693 files changed, 342185 insertions, 0 deletions

diff --git a/doc/.gitignore b/doc/.gitignore
new file mode 100644
index 000000000..0c7c74746
--- /dev/null
+++ b/doc/.gitignore
@@ -0,0 +1,2 @@
+/overview.png
+/object_store.png
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
new file mode 100644
index 000000000..e0c30d175
--- /dev/null
+++ b/doc/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(WITH_MANPAGE)
+  add_subdirectory(man)
+endif(WITH_MANPAGE)
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/doc/README.md
diff --git a/doc/_ext/ceph_commands.py b/doc/_ext/ceph_commands.py
new file mode 100644
index 000000000..861a013ea
--- /dev/null
+++ b/doc/_ext/ceph_commands.py
@@ -0,0 +1,472 @@
+import io
+import os
+import sys
+import contextlib
+
+from docutils.parsers.rst import directives
+from docutils.parsers.rst import Directive
+from jinja2 import Template
+from pcpp.preprocessor import Preprocessor
+from sphinx.util import logging
+from sphinx.util.console import bold
+from importlib import reload
+
+logger = logging.getLogger(__name__)
+
+
+class Flags:
+    NOFORWARD = (1 << 0)
+    OBSOLETE = (1 << 1)
+    DEPRECATED = (1 << 2)
+    MGR = (1 << 3)
+    POLL = (1 << 4)
+    HIDDEN = (1 << 5)
+
+    VALS = {
+        NOFORWARD: 'no_forward',
+        OBSOLETE: 'obsolete',
+        DEPRECATED: 'deprecated',
+        MGR: 'mgr',
+        POLL: 'poll',
+        HIDDEN: 'hidden',
+    }
+
+    def __init__(self, fs):
+        self.fs = fs
+
+    def __contains__(self, other):
+        return other in str(self)
+
+    def __str__(self):
+        keys = Flags.VALS.keys()
+        es = {Flags.VALS[k] for k in keys if self.fs & k == k}
+        return ', '.join(sorted(es))
+
+    def __bool__(self):
+        return bool(str(self))
+
+
+class CmdParam(object):
+    t = {
+        'CephInt': 'int',
+        'CephString': 'str',
+        'CephChoices': 'str',
+        'CephPgid': 'str',
+        'CephOsdName': 'str',
+        'CephPoolname': 'str',
+        'CephObjectname': 'str',
+        'CephUUID': 'str',
+        'CephEntityAddr': 'str',
+        'CephIPAddr': 'str',
+        'CephName': 'str',
+        'CephBool': 'bool',
+        'CephFloat': 'float',
+        'CephFilepath': 'str',
+    }
+
+    bash_example = {
+        'CephInt': '1',
+        'CephString': 'string',
+        'CephChoices': 'choice',
+        'CephPgid': '0',
+        'CephOsdName': 'osd.0',
+        'CephPoolname': 'poolname',
+        'CephObjectname': 'objectname',
+        'CephUUID': 'uuid',
+        'CephEntityAddr': 'entityaddr',
+        'CephIPAddr': '0.0.0.0',
+        'CephName': 'name',
+        'CephBool': 'true',
+        'CephFloat': '0.0',
+        'CephFilepath': '/path/to/file',
+    }
+
+    def __init__(self, type, name,
+                 who=None, n=None, req=True, range=None, strings=None,
+                 goodchars=None, positional=True):
+        self.type = type
+        self.name = name
+        self.who = who
+        self.n = n == 'N'
+        self.req = req != 'false'
+        self.range = range.split('|') if range else []
+        self.strings = strings.split('|') if strings else []
+        self.goodchars = goodchars
+        self.positional = positional != 'false'
+
+        assert who == None
+
+    def help(self):
+        advanced = []
+        if self.type != 'CephString':
+            advanced.append(self.type + ' ')
+        if self.range:
+            advanced.append('range= ``{}`` '.format('..'.join(self.range)))
+        if self.strings:
+            advanced.append('strings=({}) '.format(' '.join(self.strings)))
+        if self.goodchars:
+            advanced.append('goodchars= ``{}`` '.format(self.goodchars))
+        if self.n:
+            advanced.append('(can be repeated)')
+
+        advanced = advanced or ["(string)"]
+        return ' '.join(advanced)
+
+    def mk_example_value(self):
+        if self.type == 'CephChoices' and self.strings:
+            return self.strings[0]
+        if self.range:
+            return self.range[0]
+        return CmdParam.bash_example[self.type]
+
+    def mk_bash_example(self, simple):
+        val = self.mk_example_value()
+
+        if self.type == 'CephBool':
+            return '--' + self.name
+        if simple:
+            if self.type == "CephChoices" and self.strings:
+                return val
+            elif self.type == "CephString" and self.name != 'who':
+                return 'my_' + self.name
+            else:
+                return CmdParam.bash_example[self.type]
+        else:
+            return '--{}={}'.format(self.name, val)
+
+
+class CmdCommand(object):
+    def __init__(self, prefix, args, desc,
+                 module=None, perm=None, flags=0, poll=None):
+        self.prefix = prefix
+        self.params = sorted([CmdParam(**arg) for arg in args],
+                             key=lambda p: p.req, reverse=True)
+        self.help = desc
+        self.module = module
+        self.perm = perm
+        self.flags = Flags(flags)
+        self.needs_overload = False
+
+    def is_reasonably_simple(self):
+        if len(self.params) > 3:
+            return False
+        if any(p.n for p in self.params):
+            return False
+        return True
+
+    def mk_bash_example(self):
+        simple = self.is_reasonably_simple()
+        line = ' '.join(['ceph', self.prefix] + [p.mk_bash_example(simple) for p in self.params])
+        return line
+
+
+class Sig:
+    @staticmethod
+    def _parse_arg_desc(desc):
+        try:
+            return dict(kv.split('=', 1) for kv in desc.split(',') if kv)
+        except ValueError:
+            return desc
+
+    @staticmethod
+    def parse_cmd(cmd):
+        parsed = [Sig._parse_arg_desc(s) or s for s in cmd.split()]
+        prefix = [s for s in parsed if isinstance(s, str)]
+        params = [s for s in parsed if not isinstance(s, str)]
+        return ' '.join(prefix), params
+
+    @staticmethod
+    def parse_args(args):
+        return [Sig._parse_arg_desc(arg) for arg in args.split()]
+
+
+TEMPLATE = '''
+{%- set punct_char = '-' -%}
+{# add a header if we have multiple commands in this section #}
+{% if commands | length > 1 %}
+{{ section }}
+{{ section | length * '-' }}
+{# and demote the subsection #}
+{% set punct_char = '^' %}
+{% endif %}
+{% for command in commands %}
+{{ command.prefix }}
+{{ command.prefix | length * punct_char }}
+
+{{ command.help | wordwrap(70) }}
+
+:Example command:
+    .. code-block:: bash
+
+       {{ command.mk_bash_example() | wordwrap(70) | indent(9) }}
+
+{%- if command.params %}
+:Parameters:{% for param in command.params -%}
+{{" -" | indent(12, not loop.first) }} **{% if param.positional %}{{param.name}}{% else %}--{{param.name}}{% endif %}**: {{ param.help() }}
+{% endfor %}
+{% endif %}
+:Ceph Module: {{ command.module }}
+:Required Permissions: ``{{ command.perm }}``
+{%- if command.flags %}
+:Command Flags: ``{{ command.flags }}``
+{% endif %}
+{% endfor %}
+'''
+
+
+def group_by_prefix(commands):
+    last_prefix = None
+    grouped = []
+    for cmd in commands:
+        prefix = cmd.prefix.split(' ', 1)[0]
+        if prefix == last_prefix:
+            grouped.append(cmd)
+        elif last_prefix is None:
+            last_prefix = prefix
+            grouped = [cmd]
+        else:
+            yield last_prefix, grouped
+            last_prefix = prefix
+            grouped = [cmd]
+    assert grouped
+    yield last_prefix, grouped
+
+
+def render_commands(commands):
+    rendered = io.StringIO()
+    for section, grouped in group_by_prefix(commands):
+        logger.debug('rendering commands: %s: %d', section, len(grouped))
+        rendered.write(Template(TEMPLATE).render(
+            section=section,
+            commands=grouped))
+    return rendered.getvalue().split('\n')
+
+
+class CephMgrCommands(Directive):
+    """
+    extracts commands from specified mgr modules
+    """
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {'python_path': directives.unchanged}
+
+    def _normalize_path(self, dirname):
+        my_dir = os.path.dirname(os.path.realpath(__file__))
+        src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
+        return os.path.join(src_dir, dirname)
+
+    def _is_mgr_module(self, dirname, name):
+        if not os.path.isdir(os.path.join(dirname, name)):
+            return False
+        if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
+            return False
+        return name not in ['tests']
+
+    @contextlib.contextmanager
+    def mocked_modules(self):
+        # src/pybind/mgr/tests
+        from tests import mock
+        mock_imports = ['rados',
+                        'rbd',
+                        'cephfs',
+                        'dateutil',
+                        'dateutil.parser']
+        # make dashboard happy
+        mock_imports += ['OpenSSL',
+                         'jwt',
+                         'bcrypt',
+                         'jsonpatch',
+                         'rook.rook_client',
+                         'rook.rook_client.ceph',
+                         'rook.rook_client._helper',
+                         'cherrypy=3.2.3']
+        # make diskprediction_local happy
+        mock_imports += ['numpy',
+                         'scipy']
+        # make restful happy
+        mock_imports += ['pecan',
+                         'pecan.rest',
+                         'pecan.hooks',
+                         'werkzeug',
+                         'werkzeug.serving']
+
+        for m in mock_imports:
+            args = {}
+            parts = m.split('=', 1)
+            mocked = parts[0]
+            if len(parts) > 1:
+                args['__version__'] = parts[1]
+            sys.modules[mocked] = mock.Mock(**args)
+
+        try:
+            yield
+        finally:
+            for m in mock_imports:
+                mocked = m.split('=', 1)[0]
+                sys.modules.pop(mocked)
+
+    def _collect_module_commands(self, name):
+        with self.mocked_modules():
+            logger.info(bold(f"loading mgr module '{name}'..."))
+            mgr_mod = __import__(name, globals(), locals(), [], 0)
+            reload(mgr_mod)
+            from tests import M
+
+            def subclass(x):
+                try:
+                    return issubclass(x, M)
+                except TypeError:
+                    return False
+            ms = [c for c in mgr_mod.__dict__.values()
+                  if subclass(c) and 'Standby' not in c.__name__]
+            [m] = ms
+            assert isinstance(m.COMMANDS, list)
+            return m.COMMANDS
+
+    def _normalize_command(self, command):
+        if 'handler' in command:
+            del command['handler']
+        if 'cmd' in command:
+            command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
+            del command['cmd']
+        else:
+            command['args'] = Sig.parse_args(command['args'])
+        command['flags'] = (1 << 3)
+        command['module'] = 'mgr'
+        return command
+
+    def _render_cmds(self, commands):
+        lines = render_commands(commands)
+        assert lines
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+
+    def run(self):
+        module_path = self._normalize_path(self.arguments[0])
+        sys.path.insert(0, module_path)
+        for path in self.options.get('python_path', '').split(':'):
+            sys.path.insert(0, self._normalize_path(path))
+        os.environ['UNITTEST'] = 'true'
+        modules = [name for name in os.listdir(module_path)
+                   if self._is_mgr_module(module_path, name)]
+        commands = sum([self._collect_module_commands(name) for name in modules], [])
+        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
+        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
+        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
+        self._render_cmds(cmds)
+
+        orig_rgw_mod = sys.modules['pybind_rgw_mod']
+        sys.modules['rgw'] = orig_rgw_mod
+
+        return []
+
+
+class MyProcessor(Preprocessor):
+    def __init__(self):
+        super().__init__()
+        self.cmds = []
+        self.undef('__DATE__')
+        self.undef('__TIME__')
+        self.expand_linemacro = False
+        self.expand_filemacro = False
+        self.expand_countermacro = False
+        self.line_directive = '#line'
+        self.define("__PCPP_VERSION__ " + '')
+        self.define("__PCPP_ALWAYS_FALSE__ 0")
+        self.define("__PCPP_ALWAYS_TRUE__ 1")
+
+    def eval(self, src):
+        _cmds = []
+
+        NONE = 0
+        NOFORWARD = (1 << 0)
+        OBSOLETE = (1 << 1)
+        DEPRECATED = (1 << 2)
+        MGR = (1 << 3)
+        POLL = (1 << 4)
+        HIDDEN = (1 << 5)
+        TELL = (1 << 6)
+
+        def FLAG(a):
+            return a
+
+        def COMMAND(cmd, desc, module, perm):
+            _cmds.append({
+                'cmd': cmd,
+                'desc': desc,
+                'module': module,
+                'perm': perm
+            })
+
+        def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
+            _cmds.append({
+                'cmd': cmd,
+                'desc': desc,
+                'module': module,
+                'perm': perm,
+                'flags': flag
+            })
+
+        self.parse(src)
+        out = io.StringIO()
+        self.write(out)
+        out.seek(0)
+        s = out.read()
+        exec(s, globals(), locals())
+        return _cmds
+
+
+class CephMonCommands(Directive):
+    """
+    extracts commands from specified header file
+    """
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+
+    def _src_dir(self):
+        my_dir = os.path.dirname(os.path.realpath(__file__))
+        return os.path.abspath(os.path.join(my_dir, '../..'))
+
+    def _parse_headers(self, headers):
+        src_dir = self._src_dir()
+        src = '\n'.join(f'#include "{src_dir}/{header}"' for header in headers)
+        return MyProcessor().eval(src)
+
+    def _normalize_command(self, command):
+        if 'handler' in command:
+            del command['handler']
+        command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
+        del command['cmd']
+        return command
+
+    def _render_cmds(self, commands):
+        lines = render_commands(commands)
+        assert lines
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+
+    def run(self):
+        headers = self.arguments[0].split()
+        commands = self._parse_headers(headers)
+        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
+        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
+        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
+        self._render_cmds(cmds)
+        return []
+
+
+def setup(app):
+    app.add_directive("ceph-mgr-commands", CephMgrCommands)
+    app.add_directive("ceph-mon-commands", CephMonCommands)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
diff --git a/doc/_ext/ceph_confval.py b/doc/_ext/ceph_confval.py
new file mode 100644
index 000000000..cde538b45
--- /dev/null
+++ b/doc/_ext/ceph_confval.py
@@ -0,0 +1,459 @@
+import io
+import contextlib
+import os
+import sys
+from typing import Any, Dict, List, Union
+
+from docutils.nodes import Node
+from docutils.parsers.rst import directives
+from docutils.statemachine import StringList
+
+from sphinx import addnodes
+from sphinx.directives import ObjectDescription
+from sphinx.domains.python import PyField
+from sphinx.environment import BuildEnvironment
+from sphinx.locale import _
+from sphinx.util import logging, status_iterator, ws_re
+from sphinx.util.docutils import switch_source_input, SphinxDirective
+from sphinx.util.docfields import Field
+from sphinx.util.nodes import make_id
+import jinja2
+import jinja2.filters
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+TEMPLATE = '''
+{% if desc %}
+   {{ desc | wordwrap(70) | indent(3) }}
+{% endif %}
+   :type: ``{{opt.type}}``
+{%- if default is not none %}
+  {%- if opt.type == 'size' %}
+   :default: ``{{ default | eval_size | iec_size }}``
+  {%- elif opt.type == 'secs' %}
+   :default: ``{{ default | readable_duration(opt.type) }}``
+  {%- elif opt.type in ('uint', 'int', 'float') %}
+   :default: ``{{ default | readable_num(opt.type) }}``
+  {%- elif opt.type == 'millisecs' %}
+   :default: ``{{ default }}`` milliseconds
+  {%- elif opt.type == 'bool' %}
+   :default: ``{{ default | string | lower }}``
+  {%- else %}
+   :default: {{ default | literal }}
+  {%- endif -%}
+{%- endif %}
+{%- if opt.enum_values %}
+   :valid choices:{% for enum_value in opt.enum_values -%}
+{{" -" | indent(18, not loop.first) }} {{ enum_value | literal }}
+{% endfor %}
+{%- endif %}
+{%- if opt.min is defined and opt.max is defined %}
+   :allowed range: ``[{{ opt.min }}, {{ opt.max }}]``
+{%- elif opt.min is defined %}
+   :min: ``{{ opt.min }}``
+{%- elif opt.max is defined %}
+   :max: ``{{ opt.max }}``
+{%- endif %}
+{%- if opt.constraint %}
+   :constraint: {{ opt.constraint }}
+{% endif %}
+{%- if opt.policies %}
+   :policies: {{ opt.policies }}
+{% endif %}
+{%- if opt.example %}
+   :example: {{ opt.example }}
+{%- endif %}
+{%- if opt.see_also %}
+   :see also: {{ opt.see_also | map('ref_confval') | join(', ') }}
+{%- endif %}
+{% if opt.note %}
+   .. note::
+      {{ opt.note }}
+{%- endif -%}
+{%- if opt.warning %}
+   .. warning::
+      {{ opt.warning }}
+{%- endif %}
+'''
+
+
+def eval_size(value) -> int:
+    try:
+        return int(value)
+    except ValueError:
+        times = dict(_K=1 << 10,
+                     _M=1 << 20,
+                     _G=1 << 30,
+                     _T=1 << 40)
+        for unit, m in times.items():
+            if value.endswith(unit):
+                return int(value[:-len(unit)]) * m
+        raise ValueError(f'unknown value: {value}')
+
+
+def readable_duration(value: str, typ: str) -> str:
+    try:
+        if typ == 'sec':
+            v = int(value)
+            postfix = 'second' if v == 1 else 'seconds'
+            return f'{v} {postfix}'
+        elif typ == 'float':
+            return str(float(value))
+        else:
+            return str(int(value))
+    except ValueError:
+        times = dict(_min=['minute', 'minutes'],
+                     _hr=['hour', 'hours'],
+                     _day=['day', 'days'])
+        for unit, readables in times.items():
+            if value.endswith(unit):
+                v = int(value[:-len(unit)])
+                postfix = readables[0 if v == 1 else 1]
+                return f'{v} {postfix}'
+        raise ValueError(f'unknown value: {value}')
+
+
+def do_plain_num(value: str, typ: str) -> str:
+    if typ == 'float':
+        return str(float(value))
+    else:
+        return str(int(value))
+
+
+def iec_size(value: int) -> str:
+    if value == 0:
+        return '0B'
+    units = dict(Ei=60,
+                 Pi=50,
+                 Ti=40,
+                 Gi=30,
+                 Mi=20,
+                 Ki=10,
+                 B=0)
+    for unit, bits in units.items():
+        m = 1 << bits
+        if value % m == 0:
+            value //= m
+            return f'{value}{unit}'
+    raise Exception(f'iec_size() failed to convert {value}')
+
+
+def do_fileize_num(value: str, typ: str) -> str:
+    v = eval_size(value)
+    return iec_size(v)
+
+
+def readable_num(value: str, typ: str) -> str:
+    e = ValueError()
+    for eval_func in [do_plain_num,
+                      readable_duration,
+                      do_fileize_num]:
+        try:
+            return eval_func(value, typ)
+        except ValueError as ex:
+            e = ex
+    raise e
+
+
+def literal(name) -> str:
+    if name:
+        return f'``{name}``'
+    else:
+        return f'<empty string>'
+
+
+def ref_confval(name) -> str:
+    return f':confval:`{name}`'
+
+
+def jinja_template() -> jinja2.Template:
+    env = jinja2.Environment()
+    env.filters['eval_size'] = eval_size
+    env.filters['iec_size'] = iec_size
+    env.filters['readable_duration'] = readable_duration
+    env.filters['readable_num'] = readable_num
+    env.filters['literal'] = literal
+    env.filters['ref_confval'] = ref_confval
+    return env.from_string(TEMPLATE)
+
+
+FieldValueT = Union[bool, float, int, str]
+
+
+class CephModule(SphinxDirective):
+    """
+    Directive to name the mgr module for which options are documented.
+    """
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+
+    def run(self) -> List[Node]:
+        module = self.arguments[0].strip()
+        if module == 'None':
+            self.env.ref_context.pop('ceph:module', None)
+        else:
+            self.env.ref_context['ceph:module'] = module
+        return []
+
+
+class CephOption(ObjectDescription):
+    """
+    emit option loaded from given command/options/<name>.yaml.in file
+    """
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {
+        'module': directives.unchanged,
+        'default': directives.unchanged
+    }
+
+
+    doc_field_types = [
+        Field('default',
+              label=_('Default'),
+              has_arg=False,
+              names=('default',)),
+        Field('type',
+              label=_('Type'),
+              has_arg=False,
+              names=('type',),
+              bodyrolename='class'),
+    ]
+
+    template = jinja_template()
+    opts: Dict[str, Dict[str, FieldValueT]] = {}
+    mgr_opts: Dict[str,            # module name
+                   Dict[str,       # option name
+                        Dict[str,  # field_name
+                             FieldValueT]]] = {}
+
+    def _load_yaml(self) -> Dict[str, Dict[str, FieldValueT]]:
+        if CephOption.opts:
+            return CephOption.opts
+        opts = []
+        for fn in status_iterator(self.config.ceph_confval_imports,
+                                  'loading options...', 'red',
+                                  len(self.config.ceph_confval_imports),
+                                  self.env.app.verbosity):
+            self.env.note_dependency(fn)
+            try:
+                with open(fn, 'r') as f:
+                    yaml_in = io.StringIO()
+                    for line in f:
+                        if '@' not in line:
+                            yaml_in.write(line)
+                    yaml_in.seek(0)
+                    opts += yaml.safe_load(yaml_in)['options']
+            except OSError as e:
+                message = f'Unable to open option file "{fn}": {e}'
+                raise self.error(message)
+        CephOption.opts = dict((opt['name'], opt) for opt in opts)
+        return CephOption.opts
+
+    def _normalize_path(self, dirname):
+        my_dir = os.path.dirname(os.path.realpath(__file__))
+        src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
+        return os.path.join(src_dir, dirname)
+
+    def _is_mgr_module(self, dirname, name):
+        if not os.path.isdir(os.path.join(dirname, name)):
+            return False
+        if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
+            return False
+        return name not in ['tests']
+
+    @contextlib.contextmanager
+    def mocked_modules(self):
+        # src/pybind/mgr/tests
+        from tests import mock
+        mock_imports = ['rados',
+                        'rbd',
+                        'cephfs',
+                        'dateutil',
+                        'dateutil.parser']
+        # make dashboard happy
+        mock_imports += ['OpenSSL',
+                         'jwt',
+                         'bcrypt',
+                         'jsonpatch',
+                         'rook.rook_client',
+                         'rook.rook_client.ceph',
+                         'rook.rook_client._helper',
+                         'cherrypy=3.2.3']
+        # make diskprediction_local happy
+        mock_imports += ['numpy',
+                         'scipy']
+        # make restful happy
+        mock_imports += ['pecan',
+                         'pecan.rest',
+                         'pecan.hooks',
+                         'werkzeug',
+                         'werkzeug.serving']
+
+        for m in mock_imports:
+            args = {}
+            parts = m.split('=', 1)
+            mocked = parts[0]
+            if len(parts) > 1:
+                args['__version__'] = parts[1]
+            sys.modules[mocked] = mock.Mock(**args)
+
+        try:
+            yield
+        finally:
+            for m in mock_imports:
+                mocked = m.split('=', 1)[0]
+                sys.modules.pop(mocked)
+
+    def _collect_options_from_module(self, name):
+        with self.mocked_modules():
+            mgr_mod = __import__(name, globals(), locals(), [], 0)
+            # import 'M' from src/pybind/mgr/tests
+            from tests import M
+
+            def subclass(x):
+                try:
+                    return issubclass(x, M)
+                except TypeError:
+                    return False
+            ms = [c for c in mgr_mod.__dict__.values()
+                  if subclass(c) and 'Standby' not in c.__name__]
+            [m] = ms
+            assert isinstance(m.MODULE_OPTIONS, list)
+            return m.MODULE_OPTIONS
+
+    def _load_module(self, module) -> Dict[str, Dict[str, FieldValueT]]:
+        mgr_opts = CephOption.mgr_opts.get(module)
+        if mgr_opts is not None:
+            return mgr_opts
+        python_path = self.config.ceph_confval_mgr_python_path
+        for path in python_path.split(':'):
+            sys.path.insert(0, self._normalize_path(path))
+        module_path = self.env.config.ceph_confval_mgr_module_path
+        module_path = self._normalize_path(module_path)
+        sys.path.insert(0, module_path)
+        if not self._is_mgr_module(module_path, module):
+            raise self.error(f'module "{module}" not found under {module_path}')
+        fn = os.path.join(module_path, module, 'module.py')
+        if os.path.exists(fn):
+            self.env.note_dependency(fn)
+        os.environ['UNITTEST'] = 'true'
+        opts = self._collect_options_from_module(module)
+        CephOption.mgr_opts[module] = dict((opt['name'], opt) for opt in opts)
+        return CephOption.mgr_opts[module]
+
+    def _current_module(self) -> str:
+        return self.options.get('module',
+                                self.env.ref_context.get('ceph:module'))
+
+    def _render_option(self, name) -> str:
+        cur_module = self._current_module()
+        if cur_module:
+            try:
+                opt = self._load_module(cur_module).get(name)
+            except Exception as e:
+                message = f'Unable to load module "{cur_module}": {e}'
+                raise self.error(message)
+        else:
+            opt = self._load_yaml().get(name)
+        if opt is None:
+            raise self.error(f'Option "{name}" not found!')
+        if cur_module and 'type' not in opt:
+            # the type of module option defaults to 'str'
+            opt['type'] = 'str'
+        desc = opt.get('fmt_desc') or opt.get('long_desc') or opt.get('desc')
+        opt_default = opt.get('default')
+        default = self.options.get('default', opt_default)
+        try:
+            return self.template.render(opt=opt,
+                                        desc=desc,
+                                        default=default)
+        except Exception as e:
+            message = (f'Unable to render option "{name}": {e}. ',
+                       f'opt={opt}, desc={desc}, default={default}')
+            raise self.error(message)
+
+    def handle_signature(self,
+                         sig: str,
+                         signode: addnodes.desc_signature) -> str:
+        signode.clear()
+        signode += addnodes.desc_name(sig, sig)
+        # normalize whitespace like XRefRole does
+        name = ws_re.sub(' ', sig)
+        cur_module = self._current_module()
+        if cur_module:
+            return '/'.join(['mgr', cur_module, name])
+        else:
+            return name
+
+    def transform_content(self, contentnode: addnodes.desc_content) -> None:
+        name = self.arguments[0]
+        source, lineno = self.get_source_info()
+        source = f'{source}:{lineno}:<confval>'
+        fields = StringList(self._render_option(name).splitlines() + [''],
+                            source=source, parent_offset=lineno)
+        with switch_source_input(self.state, fields):
+            self.state.nested_parse(fields, 0, contentnode)
+
+    def add_target_and_index(self,
+                             name: str,
+                             sig: str,
+                             signode: addnodes.desc_signature) -> None:
+        node_id = make_id(self.env, self.state.document, self.objtype, name)
+        signode['ids'].append(node_id)
+        self.state.document.note_explicit_target(signode)
+        entry = f'{name}; configuration option'
+        self.indexnode['entries'].append(('pair', entry, node_id, '', None))
+        std = self.env.get_domain('std')
+        std.note_object(self.objtype, name, node_id, location=signode)
+
+
+def _reset_ref_context(app, env, docname):
+    env.ref_context.pop('ceph:module', None)
+
+
+def setup(app) -> Dict[str, Any]:
+    app.add_config_value('ceph_confval_imports',
+                         default=[],
+                         rebuild='html',
+                         types=[str])
+    app.add_config_value('ceph_confval_mgr_module_path',
+                         default=[],
+                         rebuild='html',
+                         types=[str])
+    app.add_config_value('ceph_confval_mgr_python_path',
+                         default=[],
+                         rebuild='',
+                         types=[str])
+    app.add_object_type(
+        'confsec',
+        'confsec',
+        objname='configuration section',
+        indextemplate='pair: %s; configuration section',
+        doc_field_types=[
+            Field(
+                'example',
+                label=_('Example'),
+                has_arg=False,
+            )]
+    )
+    app.add_object_type(
+        'confval',
+        'confval',
+        objname='configuration option',
+    )
+    app.add_directive_to_domain('std', 'mgr_module', CephModule)
+    app.add_directive_to_domain('std', 'confval', CephOption, override=True)
+    app.connect('env-purge-doc', _reset_ref_context)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
diff --git a/doc/_ext/ceph_releases.py b/doc/_ext/ceph_releases.py
new file mode 100644
index 000000000..94e92ffdd
--- /dev/null
+++ b/doc/_ext/ceph_releases.py
@@ -0,0 +1,351 @@
+# cobbled together from:
+#   https://github.com/sphinx-contrib/documentedlist/blob/master/sphinxcontrib/documentedlist.py
+#   https://github.com/sphinx-doc/sphinx/blob/v1.6.3/sphinx/ext/graphviz.py
+#   https://github.com/thewtex/sphinx-contrib/blob/master/exceltable/sphinxcontrib/exceltable.py
+#   https://bitbucket.org/prometheus/sphinxcontrib-htsql/src/331a542c29a102eec9f8cba44797e53a49de2a49/sphinxcontrib/htsql.py?at=default&fileviewer=file-view-default
+# into the glory that follows:
+import json
+import yaml
+import jinja2
+import sphinx
+import datetime
+from docutils.parsers.rst import Directive
+from docutils import nodes
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+
+class CephReleases(Directive):
+    has_content = False
+    required_arguments = 2
+    optional_arguments = 0
+    option_spec = {}
+
+    def run(self):
+        filename = self.arguments[0]
+        current = self.arguments[1] == 'current'
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        try:
+            with open(filename, 'r') as fp:
+                releases = yaml.safe_load(fp)
+                releases = releases["releases"]
+        except Exception as e:
+            return [document.reporter.warning(
+                "Failed to open Ceph releases file {}: {}".format(filename, e),
+                line=self.lineno)]
+
+        table = nodes.table()
+        tgroup = nodes.tgroup(cols=3)
+        table += tgroup
+
+        tgroup.extend(
+            nodes.colspec(colwidth=30, colname='c'+str(idx))
+            for idx, _ in enumerate(range(4)))
+
+        thead = nodes.thead()
+        tgroup += thead
+        row_node = nodes.row()
+        thead += row_node
+        row_node.extend(
+            nodes.entry(h, nodes.paragraph(text=h))
+            for h in ["Name", "Initial release", "Latest",
+                      "End of life (estimated)" if current else "End of life"])
+
+        releases = releases.items()
+        releases = sorted(releases, key=lambda t: t[0], reverse=True)
+
+        tbody = nodes.tbody()
+        tgroup += tbody
+
+        rows = []
+        for code_name, info in releases:
+            actual_eol = info.get("actual_eol", None)
+
+            if current:
+                if actual_eol and actual_eol <= datetime.datetime.now().date():
+                    continue
+            else:
+                if not actual_eol:
+                    continue
+
+            trow = nodes.row()
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text=f"`{code_name.title()} <{code_name}>`_")
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            #entry += para
+            trow += entry
+
+            sorted_releases = sorted(info["releases"],
+                    key=lambda t: [t["released"]] + list(map(lambda v: int(v), t["version"].split("."))))
+            oldest_release = sorted_releases[0]
+            newest_release = sorted_releases[-1]
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text="{}".format(
+                oldest_release["released"]))
+            entry += para
+            trow += entry
+
+            entry = nodes.entry()
+            if newest_release.get("skip_ref", False):
+                para = nodes.paragraph(text="{}".format(
+                    newest_release["version"]))
+            else:
+                para = nodes.paragraph(text="`{}`_".format(
+                    newest_release["version"]))
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            #entry += para
+            trow += entry
+
+            entry = nodes.entry()
+            if current:
+                para = nodes.paragraph(text=info.get("target_eol", '--'))
+            else:
+                para = nodes.paragraph(text=info.get('actual_eol', '--'))
+            entry += para
+            trow += entry
+
+            rows.append(trow)
+
+        tbody.extend(rows)
+
+        return [table]
+
+
+RELEASES_TEMPLATE = '''
+.. mermaid::
+
+   gantt
+       dateFormat  YYYY-MM-DD
+       axisFormat  %Y
+       section Active Releases
+{% for release in active_releases %}
+       {{ release.code_name }} (latest {{ release.last_version }}): done, {{ release.debute_date }},{{ release.lifetime }}d
+{% endfor %}
+       section Archived Releases
+{% for release in archived_releases %}
+       {{ release.code_name }} (latest {{ release.last_version }}): done, {{ release.debute_date }},{{ release.lifetime }}d
+{% endfor %}
+'''
+
+
+class ReleasesGantt(Directive):
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+
+    template = jinja2.Environment().from_string(RELEASES_TEMPLATE)
+
+    def _render_time_line(self, filename):
+        try:
+            with open(filename) as f:
+                releases = yaml.safe_load(f)['releases']
+        except Exception as e:
+            message = f'Unable read release file: "{filename}": {e}'
+            self.error(message)
+
+        active_releases = []
+        archived_releases = []
+        # just update `releases` with extracted info
+        for code_name, info in releases.items():
+            last_release = info['releases'][0]
+            first_release = info['releases'][-1]
+            last_version = last_release['version']
+            debute_date = first_release['released']
+            if 'actual_eol' in info:
+                lifetime = info['actual_eol'] - first_release['released']
+            else:
+                lifetime = info['target_eol'] - first_release['released']
+            release = dict(code_name=code_name,
+                           last_version=last_version,
+                           debute_date=debute_date,
+                           lifetime=lifetime.days)
+            if 'actual_eol' in info:
+                archived_releases.append(release)
+            else:
+                active_releases.append(release)
+        rendered = self.template.render(active_releases=active_releases,
+                                        archived_releases=archived_releases)
+        return rendered.splitlines()
+
+    def run(self):
+        filename = self.arguments[0]
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        lines = self._render_time_line(filename)
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+        return []
+
+
+class CephTimeline(Directive):
+    has_content = False
+    required_arguments = 3
+    optional_arguments = 0
+    option_spec = {}
+
+    def run(self):
+        filename = self.arguments[0]
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        try:
+            with open(filename, 'r') as fp:
+                releases = yaml.safe_load(fp)
+        except Exception as e:
+            return [document.reporter.warning(
+                "Failed to open Ceph releases file {}: {}".format(filename, e),
+                line=self.lineno)]
+
+        display_releases = self.arguments[1:]
+
+        timeline = []
+        for code_name, info in releases["releases"].items():
+            if code_name in display_releases:
+                for release in info.get("releases", []):
+                    released = release["released"]
+                    timeline.append((released, code_name, release["version"],
+                        release.get("skip_ref", False)))
+
+        assert "development" not in releases["releases"]
+        if "development" in display_releases:
+            for release in releases["development"]["releases"]:
+                released = release["released"]
+                timeline.append((released, "development", release["version"],
+                    release.get("skip_ref", False)))
+
+        timeline = sorted(timeline, key=lambda t: t[0], reverse=True)
+
+        table = nodes.table()
+        tgroup = nodes.tgroup(cols=3)
+        table += tgroup
+
+        columns = ["Date"] + display_releases
+        tgroup.extend(
+            nodes.colspec(colwidth=30, colname='c'+str(idx))
+            for idx, _ in enumerate(range(len(columns))))
+
+        thead = nodes.thead()
+        tgroup += thead
+        row_node = nodes.row()
+        thead += row_node
+        for col in columns:
+            entry = nodes.entry()
+            if col.lower() in ["date", "development"]:
+                para = nodes.paragraph(text=col.title())
+            else:
+                para = nodes.paragraph(text=f"`{col.title()} <{col}>`_".format(col))
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            row_node += entry
+
+        tbody = nodes.tbody()
+        tgroup += tbody
+
+        rows = []
+        for row_info in timeline:
+            trow = nodes.row()
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text=row_info[0])
+            entry += para
+            trow += entry
+
+            for release in display_releases:
+                entry = nodes.entry()
+                if row_info[1] == release:
+                    if row_info[3]: # if skip ref
+                        para = nodes.paragraph(text=row_info[2])
+                    else:
+                        para = nodes.paragraph(text="`{}`_".format(row_info[2]))
+                    sphinx.util.nodes.nested_parse_with_titles(
+                            self.state, para, entry)
+                else:
+                    para = nodes.paragraph(text="--")
+                    entry += para
+                trow += entry
+            rows.append(trow)
+
+        tbody.extend(rows)
+
+        return [table]
+
+
+TIMELINE_TEMPLATE = '''
+.. mermaid::
+
+   gantt
+       dateFormat  YYYY-MM-DD
+       axisFormat  %Y-%m
+{% if title %}
+       title       {{title}}
+{% endif %}
+{% for display_release in display_releases %}
+       section {{ display_release }}
+{%if releases[display_release].actual_eol %}
+       End of life:             crit,            {{ releases[display_release].actual_eol }},4d
+{% else %}
+       End of life (estimated): crit,            {{ releases[display_release].target_eol }},4d
+{% endif %}
+{% for release in releases[display_release].releases | sort(attribute='released', reverse=True) %}
+       {{ release.version }}:   milestone, done, {{ release.released }},0d
+{% endfor %}
+{% endfor %}
+'''
+
+
+class TimeLineGantt(Directive):
+    has_content = True
+    required_arguments = 2
+    optional_arguments = 0
+    final_argument_whitespace = True
+
+    template = jinja2.Environment().from_string(TIMELINE_TEMPLATE)
+
+    def _render_time_line(self, filename, display_releases):
+        try:
+            with open(filename) as f:
+                releases = yaml.safe_load(f)['releases']
+        except Exception as e:
+            message = f'Unable read release file: "{filename}": {e}'
+            self.error(message)
+
+        rendered = self.template.render(display_releases=display_releases,
+                                        releases=releases)
+        return rendered.splitlines()
+
+    def run(self):
+        filename = self.arguments[0]
+        display_releases = self.arguments[1].split()
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        lines = self._render_time_line(filename, display_releases)
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+        return []
+
+
+def setup(app):
+    app.add_directive('ceph_releases', CephReleases)
+    app.add_directive('ceph_releases_gantt', ReleasesGantt)
+    app.add_directive('ceph_timeline', CephTimeline)
+    app.add_directive('ceph_timeline_gantt', TimeLineGantt)
+    return {
+        'parallel_read_safe': True,
+        'parallel_write_safe': True
+    }
diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css
new file mode 100644
index 000000000..2a37cab99
--- /dev/null
+++ b/doc/_static/css/custom.css
@@ -0,0 +1,28 @@
+dt { 
+    scroll-margin-top: 3em; 
+}
+
+h2 {
+    scroll-margin-top: 4em; 
+}
+
+h3 {
+    scroll-margin-top: 4em; 
+}
+
+section {
+    scroll-margin-top: 4em; 
+}
+
+span {
+    scroll-margin-top: 2em; 
+}
+
+ul.simple > li > ul > li:last-child {
+    margin-block-end : 1em;
+}
+
+div.section > ul > li > p {
+    margin-block-start : 0.6em;
+    margin-block-end : 0.6em;
+}
diff --git a/doc/_templates/page.html b/doc/_templates/page.html
new file mode 100644
index 000000000..3fd83b623
--- /dev/null
+++ b/doc/_templates/page.html
@@ -0,0 +1,25 @@
+{% extends "!page.html" %}
+{% block body %}
+
+{%- if release == 'dev' %}
+<div id="dev-warning" class="admonition note">
+  <p class="first admonition-title">Notice</p>
+  <p class="last">This document is for a development version of Ceph.</p>
+</div>
+{%- endif %}
+
+{%- if is_release_eol %}
+<div id="eol-warning" class="admonition warning">
+  <p class="first admonition-title">Warning</p>
+  <p class="last">This document is for an unsupported version of Ceph.</p>
+</div>
+{%- endif %}
+
+{%- if not is_release_eol %}
+  <div id="docubetter" align="right" style="padding: 5px; font-weight: bold;">
+    <a href="https://pad.ceph.com/p/Report_Documentation_Bugs">Report a Documentation Bug</a>
+  </div>
+{%- endif %}
+
+  {{ super() }}
+{% endblock %}
diff --git a/doc/_templates/smarttoc.html b/doc/_templates/smarttoc.html
new file mode 100644
index 000000000..7a3fd3f2b
--- /dev/null
+++ b/doc/_templates/smarttoc.html
@@ -0,0 +1,16 @@
+{#
+    Sphinx sidebar template: smart table of contents.
+
+    Shows a sidebar ToC that gives you a more global view of the
+    documentation, and not the confusing cur/prev/next which is the
+    default sidebar.
+
+    The ToC will open and collapse automatically to show the part of the
+    hierarchy you are in. Top-level items will always be visible.
+
+#}
+<h3><a href="{{ pathto(master_doc) }}">{{ _('Table Of Contents') }}</a></h3>
+{{ toctree(maxdepth=-1, includehidden=True) }}
+
+<!-- ugly kludge to make genindex look like it's part of the toc -->
+<ul style="margin-top: -10px"><li class="toctree-l1"><a class="reference internal" href="{{ pathto('genindex') }}">Index</a></li></ul>
diff --git a/doc/_themes/ceph/layout.html b/doc/_themes/ceph/layout.html
new file mode 100644
index 000000000..f89edfe37
--- /dev/null
+++ b/doc/_themes/ceph/layout.html
@@ -0,0 +1,252 @@
+{# TEMPLATE VAR SETTINGS #}
+{%- set url_root = pathto('', 1) %}
+{%- if url_root == '#' %}{% set url_root = '' %}{% endif %}
+{%- if not embedded and docstitle %}
+  {%- set titlesuffix = " &mdash; "|safe + docstitle|e %}
+{%- else %}
+  {%- set titlesuffix = "" %}
+{%- endif %}
+{%- set lang_attr = 'en' if language == None else (language | replace('_', '-')) %}
+{%- set sphinx_writer = 'writer-html5' if html5_doctype else 'writer-html4' %}
+
+<!DOCTYPE html>
+<html class="{{ sphinx_writer }}" lang="{{ lang_attr }}" >
+<head>
+  <meta charset="utf-8" />
+  {{ metatags }}
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  {% block htmltitle %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endblock %}
+
+  {# CSS #}
+  <link rel="stylesheet" href="{{ pathto('_static/' + style, 1) }}" type="text/css" />
+  <link rel="stylesheet" href="{{ pathto('_static/pygments.css', 1) }}" type="text/css" />
+  {%- for css in css_files %}
+    {%- if css|attr("rel") %}
+  <link rel="{{ css.rel }}" href="{{ pathto(css.filename, 1) }}" type="text/css"{% if css.title is not none %} title="{{ css.title }}"{% endif %} />
+    {%- else %}
+  <link rel="stylesheet" href="{{ pathto(css, 1) }}" type="text/css" />
+    {%- endif %}
+  {%- endfor %}
+
+  {%- for cssfile in extra_css_files %}
+    <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
+  {%- endfor %}
+
+  {# FAVICON #}
+  {% if favicon %}
+    <link rel="shortcut icon" href="{{ pathto('_static/' + favicon, 1) }}"/>
+  {% endif %}
+
+  {# CANONICAL URL (deprecated) #}
+  {% if theme_canonical_url and not pageurl %}
+    <link rel="canonical" href="{{ theme_canonical_url }}{{ pagename }}.html"/>
+  {% endif %}
+
+  {# CANONICAL URL #}
+  {%- if pageurl %}
+    <link rel="canonical" href="{{ pageurl|e }}" />
+  {%- endif %}
+
+  {# JAVASCRIPTS #}
+  {%- block scripts %}
+  <!--[if lt IE 9]>
+    <script src="{{ pathto('_static/js/html5shiv.min.js', 1) }}"></script>
+  <![endif]-->
+  {%- if not embedded %}
+  {# XXX Sphinx 1.8.0 made this an external js-file, quick fix until we refactor the template to inherit more blocks directly from sphinx #}
+    {% if sphinx_version >= "1.8.0" %}
+      <script type="text/javascript" id="documentation_options" data-url_root="{{ url_root }}" src="{{ pathto('_static/documentation_options.js', 1) }}"></script>
+      {%- for scriptfile in script_files %}
+        {{ js_tag(scriptfile) }}
+      {%- endfor %}
+    {% else %}
+      <script type="text/javascript">
+          var DOCUMENTATION_OPTIONS = {
+              URL_ROOT:'{{ url_root }}',
+              VERSION:'{{ release|e }}',
+              LANGUAGE:'{{ language }}',
+              COLLAPSE_INDEX:false,
+              FILE_SUFFIX:'{{ '' if no_search_suffix else file_suffix }}',
+              HAS_SOURCE:  {{ has_source|lower }},
+              SOURCELINK_SUFFIX: '{{ sourcelink_suffix }}'
+          };
+      </script>
+      {%- for scriptfile in script_files %}
+        <script type="text/javascript" src="{{ pathto(scriptfile, 1) }}"></script>
+      {%- endfor %}
+    {% endif %}
+    <script type="text/javascript" src="{{ pathto('_static/js/theme.js', 1) }}"></script>
+
+    {# OPENSEARCH #}
+    {%- if use_opensearch %}
+    <link rel="search" type="application/opensearchdescription+xml"
+          title="{% trans docstitle=docstitle|e %}Search within {{ docstitle }}{% endtrans %}"
+          href="{{ pathto('_static/opensearch.xml', 1) }}"/>
+    {%- endif %}
+  {%- endif %}
+  {%- endblock %}
+
+  {%- block linktags %}
+    {%- if hasdoc('about') %}
+    <link rel="author" title="{{ _('About these documents') }}" href="{{ pathto('about') }}" />
+    {%- endif %}
+    {%- if hasdoc('genindex') %}
+    <link rel="index" title="{{ _('Index') }}" href="{{ pathto('genindex') }}" />
+    {%- endif %}
+    {%- if hasdoc('search') %}
+    <link rel="search" title="{{ _('Search') }}" href="{{ pathto('search') }}" />
+    {%- endif %}
+    {%- if hasdoc('copyright') %}
+    <link rel="copyright" title="{{ _('Copyright') }}" href="{{ pathto('copyright') }}" />
+    {%- endif %}
+    {%- if next %}
+    <link rel="next" title="{{ next.title|striptags|e }}" href="{{ next.link|e }}" />
+    {%- endif %}
+    {%- if prev %}
+    <link rel="prev" title="{{ prev.title|striptags|e }}" href="{{ prev.link|e }}" />
+    {%- endif %}
+  {%- endblock %}
+  {%- block extrahead %} {% endblock %}
+</head>
+
+<body class="wy-body-for-nav">
+
+  {% block extrabody %} {% endblock %}
+  <header class="top-bar">
+    {% include "breadcrumbs.html" %}
+  </header>
+  <div class="wy-grid-for-nav">
+    {# SIDE NAV, TOGGLES ON MOBILE #}
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" {% if theme_style_nav_header_background %} style="background: {{theme_style_nav_header_background}}" {% endif %}>
+          {% block sidebartitle %}
+
+          {% if logo and theme_logo_only %}
+            <a href="{{ pathto(master_doc) }}">
+          {% else %}
+            <a href="{{ pathto(master_doc) }}" class="icon icon-home"> {{ project }}
+          {% endif %}
+
+          {% if logo %}
+            {# Not strictly valid HTML, but it's the only way to display/scale
+               it properly, without weird scripting or heaps of work
+            #}
+            <img src="{{ pathto('_static/' + logo, 1) }}" class="logo" alt="{{ _('Logo') }}"/>
+          {% endif %}
+          </a>
+
+          {% if theme_display_version %}
+            {%- set nav_version = version %}
+            {% if READTHEDOCS and current_version %}
+              {%- set nav_version = current_version %}
+            {% endif %}
+            {% if nav_version %}
+              <div class="version">
+                {{ nav_version }}
+              </div>
+            {% endif %}
+          {% endif %}
+
+          {% include "searchbox.html" %}
+
+          {% endblock %}
+        </div>
+
+        {% block navigation %}
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          {% block menu %}
+            {#
+              The singlehtml builder doesn't handle this toctree call when the
+              toctree is empty. Skip building this for now.
+            #}
+            {% if 'singlehtml' not in builder %}
+              {% set global_toc = toctree(maxdepth=theme_navigation_depth|int,
+                                          collapse=theme_collapse_navigation|tobool,
+                                          includehidden=theme_includehidden|tobool,
+                                          titles_only=theme_titles_only|tobool) %}
+            {% endif %}
+            {% if global_toc %}
+              {{ global_toc }}
+            {% else %}
+              <!-- Local TOC -->
+              <div class="local-toc">{{ toc }}</div>
+            {% endif %}
+          {% endblock %}
+        </div>
+        {% endblock %}
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      {# MOBILE NAV, TRIGGLES SIDE NAV ON TOGGLE #}
+      <nav class="wy-nav-top" aria-label="top navigation">
+        {% block mobile_nav %}
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="{{ pathto(master_doc) }}">{{ project }}</a>
+        {% endblock %}
+      </nav>
+
+
+      <div class="wy-nav-content">
+      {%- block content %}
+        {% if theme_style_external_links|tobool %}
+        <div class="rst-content style-external-links">
+        {% else %}
+        <div class="rst-content">
+        {% endif %}
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+          {%- block document %}
+           <div itemprop="articleBody">
+            {% block body %}{% endblock %}
+           </div>
+           {% if self.comments()|trim %}
+           <div class="articleComments">
+            {% block comments %}{% endblock %}
+           </div>
+           {% endif%}
+          </div>
+          {%- endblock %}
+          {% include "footer.html" %}
+        </div>
+      {%- endblock %}
+      </div>
+
+    </section>
+
+  </div>
+  {% include "versions.html" %}
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable({{ 'true' if theme_sticky_navigation|tobool else 'false' }});
+      });
+  </script>
+
+  {# Do not conflict with RTD insertion of analytics script #}
+  {% if not READTHEDOCS %}
+    {% if theme_analytics_id %}
+    <!-- Theme Analytics -->
+    <script>
+    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+    ga('create', '{{ theme_analytics_id }}', 'auto');
+    {% if theme_analytics_anonymize_ip|tobool %}
+    ga('set', 'anonymizeIp', true);
+    {% endif %}
+    ga('send', 'pageview');
+    </script>
+
+    {% endif %}
+  {% endif %}
+
+  {%- block footer %} {% endblock %}
+
+</body>
+</html>
diff --git a/doc/_themes/ceph/static/ceph.css_t b/doc/_themes/ceph/static/ceph.css_t
new file mode 100644
index 000000000..1be74fd45
--- /dev/null
+++ b/doc/_themes/ceph/static/ceph.css_t
@@ -0,0 +1,199 @@
+@import url("css/theme.css");
+@import url('https://fonts.googleapis.com/css?family=Titillium+Web:400,300,700');
+
+.top-bar {
+  position: fixed;
+  min-width: auto;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: 5050;
+}
+
+.top-bar > div[role=navigation] {
+  border-top: 5px solid #F05C56;
+}
+
+.top-bar > div[role=navigation] > ul {
+  border-top: 5px solid #9C4850;
+  padding: 0 1em;
+}
+
+.top-bar > div[role=navigation] > hr {
+  margin: 0 0 10px;
+}
+
+body {
+  padding-top: 40px;
+}
+
+.wy-nav-side {
+  padding-top: 33px;
+}
+
+.wy-nav-content {
+  padding-top: 0;
+  min-width: 450px;
+  max-width: 940px;
+}
+
+.wy-breadcrumbs {
+  background-color: #80D2DC;
+  font-size: 0.9em;
+}
+
+.wy-nav-content-wrap.shift {
+  top: 2em;
+}
+
+section h1,
+section h2,
+section h3,
+section h4,
+section h5,
+section h6 {
+  font-weight: normal;
+  margin: 30px 0px 10px 0px;
+  padding: 5px 0 5px 10px;
+  color: #212224;
+  text-transform: uppercase;
+}
+
+section h1 {
+  border-top: 20px solid white; margin-top: 0;
+}
+
+section h1 { font-family: Titillium Web; font-size: 200%; background-color: #99DAE3;}
+section h2 { font-family: Titillium Web; font-size: 150%; background-color: #B2E3EA; }
+section h3 { font-family: Titillium Web; font-size: 120%; background-color: #CCECF1; }
+section h4 { font-family: Helvetica, Arial, sans-serif; font-size: 110%; background-color: #CCECF1; }
+section h5 { font-family: Helvetica, Arial, sans-serif; font-size: 100%; background-color: #CCECF1; }
+section h6 { font-family: Helvetica, Arial, sans-serif; font-size: 100%; background-color: #CCECF1; }
+
+/* nature theme */
+div.highlight {
+  background-color: white;
+}
+
+/* admonition */
+.rst-content .tip .admonition-title {
+  background-color: #1abc9c;
+}
+
+.admonition.tip {
+  background-color: #B2EBD8;
+  border: 1px solid #55AEBA;
+}
+
+.admonition.note {
+  background-color: #A9D1EC;
+  border: 1px solid #6AB0DE;
+}
+
+
+.rst-content .important .admonition-title {
+  background-color: #F05C56;
+}
+
+.admonition.important {
+  background-color: #FBD9D4;
+  border: 1px solid #F6A99E;
+}
+
+.admonition.warning {
+  border: 1px solid #F5DBC6;
+}
+
+/* table */
+.rst-content table.docutils {
+  border: 0;
+  border-collapse: collapse;
+}
+
+.rst-content li dl:not(.docutils) dt {
+  display: inline-block;
+}
+
+.rst-content table.docutils tr p:last-child {
+  margin-bottom: 0;
+}
+
+html.writer-html5 .rst-content table.docutils th,
+.rst-content table.docutils th,
+.rst-content table.docutils td,
+.rst-content table.docutils tbody > tr:last-child td {
+  border-top: 0;
+  border-left: 0;
+  border-right: 0;
+  border-bottom: 1px solid #aaa;
+  white-space: normal;
+}
+
+.rst-content table.docutils ul {
+  margin-bottom: inherit;
+}
+
+/* lists */
+.rst-content .section dl:not(.docutils) dt {
+  display: inline-block;
+}
+
+.rst-content table.docutils ul + ul {
+  margin-top: 1em;
+}
+
+.rst-content section ul li {
+  list-style: disc;
+  margin-left: 24px;
+}
+
+.rst-content section ol li>p,
+.rst-content section ol li>p:last-child,
+.rst-content section ul li>p,
+.rst-content section ul li>p:last-child {
+  margin-bottom: 12px;
+}
+
+.rst-content section ol li>p:only-child,
+.rst-content section ol li>p:only-child:last-child,
+.rst-content section ul li>p:only-child,
+.rst-content section ul li>p:only-child:last-child {
+  margin-bottom: 0;
+}
+
+/* versions */
+.injected .rst-versions.rst-badge {
+  left: 0;
+  right: 0;
+  bottom: 0;
+}
+
+.injected .rst-versions.rst-badge .rst-current-version {
+  bottom: 0;
+  line-height: 1px;
+  padding: 12px;
+  height: auto;
+}
+
+.columns-2,
+.columns-3 {
+  display: flex;
+}
+
+.columns-2 > div,
+.columns-3 > div {
+  flex: 1;
+  padding: 0 10px 10px 0;
+}
+
+.columns-2 > div {
+  width: 50%;
+}
+
+.columns-3 > div {
+  width: 33.33%;
+}
+
+div[class*="highlight-"] {
+  margin-bottom: 12px;
+}
diff --git a/doc/_themes/ceph/theme.conf b/doc/_themes/ceph/theme.conf
new file mode 100644
index 000000000..48fb4140d
--- /dev/null
+++ b/doc/_themes/ceph/theme.conf
@@ -0,0 +1,4 @@
+[theme]
+inherit = sphinx_rtd_theme
+stylesheet = ceph.css
+pygments_style = tango
diff --git a/doc/api/index.rst b/doc/api/index.rst
new file mode 100644
index 000000000..7007b0dc1
--- /dev/null
+++ b/doc/api/index.rst
@@ -0,0 +1,47 @@
+===================
+ API Documentation
+===================
+
+Ceph RESTful API
+================
+
+See :ref:`mgr ceph api`.
+
+Ceph Storage Cluster APIs
+=========================
+
+See :ref:`rados api`
+
+
+Ceph File System APIs
+=====================
+
+See :ref:`cephfs api`
+
+
+Ceph Block Device APIs
+======================
+
+See :ref:`rbd api py`
+
+Ceph RADOS Gateway APIs
+=======================
+
+See :ref:`radosgw api`
+
+Ceph Object Store APIs
+======================
+
+- See :ref:`S3-compatible API <radosgw s3>`.
+- See :ref:`Swift-compatible API <radosgw swift>`.
+- See :ref:`Admin Ops API<radosgw admin ops>`.
+
+Ceph MON Command API
+====================
+
+- See :doc:`Mon command API <mon_command_api>`.
+
+.. toctree::
+    :hidden:
+    
+    mon_command_api
diff --git a/doc/api/mon_command_api.rst b/doc/api/mon_command_api.rst
new file mode 100644
index 000000000..51a752a84
--- /dev/null
+++ b/doc/api/mon_command_api.rst
@@ -0,0 +1,3 @@
+.. ceph-mgr-commands:: src/pybind/mgr
+   :python_path: src/pybind
+.. ceph-mon-commands:: src/mon/MonCommands.h src/mgr/MgrCommands.h
diff --git a/doc/architecture.rst b/doc/architecture.rst
new file mode 100644
index 000000000..852225ce6
--- /dev/null
+++ b/doc/architecture.rst
@@ -0,0 +1,1688 @@
+==============
+ Architecture
+==============
+
+:term:`Ceph` uniquely delivers **object, block, and file storage** in one
+unified system. Ceph is highly reliable, easy to manage, and free. The power of
+Ceph can transform your company's IT infrastructure and your ability to manage
+vast amounts of data. Ceph delivers extraordinary scalability–thousands of
+clients accessing petabytes to exabytes of data. A :term:`Ceph Node` leverages
+commodity hardware and intelligent daemons, and a :term:`Ceph Storage Cluster`
+accommodates large numbers of nodes, which communicate with each other to
+replicate and redistribute data dynamically.
+
+.. image:: images/stack.png
+
+.. _arch-ceph-storage-cluster:
+
+The Ceph Storage Cluster
+========================
+
+Ceph provides an infinitely scalable :term:`Ceph Storage Cluster` based upon
+:abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, which you can read
+about in `RADOS - A Scalable, Reliable Storage Service for Petabyte-scale
+Storage Clusters`_.
+
+A Ceph Storage Cluster consists of multiple types of daemons:
+
+- :term:`Ceph Monitor`
+- :term:`Ceph OSD Daemon`
+- :term:`Ceph Manager`
+- :term:`Ceph Metadata Server`
+
+.. _arch_monitor:
+
+Ceph Monitors maintain the master copy of the cluster map, which they provide
+to Ceph clients. Provisioning multiple monitors within the Ceph cluster ensures
+availability in the event that one of the monitor daemons or its host fails.
+The Ceph monitor provides copies of the cluster map to storage cluster clients.
+
+A Ceph OSD Daemon checks its own state and the state of other OSDs and reports 
+back to monitors.
+
+A Ceph Manager serves as an endpoint for monitoring, orchestration, and plug-in
+modules.
+
+A Ceph Metadata Server (MDS) manages file metadata when CephFS is used to
+provide file services.
+
+Storage cluster clients and :term:`Ceph OSD Daemon`\s use the CRUSH algorithm
+to compute information about data location. This means that clients and OSDs
+are not bottlenecked by a central lookup table. Ceph's high-level features
+include a native interface to the Ceph Storage Cluster via ``librados``, and a
+number of service interfaces built on top of ``librados``.
+
+Storing Data
+------------
+
+The Ceph Storage Cluster receives data from :term:`Ceph Client`\s--whether it
+comes through a :term:`Ceph Block Device`, :term:`Ceph Object Storage`, the
+:term:`Ceph File System`, or a custom implementation that you create by using
+``librados``. The data received by the Ceph Storage Cluster is stored as RADOS
+objects. Each object is stored on an :term:`Object Storage Device` (this is
+also called an "OSD"). Ceph OSDs control read, write, and replication
+operations on storage drives. The default BlueStore back end stores objects 
+in a monolithic, database-like fashion.
+
+.. ditaa::
+
+           /------\       +-----+       +-----+
+           | obj  |------>| {d} |------>| {s} |
+           \------/       +-----+       +-----+
+   
+            Object         OSD          Drive
+
+Ceph OSD Daemons store data as objects in a flat namespace. This means that
+objects are not stored in a hierarchy of directories. An object has an
+identifier, binary data, and metadata consisting of name/value pairs.
+:term:`Ceph Client`\s determine the semantics of the object data. For example,
+CephFS uses metadata to store file attributes such as the file owner, the
+created date, and the last modified date.
+
+
+.. ditaa::
+
+           /------+------------------------------+----------------\
+           | ID   | Binary Data                  | Metadata       |
+           +------+------------------------------+----------------+
+           | 1234 | 0101010101010100110101010010 | name1 = value1 | 
+           |      | 0101100001010100110101010010 | name2 = value2 |
+           |      | 0101100001010100110101010010 | nameN = valueN |
+           \------+------------------------------+----------------/    
+
+.. note:: An object ID is unique across the entire cluster, not just the local
+   filesystem.
+
+
+.. index:: architecture; high availability, scalability
+
+.. _arch_scalability_and_high_availability:
+
+Scalability and High Availability
+---------------------------------
+
+In traditional architectures, clients talk to a centralized component. This
+centralized component might be a gateway, a broker, an API, or a facade. A
+centralized component of this kind acts as a single point of entry to a complex
+subsystem. Architectures that rely upon such a centralized component have a
+single point of failure and incur limits to performance and scalability. If
+the centralized component goes down, the whole system becomes unavailable.
+
+Ceph eliminates this centralized component. This enables clients to interact
+with Ceph OSDs directly. Ceph OSDs create object replicas on other Ceph Nodes
+to ensure data safety and high availability. Ceph also uses a cluster of
+monitors to ensure high availability. To eliminate centralization, Ceph uses an
+algorithm called :abbr:`CRUSH (Controlled Replication Under Scalable Hashing)`.
+
+
+.. index:: CRUSH; architecture
+
+CRUSH Introduction
+~~~~~~~~~~~~~~~~~~
+
+Ceph Clients and Ceph OSD Daemons both use the :abbr:`CRUSH (Controlled
+Replication Under Scalable Hashing)` algorithm to compute information about
+object location instead of relying upon a central lookup table. CRUSH provides
+a better data management mechanism than do older approaches, and CRUSH enables
+massive scale by distributing the work to all the OSD daemons in the cluster
+and all the clients that communicate with them. CRUSH uses intelligent data
+replication to ensure resiliency, which is better suited to hyper-scale
+storage. The following sections provide additional details on how CRUSH works.
+For a detailed discussion of CRUSH, see `CRUSH - Controlled, Scalable,
+Decentralized Placement of Replicated Data`_.
+
+.. index:: architecture; cluster map
+
+.. _architecture_cluster_map:
+
+Cluster Map
+~~~~~~~~~~~
+
+In order for a Ceph cluster to function properly, Ceph Clients and Ceph OSDs
+must have current information about the cluster's topology. Current information
+is stored in the "Cluster Map", which is in fact a collection of five maps. The
+five maps that constitute the cluster map are:
+
+#. **The Monitor Map:** Contains the cluster ``fsid``, the position, the name,
+   the address, and the TCP port of each monitor. The monitor map specifies the
+   current epoch, the time of the monitor map's creation, and the time of the
+   monitor map's last modification.  To view a monitor map, run ``ceph mon
+   dump``.   
+   
+#. **The OSD Map:** Contains the cluster ``fsid``, the time of the OSD map's
+   creation, the time of the OSD map's last modification, a list of pools, a
+   list of replica sizes, a list of PG numbers, and a list of OSDs and their
+   statuses (for example, ``up``, ``in``). To view an OSD map, run ``ceph
+   osd dump``. 
+   
+#. **The PG Map:** Contains the PG version, its time stamp, the last OSD map
+   epoch, the full ratios, and the details of each placement group. This
+   includes the PG ID, the `Up Set`, the `Acting Set`, the state of the PG (for
+   example, ``active + clean``), and data usage statistics for each pool.
+
+#. **The CRUSH Map:** Contains a list of storage devices, the failure domain
+   hierarchy (for example, ``device``, ``host``, ``rack``, ``row``, ``room``),
+   and rules for traversing the hierarchy when storing data. To view a CRUSH
+   map, run ``ceph osd getcrushmap -o {filename}`` and then decompile it by
+   running ``crushtool -d {comp-crushmap-filename} -o
+   {decomp-crushmap-filename}``. Use a text editor or ``cat`` to view the
+   decompiled map.
+
+#. **The MDS Map:** Contains the current MDS map epoch, when the map was 
+   created, and the last time it changed. It also contains the pool for 
+   storing metadata, a list of metadata servers, and which metadata servers
+   are ``up`` and ``in``. To view an MDS map, execute ``ceph fs dump``.
+
+Each map maintains a history of changes to its operating state. Ceph Monitors
+maintain a master copy of the cluster map. This master copy includes the
+cluster members, the state of the cluster, changes to the cluster, and
+information recording the overall health of the Ceph Storage Cluster.
+
+.. index:: high availability; monitor architecture
+
+High Availability Monitors
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A Ceph Client must contact a Ceph Monitor and obtain a current copy of the
+cluster map in order to read data from or to write data to the Ceph cluster.
+
+It is possible for a Ceph cluster to function properly with only a single
+monitor, but a Ceph cluster that has only a single monitor has a single point
+of failure: if the monitor goes down, Ceph clients will be unable to read data
+from or write data to the cluster.
+
+Ceph leverages a cluster of monitors in order to increase reliability and fault
+tolerance. When a cluster of monitors is used, however, one or more of the
+monitors in the cluster can fall behind due to latency or other faults. Ceph
+mitigates these negative effects by requiring multiple monitor instances to
+agree about the state of the cluster. To establish consensus among the monitors
+regarding the state of the cluster, Ceph uses the `Paxos`_ algorithm and a
+majority of monitors (for example, one in a cluster that contains only one
+monitor, two in a cluster that contains three monitors, three in a cluster that
+contains five monitors, four in a cluster that contains six monitors, and so
+on).
+
+See the `Monitor Config Reference`_ for more detail on configuring monitors.
+
+.. index:: architecture; high availability authentication
+
+.. _arch_high_availability_authentication:
+
+High Availability Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``cephx`` authentication system is used by Ceph to authenticate users and
+daemons and to protect against man-in-the-middle attacks. 
+
+.. note:: The ``cephx`` protocol does not address data encryption in transport 
+   (for example, SSL/TLS) or encryption at rest.
+
+``cephx`` uses shared secret keys for authentication. This means that both the
+client and the monitor cluster keep a copy of the client's secret key. 
+
+The ``cephx`` protocol makes it possible for each party to prove to the other
+that it has a copy of the key without revealing it. This provides mutual
+authentication and allows the cluster to confirm (1) that the user has the
+secret key and (2) that the user can be confident that the cluster has a copy
+of the secret key.
+
+As stated in :ref:`Scalability and High Availability
+<arch_scalability_and_high_availability>`, Ceph does not have any centralized
+interface between clients and the Ceph object store. By avoiding such a
+centralized interface, Ceph avoids the bottlenecks that attend such centralized
+interfaces. However, this means that clients must interact directly with OSDs.
+Direct interactions between Ceph clients and OSDs require authenticated
+connections. The ``cephx`` authentication system establishes and sustains these
+authenticated connections.
+
+The ``cephx`` protocol operates in a manner similar to `Kerberos`_. 
+
+A user invokes a Ceph client to contact a monitor. Unlike Kerberos, each
+monitor can authenticate users and distribute keys, which means that there is
+no single point of failure and no bottleneck when using ``cephx``. The monitor
+returns an authentication data structure that is similar to a Kerberos ticket.
+This authentication data structure contains a session key for use in obtaining
+Ceph services. The session key is itself encrypted with the user's permanent
+secret key, which means that only the user can request services from the Ceph
+Monitors. The client then uses the session key to request services from the
+monitors, and the monitors provide the client with a ticket that authenticates
+the client against the OSDs that actually handle data. Ceph Monitors and OSDs
+share a secret, which means that the clients can use the ticket provided by the
+monitors to authenticate against any OSD or metadata server in the cluster. 
+
+Like Kerberos tickets, ``cephx`` tickets expire. An attacker cannot use an
+expired ticket or session key that has been obtained surreptitiously. This form
+of authentication prevents attackers who have access to the communications
+medium from creating bogus messages under another user's identity and prevents
+attackers from altering another user's legitimate messages, as long as the
+user's secret key is not divulged before it expires.
+
+An administrator must set up users before using ``cephx``.  In the following
+diagram, the ``client.admin`` user invokes ``ceph auth get-or-create-key`` from
+the command line to generate a username and secret key. Ceph's ``auth``
+subsystem generates the username and key, stores a copy on the monitor(s), and
+transmits the user's secret back to the ``client.admin`` user. This means that
+the client and the monitor share a secret key.
+
+.. note:: The ``client.admin`` user must provide the user ID and 
+   secret key to the user in a secure manner. 
+
+.. ditaa::
+
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |  request to   |
+                | create a user |
+                |-------------->|----------+ create user
+                |               |          | and                 
+                |<--------------|<---------+ store key
+                | transmit key  |
+                |               |
+
+Here is how a client authenticates with a monitor. The client passes the user
+name to the monitor. The monitor generates a session key that is encrypted with
+the secret key associated with the ``username``. The monitor transmits the
+encrypted ticket to the client. The client uses the shared secret key to
+decrypt the payload. The session key identifies the user, and this act of
+identification will last for the duration of the session.  The client requests
+a ticket for the user, and the ticket is signed with the session key. The
+monitor generates a ticket and uses the user's secret key to encrypt it. The
+encrypted ticket is transmitted to the client. The client decrypts the ticket
+and uses it to sign requests to OSDs and to metadata servers in the cluster.
+
+.. ditaa::
+
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |  authenticate |
+                |-------------->|----------+ generate and
+                |               |          | encrypt                
+                |<--------------|<---------+ session key
+                | transmit      |
+                | encrypted     |
+                | session key   |
+                |               |             
+                |-----+ decrypt |
+                |     | session | 
+                |<----+ key     |              
+                |               |
+                |  req. ticket  |
+                |-------------->|----------+ generate and
+                |               |          | encrypt                
+                |<--------------|<---------+ ticket
+                | recv. ticket  |
+                |               |             
+                |-----+ decrypt |
+                |     | ticket  | 
+                |<----+         |              
+
+
+The ``cephx`` protocol authenticates ongoing communications between the clients
+and Ceph daemons. After initial authentication, each message sent between a
+client and a daemon is signed using a ticket that can be verified by monitors,
+OSDs, and metadata daemons. This ticket is verified by using the secret shared
+between the client and the daemon.
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+     +-------+
+           |  Client |     | Monitor |     |  MDS  |     |  OSD  |
+           +---------+     +---------+     +-------+     +-------+
+                |  request to   |              |             |
+                | create a user |              |             |               
+                |-------------->| mon and      |             |
+                |<--------------| client share |             |
+                |    receive    | a secret.    |             |
+                | shared secret |              |             |
+                |               |<------------>|             |
+                |               |<-------------+------------>|
+                |               | mon, mds,    |             |
+                | authenticate  | and osd      |             |  
+                |-------------->| share        |             |
+                |<--------------| a secret     |             |
+                |  session key  |              |             |
+                |               |              |             |
+                |  req. ticket  |              |             |
+                |-------------->|              |             |
+                |<--------------|              |             |
+                | recv. ticket  |              |             |
+                |               |              |             |
+                |   make request (CephFS only) |             |
+                |----------------------------->|             |
+                |<-----------------------------|             |
+                | receive response (CephFS only)             |
+                |                                            |
+                |                make request                |
+                |------------------------------------------->|  
+                |<-------------------------------------------|
+                               receive response
+
+This authentication protects only the connections between Ceph clients and Ceph
+daemons. The authentication is not extended beyond the Ceph client. If a user
+accesses the Ceph client from a remote host, cephx authentication will not be
+applied to the connection between the user's host and the client host.
+
+See `Cephx Config Guide`_ for more on configuration details. 
+
+See `User Management`_ for more on user management.
+
+See :ref:`A Detailed Description of the Cephx Authentication Protocol
+<cephx_2012_peter>` for more on the distinction between authorization and
+authentication and for a step-by-step explanation of the setup of ``cephx``
+tickets and session keys.
+
+.. index:: architecture; smart daemons and scalability
+
+Smart Daemons Enable Hyperscale
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A feature of many storage clusters is a centralized interface that keeps track
+of the nodes that clients are permitted to access. Such centralized
+architectures provide services to clients by means of a double dispatch. At the
+petabyte-to-exabyte scale, such double dispatches are a significant
+bottleneck.
+
+Ceph obviates this bottleneck: Ceph's OSD Daemons AND Ceph clients are
+cluster-aware. Like Ceph clients, each Ceph OSD Daemon is aware of other Ceph
+OSD Daemons in the cluster. This enables Ceph OSD Daemons to interact directly
+with other Ceph OSD Daemons and to interact directly with Ceph Monitors.  Being
+cluster-aware makes it possible for Ceph clients to interact directly with Ceph
+OSD Daemons.
+
+Because Ceph clients, Ceph monitors, and Ceph OSD daemons interact with one
+another directly, Ceph OSD daemons can make use of the aggregate CPU and RAM
+resources of the nodes in the Ceph cluster. This means that a Ceph cluster can
+easily perform tasks that a cluster with a centralized interface would struggle
+to perform. The ability of Ceph nodes to make use of the computing power of
+the greater cluster provides several benefits:
+
+#. **OSDs Service Clients Directly:** Network devices can support only a
+   limited number of concurrent connections. Because Ceph clients contact
+   Ceph OSD daemons directly without first connecting to a central interface,
+   Ceph enjoys improved perfomance and increased system capacity relative to
+   storage redundancy strategies that include a central interface. Ceph clients
+   maintain sessions only when needed, and maintain those sessions with only
+   particular Ceph OSD daemons, not with a centralized interface.
+
+#. **OSD Membership and Status**: When Ceph OSD Daemons join a cluster, they
+   report their status. At the lowest level, the Ceph OSD Daemon status is
+   ``up`` or ``down``: this reflects whether the Ceph OSD daemon is running and
+   able to service Ceph Client requests. If a Ceph OSD Daemon is ``down`` and
+   ``in`` the Ceph Storage Cluster, this status may indicate the failure of the
+   Ceph OSD Daemon. If a Ceph OSD Daemon is not running because it has crashed,
+   the Ceph OSD Daemon cannot notify the Ceph Monitor that it is ``down``. The
+   OSDs periodically send messages to the Ceph Monitor (in releases prior to
+   Luminous, this was done by means of ``MPGStats``, and beginning with the
+   Luminous release, this has been done with ``MOSDBeacon``). If the Ceph
+   Monitors receive no such message after a configurable period of time,
+   then they mark the OSD ``down``. This mechanism is a failsafe, however.
+   Normally, Ceph OSD Daemons determine if a neighboring OSD is ``down`` and
+   report it to the Ceph Monitors. This contributes to making Ceph Monitors 
+   lightweight processes. See `Monitoring OSDs`_ and `Heartbeats`_ for
+   additional details.
+
+#. **Data Scrubbing:** To maintain data consistency, Ceph OSD Daemons scrub
+   RADOS objects. Ceph OSD Daemons compare the metadata of their own local
+   objects against the metadata of the replicas of those objects, which are
+   stored on other OSDs. Scrubbing occurs on a per-Placement-Group basis, finds
+   mismatches in object size and finds metadata mismatches, and is usually
+   performed daily. Ceph OSD Daemons perform deeper scrubbing by comparing the
+   data in objects, bit-for-bit, against their checksums. Deep scrubbing finds
+   bad sectors on drives that are not detectable with light scrubs. See `Data
+   Scrubbing`_ for details on configuring scrubbing.
+
+#. **Replication:** Data replication involves a collaboration between Ceph
+   Clients and Ceph OSD Daemons. Ceph OSD Daemons use the CRUSH algorithm to
+   determine the storage location of object replicas. Ceph clients use the
+   CRUSH algorithm to determine the storage location of an object, then the
+   object is mapped to a pool and to a placement group, and then the client
+   consults the CRUSH map to identify the placement group's primary OSD.
+
+   After identifying the target placement group, the client writes the object
+   to the identified placement group's primary OSD. The primary OSD then
+   consults its own copy of the CRUSH map to identify secondary and tertiary
+   OSDS, replicates the object to the placement groups in those secondary and
+   tertiary OSDs, confirms that the object was stored successfully in the
+   secondary and tertiary OSDs, and reports to the client that the object
+   was stored successfully.
+
+.. ditaa::
+
+             +----------+
+             |  Client  |
+             |          |
+             +----------+
+                 *  ^
+      Write (1)  |  |  Ack (6)
+                 |  |
+                 v  *
+            +-------------+
+            | Primary OSD |
+            |             |
+            +-------------+
+              *  ^   ^  *
+    Write (2) |  |   |  |  Write (3)
+       +------+  |   |  +------+
+       |  +------+   +------+  |
+       |  | Ack (4)  Ack (5)|  | 
+       v  *                 *  v
+ +---------------+   +---------------+
+ | Secondary OSD |   | Tertiary OSD  |
+ |               |   |               |
+ +---------------+   +---------------+
+
+By performing this act of data replication, Ceph OSD Daemons relieve Ceph
+clients of the burden of replicating data.
+
+Dynamic Cluster Management
+--------------------------
+
+In the `Scalability and High Availability`_ section, we explained how Ceph uses
+CRUSH, cluster topology, and intelligent daemons to scale and maintain high
+availability. Key to Ceph's design is the autonomous, self-healing, and
+intelligent Ceph OSD Daemon. Let's take a deeper look at how CRUSH works to
+enable modern cloud storage infrastructures to place data, rebalance the
+cluster, and adaptively place and balance data and recover from faults.
+
+.. index:: architecture; pools
+
+About Pools
+~~~~~~~~~~~
+
+The Ceph storage system supports the notion of 'Pools', which are logical
+partitions for storing objects.
+   
+Ceph Clients retrieve a `Cluster Map`_ from a Ceph Monitor, and write RADOS
+objects to pools. The way that Ceph places the data in the pools is determined
+by the pool's ``size`` or number of replicas, the CRUSH rule, and the number of
+placement groups in the pool.
+
+.. ditaa::
+
+            +--------+  Retrieves  +---------------+
+            | Client |------------>|  Cluster Map  |
+            +--------+             +---------------+
+                 |
+                 v      Writes
+              /-----\
+              | obj |
+              \-----/
+                 |      To
+                 v
+            +--------+           +---------------+
+            |  Pool  |---------->|  CRUSH Rule   |
+            +--------+  Selects  +---------------+
+                 
+
+Pools set at least the following parameters:
+
+- Ownership/Access to Objects
+- The Number of Placement Groups, and 
+- The CRUSH Rule to Use.
+
+See `Set Pool Values`_ for details.
+
+
+.. index: architecture; placement group mapping
+
+Mapping PGs to OSDs
+~~~~~~~~~~~~~~~~~~~
+
+Each pool has a number of placement groups (PGs) within it. CRUSH dynamically
+maps PGs to OSDs. When a Ceph Client stores objects, CRUSH maps each RADOS
+object to a PG. 
+
+This mapping of RADOS objects to PGs implements an abstraction and indirection
+layer between Ceph OSD Daemons and Ceph Clients. The Ceph Storage Cluster must
+be able to grow (or shrink) and redistribute data adaptively when the internal
+topology changes. 
+
+If the Ceph Client "knew" which Ceph OSD Daemons were storing which objects, a
+tight coupling would exist between the Ceph Client and the Ceph OSD Daemon.
+But Ceph avoids any such tight coupling. Instead, the CRUSH algorithm maps each
+RADOS object to a placement group and then maps each placement group to one or
+more Ceph OSD Daemons. This "layer of indirection" allows Ceph to rebalance
+dynamically when new Ceph OSD Daemons and their underlying OSD devices come
+online. The following diagram shows how the CRUSH algorithm maps objects to
+placement groups, and how it maps placement groups to OSDs.
+
+.. ditaa::
+
+           /-----\  /-----\  /-----\  /-----\  /-----\
+           | obj |  | obj |  | obj |  | obj |  | obj |
+           \-----/  \-----/  \-----/  \-----/  \-----/
+              |        |        |        |        |
+              +--------+--------+        +---+----+
+              |                              |
+              v                              v
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+               |                              |
+               |      +-----------------------+---+
+        +------+------+-------------+             |
+        |             |             |             |
+        v             v             v             v
+   /----------\  /----------\  /----------\  /----------\ 
+   |          |  |          |  |          |  |          |
+   |  OSD #1  |  |  OSD #2  |  |  OSD #3  |  |  OSD #4  |
+   |          |  |          |  |          |  |          |
+   \----------/  \----------/  \----------/  \----------/  
+
+The client uses its copy of the cluster map and the CRUSH algorithm to compute
+precisely which OSD it will use when reading or writing a particular object.
+
+.. index:: architecture; calculating PG IDs
+
+Calculating PG IDs
+~~~~~~~~~~~~~~~~~~
+
+When a Ceph Client binds to a Ceph Monitor, it retrieves the latest version of
+the `Cluster Map`_. When a client has been equipped with a copy of the cluster
+map, it is aware of all the monitors, OSDs, and metadata servers in the
+cluster. **However, even equipped with a copy of the latest version of the
+cluster map, the client doesn't know anything about object locations.** 
+
+**Object locations must be computed.**
+
+The client requies only the object ID and the name of the pool in order to
+compute the object location.
+
+Ceph stores data in named pools (for example,  "liverpool"). When a client
+stores a named object (for example, "john", "paul", "george", or "ringo") it
+calculates a placement group by using the object name, a hash code, the number
+of PGs in the pool, and the pool name. Ceph clients use the following steps to
+compute PG IDs.
+
+#. The client inputs the pool name and the object ID. (for example: pool =
+   "liverpool" and object-id = "john")
+#. Ceph hashes the object ID.
+#. Ceph calculates the hash, modulo the number of PGs (for example: ``58``), to
+   get a PG ID.
+#. Ceph uses the pool name to retrieve the pool ID: (for example: "liverpool" =
+   ``4``)
+#. Ceph prepends the pool ID to the PG ID (for example: ``4.58``).
+
+It is much faster to compute object locations than to perform object location
+query over a chatty session. The :abbr:`CRUSH (Controlled Replication Under
+Scalable Hashing)` algorithm allows a client to compute where objects are
+expected to be stored, and enables the client to contact the primary OSD to
+store or retrieve the objects.
+
+.. index:: architecture; PG Peering
+
+Peering and Sets
+~~~~~~~~~~~~~~~~
+
+In previous sections, we noted that Ceph OSD Daemons check each other's
+heartbeats and report back to Ceph Monitors. Ceph OSD daemons also 'peer',
+which is the process of bringing all of the OSDs that store a Placement Group
+(PG) into agreement about the state of all of the RADOS objects (and their
+metadata) in that PG. Ceph OSD Daemons `Report Peering Failure`_ to the Ceph
+Monitors. Peering issues usually resolve themselves; however, if the problem
+persists, you may need to refer to the `Troubleshooting Peering Failure`_
+section.
+
+.. Note:: PGs that agree on the state of the cluster do not necessarily have
+   the current data yet. 
+
+The Ceph Storage Cluster was designed to store at least two copies of an object
+(that is, ``size = 2``), which is the minimum requirement for data safety. For
+high availability, a Ceph Storage Cluster should store more than two copies of
+an object (that is, ``size = 3`` and ``min size = 2``) so that it can continue
+to run in a ``degraded`` state while maintaining data safety.
+
+.. warning:: Although we say here that R2 (replication with two copies) is the
+   minimum requirement for data safety, R3 (replication with three copies) is
+   recommended. On a long enough timeline, data stored with an R2 strategy will
+   be lost.
+
+As explained in the diagram in `Smart Daemons Enable Hyperscale`_, we do not
+name the Ceph OSD Daemons specifically (for example, ``osd.0``, ``osd.1``,
+etc.), but rather refer to them as *Primary*, *Secondary*, and so forth. By
+convention, the *Primary* is the first OSD in the *Acting Set*, and is
+responsible for orchestrating the peering process for each placement group
+where it acts as the *Primary*. The *Primary* is the **ONLY** OSD in a given
+placement group that accepts client-initiated writes to objects.
+
+The set of OSDs that is responsible for a placement group is called the
+*Acting Set*. The term "*Acting Set*" can refer either to the Ceph OSD Daemons
+that are currently responsible for the placement group, or to the Ceph OSD
+Daemons that were responsible for a particular placement group as of some
+epoch.
+
+The Ceph OSD daemons that are part of an *Acting Set* might not always be
+``up``. When an OSD in the *Acting Set* is ``up``, it is part of the *Up Set*.
+The *Up Set* is an important distinction, because Ceph can remap PGs to other
+Ceph OSD Daemons when an OSD fails. 
+
+.. note:: Consider a hypothetical *Acting Set* for a PG that contains
+   ``osd.25``, ``osd.32`` and ``osd.61``. The first OSD (``osd.25``), is the
+   *Primary*. If that OSD fails, the Secondary (``osd.32``), becomes the
+   *Primary*, and ``osd.25`` is removed from the *Up Set*.
+
+.. index:: architecture; Rebalancing
+
+Rebalancing
+~~~~~~~~~~~
+
+When you add a Ceph OSD Daemon to a Ceph Storage Cluster, the cluster map gets
+updated with the new OSD. Referring back to `Calculating PG IDs`_, this changes
+the cluster map. Consequently, it changes object placement, because it changes
+an input for the calculations. The following diagram depicts the rebalancing
+process (albeit rather crudely, since it is substantially less impactful with
+large clusters) where some, but not all of the PGs migrate from existing OSDs
+(OSD 1, and OSD 2) to the new OSD (OSD 3). Even when rebalancing, CRUSH is
+stable. Many of the placement groups remain in their original configuration,
+and each OSD gets some added capacity, so there are no load spikes on the 
+new OSD after rebalancing is complete.
+
+
+.. ditaa::
+
+           +--------+     +--------+
+   Before  |  OSD 1 |     |  OSD 2 |
+           +--------+     +--------+
+           |  PG #1 |     | PG #6  |
+           |  PG #2 |     | PG #7  |
+           |  PG #3 |     | PG #8  |
+           |  PG #4 |     | PG #9  |
+           |  PG #5 |     | PG #10 |
+           +--------+     +--------+
+
+           +--------+     +--------+     +--------+
+    After  |  OSD 1 |     |  OSD 2 |     |  OSD 3 |
+           +--------+     +--------+     +--------+
+           |  PG #1 |     | PG #7  |     |  PG #3 |
+           |  PG #2 |     | PG #8  |     |  PG #6 |
+           |  PG #4 |     | PG #10 |     |  PG #9 |
+           |  PG #5 |     |        |     |        |
+           |        |     |        |     |        |
+           +--------+     +--------+     +--------+
+
+
+.. index:: architecture; Data Scrubbing
+
+Data Consistency
+~~~~~~~~~~~~~~~~
+
+As part of maintaining data consistency and cleanliness, Ceph OSDs also scrub
+objects within placement groups. That is, Ceph OSDs compare object metadata in
+one placement group with its replicas in placement groups stored in other
+OSDs. Scrubbing (usually performed daily) catches OSD bugs or filesystem
+errors, often as a result of hardware issues.  OSDs also perform deeper
+scrubbing by comparing data in objects bit-for-bit.  Deep scrubbing (by default
+performed weekly) finds bad blocks on a drive that weren't apparent in a light
+scrub.
+
+See `Data Scrubbing`_ for details on configuring scrubbing.
+
+
+
+
+
+.. index:: erasure coding
+
+Erasure Coding
+--------------
+
+An erasure coded pool stores each object as ``K+M`` chunks. It is divided into
+``K`` data chunks and ``M`` coding chunks. The pool is configured to have a size
+of ``K+M`` so that each chunk is stored in an OSD in the acting set. The rank of
+the chunk is stored as an attribute of the object.
+
+For instance an erasure coded pool can be created to use five OSDs (``K+M = 5``) and
+sustain the loss of two of them (``M = 2``).
+
+Reading and Writing Encoded Chunks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the object **NYAN** containing ``ABCDEFGHI`` is written to the pool, the erasure
+encoding function splits the content into three data chunks simply by dividing
+the content in three: the first contains ``ABC``, the second ``DEF`` and the
+last ``GHI``. The content will be padded if the content length is not a multiple
+of ``K``. The function also creates two coding chunks: the fourth with ``YXY``
+and the fifth with ``QGC``. Each chunk is stored in an OSD in the acting set.
+The chunks are stored in objects that have the same name (**NYAN**) but reside
+on different OSDs. The order in which the chunks were created must be preserved
+and is stored as an attribute of the object (``shard_t``), in addition to its
+name. Chunk 1 contains ``ABC`` and is stored on **OSD5** while chunk 4 contains
+``YXY`` and is stored on **OSD3**.
+
+
+.. ditaa::
+
+                            +-------------------+
+                       name |       NYAN        |
+                            +-------------------+
+                    content |     ABCDEFGHI     |
+                            +--------+----------+
+                                     |
+                                     |
+                                     v
+                              +------+------+
+              +---------------+ encode(3,2) +-----------+
+              |               +--+--+---+---+           |
+              |                  |  |   |               |
+              |          +-------+  |   +-----+         |
+              |          |          |         |         |
+           +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+     name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+           +------+   +------+   +------+  +------+  +------+
+    shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+           +------+   +------+   +------+  +------+  +------+
+  content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+           +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+              |          |          |         |         |
+              |          |          v         |         |
+              |          |       +--+---+     |         |
+              |          |       | OSD1 |     |         |
+              |          |       +------+     |         |
+              |          |                    |         |
+              |          |       +------+     |         |
+              |          +------>| OSD2 |     |         |
+              |                  +------+     |         |
+              |                               |         |
+              |                  +------+     |         |
+              |                  | OSD3 |<----+         |
+              |                  +------+               |
+              |                                         |
+              |                  +------+               |
+              |                  | OSD4 |<--------------+
+              |                  +------+
+              |
+              |                  +------+
+              +----------------->| OSD5 |
+                                 +------+
+
+
+When the object **NYAN** is read from the erasure coded pool, the decoding
+function reads three chunks: chunk 1 containing ``ABC``, chunk 3 containing
+``GHI`` and chunk 4 containing ``YXY``. Then, it rebuilds the original content
+of the object ``ABCDEFGHI``. The decoding function is informed that the chunks 2
+and 5 are missing (they are called 'erasures'). The chunk 5 could not be read
+because the **OSD4** is out. The decoding function can be called as soon as
+three chunks are read: **OSD2** was the slowest and its chunk was not taken into
+account.
+
+.. ditaa::
+
+	                         +-------------------+
+	                    name |       NYAN        |
+	                         +-------------------+
+	                 content |     ABCDEFGHI     |
+	                         +---------+---------+
+	                                   ^
+	                                   |
+	                                   |
+	                           +-------+-------+
+	                           |  decode(3,2)  |
+	            +------------->+  erasures 2,5 +<-+
+	            |              |               |  |
+	            |              +-------+-------+  |
+	            |                      ^          |
+	            |                      |          | 
+	            |                      |          |
+	         +--+---+   +------+   +---+--+   +---+--+
+	   name  | NYAN |   | NYAN |   | NYAN |   | NYAN |
+	         +------+   +------+   +------+   +------+
+	  shard  |  1   |   |  2   |   |  3   |   |  4   |
+	         +------+   +------+   +------+   +------+
+	content  | ABC  |   | DEF  |   | GHI  |   | YXY  |
+	         +--+---+   +--+---+   +--+---+   +--+---+
+	            ^          .          ^          ^
+	            |    TOO   .          |          |
+	            |    SLOW  .       +--+---+      |
+	            |          ^       | OSD1 |      |
+	            |          |       +------+      |
+	            |          |                     |
+	            |          |       +------+      |
+	            |          +-------| OSD2 |      |
+	            |                  +------+      |
+	            |                                |
+	            |                  +------+      |
+	            |                  | OSD3 |------+
+	            |                  +------+
+	            |
+	            |                  +------+
+	            |                  | OSD4 | OUT
+	            |                  +------+
+	            |
+	            |                  +------+
+	            +------------------| OSD5 |
+	                               +------+
+
+
+Interrupted Full Writes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+In an erasure coded pool, the primary OSD in the up set receives all write
+operations. It is responsible for encoding the payload into ``K+M`` chunks and
+sends them to the other OSDs. It is also responsible for maintaining an
+authoritative version of the placement group logs.
+
+In the following diagram, an erasure coded placement group has been created with
+``K = 2, M = 1`` and is supported by three OSDs, two for ``K`` and one for
+``M``. The acting set of the placement group is made of **OSD 1**, **OSD 2** and
+**OSD 3**. An object has been encoded and stored in the OSDs : the chunk
+``D1v1`` (i.e. Data chunk number 1, version 1) is on **OSD 1**, ``D2v1`` on
+**OSD 2** and ``C1v1`` (i.e. Coding chunk number 1, version 1) on **OSD 3**. The
+placement group logs on each OSD are identical (i.e. ``1,1`` for epoch 1,
+version 1).
+
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |             +-------------+
+   |         log |  Write Full |             |
+   |  +----+     |<------------+ Ceph Client |
+   |  |D1v1| 1,1 |      v1     |             |
+   |  +----+     |             +-------------+
+   +------+------+
+          |
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +--------->+  +----+     |
+          |          |  |D2v1| 1,1 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +--------->|  +----+     |
+                     |  |C1v1| 1,1 |
+                     |  +----+     |
+                     +-------------+
+
+**OSD 1** is the primary and receives a **WRITE FULL** from a client, which
+means the payload is to replace the object entirely instead of overwriting a
+portion of it. Version 2 (v2) of the object is created to override version 1
+(v1). **OSD 1** encodes the payload into three chunks: ``D1v2`` (i.e. Data
+chunk number 1 version 2) will be on **OSD 1**, ``D2v2`` on **OSD 2** and
+``C1v2`` (i.e. Coding chunk number 1 version 2) on **OSD 3**. Each chunk is sent
+to the target OSD, including the primary OSD which is responsible for storing
+chunks in addition to handling write operations and maintaining an authoritative
+version of the placement group logs. When an OSD receives the message
+instructing it to write the chunk, it also creates a new entry in the placement
+group logs to reflect the change. For instance, as soon as **OSD 3** stores
+``C1v2``, it adds the entry ``1,2`` ( i.e. epoch 1, version 2 ) to its logs.
+Because the OSDs work asynchronously, some chunks may still be in flight ( such
+as ``D2v2`` ) while others are acknowledged and persisted to storage drives
+(such as ``C1v1`` and ``D1v1``).
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |             +-------------+
+   |  |D1v2| 1,2 |  Write Full |             |
+   |  +----+     +<------------+ Ceph Client |
+   |             |      v2     |             |
+   |  +----+     |             +-------------+
+   |  |D1v1| 1,1 |           
+   |  +----+     |           
+   +------+------+           
+          |                  
+          |                  
+          |           +------+------+
+          |           |    OSD 2    |
+          |  +------+ |         log |
+          +->| D2v2 | |  +----+     |
+          |  +------+ |  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           +-------------+
+          |
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+
+
+If all goes well, the chunks are acknowledged on each OSD in the acting set and
+the logs' ``last_complete`` pointer can move from ``1,1`` to ``1,2``.
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |             +-------------+
+   |  |D1v2| 1,2 |  Write Full |             |
+   |  +----+     +<------------+ Ceph Client |
+   |             |      v2     |             |
+   |  +----+     |             +-------------+
+   |  |D1v1| 1,1 |           
+   |  +----+     |           
+   +------+------+           
+          |                  
+          |           +-------------+
+          |           |    OSD 2    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |D2v2| 1,2 |
+          +---------->+  +----+     |
+          |           |             |
+          |           |  +----+     |
+          |           |  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           +-------------+
+          |                  
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+
+
+Finally, the files used to store the chunks of the previous version of the
+object can be removed: ``D1v1`` on **OSD 1**, ``D2v1`` on **OSD 2** and ``C1v1``
+on **OSD 3**.
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |
+   |  |D1v2| 1,2 |
+   |  +----+     |
+   +------+------+
+          |
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +--------->+  +----+     |
+          |          |  |D2v2| 1,2 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +--------->|  +----+     |
+                     |  |C1v2| 1,2 |
+                     |  +----+     |
+                     +-------------+
+
+
+But accidents happen. If **OSD 1** goes down while ``D2v2`` is still in flight,
+the object's version 2 is partially written: **OSD 3** has one chunk but that is
+not enough to recover. It lost two chunks: ``D1v2`` and ``D2v2`` and the
+erasure coding parameters ``K = 2``, ``M = 1`` require that at least two chunks are
+available to rebuild the third. **OSD 4** becomes the new primary and finds that
+the ``last_complete`` log entry (i.e., all objects before this entry were known
+to be available on all OSDs in the previous acting set ) is ``1,1`` and that
+will be the head of the new authoritative log.
+
+.. ditaa::
+
+   +-------------+
+   |    OSD 1    |
+   |   (down)    |
+   | c333        |
+   +------+------+
+          |                  
+          |           +-------------+
+          |           |    OSD 2    |
+          |           |         log |
+          |           |  +----+     |
+          +---------->+  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           |             |
+          |           +-------------+
+          |                  
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+     Primary OSD
+   +-------------+
+   |    OSD 4    |
+   |         log |
+   |             |
+   |         1,1 |
+   |             |
+   +------+------+
+          
+
+
+The log entry 1,2 found on **OSD 3** is divergent from the new authoritative log
+provided by **OSD 4**: it is discarded and the file containing the ``C1v2``
+chunk is removed. The ``D1v1`` chunk is rebuilt with the ``decode`` function of
+the erasure coding library during scrubbing and stored on the new primary 
+**OSD 4**.
+
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 4    |
+   |         log |
+   |  +----+     |
+   |  |D1v1| 1,1 |
+   |  +----+     |
+   +------+------+
+          ^
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +----------+  +----+     |
+          |          |  |D2v1| 1,1 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +----------|  +----+     |
+                     |  |C1v1| 1,1 |
+                     |  +----+     |
+                     +-------------+
+
+   +-------------+
+   |    OSD 1    |
+   |   (down)    |
+   | c333        |
+   +-------------+
+
+See `Erasure Code Notes`_ for additional details.
+
+
+
+Cache Tiering
+-------------
+
+.. note:: Cache tiering is deprecated in Reef.
+
+A cache tier provides Ceph Clients with better I/O performance for a subset of
+the data stored in a backing storage tier. Cache tiering involves creating a
+pool of relatively fast/expensive storage devices (e.g., solid state drives)
+configured to act as a cache tier, and a backing pool of either erasure-coded
+or relatively slower/cheaper devices configured to act as an economical storage
+tier. The Ceph objecter handles where to place the objects and the tiering
+agent determines when to flush objects from the cache to the backing storage
+tier. So the cache tier and the backing storage tier are completely transparent 
+to Ceph clients.
+
+
+.. ditaa::
+
+           +-------------+
+           | Ceph Client |
+           +------+------+
+                  ^
+     Tiering is   |  
+    Transparent   |              Faster I/O
+        to Ceph   |           +---------------+
+     Client Ops   |           |               |   
+                  |    +----->+   Cache Tier  |
+                  |    |      |               |
+                  |    |      +-----+---+-----+
+                  |    |            |   ^ 
+                  v    v            |   |   Active Data in Cache Tier
+           +------+----+--+         |   |
+           |   Objecter   |         |   |
+           +-----------+--+         |   |
+                       ^            |   |   Inactive Data in Storage Tier
+                       |            v   |
+                       |      +-----+---+-----+
+                       |      |               |
+                       +----->|  Storage Tier |
+                              |               |
+                              +---------------+
+                                 Slower I/O
+
+See `Cache Tiering`_ for additional details.  Note that Cache Tiers can be
+tricky and their use is now discouraged.
+
+
+.. index:: Extensibility, Ceph Classes
+
+Extending Ceph
+--------------
+
+You can extend Ceph by creating shared object classes called 'Ceph Classes'.
+Ceph loads ``.so`` classes stored in the ``osd class dir`` directory dynamically
+(i.e., ``$libdir/rados-classes`` by default). When you implement a class, you
+can create new object methods that have the ability to call the native methods
+in the Ceph Object Store, or other class methods you incorporate via libraries
+or create yourself.
+
+On writes, Ceph Classes can call native or class methods, perform any series of
+operations on the inbound data and generate a resulting write transaction  that
+Ceph will apply atomically.
+
+On reads, Ceph Classes can call native or class methods, perform any series of
+operations on the outbound data and return the data to the client.
+
+.. topic:: Ceph Class Example
+
+   A Ceph class for a content management system that presents pictures of a
+   particular size and aspect ratio could take an inbound bitmap image, crop it
+   to a particular aspect ratio, resize it and embed an invisible copyright or 
+   watermark to help protect the intellectual property; then, save the 
+   resulting bitmap image to the object store.
+
+See ``src/objclass/objclass.h``, ``src/fooclass.cc`` and ``src/barclass`` for 
+exemplary implementations.
+
+
+Summary
+-------
+
+Ceph Storage Clusters are dynamic--like a living organism. Whereas, many storage
+appliances do not fully utilize the CPU and RAM of a typical commodity server,
+Ceph does. From heartbeats, to  peering, to rebalancing the cluster or
+recovering from faults,  Ceph offloads work from clients (and from a centralized
+gateway which doesn't exist in the Ceph architecture) and uses the computing
+power of the OSDs to perform the work. When referring to `Hardware
+Recommendations`_ and the `Network Config Reference`_,  be cognizant of the
+foregoing concepts to understand how Ceph utilizes computing resources.
+
+.. index:: Ceph Protocol, librados
+
+Ceph Protocol
+=============
+
+Ceph Clients use the native protocol for interacting with the Ceph Storage
+Cluster. Ceph packages this functionality into the ``librados`` library so that
+you can create your own custom Ceph Clients. The following diagram depicts the
+basic architecture.
+
+.. ditaa::
+
+            +---------------------------------+
+            |  Ceph Storage Cluster Protocol  |
+            |           (librados)            |
+            +---------------------------------+
+            +---------------+ +---------------+
+            |      OSDs     | |    Monitors   |
+            +---------------+ +---------------+
+
+
+Native Protocol and ``librados``
+--------------------------------
+
+Modern applications need a simple object storage interface with asynchronous
+communication capability. The Ceph Storage Cluster provides a simple object
+storage interface with asynchronous communication capability. The interface
+provides direct, parallel access to objects throughout the cluster.
+
+
+- Pool Operations
+- Snapshots and Copy-on-write Cloning
+- Read/Write Objects
+  - Create or Remove
+  - Entire Object or Byte Range
+  - Append or Truncate
+- Create/Set/Get/Remove XATTRs
+- Create/Set/Get/Remove Key/Value Pairs
+- Compound operations and dual-ack semantics
+- Object Classes
+
+
+.. index:: architecture; watch/notify
+
+Object Watch/Notify
+-------------------
+
+A client can register a persistent interest with an object and keep a session to
+the primary OSD open. The client can send a notification message and a payload to
+all watchers and receive notification when the watchers receive the
+notification. This enables a client to use any object as a
+synchronization/communication channel.
+
+
+.. ditaa::
+
+           +----------+     +----------+     +----------+     +---------------+
+           | Client 1 |     | Client 2 |     | Client 3 |     | OSD:Object ID |
+           +----------+     +----------+     +----------+     +---------------+
+                 |                |                |                  |
+                 |                |                |                  |
+                 |                |  Watch Object  |                  |               
+                 |--------------------------------------------------->|
+                 |                |                |                  |
+                 |<---------------------------------------------------|
+                 |                |   Ack/Commit   |                  |
+                 |                |                |                  |
+                 |                |  Watch Object  |                  |
+                 |                |---------------------------------->|
+                 |                |                |                  |
+                 |                |<----------------------------------|
+                 |                |   Ack/Commit   |                  |
+                 |                |                |   Watch Object   |
+                 |                |                |----------------->|
+                 |                |                |                  |
+                 |                |                |<-----------------|
+                 |                |                |    Ack/Commit    |
+                 |                |     Notify     |                  |               
+                 |--------------------------------------------------->|
+                 |                |                |                  |
+                 |<---------------------------------------------------|
+                 |                |     Notify     |                  |
+                 |                |                |                  |
+                 |                |<----------------------------------|
+                 |                |     Notify     |                  |
+                 |                |                |<-----------------|
+                 |                |                |      Notify      |
+                 |                |       Ack      |                  |               
+                 |----------------+---------------------------------->|
+                 |                |                |                  |
+                 |                |       Ack      |                  |
+                 |                +---------------------------------->|
+                 |                |                |                  |
+                 |                |                |        Ack       |
+                 |                |                |----------------->|
+                 |                |                |                  | 
+                 |<---------------+----------------+------------------|
+                 |                     Complete
+
+.. index:: architecture; Striping
+
+Data Striping
+-------------
+
+Storage devices have throughput limitations, which impact performance and
+scalability. So storage systems often support `striping`_--storing sequential
+pieces of information across multiple storage devices--to increase throughput
+and performance. The most common form of data striping comes from `RAID`_.
+The RAID type most similar to Ceph's striping is `RAID 0`_, or a 'striped
+volume'. Ceph's striping offers the throughput of RAID 0 striping, the
+reliability of n-way RAID mirroring and faster recovery.
+
+Ceph provides three types of clients: Ceph Block Device, Ceph File System, and
+Ceph Object Storage. A Ceph Client converts its data from the representation 
+format it provides to its users (a block device image, RESTful objects, CephFS
+filesystem directories) into objects for storage in the Ceph Storage Cluster. 
+
+.. tip:: The objects Ceph stores in the Ceph Storage Cluster are not striped. 
+   Ceph Object Storage, Ceph Block Device, and the Ceph File System stripe their 
+   data over multiple Ceph Storage Cluster objects. Ceph Clients that write 
+   directly to the Ceph Storage Cluster via ``librados`` must perform the
+   striping (and parallel I/O) for themselves to obtain these benefits.
+
+The simplest Ceph striping format involves a stripe count of 1 object. Ceph
+Clients write stripe units to a Ceph Storage Cluster object until the object is
+at its maximum capacity, and then create another object for additional stripes
+of data. The simplest form of striping may be sufficient for small block device
+images, S3 or Swift objects and CephFS files. However, this simple form doesn't
+take maximum advantage of Ceph's ability to distribute data across placement
+groups, and consequently doesn't improve performance very much. The following
+diagram depicts the simplest form of striping:
+
+.. ditaa::
+
+                        +---------------+
+                        |  Client Data  |
+                        |     Format    |
+                        | cCCC          |
+                        +---------------+
+                                |
+                       +--------+-------+
+                       |                |
+                       v                v
+                 /-----------\    /-----------\
+                 | Begin cCCC|    | Begin cCCC|
+                 | Object  0 |    | Object  1 |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 1   |    |  unit 5   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 2   |    |  unit 6   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 3   |    |  unit 7   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 4   |    |  unit 8   |
+                 +-----------+    +-----------+
+                 | End cCCC  |    | End cCCC  |
+                 | Object 0  |    | Object 1  |
+                 \-----------/    \-----------/
+   
+
+If you anticipate large images sizes, large S3 or Swift objects (e.g., video),
+or large CephFS directories, you may see considerable read/write performance
+improvements by striping client data over multiple objects within an object set.
+Significant write performance occurs when the client writes the stripe units to
+their corresponding objects in parallel. Since objects get mapped to different
+placement groups and further mapped to different OSDs, each write occurs in
+parallel at the maximum write speed. A write to a single drive would be limited
+by the head movement (e.g. 6ms per seek) and bandwidth of that one device (e.g.
+100MB/s).  By spreading that write over multiple objects (which map to different
+placement groups and OSDs) Ceph can reduce the number of seeks per drive and
+combine the throughput of multiple drives to achieve much faster write (or read)
+speeds.
+
+.. note:: Striping is independent of object replicas. Since CRUSH
+   replicates objects across OSDs, stripes get replicated automatically.
+
+In the following diagram, client data gets striped across an object set
+(``object set 1`` in the following diagram) consisting of 4 objects, where the
+first stripe unit is ``stripe unit 0`` in ``object 0``, and the fourth stripe
+unit is ``stripe unit 3`` in ``object 3``. After writing the fourth stripe, the
+client determines if the object set is full. If the object set is not full, the
+client begins writing a stripe to the first object again (``object 0`` in the
+following diagram). If the object set is full, the client creates a new object
+set (``object set 2`` in the following diagram), and begins writing to the first
+stripe (``stripe unit 16``) in the first object in the new object set (``object
+4`` in the diagram below).
+
+.. ditaa::
+
+                          +---------------+
+                          |  Client Data  |
+                          |     Format    |
+                          | cCCC          |
+                          +---------------+
+                                  |
+       +-----------------+--------+--------+-----------------+
+       |                 |                 |                 |     +--\
+       v                 v                 v                 v        |
+ /-----------\     /-----------\     /-----------\     /-----------\  |   
+ | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
+ | Object 0  |     | Object  1 |     | Object  2 |     | Object  3 |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 0   |     |  unit 1   |     |  unit 2   |     |  unit 3   |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  unit 4   |     |  unit 5   |     |  unit 6   |     |  unit 7   |    | Object
+ +-----------+     +-----------+     +-----------+     +-----------+    +- Set 
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   1
+ |  unit 8   |     |  unit 9   |     |  unit 10  |     |  unit 11  |  +-/
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 12  |     |  unit 13  |     |  unit 14  |     |  unit 15  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
+ | Object 0  |     | Object 1  |     | Object 2  |     | Object 3  |  |  
+ \-----------/     \-----------/     \-----------/     \-----------/  |
+                                                                      |
+                                                                   +--/
+  
+                                                                   +--\
+                                                                      |
+ /-----------\     /-----------\     /-----------\     /-----------\  |   
+ | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
+ | Object  4 |     | Object  5 |     | Object  6 |     | Object  7 |  |  
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 16  |     |  unit 17  |     |  unit 18  |     |  unit 19  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  unit 20  |     |  unit 21  |     |  unit 22  |     |  unit 23  |    | Object
+ +-----------+     +-----------+     +-----------+     +-----------+    +- Set
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   2 
+ |  unit 24  |     |  unit 25  |     |  unit 26  |     |  unit 27  |  +-/
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 28  |     |  unit 29  |     |  unit 30  |     |  unit 31  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
+ | Object 4  |     | Object 5  |     | Object 6  |     | Object 7  |  |  
+ \-----------/     \-----------/     \-----------/     \-----------/  |
+                                                                      |
+                                                                   +--/
+
+Three important variables determine how Ceph stripes data: 
+
+- **Object Size:** Objects in the Ceph Storage Cluster have a maximum
+  configurable size (e.g., 2MB, 4MB, etc.). The object size should be large
+  enough to accommodate many stripe units, and should be a multiple of
+  the stripe unit.
+
+- **Stripe Width:** Stripes have a configurable unit size (e.g., 64kb).
+  The Ceph Client divides the data it will write to objects into equally 
+  sized stripe units, except for the last stripe unit. A stripe width, 
+  should be a fraction of the Object Size so that an object may contain 
+  many stripe units.
+
+- **Stripe Count:** The Ceph Client writes a sequence of stripe units
+  over a series of objects determined by the stripe count. The series 
+  of objects is called an object set. After the Ceph Client writes to 
+  the last object in the object set, it returns to the first object in
+  the object set.
+  
+.. important:: Test the performance of your striping configuration before
+   putting your cluster into production. You CANNOT change these striping
+   parameters after you stripe the data and write it to objects.
+
+Once the Ceph Client has striped data to stripe units and mapped the stripe
+units to objects, Ceph's CRUSH algorithm maps the objects to placement groups,
+and the placement groups to Ceph OSD Daemons before the objects are stored as 
+files on a storage drive.
+
+.. note:: Since a client writes to a single pool, all data striped into objects
+   get mapped to placement groups in the same pool. So they use the same CRUSH
+   map and the same access controls.
+
+
+.. index:: architecture; Ceph Clients
+
+Ceph Clients
+============
+
+Ceph Clients include a number of service interfaces. These include:
+
+- **Block Devices:** The :term:`Ceph Block Device` (a.k.a., RBD) service
+  provides resizable, thin-provisioned block devices that can be snapshotted
+  and cloned. Ceph stripes a block device across the cluster for high
+  performance. Ceph supports both kernel objects (KO) and a QEMU hypervisor
+  that uses ``librbd`` directly--avoiding the kernel object overhead for
+  virtualized systems.
+
+- **Object Storage:** The :term:`Ceph Object Storage` (a.k.a., RGW) service 
+  provides RESTful APIs with interfaces that are compatible with Amazon S3
+  and OpenStack Swift. 
+  
+- **Filesystem**: The :term:`Ceph File System` (CephFS) service provides 
+  a POSIX compliant filesystem usable with ``mount`` or as 
+  a filesystem in user space (FUSE).
+
+Ceph can run additional instances of OSDs, MDSs, and monitors for scalability
+and high availability. The following diagram depicts the high-level
+architecture. 
+
+.. ditaa::
+
+            +--------------+  +----------------+  +-------------+
+            | Block Device |  | Object Storage |  |   CephFS    |
+            +--------------+  +----------------+  +-------------+            
+
+            +--------------+  +----------------+  +-------------+
+            |    librbd    |  |     librgw     |  |  libcephfs  |
+            +--------------+  +----------------+  +-------------+
+
+            +---------------------------------------------------+
+            |      Ceph Storage Cluster Protocol (librados)     |
+            +---------------------------------------------------+
+
+            +---------------+ +---------------+ +---------------+
+            |      OSDs     | |      MDSs     | |    Monitors   |
+            +---------------+ +---------------+ +---------------+
+
+
+.. index:: architecture; Ceph Object Storage
+
+Ceph Object Storage
+-------------------
+
+The Ceph Object Storage daemon, ``radosgw``, is a FastCGI service that provides
+a RESTful_ HTTP API to store objects and metadata. It layers on top of the Ceph
+Storage Cluster with its own data formats, and maintains its own user database,
+authentication, and access control. The RADOS Gateway uses a unified namespace,
+which means you can use either the OpenStack Swift-compatible API or the Amazon
+S3-compatible API. For example, you can write data using the S3-compatible API
+with one application and then read data using the Swift-compatible API with
+another application.
+
+.. topic:: S3/Swift Objects and Store Cluster Objects Compared
+
+   Ceph's Object Storage uses the term *object* to describe the data it stores.
+   S3 and Swift objects are not the same as the objects that Ceph writes to the 
+   Ceph Storage Cluster. Ceph Object Storage objects are mapped to Ceph Storage
+   Cluster objects. The S3 and Swift objects do not necessarily 
+   correspond in a 1:1 manner with an object stored in the storage cluster. It 
+   is possible for an S3 or Swift object to map to multiple Ceph objects.
+
+See `Ceph Object Storage`_ for details.
+
+
+.. index:: Ceph Block Device; block device; RBD; Rados Block Device
+
+Ceph Block Device
+-----------------
+
+A Ceph Block Device stripes a block device image over multiple objects in the
+Ceph Storage Cluster, where each object gets mapped to a placement group and
+distributed, and the placement groups are spread across separate ``ceph-osd``
+daemons throughout the cluster.
+
+.. important:: Striping allows RBD block devices to perform better than a single 
+   server could!
+
+Thin-provisioned snapshottable Ceph Block Devices are an attractive option for
+virtualization and cloud computing. In virtual machine scenarios, people
+typically deploy a Ceph Block Device with the ``rbd`` network storage driver in
+QEMU/KVM, where the host machine uses ``librbd`` to provide a block device
+service to the guest. Many cloud computing stacks use ``libvirt`` to integrate
+with hypervisors. You can use thin-provisioned Ceph Block Devices with QEMU and
+``libvirt`` to support OpenStack and CloudStack among other solutions.
+
+While we do not provide ``librbd`` support with other hypervisors at this time,
+you may also use Ceph Block Device kernel objects to provide a block device to a
+client. Other virtualization technologies such as Xen can access the Ceph Block
+Device kernel object(s). This is done with the  command-line tool ``rbd``.
+
+
+.. index:: CephFS; Ceph File System; libcephfs; MDS; metadata server; ceph-mds
+
+.. _arch-cephfs:
+
+Ceph File System
+----------------
+
+The Ceph File System (CephFS) provides a POSIX-compliant filesystem as a
+service that is layered on top of the object-based Ceph Storage Cluster.
+CephFS files get mapped to objects that Ceph stores in the Ceph Storage
+Cluster. Ceph Clients mount a CephFS filesystem as a kernel object or as
+a Filesystem in User Space (FUSE).
+
+.. ditaa::
+
+            +-----------------------+  +------------------------+
+            | CephFS Kernel Object  |  |      CephFS FUSE       |
+            +-----------------------+  +------------------------+            
+
+            +---------------------------------------------------+
+            |            CephFS Library (libcephfs)             |
+            +---------------------------------------------------+
+
+            +---------------------------------------------------+
+            |      Ceph Storage Cluster Protocol (librados)     |
+            +---------------------------------------------------+
+
+            +---------------+ +---------------+ +---------------+
+            |      OSDs     | |      MDSs     | |    Monitors   |
+            +---------------+ +---------------+ +---------------+
+
+
+The Ceph File System service includes the Ceph Metadata Server (MDS) deployed
+with the Ceph Storage cluster. The purpose of the MDS is to store all the
+filesystem metadata (directories, file ownership, access modes, etc) in
+high-availability Ceph Metadata Servers where the metadata resides in memory.
+The reason for the MDS (a daemon called ``ceph-mds``) is that simple filesystem
+operations like listing a directory or changing a directory (``ls``, ``cd``)
+would tax the Ceph OSD Daemons unnecessarily. So separating the metadata from
+the data means that the Ceph File System can provide high performance services
+without taxing the Ceph Storage Cluster.
+
+CephFS separates the metadata from the data, storing the metadata in the MDS,
+and storing the file data in one or more objects in the Ceph Storage Cluster.
+The Ceph filesystem aims for POSIX compatibility. ``ceph-mds`` can run as a
+single process, or it can be distributed out to multiple physical machines,
+either for high availability or for scalability. 
+
+- **High Availability**: The extra ``ceph-mds`` instances can be `standby`, 
+  ready to take over the duties of any failed ``ceph-mds`` that was
+  `active`. This is easy because all the data, including the journal, is
+  stored on RADOS. The transition is triggered automatically by ``ceph-mon``.
+
+- **Scalability**: Multiple ``ceph-mds`` instances can be `active`, and they
+  will split the directory tree into subtrees (and shards of a single
+  busy directory), effectively balancing the load amongst all `active`
+  servers.
+
+Combinations of `standby` and `active` etc are possible, for example
+running 3 `active` ``ceph-mds`` instances for scaling, and one `standby`
+instance for high availability.
+
+
+
+.. _RADOS - A Scalable, Reliable Storage Service for Petabyte-scale Storage Clusters: https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
+.. _Monitor Config Reference: ../rados/configuration/mon-config-ref
+.. _Monitoring OSDs and PGs: ../rados/operations/monitoring-osd-pg
+.. _Heartbeats: ../rados/configuration/mon-osd-interaction
+.. _Monitoring OSDs: ../rados/operations/monitoring-osd-pg/#monitoring-osds
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
+.. _Data Scrubbing: ../rados/configuration/osd-config-ref#scrubbing
+.. _Report Peering Failure: ../rados/configuration/mon-osd-interaction#osds-report-peering-failure
+.. _Troubleshooting Peering Failure: ../rados/troubleshooting/troubleshooting-pg#placement-group-down-peering-failure
+.. _Ceph Authentication and Authorization: ../rados/operations/auth-intro/
+.. _Hardware Recommendations: ../start/hardware-recommendations
+.. _Network Config Reference: ../rados/configuration/network-config-ref
+.. _Data Scrubbing: ../rados/configuration/osd-config-ref#scrubbing
+.. _striping: https://en.wikipedia.org/wiki/Data_striping
+.. _RAID: https://en.wikipedia.org/wiki/RAID
+.. _RAID 0: https://en.wikipedia.org/wiki/RAID_0#RAID_0
+.. _Ceph Object Storage: ../radosgw/
+.. _RESTful: https://en.wikipedia.org/wiki/RESTful
+.. _Erasure Code Notes: https://github.com/ceph/ceph/blob/40059e12af88267d0da67d8fd8d9cd81244d8f93/doc/dev/osd_internals/erasure_coding/developer_notes.rst
+.. _Cache Tiering: ../rados/operations/cache-tiering
+.. _Set Pool Values: ../rados/operations/pools#set-pool-values
+.. _Kerberos: https://en.wikipedia.org/wiki/Kerberos_(protocol)
+.. _Cephx Config Guide: ../rados/configuration/auth-config-ref
+.. _User Management: ../rados/operations/user-management
diff --git a/doc/ceph-volume/drive-group.rst b/doc/ceph-volume/drive-group.rst
new file mode 100644
index 000000000..f9d1cf3c3
--- /dev/null
+++ b/doc/ceph-volume/drive-group.rst
@@ -0,0 +1,12 @@
+.. _ceph-volume-drive-group:
+
+``drive-group``
+===============
+The drive-group subcommand allows for passing :ref:`drivegroups` specifications
+straight to ceph-volume as json. ceph-volume will then attempt to deploy this
+drive groups via the batch subcommand.
+
+The specification can be passed via a file, string argument or on stdin.
+See the subcommand help for further details::
+
+    # ceph-volume drive-group --help
diff --git a/doc/ceph-volume/index.rst b/doc/ceph-volume/index.rst
new file mode 100644
index 000000000..9271bc2a0
--- /dev/null
+++ b/doc/ceph-volume/index.rst
@@ -0,0 +1,87 @@
+.. _ceph-volume:
+
+ceph-volume
+===========
+Deploy OSDs with different device technologies like lvm or physical disks using
+pluggable tools (:doc:`lvm/index` itself is treated like a plugin) and trying to
+follow a predictable, and robust way of preparing, activating, and starting OSDs.
+
+:ref:`Overview <ceph-volume-overview>` |
+:ref:`Plugin Guide <ceph-volume-plugins>` |
+
+
+**Command Line Subcommands**
+
+There is currently support for ``lvm``, and plain disks (with GPT partitions)
+that may have been deployed with ``ceph-disk``.
+
+``zfs`` support is available for running a FreeBSD cluster.
+
+* :ref:`ceph-volume-lvm`
+* :ref:`ceph-volume-simple`
+* :ref:`ceph-volume-zfs`
+
+**Node inventory**
+
+The :ref:`ceph-volume-inventory` subcommand provides information and metadata
+about a nodes physical disk inventory.
+
+
+Migrating
+---------
+Starting on Ceph version 13.0.0, ``ceph-disk`` is deprecated. Deprecation
+warnings will show up that will link to this page. It is strongly suggested
+that users start consuming ``ceph-volume``. There are two paths for migrating:
+
+#. Keep OSDs deployed with ``ceph-disk``: The :ref:`ceph-volume-simple` command
+   provides a way to take over the management while disabling ``ceph-disk``
+   triggers.
+#. Redeploy existing OSDs with ``ceph-volume``: This is covered in depth on
+   :ref:`rados-replacing-an-osd`
+
+For details on why ``ceph-disk`` was removed please see the :ref:`Why was
+ceph-disk replaced? <ceph-disk-replaced>` section.
+
+
+New deployments
+^^^^^^^^^^^^^^^
+For new deployments, :ref:`ceph-volume-lvm` is recommended, it can use any
+logical volume as input for data OSDs, or it can setup a minimal/naive logical
+volume from a device.
+
+Existing OSDs
+^^^^^^^^^^^^^
+If the cluster has OSDs that were provisioned with ``ceph-disk``, then
+``ceph-volume`` can take over the management of these with
+:ref:`ceph-volume-simple`. A scan is done on the data device or OSD directory,
+and ``ceph-disk`` is fully disabled. Encryption is fully supported.
+
+
+.. toctree::
+   :hidden:
+   :maxdepth: 3
+   :caption: Contents:
+
+   intro
+   systemd
+   inventory
+   drive-group
+   lvm/index
+   lvm/activate
+   lvm/batch
+   lvm/encryption
+   lvm/prepare
+   lvm/create
+   lvm/scan
+   lvm/systemd
+   lvm/list
+   lvm/zap
+   lvm/migrate
+   lvm/newdb
+   lvm/newwal
+   simple/index
+   simple/activate
+   simple/scan
+   simple/systemd
+   zfs/index
+   zfs/inventory
diff --git a/doc/ceph-volume/intro.rst b/doc/ceph-volume/intro.rst
new file mode 100644
index 000000000..c36f12a77
--- /dev/null
+++ b/doc/ceph-volume/intro.rst
@@ -0,0 +1,84 @@
+.. _ceph-volume-overview:
+
+Overview
+--------
+The ``ceph-volume`` tool aims to be a single purpose command line tool to deploy
+logical volumes as OSDs, trying to maintain a similar API to ``ceph-disk`` when
+preparing, activating, and creating OSDs.
+
+It deviates from ``ceph-disk`` by not interacting or relying on the udev rules
+that come installed for Ceph. These rules allow automatic detection of
+previously setup devices that are in turn fed into ``ceph-disk`` to activate
+them.
+
+.. _ceph-disk-replaced:
+
+Replacing ``ceph-disk``
+-----------------------
+The ``ceph-disk`` tool was created at a time when the project was required to
+support many different types of init systems (upstart, sysvinit, etc...) while
+being able to discover devices. This caused the tool to concentrate initially
+(and exclusively afterwards) on GPT partitions. Specifically on GPT GUIDs,
+which were used to label devices in a unique way to answer questions like:
+
+* is this device a Journal?
+* an encrypted data partition?
+* was the device left partially prepared?
+
+To solve these, it used ``UDEV`` rules to match the GUIDs, that would call
+``ceph-disk``, and end up in a back and forth between the ``ceph-disk`` systemd
+unit and the ``ceph-disk`` executable. The process was very unreliable and time
+consuming (a timeout of close to three hours **per OSD** had to be put in
+place), and would cause OSDs to not come up at all during the boot process of
+a node.
+
+It was hard to debug, or even replicate these problems given the asynchronous
+behavior of ``UDEV``.
+
+Since the world-view of ``ceph-disk`` had to be GPT partitions exclusively, it meant
+that it couldn't work with other technologies like LVM, or similar device
+mapper devices. It was ultimately decided to create something modular, starting
+with LVM support, and the ability to expand on other technologies as needed.
+
+
+GPT partitions are simple?
+--------------------------
+Although partitions in general are simple to reason about, ``ceph-disk``
+partitions were not simple by any means. It required a tremendous amount of
+special flags in order to get them to work correctly with the device discovery
+workflow. Here is an example call to create a data partition::
+
+    /sbin/sgdisk --largest-new=1 --change-name=1:ceph data --partition-guid=1:f0fc39fd-eeb2-49f1-b922-a11939cf8a0f --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be --mbrtogpt -- /dev/sdb
+
+Not only creating these was hard, but these partitions required devices to be
+exclusively owned by Ceph. For example, in some cases a special partition would
+be created when devices were encrypted, which would contain unencrypted keys.
+This was ``ceph-disk`` domain knowledge, which would not translate to a "GPT
+partitions are simple" understanding. Here is an example of that special
+partition being created::
+
+    /sbin/sgdisk --new=5:0:+10M --change-name=5:ceph lockbox --partition-guid=5:None --typecode=5:fb3aabf9-d25f-47cc-bf5e-721d181642be --mbrtogpt -- /dev/sdad
+
+
+Modularity
+----------
+``ceph-volume`` was designed to be a modular tool because we anticipate that
+there are going to be lots of ways that people provision the hardware devices
+that we need to consider. There are already two: legacy ceph-disk devices that
+are still in use and have GPT partitions (handled by :ref:`ceph-volume-simple`),
+and lvm. SPDK devices where we manage NVMe devices directly from userspace are
+on the immediate horizon, where LVM won't work there since the kernel isn't
+involved at all.
+
+``ceph-volume lvm``
+-------------------
+By making use of :term:`LVM tags`, the :ref:`ceph-volume-lvm` sub-command is
+able to store and later re-discover and query devices associated with OSDs so
+that they can later be activated.
+
+LVM performance penalty
+-----------------------
+In short: we haven't been able to notice any significant performance penalties
+associated with the change to LVM. By being able to work closely with LVM, the
+ability to work with other device mapper technologies was a given: there is no
+technical difficulty in working with anything that can sit below a Logical Volume.
diff --git a/doc/ceph-volume/inventory.rst b/doc/ceph-volume/inventory.rst
new file mode 100644
index 000000000..edb1fd205
--- /dev/null
+++ b/doc/ceph-volume/inventory.rst
@@ -0,0 +1,17 @@
+.. _ceph-volume-inventory:
+
+``inventory``
+=============
+The ``inventory`` subcommand queries a host's disc inventory and provides
+hardware information and metadata on every physical device.
+
+By default the command returns a short, human-readable report of all physical disks.
+
+For programmatic consumption of this report pass ``--format json`` to generate a
+JSON formatted report. This report includes extensive information on the
+physical drives such as disk metadata (like model and size), logical volumes
+and whether they are used by ceph, and if the disk is usable by ceph and
+reasons why not.
+
+A device path can be specified to report extensive information on a device in
+both plain and json format.
diff --git a/doc/ceph-volume/lvm/activate.rst b/doc/ceph-volume/lvm/activate.rst
new file mode 100644
index 000000000..fe34ecb71
--- /dev/null
+++ b/doc/ceph-volume/lvm/activate.rst
@@ -0,0 +1,112 @@
+.. _ceph-volume-lvm-activate:
+
+``activate``
+============
+
+After :ref:`ceph-volume-lvm-prepare` has completed its run, the volume can be
+activated. 
+
+Activating the volume involves enabling a ``systemd`` unit that persists the
+``OSD ID`` and its ``UUID`` (which is also called the ``fsid`` in the Ceph CLI
+tools). After this information has been persisted, the cluster can determine
+which OSD is enabled and must be mounted.
+
+.. note:: The execution of this call is fully idempotent. This means that the
+   call can be executed multiple times without changing the result of its first
+   successful execution.
+
+For information about OSDs deployed by cephadm, refer to
+:ref:`cephadm-osd-activate`.
+
+New OSDs
+--------
+To activate newly prepared OSDs both the :term:`OSD id` and :term:`OSD uuid`
+need to be supplied. For example::
+
+    ceph-volume lvm activate --bluestore 0 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8
+
+.. note:: The UUID is stored in the ``fsid`` file in the OSD path, which is
+          generated when :ref:`ceph-volume-lvm-prepare` is used.
+
+Activating all OSDs
+-------------------
+
+.. note:: For OSDs deployed by cephadm, please refer to :ref:`cephadm-osd-activate`
+          instead.
+
+It is possible to activate all existing OSDs at once by using the ``--all``
+flag. For example::
+
+    ceph-volume lvm activate --all
+
+This call will inspect all the OSDs created by ceph-volume that are inactive
+and will activate them one by one. If any of the OSDs are already running, it
+will report them in the command output and skip them, making it safe to rerun
+(idempotent).
+
+requiring uuids
+^^^^^^^^^^^^^^^
+The :term:`OSD uuid` is being required as an extra step to ensure that the
+right OSD is being activated. It is entirely possible that a previous OSD with
+the same id exists and would end up activating the incorrect one.
+
+
+dmcrypt
+^^^^^^^
+If the OSD was prepared with dmcrypt by ceph-volume, there is no need to
+specify ``--dmcrypt`` on the command line again (that flag is not available for
+the ``activate`` subcommand). An encrypted OSD will be automatically detected.
+
+
+Discovery
+---------
+With OSDs previously created by ``ceph-volume``, a *discovery* process is
+performed using :term:`LVM tags` to enable the systemd units.
+
+The systemd unit will capture the :term:`OSD id` and :term:`OSD uuid` and
+persist it. Internally, the activation will enable it like::
+
+    systemctl enable ceph-volume@lvm-$id-$uuid
+
+For example::
+
+    systemctl enable ceph-volume@lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Would start the discovery process for the OSD with an id of ``0`` and a UUID of
+``8715BEB4-15C5-49DE-BA6F-401086EC7B41``.
+
+.. note:: for more details on the systemd workflow see :ref:`ceph-volume-lvm-systemd`
+
+The systemd unit will look for the matching OSD device, and by looking at its
+:term:`LVM tags` will proceed to:
+
+#. Mount the device in the corresponding location (by convention this is
+``/var/lib/ceph/osd/<cluster name>-<osd id>/``)
+
+#. Ensure that all required devices are ready for that OSD.
+
+#. Start the ``ceph-osd@0`` systemd unit
+
+.. note:: The system infers the objectstore type by
+          inspecting the LVM tags applied to the OSD devices
+
+Existing OSDs
+-------------
+For existing OSDs that have been deployed with ``ceph-disk``, they need to be
+scanned and activated :ref:`using the simple sub-command <ceph-volume-simple>`.
+If a different tool was used then the only way to port them over to the new
+mechanism is to prepare them again (losing data). See
+:ref:`ceph-volume-lvm-existing-osds` for details on how to proceed.
+
+Summary
+-------
+To recap the ``activate`` process for :term:`bluestore`:
+
+#. Require both :term:`OSD id` and :term:`OSD uuid`
+#. Enable the system unit with matching id and uuid
+#. Create the ``tmpfs`` mount at the OSD directory in
+   ``/var/lib/ceph/osd/$cluster-$id/``
+#. Recreate all the files needed with ``ceph-bluestore-tool prime-osd-dir`` by
+   pointing it to the OSD ``block`` device.
+#. The systemd unit will ensure all devices are ready and linked
+#. The matching ``ceph-osd`` systemd unit will get started
diff --git a/doc/ceph-volume/lvm/batch.rst b/doc/ceph-volume/lvm/batch.rst
new file mode 100644
index 000000000..2114518bf
--- /dev/null
+++ b/doc/ceph-volume/lvm/batch.rst
@@ -0,0 +1,179 @@
+.. _ceph-volume-lvm-batch:
+
+``batch``
+===========
+The subcommand allows to create multiple OSDs at the same time given
+an input of devices. The ``batch`` subcommand is closely related to
+drive-groups. One individual drive group specification translates to a single
+``batch`` invocation.
+
+The subcommand is based to :ref:`ceph-volume-lvm-create`, and will use the very
+same code path. All ``batch`` does is to calculate the appropriate sizes of all
+volumes and skip over already created volumes.
+
+All the features that ``ceph-volume lvm create`` supports, like ``dmcrypt``,
+avoiding ``systemd`` units from starting, defining bluestore,
+is supported.
+
+
+.. _ceph-volume-lvm-batch_auto:
+
+Automatic sorting of disks
+--------------------------
+If ``batch`` receives only a single list of data devices and other options are
+passed , ``ceph-volume`` will auto-sort disks by its rotational
+property and use non-rotating disks for ``block.db`` or ``journal`` depending
+on the objectstore used. If all devices are to be used for standalone OSDs,
+no matter if rotating or solid state, pass ``--no-auto``.
+For example assuming :term:`bluestore` is used and ``--no-auto`` is not passed,
+the deprecated behavior would deploy the following, depending on the devices
+passed:
+
+#. Devices are all spinning HDDs: 1 OSD is created per device
+#. Devices are all SSDs: 2 OSDs are created per device
+#. Devices are a mix of HDDs and SSDs: data is placed on the spinning device,
+   the ``block.db`` is created on the SSD, as large as possible.
+
+.. note:: Although operations in ``ceph-volume lvm create`` allow usage of
+          ``block.wal`` it isn't supported with the ``auto`` behavior.
+
+This default auto-sorting behavior is now DEPRECATED and will be changed in future releases.
+Instead devices are not automatically sorted unless the ``--auto`` option is passed
+
+It is recommended to make use of the explicit device lists for ``block.db``,
+   ``block.wal`` and ``journal``.
+
+.. _ceph-volume-lvm-batch_bluestore:
+
+Reporting
+=========
+By default ``batch`` will print a report of the computed OSD layout and ask the
+user to confirm. This can be overridden by passing ``--yes``.
+
+If one wants to try out several invocations with being asked to deploy
+``--report`` can be passed. ``ceph-volume`` will exit after printing the report.
+
+Consider the following invocation::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+
+This will deploy three OSDs with external ``db`` and ``wal`` volumes on
+an NVME device.
+
+Pretty reporting
+----------------
+
+The ``pretty`` report format (the default) would
+look like this::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+    --> passed data devices: 3 physical, 0 LVM
+    --> relative data size: 1.0
+    --> passed block_db devices: 1 physical, 0 LVM
+
+    Total OSDs: 3
+
+      Type            Path                                                    LV Size         % of device
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdb                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdc                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdd                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+
+
+JSON reporting
+--------------
+
+Reporting can produce a structured output with ``--format json`` or
+``--format json-pretty``::
+
+    $ ceph-volume lvm batch --report --format json-pretty /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+    --> passed data devices: 3 physical, 0 LVM
+    --> relative data size: 1.0
+    --> passed block_db devices: 1 physical, 0 LVM
+    [
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdb",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        },
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdc",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        },
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdd",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        }
+    ]
+
+Sizing
+======
+When no sizing arguments are passed, `ceph-volume` will derive the sizing from
+the passed device lists (or the sorted lists when using the automatic sorting).
+`ceph-volume batch` will attempt to fully utilize a device's available capacity.
+Relying on automatic sizing is recommended.
+
+If one requires a different sizing policy for wal, db or journal devices,
+`ceph-volume` offers implicit and explicit sizing rules.
+
+Implicit sizing
+---------------
+Scenarios in which either devices are under-committed or not all data devices are
+currently ready for use (due to a broken disk for example), one can still rely
+on `ceph-volume` automatic sizing.
+Users can provide hints to `ceph-volume` as to how many data devices should have
+their external volumes on a set of fast devices. These options are:
+
+* ``--block-db-slots``
+* ``--block-wal-slots``
+* ``--journal-slots``
+
+For example, consider an OSD host that is supposed to contain 5 data devices and
+one device for wal/db volumes. However, one data device is currently broken and
+is being replaced. Instead of calculating the explicit sizes for the wal/db
+volume, one can simply call::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd /dev/sde --db-devices /dev/nvme0n1 --block-db-slots 5
+
+Explicit sizing
+---------------
+It is also possible to provide explicit sizes to `ceph-volume` via the arguments
+
+* ``--block-db-size``
+* ``--block-wal-size``
+* ``--journal-size``
+
+`ceph-volume` will try to satisfy the requested sizes given the passed disks. If
+this is not possible, no OSDs will be deployed.
+
+
+Idempotency and disk replacements
+=================================
+`ceph-volume lvm batch` intends to be idempotent, i.e. calling the same command
+repeatedly must result in the same outcome. For example calling::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+
+will result in three deployed OSDs (if all disks were available). Calling this
+command again, you will still end up with three OSDs and ceph-volume will exit
+with return code 0.
+
+Suppose /dev/sdc goes bad and needs to be replaced. After destroying the OSD and
+replacing the hardware, you can again call the same command and `ceph-volume`
+will detect that only two out of the three wanted OSDs are setup and re-create
+the missing OSD.
+
+This idempotency notion is tightly coupled to and extensively used by :ref:`drivegroups`.
diff --git a/doc/ceph-volume/lvm/create.rst b/doc/ceph-volume/lvm/create.rst
new file mode 100644
index 000000000..17fe9fa5a
--- /dev/null
+++ b/doc/ceph-volume/lvm/create.rst
@@ -0,0 +1,23 @@
+.. _ceph-volume-lvm-create:
+
+``create``
+===========
+This subcommand wraps the two-step process to provision a new osd (calling
+``prepare`` first and then ``activate``) into a single
+one. The reason to prefer ``prepare`` and then ``activate`` is to gradually
+introduce new OSDs into a cluster, and avoiding large amounts of data being
+rebalanced.
+
+The single-call process unifies exactly what :ref:`ceph-volume-lvm-prepare` and
+:ref:`ceph-volume-lvm-activate` do, with the convenience of doing it all at
+once.
+
+There is nothing different to the process except the OSD will become up and in
+immediately after completion.
+
+The backing objectstore can be specified with:
+
+* :ref:`--bluestore <ceph-volume-lvm-prepare_bluestore>`
+
+All command line flags and options are the same as ``ceph-volume lvm prepare``.
+Please refer to :ref:`ceph-volume-lvm-prepare` for details.
diff --git a/doc/ceph-volume/lvm/encryption.rst b/doc/ceph-volume/lvm/encryption.rst
new file mode 100644
index 000000000..4564a7ffe
--- /dev/null
+++ b/doc/ceph-volume/lvm/encryption.rst
@@ -0,0 +1,84 @@
+.. _ceph-volume-lvm-encryption:
+
+Encryption
+==========
+
+Logical volumes can be encrypted using ``dmcrypt`` by specifying the
+``--dmcrypt`` flag when creating OSDs. When using LVM, logical volumes can be
+encrypted in different ways. ``ceph-volume`` does not offer as many options as
+LVM does, but it encrypts logical volumes in a way that  is consistent and
+robust.
+
+In this case, ``ceph-volume lvm`` follows this constraint:
+
+* Non-LVM devices (such as partitions) are encrypted with the same OSD key.
+
+
+LUKS
+----
+There are currently two versions of LUKS, 1 and 2. Version 2 is a bit easier to
+implement but not widely available in all Linux distributions supported by
+Ceph. 
+
+.. note:: Version 1 of LUKS is referred to in this documentation as "LUKS".
+   Version 2 is of LUKS is referred to in this documentation as "LUKS2".
+
+
+LUKS on LVM
+-----------
+Encryption is done on top of existing logical volumes (this is not the same as
+encrypting the physical device). Any single logical volume can be encrypted,
+leaving other volumes unencrypted. This method also allows for flexible logical
+volume setups, since encryption will happen once the LV is created.
+
+
+Workflow
+--------
+When setting up the OSD, a secret key is created. That secret key is passed
+to the monitor in JSON format as ``stdin`` to prevent the key from being
+captured in the logs.
+
+The JSON payload looks something like this::
+
+        {
+            "cephx_secret": CEPHX_SECRET,
+            "dmcrypt_key": DMCRYPT_KEY,
+            "cephx_lockbox_secret": LOCKBOX_SECRET,
+        }
+
+The naming convention for the keys is **strict**, and they are named like that
+for the hardcoded (legacy) names used by ceph-disk.
+
+* ``cephx_secret`` : The cephx key used to authenticate
+* ``dmcrypt_key`` : The secret (or private) key to unlock encrypted devices
+* ``cephx_lockbox_secret`` : The authentication key used to retrieve the
+  ``dmcrypt_key``. It is named *lockbox* because ceph-disk used to have an
+  unencrypted partition named after it, which was used to store public keys and
+  other OSD metadata.
+
+The naming convention is strict because Monitors supported the naming
+convention of ceph-disk, which used these key names. In order to maintain 
+compatibility and prevent ceph-disk from breaking, ceph-volume uses the same
+naming convention *although it does not make sense for the new encryption
+workflow*.
+
+After the common steps of setting up the OSD during the "prepare stage" (
+with :term:`bluestore`), the logical volume is left ready
+to be activated, regardless of the state of the device (encrypted or
+decrypted).
+
+At the time of its activation, the logical volume is decrypted. The OSD starts
+after the process completes correctly.
+
+Summary of the encryption workflow for creating a new OSD
+----------------------------------------------------------
+
+#. OSD is created. Both lockbox and dmcrypt keys are created and sent to the
+   monitors in JSON format, indicating an encrypted OSD.
+
+#. All complementary devices (like journal, db, or wal) get created and
+   encrypted with the same OSD key. Key is stored in the LVM metadata of the
+   OSD.
+
+#. Activation continues by ensuring devices are mounted, retrieving the dmcrypt
+   secret key from the monitors, and decrypting before the OSD gets started.
diff --git a/doc/ceph-volume/lvm/index.rst b/doc/ceph-volume/lvm/index.rst
new file mode 100644
index 000000000..962e51a51
--- /dev/null
+++ b/doc/ceph-volume/lvm/index.rst
@@ -0,0 +1,34 @@
+.. _ceph-volume-lvm:
+
+``lvm``
+=======
+Implements the functionality needed to deploy OSDs from the ``lvm`` subcommand:
+``ceph-volume lvm``
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-lvm-prepare`
+
+* :ref:`ceph-volume-lvm-activate`
+
+* :ref:`ceph-volume-lvm-create`
+
+* :ref:`ceph-volume-lvm-list`
+
+* :ref:`ceph-volume-lvm-migrate`
+
+* :ref:`ceph-volume-lvm-newdb`
+
+* :ref:`ceph-volume-lvm-newwal`
+
+.. not yet implemented
+.. * :ref:`ceph-volume-lvm-scan`
+
+**Internal functionality**
+
+There are other aspects of the ``lvm`` subcommand that are internal and not
+exposed to the user, these sections explain how these pieces work together,
+clarifying the workflows of the tool.
+
+:ref:`Systemd Units <ceph-volume-lvm-systemd>` |
+:ref:`lvm <ceph-volume-lvm-api>`
diff --git a/doc/ceph-volume/lvm/list.rst b/doc/ceph-volume/lvm/list.rst
new file mode 100644
index 000000000..718154b10
--- /dev/null
+++ b/doc/ceph-volume/lvm/list.rst
@@ -0,0 +1,184 @@
+.. _ceph-volume-lvm-list:
+
+``list``
+========
+This subcommand will list any devices (logical and physical) that may be
+associated with a Ceph cluster, as long as they contain enough metadata to
+allow for that discovery.
+
+Output is grouped by the OSD ID associated with the devices, and unlike
+``ceph-disk`` it does not provide any information for devices that aren't
+associated with Ceph.
+
+Command line options:
+
+* ``--format`` Allows a ``json`` or ``pretty`` value. Defaults to ``pretty``
+  which will group the device information in a human-readable format.
+
+Full Reporting
+--------------
+When no positional arguments are used, a full reporting will be presented. This
+means that all devices and logical volumes found in the system will be
+displayed.
+
+Full ``pretty`` reporting for two OSDs, one with a lv as a journal, and another
+one with a physical device may look similar to::
+
+    # ceph-volume lvm list
+
+
+    ====== osd.1 =======
+
+      [journal]    /dev/journals/journal1
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      journal
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sda
+
+      [data]    /dev/test_group/data-lv2
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sdb
+
+    ====== osd.0 =======
+
+      [data]    /dev/test_group/data-lv1
+
+          journal uuid              cd72bd28-002a-48da-bdf6-d5b993e84f3f
+          osd id                    0
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  943949f0-ce37-47ca-a33c-3413d46ee9ec
+          data uuid                 TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00
+          journal device            /dev/sdd1
+          data device               /dev/test_group/data-lv1
+          devices                   /dev/sdc
+
+      [journal]    /dev/sdd1
+
+          PARTUUID                  cd72bd28-002a-48da-bdf6-d5b993e84f3f
+
+
+For logical volumes the ``devices`` key is populated with the physical devices
+associated with the logical volume. Since LVM allows multiple physical devices
+to be part of a logical volume, the value will be comma separated when using
+``pretty``, but an array when using ``json``.
+
+.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored
+          as a ``ceph.osd_id`` tag. For more information on lvm tag conventions
+          see :ref:`ceph-volume-lvm-tag-api`
+
+Single Reporting
+----------------
+Single reporting can consume both devices and logical volumes as input
+(positional parameters). For logical volumes, it is required to use the group
+name as well as the logical volume name.
+
+For example the ``data-lv2`` logical volume, in the ``test_group`` volume group
+can be listed in the following way::
+
+    # ceph-volume lvm list test_group/data-lv2
+
+
+    ====== osd.1 =======
+
+      [data]    /dev/test_group/data-lv2
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sdc
+
+
+.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored
+          as a ``ceph.osd_id`` tag. For more information on lvm tag conventions
+          see :ref:`ceph-volume-lvm-tag-api`
+
+
+For plain disks, the full path to the device is required. For example, for
+a device like ``/dev/sdd1`` it can look like::
+
+
+    # ceph-volume lvm list /dev/sdd1
+
+
+    ====== osd.0 =======
+
+      [journal]    /dev/sdd1
+
+          PARTUUID                  cd72bd28-002a-48da-bdf6-d5b993e84f3f
+
+
+
+``json`` output
+---------------
+All output using ``--format=json`` will show everything the system has stored
+as metadata for the devices, including tags.
+
+No changes for readability are done with ``json`` reporting, and all
+information is presented as-is. Full output as well as single devices can be
+listed.
+
+For brevity, this is how a single logical volume would look with ``json``
+output (note how tags aren't modified)::
+
+    # ceph-volume lvm list --format=json test_group/data-lv1
+    {
+        "0": [
+            {
+                "devices": ["/dev/sda"],
+                "lv_name": "data-lv1",
+                "lv_path": "/dev/test_group/data-lv1",
+                "lv_tags": "ceph.cluster_fsid=ce454d91-d748-4751-a318-ff7f7aa18ffd,ceph.data_device=/dev/test_group/data-lv1,ceph.data_uuid=TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00,ceph.journal_device=/dev/sdd1,ceph.journal_uuid=cd72bd28-002a-48da-bdf6-d5b993e84f3f,ceph.osd_fsid=943949f0-ce37-47ca-a33c-3413d46ee9ec,ceph.osd_id=0,ceph.type=data",
+                "lv_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00",
+                "name": "data-lv1",
+                "path": "/dev/test_group/data-lv1",
+                "tags": {
+                    "ceph.cluster_fsid": "ce454d91-d748-4751-a318-ff7f7aa18ffd",
+                    "ceph.data_device": "/dev/test_group/data-lv1",
+                    "ceph.data_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00",
+                    "ceph.journal_device": "/dev/sdd1",
+                    "ceph.journal_uuid": "cd72bd28-002a-48da-bdf6-d5b993e84f3f",
+                    "ceph.osd_fsid": "943949f0-ce37-47ca-a33c-3413d46ee9ec",
+                    "ceph.osd_id": "0",
+                    "ceph.type": "data"
+                },
+                "type": "data",
+                "vg_name": "test_group"
+            }
+        ]
+    }
+
+
+Synchronized information
+------------------------
+Before any listing type, the lvm API is queried to ensure that physical devices
+that may be in use haven't changed naming. It is possible that non-persistent
+devices like ``/dev/sda1`` could change to ``/dev/sdb1``.
+
+The detection is possible because the ``PARTUUID`` is stored as part of the
+metadata in the logical volume for the data lv. Even in the case of a journal
+that is a physical device, this information is still stored on the data logical
+volume associated with it.
+
+If the name is no longer the same (as reported by ``blkid`` when using the
+``PARTUUID``), the tag will get updated and the report will use the newly
+refreshed information.
diff --git a/doc/ceph-volume/lvm/migrate.rst b/doc/ceph-volume/lvm/migrate.rst
new file mode 100644
index 000000000..983d2e797
--- /dev/null
+++ b/doc/ceph-volume/lvm/migrate.rst
@@ -0,0 +1,47 @@
+.. _ceph-volume-lvm-migrate:
+
+``migrate``
+===========
+
+Moves BlueFS data from source volume(s) to the target one, source volumes
+(except the main, i.e. data or block one) are removed on success.
+
+LVM volumes are permitted for Target only, both already attached or new one.
+
+In the latter case it is attached to the OSD replacing one of the source
+devices.
+
+Following replacement rules apply (in the order of precedence, stop
+on the first match):
+
+    - if source list has DB volume - target device replaces it.
+    - if source list has WAL volume - target device replaces it.
+    - if source list has slow volume only - operation is not permitted,
+      requires explicit allocation via new-db/new-wal command.
+
+Moves BlueFS data from main device to LV already attached as DB::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data --target vgname/db
+
+Moves BlueFS data from shared main device to LV which will be attached as a
+new DB::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data --target vgname/new_db
+
+Moves BlueFS data from DB device to new LV, DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from db --target vgname/new_db
+
+Moves BlueFS data from main and DB devices to new LV, DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data db --target vgname/new_db
+
+Moves BlueFS data from main, DB and WAL devices to new LV, WAL is  removed and
+DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data db wal --target vgname/new_db
+
+Moves BlueFS data from main, DB and WAL devices to main device, WAL and DB are
+removed::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from db wal --target vgname/data
diff --git a/doc/ceph-volume/lvm/newdb.rst b/doc/ceph-volume/lvm/newdb.rst
new file mode 100644
index 000000000..dcc87fc8a
--- /dev/null
+++ b/doc/ceph-volume/lvm/newdb.rst
@@ -0,0 +1,11 @@
+.. _ceph-volume-lvm-newdb:
+
+``new-db``
+===========
+
+Attaches the given logical volume to OSD as a DB.
+Logical volume name format is vg/lv. Fails if OSD has already got attached DB.
+
+Attach vgname/lvname as a DB volume to OSD 1::
+
+    ceph-volume lvm new-db --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_db
diff --git a/doc/ceph-volume/lvm/newwal.rst b/doc/ceph-volume/lvm/newwal.rst
new file mode 100644
index 000000000..05f87fff6
--- /dev/null
+++ b/doc/ceph-volume/lvm/newwal.rst
@@ -0,0 +1,11 @@
+.. _ceph-volume-lvm-newwal:
+
+``new-wal``
+===========
+
+Attaches the given logical volume to the given OSD as a WAL volume.
+Logical volume format is vg/lv. Fails if OSD has already got attached DB.
+
+Attach vgname/lvname as a WAL volume to OSD 1::
+
+    ceph-volume lvm new-wal --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_wal
diff --git a/doc/ceph-volume/lvm/prepare.rst b/doc/ceph-volume/lvm/prepare.rst
new file mode 100644
index 000000000..2faf12a4e
--- /dev/null
+++ b/doc/ceph-volume/lvm/prepare.rst
@@ -0,0 +1,332 @@
+.. _ceph-volume-lvm-prepare:
+
+``prepare``
+===========
+Before you run ``ceph-volume lvm prepare``, we recommend that you provision a
+logical volume. Then you can run ``prepare`` on that logical volume. 
+
+``prepare`` adds metadata to logical volumes but does not alter them in any
+other way. 
+
+.. note:: This is part of a two-step process to deploy an OSD. If you prefer 
+   to deploy an OSD by using only one command, see :ref:`ceph-volume-lvm-create`.
+
+``prepare`` uses :term:`LVM tags` to assign several pieces of metadata to a
+logical volume. Volumes tagged in this way are easier to identify and easier to
+use with Ceph. :term:`LVM tags` identify logical volumes by the role that they
+play in the Ceph cluster (for example: BlueStore data or BlueStore WAL+DB).
+
+:term:`BlueStore<bluestore>` is the default backend. Ceph permits changing
+the backend, which can be done by using the following flags and arguments:
+
+* :ref:`--bluestore <ceph-volume-lvm-prepare_bluestore>`
+
+.. _ceph-volume-lvm-prepare_bluestore:
+
+``bluestore``
+-------------
+:term:`Bluestore<bluestore>` is the default backend for new OSDs.  Bluestore
+supports the following configurations:
+
+* a block device, a block.wal device, and a block.db device
+* a block device and a block.wal device
+* a block device and a block.db device
+* a single block device
+
+The ``bluestore`` subcommand accepts physical block devices, partitions on physical
+block devices, or logical volumes as arguments for the various device
+parameters. If a physical block device is provided, a logical volume will be
+created. If the provided volume group's name begins with `ceph`, it will be
+created if it does not yet exist and it will be clobbered and reused if it
+already exists. This allows for a simpler approach to using LVM but at the
+cost of flexibility: no option or configuration can be used to change how the
+logical volume is created.
+
+The ``block`` is specified with the ``--data`` flag, and in its simplest use
+case it looks like:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --data vg/lv
+
+A raw device can be specified in the same way:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --data /path/to/device
+
+For enabling :ref:`encryption <ceph-volume-lvm-encryption>`, the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --dmcrypt --data vg/lv
+
+If a ``block.db`` device or a ``block.wal`` device is needed, it can be
+specified with ``--block.db`` or ``--block.wal``. These can be physical
+devices, partitions, or logical volumes. ``block.db`` and ``block.wal`` are
+optional for bluestore.
+
+For both ``block.db`` and ``block.wal``, partitions can be used as-is, and 
+therefore are not made into logical volumes.
+
+While creating the OSD directory, the process uses a ``tmpfs`` mount to hold
+the files needed for the OSD. These files are created by ``ceph-osd --mkfs``
+and are ephemeral.
+
+A symlink is created for the ``block`` device, and is optional for ``block.db``
+and ``block.wal``. For a cluster with a default name and an OSD ID of 0, the
+directory looks like this::
+
+    # ls -l /var/lib/ceph/osd/ceph-0
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block -> /dev/ceph-be2b6fbd-bcf2-4c51-b35d-a35a162a02f0/osd-block-25cf0a05-2bc6-44ef-9137-79d65bd7ad62
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.db -> /dev/sda1
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.wal -> /dev/ceph/osd-wal-0
+    -rw-------. 1 ceph ceph 37 Oct 20 13:05 ceph_fsid
+    -rw-------. 1 ceph ceph 37 Oct 20 13:05 fsid
+    -rw-------. 1 ceph ceph 55 Oct 20 13:05 keyring
+    -rw-------. 1 ceph ceph  6 Oct 20 13:05 ready
+    -rw-------. 1 ceph ceph 10 Oct 20 13:05 type
+    -rw-------. 1 ceph ceph  2 Oct 20 13:05 whoami
+
+In the above case, a device was used for ``block``, so ``ceph-volume`` created
+a volume group and a logical volume using the following conventions:
+
+* volume group name: ``ceph-{cluster fsid}`` (or if the volume group already
+  exists: ``ceph-{random uuid}``)
+
+* logical volume name: ``osd-block-{osd_fsid}``
+
+
+.. _ceph-volume-lvm-prepare_filestore:
+
+``filestore``
+-------------
+.. warning:: Filestore has been deprecated in the Reef release and is no longer supported.
+
+``Filestore<filestore>`` is the OSD backend that prepares logical volumes for a
+`filestore`-backed object-store OSD.
+
+
+``Filestore<filestore>`` uses a logical volume to store OSD data and it uses
+physical devices, partitions, or logical volumes to store the journal.  If a
+physical device is used to create a filestore backend, a logical volume will be
+created on that physical device. If the provided volume group's name begins
+with `ceph`, it will be created if it does not yet exist and it will be
+clobbered and reused if it already exists. No special preparation is needed for
+these volumes, but be sure to meet the minimum size requirements for OSD data and
+for the journal.
+
+Use the following command to create a basic filestore OSD:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data <data block device>
+
+Use this command to deploy filestore with an external journal:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data <data block device> --journal <journal block device>
+
+Use this command to enable :ref:`encryption <ceph-volume-lvm-encryption>`, and note that the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --dmcrypt --data <data block device> --journal <journal block device>
+
+The data block device and the journal can each take one of three forms: 
+
+* a physical block device
+* a partition on a physical block device
+* a logical volume
+
+If you use a logical volume to deploy filestore, the value that you pass in the
+command *must* be of the format ``volume_group/logical_volume_name``. Since logical
+volume names are not enforced for uniqueness, using this format is an important 
+safeguard against accidentally choosing the wrong volume (and clobbering its data).
+
+If you use a partition to deploy filestore, the partition *must* contain a
+``PARTUUID`` that can be discovered by ``blkid``. This ensures that the
+partition can be identified correctly regardless of the device's name (or path).
+
+For example, to use a logical volume for OSD data and a partition
+(``/dev/sdc1``) for the journal, run a command of this form:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data volume_group/logical_volume_name --journal /dev/sdc1
+
+Or, to use a bare device for data and a logical volume for the journal:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data /dev/sdc --journal volume_group/journal_lv
+
+A generated UUID is used when asking the cluster for a new OSD. These two
+pieces of information (the OSD ID and the OSD UUID) are necessary for
+identifying a given OSD and will later be used throughout the
+:ref:`activation<ceph-volume-lvm-activate>` process.
+
+The OSD data directory is created using the following convention::
+
+    /var/lib/ceph/osd/<cluster name>-<osd id>
+
+To link the journal volume to the mounted data volume, use this command:
+
+.. prompt:: bash #
+
+   ln -s /path/to/journal /var/lib/ceph/osd/<cluster_name>-<osd-id>/journal
+
+To fetch the monmap by using the bootstrap key from the OSD, use this command:
+
+.. prompt:: bash #
+
+   /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring
+   /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap
+
+To populate the OSD directory (which has already been mounted), use this ``ceph-osd`` command:  
+.. prompt:: bash #
+
+   ceph-osd --cluster ceph --mkfs --mkkey -i <osd id> \ --monmap
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap --osd-data \
+   /var/lib/ceph/osd/<cluster name>-<osd id> --osd-journal
+   /var/lib/ceph/osd/<cluster name>-<osd id>/journal \ --osd-uuid <osd uuid>
+   --keyring /var/lib/ceph/osd/<cluster name>-<osd id>/keyring \ --setuser ceph
+   --setgroup ceph
+
+All of the information from the previous steps is used in the above command.      
+
+
+
+.. _ceph-volume-lvm-partitions:
+
+Partitioning
+------------
+``ceph-volume lvm`` does not currently create partitions from a whole device.
+If using device partitions the only requirement is that they contain the
+``PARTUUID`` and that it is discoverable by ``blkid``. Both ``fdisk`` and
+``parted`` will create that automatically for a new partition.
+
+For example, using a new, unformatted drive (``/dev/sdd`` in this case) we can
+use ``parted`` to create a new partition. First we list the device
+information::
+
+    $ parted --script /dev/sdd print
+    Model: VBOX HARDDISK (scsi)
+    Disk /dev/sdd: 11.5GB
+    Sector size (logical/physical): 512B/512B
+    Disk Flags:
+
+This device is not even labeled yet, so we can use ``parted`` to create
+a ``gpt`` label before we create a partition, and verify again with ``parted
+print``::
+
+    $ parted --script /dev/sdd mklabel gpt
+    $ parted --script /dev/sdd print
+    Model: VBOX HARDDISK (scsi)
+    Disk /dev/sdd: 11.5GB
+    Sector size (logical/physical): 512B/512B
+    Partition Table: gpt
+    Disk Flags:
+
+Now lets create a single partition, and verify later if ``blkid`` can find
+a ``PARTUUID`` that is needed by ``ceph-volume``::
+
+    $ parted --script /dev/sdd mkpart primary 1 100%
+    $ blkid /dev/sdd1
+    /dev/sdd1: PARTLABEL="primary" PARTUUID="16399d72-1e1f-467d-96ee-6fe371a7d0d4"
+
+
+.. _ceph-volume-lvm-existing-osds:
+
+Existing OSDs
+-------------
+For existing clusters that want to use this new system and have OSDs that are
+already running there are a few things to take into account:
+
+.. warning:: this process will forcefully format the data device, destroying
+             existing data, if any.
+
+* OSD paths should follow this convention::
+
+     /var/lib/ceph/osd/<cluster name>-<osd id>
+
+* Preferably, no other mechanisms to mount the volume should exist, and should
+  be removed (like fstab mount points)
+
+The one time process for an existing OSD, with an ID of 0 and using
+a ``"ceph"`` cluster name would look like (the following command will **destroy
+any data** in the OSD)::
+
+    ceph-volume lvm prepare --filestore --osd-id 0 --osd-fsid E3D291C1-E7BF-4984-9794-B60D9FA139CB
+
+The command line tool will not contact the monitor to generate an OSD ID and
+will format the LVM device in addition to storing the metadata on it so that it
+can be started later (for detailed metadata description see
+:ref:`ceph-volume-lvm-tags`).
+
+
+Crush device class
+------------------
+
+To set the crush device class for the OSD, use the ``--crush-device-class`` flag. 
+
+    ceph-volume lvm prepare --bluestore --data vg/lv --crush-device-class foo
+
+
+.. _ceph-volume-lvm-multipath:
+
+``multipath`` support
+---------------------
+``multipath`` devices are supported if ``lvm`` is configured properly.
+
+**Leave it to LVM**
+
+Most Linux distributions should ship their LVM2 package with
+``multipath_component_detection = 1`` in the default configuration. With this
+setting ``LVM`` ignores any device that is a multipath component and
+``ceph-volume`` will accordingly not touch these devices.
+
+**Using filters**
+
+Should this setting be unavailable, a correct ``filter`` expression must be
+provided in ``lvm.conf``. ``ceph-volume`` must not be able to use both the
+multipath device and its multipath components.
+
+Storing metadata
+----------------
+The following tags will get applied as part of the preparation process
+regardless of the type of volume (journal or data) or OSD objectstore:
+
+* ``cluster_fsid``
+* ``encrypted``
+* ``osd_fsid``
+* ``osd_id``
+* ``crush_device_class``
+
+For :term:`bluestore` these tags will be added:
+
+* ``block_device``
+* ``block_uuid``
+* ``db_device``
+* ``db_uuid``
+* ``wal_device``
+* ``wal_uuid``
+
+.. note:: For the complete lvm tag conventions see :ref:`ceph-volume-lvm-tag-api`
+
+
+Summary
+-------
+To recap the ``prepare`` process for :term:`bluestore`:
+
+#. Accepts raw physical devices, partitions on physical devices or logical volumes as arguments.
+#. Creates logical volumes on any raw physical devices.
+#. Generate a UUID for the OSD
+#. Ask the monitor get an OSD ID reusing the generated UUID
+#. OSD data directory is created on a tmpfs mount.
+#. ``block``, ``block.wal``, and ``block.db`` are symlinked if defined.
+#. monmap is fetched for activation
+#. Data directory is populated by ``ceph-osd``
+#. Logical Volumes are assigned all the Ceph metadata using lvm tags
diff --git a/doc/ceph-volume/lvm/scan.rst b/doc/ceph-volume/lvm/scan.rst
new file mode 100644
index 000000000..aa9990f71
--- /dev/null
+++ b/doc/ceph-volume/lvm/scan.rst
@@ -0,0 +1,9 @@
+scan
+====
+This sub-command will allow to discover Ceph volumes previously setup by the
+tool by looking into the system's logical volumes and their tags.
+
+As part of the :ref:`ceph-volume-lvm-prepare` process, the logical volumes are assigned
+a few tags with important pieces of information.
+
+.. note:: This sub-command is not yet implemented
diff --git a/doc/ceph-volume/lvm/systemd.rst b/doc/ceph-volume/lvm/systemd.rst
new file mode 100644
index 000000000..30260de7e
--- /dev/null
+++ b/doc/ceph-volume/lvm/systemd.rst
@@ -0,0 +1,28 @@
+.. _ceph-volume-lvm-systemd:
+
+systemd
+=======
+Upon startup, it will identify the logical volume using :term:`LVM tags`,
+finding a matching ID and later ensuring it is the right one with
+the :term:`OSD uuid`.
+
+After identifying the correct volume it will then proceed to mount it by using
+the OSD destination conventions, that is::
+
+    /var/lib/ceph/osd/<cluster name>-<osd id>
+
+For our example OSD with an id of ``0``, that means the identified device will
+be mounted at::
+
+
+    /var/lib/ceph/osd/ceph-0
+
+
+Once that process is complete, a call will be made to start the OSD::
+
+    systemctl start ceph-osd@0
+
+The systemd portion of this process is handled by the ``ceph-volume lvm
+trigger`` sub-command, which is only in charge of parsing metadata coming from
+systemd and startup, and then dispatching to ``ceph-volume lvm activate`` which
+would proceed with activation.
diff --git a/doc/ceph-volume/lvm/zap.rst b/doc/ceph-volume/lvm/zap.rst
new file mode 100644
index 000000000..e737fc386
--- /dev/null
+++ b/doc/ceph-volume/lvm/zap.rst
@@ -0,0 +1,65 @@
+.. _ceph-volume-lvm-zap:
+
+``zap``
+=======
+
+This subcommand is used to zap lvs, partitions or raw devices that have been used
+by ceph OSDs so that they may be reused. If given a path to a logical
+volume it must be in the format of vg/lv. Any file systems present
+on the given lv or partition will be removed and all data will be purged.
+
+.. note:: The lv or partition will be kept intact.
+
+.. note:: If the logical volume, raw device or partition is being used for any ceph related
+          mount points they will be unmounted.
+
+Zapping a logical volume::
+
+    ceph-volume lvm zap {vg name/lv name}
+
+Zapping a partition::
+
+    ceph-volume lvm zap /dev/sdc1
+
+Removing Devices
+----------------
+When zapping, and looking for full removal of the device (lv, vg, or partition)
+use the ``--destroy`` flag. A common use case is to simply deploy OSDs using
+a whole raw device. If you do so and then wish to reuse that device for another
+OSD you must use the ``--destroy`` flag when zapping so that the vgs and lvs
+that ceph-volume created on the raw device will be removed.
+
+.. note:: Multiple devices can be accepted at once, to zap them all
+
+Zapping a raw device and destroying any vgs or lvs present::
+
+    ceph-volume lvm zap /dev/sdc --destroy
+
+
+This action can be performed on partitions, and logical volumes as well::
+
+    ceph-volume lvm zap /dev/sdc1 --destroy
+    ceph-volume lvm zap osd-vg/data-lv --destroy
+
+
+Finally, multiple devices can be detected if filtering by OSD ID and/or OSD
+FSID. Either identifier can be used or both can be used at the same time. This
+is useful in situations where multiple devices associated with a specific ID
+need to be purged. When using the FSID, the filtering is stricter, and might
+not match other (possibly invalid) devices associated to an ID.
+
+By ID only::
+
+    ceph-volume lvm zap --destroy --osd-id 1
+
+By FSID::
+
+    ceph-volume lvm zap --destroy --osd-fsid 2E8FBE58-0328-4E3B-BFB7-3CACE4E9A6CE
+
+By both::
+
+    ceph-volume lvm zap --destroy --osd-fsid 2E8FBE58-0328-4E3B-BFB7-3CACE4E9A6CE --osd-id 1
+
+
+.. warning:: If the systemd unit associated with the OSD ID to be zapped is
+             detected as running, the tool will refuse to zap until the daemon is stopped.
diff --git a/doc/ceph-volume/simple/activate.rst b/doc/ceph-volume/simple/activate.rst
new file mode 100644
index 000000000..8c7737162
--- /dev/null
+++ b/doc/ceph-volume/simple/activate.rst
@@ -0,0 +1,79 @@
+.. _ceph-volume-simple-activate:
+
+``activate``
+============
+Once :ref:`ceph-volume-simple-scan` has been completed, and all the metadata
+captured for an OSD has been persisted to ``/etc/ceph/osd/{id}-{uuid}.json``
+the OSD is now ready to get "activated".
+
+This activation process **disables** all ``ceph-disk`` systemd units by masking
+them, to prevent the UDEV/ceph-disk interaction that will attempt to start them
+up at boot time.
+
+The disabling of ``ceph-disk`` units is done only when calling ``ceph-volume
+simple activate`` directly, but is avoided when being called by systemd when
+the system is booting up.
+
+The activation process requires using both the :term:`OSD id` and :term:`OSD uuid`
+To activate parsed OSDs::
+
+    ceph-volume simple activate 0 6cc43680-4f6e-4feb-92ff-9c7ba204120e
+
+The above command will assume that a JSON configuration will be found in::
+
+    /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json
+
+Alternatively, using a path to a JSON file directly is also possible::
+
+    ceph-volume simple activate --file /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json
+
+requiring uuids
+^^^^^^^^^^^^^^^
+The :term:`OSD uuid` is being required as an extra step to ensure that the
+right OSD is being activated. It is entirely possible that a previous OSD with
+the same id exists and would end up activating the incorrect one.
+
+
+Discovery
+---------
+With OSDs previously scanned by ``ceph-volume``, a *discovery* process is
+performed using ``blkid`` and ``lvm``. There is currently support only for
+devices with GPT partitions and LVM logical volumes.
+
+The GPT partitions will have a ``PARTUUID`` that can be queried by calling out
+to ``blkid``, and the logical volumes will have a ``lv_uuid`` that can be
+queried against ``lvs`` (the LVM tool to list logical volumes).
+
+This discovery process ensures that devices can be correctly detected even if
+they are repurposed into another system or if their name changes (as in the
+case of non-persisting names like ``/dev/sda1``)
+
+The JSON configuration file used to map what devices go to what OSD will then
+coordinate the mounting and symlinking as part of activation.
+
+To ensure that the symlinks are always correct, if they exist in the OSD
+directory, the symlinks will be re-done.
+
+A systemd unit will capture the :term:`OSD id` and :term:`OSD uuid` and
+persist it. Internally, the activation will enable it like::
+
+    systemctl enable ceph-volume@simple-$id-$uuid
+
+For example::
+
+    systemctl enable ceph-volume@simple-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Would start the discovery process for the OSD with an id of ``0`` and a UUID of
+``8715BEB4-15C5-49DE-BA6F-401086EC7B41``.
+
+
+The systemd process will call out to activate passing the information needed to
+identify the OSD and its devices, and it will proceed to:
+
+# mount the device in the corresponding location (by convention this is
+  ``/var/lib/ceph/osd/<cluster name>-<osd id>/``)
+
+# ensure that all required devices are ready for that OSD and properly linked. 
+The symbolic link will **always** be re-done to ensure that the correct device is linked.
+
+# start the ``ceph-osd@0`` systemd unit
diff --git a/doc/ceph-volume/simple/index.rst b/doc/ceph-volume/simple/index.rst
new file mode 100644
index 000000000..315dea99a
--- /dev/null
+++ b/doc/ceph-volume/simple/index.rst
@@ -0,0 +1,32 @@
+.. _ceph-volume-simple:
+
+``simple``
+==========
+Implements the functionality needed to manage OSDs from the ``simple`` subcommand:
+``ceph-volume simple``
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-simple-scan`
+
+* :ref:`ceph-volume-simple-activate`
+
+* :ref:`ceph-volume-simple-systemd`
+
+
+By *taking over* management, it disables all ``ceph-disk`` systemd units used
+to trigger devices at startup, relying on basic (customizable) JSON
+configuration and systemd for starting up OSDs.
+
+This process involves two steps:
+
+#. :ref:`Scan <ceph-volume-simple-scan>` the running OSD or the data device
+#. :ref:`Activate <ceph-volume-simple-activate>` the scanned OSD
+
+The scanning will infer everything that ``ceph-volume`` needs to start the OSD,
+so that when activation is needed, the OSD can start normally without getting
+interference from ``ceph-disk``.
+
+As part of the activation process the systemd units for ``ceph-disk`` in charge
+of reacting to ``udev`` events, are linked to ``/dev/null`` so that they are
+fully inactive.
diff --git a/doc/ceph-volume/simple/scan.rst b/doc/ceph-volume/simple/scan.rst
new file mode 100644
index 000000000..2749b14b6
--- /dev/null
+++ b/doc/ceph-volume/simple/scan.rst
@@ -0,0 +1,176 @@
+.. _ceph-volume-simple-scan:
+
+``scan``
+========
+Scanning allows to capture any important details from an already-deployed OSD
+so that ``ceph-volume`` can manage it without the need of any other startup
+workflows or tools (like ``udev`` or ``ceph-disk``). Encryption with LUKS or
+PLAIN formats is fully supported.
+
+The command has the ability to inspect a running OSD, by inspecting the
+directory where the OSD data is stored, or by consuming the data partition.
+The command can also scan all running OSDs if no path or device is provided.
+
+Once scanned, information will (by default) persist the metadata as JSON in
+a file in ``/etc/ceph/osd``. This ``JSON`` file will use the naming convention
+of: ``{OSD ID}-{OSD FSID}.json``. An OSD with an id of 1, and an FSID like
+``86ebd829-1405-43d3-8fd6-4cbc9b6ecf96`` the absolute path of the file would
+be::
+
+    /etc/ceph/osd/1-86ebd829-1405-43d3-8fd6-4cbc9b6ecf96.json
+
+The ``scan`` subcommand will refuse to write to this file if it already exists.
+If overwriting the contents is needed, the ``--force`` flag must be used::
+
+    ceph-volume simple scan --force {path}
+
+If there is no need to persist the ``JSON`` metadata, there is support to send
+the contents to ``stdout`` (no file will be written)::
+
+    ceph-volume simple scan --stdout {path}
+
+
+.. _ceph-volume-simple-scan-directory:
+
+Running OSDs scan
+-----------------
+Using this command without providing an OSD directory or device will scan the
+directories of any currently running OSDs. If a running OSD was not created
+by ceph-disk it will be ignored and not scanned.
+
+To scan all running ceph-disk OSDs, the command would look like::
+
+    ceph-volume simple scan
+
+Directory scan
+--------------
+The directory scan will capture OSD file contents from interesting files. There
+are a few files that must exist in order to have a successful scan:
+
+* ``ceph_fsid``
+* ``fsid``
+* ``keyring``
+* ``ready``
+* ``type``
+* ``whoami``
+
+If the OSD is encrypted, it will additionally add the following keys:
+
+* ``encrypted``
+* ``encryption_type``
+* ``lockbox_keyring``
+
+In the case of any other file, as long as it is not a binary or a directory, it
+will also get captured and persisted as part of the JSON object.
+
+The convention for the keys in the JSON object is that any file name will be
+a key, and its contents will be its value. If the contents are a single line
+(like in the case of the ``whoami``) the contents are trimmed, and the newline
+is dropped. For example with an OSD with an id of 1, this is how the JSON entry
+would look like::
+
+    "whoami": "1",
+
+For files that may have more than one line, the contents are left as-is, except
+for keyrings which are treated specially and parsed to extract the keyring. For
+example, a ``keyring`` that gets read as::
+
+    [osd.1]\n\tkey = AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==\n
+
+Would get stored as::
+
+    "keyring": "AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==",
+
+
+For a directory like ``/var/lib/ceph/osd/ceph-1``, the command could look
+like::
+
+    ceph-volume simple scan /var/lib/ceph/osd/ceph1
+
+
+.. _ceph-volume-simple-scan-device:
+
+Device scan
+-----------
+When an OSD directory is not available (OSD is not running, or device is not
+mounted) the ``scan`` command is able to introspect the device to capture
+required data. Just like :ref:`ceph-volume-simple-scan-directory`, it would
+still require a few files present. This means that the device to be scanned
+**must be** the data partition of the OSD.
+
+As long as the data partition of the OSD is being passed in as an argument, the
+sub-command can scan its contents.
+
+In the case where the device is already mounted, the tool can detect this
+scenario and capture file contents from that directory.
+
+If the device is not mounted, a temporary directory will be created, and the
+device will be mounted temporarily just for scanning the contents. Once
+contents are scanned, the device will be unmounted.
+
+For a device like ``/dev/sda1`` which **must** be a data partition, the command
+could look like::
+
+    ceph-volume simple scan /dev/sda1
+
+
+.. _ceph-volume-simple-scan-json:
+
+``JSON`` contents
+-----------------
+The contents of the JSON object is very simple. The scan not only will persist
+information from the special OSD files and their contents, but will also
+validate paths and device UUIDs. Unlike what ``ceph-disk`` would do, by storing
+them in ``{device type}_uuid`` files, the tool will persist them as part of the
+device type key.
+
+For example, a ``block.db`` device would look something like::
+
+    "block.db": {
+        "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+        "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e"
+    },
+
+But it will also persist the ``ceph-disk`` special file generated, like so::
+
+    "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+
+This duplication is in place because the tool is trying to ensure the
+following:
+
+# Support OSDs that may not have ceph-disk special files
+# Check the most up-to-date information on the device, by querying against LVM
+and ``blkid``
+# Support both logical volumes and GPT devices
+
+This is a sample ``JSON`` metadata, from an OSD that is using ``bluestore``::
+
+    {
+        "active": "ok",
+        "block": {
+            "path": "/dev/disk/by-partuuid/40fd0a64-caa5-43a3-9717-1836ac661a12",
+            "uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12"
+        },
+        "block.db": {
+            "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+            "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e"
+        },
+        "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+        "block_uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12",
+        "bluefs": "1",
+        "ceph_fsid": "c92fc9eb-0610-4363-aafc-81ddf70aaf1b",
+        "cluster_name": "ceph",
+        "data": {
+            "path": "/dev/sdr1",
+            "uuid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96"
+        },
+        "fsid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96",
+        "keyring": "AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==",
+        "kv_backend": "rocksdb",
+        "magic": "ceph osd volume v026",
+        "mkfs_done": "yes",
+        "ready": "ready",
+        "systemd": "",
+        "type": "bluestore",
+        "whoami": "3"
+    }
diff --git a/doc/ceph-volume/simple/systemd.rst b/doc/ceph-volume/simple/systemd.rst
new file mode 100644
index 000000000..aa5bebffe
--- /dev/null
+++ b/doc/ceph-volume/simple/systemd.rst
@@ -0,0 +1,28 @@
+.. _ceph-volume-simple-systemd:
+
+systemd
+=======
+Upon startup, it will identify the logical volume by loading the JSON file in
+``/etc/ceph/osd/{id}-{uuid}.json`` corresponding to the instance name of the
+systemd unit.
+
+After identifying the correct volume it will then proceed to mount it by using
+the OSD destination conventions, that is::
+
+    /var/lib/ceph/osd/{cluster name}-{osd id}
+
+For our example OSD with an id of ``0``, that means the identified device will
+be mounted at::
+
+
+    /var/lib/ceph/osd/ceph-0
+
+
+Once that process is complete, a call will be made to start the OSD::
+
+    systemctl start ceph-osd@0
+
+The systemd portion of this process is handled by the ``ceph-volume simple
+trigger`` sub-command, which is only in charge of parsing metadata coming from
+systemd and startup, and then dispatching to ``ceph-volume simple activate`` which
+would proceed with activation.
diff --git a/doc/ceph-volume/systemd.rst b/doc/ceph-volume/systemd.rst
new file mode 100644
index 000000000..5b5273c9c
--- /dev/null
+++ b/doc/ceph-volume/systemd.rst
@@ -0,0 +1,49 @@
+.. _ceph-volume-systemd:
+
+systemd
+=======
+As part of the activation process (either with :ref:`ceph-volume-lvm-activate`
+or :ref:`ceph-volume-simple-activate`), systemd units will get enabled that
+will use the OSD id and uuid as part of their name. These units will be run
+when the system boots, and will proceed to activate their corresponding
+volumes via their sub-command implementation.
+
+The API for activation is a bit loose, it only requires two parts: the
+subcommand to use and any extra meta information separated by a dash. This
+convention makes the units look like::
+
+    ceph-volume@{command}-{extra metadata}
+
+The *extra metadata* can be anything needed that the subcommand implementing
+the processing might need. In the case of :ref:`ceph-volume-lvm` and
+:ref:`ceph-volume-simple`, both look to consume the :term:`OSD id` and :term:`OSD uuid`,
+but this is not a hard requirement, it is just how the sub-commands are
+implemented.
+
+Both the command and extra metadata gets persisted by systemd as part of the
+*"instance name"* of the unit.  For example an OSD with an ID of 0, for the
+``lvm`` sub-command would look like::
+
+    systemctl enable ceph-volume@lvm-0-0A3E1ED2-DA8A-4F0E-AA95-61DEC71768D6
+
+The enabled unit is a :term:`systemd oneshot` service, meant to start at boot
+after the local file system is ready to be used.
+
+
+Failure and Retries
+-------------------
+It is common to have failures when a system is coming up online. The devices
+are sometimes not fully available and this unpredictable behavior may cause an
+OSD to not be ready to be used.
+
+There are two configurable environment variables used to set the retry
+behavior:
+
+* ``CEPH_VOLUME_SYSTEMD_TRIES``: Defaults to 30
+* ``CEPH_VOLUME_SYSTEMD_INTERVAL``: Defaults to 5
+
+The *"tries"* is a number that sets the maximum number of times the unit will
+attempt to activate an OSD before giving up.
+
+The *"interval"* is a value in seconds that determines the waiting time before
+initiating another try at activating the OSD.
diff --git a/doc/ceph-volume/zfs/index.rst b/doc/ceph-volume/zfs/index.rst
new file mode 100644
index 000000000..c06228de9
--- /dev/null
+++ b/doc/ceph-volume/zfs/index.rst
@@ -0,0 +1,31 @@
+.. _ceph-volume-zfs:
+
+``zfs``
+=======
+Implements the functionality needed to deploy OSDs from the ``zfs`` subcommand:
+``ceph-volume zfs``
+
+The current implementation only works for ZFS on FreeBSD
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-zfs-inventory`
+
+.. not yet implemented
+.. * :ref:`ceph-volume-zfs-prepare`
+
+.. * :ref:`ceph-volume-zfs-activate`
+
+.. * :ref:`ceph-volume-zfs-create`
+
+.. * :ref:`ceph-volume-zfs-list`
+
+.. * :ref:`ceph-volume-zfs-scan`
+
+**Internal functionality**
+
+There are other aspects of the ``zfs`` subcommand that are internal and not
+exposed to the user, these sections explain how these pieces work together,
+clarifying the workflows of the tool.
+
+:ref:`zfs <ceph-volume-zfs-api>`
diff --git a/doc/ceph-volume/zfs/inventory.rst b/doc/ceph-volume/zfs/inventory.rst
new file mode 100644
index 000000000..fd00325b6
--- /dev/null
+++ b/doc/ceph-volume/zfs/inventory.rst
@@ -0,0 +1,19 @@
+.. _ceph-volume-zfs-inventory:
+
+``inventory``
+=============
+The ``inventory`` subcommand queries a host's disc inventory through GEOM and provides
+hardware information and metadata on every physical device.
+
+This only works on a FreeBSD platform.
+
+By default the command returns a short, human-readable report of all physical disks.
+
+For programmatic consumption of this report pass ``--format json`` to generate a
+JSON formatted report. This report includes extensive information on the
+physical drives such as disk metadata (like model and size), logical volumes
+and whether they are used by ceph, and if the disk is usable by ceph and
+reasons why not.
+
+A device path can be specified to report extensive information on a device in
+both plain and json format.
diff --git a/doc/cephadm/adoption.rst b/doc/cephadm/adoption.rst
new file mode 100644
index 000000000..86254a16c
--- /dev/null
+++ b/doc/cephadm/adoption.rst
@@ -0,0 +1,212 @@
+.. _cephadm-adoption:
+
+Converting an existing cluster to cephadm
+=========================================
+
+It is possible to convert some existing clusters so that they can be managed
+with ``cephadm``. This statement applies to some clusters that were deployed
+with ``ceph-deploy``, ``ceph-ansible``, or ``DeepSea``.
+
+This section of the documentation explains how to determine whether your
+clusters can be converted to a state in which they can be managed by
+``cephadm`` and how to perform those conversions.
+
+Limitations
+-----------
+
+* Cephadm works only with BlueStore OSDs.
+
+Preparation
+-----------
+
+#. Make sure that the ``cephadm`` command line tool is available on each host
+   in the existing cluster.  See :ref:`get-cephadm` to learn how.
+
+#. Prepare each host for use by ``cephadm`` by running this command:
+
+   .. prompt:: bash #
+
+      cephadm prepare-host
+
+#. Choose a version of Ceph to use for the conversion. This procedure will work
+   with any release of Ceph that is Octopus (15.2.z) or later, inclusive.  The
+   latest stable release of Ceph is the default. You might be upgrading from an
+   earlier Ceph release at the same time that you're performing this
+   conversion; if you are upgrading from an earlier release, make sure to
+   follow any upgrade-related instructions for that release.
+
+   Pass the image to cephadm with the following command:
+
+   .. prompt:: bash #
+
+      cephadm --image $IMAGE <rest of command goes here>
+
+   The conversion begins.
+
+#. Confirm that the conversion is underway by running ``cephadm ls`` and
+   making sure that the style of the daemons is changed:
+
+   .. prompt:: bash #
+
+      cephadm ls
+
+   Before starting the conversion process, ``cephadm ls`` shows all existing
+   daemons to have a style of ``legacy``. As the adoption process progresses,
+   adopted daemons will appear with a style of ``cephadm:v1``.
+
+
+Adoption process
+----------------
+
+#. Make sure that the ceph configuration has been migrated to use the cluster
+   config database.  If the ``/etc/ceph/ceph.conf`` is identical on each host,
+   then the following command can be run on one single host and will affect all
+   hosts:
+
+   .. prompt:: bash #
+
+      ceph config assimilate-conf -i /etc/ceph/ceph.conf
+
+   If there are configuration variations between hosts, you will need to repeat
+   this command on each host. During this adoption process, view the cluster's
+   configuration to confirm that it is complete by running the following
+   command:
+
+   .. prompt:: bash #
+
+      ceph config dump
+
+#. Adopt each monitor:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name mon.<hostname>
+
+   Each legacy monitor should stop, quickly restart as a cephadm
+   container, and rejoin the quorum.
+
+#. Adopt each manager:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name mgr.<hostname>
+
+#. Enable cephadm:
+
+   .. prompt:: bash #
+
+      ceph mgr module enable cephadm
+      ceph orch set backend cephadm
+
+#. Generate an SSH key:
+
+   .. prompt:: bash #
+
+      ceph cephadm generate-key
+      ceph cephadm get-pub-key > ~/ceph.pub
+
+#. Install the cluster SSH key on each host in the cluster:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i ~/ceph.pub root@<host>
+
+   .. note::
+     It is also possible to import an existing SSH key. See
+     :ref:`SSH errors <cephadm-ssh-errors>` in the troubleshooting
+     document for instructions that describe how to import existing
+     SSH keys.
+
+   .. note::
+     It is also possible to have cephadm use a non-root user to SSH 
+     into cluster hosts. This user needs to have passwordless sudo access.
+     Use ``ceph cephadm set-user <user>`` and copy the SSH key to that user.
+     See :ref:`cephadm-ssh-user`
+
+#. Tell cephadm which hosts to manage:
+
+   .. prompt:: bash #
+
+      ceph orch host add <hostname> [ip-address]
+
+   This will perform a ``cephadm check-host`` on each host before adding it;
+   this check ensures that the host is functioning properly. The IP address
+   argument is recommended; if not provided, then the host name will be resolved
+   via DNS.
+
+#. Verify that the adopted monitor and manager daemons are visible:
+
+   .. prompt:: bash #
+
+      ceph orch ps
+
+#. Adopt all OSDs in the cluster:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name <name>
+
+   For example:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name osd.1
+      cephadm adopt --style legacy --name osd.2
+
+#. Redeploy MDS daemons by telling cephadm how many daemons to run for
+   each file system. List file systems by name with the command ``ceph fs
+   ls``. Run the following command on the master nodes to redeploy the MDS
+   daemons:
+
+   .. prompt:: bash #
+
+      ceph orch apply mds <fs-name> [--placement=<placement>]
+
+   For example, in a cluster with a single file system called `foo`:
+
+   .. prompt:: bash #
+
+      ceph fs ls
+
+   .. code-block:: bash
+
+      name: foo, metadata pool: foo_metadata, data pools: [foo_data ]
+
+   .. prompt:: bash #
+
+      ceph orch apply mds foo 2
+
+   Confirm that the new MDS daemons have started:
+
+   .. prompt:: bash #
+
+      ceph orch ps --daemon-type mds
+
+   Finally, stop and remove the legacy MDS daemons:
+
+   .. prompt:: bash #
+
+      systemctl stop ceph-mds.target
+      rm -rf /var/lib/ceph/mds/ceph-*
+
+#. Redeploy RGW daemons. Cephadm manages RGW daemons by zone. For each
+   zone, deploy new RGW daemons with cephadm:
+
+   .. prompt:: bash #
+
+      ceph orch apply rgw <svc_id> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>]
+
+   where *<placement>* can be a simple daemon count, or a list of
+   specific hosts (see :ref:`orchestrator-cli-placement-spec`), and the
+   zone and realm arguments are needed only for a multisite setup.
+
+   After the daemons have started and you have confirmed that they are
+   functioning, stop and remove the old, legacy daemons:
+
+   .. prompt:: bash #
+
+      systemctl stop ceph-rgw.target
+      rm -rf /var/lib/ceph/radosgw/ceph-*
+
+#. Check the output of the command ``ceph health detail`` for cephadm warnings
+   about stray cluster daemons or hosts that are not yet managed by cephadm.
diff --git a/doc/cephadm/client-setup.rst b/doc/cephadm/client-setup.rst
new file mode 100644
index 000000000..f98ba798b
--- /dev/null
+++ b/doc/cephadm/client-setup.rst
@@ -0,0 +1,45 @@
+=======================
+Basic Ceph Client Setup
+=======================
+Client machines require some basic configuration to interact with
+Ceph clusters. This section describes how to configure a client machine
+so that it can interact with a Ceph cluster.
+
+.. note:: 
+   Most client machines need to install only the `ceph-common` package
+   and its dependencies. Such a setup supplies the basic `ceph` and
+   `rados` commands, as well as other commands including `mount.ceph`
+   and `rbd`.
+
+Config File Setup
+=================
+Client machines usually require smaller configuration files (here
+sometimes called "config files") than do full-fledged cluster members.
+To generate a minimal config file, log into a host that has been
+configured as a client or that is running a cluster daemon, and then run the following command:
+
+.. prompt:: bash #
+
+  ceph config generate-minimal-conf
+
+This command generates a minimal config file that tells the client how
+to reach the Ceph monitors. The contents of this file should usually 
+be installed in ``/etc/ceph/ceph.conf``.
+
+Keyring Setup
+=============
+Most Ceph clusters run with authentication enabled. This means that
+the client needs keys in order to communicate with the machines in the
+cluster. To generate a keyring file with credentials for `client.fs`,
+log into an running cluster member and run the following command:
+
+.. prompt:: bash $
+
+  ceph auth get-or-create client.fs
+
+The resulting output is directed into a keyring file, typically
+``/etc/ceph/ceph.keyring``.
+
+To gain a broader understanding of client keyring distribution and administration, you should read :ref:`client_keyrings_and_configs`.
+
+To see an example that explains how to distribute ``ceph.conf`` configuration files to hosts that are tagged with the ``bare_config`` label, you should read the section called "Distributing ceph.conf to hosts tagged with bare_config" in the section called :ref:`etc_ceph_conf_distribution`.
diff --git a/doc/cephadm/compatibility.rst b/doc/cephadm/compatibility.rst
new file mode 100644
index 000000000..46ab62a62
--- /dev/null
+++ b/doc/cephadm/compatibility.rst
@@ -0,0 +1,66 @@
+
+===========================
+Compatibility and Stability
+===========================
+
+.. _cephadm-compatibility-with-podman:
+
+Compatibility with Podman Versions
+----------------------------------
+
+Podman and Ceph have different end-of-life strategies. This means that care
+must be taken in finding a version of Podman that is compatible with Ceph.
+
+This table shows which version pairs are expected to work or not work together:
+
+
++-----------+-----------------------------------------------+
+|  Ceph     |                 Podman                        |
++-----------+-------+-------+-------+-------+-------+-------+
+|           | 1.9   |  2.0  |  2.1  |  2.2  |  3.0  | > 3.0 |
++===========+=======+=======+=======+=======+=======+=======+
+| <= 15.2.5 | True  | False | False | False | False | False |
++-----------+-------+-------+-------+-------+-------+-------+
+| >= 15.2.6 | True  | True  | True  | False | False | False |
++-----------+-------+-------+-------+-------+-------+-------+
+| >= 16.2.1 | False | True  | True  | False | True  | True  |
++-----------+-------+-------+-------+-------+-------+-------+
+| >= 17.2.0 | False | True  | True  | False | True  | True  |
++-----------+-------+-------+-------+-------+-------+-------+
+
+.. note::
+
+  While not all podman versions have been actively tested against
+  all Ceph versions, there are no known issues with using podman
+  version 3.0 or greater with Ceph Quincy and later releases.
+
+.. warning:: 
+
+   To use Podman with Ceph Pacific, you must use **a version of Podman that
+   is 2.0.0 or higher**. However, **Podman version 2.2.1 does not work with
+   Ceph Pacific**.
+   
+   "Kubic stable" is known to work with Ceph Pacific, but it must be run
+   with a newer kernel.
+
+
+.. _cephadm-stability:
+
+Stability
+---------
+
+Cephadm is relatively stable but new functionality is still being
+added and bugs are occasionally discovered. If issues are found, please
+open a tracker issue under the Orchestrator component (https://tracker.ceph.com/projects/orchestrator/issues)
+
+Cephadm support remains under development for the following features:
+
+- ceph-exporter deployment
+- stretch mode integration
+- monitoring stack (moving towards prometheus service discover and providing TLS)
+- RGW multisite deployment support (requires lots of manual steps currently)
+- cephadm agent
+
+If a cephadm command fails or a service stops running properly, see
+:ref:`cephadm-pause` for instructions on how to pause the Ceph cluster's
+background activity and how to disable cephadm.
diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst
new file mode 100644
index 000000000..4b964c5f4
--- /dev/null
+++ b/doc/cephadm/host-management.rst
@@ -0,0 +1,605 @@
+.. _orchestrator-cli-host-management:
+
+===============
+Host Management
+===============
+
+Listing Hosts
+=============
+
+Run a command of this form to list hosts associated with the cluster:
+
+.. prompt:: bash #
+
+   ceph orch host ls [--format yaml] [--host-pattern <name>] [--label <label>] [--host-status <status>] [--detail]
+
+In commands of this form, the arguments "host-pattern", "label", and
+"host-status" are optional and are used for filtering. 
+
+- "host-pattern" is a regex that matches against hostnames and returns only
+  matching hosts.
+- "label" returns only hosts with the specified label.
+- "host-status" returns only hosts with the specified status (currently
+  "offline" or "maintenance").
+- Any combination of these filtering flags is valid. It is possible to filter
+  against name, label and status simultaneously, or to filter against any
+  proper subset of name, label and status.
+
+The "detail" parameter provides more host related information for cephadm based
+clusters. For example:
+
+.. prompt:: bash #
+
+   ceph orch host ls --detail
+
+::
+
+    HOSTNAME     ADDRESS         LABELS  STATUS  VENDOR/MODEL                           CPU    HDD      SSD  NIC
+    ceph-master  192.168.122.73  _admin          QEMU (Standard PC (Q35 + ICH9, 2009))  4C/4T  4/1.6TB  -    1
+    1 hosts in cluster
+
+.. _cephadm-adding-hosts:    
+    
+Adding Hosts
+============
+
+Hosts must have these :ref:`cephadm-host-requirements` installed.
+Hosts without all the necessary requirements will fail to be added to the cluster.
+
+To add each new host to the cluster, perform two steps:
+
+#. Install the cluster's public SSH key in the new host's root user's ``authorized_keys`` file:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@*<new-host>*
+
+   For example:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@host2
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@host3
+
+#. Tell Ceph that the new node is part of the cluster:
+
+   .. prompt:: bash #
+
+      ceph orch host add *<newhost>* [*<ip>*] [*<label1> ...*]
+
+   For example:
+
+   .. prompt:: bash #
+
+      ceph orch host add host2 10.10.0.102
+      ceph orch host add host3 10.10.0.103
+
+   It is best to explicitly provide the host IP address.  If an IP is
+   not provided, then the host name will be immediately resolved via
+   DNS and that IP will be used.
+
+   One or more labels can also be included to immediately label the
+   new host.  For example, by default the ``_admin`` label will make
+   cephadm maintain a copy of the ``ceph.conf`` file and a
+   ``client.admin`` keyring file in ``/etc/ceph``:
+
+   .. prompt:: bash #
+
+      ceph orch host add host4 10.10.0.104 --labels _admin
+
+.. _cephadm-removing-hosts:
+
+Removing Hosts
+==============
+
+A host can safely be removed from the cluster after all daemons are removed
+from it.
+
+To drain all daemons from a host, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph orch host drain *<host>*
+
+The ``_no_schedule`` and ``_no_conf_keyring`` labels will be applied to the
+host. See :ref:`cephadm-special-host-labels`.
+
+If you only want to drain daemons but leave managed ceph conf and keyring
+files on the host, you may pass the ``--keep-conf-keyring`` flag to the
+drain command.
+
+.. prompt:: bash #
+
+   ceph orch host drain *<host>* --keep-conf-keyring
+
+This will apply the ``_no_schedule`` label to the host but not the
+``_no_conf_keyring`` label.
+
+All OSDs on the host will be scheduled to be removed. You can check the progress of the OSD removal operation with the following command:
+
+.. prompt:: bash #
+
+   ceph orch osd rm status
+
+See :ref:`cephadm-osd-removal` for more details about OSD removal.
+
+The ``orch host drain`` command also supports a ``--zap-osd-devices``
+flag. Setting this flag while draining a host will cause cephadm to zap
+the devices of the OSDs it is removing as part of the drain process
+
+.. prompt:: bash #
+
+   ceph orch host drain *<host>* --zap-osd-devices
+
+Use the following command to determine whether any daemons are still on the
+host:
+
+.. prompt:: bash #
+
+   ceph orch ps <host> 
+
+After all daemons have been removed from the host, remove the host from the
+cluster by running the following command: 
+
+.. prompt:: bash #
+
+   ceph orch host rm <host>
+
+Offline host removal
+--------------------
+
+Even if a host is offline and can not be recovered, it can be removed from the
+cluster by running a command of the following form:
+
+.. prompt:: bash #
+
+   ceph orch host rm <host> --offline --force
+
+.. warning:: This can potentially cause data loss. This command forcefully
+   purges OSDs from the cluster by calling ``osd purge-actual`` for each OSD.
+   Any service specs that still contain this host should be manually updated.
+
+.. _orchestrator-host-labels:
+
+Host labels
+===========
+
+The orchestrator supports assigning labels to hosts. Labels
+are free form and have no particular meaning by itself and each host
+can have multiple labels. They can be used to specify placement
+of daemons. See :ref:`orch-placement-by-labels`
+
+Labels can be added when adding a host with the ``--labels`` flag:
+
+.. prompt:: bash #
+
+   ceph orch host add my_hostname --labels=my_label1
+   ceph orch host add my_hostname --labels=my_label1,my_label2
+
+To add a label a existing host, run:
+
+.. prompt:: bash #
+
+   ceph orch host label add my_hostname my_label
+
+To remove a label, run:
+
+.. prompt:: bash #
+
+   ceph orch host label rm my_hostname my_label
+
+
+.. _cephadm-special-host-labels:
+
+Special host labels
+-------------------
+
+The following host labels have a special meaning to cephadm.  All start with ``_``.
+
+* ``_no_schedule``: *Do not schedule or deploy daemons on this host*.
+
+  This label prevents cephadm from deploying daemons on this host.  If it is added to
+  an existing host that already contains Ceph daemons, it will cause cephadm to move
+  those daemons elsewhere (except OSDs, which are not removed automatically).
+
+* ``_no_conf_keyring``: *Do not deploy config files or keyrings on this host*.
+
+  This label is effectively the same as ``_no_schedule`` but instead of working for
+  daemons it works for client keyrings and ceph conf files that are being managed
+  by cephadm
+
+* ``_no_autotune_memory``: *Do not autotune memory on this host*.
+
+  This label will prevent daemon memory from being tuned even when the
+  ``osd_memory_target_autotune`` or similar option is enabled for one or more daemons
+  on that host.
+
+* ``_admin``: *Distribute client.admin and ceph.conf to this host*.
+
+  By default, an ``_admin`` label is applied to the first host in the cluster (where
+  bootstrap was originally run), and the ``client.admin`` key is set to be distributed
+  to that host via the ``ceph orch client-keyring ...`` function.  Adding this label
+  to additional hosts will normally cause cephadm to deploy config and keyring files
+  in ``/etc/ceph``. Starting from versions 16.2.10 (Pacific) and 17.2.1 (Quincy) in
+  addition to the default location ``/etc/ceph/`` cephadm also stores config and keyring
+  files in the ``/var/lib/ceph/<fsid>/config`` directory.
+
+Maintenance Mode
+================
+
+Place a host in and out of maintenance mode (stops all Ceph daemons on host):
+
+.. prompt:: bash #
+
+   ceph orch host maintenance enter <hostname> [--force] [--yes-i-really-mean-it]
+   ceph orch host maintenance exit <hostname>
+
+The ``--force`` flag allows the user to bypass warnings (but not alerts). The ``--yes-i-really-mean-it``
+flag bypasses all safety checks and will attempt to force the host into maintenance mode no
+matter what.
+
+.. warning:: Using the --yes-i-really-mean-it flag to force the host to enter maintenance
+   mode can potentially cause loss of data availability, the mon quorum to break down due
+   to too few running monitors, mgr module commands (such as ``ceph orch . . .`` commands)
+   to be become unresponsive, and a number of other possible issues. Please only use this
+   flag if you're absolutely certain you know what you're doing.
+
+See also :ref:`cephadm-fqdn`
+
+Rescanning Host Devices
+=======================
+
+Some servers and external enclosures may not register device removal or insertion with the
+kernel. In these scenarios, you'll need to perform a host rescan. A rescan is typically
+non-disruptive, and can be performed with the following CLI command:
+
+.. prompt:: bash #
+
+   ceph orch host rescan <hostname> [--with-summary]
+
+The ``with-summary`` flag provides a breakdown of the number of HBAs found and scanned, together
+with any that failed:
+
+.. prompt:: bash [ceph:root@rh9-ceph1/]#
+
+   ceph orch host rescan rh9-ceph1 --with-summary
+   
+::
+
+   Ok. 2 adapters detected: 2 rescanned, 0 skipped, 0 failed (0.32s)
+
+Creating many hosts at once
+===========================
+
+Many hosts can be added at once using
+``ceph orch apply -i`` by submitting a multi-document YAML file:
+
+.. code-block:: yaml
+
+    service_type: host
+    hostname: node-00
+    addr: 192.168.0.10
+    labels:
+    - example1
+    - example2
+    ---
+    service_type: host
+    hostname: node-01
+    addr: 192.168.0.11
+    labels:
+    - grafana
+    ---
+    service_type: host
+    hostname: node-02
+    addr: 192.168.0.12
+
+This can be combined with :ref:`service specifications<orchestrator-cli-service-spec>`
+to create a cluster spec file to deploy a whole cluster in one command.  see
+``cephadm bootstrap --apply-spec`` also to do this during bootstrap. Cluster
+SSH Keys must be copied to hosts prior to adding them.
+
+Setting the initial CRUSH location of host
+==========================================
+
+Hosts can contain a ``location`` identifier which will instruct cephadm to 
+create a new CRUSH host located in the specified hierarchy.
+
+.. code-block:: yaml
+
+    service_type: host
+    hostname: node-00
+    addr: 192.168.0.10
+    location:
+      rack: rack1
+
+.. note:: 
+
+  The ``location`` attribute will be only affect the initial CRUSH location. Subsequent
+  changes of the ``location`` property will be ignored. Also, removing a host will not remove
+  any CRUSH buckets.
+
+See also :ref:`crush_map_default_types`.
+
+OS Tuning Profiles
+==================
+
+Cephadm can be used to manage operating-system-tuning profiles that apply sets
+of sysctl settings to sets of hosts. 
+
+Create a YAML spec file in the following format:
+
+.. code-block:: yaml
+
+    profile_name: 23-mon-host-profile
+    placement:
+      hosts:
+        - mon-host-01
+        - mon-host-02
+    settings:
+      fs.file-max: 1000000
+      vm.swappiness: '13'
+
+Apply the tuning profile with the following command:
+
+.. prompt:: bash #
+
+   ceph orch tuned-profile apply -i <tuned-profile-file-name>
+
+This profile is written to ``/etc/sysctl.d/`` on each host that matches the
+hosts specified in the placement block of the yaml, and ``sysctl --system`` is
+run on the host.
+
+.. note::
+
+  The exact filename that the profile is written to within ``/etc/sysctl.d/``
+  is ``<profile-name>-cephadm-tuned-profile.conf``, where ``<profile-name>`` is
+  the ``profile_name`` setting that you specify in the YAML spec. Because
+  sysctl settings are applied in lexicographical order (sorted by the filename
+  in which the setting is specified), you may want to set the ``profile_name``
+  in your spec so that it is applied before or after other conf files.
+
+.. note::
+
+  These settings are applied only at the host level, and are not specific
+  to any particular daemon or container.
+
+.. note::
+
+  Applying tuned profiles is idempotent when the ``--no-overwrite`` option is
+  passed. Moreover, if the ``--no-overwrite`` option is passed, existing
+  profiles with the same name are not overwritten.
+
+
+Viewing Profiles
+----------------
+
+Run the following command to view all the profiles that cephadm currently manages:
+
+.. prompt:: bash #
+
+   ceph orch tuned-profile ls
+
+.. note:: 
+
+  To make modifications and re-apply a profile, pass ``--format yaml`` to the
+  ``tuned-profile ls`` command. The ``tuned-profile ls --format yaml`` command
+  presents the profiles in a format that is easy to copy and re-apply.
+
+
+Removing Profiles
+-----------------
+
+To remove a previously applied profile, run this command:
+
+.. prompt:: bash #
+
+   ceph orch tuned-profile rm <profile-name>
+
+When a profile is removed, cephadm cleans up the file previously written to ``/etc/sysctl.d``.
+
+
+Modifying Profiles
+------------------
+
+Profiles can be modified by re-applying a YAML spec with the same name as the
+profile that you want to modify, but settings within existing profiles can be
+adjusted with the following commands.
+
+To add or modify a setting in an existing profile:
+
+.. prompt:: bash #
+
+   ceph orch tuned-profile add-setting <profile-name> <setting-name> <value>
+
+To remove a setting from an existing profile:
+
+.. prompt:: bash #
+
+   ceph orch tuned-profile rm-setting <profile-name> <setting-name>
+
+.. note:: 
+
+  Modifying the placement requires re-applying a profile with the same name.
+  Remember that profiles are tracked by their names, so when a profile with the
+  same name as an existing profile is applied, it overwrites the old profile
+  unless the ``--no-overwrite`` flag is passed.
+
+SSH Configuration
+=================
+
+Cephadm uses SSH to connect to remote hosts.  SSH uses a key to authenticate
+with those hosts in a secure way.
+
+
+Default behavior
+----------------
+
+Cephadm stores an SSH key in the monitor that is used to
+connect to remote hosts.  When the cluster is bootstrapped, this SSH
+key is generated automatically and no additional configuration
+is necessary.
+
+A *new* SSH key can be generated with:
+
+.. prompt:: bash #
+
+   ceph cephadm generate-key
+
+The public portion of the SSH key can be retrieved with:
+
+.. prompt:: bash #
+
+   ceph cephadm get-pub-key
+
+The currently stored SSH key can be deleted with:
+
+.. prompt:: bash #
+
+   ceph cephadm clear-key
+
+You can make use of an existing key by directly importing it with:
+
+.. prompt:: bash #
+
+   ceph config-key set mgr/cephadm/ssh_identity_key -i <key>
+   ceph config-key set mgr/cephadm/ssh_identity_pub -i <pub>
+
+You will then need to restart the mgr daemon to reload the configuration with:
+
+.. prompt:: bash #
+
+   ceph mgr fail
+
+.. _cephadm-ssh-user:
+
+Configuring a different SSH user
+----------------------------------
+
+Cephadm must be able to log into all the Ceph cluster nodes as an user
+that has enough privileges to download container images, start containers
+and execute commands without prompting for a password. If you do not want
+to use the "root" user (default option in cephadm), you must provide
+cephadm the name of the user that is going to be used to perform all the
+cephadm operations. Use the command:
+
+.. prompt:: bash #
+
+   ceph cephadm set-user <user>
+
+Prior to running this the cluster SSH key needs to be added to this users
+authorized_keys file and non-root users must have passwordless sudo access.
+
+
+Customizing the SSH configuration
+---------------------------------
+
+Cephadm generates an appropriate ``ssh_config`` file that is
+used for connecting to remote hosts.  This configuration looks
+something like this::
+
+  Host *
+  User root
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+
+There are two ways to customize this configuration for your environment:
+
+#. Import a customized configuration file that will be stored
+   by the monitor with:
+
+   .. prompt:: bash #
+
+      ceph cephadm set-ssh-config -i <ssh_config_file>
+
+   To remove a customized SSH config and revert back to the default behavior:
+
+   .. prompt:: bash #
+
+      ceph cephadm clear-ssh-config
+
+#. You can configure a file location for the SSH configuration file with:
+
+   .. prompt:: bash #
+
+      ceph config set mgr mgr/cephadm/ssh_config_file <path>
+
+   We do *not recommend* this approach.  The path name must be
+   visible to *any* mgr daemon, and cephadm runs all daemons as
+   containers. That means that the file either need to be placed
+   inside a customized container image for your deployment, or
+   manually distributed to the mgr data directory
+   (``/var/lib/ceph/<cluster-fsid>/mgr.<id>`` on the host, visible at
+   ``/var/lib/ceph/mgr/ceph-<id>`` from inside the container).
+
+Setting up CA signed keys for the cluster
+-----------------------------------------
+
+Cephadm also supports using CA signed keys for SSH authentication
+across cluster nodes. In this setup, instead of needing a private
+key and public key, we instead need a private key and certificate
+created by signing that private key with a CA key. For more info
+on setting up nodes for authentication using a CA signed key, see
+:ref:`cephadm-bootstrap-ca-signed-keys`. Once you have your private
+key and signed cert, they can be set up for cephadm to use by running:
+
+.. prompt:: bash #
+
+   ceph config-key set mgr/cephadm/ssh_identity_key -i <private-key-file>
+   ceph config-key set mgr/cephadm/ssh_identity_cert -i <signed-cert-file>
+
+.. _cephadm-fqdn:
+
+Fully qualified domain names vs bare host names
+===============================================
+
+.. note::
+
+  cephadm demands that the name of the host given via ``ceph orch host add`` 
+  equals the output of ``hostname`` on remote hosts.
+
+Otherwise cephadm can't be sure that names returned by
+``ceph * metadata`` match the hosts known to cephadm. This might result
+in a :ref:`cephadm-stray-host` warning.
+
+When configuring new hosts, there are two **valid** ways to set the 
+``hostname`` of a host:
+
+1. Using the bare host name. In this case:
+
+-  ``hostname`` returns the bare host name.
+-  ``hostname -f`` returns the FQDN.
+
+2. Using the fully qualified domain name as the host name. In this case:
+
+-  ``hostname`` returns the FQDN
+-  ``hostname -s`` return the bare host name
+
+Note that ``man hostname`` recommends ``hostname`` to return the bare
+host name:
+
+    The FQDN (Fully Qualified Domain Name) of the system is the
+    name that the resolver(3) returns for the host name, such as,
+    ursula.example.com. It is usually the hostname followed by the DNS
+    domain name (the part after the first dot). You can check the FQDN
+    using ``hostname --fqdn`` or the domain name using ``dnsdomainname``.
+
+    .. code-block:: none
+
+          You cannot change the FQDN with hostname or dnsdomainname.
+
+          The recommended method of setting the FQDN is to make the hostname
+          be an alias for the fully qualified name using /etc/hosts, DNS, or
+          NIS. For example, if the hostname was "ursula", one might have
+          a line in /etc/hosts which reads
+
+                 127.0.1.1    ursula.example.com ursula
+
+Which means, ``man hostname`` recommends ``hostname`` to return the bare
+host name. This in turn means that Ceph will return the bare host names
+when executing ``ceph * metadata``. This in turn means cephadm also
+requires the bare host name when adding a host to the cluster: 
+``ceph orch host add <bare-name>``.
+
+..
+  TODO: This chapter needs to provide way for users to configure
+  Grafana in the dashboard, as this is right now very hard to do.
diff --git a/doc/cephadm/index.rst b/doc/cephadm/index.rst
new file mode 100644
index 000000000..bfa3a4bb2
--- /dev/null
+++ b/doc/cephadm/index.rst
@@ -0,0 +1,50 @@
+.. _cephadm:
+
+=======
+Cephadm
+=======
+
+``cephadm`` is a utility that is used to manage a Ceph cluster. 
+
+Here is a list of some of the things that ``cephadm`` can do:
+
+- ``cephadm`` can add a Ceph container to the cluster.
+- ``cephadm`` can remove a Ceph container from the cluster.
+- ``cephadm`` can update Ceph containers.
+
+``cephadm`` does not rely on external configuration tools like Ansible, Rook,
+or Salt. However, those external configuration tools can be used to automate
+operations not performed by cephadm itself. To learn more about these external
+configuration tools, visit their pages:
+
+ * https://github.com/ceph/cephadm-ansible
+ * https://rook.io/docs/rook/v1.10/Getting-Started/intro/
+ * https://github.com/ceph/ceph-salt
+
+``cephadm`` manages the full lifecycle of a Ceph cluster. This lifecycle starts
+with the bootstrapping process, when ``cephadm`` creates a tiny Ceph cluster on
+a single node. This cluster consists of one monitor and one manager.
+``cephadm`` then uses the orchestration interface to expand the cluster, adding
+hosts and provisioning Ceph daemons and services. Management of this lifecycle
+can be performed either via the Ceph command-line interface (CLI) or via the
+dashboard (GUI).
+
+To use ``cephadm`` to get started with Ceph, follow the instructions in
+:ref:`cephadm_deploying_new_cluster`.
+
+``cephadm`` was introduced in Ceph release v15.2.0 (Octopus) and does not
+support older versions of Ceph.
+
+.. toctree::
+    :maxdepth: 2
+
+    compatibility
+    install
+    adoption
+    host-management
+    Service Management <services/index>
+    upgrade
+    Cephadm operations <operations>
+    Client Setup <client-setup>
+    troubleshooting
+    Cephadm Feature Planning <../dev/cephadm/index>
diff --git a/doc/cephadm/install.rst b/doc/cephadm/install.rst
new file mode 100644
index 000000000..b1aa736e2
--- /dev/null
+++ b/doc/cephadm/install.rst
@@ -0,0 +1,578 @@
+.. _cephadm_deploying_new_cluster:
+
+============================
+Deploying a new Ceph cluster
+============================
+
+Cephadm creates a new Ceph cluster by "bootstrapping" on a single
+host, expanding the cluster to encompass any additional hosts, and
+then deploying the needed services.
+
+.. highlight:: console
+
+.. _cephadm-host-requirements:
+
+Requirements
+============
+
+- Python 3
+- Systemd
+- Podman or Docker for running containers
+- Time synchronization (such as chrony or NTP)
+- LVM2 for provisioning storage devices
+
+Any modern Linux distribution should be sufficient.  Dependencies
+are installed automatically by the bootstrap process below.
+
+See the section :ref:`Compatibility With Podman
+Versions<cephadm-compatibility-with-podman>` for a table of Ceph versions that
+are compatible with Podman. Not every version of Podman is compatible with
+Ceph.
+
+
+
+.. _get-cephadm:
+
+Install cephadm
+===============
+
+There are two ways to install ``cephadm``:
+
+#. a :ref:`curl-based installation<cephadm_install_curl>` method
+#. :ref:`distribution-specific installation methods<cephadm_install_distros>`
+
+.. important:: These methods of installing ``cephadm`` are mutually exclusive.
+   Choose either the distribution-specific method or the curl-based method. Do
+   not attempt to use both these methods on one system.
+
+.. _cephadm_install_distros:
+
+distribution-specific installations
+-----------------------------------
+
+Some Linux distributions  may already include up-to-date Ceph packages.  In
+that case, you can install cephadm directly. For example:
+
+  In Ubuntu:
+
+  .. prompt:: bash #
+
+     apt install -y cephadm
+
+  In CentOS Stream:
+
+  .. prompt:: bash #
+     :substitutions:
+
+     dnf search release-ceph
+     dnf install --assumeyes centos-release-ceph-|stable-release|
+     dnf install --assumeyes cephadm
+
+  In Fedora:
+
+  .. prompt:: bash #
+
+     dnf -y install cephadm
+
+  In SUSE:
+
+  .. prompt:: bash #
+
+     zypper install -y cephadm
+
+.. _cephadm_install_curl:
+
+curl-based installation
+-----------------------
+
+* First, determine what version of Ceph you will need. You can use the releases
+  page to find the `latest active releases <https://docs.ceph.com/en/latest/releases/#active-releases>`_.
+  For example, we might look at that page and find that ``18.2.0`` is the latest
+  active release.
+
+* Use ``curl`` to fetch a build of cephadm for that release.
+
+  .. prompt:: bash #
+     :substitutions:
+
+     CEPH_RELEASE=18.2.0 # replace this with the active release
+     curl --silent --remote-name --location https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm
+
+  Ensure the ``cephadm`` file is executable:
+
+  .. prompt:: bash #
+
+   chmod +x cephadm
+
+  This file can be run directly from the current directory:
+
+  .. prompt:: bash #
+
+   ./cephadm <arguments...>
+
+* If you encounter any issues with running cephadm due to errors including
+  the message ``bad interpreter``, then you may not have Python or
+  the correct version of Python installed. The cephadm tool requires Python 3.6
+  and above. You can manually run cephadm with a particular version of Python by
+  prefixing the command with your installed Python version. For example:
+
+  .. prompt:: bash #
+     :substitutions:
+
+     python3.8 ./cephadm <arguments...>
+
+.. _cephadm_update:
+
+update cephadm
+--------------
+
+The cephadm binary can be used to bootstrap a cluster and for a variety
+of other management and debugging tasks. The Ceph team strongly recommends
+using an actively supported version of cephadm. Additionally, although
+the standalone cephadm is sufficient to get a cluster started, it is
+convenient to have the ``cephadm`` command installed on the host. Older or LTS
+distros may also have ``cephadm`` packages that are out-of-date and
+running the commands below can help install a more recent version
+from the Ceph project's repositories.
+
+To install the packages provided by the Ceph project that provide the
+``cephadm`` command, run the following commands:
+
+.. prompt:: bash #
+   :substitutions:
+
+   ./cephadm add-repo --release |stable-release|
+   ./cephadm install
+
+Confirm that ``cephadm`` is now in your PATH by running ``which`` or
+``command -v``:
+
+.. prompt:: bash #
+
+   which cephadm
+
+A successful ``which cephadm`` command will return this:
+
+.. code-block:: bash
+
+   /usr/sbin/cephadm
+
+Bootstrap a new cluster
+=======================
+
+What to know before you bootstrap
+---------------------------------
+
+The first step in creating a new Ceph cluster is running the ``cephadm
+bootstrap`` command on the Ceph cluster's first host. The act of running the
+``cephadm bootstrap`` command on the Ceph cluster's first host creates the Ceph
+cluster's first "monitor daemon", and that monitor daemon needs an IP address.
+You must pass the IP address of the Ceph cluster's first host to the ``ceph
+bootstrap`` command, so you'll need to know the IP address of that host.
+
+.. important:: ``ssh`` must be installed and running in order for the
+   bootstrapping procedure to succeed. 
+
+.. note:: If there are multiple networks and interfaces, be sure to choose one
+   that will be accessible by any host accessing the Ceph cluster.
+
+Running the bootstrap command
+-----------------------------
+
+Run the ``ceph bootstrap`` command:
+
+.. prompt:: bash #
+
+   cephadm bootstrap --mon-ip *<mon-ip>*
+
+This command will:
+
+* Create a monitor and manager daemon for the new cluster on the local
+  host.
+* Generate a new SSH key for the Ceph cluster and add it to the root
+  user's ``/root/.ssh/authorized_keys`` file.
+* Write a copy of the public key to ``/etc/ceph/ceph.pub``.
+* Write a minimal configuration file to ``/etc/ceph/ceph.conf``. This
+  file is needed to communicate with the new cluster.
+* Write a copy of the ``client.admin`` administrative (privileged!)
+  secret key to ``/etc/ceph/ceph.client.admin.keyring``.
+* Add the ``_admin`` label to the bootstrap host.  By default, any host
+  with this label will (also) get a copy of ``/etc/ceph/ceph.conf`` and
+  ``/etc/ceph/ceph.client.admin.keyring``.
+
+.. _cephadm-bootstrap-further-info:
+
+Further information about cephadm bootstrap
+-------------------------------------------
+
+The default bootstrap behavior will work for most users. But if you'd like
+immediately to know more about ``cephadm bootstrap``, read the list below.
+
+Also, you can run ``cephadm bootstrap -h`` to see all of ``cephadm``'s
+available options.
+
+* By default, Ceph daemons send their log output to stdout/stderr, which is picked
+  up by the container runtime (docker or podman) and (on most systems) sent to
+  journald.  If you want Ceph to write traditional log files to ``/var/log/ceph/$fsid``,
+  use the ``--log-to-file`` option during bootstrap.
+
+* Larger Ceph clusters perform better when (external to the Ceph cluster)
+  public network traffic is separated from (internal to the Ceph cluster)
+  cluster traffic. The internal cluster traffic handles replication, recovery,
+  and heartbeats between OSD daemons.  You can define the :ref:`cluster
+  network<cluster-network>` by supplying the ``--cluster-network`` option to the ``bootstrap``
+  subcommand. This parameter must define a subnet in CIDR notation (for example
+  ``10.90.90.0/24`` or ``fe80::/64``).
+
+* ``cephadm bootstrap`` writes to ``/etc/ceph`` the files needed to access
+  the new cluster. This central location makes it possible for Ceph
+  packages installed on the host (e.g., packages that give access to the
+  cephadm command line interface) to find these files.
+
+  Daemon containers deployed with cephadm, however, do not need
+  ``/etc/ceph`` at all.  Use the ``--output-dir *<directory>*`` option
+  to put them in a different directory (for example, ``.``). This may help
+  avoid conflicts with an existing Ceph configuration (cephadm or
+  otherwise) on the same host.
+
+* You can pass any initial Ceph configuration options to the new
+  cluster by putting them in a standard ini-style configuration file
+  and using the ``--config *<config-file>*`` option.  For example::
+
+      $ cat <<EOF > initial-ceph.conf
+      [global]
+      osd crush chooseleaf type = 0
+      EOF
+      $ ./cephadm bootstrap --config initial-ceph.conf ...
+
+* The ``--ssh-user *<user>*`` option makes it possible to choose which SSH
+  user cephadm will use to connect to hosts. The associated SSH key will be
+  added to ``/home/*<user>*/.ssh/authorized_keys``. The user that you
+  designate with this option must have passwordless sudo access.
+
+* If you are using a container on an authenticated registry that requires
+  login, you may add the argument:
+
+  * ``--registry-json <path to json file>``
+
+  example contents of JSON file with login info::
+
+      {"url":"REGISTRY_URL", "username":"REGISTRY_USERNAME", "password":"REGISTRY_PASSWORD"}
+
+  Cephadm will attempt to log in to this registry so it can pull your container
+  and then store the login info in its config database. Other hosts added to
+  the cluster will then also be able to make use of the authenticated registry.
+
+* See :ref:`cephadm-deployment-scenarios` for additional examples for using ``cephadm bootstrap``.
+
+.. _cephadm-enable-cli:
+
+Enable Ceph CLI
+===============
+
+Cephadm does not require any Ceph packages to be installed on the
+host.  However, we recommend enabling easy access to the ``ceph``
+command.  There are several ways to do this:
+
+* The ``cephadm shell`` command launches a bash shell in a container
+  with all of the Ceph packages installed. By default, if
+  configuration and keyring files are found in ``/etc/ceph`` on the
+  host, they are passed into the container environment so that the
+  shell is fully functional. Note that when executed on a MON host,
+  ``cephadm shell`` will infer the ``config`` from the MON container
+  instead of using the default configuration. If ``--mount <path>``
+  is given, then the host ``<path>`` (file or directory) will appear
+  under ``/mnt`` inside the container:
+
+  .. prompt:: bash #
+
+     cephadm shell
+
+* To execute ``ceph`` commands, you can also run commands like this:
+
+  .. prompt:: bash #
+
+     cephadm shell -- ceph -s
+
+* You can install the ``ceph-common`` package, which contains all of the
+  ceph commands, including ``ceph``, ``rbd``, ``mount.ceph`` (for mounting
+  CephFS file systems), etc.:
+
+  .. prompt:: bash #
+     :substitutions:
+
+     cephadm add-repo --release |stable-release|
+     cephadm install ceph-common
+
+Confirm that the ``ceph`` command is accessible with:
+
+.. prompt:: bash #
+
+  ceph -v
+
+
+Confirm that the ``ceph`` command can connect to the cluster and also
+its status with:
+
+.. prompt:: bash #
+
+  ceph status
+
+Adding Hosts
+============
+
+Add all hosts to the cluster by following the instructions in
+:ref:`cephadm-adding-hosts`.
+
+By default, a ``ceph.conf`` file and a copy of the ``client.admin`` keyring are
+maintained in ``/etc/ceph`` on all hosts that have the ``_admin`` label. This
+label is initially applied only to the bootstrap host. We usually recommend
+that one or more other hosts be given the ``_admin`` label so that the Ceph CLI
+(for example, via ``cephadm shell``) is easily accessible on multiple hosts. To add
+the ``_admin`` label to additional host(s), run a command of the following form:
+
+  .. prompt:: bash #
+
+    ceph orch host label add *<host>* _admin
+
+
+Adding additional MONs
+======================
+
+A typical Ceph cluster has three or five monitor daemons spread
+across different hosts.  We recommend deploying five
+monitors if there are five or more nodes in your cluster.
+
+Please follow :ref:`deploy_additional_monitors` to deploy additional MONs.
+
+Adding Storage
+==============
+
+To add storage to the cluster, you can tell Ceph to consume any
+available and unused device(s):
+
+  .. prompt:: bash #
+
+    ceph orch apply osd --all-available-devices
+
+See :ref:`cephadm-deploy-osds` for more detailed instructions.
+
+Enabling OSD memory autotuning
+------------------------------
+
+.. warning:: By default, cephadm enables ``osd_memory_target_autotune`` on bootstrap, with ``mgr/cephadm/autotune_memory_target_ratio`` set to ``.7`` of total host memory.
+
+See :ref:`osd_autotune`.
+
+To deploy hyperconverged Ceph with TripleO, please refer to the TripleO documentation: `Scenario: Deploy Hyperconverged Ceph <https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/features/cephadm.html#scenario-deploy-hyperconverged-ceph>`_
+
+In other cases where the cluster hardware is not exclusively used by Ceph (hyperconverged),
+reduce the memory consumption of Ceph like so:
+
+  .. prompt:: bash #
+
+    # hyperconverged only:
+    ceph config set mgr mgr/cephadm/autotune_memory_target_ratio 0.2
+
+Then enable memory autotuning:
+
+  .. prompt:: bash #
+
+    ceph config set osd osd_memory_target_autotune true
+
+
+Using Ceph
+==========
+
+To use the *Ceph Filesystem*, follow :ref:`orchestrator-cli-cephfs`.
+
+To use the *Ceph Object Gateway*, follow :ref:`cephadm-deploy-rgw`.
+
+To use *NFS*, follow :ref:`deploy-cephadm-nfs-ganesha`
+
+To use *iSCSI*, follow :ref:`cephadm-iscsi`
+
+.. _cephadm-deployment-scenarios:
+
+Different deployment scenarios
+==============================
+
+Single host
+-----------
+
+To configure a Ceph cluster to run on a single host, use the
+``--single-host-defaults`` flag when bootstrapping. For use cases of this, see
+:ref:`one-node-cluster`.
+
+The ``--single-host-defaults`` flag sets the following configuration options::
+
+  global/osd_crush_chooseleaf_type = 0
+  global/osd_pool_default_size = 2
+  mgr/mgr_standby_modules = False
+
+For more information on these options, see :ref:`one-node-cluster` and
+``mgr_standby_modules`` in :ref:`mgr-administrator-guide`.
+
+.. _cephadm-airgap:
+
+Deployment in an isolated environment
+-------------------------------------
+
+You might need to install cephadm in an environment that is not connected
+directly to the internet (such an environment is also called an "isolated
+environment"). This can be done if a custom container registry is used. Either
+of two kinds of custom container registry can be used in this scenario: (1) a
+Podman-based or Docker-based insecure registry, or (2) a secure registry.
+
+The practice of installing software on systems that are not connected directly
+to the internet is called "airgapping" and registries that are not connected
+directly to the internet are referred to as "airgapped".
+
+Make sure that your container image is inside the registry. Make sure that you
+have access to all hosts that you plan to add to the cluster.
+
+#. Run a local container registry:
+
+   .. prompt:: bash #
+
+      podman run --privileged -d --name registry -p 5000:5000 -v /var/lib/registry:/var/lib/registry --restart=always registry:2
+
+#. If you are using an insecure registry, configure Podman or Docker with the
+   hostname and port where the registry is running.
+
+   .. note:: You must repeat this step for every host that accesses the local
+             insecure registry.
+
+#. Push your container image to your local registry. Here are some acceptable
+   kinds of container images:
+
+   * Ceph container image. See :ref:`containers`.
+   * Prometheus container image
+   * Node exporter container image
+   * Grafana container image
+   * Alertmanager container image
+
+#. Create a temporary configuration file to store the names of the monitoring
+   images. (See :ref:`cephadm_monitoring-images`):
+
+   .. prompt:: bash $
+
+      cat <<EOF > initial-ceph.conf
+
+   ::
+
+      [mgr]
+      mgr/cephadm/container_image_prometheus = *<hostname>*:5000/prometheus
+      mgr/cephadm/container_image_node_exporter = *<hostname>*:5000/node_exporter
+      mgr/cephadm/container_image_grafana = *<hostname>*:5000/grafana
+      mgr/cephadm/container_image_alertmanager = *<hostname>*:5000/alertmanger
+
+#. Run bootstrap using the ``--image`` flag and pass the name of your
+   container image as the argument of the image flag. For example:
+
+   .. prompt:: bash #
+
+      cephadm --image *<hostname>*:5000/ceph/ceph bootstrap --mon-ip *<mon-ip>*
+
+.. _cluster network: ../rados/configuration/network-config-ref#cluster-network
+
+.. _cephadm-bootstrap-custom-ssh-keys:
+
+Deployment with custom SSH keys
+-------------------------------
+
+Bootstrap allows users to create their own private/public SSH key pair
+rather than having cephadm generate them automatically.
+
+To use custom SSH keys, pass the ``--ssh-private-key`` and ``--ssh-public-key``
+fields to bootstrap. Both parameters require a path to the file where the
+keys are stored:
+
+.. prompt:: bash #
+
+  cephadm bootstrap --mon-ip <ip-addr> --ssh-private-key <private-key-filepath> --ssh-public-key <public-key-filepath>
+
+This setup allows users to use a key that has already been distributed to hosts
+the user wants in the cluster before bootstrap.
+
+.. note:: In order for cephadm to connect to other hosts you'd like to add
+   to the cluster, make sure the public key of the key pair provided is set up
+   as an authorized key for the ssh user being used, typically root. If you'd
+   like more info on using a non-root user as the ssh user, see :ref:`cephadm-bootstrap-further-info`
+
+.. _cephadm-bootstrap-ca-signed-keys:
+
+Deployment with CA signed SSH keys
+----------------------------------
+
+As an alternative to standard public key authentication, cephadm also supports
+deployment using CA signed keys. Before bootstrapping it's recommended to set up
+the CA public key as a trusted CA key on hosts you'd like to eventually add to
+the cluster. For example:
+
+.. prompt:: bash
+
+  # we will act as our own CA, therefore we'll need to make a CA key
+  [root@host1 ~]# ssh-keygen -t rsa -f ca-key -N ""
+
+  # make the ca key trusted on the host we've generated it on
+  # this requires adding in a line in our /etc/sshd_config
+  # to mark this key as trusted
+  [root@host1 ~]# cp ca-key.pub /etc/ssh
+  [root@host1 ~]# vi /etc/ssh/sshd_config
+  [root@host1 ~]# cat /etc/ssh/sshd_config | grep ca-key
+  TrustedUserCAKeys /etc/ssh/ca-key.pub
+  # now restart sshd so it picks up the config change
+  [root@host1 ~]# systemctl restart sshd
+
+  # now, on all other hosts we want in the cluster, also install the CA key
+  [root@host1 ~]# scp /etc/ssh/ca-key.pub host2:/etc/ssh/
+
+  # on other hosts, make the same changes to the sshd_config
+  [root@host2 ~]# vi /etc/ssh/sshd_config
+  [root@host2 ~]# cat /etc/ssh/sshd_config | grep ca-key
+  TrustedUserCAKeys /etc/ssh/ca-key.pub
+  # and restart sshd so it picks up the config change
+  [root@host2 ~]# systemctl restart sshd
+
+Once the CA key has been installed and marked as a trusted key, you are ready
+to use a private key/CA signed cert combination for SSH. Continuing with our
+current example, we will create a new key-pair for for host access and then
+sign it with our CA key
+
+.. prompt:: bash
+
+  # make a new key pair
+  [root@host1 ~]# ssh-keygen -t rsa -f cephadm-ssh-key -N ""
+  # sign the private key. This will create a new cephadm-ssh-key-cert.pub
+  # note here we're using user "root". If you'd like to use a non-root
+  # user the arguments to the -I and -n params would need to be adjusted
+  # Additionally, note the -V param indicates how long until the cert
+  # this creates will expire
+  [root@host1 ~]# ssh-keygen -s ca-key -I user_root -n root -V +52w cephadm-ssh-key
+  [root@host1 ~]# ls
+  ca-key  ca-key.pub  cephadm-ssh-key  cephadm-ssh-key-cert.pub  cephadm-ssh-key.pub
+
+  # verify our signed key is working. To do this, make sure the generated private
+  # key ("cephadm-ssh-key" in our example) and the newly signed cert are stored
+  # in the same directory. Then try to ssh using the private key
+  [root@host1 ~]# ssh -i cephadm-ssh-key host2
+
+Once you have your private key and corresponding CA signed cert and have tested
+SSH authentication using that key works, you can pass those keys to bootstrap
+in order to have cephadm use them for SSHing between cluster hosts
+
+.. prompt:: bash
+
+  [root@host1 ~]# cephadm bootstrap --mon-ip <ip-addr> --ssh-private-key cephadm-ssh-key --ssh-signed-cert cephadm-ssh-key-cert.pub
+
+Note that this setup does not require installing the corresponding public key
+from the private key passed to bootstrap on other nodes. In fact, cephadm will
+reject the ``--ssh-public-key`` argument when passed along with ``--ssh-signed-cert``.
+Not because having the public key breaks anything, but because it is not at all needed
+for this setup and it helps bootstrap differentiate if the user wants the CA signed
+keys setup or standard pubkey encryption. What this means is, SSH key rotation
+would simply be a matter of getting another key signed by the same CA and providing
+cephadm with the new private key and signed cert. No additional distribution of
+keys to cluster nodes is needed after the initial setup of the CA key as a trusted key,
+no matter how many new private key/signed cert pairs are rotated in.
diff --git a/doc/cephadm/operations.rst b/doc/cephadm/operations.rst
new file mode 100644
index 000000000..5d8fdaca8
--- /dev/null
+++ b/doc/cephadm/operations.rst
@@ -0,0 +1,616 @@
+==================
+Cephadm Operations
+==================
+
+.. _watching_cephadm_logs:
+
+Watching cephadm log messages
+=============================
+
+Cephadm writes logs to the ``cephadm`` cluster log channel. You can
+monitor Ceph's activity in real time by reading the logs as they fill
+up. Run the following command to see the logs in real time:
+
+.. prompt:: bash #
+
+  ceph -W cephadm
+
+By default, this command shows info-level events and above.  To see
+debug-level messages as well as info-level events, run the following
+commands:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/log_to_cluster_level debug
+  ceph -W cephadm --watch-debug
+
+.. warning::
+
+  The debug messages are very verbose!
+
+You can see recent events by running the following command:
+
+.. prompt:: bash #
+
+  ceph log last cephadm
+
+These events are also logged to the ``ceph.cephadm.log`` file on
+monitor hosts as well as to the monitor daemons' stderr.
+
+
+.. _cephadm-logs:
+
+
+Ceph daemon control
+===================
+
+Starting and stopping daemons
+-----------------------------
+
+You can stop, start, or restart a daemon with:
+
+.. prompt:: bash #
+
+   ceph orch daemon stop <name>
+   ceph orch daemon start <name>
+   ceph orch daemon restart <name>
+
+You can also do the same for all daemons for a service with:   
+
+.. prompt:: bash #
+
+   ceph orch stop <name>
+   ceph orch start <name>
+   ceph orch restart <name>
+
+
+Redeploying or reconfiguring a daemon
+-------------------------------------
+
+The container for a daemon can be stopped, recreated, and restarted with
+the ``redeploy`` command:
+
+.. prompt:: bash #
+
+   ceph orch daemon redeploy <name> [--image <image>]
+
+A container image name can optionally be provided to force a
+particular image to be used (instead of the image specified by the
+``container_image`` config value).
+
+If only the ceph configuration needs to be regenerated, you can also
+issue a ``reconfig`` command, which will rewrite the ``ceph.conf``
+file but will not trigger a restart of the daemon.
+
+.. prompt:: bash #
+
+   ceph orch daemon reconfig <name>
+
+
+Rotating a daemon's authenticate key
+------------------------------------
+
+All Ceph and gateway daemons in the cluster have a secret key that is used to connect
+to and authenticate with the cluster.  This key can be rotated (i.e., replaced with a
+new key) with the following command:
+
+.. prompt:: bash #
+
+   ceph orch daemon rotate-key <name>
+
+For MDS, OSD, and MGR daemons, this does not require a daemon restart.  For other
+daemons, however (e.g., RGW), the daemon may be restarted to switch to the new key.
+
+
+Ceph daemon logs
+================
+
+Logging to journald
+-------------------
+
+Ceph daemons traditionally write logs to ``/var/log/ceph``. Ceph daemons log to
+journald by default and Ceph logs are captured by the container runtime
+environment. They are accessible via ``journalctl``.
+
+.. note:: Prior to Quincy, ceph daemons logged to stderr.
+
+Example of logging to journald
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For example, to view the logs for the daemon ``mon.foo`` for a cluster
+with ID ``5c5a50ae-272a-455d-99e9-32c6a013e694``, the command would be
+something like:
+
+.. prompt:: bash #
+
+  journalctl -u ceph-5c5a50ae-272a-455d-99e9-32c6a013e694@mon.foo
+
+This works well for normal operations when logging levels are low.
+
+Logging to files
+----------------
+
+You can also configure Ceph daemons to log to files instead of to
+journald if you prefer logs to appear in files (as they did in earlier,
+pre-cephadm, pre-Octopus versions of Ceph).  When Ceph logs to files,
+the logs appear in ``/var/log/ceph/<cluster-fsid>``. If you choose to
+configure Ceph to log to files instead of to journald, remember to
+configure Ceph so that it will not log to journald (the commands for
+this are covered below).
+
+Enabling logging to files
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To enable logging to files, run the following commands:
+
+.. prompt:: bash #
+
+  ceph config set global log_to_file true
+  ceph config set global mon_cluster_log_to_file true
+
+Disabling logging to journald
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you choose to log to files, we recommend disabling logging to journald or else
+everything will be logged twice. Run the following commands to disable logging
+to stderr:
+
+.. prompt:: bash #
+
+  ceph config set global log_to_stderr false
+  ceph config set global mon_cluster_log_to_stderr false
+  ceph config set global log_to_journald false
+  ceph config set global mon_cluster_log_to_journald false
+
+.. note:: You can change the default by passing --log-to-file during
+   bootstrapping a new cluster.
+
+Modifying the log retention schedule
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cephadm sets up log rotation on each host to rotate these
+files.  You can configure the logging retention schedule by modifying
+``/etc/logrotate.d/ceph.<cluster-fsid>``.
+
+
+Data location
+=============
+
+Cephadm stores daemon data and logs in different locations than did
+older, pre-cephadm (pre Octopus) versions of ceph:
+
+* ``/var/log/ceph/<cluster-fsid>`` contains all cluster logs. By
+  default, cephadm logs via stderr and the container runtime. These
+  logs will not exist unless you have enabled logging to files as
+  described in `cephadm-logs`_.
+* ``/var/lib/ceph/<cluster-fsid>`` contains all cluster daemon data
+  (besides logs).
+* ``/var/lib/ceph/<cluster-fsid>/<daemon-name>`` contains all data for
+  an individual daemon.
+* ``/var/lib/ceph/<cluster-fsid>/crash`` contains crash reports for
+  the cluster.
+* ``/var/lib/ceph/<cluster-fsid>/removed`` contains old daemon
+  data directories for stateful daemons (e.g., monitor, prometheus)
+  that have been removed by cephadm.
+
+Disk usage
+----------
+
+Because a few Ceph daemons (notably, the monitors and prometheus) store a
+large amount of data in ``/var/lib/ceph`` , we recommend moving this
+directory to its own disk, partition, or logical volume so that it does not
+fill up the root file system.
+
+
+Health checks
+=============
+The cephadm module provides additional health checks to supplement the
+default health checks provided by the Cluster. These additional health
+checks fall into two categories:
+
+- **cephadm operations**: Health checks in this category are always
+  executed when the cephadm module is active.
+- **cluster configuration**: These health checks are *optional*, and
+  focus on the configuration of the hosts in the cluster.
+
+CEPHADM Operations
+------------------
+
+CEPHADM_PAUSED
+~~~~~~~~~~~~~~
+
+This indicates that cephadm background work has been paused with
+``ceph orch pause``.  Cephadm continues to perform passive monitoring
+activities (like checking host and daemon status), but it will not
+make any changes (like deploying or removing daemons).
+
+Resume cephadm work by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch resume
+
+.. _cephadm-stray-host:
+
+CEPHADM_STRAY_HOST
+~~~~~~~~~~~~~~~~~~
+
+This indicates that one or more hosts have Ceph daemons that are
+running, but are not registered as hosts managed by *cephadm*.  This
+means that those services cannot currently be managed by cephadm
+(e.g., restarted, upgraded, included in `ceph orch ps`).
+
+* You can manage the host(s) by running the following command:
+
+  .. prompt:: bash #
+
+    ceph orch host add *<hostname>*
+
+  .. note::
+
+    You might need to configure SSH access to the remote host
+    before this will work.
+
+* See :ref:`cephadm-fqdn` for more information about host names and
+  domain names.
+
+* Alternatively, you can manually connect to the host and ensure that
+  services on that host are removed or migrated to a host that is
+  managed by *cephadm*.
+
+* This warning can be disabled entirely by running the following
+  command:
+
+  .. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/warn_on_stray_hosts false
+
+CEPHADM_STRAY_DAEMON
+~~~~~~~~~~~~~~~~~~~~
+
+One or more Ceph daemons are running but not are not managed by
+*cephadm*.  This may be because they were deployed using a different
+tool, or because they were started manually.  Those
+services cannot currently be managed by cephadm (e.g., restarted,
+upgraded, or included in `ceph orch ps`).
+
+* If the daemon is a stateful one (monitor or OSD), it should be adopted
+  by cephadm; see :ref:`cephadm-adoption`.  For stateless daemons, it is
+  usually easiest to provision a new daemon with the ``ceph orch apply``
+  command and then stop the unmanaged daemon.
+
+* If the stray daemon(s) are running on hosts not managed by cephadm, you can manage the host(s) by running the following command:
+
+  .. prompt:: bash #
+
+    ceph orch host add *<hostname>*
+
+  .. note::
+
+    You might need to configure SSH access to the remote host
+    before this will work.
+
+* See :ref:`cephadm-fqdn` for more information about host names and
+  domain names.
+
+* This warning can be disabled entirely by running the following command:
+
+  .. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/warn_on_stray_daemons false
+
+CEPHADM_HOST_CHECK_FAILED
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One or more hosts have failed the basic cephadm host check, which verifies
+that (1) the host is reachable and cephadm can be executed there, and (2)
+that the host satisfies basic prerequisites, like a working container
+runtime (podman or docker) and working time synchronization.
+If this test fails, cephadm will no be able to manage services on that host.
+
+You can manually run this check by running the following command:
+
+.. prompt:: bash #
+
+  ceph cephadm check-host *<hostname>*
+
+You can remove a broken host from management by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch host rm *<hostname>*
+
+You can disable this health warning by running the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/warn_on_failed_host_check false
+
+Cluster Configuration Checks
+----------------------------
+Cephadm periodically scans each of the hosts in the cluster in order
+to understand the state of the OS, disks, NICs etc. These facts can
+then be analysed for consistency across the hosts in the cluster to
+identify any configuration anomalies.
+
+Enabling Cluster Configuration Checks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks are an **optional** feature, and are enabled
+by running the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/config_checks_enabled true
+
+States Returned by Cluster Configuration Checks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks are triggered after each host scan (1m). The
+cephadm log entries will show the current state and outcome of the
+configuration checks as follows:
+
+Disabled state (config_checks_enabled false):
+
+.. code-block:: bash 
+
+  ALL cephadm checks are disabled, use 'ceph config set mgr mgr/cephadm/config_checks_enabled true' to enable
+
+Enabled state (config_checks_enabled true):
+
+.. code-block:: bash 
+
+  CEPHADM 8/8 checks enabled and executed (0 bypassed, 0 disabled). No issues detected
+
+Managing Configuration Checks (subcommands)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks themselves are managed through several cephadm subcommands.
+
+To determine whether the configuration checks are enabled, run the following command:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check status
+
+This command returns the status of the configuration checker as either "Enabled" or "Disabled".
+
+
+To list all the configuration checks and their current states, run the following command:
+
+.. code-block:: console
+
+  # ceph cephadm config-check ls
+
+    NAME             HEALTHCHECK                      STATUS   DESCRIPTION
+  kernel_security  CEPHADM_CHECK_KERNEL_LSM         enabled  checks SELINUX/Apparmor profiles are consistent across cluster hosts
+  os_subscription  CEPHADM_CHECK_SUBSCRIPTION       enabled  checks subscription states are consistent for all cluster hosts
+  public_network   CEPHADM_CHECK_PUBLIC_MEMBERSHIP  enabled  check that all hosts have a NIC on the Ceph public_network
+  osd_mtu_size     CEPHADM_CHECK_MTU                enabled  check that OSD hosts share a common MTU setting
+  osd_linkspeed    CEPHADM_CHECK_LINKSPEED          enabled  check that OSD hosts share a common linkspeed
+  network_missing  CEPHADM_CHECK_NETWORK_MISSING    enabled  checks that the cluster/public networks defined exist on the Ceph hosts
+  ceph_release     CEPHADM_CHECK_CEPH_RELEASE       enabled  check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active)
+  kernel_version   CEPHADM_CHECK_KERNEL_VERSION     enabled  checks that the MAJ.MIN of the kernel on Ceph hosts is consistent
+
+The name of each configuration check can be used to enable or disable a specific check by running a command of the following form:
+:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check disable <name>
+
+For example:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check disable kernel_security
+
+CEPHADM_CHECK_KERNEL_LSM
+~~~~~~~~~~~~~~~~~~~~~~~~
+Each host within the cluster is expected to operate within the same Linux
+Security Module (LSM) state. For example, if the majority of the hosts are
+running with SELINUX in enforcing mode, any host not running in this mode is
+flagged as an anomaly and a healthcheck (WARNING) state raised.
+
+CEPHADM_CHECK_SUBSCRIPTION
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+This check relates to the status of vendor subscription. This check is
+performed only for hosts using RHEL, but helps to confirm that all hosts are
+covered by an active subscription, which ensures that patches and updates are
+available.
+
+CEPHADM_CHECK_PUBLIC_MEMBERSHIP
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+All members of the cluster should have NICs configured on at least one of the
+public network subnets. Hosts that are not on the public network will rely on
+routing, which may affect performance.
+
+CEPHADM_CHECK_MTU
+~~~~~~~~~~~~~~~~~
+The MTU of the NICs on OSDs can be a key factor in consistent performance. This
+check examines hosts that are running OSD services to ensure that the MTU is
+configured consistently within the cluster. This is determined by establishing
+the MTU setting that the majority of hosts is using. Any anomalies result in a
+Ceph health check.
+
+CEPHADM_CHECK_LINKSPEED
+~~~~~~~~~~~~~~~~~~~~~~~
+This check is similar to the MTU check. Linkspeed consistency is a factor in
+consistent cluster performance, just as the MTU of the NICs on the OSDs is.
+This check determines the linkspeed shared by the majority of OSD hosts, and a
+health check is run for any hosts that are set at a lower linkspeed rate.
+
+CEPHADM_CHECK_NETWORK_MISSING
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The `public_network` and `cluster_network` settings support subnet definitions
+for IPv4 and IPv6. If these settings are not found on any host in the cluster,
+a health check is raised.
+
+CEPHADM_CHECK_CEPH_RELEASE
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Under normal operations, the Ceph cluster runs daemons under the same ceph
+release (that is, the Ceph cluster runs all daemons under (for example)
+Octopus).  This check determines the active release for each daemon, and
+reports any anomalies as a healthcheck. *This check is bypassed if an upgrade
+process is active within the cluster.*
+
+CEPHADM_CHECK_KERNEL_VERSION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The OS kernel version (maj.min) is checked for consistency across the hosts.
+The kernel version of the majority of the hosts is used as the basis for
+identifying anomalies.
+
+.. _client_keyrings_and_configs:
+
+Client keyrings and configs
+===========================
+Cephadm can distribute copies of the ``ceph.conf`` file and client keyring
+files to hosts. Starting from versions 16.2.10 (Pacific) and 17.2.1 (Quincy),
+in addition to the default location ``/etc/ceph/`` cephadm also stores config
+and keyring files in the ``/var/lib/ceph/<fsid>/config`` directory. It is usually
+a good idea to store a copy of the config and ``client.admin`` keyring on any host
+used to administer the cluster via the CLI. By default, cephadm does this for any
+nodes that have the ``_admin`` label (which normally includes the bootstrap host).
+
+.. note:: Ceph daemons will still use files on ``/etc/ceph/``. The new configuration
+   location ``/var/lib/ceph/<fsid>/config`` is used by cephadm only. Having this config
+   directory under the fsid helps cephadm to load the configuration associated with
+   the cluster.
+
+
+When a client keyring is placed under management, cephadm will:
+
+  - build a list of target hosts based on the specified placement spec (see
+    :ref:`orchestrator-cli-placement-spec`)
+  - store a copy of the ``/etc/ceph/ceph.conf`` file on the specified host(s)
+  - store a copy of the ``ceph.conf`` file at ``/var/lib/ceph/<fsid>/config/ceph.conf`` on the specified host(s)
+  - store a copy of the ``ceph.client.admin.keyring`` file at ``/var/lib/ceph/<fsid>/config/ceph.client.admin.keyring`` on the specified host(s)
+  - store a copy of the keyring file on the specified host(s)
+  - update the ``ceph.conf`` file as needed (e.g., due to a change in the cluster monitors)
+  - update the keyring file if the entity's key is changed (e.g., via ``ceph
+    auth ...`` commands)
+  - ensure that the keyring file has the specified ownership and specified mode
+  - remove the keyring file when client keyring management is disabled
+  - remove the keyring file from old hosts if the keyring placement spec is
+    updated (as needed)
+
+Listing Client Keyrings
+-----------------------
+
+To see the list of client keyrings are currently under management, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch client-keyring ls
+
+Putting a Keyring Under Management
+----------------------------------
+
+To put a keyring under management, run a command of the following form: 
+
+.. prompt:: bash #
+
+  ceph orch client-keyring set <entity> <placement> [--mode=<mode>] [--owner=<uid>.<gid>] [--path=<path>]
+
+- By default, the *path* is ``/etc/ceph/client.{entity}.keyring``, which is
+  where Ceph looks by default.  Be careful when specifying alternate locations,
+  as existing files may be overwritten.
+- A placement of ``*`` (all hosts) is common.
+- The mode defaults to ``0600`` and ownership to ``0:0`` (user root, group root).
+
+For example, to create a ``client.rbd`` key and deploy it to hosts with the
+``rbd-client`` label and make it group readable by uid/gid 107 (qemu), run the
+following commands:
+
+.. prompt:: bash #
+
+  ceph auth get-or-create-key client.rbd mon 'profile rbd' mgr 'profile rbd' osd 'profile rbd pool=my_rbd_pool'
+  ceph orch client-keyring set client.rbd label:rbd-client --owner 107:107 --mode 640
+
+The resulting keyring file is:
+
+.. code-block:: console
+
+  -rw-r-----. 1 qemu qemu 156 Apr 21 08:47 /etc/ceph/client.client.rbd.keyring
+
+Disabling Management of a Keyring File
+--------------------------------------
+
+To disable management of a keyring file, run a command of the following form:
+
+.. prompt:: bash #
+
+  ceph orch client-keyring rm <entity>
+
+.. note::
+
+  This deletes any keyring files for this entity that were previously written
+  to cluster nodes.
+
+.. _etc_ceph_conf_distribution:
+
+/etc/ceph/ceph.conf
+===================
+
+Distributing ceph.conf to hosts that have no keyrings
+-----------------------------------------------------
+
+It might be useful to distribute ``ceph.conf`` files to hosts without an
+associated client keyring file.  By default, cephadm deploys only a
+``ceph.conf`` file to hosts where a client keyring is also distributed (see
+above).  To write config files to hosts without client keyrings, run the
+following command:
+
+.. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true
+
+Using Placement Specs to specify which hosts get keyrings
+---------------------------------------------------------
+
+By default, the configs are written to all hosts (i.e., those listed by ``ceph
+orch host ls``).  To specify which hosts get a ``ceph.conf``, run a command of
+the following form:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts <placement spec>
+
+For example, to distribute configs to hosts with the ``bare_config`` label, run
+the following command:
+
+Distributing ceph.conf to hosts tagged with bare_config 
+-------------------------------------------------------
+
+For example, to distribute configs to hosts with the ``bare_config`` label, run the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts label:bare_config
+
+(See :ref:`orchestrator-cli-placement-spec` for more information about placement specs.)
+
+Purging a cluster
+=================
+
+.. danger:: THIS OPERATION WILL DESTROY ALL DATA STORED IN THIS CLUSTER
+
+In order to destroy a cluster and delete all data stored in this cluster, disable
+cephadm to stop all orchestration operations (so we avoid deploying new daemons).
+
+.. prompt:: bash #
+
+  ceph mgr module disable cephadm
+
+Then verify the FSID of the cluster:
+
+.. prompt:: bash #
+
+  ceph fsid
+
+Purge ceph daemons from all hosts in the cluster
+
+.. prompt:: bash #
+
+  # For each host:
+  cephadm rm-cluster --force --zap-osds --fsid <fsid>
diff --git a/doc/cephadm/services/custom-container.rst b/doc/cephadm/services/custom-container.rst
new file mode 100644
index 000000000..3ece248c5
--- /dev/null
+++ b/doc/cephadm/services/custom-container.rst
@@ -0,0 +1,79 @@
+========================
+Custom Container Service
+========================
+
+The orchestrator enables custom containers to be deployed using a YAML file.
+A corresponding :ref:`orchestrator-cli-service-spec` must look like:
+
+.. code-block:: yaml
+
+    service_type: container
+    service_id: foo
+    placement:
+        ...
+    spec:
+      image: docker.io/library/foo:latest
+      entrypoint: /usr/bin/foo
+      uid: 1000
+      gid: 1000
+      args:
+        - "--net=host"
+        - "--cpus=2"
+      ports:
+        - 8080
+        - 8443
+      envs:
+        - SECRET=mypassword
+        - PORT=8080
+        - PUID=1000
+        - PGID=1000
+      volume_mounts:
+        CONFIG_DIR: /etc/foo
+      bind_mounts:
+        - ['type=bind', 'source=lib/modules', 'destination=/lib/modules', 'ro=true']
+      dirs:
+        - CONFIG_DIR
+      files:
+        CONFIG_DIR/foo.conf:
+          - refresh=true
+          - username=xyz
+          - "port: 1234"
+
+where the properties of a service specification are:
+
+* ``service_id``
+    A unique name of the service.
+* ``image``
+    The name of the Docker image.
+* ``uid``
+    The UID to use when creating directories and files in the host system.
+* ``gid``
+    The GID to use when creating directories and files in the host system.
+* ``entrypoint``
+    Overwrite the default ENTRYPOINT of the image.
+* ``args``
+    A list of additional Podman/Docker command line arguments.
+* ``ports``
+    A list of TCP ports to open in the host firewall.
+* ``envs``
+    A list of environment variables.
+* ``bind_mounts``
+    When you use a bind mount, a file or directory on the host machine
+    is mounted into the container. Relative `source=...` paths will be
+    located below `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``volume_mounts``
+    When you use a volume mount, a new directory is created within
+    Docker’s storage directory on the host machine, and Docker manages
+    that directory’s contents. Relative source paths will be located below
+    `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``dirs``
+    A list of directories that are created below
+    `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``files``
+    A dictionary, where the key is the relative path of the file and the
+    value the file content. The content must be double quoted when using
+    a string. Use '\\n' for line breaks in that case. Otherwise define
+    multi-line content as list of strings. The given files will be created
+    below the directory `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+    The absolute path of the directory where the file will be created must
+    exist. Use the `dirs` property to create them if necessary.
diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst
new file mode 100644
index 000000000..82f83bfac
--- /dev/null
+++ b/doc/cephadm/services/index.rst
@@ -0,0 +1,871 @@
+==================
+Service Management
+==================
+
+A service is a group of daemons configured together. See these chapters
+for details on individual services:
+
+.. toctree::
+    :maxdepth: 1
+
+    mon
+    mgr
+    osd
+    rgw
+    mds
+    nfs
+    iscsi
+    custom-container
+    monitoring
+    snmp-gateway
+    tracing
+
+Service Status
+==============
+
+
+To see the status of one
+of the services running in the Ceph cluster, do the following:
+
+#. Use the command line to print a list of services.
+#. Locate the service whose status you want to check.
+#. Print the status of the service.
+
+The following command prints a list of services known to the orchestrator. To
+limit the output to services only on a specified host, use the optional
+``--host`` parameter. To limit the output to services of only a particular
+type, use the optional ``--type`` parameter (mon, osd, mgr, mds, rgw):
+
+   .. prompt:: bash #
+
+     ceph orch ls [--service_type type] [--service_name name] [--export] [--format f] [--refresh]
+
+Discover the status of a particular service or daemon:
+
+   .. prompt:: bash #
+
+     ceph orch ls --service_type type --service_name <name> [--refresh]
+
+To export the service specifications knows to the orchestrator, run the following command.
+
+   .. prompt:: bash #
+
+     ceph orch ls --export
+
+The service specifications exported with this command will be exported as yaml
+and that yaml can be used with the ``ceph orch apply -i`` command.
+
+For information about retrieving the specifications of single services (including examples of commands), see :ref:`orchestrator-cli-service-spec-retrieve`.
+
+Daemon Status
+=============
+
+A daemon is a systemd unit that is running and part of a service.
+
+To see the status of a daemon, do the following:
+
+#. Print a list of all daemons known to the orchestrator.
+#. Query the status of the target daemon.
+
+First, print a list of all daemons known to the orchestrator:
+
+   .. prompt:: bash #
+
+    ceph orch ps [--hostname host] [--daemon_type type] [--service_name name] [--daemon_id id] [--format f] [--refresh]
+
+Then query the status of a particular service instance (mon, osd, mds, rgw).
+For OSDs the id is the numeric OSD ID. For MDS services the id is the file
+system name:
+
+   .. prompt:: bash #
+
+    ceph orch ps --daemon_type osd --daemon_id 0
+
+.. note::
+   The output of the command ``ceph orch ps`` may not reflect the current status of the daemons. By default,
+   the status is updated every 10 minutes. This interval can be shortened by modifying the ``mgr/cephadm/daemon_cache_timeout``
+   configuration variable (in seconds) e.g: ``ceph config set mgr mgr/cephadm/daemon_cache_timeout 60`` would reduce the refresh
+   interval to one minute. The information is updated every ``daemon_cache_timeout`` seconds unless the ``--refresh`` option
+   is used. This option would trigger a request to refresh the information, which may take some time depending on the size of
+   the cluster. In general ``REFRESHED`` value indicates how recent the information displayed by ``ceph orch ps`` and similar
+   commands is.
+
+.. _orchestrator-cli-service-spec:
+
+Service Specification
+=====================
+
+A *Service Specification* is a data structure that is used to specify the
+deployment of services. In addition to parameters such as `placement` or
+`networks`, the user can set initial values of service configuration parameters
+by means of the `config` section. For each param/value configuration pair,
+cephadm calls the following command to set its value:
+
+   .. prompt:: bash #
+
+    ceph config set <service-name> <param> <value>
+
+cephadm raises health warnings in case invalid configuration parameters are
+found in the spec (`CEPHADM_INVALID_CONFIG_OPTION`) or if any error while
+trying to apply the new configuration option(s) (`CEPHADM_FAILED_SET_OPTION`).
+
+Here is an example of a service specification in YAML:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    service_id: realm.zone
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    config:
+      param_1: val_1
+      ...
+      param_N: val_N
+    unmanaged: false
+    networks:
+    - 192.169.142.0/24
+    spec:
+      # Additional service specific attributes.
+
+In this example, the properties of this service specification are:
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: ServiceSpec
+   :members:
+
+Each service type can have additional service-specific properties.
+
+Service specifications of type ``mon``, ``mgr``, and the monitoring
+types do not require a ``service_id``.
+
+A service of type ``osd`` is described in :ref:`drivegroups`
+
+Many service specifications can be applied at once using ``ceph orch apply -i``
+by submitting a multi-document YAML file::
+
+    cat <<EOF | ceph orch apply -i -
+    service_type: mon
+    placement:
+      host_pattern: "mon*"
+    ---
+    service_type: mgr
+    placement:
+      host_pattern: "mgr*"
+    ---
+    service_type: osd
+    service_id: default_drive_group
+    placement:
+      host_pattern: "osd*"
+    data_devices:
+      all: true
+    EOF
+
+.. _orchestrator-cli-service-spec-retrieve:
+
+Retrieving the running Service Specification
+--------------------------------------------
+
+If the services have been started via ``ceph orch apply...``, then directly changing
+the Services Specification is complicated. Instead of attempting to directly change
+the Services Specification, we suggest exporting the running Service Specification by
+following these instructions:
+
+   .. prompt:: bash #
+
+    ceph orch ls --service-name rgw.<realm>.<zone> --export > rgw.<realm>.<zone>.yaml
+    ceph orch ls --service-type mgr --export > mgr.yaml
+    ceph orch ls --export > cluster.yaml
+
+The Specification can then be changed and re-applied as above.
+
+Updating Service Specifications
+-------------------------------
+
+The Ceph Orchestrator maintains a declarative state of each
+service in a ``ServiceSpec``. For certain operations, like updating
+the RGW HTTP port, we need to update the existing
+specification.
+
+1. List the current ``ServiceSpec``:
+
+   .. prompt:: bash #
+
+    ceph orch ls --service_name=<service-name> --export > myservice.yaml
+
+2. Update the yaml file:
+
+   .. prompt:: bash #
+
+    vi myservice.yaml
+
+3. Apply the new ``ServiceSpec``:
+
+   .. prompt:: bash #
+
+    ceph orch apply -i myservice.yaml [--dry-run]
+
+.. _orchestrator-cli-placement-spec:
+
+Daemon Placement
+================
+
+For the orchestrator to deploy a *service*, it needs to know where to deploy
+*daemons*, and how many to deploy.  This is the role of a placement
+specification.  Placement specifications can either be passed as command line arguments
+or in a YAML files.
+
+.. note::
+
+   cephadm will not deploy daemons on hosts with the ``_no_schedule`` label; see :ref:`cephadm-special-host-labels`.
+
+.. note::
+   The **apply** command can be confusing. For this reason, we recommend using
+   YAML specifications.
+
+   Each ``ceph orch apply <service-name>`` command supersedes the one before it.
+   If you do not use the proper syntax, you will clobber your work
+   as you go.
+
+   For example:
+
+   .. prompt:: bash #
+
+        ceph orch apply mon host1
+        ceph orch apply mon host2
+        ceph orch apply mon host3
+
+   This results in only one host having a monitor applied to it: host 3.
+
+   (The first command creates a monitor on host1. Then the second command
+   clobbers the monitor on host1 and creates a monitor on host2. Then the
+   third command clobbers the monitor on host2 and creates a monitor on
+   host3. In this scenario, at this point, there is a monitor ONLY on
+   host3.)
+
+   To make certain that a monitor is applied to each of these three hosts,
+   run a command like this:
+
+   .. prompt:: bash #
+
+     ceph orch apply mon "host1,host2,host3"
+
+   There is another way to apply monitors to multiple hosts: a ``yaml`` file
+   can be used. Instead of using the "ceph orch apply mon" commands, run a
+   command of this form:
+
+   .. prompt:: bash #
+
+      ceph orch apply -i file.yaml
+
+   Here is a sample **file.yaml** file
+
+   .. code-block:: yaml
+
+        service_type: mon
+        placement:
+          hosts:
+           - host1
+           - host2
+           - host3
+
+Explicit placements
+-------------------
+
+Daemons can be explicitly placed on hosts by simply specifying them:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement="host1 host2 host3"
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+
+MONs and other services may require some enhanced network specifications:
+
+   .. prompt:: bash #
+
+    ceph orch daemon add mon --placement="myhost:[v2:1.2.3.4:3300,v1:1.2.3.4:6789]=name"
+
+where ``[v2:1.2.3.4:3300,v1:1.2.3.4:6789]`` is the network address of the monitor
+and ``=name`` specifies the name of the new monitor.
+
+.. _orch-placement-by-labels:
+
+Placement by labels
+-------------------
+
+Daemon placement can be limited to hosts that match a specific label. To set
+a label ``mylabel`` to the appropriate hosts, run this command:
+
+  .. prompt:: bash #
+
+    ceph orch host label add *<hostname>* mylabel
+
+  To view the current hosts and labels, run this command:
+
+  .. prompt:: bash #
+
+    ceph orch host ls
+
+  For example:
+
+  .. prompt:: bash #
+
+    ceph orch host label add host1 mylabel
+    ceph orch host label add host2 mylabel
+    ceph orch host label add host3 mylabel
+    ceph orch host ls
+
+  .. code-block:: bash
+
+    HOST   ADDR   LABELS  STATUS
+    host1         mylabel
+    host2         mylabel
+    host3         mylabel
+    host4
+    host5
+
+Now, Tell cephadm to deploy daemons based on the label by running
+this command:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement="label:mylabel"
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      label: "mylabel"
+
+* See :ref:`orchestrator-host-labels`
+
+Placement by pattern matching
+-----------------------------
+
+Daemons can be placed on hosts as well:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement='myhost[1-3]'
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      host_pattern: "myhost[1-3]"
+
+To place a service on *all* hosts, use ``"*"``:
+
+   .. prompt:: bash #
+
+    ceph orch apply node-exporter --placement='*'
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: node-exporter
+    placement:
+      host_pattern: "*"
+
+
+Changing the number of daemons
+------------------------------
+
+By specifying ``count``, only the number of daemons specified will be created:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement=3
+
+To deploy *daemons* on a subset of hosts, specify the count:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement="2 host1 host2 host3"
+
+If the count is bigger than the amount of hosts, cephadm deploys one per host:
+
+   .. prompt:: bash #
+
+    ceph orch apply prometheus --placement="3 host1 host2"
+
+The command immediately above results in two Prometheus daemons.
+
+YAML can also be used to specify limits, in the following way:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 3
+
+YAML can also be used to specify limits on hosts:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 2
+      hosts:
+        - host1
+        - host2
+        - host3
+
+.. _cephadm_co_location:
+
+Co-location of daemons
+----------------------
+
+Cephadm supports the deployment of multiple daemons on the same host:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    placement:
+      label: rgw
+      count_per_host: 2
+
+The main reason for deploying multiple daemons per host is an additional
+performance benefit for running multiple RGW and MDS daemons on the same host.
+
+See also: 
+
+* :ref:`cephadm_mgr_co_location`.
+* :ref:`cephadm-rgw-designated_gateways`.
+
+This feature was introduced in Pacific.
+
+Algorithm description
+---------------------
+
+Cephadm's declarative state consists of a list of service specifications
+containing placement specifications.
+
+Cephadm continually compares a list of daemons actually running in the cluster
+against the list in the service specifications. Cephadm adds new daemons and
+removes old daemons as necessary in order to conform to the service
+specifications.
+
+Cephadm does the following to maintain compliance with the service
+specifications.
+
+Cephadm first selects a list of candidate hosts. Cephadm seeks explicit host
+names and selects them. If cephadm finds no explicit host names, it looks for
+label specifications. If no label is defined in the specification, cephadm
+selects hosts based on a host pattern. If no host pattern is defined, as a last
+resort, cephadm selects all known hosts as candidates.
+
+Cephadm is aware of existing daemons running services and tries to avoid moving
+them.
+
+Cephadm supports the deployment of a specific amount of services.
+Consider the following service specification:
+
+.. code-block:: yaml
+
+    service_type: mds
+    service_name: myfs
+    placement:
+      count: 3
+      label: myfs
+
+This service specification instructs cephadm to deploy three daemons on hosts
+labeled ``myfs`` across the cluster.
+
+If there are fewer than three daemons deployed on the candidate hosts, cephadm
+randomly chooses hosts on which to deploy new daemons.
+
+If there are more than three daemons deployed on the candidate hosts, cephadm
+removes existing daemons.
+
+Finally, cephadm removes daemons on hosts that are outside of the list of
+candidate hosts.
+
+.. note::
+
+   There is a special case that cephadm must consider.
+
+   If there are fewer hosts selected by the placement specification than
+   demanded by ``count``, cephadm will deploy only on the selected hosts.
+
+.. _cephadm-extra-container-args:
+
+Extra Container Arguments
+=========================
+
+.. warning:: 
+  The arguments provided for extra container args are limited to whatever arguments are available for
+  a `run` command from whichever container engine you are using. Providing any arguments the `run`
+  command does not support (or invalid values for arguments) will cause the daemon to fail to start.
+
+.. note::
+
+  For arguments passed to the process running inside the container rather than the for
+  the container runtime itself, see :ref:`cephadm-extra-entrypoint-args`
+
+
+Cephadm supports providing extra miscellaneous container arguments for
+specific cases when they may be necessary. For example, if a user needed
+to limit the amount of cpus their mon daemons make use of they could apply
+a spec like
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    extra_container_args:
+      - "--cpus=2"
+
+which would cause each mon daemon to be deployed with `--cpus=2`.
+
+There are two ways to express arguments in the ``extra_container_args`` list.
+To start, an item in the list can be a string. When passing an argument
+as a string and the string contains spaces, Cephadm will automatically split it
+into multiple arguments. For example, ``--cpus 2`` would become ``["--cpus",
+"2"]`` when processed. Example:
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    extra_container_args:
+      - "--cpus 2"
+
+As an alternative, an item in the list can be an object (mapping) containing
+the required key "argument" and an optional key "split". The value associated
+with the ``argument`` key must be a single string. The value associated with
+the ``split`` key is a boolean value. The ``split`` key explicitly controls if
+spaces in the argument value cause the value to be split into multiple
+arguments. If ``split`` is true then Cephadm will automatically split the value
+into multiple arguments.  If ``split`` is false then spaces in the value will
+be retained in the argument.  The default, when ``split`` is not provided, is
+false. Examples:
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      hosts:
+        - tiebreaker
+    extra_container_args:
+      # No spaces, always treated as a single argument
+      - argument: "--timout=3000"
+      # Splitting explicitly disabled, one single argument
+      - argument: "--annotation=com.example.name=my favorite mon"
+        split: false
+      # Splitting explicitly enabled, will become two arguments
+      - argument: "--cpuset-cpus 1-3,7-11"
+        split: true
+      # Splitting implicitly disabled, one single argument
+      - argument: "--annotation=com.example.note=a simple example"
+
+Mounting Files with Extra Container Arguments
+---------------------------------------------
+
+A common use case for extra container arguments is to mount additional
+files within the container. Older versions of Ceph did not support spaces
+in arguments and therefore the examples below apply to the widest range
+of Ceph versions.
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/absolute/file/path/on/host:/absolute/file/path/in/container"
+
+For example:
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/opt/ceph_cert/host.cert:/etc/grafana/certs/cert_file:ro"
+
+.. _cephadm-extra-entrypoint-args:
+
+Extra Entrypoint Arguments
+==========================
+
+.. note::
+
+  For arguments intended for the container runtime rather than the process inside
+  it, see :ref:`cephadm-extra-container-args`
+
+Similar to extra container args for the container runtime, Cephadm supports
+appending to args passed to the entrypoint process running
+within a container. For example, to set the collector textfile directory for
+the node-exporter service , one could apply a service spec like
+
+.. code-block:: yaml
+
+  service_type: node-exporter
+  service_name: node-exporter
+  placement:
+    host_pattern: '*'
+  extra_entrypoint_args:
+    - "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector2"
+
+There are two ways to express arguments in the ``extra_entrypoint_args`` list.
+To start, an item in the list can be a string. When passing an argument as a
+string and the string contains spaces, cephadm will automatically split it into
+multiple arguments. For example, ``--debug_ms 10`` would become
+``["--debug_ms", "10"]`` when processed. Example:
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    extra_entrypoint_args:
+      - "--debug_ms 2"
+
+As an alternative, an item in the list can be an object (mapping) containing
+the required key "argument" and an optional key "split". The value associated
+with the ``argument`` key must be a single string. The value associated with
+the ``split`` key is a boolean value. The ``split`` key explicitly controls if
+spaces in the argument value cause the value to be split into multiple
+arguments. If ``split`` is true then cephadm will automatically split the value
+into multiple arguments.  If ``split`` is false then spaces in the value will
+be retained in the argument.  The default, when ``split`` is not provided, is
+false. Examples:
+
+.. code-block:: yaml
+
+    # An theoretical data migration service
+    service_type: pretend
+    service_name: imagine1
+    placement:
+      hosts:
+        - host1
+    extra_entrypoint_args:
+      # No spaces, always treated as a single argument
+      - argument: "--timout=30m"
+      # Splitting explicitly disabled, one single argument
+      - argument: "--import=/mnt/usb/My Documents"
+        split: false
+      # Splitting explicitly enabled, will become two arguments
+      - argument: "--tag documents"
+        split: true
+      # Splitting implicitly disabled, one single argument
+      - argument: "--title=Imported Documents"
+
+
+Custom Config Files
+===================
+
+Cephadm supports specifying miscellaneous config files for daemons.
+To do so, users must provide both the content of the config file and the
+location within the daemon's container at which it should be mounted. After
+applying a YAML spec with custom config files specified and having cephadm
+redeploy the daemons for which the config files are specified, these files will
+be mounted within the daemon's container at the specified location.
+
+Example service spec:
+
+.. code-block:: yaml
+
+    service_type: grafana
+    service_name: grafana
+    custom_configs:
+      - mount_path: /etc/example.conf
+        content: |
+          setting1 = value1
+          setting2 = value2
+      - mount_path: /usr/share/grafana/example.cert
+        content: |
+          -----BEGIN PRIVATE KEY-----
+          V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+          ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+          IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+          YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+          ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+          -----END PRIVATE KEY-----
+          -----BEGIN CERTIFICATE-----
+          V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+          ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+          IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+          YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+          ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+          -----END CERTIFICATE-----
+
+To make these new config files actually get mounted within the
+containers for the daemons
+
+.. prompt:: bash
+
+  ceph orch redeploy <service-name>
+
+For example:
+
+.. prompt:: bash
+
+  ceph orch redeploy grafana
+
+.. _orch-rm:
+
+Removing a Service
+==================
+
+In order to remove a service including the removal
+of all daemons of that service, run
+
+.. prompt:: bash
+
+  ceph orch rm <service-name>
+
+For example:
+
+.. prompt:: bash
+
+  ceph orch rm rgw.myrgw
+
+.. _cephadm-spec-unmanaged:
+
+Disabling automatic deployment of daemons
+=========================================
+
+Cephadm supports disabling the automated deployment and removal of daemons on a
+per service basis. The CLI supports two commands for this.
+
+In order to fully remove a service, see :ref:`orch-rm`.
+
+Disabling automatic management of daemons
+-----------------------------------------
+
+To disable the automatic management of dameons, set ``unmanaged=True`` in the
+:ref:`orchestrator-cli-service-spec` (``mgr.yaml``).
+
+``mgr.yaml``:
+
+.. code-block:: yaml
+
+  service_type: mgr
+  unmanaged: true
+  placement:
+    label: mgr
+
+
+.. prompt:: bash #
+
+   ceph orch apply -i mgr.yaml
+
+Cephadm also supports setting the unmanaged parameter to true or false
+using the ``ceph orch set-unmanaged`` and ``ceph orch set-managed`` commands.
+The commands take the service name (as reported in ``ceph orch ls``) as
+the only argument. For example,
+
+.. prompt:: bash #
+
+   ceph orch set-unmanaged mon
+
+would set ``unmanaged: true`` for the mon service and
+
+.. prompt:: bash #
+
+   ceph orch set-managed mon
+
+would set ``unmanaged: false`` for the mon service
+
+.. note::
+
+  After you apply this change in the Service Specification, cephadm will no
+  longer deploy any new daemons (even if the placement specification matches
+  additional hosts).
+
+.. note::
+
+  The "osd" service used to track OSDs that are not tied to any specific
+  service spec is special and will always be marked unmanaged. Attempting
+  to modify it with ``ceph orch set-unmanaged`` or ``ceph orch set-managed``
+  will result in a message ``No service of name osd found. Check "ceph orch ls" for all known services``
+
+Deploying a daemon on a host manually
+-------------------------------------
+
+.. note::
+
+  This workflow has a very limited use case and should only be used
+  in rare circumstances.
+
+To manually deploy a daemon on a host, follow these steps:
+
+Modify the service spec for a service by getting the
+existing spec, adding ``unmanaged: true``, and applying the modified spec.
+
+Then manually deploy the daemon using the following:
+
+   .. prompt:: bash #
+
+     ceph orch daemon add <daemon-type>  --placement=<placement spec>
+
+For example :
+
+   .. prompt:: bash #
+
+     ceph orch daemon add mgr --placement=my_host
+
+.. note::
+
+  Removing ``unmanaged: true`` from the service spec will
+  enable the reconciliation loop for this service and will
+  potentially lead to the removal of the daemon, depending
+  on the placement spec.
+
+Removing a daemon from a host manually
+--------------------------------------
+
+To manually remove a daemon, run a command of the following form:
+
+   .. prompt:: bash #
+
+     ceph orch daemon rm <daemon name>... [--force]
+
+For example:
+
+   .. prompt:: bash #
+
+     ceph orch daemon rm mgr.my_host.xyzxyz
+
+.. note::
+
+  For managed services (``unmanaged=False``), cephadm will automatically
+  deploy a new daemon a few seconds later.
+
+See also
+--------
+
+* See :ref:`cephadm-osd-declarative` for special handling of unmanaged OSDs.
+* See also :ref:`cephadm-pause`
diff --git a/doc/cephadm/services/iscsi.rst b/doc/cephadm/services/iscsi.rst
new file mode 100644
index 000000000..bb2de7921
--- /dev/null
+++ b/doc/cephadm/services/iscsi.rst
@@ -0,0 +1,87 @@
+=============
+iSCSI Service
+=============
+
+.. _cephadm-iscsi:
+
+Deploying iSCSI
+===============
+
+To deploy an iSCSI gateway, create a yaml file containing a
+service specification for iscsi:
+
+.. code-block:: yaml
+
+    service_type: iscsi
+    service_id: iscsi
+    placement:
+      hosts:
+        - host1
+        - host2
+    spec:
+      pool: mypool  # RADOS pool where ceph-iscsi config data is stored.
+      trusted_ip_list: "IP_ADDRESS_1,IP_ADDRESS_2"
+      api_port: ... # optional
+      api_user: ... # optional
+      api_password: ... # optional
+      api_secure: true/false # optional
+      ssl_cert: | # optional
+        ...
+      ssl_key: | # optional
+        ...
+
+For example:
+
+.. code-block:: yaml
+
+    service_type: iscsi
+    service_id: iscsi
+    placement:
+      hosts:
+      - [...]
+    spec:
+      pool: iscsi_pool
+      trusted_ip_list: "IP_ADDRESS_1,IP_ADDRESS_2,IP_ADDRESS_3,..."
+      api_user: API_USERNAME
+      api_password: API_PASSWORD
+      ssl_cert: |
+        -----BEGIN CERTIFICATE-----
+        MIIDtTCCAp2gAwIBAgIYMC4xNzc1NDQxNjEzMzc2MjMyXzxvQ7EcMA0GCSqGSIb3
+        DQEBCwUAMG0xCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRcwFQYDVQQHDA5T
+        [...]
+        -----END CERTIFICATE-----
+      ssl_key: |
+        -----BEGIN PRIVATE KEY-----
+        MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5jdYbjtNTAKW4
+        /CwQr/7wOiLGzVxChn3mmCIF3DwbL/qvTFTX2d8bDf6LjGwLYloXHscRfxszX/4h
+        [...]
+        -----END PRIVATE KEY-----
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: IscsiServiceSpec
+   :members:
+
+
+The specification can then be applied using:
+
+.. prompt:: bash #
+
+   ceph orch apply -i iscsi.yaml
+
+
+See :ref:`orchestrator-cli-placement-spec` for details of the placement specification.
+
+See also: :ref:`orchestrator-cli-service-spec`.
+
+Configuring iSCSI client
+========================
+
+The containerized iscsi service can be used from any host by
+:ref:`configuring-the-iscsi-initiators`, which will use TCP/IP to send SCSI
+commands to the iSCSI target (gateway).
+
+Further Reading
+===============
+
+* Ceph iSCSI Overview: :ref:`ceph-iscsi`
diff --git a/doc/cephadm/services/mds.rst b/doc/cephadm/services/mds.rst
new file mode 100644
index 000000000..96b7c2dda
--- /dev/null
+++ b/doc/cephadm/services/mds.rst
@@ -0,0 +1,61 @@
+===========
+MDS Service
+===========
+
+
+.. _orchestrator-cli-cephfs:
+
+Deploy CephFS
+=============
+
+One or more MDS daemons is required to use the :term:`CephFS` file system.
+These are created automatically if the newer ``ceph fs volume``
+interface is used to create a new file system. For more information,
+see :ref:`fs-volumes-and-subvolumes`.
+
+For example:
+
+.. prompt:: bash #
+
+  ceph fs volume create <fs_name> --placement="<placement spec>"
+
+where ``fs_name`` is the name of the CephFS and ``placement`` is a
+:ref:`orchestrator-cli-placement-spec`. For example, to place
+MDS daemons for the new ``foo`` volume on hosts labeled with ``mds``:
+
+.. prompt:: bash #
+
+  ceph fs volume create foo --placement="label:mds"
+
+You can also update the placement after-the-fact via:
+
+.. prompt:: bash #
+
+  ceph orch apply mds foo 'mds-[012]'
+
+For manually deploying MDS daemons, use this specification:
+
+.. code-block:: yaml
+
+    service_type: mds
+    service_id: fs_name
+    placement:
+      count: 3
+      label: mds
+
+
+The specification can then be applied using:
+
+.. prompt:: bash #
+
+   ceph orch apply -i mds.yaml
+
+See :ref:`orchestrator-cli-stateless-services` for manually deploying
+MDS daemons on the CLI.
+
+Further Reading
+===============
+
+* :ref:`ceph-file-system`
+
+
diff --git a/doc/cephadm/services/mgr.rst b/doc/cephadm/services/mgr.rst
new file mode 100644
index 000000000..9baff3a7a
--- /dev/null
+++ b/doc/cephadm/services/mgr.rst
@@ -0,0 +1,43 @@
+.. _mgr-cephadm-mgr:
+
+===========
+MGR Service
+===========
+
+The cephadm MGR service hosts multiple modules. These include the
+:ref:`mgr-dashboard` and the cephadm manager module.
+
+.. _cephadm-mgr-networks:
+
+Specifying Networks
+-------------------
+
+The MGR service supports binding only to a specific IP within a network.
+
+example spec file (leveraging a default placement):
+
+.. code-block:: yaml
+
+    service_type: mgr
+    networks:
+    - 192.169.142.0/24
+
+.. _cephadm_mgr_co_location:
+
+Allow co-location of MGR daemons
+================================
+
+In deployment scenarios with just a single host, cephadm still needs
+to deploy at least two MGR daemons in order to allow an automated
+upgrade of the cluster. See ``mgr_standby_modules`` in
+the :ref:`mgr-administrator-guide` for further details.
+
+See also: :ref:`cephadm_co_location`.
+
+
+Further Reading
+===============
+
+* :ref:`ceph-manager-daemon`
+* :ref:`cephadm-manually-deploy-mgr`
+
diff --git a/doc/cephadm/services/mon.rst b/doc/cephadm/services/mon.rst
new file mode 100644
index 000000000..389dc450e
--- /dev/null
+++ b/doc/cephadm/services/mon.rst
@@ -0,0 +1,237 @@
+===========
+MON Service
+===========
+
+.. _deploy_additional_monitors:
+
+Deploying additional monitors 
+=============================
+
+A typical Ceph cluster has three or five monitor daemons that are spread
+across different hosts.  We recommend deploying five monitors if there are
+five or more nodes in your cluster.
+
+.. _CIDR: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation
+
+Ceph deploys monitor daemons automatically as the cluster grows and Ceph
+scales back monitor daemons automatically as the cluster shrinks. The
+smooth execution of this automatic growing and shrinking depends upon
+proper subnet configuration.
+
+The cephadm bootstrap procedure assigns the first monitor daemon in the
+cluster to a particular subnet. ``cephadm`` designates that subnet as the
+default subnet of the cluster. New monitor daemons will be assigned by
+default to that subnet unless cephadm is instructed to do otherwise. 
+
+If all of the ceph monitor daemons in your cluster are in the same subnet,
+manual administration of the ceph monitor daemons is not necessary.
+``cephadm`` will automatically add up to five monitors to the subnet, as
+needed, as new hosts are added to the cluster.
+
+By default, cephadm will deploy 5 daemons on arbitrary hosts. See
+:ref:`orchestrator-cli-placement-spec` for details of specifying
+the placement of daemons.
+
+Designating a Particular Subnet for Monitors
+--------------------------------------------
+
+To designate a particular IP subnet for use by ceph monitor daemons, use a
+command of the following form, including the subnet's address in `CIDR`_
+format (e.g., ``10.1.2.0/24``):
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24
+
+Cephadm deploys new monitor daemons only on hosts that have IP addresses in
+the designated subnet.
+
+You can also specify two public networks by using a list of networks:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network1>,<mon-cidr-network2>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24,192.168.0.1/24
+
+
+Deploying Monitors on a Particular Network 
+------------------------------------------
+
+You can explicitly specify the IP address or CIDR network for each monitor and
+control where each monitor is placed.  To disable automated monitor deployment,
+run this command:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+
+  To deploy each additional monitor:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add mon *<host1:ip-or-network1>
+
+  For example, to deploy a second monitor on ``newhost1`` using an IP
+  address ``10.1.2.123`` and a third monitor on ``newhost2`` in
+  network ``10.1.2.0/24``, run the following commands:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+    ceph orch daemon add mon newhost1:10.1.2.123
+    ceph orch daemon add mon newhost2:10.1.2.0/24
+
+  Now, enable automatic placement of Daemons
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run
+
+  See :ref:`orchestrator-cli-placement-spec` for details of specifying
+  the placement of daemons.
+
+  Finally apply this new placement by dropping ``--dry-run``
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3"
+
+
+Moving Monitors to a Different Network
+--------------------------------------
+
+To move Monitors to a new network, deploy new monitors on the new network and
+subsequently remove monitors from the old network. It is not advised to
+modify and inject the ``monmap`` manually.
+
+First, disable the automated placement of daemons:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+
+To deploy each additional monitor:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add mon *<newhost1:ip-or-network1>*
+
+For example, to deploy a second monitor on ``newhost1`` using an IP
+address ``10.1.2.123`` and a third monitor on ``newhost2`` in
+network ``10.1.2.0/24``, run the following commands:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+    ceph orch daemon add mon newhost1:10.1.2.123
+    ceph orch daemon add mon newhost2:10.1.2.0/24
+
+  Subsequently remove monitors from the old network:
+
+  .. prompt:: bash #
+
+    ceph orch daemon rm *mon.<oldhost1>*
+
+  Update the ``public_network``:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24
+
+  Now, enable automatic placement of Daemons
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run
+
+  See :ref:`orchestrator-cli-placement-spec` for details of specifying
+  the placement of daemons.
+
+  Finally apply this new placement by dropping ``--dry-run``
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" 
+
+
+Setting Crush Locations for Monitors
+------------------------------------
+
+Cephadm supports setting CRUSH locations for mon daemons
+using the mon service spec. The CRUSH locations are set
+by hostname. When cephadm deploys a mon on a host that matches
+a hostname specified in the CRUSH locations, it will add
+``--set-crush-location <CRUSH-location>`` where the CRUSH location
+is the first entry in the list of CRUSH locations for that
+host. If multiple CRUSH locations are set for one host, cephadm
+will attempt to set the additional locations using the
+"ceph mon set_location" command.
+
+.. note::
+
+   Setting the CRUSH location in the spec is the recommended way of
+   replacing tiebreaker mon daemons, as they require having a location
+   set when they are added.
+
+ .. note::
+
+   Tiebreaker mon daemons are a part of stretch mode clusters. For more
+   info on stretch mode clusters see :ref:`stretch_mode`
+
+Example syntax for setting the CRUSH locations:
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      count: 5
+    spec:
+      crush_locations:
+        host1:
+        - datacenter=a
+        host2:
+        - datacenter=b
+        - rack=2
+        host3:
+        - datacenter=a
+
+.. note::
+
+   Sometimes, based on the timing of mon daemons being admitted to the mon
+   quorum, cephadm may fail to set the CRUSH location for some mon daemons
+   when multiple locations are specified. In this case, the recommended
+   action is to re-apply the same mon spec to retrigger the service action.
+
+.. note::
+
+   Mon daemons will only get the ``--set-crush-location`` flag set when cephadm
+   actually deploys them. This means if a spec is applied that includes a CRUSH
+   location for a mon that is already deployed, the flag may not be set until
+   a redeploy command is issued for that mon daemon.
+
+
+Further Reading
+===============
+
+* :ref:`rados-operations`
+* :ref:`rados-troubleshooting-mon`
+* :ref:`cephadm-restore-quorum`
+
diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst
new file mode 100644
index 000000000..a17a5ba03
--- /dev/null
+++ b/doc/cephadm/services/monitoring.rst
@@ -0,0 +1,543 @@
+.. _mgr-cephadm-monitoring:
+
+Monitoring Services
+===================
+
+Ceph Dashboard uses `Prometheus <https://prometheus.io/>`_, `Grafana
+<https://grafana.com/>`_, and related tools to store and visualize detailed
+metrics on cluster utilization and performance.  Ceph users have three options:
+
+#. Have cephadm deploy and configure these services.  This is the default
+   when bootstrapping a new cluster unless the ``--skip-monitoring-stack``
+   option is used.
+#. Deploy and configure these services manually.  This is recommended for users
+   with existing prometheus services in their environment (and in cases where
+   Ceph is running in Kubernetes with Rook).
+#. Skip the monitoring stack completely.  Some Ceph dashboard graphs will
+   not be available.
+
+The monitoring stack consists of `Prometheus <https://prometheus.io/>`_,
+Prometheus exporters (:ref:`mgr-prometheus`, `Node exporter
+<https://prometheus.io/docs/guides/node-exporter/>`_), `Prometheus Alert
+Manager <https://prometheus.io/docs/alerting/alertmanager/>`_ and `Grafana
+<https://grafana.com/>`_.
+
+.. note::
+
+  Prometheus' security model presumes that untrusted users have access to the
+  Prometheus HTTP endpoint and logs. Untrusted users have access to all the
+  (meta)data Prometheus collects that is contained in the database, plus a
+  variety of operational and debugging information.
+
+  However, Prometheus' HTTP API is limited to read-only operations.
+  Configurations can *not* be changed using the API and secrets are not
+  exposed. Moreover, Prometheus has some built-in measures to mitigate the
+  impact of denial of service attacks.
+
+  Please see `Prometheus' Security model
+  <https://prometheus.io/docs/operating/security/>` for more detailed
+  information.
+
+Deploying monitoring with cephadm
+---------------------------------
+
+The default behavior of ``cephadm`` is to deploy a basic monitoring stack.  It
+is however possible that you have a Ceph cluster without a monitoring stack,
+and you would like to add a monitoring stack to it. (Here are some ways that
+you might have come to have a Ceph cluster without a monitoring stack: You
+might have passed the ``--skip-monitoring stack`` option to ``cephadm`` during
+the installation of the cluster, or you might have converted an existing
+cluster (which had no monitoring stack) to cephadm management.)
+
+To set up monitoring on a Ceph cluster that has no monitoring, follow the
+steps below:
+
+#. Deploy a node-exporter service on every node of the cluster.  The node-exporter provides host-level metrics like CPU and memory utilization:
+
+   .. prompt:: bash #
+
+     ceph orch apply node-exporter
+
+#. Deploy alertmanager:
+
+   .. prompt:: bash #
+
+     ceph orch apply alertmanager
+
+#. Deploy Prometheus. A single Prometheus instance is sufficient, but
+   for high availability (HA) you might want to deploy two:
+
+   .. prompt:: bash #
+
+     ceph orch apply prometheus
+
+   or
+
+   .. prompt:: bash #
+
+     ceph orch apply prometheus --placement 'count:2'
+
+#. Deploy grafana:
+
+   .. prompt:: bash #
+
+     ceph orch apply grafana
+
+.. _cephadm-monitoring-centralized-logs:
+
+Centralized Logging in Ceph
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ceph now provides centralized logging with Loki & Promtail. Centralized Log Management (CLM) consolidates all log data and pushes it to a central repository, 
+with an accessible and easy-to-use interface. Centralized logging is designed to make your life easier. 
+Some of the advantages are:
+
+#. **Linear event timeline**: it is easier to troubleshoot issues analyzing a single chain of events than thousands of different logs from a hundred nodes.
+#. **Real-time live log monitoring**: it is impractical to follow logs from thousands of different sources.
+#. **Flexible retention policies**: with per-daemon logs, log rotation is usually set to a short interval (1-2 weeks) to save disk usage.
+#. **Increased security & backup**: logs can contain sensitive information and expose usage patterns. Additionally, centralized logging allows for HA, etc.
+
+Centralized Logging in Ceph is implemented using two new services - ``loki`` & ``promtail``.
+
+Loki: It is basically a log aggregation system and is used to query logs. It can be configured as a datasource in Grafana. 
+
+Promtail: It acts as an agent that gathers logs from the system and makes them available to Loki.
+
+These two services are not deployed by default in a Ceph cluster. To enable the centralized logging you can follow the steps mentioned here :ref:`centralized-logging`.
+
+.. _cephadm-monitoring-networks-ports:
+
+Networks and Ports
+~~~~~~~~~~~~~~~~~~
+
+All monitoring services can have the network and port they bind to configured with a yaml service specification. By default
+cephadm will use ``https`` protocol when configuring Grafana daemons unless the user explicitly sets the protocol to ``http``.
+
+example spec file:
+
+.. code-block:: yaml
+
+    service_type: grafana
+    service_name: grafana
+    placement:
+      count: 1
+    networks:
+    - 192.169.142.0/24
+    spec:
+      port: 4200
+      protocol: http
+
+.. _cephadm_monitoring-images:
+
+Using custom images
+~~~~~~~~~~~~~~~~~~~
+
+It is possible to install or upgrade monitoring components based on other
+images.  To do so, the name of the image to be used needs to be stored in the
+configuration first.  The following configuration options are available.
+
+- ``container_image_prometheus``
+- ``container_image_grafana``
+- ``container_image_alertmanager``
+- ``container_image_node_exporter``
+
+Custom images can be set with the ``ceph config`` command
+
+.. code-block:: bash
+
+     ceph config set mgr mgr/cephadm/<option_name> <value>
+
+For example
+
+.. code-block:: bash
+
+     ceph config set mgr mgr/cephadm/container_image_prometheus prom/prometheus:v1.4.1
+
+If there were already running monitoring stack daemon(s) of the type whose
+image you've changed, you must redeploy the daemon(s) in order to have them
+actually use the new image.
+
+For example, if you had changed the prometheus image
+
+.. prompt:: bash #
+
+     ceph orch redeploy prometheus
+
+
+.. note::
+
+     By setting a custom image, the default value will be overridden (but not
+     overwritten).  The default value changes when updates become available.
+     By setting a custom image, you will not be able to update the component
+     you have set the custom image for automatically.  You will need to
+     manually update the configuration (image name and tag) to be able to
+     install updates.
+
+     If you choose to go with the recommendations instead, you can reset the
+     custom image you have set before.  After that, the default value will be
+     used again.  Use ``ceph config rm`` to reset the configuration option
+
+     .. code-block:: bash
+
+          ceph config rm mgr mgr/cephadm/<option_name>
+
+     For example
+
+     .. code-block:: bash
+
+          ceph config rm mgr mgr/cephadm/container_image_prometheus
+
+See also :ref:`cephadm-airgap`.
+
+.. _cephadm-overwrite-jinja2-templates:
+
+Using custom configuration files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By overriding cephadm templates, it is possible to completely customize the
+configuration files for monitoring services.
+
+Internally, cephadm already uses `Jinja2
+<https://jinja.palletsprojects.com/en/2.11.x/>`_ templates to generate the
+configuration files for all monitoring components. Starting from version 17.2.3,
+cephadm supports Prometheus http service discovery, and uses this endpoint for the
+definition and management of the embedded Prometheus service. The endpoint listens on
+``https://<mgr-ip>:8765/sd/`` (the port is
+configurable through the variable ``service_discovery_port``) and returns scrape target
+information in `http_sd_config format
+<https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config/>`_
+
+Customers with external monitoring stack can use `ceph-mgr` service discovery endpoint
+to get scraping configuration. Root certificate of the server can be obtained by the
+following command:
+
+   .. prompt:: bash #
+
+     ceph orch sd dump cert
+
+The configuration of Prometheus, Grafana, or Alertmanager may be customized by storing
+a Jinja2 template for each service. This template will be evaluated every time a service
+of that kind is deployed or reconfigured. That way, the custom configuration is preserved
+and automatically applied on future deployments of these services.
+
+.. note::
+
+  The configuration of the custom template is also preserved when the default
+  configuration of cephadm changes. If the updated configuration is to be used,
+  the custom template needs to be migrated *manually* after each upgrade of Ceph.
+
+Option names
+""""""""""""
+
+The following templates for files that will be generated by cephadm can be
+overridden. These are the names to be used when storing with ``ceph config-key
+set``:
+
+- ``services/alertmanager/alertmanager.yml``
+- ``services/grafana/ceph-dashboard.yml``
+- ``services/grafana/grafana.ini``
+- ``services/prometheus/prometheus.yml``
+- ``services/prometheus/alerting/custom_alerts.yml``
+- ``services/loki.yml``
+- ``services/promtail.yml``
+
+You can look up the file templates that are currently used by cephadm in
+``src/pybind/mgr/cephadm/templates``:
+
+- ``services/alertmanager/alertmanager.yml.j2``
+- ``services/grafana/ceph-dashboard.yml.j2``
+- ``services/grafana/grafana.ini.j2``
+- ``services/prometheus/prometheus.yml.j2``
+- ``services/loki.yml.j2``
+- ``services/promtail.yml.j2``
+
+Usage
+"""""
+
+The following command applies a single line value:
+
+.. code-block:: bash
+
+  ceph config-key set mgr/cephadm/<option_name> <value>
+
+To set contents of files as template use the ``-i`` argument:
+
+.. code-block:: bash
+
+  ceph config-key set mgr/cephadm/<option_name> -i $PWD/<filename>
+
+.. note::
+
+  When using files as input to ``config-key`` an absolute path to the file must
+  be used.
+
+
+Then the configuration file for the service needs to be recreated.
+This is done using `reconfig`. For more details see the following example.
+
+Example
+"""""""
+
+.. code-block:: bash
+
+  # set the contents of ./prometheus.yml.j2 as template
+  ceph config-key set mgr/cephadm/services/prometheus/prometheus.yml \
+    -i $PWD/prometheus.yml.j2
+
+  # reconfig the prometheus service
+  ceph orch reconfig prometheus
+
+.. code-block:: bash
+
+  # set additional custom alerting rules for Prometheus
+  ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \
+    -i $PWD/custom_alerts.yml
+
+  # Note that custom alerting rules are not parsed by Jinja and hence escaping
+  # will not be an issue.
+
+Deploying monitoring without cephadm
+------------------------------------
+
+If you have an existing prometheus monitoring infrastructure, or would like
+to manage it yourself, you need to configure it to integrate with your Ceph
+cluster.
+
+* Enable the prometheus module in the ceph-mgr daemon
+
+  .. code-block:: bash
+
+     ceph mgr module enable prometheus
+
+  By default, ceph-mgr presents prometheus metrics on port 9283 on each host
+  running a ceph-mgr daemon.  Configure prometheus to scrape these.
+
+To make this integration easier, cephadm provides a service discovery endpoint at
+``https://<mgr-ip>:8765/sd/``. This endpoint can be used by an external
+Prometheus server to retrieve target information for a specific service. Information returned
+by this endpoint uses the format specified by the Prometheus `http_sd_config option
+<https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config/>`_
+
+Here's an example prometheus job definition that uses the cephadm service discovery endpoint
+
+  .. code-block:: bash
+
+     - job_name: 'ceph-exporter'  
+       http_sd_configs:  
+       - url: http://<mgr-ip>:8765/sd/prometheus/sd-config?service=ceph-exporter
+
+
+* To enable the dashboard's prometheus-based alerting, see :ref:`dashboard-alerting`.
+
+* To enable dashboard integration with Grafana, see :ref:`dashboard-grafana`.
+
+Disabling monitoring
+--------------------
+
+To disable monitoring and remove the software that supports it, run the following commands:
+
+.. code-block:: console
+
+  $ ceph orch rm grafana
+  $ ceph orch rm prometheus --force   # this will delete metrics data collected so far
+  $ ceph orch rm node-exporter
+  $ ceph orch rm alertmanager
+  $ ceph mgr module disable prometheus
+
+See also :ref:`orch-rm`.
+
+Setting up RBD-Image monitoring
+-------------------------------
+
+Due to performance reasons, monitoring of RBD images is disabled by default. For more information please see
+:ref:`prometheus-rbd-io-statistics`. If disabled, the overview and details dashboards will stay empty in Grafana
+and the metrics will not be visible in Prometheus.
+
+Setting up Prometheus
+-----------------------
+
+Setting Prometheus Retention Size and Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cephadm can configure Prometheus TSDB retention by specifying ``retention_time``
+and ``retention_size`` values in the Prometheus service spec.
+The retention time value defaults to 15 days (15d). Users can set a different value/unit where
+supported units are: 'y', 'w', 'd', 'h', 'm' and 's'. The retention size value defaults
+to 0 (disabled). Supported units in this case are: 'B', 'KB', 'MB', 'GB', 'TB', 'PB' and 'EB'.
+
+In the following example spec we set the retention time to 1 year and the size to 1GB.
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 1
+    spec:
+      retention_time: "1y"
+      retention_size: "1GB"
+
+.. note::
+
+  If you already had Prometheus daemon(s) deployed before and are updating an
+  existent spec as opposed to doing a fresh Prometheus deployment, you must also
+  tell cephadm to redeploy the Prometheus daemon(s) to put this change into effect.
+  This can be done with a ``ceph orch redeploy prometheus`` command.
+
+Setting up Grafana
+------------------
+
+Manually setting the Grafana URL
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cephadm automatically configures Prometheus, Grafana, and Alertmanager in
+all cases except one.
+
+In a some setups, the Dashboard user's browser might not be able to access the
+Grafana URL that is configured in Ceph Dashboard. This can happen when the
+cluster and the accessing user are in different DNS zones.
+
+If this is the case, you can use a configuration option for Ceph Dashboard
+to set the URL that the user's browser will use to access Grafana. This
+value will never be altered by cephadm. To set this configuration option,
+issue the following command:
+
+   .. prompt:: bash $
+
+     ceph dashboard set-grafana-frontend-api-url <grafana-server-api>
+
+It might take a minute or two for services to be deployed. After the
+services have been deployed, you should see something like this when you issue the command ``ceph orch ls``:
+
+.. code-block:: console
+
+  $ ceph orch ls
+  NAME           RUNNING  REFRESHED  IMAGE NAME                                      IMAGE ID        SPEC
+  alertmanager       1/1  6s ago     docker.io/prom/alertmanager:latest              0881eb8f169f  present
+  crash              2/2  6s ago     docker.io/ceph/daemon-base:latest-master-devel  mix           present
+  grafana            1/1  0s ago     docker.io/pcuzner/ceph-grafana-el8:latest       f77afcf0bcf6   absent
+  node-exporter      2/2  6s ago     docker.io/prom/node-exporter:latest             e5a616e4b9cf  present
+  prometheus         1/1  6s ago     docker.io/prom/prometheus:latest                e935122ab143  present
+
+Configuring SSL/TLS for Grafana
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cephadm`` deploys Grafana using the certificate defined in the ceph
+key/value store. If no certificate is specified, ``cephadm`` generates a
+self-signed certificate during the deployment of the Grafana service. Each
+certificate is specific for the host it was generated on.
+
+A custom certificate can be configured using the following commands:
+
+.. prompt:: bash #
+
+  ceph config-key set mgr/cephadm/{hostname}/grafana_key -i $PWD/key.pem
+  ceph config-key set mgr/cephadm/{hostname}/grafana_crt -i $PWD/certificate.pem
+
+Where `hostname` is the hostname for the host where grafana service is deployed.
+
+If you have already deployed Grafana, run ``reconfig`` on the service to
+update its configuration:
+
+.. prompt:: bash #
+
+  ceph orch reconfig grafana
+
+The ``reconfig`` command also sets the proper URL for Ceph Dashboard.
+
+Setting the initial admin password
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, Grafana will not create an initial
+admin user. In order to create the admin user, please create a file
+``grafana.yaml`` with this content:
+
+.. code-block:: yaml
+
+  service_type: grafana
+  spec:
+    initial_admin_password: mypassword
+
+Then apply this specification:
+
+.. code-block:: bash
+
+  ceph orch apply -i grafana.yaml
+  ceph orch redeploy grafana
+
+Grafana will now create an admin user called ``admin`` with the
+given password.
+
+Turning off anonymous access
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cephadm allows anonymous users (users who have not provided any
+login information) limited, viewer only access to the grafana dashboard. In
+order to set up grafana to only allow viewing from logged in users, you can
+set ``anonymous_access: False`` in your grafana spec.
+
+.. code-block:: yaml
+
+  service_type: grafana
+  placement:
+    hosts:
+    - host1
+  spec:
+    anonymous_access: False
+    initial_admin_password: "mypassword"
+
+Since deploying grafana with anonymous access set to false without an initial
+admin password set would make the dashboard inaccessible, cephadm requires
+setting the ``initial_admin_password`` when ``anonymous_access`` is set to false.
+
+
+Setting up Alertmanager
+-----------------------
+
+Adding Alertmanager webhooks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To add new webhooks to the Alertmanager configuration, add additional
+webhook urls like so:
+
+.. code-block:: yaml
+
+    service_type: alertmanager
+    spec:
+      user_data:
+        default_webhook_urls:
+        - "https://foo"
+        - "https://bar"
+
+Where ``default_webhook_urls`` is a list of additional URLs that are
+added to the default receivers' ``<webhook_configs>`` configuration.
+
+Run ``reconfig`` on the service to update its configuration:
+
+.. prompt:: bash #
+
+  ceph orch reconfig alertmanager
+
+Turn on Certificate Validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are using certificates for alertmanager and want to make sure
+these certs are verified, you should set the "secure" option to
+true in your alertmanager spec (this defaults to false).
+
+.. code-block:: yaml
+
+    service_type: alertmanager
+    spec:
+      secure: true
+
+If you already had alertmanager daemons running before applying the spec
+you must reconfigure them to update their configuration
+
+.. prompt:: bash #
+
+  ceph orch reconfig alertmanager
+
+Further Reading
+---------------
+
+* :ref:`mgr-prometheus`
diff --git a/doc/cephadm/services/nfs.rst b/doc/cephadm/services/nfs.rst
new file mode 100644
index 000000000..2f12c5916
--- /dev/null
+++ b/doc/cephadm/services/nfs.rst
@@ -0,0 +1,215 @@
+.. _deploy-cephadm-nfs-ganesha:
+
+===========
+NFS Service
+===========
+
+.. note:: Only the NFSv4 protocol is supported.
+
+The simplest way to manage NFS is via the ``ceph nfs cluster ...``
+commands; see :ref:`mgr-nfs`.  This document covers how to manage the
+cephadm services directly, which should only be necessary for unusual NFS
+configurations.
+
+Deploying NFS ganesha
+=====================
+
+Cephadm deploys NFS Ganesha daemon (or set of daemons).  The configuration for
+NFS is stored in the ``nfs-ganesha`` pool and exports are managed via the
+``ceph nfs export ...`` commands and via the dashboard.
+
+To deploy a NFS Ganesha gateway, run the following command:
+
+.. prompt:: bash #
+
+    ceph orch apply nfs *<svc_id>* [--port *<port>*] [--placement ...]
+
+For example, to deploy NFS with a service id of *foo* on the default
+port 2049 with the default placement of a single daemon:
+
+.. prompt:: bash #
+
+   ceph orch apply nfs foo
+
+See :ref:`orchestrator-cli-placement-spec` for the details of the placement
+specification.
+
+Service Specification
+=====================
+
+Alternatively, an NFS service can be applied using a YAML specification. 
+
+.. code-block:: yaml
+
+    service_type: nfs
+    service_id: mynfs
+    placement:
+      hosts:
+        - host1
+        - host2
+    spec:
+      port: 12345
+
+In this example, we run the server on the non-default ``port`` of
+12345 (instead of the default 2049) on ``host1`` and ``host2``.
+
+The specification can then be applied by running the following command:
+
+.. prompt:: bash #
+
+   ceph orch apply -i nfs.yaml
+
+.. _cephadm-ha-nfs:
+
+High-availability NFS
+=====================
+
+Deploying an *ingress* service for an existing *nfs* service will provide:
+
+* a stable, virtual IP that can be used to access the NFS server
+* fail-over between hosts if there is a host failure
+* load distribution across multiple NFS gateways (although this is rarely necessary)
+
+Ingress for NFS can be deployed for an existing NFS service
+(``nfs.mynfs`` in this example) with the following specification:
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: nfs.mynfs
+    placement:
+      count: 2
+    spec:
+      backend_service: nfs.mynfs
+      frontend_port: 2049
+      monitor_port: 9000
+      virtual_ip: 10.0.0.123/24
+
+A few notes:
+
+  * The *virtual_ip* must include a CIDR prefix length, as in the
+    example above.  The virtual IP will normally be configured on the
+    first identified network interface that has an existing IP in the
+    same subnet.  You can also specify a *virtual_interface_networks*
+    property to match against IPs in other networks; see
+    :ref:`ingress-virtual-ip` for more information.
+  * The *monitor_port* is used to access the haproxy load status
+    page.  The user is ``admin`` by default, but can be modified by
+    via an *admin* property in the spec.  If a password is not
+    specified via a *password* property in the spec, the auto-generated password
+    can be found with:
+
+    .. prompt:: bash #
+
+	ceph config-key get mgr/cephadm/ingress.*{svc_id}*/monitor_password
+
+    For example:
+
+    .. prompt:: bash #
+
+	ceph config-key get mgr/cephadm/ingress.nfs.myfoo/monitor_password
+	
+  * The backend service (``nfs.mynfs`` in this example) should include
+    a *port* property that is not 2049 to avoid conflicting with the
+    ingress service, which could be placed on the same host(s).
+
+NFS with virtual IP but no haproxy
+----------------------------------
+
+Cephadm also supports deploying nfs with keepalived but not haproxy. This
+offers a virtual ip supported by keepalived that the nfs daemon can directly bind
+to instead of having traffic go through haproxy.
+
+In this setup, you'll either want to set up the service using the nfs module
+(see :ref:`nfs-module-cluster-create`) or place the ingress service first, so
+the virtual IP is present for the nfs daemon to bind to. The ingress service
+should include the attribute ``keepalive_only`` set to true. For example
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: nfs.foo
+    placement:
+      count: 1
+      hosts:
+      - host1
+      - host2
+      - host3
+    spec:
+      backend_service: nfs.foo
+      monitor_port: 9049
+      virtual_ip: 192.168.122.100/24
+      keepalive_only: true
+
+Then, an nfs service could be created that specifies a ``virtual_ip`` attribute
+that will tell it to bind to that specific IP.
+
+.. code-block:: yaml
+
+    service_type: nfs
+    service_id: foo
+    placement:
+      count: 1
+      hosts:
+      - host1
+      - host2
+      - host3
+    spec:
+      port: 2049
+      virtual_ip: 192.168.122.100
+
+Note that in these setups, one should make sure to include ``count: 1`` in the
+nfs placement, as it's only possible for one nfs daemon to bind to the virtual IP.
+
+NFS with HAProxy Protocol Support
+----------------------------------
+
+Cephadm supports deploying NFS in High-Availability mode with additional
+HAProxy protocol support. This works just like High-availability NFS but also
+supports client IP level configuration on NFS Exports.  This feature requires
+`NFS-Ganesha v5.0`_ or later.
+
+.. _NFS-Ganesha v5.0: https://github.com/nfs-ganesha/nfs-ganesha/wiki/ReleaseNotes_5
+
+To use this mode, you'll either want to set up the service using the nfs module
+(see :ref:`nfs-module-cluster-create`) or manually create services with the
+extra parameter ``enable_haproxy_protocol`` set to true. Both NFS Service and
+Ingress service must have ``enable_haproxy_protocol`` set to the same value.
+For example:
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: nfs.foo
+    placement:
+      count: 1
+      hosts:
+      - host1
+      - host2
+      - host3
+    spec:
+      backend_service: nfs.foo
+      monitor_port: 9049
+      virtual_ip: 192.168.122.100/24
+      enable_haproxy_protocol: true
+
+.. code-block:: yaml
+
+    service_type: nfs
+    service_id: foo
+    placement:
+      count: 1
+      hosts:
+      - host1
+      - host2
+      - host3
+    spec:
+      port: 2049
+      enable_haproxy_protocol: true
+
+
+Further Reading
+===============
+
+* CephFS: :ref:`cephfs-nfs`
+* MGR: :ref:`mgr-nfs`
diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst
new file mode 100644
index 000000000..f62b0f831
--- /dev/null
+++ b/doc/cephadm/services/osd.rst
@@ -0,0 +1,997 @@
+***********
+OSD Service
+***********
+.. _device management: ../rados/operations/devices
+.. _libstoragemgmt: https://github.com/libstorage/libstoragemgmt
+
+List Devices
+============
+
+``ceph-volume`` scans each host in the cluster from time to time in order
+to determine which devices are present and whether they are eligible to be
+used as OSDs.
+
+To print a list of devices discovered by ``cephadm``, run this command:
+
+.. prompt:: bash #
+
+  ceph orch device ls [--hostname=...] [--wide] [--refresh]
+
+Example::
+
+  Hostname  Path      Type  Serial              Size   Health   Ident  Fault  Available
+  srv-01    /dev/sdb  hdd   15P0A0YFFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sdc  hdd   15R0A08WFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sdd  hdd   15R0A07DFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sde  hdd   15P0A0QDFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdb  hdd   15R0A033FRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdc  hdd   15R0A05XFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sde  hdd   15R0A0ANFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdf  hdd   15R0A06EFRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdb  hdd   15R0A0OGFRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdc  hdd   15R0A0P7FRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdd  hdd   15R0A0O7FRD6         300G  Unknown  N/A    N/A    No
+
+Using the ``--wide`` option provides all details relating to the device,
+including any reasons that the device might not be eligible for use as an OSD.
+
+In the above example you can see fields named "Health", "Ident", and "Fault".
+This information is provided by integration with `libstoragemgmt`_. By default,
+this integration is disabled (because `libstoragemgmt`_ may not be 100%
+compatible with your hardware).  To make ``cephadm`` include these fields,
+enable cephadm's "enhanced device scan" option as follows;
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/device_enhanced_scan true
+
+.. warning::
+    Although the libstoragemgmt library performs standard SCSI inquiry calls,
+    there is no guarantee that your firmware fully implements these standards.
+    This can lead to erratic behaviour and even bus resets on some older
+    hardware. It is therefore recommended that, before enabling this feature,
+    you test your hardware's compatibility with libstoragemgmt first to avoid
+    unplanned interruptions to services.
+
+    There are a number of ways to test compatibility, but the simplest may be
+    to use the cephadm shell to call libstoragemgmt directly - ``cephadm shell
+    lsmcli ldl``. If your hardware is supported you should see something like
+    this: 
+
+    ::
+
+      Path     | SCSI VPD 0x83    | Link Type | Serial Number      | Health Status
+      ----------------------------------------------------------------------------
+      /dev/sda | 50000396082ba631 | SAS       | 15P0A0R0FRD6       | Good
+      /dev/sdb | 50000396082bbbf9 | SAS       | 15P0A0YFFRD6       | Good
+
+
+After you have enabled libstoragemgmt support, the output will look something
+like this:
+
+::
+
+  # ceph orch device ls
+  Hostname   Path      Type  Serial              Size   Health   Ident  Fault  Available
+  srv-01     /dev/sdb  hdd   15P0A0YFFRD6         300G  Good     Off    Off    No
+  srv-01     /dev/sdc  hdd   15R0A08WFRD6         300G  Good     Off    Off    No
+  :
+
+In this example, libstoragemgmt has confirmed the health of the drives and the ability to
+interact with the Identification and Fault LEDs on the drive enclosures. For further
+information about interacting with these LEDs, refer to `device management`_.
+
+.. note::
+    The current release of `libstoragemgmt`_ (1.8.8) supports SCSI, SAS, and SATA based
+    local disks only. There is no official support for NVMe devices (PCIe)
+
+.. _cephadm-deploy-osds:
+
+Deploy OSDs
+===========
+
+Listing Storage Devices
+-----------------------
+
+In order to deploy an OSD, there must be a storage device that is *available* on
+which the OSD will be deployed.
+
+Run this command to display an inventory of storage devices on all cluster hosts:
+
+.. prompt:: bash #
+
+  ceph orch device ls
+
+A storage device is considered *available* if all of the following
+conditions are met:
+
+* The device must have no partitions.
+* The device must not have any LVM state.
+* The device must not be mounted.
+* The device must not contain a file system.
+* The device must not contain a Ceph BlueStore OSD.
+* The device must be larger than 5 GB.
+
+Ceph will not provision an OSD on a device that is not available.
+
+Creating New OSDs
+-----------------
+
+There are a few ways to create new OSDs:
+
+* Tell Ceph to consume any available and unused storage device:
+
+  .. prompt:: bash #
+
+    ceph orch apply osd --all-available-devices
+
+* Create an OSD from a specific device on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd *<host>*:*<device-path>*
+
+  For example:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:/dev/sdb
+
+  Advanced OSD creation from specific devices on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:data_devices=/dev/sda,/dev/sdb,db_devices=/dev/sdc,osds_per_device=2
+
+* Create an OSD on a specific LVM logical volume on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd *<host>*:*<lvm-path>*
+
+  For example:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:/dev/vg_osd/lvm_osd1701
+
+* You can use :ref:`drivegroups` to categorize device(s) based on their
+  properties. This might be useful in forming a clearer picture of which
+  devices are available to consume. Properties include device type (SSD or
+  HDD), device model names, size, and the hosts on which the devices exist:
+
+  .. prompt:: bash #
+
+    ceph orch apply -i spec.yml
+
+Dry Run
+-------
+
+The ``--dry-run`` flag causes the orchestrator to present a preview of what
+will happen without actually creating the OSDs.
+
+For example:
+
+.. prompt:: bash #
+
+  ceph orch apply osd --all-available-devices --dry-run
+
+::
+
+  NAME                  HOST  DATA      DB  WAL
+  all-available-devices node1 /dev/vdb  -   -
+  all-available-devices node2 /dev/vdc  -   -
+  all-available-devices node3 /dev/vdd  -   -
+
+.. _cephadm-osd-declarative:
+
+Declarative State
+-----------------
+
+The effect of ``ceph orch apply`` is persistent. This means that drives that
+are added to the system after the ``ceph orch apply`` command completes will be
+automatically found and added to the cluster.  It also means that drives that
+become available (by zapping, for example) after the ``ceph orch apply``
+command completes will be automatically found and added to the cluster.
+
+We will examine the effects of the following command:
+
+.. prompt:: bash #
+
+  ceph orch apply osd --all-available-devices
+
+After running the above command: 
+
+* If you add new disks to the cluster, they will automatically be used to
+  create new OSDs.
+* If you remove an OSD and clean the LVM physical volume, a new OSD will be
+  created automatically.
+
+If you want to avoid this behavior (disable automatic creation of OSD on available devices), use the ``unmanaged`` parameter:
+
+.. prompt:: bash #
+
+  ceph orch apply osd --all-available-devices --unmanaged=true
+
+.. note::
+
+    Keep these three facts in mind:
+
+    - The default behavior of ``ceph orch apply`` causes cephadm constantly to reconcile. This means that cephadm creates OSDs as soon as new drives are detected.
+
+    - Setting ``unmanaged: True`` disables the creation of OSDs. If ``unmanaged: True`` is set, nothing will happen even if you apply a new OSD service.
+
+    - ``ceph orch daemon add`` creates OSDs, but does not add an OSD service.
+
+* For cephadm, see also :ref:`cephadm-spec-unmanaged`.
+
+.. _cephadm-osd-removal:
+
+Remove an OSD
+=============
+
+Removing an OSD from a cluster involves two steps:
+
+#. evacuating all placement groups (PGs) from the cluster
+#. removing the PG-free OSD from the cluster
+
+The following command performs these two steps:
+
+.. prompt:: bash #
+
+  ceph orch osd rm <osd_id(s)> [--replace] [--force]
+
+Example:
+
+.. prompt:: bash #
+
+  ceph orch osd rm 0
+
+Expected output::
+
+  Scheduled OSD(s) for removal
+
+OSDs that are not safe to destroy will be rejected.
+
+.. note::
+    After removing OSDs, if the drives the OSDs were deployed on once again
+    become available, cephadm may automatically try to deploy more OSDs
+    on these drives if they match an existing drivegroup spec. If you deployed
+    the OSDs you are removing with a spec and don't want any new OSDs deployed on
+    the drives after removal, it's best to modify the drivegroup spec before removal.
+    Either set ``unmanaged: true`` to stop it from picking up new drives at all,
+    or modify it in some way that it no longer matches the drives used for the
+    OSDs you wish to remove. Then re-apply the spec. For more info on drivegroup
+    specs see :ref:`drivegroups`. For more info on the declarative nature of
+    cephadm in reference to deploying OSDs, see :ref:`cephadm-osd-declarative`
+
+Monitoring OSD State
+--------------------
+
+You can query the state of OSD operation with the following command:
+
+.. prompt:: bash #
+
+  ceph orch osd rm status
+
+Expected output::
+
+  OSD_ID  HOST         STATE                    PG_COUNT  REPLACE  FORCE  STARTED_AT
+  2       cephadm-dev  done, waiting for purge  0         True     False  2020-07-17 13:01:43.147684
+  3       cephadm-dev  draining                 17        False    True   2020-07-17 13:01:45.162158
+  4       cephadm-dev  started                  42        False    True   2020-07-17 13:01:45.162158
+
+
+When no PGs are left on the OSD, it will be decommissioned and removed from the cluster.
+
+.. note::
+    After removing an OSD, if you wipe the LVM physical volume in the device used by the removed OSD, a new OSD will be created.
+    For more information on this, read about the ``unmanaged`` parameter in :ref:`cephadm-osd-declarative`.
+
+Stopping OSD Removal
+--------------------
+
+It is possible to stop queued OSD removals by using the following command:
+
+.. prompt:: bash #
+
+  ceph orch osd rm stop <osd_id(s)>
+
+Example:
+
+.. prompt:: bash #
+
+  ceph orch osd rm stop 4
+
+Expected output::
+
+  Stopped OSD(s) removal
+
+This resets the initial state of the OSD and takes it off the removal queue.
+
+.. _cephadm-replacing-an-osd:
+
+Replacing an OSD
+----------------
+
+.. prompt:: bash #
+
+  ceph orch osd rm <osd_id(s)> --replace [--force]
+
+Example:
+
+.. prompt:: bash #
+
+  ceph orch osd rm 4 --replace
+
+Expected output::
+
+  Scheduled OSD(s) for replacement
+
+This follows the same procedure as the procedure in the "Remove OSD" section, with
+one exception: the OSD is not permanently removed from the CRUSH hierarchy, but is
+instead assigned a 'destroyed' flag.
+
+.. note::
+    The new OSD that will replace the removed OSD must be created on the same host 
+    as the OSD that was removed.
+
+**Preserving the OSD ID**
+
+The 'destroyed' flag is used to determine which OSD ids will be reused in the
+next OSD deployment.
+
+If you use OSDSpecs for OSD deployment, your newly added disks will be assigned
+the OSD ids of their replaced counterparts. This assumes that the new disks
+still match the OSDSpecs.
+
+Use the ``--dry-run`` flag to make certain that the ``ceph orch apply osd`` 
+command does what you want it to. The ``--dry-run`` flag shows you what the
+outcome of the command will be without making the changes you specify. When
+you are satisfied that the command will do what you want, run the command
+without the ``--dry-run`` flag.
+
+.. tip::
+
+  The name of your OSDSpec can be retrieved with the command ``ceph orch ls``
+
+Alternatively, you can use your OSDSpec file:
+
+.. prompt:: bash #
+
+  ceph orch apply -i <osd_spec_file> --dry-run
+
+Expected output::
+
+  NAME                  HOST  DATA     DB WAL
+  <name_of_osd_spec>    node1 /dev/vdb -  -
+
+
+When this output reflects your intention, omit the ``--dry-run`` flag to
+execute the deployment.
+
+
+Erasing Devices (Zapping Devices)
+---------------------------------
+
+Erase (zap) a device so that it can be reused. ``zap`` calls ``ceph-volume
+zap`` on the remote host.
+
+.. prompt:: bash #
+
+  ceph orch device zap <hostname> <path>
+
+Example command:
+
+.. prompt:: bash #
+
+  ceph orch device zap my_hostname /dev/sdx
+
+.. note::
+    If the unmanaged flag is unset, cephadm automatically deploys drives that
+    match the OSDSpec.  For example, if you use the
+    ``all-available-devices`` option when creating OSDs, when you ``zap`` a
+    device the cephadm orchestrator automatically creates a new OSD in the
+    device.  To disable this behavior, see :ref:`cephadm-osd-declarative`.
+
+
+.. _osd_autotune:
+
+Automatically tuning OSD memory
+===============================
+
+OSD daemons will adjust their memory consumption based on the
+``osd_memory_target`` config option (several gigabytes, by
+default).  If Ceph is deployed on dedicated nodes that are not sharing
+memory with other services, cephadm can automatically adjust the per-OSD
+memory consumption based on the total amount of RAM and the number of deployed
+OSDs.
+
+.. warning:: Cephadm sets ``osd_memory_target_autotune`` to ``true`` by default which is unsuitable for hyperconverged infrastructures.
+
+Cephadm will start with a fraction
+(``mgr/cephadm/autotune_memory_target_ratio``, which defaults to
+``.7``) of the total RAM in the system, subtract off any memory
+consumed by non-autotuned daemons (non-OSDs, for OSDs for which
+``osd_memory_target_autotune`` is false), and then divide by the
+remaining OSDs.
+
+The final targets are reflected in the config database with options like::
+
+  WHO   MASK      LEVEL   OPTION              VALUE
+  osd   host:foo  basic   osd_memory_target   126092301926
+  osd   host:bar  basic   osd_memory_target   6442450944
+
+Both the limits and the current memory consumed by each daemon are visible from
+the ``ceph orch ps`` output in the ``MEM LIMIT`` column::
+
+  NAME        HOST  PORTS  STATUS         REFRESHED  AGE  MEM USED  MEM LIMIT  VERSION                IMAGE ID      CONTAINER ID  
+  osd.1       dael         running (3h)     10s ago   3h    72857k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  9e183363d39c  
+  osd.2       dael         running (81m)    10s ago  81m    63989k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  1f0cc479b051  
+  osd.3       dael         running (62m)    10s ago  62m    64071k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  ac5537492f27  
+
+To exclude an OSD from memory autotuning, disable the autotune option
+for that OSD and also set a specific memory target.  For example,
+
+.. prompt:: bash #
+
+  ceph config set osd.123 osd_memory_target_autotune false
+  ceph config set osd.123 osd_memory_target 16G
+
+
+.. _drivegroups:
+
+Advanced OSD Service Specifications
+===================================
+
+:ref:`orchestrator-cli-service-spec`\s of type ``osd`` are a way to describe a
+cluster layout, using the properties of disks. Service specifications give the
+user an abstract way to tell Ceph which disks should turn into OSDs with which
+configurations, without knowing the specifics of device names and paths.
+
+Service specifications make it possible to define a yaml or json file that can
+be used to reduce the amount of manual work involved in creating OSDs.
+
+For example, instead of running the following command:
+
+.. prompt:: bash [monitor.1]#
+
+  ceph orch daemon add osd *<host>*:*<path-to-device>*
+
+for each device and each host, we can define a yaml or json file that allows us
+to describe the layout. Here's the most basic example.
+
+Create a file called (for example) ``osd_spec.yml``:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: default_drive_group  # custom name of the osd spec
+    placement:
+      host_pattern: '*'              # which hosts to target
+    spec:
+      data_devices:                  # the type of devices you are applying specs to
+        all: true                    # a filter, check below for a full list
+
+This means :
+
+#. Turn any available device (ceph-volume decides what 'available' is) into an
+   OSD on all hosts that match the glob pattern '*'. (The glob pattern matches
+   against the registered hosts from `host ls`) A more detailed section on
+   host_pattern is available below.
+
+#. Then pass it to `osd create` like this:
+
+   .. prompt:: bash [monitor.1]#
+
+     ceph orch apply -i /path/to/osd_spec.yml
+
+   This instruction will be issued to all the matching hosts, and will deploy
+   these OSDs.
+
+   Setups more complex than the one specified by the ``all`` filter are
+   possible. See :ref:`osd_filters` for details.
+
+   A ``--dry-run`` flag can be passed to the ``apply osd`` command to display a
+   synopsis of the proposed layout.
+
+Example
+
+.. prompt:: bash [monitor.1]#
+
+  ceph orch apply -i /path/to/osd_spec.yml --dry-run
+
+
+
+.. _osd_filters:
+
+Filters
+-------
+
+.. note::
+    Filters are applied using an `AND` gate by default. This means that a drive
+    must fulfill all filter criteria in order to get selected. This behavior can
+    be adjusted by setting ``filter_logic: OR`` in the OSD specification. 
+
+Filters are used to assign disks to groups, using their attributes to group
+them. 
+
+The attributes are based off of ceph-volume's disk query. You can retrieve
+information about the attributes with this command:
+
+.. code-block:: bash
+
+    ceph-volume inventory </path/to/disk>
+
+Vendor or Model
+^^^^^^^^^^^^^^^
+
+Specific disks can be targeted by vendor or model:
+
+.. code-block:: yaml
+
+    model: disk_model_name
+
+or
+
+.. code-block:: yaml
+
+    vendor: disk_vendor_name
+
+
+Size
+^^^^
+
+Specific disks can be targeted by `Size`:
+
+.. code-block:: yaml
+
+    size: size_spec
+
+Size specs
+__________
+
+Size specifications can be of the following forms:
+
+* LOW:HIGH
+* :HIGH
+* LOW:
+* EXACT
+
+Concrete examples:
+
+To include disks of an exact size
+
+.. code-block:: yaml
+
+    size: '10G'
+
+To include disks within a given range of size: 
+
+.. code-block:: yaml
+
+    size: '10G:40G'
+
+To include disks that are less than or equal to 10G in size:
+
+.. code-block:: yaml
+
+    size: ':10G'
+
+To include disks equal to or greater than 40G in size:
+
+.. code-block:: yaml
+
+    size: '40G:'
+
+Sizes don't have to be specified exclusively in Gigabytes(G).
+
+Other units of size are supported: Megabyte(M), Gigabyte(G) and Terabyte(T).
+Appending the (B) for byte is also supported: ``MB``, ``GB``, ``TB``.
+
+
+Rotational
+^^^^^^^^^^
+
+This operates on the 'rotational' attribute of the disk.
+
+.. code-block:: yaml
+
+    rotational: 0 | 1
+
+`1` to match all disks that are rotational
+
+`0` to match all disks that are non-rotational (SSD, NVME etc)
+
+
+All
+^^^
+
+This will take all disks that are 'available'
+
+.. note:: This is exclusive for the data_devices section.
+
+.. code-block:: yaml
+
+    all: true
+
+
+Limiter
+^^^^^^^
+
+If you have specified some valid filters but want to limit the number of disks that they match, use the ``limit`` directive:
+
+.. code-block:: yaml
+
+    limit: 2
+
+For example, if you used `vendor` to match all disks that are from `VendorA`
+but want to use only the first two, you could use `limit`:
+
+.. code-block:: yaml
+
+    data_devices:
+      vendor: VendorA
+      limit: 2
+
+.. note:: `limit` is a last resort and shouldn't be used if it can be avoided.
+
+
+Additional Options
+------------------
+
+There are multiple optional settings you can use to change the way OSDs are deployed.
+You can add these options to the base level of an OSD spec for it to take effect.
+
+This example would deploy all OSDs with encryption enabled.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: example_osd_spec
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        all: true
+      encrypted: true
+
+See a full list in the DriveGroupSpecs
+
+.. py:currentmodule:: ceph.deployment.drive_group
+
+.. autoclass:: DriveGroupSpec
+   :members:
+   :exclude-members: from_json
+
+
+Examples
+========
+
+The simple case
+---------------
+
+All nodes with the same setup
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: HDD-123-foo
+    Size: 4TB
+
+    2 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+This is a common setup and can be described quite easily:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: HDD-123-foo # Note, HDD-123 would also be valid
+      db_devices:
+        model: MC-55-44-XZ # Same here, MC-55-44 is valid
+
+However, we can improve it by reducing the filters on core properties of the drives:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        rotational: 0
+
+Now, we enforce all rotating devices to be declared as 'data devices' and all non-rotating devices will be used as shared_devices (wal, db)
+
+If you know that drives with more than 2TB will always be the slower data devices, you can also filter by size:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        size: '2TB:'
+      db_devices:
+        size: ':2TB'
+
+.. note:: All of the above OSD specs  are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change.
+
+
+Multiple OSD specs for a single host
+------------------------------------
+
+Here we have two distinct setups
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: HDD-123-foo
+    Size: 4TB
+
+    12 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+    2 NVMEs
+    Vendor: VendorC
+    Model: NVME-QQQQ-987
+    Size: 256GB
+
+
+* 20 HDDs should share 2 SSDs
+* 10 SSDs should share 2 NVMes
+
+This can be described with two layouts.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_hdd
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        model: MC-55-44-XZ
+        limit: 2 # db_slots is actually to be favoured here, but it's not implemented yet
+    ---
+    service_type: osd
+    service_id: osd_spec_ssd
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        vendor: VendorC
+
+This would create the desired layout by using all HDDs as data_devices with two SSD assigned as dedicated db/wal devices.
+The remaining SSDs(10) will be data_devices that have the 'VendorC' NVMEs assigned as dedicated db/wal devices.
+
+Multiple hosts with the same disk layout
+----------------------------------------
+
+Assuming the cluster has different kinds of hosts each with similar disk 
+layout, it is recommended to apply different OSD specs matching only one
+set of hosts. Typically you will have a spec for multiple hosts with the 
+same layout. 
+
+The service id as the unique key: In case a new OSD spec with an already
+applied service id is applied, the existing OSD spec will be superseded.
+cephadm will now create new OSD daemons based on the new spec
+definition. Existing OSD daemons will not be affected. See :ref:`cephadm-osd-declarative`.
+
+Node1-5
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: SSD-123-foo
+    Size: 4TB
+    2 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+Node6-10
+
+.. code-block:: none
+
+    5 NVMEs
+    Vendor: VendorA
+    Model: SSD-123-foo
+    Size: 4TB
+    20 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+You can use the 'placement' key in the layout to target certain nodes.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: disk_layout_a
+    placement:
+      label: disk_layout_a
+    spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        rotational: 0
+    ---
+    service_type: osd
+    service_id: disk_layout_b
+    placement:
+      label: disk_layout_b
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        model: SSD-123-foo
+
+
+This applies different OSD specs to different hosts depending on the `placement` key.
+See :ref:`orchestrator-cli-placement-spec`
+
+.. note::
+
+    Assuming each host has a unique disk layout, each OSD 
+    spec needs to have a different service id
+
+
+Dedicated wal + db
+------------------
+
+All previous cases co-located the WALs with the DBs.
+It's however possible to deploy the WAL on a dedicated device as well, if it makes sense.
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: SSD-123-foo
+    Size: 4TB
+
+    2 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+    2 NVMEs
+    Vendor: VendorC
+    Model: NVME-QQQQ-987
+    Size: 256GB
+
+
+The OSD spec for this case would look like the following (using the `model` filter):
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        model: SSD-123-foo
+      wal_devices:
+        model: NVME-QQQQ-987
+
+
+It is also possible to specify directly device paths in specific hosts like the following:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_using_paths
+    placement:
+      hosts:
+        - Node01
+        - Node02
+    spec:
+      data_devices:
+        paths:
+        - /dev/sdb
+      db_devices:
+        paths:
+        - /dev/sdc
+      wal_devices:
+        paths:
+        - /dev/sdd
+
+
+This can easily be done with other filters, like `size` or `vendor` as well.
+
+It's possible to specify the `crush_device_class` parameter within the
+DriveGroup spec, and it's applied to all the devices defined by the `paths`
+keyword:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_using_paths
+    placement:
+      hosts:
+        - Node01
+        - Node02
+    crush_device_class: ssd
+    spec:
+      data_devices:
+        paths:
+        - /dev/sdb
+        - /dev/sdc
+      db_devices:
+        paths:
+        - /dev/sdd
+      wal_devices:
+        paths:
+        - /dev/sde
+
+The `crush_device_class` parameter, however, can be defined for each OSD passed
+using the `paths` keyword with the following syntax:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_using_paths
+    placement:
+      hosts:
+        - Node01
+        - Node02
+    crush_device_class: ssd
+    spec:
+      data_devices:
+        paths:
+        - path: /dev/sdb
+          crush_device_class: ssd
+        - path: /dev/sdc
+          crush_device_class: nvme
+      db_devices:
+        paths:
+        - /dev/sdd
+      wal_devices:
+        paths:
+        - /dev/sde
+
+.. _cephadm-osd-activate:
+
+Activate existing OSDs
+======================
+
+In case the OS of a host was reinstalled, existing OSDs need to be activated
+again. For this use case, cephadm provides a wrapper for :ref:`ceph-volume-lvm-activate` that
+activates all existing OSDs on a host.
+
+.. prompt:: bash #
+
+  ceph cephadm osd activate <host>...
+
+This will scan all existing disks for OSDs and deploy corresponding daemons.
+
+Further Reading
+===============
+
+* :ref:`ceph-volume`
+* :ref:`rados-index`
diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
new file mode 100644
index 000000000..20ec39a88
--- /dev/null
+++ b/doc/cephadm/services/rgw.rst
@@ -0,0 +1,371 @@
+===========
+RGW Service
+===========
+
+.. _cephadm-deploy-rgw:
+
+Deploy RGWs
+===========
+
+Cephadm deploys radosgw as a collection of daemons that manage a
+single-cluster deployment or a particular *realm* and *zone* in a
+multisite deployment.  (For more information about realms and zones,
+see :ref:`multisite`.)
+
+Note that with cephadm, radosgw daemons are configured via the monitor
+configuration database instead of via a `ceph.conf` or the command line.  If
+that configuration isn't already in place (usually in the
+``client.rgw.<something>`` section), then the radosgw
+daemons will start up with default settings (e.g., binding to port
+80).
+
+To deploy a set of radosgw daemons, with an arbitrary service name
+*name*, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch apply rgw *<name>* [--realm=*<realm-name>*] [--zone=*<zone-name>*] --placement="*<num-daemons>* [*<host1>* ...]"
+
+Trivial setup
+-------------
+
+For example, to deploy 2 RGW daemons (the default) for a single-cluster RGW deployment
+under the arbitrary service id *foo*:
+
+.. prompt:: bash #
+
+   ceph orch apply rgw foo
+
+.. _cephadm-rgw-designated_gateways:
+
+Designated gateways
+-------------------
+
+A common scenario is to have a labeled set of hosts that will act
+as gateways, with multiple instances of radosgw running on consecutive
+ports 8000 and 8001:
+
+.. prompt:: bash #
+
+   ceph orch host label add gwhost1 rgw  # the 'rgw' label can be anything
+   ceph orch host label add gwhost2 rgw
+   ceph orch apply rgw foo '--placement=label:rgw count-per-host:2' --port=8000
+
+See also: :ref:`cephadm_co_location`.
+
+.. _cephadm-rgw-networks:
+
+Specifying Networks
+-------------------
+
+The RGW service can have the network they bind to configured with a yaml service specification.
+
+example spec file:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    service_id: foo
+    placement:
+      label: rgw
+      count_per_host: 2
+    networks:
+    - 192.169.142.0/24
+    spec:
+      rgw_frontend_port: 8080
+
+Passing Frontend Extra Arguments
+--------------------------------
+
+The RGW service specification can be used to pass extra arguments to the rgw frontend by using
+the `rgw_frontend_extra_args` arguments list.
+
+example spec file:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    service_id: foo
+    placement:
+      label: rgw
+      count_per_host: 2
+    spec:
+      rgw_realm: myrealm
+      rgw_zone: myzone
+      rgw_frontend_type: "beast"
+      rgw_frontend_port: 5000
+      rgw_frontend_extra_args:
+      - "tcp_nodelay=1"
+      - "max_header_size=65536"
+
+.. note:: cephadm combines the arguments from the `spec` section and the ones from
+	  the `rgw_frontend_extra_args` into a single space-separated arguments list
+	  which is used to set the value of `rgw_frontends` configuration parameter.
+
+Multisite zones
+---------------
+
+To deploy RGWs serving the multisite *myorg* realm and the *us-east-1* zone on
+*myhost1* and *myhost2*:
+
+.. prompt:: bash #
+
+   ceph orch apply rgw east --realm=myorg --zonegroup=us-east-zg-1 --zone=us-east-1 --placement="2 myhost1 myhost2"
+
+Note that in a multisite situation, cephadm only deploys the daemons.  It does not create
+or update the realm or zone configurations.  To create a new realms, zones and zonegroups
+you can use :ref:`mgr-rgw-module` or manually using something like:
+
+.. prompt:: bash #
+
+  radosgw-admin realm create --rgw-realm=<realm-name>
+
+.. prompt:: bash #
+
+  radosgw-admin zonegroup create --rgw-zonegroup=<zonegroup-name>  --master
+
+.. prompt:: bash #
+
+  radosgw-admin zone create --rgw-zonegroup=<zonegroup-name> --rgw-zone=<zone-name> --master
+
+.. prompt:: bash #
+
+  radosgw-admin period update --rgw-realm=<realm-name> --commit
+
+See :ref:`orchestrator-cli-placement-spec` for details of the placement
+specification.  See :ref:`multisite` for more information of setting up multisite RGW.
+
+See also :ref:`multisite`.
+
+Setting up HTTPS
+----------------
+
+In order to enable HTTPS for RGW services, apply a spec file following this scheme:
+
+.. code-block:: yaml
+
+  service_type: rgw
+  service_id: myrgw
+  spec:
+    rgw_frontend_ssl_certificate: | 
+      -----BEGIN PRIVATE KEY-----
+      V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+      ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+      IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+      YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+      ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+      -----END PRIVATE KEY-----
+      -----BEGIN CERTIFICATE-----
+      V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+      ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+      IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+      YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+      ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+      -----END CERTIFICATE-----
+    ssl: true
+
+Then apply this yaml document:
+
+.. prompt:: bash #
+
+  ceph orch apply -i myrgw.yaml
+
+Note the value of ``rgw_frontend_ssl_certificate`` is a literal string as
+indicated by a ``|`` character preserving newline characters.
+
+Service specification
+---------------------
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: RGWSpec
+   :members:
+
+.. _orchestrator-haproxy-service-spec:
+
+High availability service for RGW
+=================================
+
+The *ingress* service allows you to create a high availability endpoint
+for RGW with a minimum set of configuration options.  The orchestrator will
+deploy and manage a combination of haproxy and keepalived to provide load
+balancing on a floating virtual IP.
+
+If the RGW service is configured with SSL enabled, then the ingress service
+will use the `ssl` and `verify none` options in the backend configuration.
+Trust verification is disabled because the backends are accessed by IP 
+address instead of FQDN.
+
+.. image:: ../../images/HAProxy_for_RGW.svg
+
+There are N hosts where the ingress service is deployed.  Each host
+has a haproxy daemon and a keepalived daemon.  A virtual IP is
+automatically configured on only one of these hosts at a time.
+
+Each keepalived daemon checks every few seconds whether the haproxy
+daemon on the same host is responding.  Keepalived will also check
+that the master keepalived daemon is running without problems.  If the
+"master" keepalived daemon or the active haproxy is not responding,
+one of the remaining keepalived daemons running in backup mode will be
+elected as master, and the virtual IP will be moved to that node.
+
+The active haproxy acts like a load balancer, distributing all RGW requests
+between all the RGW daemons available.
+
+Prerequisites
+-------------
+
+* An existing RGW service.
+
+Deploying
+---------
+
+Use the command::
+
+    ceph orch apply -i <ingress_spec_file>
+
+Service specification
+---------------------
+
+It is a yaml format file with the following properties:
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: rgw.something    # adjust to match your existing RGW service
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    spec:
+      backend_service: rgw.something            # adjust to match your existing RGW service
+      virtual_ip: <string>/<string>             # ex: 192.168.20.1/24
+      frontend_port: <integer>                  # ex: 8080
+      monitor_port: <integer>                   # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ]       # optional: list of CIDR networks
+      use_keepalived_multicast: <bool>          # optional: Default is False.
+      vrrp_interface_network: <string>/<string> # optional: ex: 192.168.20.0/24
+      ssl_cert: |                               # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: rgw.something    # adjust to match your existing RGW service
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    spec:
+      backend_service: rgw.something      # adjust to match your existing RGW service
+      virtual_ips_list:
+      - <string>/<string>                 # ex: 192.168.20.1/24
+      - <string>/<string>                 # ex: 192.168.20.2/24
+      - <string>/<string>                 # ex: 192.168.20.3/24
+      frontend_port: <integer>            # ex: 8080
+      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
+      first_virtual_router_id: <integer>  # optional: default 50
+      ssl_cert: |                         # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+
+where the properties of this service specification are:
+
+* ``service_type``
+    Mandatory and set to "ingress"
+* ``service_id``
+    The name of the service.  We suggest naming this after the service you are
+    controlling ingress for (e.g., ``rgw.foo``).
+* ``placement hosts``
+    The hosts where it is desired to run the HA daemons. An haproxy and a
+    keepalived container will be deployed on these hosts.  These hosts do not need
+    to match the nodes where RGW is deployed.
+* ``virtual_ip``
+    The virtual IP (and network) in CIDR format where the ingress service will be available.
+* ``virtual_ips_list``
+    The virtual IP address in CIDR format where the ingress service will be available.
+    Each virtual IP address will be primary on one node running the ingress service. The number
+    of virtual IP addresses must be less than or equal to the number of ingress nodes.
+* ``virtual_interface_networks``
+    A list of networks to identify which ethernet interface to use for the virtual IP.
+* ``frontend_port``
+    The port used to access the ingress service.
+* ``ssl_cert``:
+    SSL certificate, if SSL is to be enabled. This must contain the both the certificate and
+    private key blocks in .pem format.
+* ``use_keepalived_multicast``
+    Default is False. By default, cephadm will deploy keepalived config to use unicast IPs,
+    using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect
+    to the machines. But if multicast is prefered, we can set ``use_keepalived_multicast``
+    to ``True`` and Keepalived will use multicast IP (224.0.0.18) to communicate between instances,
+    using the same interfaces as where the VIPs are.
+* ``vrrp_interface_network``
+    By default, cephadm will configure keepalived to use the same interface where the VIPs are
+    for VRRP communication. If another interface is needed, it can be set via ``vrrp_interface_network``
+    with a network to identify which ethernet interface to use.
+* ``first_virtual_router_id``
+    Default is 50. When deploying more than 1 ingress, this parameter can be used to ensure each
+    keepalived will have different virtual_router_id. In the case of using ``virtual_ips_list``,
+    each IP will create its own virtual router. So the first one will have ``first_virtual_router_id``,
+    second one will have ``first_virtual_router_id`` + 1, etc. Valid values go from 1 to 255.
+
+.. _ingress-virtual-ip:
+
+Selecting ethernet interfaces for the virtual IP
+------------------------------------------------
+
+You cannot simply provide the name of the network interface on which
+to configure the virtual IP because interface names tend to vary
+across hosts (and/or reboots).  Instead, cephadm will select
+interfaces based on other existing IP addresses that are already
+configured.
+
+Normally, the virtual IP will be configured on the first network
+interface that has an existing IP in the same subnet.  For example, if
+the virtual IP is 192.168.0.80/24 and eth2 has the static IP
+192.168.0.40/24, cephadm will use eth2.
+
+In some cases, the virtual IP may not belong to the same subnet as an existing static
+IP.  In such cases, you can provide a list of subnets to match against existing IPs,
+and cephadm will put the virtual IP on the first network interface to match.  For example,
+if the virtual IP is 192.168.0.80/24 and we want it on the same interface as the machine's
+static IP in 10.10.0.0/16, you can use a spec like::
+
+  service_type: ingress
+  service_id: rgw.something
+  spec:
+    virtual_ip: 192.168.0.80/24
+    virtual_interface_networks:
+      - 10.10.0.0/16
+    ...
+
+A consequence of this strategy is that you cannot currently configure the virtual IP
+on an interface that has no existing IP address.  In this situation, we suggest
+configuring a "dummy" IP address is an unroutable network on the correct interface
+and reference that dummy network in the networks list (see above).
+
+
+Useful hints for ingress
+------------------------
+
+* It is good to have at least 3 RGW daemons.
+* We recommend at least 3 hosts for the ingress service.
+
+Further Reading
+===============
+
+* :ref:`object-gateway`
+* :ref:`mgr-rgw-module`
diff --git a/doc/cephadm/services/snmp-gateway.rst b/doc/cephadm/services/snmp-gateway.rst
new file mode 100644
index 000000000..f927fdfd0
--- /dev/null
+++ b/doc/cephadm/services/snmp-gateway.rst
@@ -0,0 +1,171 @@
+====================
+SNMP Gateway Service
+====================
+
+SNMP_ is still a widely used protocol, to monitor distributed systems and devices across a variety of hardware
+and software platforms. Ceph's SNMP integration focuses on forwarding alerts from it's Prometheus Alertmanager
+cluster to a gateway daemon. The gateway daemon, transforms the alert into an SNMP Notification and sends
+it on to a designated SNMP management platform. The gateway daemon is from the snmp_notifier_ project,
+which provides SNMP V2c and V3 support (authentication and encryption).
+
+Ceph's SNMP gateway service deploys one instance of the gateway by default. You may increase this
+by providing placement information. However, bear in mind that if you enable multiple SNMP gateway daemons,
+your SNMP management platform will receive multiple notifications for the same event.
+
+.. _SNMP: https://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
+.. _snmp_notifier: https://github.com/maxwo/snmp_notifier
+
+Compatibility
+=============
+The table below shows the SNMP versions that are supported by the gateway implementation
+
+================ =========== ===============================================
+ SNMP Version     Supported  Notes
+================ =========== ===============================================
+ V1                  �      Not supported by snmp_notifier
+ V2c                  ✔
+ V3 authNoPriv        ✔      uses username/password authentication, without
+                             encryption (NoPriv = no privacy)
+ V3 authPriv          ✔      uses username/password authentication with
+                             encryption to the SNMP management platform
+================ =========== ===============================================
+
+
+Deploying an SNMP Gateway
+=========================
+Both SNMP V2c and V3 provide credentials support. In the case of V2c, this is just the community string - but for V3
+environments you must provide additional authentication information. These credentials are not supported on the command
+line when deploying the service. Instead, you must create the service using a credentials file (in yaml format), or
+specify the complete service definition in a yaml file.
+
+Command format
+--------------
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway <snmp_version:V2c|V3> <destination> [<port:int>] [<engine_id>] [<auth_protocol: MD5|SHA>] [<privacy_protocol:DES|AES>] [<placement>] ...
+
+
+Usage Notes
+
+- you must supply the ``--snmp-version`` parameter
+- the ``--destination`` parameter must be of the format hostname:port (no default)
+- you may omit ``--port``. It defaults to 9464
+- the ``--engine-id`` is a unique identifier for the device (in hex) and required for SNMP v3 only.
+  Suggested value: 8000C53F<fsid> where the fsid is from your cluster, without the '-' symbols
+- for SNMP V3, the ``--auth-protocol`` setting defaults to **SHA**
+- for SNMP V3, with encryption you must define the ``--privacy-protocol``
+- you **must** provide a -i <filename> to pass the secrets/passwords to the orchestrator
+
+Deployment Examples
+===================
+
+SNMP V2c
+--------
+Here's an example for V2c, showing CLI and service based deployments
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --port 9464 --snmp_version=V2c --destination=192.168.122.73:162 -i ./snmp_creds.yaml
+
+with a credentials file that contains;
+
+.. code-block:: yaml
+
+   ---
+   snmp_community: public
+
+Alternatively, you can create a yaml definition for the gateway and apply it from a single file
+
+.. prompt:: bash #
+
+   ceph orch apply -i snmp-gateway.yml
+
+with the file containing the following configuration
+
+.. code-block:: yaml
+
+    service_type: snmp-gateway
+    service_name: snmp-gateway
+    placement:
+      count: 1
+    spec:
+      credentials:
+        snmp_community: public
+      port: 9464
+      snmp_destination: 192.168.122.73:162
+      snmp_version: V2c
+
+
+SNMP V3 (authNoPriv)
+--------------------
+Deploying an snmp-gateway service supporting SNMP V3 with authentication only, would look like this;
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --snmp-version=V3 --engine-id=800C53F000000 --destination=192.168.122.1:162 -i ./snmpv3_creds.yml
+
+with a credentials file as;
+
+.. code-block:: yaml
+
+   ---
+   snmp_v3_auth_username: myuser
+   snmp_v3_auth_password: mypassword
+
+or as a service configuration file
+
+.. code-block:: yaml
+
+   service_type: snmp-gateway
+   service_name: snmp-gateway
+   placement:
+     count: 1
+   spec:
+     credentials:
+       snmp_v3_auth_password: mypassword
+       snmp_v3_auth_username: myuser
+     engine_id: 800C53F000000
+     port: 9464
+     snmp_destination: 192.168.122.1:162
+     snmp_version: V3
+
+
+SNMP V3 (authPriv)
+------------------
+
+Defining an SNMP V3 gateway service that implements authentication and privacy (encryption), requires two additional values
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --snmp-version=V3 --engine-id=800C53F000000 --destination=192.168.122.1:162 --privacy-protocol=AES -i ./snmpv3_creds.yml
+
+with a credentials file as;
+
+.. code-block:: yaml
+
+   ---
+   snmp_v3_auth_username: myuser
+   snmp_v3_auth_password: mypassword
+   snmp_v3_priv_password: mysecret
+
+
+.. note::
+
+   The credentials are stored on the host, restricted to the root user and passed to the snmp_notifier daemon as
+   an environment file (``--env-file``), to limit exposure.
+
+
+AlertManager Integration
+========================
+When an SNMP gateway service is deployed or updated, the Prometheus Alertmanager configuration is automatically updated to forward any
+alert that has an OID_ label to the SNMP gateway daemon for processing.
+
+.. _OID: https://en.wikipedia.org/wiki/Object_identifier
+
+Implementing the MIB
+======================
+To make sense of the SNMP Notification/Trap, you'll need to apply the MIB to your SNMP management platform. The MIB (CEPH-MIB.txt) can
+downloaded from the main Ceph repo_
+
+.. _repo: https://github.com/ceph/ceph/tree/master/monitoring/snmp
diff --git a/doc/cephadm/services/tracing.rst b/doc/cephadm/services/tracing.rst
new file mode 100644
index 000000000..e96d601d2
--- /dev/null
+++ b/doc/cephadm/services/tracing.rst
@@ -0,0 +1,45 @@
+================
+Tracing Services
+================
+
+.. _cephadm-tracing:
+
+
+Jaeger Tracing
+==============
+
+Ceph uses Jaeger as the tracing backend. in order to use tracing, we need to deploy those services.
+
+Further details on tracing in ceph:
+
+`Ceph Tracing documentation <https://docs.ceph.com/en/latest/jaegertracing/#jaeger-distributed-tracing/>`_
+
+Deployment
+==========
+
+Jaeger services consist of 3 services:
+
+1. Jaeger Agent
+
+2. Jaeger Collector
+
+3. Jaeger Query
+
+Jaeger requires a database for the traces. we use ElasticSearch (version 6) by default.
+
+
+To deploy jaeger tracing service, when not using your own ElasticSearch:
+
+#. Deploy jaeger services, with a new elasticsearch container:
+
+    .. prompt:: bash #
+
+        ceph orch apply jaeger
+
+
+#. Deploy jaeger services, with existing elasticsearch cluster and existing jaeger query (deploy agents and collectors):
+
+     .. prompt:: bash #
+
+        ceph orch apply jaeger --without-query --es_nodes=ip:port,..
+
diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
new file mode 100644
index 000000000..d891ebaf2
--- /dev/null
+++ b/doc/cephadm/troubleshooting.rst
@@ -0,0 +1,514 @@
+Troubleshooting
+===============
+
+This section explains how to investigate why a cephadm command failed or why a
+certain service no longer runs properly.
+
+Cephadm deploys daemons within containers. Troubleshooting containerized
+daemons requires a different process than does troubleshooting traditional
+daemons that were installed by means of packages.
+
+Here are some tools and commands to help you troubleshoot your Ceph
+environment.
+
+.. _cephadm-pause:
+
+Pausing or Disabling cephadm
+----------------------------
+
+If something goes wrong and cephadm is behaving badly, pause most of the Ceph
+cluster's background activity by running the following command: 
+
+.. prompt:: bash #
+
+  ceph orch pause
+
+This stops all changes in the Ceph cluster, but cephadm will still periodically
+check hosts to refresh its inventory of daemons and devices. Disable cephadm
+completely by running the following commands:
+
+.. prompt:: bash #
+
+  ceph orch set backend ''
+  ceph mgr module disable cephadm
+
+These commands disable all of the ``ceph orch ...`` CLI commands. All
+previously deployed daemon containers continue to run and will start just as
+they were before you ran these commands.
+
+See :ref:`cephadm-spec-unmanaged` for more on disabling individual services.
+
+
+Per-service and Per-daemon Events
+---------------------------------
+
+To make it easier to debug failed daemons, cephadm stores events per service
+and per daemon. These events often contain information relevant to
+the troubleshooting of your Ceph cluster. 
+
+Listing Service Events
+~~~~~~~~~~~~~~~~~~~~~~
+
+To see the events associated with a certain service, run a command of the 
+following form:
+
+.. prompt:: bash #
+
+  ceph orch ls --service_name=<service-name> --format yaml
+
+This will return something in the following form:
+
+.. code-block:: yaml
+
+  service_type: alertmanager
+  service_name: alertmanager
+  placement:
+    hosts:
+    - unknown_host
+  status:
+    ...
+    running: 1
+    size: 1
+  events:
+  - 2021-02-01T08:58:02.741162 service:alertmanager [INFO] "service was created"
+  - '2021-02-01T12:09:25.264584 service:alertmanager [ERROR] "Failed to apply: Cannot
+    place <AlertManagerSpec for service_name=alertmanager> on unknown_host: Unknown hosts"'
+
+Listing Daemon Events
+~~~~~~~~~~~~~~~~~~~~~
+
+To see the events associated with a certain daemon, run a command of the
+following form:
+
+.. prompt:: bash #
+
+  ceph orch ps --service-name <service-name> --daemon-id <daemon-id> --format yaml
+
+This will return something in the following form:
+
+.. code-block:: yaml
+
+  daemon_type: mds
+  daemon_id: cephfs.hostname.ppdhsz
+  hostname: hostname
+  status_desc: running
+  ...
+  events:
+  - 2021-02-01T08:59:43.845866 daemon:mds.cephfs.hostname.ppdhsz [INFO] "Reconfigured
+    mds.cephfs.hostname.ppdhsz on host 'hostname'"
+
+
+Checking Cephadm Logs
+---------------------
+
+To learn how to monitor cephadm logs as they are generated, read
+:ref:`watching_cephadm_logs`.
+
+If your Ceph cluster has been configured to log events to files, there will be
+a ``ceph.cephadm.log`` file on all monitor hosts. See :ref:`cephadm-logs` for a
+more complete explanation.
+
+Gathering Log Files
+-------------------
+
+Use ``journalctl`` to gather the log files of all daemons:
+
+.. note:: By default cephadm now stores logs in journald. This means
+   that you will no longer find daemon logs in ``/var/log/ceph/``.
+
+To read the log file of one specific daemon, run a command of the following
+form:
+
+.. prompt:: bash
+
+   cephadm logs --name <name-of-daemon>
+
+.. Note:: This works only when run on the same host that is running the daemon.
+   To get the logs of a daemon that is running on a different host, add the
+   ``--fsid`` option to the command, as in the following example:
+
+   .. prompt:: bash
+
+      cephadm logs --fsid <fsid> --name <name-of-daemon>
+
+   In this example, ``<fsid>`` corresponds to the cluster ID returned by the
+   ``ceph status`` command.
+
+To fetch all log files of all daemons on a given host, run the following
+for-loop::
+
+    for name in $(cephadm ls | jq -r '.[].name') ; do
+      cephadm logs --fsid <fsid> --name "$name" > $name;
+    done
+
+Collecting Systemd Status
+-------------------------
+
+To print the state of a systemd unit, run a command of the following form: 
+
+.. prompt:: bash
+
+   systemctl status "ceph-$(cephadm shell ceph fsid)@<service name>.service";
+
+
+To fetch the state of all daemons of a given host, run the following shell
+script::
+
+   fsid="$(cephadm shell ceph fsid)"
+   for name in $(cephadm ls | jq -r '.[].name') ; do
+     systemctl status "ceph-$fsid@$name.service" > $name;
+   done
+
+
+List all Downloaded Container Images
+------------------------------------
+
+To list all container images that are downloaded on a host, run the following
+commands:
+
+.. prompt:: bash #
+
+   podman ps -a --format json | jq '.[].Image' "docker.io/library/centos:8" "registry.opensuse.org/opensuse/leap:15.2"
+
+.. note:: ``Image`` might also be called ``ImageID``.
+
+
+Manually Running Containers
+---------------------------
+
+Cephadm uses small wrappers when running containers. Refer to
+``/var/lib/ceph/<cluster-fsid>/<service-name>/unit.run`` for the container
+execution command.
+
+.. _cephadm-ssh-errors:
+
+SSH Errors
+----------
+
+Error message::
+
+  execnet.gateway_bootstrap.HostNotFound: -F /tmp/cephadm-conf-73z09u6g -i /tmp/cephadm-identity-ky7ahp_5 root@10.10.1.2
+  ...
+  raise OrchestratorError(msg) from e
+  orchestrator._interface.OrchestratorError: Failed to connect to 10.10.1.2 (10.10.1.2).
+  Please make sure that the host is reachable and accepts connections using the cephadm SSH key
+  ...
+
+If you receive the above error message, try the following things to
+troubleshoot the SSH connection between ``cephadm`` and the monitor:
+
+1. Ensure that ``cephadm`` has an SSH identity key::
+
+     [root@mon1~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key
+     INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98
+     INFO:cephadm:Using recent ceph image docker.io/ceph/ceph:v15 obtained 'mgr/cephadm/ssh_identity_key'
+     [root@mon1 ~] # chmod 0600 ~/cephadm_private_key
+
+ If this fails, cephadm doesn't have a key. Fix this by running the following command::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm generate-ssh-key
+
+ or::
+
+     [root@mon1 ~]# cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssh-key -i -
+
+2. Ensure that the SSH config is correct::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm get-ssh-config > config
+
+3. Verify that it is possible to connect to the host::
+
+     [root@mon1 ~]# ssh -F config -i ~/cephadm_private_key root@mon1
+
+Verifying that the Public Key is Listed in the authorized_keys file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify that the public key is in the ``authorized_keys`` file, run the
+following commands::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm get-pub-key > ~/ceph.pub
+     [root@mon1 ~]# grep "`cat ~/ceph.pub`"  /root/.ssh/authorized_keys
+
+Failed to Infer CIDR network error
+----------------------------------
+
+If you see this error::
+
+   ERROR: Failed to infer CIDR network for mon ip ***; pass --skip-mon-network to configure it later
+
+Or this error::
+
+   Must set public_network config option or specify a CIDR network, ceph addrvec, or plain IP
+
+This means that you must run a command of this form:
+
+.. prompt:: bash
+
+   ceph config set mon public_network <mon_network>
+
+For more detail on operations of this kind, see
+:ref:`deploy_additional_monitors`.
+
+Accessing the Admin Socket
+--------------------------
+
+Each Ceph daemon provides an admin socket that bypasses the MONs (See
+:ref:`rados-monitoring-using-admin-socket`).
+
+#. To access the admin socket, enter the daemon container on the host::
+
+   [root@mon1 ~]# cephadm enter --name <daemon-name>
+
+#. Run a command of the following form to see the admin socket's configuration::
+  
+   [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok config show
+
+Running Various Ceph Tools
+--------------------------------
+
+To run Ceph tools such as ``ceph-objectstore-tool`` or 
+``ceph-monstore-tool``, invoke the cephadm CLI with
+``cephadm shell --name <daemon-name>``.  For example::
+
+    root@myhostname # cephadm unit --name mon.myhostname stop
+    root@myhostname # cephadm shell --name mon.myhostname
+    [ceph: root@myhostname /]# ceph-monstore-tool /var/lib/ceph/mon/ceph-myhostname get monmap > monmap         
+    [ceph: root@myhostname /]# monmaptool --print monmap
+    monmaptool: monmap file monmap
+    epoch 1
+    fsid 28596f44-3b56-11ec-9034-482ae35a5fbb
+    last_changed 2021-11-01T20:57:19.755111+0000
+    created 2021-11-01T20:57:19.755111+0000
+    min_mon_release 17 (quincy)
+    election_strategy: 1
+    0: [v2:127.0.0.1:3300/0,v1:127.0.0.1:6789/0] mon.myhostname
+
+The cephadm shell sets up the environment in a way that is suitable for
+extended daemon maintenance and for the interactive running of daemons. 
+
+.. _cephadm-restore-quorum:
+
+Restoring the Monitor Quorum
+----------------------------
+
+If the Ceph Monitor daemons (mons) cannot form a quorum, ``cephadm`` will not
+be able to manage the cluster until quorum is restored.
+
+In order to restore the quorum, remove unhealthy monitors
+form the monmap by following these steps:
+
+1. Stop all Monitors. Use ``ssh`` to connect to each Monitor's host, and then
+   while connected to the Monitor's host use ``cephadm`` to stop the Monitor
+   daemon:
+
+   .. prompt:: bash
+
+      ssh {mon-host}
+      cephadm unit --name {mon.hostname} stop
+
+
+2. Identify a surviving Monitor and log in to its host:
+
+   .. prompt:: bash
+
+      ssh {mon-host}
+      cephadm enter --name {mon.hostname}
+
+3. Follow the steps in :ref:`rados-mon-remove-from-unhealthy`.
+
+.. _cephadm-manually-deploy-mgr:
+
+Manually Deploying a Manager Daemon
+-----------------------------------
+At least one Manager (``mgr``) daemon is required by cephadm in order to manage
+the cluster. If the last remaining Manager has been removed from the Ceph
+cluster, follow these steps in order to deploy a fresh Manager on an arbitrary
+host in your cluster. In this example, the freshly-deployed Manager daemon is
+called ``mgr.hostname.smfvfd``.
+
+#. Disable the cephadm scheduler, in order to prevent ``cephadm`` from removing
+   the new Manager. See :ref:`cephadm-enable-cli`:
+
+   .. prompt:: bash #
+
+      ceph config-key set mgr/cephadm/pause true
+
+#. Retrieve or create the "auth entry" for the new Manager:
+
+   .. prompt:: bash #
+
+      ceph auth get-or-create mgr.hostname.smfvfd mon "profile mgr" osd "allow *" mds "allow *"
+
+#. Retrieve the Monitor's configuration:
+
+   .. prompt:: bash #
+
+      ceph config generate-minimal-conf
+
+#. Retrieve the container image:
+
+   .. prompt:: bash #
+
+      ceph config get "mgr.hostname.smfvfd" container_image
+
+#. Create a file called ``config-json.json``, which contains the information
+   necessary to deploy the daemon:
+
+   .. code-block:: json
+
+     {
+       "config": "# minimal ceph.conf for 8255263a-a97e-4934-822c-00bfe029b28f\n[global]\n\tfsid = 8255263a-a97e-4934-822c-00bfe029b28f\n\tmon_host = [v2:192.168.0.1:40483/0,v1:192.168.0.1:40484/0]\n",
+       "keyring": "[mgr.hostname.smfvfd]\n\tkey = V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4=\n"
+     }
+
+#. Deploy the Manager daemon:
+
+   .. prompt:: bash #
+
+      cephadm --image <container-image> deploy --fsid <fsid> --name mgr.hostname.smfvfd --config-json config-json.json
+
+Capturing Core Dumps
+---------------------
+
+A Ceph cluster that uses ``cephadm`` can be configured to capture core dumps.
+The initial capture and processing of the coredump is performed by
+`systemd-coredump
+<https://www.man7.org/linux/man-pages/man8/systemd-coredump.8.html>`_.
+
+
+To enable coredump handling, run the following command
+
+.. prompt:: bash #
+
+   ulimit -c unlimited
+
+
+.. note::
+
+  Core dumps are not namespaced by the kernel. This means that core dumps are
+  written to ``/var/lib/systemd/coredump`` on the container host. The ``ulimit
+  -c unlimited`` setting  will persist  only until the system is rebooted.
+
+Wait for the crash to happen again. To simulate the crash of a daemon, run for
+example ``killall -3 ceph-mon``.
+
+
+Running the Debugger with cephadm
+----------------------------------
+
+Running a single debugging session
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Initiate a debugging session by using the ``cephadm shell`` command.
+From within the shell container we need to install the debugger and debuginfo
+packages. To debug a core file captured by systemd, run the following:
+
+
+#. Start the shell session:
+
+   .. prompt:: bash #
+
+      cephadm shell --mount /var/lib/system/coredump
+
+#. From within the shell session, run the following commands:
+
+   .. prompt:: bash #
+
+      dnf install ceph-debuginfo gdb zstd
+
+   .. prompt:: bash #
+      
+    unzstd /var/lib/systemd/coredump/core.ceph-*.zst
+
+   .. prompt:: bash #
+
+    gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-*.zst
+
+#. Run debugger commands at gdb's prompt:
+
+   .. prompt:: bash (gdb)
+
+      bt
+      
+   ::
+
+      #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
+      #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
+      #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
+      #3  0x0000563085ca3d7e in main ()
+
+
+Running repeated debugging sessions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When using ``cephadm shell``, as in the example above, any changes made to the
+container that is spawned by the shell command are ephemeral. After the shell
+session exits, the files that were downloaded and installed cease to be
+available. You can simply re-run the same commands every time ``cephadm
+shell`` is invoked, but in order to save time and resources one can create a
+new container image and use it for repeated debugging sessions.
+
+In the following example, we create a simple file that will construct the
+container image. The command below uses podman but it is expected to work
+correctly even if ``podman`` is replaced with ``docker``::
+
+  cat >Containerfile <<EOF
+  ARG BASE_IMG=quay.io/ceph/ceph:v18
+  FROM \${BASE_IMG}
+  # install ceph debuginfo packages, gdb and other potentially useful packages
+  RUN dnf install --enablerepo='*debug*' -y ceph-debuginfo gdb zstd strace python3-debuginfo
+  EOF
+  podman build -t ceph:debugging -f Containerfile .
+  # pass --build-arg=BASE_IMG=<your image> to customize the base image
+
+The above file creates a new local image named ``ceph:debugging``. This image
+can be used on the same machine that built it. The image can also be pushed to
+a container repository or saved and copied to a node runing other Ceph
+containers. Consult the ``podman`` or ``docker`` documentation for more
+information about the container workflow.
+
+After the image has been built, it can be used to initiate repeat debugging
+sessions. By using an image in this way, you avoid the trouble of having to
+re-install the debug tools and debuginfo packages every time you need to run a
+debug session. To debug a core file using this image, in the same way as
+previously described, run:
+
+.. prompt:: bash #
+
+    cephadm --image ceph:debugging shell --mount /var/lib/system/coredump
+
+
+Debugging live processes
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The gdb debugger can attach to running processes to debug them. This can be
+achieved with a containerized process by using the debug image and attaching it
+to the same PID namespace in which the process to be debugged resides.
+
+This requires running a container command with some custom arguments. We can
+generate a script that can debug a process in a running container.
+
+.. prompt:: bash #
+
+   cephadm --image ceph:debugging shell --dry-run > /tmp/debug.sh
+
+This creates a script that includes the container command that ``cephadm``
+would use to create a shell. Modify the script by removing the ``--init``
+argument and replace it with the argument that joins to the namespace used for
+a running running container. For example, assume we want to debug the Manager
+and have determnined that the Manager is running in a container named
+``ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``. In this case,
+the argument
+``--pid=container:ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``
+should be used.
+
+We can run our debugging container with ``sh /tmp/debug.sh``. Within the shell,
+we can run commands such as ``ps`` to get the PID of the Manager process. In
+the following example this is ``2``. While running gdb, we can attach to the
+running process:
+
+.. prompt:: bash (gdb)
+
+   attach 2
+   info threads
+   bt
diff --git a/doc/cephadm/upgrade.rst b/doc/cephadm/upgrade.rst
new file mode 100644
index 000000000..e0a9f610a
--- /dev/null
+++ b/doc/cephadm/upgrade.rst
@@ -0,0 +1,295 @@
+==============
+Upgrading Ceph
+==============
+
+Cephadm can safely upgrade Ceph from one bugfix release to the next.  For
+example, you can upgrade from v15.2.0 (the first Octopus release) to the next
+point release, v15.2.1.
+
+The automated upgrade process follows Ceph best practices.  For example:
+
+* The upgrade order starts with managers, monitors, then other daemons.
+* Each daemon is restarted only after Ceph indicates that the cluster
+  will remain available.
+
+.. note::
+
+   The Ceph cluster health status is likely to switch to
+   ``HEALTH_WARNING`` during the upgrade.
+
+.. note:: 
+
+   In case a host of the cluster is offline, the upgrade is paused.
+
+
+Starting the upgrade
+====================
+
+.. note::
+   .. note::
+      `Staggered Upgrade`_ of the mons/mgrs may be necessary to have access
+      to this new feature.
+
+   Cephadm by default reduces `max_mds` to `1`. This can be disruptive for large
+   scale CephFS deployments because the cluster cannot quickly reduce active MDS(s)
+   to `1` and a single active MDS cannot easily handle the load of all clients
+   even for a short time. Therefore, to upgrade MDS(s) without reducing `max_mds`,
+   the `fail_fs` option can to be set to `true` (default value is `false`) prior
+   to initiating the upgrade:
+
+   .. prompt:: bash #
+
+      ceph config set mgr mgr/orchestrator/fail_fs true
+
+   This would:
+               #. Fail CephFS filesystems, bringing active MDS daemon(s) to
+                  `up:standby` state.
+
+               #. Upgrade MDS daemons safely.
+
+               #. Bring CephFS filesystems back up, bringing the state of active
+                  MDS daemon(s) from `up:standby` to `up:active`.
+
+Before you use cephadm to upgrade Ceph, verify that all hosts are currently online and that your cluster is healthy by running the following command:
+
+.. prompt:: bash #
+
+   ceph -s
+
+To upgrade (or downgrade) to a specific release, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --ceph-version <version>
+
+For example, to upgrade to v16.2.6, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --ceph-version 16.2.6
+
+.. note::
+
+    From version v16.2.6 the Docker Hub registry is no longer used, so if you use Docker you have to point it to the image in the quay.io registry:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image quay.io/ceph/ceph:v16.2.6
+
+
+Monitoring the upgrade
+======================
+
+Determine (1) whether an upgrade is in progress and (2) which version the
+cluster is upgrading to by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade status
+
+Watching the progress bar during a Ceph upgrade
+-----------------------------------------------
+
+During the upgrade, a progress bar is visible in the ceph status output. It
+looks like this:
+
+.. code-block:: console
+
+  # ceph -s
+
+  [...]
+    progress:
+      Upgrade to docker.io/ceph/ceph:v15.2.1 (00h 20m 12s)
+        [=======.....................] (time remaining: 01h 43m 31s)
+
+Watching the cephadm log during an upgrade
+------------------------------------------
+
+Watch the cephadm log by running the following command:
+
+.. prompt:: bash #
+
+  ceph -W cephadm
+
+
+Canceling an upgrade
+====================
+
+You can stop the upgrade process at any time by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade stop
+
+Post upgrade actions
+====================
+
+In case the new version is based on ``cephadm``, once done with the upgrade the user
+has to update the ``cephadm`` package (or ceph-common package in case the user
+doesn't use ``cephadm shell``) to a version compatible with the new version.
+
+Potential problems
+==================
+
+There are a few health alerts that can arise during the upgrade process.
+
+UPGRADE_NO_STANDBY_MGR
+----------------------
+
+This alert (``UPGRADE_NO_STANDBY_MGR``) means that Ceph does not detect an
+active standby manager daemon. In order to proceed with the upgrade, Ceph
+requires an active standby manager daemon (which you can think of in this
+context as "a second manager").
+
+You can ensure that Cephadm is configured to run 2 (or more) managers by
+running the following command:
+
+.. prompt:: bash #
+
+  ceph orch apply mgr 2  # or more
+
+You can check the status of existing mgr daemons by running the following
+command:
+
+.. prompt:: bash #
+
+  ceph orch ps --daemon-type mgr
+
+If an existing mgr daemon has stopped, you can try to restart it by running the
+following command: 
+
+.. prompt:: bash #
+
+  ceph orch daemon restart <name>
+
+UPGRADE_FAILED_PULL
+-------------------
+
+This alert (``UPGRADE_FAILED_PULL``) means that Ceph was unable to pull the
+container image for the target version. This can happen if you specify a
+version or container image that does not exist (e.g. "1.2.3"), or if the
+container registry can not be reached by one or more hosts in the cluster.
+
+To cancel the existing upgrade and to specify a different target version, run
+the following commands: 
+
+.. prompt:: bash #
+
+  ceph orch upgrade stop
+  ceph orch upgrade start --ceph-version <version>
+
+
+Using customized container images
+=================================
+
+For most users, upgrading requires nothing more complicated than specifying the
+Ceph version number to upgrade to.  In such cases, cephadm locates the specific
+Ceph container image to use by combining the ``container_image_base``
+configuration option (default: ``docker.io/ceph/ceph``) with a tag of
+``vX.Y.Z``.
+
+But it is possible to upgrade to an arbitrary container image, if that's what
+you need. For example, the following command upgrades to a development build:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image quay.io/ceph-ci/ceph:recent-git-branch-name
+
+For more information about available container images, see :ref:`containers`.
+
+Staggered Upgrade
+=================
+
+Some users may prefer to upgrade components in phases rather than all at once.
+The upgrade command, starting in 16.2.11 and 17.2.1 allows parameters
+to limit which daemons are upgraded by a single upgrade command. The options in
+include ``daemon_types``, ``services``, ``hosts`` and ``limit``. ``daemon_types``
+takes a comma-separated list of daemon types and will only upgrade daemons of those
+types. ``services`` is mutually exclusive with ``daemon_types``, only takes services
+of one type at a time (e.g. can't provide an OSD and RGW service at the same time), and
+will only upgrade daemons belonging to those services. ``hosts`` can be combined
+with ``daemon_types`` or ``services`` or provided on its own. The ``hosts`` parameter
+follows the same format as the command line options for :ref:`orchestrator-cli-placement-spec`.
+``limit`` takes an integer > 0 and provides a numerical limit on the number of
+daemons cephadm will upgrade. ``limit`` can be combined with any of the other
+parameters. For example, if you specify to upgrade daemons of type osd on host
+Host1 with ``limit`` set to 3, cephadm will upgrade (up to) 3 osd daemons on
+Host1.
+
+Example: specifying daemon types and hosts:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --daemon-types mgr,mon --hosts host1,host2
+
+Example: specifying services and using limit:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --services rgw.example1,rgw.example2 --limit 2
+
+.. note::
+
+   Cephadm strictly enforces an order to the upgrade of daemons that is still present
+   in staggered upgrade scenarios. The current upgrade ordering is
+   ``mgr -> mon -> crash -> osd -> mds -> rgw -> rbd-mirror -> cephfs-mirror -> iscsi -> nfs``.
+   If you specify parameters that would upgrade daemons out of order, the upgrade
+   command will block and note which daemons will be missed if you proceed.
+
+.. note::
+
+  Upgrade commands with limiting parameters will validate the options before beginning the
+  upgrade, which may require pulling the new container image. Do not be surprised
+  if the upgrade start command takes a while to return when limiting parameters are provided.
+
+.. note::
+
+   In staggered upgrade scenarios (when a limiting parameter is provided) monitoring
+   stack daemons including Prometheus and node-exporter are refreshed after the Manager
+   daemons have been upgraded. Do not be surprised if Manager upgrades thus take longer
+   than expected. Note that the versions of monitoring stack daemons may not change between
+   Ceph releases, in which case they are only redeployed.
+
+Upgrading to a version that supports staggered upgrade from one that doesn't
+----------------------------------------------------------------------------
+
+While upgrading from a version that already supports staggered upgrades the process
+simply requires providing the necessary arguments. However, if you wish to upgrade
+to a version that supports staggered upgrade from one that does not, there is a
+workaround. It requires first manually upgrading the Manager daemons and then passing
+the limiting parameters as usual.
+
+.. warning::
+  Make sure you have multiple running mgr daemons before attempting this procedure.
+
+To start with, determine which Manager is your active one and which are standby. This
+can be done in a variety of ways such as looking at the ``ceph -s`` output. Then,
+manually upgrade each standby mgr daemon with:
+
+.. prompt:: bash #
+
+  ceph orch daemon redeploy mgr.example1.abcdef --image <new-image-name>
+
+.. note::
+
+   If you are on a very early version of cephadm (early Octopus) the ``orch daemon redeploy``
+   command may not have the ``--image`` flag. In that case, you must manually set the
+   Manager container image ``ceph config set mgr container_image <new-image-name>`` and then
+   redeploy the Manager ``ceph orch daemon redeploy mgr.example1.abcdef``
+
+At this point, a Manager fail over should allow us to have the active Manager be one
+running the new version.
+
+.. prompt:: bash #
+
+  ceph mgr fail
+
+Verify the active Manager is now one running the new version. To complete the Manager
+upgrading:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <new-image-name> --daemon-types mgr
+
+You should now have all your Manager daemons on the new version and be able to
+specify the limiting parameters for the rest of the upgrade.
diff --git a/doc/cephfs/add-remove-mds.rst b/doc/cephfs/add-remove-mds.rst
new file mode 100644
index 000000000..4f5ee06aa
--- /dev/null
+++ b/doc/cephfs/add-remove-mds.rst
@@ -0,0 +1,118 @@
+.. _cephfs_add_remote_mds:
+
+.. note::
+   It is highly recommended to use :doc:`/cephadm/index` or another Ceph
+   orchestrator for setting up the ceph cluster. Use this approach only if you
+   are setting up the ceph cluster manually. If one still intends to use the
+   manual way for deploying MDS daemons, :doc:`/cephadm/services/mds/` can
+   also be used.
+
+============================
+ Deploying Metadata Servers
+============================
+
+Each CephFS file system requires at least one MDS. The cluster operator will
+generally use their automated deployment tool to launch required MDS servers as
+needed.  Rook and ansible (via the ceph-ansible playbooks) are recommended
+tools for doing this. For clarity, we also show the systemd commands here which
+may be run by the deployment technology if executed on bare-metal.
+
+See `MDS Config Reference`_ for details on configuring metadata servers.
+
+
+Provisioning Hardware for an MDS
+================================
+
+The present version of the MDS is single-threaded and CPU-bound for most
+activities, including responding to client requests. An MDS under the most
+aggressive client loads uses about 2 to 3 CPU cores. This is due to the other
+miscellaneous upkeep threads working in tandem.
+
+Even so, it is recommended that an MDS server be well provisioned with an
+advanced CPU with sufficient cores. Development is on-going to make better use
+of available CPU cores in the MDS; it is expected in future versions of Ceph
+that the MDS server will improve performance by taking advantage of more cores.
+
+The other dimension to MDS performance is the available RAM for caching. The
+MDS necessarily manages a distributed and cooperative metadata cache among all
+clients and other active MDSs. Therefore it is essential to provide the MDS
+with sufficient RAM to enable faster metadata access and mutation. The default
+MDS cache size (see also :doc:`/cephfs/cache-configuration`) is 4GB. It is
+recommended to provision at least 8GB of RAM for the MDS to support this cache
+size.
+
+Generally, an MDS serving a large cluster of clients (1000 or more) will use at
+least 64GB of cache. An MDS with a larger cache is not well explored in the
+largest known community clusters; there may be diminishing returns where
+management of such a large cache negatively impacts performance in surprising
+ways. It would be best to do analysis with expected workloads to determine if
+provisioning more RAM is worthwhile.
+
+In a bare-metal cluster, the best practice is to over-provision hardware for
+the MDS server. Even if a single MDS daemon is unable to fully utilize the
+hardware, it may be desirable later on to start more active MDS daemons on the
+same node to fully utilize the available cores and memory. Additionally, it may
+become clear with workloads on the cluster that performance improves with
+multiple active MDS on the same node rather than over-provisioning a single
+MDS.
+
+Finally, be aware that CephFS is a highly-available file system by supporting
+standby MDS (see also :ref:`mds-standby`) for rapid failover. To get a real
+benefit from deploying standbys, it is usually necessary to distribute MDS
+daemons across at least two nodes in the cluster. Otherwise, a hardware failure
+on a single node may result in the file system becoming unavailable.
+
+Co-locating the MDS with other Ceph daemons (hyperconverged) is an effective
+and recommended way to accomplish this so long as all daemons are configured to
+use available hardware within certain limits.  For the MDS, this generally
+means limiting its cache size.
+
+
+Adding an MDS
+=============
+
+#. Create an mds directory ``/var/lib/ceph/mds/ceph-${id}``. The daemon only uses this directory to store its keyring.
+
+#. Create the authentication key, if you use CephX: ::
+
+	$ sudo ceph auth get-or-create mds.${id} mon 'profile mds' mgr 'profile mds' mds 'allow *' osd 'allow *' > /var/lib/ceph/mds/ceph-${id}/keyring
+
+#. Start the service: ::
+
+	$ sudo systemctl start ceph-mds@${id}
+
+#. The status of the cluster should show: ::
+
+	mds: ${id}:1 {0=${id}=up:active} 2 up:standby
+
+#. Optionally, configure the file system the MDS should join (:ref:`mds-join-fs`): ::
+
+    $ ceph config set mds.${id} mds_join_fs ${fs}
+
+
+Removing an MDS
+===============
+
+If you have a metadata server in your cluster that you'd like to remove, you may use
+the following method.
+
+#. (Optionally:) Create a new replacement Metadata Server. If there are no
+   replacement MDS to take over once the MDS is removed, the file system will
+   become unavailable to clients.  If that is not desirable, consider adding a
+   metadata server before tearing down the metadata server you would like to
+   take offline.
+
+#. Stop the MDS to be removed. ::
+
+	$ sudo systemctl stop ceph-mds@${id}
+
+   The MDS will automatically notify the Ceph monitors that it is going down.
+   This enables the monitors to perform instantaneous failover to an available
+   standby, if one exists. It is unnecessary to use administrative commands to
+   effect this failover, e.g. through the use of ``ceph mds fail mds.${id}``.
+
+#. Remove the ``/var/lib/ceph/mds/ceph-${id}`` directory on the MDS. ::
+
+	$ sudo rm -rf /var/lib/ceph/mds/ceph-${id}
+
+.. _MDS Config Reference: ../mds-config-ref
diff --git a/doc/cephfs/administration.rst b/doc/cephfs/administration.rst
new file mode 100644
index 000000000..cd912b42a
--- /dev/null
+++ b/doc/cephfs/administration.rst
@@ -0,0 +1,392 @@
+.. _cephfs-administration:
+
+CephFS Administrative commands
+==============================
+
+File Systems
+------------
+
+.. note:: The names of the file systems, metadata pools, and data pools can
+          only have characters in the set [a-zA-Z0-9\_-.].
+
+These commands operate on the CephFS file systems in your Ceph cluster.
+Note that by default only one file system is permitted: to enable
+creation of multiple file systems use ``ceph fs flag set enable_multiple true``.
+
+::
+
+    ceph fs new <file system name> <metadata pool name> <data pool name>
+
+This command creates a new file system. The file system name and metadata pool
+name are self-explanatory. The specified data pool is the default data pool and
+cannot be changed once set. Each file system has its own set of MDS daemons
+assigned to ranks so ensure that you have sufficient standby daemons available
+to accommodate the new file system.
+
+::
+
+    ceph fs ls
+
+List all file systems by name.
+
+::
+
+    ceph fs lsflags <file system name>
+
+List all the flags set on a file system.
+
+::
+
+    ceph fs dump [epoch]
+
+This dumps the FSMap at the given epoch (default: current) which includes all
+file system settings, MDS daemons and the ranks they hold, and the list of
+standby MDS daemons.
+
+
+::
+
+    ceph fs rm <file system name> [--yes-i-really-mean-it]
+
+Destroy a CephFS file system. This wipes information about the state of the
+file system from the FSMap. The metadata pool and data pools are untouched and
+must be destroyed separately.
+
+::
+
+    ceph fs get <file system name>
+
+Get information about the named file system, including settings and ranks. This
+is a subset of the same information from the ``ceph fs dump`` command.
+
+::
+
+    ceph fs set <file system name> <var> <val>
+
+Change a setting on a file system. These settings are specific to the named
+file system and do not affect other file systems.
+
+::
+
+    ceph fs add_data_pool <file system name> <pool name/id>
+
+Add a data pool to the file system. This pool can be used for file layouts
+as an alternate location to store file data.
+
+::
+
+    ceph fs rm_data_pool <file system name> <pool name/id>
+
+This command removes the specified pool from the list of data pools for the
+file system.  If any files have layouts for the removed data pool, the file
+data will become unavailable. The default data pool (when creating the file
+system) cannot be removed.
+
+::
+
+    ceph fs rename <file system name> <new file system name> [--yes-i-really-mean-it]
+
+Rename a Ceph file system. This also changes the application tags on the data
+pools and metadata pool of the file system to the new file system name.
+The CephX IDs authorized to the old file system name need to be reauthorized
+to the new name. Any on-going operations of the clients using these IDs may be
+disrupted. Mirroring is expected to be disabled on the file system.
+
+
+Settings
+--------
+
+::
+
+    ceph fs set <fs name> max_file_size <size in bytes>
+
+CephFS has a configurable maximum file size, and it's 1TB by default.
+You may wish to set this limit higher if you expect to store large files
+in CephFS. It is a 64-bit field.
+
+Setting ``max_file_size`` to 0 does not disable the limit. It would
+simply limit clients to only creating empty files.
+
+
+Maximum file sizes and performance
+----------------------------------
+
+CephFS enforces the maximum file size limit at the point of appending to
+files or setting their size. It does not affect how anything is stored.
+
+When users create a file of an enormous size (without necessarily
+writing any data to it), some operations (such as deletes) cause the MDS
+to have to do a large number of operations to check if any of the RADOS
+objects within the range that could exist (according to the file size)
+really existed.
+
+The ``max_file_size`` setting prevents users from creating files that
+appear to be eg. exabytes in size, causing load on the MDS as it tries
+to enumerate the objects during operations like stats or deletes.
+
+
+Taking the cluster down
+-----------------------
+
+Taking a CephFS cluster down is done by setting the down flag:
+ 
+:: 
+ 
+    ceph fs set <fs_name> down true
+ 
+To bring the cluster back online:
+ 
+:: 
+
+    ceph fs set <fs_name> down false
+
+This will also restore the previous value of max_mds. MDS daemons are brought
+down in a way such that journals are flushed to the metadata pool and all
+client I/O is stopped.
+
+
+Taking the cluster down rapidly for deletion or disaster recovery
+-----------------------------------------------------------------
+
+To allow rapidly deleting a file system (for testing) or to quickly bring the
+file system and MDS daemons down, use the ``ceph fs fail`` command:
+
+::
+
+    ceph fs fail <fs_name>
+
+This command sets a file system flag to prevent standbys from
+activating on the file system (the ``joinable`` flag).
+
+This process can also be done manually by doing the following:
+
+::
+
+    ceph fs set <fs_name> joinable false
+
+Then the operator can fail all of the ranks which causes the MDS daemons to
+respawn as standbys. The file system will be left in a degraded state.
+
+::
+
+    # For all ranks, 0-N:
+    ceph mds fail <fs_name>:<n>
+
+Once all ranks are inactive, the file system may also be deleted or left in
+this state for other purposes (perhaps disaster recovery).
+
+To bring the cluster back up, simply set the joinable flag:
+
+::
+
+    ceph fs set <fs_name> joinable true
+
+
+Daemons
+-------
+
+Most commands manipulating MDSs take a ``<role>`` argument which can take one
+of three forms:
+
+::
+
+    <fs_name>:<rank>
+    <fs_id>:<rank>
+    <rank>
+
+Commands to manipulate MDS daemons:
+
+::
+
+    ceph mds fail <gid/name/role>
+
+Mark an MDS daemon as failed.  This is equivalent to what the cluster
+would do if an MDS daemon had failed to send a message to the mon
+for ``mds_beacon_grace`` second.  If the daemon was active and a suitable
+standby is available, using ``ceph mds fail`` will force a failover to the
+standby.
+
+If the MDS daemon was in reality still running, then using ``ceph mds fail``
+will cause the daemon to restart.  If it was active and a standby was
+available, then the "failed" daemon will return as a standby.
+
+
+::
+
+    ceph tell mds.<daemon name> command ...
+
+Send a command to the MDS daemon(s). Use ``mds.*`` to send a command to all
+daemons. Use ``ceph tell mds.* help`` to learn available commands.
+
+::
+
+    ceph mds metadata <gid/name/role>
+
+Get metadata about the given MDS known to the Monitors.
+
+::
+
+    ceph mds repaired <role>
+
+Mark the file system rank as repaired. Unlike the name suggests, this command
+does not change a MDS; it manipulates the file system rank which has been
+marked damaged.
+
+
+Required Client Features
+------------------------
+
+It is sometimes desirable to set features that clients must support to talk to
+CephFS. Clients without those features may disrupt other clients or behave in
+surprising ways. Or, you may want to require newer features to prevent older
+and possibly buggy clients from connecting.
+
+Commands to manipulate required client features of a file system:
+
+::
+
+    ceph fs required_client_features <fs name> add reply_encoding
+    ceph fs required_client_features <fs name> rm reply_encoding
+
+To list all CephFS features
+
+::
+
+    ceph fs feature ls
+
+Clients that are missing newly added features will be evicted automatically.
+
+Here are the current CephFS features and first release they came out:
+
++------------------+--------------+-----------------+
+| Feature          | Ceph release | Upstream Kernel |
++==================+==============+=================+
+| jewel            | jewel        | 4.5             |
++------------------+--------------+-----------------+
+| kraken           | kraken       | 4.13            |
++------------------+--------------+-----------------+
+| luminous         | luminous     | 4.13            |
++------------------+--------------+-----------------+
+| mimic            | mimic        | 4.19            |
++------------------+--------------+-----------------+
+| reply_encoding   | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| reclaim_client   | nautilus     | N/A             |
++------------------+--------------+-----------------+
+| lazy_caps_wanted | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| multi_reconnect  | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| deleg_ino        | octopus      | 5.6             |
++------------------+--------------+-----------------+
+| metric_collect   | pacific      | N/A             |
++------------------+--------------+-----------------+
+| alternate_name   | pacific      | PLANNED         |
++------------------+--------------+-----------------+
+
+CephFS Feature Descriptions
+
+
+::
+
+    reply_encoding
+
+MDS encodes request reply in extensible format if client supports this feature.
+
+
+::
+
+    reclaim_client
+
+MDS allows new client to reclaim another (dead) client's states. This feature
+is used by NFS-Ganesha.
+
+
+::
+
+    lazy_caps_wanted
+
+When a stale client resumes, if the client supports this feature, mds only needs
+to re-issue caps that are explicitly wanted.
+
+
+::
+
+    multi_reconnect
+
+When mds failover, client sends reconnect messages to mds, to reestablish cache
+states. If MDS supports this feature, client can split large reconnect message
+into multiple ones.
+
+
+::
+
+    deleg_ino
+
+MDS delegate inode numbers to client if client supports this feature. Having
+delegated inode numbers is a prerequisite for client to do async file creation.
+
+
+::
+
+    metric_collect
+
+Clients can send performance metric to MDS if MDS support this feature.
+
+::
+
+    alternate_name
+
+Clients can set and understand "alternate names" for directory entries. This is
+to be used for encrypted file name support.
+
+
+Global settings
+---------------
+
+
+::
+
+    ceph fs flag set <flag name> <flag val> [<confirmation string>]
+
+Sets a global CephFS flag (i.e. not specific to a particular file system).
+Currently, the only flag setting is 'enable_multiple' which allows having
+multiple CephFS file systems.
+
+Some flags require you to confirm your intentions with "--yes-i-really-mean-it"
+or a similar string they will prompt you with. Consider these actions carefully
+before proceeding; they are placed on especially dangerous activities.
+
+.. _advanced-cephfs-admin-settings:
+
+Advanced
+--------
+
+These commands are not required in normal operation, and exist
+for use in exceptional circumstances.  Incorrect use of these
+commands may cause serious problems, such as an inaccessible
+file system.
+
+::
+
+    ceph mds rmfailed
+
+This removes a rank from the failed set.
+
+::
+
+    ceph fs reset <file system name>
+
+This command resets the file system state to defaults, except for the name and
+pools. Non-zero ranks are saved in the stopped set.
+
+
+::
+
+    ceph fs new <file system name> <metadata pool name> <data pool name> --fscid <fscid> --force
+
+This command creates a file system with a specific **fscid** (file system cluster ID).
+You may want to do this when an application expects the file system's ID to be
+stable after it has been recovered, e.g., after monitor databases are lost and
+rebuilt. Consequently, file system IDs don't always keep increasing with newer
+file systems.
diff --git a/doc/cephfs/api/index.rst b/doc/cephfs/api/index.rst
new file mode 100644
index 000000000..bb6250200
--- /dev/null
+++ b/doc/cephfs/api/index.rst
@@ -0,0 +1,11 @@
+.. _cephfs api:
+
+============
+ CephFS APIs
+============
+
+.. toctree::
+   :maxdepth: 2
+
+   libcephfs (Java) <libcephfs-java>
+   libcephfs (Python) <libcephfs-py>
diff --git a/doc/cephfs/api/libcephfs-java.rst b/doc/cephfs/api/libcephfs-java.rst
new file mode 100644
index 000000000..83b5a6638
--- /dev/null
+++ b/doc/cephfs/api/libcephfs-java.rst
@@ -0,0 +1,18 @@
+===================
+Libcephfs (JavaDoc)
+===================
+
+.. warning::
+
+    CephFS Java bindings are no longer tested by CI. They may not work properly
+    or corrupt data.
+
+    Developers interested in reviving these bindings by fixing and writing tests
+    are encouraged to contribute!
+
+..
+    The admin/build-docs script runs Ant to build the JavaDoc files, and
+    copies them to api/libcephfs-java/javadoc/.
+
+
+View the auto-generated `JavaDoc pages for the CephFS Java bindings <javadoc/>`_.
diff --git a/doc/cephfs/api/libcephfs-py.rst b/doc/cephfs/api/libcephfs-py.rst
new file mode 100644
index 000000000..039401f22
--- /dev/null
+++ b/doc/cephfs/api/libcephfs-py.rst
@@ -0,0 +1,13 @@
+===================
+ LibCephFS (Python)
+===================
+
+.. highlight:: python
+
+The `cephfs` python module provides access to CephFS service.
+
+API calls
+=========
+
+.. automodule:: cephfs
+    :members: DirEntry, DirResult, LibCephFS
diff --git a/doc/cephfs/app-best-practices.rst b/doc/cephfs/app-best-practices.rst
new file mode 100644
index 000000000..50bd3b689
--- /dev/null
+++ b/doc/cephfs/app-best-practices.rst
@@ -0,0 +1,82 @@
+
+Application best practices for distributed file systems
+=======================================================
+
+CephFS is POSIX compatible, and therefore should work with any existing
+applications that expect a POSIX file system.  However, because it is a
+network file system (unlike e.g. XFS) and it is highly consistent (unlike
+e.g. NFS), there are some consequences that application authors may
+benefit from knowing about.
+
+The following sections describe some areas where distributed file systems
+may have noticeably different performance behaviours compared with
+local file systems.
+
+
+ls -l
+-----
+
+When you run "ls -l", the ``ls`` program
+is first doing a directory listing, and then calling ``stat`` on every
+file in the directory.
+
+This is usually far in excess of what an application really needs, and
+it can be slow for large directories.  If you don't really need all
+this metadata for each file, then use a plain ``ls``.
+
+ls/stat on files being extended
+-------------------------------
+
+If another client is currently extending files in the listed directory,
+then an ``ls -l`` may take an exceptionally long time to complete, as
+the lister must wait for the writer to flush data in order to do a valid
+read of the every file's size.  So unless you *really* need to know the
+exact size of every file in the directory, just don't do it!
+
+This would also apply to any application code that was directly
+issuing ``stat`` system calls on files being appended from
+another node.
+
+Very large directories
+----------------------
+
+Do you really need that 10,000,000 file directory?  While directory
+fragmentation enables CephFS to handle it, it is always going to be
+less efficient than splitting your files into more modest-sized directories.
+
+Even standard userspace tools can become quite slow when operating on very
+large directories. For example, the default behaviour of ``ls``
+is to give an alphabetically ordered result, but ``readdir`` system
+calls do not give an ordered result (this is true in general, not just
+with CephFS).  So when you ``ls`` on a million file directory, it is
+loading a list of a million names into memory, sorting the list, then writing
+it out to the display.
+
+Hard links
+----------
+
+Hard links have an intrinsic cost in terms of the internal housekeeping
+that a file system has to do to keep two references to the same data.  In
+CephFS there is a particular performance cost, because with normal files
+the inode is embedded in the directory (i.e. there is no extra fetch of
+the inode after looking up the path).
+
+Working set size
+----------------
+
+The MDS acts as a cache for the metadata stored in RADOS.  Metadata
+performance is very different for workloads whose metadata fits within
+that cache.
+
+If your workload has more files than fit in your cache (configured using 
+``mds_cache_memory_limit`` settings), then make sure you test it
+appropriately: don't test your system with a small number of files and then
+expect equivalent performance when you move to a much larger number of files.
+
+Do you need a file system?
+--------------------------
+
+Remember that Ceph also includes an object storage interface.  If your
+application needs to store huge flat collections of files where you just
+read and write whole files at once, then you might well be better off
+using the :ref:`Object Gateway <object-gateway>`
diff --git a/doc/cephfs/cache-configuration.rst b/doc/cephfs/cache-configuration.rst
new file mode 100644
index 000000000..3fc757005
--- /dev/null
+++ b/doc/cephfs/cache-configuration.rst
@@ -0,0 +1,211 @@
+=======================
+MDS Cache Configuration
+=======================
+
+The Metadata Server coordinates a distributed cache among all MDS and CephFS
+clients. The cache serves to improve metadata access latency and allow clients
+to safely (coherently) mutate metadata state (e.g. via `chmod`). The MDS issues
+**capabilities** and **directory entry leases** to indicate what state clients
+may cache and what manipulations clients may perform (e.g. writing to a file).
+
+The MDS and clients both try to enforce a cache size. The mechanism for
+specifying the MDS cache size is described below. Note that the MDS cache size
+is not a hard limit. The MDS always allows clients to lookup new metadata
+which is loaded into the cache. This is an essential policy as it avoids
+deadlock in client requests (some requests may rely on held capabilities before
+capabilities are released).
+
+When the MDS cache is too large, the MDS will **recall** client state so cache
+items become unpinned and eligible to be dropped. The MDS can only drop cache
+state when no clients refer to the metadata to be dropped. Also described below
+is how to configure the MDS recall settings for your workload's needs. This is
+necessary if the internal throttles on the MDS recall can not keep up with the
+client workload.
+
+
+MDS Cache Size
+--------------
+
+You can limit the size of the Metadata Server (MDS) cache by a byte count. This
+is done through the `mds_cache_memory_limit` configuration:
+
+.. confval:: mds_cache_memory_limit
+
+In addition, you can specify a cache reservation by using the
+`mds_cache_reservation` parameter for MDS operations:
+
+.. confval:: mds_cache_reservation
+
+The cache reservation is
+limited as a percentage of the memory and is set to 5% by default. The intent
+of this parameter is to have the MDS maintain an extra reserve of memory for
+its cache for new metadata operations to use. As a consequence, the MDS should
+in general operate below its memory limit because it will recall old state from
+clients in order to drop unused metadata in its cache.
+
+If the MDS cannot keep its cache under the target size, the MDS will send a
+health alert to the Monitors indicating the cache is too large. This is
+controlled by the `mds_health_cache_threshold` configuration which is by
+default 150% of the maximum cache size:
+
+.. confval:: mds_health_cache_threshold
+
+Because the cache limit is not a hard limit, potential bugs in the CephFS
+client, MDS, or misbehaving applications might cause the MDS to exceed its
+cache size. The health warnings are intended to help the operator detect this
+situation and make necessary adjustments or investigate buggy clients.
+
+MDS Cache Trimming
+------------------
+
+There are two configurations for throttling the rate of cache trimming in the MDS:
+
+.. confval:: mds_cache_trim_threshold
+
+.. confval:: mds_cache_trim_decay_rate
+
+The intent of the throttle is to prevent the MDS from spending too much time
+trimming its cache. This may limit its ability to handle client requests or
+perform other upkeep.
+
+The trim configurations control an internal **decay counter**. Anytime metadata
+is trimmed from the cache, the counter is incremented.  The threshold sets the
+maximum size of the counter while the decay rate indicates the exponential half
+life for the counter. If the MDS is continually removing items from its cache,
+it will reach a steady state of ``-ln(0.5)/rate*threshold`` items removed per
+second.
+
+.. note:: Increasing the value of the configuration setting
+          ``mds_cache_trim_decay_rate`` leads to the MDS spending less time
+          trimming the cache. To increase the cache trimming rate, set a lower
+          value.
+
+The defaults are conservative and may need to be changed for production MDS with
+large cache sizes.
+
+
+MDS Recall
+----------
+
+MDS limits its recall of client state (capabilities/leases) to prevent creating
+too much work for itself handling release messages from clients. This is controlled
+via the following configurations:
+
+
+The maximum number of capabilities to recall from a single client in a given recall
+event:
+
+.. confval:: mds_recall_max_caps
+
+The threshold and decay rate for the decay counter on a session:
+
+.. confval:: mds_recall_max_decay_threshold
+
+.. confval:: mds_recall_max_decay_rate
+
+The session decay counter controls the rate of recall for an individual
+session. The behavior of the counter works the same as for cache trimming
+above. Each capability that is recalled increments the counter.
+
+There is also a global decay counter that throttles for all session recall:
+
+.. confval:: mds_recall_global_max_decay_threshold
+
+its decay rate is the same as ``mds_recall_max_decay_rate``. Any recalled
+capability for any session also increments this counter.
+
+If clients are slow to release state, the warning "failing to respond to cache
+pressure" or ``MDS_HEALTH_CLIENT_RECALL`` will be reported. Each session's rate
+of release is monitored by another decay counter configured by:
+
+.. confval:: mds_recall_warning_threshold
+
+.. confval:: mds_recall_warning_decay_rate
+
+Each time a capability is released, the counter is incremented.  If clients do
+not release capabilities quickly enough and there is cache pressure, the
+counter will indicate if the client is slow to release state.
+
+Some workloads and client behaviors may require faster recall of client state
+to keep up with capability acquisition. It is recommended to increase the above
+counters as needed to resolve any slow recall warnings in the cluster health
+state.
+
+
+MDS Cap Acquisition Throttle
+----------------------------
+
+A trivial "find" command on a large directory hierarchy will cause the client
+to receive caps significantly faster than it will release. The MDS will try
+to have the client reduce its caps below the ``mds_max_caps_per_client`` limit
+but the recall throttles prevent it from catching up to the pace of acquisition.
+So the readdir is throttled to control cap acquisition via the following
+configurations:
+
+
+The threshold and decay rate for the readdir cap acquisition decay counter:
+
+.. confval:: mds_session_cap_acquisition_throttle
+
+.. confval:: mds_session_cap_acquisition_decay_rate
+
+The cap acquisition decay counter controls the rate of cap acquisition via
+readdir. The behavior of the decay counter is the same as for cache trimming or
+caps recall. Each readdir call increments the counter by the number of files in
+the result.
+
+.. confval:: mds_session_max_caps_throttle_ratio
+
+.. confval:: mds_cap_acquisition_throttle_retry_request_timeout
+
+If the number of caps acquired by the client per session is greater than the
+``mds_session_max_caps_throttle_ratio`` and cap acquisition decay counter is
+greater than ``mds_session_cap_acquisition_throttle``, the readdir is throttled.
+The readdir request is retried after ``mds_cap_acquisition_throttle_retry_request_timeout``
+seconds.
+
+
+Session Liveness
+----------------
+
+The MDS also keeps track of whether sessions are quiescent. If a client session
+is not utilizing its capabilities or is otherwise quiet, the MDS will begin
+recalling state from the session even if it's not under cache pressure. This
+helps the MDS avoid future work when the cluster workload is hot and cache
+pressure is forcing the MDS to recall state. The expectation is that a client
+not utilizing its capabilities is unlikely to use those capabilities anytime
+in the near future.
+
+Determining whether a given session is quiescent is controlled by the following
+configuration variables:
+
+.. confval:: mds_session_cache_liveness_magnitude
+
+.. confval:: mds_session_cache_liveness_decay_rate
+
+The configuration ``mds_session_cache_liveness_decay_rate`` indicates the
+half-life for the decay counter tracking the use of capabilities by the client.
+Each time a client manipulates or acquires a capability, the MDS will increment
+the counter. This is a rough but effective way to monitor the utilization of the
+client cache.
+
+The ``mds_session_cache_liveness_magnitude`` is a base-2 magnitude difference
+of the liveness decay counter and the number of capabilities outstanding for
+the session. So if the client has ``1*2^20`` (1M) capabilities outstanding and
+only uses **less** than ``1*2^(20-mds_session_cache_liveness_magnitude)`` (1K
+using defaults), the MDS will consider the client to be quiescent and begin
+recall.
+
+
+Capability Limit
+----------------
+
+The MDS also tries to prevent a single client from acquiring too many
+capabilities. This helps prevent recovery from taking a long time in some
+situations.  It is not generally necessary for a client to have such a large
+cache. The limit is configured via:
+
+.. confval:: mds_max_caps_per_client
+
+It is not recommended to set this value above 5M but it may be helpful with
+some workloads.
diff --git a/doc/cephfs/capabilities.rst b/doc/cephfs/capabilities.rst
new file mode 100644
index 000000000..79dfd3ace
--- /dev/null
+++ b/doc/cephfs/capabilities.rst
@@ -0,0 +1,182 @@
+======================
+Capabilities in CephFS
+======================
+When a client wants to operate on an inode, it will query the MDS in various
+ways, which will then grant the client a set of **capabilities**. This
+grants the client permissions to operate on the inode in various ways. One
+of the major differences from other network file systems (e.g NFS or SMB) is
+that the capabilities granted are quite granular, and it's possible that
+multiple clients can hold different capabilities on the same inodes.
+
+Types of Capabilities
+---------------------
+There are several "generic" capability bits. These denote what sort of ability
+the capability grants.
+
+::
+
+        /* generic cap bits */
+        #define CEPH_CAP_GSHARED     1  /* (metadata) client can read (s) */
+        #define CEPH_CAP_GEXCL       2  /* (metadata) client can read and update (x) */
+        #define CEPH_CAP_GCACHE      4  /* (file) client can cache reads (c) */
+        #define CEPH_CAP_GRD         8  /* (file) client can read (r) */
+        #define CEPH_CAP_GWR        16  /* (file) client can write (w) */
+        #define CEPH_CAP_GBUFFER    32  /* (file) client can buffer writes (b) */
+        #define CEPH_CAP_GWREXTEND  64  /* (file) client can extend EOF (a) */
+        #define CEPH_CAP_GLAZYIO   128  /* (file) client can perform lazy io (l) */
+
+These are then shifted by a particular number of bits. These denote a part of
+the inode's data or metadata on which the capability is being granted:
+
+::
+
+        /* per-lock shift */
+        #define CEPH_CAP_SAUTH      2 /* A */
+        #define CEPH_CAP_SLINK      4 /* L */
+        #define CEPH_CAP_SXATTR     6 /* X */
+        #define CEPH_CAP_SFILE      8 /* F */
+
+Only certain generic cap types are ever granted for some of those "shifts",
+however. In particular, only the FILE shift ever has more than the first two
+bits.
+
+::
+
+        | AUTH | LINK | XATTR | FILE
+        2      4      6       8
+
+From the above, we get a number of constants, that are generated by taking
+each bit value and shifting to the correct bit in the word:
+
+::
+
+        #define CEPH_CAP_AUTH_SHARED  (CEPH_CAP_GSHARED  << CEPH_CAP_SAUTH)
+
+These bits can then be or'ed together to make a bitmask denoting a set of
+capabilities.
+
+There is one exception:
+
+::
+
+        #define CEPH_CAP_PIN  1  /* no specific capabilities beyond the pin */
+
+The "pin" just pins the inode into memory, without granting any other caps.
+
+Graphically:
+
+::
+
+    +---+---+---+---+---+---+---+---+
+    | p | _ |As   x |Ls   x |Xs   x |
+    +---+---+---+---+---+---+---+---+
+    |Fs   x   c   r   w   b   a   l |
+    +---+---+---+---+---+---+---+---+
+
+The second bit is currently unused.
+
+Abilities granted by each cap
+-----------------------------
+While that is how capabilities are granted (and communicated), the important
+bit is what they actually allow the client to do:
+
+* **PIN**: This just pins the inode into memory. This is sufficient to allow
+  the client to get to the inode number, as well as other immutable things like
+  major or minor numbers in a device inode, or symlink contents.
+
+* **AUTH**: This grants the ability to get to the authentication-related metadata.
+  In particular, the owner, group and mode. Note that doing a full permission
+  check may require getting at ACLs as well, which are stored in xattrs.
+
+* **LINK**: The link count of the inode.
+
+* **XATTR**: Ability to access or manipulate xattrs. Note that since ACLs are
+  stored in xattrs, it's also sometimes necessary to access them when checking
+  permissions.
+
+* **FILE**: This is the big one. This allows the client to access and manipulate
+  file data. It also covers certain metadata relating to file data -- the
+  size, mtime, atime and ctime, in particular.
+
+Shorthand
+---------
+Note that the client logging can also present a compact representation of the
+capabilities. For example:
+
+::
+
+        pAsLsXsFs
+
+The 'p' represents the pin. Each capital letter corresponds to the shift
+values, and the lowercase letters after each shift are for the actual
+capabilities granted in each shift.
+
+The relation between the lock states and the capabilities
+---------------------------------------------------------
+In MDS there are four different locks for each inode, they are simplelock,
+scatterlock, filelock and locallock. Each lock has several different lock
+states, and the MDS will issue capabilities to clients based on the lock
+state.
+
+In each state the MDS Locker will always try to issue all the capabilities to the
+clients allowed, even some capabilities are not needed or wanted by the clients,
+as pre-issuing capabilities could reduce latency in some cases.
+
+If there is only one client, usually it will be the loner client for all the inodes.
+While in multiple clients case, the MDS will try to calculate a loner client out for
+each inode depending on the capabilities the clients (needed | wanted), but usually
+it will fail. The loner client will always get all the capabilities.
+
+The filelock will control files' partial metadatas' and the file contents' access
+permissions. The metadatas include **mtime**, **atime**, **size**, etc.
+
+* **Fs**: Once a client has it, all other clients are denied **Fw**.
+
+* **Fx**: Only the loner client is allowed this capability. Once the lock state
+  transitions to LOCK_EXCL, the loner client is granted this along with all other
+  file capabilities except the **Fl**.
+
+* **Fr**: Once a client has it, the **Fb** capability will be already revoked from
+  all the other clients.
+
+  If clients only request to read the file, the lock state will be transferred
+  to LOCK_SYNC stable state directly. All the clients can be granted **Fscrl**
+  capabilities from the auth MDS and **Fscr** capabilities from the replica MDSes.
+
+  If multiple clients read from and write to the same file, then the lock state
+  will be transferred to LOCK_MIX stable state finally and all the clients could
+  have the **Frwl** capabilities from the auth MDS, and the **Fr** from the replica
+  MDSes. The **Fcb** capabilities won't be granted to all the clients and the
+  clients will do sync read/write.
+
+* **Fw**: If there is no loner client and once a client have this capability, the
+  **Fsxcb** capabilities won't be granted to other clients.
+
+  If multiple clients read from and write to the same file, then the lock state
+  will be transferred to LOCK_MIX stable state finally and all the clients could
+  have the **Frwl** capabilities from the auth MDS, and the **Fr** from the replica
+  MDSes. The **Fcb** capabilities won't be granted to all the clients and the
+  clients will do sync read/write.
+
+* **Fc**: This capability means the clients could cache file read and should be
+  issued together with **Fr** capability and only in this use case will it make
+  sense.
+
+  While actually in some stable or interim transitional states they tend to keep
+  the **Fc** allowed even the **Fr** capability isn't granted as this can avoid
+  forcing clients to drop full caches, for example on a simple file size extension
+  or truncating use case.
+
+* **Fb**: This capability means the clients could buffer file write and should be
+  issued together with **Fw** capability and only in this use case will it make
+  sense.
+
+  While actually in some stable or interim transitional states they tend to keep
+  the **Fc** allowed even the **Fw** capability isn't granted as this can avoid
+  forcing clients to drop dirty buffers, for example on a simple file size extension
+  or truncating use case.
+
+* **Fl**: This capability means the clients could perform lazy io. LazyIO relaxes
+  POSIX semantics. Buffered reads/writes are allowed even when a file is opened by
+  multiple applications on multiple clients. Applications are responsible for managing
+  cache coherency themselves.
diff --git a/doc/cephfs/ceph-dokan.rst b/doc/cephfs/ceph-dokan.rst
new file mode 100644
index 000000000..b9fb6c592
--- /dev/null
+++ b/doc/cephfs/ceph-dokan.rst
@@ -0,0 +1,102 @@
+.. _ceph-dokan:
+
+=======================
+Mount CephFS on Windows
+=======================
+
+``ceph-dokan`` can be used for mounting CephFS filesystems on Windows.
+It leverages Dokany, a Windows driver that allows implementing filesystems in
+userspace, pretty much like FUSE.
+
+Please check the `installation guide`_ to get started.
+
+Usage
+=====
+
+Mounting filesystems
+--------------------
+
+In order to mount a ceph filesystem, the following command can be used::
+
+    ceph-dokan.exe -c c:\ceph.conf -l x
+
+This will mount the default ceph filesystem using the drive letter ``x``.
+If ``ceph.conf`` is placed at the default location, which is
+``%ProgramData%\ceph\ceph.conf``, then this argument becomes optional.
+
+The ``-l`` argument also allows using an empty folder as a mountpoint
+instead of a drive letter.
+
+The uid and gid used for mounting the filesystem default to 0 and may be
+changed using the following ``ceph.conf`` options::
+
+    [client]
+    # client_permissions = true
+    client_mount_uid = 1000
+    client_mount_gid = 1000
+
+If you have more than one FS on your Ceph cluster, use the option
+``--client_fs`` to mount the non-default FS::
+
+    mkdir -Force C:\mnt\mycephfs2
+    ceph-dokan.exe --mountpoint C:\mnt\mycephfs2 --client_fs mycephfs2
+
+CephFS subdirectories can be mounted using the ``--root-path`` parameter::
+
+    ceph-dokan -l y --root-path /a
+
+If the ``-o --removable`` flags are set, the mounts will show up in the
+``Get-Volume`` results::
+
+    PS C:\> Get-Volume -FriendlyName "Ceph*" | `
+            Select-Object -Property @("DriveLetter", "Filesystem", "FilesystemLabel")
+
+    DriveLetter Filesystem FilesystemLabel
+    ----------- ---------- ---------------
+              Z Ceph       Ceph
+              W Ceph       Ceph - new_fs
+
+Please use ``ceph-dokan --help`` for a full list of arguments.
+
+Credentials
+-----------
+
+The ``--id`` option passes the name of the CephX user whose keyring we intend to
+use for mounting CephFS. The following commands are equivalent::
+
+    ceph-dokan --id foo -l x
+    ceph-dokan --name client.foo -l x
+
+Unmounting filesystems
+----------------------
+
+The mount can be removed by either issuing ctrl-c or using the unmap command,
+like so::
+
+    ceph-dokan.exe unmap -l x
+
+Note that when unmapping Ceph filesystems, the exact same mountpoint argument
+must be used as when the mapping was created.
+
+Limitations
+-----------
+
+Be aware that Windows ACLs are ignored. Posix ACLs are supported but cannot be
+modified using the current CLI. In the future, we may add some command actions
+to change file ownership or permissions.
+
+Another thing to note is that cephfs doesn't support mandatory file locks, which
+Windows is heavily rely upon. At the moment, we're letting Dokan handle file
+locks, which are only enforced locally.
+
+Unlike ``rbd-wnbd``, ``ceph-dokan`` doesn't currently provide a ``service``
+command. In order for the cephfs mount to survive host reboots, consider using
+``NSSM``.
+
+Troubleshooting
+===============
+
+Please consult the `Windows troubleshooting`_ page.
+
+.. _Windows troubleshooting: ../../install/windows-troubleshooting
+.. _installation guide: ../../install/windows-install
diff --git a/doc/cephfs/cephfs-architecture.svg b/doc/cephfs/cephfs-architecture.svg
new file mode 100644
index 000000000..e44c52a3f
--- /dev/null
+++ b/doc/cephfs/cephfs-architecture.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 784.4199475065617 481.56692913385825" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l784.4199 0l0 481.56693l-784.4199 0l0 -481.56693z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l784.4199 0l0 481.56693l-784.4199 0z" fill-rule="evenodd"/><path fill="#b6d7a8" d="m30.272966 79.10533l0 0c0 -31.388912 24.684267 -56.83465 55.133858 -56.83465l0 0c14.622414 0 28.64592 5.987919 38.985527 16.646484c10.3396 10.658562 16.14833 25.114677 16.14833 40.188164l0 0c0 31.388908 -24.684265 56.83464 -55.133858 56.83464l0 0c-30.449589 0 -55.133858 -25.445732 -55.133858 -56.83464z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m30.272966 79.10533l0 0c0 -31.388912 24.684267 -56.83465 55.133858 -56.83465l0 0c14.622414 0 28.64592 5.987919 38.985527 16.646484c10.3396 10.658562 16.14833 25.114677 16.14833 40.188164l0 0c0 31.388908 -24.684265 56.83464 -55.133858 56.83464l0 0c-30.449589 0 -55.133858 -25.445732 -55.133858 -56.83464z" fill-rule="evenodd"/><path fill="#000000" d="m62.688076 85.08783q0.6875 0 1.3124962 -0.375q0.640625 -0.390625 1.03125 -1.03125l0.96875 0.625q-0.578125 0.953125 -1.359375 1.4375q-0.7812462 0.46875 -1.9062462 0.46875q-1.296875 0 -2.3125 -0.640625q-1.015625 -0.640625 -1.625 -1.96875q-0.609375 -1.328125 -0.609375 -3.328125q0 -2.140625 0.65625 -3.46875q0.65625 -1.34375 1.65625 -1.921875q1.0 -0.578125 2.171875 -0.578125q1.21875 0 2.1249962 0.625q0.90625 0.609375 1.375 1.671875l-1.125 0.515625q-0.015625 0 -0.015625 0q0 -0.015625 0 -0.015625q-0.484375 -0.96875 -1.0781212 -1.359375q-0.59375 -0.390625 -1.328125 -0.390625q-1.46875 0 -2.328125 1.28125q-0.84375 1.28125 -0.84375 3.546875q0 1.4375 0.421875 2.5625q0.4375 1.109375 1.171875 1.734375q0.734375 0.609375 1.640625 0.609375zm2.1406212 -8.015625q0.03125 -0.046875 0.03125 -0.046875q0.046875 0 0.15625 0.125l-0.125 0.046875l-0.0625 -0.125zm0.21875 0.046875q0.078125 0.171875 -0.03125 0.03125l0.03125 -0.03125zm3.171875 8.90625l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0zm9.71875 0l0 -1.078125l2.1875076 0l0 -6.359375l-2.0625076 0l0 -1.09375l3.4062576 0l0 7.453125l2.0 0l0 1.078125l-5.5312576 0zm2.7812576 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm9.765625 10.5q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0zm3.734375 -3.421875l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm16.265625 7.75q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625z" fill-rule="nonzero"/><path fill="#e06666" d="m245.87401 93.125015l0 0c0 -33.441494 27.109695 -60.55118 60.551193 -60.55118l0 0c16.059174 0 31.460602 6.3794823 42.81613 17.735031c11.35556 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.551178 60.551178l0 0c-33.441498 0 -60.551193 -27.109688 -60.551193 -60.551178z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m245.87401 93.125015l0 0c0 -33.441494 27.109695 -60.55118 60.551193 -60.55118l0 0c16.059174 0 31.460602 6.3794823 42.81613 17.735031c11.35556 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.551178 60.551178l0 0c-33.441498 0 -60.551193 -27.109688 -60.551193 -60.551178z" fill-rule="evenodd"/><path fill="#000000" d="m286.12833 89.04501l-1.171875 -3.390625l-3.75 0l-1.1875 3.390625l-1.28125 0l4.296875 -11.828125l0.140625 0l4.296875 11.828125l-1.34375 0zm-4.578125 -4.40625l3.0625 0l-1.53125 -4.4375l-1.53125 4.4375zm11.609375 3.40625q1.28125 0 2.21875 -1.046875l0.78125 0.90625q-1.25 1.34375 -3.078125 1.34375q-1.234375 0 -2.203125 -0.578125q-0.96875 -0.578125 -1.515625 -1.59375q-0.546875 -1.015625 -0.546875 -2.28125q0 -1.265625 0.546875 -2.265625q0.546875 -1.015625 1.515625 -1.59375q0.96875 -0.578125 2.1875 -0.578125q1.03125 0 1.859375 0.421875q0.84375 0.40625 1.375 1.15625l-0.84375 0.828125l-0.015625 0.015625q-0.546875 -0.71875 -1.125 -1.0q-0.5625 -0.296875 -1.390625 -0.296875q-0.734375 0 -1.359375 0.40625q-0.625 0.40625 -1.0 1.140625q-0.375 0.71875 -0.375 1.671875q0 0.953125 0.375 1.71875q0.390625 0.765625 1.0625 1.203125q0.6875 0.421875 1.53125 0.421875zm2.078125 -5.25q0 -0.109375 0.15625 0.015625l-0.0625 0.078125l-0.09375 -0.09375zm0.203125 -0.015625q0.09375 0.125 0.046875 0.09375q-0.046875 -0.046875 -0.09375 -0.0625l0.046875 -0.03125zm9.9375 5.484375q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm2.90625 0.78125l0 -1.078125l2.1875 0l0 -6.359375l-2.0625 0l0 -1.09375l3.40625 0l0 7.453125l2.0 0l0 1.078125l-5.53125 0zm2.78125 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm13.171875 1.78125q-0.296875 1.515625 -1.25 3.734375l-2.0625 4.796875l-1.046875 0l-3.375 -8.53125l1.34375 0l2.625 6.6875l1.375 -3.15625q0.859375 -1.9375 1.125 -3.53125l1.265625 0zm5.921875 8.71875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" d="m293.19864 111.04501l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#6fa8dc" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035l0 -49.021072zm7.809265 -7.1924896l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238l0 -48.924393z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035m-84.72504 -56.213562l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m247.85564 269.3983c50.423965 10.6727295 50.423965 -10.672699 100.84248 -10.672699l0 -6.457779c0 0 3.812439 -0.3867035 7.6194763 -0.3867035l0 -6.902466c0 0 4.3384705 -0.2900238 8.676941 -0.2900238l0 -55.652832l-101.021454 0l0 7.018463l-7.809265 0l0 7.1924896l-8.308182 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035m-84.72504 -56.213562l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238" fill-rule="evenodd"/><path fill="#000000" d="m272.36282 232.7269l0 6.609375q0.03125 1.421875 -0.375 2.34375q-0.40625 0.90625 -1.109375 1.34375q-0.703125 0.421875 -1.5625 0.421875q-1.703125 0 -2.765625 -1.28125l0.734375 -0.9375l0.015625 -0.015625l0.03125 0.015625q0.515625 0.5625 0.953125 0.8125q0.4375 0.25 1.0 0.25q0.953125 0 1.375 -0.65625q0.421875 -0.671875 0.421875 -2.265625l0 -6.640625l-2.25 0l0 -1.109375l5.328125 0l0 1.109375l-1.796875 0zm-4.84375 8.421875q0 0.078125 -0.0625 0.078125q-0.0625 -0.015625 -0.109375 -0.0625l0.078125 -0.09375l0.09375 0.078125zm-0.21875 0.0625q-0.09375 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm12.328125 2.1875q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm8.78125 1.171875q-1.34375 0 -2.15625 -1.0q-0.796875 -1.0 -0.78125 -2.96875l0.03125 -4.765625l1.296875 0l0 4.765625q0 1.53125 0.53125 2.21875q0.53125 0.671875 1.40625 0.671875q0.96875 0 1.625 -0.75q0.671875 -0.765625 0.671875 -2.203125l0 -4.703125l1.3125 0l0 7.203125q0 0.453125 0.015625 0.75q0.03125 0.28125 0.171875 0.578125l-1.296875 0q-0.125 -0.28125 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.734375q-0.40625 0.71875 -1.109375 1.125q-0.6875 0.390625 -1.53125 0.390625zm13.15625 -6.953125l0 0.015625q-0.546875 -0.5 -0.921875 -0.671875q-0.359375 -0.171875 -0.84375 -0.171875q-0.671875 0 -1.265625 0.34375q-0.59375 0.328125 -0.96875 1.015625q-0.375 0.671875 -0.375 1.703125l0 4.53125l-1.359375 0l0 -8.546875l1.40625 0l-0.046875 1.578125q0.359375 -0.84375 1.09375 -1.3125q0.734375 -0.46875 1.609375 -0.46875q1.375 0 2.265625 0.9375l-0.59375 1.046875zm0 0.015625q0.125 0.109375 0.0625 0.09375q-0.0625 -0.015625 -0.109375 -0.03125l0.046875 -0.0625zm-0.203125 0.0625q0 -0.0625 0.046875 -0.046875q0.0625 0 0.109375 0.046875l-0.03125 0.09375l-0.125 -0.078125l0 -0.015625zm2.921875 -1.859375l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.53125 -0.1875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.734375 0.828125l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m306.4252 153.6762l8.062988 35.370087" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m306.4252 153.6762l6.729431 29.520172" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m311.54422 183.56346l2.619049 4.05748l0.6017761 -4.7917023z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m673.7428 143.05284l100.28345 0l0 73.44881l-100.28345 0z" fill-rule="evenodd"/><path fill="#000000" d="m683.5084 169.97284l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm13.59375 0.1875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm9.75 4.328125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125zm10.953125 -7.734375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625z" fill-rule="nonzero"/><path fill="#000000" d="m683.5084 191.97284l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm12.640625 0.203125q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm13.703125 -0.984375q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm2.734375 0.78125l0.234375 -1.078125l2.1875 0l1.34375 -6.359375l-2.0625 0l0.234375 -1.09375l3.40625 0l-1.578125 7.453125l2.0 0l-0.234375 1.078125l-5.53125 0zm4.96875 -10.3125q-0.390625 0 -0.609375 -0.28125q-0.21875 -0.28125 -0.140625 -0.671875q0.09375 -0.40625 0.421875 -0.6875q0.328125 -0.28125 0.734375 -0.28125q0.390625 0 0.625 0.296875q0.234375 0.28125 0.140625 0.671875q-0.078125 0.390625 -0.4375 0.671875q-0.34375 0.28125 -0.734375 0.28125zm7.140625 10.46875q-1.140625 0 -1.921875 -0.5625q-0.78125 -0.5625 -1.0625 -1.5625q-0.28125 -1.015625 -0.015625 -2.296875q0.28125 -1.296875 0.984375 -2.296875q0.71875 -1.015625 1.734375 -1.578125q1.03125 -0.578125 2.171875 -0.578125q1.125 0 1.890625 0.578125q0.78125 0.5625 1.0625 1.578125q0.296875 1.0 0.015625 2.296875q-0.265625 1.28125 -0.984375 2.296875q-0.71875 1.0 -1.734375 1.5625q-1.015625 0.5625 -2.140625 0.5625zm0.234375 -1.125q0.71875 0 1.375 -0.421875q0.671875 -0.4375 1.15625 -1.1875q0.484375 -0.765625 0.6875 -1.71875q0.3125 -1.453125 -0.203125 -2.375q-0.515625 -0.921875 -1.609375 -0.921875q-1.109375 0 -2.015625 0.921875q-0.90625 0.921875 -1.21875 2.375q-0.203125 0.953125 -0.03125 1.71875q0.171875 0.75 0.640625 1.1875q0.484375 0.421875 1.21875 0.421875zm7.609375 -7.5625l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125z" fill-rule="nonzero"/><path fill="#000000" d="m689.08655 208.95721q1.515625 0.484375 2.046875 1.0625q0.53125 0.5625 0.34375 1.46875q-0.25 1.15625 -1.3125 1.921875q-1.0625 0.75 -2.78125 0.75q-2.171875 0 -3.328125 -1.34375l1.015625 -1.265625l0.015625 -0.015625l0.015625 0.015625q0.421875 0.75 0.953125 1.125q0.546875 0.359375 1.609375 0.359375q1.015625 0 1.671875 -0.359375q0.65625 -0.359375 0.78125 -1.0q0.109375 -0.53125 -0.28125 -0.890625q-0.390625 -0.359375 -1.515625 -0.75q-3.015625 -0.90625 -2.65625 -2.640625q0.21875 -1.0 1.15625 -1.578125q0.9375 -0.578125 2.453125 -0.578125q1.15625 0 1.875 0.328125q0.71875 0.328125 1.234375 1.015625l-0.96875 0.9375l-0.015625 0.015625q-0.265625 -0.59375 -0.921875 -0.9375q-0.640625 -0.34375 -1.390625 -0.34375q-0.796875 0 -1.375 0.28125q-0.578125 0.28125 -0.671875 0.78125q-0.109375 0.46875 0.328125 0.859375q0.453125 0.390625 1.71875 0.78125zm2.09375 -1.390625q0.015625 -0.09375 0.140625 0.03125l-0.078125 0.0625l-0.0625 -0.09375zm0.21875 -0.03125q0.03125 0.078125 0.015625 0.09375q-0.015625 0.015625 -0.046875 0q-0.03125 -0.015625 -0.046875 -0.03125l0.078125 -0.0625zm-6.09375 3.9375q-0.015625 0.078125 -0.171875 0l0.078125 -0.09375l0.09375 0.078125l0 0.015625zm-0.21875 0.0625q-0.046875 -0.09375 -0.03125 -0.09375q0.03125 0 0.078125 0.03125l-0.046875 0.0625z" fill-rule="nonzero"/><path fill="#e06666" d="m424.69684 98.478035l0 0c0 -33.441498 27.10971 -60.55118 60.551178 -60.55118l0 0c16.059174 0 31.460602 6.3794785 42.816193 17.735027c11.35553 11.355553 17.734985 26.75698 17.734985 42.816154l0 0c0 33.44149 -27.10968 60.551186 -60.551178 60.551186l0 0c-33.441467 0 -60.551178 -27.109695 -60.551178 -60.551186z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m424.69684 98.478035l0 0c0 -33.441498 27.10971 -60.55118 60.551178 -60.55118l0 0c16.059174 0 31.460602 6.3794785 42.816193 17.735027c11.35553 11.355553 17.734985 26.75698 17.734985 42.816154l0 0c0 33.44149 -27.10968 60.551186 -60.551178 60.551186l0 0c-33.441467 0 -60.551178 -27.109695 -60.551178 -60.551186z" fill-rule="evenodd"/><path fill="#000000" d="m458.16208 87.97616q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125zm15.96875 1.703125q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.609375 -7.703125l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.34375 8.71875q-0.90625 0 -1.71875 -0.515625q-0.796875 -0.515625 -1.296875 -1.53125q-0.5 -1.03125 -0.5 -2.484375q0 -1.484375 0.515625 -2.46875q0.53125 -0.984375 1.34375 -1.453125q0.828125 -0.484375 1.734375 -0.484375q0.859375 0 1.5 0.390625q0.640625 0.390625 0.984375 1.078125l0 -5.125l1.421875 0l0 0.125q-0.109375 0.125 -0.140625 0.25q-0.03125 0.125 -0.03125 0.453125l0.015625 10.25q0 0.453125 0.03125 0.75q0.03125 0.28125 0.15625 0.578125l-1.328125 0q-0.125 -0.296875 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.75q-0.40625 0.71875 -1.046875 1.125q-0.640625 0.390625 -1.453125 0.390625zm0.21875 -1.171875q1.15625 0 1.6875 -0.90625q0.546875 -0.921875 0.546875 -2.421875q0 -1.515625 -0.59375 -2.421875q-0.578125 -0.921875 -1.765625 -0.921875q-1.109375 0 -1.71875 0.84375q-0.59375 0.828125 -0.59375 2.265625q0 1.640625 0.625 2.609375q0.625 0.953125 1.8125 0.953125zm9.71875 1.1875q-0.765625 0 -1.421875 -0.34375q-0.65625 -0.34375 -1.109375 -0.984375l-0.453125 1.125l-0.859375 0l0 -12.40625l1.53125 0l0 0.125q-0.125 0.125 -0.15625 0.25q-0.015625 0.125 -0.015625 0.453125l0 4.359375q0.40625 -0.6875 1.109375 -1.09375q0.703125 -0.421875 1.421875 -0.421875q1.609375 0 2.5625 1.125q0.953125 1.109375 0.953125 3.265625q0 1.453125 -0.515625 2.484375q-0.5 1.03125 -1.328125 1.546875q-0.8125 0.515625 -1.71875 0.515625zm-0.171875 -1.1875q1.0 0 1.671875 -0.796875q0.671875 -0.796875 0.671875 -2.484375q0 -1.640625 -0.625 -2.46875q-0.625 -0.84375 -1.6875 -0.84375q-1.046875 0 -1.703125 0.9375q-0.640625 0.9375 -0.640625 2.40625q0 3.25 2.3125 3.25zm12.7969055 -7.546875q-0.109375 0.765625 -0.34375 1.515625q-0.234375 0.75 -0.671875 1.984375l-2.125 6.0q-0.421875 1.203125 -1.109375 1.734375q-0.6875305 0.546875 -1.6406555 0.546875q-1.203125 0 -1.96875 -0.734375l0.578125 -0.90625l0.015625 -0.03125l0.03125 0.03125q0.296875 0.28125 0.625 0.40625q0.34375 0.125 0.78125 0.125q0.703125 0 1.1094055 -0.484375q0.40625 -0.484375 0.796875 -1.59375l-3.4844055 -8.59375l1.390625 0l2.6562805 6.828125l1.125 -3.421875q0.40625 -1.296875 0.59375 -1.96875q0.203125 -0.6875 0.296875 -1.4375l1.34375 0zm-7.0469055 10.0625q0 0.03125 -0.046875 0.03125l-0.140625 -0.03125l0.0625 -0.09375l0.125 0.078125l0 0.015625zm-0.21875 0.046875q-0.078125 -0.078125 -0.046875 -0.0625q0.046875 0.015625 0.078125 0.015625l-0.03125 0.046875z" fill-rule="nonzero"/><path fill="#000000" d="m472.02145 116.39803l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#e06666" d="m603.52234 93.125015l0 0c0 -33.441494 27.10968 -60.55118 60.551147 -60.55118l0 0c16.059204 0 31.460632 6.3794823 42.816162 17.735031c11.35553 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.55121 60.551178l0 0c-33.441467 0 -60.551147 -27.109688 -60.551147 -60.551178z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m603.52234 93.125015l0 0c0 -33.441494 27.10968 -60.55118 60.551147 -60.55118l0 0c16.059204 0 31.460632 6.3794823 42.816162 17.735031c11.35553 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.55121 60.551178l0 0c-33.441467 0 -60.551147 -27.109688 -60.551147 -60.551178z" fill-rule="evenodd"/><path fill="#000000" d="m643.7766 89.04501l-1.171875 -3.390625l-3.75 0l-1.1875 3.390625l-1.28125 0l4.296875 -11.828125l0.140625 0l4.296875 11.828125l-1.34375 0zm-4.578125 -4.40625l3.0625 0l-1.53125 -4.4375l-1.53125 4.4375zm11.609375 3.40625q1.28125 0 2.21875 -1.046875l0.78125 0.90625q-1.25 1.34375 -3.078125 1.34375q-1.234375 0 -2.203125 -0.578125q-0.96875 -0.578125 -1.515625 -1.59375q-0.546875 -1.015625 -0.546875 -2.28125q0 -1.265625 0.546875 -2.265625q0.546875 -1.015625 1.515625 -1.59375q0.96875 -0.578125 2.1875 -0.578125q1.03125 0 1.859375 0.421875q0.84375 0.40625 1.375 1.15625l-0.84375 0.828125l-0.015625 0.015625q-0.546875 -0.71875 -1.125 -1.0q-0.5625 -0.296875 -1.390625 -0.296875q-0.734375 0 -1.359375 0.40625q-0.625 0.40625 -1.0 1.140625q-0.375 0.71875 -0.375 1.671875q0 0.953125 0.375 1.71875q0.390625 0.765625 1.0625 1.203125q0.6875 0.421875 1.53125 0.421875zm2.078125 -5.25q0 -0.109375 0.15625 0.015625l-0.0625 0.078125l-0.09375 -0.09375zm0.203125 -0.015625q0.09375 0.125 0.046875 0.09375q-0.046875 -0.046875 -0.09375 -0.0625l0.046875 -0.03125zm9.9375 5.484375q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm2.90625 0.78125l0 -1.078125l2.1875 0l0 -6.359375l-2.0625 0l0 -1.09375l3.40625 0l0 7.453125l2.0 0l0 1.078125l-5.53125 0zm2.78125 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm13.171875 1.78125q-0.296875 1.515625 -1.25 3.734375l-2.0625 4.796875l-1.046875 0l-3.375 -8.53125l1.34375 0l2.625 6.6875l1.375 -3.15625q0.859375 -1.9375 1.125 -3.53125l1.265625 0zm5.921875 8.71875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" d="m650.8469 111.04501l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#6fa8dc" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823l0 -49.021057zm7.809265 -7.1924896l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854l0 -48.924393z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823m-84.7251 -56.213547l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m597.18896 281.37335c50.42395 10.6727295 50.42395 -10.6727295 100.84253 -10.6727295l0 -6.4577637c0 0 3.812439 -0.38668823 7.619446 -0.38668823l0 -6.902466c0 0 4.33844 -0.29000854 8.676941 -0.29000854l0 -55.652863l-101.021484 0l0 7.0184784l-7.809265 0l0 7.1924896l-8.3081665 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823m-84.7251 -56.213547l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854" fill-rule="evenodd"/><path fill="#000000" d="m621.69617 244.70195l0 6.609375q0.03125 1.421875 -0.375 2.34375q-0.40625 0.90625 -1.109375 1.34375q-0.703125 0.421875 -1.5625 0.421875q-1.703125 0 -2.765625 -1.28125l0.734375 -0.9375l0.015625 -0.015625l0.03125 0.015625q0.515625 0.5625 0.953125 0.8125q0.4375 0.25 1.0 0.25q0.953125 0 1.375 -0.65625q0.421875 -0.671875 0.421875 -2.265625l0 -6.640625l-2.25 0l0 -1.109375l5.328125 0l0 1.109375l-1.796875 0zm-4.84375 8.421875q0 0.078125 -0.0625 0.078125q-0.0625 -0.015625 -0.109375 -0.0625l0.078125 -0.09375l0.09375 0.078125zm-0.21875 0.0625q-0.09375 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm12.328125 2.1875q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm8.78125 1.171875q-1.34375 0 -2.15625 -1.0q-0.796875 -1.0 -0.78125 -2.96875l0.03125 -4.765625l1.296875 0l0 4.765625q0 1.53125 0.53125 2.21875q0.53125 0.671875 1.40625 0.671875q0.96875 0 1.625 -0.75q0.671875 -0.765625 0.671875 -2.203125l0 -4.703125l1.3125 0l0 7.203125q0 0.453125 0.015625 0.75q0.03125 0.28125 0.171875 0.578125l-1.296875 0q-0.125 -0.28125 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.734375q-0.40625 0.71875 -1.109375 1.125q-0.6875 0.390625 -1.53125 0.390625zm13.15625 -6.953125l0 0.015625q-0.546875 -0.5 -0.921875 -0.671875q-0.359375 -0.171875 -0.84375 -0.171875q-0.671875 0 -1.265625 0.34375q-0.59375 0.328125 -0.96875 1.015625q-0.375 0.671875 -0.375 1.703125l0 4.53125l-1.359375 0l0 -8.546875l1.40625 0l-0.046875 1.578125q0.359375 -0.84375 1.09375 -1.3125q0.734375 -0.46875 1.609375 -0.46875q1.375 0 2.265625 0.9375l-0.59375 1.046875zm0 0.015625q0.125 0.109375 0.0625 0.09375q-0.0625 -0.015625 -0.109375 -0.03125l0.046875 -0.0625zm-0.203125 0.0625q0 -0.0625 0.046875 -0.046875q0.0625 0 0.109375 0.046875l-0.03125 0.09375l-0.125 -0.078125l0 -0.015625zm2.921875 -1.859375l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.53125 -0.1875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.734375 0.828125l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m664.0735 153.6762l-0.25195312 47.338577" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m664.0735 153.6762l-0.22003174 41.33867" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m662.2017 195.00607l1.6275635 4.546829l1.6759033 -4.529236z" fill-rule="evenodd"/><path fill="#e6b8af" d="m3.406824 307.91898l777.6063 0l0 169.13385l-777.6063 0z" fill-rule="evenodd"/><path fill="#000000" d="m344.085 381.96402l7.890625 0q4.46875 0 6.59375 1.96875q2.125 1.953125 2.125 5.46875q0 1.375 -0.609375 2.75q-0.59375 1.359375 -1.671875 2.40625q-1.078125 1.03125 -2.484375 1.484375l5.828125 10.84375l-5.03125 0l-5.15625 -10.3125l-2.90625 0l0 10.3125l-4.578125 0l0 -24.921875zm8.171875 10.796875q1.921875 0 2.875 -0.890625q0.96875 -0.90625 0.96875 -2.46875q0 -1.75 -0.9375 -2.609375q-0.921875 -0.859375 -2.90625 -0.859375l-3.59375 0l0 6.828125l3.59375 0zm24.921875 14.125l-1.78125 -6.453125l-6.375 0l-1.8125 6.453125l-4.609375 0l8.328125 -24.921875l2.609375 0l8.328125 24.921875l-4.6875 0zm-7.265625 -9.546875l4.625 0l-2.28125 -8.28125l-2.34375 8.28125zm14.0625 -15.375l6.4375 0q3.1875 0 5.0625 0.859375q1.890625 0.84375 3.21875 2.609375q2.5625 3.484375 2.5625 9.171875q0 5.890625 -2.75 9.09375q-2.734375 3.1875 -8.578125 3.1875l-5.953125 0l0 -24.921875zm5.875 21.40625q3.5 0 5.203125 -2.203125q1.703125 -2.21875 1.703125 -6.453125q0 -9.015625 -6.5 -9.015625l-1.9375 0l0 17.671875l1.53125 0zm22.484375 4.234375q-2.953125 0 -5.078125 -1.5625q-2.109375 -1.578125 -3.234375 -4.546875q-1.125 -2.984375 -1.125 -7.125q0 -4.09375 1.125 -7.015625q1.125 -2.9375 3.234375 -4.484375q2.125 -1.546875 5.078125 -1.546875q2.9375 0 5.0625 1.546875q2.125 1.546875 3.234375 4.484375q1.125 2.921875 1.125 7.015625q0 4.140625 -1.125 7.125q-1.109375 2.96875 -3.234375 4.546875q-2.125 1.5625 -5.0625 1.5625zm0 -3.59375q1.484375 0 2.5625 -1.21875q1.078125 -1.21875 1.65625 -3.453125q0.578125 -2.234375 0.578125 -5.203125q0 -2.765625 -0.578125 -4.84375q-0.578125 -2.078125 -1.65625 -3.21875q-1.078125 -1.140625 -2.5625 -1.140625q-1.46875 0 -2.5625 1.140625q-1.09375 1.140625 -1.671875 3.21875q-0.578125 2.078125 -0.578125 4.84375q0 2.96875 0.578125 5.203125q0.578125 2.234375 1.65625 3.453125q1.09375 1.21875 2.578125 1.21875zm21.75 -11.5625q2.546875 1.21875 3.953125 2.34375q1.40625 1.109375 1.984375 2.34375q0.59375 1.234375 0.59375 2.875q0 1.984375 -0.921875 3.71875q-0.921875 1.734375 -2.90625 2.8125q-1.984375 1.0625 -5.03125 1.0625q-4.859375 0 -8.4375 -3.46875l1.953125 -3.40625l0.03125 -0.03125l0.03125 0.03125q2.90625 3.03125 6.796875 3.03125q1.875 0 2.890625 -0.875q1.03125 -0.890625 1.03125 -2.640625q0 -0.875 -0.4375 -1.59375q-0.4375 -0.71875 -1.5 -1.4375q-1.0625 -0.71875 -2.9375 -1.5625q-3.484375 -1.5 -5.046875 -3.375q-1.5625 -1.875 -1.5625 -4.359375q0 -1.796875 1.0 -3.3125q1.015625 -1.515625 2.78125 -2.390625q1.765625 -0.890625 3.984375 -0.890625q2.46875 0 4.265625 0.828125q1.796875 0.8125 3.4375 2.53125l-2.234375 3.125l-0.03125 0.03125q-0.015625 0 -0.015625 0q0 -0.015625 0 -0.015625q-0.921875 -1.46875 -2.34375 -2.140625q-1.421875 -0.671875 -3.125 -0.671875q-1.09375 0 -1.890625 0.390625q-0.78125 0.375 -1.1875 1.0q-0.390625 0.625 -0.390625 1.359375q0 0.875 0.453125 1.5625q0.453125 0.671875 1.5625 1.390625q1.125 0.703125 3.25 1.734375zm3.140625 -4.671875q0 -0.03125 0.046875 -0.03125q0.03125 0 0.15625 0.09375q0.140625 0.09375 0.234375 0.15625l-0.0625 0.09375l-0.375 -0.25l0 -0.0625zm0.546875 0.0625q0.140625 0.21875 0.140625 0.28125q0 0.015625 -0.015625 0.015625q-0.0625 0 -0.234375 -0.140625l0.109375 -0.15625zm-11.90625 12.875q0 0.03125 -0.046875 0.03125q-0.046875 0 -0.203125 -0.078125q-0.140625 -0.078125 -0.234375 -0.109375l0.078125 -0.140625l0.390625 0.234375l0.015625 0.0625zm-0.5625 -0.015625q-0.15625 -0.15625 -0.15625 -0.21875q0 0 0 0q0 -0.015625 0.015625 -0.015625l0.21875 0.09375l-0.078125 0.140625z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m168.86089 338.05414l0 0c0 7.6123962 24.68428 13.783478 55.133865 13.783478c30.449585 0 55.13385 -6.1710815 55.13385 -13.783478l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478z" fill-rule="evenodd"/><path fill="#e2edf7" d="m168.86089 338.05414l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m279.1286 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478l0 -119.078735" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m279.1286 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478l0 -119.078735" fill-rule="evenodd"/><path fill="#000000" d="m183.01819 399.78024l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm10.4375 -7.71875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm13.546875 0.046875q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm15.6875 -10.796875l3.734375 0q1.84375 0 2.75 0.921875q0.921875 0.90625 0.921875 2.359375q0 1.4375 -0.890625 2.328125q-0.890625 0.890625 -2.6875 0.890625l-2.484375 0l0 5.125l-1.34375 0l0 -11.625zm3.6875 5.34375q1.21875 0 1.796875 -0.53125q0.578125 -0.53125 0.578125 -1.484375q0 -0.921875 -0.59375 -1.5q-0.59375 -0.59375 -1.765625 -0.59375l-2.359375 0l0 4.109375l2.34375 0zm9.21875 6.4375q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm9.328125 1.125q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm6.125 0.96875l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m500.87402 338.05414l0 0c0 7.6123962 24.684265 13.783478 55.13385 13.783478c30.449585 0 55.13385 -6.1710815 55.13385 -13.783478l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478z" fill-rule="evenodd"/><path fill="#e2edf7" d="m500.87402 338.05414l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m611.1417 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478l0 -119.078735" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m611.1417 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478l0 -119.078735" fill-rule="evenodd"/><path fill="#000000" d="m519.461 400.40524l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm13.640625 0.1875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0zm10.671875 4.328125q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm9.625 1.015625q-0.90625 0 -1.71875 -0.515625q-0.796875 -0.515625 -1.296875 -1.53125q-0.5 -1.03125 -0.5 -2.484375q0 -1.484375 0.515625 -2.46875q0.53125 -0.984375 1.34375 -1.453125q0.828125 -0.484375 1.734375 -0.484375q0.859375 0 1.5 0.390625q0.640625 0.390625 0.984375 1.078125l0 -5.125l1.421875 0l0 0.125q-0.109375 0.125 -0.140625 0.25q-0.03125 0.125 -0.03125 0.453125l0.015625 10.25q0 0.453125 0.03125 0.75q0.03125 0.28125 0.15625 0.578125l-1.328125 0q-0.125 -0.296875 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.75q-0.40625 0.71875 -1.046875 1.125q-0.640625 0.390625 -1.453125 0.390625zm0.21875 -1.171875q1.15625 0 1.6875 -0.90625q0.546875 -0.921875 0.546875 -2.421875q0 -1.515625 -0.59375 -2.421875q-0.578125 -0.921875 -1.765625 -0.921875q-1.109375 0 -1.71875 0.84375q-0.59375 0.828125 -0.59375 2.265625q0 1.640625 0.625 2.609375q0.625 0.953125 1.8125 0.953125zm9.296875 -7.734375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm13.546875 0.046875q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625z" fill-rule="nonzero"/><path fill="#000000" d="m538.4454 410.78024l3.734375 0q1.84375 0 2.75 0.921875q0.921875 0.90625 0.921875 2.359375q0 1.4375 -0.890625 2.328125q-0.890625 0.890625 -2.6875 0.890625l-2.484375 0l0 5.125l-1.34375 0l0 -11.625zm3.6875 5.34375q1.21875 0 1.796875 -0.53125q0.578125 -0.53125 0.578125 -1.484375q0 -0.921875 -0.59375 -1.5q-0.59375 -0.59375 -1.765625 -0.59375l-2.359375 0l0 4.109375l2.34375 0zm9.21875 6.4375q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm9.328125 1.125q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm6.125 0.96875l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m85.40682 135.93997l138.58267 188.34647" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m85.40682 135.93997l135.0268 183.51367" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m219.10321 320.43256l4.0198975 2.676361l-1.3590851 -4.6341553z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m47.26509 193.37305l76.28346 0l0 73.44882l-76.28346 0z" fill-rule="evenodd"/><path fill="#000000" d="m76.1412 213.54305l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm5.1875 6.859375q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm7.703125 -3.609375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125z" fill-rule="nonzero"/><path fill="#000000" d="m64.195885 233.7618l1.171875 0l-0.0625 6.859375l2.734375 -6.046875l0.8125 0l0.53125 6.015625q1.28125 -3.484375 1.625 -4.578125q0.359375 -1.09375 0.546875 -1.96875l0.0625 -0.28125l1.25 0q-1.484375 4.328125 -3.21875 8.546875l-1.28125 0l-0.390625 -5.515625l-2.65625 5.515625l-1.21875 0l0.09375 -8.546875zm16.609375 1.78125l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm2.015625 6.671875l0.234375 -1.078125l2.1875 0l1.34375 -6.359375l-2.0625 0l0.234375 -1.09375l3.40625 0l-1.578125 7.453125l2.0 0l-0.234375 1.078125l-5.53125 0zm4.96875 -10.3125q-0.390625 0 -0.609375 -0.28125q-0.21875 -0.28125 -0.140625 -0.671875q0.09375 -0.40625 0.421875 -0.6875q0.328125 -0.28125 0.734375 -0.28125q0.390625 0 0.625 0.296875q0.234375 0.28125 0.140625 0.671875q-0.078125 0.390625 -0.4375 0.671875q-0.34375 0.28125 -0.734375 0.28125zm10.953125 9.53125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm5.90625 0.96875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m647.6129 281.37335l-91.590515 42.897644" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m647.6129 281.37335l-86.15698 40.352753" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m560.7554 320.23032l-3.4091187 3.4206238l4.8103027 -0.4290161z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m298.2796 269.3983l257.7323 54.86615" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m298.2796 269.3983l251.86383 53.616882" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m549.7995 324.63068l4.7825317 -0.6706238l-4.0947266 -2.5604248z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m621.2573 135.94116l-65.25983 188.31497" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m621.2573 135.94116l-63.295166 182.64575" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m556.4015 318.04605l0.07470703 4.828766l3.0466309 -3.7470703z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m420.3832 219.50296l161.32285 0l0 52.503937l-161.32285 0z" fill-rule="evenodd"/><path fill="#000000" d="m460.40915 235.90733l-1.40625 6.609375q-0.265625 1.421875 -0.875 2.34375q-0.59375 0.90625 -1.390625 1.34375q-0.796875 0.421875 -1.65625 0.421875q-1.703125 0 -2.484375 -1.28125l0.921875 -0.9375l0.03125 -0.015625l0.015625 0.015625q0.40625 0.5625 0.78125 0.8125q0.390625 0.25 0.953125 0.25q0.953125 0 1.515625 -0.65625q0.5625 -0.671875 0.90625 -2.265625l1.40625 -6.640625l-2.25 0l0.234375 -1.109375l5.328125 0l-0.234375 1.109375l-1.796875 0zm-6.640625 8.421875q-0.015625 0.078125 -0.078125 0.078125q-0.046875 -0.015625 -0.09375 -0.0625l0.109375 -0.09375l0.0625 0.078125zm-0.21875 0.0625q-0.078125 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm11.859375 2.1875q-1.140625 0 -1.921875 -0.5625q-0.78125 -0.5625 -1.0625 -1.5625q-0.28125 -1.015625 -0.015625 -2.296875q0.28125 -1.296875 0.984375 -2.296875q0.71875 -1.015625 1.734375 -1.578125q1.03125 -0.578125 2.171875 -0.578125q1.125 0 1.890625 0.578125q0.78125 0.5625 1.0625 1.578125q0.296875 1.0 0.015625 2.296875q-0.265625 1.28125 -0.984375 2.296875q-0.71875 1.0 -1.734375 1.5625q-1.015625 0.5625 -2.140625 0.5625zm0.234375 -1.125q0.71875 0 1.375 -0.421875q0.671875 -0.4375 1.15625 -1.1875q0.484375 -0.765625 0.6875 -1.71875q0.3125 -1.453125 -0.203125 -2.375q-0.515625 -0.921875 -1.609375 -0.921875q-1.109375 0 -2.015625 0.921875q-0.90625 0.921875 -1.21875 2.375q-0.203125 0.953125 -0.03125 1.71875q0.171875 0.75 0.640625 1.1875q0.484375 0.421875 1.21875 0.421875zm8.53125 1.171875q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm14.640625 -6.953125l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm3.3125 -1.859375l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125zm12.578125 -0.1875q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm6.5625 0.828125l0.234375 -1.078125l2.53125 0l2.1719055 -10.25l-2.4219055 0l0.234375 -1.078125l3.7812805 0l-2.40625 11.328125l2.5 0l-0.234375 1.078125l-6.3906555 0zm18.640656 0l2.46875 -11.640625l6.703125 0l-0.234375 1.140625l-5.390625 0l-0.78125 3.65625l4.34375 0l-0.234375 1.140625l-4.34375 0l-1.21875 5.703125l-1.3125 0zm9.34375 0l0.234375 -1.078125l2.53125 0l2.171875 -10.25l-2.421875 0l0.234375 -1.078125l3.78125 0l-2.40625 11.328125l2.5 0l-0.234375 1.078125l-6.390625 0zm11.9375 0.203125q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm11.59375 -5.21875q1.515625 0.484375 2.046875 1.0625q0.53125 0.5625 0.34375 1.46875q-0.25 1.15625 -1.3125 1.921875q-1.0625 0.75 -2.78125 0.75q-2.171875 0 -3.328125 -1.34375l1.015625 -1.265625l0.015625 -0.015625l0.015625 0.015625q0.421875 0.75 0.953125 1.125q0.546875 0.359375 1.609375 0.359375q1.015625 0 1.671875 -0.359375q0.65625 -0.359375 0.78125 -1.0q0.109375 -0.53125 -0.28125 -0.890625q-0.390625 -0.359375 -1.515625 -0.75q-3.015625 -0.90625 -2.65625 -2.640625q0.21875 -1.0 1.15625 -1.578125q0.9375 -0.578125 2.453125 -0.578125q1.15625 0 1.875 0.328125q0.71875 0.328125 1.234375 1.015625l-0.96875 0.9375l-0.015625 0.015625q-0.265625 -0.59375 -0.921875 -0.9375q-0.640625 -0.34375 -1.390625 -0.34375q-0.796875 0 -1.375 0.28125q-0.578125 0.28125 -0.671875 0.78125q-0.109375 0.46875 0.328125 0.859375q0.453125 0.390625 1.71875 0.78125zm2.09375 -1.390625q0.015625 -0.09375 0.140625 0.03125l-0.078125 0.0625l-0.0625 -0.09375zm0.21875 -0.03125q0.03125 0.078125 0.015625 0.09375q-0.015625 0.015625 -0.046875 0q-0.03125 -0.015625 -0.046875 -0.03125l0.078125 -0.0625zm-6.09375 3.9375q-0.015625 0.078125 -0.171875 0l0.078125 -0.09375l0.09375 0.078125l0 0.015625zm-0.21875 0.0625q-0.046875 -0.09375 -0.03125 -0.09375q0.03125 0 0.078125 0.03125l-0.046875 0.0625zm10.984375 -9.96875l1.46875 0l-0.03125 0.125q-0.140625 0.125 -0.1875 0.25q-0.046875 0.125 -0.109375 0.453125l-0.984375 4.5625q0.671875 -0.78125 1.53125 -1.25q0.859375 -0.46875 1.640625 -0.46875q1.203125 0 1.703125 0.859375q0.515625 0.859375 0.125 2.703125l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.59375 0 -1.25 0.328125q-0.640625 0.328125 -1.15625 0.9375q-0.515625 0.609375 -0.671875 1.390625l-1.046875 4.90625l-1.3125 0l2.640625 -12.40625z" fill-rule="nonzero"/><path fill="#ffffff" d="m341.60367 41.995308l0 0c0 -5.917248 4.796875 -10.714127 10.714142 -10.714127l42.855194 0l0 0c2.8415833 0 5.566742 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.138092 4.7344666 3.138092 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714142 10.714119l-42.855194 0c-5.917267 0 -10.714142 -4.796875 -10.714142 -10.714119z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m341.60367 41.995308l0 0c0 -5.917248 4.796875 -10.714127 10.714142 -10.714127l42.855194 0l0 0c2.8415833 0 5.566742 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.138092 4.7344666 3.138092 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714142 10.714119l-42.855194 0c-5.917267 0 -10.714142 -4.796875 -10.714142 -10.714119z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m371.98032 52.367798l22.11023 22.110237" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m374.66028 55.04773l16.750305 16.750374" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m372.33386 52.72135c0.64242554 -0.6424141 1.68396 -0.6424141 2.3263855 0c0.64242554 0.6424103 0.64242554 1.6839676 0 2.3263817c-0.64242554 0.6424103 -1.68396 0.6424103 -2.3263855 0c-0.642395 -0.6424141 -0.642395 -1.6839714 0 -2.3263817z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m393.737 74.12448c-0.64242554 0.6424103 -1.68396 0.6424103 -2.3263855 0c-0.642395 -0.6424103 -0.642395 -1.6839676 0 -2.3263855c0.64242554 -0.6424103 1.68396 -0.6424103 2.3263855 0c0.64242554 0.6424179 0.64242554 1.6839752 0 2.3263855z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m374.6601 52.79299l-21.259827 21.259842" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m374.6601 52.792995l-18.579865 18.579906" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m357.24338 72.536095l-2.3263855 2.3263779l-2.326355 -2.3263779l2.326355 -2.3263855l2.3263855 2.3263855z" fill-rule="nonzero"/><path fill="#ffffff" d="m702.1522 41.993996l0 0c0 -5.917248 4.796875 -10.714127 10.714111 -10.714127l42.855225 0l0 0c2.8415527 0 5.5667725 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.1380615 4.7344666 3.1380615 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714111 10.714119l-42.855225 0c-5.9172363 0 -10.714111 -4.796875 -10.714111 -10.714119z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m702.1522 41.993996l0 0c0 -5.917248 4.796875 -10.714127 10.714111 -10.714127l42.855225 0l0 0c2.8415527 0 5.5667725 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.1380615 4.7344666 3.1380615 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714111 10.714119l-42.855225 0c-5.9172363 0 -10.714111 -4.796875 -10.714111 -10.714119z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m716.458 43.87436l22.11023 22.110237" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m719.13794 46.554295l16.750366 16.750366" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m716.8116 44.227913c0.642395 -0.6424141 1.68396 -0.6424141 2.326355 0c0.642395 0.6424103 0.642395 1.6839676 0 2.3263817c-0.642395 0.6424103 -1.68396 0.6424103 -2.326355 0c-0.64245605 -0.6424141 -0.64245605 -1.6839714 0 -2.3263817z" fill-rule="nonzero"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m738.21466 65.63104c-0.642395 0.6424103 -1.68396 0.6424103 -2.326355 0c-0.642395 -0.6424103 -0.642395 -1.6839676 0 -2.3263817c0.642395 -0.6424141 1.68396 -0.6424141 2.326355 0c0.64245605 0.6424141 0.64245605 1.6839714 0 2.3263817z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m738.14307 61.709003l-21.259888 21.259846" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m738.14307 61.709003l-18.579895 18.579914" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m720.7263 81.4521l-2.326355 2.3263855l-2.326416 -2.3263855l2.326416 -2.3263779l2.326355 2.3263779z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m735.96716 61.34155l21.259888 21.259846" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m735.96716 61.34155l18.579895 18.579914" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m755.71027 78.75827l2.326416 2.3263779l-2.326416 2.3263855l-2.326355 -2.3263855l2.326355 -2.3263779z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m361.7454 127.52721c13.486847 7.3713837 49.210907 36.856964 80.92093 44.228348c31.710052 7.371399 81.85028 8.16481 109.33923 0c27.489014 -8.164795 46.328796 -40.82402 55.594604 -48.988815" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m366.87332 130.64238l0.061187744 0.03933716c0.28182983 0.18249512 0.5683594 0.36921692 0.8595886 0.5599823c2.3294678 1.5260925 4.9551697 3.3113708 7.8257446 5.269394c5.7411804 3.9160461 12.462006 8.523178 19.75232 13.130295c14.580627 9.214233 31.439148 18.428467 47.29419 22.114166c31.709991 7.371399 81.85025 8.16481 109.3392 0c13.744507 -4.0823975 25.326721 -14.288406 34.63098 -24.494415c4.6521606 -5.102997 8.734863 -10.205994 12.233582 -14.543549c0.8746948 -1.0843811 1.7128906 -2.1209412 2.5143433 -3.0976868c0.40075684 -0.4883728 0.79229736 -0.961792 1.1746216 -1.4187622c0.19122314 -0.22849274 0.38006592 -0.45287323 0.5666504 -0.6729584l0.34716797 -0.40646362" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m367.7309 129.23071l-4.7360535 -0.9445038l3.0209045 3.7678375z" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m604.6718 128.2579l1.9227295 -4.4300766l-4.3204956 2.1577148z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m388.79266 166.79956l192.91339 0l0 43.338577l-192.91339 0z" fill-rule="evenodd"/><path fill="#000000" d="m406.72592 193.71956l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm13.59375 0.1875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm9.75 4.328125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125zm10.953125 -7.734375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm15.515625 0.828125l2.46875 -11.640625l7.140625 0l-0.234375 1.15625l-5.90625 0l-0.8125 3.828125l4.875 0l-0.265625 1.1875l-4.875 0l-0.90625 4.3125l5.84375 0l-0.25 1.15625l-7.078125 0zm9.109375 0l3.921875 -4.328125l-2.046875 -4.203125l1.46875 0l1.546875 3.15625l2.796875 -3.15625l1.390625 0l-3.640625 4.140625l2.1875 4.390625l-1.515625 0l-1.65625 -3.328125l-2.921875 3.328125l-1.53125 0zm14.0624695 -1.0q1.28125 0 2.4375 -1.046875l0.59375 0.90625q-1.546875 1.34375 -3.375 1.34375q-1.234375 0 -2.0780945 -0.578125q-0.84375 -0.578125 -1.171875 -1.59375q-0.328125 -1.015625 -0.0625 -2.28125q0.265625 -1.265625 1.015625 -2.265625q0.7655945 -1.015625 1.8593445 -1.59375q1.09375 -0.578125 2.3125 -0.578125q1.03125 0 1.78125 0.421875q0.75 0.40625 1.125 1.15625l-1.03125 0.828125l-0.015625 0.015625q-0.390625 -0.71875 -0.90625 -1.0q-0.5 -0.296875 -1.328125 -0.296875q-0.734375 0 -1.453125 0.40625q-0.703125 0.40625 -1.234375 1.140625q-0.53125 0.71875 -0.7343445 1.671875q-0.203125 0.953125 0.015625 1.71875q0.21871948 0.765625 0.8124695 1.203125q0.59375 0.421875 1.4375 0.421875zm3.1875 -5.25q0.03125 -0.109375 0.15625 0.015625l-0.078125 0.078125l-0.078125 -0.09375zm0.203125 -0.015625q0.078125 0.125 0.03125 0.09375q-0.03125 -0.046875 -0.078125 -0.0625l0.046875 -0.03125zm4.328125 -6.140625l1.46875 0l-0.03125 0.125q-0.140625 0.125 -0.1875 0.25q-0.046875 0.125 -0.109375 0.453125l-0.984375 4.5625q0.671875 -0.78125 1.53125 -1.25q0.859375 -0.46875 1.640625 -0.46875q1.203125 0 1.703125 0.859375q0.515625 0.859375 0.125 2.703125l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.59375 0 -1.25 0.328125q-0.640625 0.328125 -1.15625 0.9375q-0.515625 0.609375 -0.671875 1.390625l-1.046875 4.90625l-1.3125 0l2.640625 -12.40625zm11.734375 3.6875q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm8.25 -7.703125l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125zm16.65625 0.875q-0.171875 -0.015625 -0.484375 -0.015625q-0.890625 0 -1.5625 0.390625q0.171875 0.640625 0.015625 1.40625q-0.1875 0.84375 -0.703125 1.53125q-0.5 0.671875 -1.265625 1.0625q-0.765625 0.390625 -1.65625 0.390625q-0.71875 0 -1.265625 -0.265625q-0.421875 0.40625 -0.5 0.78125q-0.09375 0.453125 0.421875 0.65625q0.53125 0.1875 2.015625 0.1875q1.8125 0 2.34375 0.5625q0.546875 0.546875 0.3125 1.609375q-0.234375 1.09375 -1.28125 1.828125q-1.046875 0.734375 -2.953125 0.734375q-1.828125 0 -2.75 -0.5625q-0.90625 -0.546875 -0.6875 -1.640625q0.15625 -0.671875 0.625 -1.15625q0.484375 -0.484375 1.125 -0.765625q-0.53125 -0.40625 -0.375 -1.125q0.15625 -0.75 1.09375 -1.578125q-0.375 -0.40625 -0.5 -0.984375q-0.109375 -0.578125 0.03125 -1.265625q0.171875 -0.828125 0.671875 -1.5q0.515625 -0.6875 1.28125 -1.078125q0.765625 -0.390625 1.65625 -0.390625q1.359375 0 2.0 0.875q0.53125 -0.40625 1.046875 -0.59375q0.515625 -0.203125 1.09375 -0.203125l0.3125 0.015625l-0.0625 1.09375zm-5.40625 3.640625q0.796875 0 1.4375 -0.53125q0.640625 -0.546875 0.8125 -1.328125q0.171875 -0.78125 -0.25 -1.328125q-0.421875 -0.546875 -1.21875 -0.546875q-0.796875 0 -1.453125 0.546875q-0.640625 0.546875 -0.8125 1.328125q-0.171875 0.78125 0.25 1.328125q0.4375 0.53125 1.234375 0.53125zm1.828125 4.859375q0.09375 -0.453125 -0.03125 -0.703125q-0.109375 -0.25 -0.546875 -0.375q-0.421875 -0.125 -1.359375 -0.125q-1.34375 0 -2.09375 -0.234375q-0.515625 0.296875 -0.765625 0.609375q-0.234375 0.3125 -0.359375 0.84375q-0.125 0.640625 0.5 0.984375q0.640625 0.359375 1.90625 0.359375q1.234375 0 1.921875 -0.375q0.6875 -0.359375 0.828125 -0.984375zm7.453125 -0.65625q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m110.45702 4.5455894l171.33856 -0.031496048l0 73.44882l-171.33856 0.03149414z" fill-rule="evenodd"/><path fill="#000000" d="m182.11069 31.608667q-1.140625 2.07901E-4 -1.921875 -0.56214714q-0.78125 -0.56235695 -1.0625 -1.5623055q-0.28125 -1.0155735 -0.015625 -2.2968712q0.28125 -1.2969265 0.984375 -2.2970562q0.71875 -1.0157585 1.734375 -1.5784435q1.03125 -0.57831573 2.171875 -0.57852554q1.125 -2.0599365E-4 1.890625 0.57777786q0.78125 0.56235695 1.0625 1.5779305q0.296875 0.9999447 0.015625 2.2968712q-0.265625 1.2812996 -0.984375 2.2970562q-0.71875 1.0001316 -1.734375 1.5628185q-1.015625 0.5626869 -2.140625 0.5628948zm0.234375 -1.1250439q0.71875 -1.3160706E-4 1.375 -0.42212868q0.671875 -0.43762207 1.15625 -1.1877117q0.484375 -0.76571465 0.6875 -1.7188759q0.3125 -1.4531822 -0.203125 -2.3749638q-0.515625 -0.92177963 -1.609375 -0.92157936q-1.109375 2.040863E-4 -2.015625 0.92224693q-0.90625 0.92204094 -1.21875 2.3752232q-0.203125 0.95316315 -0.03125 1.7187557q0.171875 0.7499695 0.640625 1.1873817q0.484375 0.42178726 1.21875 0.42165184zm7.390625 -7.563858l1.359375 -2.4986267E-4l-0.359375 1.6875648q0.484375 -0.93758774 1.265625 -1.4064827q0.796875 -0.46889687 1.734375 -0.46906853q0.96875 -1.7738342E-4 1.671875 0.49969292q0.703125 0.4998703 0.96875 1.4841976q0.28125 0.9843235 -0.015625 2.406252q-0.3125 1.4375572 -1.046875 2.4689426q-0.71875 1.0313816 -1.65625 1.5628052q-0.9375 0.51579666 -1.875 0.5159683q-0.765625 1.411438E-4 -1.359375 -0.34350014q-0.59375 -0.35926437 -0.890625 -0.99983597l-0.890625 4.234541l-1.375 2.5177002E-4l2.46875 -11.641079zm2.078125 7.5621166q1.03125 -1.8882751E-4 1.890625 -0.76597214q0.859375 -0.7814083 1.234375 -2.531477q0.34375 -1.593813 -0.09375 -2.4374828q-0.4375 -0.8592949 -1.5625 -0.859087q-1.0625 1.9454956E-4 -1.921875 0.89097786q-0.84375 0.8907795 -1.234375 2.703352q-0.28125 1.4688015 0.171875 2.2343426q0.453125 0.765543 1.515625 0.7653465zm9.34375 1.1545334q-1.90625 3.5095215E-4 -2.78125 -1.1557388q-0.875 -1.1560898 -0.421875 -3.2655468q0.296875 -1.4063053 1.015625 -2.422062q0.734375 -1.0157604 1.734375 -1.5471954q1.015625 -0.531435 2.109375 -0.5316372q1.546875 -2.8419495E-4 2.28125 1.0308304q0.75 1.0311127 0.328125 3.0155659q-0.046875 0.20313263 -0.171875 0.6250305l-6.0625 0.001115799q-0.25 1.5625458 0.375 2.3749294q0.625 0.79676056 1.859375 0.7965336q1.34375 -2.4604797E-4 2.40625 -0.9535675l0.59375 0.7186413q-1.359375 1.3127499 -3.265625 1.3131008zm3.0 -5.297426q0.25 -1.2031708 -0.203125 -1.8905888q-0.453125 -0.7030411 -1.453125 -0.702858q-0.921875 1.6975403E-4 -1.765625 0.65657425q-0.828125 0.6407776 -1.265625 1.9377327l4.6875 -8.6021423E-4zm4.453125 -3.4226952l1.3125 -2.4032593E-4l-0.328125 1.515686q0.671875 -0.781374 1.53125 -1.2502823q0.875 -0.46891022 1.734375 -0.46906853q1.125 -2.07901E-4 1.625 0.87470055q0.5 0.85928345 0.109375 2.687481l-1.09375 5.1720753l-1.3125 2.4223328E-4l1.09375 -5.125202q0.265625 -1.2812977 -0.046875 -1.8593655q-0.296875 -0.5780716 -1.0 -0.5779419q-0.578125 1.0681152E-4 -1.234375 0.34397697q-0.65625 0.32824516 -1.171875 0.93771553q-0.515625 0.60947037 -0.671875 1.375124l-1.046875 4.9064426l-1.3125 2.4032593E-4l1.8125 -8.531584z" fill-rule="nonzero"/><path fill="#000000" d="m175.32162 44.922413l1.234375 -2.2506714E-4l-0.171875 0.8437805q0.390625 -0.46882248 0.921875 -0.75016785q0.53125 -0.29697418 1.03125 -0.29706573q0.5625 -1.02996826E-4 0.90625 0.35920715q0.34375 0.35931396 0.34375 0.89056396q0.34375 -0.56256485 0.9375 -0.90642166q0.609375 -0.34386444 1.25 -0.34397888q0.84375 -1.5640259E-4 1.125 0.6091652q0.28125 0.6093254 0.046875 1.5781174l-1.390625 6.5471306l-1.234375 2.2888184E-4l1.28125 -6.0471115q0.234375 -1.062542 0.09375 -1.3750191q-0.140625 -0.3280983 -0.53125 -0.32802582q-0.34375 6.1035156E-5 -0.71875 0.3282585q-0.375 0.31256866 -0.6875 0.8126259q-0.296875 0.4844284 -0.390625 0.96881866l-1.203125 5.6408463l-1.25 2.3269653E-4l1.265625 -5.9377327q0.21875 -1.046917 0.09375 -1.4062691q-0.109375 -0.35935593 -0.65625 -0.35925293q-0.28125 4.9591064E-5 -0.625 0.25011444q-0.34375 0.25006104 -0.640625 0.71886826q-0.28125 0.4531746 -0.40625 1.0469475l-1.203125 5.6877213l-1.234375 2.2888184E-4l1.8125 -8.531586zm11.4375 4.263523l-1.34375 1.031498l-0.6875 3.234501l-1.359375 2.5177002E-4l2.640625 -12.406738l1.53125 -2.784729E-4l-0.03125 0.12500381q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12500763 -0.109375 0.45314407l-1.484375 6.9690247l5.09375 -3.9540634q0.453125 0.09366608 0.984375 0.09357071l0.484375 -8.773804E-5l-4.3125 3.3757896l3.109375 5.0931816l-1.75 0.078445435l-2.5625 -4.343281zm9.015625 4.4514694q-0.90625 1.6784668E-4 -1.609375 -0.51533127q-0.6875 -0.5154953 -0.96875 -1.5310707q-0.28125 -1.0311966 0.03125 -2.4843788q0.3125 -1.4844322 1.046875 -2.4689445q0.734375 -0.9845085 1.65625 -1.4534302q0.921875 -0.48454285 1.828125 -0.4847107q0.859375 -1.5640259E-4 1.40625 0.3903656q0.5625 0.390522 0.765625 1.0779877l1.09375 -5.125202l1.421875 -2.632141E-4l-0.03125 0.12500763q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12500763 -0.109375 0.45314407l-2.171875 10.250401q-0.09375 0.45314026 -0.125 0.7500229q-0.03125 0.2812538 0.03125 0.5781174l-1.328125 2.4414062E-4q-0.0625 -0.29686356 -0.03125 -0.5781174q0.03125 -0.29688263 0.125 -0.7500229q-0.5625 0.718853 -1.296875 1.1252365q-0.71875 0.3907585 -1.53125 0.3909073zm0.46875 -1.1719627q1.15625 -2.0980835E-4 1.890625 -0.90659714q0.734375 -0.9220085 1.0625 -2.4220695q0.3125 -1.5156822 -0.078125 -2.4218597q-0.390625 -0.9218025 -1.578125 -0.9215851q-1.109375 2.0217896E-4 -1.890625 0.84409714q-0.78125 0.82826996 -1.078125 2.2658234q-0.34375 1.640686 0.0625 2.6093636q0.421875 0.9530487 1.609375 0.95282745zm6.40625 0.9832001l0.234375 -1.0781708l2.1875 -4.005432E-4l1.34375 -6.359623l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.078167l-5.53125 0.0010185242zm4.96875 -10.313416q-0.390625 7.247925E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626526 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.2813835q0.390625 -7.247925E-5 0.625 0.29675674q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39063644 -0.4375 0.6719551q-0.34375 0.28131104 -0.734375 0.2813835zm11.890625 3.5603142l0 0.015625q-0.453125 -0.49991608 -0.78125 -0.67173004q-0.328125 -0.17181396 -0.8125 -0.17172623q-0.671875 1.2207031E-4 -1.34375 0.34399796q-0.65625 0.32824326 -1.1875 1.0158424q-0.515625 0.67197037 -0.734375 1.7032585l-0.953125 4.5314255l-1.359375 2.5177002E-4l1.8125 -8.547211l1.40625 -2.5558472E-4l-0.375 1.5781937q0.53125 -0.8438492 1.359375 -1.3127518q0.84375 -0.4689026 1.71875 -0.46906662q1.375 -2.5177002E-4 2.0625 0.93712234l-0.8125 1.0470238zm0 0.015625q0.09375 0.10935974 0.03125 0.093746185q-0.046875 -0.015617371 -0.09375 -0.031234741l0.0625 -0.062511444zm-0.21875 0.06254196q0.015625 -0.062503815 0.0625 -0.046886444q0.046875 -1.1444092E-5 0.09375 0.046855927l-0.0625 0.093761444l-0.09375 -0.07810593l0 -0.015625z" fill-rule="nonzero"/><path fill="#000000" d="m164.94662 75.455574l0.234375 -1.0781708l2.53125 -4.6539307E-4l2.171875 -10.250397l-2.421875 4.4250488E-4l0.234375 -1.078167l3.78125 -6.942749E-4l-2.40625 11.328564l2.5 -4.5776367E-4l-0.234375 1.0781708l-6.390625 0.0011749268zm9.71875 -0.0017852783l0.234375 -1.0781708l2.1875 -4.043579E-4l1.34375 -6.359619l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.0781631l-5.53125 0.0010223389zm4.96875 -10.313416q-0.390625 6.866455E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626907 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.28138733q0.390625 -7.247925E-5 0.625 0.29676056q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39064026 -0.4375 0.6719513q-0.34375 0.28131104 -0.734375 0.28138733zm8.84375 5.29525q1.515625 0.4840927 2.046875 1.0621185q0.53125 0.56240845 0.34375 1.468689q-0.25 1.1562958 -1.3125 1.9221191q-1.0625 0.75019073 -2.78125 0.75051117q-2.171875 3.9672852E-4 -3.328125 -1.3431396l1.015625 -1.2658081l0.015625 -0.01563263l0.015625 0.015625q0.421875 0.7499237 0.953125 1.1248245q0.546875 0.35927582 1.609375 0.35907745q1.015625 -1.8310547E-4 1.671875 -0.35968018q0.65625 -0.35949707 0.78125 -1.000145q0.109375 -0.5312729 -0.28125 -0.8905716q-0.390625 -0.35930634 -1.515625 -0.74972534q-3.015625 -0.90569305 -2.65625 -2.6401367q0.21875 -1.0000381 1.15625 -1.5783386q0.9375 -0.57829285 2.453125 -0.57857513q1.15625 -2.1362305E-4 1.875 0.32778168q0.71875 0.3279953 1.234375 1.0153961l-0.96875 0.9376831l-0.015625 0.015625q-0.265625 -0.5937042 -0.921875 -0.93733215q-0.640625 -0.34362793 -1.390625 -0.3434906q-0.796875 1.449585E-4 -1.375 0.28150177q-0.578125 0.2813568 -0.671875 0.7813721q-0.109375 0.4687729 0.328125 0.85931396q0.453125 0.39054108 1.71875 0.7809372zm2.09375 -1.3910141q0.015625 -0.09375 0.140625 0.031227112l-0.078125 0.06251526l-0.0625 -0.09374237zm0.21875 -0.031288147q0.03125 0.07811737 0.015625 0.09375q-0.015625 0.015625 -0.046875 7.6293945E-6q-0.03125 -0.015617371 -0.046875 -0.03124237l0.078125 -0.06251526zm-6.09375 3.9386215q-0.015625 0.078125 -0.171875 3.0517578E-5l0.078125 -0.09376526l0.09375 0.07810974l0 0.015625zm-0.21875 0.06253815q-0.046875 -0.09374237 -0.03125 -0.09374237q0.03125 -7.6293945E-6 0.078125 0.031234741l-0.046875 0.06250763zm15.4375 1.6534119q-1.4375 0.906517 -2.90625 0.90678406q-1.421875 2.670288E-4 -1.828125 -0.8434143q-0.40625 -0.8592987 0.015625 -2.8125q0.0625 -0.31251526 0.265625 -1.0937958l0.765625 -2.79702l-1.875 3.4332275E-4l0.234375 -1.1094131l1.953125 -3.5858154E-4l0.609375 -2.2657394l1.546875 -0.2502823l0.1875 -0.015663147l0 0.109375q-0.171875 0.18753052 -0.265625 0.32817078q-0.09375 0.12502289 -0.171875 0.40628815l-0.5625 1.6875992l2.828125 -5.187988E-4l-0.234375 1.1094208l-2.90625 5.340576E-4l-0.78125 2.89077q-0.203125 0.7500305 -0.25 0.9844208q-0.3125 1.5000534 -0.09375 2.0312653q0.234375 0.5312042 1.0 0.5310669q0.5625 -1.0681152E-4 1.078125 -0.20332336q0.515625 -0.21884918 1.203125 -0.65647125l0.1875 1.0624619zm5.1875 0.9677963q-0.90625 1.6784668E-4 -1.609375 -0.51532745q-0.6875 -0.5154953 -0.96875 -1.5310745q-0.28125 -1.0311966 0.03125 -2.484375q0.3125 -1.484436 1.046875 -2.4689484q0.734375 -0.9845047 1.65625 -1.4534302q0.921875 -0.48454285 1.828125 -0.4847107q0.859375 -1.5258789E-4 1.40625 0.39037323q0.5625 0.3905182 0.765625 1.07798l1.09375 -5.125202l1.421875 -2.593994E-4l-0.03125 0.12500381q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12501144 -0.109375 0.4531479l-2.171875 10.250397q-0.09375 0.45314026 -0.125 0.7500229q-0.03125 0.28125763 0.03125 0.5781174l-1.328125 2.4414062E-4q-0.0625 -0.29685974 -0.03125 -0.5781174q0.03125 -0.29688263 0.125 -0.7500229q-0.5625 0.7188568 -1.296875 1.1252365q-0.71875 0.39076233 -1.53125 0.3909073zm0.46875 -1.1719589q1.15625 -2.1362305E-4 1.890625 -0.90660095q0.734375 -0.9220047 1.0625 -2.4220657q0.3125 -1.515686 -0.078125 -2.4218597q-0.390625 -0.92180634 -1.578125 -0.9215851q-1.109375 1.9836426E-4 -1.890625 0.8440933q-0.78125 0.82826996 -1.078125 2.2658234q-0.34375 1.6406937 0.0625 2.6093674q0.421875 0.9530487 1.609375 0.95282745zm6.40625 0.9832001l0.234375 -1.0781708l2.1875 -4.043579E-4l1.34375 -6.359619l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.0781631l-5.53125 0.0010223389zm4.96875 -10.313416q-0.390625 6.866455E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626907 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.28138733q0.390625 -7.247925E-5 0.625 0.29676056q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39064026 -0.4375 0.6719513q-0.34375 0.28131104 -0.734375 0.28138733zm11.890625 3.5603104l0 0.015625q-0.453125 -0.49991608 -0.78125 -0.67173004q-0.328125 -0.17181396 -0.8125 -0.17172241q-0.671875 1.2207031E-4 -1.34375 0.34399414q-0.65625 0.32824707 -1.1875 1.0158463q-0.515625 0.67196655 -0.734375 1.7032547l-0.953125 4.5314255l-1.359375 2.5177002E-4l1.8125 -8.547203l1.40625 -2.593994E-4l-0.375 1.5781937q0.53125 -0.8438492 1.359375 -1.3127518q0.84375 -0.4689026 1.71875 -0.46907043q1.375 -2.5177002E-4 2.0625 0.93712616l-0.8125 1.04702zm0 0.015625q0.09375 0.10935974 0.03125 0.09375q-0.046875 -0.015617371 -0.09375 -0.031234741l0.0625 -0.06251526zm-0.21875 0.06254578q0.015625 -0.06250763 0.0625 -0.04689026q0.046875 -7.6293945E-6 0.09375 0.04685974l-0.0625 0.09375763l-0.09375 -0.07810211l0 -0.015625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m124.98425 40.490856c3.682846 -3.459755 6.026245 -16.741034 22.097107 -20.758532c16.070877 -4.017497 54.685913 -8.257217 74.328094 -3.3464565c19.642166 4.910761 36.270782 27.342522 43.524933 32.811024" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m124.98425 40.490856c3.6828613 -3.459755 6.026245 -16.741034 22.097107 -20.758532c16.070877 -4.017497 54.68593 -8.257218 74.328094 -3.3464565c9.82106 2.4553795 18.888779 9.29101 26.407974 16.196358c3.7595825 3.4526749 7.1320496 6.922779 10.018021 9.871494c0.72146606 0.7371788 1.4125671 1.4417686 2.0716858 2.1053543l0.594635 0.594059" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m259.38858 46.373383l4.46579 1.8382454l-2.23938 -4.27874z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m220.74016 15.71573c65.288284 0 317.17804 -2.5669298 391.7296 0c74.551636 2.566928 46.31671 12.834645 55.58008 15.401574" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m220.74016 15.71573c65.288284 0 317.17804 -2.5669298 391.7296 0c37.27588 1.2834635 48.85504 4.4921255 52.273132 7.7007875c0.42730713 0.401083 0.7270508 0.802166 0.9335327 1.1994877c0.10321045 0.1986618 0.1831665 0.3963833 0.24407959 0.59269524c0.015197754 0.04907608 0.02923584 0.09806633 0.042175293 0.14696121l0.030273438 0.12522316" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m664.44135 26.04712l3.1073608 3.6968708l-0.0040893555 -4.829342z" fill-rule="evenodd"/></g></svg>
\ No newline at end of file
diff --git a/doc/cephfs/cephfs-io-path.rst b/doc/cephfs/cephfs-io-path.rst
new file mode 100644
index 000000000..8c7810ba0
--- /dev/null
+++ b/doc/cephfs/cephfs-io-path.rst
@@ -0,0 +1,50 @@
+=========================
+ Ceph File System IO Path
+=========================
+
+All file data in CephFS is stored as RADOS objects. CephFS clients can directly
+access RADOS to operate on file data. MDS only handles metadata operations.
+
+To read/write a CephFS file, client needs to have 'file read/write' capabilities
+for corresponding inode. If client does not have required capabilities, it sends
+a 'cap message' to MDS, telling MDS what it wants. MDS will issue capabilities
+to client when it is possible. Once client has 'file read/write' capabilities,
+it can directly access RADOS to read/write file data. File data are stored as
+RADOS objects in the form of <inode number>.<object index>. See 'Data Striping'
+section of `Architecture`_ for more information. If the file is only opened by
+one client, MDS also issues 'file cache/buffer' capabilities to the only client.
+The 'file cache' capability means that file read can be satisfied by client
+cache. The 'file buffer' capability means that file write can be buffered in
+client cache.
+
+
+.. ditaa::
+
+            +---------------------+
+            |     Application     |
+            +---------------------+
+              |
+              V
+            +---------------------+  Data IOs  +--------------------+
+            |    CephFS Library   | ---------> |       LibRados     |
+            +---------------------+            +--------------------+
+              |                                  |
+              | Metadata Operations              | Objects Read/Write
+              V                                  V
+            +---------------------+            +--------------------+
+            |         MDSs        | -=-------> |         OSDs       |
+            +---------------------+            +--------------------+
+
+
+            +----------------------+           +---------------------+
+            | CephFS kernel client | Data IOs  | Ceph kernel library |
+            |      (ceph.ko)       | --------> |     (libceph.ko)    |
+            +----------------------+           +---------------------+
+              |                                  |
+              | Metadata Operations              | Objects Read/Write
+              v                                  v
+            +---------------------+            +--------------------+
+            |         MDSs        | -=-------> |         OSDs       |
+            +---------------------+            +--------------------+
+
+.. _Architecture: ../architecture
diff --git a/doc/cephfs/cephfs-journal-tool.rst b/doc/cephfs/cephfs-journal-tool.rst
new file mode 100644
index 000000000..64a113091
--- /dev/null
+++ b/doc/cephfs/cephfs-journal-tool.rst
@@ -0,0 +1,238 @@
+
+cephfs-journal-tool
+===================
+
+Purpose
+-------
+
+If a CephFS journal has become damaged, expert intervention may be required
+to restore the file system to a working state.
+
+The ``cephfs-journal-tool`` utility provides functionality to aid experts in
+examining, modifying, and extracting data from journals.
+
+.. warning::
+
+    This tool is **dangerous** because it directly modifies internal
+    data structures of the file system.  Make backups, be careful, and
+    seek expert advice.  If you are unsure, do not run this tool.
+
+Syntax
+------
+
+::
+
+    cephfs-journal-tool journal <inspect|import|export|reset>
+    cephfs-journal-tool header <get|set>
+    cephfs-journal-tool event <get|splice|apply> [filter] <list|json|summary|binary>
+
+
+The tool operates in three modes: ``journal``, ``header`` and ``event``,
+meaning the whole journal, the header, and the events within the journal
+respectively.
+
+Journal mode
+------------
+
+This should be your starting point to assess the state of a journal.
+
+* ``inspect`` reports on the health of the journal.  This will identify any
+  missing objects or corruption in the stored journal.  Note that this does
+  not identify inconsistencies in the events themselves, just that events are
+  present and can be decoded.
+
+* ``import`` and ``export`` read and write binary dumps of the journal
+  in a sparse file format.  Pass the filename as the last argument.  The
+  export operation may not work reliably for journals which are damaged (missing
+  objects).
+
+* ``reset`` truncates a journal, discarding any information within it.
+
+
+Example: journal inspect
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool journal inspect
+    Overall journal integrity: DAMAGED
+    Objects missing:
+      0x1
+    Corrupt regions:
+      0x400000-ffffffffffffffff
+
+Example: Journal import/export
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool journal export myjournal.bin
+    journal is 4194304~80643
+    read 80643 bytes at offset 4194304
+    wrote 80643 bytes at offset 4194304 to myjournal.bin
+    NOTE: this is a _sparse_ file; you can
+        $ tar cSzf myjournal.bin.tgz myjournal.bin
+          to efficiently compress it while preserving sparseness.
+
+    # cephfs-journal-tool journal import myjournal.bin
+    undump myjournal.bin
+    start 4194304 len 80643
+    writing header 200.00000000
+     writing 4194304~80643
+    done.
+
+.. note::
+
+    It is wise to use the ``journal export <backup file>`` command to make a journal backup
+    before any further manipulation.
+
+Header mode
+-----------
+
+* ``get`` outputs the current content of the journal header
+
+* ``set`` modifies an attribute of the header.  Allowed attributes are
+  ``trimmed_pos``, ``expire_pos`` and ``write_pos``.
+
+Example: header get/set
+~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool header get
+    { "magic": "ceph fs volume v011",
+      "write_pos": 4274947,
+      "expire_pos": 4194304,
+      "trimmed_pos": 4194303,
+      "layout": { "stripe_unit": 4194304,
+          "stripe_count": 4194304,
+          "object_size": 4194304,
+          "cas_hash": 4194304,
+          "object_stripe_unit": 4194304,
+          "pg_pool": 4194304}}
+
+    # cephfs-journal-tool header set trimmed_pos 4194303
+    Updating trimmed_pos 0x400000 -> 0x3fffff
+    Successfully updated header.
+
+
+Event mode
+----------
+
+Event mode allows detailed examination and manipulation of the contents of the journal.  Event
+mode can operate on all events in the journal, or filters may be applied.
+
+The arguments following ``cephfs-journal-tool event`` consist of an action, optional filter
+parameters, and an output mode:
+
+::
+
+    cephfs-journal-tool event <action> [filter] <output>
+
+Actions:
+
+* ``get`` read the events from the log
+* ``splice`` erase events or regions in the journal
+* ``apply`` extract file system metadata from events and attempt to apply it to the metadata store.
+
+Filtering:
+
+* ``--range <int begin>..[int end]`` only include events within the range begin (inclusive) to end (exclusive)
+* ``--path <path substring>`` only include events referring to metadata containing the specified string
+* ``--inode <int>`` only include events referring to metadata containing the specified inode
+* ``--type <type string>`` only include events of this type
+* ``--frag <ino>[.frag id]`` only include events referring to this directory fragment
+* ``--dname <string>`` only include events referring to this named dentry within a directory
+  fragment (may only be used in conjunction with ``--frag``
+* ``--client <int>`` only include events from this client session ID
+
+Filters may be combined on an AND basis (i.e. only the intersection of events from each filter).
+
+Output modes:
+
+* ``binary``: write each event as a binary file, within a folder whose name is controlled by ``--path``
+* ``json``: write all events to a single file, as a JSON serialized list of objects
+* ``summary``: write a human readable summary of the events read to standard out
+* ``list``: write a human readable terse listing of the type of each event, and
+  which file paths the event affects.
+
+
+Example: event mode
+~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool event get json --path output.json
+    Wrote output to JSON file 'output.json'
+
+    # cephfs-journal-tool event get summary
+    Events by type:
+      NOOP: 2
+      OPEN: 2
+      SESSION: 2
+      SUBTREEMAP: 1
+      UPDATE: 43
+
+    # cephfs-journal-tool event get list
+    0x400000 SUBTREEMAP:  ()
+    0x400308 SESSION:  ()
+    0x4003de UPDATE:  (setattr)
+      /
+    0x40068b UPDATE:  (mkdir)
+      diralpha
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+    0x40228d UPDATE:  (unlink_local)
+      diralpha
+      stray0/10000000000
+    0x402bf9 UPDATE:  (scatter_writebehind)
+      stray0
+    0x403150 UPDATE:  (mkdir)
+      dirbravo
+    0x4037e0 UPDATE:  (openc)
+      dirbravo/.filebravo1.swp
+    0x404032 UPDATE:  (openc)
+      dirbravo/.filebravo1.swpx
+
+    # cephfs-journal-tool event get --path filebravo1 list
+    0x40785a UPDATE:  (openc)
+      dirbravo/filebravo1
+    0x4103ee UPDATE:  (cap update)
+      dirbravo/filebravo1
+
+    # cephfs-journal-tool event splice --range 0x40f754..0x410bf1 summary
+    Events by type:
+      OPEN: 1
+      UPDATE: 2
+
+    # cephfs-journal-tool event apply --range 0x410bf1.. summary
+    Events by type:
+      NOOP: 1
+      SESSION: 1
+      UPDATE: 9
+
+    # cephfs-journal-tool event get --inode=1099511627776 list
+    0x40068b UPDATE:  (mkdir)
+      diralpha
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+    0x40228d UPDATE:  (unlink_local)
+      diralpha
+      stray0/10000000000
+
+    # cephfs-journal-tool event get --frag=1099511627776 --dname=filealpha1 list
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+
+    # cephfs-journal-tool event get binary --path bin_events
+    Wrote output to binary files in directory 'bin_events'
+
diff --git a/doc/cephfs/cephfs-mirroring.rst b/doc/cephfs/cephfs-mirroring.rst
new file mode 100644
index 000000000..fd00a1eef
--- /dev/null
+++ b/doc/cephfs/cephfs-mirroring.rst
@@ -0,0 +1,414 @@
+.. _cephfs-mirroring:
+
+=========================
+CephFS Snapshot Mirroring
+=========================
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file system via
+the `cephfs-mirror` tool. Snapshots are synchronized by mirroring snapshot data followed by
+creating a remote snapshot with the same name (for a given directory on the remote file system) as
+the source snapshot.
+
+Requirements
+------------
+
+The primary (local) and secondary (remote) Ceph clusters version should be Pacific or later.
+
+.. _cephfs_mirroring_creating_users:
+
+Creating Users
+--------------
+
+Start by creating a Ceph user (on the primary/local cluster) for the `cephfs-mirror` daemon. This user
+requires write capability on the metadata pool to create RADOS objects (index objects)
+for watch/notify operation and read capability on the data pool(s)::
+
+  $ ceph auth get-or-create client.mirror mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'
+
+Create a Ceph user for each file system peer (on the secondary/remote cluster). This user needs
+to have full capabilities on the MDS (to take snapshots) and the OSDs::
+
+  $ ceph fs authorize <fs_name> client.mirror_remote / rwps
+
+This user will be supplied as part of the peer specification when adding a peer.
+
+Starting Mirror Daemon
+----------------------
+
+The mirror daemon should be spawned using `systemctl(1)` unit files::
+
+  $ systemctl enable cephfs-mirror@mirror
+  $ systemctl start cephfs-mirror@mirror
+
+`cephfs-mirror` daemon can be run in foreground using::
+
+  $ cephfs-mirror --id mirror --cluster site-a -f
+
+.. note:: The user specified here is `mirror`, the creation of which is
+   described in the :ref:`Creating Users<cephfs_mirroring_creating_users>`
+   section.
+
+Multiple ``cephfs-mirror`` daemons may be deployed for concurrent
+synchronization and high availability. Mirror daemons share the synchronization
+load using a simple ``M/N`` policy, where ``M`` is the number of directories
+and ``N`` is the number of ``cephfs-mirror`` daemons.
+
+When ``cephadm`` is used to manage a Ceph cluster, ``cephfs-mirror`` daemons can be
+deployed by running the following command:
+
+.. prompt:: bash $
+
+   ceph orch apply cephfs-mirror
+
+To deploy multiple mirror daemons, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph orch apply cephfs-mirror --placement=<placement-spec>
+
+For example, to deploy 3 `cephfs-mirror` daemons on different hosts, run a command of the following form:
+
+.. prompt:: bash $
+
+  $ ceph orch apply cephfs-mirror --placement="3 host1,host2,host3"
+
+Interface
+---------
+
+The `Mirroring` module (manager plugin) provides interfaces for managing
+directory snapshot mirroring. These are (mostly) wrappers around monitor
+commands for managing file system mirroring and is the recommended control
+interface.
+
+Mirroring Module
+----------------
+
+The mirroring module is responsible for assigning directories to mirror daemons
+for synchronization. Multiple mirror daemons can be spawned to achieve
+concurrency in directory snapshot synchronization. When mirror daemons are
+spawned (or terminated), the mirroring module discovers the modified set of
+mirror daemons and rebalances directory assignments across the new set, thus
+providing high-availability.
+
+.. note:: Deploying a single mirror daemon is recommended. Running multiple
+   daemons is untested.
+
+The mirroring module is disabled by default. To enable the mirroring module,
+run the following command:
+
+.. prompt:: bash $
+
+   ceph mgr module enable mirroring
+
+The mirroring module provides a family of commands that can be used to control
+the mirroring of directory snapshots. To add or remove directories, mirroring
+must be enabled for a given file system. To enable mirroring for a given file
+system, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror enable <fs_name>
+
+.. note:: "Mirroring module" commands are prefixed with ``fs snapshot mirror``.
+   This distinguishes them from "monitor commands", which are prefixed with ``fs
+   mirror``. Be sure (in this context) to use module commands.
+
+To disable mirroring for a given file system, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror disable <fs_name>
+
+After mirroring is enabled, add a peer to which directory snapshots are to be
+mirrored. Peers are specified by the ``<client>@<cluster>`` format, which is
+referred to elsewhere in this document as the ``remote_cluster_spec``. Peers
+are assigned a unique-id (UUID) when added. See the :ref:`Creating
+Users<cephfs_mirroring_creating_users>` section for instructions that describe
+how to create Ceph users for mirroring.
+
+To add a peer, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror peer_add <fs_name> <remote_cluster_spec> [<remote_fs_name>] [<remote_mon_host>] [<cephx_key>]
+
+``<remote_cluster_spec>`` is of the format ``client.<id>@<cluster_name>``.
+
+``<remote_fs_name>`` is optional, and defaults to `<fs_name>` (on the remote
+cluster).
+
+For this command to succeed, the remote cluster's Ceph configuration and user
+keyring must be available in the primary cluster. For example, if a user named
+``client_mirror`` is created on the remote cluster which has ``rwps``
+permissions for the remote file system named ``remote_fs`` (see `Creating
+Users`) and the remote cluster is named ``remote_ceph`` (that is, the remote
+cluster configuration file is named ``remote_ceph.conf`` on the primary
+cluster), run the following command to add the remote filesystem as a peer to
+the primary filesystem ``primary_fs``:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror peer_add primary_fs client.mirror_remote@remote_ceph remote_fs
+
+To avoid having to maintain the remote cluster configuration file and remote
+ceph user keyring in the primary cluster, users can bootstrap a peer (which
+stores the relevant remote cluster details in the monitor config store on the
+primary cluster). See the :ref:`Bootstrap
+Peers<cephfs_mirroring_bootstrap_peers>` section.
+
+The ``peer_add`` command supports passing the remote cluster monitor address
+and the user key. However, bootstrapping a peer is the recommended way to add a
+peer.
+
+.. note:: Only a single peer is currently supported.
+
+To remove a peer, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror peer_remove <fs_name> <peer_uuid>
+
+To list file system mirror peers, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror peer_list <fs_name>
+
+To configure a directory for mirroring, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror add <fs_name> <path>
+
+To stop mirroring directory snapshots, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs snapshot mirror remove <fs_name> <path>
+
+Only absolute directory paths are allowed. 
+
+Paths are normalized by the mirroring module. This means that ``/a/b/../b`` is
+equivalent to ``/a/b``. Paths always start from the CephFS file-system root and
+not from the host system mount point.
+
+For example::
+
+  $ mkdir -p /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2
+  {}
+  $ ceph fs snapshot mirror add cephfs /d0/d1/../d1/d2
+  Error EEXIST: directory /d0/d1/d2 is already tracked
+
+After a directory is added for mirroring, the additional mirroring of
+subdirectories or ancestor directories is disallowed::
+
+  $ ceph fs snapshot mirror add cephfs /d0/d1
+  Error EINVAL: /d0/d1 is a ancestor of tracked path /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2/d3
+  Error EINVAL: /d0/d1/d2/d3 is a subtree of tracked path /d0/d1/d2
+
+The :ref:`Mirroring Status<cephfs_mirroring_mirroring_status>` section contains
+information about the commands for checking the directory mapping (to mirror
+daemons) and for checking the directory distribution. 
+
+.. _cephfs_mirroring_bootstrap_peers:
+
+Bootstrap Peers
+---------------
+
+Adding a peer (via `peer_add`) requires the peer cluster configuration and user keyring
+to be available in the primary cluster (manager host and hosts running the mirror daemon).
+This can be avoided by bootstrapping and importing a peer token. Peer bootstrap involves
+creating a bootstrap token on the peer cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap create backup_fs client.mirror_remote site-remote
+  {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="}
+
+`site-name` refers to a user-defined string to identify the remote filesystem. In context
+of `peer_add` interface, `site-name` is the passed in `cluster` name from `remote_cluster_spec`.
+
+Import the bootstrap token in the primary cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ==
+
+
+.. _cephfs_mirroring_mirroring_status:
+
+Mirroring Status
+----------------
+
+CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon status::
+
+  $ ceph fs snapshot mirror daemon status
+  [
+    {
+      "daemon_id": 284167,
+      "filesystems": [
+        {
+          "filesystem_id": 1,
+          "name": "a",
+          "directory_count": 1,
+          "peers": [
+            {
+              "uuid": "02117353-8cd1-44db-976b-eb20609aa160",
+              "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "ceph",
+                "fs_name": "backup_fs"
+              },
+              "stats": {
+                "failure_count": 1,
+                "recovery_count": 0
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+
+An entry per mirror daemon instance is displayed along with information such as configured
+peers and basic stats. For more detailed stats, use the admin socket interface as detailed
+below.
+
+CephFS mirror daemons provide admin socket commands for querying mirror status. To check
+available commands for mirror status use::
+
+  $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help
+  {
+      ....
+      ....
+      "fs mirror status cephfs@360": "get filesystem mirror status",
+      ....
+      ....
+  }
+
+Commands prefixed with`fs mirror status` provide mirror status for mirror enabled
+file systems. Note that `cephfs@360` is of format `filesystem-name@filesystem-id`.
+This format is required since mirror daemons get asynchronously notified regarding
+file system mirror status (A file system can be deleted and recreated with the same
+name).
+
+This command currently provides minimal information regarding mirror status::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror status cephfs@360
+  {
+    "rados_inst": "192.168.0.5:0/1476644347",
+    "peers": {
+        "a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
+            "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "site-a",
+                "fs_name": "backup_fs"
+            }
+        }
+    },
+    "snap_dirs": {
+        "dir_count": 1
+    }
+  }
+
+The `Peers` section in the command output above shows the peer information including the unique
+peer-id (UUID) and specification. The peer-id is required when removing an existing peer
+as mentioned in the `Mirror Module and Interface` section.
+
+Commands prefixed with `fs mirror peer status` provide peer synchronization status. This
+command is of format `filesystem-name@filesystem-id peer-uuid`::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+Synchronization stats including `snaps_synced`, `snaps_deleted` and `snaps_renamed` are reset
+on daemon restart and/or when a directory is reassigned to another mirror daemon (when
+multiple mirror daemons are deployed).
+
+A directory can be in one of the following states::
+
+  - `idle`: The directory is currently not being synchronized
+  - `syncing`: The directory is currently being synchronized
+  - `failed`: The directory has hit upper limit of consecutive failures
+
+When a directory experiences a configured number of consecutive synchronization failures, the
+mirror daemon marks it as `failed`. Synchronization for these directories is retried.
+By default, the number of consecutive failures before a directory is marked as failed
+is controlled by `cephfs_mirror_max_consecutive_failures_per_directory` configuration
+option (default: 10) and the retry interval for failed directories is controlled via
+`cephfs_mirror_retry_failed_directories_interval` configuration option (default: 60s).
+
+E.g., adding a regular file for synchronization would result in failed status::
+
+  $ ceph fs snapshot mirror add cephfs /f0
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    },
+    "/f0": {
+        "state": "failed",
+        "snaps_synced": 0,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+This allows a user to add a non-existent directory for synchronization. The mirror daemon
+will mark such a directory as failed and retry (less frequently). When the directory is
+created, the mirror daemon will clear the failed state upon successful synchronization.
+
+When mirroring is disabled, the respective `fs mirror status` command for the file system
+will not show up in command help.
+
+Configuration Options
+---------------------
+
+.. confval:: cephfs_mirror_max_concurrent_directory_syncs
+.. confval:: cephfs_mirror_action_update_interval
+.. confval:: cephfs_mirror_restart_mirror_on_blocklist_interval
+.. confval:: cephfs_mirror_max_snapshot_sync_per_cycle
+.. confval:: cephfs_mirror_directory_scan_interval
+.. confval:: cephfs_mirror_max_consecutive_failures_per_directory
+.. confval:: cephfs_mirror_retry_failed_directories_interval
+.. confval:: cephfs_mirror_restart_mirror_on_failure_interval
+.. confval:: cephfs_mirror_mount_timeout
+
+Re-adding Peers
+---------------
+
+When re-adding (reassigning) a peer to a file system in another cluster, ensure that
+all mirror daemons have stopped synchronization to the peer. This can be checked
+via `fs mirror status` admin socket command (the `Peer UUID` should not show up
+in the command output). Also, it is recommended to purge synchronized directories
+from the peer  before re-adding it to another file system (especially those directories
+which might exist in the new primary file system). This is not required if re-adding
+a peer to the same primary file system it was earlier synchronized from.
diff --git a/doc/cephfs/cephfs-top.png b/doc/cephfs/cephfs-top.png
new file mode 100644
index 000000000..5215eb98c
--- /dev/null
+++ b/doc/cephfs/cephfs-top.png
diff --git a/doc/cephfs/cephfs-top.rst b/doc/cephfs/cephfs-top.rst
new file mode 100644
index 000000000..49439a4bd
--- /dev/null
+++ b/doc/cephfs/cephfs-top.rst
@@ -0,0 +1,116 @@
+.. _cephfs-top:
+
+==================
+CephFS Top Utility
+==================
+
+CephFS provides `top(1)` like utility to display various Ceph Filesystem metrics
+in realtime. `cephfs-top` is a curses based python script which makes use of `stats`
+plugin in Ceph Manager to fetch (and display) metrics.
+
+Manager Plugin
+==============
+
+Ceph Filesystem clients periodically forward various metrics to Ceph Metadata Servers (MDS)
+which in turn get forwarded to Ceph Manager by MDS rank zero. Each active MDS forward its
+respective set of metrics to MDS rank zero. Metrics are aggregated and forwarded to Ceph
+Manager.
+
+Metrics are divided into two categories - global and per-mds. Global metrics represent
+set of metrics for the filesystem as a whole (e.g., client read latency) whereas per-mds
+metrics are for a particular MDS rank (e.g., number of subtrees handled by an MDS).
+
+.. note:: Currently, only global metrics are tracked.
+
+`stats` plugin is disabled by default and should be enabled via::
+
+  $ ceph mgr module enable stats
+
+Once enabled, Ceph Filesystem metrics can be fetched via::
+
+  $ ceph fs perf stats
+
+The output format is JSON and contains fields as follows:
+
+- `version`: Version of stats output
+- `global_counters`: List of global performance metrics
+- `counters`: List of per-mds performance metrics
+- `client_metadata`: Ceph Filesystem client metadata
+- `global_metrics`: Global performance counters
+- `metrics`: Per-MDS performance counters (currently, empty) and delayed ranks
+
+.. note:: `delayed_ranks` is the set of active MDS ranks that are reporting stale metrics.
+          This can happen in cases such as (temporary) network issue between MDS rank zero
+          and other active MDSs.
+
+Metrics can be fetched for a particular client and/or for a set of active MDSs. To fetch metrics
+for a particular client (e.g., for client-id: 1234)::
+
+  $ ceph fs perf stats --client_id=1234
+
+To fetch metrics only for a subset of active MDSs (e.g., MDS rank 1 and 2)::
+
+  $ ceph fs perf stats --mds_rank=1,2
+
+`cephfs-top`
+============
+
+`cephfs-top` utility relies on `stats` plugin to fetch performance metrics and display in
+`top(1)` like format. `cephfs-top` is available as part of `cephfs-top` package.
+
+By default, `cephfs-top` uses `client.fstop` user to connect to a Ceph cluster::
+
+  $ ceph auth get-or-create client.fstop mon 'allow r' mds 'allow r' osd 'allow r' mgr 'allow r'
+  $ cephfs-top
+
+Command-Line Options
+--------------------
+
+To use a non-default user (other than `client.fstop`) use::
+
+  $ cephfs-top --id <name>
+
+By default, `cephfs-top` connects to cluster name `ceph`. To use a non-default cluster name::
+
+  $ cephfs-top --cluster <cluster>
+
+`cephfs-top` refreshes stats every second by default. To choose a different refresh interval use::
+
+  $ cephfs-top -d <seconds>
+
+Refresh interval should be a positive integer.
+
+To dump the metrics to stdout without creating a curses display use::
+
+  $ cephfs-top --dump
+
+To dump the metrics of the given filesystem to stdout without creating a curses display use::
+
+  $ cephfs-top --dumpfs <fs_name>
+
+Interactive Commands
+--------------------
+
+1. m : Filesystem selection
+      Displays a menu of filesystems for selection.
+
+2. s : Sort field selection
+      Designates the sort field.  'cap_hit' is the default.
+
+3. l : Client limit
+      Sets the limit on the number of clients to be displayed.
+
+4. r : Reset
+      Resets the sort field and limit value to the default.
+
+5. q : Quit
+      Exit the utility if you are at the home screen (all filesystem info),
+      otherwise escape back to the home screen.
+
+The metrics display can be scrolled using the Arrow Keys, PgUp/PgDn, Home/End and mouse.
+
+Sample screenshot running `cephfs-top` with 2 filesystems:
+
+.. image:: cephfs-top.png
+
+.. note:: Minimum compatible python version for cephfs-top is 3.6.0. cephfs-top is supported on distros RHEL 8, Ubuntu 18.04, CentOS 8 and above.
diff --git a/doc/cephfs/client-auth.rst b/doc/cephfs/client-auth.rst
new file mode 100644
index 000000000..a7dea5251
--- /dev/null
+++ b/doc/cephfs/client-auth.rst
@@ -0,0 +1,261 @@
+================================
+CephFS Client Capabilities
+================================
+
+Use Ceph authentication capabilities to restrict your file system clients
+to the lowest possible level of authority needed.
+
+.. note:: Path restriction and layout modification restriction are new features
+    in the Jewel release of Ceph.
+
+.. note:: Using Erasure Coded(EC) pools with CephFS is supported only with the
+   BlueStore Backend. They cannot be used as metadata pools and overwrites must
+   be enabled on the data pools.
+
+
+Path restriction
+================
+
+By default, clients are not restricted in what paths they are allowed to
+mount. Further, when clients mount a subdirectory, e.g., ``/home/user``, the
+MDS does not by default verify that subsequent operations are ‘locked’ within
+that directory.
+
+To restrict clients to only mount and work within a certain directory, use
+path-based MDS authentication capabilities.
+
+Note that this restriction *only* impacts the filesystem hierarchy -- the metadata
+tree managed by the MDS. Clients will still be able to access the underlying
+file data in RADOS directly. To segregate clients fully, you must also isolate
+untrusted clients in their own RADOS namespace. You can place a client's
+filesystem subtree in a particular namespace using `file layouts`_ and then
+restrict their RADOS access to that namespace using `OSD capabilities`_
+
+.. _file layouts: ./file-layouts
+.. _OSD capabilities: ../rados/operations/user-management/#authorization-capabilities
+
+Syntax
+------
+
+To grant rw access to the specified directory only, we mention the specified
+directory while creating key for a client using the following syntax::
+
+ ceph fs authorize <fs_name> client.<client_id> <path-in-cephfs> rw
+
+For example, to restrict client ``foo`` to writing only in the ``bar``
+directory of file system ``cephfs_a``, use ::
+
+ ceph fs authorize cephfs_a client.foo / r /bar rw
+
+ results in:
+
+ client.foo
+   key: *key*
+   caps: [mds] allow r, allow rw path=/bar
+   caps: [mon] allow r
+   caps: [osd] allow rw tag cephfs data=cephfs_a
+
+To completely restrict the client to the ``bar`` directory, omit the
+root directory ::
+
+ ceph fs authorize cephfs_a client.foo /bar rw
+
+Note that if a client's read access is restricted to a path, they will only
+be able to mount the file system when specifying a readable path in the
+mount command (see below).
+
+Supplying ``all`` or ``*`` as the file system name will grant access to every
+file system. Note that it is usually necessary to quote ``*`` to protect it
+from the shell.
+
+See `User Management - Add a User to a Keyring`_. for additional details on
+user management
+
+To restrict a client to the specified sub-directory only, we mention the
+specified directory while mounting using the following syntax::
+
+ ceph-fuse -n client.<client_id> <mount-path> -r *directory_to_be_mounted*
+
+For example, to restrict client ``foo`` to ``mnt/bar`` directory, we will
+use::
+
+ ceph-fuse -n client.foo mnt -r /bar
+
+Free space reporting
+--------------------
+
+By default, when a client is mounting a sub-directory, the used space (``df``)
+will be calculated from the quota on that sub-directory, rather than reporting
+the overall amount of space used on the cluster.
+
+If you would like the client to report the overall usage of the file system,
+and not just the quota usage on the sub-directory mounted, then set the
+following config option on the client::
+
+
+    client quota df = false
+
+If quotas are not enabled, or no quota is set on the sub-directory mounted,
+then the overall usage of the file system will be reported irrespective of
+the value of this setting.
+
+Layout and Quota restriction (the 'p' flag)
+===========================================
+
+To set layouts or quotas, clients require the 'p' flag in addition to 'rw'.
+This restricts all the attributes that are set by special extended attributes
+with a "ceph." prefix, as well as restricting other means of setting
+these fields (such as openc operations with layouts).
+
+For example, in the following snippet client.0 can modify layouts and quotas
+on the file system cephfs_a, but client.1 cannot::
+
+    client.0
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rwp
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+    client.1
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rw
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+
+Snapshot restriction (the 's' flag)
+===========================================
+
+To create or delete snapshots, clients require the 's' flag in addition to
+'rw'. Note that when capability string also contains the 'p' flag, the 's'
+flag must appear after it (all flags except 'rw' must be specified in
+alphabetical order).
+
+For example, in the following snippet client.0 can create or delete snapshots
+in the ``bar`` directory of file system ``cephfs_a``::
+
+    client.0
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rw, allow rws path=/bar
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+
+.. _User Management - Add a User to a Keyring: ../../rados/operations/user-management/#add-a-user-to-a-keyring
+
+Network restriction
+===================
+
+::
+
+ client.foo
+   key: *key*
+   caps: [mds] allow r network 10.0.0.0/8, allow rw path=/bar network 10.0.0.0/8
+   caps: [mon] allow r network 10.0.0.0/8
+   caps: [osd] allow rw tag cephfs data=cephfs_a network 10.0.0.0/8
+
+The optional ``{network/prefix}`` is a standard network name and
+prefix length in CIDR notation (e.g., ``10.3.0.0/16``).  If present,
+the use of this capability is restricted to clients connecting from
+this network.
+
+.. _fs-authorize-multifs:
+
+File system Information Restriction
+===================================
+
+If desired, the monitor cluster can present a limited view of the file systems
+available. In this case, the monitor cluster will only inform clients about
+file systems specified by the administrator. Other file systems will not be
+reported and commands affecting them will fail as if the file systems do
+not exist.
+
+Consider following example. The Ceph cluster has 2 FSs::
+
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+    name: cephfs2, metadata pool: cephfs2_metadata, data pools: [cephfs2_data ]
+
+But we authorize client ``someuser`` for only one FS::
+
+    $ ceph fs authorize cephfs client.someuser / rw
+    [client.someuser]
+        key = AQAmthpf89M+JhAAiHDYQkMiCq3x+J0n9e8REQ==
+    $ cat ceph.client.someuser.keyring
+    [client.someuser]
+        key = AQAmthpf89M+JhAAiHDYQkMiCq3x+J0n9e8REQ==
+        caps mds = "allow rw fsname=cephfs"
+        caps mon = "allow r fsname=cephfs"
+        caps osd = "allow rw tag cephfs data=cephfs"
+
+And the client can only see the FS that it has authorization for::
+
+    $ ceph fs ls -n client.someuser -k ceph.client.someuser.keyring
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+
+Standby MDS daemons will always be displayed. Note that the information about
+restricted MDS daemons and file systems may become available by other means,
+such as ``ceph health detail``.
+
+MDS communication restriction
+=============================
+
+By default, user applications may communicate with any MDS, whether or not
+they are allowed to modify data on an associated file system (see
+`Path restriction` above). Client's communication can be restricted to MDS
+daemons associated with particular file system(s) by adding MDS caps for that
+particular file system. Consider the following example where the Ceph cluster
+has 2 FSs::
+
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+    name: cephfs2, metadata pool: cephfs2_metadata, data pools: [cephfs2_data ]
+
+Client ``someuser`` is authorized only for one FS::
+
+    $ ceph fs authorize cephfs client.someuser / rw
+    [client.someuser]
+        key = AQBPSARfg8hCJRAAEegIxjlm7VkHuiuntm6wsA==
+    $ ceph auth get client.someuser > ceph.client.someuser.keyring
+    exported keyring for client.someuser
+    $ cat ceph.client.someuser.keyring
+    [client.someuser]
+        key = AQBPSARfg8hCJRAAEegIxjlm7VkHuiuntm6wsA==
+        caps mds = "allow rw fsname=cephfs"
+        caps mon = "allow r"
+        caps osd = "allow rw tag cephfs data=cephfs"
+
+Mounting ``cephfs1`` with ``someuser`` works::
+
+    $ sudo ceph-fuse /mnt/cephfs1 -n client.someuser -k ceph.client.someuser.keyring --client-fs=cephfs
+    ceph-fuse[96634]: starting ceph client
+    ceph-fuse[96634]: starting fuse
+    $ mount | grep ceph-fuse
+    ceph-fuse on /mnt/cephfs1 type fuse.ceph-fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
+
+But mounting ``cephfs2`` does not::
+
+    $ sudo ceph-fuse /mnt/cephfs2 -n client.someuser -k ceph.client.someuser.keyring --client-fs=cephfs2
+    ceph-fuse[96599]: starting ceph client
+    ceph-fuse[96599]: ceph mount failed with (1) Operation not permitted
+
+Root squash
+===========
+
+The ``root squash`` feature is implemented as a safety measure to prevent
+scenarios such as accidental ``sudo rm -rf /path``. You can enable
+``root_squash`` mode in MDS caps to disallow clients with uid=0 or gid=0 to
+perform write access operations -- e.g., rm, rmdir, rmsnap, mkdir, mksnap.
+However, the mode allows the read operations of a root client unlike in
+other file systems.
+
+Following is an example of enabling root_squash in a filesystem except within
+'/volumes' directory tree in the filesystem::
+
+    $ ceph fs authorize a client.test_a / rw root_squash /volumes rw
+    $ ceph auth get client.test_a
+    [client.test_a]
+	key = AQBZcDpfEbEUKxAADk14VflBXt71rL9D966mYA==
+	caps mds = "allow rw fsname=a root_squash, allow rw fsname=a path=/volumes"
+	caps mon = "allow r fsname=a"
+	caps osd = "allow rw tag cephfs data=a"
diff --git a/doc/cephfs/client-config-ref.rst b/doc/cephfs/client-config-ref.rst
new file mode 100644
index 000000000..5167906b3
--- /dev/null
+++ b/doc/cephfs/client-config-ref.rst
@@ -0,0 +1,75 @@
+Client Configuration
+====================
+
+Updating Client Configuration
+-----------------------------
+
+Certain client configurations can be applied at runtime. To check if a configuration option can be applied (taken into affect by a client) at runtime, use the `config help` command::
+
+   ceph config help debug_client
+    debug_client - Debug level for client
+    (str, advanced)                                                                                                                      Default: 0/5
+    Can update at runtime: true
+
+    The value takes the form 'N' or 'N/M' where N and M are values between 0 and 99.  N is the debug level to log (all values below this are included), and M is the level to gather and buffer in memory.  In the event of a crash, the most recent items <= M are dumped to the log file.
+
+`config help` tells if a given configuration can be applied at runtime along with the defaults and a description of the configuration option.
+
+To update a configuration option at runtime, use the `config set` command::
+
+   ceph config set client debug_client 20/20
+
+Note that this changes a given configuration for all clients.
+
+To check configured options use the `config get` command::
+
+   ceph config get client
+    WHO    MASK LEVEL    OPTION                    VALUE     RO 
+    client      advanced debug_client              20/20          
+    global      advanced osd_pool_default_min_size 1            
+    global      advanced osd_pool_default_size     3            
+
+Client Config Reference
+------------------------
+
+.. confval:: client_acl_type
+.. confval:: client_cache_mid
+.. confval:: client_cache_size
+.. confval:: client_caps_release_delay
+.. confval:: client_debug_force_sync_read
+.. confval:: client_dirsize_rbytes
+.. confval:: client_max_inline_size
+.. confval:: client_metadata
+.. confval:: client_mount_gid
+.. confval:: client_mount_timeout
+.. confval:: client_mount_uid
+.. confval:: client_mountpoint
+.. confval:: client_oc
+.. confval:: client_oc_max_dirty
+.. confval:: client_oc_max_dirty_age
+.. confval:: client_oc_max_objects
+.. confval:: client_oc_size
+.. confval:: client_oc_target_dirty
+.. confval:: client_permissions
+.. confval:: client_quota_df
+.. confval:: client_readahead_max_bytes
+.. confval:: client_readahead_max_periods
+.. confval:: client_readahead_min
+.. confval:: client_reconnect_stale
+.. confval:: client_snapdir
+.. confval:: client_tick_interval
+.. confval:: client_use_random_mds
+.. confval:: fuse_default_permissions
+.. confval:: fuse_max_write
+.. confval:: fuse_disable_pagecache
+
+Developer Options
+#################
+
+.. important:: These options are internal. They are listed here only to complete the list of options.
+
+.. confval:: client_debug_getattr_caps
+.. confval:: client_debug_inject_tick_delay
+.. confval:: client_inject_fixed_oldest_tid
+.. confval:: client_inject_release_failure
+.. confval:: client_trace
diff --git a/doc/cephfs/createfs.rst b/doc/cephfs/createfs.rst
new file mode 100644
index 000000000..4a282e562
--- /dev/null
+++ b/doc/cephfs/createfs.rst
@@ -0,0 +1,135 @@
+=========================
+Create a Ceph file system
+=========================
+
+Creating pools
+==============
+
+A Ceph file system requires at least two RADOS pools, one for data and one for metadata.
+There are important considerations when planning these pools:
+
+- We recommend configuring *at least* 3 replicas for the metadata pool,
+  as data loss in this pool can render the entire file system inaccessible.
+  Configuring 4 would not be extreme, especially since the metadata pool's
+  capacity requirements are quite modest.
+- We recommend the fastest feasible low-latency storage devices (NVMe, Optane,
+  or at the very least SAS/SATA SSD) for the metadata pool, as this will
+  directly affect the latency of client file system operations.
+- We strongly suggest that the CephFS metadata pool be provisioned on dedicated
+  SSD / NVMe OSDs. This ensures that high client workload does not adversely
+  impact metadata operations. See :ref:`device_classes` to configure pools this
+  way.
+- The data pool used to create the file system is the "default" data pool and
+  the location for storing all inode backtrace information, which is used for hard link
+  management and disaster recovery. For this reason, all CephFS inodes
+  have at least one object in the default data pool. If erasure-coded
+  pools are planned for file system data, it is best to configure the default as
+  a replicated pool to improve small-object write and
+  read performance when updating backtraces. Separately, another erasure-coded
+  data pool can be added (see also :ref:`ecpool`) that can be used on an entire
+  hierarchy of directories and files (see also :ref:`file-layouts`).
+
+Refer to :doc:`/rados/operations/pools` to learn more about managing pools.  For
+example, to create two pools with default settings for use with a file system, you
+might run the following commands:
+
+.. code:: bash
+
+    $ ceph osd pool create cephfs_data
+    $ ceph osd pool create cephfs_metadata
+
+The metadata pool will typically hold at most a few gigabytes of data. For
+this reason, a smaller PG count is usually recommended. 64 or 128 is commonly
+used in practice for large clusters.
+
+.. note:: The names of the file systems, metadata pools, and data pools can
+          only have characters in the set [a-zA-Z0-9\_-.].
+
+Creating a file system
+======================
+
+Once the pools are created, you may enable the file system using the ``fs new`` command:
+
+.. code:: bash
+
+    $ ceph fs new <fs_name> <metadata> <data> [--force] [--allow-dangerous-metadata-overlay] [<fscid:int>] [--recover]
+
+This command creates a new file system with specified metadata and data pool.
+The specified data pool is the default data pool and cannot be changed once set.
+Each file system has its own set of MDS daemons assigned to ranks so ensure that
+you have sufficient standby daemons available to accommodate the new file system.
+
+The ``--force`` option is used to achieve any of the following:
+
+- To set an erasure-coded pool for the default data pool. Use of an EC pool for the
+  default data pool is discouraged. Refer to `Creating pools`_ for details.
+- To set non-empty pool (pool already contains some objects) for the metadata pool.
+- To create a file system with a specific file system's ID (fscid).
+  The --force option is required with --fscid option.
+
+The ``--allow-dangerous-metadata-overlay`` option permits the reuse metadata and
+data pools if it is already in-use. This should only be done in emergencies and
+after careful reading of the documentation.
+
+If the ``--fscid`` option is provided then this creates a file system with a
+specific fscid. This can be used when an application expects the file system's ID
+to be stable after it has been recovered, e.g., after monitor databases are
+lost and rebuilt. Consequently, file system IDs don't always keep increasing
+with newer file systems.
+
+The ``--recover`` option sets the state of file system's rank 0 to existing but
+failed. So when a MDS daemon eventually picks up rank 0, the daemon reads the
+existing in-RADOS metadata and doesn't overwrite it. The flag also prevents the
+standby MDS daemons to join the file system.
+
+For example:
+
+.. code:: bash
+
+    $ ceph fs new cephfs cephfs_metadata cephfs_data
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+
+Once a file system has been created, your MDS(s) will be able to enter
+an *active* state.  For example, in a single MDS system:
+
+.. code:: bash
+
+    $ ceph mds stat
+    cephfs-1/1/1 up {0=a=up:active}
+
+Once the file system is created and the MDS is active, you are ready to mount
+the file system.  If you have created more than one file system, you will
+choose which to use when mounting.
+
+  - `Mount CephFS`_
+  - `Mount CephFS as FUSE`_
+  - `Mount CephFS on Windows`_
+
+.. _Mount CephFS: ../../cephfs/mount-using-kernel-driver
+.. _Mount CephFS as FUSE: ../../cephfs/mount-using-fuse
+.. _Mount CephFS on Windows: ../../cephfs/ceph-dokan
+
+If you have created more than one file system, and a client does not
+specify a file system when mounting, you can control which file system
+they will see by using the ``ceph fs set-default`` command.
+
+Adding a Data Pool to the File System 
+-------------------------------------
+
+See :ref:`adding-data-pool-to-file-system`.
+
+
+Using Erasure Coded pools with CephFS
+=====================================
+
+You may use Erasure Coded pools as CephFS data pools as long as they have overwrites enabled, which is done as follows:
+
+.. code:: bash
+
+    ceph osd pool set my_ec_pool allow_ec_overwrites true
+    
+Note that EC overwrites are only supported when using OSDs with the BlueStore backend.
+
+You may not use Erasure Coded pools as CephFS metadata pools, because CephFS metadata is stored using RADOS *OMAP* data structures, which EC pools cannot store.
+
diff --git a/doc/cephfs/dirfrags.rst b/doc/cephfs/dirfrags.rst
new file mode 100644
index 000000000..9f177c19f
--- /dev/null
+++ b/doc/cephfs/dirfrags.rst
@@ -0,0 +1,101 @@
+
+===================================
+Configuring Directory fragmentation
+===================================
+
+In CephFS, directories are *fragmented* when they become very large
+or very busy.  This splits up the metadata so that it can be shared
+between multiple MDS daemons, and between multiple objects in the
+metadata pool.
+
+In normal operation, directory fragmentation is invisible to
+users and administrators, and all the configuration settings mentioned
+here should be left at their default values.
+
+While directory fragmentation enables CephFS to handle very large
+numbers of entries in a single directory, application programmers should
+remain conservative about creating very large directories, as they still
+have a resource cost in situations such as a CephFS client listing
+the directory, where all the fragments must be loaded at once.
+
+.. tip:: The root directory cannot be fragmented.
+
+All directories are initially created as a single fragment.  This fragment
+may be *split* to divide up the directory into more fragments, and these
+fragments may be *merged* to reduce the number of fragments in the directory.
+
+Splitting and merging
+=====================
+
+When an MDS identifies a directory fragment to be split, it does not
+do the split immediately.  Because splitting interrupts metadata IO,
+a short delay is used to allow short bursts of client IO to complete
+before the split begins.  This delay is configured with
+``mds_bal_fragment_interval``, which defaults to 5 seconds.
+
+When the split is done, the directory fragment is broken up into
+a power of two number of new fragments.  The number of new
+fragments is given by two to the power ``mds_bal_split_bits``, i.e.
+if ``mds_bal_split_bits`` is 2, then four new fragments will be
+created.  The default setting is 3, i.e. splits create 8 new fragments.
+
+The criteria for initiating a split or a merge are described in the
+following sections.
+
+Size thresholds
+===============
+
+A directory fragment is eligible for splitting when its size exceeds
+``mds_bal_split_size`` (default 10000 directory entries).  Ordinarily this
+split is delayed by ``mds_bal_fragment_interval``, but if the fragment size
+exceeds a factor of ``mds_bal_fragment_fast_factor`` the split size,
+the split will happen immediately (holding up any client metadata
+IO on the directory).
+
+``mds_bal_fragment_size_max`` is the hard limit on the size of
+directory fragments.  If it is reached, clients will receive
+ENOSPC errors if they try to create files in the fragment.  On
+a properly configured system, this limit should never be reached on
+ordinary directories, as they will have split long before.  By default,
+this is set to 10 times the split size, giving a dirfrag size limit of
+100000 directory entries.  Increasing this limit may lead to oversized
+directory fragment objects in the metadata pool, which the OSDs may not
+be able to handle.
+
+A directory fragment is eligible for merging when its size is less
+than ``mds_bal_merge_size``.  There is no merge equivalent of the
+"fast splitting" explained above: fast splitting exists to avoid
+creating oversized directory fragments, there is no equivalent issue
+to avoid when merging.  The default merge size is 50 directory entries.
+
+Activity thresholds
+===================
+
+In addition to splitting fragments based
+on their size, the MDS may split directory fragments if their
+activity exceeds a threshold.
+
+The MDS maintains separate time-decaying load counters for read and write
+operations on directory fragments.  The decaying load counters have an
+exponential decay based on the ``mds_decay_halflife`` setting.
+
+On writes, the write counter is
+incremented, and compared with ``mds_bal_split_wr``, triggering a
+split if the threshold is exceeded.  Write operations include metadata IO
+such as renames, unlinks and creations.
+
+The ``mds_bal_split_rd`` threshold is applied based on the read operation
+load counter, which tracks readdir operations.
+
+The ``mds_bal_split_rd`` and ``mds_bal_split_wr`` configs represent the
+popularity threshold. In the MDS these are measured as "read/write temperatures"
+which is closely related to the number of respective read/write operations.
+By default, the read threshold is 25000 operations and the write
+threshold is 10000 operations, i.e. 2.5x as many reads as writes would be
+required to trigger a split.
+
+After fragments are split due to the activity thresholds, they are only
+merged based on the size threshold (``mds_bal_merge_size``), so 
+a spike in activity may cause a directory to stay fragmented
+forever unless some entries are unlinked.
+
diff --git a/doc/cephfs/disaster-recovery-experts.rst b/doc/cephfs/disaster-recovery-experts.rst
new file mode 100644
index 000000000..c881c2423
--- /dev/null
+++ b/doc/cephfs/disaster-recovery-experts.rst
@@ -0,0 +1,322 @@
+
+.. _disaster-recovery-experts:
+
+Advanced: Metadata repair tools
+===============================
+
+.. warning::
+
+    If you do not have expert knowledge of CephFS internals, you will
+    need to seek assistance before using any of these tools.
+
+    The tools mentioned here can easily cause damage as well as fixing it.
+
+    It is essential to understand exactly what has gone wrong with your
+    file system before attempting to repair it.
+
+    If you do not have access to professional support for your cluster,
+    consult the ceph-users mailing list or the #ceph IRC channel.
+
+
+Journal export
+--------------
+
+Before attempting dangerous operations, make a copy of the journal like so:
+
+::
+
+    cephfs-journal-tool journal export backup.bin
+
+Note that this command may not always work if the journal is badly corrupted,
+in which case a RADOS-level copy should be made (http://tracker.ceph.com/issues/9902).
+
+
+Dentry recovery from journal
+----------------------------
+
+If a journal is damaged or for any reason an MDS is incapable of replaying it,
+attempt to recover what file metadata we can like so:
+
+::
+
+    cephfs-journal-tool event recover_dentries summary
+
+This command by default acts on MDS rank 0, pass --rank=<n> to operate on other ranks.
+
+This command will write any inodes/dentries recoverable from the journal
+into the backing store, if these inodes/dentries are higher-versioned
+than the previous contents of the backing store.  If any regions of the journal
+are missing/damaged, they will be skipped.
+
+Note that in addition to writing out dentries and inodes, this command will update
+the InoTables of each 'in' MDS rank, to indicate that any written inodes' numbers
+are now in use.  In simple cases, this will result in an entirely valid backing
+store state.
+
+.. warning::
+
+    The resulting state of the backing store is not guaranteed to be self-consistent,
+    and an online MDS scrub will be required afterwards.  The journal contents
+    will not be modified by this command, you should truncate the journal
+    separately after recovering what you can.
+
+Journal truncation
+------------------
+
+If the journal is corrupt or MDSs cannot replay it for any reason, you can
+truncate it like so:
+
+::
+
+    cephfs-journal-tool [--rank=N] journal reset
+
+Specify the MDS rank using the ``--rank`` option when the file system has/had
+multiple active MDS.
+
+.. warning::
+
+    Resetting the journal *will* lose metadata unless you have extracted
+    it by other means such as ``recover_dentries``.  It is likely to leave
+    some orphaned objects in the data pool.  It may result in re-allocation
+    of already-written inodes, such that permissions rules could be violated.
+
+MDS table wipes
+---------------
+
+After the journal has been reset, it may no longer be consistent with respect
+to the contents of the MDS tables (InoTable, SessionMap, SnapServer).
+
+To reset the SessionMap (erase all sessions), use:
+
+::
+
+    cephfs-table-tool all reset session
+
+This command acts on the tables of all 'in' MDS ranks.  Replace 'all' with an MDS
+rank to operate on that rank only.
+
+The session table is the table most likely to need resetting, but if you know you
+also need to reset the other tables then replace 'session' with 'snap' or 'inode'.
+
+MDS map reset
+-------------
+
+Once the in-RADOS state of the file system (i.e. contents of the metadata pool)
+is somewhat recovered, it may be necessary to update the MDS map to reflect
+the contents of the metadata pool.  Use the following command to reset the MDS
+map to a single MDS:
+
+::
+
+    ceph fs reset <fs name> --yes-i-really-mean-it
+
+Once this is run, any in-RADOS state for MDS ranks other than 0 will be ignored:
+as a result it is possible for this to result in data loss.
+
+One might wonder what the difference is between 'fs reset' and 'fs remove; fs new'.  The
+key distinction is that doing a remove/new will leave rank 0 in 'creating' state, such
+that it would overwrite any existing root inode on disk and orphan any existing files.  In
+contrast, the 'reset' command will leave rank 0 in 'active' state such that the next MDS
+daemon to claim the rank will go ahead and use the existing in-RADOS metadata.
+
+Recovery from missing metadata objects
+--------------------------------------
+
+Depending on what objects are missing or corrupt, you may need to
+run various commands to regenerate default versions of the
+objects.
+
+::
+
+    # Session table
+    cephfs-table-tool 0 reset session
+    # SnapServer
+    cephfs-table-tool 0 reset snap
+    # InoTable
+    cephfs-table-tool 0 reset inode
+    # Journal
+    cephfs-journal-tool --rank=0 journal reset
+    # Root inodes ("/" and MDS directory)
+    cephfs-data-scan init
+
+Finally, you can regenerate metadata objects for missing files
+and directories based on the contents of a data pool.  This is
+a three-phase process.  First, scanning *all* objects to calculate
+size and mtime metadata for inodes.  Second, scanning the first
+object from every file to collect this metadata and inject it into
+the metadata pool. Third, checking inode linkages and fixing found
+errors.
+
+::
+
+    cephfs-data-scan scan_extents [<data pool> [<extra data pool> ...]]
+    cephfs-data-scan scan_inodes [<data pool>]
+    cephfs-data-scan scan_links
+
+'scan_extents' and 'scan_inodes' commands may take a *very long* time
+if there are many files or very large files in the data pool.
+
+To accelerate the process, run multiple instances of the tool.
+
+Decide on a number of workers, and pass each worker a number within
+the range 0-(worker_m - 1).
+
+The example below shows how to run 4 workers simultaneously:
+
+::
+
+    # Worker 0
+    cephfs-data-scan scan_extents --worker_n 0 --worker_m 4
+    # Worker 1
+    cephfs-data-scan scan_extents --worker_n 1 --worker_m 4
+    # Worker 2
+    cephfs-data-scan scan_extents --worker_n 2 --worker_m 4
+    # Worker 3
+    cephfs-data-scan scan_extents --worker_n 3 --worker_m 4
+
+    # Worker 0
+    cephfs-data-scan scan_inodes --worker_n 0 --worker_m 4
+    # Worker 1
+    cephfs-data-scan scan_inodes --worker_n 1 --worker_m 4
+    # Worker 2
+    cephfs-data-scan scan_inodes --worker_n 2 --worker_m 4
+    # Worker 3
+    cephfs-data-scan scan_inodes --worker_n 3 --worker_m 4
+
+It is **important** to ensure that all workers have completed the
+scan_extents phase before any workers enter the scan_inodes phase.
+
+After completing the metadata recovery, you may want to run cleanup
+operation to delete ancillary data generated during recovery.
+
+::
+
+    cephfs-data-scan cleanup [<data pool>]
+
+Note, the data pool parameters for 'scan_extents', 'scan_inodes' and
+'cleanup' commands are optional, and usually the tool will be able to
+detect the pools automatically. Still you may override this. The
+'scan_extents' command needs all data pools to be specified, while
+'scan_inodes' and 'cleanup' commands need only the main data pool.
+
+
+Using an alternate metadata pool for recovery
+---------------------------------------------
+
+.. warning::
+
+   There has not been extensive testing of this procedure. It should be
+   undertaken with great care.
+
+If an existing file system is damaged and inoperative, it is possible to create
+a fresh metadata pool and attempt to reconstruct the file system metadata into
+this new pool, leaving the old metadata in place. This could be used to make a
+safer attempt at recovery since the existing metadata pool would not be
+modified.
+
+.. caution::
+
+   During this process, multiple metadata pools will contain data referring to
+   the same data pool. Extreme caution must be exercised to avoid changing the
+   data pool contents while this is the case. Once recovery is complete, the
+   damaged metadata pool should be archived or deleted.
+
+To begin, the existing file system should be taken down, if not done already,
+to prevent further modification of the data pool. Unmount all clients and then
+mark the file system failed:
+
+::
+
+    ceph fs fail <fs_name>
+
+.. note::
+
+   <fs_name> here and below indicates the original, damaged file system.
+
+Next, create a recovery file system in which we will populate a new metadata pool
+backed by the original data pool.
+
+::
+
+    ceph osd pool create cephfs_recovery_meta
+    ceph fs new cephfs_recovery cephfs_recovery_meta <data_pool> --recover --allow-dangerous-metadata-overlay
+
+.. note::
+
+   You may rename the recovery metadata pool and file system at a future time.
+   The ``--recover`` flag prevents any MDS from joining the new file system.
+
+Next, we will create the intial metadata for the fs:
+
+::
+
+    cephfs-table-tool cephfs_recovery:0 reset session
+    cephfs-table-tool cephfs_recovery:0 reset snap
+    cephfs-table-tool cephfs_recovery:0 reset inode
+    cephfs-journal-tool --rank cephfs_recovery:0 journal reset --force
+
+Now perform the recovery of the metadata pool from the data pool:
+
+::
+
+    cephfs-data-scan init --force-init --filesystem cephfs_recovery --alternate-pool cephfs_recovery_meta
+    cephfs-data-scan scan_extents --alternate-pool cephfs_recovery_meta --filesystem <fs_name>
+    cephfs-data-scan scan_inodes --alternate-pool cephfs_recovery_meta --filesystem <fs_name> --force-corrupt
+    cephfs-data-scan scan_links --filesystem cephfs_recovery
+
+.. note::
+
+   Each scan procedure above goes through the entire data pool. This may take a
+   significant amount of time. See the previous section on how to distribute
+   this task among workers.
+
+If the damaged file system contains dirty journal data, it may be recovered next
+with:
+
+::
+
+    cephfs-journal-tool --rank=<fs_name>:0 event recover_dentries list --alternate-pool cephfs_recovery_meta
+
+After recovery, some recovered directories will have incorrect statistics.
+Ensure the parameters ``mds_verify_scatter`` and ``mds_debug_scatterstat`` are
+set to false (the default) to prevent the MDS from checking the statistics:
+
+::
+
+    ceph config rm mds mds_verify_scatter
+    ceph config rm mds mds_debug_scatterstat
+
+.. note::
+
+    Also verify the config has not been set globally or with a local ceph.conf file.
+
+Now, allow an MDS to join the recovery file system:
+
+::
+
+    ceph fs set cephfs_recovery joinable true
+
+Finally, run a forward :doc:`scrub </cephfs/scrub>` to repair recursive statistics.
+Ensure you have an MDS running and issue:
+
+::
+
+    ceph tell mds.cephfs_recovery:0 scrub start / recursive,repair,force
+
+.. note::
+
+   The `Symbolic link recovery <https://tracker.ceph.com/issues/46166>`_ is supported from Quincy.
+   Symbolic links were recovered as empty regular files before.
+
+It is recommended to migrate any data from the recovery file system as soon as
+possible. Do not restore the old file system while the recovery file system is
+operational.
+
+.. note::
+
+    If the data pool is also corrupt, some files may not be restored because
+    backtrace information is lost. If any data objects are missing (due to
+    issues like lost Placement Groups on the data pool), the recovered files
+    will contain holes in place of the missing data.
+
+.. _Symbolic link recovery: https://tracker.ceph.com/issues/46166
diff --git a/doc/cephfs/disaster-recovery.rst b/doc/cephfs/disaster-recovery.rst
new file mode 100644
index 000000000..a728feb55
--- /dev/null
+++ b/doc/cephfs/disaster-recovery.rst
@@ -0,0 +1,61 @@
+.. _cephfs-disaster-recovery:
+
+Disaster recovery
+=================
+
+Metadata damage and repair
+--------------------------
+
+If a file system has inconsistent or missing metadata, it is considered
+*damaged*.  You may find out about damage from a health message, or in some
+unfortunate cases from an assertion in a running MDS daemon.
+
+Metadata damage can result either from data loss in the underlying RADOS
+layer (e.g. multiple disk failures that lose all copies of a PG), or from
+software bugs.
+
+CephFS includes some tools that may be able to recover a damaged file system,
+but to use them safely requires a solid understanding of CephFS internals.
+The documentation for these potentially dangerous operations is on a
+separate page: :ref:`disaster-recovery-experts`.
+
+Data pool damage (files affected by lost data PGs)
+--------------------------------------------------
+
+If a PG is lost in a *data* pool, then the file system will continue
+to operate normally, but some parts of some files will simply
+be missing (reads will return zeros).
+
+Losing a data PG may affect many files.  Files are split into many objects,
+so identifying which files are affected by loss of particular PGs requires
+a full scan over all object IDs that may exist within the size of a file. 
+This type of scan may be useful for identifying which files require
+restoring from a backup.
+
+.. danger::
+
+    This command does not repair any metadata, so when restoring files in
+    this case you must *remove* the damaged file, and replace it in order
+    to have a fresh inode.  Do not overwrite damaged files in place.
+
+If you know that objects have been lost from PGs, use the ``pg_files``
+subcommand to scan for files that may have been damaged as a result:
+
+::
+
+    cephfs-data-scan pg_files <path> <pg id> [<pg id>...]
+
+For example, if you have lost data from PGs 1.4 and 4.5, and you would like
+to know which files under /home/bob might have been damaged:
+
+::
+
+    cephfs-data-scan pg_files /home/bob 1.4 4.5
+
+The output will be a list of paths to potentially damaged files, one
+per line.
+
+Note that this command acts as a normal CephFS client to find all the
+files in the file system and read their layouts, so the MDS must be
+up and running.
+
diff --git a/doc/cephfs/dynamic-metadata-management.rst b/doc/cephfs/dynamic-metadata-management.rst
new file mode 100644
index 000000000..6e7ada9fc
--- /dev/null
+++ b/doc/cephfs/dynamic-metadata-management.rst
@@ -0,0 +1,90 @@
+==================================
+CephFS Dynamic Metadata Management
+==================================
+Metadata operations usually take up more than 50 percent of all
+file system operations. Also the metadata scales in a more complex
+fashion when compared to scaling storage (which in turn scales I/O
+throughput linearly). This is due to the hierarchical and
+interdependent nature of the file system metadata. So in CephFS,
+the metadata workload is decoupled from data workload so as to
+avoid placing unnecessary strain on the RADOS cluster. The metadata
+is hence handled by a cluster of Metadata Servers (MDSs). 
+CephFS distributes metadata across MDSs via `Dynamic Subtree Partitioning <https://ceph.com/assets/pdfs/weil-mds-sc04.pdf>`__.
+
+Dynamic Subtree Partitioning
+----------------------------
+In traditional subtree partitioning, subtrees of the file system
+hierarchy are assigned to individual MDSs. This metadata distribution
+strategy provides good hierarchical locality, linear growth of
+cache and horizontal scaling across MDSs and a fairly good distribution
+of metadata across MDSs.
+
+.. image:: subtree-partitioning.svg
+
+The problem with traditional subtree partitioning is that the workload
+growth by depth (across a single MDS) leads to a hotspot of activity.
+This results in lack of vertical scaling and wastage of non-busy resources/MDSs. 
+
+This led to the adoption of a more dynamic way of handling
+metadata: Dynamic Subtree Partitioning, where load intensive portions
+of the directory hierarchy from busy MDSs are migrated to non busy MDSs. 
+
+This strategy ensures that activity hotspots are relieved as they
+appear and so leads to vertical scaling of the metadata workload in
+addition to horizontal scaling.
+
+Export Process During Subtree Migration
+---------------------------------------
+
+Once the exporter verifies that the subtree is permissible to be exported
+(Non degraded cluster, non-frozen subtree root), the subtree root
+directory is temporarily auth pinned, the subtree freeze is initiated,
+and the exporter is committed to the subtree migration, barring an
+intervening failure of the importer or itself.
+
+The MExportDiscover message is exchanged to ensure that the inode for the
+base directory being exported is open on the destination node. It is
+auth pinned by the importer to prevent it from being trimmed. This occurs
+before the exporter completes the freeze of the subtree to ensure that
+the importer is able to replicate the necessary metadata. When the
+exporter receives the MDiscoverAck, it allows the freeze to proceed by
+removing its temporary auth pin.
+
+A warning stage occurs only if the base subtree directory is open by
+nodes other than the importer and exporter. If it is not, then this
+implies that no metadata within or nested beneath the subtree is
+replicated by any node other than the importer and exporter. If it is,
+then an MExportWarning message informs any bystanders that the
+authority for the region is temporarily ambiguous, and lists both the
+exporter and importer as authoritative MDS nodes. In particular,
+bystanders who are trimming items from their cache must send
+MCacheExpire messages to both the old and new authorities. This is
+necessary to ensure that the surviving authority reliably receives all
+expirations even if the importer or exporter fails. While the subtree
+is frozen (on both the importer and exporter), expirations will not be
+immediately processed; instead, they will be queued until the region
+is unfrozen and it can be determined that the node is or is not
+authoritative.
+
+The exporter then packages an MExport message containing all metadata
+of the subtree and flags the objects as non-authoritative. The MExport message sends
+the actual subtree metadata to the importer. Upon receipt, the
+importer inserts the data into its cache, marks all objects as
+authoritative, and logs a copy of all metadata in an EImportStart
+journal message. Once that has safely flushed, it replies with an
+MExportAck. The exporter can now log an EExport journal entry, which
+ultimately specifies that the export was a success. In the presence
+of failures, it is the existence of the EExport entry only that
+disambiguates authority during recovery.
+
+Once logged, the exporter will send an MExportNotify to any
+bystanders, informing them that the authority is no longer ambiguous
+and cache expirations should be sent only to the new authority (the
+importer). Once these are acknowledged back to the exporter,
+implicitly flushing the bystander to exporter message streams of any
+stray expiration notices, the exporter unfreezes the subtree, cleans
+up its migration-related state, and sends a final MExportFinish to the
+importer. Upon receipt, the importer logs an EImportFinish(true)
+(noting locally that the export was indeed a success), unfreezes its
+subtree, processes any queued cache expirations, and cleans up its
+state.
diff --git a/doc/cephfs/eviction.rst b/doc/cephfs/eviction.rst
new file mode 100644
index 000000000..eb6f70a8e
--- /dev/null
+++ b/doc/cephfs/eviction.rst
@@ -0,0 +1,190 @@
+
+================================
+Ceph file system client eviction
+================================
+
+When a file system client is unresponsive or otherwise misbehaving, it
+may be necessary to forcibly terminate its access to the file system.  This
+process is called *eviction*.
+
+Evicting a CephFS client prevents it from communicating further with MDS
+daemons and OSD daemons.  If a client was doing buffered IO to the file system,
+any un-flushed data will be lost.
+
+Clients may either be evicted automatically (if they fail to communicate
+promptly with the MDS), or manually (by the system administrator).
+
+The client eviction process applies to clients of all kinds, this includes
+FUSE mounts, kernel mounts, nfs-ganesha gateways, and any process using
+libcephfs.
+
+Automatic client eviction
+=========================
+
+There are three situations in which a client may be evicted automatically.
+
+#. On an active MDS daemon, if a client has not communicated with the MDS for over
+   ``session_autoclose`` (a file system variable) seconds (300 seconds by
+   default), then it will be evicted automatically.
+
+#. On an active MDS daemon, if a client has not responded to cap revoke messages
+   for over ``mds_cap_revoke_eviction_timeout`` (configuration option) seconds.
+   This is disabled by default.
+
+#. During MDS startup (including on failover), the MDS passes through a
+   state called ``reconnect``.  During this state, it waits for all the
+   clients to connect to the new MDS daemon.  If any clients fail to do
+   so within the time window (``mds_reconnect_timeout``, 45 seconds by default)
+   then they will be evicted.
+
+A warning message is sent to the cluster log if either of these situations
+arises.
+
+Manual client eviction
+======================
+
+Sometimes, the administrator may want to evict a client manually.  This
+could happen if a client has died and the administrator does not
+want to wait for its session to time out, or it could happen if
+a client is misbehaving and the administrator does not have access to
+the client node to unmount it.
+
+It is useful to inspect the list of clients first:
+
+::
+
+    ceph tell mds.0 client ls
+
+    [
+        {
+            "id": 4305,
+            "num_leases": 0,
+            "num_caps": 3,
+            "state": "open",
+            "replay_requests": 0,
+            "completed_requests": 0,
+            "reconnecting": false,
+            "inst": "client.4305 172.21.9.34:0/422650892",
+            "client_metadata": {
+                "ceph_sha1": "ae81e49d369875ac8b569ff3e3c456a31b8f3af5",
+                "ceph_version": "ceph version 12.0.0-1934-gae81e49 (ae81e49d369875ac8b569ff3e3c456a31b8f3af5)",
+                "entity_id": "0",
+                "hostname": "senta04",
+                "mount_point": "/tmp/tmpcMpF1b/mnt.0",
+                "pid": "29377",
+                "root": "/"
+            }
+        }
+    ]
+    
+
+
+Once you have identified the client you want to evict, you can
+do that using its unique ID, or various other attributes to identify it:
+
+::
+    
+    # These all work
+    ceph tell mds.0 client evict id=4305
+    ceph tell mds.0 client evict client_metadata.=4305
+
+
+Advanced: Un-blocklisting a client
+==================================
+
+Ordinarily, a blocklisted client may not reconnect to the servers: it
+must be unmounted and then mounted anew.
+
+However, in some situations it may be useful to permit a client that
+was evicted to attempt to reconnect.
+
+Because CephFS uses the RADOS OSD blocklist to control client eviction,
+CephFS clients can be permitted to reconnect by removing them from
+the blocklist:
+
+::
+
+    $ ceph osd blocklist ls
+    listed 1 entries
+    127.0.0.1:0/3710147553 2018-03-19 11:32:24.716146
+    $ ceph osd blocklist rm 127.0.0.1:0/3710147553
+    un-blocklisting 127.0.0.1:0/3710147553
+
+
+Doing this may put data integrity at risk if other clients have accessed
+files that the blocklisted client was doing buffered IO to.  It is also not
+guaranteed to result in a fully functional client -- the best way to get
+a fully healthy client back after an eviction is to unmount the client
+and do a fresh mount.
+
+If you are trying to reconnect clients in this way, you may also
+find it useful to set ``client_reconnect_stale`` to true in the
+FUSE client, to prompt the client to try to reconnect.
+
+Advanced: Configuring blocklisting
+==================================
+
+If you are experiencing frequent client evictions, due to slow
+client hosts or an unreliable network, and you cannot fix the underlying
+issue, then you may want to ask the MDS to be less strict.
+
+It is possible to respond to slow clients by simply dropping their
+MDS sessions, but permit them to re-open sessions and permit them
+to continue talking to OSDs.  To enable this mode, set
+``mds_session_blocklist_on_timeout`` to false on your MDS nodes.
+
+For the equivalent behaviour on manual evictions, set
+``mds_session_blocklist_on_evict`` to false.
+
+Note that if blocklisting is disabled, then evicting a client will
+only have an effect on the MDS you send the command to.  On a system
+with multiple active MDS daemons, you would need to send an
+eviction command to each active daemon.  When blocklisting is enabled 
+(the default), sending an eviction command to just a single
+MDS is sufficient, because the blocklist propagates it to the others.
+
+.. _background_blocklisting_and_osd_epoch_barrier:
+
+Background: Blocklisting and OSD epoch barrier
+==============================================
+
+After a client is blocklisted, it is necessary to make sure that
+other clients and MDS daemons have the latest OSDMap (including
+the blocklist entry) before they try to access any data objects
+that the blocklisted client might have been accessing.
+
+This is ensured using an internal "osdmap epoch barrier" mechanism.
+
+The purpose of the barrier is to ensure that when we hand out any
+capabilities which might allow touching the same RADOS objects, the
+clients we hand out the capabilities to must have a sufficiently recent
+OSD map to not race with cancelled operations (from ENOSPC) or
+blocklisted clients (from evictions).
+
+More specifically, the cases where an epoch barrier is set are:
+
+ * Client eviction (where the client is blocklisted and other clients
+   must wait for a post-blocklist epoch to touch the same objects).
+ * OSD map full flag handling in the client (where the client may
+   cancel some OSD ops from a pre-full epoch, so other clients must
+   wait until the full epoch or later before touching the same objects).
+ * MDS startup, because we don't persist the barrier epoch, so must
+   assume that latest OSD map is always required after a restart.
+
+Note that this is a global value for simplicity. We could maintain this on
+a per-inode basis. But we don't, because:
+
+ * It would be more complicated.
+ * It would use an extra 4 bytes of memory for every inode.
+ * It would not be much more efficient as, almost always, everyone has
+   the latest OSD map. And, in most cases everyone will breeze through this
+   barrier rather than waiting.
+ * This barrier is done in very rare cases, so any benefit from per-inode
+   granularity would only very rarely be seen.
+
+The epoch barrier is transmitted along with all capability messages, and
+instructs the receiver of the message to avoid sending any more RADOS
+operations to OSDs until it has seen this OSD epoch.  This mainly applies
+to clients (doing their data writes directly to files), but also applies
+to the MDS because things like file size probing and file deletion are
+done directly from the MDS.
diff --git a/doc/cephfs/experimental-features.rst b/doc/cephfs/experimental-features.rst
new file mode 100644
index 000000000..ba60d12c7
--- /dev/null
+++ b/doc/cephfs/experimental-features.rst
@@ -0,0 +1,42 @@
+=====================
+Experimental Features
+=====================
+
+CephFS includes a number of experimental features which are not fully
+stabilized or qualified for users to turn on in real deployments. We generally
+do our best to clearly demarcate these and fence them off so they cannot be
+used by mistake.
+
+Some of these features are closer to being done than others, though. We
+describe each of them with an approximation of how risky they are and briefly
+describe what is required to enable them. Note that doing so will
+*irrevocably* flag maps in the monitor as having once enabled this flag to
+improve debugging and support processes.
+
+Inline data
+-----------
+By default, all CephFS file data is stored in RADOS objects. The inline data
+feature enables small files (generally <2KB) to be stored in the inode
+and served out of the MDS. This may improve small-file performance but increases
+load on the MDS. It is not sufficiently tested to support at this time, although
+failures within it are unlikely to make non-inlined data inaccessible
+
+Inline data has always been off by default and requires setting
+the ``inline_data`` flag.
+
+Inline data has been declared deprecated for the Octopus release, and will
+likely be removed altogether in the Q release.
+
+Mantle: Programmable Metadata Load Balancer
+-------------------------------------------
+
+Mantle is a programmable metadata balancer built into the MDS. The idea is to
+protect the mechanisms for balancing load (migration, replication,
+fragmentation) but stub out the balancing policies using Lua. For details, see
+:doc:`/cephfs/mantle`.
+
+LazyIO
+------
+LazyIO relaxes POSIX semantics. Buffered reads/writes are allowed even when a
+file is opened by multiple applications on multiple clients. Applications are
+responsible for managing cache coherency themselves.
diff --git a/doc/cephfs/file-layouts.rst b/doc/cephfs/file-layouts.rst
new file mode 100644
index 000000000..2cdb26efc
--- /dev/null
+++ b/doc/cephfs/file-layouts.rst
@@ -0,0 +1,272 @@
+.. _file-layouts:
+
+File layouts
+============
+
+The layout of a file controls how its contents are mapped to Ceph RADOS objects.  You can
+read and write a file's layout using *virtual extended attributes* or xattrs.
+
+The name of the layout xattrs depends on whether a file is a regular file or a directory.  Regular
+files' layout xattrs are called ``ceph.file.layout``, whereas directories' layout xattrs are called
+``ceph.dir.layout``.  Where subsequent examples refer to ``ceph.file.layout``, substitute ``dir`` as appropriate
+when dealing with directories.
+
+.. tip::
+
+    Your linux distribution may not ship with commands for manipulating xattrs by default,
+    the required package is usually called ``attr``.
+
+Layout fields
+-------------
+
+pool
+    String, giving ID or name. String can only have characters in the set [a-zA-Z0-9\_-.]. Which RADOS pool a file's data objects will be stored in.
+
+pool_id
+    String of digits. This is the system assigned pool id for the RADOS pool whenever it is created.
+
+pool_name
+    String, given name. This is the user defined name for the RADOS pool whenever user creates it.
+
+pool_namespace
+    String with only characters in the set [a-zA-Z0-9\_-.].  Within the data pool, which RADOS namespace the objects will
+    be written to.  Empty by default (i.e. default namespace).
+
+stripe_unit
+    Integer in bytes.  The size (in bytes) of a block of data used in the RAID 0 distribution of a file. All stripe units for a file have equal size. The last stripe unit is typically incomplete–i.e. it represents the data at the end of the file as well as unused “space� beyond it up to the end of the fixed stripe unit size.
+
+stripe_count
+    Integer.  The number of consecutive stripe units that constitute a RAID 0 “stripe� of file data.
+
+object_size
+    Integer in bytes.  File data is chunked into RADOS objects of this size.
+
+.. tip::
+
+    RADOS enforces a configurable limit on object sizes: if you increase CephFS
+    object sizes beyond that limit then writes may not succeed.  The OSD
+    setting is ``osd_max_object_size``, which is 128MB by default.
+    Very large RADOS objects may prevent smooth operation of the cluster,
+    so increasing the object size limit past the default is not recommended.
+
+Reading layouts with ``getfattr``
+---------------------------------
+
+Read the layout information as a single string:
+
+.. code-block:: bash
+
+    $ touch file
+    $ getfattr -n ceph.file.layout file
+    # file: file
+    ceph.file.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data"
+
+Read individual layout fields:
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.file.layout.pool_name file
+    # file: file
+    ceph.file.layout.pool_name="cephfs_data"
+    $ getfattr -n ceph.file.layout.pool_id file
+    # file: file
+    ceph.file.layout.pool_id="5"
+    $ getfattr -n ceph.file.layout.pool file
+    # file: file
+    ceph.file.layout.pool="cephfs_data"
+    $ getfattr -n ceph.file.layout.stripe_unit file
+    # file: file
+    ceph.file.layout.stripe_unit="4194304"
+    $ getfattr -n ceph.file.layout.stripe_count file
+    # file: file
+    ceph.file.layout.stripe_count="1"
+    $ getfattr -n ceph.file.layout.object_size file
+    # file: file
+    ceph.file.layout.object_size="4194304"    
+
+.. note::
+
+    When reading layouts, the pool will usually be indicated by name.  However, in 
+    rare cases when pools have only just been created, the ID may be output instead.
+
+Directories do not have an explicit layout until it is customized.  Attempts to read
+the layout will fail if it has never been modified: this indicates that layout of the
+next ancestor directory with an explicit layout will be used.
+
+.. code-block:: bash
+
+    $ mkdir dir
+    $ getfattr -n ceph.dir.layout dir
+    dir: ceph.dir.layout: No such attribute
+    $ setfattr -n ceph.dir.layout.stripe_count -v 2 dir
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+Getting the layout in json format. If there's no specific layout set for the
+particular inode, the system traverses the directory path backwards and finds
+the closest ancestor directory with a layout and returns it in json format.
+A file layout also can be retrieved in json format using ``ceph.file.layout.json`` vxattr.
+
+A virtual field named ``inheritance`` is added to the json output to show the status of layout.
+The ``inheritance`` field can have the following values:
+
+``@default`` implies the system default layout
+``@set`` implies that a specific layout has been set for that particular inode
+``@inherited`` implies that the returned layout has been inherited from an ancestor
+
+.. code-block:: bash
+
+   $ getfattr -n ceph.dir.layout.json --only-values /mnt/mycephs/accounts
+   {"stripe_unit": 4194304, "stripe_count": 1, "object_size": 4194304, "pool_name": "cephfs.a.data", "pool_id": 3, "pool_namespace": "", "inheritance": "@default"}
+
+
+Writing layouts with ``setfattr``
+---------------------------------
+
+Layout fields are modified using ``setfattr``:
+
+.. code-block:: bash
+
+    $ ceph osd lspools
+    0 rbd
+    1 cephfs_data
+    2 cephfs_metadata
+
+    $ setfattr -n ceph.file.layout.stripe_unit -v 1048576 file2
+    $ setfattr -n ceph.file.layout.stripe_count -v 8 file2
+    $ setfattr -n ceph.file.layout.object_size -v 10485760 file2
+    $ setfattr -n ceph.file.layout.pool -v 1 file2  # Setting pool by ID
+    $ setfattr -n ceph.file.layout.pool -v cephfs_data file2  # Setting pool by name
+    $ setfattr -n ceph.file.layout.pool_id -v 1 file2  # Setting pool by ID
+    $ setfattr -n ceph.file.layout.pool_name -v cephfs_data file2  # Setting pool by name
+
+.. note::
+
+    When the layout fields of a file are modified using ``setfattr``, this file must be empty, otherwise an error will occur.
+
+.. code-block:: bash
+
+    # touch an empty file
+    $ touch file1
+    # modify layout field successfully
+    $ setfattr -n ceph.file.layout.stripe_count -v 3 file1
+
+    # write something to file1
+    $ echo "hello world" > file1
+    $ setfattr -n ceph.file.layout.stripe_count -v 4 file1
+    setfattr: file1: Directory not empty
+
+File and Directory layouts can also be set using the json format.
+The ``inheritance`` field is ignored when setting the layout.
+Also, if both, ``pool_name`` and ``pool_id`` fields are specified, then the
+``pool_name`` is given preference for better disambiguation.
+
+.. code-block:: bash
+
+   $ setfattr -n ceph.file.layout.json -v '{"stripe_unit": 4194304, "stripe_count": 1, "object_size": 4194304, "pool_name": "cephfs.a.data", "pool_id": 3, "pool_namespace": "", "inheritance": "@default"}' file1
+    
+Clearing layouts
+----------------
+
+If you wish to remove an explicit layout from a directory, to revert to
+inheriting the layout of its ancestor, you can do so:
+
+.. code-block:: bash
+
+    setfattr -x ceph.dir.layout mydir
+
+Similarly, if you have set the ``pool_namespace`` attribute and wish
+to modify the layout to use the default namespace instead:
+
+.. code-block:: bash
+
+    # Create a dir and set a namespace on it
+    mkdir mydir
+    setfattr -n ceph.dir.layout.pool_namespace -v foons mydir
+    getfattr -n ceph.dir.layout mydir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_a pool_namespace=foons"
+
+    # Clear the namespace from the directory's layout
+    setfattr -x ceph.dir.layout.pool_namespace mydir
+    getfattr -n ceph.dir.layout mydir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_a"
+
+
+Inheritance of layouts
+----------------------
+
+Files inherit the layout of their parent directory at creation time.  However, subsequent
+changes to the parent directory's layout do not affect children.
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # Demonstrate file1 inheriting its parent's layout
+    $ touch dir/file1
+    $ getfattr -n ceph.file.layout dir/file1
+    # file: dir/file1
+    ceph.file.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # Now update the layout of the directory before creating a second file
+    $ setfattr -n ceph.dir.layout.stripe_count -v 4 dir
+    $ touch dir/file2
+
+    # Demonstrate that file1's layout is unchanged
+    $ getfattr -n ceph.file.layout dir/file1
+    # file: dir/file1
+    ceph.file.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # ...while file2 has the parent directory's new layout
+    $ getfattr -n ceph.file.layout dir/file2
+    # file: dir/file2
+    ceph.file.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+
+
+Files created as descendents of the directory also inherit the layout, if the intermediate
+directories do not have layouts set:
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+    $ mkdir dir/childdir
+    $ getfattr -n ceph.dir.layout dir/childdir
+    dir/childdir: ceph.dir.layout: No such attribute
+    $ touch dir/childdir/grandchild
+    $ getfattr -n ceph.file.layout dir/childdir/grandchild
+    # file: dir/childdir/grandchild
+    ceph.file.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+
+
+.. _adding-data-pool-to-file-system:
+    
+Adding a data pool to the File System 
+-------------------------------------
+
+Before you can use a pool with CephFS you have to add it to the Metadata Servers.
+
+.. code-block:: bash
+
+    $ ceph fs add_data_pool cephfs cephfs_data_ssd
+    $ ceph fs ls  # Pool should now show up
+    .... data pools: [cephfs_data cephfs_data_ssd ]
+
+Make sure that your cephx keys allows the client to access this new pool.
+
+You can then update the layout on a directory in CephFS to use the pool you added:
+
+.. code-block:: bash
+
+    $ mkdir /mnt/cephfs/myssddir
+    $ setfattr -n ceph.dir.layout.pool -v cephfs_data_ssd /mnt/cephfs/myssddir
+
+All new files created within that directory will now inherit its layout and place their data in your newly added pool. 
+
+You may notice that object counts in your primary data pool (the one passed to ``fs new``) continue to increase, even if files are being created in the pool you added.  This is normal: the file data is stored in the pool specified by the layout, but a small amount of metadata is kept in the primary data pool for all files.
+
+
diff --git a/doc/cephfs/fs-volumes.rst b/doc/cephfs/fs-volumes.rst
new file mode 100644
index 000000000..e7fd377bf
--- /dev/null
+++ b/doc/cephfs/fs-volumes.rst
@@ -0,0 +1,653 @@
+.. _fs-volumes-and-subvolumes:
+
+FS volumes and subvolumes
+=========================
+
+The volumes module of the :term:`Ceph Manager` daemon (ceph-mgr) provides a
+single source of truth for CephFS exports. The OpenStack shared file system
+service (manila_) and the Ceph Container Storage Interface (CSI_) storage
+administrators use the common CLI provided by the ceph-mgr ``volumes`` module
+to manage CephFS exports.
+
+The ceph-mgr ``volumes`` module implements the following file system export
+abstractions:
+
+* FS volumes, an abstraction for CephFS file systems
+
+* FS subvolumes, an abstraction for independent CephFS directory trees
+
+* FS subvolume groups, an abstraction for a directory level higher than FS
+  subvolumes. Used to effect policies (e.g., :doc:`/cephfs/file-layouts`)
+  across a set of subvolumes
+
+Some possible use-cases for the export abstractions:
+
+* FS subvolumes used as Manila shares or CSI volumes
+
+* FS subvolume groups used as Manila share groups
+
+Requirements
+------------
+
+* Nautilus (14.2.x) or later Ceph release
+
+* Cephx client user (see :doc:`/rados/operations/user-management`) with
+  at least the following capabilities::
+
+    mon 'allow r'
+    mgr 'allow rw'
+
+FS Volumes
+----------
+
+Create a volume by running the following command:
+
+.. prompt:: bash #
+
+   ceph fs volume create <vol_name> [placement]
+
+This creates a CephFS file system and its data and metadata pools. It can also
+deploy MDS daemons for the filesystem using a ceph-mgr orchestrator module (for
+example Rook). See :doc:`/mgr/orchestrator`.
+
+``<vol_name>`` is the volume name (an arbitrary string). ``[placement]`` is an
+optional string that specifies the :ref:`orchestrator-cli-placement-spec` for
+the MDS. See also :ref:`orchestrator-cli-cephfs` for more examples on
+placement.
+
+.. note:: Specifying placement via a YAML file is not supported through the
+          volume interface.
+
+To remove a volume, run the following command:
+
+    $ ceph fs volume rm <vol_name> [--yes-i-really-mean-it]
+
+This removes a file system and its data and metadata pools. It also tries to
+remove MDS daemons using the enabled ceph-mgr orchestrator module.
+
+.. note:: After volume deletion, it is recommended to restart `ceph-mgr`
+   if a new file system is created on the same cluster and subvolume interface
+   is being used. Please see https://tracker.ceph.com/issues/49605#note-5
+   for more details.
+
+List volumes by running the following command:
+
+    $ ceph fs volume ls
+
+Rename a volume by running the following command:
+
+    $ ceph fs volume rename <vol_name> <new_vol_name> [--yes-i-really-mean-it]
+
+Renaming a volume can be an expensive operation that requires the following:
+
+- Renaming the orchestrator-managed MDS service to match the <new_vol_name>.
+  This involves launching a MDS service with ``<new_vol_name>`` and bringing
+  down the MDS service with ``<vol_name>``.
+- Renaming the file system matching ``<vol_name>`` to ``<new_vol_name>``.
+- Changing the application tags on the data and metadata pools of the file system
+  to ``<new_vol_name>``.
+- Renaming the metadata and data pools of the file system.
+
+The CephX IDs that are authorized for ``<vol_name>`` must be reauthorized for
+``<new_vol_name>``. Any ongoing operations of the clients using these IDs may
+be disrupted. Ensure that mirroring is disabled on the volume.
+
+To fetch the information of a CephFS volume, run the following command:
+
+    $ ceph fs volume info vol_name [--human_readable]
+
+The ``--human_readable`` flag shows used and available pool capacities in KB/MB/GB.
+
+The output format is JSON and contains fields as follows:
+
+* ``pools``: Attributes of data and metadata pools
+        * ``avail``: The amount of free space available in bytes
+        * ``used``: The amount of storage consumed in bytes
+        * ``name``: Name of the pool
+* ``mon_addrs``: List of Ceph monitor addresses
+* ``used_size``: Current used size of the CephFS volume in bytes
+* ``pending_subvolume_deletions``: Number of subvolumes pending deletion
+
+Sample output of the ``volume info`` command::
+
+  $ ceph fs volume info vol_name
+  {
+      "mon_addrs": [
+          "192.168.1.7:40977"
+      ],
+      "pending_subvolume_deletions": 0,
+      "pools": {
+          "data": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.data",
+                  "used": 4096
+              }
+          ],
+          "metadata": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.meta",
+                  "used": 155648
+              }
+          ]
+      },
+      "used_size": 0
+  }
+
+FS Subvolume groups
+-------------------
+
+Create a subvolume group by running the following command:
+
+    $ ceph fs subvolumegroup create <vol_name> <group_name> [--size <size_in_bytes>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
+
+The command succeeds even if the subvolume group already exists.
+
+When creating a subvolume group you can specify its data pool layout (see
+:doc:`/cephfs/file-layouts`), uid, gid, file mode in octal numerals, and
+size in bytes. The size of the subvolume group is specified by setting
+a quota on it (see :doc:`/cephfs/quota`). By default, the subvolume group
+is created with octal file mode ``755``, uid ``0``, gid ``0`` and the data pool
+layout of its parent directory.
+
+Remove a subvolume group by running a command of the following form:
+
+    $ ceph fs subvolumegroup rm <vol_name> <group_name> [--force]
+
+The removal of a subvolume group fails if the subvolume group is not empty or
+is non-existent. The ``--force`` flag allows the non-existent "subvolume group remove
+command" to succeed.
+
+Fetch the absolute path of a subvolume group by running a command of the
+following form:
+
+    $ ceph fs subvolumegroup getpath <vol_name> <group_name>
+
+List subvolume groups by running a command of the following form:
+
+    $ ceph fs subvolumegroup ls <vol_name>
+
+.. note:: Subvolume group snapshot feature is no longer supported in mainline
+   CephFS (existing group snapshots can still be listed and deleted)
+
+Fetch the metadata of a subvolume group by running a command of the following form:
+
+.. prompt:: bash #
+
+   ceph fs subvolumegroup info <vol_name> <group_name>
+
+The output format is JSON and contains fields as follows:
+
+* ``atime``: access time of the subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* ``mtime``: modification time of the subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* ``ctime``: change time of the subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* ``uid``: uid of the subvolume group path
+* ``gid``: gid of the subvolume group path
+* ``mode``: mode of the subvolume group path
+* ``mon_addrs``: list of monitor addresses
+* ``bytes_pcent``: quota used in percentage if quota is set, else displays "undefined"
+* ``bytes_quota``: quota size in bytes if quota is set, else displays "infinite"
+* ``bytes_used``: current used size of the subvolume group in bytes
+* ``created_at``: creation time of the subvolume group in the format "YYYY-MM-DD HH:MM:SS"
+* ``data_pool``: data pool to which the subvolume group belongs
+
+Check the presence of any subvolume group by running a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph fs subvolumegroup exist <vol_name>
+
+The ``exist`` command outputs:
+
+* "subvolumegroup exists": if any subvolumegroup is present
+* "no subvolumegroup exists": if no subvolumegroup is present
+
+.. note:: This command checks for the presence of custom groups and not
+   presence of the default one. To validate the emptiness of the volume, a
+   subvolumegroup existence check alone is not sufficient. Subvolume existence
+   also needs to be checked as there might be subvolumes in the default group.
+
+Resize a subvolume group by running a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs subvolumegroup resize <vol_name> <group_name> <new_size> [--no_shrink]
+
+The command resizes the subvolume group quota, using the size specified by
+``new_size``.  The ``--no_shrink`` flag prevents the subvolume group from
+shrinking below the current used size.
+
+The subvolume group may be resized to an infinite size by passing ``inf`` or
+``infinite`` as the ``new_size``.
+
+Remove a snapshot of a subvolume group by running a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs subvolumegroup snapshot rm <vol_name> <group_name> <snap_name> [--force]
+
+Supplying the ``--force`` flag allows the command to succeed when it would
+otherwise fail due to the nonexistence of the snapshot.
+
+List snapshots of a subvolume group by running a command of the following form:
+
+.. prompt:: bash $
+
+   ceph fs subvolumegroup snapshot ls <vol_name> <group_name>
+
+
+FS Subvolumes
+-------------
+
+Create a subvolume using::
+
+    $ ceph fs subvolume create <vol_name> <subvol_name> [--size <size_in_bytes>] [--group_name <subvol_group_name>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>] [--namespace-isolated]
+
+
+The command succeeds even if the subvolume already exists.
+
+When creating a subvolume you can specify its subvolume group, data pool layout,
+uid, gid, file mode in octal numerals, and size in bytes. The size of the subvolume is
+specified by setting a quota on it (see :doc:`/cephfs/quota`). The subvolume can be
+created in a separate RADOS namespace by specifying --namespace-isolated option. By
+default a subvolume is created within the default subvolume group, and with an octal file
+mode '755', uid of its subvolume group, gid of its subvolume group, data pool layout of
+its parent directory and no size limit.
+
+Remove a subvolume using::
+
+    $ ceph fs subvolume rm <vol_name> <subvol_name> [--group_name <subvol_group_name>] [--force] [--retain-snapshots]
+
+The command removes the subvolume and its contents. It does this in two steps.
+First, it moves the subvolume to a trash folder, and then asynchronously purges
+its contents.
+
+The removal of a subvolume fails if it has snapshots, or is non-existent.
+'--force' flag allows the non-existent subvolume remove command to succeed.
+
+A subvolume can be removed retaining existing snapshots of the subvolume using the
+'--retain-snapshots' option. If snapshots are retained, the subvolume is considered
+empty for all operations not involving the retained snapshots.
+
+.. note:: Snapshot retained subvolumes can be recreated using 'ceph fs subvolume create'
+
+.. note:: Retained snapshots can be used as a clone source to recreate the subvolume, or clone to a newer subvolume.
+
+Resize a subvolume using::
+
+    $ ceph fs subvolume resize <vol_name> <subvol_name> <new_size> [--group_name <subvol_group_name>] [--no_shrink]
+
+The command resizes the subvolume quota using the size specified by ``new_size``.
+The `--no_shrink`` flag prevents the subvolume from shrinking below the current  used size of the subvolume.
+
+The subvolume can be resized to an unlimited (but sparse) logical size by passing ``inf`` or ``infinite`` as `` new_size``.
+
+Authorize cephx auth IDs, the read/read-write access to fs subvolumes::
+
+    $ ceph fs subvolume authorize <vol_name> <sub_name> <auth_id> [--group_name=<group_name>] [--access_level=<access_level>]
+
+The ``access_level`` takes ``r`` or ``rw`` as value.
+
+Deauthorize cephx auth IDs, the read/read-write access to fs subvolumes::
+
+    $ ceph fs subvolume deauthorize <vol_name> <sub_name> <auth_id> [--group_name=<group_name>]
+
+List cephx auth IDs authorized to access fs subvolume::
+
+    $ ceph fs subvolume authorized_list <vol_name> <sub_name> [--group_name=<group_name>]
+
+Evict fs clients based on auth ID and subvolume mounted::
+
+    $ ceph fs subvolume evict <vol_name> <sub_name> <auth_id> [--group_name=<group_name>]
+
+Fetch the absolute path of a subvolume using::
+
+    $ ceph fs subvolume getpath <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Fetch the information of a subvolume using::
+
+    $ ceph fs subvolume info <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+The output format is JSON and contains fields as follows.
+
+* ``atime``: access time of the subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* ``mtime``: modification time of the subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* ``ctime``: change time of the subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* ``uid``: uid of the subvolume path
+* ``gid``: gid of the subvolume path
+* ``mode``: mode of the subvolume path
+* ``mon_addrs``: list of monitor addresses
+* ``bytes_pcent``: quota used in percentage if quota is set, else displays ``undefined``
+* ``bytes_quota``: quota size in bytes if quota is set, else displays ``infinite``
+* ``bytes_used``: current used size of the subvolume in bytes
+* ``created_at``: creation time of the subvolume in the format "YYYY-MM-DD HH:MM:SS"
+* ``data_pool``: data pool to which the subvolume belongs
+* ``path``: absolute path of a subvolume
+* ``type``: subvolume type indicating whether it's clone or subvolume
+* ``pool_namespace``: RADOS namespace of the subvolume
+* ``features``: features supported by the subvolume
+* ``state``: current state of the subvolume
+
+If a subvolume has been removed retaining its snapshots, the output contains only fields as follows.
+
+* ``type``: subvolume type indicating whether it's clone or subvolume
+* ``features``: features supported by the subvolume
+* ``state``: current state of the subvolume
+
+A subvolume's ``features`` are based on the internal version of the subvolume and are
+a subset of the following:
+
+* ``snapshot-clone``: supports cloning using a subvolumes snapshot as the source
+* ``snapshot-autoprotect``: supports automatically protecting snapshots, that are active clone sources, from deletion
+* ``snapshot-retention``: supports removing subvolume contents, retaining any existing snapshots
+
+A subvolume's ``state`` is based on the current state of the subvolume and contains one of the following values.
+
+* ``complete``: subvolume is ready for all operations
+* ``snapshot-retained``: subvolume is removed but its snapshots are retained
+
+List subvolumes using::
+
+    $ ceph fs subvolume ls <vol_name> [--group_name <subvol_group_name>]
+
+.. note:: subvolumes that are removed but have snapshots retained, are also listed.
+
+Check the presence of any subvolume using::
+
+    $ ceph fs subvolume exist <vol_name> [--group_name <subvol_group_name>]
+
+These are the possible results of the ``exist`` command:
+
+* ``subvolume exists``: if any subvolume of given group_name is present
+* ``no subvolume exists``: if no subvolume of given group_name is present
+
+Set custom metadata on the subvolume as a key-value pair using::
+
+    $ ceph fs subvolume metadata set <vol_name> <subvol_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: key_name and value should be a string of ASCII characters (as specified in python's string.printable). key_name is case-insensitive and always stored in lower case.
+
+.. note:: Custom metadata on a subvolume is not preserved when snapshotting the subvolume, and hence, is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata get <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the subvolume using::
+
+    $ ceph fs subvolume metadata ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata rm <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the ``--force`` flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
+
+Create a snapshot of a subvolume using::
+
+    $ ceph fs subvolume snapshot create <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+Remove a snapshot of a subvolume using::
+
+    $ ceph fs subvolume snapshot rm <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>] [--force]
+
+Using the ``--force`` flag allows the command to succeed that would otherwise
+fail if the snapshot did not exist.
+
+.. note:: if the last snapshot within a snapshot retained subvolume is removed, the subvolume is also removed
+
+List snapshots of a subvolume using::
+
+    $ ceph fs subvolume snapshot ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Fetch the information of a snapshot using::
+
+    $ ceph fs subvolume snapshot info <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+The output format is JSON and contains fields as follows.
+
+* ``created_at``: creation time of the snapshot in the format "YYYY-MM-DD HH:MM:SS:ffffff"
+* ``data_pool``: data pool to which the snapshot belongs
+* ``has_pending_clones``: ``yes`` if snapshot clone is in progress, otherwise ``no``
+* ``pending_clones``: list of in-progress or pending clones and their target group if any exist, otherwise this field is not shown
+* ``orphan_clones_count``: count of orphan clones if the snapshot has orphan clones, otherwise this field is not shown
+
+Sample output when snapshot clones are in progress or pending::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "yes",
+      "pending_clones": [
+          {
+              "name": "clone_1",
+              "target_group": "target_subvol_group"
+          },
+          {
+              "name": "clone_2"
+          },
+          {
+              "name": "clone_3",
+              "target_group": "target_subvol_group"
+          }
+      ]
+  }
+
+Sample output when no snapshot clone is in progress or pending::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "no"
+  }
+
+Set custom key-value metadata on the snapshot by running::
+
+    $ ceph fs subvolume snapshot metadata set <vol_name> <subvol_name> <snap_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: The key_name and value should be a strings of ASCII characters (as specified in Python's ``string.printable``). The key_name is case-insensitive and always stored in lowercase.
+
+.. note:: Custom metadata on a snapshot is not preserved when snapshotting the subvolume, and hence is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata get <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the snapshot using::
+
+    $ ceph fs subvolume snapshot metadata ls <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata rm <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the ``--force`` flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
+
+Cloning Snapshots
+-----------------
+
+Subvolumes can be created by cloning subvolume snapshots. Cloning is an asynchronous operation that copies
+data from a snapshot to a subvolume. Due to this bulk copying, cloning is inefficient for very large
+data sets.
+
+.. note:: Removing a snapshot (source subvolume) would fail if there are pending or in progress clone operations.
+
+Protecting snapshots prior to cloning was a prerequisite in the Nautilus release, and the commands to protect/unprotect
+snapshots were introduced for this purpose. This prerequisite, and hence the commands to protect/unprotect, is being
+deprecated and may be removed from a future release.
+
+The commands being deprecated are:
+
+.. prompt:: bash #
+
+   ceph fs subvolume snapshot protect <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+   ceph fs subvolume snapshot unprotect <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+.. note:: Using the above commands will not result in an error, but they have no useful purpose.
+
+.. note:: Use the ``subvolume info`` command to fetch subvolume metadata regarding supported ``features`` to help decide if protect/unprotect of snapshots is required, based on the availability of the ``snapshot-autoprotect`` feature.
+
+To initiate a clone operation use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name>
+
+If a snapshot (source subvolume) is a part of non-default group, the group name needs to be specified::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --group_name <subvol_group_name>
+
+Cloned subvolumes can be a part of a different group than the source snapshot (by default, cloned subvolumes are created in default group). To clone to a particular group use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --target_group_name <subvol_group_name>
+
+Similar to specifying a pool layout when creating a subvolume, pool layout can be specified when creating a cloned subvolume. To create a cloned subvolume with a specific pool layout use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --pool_layout <pool_layout>
+
+Configure the maximum number of concurrent clones. The default is 4::
+
+  $ ceph config set mgr mgr/volumes/max_concurrent_clones <value>
+
+To check the status of a clone operation use::
+
+  $ ceph fs clone status <vol_name> <clone_name> [--group_name <group_name>]
+
+A clone can be in one of the following states:
+
+#. ``pending``     : Clone operation has not started
+#. ``in-progress`` : Clone operation is in progress
+#. ``complete``    : Clone operation has successfully finished
+#. ``failed``      : Clone operation has failed
+#. ``canceled``    : Clone operation is cancelled by user
+
+The reason for a clone failure is shown as below:
+
+#. ``errno``     : error number
+#. ``error_msg`` : failure error string
+
+Here is an example of an ``in-progress`` clone::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "in-progress",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+      }
+    }
+  }
+
+.. note:: The ``failure`` section will be shown only if the clone's state is ``failed`` or ``cancelled``
+
+Here is an example of a ``failed`` clone::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "failed",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+        "size": "104857600"
+      },
+      "failure": {
+        "errno": "122",
+        "errstr": "Disk quota exceeded"
+      }
+    }
+  }
+
+(NOTE: since ``subvol1`` is in the default group, the ``source`` object's  ``clone status`` does not include the group name)
+
+.. note:: Cloned subvolumes are accessible only after the clone operation has successfully completed.
+
+After a successful clone operation, ``clone status`` will look like the below::
+
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "complete"
+    }
+  }
+
+If a clone operation is unsuccessful, the ``state`` value will be  ``failed``.
+
+To retry a failed clone operation, the incomplete clone must be deleted and the clone operation must be issued again.
+To delete a partial clone use::
+
+  $ ceph fs subvolume rm <vol_name> <clone_name> [--group_name <group_name>] --force
+
+.. note:: Cloning synchronizes only directories, regular files and symbolic links. Inode timestamps (access and
+          modification times) are synchronized up to seconds granularity.
+
+An ``in-progress`` or a ``pending`` clone operation may be canceled. To cancel a clone operation use the ``clone cancel`` command::
+
+  $ ceph fs clone cancel <vol_name> <clone_name> [--group_name <group_name>]
+
+On successful cancellation, the cloned subvolume is moved to the ``canceled`` state::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone cancel cephfs clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "canceled",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+      }
+    }
+  }
+
+.. note:: The canceled cloned may be deleted by supplying the ``--force`` option to the `fs subvolume rm` command.
+
+
+.. _subvol-pinning:
+
+Pinning Subvolumes and Subvolume Groups
+---------------------------------------
+
+Subvolumes and subvolume groups may be automatically pinned to ranks according
+to policies. This can distribute load across MDS ranks in predictable and
+stable ways.  Review :ref:`cephfs-pinning` and :ref:`cephfs-ephemeral-pinning`
+for details on how pinning works.
+
+Pinning is configured by::
+
+  $ ceph fs subvolumegroup pin <vol_name> <group_name> <pin_type> <pin_setting>
+
+or for subvolumes::
+
+  $ ceph fs subvolume pin <vol_name> <group_name> <pin_type> <pin_setting>
+
+Typically you will want to set subvolume group pins. The ``pin_type`` may be
+one of ``export``, ``distributed``, or ``random``. The ``pin_setting``
+corresponds to the extended attributed "value" as in the pinning documentation
+referenced above.
+
+So, for example, setting a distributed pinning strategy on a subvolume group::
+
+  $ ceph fs subvolumegroup pin cephfilesystem-a csi distributed 1
+
+Will enable distributed subtree partitioning policy for the "csi" subvolume
+group.  This will cause every subvolume within the group to be automatically
+pinned to one of the available ranks on the file system.
+
+
+.. _manila: https://github.com/openstack/manila
+.. _CSI: https://github.com/ceph/ceph-csi
diff --git a/doc/cephfs/full.rst b/doc/cephfs/full.rst
new file mode 100644
index 000000000..fe0616cb6
--- /dev/null
+++ b/doc/cephfs/full.rst
@@ -0,0 +1,60 @@
+
+Handling a full Ceph file system
+================================
+
+When a RADOS cluster reaches its ``mon_osd_full_ratio`` (default
+95%) capacity, it is marked with the OSD full flag.  This flag causes
+most normal RADOS clients to pause all operations until it is resolved
+(for example by adding more capacity to the cluster).
+
+The file system has some special handling of the full flag, explained below.
+
+Hammer and later
+----------------
+
+Since the hammer release, a full file system will lead to ENOSPC
+results from:
+
+ * Data writes on the client
+ * Metadata operations other than deletes and truncates
+
+Because the full condition may not be encountered until
+data is flushed to disk (sometime after a ``write`` call has already
+returned 0), the ENOSPC error may not be seen until the application
+calls ``fsync`` or ``fclose`` (or equivalent) on the file handle.
+
+Calling ``fsync`` is guaranteed to reliably indicate whether the data
+made it to disk, and will return an error if it doesn't.  ``fclose`` will
+only return an error if buffered data happened to be flushed since
+the last write -- a successful ``fclose`` does not guarantee that the
+data made it to disk, and in a full-space situation, buffered data
+may be discarded after an ``fclose`` if no space is available to persist it.
+
+.. warning::
+    If an application appears to be misbehaving on a full file system,
+    check that it is performing ``fsync()`` calls as necessary to ensure
+    data is on disk before proceeding.
+
+Data writes may be cancelled by the client if they are in flight at the
+time the OSD full flag is sent.  Clients update the ``osd_epoch_barrier``
+when releasing capabilities on files affected by cancelled operations, in
+order to ensure that these cancelled operations do not interfere with
+subsequent access to the data objects by the MDS or other clients.  For
+more on the epoch barrier mechanism, see :ref:`background_blocklisting_and_osd_epoch_barrier`.
+
+Legacy (pre-hammer) behavior
+----------------------------
+
+In versions of Ceph earlier than hammer, the MDS would ignore
+the full status of the RADOS cluster, and any data writes from
+clients would stall until the cluster ceased to be full.
+
+There are two dangerous conditions to watch for with this behaviour:
+
+* If a client had pending writes to a file, then it was not possible
+  for the client to release the file to the MDS for deletion: this could
+  lead to difficulty clearing space on a full file system
+* If clients continued to create a large number of empty files, the
+  resulting metadata writes from the MDS could lead to total exhaustion
+  of space on the OSDs such that no further deletions could be performed.
+
diff --git a/doc/cephfs/health-messages.rst b/doc/cephfs/health-messages.rst
new file mode 100644
index 000000000..7edc1262f
--- /dev/null
+++ b/doc/cephfs/health-messages.rst
@@ -0,0 +1,240 @@
+
+.. _cephfs-health-messages:
+
+======================
+CephFS health messages
+======================
+
+Cluster health checks
+=====================
+
+The Ceph monitor daemons will generate health messages in response
+to certain states of the file system map structure (and the enclosed MDS maps).
+
+Message: mds rank(s) *ranks* have failed
+Description: One or more MDS ranks are not currently assigned to
+an MDS daemon; the cluster will not recover until a suitable replacement
+daemon starts.
+
+Message: mds rank(s) *ranks* are damaged
+Description: One or more MDS ranks has encountered severe damage to
+its stored metadata, and cannot start again until it is repaired.
+
+Message: mds cluster is degraded
+Description: One or more MDS ranks are not currently up and running, clients
+may pause metadata IO until this situation is resolved.  This includes
+ranks being failed or damaged, and additionally includes ranks
+which are running on an MDS but have not yet made it to the *active*
+state (e.g. ranks currently in *replay* state).
+
+Message: mds *names* are laggy
+Description: The named MDS daemons have failed to send beacon messages
+to the monitor for at least ``mds_beacon_grace`` (default 15s), while
+they are supposed to send beacon messages every ``mds_beacon_interval``
+(default 4s).  The daemons may have crashed.  The Ceph monitor will
+automatically replace laggy daemons with standbys if any are available.
+
+Message: insufficient standby daemons available
+Description: One or more file systems are configured to have a certain number
+of standby daemons available (including daemons in standby-replay) but the
+cluster does not have enough standby daemons. The standby daemons not in replay
+count towards any file system (i.e. they may overlap). This warning can
+configured by setting ``ceph fs set <fs> standby_count_wanted <count>``.  Use
+zero for ``count`` to disable.
+
+
+Daemon-reported health checks
+=============================
+
+MDS daemons can identify a variety of unwanted conditions, and
+indicate these to the operator in the output of ``ceph status``.
+These conditions have human readable messages, and additionally
+a unique code starting with ``MDS_``.
+
+.. highlight:: console
+
+``ceph health detail`` shows the details of the conditions. Following
+is a typical health report from a cluster experiencing MDS related
+performance issues::
+
+  $ ceph health detail
+  HEALTH_WARN 1 MDSs report slow metadata IOs; 1 MDSs report slow requests
+  MDS_SLOW_METADATA_IO 1 MDSs report slow metadata IOs
+     mds.fs-01(mds.0): 3 slow metadata IOs are blocked > 30 secs, oldest blocked for 51123 secs
+  MDS_SLOW_REQUEST 1 MDSs report slow requests
+     mds.fs-01(mds.0): 5 slow requests are blocked > 30 secs
+
+Where, for instance, ``MDS_SLOW_REQUEST`` is the unique code representing the
+condition where requests are taking long time to complete. And the following
+description shows its severity and the MDS daemons which are serving these
+slow requests.
+
+This page lists the health checks raised by MDS daemons. For the checks from
+other daemons, please see :ref:`health-checks`.
+
+``MDS_TRIM``
+------------
+
+  Message
+    "Behind on trimming..."
+  Description
+    CephFS maintains a metadata journal that is divided into
+    *log segments*.  The length of journal (in number of segments) is controlled
+    by the setting ``mds_log_max_segments``, and when the number of segments
+    exceeds that setting the MDS starts writing back metadata so that it
+    can remove (trim) the oldest segments.  If this writeback is happening
+    too slowly, or a software bug is preventing trimming, then this health
+    message may appear.  The threshold for this message to appear is controlled by
+    the config option ``mds_log_warn_factor``, the default is 2.0.
+
+``MDS_HEALTH_CLIENT_LATE_RELEASE``, ``MDS_HEALTH_CLIENT_LATE_RELEASE_MANY``
+---------------------------------------------------------------------------
+
+  Message
+    "Client *name* failing to respond to capability release"
+  Description
+    CephFS clients are issued *capabilities* by the MDS, which
+    are like locks.  Sometimes, for example when another client needs access,
+    the MDS will request clients release their capabilities.  If the client
+    is unresponsive or buggy, it might fail to do so promptly or fail to do
+    so at all.  This message appears if a client has taken longer than
+    ``session_timeout`` (default 60s) to comply.
+
+``MDS_CLIENT_RECALL``, ``MDS_HEALTH_CLIENT_RECALL_MANY``
+--------------------------------------------------------
+
+  Message
+    "Client *name* failing to respond to cache pressure"
+  Description
+    Clients maintain a metadata cache.  Items (such as inodes) in the
+    client cache are also pinned in the MDS cache, so when the MDS needs to shrink
+    its cache (to stay within ``mds_cache_memory_limit``), it sends messages to
+    clients to shrink their caches too.  If the client is unresponsive or buggy,
+    this can prevent the MDS from properly staying within its cache limits and it
+    may eventually run out of memory and crash.  This message appears if a client
+    has failed to release more than
+    ``mds_recall_warning_threshold`` capabilities (decaying with a half-life of
+    ``mds_recall_max_decay_rate``) within the last
+    ``mds_recall_warning_decay_rate`` second.
+
+``MDS_CLIENT_OLDEST_TID``, ``MDS_CLIENT_OLDEST_TID_MANY``
+---------------------------------------------------------
+
+  Message
+    "Client *name* failing to advance its oldest client/flush tid"
+  Description
+    The CephFS client-MDS protocol uses a field called the
+    *oldest tid* to inform the MDS of which client requests are fully
+    complete and may therefore be forgotten about by the MDS.  If a buggy
+    client is failing to advance this field, then the MDS may be prevented
+    from properly cleaning up resources used by client requests.  This message
+    appears if a client appears to have more than ``max_completed_requests``
+    (default 100000) requests that are complete on the MDS side but haven't
+    yet been accounted for in the client's *oldest tid* value.
+
+``MDS_DAMAGE``
+--------------
+
+  Message
+    "Metadata damage detected"
+  Description
+    Corrupt or missing metadata was encountered when reading
+    from the metadata pool.  This message indicates that the damage was
+    sufficiently isolated for the MDS to continue operating, although
+    client accesses to the damaged subtree will return IO errors.  Use
+    the ``damage ls`` admin socket command to get more detail on the damage.
+    This message appears as soon as any damage is encountered.
+
+``MDS_HEALTH_READ_ONLY``
+------------------------
+
+  Message
+    "MDS in read-only mode"
+  Description
+    The MDS has gone into readonly mode and will return EROFS
+    error codes to client operations that attempt to modify any metadata.  The
+    MDS will go into readonly mode if it encounters a write error while
+    writing to the metadata pool, or if forced to by an administrator using
+    the *force_readonly* admin socket command.
+
+``MDS_SLOW_REQUEST``
+--------------------
+
+  Message
+    "*N* slow requests are blocked"
+
+  Description
+    One or more client requests have not been completed promptly,
+    indicating that the MDS is either running very slowly, or that the RADOS
+    cluster is not acknowledging journal writes promptly, or that there is a bug.
+    Use the ``ops`` admin socket command to list outstanding metadata operations.
+    This message appears if any client requests have taken longer than
+    ``mds_op_complaint_time`` (default 30s).
+
+``MDS_CACHE_OVERSIZED``
+-----------------------
+
+  Message
+    "Too many inodes in cache"
+  Description
+    The MDS is not succeeding in trimming its cache to comply with the
+    limit set by the administrator.  If the MDS cache becomes too large, the daemon
+    may exhaust available memory and crash.  By default, this message appears if
+    the actual cache size (in memory) is at least 50% greater than
+    ``mds_cache_memory_limit`` (default 4GB). Modify ``mds_health_cache_threshold``
+    to set the warning ratio.
+
+``FS_WITH_FAILED_MDS``
+----------------------
+
+  Message
+    "Some MDS ranks do not have standby replacements"
+
+  Description
+    Normally, a failed MDS rank will be replaced by a standby MDS. This situation
+    is transient and is not considered critical. However, if there are no standby
+    MDSs available to replace an active MDS rank, this health warning is generated.
+
+``MDS_INSUFFICIENT_STANDBY``
+----------------------------
+
+  Message
+    "Insufficient number of available standby(-replay) MDS daemons than configured"
+
+  Description
+    The minimum number of standby(-replay) MDS daemons can be configured by setting
+    ``standby_count_wanted`` configuration variable. This health warning is generated
+    when the configured value mismatches the number of standby(-replay) MDS daemons
+    available.
+
+``FS_DEGRADED``
+----------------------------
+
+  Message
+    "Some MDS ranks have been marked failed or damaged"
+
+  Description
+    When one or more MDS rank ends up in failed or damaged state due to
+    an unrecoverable error. The file system may be partially or fully
+    unavailable when one (or more) ranks are offline.
+
+``MDS_UP_LESS_THAN_MAX``
+----------------------------
+
+  Message
+    "Number of active ranks are less than configured number of maximum MDSs"
+
+  Description
+    The maximum number of MDS ranks can be configured by setting ``max_mds``
+    configuration variable. This health warning is generated when the number
+    of MDS ranks falls below this configured value.
+
+``MDS_ALL_DOWN``
+----------------------------
+
+  Message
+    "None of the MDS ranks are available (file system offline)"
+
+  Description
+    All MDS ranks are unavailable resulting in the file system to be completely
+    offline.
diff --git a/doc/cephfs/index.rst b/doc/cephfs/index.rst
new file mode 100644
index 000000000..3d52aef38
--- /dev/null
+++ b/doc/cephfs/index.rst
@@ -0,0 +1,213 @@
+.. _ceph-file-system:
+
+=================
+ Ceph File System
+=================
+
+The Ceph File System, or **CephFS**, is a POSIX-compliant file system built on
+top of Ceph's distributed object store, **RADOS**. CephFS endeavors to provide
+a state-of-the-art, multi-use, highly available, and performant file store for
+a variety of applications, including traditional use-cases like shared home
+directories, HPC scratch space, and distributed workflow shared storage.
+
+CephFS achieves these goals through the use of some novel architectural
+choices.  Notably, file metadata is stored in a separate RADOS pool from file
+data and served via a resizable cluster of *Metadata Servers*, or **MDS**,
+which may scale to support higher throughput metadata workloads.  Clients of
+the file system have direct access to RADOS for reading and writing file data
+blocks. For this reason, workloads may linearly scale with the size of the
+underlying RADOS object store; that is, there is no gateway or broker mediating
+data I/O for clients.
+
+Access to data is coordinated through the cluster of MDS which serve as
+authorities for the state of the distributed metadata cache cooperatively
+maintained by clients and MDS. Mutations to metadata are aggregated by each MDS
+into a series of efficient writes to a journal on RADOS; no metadata state is
+stored locally by the MDS. This model allows for coherent and rapid
+collaboration between clients within the context of a POSIX file system.
+
+.. image:: cephfs-architecture.svg
+
+CephFS is the subject of numerous academic papers for its novel designs and
+contributions to file system research. It is the oldest storage interface in
+Ceph and was once the primary use-case for RADOS.  Now it is joined by two
+other storage interfaces to form a modern unified storage system: RBD (Ceph
+Block Devices) and RGW (Ceph Object Storage Gateway).
+
+
+Getting Started with CephFS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For most deployments of Ceph, setting up your first CephFS file system is as simple as:
+
+.. prompt:: bash
+
+    # Create a CephFS volume named (for example) "cephfs":
+    ceph fs volume create cephfs
+
+The Ceph `Orchestrator`_  will automatically create and configure MDS for
+your file system if the back-end deployment technology supports it (see
+`Orchestrator deployment table`_). Otherwise, please `deploy MDS manually
+as needed`_. You can also `create other CephFS volumes`_.
+
+Finally, to mount CephFS on your client nodes, see `Mount CephFS:
+Prerequisites`_ page. Additionally, a command-line shell utility is available
+for interactive access or scripting via the `cephfs-shell`_.
+
+.. _Orchestrator: ../mgr/orchestrator
+.. _deploy MDS manually as needed: add-remove-mds
+.. _create other CephFS volumes: fs-volumes
+.. _Orchestrator deployment table: ../mgr/orchestrator/#current-implementation-status
+.. _Mount CephFS\: Prerequisites: mount-prerequisites
+.. _cephfs-shell: ../man/8/cephfs-shell
+
+
+.. raw:: html
+
+   <!---
+
+Administration
+^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Create a CephFS file system <createfs>
+    Administrative commands <administration>
+    Creating Multiple File Systems <multifs>
+    Provision/Add/Remove MDS(s) <add-remove-mds>
+    MDS failover and standby configuration <standby>
+    MDS Cache Configuration <cache-configuration>
+    MDS Configuration Settings <mds-config-ref>
+    Manual: ceph-mds <../../man/8/ceph-mds>
+    Export over NFS <nfs>
+    Application best practices <app-best-practices>
+    FS volume and subvolumes <fs-volumes>
+    CephFS Quotas <quota>
+    Health messages <health-messages>
+    Upgrading old file systems <upgrading>
+    CephFS Top Utility <cephfs-top>
+    Scheduled Snapshots <snap-schedule>
+    CephFS Snapshot Mirroring <cephfs-mirroring>
+
+.. raw:: html
+
+   <!---
+
+Mounting CephFS
+^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Client Configuration Settings <client-config-ref>
+    Client Authentication <client-auth>
+    Mount CephFS: Prerequisites <mount-prerequisites>
+    Mount CephFS using Kernel Driver <mount-using-kernel-driver>
+    Mount CephFS using FUSE <mount-using-fuse>
+    Mount CephFS on Windows <ceph-dokan>
+    Use the CephFS Shell <../../man/8/cephfs-shell>
+    Supported Features of Kernel Driver <kernel-features>
+    Manual: ceph-fuse <../../man/8/ceph-fuse>
+    Manual: mount.ceph <../../man/8/mount.ceph>
+    Manual: mount.fuse.ceph <../../man/8/mount.fuse.ceph>
+
+
+.. raw:: html
+
+   <!---
+
+CephFS Concepts
+^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    MDS States <mds-states>
+    POSIX compatibility <posix>
+    MDS Journaling <mds-journaling>
+    File layouts <file-layouts>
+    Distributed Metadata Cache <mdcache>
+    Dynamic Metadata Management in CephFS <dynamic-metadata-management>
+    CephFS IO Path <cephfs-io-path>
+    LazyIO <lazyio>
+    Directory fragmentation <dirfrags>
+    Multiple active MDS daemons <multimds>
+
+
+.. raw:: html
+
+   <!---
+
+Troubleshooting and Disaster Recovery
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :hidden:
+
+    Client eviction <eviction>
+    Scrubbing the File System <scrub>
+    Handling full file systems <full>
+    Metadata repair <disaster-recovery-experts>
+    Troubleshooting <troubleshooting>
+    Disaster recovery <disaster-recovery>
+    cephfs-journal-tool <cephfs-journal-tool>
+    Recovering file system after monitor store loss <recover-fs-after-mon-store-loss>
+
+
+.. raw:: html
+
+   <!---
+
+Developer Guides
+^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Journaler Configuration <journaler>
+    Client's Capabilities <capabilities>
+    Java and Python bindings <api/index>
+    Mantle <mantle>
+
+
+.. raw:: html
+
+   <!---
+
+Additional Details
+^^^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+    Experimental Features <experimental-features>
diff --git a/doc/cephfs/journaler.rst b/doc/cephfs/journaler.rst
new file mode 100644
index 000000000..d2fb2b0ab
--- /dev/null
+++ b/doc/cephfs/journaler.rst
@@ -0,0 +1,7 @@
+===========
+ Journaler
+===========
+
+.. confval:: journaler_write_head_interval
+.. confval:: journaler_prefetch_periods
+.. confval:: journaler_prezero_periods
diff --git a/doc/cephfs/kernel-features.rst b/doc/cephfs/kernel-features.rst
new file mode 100644
index 000000000..c025c7a52
--- /dev/null
+++ b/doc/cephfs/kernel-features.rst
@@ -0,0 +1,47 @@
+
+Supported Features of the Kernel Driver
+========================================
+The kernel driver is developed separately from the core ceph code, and as
+such it sometimes differs from the FUSE driver in feature implementation.
+The following details the implementation status of various CephFS features
+in the kernel driver.
+
+Inline data
+-----------
+Inline data was introduced by the Firefly release. This feature is being
+deprecated in mainline CephFS, and may be removed from a future kernel
+release.
+
+Linux kernel clients >= 3.19 can read inline data and convert existing
+inline data to RADOS objects when file data is modified. At present,
+Linux kernel clients do not store file data as inline data.
+
+See `Experimental Features`_ for more information.
+
+Quotas
+------
+Quota was first introduced by the hammer release. Quota disk format got renewed
+by the Mimic release. Linux kernel clients >= 4.17 can support the new format
+quota. At present, no Linux kernel client support the old format quota.
+
+See `Quotas`_ for more information.
+
+Multiple file systems within a Ceph cluster
+-------------------------------------------
+The feature was introduced by the Jewel release. Linux kernel clients >= 4.7
+can support it.
+
+See `Experimental Features`_ for more information.
+
+Multiple active metadata servers
+--------------------------------
+The feature has been supported since the Luminous release. It is recommended to
+use Linux kernel clients >= 4.14 when there are multiple active MDS.
+
+Snapshots
+---------
+The feature has been supported since the Mimic release. It is recommended to
+use Linux kernel clients >= 4.17 if snapshot is used.
+
+.. _Experimental Features: ../experimental-features
+.. _Quotas: ../quota
diff --git a/doc/cephfs/lazyio.rst b/doc/cephfs/lazyio.rst
new file mode 100644
index 000000000..b30058774
--- /dev/null
+++ b/doc/cephfs/lazyio.rst
@@ -0,0 +1,76 @@
+======
+LazyIO
+======
+
+LazyIO relaxes POSIX semantics. Buffered reads/writes are allowed even when a
+file is opened by multiple applications on multiple clients. Applications are
+responsible for managing cache coherency themselves.
+
+Libcephfs supports LazyIO since nautilus release.
+
+Enable LazyIO
+=============
+
+LazyIO can be enabled by following ways.
+
+- ``client_force_lazyio`` option enables LAZY_IO globally for libcephfs and
+  ceph-fuse mount.
+
+- ``ceph_lazyio(...)`` and ``ceph_ll_lazyio(...)`` enable LAZY_IO for file handle
+  in libcephfs.
+
+Using LazyIO
+============
+
+LazyIO includes two methods ``lazyio_propagate()`` and ``lazyio_synchronize()``.
+With LazyIO enabled, writes may not be visible to other clients until
+``lazyio_propagate()`` is called. Reads may come from local cache (irrespective of
+changes to the file by other clients) until ``lazyio_synchronize()`` is called.
+
+- ``lazyio_propagate(int fd, loff_t offset, size_t count)`` - Ensures that any
+  buffered writes of the client, in the specific region (offset to offset+count),
+  has been propagated to the shared file. If offset and count are both 0, the 
+  operation is performed on the entire file. Currently only this is supported.
+
+- ``lazyio_synchronize(int fd, loff_t offset, size_t count)`` - Ensures that the
+  client is, in a subsequent read call, able to read the updated file with all 
+  the propagated writes of the other clients. In CephFS this is facilitated by
+  invalidating the file caches pertaining to the inode and hence forces the
+  client to refetch/recache the data from the updated file. Also if the write cache
+  of the calling client is dirty (not propagated), lazyio_synchronize() flushes it as well.
+
+An example usage (utilizing libcephfs) is given below. This is a sample I/O loop for a
+particular client/file descriptor in a parallel application:
+
+::
+
+        /* Client a (ca) opens the shared file file.txt */
+        int fda = ceph_open(ca, "shared_file.txt", O_CREAT|O_RDWR, 0644); 
+
+        /* Enable LazyIO for fda */
+        ceph_lazyio(ca, fda, 1));
+
+        for(i = 0; i < num_iters; i++) {
+            char out_buf[] = "fooooooooo";
+            
+            ceph_write(ca, fda, out_buf, sizeof(out_buf), i);
+            /* Propagate the writes associated with fda to the backing storage*/
+            ceph_propagate(ca, fda, 0, 0);
+            
+            /* The barrier makes sure changes associated with all file descriptors
+            are propagated so that there is certainty that the backing file 
+            is up to date */
+            application_specific_barrier();
+
+            char in_buf[40];
+            /* Calling ceph_lazyio_synchronize here will ascertain that ca will
+            read the updated file with the propagated changes and not read
+            stale cached data */
+            ceph_lazyio_synchronize(ca, fda, 0, 0);
+            ceph_read(ca, fda, in_buf, sizeof(in_buf), 0);
+
+            /* A barrier is required here before returning to the next write
+            phase so as to avoid overwriting the portion of the shared file still
+            being read by another file descriptor */
+            application_specific_barrier();
+        }
diff --git a/doc/cephfs/mantle.rst b/doc/cephfs/mantle.rst
new file mode 100644
index 000000000..dc9e62461
--- /dev/null
+++ b/doc/cephfs/mantle.rst
@@ -0,0 +1,263 @@
+Mantle
+======
+
+.. warning::
+
+    Mantle is for research and development of metadata balancer algorithms,
+    not for use on production CephFS clusters.
+
+Multiple, active MDSs can migrate directories to balance metadata load. The
+policies for when, where, and how much to migrate are hard-coded into the
+metadata balancing module. Mantle is a programmable metadata balancer built
+into the MDS. The idea is to protect the mechanisms for balancing load
+(migration, replication, fragmentation) but stub out the balancing policies
+using Lua. Mantle is based on [1] but the current implementation does *NOT*
+have the following features from that paper:
+
+1. Balancing API: in the paper, the user fills in when, where, how much, and
+   load calculation policies; currently, Mantle only requires that Lua policies
+   return a table of target loads (e.g., how much load to send to each MDS)
+2. "How much" hook: in the paper, there was a hook that let the user control
+   the fragment selector policy; currently, Mantle does not have this hook
+3. Instantaneous CPU utilization as a metric
+
+[1] Supercomputing '15 Paper:
+http://sc15.supercomputing.org/schedule/event_detail-evid=pap168.html
+
+Quickstart with vstart
+----------------------
+
+.. warning::
+
+    Developing balancers with vstart is difficult because running all daemons
+    and clients on one node can overload the system. Let it run for a while, even
+    though you will likely see a bunch of lost heartbeat and laggy MDS warnings.
+    Most of the time this guide will work but sometimes all MDSs lock up and you
+    cannot actually see them spill. It is much better to run this on a cluster.
+
+As a prerequisite, we assume you have installed `mdtest
+<https://sourceforge.net/projects/mdtest/>`_ or pulled the `Docker image
+<https://hub.docker.com/r/michaelsevilla/mdtest/>`_. We use mdtest because we
+need to generate enough load to get over the MIN_OFFLOAD threshold that is
+arbitrarily set in the balancer. For example, this does not create enough
+metadata load:
+
+::
+
+    while true; do
+      touch "/cephfs/blah-`date`"
+    done
+
+
+Mantle with `vstart.sh`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Start Ceph and tune the logging so we can see migrations happen:
+
+::
+
+    cd build
+    ../src/vstart.sh -n -l
+    for i in a b c; do 
+      bin/ceph --admin-daemon out/mds.$i.asok config set debug_ms 0
+      bin/ceph --admin-daemon out/mds.$i.asok config set debug_mds 2
+      bin/ceph --admin-daemon out/mds.$i.asok config set mds_beacon_grace 1500
+    done
+
+
+2. Put the balancer into RADOS:
+
+::
+
+    bin/rados put --pool=cephfs_metadata_a greedyspill.lua ../src/mds/balancers/greedyspill.lua
+
+
+3. Activate Mantle:
+
+::
+
+    bin/ceph fs set cephfs max_mds 5
+    bin/ceph fs set cephfs_a balancer greedyspill.lua
+
+
+4. Mount CephFS in another window:
+
+::
+
+     bin/ceph-fuse /cephfs -o allow_other &
+     tail -f out/mds.a.log
+
+
+   Note that if you look at the last MDS (which could be a, b, or c -- it's
+   random), you will see an attempt to index a nil value. This is because the
+   last MDS tries to check the load of its neighbor, which does not exist.
+
+5. Run a simple benchmark. In our case, we use the Docker mdtest image to
+   create load:
+
+::
+
+    for i in 0 1 2; do
+      docker run -d \
+        --name=client$i \
+        -v /cephfs:/cephfs \
+        michaelsevilla/mdtest \
+        -F -C -n 100000 -d "/cephfs/client-test$i"
+    done
+
+
+6. When you are done, you can kill all the clients with:
+
+::
+
+    for i in 0 1 2 3; do docker rm -f client$i; done
+
+
+Output
+~~~~~~
+
+Looking at the log for the first MDS (could be a, b, or c), we see that
+everyone has no load:
+
+::
+
+    2016-08-21 06:44:01.763930 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=1.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.763966 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.763982 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.764010 7fd03aaf7700  2 lua.balancer when: not migrating! my_load=0.0 hisload=0.0
+    2016-08-21 06:44:01.764033 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={}
+
+
+After the jobs starts, MDS0 gets about 1953 units of load. The greedy spill
+balancer dictates that half the load goes to your neighbor MDS, so we see that
+Mantle tries to send 1953 load units to MDS1.
+
+::
+
+    2016-08-21 06:45:21.869994 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=5834.188908912 all.meta_load=1953.3492228857 req_rate=12591.0 queue_len=1075.0 cpu_load_avg=3.05 > load=1953.3492228857
+    2016-08-21 06:45:21.870017 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=3.05 > load=0.0
+    2016-08-21 06:45:21.870027 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=3.05 > load=0.0
+    2016-08-21 06:45:21.870034 7fd03aaf7700  2 lua.balancer when: migrating! my_load=1953.3492228857 hisload=0.0
+    2016-08-21 06:45:21.870050 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={0=0,1=976.675,2=0}
+    2016-08-21 06:45:21.870094 7fd03aaf7700  0 mds.0.bal    - exporting [0,0.52287 1.04574] 1030.88 to mds.1 [dir 100000006ab /client-test2/ [2,head] auth pv=33 v=32 cv=32/0 ap=2+3+4 state=1610612802|complete f(v0 m2016-08-21 06:44:20.366935 1=0+1) n(v2 rc2016-08-21 06:44:30.946816 3790=3788+2) hs=1+0,ss=0+0 dirty=1 | child=1 dirty=1 authpin=1 0x55d2762fd690]
+    2016-08-21 06:45:21.870151 7fd03aaf7700  0 mds.0.migrator nicely exporting to mds.1 [dir 100000006ab /client-test2/ [2,head] auth pv=33 v=32 cv=32/0 ap=2+3+4 state=1610612802|complete f(v0 m2016-08-21 06:44:20.366935 1=0+1) n(v2 rc2016-08-21 06:44:30.946816 3790=3788+2) hs=1+0,ss=0+0 dirty=1 | child=1 dirty=1 authpin=1 0x55d2762fd690]
+
+
+Eventually load moves around:
+
+::
+
+    2016-08-21 06:47:10.210253 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=415.77414300449 all.meta_load=415.79000078186 req_rate=82813.0 queue_len=0.0 cpu_load_avg=11.97 > load=415.79000078186
+    2016-08-21 06:47:10.210277 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=228.72023977691 all.meta_load=186.5606496623 req_rate=28580.0 queue_len=0.0 cpu_load_avg=11.97 > load=186.5606496623
+    2016-08-21 06:47:10.210290 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=1.0 queue_len=0.0 cpu_load_avg=11.97 > load=0.0
+    2016-08-21 06:47:10.210298 7fd03aaf7700  2 lua.balancer when: not migrating! my_load=415.79000078186 hisload=186.5606496623
+    2016-08-21 06:47:10.210311 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={}
+
+
+Implementation Details
+----------------------
+
+Most of the implementation is in MDBalancer. Metrics are passed to the balancer
+policies via the Lua stack and a list of loads is returned back to MDBalancer.
+It sits alongside the current balancer implementation and it's enabled with a
+Ceph CLI command ("ceph fs set cephfs balancer mybalancer.lua"). If the Lua policy
+fails (for whatever reason), we fall back to the original metadata load
+balancer. The balancer is stored in the RADOS metadata pool and a string in the
+MDSMap tells the MDSs which balancer to use.
+
+Exposing Metrics to Lua
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Metrics are exposed directly to the Lua code as global variables instead of
+using a well-defined function signature. There is a global "mds" table, where
+each index is an MDS number (e.g., 0) and each value is a dictionary of metrics
+and values. The Lua code can grab metrics using something like this:
+
+::
+
+    mds[0]["queue_len"]
+
+
+This is in contrast to cls-lua in the OSDs, which has well-defined arguments
+(e.g., input/output bufferlists). Exposing the metrics directly makes it easier
+to add new metrics without having to change the API on the Lua side; we want
+the API to grow and shrink as we explore which metrics matter. The downside of
+this approach is that the person programming Lua balancer policies has to look
+at the Ceph source code to see which metrics are exposed. We figure that the
+Mantle developer will be in touch with MDS internals anyways.
+
+The metrics exposed to the Lua policy are the same ones that are already stored
+in mds_load_t: auth.meta_load(), all.meta_load(), req_rate, queue_length,
+cpu_load_avg.
+
+Compile/Execute the Balancer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Here we use `lua_pcall` instead of `lua_call` because we want to handle errors
+in the MDBalancer. We do not want the error propagating up the call chain. The
+cls_lua class wants to handle the error itself because it must fail gracefully.
+For Mantle, we don't care if a Lua error crashes our balancer -- in that case,
+we will fall back to the original balancer.
+
+The performance improvement of using `lua_call` over `lua_pcall` would not be
+leveraged here because the balancer is invoked every 10 seconds by default. 
+
+Returning Policy Decision to C++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We force the Lua policy engine to return a table of values, corresponding to
+the amount of load to send to each MDS. These loads are inserted directly into
+the MDBalancer "my_targets" vector. We do not allow the MDS to return a table
+of MDSs and metrics because we want the decision to be completely made on the
+Lua side.
+
+Iterating through tables returned by Lua is done through the stack. In Lua
+jargon: a dummy value is pushed onto the stack and the next iterator replaces
+the top of the stack with a (k, v) pair. After reading each value, pop that
+value but keep the key for the next call to `lua_next`. 
+
+Reading from RADOS
+~~~~~~~~~~~~~~~~~~
+
+All MDSs will read balancing code from RADOS when the balancer version changes
+in the MDS Map. The balancer pulls the Lua code from RADOS synchronously. We do
+this with a timeout: if the asynchronous read does not come back within half
+the balancing tick interval the operation is cancelled and a Connection Timeout
+error is returned. By default, the balancing tick interval is 10 seconds, so
+Mantle will use a 5 second timeout. This design allows Mantle to
+immediately return an error if anything RADOS-related goes wrong.
+
+We use this implementation because we do not want to do a blocking OSD read
+from inside the global MDS lock. Doing so would bring down the MDS cluster if
+any of the OSDs are not responsive -- this is tested in the ceph-qa-suite by
+setting all OSDs to down/out and making sure the MDS cluster stays active.
+
+One approach would be to asynchronously fire the read when handling the MDS Map
+and fill in the Lua code in the background. We cannot do this because the MDS
+does not support daemon-local fallbacks and the balancer assumes that all MDSs
+come to the same decision at the same time (e.g., importers, exporters, etc.).
+
+Debugging
+~~~~~~~~~
+
+Logging in a Lua policy will appear in the MDS log. The syntax is the same as
+the cls logging interface:
+
+::
+
+    BAL_LOG(0, "this is a log message")
+
+
+It is implemented by passing a function that wraps the `dout` logging framework
+(`dout_wrapper`) to Lua with the `lua_register()` primitive. The Lua code is
+actually calling the `dout` function in C++.
+
+Warning and Info messages are centralized using the clog/Beacon. Successful
+messages are only sent on version changes by the first MDS to avoid spamming
+the `ceph -w` utility. These messages are used for the integration tests.
+
+Testing
+~~~~~~~
+
+Testing is done with the ceph-qa-suite (tasks.cephfs.test_mantle). We do not
+test invalid balancer logging and loading the actual Lua VM.
diff --git a/doc/cephfs/mdcache.rst b/doc/cephfs/mdcache.rst
new file mode 100644
index 000000000..f2e20238c
--- /dev/null
+++ b/doc/cephfs/mdcache.rst
@@ -0,0 +1,77 @@
+=================================
+CephFS Distributed Metadata Cache
+=================================
+While the data for inodes in a Ceph file system is stored in RADOS and
+accessed by the clients directly, inode metadata and directory
+information is managed by the Ceph metadata server (MDS). The MDS's
+act as mediator for all metadata related activity, storing the resulting
+information in a separate RADOS pool from the file data.
+
+CephFS clients can request that the MDS fetch or change inode metadata
+on its behalf, but an MDS can also grant the client **capabilities**
+(aka **caps**) for each inode (see :doc:`/cephfs/capabilities`).
+
+A capability grants the client the ability to cache and possibly
+manipulate some portion of the data or metadata associated with the
+inode. When another client needs access to the same information, the MDS
+will revoke the capability and the client will eventually return it,
+along with an updated version of the inode's metadata (in the event that
+it made changes to it while it held the capability).
+
+Clients can request capabilities and will generally get them, but when
+there is competing access or memory pressure on the MDS, they may be
+**revoked**. When a capability is revoked, the client is responsible for
+returning it as soon as it is able. Clients that fail to do so in a
+timely fashion may end up **blocklisted** and unable to communicate with
+the cluster.
+
+Since the cache is distributed, the MDS must take great care to ensure
+that no client holds capabilities that may conflict with other clients'
+capabilities, or operations that it does itself. This allows cephfs
+clients to rely on much greater cache coherence than a filesystem like
+NFS, where the client may cache data and metadata beyond the point where
+it has changed on the server.
+
+Client Metadata Requests
+------------------------
+When a client needs to query/change inode metadata or perform an
+operation on a directory, it has two options. It can make a request to
+the MDS directly, or serve the information out of its cache. With
+CephFS, the latter is only possible if the client has the necessary
+caps.
+
+Clients can send simple requests to the MDS to query or request changes
+to certain metadata. The replies to these requests may also grant the
+client a certain set of caps for the inode, allowing it to perform
+subsequent requests without consulting the MDS.
+
+Clients can also request caps directly from the MDS, which is necessary
+in order to read or write file data.
+
+Distributed Locks in an MDS Cluster
+-----------------------------------
+When an MDS wants to read or change information about an inode, it must
+gather the appropriate locks for it. The MDS cluster may have a series
+of different types of locks on the given inode and each MDS may have
+disjoint sets of locks.
+
+If there are outstanding caps that would conflict with these locks, then
+they must be revoked before the lock can be acquired. Once the competing
+caps are returned to the MDS, then it can get the locks and do the
+operation.
+
+On a filesystem served by multiple MDS', the metadata cache is also
+distributed among the MDS' in the cluster. For every inode, at any given
+time, only one MDS in the cluster is considered **authoritative**. Any
+requests to change that inode must be done by the authoritative MDS,
+though non-authoritative MDS can forward requests to the authoritative
+one.
+
+Non-auth MDS' can also obtain read locks that prevent the auth MDS from
+changing the data until the lock is dropped, so that they can serve
+inode info to the clients.
+
+The auth MDS for an inode can change over time as well. The MDS' will
+actively balance responsibility for the inode cache amongst
+themselves, but this can be overridden by **pinning** certain subtrees
+to a single MDS.
diff --git a/doc/cephfs/mds-config-ref.rst b/doc/cephfs/mds-config-ref.rst
new file mode 100644
index 000000000..5b68053a0
--- /dev/null
+++ b/doc/cephfs/mds-config-ref.rst
@@ -0,0 +1,67 @@
+======================
+ MDS Config Reference
+======================
+
+.. confval:: mds_cache_mid
+.. confval:: mds_dir_max_commit_size
+.. confval:: mds_dir_max_entries
+.. confval:: mds_decay_halflife
+.. confval:: mds_beacon_interval
+.. confval:: mds_beacon_grace
+.. confval:: mon_mds_blocklist_interval
+.. confval:: mds_reconnect_timeout
+.. confval:: mds_tick_interval
+.. confval:: mds_dirstat_min_interval
+.. confval:: mds_scatter_nudge_interval
+.. confval:: mds_client_prealloc_inos
+.. confval:: mds_early_reply
+.. confval:: mds_default_dir_hash
+.. confval:: mds_log_skip_corrupt_events
+.. confval:: mds_log_max_events
+.. confval:: mds_log_max_segments
+.. confval:: mds_bal_sample_interval
+.. confval:: mds_bal_replicate_threshold
+.. confval:: mds_bal_unreplicate_threshold
+.. confval:: mds_bal_split_size
+.. confval:: mds_bal_split_rd
+.. confval:: mds_bal_split_wr
+.. confval:: mds_bal_split_bits
+.. confval:: mds_bal_merge_size
+.. confval:: mds_bal_interval
+.. confval:: mds_bal_fragment_interval
+.. confval:: mds_bal_fragment_fast_factor
+.. confval:: mds_bal_fragment_size_max
+.. confval:: mds_bal_idle_threshold
+.. confval:: mds_bal_max
+.. confval:: mds_bal_max_until
+.. confval:: mds_bal_mode
+.. confval:: mds_bal_min_rebalance
+.. confval:: mds_bal_min_start
+.. confval:: mds_bal_need_min
+.. confval:: mds_bal_need_max
+.. confval:: mds_bal_midchunk
+.. confval:: mds_bal_minchunk
+.. confval:: mds_replay_interval
+.. confval:: mds_shutdown_check
+.. confval:: mds_thrash_exports
+.. confval:: mds_thrash_fragments
+.. confval:: mds_dump_cache_on_map
+.. confval:: mds_dump_cache_after_rejoin
+.. confval:: mds_verify_scatter
+.. confval:: mds_debug_scatterstat
+.. confval:: mds_debug_frag
+.. confval:: mds_debug_auth_pins
+.. confval:: mds_debug_subtrees
+.. confval:: mds_kill_mdstable_at
+.. confval:: mds_kill_export_at
+.. confval:: mds_kill_import_at
+.. confval:: mds_kill_link_at
+.. confval:: mds_kill_rename_at
+.. confval:: mds_inject_skip_replaying_inotable
+.. confval:: mds_kill_skip_replaying_inotable
+.. confval:: mds_wipe_sessions
+.. confval:: mds_wipe_ino_prealloc
+.. confval:: mds_skip_ino
+.. confval:: mds_min_caps_per_client
+.. confval:: mds_symlink_recovery
+.. confval:: mds_extraordinary_events_dump_interval
diff --git a/doc/cephfs/mds-journaling.rst b/doc/cephfs/mds-journaling.rst
new file mode 100644
index 000000000..92b32bb45
--- /dev/null
+++ b/doc/cephfs/mds-journaling.rst
@@ -0,0 +1,90 @@
+MDS Journaling
+==============
+
+CephFS Metadata Pool
+--------------------
+
+CephFS uses a separate (metadata) pool for managing file metadata (inodes and
+dentries) in a Ceph File System. The metadata pool has all the information about
+files in a Ceph File System including the File System hierarchy. Additionally,
+CephFS maintains meta information related to other entities in a file system
+such as file system journals, open file table, session map, etc.
+
+This document describes how Ceph Metadata Servers use and rely on journaling.
+
+CephFS MDS Journaling
+---------------------
+
+CephFS metadata servers stream a journal of metadata events into RADOS in the metadata
+pool prior to executing a file system operation. Active MDS daemon(s) manage metadata
+for files and directories in CephFS.
+
+CephFS uses journaling for couple of reasons:
+
+#. Consistency: On an MDS failover, the journal events can be replayed to reach a
+   consistent file system state. Also, metadata operations that require multiple
+   updates to the backing store need to be journaled for crash consistency (along
+   with other consistency mechanisms such as locking, etc..).
+
+#. Performance: Journal updates are (mostly) sequential, hence updates to journals
+   are fast. Furthermore, updates can be batched into single write, thereby saving
+   disk seek time involved in updates to different parts of a file. Having a large
+   journal also helps a standby MDS to warm its cache which helps indirectly during
+   MDS failover.
+
+Each active metadata server maintains its own journal in the metadata pool. Journals
+are striped over multiple objects. Journal entries which are not required (deemed as
+old) are trimmed by the metadata server.
+
+Journal Events
+--------------
+
+Apart from journaling file system metadata updates, CephFS journals various other events
+such as client session info and directory import/export state to name a few. These events
+are used by the metadata sever to reestablish correct state as required, e.g., Ceph MDS
+tries to reconnect clients on restart when journal events get replayed and a specific
+event type in the journal specifies that a client entity type has a session with the MDS
+before it was restarted.
+
+To examine the list of such events recorded in the journal, CephFS provides a command
+line utility `cephfs-journal-tool` which can be used as follows:
+
+::
+
+   cephfs-journal-tool --rank=<fs>:<rank> event get list
+
+`cephfs-journal-tool` is also used to discover and repair a damaged Ceph File System.
+(See :doc:`/cephfs/cephfs-journal-tool` for more details)
+
+Journal Event Types
+-------------------
+
+Following are various event types that are journaled by the MDS.
+
+#. `EVENT_COMMITTED`: Mark a request (id) as committed.
+
+#. `EVENT_EXPORT`: Maps directories to an MDS rank.
+
+#. `EVENT_FRAGMENT`: Tracks various stages of directory fragmentation (split/merge).
+
+#. `EVENT_IMPORTSTART`: Logged when an MDS rank starts importing directory fragments.
+
+#. `EVENT_IMPORTFINISH`: Logged when an MDS rank finishes importing directory fragments.
+
+#. `EVENT_NOOP`: No operation event type for skipping over a journal region.
+
+#. `EVENT_OPEN`: Tracks which inodes have open file handles.
+
+#. `EVENT_RESETJOURNAL`: Used to mark a journal as `reset` post truncation.
+
+#. `EVENT_SESSION`: Tracks open client sessions.
+
+#. `EVENT_SLAVEUPDATE`: Logs various stages of an operation that has been forwarded to a (slave) mds.
+
+#. `EVENT_SUBTREEMAP`: Map of directory inodes to directory contents (subtree partition).
+
+#. `EVENT_TABLECLIENT`: Log transition states of MDSs view of client tables (snap/anchor).
+
+#. `EVENT_TABLESERVER`: Log transition states of MDSs view of server tables (snap/anchor).
+
+#. `EVENT_UPDATE`: Log file operations on an inode.
diff --git a/doc/cephfs/mds-state-diagram.dot b/doc/cephfs/mds-state-diagram.dot
new file mode 100644
index 000000000..8c9fa25d0
--- /dev/null
+++ b/doc/cephfs/mds-state-diagram.dot
@@ -0,0 +1,75 @@
+digraph {
+
+node [shape=circle,style=unfilled,fixedsize=true,width=2.0]
+
+node [color=blue,peripheries=1];
+N0 [label="up:boot"]
+
+node [color=green,peripheries=1];
+S1 [label="up:standby"]
+N0 -> S1 [color=green,penwidth=2.0];
+S2 [label="up:standby_replay"]
+S1 -> S2 [color=green,penwidth=2.0];
+
+node [color=orange,peripheries=2];
+N1 [label="up:creating"]
+S1 -> N1 [color=orange,penwidth=2.0];
+N2 [label="up:starting"]
+S1 -> N2 [color=orange,penwidth=2.0];
+N3 [label="up:replay"]
+S1 -> N3 [color=orange,penwidth=2.0];
+S2 -> N3 [color=orange,penwidth=2.0];
+N4 [label="up:resolve"]
+N3 -> N4 [color=orange,penwidth=2.0];
+N5 [label="up:reconnect"]
+N3 -> N5 [color=orange,penwidth=2.0];
+N4 -> N5 [color=orange,penwidth=2.0];
+N6 [label="up:rejoin"]
+N5 -> N6 [color=orange,penwidth=2.0];
+N7 [label="up:clientreplay"]
+N6 -> N7 [color=orange,penwidth=2.0];
+
+node [color=green,peripheries=2];
+S0 [label="up:active"]
+N7 -> S0 [color=green,penwidth=2.0];
+N1 -> S0 [color=green,penwidth=2.0];
+N2 -> S0 [color=green,penwidth=2.0];
+N6 -> S0 [color=green,penwidth=2.0];
+
+// going down but still accessible by clients
+node [color=purple,peripheries=2];
+S3 [label="up:stopping"]
+S0 -> S3 [color=purple,penwidth=2.0];
+
+// terminal (but "in")
+node [shape=polygon,sides=6,color=red,peripheries=2];
+D0 [label="down:failed"]
+N2 -> D0 [color=red,penwidth=2.0];
+N3 -> D0 [color=red,penwidth=2.0];
+N4 -> D0 [color=red,penwidth=2.0];
+N5 -> D0 [color=red,penwidth=2.0];
+N6 -> D0 [color=red,penwidth=2.0];
+N7 -> D0 [color=red,penwidth=2.0];
+S0 -> D0 [color=red,penwidth=2.0];
+S3 -> D0 [color=red,penwidth=2.0];
+D0 -> N3 [color=red,penwidth=2.0];
+
+// terminal (but not "in")
+node [shape=polygon,sides=6,color=black,peripheries=1];
+D1 [label="down:damaged"]
+S2 -> D1 [color=black,penwidth=2.0];
+N3 -> D1 [color=black,penwidth=2.0];
+N4 -> D1 [color=black,penwidth=2.0];
+N5 -> D1 [color=black,penwidth=2.0];
+N6 -> D1 [color=black,penwidth=2.0];
+N7 -> D1 [color=black,penwidth=2.0];
+S0 -> D1 [color=black,penwidth=2.0];
+S3 -> D1 [color=black,penwidth=2.0];
+D1 -> D0 [color=red,penwidth=2.0]
+
+node [shape=polygon,sides=6,color=purple,peripheries=1];
+D3 [label="down:stopped"]
+S3 -> D3 [color=purple,penwidth=2.0];
+N6 -> D3 [color=purple,penwidth=2.0];
+
+}
diff --git a/doc/cephfs/mds-states.rst b/doc/cephfs/mds-states.rst
new file mode 100644
index 000000000..c7da639a0
--- /dev/null
+++ b/doc/cephfs/mds-states.rst
@@ -0,0 +1,230 @@
+
+MDS States
+==========
+
+
+The Metadata Server (MDS) goes through several states during normal operation
+in CephFS. For example, some states indicate that the MDS is recovering from a
+failover by a previous instance of the MDS. Here we'll document all of these
+states and include a state diagram to visualize the transitions.
+
+State Descriptions
+------------------
+
+Common states
+~~~~~~~~~~~~~~
+
+
+::
+
+    up:active
+
+This is the normal operating state of the MDS. It indicates that the MDS
+and its rank in the file system is available.
+
+
+::
+
+    up:standby
+
+The MDS is available to takeover for a failed rank (see also :ref:`mds-standby`).
+The monitor will automatically assign an MDS in this state to a failed rank
+once available.
+
+
+::
+
+    up:standby_replay
+
+The MDS is following the journal of another ``up:active`` MDS. Should the
+active MDS fail, having a standby MDS in replay mode is desirable as the MDS is
+replaying the live journal and will more quickly takeover. A downside to having
+standby replay MDSs is that they are not available to takeover for any other
+MDS that fails, only the MDS they follow.
+
+
+Less common or transitory states
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+::
+
+    up:boot
+
+This state is broadcast to the Ceph monitors during startup. This state is
+never visible as the Monitor immediately assign the MDS to an available rank or
+commands the MDS to operate as a standby. The state is documented here for
+completeness.
+
+
+::
+
+    up:creating
+
+The MDS is creating a new rank (perhaps rank 0) by constructing some per-rank
+metadata (like the journal) and entering the MDS cluster.
+
+
+::
+
+    up:starting
+
+The MDS is restarting a stopped rank. It opens associated per-rank metadata
+and enters the MDS cluster.
+
+
+::
+
+    up:stopping
+
+When a rank is stopped, the monitors command an active MDS to enter the
+``up:stopping`` state. In this state, the MDS accepts no new client
+connections, migrates all subtrees to other ranks in the file system, flush its
+metadata journal, and, if the last rank (0), evict all clients and shutdown
+(see also :ref:`cephfs-administration`).
+
+
+::
+
+    up:replay
+
+The MDS taking over a failed rank. This state represents that the MDS is
+recovering its journal and other metadata.
+
+
+::
+
+    up:resolve
+
+The MDS enters this state from ``up:replay`` if the Ceph file system has
+multiple ranks (including this one), i.e. it's not a single active MDS cluster.
+The MDS is resolving any uncommitted inter-MDS operations. All ranks in the
+file system must be in this state or later for progress to be made, i.e. no
+rank can be failed/damaged or ``up:replay``.
+
+
+::
+
+    up:reconnect
+
+An MDS enters this state from ``up:replay`` or ``up:resolve``. This state is to
+solicit reconnections from clients. Any client which had a session with this
+rank must reconnect during this time, configurable via
+``mds_reconnect_timeout``.
+
+
+::
+
+    up:rejoin
+
+The MDS enters this state from ``up:reconnect``. In this state, the MDS is
+rejoining the MDS cluster cache. In particular, all inter-MDS locks on metadata
+are reestablished.
+
+If there are no known client requests to be replayed, the MDS directly becomes
+``up:active`` from this state.
+
+
+::
+
+    up:clientreplay
+
+The MDS may enter this state from ``up:rejoin``. The MDS is replaying any
+client requests which were replied to but not yet durable (not journaled).
+Clients resend these requests during ``up:reconnect`` and the requests are
+replayed once again. The MDS enters ``up:active`` after completing replay.
+
+
+Failed states
+~~~~~~~~~~~~~
+
+::
+
+    down:failed
+
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  0
+    ...
+
+Rank 0 is part of the failed set and is pending to be taken over by a standby
+MDS. If this state persists, it indicates no suitable MDS daemons found to be
+assigned to this rank. This may be caused by not enough standby daemons, or all
+standby daemons have incompatible compat (see also :ref:`upgrade-mds-cluster`).
+
+
+::
+
+    down:damaged
+
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  
+    damaged 0
+    ...
+
+Rank 0 has become damaged (see also :ref:`cephfs-disaster-recovery`) and placed in
+the ``damaged`` set. An MDS which was running as rank 0 found metadata damage
+that could not be automatically recovered. Operator intervention is required.
+
+
+::
+
+    down:stopped
+    
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  
+    damaged 
+    stopped 1
+    ...
+
+The rank has been stopped by reducing ``max_mds`` (see also :ref:`cephfs-multimds`).
+
+State Diagram
+-------------
+
+This state diagram shows the possible state transitions for the MDS/rank. The legend is as follows:
+
+Color
+~~~~~
+
+- Green: MDS is active.
+- Orange: MDS is in transient state trying to become active.
+- Red: MDS is indicating a state that causes the rank to be marked failed.
+- Purple: MDS and rank is stopping.
+- Black: MDS is indicating a state that causes the rank to be marked damaged.
+
+Shape
+~~~~~
+
+- Circle: an MDS holds this state.
+- Hexagon: no MDS holds this state (it is applied to the rank).
+
+Lines
+~~~~~
+
+- A double-lined shape indicates the rank is "in".
+
+.. graphviz:: mds-state-diagram.dot
diff --git a/doc/cephfs/mount-prerequisites.rst b/doc/cephfs/mount-prerequisites.rst
new file mode 100644
index 000000000..6ed8a19b6
--- /dev/null
+++ b/doc/cephfs/mount-prerequisites.rst
@@ -0,0 +1,75 @@
+Mount CephFS: Prerequisites
+===========================
+
+You can use CephFS by mounting it to your local filesystem or by using
+`cephfs-shell`_. Mounting CephFS requires superuser privileges to trim
+dentries by issuing a remount of itself. CephFS can be mounted
+`using kernel`_ as well as `using FUSE`_. Both have their own
+advantages. Read the following section to understand more about both of
+these ways to mount CephFS.
+
+For Windows CephFS mounts, please check the `ceph-dokan`_ page.
+
+Which CephFS Client?
+--------------------
+
+The FUSE client is the most accessible and the easiest to upgrade to the
+version of Ceph used by the storage cluster, while the kernel client will
+always gives better performance.
+
+When encountering bugs or performance issues, it is often instructive to
+try using the other client, in order to find out whether the bug was
+client-specific or not (and then to let the developers know).
+
+General Pre-requisite for Mounting CephFS
+-----------------------------------------
+Before mounting CephFS, ensure that the client host (where CephFS has to be
+mounted and used) has a copy of the Ceph configuration file (i.e.
+``ceph.conf``) and a keyring of the CephX user that has permission to access
+the MDS. Both of these files must already be present on the host where the
+Ceph MON resides.
+
+#. Generate a minimal conf file for the client host and place it at a
+   standard location::
+
+    # on client host
+    mkdir -p -m 755 /etc/ceph
+    ssh {user}@{mon-host} "sudo ceph config generate-minimal-conf" | sudo tee /etc/ceph/ceph.conf
+
+   Alternatively, you may copy the conf file. But the above method generates
+   a conf with minimal details which is usually sufficient. For more
+   information, see `Client Authentication`_ and :ref:`bootstrap-options`.
+
+#. Ensure that the conf has appropriate permissions::
+
+    chmod 644 /etc/ceph/ceph.conf
+
+#. Create a CephX user and get its secret key::
+
+    ssh {user}@{mon-host} "sudo ceph fs authorize cephfs client.foo / rw" | sudo tee /etc/ceph/ceph.client.foo.keyring
+
+   In above command, replace ``cephfs`` with the name of your CephFS, ``foo``
+   by the name you want for your CephX user and ``/`` by the path within your
+   CephFS for which you want to allow access to the client host and ``rw``
+   stands for both read and write permissions. Alternatively, you may copy the
+   Ceph keyring from the MON host to client host at ``/etc/ceph`` but creating
+   a keyring specific to the client host is better. While creating a CephX
+   keyring/client, using same client name across multiple machines is perfectly
+   fine.
+
+   .. note:: If you get 2 prompts for password while running above any of 2
+             above command, run ``sudo ls`` (or any other trivial command with
+             sudo) immediately before these commands.
+
+#. Ensure that the keyring has appropriate permissions::
+
+    chmod 600 /etc/ceph/ceph.client.foo.keyring
+
+.. note:: There might be few more prerequisites for kernel and FUSE mounts
+   individually, please check respective mount documents.
+
+.. _Client Authentication: ../client-auth
+.. _cephfs-shell: ../cephfs-shell
+.. _using kernel: ../mount-using-kernel-driver
+.. _using FUSE: ../mount-using-fuse
+.. _ceph-dokan: ../ceph-dokan
diff --git a/doc/cephfs/mount-using-fuse.rst b/doc/cephfs/mount-using-fuse.rst
new file mode 100644
index 000000000..f3ac054c9
--- /dev/null
+++ b/doc/cephfs/mount-using-fuse.rst
@@ -0,0 +1,103 @@
+========================
+ Mount CephFS using FUSE
+========================
+
+`ceph-fuse`_ is an alternate way of mounting CephFS, although it mounts it
+in userspace. Therefore, performance of FUSE can be relatively lower but FUSE
+clients can be more manageable, especially while upgrading CephFS.
+
+Prerequisites
+=============
+
+Go through the prerequisites required by both, kernel as well as FUSE mounts,
+in `Mount CephFS: Prerequisites`_ page.
+
+.. note:: Mounting CephFS using FUSE requires superuser privileges to trim dentries
+   by issuing a remount of itself.
+
+Synopsis
+========
+In general, the command to mount CephFS via FUSE looks like this::
+
+    ceph-fuse {mountpoint} {options}
+
+Mounting CephFS
+===============
+To FUSE-mount the Ceph file system, use the ``ceph-fuse`` command::
+
+    mkdir /mnt/mycephfs
+    ceph-fuse --id foo /mnt/mycephfs
+
+Option ``--id`` passes the name of the CephX user whose keyring we intend to
+use for mounting CephFS. In the above command, it's ``foo``. You can also use
+``-n`` instead, although ``--id`` is evidently easier::
+
+    ceph-fuse -n client.foo /mnt/mycephfs
+
+In case the keyring is not present in standard locations, you may pass it
+too::
+
+    ceph-fuse --id foo -k /path/to/keyring /mnt/mycephfs
+
+You may pass the MON's socket too, although this is not mandatory::
+
+    ceph-fuse --id foo -m 192.168.0.1:6789 /mnt/mycephfs
+
+You can also mount a specific directory within CephFS instead of mounting
+root of CephFS on your local FS::
+
+    ceph-fuse --id foo -r /path/to/dir /mnt/mycephfs
+
+If you have more than one FS on your Ceph cluster, use the option
+``--client_fs`` to mount the non-default FS::
+
+    ceph-fuse --id foo --client_fs mycephfs2 /mnt/mycephfs2
+
+You may also add a ``client_fs`` setting to your ``ceph.conf``. Alternatively, the option
+``--client_mds_namespace`` is supported for backward compatibility.
+
+Unmounting CephFS
+=================
+
+Use ``umount`` to unmount CephFS like any other FS::
+
+    umount /mnt/mycephfs
+
+.. tip:: Ensure that you are not within the file system directories before
+   executing this command.
+
+Persistent Mounts
+=================
+
+To mount CephFS as a file system in user space, add the following to ``/etc/fstab``::
+
+       #DEVICE PATH       TYPE      OPTIONS
+       none    /mnt/mycephfs  fuse.ceph ceph.id={user-ID}[,ceph.conf={path/to/conf.conf}],_netdev,defaults  0 0
+
+For example::
+
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,_netdev,defaults  0 0
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,ceph.conf=/etc/ceph/foo.conf,_netdev,defaults  0 0
+
+Ensure you use the ID (e.g., ``myuser``, not ``client.myuser``). You can pass
+any valid ``ceph-fuse`` option to the command line this way.
+
+To mount a subdirectory of the CephFS, add the following to ``/etc/fstab``::
+
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,ceph.client_mountpoint=/path/to/dir,_netdev,defaults  0 0
+
+``ceph-fuse@.service`` and ``ceph-fuse.target`` systemd units are available.
+As usual, these unit files declare the default dependencies and recommended
+execution context for ``ceph-fuse``. After making the fstab entry shown above,
+run following commands::
+
+    systemctl start ceph-fuse@/mnt/mycephfs.service
+    systemctl enable ceph-fuse.target
+    systemctl enable ceph-fuse@-mnt-mycephfs.service
+
+See :ref:`User Management <user-management>` for details on CephX user management and `ceph-fuse`_
+manual for more options it can take. For troubleshooting, see
+:ref:`ceph_fuse_debugging`.
+
+.. _ceph-fuse: ../../man/8/ceph-fuse/#options
+.. _Mount CephFS\: Prerequisites: ../mount-prerequisites
diff --git a/doc/cephfs/mount-using-kernel-driver.rst b/doc/cephfs/mount-using-kernel-driver.rst
new file mode 100644
index 000000000..9d9a4a683
--- /dev/null
+++ b/doc/cephfs/mount-using-kernel-driver.rst
@@ -0,0 +1,159 @@
+=================================
+ Mount CephFS using Kernel Driver
+=================================
+
+The CephFS kernel driver is part of the Linux kernel. It allows mounting
+CephFS as a regular file system with native kernel performance. It is the
+client of choice for most use-cases.
+
+.. note:: CephFS mount device string now uses a new (v2) syntax. The mount
+          helper (and the kernel) is backward compatible with the old syntax.
+          This means that the old syntax can still be used for mounting with
+          newer mount helpers and kernel. However, it is recommended to use
+          the new syntax whenever possible.
+
+Prerequisites
+=============
+
+Complete General Prerequisites
+------------------------------
+Go through the prerequisites required by both, kernel as well as FUSE mounts,
+in `Mount CephFS: Prerequisites`_ page.
+
+Is mount helper is present?
+---------------------------
+``mount.ceph`` helper is installed by Ceph packages. The helper passes the
+monitor address(es) and CephX user keyrings automatically saving the Ceph
+admin the effort to pass these details explicitly while mounting CephFS. In
+case the helper is not present on the client machine, CephFS can still be
+mounted using kernel but by passing these details explicitly to the ``mount``
+command. To check whether it is present on your system, do::
+
+    stat /sbin/mount.ceph
+
+Which Kernel Version?
+---------------------
+
+Because the kernel client is distributed as part of the linux kernel (not
+as part of packaged ceph releases), you will need to consider which kernel
+version to use on your client nodes. Older kernels are known to include buggy
+ceph clients, and may not support features that more recent Ceph clusters
+support.
+
+Remember that the "latest" kernel in a stable linux distribution is likely
+to be years behind the latest upstream linux kernel where Ceph development
+takes place (including bug fixes).
+
+As a rough guide, as of Ceph 10.x (Jewel), you should be using a least a 4.x
+kernel. If you absolutely have to use an older kernel, you should use the
+fuse client instead of the kernel client.
+
+This advice does not apply if you are using a linux distribution that
+includes CephFS support, as in this case the distributor will be responsible
+for backporting fixes to their stable kernel: check with your vendor.
+
+Synopsis
+========
+In general, the command to mount CephFS via kernel driver looks like this::
+
+  mount -t ceph {device-string}={path-to-mounted} {mount-point} -o {key-value-args} {other-args}
+
+Mounting CephFS
+===============
+On Ceph clusters, CephX is enabled by default. Use ``mount`` command to
+mount CephFS with the kernel driver::
+
+  mkdir /mnt/mycephfs
+  mount -t ceph <name>@<fsid>.<fs_name>=/ /mnt/mycephfs
+
+``name`` is the username of the CephX user we are using to mount CephFS.
+``fsid`` is the FSID of the ceph cluster which can be found using
+``ceph fsid`` command. ``fs_name`` is the file system to mount. The kernel
+driver requires MON's socket and the secret key for the CephX user, e.g.::
+
+  mount -t ceph cephuser@b3acfc0d-575f-41d3-9c91-0e7ed3dbb3fa.cephfs=/ -o mon_addr=192.168.0.1:6789,secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==
+
+When using the mount helper, monitor hosts and FSID are optional. ``mount.ceph``
+helper figures out these details automatically by finding and reading ceph conf
+file, .e.g::
+
+  mount -t ceph cephuser@.cephfs=/ -o secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==
+
+.. note:: Note that the dot (``.``) still needs to be a part of the device string.
+
+A potential problem with the above command is that the secret key is left in your
+shell's command history. To prevent that you can copy the secret key inside a file
+and pass the file by using the option ``secretfile`` instead of ``secret``::
+
+  mount -t ceph cephuser@.cephfs=/ /mnt/mycephfs -o secretfile=/etc/ceph/cephuser.secret
+
+Ensure the permissions on the secret key file are appropriate (preferably, ``600``).
+
+Multiple monitor hosts can be passed by separating each address with a ``/``::
+
+  mount -t ceph cephuser@.cephfs=/ /mnt/mycephfs -o mon_addr=192.168.0.1:6789/192.168.0.2:6789,secretfile=/etc/ceph/cephuser.secret
+
+In case CephX is disabled, you can omit any credential related options::
+
+  mount -t ceph cephuser@.cephfs=/ /mnt/mycephfs
+
+.. note:: The ceph user name still needs to be passed as part of the device string.
+
+To mount a subtree of the CephFS root, append the path to the device string::
+
+  mount -t ceph cephuser@.cephfs=/subvolume/dir1/dir2 /mnt/mycephfs -o secretfile=/etc/ceph/cephuser.secret
+
+Backward Compatibility
+======================
+The old syntax is supported for backward compatibility.
+
+To mount CephFS with the kernel driver::
+
+    mkdir /mnt/mycephfs
+    mount -t ceph :/ /mnt/mycephfs -o name=admin
+
+The key-value argument right after option ``-o`` is CephX credential;
+``name`` is the username of the CephX user we are using to mount CephFS.
+
+To mount a non-default FS ``cephfs2``, in case the cluster has multiple FSs::
+
+    mount -t ceph :/ /mnt/mycephfs -o name=admin,fs=cephfs2
+
+    or
+
+    mount -t ceph :/ /mnt/mycephfs -o name=admin,mds_namespace=cephfs2
+
+.. note:: The option ``mds_namespace`` is deprecated. Use ``fs=`` instead when using the old syntax for mounting.
+
+Unmounting CephFS
+=================
+To unmount the Ceph file system, use the ``umount`` command as usual::
+
+    umount /mnt/mycephfs
+
+.. tip:: Ensure that you are not within the file system directories before
+   executing this command.
+
+Persistent Mounts
+==================
+
+To mount CephFS in your file systems table as a kernel driver, add the
+following to ``/etc/fstab``::
+
+  {name}@.{fs_name}=/ {mount}/{mountpoint} ceph [mon_addr={ipaddress},secret=secretkey|secretfile=/path/to/secretfile],[{mount.options}]  {fs_freq}  {fs_passno}
+
+For example::
+
+  cephuser@.cephfs=/     /mnt/ceph    ceph    mon_addr=192.168.0.1:6789,noatime,_netdev    0       0
+
+If the ``secret`` or ``secretfile`` options are not specified then the mount helper
+will attempt to find a secret for the given ``name`` in one of the configured keyrings.
+
+See `User Management`_ for details on CephX user management and mount.ceph_
+manual for more options it can take. For troubleshooting, see
+:ref:`kernel_mount_debugging`.
+
+.. _fstab: ../fstab/#kernel-driver
+.. _Mount CephFS\: Prerequisites: ../mount-prerequisites
+.. _mount.ceph: ../../man/8/mount.ceph/
+.. _User Management: ../../rados/operations/user-management/
diff --git a/doc/cephfs/multifs.rst b/doc/cephfs/multifs.rst
new file mode 100644
index 000000000..2dcba7ae0
--- /dev/null
+++ b/doc/cephfs/multifs.rst
@@ -0,0 +1,54 @@
+.. _cephfs-multifs:
+
+Multiple Ceph File Systems
+==========================
+
+
+Beginning with the Pacific release, multiple file system support is stable
+and ready to use. This functionality allows configuring separate file systems
+with full data separation on separate pools.
+
+Existing clusters must set a flag to enable multiple file systems::
+
+    ceph fs flag set enable_multiple true
+
+New Ceph clusters automatically set this.
+
+
+Creating a new Ceph File System
+-------------------------------
+
+The new ``volumes`` plugin interface (see: :doc:`/cephfs/fs-volumes`) automates
+most of the work of configuring a new file system. The "volume" concept is
+simply a new file system. This can be done via::
+
+    ceph fs volume create <fs_name>
+
+Ceph will create the new pools and automate the deployment of new MDS to
+support the new file system. The deployment technology used, e.g. cephadm, will
+also configure the MDS affinity (see: :ref:`mds-join-fs`) of new MDS daemons to
+operate the new file system.
+
+
+Securing access
+---------------
+
+The ``fs authorize`` command allows configuring the client's access to a
+particular file system. See also in :ref:`fs-authorize-multifs`. The client will
+only have visibility of authorized file systems and the Monitors/MDS will
+reject access to clients without authorization.
+
+
+Other Notes
+-----------
+
+Multiple file systems do not share pools. This is particularly important for
+snapshots but also because no measures are in place to prevent duplicate
+inodes. The Ceph commands prevent this dangerous configuration.
+
+Each file system has its own set of MDS ranks. Consequently, each new file
+system requires more MDS daemons to operate and increases operational costs.
+This can be useful for increasing metadata throughput by application or user
+base but also adds cost to the creation of a file system. Generally, a single
+file system with subtree pinning is a better choice for isolating load between
+applications.
diff --git a/doc/cephfs/multimds.rst b/doc/cephfs/multimds.rst
new file mode 100644
index 000000000..e50a5148e
--- /dev/null
+++ b/doc/cephfs/multimds.rst
@@ -0,0 +1,286 @@
+.. _cephfs-multimds:
+
+Configuring multiple active MDS daemons
+---------------------------------------
+
+*Also known as: multi-mds, active-active MDS*
+
+Each CephFS file system is configured for a single active MDS daemon
+by default.  To scale metadata performance for large scale systems, you
+may enable multiple active MDS daemons, which will share the metadata
+workload with one another.
+
+When should I use multiple active MDS daemons?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You should configure multiple active MDS daemons when your metadata performance
+is bottlenecked on the single MDS that runs by default.
+
+Adding more daemons may not increase performance on all workloads.  Typically,
+a single application running on a single client will not benefit from an
+increased number of MDS daemons unless the application is doing a lot of
+metadata operations in parallel.
+
+Workloads that typically benefit from a larger number of active MDS daemons
+are those with many clients, perhaps working on many separate directories.
+
+
+Increasing the MDS active cluster size
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each CephFS file system has a *max_mds* setting, which controls how many ranks
+will be created.  The actual number of ranks in the file system will only be
+increased if a spare daemon is available to take on the new rank. For example,
+if there is only one MDS daemon running, and max_mds is set to two, no second
+rank will be created. (Note that such a configuration is not Highly Available
+(HA) because no standby is available to take over for a failed rank. The
+cluster will complain via health warnings when configured this way.)
+
+Set ``max_mds`` to the desired number of ranks.  In the following examples
+the "fsmap" line of "ceph status" is shown to illustrate the expected
+result of commands.
+
+::
+
+    # fsmap e5: 1/1/1 up {0=a=up:active}, 2 up:standby
+
+    ceph fs set <fs_name> max_mds 2
+
+    # fsmap e8: 2/2/2 up {0=a=up:active,1=c=up:creating}, 1 up:standby
+    # fsmap e9: 2/2/2 up {0=a=up:active,1=c=up:active}, 1 up:standby
+
+The newly created rank (1) will pass through the 'creating' state
+and then enter this 'active state'.
+
+Standby daemons
+~~~~~~~~~~~~~~~
+
+Even with multiple active MDS daemons, a highly available system **still
+requires standby daemons** to take over if any of the servers running
+an active daemon fail.
+
+Consequently, the practical maximum of ``max_mds`` for highly available systems
+is at most one less than the total number of MDS servers in your system.
+
+To remain available in the event of multiple server failures, increase the
+number of standby daemons in the system to match the number of server failures
+you wish to withstand.
+
+Decreasing the number of ranks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Reducing the number of ranks is as simple as reducing ``max_mds``:
+
+::
+    
+    # fsmap e9: 2/2/2 up {0=a=up:active,1=c=up:active}, 1 up:standby
+    ceph fs set <fs_name> max_mds 1
+    # fsmap e10: 2/2/1 up {0=a=up:active,1=c=up:stopping}, 1 up:standby
+    # fsmap e10: 2/2/1 up {0=a=up:active,1=c=up:stopping}, 1 up:standby
+    ...
+    # fsmap e10: 1/1/1 up {0=a=up:active}, 2 up:standby
+
+The cluster will automatically stop extra ranks incrementally until ``max_mds``
+is reached.
+
+See :doc:`/cephfs/administration` for more details which forms ``<role>`` can
+take.
+
+Note: stopped ranks will first enter the stopping state for a period of
+time while it hands off its share of the metadata to the remaining active
+daemons.  This phase can take from seconds to minutes.  If the MDS appears to
+be stuck in the stopping state then that should be investigated as a possible
+bug.
+
+If an MDS daemon crashes or is killed while in the ``up:stopping`` state, a
+standby will take over and the cluster monitors will against try to stop
+the daemon.
+
+When a daemon finishes stopping, it will respawn itself and go back to being a
+standby.
+
+
+.. _cephfs-pinning:
+
+Manually pinning directory trees to a particular rank
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In multiple active metadata server configurations, a balancer runs which works
+to spread metadata load evenly across the cluster. This usually works well
+enough for most users but sometimes it is desirable to override the dynamic
+balancer with explicit mappings of metadata to particular ranks. This can allow
+the administrator or users to evenly spread application load or limit impact of
+users' metadata requests on the entire cluster.
+
+The mechanism provided for this purpose is called an ``export pin``, an
+extended attribute of directories. The name of this extended attribute is
+``ceph.dir.pin``.  Users can set this attribute using standard commands:
+
+::
+
+    setfattr -n ceph.dir.pin -v 2 path/to/dir
+
+The value of the extended attribute is the rank to assign the directory subtree
+to. A default value of ``-1`` indicates the directory is not pinned.
+
+A directory's export pin is inherited from its closest parent with a set export
+pin.  In this way, setting the export pin on a directory affects all of its
+children. However, the parents pin can be overridden by setting the child
+directory's export pin. For example:
+
+::
+
+    mkdir -p a/b
+    # "a" and "a/b" both start without an export pin set
+    setfattr -n ceph.dir.pin -v 1 a/
+    # a and b are now pinned to rank 1
+    setfattr -n ceph.dir.pin -v 0 a/b
+    # a/b is now pinned to rank 0 and a/ and the rest of its children are still pinned to rank 1
+
+
+.. _cephfs-ephemeral-pinning:
+
+Setting subtree partitioning policies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is also possible to setup **automatic** static partitioning of subtrees via
+a set of **policies**. In CephFS, this automatic static partitioning is
+referred to as **ephemeral pinning**. Any directory (inode) which is
+ephemerally pinned will be automatically assigned to a particular rank
+according to a consistent hash of its inode number. The set of all
+ephemerally pinned directories should be uniformly distributed across all
+ranks.
+
+Ephemerally pinned directories are so named because the pin may not persist
+once the directory inode is dropped from cache. However, an MDS failover does
+not affect the ephemeral nature of the pinned directory. The MDS records what
+subtrees are ephemerally pinned in its journal so MDS failovers do not drop
+this information.
+
+A directory is either ephemerally pinned or not. Which rank it is pinned to is
+derived from its inode number and a consistent hash. This means that
+ephemerally pinned directories are somewhat evenly spread across the MDS
+cluster. The **consistent hash** also minimizes redistribution when the MDS
+cluster grows or shrinks. So, growing an MDS cluster may automatically increase
+your metadata throughput with no other administrative intervention.
+
+Presently, there are two types of ephemeral pinning:
+
+**Distributed Ephemeral Pins**: This policy causes a directory to fragment
+(even well below the normal fragmentation thresholds) and distribute its
+fragments as ephemerally pinned subtrees. This has the effect of distributing
+immediate children across a range of MDS ranks.  The canonical example use-case
+would be the ``/home`` directory: we want every user's home directory to be
+spread across the entire MDS cluster. This can be set via:
+
+::
+
+    setfattr -n ceph.dir.pin.distributed -v 1 /cephfs/home
+
+
+**Random Ephemeral Pins**: This policy indicates any descendent sub-directory
+may be ephemerally pinned. This is set through the extended attribute
+``ceph.dir.pin.random`` with the value set to the percentage of directories
+that should be pinned. For example:
+
+::
+
+    setfattr -n ceph.dir.pin.random -v 0.5 /cephfs/tmp
+
+Would cause any directory loaded into cache or created under ``/tmp`` to be
+ephemerally pinned 50 percent of the time.
+
+It is recommended to only set this to small values, like ``.001`` or ``0.1%``.
+Having too many subtrees may degrade performance. For this reason, the config
+``mds_export_ephemeral_random_max`` enforces a cap on the maximum of this
+percentage (default: ``.01``). The MDS returns ``EINVAL`` when attempting to
+set a value beyond this config.
+
+Both random and distributed ephemeral pin policies are off by default in
+Octopus. The features may be enabled via the
+``mds_export_ephemeral_random`` and ``mds_export_ephemeral_distributed``
+configuration options.
+
+Ephemeral pins may override parent export pins and vice versa. What determines
+which policy is followed is the rule of the closest parent: if a closer parent
+directory has a conflicting policy, use that one instead. For example:
+
+::
+
+    mkdir -p foo/bar1/baz foo/bar2
+    setfattr -n ceph.dir.pin -v 0 foo
+    setfattr -n ceph.dir.pin.distributed -v 1 foo/bar1
+
+The ``foo/bar1/baz`` directory will be ephemerally pinned because the
+``foo/bar1`` policy overrides the export pin on ``foo``. The ``foo/bar2``
+directory will obey the pin on ``foo`` normally.
+
+For the reverse situation:
+
+::
+
+    mkdir -p home/{patrick,john}
+    setfattr -n ceph.dir.pin.distributed -v 1 home
+    setfattr -n ceph.dir.pin -v 2 home/patrick
+
+The ``home/patrick`` directory and its children will be pinned to rank 2
+because its export pin overrides the policy on ``home``.
+
+To remove a partitioning policy, remove the respective extended attribute
+or set the value to 0.
+
+.. code::bash
+   $ setfattr -n ceph.dir.pin.distributed -v 0 home
+   # or
+   $ setfattr -x ceph.dir.pin.distributed home
+
+For export pins, remove the extended attribute or set the extended attribute
+value to `-1`.
+
+.. code::bash
+   $ setfattr -n ceph.dir.pin -v -1 home
+
+
+Dynamic subtree partitioning with Balancer on specific ranks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The CephFS file system provides the ``bal_rank_mask`` option to enable the balancer
+to dynamically rebalance subtrees within particular active MDS ranks. This
+allows administrators to employ both the dynamic subtree partitioning and
+static pining schemes in different active MDS ranks so that metadata loads
+are optimized based on user demand. For instance, in realistic cloud
+storage environments, where a lot of subvolumes are allotted to multiple
+computing nodes (e.g., VMs and containers), some subvolumes that require
+high performance are managed by static partitioning, whereas most subvolumes
+that experience a moderate workload are managed by the balancer. As the balancer
+evenly spreads the metadata workload to all active MDS ranks, performance of
+static pinned subvolumes inevitably may be affected or degraded. If this option
+is enabled, subtrees managed by the balancer are not affected by
+static pinned subtrees.
+
+This option can be configured with the ``ceph fs set`` command. For example:
+
+::
+
+    ceph fs set <fs_name> bal_rank_mask <hex> 
+
+Each bitfield of the ``<hex>`` number represents a dedicated rank. If the ``<hex>`` is
+set to ``0x3``, the balancer runs on active ``0`` and ``1`` ranks. For example:
+
+::
+
+    ceph fs set <fs_name> bal_rank_mask 0x3
+
+If the ``bal_rank_mask`` is set to ``-1`` or ``all``, all active ranks are masked
+and utilized by the balancer. As an example:
+
+::
+
+    ceph fs set <fs_name> bal_rank_mask -1
+
+On the other hand, if the balancer needs to be disabled,
+the ``bal_rank_mask`` should be set to ``0x0``. For example:
+
+::
+
+    ceph fs set <fs_name> bal_rank_mask 0x0
diff --git a/doc/cephfs/nfs.rst b/doc/cephfs/nfs.rst
new file mode 100644
index 000000000..75e804e20
--- /dev/null
+++ b/doc/cephfs/nfs.rst
@@ -0,0 +1,98 @@
+.. _cephfs-nfs:
+
+===
+NFS
+===
+
+CephFS namespaces can be exported over NFS protocol using the `NFS-Ganesha NFS
+server`_.  This document provides information on configuring NFS-Ganesha
+clusters manually.  The simplest and preferred way of managing NFS-Ganesha
+clusters and CephFS exports is using ``ceph nfs ...`` commands. See
+:doc:`/mgr/nfs` for more details. As the deployment is done using cephadm or
+rook.
+
+Requirements
+============
+
+-  Ceph file system
+-  ``libcephfs2``, ``nfs-ganesha`` and ``nfs-ganesha-ceph`` packages on NFS
+   server host machine.
+-  NFS-Ganesha server host connected to the Ceph public network
+
+.. note::
+   It is recommended to use 3.5 or later stable version of NFS-Ganesha
+   packages with pacific (16.2.x) or later stable version of Ceph packages.
+
+Configuring NFS-Ganesha to export CephFS
+========================================
+
+NFS-Ganesha provides a File System Abstraction Layer (FSAL) to plug in
+different storage backends. FSAL_CEPH_ is the plugin FSAL for CephFS. For
+each NFS-Ganesha export, FSAL_CEPH_ uses a libcephfs client to mount the
+CephFS path that NFS-Ganesha exports.
+
+Setting up NFS-Ganesha with CephFS, involves setting up NFS-Ganesha's and
+Ceph's configuration file and CephX access credentials for the Ceph clients
+created by NFS-Ganesha to access CephFS.
+
+NFS-Ganesha configuration
+-------------------------
+
+Here's a `sample ganesha.conf`_ configured with FSAL_CEPH_. It is suitable
+for a standalone NFS-Ganesha server, or an active/passive configuration of
+NFS-Ganesha servers, to be managed by some sort of clustering software
+(e.g., Pacemaker). Important details about the options are added as comments
+in the sample conf. There are options to do the following:
+
+- minimize Ganesha caching wherever possible since the libcephfs clients
+  (of FSAL_CEPH_) also cache aggressively
+
+- read from Ganesha config files stored in RADOS objects
+
+- store client recovery data in RADOS OMAP key-value interface
+
+- mandate NFSv4.1+ access
+
+- enable read delegations (need at least v13.0.1 ``libcephfs2`` package
+  and v2.6.0 stable ``nfs-ganesha`` and ``nfs-ganesha-ceph`` packages)
+
+.. important::
+
+   Under certain conditions, NFS access using the CephFS FSAL fails. This
+   causes an error to be thrown that reads "Input/output error". Under these
+   circumstances, the application metadata must be set for the CephFS metadata
+   and CephFS data pools. Do this by running the following command:
+
+   .. prompt:: bash $
+
+      ceph osd pool application set <cephfs_metadata_pool> cephfs <cephfs_data_pool> cephfs
+
+
+Configuration for libcephfs clients
+-----------------------------------
+
+``ceph.conf`` for libcephfs clients includes a ``[client]`` section with
+``mon_host`` option set to let the clients connect to the Ceph cluster's
+monitors, usually generated via ``ceph config generate-minimal-conf``.
+For example::
+
+    [client]
+            mon host = [v2:192.168.1.7:3300,v1:192.168.1.7:6789], [v2:192.168.1.8:3300,v1:192.168.1.8:6789], [v2:192.168.1.9:3300,v1:192.168.1.9:6789]
+
+Mount using NFSv4 clients
+=========================
+
+It is preferred to mount the NFS-Ganesha exports using NFSv4.1+ protocols
+to get the benefit of sessions.
+
+Conventions for mounting NFS resources are platform-specific. The
+following conventions work on Linux and some Unix platforms:
+
+.. code:: bash
+
+    mount -t nfs -o nfsvers=4.1,proto=tcp <ganesha-host-name>:<ganesha-pseudo-path> <mount-point>
+
+
+.. _FSAL_CEPH: https://github.com/nfs-ganesha/nfs-ganesha/tree/next/src/FSAL/FSAL_CEPH
+.. _NFS-Ganesha NFS server: https://github.com/nfs-ganesha/nfs-ganesha/wiki
+.. _sample ganesha.conf: https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/ceph.conf
diff --git a/doc/cephfs/posix.rst b/doc/cephfs/posix.rst
new file mode 100644
index 000000000..d79730c6f
--- /dev/null
+++ b/doc/cephfs/posix.rst
@@ -0,0 +1,104 @@
+========================
+ Differences from POSIX
+========================
+
+CephFS aims to adhere to POSIX semantics wherever possible.  For
+example, in contrast to many other common network file systems like
+NFS, CephFS maintains strong cache coherency across clients.  The goal
+is for processes communicating via the file system to behave the same
+when they are on different hosts as when they are on the same host.
+
+However, there are a few places where CephFS diverges from strict
+POSIX semantics for various reasons:
+
+- If a client is writing to a file and fails, its writes are not
+  necessarily atomic. That is, the client may call write(2) on a file
+  opened with O_SYNC with an 8 MB buffer and then crash and the write
+  may be only partially applied.  (Almost all file systems, even local
+  file systems, have this behavior.)
+- In shared simultaneous writer situations, a write that crosses
+  object boundaries is not necessarily atomic. This means that you
+  could have writer A write "aa|aa" and writer B write "bb|bb"
+  simultaneously (where | is the object boundary), and end up with
+  "aa|bb" rather than the proper "aa|aa" or "bb|bb".
+- Sparse files propagate incorrectly to the stat(2) st_blocks field.
+  Because CephFS does not explicitly track which parts of a file are
+  allocated/written, the st_blocks field is always populated by the
+  file size divided by the block size.  This will cause tools like
+  du(1) to overestimate consumed space.  (The recursive size field,
+  maintained by CephFS, also includes file "holes" in its count.)
+- When a file is mapped into memory via mmap(2) on multiple hosts,
+  writes are not coherently propagated to other clients' caches.  That
+  is, if a page is cached on host A, and then updated on host B, host
+  A's page is not coherently invalidated.  (Shared writable mmap
+  appears to be quite rare--we have yet to hear any complaints about this
+  behavior, and implementing cache coherency properly is complex.)
+- CephFS clients present a hidden ``.snap`` directory that is used to
+  access, create, delete, and rename snapshots.  Although the virtual
+  directory is excluded from readdir(2), any process that tries to
+  create a file or directory with the same name will get an error
+  code.  The name of this hidden directory can be changed at mount
+  time with ``-o snapdirname=.somethingelse`` (Linux) or the config
+  option ``client_snapdir`` (libcephfs, ceph-fuse).
+- CephFS does not currently maintain the ``atime`` field. Most applications
+  do not care, though this impacts some backup and data tiering
+  applications that can move unused data to a secondary storage system.
+  You may be able to workaround this for some use cases, as CephFS does
+  support setting ``atime`` via the ``setattr`` operation.
+
+Perspective
+-----------
+
+People talk a lot about "POSIX compliance," but in reality most file
+system implementations do not strictly adhere to the spec, including
+local Linux file systems like ext4 and XFS.  For example, for
+performance reasons, the atomicity requirements for reads are relaxed:
+processing reading from a file that is also being written may see torn
+results.
+
+Similarly, NFS has extremely weak consistency semantics when multiple
+clients are interacting with the same files or directories, opting
+instead for "close-to-open".  In the world of network attached
+storage, where most environments use NFS, whether or not the server's
+file system is "fully POSIX" may not be relevant, and whether client
+applications notice depends on whether data is being shared between
+clients or not.  NFS may also "tear" the results of concurrent writers
+as client data may not even be flushed to the server until the file is
+closed (and more generally writes will be significantly more
+time-shifted than CephFS, leading to less predictable results).
+
+Regardless, these are all similar enough to POSIX, and applications still work
+most of the time. Many other storage systems (e.g., HDFS) claim to be
+"POSIX-like" but diverge significantly from the standard by dropping support
+for things like in-place file modifications, truncate, or directory renames.
+
+Bottom line
+-----------
+
+CephFS relaxes more than local Linux kernel file systems (for example, writes
+spanning object boundaries may be torn).  It relaxes strictly less
+than NFS when it comes to multiclient consistency, and generally less
+than NFS when it comes to write atomicity.
+
+In other words, when it comes to POSIX, ::
+
+  HDFS < NFS < CephFS < {XFS, ext4}
+
+
+fsync() and error reporting
+---------------------------
+
+POSIX is somewhat vague about the state of an inode after fsync reports
+an error. In general, CephFS uses the standard error-reporting
+mechanisms in the client's kernel, and therefore follows the same
+conventions as other file systems.
+
+In modern Linux kernels (v4.17 or later), writeback errors are reported
+once to every file description that is open at the time of the error. In
+addition, unreported errors that occurred before the file description was
+opened will also be returned on fsync.
+
+See `PostgreSQL's summary of fsync() error reporting across operating systems
+<https://wiki.postgresql.org/wiki/Fsync_Errors>`_ and `Matthew Wilcox's
+presentation on Linux IO error handling
+<https://www.youtube.com/watch?v=74c19hwY2oE>`_ for more information.
diff --git a/doc/cephfs/quota.rst b/doc/cephfs/quota.rst
new file mode 100644
index 000000000..0bc56be12
--- /dev/null
+++ b/doc/cephfs/quota.rst
@@ -0,0 +1,111 @@
+CephFS Quotas
+=============
+
+CephFS allows quotas to be set on any directory in the file system.  The
+quota can restrict the number of *bytes* or the number of *files*
+stored beneath that point in the directory hierarchy.
+
+Like most other things in CephFS, quotas are configured using virtual
+extended attributes:
+
+ * ``ceph.quota.max_files`` -- file limit
+ * ``ceph.quota.max_bytes`` -- byte limit
+
+If the extended attributes appear on a directory that means a quota is
+configured there. If they are not present then no quota is set on that
+directory (although one may still be configured on a parent directory).
+
+To set a quota, set the extended attribute on a CephFS directory with a
+value::
+
+  setfattr -n ceph.quota.max_bytes -v 100000000 /some/dir     # 100 MB
+  setfattr -n ceph.quota.max_files -v 10000 /some/dir         # 10,000 files
+
+To view quota limit::
+
+  $ getfattr -n ceph.quota.max_bytes /some/dir
+  # file: dir1/
+  ceph.quota.max_bytes="100000000"
+  $
+  $ getfattr -n ceph.quota.max_files /some/dir
+  # file: dir1/
+  ceph.quota.max_files="10000"
+
+.. note:: Running ``getfattr /some/dir -d -m -`` for a CephFS directory will
+   print none of the CephFS extended attributes. This is because the CephFS
+   kernel and FUSE clients hide this information from the ``listxattr(2)``
+   system call. Instead, a specific CephFS extended attribute can be viewed by
+   running ``getfattr /some/dir -n ceph.<some-xattr>``.
+
+To remove a quota, set the value of extended attribute to ``0``::
+
+  $ setfattr -n ceph.quota.max_bytes -v 0 /some/dir
+  $ getfattr /some/dir -n ceph.quota.max_bytes
+  dir1/: ceph.quota.max_bytes: No such attribute
+  $
+  $ setfattr -n ceph.quota.max_files -v 0 /some/dir
+  $ getfattr dir1/ -n ceph.quota.max_files
+  dir1/: ceph.quota.max_files: No such attribute
+
+Space Usage Reporting and CephFS Quotas
+---------------------------------------
+When the root directory of the CephFS mount has quota set on it, the available
+space on the CephFS reported by space usage report tools (like ``df``) is
+based on quota limit. That is, ``available space = quota limit - used space``
+instead of ``available space = total space - used space``.
+
+This behaviour can be disabled by setting following option in client section
+of ``ceph.conf``::
+
+    client quota df = false
+
+Limitations
+-----------
+
+#. *Quotas are cooperative and non-adversarial.* CephFS quotas rely on
+   the cooperation of the client who is mounting the file system to
+   stop writers when a limit is reached.  A modified or adversarial
+   client cannot be prevented from writing as much data as it needs.
+   Quotas should not be relied on to prevent filling the system in
+   environments where the clients are fully untrusted.
+
+#. *Quotas are imprecise.* Processes that are writing to the file
+   system will be stopped a short time after the quota limit is
+   reached.  They will inevitably be allowed to write some amount of
+   data over the configured limit.  How far over the quota they are
+   able to go depends primarily on the amount of time, not the amount
+   of data.  Generally speaking writers will be stopped within 10s of
+   seconds of crossing the configured limit.
+
+#. *Quotas are implemented in the kernel client 4.17 and higher.*
+   Quotas are supported by the userspace client (libcephfs, ceph-fuse).
+   Linux kernel clients >= 4.17 support CephFS quotas but only on
+   mimic+ clusters.  Kernel clients (even recent versions) will fail
+   to handle quotas on older clusters, even if they may be able to set
+   the quotas extended attributes.
+
+#. *Quotas must be configured carefully when used with path-based
+   mount restrictions.* The client needs to have access to the
+   directory inode on which quotas are configured in order to enforce
+   them.  If the client has restricted access to a specific path
+   (e.g., ``/home/user``) based on the MDS capability, and a quota is
+   configured on an ancestor directory they do not have access to
+   (e.g., ``/home``), the client will not enforce it.  When using
+   path-based access restrictions be sure to configure the quota on
+   the directory the client is restricted too (e.g., ``/home/user``)
+   or something nested beneath it.
+
+   In case of a kernel client, it needs to have access to the parent
+   of the directory inode on which quotas are configured in order to
+   enforce them. If quota is configured on a directory path
+   (e.g., ``/home/volumes/group``), the kclient needs to have access
+   to the parent (e.g., ``/home/volumes``).
+
+   An example command to create such an user is as below::
+
+     $ ceph auth get-or-create client.guest mds 'allow r path=/home/volumes, allow rw path=/home/volumes/group' mgr 'allow rw' osd 'allow rw tag cephfs metadata=*' mon 'allow r'
+
+   See also: https://tracker.ceph.com/issues/55090
+
+#. *Snapshot file data which has since been deleted or changed does not count
+   towards the quota.* See also: http://tracker.ceph.com/issues/24284
diff --git a/doc/cephfs/recover-fs-after-mon-store-loss.rst b/doc/cephfs/recover-fs-after-mon-store-loss.rst
new file mode 100644
index 000000000..d44269c83
--- /dev/null
+++ b/doc/cephfs/recover-fs-after-mon-store-loss.rst
@@ -0,0 +1,51 @@
+Recovering the file system after catastrophic Monitor store loss
+================================================================
+
+During rare occasions, all the monitor stores of a cluster may get corrupted
+or lost. To recover the cluster in such a scenario, you need to rebuild the
+monitor stores using the OSDs (see :ref:`mon-store-recovery-using-osds`),
+and get back the pools intact (active+clean state). However, the rebuilt monitor
+stores don't restore the file system maps ("FSMap"). Additional steps are required
+to bring back the file system. The steps to recover a multiple active MDS file
+system or multiple file systems are yet to be identified. Currently, only the steps
+to recover a **single active MDS** file system with no additional file systems
+in the cluster have been identified and tested. Briefly the steps are:
+recreate the FSMap with basic defaults; and allow MDSs to recover from
+the journal/metadata stored in the filesystem's pools. The steps are described
+in more detail below.
+
+First up, recreate the file system using the recovered file system pools. The
+new FSMap will have the filesystem's default settings. However, the user defined
+file system settings such as ``standby_count_wanted``, ``required_client_features``,
+extra data pools, etc., are lost and need to be reapplied later.
+
+::
+
+    ceph fs new <fs_name> <metadata_pool> <data_pool> --force --recover
+
+The ``recover`` flag sets the state of file system's rank 0 to existing but
+failed. So when a MDS daemon eventually picks up rank 0, the daemon reads the
+existing in-RADOS metadata and doesn't overwrite it. The flag also prevents the
+standby MDS daemons to activate the file system.
+
+The file system cluster ID, fscid, of the file system will not be preserved.
+This behaviour may not be desirable for certain applications (e.g., Ceph CSI)
+that expect the file system to be unchanged across recovery. To fix this, you
+can optionally set the ``fscid`` option in the above command (see
+:ref:`advanced-cephfs-admin-settings`).
+
+Allow standby MDS daemons to join the file system.
+
+::
+
+    ceph fs set <fs_name> joinable true
+
+
+Check that the file system is no longer in degraded state and has an active
+MDS.
+
+::
+
+    ceph fs status
+
+Reapply any other custom file system settings.
diff --git a/doc/cephfs/scrub.rst b/doc/cephfs/scrub.rst
new file mode 100644
index 000000000..5b813f1c4
--- /dev/null
+++ b/doc/cephfs/scrub.rst
@@ -0,0 +1,156 @@
+.. _mds-scrub:
+
+======================
+Ceph File System Scrub
+======================
+
+CephFS provides the cluster admin (operator) to check consistency of a file system
+via a set of scrub commands. Scrub can be classified into two parts:
+
+#. Forward Scrub: In which the scrub operation starts at the root of the file system
+   (or a sub directory) and looks at everything that can be touched in the hierarchy
+   to ensure consistency.
+
+#. Backward Scrub: In which the scrub operation looks at every RADOS object in the
+   file system pools and maps it back to the file system hierarchy.
+
+This document details commands to initiate and control forward scrub (referred as
+scrub thereafter).
+
+.. warning::
+
+   CephFS forward scrubs are started and manipulated on rank 0. All scrub
+   commands must be directed at rank 0.
+
+Initiate File System Scrub
+==========================
+
+To start a scrub operation for a directory tree use the following command::
+
+   ceph tell mds.<fsname>:0 scrub start <path> [scrubopts] [tag]
+
+where ``scrubopts`` is a comma delimited list of ``recursive``, ``force``, or
+``repair`` and ``tag`` is an optional custom string tag (the default is a generated
+UUID). An example command is::
+
+   ceph tell mds.cephfs:0 scrub start / recursive
+   {
+       "return_code": 0,
+       "scrub_tag": "6f0d204c-6cfd-4300-9e02-73f382fd23c1",
+       "mode": "asynchronous"
+   }
+
+Recursive scrub is asynchronous (as hinted by `mode` in the output above).
+Asynchronous scrubs must be polled using ``scrub status`` to determine the
+status.
+
+The scrub tag is used to differentiate scrubs and also to mark each inode's
+first data object in the default data pool (where the backtrace information is
+stored) with a ``scrub_tag`` extended attribute with the value of the tag. You
+can verify an inode was scrubbed by looking at the extended attribute using the
+RADOS utilities.
+
+Scrubs work for multiple active MDS (multiple ranks). The scrub is managed by
+rank 0 and distributed across MDS as appropriate.
+
+
+Monitor (ongoing) File System Scrubs
+====================================
+
+Status of ongoing scrubs can be monitored and polled using in `scrub status`
+command. This commands lists out ongoing scrubs (identified by the tag) along
+with the path and options used to initiate the scrub::
+
+   ceph tell mds.cephfs:0 scrub status
+   {
+       "status": "scrub active (85 inodes in the stack)",
+       "scrubs": {
+           "6f0d204c-6cfd-4300-9e02-73f382fd23c1": {
+               "path": "/",
+               "options": "recursive"
+           }
+       }
+   }
+
+`status` shows the number of inodes that are scheduled to be scrubbed at any point in time,
+hence, can change on subsequent `scrub status` invocations. Also, a high level summary of
+scrub operation (which includes the operation state and paths on which scrub is triggered)
+gets displayed in `ceph status`::
+
+   ceph status
+   [...]
+
+   task status:
+     scrub status:
+         mds.0: active [paths:/]
+
+   [...]
+
+A scrub is complete when it no longer shows up in this list (although that may
+change in future releases). Any damage will be reported via cluster health warnings.
+
+Control (ongoing) File System Scrubs
+====================================
+
+- Pause: Pausing ongoing scrub operations results in no new or pending inodes being
+  scrubbed after in-flight RADOS ops (for the inodes that are currently being scrubbed)
+  finish::
+
+   ceph tell mds.cephfs:0 scrub pause
+   {
+       "return_code": 0
+   }
+
+  The ``scrub status`` after pausing reflects the paused state. At this point,
+  initiating new scrub operations (via ``scrub start``) would just queue the
+  inode for scrub::
+
+   ceph tell mds.cephfs:0 scrub status
+   {
+       "status": "PAUSED (66 inodes in the stack)",
+       "scrubs": {
+           "6f0d204c-6cfd-4300-9e02-73f382fd23c1": {
+               "path": "/",
+               "options": "recursive"
+           }
+       }
+   }
+
+- Resume: Resuming kick starts a paused scrub operation::
+
+   ceph tell mds.cephfs:0 scrub resume
+   {
+       "return_code": 0
+   }
+
+- Abort: Aborting ongoing scrub operations removes pending inodes from the scrub
+  queue (thereby aborting the scrub) after in-flight RADOS ops (for the inodes that
+  are currently being scrubbed) finish::
+
+   ceph tell mds.cephfs:0 scrub abort
+   {
+       "return_code": 0
+   }
+
+Damages
+=======
+
+The types of damage that can be reported and repaired by File System Scrub are:
+
+* DENTRY : Inode's dentry is missing.
+
+* DIR_FRAG : Inode's directory fragment(s) is missing.
+
+* BACKTRACE : Inode's backtrace in the data pool is corrupted.
+
+Evaluate strays using recursive scrub
+=====================================
+
+- In order to evaluate strays i.e. purge stray directories in ``~mdsdir`` use the following command::
+
+    ceph tell mds.<fsname>:0 scrub start ~mdsdir recursive
+
+- ``~mdsdir`` is not enqueued by default when scrubbing at the CephFS root. In order to perform stray evaluation
+  at root, run scrub with flags ``scrub_mdsdir`` and ``recursive``::
+
+    ceph tell mds.<fsname>:0 scrub start / recursive,scrub_mdsdir
diff --git a/doc/cephfs/snap-schedule.rst b/doc/cephfs/snap-schedule.rst
new file mode 100644
index 000000000..2b8873699
--- /dev/null
+++ b/doc/cephfs/snap-schedule.rst
@@ -0,0 +1,209 @@
+.. _snap-schedule:
+
+==========================
+Snapshot Scheduling Module
+==========================
+This module implements scheduled snapshots for CephFS.
+It provides a user interface to add, query and remove snapshots schedules and
+retention policies, as well as a scheduler that takes the snapshots and prunes
+existing snapshots accordingly.
+
+
+How to enable
+=============
+
+The *snap_schedule* module is enabled with::
+
+  ceph mgr module enable snap_schedule
+
+Usage
+=====
+
+This module uses :doc:`/dev/cephfs-snapshots`, please consider this documentation
+as well.
+
+This module's subcommands live under the `ceph fs snap-schedule` namespace.
+Arguments can either be supplied as positional arguments or as keyword
+arguments. Once a keyword argument was encountered, all following arguments are
+assumed to be keyword arguments too.
+
+Snapshot schedules are identified by path, their repeat interval and their start
+time. The
+repeat interval defines the time between two subsequent snapshots. It is
+specified by a number and a period multiplier, one of `h(our)`, `d(ay)` and
+`w(eek)`. E.g. a repeat interval of `12h` specifies one snapshot every 12
+hours.
+The start time is specified as a time string (more details about passing times
+below). By default
+the start time is last midnight. So when a snapshot schedule with repeat
+interval `1h` is added at 13:50
+with the default start time, the first snapshot will be taken at 14:00.
+The time zone is assumed to be UTC if none is explicitly included in the string.
+An explicit time zone will be mapped to UTC at execution.
+The start time must be in ISO8601 format. Examples below:
+
+UTC: 2022-08-08T05:30:00 i.e. 5:30 AM UTC, without explicit time zone offset
+IDT: 2022-08-08T09:00:00+03:00 i.e. 6:00 AM UTC
+EDT: 2022-08-08T05:30:00-04:00 i.e. 9:30 AM UTC
+
+Retention specifications are identified by path and the retention spec itself. A
+retention spec consists of either a number and a time period separated by a
+space or concatenated pairs of `<number><time period>`.
+The semantics are that a spec will ensure `<number>` snapshots are kept that are
+at least `<time period>` apart. For Example `7d` means the user wants to keep 7
+snapshots that are at least one day (but potentially longer) apart from each other.
+The following time periods are recognized: `h(our), d(ay), w(eek), m(onth),
+y(ear)` and `n`. The latter is a special modifier where e.g. `10n` means keep
+the last 10 snapshots regardless of timing,
+
+All subcommands take optional `fs` argument to specify paths in
+multi-fs setups and :doc:`/cephfs/fs-volumes` managed setups. If not
+passed `fs` defaults to the first file system listed in the fs_map.
+When using :doc:`/cephfs/fs-volumes` the argument `fs` is equivalent to a
+`volume`.
+
+When a timestamp is passed (the `start` argument in the `add`, `remove`,
+`activate` and `deactivate` subcommands) the ISO format `%Y-%m-%dT%H:%M:%S` will
+always be accepted. When either python3.7 or newer is used or
+https://github.com/movermeyer/backports.datetime_fromisoformat is installed, any
+valid ISO timestamp that is parsed by python's `datetime.fromisoformat` is valid.
+
+When no subcommand is supplied a synopsis is printed::
+
+  #> ceph fs snap-schedule
+  no valid command found; 8 closest matches:
+  fs snap-schedule status [<path>] [<fs>] [<format>]
+  fs snap-schedule list <path> [--recursive] [<fs>] [<format>]
+  fs snap-schedule add <path> <snap_schedule> [<start>] [<fs>]
+  fs snap-schedule remove <path> [<repeat>] [<start>] [<fs>]
+  fs snap-schedule retention add <path> <retention_spec_or_period> [<retention_count>] [<fs>]
+  fs snap-schedule retention remove <path> <retention_spec_or_period> [<retention_count>] [<fs>]
+  fs snap-schedule activate <path> [<repeat>] [<start>] [<fs>]
+  fs snap-schedule deactivate <path> [<repeat>] [<start>] [<fs>]
+  Error EINVAL: invalid command
+
+Note:
+^^^^^
+A `subvolume` argument is no longer accepted by the commands.
+
+
+Inspect snapshot schedules
+--------------------------
+
+The module offers two subcommands to inspect existing schedules: `list` and
+`status`. Bother offer plain and json output via the optional `format` argument.
+The default is plain.
+The `list` sub-command will list all schedules on a path in a short single line
+format. It offers a `recursive` argument to list all schedules in the specified
+directory and all contained directories.
+The `status` subcommand prints all available schedules and retention specs for a
+path.
+
+Examples::
+
+  ceph fs snap-schedule status /
+  ceph fs snap-schedule status /foo/bar --format=json
+  ceph fs snap-schedule list /
+  ceph fs snap-schedule list / --recursive=true # list all schedules in the tree
+
+
+Add and remove schedules
+------------------------
+The `add` and `remove` subcommands add and remove snapshots schedules
+respectively. Both require at least a `path` argument, `add` additionally
+requires a `schedule` argument as described in the USAGE section.
+
+Multiple different schedules can be added to a path. Two schedules are considered
+different from each other if they differ in their repeat interval and their
+start time.
+
+If multiple schedules have been set on a path, `remove` can remove individual
+schedules on a path by specifying the exact repeat interval and start time, or
+the subcommand can remove all schedules on a path when just a `path` is
+specified.
+
+Examples::
+
+  ceph fs snap-schedule add / 1h
+  ceph fs snap-schedule add / 1h 11:55
+  ceph fs snap-schedule add / 2h 11:55
+  ceph fs snap-schedule remove / 1h 11:55 # removes one single schedule
+  ceph fs snap-schedule remove / 1h # removes all schedules with --repeat=1h
+  ceph fs snap-schedule remove / # removes all schedules on path /
+
+Add and remove retention policies
+---------------------------------
+The `retention add` and `retention remove` subcommands allow to manage
+retention policies. One path has exactly one retention policy. A policy can
+however contain multiple count-time period pairs in order to specify complex
+retention policies.
+Retention policies can be added and removed individually or in bulk via the
+forms `ceph fs snap-schedule retention add <path> <time period> <count>` and
+`ceph fs snap-schedule retention add <path> <countTime period>[countTime period]`
+
+Examples::
+
+  ceph fs snap-schedule retention add / h 24 # keep 24 snapshots at least an hour apart
+  ceph fs snap-schedule retention add / d 7 # and 7 snapshots at least a day apart
+  ceph fs snap-schedule retention remove / h 24 # remove retention for 24 hourlies
+  ceph fs snap-schedule retention add / 24h4w # add 24 hourly and 4 weekly to retention
+  ceph fs snap-schedule retention remove / 7d4w # remove 7 daily and 4 weekly, leaves 24 hourly
+
+.. note: When adding a path to snap-schedule, remember to strip off the mount
+   point path prefix. Paths to snap-schedule should start at the appropriate
+   CephFS file system root and not at the host file system root.
+   e.g. if the Ceph File System is mounted at ``/mnt`` and the path under which
+   snapshots need to be taken is ``/mnt/some/path`` then the acutal path required
+   by snap-schedule is only ``/some/path``.
+
+.. note: It should be noted that the "created" field in the snap-schedule status
+   command output is the timestamp at which the schedule was created. The "created"
+   timestamp has nothing to do with the creation of actual snapshots. The actual
+   snapshot creation is accounted for in the "created_count" field, which is a
+   cumulative count of the total number of snapshots created so far.
+
+.. note: The maximum number of snapshots to retain per directory is limited by the
+   config tunable `mds_max_snaps_per_dir`. This tunable defaults to 100.
+   To ensure a new snapshot can be created, one snapshot less than this will be
+   retained. So by default, a maximum of 99 snapshots will be retained.
+
+.. note: The --fs argument is now required if there is more than one file system.
+
+Active and inactive schedules
+-----------------------------
+Snapshot schedules can be added for a path that doesn't exist yet in the
+directory tree. Similarly a path can be removed without affecting any snapshot
+schedules on that path.
+If a directory is not present when a snapshot is scheduled to be taken, the
+schedule will be set to inactive and will be excluded from scheduling until
+it is activated again.
+A schedule can manually be set to inactive to pause the creating of scheduled
+snapshots.
+The module provides the `activate` and `deactivate` subcommands for this
+purpose.
+
+Examples::
+
+  ceph fs snap-schedule activate / # activate all schedules on the root directory
+  ceph fs snap-schedule deactivate / 1d # deactivates daily snapshots on the root directory
+
+Limitations
+-----------
+Snapshots are scheduled using python Timers. Under normal circumstances
+specifying 1h as the schedule will result in snapshots 1 hour apart fairly
+precisely. If the mgr daemon is under heavy load however, the Timer threads
+might not get scheduled right away, resulting in a slightly delayed snapshot. If
+this happens, the next snapshot will be schedule as if the previous one was not
+delayed, i.e. one or more delayed snapshots will not cause drift in the overall
+schedule.
+
+In order to somewhat limit the overall number of snapshots in a file system, the
+module will only keep a maximum of 50 snapshots per directory. If the retention
+policy results in more then 50 retained snapshots, the retention list will be
+shortened to the newest 50 snapshots.
+
+Data storage
+------------
+The snapshot schedule data is stored in a rados object in the cephfs metadata
+pool. At runtime all data lives in a sqlite database that is serialized and
+stored as a rados object.
diff --git a/doc/cephfs/standby.rst b/doc/cephfs/standby.rst
new file mode 100644
index 000000000..367c6762b
--- /dev/null
+++ b/doc/cephfs/standby.rst
@@ -0,0 +1,187 @@
+.. _mds-standby:
+
+Terminology
+-----------
+
+A Ceph cluster may have zero or more CephFS *file systems*.  Each CephFS has
+a human readable name (set at creation time with ``fs new``) and an integer
+ID.  The ID is called the file system cluster ID, or *FSCID*.
+
+Each CephFS file system has a number of *ranks*, numbered beginning with zero.
+By default there is one rank per file system.  A rank may be thought of as a
+metadata shard.  Management of ranks is described in :doc:`/cephfs/multimds` .
+
+Each CephFS ``ceph-mds`` daemon starts without a rank.  It may be assigned one
+by the cluster's monitors. A daemon may only hold one rank at a time, and only
+give up a rank when the ``ceph-mds`` process stops.
+
+If a rank is not associated with any daemon, that rank is considered ``failed``.
+Once a rank is assigned to a daemon, the rank is considered ``up``.
+
+Each ``ceph-mds`` daemon has a *name* that is assigned statically by the
+administrator when the daemon is first configured.  Each daemon's *name* is
+typically that of the hostname where the process runs.
+
+A ``ceph-mds`` daemon may be assigned to a specific file system by
+setting its ``mds_join_fs`` configuration option to the file system's
+``name``.
+
+When a ``ceph-mds`` daemon starts, it is also assigned an integer ``GID``,
+which is unique to this current daemon's process.  In other words, when a
+``ceph-mds`` daemon is restarted, it runs as a new process and is assigned a
+*new* ``GID`` that is different from that of the previous process.
+
+Referring to MDS daemons
+------------------------
+
+Most administrative commands that refer to a ``ceph-mds`` daemon (MDS)
+accept a flexible argument format that may specify a ``rank``, a ``GID``
+or a ``name``.
+
+Where a ``rank`` is used, it  may optionally be qualified by
+a leading file system ``name`` or ``GID``.  If a daemon is a standby (i.e.
+it is not currently assigned a ``rank``), then it may only be
+referred to by ``GID`` or ``name``.
+
+For example, say we have an MDS daemon with ``name`` 'myhost' and
+``GID`` 5446, and which is assigned ``rank`` 0 for the file system 'myfs'
+with ``FSCID`` 3.  Any of the following are suitable forms of the ``fail``
+command:
+
+::
+
+    ceph mds fail 5446     # GID
+    ceph mds fail myhost   # Daemon name
+    ceph mds fail 0        # Unqualified rank
+    ceph mds fail 3:0      # FSCID and rank
+    ceph mds fail myfs:0   # File System name and rank
+
+Managing failover
+-----------------
+
+If an MDS daemon stops communicating with the cluster's monitors, the monitors
+will wait ``mds_beacon_grace`` seconds (default 15) before marking the daemon as
+*laggy*.  If a standby MDS is available, the monitor will immediately replace the
+laggy daemon.
+
+Each file system may specify a minimum number of standby daemons in order to be
+considered healthy. This number includes daemons in the ``standby-replay`` state
+waiting for a ``rank`` to fail. Note that a ``standby-replay`` daemon will not
+be assigned to take over a failure for another ``rank`` or a failure in a
+different CephFS file system). The pool of standby daemons not in ``replay``
+counts towards any file system count.
+Each file system may set the desired number of standby daemons by:
+
+::
+
+    ceph fs set <fs name> standby_count_wanted <count>
+
+Setting ``count`` to 0 will disable the health check.
+
+
+.. _mds-standby-replay:
+
+Configuring standby-replay
+--------------------------
+
+Each CephFS file system may be configured to add ``standby-replay`` daemons.
+These standby daemons follow the active MDS's metadata journal in order to
+reduce failover time in the event that the active MDS becomes unavailable. Each
+active MDS may have only one ``standby-replay`` daemon following it.
+
+Configuration of ``standby-replay`` on a file system is done using the below:
+
+::
+
+    ceph fs set <fs name> allow_standby_replay <bool>
+
+Once set, the monitors will assign available standby daemons to follow the
+active MDSs in that file system.
+
+Once an MDS has entered the ``standby-replay`` state, it will only be used as a
+standby for the ``rank`` that it is following. If another ``rank`` fails, this
+``standby-replay`` daemon will not be used as a replacement, even if no other
+standbys are available. For this reason, it is advised that if ``standby-replay``
+is used then *every* active MDS should have a ``standby-replay`` daemon.
+
+.. _mds-join-fs:
+
+Configuring MDS file system affinity
+------------------------------------
+
+You might elect to dedicate an MDS to a particular file system. Or, perhaps you
+have MDSs that run on better hardware that should be preferred over a last-resort
+standby on modest or over-provisioned systems. To configure this preference,
+CephFS provides a configuration option for MDS called ``mds_join_fs`` which
+enforces this affinity.
+
+When failing over MDS daemons, a cluster's monitors will prefer standby daemons with
+``mds_join_fs`` equal to the file system ``name`` with the failed ``rank``.  If no
+standby exists with ``mds_join_fs`` equal to the file system ``name``, it will
+choose an unqualified standby (no setting for ``mds_join_fs``) for the replacement,
+or any other available standby, as a last resort. Note, this does not change the
+behavior that ``standby-replay`` daemons are always selected before
+other standbys.
+
+Even further, the monitors will regularly examine the CephFS file systems even when
+stable to check if a standby with stronger affinity is available to replace an
+MDS with lower affinity. This process is also done for ``standby-replay`` daemons:
+if a regular standby has stronger affinity than the ``standby-replay`` MDS, it will
+replace the standby-replay MDS.
+
+For example, given this stable and healthy file system:
+
+::
+
+    $ ceph fs dump
+    dumped fsmap epoch 399
+    ...
+    Filesystem 'cephfs' (27)
+    ...
+    e399
+    max_mds 1
+    in      0
+    up      {0=20384}
+    failed
+    damaged
+    stopped
+    ...
+    [mds.a{0:20384} state up:active seq 239 addr [v2:127.0.0.1:6854/966242805,v1:127.0.0.1:6855/966242805]]
+
+    Standby daemons:
+
+    [mds.b{-1:10420} state up:standby seq 2 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]]
+
+
+You may set ``mds_join_fs`` on the standby to enforce your preference: ::
+
+    $ ceph config set mds.b mds_join_fs cephfs
+
+after automatic failover: ::
+
+    $ ceph fs dump
+    dumped fsmap epoch 405
+    e405
+    ...
+    Filesystem 'cephfs' (27)
+    ...
+    max_mds 1
+    in      0
+    up      {0=10420}
+    failed
+    damaged
+    stopped
+    ...
+    [mds.b{0:10420} state up:active seq 274 join_fscid=27 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]]
+
+    Standby daemons:
+
+    [mds.a{-1:10720} state up:standby seq 2 addr [v2:127.0.0.1:6854/1340357658,v1:127.0.0.1:6855/1340357658]]
+
+Note in the above example that ``mds.b`` now has ``join_fscid=27``. In this
+output, the file system name from ``mds_join_fs`` is changed to the file system
+identifier (27). If the file system is recreated with the same name, the
+standby will follow the new file system as expected.
+
+Finally, if the file system is degraded or undersized, no failover will occur
+to enforce ``mds_join_fs``.
diff --git a/doc/cephfs/subtree-partitioning.svg b/doc/cephfs/subtree-partitioning.svg
new file mode 100644
index 000000000..20c60de92
--- /dev/null
+++ b/doc/cephfs/subtree-partitioning.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 908.7690288713911 373.36482939632543" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l908.76904 0l0 373.36484l-908.76904 0l0 -373.36484z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l908.76904 0l0 373.36484l-908.76904 0z" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.1)"><use xlink:href="#p.1" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.1" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.1"><path fill="#ebeded" d="m259.5493 158.67822c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.5493 158.67822c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.2)"><use xlink:href="#p.2" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.2" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.2"><path fill="#717d84" d="m437.25455 5.221119c-0.43972778 0 -0.8501892 0.36909866 -0.8501892 0.7646594l0 13.396389c0 0.39556122 0.41046143 0.79112244 0.8501892 0.79112244l58.752655 0c0.44024658 0 0.87994385 -0.39556122 0.87994385 -0.79112244l0 -13.396389c0 -0.39556074 -0.43969727 -0.7646594 -0.87994385 -0.7646594l-58.752655 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m437.25455 5.221119c-0.43972778 0 -0.8501892 0.36909866 -0.8501892 0.7646594l0 13.396389c0 0.39556122 0.41046143 0.79112244 0.8501892 0.79112244l58.752655 0c0.44024658 0 0.87994385 -0.39556122 0.87994385 -0.79112244l0 -13.396389c0 -0.39556074 -0.43969727 -0.7646594 -0.87994385 -0.7646594l-58.752655 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.3)"><use xlink:href="#p.3" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.3" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.3"><path fill="#90dae3" d="m595.4631 55.41167c-0.46917725 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4395752 0.8442764 0.90875244 0.8442764l41.4339 0c0.46917725 0 0.9383545 -0.42214203 0.9383545 -0.8442764l0 -14.326885c0 -0.42214203 -0.46917725 -0.81785583 -0.9383545 -0.81785583l-41.4339 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m595.4631 55.41167c-0.46917725 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4395752 0.8442764 0.90875244 0.8442764l41.4339 0c0.46917725 0 0.9383545 -0.42214203 0.9383545 -0.8442764l0 -14.326885c0 -0.42214203 -0.46917725 -0.81785583 -0.9383545 -0.81785583l-41.4339 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m315.78055 55.438137l150.90213 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.78055 55.438137l150.90213 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m466.6484 5.2211194l149.58078 50.208202" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m466.6484 5.2211194l149.58078 50.208202" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.4)"><use xlink:href="#p.4" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.4" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.4"><path fill="#717d84" d="m295.0487 55.41167c-0.46920776 0 -0.9088135 0.3957138 -0.9088135 0.81785583l0 14.326885c0 0.4221344 0.4396057 0.8442764 0.9088135 0.8442764l41.433838 0c0.46920776 0 0.9384155 -0.42214203 0.9384155 -0.8442764l0 -14.326885c0 -0.42214203 -0.46920776 -0.81785583 -0.9384155 -0.81785583l-41.433838 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m295.0487 55.41167c-0.46920776 0 -0.9088135 0.3957138 -0.9088135 0.81785583l0 14.326885c0 0.4221344 0.4396057 0.8442764 0.9088135 0.8442764l41.433838 0c0.46920776 0 0.9384155 -0.42214203 0.9384155 -0.8442764l0 -14.326885c0 -0.42214203 -0.46920776 -0.81785583 -0.9384155 -0.81785583l-41.433838 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.5)"><use xlink:href="#p.5" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.5" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.5"><path fill="#717d84" d="m302.62457 101.419685c-0.29312134 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.5566406 0.52716064l26.135773 0c0.2928772 0 0.585968 -0.26358032 0.585968 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.585968 -0.5007782l-26.135773 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m302.62457 101.419685c-0.29312134 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.5566406 0.52716064l26.135773 0c0.2928772 0 0.585968 -0.26358032 0.585968 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.585968 -0.5007782l-26.135773 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.6)"><use xlink:href="#p.6" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.6" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.6"><path fill="#ebeded" d="m445.97543 55.41167c-0.46939087 0 -0.90893555 0.3957138 -0.90893555 0.81785583l0 14.326885c0 0.4221344 0.43954468 0.8442764 0.90893555 0.8442764l41.286926 0c0.4694214 0 0.93844604 -0.42214203 0.93844604 -0.8442764l0 -14.326885c0 -0.42214203 -0.46902466 -0.81785583 -0.93844604 -0.81785583l-41.286926 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m445.97543 55.41167c-0.46939087 0 -0.90893555 0.3957138 -0.90893555 0.81785583l0 14.326885c0 0.4221344 0.43954468 0.8442764 0.90893555 0.8442764l41.286926 0c0.4694214 0 0.93844604 -0.42214203 0.93844604 -0.8442764l0 -14.326885c0 -0.42214203 -0.46902466 -0.81785583 -0.93844604 -0.81785583l-41.286926 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.7)"><use xlink:href="#p.7" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.7" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.7"><path fill="#717d84" d="m361.4975 101.419685c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.556427 0.52716064l26.160553 0c0.2929077 0 0.5858154 -0.26358032 0.5858154 -0.52716064l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160553 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m361.4975 101.419685c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.556427 0.52716064l26.160553 0c0.2929077 0 0.5858154 -0.26358032 0.5858154 -0.52716064l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160553 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.8)"><use xlink:href="#p.8" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.8" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.8"><path fill="#90dae3" d="m542.25726 101.419685c-0.29309082 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635498 0.52716064 0.5566406 0.52716064l26.135742 0c0.2929077 0 0.58599854 -0.26358032 0.58599854 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m542.25726 101.419685c-0.29309082 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635498 0.52716064 0.5566406 0.52716064l26.135742 0c0.2929077 0 0.58599854 -0.26358032 0.58599854 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.9)"><use xlink:href="#p.9" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.9" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.9"><path fill="#ebeded" d="m302.62503 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m302.62503 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.10)"><use xlink:href="#p.10" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.10" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.10"><path fill="#ebeded" d="m602.9806 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.359985 0c0.26379395 0 0.55718994 -0.26358032 0.55718994 -0.52716064l0 -9.093201c0 -0.26358032 -0.293396 -0.5007782 -0.55718994 -0.5007782l-26.359985 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m602.9806 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.359985 0c0.26379395 0 0.55718994 -0.26358032 0.55718994 -0.52716064l0 -9.093201c0 -0.26358032 -0.293396 -0.5007782 -0.55718994 -0.5007782l-26.359985 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.11)"><use xlink:href="#p.11" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.11" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.11"><path fill="#ff4b47" d="m581.3394 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m581.3394 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.12)"><use xlink:href="#p.12" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.12" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.12"><path fill="#ebeded" d="m245.95436 101.419685c-0.29327393 0 -0.55729675 0.23719788 -0.55729675 0.5007782l0 9.093201c0 0.26358032 0.26402283 0.52716064 0.55729675 0.52716064l26.285995 0c0.2640381 0 0.557312 -0.26358032 0.557312 -0.52716064l0 -9.093201c0 -0.26358032 -0.29327393 -0.5007782 -0.557312 -0.5007782l-26.285995 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m245.95436 101.419685c-0.29327393 0 -0.55729675 0.23719788 -0.55729675 0.5007782l0 9.093201c0 0.26358032 0.26402283 0.52716064 0.55729675 0.52716064l26.285995 0c0.2640381 0 0.557312 -0.26358032 0.557312 -0.52716064l0 -9.093201c0 -0.26358032 -0.29327393 -0.5007782 -0.557312 -0.5007782l-26.285995 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.13)"><use xlink:href="#p.13" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.13" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.13"><path fill="#717d84" d="m259.5493 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.5493 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.14)"><use xlink:href="#p.14" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.14" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.14"><path fill="#ebeded" d="m541.8753 131.78285c-0.2929077 0 -0.5563965 0.2373352 -0.5563965 0.50120544l0 9.074539c0 0.23744202 0.26348877 0.5012207 0.5563965 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m541.8753 131.78285c-0.2929077 0 -0.5563965 0.2373352 -0.5563965 0.50120544l0 9.074539c0 0.23744202 0.26348877 0.5012207 0.5563965 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.15)"><use xlink:href="#p.15" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.15" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.15"><path fill="#ebeded" d="m581.3394 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m581.3394 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.16)"><use xlink:href="#p.16" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.16" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.16"><path fill="#ebeded" d="m622.8298 158.67822c-0.29309082 0 -0.5566406 0.2373352 -0.5566406 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5566406 0.5012207l26.28247 0c0.29309082 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29284668 -0.50120544 -0.5859375 -0.50120544l-26.28247 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m622.8298 158.67822c-0.29309082 0 -0.5566406 0.2373352 -0.5566406 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5566406 0.5012207l26.28247 0c0.29309082 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29284668 -0.50120544 -0.5859375 -0.50120544l-26.28247 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m315.7218 101.44615l0.09790039 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7218 101.44615l0.09790039 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m374.6243 101.44615l-58.82419 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m374.6243 101.44615l-58.82419 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m259.10968 101.44615l56.695343 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.10968 101.44615l56.695343 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m315.7218 131.80933l0.024475098 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7218 131.80933l0.024475098 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m376.35672 131.80933l-1.7617798 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m376.35672 131.80933l-1.7617798 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m315.7512 101.419685l-43.1149 30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7512 101.419685l-43.1149 30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m272.6461 158.70467l0.02444458 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m272.6461 158.70467l0.02444458 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m466.6484 55.438137l0.024475098 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m466.6484 55.438137l0.024475098 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m555.3545 101.44615l60.879578 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m555.3545 101.44615l60.879578 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m616.1655 101.44615l0.07342529 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m616.1655 101.44615l0.07342529 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m679.0027 101.975586l-62.812683 -46.590385" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m679.0027 101.975586l-62.812683 -46.590385" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m554.9728 131.80933l0.4159546 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m554.9728 131.80933l0.4159546 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m594.5543 131.80933l-39.224304 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m594.5543 131.80933l-39.224304 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m636.0151 158.70467l-41.49994 -26.890945" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m636.0151 158.70467l-41.49994 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m594.5249 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m594.5249 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m510.6638 131.80933l44.729828 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m510.6638 131.80933l44.729828 -30.39846" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.17)"><use xlink:href="#p.17" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.17" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.17"><path fill="#ebeded" d="m407.98 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.5572815 -0.2637787 0.5572815 -0.5012207l0 -9.074539c0 -0.26387024 -0.29330444 -0.50120544 -0.5572815 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m407.98 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.5572815 -0.2637787 0.5572815 -0.5012207l0 -9.074539c0 -0.26387024 -0.29330444 -0.50120544 -0.5572815 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m421.10617 131.80933l-46.516113 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m421.10617 131.80933l-46.516113 -30.39846" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.18)"><use xlink:href="#p.18" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.18" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.18"><path fill="#717d84" d="m363.2306 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m363.2306 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.19)"><use xlink:href="#p.19" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.19" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.19"><path fill="#f9afaa" d="m192.04312 101.419685c-0.29301453 0 -0.5567932 0.23719788 -0.5567932 0.5007782l0 9.093201c0 0.26358032 0.2637787 0.52716064 0.5567932 0.52716064l26.233093 0c0.2932434 0 0.58625793 -0.26358032 0.58625793 -0.52716064l0 -9.093201c0 -0.26358032 -0.29301453 -0.5007782 -0.58625793 -0.5007782l-26.233093 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m192.04312 101.419685c-0.29301453 0 -0.5567932 0.23719788 -0.5567932 0.5007782l0 9.093201c0 0.26358032 0.2637787 0.52716064 0.5567932 0.52716064l26.233093 0c0.2932434 0 0.58625793 -0.26358032 0.58625793 -0.52716064l0 -9.093201c0 -0.26358032 -0.29301453 -0.5007782 -0.58625793 -0.5007782l-26.233093 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.20)"><use xlink:href="#p.20" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.20" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.20"><path fill="#f9afaa" d="m184.55559 55.41167c-0.46939087 0 -0.9092102 0.3957138 -0.9092102 0.81785583l0 14.326885c0 0.4221344 0.43981934 0.8442764 0.9092102 0.8442764l41.384506 0c0.46903992 0 0.9384308 -0.42214203 0.9384308 -0.8442764l0 -14.326885c0 -0.42214203 -0.46939087 -0.81785583 -0.9384308 -0.81785583l-41.384506 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m184.55559 55.41167c-0.46939087 0 -0.9092102 0.3957138 -0.9092102 0.81785583l0 14.326885c0 0.4221344 0.43981934 0.8442764 0.9092102 0.8442764l41.384506 0c0.46903992 0 0.9384308 -0.42214203 0.9384308 -0.8442764l0 -14.326885c0 -0.42214203 -0.46939087 -0.81785583 -0.9384308 -0.81785583l-41.384506 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.21)"><use xlink:href="#p.21" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.21" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.21"><path fill="#ebeded" d="m736.84656 55.41167c-0.46899414 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4397583 0.8442764 0.90875244 0.8442764l41.483215 0c0.46899414 0 0.9379883 -0.42214203 0.9379883 -0.8442764l0 -14.326885c0 -0.42214203 -0.46899414 -0.81785583 -0.9379883 -0.81785583l-41.483215 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m736.84656 55.41167c-0.46899414 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4397583 0.8442764 0.90875244 0.8442764l41.483215 0c0.46899414 0 0.9379883 -0.42214203 0.9379883 -0.8442764l0 -14.326885c0 -0.42214203 -0.46899414 -0.81785583 -0.9379883 -0.81785583l-41.483215 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.22)"><use xlink:href="#p.22" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.22" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.22"><path fill="#90dae3" d="m665.8761 101.94912c-0.29315186 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.26348877 0.5271683 0.5566406 0.5271683l26.135742 0c0.2929077 0 0.58599854 -0.26358795 0.58599854 -0.5271683l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m665.8761 101.94912c-0.29315186 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.26348877 0.5271683 0.5566406 0.5271683l26.135742 0c0.2929077 0 0.58599854 -0.26358795 0.58599854 -0.5271683l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.23)"><use xlink:href="#p.23" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.23" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.23"><path fill="#ebeded" d="m192.04282 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331833 0c0.2929077 0 0.5858307 -0.2637787 0.5858307 -0.5012207l0 -9.074539c0 -0.26387024 -0.29292297 -0.50120544 -0.5858307 -0.50120544l-26.331833 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m192.04282 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331833 0c0.2929077 0 0.5858307 -0.2637787 0.5858307 -0.5012207l0 -9.074539c0 -0.26387024 -0.29292297 -0.50120544 -0.5858307 -0.50120544l-26.331833 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.24)"><use xlink:href="#p.24" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.24" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.24"><path fill="#ebeded" d="m149.17256 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.25)"><use xlink:href="#p.25" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.25" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.25"><path fill="#ebeded" d="m665.8765 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m665.8765 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.26)"><use xlink:href="#p.26" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.26" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.26"><path fill="#ebeded" d="m707.5716 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m707.5716 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.27)"><use xlink:href="#p.27" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.27" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.27"><path fill="#ebeded" d="m750.88226 158.67822c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m750.88226 158.67822c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.28)"><use xlink:href="#p.28" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.28" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.28"><path fill="#ebeded" d="m106.53792 158.67822c-0.29310608 0 -0.55708313 0.2373352 -0.55708313 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.55708313 0.5012207l26.163986 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.163986 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m106.53792 158.67822c-0.29310608 0 -0.55708313 0.2373352 -0.55708313 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.55708313 0.5012207l26.163986 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.163986 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.29)"><use xlink:href="#p.29" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.29" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.29"><path fill="#ebeded" d="m191.01506 158.67822c-0.2928772 0 -0.55644226 0.2373352 -0.55644226 0.50120544l0 9.074539c0 0.23744202 0.26356506 0.5012207 0.55644226 0.5012207l26.307266 0c0.2928772 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29312134 -0.50120544 -0.58599854 -0.50120544l-26.307266 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m191.01506 158.67822c-0.2928772 0 -0.55644226 0.2373352 -0.55644226 0.50120544l0 9.074539c0 0.23744202 0.26356506 0.5012207 0.55644226 0.5012207l26.307266 0c0.2928772 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29312134 -0.50120544 -0.58599854 -0.50120544l-26.307266 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.30)"><use xlink:href="#p.30" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.30" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.30"><path fill="#0433ff" d="m303.35846 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160553 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160553 0" fill-rule="evenodd"/><path stroke="#021ca1" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m303.35846 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160553 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160553 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.31)"><use xlink:href="#p.31" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.31" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.31"><path fill="#ebeded" d="m363.2306 158.67822c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m363.2306 158.67822c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.32)"><use xlink:href="#p.32" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.32" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.32"><path fill="#ebeded" d="m409.41852 158.67822c-0.2932129 0 -0.5567932 0.2373352 -0.5567932 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.5567932 0.5012207l26.380066 0c0.29296875 0 0.5861511 -0.2637787 0.5861511 -0.5012207l0 -9.074539c0 -0.26387024 -0.29318237 -0.50120544 -0.5861511 -0.50120544l-26.380066 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m409.41852 158.67822c-0.2932129 0 -0.5567932 0.2373352 -0.5567932 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.5567932 0.5012207l26.380066 0c0.29296875 0 0.5861511 -0.2637787 0.5861511 -0.5012207l0 -9.074539c0 -0.26387024 -0.29318237 -0.50120544 -0.5861511 -0.50120544l-26.380066 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.33)"><use xlink:href="#p.33" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.33" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.33"><path fill="#ebeded" d="m707.5716 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m707.5716 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.34)"><use xlink:href="#p.34" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.34" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.34"><path fill="#ebeded" d="m497.4783 158.67822c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m497.4783 158.67822c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.35)"><use xlink:href="#p.35" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.35" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.35"><path fill="#f9afaa" d="m149.17256 131.78285c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 131.78285c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.36)"><use xlink:href="#p.36" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.36" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.36"><path fill="#90dae3" d="m497.4783 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m497.4783 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.37)"><use xlink:href="#p.37" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.37" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.37"><path fill="#90dae3" d="m750.88226 131.78285c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m750.88226 131.78285c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.38)"><use xlink:href="#p.38" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.38" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.38"><path fill="#ebeded" d="m459.74677 158.67822c-0.29284668 0 -0.5565796 0.2373352 -0.5565796 0.50120544l0 9.074539c0 0.23744202 0.2637329 0.5012207 0.5565796 0.5012207l26.111206 0c0.2930603 0 0.5861206 -0.2637787 0.5861206 -0.5012207l0 -9.074539c0 -0.26387024 -0.2930603 -0.50120544 -0.5861206 -0.50120544l-26.111206 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m459.74677 158.67822c-0.29284668 0 -0.5565796 0.2373352 -0.5565796 0.50120544l0 9.074539c0 0.23744202 0.2637329 0.5012207 0.5565796 0.5012207l26.111206 0c0.2930603 0 0.5861206 -0.2637787 0.5861206 -0.5012207l0 -9.074539c0 -0.26387024 -0.2930603 -0.50120544 -0.5861206 -0.50120544l-26.111206 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.39)"><use xlink:href="#p.39" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.39" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.39"><path fill="#ebeded" d="m536.7074 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m536.7074 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m205.25764 55.438137l261.40546 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.25764 55.438137l261.40546 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m757.63727 55.438137l-290.98883 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m757.63727 55.438137l-290.98883 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m205.16956 101.44615l0.12234497 -46.03889" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.16956 101.44615l0.12234497 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m205.25764 131.80933l-0.09786987 -30.39846" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.25764 131.80933l-0.09786987 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m162.26999 101.975586l43.017014 -46.590385" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m162.26999 101.975586l43.017014 -46.590385" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m162.26999 131.80933l42.919144 -30.39846" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m162.26999 131.80933l42.919144 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m678.9733 131.80933l0.024475098 -29.869026" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m678.9733 131.80933l0.024475098 -29.869026" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m720.69836 131.80933l43.6532 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m720.69836 131.80933l43.6532 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m764.06775 131.80933l0.29364014 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.06775 131.80933l0.29364014 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m119.634705 158.70467l42.67444 -26.890945" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m119.634705 158.70467l42.67444 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m204.22993 158.70467l-41.96483 -26.890945" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m204.22993 158.70467l-41.96483 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m316.48526 158.70467l-43.84897 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m316.48526 158.70467l-43.84897 -26.890945" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.40)"><use xlink:href="#p.40" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.40" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.40"><path fill="#ebeded" d="m149.17256 101.94912c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.5271683 0.556427 0.5271683l26.160568 0c0.2929077 0 0.5858154 -0.26358795 0.5858154 -0.5271683l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 101.94912c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.5271683 0.556427 0.5271683l26.160568 0c0.2929077 0 0.5858154 -0.26358795 0.5858154 -0.5271683l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160568 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m422.66238 158.70467l-46.320343 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m422.66238 158.70467l-46.320343 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m549.8342 158.70467l-39.175354 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m549.8342 158.70467l-39.175354 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m472.81467 158.70467l37.87845 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m472.81467 158.70467l37.87845 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m720.69836 158.70467l43.40851 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m720.69836 158.70467l43.40851 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m804.6476 158.70467l-40.57007 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m804.6476 158.70467l-40.57007 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m764.06775 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.06775 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m376.32736 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m376.32736 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m510.6638 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m510.6638 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.41)"><use xlink:href="#p.41" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.41" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.41"><path fill="#90dae3" d="m751.14703 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.360046 0c0.26379395 0 0.5571289 -0.26358032 0.5571289 -0.52716064l0 -9.093201c0 -0.26358032 -0.29333496 -0.5007782 -0.5571289 -0.5007782l-26.360046 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m751.14703 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.360046 0c0.26379395 0 0.5571289 -0.26358032 0.5571289 -0.52716064l0 -9.093201c0 -0.26358032 -0.29333496 -0.5007782 -0.5571289 -0.5007782l-26.360046 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m764.3614 101.44615l-148.16156 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.3614 101.44615l-148.16156 -46.03889" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.42)"><use xlink:href="#p.42" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.42" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.42"><path fill="#d7dada" d="m276.6083 192.50917c-1.1743164 0 -2.3179016 1.0310669 -2.3179016 2.0888672l0 36.888687c0 1.0578003 1.1435852 2.115265 2.3179016 2.115265l107.05585 0c1.1734009 0 2.3477173 -1.0574646 2.3477173 -2.115265l0 -36.888687c0 -1.0578003 -1.1743164 -2.0888672 -2.3477173 -2.0888672l-107.05585 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m276.6083 192.50917c-1.1743164 0 -2.3179016 1.0310669 -2.3179016 2.0888672l0 36.888687c0 1.0578003 1.1435852 2.115265 2.3179016 2.115265l107.05585 0c1.1734009 0 2.3477173 -1.0574646 2.3477173 -2.115265l0 -36.888687c0 -1.0578003 -1.1743164 -2.0888672 -2.3477173 -2.0888672l-107.05585 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.43)"><use xlink:href="#p.43" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.43" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.43"><path fill="#717d84" d="m287.06165 197.2741c-0.9097595 0 -1.8187866 0.8192291 -1.8187866 1.6384583l0 28.965668c0 0.8192291 0.9090271 1.6648407 1.8187866 1.6648407l85.71365 0c0.9097595 0 1.8485718 -0.8456116 1.8485718 -1.6648407l0 -28.965668c0 -0.8192291 -0.93881226 -1.6384583 -1.8485718 -1.6384583l-85.71365 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m287.06165 197.2741c-0.9097595 0 -1.8187866 0.8192291 -1.8187866 1.6384583l0 28.965668c0 0.8192291 0.9090271 1.6648407 1.8187866 1.6648407l85.71365 0c0.9097595 0 1.8485718 -0.8456116 1.8485718 -1.6648407l0 -28.965668c0 -0.8192291 -0.93881226 -1.6384583 -1.8485718 -1.6384583l-85.71365 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m305.03363 202.62143l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 202.62143l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 206.5922l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 206.5922l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 210.53648l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 210.53648l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 214.50725l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 214.50725l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 218.45154l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 218.45154l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.44)"><use xlink:href="#p.44" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.44" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.44"><path fill="#d7dada" d="m139.9815 192.50917c-1.1738129 0 -2.3178253 1.0310669 -2.3178253 2.0888672l0 36.888687c0 1.0578003 1.1440125 2.115265 2.3178253 2.115265l107.00708 0c1.1738129 0 2.3476105 -1.0574646 2.3476105 -2.115265l0 -36.888687c0 -1.0578003 -1.1737976 -2.0888672 -2.3476105 -2.0888672l-107.00708 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m139.9815 192.50917c-1.1738129 0 -2.3178253 1.0310669 -2.3178253 2.0888672l0 36.888687c0 1.0578003 1.1440125 2.115265 2.3178253 2.115265l107.00708 0c1.1738129 0 2.3476105 -1.0574646 2.3476105 -2.115265l0 -36.888687c0 -1.0578003 -1.1737976 -2.0888672 -2.3476105 -2.0888672l-107.00708 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.45)"><use xlink:href="#p.45" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.45" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.45"><path fill="#717d84" d="m150.37743 197.2741c-0.9100189 0 -1.8200226 0.8192291 -1.8200226 1.6384583l0 28.965668c0 0.8192291 0.91000366 1.6648407 1.8200226 1.6648407l85.838806 0c0.91000366 0 1.8200073 -0.8456116 1.8200073 -1.6648407l0 -28.965668c0 -0.8192291 -0.91000366 -1.6384583 -1.8200073 -1.6384583l-85.838806 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m150.37743 197.2741c-0.9100189 0 -1.8200226 0.8192291 -1.8200226 1.6384583l0 28.965668c0 0.8192291 0.91000366 1.6648407 1.8200226 1.6648407l85.838806 0c0.91000366 0 1.8200073 -0.8456116 1.8200073 -1.6648407l0 -28.965668c0 -0.8192291 -0.91000366 -1.6384583 -1.8200073 -1.6384583l-85.838806 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m168.37753 202.62143l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 202.62143l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 206.5922l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 206.5922l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 210.53648l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 210.53648l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 214.50725l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 214.50725l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 218.45154l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 218.45154l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.46)"><use xlink:href="#p.46" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.46" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.46"><path fill="#d7dada" d="m413.99884 192.72095c-1.174469 0 -2.3182373 1.0310669 -2.3182373 2.0888672l0 36.888687c0 1.0578003 1.1437683 2.115265 2.3182373 2.115265l106.98181 0c1.173523 0 2.3480225 -1.0574646 2.3480225 -2.115265l0 -36.888687c0 -1.0578003 -1.1744995 -2.0888672 -2.3480225 -2.0888672l-106.98181 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m413.99884 192.72095c-1.174469 0 -2.3182373 1.0310669 -2.3182373 2.0888672l0 36.888687c0 1.0578003 1.1437683 2.115265 2.3182373 2.115265l106.98181 0c1.173523 0 2.3480225 -1.0574646 2.3480225 -2.115265l0 -36.888687c0 -1.0578003 -1.1744995 -2.0888672 -2.3480225 -2.0888672l-106.98181 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.47)"><use xlink:href="#p.47" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.47" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.47"><path fill="#717d84" d="m424.36423 197.45943c-0.91000366 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.90927124 1.6646271 1.8192749 1.6646271l85.810455 0c0.91000366 0 1.8491211 -0.8453827 1.8491211 -1.6646271l0 -28.988144c0 -0.81925964 -0.93911743 -1.6382294 -1.8491211 -1.6382294l-85.810455 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m424.36423 197.45943c-0.91000366 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.90927124 1.6646271 1.8192749 1.6646271l85.810455 0c0.91000366 0 1.8491211 -0.8453827 1.8491211 -1.6646271l0 -28.988144c0 -0.81925964 -0.93911743 -1.6382294 -1.8491211 -1.6382294l-85.810455 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m442.36508 202.83318l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 202.83318l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 206.7775l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 206.7775l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 210.74826l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 210.74826l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 214.69255l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 214.69255l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 218.66333l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 218.66333l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.48)"><use xlink:href="#p.48" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.48" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.48"><path fill="#d7dada" d="m551.53564 192.72095c-1.1734619 0 -2.3180542 1.0310669 -2.3180542 2.0888672l0 36.888687c0 1.0578003 1.1445923 2.115265 2.3180542 2.115265l107.15332 0c1.1744385 0 2.3479004 -1.0574646 2.3479004 -2.115265l0 -36.888687c0 -1.0578003 -1.1734619 -2.0888672 -2.3479004 -2.0888672l-107.15332 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m551.53564 192.72095c-1.1734619 0 -2.3180542 1.0310669 -2.3180542 2.0888672l0 36.888687c0 1.0578003 1.1445923 2.115265 2.3180542 2.115265l107.15332 0c1.1744385 0 2.3479004 -1.0574646 2.3479004 -2.115265l0 -36.888687c0 -1.0578003 -1.1734619 -2.0888672 -2.3479004 -2.0888672l-107.15332 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.49)"><use xlink:href="#p.49" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.49" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.49"><path fill="#717d84" d="m561.9303 197.45943c-0.90948486 0 -1.8189697 0.8189697 -1.8189697 1.6382294l0 28.988144c0 0.8192444 0.90948486 1.6646271 1.8189697 1.6646271l85.909546 0c0.90948486 0 1.8481445 -0.8453827 1.8481445 -1.6646271l0 -28.988144c0 -0.81925964 -0.93865967 -1.6382294 -1.8481445 -1.6382294l-85.909546 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m561.9303 197.45943c-0.90948486 0 -1.8189697 0.8189697 -1.8189697 1.6382294l0 28.988144c0 0.8192444 0.90948486 1.6646271 1.8189697 1.6646271l85.909546 0c0.90948486 0 1.8481445 -0.8453827 1.8481445 -1.6646271l0 -28.988144c0 -0.81925964 -0.93865967 -1.6382294 -1.8481445 -1.6382294l-85.909546 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m579.9902 202.83318l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 202.83318l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 206.7775l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 206.7775l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 210.74826l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 210.74826l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 214.69255l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 214.69255l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 218.66333l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 218.66333l66.65436 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.50)"><use xlink:href="#p.50" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.50" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.50"><path fill="#d7dada" d="m686.0774 192.72095c-1.1738281 0 -2.317871 1.0310669 -2.317871 2.0888672l0 36.888687c0 1.0578003 1.144043 2.115265 2.317871 2.115265l107.00708 0c1.1737671 0 2.3475952 -1.0574646 2.3475952 -2.115265l0 -36.888687c0 -1.0578003 -1.1738281 -2.0888672 -2.3475952 -2.0888672l-107.00708 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m686.0774 192.72095c-1.1738281 0 -2.317871 1.0310669 -2.317871 2.0888672l0 36.888687c0 1.0578003 1.144043 2.115265 2.317871 2.115265l107.00708 0c1.1737671 0 2.3475952 -1.0574646 2.3475952 -2.115265l0 -36.888687c0 -1.0578003 -1.1738281 -2.0888672 -2.3475952 -2.0888672l-107.00708 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.51)"><use xlink:href="#p.51" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.51" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.51"><path fill="#717d84" d="m696.53125 197.45943c-0.9100342 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.9092407 1.6646271 1.8192749 1.6646271l85.810486 0c0.90997314 0 1.84906 -0.8453827 1.84906 -1.6646271l0 -28.988144c0 -0.81925964 -0.9390869 -1.6382294 -1.84906 -1.6382294l-85.810486 0" fill-rule="evenodd"/><path stroke="#021ca1" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m696.53125 197.45943c-0.9100342 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.9092407 1.6646271 1.8192749 1.6646271l85.810486 0c0.90997314 0 1.84906 -0.8453827 1.84906 -1.6646271l0 -28.988144c0 -0.81925964 -0.9390869 -1.6382294 -1.84906 -1.6382294l-85.810486 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m714.50275 202.83318l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 202.83318l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 206.7775l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 206.7775l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 210.74826l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 210.74826l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 214.69255l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 214.69255l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 218.66333l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 218.66333l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m158.46217 234.2839l70.10451 0l0 32.91327l-70.10451 0z" fill-rule="evenodd"/><path fill="#000000" d="m168.8528 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113571 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m168.24342 276.61014q0 -2.359375 0.484375 -3.796875q0.484375 -1.453125 1.4375 -2.234375q0.96875 -0.78125 2.421875 -0.78125q1.078125 0 1.890625 0.4375q0.8125 0.421875 1.328125 1.25q0.53125 0.8125 0.828125 1.984375q0.3125 1.15625 0.3125 3.140625q0 2.359375 -0.484375 3.8125q-0.484375 1.4375 -1.453125 2.234375q-0.953125 0.78125 -2.421875 0.78125q-1.921875 0 -3.03125 -1.390625q-1.3125 -1.671875 -1.3125 -5.4375zm1.671875 0q0 3.296875 0.765625 4.390625q0.78125 1.078125 1.90625 1.078125q1.140625 0 1.90625 -1.09375q0.765625 -1.09375 0.765625 -4.375q0 -3.296875 -0.765625 -4.375q-0.765625 -1.078125 -1.921875 -1.078125q-1.125 0 -1.796875 0.953125q-0.859375 1.21875 -0.859375 4.5z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m295.11298 234.2839l70.10452 0l0 32.91327l-70.10452 0z" fill-rule="evenodd"/><path fill="#000000" d="m305.5036 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113556 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m311.0661 283.2039l-1.640625 0l0 -10.453125q-0.59375 0.5625 -1.5625 1.140625q-0.953125 0.5625 -1.71875 0.84375l0 -1.59375q1.375 -0.640625 2.40625 -1.5625q1.03125 -0.921875 1.453125 -1.78125l1.0625 0l0 13.40625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m437.7791 234.2839l70.10452 0l0 32.91327l-70.10452 0z" fill-rule="evenodd"/><path fill="#000000" d="m448.16974 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113586 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m456.16974 281.62576l0 1.578125l-8.828125 0q-0.015625 -0.59375 0.1875 -1.140625q0.34375 -0.90625 1.078125 -1.78125q0.75 -0.875 2.15625 -2.015625q2.171875 -1.78125 2.9375 -2.828125q0.765625 -1.046875 0.765625 -1.96875q0 -0.984375 -0.703125 -1.640625q-0.6875 -0.671875 -1.8125 -0.671875q-1.1875 0 -1.90625 0.71875q-0.703125 0.703125 -0.703125 1.953125l-1.6875 -0.171875q0.171875 -1.890625 1.296875 -2.875q1.140625 -0.984375 3.03125 -0.984375q1.921875 0 3.046875 1.0625q1.125 1.0625 1.125 2.640625q0 0.796875 -0.328125 1.578125q-0.328125 0.78125 -1.09375 1.640625q-0.75 0.84375 -2.53125 2.34375q-1.46875 1.234375 -1.890625 1.6875q-0.421875 0.4375 -0.6875 0.875l6.546875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m570.0907 234.2839l70.10455 0l0 32.91327l-70.10455 0z" fill-rule="evenodd"/><path fill="#000000" d="m580.4813 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.5877075 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113525 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m579.87195 279.67264l1.640625 -0.21875q0.28125 1.40625 0.953125 2.015625q0.6875 0.609375 1.65625 0.609375q1.15625 0 1.953125 -0.796875q0.796875 -0.796875 0.796875 -1.984375q0 -1.125 -0.734375 -1.859375q-0.734375 -0.734375 -1.875 -0.734375q-0.46875 0 -1.15625 0.171875l0.1875 -1.4375q0.15625 0.015625 0.265625 0.015625q1.046875 0 1.875 -0.546875q0.84375 -0.546875 0.84375 -1.671875q0 -0.90625 -0.609375 -1.5q-0.609375 -0.59375 -1.578125 -0.59375q-0.953125 0 -1.59375 0.609375q-0.640625 0.59375 -0.8125 1.796875l-1.640625 -0.296875q0.296875 -1.640625 1.359375 -2.546875q1.0625 -0.90625 2.65625 -0.90625q1.09375 0 2.0 0.46875q0.921875 0.46875 1.40625 1.28125q0.5 0.8125 0.5 1.71875q0 0.859375 -0.46875 1.578125q-0.46875 0.703125 -1.375 1.125q1.1875 0.28125 1.84375 1.140625q0.65625 0.859375 0.65625 2.15625q0 1.734375 -1.28125 2.953125q-1.265625 1.21875 -3.21875 1.21875q-1.765625 0 -2.921875 -1.046875q-1.15625 -1.046875 -1.328125 -2.71875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m704.5576 234.2839l70.10449 0l0 32.91327l-70.10449 0z" fill-rule="evenodd"/><path fill="#000000" d="m714.94824 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.5876465 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113586 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m719.58887 283.2039l0 -3.203125l-5.796875 0l0 -1.5l6.09375 -8.65625l1.34375 0l0 8.65625l1.796875 0l0 1.5l-1.796875 0l0 3.203125l-1.640625 0zm0 -4.703125l0 -6.015625l-4.1875 6.015625l4.1875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m27.929134 282.41995l877.6693 0l0 27.937012l-877.6693 0z" fill-rule="evenodd"/><path fill="#000000" d="m38.429134 306.77994l0 -11.453125l1.515625 0l0 11.453125l-1.515625 0zm4.0078125 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm16.40625 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.4843712 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.2968712 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.898434 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.3359375 2.484375l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm4.1171875 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5234375 0l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5 5.6875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm7.5 -5.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.5468826 0.4375 1.5000076 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.5468826 -0.390625 -2.1562576 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.5781326 -0.171875 1.2187576 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.9687576 0 -1.3906326 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.2500076 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.6250076 0 -2.4687576 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm11.625008 1.21875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm7.0546875 -1.40625l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 4.953125l0 -8.296875l1.25 0l0 1.15625q0.390625 -0.609375 1.03125 -0.96875q0.65625 -0.375 1.484375 -0.375q0.921875 0 1.515625 0.390625q0.59375 0.375 0.828125 1.0625q0.984375 -1.453125 2.5625 -1.453125q1.234375 0 1.890625 0.6875q0.671875 0.671875 0.671875 2.09375l0 5.703125l-1.390625 0l0 -5.234375q0 -0.84375 -0.140625 -1.203125q-0.140625 -0.375 -0.5 -0.59375q-0.359375 -0.234375 -0.84375 -0.234375q-0.875 0 -1.453125 0.578125q-0.578125 0.578125 -0.578125 1.859375l0 4.828125l-1.40625 0l0 -5.390625q0 -0.9375 -0.34375 -1.40625q-0.34375 -0.46875 -1.125 -0.46875q-0.59375 0 -1.09375 0.3125q-0.5 0.3125 -0.734375 0.921875q-0.21875 0.59375 -0.21875 1.71875l0 4.3125l-1.40625 0zm17.773438 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8984375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm9.2265625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8203125 4.953125l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8359375 3.203125l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm15.5703125 -4.46875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm8.1953125 4.953125l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 4.140625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3671875 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm13.3515625 4.15625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm12.9921875 2.484375l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 8.140625l0 -11.484375l1.28125 0l0 1.078125q0.453125 -0.640625 1.015625 -0.953125q0.578125 -0.3125 1.390625 -0.3125q1.0625 0 1.875 0.546875q0.8125 0.546875 1.21875 1.546875q0.421875 0.984375 0.421875 2.171875q0 1.28125 -0.46875 2.296875q-0.453125 1.015625 -1.328125 1.5625q-0.859375 0.546875 -1.828125 0.546875q-0.703125 0 -1.265625 -0.296875q-0.546875 -0.296875 -0.90625 -0.75l0 4.046875l-1.40625 0zm1.265625 -7.296875q0 1.609375 0.640625 2.375q0.65625 0.765625 1.578125 0.765625q0.9375 0 1.609375 -0.796875q0.671875 -0.796875 0.671875 -2.453125q0 -1.59375 -0.65625 -2.375q-0.65625 -0.796875 -1.5625 -0.796875q-0.890625 0 -1.59375 0.84375q-0.6875 0.84375 -0.6875 2.4375zm7.6171875 4.109375l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.234375 -0.1875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 4.953125l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm11.9609375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm11.203125 1.265625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9609375 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5390625 0l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2421875 1.90625l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm5.65625 1.78125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm0.8515625 -2.890625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.34375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm9.2265625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm18.414062 1.453125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm17.84375 4.140625l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm3.4609375 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3359375 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 4.140625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3671875 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm13.3515625 4.15625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.3359375 2.484375l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm4.1171875 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5234375 0l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.953125 2.484375l0 -1.609375l1.609375 0l0 1.609375l-1.609375 0zm11.3046875 0l0 -10.109375l-3.78125 0l0 -1.34375l9.078125 0l0 1.34375l-3.78125 0l0 10.109375l-1.515625 0zm6.6796875 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm18.414062 1.453125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875z" fill-rule="nonzero"/><path fill="#000000" d="m37.991634 328.96744l0 -11.484375l1.28125 0l0 1.078125q0.453125 -0.640625 1.015625 -0.953125q0.578125 -0.3125 1.390625 -0.3125q1.0625 0 1.875 0.546875q0.8125 0.546875 1.21875 1.546875q0.421875 0.984375 0.421875 2.171875q0 1.28125 -0.46875 2.296875q-0.453125 1.015625 -1.328125 1.5625q-0.859375 0.546875 -1.828125 0.546875q-0.703125 0 -1.265625 -0.296875q-0.546875 -0.296875 -0.90625 -0.75l0 4.046875l-1.40625 0zm1.265625 -7.296875q0 1.609375 0.640625 2.375q0.65625 0.765625 1.578125 0.765625q0.9375 0 1.609375 -0.796875q0.671875 -0.796875 0.671875 -2.453125q0 -1.59375 -0.65625 -2.375q-0.65625 -0.796875 -1.5625 -0.796875q-0.890625 0 -1.59375 0.84375q-0.6875 0.84375 -0.6875 2.4375zm13.0390625 3.078125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm8.406246 -1.265625l0.203125 1.25q-0.5937462 0.125 -1.0624962 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.4062462 0l0 1.09375l-1.4062462 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.6093712 -0.0625zm1.3828125 -8.578125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm6.6171875 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 -8.578125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.0234375 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9765625 4.15625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5546875 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm12.781258 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm8.8671875 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm18.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm9.578125 4.171875l-2.546875 -8.296875l1.453125 0l1.328125 4.78125l0.484375 1.78125q0.03125 -0.125 0.4375 -1.703125l1.3125 -4.859375l1.453125 0l1.234375 4.8125l0.421875 1.578125l0.46875 -1.59375l1.421875 -4.796875l1.375 0l-2.59375 8.296875l-1.46875 0l-1.3125 -4.96875l-0.328125 -1.421875l-1.671875 6.390625l-1.46875 0zm15.4296875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5390625 7.375l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm15.5703125 -4.46875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm6.6640625 2.90625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm8.890625 0l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.28125 4.953125l0 -8.296875l1.25 0l0 1.15625q0.390625 -0.609375 1.03125 -0.96875q0.65625 -0.375 1.484375 -0.375q0.921875 0 1.515625 0.390625q0.59375 0.375 0.828125 1.0625q0.984375 -1.453125 2.5625 -1.453125q1.234375 0 1.890625 0.6875q0.671875 0.671875 0.671875 2.09375l0 5.703125l-1.390625 0l0 -5.234375q0 -0.84375 -0.140625 -1.203125q-0.140625 -0.375 -0.5 -0.59375q-0.359375 -0.234375 -0.84375 -0.234375q-0.875 0 -1.453125 0.578125q-0.578125 0.578125 -0.578125 1.859375l0 4.828125l-1.40625 0l0 -5.390625q0 -0.9375 -0.34375 -1.40625q-0.34375 -0.46875 -1.125 -0.46875q-0.59375 0 -1.09375 0.3125q-0.5 0.3125 -0.734375 0.921875q-0.21875 0.59375 -0.21875 1.71875l0 4.3125l-1.40625 0zm19.0 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm10.8984375 3.6875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm6.7890625 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.3671875 3.109375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm6.6640625 2.90625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm6.7890625 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.375 4.171875l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm3.5859375 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm12.8515625 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.015625 5.640625l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9765625 4.296875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.7734375 8.15625l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm11.9453125 -5.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.28125 -4.890625l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.0078125 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9296875 4.140625l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.703125 4.140625l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm15.0703125 -1.96875q0 -2.03125 0.40625 -3.265625q0.421875 -1.234375 1.25 -1.90625q0.828125 -0.671875 2.078125 -0.671875q0.921875 0 1.609375 0.375q0.703125 0.359375 1.15625 1.0625q0.453125 0.703125 0.703125 1.703125q0.265625 1.0 0.265625 2.703125q0 2.015625 -0.421875 3.265625q-0.40625 1.234375 -1.234375 1.921875q-0.828125 0.671875 -2.078125 0.671875q-1.65625 0 -2.609375 -1.203125q-1.125 -1.421875 -1.125 -4.65625zm1.4375 0q0 2.828125 0.65625 3.765625q0.671875 0.921875 1.640625 0.921875q0.96875 0 1.625 -0.9375q0.65625 -0.9375 0.65625 -3.75q0 -2.828125 -0.65625 -3.75q-0.65625 -0.9375 -1.640625 -0.9375q-0.96875 0 -1.546875 0.828125q-0.734375 1.046875 -0.734375 3.859375zm8.2109375 5.65625l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.25 4.953125l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm3.5859375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.2890625 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9921875 4.296875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm11.9609375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm7.125 1.265625l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5859375 4.21875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.0234375 0l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm9.1328125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm20.367188 3.6875l-1.40625 0l0 -8.96875q-0.515625 0.484375 -1.34375 0.96875q-0.8125 0.484375 -1.46875 0.734375l0 -1.359375q1.171875 -0.5625 2.046875 -1.34375q0.890625 -0.796875 1.265625 -1.53125l0.90625 0l0 11.5zm4.3515625 0l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0z" fill-rule="nonzero"/><path fill="#000000" d="m41.054134 343.5143l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.749996 0.796875q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.6328125 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm13.6640625 1.625l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.000008 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.6250076 -0.28125 -0.9375076 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.4062576 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm22.460938 2.328125l0 1.359375l-7.578125 0q-0.015625 -0.515625 0.171875 -0.984375q0.28125 -0.765625 0.921875 -1.515625q0.640625 -0.75 1.84375 -1.734375q1.859375 -1.53125 2.515625 -2.421875q0.65625 -0.90625 0.65625 -1.703125q0 -0.828125 -0.59375 -1.40625q-0.59375 -0.578125 -1.5625 -0.578125q-1.015625 0 -1.625 0.609375q-0.609375 0.609375 -0.609375 1.6875l-1.453125 -0.140625q0.15625 -1.625 1.125 -2.46875q0.96875 -0.84375 2.59375 -0.84375q1.65625 0 2.609375 0.921875q0.96875 0.90625 0.96875 2.25q0 0.6875 -0.28125 1.359375q-0.28125 0.65625 -0.9375 1.390625q-0.65625 0.734375 -2.171875 2.015625q-1.265625 1.0625 -1.625 1.453125q-0.359375 0.375 -0.59375 0.75l5.625 0zm2.2578125 1.359375l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.265625 4.953125l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm11.84375 1.65625l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm15.0703125 0.65625l1.40625 -0.1875q0.25 1.203125 0.828125 1.734375q0.578125 0.515625 1.421875 0.515625q0.984375 0 1.671875 -0.6875q0.6875 -0.6875 0.6875 -1.703125q0 -0.96875 -0.640625 -1.59375q-0.625 -0.625 -1.609375 -0.625q-0.390625 0 -0.984375 0.15625l0.15625 -1.234375q0.140625 0.015625 0.21875 0.015625q0.90625 0 1.625 -0.46875q0.71875 -0.46875 0.71875 -1.453125q0 -0.765625 -0.53125 -1.265625q-0.515625 -0.515625 -1.34375 -0.515625q-0.828125 0 -1.375 0.515625q-0.546875 0.515625 -0.703125 1.546875l-1.40625 -0.25q0.265625 -1.421875 1.171875 -2.1875q0.921875 -0.78125 2.28125 -0.78125q0.9375 0 1.71875 0.40625q0.796875 0.390625 1.203125 1.09375q0.421875 0.6875 0.421875 1.46875q0 0.75 -0.40625 1.359375q-0.390625 0.609375 -1.171875 0.96875q1.015625 0.234375 1.578125 0.96875q0.5625 0.734375 0.5625 1.84375q0 1.5 -1.09375 2.546875q-1.09375 1.046875 -2.765625 1.046875q-1.5 0 -2.5 -0.890625q-1.0 -0.90625 -1.140625 -2.34375zm19.140625 2.0q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm15.46875 2.875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm17.65625 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.3671875 3.109375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.34375 0l0 -11.453125l1.40625 0l0 6.53125l3.328125 -3.375l1.828125 0l-3.171875 3.078125l3.484375 5.21875l-1.734375 0l-2.734375 -4.25l-1.0 0.953125l0 3.296875l-1.40625 0zm13.7421875 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5859375 4.21875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.0234375 0l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm9.1328125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm10.5234375 8.328125q-1.171875 -1.46875 -1.984375 -3.4375q-0.796875 -1.984375 -0.796875 -4.09375q0 -1.859375 0.609375 -3.5625q0.703125 -1.96875 2.171875 -3.9375l1.0 0q-0.9375 1.625 -1.25 2.328125q-0.46875 1.078125 -0.75 2.25q-0.328125 1.453125 -0.328125 2.9375q0 3.75 2.328125 7.515625l-1.0 0zm8.6171875 -3.375l-2.546875 -8.296875l1.453125 0l1.328125 4.78125l0.484375 1.78125q0.03125 -0.125 0.4375 -1.703125l1.3125 -4.859375l1.453125 0l1.234375 4.8125l0.421875 1.578125l0.46875 -1.59375l1.421875 -4.796875l1.375 0l-2.59375 8.296875l-1.46875 0l-1.3125 -4.96875l-0.328125 -1.421875l-1.671875 6.390625l-1.46875 0zm10.0234375 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8984375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm8.9609375 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm13.34375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm12.9921875 -7.34375l0 -1.625l1.40625 0l0 1.625l-1.40625 0zm-1.78125 13.046875l0.265625 -1.1875q0.421875 0.109375 0.671875 0.109375q0.421875 0 0.625 -0.296875q0.21875 -0.28125 0.21875 -1.421875l0 -8.71875l1.40625 0l0 8.75q0 1.53125 -0.390625 2.140625q-0.515625 0.78125 -1.6875 0.78125q-0.578125 0 -1.109375 -0.15625zm10.7890625 -3.21875l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm2.8984375 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm11.625 1.21875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm11.234375 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm7.484375 1.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 -7.359375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5546875 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.6328125 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9609375 4.296875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm17.65625 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9609375 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5390625 0l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2421875 1.90625l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm5.65625 1.78125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm0.8515625 -2.890625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.28125 3.203125l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm8.984375 0.171875l-1.015625 0q2.34375 -3.765625 2.34375 -7.515625q0 -1.46875 -0.34375 -2.921875q-0.265625 -1.171875 -0.734375 -2.25q-0.3125 -0.703125 -1.265625 -2.34375l1.015625 0q1.46875 1.96875 2.171875 3.9375q0.59375 1.703125 0.59375 3.5625q0 2.109375 -0.8125 4.09375q-0.796875 1.96875 -1.953125 3.4375zm10.1484375 -3.375l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875z" fill-rule="nonzero"/><path fill="#000000" d="m38.116634 366.33994l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.093746185 0.859375 0.4687462 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8124962 -0.4375 -1.2187462 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4374962 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.0468712 -0.46875 -1.6562462 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm19.570309 3.6875l0 -2.75l-4.96875 0l0 -1.28125l5.234375 -7.421875l1.140625 0l0 7.421875l1.546875 0l0 1.28125l-1.546875 0l0 2.75l-1.40625 0zm0 -4.03125l0 -5.171875l-3.578125 5.171875l3.578125 0zm5.1796875 4.03125l0 -1.609375l1.609375 0l0 1.609375l-1.609375 0z" fill-rule="nonzero"/></g></svg>
\ No newline at end of file
diff --git a/doc/cephfs/troubleshooting.rst b/doc/cephfs/troubleshooting.rst
new file mode 100644
index 000000000..0e511526b
--- /dev/null
+++ b/doc/cephfs/troubleshooting.rst
@@ -0,0 +1,429 @@
+=================
+ Troubleshooting
+=================
+
+Slow/stuck operations
+=====================
+
+If you are experiencing apparent hung operations, the first task is to identify
+where the problem is occurring: in the client, the MDS, or the network connecting
+them. Start by looking to see if either side has stuck operations
+(:ref:`slow_requests`, below), and narrow it down from there.
+
+We can get hints about what's going on by dumping the MDS cache ::
+
+  ceph daemon mds.<name> dump cache /tmp/dump.txt
+
+.. note:: The file `dump.txt` is on the machine executing the MDS and for systemd
+	  controlled MDS services, this is in a tmpfs in the MDS container.
+	  Use `nsenter(1)` to locate `dump.txt` or specify another system-wide path.
+
+If high logging levels are set on the MDS, that will almost certainly hold the
+information we need to diagnose and solve the issue.
+
+Stuck during recovery
+=====================
+
+Stuck in up:replay
+------------------
+
+If your MDS is stuck in ``up:replay`` then it is likely that the journal is
+very long. Did you see ``MDS_HEALTH_TRIM`` cluster warnings saying the MDS is
+behind on trimming its journal? If the journal has grown very large, it can
+take hours to read the journal. There is no working around this but there
+are things you can do to speed things along:
+
+Reduce MDS debugging to 0. Even at the default settings, the MDS logs some
+messages to memory for dumping if a fatal error is encountered. You can avoid
+this:
+
+.. code:: bash
+
+   ceph config set mds debug_mds 0
+   ceph config set mds debug_ms 0
+   ceph config set mds debug_monc 0
+
+Note if the MDS fails then there will be virtually no information to determine
+why. If you can calculate when ``up:replay`` will complete, you should restore
+these configs just prior to entering the next state:
+
+.. code:: bash
+
+   ceph config rm mds debug_mds
+   ceph config rm mds debug_ms
+   ceph config rm mds debug_monc
+
+Once you've got replay moving along faster, you can calculate when the MDS will
+complete. This is done by examining the journal replay status:
+
+.. code:: bash
+
+   $ ceph tell mds.<fs_name>:0 status | jq .replay_status
+   {
+     "journal_read_pos": 4195244,
+     "journal_write_pos": 4195244,
+     "journal_expire_pos": 4194304,
+     "num_events": 2,
+     "num_segments": 2
+   }
+
+Replay completes when the ``journal_read_pos`` reaches the
+``journal_write_pos``. The write position will not change during replay. Track
+the progression of the read position to compute the expected time to complete.
+
+
+Avoiding recovery roadblocks
+----------------------------
+
+When trying to urgently restore your file system during an outage, here are some
+things to do:
+
+* **Deny all reconnect to clients.** This effectively blocklists all existing
+  CephFS sessions so all mounts will hang or become unavailable.
+
+.. code:: bash
+
+   ceph config set mds mds_deny_all_reconnect true
+
+  Remember to undo this after the MDS becomes active.
+
+.. note:: This does not prevent new sessions from connecting. For that, see the ``refuse_client_session`` file system setting.
+
+* **Extend the MDS heartbeat grace period**. This avoids replacing an MDS that appears
+  "stuck" doing some operation. Sometimes recovery of an MDS may involve an
+  operation that may take longer than expected (from the programmer's
+  perspective). This is more likely when recovery is already taking a longer than
+  normal amount of time to complete (indicated by your reading this document).
+  Avoid unnecessary replacement loops by extending the heartbeat graceperiod:
+
+.. code:: bash
+
+   ceph config set mds mds_heartbeat_grace 3600
+
+  This has the effect of having the MDS continue to send beacons to the monitors
+  even when its internal "heartbeat" mechanism has not been reset (beat) in one
+  hour. Note the previous mechanism for achieving this was via the
+  `mds_beacon_grace` monitor setting.
+
+* **Disable open file table prefetch.** Normally, the MDS will prefetch
+  directory contents during recovery to heat up its cache. During long
+  recovery, the cache is probably already hot **and large**. So this behavior
+  can be undesirable. Disable using:
+
+.. code:: bash
+
+   ceph config set mds mds_oft_prefetch_dirfrags false
+
+* **Turn off clients.** Clients reconnecting to the newly ``up:active`` MDS may
+  cause new load on the file system when it's just getting back on its feet.
+  There will likely be some general maintenance to do before workloads should be
+  resumed. For example, expediting journal trim may be advisable if the recovery
+  took a long time because replay was reading a overly large journal.
+
+  You can do this manually or use the new file system tunable:
+
+.. code:: bash
+
+   ceph fs set <fs_name> refuse_client_session true
+
+  That prevents any clients from establishing new sessions with the MDS.
+
+
+
+Expediting MDS journal trim
+===========================
+
+If your MDS journal grew too large (maybe your MDS was stuck in up:replay for a
+long time!), you will want to have the MDS trim its journal more frequently.
+You will know the journal is too large because of ``MDS_HEALTH_TRIM`` warnings.
+
+The main tunable available to do this is to modify the MDS tick interval. The
+"tick" interval drives several upkeep activities in the MDS. It is strongly
+recommended no significant file system load be present when modifying this tick
+interval. This setting only affects an MDS in ``up:active``. The MDS does not
+trim its journal during recovery.
+
+.. code:: bash
+
+   ceph config set mds mds_tick_interval 2
+
+
+RADOS Health
+============
+
+If part of the CephFS metadata or data pools is unavailable and CephFS is not
+responding, it is probably because RADOS itself is unhealthy. Resolve those
+problems first (:doc:`../../rados/troubleshooting/index`).
+
+The MDS
+=======
+
+If an operation is hung inside the MDS, it will eventually show up in ``ceph health``,
+identifying "slow requests are blocked". It may also identify clients as
+"failing to respond" or misbehaving in other ways. If the MDS identifies
+specific clients as misbehaving, you should investigate why they are doing so.
+
+Generally it will be the result of
+
+#. Overloading the system (if you have extra RAM, increase the
+   "mds cache memory limit" config from its default 1GiB; having a larger active
+   file set than your MDS cache is the #1 cause of this!).
+
+#. Running an older (misbehaving) client.
+
+#. Underlying RADOS issues.
+
+Otherwise, you have probably discovered a new bug and should report it to
+the developers!
+
+.. _slow_requests:
+
+Slow requests (MDS)
+-------------------
+You can list current operations via the admin socket by running::
+
+  ceph daemon mds.<name> dump_ops_in_flight
+
+from the MDS host. Identify the stuck commands and examine why they are stuck.
+Usually the last "event" will have been an attempt to gather locks, or sending
+the operation off to the MDS log. If it is waiting on the OSDs, fix them. If
+operations are stuck on a specific inode, you probably have a client holding
+caps which prevent others from using it, either because the client is trying
+to flush out dirty data or because you have encountered a bug in CephFS'
+distributed file lock code (the file "capabilities" ["caps"] system).
+
+If it's a result of a bug in the capabilities code, restarting the MDS
+is likely to resolve the problem.
+
+If there are no slow requests reported on the MDS, and it is not reporting
+that clients are misbehaving, either the client has a problem or its
+requests are not reaching the MDS.
+
+.. _ceph_fuse_debugging:
+
+ceph-fuse debugging
+===================
+
+ceph-fuse also supports ``dump_ops_in_flight``. See if it has any and where they are
+stuck.
+
+Debug output
+------------
+
+To get more debugging information from ceph-fuse, try running in the foreground
+with logging to the console (``-d``) and enabling client debug
+(``--debug-client=20``), enabling prints for each message sent
+(``--debug-ms=1``).
+
+If you suspect a potential monitor issue, enable monitor debugging as well
+(``--debug-monc=20``).
+
+.. _kernel_mount_debugging:
+
+Kernel mount debugging
+======================
+
+If there is an issue with the kernel client, the most important thing is
+figuring out whether the problem is with the kernel client or the MDS. Generally,
+this is easy to work out. If the kernel client broke directly, there will be
+output in ``dmesg``. Collect it and any inappropriate kernel state.
+
+Slow requests
+-------------
+
+Unfortunately the kernel client does not support the admin socket, but it has
+similar (if limited) interfaces if your kernel has debugfs enabled. There
+will be a folder in ``sys/kernel/debug/ceph/``, and that folder (whose name will
+look something like ``28f7427e-5558-4ffd-ae1a-51ec3042759a.client25386880``)
+will contain a variety of files that output interesting output when you ``cat``
+them. These files are described below; the most interesting when debugging
+slow requests are probably the ``mdsc`` and ``osdc`` files.
+
+* bdi: BDI info about the Ceph system (blocks dirtied, written, etc)
+* caps: counts of file "caps" structures in-memory and used
+* client_options: dumps the options provided to the CephFS mount
+* dentry_lru: Dumps the CephFS dentries currently in-memory
+* mdsc: Dumps current requests to the MDS
+* mdsmap: Dumps the current MDSMap epoch and MDSes
+* mds_sessions: Dumps the current sessions to MDSes
+* monc: Dumps the current maps from the monitor, and any "subscriptions" held
+* monmap: Dumps the current monitor map epoch and monitors
+* osdc: Dumps the current ops in-flight to OSDs (ie, file data IO)
+* osdmap: Dumps the current OSDMap epoch, pools, and OSDs
+
+If the data pool is in a NEARFULL condition, then the kernel cephfs client
+will switch to doing writes synchronously, which is quite slow.
+
+Disconnected+Remounted FS
+=========================
+Because CephFS has a "consistent cache", if your network connection is
+disrupted for a long enough time, the client will be forcibly
+disconnected from the system. At this point, the kernel client is in
+a bind: it cannot safely write back dirty data, and many applications
+do not handle IO errors correctly on close().
+At the moment, the kernel client will remount the FS, but outstanding file system
+IO may or may not be satisfied. In these cases, you may need to reboot your
+client system.
+
+You can identify you are in this situation if dmesg/kern.log report something like::
+
+   Jul 20 08:14:38 teuthology kernel: [3677601.123718] ceph: mds0 closed our session
+   Jul 20 08:14:38 teuthology kernel: [3677601.128019] ceph: mds0 reconnect start
+   Jul 20 08:14:39 teuthology kernel: [3677602.093378] ceph: mds0 reconnect denied
+   Jul 20 08:14:39 teuthology kernel: [3677602.098525] ceph:  dropping dirty+flushing Fw state for ffff8802dc150518 1099935956631
+   Jul 20 08:14:39 teuthology kernel: [3677602.107145] ceph:  dropping dirty+flushing Fw state for ffff8801008e8518 1099935946707
+   Jul 20 08:14:39 teuthology kernel: [3677602.196747] libceph: mds0 172.21.5.114:6812 socket closed (con state OPEN)
+   Jul 20 08:14:40 teuthology kernel: [3677603.126214] libceph: mds0 172.21.5.114:6812 connection reset
+   Jul 20 08:14:40 teuthology kernel: [3677603.132176] libceph: reset on mds0
+
+This is an area of ongoing work to improve the behavior. Kernels will soon
+be reliably issuing error codes to in-progress IO, although your application(s)
+may not deal with them well. In the longer-term, we hope to allow reconnect
+and reclaim of data in cases where it won't violate POSIX semantics (generally,
+data which hasn't been accessed or modified by other clients).
+
+Mounting
+========
+
+Mount 5 Error
+-------------
+
+A mount 5 error typically occurs if a MDS server is laggy or if it crashed.
+Ensure at least one MDS is up and running, and the cluster is ``active +
+healthy``. 
+
+Mount 12 Error
+--------------
+
+A mount 12 error with ``cannot allocate memory`` usually occurs if you  have a
+version mismatch between the :term:`Ceph Client` version and the :term:`Ceph
+Storage Cluster` version. Check the versions using::
+
+	ceph -v
+	
+If the Ceph Client is behind the Ceph cluster, try to upgrade it::
+
+	sudo apt-get update && sudo apt-get install ceph-common 
+
+You may need to uninstall, autoclean and autoremove ``ceph-common`` 
+and then reinstall it so that you have the latest version.
+
+Dynamic Debugging
+=================
+
+You can enable dynamic debug against the CephFS module.
+
+Please see: https://github.com/ceph/ceph/blob/master/src/script/kcon_all.sh
+
+In-memory Log Dump
+==================
+
+In-memory logs can be dumped by setting ``mds_extraordinary_events_dump_interval``
+during a lower level debugging (log level < 10). ``mds_extraordinary_events_dump_interval``
+is the interval in seconds for dumping the recent in-memory logs when there is an Extra-Ordinary event.
+
+The Extra-Ordinary events are classified as:
+
+* Client Eviction
+* Missed Beacon ACK from the monitors
+* Missed Internal Heartbeats
+
+In-memory Log Dump is disabled by default to prevent log file bloat in a production environment.
+The below commands consecutively enables it::
+
+  $ ceph config set mds debug_mds <log_level>/<gather_level>
+  $ ceph config set mds mds_extraordinary_events_dump_interval <seconds>
+
+The ``log_level`` should be < 10 and ``gather_level`` should be >= 10 to enable in-memory log dump.
+When it is enabled, the MDS checks for the extra-ordinary events every
+``mds_extraordinary_events_dump_interval`` seconds and if any of them occurs, MDS dumps the
+in-memory logs containing the relevant event details in ceph-mds log.
+
+.. note:: For higher log levels (log_level >= 10) there is no reason to dump the In-memory Logs and a
+          lower gather level (gather_level < 10) is insufficient to gather In-memory Logs. Thus a
+          log level >=10 or a gather level < 10 in debug_mds would prevent enabling the In-memory Log Dump.
+          In such cases, when there is a failure it's required to reset the value of
+          mds_extraordinary_events_dump_interval to 0 before enabling using the above commands.
+
+The In-memory Log Dump can be disabled using::
+
+  $ ceph config set mds mds_extraordinary_events_dump_interval 0
+
+Filesystems Become Inaccessible After an Upgrade
+================================================
+
+.. note::
+   You can avoid ``operation not permitted`` errors by running this procedure
+   before an upgrade. As of May 2023, it seems that ``operation not permitted``
+   errors of the kind discussed here occur after upgrades after Nautilus
+   (inclusive).
+
+IF
+
+you have CephFS file systems that have data and metadata pools that were
+created by a ``ceph fs new`` command (meaning that they were not created
+with the defaults)
+
+OR
+
+you have an existing CephFS file system and are upgrading to a new post-Nautilus
+major version of Ceph
+
+THEN
+
+in order for the documented ``ceph fs authorize...`` commands to function as
+documented (and to avoid 'operation not permitted' errors when doing file I/O
+or similar security-related problems for all users except the ``client.admin``
+user), you must first run:
+
+.. prompt:: bash $
+
+   ceph osd pool application set <your metadata pool name> cephfs metadata <your ceph fs filesystem name>
+
+and
+
+.. prompt:: bash $
+
+   ceph osd pool application set <your data pool name> cephfs data <your ceph fs filesystem name>
+
+Otherwise, when the OSDs receive a request to read or write data (not the
+directory info, but file data) they will not know which Ceph file system name
+to look up. This is true also of pool names, because the 'defaults' themselves
+changed in the major releases, from::
+
+   data pool=fsname
+   metadata pool=fsname_metadata
+
+to::
+
+   data pool=fsname.data and
+   metadata pool=fsname.meta
+
+Any setup that used ``client.admin`` for all mounts did not run into this
+problem, because the admin key gave blanket permissions.
+
+A temporary fix involves changing mount requests to the 'client.admin' user and
+its associated key. A less drastic but half-fix is to change the osd cap for
+your user to just ``caps osd = "allow rw"``  and delete ``tag cephfs
+data=....``
+
+Reporting Issues
+================
+
+If you have identified a specific issue, please report it with as much
+information as possible. Especially important information:
+
+* Ceph versions installed on client and server
+* Whether you are using the kernel or fuse client
+* If you are using the kernel client, what kernel version?
+* How many clients are in play, doing what kind of workload?
+* If a system is 'stuck', is that affecting all clients or just one?
+* Any ceph health messages
+* Any backtraces in the ceph logs from crashes
+
+If you are satisfied that you have found a bug, please file it on `the bug
+tracker`. For more general queries, please write to the `ceph-users mailing
+list`.
+
+.. _the bug tracker: http://tracker.ceph.com
+.. _ceph-users mailing list:  http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com/
diff --git a/doc/cephfs/upgrading.rst b/doc/cephfs/upgrading.rst
new file mode 100644
index 000000000..c06e3e8e0
--- /dev/null
+++ b/doc/cephfs/upgrading.rst
@@ -0,0 +1,90 @@
+.. _upgrade-mds-cluster:
+
+Upgrading the MDS Cluster
+=========================
+
+Currently the MDS cluster does not have built-in versioning or file system
+flags to support seamless upgrades of the MDSs without potentially causing
+assertions or other faults due to incompatible messages or other functional
+differences. For this reason, it's necessary during any cluster upgrade to
+reduce the number of active MDS for a file system to one first so that two
+active MDS do not communicate with different versions.
+
+The proper sequence for upgrading the MDS cluster is:
+
+1. For each file system, disable and stop standby-replay daemons.
+
+::
+
+    ceph fs set <fs_name> allow_standby_replay false
+
+In Pacific, the standby-replay daemons are stopped for you after running this
+command. Older versions of Ceph require you to stop these daemons manually.
+
+::
+
+    ceph fs dump # find standby-replay daemons
+    ceph mds fail mds.<X>
+
+
+2. For each file system, reduce the number of ranks to 1:
+
+::
+
+    ceph fs set <fs_name> max_mds 1
+
+3. Wait for cluster to stop non-zero ranks where only rank 0 is active and the rest are standbys.
+
+::
+
+    ceph status # wait for MDS to finish stopping
+
+4. For each MDS, upgrade packages and restart. Note: to reduce failovers, it is
+   recommended -- but not strictly necessary -- to first upgrade standby daemons.
+
+::
+
+    # use package manager to update cluster
+    systemctl restart ceph-mds.target
+
+5. For each file system, restore the previous max_mds and allow_standby_replay settings for your cluster:
+
+::
+
+    ceph fs set <fs_name> max_mds <old_max_mds>
+    ceph fs set <fs_name> allow_standby_replay <old_allow_standby_replay>
+
+
+Upgrading pre-Firefly file systems past Jewel
+=============================================
+
+.. tip::
+
+    This advice only applies to users with file systems
+    created using versions of Ceph older than *Firefly* (0.80).
+    Users creating new file systems may disregard this advice.
+
+Pre-firefly versions of Ceph used a now-deprecated format
+for storing CephFS directory objects, called TMAPs.  Support
+for reading these in RADOS will be removed after the Jewel
+release of Ceph, so for upgrading CephFS users it is important
+to ensure that any old directory objects have been converted.
+
+After installing Jewel on all your MDS and OSD servers, and restarting
+the services, run the following command:
+
+::
+    
+    cephfs-data-scan tmap_upgrade <metadata pool name>
+
+This only needs to be run once, and it is not necessary to
+stop any other services while it runs.  The command may take some
+time to execute, as it iterates overall objects in your metadata
+pool.  It is safe to continue using your file system as normal while
+it executes.  If the command aborts for any reason, it is safe
+to simply run it again.
+
+If you are upgrading a pre-Firefly CephFS file system to a newer Ceph version
+than Jewel, you must first upgrade to Jewel and run the ``tmap_upgrade``
+command before completing your upgrade to the latest version.
+
diff --git a/doc/changelog/v0.48.1argonaut.txt b/doc/changelog/v0.48.1argonaut.txt
new file mode 100644
index 000000000..cdd557f97
--- /dev/null
+++ b/doc/changelog/v0.48.1argonaut.txt
@@ -0,0 +1,1286 @@
+commit a7ad701b9bd479f20429f19e6fea7373ca6bba7c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 13 14:58:51 2012 -0700
+
+    v0.48.1argonaut
+
+commit d4849f2f8a8c213c266658467bc5f22763010bc2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 13:22:38 2012 -0700
+
+    rgw: fix usage trim call encoding
+    
+    Fixes: #2841.
+    Usage trim operation was encoding the wrong op structure (usage read).
+    Since the structures somewhat overlapped it somewhat worked, but user
+    info wasn't encoded.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 515952d07107d442889754ec3bd6a344fad25d58
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 8 15:21:53 2012 -0700
+
+    cls_rgw: fix rgw_cls_usage_log_trim_op encode/decode
+    
+    It was not encoding user, adding that and reset version
+    compatibility.
+    This changes affects command interface, makes use of
+    radosgw-admin usage trim incompatible. Use of old
+    radosgw-admin usage trim should be avoided, as it may
+    remove more data than requested. In any case, upgraded
+    server code will not handle old client's trim requests.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2e77130d5c80220be1612b5499d422de620d2d0b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Jul 31 16:17:22 2012 -0700
+
+    rgw: expand date format support
+    
+    Relaxing the date format parsing function to allow UTC
+    instead of GMT.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 14fa77d9277b5ef5d0c6683504b368773b39ccc4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 2 11:13:05 2012 -0700
+
+    rgw: complete multipart upload can handle chunked encoding
+    
+    Fixes: #2878
+    We now allow complete multipart upload to use chunked encoding
+    when sending request data. With chunked encoding the HTTP_LENGTH
+    header is not required.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a06f7783fbcc02e775fc36f30e422fe0f9e0ec2d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 11:19:32 2012 -0700
+
+    rgw_xml: xml_handle_data() appends data string
+    
+    Fixes: #2879.
+    xml_handle_data() appends data to the object instead of just
+    replacing it. Parsed data can arrive in pieces, specifically
+    when data is escaped.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a8b224b9c4877a559ce420a2e04f19f68c8c5680
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 13:09:41 2012 -0700
+
+    rgw: ETag is unquoted in multipart upload complete
+    
+    Fixes #2877.
+    Removing quotes from ETag before comparing it to what we
+    have when completing a multipart upload.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 22259c6efda9a5d55221fd036c757bf123796753
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 8 15:24:57 2012 -0700
+
+    MonMap: return error on failure in build_initial
+    
+    If mon_host fails to parse, return an error instead of success.
+    This avoids failing later on an assert monmap.size() > 0 in the
+    monmap in MonClient.
+    
+    Fixes: #2913
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 49b2c7b5a79b8fb4a3941eca2cb0dbaf22f658b7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 8 15:10:27 2012 -0700
+
+    addr_parsing: report correct error message
+    
+    getaddrinfo uses its return code to report failures.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 7084f29544f431b7c6a3286356f2448ae0333eda
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 8 14:01:53 2012 -0700
+
+    mkcephfs: use default osd_data, _journal values
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 96b1a496cdfda34a5efdb6686becf0d2e7e3a1c0
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 8 14:01:35 2012 -0700
+
+    mkcephfs: use new default keyring locations
+    
+    The ceph-conf command only parses the conf; it does not apply default
+    config values.  This breaks mkcephfs if values are not specified in the
+    config.
+    
+    Let ceph-osd create its own key, fix copying, and fix creation/copying for
+    the mds.
+    
+    Fixes: #2845
+    Reported-by: Florian Haas <florian@hastexo.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 4bd466d6ed49c7192df4a5bf0d63bda5d7d7dd9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 31 14:01:57 2012 -0700
+
+    osd: peering: detect when log source osd goes down
+    
+    The Peering state has a generic check based on the prior set osds that
+    will restart peering if one of them goes down (or one of the interesting
+    down ones comes up).  The GetLog state, however, can pull the log from
+    a peer that is not in the prior set if it got a notify from them (e.g., an
+    osd in an old interval that was down when the prior set was calculated).
+    If that osd goes down, we don't detect it and will block forward.
+    
+    Fix by adding a simple check in GetLog for the newest_update_osd going
+    down.
+    
+    (BTW GetMissing does not suffer from this problem because
+    peer_missing_requested is a subset of the prior set, so the Peering check
+    is sufficient.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 87defa88a0c6d6aafaa65437a6e4ddd92418f834
+Author: Sylvain Munaut <tnt@246tNt.com>
+Date:   Tue Jul 31 11:55:56 2012 -0700
+
+    rbd: fix off-by-one error in key name
+    
+    Fixes: #2846
+    Signed-off-by: Sylvain Munaut <tnt@246tNt.com>
+
+commit 37d5b46269c8a4227e5df61a88579d94f7b56772
+Author: Sylvain Munaut <tnt@246tNt.com>
+Date:   Tue Jul 31 11:54:29 2012 -0700
+
+    secret: return error on empty secret
+    
+    Signed-off-by: Sylvain Munaut <tnt@246tNt.com>
+
+commit 7b9d37c662313929b52011ddae47cc8abab99095
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 28 10:05:47 2012 -0700
+
+    osd: set STRAY on pg load when non-primary
+    
+    The STRAY bit indicates that we should annouce ourselves to the primary,
+    but it is only set in start_peering_interval().  We also need to set it
+    initially, so that a PG that is loaded but whose role does not change
+    (e.g., the stray replica stays a stray) will notify the primary.
+    
+    Observed:
+     - osd starts up
+     - mapping does not change, STRAY not set
+     - does not announce to primary
+     - primary does not re-check must_have_unfound, objects appear unfound
+    
+    Fix this by initializing STRAY when pg is loaded or created whenever we
+    are not the primary.
+    
+    Fixes: #2866
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 96feca450c5505a06868bc012fe998a03371b77f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 16:03:26 2012 -0700
+
+    osd: peering: make Incomplete a Peering substate
+    
+    This allows us to still catch changes in the prior set that would affect
+    our conclusions (that we are incomplete) and, when they happen, restart
+    peering.
+    
+    Consider:
+     - calc prior set, osd A is down
+     - query everyone else, no good info
+     - set down, go to Incomplete (previously WaitActingChange) state.
+     - osd A comes back up (we do nothing)
+     - osd A sends notify message with good info (we ignore)
+    
+    By making this a Peering substate, we catch the Peering AdvMap reaction,
+    which will notice a prior set down osd is now up and move to Reset.
+    
+    Fixes: #2860
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit a71e442fe620fa3a22ad9302413d8344a3a1a969
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 15:39:40 2012 -0700
+
+    osd: peering: move to Incomplete when.. incomplete
+    
+    PG::choose_acting() may return false and *not* request an acting set change
+    if it can't find any suitable peers with enough info to recover.  In that
+    case, we should move to Incomplete, not WaitActingChange, just like we do
+    a bit lower in GetLog() if we have non-contiguous logs.  The state name is
+    more accurate, and this is also needed to fix bug #2860.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 623026d9bc8ea4c845eb3b06d79e0ca9bef50deb
+Merge: 87b6e80 9db7809
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 14:00:52 2012 -0700
+
+    Merge remote-tracking branch 'gh/stable' into stable-next
+
+commit 9db78090451e609e3520ac3e57a5f53da03f9ee2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 16:35:00 2012 -0700
+
+    osd: fixing sharing of past_intervals on backfill restart
+    
+    We need to share past_intervals whenever we instantiate the PG on a peer.
+    In the PG activation case, this is based on whether our peer_info[] value
+    for that peer is dne().  However, the backfill code was updating the
+    peer info (history) in the block preceeding the dne() check, which meant
+    we never shared past_intervals in this case and the peer would have to
+    chew through a potentially large number of maps if the PG has not been
+    clean recently.
+    
+    Fix by checking dne() prior to the backfill block.  We still need to fill
+    in the message later because it isn't yet instantiated.
+    
+    Fixes: #2849
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 87b6e8045a3a1ff6439d2684e960ad0dc8988b33
+Merge: 81d72e5 7dfdf4f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:04:12 2012 -0700
+
+    Merge remote-tracking branch 'gh/wip-rbd-bid' into stable-next
+
+commit 81d72e5d7ba4713eb7c290878d901e21c0709028
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 23 10:47:10 2012 -0700
+
+    mon: make 'ceph osd rm ...' wipe out all state bits, not just EXISTS
+    
+    This ensures that when a new osd reclaims that id it behaves as if it were
+    really new.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ad9c37f2c029f6eb372efb711b234014397057e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 20:54:19 2012 -0700
+
+    test_stress_watch: just one librados instance
+    
+    This was creating a new cluster connection/session per iteration, and
+    along with it a few service threads and sockets and so forth.
+    
+    Unfortunately, librados leaks like a sieve, starting with CephContext
+    and ceph::crypto::init().  See #845 and #2067.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c60afe1842a48dd75944822c0872fce6a7229f5a
+Merge: 8833050 35b1326
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:03:50 2012 -0700
+
+    Merge commit '35b13266923f8095650f45562d66372e618c8824' into stable-next
+    
+    First batch of msgr fixes.
+
+commit 88330505cc772a5528e9405d515aa2b945b0819e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 9 15:53:31 2012 -0700
+
+    ReplicatedPG: fix replay op ordering
+    
+    After a client reconnect, the client replays outstanding ops.  The
+    OSD then immediately responds with success if the op has already
+    committed (version < ReplicatedPG::get_first_in_progress).
+    Otherwise, we stick it in waiting_for_ondisk to be replied to when
+    eval_repop concludes that waitfor_disk is empty.
+    
+    Fixes #2508
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit 682609a9343d0488788b1c6b03bc437b7905e4d6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 18 12:55:35 2012 -0700
+
+    objecter: always resend linger registrations
+    
+    If a linger op (watch) is sent to the OSD and updates the object, and then
+    the client loses the reply, it will resend the request.  The OSD will see
+    that it is a dup, however, and not set up the in-memory session state for
+    the watch.  This in turn will break the watch (i.e., notifies won't
+    get delivered).
+    
+    Instead, always resend linger registration ops, so that we always have a
+    unique reqid and do the correct session registeration for each session.
+    
+     * track the tid of the registation op for each LingerOp
+     * mark registrations ops as should_resend=false; cancel as needed
+     * when we send a new registration op, cancel the old one to ensure we
+       ignore the reply.  This is needed becuase we resend linger ops on any
+       pg change, not just a primary change.
+     * drop the first_send arg to send_linger(), as we can now infer that
+       from register_tid == 0.
+    
+    The bug was easily reproduced with ms inject socket failures = 500 and the
+    test_stress_watch utility.
+    
+    Fixes: #2796
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 4d7d3e276967d555fed8a689976047f72c96c2db
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 13:22:42 2012 -0700
+
+    osd: guard class call decoding
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7fbbe4652ffb2826978aa1f1cacce4456d2ef1fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 5 18:08:58 2012 -0700
+
+    librados: take lock when signaling notify cond
+    
+    When we are signaling the cond to indicate that a notify is complete,
+    take the appropriate lock.  This removes the possibility of a race
+    that loses our signal.  (That would be very difficult given that there
+    are network round trips involved, but this makes the lock/cond usage
+    "correct.")
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6ed01df412b4f4745c8f427a94446987c88b6bef
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 22 07:46:11 2012 -0700
+
+    workqueue: kick -> wake or _wake, depending on locking
+    
+    Break kick() into wake() and _wake() methods, depending on whether the
+    lock is already held.  (The rename ensures that we audit/fix all
+    callers.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    
+    Conflicts:
+    
+    	src/common/WorkQueue.h
+    	src/osd/OSD.cc
+
+commit d2d40dc3059d91450925534f361f2c03eec9ef88
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 4 15:11:21 2012 -0700
+
+    client: fix locking for SafeCond users
+    
+    Need to wait on flock, not client_lock.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c963a21a8620779d97d6cbb51572551bdbb50d0b
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:01:05 2012 -0700
+
+    filestore: check for EIO in read path
+    
+    Check for EIO in read methods and helpers.  Try to do checks in low-level
+    methods (e.g., lfn_*()) to avoid duplication in higher-level methods.
+    
+    The transaction apply function already checks for EIO on writes, and will
+    generate a nicer error message, so we can largely ignore the write path,
+    as long as errors get passed up correctly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6bd89aeb1bf3b1cbb663107ae6bcda8a84dd8601
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 09:07:46 2012 -0700
+
+    filestore: add 'filestore fail eio' option, default true
+    
+    By default we will assert/fail/crash on EIO from the underlying fs.  We
+    already do this in the write path, but not the read path, or in various
+    internal infrastructure.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e9b5a289838f17f75efbf9d1640b949e7485d530
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 13:53:03 2012 -0700
+
+    config: fix 'config set' admin socket command
+    
+    Fixes: #2832
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1a6cd9659abcdad0169fe802ed47967467c448b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 16:35:09 2012 -0700
+
+    osd: break potentially large transaction into pieces
+    
+    We do a similar trick elsewhere.  Control this via a tunable.  Eventually
+    we'll control the others (in a non-stable branch).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 15e1622959f5a46f7a98502cdbaebfda2247a35b
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 14:53:34 2012 -0700
+
+    osd: only commit past intervals at end of parallel build
+    
+    We don't check for gaps in the past intervals, so we should only commit
+    this when we are completely done.  Otherwise a partial run and rsetart will
+    leave the gap in place, which may confuse the peering code that relies on
+    this information.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 16302acefd8def98fc4597366d6ba2845e17fcb6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:57:35 2012 -0700
+
+    osd: generate past intervals in parallel on boot
+    
+    Even though we aggressively share past_intervals with notifies etc, it is
+    still possible for an osd to get buried behind a pile of old maps and need
+    to generate these if it has been out of the cluster for a while.  This has
+    happened to us in the past but, sadly, we did not merge the work then.
+    On the bright side, this implementation is much much much cleaner than the
+    old one because of the pg_interval_t helper we've since switched to.
+    
+    On bootup, we look at the intervals each pg needs and calclate the union,
+    and then iterate over that map range.  The inner bit of the loop is
+    functionally identical to PG::build_past_intervals(), keeping the per-pg
+    state in the pistate struct.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit fca65ff52a5f7d49bcac83b3b2232963a879e446
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:58:07 2012 -0700
+
+    osd: move calculation of past_interval range into helper
+    
+    PG::generate_past_intervals() first calculates the range over which it
+    needs to generate past intervals.  Do this in a helper function.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 5979351ef3d3d03bced9286f79cbc22524c4a8de
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:58:28 2012 -0700
+
+    osd: fix map epoch boot condition
+    
+    We only want to join the cluster if we can catch up to the latest
+    osdmap with a small number of maps, in this case a single map message.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 8c7186d02627f8255273009269d50955172efb52
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 20:18:01 2012 -0700
+
+    mon: ignore pgtemp messages from down osds
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b17f54671f350fd4247f895f7666d46860736728
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 20:16:04 2012 -0700
+
+    mon: ignore osd_alive messages from down osds
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7dfdf4f8de16155edd434534e161e06ba7c79d7d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jul 23 14:05:53 2012 -0700
+
+    librbd: replace assign_bid with client id and random number
+    
+    The assign_bid method has issues with replay because it is a write
+    that also returns data. This means that the replayed operation would
+    return success, but no data, and cause a create to fail. Instead, let
+    the client set the bid based on its global id and a random number.
+    
+    This only affects the creation of new images, since the bid is put
+    into an opaque string as part of the object prefix.
+    
+    Keep the server side assign_bid around in case there are old clients
+    still using it.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit dc2d67112163bee8b111f75ae3e3ca42884b09b4
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 9 14:11:23 2012 -0700
+
+    librados: add new constructor to form a Rados object from IoCtx
+    
+    This creates a separate reference to an existing connection, for
+    use when a client holding IoCtx needs to consult another (say,
+    for rbd cloning)
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit c99671201de9d9cdf03bbf0f4e28e8afb70c280c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 18 19:49:58 2012 -0700
+
+    add CRUSH_TUNABLES feature bit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0b579546cfddec35095b2aec753028d8e63f3533
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 18 10:24:58 2012 -0700
+
+    ObjectCacher: fix cache_bytes_hit accounting
+    
+    Misses are not hits!
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2869039b79027e530c2863ebe990662685e4bbe6
+Author: Pascal de Bruijn | Unilogic Networks B.V <pascal@unilogicnetworks.net>
+Date:   Wed Jul 11 15:23:16 2012 +0200
+
+    Robustify ceph-rbdnamer and adapt udev rules
+    
+    Below is a patch which makes the ceph-rbdnamer script more robust and
+    fixes a problem with the rbd udev rules.
+    
+    On our setup we encountered a symlink which was linked to the wrong rbd:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd1
+    
+    While that link should have gone to /dev/rbd3 (on which a
+    partition /dev/rbd3p1 was present).
+    
+    Now the old udev rule passes %n to the ceph-rbdnamer script, the problem
+    with %n is that %n results in a value of 3 (for rbd3), but in a value of
+    1 (for rbd3p1), so it seems it can't be depended upon for rbdnaming.
+    
+    In the patch below the ceph-rbdnamer script is made more robust and it
+    now it can be called in various ways:
+    
+      /usr/bin/ceph-rbdnamer /dev/rbd3
+      /usr/bin/ceph-rbdnamer /dev/rbd3p1
+      /usr/bin/ceph-rbdnamer rbd3
+      /usr/bin/ceph-rbdnamer rbd3p1
+      /usr/bin/ceph-rbdnamer 3
+    
+    Even with all these different styles of calling the modified script, it
+    should now return the same rbdname. This change "has" to be combined
+    with calling it from udev with %k though.
+    
+    With that fixed, we hit the second problem. We ended up with:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3p1
+    
+    So the rbdname was symlinked to the partition on the rbd instead of the
+    rbd itself. So what probably went wrong is udev discovering the disk and
+    running ceph-rbdnamer which resolved it to myrbd so the following
+    symlink was created:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3
+    
+    However partitions would be discovered next and ceph-rbdnamer would be
+    run with rbd3p1 (%k) as parameter, resulting in the name myrbd too, with
+    the previous correct symlink being overwritten with a faulty one:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3p1
+    
+    The solution to the problem is in differentiating between disks and
+    partitions in udev and handling them slightly differently. So with the
+    patch below partitions now get their own symlinks in the following style
+    (which is fairly consistent with other udev rules):
+    
+      /dev/rbd/mypool/myrbd-part1 -> /dev/rbd3p1
+    
+    Please let me know any feedback you have on this patch or the approach
+    used.
+    
+    Regards,
+    Pascal de Bruijn
+    Unilogic B.V.
+    
+    Signed-off-by: Pascal de Bruijn <pascal@unilogicnetworks.net>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 426384f6beccabf9e9b9601efcb8147904ec97c2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 16 16:02:14 2012 -0700
+
+    log: apply log_level to stderr/syslog logic
+    
+    In non-crash situations, we want to make sure the message is both below the
+    syslog/stderr threshold and also below the normal log threshold.  Otherwise
+    we get anything we gather on those channels, even when the log level is
+    low.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 8dafcc5c1906095cb7d15d648a7c1d7524df3768
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 16 15:40:53 2012 -0700
+
+    log: fix event gather condition
+    
+    We should gather an event if it is below the log or gather threshold.
+    
+    Previously we were only gathering if we were going to print it, which makes
+    the dump no more useful than what was already logged.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ec5cd6def9817039704b6cc010f2797a700d8500
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 16 13:11:24 2012 -0700
+
+    PG::RecoveryState::Stray::react(LogEvt&): reset last_pg_scrub
+    
+    We need to reset the last_pg_scrub data in the osd since we
+    are replacing the info.
+    
+    Probably fixes #2453
+    
+    In cases like 2453, we hit the following backtrace:
+    
+         0> 2012-05-19 17:24:09.113684 7fe66be3d700 -1 osd/OSD.h: In function 'void OSD::unreg_last_pg_scrub(pg_t, utime_t)' thread 7fe66be3d700 time 2012-05-19 17:24:09.095719
+    osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p))
+    
+     ceph version 0.46-313-g4277d4d (commit:4277d4d3378dde4264e2b8d211371569219c6e4b)
+     1: (OSD::unreg_last_pg_scrub(pg_t, utime_t)+0x149) [0x641f49]
+     2: (PG::proc_primary_info(ObjectStore::Transaction&, pg_info_t const&)+0x5e) [0x63383e]
+     3: (PG::RecoveryState::ReplicaActive::react(PG::RecoveryState::MInfoRec const&)+0x4a) [0x633eda]
+     4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::RecoveryState::MQuery>, boost::statechart::custom_reaction<PG::RecoveryState::MInfoRec>, boost::statechart::custom_reaction<PG::RecoveryState::MLogRec> >, boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x130) [0x6466a0]
+     5: (boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x81) [0x646791]
+     6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x63dfcb]
+     7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x63e0f1]
+     8: (PG::RecoveryState::handle_info(int, pg_info_t&, PG::RecoveryCtx*)+0x177) [0x616987]
+     9: (OSD::handle_pg_info(std::tr1::shared_ptr<OpRequest>)+0x665) [0x5d3d15]
+     10: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x2a0) [0x5d7370]
+     11: (OSD::_dispatch(Message*)+0x191) [0x5dd4a1]
+     12: (OSD::ms_dispatch(Message*)+0x153) [0x5ddda3]
+     13: (SimpleMessenger::dispatch_entry()+0x863) [0x77fbc3]
+     14: (SimpleMessenger::DispatchThread::entry()+0xd) [0x746c5d]
+     15: (()+0x7efc) [0x7fe679b1fefc]
+     16: (clone()+0x6d) [0x7fe67815089d]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Because we don't clear the scrub state before reseting info,
+    the last_scrub_stamp state in the info.history structure
+    changes without updating the osd state resulting in the
+    above assert failure.
+    
+    Backport: stable
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 248cfaddd0403c7bae8e1533a3d2e27d1a335b9b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 9 17:57:03 2012 -0700
+
+    ReplicatedPG: don't warn if backfill peer stats don't match
+    
+    pinfo.stats might be wrong if we did log-based recovery on the
+    backfilled portion in addition to continuing backfill.
+    
+    bug #2750
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit bcb1073f9171253adc37b67ee8d302932ba1667b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 15 20:30:34 2012 -0700
+
+    mon/MonitorStore: always O_TRUNC when writing states
+    
+    It is possible for a .new file to already exist, potentially with a
+    larger size.  This would happen if:
+    
+     - we were proposing a different value
+     - we crashed (or were stopped) before it got renamed into place
+     - after restarting, a different value was proposed and accepted.
+    
+    This isn't so unlikely for the log state machine, where we're
+    aggregating random messages.  O_TRUNC ensure we avoid getting the tail
+    end of some previous junk.
+    
+    I observed #2593 and found that a logm state value had a larger size on
+    one mon (after slurping) than the others, pointing to put_bl_sn_map().
+    
+    While we are at it, O_TRUNC put_int() too; the same type of bug is
+    possible there, too.
+    
+    Fixes: #2593
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 41a570778a51fe9a36a5b67a177d173889e58363
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 14 14:31:34 2012 -0700
+
+    osd: based misdirected op role calc on acting set
+    
+    We want to look at the acting set here, nothing else.  This was causing us
+    to erroneously queue ops for later (wasting memory) and to erroneously
+    print out a 'misdrected op' message in the cluster log (confusion and
+    incorrect [but ignored] -ENXIO reply).
+    
+    Fixes: #2022
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b3d077c61e977e8ebb91288aa2294fb21c197fe7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Jul 13 09:42:20 2012 -0700
+
+    qa: download tests from specified branch
+    
+    These python tests aren't installed, so they need to be downloaded
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit e855cb247b5a9eda6845637e2da5b6358f69c2ed
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 25 09:47:37 2012 -0700
+
+    rgw: don't override subuser perm mask if perm not specified
+    
+    Bug #2650. We were overriding subuser perm mask whenever subuser
+    was modified, even if perm mask was not passed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d6c766ea425d87a2f2405c08dcec66f000a4e1a0
+Author: James Page <james.page@ubuntu.com>
+Date:   Wed Jul 11 11:34:21 2012 -0700
+
+    debian: fix ceph-fs-common-dbg depends
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+
+commit 95e8d87bc3fb12580e4058401674b93e19df6e02
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 11 11:52:24 2012 -0700
+
+    rados tool: remove -t param option for target pool
+    
+    Bug #2772. This fixes an issue that was introduced when we
+    added the 'rados cp' command. The -t param was already used
+    for rados bench. With this change the only way to specify
+    a target pool is using --target-pool.
+    Though this problem is post argonaut, the 'rados cp' command
+    has been backported, so we need this fix there too.
+    
+    Backport: argonaut
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 5b10778399d5bee602e57035df7d40092a649c06
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 11 09:19:00 2012 -0700
+
+    Makefile: don't install crush headers
+    
+    This is leftover from when we built a libcrush.so.  We can re-add when we
+    start doing that again.
+    
+    Reported-by: Laszlo Boszormenyi <gcs@debian.hu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 35b13266923f8095650f45562d66372e618c8824
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 10 13:18:27 2012 -0700
+
+    msgr: take over existing Connection on Pipe replacement
+    
+    If a new pipe/socket is taking over an existing session, it should also
+    take over the Connection* associated with the existing session.  Because
+    we cannot clear existing->connection_state, we just take another reference.
+    
+    Clean up the comments a bit while we're here.
+    
+    This affects MDS<->client sessions when reconnecting after a socket fault.
+    It probably also affects intra-cluster (osd/osd, mds/mds, mon/mon)
+    sessions as well, but I did not confirm that.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b387077b1d019ee52b28bc3bc5305bfb53dfd892
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 8 20:33:12 2012 -0700
+
+    debian: include librados-config in librados-dev
+    
+    Reported-by: Laszlo Boszormenyi <gcs@debian.hu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 03c2dc244af11b711e2514fd5f32b9bfa34183f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 13:04:28 2012 -0700
+
+    lockdep: increase max locks
+    
+    Hit this limit with the rados api tests.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b554d112c107efe78ec64f85b5fe588f1e7137ce
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 12:07:28 2012 -0700
+
+    config: add unlocked version of get_my_sections; use it internally
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 01da287b8fdc07262be252f1a7c115734d3cc328
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 08:20:06 2012 -0700
+
+    config: fix lock recursion in get_val_from_conf_file()
+    
+    Introduce a private, already-locked version.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c73c64a0f722477a5b0db93da2e26e313a5f52ba
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 08:15:08 2012 -0700
+
+    config: fix recursive lock in parse_config_files()
+    
+    The _impl() helper is only called from parse_config_files(); don't retake
+    the lock.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6646e891ff0bd31c935d1ce0870367b1e086ddfd
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 18:51:02 2012 -0700
+
+    rgw: initialize fields of RGWObjEnt
+    
+    This fixes various valgrind warnings triggered by the s3test
+    test_object_create_unreadable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b33553aae63f70ccba8e3d377ad3068c6144c99a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jul 6 13:14:53 2012 -0700
+
+    rgw: handle response-* params
+    
+    Handle response-* params that set response header field values.
+    Fixes #2734, #2735.
+    Backport: argonaut
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 74f687501a8a02ef248a76f061fbc4d862a9abc4
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 4 13:59:04 2012 -0700
+
+    osd: add missing formatter close_section() to scrub status
+    
+    Also add braces to make the open/close matchups easier to see.  Broken
+    by f36617392710f9b3538bfd59d45fd72265993d57.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 020b29961303b12224524ddf78c0c6763a61242e
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Wed Jun 27 14:14:30 2012 -0700
+
+    pg: report scrub status
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit db6d83b3ed51c07b361b27d2e5ce3227a51e2c60
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Wed Jun 27 13:30:45 2012 -0700
+
+    pg: track who we are waiting for maps from
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit e1d4855fa18b1cda85923ad9debd95768260d4eb
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Tue Jun 26 16:25:27 2012 -0700
+
+    pg: reduce scrub write lock window
+    
+    Wait for all replicas to construct the base scrub map before finalizing
+    the scrub and locking out writes.
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit 27409aa1612c1512bf393de22b62bbfe79b104c1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 15:52:51 2012 -0700
+
+    rgw: don't store bucket info indexed by bucket_id
+    
+    Issue #2701. This info wasn't really used anywhere and we weren't
+    removing it. It was also sharing the same pool namespace as the
+    info indexed by bucket name, which is bad.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 9814374a2b40e15c13eb03ce6b8e642b0f7f93e4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 14:59:22 2012 -0700
+
+    test_rados_tool.sh: test copy pool
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d75100667a539baf47c79d752b787ed5dcb51d7a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 13:42:23 2012 -0700
+
+    rados tool: copy object in chunks
+    
+    Instead of reading the entire object and then writing it,
+    we read it in chunks.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 16ea64fbdebb7a74e69e80a18d98f35d68b8d9a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 29 14:43:00 2012 -0700
+
+    rados tool: copy entire pool
+    
+    A new rados tool command that copies an entire pool
+    into another existing pool.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 960c2124804520e81086df97905a299c8dd4e08c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 29 14:09:08 2012 -0700
+
+    rados tool: copy object
+    
+    New rados command: rados cp <src-obj> [dest-obj]
+    
+    Requires specifying source pool. Target pool and locator can be specified.
+    The new command preserves object xattrs and omap data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 23d31d3e2aa7f2b474a7b8e9d40deb245d8be9de
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 6 08:47:44 2012 -0700
+
+    ceph.spec.in: add ceph-disk-{activate,prepare}
+    
+    Reported-by: Jimmy Tang <jtang@tchpc.tcd.ie>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ea11c7f9d8fd9795e127cfd7e8a1f28d4f5472e9
+Author: Wido den Hollander <wido@widodh.nl>
+Date:   Thu Jul 5 15:29:54 2012 +0200
+
+    Allow URL-safe base64 cephx keys to be decoded.
+    
+    In these cases + and / are replaced by - and _ to prevent problems when using
+    the base64 strings in URLs.
+    
+    Signed-off-by: Wido den Hollander <wido@widodh.nl>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f67fe4e368b5f250f0adfb183476f5f294e8a529
+Author: Wido den Hollander <wido@widodh.nl>
+Date:   Wed Jul 4 15:46:04 2012 +0200
+
+    librados: Bump the version to 0.48
+    
+    Signed-off-by: Wido den Hollander <wido@widodh.nl>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 35b9ec881aecf84b3a49ec0395d7208de36dc67d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Jun 26 17:28:51 2012 -0700
+
+    rgw-admin: use correct modifier with strptime
+    
+    Bug #2658: used %I (12h) instead of %H (24h)
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit da251fe88503d32b86113ee0618db7c446d34853
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 15:40:27 2012 -0700
+
+    rgw: send both swift x-storage-token and x-auth-token
+    
+    older clients need x-storage-token, newer x-auth-token
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 4c19ecb9a34e77e71d523a0a97e17f747bd5767d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 15:17:19 2012 -0700
+
+    rgw: radosgw-admin date params now also accept time
+    
+    The date format now is "YYYY-MM-DD[ hh:mm:ss]". Got rid of
+    the --time param for the old ops log stuff.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    
+    Conflicts:
+    
+    	src/test/cli/radosgw-admin/help.t
+
+commit 6958aeb898fc683159483bfbb798f069a9b5330a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 13:14:47 2012 -0700
+
+    rgw-admin: fix usage help
+    
+    s/show/trim
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 83c043f803ab2ed74fa9a84ae9237dd7df2a0c57
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 14:07:16 2012 -0700
+
+    radosgw-admin: fix clit test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 5674158163e9c1d50985796931240b237676b74d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 11:32:57 2012 -0700
+
+    ceph: fix cli help test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 151bf0eef59acae2d1fcf3f0feb8b6aa963dc2f6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 3 11:23:16 2012 -0700
+
+    ReplicatedPG: remove faulty scrub assert in sub_op_modify_applied
+    
+    This assert assumed that all ops submitted before MOSDRepScrub was
+    submitted were processed by the time that MOSDRepScrub was
+    processed.  In fact, MOSDRepScrub's scrub_to may refer to a
+    last_update yet to be seen by the replica.
+    
+    Bug #2693
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 32833e88a1ad793fa4be86101ce9c22b6f677c06
+Author: Kyle Bader <kyle.bader@dreamhost.com>
+Date:   Tue Jul 3 11:20:38 2012 -0700
+
+    ceph: better usage
+    
+    Signed-off-by: Kyle Bader <kyle.bader@dreamhost.com>
+
+commit 67455c21879c9c117f6402259b5e2da84524e169
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 09:20:35 2012 -0700
+
+    debian: strip new ceph-mds package
+    
+    Reported-by: Amon Ott <a.ott@m-privacy.de>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b53cdb97d15f9276a9b26bec9f29034149f93358
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 06:46:10 2012 -0700
+
+    config: remove bad argparse_flag argument in parse_option()
+    
+    This is wrong, and thankfully valgrind picks it up.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f7d4e39740fd2afe82ac40c711bd3fe7a282e816
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 1 17:23:28 2012 -0700
+
+    msgr: restart_queue when replacing existing pipe and taking over the queue
+    
+    The queue may have been previously stopped (by discard_queue()), and needs
+    to be restarted.
+    
+    Fixes consistent failures from the mon_recovery.py integration tests.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 5dfd2a512d309f7f641bcf7c43277f08cf650b01
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 1 15:37:31 2012 -0700
+
+    msgr: choose incoming connection if ours is STANDBY
+    
+    If the connect_seq matches, but our existing connection is in STANDBY, take
+    the incoming one.  Otherwise, the other end will wait indefinitely for us
+    to connect but we won't.
+    
+    Alternatively, we could "win" the race and trigger a connection by sending
+    a keepalive (or similar), but that is more work; we may as well accept the
+    incoming connection we have now.
+    
+    This removes STANDBY from the acceptable WAIT case states.  It also keeps
+    responsibility squarely on the shoulders of the peer with something to
+    deliver.
+    
+    Without this patch, a 3-osd vstart cluster with
+    'ms inject socket failures = 100' and rados bench write -b 4096 would start
+    generating slow request warnings after a few minutes due to the osds
+    failing to connect to each other.  With the patch, I complete a 10 minute
+    run without problems.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b7007a159f6d941fa8313a24af5810ce295b36ca
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:50:47 2012 -0700
+
+    msgr: preserve incoming message queue when replacing pipes
+    
+    If we replace an existing pipe with a new one, move the incoming queue
+    of messages that have not yet been dispatched over to the new Pipe so that
+    they are not lost.  This prevents messages from being lost.
+    
+    Alternatively, we could set in_seq = existing->in_seq - existing->in_qlen,
+    but that would make the other end resend those messages, which is a waste
+    of bandwidth.
+    
+    Very easy to reproduce the original bug with 'ms inject socket failures'.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1f3a722e150f9f27fe7919e9579b5a88dcd15639
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:45:24 2012 -0700
+
+    msgr: move dispatch_entry into DispatchQueue class
+    
+    A bit cleaner.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 03445290dad5b1213dd138cacf46e379400201c9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:38:34 2012 -0700
+
+    msgr: move incoming queue to separate class
+    
+    This extricates the incoming queue and its funky relationship with
+    DispatchQueue from Pipe and moves it into IncomingQueue.  There is now a
+    single IncomingQueue attached to each Pipe.  DispatchQueue is now no
+    longer tied to Pipe.
+    
+    This modularizes the code a bit better (tho that is still a work in
+    progress) and (more importantly) will make it possible to move the
+    incoming messages from one pipe to another in accept().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0dbc54169512da776c16161ec3b8fa0b3f08e248
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 27 17:06:40 2012 -0700
+
+    msgr: make D_CONNECT constant non-zero, fix ms_handle_connect() callback
+    
+    A while ago we inadvertantly broke ms_handle_connect() callbacks because
+    of a check for m being non-zero in the dispatch_entry() thread.  Adjust the
+    enums so that they get delivered again.
+    
+    This fixes hangs when, for example, the ceph tool sends a command, gets a
+    connection reset, and doesn't get the connect callback to resend after
+    reconnecting to a new monitor.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 2429556a51e8f60b0d9bdee71ef7b34b367f2f38
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:10:40 2012 -0700
+
+    msgr: fix pipe replacement assert
+    
+    We may replace an existing pipe in the STANDBY state if the previous
+    attempt failed during accept() (see previous patches).
+    
+    This might fix #1378.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 204bc594be1a6046d1b362693d086b49294c2a27
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:07:31 2012 -0700
+
+    msgr: do not try to reconnect con with CLOSED pipe
+    
+    If we have a con with a closed pipe, drop the message.  For lossless
+    sessions, the state will be STANDBY if we should reconnect.  For lossy
+    sessions, we will end up with CLOSED and we *should* drop the message.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e6ad6d25a58b8e34a220d090d01e26293c2437b4
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:06:41 2012 -0700
+
+    msgr: move to STANDBY if we replace during accept and then fail
+    
+    If we replace an existing pipe during accept() and then fail, move to
+    STANDBY so that our connection state (connect_seq, etc.) is preserved.
+    Otherwise, we will throw out that information and falsely trigger a
+    RESETSESSION on the next connection attempt.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.48.2argonaut.txt b/doc/changelog/v0.48.2argonaut.txt
new file mode 100644
index 000000000..2884e60db
--- /dev/null
+++ b/doc/changelog/v0.48.2argonaut.txt
@@ -0,0 +1,476 @@
+commit 3e02b2fad88c2a95d9c0c86878f10d1beb780bfe
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 11 13:04:50 2012 -0700
+
+    v0.48.2argonaut
+
+commit 343e607980c923324de241d2cf8396d7c4857dd8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 18 13:45:27 2012 -0700
+
+    cls_rgw: if stats drop below zero, set them to zero
+    
+    This complements fix for #3127. This is only a band aid
+    solution for argonaut, the real solution fixes the original
+    issue that made this possible.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 92be908c7f22c0ccf2092024fd54096362032394
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 12 16:41:17 2012 -0700
+
+    cls_rgw: change scoping of suggested changes vars
+    
+    Fixes: #3127
+    Bad variable scoping made it so that specific variables
+    weren't initialized between suggested changes iterations.
+    This specifically affected a case where in a specific
+    change we had an updated followed by a remove, and the
+    remove was on a non-existent key (e.g., was already
+    removed earlier). We ended up re-substracting the
+    object stats, as the entry wasn't reset between
+    the iterations (and we didn't read it because the
+    key didn't exist).
+    
+    backport:argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d3a8497b44296484fa18ac10e776c93701365a8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 4 11:29:21 2012 -0700
+
+    objecter: fix osdmap wait
+    
+    When we get a pool_op_reply, we find out which osdmap we need to wait for.
+    The wait_for_new_map() code was feeding that epoch into
+    maybe_request_map(), which was feeding it to the monitor with the subscribe
+    request.  However, that epoch is the *start* epoch, not what we want.  Fix
+    this code to always subscribe to what we have (+1), and ensure we keep
+    asking for more until we catch up to what we know we should eventually
+    get.
+    
+    Bug: #3075
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e09b26555c6132ffce08b565780a39e4177cbc1c)
+
+commit de026f31f7f31c2b79c41910b83570f052f354d5
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 27 07:38:34 2012 -0700
+
+    objecter: send queued requests when we get first osdmap
+    
+    If we get our first osdmap and already have requests queued, send them.
+    
+    Backported from 8d1efd1b829ae50eab7f7f4c07da04e03fce7c45.
+    
+    Fixes: #3050
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 379aa80ac3a313025e433cefd239ddbeec45f9e7
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 21 21:12:33 2012 -0700
+
+    objecter: use ordered map<> for tracking tids to preserve order on resend
+    
+    We are using a hash_map<> to map tids to Op*'s.  In handle_osd_map(),
+    we will recalc_op_target() on each Op in a random (hash) order.  These
+    will get put in a temp map<tid,Op*> to ensure they are resent in the
+    correct order, but their order on the session->ops list will be random.
+    
+    Then later, if we reset an OSD connection, we will resend everything for
+    that session in ops order, which is be incorrect.
+    
+    Fix this by explicitly reordering the requests to resend in
+    kick_requests(), much like we do in handle_osd_map().  This lets us
+    continue to use a hash_map<>, which is faster for reasonable numbers of
+    requests.  A simpler but slower fix would be to just use map<> instead.
+    
+    This is one of many bugs contributing to #2947.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1113a6c56739a56871f01fa13da881dab36a32c4)
+
+commit 54788d0da40608fb8ccf8f16039536729881d542
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Aug 20 15:02:57 2012 -0700
+
+    rbd: force all exiting paths through main()/return
+    
+    This properly destroys objects.  In the process, remove usage_exit();
+    also kill error-handling in set_conf_param (never relevant for rbd.cc,
+    and if you call it with both pointers NULL, well...)
+    Also switch to EXIT_FAILURE for consistency.
+    
+    Backported from fed8aea662bf919f35a5a72e4e2a2a685af2b2ed.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Fixes: #2948
+
+commit a179ccee8848fe9bc8d622cba13fab1a99f6eb63
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Sep 18 09:37:44 2012 -0700
+
+    rbd: only open the destination pool for import
+    
+    Otherwise importing into another pool when the default pool, rbd,
+    doesn't exist results in an error trying to open the rbd pool.
+    
+    Reported-by: Sébastien Han <han.sebastien@gmail.com>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 16aca749989b006f4d5d2190ed1e8480c1bf0282
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Mon Sep 17 08:55:14 2012 -0700
+
+    ceph-disk-activate, upstart: Use "initctl emit" to start OSDs.
+    
+    This avoids an error if the daemon was running already, and is
+    already being done with the other services.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 34d8eec9e23edec174a96d16bf42a0b8df118183
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Sep 14 17:13:57 2012 -0700
+
+    rbd: make --pool/--image args easier to understand for import
+    
+    There's no need to set the default pool in set_pool_image_name - this
+    is done later, in a way that doesn't ignore --pool if --dest-pool
+    is not specified.
+    
+    This means --pool and --image can be used with import, just like
+    the rest of the commands. Without this change, --dest and --dest-pool
+    had to be used, and --pool would be silently ignored for rbd import.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit f390ee68233e9c6db1202aa95665adf27ba4399a
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Sep 13 14:06:04 2012 -0700
+
+    ceph-create-keys: Create a bootstrap-osd key too.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 9348ea0de953a4fd2eed239437d14c366c8e2cdd
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Sep 13 11:34:03 2012 -0700
+
+    ceph-create-keys: Refactor to share wait_for_quorum call.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit c9d4c58b9d71e6d84b644376684bcb72eae7e11c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Sep 12 11:38:07 2012 -0700
+
+    objecter: fix skipped map handling
+    
+    If we skip a map, we want to translate NO_ACTION to NEED_RESEND, but leave
+    POOL_DNE alone.
+    
+    Backported from 2a3b7961c021b19a035f8a6cc4fc3cc90f88f367.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit d8dff93a2497bd578116eb20ed65a0595acdf341
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jul 30 15:19:29 2012 -0700
+
+    librbd, cls_rbd: close snapshot creation race with old format
+    
+    If two clients created a snapshot at the same time, the one with the
+    higher snapshot id might be created first, so the lower snapshot id
+    would be added to the snapshot context and the snaphot seq would be
+    set to the lower one.
+    
+    Instead of allowing this to happen, return -ESTALE if the snapshot id
+    is lower than the currently stored snapshot sequence number. On the
+    client side, get a new id and retry if this error is encountered.
+    
+    Backport: argonaut
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 771ca71c0357bd8149677795ac934ab09945a3a3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 16:31:57 2012 -0700
+
+    upstart: Give everything a stop on stanza.
+    
+    These are all tasks, and expected to exit somewhat quickly,
+    but e.g. ceph-create-keys has a loop where it waits for mon
+    to reach quorum, so it might still be in that loop when the
+    machine is shut down.
+
+commit 9f967e3ac255b0037b598061f5cbb4050db79472
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 16:28:41 2012 -0700
+
+    upstart: Start mds,mon,radosgw after a reboot.
+    
+    They had no "start on" stanzas, so they didn't get started earlier.
+
+commit ce1e7d1b0291759950abf02f5bae064994d2ec34
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 15:31:06 2012 -0700
+
+    upstart: Add ceph-create-keys.conf to package.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 0ff22ba0508d43cd78aeae6736f2db002cc8de8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 11 14:50:53 2012 -0700
+
+    obsync: if OrdinaryCallingFormat fails, try SubdomainCallingFormat
+    
+    This blindly tries the Subdomain calling format if the ordinary method
+    fails.  In particular, this works around buckets that present a
+    PermanentRedirect message.
+    
+    See bug #3128.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+
+commit 0e58d95134dd95f33b9b1a3b1286a0719ad20707
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 17 16:04:20 2012 -0700
+
+    librbd: add test for discard of nonexistent objects
+    
+    This verifies librbd properly handles ENOENT during discard.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 97d8a734ce937ce2a683c7c36f5b72395c6456c2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 10 13:19:53 2012 -0700
+
+    librbd: ignore -ENOENT during discard
+    
+    This is a backport of a3ad98a3eef062e9ed51dd2d1e58c593e12c9703
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 61d705e2d67f83c81aa7c6362ec6703e014ecb87
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 15 18:42:56 2012 -0700
+
+    objectcacher: fix bh leak on discard
+    
+    Fixes: #2950
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 787a78e8b1d8e5d92ee447b970ec8b79baa46f0b
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:16:52 2012 -0400
+
+    upstart, ceph-create-keys: Make client.admin key generation automatic.
+    
+    This should help simplify Chef etc deployments. Now (when using the
+    Upstart jobs), when a ceph-mon is started, ceph-create-admin-key is
+    triggered. If /etc/ceph/$cluster.client.admin.keyring already exists,
+    it does nothing; otherwise, it waits for ceph-mon to reach quorum, and
+    then does a "ceph auth get-or-create" to create the key, and writes it
+    atomically to disk.
+    
+    The equivalent code can be removed from the Chef cookbook once this is
+    in.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 66aecb1e410a59f3e0ed89a30212b8dc546a3d4a
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:21:29 2012 -0400
+
+    config: Add a per-name default keyring to front of keyring search path.
+    
+    This lets us have e.g. /etc/ceph/ceph.client.admin.keyring that is
+    owned by root:admin and mode u=rw,g=r,o= without making every non-root
+    run of the command line tools complain and fail.
+    
+    This is what the Chef cookbook has been doing for a while already.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 6e34b8c109322690151b42efb745bc96a210dda4
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:11:09 2012 -0400
+
+    upstart: Make instance jobs export their cluster and id variables.
+    
+    This allows other jobs listening to Upstart "started ceph-mon" events
+    to see what instance started.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 27372dc4e99c17a7a2d5ad6646e5ae54392d5955
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Jul 12 10:47:29 2012 -0700
+
+    upstart: Make ceph-osd always set the crush location.
+    
+    This used to be conditional on config having osd_crush_location set,
+    but with that, minimal configuration left the OSD completely out of
+    the crush map, and prevented the OSD from starting properly.
+    
+    Note: Ceph does not currently let this mechanism automatically move
+    hosts to another location in the CRUSH hierarchy. This means if you
+    let this run with defaults, setting osd_crush_location later will not
+    take effect. Set up your config file (or Chef environment) fully
+    before starting the OSDs the first time.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 471105a966f873aef2361b1ed48d088c490fe1aa
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Jul 3 15:24:26 2012 -0700
+
+    ceph-disk-prepare: Partition and format OSD data disks automatically.
+    
+    Uses gdisk, as it seems to be the only tool that can automate GPT uuid
+    changes. Needs to run as root.
+    
+    Adds Recommends: gdisk to ceph.deb.
+    
+    Closes: #2547
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 24fe265c594308d5679e845fcfe8b0fe6a7be612
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Jul 3 09:22:28 2012 -0700
+
+    ceph-disk-prepare: Take fsid from config file.
+    
+    Closes: #2546.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 1f06b490b4187055a5bfabc20746c87dfa075e0d
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Mon Jun 25 15:14:33 2012 -0700
+
+    upstart: fix regex
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 657ca118a7658617b9117311d9ee1cbe00103c06
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 28 16:17:21 2012 -0700
+
+    rgw: clear usage map before reading usage
+    
+    Fixes: #3057
+    Since we read usage in chunks we need to clear the
+    usage map before reading the next chunk, otherwise
+    we're going to aggregate the old data as well.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit c49284c98de1df1a889e0c88e6b1157133f5e0a6
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 23 11:48:50 2012 -0700
+
+    Don't package crush header files.
+
+commit b6fb3e37a68d48434a81ee4356cde48cad187e94
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 17 17:34:23 2012 -0700
+
+    rgw: dump content_range using 64 bit formatters
+    
+    Fixes: #2961
+    Also make sure that size is 64 bit.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 08d975146cbe6796799266f03361240a62acb297
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 21 10:58:38 2012 -0700
+
+    Revert "rgw: dump content_range using 64 bit formatters"
+    
+    This reverts commit faf9fa5744b459abc2eda829a48a4e07b9c97a08.
+
+commit faf9fa5744b459abc2eda829a48a4e07b9c97a08
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 17 17:34:23 2012 -0700
+
+    rgw: dump content_range using 64 bit formatters
+    
+    Fixes: #2961
+    Also make sure that size is 64 bit.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 47b24c0562bcb44964a0b8f6c4847bb0f05924e0
+Author: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+Date:   Tue Jul 31 19:13:03 2012 -0700
+
+    obsync: add missing package specifier to format_exc
+    
+    Fixes: #2873
+    Signed-off-by: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+
+commit a14214dc9b7c15581a0664dbe259389867f88e72
+Author: Danny Kukawka <danny.kukawka@bisect.de>
+Date:   Thu Aug 16 12:56:58 2012 +0200
+
+    fix keyring generation for mds and osd
+    
+        [ The following text is in the "UTF-8" character set. ]
+        [ Your display is set for the "ANSI_X3.4-1968" character set.  ]
+        [ Some characters may be displayed incorrectly. ]
+    
+    Fix config keys for OSD/MDS data dirs. As in documentation and other
+    places of the scripts the keys are 'osd data'/'mds data' and not
+    'osd_data'
+    
+    In case if MDS: if 'mds data' doesn't exist, create it.
+    
+    Signed-off-by: Danny Kukawka <danny.kukawka@bisect.de>
+
+commit d1a31ce1ca2a5e1c53146c6e9063a123a1fe8cdb
+Author: Danny Kukawka <danny.kukawka@bisect.de>
+Date:   Thu Aug 16 12:56:32 2012 +0200
+
+    fix ceph osd create help
+    
+        [ The following text is in the "UTF-8" character set. ]
+        [ Your display is set for the "ANSI_X3.4-1968" character set.  ]
+        [ Some characters may be displayed incorrectly. ]
+    
+    Change ceph osd create <osd-id> to ceph osd create <uuid>, since this
+    is what the command is really doing.
+    
+    Signed-off-by: Danny Kukawka <danny.kukawka@bisect.de>
+
+commit dc2a232bd37b7202c3d6e94396b3d85cec5225cd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 17:24:19 2012 -0700
+
+    mon: simplify logmonitor check_subs; less noise
+    
+     * simple helper to translate name to id
+     * verify sub type is valid in caller
+     * assert sub type is valid in method
+     * simplify iterator usage
+    
+    Among other things, this gets rid of this noise in the logs:
+    
+    2012-07-10 20:51:42.617152 7facb23f1700  1 mon.a@1(peon).log v310 check_sub sub monmap not log type
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.48.3argonaut.txt b/doc/changelog/v0.48.3argonaut.txt
new file mode 100644
index 000000000..22f0dc967
--- /dev/null
+++ b/doc/changelog/v0.48.3argonaut.txt
@@ -0,0 +1,895 @@
+commit 920f82e805efec2cae05b79c155c07df0f3ed5dd
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Jan 7 21:08:08 2013 -0800
+
+    v0.48.3argonaut
+
+commit 213e3559dd260a2e19324f2a671c808261249f96
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 20:43:21 2013 -0800
+
+    osd: fix race in do_recovery()
+    
+    Verify that the PG is still RECOVERING or BACKFILL when we take the pg
+    lock in the recovery thread.  This prevents a crash from an invalid
+    state machine event when the recovery queue races with a PG state change
+    (e.g., due to peering).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit e410d1a066b906cad3103a5bbfa5b4509be9ac37
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 4 21:19:45 2013 -0800
+
+    ReplicatedPG: requeue waiting_for_ondisk in apply_and_flush_repops
+    
+    Fixes: #3722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit cd5f2bfdbb7fbf9237bef482845644cc41fa66de
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Nov 15 16:20:33 2012 -0800
+
+    ObjectCacher: fix off-by-one error in split
+    
+    This error left a completion that should have been attached
+    to the right BufferHead on the left BufferHead, which would
+    result in the completion never being called unless the buffers
+    were merged before it's original read completed. This would cause
+    a hang in any higher level waiting for a read to complete.
+    
+    The existing loop went backwards (using a forward iterator),
+    but stopped when the iterator reached the beginning of the map,
+    or when a waiter belonged to the left BufferHead.
+    
+    If the first list of waiters should have been moved to the right
+    BufferHead, it was skipped because at that point the iterator
+    was at the beginning of the map, which was the main condition
+    of the loop.
+    
+    Restructure the waiters-moving loop to go forward in the map instead,
+    so it's harder to make an off-by-one error.
+    
+    Possibly-fixes: #3286
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2e862f4d183d8b57b43b0777737886f18f68bf00)
+
+commit 802c486fd0913e9fe58ae7ff72df135f99712358
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 11:07:48 2013 -0800
+
+    config: change default log_max_recent to 10,000
+    
+    Commit c34e38bcdc0460219d19b21ca7a0554adf7f7f84 meant to do this but got
+    the wrong number of zeros.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 39a734fbf34ccd121f17023bcec814e61c8bdaab
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 3 17:15:07 2013 -0800
+
+    os/FileStore: fix non-btrfs op_seq commit order
+    
+    The op_seq file is the starting point for journal replay.  For stable btrfs
+    commit mode, which is using a snapshot as a reference, we should write this
+    file before we take the snap.  We normally ignore current/ contents anyway.
+    
+    On non-btrfs file systems, however, we should only write this file *after*
+    we do a full sync, and we should then fsync(2) it before we continue
+    (and potentially trim anything from the journal).
+    
+    This fixes a serious bug that could cause data loss and corruption after
+    a power loss event.  For a 'kill -9' or crash, however, there was little
+    risk, since the writes were still captured by the host's cache.
+    
+    Fixes: #3721
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 28d59d374b28629a230d36b93e60a8474c902aa5)
+
+commit 49416619d733572368e5d2ba7f2b34150c754b23
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 13:07:18 2012 -0800
+
+    log: broadcast cond signals
+    
+    We were using a single cond, and only signalling one waiter.  That means
+    that if the flusher and several logging threads are waiting, and we hit
+    a limit, we the logger could signal another logger instead of the flusher,
+    and we could deadlock.
+    
+    Similarly, if the flusher empties the queue, it might signal only a single
+    logger, and that logger could re-signal the flusher, and the other logger
+    could wait forever.
+    
+    Intead, break the single cond into two: one for loggers, and one for the
+    flusher.  Always signal the (one) flusher, and always broadcast to all
+    loggers.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 813787af3dbb99e42f481af670c4bb0e254e4432)
+
+commit f8bb4814d0fd7e08c9b6cdcde02940bf1b42cadd
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 13:58:44 2013 -0800
+
+    log: fix locking typo/stupid for dump_recent()
+    
+    We weren't locking m_flush_mutex properly, which in turn was leading to
+    racing threads calling dump_recent() and garbling the crash dump output.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 43cba617aa0247d714632bddf31b9271ef3a1b50)
+
+commit 9483a032f750572586f146c696ec6501d3df0383
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Dec 27 16:06:24 2012 -0800
+
+    init-ceph: fix status version check across machines
+    
+    The local state isn't propagated into the backtick shell, resulting in
+    'unknown' for all remote daemons.  Avoid backticks altogether.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6c7b667badc5e7608b69c533a119a2afc062e257)
+
+commit 8fef9360fe949fbf4ee6e468c36cc1c84b369b6e
+Author: Travis Rhoden <trhoden@gmail.com>
+Date:   Mon Aug 20 13:29:11 2012 -0700
+
+    init-ceph: use SSH in "service ceph status -a" to get version
+    
+    When running "service ceph status -a", a version number was never
+    returned for remote hosts, only for the local.  This was because
+    the command to query the version number didn't use the do_cmd
+    function, which is responsible for running the command over SSH
+    when needed.
+    
+    Modify the ceph init.d script to use do_cmd for querying the
+    Ceph version.
+    
+    Signed-off-by: Travis Rhoden <trhoden@gmail.com>
+    (cherry picked from commit 60fdb6fda6233b01dae4ed8a34427d5960840b84)
+
+commit c34e38bcdc0460219d19b21ca7a0554adf7f7f84
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Nov 28 13:00:36 2012 -0800
+
+    log: 10,000 recent log entries
+    
+    This is what we were (wrongly) doing before, so there are no memory
+    utilization surprises.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 78286b1403a5e0f14f95fe6b92f2fdb163e909f1)
+
+commit 4daede79f578cccd340210752f912e685b1ff03d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Nov 28 12:59:43 2012 -0800
+
+    log: fix log_max_recent config
+    
+    <facepalm>
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4de7748b72d4f90eb1197a70015c199c15203354)
+
+commit fdae0552a0b8b1a634775609b473aff9221904f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Dec 20 13:48:06 2012 -0800
+
+    log: fix flush/signal race
+    
+    We need to signal the cond in the same interval where we hold the lock
+    *and* modify the queue.  Otherwise, we can have a race like:
+    
+     queue has 1 item, max is 1.
+     A: enter submit_entry, signal cond, wait on condition
+     B: enter submit_entry, signal cond, wait on condition
+     C: flush wakes up, flushes 1 previous item
+     A: retakes lock, enqueues something, exits
+     B: retakes lock, condition fails, waits
+      -> C is never woken up as there are 2 items waiting
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 50914e7a429acddb981bc3344f51a793280704e6)
+
+commit bdcf6647dec05617d3da46ad00459498ede94f1d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 17:12:07 2012 -0800
+
+    .gitignore:  Add ar-lib to ignore list
+
+commit 51a900cfd7cc009f03fdb80be028015fc34547d2
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 16:55:27 2012 -0800
+
+    autogen.sh:  Create m4 directory for leveldb
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit 74473bb667df0ee0bb1adc6c43242733b80aa10d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 16:17:33 2012 -0800
+
+    leveldb:  Update submodule
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit bc4f74c7b9ec4e6bbcf4ab4ab68a82a69b8eaca5
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Dec 20 16:49:32 2012 -0800
+
+    ceph.spec.in:  Fedora builds debuginfo by default.
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit 3ed2d59e98494a0da29f4bb8537471237a6f8e93
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 19 17:07:18 2012 -0800
+
+    rgw: fix error handling with swift
+    
+    Fixes: #3649
+    verify_swift_token returns a bool and not an int.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit dfd310368267df848f2b65cc536b4ffcb039d353
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Mon Sep 24 09:55:25 2012 -0700
+
+    client: Fix for #3184 cfuse segv with no keyring
+    
+    Fixes bug #3184 where the ceph-fuse client segfaults if authx is
+    enabled but no keyring file is present.  This was due to the
+    client->init() return value not getting checked.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 47983df4cbd31f299eef896b4612d3837bd7c7bd)
+
+commit acebcce91448421c34a72850a380fcd1aabe9f41
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Oct 9 21:25:54 2012 +0100
+
+    mon: Monitor: resolve keyring option to a file before loading keyring
+    
+    Otherwise our keyring default location, or any other similarly formatted
+    location, will be taken as the actual location for the keyring and fail.
+    
+    Reported-by: tziOm (at) #ceph
+    Fixes: 3276
+    Backport: argonaut
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7ef0df25e001bfae303feb3ae36514608767b1f2)
+
+commit 7ab00a796d2233c3804f52fbba16c2f3ed0c297b
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Dec 5 19:39:11 2012 -0800
+
+    .gitignore:  Add m4 macro directories to ignore list
+
+commit b781ac00e16fd802b6e055dd9d453271db6efa39
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Nov 8 12:43:24 2012 -0800
+
+    build:  Add RPM release string generated from git describe.
+    
+    Fix for bug 3451.  Use the commit count and sha1 from git describe to
+    construct a release string for rpm packages.
+    
+    Conflicts:
+    
+    	configure.ac
+
+commit f60c9b6600a85b840a02e145bac77326d60fd03f
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Fri Nov 9 13:28:13 2012 -0800
+
+    ceph.spec.in:  Build debuginfo subpackage.
+    
+    This is a partial fix for bug 3471.  Enable building of debuginfo package.
+    Some distributions enable this automatically by installing additional rpm
+    macros, on others it needs to be explicity added to the spec file.
+
+commit 7fbf9476cb44ce256f4bb2366f46fa20cb570eda
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Dec 3 14:32:28 2012 -0800
+
+    rgw: fix swift auth concurrency issue
+    
+    Fixes: #3565
+    Originally ops were using static structures, but that
+    has since changed. Switching swift auth handler to do
+    the same.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 0639cd9c479d69b077175f0385eb569ebb839349
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 29 13:39:22 2012 -0800
+
+    rgw: fix rgw_tools get_obj()
+    
+    The original implementation broke whenever data exceeded
+    the chunk size. Also don't keep cache for objects that
+    exceed the chunk size as cache is not designed for
+    it. Increased chunk size to 512k.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 84983387a220a29540aca64af774bbe7bc6b5ee6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 29 12:47:59 2012 -0800
+
+    rgw: fix PUT acls
+    
+    This fixes a regression introduced at
+    17e4c0df44781f5ff1d74f3800722452b6a0fc58. The original
+    patch fixed error leak, however it also removed the
+    operation's send_response() call.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 6c1e2b663227cadff4255db3321976882df6b261
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 17:10:11 2012 -0800
+
+    rgw: fix xml parser leak
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f86522cdfcd81b2d28c581ac8b8de6226bc8d1a4)
+
+commit 17e4c0df44781f5ff1d74f3800722452b6a0fc58
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 16:52:38 2012 -0800
+
+    rgw: fix memory leaks
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 98a04d76ebffa61c3ba4b033cdd57ac57b2f29f3)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+    	src/rgw/rgw_op.h
+
+commit cd18b9f43cce68ac3d8f109e82a5b603f9caad9b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 7 13:21:15 2012 -0800
+
+    rgw: don't convert object mtime to UTC
+    
+    Fixes: #3452
+    When we read object info, don't try to convert mtime to
+    UTC, it's already in UTC.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit ddb40dec6e8d930fd63a71ef1c98279d6472e3e4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 14 11:30:34 2012 -0800
+
+    rgw: relax date format check
+    
+    Don't try to parse beyond the GMT or UTC. Some clients use
+    special date formatting. If we end up misparsing the date
+    it'll fail in the authorization, so don't need to be too
+    restrictive.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit e35e7e2bcb749e3ee00f1f93bc6b17bde8329414
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 30 14:17:56 2012 -0700
+
+    ceph-disk-activate: avoid duplicating mounts if already activated
+    
+    If the given device is already mounted at the target location, do not
+    mount --move it again and create a bunch of dup entries in the /etc/mtab
+    and kernel mount table.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c435d314caeb5424c1f4482ad02f8a085317ad5b)
+
+commit 0c9bbb3b4b4dbe6f0a12960785e35af9c748701a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 21:21:18 2012 -0700
+
+    ceph-disk-prepare: poke kernel into refreshing partition tables
+    
+    Prod the kernel to refresh the partition table after we create one.  The
+    partprobe program is packaged with parted, which we already use, so this
+    introduces no new dependency.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 402e1f5319a52c309eca936081fddede1f107268)
+
+commit b4004806ec387915ba4e039b9bc1c563273ef08f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 21:20:21 2012 -0700
+
+    ceph-disk-prepare: fix journal partition creation
+    
+    The end value needs to have + to indicate it is relative to wherever the
+    start is.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2e32a0ee2d9e2a3bf5b138f50efc5fba8d5b8660)
+
+commit efac1bb19f1ca3851c7b51d96cb4e82d8d142a45
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 18:14:47 2012 -0700
+
+    ceph-disk-prepare: assume parted failure means no partition table
+    
+    If the disk has no valid label we get an error like
+    
+      Error: /dev/sdi: unrecognised disk label
+    
+    Assume any error we get is that and go with an id label of 1.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8921fc7c7bc28fb98334c06f1f0c10af58085085)
+
+commit 39b9da403541617fb088aa4ae03925850aa2de65
+Merge: 83ab366 8c115d3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 12 11:24:00 2012 -0800
+
+    Merge remote-tracking branch 'gh/wip-mds-stable' into stable
+
+commit 8c115d3e81543a4c723bae217ba5b9d2c0e988f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Nov 9 05:28:12 2012 -0800
+
+    mds: re-try_set_loner() after doing evals in eval(CInode*, int mask)
+    
+    Consider a case where current loner is A and wanted loner is B.
+    At the top of the function we try to set the loner, but that may fail
+    because we haven't processed the gathered caps yet for the previous
+    loner.  In the body we do that and potentially drop the old loner, but we
+    do not try_set_loner() again on the desired loner.
+    
+    Try after our drop.  If it succeeds, loop through the eval's one more time
+    so that we can issue caps approriately.
+    
+    This fixes a hang induced by a simple loop like:
+    
+     while true ; do echo asdf >> mnt.a/foo ; tail mnt.b/foo ; done &
+     while true ; do ls mnt.a mnt.b ; done
+    
+    (The second loop may not be necessary.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1a23bd6981e2a308be4a905021453a6d5feac9e3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jul 13 14:23:27 2012 -0700
+
+    CompatSet: users pass bit indices rather than masks
+    
+    CompatSet users number the Feature objects rather than
+    providing masks.  Thus, we should do
+    
+    mask |= (1 << f.id) rather than mask |= f.id.
+    
+    In order to detect old, broken encodings, the lowest
+    bit will be set in memory but not set in the encoding.
+    We can reconstruct the correct mask from the names map.
+    
+    This bug can cause an incompat bit to not be detected
+    since 1|2 == 1|2|3.
+    
+    fixes: #2748
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 83ab36655f397b7d781a611b4d82679c8484710f
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Nov 7 12:41:10 2012 -0800
+
+    ceph.spec.in:  Remove ceph version requirement from ceph-fuse package.
+    
+    The ceph-fuse rpm package now only requires ceph as a pre-req, not a specific
+    version.
+
+commit 578b5764d5b26d190c295da88c1be5f0c77682a7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Oct 24 13:15:46 2012 -0700
+
+    rgw: fix multipart overwrite
+    
+    Fixes: #3400
+    Removed a few lines of code that prematurely created the head
+    part of the final object (before creating the manifest).
+    
+    backport:argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 96888ddf154231daa6c7138bb2b97674771cc610
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 5 23:27:13 2012 -0800
+
+    mds: move to from loner -> mix if *anyone* wants rd|wr
+    
+    We were either going to MIX or SYNC depending on whether non-loners wanted
+    to read/write, but it may be that the loner wants to if our logic for
+    choosing loner vs not longer is based on anything other that just rd|wr
+    wanted.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b940d7750d14285fc85ce63b0f0444b58ad3ad93
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 5 23:26:09 2012 -0800
+
+    mds: base loner decision on wanted RD|WR|EXCL, not CACHE|BUFFER
+    
+    Observed instance where one client wanted the Fc cap and prevented the
+    loner from getting RD|WR caps.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7a82a3214a38cbc7b5750adba23b760a4f51643a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 30 09:00:11 2012 -0700
+
+    osd: make pool_snap_info_t encoding backward compatible
+    
+    Way back in fc869dee1e8a1c90c93cb7e678563772fb1c51fb (v0.42) when we redid
+    the osd type encoding we forgot to make this conditionally encode the old
+    format for old clients.  In particular, this means that kernel clients
+    will fail to decode the osdmap if there is a rados pool with a pool-level
+    snapshot defined.
+    
+    Fixes: #3290
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    
+    Conflicts:
+    
+    	src/test/encoding/types.h
+
+commit 8d7ca76182fdc04aaf8b698afbbd2d9b22e07fdc
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Sep 7 13:49:27 2012 +0800
+
+    osd/OSD.cc: Fix typo in OSD::heartbeat_check()
+    
+    The check 'p->second.last_tx > cutoff' should always be false
+    since last_tx is periodically updated by OSD::heartbeat()
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit fc2b3ced4f7ed8d6e065c159fbeb8670fcdaca2a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 22 16:52:11 2012 -0700
+
+    rgw: dump an error message if FCGX_Accept fails
+    
+    Adding missing debug info.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d9b51b3b7011f4551fe27e33301b26cd23a72378
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 22 15:38:30 2012 -0700
+
+    workqueue: make debug output include active threads
+    
+    Include active thread count in threadpool debug output.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 852e27d3e19239a696fa693e2a30c3226d27a611
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 22 13:16:59 2012 -0700
+
+    rgw: don't continue processing of GET request on error
+    
+    Fixes #3381
+    We continued processing requests long after the client
+    has died. This fix appliese to both s3 and swift.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 5f2b9b451da5a96fdbf9cf292648feb40dc13d28
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Oct 19 08:46:19 2012 -0700
+
+    osd: be quiet about watches
+    
+    Useless log noise.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ea42eb1f7866fed0e843257ce05e34503fd0129c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Oct 17 17:44:12 2012 -0700
+
+    addr_parsing: make , and ; and ' ' all delimiters
+    
+    Instead of just ,.  Currently "foo.com, bar.com" will fail because of the
+    space after the comma.  This patches fixes that, and makes all delim
+    chars interchangeable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b0164d9902c834c95187f6b277f58a30d26cc4d3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 10:57:42 2012 -0700
+
+    ceph-disk-prepare, debian/control: Support external journals.
+    
+    Previously, ceph-disk-* would only let you use a journal that was a
+    file inside the OSD data directory. With this, you can do:
+    
+      ceph-disk-prepare /dev/sdb /dev/sdb
+    
+    to put the journal as a second partition on the same disk as the OSD
+    data (might save some file system overhead), or, more interestingly:
+    
+      ceph-disk-prepare /dev/sdb /dev/sdc
+    
+    which makes it create a new partition on /dev/sdc to use as the
+    journal. Size of the partition is decided by $osd_journal_size.
+    /dev/sdc must be a GPT-format disk. Multiple OSDs may share the same
+    journal disk (using separate partitions); this way, a single fast SSD
+    can serve as journal for multiple spinning disks.
+    
+    The second use case currently requires parted, so a Recommends: for
+    parted has been added to Debian packaging.
+    
+    Closes: #3078
+    Closes: #3079
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 5a3076fd51706e32bc55b734099037520299003b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 15 09:43:47 2012 -0700
+
+    rgw: don't add port to url if already has one
+    
+    Fixes: #3296
+    Specifically, is host name string already has ':', then
+    don't try to append theport (swift auth).
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d76b71caa99a39035ea6ed9db6ca0994975d38af
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 15 16:37:05 2012 -0700
+
+    admin_socket: fix '0' protocol version
+    
+    Broken by 895e24d198ced83ab7fed3725f12f75e3bc97b0b.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 714254cb0522b194118eb854f90d4c3971e8c6b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 8 17:14:22 2012 -0700
+
+    mon: drop command replies on paxos reset
+    
+    If paxos resets, do not send the reply for the commit we were waiting for;
+    let the command be reprocessed and re-proposed.
+    
+    Among other things, this could lead to nondeterministic results for
+    'ceph osd create <uuid>'.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit d9e981f4209da9a0e000d434e28686a133b25f11
+Merge: 5d84b4a 13b448b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 8 21:02:51 2012 -0700
+
+    Merge remote-tracking branch 'gh/for-stable-fstypes-and-ext-journal' into stable
+
+commit 5d84b4a22b2ff975679711719f336a312a6edff3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 2 13:02:04 2012 -0700
+
+    ceph-authtool: Fix usage, it's --print-key not --print.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 13b448b8ce959f31d07cd1827614c445c410aa62
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 09:22:34 2012 -0700
+
+    upstart: OSD journal can be a symlink; if it's dangling, don't start.
+    
+    This lets a $osd_data/journal symlink point to
+    /dev/disk/by-partuuid/UUID and the osd will not attempt to start until
+    that disk is available.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit d9dfa2d17859ca07d735b8e35c5dc00a25995153
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Oct 5 09:10:31 2012 -0700
+
+    osd: Make --get-journal-fsid not really start the osd.
+    
+    This way, it won't need -i ID and it won't access the osd_data_dir.
+    That makes it useful for locating the right osd to use with an
+    external journal partition.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit f65daf3f24830a28e85ebe7c6fce2642f5fc6dd8
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 09:08:56 2012 -0700
+
+    osd: Make --get-journal-fsid not attempt aio or direct_io.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 86546d7dec6cd5080162d1f3d7de2c579736dd65
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Oct 4 16:03:40 2012 -0700
+
+    ceph-disk-prepare: Use the OSD uuid as the partition GUID.
+    
+    This will make locating the right data partition for a given journal
+    partition a lot easier.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit fa3aa41ead0be5a27695e3c208c5af00af90bc53
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 12:38:38 2012 -0700
+
+    debian/control, ceph-disk-prepare: Depend on xfsprogs, use xfs by default.
+    
+    Ext4 as a default is a bad choice, as we don't perform enough QA with
+    it. To use XFS as the default for ceph-disk-prepare, we need to depend
+    on xfsprogs.
+    
+    btrfs-tools is already recommended, so no change there. If you set
+    osd_fs_type=btrfs, and don't have the package installed, you'll just
+    get an error message.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 4c3997dc8bbd3985a7b86d8c54b71fbe15af1591
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 10:13:17 2012 -0700
+
+    ceph-disk-{prepare,activate}: Default mkfs arguments and mount options.
+    
+    The values for the settings were copied from teuthology task "ceph".
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit a6229b901785ef0a88d1b1886b5b5ae339ac7fcb
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 08:47:20 2012 -0700
+
+    ceph-disk-prepare: Avoid triggering activate before prepare is done.
+    
+    Earlier testing never saw this, but now a mount of a disk triggers a
+    udev blockdev-added event, causing ceph-disk-activate to run even
+    before ceph-disk-prepare has had a chance to write the files and
+    unmount the disk.
+    
+    Avoid this by using a temporary partition type uuid ("ceph 2 be"), and
+    only setting it to the permanent ("ceph osd"). The hotplug event won't
+    match the type uuid, and thus won't trigger ceph-disk-activate.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 03cec77e2b0b4536349880de6890ad9537da0783
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 17:06:11 2012 -0700
+
+    ceph-disk-activate: Add a comment about user_xattr being default now.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 4aa13c6d228b9256870e02a57496f00a75e17349
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:53:35 2012 -0700
+
+    ceph-disk-activate: Use mount options from ceph.conf
+    
+    Always uses default cluster name ("ceph") for now, see
+    http://tracker.newdream.net/issues/3253
+    
+    Closes: #2548
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit b684ab1d1727b23f75f232123f1da947df2e5e63
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:43:08 2012 -0700
+
+    ceph-disk-activate: Refactor to extract detect_fstype call.
+    
+    This allows us to use the fstype for a config lookup.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit ce5e1f8dfaa8675b0a1e9fa22ebf91be9652aa01
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:37:07 2012 -0700
+
+    ceph-disk-activate: Unmount on errors (if it did the mount).
+    
+    This cleans up the error handling to not leave disks mounted
+    in /var/lib/ceph/tmp/mnt.* when something fails, e.g. when
+    the ceph command line tool can't talk to mons.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit da86e4e43676ceaf0b39635954489c297de88025
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:23:55 2012 -0700
+
+    ceph-disk-prepare: Allow setting mkfs arguments and mount options in ceph.conf
+    
+    Tested with meaningless but easy-to-verify values:
+    
+      [global]
+      osd_fs_type = xfs
+      osd_fs_mkfs_arguments_xfs = -i size=512
+      osd_fs_mount_options_xfs = noikeep
+    
+    ceph-disk-activate does not respect the mount options yet.
+    
+    Closes: #2549
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit cf2e3b15cea99cc100169e7a67103676d8990aee
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:04:15 2012 -0700
+
+    ceph-disk-prepare: Allow specifying fs type to use.
+    
+    Either use ceph.conf variable osd_fs_type or command line option
+    --fs-type=
+    
+    Default is still ext4, as currently nothing guarantees xfsprogs
+    or btrfs-tools are installed.
+    
+    Currently both btrfs and xfs seems to trigger a disk hotplug event at
+    mount time, thus triggering a useless and unwanted ceph-disk-activate
+    run. This will be worked around in a later commit.
+    
+    Currently mkfs and mount options cannot be configured.
+    
+    Bug: #2549
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit f031046c4d600ce0a0c2c9ad5856dc100b0901fb
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 26 15:43:56 2012 -0700
+
+    rgw: copy_object should not override ETAG implicitly
+    
+    When copying an object with new attrs, we still need to
+    maintain the ETAG.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2fddfb0a95288d39cee2e82376297ce43296743b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 24 18:10:24 2012 -0700
+
+    rgw: url_decode should allocate extra byte for dest
+    
+    Was missing extra byte for null termination
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.56.1.txt b/doc/changelog/v0.56.1.txt
new file mode 100644
index 000000000..e2cfe9bbc
--- /dev/null
+++ b/doc/changelog/v0.56.1.txt
@@ -0,0 +1,316 @@
+commit e4a541624df62ef353e754391cbbb707f54b16f7
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Jan 7 13:33:30 2013 -0800
+
+    v0.56.1
+
+commit 9aecacda7fbf07f12b210f87cf3dbb53021b068d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:38:27 2013 -0800
+
+    msg/Pipe: prepare Message data for wire under pipe_lock
+    
+    We cannot trust the Message bufferlists or other structures to be
+    stable without pipe_lock, as another Pipe may claim and modify the sent
+    list items while we are writing to the socket.
+    
+    Related to #3678.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d16ad9263d7b1d3c096f56c56e9631fae8509651)
+
+commit 299dbad490df5e98c04f17fa8e486a718f3c121f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:33:01 2013 -0800
+
+    msgr: update Message envelope in encode, not write_message
+    
+    Fill out the Message header, footer, and calculate CRCs during
+    encoding, not write_message().  This removes most modifications from
+    Pipe::write_message().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 40706afc66f485b2bd40b2b4b1cd5377244f8758)
+
+commit 35d2f58305eab6c9b57a92269598b9729e2d8681
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:25:40 2013 -0800
+
+    msg/Pipe: encode message inside pipe_lock
+    
+    This modifies bufferlists in the Message struct, and it is possible
+    for multiple instances of the Pipe to get references on the Message;
+    make sure they don't modify those bufferlists concurrently.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4cfc4903c6fb130b6ac9105baf1f66fbda797f14)
+
+commit 9b23f195df43589d062da95a11abc07c79f3109b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 10:39:08 2013 -0800
+
+    msg/Pipe: associate sending msgs to con inside lock
+    
+    Associate a sending message with the connection inside the pipe_lock.
+    This way if a racing thread tries to steal these messages it will
+    be sure to reset the con point *after* we do such that it the con
+    pointer is valid in encode_payload() (and later).
+    
+    This may be part of #3678.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a058f16113efa8f32eb5503d5443aa139754d479)
+
+commit 6229b5a06f449a470d3211ea94c1c5faf7100876
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 09:29:50 2013 -0800
+
+    msg/Pipe: fix msg leak in requeue_sent()
+    
+    The sent list owns a reference to each message.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2a1eb466d3f8e25ec8906b3ca6118a14c4e269d2)
+
+commit 6a00ce0dc24626fdfa210ddec6334bde3c8a20db
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 7 12:58:39 2013 -0800
+
+    osdc/Objecter: fix linger_ops iterator invalidation on pool deletion
+    
+    The call to check_linger_pool_dne() may unregister the linger request,
+    invalidating the iterator.  To avoid this, increment the iterator at
+    the top of the loop.
+    
+    This mirror the fix in 4bf9078286d58c2cd4e85cb8b31411220a377092 for
+    regular non-linger ops.
+    
+    Fixes: #3734
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 62586884afd56f2148205bdadc5a67037a750a9b)
+
+commit a10950f91e6ba9c1620d8fd00a84fc59f983fcee
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 20:53:49 2013 -0800
+
+    os/FileJournal: include limits.h
+    
+    Needed for IOV_MAX.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce49968938ca3636f48fe543111aa219f36914d8)
+
+commit cd194ef3c7082993cae0892a97494f2a917ce2a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 17:43:41 2013 -0800
+
+    osd: special case CALL op to not have RD bit effects
+    
+    In commit 20496b8d2b2c3779a771695c6f778abbdb66d92a we treat a CALL as
+    different from a normal "read", but we did not adjust the behavior
+    determined by the RD bit in the op.  We tried to fix that in
+    91e941aef9f55425cc12204146f26d79c444cfae, but changing the op code breaks
+    compatibility, so that was reverted.
+    
+    Instead, special-case CALL in the helper--the only point in the code that
+    actually checks for the RD bit.  (And fix one lingering user to use that
+    helper appropriately.)
+    
+    Fixes: #3731
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 988a52173522e9a410ba975a4e8b7c25c7801123)
+
+commit 921e06decebccc913c0e4f61916d00e62e7e1635
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 20:46:48 2013 -0800
+
+    Revert "OSD: remove RD flag from CALL ops"
+    
+    This reverts commit 91e941aef9f55425cc12204146f26d79c444cfae.
+    
+    We cannot change this op code without breaking compatibility
+    with old code (client and server).  We'll have to special case
+    this op code instead.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit d3abd0fe0bb402ff403259d4b1a718a56331fc39)
+
+commit 7513e9719a532dc538d838f68e47c83cc51fef82
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 4 12:43:52 2013 -0800
+
+    ReplicatedPG: remove old-head optization from push_to_replica
+    
+    This optimization allowed the primary to push a clone as a single push in the
+    case that the head object on the replica is old and happens to be at the same
+    version as the clone.  In general, using head in clone_subsets is tricky since
+    we might be writing to head during the push.  calc_clone_subsets does not
+    consider head (probably for this reason).  Handling the clone from head case
+    properly would require blocking writes on head in the interim which is probably
+    a bad trade off anyway.
+    
+    Because the old-head optimization only comes into play if the replica's state
+    happens to fall on the last write to head prior to the snap that caused the
+    clone in question, it's not worth the complexity.
+    
+    Fixes: #3698
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e89b6ade63cdad315ab754789de24008cfe42b37)
+
+commit c63c66463a567e8095711e7c853ac8feb065c5c5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 3 17:15:07 2013 -0800
+
+    os/FileStore: fix non-btrfs op_seq commit order
+    
+    The op_seq file is the starting point for journal replay.  For stable btrfs
+    commit mode, which is using a snapshot as a reference, we should write this
+    file before we take the snap.  We normally ignore current/ contents anyway.
+    
+    On non-btrfs file systems, however, we should only write this file *after*
+    we do a full sync, and we should then fsync(2) it before we continue
+    (and potentially trim anything from the journal).
+    
+    This fixes a serious bug that could cause data loss and corruption after
+    a power loss event.  For a 'kill -9' or crash, however, there was little
+    risk, since the writes were still captured by the host's cache.
+    
+    Fixes: #3721
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 28d59d374b28629a230d36b93e60a8474c902aa5)
+
+commit b8f061dcdb808a6fc5ec01535b37560147b537de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 3 09:59:45 2013 -0800
+
+    OSD: for old osds, dispatch peering messages immediately
+    
+    Normally, we batch up peering messages until the end of
+    process_peering_events to allow us to combine many notifies, etc
+    to the same osd into the same message.  However, old osds assume
+    that the actiavtion message (log or info) will be _dispatched
+    before the first sub_op_modify of the interval.  Thus, for those
+    peers, we need to send the peering messages before we drop the
+    pg lock, lest we issue a client repop from another thread before
+    activation message is sent.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4ae4dce5c5bb547c1ff54d07c8b70d287490cae9)
+
+commit 67968d115daf51762dce65af46b9b843eda592b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 22:38:53 2013 -0800
+
+    osd: move common active vs booting code into consume_map
+    
+    Push osdmaps to PGs in separate method from activate_map() (whose name
+    is becoming less and less accurate).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a32d6c5dca081dcd8266f4ab51581ed6b2755685)
+
+commit 34266e6bde9f36b1c46144d2341b13605eaa9abe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 22:20:06 2013 -0800
+
+    osd: let pgs process map advances before booting
+    
+    The OSD deliberate consumes and processes most OSDMaps from while it
+    was down before it marks itself up, as this is can be slow.  The new
+    threading code does this asynchronously in peering_wq, though, and
+    does not let it drain before booting the OSD.  The OSD can get into
+    a situation where it marks itself up but is not responsive or useful
+    because of the backlog, and only makes the situation works by
+    generating more osdmaps as result.
+    
+    Fix this by calling activate_map() even when booting, and when booting
+    draining the peering_wq on each call.  This is harmless since we are
+    not yet processing actual ops; we only need to be async when active.
+    
+    Fixes: #3714
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0bfad8ef2040a0dd4a0dc1d3abf3ab5b2019d179)
+
+commit 4034f6c817d1efce5fb9eb8cc0a9327f9f7d7910
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 13:07:18 2012 -0800
+
+    log: broadcast cond signals
+    
+    We were using a single cond, and only signalling one waiter.  That means
+    that if the flusher and several logging threads are waiting, and we hit
+    a limit, we the logger could signal another logger instead of the flusher,
+    and we could deadlock.
+    
+    Similarly, if the flusher empties the queue, it might signal only a single
+    logger, and that logger could re-signal the flusher, and the other logger
+    could wait forever.
+    
+    Intead, break the single cond into two: one for loggers, and one for the
+    flusher.  Always signal the (one) flusher, and always broadcast to all
+    loggers.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 813787af3dbb99e42f481af670c4bb0e254e4432)
+
+commit 2141454eee3a1727706d48f8efef92f8a2b98278
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 13:58:44 2013 -0800
+
+    log: fix locking typo/stupid for dump_recent()
+    
+    We weren't locking m_flush_mutex properly, which in turn was leading to
+    racing threads calling dump_recent() and garbling the crash dump output.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 43cba617aa0247d714632bddf31b9271ef3a1b50)
+
+commit 936560137516a1fd5e55b52ccab59c408ac2c245
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:48:22 2012 -0800
+
+    test_filejournal: optionally specify journal filename as an argument
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 483c6f76adf960017614a8641c4dcdbd7902ce33)
+
+commit be0473bbb1feb8705be4fa8f827704694303a930
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:48:05 2012 -0800
+
+    test_filejournal: test journaling bl with >IOV_MAX segments
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c461e7fc1e34fdddd8ff8833693d067451df906b)
+
+commit de61932793c5791c770855e470e3b5b9ebb53dba
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:47:28 2012 -0800
+
+    os/FileJournal: limit size of aio submission
+    
+    Limit size of each aio submission to IOV_MAX-1 (to be safe).  Take care to
+    only mark the last aio with the seq to signal completion.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dda7b651895ab392db08e98bf621768fd77540f0)
+
+commit ded454c669171d4038b087cfdad52a57da222c1f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 15:44:51 2012 -0800
+
+    os/FileJournal: logger is optional
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 076b418c7f03c5c62f811fdc566e4e2b776389b7)
diff --git a/doc/changelog/v0.56.2.txt b/doc/changelog/v0.56.2.txt
new file mode 100644
index 000000000..cd8e402f7
--- /dev/null
+++ b/doc/changelog/v0.56.2.txt
@@ -0,0 +1,1294 @@
+commit 586538e22afba85c59beda49789ec42024e7a061
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Jan 29 23:54:47 2013 -0800
+
+    v0.56.2
+
+commit bcb8dfad9cbb4c6af7ae7f9584e36449a03cd1b6
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 29 23:05:49 2013 -0800
+
+    cls_rbd, cls_rgw: use PRI*64 when printing/logging 64-bit values
+
+    caused segfaults in 32-bit build
+
+    Fixes: #3961
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e253830abac76af03c63239302691f7fac1af381)
+
+commit 5a7c5088cc8f57f75eb594a21bf5fb6661e50978
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 29 15:18:53 2013 -0800
+
+    init-ceph: make ulimit -n be part of daemon command
+
+    ulimit -n from 'max open files' was being set only on the machine
+    running /etc/init.d/ceph.  It needs to be added to the commands to
+    start the daemons, and run both locally and remotely.
+
+    Verified by examining /proc/<pid>/limits on local and remote hosts
+
+    Fixes: #3900
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Loïc Dachary <loic@dachary.org>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 84a024b647c0ac2ee5a91bacdd4b8c966e44175c)
+
+commit 95677fc599b9bf37ab4c2037b3675fd68f92ebcf
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sat Jan 12 01:06:36 2013 +0000
+
+    mon: OSDMonitor: only share osdmap with up OSDs
+
+    Try to share the map with a randomly picked OSD; if the picked monitor is
+    not 'up', then try to find the nearest 'up' OSD in the map by doing a
+    backward and a forward linear search on the map -- this would be O(n) in
+    the worst case scenario, as we only do a single iteration starting on the
+    picked position, incrementing and decrementing two different iterators
+    until we find an appropriate OSD or we exhaust the map.
+
+    Fixes: #3629
+    Backport: bobtail
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3610e72e4f9117af712f34a2e12c5e9537a5746f)
+
+commit e4d76cb8594c0ec901f89c2f2e8cc53e00eb2a06
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sun Jan 27 21:57:31 2013 +0100
+
+    utime: fix narrowing conversion compiler warning in sleep()
+
+    Fix compiler warning:
+    ./include/utime.h: In member function 'void utime_t::sleep()':
+    ./include/utime.h:139:50: warning: narrowing conversion of
+     '((utime_t*)this)->utime_t::tv.utime_t::<anonymous struct>::tv_sec' from
+     '__u32 {aka unsigned int}' to '__time_t {aka long int}' inside { } is
+     ill-formed in C++11 [-Wnarrowing]
+    ./include/utime.h:139:50: warning: narrowing conversion of
+     '((utime_t*)this)->utime_t::tv.utime_t::<anonymous struct>::tv_nsec' from
+     '__u32 {aka unsigned int}' to 'long int' inside { } is
+     ill-formed in C++11 [-Wnarrowing]
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 014fc6d6c1c68e2e3ad0117d08c4e46e4030d49e)
+
+commit a8964107ddf02ac4a6707a997e1b634c1084a3b9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 28 17:13:23 2013 -0800
+
+    rgw: fix crash when missing content-type in POST object
+
+    Fixes: #3941
+    This fixes a crash when handling S3 POST request and content type
+    is not provided.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f41010c44b3a4489525d25cd35084a168dc5f537)
+
+commit 11e1f3acf0953e9ac38322c0423144eaabd7bb61
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 15:00:02 2013 -0800
+
+    ReplicatedPG: make_snap_collection when moving snap link in snap_trimmer
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 88956e3186798058a1170803f8abfc0f3cf77a07)
+
+commit c9201d0e9de5f4766a2d9f4715eb7c69691964de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 16:43:14 2013 -0800
+
+    ReplicatedPG: correctly handle new snap collections on replica
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9e44fca13bf1ba39dbcad29111b29f46c49d59f7)
+
+commit 2efdfb41c1bc9128b76416630ee00a75de90c020
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Jan 27 18:08:15 2013 +0000
+
+    mon: Elector: reset the acked leader when the election finishes and we lost
+
+    Failure to do so will mean that we will always ack the same leader during
+    an election started by another monitor.  This had been working so far
+    because we were still acking the existing leader if he was supposed to
+    still be the leader; or we were acking a new potentially leader; or we
+    would eventually fall behind on an election and start a new election
+    ourselves, thus resetting the previously acked leader.  While this wasn't
+    something that mattered much until now, the timechecks code stumbled into
+    this tiny issue and was failing hard at completing a round because there
+    wouldn't be a reset before the election started -- timechecks are bound
+    to election epochs.
+
+    Fixes: #3854
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c54781618569680898e77e151dd7364f22ac4aa1)
+
+commit a16c6f3dc278e19e66776ffde45de3ff0db46a6c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Dec 26 14:24:22 2012 -0800
+
+    rbd: fix bench-write infinite loop
+
+    I/O was continously submitted as long as there were few enough ops in
+    flight. If the number of 'threads' was high, or caching was turned on,
+    there would never be that many ops in flight, so the loop would continue
+    indefinitely. Instead, submit at most io_threads ops per offset.
+
+    Fixes: #3413
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit d81ac8418f9e6bbc9adcc69b2e7cb98dd4db6abb)
+
+commit 76f93751d3603e3fb5c4b9e14bfdac406d8d1a58
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri Jan 4 18:00:24 2013 -0800
+
+    rbd: Don't call ProgressContext's finish() if there's an error.
+
+    do_copy was different from the others; call pc.fail() on error and
+    do not call pc.finish().
+
+    Fixes: #3729
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 0978dc4963fe441fb67afecb074bc7b01798d59d)
+
+commit 10053b14623f9c19727cb4d2d3a6b62945bef5c1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jan 2 14:15:24 2013 -0800
+
+    librbd: establish watch before reading header
+
+    This eliminates a window in which a race could occur when we have an
+    image open but no watch established. The previous fix (using
+    assert_version) did not work well with resend operations.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c4370ff03f8ab655a009cfd9ba3a0827d8c58b11)
+
+commit f666c617f6a5f8d94ce81461942c9f94a0775fb2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jan 2 12:32:33 2013 -0800
+
+    Revert "librbd: ensure header is up to date after initial read"
+
+    Using assert version for linger ops doesn't work with retries,
+    since the version will change after the first send.
+    This reverts commit e1776809031c6dad441cfb2b9fac9612720b9083.
+
+    Conflicts:
+
+	qa/workunits/rbd/watch_correct_version.sh
+    (cherry picked from commit e0858fa89903cf4055889c405f17515504e917a0)
+
+commit 575a58666adbca83d15468899272e8c369e903e1
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 23 22:16:49 2013 -0800
+
+    os/FileStore: only adjust up op queue for btrfs
+
+    We only need to adjust up the op queue limits during commit for btrfs,
+    because the snapshot initiation (async create) is currently
+    high-latency and the op queue is quiesced during that period.
+
+    This lets us revert 44dca5c, which disabled the extra allowance because
+    it is generally bad for non-btrfs writeahead mode.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 38871e27eca5a34de78db23aa3663f6cb045d461)
+
+commit c9eb1b0a99b0e55f7d7343176dad17d1a53589a1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 24 10:52:46 2013 -0800
+
+    common/HeartbeatMap: fix uninitialized variable
+
+    Introduced by me in 132045ce085e8584a3e177af552ee7a5205b13d8.  Thank you,
+    valgrind!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 00cfe1d3af286ffab7660933415684f18449720c)
+
+commit e6bceeedb0b77d23416560bd951326587470aacb
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 25 11:31:29 2013 -0800
+
+    sharedptr_registry: remove extaneous Mutex::Locker declaration
+
+    For some reason, the lookup() retry loop (for when happened to
+    race with a removal and grab an invalid WeakPtr) locked
+    the lock again.  This causes the #3836 crash since the lock
+    is already locked.  It's rare since it requires a lookup between
+    invalidation of the WeakPtr and removal of the WeakPtr entry.
+
+    Fixes: #3836
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 037900dc7a051ce2293a4ef9d0e71911b29ec159)
+
+commit 60888cafdc53d6b381cd634170646c12669e1754
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 12:02:09 2013 -0800
+
+    FileStore: ping TPHandle after each operation in _do_transactions
+
+    Each completed operation in the transaction proves thread
+    liveness, a stuck thread should still trigger the timeouts.
+
+    Fixes: #3928
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0c1cc687b6a40d3c6a26671f0652e1b51c3fd1af)
+
+commit 6b8a673f88cbaca2891834dd5d2137a0e076fd1e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 11:07:37 2013 -0800
+
+    OSD: use TPHandle in peering_wq
+
+    Implement _process overload with TPHandle argument and use
+    that to ping the hb map between pgs and between map epochs
+    when advancing a pg.  The thread will still timeout if
+    genuinely stuck at any point.
+
+    Fixes: 3905
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e0511f4f4773766d04e845af2d079f82f3177cb6)
+
+commit aa6d20aac22d4c14ff059dbc28e06b7a5e5d6de1
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 11:04:04 2013 -0800
+
+    WorkQueue: add TPHandle to allow _process to ping the hb map
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4f653d23999b24fc8c65a59f14905db6630be5b5)
+
+commit e66a75052a340b15693f08b05f7f9f5d975b0978
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:49:04 2013 -0800
+
+    ReplicatedPG: handle omap > max_recovery_chunk
+
+    span_of fails if len == 0.
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8a97eef1f7004988449bd7ace4c69d5796495139)
+
+commit 44f0407a6b259e87803539ec9e942043de0cf35d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:18:31 2013 -0800
+
+    ReplicatedPG: correctly handle omap key larger than max chunk
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c3dec3e30a85ecad0090c75a38f28cb83e36232e)
+
+commit 50fd6ac9f147a4418d64dfe08843402e7cfb4910
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:15:10 2013 -0800
+
+    ReplicatedPG: start scanning omap at omap_recovered_to
+
+    Previously, we started scanning omap after omap_recovered_to.
+    This is a problem since the break in the loop implies that
+    omap_recovered_to is the first key not recovered.
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 09c71f2f5ee9929ac4574f4c35fb8c0211aad097)
+
+commit 4b32eecba2e2bd8e8ea17e1888e6971d31e71439
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:50:13 2013 -0800
+
+    ReplicatedPG: don't finish_recovery_op until the transaction completes
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 62a4b96831c1726043699db86a664dc6a0af8637)
+
+commit da34c77b93e3f880c01329711ab8eca7776b1830
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:35:47 2013 -0800
+
+    ReplicatedPG: ack push only after transaction has completed
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 20278c4f77b890d5b2b95d2ccbeb4fbe106667ac)
+
+commit f9381c74931b80294e5df60f6d2e69c946b8fe88
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:13:28 2013 -0800
+
+    ObjectStore: add queue_transactions with oncomplete
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4d6ba06309b80fb21de7bb5d12d5482e71de5f16)
+
+commit e2560554f0568c30c786632723c5ce0c86043359
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 21:18:45 2013 -0800
+
+    common/HeartbeatMap: inject unhealthy heartbeat for N seconds
+
+    This lets us test code that is triggered by an unhealthy heartbeat in a
+    generic way.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 132045ce085e8584a3e177af552ee7a5205b13d8)
+
+commit cbe8b5bca40fd63a382b1a903087e7c34b314985
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:08:22 2013 -0800
+
+    os/FileStore: add stall injection into filestore op queue
+
+    Allow admin to artificially induce a stall in the op queue.  Forces the
+    thread(s) to sleep for N seconds.  We pause for 1 second increments and
+    recheck the value so that a previously stalled thread can be unwedged by
+    reinjecting a lower value (or 0).  To stall indefinitely, just injust
+    very large number.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 657df852e9c89bfacdbce25ea014f7830d61e6aa)
+
+commit beb6ca44cd0e7fc405360e6da974252cb76e7039
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:03:10 2013 -0800
+
+    osd: do not join cluster if not healthy
+
+    If our internal heartbeats are failing, do not send a boot message and try
+    to join the cluster.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a4e78652cdd1698e8dd72dda51599348d013e5e0)
+
+commit 1ecdfca3a3b4985ebd182a5f399c7b15af258663
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:01:07 2013 -0800
+
+    osd: hold lock while calling start_boot on startup
+
+    This probably doesn't strictly matter because start_boot doesn't need the
+    lock (currently) and few other threads should be running, but it is
+    better to be consistent.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c406476c0309792c43df512dddb2fe0f19835e71)
+
+commit e120bf20b3c7213fbde20907e158792dd36c8e54
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 17:56:32 2013 -0800
+
+    osd: do not reply to ping if internal heartbeat is not healthy
+
+    If we find that our internal threads are stalled, do not reply to ping
+    requests.  If we do this long enough, peers will mark us down.  If we are
+    only transiently unhealthy, we will reply to the next ping and they will
+    be satisfied.  If we are unhealthy and marked down, and eventually recover,
+    we will mark ourselves back up.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad6b231127a6bfcbed600a7493ca3b66c68484d2)
+
+commit 5f396e2b9360401dfe4dc2afa6acc37df8580c80
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 17:53:40 2013 -0800
+
+    osd: reduce op thread heartbeat default 30 -> 15 seconds
+
+    If the thread stalls for 15 seconds, let our internal heartbeat fail.
+    This will let us internally respond more quickly to a stalled or failing
+    disk.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 61eafffc3242357d9add48be9308222085536898)
+
+commit fca288b718ef4582d65ff4b9d1fc87ba53d7fd8d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 21:02:01 2013 -0800
+
+    osd: improve sub_op flag points
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 73a969366c8bbd105579611320c43e2334907fef)
+
+commit f13ddc8a2df401c37f6dc792eb93fc0cc45705e2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 20:55:20 2013 -0800
+
+    osd: refactor ReplicatedPG::do_sub_op
+
+    PULL is the only case where we don't wait for active.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 23c02bce90c9725ccaf4295de3177e8146157723)
+
+commit d5e00f963f177745f0e0684d5977460b7ab59fbd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 16:36:36 2013 -0800
+
+    osd: make last state for slow requests more informative
+
+    Report on the last event string, and pass in important context for the
+    op event list, including:
+
+     - which peers were sent sub ops and we are waiting for
+     - which pg queue we are delayed by
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a1137eb3e168c2d00f93789e4d565c1584790df0)
+
+commit ab3a110cbe16b548bb96225656b64507aa67e78f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:59:07 2013 -0800
+
+    osd: dump op priority queue state via admin socket
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 24d0d7eb0165c8b8f923f2d8896b156bfb5e0e60)
+
+commit 43a65d04d8a13621a856baec85fb741971c13cb0
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:50:33 2013 -0800
+
+    osd: simplify asok to single callback
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 33efe32151e04beaafd9435d7f86dc2eb046214d)
+
+commit d040798637da03e3df937181de156714fc62a550
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:58:57 2013 -0800
+
+    common/PrioritizedQueue: dump state to Formatter
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 514af15e95604bd241d2a98a97b938889c6876db)
+
+commit 691fd505ad606bd8befd2b19113ee51a17a0a543
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:29:28 2013 -0800
+
+    common/PrioritizedQueue: add min cost, max tokens per bucket
+
+    Two problems.
+
+    First, we need to cap the tokens per bucket.  Otherwise, a stream of
+    items at one priority over time will indefinitely inflate the tokens
+    available at another priority.  The cap should represent how "bursty"
+    we allow a given bucket to be.  Start with 4MB for now.
+
+    Second, set a floor on the item cost.  Otherwise, we can have an
+    infinite queue of 0 cost items that start over queues.  More
+    realistically, we need to balance the overhead of processing small items
+    with the cost of large items.  I.e., a 4 KB item is not 1/1000th as
+    expensive as a 4MB item.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6e3363b20e590cd9df89f2caebe71867b94cc291)
+
+commit a2b03fe08044b5c121ea6b4c2f9d19e73e4c83d1
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:52:54 2013 -0800
+
+    common/PrioritizedQueue: buckets -> tokens
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c549a0cf6fae78c8418a3b4b0702fd8a1e4ce482)
+
+commit 612d75cdee0daf9dfca97831c249e1ac3fbd59fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:31:00 2013 -0800
+
+    note puller's max chunk in pull requests
+
+    this lets us calculate a cost value
+    (cherry picked from commit 128fcfcac7d3fb66ca2c799df521591a98b82e05)
+
+commit 2224e413fba11795693025fa8f11c3f1fba4bbaa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:14:25 2013 -0800
+
+    osd: add OpRequest flag point when commit is sent
+
+    With writeahead journaling in particular, we can get requests that
+    stay in the queue for a long time even after the commit is sent to the
+    client while we are waiting for the transaction to apply to the fs.
+    Instead of showing up as 'waiting for subops', make it clear that the
+    client has gotten its reply and it is local state that is slow.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b685f727d4c37a26cb78bd4a04cce041428ceb52)
+
+commit 5b5ca5926258e4f0b5041fb2c15b1c2f904c4adb
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:57:59 2013 -0800
+
+    osd: set PULL subop cost to size of requested data
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a1bf8220e545f29b83d965f07b1abfbea06238b3)
+
+commit 10651e4f500d7b55d8c689a10a61d2239b3ecd26
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:57:38 2013 -0800
+
+    osd: use Message::get_cost() function for queueing
+
+    The data payload is a decent proxy for cost in most cases, but not all.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8e0da1a577e24cd4aad71fb94d8b244e2ac7300)
+
+commit 9735c6b163f4d226d8de6508d5c1534d18f1c300
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:25:21 2013 -0800
+
+    osd: debug msg prio, cost, latency
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bec96a234c160bebd9fd295df5b431dc70a2cfb3)
+
+commit c48279da7ad98013ce97eab89c17fe9fae1ba866
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 21:05:00 2013 -0800
+
+    filestore: filestore_queue_max_ops 500 -> 50
+
+    Having a deep queue limits the effectiveness of the priority queues
+    above by adding additional latency.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 40654d6d53436c210b2f80911217b044f4d7643a)
+
+commit f47b2e8b607cc0d56a42ec7b1465ce6b8c0ca68c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 20:00:26 2013 -0800
+
+    osd: target transaction size 300 -> 30
+
+    Small transactions make pg removal nicer to the op queue.  It also slows
+    down PG deletion a bit, which may exacerbate the PG resurrection case
+    until #3884 is addressed.
+
+    At least on user reported this fixed an osd that kept failing due to
+    an internal heartbeat failure.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1233e8617098766c95100aa9a6a07db1a688e290)
+
+commit 4947f0efadf9ef209d02fd17f5f86b9a7d6523ef
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 19:55:26 2013 -0800
+
+    os/FileStore: allow filestore_queue_max_{ops,bytes} to be adjusted at runtime
+
+    The 'committing' ones too.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cfe4b8519363f92f84f724a812aa41257402865f)
+
+commit ad6e6c91f61c092bfc9f88b788ccbee6438fd40b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 22:06:27 2013 -0800
+
+    osd: make osd_max_backfills dynamically adjustable
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 101955a6b8bfdf91f4229f4ecb5d5b3da096e160)
+
+commit 939b1855245bc9cb31f5762027f2ed3f2317eb55
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 18:28:35 2013 -0800
+
+    osd: make OSD a config observer
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9230c863b3dc2bdda12c23202682a84c48f070a1)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit b0f27a8f81feb401407bed784bf5d4d799998ee0
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 8 11:21:22 2013 -0800
+
+    librbd: Allow get_lock_info to fail
+
+    If the lock class isn't present, EOPNOTSUPP is returned for lock calls
+    on newer OSDs, but sadly EIO on older; we need to treat both as
+    acceptable failures for RBD images.  rados lock list will still fail.
+
+    Fixes #3744.
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4483285c9fb16f09986e2e48b855cd3db869e33c)
+
+commit 022a5254b4fac3f76220abdde2a2e81de33cb8dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 13:00:56 2013 -0800
+
+    osd: drop newlines from event descriptions
+
+    These produce extra newlines in the log.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 9a1f574283804faa6dbba9165a40558e1a6a1f13)
+
+commit ebc93a878c8b0697004a619d6aa957a80b8b7e35
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 18 14:35:51 2013 -0800
+
+    OSD: do deep_scrub for repair
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 0cb760f31b0cb26f022fe8b9341e41cd5351afac)
+
+commit 32527fa3eb48a7d7d5d67c39bfa05087dbc0e41b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jan 14 12:52:04 2013 -0800
+
+    ReplicatedPG: ignore snap link info in scrub if nlinks==0
+
+    links==0 implies that the replica did not sent snap link information.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 70c3512037596a42ba6eb5eb7f96238843095db9)
+
+commit 13e42265db150b19511a5a618c7a95ad801290c8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 11 12:25:22 2013 -0800
+
+    osd/PG: fix osd id in error message on snap collection errors
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 381e25870f26fad144ecc2fb99710498e3a7a1d4)
+
+commit e3b6191fc45c7d2c27ec75c867be822a6da17e9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 9 22:34:12 2013 -0800
+
+    osd/ReplicatedPG: validate ino when scrubbing snap collections
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 665577a88b98390b9db0f9991836d10ebdd8f4cf)
+
+commit 353b7341caff86f936a429669de52e6949a89c2b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 16:41:40 2013 -0800
+
+    ReplicatedPG: compare nlinks to snapcolls
+
+    nlinks gives us the number of hardlinks to the object.
+    nlinks should be 1 + snapcolls.size().  This will allow
+    us to detect links which remain in an erroneous snap
+    collection.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e65ea70ea64025fbb0709ee8596bb2878be0bbdc)
+
+commit 33d5cfc8c080a270d65275f8e010a6468c77381a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 10 15:35:10 2013 -0800
+
+    ReplicatedPG/PG: check snap collections during _scan_list
+
+    During _scan_list check the snapcollections corresponding to the
+    object_info attr on the object.  Report inconsistencies during
+    scrub_finalize.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 57352351bb86e0ae9f64f9ba0d460c532d882de6)
+
+commit bea783bd722d862a5018477a637c843fe4b18a58
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 11:53:52 2013 -0800
+
+    osd_types: add nlink and snapcolls fields to ScrubMap::object
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b85687475fa2ec74e5429d92ee64eda2051a256c)
+
+commit 0c48407bf46b39b2264a7be14e9d3caa2c1e5875
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 3 20:16:50 2013 -0800
+
+    PG: move auth replica selection to helper in scrub
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 39bc65492af1bf1da481a8ea0a70fe7d0b4b17a3)
+
+commit c3433ce60ec3683217d8b4cd2b6e75fb749af2c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:23:52 2013 -0800
+
+    mon: note scrub errors in health summary
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8e33a8b9e1fef757bbd901d55893e9b84ce6f3fc)
+
+commit 90c6edd0155b327c48a5b178d848d9e5839bd928
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:31:06 2013 -0800
+
+    osd: fix rescrub after repair
+
+    We were rescrubbing if INCONSISTENT is set, but that is now persistent.
+    Add a new scrub_after_recovery flag that is reset on each peering interval
+    and set that when repair encounters errors.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a586966a3cfb10b5ffec0e9140053a7e4ff105d2)
+
+commit 0696cf57283e6e9a3500c56ca5fc9f981475ca26
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:22:02 2013 -0800
+
+    osd: note must_scrub* flags in PG operator<<
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d56af797f996ac92bf4e0886d416fd358a2aa08e)
+
+commit 1541ffe4bec6cce607c505271ff074fd0a292d30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:21:46 2013 -0800
+
+    osd: based INCONSISTENT pg state on persistent scrub errors
+
+    This makes the state persistent across PG peering and OSD restarts.
+
+    This has the side-effect that, on recovery, we rescrub any PGs marked
+    inconsistent.  This is new behavior!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2baf1253eed630a7c4ae4cb43aab6475efd82425)
+
+commit 609101255c81d977072b2ab741ac47167d9b1b16
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:20:29 2013 -0800
+
+    osd: fix scrub scheduling for 0.0
+
+    The initial value for pair<utime_t,pg_t> can match pg 0.0, preventing it
+    from being manually scrubbed.  Fix!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 26a63df97b2a12fd1a7c1e3cc9ccd34ca2ef9834)
+
+commit 0961a3a85c286a31ec2e8bba23217bbd3974572c
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 23:03:01 2013 -0800
+
+    osd: note last_clean_scrub_stamp, last_scrub_errors
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 389bed5d338cf32ab14c9fc2abbc7bcc386b8a28)
+
+commit 8d823045538bf4c51506e349b5c6705fd76450f8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:59:39 2013 -0800
+
+    osd: add num_scrub_errors to object_stat_t
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2475066c3247774a2ad048a2e32968e47da1b0f5)
+
+commit 3a1cd6e07b4e2a4714de159f69afd689495e2927
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:43:35 2013 -0800
+
+    osd: add last_clean_scrub_stamp to pg_stat_t, pg_history_t
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d738328488de831bf090f23e3fa6d25f6fa819df)
+
+commit 7e5a899bdcf6c08a5f6f5c98cd2fff7fa2dacaca
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:56:14 2013 -0800
+
+    osd: fix object_stat_sum_t dump signedness
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6f6a41937f1bd05260a8d70b4c4a58ecadb34a2f)
+
+commit e252a313d465006d3fe4db97939ad307ebe91c71
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:04:58 2013 -0800
+
+    osd: change scrub min/max thresholds
+
+    The previous 'osd scrub min interval' was mostly meaningless and useless.
+    Meanwhile, the 'osd scrub max interval' would only trigger a scrub if the
+    load was sufficiently low; if it was high, the PG might *never* scrub.
+
+    Instead, make the 'min' what the max used to be.  If it has been more than
+    this many seconds, and the load is low, scrub.  And add an additional
+    condition that if it has been more than the max threshold, scrub the PG
+    no matter what--regardless of the load.
+
+    Note that this does not change the default scrub interval for less-loaded
+    clusters, but it *does* change the meaning of existing config options.
+
+    Fixes: #3786
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 299548024acbf8123a4e488424c06e16365fba5a)
+
+    Conflicts:
+
+	PendingReleaseNotes
+
+commit 33aa64eee34f4759f6000130de4d1306de49d087
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 20:27:59 2013 -0800
+
+    osd/PG: remove useless osd_scrub_min_interval check
+
+    This was already a no-op: we don't call PG::scrub_sched() unless it has
+    been osd_scrub_max_interval seconds since we last scrubbed.  Unless we
+    explicitly requested in, in which case we don't want this check anyway.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 16d67c798b6f752a6e03084bafe861396b86baae)
+
+commit fdd0c1ec3519376980a205b94e65187833634e2e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 20:25:39 2013 -0800
+
+    osd: move scrub schedule random backoff to seperate helper
+
+    Separate this from the load check, which will soon vary dependon on the
+    PG.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a148120776d0930b265411332a60e93abfbf0423)
+
+commit 9ffbe268f785e1a74c0d893735117edb7a3ef377
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:18:38 2013 -0800
+
+    osd/PG: trigger scrub via scrub schedule, must_ flags
+
+    When a scrub is requested, flag it and move it to the front of the
+    scrub schedule instead of immediately queuing it.  This avoids
+    bypassing the scrub reservation framework, which can lead to a heavier
+    impact on performance.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 62ee6e099a8e4873287b54f9bba303ea9523d040)
+
+commit cffb1b22d5df7300ec411d2b620bf3c4a08351cd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:15:16 2013 -0800
+
+    osd/PG: introduce flags to indicate explicitly requested scrubs
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1441095d6babfacd781929e8a54ed2f8a4444467)
+
+commit 438e3dfc88bfdc8eb36b5b5f7b728b2610476724
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:14:01 2013 -0800
+
+    osd/PG: move scrub schedule registration into a helper
+
+    Simplifies callers, and will let us easily modify the decision of when
+    to schedule the PG for scrub.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 796907e2159371f84a16cbd35f6caa8ac868acf6)
+
+commit acb47e4d7dc9682937984661a9d754131d806630
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 12:14:48 2013 -0800
+
+    os/FileStore: only flush inline if write is sufficiently large
+
+    Honor filestore_flush_min in the inline flush case.
+
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 49726dcf973c38c7313ab78743b45ccc879671ea)
+
+commit 15a1ced859629c361da127799b05620bee84c9a8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 12:14:40 2013 -0800
+
+    os/FileStore: fix compile when sync_file_range is missing;
+
+    If sync_file_range is not present, we always close inline, and flush
+    via fdatasync(2).
+
+    Fixes compile on ancient platforms like RHEL5.8.
+
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8ddb55d34c72e6df1023cf427cbd41f3f98da402)
+
+commit 9dddb9d855e6d5fd804b54bff1f726c1d2fb566c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:23:22 2013 -0800
+
+    osd: set pg removal transactions based on configurable
+
+    Use the osd_target_transaction_size knob, and gracefully tolerate bogus
+    values (e.g., <= 0).
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e00af406b89c9817e9a429f92a05ca9c29b19c3)
+
+commit c30d231e40a17c3fb08d1db5e01133466170e90c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:30:06 2013 -0800
+
+    osd: make pg removal thread more friendly
+
+    For a large PG these are saturating the filestore and journal queues.  Do
+    them synchronously to make them more friendly.  They don't need to be fast.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4712e984d3f62cdf51ea67da8197eed18a5983dd)
+
+commit b2bc4b95fefaeb0cfc31ce0bc95b77062d0777c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:27:24 2013 -0800
+
+    os: move apply_transactions() sync wrapper into ObjectStore
+
+    This has nothing to do with the backend implementation.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bc994045ad67fb70c7a0457b8cd29273dd5d1654)
+
+commit 6d161b57979246ddea4e6309e0e489ab729eec4b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:28:24 2013 -0800
+
+    os: add apply_transaction() variant that takes a sequencer
+
+    Also, move the convenience wrappers into the interface and funnel through
+    a single implementation.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f6c69c3f1ac35546b90315fff625993ba5cd8c07)
+
+commit c5fe0965572c074a2a33660719ce3222d18c1464
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 20 16:11:10 2013 -0800
+
+    osd: calculate initial PG mapping from PG's osdmap
+
+    The initial values of up/acting need to be based on the PG's osdmap, not
+    the OSD's latest.  This can cause various confusion in
+    pg_interval_t::check_new_interval() when calling OSDMap methods due to the
+    up/acting OSDs not existing yet (for example).
+
+    Fixes: #3879
+    Reported-by: Jens Kristian S?gaard <jens@mermaidconsulting.dk>
+    Tested-by: Jens Kristian S?gaard <jens@mermaidconsulting.dk>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 17160843d0c523359d8fa934418ff2c1f7bffb25)
+
+commit 6008b1d8e4587d5a3aea60684b1d871401496942
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 17 15:01:35 2013 -0800
+
+    osdmap: make replica separate in default crush map configurable
+
+    Add 'osd crush chooseleaf type' option to control what the default
+    CRUSH rule separates replicas across.  Default to 1 (host), and set it
+    to 0 in vstart.sh.
+
+    Fixes: #3785
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c236a51a8040508ee893e4c64b206e40f9459a62)
+
+commit 5fb77bf1d1b241b4f9c1fe9e57288bbc84d8d97d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 14:09:53 2013 -0800
+
+    ceph: adjust crush tunables via 'ceph osd crush tunables <profile>'
+
+    Make it easy to adjust crush tunables.  Create profiles:
+
+     legacy: the legacy values
+     argonaut: the argonaut defaults, and what is supported.. legacy! (*(
+     bobtail: best that bobtail supports
+     optimal: the current optimal values
+     default: the current default values
+
+    * In actuality, argonaut supports some of the tunables, but it doesn't
+      say so via the feature bits.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 19ee23111585f15a39ee2907fa79e2db2bf523f0)
+
+commit 8c0d702e6f2ba0ed0fe31c06c7a028260ae08e42
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 17:20:43 2012 -0800
+
+    msg/Pipe: use state_closed atomic_t for _lookup_pipe
+
+    We shouldn't look at Pipe::state in SimpleMessenger::_lookup_pipe() without
+    holding pipe_lock.  Instead, use an atomic that we set to non-zero only
+    when transitioning to the terminal STATE_CLOSED state.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82f8bcddb5fa09913eb477ee26c71d6b4bb8d97c)
+
+commit 8e0359c3e586c0edcce769c8ed1a03444a521165
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 13:43:15 2012 -0800
+
+    msgr: inject delays at inconvenient times
+
+    Exercise some rare races by injecting delays before taking locks
+    via the 'ms inject internal delays' option.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a5d692a7b9b4bec2c27993ca37aa3fec4065292b)
+
+commit 34e2d4024700f633c2c586265efb61537342db18
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 09:22:18 2012 -0800
+
+    msgr: fix race on Pipe removal from hash
+
+    When a pipe is faulting and shutting down, we have to drop pipe_lock to
+    take msgr lock and then remove the entry.  The Pipe in this case will
+    have STATE_CLOSED.  Handle this case in all places we do a lookup on
+    the rank_pipe hash so that we effectively ignore entries that are
+    CLOSED.
+
+    This fixes a race introduced by the previous commit where we won't use
+    the CLOSED pipe and try to register a new one, but the old one is still
+    registered.
+
+    See bug #3675.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e99b4a307b4427945a4eb5ec50e65d6239af4337)
+
+commit ae1882e7efc91b770ac0ac8682ee6c5792a63a93
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 09:19:05 2012 -0800
+
+    msgr: don't queue message on closed pipe
+
+    If we have a con that refs a pipe but it is closed, don't use it.  If
+    the ref is still there, it is only because we are racing with fault()
+    and it is about to (or just was) be detached.  Either way,
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6339c5d43974f4b495f15d199e01a141e74235f5)
+
+commit 373f1671b6cb64dba5a9172967b27177515be1fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Dec 22 21:24:52 2012 -0800
+
+    msgr: atomically queue first message with connect_rank
+
+    Atomically queue the first message on the new pipe, without dropping
+    and retaking pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7bf0b0854d1f2706a3a2302bcbf92dd5c8c888ef)
+
+commit 82f22b38c5dc0b636574679ba1fee1b36a3c0478
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 10 11:06:02 2013 -0800
+
+    config_opts.h: default osd_recovery_delay_start to 0
+
+    This setting was intended to prevent recovery from overwhelming peering traffic
+    by delaying the recovery_wq until osd_recovery_delay_start seconds after pgs
+    stop being added to it.  This should be less necessary now that recovery
+    messages are sent with strictly lower priority then peering messages.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Gregory Farnum <greg@inktank.com>
+    (cherry picked from commit 44625d4460f61effe2d63d8280752f10f159e7b4)
+
+commit 81e8bb55e28384048fd82116a791a65ca52ef999
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 21:19:18 2013 -0800
+
+    osdmaptool: more fix cli test
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b0162fab3d927544885f2b9609b9ab3dc4aaff74)
+
+commit 2b5b2657579abdf5b1228f4c5c5ac8cec3706726
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 21:10:26 2013 -0800
+
+    osdmaptool: fix cli test
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bd8765c918174aea606069124e43c480c809943)
+
+commit f739d1238a8a67598c037b6e2ed5d539a2d79996
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 16 14:21:47 2013 -0800
+
+    osdmaptool: allow user to specify pool for test-map-object
+
+    Fixes: #3820
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Gregory Farnum <greg@inktank.com>
+    (cherry picked from commit 85eb8e382a26dfc53df36ae1a473185608b282aa)
+
+commit 00759ee08f5dc62cbe4f237399f298472f6d8f4a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jan 16 12:41:16 2013 -0800
+
+    rados.cc: fix rmomapkey usage: val not needed
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <samuel.just@inktank.com>
+    (cherry picked from commit 625c3cb9b536a0cff7249b8181b7a4f09b1b4f4f)
+
+commit 06b3270f679be496df41810dacf863128b0cfcaa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:27:23 2013 -0800
+
+    librados.hpp: fix omap_get_vals and omap_get_keys comments
+
+    We list keys greater than start_after.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3f0ad497b3c4a5e9bef61ecbae5558ae72d4ce8b)
+
+commit 75072965201380aa55a8e15f9db4ccaf4d34d954
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:26:22 2013 -0800
+
+    rados.cc: use omap_get_vals_by_keys in getomapval
+
+    Fixes: #3811
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit cb5e2be418924cf8b2c6a6d265a7a0327f08d00a)
+
+commit a3c2980fccfe95b7d094a7c93945437c3911b858
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:24:50 2013 -0800
+
+    rados.cc: fix listomapvals usage: key,val are not needed
+
+    Fixes: #3812
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 44c45e520cc2e60c6c803bb245edb9330bff37e4)
+
+commit 20b27a1ce71c379a3b2a29d282dc0689a3a0df46
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jan 16 15:01:47 2013 -0800
+
+    rgw: copy object should not copy source acls
+
+    Fixes: #3802
+    Backport: argonaut, bobtail
+
+    When using the S3 api and x-amz-metadata-directive is
+    set to COPY we used to copy complete metadata of source
+    object. However, this shouldn't include the source ACLs.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 37dbf7d9df93dd0e92019be31eaa1a19dd9569c7)
+
+commit 3293b31b44c9adad2b5e37da9d5342a6e4b72ade
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 11:02:15 2013 -0800
+
+    OSD: only trim up to the oldest map still in use by a pg
+
+    map_cache.cached_lb() provides us with a lower bound across
+    all pgs for in-use osdmaps.  We cannot trim past this since
+    those maps are still in use.
+
+    backport: bobtail
+    Fixes: #3770
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 66eb93b83648b4561b77ee6aab5b484e6dba4771)
+
+commit 898a4b19ecc6fffc33feb198f37182ec0a6e77e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 08:15:02 2013 -0800
+
+    Revert "osdmap: spread replicas across hosts with default crush map"
+
+    This reverts commit 503917f0049d297218b1247dc0793980c39195b3.
+
+    This breaks vstart and teuthology configs.  A better fix is coming.
+
+commit 55b7dd3248f35929ea097525798e8667fafbf161
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jan 10 18:54:12 2013 +0000
+
+    mon: OSDMonitor: don't output to stdout in plain text if json is specified
+
+    Fixes: #3748
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 410906e04936c935903526f26fb7db16c412a711)
+
+commit 015a454a0c046cb678991cc4f4d53fb58c41dbe4
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 11 17:23:22 2013 -0800
+
+    osdmap: spread replicas across hosts with default crush map
+
+    This is more often the case than not, and we don't have a good way to
+    magically know what size of cluster the user will be creating.  Better to
+    err on the side of doing the right thing for more people.
+
+    Fixes: #3785
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7ea5d84fa3d0ed3db61eea7eb9fa8dbee53244b6)
+
+commit d882d053927c319274be38a247f2beabb4e06b64
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 19:17:23 2013 -0800
+
+    ReplicatedPG: fix snapdir trimming
+
+    The previous logic was both complicated and not correct.  Consequently,
+    we have been tending to drop snapcollection links in some cases.  This
+    has resulted in clones incorrectly not being trimmed.  This patch
+    replaces the logic with something less efficient but hopefully a bit
+    clearer.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f42c37359d976d1fe90f2d3b877b9b0268adc0b)
diff --git a/doc/changelog/v0.56.3.txt b/doc/changelog/v0.56.3.txt
new file mode 100644
index 000000000..c87675a96
--- /dev/null
+++ b/doc/changelog/v0.56.3.txt
@@ -0,0 +1,562 @@
+commit 6eb7e15a4783b122e9b0c85ea9ba064145958aa5
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Feb 13 10:10:20 2013 -0800
+
+    v0.56.3
+
+commit f5eb845a0f7a2c28d3a88a37479bcb34f882f40c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 8 13:14:49 2013 -0800
+
+    rgw: change json formatting for swift list container
+    
+    Fixes: #4048
+    There is some difference in the way swift formats the
+    xml output and the json output for list container. In
+    xml the entity is named 'name' and in json it is named
+    'subdir'.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3e4d79fe42dfc3ca70dc4d5d2aff5223f62eb34b)
+
+commit f21543f0d88f7bacb69cef3712b0ce087f386e93
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 11 17:08:55 2013 -0800
+
+    librbd: unprotect any non-unprotected snapshot
+    
+    Include snapshots in the UNPROTECTING state as well, which can occur
+    after an unprotect is interrupted.
+    
+    Fixes: #4100
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit fe283813b44a7c45def6768ea0788a3a0635957e)
+
+commit 65969f8fbef02ee39f6c2365fffbcd3f633f4b37
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 8 21:36:13 2013 -0800
+
+    java: make CephMountTest use user.* xattr names
+    
+    Changes to the xattr code in Ceph require
+    a few tweaks to existing test cases.
+    Specifically, there is now a ceph.file.layout
+    xattr by default and user defined xattrs
+    are prepended with "user."
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joe Buck <jbbuck@gmail.com>
+    Reviewed-by: Noah Watkins <noahwatkins@gmail.com>
+
+commit 14fddc3ce85d3695aad9d3597f8f50dba5960a86
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 8 09:59:25 2013 -0800
+
+    mon: fix typo in C_Stats
+    
+    Broken by previous commit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3cf3710be0b4cccc8de152a97be50d983c35116d)
+
+commit 0453140d187016a61950a8836da57f54d2c34602
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 23:13:11 2013 -0800
+
+    mon: retry PGStats message on EAGAIN
+    
+    If we get EAGAIN from a paxos restart/election/whatever, we should
+    restart the message instead of just blindly acking it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4837063d447afb45554f55bb6fde1c97559acd4b)
+
+commit e68fcec78286363935cf731015108b9ea36b50a6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 22:06:14 2013 -0800
+
+    mon: handle -EAGAIN in completion contexts
+    
+    We can get ECANCELED, EAGAIN, or success out of the completion contexts,
+    but in the EAGAIN case (meaning there was an election) we were sending
+    a success to the client.  This resulted in client hangs and all-around
+    confusion when the monitor cluster was thrashing.
+    
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 17827769f1fe6d7c4838253fcec3b3a4ad288f41)
+
+commit 20ec490555728251444833520a40b20dc8015216
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 14:11:09 2013 -0800
+
+    osd: only share maps on hb connection of OSD_HBMSGS feature is set
+    
+    Back in 1bc419a7affb056540ba8f9b332b6ff9380b37af we started sharing maps
+    with dead osds via the heartbeat connection, but old code will crash on an
+    unexpected message.  Only do this if the OSD_HBMSGS feature is present.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 302b26ff70ee5539da3dcb2e5614e2b7e83b9dcd)
+
+commit cbf63b633e7a59456f503af487fd4ad2607bbd76
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 14:10:51 2013 -0800
+
+    osd: tolerate unexpected messages on the heartbeat interface
+    
+    We should note but not crash on unexpected messages.  Announce this awesome
+    new "capability" via a feature bit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit afda30aeaae0a65f83c6886658354ad2b57c4c43)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+
+commit 102a519632f1b7a0fede9a3fbd4a5c1df0e732a5
+Merge: 2c6afa0 2ebf4d0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 13:39:52 2013 -0800
+
+    Merge remote-tracking branch 'gh/wip-bobtail-osd-msgr' into bobtail
+
+commit 2c6afa058e8b1738c1400392320482945834de86
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 30 11:32:23 2013 -0800
+
+    test_libcephfs: fix xattr test
+    
+    Ignore the ceph.*.layout xattrs.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b0d4dd21c7be86eb47728a4702a3c67ca44424ac)
+
+commit f11beb954976f66bfae75e847937f84958ebeaf3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 22:51:29 2013 -0800
+
+    radosgw-admin: fix cli test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1b05b0edbac09d1d7cf0da2e536829df05e48573)
+
+commit ec1085e534eb39d999775bebdcdb997f893a04ae
+Merge: 66d7758 62ed62f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 23:25:30 2013 -0800
+
+    Merge remote-tracking branch 'gh/wip-bobtail-vxattrs' into bobtail
+
+commit 66d775858004d1d4e8a138b8d33a3799e03ce26e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 4 09:14:39 2013 -0800
+
+    mon: enforce reweight be between 0..1
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4e29c95d6f61daa838888840cef0cceedc0fcfdd)
+
+commit 8bab3a1c3d0d2f619ddf885bb9050ad9a1c43517
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 7 10:38:00 2013 -0800
+
+    PG: dirty_info on handle_activate_map
+    
+    We need to make sure the pg epoch is persisted during
+    activate_map.
+    
+    Backport: bobtail
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit dbce1d0dc919e221523bd44e1d0834711da1577d)
+
+commit dffa386bc13370c0ef56acf740b5200b2054980f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 10:21:49 2013 -0800
+
+    osd: flush peering queue (consume maps) prior to boot
+    
+    If the osd itself is behind on many maps during boot, it will get more and
+    (as part of that) flush the peering wq to ensure the pgs consume them.
+    However, it is possible for OSD to have latest/recnet maps, but pgs to be
+    behind, and to jump directly to boot and join.  The OSD is then laggy and
+    unresponsive because the peering wq is way behind.
+    
+    To avoid this, call consume_map() (kick the peering wq) at the end of
+    init and flush it to ensure we are *internally* all caught up before we
+    consider joining the cluster.
+    
+    I'm pretty sure this is the root cause of #3905 and possibly #3995.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit af95d934b039d65d3667fc022e2ecaebba107b01)
+
+commit 47c9f46aac4afac37fb6ec72f0482e61f5e0d798
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 6 17:10:00 2013 -0800
+
+    rgw: a tool to fix clobbered bucket info in user's bucket list
+    
+    This fixes bad entries in user's bucket list that may have occured
+    due to issue #4039. Syntax:
+    
+     $ radosgw-admin user check --uid=<uid> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9cb6c33f0e2281b66cc690a28e08459f2e62ca13)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+
+commit 6c8d63819fde1b6854f8fc03351465b420ff1bdc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 6 16:43:48 2013 -0800
+
+    rgw: bucket recreation should not clobber bucket info
+    
+    Fixes: #4039
+    User's list of buckets is getting modified even if bucket already
+    exists. This fix removes the newly created directory object, and
+    makes sure that user info's data points at the correct bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9d006ec40ced9d97b590ee07ca9171f0c9bec6e9)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+    	src/rgw/rgw_rados.cc
+
+commit cc167914ac9603f87083c63f2cbc8dac9441329f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 5 14:50:54 2013 -0800
+
+    rgw: a tool to fix buckets with leaked multipart references
+    
+    Checks specified bucket for the #4011 symptoms, optionally fix
+    the issue.
+    
+    sytax:
+      radosgw-admin bucket check --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2d8faf8e5f15e833e6b556b0f3c4ac92e4a4151e)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_rados.h
+
+commit 4d6964fc7ddd23806e225c95bcb90ef93e4d23a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 5 13:54:11 2013 -0800
+
+    rgw: radosgw-admin object unlink
+    
+    Add a radosgw-admin option to remove object from bucket index
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 16235a7acb9543d60470170bb2a09956364626cd)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_rados.h
+    	src/test/cli/radosgw-admin/help.t
+
+commit 2ebf4d065af3dc2e581a25b921071af3efb57f8a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:30:00 2013 -0800
+
+    osd: kill unused addr-based send_map()
+    
+    Not used, old API, bad.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e359a862199c8a94cb238f7271ba1b0edcc0863c)
+
+commit bac5b144b27f32da306161ae7018ccc337704121
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:29:37 2013 -0800
+
+    osd: share incoming maps via Connection*, not addrs
+    
+    Kill a set of parallel methods that are using the old addr/inst-based
+    msgr APIs, and instead use Connection handles.  This is much safer and gets
+    us closer to killing the old msgr API.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e2fab54a4fdf2f59e2b635cbddef8a5909acb7c)
+
+commit 9ca3a165ded62313ba153d7bab89dadf3f73999f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:27:00 2013 -0800
+
+    osd: pass new maps to dead osds via existing Connection
+    
+    Previously we were sending these maps to dead osds via their old addrs
+    using a new outgoing connection and setting the flags so that the msgr
+    would clean up.  That mechanism is possibly buggy and fragile, and we can
+    avoid it entirely if we just reuse the existing heartbeat Connection.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1bc419a7affb056540ba8f9b332b6ff9380b37af)
+
+commit 4cb28b6ed5a702fdac99b8ec71233ef7f877a7a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:25:28 2013 -0800
+
+    osd: requeue osdmaps on heartbeat connections for cluster connection
+    
+    If we receive an OSDMap on the cluster connection, requeue it for the
+    cluster messenger, and process it there where we normally do.  This avoids
+    any concerns about locking and ordering rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76705ace2e9767939aa9acf5d9257c800f838854)
+
+commit e4f7ff8c288eac8a8b57382f11a4b6f93682315a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:23:23 2013 -0800
+
+    msgr: add get_loopback_connection() method
+    
+    Return the Connection* for ourselves, so we can queue messages for
+    ourselves.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7059eb3f3922cf08c1e5bb5958acc2d45952482)
+
+commit 62ed62f5e2fb068cee38612d7974526aa1b3c759
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 11:33:04 2013 -0800
+
+    qa: add layout_vxattrs.sh test script
+    
+    Test virtual xattrs for file and directory layouts.
+    
+    TODO: create a data pool, add it to the fs, and make sure we can use it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 61fbe27a52d12ecd98ddeb5fc0965c4f8ee7841a)
+
+commit d386622c3961a3b57eea42fdb82611cd2e904f4d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:11:18 2013 -0800
+
+    mds: allow dir layout/policy to be removed via removexattr on ceph.dir.layout
+    
+    This lets a user remove a policy that was previously set on a dir.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit db31a1f9f27416e4d531fda716e32d42a275e84f)
+
+commit 6af5da7ae2c4ef95c16c6460770b6244d1aa1a6e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:09:39 2013 -0800
+
+    mds: handle ceph.*.layout.* setxattr
+    
+    Allow individual fields of file or dir layouts to be set via setxattr.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ebebf72f0993d028e795c78a986e1aee542ca5e0)
+
+commit c0af056eb9bdb62cfd8a6f9054a3a3c78c8e7447
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 4 22:03:32 2013 -0800
+
+    mdsmap: backported is_data_pool()
+    
+    This roughly corresponds to mainline commit 99d9e1d.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0407af4641ea19697f8feb0f48a92cde8dd4fbe4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:04:05 2013 -0800
+
+    mds: fix client view of dir layout when layout is removed
+    
+    We weren't handling the case where the projected node has NULL for the
+    layout properly.  Fixes the client's view when we remove the dir layout.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09f28541e374ffac198e4d48082b064aae93cb2c)
+
+commit 8ce834d3f50b00fdd59cd237f3fb5fef1d57e1dd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:04:39 2013 -0800
+
+    client: note presence of dir layout in inode operator<<
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 84751489ca208964e617516e04556722008ddf67)
+
+commit 99824b93cec93daaa0d536f031eb3b6180f94e3b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 09:05:59 2013 -0800
+
+    client: list only aggregate xattr, but allow setting subfield xattrs
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ba32ea9454d36072ec5ea3e6483dc3daf9199903)
+
+commit 809cff488ea1ffa299edd678ba6260993771bde3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 22:26:00 2013 -0800
+
+    client: implement ceph.file.* and ceph.dir.* vxattrs
+    
+    Display ceph.file.* vxattrs on any regular file, and ceph.dir.* vxattrs
+    on any directory that has a policy set.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3f82912a891536dd7e930f98e28d9a8c18fab756)
+
+commit 13babca354d9fbe255de8bae9608a0c158bf6c40
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 17:21:37 2013 -0800
+
+    client: move xattr namespace enforcement into internal method
+    
+    This captures libcephfs users now too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit febb96509559084357bfaabf7e4d28e494c274aa)
+
+commit 65ab51740175254ba3ee050f0fd97332dffe2eb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 17:20:22 2013 -0800
+
+    client: allow ceph.* xattrs
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad7ebad70bf810fde45067f78f316f130a243b9c)
+
+commit 6f3c1cd2cc07d951dfc23e523b9c6400b7c77c72
+Author: caleb miles <caselim@gmail.com>
+Date:   Mon Jan 14 12:16:12 2013 -0500
+
+    rgw_rest: Make fallback uri configurable.
+    
+    Some HTTP servers, notabily lighttp, do not set SCRIPT_URI, make the fallback
+    string configurable.
+    
+    Signed-off-by: caleb miles <caleb.miles@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b3a2e7e955547a863d29566aab62bcc480e27a65)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+
+commit f57d1b4c8cc4d08c6147423d7881be55ed2e88d9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 1 10:56:11 2013 -0800
+
+    rgw: fix setting of NULL to string
+    
+    Fixes: #3777
+    s->env->get() returns char * and not string and can return NULL.
+    Also, remove some old unused code.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9019fbbe8f84f530b6a8700dfe99dfeb03e0ed3d)
+
+commit 55687240b2de20185524de07e67f42c3b1ae6592
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 10:44:04 2013 -0800
+
+    OSD: check for empty command in do_command
+    
+    Fixes: #3878
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 8cf79f252a1bcea5713065390180a36f31d66dfd)
+
+commit c3468f76a5e68a6426f03e508d8ecf26950fca2a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jan 30 18:52:24 2013 +0100
+
+    PGMap: fix -Wsign-compare warning
+    
+    Fix -Wsign-compare compiler warning:
+    
+    mon/PGMap.cc: In member function 'void PGMap::apply_incremental
+     (CephContext*, const PGMap::Incremental&)':
+    mon/PGMap.cc:247:30: warning: comparison between signed and
+     unsigned integer expressions [-Wsign-compare]
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b571f8ee2d22a3894120204bc5f119ff37e1de53)
+
+commit 5a6b9af90f00d08ef97b34ee0b5abc7b0b63e72b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 28 19:46:33 2013 -0800
+
+    mon: smooth pg stat rates over last N pgmaps
+    
+    This smooths the recovery and throughput stats over the last N pgmaps,
+    defaulting to 2.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7d15afb529615db56bae038b18b66e60d827a96)
+
+commit 7fd7a5eed19d5ab508d5fe11ff8734bc2bc8c565
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:51:40 2013 -0800
+
+    mon/PGMap: report IO rates
+    
+    This does not appear to be very accurate; probably the stat values we're
+    displaying are not being calculated correctly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3f6837e022176ec4b530219043cf12e009d1ed6e)
+
+commit 7f149cf6730280f0e633d9f5ef3f0f95c5a5e430
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:51:14 2013 -0800
+
+    mon/PGMap: report recovery rates
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 208b02a748d97378f312beaa5110d8630c853ced)
+
+commit 8d2d396c6d02bff72aca53920e9ac93fe91428d3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:50:45 2013 -0800
+
+    mon/PGMap: include timestamp
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76e9fe5f06411eb0e96753dcd708dd6e43ab2c02)
+
+commit 8ab77bd4b510149f4df6b3134de0ef59272cec71
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:49:16 2013 -0800
+
+    osd: track recovery ops in stats
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2495f658c6d17f56ea0a2ab1043299a59a7115b)
+
+commit 8fd8534b4b808292a4b7c6b9f2f866c431cf9645
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:06:52 2013 -0800
+
+    osd_types: add recovery counts to object_sum_stats_t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4aea19ee60fbe1106bdd71de2d172aa2941e8aab)
diff --git a/doc/changelog/v0.56.4.txt b/doc/changelog/v0.56.4.txt
new file mode 100644
index 000000000..f48fd834a
--- /dev/null
+++ b/doc/changelog/v0.56.4.txt
@@ -0,0 +1,1126 @@
+commit 63b0f854d1cef490624de5d6cf9039735c7de5ca
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Mar 25 11:02:31 2013 -0700
+
+    v0.56.4
+
+commit 90ab120a3c08f30654207c9c67fb0a4ff1476aff
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Mar 25 09:50:33 2013 -0700
+
+    rgw: bucket index ops on system buckets shouldn't do anything
+    
+    Fixes: #4508
+    Backport: bobtail
+    On certain bucket index operations we didn't check whether
+    the bucket was a system bucket, which caused the operations
+    to fail. This triggered an error message on bucket removal
+    operations.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 70e0ee8ba955322832f0c366537ddf7a0288761e)
+
+commit b8657fa08d0fe176c71a60c1654c53c3156a9c53
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 15:02:50 2013 -0800
+
+    systest: restrict list error acceptance
+    
+    Only ignore errors after the midway point if the midway_sem_post is
+    defined.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5b24a68b6e7d57bac688021b822fb2f73494c3e9)
+
+commit 5af3d64f62b30cbaac13ccc7c23fc884ee1b7ae3
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 14:55:34 2013 -0800
+
+    systest: fix race with pool deletion
+    
+    The second test have pool deletion and object listing wait on the same
+    semaphore to connect and start. This led to errors sometimes when the
+    pool was deleted before it could be opened by the listing process. Add
+    another semaphore so the pool deletion happens only after the listing
+    has begun.
+    
+    Fixes: #4147
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit b0271e390564119e998e18189282252d54f75eb6)
+
+commit 3569489b541ac0643520d20b08c788c26dfaff7f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 19 14:26:16 2013 -0700
+
+    os/FileJournal: fix aio self-throttling deadlock
+    
+    This block of code tries to limit the number of aios in flight by waiting
+    for the amount of data to be written to grow relative to a function of the
+    number of aios.  Strictly speaking, the condition we are waiting for is a
+    function of both aio_num and the write queue, but we are only woken by
+    changes in aio_num, and were (in rare cases) waiting when aio_num == 0 and
+    there was no possibility of being woken.
+    
+    Fix this by verifying that aio_num > 0, and restructuring the loop to
+    recheck that condition on each wakeup.
+    
+    Fixes: #4079
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e5940da9a534821d0d8f872c13f9ac26fb05a0f5)
+
+commit 664ffa7d2178e486b00fa0706067f19b1bb9ab82
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:49 2013 -0700
+
+    common/MemoryModel: remove logging to /tmp/memlog
+    
+    This was a hack for dev purposes ages ago; remove it.  The predictable
+    filename is a security issue.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit c524e2e01da41ab5b6362c117939ea1efbd98095)
+
+commit 901fa435bb4be29b72cec39e5ff77570d13cbc6c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:43 2013 -0700
+
+    init-ceph: clean up temp ceph.conf filename on exit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6a7ad2eac1db6abca3d7edb23ca9b80751400a23)
+
+commit 951f7e562afb4ae91b8d037ef793a96779461e96
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:33 2013 -0700
+
+    init-ceph: push temp conf file to a unique location on remote host
+    
+    The predictable file name is a security problem.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 051734522fea92878dd8139f28ec4e6b01371ede)
+
+commit dd670afccfd3ae58c03ea3b4c213dd8db22f618e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:23 2013 -0700
+
+    mkcephfs: make remote temp directory name unique
+    
+    The predictable file name is a security problem.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f463ef78d77b11b5ad78b31e9a3a88d0a6e62bca)
+
+commit 1cd51eb63a91fa0aa395c41572cfee4d53fbc7f5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 22 13:51:14 2013 -0700
+
+    PG::GetMissing: need to check need_up_thru in MLogRec handler
+    
+    Backport: bobtail
+    Fixes: #4534
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4fe4deafbe1758a6b3570048aca57485bd562440)
+
+commit d866814285667c7f9cd9205dca6cf95f3634c5fc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 22 13:48:49 2013 -0700
+
+    PG,osd_types: improve check_new_interval debugging
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d611eba9caf45f2d977c627b123462a073f523a4)
+
+commit fc3406f395103737e5784611e2b38fd1c4f26369
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 16:06:20 2013 -0800
+
+    FileStore: fix reversed collection_empty return value
+    
+    Backport: bobtail
+    Fixes: #4380
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7a434d10da0f77e1b329de0b06b6645cd73cc81b)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit 9f7c6ad395653b1039ea957cfaa7689699f510e7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Feb 11 12:52:07 2013 -0800
+
+    FileStore: set replay guard on create_collection
+    
+    This should prevent sequences like:
+    
+    rmcoll a
+    mkcoll a
+    touch a foo
+    <crash>
+    
+    from causing trouble by preventing the rmcoll
+    and mkcoll from being replayed.
+    
+    Fixes: 4064
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 411770c45734c9827745ddc4018d86c14f2858a6)
+
+commit 9932c9f0a0503f1902549a3c39e5d51488fff3ef
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Feb 11 12:24:14 2013 -0800
+
+    FileStore: _split_collection should not create the collection
+    
+    This will simplify adding a replay guard to create_collection.
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b184ff581a08c9e6ce5b858f06ccbe9d0e2a170b)
+
+commit 7e7ea8266ee45d96863e1edc5a567dcc2977680c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 15:15:27 2013 -0800
+
+    client: use 4MB f_bsize and f_frsize for statfs
+    
+    Old stat(1) reports:
+    
+      Block size: 1048576    Fundamental block size: 1048576
+    
+    and the df(1) arithmetic works out.  New stat(1) reports:
+    
+      Block size: 1048576    Fundamental block size: 4096
+    
+    which is what we are shoving into statvfs, but we have the b_size and
+    fr_size arithmetic swapped.  However, doing the *correct* reporting would
+    then break the old stat by making both sizes appear to be 4KB (or
+    whatever).
+    
+    Sidestep the issue by making *both* values 4MB.. which is both large enough
+    to report large FS sizes, and also the default stripe size and thus a
+    "reasonable" value to report for a block size.
+    
+    Perhaps in the future, when we no longer care about old userland, we can
+    report the page size for f_bsize, which is probably the "most correct"
+    thing to do.
+    
+    Fixes: #3794.  See also #3793.
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7c94083643891c9d66a117352f312b268bdb1135)
+
+commit 66dca7730006dffbce12e6b27016ea5469b7691b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 18 17:39:46 2013 -0800
+
+    os/FileStore: check replay guard on src for collection rename
+    
+    This avoids a problematic sequence like:
+    
+         - rename A/ -> B/
+         - remove B/1...100
+         - destroy B/
+         - create A/
+         - write A/101...
+         <crash>
+         - replay A/ -> B/
+         - remove B/1...100  (fails but tolerated)
+         - destroy B/        (fails with ENOTEMPTY)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5fc83c8d9887d2a916af11436ccc94fcbfe59b7a)
+
+commit ea570591b0d9dda102bece0f4cc7d263c3e2a3b0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Feb 22 14:12:28 2013 -0800
+
+    PG::proc_replica_log: oinfo.last_complete must be *before* first entry in omissing
+    
+    Fixes: #4189
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2dae6a68ee85a20220ee940dbe33a2144d43457b)
+
+commit aa38d87c6237ccfb8b3b03bc7c28dcadb927237b
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 17:55:21 2013 -0800
+
+    osd/PG: fix typo, missing -> omissing
+    
+    From ce7ffc34408bf32c66dc07e6f42d54b7ec489d41.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dc181224abf6fb8fc583730ae3d90acdf0b80f39)
+
+commit ac914834ba4fb0e4770f63c60713be7a48aa3fa1
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 21 15:31:36 2013 -0800
+
+    PG::proc_replica_log: adjust oinfo.last_complete based on omissing
+    
+    Otherwise, search_for_missing may neglect to check the missing
+    set for some objects assuming that if the need version is
+    prior to last_complete, the replica must have it.
+    
+    Fixes: #4994
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ce7ffc34408bf32c66dc07e6f42d54b7ec489d41)
+
+commit 7f39704b0903fb11e3f1e102599bf0c540f93623
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 9 00:05:33 2013 -0800
+
+    osd: fix load_pgs collection handling
+    
+    On a _TEMP pg, is_pg() would succeed, which meant we weren't actually
+    hitting the cleanup checks.  Instead, restructure this loop as positive
+    checks and handle each type of collection we understand.
+    
+    This fixes _TEMP cleanup.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b19b6dced85617d594c15631571202aab2f94ae8)
+
+commit 0a09be99f3e9d2b68a052ff63ea858a7c8e5626c
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 9 00:04:29 2013 -0800
+
+    osd: fix load_pgs handling of pg dirs without a head
+    
+    If there is a pgid that passes coll_t::is_pg() but there is no head, we
+    will populate the pgs map but then fail later when we try to do
+    read_state.  This is a side-effect of 55f8579.
+    
+    Take explicit note of _head collections we see, and then warn when we
+    find stray snap collections.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1f80a0b576c0af1931f743ad988b6293cbf2d6d9)
+
+commit 2dc25dfd47de8d3602bdb3699c82b8b118b0fb28
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 7 13:34:47 2013 -0800
+
+    OSD::load_pgs: first scan colls before initing PGs
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 073f58ede2e473af91f76d01679631c169274af7)
+
+commit 2fd8db29ccf56fd5473826f20e351edfe200419d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Jan 8 19:24:13 2013 -0800
+
+    osd: Add digest of omap for deep-scrub
+    
+    Add ScrubMap encode/decode v4 message with omap digest
+    Compute digest of header and key/value.  Use bufferlist
+    to reflect structure and compute as we go, clearing
+    bufferlist to reduce memory usage.
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 509a93e89f04d7e9393090563cf7be8e0ea53891)
+
+commit ee59f517ccc195e591e0b1b6ddf6d1d054c4e2fd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 15 15:13:46 2013 -0700
+
+    OSD: split temp collection as well
+    
+    Otherwise, when we eventually remove the temp collection, there might be
+    objects in the temp collection which were independently pulled into the child
+    pg collection.  Thus, removing the old stale parent link from its temp
+    collection also blasts the omap entries and snap mappings for the real child
+    object.
+    
+    Backport: bobtail
+    Fixes: #4452
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f8d66e87a5c155b027cc6249006b83b4ac9b6c9b)
+
+commit 7ec68e2d45b431a3a77458939622e6ea1e396d0e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 14 19:59:36 2013 -0700
+
+    PG: ignore non MISSING pg query in ReplicaActive
+    
+    1) Replica sends notify
+    2) Prior to processing notify, primary queues query to replica
+    3) Primary processes notify and activates sending MOSDPGLog
+    to replica.
+    4) Primary does do_notifies at end of process_peering_events
+    and sends to Query.
+    5) Replica sees MOSDPGLog and activates
+    6) Replica sees Query and asserts.
+    
+    In the above case, the Replica should simply ignore the old
+    Query.
+    
+    Fixes: #4050
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8222cbc8f35c359a35f8381ad90ff0eed5615dac)
+
+commit b279b148b38c755d2dcca6524d7b8a95daf123a0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Mar 13 16:04:23 2013 -0700
+
+    FileJournal: queue_pos \in [get_top(), header.max_size)
+    
+    If queue_pos == header.max_size when we create the entry
+    header magic, the entry will be rejected at get_top() on
+    replay.
+    
+    Fixes: #4436
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit de8edb732e3a5ce4471670e43cfe6357ae6a2758)
+
+commit 56e0a129b0d9ad463f4300999ed2c15173cbe603
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 14 18:52:02 2013 -0700
+
+    OSD: expand_pg_num after pg removes
+    
+    Otherwise:
+    1) expand_pg_num removes a splitting pg entry
+    2) peering thread grabs pg lock and starts split
+    3) OSD::consume_map grabs pg lock and starts removal
+    
+    At step 2), we run afoul of the assert(is_splitting)
+    check in split_pgs.  This way, the would be splitting
+    pg is marked as removed prior to the splitting state
+    being updated.
+    
+    Backport: bobtail
+    Fixes: #4449
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f1b031b3cf195cf6df3d3c47c7d606fba63ed4c4)
+
+commit d6e678a62950ccc16d010a76500705e176ba9c73
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 11 06:23:54 2013 -0800
+
+    osd: update snap collections for sub_op_modify log records conditionaly
+    
+    The only remaining caller is sub_op_modify().  If we do have a non-empty
+    op transaction, we want to do this update, regardless of what we think
+    last_backfill is (our notion may be not completely in sync with the
+    primary).  In particular, our last_backfill may be the same object but
+    a different snapid, but the primary disagrees and is pushing an op
+    transaction through.
+    
+    Instead, update the collections if we have a non-empty transaction.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 31e911b63d326bdd06981ec4029ad71b7479ed70)
+
+commit 51c2c4d7ccb25617967689d7e531eb145ee7011d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 17:02:45 2013 -0800
+
+    osd: include snaps in pg_log_entry_t::dump()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 715d8717a0e8a08fbe97a3e7d3ffd33aa9529d90)
+
+commit f0bf68ff500f7337103aef2a9d6d10c3597e410f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 16:59:48 2013 -0800
+
+    osd: unconditionally encode snaps buffer
+    
+    Previously we would only encode the updated snaps vector for CLONE ops.
+    This doesn't work for MODIFY ops generated by the snap trimmer, which
+    may also adjust the clone collections.  It is also possible that other
+    operations may need to populate this field in the future (e.g.,
+    LOST_REVERT may, although it currently does not).
+    
+    Fixes: #4071, and possibly #4051.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 54b6dd924fea3af982f3d729150b6449f318daf2)
+
+commit aa74fabd90b776e9938f3d59ea5ed04bd4027dbb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 10:57:12 2013 -0800
+
+    osd: improve debug output on snap collections
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8b05492ca5f1479589bb19c1ce058b0d0988b74f)
+
+commit a30903c6adaa023587d3147179d6038ad37ca520
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 7 12:53:51 2013 -0800
+
+    PG: check_recovery_sources must happen even if not active
+    
+    missing_loc/missing_loc_sources also must be cleaned up
+    if a peer goes down during peering:
+    
+    1) pg is in GetInfo, acting is [3,1]
+    2) we find object A on osd [0] in GetInfo
+    3) 0 goes down, no new peering interval since it is neither up nor
+    acting, but peer_missing[0] is removed.
+    4) pg goes active and try to pull A from 0 since missing_loc did not get
+    cleaned up.
+    
+    Backport: bobtail
+    Fixes: #4371
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit de22b186c497ce151217aecf17a8d35cdbf549bb)
+
+commit 19f6df28337307ce6d8617524f88f222c0ffcab3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 15:49:26 2013 -0800
+
+    HashIndex: _collection_list_partial must tolerate NULL next
+    
+    Backport: bobtail
+    Fixes: #4379
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ce4432adc67dc2fc06dd21ea08e59d179496bcc6)
+
+commit 1b5c1564bad3574db257cb17d48190958d870f51
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 14:35:39 2013 -0800
+
+    OSD: lock not needed in ~DeletingState()
+    
+    No further refs to the object can remain at this point.
+    Furthermore, the callbacks might lock mutexes of their
+    own.
+    
+    Backport: bobtail
+    Fixes: #4378
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e4bf1bcab159d7c5b720f5da01877c0f67c16d16)
+
+commit 50d2d4a77f339d489703c1125dc5c1159d08ca8a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Mar 10 12:50:01 2013 -0700
+
+    ReplicatedPG: don't leak reservation on removal
+    
+    Fixes: 4431
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 32bf131e0141faf407b5ff993f75f97516b27c12)
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit c6b7c4be3fc44d065c29ea76b744b39272c2ba05
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Mar 12 12:56:01 2013 -0700
+
+    rgw: set up curl with CURL_NOSIGNAL
+    
+    Fixes: #4425
+    Backport: bobtail
+    Apparently, libcurl needs that in order to be thread safe. Side
+    effect is that if libcurl is not compiled with c-ares support,
+    domain name lookups are not going to time out.
+    Issue affected keystone.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 88725316ddcfa02ff110e659f7a8131dc1ea2cfc)
+
+commit 8c16a376a9cbfd6812824fb71e2cc9935e14e667
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 8 08:56:44 2013 -0800
+
+    osd: mark down connections from old peers
+    
+    Close out any connection with an old peer.  This avoids a race like:
+    
+    - peer marked down
+    - we get map, mark down the con
+    - they reconnect and try to send us some stuff
+    - we share our map to tell them they are old and dead, but leave the con
+      open
+    ...
+    - peer marks itself up a few times, eventually reuses the same port
+    - sends messages on their fresh con
+    - we discard because of our old con
+    
+    This could cause a tight reconnect loop, but it is better than wrong
+    behavior.
+    
+    Other possible fixes:
+     - make addr nonce truly unique (augment pid in nonce)
+     - make a smarter 'disposable' msgr state (bleh)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 881e9d850c6762290f8be24da9e74b9dc112f1c9)
+
+commit 1affc8b6e8cb82510aed524f90a6165c6edf7513
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 8 08:53:40 2013 -0800
+
+    osd/PG: rename require_same_or_newer_map -> is_same_or_newer_map
+    
+    This avoids confusion with the OSD method of the same name, and better
+    matches what the function tests (and does not do).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ba7e815a18cad110525f228db1b3fe39e011409e)
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit 7ca55d36c4007be5880102fa1ed314d3d66e9d87
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 16:25:16 2013 -0700
+
+    log: drop default 'log max recent' from 100k -> 10k
+    
+    Use less memory.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c021c5ccf0c063cccd7314964420405cea6406de)
+
+commit 33ac786ddc14d0904f4835cbf684a51f4815d45b
+Author: Jan Harkes <jaharkes@cs.cmu.edu>
+Date:   Fri Mar 8 12:45:57 2013 -0500
+
+    Fix radosgw actually reloading after rotating logs.
+    
+    The --signal argument to Debian's start-stop-daemon doesn't
+    make it send a signal, but defines which signal should be send
+    when --stop is specified.
+    
+    Signed-off-by: Jan Harkes <jaharkes@cs.cmu.edu>
+    (cherry picked from commit 44f1cc5bc42f9bb6d5a386037408d2de17dc5413)
+
+commit 6838b8db9a3d4ceaf121a261e2ded4af9bfeaeb9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 6 17:42:03 2013 -0800
+
+    common: reduce default in-memory logs for non-daemons
+    
+    The default of 100000 can result in hundreds of MBs of extra memory
+    used. This was most obvious when using librbd with caching enabled,
+    since there was a dout(0) accidentally left in the ObjectCacher.
+    
+    refs: #4352
+    backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 7c208d2f8e3f28f4055a4ae51eceae892dcef1dc)
+
+commit f80f64cf024bd7519d5a1fb2a5698db97a003ce8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 17:01:53 2013 -0800
+
+    osd: allow (some) log trim when degraded, but not during recovery
+    
+    We allow some trim during degraded, although we keep more entries around to
+    improve our chances of a restarting OSD of doing log-based recovery.
+    
+    Still disallow during recovery...
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6d89b34e5608c71b49ef33ab58340e90bd8da6e4)
+
+commit e502a65da1779564059e74c09ab87aca1d901bd2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 25 15:33:35 2013 -0800
+
+    osd: restructure calc_trim
+    
+    No functional change, except that we log more debug, yay!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 86df164d04f6e31a0f20bbb94dbce0599c0e8b3d)
+
+commit 3a3123033d7d9e1574be3fb18c807eeb0f4678eb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 16:48:02 2013 -0800
+
+    osd: allow pg log trim during (non-classic) scrub
+    
+    Chunky (and deep) scrub do not care about PG log trimming.  Classic scrub
+    still does.
+    
+    Deep scrub can take a long time, so not trimming the log during that period
+    may eat lots of RAM; avoid that!
+    
+    Might fix: #4179
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0ba8db6b664205348d5499937759916eac0997bf)
+
+commit cde0f224dac21b2e17e06d396ac52e18034a4262
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 28 12:46:00 2013 -0800
+
+    msgr: drop messages on cons with CLOSED Pipes
+    
+    Back in commit 6339c5d43974f4b495f15d199e01a141e74235f5, we tried to make
+    this deal with a race between a faulting pipe and new messages being
+    queued.  The sequence is
+    
+    - fault starts on pipe
+    - fault drops pipe_lock to unregister the pipe
+    - user (objecter) queues new message on the con
+    - submit_message reopens a Pipe (due to this bug)
+    - the message managed to make it out over the wire
+    - fault finishes faulting, calls ms_reset
+    - user (objecter) closes the con
+    - user (objecter) resends everything
+    
+    It appears as though the previous patch *meant* to drop *m on the floor in
+    this case, which is what this patch does.  And that fixes the crash I am
+    hitting; see #4271.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 0f42eddef5da6c1babe9ed51ceaa3212a42c2ec4)
+
+commit 2fa2546c90e7a6eab9a3de7ab4e4271f8b25bad6
+Author: Concubidated <tyler.brekke@inktank.com>
+Date:   Fri Mar 8 13:44:39 2013 -0800
+
+    Fix output of 'ceph osd tree --format=json'
+    
+    Signed-off-by: Tyler Brekke <tyler.brekke@inktank.com>
+    (cherry picked from commit 9bcba944c6586ad5f007c0a30e69c6b5a886510b)
+
+commit b70e2c270b9eb3fce673b7e51b527ebf88214f14
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Tue Feb 12 11:32:29 2013 -0600
+
+    deb:  Add ceph-coverage to ceph-test deb package
+    
+    Teuthology uses the ceph-coverage script extensively
+    and expects it to be installed by the ceph task.  Add
+    the script to the ceph-test debian package so that it
+    gets installed for that use case.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 376cca2d4d4f548ce6b00b4fc2928d2e6d41038f)
+
+commit ca9aac785eb9e2d1ee955792d2f4d1d911727fb3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 22 15:04:37 2013 -0800
+
+    rgw: set attrs on various list bucket xml results (swift)
+    
+    Fixes: #4247
+    The list buckets operation was missing some attrs on the different
+    xml result entities. This fixes it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 4384e59ad046afc9ec53a2d2f1fff6a86e645505)
+
+commit e39660c901756d5e722308e72a9d8ee4893f70f7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 22 15:02:02 2013 -0800
+
+    formatter: add the ability to dump attrs in xml entities
+    
+    xml entities may have attrs assigned to them. Add the ability
+    to set them. A usage example:
+    
+    formatter->open_array_section_with_attrs("container",
+         FormatterAttrs("name", "foo", NULL));
+    
+    This will generate the following xml entity:
+    <container name="foo">
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7cb6ee28073824591d8132a87ea09a11c44efd66)
+    
+    Conflicts:
+    	src/common/Formatter.cc
+
+commit 0304b85e5c12c30459b94ec5d332dfaa04d20d5b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Mar 6 19:32:21 2013 -0800
+
+    rgw: don't iterate through all objects when in namespace
+    
+    Fixes: #4363
+    Backport: argonaut, bobtail
+    When listing objects in namespace don't iterate through all the
+    objects, only go though the ones that starts with the namespace
+    prefix
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6669e73fa50e3908ec825ee030c31a6dbede6ac0)
+
+commit d0dafaa77f96fa91785df0796806ea07fd93f319
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 28 12:13:45 2013 -0800
+
+    ObjectCacher: fix debug log level in split
+    
+    Level 0 should never be used for this kind of debugging.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit cb3ee33532fb60665f39f6ccb1d69d67279fd5e1)
+
+commit ff014ed26bfa4fbf40acffec4eb4805b95a9982c
+Author: Dan Mick <dmick@danceorelse.org>
+Date:   Thu Jan 24 13:38:25 2013 -0800
+
+    rados: remove unused "check_stdio" parameter
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit bb860e49a7faeaf552538a9492ef0ba738c99760)
+
+commit e89884da9f76b713372a79b772ba3cc2f3b03048
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 23 21:31:11 2013 -0800
+
+    rados: obey op_size for 'get'
+    
+    Otherwise we try to read the whole object in one go, which doesn't bode
+    well for large objects (either non-optimal or simply broken).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 234becd3447a679a919af458440bc31c8bd6b84f)
+
+commit ee943c8bcf36f1e2218d8e25edfa38ec5fe4bec2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Feb 27 16:58:45 2013 -0800
+
+    FileJournal::wrap_read_bl: adjust pos before returning
+    
+    Otherwise, we may feed an offset past the end of the journal to
+    check_header in read_entry and incorrectly determine that the entry is
+    corrupt.
+    
+    Fixes: 4296
+    Backport: bobtail
+    Backport: argonaut
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5d54ab154ca790688a6a1a2ad5f869c17a23980a)
+
+commit 8c6f52215240f48b5e4d5bb99a5f2f451e7ce70a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 13:14:00 2013 -0800
+
+    osd: leave osd_lock locked in shutdown()
+    
+    No callers expect the lock to be dropped.
+    
+    Fixes: #3816
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98a763123240803741ac9f67846b8f405f1b005b)
+
+commit a324d999804b4648f245ee36b3bd611b3d139d5d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 26 14:07:12 2013 -0800
+
+    msg: fix entity_addr_t::is_same_host() for IPv6
+    
+    We weren't checking the memcmp return value properly!  Aie...
+    
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8dd2b67b39a8c70e48441ecd1a5cc3c6200ae97)
+
+commit 79d68ae8581141c3fb4cfafd76d5111ff009b762
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 17 22:35:50 2013 -0800
+
+    osd: requeue pg waiters at the front of the finished queue
+    
+    We could have a sequence like:
+    
+    - op1
+    - notify
+    - op2
+    
+    in the finished queue.  Op1 gets put on waiting_for_pg, the notify
+    creates the pg and requeues op1 (and the end), op2 is handled, and
+    finally op1 is handled.  That breaks ordering; see #2947.
+    
+    Instead, when we wake up a pg, queue the waiting messages at the front
+    of the dispatch queue.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 56c5a07708d52de1699585c9560cff8b4e993d0a)
+
+commit 79c4e7e91becc497843d96251776bdc176706aa0
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 17 20:49:52 2013 -0800
+
+    osd: pull requeued requests off one at a time
+    
+    Pull items off the finished queue on at a time.  In certain cases, an
+    event may result in new items betting added to the finished queue that
+    will be put at the *front* instead of the back.  See latest incarnation
+    of #2947.
+    
+    Note that this is a significant changed in behavior in that we can
+    theoretically starve if an event keeps resulting in new events getting
+    generated.  Beware!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit f1841e4189fce70ef5722d508289e516faa9af6a)
+
+commit 3a6ce5d0355beaa56199465e94666cae40bd8da1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 17 22:00:42 2013 -0800
+
+    mds: open mydir after replay
+    
+    In certain cases, we may replay the journal and not end up with the
+    dirfrag for mydir open.  This is fine--we just need to open it up and
+    fetch it below.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e51299fbce6bdc3d6ec736e949ba8643afc965ec)
+
+commit 36ed407e0f939a9bca57c3ffc0ee5608d50ab7ed
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Feb 21 09:21:01 2013 -0800
+
+    mds: use inode_t::layout for dir layout policy
+    
+    Remove the default_file_layout struct, which was just a ceph_file_layout,
+    and store it in the inode_t.  Rip out all the annoying code that put this
+    on the heap.
+    
+    To aid in this usage, add a clear_layout() function to inode_t.
+    
+    Signed-off-by: Sage Weil <sage.weil@dreamhost.com>
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit b4fd196cae5ab76aeb8f82f0581d0a6a2133a3ed
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 20 21:53:37 2013 -0800
+
+    mds: parse ceph.*.layout vxattr key/value content
+    
+    Use qi to parse a strictly formatted set of key/value pairs.  Be picky
+    about whitespace.  Any subset of recognized keys is allowed.  Parse the
+    same set of keys as the ceph.*.layout.* vxattrs.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5551aa5b3b5c2e9e7006476b9cd8cc181d2c9a04)
+
+commit 5f92adca26b509aed5655e85ad53174fa7b574b9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Feb 18 09:10:43 2013 -0800
+
+    rgw: fix multipart uploads listing
+    
+    Fixes: #4177
+    Backport: bobtail
+    Listing multipart uploads had a typo, and was requiring the
+    wrong resource (uploadId instead of uploads).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit db99fb4417b87301a69cb37b00c35c838b77197e)
+
+commit a44df9343594099fecb3897df393249d3d1992e2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 15 10:22:54 2013 -0800
+
+    rgw: don't copy object when it's copied into itself
+    
+    Fixes: #4150
+    Backport: bobtail
+    
+    When object copied into itself, object will not be fully copied: tail
+    reference count stays the same, head part is rewritten.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 34f885be536d0ac89c10fd29b1518751d2ffc547)
+
+commit 91d6be8353259ca30dc87062422e9ae334c3c344
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 19 10:49:33 2013 -0800
+
+    PG: remove weirdness log for last_complete < log.tail
+    
+    In the case of a divergent object prior to log.tail,
+    last_complete may end up before log.tail.
+    
+    Backport: bobtail
+    Fixes #4174
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dbadb3e2921297882c5836c67ca32bb8ecdc75db)
+    
+    Conflicts:
+    
+    	src/osd/PG.cc
+
+commit 7889c5412deaab1e7ba53d66814d0f25659229c1
+Author: James Page <james.page@ubuntu.com>
+Date:   Mon Feb 18 16:24:54 2013 +0000
+
+    Strip any trailing whitespace from rbd showmapped
+    
+    More recent versions of ceph append a bit of whitespace to the line
+    after the name of the /dev/rbdX device; this causes the monitor check
+    to fail as it can't find the device name due to the whitespace.
+    
+    This fix excludes any characters after the /dev/rbdN match.
+    (cherry picked from commit ad84ea07cac5096de38b51b8fc452c99f016b8d8)
+
+commit 42fba772c3c0e2396124a9d6ec39297dd9af4cc9
+Merge: 290a352 ad6655e
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Feb 17 21:52:55 2013 -0800
+
+    Merge pull request #64 from dalgaaf/wip-bobtail-memleaks
+    
+    cherry-pick some memleak fixes from master to bobtail
+
+commit ad6655e976916d3b62e0c91fd469eeb49fe7da3e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 17:17:59 2013 +0100
+
+    rgw/rgw_rest.cc: fix 4K memory leak
+    
+    Fix 4K memory leak in case RGWClientIO::read() fails in
+    read_all_chunked_input().
+    
+    Error from cppcheck was:
+    Checking src/rgw/rgw_rest.cc...
+    [src/rgw/rgw_rest.cc:688]: (error) Memory leak: data
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 89df090e04ef9fc5aae29122df106b0347786fab)
+
+commit 3de692753c28ddbeb3c86b51466be16e94c9c458
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 17:14:19 2013 +0100
+
+    SyntheticClient.cc: fix some memory leaks in the error handling
+    
+    Fix some memory leaks in case of error handling due to failed
+    client->open() calls.
+    
+    Error from cppcheck was:
+    [src/client/SyntheticClient.cc:1980]: (error) Memory leak: buf
+    [src/client/SyntheticClient.cc:2040]: (error) Memory leak: buf
+    [src/client/SyntheticClient.cc:2090]: (error) Memory leak: buf
+    (cherry picked from commit f0ba80756d1c3c313014ad7be18191981fb545be)
+
+commit f19d228c6a49222659c769099aaa4e755b80331d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:57:20 2013 +0100
+
+    rgw/rgw_xml.cc: fix realloc memory leak in error case
+    
+    Fix error from cppcheck:
+    
+    [src/rgw/rgw_xml.cc:212]: (error) Common realloc mistake: 'buf'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit d48cc789ea075ba2745754035640ada4131b2119)
+
+commit b0c6be95b03d9f3dd2badcdcff359ae7bc9684f4
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:54:33 2013 +0100
+
+    os/FileStore.cc: fix realloc memory leak in error case
+    
+    Fix error from cppcheck:
+    
+    [src/os/FileStore.cc:512]: (error) Common realloc mistake: 'fiemap'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c92a0f552587a232f66620170660d6b2ab6fb3a5)
+
+commit f3e5cedbc11005701ac0a8e70909a6372cd2fe6f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:49:36 2013 +0100
+
+    common/fiemap.cc: fix realloc memory leak
+    
+    Fix error from cppcheck:
+    
+    [src/common/fiemap.cc:73]: (error) Common realloc mistake: 'fiemap'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit f26f1470e7af36fa1eb8dc59c8a7c62c3c3a22ba)
+
+commit 290a352c3f9e241deac562e980ac8c6a74033ba6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 11:37:57 2013 -0800
+
+    osd/OSDCap: add unit test for parsing pools/objects with _ and -
+    
+    Hunting #4122, where a user saw
+    
+    2013-02-13 19:39:25.467916 7f766fdb4700 10 osd.0 10  session 0x2c8cc60 client.libvirt has caps osdcap[grant(object_prefix rbd^@children  class-read),grant(pool libvirt^@pool^@test rwx)] 'allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ce28ef1d7f95e71e1043912dfa269ea3b0d1599)
+    (cherry picked from commit a6534bc8a0247418d5263b765772d5266f99229c)
+
+commit f47916353547c77a65c7b70e609a9096f09c6d28
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 15:39:43 2013 -0800
+
+    osd/OSDCap: tweak unquoted_word parsing in osd caps
+    
+    Newer versions of spirit (1.49.0-3.1ubuntu1.1 in quantal, in particular)
+    dislike the construct with alnum and replace the - and _ with '\0' in the
+    resulting string.
+    
+    Fixes: #4122
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 6c504d96c1e4fbb67578fba0666ca453b939c218)
diff --git a/doc/changelog/v0.56.5.txt b/doc/changelog/v0.56.5.txt
new file mode 100644
index 000000000..0e9ce5ed2
--- /dev/null
+++ b/doc/changelog/v0.56.5.txt
@@ -0,0 +1,1972 @@
+commit df884bb7b59e5796c996ab806f5ec8b8322e292f
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu May 2 18:08:06 2013 -0700
+
+    v0.56.5
+
+commit b38cbabb2a38e4476de1135c33c25c673d62cd67
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Apr 4 18:23:40 2013 +0200
+
+    ceph.spec.in: fix udev rules.d files handling
+    
+    Move 50-rbd.rules into the ceph base package since the related
+    ceph-rbdnamer binary is part of this package. Use correct install
+    pattern.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit d0678a062a1baf738ce96114114d99495be19478
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Apr 29 17:01:55 2013 -0700
+
+    debian: only start/stop upstart jobs if upstart is present
+    
+    This avoids errors on non-upstart distros (like wheezy).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 030bf8aaa15837f898e453161eeaf1d52fc5779d)
+    
+    Conflicts:
+    	debian/ceph-mds.postinst
+
+commit 209ce34a420cc5cdbe7219393f2f150ac28f612c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 17:09:37 2013 -0700
+
+    debian: stop ceph-mds before uninstalling ceph-mds
+    
+    Fixes: #4384
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9eb0d91b867ab980135d7c6ff6347d69d0a8a794)
+
+commit 0c91becfa6cc5f05c5b2eaa89299dbf1aa202344
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Apr 25 15:53:51 2013 -0700
+
+    Makefile.am: Add -lpthread to fix build on newer ld in Raring Ringtail
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98f532e8000af281fa03b24da9ad2fda81755270)
+    
+    Conflicts:
+    	src/Makefile.am
+
+commit 15e6544f0892e8472f436149d15fc30257496c0b
+Merge: fdc0534 418cff5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 1 12:57:43 2013 -0700
+
+    Merge remote-tracking branch 'gh/bobtail-deploy' into bobtail-next
+
+commit fdc05346177a60f064fe351ca81c6078cd065179
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Apr 25 11:13:33 2013 -0700
+
+    init-ceph: use remote config when starting daemons on remote nodes (-a)
+    
+    If you use -a to start a remote daemon, assume the remote config is present
+    instead of pushing the local config.  This makes more sense and simplifies
+    things.
+    
+    Note that this means that -a in concert with -c foo means that foo must
+    also be present on the remote node in the same path.  That, however, is a
+    use case that I don't particularly care about right now.  :)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit cd7e52cc76878eed0f084f7b9a6cf7c792b716c6)
+
+commit 55c87e821b6acc1cd46056ca27c6e7f7030466e7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Apr 24 12:20:17 2013 -0700
+
+    PG: call check_recovery_sources in remove_down_peer_info
+    
+    If we transition out of peering due to affected
+    prior set, we won't trigger start_peering_interval
+    and check_recovery_sources won't get called.  This
+    will leave an entry in missing_loc_sources without
+    a matching missing set.  We always want to
+    check_recovery_sources with remove_down_peer_info.
+    
+    Fixes: 4805
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81a6165c13c533e9c1c6684ab7beac09d52ca6b5)
+
+commit a28c2f55bfc768a502fbc8ec35265e7f1a53e044
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Apr 25 14:08:57 2013 -0700
+
+    PG: clear want_acting when we leave Primary
+    
+    This is somewhat annoying actually.  Intuitively we want to
+    clear_primary_state when we leave primary, but when we restart
+    peering due to a change in prior set status, we can't afford
+    to forget most of our peering state.  want_acting, on the
+    other hand, should never persist across peering attempts.
+    In fact, in the future, want_acting should be pulled into
+    the Primary state structure.
+    
+    Fixes: #3904
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit a5cade1fe7338602fb2bbfa867433d825f337c87)
+
+commit 849ed5982d4547dd651313919fb090944f568214
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Apr 30 18:12:10 2013 -0700
+
+    mon: communicate the quorum_features properly when declaring victory.
+    
+    Fixes #4747.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe68afe9d10bc5d49a05a8bafa644d57783447cf)
+
+commit ea9c76b8f457d44e63551d47b6dd30bd16a56468
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Apr 30 11:01:54 2013 -0700
+
+    elector: trigger a mon reset whenever we bump the epoch
+    
+    We need to call reset during every election cycle; luckily we
+    can call it more than once. bump_epoch is (by definition!) only called
+    once per cycle, and it's called at the beginning, so we put it there.
+    
+    Fixes #4858.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6ae9bbb5d03cb5695a4ebb7a3c20f729de1bd67a)
+    
+    Conflicts:
+    
+    	src/mon/Elector.cc
+
+commit 418cff585bf6fc292d260a464369d82f7cdb3b79
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Apr 19 15:09:28 2013 -0700
+
+    Fix journal partition creation
+    
+    With OSD sharing data and journal, the previous code created the
+    journal partiton from the end of the device. A uint32_t is
+    used in sgdisk to get the last sector, with large HD, uint32_t
+    is too small.
+    The journal partition will be created backwards from the
+    a sector in the midlle of the disk leaving space before
+    and after it. The data partition will use whichever of
+    these spaces is greater. The remaining will not be used.
+    
+    This patch creates the journal partition from the start as a workaround.
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 56619ab91732bd2eacbef388311954f4e0230a30)
+
+commit 1a6b87ea48c3bcd3904d888a2ad1e5774aa2de8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 26 15:37:21 2013 -0700
+
+    ceph.spec.in: put ceph-disk-* et al in correct sbindir
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 86337936059d6b17c887e82f79167324eafa083c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 26 15:29:12 2013 -0700
+
+    debian: fix ceph.install
+    
+    This got out of sync somewhere in cherry-picking all of these patches.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0b42b1edb306a9763bcd02bd962bd284f6b7b3a3
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Apr 4 15:54:31 2013 +0200
+
+    Makefile.am: install ceph-* python scripts to /usr/bin directly
+    
+    Install ceph-* scripts directly to $(prefix)$(sbindir) (which
+    normaly would be /usr/sbin) instead of moving it around after
+    installation in SPEC file or debian files.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4d16f38f48e276497190c8bc03abc55c40e18eed)
+
+commit c71fb8d5b39667fc2c9f7e898ced53059d3f1eec
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Tue Apr 2 12:11:10 2013 -0700
+
+    ceph-disk:  CalledProcessError has no output keyword on 2.6
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit a793853850ee135de14b9237f7023cadcdb8575c)
+
+commit 1b86b1c7618b1ab93e25e1abf79bbe0a992c8526
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:54:53 2013 +0200
+
+    ceph-disk: fix some (local) variable names
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c4eb7e6ddd593cd45ab8343da01355be7382723e)
+
+commit ee452ebe6ff37087714c085dacd1ca2bcf66bd49
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:36:37 2013 +0200
+
+    ceph-disk: fix naming of local variable in is_mounted()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3dd8b461219e64bb0f7a210dba5a9ab7c644f014)
+
+commit 0e47d312dcec2deb26a960e6097708d8742d85cb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:33:08 2013 +0200
+
+    ceph-disk: merge twice defined function is_mounted(dev)
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit eaf31bf9f90ba9709a57a6870dbafa21142dae2c)
+
+commit 7326ea6397160607d78914444f7f0598a6ba346f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:26:12 2013 +0200
+
+    ceph-disk: define exception type
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4c6d6442a89adc5b56e99cb4d2ed572f2ad192c9)
+
+commit 14a348dc8ea9c012b6843825fd7beb5896a6d92e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:17:38 2013 +0200
+
+    ceph-disk: fix Redefining name 'uuid' from outer scope
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 058eb923c5b7dab611901fdd1724ce2a7c180827)
+
+commit 9419dca69292146ea1fd216a1ad08b2e07bdc1a0
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:14:23 2013 +0200
+
+    ceph-disk: add missing space after comma
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0080d1df7c7950e051840a543fc4bdabe6cc53e5)
+
+commit 7406981aa5515a3be8af0b9a0abef83a40c432b2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 29 17:30:28 2013 -0700
+
+    ceph-disk list: say 'unknown cluster $UUID' when cluster is unknown
+    
+    This makes it clearer that an old osd is in fact old.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e9b3f2e6e93426d631d4f8101fb431a523b0d88a)
+
+commit 0182973b66d449552d17248e27cd12c470137ea9
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 29 13:59:04 2013 -0700
+
+    ceph-disk: handle missing journal_uuid field gracefully
+    
+    Only lower if we know it's not None.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 562e1716bd4b5372716e502a5e4dd872d381bfb7)
+
+commit 24d729c591430349b033992f87141e596dcd6bff
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 28 20:49:24 2013 -0700
+
+    ceph-disk: implement 'list'
+    
+    This is based on Sandon's initial patch, but much-modified.
+    
+    Mounts ceph data volumes temporarily to see what is inside.  Attempts to
+    associated journals with osds.
+    
+    Resolves: #3120
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6a65b9131c444041d16b880c6a7f332776063a78)
+
+commit bf3f8702c580c6b9647878837355688a2a4f954c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 18:44:32 2013 -0700
+
+    ceph-disk: reimplement list_all_partitions
+    
+    Use /dev/disk/by-id to list disks and their partitions.  This is more
+    accurate and correct than the previous (as-yet unused) implementation.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d3e49047ff405573aa41f45864cf315be23f5c50)
+
+commit 9da81e4e798b8e5593f3a7eda6dfa8586307121f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 18:43:59 2013 -0700
+
+    ceph-disk: reimplement is_partition
+    
+    Previously we were assuming any device that ended in a digit was a
+    partition, but this is not at all correct (e.g., /dev/sr0, /dev/rbd1).
+    Instead, look in /dev/disk/by-id and see if there is a symlink that ends in
+    -partNN that links to our device.
+    
+    There is probably still a better way...
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20d594a889d62110ad03b761d8703f79f8eea6ad)
+
+commit 0c8efc0664f068a1952fe54b8034d005cf3f9ce8
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 12:45:29 2013 -0700
+
+    ceph-disk: conditionally remove mount path
+    
+    umount removes it on success; only remove it here if it is still there.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4362934a94c5a6f7521f06aa1255e111094e1d88)
+
+commit 153994cdcf9bc4a9719849d951bec5f4983340b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 26 17:04:14 2013 -0700
+
+    ceph-disk: ignore udevadm settle return code
+    
+    If we time out, just continue and let the next step fail.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ae297ccb812c00204ec0c4dc614821c506062b9)
+
+commit d714049d530dc9611cf5f2fcb9b272c355dbe9f1
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 17:45:32 2013 +0100
+
+    ceph-disk: rename some local variabels in list_*partitions
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b7d7e6894c550a7afa8dfb5bfa2bc54b5d331178)
+
+commit ecb34b81b843adaf9455503eb3ecc69ea94652a3
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:24:00 2013 +0100
+
+    ceph-disk: fix naming of a local variable in find_cluster_by_uuid
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0b5fcfffe6d2f69bd4318cc93ef73195d948b9c5)
+
+commit 63eb85072f6a61552725bd04dfb261e5cd52a60a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:18:17 2013 +0100
+
+    ceph-disk: rename some constants to upper case variable names
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8a999ded088e688fd3f4a7c27127b7c06f0b2f66)
+
+commit d26a03422a37f16d609de12f8973f3c32ffedae0
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:15:29 2013 +0100
+
+    ceph-disk: add some more docstrings
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 86e55f5448c4b5b46b74d2d89b01d1e64b1ea826)
+
+commit bd8bb984806a1dbc3514c3a2a8980a03cfb2bc23
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 14:36:41 2013 +0100
+
+    ceph-disk: print subprocess.CalledProcessError on error
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 543327b1f2a9efe8083bb196433c4bcf83883269)
+
+commit 970348fc46a7e100731ffdc63d20f343dcb1b787
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 13:55:56 2013 +0100
+
+    ceph-disk: fix indention
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 2d26bcc07162a5176cdbc1748b829e3f39653697)
+
+commit 0113e5335e1552294bbf075cfff85c204b164e95
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Tue Mar 26 11:31:16 2013 -0700
+
+    ceph-disk:  udevadm settle before partprobe
+    
+    After changing the partition table, allow the udev event to be
+    processed before calling partprobe.  This helps prevent partprobe
+    from getting a resource busy error on some platforms.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit 9eda8e5d5abf0743a2ad484806cfb2018243515f)
+
+commit 02d48351f73330917185d8980f8ad123b99138b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 10:09:55 2013 -0700
+
+    ceph-disk: re-add python 2.7 dependency comment
+    
+    FIXME!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6d63752c8fde91cdab306d1ca689690b269fe977)
+
+commit e92baf5081a43ff3290eb934612a11fbed946469
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:36:58 2013 +0100
+
+    ceph-disk: cast output of subprocess.Popen() to str()
+    
+    Cast output of subprocess.Popen() to str() to be able to use
+    str.split() and str.splitlines() without warnings from pylint.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 87691dc46edfba11c370592dbb533772190be4b2)
+
+commit 1ffc89af1954643426708987a09291463bd7525b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:02:28 2013 +0100
+
+    ceph-disk: fix adjust_symlink() replace 'journal' with 'target'
+    
+    Replace 'journal' variable with 'target' since journal doesn't
+    exist in this function.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3575feb7d2e5f4e35c5df193a1f8c9f08f88fcf4)
+
+commit 690ab6b3e284e9beed74b3e77612e804464e467d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:01:02 2013 +0100
+
+    ceph-disk: fix adjust_symlink() replace 'canonical' with 'path'
+    
+    Replace 'canonical' variable with 'path' since canonical doesn't
+    exist in this function.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a27cb85b21610b4a9f8eb24f8f6cd04902349728)
+
+commit 329f279cd108c837e434ee3aae6c8ad502ba802f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:56:23 2013 +0100
+
+    ceph-disk: there is no os.path.lstat use os.lstat
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 6d3247b5c02c39a66666a5833106dbc23044e436)
+
+commit ffe024b8b5024cdac256bacb9df433b69385cc0a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:30:21 2013 +0100
+
+    ceph-disk: remove unused variable key from prepare_journal_dev()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 97b4f8d66bef2328fa53f9e508eb38f8b8d49632)
+
+commit 9464284f3ac9737345c595847e165db48adc0a70
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:27:27 2013 +0100
+
+    ceph-disk: fix except to catch OSError
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9daf6cfce2d57509d896eae28bb97146a682a306)
+
+commit 01152115f7bf4a307296c9126174ce09bb0ce22d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:15:36 2013 +0100
+
+    ceph-disk: add missing space after >> operator
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0ada43f79d2b1f9f84367e558c6d1a3e90e5c9b2)
+
+commit b9f86d96f5dd6ca18eb39882caf69e76369b5e0b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:33:09 2013 +0100
+
+    fix: Redefining name 'uuid' from outer scope (line 14)
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit d3c60dc8cad1db1d5df1c740bc805aaf9ba606ba)
+
+commit 6fa6cd85b55935739e16b8ba537f7848dbb0218f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:17:45 2013 +0100
+
+    ceph-disk: remove unused variables from list_partitions()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 6a8120d4b0c4cfa851d473532eb2366534f8653d)
+
+commit 0b4e85fe23e8aa24e8aef8548df2b508bda24752
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:10:22 2013 +0100
+
+    ceph-disk: fix /dev/dm-[0-9] handling list_all_partitions()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3af7a1ac5bf24bf832d7180002281d6b5853e85a)
+
+commit 3ec61f85f6a8a8af8a2b921f94e77a77be77a1d7
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:07:36 2013 +0100
+
+    ceph-disk: rename local variable shadowing builtin
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9bcf5b64f45ab6c4bdedf820ed111319b2dbd778)
+
+commit bd1036dd83fa2ee4807047fb9c2b414ccc9375eb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:05:23 2013 +0100
+
+    ceph-disk: remove twice defined identical function unmount
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4adf088107586be7b356d1e963570cdab232809e)
+
+commit 8dd8cbac6f88fcb9d2c6838b9f961c70cccb9288
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:02:49 2013 +0100
+
+    ceph-disk: remove twice defined function mount
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit ea26ea0d81a23aa76076ad5441c3b1aadfba3b44)
+
+commit 0da87db1c0e4b44b69c19b718d89e7a621979009
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 21:58:22 2013 +0100
+
+    ceph-disk: remove double defined function get_conf
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c57daa3c6e03e8974e133d3a2d9bc3d6f06f4faf)
+
+commit 9c46dfb2512d3b55345faed1bebb9204ba07d030
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 21:51:54 2013 +0100
+
+    ceph-disk: rename local variable shadowing builtin
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 57dde5c8b18ff4ccd53a30bb94119c0ffceb52b7)
+
+commit b807d8ba0a0c23c7dab6893bc5297e07c0fd902f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 18 14:55:24 2013 -0700
+
+    ceph-disk: install and package
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c2602d749023b24ac22d8cfce6e04889078f14d8)
+    
+    Conflicts:
+    
+    	debian/ceph.install
+
+commit 8901e02db05837b12d5e1d0ec6f14f909cc03315
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:50:05 2013 -0700
+
+    ceph-disk: simplify command dispatch
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f287c6f90af0dfdd41358846b069aa3c54b600b3)
+
+commit 3cbc0d0c92ad19d5022c94d9d1ac00aff4eb9b2d
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:40:32 2013 -0700
+
+    ceph-disk: consolidate exceptions
+    
+    Use a single exception type, and catch it at the top level.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a019753bd3897ee0f5d9c3563c653c5457d5e67d)
+
+commit b4176bafa2b0721664332a1bb8b73da54151f685
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:36:45 2013 -0700
+
+    ceph-disk: consolidate ceph-disk-* into a single binary
+    
+     ceph-disk prepare ...
+     ceph-disk activate ...
+     ceph-disk ...
+    
+    This let's us share code (we were already duplicating a bunch of stuff!)
+    and to add new commands easily.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20e4ba5cebb7b4989d1745932f7a412c28504ee1)
+
+commit 455cb32543df8036a605dc7b6c194e58b10bc4ec
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 18 21:13:34 2013 -0700
+
+    ceph-disk-prepare: 'mkfs -t' instead of 'mkfs --type='
+    
+    Older mkfs (el6) doesn't like --type=.
+    
+    Fixes: #4495
+    Reported-by: Alexandre Maragone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d47759429a6e2fdd392265e90b67046302c97644)
+
+commit 19a2cf58504a3305ac9afe3d6642cf68e1b0f6c1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 21:05:07 2013 -0700
+
+    ceph-disk-activate: identify cluster .conf by fsid
+    
+    Determine what cluster the disk belongs to by checking the fsid defined
+    in /etc/ceph/*.conf.  Previously we hard-coded 'ceph'.
+    
+    Note that this has the nice side-effect that if we have a disk with a
+    bad/different fsid, we now fail to activate it.  Previously, we would
+    mount and start ceph-osd, but the daemon would fail to authenticate
+    because it was part of the wrong cluster.
+    
+    Fixes: #3253
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 80af5fb887f30792c342ac16da9ed95d7e89e491)
+
+commit 568485bea8ae9f0f36571de259c7254fa4227b15
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 16:18:26 2013 -0700
+
+    ceph-disk-activate: abort if target position is already mounted
+    
+    If the target position is already a mount point, fail to move our mount
+    over to it.  This usually indicates that a different osd.N from a
+    different cluster instances is in that position.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7370b5564606474f11b9ac5afb7cc60e0ac36ed1)
+
+commit a6ecf9288af248656bc712c1a4de910ff107a9a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 12:05:52 2013 -0700
+
+    Revert "ceph-disk-activate: rely on default/configured keyring path"
+    
+    This reverts commit 936b8f20af1d390976097c427b6e92da4b39b218.
+    
+    This is necessary because we mount the osd in a temporary location.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b6102c0945a0d68b7c5ff7f2254a1a994788ca0f)
+
+commit 7c1edc0c927a27c07b6f1611c2cf9c48f2dc95f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 12:04:44 2013 -0700
+
+    Revert "ceph-disk-activate: don't override default or configured osd journal path"
+    
+    This reverts commit 813e9fe2b4291a1c1922ef78f031daa9b78fe53b.
+    
+    We run --mkfs with the osd disk mounted in a temporary location, so it is
+    necessary to explicitly pass in these paths.
+    
+    If we want to support journals in a different location, we need to make
+    ceph-disk-prepare update the journal symlink accordingly.. not control it via
+    the config option.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e628eee770508e750f64ea50179bbce52e7b8e0)
+
+commit 739b013cfa4b84d36750b80ce44579498e7ea4f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 21:15:11 2013 -0700
+
+    ceph-disk-activate: rely on default/configured keyring path
+    
+    No reason to override the default or configured value here.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 936b8f20af1d390976097c427b6e92da4b39b218)
+
+commit 897413f90c0e802c6824c0ee8d06bdae33979293
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 21:14:53 2013 -0700
+
+    ceph-disk-activate: don't override default or configured osd journal path
+    
+    There is no reason not to rely on the default or obey any configured
+    value here.
+    
+    Fixes: #4031
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 813e9fe2b4291a1c1922ef78f031daa9b78fe53b)
+
+commit 35eac085182e569b17c287c86e1415880909be22
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 5 13:08:26 2013 -0800
+
+    ceph-disk-prepare: move in-use checks to the top, before zap
+    
+    Move the in-use checks to the very top, before we (say) zap!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 32407c994f309cd788bf13fe9af27e17a422309a)
+
+commit 5ad4120a4825d6615778a005cff990afccf30691
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 20:57:52 2013 -0800
+
+    ceph-disk-prepare: verify device is not in use by device-mapper
+    
+    Be nice and tell the user which devices/mappings are consuming the device,
+    too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a6196de9e2f3ca9d67691f79d44e9a9f669443e9)
+
+commit e4a520029196cc76f1ec45695554ca6330bf14aa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:38:57 2013 -0800
+
+    ceph-disk-prepare: verify device is not mounted before using
+    
+    Make sure the data and/or journal device(s) are not in use (mounted)
+    before using them.  Make room for additional "in-use" checks in the future.
+    
+    Closes: #3256
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3bd0ac0ab011c4cdf0121f0d9732938d085fb8bf)
+
+commit d05b439142a24d28f35c6a09a8ab7c9eed503cd5
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:29:57 2013 -0800
+
+    ceph-disk-prepare: clean up stupid check for a digit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f03f62697f170d42b4b62c53d2860ff2f24a2d73)
+
+commit ea07b0e1047a4ac1b4dde88c795709190bd048be
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:08:15 2013 -0800
+
+    ceph-disk-prepare: use os.path.realpath()
+    
+    My janky symlink resolution is broken in various ways.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 59505546e52a175435881b431bd349d532ae627e)
+
+commit 8f7e3e7dbbb9cab869366927c1687c38c012a1ef
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Feb 28 14:16:26 2013 +0100
+
+    ceph.spec.in: add new Requires from ceph-disk-prepare
+    
+    Added new Requires from ceph-disk-prepare: cryptsetup, gptfdisk,
+    parted and util-linux.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 5c3f5c5b69a9edc99138d4f1ddb016689303dc28)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit 3441acf3749dbd99f1e00e5306e8180647085d2a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 27 13:31:12 2013 -0800
+
+    debian: require cryptsetup-bin
+    
+    This is needed for ceph-disk-prepare's dmcrypt support.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cfcaceac44d6b7b7c55e81d0bfb05f4893f3b1d0)
+    
+    Conflicts:
+    
+    	debian/control
+
+commit abdac6fddd25bc71f59c1097246cfa32b33b0f5f
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Mon Apr 15 15:57:00 2013 -0700
+
+    Fix: use absolute path with udev
+    
+    Avoids the following: udevd[61613]: failed to execute '/lib/udev/bash'
+    'bash -c 'while [ ! -e /dev/mapper/....
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 785b25f53dc7f8035eeba2aae8a196e3b102d930)
+
+commit d1775daf68d9aa887116a357cbcaf7c44e091cce
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 16 20:55:03 2013 -0800
+
+    ceph-disk-prepare: -f for mkfs.xfs only
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fecc3c3abf1176f4c7938e161559ea2db59f1cff)
+
+commit 405e0ea1c692b1d62f948734e07c4f919b90d4e0
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 16 16:49:50 2013 -0800
+
+    debian: fix start of ceph-all
+    
+    Tolerate failure, and do ceph-all, not ceph-osd-all.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit aff0bb6fdc8ca358f7ac1e941bb9cfecbefb4bb6)
+
+commit 632be442261cabd56da0dc897b8a47a8e5493dce
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Feb 15 12:24:01 2013 -0800
+
+    ceph-disk-prepare: always force mkfs.xfs
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit d950d83250db3a179c4b629fd32cd7bc8149997e)
+
+commit 28d1193836edee622763e308aad69c7bfe7a4971
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 18:22:45 2013 -0800
+
+    udev: trigger on dmcrypted osd partitions
+    
+    Automatically map encrypted journal partitions.
+    
+    For encrypted OSD partitions, map them, wait for the mapped device to
+    appear, and then ceph-disk-activate.
+    
+    This is much simpler than doing the work in ceph-disk-activate.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e090a92a20f4161f473d16bc966f7d6aacac75ee)
+
+commit 5c5021b43ab70a08d791020262f76879dc77effb
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 21:35:56 2013 -0800
+
+    ceph-disk-prepare: add initial support for dm-crypt
+    
+    Keep keys in /etc/ceph/dmcrypt-keys.
+    
+    Identify partition instances by the partition UUID.  Identify encrypted
+    partitions by a parallel set of type UUIDs.
+    
+    Signed-off-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c6ac0ddf91915ba2aeae46d21367f017e18e82cd)
+
+commit d7084037d7a2395acfe8181f94963c1337c0f95b
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Feb 15 12:22:33 2013 -0800
+
+    ceph-disk-activate: pull mount options from ceph.conf
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit e7040f55f01db3de7d5cebfc79de50c8b6ad5d45)
+
+commit 34fba357ca8b662c7b943d426ed37875c173003f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 17:05:32 2013 -0800
+
+    ceph-disk-activate: use full paths for everything
+    
+    We are run from udev, which doesn't get a decent PATH.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b1c0fccba42dd184a2891ee873c0d6d8f8c79d14)
+
+commit caad18745cd8553a012d9e7f92e59d5b4450dcf2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 17:04:55 2013 -0800
+
+    ceph-disk-prepare: do partprobe after setting final partition type
+    
+    This is necessary to kick udev into processing the updated partition and
+    running its rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 95835de9f80712eb26652ac6b66ba9c5eeb093d6)
+
+commit e1624e4626b4e67909f3bb83920c99ad323ed083
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 21:47:30 2013 -0800
+
+    debian: start/stop ceph-all event on install/uninstall
+    
+    This helps us avoid the confusing situation with upstart where an individual
+    daemon job is running (like ceph-osd id=2) but the container jobs ceph-osd-all
+    and ceph-all are not.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b7b9af5c0d531dcee7ce9b10043a29b0a1b31f47)
+
+commit 656305f65eb33d68b95f4d29f37b031cf6cf889a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 21:37:08 2013 -0800
+
+    ceph-disk-activate: catch daemon start errors
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 690ae05309db118fb3fe390a48df33355fd068a0)
+
+commit 8b771bf929c4a9ab3fd4eb6f40b9222a463fd1a4
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 17:30:19 2013 -0800
+
+    udev: trigger ceph-disk-activate directly from udev
+    
+    There is no need to depend on upstart for this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bd85ee5aa31bfd1f4f0e434f08c2a19414358ef)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit ffb0613efaa9cf919992640322785eedf95ca25f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 16:02:10 2013 -0800
+
+    ceph-disk-activate: auto detect init system
+    
+    Look for an option 'init' in ceph.conf.  Otherwise, check if we're ubuntu.
+    If so, use upstart.  Otherwise, use sysvinit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d1904b2a848af3c02d2065ac2a42abe0e2699d0f)
+
+commit 74b562703c422890c67b2a88d7b18242f64949dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 15:49:42 2013 -0800
+
+    ceph-disk-activate: specify full path for blkid, initctl, service
+    
+    /sbin apparently isn't in the path when udev runs us.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f06b45e66315310abb0720e021da377186455048)
+
+commit 9ea32e5f066bee75923212d8009fb37a365ef32e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:18:23 2013 -0800
+
+    upstart: ceph-hotplug -> ceph-osd-activate
+    
+    This is a more meaningful name.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e011ad128e7f302cb6955d9a7171ac0ec8890ddf)
+
+commit 494533a55010a59a52f6dcbe64f76e677a7dfe8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:14:22 2013 -0800
+
+    upstart/ceph-hotplug: tell activate to start via upstart
+    
+    This will mark the OSD data dir as upstart-managed.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 792e45c63dd7a9622fddd6e15ee4c075f995ea56)
+
+commit 5e0892fd8cbc2571d94cffc0b64daaa4ec6c38f8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:08:22 2013 -0800
+
+    ceph-disk-prepare: refactor to support DIR, DISK, or PARTITION for data or journal
+    
+    Lots of code reorganization collapsed into a single commit here.
+    
+    - detect whether the user gave us a directory, disk, or partition, and Do The
+    Right Thing
+    - allow them to force that the input was of type X, for the careful/paranoid.
+    - make --zap-disk an option -- no longer the default
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b2ff6e8c9d96dee2c063b126de7030a5c2ae0d02)
+
+commit aa428017e4aad9872a54cd87036ef2bb528aac7b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 20:33:16 2013 -0800
+
+    ceph-disk-activate: detect whether PATH is mount or dir
+    
+    remove in-the-way symlinks in /var/lib/ceph/osd
+    
+    This is simpler.  Just detect what the path is and Do The Right Thing.
+    
+    Closes #3341 (which wanted to make --mount the default)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 191d5f7535f8d96d493e1b35b43a421c67c168ea)
+
+commit e6d5aa053dbaaf07c94e35406c23f4ca009fe2da
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 20:32:47 2013 -0800
+
+    ceph-disk-activate: add --mark-init INITSYSTEM option
+    
+    Do not assume we will manage via upstart; let that be passed down via the
+    command line.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fd4a921085a861e4aa428376219bb39055731f2b)
+
+commit fa23919eae363291b4807afe3fadd34f94e9ddcb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 14:44:26 2013 -0800
+
+    ceph-disk-activate: factor mounting out of activate
+    
+    The activate stuff is generic for any OSD, regardless of whether we want
+    to mount it or not.  Pull that part out.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 07655288281c9c6f691f87352dc26b7c11ae07e8)
+
+commit 181ebdee0addfe3d13c13342def94ecf9415d6d5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 18:17:55 2013 -0800
+
+    debian: put ceph-mds upstart conf in ceph-mds package
+    
+    Fixes: #3157
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 23ad3a46a0099e263f43e0f0c1df1d21cfe58b3f)
+
+commit 919b0aedf7286e85e6ddc23f99f80f29e1d35db6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 17:02:52 2013 -0800
+
+    debian: include /var/lib/ceph/bootstrap-mds in package
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e80675a0f333c04452d4822fd0eb3c6e92eda3df)
+
+commit f97f49b1e7ca8555c945b93fbc8241f5fa620dd8
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 16:26:14 2013 -0800
+
+    ceph-create-keys: create mds bootstrap key
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 809143f16c70483ba5bb429dea812d31b67f2b49)
+
+commit f43c339d67b78e6afc3737d26cea4bc05dcea900
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 14:45:43 2013 -0800
+
+    upstart/ceph-hotplug: drop -- in ceph-disk-activate args
+    
+    We would like to transition to
+    
+     ceph-disk-activate --mount DEV
+    
+    and away from a generic multi-definition PATH argument.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4698b6a1035dee8509ce2d4dab7b34a16b78f7cd)
+
+commit 39df4c81f2166b7e9ac89c591b268b60de4042e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 22:53:23 2013 -0800
+
+    init-ceph: iterate/locate local sysvinit-tagged directories
+    
+    Search /var/lib/ceph/$type/ceph-$id and start/stop those daemons if
+    present and tagged with the sysvinit file.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8f528a4070dd3aa0b25c435c6234032aee39b21)
+
+commit 05efb7ab5b289472a1af05d6aa100a7385be304e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 22:52:03 2013 -0800
+
+    init-ceph: consider sysvinit-tagged dirs as local
+    
+    If there is a 'sysvinit' file in the daemon directory in the default
+    location (/var/lib/ceph/$type/ceph-$id), consider it sysvinit-managed.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b8aa4769a62e0d88174678cbefd89d9ee2baceea)
+
+commit 8c4c53abf84d30ff4a605225a22575f56d946971
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 11 17:39:03 2013 -0800
+
+    ceph-disk-prepare: align mkfs, mount config options with mkcephfs
+    
+    'osd mkfs ...', not 'osd fs mkfs ...'.  Sigh.  Support both.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit af2372ca4a702da70275edd1b1357fcff51e6ae2)
+
+commit a8e7e9df61a7229d9e2b4b4dedc68b5c1bf15c38
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 23 10:00:38 2013 -0700
+
+    init-ceph: fix (and simplify) pushing ceph.conf to remote unique name
+    
+    The old code would only do the push once per remote node (due to the
+    list in $pushed_to) but would reset $unique on each attempt.  This would
+    break if a remote host was processed twice.
+    
+    Fix by just skipping the $pushed_to optimization entirely.
+    
+    Fixes: #4794
+    Reported-by: Andreas Friedrich <andreas.friedrich@ts.fujitsu.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ccbc4dbc6edf09626459ca52a53a72682f541e86)
+
+commit 9374baccb8a71e1fe5ea2e614feac93e865dd027
+Merge: b6b4ebe aa37726
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 23 12:04:45 2013 -0700
+
+    Merge pull request #238 from ceph/wip-bobtail-rbd-backports-req-order
+    
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+
+commit aa37726b8d43d08b495d55dc2aef93854e9539c5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 23 09:18:30 2013 -0700
+
+    rbd: only set STRIPINGV2 feature when needed
+    
+    Only set the STRIPINGV2 feature if the striping parameters are non-default.
+    Specifically, fix the case where the passed-in size and count are == 0.
+    
+    Fixes: #4710
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5926ffa576e9477324ca00eaec731a224195e7db)
+    
+    Conflicts:
+    
+    	src/rbd.cc
+
+commit 0e2266dba912cde2c4a86ba7055c8bec3a5bc1a5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 14:16:56 2013 -0700
+
+    LibrbdWriteback:  complete writes strictly in order
+    
+    RADOS returns writes to the same object in the same order. The
+    ObjectCacher relies on this assumption to make sure previous writes
+    are complete and maintain consistency. Reads, however, may be
+    reordered with respect to each other. When writing to an rbd clone,
+    reads to the parent must be performed when the object does not exist
+    in the child yet. These reads may be reordered, resulting in the
+    original writes being reordered. This breaks the assmuptions of the
+    ObjectCacher, causing an assert to fail.
+    
+    To fix this, keep a per-object queue of outstanding writes to an
+    object in the LibrbdWriteback handler, and finish them in the order in
+    which they were sent.
+    
+    Fixes: #4531
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 06d05e5ed7e09fa873cc05021d16f21317a1f8ef)
+
+commit 7a11c2505a8b74babee90cca945c2a068a656ec1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:22:02 2013 -0700
+
+    LibrbdWriteback: removed unused and undefined method
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 909dfb7d183f54f7583a70c05550bec07856d4e4)
+
+commit 884438febaf12dc380d4a1a3c3c8d3cbefd32399
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:06:36 2013 -0700
+
+    LibrbdWriteback: use a tid_t for tids
+    
+    An int could be much smaller, leading to overflow and bad behavior.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9d19961539b2d50d0c9edee1e3d5ac6912a37f24)
+
+commit 124f81cc5fa5a3ac9e7777ad49dd42d864fd7f68
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:03:04 2013 -0700
+
+    WritebackHandler: make read return nothing
+    
+    The tid returned by reads is ignored, and would make tracking writes
+    internally more difficult by using the same id-space as them. Make read
+    void and update all implementations.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 870f9cd421ca7b0094f9f89e13b1898a8302c494)
+
+commit 13ba07a0f3af479861c0e83e3e6441fef1bc8a99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Apr 1 14:51:46 2013 -0700
+
+    ObjectCacher: deduplicate final part of flush_set()
+    
+    Both versions of flush_set() did the same thing. Move it into a
+    helper called from both.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f5b81d8d167d1aa7f82a5776bbb1f319063ab809)
+
+commit 7bc8df1f3e3b60b59672a082eba374729a6866b8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 11:35:46 2013 -0700
+
+    test_stress_watch: remove bogus asserts
+    
+    There's no reason to check the duration of a watch. The notify will
+    timeout after 30s on the OSD, but there's no guarantee the client will
+    see that in any bounded time. This test is really meant as a stress
+    test of the OSDs anyway, not of the clients, so just remove asserts
+    about operation duration.
+    
+    Fixes: #4591
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sam Just <sam.just@inktank.com>
+    (cherry picked from commit 4b656730ffff21132f358c2b9a63504dfbf0998d)
+
+commit 4a1c27c0e6e2e70be2b58e01872e321352c8e484
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Apr 1 11:09:52 2013 -0700
+
+    librados: don't use lockdep for AioCompletionImpl
+    
+    This is a quick workaround for the next branch. A more complete fix
+    will be done for the master branch. This does not affect correctness,
+    just what qa runs with lockdep enabled do.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 267ce0d90b8f3afaaddfdc0556c9bafbf4628426)
+
+commit d36c5b5bd30a9d7e60403a09cedaea28bdc29e99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 29 12:46:27 2013 -0700
+
+    librados: move snapc creation to caller for aio_operate
+    
+    The common case already has a snapshot context, so avoid duplicating
+    it (copying a potentially large vector) in IoCtxImpl::aio_operate().
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4c4d5591bdb048cd9ffa25b529c6127356e7f9a7)
+
+commit 31a45e8e23ebf9815ce5a4e0c89ddbf384702aad
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 21 16:04:10 2013 -0700
+
+    librbd: add an async flush
+    
+    At this point it's a simple wrapper around the ObjectCacher or
+    librados.
+    
+    This is needed for QEMU so that its main thread can continue while a
+    flush is occurring. Since this will be backported, don't update the
+    librbd version yet, just add a #define that QEMU and others can use to
+    detect the presence of aio_flush().
+    
+    Refs: #3737
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 302b93c478b3f4bc2c82bfb08329e3c98389dd97)
+
+commit f2e490cb5b1f322e637d787e788733ac4b2496f4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:42:10 2013 -0700
+
+    librbd: use the same IoCtx for each request
+    
+    Before we were duplicating the IoCtx for each new request since they
+    could have a different snapshot context or read from a different
+    snapshot id. Since librados now supports setting these explicitly
+    for a given request, do that instead.
+    
+    Since librados tracks outstanding requests on a per-IoCtx basis, this
+    also fixes a bug that causes flush() without caching to ignore
+    all the outstanding requests, since they were to separate,
+    duplicate IoCtxs.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 860493e7ff0d87d02069b243fc1c8326ce0721f9)
+
+commit cbb37fb5470f22949a4dbb747f428943cce4aca7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:37:27 2013 -0700
+
+    librbd: add an is_complete() method to AioCompletions
+    
+    Mainly this is useful for testing, like flushing and checking that
+    all pending writes are complete after the flush finishes.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2ae32068dee22a0ca0698e230ead98f2eeeff3e6)
+
+commit f9bcffa2482220202e68d6bac70216d44ee5046b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:32:29 2013 -0700
+
+    librados: add versions of a couple functions taking explicit snap args
+    
+    Usually the snapid to read from or the snapcontext to send with a write
+    are determined implicitly by the IoCtx the operations are done on.
+    
+    This makes it difficult to have multiple ops in flight to the same
+    IoCtx using different snapcontexts or reading from different snapshots,
+    particularly when more than one operation may be needed past the initial
+    scheduling.
+    
+    Add versions of aio_read, aio_sparse_read, and aio_operate
+    that don't depend on the snap id or snapcontext stored in the IoCtx,
+    but get them from the caller. Specifying this information for each
+    operation can be a more useful interface in general, but for now just
+    add it for the methods used by librbd.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f06debef6c293750539501ec4e6103e5ae078392)
+
+commit fb95b800a4cc431a8bd98692fffbfc150f6f8d67
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 14:48:31 2013 -0700
+
+    librados: add async flush interface
+    
+    Sometimes you don't want flush to block, and can't modify
+    already scheduled aio_writes. This will be useful for a
+    librbd async flush interface.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 7cc0940f89070dadab5b9102b1e78362f762f402)
+    
+    Conflicts:
+    
+    	src/include/rados/librados.h
+    	src/include/rados/librados.hpp
+
+commit 3a61d17b6729b73d0bf2df106f54dafb603af646
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 28 10:34:37 2013 -0700
+
+    ObjectCacher: remove unneeded var from flush_set()
+    
+    The gather will only have subs if there is something to flush. Remove
+    the safe variable, which indicates the same thing, and convert the
+    conditionals that used it to an else branch. Movinig gather.activate()
+    inside the has_subs() check has no effect since activate() does
+    nothing when there are no subs.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 553aaac8a19e2359acf6d9d2e1bb4ef0bdba7801)
+
+commit ee7bf281005cce671e467ca9b9eff0ba7eb9899e
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 22 12:17:43 2013 -0700
+
+    ObjectCacher: remove NULL checks in flush_set()
+    
+    Callers will always pass a callback, so assert this and remove the
+    checks for it being NULL.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 41568b904de6d155e5ee87c68e9c31cbb69508e5)
+
+commit d86f9b1dd370563170ab875c6a7de511f05acbf7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 22 12:13:36 2013 -0700
+
+    ObjectCacher: always complete flush_set() callback
+    
+    This removes the last remnants of
+    b5e9995f59d363ba00d9cac413d9b754ee44e370. If there's nothing to flush,
+    immediately call the callback instead of deleting it. Callers were
+    assuming they were responsible for completing the callback whenever
+    flush_set() returned true, and always called complete(0) in this
+    case. Simplify the interface and just do this in flush_set(), so that
+    it always calls the callback.
+    
+    Since C_GatherBuilder deletes its finisher if there are no subs,
+    only set its finisher when subs are present. This way we can still
+    call ->complete() for the callback.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 92db06c05dc2cad8ed31648cb08866781aee2855)
+    
+    Conflicts:
+    
+    	src/client/Client.cc
+
+commit 00dfb3f06fb6c2581a3f71126adcde0126e5e207
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jan 29 14:22:15 2013 -0800
+
+    ObjectCacher: fix flush_set when no flushing is needed
+    
+    C_GatherBuilder takes ownership of the Context we pass it. Deleting it
+    in flush_set after constructing the C_GatherBuilder results in a
+    double delete.
+    
+    Fixes: #3946
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 3bc21143552b35698c9916c67494336de8964d2a)
+
+commit 0f2e5d36e4dcd073174715c310fa22a835ca1259
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Fri Jan 18 14:59:12 2013 -0600
+
+    objectcacher: Remove commit_set, use flush_set
+    
+    commit_set() and flush_set() are identical in functionality,
+    so use flush_set everywhere and remove commit_set from
+    the code.
+    
+    Also fixes a bug in flush_set where the finisher context was
+    getting freed twice if no objects needed to be flushed.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 72147fd3a1da8ecbcb31ddf6b66a158d71933909)
+
+commit 3b0c565dceb41dcaf53965321ef420641bcb699b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 09:42:43 2013 -0700
+
+    librbd: make aio_writes to the cache always non-blocking by default
+    
+    When the ObjectCacher's writex blocks, it affects the thread requesting
+    the aio, which can cause starvation for other I/O when used by QEMU.
+    
+    Preserve the old behavior via a config option in case this has any
+    bad side-effects, like too much memory usage under heavy write loads.
+    
+    Fixes: #4091
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 03ac01fa6a94fa7a66ede057e9267e0a562c3cdb)
+
+commit e237dfc742bba76664861a8868c502946fe53548
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 09:37:21 2013 -0700
+
+    ObjectCacher: optionally make writex always non-blocking
+    
+    Add a callback argument to writex, and a finisher to run the
+    callbacks. Move the check for dirty+tx > max_dirty into a helper that
+    can be called from a wrapper around the callbacks from writex, or from
+    the current place in _wait_for_write().
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c21250406eced8e5c467f492a2148c57978634f4)
+
+commit 7bc1596b90d2d5853659cb3ffbe9653b633345f4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 17:30:42 2013 -0700
+
+    librbd: flush cache when set_snap() is called
+    
+    If there are writes pending, they should be sent while the image
+    is still writeable. If the image becomes read-only, flushing the
+    cache will just mark everything dirty again due to -EROFS.
+    
+    Fixes: #4525
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 613b7085bb48cde1e464b7a97c00b8751e0e917f)
+
+commit 9facdcac7bb8ab92f1ab486eaabee681e6b6519d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 15 17:28:13 2013 -0700
+
+    librbd: optionally wait for a flush before enabling writeback
+    
+    Older guests may not send flushes properly (i.e. never), so if this is
+    enabled, rbd_cache=true is safe for them transparently.
+    
+    Disable by default, since it will unnecessarily slow down newer guest
+    boot, and prevent writeback caching for things that don't need to send
+    flushes, like the command line tool.
+    
+    Refs: #3817
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1597b3e3a1d776b56e05c57d7c3de396f4f2b5b2)
+
+commit 1c44b66fb371a5864860c067bb5f59374f43005d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 18:57:24 2013 -0800
+
+    librbd: invalidate cache when flattening
+    
+    The cache stores which objects don't exist. Flatten bypasses the cache
+    when doing its copyups, so when it is done the -ENOENT from the cache
+    is treated as zeroes instead of 'need to read from parent'.
+    
+    Clients that have the image open need to forgot about the cached
+    non-existent objects as well. Do this during ictx_refresh, while the
+    parent_lock is held exclusively so no new reads from the parent can
+    happen until the updated parent metadata is visible, so no new reads
+    from the parent will occur.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 46e8fc00b2dc8eb17d8777b6ef5ad1cfcc389cea)
+
+commit d9ca1b00a4dacb4715a69a276a644d645d0d5bc1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 17:53:31 2013 -0800
+
+    ObjectCacher: add a method to clear -ENOENT caching
+    
+    Clear the exists and complete flags for any objects that have exists
+    set to false, and force any in-flight reads to retry if they get
+    -ENOENT instead of generating zeros.
+    
+    This is useful for getting the cache into a consistent state for rbd
+    after an image has been flattened, since many objects which previously
+    did not exist and went up to the parent to retrieve data may now exist
+    in the child.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f2a23dc0b092c5ac081893e8f28c6d4bcabd0c2e)
+
+commit 1e51be05b7909a676ae499ca84e4cce94bb50da4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 17:49:27 2013 -0800
+
+    ObjectCacher: keep track of outstanding reads on an object
+    
+    Reads always use C_ReadFinish as a callback (and they are the only
+    user of this callback). Keep an xlist of these for each object, so
+    they can remove themselves as they finish. To prevent racing requests
+    and with discard removing objects from the cache, clear the xlist in
+    the object destructor, so if the Object is still valid the set_item
+    will still be on the list.
+    
+    Make the ObjectCacher constructor take an Object* instead of the pool
+    and object id, which are derived from the Object* anyway.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f6f876fe51e40570596c25ac84ba3689f72776c2)
+
+commit f2bcf241257500f5784b216b7a377ddec4669531
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 16:09:26 2013 -0800
+
+    test_rbd: move flatten tests back into TestClone
+    
+    They need the same setup, and it's easy enough to run specific
+    subtests. Making them a separate subclass accidentally duplicated
+    tests from TestClone.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9c693d7e8312026f6d8d9586381b026ada35d808)
+
+commit cd989681823a2b8521c789f5d4fea5525ac53911
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 26 13:20:08 2013 -0800
+
+    librbd: fix rollback size
+    
+    The duplicate calls to get_image_size() and get_snap_size() replaced
+    by 5806226cf0743bb44eaf7bc815897c6846d43233 uncovered this. The first
+    call was using the currently set snap_id instead of the snapshot being
+    rolled back to.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d6c126e2131fefab6df676f2b9d0addf78f7a488)
+
+commit 796066b7a8324b12a660d9b80b927e643db7529e
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 12:05:16 2013 -0800
+
+    Merge branch 'wip-4249' into wip-4249-master
+    
+    Make snap_rollback() only take a read lock on snap_lock, since
+    it does not modify snapshot-related fields.
+    Conflicts:
+    	src/librbd/internal.cc
+    (cherry picked from commit db5fc2270f91aae220fc3c97b0c62e92e263527b)
+
+commit 34e9030e50fe15a320cbfbc9854c86deb3a9c57c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:26:45 2013 -0800
+
+    librbd: make sure racing flattens don't crash
+    
+    The only way for a parent to disappear is a racing flatten completing,
+    or possibly in the future the image being forcibly removed. In either
+    case, continuing to flatten makes no sense, so stop early.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit a1ae8562877d1b902918e866a1699214090c40bd)
+
+commit 6e6636d50840e97022c8e4a1072f42fee15050a9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:17:18 2013 -0800
+
+    librbd: use rwlocks instead of mutexes for several fields
+    
+    Image metadata like snapshots, size, and parent is frequently read,
+    but rarely updated. During flatten, we were depending on the parent
+    lock to prevent the parent ImageCtx from disappearing out from under
+    us while we read from it. The copy-up path also needed the parent lock
+    to be able to read from the parent image, which lead to a deadlock.
+    
+    Convert parent_lock, snap_lock, and md_lock to RWLocks, and change
+    their use to read instead of exclusive locks where appropriate. The
+    main place exclusive locks are needed is in ictx_refresh, so this is
+    pretty simple. This fixes the deadlock, since parent_lock is only
+    needed for read access in both flatten and the copy-up operation.
+    
+    cache_lock and refresh_lock are only really used for exclusive access,
+    so leave them as regular mutexes.
+    
+    One downside to this is that there's no way to assert is_locked()
+    for RWLocks, so we'll have to be very careful about changing code
+    in the future.
+    
+    Fixes: #3665
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 995ff0e3eaa560b242da8c019a2e11e735e854f7)
+
+commit 9b292199ef63822462c67cb6cf1cca6a53cf9fd7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:15:41 2013 -0800
+
+    common: add lockers for RWLocks
+    
+    This makes them easier to use, especially instead of existing mutexes.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e0f8e5a80d6d22bd4dee79a4996ea7265d11b0c1)
+
+commit d8ac6cbf883fce0f39d661a879ff2462479d5494
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 23:22:59 2013 -0800
+
+    objecter: initialize linger op snapid
+    
+    Since they are write ops now, it must be CEPH_NOSNAP or the OSD
+    returns EINVAL.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 15bb9ba9fbb4185708399ed6deee070d888ef6d2)
+
+commit 9ea4dac1db46c20cd83c5af5d29d9ce7a5768380
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 15:44:19 2013 -0800
+
+    objecter: separate out linger_read() and linger_mutate()
+    
+    A watch is a mutation, while a notify is a read.  The mutations need to
+    pass in a proper snap context to be fully correct.
+    
+    Also, make the WRITE flag implicit so the caller doesn't need to pass it
+    in.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6c08c7c1c6d354d090eb16df279d4b63ca7a355a)
+
+commit d9636faabfdb2c0a876f0afa092a2ab925cf8983
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 15:31:08 2013 -0800
+
+    osd: make watch OSDOp print sanely
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit de4fa95f03b99a55b5713911c364d7e2a4588679)
+
+commit 959bfe9097a2cfa48e2ab21f11bd32c1349807aa
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 13:28:47 2013 -0800
+
+    osdc/Objecter: unwatch is a mutation, not a read
+    
+    This was causing librados to unblock after the ACK on unwatch, which meant
+    that librbd users raced and tried to delete the image before the unwatch
+    change was committed..and got EBUSY.  See #3958.
+    
+    The watch operation has a similar problem.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fea77682a6cf9c7571573bc9791c03373d1d976d)
+    
+    Conflicts:
+    
+    	src/librados/IoCtxImpl.cc
+
+commit b6b4ebed559db8bf74da4f482201948b5abc9225
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 11:15:58 2013 -0800
+
+    osd: an interval can't go readwrite if its acting is empty
+    
+    Let's not forget that min_size can be zero.
+    
+    Fixes: #4159
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4277265d99647c9fe950ba627e5d86234cfd70a9)
+
+commit 055d746c3bca0d7067b71ed1ff6d1c1cce42b5ed
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 19 08:29:53 2013 -0800
+
+    mon: restrict pool size to 1..10
+    
+    See: #4159
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 30b8d653751acb4bc4be5ca611f154e19afe910a)
+
+commit c073bd25610ba556f3b30e6c583d1d74a10f09e2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 19 13:05:43 2013 -0700
+
+    init-ceph: do not stop start on first failure
+    
+    When starting we often loop over many daemon instances.  Currently we stop
+    on the first error and do not try to start other daemons.
+    
+    Instead, try them all, but return a failure if anything did not start.
+    
+    Fixes: #2545
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit d395aa521e8a4b295ed2b08dd7cfb7d9f995fcf7)
+    
+    Conflicts:
+    
+    	src/init-ceph.in
+
+commit daa6ed2bfe790f83f986f51d0917d6268d3e9d78
+Merge: 1d0bf61 9fe57e2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 11 13:00:27 2013 -0700
+
+    Merge pull request #210 from dalgaaf/wip-da-bobtail-pybind
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 9fe57e2bea89243a0080e667f90b7d4fb15ed883
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Apr 5 15:55:34 2013 +0200
+
+    rados.py: fix create_pool()
+    
+    Call rados_pool_create_with_all() only if auid and crush_rule
+    are set properly. In case only crush_rule is set call
+    rados_pool_create_with_crush_rule() on librados, not the other
+    way around.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 94a1f25e7230a700f06a2699c9c2b99ec1bf7144)
+
+commit 1d0bf61da98bec7318f44b95a49298e13b792b70
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Apr 8 13:52:32 2013 -0700
+
+    mon: Use _daemon version of argparse functions
+    
+    Allow argparse functions to fail if no argument given by using
+    special versions that avoid the default CLI behavior of "cerr/exit"
+    
+    Fixes: #4678
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit be801f6c506d9fbfb6c06afe94663abdb0037be5)
+    
+    Conflicts:
+    	src/mon/Monitor.cc
+
+commit 3769250acf42a751ee8dfa95207a4ffafd25574a
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Apr 8 13:49:22 2013 -0700
+
+    ceph_argparse: add _daemon versions of argparse calls
+    
+    mon needs to call argparse for a couple of -- options, and the
+    argparse_witharg routines were attempting to cerr/exit on missing
+    arguments.  This is appropriate for the CLI usage, but not the daemon
+    usage.  Add a 'cli' flag that can be set false for the daemon usage
+    (and cause the parsing routine to return false instead of exit).
+    
+    The daemon's parsing code due for a rewrite soon.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c76bbc2e6df16d283cac3613628a44937e38bed8)
+
+commit 1f3b4917cc5c4d796a96637912d2ae5ce2877861
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Wed Feb 6 15:27:13 2013 -0200
+
+    silence logrotate some more
+    
+    I was getting email with logrotate error output from “which invoke-rc.d�
+    on systems without an invoke-rc.d.  This patch silences it.
+    
+    Silence stderr from which when running logrotate
+    
+    From: Alexandre Oliva <oliva@gnu.org>
+    
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    (cherry picked from commit d02340d90c9d30d44c962bea7171db3fe3bfba8e)
+
+commit e6eace2334d04fc9292c96a843304d1748b5bbe7
+Merge: 8cb13a1 1507443
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 29 12:14:22 2013 -0700
+
+    Merge remote-tracking branch 'upstream/bobtail-4556' into bobtail
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 8cb13a195c592fd2f17889e77d717c1864f97dca
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 14 14:03:56 2013 -0800
+
+    OSD: always activate_map in advance_pgs, only send messages if up
+    
+    We should always handle_activate_map() after handle_advance_map() in
+    order to kick the pg into a valid peering state for processing requests
+    prior to dropping the lock.
+    
+    Additionally, we would prefer to avoid sending irrelevant messages
+    during boot, so only send if we are up according to the current service
+    osdmap.
+    
+    Fixes: #4572
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4dfcad44431855ba7d68a1ccb41dc3cb5db6bb50)
+
+commit d89ab0ea6fa8d0961cad82f6a81eccbd3bbd3f55
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 28 14:09:17 2013 -0700
+
+    PG: update PGPool::name in PGPool::update
+    
+    Fixes: #4471
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f804892d725cfa25c242bdc577b12ee81dcc0dcc)
+
+commit 750626108616761512271d5a4f10dee82a54e460
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 26 15:10:37 2013 -0700
+
+    ReplicatedPG: send entire stats on OP_BACKFILL_FINISH
+    
+    Otherwise, we update the stat.stat structure, but not the
+    stat.invalid_stats part.  This will result in a recently
+    split primary propogating the invalid stats but not the
+    invalid marker.  Sending the whole pg_stat_t structure
+    also mirrors MOSDSubOp.
+    
+    Fixes: #4557
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76b296f01fd0d337c8fc9f79013883e62146f0c6)
+
+commit 1507443271fda933032ef0877aff1890d4fd5b63
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:19:03 2013 -0700
+
+    osd: disallow classes with flags==0
+    
+    They must be RD, WR, or something....
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 89c69016e1dddb9f3ca40fd699e4a995ef1e3eee)
+
+commit 6b6e0cef99e66b46ade35ab262f071982049bc22
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 12:59:41 2013 -0700
+
+    osd: EINVAL when rmw_flags is 0
+    
+    A broken client (e.g., v0.56) can send a request that ends up with an
+    rmw_flags of 0.  Treat this as invalid and return EINVAL.
+    
+    Fixes: #4556
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f2dda43c9ed4fda9cfa87362514985ee79e0ae15)
+
+commit 4bdd37495cedb1cf30a5311548492fe3f5db6e92
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:08:38 2013 -0700
+
+    osd: fix detection of non-existent class method
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 50b831e3641c21cd5b145271688189e199f432d1)
+
+commit f28800f8b65e90a3b96429f07197236e6c9bf1a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:12:38 2013 -0700
+
+    osd: tolerate rmw_flags==0
+    
+    We will let OSD return a proper error instead of asserting.
+    
+    This is effectively a backport of c313423cfda55a2231e000cd5ff20729310867f8.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 94321ccdff81d5d6cea1acdb54344c3d930a49eb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 17:39:19 2013 -0800
+
+    test_librbd_fsx: fix image closing
+    
+    Always close the image we opened in check_clone(), and check the
+    return code of the rbd_close() called before cloning.
+    
+    Refs: #3958
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 94ae72546507799667197fd941633bb1fd2520c2)
+
+commit 7fbc1ab69267e6be96665b0d4e277dc4a07f9220
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 17:05:42 2013 -0700
+
+    rbd: remove fiemap use from import
+    
+    On some kernels and filesystems fiemap can be racy and provide
+    incorrect data even after an fsync. Later we can use SEEK_HOLE and
+    SEEK_DATA, but for now just detect zero runs like we do with stdin.
+    
+    Basically this adapts import from stdin to work in the case of a file
+    or block device, and gets rid of other cruft in the import that used
+    fiemap.
+    
+    Fixes: #4388
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3091283895e8ffa3e4bda13399318a6e720d498f)
diff --git a/doc/changelog/v0.56.6.txt b/doc/changelog/v0.56.6.txt
new file mode 100644
index 000000000..17818d278
--- /dev/null
+++ b/doc/changelog/v0.56.6.txt
@@ -0,0 +1,40 @@
+commit 95a0bda7f007a33b0dc7adf4b330778fa1e5d70c
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri May 3 12:53:00 2013 -0700
+
+    v0.56.6
+
+commit 6dbdcf5a210febb5e0dd585e0e599ac807642210
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri May 3 12:45:24 2013 -0700
+
+    ceph.spec.in:  Fix platform dependecies
+
+    Picked up an incorrect dependency merging the rbd udev rules update.
+
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 05af17e697eb95b2a807d9c05cde39106c5ecee9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 22 12:48:56 2013 -0700
+
+    rgw: don't send tail to gc if copying object to itself
+
+    Fixes: #4776
+    Backport: bobtail
+    Need to make sure that when copying an object into itself we don't
+    send the tail to the garbage collection.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit de5d1da810732ee48f41e8be18257053d862301b)
+
+commit f0eb20a7b0f7c8afadc21cc063f1f289b5092bab
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 3 12:24:21 2013 -0700
+
+    ceph_common.sh: re-sync get_name_list with master
+
+    We backported various items but didn't catch all the changes!  :(
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.56.7.txt b/doc/changelog/v0.56.7.txt
new file mode 100644
index 000000000..b01c0de10
--- /dev/null
+++ b/doc/changelog/v0.56.7.txt
@@ -0,0 +1,454 @@
+commit 14f23ab86b0058a8651895b3dc972a29459f3a33
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 10:44:32 2013 -0700
+
+    v0.56.7
+
+commit 8551be345c86837e0893fdf6c9c5b0af523f50f8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
+
+commit 1670a73b56e0b407b65334d8f03d3ebb9558ac8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+
+    we can hang forever.
+
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+
+    Refs: #5919
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit 6cf05375abea26d2645e3301c9082c64ddf31a55
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
+
+commit 6a37a62b6f794026b82b88630519ec2cde4f20d6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 12 10:05:44 2013 -0700
+
+    rgw: fix multi delete
+
+    Fixes: #5931
+    Backport: bobtail, cuttlefish
+
+    Fix a bad check, where we compare the wrong field. Instead of
+    comparing the ret code to 0, we compare the string value to 0
+    which generates implicit casting, hence the crash.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f9f1c48ad799da2b4be0077bf9d61ae116da33d7)
+
+    Conflicts:
+	src/rgw/rgw_rest_s3.cc
+
+commit 586c68f544c95f9e379df7e4d2705a3090baca49
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 21:21:09 2013 -0700
+
+    ceph-fuse: create finisher threads after fork()
+
+    The ObjectCacher and MonClient classes both instantiate Finisher
+    threads.  We need to make sure they are created *after* the fork(2)
+    or else the process will fail to join() them on shutdown, and the
+    threads will not exist while fuse is doing useful work.
+
+    Put CephFuse on the heap and move all this initalization into the child
+    block, and make sure errors are passed back to the parent.
+
+    Fix-proposed-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c1198d680587928b390bb82c87442384331afd40
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 21 16:22:08 2013 -0700
+
+    debian: update postinst, prerm hooks
+
+    This syncs up the hooks with the latest master versions.  In particular,
+    do not blindly stop/restart daemons on package upgrade!
+
+    Fixes: #5414
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit de8900dcd079207852b6ce0b51473037be9ae956
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:33:09 2013 -0700
+
+    os/FileStore: use fdatasync(2) instead of sync_file_range(2)
+
+    This fixes data corruption on XFS.  Backported from
+    ffade3c85dfffa13a16edd9630a52d99eb8a413d.
+
+    Fixes: #4976
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit cbbad5b5d917fe74d6cbc50a259f9dbaeda54ca8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:06:59 2013 -0700
+
+    chmod +x iogen.sh
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit fcd65139135d907098a47427d8503ac6d6042f81
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 14 14:53:54 2013 -0700
+
+    rgw: escape prefix correctly when listing objects
+
+    Fixes: #5362
+    When listing objects prefix needs to be escaped correctly (the
+    same as with the marker). Otherwise listing objects with prefix
+    that starts with underscore doesn't work.
+    Backport: bobtail, cuttlefish
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit a8f9d57a15ad7a69d53aa8fc6090fd1b394b616a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Sun Mar 31 00:02:15 2013 -0700
+
+    rgw: translate object marker to raw format
+
+    Fixes: #4600
+    Object marker should be treated as an object, so that name is formatted
+    correctly when getting the raw oid.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b083dece36a050ec15ac41a275aeef0ece1ac009)
+
+commit e1d41901cde97a77fc1fda2d7f6e78a7cea61c5c
+Author: tamil <tamil.muthamizhan@inktank.com>
+Date:   Thu Jun 13 13:50:56 2013 -0700
+
+    adding iogen.sh
+
+    Signed-off-by: tamil <tamil.muthamizhan@inktank.com>
+
+commit 32b2f11366418f9a0cc0659be22cf15fbafbe3b1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 16:35:54 2013 -0700
+
+    osd: do not include logbl in scrub map
+
+    This is a potentially use object/file, usually prefixed by a zeroed region
+    on disk, that is not used by scrub at all.  It dates back to
+    f51348dc8bdd5071b7baaf3f0e4d2e0496618f08 (2008) and the original version of
+    scrub.
+
+    This *might* fix #4179.  It is not a leak per se, but I observed 1GB
+    scrub messages going over the write.  Maybe the allocations are causing
+    fragmentation, or the sub_op queues are growing.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0b036ecddbfd82e651666326d6f16b3c000ade18)
+
+commit 5047a4ff16f556ec25b6624fe8f1a07e3ec5f864
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:53:00 2013 -0700
+
+    rgw: handle deep uri resources
+
+    In case of deep uri resources (ones created beyond a single level
+    of hierarchy, e.g. auth/v1.0) we want to create a new empty
+    handlers for the path if no handlers exists. E.g., for
+    auth/v1.0 we need to have a handler for 'auth', otherwise
+    the default S3 handler will be used, which we don't want.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ad3934e335399f7844e45fcfd17f7802800d2cb3)
+
+commit 3ea45533c18174cb49af52024dae27533971fa01
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:47:21 2013 -0700
+
+    rgw: fix get_resource_mgr() to correctly identify resource
+
+    Fixes: #5262
+    The original test was not comparing the correct string, ended up
+    with the effect of just checking the substring of the uri to match
+    the resource.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8d55b87f95d59dbfcfd0799c4601ca37ebb025f5)
+
+commit 4ee638cc3b1d74e678a3f78b3a57baf57a5d407c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Apr 15 16:33:48 2013 -0700
+
+    PG: don't write out pg map epoch every handle_activate_map
+
+    We don't actually need to write out the pg map epoch on every
+    activate_map as long as:
+    a) the osd does not trim past the oldest pg map persisted
+    b) the pg does update the persisted map epoch from time
+    to time.
+
+    To that end, we now keep a reference to the last map persisted.
+    The OSD already does not trim past the oldest live OSDMapRef.
+    Second, handle_activate_map will trim if the difference between
+    the current map and the last_persisted_map is large enough.
+
+    Fixes: #4731
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 8c6a912ae46c4d3aeb7c1000d221f67e158ec5c8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 12:58:11 2013 -0700
+
+    rgw: only append prefetched data if reading from head
+
+    Fixes: #5209
+    Backport: bobtail, cuttlefish
+    If the head object wrongfully contains data, but according to the
+    manifest we don't read from the head, we shouldn't copy the prefetched
+    data. Also fix the length calculation for that data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c5fc52ae0fc851444226abd54a202af227d7cf17)
+
+commit f42e84ee3148dcc59a05c8a12ce39996eb854e26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 09:34:21 2013 -0700
+
+    rgw: don't copy object idtag when copying object
+
+    Fixes: #5204
+    When copying object we ended up also copying the original
+    object idtag which overrode the newly generated one. When
+    refcount put is called with the wrong idtag the count
+    does't go down.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b1312f94edc016e604f1d05ccfe2c788677f51d1)
+
+commit d86b9a852b57553448709c2b978ac3a8637cd63f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 16:50:04 2013 -0700
+
+    osd: initialize new_state field when we use it
+
+    If we use operator[] on a new int field its value is undefined; avoid
+    reading it or using |= et al until we initialize it.
+
+    Fixes: #4967
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50ac8917f175d1b107c18ecb025af1a7b103d634)
+
+commit fca54bfbb35f0c200fac52963cd2d1780467f59a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 28 11:10:05 2013 -0700
+
+    HashIndex: sync top directory during start_split,merge,col_split
+
+    Otherwise, the links might be ordered after the in progress
+    operation tag write.  We need the in progress operation tag to
+    correctly recover from an interrupted merge, split, or col_split.
+
+    Fixes: #5180
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bca9c38ef5187c7a97916970a7fa73b342755ac)
+
+commit ac6c77830ad9182c7345f10943ea4f537c1629b0
+Merge: 3984b98 0b50d07
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 19:29:06 2013 -0700
+
+    Merge remote-tracking branch 'upstream/wip_scrub_tphandle_bobtail' into bobtail
+
+    Fixes: #5159
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 0b50d073fab11f64682e20fec7bb71d4e931bd3c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 17:40:44 2013 -0700
+
+    PG: ping tphandle during omap loop as well
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 5979a26d521918d74fc3b62b75c9fb6afe109933
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:24:39 2013 -0700
+
+    PG: reset timeout in _scan_list for each object, read chunk
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit bde6b58baf5e78550980ee5076bbc2d72a52c0fc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:23:05 2013 -0700
+
+    OSD,PG: pass tphandle down to _scan_list
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 3984b98804bf18711a7a4fc3940143ab599ad5b5
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 22 21:34:52 2013 -0700
+
+    rgw: iterate usage entries from correct entry
+
+    Fixes: #5152
+    When iterating through usage entries, and when user id was
+    provided, we started at the user's first entry and not from
+    the entry indexed by the request start time.
+    This commit fixes the issue.
+
+    Backport: bobtail
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8b3a04dec8be13559716667d4b16cde9e9543feb)
+
+commit 3d7f8f840fb9fd975089af32c85093eeb1eac338
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 12:45:27 2013 -0700
+
+    mon: be a bit more verbose about osd mark down events
+
+    Put these in the cluster log; they are interesting.
+
+    This is a backport of 87767fb1fb9a52d11b11f0b641cebbd9998f089e.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f21a7f7bf3db61bd533e777297bff1346112a0db
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 3 12:57:00 2013 -0700
+
+    rgw: protect ops log socket formatter
+
+    Fixes: #4905
+    Ops log (through the unix domain socket) uses a formatter, which wasn't
+    protected.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d48f1edb07a4d8727ac956f70e663c1b4e33e1dd)
+
+commit 49c04c623466df45003ed2a18445c354c424a078
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:28:40 2013 -0700
+
+    librbd: make image creation defaults configurable
+
+    Programs using older versions of the image creation functions can't
+    set newer parameters like image format and fancier striping.
+
+    Setting these options lets them use all the new functionality without
+    being patched and recompiled to use e.g. rbd_create3().
+    This is particularly useful for things like qemu-img, which does not
+    know how to create format 2 images yet.
+
+    Refs: #5067
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit aacc9adc4e9ca90bbe73ac153cc754a3a5b2c0a1)
+
+commit 4d7058fe2254f335969f05bef649b1a27d470aa4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:21:24 2013 -0700
+
+    rbd.py: fix stripe_unit() and stripe_count()
+
+    These matched older versions of the functions, but would segfault
+    using the current versions.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 53ee6f965e8f06c7256848210ad3c4f89d0cb5a0)
+
+commit 82a16c32a37dc46e3019cedc2a5407ae34f806e2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:19:46 2013 -0700
+
+    cls_rbd: make sure stripe_unit is not larger than object size
+
+    Test a few other cases too.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 810306a2a76eec1c232fd28ec9c351e827fa3031)
+
+commit e0de00897468a434e94790a86fc812b77a59614c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 10 22:14:05 2013 -0700
+
+    mon: fix validatation of mds ids in mon commands
+
+    Fixes: #4996
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c305d63043762027323052b4bb3ae3063665c6f)
+
+    Conflicts:
+
+	src/mon/MDSMonitor.cc
diff --git a/doc/changelog/v0.61.1.txt b/doc/changelog/v0.61.1.txt
new file mode 100644
index 000000000..4dbbe9725
--- /dev/null
+++ b/doc/changelog/v0.61.1.txt
@@ -0,0 +1,139 @@
+commit 56c4847ba82a92023700e2d4920b59cdaf23428d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed May 8 17:23:47 2013 -0700
+
+    v0.61.1
+
+commit 3b94f03ec58abe3d7a6d0359ff9b4d75826f3777
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 2 14:13:07 2013 -0700
+
+    mon: dump MonitorDBStore transactions to file
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 797089ef082b99910eebfd9454c03d1f027c93bb)
+
+commit 9143d6d0d0ebaaee81622587dc21a3d562b05c9c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 14:21:28 2013 -0700
+
+    osd: optionally enable leveldb logging
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0b4c5c1a3349670d11cc3c4fb3c4b3c1a80b2502)
+
+commit 8f456e89ec38f37ae4b444f5cabccd9fc0e415b8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 14:13:50 2013 -0700
+
+    mon: allow leveldb logging
+    
+    'mon leveldb log = filename'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c1d5f815546b731e10bfcb81cbcd48b7d432e9c4)
+
+commit 306ebc6a9164feab084123f91a0fe928125f4b74
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed May 8 16:33:05 2013 -0700
+
+    debian/control:  squeeze requres cryptsetup package
+    
+    Squeeze requires the cryptsetup package which has been renamed
+    cryptsetup-bin in later versions.  Allow either package to
+    satisfy the dependency.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit 83bbae415de16f708ca1cb24861ddbb0bd514a7f)
+
+commit e0c0a5c1a711263d234b70b454aca07c23e98243
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 8 12:18:49 2013 -0700
+
+    osd: don't assert if get_omap_iterator() returns NULL
+    
+    Fixes: #4949
+    This can happen if the object does not exist and it's
+    a write operation. Just return -ENOENT.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 36ec6f9bce63641f4fc2e4ab04d03d3ec1638ea0)
+
+commit 82b9299512537b8d921caa044107472adacbdabe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 14:54:33 2013 -0700
+
+    ceph-create-keys: gracefully handle no data from admin socket
+    
+    Old ceph-mon (prior to 393c9372f82ef37fc6497dd46fc453507a463d42) would
+    return an empty string and success if the command was not registered yet.
+    Gracefully handle that case by retrying.
+    
+    If we still fail to parse, exit entirely with EINVAL.
+    
+    Fixes: #4952
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@intank.com>
+    (cherry picked from commit e2528ae42c455c522154c9f68b5032a3362fca8e)
+
+commit ee3da880fbc0488467c30b1d0b635e3c50e23cc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 14:35:54 2013 -0700
+
+    init-ceph: fix osd_data location when checking df utilization
+    
+    Do not assume default osd data location.
+    
+    Fixes: #4951
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowelll <gary.lowell@inktank.com>
+    (cherry picked from commit f2a54cc9c98a9f31aef049c74ea932b2d9000d3c)
+
+commit 4848fac24224879bcdc4fcf202d6ab689d8d990f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:41:22 2013 -0700
+
+    OSD: handle stray snap collections from upgrade bug
+    
+    Previously, we failed to clear snap_collections, which causes split to
+    spawn a bunch of snap collections.  In load_pgs, we now clear any such
+    snap collections and then snap_collections field on the PG itself.
+    
+    Related: #4927
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8e89db89cb36a217fd97cbc1f24fd643b62400dc)
+
+commit dc6b9e6b2e8c1b41e982430795ff41a65a5813dc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:35:57 2013 -0700
+
+    PG: clear snap_collections on upgrade
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 252d71a81ef4536830a74897c84a7015ae6ec9fe)
+
+commit b514941ba5f49b6465082e4906b7f4022631bfb3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:34:57 2013 -0700
+
+    OSD: snap collections can be ignored on split
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 438d9aa152e546b2008ec355b481df71aa1c51a5)
+
+commit 30ffca77df006a244044604074779af538721f14
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 11:05:29 2013 -0700
+
+    ceph: return error code when failing to get result from admin socket
+    
+    Make sure we return a non-zero result code when we fail to read something
+    from the admin socket.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 393c9372f82ef37fc6497dd46fc453507a463d42)
diff --git a/doc/changelog/v0.61.2.txt b/doc/changelog/v0.61.2.txt
new file mode 100644
index 000000000..b57c7e634
--- /dev/null
+++ b/doc/changelog/v0.61.2.txt
@@ -0,0 +1,27 @@
+commit fea782543a844bb277ae94d3391788b76c5bee60
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon May 13 11:58:35 2013 -0700
+
+    v0.61.2
+
+commit 8464c06412b72673ca9d01d35e8f68451e1982b3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon May 13 15:36:59 2013 +0100
+
+    mon: Monitor: tolerate GV duplicates during conversion
+    
+    Fixes: #4974
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit ba05b16ee2b6e25141f2ab88265a1cf92dcd493c)
+
+commit 703bc2fd8aec512fb2b58d5ef263e37647c1f047
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri May 10 20:09:34 2013 -0700
+
+    config_opts: default mon_debug_dump_transactions to 'false'
+    
+    otherwise, it chews mon log space at an alarming rate.
+    
+    Fixes: #5024
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
diff --git a/doc/changelog/v0.61.3.txt b/doc/changelog/v0.61.3.txt
new file mode 100644
index 000000000..c190d40ed
--- /dev/null
+++ b/doc/changelog/v0.61.3.txt
@@ -0,0 +1,831 @@
+commit 92b1e398576d55df8e5888dd1a9545ed3fd99532
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jun 5 11:10:05 2013 -0700
+
+    v0.61.3
+
+commit 7d549cb82ab8ebcf1cc104fc557d601b486c7635
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 4 10:42:13 2013 -0700
+
+    os/LevelDBStore: only remove logger if non-null
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce67c58db7d3e259ef5a8222ef2ebb1febbf7362)
+    Fixes: #5255
+
+commit 8544ea751884617616addc17b4467b9a86bd9d8a
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jun 3 15:57:23 2013 -0700
+
+    test_librbd: use correct type for varargs snap test
+
+    uint64_t is passed in, but int was extracted. This fails on 32-bit builds.
+
+    Fixes: #5220
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 17029b270dee386e12e5f42c2494a5feffd49b08)
+
+commit b226e117b5a72c3b04b74aec50a9198601f3730b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 18:07:34 2013 -0700
+
+    os/LevelDBStore: fix merge loop
+
+    We were double-incrementing p, both in the for statement and in the
+    body.  While we are here, drop the unnecessary else's.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eb6d5fcf994d2a25304827d7384eee58f40939af)
+
+commit d017afb5b075a1958cb19213dd95e41d54065e45
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 17:27:10 2013 -0700
+
+    msgr: add get_messenger() to Connection
+
+    This was part of commit 27381c0c6259ac89f5f9c592b4bfb585937a1cfc.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ffb87918fa7b829a5199eec08804dc540a819bf2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 17:09:19 2013 -0700
+
+    mon: start lease timer from peon_init()
+
+    In the scenario:
+
+     - leader wins, peons lose
+     - leader sees it is too far behind on paxos and bootstraps
+     - leader tries to sync with someone, waits for a quorum of the others
+     - peons sit around forever waiting
+
+    The problem is that they never time out because paxos never issues a lease,
+    which is the normal timeout that lets them detect a leader failure.
+
+    Avoid this by starting the lease timeout as soon as we lose the election.
+    The timeout callback just does a bootstrap and does not rely on any other
+    state.
+
+    I see one possible danger here: there may be some "normal" cases where the
+    leader takes a long time to issue its first lease that we currently
+    tolerate, but won't with this new check in place.  I hope that raising
+    the lease interval/timeout or reducing the allowed paxos drift will make
+    that a non-issue.  If it is problematic, we will need a separate explicit
+    "i am alive" from the leader while it is getting ready to issue the lease
+    to prevent a live-lock.
+
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit f1ccb2d808453ad7ef619c2faa41a8f6e0077bd9)
+
+commit 38f8d850d35500e3d8751cd14c5cdaaff682c7d7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 22:52:21 2013 -0700
+
+    mon: discard messages from disconnected clients
+
+    If the client is not connected, discard the message.  They will
+    reconnect and resend anyway, so there is no point in processing it
+    twice (now and later).
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fb3cd0c2a8f27a1c8d601a478fd896cc0b609011)
+
+commit 36d948981685114d2fe807f480c19aade7497194
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 08:13:21 2013 -0700
+
+    msgr: add Messenger reference to Connection
+
+    This allows us to get the messenger associated with a connection.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 92a558bf0e5fee6d5250e1085427bff22fe4bbe4)
+
+commit 61135964419ecf5165366724d064b623b517fb4e
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 15:59:49 2013 -0700
+
+    mon/Paxos: adjust trimming defaults up; rename options
+
+    - trim more at a time (by an order of magnitude)
+    - rename fields to paxos_trim_{min,max}; only trim when there are min items
+      that are trimmable, and trim at most max items at a time.
+    - adjust the paxos_service_trim_{min,max} values up by a factor of 2.
+
+    Since we are compacting every time we trim, adjusting these up mean less
+    frequent compactions and less overall work for the monitor.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 6b8e74f0646a7e0d31db24eb29f3663fafed4ecc)
+
+commit 2dc402815f71204cfe592cfb3d6758486d84166d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 16:42:24 2013 -0700
+
+    common/Preforker: fix warnings
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a284c9ece85f11d020d492120be66a9f4c997416)
+
+commit 48ee9283de288c101d3387cc48df6eb8ea889fb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 15:53:35 2013 -0700
+
+    fix test users of LevelDBStore
+
+    Need to pass in cct.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 446e0770c77de5d72858dcf7a95c5b19f642cf98)
+
+commit 3372235d307172b404a48e4c4d24702a96116d00
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 21:43:50 2013 -0700
+
+    mon: destroy MonitorDBStore before g_ceph_context
+
+    Put it on the heap so that we can destroy it before the g_ceph_context
+    cct that it references.  This fixes a crash like
+
+    *** Caught signal (Segmentation fault) **
+    in thread 4034a80
+    ceph version 0.63-204-gcf9aa7a (cf9aa7a0037e56eada8b3c1bb59d59d0bfe7bba5)
+    1: ceph-mon() [0x59932a]
+    2: (()+0xfcb0) [0x4e41cb0]
+    3: (Mutex::Lock(bool)+0x1b) [0x6235bb]
+    4: (PerfCountersCollection::remove(PerfCounters*)+0x27) [0x6a0877]
+    5: (LevelDBStore::~LevelDBStore()+0x1b) [0x582b2b]
+    6: (LevelDBStore::~LevelDBStore()+0x9) [0x582da9]
+    7: (main()+0x1386) [0x48db16]
+    8: (__libc_start_main()+0xed) [0x658076d]
+    9: ceph-mon() [0x4909ad]
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit df2d06db6f3f7e858bdadcc8cd2b0ade432df413)
+
+commit d284eaf9ce7d2022ba62562236d5fa41c26c1eb3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 11:07:06 2013 -0700
+
+    mon: fix leak of health_monitor and config_key_service
+
+    Switch to using regular pointers here.  The lifecycle of these services is
+    very simple such that refcounting is overkill.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c888d1d3f1b77e62d1a8796992e918d12a009b9d)
+
+commit cc88cdfe8de02da675f9051e95b70da11f7fbe9e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 17:54:17 2013 -0700
+
+    mon: return instead of exit(3) via preforker
+
+    This lets us run all the locally-scoped dtors so that leak checking will
+    work.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3c5706163b72245768958155d767abf561e6d96d)
+
+commit 85d629a5f8e1deef9a562d9a7b371707d90c5ba1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:57:42 2013 -0700
+
+    os/LevelDBStore: add perfcounters
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7802292e0a49be607d7ba139b44d5ea1f98e07e6)
+
+commit 0109fa8ae70671c2d8ca19bcc95662d5f41cac66
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:36:41 2013 -0700
+
+    mon: make compaction bounds overlap
+
+    When we trim items N to M, compact over range (N-1) to M so that the
+    items in the queue will share bounds and get merged.  There is no harm in
+    compacting over a larger range here when the lower bound is a key that
+    doesn't exist anyway.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a47ca583980523ee0108774b466718b303bd3f46)
+
+commit 83b1edac07dd74b91ba2cdfe8b63236d7930c9b1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:26:42 2013 -0700
+
+    os/LevelDBStore: merge adjacent ranges in compactionqueue
+
+    If we get behind and multiple adjacent ranges end up in the queue, merge
+    them so that we fire off compaction on larger ranges.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f628dd0e4a5ace079568773edfab29d9f764d4f0)
+
+commit d14665e550d9b2dfc47684b73427042b0744127f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:40:32 2013 -0700
+
+    mon: compact trimmed range, not entire prefix
+
+    This will reduce the work that leveldb is asked to do by only triggering
+    compaction of the keys that were just trimmed.
+
+    We ma want to further reduce the work by compacting less frequently, but
+    this is at least a step in that direction.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6da4b20ca53fc8161485c8a99a6b333e23ace30e)
+
+commit dcd9b793fb0b05976b55be029315114d6f1df0e5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:35:44 2013 -0700
+
+    mon/MonitorDBStore: allow compaction of ranges
+
+    Allow a transaction to describe the compaction of a range of keys.  Do this
+    in a backward compatible say, such that older code will interpret the
+    compaction of a prefix + range as compaction of the entire prefix.  This
+    allows us to avoid introducing any new feature bits.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ab09f1e5c1305a64482ebbb5a6156a0bb12a63a4)
+
+    Conflicts:
+
+	src/mon/MonitorDBStore.h
+
+commit 6543da740a12c6ad085b807c9038d5b7b5aeaba6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:34:13 2013 -0700
+
+    os/LevelDBStore: allow compaction of key ranges
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e20c9a3f79ccfeb816ed634ca25de29fc5975ea8)
+
+commit 87dcba2dd12a673fdc63ad64fb23e6e9f841d74f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 28 16:35:55 2013 -0700
+
+    os/LevelDBStore: do compact_prefix() work asynchronously
+
+    We generally do not want to block while compacting a range of leveldb.
+    Push the blocking+waiting off to a separate thread.  (leveldb will do what
+    it can to avoid blocking internally; no reason for us to wait explicitly.)
+
+    This addresses part of #5176.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4af917d4478ec07734a69447420280880d775fa2)
+
+commit bac5720b2a583e799c6961c733c4a9132a002440
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat May 11 17:36:13 2013 -0700
+
+    qa: rsync test: exclude /usr/local
+
+    Some plana have non-world-readable crap in /usr/local/samba.  Avoid
+    /usr/local entirely for that and any similar landmines.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82211f2197241c4f3d3135fd5d7f0aa776eaeeb6)
+
+commit 9f1afe1a8e7906bdc54158a4813f011933f6a78f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 21:16:54 2013 -0700
+
+    mon: fix uninitialized fields in MMonHealth
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d7e2ab1451e284cd4273cca47eec75e1d323f113)
+
+commit d5b67d49c87b4fe6a90cdd0bf4fac4f9d9377e81
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 31 13:44:39 2013 -0700
+
+    PGLog: only add entry to caller_ops in add() if reqid_is_indexed()
+
+    Fixes: #5216
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit a5f6d8967fbbe87f34b23cfcceea84317b647f62
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Apr 15 16:33:48 2013 -0700
+
+    PG: don't write out pg map epoch every handle_activate_map
+
+    We don't actually need to write out the pg map epoch on every
+    activate_map as long as:
+    a) the osd does not trim past the oldest pg map persisted
+    b) the pg does update the persisted map epoch from time
+    to time.
+
+    To that end, we now keep a reference to the last map persisted.
+    The OSD already does not trim past the oldest live OSDMapRef.
+    Second, handle_activate_map will trim if the difference between
+    the current map and the last_persisted_map is large enough.
+
+    Fixes: #4731
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 2c5a9f0e178843e7ed514708bab137def840ab89)
+
+    Conflicts:
+
+	src/common/config_opts.h
+	src/osd/PG.cc
+	- last_persisted_osdmap_ref gets set in the non-static
+	  PG::write_info
+
+commit 9aaff1492a00e137f84af9b03e3a4608cea4b520
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri May 31 12:33:11 2013 -0700
+
+    upstart: handle upper case in cluster name and id
+
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 851619ab6645967e5d7659d9b0eea63d5c402b15)
+
+commit f87a19d34f9a03493eaca654dd176992676c5812
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 21 15:22:56 2013 -0700
+
+    OSDMonitor: skip new pools in update_pools_status() and get_pools_health()
+
+    New pools won't be full.  mon->pgmon()->pg_map.pg_pool_sum[poolid] will
+    implicitly create an entry for poolid causing register_new_pgs() to assume that
+    the newly created pgs in the new pool are in fact a result of a split
+    preventing MOSDPGCreate messages from being sent out.
+
+    Fixes: #4813
+    Backport: cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0289c445be0269157fa46bbf187c92639a13db46)
+
+commit 34733bdc3b9da13195dfac8a25f98c6de64070d7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 12:58:11 2013 -0700
+
+    rgw: only append prefetched data if reading from head
+
+    Fixes: #5209
+    Backport: bobtail, cuttlefish
+    If the head object wrongfully contains data, but according to the
+    manifest we don't read from the head, we shouldn't copy the prefetched
+    data. Also fix the length calculation for that data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c5fc52ae0fc851444226abd54a202af227d7cf17)
+
+commit 3d91301e5d788f9f9f52ba4067006f39a89e1531
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 09:34:21 2013 -0700
+
+    rgw: don't copy object idtag when copying object
+
+    Fixes: #5204
+    When copying object we ended up also copying the original
+    object idtag which overrode the newly generated one. When
+    refcount put is called with the wrong idtag the count
+    does't go down.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b1312f94edc016e604f1d05ccfe2c788677f51d1)
+
+commit 131dca7d10d5b0b7dca5c5cbe9b0b54938805015
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 08:53:22 2013 -0700
+
+    debian: sync up postinst and prerm with latest
+
+    - do not use invoke-rc.d for upstart
+    - do not stop daemons on upgrade
+    - misc other cleanups
+
+    This corresponds to the state of master as of cf9aa7a.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 8b7ca687ded06fe0b67d98e81fa1dabbed440853
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu May 30 18:17:28 2013 +0100
+
+    mon: Monitor: backup monmap using all ceph features instead of quorum's
+
+    When a monitor is freshly created and for some reason its initial sync is
+    aborted, it will end up with an incorrect backup monmap.  This monmap is
+    incorrect in the sense that it will not contain the monitor's names as
+    it will expect on the next run.
+
+    This results from us being using the quorum features to encode the monmap
+    when backing it up, instead of CEPH_FEATURES_ALL.
+
+    Fixes: #5203
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 626de387e617db457d6d431c16327c275b0e8a34)
+
+commit 02ef6e918e4fe0368b02fbc700a4d921ae298dc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 09:49:11 2013 -0700
+
+    osd: do not assume head obc object exists when getting snapdir
+
+    For a list-snaps operation on the snapdir, do not assume that the obc for the
+    head means the object exists.  This fixes a race between a head deletion and
+    a list-snaps that wrongly returns ENOENT, triggered by the DiffItersateStress
+    test when thrashing OSDs.
+
+    Fixes: #5183
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 29e4e7e316fe3f3028e6930bb5987cfe3a5e59ab)
+
+commit 85ad65e294f2b3d4bd1cfef6ae613e31d1cea635
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 16:50:04 2013 -0700
+
+    osd: initialize new_state field when we use it
+
+    If we use operator[] on a new int field its value is undefined; avoid
+    reading it or using |= et al until we initialize it.
+
+    Fixes: #4967
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50ac8917f175d1b107c18ecb025af1a7b103d634)
+
+commit 65236a4a2541799a0d946df7f3048d4404678f64
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 28 11:10:05 2013 -0700
+
+    HashIndex: sync top directory during start_split,merge,col_split
+
+    Otherwise, the links might be ordered after the in progress
+    operation tag write.  We need the in progress operation tag to
+    correctly recover from an interrupted merge, split, or col_split.
+
+    Fixes: #5180
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bca9c38ef5187c7a97916970a7fa73b342755ac)
+
+commit 0e127dc4ef16d19d56a3603ca90fa6b076d905b0
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed May 22 13:59:08 2013 +0100
+
+    mon: Paxos: get rid of the 'prepare_bootstrap()' mechanism
+
+    We don't need it after all.  If we are in the middle of some proposal,
+    then we guarantee that said proposal is likely to be retried.  If we
+    haven't yet proposed, then it's forever more likely that a client will
+    eventually retry the message that triggered this proposal.
+
+    Basically, this mechanism attempted at fixing a non-problem, and was in
+    fact triggering some unforeseen issues that would have required increasing
+    the code complexity for no good reason.
+
+    Fixes: #5102
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e15d29094503f279d444eda246fc45c09f5535c9)
+
+commit f29206bdd950d1e916a1422b4826caaf4a8cef3a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed May 22 13:51:13 2013 +0100
+
+    mon: Paxos: finish queued proposals instead of clearing the list
+
+    By finishing these Contexts, we make sure the Contexts they enclose (to be
+    called once the proposal goes through) will behave as their were initially
+    planned:  for instance, a C_Command() may retry the command if a -EAGAIN
+    is passed to 'finish_contexts', while a C_Trimmed() will simply set
+    'going_to_trim' to false.
+
+    This aims at fixing at least a bug in which Paxos will stop trimming if an
+    election is triggered while a trim is queued but not yet finished.  Such
+    happens because it is the C_Trimmed() context that is responsible for
+    resetting 'going_to_trim' back to false.  By clearing all the contexts on
+    the proposal list instead of finishing them, we stay forever unable to
+    trim Paxos again as 'going_to_trim' will stay True till the end of time as
+    we know it.
+
+    Fixes: #4895
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 586e8c2075f721456fbd40f738dab8ccfa657aa8)
+
+commit b73d7c6b7e83fe8ce29e7150f25a4cca7647cccd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 17 18:23:36 2013 +0100
+
+    mon: Paxos: finish_proposal() when we're finished recovering
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 2ff23fe784245f3b86bc98e0434b21a5318e0a7b)
+
+commit 79eb61c4eadd9d94b3b8087d85b7210f4ab71d54
+Merge: a460e53 8682248
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 20:09:29 2013 -0700
+
+    Merge branch 'wip_scrub_tphandle' into cuttlefish
+
+    Fixes: #5159
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 86822485e518d61d7b2c02a6ff25eb2c4b4bc307
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 17:40:44 2013 -0700
+
+    PG: ping tphandle during omap loop as well
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d62716dd4ceb29032759bf84e864d214fe38a17c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:24:39 2013 -0700
+
+    PG: reset timeout in _scan_list for each object, read chunk
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit b8a25e08a638c31b9cfc2c1bf6d9bad40e921a9f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:23:05 2013 -0700
+
+    OSD,PG: pass tphandle down to _scan_list
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit a460e53ecac03e9c8f54c402a790e6d8cf75b38c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 22 21:34:52 2013 -0700
+
+    rgw: iterate usage entries from correct entry
+
+    Fixes: #5152
+    When iterating through usage entries, and when user id was
+    provided, we started at the user's first entry and not from
+    the entry indexed by the request start time.
+    This commit fixes the issue.
+
+    Backport: bobtail
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8b3a04dec8be13559716667d4b16cde9e9543feb)
+
+commit 48e1b11bcbdbdf8661cb560c8563f91738034287
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 20:37:05 2013 -0700
+
+    sysvinit: fix enumeration of local daemons when specifying type only
+
+    - prepend $local to the $allconf list at the top
+    - remove $local special case for all case
+    - fix the type prefix checks to explicitly check for prefixes
+
+    Fugly bash, but works!
+
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit c80c6a032c8112eab4f80a01ea18e1fa2c7aa6ed)
+
+commit 6e5f0db75a2bf591bd1fc0a5c2a93051f83ebc5d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 09:47:29 2013 -0700
+
+    sysvinit: fix osd weight calculation on remote hosts
+
+    We need to do df on the remote host, not locally.
+
+    Simlarly, the ceph command uses the osd key, which exists remotely; run it there.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d81d0ea5c442699570bd93a90bea0d97a288a1e9)
+
+commit 674e75bc0783cd9befe9c20e36fbc2cfdac62e5a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 09:47:03 2013 -0700
+
+    sysvinit: use known hostname $host instead of (incorrectly) recalculating
+
+    We would need to do hostname -s on the remote node, not the local one.
+    But we already have $host; use it!
+
+    Reported-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit caa15a34cb5d918c0c8b052cd012ec8a12fca150)
+
+commit f4cd61d339419738355a26d7f4fce04eac9dedee
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 20 12:41:30 2013 -0700
+
+    mon: be a bit more verbose about osd mark down events
+
+    Put these in the cluster log; they are interesting.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 87767fb1fb9a52d11b11f0b641cebbd9998f089e)
+
+commit e04b1894ed7f3a6e95007e58dae5b35357e5c507
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon May 13 14:23:00 2013 -0700
+
+    PG: subset_last_update must be at least log.tail
+
+    Fixes: 5020
+    Backport: bobtail, cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 72bf5f4813c273210b5ced7f7793bc1bf813690c)
+
+commit 3a02e00d1d3b904b312db283faedf4dff37b0f6f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 14 16:35:48 2013 -0700
+
+    FileJournal: adjust write_pos prior to unlocking write_lock
+
+    In committed_thru, we use write_pos to reset the header.start value in cases
+    where seq is past the end of our journalq.  It is therefore important that the
+    journalq be updated atomically with write_pos (that is, under the write_lock).
+
+    The call to align_bl() is moved into do_write in order to ensure that write_pos
+    is adjusted correctly prior to write_bl().
+
+    Also, we adjust pos at the end of write_bl() such that pos \in [get_top(),
+    header.max_size) after write_bl().
+
+    Fixes: #5020
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eaf3abf3f9a7b13b81736aa558c9084a8f07fdbe)
+
+commit 8a030eb0e1b61121f7b1e2b7a551bb213d1c428b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 21 14:36:11 2013 -0700
+
+    mon: implement --extract-monmap <filename>
+
+    This will make for a simpler process for
+      http://docs.ceph.com/docs/master/rados/operations/add-or-rm-mons/#removing-monitors-from-an-unhealthy-cluster
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c0268e27497a4d8228ef54da9d4ca12f3ac1f1bf)
+
+commit 4ad13c945fd29a2d183f7ea6c6ac8a51d24dabe1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:28:40 2013 -0700
+
+    librbd: make image creation defaults configurable
+
+    Programs using older versions of the image creation functions can't
+    set newer parameters like image format and fancier striping.
+
+    Setting these options lets them use all the new functionality without
+    being patched and recompiled to use e.g. rbd_create3().
+    This is particularly useful for things like qemu-img, which does not
+    know how to create format 2 images yet.
+
+    Refs: #5067
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit aacc9adc4e9ca90bbe73ac153cc754a3a5b2c0a1)
+
+commit 684444f88f2a7cf28f2e685c18f0771730a1d48f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:21:24 2013 -0700
+
+    rbd.py: fix stripe_unit() and stripe_count()
+
+    These matched older versions of the functions, but would segfault
+    using the current versions.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 53ee6f965e8f06c7256848210ad3c4f89d0cb5a0)
+
+commit 9c7faf957fffb2721ccb915b68ca90ffb0d04a9f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:19:46 2013 -0700
+
+    cls_rbd: make sure stripe_unit is not larger than object size
+
+    Test a few other cases too.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 810306a2a76eec1c232fd28ec9c351e827fa3031)
+
+commit 4071d7a79585ee2768d0a63819d99405a083369f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 3 12:57:00 2013 -0700
+
+    rgw: protect ops log socket formatter
+
+    Fixes: #4905
+    Ops log (through the unix domain socket) uses a formatter, which wasn't
+    protected.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d48f1edb07a4d8727ac956f70e663c1b4e33e1dd)
+
+commit 57a6a54db4020debd826a493cccca4695934f693
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 15 23:02:10 2013 -0700
+
+    Makefle: force char to be signed
+
+    On an armv7l build, we see errors like
+
+     warning: rgw/rgw_common.cc:626:16: comparison is always false due to limited range of data type [-Wtype-limits]
+
+    from code
+
+          char c1 = hex_to_num(*src++);
+    ...
+          if (c1 < 0)
+
+    Force char to be signed (regardless of any weird architecture's default)
+    to avoid risk of this leading to misbehavior.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 769a16d6674122f3b537f03e17514ad974bf2a2f)
+
+commit 85fb422a084785176af3b694882964841e02195d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 20 13:34:27 2013 -0700
+
+    debian: stop sysvinit on ceph.prerm
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2f193fb931ed09d921e6fa5a985ab87aa4874589)
+
+commit eac3cd2d8334db7c70cbba433610136ff76e447f
+Author: Mike Kelly <pioto@pioto.org>
+Date:   Thu May 16 12:29:50 2013 -0400
+
+    ceph df: fix si units for 'global' stats
+
+    si_t expects bytes, but it was being given kilobytes.
+
+    Signed-off-by: Mike Kelly <pioto@pioto.org>
+    (cherry picked from commit 0c2b738d8d07994fee4c73dd076ac9364a64bdb2)
+
+commit 7bc7c9d4bc6f5bbadbb5e6f9844a0e1f66373c90
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 18:40:29 2013 -0700
+
+    udev: install disk/by-partuuid rules
+
+    Wheezy's udev (175-7.2) has broken rules for the /dev/disk/by-partuuid/
+    symlinks that ceph-disk relies on.  Install parallel rules that work.  On
+    new udev, this is harmless; old older udev, this will make life better.
+
+    Fixes: #4865
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d8d7113c35b59902902d487738888567e3a6b933)
+
+commit c2acecbdab93177227a80a114ecc4ce71dbdbe49
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 13:17:45 2013 -0700
+
+    debian: make radosgw require matching version of librados2
+
+    ...indirectly via ceph-common.  We get bad behavior when they diverge, I
+    think because of libcommon.la being linked both statically and dynamically.
+
+    Fixes: #4997
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 604c83ff18f9a40c4f44bc8483ef22ff41efc8ad)
+
+commit 4c0d3eb72c045c39e8b38fd5abf9ddfbb2219f9c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 10 22:14:05 2013 -0700
+
+    mon: fix validatation of mds ids in mon commands
+
+    Fixes: #4996
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c305d63043762027323052b4bb3ae3063665c6f)
diff --git a/doc/changelog/v0.61.4.txt b/doc/changelog/v0.61.4.txt
new file mode 100644
index 000000000..0964274f0
--- /dev/null
+++ b/doc/changelog/v0.61.4.txt
@@ -0,0 +1,823 @@
+commit 1669132fcfc27d0c0b5e5bb93ade59d147e23404
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jun 19 13:51:38 2013 -0700
+
+    v0.61.4
+
+commit b76540f6e2db7a08dee86f84358d56c4ea0b3293
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 14:29:37 2013 -0700
+
+    messages/MOSDMarkMeDown: fix uninit field
+    
+    Fixes valgrind warning:
+    ==14803== Use of uninitialised value of size 8
+    ==14803==    at 0x12E7614: sctp_crc32c_sb8_64_bit (sctp_crc32.c:567)
+    ==14803==    by 0x12E76F8: update_crc32 (sctp_crc32.c:609)
+    ==14803==    by 0x12E7720: ceph_crc32c_le (sctp_crc32.c:733)
+    ==14803==    by 0x105085F: ceph::buffer::list::crc32c(unsigned int) (buffer.h:427)
+    ==14803==    by 0x115D7B2: Message::calc_front_crc() (Message.h:441)
+    ==14803==    by 0x1159BB0: Message::encode(unsigned long, bool) (Message.cc:170)
+    ==14803==    by 0x1323934: Pipe::writer() (Pipe.cc:1524)
+    ==14803==    by 0x13293D9: Pipe::Writer::entry() (Pipe.h:59)
+    ==14803==    by 0x120A398: Thread::_entry_func(void*) (Thread.cc:41)
+    ==14803==    by 0x503BE99: start_thread (pthread_create.c:308)
+    ==14803==    by 0x6C6E4BC: clone (clone.S:112)
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eb91f41042fa31df2bef9140affa6eac726f6187)
+
+commit 2434ecca6f8e1f2af62c74cb1b7df277f4a076f7
+Merge: f8973bd 3f32ded
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 19 10:56:51 2013 -0700
+
+    Merge remote-tracking branch 'gh/wip-4976-cuttlefish' into cuttlefish
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit f8973bda801dbb2412c4cd83869c0203930f38e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:32:15 2013 -0700
+
+    common/Preforker: fix warning
+    
+    common/Preforker.h: In member function ‘int Preforker::signal_exit(int)’:
+    warning: common/Preforker.h:82:45: ignoring return value of ‘ssize_t safe_write(int, const void*, size_t)’, declared with attribute warn_unused_result [-Wunused-result]
+    
+    This is harder than it should be to fix.  :(
+      http://stackoverflow.com/questions/3614691/casting-to-void-doesnt-remove-warn-unused-result-error
+    
+    Whatever, I guess we can do something useful with this return value.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit ce7b5ea7d5c30be32e4448ab0e7e6bb6147af548)
+
+commit 6427985b746b7c29f5337d87b60aab9008a8bc6e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:50:45 2013 +0100
+
+    mon: Monitor: make sure we backup a monmap during sync start
+    
+    First of all, we must find a monmap to backup.  The newest version.
+    
+    Secondly, we must make sure we back it up before clearing the store.
+    
+    Finally, we must make sure that we don't remove said backup while
+    clearing the store; otherwise, we would be out of a backup monmap if the
+    sync happened to fail (and if the monitor happened to be killed before a
+    new sync had finished).
+    
+    This patch makes sure these conditions are met.
+    
+    Fixes: #5256 (partially)
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e6dc4ea21b452e34599678792cd36ce1ba3edb3)
+
+commit f8f83e78218ee5b8602cfe6758caf2ecd3e06602
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:36:44 2013 +0100
+
+    mon: Monitor: obtain latest monmap on sync store init
+    
+    Always use the highest version amongst all the typically available
+    monmaps: whatever we have in memory, whatever we have under the
+    MonmapMonitor's store, and whatever we have backed up from a previous
+    sync.  This ensures we always use the newest version we came across
+    with.
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6284fdce794b73adcc757fee910e975b6b4bd054)
+
+commit 41e1a6beeef69136fe6ec3f31d07ea8c33d64cba
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:21:58 2013 +0100
+
+    mon: Monitor: don't remove 'mon_sync' when clearing the store during abort
+    
+    Otherwise, we will end up losing the monmap we backed up when we started
+    the sync, and the monitor may be unable to start if it is killed or
+    crashes in-between the sync abort and finishing a new sync.
+    
+    Fixes: #5256 (partially)
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit af5a9861d7c6b4527b0d2312d0efa792910bafd9)
+
+commit 3f32ded41818ad3c607339d723c095f166e36f7e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:31:23 2013 -0700
+
+    os/FileStore: drop posix_fadvise(...DONTNEED)
+    
+    On XFS this call is problematic because it directly calls the filemap
+    writeback without vectoring through xfs.  This can break the delicate
+    ordering of writeback and range zeroing; see #4976 and this thread
+    
+          http://oss.sgi.com/archives/xfs/2013-06/msg00066.html
+    
+    Drop this behavior for now to avoid subtle data corruption.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ffade3c85dfffa13a16edd9630a52d99eb8a413d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:24:16 2013 -0700
+
+    os/FileStore: use fdatasync(2) instead of sync_file_range(2)
+    
+    The use of sync_file_range(2) on XFS screws up XFS' delicate ordering
+    of writeback and range zeroing; see #4976 and this thread:
+    
+      http://oss.sgi.com/archives/xfs/2013-06/msg00066.html
+    
+    Instead, replace all sync_file_range(2) calls with fdatasync(2), which
+    *does* do ordered writeback and should not leak unzeroed blocks.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 3383ba6326aeff6500df4792fcef65740dd6b3ea
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 12 21:47:09 2013 -0700
+
+    config: fix run_dir typo
+    
+    From 654299108bfb11e7dce45f54946d1505f71d2de8.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e9689ac6f5f50b077a6ac874f811d204ef996c96)
+
+commit 41e1416c3585507db75f10ed69f65a5a80370827
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 14:51:08 2013 -0700
+
+    ceph.spec: create /var/run on package install
+    
+    The %ghost %dir ... line will make this get cleaned up but won't install
+    it.
+    
+    Reported-by: Derek Yarnell <derek@umiacs.umd.edu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 64ee0148a5b7324c7df7de2d5f869b880529d452)
+
+commit 2df3619169d2e83617fc3005b68de395e982b39a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 17:03:41 2013 -0700
+
+    global: create /var/run/ceph on daemon startup
+    
+    This handles cases where the daemon is started without the benefit of
+    sysvinit or upstart (as with teuthology or ceph-fuse).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 654299108bfb11e7dce45f54946d1505f71d2de8)
+
+commit eace9987c19f4da5cc89ca84abec5e54035b4887
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jun 5 11:10:34 2013 -0700
+
+    PG: don't dirty log unconditionally in activate()
+    
+    merge_log and friends all take care of dirtying the log
+    as necessary.
+    
+    Fixes: #5238
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5deece1d034749bf72b7bd04e4e9c5d97e5ad6ce)
+
+commit fda618e96690b2693c7a8dd523ece7c1c5a90b67
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 14 17:11:43 2013 +0100
+
+    mon: OSDMonitor: don't ignore apply_incremental()'s return on UfP [1]
+    
+    apply_incremental() may return -EINVAL.  Don't ignore it.
+    
+    [1] UfP = Update from Paxos
+    
+    Fixes: #5343
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e3c33f4315cbf8718f61eb79e15dd6d44fc908b7)
+
+commit c14dd154584eef97075e4a260719b6cbe686d4c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 16:38:26 2013 -0700
+
+    client: handle reset during initial mds session open
+    
+    If we get a reset during our attempt to open an MDS session, close out the
+    Connection* and retry to open the session, moving the waiters over.
+    
+    Fixes: #5379
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit df8a3e5591948dfd94de2e06640cfe54d2de4322)
+
+commit aa8834831dbfcdebb96841686e3707e552c349f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 15:43:40 2013 -0700
+
+    ceph-disk: add some notes on wth we are up to
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c6b24e9039079e897108f28d6af58cbc703a15a)
+
+commit 261964adc5a7efe7c0efac2d527bc10a95498e45
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 16:29:10 2013 -0700
+
+    ceph-disk: clear TERM to avoid libreadline hijinx
+    
+    The weird output from libreadline users is related to the TERM variable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e538829f16ce19d57d63229921afa01cc687eb86)
+
+commit e36fb3267cdbe0ec5bb01f8a012485de1bbf1652
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 09:49:46 2013 -0700
+
+    ceph-disk-udev: set up by-partuuid, -typeuuid symlinks on ancient udev
+    
+    Make the ancient-udev/blkid workaround script for RHEL/CentOS create the
+    symlinks for us too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d7f7d613512fe39ec883e11d201793c75ee05db1)
+
+commit c3936ebb159b419115bfdc5dc7a23f5807cff9fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 20:06:33 2013 -0700
+
+    ceph-disk: do not stop activate-all on first failure
+    
+    Keep going even if we hit one activation error.  This avoids failing to
+    start some disks when only one of them won't start (e.g., because it
+    doesn't belong to the current cluster).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9074375bfbe1e3757b9c423a5ff60e8013afbce)
+
+commit 3cb783c31541cebe928b97cf27eb326bb4b54f9b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 16:30:24 2013 -0700
+
+    ceph.spec: include partuuid rules in package
+    
+    Commit f3234c147e083f2904178994bc85de3d082e2836 missed this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 253069e04707c5bf46869f4ff5a47ea6bb0fde3e)
+
+commit 2d771b57082545e139086f3c5c9f79ba47101a14
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 15:01:14 2013 -0700
+
+    ceph.spec: install/uninstall init script
+    
+    This was commented out almost years ago in commit 9baf5ef4 but it is not
+    clear to me that it was correct to do so.  In any case, we are not
+    installing the rc.d links for ceph, which means it does not start up after
+    a reboot.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc9b83a80262d014cc37f0c974963cf7402a577a)
+
+commit dfe421e44cfa13178950258509a0cf71e308daa1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:39:03 2013 -0700
+
+    sysvinit, upstart: ceph-disk activate-all on start
+    
+    On 'service ceph start' or 'service ceph start osd' or start ceph-osd-all
+    we should activate any osd GPT partitions.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 13680976ef6899cb33109f6f841e99d4d37bb168)
+
+commit ef47cf9c9dd7f4a1f549c14f984cd052fc95bb64
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:34:40 2013 -0700
+
+    ceph-disk: add 'activate-all'
+    
+    Scan /dev/disk/by-parttypeuuid for ceph OSDs and activate them all.  This
+    is useful when the event didn't trigger on the initial udev event for
+    some reason.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c7a23687a1a21bec5cca7b302ac4ba47c78e041)
+
+commit 65ec1d87f96694ab08c99a7104329065c7c5ebae
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:23:52 2013 -0700
+
+    udev: /dev/disk/by-parttypeuuid/$type-$uuid
+    
+    We need this to help trigger OSD activations.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d512dc9eddef3299167d4bf44e2018b3b6031a22)
+
+commit 4a739b13f420acf4f0af21b4cf133ac08c97ee18
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 14 14:53:54 2013 -0700
+
+    rgw: escape prefix correctly when listing objects
+    
+    Fixes: #5362
+    When listing objects prefix needs to be escaped correctly (the
+    same as with the marker). Otherwise listing objects with prefix
+    that starts with underscore doesn't work.
+    Backport: bobtail, cuttlefish
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit d582ee2438a3bd307324c5f44491f26fd6a56704)
+
+commit 6edba9e9ad1c115a462aeab858063fe3939a40e4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 10 17:28:22 2013 -0700
+
+    messages/MMonSync: initialize crc in ctor
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cd1c289b96a874ff99a83a44955d05efc9f2765a)
+
+commit 2ad85576850bcb61c3ac8fee38609f9cd51bfac2
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 08:48:37 2013 -0700
+
+    client: fix ancient typo in caps revocation path
+    
+    If we have dropped all references to a revoked capability, send the ack
+    to the MDS.  This typo has been there since v0.7 (early 2009)!
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b7143c2f84daafbe2c27d5b2a2d5dc40c3a68d15)
+
+commit d3d88983bceaf508f2585cc6bf8fc304a0943ffe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 5 08:42:25 2013 -0700
+
+    messages/MMonHealth: remove unused flag field
+    
+    This was initialized in (one of) the ctor(s), but not encoded/decoded,
+    and not used.  Remove it.  This makes valgrind a happy.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 08bb8d510b5abd64f5b9f8db150bfc8bccaf9ce8)
+
+commit 646d9ccc575bf91cf73fbfaf91ab4d04ed2b5981
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 5 08:34:20 2013 -0700
+
+    messages/MMonProbe: fix uninitialized variables
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4974b29e251d433101b69955091e22393172bcd8)
+
+commit 08304a7c46da7517319b7db0b64d1c4f54771472
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 08:14:40 2013 -0700
+
+    common/Preforker: fix broken recursion on exit(3)
+    
+    If we exit via preforker, call exit(3) and not recursively back into
+    Preforker::exit(r).  Otherwise you get a hang with the child blocked
+    at:
+    
+    Thread 1 (Thread 0x7fa08962e7c0 (LWP 5419)):
+    #0  0x000000309860e0cd in write () from /lib64/libpthread.so.0
+    #1  0x00000000005cc906 in Preforker::exit(int) ()
+    #2  0x00000000005c8dfb in main ()
+    
+    and the parent at
+    
+    #0  0x000000309860eba7 in waitpid () from /lib64/libpthread.so.0
+    #1  0x00000000005cc87a in Preforker::parent_wait() ()
+    #2  0x00000000005c75ae in main ()
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7e7ff7532d343c473178799e37f4b83cf29c4eee)
+
+commit f202d332dc86844caa9076c7866cc009d87f76b6
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Jun 13 16:38:26 2013 -0700
+
+    rules:  Don't disable tcmalloc on ARM (and other non-intel)
+    
+    Fixes #5342
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit cbca9232618e85bd15b8677c9306042caad876a6
+Author: Guilhem Lettron <guilhem+github@lettron.fr>
+Date:   Mon May 27 12:41:53 2013 +0200
+
+    Remove mon socket in post-stop
+    
+    If ceph-mon segfault, socket file isn't removed.
+    
+    By adding a remove in post-stop, upstart clean run directory properly.
+    
+    Signed-off-by: Guilhem Lettron <guilhem@lettron.fr>
+    (cherry picked from commit 554b41b171eab997038e83928c462027246c24f4)
+
+commit 3f8b3cf1d4042ae6bb64a840ff19f859e6ce6dc2
+Author: James Page <james.page@ubuntu.com>
+Date:   Mon May 20 10:26:06 2013 +0100
+
+    Remove stop on from upstart tasks
+    
+    Upstart tasks don't have to concept of 'stop on' as they
+    are not long running.
+    (cherry picked from commit 17f6fccabc262b9a6d59455c524b550e77cd0fe3)
+
+commit 0b9b8368aec049bfdd4cf72b481f3a4aa8cb031e
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jun 12 22:22:42 2013 -0700
+
+    ceph-disk: extra dash in error message
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f86b4e7a4831c684033363ddd335d2f3fb9a189a)
+
+commit ecddd1e0062ae58e8cc75e125cc683fc04b8befe
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:41:11 2013 +0200
+
+    ceph-disk: cast output of _check_output()
+    
+    Cast output of _check_output() to str() to be able to use
+    str.split().
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 16ecae153d260407085aaafbad1c1c51f4486c9a)
+
+commit 43855e3788cc90900f1b3424c4e4e2c28e8f351e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:46:15 2013 +0200
+
+    ceph-disk: remove unnecessary semicolons
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9785478a2aae7bf5234fbfe443603ba22b5a50d2)
+
+commit 53927f94f3dcf7c57b73b66159d924557d92506e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:33:16 2013 +0200
+
+    ceph-disk: fix undefined variable
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9429ff90a06368fc98d146e065a7b9d1b68e9822)
+
+commit f1beeb267d6e19754a9f2f483b5849a6b900f3ce
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:29:07 2013 +0200
+
+    ceph-disk: add missing spaces around operator
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c127745cc021c8b244d721fa940319158ef9e9d4)
+
+commit fe23ae0042629f493792eadb88a30514c29fc1a0
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:02:03 2013 -0700
+
+    udev: drop useless --mount argument to ceph-disk
+    
+    It doesn't mean anything anymore; drop it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcfd2f31a50d27038bc02e645795f0ec99dd3b32)
+
+commit d99e0a8908d4feccc0a73952ee31586c83174589
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:01:34 2013 -0700
+
+    ceph-disk-udev: activate-journal
+    
+    Trigger 'ceph-disk activate-journal' from the alt udev rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b139152039bfc0d190f855910d44347c9e79b22a)
+
+commit dfbd20104e01ce22c7e501f748b2bf13382ee7f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 21:56:23 2013 -0700
+
+    ceph-disk: do not use mount --move (or --bind)
+    
+    The kernel does not let you mount --move when the parent mount is
+    shared (see, e.g., https://bugzilla.redhat.com/show_bug.cgi?id=917008
+    for another person this also confused).  We can't use --bind either
+    since that (on RHEL at least) screws up /etc/mtab so that the final
+    result looks like
+    
+     /var/lib/ceph/tmp/mnt.HNHoXU /var/lib/ceph/osd/ceph-0 none rw,bind 0 0
+    
+    Instead, mount the original dev in the final location and then umount
+    from the old location.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e5ffe0d2484eb6cbcefcaeb5d52020b1130871a5)
+
+commit 98a990d796e71c62fa64c13230a18a4a40aedcdf
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 21:22:53 2013 -0700
+
+    ceph.spec: include by-partuuid udev workaround rules
+    
+    These are need for old or buggy udev.  Having them for new and unbroken
+    udev is harmless.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3234c147e083f2904178994bc85de3d082e2836)
+
+commit 1504e23f177fff00a794c0db76db8a8c72bc8eb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 12:10:49 2013 -0700
+
+    ceph-disk: work around buggy rhel/centos parted
+    
+    parted on RHEL/Centos prefixes the *machine readable output* with
+    
+     1b 5b 3f 31 30 33 34 68
+    
+    Note that the same thing happens when you 'import readline' in python.
+    
+    Work around it!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82ff72f827b9bd7f91d30a09d35e42b25d2a7344)
+
+commit 0b9482bd631aada9e2ca730b6cddd51d7954fb11
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 15:54:58 2013 -0700
+
+    ceph-disk: implement 'activate-journal'
+    
+    Activate an osd via its journal device.  udev populates its symlinks and
+    triggers events in an order that is not related to whether the device is
+    an osd data partition or a journal.  That means that triggering
+    'ceph-disk activate' can happen before the journal (or journal symlink)
+    is present and then fail.
+    
+    Similarly, it may be that they are on different disks that are hotplugged
+    with the journal second.
+    
+    This can be wired up to the journal partition type to ensure that osds are
+    started when the journal appears second.
+    
+    Include the udev rules to trigger this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2a78e8d16db0a71b13fc15457abc5fe0091c84c)
+
+commit 062b9296c98a03f4f291615dce010aebc55aaa32
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 11 18:35:01 2013 -0700
+
+    ceph-disk: call partprobe outside of the prepare lock; drop udevadm settle
+    
+    After we change the final partition type, sgdisk may or may not trigger a
+    udev event, depending on how well udev is behaving (it varies between
+    distros, it seems).  The old code would often settle and wait for udev to
+    activate the device, and then partprobe would uselessly fail because it
+    was already mounted.
+    
+    Call partprobe only at the very end, after prepare is done.  This ensures
+    that if partprobe calls udevadm settle (which is sometimes does) we do not
+    get stuck.
+    
+    Drop the udevadm settle.  I'm not sure what this accomplishes; take it out,
+    at least until we determine we need it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8b3b59e01432090f7ae774e971862316203ade68)
+
+commit 9f9dc8b85a81529deb6865ec5f1cebb45f04ddc4
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 11:03:37 2013 -0700
+
+    ceph-disk: add 'zap' command
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 10ba60cd088c15d4b4ea0b86ad681aa57f1051b6)
+
+commit 4fab29c1c5e4dec79f089e9803adf0e855e5ec4f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 21 12:52:03 2013 -0700
+
+    ceph-disk: fix stat errors with new suppress code
+    
+    Broken by 225fefe5e7c997b365f481b6c4f66312ea28ed61.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcc8bfdb672654c6a6b48a2aa08267a894debc32)
+
+commit 01ba391b079a494272a071b76f8dee89b5f5c44d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 13 12:35:32 2013 -0700
+
+    ceph-disk: add '[un]suppress-activate <dev>' command
+    
+    It is often useful to prepare but not activate a device, for example when
+    preparing a bunch of spare disks.  This marks a device as 'do not
+    activate' so that it can be prepared without activating.
+    
+    Fixes: #3255
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 225fefe5e7c997b365f481b6c4f66312ea28ed61)
+
+commit 41a15a63349abc5f3e1c0b8de8fcc543e2567035
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 11:21:25 2013 -0700
+
+    upstart: start ceph-all on runlevel [2345]
+    
+    Starting when only one network interface has started breaks machines with
+    multiple nics in very problematic ways.
+    
+    There may be an earlier trigger that we can use for cases where other
+    services on the local machine depend on ceph, but for now this is better
+    than the existing behavior.
+    
+    See #5248
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7e08ed1bf154f5556b3c4e49f937c1575bf992b8)
+
+commit 60e4bb0acb41febb75c866646c7102e9a25782f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 8 17:38:07 2013 -0700
+
+    client: set issue_seq (not seq) in cap release
+    
+    We regularly have been observing a stall where the MDS is blocked waiting
+    for a cap revocation (Ls, in our case) and never gets a reply.  We finally
+    tracked down the sequence:
+    
+     - mds issues cap seq 1 to client
+     - mds does revocation (seq 2)
+     - client replies
+     - much time goes by
+     - client trims inode from cache, sends release with seq == 2
+     - mds ignores release because its issue_seq is 1
+     - mds later tries to revoke other caps
+     - client discards message because it doesn't have the inode in cache
+    
+    The problem is simply that we are using seq instead of issue_seq in the
+    cap release message.  Note that the other release call site in
+    encode_inode_release() is correct.  That one is much more commonly
+    triggered by short tests, as compared to this case where the inode needs to
+    get pushed out of the client cache.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9b012e234a924efd718826ab6a53b9aeb7cd6649)
+
+commit ae522291d2a9b994a65fed5e63e9fec74eece5be
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 15:03:50 2013 -0700
+
+    osd: skip mark-me-down message if osd is not up
+    
+    Fixes crash when the OSD has not successfully booted and gets a
+    SIGINT or SIGTERM.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c2e262fc9493b4bb22c2b7b4990aa1ee7846940e)
+
+commit 48f8d6e3ffe9fc5bb71b362e3059ffb2da69926b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 21:21:51 2013 -0700
+
+    ceph-fuse: create finisher threads after fork()
+    
+    The ObjectCacher and MonClient classes both instantiate Finisher
+    threads.  We need to make sure they are created *after* the fork(2)
+    or else the process will fail to join() them on shutdown, and the
+    threads will not exist while fuse is doing useful work.
+    
+    Put CephFuse on the heap and move all this initalization into the child
+    block, and make sure errors are passed back to the parent.
+    
+    Fix-proposed-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4fa5f99a40792341d247e51488c37301da3c4e4f)
+
+commit 7012d192e6a31d91e57bc9f5b00af6c541ceb3e0
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 16:35:54 2013 -0700
+
+    osd: do not include logbl in scrub map
+    
+    This is a potentially use object/file, usually prefixed by a zeroed region
+    on disk, that is not used by scrub at all.  It dates back to
+    f51348dc8bdd5071b7baaf3f0e4d2e0496618f08 (2008) and the original version of
+    scrub.
+    
+    This *might* fix #4179.  It is not a leak per se, but I observed 1GB
+    scrub messages going over the write.  Maybe the allocations are causing
+    fragmentation, or the sub_op queues are growing.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0b036ecddbfd82e651666326d6f16b3c000ade18)
+
+commit c75760e39d8df5b1971343e9f9186fd67e154af6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:53:00 2013 -0700
+
+    rgw: handle deep uri resources
+    
+    In case of deep uri resources (ones created beyond a single level
+    of hierarchy, e.g. auth/v1.0) we want to create a new empty
+    handlers for the path if no handlers exists. E.g., for
+    auth/v1.0 we need to have a handler for 'auth', otherwise
+    the default S3 handler will be used, which we don't want.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ad3934e335399f7844e45fcfd17f7802800d2cb3)
+
+commit bd12e81e48014024171c55f5984c9183c8e363cb
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:47:21 2013 -0700
+
+    rgw: fix get_resource_mgr() to correctly identify resource
+    
+    Fixes: #5262
+    The original test was not comparing the correct string, ended up
+    with the effect of just checking the substring of the uri to match
+    the resource.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8d55b87f95d59dbfcfd0799c4601ca37ebb025f5)
+
+commit b1d436e752c9c20e7dbff91b769cb2ba47383571
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 11:22:38 2013 -0700
+
+    rgw: add 'cors' to the list of sub-resources
+    
+    Fixes: #5261
+    Backport: cuttlefish
+    Add 'cors' to the list of sub-resources, otherwise auth signing
+    is wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9a0a9c205b8c24ca9c1e05b0cf9875768e867a9e)
+
+commit 4bf75015ae7539a9dbabbbfc0c784dbd58aa6004
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 21:23:45 2013 -0700
+
+    mon: fix preforker exit behavior behavior
+    
+    In 3c5706163b72245768958155d767abf561e6d96d we made exit() not actually
+    exit so that the leak checking would behave for a non-forking case.
+    That is only needed for the normal exit case; every other case expects
+    exit() to actually terminate and not continue execution.
+    
+    Instead, make a signal_exit() method that signals the parent (if any)
+    and then lets you return.  exit() goes back to it's usual behavior,
+    fixing the many other calls in main().
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 92d085f7fd6224ffe5b7651c1f83b093f964b5cd)
+
+commit d398a1850d5895ad409f5229f0dc06651c3d4d75
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jun 4 13:23:36 2013 -0700
+
+    rados.py: correct some C types
+    
+    trunc was getting size_t instead of uint64_t, leading to bad results
+    in 32-bit environments. Explicitly cast to the desired type
+    everywhere, so it's clear the correct type is being used.
+    
+    Fixes: #5233
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 6dd7d469000144b499af84bda9b735710bb5cec3)
diff --git a/doc/changelog/v0.61.5.txt b/doc/changelog/v0.61.5.txt
new file mode 100644
index 000000000..9d99f910c
--- /dev/null
+++ b/doc/changelog/v0.61.5.txt
@@ -0,0 +1,1199 @@
+commit 8ee10dc4bb73bdd918873f29c70eedc3c7ef1979
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jul 17 16:39:08 2013 -0700
+
+    v0.61.5
+
+commit 39bffac6b6c898882d03de392f7f2218933d942b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:14:50 2013 -0700
+
+    ceph-disk: rely on /dev/disk/by-partuuid instead of special-casing journal symlinks
+    
+    This was necessary when ceph-disk-udev didn't create the by-partuuid (and
+    other) symlinks for us, but now it is fragile and error-prone.  (It also
+    appears to be broken on a certain customer RHEL VM.)  See
+    d7f7d613512fe39ec883e11d201793c75ee05db1.
+    
+    Instead, just use the by-partuuid symlinks that we spent all that ugly
+    effort generating.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 64379e701b3ed862c05f156539506d3382f77aa8)
+
+commit 363d54288254b5e2311cd28fce5988d68cfd5773
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 16 16:49:48 2013 +0100
+
+    mon: Monitor: StoreConverter: clearer debug message on 'needs_conversion()'
+    
+    The previous debug message outputted the function's name, as often our
+    functions do.  This was however a source of bewilderment, as users would
+    see those in logs and think their stores would need conversion.  Changing
+    this message is trivial enough and it will make ceph users happier log
+    readers.
+    
+    Backport: cuttlefish
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad1392f68170b391d11df0ce5523c2d1fb57f60e)
+
+commit 0ea89760def73f76d8100889eca3c25b0a6eb772
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 16 16:45:39 2013 +0100
+
+    mon: Monitor: do not reopen MonitorDBStore during conversion
+    
+    We already open the store on ceph_mon.cc, before we start the conversion.
+    Given we are unable to reproduce this every time a conversion is triggered,
+    we are led to believe that this causes a race in leveldb that will lead
+    to 'store.db/LOCK' being locked upon the open this patch removes.
+    
+    Regardless, reopening the db here is pointless as we already did it when
+    we reach Monitor::StoreConverter::convert().
+    
+    Fixes: #5640
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 036e6739a4e873863bae3d7d00f310c015dfcdb3)
+
+commit 595c09df9134fb0d62144fe1594914c90e567dca
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 14 15:54:29 2013 -0700
+
+    messages/MClientReconnect: clear data when encoding
+    
+    The MClientReconnect puts everything in the data payload portion of
+    the message and nothing in the front portion.  That means that if the
+    message is resent (socket failure or something), the messenger thinks it
+    hasn't been encoded yet (front empty) and reencodes, which means
+    everything gets added (again) to the data portion.
+    
+    Decoding keep decoding until it runs out of data, so the second copy
+    means we decode garbage snap realms, leading to the crash in bug
+    
+    Clearing data each time around resolves the problem, although it does
+    mean we do the encoding work multiple times.  We could alternatively
+    (or also) stick some data in the front portion of the payload
+    (ignored), but that changes the wire protocol and I would rather not
+    do that.
+    
+    Fixes: #4565
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 314cf046b0b787ca69665e8751eab6fe7adb4037)
+
+commit 8c178e0d39d8d4a4820eb061f79d74f95e60199f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 10:05:37 2013 -0700
+
+    mon: once sync full is chosen, make sure we don't change our mind
+    
+    It is possible for a sequence like:
+    
+     - probe
+     - first probe reply has paxos trim that indicates a full sync is
+       needed
+     - start sync
+     - clear store
+     - something happens that makes us abort and bootstrap (e.g., the
+       provider mon restarts
+     - probe
+     - first probe reply has older paxos trim bound and we call an election
+     - on election completion, we crash because we have no data.
+    
+    Non-determinism of the probe decision aside, we need to ensure that
+    the info we share during probe (fc, lc) is accurate, and that once we
+    clear the store we know we *must* do a full sync.
+    
+    This is a backport of aa60f940ec1994a61624345586dc70d261688456.
+    
+    Fixes: #5621
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 6af0ed9bc4cc955f8c30ad9dc6e9095599f323d0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 9 14:12:15 2013 -0700
+
+    mon: do not scrub if scrub is in progress
+    
+    This prevents an assert from unexpected scrub results from the previous
+    scrub on the leader.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 00ae543b3e32f89d906a0e934792cc5309f57696)
+
+commit 5642155ba5ca9b384a7af058a6538ac00c2a592d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 10:06:20 2013 -0700
+
+    messages/MPGStats: do not set paxos version to osdmap epoch
+    
+    The PaxosServiceMessage version field is meant for client-coordinated
+    ordering of messages when switching between monitors (and is rarely
+    used).  Do not fill it with the osdmap epoch lest it be compared to a
+    pgmap version, which may cause the mon to (near) indefinitely put it on
+    a wait queue until the pgmap version catches up.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b36338be43f43b6dd4ee87c97f2eaa23b467c386)
+
+commit 06c65988bb0b1d1ec626fe31e9d806a1c4e24b28
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 11 18:43:24 2013 -0700
+
+    osd/OSDmap: fix OSDMap::Incremental::dump() for new pool names
+    
+    The name is always present when pools are created, but not when they are
+    modified.  Also, a name may be present with a new_pools entry if the pool
+    is just renamed.  Separate it out completely in the dump.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e4a29111e89588385e63f8d92ce3d67739dd679)
+
+commit 658240710baaf9c661b8fbf856322907a0d394ee
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 10:49:28 2013 -0700
+
+    mon/PaxosService: prevent reads until initial service commit is done
+    
+    Do not process reads (or, by PaxosService::dispatch() implication, writes)
+    until we have committed the initial service state.  This avoids things like
+    EPERM due to missing keys when we race with mon creation, triggered by
+    teuthology tests doing their health check after startup.
+    
+    Fixes: #5515
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit d08b6d6df7dba06dad73bdec2c945f24afc02717)
+
+commit 5c3ff33771e227b3fb5cc354323846fe8db4ecc1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 28 12:21:58 2013 -0700
+
+    client: send all request put's through put_request()
+    
+    Make sure all MetaRequest reference put's go through the same path that
+    releases inode references, including all of the error paths.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 87217e1e3cb2785b79d0dec49bd3f23a827551f5)
+
+commit 1df78ad73df581bc7537688ae28bda820b089a13
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 28 11:50:11 2013 -0700
+
+    client: fix remaining Inode::put() caller, and make method psuedo-private
+    
+    Not sure I can make this actually private and make Client::put_inode() a
+    friend method (making all of Client a friend would defeat the purpose).
+    This works well enough, though!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9af3b86b25574e4d2cdfd43e61028cffa19bdeb1)
+
+commit fea024cc3dd2c6fd9ff322d1cd15e0d75c92eca5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 27 21:39:35 2013 -0700
+
+    client: use put_inode on MetaRequest inode refs
+    
+    When we drop the request inode refs, we need to use put_inode() to ensure
+    they get cleaned up properly (removed from inode_map, caps released, etc.).
+    Do this explicitly here (as we do with all other inode put() paths that
+    matter).
+    
+    Fixes: #5381
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81bee6487fb1ce9e090b030d61bda128a3cf4982)
+
+commit 62ae39ec8f208cb8f89e43ba844b9a20b4315c61
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:57:48 2013 -0700
+
+    mon: be smarter about calculating last_epoch_clean lower bound
+    
+    We need to take PGs whose mapping has not changed in a long time into
+    account.  For them, the pg state will indicate it was clean at the time of
+    the report, in which case we can use that as a lower-bound on their actual
+    latest epoch clean.  If they are not currently clean (at report time), use
+    the last_epoch_clean value.
+    
+    Fixes: #5519
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc0006deee3153e06ddd220bf8a40358ba830135)
+
+commit da725852190245d2f91b7b21e72baee70e4342bd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 13:27:58 2013 -0700
+
+    osd: report pg stats to mon at least every N (=500) epochs
+    
+    The mon needs a moderately accurate last_epoch_clean value in order to trim
+    old osdmaps.  To prevent a PG that hasn't peered or received IO in forever
+    from preventing this, send pg stats at some minimum frequency.  This will
+    increase the pg stat report workload for the mon over an idle pool, but
+    should be no worse that a cluster that is getting actual IO and sees these
+    updates from normal stat updates.
+    
+    This makes the reported update a bit more aggressive/useful in that the epoch
+    is the last map epoch processed by this PG and not just one that is >= the
+    currenting interval.  Note that the semantics of this field are pretty useless
+    at this point.
+    
+    See #5519
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit da81228cc73c95737f26c630e5c3eccf6ae1aaec)
+
+commit 757af91b2af0da6bbfeeb53551fa1ef4ef9118ea
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:32:34 2013 -0700
+
+    osd: fix warning
+    
+    From 653e04a79430317e275dd77a46c2b17c788b860b
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bc291d3fc3fc1cac838565cbe0f25f71d855a6e3)
+
+commit 65af2538329472d2fd078bb961863c40cdabda12
+Merge: e537699 804314b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 15:21:20 2013 -0700
+
+    Merge remote-tracking branch 'gh/wip-mon-sync-2' into cuttlefish
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit e537699b33f84c14f027b56372fbcb0a99bbe88d
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Wed Jul 10 14:55:52 2013 -0700
+
+    Get device-by-path by looking for it instead of assuming 3rd entry.
+    
+    On some systems (virtual machines so far) the device-by-path entry
+    from udevadm is not always in the same spot so instead actually
+    look for the right output instead of blindy assuming that its a
+    specific field in the output.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 804314b8bfa5ec75cc9653e2928874c457395c92
+Merge: 6ad9fe1 78f2266
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:40:37 2013 -0700
+
+    Merge remote-tracking branch 'gh/cuttlefish' into wip-mon-sync-2
+
+commit 78f226634bd80f6678b1f74ccf785bc52fcd6b62
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:02:08 2013 -0700
+
+    osd: limit number of inc osdmaps send to peers, clients
+    
+    We should not send an unbounded number of inc maps to our peers or clients.
+    In particular, if a peer is not contacted for a while, we may think they
+    have a very old map (say, 10000 epochs ago) and send thousands of inc maps
+    when the distribution shifts and we need to peer.
+    
+    Note that if we do not send enough maps, the peers will make do by
+    requesting the map from somewhere else (currently the mon).  Regardless
+    of the source, however, we must limit the amount that we speculatively
+    share as it usually is not needed.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 653e04a79430317e275dd77a46c2b17c788b860b)
+
+commit 54ee2dc80ed032c286546da51442340ec9991cdf
+Author: Christophe Courtaut <christophe.courtaut@gmail.com>
+Date:   Mon Jul 1 14:57:17 2013 +0200
+
+    rgw: Fix return value for swift user not found
+    
+    http://tracker.ceph.com/issues/1779 fixes #1779
+    
+    Adjust the return value from rgw_get_user_info_by_swift call
+    in RGW_SWIFT_Auth_Get::execute() to have the correct
+    return code in response.
+    (cherry picked from commit 4089001de1f22d6acd0b9f09996b71c716235551)
+
+commit 47852c263831707fff1570317a7446b0700c5962
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 9 21:55:51 2013 -0700
+
+    mon/OSDMonitor: make 'osd crush rm ...' slightly more idempotent
+    
+    This is a manual backport of 18a624fd8b90d9959de51f07622cf0839e6bd9aa.
+    Do not return immediately if we are looking at uncommitted state.t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit bfc26c656d183fbcc90a352391e47f9f51c96052
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 17:46:40 2013 -0700
+
+    mon/OSDMonitor: fix base case for loading full osdmap
+    
+    Right after cluster creation, first_committed is 1 and latest stashed in 0,
+    but we don't have the initial full map yet.  Thereafter, we do (because we
+    write it with trim).  Fixes afd6c7d8247075003e5be439ad59976c3d123218.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 43fa7aabf1f7e5deb844c1f52d451bab9e7d1006)
+
+commit 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:04:59 2013 -0700
+
+    mon: fix osdmap stash, trim to retain complete history of full maps
+    
+    The current interaction between sync and stashing full osdmaps only on
+    active mons means that a sync can result in an incomplete osdmap_full
+    history:
+    
+     - mon.c starts a full sync
+     - during sync, active osdmap service should_stash_full() is true and
+       includes a full in the txn
+     - mon.c sync finishes
+     - mon.c update_from_paxos gets "latest" stashed that it got from the
+       paxos txn
+     - mon.c does *not* walk to previous inc maps to complete it's collection
+       of full maps.
+    
+    To fix this, we disable the periodic/random stash of full maps by the
+    osdmap service.
+    
+    This introduces a new problem: we must have at least one full map (the first
+    one) in order for a mon that just synced to build it's full collection.
+    Extend the encode_trim() process to allow the osdmap service to include
+    the oldest full map with the trim txn.  This is more complex than just
+    writing the full maps in the txn, but cheaper--we only write the full
+    map at trim time.
+    
+    This *might* be related to previous bugs where the full osdmap was
+    missing, or case where leveldb keys seemed to 'disappear'.
+    
+    Fixes: #5512
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit afd6c7d8247075003e5be439ad59976c3d123218)
+
+commit 24f90b832c695ef13021db66a178c18369ac356d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:07:57 2013 -0700
+
+    mon: implement simple 'scrub' command
+    
+    Compare all keys within the sync'ed prefixes across members of the quorum
+    and compare the key counts and CRC for inconsistencies.
+    
+    Currently this is a one-shot inefficient hammer.  We'll want to make this
+    work in chunks before it is usable in production environments.
+    
+    Protect with a feature bit to avoid sending MMonScrub to mons who can't
+    decode it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a9906641a1dce150203b72682da05651e4d68ff5)
+    
+    Conflicts:
+    
+    	src/mon/MonCommands.h
+    	src/mon/Monitor.cc
+
+commit 926f723c12428a034545c6c4ff6641e1d5e05d24
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 3 11:18:33 2013 -0700
+
+    Elector.h: features are 64 bit
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3564e304e3f50642e4d9ff25e529d5fc60629093)
+
+commit c2b38291e706c9d1d4d337cee3a944f34bf66525
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 3 11:18:19 2013 -0700
+
+    ceph_features.h: declare all features as ULL
+    
+    Otherwise, the first 32 get |'d together as ints.  Then, the result
+    ((int)-1) is sign extended to ((long long int)-1) before being |'d
+    with the 1LL entries.  This results in ~((uint64_t)0).
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4255b5c2fb54ae40c53284b3ab700fdfc7e61748)
+
+commit 95ef961d8537fc369efd0634262ffb8f288d6e9e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 2 21:09:36 2013 -0700
+
+    Pipe: use uint64_t not unsigned when setting features
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit bc3e2f09f8860555d8b3b49b2eea164b4118d817)
+
+commit 09d258b70a28e5cea555b9d7e215fe41d6b84577
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 11:24:48 2013 -0700
+
+    client: remove O_LAZY
+    
+    The once-upon-a-time unique O_LAZY value I chose forever ago is now
+    O_NOATIME, which means that some clients are choosing relaxed
+    consistency without meaning to.
+    
+    It is highly unlikely that a real O_LAZY will ever exist, and we can
+    select it in the ceph case with the ioctl or libcephfs call, so drop
+    any support for doing this via open(2) flags.
+    
+    Update doc/lazy_posix.txt file re: lazy io.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 94afedf02d07ad4678222aa66289a74b87768810)
+
+commit c3b684932bad31fc853ad556d16e1e4a9926486e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 12:55:20 2013 -0700
+
+    osd/osd_types: fix pg_stat_t::dump for last_epoch_clean
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 69a55445439fce0dd6a3d32ff4bf436da42f1b11)
+
+commit a02f2510fcc800b9f2cf2a06401a7b97d5985409
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 5 16:03:49 2013 -0700
+
+    mon: remove bad assert about monmap version
+    
+    It is possible to start a sync when our newest monmap is 0.  Usually we see
+    e0 from probe, but that isn't always published as part of the very first
+    paxos transaction due to the way PaxosService::_active generates it's
+    first initial commit.
+    
+    In any case, having e0 here is harmless.
+    
+    Fixes: #5509
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 85a1d6cc5d3852c94d1287b566656c5b5024fa13)
+
+commit 6ad9fe17a674ba65bbeb4052cb1ac47f3113e7bf
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 19:33:06 2013 -0700
+
+    mon/Paxos: fix sync restart
+    
+    If we have a sync going, and an election intervenes, the client will
+    try to continue by sending a new start_chunks request.  In order to
+    ensure that we get all of the paxos commits from our original starting
+    point (and thus properly update the keys from which they started),
+    only pay attention if they *also* send their current last_committed
+    version.  Otherwise, start them at the beginning.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c5812b1c893305a7d20f9eaec2695c8b1691f0c9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:57:06 2013 -0700
+
+    mon: uninline _trim_enable and Paxos::trim_{enable,disable} so we can debug them
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6fbcbd7fddf35a5be4b38e536871903bff4f9bf1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:55:34 2013 -0700
+
+    mon/Paxos: increase paxos max join drift
+    
+    A value of 10 is too aggressive for large, long-running syncs. 100 is
+    about 2 minutes of activity at most, which should be a more forgiving
+    buffer.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f3a51fa30e5ce1656853b40d831409f195f6e4ca
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:21:04 2013 -0700
+
+    mon/Paxos: configure minimum paxos txns separately
+    
+    We were using paxos_max_join_drift to control the minimum number of
+    paxos transactions to keep around.  Instead, make this explicit, and
+    separate from the join drift.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1156721f22f5f337241eef3d0276ca74fe6352d1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 17:09:07 2013 -0700
+
+    mon: include any new paxos commits in each sync CHUNK message
+    
+    We already take note of the paxos version when we begin the sync.  As
+    sync progresses and there are new paxos commits/txns, include those
+    and update last_committed, so that when sync completes we will have
+    a full view of everything that happened during sync.
+    
+    Note that this does not introduce any compatibility change.  This change
+    *only* affects the provider.  The key difference is that at the end
+    of the sync, the provide will set version to the latest version, and
+    not the version from the start of the sync (as was done previously).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 40672219a081f0dc2dd536977290ef05cfc9f097
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 12:17:28 2013 -0700
+
+    mon/MonitorDBStore: expose get_chunk_tx()
+    
+    Allow users get the transaction unencoded.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit db2bb270e93ed44f9252d65d1d4c9b36875d0ea5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 17:15:56 2013 -0700
+
+    mon: enable leveldb cache by default
+    
+    256 is not as large as the upstream 512 MB, but will help signficiantly and
+    be less disruptive for existing cuttlefish clusters.
+    
+    Sort-of backport of e93730b7ffa48b53c8da2f439a60cb6805facf5a.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 123f676e3ae8154ca94cb076c4c4ec5389d2a643
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 16:56:06 2013 -0700
+
+    mon/Paxos: make 'paxos trim disabled max versions' much much larger
+    
+    108000 is about 3 hours if paxos is going full-bore (1 proposal/second).
+    That ought to be pretty safe.  Otherwise, we start trimming to soon and a
+    slow sync will just have to restart when it finishes.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 71ebfe7e1abe4795b46cf00dfe1b03d1893368b0)
+    
+    Conflicts:
+    
+    	src/common/config_opts.h
+
+commit 03393c0df9f54e4f1db60e1058ca5a7cd89f44e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:01:40 2013 -0700
+
+    mon: do not reopen MonitorDBStore during startup
+    
+    level doesn't seem to like this when it races with an internal compaction
+    attempt (see below).  Instead, let the store get opened by the ceph_mon
+    caller, and pull a bit of the logic into the caller to make the flow a
+    little easier to follow.
+    
+        -2> 2013-06-25 17:49:25.184490 7f4d439f8780 10 needs_conversion
+        -1> 2013-06-25 17:49:25.184495 7f4d4065c700  5 asok(0x13b1460) entry start
+         0> 2013-06-25 17:49:25.316908 7f4d3fe5b700 -1 *** Caught signal (Segmentation fault) **
+     in thread 7f4d3fe5b700
+    
+     ceph version 0.64-667-g089cba8 (089cba8fc0e8ae8aef9a3111cba7342ecd0f8314)
+     1: ceph-mon() [0x649f0a]
+     2: (()+0xfcb0) [0x7f4d435dccb0]
+     3: (leveldb::Table::BlockReader(void*, leveldb::ReadOptions const&, leveldb::Slice const&)+0x154) [0x806e54]
+     4: ceph-mon() [0x808840]
+     5: ceph-mon() [0x808b39]
+     6: ceph-mon() [0x806540]
+     7: (leveldb::DBImpl::DoCompactionWork(leveldb::DBImpl::CompactionState*)+0xdd) [0x7f363d]
+     8: (leveldb::DBImpl::BackgroundCompaction()+0x2c0) [0x7f4210]
+     9: (leveldb::DBImpl::BackgroundCall()+0x68) [0x7f4cc8]
+     10: ceph-mon() [0x80b3af]
+     11: (()+0x7e9a) [0x7f4d435d4e9a]
+     12: (clone()+0x6d) [0x7f4d4196bccd]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ea1f316e5de21487ae034a1aa929068ba23ac525)
+
+commit 0143acc49bc5834836afc2c5a9d8f67030bec85f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 2 14:43:17 2013 -0700
+
+    sysvinit, upstart: handle symlinks to dirs in /var/lib/ceph/*
+    
+    Match a symlink to a dir, not just dirs.  This fixes the osd case of e.g.,
+    creating an osd in /data/osd$id in which ceph-disk makes a symlink from
+    /var/lib/ceph/osd/ceph-$id.
+    
+    Fix proposed by Matt Thompson <matt.thompson@mandiant.com>; extended to
+    include the upstart users too.
+    
+    Fixes: #5490
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 87c98e92d1375c8bc76196bbbf06f677bef95e64)
+
+commit 7e878bcc8c1b51538f3c05f854a9dac74c09b116
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 1 17:33:11 2013 -0700
+
+    rgw: add RGWFormatter_Plain allocation to sidestep cranky strlen()
+    
+    Valgrind complains about an invalid read when we don't pad the allocation,
+    and because it is inlined we can't whitelist it for valgrind.  Workaround
+    the warning by just padding our allocations a bit.
+    
+    Fixes: #5346
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 49ff63b1750789070a8c6fef830c9526ae0f6d9f)
+
+commit ca61402855966210ba1598239eaf454eaad0f5f2
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Wed May 15 11:24:36 2013 +0800
+
+    mds: warn on unconnected snap realms
+    
+    When there are more than one active MDS, restarting MDS triggers
+    assertion "reconnected_snaprealms.empty()" quite often. If there
+    is no snapshot in the FS, the items left in reconnected_snaprealms
+    should be other MDS' mdsdir. I think it's harmless.
+    
+    If there are snapshots in the FS, the assertion probably can catch
+    real bugs. But at present, snapshot feature is broken, fixing it is
+    non-trivial. So replace the assertion with a warning.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 26effc0e583b0a3dade6ec81ef26dec1c94ac8b2)
+
+commit e11f258831e14dc3755e09c0fd4f9bfdf79022a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:53:08 2013 -0700
+
+    mon/PGMonitor: use post_paxos_update, not init, to refresh from osdmap
+    
+    We do two things here:
+     - make init an one-time unconditional init method, which is what the
+       health service expects/needs.
+     - switch PGMonitor::init to be post_paxos_update() which is called after
+       the other services update, which is what PGMonitor really needs.
+    
+    This is a new version of the fix originally in commit
+    a2fe0137946541e7b3b537698e1865fbce974ca6 (and those around it).  That is,
+    this re-fixes a problem where osds do not see pg creates from their
+    subscribe due to map_pg_creates() not getting called.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e635c47851d185eda557e36bdc4bf3775f7b87a2)
+    
+    Conflicts:
+    	src/mon/PGMonitor.cc
+    	src/mon/PGMonitor.h
+
+commit 4d07fb014178da3c88edeb8765e1aaacb8cb8ffa
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:52:01 2013 -0700
+
+    mon/PaxosService: add post_paxos_update() hook
+    
+    Some services need to update internal state based on other service's
+    state, and thus need to be run after everyone has pulled their info out of
+    paxos.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 131686980f0a930d5de7cbce8234fead5bd438b6)
+
+commit 90f5c448abeb127ae5a5528a79bd7bdbc74cb497
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jun 27 14:58:14 2013 -0700
+
+    ceph-disk: s/else if/elif/
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit bd8255a750de08c1b8ee5e9c9a0a1b9b16171462)
+    (cherry picked from commit 9e604ee6943fdb131978afbec51321050faddfc6)
+
+commit 5c4bb463dca5aa61ea5f02f7592d5a3cc82cf6f4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 26 11:28:57 2013 -0700
+
+    rgw: fix radosgw-admin buckets list
+    
+    Fixes: #5455
+    Backport: cuttlefish
+    This commit fixes a regression, where radosgw-admin buckets list
+    operation wasn't returning any data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e1f9fe58d2860fcbb18c92d3eb3946236b49a6ce)
+
+commit b2fb48762f32279e73feb83b220339fea31275e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 19 17:27:49 2013 -0700
+
+    ceph-disk: use unix lock instead of lockfile class
+    
+    The lockfile class relies on file system trickery to get safe mutual
+    exclusion.  However, the unix syscalls do this for us.  More
+    importantly, the unix locks go away when the owning process dies, which
+    is behavior that we want here.
+    
+    Fixes: #5387
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 2a4953b697a3464862fd3913336edfd7eede2487)
+
+commit 26e7a6fffde4abcb685f34247e8491c05ee2a68d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 18:27:49 2013 -0700
+
+    ceph-disk: do not mount over an osd directly in /var/lib/ceph/osd/$cluster-$id
+    
+    If we see a 'ready' file in the target OSD dir, do not mount our device
+    on top of it.
+    
+    Among other things, this prevents ceph-disk activate on stray disks from
+    stepping on teuthology osds.
+    
+    Fixes: #5445
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8a17f33b14d858235dfeaa42be1f4842dcfd66d2)
+
+commit ccb3dd5ad5533ca4e9b656b4e3df31025a5f2017
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Tue Apr 2 15:46:51 2013 +0800
+
+    mds: fix underwater dentry cleanup
+    
+    If the underwater dentry is a remove link, we shouldn't mark the
+    inode clean
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 81d073fecb58e2294df12b71351321e6d2e69652)
+
+commit 3020c5ea07a91475a7261dc2b810f5b61a1ae1f2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 18:51:07 2013 -0700
+
+    mon/Elector: cancel election timer if we bootstrap
+    
+    If we short-circuit and bootstrap, cancel our timer.  Otherwise it will
+    go off some time later when we are in who knows what state.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9ae0ec83dabe37ac15e5165559debdfef7a5f91d)
+
+commit 305f0c50a5f0ffabc73e10bdf4590217d5d5d211
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 18:12:11 2013 -0700
+
+    mon: cancel probe timeout on reset
+    
+    If we are probing and get (say) an election timeout that calls reset(),
+    cancel the timer.  Otherwise, we assert later with a splat like
+    
+    2013-06-24 01:09:33.675882 7fb9627e7700  4 mon.b@0(leader) e1 probe_timeout 0x307a520
+    2013-06-24 01:09:33.676956 7fb9627e7700 -1 mon/Monitor.cc: In function 'void Monitor::probe_timeout(int)' thread 7fb9627e7700 time 2013-06-24 01:09:43.675904
+    mon/Monitor.cc: 1888: FAILED assert(is_probing() || is_synchronizing())
+    
+     ceph version 0.64-613-g134d08a (134d08a9654f66634b893d493e4a92f38acc63cf)
+     1: (Monitor::probe_timeout(int)+0x161) [0x56f5c1]
+     2: (Context::complete(int)+0xa) [0x574a2a]
+     3: (SafeTimer::timer_thread()+0x425) [0x7059a5]
+     4: (SafeTimerThread::entry()+0xd) [0x7065dd]
+     5: (()+0x7e9a) [0x7fb966f62e9a]
+     6: (clone()+0x6d) [0x7fb9652f9ccd]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Fixes: #5438
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 03d3be3eaa96a8e72754c36abd6f355c68d52d59)
+
+commit a8f601d543168f4cdbddf674479d8de4b8dfc732
+Author: Alexandre Maragone <alexandre.marangone@inktank.com>
+Date:   Tue Jun 18 16:18:01 2013 -0700
+
+    ceph-disk: make list_partition behave with unusual device names
+    
+    When you get device names like sdaa you do not want to mistakenly conclude that
+    sdaa is a partition of sda.  Use /sys/block/$device/$partition existence
+    instead.
+    
+    Fixes: #5211
+    Backport: cuttlefish
+    Signed-off-by: Alexandre Maragone <alexandre.maragone@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c0daafe003935881c5192e0b6b59b949269e5ae)
+
+commit 1c890f5cdfc596588e54fffeb016b4a5e9e2124c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:28:24 2013 -0700
+
+    client: fix warning
+    
+    client/Client.cc: In member function 'virtual void Client::ms_handle_remote_reset(Connection*)':
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_NEW' not handled in switch [-Wswitch]
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_OPEN' not handled in switch [-Wswitch]
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_CLOSED' not handled in switch [-Wswitch]
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 8bd936f077530dfeb2e699164e4492b1c0973088)
+
+commit c3b97591fd8206825bcfe65bdb24fbc75a2a9b42
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 17:58:48 2013 -0700
+
+    mon/AuthMonitor: ensure initial rotating keys get encoded when create_initial called 2x
+    
+    The create_initial() method may get called multiple times; make sure it
+    will unconditionally generate new/initial rotating keys.  Move the block
+    up so that we can easily assert as much.
+    
+    Broken by commit cd98eb0c651d9ee62e19c2cc92eadae9bed678cd.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 521fdc2a4e65559b3da83283e6ca607b6e55406f)
+
+commit 0cc826c385edb2e327505696491d3ff1c3bfe8fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 17:42:04 2013 -0700
+
+    init-radosgw.sysv: remove -x debug mode
+    
+    Fixes: #5443
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 31d6062076fdbcd2691c07a23b381b26abc59f65)
+
+commit 4d57c12faceb7f591f10776c6850d98da55c667b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 12:52:44 2013 -0700
+
+    common/pick_addresses: behave even after internal_safe_to_start_threads
+    
+    ceph-mon recently started using Preforker to working around forking issues.
+    As a result, internal_safe_to_start_threads got set sooner and calls to
+    pick_addresses() which try to set string config values now fail because
+    there are no config observers for them.
+    
+    Work around this by observing the change while we adjust the value.  We
+    assume pick_addresses() callers are smart enough to realize that their
+    result will be reflected by cct->_conf and not magically handled elsewhere.
+    
+    Fixes: #5195, #5205
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eb86eebe1ba42f04b46f7c3e3419b83eb6fe7f9a)
+
+commit e1ac7c6c3ca673d08710829aa5a3c03735710486
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 20 15:39:23 2013 -0700
+
+    mon/PaxosService: allow paxos service writes while paxos is updating
+    
+    In commit f985de28f86675e974ac7842a49922a35fe24c6c I mistakenly made
+    is_writeable() false while paxos was updating due to a misread of
+    Paxos::propose_new_value() (I didn't see that it would queue).
+    This is problematic because it narrows the window during which each service
+    is writeable for no reason.
+    
+    Allow service to be writeable both when paxos is active and updating.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 11169693d086e67dcf168ce65ef6e13eebd1a1ab)
+
+commit 02b0b4a9acb439b2ee5deadc8b02492006492931
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:41:21 2013 -0700
+
+    mon/PaxosService: not active during paxos UPDATING_PREVIOUS
+    
+    Treat this as an extension of the recovery process, e.g.
+    
+     RECOVERING -> ACTIVE
+    or
+     RECOVERING -> UPDATING_PREVIOUS -> ACTIVE
+    
+    and we are not active until we get to "the end" in both cases.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 392a8e21f8571b410c85be2129ef62dd6fc52b54)
+
+commit c6d5dc4d47838c8c8f4d059b7d018dea3f9c4425
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:40:22 2013 -0700
+
+    mon: simplify states
+    
+    - make states mutually exclusive (an enum)
+    - rename locked -> updating_previous
+    - set state prior to begin() to simplify things a bit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ee34a219605d1943740fdae0d84cfb9020302dd6)
+
+commit c43b1f4dff254df96144b0b4d569cc72421a8fff
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:14:58 2013 -0700
+
+    mon/Paxos: not readable when LOCKED
+    
+    If we are re-proposing a previously accepted value from a previous quorum,
+    we should not consider it readable, because it is possible it was exposed
+    to clients as committed (2/3 accepted) but not recored to be committed, and
+    we do not want to expose old state as readable when new state was
+    previously readable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ec2ea86ed55e00265c2cc5ad0c94460b4c92865c)
+
+commit 10d41200622d76dbf276602828584e7153cb22b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:07:38 2013 -0700
+
+    mon/Paxos: cleanup: drop unused PREPARING state bit
+    
+    This is never set when we block, and nobody looks at it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7b7ea8e30e20704caad9a841332ecb2e39819a41)
+
+commit 9d7c40e3f4ea2dd969aa0264ea8a6ad74f3e678a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 15:20:05 2013 -0700
+
+    mon/PaxosService: simplify is_writeable
+    
+    Recast this in terms of paxos check + our conditions, and make it
+    match wait_for_writeable().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f985de28f86675e974ac7842a49922a35fe24c6c)
+
+commit 35745cba8985c5f3238e3c28fd28b194fae043d9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 4 17:03:15 2013 -0700
+
+    mon/PaxosService: simplify readable check
+    
+    Recast this in terms of the paxos check and our additional conditions,
+    which match wait_for_readable().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3aa61a0beb540e48bf61ceded766d6ff52c95eb2)
+
+commit 57c89291a48c319907fb3029746d9f5a4bd9dd61
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 16:45:08 2013 -0700
+
+    mon: simplify Monitor::init_paxos()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e832e76a4af04b091c806ad412bcfd0326d75a2d)
+
+commit fd1769cb2d61e8f2c7921a78760e8f12b28258fb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 16:39:37 2013 -0700
+
+    mon/Paxos: go active *after* refreshing
+    
+    The update_from_paxos() methods occasionally like to trigger new activity.
+    As long as they check is_readable() and is_writeable(), they will defer
+    until we go active and that activity will happen in the normal callbacks.
+    
+    This fixes the problem where we active but is_writeable() is still false,
+    triggered by PGMonitor::check_osd_map().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e68b1bd36ed285e38a558899f83cf224d3aa60ed)
+
+commit cf75478d027dfd377424988745230d096dae79ac
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 15:32:06 2013 -0700
+
+    mon: safely signal bootstrap from MonmapMonitor::update_from_paxos()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dc83430124a5fd37573202a4cc0986c3c03739ef)
+
+commit 6ac58cd9c1f9c80c5f3cbe97e19cfcd8427db46d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:57:11 2013 -0700
+
+    mon/Paxos: do paxos refresh in finish_proposal; and refactor
+    
+    Do the paxos refresh inside finish_proposal, ordered *after* the leader
+    assertion so that MonmapMonitor::update_from_paxos() calling bootstrap()
+    does not kill us.
+    
+    Also, remove unnecessary finish_queued_proposal() and move the logic inline
+    where the bad leader assertion is obvious.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a42d7582f816b45f5d19c393fd45447555e78fdd)
+
+commit 054e96d96533b1c4078402e43184f13b97329905
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Jun 2 16:15:02 2013 -0700
+
+    mon/PaxosService: cache {first,last}_committed
+    
+    Refresh the in-memory values when we are told the on-disk paxos state
+    may have changed.
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 2fccb300bdf6ffd44db3462eb05115da11322ed4)
+
+commit 265212a7384399bf85e15e6978bc7543824c0e92
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 14:30:48 2013 -0700
+
+    mon: no need to refresh from _active
+    
+    The refresh is done explicitly by the monitor, independent of the more
+    fragile PaxosService callbacks.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d941363d6e4249e97b64faff0e573f75e918ac0c)
+
+commit 1d8662504299babec22c714662cefbb86a0acb8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:10:57 2013 -0700
+
+    mon: remove unnecessary update_from_paxos calls
+    
+    The refresh() will do this when the state changes; no need to
+    opportunistically call this method all of the time.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 03014a4ecc06cde420fad0c6c2a0177ebd7b839d)
+
+commit 34acc5a3161b6bcda2b9f7ce18d89a8618fff1c5
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:14:01 2013 -0700
+
+    mon: explicitly refresh_from_paxos() when leveldb state changes
+    
+    Instead of opportunistically calling each service's update_from_paxos(),
+    instead explicitly refresh all in-memory state whenever we know the
+    paxos state may have changed.  This is simpler and less fragile.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc339c07312006e65854207523f50542d00ecf87)
+
+commit 4474a0cc6c009a566ecf46efadb39d80343a7c68
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 23 09:25:55 2013 -0700
+
+    mon/AuthMonitor: make initial auth include rotating keys
+    
+    This closes a very narrow race during mon creation where there are no
+    service keys.
+    
+    Fixes: #5427
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cd98eb0c651d9ee62e19c2cc92eadae9bed678cd)
+
+commit d572cf6f77418f217a5a8e37f1124dc566e24d0b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 21 11:53:29 2013 -0700
+
+    mds: fix iterator invalidation for backtrace removal
+    
+    - Don't increment before we dereference!
+    - We need to update the iterator before we delete the item.
+    
+    This code is changed in master, so this fix is for cuttlefish only.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 50957772c3582290331f69ba4a985b1cdf86834d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 9 09:44:20 2013 -0700
+
+    osd: init test_ops_hook
+    
+    CID 1019628 (#1 of 1): Uninitialized pointer field (UNINIT_CTOR)
+    2. uninit_member: Non-static class member "test_ops_hook" is not initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e30a03210c3efb768b1653df5ae58917ef26e579)
+
+commit 17d2745f095e7bb640dece611d7824d370ea3b81
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 9 09:45:51 2013 -0700
+
+    osd: initialize OSDService::next_notif_id
+    
+    CID 1019627 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR)
+    2. uninit_member: Non-static class member "next_notif_id" is not initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 499edd8bfc355c2d590f5fa1ef197d1ea5680351)
+
+commit ffdb7236a994aa20b5f75860b9c81dac0f131f9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 20 09:46:42 2013 -0700
+
+    mon: more fix dout use in sync_requester_abort()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d60534b8f59798feaeeaa17adba2a417d7777cbf)
+
+commit 38ddae04bb974a93f1718c509363f1afbe6b612d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 10 11:48:25 2013 -0700
+
+    mon: fix raw use of *_dout in sync_requester_abort()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8a4ed58e39b287fd8667c62b45848487515bdc80)
diff --git a/doc/changelog/v0.61.6.txt b/doc/changelog/v0.61.6.txt
new file mode 100644
index 000000000..3a1e5bd4b
--- /dev/null
+++ b/doc/changelog/v0.61.6.txt
@@ -0,0 +1,75 @@
+commit 59ddece17e36fef69ecf40e239aeffad33c9db35
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Jul 23 13:52:19 2013 -0700
+
+    v0.61.6
+
+commit 38c3271d3fc415919f0856398bd94eb87a0776b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 23 13:32:12 2013 -0700
+
+    mon/OSDMonitor: fix base case for 7fb3804fb workaround
+    
+    After cluster creation, we have no full map stored and first_committed ==
+    1.  In that case, there is no need for a full map, since we can get there
+    from OSDMap() and the incrementals.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao@inktank.com>
+    (cherry picked from commit e807770784175b05130bba938113fdbf874f152e)
+
+commit f94956cb1a56ff62e01b7ae218a93c4004470ae5
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 23 17:25:13 2013 +0100
+
+    mon: OSDMonitor: work around a full version bug introduced in 7fb3804fb
+    
+    In 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6 we moved the full version
+    stashing logic to the encode_trim_extra() function.  However, we forgot
+    to update the osdmap's 'latest_full' key that should always point to
+    the latest osdmap full version.  This eventually degenerated in a missing
+    full version after a trim.  This patch works around this bug by looking
+    for the latest available full osdmap version in the store and updating
+    'latest_full' to its proper value.
+    
+    Related-to: #5704
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 97462a3213e5e15812c79afc0f54d697b6c498b1)
+
+commit 10e1de45dc8ace793ecf921f884f90c9daa99c48
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 23 16:36:52 2013 +0100
+
+    mon: OSDMonitor: update the osdmap's latest_full with the new full version
+    
+    We used to do this on encode_full(), but since [1] we no longer rely on
+    PaxosService to manage the full maps for us.  And we forgot to write down
+    the latest_full version to the store, leaving it in a truly outdated state.
+    
+    [1] - 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6
+    
+    Fixes: #5704
+    Backport: cuttlefish
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a815547ed3e5ffdbbb96c8c0c1b8d6dd8c62bfba)
+
+commit a0cb40b45c4f2f921a63c2d7bb5a28572381d793
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 18 14:35:19 2013 -0700
+
+    mon: decline to scrub when paxos is not active
+    
+    In f1ce8d7c955a2443111bf7d9e16b4c563d445712 we close a race between scrub
+    and paxos commit completion on the leader.  The fix is nontrivial to
+    backport and probably not worthwhile; just avoid scrubbing at that time
+    for now.
+    
+    Note that the actual fix for this is in commit
+    f1ce8d7c955a2443111bf7d9e16b4c563d445712.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.61.7.txt b/doc/changelog/v0.61.7.txt
new file mode 100644
index 000000000..5836c6a32
--- /dev/null
+++ b/doc/changelog/v0.61.7.txt
@@ -0,0 +1,220 @@
+commit 8f010aff684e820ecc837c25ac77c7a05d7191ff
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jul 24 20:44:12 2013 -0700
+
+    v0.61.7
+
+commit 24a56a9637afd8c64b71d264359c78a25d52be02
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 24 14:46:24 2013 -0700
+
+    ceph-disk: use new get_dev_path helper for list
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Tested-by: Olivier Bonvalet <ob.ceph@daevel.fr>
+    (cherry picked from commit fd1fd664d6102a2a96b27e8ca9933b54ac626ecb)
+
+commit 1f8e4b15eeb132fd7f389318009b19f8f13adbf5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 11 12:59:56 2013 -0700
+
+    ceph-disk: use /sys/block to determine partition device names
+    
+    Not all devices are basename + number; some have intervening character(s),
+    like /dev/cciss/c0d1p2.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ea8fac441141d64ee0d26c5dd2b441f9782d840)
+
+commit 0a08c8226cb3e461301beade9bab2e264d1b960e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 11:01:58 2013 -0700
+
+    ceph-disk: reimplement is_partition() using /sys/block
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5b031e100b40f597752b4917cdbeebb366eb98d7)
+
+commit 056000346db09ea7274a22e57cf4b86a7ea4090e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 11:01:39 2013 -0700
+
+    ceph-disk: use get_dev_name() helper throughout
+    
+    This is more robust than the broken split trick.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3359aaedde838c98d1155611e157fd2da9e8b9f5)
+
+commit f3ee3e690c42769229a6cd9ae8dec43f2aa22ecd
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 10:55:36 2013 -0700
+
+    ceph-disk: refactor list_[all_]partitions
+    
+    Make these methods work in terms of device *names*, not paths, and fix up
+    the only direct list_partitions() caller to do the same.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 35d3f2d84808efda3d2ac868afe03e6959d51c03)
+
+commit be12811b4cb98ff1c2c691c67af7ad3586c436ff
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 10:52:29 2013 -0700
+
+    ceph-disk: add get_dev_name, path helpers
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e0401591e352ea9653e3276d66aebeb41801eeb3)
+
+commit f46dbc462f623e9ab6c00394abb4d890e5d90890
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 16:21:48 2013 -0700
+
+    ceph-disk: handle /dev/foo/bar devices throughout
+    
+    Assume the last component is the unique device name, even if it appears
+    under a subdir of /dev.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cb97338b1186939deecb78e9d949c38c3ef59026)
+
+commit f799dac7bdf7cf0824a177131473cf59ef3c5205
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:54:15 2013 -0700
+
+    ceph-disk: make is_held() smarter about full disks
+    
+    Handle the case where the device is a full disk.  Make the partition
+    check a bit more robust (don't make assumptions about naming aside from
+    the device being a prefix of the partition).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e082f1247fb6ddfb36c4223cbfdf500d6b45c978)
+
+commit 27f31895664fa7f10c1617d486f2a6ece0f97091
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 24 11:55:42 2013 -0700
+
+    mon/OSDMonitor: search for latest full osdmap if record version is missing
+    
+    In 97462a3213e5e15812c79afc0f54d697b6c498b1 we tried to search for a
+    recent full osdmap but were looking at the wrong key.  If full_0 was
+    present we could record that the latest full map was last_committed even
+    though it wasn't present.  This is fixed in 76cd7ac1c, but we need to
+    compensate for when get_version_latest_full() gives us a back version
+    number by repeating the search.
+    
+    Fixes: #5737
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c2131d4047156aa2964581c9dbd93846382a07e7)
+
+commit 5b0967f03efb1be210b52f24f095f023fe1bc539
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Jun 17 14:43:36 2013 +0100
+
+    test: test_store_tool: global init before using LevelDBStore
+    
+    Fixes a segfault
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7a7d3fc8a2ba4a30ef136a32f2903d157b3e19a)
+
+commit 115468c73f121653eec2efc030d5ba998d834e43
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 24 12:00:28 2013 +0100
+
+    mon: OSDMonitor: fix a bug introduced on 97462a32
+    
+    Fixes: #5737
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76cd7ac1c2094b34ad36bea89b2246fa90eb2f6d)
+
+commit 938a639e2cb6abd22c2c588e619c1aae32c6521f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:48:18 2013 -0700
+
+    mon/Paxos: fix pn for uncommitted value during collect/last phase
+    
+    During the collect/last exchange, peers share any uncommitted values
+    with the leader.  They are supposed to also share the pn under which
+    that value was accepted, but were instead using the just-accepted pn
+    value.  This effectively meant that we *always* took the uncommitted
+    value; if there were multiples, which one we accepted depended on what
+    order the LAST messages arrived, not which pn the values were generated
+    under.
+    
+    The specific failure sequence I observed:
+    
+     - collect
+      - learned uncommitted value for 262 from myself
+      - send collect with pn 901
+     - got last with pn 901 (incorrect) for 200 (old) from peer
+      - discard our own value, remember the other
+     - finish collect phase
+      - ignore old uncommitted value
+    
+    Fix this by storing a pending_v and pending_pn value whenever we accept
+    a value.  Use this to send an appropriate pn value in the LAST reply
+    so that the leader can make it's decision about which uncommitted value
+    to accept based on accurate information.  Also use it when we learn
+    the uncommitted value from ourselves.
+    
+    We could probably be more clever about storing less information here,
+    for example by omitting pending_v and clearing pending_pn at the
+    appropriate point, but that would be more fragile.  Similarly, we could
+    store a pn for *every* commit if we wanted to lay some groundwork for
+    having multiple uncommitted proposals in flight, but I don't want to
+    speculate about what is necessary or sufficient for a correct solution
+    there.
+    
+    Fixes: #5698
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20baf662112dd5f560bc3a2d2114b469444c3de8)
+
+commit 18596340f020be1f21bdc9bcc752ae1da4a93a46
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:12:46 2013 -0700
+
+    mon/Paxos: debug ignored uncommitted values
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 19b29788966eb80ed847630090a16a3d1b810969)
+
+commit f598245f1355d7791162c03d90bdd97b013e56f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:11:22 2013 -0700
+
+    mon/Paxos: only learn uncommitted value if it is in the future
+    
+    If an older peer sends an uncommitted value, make sure we only take it
+    if it is in the future, and at least as new as any current uncommitted
+    value.
+    
+    (Prior to the previous patch, peers could send values from long-past
+    rounds.  The pn values are also bogus.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b3253a453c057914753846c77499f98d3845c58e)
+
+commit 732286a28cd8a643593d490a7a84a590d372f78d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 22 14:13:23 2013 -0700
+
+    mon/Paxos: only share uncommitted value if it is next
+    
+    We may have an uncommitted value from our perspective (it is our lc + 1)
+    when the collector has a much larger lc (because we have been out for
+    the last few rounds).  Only share an uncommitted value if it is in fact
+    the next value.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b26b7f6e5e02ac6beb66e3e34e177e6448cf91cf)
diff --git a/doc/changelog/v0.61.8.txt b/doc/changelog/v0.61.8.txt
new file mode 100644
index 000000000..ef01d4551
--- /dev/null
+++ b/doc/changelog/v0.61.8.txt
@@ -0,0 +1,810 @@
+commit d783e33b672ec324eb48d588f956da0c51ff5dac
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Sun Aug 18 23:54:49 2013 -0700
+
+    v0.61.8
+
+commit 21a6e2479133a3debb9ab9057ff9fae70c9eede9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 8 15:12:46 2013 -0700
+
+    RadosClient: shutdown monclient after dropping lock
+
+    Otherwise, the monclient shutdown may deadlock waiting
+    on a context trying to take the RadosClient lock.
+
+    Fixes: #5897
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0aacd10e2557c55021b5be72ddf39b9cea916be4)
+
+commit 64bef4ae4bab28b0b82a1481381b0c68a22fe1a4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Aug 17 09:05:32 2013 -0700
+
+    mon/OSDMonitor: make 'osd pool mksnap ...' not expose uncommitted state
+
+    [This is a backport of d1501938f5d07c067d908501fc5cfe3c857d7281]
+
+    We were returning success without waiting if the pending pool state had
+    the snap.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 411871f6bcc9a4b81140c2e98d13dc123860f6f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 16 10:52:02 2013 -0700
+
+    mon/OSDMonitor: make 'osd pool rmsnap ...' not racy/crashy
+
+    NOTE: This is a manual backport of d90683fdeda15b726dcf0a7cab7006c31e99f14.
+    Due to all kinds of collateral changes in the mon the original patch
+    doesn't cleanly apply.
+
+    Ensure that the snap does in fact exist before we try to remove it.  This
+    avoids a crash where a we get two dup rmsnap requests (due to thrashing, or
+    a reconnect, or something), the committed (p) value does have the snap, but
+    the uncommitted (pp) does not.  This fails the old test such that we try
+    to remove it from pp again, and assert.
+
+    Restructure the flow so that it is easier to distinguish the committed
+    short return from the uncommitted return (which must still wait for the
+    commit).
+
+             0> 2013-07-16 14:21:27.189060 7fdf301e9700 -1 osd/osd_types.cc: In function 'void pg_pool_t::remove_snap(snapid_t)' thread 7fdf301e9700 time 2013-07-16 14:21:27.187095
+        osd/osd_types.cc: 662: FAILED assert(snaps.count(s))
+
+         ceph version 0.66-602-gcd39d8a (cd39d8a6727d81b889869e98f5869e4227b50720)
+         1: (pg_pool_t::remove_snap(snapid_t)+0x6d) [0x7ad6dd]
+         2: (OSDMonitor::prepare_command(MMonCommand*)+0x6407) [0x5c1517]
+         3: (OSDMonitor::prepare_update(PaxosServiceMessage*)+0x1fb) [0x5c41ab]
+         4: (PaxosService::dispatch(PaxosServiceMessage*)+0x937) [0x598c87]
+         5: (Monitor::handle_command(MMonCommand*)+0xe56) [0x56ec36]
+         6: (Monitor::_ms_dispatch(Message*)+0xd1d) [0x5719ad]
+         7: (Monitor::handle_forward(MForward*)+0x821) [0x572831]
+         8: (Monitor::_ms_dispatch(Message*)+0xe44) [0x571ad4]
+         9: (Monitor::ms_dispatch(Message*)+0x32) [0x588c52]
+         10: (DispatchQueue::entry()+0x549) [0x7cf1d9]
+         11: (DispatchQueue::DispatchThread::entry()+0xd) [0x7060fd]
+         12: (()+0x7e9a) [0x7fdf35165e9a]
+         13: (clone()+0x6d) [0x7fdf334fcccd]
+         NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 50698d1862065c8d74338fd08c7e5af66e222490
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+
+    we can hang forever.
+
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+
+    Refs: #5919
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit ef731dfc84a71d3c3262f5cff9a9d33a60255485
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
+
+commit 32631685199f2e47c2ba0ed27d16eff80fa6917d
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 14:27:04 2013 -0700
+
+    mon/Paxos: bootstrap peon too if monmap updates
+
+    If we get a monmap update, the leader bootstraps.  Peons should do the
+    same.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit efe5b67bb700ef6218d9579abf43cc9ecf25ef52)
+
+commit 1ea6b56170fc9e223e7c30635db02fa2ad8f4b4e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 25 13:16:45 2013 -0700
+
+    osd: fix race when queuing recovery ops
+
+    Previously we would sample how many ops to start under the lock, drop it,
+    and start that many.  This is racy because multiple threads can jump in
+    and we start too many ops.  Instead, claim as many slots as we can and
+    release them back later if we do not end up using them.
+
+    Take care to re-wake the work-queue since we are releasing more resources
+    for wq use.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 01d3e094823d716be0b39e15323c2506c6f0cc3b)
+
+commit 4433f9ad8b338b6a55e205602434b307287bfaa3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 16:37:29 2013 -0700
+
+    osd: tolerate racing threads starting recovery ops
+
+    We sample the (max - active) recovery ops to know how many to start, but
+    do not hold the lock over the full duration, such that it is possible to
+    start too many ops.  This isn't problematic except that our condition
+    checks for being == max but not beyond it, and we will continue to start
+    recovery ops when we shouldn't.  Fix this by adjusting the conditional
+    to be <=.
+
+    Reported-by: Stefan Priebe <s.priebe@profihost.ag>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3791a1e55828ba541f9d3e8e3df0da8e79c375f9)
+
+commit 0964d53ef3e8e386e0a1635d2240aefad7b8e2c1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 18:02:32 2013 -0700
+
+    ceph-disk: fix mount options passed to move_mount
+
+    Commit 6cbe0f021f62b3ebd5f68fcc01a12fde6f08cff5 added a mount_options but
+    in certain cases it may be blank.  Fill in with the defaults, just as we
+    do in mount().
+
+    Backport: cuttlefish
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cb50b5a7f1ab2d4e7fdad623a0e7769000755a70)
+
+commit d6be5ed2601b8cf45570afe7ca75ce5aba3f8b4f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 12 10:05:44 2013 -0700
+
+    rgw: fix multi delete
+
+    Fixes: #5931
+    Backport: bobtail, cuttlefish
+
+    Fix a bad check, where we compare the wrong field. Instead of
+    comparing the ret code to 0, we compare the string value to 0
+    which generates implicit casting, hence the crash.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f9f1c48ad799da2b4be0077bf9d61ae116da33d7)
+
+    Conflicts:
+	src/rgw/rgw_rest_s3.cc
+
+commit ecaa46a13837305b9382ab319d43890729c54f1e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jul 23 21:56:09 2013 +0200
+
+    ceph.spec.in: obsolete ceph-libs only on the affected distro
+
+    The ceph-libs package existed only on Redhat based distro,
+    there was e.g. never such a package on SUSE. Therefore: make
+    sure the 'Obsoletes' is only set on these affected distros.
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit 81aa68c309a6d4eaecc54f8d735efde8843fed8c
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Jul 3 11:28:28 2013 -0700
+
+    ceph.spec.in:  Obsolete ceph-libs
+
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 2a34df68bb02d14f6a25bd13dff600a4d629ad05
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Aug 9 14:48:15 2013 -0700
+
+    common: pick_addresses: fix bug with observer class that triggered #5205
+
+    The Observer class we defined to observe conf changes and thus avoid
+    triggering #5205 (as fixed by eb86eebe1ba42f04b46f7c3e3419b83eb6fe7f9a),
+    was returning always the same const static array, which would lead us to
+    always populate the observer's list with an observer for 'public_addr'.
+
+    This would of course become a problem when trying to obtain the observer
+    for 'cluster_add' during md_config_t::set_val() -- thus triggering the
+    same assert as initially reported on #5205.
+
+    Backport: cuttlefish
+    Fixes: #5205
+
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7ed6de9dd7aed59f3c5dd93e012cf080bcc36d8a)
+
+commit 1243c9749ed27850c5d041023780efcdf7b31a68
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Aug 8 16:09:26 2013 -0700
+
+    make sure we are using the mount options
+
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 34831d0989d4bcec4920068b6ee09ab6b3234c91)
+
+commit a9a370be2d8155b696ebe2866febb0571da5740f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Aug 2 11:58:52 2013 -0700
+
+    PG: set !flushed in Reset()
+
+    Otherwise, we might serve a pull before we start_flush in the
+    ReplicaActive constructor.
+
+    Fixes: #5799
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9e7d6d547e0e8a6db6ba611882afa9bf74ea0195)
+
+commit 65bfa4941f983c988837cd010f731966ff53fd19
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 14:02:07 2013 -0700
+
+    osd: make open classes on start optional
+
+    This is cuttlefish; default to the old behavior!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6f996223fb34650771772b88355046746f238cf2)
+
+commit e8253ae5451b1c8e3d7d50199b8db7b2d4c66486
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 13:58:46 2013 -0700
+
+    osd: load all classes on startup
+
+    This avoid creating a wide window between when ceph-osd is started and
+    when a request arrives needing a class and it is loaded.  In particular,
+    upgrading the packages in that window may cause linkage errors (if the
+    class API has changed, for example).
+
+    Fixes: #5752
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c24e652d8c5e693498814ebe38c6adbec079ea36)
+
+    Conflicts:
+	src/osd/ClassHandler.cc
+
+commit 7a1d6d3e727fd8b6947c658e171bf7ec31cd7966
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 15:42:08 2013 -0700
+
+    ceph_test_rados: print version banner on startup
+
+    It is helpful when looking at qa run logs to see what version of the
+    tester is running.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 12c1f1157c7b9513a3d9f716a8ec62fce00d28f5)
+
+commit 86769f05ccc54bfec403bb9ea9a3a951bbcea301
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:08:36 2013 -0700
+
+    ceph_test_rados: add --pool <name> arg
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcfbd0a3ffae6947464d930f636c8b35d1331e9d)
+
+commit b70a9abc5e3ae01204256f414bd7e69d083ed7c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 14:07:02 2013 -0700
+
+    upstart: stop ceph-create-keys when the monitor stops
+
+    This avoids lingering ceph-create-keys tasks.
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a90a2b42db8de134b8ea5d81cab7825fb9ec50b4)
+
+commit 5af48dc7c7e3a0d7f7bc22af58831d58d165e657
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jul 26 13:42:27 2013 -0700
+
+    FileStore: fix fd leak in _check_global_replay_guard
+
+    Bug introduced in f3f92fe21061e21c8b259df5ef283a61782a44db.
+
+    Fixes: #5766
+    Backport: cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c562b72e703f671127d0ea2173f6a6907c825cd1)
+
+commit 17aa2d6d16c77028bae1d2a77903cdfd81efa096
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 25 11:10:53 2013 -0700
+
+    mon/Paxos: share uncommitted value when leader is/was behind
+
+    If the leader has and older lc than we do, and we are sharing states to
+    bring them up to date, we still want to also share our uncommitted value.
+    This particular case was broken by b26b7f6e, which was only contemplating
+    the case where the leader was ahead of us or at the same point as us, but
+    not the case where the leader was behind.  Note that the call to
+    share_state() a few lines up will bring them fully up to date, so
+    after they receive and store_state() for this message they will be at the
+    same lc as we are.
+
+    Fixes: #5750
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 05b6c7e8645081f405c616735238ae89602d3cc6)
+
+commit 09a664e25391dbad9a479bae33904d28231f429d
+Merge: 8f010af b0535fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 25 15:21:31 2013 -0700
+
+    Merge remote-tracking branch 'gh/cuttlefish-next' into cuttlefish
+
+commit b0535fcf854c5042d6b5ff481aabca08026d8f7f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 18:04:40 2013 -0700
+
+    HashIndex: reset attr upon split or merge completion
+
+    A replay of an in progress merge or split might make
+    our counts unreliable.
+
+    Fixes: #5723
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0dc3efdd885377a07987d868af5bb7a38245c90b)
+
+commit 8f73302b4e637ca8b85d68ea7503279faecb57d8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 17:34:25 2013 -0700
+
+    test/filestore/store_test: add test for 5723
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 37a4c4af54879512429bb114285bcb4c7c3488d5)
+
+    Conflicts:
+	src/os/LFNIndex.cc
+	src/test/filestore/store_test.cc
+
+commit 6a7b9e5f0c1d2344209c69ab9992f94221a16468
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 13:51:26 2013 -0700
+
+    FileStore::_collection_rename: fix global replay guard
+
+    If the replay is being replayed, we might have already
+    performed the rename, skip it.  Also, we must set the
+    collection replay guard only after we have done the
+    rename.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 870c474c5348831fcb13797d164f49682918fb30)
+
+commit 7d98651775265896c22bacfc4afcfccbb0128470
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 22 13:46:10 2013 -0700
+
+    PGLog::rewind_divergent_log: unindex only works from tail, index() instead
+
+    Fixes: #5714
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6957dbc75cc2577652b542aa3eae69f03060cb63)
+
+    The original patch covered the same code in PGLog.cc.
+
+    Conflicts:
+
+	src/osd/PGLog.cc
+	src/osd/PG.cc
+
+commit 611a06ae6c9cba468db206dfc82ec883c7a394af
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 18 09:55:43 2013 -0700
+
+    msg/Pipe: do not hold pipe_lock for verify_authorizer()
+
+    We shouldn't hold the pipe_lock while doing the ms_verify_authorizer
+    upcalls.
+
+    Fix by unlocking a bit earlier, and verifying our state is still correct
+    in the failure path.
+
+    This regression was introduced by ecab4bb9513385bd765cca23e4e2fadb7ac4bac2.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 723d691f7a1f53888618dfc311868d1988f61f56)
+
+    Conflicts:
+
+	src/msg/Pipe.cc
+
+commit 45bda482fa8a23f4b80d115e29d6f04cb5e226d6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 14:17:05 2013 -0700
+
+    msg/Pipe: a bit of additional debug output
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 16568d9e1fb8ac0c06ebaa1e1dc1d6a432a5e4d4)
+
+commit 806eab59ad1a32aedb662c51de3b4a1d61fcbb62
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:13:46 2013 -0700
+
+    msg/Pipe: hold pipe_lock during important parts of accept()
+
+    Previously we did not bother with locking for accept() because we were
+    not visible to any other threads.  However, we need to close accepting
+    Pipes from mark_down_all(), which means we need to handle interference.
+
+    Fix up the locking so that we hold pipe_lock when looking at Pipe state
+    and verify that we are still in the ACCEPTING state any time we retake
+    the lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ecab4bb9513385bd765cca23e4e2fadb7ac4bac2)
+
+commit ce6a0b74459996f91a0511a4a7147179bcd47876
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Jul 17 15:23:12 2013 -0700
+
+    msgr: fix a typo/goto-cross from dd4addef2d
+
+    We didn't build or review carefully enough!
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1a84411209b13084b3edb87897d5d678937e3299)
+
+commit 1ed51ad535612d5c444a3cc35a331f5e6a68ce30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:16:23 2013 -0700
+
+    msgr: close accepting_pipes from mark_down_all()
+
+    We need to catch these pipes too, particularly when doing a rebind(),
+    to avoid them leaking through.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 687fe888b32ac9d41595348dfc82111c8dbf2fcb)
+
+commit 2f696f17a413015a3038d5aa76d18fe94f503f03
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:14:25 2013 -0700
+
+    msgr: maintain list of accepting pipes
+
+    New pipes exist in a sort of limbo before we know who the peer is and
+    add them to rank_pipe.  Keep a list of them in accepting_pipes for that
+    period.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd4addef2d5b457cc9a58782fe42af6b13c68b81)
+
+commit 540a6f49d402c1990f0e0fe9f8897dd664e79501
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 16:25:28 2013 -0700
+
+    msgr: adjust nonce on rebind()
+
+    We can have a situation where:
+
+     - we have a pipe to a peer
+     - pipe goes to standby (on peer)
+     - we rebind to a new port
+     - ....
+     - we rebind again to the same old port
+     - we connect to peer
+
+    and get reattached to the ancient pipe from two instances back.  Avoid that
+    by picking a new nonce each time we rebind.
+
+    Add 1,000,000 each time so that the port is still legible in the printed
+    output.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 994e2bf224ab7b7d5b832485ee14de05354d2ddf)
+
+    Conflicts:
+
+	src/msg/Accepter.cc
+
+commit f938a5bf604885ffba65a9b86e19258ca254e58c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:10:23 2013 -0700
+
+    msgr: mark_down_all() after, not before, rebind
+
+    If we are shutting down all old connections and binding to new ports,
+    we want to avoid a sequence like:
+
+     - close all prevoius connections
+     - new connection comes in on old port
+     - rebind to new ports
+     -> connection from old port leaks through
+
+    As a first step, close all connections after we shut down the old
+    accepter and before we start the new one.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 07a0860a1899c7353bb506e33de72fdd22b857dd)
+
+    Conflicts:
+
+	src/msg/SimpleMessenger.cc
+
+commit 07b9ebf4212d53606ce332ff927a2ff68ed26978
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:01:18 2013 -0700
+
+    msg/Pipe: unlock msgr->lock earlier in accept()
+
+    Small cleanup.  Nothing needs msgr->lock for the previously larger
+    window.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad548e72fd94b4a16717abd3b3f1d1be4a3476cf)
+
+commit ae85a0a101d624363fe761c06ecd52d3d38ba4a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 10:09:02 2013 -0700
+
+    msg/Pipe: avoid creating empty out_q entry
+
+    We need to maintain the invariant that all sub queues in out_q are never
+    empty.  Fix discard_requeued_up_to() to avoid creating an entry unless we
+    know it is already present.
+
+    This bug leads to an incorrect reconnect attempt when
+
+     - we accept a pipe (lossless peer)
+     - they send some stuff, maybe
+     - fault
+     - we initiate reconnect, even tho we have nothing queued
+
+    In particular, we shouldn't reconnect because we aren't checking for
+    resets, and the fact that our out_seq is 0 while the peer's might be
+    something else entirely will trigger asserts later.
+
+    This fixes at least one source of #5626, and possibly #5517.
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9f1c27261811733f40acf759a72958c3689c8516)
+
+commit 21e27262edc6f5f090ea8915517ee867e30b9066
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 14:47:05 2013 -0700
+
+    msg/Pipe: assert lock is held in various helpers
+
+    These all require that we hold pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 579d858aabbe5df88543d096ef4dbddcfc023cca)
+
+commit 25f4786ac41869b3f135bd072000634765bb8919
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 14 08:55:52 2013 -0700
+
+    msg/Pipe: be a bit more explicit about encoding outgoing messages
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4282971d47b90484e681ff1a71ae29569dbd1d32)
+
+commit 48105a32605aa59b6970eb89fce4ecc4201e8d04
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 16:21:24 2013 -0700
+
+    msg/Pipe: fix RECONNECT_SEQ behavior
+
+    Calling handle_ack() here has no effect because we have already
+    spliced sent messages back into our out queue.  Instead, pull them out
+    of there and discard.  Add a few assertions along the way.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 495ee108dbb39d63e44cd3d4938a6ec7d11b12e3)
+
+commit 1eab069017ce6b71e4bc2bb9679dbe31b50ae938
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 13:32:38 2013 -0700
+
+    msgr: reaper: make sure pipe has been cleared (under pipe_lock)
+
+    All paths to pipe shutdown should have cleared the con->pipe reference
+    already.  Assert as much.
+
+    Also, do it under pipe_lock!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9586305a2317c7d6bbf31c9cf5b67dc93ccab50d)
+
+commit db06a5092bc45d0479fe492a5d592713a7c53494
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 14:14:02 2013 -0700
+
+    msg/Pipe: goto fail_unlocked on early failures in accept()
+
+    Instead of duplicating an incomplete cleanup sequence (that does not
+    clear_pipe()), goto fail_unlocked and do the cleanup in a generic way.
+    s/rc/r/ while we are here.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ec612a5bda119cea52bbac9b2a49ecf1e83b08e5)
+
+commit 8612e50fd70bfceebd6c291e6cab10d9dfd39e8c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 13:32:07 2013 -0700
+
+    msgr: clear con->pipe inside pipe_lock on mark_down
+
+    We need to do this under protection of the pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit afafb87e8402242d3897069f4b94ba46ffe0c413)
+
+commit 8aafe131acadc22cb069f3d98bba6922ab09c749
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 12:47:11 2013 -0700
+
+    msgr: clear_pipe inside pipe_lock on mark_down_all
+
+    Observed a segfault in rebind -> mark_down_all -> clear_pipe -> put that
+    may have been due to a racing thread clearing the connection_state pointer.
+    Do the clear_pipe() call under the protection of pipe_lock, as we do in
+    all other contexts.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5fc1dabfb3b2cbffdee3214d24d7769d6e440e45)
+
+    Conflicts:
+
+	src/msg/SimpleMessenger.cc
+
+commit 2f7979d1262e9d4899be76963a1620db46b334e8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 19:26:02 2013 -0700
+
+    ReplicatedPG: track temp collection contents, clear during on_change
+
+    We also assert in on_flushed() that the temp collection is actually
+    empty.
+
+    Fixes: #5670
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 47516d9c4b7f023f3a16e166749fa7b1c7b3b24c)
+
+    Conflicts:
+
+	src/osd/ReplicatedPG.cc
+
+commit c7e2945a42541f966017180684dd969389eef3ac
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 19:25:14 2013 -0700
+
+    PG, ReplicatedPG: pass a transaction down to ReplicatedPG::on_change
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9f56a7b8bfcb63cb4fbbc0c9b8ff01de9e518c57)
+
+commit 7ffc65fc4d7d842954cf791c016fd2711f644a9c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 17 15:04:10 2013 -0700
+
+    PG: start flush on primary only after we process the master log
+
+    Once we start serving reads, stray objects must have already
+    been removed.  Therefore, we have to flush all operations
+    up to the transaction writing out the authoritative log.
+    On replicas, we flush in Stray() if we will not eventually
+    be activated and in ReplicaActive if we are in the acting
+    set.  This way a replica won't serve a replica read until
+    the store is consistent.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b41f1ba48563d1d3fd17c2f62d10103b5d63f305)
+
+commit 850da0890da5df7e670df9268afe420d0c906c38
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 17 12:51:19 2013 -0700
+
+    ReplicatedPG: replace clean_up_local with a debug check
+
+    Stray objects should have been cleaned up in the merge_log
+    transactions.  Only on the primary have those operations
+    necessarily been flushed at activate().
+
+    Fixes: 5084
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 278c7b59228f614addf830cb0afff4988c9bc8cb)
+
+commit 95b1b5da439f1b7e2fb1886aaeec2d61532183f0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 10:12:17 2013 -0700
+
+    FileStore: add global replay guard for split, collection_rename
+
+    In the event of a split or collection rename, we need to ensure that
+    we don't replay any operations on objects within those collections
+    prior to that point.  Thus, we mark a global replay guard on the
+    collection after doing a syncfs and make sure to check that in
+    _check_replay_guard() for all object operations.
+
+    Fixes: #5154
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3f92fe21061e21c8b259df5ef283a61782a44db)
+
+    Conflicts:
+
+	src/os/FileStore.cc
+
+commit d92a43d8ff0123b234e47a94c2ce73fcaae7f625
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 15 13:44:20 2013 -0700
+
+    OSD: add config option for peering_wq batch size
+
+    Large peering_wq batch sizes may excessively delay
+    peering messages resulting in unreasonably long
+    peering.  This may speed up peering.
+
+    Backport: cuttlefish
+    Related: #5084
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 39e5a2a406b77fa82e9a78c267b679d49927e3c3)
diff --git a/doc/changelog/v0.61.9.txt b/doc/changelog/v0.61.9.txt
new file mode 100644
index 000000000..fe2a7e733
--- /dev/null
+++ b/doc/changelog/v0.61.9.txt
@@ -0,0 +1,571 @@
+commit 7440dcd135750839fa0f00263f80722ff6f51e90
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Oct 16 18:57:51 2013 +0000
+
+    v0.61.9
+
+commit fcf5f117a9111c2d88b8fa5d00c975a8e377df7e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:20:48 2013 -0700
+
+    rgw: fix authenticated users acl group check
+    
+    Fixes: #6553
+    Backport: bobtail, cuttlefish, dumpling
+    Authenticated users group acl bit was not working correctly. Check to
+    test whether user is anonymous was wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bebbd6cb7b71697b34b8f27652cabdc40c97a33b)
+
+commit 991ed515480114c476cd3c4d761f256d1708fb39
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:55:07 2013 -0700
+
+    rgw: change default log level
+    
+    Fixes: #6554
+    Backport: cuttlefish, dumpling
+    Default log level was just too high, bring it down a bit.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8d7dbf85472cfca9268d81ecf057ea078cf345b3)
+
+commit ebb9b0cb7e4ab60fdbbc410ecfb35e51cf11434d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 6 09:21:47 2013 -0700
+
+    mds: do not allow GLAZYIO in mix->sync state
+    
+    GLAZYIO is not allowed in SYNC, so we cannot allow it in the preceding
+    gather state.
+    
+    I verified the other GLAZYIO rules look ok.  We should make a validater
+    to confirm that no gather state includes caps that its target state
+    does not... or at least assert as much in eval_gather().
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b88938e5a646fbf175a7135e872bcb2d1afafbb8)
+
+commit 33da08f683d40f33061cefa0cf145f3ff21ea089
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 10:36:39 2013 +0800
+
+    osdc/ObjectCacher: finish contexts after dropping object reference
+    
+    The context to finish can be class C_Client_PutInode, which may drop
+    inode's last reference. So we should first drop object's reference,
+    then finish contexts.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit b66ac77fa7aa3ff37804918c4308a348f239af09)
+
+commit 346b43d80f728e6b389208ccd8054d96b76b093c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 22:04:09 2013 -0700
+
+    mds: fix filelock eval_gather
+    
+    Broken by a08d62045657713bf0a5372bf14136082ec3b17e
+    
+    Reported-by: Yan, Zheng <yan.zheng@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8300d0afb5154d4d13536abdcf47bd5cc8ce810)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit ffdc7fce132b3b98463b4222d2c51ccef6b94d82
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 21:38:56 2013 -0700
+
+    mds: do not double-queue file recovery in eval_gather
+    
+    This fixes a specific case of double-queuing seen in #4832:
+    
+     - client goes stale, inode marked NEEDSRECOVER
+     - eval does sync, queued, -> RECOVERING
+     - client resumes
+     - client goes stale (again), inode marked NEEDSRECOVER
+     - eval_gather queues *again*
+    
+    Note that a cursory look at the recovery code makes me think this needs
+    a much more serious overhaul.  In particular, I don't think we should
+    be triggering recovery when transitioning *from* a stable state, but
+    explicitly when we are flagged, or when gathering.  We should probably
+    also hold a wrlock over the recovery period and remove the force_wrlock
+    kludge from the final size check.  Opened ticket #5268.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a08d62045657713bf0a5372bf14136082ec3b17e)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 60033c31381d36cbbc6c873d7055cbe735f5deb2
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Oct 8 11:58:57 2013 -0700
+
+    Go back to $PWD in fsstress.sh if compiling from source.
+    
+    Although fsstress was being called with a static path the directory
+    it was writing to was in the current directory so doing a cd to the
+    source directory that is made in /tmp and then removing it later
+    caused it to be unable to write the files in a non-existent dir.
+    
+    This change gets the current path first and cd's back into it after
+    it is done compiling fsstress.
+    
+    Issue #6479.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit eb06f3738851d27914704821897ed80104c4c29c
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 09:53:12 2013 -0700
+
+    ceph.spec.in:  radosgw package doesn't require mod_fcgi
+    
+    Fixes #5702
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 5a426a1f1f34d3f5a510009cc3f3b219d3cbc74b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 15:53:42 2013 -0700
+
+    crush: invalidate rmap on create (and thus decode)
+    
+    If we have an existing CrushWrapper object and decode from a bufferlist,
+    reset build_rmaps so that they get rebuilt.
+    
+    Remove the build_rmaps() all in decode that was useless on a redecode
+    (because have_rmaps == true in that case and it did nothing).
+    
+    Fixes: #6442
+    Backport: dumpling, maybe cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9b7a2ae329b6a511064dd3d6e549ba61f52cfd21)
+
+commit 6f342872cdd211e24deb19f5e00380494514c437
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 19:04:23 2013 +0200
+
+    osd: change warn_interval_multiplier to uint32_t
+    
+    to prevent overflow in OpTracker::check_ops_in_flight when
+    multiplying warn_interval_multiplier *= 2
+    
+    Backport: cuttlefish, dumpling
+    
+    http://tracker.ceph.com/issues/6370 fixes #6370
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1bce1f009bffd3e28025a08775fec189907a81db)
+
+commit be2907ef85a31c2be8be7446fe71f5d2e1410ec0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 22:30:12 2013 -0700
+
+    rgw: don't call list::size() in ObjectCache
+    
+    Fixes: #6286
+    Use an external counter instead of calling list::size()
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31e3a51e933429d286104fe077e98ea883437ad6)
+
+commit bbfbb097e2f9efbf4f7ec997c70befa20c79d27c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 10 12:18:55 2013 -0700
+
+    rgw: drain pending requests before completing write
+    
+    Fixes: #6268
+    When doing aio write of objects (either regular or multipart parts) we
+    need to drain pending aio requests. Otherwise if gateway goes down then
+    object might end up corrupted.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit b16f812362ccb1d9bdd4900d155e248d695ef0d7
+Merge: 97a97c3 5f16ea6
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Sep 7 13:32:40 2013 -0700
+
+    Merge pull request #573 from dalgaaf/fix/da-cuttlefish-fixes-and-cherry-picks
+    
+    Cherry-pick some smaller changes from master to cuttlefish
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 5f16ea62cee4fad9be6e44f3562da31908303ae5
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 20:32:40 2013 +0200
+
+    tools/ceph.cc: add missig 'ceph osd lspools' command to help
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit 59f02ecf0b91a2248d8b7b75dc27b517f04ac292
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 11:30:15 2013 +0200
+
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b5137baf651eaaa9f67e3864509e437f9d5c3d5a)
+
+commit c25770c39ae006ab4ad14a5d75bf7a2dffe0279e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:34:54 2013 +0200
+
+    init-radosgw*: add all sections to usage output
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a0c5095be3640e98d5541920c19387bf3764a350)
+
+commit 1a8347e0d1cafc38259adc1f1a6154fa0d48f1d2
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:33:23 2013 +0200
+
+    init-radosgw*: add status
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 385457f8d871238a896229d0c2cbb25646969f6a)
+
+commit b1c2aa2c4a8c0266a01903eab5539e7929ea0431
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:21:30 2013 +0200
+
+    fix init-radosgw* to use the same indentation
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b4d4e92ed2deae435a24b36d086c1a73e5997855)
+
+commit 794ed1faec7ced23b5b46d114f5320d718c9e9fb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sun Jul 28 23:25:58 2013 +0200
+
+    ceph_authtool.cc: update help/usage text
+    
+    Added implemented but not listed commands to the help/usage text:
+    * -g  shortcut for --gen-key
+    * -a  shortcut for --add-key
+    * -u/--set-uid to set auid
+    * --gen-print-key
+    * --import-keyring
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9a9a0ef3f9f39909eaeb95eb99db4711a2425af5)
+
+commit 97a97c3c554f689dd3f987e63eaa2b9c5ec1dd0a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 19:46:43 2013 -0700
+
+    rgw: check object name after rebuilding it in S3 POST
+    
+    Fixes: #6088
+    Backport: bobtail, cuttlefish, dumpling
+    
+    When posting an object it is possible to provide a key
+    name that refers to the original filename, however we
+    need to verify that in the end we don't end up with an
+    empty object name.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c8ec532fadc0df36e4b265fe20a2ff3e35319744)
+
+commit 7a0bd5bc2c6e5464f70b19154834448ac1e4c369
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 13:29:32 2013 -0700
+
+    ceph.spec.in:  remove trailing paren in previous commit
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit f1507d23707e7929f7a55fe2ea9418dcc715d8b2
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 11:07:16 2013 -0700
+
+    ceph.spec.in:  Don't invoke debug_package macro on centos.
+    
+    If the redhat-rpm-config package is installed, the debuginfo rpms will
+    be built by default.   The build will fail when the package installed
+    and the specfile also invokes the macro.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 65a10862feec199d14f17627d0c42fa7c85766fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 08:59:21 2013 -0700
+
+    osd: get initial full map after a map gap
+    
+    If there is a gap in our map history, get the full range of maps that
+    the mon has.  Make sure the first one is a full map.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit a6cd9fea50a4bd7048a222617a2bfe0680f7a969)
+
+commit aceef04f7fd56935e691c7deb05f25ace653bb76
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 08:55:38 2013 -0700
+
+    osd: fix off-by-one in map gap logic
+    
+    If we have map 250, and monitor's first is 251, but sends 260, we can
+    request the intervening range.
+    
+    Fixes: #5784
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e24b50225c841a650d9303041bbe811e04bdd668)
+
+commit cdbfd66249cdf91c02a88af5df5a6517688a78df
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 22 16:00:07 2013 -0700
+
+    OSD: tolerate holes in stored maps
+    
+    We may have holes in stored maps during init_splits_between
+    and advance_pg.  In either case, we should simply skip the
+    missing maps.
+    
+    Fixes: #5677
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6951d2345a5d837c3b14103bd4d8f5ee4407c937)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 234d68c68028fcf9c2665cb9f45b9b76556241ba
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 22:39:09 2013 -0700
+
+    ceph-disk: partprobe after creating journal partition
+    
+    At least one user reports that a partprobe is needed after creating the
+    journal partition.  It is not clear why sgdisk is not doing it, but this
+    fixes ceph-disk for them, and should be harmless for other users.
+    
+    Fixes: #5599
+    Tested-by: lurbs in #ceph
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2af59d5e81c5e3e3d7cfc50d9330d7364659c5eb)
+
+commit cf2f31ac23b6eb43a81a1c8157907b9cae4d58a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:48:06 2013 -0700
+
+    osdc/ObjectCacher: do not merge rx buffers
+    
+    We do not try to merge rx buffers currently.  Make that explicit and
+    documented in the code that it is not supported.  (Otherwise the
+    last_read_tid values will get lost and read results won't get applied
+    to the cache properly.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c50c446152ab0e571ae5508edb4ad7c7614c310)
+
+commit 02da55757a9fb53df4746db5dd14724e77da95b6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:47:18 2013 -0700
+
+    osdc/ObjectCacher: match reads with their original rx buffers
+    
+    Consider a sequence like:
+    
+     1- start read on 100~200
+           100~200 state rx
+     2- truncate to 200
+           100~100 state rx
+     3- start read on 200~200
+           100~100 state rx
+           200~200 state rx
+     4- get 100~200 read result
+    
+    Currently this makes us crash on
+    
+    osdc/ObjectCacher.cc: 738: FAILED assert(bh->length() <= start+(loff_t)length-opos)
+    
+    when processing the second 200~200 bufferhead (it is too big).  The
+    larger issue, though, is that we should not be looking at this data at
+    all; it has been truncated away.
+    
+    Fix this by marking each rx buffer with the read request that is sent to
+    fill it, and only fill it from that read request.  Then the first reply
+    will fill the first 100~100 extend but not touch the other extent; the
+    second read will do that.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b59f930ae147767eb4c9ff18c3821f6936a83227)
+
+commit 43e7ad989dcb4deb18b32ec31f76c8755354d2a6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 15:54:48 2013 -0700
+
+    mon/Paxos: fix another uncommitted value corner case
+    
+    It is possible that we begin the paxos recovery with an uncommitted
+    value for, say, commit 100.  During last/collect we discover 100 has been
+    committed already.  But also, another node provides an uncommitted value
+    for 101 with the same pn.  Currently, we refuse to learn it, because the
+    pn is not strictly > than our current uncommitted pn... even though it is
+    the next last_committed+1 value that we need.
+    
+    There are two possible fixes here:
+    
+     - make this a >= as we can accept newer values from the same pn.
+     - discard our uncommitted value metadata when we commit the value.
+    
+    Let's do both!
+    
+    Fixes: #6090
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe5010380a3a18ca85f39403e8032de1dddbe905)
+
+commit 2de1515289f49f2e388448506f4788db56d0e25a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:35 2013 -0700
+
+    os: make readdir_r buffers larger
+    
+    PATH_MAX isn't quite big enough.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 99a2ff7da99f8cf70976f05d4fe7aa28dd7afae5)
+
+commit af9818c486484c7617c07f26beaded8a3bc88043
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:08 2013 -0700
+
+    os: fix readdir_r buffer size
+    
+    The buffer needs to be big or else we're walk all over the stack.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2df66d9fa214e90eb5141df4d5755b57e8ba9413)
+    
+    Conflicts:
+    
+    	src/os/BtrfsFileStoreBackend.cc
+
+commit cce1d1f9cd8b034deee29d8566780763beb0155f
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 23 08:56:07 2013 -0400
+
+    ceph-disk: specify the filetype when mounting
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f040020fb2a7801ebbed23439159755ff8a3edbd)
+
+commit c25e7da57d704d4a8db59a2e97fb687968520c69
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Aug 22 19:44:40 2013 -0700
+
+    QA: Compile fsstress if missing on machine.
+    
+    Some distro's have a lack of ltp-kernel packages and all we need is
+    fstress. This just modified the shell script to download/compile
+    fstress from source and copy it to the right location if it doesn't
+    currently exist where it is expected. It is a very small/quick
+    compile and currently only SLES and debian do not have it already.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+
+commit c807f27c391d336a7223fcfdd3daad9bb374a3dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 5 12:52:44 2013 -0700
+
+    mds: fix locking, use-after-free/race in handle_accept
+    
+    We need to hold mds_lock here.
+    
+    Normally the con also holds a reference, but an ill-timed connection reset
+    could drop it.
+    
+    Fixes: #5883
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a0929955cb84fb8cfdeb551d6863e4955b8e2a71)
+
+commit bd71192eaa6f884e879b1711e5937b1e3609d86d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 10:14:59 2013 -0700
+
+    .gitignore: ignore test-driver
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit edf2c3449ec96d91d3d7ad01c50f7a79b7b2f7cc)
+    
+    Conflicts:
+    
+    	.gitignore
+
+commit bc997ebea3263c2bc7df83661ae3a966470ba35e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:42:49 2013 -0700
+
+    fuse: fix warning when compiled against old fuse versions
+    
+    client/fuse_ll.cc: In function 'void invalidate_cb(void*, vinodeno_t, int64_t, int64_t)':
+    warning: client/fuse_ll.cc:540: unused variable 'fino'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9833e9dabe010e538cb98c51d79b6df58ce28f9e)
+
+commit 9cb2c2eb3627b52c3413b39b45e7fb7e0e9a074c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:40:34 2013 -0700
+
+    json_spirit: remove unused typedef
+    
+    In file included from json_spirit/json_spirit_writer.cpp:7:0:
+    json_spirit/json_spirit_writer_template.h: In function 'String_type json_spirit::non_printable_to_string(unsigned int)':
+    json_spirit/json_spirit_writer_template.h:37:50: warning: typedef 'Char_type' locally defined but not used [-Wunused-local-typedefs]
+             typedef typename String_type::value_type Char_type;
+    
+    (Also, ha ha, this file uses \r\n.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6abae35a3952e5b513895267711fea63ff3bad09)
+
+commit d774559f118d26cd15ecf1a49468ce1a3d260efc
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:31:41 2013 -0700
+
+    gtest: add build-aux/test-driver to .gitignore
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9cdd19d1cd88b84e8a867f5ab85cb51fdc6f8e4)
+
+commit 1a2d9edde0311b51d3d68b87c20dea3061b2395b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+    
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+    
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+    
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
diff --git a/doc/changelog/v0.67.1.txt b/doc/changelog/v0.67.1.txt
new file mode 100644
index 000000000..9b9405f82
--- /dev/null
+++ b/doc/changelog/v0.67.1.txt
@@ -0,0 +1,142 @@
+commit e23b817ad0cf1ea19c0a7b7c9999b30bed37d533
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Aug 16 19:39:41 2013 -0700
+
+    v0.67.1
+
+commit 1aa01910957e967e87e46b86f1e67844148703e3
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Aug 15 17:10:56 2013 -0700
+
+    ceph.in: --admin-daemon was not returning EINVAL on bad command
+    
+    Fix by restructuring code to hoist common code and have only one
+    place where admin_socket is actually called.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 266460e97ec9ef9711e9eaa4bd954f3188d8da69)
+
+commit d290a91525c3ebc6941dae2bb55f0bfbf120cb60
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:37:07 2013 -0700
+
+    mon: use str_join instead of std::copy
+    
+    The std::copy method leaves a trailing separator.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 35565ee64e41d7fddc7849c6006692c78227132c)
+
+commit b99921746ef2e1e15777c748a15e929c72888db1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:36:57 2013 -0700
+
+    config: fix stringification of config values
+    
+    The std::copy construct leaves a trailing separator character, which breaks
+    parsing for booleans (among other things) and probably mangles everything
+    else too.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit fc23cfe3fe567b30413d8af0c614a32fec238939)
+
+commit d212bba6bd0d7d234097122988e4d973064b5645
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:36:49 2013 -0700
+
+    common: add str_join helper
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit ce3a0944d9b47f7b178fe7775c9d105305b238e0)
+
+commit a99fef9189086f5dd6ddacaecf967619dc5fe407
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 14 15:50:59 2013 -0700
+
+    rados.py: fix Rados() unicode checking
+    
+    Check new parameters and check that rados_id is not None again to
+    catch the empty string.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 4422f21a6586467a63ce6841552d0f60aa849cf1)
+
+commit dd0df583e2661444287a36acc240a8ac0ec381e8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 14 15:28:19 2013 -0700
+
+    rados.py: fix Rados() backwards compatibility
+    
+    Previously it had no name parameter, so the default will be used by
+    old clients. However, if an old client set rados_id, a new check that
+    both rados_id and name are set would result in an error. Fix this by
+    only applying the default names after the check, and add tests of this
+    behavior.
+    
+    This was introduced in 783b7ec847c7f987ac1814c9c41c91921cac4eba,
+    so it does not affect cuttlefish.
+    
+    Fixes: #5970
+    Reported-by: Michael Morgan <mmorgan@dca.net>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 34da9cbc33205623cf64aee1989f53dfb2c5bddd)
+
+commit b9d1bf51610159a88ad257f29a81691e6b178e17
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+    
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+    
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+    
+    we can hang forever.
+    
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+    
+    Refs: #5919
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit 29ae033b435581a64fc92a26106b4ea1c3dfc0b1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+    
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+    
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+    
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+    
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
diff --git a/doc/changelog/v0.67.10.txt b/doc/changelog/v0.67.10.txt
new file mode 100644
index 000000000..165c8e0bb
--- /dev/null
+++ b/doc/changelog/v0.67.10.txt
@@ -0,0 +1,669 @@
+commit 9d446bd416c52cd785ccf048ca67737ceafcdd7f (tag: refs/tags/v0.67.10)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Aug 11 05:30:10 2014 -0700
+
+    0.67.10
+
+commit a86e497a343b24425a1237e65ec2776398d07a95
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Jul 14 14:27:17 2014 +0800
+
+    Add rbdcache max dirty object option
+    
+    Librbd will calculate max dirty object according to rbd_cache_max_size, it
+    doesn't suitable for every case. If user set image order 24, the calculating
+    result is too small for reality. It will increase the overhead of trim call
+    which is called each read/write op.
+    
+    Now we make it as option for tunning, by default this value is calculated.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3c7229a2fea98b30627878c86b1410c8eef2b5d7)
+
+commit d02381316ba96746a00b2b6f8c5c665179fe7a79
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 4 23:22:18 2014 +0200
+
+    librbd/internal.cc: check earlier for null pointer
+    
+    Fix potential null ponter deref, move check for 'order != NULL'
+    to the beginning of the function to prevent a) deref in ldout() call
+    and b) to leave function as early as possible if check fails.
+    
+    [src/librbd/internal.cc:843] -> [src/librbd/internal.cc:865]: (warning)
+     Possible null pointer dereference: order - otherwise it is redundant
+     to check it against null.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3ee3e66a9520a5fcafa7d8c632586642f7bdbd29)
+
+commit dda98b9afb5d74c8c5f101b792daca9ca344fe87
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:47:24 2014 -0700
+
+    librbd: add an interface to invalidate cached data
+    
+    This is useful for qemu to guarantee live migration with caching is
+    safe, by invalidating the cache on the destination before starting it.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5d340d26dd70192eb0e4f3f240e3433fb9a24154)
+
+commit d93579514db74eed61580a6166c2787304a8a9fb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:43:35 2014 -0700
+
+    librbd: check return code and error out if invalidate_cache fails
+    
+    This will only happen when shrinking or rolling back an image is done
+    while other I/O is in flight to the same ImageCtx. This is unsafe, so
+    return an error before performing the resize or rollback.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e08b8b66c77be3a3d7f79d91c20b1619571149ee)
+
+commit 617914209694f67150264358da888c1ca9ff42e1
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed May 21 18:12:22 2014 +0800
+
+    Avoid extra check for clean object
+    
+    We needn't to check clean object via buffer state, skip the clean object.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit f51e33bd9c5a8e1cfc7065b30785696dc45918bc)
+
+commit d474443b12b978dbe267e63cd8e34e8c835e4f6c
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri Mar 28 18:10:43 2014 -0700
+
+    rbd.cc: yes, cover formatted output as well.  sigh.
+    
+    Fixes: #7577
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit bd6e35c1b171e46cc3e026c59b076b73440a8502)
+
+commit 051d1b4e56b37b2293b54d8cdb58a974684c8464
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Mar 25 17:09:48 2014 -0700
+
+    rbd.cc: tolerate lack of NUL-termination on block_name_prefix
+    
+    Fixes: #7577
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fd76fec589be13a4a6362ef388929d3e3d1d21f6)
+
+commit 61e2219dd07ebb856a61f54e8cd992dc1e16f5d9
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Wed Jan 29 16:12:01 2014 +0200
+
+    rbd: don't forget to call close_image() if remove_child() fails
+    
+    close_image() among other things unregisters a watcher that's been
+    registered by open_image().  Even though it'll timeout in 30 or so
+    seconds, it's not nice now that we check for watchers before starting
+    the removal process.
+    
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit 4ebc32f37a4860bdc676491bf8b042c18fd619cf)
+
+commit 2c6c23cee7ff2a8f56f3986051d43824b7bd3526
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 9 13:23:06 2014 -0700
+
+    os/FileStore: dump open fds before asserting
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e8de1792731cf30f2744ab0659d036adc0565a3)
+
+commit 3663233a63df3a1fb1584d9800e1c6a8feac2fe0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 18 16:43:48 2014 -0800
+
+    rgw: return error if accessing object in non-existent bucket
+    
+    Fixes: #7064
+    Instead of trying to access the object, which is impossible as we don't
+    even have a proper bucket info. Up until now we ended up creating an
+    empty pool and eventually returning ENOENT, this fix catches the issue
+    earlier in the process.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3ed68eb9fac9b3d0bf9111867d609f9ea08fb202)
+
+commit 0a1cea58641c4584b85165755145d8d0288f4f22
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 6 17:04:02 2014 -0700
+
+    os/FileStore: force any new xattr into omap on E2BIG
+    
+    If we have a huge xattr (or many little ones), the _fgetattrs() for the
+    inline_set will fail with E2BIG.  The conditions later where we decide
+    whether to clean up the old xattr will then also fail.  Will *will* put
+    the xattr in omap, but the non-omap version isn't cleaned up.
+    
+    Fix this by setting a flag if we get E2BIG that the inline_set is known
+    to be incomplete.  In that case, take the conservative step of assuming
+    the xattr might be present and chain_fremovexattr().  Ignore any error
+    because it might not be there.
+    
+    This is clearly harmless in the general case because it won't be there.  If
+    it is, we will hopefully remove enough xattrs that the E2BIG condition
+    will go away (usually by removing some really big chained xattr).
+    
+    See bug #7779.
+    
+    This is a backport of 26750fcfe8d766874513e57981565adde2e6d8c7.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 64d5c406995bedbb6a4bc9c851f5d25fe94749ee
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 23 14:58:54 2014 -0700
+
+    rgw: calc md5 and compare if user provided appropriate header
+    
+    Fixes: #8436
+    Backport: firefly
+    
+    This was broken in ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782. The fix
+    resurrects and old check that was dropped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9c56c86bdac6bcb8e76c3f04e7d393e4eaadd721)
+
+commit ebecd80d3a082ac87e7cd6a63cbd00259c5a8baf
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:07:12 2014 -0700
+
+    rgw: calculate user manifest
+    
+    Fixes: #8169
+    Backport: firefly
+    We didn't calculate the user manifest's object etag at all. The etag
+    needs to be the md5 of the contantenation of all the parts' etags.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+
+commit 1f3f2982224f68109cce8b5564d99977b22c8f60
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 29 15:25:47 2014 -0700
+
+    rgw: fix crash in swift CORS preflight request
+    
+    Fixes: #8586
+    
+    This fixes error handling, in accordance with commit 6af5a537 that fixed
+    the same issue for the S3 case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 18ea2a869791b4894f93fdafde140285f2e4fb65)
+
+commit 6e4921500aeb256da0fa20da6ee47fb7d4e171f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 11:53:16 2014 -0700
+
+    cls_rgw: fix object name of objects removed on object creation
+    
+    Fixes: #8972
+    Backport: firefly, dumpling
+    
+    Reported-by: Patrycja Szabłowska <szablowska.patrycja@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0f8929a68aed9bc3e50cf15765143a9c55826cd2)
+
+commit 4aa2b87385e94be0147401918c332d526029e481
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 2 10:38:43 2014 -0700
+
+    qa/workunits/rest/test.py: make osd create test idempotent
+    
+    Avoid possibility that we create multiple OSDs do to retries by passing in
+    the optional uuid arg.  (A stray osd id will make the osd tell tests a
+    few lines down fail.)
+    
+    Fixes: #8728
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bb3e1c92b6682ed39968dc5085b69c117f43cbb0)
+
+commit 935e75a850b3363536cc09b68f45f2a497da4bb4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Apr 10 15:14:19 2014 +0100
+
+    mon: Monitor: suicide on start if mon has been removed from monmap
+    
+    If the monitor has been marked as having been part of an existing quorum
+    and is no longer in the monmap, then it is safe to assume the monitor
+    was removed from the monmap.  In that event, do not allow the monitor
+    to start, as it will try to find its way into the quorum again (and
+    someone clearly stated they don't really want them there), unless
+    'mon force quorum join' is specified.
+    
+    Fixes: 6789
+    Backport: dumpling, emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 86b85947a2148c2e73886c1a7edd09093966ada0)
+    
+    Conflicts:
+    	src/common/config_opts.h
+
+commit 4aca6c0d7540c2bf5fb54df2d70c4649a9b94100
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 16:05:58 2014 -0700
+
+    utf8: export encode_utf8() and decode_utf8()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 49fc68cf8c3122c878ea9503c9c74d7046bc9c6f)
+
+commit ffa1f5200061c48de6315b5e7bd335ab5f718a6f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 18 14:52:48 2014 -0700
+
+    rgw: dump prefix unconditionally
+    
+    As part of issue #8858, and to be more in line with S3, dump the Prefix
+    field when listing bucket even if bucket is empty.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7209c11251d42227608bc54cc69232ef62ffe80)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit 6df17a92283da9933bf5a3fca1278b728c79b8ee
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 15:48:26 2014 -0700
+
+    rgw: list extra objects to set truncation flag correctly
+    
+    Otherwise we end up returning wrong truncated value, and no data on the
+    next iteration.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit dc417e477d4ad262885c6b5f5987cf06d63b159d)
+
+commit 77c20879d02c9793241da508b5ad59f8b927e9f7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:45:44 2014 -0700
+
+    rgw: account common prefixes for MaxKeys in bucket listing
+    
+    To be more in line with the S3 api. Beforehand we didn't account the
+    common prefixes towards the MaxKeys (a single common prefix counts as a
+    single key). Also need to adjust the marker now if it is pointing at a
+    common prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 82d2d612e700f94a0bb2d9fb7555abf327be379b)
+
+commit 7068d8595ff462323dedd9b56bce28a76405585a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:24:51 2014 -0700
+
+    rgw: add NextMarker param for bucket listing
+    
+    Partially fixes #8858.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb)
+
+commit 9604425b86f5839a109faa1f396b0d114e9b9391
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 15:21:09 2014 -0700
+
+    rgw: improve delmited listing of bucket
+    
+    If found a prefix, calculate a string greater than that so that next
+    request we can skip to that. This is still not the most efficient way to
+    do it. It'll be better to push it down to the objclass, but that'll
+    require a much bigger change.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)
+
+commit b287c677ba1a5573e66c8a287f6a1f420c3ba46f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 12:23:31 2014 -0700
+
+    rgw: don't try to wait for pending if list is empty
+    
+    Fixes: #8846
+    Backport: firefly, dumpling
+    
+    This was broken at ea68b9372319fd0bab40856db26528d36359102e. We ended
+    up calling wait_pending_front() when pending list was empty.
+    This commit also moves the need_to_wait check to a different place,
+    where we actually throttle (and not just drain completed IOs).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f9f2417d7db01ecf2425039539997901615816a9)
+
+commit 4b13caf5be33d4644811d387d1350f2f11ac5409
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Apr 24 12:55:26 2014 -0700
+
+    Use new git mirror for qemu-iotests
+    
+    Fixes: 8191
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit ddf37d903f826f3e153d8009c716780453b68b05)
+
+commit 3f251024bf056f5a8c22f29333b3a81589431424
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Wed Apr 23 13:20:14 2014 -0700
+
+    Support latest qemu iotest code
+    
+    Modified qemu-iotests workunit script to check for versions
+    that use the latest qemu (currently only Trusty).  Limit the
+    tests to those that are applicable to rbd.
+    
+    Fixes: 7882
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit 606e725eb5204e76e602d26ffd113e40af2ee812)
+
+commit d06137841f9911ce4dc0d9ab9be73a131ede879d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Mar 31 14:53:31 2014 -0700
+
+    librbd: skip zeroes when copying an image
+    
+    This is the simple coarse-grained solution, but it works well in
+    common cases like a small base image resized with a bunch of empty
+    space at the end. Finer-grained sparseness can be copied by using rbd
+    {export,import}-diff.
+    
+    Fixes: #6257
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 824da2029613a6f4b380b6b2f16a0bd0903f7e3c)
+
+commit 8649cbbc96a4de9de169b0203f35e0ac6c36a2ef
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Jul 1 15:19:21 2014 -0700
+
+    Revert "qa/workunits/suites/fsx.sh: don't use zero range"
+    
+    This reverts commit 583e6e3ef7f28bf34fe038e8a2391f9325a69adf.
+    
+    We're using a different fsx source, which doesn't support the
+    same options as our git-based one does.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 583e6e3ef7f28bf34fe038e8a2391f9325a69adf
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 07:05:04 2014 -0700
+
+    qa/workunits/suites/fsx.sh: don't use zero range
+    
+    Zero range is not supported by cephfs.
+    
+    Fixes: #8542
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2dec8a810060f65d022c06e82090b4aa5ccec0cb)
+
+commit 49c0b46153f96a94407ef2487b2bd9525dfaa1e6
+Merge: 3c63601 4ca12b4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 30 19:19:24 2014 +0200
+
+    Merge pull request #2014 from ceph/wip-scrub-dumpling
+    
+    osd: scrub priority updates for dumpling
+    
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+
+commit 3c63601888c5c825f33ffc6797474eb8dc6935a4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 16 11:48:24 2014 -0700
+
+    rgw: allocate enough space for bucket instance id
+    
+    Fixes: #8608
+    Backport: dumpling, firefly
+    Bucket instance id is a concatenation of zone name, rados instance id,
+    and a running counter. We need to allocate enough space to account zone
+    name length.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d2e86a66ca55685e04ffbfaa58452af59f381277)
+
+commit 15360a900c8746c28006d6f5c8b093685ddac796
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 08:52:51 2014 -0700
+
+    ceph-disk: partprobe before settle when preparing dev
+    
+    Two users have reported this fixes a problem with using --dmcrypt.
+    
+    Fixes: #6966
+    Tested-by: Eric Eastman <eric0e@aol.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f196265f049d432e399197a3af3f90d2e916275)
+
+commit ef736f530405a2c4f9a9a1b4cc98d1160524e1ab
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 13:33:14 2014 -0700
+
+    osd: fix filestore perf stats update
+    
+    Update the struct we are about to send, not the (unlocked!) one we will
+    send the next time around.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4afffb4a10a0bbf7f2018ef3ed6b167c7921e46b)
+
+commit 4ca12b40206f64d32369a78ce443a0dbf5d2b757
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:09 2014 -0700
+
+    common/WorkQueue: allow io priority to be set for wq
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e4b3b1f1cb870f39fc7cfb3adeae93e078d9057)
+    
+    Conflicts:
+    	src/common/WorkQueue.cc
+
+commit c9b3067f9b78a65e0b9c2ce2488d8906eb73c869
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:42 2014 -0700
+
+    common/Thread: allow io priority to be set for a Thread
+    
+    Ideally, set this before starting the thread.  If you set it after, we
+    could potentially race with create() itself.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 01533183e7455b713640e001962339907fb6f980)
+
+commit 0796057f1c13c467dd3eadca036e3034da20e500
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:09 2014 -0700
+
+    common/io_priority: wrap ioprio_set() and gettid()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 705713564bebd84ad31cc91698311cf2fbd51a48)
+    
+    Conflicts:
+    	src/common/Makefile.am
+
+commit c46fe8e9ec54d4decd1f70256a8ab05c9b183f8a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 10:47:24 2014 -0700
+
+    osd: introduce simple sleep during scrub
+    
+    This option is similar to osd_snap_trim_sleep: simply inject an optional
+    sleep in the thread that is doing scrub work.  This is a very kludgey and
+    coarse knob for limiting the impact of scrub on the cluster, but can help
+    until we have a more robust and elegant solution.
+    
+    Only sleep if we are in the NEW_CHUNK state to avoid delaying processing of
+    an in-progress chunk.  In this state nothing is blocked on anything.
+    Conveniently, chunky_scrub() requeues itself for each new chunk.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c4e8451cc5b4ec5ed07e09c08fb13221e31a7ac6)
+
+commit 3cf01a0e56c5599731e25898c1d9f2d2b3d6a9ba
+Merge: 8fd2fc1 c340085
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 09:27:03 2014 -0700
+
+    Merge pull request #1963 from dachary/wip-8599-ruleset-dumpling
+    
+    mon: pool set <pool> crush_ruleset must not use rule_exists (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit c34008532362c3afadff6287bc8746c141a221a3
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:50:18 2014 +0100
+
+    mon: pool set <pool> crush_ruleset must not use rule_exists
+    
+    Implement CrushWrapper::ruleset_exists that iterates over the existing
+    rulesets to find the one matching the ruleset argument.
+    
+    ceph osd pool set <pool> crush_ruleset must not use
+    CrushWrapper::rule_exists, which checks for a *rule* existing, whereas
+    the value being set is a *ruleset*. (cherry picked from commit
+    fb504baed98d57dca8ec141bcc3fd021f99d82b0)
+    
+    A test via ceph osd pool set data crush_ruleset verifies the ruleset
+    argument is accepted.
+    
+    http://tracker.ceph.com/issues/8599 fixes: #8599
+    
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d02d46e25080d5f7bb8ddd4874d9019a078b816b)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+
+commit 8fd2fc1aec4ba08c086a7db2208dbdebfcb4e92e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 3 07:03:01 2014 -0800
+
+    osd: 'status' admin socket command
+    
+    Basic stuff, like what state is the OSD in, and what osdmap epoch are
+    we on.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09099c9e4c7d2aa31eb8a0b7c18e43272fae7ce2)
+
+commit 323492451c2d29b90c30a4e133dc8ca9780035a5
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 20:18:49 2014 -0700
+
+    init-ceph: continue after failure doing osd data mount
+    
+    If we are starting many daemons and hit an error, we normally note it and
+    move on.  Do the same when doing the pre-mount step.
+    
+    Fixes: #8554
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6a7e20147cc39ed4689809ca7d674d3d408f2a17)
+
+commit 558249c519225d5e6d0784da20fe12d8f5d71cd2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 11:06:29 2014 -0700
+
+    rgw: cut short object read if a chunk returns error
+    
+    Fixes: #8289
+    Backport: firefly, dumpling
+    When reading an object, if we hit an error when trying to read one of
+    the rados objects then we should just stop. Otherwise we're just going
+    to continue reading the rest of the object, and since it can't be sent
+    back to the client (as we have a hole in the middle), we end up
+    accumulating everything in memory.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 03b0d1cfb7bd30a77fedcf75eb06476b21b14e95)
+
+commit a948ce2f9ccf67d5ccc690b5c326af19b6e6da4d
+Merge: 95a464a 7ad2fbf
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 21:12:32 2014 -0700
+
+    Merge pull request #1931 from ceph/wip-7068-dumpling
+    
+    Wip 7068 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 95a464a443e3a249889151807065736fff44ea3a
+Merge: a79a787 94e6c08
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 6 08:45:58 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8269-dumpling' into dumpling
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit a79a787b924ef7e04efb7f3c13b9559e06bebfd8
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:41:41 2014 -0700
+
+    doc: Added requiretty comment to preflight checklist.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 5c5eb21fd1a12666e1ce79718c8166eb7bdd6748
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:34:46 2014 -0700
+
+    doc: Added Disable requiretty to quick start.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 7ad2fbf3bd3208f47aa2a76d3fd16479e2c845ab
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 2 18:00:04 2013 -0700
+
+    ReplicatedPG: lock snapdir obc during write
+    
+    Otherwise, we won't block properly in prep_push_backfill_object.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b87bc2311aa4da065477f402a869e2edc1558e2f)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h
+
+commit 94e6c0858fb07c0edb8fc4d08e02542d22823ed2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 2 17:06:05 2014 -0700
+
+    rgw: don't allow multiple writers to same multiobject part
+    
+    Fixes: #8269
+    
+    A client might need to retry a multipart part write. The original thread
+    might race with the new one, trying to clean up after it, clobbering the
+    part's data.
+    The fix is to detect whether an original part already existed, and if so
+    use a different part name for it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.67.11.txt b/doc/changelog/v0.67.11.txt
new file mode 100644
index 000000000..6e581ece8
--- /dev/null
+++ b/doc/changelog/v0.67.11.txt
@@ -0,0 +1,215 @@
+commit bc8b67bef6309a32361be76cd11fb56b057ea9d2 (tag: refs/tags/v0.67.11, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Sep 24 06:25:55 2014 -0700
+
+    0.67.11
+
+commit 0e90b04d9452999d7d91305c80585782d492c91b (refs/remotes/gh/dumpling-jni)
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed May 21 21:41:23 2014 -0700
+
+    cephfs-java: build against older jni headers
+    
+    Older versions of the JNI interface expected non-const parameters
+    to their memory move functions. It's unpleasant, but won't actually
+    change the memory in question, to do a cast_const in order to satisfy
+    those older headers. (And even if it *did* modify the memory, that
+    would be okay given our single user.)
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 4d4b77e5b6b923507ec4a0ad9d5c7018e4542a3c)
+
+commit 3f020443c8d92e61d8593049147a79a6696c9c93
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Jul 24 15:29:40 2014 -0700
+
+    librbd: fix crash using clone of flattened image
+    
+    The crash occurs due to ImageCtx->parent->parent being uninitialized,
+    since the inital open_parent() -> open_image(parent) ->
+    ictx_refresh(parent) occurs before ImageCtx->parent->snap_id is set,
+    so refresh_parent() is not called to open an ImageCtx for the parent
+    of the parent. This leaves the ImageCtx->parent->parent NULL, but the
+    rest of ImageCtx->parent updated to point at the correct parent snapshot.
+    
+    Setting the parent->snap_id earlier has some unintended side effects
+    currently, so for now just call refresh_parent() during
+    open_parent(). This is the easily backportable version of the
+    fix. Further patches can clean up this whole initialization process.
+    
+    Fixes: #8845
+    Backport: firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2545e80d274b23b6715f4d8b1f4c6b96182996fb)
+
+commit d3e880af5f3ae71d13159514c33c6b41fc648d54
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 10 08:00:50 2014 -0700
+
+    test/cli-integration/rbd: fix trailing space
+    
+    Newer versions of json.tool remove the trailing ' ' after the comma.  Add
+    it back in with sed so that the .t works on both old and new versions, and
+    so that we don't have to remove the trailing spaces from all of the test
+    cases.
+    
+    Fixes: #8920
+    
+    Backports commit 605064dc685aa25cc7d58ec18b6449a3ce476d01
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5336398618f49d10e497392ab942d43062ed772d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 16 12:42:33 2014 -0700
+
+    os/FileStore: fix mount/remount force_sync race
+    
+    Consider:
+    
+     - mount
+     - sync_entry is doing some work
+     - umount
+       - set force_sync = true
+       - set done = true
+     - sync_entry exits (due to done)
+       - ..but does not set force_sync = false
+     - mount
+     - journal replay starts
+     - sync_entry sees force_sync and does a commit while op_seq == 0
+     ...crash...
+    
+    Fixes: #9144
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd11042f969b94f7a461d02e1475794031c79f61)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit f7498c47ed90db71e50a3d4ff9ce13626e863719
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:44:57 2014 -0700
+
+    osdc/Objecter: revoke rx_buffer on op_cancel
+    
+    If we cancel a read, revoke the rx buffers to avoid a use-after-free and/or
+    other undefined badness by using user buffers that may no longer be
+    present.
+    
+    Fixes: #9362
+    Backport: firefly, dumpling
+    Reported-by: Matthias Kiefer <matthias.kiefer@1und1.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2305b2897acba38384358c33ca3bbfcae6f1c74e)
+    
+    (adjusted for op->con instead of s->con)
+
+commit 4dbb5010bd73ea8c7f9ed85e90e645ba1e85ea0a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 06:58:45 2014 -0700
+
+    mon/Paxos: don't spam log with is_readable at dout level 1
+    
+    Backport: firefly, dumpling
+    Reported-by: Aanchal Agrawal <Aanchal.Agrawal@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 62ca27d0b119b597ebad40dde64c4d86599e466d)
+
+commit 5315cf0a47e0a21e514df0d85be170dbca7ffc92
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Sep 4 13:58:14 2014 -0400
+
+    doc: add note on soft JS dependency for navigating docs
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 657be818375bea2d8b5998ea1e5505eedc2f294d)
+
+commit 707076e3a9294169c3042afeb11b71b84727094b
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 21:21:45 2014 -0400
+
+    doc: fix missing bracket
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 69638dfaeb0dcd96dac4b5f5c00ed08042432487)
+
+commit 2ebdcfd5490462d77e32ffe3ef162f70183d53ab
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 20:47:54 2014 -0400
+
+    doc: attempt to get the ayni JS into all head tags
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 35663fa55ac1579a3b0c8b67028a3a8dfea87b48)
+
+commit c7948afa8232117c8a6e13b17774045a5e1bea6d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 17 20:54:28 2014 -0700
+
+    qa/workunits/rbd/qemu-iotests: touch common.env
+    
+    This seems to be necessary on trusty.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 055be68cf8e1b84287ab3631a02e89a9f3ae6cca)
+
+commit aaeebceb2189336f5f957f2a797d0448c2754b15
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 13:32:29 2014 -0700
+
+    mon: fix divide by zero when pg_num adjusted and no osds
+    
+    Fixes: #9052
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    
+    Manual backport of 239401db7b51541a57c59a261b89e0f05347c32d
+
+commit e7c5fb58d85ed0d1be359e616137666c5e207c38
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 07:05:34 2014 -0700
+
+    common/LogClient: fix sending dup log items
+    
+    We need to skip even the most recently sent item in order to get to the
+    ones we haven't sent yet.
+    
+    Fixes: #9080
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 057c6808be5bc61c3f1ac2b956c1522f18411245)
+
+commit cbc9218edc122337fd6d9dbbdca18670e20c6169
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 11 16:41:26 2014 -0700
+
+    librbd: fix error path cleanup for opening an image
+    
+    If the image doesn't exist and caching is enabled, the ObjectCacher
+    was not being shutdown, and the ImageCtx was leaked. The IoCtx could
+    later be closed while the ObjectCacher was still running, resulting in
+    a segfault. Simply use the usual cleanup path in open_image(), which
+    works fine here.
+    
+    Fixes: #8912
+    Backport: dumpling, firefly
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3dfa72d5b9a1f54934dc8289592556d30430959d)
+
+commit b5dafe1c0f7ecf7c3a25d0be5dfddcbe3d07e69e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:58 2014 -0700
+
+    osd: allow io priority to be set for the disk_tp
+    
+    The disk_tp covers scrubbing, pg deletion, and snap trimming
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 84b3003119eeb8acfb3faacf357e6c6a452950e3)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    
+    (cherry picked from commit 987ad133415aa988061c95259f9412b05ce8ac7e)
diff --git a/doc/changelog/v0.67.2.txt b/doc/changelog/v0.67.2.txt
new file mode 100644
index 000000000..e5d0f3d5e
--- /dev/null
+++ b/doc/changelog/v0.67.2.txt
@@ -0,0 +1,207 @@
+commit eb4380dd036a0b644c6283869911d615ed729ac8
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Aug 22 19:10:55 2013 -0700
+
+    v0.67.2
+
+commit 242e43dae5b7c935b8f92c09e8dfe4704ba13787
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:49:57 2013 -0700
+
+    .gitignore: ignore test-driver
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit edf2c3449ec96d91d3d7ad01c50f7a79b7b2f7cc)
+
+commit 88aef702fb77c0a176caf37646a11ef480621412
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:42:49 2013 -0700
+
+    fuse: fix warning when compiled against old fuse versions
+    
+    client/fuse_ll.cc: In function 'void invalidate_cb(void*, vinodeno_t, int64_t, int64_t)':
+    warning: client/fuse_ll.cc:540: unused variable 'fino'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9833e9dabe010e538cb98c51d79b6df58ce28f9e)
+
+commit 48e104c9486f7a532455df108dbc225c00796097
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:40:34 2013 -0700
+
+    json_spirit: remove unused typedef
+    
+    In file included from json_spirit/json_spirit_writer.cpp:7:0:
+    json_spirit/json_spirit_writer_template.h: In function 'String_type json_spirit::non_printable_to_string(unsigned int)':
+    json_spirit/json_spirit_writer_template.h:37:50: warning: typedef 'Char_type' locally defined but not used [-Wunused-local-typedefs]
+             typedef typename String_type::value_type Char_type;
+    
+    (Also, ha ha, this file uses \r\n.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6abae35a3952e5b513895267711fea63ff3bad09)
+
+commit ae42619ca710d737bf4d8c63f39d1102326c903c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:31:41 2013 -0700
+
+    gtest: add build-aux/test-driver to .gitignore
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9cdd19d1cd88b84e8a867f5ab85cb51fdc6f8e4)
+
+commit 2c122be08db2f233d66214eb804734ae45646084
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+    
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+    
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+    
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
+
+commit f6fe74ff51f679e7245b02462822d9ef1e15d28c
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:23:46 2013 -0700
+
+    pybind: fix Rados.conf_parse_env test
+    
+    This happens after we connect, which means we get ENOSYS always.
+    Instead, parse_env inside the normal setup method, which had the added
+    benefit of being able to debug these tests.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6ef1970340c57d6e02f947348fb38882b51d131c)
+
+commit 47c89497b7f69cbf1557cd05b89837c388e2ba2f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 13:14:59 2013 -0700
+
+    librados: fix MWatchNotify leak
+    
+    Do not leak the message if the watcher is not registered.  Also, simplify
+    this block.
+    
+    Fixes (part of): #5949
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6f5d8036f3e70c5e30edf7e36fb8ff9a56197f60)
+
+commit b3a9a8c4e5edff5431d8da71033047eced6bf985
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 19 17:23:44 2013 -0700
+
+    PG: remove old log when we upgrade log version
+    
+    Otherwise the log_oid will be non-empty and the next
+    boot will cause us to try to upgrade again.
+    
+    Fixes: #6057
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1f851cb2489a95526de932ec6734ebf413e43490)
+
+commit c6005ccbaa482c62d7a6cbb387bdcf17f0e308d5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 19 00:02:24 2013 -0700
+
+    PGLog: add a config to disable PGLog::check()
+    
+    This is a debug check which may be causing excessive
+    cpu usage.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 00080d785f6695b800f71317a3048a21064e61cb)
+
+commit 96d719eeecceaa06078a29c2f868e50e6bc9ab31
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 19 12:48:50 2013 -0700
+
+    ceph: parse CEPH_ARGS environment variable
+    
+    Fixes: #6052
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 67a95b9880c9bc6e858150352318d68d64ed74ad)
+
+commit d348cf5d135d099fe0490c1519196cd83a04831e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 19 12:48:40 2013 -0700
+
+    rados pybind: add conf_parse_env()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eef7cacdb19313907a9367187b742db5382ee584)
+
+commit 290bcd8a718887eb0e28aa2d97bceeee79068ea9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 13 13:16:07 2013 -0700
+
+    rgw: drain requests before exiting
+    
+    Fixes: #5953
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3cbf6a7b031c2ce8072733c5c0b7ceb53fdcb090)
+
+commit 863df08a43dff99797453040eb1ef6071b0432f9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 11:16:17 2013 -0700
+
+    rgw: do not leak handler in get_handler() error path
+    
+    If we fail to initialize, delete the handler.
+    
+    Fixes (part of): #5949
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 810c52de36719c3ee6cf2bdf59d5cde8840bbe55)
+
+commit 9ac003f793b6cc72059110aac44014ddf2372bee
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:20:54 2013 -0700
+
+    rgw: fix leak of RGWDataChangesLog::renew_thread
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 25948319c4d256c4aeb0137eb88947e54d14cc79)
+
+commit 89cd9dc403e97b4bd08920fbb5d6e2b8b9b7dac2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:17:10 2013 -0700
+
+    rgw: free resolver on shutdown
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a31356338b8ae55df59d829d9080ffad70b97d10)
+
+commit 5b26ca7fa5beb87cbbe6bbb26d70789ff2aa7661
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:22:20 2013 -0700
+
+    rgw: fix up signal handling
+    
+    OMG libfcgi is annoying with shutdown and signals.  You need to close
+    the fd *and* resend a signal to ensure that you kick the accept loop
+    hard enough to make it shut down.
+    
+    Document this, and switch to the async signal handlers.  Put them
+    tightly around the runtime loop as we do with other daemons.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 59b13cebee600dad2551d2c7dc3482b05eaf8b22)
diff --git a/doc/changelog/v0.67.3.txt b/doc/changelog/v0.67.3.txt
new file mode 100644
index 000000000..d6b1f2b27
--- /dev/null
+++ b/doc/changelog/v0.67.3.txt
@@ -0,0 +1,700 @@
+commit 408cd61584c72c0d97b774b3d8f95c6b1b06341a
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Sep 9 12:50:11 2013 -0700
+
+    v0.67.3
+
+commit 17a7342b3b935c06610c58ab92a9a1d086923d32
+Merge: b4252bf 10433bb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Sep 7 13:34:45 2013 -0700
+
+    Merge pull request #574 from dalgaaf/fix/da-dumpling-cherry-picks
+    
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 10433bbe72dbf8eae8fae836e557a043610eb54e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 11:30:15 2013 +0200
+
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b5137baf651eaaa9f67e3864509e437f9d5c3d5a)
+
+commit b4252bff79150a95e9d075dd0b5e146ba9bf2ee5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 22 11:19:37 2013 -0700
+
+    FileStore: add config option to disable the wbthrottle
+    
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3528100a53724e7ae20766344e467bf762a34163)
+
+commit 699324e0910e5e07a1ac68df8cf1108e5671ec15
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 22 11:19:52 2013 -0700
+
+    WBThrottle: use fdatasync instead of fsync
+    
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d571825080f0bff1ed3666e95e19b78a738ecfe8)
+
+commit 074717b4b49ae1a55bc867e5c34d43c51edc84a5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 29 15:08:58 2013 -0700
+
+    PGLog: initialize writeout_from in PGLog constructor
+    
+    Fixes: 6151
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Introduced: f808c205c503f7d32518c91619f249466f84c4cf
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 42d65b0a7057696f4b8094f7c686d467c075a64d)
+
+commit c22d980cf42e580818dc9f526327518c0ddf8ff5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 27 08:49:14 2013 -0700
+
+    PGLog: maintain writeout_from and trimmed
+    
+    This way, we can avoid omap_rmkeyrange in the common append
+    and trim cases.
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit f808c205c503f7d32518c91619f249466f84c4cf)
+
+commit 53c7ab4db00ec7034f5aa555231f9ee167f43201
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 27 07:27:26 2013 -0700
+
+    PGLog: don't maintain log_keys_debug if the config is disabled
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1c0d75db1075a58d893d30494a5d7280cb308899)
+
+commit 40dc489351383c2e35b91c3d4e76b633309716df
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 26 23:19:45 2013 -0700
+
+    PGLog: move the log size check after the early return
+    
+    There really are stl implementations (like the one on my ubuntu 12.04
+    machine) which have a list::size() which is linear in the size of the
+    list.  That assert, therefore, is quite expensive!
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit fe68b15a3d82349f8941f5b9f70fcbb5d4bc7f97)
+
+commit 4261eb5ec105b9c27605360910602dc367fd79f5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 17:16:08 2013 -0700
+
+    rbd.cc: relicense as LGPL2
+    
+    All past authors for rbd.cc have consented to relicensing from GPL to
+    LGPL2 via email:
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 01:59:36 +0200
+    From: Sylvain Munaut <s.munaut@whatever-company.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I hereby consent to the relicensing of any contribution I made to the
+    aforementioned rbd.cc file from GPL to LGPL2.1.
+    
+    (I hope that'll be impressive enough, I did my best :p)
+    
+    btw, tnt@246tNt.com and s.munaut@whatever-company.com are both me.
+    
+    Cheers,
+    
+        Sylvain
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 17:00:48 -0700
+    From: Yehuda Sadeh <yehuda@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 17:02:24 -0700
+    From: Josh Durgin <josh.durgin@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 18:17:46 -0700
+    From: Stanislav Sedov <stas@freebsd.org>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    Thanks for taking care of it!
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 18:24:15 -0700
+    From: Colin McCabe <cmccabe@alumni.cmu.edu>
+    
+    I consent.
+    
+    cheers,
+    Colin
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 07:08:12 +0200
+    From: Christian Brunner <christian@brunner-muc.de>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent
+    
+    Christian
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 12:17:34 +0300
+    From: Stratos Psomadakis <psomas@grnet.gr>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    Hi,
+    
+    I consent with the GPL -> LGL2.1 re-licensing.
+    
+    Thanks
+    Stratos
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 16:13:13 +0200
+    From: Wido den Hollander <wido@42on.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent!
+    
+    You have my permission to re-license the code I wrote for rbd.cc to LGPL2.1
+    
+    ---
+    
+    Date: Sun, 11 Aug 2013 10:40:32 +0200
+    From: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Subject: Re: btw
+    
+    Hi Sage,
+    
+    I agree to switch the license of ceph_argparse.py and rbd.cc from GPL2
+    to LGPL2.
+    
+    Regards
+    
+    Danny Al-Gaaf
+    
+    ---
+    
+    Date: Tue, 13 Aug 2013 17:15:24 -0700
+    From: Dan Mick <dan.mick@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent to relicense any contributed code that I wrote under LGPL2.1 license.
+    
+    ---
+    
+    ...and I consent too.  Drop the exception from COPYING and debian/copyright
+    files.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2206f55761c675b31078dea4e7dd66f2666d7d03)
+
+commit 211c5f13131e28b095a1f3b72426128f1db22218
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 23 15:39:20 2013 -0700
+
+    rgw: flush pending data when completing multipart part upload
+    
+    Fixes: #6111
+    Backport: dumpling
+    When completing the part upload we need to flush any data that we
+    aggregated and didn't flush yet. With earlier code didn't have to deal
+    with it as for multipart upload we didn't have any pending data.
+    What we do now is we call the regular atomic data completion
+    function that takes care of it.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9a551296e0811f2b65972377b25bb28dbb42f575)
+
+commit 1a9651010aab51c9be2edeccd80e9bd11f5177ce
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 19:46:43 2013 -0700
+
+    rgw: check object name after rebuilding it in S3 POST
+    
+    Fixes: #6088
+    Backport: bobtail, cuttlefish, dumpling
+    
+    When posting an object it is possible to provide a key
+    name that refers to the original filename, however we
+    need to verify that in the end we don't end up with an
+    empty object name.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c8ec532fadc0df36e4b265fe20a2ff3e35319744)
+
+commit 1bd74a020b93f154b2d4129d512f6334387de7c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 17:46:45 2013 -0700
+
+    mon/MonClient: release pending outgoing messages on shutdown
+    
+    This fixes a small memory leak when we have messages queued for the mon
+    when we shut down.  It is harmless except for the valgrind leak check
+    noise that obscures real leaks.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 309569a6d0b7df263654b7f3f15b910a72f2918d)
+
+commit 24f2669783e2eb9d9af5ecbe106efed93366ba63
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 29 13:06:33 2013 -0700
+
+    rgw: change watch init ordering, don't distribute if can't
+    
+    Backport: dumpling
+    
+    Moving back the watch initialization after the zone init,
+    as the zone info holds the control pool name. Since zone
+    init might need to create a new system object (that needs
+    to distribute cache), don't try to distribute cache if
+    watch is not yet initialized.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 1d1f7f18dfbdc46fdb09a96ef973475cd29feef5)
+
+commit a708c8ab52e5b1476405a1f817c23b8845fbaab3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 30 09:41:29 2013 -0700
+
+    ceph-post-file: use mktemp instead of tempfile
+    
+    tempfile is a debian thing, apparently; mktemp is present everywhere.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e60d4e09e9f11e3c34a05cd122341e06c7c889bb)
+
+commit 625f13ee0d6cca48d61dfd65e00517d092552d1c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 28 09:50:11 2013 -0700
+
+    mon: discover mon addrs, names during election state too
+    
+    Currently we only detect new mon addrs and names during the probing phase.
+    For non-trivial clusters, this means we can get into a sticky spot when
+    we discover enough peers to form an quorum, but not all of them, and the
+    undiscovered ones are enough to break the mon ranks and prevent an
+    election.
+    
+    One way to work around this is to continue addr and name discovery during
+    the election.  We should also consider making the ranks less sensitive to
+    the undefined addrs; that is a separate change.
+    
+    Fixes: #4924
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Tested-by: Bernhard Glomm <bernhard.glomm@ecologic.eu>
+    (cherry picked from commit c24028570015cacf1d9e154ffad80bec06a61e7c)
+
+commit 83cfd4386c1fd0fa41aea345704e27f82b524ece
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Aug 22 17:30:24 2013 -0700
+
+    ceph_rest_api.py: create own default for log_file
+    
+    common/config thinks the default log_file for non-daemons should be "".
+    Override that so that the default is
+        /var/log/ceph/{cluster}-{name}.{pid}.log
+    since ceph-rest-api is more of a daemon than a client.
+    
+    Fixes: #6099
+    Backport: dumpling
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 2031f391c3df68e0d9e381a1ef3fe58d8939f0a8)
+
+commit 8a1da62d9564a32f7b8963fe298e1ac3ad0ea3d9
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 16 17:59:11 2013 -0700
+
+    ceph-post-file: single command to upload a file to cephdrop
+    
+    Use sftp to upload to a directory that only this user and ceph devs can
+    access.
+    
+    Distribute an ssh key to connect to the account.  This will let us revoke
+    the key in the future if we feel the need.  Also distribute a known_hosts
+    file so that users have some confidence that they are connecting to the
+    real ceph drop account and not some third party.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit d08e05e463f1f7106a1f719d81b849435790a3b9)
+
+commit 3f8663477b585dcb528fdd7047c50d9a52d24b95
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 13:29:32 2013 -0700
+
+    ceph.spec.in:  remove trailing paren in previous commit
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 23fb908cb3ac969c874ac12755d20ed2f636e1b9
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 11:07:16 2013 -0700
+
+    ceph.spec.in:  Don't invoke debug_package macro on centos.
+    
+    If the redhat-rpm-config package is installed, the debuginfo rpms will
+    be built by default.   The build will fail when the package installed
+    and the specfile also invokes the macro.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 11f5853d8178ab60ab948d373c1a1f67324ce3bd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Aug 24 14:04:09 2013 -0700
+
+    osd: install admin socket commands after signals
+    
+    This lets us tell by the presence of the admin socket commands whether
+    a signal will make us shut down cleanly.  See #5924.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c5b5ce120a8ce9116be52874dbbcc39adec48b5c)
+
+commit 39adc0195e6016ce36828885515be1bffbc10ae1
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 22:39:09 2013 -0700
+
+    ceph-disk: partprobe after creating journal partition
+    
+    At least one user reports that a partprobe is needed after creating the
+    journal partition.  It is not clear why sgdisk is not doing it, but this
+    fixes ceph-disk for them, and should be harmless for other users.
+    
+    Fixes: #5599
+    Tested-by: lurbs in #ceph
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2af59d5e81c5e3e3d7cfc50d9330d7364659c5eb)
+    (cherry picked from commit 3e42df221315679605d68b2875aab6c7eb6b3cc4)
+
+commit 6a4fe7b9b068ae990d6404921a46631fe9ebcd31
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:27:23 2013 -0700
+
+    mon/Paxos: always refresh after any store_state
+    
+    If we store any new state, we need to refresh the services, even if we
+    are still in the midst of Paxos recovery.  This is because the
+    subscription path will share any committed state even when paxos is
+    still recovering.  This prevents a race like:
+    
+     - we have maps 10..20
+     - we drop out of quorum
+     - we are elected leader, paxos recovery starts
+     - we get one LAST with committed states that trim maps 10..15
+     - we get a subscribe for map 10..20
+       - we crash because 10 is no longer on disk because the PaxosService
+         is out of sync with the on-disk state.
+    
+    Fixes: #6045
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 981eda9f7787c83dc457f061452685f499e7dd27)
+
+commit 13d396e46ed9200e4b9f21db2f0a8efbc5998d82
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:27:09 2013 -0700
+
+    mon/Paxos: return whether store_state stored anything
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7e0848d8f88f156a05eef47a9f730b772b64fbf2)
+
+commit f248383bacff76203fa94716cfdf6cf766da24a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:26:57 2013 -0700
+
+    mon/Paxos: cleanup: use do_refresh from handle_commit
+    
+    This avoid duplicated code by using the helper created exactly for this
+    purpose.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b9dee2285d9fe8533fa98c940d5af7b0b81f3d33)
+
+commit 02608a12d4e7592784148a62a47d568efc24079d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:48:06 2013 -0700
+
+    osdc/ObjectCacher: do not merge rx buffers
+    
+    We do not try to merge rx buffers currently.  Make that explicit and
+    documented in the code that it is not supported.  (Otherwise the
+    last_read_tid values will get lost and read results won't get applied
+    to the cache properly.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c50c446152ab0e571ae5508edb4ad7c7614c310)
+
+commit 0e2bfe71965eeef29b47e8032637ea820a7ce49c
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:47:18 2013 -0700
+
+    osdc/ObjectCacher: match reads with their original rx buffers
+    
+    Consider a sequence like:
+    
+     1- start read on 100~200
+           100~200 state rx
+     2- truncate to 200
+           100~100 state rx
+     3- start read on 200~200
+           100~100 state rx
+           200~200 state rx
+     4- get 100~200 read result
+    
+    Currently this makes us crash on
+    
+    osdc/ObjectCacher.cc: 738: FAILED assert(bh->length() <= start+(loff_t)length-opos)
+    
+    when processing the second 200~200 bufferhead (it is too big).  The
+    larger issue, though, is that we should not be looking at this data at
+    all; it has been truncated away.
+    
+    Fix this by marking each rx buffer with the read request that is sent to
+    fill it, and only fill it from that read request.  Then the first reply
+    will fill the first 100~100 extend but not touch the other extent; the
+    second read will do that.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b59f930ae147767eb4c9ff18c3821f6936a83227)
+
+commit 6b51c960715971a0351e8203d4896cb0c4138a3f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 15:54:48 2013 -0700
+
+    mon/Paxos: fix another uncommitted value corner case
+    
+    It is possible that we begin the paxos recovery with an uncommitted
+    value for, say, commit 100.  During last/collect we discover 100 has been
+    committed already.  But also, another node provides an uncommitted value
+    for 101 with the same pn.  Currently, we refuse to learn it, because the
+    pn is not strictly > than our current uncommitted pn... even though it is
+    the next last_committed+1 value that we need.
+    
+    There are two possible fixes here:
+    
+     - make this a >= as we can accept newer values from the same pn.
+     - discard our uncommitted value metadata when we commit the value.
+    
+    Let's do both!
+    
+    Fixes: #6090
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe5010380a3a18ca85f39403e8032de1dddbe905)
+
+commit b3a280d5af9d06783d2698bd434940de94ab0fda
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:35 2013 -0700
+
+    os: make readdir_r buffers larger
+    
+    PATH_MAX isn't quite big enough.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 99a2ff7da99f8cf70976f05d4fe7aa28dd7afae5)
+
+commit 989a664ef0d1c716cab967f249112f595cf98c43
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:08 2013 -0700
+
+    os: fix readdir_r buffer size
+    
+    The buffer needs to be big or else we're walk all over the stack.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2df66d9fa214e90eb5141df4d5755b57e8ba9413)
+    
+    Conflicts:
+    
+    	src/os/BtrfsFileStoreBackend.cc
+
+commit a4cca31c82bf0e84272e01eb1b3188dfdb5b5615
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 10:53:12 2013 -0700
+
+    rgw: fix crash when creating new zone on init
+    
+    Moving the watch/notify init before the zone init,
+    as we might need to send a notification.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3d55534268de7124d29bd365ea65da8d2f63e501)
+
+commit 4cf6996803ef66f2b6083f73593259d45e2740a3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 19 08:40:16 2013 -0700
+
+    rgw: change cache / watch-notify init sequence
+    
+    Fixes: #6046
+    We were initializing the watch-notify (through the cache
+    init) before reading the zone info which was much too
+    early, as we didn't have the control pool name yet. Now
+    simplifying init/cleanup a bit, cache doesn't call watch/notify
+    init and cleanup directly, but rather states its need
+    through a virtual callback.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d26ba3ab0374e77847c742dd00cb3bc9301214c2)
+
+commit aea6de532b0b843c3a8bb76d10bab8476f0d7c09
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Thu Aug 22 03:40:22 2013 -0300
+
+    enable mds rejoin with active inodes' old parent xattrs
+    
+    When the parent xattrs of active inodes that the mds attempts to open
+    during rejoin lack pool info (struct_v < 5), this field will be filled
+    in with -1, causing the mds to retry fetching a backtrace with a pool
+    number that matches the expected value, which fails and causes the
+    err==-ENOENT branch to be taken and retry pool 1, which succeeds, but
+    with pool -1, and so keeps on bouncing between the two retry cases
+    forever.
+    
+    This patch arranges for the mds to go along with pool -1 instead of
+    insisting that it be refetched, enabling it to complete recovery
+    instead of eating cpu, network bandwidth and metadata osd's resources
+    like there's no tomorrow, in what AFAICT is an infinite and very busy
+    loop.
+    
+    This is not a new problem: I've had it even before upgrading from
+    Cuttlefish to Dumpling, I'd just never managed to track it down, and
+    force-unmounting the filesystem and then restarting the mds was an
+    easier (if inconvenient) work-around, particularly because it always
+    hit when the filesystem was under active, heavy-ish use (or there
+    wouldn't be much reason for caps recovery ;-)
+    
+    There are two issues not addressed in this patch, however.  One is
+    that nothing seems to proactively update the parent xattr when it is
+    found to be outdated, so it remains out of date forever.  Not even
+    renaming top-level directories causes the xattrs to be recursively
+    rewritten.  AFAICT that's a bug.
+    
+    The other is that inodes that don't have a parent xattr (created by
+    even older versions of ceph) are reported as non-existing in the mds
+    rejoin message, because the absence of the parent xattr is signaled as
+    a missing inode (?failed to reconnect caps for missing inodes?).  I
+    suppose this may cause more serious recovery problems.
+    
+    I suppose a global pass over the filesystem tree updating parent
+    xattrs that are out-of-date would be desirable, if we find any parent
+    xattrs still lacking current information; it might make sense to
+    activate it as a background thread from the backtrace decoding
+    function, when it finds a parent xattr that's too out-of-date, or as a
+    separate client (ceph-fsck?).
+    
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    Reviewed-by: Zheng, Yan <zheng.z.yan@intel.com>
+    (cherry picked from commit 617dc36d477fd83b2d45034fe6311413aa1866df)
+
+commit 0738bdf92f5e5eb93add152a4135310ac7ea1c91
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Mon Jul 29 17:05:44 2013 +0200
+
+    mds: remove waiting lock before merging with neighbours
+    
+    CephFS currently deadlocks under CTDB's ping_pong POSIX locking test
+    when run concurrently on multiple nodes.
+    The deadlock is caused by failed removal of a waiting_locks entry when
+    the waiting lock is merged with an existing lock, e.g:
+    
+    Initial MDS state (two clients, same file):
+    held_locks -- start: 0, length: 1, client: 4116, pid: 7899, type: 2
+    	      start: 2, length: 1, client: 4110, pid: 40767, type: 2
+    waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2
+    
+    Waiting lock entry 4116@1:1 fires:
+    handle_client_file_setlock: start: 1, length: 1,
+    			    client: 4116, pid: 7899, type: 2
+    
+    MDS state after lock is obtained:
+    held_locks -- start: 0, length: 2, client: 4116, pid: 7899, type: 2
+    	      start: 2, length: 1, client: 4110, pid: 40767, type: 2
+    waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2
+    
+    Note that the waiting 4116@1:1 lock entry is merged with the existing
+    4116@0:1 held lock to become a 4116@0:2 held lock. However, the now
+    handled 4116@1:1 waiting_locks entry remains.
+    
+    When handling a lock request, the MDS calls adjust_locks() to merge
+    the new lock with available neighbours. If the new lock is merged,
+    then the waiting_locks entry is not located in the subsequent
+    remove_waiting() call because adjust_locks changed the new lock to
+    include the old locks.
+    This fix ensures that the waiting_locks entry is removed prior to
+    modification during merge.
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 476e4902907dfadb3709ba820453299ececf990b)
+
+commit a0ac88272511d670b5c3756dda2d02c93c2e9776
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 20 11:10:42 2013 -0700
+
+    mon/PGMap: OSD byte counts 4x too large (conversion to bytes overzealous)
+    
+    Fixes: #6049
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eca53bbf583027397f0d5e050a76498585ecb059)
+
+commit 87b19c33ce29e2ca4fc49a2adeb12d3f14ca90a9
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 23 08:56:07 2013 -0400
+
+    ceph-disk: specify the filetype when mounting
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f040020fb2a7801ebbed23439159755ff8a3edbd)
diff --git a/doc/changelog/v0.67.4.txt b/doc/changelog/v0.67.4.txt
new file mode 100644
index 000000000..73b997ea3
--- /dev/null
+++ b/doc/changelog/v0.67.4.txt
@@ -0,0 +1,550 @@
+commit ad85b8bfafea6232d64cb7ba76a8b6e8252fa0c7
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Oct 3 22:41:31 2013 +0000
+
+    v0.67.4
+
+commit 5cd66d3b4bca92b402c95ab256fbc3f0329c446f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 20 14:04:47 2013 -0700
+
+    rgw: fix keystone token expiration test
+    
+    Fixes: #6360
+    The test was inverted, need expiration to be greater than
+    current time in order for token to be valid.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit e0203c61a3f45fdd6d3d3ece26fef6152bdc036d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Sep 11 16:55:06 2013 -0700
+
+    osd/OSD.cc: Use MIN() so that we don't exceed osd_recovery_max_active
+    
+    Caused by 944f3b73531af791c90f0f061280160003545c63
+    
+    Fixes: #6291
+    
+    Backport: dumpling
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 139a714e13aa3c7f42091270b55dde8a17b3c4b8)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit c376708358cedb5561fbb43e9b9e622df3ea7a58
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Sep 25 22:08:24 2013 +0100
+
+    mon: OSDMonitor: do not write full_latest during trim
+    
+    On commit 81983bab we patched OSDMonitor::update_from_paxos() such that we
+    write the latest full map version to 'full_latest' each time the latest
+    full map was built from the incremental versions.
+    
+    This change however clashed with OSDMonitor::encode_trim_extra(), which
+    also wrote to 'full_latest' on each trim, writing instead the version of
+    the *oldest* full map.  This duality of behaviors could lead the store
+    to an inconsistent state across the monitors (although there's no sign of
+    it actually imposing any issues besides rebuilding already existing full
+    maps on some monitors).
+    
+    We now stop OSDMonitor::encode_trim_extra() from writing to 'full_latest'.
+    This function will still write out the oldest full map it has in the store,
+    but it will no longer write to full_latest, instead leaving it up to
+    OSDMonitor::update_from_paxos() to figure it out -- and it already does.
+    
+    Fixes: #6378
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bd0f29a2c28cca496ec830eac932477ebf3182ba)
+
+commit de40d0b3e35ab0124cd3c4ebfcaa435ab8abfab9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 15:53:42 2013 -0700
+
+    crush: invalidate rmap on create (and thus decode)
+    
+    If we have an existing CrushWrapper object and decode from a bufferlist,
+    reset build_rmaps so that they get rebuilt.
+    
+    Remove the build_rmaps() all in decode that was useless on a redecode
+    (because have_rmaps == true in that case and it did nothing).
+    
+    Fixes: #6442
+    Backport: dumpling, maybe cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9b7a2ae329b6a511064dd3d6e549ba61f52cfd21)
+
+commit 32f5233288c47d95b87c0a9cab5f9c2ffcf15417
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Sep 30 14:58:11 2013 -0700
+
+    Invoke python with /usr/bin/env python instead of directly
+    
+    Fixes: #6311
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit b9000b314b9166845ff302d4a827a996775d9a14)
+
+commit 66aeca5a9079be398403bbff67bd5bf68c6fb111
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Sep 25 10:10:21 2013 -0700
+
+    qa/workunits/mon/crush_ops.sh: fix test
+    
+    Fix root.
+    
+    Fixes: #6392
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8cae87e9e08468cc86145e0fd60c05d12826239)
+
+commit beb366302a125dd422c4f092b12eb541cb3bc788
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 09:04:34 2013 -0700
+
+    Revert "ceph: parse CEPH_ARGS environment variable"
+    
+    This reverts commit 67a95b9880c9bc6e858150352318d68d64ed74ad.
+    
+    We now put CEPH_ARGS in the actual args we parse in python, which are passed
+    to rados piecemeal later.  This lets you put things like --id ... in there
+    that need to be parsed before librados is initialized.
+    (cherry picked from commit 97f462be4829f0167ed3d65e6694dfc16f1f3243)
+
+commit b475ff9576f145d31c053213c699e13df76d2bcb
+Author: Benoît Knecht <benoit.knecht@fsfe.org>
+Date:   Mon Sep 23 15:58:42 2013 +0200
+
+    Add CEPH_ARGS at the end of sys.argv
+    
+    This allows, for instance, to pass a different client name to ceph by
+    exporting CEPH_ARGS="--id client_id".
+    
+    Signed-off-by: Benoît Knecht <benoit.knecht@fsfe.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 30abe3244c86cbbe1f5b005850c29c9c0eafcad4)
+
+commit 94548b4b67cca37366c7d8719209a6d2e7956811
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 24 15:26:03 2013 -0700
+
+    mon/OSDMonitor: fix 'ceph osd crush reweight ...'
+    
+    The adjust method returns a count of adjusted items.
+    
+    Add a test.
+    
+    Fixes: #6382
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 3de32562b55c6ece3a6ed783c36f8b9f21460339)
+
+commit 00ff7f5c20e13869d0694379739ba4e61d44b97c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Sep 10 00:20:41 2013 +0100
+
+    qa: workunits: mon: crush_ops: test 'ceph osd crush move'
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3bc618b7b46496c5110edde0da9cae5d3e68e0e1)
+
+commit 0ff5b4a96833681e92cc41f019a569134474f4cf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 19:04:23 2013 +0200
+
+    osd: change warn_interval_multiplier to uint32_t
+    
+    to prevent overflow in OpTracker::check_ops_in_flight when
+    multiplying warn_interval_multiplier *= 2
+    
+    Backport: cuttlefish, dumpling
+    
+    http://tracker.ceph.com/issues/6370 fixes #6370
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1bce1f009bffd3e28025a08775fec189907a81db)
+
+commit fb15040b6cec6221baa550ddfffade823f784c4a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon Sep 9 13:01:12 2013 -0700
+
+    crushtool: do not dump core with non-unique bucket IDs
+    
+    Return -EEXIST on duplicate ID
+    BUG FIX: crush_add_bucket() mixes error returns and IDs
+      Add optional argument to return generated ID
+    
+    Fixes: #6246
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c76f3a0f9cf100ea2c941dc2b61c470aa5033d7)
+
+commit 410db3f30c6eb54b807908c1f251ad4026e7d446
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 17:06:30 2013 +0100
+
+    qa: workunits: cephtool: check if 'heap' commands are parseable
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit b1eeaddd5f214c1b0883b44fc8cae07c649be7c4)
+
+commit 062060a38bb26ff260cc51accc534413d726de49
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 17:50:27 2013 +0100
+
+    osd: OSD: add 'heap' command to known osd commands array
+    
+    Must have been forgotten during the cli rework.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit 296f2d0db31e9f5a59a3a62a1e95b6c440430fa3)
+
+commit 3f32f57b98e0224a1d30b2a81d7d260be0f53800
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 16:43:27 2013 +0100
+
+    mds: MDS: pass only heap profiler commands instead of the whole cmd vector
+    
+    The heap profiler doesn't care, nor should it, what our command name is.
+    It only cares about the commands it handles.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit 238fe272c6bdb62d4e57fd8555c0136de99c8129)
+
+commit 46dcc46617d8f35ab8433540b22343ddcbcc3716
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 16:41:14 2013 +0100
+
+    perfglue/heap_profiler.cc: expect args as first element on cmd vector
+    
+    We used to pass 'heap' as the first element of the cmd vector when
+    handling commands.  We haven't been doing so for a while now, so we
+    needed to fix this.
+    
+    Not expecting 'heap' also makes sense, considering that what we need to
+    know when we reach this function is what command we should handle, and
+    we should not care what the caller calls us when handling his business.
+    
+    Fixes: #6361
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit c98b910d49bd2b46ceafdc430044a31524c29f5b)
+
+commit 9dc5f15fbae22244ad1f62925e17c9d81e856e55
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 16 14:35:25 2013 -0700
+
+    rgw: destroy get_obj handle in copy_obj()
+    
+    Fixes: #6176
+    Backport: dumpling
+    We take different code paths in copy_obj, make sure we close the handle
+    when we exit the function. Move the call to finish_get_obj() out of
+    copy_obj_data() as we don't create the handle there, so that should
+    makes code less confusing and less prone to errors.
+    Also, note that RGWRados::get_obj() also calls finish_get_obj(). For
+    everything to work in concert we need to pass a pointer to the handle
+    and not the handle itself. Therefore we needed to also change the call
+    to copy_obj_data().
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9e98620e4325d15c88440a890b267131613e1aa1)
+
+commit 471233e98a9f64ad513a4a196b7661b80534cb00
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Sep 9 23:14:11 2013 +0100
+
+    mon: MonCommands: expect a CephString as 1st arg for 'osd crush move'
+    
+    Fixes: #6230
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7d3799fde19138f957f26ec6be10a8a0000fc1f0)
+
+commit 2908225092bd2aa1b8afcb7848c1cdac5bd9e638
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 16:23:33 2013 -0700
+
+    osd: revert 'osd max xattr size' limit
+    
+    Set it to 0 (unlimited) for now.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit abb88d70643c3a76435b7a9d5b04ff29f7502361)
+
+commit b3d3b3747c1eef695138dac828e5fcb435309c7b
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Sep 11 16:24:32 2013 -0700
+
+    mds: be more careful about decoding LogEvents
+    
+    We need to wrap the full decode section or we can abort the process
+    if there's an issue (which we may want to just skip by).
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 73289b34b0be5b6612e38944794d59b5e789f841)
+
+commit 06c58132199ed22413b509dfa751321ccdb24225
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Sep 17 17:58:20 2013 +0100
+
+    mon: OSDMonitor: multiple rebuilt full maps per transaction
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 0d20cae0be701c5b6151a26ee5e4fe24d89aa20a)
+
+commit 65bbcaf4b68790dae4506c1f5db237077e1ff0ae
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Sep 15 21:03:50 2013 +0100
+
+    mon: OSDMonitor: update latest_full while rebuilding full maps
+    
+    Not doing so will make the monitor rebuild the osdmap full versions, even
+    though they may have been rebuilt before, every time the monitor starts.
+    
+    This mostly happens when the cluster is left in an unhealthy state for
+    a long period of time and incremental versions build up.  Even though we
+    build the full maps on update_from_paxos(), not updating 'full_latest'
+    leads to the situation initially described.
+    
+    Fixes: #6322
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 81983bab3630520d6c7ee9b7e4a747bc17b8c5c3)
+
+commit 9b9edb04581cca15e67c567332529f5b3f426743
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Sep 15 21:00:55 2013 +0100
+
+    mon: OSDMonitor: smaller transactions when rebuilding full versions
+    
+    Otherwise, for considerably sized rebuilds, the monitor will not only
+    consume vast amounts of memory, but it will also have troubles committing
+    the transaction.  Anyway, it's also a good idea to adjust transactions to
+    the granularity we want, and to be fair we care that each rebuilt full map
+    gets to disk, even if subsequent full maps don't (those can be rebuilt
+    later).
+    
+    Fixes: #6323
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4ac1570c5cdcd6556dc291cc6d7878fd92d343ae)
+
+commit 298811f7a15541b9ec1015c416ad2aa075be5691
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Wed Aug 28 15:51:01 2013 +0100
+
+    mon: OSDMonitor: check if pool is on unmanaged snaps mode on mk/rmsnap
+    
+    Backport: dumpling
+    Fixes: #6047
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit fab79543c54c2e446d3f76520d7906645c6b0075)
+
+commit a992664435db9dde3745eb7f354cce3fc5400a47
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Sep 12 14:32:17 2013 -0700
+
+    lru_map: don't use list::size()
+    
+    replace list::size() with map::size(), which should have
+    a constant time complexity.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 7c1d2ded8fa8061bf3f14932800998b963745dd1)
+
+commit 788546ea71c994ff35323747294ed9c177fe7020
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Sep 12 14:30:19 2013 -0700
+
+    common/lru_map: rename tokens to entries
+    
+    This code was originally used in a token cache, now
+    as a generic infrastructure rename token fields.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 532e41a9985a16b35a6e49cdcba38af0ad166fa8)
+
+commit babeb00c42af760b3e7575166479e95365cfcc0a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 18 10:37:21 2013 -0700
+
+    rgw: use bufferlist::append() instead of bufferlist::push_back()
+    
+    push_back() expects char *, whereas append can append a single char.
+    Appending a NULL char to push_back is cast as a NULL pointer which is
+    bad.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 08fe028bad13096d482454a2f303158727c363ff)
+
+commit daf85c45dd4d158bc7c33a2fb784857bc7db35cd
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 13:46:31 2013 -0700
+
+    rgw: NULL terminate buffer before parsing it
+    
+    Fixes: #6175
+    Backport: dumpling
+    We get a buffer off the remote gateway which might
+    not be NULL terminated. The JSON parser needs the
+    buffer to be NULL terminated even though we provide
+    a buffer length as it calls strlen().
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e7f7483192cddca1159aba439ce62b1e78669d51)
+
+commit c73040a5518971813b9ebaae1624c5bacef315d0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 22:30:12 2013 -0700
+
+    rgw: don't call list::size() in ObjectCache
+    
+    Fixes: #6286
+    Use an external counter instead of calling list::size()
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31e3a51e933429d286104fe077e98ea883437ad6)
+
+commit a855aba9d18936e9a060119e041518790cd4b831
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 10 12:18:55 2013 -0700
+
+    rgw: drain pending requests before completing write
+    
+    Fixes: #6268
+    When doing aio write of objects (either regular or multipart parts) we
+    need to drain pending aio requests. Otherwise if gateway goes down then
+    object might end up corrupted.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 626669afaa333d73707553a85f5c874e99e9cbd8)
+
+commit 670db7e80ddc9c26c43a4f66907a5996ce207c4d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 6 22:33:38 2013 -0700
+
+    rgw: fix get cors, delete cors
+    
+    Remove a couple of variables that overrode class member. Not
+    really clear how it was working before, might have been a bad
+    merge / rebase.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 13872785aeeddbe1b8dd97e49fd6a2d879514f8d)
+
+commit a304016fa01b02efd500135c00b9bf3407a9999c
+Merge: 408cd61 ac0a30f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 09:47:10 2013 -0700
+
+    Merge branch 'wip-6078-dumpling' into dumpling
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit ac0a30feb8c64a3b80d9c519a7b561213403afab
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 28 21:25:20 2013 -0700
+
+    rgw: fix certain return status cases in CORS
+    
+    Change return values in certain cases, reorder
+    checks, etc.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 13b28cc3f1eb8ef42875b630c485ee0105cd244a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 28 21:24:36 2013 -0700
+
+    rgw: add COPY method to be handled by CORS
+    
+    Was missing this http method.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d45c87ea738807487e72c0719b0d3d459cbe19e9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 27 19:38:45 2013 -0700
+
+    rgw: fix CORS rule check
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 986fa92a7a1d88111ba28457160adfcfdaabc5d2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 27 19:38:18 2013 -0700
+
+    rgw: don't handle CORS if rule not found (is NULL)
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 71873aba6553492d3ad71596cefd7c841030a277
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 13:38:55 2013 -0700
+
+    rgw: tie CORS header response to all relevant operations
+    
+    Have the CORS responses on all relevant operations. Also add headers
+    on failure cases.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 94e7b594d85dbd26e58d823b41f418032e9f163f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 10:00:53 2013 -0700
+
+    rgw: add a generic CORS response handling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit c3385d8a102faf5379559bb98cf89637ceda1579
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 21 17:22:46 2013 -0700
+
+    rgw: OPTIONS request doesn't need to read object info
+    
+    This is a bucket-only operation, so we shouldn't look at the
+    object. Object may not exist and we might respond with Not
+    Exists response which is not what we want.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a5fdd44e5d8ce4b8d82273d83e27aea19e63aa7c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 21 14:43:28 2013 -0700
+
+    rgw: remove use of s->bucket_cors
+    
+    Some old code still tried to use s->bucket_cors, which was
+    abandoned in a cleanup work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.67.5.txt b/doc/changelog/v0.67.5.txt
new file mode 100644
index 000000000..2767fb85e
--- /dev/null
+++ b/doc/changelog/v0.67.5.txt
@@ -0,0 +1,711 @@
+commit a60ac9194718083a4b6a225fc17cad6096c69bd1
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Dec 19 20:53:10 2013 +0000
+
+    v0.67.5
+
+commit 8745ba63533ba77b2ec9d40780cad822a577089c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:11:01 2013 -0800
+
+    rgw: fix use-after-free when releasing completion handle
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8890ab2d46fe8e12200a0d2f9eab31c461fb871)
+
+commit 7587ee52161d74a41bf89dc741e8ece9c964fb67
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:10:21 2013 -0800
+
+    rgw: don't return data within the librados cb
+    
+    Fixes: #7030
+    The callback is running within a single Finisher thread, thus we
+    shouldn't block there. Append read data to a list and flush it within
+    the iterate context.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)
+
+commit 9bd1ebe960aae69f6528a33809a58c13481f8df3
+Author: Christophe Courtaut <christophe.courtaut@gmail.com>
+Date:   Mon Jul 22 15:15:38 2013 +0200
+
+    rgw: Allow wildcard in supported keystone roles.
+    
+    http://tracker.ceph.com/issues/4365 fixes #4365
+    
+    Signed-off-by: Christophe Courtaut <christophe.courtaut@gmail.com>
+    (cherry picked from commit 60195581273aee259e8c83d0b471af859d928342)
+
+commit e52b9b76844f935daf2dc060905f869248bba60b
+Author: Roald J. van Loon <roaldvanloon@gmail.com>
+Date:   Fri Aug 9 13:31:10 2013 +0200
+
+    Validate S3 tokens against Keystone
+    
+    - Added config option to allow S3 to use Keystone auth
+    - Implemented JSONDecoder for KeystoneToken
+    - RGW_Auth_S3::authorize now uses rgw_store_user_info on keystone auth
+    - Minor fix in get_canon_resource; dout is now after the assignment
+    
+    Reviewed-by: Yehuda Sadeh<yehuda@inktank.com>
+    Signed-off-by: Roald J. van Loon <roaldvanloon@gmail.com>
+    (cherry picked from commit a200e184b15a03a4ca382e94caf01efb41cb9db7)
+    
+    Conflicts:
+    	src/rgw/rgw_swift.h
+
+commit 5df2f35ee5797ffe85e8398b1ecf75f2eefb5031
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 18:02:32 2013 -0700
+
+    rgw: init src_bucket_name, src_object in the S3 handler
+    
+    Be consistent and initialize these fields also in the S3 case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ccaab2abf9e49ce3e50c15df793a3a3ca6b29bb8)
+
+commit 0edee8e9c9b140a280dc782b491c330c7f86edfc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 14:45:59 2013 -0700
+
+    rgw: get rid of req_state.bucket_name
+    
+    No need for this field, as we already have req_state.bucket_name_str.
+    This saves us some memory allocation / freeing through every request
+    processing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit db7eb771226beaa2372ef5860ce742cb3457cd89)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit 0eae05f31e4a03e923d51521ecd0078657eda9ba
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 14:17:12 2013 -0700
+
+    rgw: turn swift COPY into PUT
+    
+    Fixes: #6606
+    The swift COPY operation is unique in a sense that it's a write
+    operation that has its destination not set by the URI target, but by a
+    different HTTP header. This is problematic as there are some hidden
+    assumptions in the code that the specified bucket/object in the URI is
+    the operation target. E.g., certain initialization functions, quota,
+    etc. Instead of creating a specialized code everywhere for this case
+    just turn it into a regular copy operation, that is, a PUT with
+    a specified copy source.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit cbf8f9a51737e7d89fb0ec0832b50fd47c35b08d)
+
+commit 455f28a78ee22b6a535831ff70a330fa1379567c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 20:32:14 2013 -0700
+
+    OSDMonitor: add 'osd perf' command to dump recent osd perf information
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 94c3f29a32cbf169d896015da6765febd3c724e0)
+
+commit ec086df1beccda6d5a6c12c727c84c2555b0b2cd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 17:36:57 2013 -0700
+
+    ObjectStore: add ObjectStore::get_cur_stats and impl in FileStore
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ebde89d5602536b4bc651737e4280fdfb6634c32)
+
+commit 5bf7e7be1fa39ec39726635581e98a0e3a1df20f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 17:21:39 2013 -0700
+
+    osd_types: add osd_perf_stat_t type
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 95f3353a6ab3a4dd2bf28eaca7122487942e913e)
+
+commit b37c5d208778ee52292757e96265d59d75f19654
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Dec 3 11:22:56 2013 -0800
+
+    rgw: do not use UNPRIVILEGED_DAEMON option
+    
+    This is sort of a backport for 4f403c26dc0048ad63e20d20369fa86bfb31c50e
+    in that we get the same behavior change, but only for rgw.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit a447fb7d04fbad84f9ecb57726396bb6ca29d8f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Dec 3 21:51:26 2013 -0800
+
+    osd/OSDMonitor: accept 'osd pool set ...' value as string
+    
+    Newer monitors take this as a CephString.  Accept that so that if we are
+    mid-upgrade and get a forwarded message using the alternate schema from
+    a future mon we will handle it properly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 9875c8b1992c59cc0c40901a44573676cdff2669
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 25 13:43:43 2013 -0800
+
+    init, upstart: prevent daemons being started by both
+    
+    There can be only one init system starting a daemon. If there is a
+    host entry in ceph.conf for a daemon, sysvinit would try to start it
+    even if the daemon's directory did not include a sysvinit file. This
+    preserves backwards compatibility with older installs using sysvinit,
+    but if an upstart file is present in the daemon's directory, upstart
+    will try to start them, regardless of host entries in ceph.conf.
+    
+    If there's an upstart file in a daemon's directory and a host entry
+    for that daemon in ceph.conf, both sysvinit and upstart would attempt
+    to manage it.
+    
+    Fix this by only starting daemons if the marker file for the other
+    init system is not present. This maintains backwards compatibility
+    with older installs using neither sysvinit or upstart marker files,
+    and does not break any valid configurations. The only configuration
+    that would break is one with both sysvinit and upstart files present
+    for the same daemon.
+    
+    Backport: emperor, dumpling
+    Reported-by: Tim Spriggs <tims@uahirise.org>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5e34beb61b3f5a1ed4afd8ee2fe976de40f95ace)
+
+commit bd08d332f5d23b6097a7e8344a6183896b2303a9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Nov 3 21:02:36 2013 -0800
+
+    OSD: allow project_pg_history to handle a missing map
+    
+    If we get a peering message for an old map we don't have, we
+    can throwit out: the sending OSD will learn about the newer
+    maps and update itself accordingly, and we don't have the
+    information to know if the message is valid. This situation
+    can only happen if the sender was down for a long enough time
+    to create a map gap and its PGs have not yet advanced from
+    their boot-up maps to the current ones, so we can rely on it
+    
+    Fixes: #6712
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit cd0d612e1abdf5c87082eeeccd4ca09dd14fd737)
+
+commit 8cd33e3a8ebf7c2aa796ec9f92d6b554c39ff705
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 19 16:56:27 2013 -0700
+
+    rgw: bucket meta remove don't overwrite entry point first
+    
+    Fixes: #6056
+    
+    When removing a bucket metadata entry we first unlink the bucket
+    and then we remove the bucket entrypoint object. Originally
+    when unlinking the bucket we first overwrote the bucket entrypoint
+    entry marking it as 'unlinked'. However, this is not really needed
+    as we're just about to remove it. The original version triggered
+    a bug, as we needed to propagate the new header version first (which
+    we didn't do, so the subsequent bucket removal failed).
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 0373d749cea7d9b532069ba8ebca2f005b2c9f59)
+
+commit a24f1326adcdda1d387e0f4e3c18dda5ec4aa055
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 07:04:01 2013 -0800
+
+    rgw: lower some debug message
+    
+    Fixes: #6084
+    Backport: dumpling, emperor
+    
+    Reported-by: Ron Allred <rallred@itrefined.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b35fc1bc2ec8c8376ec173eb1c3e538e02c1694e)
+
+commit 9e2d0d311437c1ffdd43fc8c961c60aacfea339c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 18 14:39:12 2013 -0800
+
+    osd: fix bench block size
+    
+    The command was declared to take 'size' in dumpling, but was trying to
+    read 'bsize' instead, so it always used the default of 4MiB. Change
+    the bench command to read 'size', so it matches what existing clients
+    are sending.
+    
+    Fixes: #6795
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 40a76ef0d09f8ecbea13712410d9d34f25b91935)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 35c91cac9a2b414ba3d52ea06cf6f71a0a1d4ba3
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Sep 25 09:19:16 2013 -0700
+
+    os, osd, tools: Add backportable compatibility checking for sharded objects
+    
+    OSD
+      New CEPH_OSD_FEATURE_INCOMPAT_SHARDS
+    FileStore
+      NEW CEPH_FS_FEATURE_INCOMPAT_SHARDS
+      Add FSSuperblock with feature CompatSet in it
+      Store sharded_objects state using CompatSet
+      Add set_allow_sharded_objects() and get_allow_sharded_objects() to FileStore/ObjectStore
+      Add read_superblock()/write_superblock() internal filestore functions
+    ceph_filestore_dump
+      Add OSDsuperblock to export format
+      Use CompatSet from OSD code itself in filestore-dump tool
+      Always check compatibility of OSD features with on-disk features
+      On import verify compatibility of on-disk features with export data
+      Bump super_ver due to export format change
+    
+    Backport: dumpling, cuttlefish
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c6b83180f9f769de27ca7890f5f8ec507ee743ca)
+    
+    Conflicts:
+    
+    	src/os/FileStore.cc
+    	src/os/FileStore.h
+    	src/osd/OSD.cc
+    
+    Excluded from cherry-pick:
+      Didn't add set_allow_sharded_objects() and get_allow_sharded_objects() to FileStore/ObjectStore
+      Didn't add code to check for incomplete transition to sharded objects in ceph-filestore-dump
+
+commit 372f62717c56d9ab883ae2942e13d6d8d37c2925
+Author: Ray Lv <xiangyulv@gmail.com>
+Date:   Tue Oct 29 19:34:51 2013 +0800
+
+    rgw: Wrap hex_to_num table into class HexTable
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Ray Lv <xiangyulv@gmail.com>
+    (cherry picked from commit 588ed60a8ec1d8db5e096fd8f7b762f2afcf7dd3)
+
+commit 6917b02530103b8c86ed75592da33144b0dea168
+Merge: 7ef210a f1fa811
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Nov 5 12:27:42 2013 -0800
+
+    Merge pull request #805 from ceph/wip-rgw-replica-log-dumpling
+    
+    backport a bunch of rgw stuff to dumpling
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit f1fa8116d441924d44c99624829f3daa090c821c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Nov 1 16:12:52 2013 -0700
+
+    rgw: don't turn 404 into 400 for the replicalog api
+    
+    404 is not actually a problem to clients like radosgw-agent, but 400
+    implies something about the request was incorrect.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 306ec7142d699c26ce874b11dc02ccdb3cf296c7)
+
+commit 739e7861f6d5fde5c4a2b59170377b6147fecfa3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 3 13:27:21 2013 -0700
+
+    rgw: when failing read from client, return correct error
+    
+    Fixes: #6214
+    When getting a failed read from client when putting an object
+    we returned the wrong value (always 0), which in the chunked-
+    upload case ended up in assuming that the write was done
+    successfully.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0499948aad64c6c66668b43963403fb0cb1a2737)
+
+commit 465087e64ab2bd85d7f5e183565e3619b3aa76fc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 30 14:15:19 2013 -0700
+
+    rgw: fix leak in RGWMetadataManager::remove()
+    
+    Backport: dumpling
+    Fixes: #6445
+    handler->get() returns a newly allocated obj, need to delete it when
+    done.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 8dd7ea3fadc06b5ebb330af41d686d3da155fb5d)
+
+commit a4dc2694db6a7e45109f111dcdfd077da67b6030
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 11:16:08 2013 -0700
+
+    rgw: quiet down warning message
+    
+    Fixes: #6123
+    We don't want to know about failing to read region map info
+    if it's not found, only if failed on some other error. In
+    any case it's just a warning.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 055e31359ac977943f014270cc56a1b7786abee3)
+
+commit 7f57d9dda74a1feb2da85be3c77d61c6f81ec1b2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 13 14:43:54 2013 -0700
+
+    rgw: try to create log pool if doesn't exist
+    
+    When using replica log, if the log pool doesn't exist all operations are
+    going to fail. Try to create it if doesn't exist.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 4216eac0f59af60f60d4ce909b9ace87a7b64ccc)
+
+commit 4fb233cf702957e782083d760877afa48d4f6965
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Oct 11 16:56:46 2013 -0700
+
+    formatter: dump_bool dumps unquoted strings
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ad409f8a6d230e9b1199226a333bb54159c2c910)
+
+commit de7ede925acc47a5aee84b9fe82b9cf0368aa1b2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 27 13:15:35 2013 -0700
+
+    Formatter: add dump_bool()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 28949d5d43beba7cef37cb2f83e3399d978061a6)
+
+commit c5991f26febd86878aeb78baab33c071660fdee6
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:42:48 2013 -0700
+
+    rgw: escape bucket and object names in StreamReadRequests
+    
+    This fixes copy operations for objects that contain unsafe characters,
+    like a newline, which would return a 403 otherwise, since the GET to
+    the source rgw would be unable to verify the signature on a partially
+    valid bucket name.
+    
+    Fixes: #6604
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ec45b3b88c485140781b23d2c4f582f2cc26ea43)
+
+commit 672679d26110dd93393bc18a681ae41175e4a299
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:37:25 2013 -0700
+
+    rgw: move url escaping to a common place
+    
+    This is useful outside of the s3 interface. Rename url_escape()
+    url_encode() for consistency with the exsting common url_decode()
+    function. This is in preparation for the next commit, which needs
+    to escape url-unsafe characters in another place.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit dd308cd481b368f90a64220847b91fc233d92a59)
+
+commit 91997fca6ec6c06beecd63c3e2ae521319fca7c2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:34:24 2013 -0700
+
+    rgw: update metadata log list to match data log list
+    
+    Send the last marker whether the log is truncated in the same format
+    as data log list, so clients don't have more needless complexity
+    handling the difference.  Keep bucket index logs the same, since they
+    contain the marker already, and are not used in exactly the same way
+    metadata and data logs are.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e0e8fb1b2b4a308b2a9317e10c6fd53ad48dbfaf)
+
+commit 8916648521e2362a54eed0f5ebe078a269e4d703
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:26:19 2013 -0700
+
+    rgw: include marker and truncated flag in data log list api
+    
+    Consumers of this api need to know their position in the log. It's
+    readily available when fetching the log, so return it.  Without the
+    marker in this call, a client could not easily or efficiently figure
+    out its position in the log, since it would require getting the global
+    last marker in the log, and then reading all the log entries.
+    
+    This would be slow for large logs, and would be subject to races that
+    would cause potentially very expensive duplicate work.
+    
+    Returning this atomically while fetching the log entries simplifies
+    all of this.
+    
+    Fixes: #6615
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c275912509255f8bb4c854e181318b45ab0f8564)
+
+commit a49c064b7a1b4083818b6f950d7e0aade71c6afe
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:18:19 2013 -0700
+
+    cls_log: always return final marker from log_list
+    
+    There's no reason to restrict returning the marker to the case where
+    less than the whole log is returned, since there's already a truncated
+    flag to tell the client what happened.
+    
+    Giving the client the last marker makes it easy to consume when the
+    log entries do not contain their own marker. If the last marker is not
+    returned, the client cannot get the last marker without racing with
+    updates to the log.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e74776f4176470122485a79a4c07e9c12c9fc036)
+
+commit 90101aea6818c2f43bf31243d912ec9143f68882
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 10 15:50:39 2013 -0700
+
+    rgw: skip read_policy checks for system_users
+    
+    A system user should still be able to examine suspended buckets, and
+    get -ENOENT instead of -EACCESS for a deleted object.
+    
+    Fixes: #6616
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ea816c1c2fd47eab647d6fab96c9ca4bfeecd5bb)
+
+commit 7ef210adbd4645bd45e5387c9b03a122a7ca82df
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 29 08:39:27 2013 -0700
+
+    common/crc32c: fix #ifdef to be x86_64 specific
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 29f4eebf63bcba9ea86b9a2e4d093b782b5fbf1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 08:05:38 2013 +0200
+
+    rgw: fix rgw test to reflect usage change
+    
+    otherwise src/test/cli/radosgw-admin/help.t fails when running make
+    check when run after a configure --with-radosgw
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit e50343e4423e20130035c860ba47a0edea876f7c)
+
+commit 0106569a88a92aa94fbc5efc6a0a142ad0de325d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 17:25:16 2013 -0700
+
+    rbd.py: increase parent name size limit
+    
+    64 characters isn't all that long. 4096 ought to be enough for anyone.
+    
+    Fixes: #6072
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3c0042cde5a12de0f554a16b227ab437c6254ddd)
+
+commit 352d8c966c323a008362130eb979ae5dfa47494f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 14:10:36 2013 -0700
+
+    common/config: include --cluster in default usage message
+    
+    Clean it up a bit too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 157754b3a0191c5ff534a84adbeed88025615898)
+
+commit 299ddd31b29e332dc5e76bc4f871e4769698665d
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Oct 10 10:35:48 2013 +0800
+
+    mds: fix infinite loop of MDCache::populate_mydir().
+    
+    make MDCache::populate_mydir() only fetch bare-bone stray dirs.
+    After all stray dirs are populated, call MDCache::scan_stray_dir(),
+    it fetches incomplete stray dirs.
+    
+    Fixes: #4405
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 007f06ec174d4ee5cfb578c8b3f1c96b2bb0c238)
+    
+    Conflicts:
+    
+    	src/mds/MDCache.h
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 7d99940467019bda74da83f3f1ebc2da46bd6967
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:20:48 2013 -0700
+
+    rgw: fix authenticated users acl group check
+    
+    Fixes: #6553
+    Backport: bobtail, cuttlefish, dumpling
+    Authenticated users group acl bit was not working correctly. Check to
+    test whether user is anonymous was wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bebbd6cb7b71697b34b8f27652cabdc40c97a33b)
+
+commit e5736de834027a281d29afcf6743347a8db97d26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:55:07 2013 -0700
+
+    rgw: change default log level
+    
+    Fixes: #6554
+    Backport: cuttlefish, dumpling
+    Default log level was just too high, bring it down a bit.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8d7dbf85472cfca9268d81ecf057ea078cf345b3)
+
+commit e6e9ee7145bcfc1774af738af607a653fe8dfe46
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Oct 4 13:00:26 2013 -0700
+
+    rgw: swift update obj metadata also add generic attrs
+    
+    Fixes: #6462
+    We were missing the generic attributes when we updated the object
+    metadata (operation that only exists in the swift api).
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f2645e1c6d7383a0ace3b239f4304e353249c4bb)
+
+commit cd1c3c9e00e90b19e83c1f11a48e516a7de93665
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Wed Oct 2 11:43:12 2013 +0800
+
+    mds: return -EAGAIN if standby replay falls behind
+    
+    standby replay may fall behind and get -ENOENT when reading the
+    journal. return -EAGAIN in this case, it makes the MDS respawn itself.
+    
+    fixes: #5458
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d2cb2bf6bac83ac6db9df6cb876317d30e7493cc)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit bd073eeac28d8cba969e5746c5e6adcb95820fdf
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 09:50:51 2013 +0800
+
+    mon/MDSMonitor: don't reset incarnation when creating newfs
+    
+    Fixes: #6279
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 40613b700b87b495d67793101ae05d6ba58b2a9a)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 3a3724b782c53c92c45624ad02df9f0de77f1545
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 10:36:39 2013 +0800
+
+    osdc/ObjectCacher: finish contexts after dropping object reference
+    
+    The context to finish can be class C_Client_PutInode, which may drop
+    inode's last reference. So we should first drop object's reference,
+    then finish contexts.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit b66ac77fa7aa3ff37804918c4308a348f239af09)
+
+commit 102f63993cacf3b3404256e62a3096c5836f2282
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Oct 8 12:08:08 2013 -0700
+
+    Go back to $PWD in fsstress.sh if compiling from source.
+    
+    Although fsstress was being called with a static path the directory
+    it was writing to was in the current directory so doing a cd to the
+    source directory that is made in /tmp and then removing it later
+    caused it to be unable to write the files in a non-existent dir.
+    
+    This change gets the current path first and cd's back into it after
+    it is done compiling fsstress.
+    
+    Issue #6479.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit 62db186a19f902b0a23d14034552c83db8de7ece
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 09:53:12 2013 -0700
+
+    ceph.spec.in:  radosgw package doesn't require mod_fcgi
+    
+    Fixes #5702
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 42fc9b49a7bbc242096946b381aafd435ebdd530
+Merge: ad85b8b 67ac3cf
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Oct 4 19:51:32 2013 +0000
+
+    Merge branch 'dumpling' of jenkins:ceph/ceph into dumpling
+
+commit 67ac3cffa1066030ad840937958bd534e8b26272
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 3 21:47:26 2013 -0700
+
+    ceph_test_rados: do not let rollback race with snap delete
+    
+    Note that the OSD behaves in a weird way when you rollback to a non-
+    existent snap, so the test probably isn't the only party at fault here.
+    
+    Fixes (test half of): #6254
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 55d279b98553ba4542219b126fc7159b20b18b1f)
+    
+    Conflicts:
+    
+    	src/test/osd/RadosModel.h
+    	src/test/osd/TestRados.cc
diff --git a/doc/changelog/v0.67.6.txt b/doc/changelog/v0.67.6.txt
new file mode 100644
index 000000000..510893a19
--- /dev/null
+++ b/doc/changelog/v0.67.6.txt
@@ -0,0 +1,1258 @@
+commit c71f8a60cf49fb3d4efb5ea75b2366cd7f167497 (tag: refs/tags/v0.67.6)
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Feb 13 20:20:42 2014 +0000
+
+    release build 67.6
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit 1c42a6e34bb8e318e001384270cda003add946a7
+Merge: 3e473d4 c5e12f9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 13 06:55:37 2014 -0800
+
+    Merge pull request #1232 from ceph/dumpling-7334
+    
+    backport ceph-disk improvements to dumpling
+    
+    http://pulpito.ceph.com/ubuntu-2014-02-12_16:52:33-ceph-deploy-dumpling-7334-testing-basic-plana/
+
+commit 3e473d4237697872498094b41fed19b414f8d3c4
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri Sep 27 16:23:09 2013 -0700
+
+    common,os: Remove filestore_xattr_use_omap option
+    
+    Now we operate just like when this was set to true
+    
+    Fixes: #6143
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit dc0dfb9e01d593afdd430ca776cf4da2c2240a20)
+
+commit c5e12f94efc07f0ff174c0f19494ee2e06d88dfa
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 12 16:43:59 2014 -0500
+
+    add support for absence of PATH
+    
+    Note that this commit is actually bisecting the changes from
+    Loic Dachary that touch ceph-disk only (ad515bf). As that changeset
+    also touches other files it causes conflicts that are not resolvable
+    for backporting it to dumpling.
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+
+commit c531d75f7e86cee619e44c247298b50c37b81539
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 9 22:27:23 2013 -0700
+
+    ceph-disk: make initial journal files 0 bytes
+    
+    The ceph-osd will resize journal files up and properly fallocate() them
+    so that the blocks are preallocated and (hopefully) contiguous.  We
+    don't need to do it here too, and getting fallocate() to work from
+    python is a pain in the butt.
+    
+    Fixes: #5981
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a786ad773cd33880075f1deb3691528d1afd03ec)
+
+commit 836ead3e42491597c192c63d3171021c5daecbc0
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Feb 10 15:07:55 2014 -0500
+
+    alert the user about error messages from partx
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 9bcc42a3e6b08521694b5c0228b2c6ed7b3d312e)
+
+commit cff3547c60dc6d164f9a95eaa2706633e32756d0
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Feb 7 11:55:01 2014 -0500
+
+    use partx for red hat or centos instead of partprobe
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 42900ff9da9f5adcac239a84ebf4d2e407c29699)
+
+commit 5ebd7c4520a9805f985cab1a0ba9311e19efa041
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jan 28 17:26:58 2014 -0800
+
+    ceph-disk: run the right executables from udev
+    
+    When run by the udev rules, PATH is not defined. Thus,
+    ceph-disk-activate relies on its which() function to locate the
+    correct executable.  The which() function used os.defpath if none was
+    set, and this worked for anything using it.
+    
+    ad6b4b4b08b6ef7ae8086f2be3a9ef521adaa88c added a new default value to
+    PATH, so only /usr/bin was checked by callers that did not use
+    which(). This resulted in the mount command not being found when
+    ceph-disk-activate was run by udev, and thus osds failing to start
+    after being prepared by ceph-deploy.
+    
+    Make ceph-disk consistently use the existing helpers (command() and
+    command_check_call()) that use which(), so lack of PATH does not
+    matter. Simplify _check_output() to use command(),
+    another wrapper around subprocess.Popen.
+    
+    Fixes: #7258
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d7b0c7faafd37e4ae8a1680edfa60c22b419cbd8)
+
+commit 4a7c4e0a81753dfb811396e7bfa1df72135ee12b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 3 16:06:55 2014 +0100
+
+    ceph-disk: implement --sysconfdir as /etc/ceph
+    
+    Replace hardcoded /etc/ceph with the SYSCONFDIR global variable and
+    implement the --sysconfdir option to override the default value.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a71025d33621257b6fd6632516cfed2849ff1637)
+
+commit c8b9798a69bd1655a7131c7bce8a14c2c74aba0c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 3 16:03:09 2014 +0100
+
+    ceph-disk: implement --statedir as /var/lib/ceph
+    
+    Replace hardcoded /var/lib/ceph with the STATEDIR global variable and
+    implement the --statedir option to override the default value.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ca713f48ae7a1fece2869f1a1c97d23ab33fb441)
+
+commit 79564874be35d4414092fbb80df3214d7f282b1c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:05:03 2014 +0100
+
+    ceph-disk: add copyright notice
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 51ee3c04777aaf6b9609dde9bc318b5c66c70787)
+
+commit c4b66bbdf09a8bb85ed14fbb643f40f87345463b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:11:30 2014 +0100
+
+    ceph-disk: create the data directory if it does not exist
+    
+    Instead of failing if the OSD data directory does not exist, create
+    it. Only do so if the data directory is not enforced to be a device via
+    the use of the --data-dev flag. The directory is not recursively created.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 306b099ab093bfac466d68fe1cb87367bc01e577)
+
+commit c473c65d4b6235fe20a97bd1d4741c12f86858e8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Dec 31 00:03:42 2013 +0100
+
+    ceph-disk: run ceph-osd when --mark-init=none
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 0fcc081858fae4febbb6a613a93cfbbcedd5a320)
+
+commit 6ea59a3a0c38c9629036442aff652489eecce51a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 23:57:39 2013 +0100
+
+    ceph-disk: implement --mark-init=none
+    
+    It is meant to be used when preparing and activating a directory that is
+    not to be used with init. No file is created to identify the init
+    system, no symbolic link is made to the directory in /var/lib/ceph
+    and the init scripts are not called.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit e773b68f4c89ac56b425c710d7dcdc3d74a92926)
+
+commit 3aae5b764c5698c665a708a3f6f2243dfee1a28d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:07:57 2014 +0100
+
+    ceph-disk: fsid is a known configuration option
+    
+    Use get_conf_with_default instead of get_conf because fsid is a known
+    ceph configuration option. It allows overriding via CEPH_ARGS which is
+    convenient for testing. Only options that are not found in config_opts.h
+    are fetch via get_conf.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b65eb377f5e93ea85644e4c0939365fd7ac36072)
+
+commit a33ad6d150206232806ee466f301e41f15c631e9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:06:29 2014 +0100
+
+    ceph-disk: use CalledProcessError.returncode
+    
+    CalledProcessError has no errno data member
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 97f516a1ddfb2d014e1f7e762c4155e4b9bcb90b)
+
+commit 6ae96d3ce757b05daeb7c1437778149b07248d47
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:13:15 2014 +0100
+
+    ceph-disk: display the command output on OSD creation failure
+    
+    The string form of a CalledProcessError instance does not include the
+    output datamember. Add it to the Error exception for debug purposes.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d09af0fa50f322c9e59765f3badd497f5ca184d4)
+
+commit deefb3a05788b308631812c652e9eccaeb999d7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 23:07:27 2013 +0100
+
+    ceph-disk: which() uses PATH first
+    
+    Instead of relying on a hardcoded set of if paths. Although this has the
+    potential of changing the location of the binary being used by ceph-disk
+    on an existing installation, it is currently only used for sgdisk. It
+    could be disruptive for someone using a modified version of sgdisk but
+    the odds of this happening are very low.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 2b935bbf60bafb6dd488c0eb30f156fce1b9d197)
+
+commit 06bbea187293ac52aefbcf4e98c162bb3286341d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 22:48:46 2013 +0100
+
+    ceph-disk: add --prepend-to-path to control execution
+    
+    /usr/bin is hardcoded in front of some ceph programs which makes it
+    impossible to control where they are located via the PATH.
+    
+    The hardcoded path cannot be removed altogether because it will most
+    likely lead to unexpected and difficult to diagnose problems for
+    existing installations where the PATH finds the program elsewhere.
+    
+    The --prepend-to-path flag is added and defaults to /usr/bin : it prepends
+    to the PATH environment variable. The hardcoded path is removed
+    and the PATH will be used: since /usr/bin is searched first, the
+    legacy behavior will not change.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ad6b4b4b08b6ef7ae8086f2be3a9ef521adaa88c)
+
+commit ccf182241dc04c79e2377b6ba2f060dbfe6dddae
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:15:29 2013 +0100
+
+    ceph-disk: make exception handling terse in main_activate_journal
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 908348b8047e8577ecf9133f2683f91423694416)
+
+commit d60e952f3c00d0e551b04c0d6486b11d9761be35
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:13:47 2013 +0100
+
+    ceph-disk: do not hide main_activate() exceptions
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 55ca7bb2da73f1be1293710a635cfea42abd7682)
+
+commit 827edb93e3971fc7549fe831894776c6bf163346
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:11:30 2013 +0100
+
+    ceph-disk: fix activate() indent
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 324804a81c37ff89f2488e2ba106033c0e6e119e)
+
+commit 27988cf9ffedf3289099e51f1e7c3f5b8d79d667
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:09:35 2013 +0100
+
+    ceph-disk: remove noop try:
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit de0050596b5f56863c3486c1cd5e7ffea62e3d00)
+
+commit 75bdb5430d9403bed8bd2ccd6a9c5a30d63a189e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 16:57:36 2013 +0100
+
+    ceph-disk: fix Error() messages formatting
+    
+    Mainly using % instead of ,
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b82ccfbfa786cd5436b48ec38276c5a48028ce1d)
+
+commit 5df1eaa55bc7ebe68473831d06cf9819fa39baa6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 12:26:20 2013 +0100
+
+    ceph-disk: prepare --data-dir must not override files
+    
+    ceph-disk does nothing when given a device that is already prepared. If
+    given a directory that already contains a successfully prepared OSD, it
+    will however override it.
+    
+    Instead of overriding the files in the osd data directory, return
+    immediately if the magic file exists. Make it so the magic file is
+    created last to accurately reflect the success of the OSD preparation.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7dfe550ce18623cde4ae43a2416e31ef81381ab9)
+
+commit 2e8496a6e0b74ff351bc2a4f138ddc665d8c30ca
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 15:34:17 2013 +0100
+
+    ceph-disk: zap needs at least one device
+    
+    If given no argument, ceph-disk zap should display the usage instead of
+    silently doing nothing. Silence can be confused with "I zapped all the
+    disks".
+    
+    http://tracker.ceph.com/issues/6981 fixes #6981
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 07888ef3fd4440332c8287d0faa9f23a32cf141c)
+
+commit 0fe2ac47a82d3caa9dca2a9009b68a14b577e35c
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Dec 13 12:06:25 2013 -0500
+
+    use the new get_command helper in check_call
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 897dfc113fe3b86f3dda53172933bfd4f8089869)
+
+commit 8fc9f1b791248f08a311d619addc4465cac298f9
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 12 11:16:38 2013 -0500
+
+    use the absolute path for executables if found
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit a9334a1c8c6681305e76b361377864d0dd1e3d34)
+
+commit a2de2f718c9e341c0b6ec2ebffdf03d0f9da0fbd
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 12 10:26:05 2013 -0500
+
+    remove trailing semicolon
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 43561f791607f5fd6f03d5421e1f30a29fb4194e)
+
+commit 2cd22b5d39760ac4b5baad0b05c676b833ae9213
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Dec 11 15:41:45 2013 -0500
+
+    replace sgdisk subprocess calls with a helper
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit e19e38012bc4579054f63865e682c8c3a7829c7b)
+
+commit e8cc27cc8eb981c71b8cafb58964317687ca5787
+Author: Jonathan Davies <jonathan.davies@canonical.com>
+Date:   Tue Dec 3 21:26:43 2013 +0000
+
+    Call --mbrtogpt on journal run of sgdisk should the drive require a GPT table.
+    
+    Signed-off-by: Jonathan Davies <jonathan.davies@canonical.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35011e0b01d65e4c001876882d597451f2028345)
+
+commit 3942a93fae3331b2d08f93181eb2c9cf90b50440
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 21 13:21:22 2013 +0100
+
+    ceph-disk: blacklist /dev/fd0
+    
+    blkid -s TYPE /dev/fd0 has been verified to hang forever on a
+    H8DMR-82 supermicro motherboard running
+    
+    3.8.0-33-generic #48~precise1-Ubuntu SMP Thu Oct 24 16:28:06 UTC 2013
+    x86_64
+    
+    It is unlikely that ceph will ever be used on floppy disks, they
+    can be blacklisted.
+    
+    http://tracker.ceph.com/issues/6827 fixes: #6827
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 65701978715237ff5a4c68573c0696fd9d438e4f)
+
+commit ccc6b0c5e6d47eefef5cf1fa17bf69c47c9459c4
+Author: git-harry <git-harry@live.co.uk>
+Date:   Thu Oct 3 10:32:50 2013 +0100
+
+    Make fsid comparison case-insensitive
+    
+    get_fsid and find_cluster_by_uuid are modified so ceph-disk activate and
+    ceph-disk activate-all will work if the fsid uses uppercase characters.
+    
+    Signed-off-by: Harry Harrington <git-harry@live.co.uk>
+    (cherry picked from commit 22f8325dbfce7ef2e97bf015c0f8bba53e75dfe9)
+
+commit 367cf1bbf86233eb20ff2304e7d6caab77b84fcc
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 10:14:36 2014 -0800
+
+    librbd: remove limit on number of objects in the cache
+    
+    The number of objects is not a significant indicated of when data
+    should be written out for rbd. Use the highest possible value for
+    number of objects and just rely on the dirty data limits to trigger
+    flushing. When the number of objects is low, and many start being
+    flushed before they accumulate many requests, it hurts average request
+    size and performance for many concurrent sequential writes.
+    
+    Fixes: #7385
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 0559d31db29ea83bdb6cec72b830d16b44e3cd35)
+
+commit 4856fc2950442fafc208c967af5131f7874f5441
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 11:53:00 2014 -0800
+
+    ObjectCacher: use uint64_t for target and max values
+    
+    All the options are uint64_t, but the ObjectCacher was converting them
+    to int64_t. There's never any reason for these to be negative, so
+    change the type.
+    
+    Adjust a few conditionals so that they only convert known-positive
+    signed values to uint64_t before comparing with the target and max
+    values. Leave the actual stats accounting as loff_t for now, since
+    bugs in accounting will have bad effects if negative values wrap
+    around.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit db034acf546a72739ff6543241543f3bd651f3ae)
+
+commit a572aafe178dfa3ab4d82aa5c7f4a72a63590ccf
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 10:35:14 2014 -0800
+
+    ObjectCacher: remove max_bytes and max_ob arguments to trim()
+    
+    These are never passed, so replace them with the defaults.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit bf8cf2d6d21a204a099347f3dcd5b48100b8c445)
+
+commit 4889234f80dbd95e5b2012ff9bfc2d87960034e2
+Merge: 6af657b 30a604d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 11 08:32:37 2014 -0800
+
+    Merge pull request #1210 from dachary/dumpling
+    
+    common: admin socket fallback to json-pretty format (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 30a604d2d6a5de7bcbfa0136e9a766cb34f96c0a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 10 23:42:38 2014 +0100
+
+    common: admin socket fallback to json-pretty format
+    
+    If the format argument to a command sent to the admin socket is not
+    among the supported formats ( json, json-pretty, xml, xml-pretty ) the
+    new_formatter function will return null and the AdminSocketHook::call
+    function must fall back to a sensible default.
+    
+    The CephContextHook::call and HelpHook::call failed to do that and a
+    malformed format argument would cause the mon to crash. A check is added
+    to each of them and fallback to json-pretty if the format is not
+    recognized.
+    
+    To further protect AdminSocketHook::call implementations from similar
+    problems the format argument is checked immediately after accepting the
+    command in AdminSocket::do_accept and replaced with json-pretty if it is
+    not known.
+    
+    A test case is added for both CephContextHook::call and HelpHook::call
+    to demonstrate the problem exists and is fixed by the patch.
+    
+    Three other instances of unsafe calls to new_formatter were found and
+    a fallback to json-pretty was added. All other calls have been audited
+    and appear to be safe.
+    
+    http://tracker.ceph.com/issues/7378 fixes #7378
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 165e76d4d03ffcc490fd3c2ba60fb37372990d0a)
+
+commit 6af657bd4cf87f6f7afe84257941e7e2366183d9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:26:02 2014 -0800
+
+    qa: add script for testing rados client timeout options
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9e62beb80b6c92a97ec36c0db5ea39e417661b35)
+
+commit 21a0b67440eaa07fa051d26321a88654ee9e8ae7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:25:24 2014 -0800
+
+    rados: check return values for commands that can now fail
+    
+    A few places were not checking the return values of commands, since
+    they could not fail before timeouts were added.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 79c1874346ff55e2dc74ef860db16ce70242fd00)
+
+commit 2c42dd26b49d01922bd3df3ba26af70da0387a99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:24:16 2014 -0800
+
+    librados: check and return on error so timeouts work
+    
+    Some functions could not previously return errors, but they had an
+    int return value, which can now receive ETIMEDOUT.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 8e9459e897b1bc2f97d52ee07701fd22069efcf3)
+
+commit 28fe564cc298b96c063e6cca44304046a86942cd
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:22:14 2014 -0800
+
+    msg/Pipe: add option to restrict delay injection to specific msg type
+    
+    This makes it possible to test timeouts reliably by delaying certain
+    messages effectively forever, but still being able to e.g. connect and
+    authenticate to the monitors.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d389e617c1019e44848330bf9570138ac7b0e5d4)
+
+commit 1d2ea9efb844cb902d282c3a752acc45fefeb04c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 18:30:00 2014 -0800
+
+    MonClient: add a timeout on commands for librados
+    
+    Just use the conf option directly, since librados is the only caller.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 671a76d64bc50e4f15f4c2804d99887e22dcdb69)
+
+commit 30dafacd0b54bb98b01284851e0d5abf76324e95
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 17:59:21 2014 -0800
+
+    Objecter: implement mon and osd operation timeouts
+    
+    This captures almost all operations from librados other than mon_commands().
+    
+    Get the values for the timeouts from the Objecter constructor, so only
+    librados uses them.
+    
+    Add C_Cancel_*_Op, finish_*_op(), and *_op_cancel() for each type of
+    operation, to mirror those for Op. Create a callback and schedule it
+    in the existing timer thread if the timeouts are specified.
+    
+    Fixes: #6507
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3e1f7bbb4217d322f4e0ece16e676cd30ee42a20)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	src/osd/ReplicatedPG.cc
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit 2aa0d7589283fe5879a966601fbfe690816cc4a1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 12:53:15 2014 -0800
+
+    librados: add timeout to wait_for_osdmap()
+    
+    This is used by several pool operations independent of the objecter,
+    including rados_ioctx_create() to look up the pool id in the first
+    osdmap.
+    
+    Unfortunately we can't just rely on WaitInterval returning ETIMEDOUT,
+    since it may also get interrupted by a signal, so we can't avoid
+    keeping track of time explicitly here.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1829d2c9fd13f2cbae4e192c9feb553047dad42c)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+
+commit 55d254d5ea113d8be29afe18dc9940f14d93ebb4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 12:09:34 2014 -0800
+
+    conf: add options for librados timeouts
+    
+    These will be implemented in subsequent patches.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 0dcceff1378d85ca6d81d102d201890b8a71af6b)
+
+commit ccd9b5eb9bcfe2360e874b306d4decd8815ecf46
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 9 20:53:00 2014 -0800
+
+    test_striper: fix warning
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit bb44c0d5e06c2b699c81a76373a0edde1bf90445
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 8 12:21:26 2014 -0800
+
+    crushtool: add cli test for off-by-one tries vs retries bug
+    
+    See bug #7370.  This passes on dumpling and breaks prior to the #7370 fix.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ed32c4002fb5cb1dd546331651eaf7de1a017471)
+
+commit c301ec689caac5bbcd21e02226045d3454cc82fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 3 08:54:14 2014 -0800
+
+    client: use 64-bit value in sync read eof logic
+    
+    The file size can jump to a value that is very much larger than our current
+    position (for example, it could be a disk image file that gets a sparse
+    write at a large offset).  Use a 64-bit value so that 'some' doesn't
+    overflow.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit 7ff2b541c24d1c81c3bcfbcb347694c2097993d7)
+
+commit 183deb899bc6b1b7b2a1ec639425e45786e56b01
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:39:35 2014 -0800
+
+    osd: do not send peering messages during init
+    
+    Do not send any peering messages while we are still working our way
+    through init().
+    
+    Fixes: #7093
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 35da8f9d80e0c6c33fb6c6e00f0bf38f1eb87d0e)
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 3a1fb17401897b0717cf1d264c6e6fdf99fc703f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 18:46:37 2014 -0800
+
+    OSDMap: fix deepish_copy_from
+    
+    Start with a shallow copy!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d0f13f54146694a197535795da15b8832ef4b56f)
+    
+    Conflicts:
+    
+    	src/osd/OSDMap.h
+
+commit f01bd8a94adc4ce8e7131f74282de5b434a7316a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 23 13:48:28 2014 -0800
+
+    rgw: fix listing of multipart upload parts
+    
+    Fixes: #7169
+    
+    There are two issues here. One is that we may return more entries than
+    we should (as specified by max_parts). Second issue is that the
+    NextPartNumberMarker is set incorrectly. Both of these issues mainly
+    affect uploads with > 1000 parts, although can be triggered with less
+    than that.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 6b657992d2397e399eacf554357ed807fda399fa
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 21 06:52:48 2013 -0800
+
+    rgw: initialize RGWUserAdminOpState::system_specified
+    
+    Fixes: #6829
+    Backport: dumpling, emperor
+    We didn't init this member variable, which might cause that when
+    modifying user info that has this flag set the 'system' flag might
+    inadvertently reset.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 561e7b0b287e65e90b80699e45a52ae44e94684f)
+
+commit 585e0e7eec1bbee60fe352166b593d53476003f8
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 12:26:19 2013 -0800
+
+    rgw: Fix CORS allow-headers validation
+    
+    This fix is needed because Ceph presently validates CORS headers in a
+    case-sensitive manner. Keeps a local cache of lowercased allowed headers
+    to avoid converting the allowed headers to lowercase each time.
+    
+    CORS 6.2.6: If any of the header field-names is not a ASCII
+    case-insensitive match for any of the values in list of headers do not
+    set any additional headers and terminate this set of steps.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31b60bfd9347a386ff12b4e4f1812d664bcfff01)
+
+commit da2267a87e37afa28385e915a566de26fd784b70
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 11:40:31 2013 -0800
+
+    rgw: Clarify naming of case-change functions
+    
+    It is not clear that the lowercase_http_attr & uppercase_http_attr
+    functions replace dashes with underscores. Rename them to match the
+    pattern established by the camelcase_dash_http_attr function in
+    preperation for more case-change functions as needed by later fixes.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6a7edab2673423c53c6a422a10cb65fe07f9b235)
+
+commit fdea76ef6f7d7c4ee89c30d430f5495072023957
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 11:27:49 2013 -0800
+
+    rgw: Look at correct header about headers for CORS
+    
+    The CORS standard dictates that preflight requests are made with the
+    Access-Control-Request-Headers header containing the headers of the
+    author request. The Access-Control-Allow-Headers header is sent in the
+    response.
+    
+    The present code looks for Access-Control-Allow-Headers in request, so
+    fix it to look at Access-Control-Request-Headers instead.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2abacd9678ae04cefac457882ba718a454948915)
+
+commit a5739e3903f8c30c8168199413818922b4a29bad
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Dec 6 11:07:09 2013 -0800
+
+    rgw: fix reading bucket policy in RGWBucket::get_policy()
+    
+    Fixes: 6940
+    Backport: dumpling, emperor
+    
+    We changed the way we keep the bucket policy, and we shouldn't try to
+    access the bucket object directly. This had changed when we added the
+    bucket instance object around dumpling.
+    
+    Reported-by: Gao, Wei M <wei.m.gao@intel.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 7a9a088d82d04f6105d72f6347673724ac16c9f8)
+
+commit ba792cdc8d0c006e260e6a02a33ca1491b35901c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 16 11:45:27 2014 -0800
+
+    rgw: handle racing object puts when object doesn't exist
+    
+    If the object didn't exist before and now we have multiple puts coming
+    in concurrently, we need to make sure that we behave correctly. Only one
+    needs to win, the other one can fail silently. We do that by setting
+    exclusive flag on the object creation and handling the error correctly.
+    Note that we still want to return -EEXIST in some cases (when the
+    exclusive flag is passed to put_obj_meta(), e.g., on bucket creation).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 86c15480fc3e33b9a3b84d0af68d8398fc732bae)
+
+commit 1e2984ac4a4ed0508e1b5efc8b69ae4da0cac824
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 16 11:33:49 2014 -0800
+
+    rgw: don't return -ENOENT in put_obj_meta()
+    
+    Fixes: #7168
+    
+    An object put may race with the same object's delete. In this case just
+    ignore the error, same behavior as if object was created and then
+    removed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5c24a7ea905587fd4077e3b0cfc0f5ad2b178c29)
+
+commit 5f1eea2ccf1ee4296530fc754bfd5dfc1571cd62
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sat Jan 18 18:01:20 2014 -0800
+
+    rgw: Use correct secret key for POST authn
+    
+    The POST authentication by signature validation looked up a user based
+    on the access key, then used the first secret key for the user. If the
+    access key used was not the first access key, then the expected
+    signature would be wrong, and the POST would be rejected.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 586ad1f69e6260ef298805647911b3b49f8be7cf)
+
+commit 14e933eb2e8adf3e7df2aef823d863c593067928
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jan 28 15:54:33 2014 +0000
+
+    osd: OSDMonitor: ignore pgtemps from removed pool
+    
+    There's a window in-between receiving an MOSDPGTemp message from an OSD
+    and actually handling it that may lead to the pool the pg temps refer to
+    no longer existing. This may happen if the MOSDPGTemp message is queued
+    pending dispatching due to an on-going proposal (maybe even the pool
+    removal).
+    
+    This patch fixes such behavior in two steps:
+    
+    1. Check if the pool exists in the osdmap upon preprocessing
+     - if pool does not exist in the osdmap, then the pool must have been
+       removed prior to handling the message, but after the osd sent it.
+     - safe to ignore the pg update
+    2. If all pg updates in the message have been ignored, ignore the whole
+       message.  Otherwise, let prepare handle the rest.
+    
+    3. Recheck if pool exists in the osdmap upon prepare
+     - We may have ignored this pg back in preprocess, but other pgs in the
+       message may have led the message to be passed on to prepare; ignore
+       pg update once more.
+    4. Check if pool is pending removal and ignore pg update if so.
+    
+    We delegate checking the pending value to prepare_pgtemp() because in this
+    case we should only ignore the update IFF the pending value is in fact
+    committed.  Otherwise we should retry the message.  prepare_pgtemp() is
+    the appropriate place to do so.
+    
+    Fixes: 7116
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f513f66f48383a07c70ca18a4dba6c2449ea9860)
+
+commit 3672fc49bea5efcef2069673c88fd8e526eeb7e5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 11:00:34 2014 -0800
+
+    OSDMonitor: use deepish_copy_from for remove_down_pg_temp
+    
+    This is a backport of 368852f6c0a884b8fdc80a5cd6f9ab72e814d412.
+    
+    Make a deep copy of the OSDMap to avoid clobbering the in-memory copy with
+    the call to apply_incremental.
+    
+    Fixes: #7060
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e6fef68ee51a2e3be6537086246ce860819788eb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 24 11:03:26 2014 -0800
+
+    OSDMap: deepish_copy_from()
+    
+    Make a deep(ish) copy of another OSDMap.  Unfortunatley we can't make the
+    compiler-generated copy operator/constructors private until c++11.  :(
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bd54b9841b9255406e56cdc7269bddb419453304)
+
+commit 2528186c0dd327607b1e76a51581df72aeea52f2
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 10:26:12 2014 -0800
+
+    buffer: make 0-length splice() a no-op
+    
+    This was causing a problem in the Striper, but fixing it here will avoid
+    corner cases all over the tree.  Note that we have to bail out before
+    the end-of-buffer check to avoid hitting that check when the bufferlist is
+    also empty.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ff5abfbdae07ae8a56fa83ebaa92000896f793c2)
+
+commit fe9539e118a830aa71bc5846ad2693543213fc59
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 10:09:17 2014 -0800
+
+    osdc/Striper: test zero-length add_partial_result
+    
+    If we add a partial result that is 0-length, we used to hit an assert in
+    buffer::list::splice().  Add a unit test to verify the fix.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 28c7388d320a47657c2e12c46907f1bf40672b08)
+
+commit 9801718e6e7a335d1aeffb7f3d958b012d8dd79c
+Author: Derek Yarnell <derek@umiacs.umd.edu>
+Date:   Mon Jan 27 12:27:51 2014 -0700
+
+    packaging: apply udev hack rule to RHEL
+    
+    In the RPM spec file there is a test to deploy the uuid hack udev rules
+    for older udev operating systems. This includes CentOS and RHEL, but the
+    check currently only is for CentOS, causing RHEL clients to get a bogus
+    osd rules file.
+    
+    Adjust the conditional to apply to RHEL as well as CentOS. (The %{rhel}
+    macro is defined in both platforms' redhat-rpm-config package.)
+    
+    Fixes http://tracker.ceph.com/issues/7245
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+    (cherry picked from commit 64a0b4fa563795bc22753940aa3a4a2946113109)
+
+commit 5817078ba9b2aa38f39e1f62d8d08e943646c0bb
+Merge: 9fa8d2d abf6441
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 23 07:12:37 2014 -0800
+
+    Merge pull request #1129 from ceph/wip-dumpling-backport-6620
+    
+    mds: MDSMap: adjust buffer size for uint64 values with more than 5 chars
+
+commit abf64417bbc63b006c53a07262a5774d0d40550e
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Thu Jan 23 11:36:16 2014 +0000
+
+    mds: MDSMap: adjust buffer size for uint64 values with more than 5 chars
+    
+    Fixes: #6620
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 0e8182edd850f061421777988974efbaa3575b9f)
+
+commit 9fa8d2d1bd4a8cf56fdebae484bee35c3930fb62
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 21 11:29:56 2014 -0800
+
+    mon/MDSMonitor: do not generate mdsmaps from already-laggy mds
+    
+    There is one path where a mds that is not sending its beacon (e.g.,
+    because it is not running at all) will lead to proposal of new mdsmaps.
+    Fix it.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 584c2dd6bea3fe1a3c7f306874c054ce0cf0d2b5)
+
+commit a2e34b8d322e571600d6c0a3ff363f9910196f28
+Author: John Spray <john.spray@inktank.com>
+Date:   Mon Jan 20 11:08:27 2014 +0000
+
+    Fix #7187: Include all summary items in JSON health output
+    
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit fdf3b5520d150f14d90bdfc569b70c07b0579b38)
+
+commit ee9fc590a556354b54a174a4790c4142949d7409
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 6 18:32:42 2014 -0800
+
+    rgw: convert bucket info if needed
+    
+    Fixes: #7110
+    In dumpling, the bucket info was separated into bucket entry point and
+    bucket instance objects. When setting bucket attrs we only ended up
+    updating the bucket instance object. However, pre-dumpling buckets still
+    keep everything at the entry-point object, so acl changes didn't affect
+    anything (because we never updated the entry point). This change just
+    converts the bucket info into the new format.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit a5f8cc7ec9ec8bef4fbc656066b4d3a08e5b215b)
+
+commit 69a99e6edd67aadaea77fb5ebcf46190dbfcdbc6
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:40:43 2014 -0800
+
+    osd: ignore OSDMap messages while we are initializing
+    
+    The mon may occasionally send OSDMap messages to random OSDs, but is not
+    very descriminating in that we may not have authenticated yet.  Ignore any
+    messages if that is the case; we will reqeust whatever we need during the
+    BOOTING state.
+    
+    Fixes: #7093
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f68de9f352d53e431b1108774e4a23adb003fe3f)
+
+commit 1545ddbd3b6dfac897d483bd58a4335d68b9b62a
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:43:26 2014 -0800
+
+    mon: only send messages to current OSDs
+    
+    When choosing a random OSD to send a message to, verify not only that
+    the OSD id is up but that the session is for the same instance of that OSD
+    by checking that the address matches.
+    
+    Fixes: #7093
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98ed9ac5fed6eddf68f163086df72faabd9edcde)
+
+commit 4d88dd10bfab4e5fb45632245be5f79eeba73a30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 26 13:58:47 2013 -0700
+
+    osd: discriminate based on connection messenger, not peer type
+    
+    Replace ->get_source().is_osd() checks and instead see if it is the
+    cluster_messenger so that we do not confuse ourselves when we get
+    legit requests from other OSDs on our public interface.
+    
+    NOTE: backporting this because a mixed cluster may send OSD requests
+    via the client interface, even though dumpling doesn't do this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a6b04c5d8ba043727a2e39a62e9d4126485bcfeb)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 2354b957b7924af3e81f559d5c975262848c3bf6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 22:59:51 2013 +0100
+
+    qa: test for error when ceph osd rm is EBUSY
+    
+    http://tracker.ceph.com/issues/6824 fixes #6824
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 31507c90f0161c4569a2cc634c0b5f671179440a)
+
+commit 10ad2e01f832ca9293a1bda1c82f7d318e8c3589
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 16:27:02 2013 +0100
+
+    mon: set ceph osd (down|out|in|rm) error code on failure
+    
+    Instead of always returning true, the error code is set if at least one
+    operation fails.
+    
+    EINVAL if the OSD id is invalid (osd.foobar for instance).
+    EBUSY if trying to remove and OSD that is up.
+    
+    When used with the ceph command line, it looks like this:
+    
+        ceph -c ceph.conf osd rm osd.0
+        Error EBUSY: osd.0 is still up; must be down before removal.
+        kill PID_OF_osd.0
+        ceph -c ceph.conf osd down osd.0
+        marked down osd.0.
+        ceph -c ceph.conf osd rm osd.0 osd.1
+        Error EBUSY: removed osd.0, osd.1 is still up; must be down before removal.
+    
+    http://tracker.ceph.com/issues/6824 fixes #6824
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 15b8616b13a327701c5d48c6cb7aeab8fcc4cafc)
+
+commit db82bfec50349783c3bb708c65cf89bceff7b2ca
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Oct 29 20:30:37 2013 +0000
+
+    mon: OSDMonitor: fix some annoying whitespace
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 42c4137cbfacad5654f02c6608cc0e81b45c06be)
+
+commit 1abb16988727a21dff61edca48bfcb05c60d26f2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 26 17:38:52 2013 -0800
+
+    librbd: call user completion after incrementing perfcounters
+    
+    The perfcounters (and the ictx) are only valid while the image is
+    still open.  If the librbd user gets the callback for its last I/O,
+    then closes the image, the ictx and its perfcounters will be
+    invalid. If the AioCompletion object is has not run the rest of its
+    complete() method yet, it will access these now-invalid addresses,
+    possibly leading to a crash.
+    
+    The AioCompletion object is independent of the ictx and does not
+    access it again after incrementing perfcounters, so avoid this race by
+    calling the user's callback after this step. The AioCompletion object
+    will be cleaned up by the rest of complete_request(), independent of
+    the ImageCtx.
+    
+    Fixes: #5426
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4cea7895da7331b84d8c6079851fdc0ff2f4afb1)
+
+commit 4dae50d8a0ffe3f3491eae62950fe46229aaa0e6
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Dec 6 16:03:20 2013 -0800
+
+    objecter: don't take extra throttle budget for resent ops
+    
+    These ops have already taken their budget in the original op_submit().
+    It will be returned via put_op_budget() when they complete.
+    If there were many localized reads of missing objects from replicas,
+    or cache pool redirects, this would cause the objecter to use up all
+    of its op throttle budget and hang.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 8d0180b1b7b48662daef199931efc7f2a6a1c431)
+    
+    Conflicts:
+    
+    	src/osdc/Objecter.cc
+
+commit 4944618fe74e1875e37a5de9a5bfe3922f906942
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:44:37 2013 -0800
+
+    rbd: check write return code during bench-write
+    
+    This is allows rbd-bench to detect http://tracker.ceph.com/issues/6938
+    when combined with rapidly changing the mon osd full ratio.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3caf3effcb113f843b54e06099099909eb335453)
+
+commit 3a7a0353d7c8521a2c3c25778801d45f0ca10957
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:36:33 2013 -0800
+
+    objecter: resend all writes after osdmap loses the full flag
+    
+    Now that the osd does not respond if it gets a map with the full flag
+    set first, clients need to resend all writes.
+    
+    Clients talking to old osds are still subject to the race condition,
+    so both sides must be upgraded to avoid it.
+    
+    Refs: #6938
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e32874fc5aa6f59494766b7bbeb2b6ec3d8f190e)
+    
+    Conflicts:
+    
+    	src/osdc/Objecter.h
+
+commit f4d0fc8afac1bcbea76f46744397a8462025f373
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:34:38 2013 -0800
+
+    osd: drop writes when full instead of returning an error
+    
+    There's a race between the client and osd with a newly marked full
+    osdmap.  If the client gets the new map first, it blocks writes and
+    everything works as expected, with no errors from the osd.
+    
+    If the osd gets the map first, however, it will respond to any writes
+    with -ENOSPC. Clients will pass this up the stack, and not retry these
+    writes later.  -ENOSPC isn't handled well by all clients. RBD, for
+    example, may pass it on to qemu or kernel rbd which will both
+    interpret it as EIO.  Filesystems on top of rbd will not behave well
+    when they receive EIOs like this, especially if the cluster oscillates
+    between full and not full, so some writes succeed.
+    
+    To fix this, never return ENOSPC from the osd because of a map marked
+    full, and rely on the client to retry all writes when the map is no
+    longer marked full.
+    
+    Old clients talking to osds with this fix will hang instead of
+    propagating an error, but only if they run into this race
+    condition. ceph-fuse and rbd with caching enabled are not affected,
+    since the ObjectCacher will retry writes that return errors.
+    
+    Refs: #6938
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4111729dda7437c23f59e7100b3c4a9ec4101dd0)
+
+commit 2b109803b9434d2d495e65d6a045838ee3ebf169
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 6 16:55:52 2013 -0800
+
+    objecter: clean pause / unpause logic
+    
+    op->paused holds now whether operation should be paused or not, and it's
+    being updated when scanning requests. No need to do a second scan.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5fe3dc647bf936df8e1eb2892b53f44f68f19821)
+
+commit 7b229fca346b7ab60131b229591d35f63f64a6ad
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 6 16:15:47 2013 -0800
+
+    objecter: set op->paused in recalc_op_target(), resend in not paused
+    
+    When going through scan_requests() in handle_osd_map() we need to make
+    sure that if an op should not be paused anymore then set it on the op
+    itself, and return NEED_RESEND. Otherwise we're going to miss reset of
+    the full flag.
+    Also in handle_osd_map(), make sure that op shouldn't be paused before
+    sending it. There's a lot of cleanup around that area that we should
+    probably be doing, make the code much more tight.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 98ab7d64a191371fa39d840c5f8e91cbaaa1d7b7)
+
+commit a3fdfd77c953a8f9ace4991630599e60f56abc4a
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Nov 6 10:46:37 2013 +0800
+
+    objecter: don't resend paused ops
+    
+    Paused ops are meant to block on the client side until a new map that
+    unpauses them is recieved. If we send paused writes when the FULL flag
+    is set, we'll get -ENOSPC from the osds, which is not what Objecter
+    users expect. This may cause rbd without caching to produce an I/O
+    error instead of waiting for the cluster to have capacity.
+    
+    Fixes: #6725
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c5c399d327cfc0d232d9ec7d49ababa914d0b21a)
+
+commit 78037886ae6ae278d8fb2db52d7767b8612313fe
+Merge: a60ac91 4e47a00
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 29 21:10:30 2013 -0800
+
+    Merge pull request #808 from ceph/wip-6152-dumpling
+    
+    rgw: Fix S3 auth when using response-* query string params
+
+commit 4e47a003bb3adcee690067d4037c248a93023c20
+Author: Sylvain Munaut <s.munaut@whatever-company.com>
+Date:   Thu Aug 29 16:17:30 2013 +0200
+
+    rgw: Fix S3 auth when using response-* query string params
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    (cherry picked from commit 7a7361d7e798d94796d4985e2e4b35af22463ae2)
diff --git a/doc/changelog/v0.67.7.txt b/doc/changelog/v0.67.7.txt
new file mode 100644
index 000000000..71442f66b
--- /dev/null
+++ b/doc/changelog/v0.67.7.txt
@@ -0,0 +1,110 @@
+commit d7ab4244396b57aac8b7e80812115bbd079e6b73 (tag: refs/tags/v0.67.7, refs/remotes/gh/dumpling)
+Author: Ken Dreyer <ken.dreyer@inktank.com>
+Date:   Wed Feb 19 18:09:39 2014 +0000
+
+    v0.67.7
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+
+commit c1f42770f45205c8a3eae774d09e747408130c50
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 6 12:53:58 2014 -0800
+
+    radosgw-admin: fix object policy read op
+    
+    Fixes: #7083
+    This was broken when we fixed #6940. We use the same function to both
+    read the bucket policy and the object policy. However, each needed to be
+    treated differently. Restore old behavior for objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b1976dd00f5b29c01791272f63a18250319f2edb)
+
+commit 404500811adc74c79583bbf3122dc333e0b41b51
+Merge: 3e7ea33 fec55a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 15 22:20:45 2014 -0800
+
+    Merge pull request #1243 from dachary/wip-ceph-disk-dumpling
+    
+    ceph-disk: unit tests (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 3e7ea33cfc80031aed8d6c61e1bdca947b36d69b
+Author: Ken Dreyer <ken.dreyer@inktank.com>
+Date:   Fri Jan 3 16:38:25 2014 -0700
+
+    add autotools-generated files to .gitignore
+    
+    When running "make check", Automake generates test-suite.log, along with
+    various *.log and *.trs files in the tree. Add these files to
+    .gitignore.
+    
+    (It looks like this feature arrived in Automake 1.13.)
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+    (cherry picked from commit bb8b7503b03fac5830fb71b9723963fdc803ca90)
+
+commit fec55a2174b49ef68c816755c1218f9cd69033b6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Feb 15 18:34:04 2014 +0100
+
+    ceph-disk: unit tests
+    
+    src/test/ceph-disk.sh replaces src/test/cli/ceph-disk/data-dir.t
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 0679b11b4d2f7cb7bd9c1cd86854a064aaa83f8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 2 22:42:17 2014 +0100
+
+    ceph-disk: cannot run unit tests
+    
+    Because ceph-disk relies on hardcoded paths. The corresponding test will
+    be added back when ceph-disk can run from sources.
+    
+    Fixes: #7085
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 2ba6930d62263a39f150ab43bf8cd860b9245188)
+
+commit 9dae9993e9b36e741e05804807bdb738fbe52e6f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 23:52:06 2014 -0800
+
+    Revert "librbd: remove limit on number of objects in the cache"
+    
+    This reverts commit 367cf1bbf86233eb20ff2304e7d6caab77b84fcc.
+    
+    Removing the limit on objects means we leak memory, since Objects without
+    any buffers can exist in the cache.
+
+commit 2795339d3632df975486d51e69762f9f567fb43d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 7 05:22:20 2013 -0700
+
+    os/FileStore: fix ENOENT error code for getattrs()
+    
+    In commit dc0dfb9e01d593afdd430ca776cf4da2c2240a20 the omap xattrs code
+    moved up a block and r was no longer local to the block.  Translate
+    ENOENT -> 0 to compensate.
+    
+    Fix the same error in _rmattrs().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6da4b91c07878e07f23eee563cf1d2422f348c2f)
+
+commit c158de232fcfb0e39f4b7eaac6dacbaaaee84b5f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 15 16:00:26 2013 -0700
+
+    test/filestore/run_seed_to.sh: avoid obsolete --filestore-xattr-use-omap
+    
+    This option no longer exists.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1d4f501a015727a7ff4b2f9b20dc91f2bbd9707b)
diff --git a/doc/changelog/v0.67.8.txt b/doc/changelog/v0.67.8.txt
new file mode 100644
index 000000000..2f8e0c944
--- /dev/null
+++ b/doc/changelog/v0.67.8.txt
@@ -0,0 +1,1220 @@
+commit ffc58b4ea6a3ea10e91050aaa28d14d87dea5656 (tag: refs/tags/v0.67.8, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Thu May 1 11:18:24 2014 +0000
+
+    0.67.8
+
+commit 4b16b70c53be83481efefcf394eca99c73bb9805
+Merge: 5a6b351 fb0944e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 30 15:15:48 2014 -0700
+
+    Merge pull request #1743 from ceph/wip-mon-backports.dumpling
+    
+    mon: OSDMonitor: HEALTH_WARN on 'mon osd down out interval == 0'
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit fb0944e22acf6f8b6cefb59cc4c41dc48087bfd7
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Apr 30 17:13:30 2014 +0100
+
+    mon: OSDMonitor: HEALTH_WARN on 'mon osd down out interval == 0'
+    
+    A 'status' or 'health' request will return a HEALTH_WARN whenever the
+    monitor handling the request has the option set to zero.
+    
+    Fixes: 7784
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b2112d5087b449d3b019678cb266ff6fa897897e)
+
+commit 5a6b35160417423db7c6ff892627f084ab610dfe
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Mar 4 16:15:15 2014 -0800
+
+    Make symlink  of librbd to qemu's folder so it can detect it.
+    
+    Per issue #7293.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 65f3354903fdbdb81468a84b8049ff19c00f91ba)
+
+commit 735a90a95eea01dbcce5026758895117c2842627
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Apr 25 14:11:27 2014 -0700
+
+    rgw: fix url escaping
+    
+    Fixes: #8202
+    This fixes the radosgw side of issue #8202. Needed to cast value
+    to unsigned char, otherwise it'd get padded.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bcf92c496aba0dfde432290fc2df5620a2767313)
+
+commit 438b57890dfce04226d769389a601d35b74e11fe
+Merge: c049967 476b929
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 25 16:00:24 2014 -0700
+
+    Merge pull request #1700 from xanpeng/patch-1
+    
+    Fix error in mkcephfs.rst
+    
+    Signed-off-by: Xan Peng <xanpeng@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 476b929ecc5b7351a5be3024817b900976a90a3e
+Author: xanpeng <xanpeng@gmail.com>
+Date:   Mon Apr 21 11:30:42 2014 +0800
+
+    Update mkcephfs.rst
+    
+    There should be no blank between mount options.
+
+commit c049967af829497f8a62e0cbbd6031f85ead8a59
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 1 17:27:01 2014 -0700
+
+    auth: add rwlock to AuthClientHandler to prevent races
+    
+    For cephx, build_authorizer reads a bunch of state (especially the
+    current session_key) which can be updated by the MonClient. With no
+    locks held, Pipe::connect() calls SimpleMessenger::get_authorizer()
+    which ends up calling RadosClient::get_authorizer() and then
+    AuthClientHandler::bulid_authorizer(). This unsafe usage can lead to
+    crashes like:
+    
+    Program terminated with signal 11, Segmentation fault.
+    0x00007fa0d2ddb7cb in ceph::buffer::ptr::release (this=0x7f987a5e3070) at common/buffer.cc:370
+    370 common/buffer.cc: No such file or directory.
+    in common/buffer.cc
+    (gdb) bt
+    0x00007fa0d2ddb7cb in ceph::buffer::ptr::release (this=0x7f987a5e3070) at common/buffer.cc:370
+    0x00007fa0d2ddec00 in ~ptr (this=0x7f989c03b830) at ./include/buffer.h:171
+    ceph::buffer::list::rebuild (this=0x7f989c03b830) at common/buffer.cc:817
+    0x00007fa0d2ddecb9 in ceph::buffer::list::c_str (this=0x7f989c03b830) at common/buffer.cc:1045
+    0x00007fa0d2ea4dc2 in Pipe::connect (this=0x7fa0c4307340) at msg/Pipe.cc:907
+    0x00007fa0d2ea7d73 in Pipe::writer (this=0x7fa0c4307340) at msg/Pipe.cc:1518
+    0x00007fa0d2eb44dd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
+    0x00007fa0e0f5f9d1 in start_thread (arg=0x7f987a5e4700) at pthread_create.c:301
+    0x00007fa0de560b6d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
+    
+    and
+    
+    Error in `qemu-system-x86_64': invalid fastbin entry (free): 0x00007ff12887ff20
+    *** ======= Backtrace: =========
+    /lib/x86_64-linux-gnu/libc.so.6(+0x80a46)[0x7ff3dea1fa46]
+    /usr/lib/librados.so.2(+0x29eb03)[0x7ff3e3d43b03]
+    /usr/lib/librados.so.2(_ZNK9CryptoKey7encryptEP11CephContextRKN4ceph6buffer4listERS4_RSs+0x71)[0x7ff3e3d42661]
+    /usr/lib/librados.so.2(_Z21encode_encrypt_enc_blIN4ceph6buffer4listEEvP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xfe)[0x7ff3e3d417de]
+    /usr/lib/librados.so.2(_Z14encode_encryptIN4ceph6buffer4listEEiP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xa2)[0x7ff3e3d41912]
+    /usr/lib/librados.so.2(_ZN19CephxSessionHandler12sign_messageEP7Message+0x242)[0x7ff3e3d40de2]
+    /usr/lib/librados.so.2(_ZN4Pipe6writerEv+0x92b)[0x7ff3e3e61b2b]
+    /usr/lib/librados.so.2(_ZN4Pipe6Writer5entryEv+0xd)[0x7ff3e3e6c7fd]
+    /lib/x86_64-linux-gnu/libpthread.so.0(+0x7f8e)[0x7ff3ded6ff8e]
+    /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7ff3dea99a0d]
+    
+    Fix this by adding an rwlock to AuthClientHandler. A simpler fix would
+    be to move RadosClient::get_authorizer() into the MonClient() under
+    the MonClient lock, but this would not catch all uses of other
+    Authorizer, e.g. for verify_authorizer() and it would serialize
+    independent connection attempts.
+    
+    This mainly matters for cephx, but none and unknown can have the
+    global_id reset as well.
+    
+    Partially-fixes: #6480
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2cc76bcd12d803160e98fa73810de2cb916ef1ff)
+
+commit 2b4b00b76b245b1ac6f95e4537b1d1a4656715d5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 1 11:37:29 2014 -0700
+
+    pipe: only read AuthSessionHandler under pipe_lock
+    
+    session_security, the AuthSessionHandler for a Pipe, is deleted and
+    recreated while the pipe_lock is held. read_message() is called
+    without pipe_lock held, and examines session_security. To make this
+    safe, make session_security a shared_ptr and take a reference to it
+    while the pipe_lock is still held, and use that shared_ptr in
+    read_message().
+    
+    This may have caused crashes like:
+    
+    *** Error in `qemu-system-x86_64': invalid fastbin entry (free): 0x00007f42a4002de0 ***
+    ======= Backtrace: =========
+    /lib/x86_64-linux-gnu/libc.so.6(+0x80a46)[0x7f452f1f3a46]
+    /usr/lib/x86_64-linux-gnu/libnss3.so(PK11_FreeSymKey+0xa8)[0x7f452e72ff98]
+    /usr/lib/librados.so.2(+0x2a18cd)[0x7f453451a8cd]
+    /usr/lib/librados.so.2(_ZNK9CryptoKey7encryptEP11CephContextRKN4ceph6buffer4listERS4_RSs+0x71)[0x7f4534519421]
+    /usr/lib/librados.so.2(_Z21encode_encrypt_enc_blIN4ceph6buffer4listEEvP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xfe)[0x7f453451859e]
+    /usr/lib/librados.so.2(_Z14encode_encryptIN4ceph6buffer4listEEiP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xa2)[0x7f45345186d2]
+    /usr/lib/librados.so.2(_ZN19CephxSessionHandler23check_message_signatureEP7Message+0x246)[0x7f4534516866]
+    /usr/lib/librados.so.2(_ZN4Pipe12read_messageEPP7Message+0xdcc)[0x7f453462ecbc]
+    /usr/lib/librados.so.2(_ZN4Pipe6readerEv+0xa5c)[0x7f453464059c]
+    /usr/lib/librados.so.2(_ZN4Pipe6Reader5entryEv+0xd)[0x7f4534643ecd]
+    /lib/x86_64-linux-gnu/libpthread.so.0(+0x7f8e)[0x7f452f543f8e]
+    /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f452f26da0d]
+    
+    Partially-fixes: #6480
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1d74170a4c252f35968ccfbec8e432582e92f638)
+
+commit 48895a46015c9d6d67543816f5a400c21aa206b1 (refs/remotes/gh/wip-objectcacher-flusher-dumpling)
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 3 12:51:15 2014 -0800
+
+    osdc/ObjectCacher: back off less during flush
+    
+    In cce990efc8f2a58c8d0fa11c234ddf2242b1b856 we added a limit to avoid
+    holding the lock for too long.  However, if we back off, we currently
+    wait for a full second, which is probably a bit much--we really just want
+    to give other threads a chance.
+    
+    Backport: emperor
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e2ee52879e9de260abbf5eacbdabbd71973a6a83)
+
+commit f3b5ba6f25010291a2918bdd286f1b39570bb907
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 09:28:29 2013 -0700
+
+    osdc/ObjectCacher: limit writeback IOs generated while holding lock
+    
+    While analyzing a log from Mike Dawson I saw a long stall while librbd's
+    objectcacher was starting lots (many hundreds) of IOs.  Limit the amount of
+    time we spend doing this at a time to allow IO replies to be processed so
+    that the cache remains responsive.
+    
+    I'm not sure this warrants a tunable (which we would need to add for both
+    libcephfs and librbd).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cce990efc8f2a58c8d0fa11c234ddf2242b1b856)
+
+commit 06f27fc6446d47b853208357ec4277c5dc10d9fe
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 8 10:52:43 2014 -0700
+
+    os/FileStore: reset journal state on umount
+    
+    We observed a sequence like:
+    
+     - replay journal
+       - sets JournalingObjectStore applied_op_seq
+     - umount
+     - mount
+       - initiate commit with prevous applied_op_seq
+     - replay journal
+       - commit finishes
+       - on replay commit, we fail assert op > committed_seq
+    
+    Although strictly speaking the assert failure is harmless here, in general
+    we should not let state leak through from a previous mount into this
+    mount or else assertions are in general more difficult to reason about.
+    
+    Fixes: #8019
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4de49e8676748b6ab4716ff24fd0a465548594fc)
+
+commit b29238729f87c73dfdcf16dddcf293577678dea2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Nov 5 14:54:20 2013 -0800
+
+    rgw: deny writes to a secondary zone by non-system users
+    
+    Fixes: #6678
+    We don't want to allow regular users to write to secondary zones,
+    otherwise we'd end up with data inconsistencies.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6961b5254f16ac3362c3a51f5490328d23640dbf)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.h
+
+commit 051a17eb008d75aa6b0737873318a2e7273501ab
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Apr 5 16:58:55 2014 -0700
+
+    mon: wait for quorum for MMonGetVersion
+    
+    We should not respond to checks for map versions when we are in the
+    probing or electing states or else clients will get incorrect results when
+    they ask what the latest map version is.
+    
+    Fixes: #7997
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 67fd4218d306c0d2c8f0a855a2e5bf18fa1d659e)
+
+commit 0716516da05eee967796fb71eb2f85c86afc40f1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 19 08:59:07 2014 -0800
+
+    rgw: fix swift range response
+    
+    Fixes: #7099
+    Backport: dumpling
+    The range response header was broken in swift.
+    
+    Reported-by: Julien Calvet <julien.calvet@neurea.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 0427f61544529ab4e0792b6afbb23379fe722de1)
+
+commit 94a1deefcfe525a7e698a1ae70a3bb561b6157de
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 15:41:49 2013 -0800
+
+    rgw: don't log system requests in usage log
+    
+    Fixes: 6889
+    System requets should not be logged in the usage log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 42ef8ba543c7bf13c5aa3b6b4deaaf8a0f9c58b6)
+
+commit 23fed8fc427e7077c61f86168a42f61a5f73867d
+Author: Greg Farnum <greg@inktank.com>
+Date:   Fri Apr 4 16:06:05 2014 -0700
+
+    OSD: _share_map_outgoing whenever sending a message to a peer
+    
+    This ensures that they get new maps before an op which requires them (that
+    they would then request from the monitor).
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 232ac1a52a322d163d8d8dbc4a7da4b6a9acb709)
+
+commit c45e15fd5cbe57a34c743b2835ecc30ee5a43963
+Author: Xihui He <xihuihe@gmail.com>
+Date:   Mon Dec 30 12:04:10 2013 +0800
+
+    msgr: fix rebind() race
+    stop the accepter and mark all pipes down before rebind to avoid race
+    
+    Fixes: #6992
+    
+    Signed-off-by: Xihui He xihuihe@gmail.com
+    (cherry picked from commit f8e413f9c79a3a2a12801f5f64a2f612de3f06a0)
+
+commit 3d31cf012a59e1fea8080b13bdc06c9021ba0656
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 26 13:20:21 2013 -0800
+
+    PG: retry GetLog() each time we get a notify in Incomplete
+    
+    If for some reason there are no up OSDs in the history which
+    happen to have usable copies of the pg, it's possible that
+    there is a usable copy elsewhere on the cluster which will
+    become known to the primary if it waits.
+    
+    Fixes: #6909
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 964c8e978f86713e37a13b4884a6c0b9b41b5bae)
+
+commit 1f80bbdf45439c7224ed52e4956973fc6d007848
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 17 15:37:44 2014 -0700
+
+    os/FileJournal: return errors on make_writeable() if reopen fails
+    
+    This is why #7738 is resulting in a crash instead of an error.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit aed074401d2834a5b04edd1b7f6b4f36336f6293)
+
+commit 62d942294a54208cdc82aebf8b536d164cae5dc6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 17 16:21:17 2014 -0700
+
+    mon/Paxos: commit only after entire quorum acks
+    
+    If a subset of the quorum accepts the proposal and we commit, we will start
+    sharing the new state.  However, the mon that didn't yet reply with the
+    accept may still be sharing the old and stale value.
+    
+    The simplest way to prevent this is not to commit until the entire quorum
+    replies.  In the general case, there are no failures and this is just fine.
+    In the failure case, we will call a new election and have a smaller quorum
+    of (live) nodes and will recommit the same value.
+    
+    A more performant solution would be to have a separate message invalidate
+    the old state and commit once we have all invalidations and a majority of
+    accepts.  This will lower latency a bit in the non-failure case, but not
+    change the failure case significantly.  Later!
+    
+    Fixes: #7736
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fa1d957c115a440e162dba1b1002bc41fc1eac43)
+
+commit 2160c72b393893896be581e89a42c4e37b79cb23
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 13 14:04:19 2014 -0700
+
+    PrioritizedQueue: cap costs at max_tokens_per_subqueue
+    
+    Otherwise, you can get a recovery op in the queue which has a cost
+    higher than the max token value.  It won't get serviced until all other
+    queues also do not have enough tokens and higher priority queues are
+    empty.
+    
+    Fixes: #7706
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2722a0a487e77ea2aa0d18caec0bdac50cb6a264)
+
+commit 1dd42e0f75fe1f5363f92bd5a4506812e54b8fb5
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Apr 3 13:59:59 2014 -0700
+
+    Fix byte-order dependency in calculation of initial challenge
+    
+    Fixes: #7977
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4dc62669ecd679bc4d0ef2b996b2f0b45b8b4dc7)
+
+commit c66b61f9dcad217429e4876d27881d9fb2e7666f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Dec 31 17:00:06 2013 -0800
+
+    rbd: return 0 and an empty list when pool is entirely empty
+    
+    rbd_list will return -ENOENT when no rbd_directory object
+    exists. Handle this in the cli tool and interpret it as success with
+    an empty list.
+    
+    Add this to the release notes since it changes command line behavior.
+    
+    Fixes: #6693
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ac547a5b7dc94282f079aef78e66348d99d9d5e9)
+    
+    Conflicts:
+    	PendingReleaseNotes
+    	src/rbd.cc
+
+commit 60b7aa96abfe09f7e9a263fa3f9b72c556dee8cb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Nov 20 18:35:34 2013 -0800
+
+    test: use older names for module setup/teardown
+    
+    setUp and tearDown require nosetests 0.11, but 0.10.4 is the latest on
+    centos. Rename to use the older aliases, which still work with newer
+    versions of nosetests as well.
+    
+    Fixes: #6368
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f753d56a9edba6ce441520ac9b52b93bd8f1b5b4)
+
+commit b405bfa49ec31f0c6d8636c4bdde17ee1f81deb7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Nov 3 11:06:10 2013 -0800
+
+    OSD: don't clear peering_wait_for_split in advance_map()
+    
+    I really don't know why I added this...  Ops can be discarded from the
+    waiting_for_pg queue if we aren't primary simply because there must have
+    been an exchange of peering events before subops will be sent within a
+    particular epoch.  Thus, any events in the waiting_for_pg queue must be
+    client ops which should only be seen by the primary.  Peering events, on
+    the other hand, should only be discarded if we are in a new interval,
+    and that check might as well be performed in the peering wq.
+    
+    Fixes: #6681
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9ab513334c7ff9544bac07bd420c6d5d200cf535)
+
+commit a498c940bd630cb103d17ad8532a11122439411d
+Merge: 225fc97 80e0a0a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 2 12:57:30 2014 -0700
+
+    Merge remote-tracking branch 'gh/wip-7888-dumpling' into dumpling
+
+commit 225fc97f228490dfc13c2e4deed8fecffdb28c5e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 5 21:48:53 2013 -0800
+
+    PG: fix operator<<,log_wierdness log bound warning
+    
+    Split may cause holes such that head != tail and yet
+    log.empty().
+    
+    Fixes: #6722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c6826c1e8a301b2306530c6e5d0f4a3160c4e691)
+
+commit 26eeab43f3f703a25e7ba62c75d0382d15e38263
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 5 17:47:48 2013 -0800
+
+    PGLog::rewind_divergent_log: log may not contain newhead
+    
+    Due to split, there may be a hole at newhead.
+    
+    Fixes: #6722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f4648bc6fec89c870e0c47b38b2f13496742b10f)
+
+commit 040abd75ad45bbcc05b24c9dddbd2026dd35e659
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Mar 29 14:23:21 2014 -0700
+
+    qa/workunits/fs/misc/layout_vxattrs: ceph.file.layout is not listed
+    
+    As of 08a3d6bd428c5e78dd4a10e6ee97540f66f9729c.  A similar change was made
+    in the kernel.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4f9f7f878953b29cd5f56a8e0834832d6e3a9cec)
+
+commit f5aa492a338ff711f3a45e4ebbf0d6b187b5f78e
+Merge: fef70cb 84cb345
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 18:01:08 2014 -0700
+
+    Merge pull request #1519 from ceph/wip-6951-dumpling
+    
+    rgw: reset objv tracker on bucket recreation
+
+commit fef70cbb52cf1ad12db45998b38858d9bbc3360d
+Merge: 9bfbce3 f443ff3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 17:02:39 2014 -0700
+
+    Merge pull request #1559 from ceph/wip-7881-dumpling
+    
+    Wip 7881 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 80e0a0a8fee2f6f903f612734b2cc72eae703eae (refs/remotes/gh/wip-7888-dumpling)
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 27 21:33:21 2014 -0700
+
+    mon/MonClient: use keepalive2 to verify the mon session is live
+    
+    Verify that the mon is responding by checking the keepalive2 reply
+    timestamp.  We cannot rely solely on TCP timing out and returning an
+    error.
+    
+    Fixes: #7888
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 056151a6334c054505c54e59af40f203a0721f28)
+
+commit 8723218379e80725a449b0594a4b15eb1c236b05
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 27 21:09:13 2014 -0700
+
+    msgr: add KEEPALIVE2 feature
+    
+    This is similar to KEEPALIVE, except a timestamp is also exchanged.  It is
+    sent with the KEEPALIVE, and then returned with the ACK.  The last
+    received stamp is stored in the Connection so that it can be queried for
+    liveness.  Since all of the users of keepalive are already regularly
+    triggering a keepalive, they can check the liveness at the same time.
+    
+    See #7888.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d747d79fd5ea8662a809c5636dfd2eaaa9bf8f5d)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+
+commit a2f0974f8b3567c0385494a0b2c828ade6ca8e1c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Mar 26 15:58:10 2014 -0700
+
+    Pipe: rename keepalive->send_keepalive
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 38d4c71a456c1cc9a5044dbcae5378836a34484d)
+
+commit 9bfbce30678742515025ca235c4443bb3a69199f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 26 21:52:00 2014 -0700
+
+    client: pin Inode during readahead
+    
+    Make sure the Inode does not go away while a readahead is in progress.  In
+    particular:
+    
+     - read_async
+       - start a readahead
+       - get actual read from cache, return
+     - close/release
+       - call ObjectCacher::release_set() and get unclean > 0, assert
+    
+    Fixes: #7867
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f1c7b4ef0cd064a9cb86757f17118d17913850db)
+
+commit 232445578a2c6d0fb974e55378057fce473095f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 12:34:07 2014 -0700
+
+    osdc/ObjectCacher: call read completion even when no target buffer
+    
+    If we do no assemble a target bl, we still want to return a valid return
+    code with the number of bytes read-ahead so that the C_RetryRead completion
+    will see this as a finish and call the caller's provided Context.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 032d4ec53e125ad91ad27ce58da6f38dcf1da92e)
+
+commit f443ff3006d41a7b0a2d7b649e8def0ffef6df12
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 30 16:54:39 2013 -0700
+
+    PGLog: remove obsolete assert in merge_log
+    
+    This assert assumes that if olog.head != log.head, olog contains
+    a log entry at log.head, which may not be true since pg splitting
+    might have left the log with arbitrary holes.
+    
+    Related: 0c2769d3321bff6e85ec57c85a08ee0b8e751bcb
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 353813b2e1a98901b876790c7c531f8a202c661d)
+
+commit 636e53c0f4fc43e9bfc1c8e7214cab9e0b46a359
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 30 15:54:27 2013 -0700
+
+    PGLog: on split, leave log head alone
+    
+    This way last_update doesn't go backwards.
+    
+    Fixes: 6447
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0c2769d3321bff6e85ec57c85a08ee0b8e751bcb)
+
+commit a0a560a9f04311306a9784fa3c6ea2586d637f56
+Merge: 466cd53 41d5e9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 26 17:18:24 2014 -0700
+
+    Merge pull request #1539 from ceph/wip-6910-dumpling
+    
+    PG: don't query unfound on empty pgs
+
+commit 41d5e9ab39e69c80bec1cb0627004c3fae6dc81d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 26 19:17:59 2013 -0800
+
+    PG: don't query unfound on empty pgs
+    
+    When the replica responds, it responds with a notify
+    rather than a log, which the primary then ignores since
+    it is already in the peer_info map.  Rather than fix that
+    we'll simply not send queries to peers we already know to
+    have no unfound objects.
+    
+    Fixes: #6910
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 838b6c8387087543ce50837277f7f6b52ae87d00)
+
+commit 466cd536ed7e541a36d88bce43683a2d9e2ca283
+Merge: 2ef0d6a c188949
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 21 14:53:23 2014 -0700
+
+    Merge pull request #1313 from ceph/dumpling-osd-subscribe
+    
+    Dumpling backport: clean up osd subscriptions
+
+commit 2ef0d6a25bf8c0cfb38768c157c29ba52295f3ca
+Merge: 77e46d0 bdd96c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 21 14:52:20 2014 -0700
+
+    Merge pull request #1485 from ceph/wip-7212.dumpling
+    
+    backport 7212 fixes to dumpling
+
+commit 84cb345e4f12a9b1db5e384411492a9d88f17dd8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 19 08:11:56 2014 -0800
+
+    rgw: reset objv tracker on bucket recreation
+    
+    Fixes: #6951
+    If we cannot create a new bucket (as it already existed), we need to
+    read the old bucket's info. However, this was failing as we were holding
+    the objv tracker that we created for the bucket creation. We need to
+    clear it, as subsequent read using it will fail.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 859ed33ed7f9a96f4783dfb3e130d5eb60c622dd)
+
+commit 77e46d0d7984f2d3ee0e15f27d2961a637c20b45
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Nov 6 14:33:03 2013 -0800
+
+    ReplicatedPG: don't skip missing if sentries is empty on pgls
+    
+    Formerly, if sentries is empty, we skip missing.  In general,
+    we need to continue adding items from missing until we get
+    to next (returned from collection_list_partial) to avoid
+    missing any objects.
+    
+    Fixes: #6633
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c7a30b881151e08b37339bb025789921e7115288)
+
+commit bdd96c620f33fb8f48f30f8d543af3290e6c934a
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 15 08:59:51 2014 -0800
+
+    mon/Elector: bootstrap on timeout
+    
+    Currently if an election times out we call a new
+    election.  If we have never joined a quorum, bootstrap
+    instead. This is heavier weight, but captures the case
+    where, during bootstrap:
+    
+     - a and b have learned each others' addresses
+     - everybody calls an election
+     - a and b form a quorum
+     - c loops trying to call an election, but is ignored
+       because a and b don't see its address in the monmap
+    
+    See logs:
+      ubuntu@teuthology:/var/lib/teuthworker/archive/sage-2014-02-14_13:50:04-ceph-deploy-wip-7212-sage-b-testing-basic-plana/83194
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a4bcb1f8129a4ece97bd3419abf1ff45d260ad8e)
+    (cherry picked from commit 143ec0281aa8b640617a3fe19a430248ce3b514c)
+
+commit 68fcc63c0423a2071b7b944ea6c3448282a78a09
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 11:25:52 2014 -0800
+
+    mon: tell MonmapMonitor first about winning an election
+    
+    It is important in the bootstrap case that the very first paxos round
+    also codify the contents of the monmap itself in order to avoid any manner
+    of confusing scenarios where subsequent elections are called and people
+    try to recover and modify paxos without agreeing on who the quorum
+    participants are.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad7f5dd481a7f45dfe6b50d27ad45abc40950510)
+    (cherry picked from commit e073a062d56099b5fb4311be2a418f7570e1ffd9)
+
+commit a3e57b7231cb28c2e0a896f747537ebdbe3a4e96
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 11:13:26 2014 -0800
+
+    mon: only learn peer addresses when monmap == 0
+    
+    It is only safe to dynamically update the address for a peer mon in our
+    monmap if we are in the midst of the initial quorum formation (i.e.,
+    monmap.epoch == 0).  If it is a later epoch, we have formed our initial
+    quorum and any and all monmap changes need to be agreed upon by the quorum
+    and committed via paxos.
+    
+    Fixes: #7212
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7bd2104acfeff0c9aa5e648d82ed372f901f767f)
+    (cherry picked from commit 1996fd89fb3165a63449b135e05841579695aabd)
+
+commit 21ed54201bd4b0f02c07f6f96a63a5720057f011
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Mar 17 14:37:09 2014 +0000
+
+    ceph.in: do not allow using 'tell' with interactive mode
+    
+    This avoids a lot of hassle when dealing with to whom tell each command
+    on interactive mode, and even more so if multiple targets are specified.
+    
+    As so, 'tell' commands should be used while on interactive mode instead.
+    
+    Backport: dumpling,emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e39c213c1d230271d23b74086664c2082caecdb9)
+
+commit be0205c33ccbab3b6f105bdf4da114658a981557
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 12 22:56:44 2014 +0100
+
+    RGWListBucketMultiparts: init max_uploads/default_max with 0
+    
+    CID 717377 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR)
+     2. uninit_member: Non-static class member "max_uploads" is not initialized
+        in this constructor nor in any functions that it calls.
+     4. uninit_member: Non-static class member "default_max" is not initialized
+        in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b23a141d54ffb39958aba9da7f87544674fa0e50)
+
+commit 6c3d4fbeb9cc50eca6eba12cfe7fed64b34eec3d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 13 14:49:30 2014 -0700
+
+    ceph_test_rados: wait for commit, not ack
+    
+    First, this is what we wanted in the first place
+    
+    Second, if we wait for ACK, we may look at a user_version value that is
+    not stable.
+    
+    Fixes: #7705
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f2124c5846f1e9cb44e66eb2e957b8c7df3e19f4)
+    
+    Conflicts:
+    
+    	src/test/osd/RadosModel.h
+
+commit 2daed5ff99dab238b696da5aba3816c4f5d763e8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 13 09:50:16 2014 -0700
+
+    test-upgrade-firefly: skip watch-notify system test
+    
+    This also fails on mixed version clusters due to watch on a
+    non-existent object returning ENOENT in firefly and 0 in dumpling.
+    
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 90a21d8cf6df5fe14b2dc9b2c175983b6bb017ce
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 12 21:30:12 2014 -0700
+
+    qa/workunit/rados/test-upgrade-firefly: skip watch-notify test
+    
+    A watch on a non-existent object now returns ENOENT in firefly; skip this
+    test as it will fail on a hybrid or upgraded cluster.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 32fdca6d9c2e3b923db7f21568bd315ab2c1c4ad
+Merge: 6700dd0 cd7986c
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 11 21:33:40 2014 -0700
+
+    Merge pull request #1411 from ceph/wip-7076-dumpling
+    
+    dumpling backport of watchers check for rbd_remove()
+
+commit 6700dd068e236473343d15eee6307d44156958a3
+Author: Ray Lv <xiangyulv@gmail.com>
+Date:   Wed Feb 26 21:17:32 2014 +0800
+
+    rgw: off-by-one in rgw_trim_whitespace()
+    
+    Fixes: #7543
+    Backport: dumpling
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Ray Lv <raylv@yahoo-inc.com>
+    (cherry picked from commit 195d53a7fc695ed954c85022fef6d2a18f68fe20)
+
+commit cd7986caf6baee5f9d6498b113b3382e66dd6f77
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Wed Jan 29 16:12:01 2014 +0200
+
+    rbd: check for watchers before trimming an image on 'rbd rm'
+    
+    Check for watchers before trimming image data to try to avoid getting
+    into the following situation:
+    
+      - user does 'rbd rm' on a mapped image with an fs mounted from it
+      - 'rbd rm' trims (removes) all image data, only header is left
+      - 'rbd rm' tries to remove a header and fails because krbd has a
+        watcher registered on the header
+      - at this point image cannot be unmapped because of the mounted fs
+      - fs cannot be unmounted because all its data and metadata is gone
+    
+    Unfortunately, this fix doesn't make it impossible to happen (the
+    required atomicity isn't there), but it's a big improvement over the
+    status quo.
+    
+    Fixes: http://tracker.ceph.com/issues/7076
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit 0a553cfa81b06e75585ab3c39927e307ec0f4cb6)
+
+commit a931aaa6cc104d63b20c0cbe9e3af4006c3abfaf
+Merge: b476d32 f5668b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Mar 9 10:56:31 2014 -0700
+
+    Merge pull request #1407 from dachary/wip-7188-dumpling
+    
+    common: ping existing admin socket before unlink (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit f5668b363b0724f385bebded3cbc7f363893f985
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Feb 15 11:43:13 2014 +0100
+
+    common: ping existing admin socket before unlink
+    
+    When a daemon initializes it tries to create an admin socket and unlinks
+    any pre-existing file, regardless. If such a file is in use, it causes
+    the existing daemon to loose its admin socket.
+    
+    The AdminSocketClient::ping is implemented to probe an existing socket,
+    using the "0" message. The AdminSocket::bind_and_listen function is
+    modified to call ping() on when it finds existing file. It unlinks the
+    file only if the ping fails.
+    
+    http://tracker.ceph.com/issues/7188 fixes: #7188
+    
+    Backport: emperor, dumpling
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 45600789f1ca399dddc5870254e5db883fb29b38)
+
+commit b476d324c69d4e6018191a7ffea8c9d6c1dfa008
+Merge: d3e13a7 5133dd6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 5 14:19:31 2014 -0800
+
+    Merge pull request #1366 from ceph/wip-6820.dumpling
+    
+    mon: OSDMonitor: don't crash if formatter is invalid during osd crush dump
+
+commit d3e13a7cdab42fa33182680f45fe21b4f9dc4b20
+Merge: c218c99 9c626e0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 5 12:45:57 2014 -0800
+
+    Merge pull request #1377 from ceph/wip-7584
+    
+    qa/workunit/rados/test-upgrade-firely.sh
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 9c626e0b18f538eb60883da01713ba629220e35e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 5 12:37:10 2014 -0800
+
+    qa/workunit/rados/test-upgrade-firely.sh
+    
+    Skip the tests that don't pass when run against firefly OSDs.
+    
+    Fixes: #7584
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c218c999ecebe41e6de6fde76e85cc765cad8257
+Merge: 0eabbf1 24711cd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 4 07:28:44 2014 -0800
+
+    Merge pull request #1357 from ceph/wip-dumpling-removewq
+    
+    OSD: ping tphandle during pg removal
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 5133dd60e272d3fcbaacd5662a708ee4cf0db46d
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Nov 22 02:17:16 2013 +0000
+
+    mon: OSDMonitor: don't crash if formatter is invalid during osd crush dump
+    
+    Code would assume a formatter would always be defined. If a 'plain'
+    formatter or even an invalid formatter were to be supplied, the monitor
+    would crash and burn in poor style.
+    
+    Fixes: 6820
+    Backport: emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 49d2fb71422fe4edfe5795c001104fb5bc8c98c3)
+
+commit 24711cd49f85dbe827d41c4bcad2700cd6c42ad7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Oct 15 13:11:29 2013 -0700
+
+    OSD: ping tphandle during pg removal
+    
+    Fixes: #6528
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    
+    (cherry picked from commit c658258d9e2f590054a30c0dee14a579a51bda8c)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 0eabbf145e1c44f4d128b192cc77b708f180c968
+Merge: fe8915a 9d5d931
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:47:05 2014 -0800
+
+    Merge pull request #1316 from ceph/dumpling-6922
+    
+    Dumpling: Prevent extreme PG split multipliers
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit fe8915ae7e182340d1e22154e852895742c7da51
+Merge: 87822cc 5667566
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:45:45 2014 -0800
+
+    Merge pull request #1315 from ceph/dumpling-hashpspool
+    
+    mon: OSDMonitor: allow (un)setting 'hashpspool' flag via 'osd pool set'
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 87822ccc862b533132c1fe232dfe4b7b17b816ad
+Merge: 0ae3352 37fbcb9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:44:39 2014 -0800
+
+    Merge pull request #1314 from ceph/dumpling-osd-pgstatsack
+    
+    Dumpling osd pgstatsack
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 5667566313b69dca011e897b2fa752356ad8901b
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Oct 10 17:43:48 2013 -0700
+
+    mon: OSDMonitor: allow (un)setting 'hashpspool' flag via 'osd pool set'
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c2886964a0c005545abab0cf8feae7e06ac02a8)
+    
+    Conflicts:
+    
+    	src/mon/MonCommands.h
+    	src/mon/OSDMonitor.cc
+    
+    mon: ceph hashpspool false clears the flag
+    instead of toggling it.
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    Reviewed-by: Christophe Courtaut <christophe.courtaut@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 589e2fa485b94244c79079f249428d4d545fca18
+    
+    Replace some of the infrastructure required by this command that
+    was not present in Dumpling with single-use code.
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 9d5d931c60104823b3b20dcfb09480d65ffaa5ed
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Dec 3 10:57:09 2013 -0800
+
+    OSDMonitor: use a different approach to prevent extreme multipliers on PG splits
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d8ccd73968fbd0753ca08916ebf1062cdb4d5ac1)
+    
+    Conflicts:
+    
+    	src/mon/OSDMonitor.cc
+
+commit c0c4448dc7df7900a564a6745903398cd39be7f1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Dec 2 15:13:40 2013 -0800
+
+    OSDMonitor: prevent extreme multipliers on PG splits
+    
+    Fixes: #6922
+    Backport: emperor
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f57dad6461171c903e8b5255eaed300374b00e74)
+    
+    Conflicts:
+    
+    	src/mon/OSDMonitor.cc
+
+commit c1889497b93ae9f0a946b11d9f5f6fcc7427e934
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 22 08:08:37 2014 -0800
+
+    osd: fix off-by-one is boot subscription
+    
+    If we have osdmap N, we want to onetime subscribe
+    starting at N+1.  Among other things, it means we
+    hear when the NOUP flag is cleared.
+    
+    This appears to have broken somewhere around
+    3c76b81f2f96b790b72f2088164ed8e9d5efbba1.
+    
+    Fixes: #7511
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Sam Just <sam.just@inktank.com>
+    (cherry picked from commit 70d23b9a0ad9af5ca35a627a7f93c7e610e17549)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 4584f60653bee0305e85418323d80332ceecd0cf
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 12:51:19 2014 -0800
+
+    OSD: use the osdmap_subscribe helper
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3c76b81f2f96b790b72f2088164ed8e9d5efbba1)
+
+commit 61b2aeee7c37e03d5f6691c08c7760c48a85a2e1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 13:34:39 2014 -0800
+
+    OSD: create a helper for handling OSDMap subscriptions, and clean them up
+    
+    We've had some trouble with not clearing out subscription requests and
+    overloading the monitors (though only because of other bugs). Write a
+    helper for handling subscription requests that we can use to centralize
+    safety logic. Clear out the subscription whenever we get a map that covers
+    it; if there are more maps available than we received, we will issue another
+    subscription request based on "m->newest_map" at the end of handle_osd_map().
+    
+    Notice that the helper will no longer request old maps which we already have,
+    and that unless forced it will not dispatch multiple subscribe requests
+    to a single monitor.
+    Skipping old maps is safe:
+    1) we only trim old maps when the monitor tells us to,
+    2) we do not send messages to our peers until we have updated our maps
+    from the monitor.
+    That means only old and broken OSDs will send us messages based on maps
+    in our past, and we can (and should) ignore any directives from them anyway.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6db3ae851d1c936de045390d18b1c6ae95f2a209)
+    
+    Conflicts:
+    
+    	src/osd/OSD.h
+
+commit d93d67d1a315d8abe8d1cd9d7ea83417a19e2406
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 13:31:26 2014 -0800
+
+    monc: new fsub_want_increment( function to make handling subscriptions easier
+    
+    Provide a subscription-modifying function which will not decrement
+    the start version.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5b9c187caf6f7847aaa4a1003d200158dd32bf63)
+
+commit 37fbcb958f79bbfcba57c516b4862a14c52be398
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 11:30:15 2014 -0800
+
+    OSD: disable the PGStatsAck timeout when we are reconnecting to a monitor
+    
+    Previously, the timeout counter started as soon as we issued the reopen,
+    but if the reconnect process itself took a while, we might time out and
+    issue another reopen just as we get to the point where it's possible to
+    get work done. Since the mon client has its own reconnect timeouts (that is,
+    the OSD doesn't need to trigger those), we instead disable our timeouts
+    while the reconnect is happening, and then turn them back on again starting
+    from when we get the reconnect callback.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 64cedf6fa3ee309cc96554286bfb805e4ca89439)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 9be395e870c50d97604ef41f17667cc566fd84e1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 13:51:48 2014 -0800
+
+    monc: backoff the timeout period when reconnecting
+    
+    If the monitors are systematically slowing down, we don't want to spam
+    them with reconnect attempts every three seconds. Instead, every time
+    we issue a reconnect, multiply our timeout period by a configurable; when
+    we complete the connection, reduce that multipler by 50%. This should let
+    us respond to monitor load.
+    Of course, we don't want to do that for initial startup in the case of a
+    couple down monitors, so don't apply the backoff until we've successfully
+    connected to a monitor at least once.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 794c86fd289bd62a35ed14368fa096c46736e9a2)
+
+commit 8f4c20bdab153d1603cc99186d8d3e3970aa8976
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 13:37:50 2014 -0800
+
+    monc: set "hunting" to true when we reopen the mon session
+    
+    If we don't have a connecton to a monitor, we want to retry to another
+    monitor regardless of whether it's the first time or not.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 60da8abe0ebf17ce818d6fcc6391401878123bb7)
+
+commit c6317558e0d3c8c62aecee0d95a839f93303f681
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 17:53:56 2014 -0800
+
+    monc: let users specify a callback when they reopen their monitor session
+    
+    Then the callback is triggered when a new session is established, and the
+    daemon can do whatever it likes. There are no guarantees about how long it
+    might take to trigger, though. In particular we call the provided callback
+    while not holding our own lock in order to avoid deadlock. This could lead
+    to some funny ordering from the user's perspective if they call
+    reopen_session() again before getting the callback, but there's no way around
+    that, so they just have to use it appropriately.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1a8c43474bf36bfcf2a94bf9b7e756a2a99f33fd)
+
+commit 0ae335298b85daba5125a3da4ad26d598c76ecab (refs/remotes/gh/multi-object-delete)
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 11 16:54:05 2014 -0800
+
+    rgw: multi object delete should be idempotent
+    
+    Fixes: #7346
+    When doing a multi object delete, if an object does not exist then we
+    should return a success code for that object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 8ca3d95bf633ea9616852cec74f02285a03071d5)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
diff --git a/doc/changelog/v0.67.9.txt b/doc/changelog/v0.67.9.txt
new file mode 100644
index 000000000..8ae98cd89
--- /dev/null
+++ b/doc/changelog/v0.67.9.txt
@@ -0,0 +1,233 @@
+commit ba340a97c3dafc9155023da8d515eecc675c619a (tag: refs/tags/v0.67.9, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed May 21 16:57:02 2014 +0000
+
+    0.67.9
+
+commit b638d19d126646d2a8f6da11067c5f392a62525e
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri May 9 09:21:23 2014 +0000
+
+    msg: Fix inconsistent message sequence negotiation during connection reset
+    
+    Backport: firefly, emperor, dumpling
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit bdee119076dd0eb65334840d141ccdf06091e3c9)
+
+commit 55b1112819ed2e6fd509c09c2c1a2b434d22aac6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 10:46:34 2014 -0700
+
+    OSD::handle_pg_query: on dne pg, send lb=hobject_t() if deleting
+    
+    We will set lb=hobject_t() if we resurrect the pg.  In that case,
+    we need to have sent that to the primary before hand.  If we
+    finish the removal before the pg is recreated, we'll just end
+    up backfilling it, which is ok since the pg doesn't exist anyway.
+    
+    Fixes: #7740
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 04de781765dd5ac0e28dd1a43cfe85020c0854f8)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 3b8ab41e1ec86f2ab5c6b4bee3fb4030077e2c21
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 14:48:35 2014 -0700
+
+    mon/MonClient: remove stray _finish_hunting() calls
+    
+    Callig _finish_hunting() clears out the bool hunting flag, which means we
+    don't retry by connection to another mon periodically.  Instead, we send
+    keepalives every 10s.  But, since we aren't yet in state HAVE_SESSION, we
+    don't check that the keepalives are getting responses.  This means that an
+    ill-timed connection reset (say, after we get a MonMap, but before we
+    finish authenticating) can drop the monc into a black hole that does not
+    retry.
+    
+    Instead, we should *only* call _finish_hunting() when we complete the
+    authentication handshake.
+    
+    Fixes: #8278
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 77a6f0aefebebf057f02bfb95c088a30ed93c53f)
+
+commit a915350491092abcd2584d0bbb5f5d8610ed89fc
+Merge: bd5d6f1 d4e67ff
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 10:19:00 2014 -0700
+
+    Merge pull request #1826 from ceph/wip-8162-dumpling
+    
+    Wip 8162 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit bd5d6f116416d1b410d57ce00cb3e2abf6de102b
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu May 15 16:50:43 2014 -0700
+
+    OSD: fix an osdmap_subscribe interface misuse
+    
+    When calling osdmap_subscribe, you have to pass an epoch newer than the
+    current map's. _maybe_boot() was not doing this correctly -- we would
+    fail a check for being *in* the monitor's existing map range, and then
+    pass along the map prior to the monitor's range. But if we were exactly
+    one behind, that value would be our current epoch, and the request would
+    get dropped. So instead, make sure we are not *in contact* with the monitor's
+    existing map range.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 290ac818696414758978b78517b137c226110bb4)
+
+commit c5ae8ffc281ac538dd0966a552a6cf729728e925
+Merge: fced056 8d991fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 19 13:45:50 2014 -0700
+
+    Merge pull request #1827 from ceph/wip-6565-dumpling
+    
+    Wip 6565 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 8d991fa2c3fdbfd9178d93e4ca33d47504a37195
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 16 10:07:37 2013 -0700
+
+    OSD: check for splitting when processing recover/backfill reservations
+    
+    Fixes: 6565
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 15ec5332ba4154930a0447e2bcf1acec02691e97)
+
+commit d4e67ff3037a3cc7ae2ecc9e1d8d086c45ae515a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 8 13:25:32 2014 -0700
+
+    ReplicatedPG::recover_backfill: do not update last_backfill prematurely
+    
+    Previously, we would update last_backfill on the backfill peer to
+    
+    backfills_in_flight.empty() ? backfill_pos :
+      backfills_in_flight.begin()->first
+    
+    which is actually the next backfill to complete.  We want to update
+    last_backfill to the largest completed backfill instead.
+    
+    We use the pending_backfill_updates mapping to identify the most
+    recently completed backfill.  Due to the previous patch, deletes
+    will also be included in that mapping.
+    
+    Related sha1s from master:
+    4139e75d63b0503dbb7fea8036044eda5e8b7cf1
+    7a06a71e0f2023f66d003dfb0168f4fe51eaa058
+    
+    We don't really want to backport those due to the changes in:
+    9ec35d5ccf6a86c380865c7fc96017a1f502560a
+    
+    This patch does essentially the same thing, but using backfill_pos.
+    
+    Fixse: #8162
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d620b13c51fc69b509824177af32de55c89c2259
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 28 16:03:25 2013 -0700
+
+    ReplicatedPG: add empty stat when we remove an object in recover_backfill
+    
+    Subsequent updates to that object need to have their stats added
+    to the backfill info stats atomically with the last_backfill
+    update.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ecddd12b01be120fba87f5ac60539f98f2c69a28)
+
+commit fced0562f6b80035cd522cc81bdc844ccc92d9bc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 27 13:34:00 2013 -0800
+
+    rgw: don't error out on empty owner when setting acls
+    
+    Fixes: #6892
+    Backport: dumpling, emperor
+    s3cmd specifies empty owner field when trying to set acls on object
+    / bucket. We errored out as it didn't match the current owner name, but
+    with this change we ignore it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 14cf4caff58cc2c535101d48c53afd54d8632104)
+
+commit 0e685c68aa6a500aa7fa433cd9b8246f70c5383e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:34:04 2014 -0700
+
+    rgw: send user manifest header field
+    
+    Fixes: #8170
+    Backport: firefly
+    If user manifest header exists (swift) send it as part of the object
+    header data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5cc5686039a882ad345681133c9c5a4a2c2fd86b)
+
+commit 20383e359cdfbf146838742fa050ec4b6073271d
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Apr 11 15:03:37 2014 +0800
+
+    client: add asok command to kick sessions that were remote reset
+    
+    Fixes: #8021
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 09a1bc5a4601d356b9cc69be8541e6515d763861)
+
+commit 4e5e41deeaf91c885773d90e6f94da60f6d4efd3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 18 13:50:11 2014 -0700
+
+    osd: throttle snap trimmming with simple delay
+    
+    This is not particularly smart, but it is *a* knob that lets you make
+    the snap trimmer slow down.  It's a flow and a simple delay, so it is
+    adjustable at runtime.  Default is 0 (no change in behavior).
+    
+    Partial solution for #6278.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4413670d784efc2392359f0f22bca7c9056188f4)
+
+commit 14650b282ecd344025714a4e743b255ae01b3ce0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 1 16:01:28 2014 -0700
+
+    PG: only complete replicas should count toward min_size
+    
+    Backport: emperor,dumpling,cuttlefish
+    Fixes: #7805
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0d5d3d1a30685e7c47173b974caa12076c43a9c4)
+
+commit 3f1d7f5e0a67ad646de465335fb7ee00eb07e220
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 15:10:43 2014 -0700
+
+    mon/PGMonitor: set tid on no-op PGStatsAck
+    
+    The OSD needs to know the tid.  Both generally, and specifically because
+    the flush_pg_stats may be blocking on it.
+    
+    Fixes: #8280
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5a6ae2a978dcaf96ef89de3aaa74fe951a64def6)
diff --git a/doc/changelog/v0.72.2.txt b/doc/changelog/v0.72.2.txt
new file mode 100644
index 000000000..66fd667e0
--- /dev/null
+++ b/doc/changelog/v0.72.2.txt
@@ -0,0 +1,189 @@
+commit a913ded2ff138aefb8cb84d347d72164099cfd60
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 20 19:28:37 2013 +0000
+
+    v0.72.2
+
+commit b570c8014325fef590d16e6157cb33026155932c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:11:01 2013 -0800
+
+    rgw: fix use-after-free when releasing completion handle
+
+    Backport: emperor, dumpling
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8890ab2d46fe8e12200a0d2f9eab31c461fb871)
+
+commit 451381b4dba9d50fbee2accf5d75562615001ef3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:10:21 2013 -0800
+
+    rgw: don't return data within the librados cb
+
+    Fixes: #7030
+    The callback is running within a single Finisher thread, thus we
+    shouldn't block there. Append read data to a list and flush it within
+    the iterate context.
+
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)
+
+commit c044b218cb32d7ef2f2aa5db81ebf38ad0dc1049
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 1 22:21:31 2013 -0800
+
+    Partial revert "mon: osd pool set syntax relaxed, modify unit tests"
+
+    This reverts commit 08327fed8213a5d24cd642e12b38a171b98924cb, except
+    for the hashpspool bit.  We switched back to an integer argument in
+    commit 337195f04653eed8e8f153a5b074f3bd48408998.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e80ab94bf44e102fcd87d16dc11e38ca4c0eeadb)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 1983ed8013637e9b80ca4a0e8d2800d6a36b120e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Nov 19 23:21:11 2013 +0000
+
+    mon: OSDMonitor: drop cmdval_get() for unused variable
+
+    We don't ever use any value as a float, so just drop obtaining it.  This
+    makes it easier to partially revert 2fe0d0d9 in an upcoming patch.
+
+    Backport: emperor
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7191bb2b2485c7819ca7b9d9434d803d0c94db7a)
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 1c4759cebb11ae1aa84aa3cbdb2b31e6d111f28e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Nov 22 02:10:35 2013 +0000
+
+    mon: OSDMonitor: receive CephInt on 'osd pool set' instead on CephString
+
+    This partially reverts 2fe0d0d9 in order to allow Emperor monitors to
+    forward mon command messages to Dumpling monitors without breaking a
+    cluster.
+
+    The need for this patch became obvious after issue #6796 was triggered.
+    Basically, in a mixed cluster of Emperor/Dumpling monitors, if a client
+    happens to obtain the command descriptions from an Emperor monitor and
+    then issue an 'osd pool set' this can turn out in one of two ways:
+
+    1. client msg gets forwarded to an Emperor leader and everything's a-okay;
+    2. client msg gets forwarded to a Dumpling leader and the string fails to
+    be interpreted without the monitor noticing, thus leaving the monitor with
+    an uninitialized variable leading to trouble.
+
+    If 2 is triggered, a multitude of bad things can happen, such as thousands
+    of pg splits, due to a simple 'osd set pool foo pg_num 128' turning out
+    to be interpreted as 109120394 or some other random number.
+
+    This patch is such that we make sure the client sends an integer instead
+    of a string. We also make sure to interpret anything the client sends as
+    possibly being a string, or an integer.
+
+    Fixes: 6796
+    Backport: emperor
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 337195f04653eed8e8f153a5b074f3bd48408998)
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit c2e467b42dfaecf082a3838b10e873f9f8bd5633
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 25 13:43:43 2013 -0800
+
+    init, upstart: prevent daemons being started by both
+
+    There can be only one init system starting a daemon. If there is a
+    host entry in ceph.conf for a daemon, sysvinit would try to start it
+    even if the daemon's directory did not include a sysvinit file. This
+    preserves backwards compatibility with older installs using sysvinit,
+    but if an upstart file is present in the daemon's directory, upstart
+    will try to start them, regardless of host entries in ceph.conf.
+
+    If there's an upstart file in a daemon's directory and a host entry
+    for that daemon in ceph.conf, both sysvinit and upstart would attempt
+    to manage it.
+
+    Fix this by only starting daemons if the marker file for the other
+    init system is not present. This maintains backwards compatibility
+    with older installs using neither sysvinit or upstart marker files,
+    and does not break any valid configurations. The only configuration
+    that would break is one with both sysvinit and upstart files present
+    for the same daemon.
+
+    Backport: emperor, dumpling
+    Reported-by: Tim Spriggs <tims@uahirise.org>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5e34beb61b3f5a1ed4afd8ee2fe976de40f95ace)
+
+commit 8812c4e958b154ca401c7257fd48ccaffe013639
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 27 13:34:00 2013 -0800
+
+    rgw: don't error out on empty owner when setting acls
+
+    Fixes: #6892
+    Backport: dumpling, emperor
+    s3cmd specifies empty owner field when trying to set acls on object
+    / bucket. We errored out as it didn't match the current owner name, but
+    with this change we ignore it.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 14cf4caff58cc2c535101d48c53afd54d8632104)
+
+commit 69e055f367ae37dd771c050f158d684e2a3cb246
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 07:04:01 2013 -0800
+
+    rgw: lower some debug message
+
+    Fixes: #6084
+    Backport: dumpling, emperor
+
+    Reported-by: Ron Allred <rallred@itrefined.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b35fc1bc2ec8c8376ec173eb1c3e538e02c1694e)
+
+commit 53951fd5459d6396b6ab59b52dc33df0ebcf4414
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 12 15:15:26 2013 -0800
+
+    ReplicatedPG: test for missing head before find_object_context
+
+    find_object_context doesn't return EAGAIN for a missing head.
+    I chose not to change that behavior since it might hide bugs
+    in the future.  All other callers check for missing on head
+    before calling into find_object_context because we potentially
+    need head or snapdir to map a snapid onto a clone.
+
+    Backport: emperor
+    Fixes: 6758
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit dd9d8b020286d5e3a69455023c3724a7b436d687)
+
+commit 83ee6843de6432278dc2891526691006c51eb1fa
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 18 14:39:12 2013 -0800
+
+    osd: fix bench block size
+
+    The command was declared to take 'size' in dumpling, but was trying to
+    read 'bsize' instead, so it always used the default of 4MiB. Change
+    the bench command to read 'size', so it matches what existing clients
+    are sending.
+
+    Fixes: #6795
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 40a76ef0d09f8ecbea13712410d9d34f25b91935)
diff --git a/doc/changelog/v0.80.1.txt b/doc/changelog/v0.80.1.txt
new file mode 100644
index 000000000..70d6fd6af
--- /dev/null
+++ b/doc/changelog/v0.80.1.txt
@@ -0,0 +1,216 @@
+commit a38fe1169b6d2ac98b427334c12d7cf81f809b74 (tag: refs/tags/v0.80.1)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:12:54 2014 +0000
+
+    0.80.1
+
+commit ab873622c6d41bc8053a1d205da2e4b55e77351d
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:11:33 2014 +0000
+
+    0.80.1
+
+commit 03e7d1d550132a55e87972123d5bc3c0bb0cc981
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:10:56 2014 +0000
+
+    0.80.1
+
+commit 7a4919c7fa10020643df09e012376e4b563ca17d
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:09:01 2014 +0000
+
+    0.80.1
+
+commit ce852b29dc5cf9857a6bfeef7839293cfadf1877 (refs/remotes/gh/firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 2 16:21:26 2014 -0700
+
+    Revert "ReplicatedPG: block scrub on blocked object contexts"
+
+    This reverts commit e66f2e36c06ca00c1147f922d3513f56b122a5c0.
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+    0f3235d46c8fd6c537bd4aa8a3faec6c00f311a8 is the firefly commit
+    corresponding to e66f2e36c06ca00c1147f922d3513f56b122a5c0.
+
+    (cherry picked from commit 84728058dbb91b8ed062240b3373b18078f0c9ca)
+
+commit db8873b69c73b40110bf1512c114e4a0395671ab
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 16:55:27 2014 -0700
+
+    rgw: fix stripe_size calculation
+
+    Fixes: #8299
+    Backport: firefly
+    The stripe size calculation was broken, specifically affected cases
+    where we had manifest that described multiple parts.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9968b938b5d47fdf3a67db134bd2ea6bf3a28086)
+
+commit e0fb2e63074d5f71679bb68656336f68e8190f39
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 11:06:29 2014 -0700
+
+    rgw: cut short object read if a chunk returns error
+
+    Fixes: #8289
+    Backport: firefly, dumpling
+    When reading an object, if we hit an error when trying to read one of
+    the rados objects then we should just stop. Otherwise we're just going
+    to continue reading the rest of the object, and since it can't be sent
+    back to the client (as we have a hole in the middle), we end up
+    accumulating everything in memory.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 03b0d1cfb7bd30a77fedcf75eb06476b21b14e95)
+
+commit 328665db17bf87d36a3dcbf5f0d0d1a9410e2f96
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:34:04 2014 -0700
+
+    rgw: send user manifest header field
+
+    Fixes: #8170
+    Backport: firefly
+    If user manifest header exists (swift) send it as part of the object
+    header data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5cc5686039a882ad345681133c9c5a4a2c2fd86b)
+
+commit cdb0fac29693fb0250b515891804c4c7fd3045c7
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Apr 11 15:03:37 2014 +0800
+
+    client: add asok command to kick sessions that were remote reset
+
+    Fixes: #8021
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 09a1bc5a4601d356b9cc69be8541e6515d763861)
+
+commit 4a91196bf49936471a731f9316ef904fda982b26
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 18 13:50:11 2014 -0700
+
+    osd: throttle snap trimmming with simple delay
+
+    This is not particularly smart, but it is *a* knob that lets you make
+    the snap trimmer slow down.  It's a flow and a simple delay, so it is
+    adjustable at runtime.  Default is 0 (no change in behavior).
+
+    Partial solution for #6278.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4413670d784efc2392359f0f22bca7c9056188f4)
+
+commit 3ba2e228db1829a02f9cd31225672604fdc8adf7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 14:48:35 2014 -0700
+
+    mon/MonClient: remove stray _finish_hunting() calls
+
+    Callig _finish_hunting() clears out the bool hunting flag, which means we
+    don't retry by connection to another mon periodically.  Instead, we send
+    keepalives every 10s.  But, since we aren't yet in state HAVE_SESSION, we
+    don't check that the keepalives are getting responses.  This means that an
+    ill-timed connection reset (say, after we get a MonMap, but before we
+    finish authenticating) can drop the monc into a black hole that does not
+    retry.
+
+    Instead, we should *only* call _finish_hunting() when we complete the
+    authentication handshake.
+
+    Fixes: #8278
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 77a6f0aefebebf057f02bfb95c088a30ed93c53f)
+
+commit 383f6440ce34b0d84c0dfa06f561896dad6862f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 16:41:26 2014 -0700
+
+    osd/ReplicatedPG: fix trim of in-flight hit_sets
+
+    We normally need to stat the hit_set to know how many bytes to adjust the
+    stats by.  If the hit_set was just written, we will get ENOENT.
+
+    Get the obc instead, which will either get the in-memory copy (because the
+    repop is still in flight) or load it off of disk.
+
+    Fixes: #8283
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 72fdd557c35cb721d4b502c5a8f68c878f11a19c)
+
+commit ef35448e63916102261ff6198f2c0ab4e4a57e9b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 6 11:01:27 2014 -0700
+
+    osd/ReplicatedPG: fix whiteouts for other cache mode
+
+    We were special casing WRITEBACK mode for handling whiteouts; this needs to
+    also include the FORWARD and READONLY modes.  To avoid having to list
+    specific cache modes, though, just check != NONE.
+
+    Fixes: #8296
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e387d62ed95898db8a7d7163c2bacc363b8f617)
+
+commit e7df73dd7aaf5a0b1171f73d6695d26cd25b7b35
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 1 16:53:17 2014 -0700
+
+    osd: Prevent divide by zero in agent_choose_mode()
+
+    Fixes: #8175
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f47f867952e6b2a16a296c82bb9b585b21cde6c8)
+
+commit 022d467b5d6b77c17b6fdaeec8369cae61e9e5a4
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon Apr 21 23:52:04 2014 -0700
+
+    osd, common: If agent_work() finds no objs to work on delay 5 (default) secs
+
+    Add config osd_agent_delay_time of 5 seconds
+    Honor delay by ignoring agent_choose_mode() calls
+    Add tier_delay to logger
+    Treat restart after delay like we were previously idle
+
+    Fixes: #8113
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b7d31e5f5952c631dd4172bcb825e77a13fc60bc)
+
+commit 6a55c3bc3caf46652e962fa9434900fb494d1e6c
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu May 1 18:54:30 2014 -0700
+
+    osd/ReplicatedPG: agent_work() fix next if finished early due to start_max
+
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 9cf470cac8dd4d8f769e768f2de6b9eb67a3c3af)
+
+commit 1b899148a729235ab2835d368077f18e62a36a93
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sat May 3 12:53:06 2014 +0800
+
+    Fix clone problem
+
+    When clone happened, the origin header also will be updated in GenericObjectMap,
+    so the new header wraper(StripObjectHeader) should be updated too.
+
+    Fix #8282
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3aee1e0ffe0583f74c02d9c9e86c7fb267f3515c)
diff --git a/doc/changelog/v0.80.10.txt b/doc/changelog/v0.80.10.txt
new file mode 100644
index 000000000..be82bcc9b
--- /dev/null
+++ b/doc/changelog/v0.80.10.txt
@@ -0,0 +1,3308 @@
+commit ea6c958c38df1216bf95c927f143d8b13c4a9e70 (tag: refs/tags/v0.80.10)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Thu Jun 18 09:48:56 2015 -0700
+
+    0.80.10
+
+commit 9b7f09e280b4610b9f85ac34ec014018f0e2e1d3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 17 09:35:28 2015 -0700
+
+    qa/workunits/rados/test-upgarde-v9.0.1: fix exclude syntax
+    
+    It's -, then a list of all exclusions separated by :.  There are just 2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78d894a634d727a9367f809a1f57234e5e6935be)
+
+commit 4e28fbc52db7d808aeaede884f342beafb7fc581
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 16 21:05:29 2015 -0700
+
+    qa/workunits/rados/test-upgrade-v9.0.1: skip one more evict test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e8d60a80ce31860eac76a1f6489a35e1795a0c0)
+
+commit 34ba3719997fed6d0b8148f21e59b446bdf42962
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 15 15:12:43 2015 -0700
+
+    qa: add compatibility filtered rados api tests for upgrades
+    
+    Post-9.0.1, the evict op returns success when an object doesn't exist
+    in the cache tier. Skip the tests that are incompatible across
+    versions.
+    
+    Fixes: #11548
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 348a3d3c9880e7d022e71a2faafe51c8f771406e)
+
+commit d1f478200342f422ee6e563bf7aad54ba38b3ed3
+Merge: d0f9c5f 7f1abd9
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 10 15:03:48 2015 -0700
+
+    Merge pull request #4924 from ceph/wip-11955-firefly
+    
+    11955: ceph.spec.in: package mkcephfs on EL6
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7f1abd9053088ded6613aeca4e7c9489c44910e4
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 10 15:43:41 2015 -0600
+
+    ceph.spec.in: package mkcephfs on EL6
+    
+    Commit efbca0465c2946e113771966df08cf7cf37b1196 added mkcephfs to the
+    RPM %files listing, but this /usr/sbin path is only correct for CentOS
+    7. In CentOS 6, the utility is present at /sbin/mkcephfs instead. This
+    causes rpmbuild to fail to build the tip of the firefly branch on EL6.
+    
+    Adjust the RPM %files list so we properly package mkcephfs on both EL7
+    and EL6.
+    
+    http://tracker.ceph.com/issues/11955 Refs: #11955
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit d0f9c5f47024f53b4eccea2e0fde9b7844746362
+Merge: efbca04 9930138
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 4 19:07:03 2015 +0200
+
+    Merge pull request #4851 from ceph/wip-10873-firefly
+    
+    rgw: generate the "Date" HTTP header for civetweb.
+
+commit 99301384be96997203682679c0430ca0a53be5d3 (refs/remotes/gh/wip-10873-firefly)
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 18 15:48:43 2015 +0100
+
+    rgw: generate the "Date" HTTP header for civetweb.
+    
+    Fixes: #10873
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ea384f83b601f60e135c3d3f960fdb75a919dd84)
+
+commit efbca0465c2946e113771966df08cf7cf37b1196
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 13:03:50 2015 -0400
+
+    ceph.spec: add mkcephfs to ceph.rpm
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit e785f1a2bb4cd77b0aaaccd78060183157ad842e
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Dec 2 13:58:10 2014 -0700
+
+    packaging: package ceph-disk(8)
+    
+    The ceph-disk man page was added in
+    a450cab2b8148cb8a9b043d629feccf89e5aabac, but this was not added to the
+    RPM or DEB packaging. Add it here.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit b743a951114b00bbb6e14fb88f1928b504bc0f8b)
+
+commit 819cf6ddb986b37c5d1229c4ea330010e88bb615
+Merge: e93711a 665a857
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 2 19:53:03 2015 +0300
+
+    Merge pull request #4780 from oritwas/wip-10295-firefly
+    
+    rgw: civetweb should use unique request id
+    
+    Reviewd-by: Yehuda Sadeh <yehuda@rehat.com>
+
+commit e93711a40d680230b03fe722a4da49a6f9a9b667
+Merge: 071c943 50e8579
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 2 19:02:07 2015 +0300
+
+    Merge pull request #4829 from oritwas/wip-negative_content_length-firefly
+    
+    rgw: don't allow negative / invalid content length
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 50e85797507a3ba13193f368cff461c08e44a9b3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Aug 1 16:15:36 2014 -0700
+
+    rgw: don't allow negative / invalid content length
+    
+    Certain frontends (e.g., civetweb) don't filter such requests.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0e74b7a1d56733358e2f1d3df4386125a94c2966)
+
+commit 071c94385ee71b86c5ed8363d56cf299da1aa7b3
+Merge: 68211f6 c632ef1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 27 15:36:15 2015 -0700
+
+    Merge pull request #4766 from SUSE/wip-11673-firefly
+    
+    Debian: ceph-test and rest-bench debug packages should require their respective binary packages
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 665a85701177230365e43a351d7722cb2adbde93
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 23 17:36:47 2015 +0200
+
+    rgw: civetweb should use unique request id
+    
+    max_req_id was moved to RGWRados and changed to atomic64_t.
+    
+    The same request id resulted in gc giving the same idtag to all objects
+    resulting in a leakage of rados objects. It only kept the last deleted object in
+    it's queue, the previous objects were never freed.
+    
+    Fixes: 10295
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c262259)
+    
+    Conflicts:
+    	src/rgw/rgw_main.cc
+    	src/rgw/rgw_rados.h
+
+commit c632ef13e05654d0e5ddc79fc7e9553cad8cbed0
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon May 18 10:50:58 2015 -0600
+
+    debian: set rest-bench-dbg ceph-test-dbg dependencies
+    
+    Debian's debug packages ought to depend on their respective binary
+    packages. This was the case for many of our ceph packages, but it was
+    not the case for ceph-test-dbg or rest-bench-dbg.
+    
+    Add the dependencies on the relevant binary packages, pinned to
+    "= ${binary:Version}" per convention.
+    
+    http://tracker.ceph.com/issues/11673 Fixes: #11673
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f898ec1e4e3472b0202280f09653a769fc62c8d3)
+
+commit 68211f695941ee128eb9a7fd0d80b615c0ded6cf
+Merge: 7d11b19 cd8f183
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 18 14:25:59 2015 +0200
+
+    Merge pull request #4697 from ceph/wip-11622-firefly
+    
+    Wip 11622 firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cd8f1830eb5d7ff75b17d7f0915ee4b3b834b149 (refs/remotes/gh/wip-11622-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 13 17:05:22 2015 -0700
+
+    rgw: merge manifests correctly when there's prefix override
+    
+    Fixes: #11622
+    Backport: hammer, firefly
+    
+    Prefix override happens in a manifest when a rados object does not
+    conform to the generic prefix set on the manifest. When merging
+    manifests (specifically being used in multipart objects upload), we need
+    to check if the rule that we try to merge has a prefix that is the same
+    as the previous rule. Beforehand we checked if both had the same
+    override_prefix setting, but that might not apply as both manifests
+    might have different prefixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 389ae6739ddc6239a4dd7c5f7f9bfc9b645b8577)
+
+commit eef3d2f1c400573db90936fd417769183950b6ee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 12 16:40:10 2015 -0700
+
+    rgw: restore buffer of multipart upload after EEXIST
+    
+    Fixes #11604
+    Backport: hammer, firefly
+    
+    When we need to restart a write of part data, we need to revert to
+    buffer to before the write, otherwise we're going to skip some data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 580ccaec12daae64c38a1616d0be907bdd70a888)
+
+commit 7d11b19003503a9db1572d01f7a170e35b29017d
+Merge: 114f2e9 9b33965
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 15 10:27:42 2015 -0700
+
+    Merge pull request #4414 from xinxinsh/wip-11125-firefly
+    
+    rgw: keystone token cache does not work correctly
+
+commit 114f2e9bb5665760a5be9816785508f1c97662d5
+Merge: ac7d28a f33effc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:03:04 2015 +0200
+
+    Merge pull request #4415 from xinxinsh/wip-11244-firefly
+    
+    cancel_pull: requeue waiters
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ac7d28a6977084da0b70a3f2d0a54c8fa55b16fa
+Merge: f273792 1f6b1bb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:02:27 2015 +0200
+
+    Merge pull request #4416 from xinxinsh/wip-10976-firefly
+    
+    fix PG::all_unfound_are_queried_or_lost for non-existent osds
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f273792c675aca95694cd36794d8a854731bf308
+Merge: 8a6632b 019b28b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:02:05 2015 +0200
+
+    Merge pull request #4556 from xinxinsh/wip-11429-firefly
+    
+    OSD::load_pgs: we need to handle the case where an upgrade from earlier versions which ignored non-existent pgs resurrects a pg with a prehistoric osdmap
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8a6632b4e5ffad305f92c89656b161a521529e45
+Merge: 156c385 a71f309
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 14 09:50:28 2015 -0700
+
+    Merge pull request #4638 from SUSE/wip-11453-firefly
+    
+    run RGW as root
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 019b28b6397dbc3111faa6a8eb57349a48f408d8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Apr 20 23:45:57 2015 -0700
+
+    OSD: handle the case where we resurrected an old, deleted pg
+    
+    Prior to giant, we would skip pgs in load_pgs which were not present in
+    the current osdmap.  Those pgs would eventually refer to very old
+    osdmaps, which we no longer have causing the assertion failure in 11429
+    once the osd is finally upgraded to a version which does not skip the
+    pgs.  Instead, if we do not have the map for the pg epoch, complain to
+    the osd log and skip the pg.
+    
+    Fixes: 11429
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit fbfd50de5b9b40d71d2e768418a8eca28b1afaca)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+            resolved by add a new comment line
+
+commit 156c385ef726e56d219c3383a4015c50aec2758a
+Merge: fcd0ea3 8cc4bc1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue May 12 13:57:36 2015 +0200
+
+    Merge pull request #4379 from ceph/wip-11416
+    
+    rgw: use correct objv_tracker for bucket instance
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a71f3091a4ea810c02517642fb4ab9ce5516b452
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Apr 22 16:36:42 2015 -0600
+
+    init-radosgw: run RGW as root
+    
+    The ceph-radosgw service fails to start if the httpd package is not
+    installed. This is because the init.d file attempts to start the RGW
+    process with the "apache" UID. If a user is running civetweb, there is
+    no reason for the httpd or apache2 package to be present on the system.
+    
+    Switch the init scripts to use "root" as is done on Ubuntu.
+    
+    http://tracker.ceph.com/issues/11453 Refs: #11453
+    
+    Reported-by: Vickey Singh <vickey.singh22693@gmail.com>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 47339c5ac352d305e68a58f3d744c3ce0fd3a2ac)
+
+commit fcd0ea3383582e7a1dc8091e7a48e1d4bbaa76ee
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri May 8 11:29:44 2015 -0700
+
+    workunits: remove defunct cephfs set_layout test
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit df053b86a89edf8f390400dad8c5e654e14df327
+Merge: aef0272 62645d3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 6 13:07:11 2015 -0700
+
+    Merge pull request #4571 from ceph/wip-11256-firefly
+    
+    Wip 11256 firefly
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit aef0272d72afaef849b5d4acbf55626033369ee8
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Fri Mar 27 19:34:12 2015 -0700
+
+    java: libcephfs_jni.so is in /usr/lib64 on rhel
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit aed3434dc7c5161c72c7d5655faa3bc693fc9777)
+
+commit a551a23d36e3f30ff5b0679a98ee760166ae47ae
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Mar 2 21:04:25 2015 +0800
+
+    qa/workunits/fs/misc: fix filelock_interrupt.py
+    
+    Handle the case that kernel does not support fcntl.F_OFD_SETLK.
+    Also fix the code that checks if fnctl fails with errno == EINTR.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4ececa3dc4a21b98f61a592da9e2be60a0d71625)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 07031b151b4ac6677b1f663ac200d09088deff64
+Merge: 83f8d43 3fb97e2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:32:21 2015 +0200
+
+    Merge pull request #4385 from xinxinsh/wip-11199-firefly
+    
+    osd: ENOENT on clone
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 83f8d434a5c245711922ea43a962160177aa5a40
+Merge: f1425e0 29bc9e5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:31:48 2015 +0200
+
+    Merge pull request #4384 from xinxinsh/wip-11197-firefly
+    
+    pg stuck stale after create with activation delay
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f1425e093e00343a5ae4b9ff56911bf654a5c152
+Merge: e980e00 69d9339
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:31:32 2015 +0200
+
+    Merge pull request #4382 from xinxinsh/wip-10718-firefly
+    
+    osd/PGLog.h: 279: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e980e0036909afc392f9bdf5532ce500af602c2f
+Merge: b36229b 110c354
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:23:04 2015 +0200
+
+    Merge pull request #4185 from ldachary/wip-11156-firefly
+    
+    FAILED assert(soid < scrubber.start || soid >= scrubber.end)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit b36229b5aeec669cd1494f47120ae207a393846f
+Merge: da75978 20c2175
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 28 08:11:18 2015 -0700
+
+    Merge pull request #4475 from ceph/wip-9538.firefly
+    
+    mon: OSDMonitor: fallback to json-pretty in case of invalid formatter
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 20c2175352c9d4e4d0e709d294fe6fe0695761cc
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Sep 19 15:28:36 2014 +0200
+
+    mon: osd find / metadata --format plain fallback
+    
+    ceph --format plain osd find 1 (and metadata) are not implemented and
+    must fallback to the default (json-pretty).
+    
+    http://tracker.ceph.com/issues/9538 Fixes: #9538
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 13780d755115387591888f94ea6c58ac0db3ecc4)
+
+commit da759784ebda11275106c3b280f1d32b64ade00a
+Merge: 1a10a7e e1d5773
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 27 10:30:49 2015 -0700
+
+    Merge pull request #4453 from ceph/wip-11454
+    
+    PG::actingset should be used when checking the number of acting OSDs for...
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1a10a7eac190249a41b5200f0cc5e3863e76c31d
+Merge: a860e2b b9da6f1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:14:00 2015 +0200
+
+    Merge pull request #4245 from ceph/wip-11113-firefly
+    
+    librbd: snap_remove should ignore -ENOENT errors
+    
+    Reviewed-by: Shu, Xinxin <xinxin.shu@intel.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a860e2b3c02ef4f1402b54b08a6bacd4b02cc07f
+Merge: dd15e54 364563a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:12:40 2015 +0200
+
+    Merge pull request #4206 from ceph/wip-5488-firefly
+    
+    librbd: acquire cache_lock before refreshing parent
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit dd15e54b2fae134126e9795a8a973e391c628eb7
+Merge: 51ff2b6 5404fbf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:09:23 2015 +0200
+
+    Merge pull request #3963 from dachary/wip-10153-firefly
+    
+    Rados.shutdown() dies with Illegal instruction (core dumped)
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 51ff2b60dc576fb444bcfea8f6cd3e12043ed5b8
+Merge: 12143ff c0cfd6e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:08:15 2015 +0200
+
+    Merge pull request #4383 from xinxinsh/wip-11144-firefly
+    
+    erasure-code-profile set races with erasure-code-profile rm
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1f6b1bb51d0b40899155313165ac5e3a21dde11a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 3 08:45:58 2015 +0200
+
+    osd: fix PG::all_unfound_are_queried_or_lost for non-existent osds
+    
+    A common mistake upon osd loss is to remove the osd from the crush map
+    before marking the osd lost. This tends to make it so that the user
+    can no longer mark the osd lost to satisfy all_unfound_are_queried_or_lost.
+    
+    The simple solution is for all_unfound_are_queried_or_lost to ignore
+    the osd if it does not exist.
+    
+    Fixes: #10976
+    Backports: firefly,giant
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5bb51320138ff714806d24312149f6275d546608)
+
+commit e1d57730ddfc92b17562080b10bc22fceb86539a
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Sep 29 08:21:10 2014 +0000
+
+    PG::actingset should be used when checking the number of acting OSDs for a given PG.
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    
+    (cherry picked from commit 19be358322be48fafa17b28054619a8b5e7d403b)
+    
+    Conflicts:
+    	src/osd/PG.cc	PG::get_backfill_priority() doesn't exist in firefly
+    			Variation in code related to no "undersized" state in firefly
+    
+    Fixes: #11454
+
+commit f33effccc0592fddfcd9c5c2f5c3385462aa9b84
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 26 10:26:48 2015 -0700
+
+    ReplicatedPG::cancel_pull: requeue waiters as well
+    
+    If we are in recovery_wait, we might not recover that object as part of
+    recover_primary for some time.  Worse, if we are waiting on a backfill
+    which is blocked waiting on a copy_from on the missing object in
+    question, it can become a dead lock.
+    
+    Fixes: 11244
+    Backport: firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 0b2e272430dd7433e6763be99b8a4cb127d9be19)
+
+commit 9b3396516c00ef931bab2d4aa6288ad974ec579d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 3 11:03:35 2015 -0800
+
+    rgw: update keystone cache with token info
+    
+    Fixes: #11125
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6616294aa140ceb83cc61c6ab6f9947636f5e67d)
+
+commit 8cc4bc162c8c1c240e62840d968b967f5f47b682 (refs/remotes/gh/wip-11416)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Dec 17 17:12:43 2014 -0800
+
+    rgw: use correct objv_tracker for bucket instance
+    
+    When trying to create a bucket that already existed, use the
+    objv_tracker of the newly created instance, and not of the original
+    bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe158ecc25feefcea8aea4133118e4a84900a8ec)
+
+commit c0cfd6e56ca9d17241da159295bcee7cf44c9ba3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 18 14:17:00 2015 +0100
+
+    osd: erasure-code-profile incremental rm before set
+    
+    It is possible for an incremental change to have both a rm and a set for
+    a given erasure code profile. It only happens when a rm is followed by a
+    set. When a set is followed by a rm, the rm will remove the pending set
+    in the incremental change.
+    
+    The logic is the same for pool create and pool delete.
+    
+    We must apply the incremental erasure-code-profile removal before the
+    creation otherwise rm and set in the same proposal will ignore the set.
+    
+    This fix is minimal. A better change would be that erasure-code-profile
+    set checks if there is a pending removal and wait_for_finished_proposal
+    before creating.
+    
+    http://tracker.ceph.com/issues/11144 Fixes: #11144
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0d52aca0d0c302983d03b0f5213ffed187e4ed63)
+    
+    Conflicts:
+    	src/osd/OSDMap.cc
+            resolved by replacing i++ with ++i
+
+commit 3fb97e25b194e92112077385b10381801e02ddb9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 24 15:14:34 2015 -0700
+
+    ReplicatedPG: trim backfill intervals based on peer's last_backfill_started
+    
+    Otherwise, we fail to trim the peer's last_backfill_started and get bug 11199.
+    
+    1) osd 4 backfills up to 31bccdb2/mira01213209-286/head (henceforth: foo)
+    
+    2) Interval change happens
+    
+    3) osd 0 now finds itself backfilling to 4 (lb=foo) and osd.5
+    (lb=b6670ba2/mira01213209-160/snapdir//1, henceforth: bar)
+    
+    4) recover_backfill causes both 4 and 5 to scan forward, so 4 has an interval
+    starting at foo, 5 has an interval starting at bar.
+    
+    5) Once those have come back, recover_backfill attempts to trim off the
+    last_backfill_started, but 4's interval starts after that, so foo remains in
+    osd 4's interval (this is the bug)
+    
+    7) We serve a copyfrom on foo (sent to 4 as well).
+    
+    8) We eventually get to foo in the backfilling. Normally, they would have the
+    same version, but of course we don't update osd.4's interval from the log since
+    it should not have received writes in that interval. Thus, we end up trying to
+    recover foo on osd.4 anyway.
+    
+    9) But, an interval change happens between removing foo from osd.4 and
+    completing the recovery, leaving osd.4 without foo, but with lb >= foo
+    
+    Fixes: #11199
+    Backport: firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1388d6bd949a18e8ac0aecb0eb79ffb93d316879)
+
+commit 29bc9e56a978d4e18a507e71858baa21037964c4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 24 10:48:02 2015 -0700
+
+    PG: set/clear CREATING in Primary state entry/exit
+    
+    Previously, we did not actually set it when we got a pg creation message from
+    the mon.  It would actually get set on the first start_peering_interval after
+    that point.  If we don't get that far, but do send a stat update to the mon, we
+    can end up with 11197.  Instead, let's just set it and clear it upon entry into
+    and exit from the Primary state.
+    
+    Fixes: 11197
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ddf0292250dfb1040d3cad96fa2cf886fd65933c)
+
+commit 12143ff9b25fdd96f8d1a9cecb1329c7f354d414
+Merge: 53eff01 bf1d8e8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 6 10:24:06 2015 -0700
+
+    Merge pull request #4275 from jeanchlopez/wip-11160-firefly
+    
+    rgw: shouldn't need to disable rgw_socket_path if frontend is configured
+
+commit bf1d8e888439df75feadde76bafe7d07d0e8481a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:49:13 2015 -0700
+
+    rgw: don't use rgw_socket_path if frontend is configured
+    
+    Fixes: #11160
+    Backport: hammer, firefly
+    
+    Previously if we wanted to use the tcp fcgi socket, we needed to clear
+    rgw_socket_path.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 53eff01f1e7ecd3783bb05c47588a134d4fededc
+Merge: 899738e 80afc5e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Apr 1 16:30:23 2015 -0700
+
+    Merge pull request #4247 from ceph/wip-11303
+    
+    Fix do_autogen.sh so that -L is allowed
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 80afc5eca293e5e2f168f219931a2f554040cdd8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 1 19:10:33 2015 -0400
+
+    Fix do_autogen.sh so that -L is allowed
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit b9da6f1025683d240baa73a9fcccbf2780035231
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 18:40:49 2015 -0400
+
+    librbd: snap_remove should ignore -ENOENT errors
+    
+    If the attempt to deregister the snapshot from the parent
+    image fails with -ENOENT, ignore the error as it is safe
+    to assume that the child is not associated with the parent.
+    
+    Fixes: #11113
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cf8094942ccdba831e03e5a79451cfa5d78a135f)
+
+commit b250b3cfae69f4e8354027fae26fd85a792da0df
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 18:35:07 2015 -0400
+
+    librbd: get_parent_info should protect against invalid parent
+    
+    get_parent_info should return -ENOENT if the image does not
+    have an associated parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 21afd0ef8e1fb81c5ace5fd403513c542e2413e3)
+
+commit 69d9339fb352342a87e1148e9b5161246b27776a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 2 09:07:27 2015 -0800
+
+    PGLog: improve PGLog::check() debugging
+    
+    Related: 10718
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c656bcec2441c90b084ca50a17c37625d69942a1)
+
+commit 73cbb6f34a5a556dbbd4db2e6f81391d83447efd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 17 10:07:03 2015 -0700
+
+    PGLog::merge_log: in tail extend case, log.log might be empty
+    
+    Even if the tail != last_update, the log might be empty due to split
+    moving all entries into other logs.
+    
+    Fixes: 10718
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f5a2aef3db29b99634250fd388a0c2d9b9a23d38)
+
+commit 62645d30b0ceee075a56bc692fd65cee85b09aa5 (refs/remotes/gh/wip-11256-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Mar 30 17:34:57 2015 -0700
+
+    cls_rgw: use multimap to keep pending operations in bucket index
+    
+    Fixes: #11256
+    Multiple concurrent requests might be sent using the same tag, need the
+    entry map to be able to hold multiple entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7538319dd7aa80a3318c108d345dee8044cf20a8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Mar 27 16:32:48 2015 -0700
+
+    rgw: generate new tag for object when setting object attrs
+    
+    Fixes: #11256
+    Backport: firefly, hammer
+    
+    Beforehand we were reusing the object's tag, which is problematic as
+    this tag is used for bucket index updates, and we might be clobbering a
+    racing update (like object removal).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 364563aac979fdf5ccbb6c588051d097a26bc594
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 11:04:22 2015 -0400
+
+    librbd: acquire cache_lock before refreshing parent
+    
+    cache_lock needs to be acquired before snap_lock to avoid
+    the potential for deadlock.
+    
+    Fixes: #5488
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 110c35498942ea0feec395b6e7992f802dd740ce
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 20 15:28:15 2015 -0700
+
+    ReplicatedPG::promote_object: check scrubber and block if necessary
+    
+    Otherwise, we might attempt to promote into an in-progress scrub
+    interval causing 11156.  I would have added a return value to
+    promote_object(), but could not find an existing user which
+    cared to distinguish the cases, even with a null op passed.
+    All existing users are in maybe_handle_cache.  The ones which
+    pass a null op are for promoting the object in parallel
+    with a proxy -- a case where not actually performing the promote
+    does not really matter.
+    
+    Fixes: #11156
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 65bb4df599541cd2e0f195b905f24f529e255c00)
+
+commit 899738e10e82b50dcf7dfffe5cc83937179bf323
+Merge: 15acfe7 9437cb1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 20:38:43 2015 +0100
+
+    Merge pull request #4126 from dzafman/wip-11176-firefly
+    
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 15acfe7f226ca7bc2c942d5fbcd3a40bd41e5930
+Merge: 6c95cd2 cddbff9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 21 11:38:28 2015 +0100
+
+    Merge pull request #4079 from dachary/wip-11157-firefly
+    
+    doc,tests: force checkout of submodules
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 6c95cd24f6f4a1a933f8799754831e7a5c0a725d
+Merge: ea79a3b 43053fc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Mar 20 21:06:17 2015 -0700
+
+    Merge pull request #4129 from dzafman/wip-11139-firefly
+    
+    ceph-objectstore-tool: Use exit status 11 for incompatible import attemp...
+
+commit 43053fcd8969c406969fef67613aa37ad1cc86bf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 17:48:01 2015 -0700
+
+    ceph-objectstore-tool: Use exit status 11 for incompatible import attempt
+    
+    This is used so upgrade testing doesn't generate false failure.
+    Fixes: #11139
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 175aff8afe8215547ab57f8d8017ce8fdc0ff543)
+
+commit 9437cb1b284ec9366b51d660396c2c8a9366b31f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 16:57:40 2015 -0700
+
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Fixes: #11176
+    Backport: firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5b23f5b5892b36fb7d06efc0d77e64a24ef6e8c9)
+
+commit ea79a3b122917ddf5dc1972bb9dc5b20f78e2f18
+Merge: c1c20d8 07fc9f6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:44:39 2015 +0100
+
+    Merge pull request #3970 from ceph/firefly-11053
+    
+    mds: fix assertion caused by system clock backwards
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c1c20d89d511499d678fdba0667581e88b9b5d95
+Merge: 2fbb9a6 63b39db
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:42:58 2015 +0100
+
+    Merge pull request #4021 from ceph/wip-7737-firefly
+    
+    osd: fix negative degraded objects during backfilling
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2fbb9a63a82ac5136b033799aec303fc151b25e5
+Merge: 3160e8d 2c7eda6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:38:26 2015 +0100
+
+    Merge pull request #3952 from dachary/wip-9986-firefly
+    
+    objecter: fix map skipping
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3160e8d6918a04d1fec9fdccbc30bf007c7940c6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:55:24 2015 -0700
+
+    init-radosgw*: don't require rgw_socket_path to be defined
+    
+    Fixes: #11159
+    Backport: hammer, firefly
+    
+    Scripts required rgw_socket_path to exist in order to start radosgw.
+    This is not needed.
+    
+    Reported-by: Dan Mick <dmick@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6823bcdcd0ce72cd223e809291f46d82da76115c)
+
+commit cddbff9a3653d6257d13e0ac411cd6e4cd71feef
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Mar 19 00:32:39 2015 +0100
+
+    doc,tests: force checkout of submodules
+    
+    When updating submodules, always checkout even if the HEAD is the
+    desired commit hash (update --force) to avoid the following:
+    
+        * a directory gmock exists in hammer
+        * a submodule gmock replaces the directory gmock in master
+        * checkout master + submodule update : gmock/.git is created
+        * checkout hammer : the gmock directory still contains the .git from
+          master because it did not exist at the time and checkout won't
+          remove untracked directories
+        * checkout master + submodule update : git rev-parse HEAD is
+          at the desired commit although the content of the gmock directory
+          is from hammer
+    
+    http://tracker.ceph.com/issues/11157 Fixes: #11157
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 63b39dbd529936e60d0fd08dffd35f82b3d1729c (refs/remotes/gh/wip-7737-firefly)
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Thu Feb 26 08:13:12 2015 +0000
+
+    osd: fix negative degraded objects during backfilling
+    
+    When there is deleting requests during backfilling, the reported number of degraded
+    objects could be negative, as the primary's num_objects is the latest (locally) but
+    the number for replicas might not reflect the deletings. A simple fix is to ignore
+    the negative subtracted value.
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 14d7e36d3c978844da73d0e1c8a3a1ec863bac15)
+    
+    Conflicts:
+    	src/osd/PG.cc
+
+commit 493d285508914769cba3639b601ae6c20303af0d
+Merge: 9839726 8a25a51
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:20:36 2015 +0100
+
+    Merge pull request #3966 from theanalyst/wip-10698-backport
+    
+    rgw: fail s3 POST auth if keystone not configured
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9839726f009a3189013a23e8226d2f7618f1d56b
+Merge: dab85cb 651dc55
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:19:22 2015 +0100
+
+    Merge pull request #3934 from dachary/wip-10665-firefly
+    
+    rgw: Swift API. Dump container's custom metadata.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit dab85cb7fcc5770fe4d6f075cf583a33ea335aa3
+Merge: 0e55046 d0fd417
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:18:59 2015 +0100
+
+    Merge pull request #3929 from dachary/wip-10475-firefly
+    
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0e550461666d6e3d32981f99c8327931a6bd6d62
+Merge: 1ca6dd9 963439f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:18:24 2015 +0100
+
+    Merge pull request #3938 from dachary/wip-10770-firefly
+    
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 1ca6dd9676014a81983bd397e3154bf53243c7f2
+Merge: 66327d6 34d19e1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:17:56 2015 +0100
+
+    Merge pull request #3961 from dachary/wip-10106-firefly
+    
+    rgw: flush xml header on get acl request
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 66327d69a9bfa875d58b063c180a11b2769a093e
+Merge: de45d9e 9074eb7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:16:53 2015 +0100
+
+    Merge pull request #3958 from dachary/wip-10062-firefly
+    
+    rgw: check for timestamp for s3 keystone auth
+    
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Valery Tschopp <valery.tschopp@switch.ch>
+
+commit de45d9e2606d8fb6ea5533521163669e748e7d01
+Merge: 308f758 8ef14fc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 17:00:27 2015 +0100
+
+    Merge pull request #3942 from dachary/wip-10844-firefly
+    
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit 308f758f49d28c012f3ba765519144e5ea5794e6
+Merge: c59a081 ca42905
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:53:48 2015 +0100
+
+    Merge pull request #3936 from dachary/wip-10724-firefly
+    
+     backport ceph-disk man page to Firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c59a08135c35072fa05104e26b500553e800cdd1
+Merge: 331acae 8ec8f11
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:50:39 2015 +0100
+
+    Merge pull request #3940 from dachary/wip-8753-firefly
+    
+    ReplicatedPG::on_change: clean up callbacks_for_degraded_object
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 331acae4d6b34bb047c914945a2d9c0d9c7b9562
+Merge: fc364a7 4e32ff2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:49:56 2015 +0100
+
+    Merge pull request #3827 from dachary/wip-10259-firefly
+    
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::o...
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fc364a76f85eb5016ea460e5fd9c1603df374bcf
+Merge: 31d99d2 b0d0d44
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:48:02 2015 +0100
+
+    Merge pull request #3962 from dachary/wip-10150-firefly
+    
+    ReplicatedPG::scan_range: an object can disappear between the list and t...
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 31d99d2de714eb4a8eeb4a431cbc0d40ca749f15
+Merge: 45f0870 5865411
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:45:32 2015 +0100
+
+    Merge pull request #3948 from dachary/wip-9891-firefly
+    
+    DBObjectMap: lock header_lock on sync()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 45f087032e27c63dc459318717b05fe5f9888664
+Merge: 3050262 6207333
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:43:48 2015 +0100
+
+    Merge pull request #3949 from dachary/wip-9915-firefly
+    
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3050262d7dcb8ac20b9f86544461bb59f4b87cf5
+Merge: c0abc4d 0f31388
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:40:12 2015 +0100
+
+    Merge pull request #3944 from dachary/wip-9193-firefly
+    
+    messages/MWatchNotify: include an error code in the message
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit c0abc4d769afd00773a9f466ffd3feced9cdb17d
+Merge: f9acd3a f856739
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:39:41 2015 +0100
+
+    Merge pull request #3943 from dachary/wip-8011-firefly
+    
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f9acd3ad7397ac1e745beeeaf21b55ecd95484a1
+Merge: f95d327 ca96b59
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:39:16 2015 +0100
+
+    Merge pull request #3941 from dachary/wip-10817-firefly
+    
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f95d327cbc0750cfb77114c66082ddd5fc458b94
+Merge: 32de8ab 3782b8b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:42 2015 +0100
+
+    Merge pull request #3937 from dachary/wip-10762-firefly
+    
+    mon: ignore osd failures from before up_from
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 32de8ab146f242f7b73aca211f059bc39d38f85c
+Merge: 24a8c10 6fd3dfa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:22 2015 +0100
+
+    Merge pull request #3933 from dachary/wip-10617-firefly
+    
+    osd: do not ignore deleted pgs on startup
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 24a8c10a5a215c60b525f4170e45565390e02231
+Merge: f4e76c3 368a5a8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:01 2015 +0100
+
+    Merge pull request #3932 from dachary/wip-10546-firefly
+    
+    mon: Monitor: fix timecheck rounds period
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f4e76c3db90bf383c02cdb8cca19a37cd187095b
+Merge: fa518ff 66b13f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:37:36 2015 +0100
+
+    Merge pull request #3931 from dachary/wip-10512-firefly
+    
+    osd: requeue blocked op before flush it was blocked on
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fa518ffc5961785f01f0f91980c38a7e02686901
+Merge: ac9980e a22aa8f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:37:05 2015 +0100
+
+    Merge pull request #3930 from dachary/wip-10497-firefly
+    
+    librados: Translate operation flags from C APIs
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ac9980e074adf587b8b16af0bd08b3fc3233804d
+Merge: ccebb5f d3de8a5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:31:40 2015 +0100
+
+    Merge pull request #3955 from dachary/wip-10059-firefly
+    
+    PG: always clear_primary_state
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ccebb5f90bea0fd3bac1bd7ab93e5e07700f1301
+Merge: a30379d f4bab86
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:31:16 2015 +0100
+
+    Merge pull request #3954 from dachary/wip-10014-firefly
+    
+    ObjectStore: Don't use largest_data_off to calc data_align.
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a30379d9af89cffa4b3083ae14a94e3758c2b01c
+Merge: e575ca8 eb03e79
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:25:59 2015 +0100
+
+    Merge pull request #3947 from dachary/wip-9555-firefly
+    
+    osd: check that source OSD is valid for MOSDRepScrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e575ca84419f8be5c7fae7e2ffac549956a74f82
+Merge: cd675bb c60da2f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:25:30 2015 +0100
+
+    Merge pull request #3964 from dachary/wip-10157-firefly
+    
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit cd675bba5fdfb85689880ca7ecbd284181984a63
+Merge: 2b8e476 1a0f770
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:24:55 2015 +0100
+
+    Merge pull request #3960 from dachary/wip-6003-firefly
+    
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 2b8e4762edb7c35950f7e0d69bba2b5e1d83133a
+Merge: d434ead a746f7e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:24:03 2015 +0100
+
+    Merge pull request #3950 from dachary/wip-9985-firefly
+    
+    Get the currently atime of the object in cache pool for eviction
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Xinze Chi <xmdxcxz@gmail.com>
+
+commit d434eadadd1df6e4df18a6914015736bc09722d1
+Merge: e79e5f8 5f1245e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:20:12 2015 +0100
+
+    Merge pull request #3935 from dachary/wip-10723-firefly
+    
+    Fix memory leak in python rados bindings
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e79e5f8fbbfba6984cf21784e20d1c46cb60a397
+Merge: a9a36de 91b2aca
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 15:02:37 2015 +0100
+
+    Merge pull request #3866 from ceph/wip-cot-firefly
+    
+    Backport ceph-objectstore-tool changes to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a9a36deb89ac5719787e905469fe0b1bde5d58ca
+Merge: 83c571e 7e85722
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 15:00:56 2015 +0100
+
+    Merge pull request #3996 from dzafman/wip-10676
+    
+    Fix ceph command manpage to match ceph -h
+    
+    Reviewed-by: Xinxin Shu <xinxin.shu5040@gmail.com>
+
+commit 83c571e3067b4cad3e4567522c797d09a82d87db
+Merge: baa74b8 d5c3a14
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 14:55:37 2015 +0100
+
+    Merge pull request #3927 from dachary/wip-10351-firefly
+    
+    mount.ceph: avoid spurious error message
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 7e85722fd4c89715fc2ed79697c82d65d7ebf287
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 16:50:13 2015 -0700
+
+    doc: Minor fixes to ceph command manpage
+    
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 9ac488c1eb0e30511079ba05aaf11c79615b3940
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 12 11:39:52 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (firefly)
+    
+    Improve synopsis section
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 5f1245e131e33a98572408c8223deed2c7cf7b75
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 9 20:50:23 2015 -0800
+
+    rados.py: keep reference to python callbacks
+    
+    If we don't keep a reference to these, the librados aio calls will
+    segfault since the python-level callbacks will have been garbage
+    collected. Passing them to aio_create_completion() does not take a
+    reference to them. Keep a reference in the python Completion object
+    associated with the request, since they need the same lifetime.
+    
+    This fixes a regression from 60b019f69aa0e39d276c669698c92fc890599f50.
+    
+    Fixes: #10775
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 36d37aadbbbece28d70e827511f1a473d851463d)
+
+commit cf366fc3b21ff6f98530dbadb75a430c25672d56
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:11:22 2014 +0530
+
+    doc: Changes format style in ceph to improve readability as html.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8b796173063ac9af8c21364521fc5ee23d901196)
+
+commit 07fc9f66a69aa31d2cf8bf7a277d3e14ad6209be
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Mar 12 11:01:46 2015 +0800
+
+    mds: fix assertion caused by system clock backwards
+    
+    Fixes: #11053
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 8a25a51e42cdaed2c66dc25a6c6d0245441123a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 07:03:30 2015 -0800
+
+    rgw: fail s3 POST auth if keystone not configured
+    
+    Fixes: #10698
+    This fixes issue introduced in 8b3dfc9472022ea45ad24e02e0aa21dfdad798f8,
+    where if user does not exist, we try keystone authentication. However,
+    if keystone is not configured we justt fall through without failing.
+    This would have failed later on due to bucket permissions, unless bucket
+    had a public write permissions.
+    
+    Backports: Firefly
+    Reported-by: Valery Tschopp <valery.tschopp@switch.ch>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit eb03e792040bd22c1ae8b7dd73d94fbfd6208eec
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 13 08:06:31 2014 -0800
+
+    osd: check that source OSD is valid for MOSDRepScrub
+    
+    Make sure the message we got from the peer OSD is valid.  Specifically,
+    this avoids a race like this:
+    
+    - A marks down B
+    - B sends MOSDRepScrub
+    - A accepts connection from B as new
+    - A replies to scrub
+    - B crashes because msgr seq 1 < expected seq 1000+
+    
+    See #8880 for the most recent fix for requests.
+    
+    Fixes: #9555
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 847e5e102522d651aa9687a54aaafcebf3afc596)
+    
+    Conflicts:
+        src/osd/OSD.cc: require functions first argument is now a reference
+
+commit ae18707b3caa115dc510aff38b77f8afe8555c61
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 13 07:56:39 2014 -0800
+
+    osd: pass Message* to most require_* helpers
+    
+    These do nothing but op->get_req(); pass the Message* explicitly so that
+    non-OpRequest callers can use them.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 707a111d53efb09b3471dd3788b86d2bfee4e96f)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	src/osd/OSD.h
+            significantly changes had to be made but in a simple way and
+    	stays within the scope of the original commit
+
+commit c60da2f3c34e7325c748d2d6e55140a0a30013fd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 15:15:08 2014 -0800
+
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Fixes: #10157
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1fe8b846641486cc294fe7e1d2450132c38d2dba)
+    
+    Conflicts:
+    	src/osd/PGLog.cc
+            in the context coll_t::META_COLL was replaced with META_COLL
+
+commit 1a0f770eea18af6b276a31157f201a93166eb038
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 6 09:52:29 2015 -0800
+
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Fixes: 6003
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bae1f3eaa09c4747b8bfc6fb5dc673aa6989b695)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+            because reinterpret_cast was added near two hunks after firefly
+
+commit a746f7e5985198024067cb6e123569c09169b356
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 19:33:59 2014 -0700
+
+    osd/ReplicatedPG: fix compile error
+    
+    From 1fef4c3d541cba360738437420ebfa2447d5802e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4a9ad7dc2da6f4fa6a64235776a3f1d2799aef60)
+
+commit 5404fbfdd9b18cdb3fe5bed67146c769ec3acfa0
+Author: Federico Simoncelli <fsimonce@redhat.com>
+Date:   Sat Nov 15 14:14:04 2014 +0000
+
+    common: do not unlock rwlock on destruction
+    
+    According to pthread_rwlock_unlock(3p):
+    
+     Results are undefined if the read-write lock rwlock is not held
+     by the calling thread.
+    
+    and:
+    
+     https://sourceware.org/bugzilla/show_bug.cgi?id=17561
+    
+     Calling pthread_rwlock_unlock on an rwlock which is not locked
+     is undefined.
+    
+    calling pthread_rwlock_unlock on RWLock destruction could cause
+    an unknown behavior for two reasons:
+    
+    - the lock is acquired by another thread (undefined)
+    - the lock is not acquired (undefined)
+    
+    Moreover since glibc-2.20 calling pthread_rwlock_unlock on a
+    rwlock that is not locked results in a SIGILL that kills the
+    application.
+    
+    This patch removes the pthread_rwlock_unlock call on destruction
+    and replaces it with an assertion to check that the RWLock is
+    not in use.
+    
+    Any code that relied on the implicit release is now going to
+    break the assertion, e.g.:
+    
+     {
+       RWLock l;
+       l.get(for_write);
+     } // implicit release, wrong.
+    
+    Signed-off-by: Federico Simoncelli <fsimonce@redhat.com>
+    (cherry picked from commit cf2104d4d991361c53f6e2fea93b69de10cd654b)
+
+commit a73a4cb3889a6da21c3cfa4ddfa16d1a7059d20c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Sat May 3 08:32:19 2014 -0700
+
+    common/RWLock: track read/write locks via atomics for assertion checks
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 92615ea95a31d9fd22c3d11c860e0f502dc52c26)
+
+commit b0d0d44018de6289b64b6334edd4959ca4a0cc48
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 11 13:05:54 2014 -0800
+
+    ReplicatedPG::scan_range: an object can disappear between the list and the attr get
+    
+    The first item in the range is often last_backfill, upon which writes
+    can be occuring.  It's trimmed off on the primary side anyway.
+    
+    Fixes: 10150
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit dce6f288ad541fe7f0ef8374301cd712dd3bfa39)
+
+commit 34d19e1501b242fd8fc7cc95656592b5982f29a6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 18:42:40 2015 -0800
+
+    rgw: flush xml header on get acl request
+    
+    Fixes: #10106
+    Backport: firefly, giant
+    
+    dump_start() updates the formatter with the appropriate prefix, however,
+    we never flushed the formatter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb45f861343162e018968b8c56693a8c6f5b2cab)
+
+commit 9074eb7c46589aa1e5d10a2b9a8534f22dff2154
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Nov 17 17:37:00 2014 +0530
+
+    rgw: check for timestamp for s3 keystone auth
+    
+    This commit ensures that we check for timestamp of s3 request is within
+    acceptable grace time of radosgw
+    Addresses some failures in #10062
+    Fixes: #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 4b35ae067fef9f97b886afe112d662c61c564365)
+
+commit d3de8a5722a68d69023cf60c9076c2fb19058bf9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 19 08:20:16 2014 -0800
+
+    PG: always clear_primary_state on new interval, but only clear pg temp if not primary
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f692bfe076b8ddb679c6d1a6ea78cc47f0876326)
+
+commit 8b07236cd799b4752cbe620b737343a02a703a17
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Nov 14 15:44:20 2014 -0800
+
+    PG: always clear_primary_state when leaving Primary
+    
+    Otherwise, entries from the log collection process might leak into the next
+    epoch, where we might end up choosing a different authoritative log.  In this
+    case, it resulted in us not rolling back to log entries on one of the replicas
+    prior to trying to recover from an affected object due to the peer_missing not
+    being cleared.
+    
+    Fixes: #10059
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c87bde64dfccb5d6ee2877cc74c66fc064b1bcd7)
+
+commit f4bab86fe3b218d66c14d06883c297836d9ca19d
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Oct 27 11:22:13 2014 +0800
+
+    ObjectStore: Don't use largest_data_off to calc data_align.
+    
+    If largest_data_off % CEPH_PAGE_SIZE != 0, the get_data_aligment return
+    a erro value. This make the FileJouranl::align_bl to memcopy much data.
+    
+    Tested-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit a1aa70f2f21339feabfe9c1b3c9c9f97fbd53c9d)
+
+commit 2c7eda690ae2f71c8e8e189b8f7330196c1f385f
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:42 2014 +0800
+
+    osdc/Objecter: Fix a bug of dead looping in Objecter::handle_osd_map
+    
+    If current map epoch is less than oldest epoch, current map epoch
+    should step up to oldest epoch.
+    
+    Fixes: #9986
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit e0166a23c2cf655bfb4cf873be021a14d9b9be27)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            the debug line that follows was re-indented
+
+commit 11f8017cbdf94d4a8083412b96c251ee39286541
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:05 2014 +0800
+
+    osdc/Objecter: e shouldn't be zero in Objecter::handle_osd_map
+    
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit 31c584c8ba022cd44fe2872d221f3026618cefab)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            the debug line that follows was re-indented
+
+commit 7831582026441abbd6066dd951db4b63ffb45402
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Oct 29 07:11:11 2014 +0000
+
+    Get the currently atime of the object in cache pool for eviction
+    
+    Because if there are mutiple atime in agent_state for the same object, we should use the recently one.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 1fef4c3d541cba360738437420ebfa2447d5802e)
+
+commit 620733386bd0694960cecac8f32bd1538382d5bb
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Oct 28 09:37:11 2014 +0800
+
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reported-by: Xinze Chi <xmdxcxz@gmail.com>
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 622c5ac41707069ef8db92cb67c9185acf125d40)
+
+commit 5865411360f722ec511f2df6656d4ba975bef8eb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 20 13:43:46 2015 -0800
+
+    DBObjectMap: lock header_lock on sync()
+    
+    Otherwise, we can race with another thread updating state.seq
+    resulting in the old, smaller value getting persisted.  If there
+    is a crash at that time, we will reuse a sequence number, resulting
+    in an inconsistent node tree and bug #9891.
+    
+    Fixes: 9891
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2b63dd25fc1c73fa42e52e9ea4ab5a45dd9422a0)
+    
+    Conflicts:
+    	src/os/DBObjectMap.cc
+            because we have state.v = 1; instead of state.v = 2;
+
+commit baa74b88a4f4b6073b99b7a9774692b37405b59e
+Merge: be6559a 420ea03
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:09:23 2015 +0100
+
+    Merge pull request #3828 from dachary/wip-10425-firefly
+    
+    librados: fix resources leakage in RadosClient::connect()
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit be6559a423badde3b573b0c9055056999baae104
+Merge: f91d355 6e58732
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:06:27 2015 +0100
+
+    Merge pull request #3826 from dachary/wip-10257-firefly
+    
+    mon: PGMonitor: several stats output error fixes
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit f91d355306620cc543113ed21fddf84f4c170d6e
+Merge: f5525a1 1e58bb4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:05:13 2015 +0100
+
+    Merge pull request #3824 from dachary/wip-10353-firefly
+    
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f5525a1f41c9154b48f3ad0ccd899b5203c897bd
+Merge: b7e3614 27dbbb3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:04:33 2015 +0100
+
+    Merge pull request #3823 from dachary/wip-10787-firefly
+    
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit b7e3614ff38379aeea66ba00c64bc88ffa080963
+Merge: b5a67f0 7ed92f7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:02:16 2015 +0100
+
+    Merge pull request #3915 from dachary/wip-10080-firefly
+    
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 8ef14fcca715a33be8c611a37628c90d7fafca43
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Feb 11 23:36:01 2015 +0000
+
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    entity_name_t is tightly coupled to the messenger, while EntityName is
+    tied to auth.  When expanding profiles we want to tie the profile
+    expansion to the entity that was authenticated.  Otherwise we may incur
+    in weird behavior such as having caps validation failing because a given
+    client messenger inst does not match the auth entity it used.
+    
+    e.g., running
+    
+    ceph --name osd.0 config-key exists foo daemon-private/osd.X/foo
+    
+    has entity_name_t 'client.12345' and EntityName 'osd.0'.  Using
+    entity_name_t during profile expansion would not allow the client access
+    to daemon-private/osd.X/foo (client.12345 != osd.X).
+    
+    Fixes: #10844
+    Backport: firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 87544f68b88fb3dd17c519de3119a9ad9ab21dfb)
+
+commit 370f37f452aff3b48f9ae8a33b7ef26b572b41c8
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Nov 14 21:03:54 2014 +0000
+
+    mon: Monitor: stash auth entity name in session
+    
+    Backport: giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit ca8e1efc0be9bffcfbdce5593526d257aa498062)
+
+commit 0f31388eb2bb68c09ab270f871b421a9c368af97
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 17:51:08 2014 -0700
+
+    messages/MWatchNotify: include an error code in the message
+    
+    Document the fields, while we are here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7c7bf5fee7be397ef141b947f532a2a0b3567b42)
+
+commit f856739824bc271405a6fa35bdefc2bdc42c2f02
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 14:27:39 2014 -0800
+
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Fixes: #8011
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 9b26de3f3653d38dcdfc5b97874089f19d2a59d7)
+
+commit ca96b59db529ffbba0c834795800b6e90a7e4fce
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:11:38 2015 -0800
+
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Fixes: 10817
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5aa6f910843e98a05bfcabe6f29d612cf335edbf)
+
+commit 8ec8f1175a129624dffb511782664d74966d3c42
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 10:37:15 2015 -0800
+
+    ReplicatedPG::on_change: clean up callbacks_for_degraded_object
+    
+    Backport: dumpling, firefly, giant
+    Fixes: 8753
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a5ecaa189d47a69466a6cd55fa4180e5c3092dc2)
+
+commit 963439f1038000c3f28c728350a2e9d351341e0b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Feb 5 09:33:26 2015 -0800
+
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Fixes: #10770
+    Backport: firefly, giant
+    
+    Reported-by: baijiaruo <baijiaruo@126.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dfee96e3aebcaeef18c721ab73f0460eba69f1c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 3782b8b3652c0eb3fb8f65193ecfe09e92925170
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 5 03:07:50 2015 -0800
+
+    mon: ignore osd failures from before up_from
+    
+    If the failure was generated for an instance of the OSD prior to when
+    it came up, ignore it.
+    
+    This probably causes a fair bit of unnecessary flapping in the wild...
+    
+    Backport: giant, firefly
+    Fixes: #10762
+    Reported-by: Dan van der Ster <dan@vanderster.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 400ac237d35d0d1d53f240fea87e8483c0e2a7f5)
+
+commit ca42905a6fca8b2b404500a6f74951ae20c8a488
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Wed Nov 26 22:21:32 2014 +0530
+
+    doc: Adds the updated man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 016080d2c39919d73956e5e138ba5f079327aa44)
+
+commit 6602ab4581f27e066484b4c419d8206fcc20e045
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Wed Nov 26 22:19:01 2014 +0530
+
+    doc: Updates the man page for ceph-disk utility with some changes.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8a48847cd46d4807ca1dbd13d3a561a693cdc877)
+
+commit 459807021dd3bf86ce31d30a3b576e783da67e3f
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 21:23:08 2014 +0530
+
+    doc: Adds man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 9a118d56d4a5b0a0456e9f092f5ae9293f7bf3f3)
+
+commit 430d821b7592148ad9b2656bb3031a0484763f33
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 21:16:16 2014 +0530
+
+    doc: Removes ceph-deploy usage instances from ceph-disk man page.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 242dd1c0bbb728475a94f47740790b8a196d9804)
+
+commit 21c3256c851065521e34a179dc05d48fcc0a6e0f
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 02:06:39 2014 +0530
+
+    doc: Updates man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 7dcc85042b0c0a26e495f7574ce144d1083d15f8)
+
+commit 1a6490e38d7f5fd3ff640a810c3b911699cd4884
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Mon Nov 24 22:05:11 2014 +0530
+
+    doc: Adds man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit a450cab2b8148cb8a9b043d629feccf89e5aabac)
+
+commit 3bab47054dc77b9a00d3f47fa73f458ede7d4ab4
+Author: Billy Olsen <billy.olsen@gmail.com>
+Date:   Mon Feb 2 16:24:59 2015 -0700
+
+    Fix memory leak in python rados bindings
+    
+    A circular reference was inadvertently created when using the
+    CFUNCTYPE binding for callbacks for the asynchronous i/o callbacks.
+    This commit refactors the usage of the callbacks such that the
+    Ioctx object does not have a class reference to the callbacks.
+    
+    Fixes: #10723
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Billy Olsen <billy.olsen@gmail.com>
+    Reviewed-by: Dan Mick <dmick@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 60b019f69aa0e39d276c669698c92fc890599f50)
+
+commit d0fd417e872a73033903fb36144fe7a39e90fc9a
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Tue Feb 3 17:54:38 2015 +0200
+
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Fixes: #10475
+    Backport: hammer, firefly
+    Reported-by: Josh Durgin <jdurgin@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit f67bfa24fd6f69c2fcc0987eba8b6b426dd78320)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+            trivial merge: prototype of an unrelated function changed
+            s/is_object_op/!(s->object == NULL)/
+
+commit 651dc556047aa4ee9e95fe9fc7bcd11488973872
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Mon Feb 2 11:27:00 2015 +0200
+
+    rgw: Swift API. Dump container's custom metadata.
+    
+    Fixes: #10665
+    Backport: hammer, firefly
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit 2f8d31e9b1c4b09506bd1b0dad635c6e820783b2)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+            The first hunk conflicts because X-Storage-Policy was added
+    	after firefly. The second hunk conflicts because the type of the
+    	s->object data member changed after firefly but it is irrelevant
+    	because the patch does not need to access s->object anymore.
+
+commit 6fd3dfa7224f0af3101fd1614506e8cb2fc7f6a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 23 10:47:44 2015 -0800
+
+    osd: do not ignore deleted pgs on startup
+    
+    These need to get instantiated so that we can complete the removal process.
+    
+    Fixes: #10617
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 879fd0c192f5d3c6afd36c2df359806ea95827b8)
+
+commit 368a5a8a8739e98ffdb8ac1210d111092e31be9e
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 30 11:37:28 2015 +0000
+
+    mon: Monitor: fix timecheck rounds period
+    
+    Fixes: #10546
+    Backports: dumpling?,firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2e749599ac6e1060cf553b521761a93fafbf65bb)
+
+commit 66b13f2beff702c3b0bcb0aaa8da4e12d594eddf
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 11 17:28:04 2015 -0800
+
+    osd: requeue blocked op before flush it was blocked on
+    
+    If we have request A (say, cache-flush) that blocks things, and then
+    request B that gets blocked on it, and we have an interval change, then we
+    need to requeue B first, then A, so that the resulting queue will keep
+    A before B and preserve the order.
+    
+    This was observed on this firefly run:
+    
+      ubuntu@teuthology:/a/sage-2015-01-09_21:43:43-rados-firefly-distro-basic-multi/694675
+    
+    Backport: giant, firefly
+    Fixes: #10512
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11bdfb4131ecac16d4a364d651c6cf5d1d28c702)
+
+commit a22aa8f3228e0baf2ba08d9e79ee4924cd341a07
+Author: Matt Richards <mattjrichards@gmail.com>
+Date:   Thu Jan 8 13:16:17 2015 -0800
+
+    librados: Translate operation flags from C APIs
+    
+    The operation flags in the public C API are a distinct enum
+    and need to be translated to Ceph OSD flags, like as happens in
+    the C++ API. It seems like the C enum and the C++ enum consciously
+    use the same values, so I reused the C++ translation function.
+    
+    Signed-off-by: Matthew Richards <mattjrichards@gmail.com>
+    (cherry picked from commit 49d114f1fff90e5c0f206725a5eb82c0ba329376)
+    
+    Conflicts:
+    	src/librados/librados.cc
+            comes from lttng tracepoints introduced after firefly
+
+commit d5c3a14390f6bb2af45a1a4ad842777302dd0553
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jan 3 15:29:29 2015 +0800
+
+    mount.ceph: avoid spurious error message
+    
+    /etc/mtab in most modern distributions is a symbol link to
+    /proc/self/mounts.
+    
+    Fixes: #10351
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bdd0e3c4bda97fe18487a58dd173a7dff752e1a2)
+
+commit 7ed92f7dc5f0f327b77b6f3835e3f821fc810708
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 2 15:17:57 2014 -0800
+
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    In the past (e229f8451d37913225c49481b2ce2896ca6788a2) we decided to disable
+    reset of lossless Pipes, because lossless peers resetting caused trouble and
+    they can't forget about each other. But they actually can: if mark_down()
+    is called.
+    
+    I can't figure out how else we could forget about a remote endpoint, so I think
+    it's okay if we tell them we reset in order to clean up state. That's desirable
+    so that we don't get into strange situations with out-of-whack counters.
+    
+    Fixes: #10080
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8cd1fdd7a778eb84cb4d7161f73bc621cc394261)
+
+commit 91b2acaadee1b62c1fcac73147908ec4477840f3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 9 11:20:13 2014 -0700
+
+    osd: Get pgid ancestor from last_map when building past intervals
+    
+    Fixed OSD::build_past_intervals_parallel() and PG::generate_past_intervals()
+    
+    Fixes: #10430
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c5b66da7a9ba516340d06d9e806beb9d1040d0e)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit c09b6d9d64fdcdc3842c4f89acf10080125a4adc
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 23 12:04:26 2014 -0800
+
+    osd: Pass oldest_map when generating past intervals
+    
+    From load_pgs() the superblock hasn't been published yet
+    so we need to retrieve the value of oldest_map depending on the context.
+    
+    Fixes: #10427
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7fb721c1ceb39b38ca2e653299bcf51e109411d7)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 9235d781bf860c1ecef5af600431f1619b56fbc1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 17 16:59:09 2014 -0800
+
+    osd: build_push_op() handle short reads so recipient doesn't crash
+    
+    Fixes: #8121
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b9a04846d18e1c6621d7f2794ec1fae02875bed2)
+
+commit 7ce0cb8e33a281d5f675273d7bcbc570a32e5497
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Feb 5 16:33:08 2015 +0800
+
+    ceph_objectstore_tool: fix check_output on python2.6
+    
+    * backported the subprocess.check_output from python2.7
+    
+    Fixes: #10756
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 15350a088d84bc6fc664f0d3f5d09b35f58b2144)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit d5d34ddd1eac688d9422ca02c859d61be8e3e56e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 6 15:49:50 2015 -0800
+
+    ceph_objectstore_tool: For import get object_info_t available for each object
+    
+    Add object_info_t to object_begin so we have at object create time
+    This will be useful for importing from multiple erasure coded exports
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 05d916ed12f361da48ef73953bcc0cef465fcc2a)
+
+commit f70590b211ba28f350921b2f0c92712ed779858e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:16:03 2014 -0800
+
+    ceph_objectstore_tool: Handle import of pg which OSD has split
+    
+    Fail import if import data doesn't include OSDMap and can't find it locally
+    See if local map can be read for import's epoch
+    Jump to current epoch like a split would if local map not present
+    
+    Fixes: #9781
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit afda6e4f3b98cc1773fd014583dfb5e1f214a939)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit aedd324f8fbb031d5126ae158f03066c7342f4b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:38:33 2014 -0800
+
+    ceph_objectstore_tool: On import following a split skip objects no longer in pg
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 68b27e25a3729566c3a22c0b71f70f7f3aca29a3)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit c3de607c2ce16a26af7a78a4e557f321ffbcb44d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 5 15:12:21 2014 -0800
+
+    ceph_objectstore_tool: Verify that object pool and pgid pool match
+    
+    Also, earlier check for invalid --pgid with import op
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit da3be80497a1b1878ee5d2015f8840d202a83aa2)
+
+commit 5ec38e54f40431a5303cafe202c5097cc400fdb6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 19 11:47:36 2014 -0800
+
+    ceph_objectstore_tool: Check for splits and fail import if there were splits
+    
+    Add osdmap into metadata_section
+    On export put metadata_section before file data
+    
+    Fixes: #9780
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 19fdeea8b67091ed044ebce25799d3237b4d734a)
+
+commit aa472fa4df19d826a5af72e286365fa5ce4c71fb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 15 10:03:53 2014 -0800
+
+    ceph_objectstore_tool: Add special exit code for test infrastructure
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b1f12f09c0211b608178f5ca2e292ab1765ce620)
+
+commit 22b7c2faee8cfad1b40cef019984f4a2d112b268
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 19 11:41:39 2014 -0800
+
+    ceph_objectstore_tool: Check that pool exists before allowing import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f24f646d870703b7b79563cdbc20920248be6142)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit f65db436f6eb83d7c59fdddced4d35dd9eeeb078
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 15 15:21:11 2014 -0700
+
+    ceph_objectstore_tool: Check cluster_fsid before allowing an import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 196c8112dc9a6e8780b05d6c579cd7fbd5b07589)
+
+commit 360f68ec69b36b172d15c6206698340a5c00aafa
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 16 12:27:56 2014 -0700
+
+    ceph_objectstore_tool: Allow the metadata_section to be anywhere in the export
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 62dd912f1192b28700a15b02507a8c9efd710cb5)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit c3fcbe636ae1a936b9180628cff939b2b5dddf7c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:01:24 2014 -0800
+
+    ceph_objectstore_tool: import-rados shouldn't import internal namespace objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f727d2eaf50b0351feb9f12dcd65d50fb6eff7e9)
+
+commit 2d69076943b9724ce1d5c9f03f2f7594a77b92d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 14:58:54 2014 -0800
+
+    ceph_objectstore_tool: Get g_ceph_context available to import-rados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ddc4613ec71752e5dccbbfe6dc078b86f0f186a9)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit bbed3a728471292de625d922abeae8b39d290045
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 9 18:09:04 2014 -0800
+
+    ceph_objectstore_tool: Fix import-rados skipping of snapshots
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fe936026ed87c9f95f3b7ad235b24c22e8de5f55)
+
+commit d962c79dc1f0f2189e25743c6d253fac412c004d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 20 13:00:10 2014 -0800
+
+    ceph_objectstore_tool: read_fd() doesn't handle ^D from tty stdin, don't allow
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5cb692528e3ac0ebea3f1714b3ac43f69d176888)
+
+commit 8b7f4cb232c2f1bde6f50c1f092cb622fe2c41f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 19 13:47:32 2014 -0800
+
+    ceph-objectstore-tool: Remove --pretty-format and use new --format options
+    
+    Call new_formatter() with --format specified argument
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 22b71744bb0cb56434d5f6214ccea7d81f771860)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit ee183b8e56c6f8a88ac781cf1fedb6a7a93f1005
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 15 15:20:03 2014 -0700
+
+    ceph_objectstore_tool: Describe super_ver values
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0aeba0f216a54390288b5e3d6147deb31877f744)
+
+commit 5f82f824c7b1bb246bdb54b11a694133a49da70a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Jan 2 18:36:54 2015 +0100
+
+    ceph_objectstore_tool.cc: reduce scope of variable
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 78542f9a901c05e627b53b5306ea604be3bc82e8)
+
+commit 2f97c51b5ee5de7c1657bc8fee6ccaa474f6f478
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Jan 2 15:48:08 2015 +0100
+
+    ceph_objectstore_tool.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 252fc03ba21c7b09922598a8d40997fc639bb994)
+
+commit 2a22bfedc4fa9f461dc8cfd42c659b9c55ddafca
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 25 16:56:19 2014 -0800
+
+    ceph_objectstore_tool: Prevent tool from transitioning struct_v on rm-past-intervals
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f056bdf93980a0a8e6157dbb124a79389a8f1a3c)
+
+commit 53aa04f95c43795da81a7d9f3117d7e5816aedcb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 18:53:08 2014 -0800
+
+    ceph_objectstore_tool: Accept json object with --pgid instead of array
+    
+    It isn't anticipated that anyone would use this but keeps backward compatible
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 59b423e2e8846b098326fdec440de46b8e3d2769)
+
+commit 60e9a8120b292628ee4e5ef33fe933222609b861
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 18:27:50 2014 -0800
+
+    ceph_objectstore_tool: Improve object spec parsing error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de6384fda183801c16af1b61ed36eaed289bb4f6)
+
+commit 4f95409c0dadeed18334c00630ddc6d7c99d2819
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 17:48:28 2014 -0800
+
+    ceph_objectstore_tool: Fix errors messages in newer code
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eae7c02fded460f6c8aaf18aa83d2730b89e0eb1)
+
+commit 1703867735c8f8ab1c83aa526c84b278436f38d5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 16:00:40 2014 -0800
+
+    ceph_objectstore_tool: Remove extraneous endl on error throw messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 32c832f0c62259a492d1c934c56ac165496763a0)
+
+commit f2d2110a8cb1a1b1216c1083b54ea6212138dc93
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 14:01:39 2014 -0800
+
+    ceph_objectstore_tool: Add --format and --pretty-format support
+    
+    --pretty-format defaults true
+    Add --format so xml output can be requested
+    --op list defaults to single line of json per object
+    To override this more human readable output use --pretty-format=false
+    Add testing of --op list special handling
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cca85a534fe488ae314400e8faad55a758609467)
+
+commit a5544c12c3e982edf66f55a8edfb7fc69300520b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 17:53:11 2014 -0800
+
+    ceph_objectstore_tool: Strip _* (always _head) from pgid in list entry output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5eacd3c5f39766cb8be6b3251d139d16431cf6b6)
+
+commit 4a0a42f3e6b9a1b7ed4df3d4b6c417acfc00050a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:39:04 2014 -0800
+
+    ceph_objectstore_tool: BUG: --op list wasn't including snapshots
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b617ee2d45886ec6b3dc0db0edbf814ea5748083)
+
+commit 06579b9f269dc9864e920368c5bea9bcb9dd8be9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:38:22 2014 -0800
+
+    ceph_objectstore_tool: For terminal output of list one object per line
+    
+    Instead of a parsable array make it easier to cut and paste listed objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5a66db9418aeed31fec98999c5053dab357d9c1e)
+
+commit 22c6bf410de1b0c81f131aa4d5682f5162dd1a56
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:35:09 2014 -0800
+
+    ceph_objectstore_tool: In error case umount objectstore
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bc6ea9cb8a2b86c73f7f15bc46145177ccf91f4b)
+
+commit d802ab69a4a0f37ed9328ec385746f59643e1420
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 27 01:24:03 2014 +0100
+
+    objectstore_tool: test --op list variants
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ddba2676c4c48c2a556f5d4ffd817bfe134a9153)
+
+commit f30e053fe7b3e5efc679b20cf1b3e2f7f8ed7e54
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 27 00:11:45 2014 +0100
+
+    objectstore_tool: parse new object description format
+    
+    The object format changed from
+    
+        {json object}
+    
+    to
+    
+        [pgid,{json object}]
+    
+    The parser is updated accordingly. If the --pgid is present, check that
+    it equals the pgid from the object description.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit df9d5c5cfd8b0ff793647a592c7661965cef5c92)
+
+commit fce87c9d7dbb51989826d068d6b6657a2f3d129d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 23:35:21 2014 +0100
+
+    objectstore_tool: filter --op list and explore all PGs
+    
+    The positional object name is used to filter the output of --op list and
+    only show the objects with a matching name. If both the object name and
+    the pgid are omitted, all objects from all PGs are displayed.
+    
+    The output format is changed from
+    
+        {"oid":"GROUP","key":"","snapid":-2,
+         "hash":2659194943,"max":0,"pool":0,"namespace":""}
+    
+    to
+    
+        [["0.7_head",{"oid":"GROUP","key":"","snapid":-2,
+                      "hash":2659194943,"max":0,"pool":0,
+                      "namespace":""}]]
+    
+    where the first member is the pgid where the object is found.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit c69aaceac7f370e5369d511bf17898adc338ae43)
+
+commit 2f874fd3715d216a68658e94a5c741e665c76632
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 23:34:22 2014 +0100
+
+    objectstore_tool: lookup objects by name
+    
+    If the object is not a parsable JSON string, assume an object name and
+    look it up in all the PGs. If multiple objects have the same name, only
+    apply the command to one of them. It is primarily useful in a test
+    environment where the names of the tests objects are known and only a
+    small number of objects exists. It replaces the following:
+    
+        path='--data-path dev/osd0 --journal-path dev/osd0.journal'
+        for pgid in $(./ceph_objectstore_tool $path --op list-pgs) ; do
+          object=$(./ceph_objectstore_tool $path --pgid $pgid --op list |
+                   grep '"oid":"NAME"')
+          test -n "$object" && break
+        done
+        ./ceph_objectstore_tool $path --pgid $pgid "$object" remove
+    
+    with:
+    
+        ./ceph_objectstore_tool $path NAME remove
+    
+    http://tracker.ceph.com/issues/10192 Fixes: #10192
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7c1165f96391821c00cca1ac04b3433dbec6bb6e)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit eb48aba2119959c53ea8a103b53f0c2e07c52acb
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 17:30:30 2014 +0100
+
+    objectstore_tool: refactor list-lost and fix-lost
+    
+    Abstract out the PG exploration loops and encapsulate the list-lost and
+    fix-lost semantic in a callable object.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit d9e747b1bdb53d1fe543ef311e3db35fb78d8051)
+
+commit eab3226edc8ecc6d2d93f463e019ef4eacb9f468
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 17:26:54 2014 +0100
+
+    objectstore_tool: update usage strings
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a90233c8b79ae6c035b5169c7f8809f853631689)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 5bbe41cb1e7faf40a9c53a9bdc01db8c35670a36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 13:00:15 2014 -0800
+
+    ceph_objectstore_tool: Add feature called set-allow-sharded-objects
+    
+    Uses --op set-allow-sharded-objects option
+    This operation will be rejected if on the target OSD's osdmap there is
+        at least one OSD which does not support ERASURE CODES.
+    Prompt the user that they could import if sharded state allowed
+    Prompt the user to use new feature if sharded state found inconsistent
+    
+    Fixes: #10077
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f3dab446fc8e58b3b3d9334b8c38722e73881b9e)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 4e30d8c53c0a4952f81cc3b5680c7e92fffb1355
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 11:59:18 2014 -0800
+
+    ceph_objectstore_tool: Add utility routine get_osdmap()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b3021b0d3362000b5938d406ed2e032a8eb38405)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit f997b126afaeadebe6b6d76954fbe2008bd2a7d8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 12 21:14:11 2014 -0800
+
+    ceph_objectstore_tool: Clear ...INCOMPAT_SHARDS from feature if exporting replicated pg
+    
+    Don't require importing OSD to have shards feature for replicated pg
+    
+    http://tracker.ceph.com/issues/10077 Fixes: #10077
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 86baf2d38170ef19de2dd5e9ce3f280237d8474d)
+
+commit 3f862da57945f821ed459f5a77f8032331c9cb20
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 00:10:41 2014 -0800
+
+    tests: ceph_objectstore_tool.py test all variants of export/import
+    
+    Handle change of error message text
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9e53c3554c71121f2e2dd0234b5232da37ad5a1b)
+
+commit 2a58bdee19eb8f02a2800d728d402e76566d7f58
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:23:40 2014 -0800
+
+    ceph_objectstore_tool: Make --file option consistent by treating "-" as stdout/stdin
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8c87f3284f87d1121218cb7f41edc81b74c9df29)
+
+commit 70329be7be5d80e52d5424958cecd7f4d939add6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:43:10 2014 -0800
+
+    tests: ceph_objectstore_tool.py fix list-attr for erasure code
+    
+    Adding testing of xattr for erasure coded shards
+    Fix error message when finding an unexpected xattr key
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cbecab477a70782f2f69258f035e78fb5c829810)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit bc921c7ecbc061ccef4847d967986c9fa902111f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:46:15 2014 -0800
+
+    tests: ceph_objectstore_tool.py check for malformed JSON for erasure code objs
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 40717aa4c399e87d2c3e32038f78788eb213f87d)
+
+commit 99ffd137f17c438e3ee9dbbc0ab73cdcd3a45a5c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:44:54 2014 -0800
+
+    tests: ceph_objectstore_tool.py fix off by 1 ATTR_OBJS handling
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eaf1d1e35243566c46b478788e79e0ebf7583015)
+
+commit 951c951a0f164271c5e9b96ecbd510cf5c6663b6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 11:00:17 2014 +0100
+
+    tests: ceph_objectstore_tool.py skip if /dev/tty fails
+    
+    Some environments do not have a /dev/tty. When opening /dev/tty fails,
+    skip the test instead of returning an error.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 4c94f1778fdf483e9e0b62f89f7e46e78aeeebf3)
+
+commit 8dc263555d7aca2befa912c78c585a43c8e7592c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 19:15:50 2014 +0100
+
+    tests: ceph_objectstore_tool.py encapsulate init-ceph stop
+    
+    Call init-ceph in kill_daemons and add a call to kill_daemon when main
+    returns on error so that it never leaves daemons hanging.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit fc435ff3623d196dd7cc375302acd8dfc9eb59fd)
+
+commit b78d802f3e082b0762203ae37b3c3a44b7608907
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 19:14:49 2014 +0100
+
+    tests: ceph_objectstore_tool.py main returns
+    
+    Instead of calling sys.exit() the main function returns the desired exit
+    code.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0f3d7b1315f2b5595047d8bd13949ed0d9194bfa)
+
+commit 04e480b09b59f5318e9b206e6c3e529d8bb22328
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:32:14 2014 +0100
+
+    tests: ceph_objectstore_tool.py replace stop.sh with init-ceph
+    
+    The stop.sh will stop all ceph-* processes. Use the init-ceph script
+    instead to selectively kill the daemons run by the vstart.sh cluster
+    used for ceph_objectstore_tool.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e8f34bd62bf282144b8851fb9764cf4429a49c25)
+
+commit 51855584eb970b28dfa266ee3653963fb77a2b1e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:30:29 2014 +0100
+
+    tests: ceph_objectstore_tool.py use a dedicated directory
+    
+    Set CEPH_DIR to a directory that is specific to ceph_objectstore_tool so
+    that it can run in parallel with other vstart.sh clusters.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e303d1afde58e68c1f587962010da9e1f1278bc3)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit 454ec85ae449cb20c2ddecade421262d9d9de615
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:27:01 2014 +0100
+
+    tests: ceph_objectstore_tool.py run faster by default
+    
+    By default use only a small number of objects to speed up the tests. If
+    the argument "big" is given, use a large number of objects as it may
+    help find some problems.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 235257c257aea98b770d9637957818c8aeb6c745)
+
+commit 427fe9177f90db091685b937839dcc4dfe4c8a01
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:21:48 2014 +0100
+
+    tests: ceph_objectstore_tool.py run mon and osd on specific port
+    
+    By default vstart.sh runs MDS but they are not needed for the tests,
+    only run mon and osd instead. Instead of using the default vstart.sh
+    port which may conflict with a already running vstart.sh, set the
+    CEPH_PORT=7400 which is not used by any other test run with make check.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f04d4e7539bc8c1b6cf94db815f9dcdecc52faa2)
+
+commit 28ed34a7dc32033be61cd61178eb59471de45aac
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:16:41 2014 +0100
+
+    tests: ceph_objectstore_tool.py can use a WARN cluster
+    
+    The tests do not need HEALTH_OK exclusively, a HEALTH_WARN cluster can
+    also run them successfully.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 783378c019aaac36d542e1b12c0d64196ea21012)
+
+commit 8548d7c65e8743d4e743c77981acd702efab3fb2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:12:35 2014 +0100
+
+    tests: ceph_objectstore_tool.py use env python
+    
+    Using #/usr/bin/env python instead of a hard coded path is more flexible
+    and can also be used to run from virtualenv.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 74506d2506d03d05935cbe342fef9dc1d9022a13)
+
+commit 5b4c2ee0902d94e6fa5bee67188fddcf0e0c38a2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 12 15:22:04 2014 -0800
+
+    ceph_objectstore_tool: Fixes to make import work again
+    
+    The is_pg() call is now true even for pgs pending removal, fix broken
+        finish_remove_pgs() by removing is_pg() check.
+    Need to add create_collection() to the initial transaction on import
+    
+    Fixes: #10090
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5ce09198bf475e5c3a2df26232fa04ba9912b103)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit e0dab883d29bd3a54b5707f8d3ea830f8a1ce516
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 6 18:26:44 2014 -0700
+
+    ceph_objectstore_tool: Accept CEPH_ARGS environment arguments
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 10fe7cfe561f91717f2ac2e13aeecc06a903704e)
+
+commit 5f788eaa61054c6cb27960b5544b321078706343
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 3 15:12:28 2014 -0700
+
+    ceph_objectstore_tool: MemStore needs a CephContext
+    
+    Pass g_ceph_context to ObjectStore::create() needed by MemStore
+    
+    Fixes: #9661
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0b155d00c542f0d8b8b5b0324dac4b3cf7ff37b5)
+
+commit 7470c7ca02ef6313b925be3ce4b27437a0c2e1e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 3 10:41:28 2015 -0800
+
+    ceph_objectstore_tool: Rename generated binary to ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4f72ba545e204a24a55adead43c61cb1d4394381)
+    
+    Conflicts:
+    	debian/ceph-test.install
+    	src/.gitignore
+    	src/test/ceph-helpers.sh (doesn't exist in firefly)
+    	src/test/ceph_objectstore_tool.py
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 4b51645fdfe8a761f7ebc0faee1a87187d498fee
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 13:35:43 2014 -0800
+
+    vstart.sh: warn less
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a69b8450f642af91a352d0de4378d93828291933)
+
+commit 3075919c0dcee1f33a0beb299e98d8a88836c5f8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:02:50 2014 -0800
+
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Fixes: #10128
+    Caused by a2bd2aa7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0d5262ac2f69ed3996af76a72894b1722a27b37d)
+    (cherry picked from commit 6cb9a2499cac2645e2cc6903ab29dfd95aac26c7)
+
+commit ca9df2803f3200431f5f7ea99a713005f15b7f5a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Sep 2 14:56:10 2014 +0200
+
+    ceph_objectstore_tool.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a5468abe4459159e8a9f7a4f21d082bb414e1cdd)
+
+commit ac396f5df3c51d09b9dbf7e6be7bed1d755d2317
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Fri Aug 29 16:39:40 2014 +0800
+
+    Test: fixing a compile warning in ceph_objectstore_tool.cc
+    
+    For the compiler's sake:
+    tools/ceph_objectstore_tool.cc:2547:15: warning: ‘r’ may be used
+    uninitialized in this function [-Wmaybe-uninitialized]
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit c3e1466b46076f133b62f98e2c0b712bdde0e119)
+
+commit b863970110e7f8e835e77864ea59b81f0b026158
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 20 01:33:45 2014 -0700
+
+    ceph_objectstore_tool: Bug fixes and test improvements
+    
+    ceph_objectgstore_tool:
+    Fix bugs in the way collection_list_partial() was being called
+    which caused objects to be seen over and over again.
+    
+    Unit test:
+    Fix get_objs() to walk pg tree for pg with sub-directories
+    Create more objects to test object listing code
+    Limit number of larger objects
+    Limit number of objects which get attributes and omaps
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a03f719eb3a46f410550afce313e6720e0c27946)
+
+commit 4f83005bb8a615df370de7b6dfe5d926c7cfef7f
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Aug 5 18:26:11 2014 -0700
+
+    ceph_objectstore_tool, test: Implement import-rados feature and unit test code
+    
+    Fixes: #8276
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 23ec93a86140c4b271b45d87c62682288079cbba)
+
+commit df334617705fe862b820ef5de674ae54efad0cea
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Aug 6 19:53:43 2014 -0700
+
+    test: ceph_objectstore_tool unit test improvements
+    
+    Add namespaces to testing
+    Increase filesize so export will have multiple chunks
+    Put json for each object into the db dict
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+
+commit ecd25cf6ce1a1a34e536c7fd313225b3bdedd2e0
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 13:31:48 2014 -0700
+
+    ceph_objectstore_tool: Add operation "rm-past-intervals" for testing purposes
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3694068b67fd625495c4511390bc5bcbfbbd28f5)
+
+commit 6b50b384efc1f0735f8635a59663b50e3155de1a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 11:46:08 2014 -0700
+
+    ceph_objectstore_tool: Add past_intervals to export/import code
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 9325ec15d4b89c5537cbcbda4c7594e25dc6e7df)
+
+commit 63529079b97c99cbaa863e1be865e2693e90c556
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 14:11:21 2014 -0700
+
+    ceph_objectstore_tool: Minor improvements
+    
+    Make all non-error non-debug output to stdout
+    Fix a message
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit a2bd2aa7babb4ad45ba21c70f9d179fda27742aa)
+
+commit 5e3f89ece7bdd09ed06ca4208cfa0a0b3104f109
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Aug 5 12:26:42 2014 -0700
+
+    ceph_objectstore_tool, test: Add list-pgs operations and unit test case
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f01e334c697057158354f0ce5ecff6d6ba8e2704)
+
+commit edc9f71efeebe5241004c669cc58089905907634
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jul 30 12:39:49 2014 -0700
+
+    Complete replacement of ceph_filestore_tool and ceph_filestore_dump
+    with unified ceph_objectstore_tool
+    
+    Move list-lost-objects and fix-lost-objects features from
+      ceph_filestore_tool to ceph_objectstore_tool as list-lost, fix-lost
+    Change --type to --op for info, log, export...operations
+    Add --type for the ObjectStore type (defaults to filestore)
+    Change --filestore-path to --data-path
+    Update installation, Makefile.am, and .gitignore
+    Fix and rename test case to match
+      Add some additional invalid option checks
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 83fbc91e5c4e52cc1513f34908f99d2ac3b930df)
+
+commit a42273ecd955470105cba1cc4ac7eb782ac46833
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jul 30 11:22:29 2014 -0700
+
+    Renames and removal towards a unified ceph_objectstore_tool
+    
+    Rename ceph_filestore_dump.cc and ceph_filestore_dump.py
+    Remove ceph_filestore_tool.cc
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 77864193a1162393ade783480aee39a232934377)
+
+commit 9ee2c27096784efceb02b06a0df4325979385f44
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 11:19:19 2014 -0700
+
+    ceph_filestore_dump: Add set-omaphdr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b4d95cc85af9af64d33d541cd69c5f28fd45423b)
+
+commit c7d261d40367ab01a577bf255da776a622f8984a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 10:44:37 2014 -0700
+
+    ceph_filestore_dump: Add get-omaphdr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 30c0f3114e665acdd99e64bf0d2a7399b33e8d61)
+
+commit 271e3ca19b9e878c6c3c26a9ec461faf06e7a19d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:55:47 2014 -0700
+
+    ceph_filestore_dump: Add rm-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 0fc6bd2777edf24a044f454beacf1647cc52f9fe)
+
+commit 20165d101a30c6beb591ca56b56bdf5505f70cf3
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:47:14 2014 -0700
+
+    ceph_filestore_dump: Add set-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50cd57e902fe508f98f63fea30626780b07561d9)
+
+commit 7547f3d17bc89437c529aa96413b0bebb808da5e
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:37:01 2014 -0700
+
+    ceph_filestore_dump: Add get-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b50c43ce5e52f5bbcb3684f6793d50f34ed741d1)
+
+commit f391feb1f50632adbe94c3e8cdc4b75091d6b8e1
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:33:24 2014 -0700
+
+    ceph_filestore_dump: Add rm-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 465d77733c7499fbd65bebe7141895714c625e0d)
+
+commit 7bd2dd3a1d022df6d3f886ad12a191d0cfcef1d6
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:17:27 2014 -0700
+
+    ceph_filestore_dump: Add set-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 95554e03dcb74b6d74b2f1b2891b3570abb187b8)
+
+commit 6724da821158ddee6ef6ee7b5bac9e97dcfc2292
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu May 15 15:50:48 2014 -0700
+
+    ceph_filestore_dump: Add get-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 55d43c0e20fc853daec134449b9954248fd7ef31)
+
+commit 55c21b898834d77234227d3fc14c8580ef698663
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:52:09 2014 -0700
+
+    ceph_filestore_dump: Add set-bytes object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3c24d1f46a624d0a053ad234997a1f8c8b036db5)
+
+commit 2f1926d2f57082666350d8223b09f61da5f95b6f
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:51:29 2014 -0700
+
+    ceph_filestore_dump: Add get-bytes object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 869dd92cc8ec29a3a684f88c335d359f225bba24)
+
+commit fcabb8133af3b90d5d9e976ce658ceccfc5b89c5
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:50:16 2014 -0700
+
+    ceph_filestore_dump: Add list-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 48890c7741d76cf92b5f589f49378ca57292e88b)
+
+commit 303e4cedd91ca3553e956eec495a05e3136b3c56
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 18:32:42 2014 -0700
+
+    ceph_filestore_dump: Add list-attrs object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 00c6b0673288ca76fe144575b7af76eaa36f5857)
+
+commit aff2c995f67bdde1a592a9b24e4e96e85735d500
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:39:17 2014 -0700
+
+    ceph_filestore_dump: Add --type list to output objects in a pg in json
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 844dabb7f311e68eba0293ae9ca4c68521745877)
+
+commit 7cda8b7a0b43d709b73d875088ecd169f47d59ab
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:44:31 2014 -0700
+
+    ceph_filestore_dump: Add remove object command and usage for new commands
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 605caec64b036f8ab5ae451d7e9e7515d414f28e)
+
+commit a4694643ae4503746d3fac8a0feac706ddc13a16
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri Jun 6 17:05:53 2014 -0700
+
+    ceph_filestore_dump: Add utility function get_fd_data()
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit d4a9dafe442f139562497d746f80ba49faa954e8)
+
+commit 62dc823fb3b969c0ad52594419e8a86a3ff1e6ef
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:16:52 2014 -0700
+
+    ceph_filestore_dump: Fail import/export with a tty for stdin/stdout
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 7520e504cf2cdd3de2f236acb2cbf8a5016e6698)
+
+commit 9816f872ad59bcaa1a125b297f3991b333aad39c
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 11:56:20 2014 -0700
+
+    ceph_filstore_dump: Save if stdout is a tty and add routine to clean binary strings
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3a574cc78b0e3ec6d8dd0c39ee20e7a54ad64056)
+
+commit d4aedeb833f23bf4ce6187cb82910ab2e71d48e5
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:30:11 2014 -0700
+
+    common: Add missing ghobject_t::decode() for json
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c05f895d15a1d0e78ff5e9ae1a83f0a5424103d0)
+    
+    Changes:
+    	Adjusted for older shard_t
+
+commit dadecb1e05e528093642ba356fa7a70a0b546727
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:37:17 2014 -0700
+
+    ceph_filestore_dump: Add --skip-journal-replay and --skip-mount-omap
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 2e9dcb256509e7c921556202052f0cc1dcd91398)
+
+commit c6369987b1e1e55a9d0ab0bc328f61f52fc608d0
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:41:15 2014 -0700
+
+    os: Add optional flags to generic ObjectStore creation (SKIP_JOURNAL_REPLAY
+    and SKIP_MOUNT_OMAP)
+    
+    Only FileStore cares about these flags, so passed on during create()
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3d9fde9d92322cd8ac3e3d8bcbf5b0a01ef0528b)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit 3381aebb113d14249f6998a86ebf6b4ec6adc42d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri May 16 18:20:11 2014 -0700
+
+    ceph_filestore_dump: Improve debug output by showing actual offset
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 44b261d5d1b36528bfbcb37dbd866b615e14be99)
+
+commit 1164b2e1610028a40cddbed09f9da5649bd2023a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 12:36:37 2014 -0700
+
+    ceph_filestore_dump: Use cerr now that we aren't closing stderr
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 087c0f9d31e0f3d5bae7eac6231978105a71677e)
+
+commit f224429aa4fcba897be5e438bbb49d1025e2c482
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 12:42:21 2014 -0700
+
+    common,ceph_filestore_dump: Add ability for utilities to suppress library dout output
+    
+    Suppress dout output with CODE_ENVIRONMENT_UTILITY_NODOUT
+    ceph_filestore_dump turns on dout output if --debug specified
+    When used it can still be enable with --log-to-stderr --err-to-stderr
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f7f9b251fc377651d8da4cbfd1942c3b86f3247e)
+
+commit 3f4cabdb84e58fcec0c3f508f980881c59fba948
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 13 18:27:30 2014 -0700
+
+    ceph_filestore_dump: Export omap in batches for large omap case
+    
+    New function get_omap_batch()
+    Create a TYPE_OMAP section for each batch
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 501dd3c05b8983159a289b021943cb828b908f53)
+
+commit 0b757af5be338b65fd9124ac5158bfe02ad5f899
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 12 15:50:34 2014 -0700
+
+    ceph_filestore_dump: Remove unused bufferlist databl
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 398b418e2b9f8260bcfacac8bcebea5beffcceca)
+
+commit 4a742fe29b6d959912a38d132344c695f89dd34f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed May 7 14:12:15 2014 +0200
+
+    ceph_filestore_dump.cc: cleanup includes
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8620609884243596d35b69c571d2da751e63cf2b)
+
+commit 420ea03aa3cd52bd035d31ba111c3d6d0745352d
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jan 29 18:19:16 2015 +0100
+
+    librados: rectify the guard in RadosClient::wait_for_osdmap().
+    
+    RadosClient::wait_for_osdmap() did not signalize lack of connection
+    via -ENOTCONN error code when the Objecter instance was allocated.
+    The proper way is to check the connection state explicitly.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 34473f78f101d87d2606e0e7112682a47ff24830)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+            the modified guard was not present: add the new guard instead
+            of modifying it
+
+commit 1b2667211f90a1b630d2ddffe99b0fb00bb3c07c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Jan 19 15:07:21 2015 +0100
+
+    librados: fix resources leakage in RadosClient::connect().
+    
+    If RadosClient::connect was called a second time (which could
+    happen as a part of recovery from failure), the instances
+    of Objecter and Messenger allocated by the first call were leaked.
+    
+    Additionally, the implementation of the method wrongly reported
+    memory allocation problems -- it throwed std::bad_alloc exception
+    instead of returning -ENOMEM error code.
+    
+    Fixes: #10425
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 624c056da093c8741242892413438a291c03c7d5)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+            resolve adding (std::nothrow) that failed because the
+            prototype of the constructor is not the same
+
+commit 4e32ff2b60549742d01b18429810c89f5f707548
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Dec 5 15:29:52 2014 -0800
+
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::operator==
+    
+    Fixes: 10259
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1ac17c0a662e6079c2c57edde2b4dc947f547f57)
+
+commit 6e58732f546ec6241b198d2473902d66327cdc36
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Mon Jan 19 18:49:15 2015 +0000
+
+    mon: PGMonitor: skip zeroed osd stats on get_rule_avail()
+    
+    Fixes: #10257
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit b311e7c36273efae39aa2602c1f8bd90d39e5975)
+    
+    Conflicts:
+    	src/mon/PGMonitor.cc
+            ceph::unordered_map changed the context, simple resolution
+
+commit bcc8cfb24a96a7874a24760771755088a231a8d7
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:13:05 2015 +0000
+
+    mon: PGMonitor: available size 0 if no osds on pool's ruleset
+    
+    get_rule_avail() may return < 0, which we were using blindly assuming it
+    would always return an unsigned value.  We would end up with weird
+    values if the ruleset had no osds.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 8be6a6ab2aa5a000a39c73a98b11a0ab32fffa1c)
+
+commit 894c8ad26fd2da203dcbf8eb0ad5e2af0223d5a9
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:12:42 2015 +0000
+
+    mon: PGMonitor: fix division by zero on stats dump
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 50547dc3c00b7556e26b9a44ec68640c5c3a2384)
+
+commit 1e58bb49b99118d064c1ca92e42cbfb2786fdaff
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 17 16:06:55 2014 +0100
+
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    It is common for people to try to map 9 OSDs out of a 9 OSDs total ceph
+    cluster. The default tries (50) will frequently lead to bad mappings for
+    this use case. Changing it to 100 makes no significant CPU performance
+    difference, as tested manually by running crushtool on one million
+    mappings.
+    
+    http://tracker.ceph.com/issues/10353 Fixes: #10353
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 2f87ac807f3cc7ac55d9677d2051645bf5396a62)
+    
+    Conflicts:
+    	src/erasure-code/lrc/ErasureCodeLrc.cc
+            safely ignored because the file does not exist
+
+commit 27dbbb3c312ea68a96b011ccb12394c75c0fb0f3
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 12 13:49:50 2015 -0800
+
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    If the epoch cache says the osd has epoch 100 and the osd is asking for
+    epoch 200+, do not send it 100+.
+    
+    Fixes: #10787
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a5759e9b97107488a8508f36adf9ca1aba3fae07)
diff --git a/doc/changelog/v0.80.11.txt b/doc/changelog/v0.80.11.txt
new file mode 100644
index 000000000..beb1ea96e
--- /dev/null
+++ b/doc/changelog/v0.80.11.txt
@@ -0,0 +1,3184 @@
+commit 8424145d49264624a3b0a204aedb127835161070 (tag: refs/tags/v0.80.11, refs/remotes/gh/firefly)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Tue Nov 17 18:57:48 2015 +0000
+
+    0.80.11
+
+commit 2a14d16712fbe2ee9d7d2aef389e18fcb2809a47
+Merge: c551622 a06179a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 2 14:34:28 2015 -0800
+
+    Merge pull request #6451 from jcsp/wip-firefly-urls
+    
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+
+commit a06179acb268b073fcf67fbbf19152c1a5a275af
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 24 18:10:46 2015 -0400
+
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4d65fbd6ad15320339751c604613ac63511e645)
+
+commit c551622ca21fe044bc1083614c45d888a2a34aeb
+Merge: 241edfa bff2f47
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 21:47:27 2015 +0200
+
+    Merge pull request #5823 from SUSE/wip-11700-firefly
+    
+    make the all osd/filestore thread pool suicide timeouts separately configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 241edfabf6a6c767afc1e283dd7387377ece36c7
+Merge: 07e90f5 f39c791
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 21:01:54 2015 +0200
+
+    Merge pull request #5815 from SUSE/wip-12838-firefly
+    
+    Mutex Assert from PipeConnection::try_get_pipe
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 07e90f54318c0a6202add4de16f965c6e331d513
+Merge: eaf7dc3 d0d6727
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 18:19:39 2015 +0200
+
+    Merge pull request #5529 from SUSE/wip-12586-firefly
+    
+    FileStore calls syncfs(2) even it is not supported
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eaf7dc343eb78f09d9a4b1815f6bd6ceb4217ad7
+Merge: a960327 98769f7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 22 07:44:12 2015 -0700
+
+    Merge pull request #5532 from SUSE/wip-12635-firefly
+    
+    swift smoke test fails on TestAccountUTF8
+
+commit a96032738ba1e1cc35f21607eba06e7e241abd97
+Merge: 15f3628 86f9e55
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 22 07:33:02 2015 -0700
+
+    Merge pull request #5997 from SUSE/wip-13178-firefly
+    
+    rgw: content length parsing calls strtol() instead of strtoll()
+
+commit 15f362808b7100a5bade24cc78e718b04d391b54
+Merge: e010899 ebba1d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 10:45:48 2015 +0200
+
+    Merge pull request #5526 from SUSE/wip-12584-firefly
+    
+    OSD crash creating/deleting pools
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e010899ac9cd4060c09f0f1c754f0840b5149f6b
+Merge: 24ffe4c 8c02376
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 10:43:29 2015 +0200
+
+    Merge pull request #5287 from SUSE/wip-12389-firefly
+    
+    PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 24ffe4c5156843ce69cae30087fb767063fd489a
+Merge: f48450b c155df2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 21 10:03:13 2015 +0200
+
+    Merge pull request #6328 from dachary/wip-13551-firefly
+    
+    qemu workunit refers to apt-mirror.front.sepia.ceph.com
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c155df217287b980e176c3c98ee49753d6378740
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Fri Jun 19 00:02:20 2015 +0800
+
+    qa: Use public qemu repo
+    
+    This would allow some external tests outside of sepia lab
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit 4731c1e35539c9506ff3fe3141553fad099d0eee)
+
+commit d9ae7d80850b9ea4a84549c3afb4603cf9585b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 16 08:29:41 2015 -0700
+
+    use git://git.ceph.com
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34c467ba06345eacd2fd0e79c162dfd0b22f0486)
+    
+    Conflicts:
+    	qa/run_xfstests-obsolete.sh : does not exist in firefly
+    	qa/workunits/suites/fsx.sh  : trivial resolution
+
+commit f48450bdc70eef7cef8f76bc30f678a6d8d52b78
+Merge: f275ad2 b894b36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 21 01:02:08 2015 +0200
+
+    Merge pull request #5822 from SUSE/wip-12837-firefly
+    
+    WBThrottle::clear_object: signal on cond when we reduce throttle values
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit f275ad21a48cc4f2812f3ab9e3e9b8b05469c3c7
+Merge: 143e267 81bb6dd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:48:39 2015 +0200
+
+    Merge pull request #6325 from dachary/wip-13041-firefly
+    
+    common/Thread:pthread_attr_destroy(thread_attr) when done with it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 143e267a30ef7aac04fdfa530babab529cf30c1a
+Merge: 725401e 6d396a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:46:28 2015 +0200
+
+    Merge pull request #6091 from SUSE/wip-13252-firefly
+    
+    mon: does not check for IO errors on every transaction
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 725401e05d0fbe41d93b200fe39d73d9dddae80c
+Merge: 8fd4dd7 c85d063
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:45:19 2015 +0200
+
+    Merge pull request #6087 from SUSE/wip-13244-firefly
+    
+    client nonce collision due to unshared pid namespaces
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8fd4dd7480148c664d2c99956a7c987b8707c3c3
+Merge: 77a1b5f 6d82eb1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:44:32 2015 +0200
+
+    Merge pull request #6010 from liewegas/wip-globalid-firefly
+    
+    increase globalid default for firefly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 77a1b5f1292d8410a1c29cebb0da449b09a230d3
+Merge: a3823fe 8c14cad
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:43:33 2015 +0200
+
+    Merge pull request #5991 from SUSE/wip-13043-firefly
+    
+    ThreadPool add/remove work queue methods not thread safe
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a3823fe76171479e4c8f071f6e8eee3f13a02441
+Merge: e771054 48c929e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:42:28 2015 +0200
+
+    Merge pull request #5988 from SUSE/wip-13038-firefly
+    
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e771054241fcba536a478f1d014feb20a434fa3f
+Merge: e43d049 cd1396c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:41:19 2015 +0200
+
+    Merge pull request #5820 from SUSE/wip-12845-firefly
+    
+    osd/PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e43d0492a415be5a0feb361efbfbdda6e00a6234
+Merge: 11ed9f9 2c2ffa1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:40:28 2015 +0200
+
+    Merge pull request #5814 from SUSE/wip-12840-firefly
+    
+    recursive lock of md_config_t (0)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11ed9f9b263a1d3a8df7996d5644f0f6de7a9b1a
+Merge: aead425 2c0d7fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:39:32 2015 +0200
+
+    Merge pull request #5813 from SUSE/wip-12822-firefly
+    
+    ceph-fuse crash in test_client_recovery
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit aead4255528e60a2ccc6f4d47a6cb482bfdccc82
+Merge: 1c65d09 d3c9469
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:37:58 2015 +0200
+
+    Merge pull request #5726 from ceph/wip-12752
+    
+    is_new_interval() fixes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c65d09ef8de2732cfa6f18d22de2c7b3fac0f0f
+Merge: 1f0f4bb 411769c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:37:01 2015 +0200
+
+    Merge pull request #5698 from tchaikov/wip-12638-firefly
+    
+    mon: add a cache layer over MonitorDBStore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1f0f4bb5347cb935095dd374d6dcdec44ff175e2
+Merge: 4a1e54f 278d732
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:35:56 2015 +0200
+
+    Merge pull request #5619 from liewegas/wip-journal-firefly
+    
+    os/FileJournal: Fix journal write fail, align for direct io
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 4a1e54fc88e43885c57049d1ad4c5641621b6c29
+Merge: f272435 de53add
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:31:52 2015 +0200
+
+    Merge pull request #5360 from ceph/wip-11786.firefly
+    
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f272435c1303327e7dae248179387d12e28ea6bc
+Merge: 21452ed e7d08b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:30:04 2015 +0200
+
+    Merge pull request #5236 from SUSE/wip-12312-firefly
+    
+    mon: adding exsting pool as tier with --force-nonempty clobbers removed_snaps
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 21452ed93b707984d5301a51040ba8c311678c9c
+Merge: 14c8528 971eb3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:28:31 2015 +0200
+
+    Merge pull request #5050 from SUSE/wip-12015-firefly
+    
+    LibRadosTwoPools[EC]PP.PromoteSnap failure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 14c85282399aca29a31566e0f068376b10cb95e6
+Merge: b6d5768 7a8b6fb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:09:38 2015 +0200
+
+    Merge pull request #6207 from SUSE/wip-13417-firefly
+    
+    Fix -Wno-format and -Werror=format-security options clash
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b6d57686e1f482bf510e6c6c0ae56cff575f3eca
+Merge: fca8a17 415335d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:08:01 2015 +0200
+
+    Merge pull request #6203 from SUSE/wip-13413-firefly
+    
+    fedora21 has junit, not junit4
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit fca8a17bbb91ce5b3cb0c6a282cded03fe37956f
+Merge: 6240b61 20ad17d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:06:14 2015 +0200
+
+    Merge pull request #5992 from SUSE/wip-13168-firefly
+    
+    upstart: configuration is too generous on restarts
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6240b61fc48e4f7b306ec18eca51f40081b5160d
+Merge: d2459d8 04b2a87
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:04:40 2015 +0200
+
+    Merge pull request #5541 from dachary/wip-12418-firefly
+    
+    ceph-disk: sometimes the journal symlink is not created
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d2459d8b9c189e557f81bb7d10dd17cc53cf9af1
+Merge: 3e0e3c5 644f213
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:36:01 2015 +0200
+
+    Merge pull request #5729 from ceph/wip-11455-firefly
+    
+    rgw: init some manifest fields when handling explicit objs
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3e0e3c518bf68e9d0ad7a79d804602634b9c6466
+Merge: e44a777 e7931a7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:35:22 2015 +0200
+
+    Merge pull request #5730 from ceph/wip-12537-firefly
+    
+    RGW returns requested bucket name raw in "Bucket" response header
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e44a7775fb34ccdf7ff36e5e56aba9a309521be1
+Merge: 028da25 3be204f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:34:01 2015 +0200
+
+    Merge pull request #5831 from gnuoy/wip-11140-firefly
+    
+    /etc/init.d/radosgw restart does not work correctly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 81bb6dd7de8a191c11c16b3cdc07355ef3434435
+Author: zqkkqz <zheng.qiankun@h3c.com>
+Date:   Fri Aug 7 10:49:45 2015 +0800
+
+    Common/Thread: pthread_attr_destroy(thread_attr) when done with it
+    When a thread attributes object is no longer required, it should be destroyed using the
+    pthread_attr_destroy() function. Destroying a thread attributes object has no effect on threads that were created using that object.
+    
+    Fixes: #12570
+    Signed-off-by: zqkqkz zheng.qiankun@h3c.com
+    (cherry picked from commit 9471bb838a420ef5f439191d87e5388fecccb9e6)
+
+commit 77d6426a8323db91a60f246f385826c35f31521c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Jul 17 12:43:52 2015 +0200
+
+    Thread.cc: remove malloc/free pair
+    
+    There's no need for mallocing pthread_attr_t in Thread::try_create(),
+    it can be located on stack as it is freed in same function. This reduces
+    pressure put on memory manager.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 614ac0f4e375f3dd008c8937d3f89f6005d6e0a0)
+
+commit 7a8b6fb4c7b80c33dd3565fc4140d45fe70f0438
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Aug 15 19:34:27 2014 +0200
+
+    Fix -Wno-format and -Werror=format-security options clash
+    
+    This causes build failure in latest fedora builds, ceph_test_librbd_fsx adds -Wno-format cflag but the default AM_CFLAGS already contain -Werror=format-security, in previous releases, this was tolerated but in the latest fedora rawhide it no longer is, ceph_test_librbd_fsx builds fine without -Wno-format on x86_64 so there is likely no need for the flag anymore
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7df67a544f6401fd31157b6e2a6ca590d87d7cbc)
+    
+    Conflicts:
+    	src/test/Makefile.am
+
+commit 415335df53bf0d6b42f745b038c1c0c9c4aaa810
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 3 16:14:23 2015 +0100
+
+    ceph.spec.in: junit always except for EPEL 6
+    
+    The package was renamed a long time ago (around the Fedora 15
+    timeframe). The "junit4" name is only relevant for EPEL 6. For EPEL 7
+    and Fedora 20, the "junit" package has "Provides: junit4". And most
+    recently, in the junit package that ships in Fedora 21 and 22, the
+    package maintainer dropped the old Provides: line.
+    
+    http://tracker.ceph.com/issues/10728 Fixes: #10728
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 4f36eaee9ea71ed48018ccb2d7eac3e809c72d82)
+
+commit 98769f7ccf97d6a0c98438f1eab7a5705fb9920b
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri Sep 4 01:07:48 2015 +0000
+
+    Fix casing of Content-Type header
+    
+    It turns out, despite the HTTP spec declaring that header field names
+    should be case-insensitive, some clients treat them wrongly, and
+    consider "Content-type" to not match "Content-Type".
+    
+    CyberDuck was one of those clients, now fixed upstream in
+    https://trac.cyberduck.io/ticket/8999
+    
+    To reduce future occurances of this bug, fix the casing of the
+    Content-Type header, to strictly comply with the HTTP specification (be
+    strict about what you send, and generous about what you receive).
+    
+    Fixes: #12939
+    Backport: hammer, firefly
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 1b9fbffdc24160251b96cec820d62fb2a12b6eab)
+
+commit ccf268502b0592f5ff58cd1a5920ec02a117acfb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 13 15:02:03 2015 +0200
+
+    rgw: we should not overide Swift sent content type
+    
+    Fixes: #12363
+    backport: hammer
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 423cf136f15df3099c9266f55932542d303c2713)
+
+commit 6829ee29a484a3b7055122addcda06a233d3ce53
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jun 25 18:28:50 2015 +0200
+
+    rgw: send Content-Length in response for GET on Swift account.
+    
+    Fixes: #12158
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5f47b11791cd46ce2771046e4888575869cc9685)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+                omit dump_account_metadata() call
+
+commit c9226afd7b9e54a8998bf88dfbd6753b262fdf9d
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jun 23 11:54:09 2015 +0200
+
+    rgw: enforce Content-Type in Swift responses.
+    
+    Swift sends Content-Type HTTP header even if the response
+    doesn't contain body. We have this behaviour implemented
+    until applying some changes in end_header() function.
+    Unfortunately, lack of Content-Type causes early exits in
+    many Tempest's tests for Swift API verification.
+    
+    Fixes: #12157
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 106aeba206736d4080326f9bc191876bed63370b)
+
+commit 9139fbbc1ab3ae455d2e36b3a768601ccb216801
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 22 13:51:49 2015 +0200
+
+    rgw: force content_type for swift bucket stats request
+    
+    Fixes: 12095
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit da259e4da1329e1e448d75c48c6ee3d1318e19b2)
+
+commit 767f1f74c9f015d6bb3376266ec77a10a2d2c59a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Apr 21 12:31:19 2015 +0200
+
+    rgw: force content-type header for swift account responses without body
+    
+    This regression was introduced in commit 4cfeca03ade21861ab70ca759f94eb244c16cb39
+    
+    Fixes: #11438
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 009664e560ee37ef6dc170a0155d6c749e151b06)
+
+commit e3da03dd6be04e782c3e5786767bba4b4964fbec
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 9 20:04:59 2015 +0200
+
+    rgw: shouldn't return content-type: application/xml if content length is 0
+    
+    Fixed: #11091
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5cc92bb27cfbaee4af24dc312c9b98331f899794)
+
+commit e7d08b1bca951f9a963eb0aeb4cd88abbe5723ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 8 10:26:48 2015 -0700
+
+    test/librados/tier.cc: destroy and recreate cache pool on every test
+    
+    Namespaces are not sufficient with the checks for 11493 in the mon.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bef09e0cdb274cb1c87335a2af9ee532d14a4596)
+
+commit d7f53442260dcfc92e7afa8bba2dd116c15e030d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 29 12:34:25 2015 -0700
+
+    mon: prevent pool with snapshot state from being used as a tier
+    
+    If we add a pool with snap state as a tier the snap state gets clobbered
+    by OSDMap::Incremental::propogate_snaps_to_tiers(), and may prevent OSDs
+    from starting.  Disallow this.
+    
+    Include a test.
+    
+    Fixes: #11493
+    Backport: hammer, giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbec53edf9e585af4e20bbc9ba9057d6fdfda342)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+               properly co-exist with "# make sure we can't create an ec pool tier"
+    	src/mon/OSDMonitor.cc
+               properly co-exist with preceding "if (tp->ec_pool())"
+            (The changes to both files would have applied cleanly if
+            https://github.com/ceph/ceph/pull/5389 had not been merged first.)
+
+commit 028da252d85733fcc7d8495750b9ae97043cb0dd
+Merge: ecabc67 f71a6eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 3 09:40:33 2015 +0200
+
+    Merge pull request #5993 from SUSE/wip-13169-firefly
+    
+    update docs to point to download.ceph.com
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 971eb3f6dddb509d15e5e4bcfd5e34d42ae2e75b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 22:33:10 2014 -0800
+
+    ceph_test_rados_api_tier: fix cleanup of whiteouts
+    
+    Doing a normal DELETE on a whiteout returns ENOENT.  Use the IGNORE_CACHE
+    flag to make sure it removes the whiteout, too.
+    
+    Fixes: #10052
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8dde6a61921798aa188bd69406c8fa7afeb96b45)
+    
+    Conflicts:
+    	src/test/librados/TestCase.cc
+                for it of type ObjectIterator:
+                - use it->first instead of it->get_oid()
+                - use it->second instead of it->get_locator()
+
+commit b134a629863b4601892db03d7c4377040d556bdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 22:32:20 2014 -0800
+
+    osd/ReplicatedPG: allow whiteout deletion with IGNORE_CACHE flag
+    
+    If the client specifies IGNORE_CACHE, allow a regular DELETE to zap a
+    whiteout.  Expand test case to verify this works.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34e4d24)
+    
+        Conflicts:
+            src/test/librados/tier.cc
+                replaced NObjectIterator -> ObjectIterator
+                replaced cache_ioctx.nobjects_begin -> cache_ioctx.objects_begin
+                replaced cache_ioctx.nobjects_end -> cache_ioctx.objects_end
+                replace it->get_oid() with it->first for it of type ObjectIterator
+
+commit 6d396a322e5ad61a3ca262b1c78a670481ff740c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:01 2015 -0400
+
+    mon/Elector: do a trivial write on every election cycle
+    
+    Currently we already do a small write when the *first* election in
+    a round happens (to update the election epoch).  If the backend
+    happens to fail while we are already in the midst of elections,
+    however, we may continue to call elections without verifying we
+    are still writeable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef909ccbdc303cce8a39edef255325127832ff16)
+    
+        Conflicts:
+    	src/mon/Elector.cc
+                no MonitorDBStore::TransactionRef in firefly
+
+commit d2bfa19077422eea37205fee9ed6fd5f2692f222
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:13 2015 -0400
+
+    mon/MonitorDBStore: assert/crash if there is a write error
+    
+    Do this globally intead of relying on teh zillion mon callers to
+    check the error code.  There are no cases where we want to
+    tolerate a commit failure.
+    
+    Fixes: #13089
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fb7b1f0e33ada7c9a1be3de2f7708eb0760fcef)
+
+commit c85d06384e795ab2717868ce45f01ae93181a6be
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:46:42 2015 -0700
+
+    use simplifed messenger constructor for clients
+    
+    This is all mechanical except the calculation of the nonce, which is
+    now always randomized for clients.
+    
+    Fixes: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit a3a8c85b79afef67681c32c57b591c0e0a87a349)
+    
+    Conflicts:
+    	src/ceph_fuse.cc
+    	src/ceph_syn.cc
+    	src/libcephfs.cc
+    	src/librados/RadosClient.cc
+    	src/mds/MDSUtility.cc
+    	src/mon/MonClient.cc
+    	src/test/mon/test_mon_workloadgen.cc
+                - different arguments to Messenger::create() in firefly
+
+commit ef24b4ec7245959b1960a415617439bc3baac9a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 21:20:08 2014 -0700
+
+    librados: use Messenger factory method
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5a860bec48348ed4c09256ccd394b33ace6b4f52)
+
+commit 3a52acde3133aab2a717fefa967b0d404e50e2d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 08:50:48 2014 -0700
+
+    mon/MonClient: use generic Messenger factory
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e55cb1f68ae1a04cc5d281bea57ee79ef4a8af38)
+
+commit de8da652b6c969aa0446f0d3351f63b012063c71
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:20:53 2015 -0700
+
+    msg: add simplified messenger constructor
+    
+    Several callers create messengers using exactly the same parameters:
+    
+    - reading the ms type from cct that is also passed in
+    - a default entity_name_t::CLIENT
+    - the default features
+    
+    Additionally, the nonce should be randomized and not depend on
+    e.g. pid, as it does in several callers now. Clients running in
+    containers can easily have pid collisions, leading to hangs, so
+    randomize the nonce in this simplified constructor rather than
+    duplicating that logic in every caller.
+    
+    Daemons have meaningful entity_name_ts, and monitors currently depend
+    on using 0 as a nonce, so make this simple constructer
+    client-specific.
+    
+    Related: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e3785b0b5fbff870adbd5f121021b671912386cf)
+    
+    Conflicts:
+    	src/msg/Messenger.cc
+                - different #includes in firefly
+                - different Messenger::create() arguments in firefly
+
+commit ecabc6796f19c03947bb5b14da4e1b761ff8847f
+Merge: 741dd5b 2e09a2c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 17:09:18 2015 -0400
+
+    Merge pull request #6038 from liewegas/wip-13088
+    
+    common/HeartBeat: include types
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2e09a2c22ab885f8ec81dbc55f2c8fc0f2984543
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 17:06:52 2015 -0400
+
+    common/RWLock: include assert
+    
+    Fixes atomic_t build issue without libatomic-ops.
+    
+    Fixes: #13088
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 741dd5b9433d1592ce91aea70ebec106a1354973
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 13:03:49 2015 -0400
+
+    .gitmodules: fix ceph-object-corpus source url
+    
+    Fixes: #13196
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit a1997879aa26299519ed157017f8f5715c626dcc
+Merge: ee81812 01dc221
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 12:29:08 2015 -0400
+
+    Merge pull request #6000 from SUSE/wip-12097-firefly
+    
+    kernel_untar_build fails on EL7
+
+commit ee8181206726712729235e4a5f8d2094a6e72f71
+Merge: bb8b422 4e7de5b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 21 13:07:49 2015 -0700
+
+    Merge pull request #5234 from SUSE/wip-12244-firefly
+    
+    Civetweb RGW appears to report full size of object as downloaded when only partially downloaded
+
+commit bb8b422beac22bd8c2e9c1ecd83e265a3de1a3e6
+Merge: 4b08895 f602263
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 21 12:35:52 2015 -0700
+
+    Merge pull request #4535 from xinxinsh/wip-11416-firefly
+    
+    rgw: use correct objv_tracker for bucket instance
+
+commit 6d82eb165fdc91851f702a463022b26c50f5094b (refs/remotes/me/wip-globalid-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 18 16:49:20 2015 -0800
+
+    mon: handle case where mon_globalid_prealloc > max_global_id
+    
+    This triggers with the new larger mon_globalid_prealloc value.  It didn't
+    trigger on the existing cluster I tested on because it already had a very
+    large max.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbaf582e5bb54ae35327ffd234e4ccf0f07317db)
+
+commit f545a0f430bf6f1e26983fc0ff20a645697f017c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 18 10:39:25 2015 -0800
+
+    mon: change mon_globalid_prealloc to 10000 (from 100)
+    
+    100 ids (session 100 authentications) can be consumed quite quickly if
+    the monitor is being queried by the CLI via scripts or on a large cluster,
+    especially if the propose interval is long (many seconds).  These live in
+    a 64-bit value and are only "lost" if we have a mon election before they
+    are consumed, so there's no real risk here.
+    
+    Backport: giant, firefly
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d1215fe5f95c2bafee5b670cdae1353104636a0)
+
+commit 01dc221f280ad25aa556f246414e3c3c42762956
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 16 08:13:41 2015 -0700
+
+    qa: update to newer Linux tarball
+    
+    This should make newer gcc releases happier in their default configuration.
+    kernel.org is now distributing tarballs as .xz files so we change to that
+    as well when decompressing (it is supported by Ubuntu Precise so we should
+    be all good).
+    
+    Fixes: #11758
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 1ea3f47ab806d48ca7b045c2731d344eae3900e1)
+    
+    Conflicts:
+    	qa/workunits/kernel_untar_build.sh
+                trivial resolution
+
+commit 86f9e55f0c151c0b9a289b475f87b6a11329e6e1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 10:51:52 2015 -0800
+
+    rgw: use strict_strtoll() for content length
+    
+    instead of strict_strtol().
+    
+    Backport: giant, firefly
+    Fixes: #10701
+    
+    Reported-by: Axel Dunkel <ad@dunkel.de>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c25dbafb45caf1b261cfcec15b868a2ba6b5fef)
+
+commit f71a6ebf1b371f9389865a0a33652841726ff77b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:02 2015 -0400
+
+    doc: correct links to download.ceph.com
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 387d7800359154950431d0984c756f43f21dd9b4)
+    
+    Conflicts:
+    	doc/install/get-packages.rst
+                preferred the new version in all cases
+
+commit 20ad17d271fb443f6c40591e205e880b5014a4f3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 14:57:34 2015 -0400
+
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    It may take tens of seconds to restart each time, so 5 in 30s does not stop
+    the crash on startup respawn loop in many cases.  In particular, we'd like
+    to catch the case where the internal heartbeats fail.
+    
+    This should be enough for all but the most sluggish of OSDs and capture
+    many cases of failure shortly after startup.
+    
+    Fixes: #11798
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eaff6cb24ef052c54dfa2131811758e335f19939)
+
+commit 8c14cad0895590f19a6640c945b52213f30a9671
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 19:10:19 2015 -0400
+
+    WorkQueue: add/remove_work_queue methods now thread safe
+    
+    These methods were not acquiring the ThreadPool lock when
+    manipulating the work_queue collection.  This was causing
+    occasional crashes within librbd when opening and closing
+    images.
+    
+    Fixes: #12662
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e18449b01c1ab78d1bbfc1cf111aa9bdbef7b1f)
+
+commit 48c929e689b0fa5138922fcb959be5d05296e59a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 27 11:08:33 2015 -0700
+
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Otherwise, the is_active() checks in the hitset code can erroneously
+    return true firing off repops stamped with the new epoch which then get
+    cleared in the map change code.  The filestore callbacks then pass the
+    interval check and call into a destroyed repop structure.
+    
+    Fixes: 12809
+    Backport: hammer,firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 14e02bc90a463805f4c3e2de210892067a52514b)
+
+commit 4b0889567d7fe28ef6976f8dbb0d329b03200986
+Merge: 0a740d7 5426421
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 10 11:46:41 2015 +0200
+
+    Merge pull request #5200 from SUSE/wip-12289-firefly
+    
+    mon: scrub error (osdmap encoding mismatch?) upgrading from 0.80 to ~0.80.2
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit de53addac8234037a66cdd45cf8007deba7a0530 (refs/remotes/gh/wip-11786.firefly)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Sep 7 14:12:19 2015 +0100
+
+    mon: MonitorDBStore: make get_next_key() work properly
+    
+    We introduced a significant bug with 2cc7aee, when we fixed issue #11786.
+    Although that patch would fix the problem described in #11786, we
+    managed to not increment the iterator upon returning the current key.
+    This would have the iterator iterating over the same key, forever and
+    ever.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 70d31082fd3dc8c7857994104577f1a3631c678c)
+
+commit 0a740d7b2b62e55056fb96d6423203c47e0f3cf6
+Merge: aeea956 37d16a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 15:45:37 2015 +0200
+
+    Merge pull request #5388 from SUSE/wip-12490-firefly
+    
+    buffer: critical bufferlist::zero bug
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit aeea9567e8b047b3a67badfcc4e43c880f0efa46
+Merge: 973b74d 6b42b06
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:10:09 2015 +0200
+
+    Merge pull request #5408 from SUSE/wip-12492-firefly
+    
+    the output is wrong when runing ceph osd reweight
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 973b74d5b266f4e9e7f992a85b22f42c9fb543b1
+Merge: 79403ba 5445461
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:07:51 2015 +0200
+
+    Merge pull request #5404 from SUSE/wip-12395-firefly
+    
+    register_new_pgs() should check ruleno instead of its index
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 79403ba8ce526d65521e6cd2ca2bc06f5fe261b9
+Merge: db44bb2 dc12875
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:05:39 2015 +0200
+
+    Merge pull request #5199 from SUSE/wip-11980-firefly
+    
+    MDSMonitor: handle MDSBeacon messages properly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit db44bb29b8b036642c50f5d322b5edd9d51699bb
+Merge: fc699f4 31ff7d1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:04:15 2015 +0200
+
+    Merge pull request #5410 from SUSE/wip-12497-firefly
+    
+    get pools health'info have error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit fc699f42459ab3723f3e63b28a4aa5f481c78fc4
+Merge: 6762295 ea0d67c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:01:50 2015 +0200
+
+    Merge pull request #5409 from SUSE/wip-12495-firefly
+    
+    pgmonitor: wrong at/near target max“ reporting
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 67622955b1cfd0b97ac8d3d96b223995440af0b1
+Merge: c4fa5ef 2f35a41
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 7 14:22:52 2015 +0200
+
+    Merge pull request #5358 from ceph/wip-11470.firefly
+    
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 3be204f6a22be109d2aa8cfd5cee09ec3381d9b2
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Tue Mar 17 15:44:52 2015 -0700
+
+    rgw: init script waits until the radosgw stops
+    
+    Fixes: #11140
+    Init script waits in stop action until the radowgw daemons stop.
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit 1cca0c1e91a1e18f82f4d22855e96b2fc947f5ea)
+
+commit c4fa5ef989247404ded7adf5dad4e5316b8af04a
+Merge: beff99b c6f3780
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 7 10:20:27 2015 +0200
+
+    Merge pull request #5403 from SUSE/wip-12393-firefly
+    
+    Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit beff99bc7358ee0625079ca60adbf109986f2285
+Merge: 4a3c178 68508e2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 6 21:19:55 2015 +0200
+
+    Merge pull request #5225 from SUSE/wip-12266-firefly
+    
+    ceph.spec.in: 50-rbd.rules conditional is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4a3c17834890b69e0467b65f83b1979328f3d27c
+Merge: 6571cd9 2544a6f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 6 21:19:42 2015 +0200
+
+    Merge pull request #5217 from SUSE/wip-12268-firefly
+    
+    ceph-common needs python-argparse on older distros, but doesn't require it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bff2f477c4ad86b4bd6e3ca3e637a6168c5c8053
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Sep 6 16:30:35 2015 +0200
+
+    OSD: add scrub_finalize_wq suicide timeout
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 91d4c217e32b8b76fcac49f37879a3f78088694d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:52:40 2015 -0700
+
+    OSD: add scrub_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 547a7041edc833f3cc8e04d388574809e30a8af6)
+    
+    Conflicts:
+            src/osd/OSD.cc
+                firefly still has scrub_finalize_wq
+
+commit 7f6ec65b7c2ca0174142c1c48f18998d8c586b02
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 11:02:19 2015 -0700
+
+    OSD: add op_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3f2946aa9f9b4b432e8c018283503153a2b1eddc)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+              adapt op_wq() and peering_wq() calls in OSD::OSD
+    	src/osd/OSD.h
+              adapt OpWQ instead of ShardedOpWQ
+
+commit 9ce8ce689009cf8ef749edd320d1c2a73ecc2f90
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:54:31 2015 -0700
+
+    OSD: add remove_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f2fbfa32a16666be46359f0eab7b04ca80a753f5)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+               No next_removal_seq(0) in master
+
+commit 6926a64fbd4718b8a5df8e04545bc93c4981e413
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:51:28 2015 -0700
+
+    OSD: add snap_trim_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit e1073a4a577211672148a4112bd633831552d66f)
+
+commit d31d1f6f5b08362779fa6af72690e898d2407b90
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:50:19 2015 -0700
+
+    OSD: add recovery_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 85311b656852af75bfbbc6699f92fc6aa233c316)
+    
+    Conflicts:
+    	src/common/config_opts.h
+               place OPTION(osd_command_thread_suicide_timeout, ...)
+               directly after OPTION(osd_command_thread_timeout, ...)
+
+commit f85ec2a52e969f9a7927d0cfacda6a1cc6f2898c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:49:00 2015 -0700
+
+    OSD: add command_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit df4e5de819c30003cfbe50a071c49039cf534419)
+    
+    Conflicts:
+    	src/common/config_opts.h
+               place OPTION(osd_command_thread_suicide_timeout, ...)
+               directly after OPTION(osd_command_thread_timeout, ...)
+
+commit b894b368790de3383295372250888ba674502fb1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 08:49:54 2015 -0700
+
+    WBThrottle::clear_object: signal if we cleared an object
+    
+    Fixes: #12223
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8447b08d60d5cf7dce20f8b839cc2cb5a93782de)
+
+commit cd1396cd62c79b177e46cfb57ab6b3b6fdcd227b
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 9 10:46:10 2015 -0400
+
+    osd/PGLog: dirty_to is inclusive
+    
+    There are only two callers of mark_dirty_to who do not pass max,
+    and they are both in the merge_log extending tail path.  In that
+    case, we want to include the last version specified in the log
+    writeout.  Fix the tail extending code to always specify the
+    last entry added, inclusive.
+    
+    Fixes: #12652
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f0ca14df0641daa04eee39d98d8bd0faf46e4e6d)
+
+commit f39c7917a39e445efa8d73178657fc5960772275
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Sep 4 17:38:32 2015 +0200
+
+    osd: Keep a reference count on Connection while calling send_message()
+    
+    Fixes: #12437
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (manual backport of commit a140085)
+    
+    Conflicts:      src/osd/OSD.cc
+                       master has share_map_peer; firefly has osd->_share_map_outgoing
+
+commit 6571cd9a9616637bf32f9ac97a9bd864c0bff856
+Merge: 8edc091 bd25136
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 11:50:22 2015 -0600
+
+    Merge pull request #5224 from SUSE/wip-12304-firefly
+    
+    ceph.spec.in running fdupes unnecessarily
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 8edc091df90be6935fcfe59c18d7060c88501556
+Merge: 6053bff 601eadb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 11:47:49 2015 -0600
+
+    Merge pull request #5394 from SUSE/wip-12447-firefly
+    
+    ceph.spec.in: useless %py_requires breaks SLE11-SP3 build
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 2c2ffa1d6d1112dbf52cbbe36f4a5376e17da56a
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 24 15:40:39 2015 -0700
+
+    config: skip lockdep for intentionally recursive md_config_t lock
+    
+    lockdep can't handle recursive locks, resulting in false positive
+    reports for certain set_val_or_die() calls, like via
+    md_config_t::parse_argv() passed "-m".
+    
+    Fixes: #12614
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 415226995ebe841cc9e8639a3233150a75aecaed)
+
+commit 6053bffa8955953cf40a05f2b22684e77b6003a3
+Merge: 49d416e 5559a5f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 17:18:30 2015 +0200
+
+    Merge pull request #5043 from SUSE/wip-12007-firefly
+    
+    Objecter: pg listing can deadlock when throttling is in use
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2c0d7feeb1c7592887e0408fe4fadaa9b4f659e9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 19:41:47 2015 +0200
+
+    tests: tiering agent and proxy read
+    
+    Verify that an object promoted to a cache tier because of a proxy read
+    is evicted as expected.
+    
+    http://tracker.ceph.com/issues/12673 Refs: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7924231930732bd297d3bd034c8295e96cb81088)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+                resolved by manually adding the new test to TESTS
+
+commit aa911767d9326c8aa37671883892b7d383596960
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 13:47:24 2015 +0200
+
+    osd: trigger the cache agent after a promotion
+    
+    When a proxy read happens, the object promotion is done in parallel. The
+    agent_choose_mode function must be called to reconsider the situation
+    to protect against the following scenario:
+    
+      * proxy read
+      * agent_choose_mode finds no object exists and the agent
+        goes idle
+      * object promotion happens
+      * the agent does not reconsider and eviction does not happen
+        although it should
+    
+    http://tracker.ceph.com/issues/12673 Fixes: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e1f58feb9b1d20b72f2eb2eefdea5982e0cddccd)
+
+commit 49d416e6e2e2f3e1f317978371089f9d92358642
+Merge: 7744518 f93eadd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 17:08:41 2015 +0200
+
+    Merge pull request #4769 from SUSE/wip-11741-firefly
+    
+    crush: take crashes due to invalid arg
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 774451810ff055984b79e1d1875bd6a3468dfb7a
+Merge: fc27b0d f122788
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Sep 4 16:50:45 2015 +0200
+
+    Merge pull request #4788 from SUSE/wip-11801-firefly
+    
+    Make it more difficult to delete pools in firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fc27b0dbe074dc15450d7669ea722aa3b605b3b2
+Merge: 055624b da78889
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 16:44:57 2015 +0200
+
+    Merge pull request #5235 from SUSE/wip-12310-firefly
+    
+    read on chunk-aligned xattr not handled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 055624bf9a6480fffefb3c7cf5c3a4d4d73985c1
+Merge: 045c056 46e5dea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 16:42:28 2015 +0200
+
+    Merge pull request #5389 from SUSE/wip-12391-firefly
+    
+    EC pools are not allowed as cache pools, disallow in the mon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 045c0562f97e7d2c68278fbddb9ca1707bb8d5fd
+Merge: 99612e7 b8e3f6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 4 09:42:03 2015 -0400
+
+    Merge pull request #5406 from ceph/wip-12465-firefly
+    
+    log: take mutex while opening fd
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 99612e7ed9573f2af36604f7d258535f6feadff3
+Merge: 7f8a397 4a709a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 3 12:34:32 2015 +0200
+
+    Merge pull request #4854 from ceph/wip-11769-firefly
+    
+    librbd: aio calls may block
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7f8a397e38101017ef9783d8ed08bb393a8d8825
+Merge: ea601ef f0d2429
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 07:55:15 2015 +0200
+
+    Merge pull request #5233 from SUSE/wip-12074-firefly
+    
+    RGW logfile does not get chowned properly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 644f213c672f6fe2786e041043fdd55f8399871e (refs/remotes/gh/wip-11455-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 14:34:30 2015 -0700
+
+    rgw: init some manifest fields when handling explicit objs
+    
+    Fixes: #11455
+    When dealing with old manifest that has explicit objs, we also
+    need to set the head size and head object correctly so that
+    code that relies on this info doesn't break.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7bf8cb594e5276d1c80544f5ec954d52b159750)
+
+commit e7931a73df1ab77feb1c2ece13e3de3989ef7a0e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 30 14:47:15 2015 -0700
+
+    rgw: url encode exposed bucket
+    
+    Fixes: #12537
+    Don't send the bucket name back without url encoding it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ad5507fe0bf72ed5bdf8353e315cc9092c740144)
+
+commit 558c52955d464827630e0aa2fed970df987bb036
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Jun 3 13:13:33 2015 +0200
+
+    rgw: Do not enclose the Bucket header in quotes
+    
+    Fixes: #11860
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8af25faed93fe02d3dad585b8579ce8b041cc4e6)
+
+commit d3c94698e4e852bef3e65fbf439f5f209fbc0b25 (refs/remotes/gh/wip-12752)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 21 11:32:39 2015 -0400
+
+    Objecter: pg_interval_t::is_new_interval needs pgid from previous pool
+    
+    When increasing the pg_num of a pool, an assert would fail since the
+    calculated pgid seed would be for the pool's new pg_num value instead
+    of the previous pg_num value.
+    
+    Fixes: #10399
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f20f7a23e913d09cc7fc22fb3df07f9938ddc144)
+    
+    Conflicts: (hobject_t sort order not backported, trivial resolution)
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit 56d267b7ae02070a7d7ed247990b84124fd62411
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 26 12:35:11 2015 -0700
+
+    osd_types::is_new_interval: size change triggers new interval
+    
+    Fixes: 11771
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ff79959c037a7145f7104b06d9e6a64492fdb95f)
+
+commit 411769c1461c11611b479bd826c72c56b3ce47c5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 10 04:25:03 2015 -0700
+
+    mon: add a cache layer over MonitorDBStore
+    
+    the cache of of leveldb does not perform well under some condition,
+    so we need a cache in our own stack.
+    
+    * add an option "mon_osd_cache_size" to control the size of cache size
+      of MonitorDBStore.
+    
+    Fixes: #12638
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e99a578050976ca22b549812ac80d494fe7041d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.h
+    minor differences in OSDMonitor ctor
+
+commit 278d732ecd3594cd76e172d78ce3ec84e58e178b (refs/remotes/me/wip-journal-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 15 11:20:18 2015 -0800
+
+    os/FileJournal: Fix journal write fail, align for direct io
+    
+    when config journal_zero_on_create true, osd mkfs will fail when zeroing journal.
+    journal open with O_DIRECT, buf should align with blocksize.
+    
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Xie Rui <875016668@qq.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80473f63853593e53d72f17c098152caf17f5e9e)
+
+commit d0d6727762ebda858065101635935df3d44a18ad
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 29 15:52:19 2015 +0800
+
+    common/syncfs: fall back to sync(2) if syncfs(2) not available
+    
+    Fixes: #12512
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08210d6a85527b0759f6dc4411f742bc13a4939f)
+
+commit 40494c6e479c2ec4dfe5f6c2d6aef3b6fa841620
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Mar 12 17:38:27 2015 +0100
+
+    sync_filesystem.h: fix unreachable code
+    
+    Fix for:
+    
+    CID 1264460 (#1 of 1): Structurally dead code (UNREACHABLE)
+    unreachable: This code cannot be reached: sync();
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9921836e982818b92e20280eecb74e701523045c)
+
+commit 3dbbc86ad6d1e7131bbe49a4eff1557d7da9822f
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Jan 8 10:29:37 2015 +0800
+
+    mon, os: check the result of sync_filesystem.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 27cb78bd3b2dde40c70de245769d1c0ec252533b)
+
+commit 7fa6fdc6c5e52f11456e4bea4ae32fd62248c80b
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jan 7 15:28:37 2015 +0800
+
+    common: Don't call ioctl(BTRFS_IOC_SYNC) in sync_filesystem.
+    
+    From the manual of syncfs, it first appeared in Linux 2.6.39. At this
+    point, btrfs didn't as a stable production. So remove this.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 397b261630dccc656f6eca509f4c70173c8d8d8f)
+
+commit 9f15ed5bb5a837727cb3bef70508e056c125a518
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jan 7 15:26:08 2015 +0800
+
+    common: Directly return the result of syncfs().
+    
+    In commit 808c644248e4, it will try sync() if syncfs() return error.
+    No evidence prove this way can work. And sync() don't return result
+    so make this function always return zero which cause filestore omit the
+    error.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 3f7faa4d8d3af3d2e3889c23290b016f27d36386)
+
+commit ea601efc6c693ba8041509c98756fa8c49b9c13a
+Merge: 50fa963 a128ce3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 12 09:43:19 2015 +0200
+
+    Merge pull request #5390 from SUSE/wip-12503-firefly
+    
+    rgw/logrotate.conf calls service with wrong init script name
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 50fa9636d0c5e8946fa410891623e2c8363e2067
+Merge: abe6c03 d4ef556
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 11 11:27:16 2015 -0700
+
+    Merge pull request #4765 from SUSE/wip-11721-firefly
+    
+    rgw: Keystone PKI token expiration is not enforced
+
+commit 04b2a878b1329202758465cf8e9b0f874cbeeef5
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Tue Nov 18 15:51:46 2014 +0100
+
+    ceph-disk: don't change the journal partition uuid
+    
+    We observe that the new /dev/disk/by-partuuid/<journal_uuid>
+    symlink is not always created by udev when reusing a journal
+    partition. Fix by not changing the uuid of a journal partition
+    in this case -- instead we can reuse the existing uuid (and
+    journal_symlink) instead. We also now assert that the symlink
+    exists before further preparing the OSD.
+    
+    Fixes: #10146
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Tested-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 29eb1350b4acaeabfe1d2b19efedbce22641d8cc)
+
+commit e57e6f5da10a62f2f4d7b1a6a734a095ed494ebe
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Mon Sep 29 13:20:10 2014 +0200
+
+    ceph-disk: set guid if reusing a journal partition
+    
+    When reusing a journal partition (e.g. /dev/sda2) we should set a
+    new partition guid and link it correctly with the OSD. This way
+    the journal is symlinked by its persistent name and ceph-disk list
+    works correctly.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit beff616f506b96eb52285f0d2e268e10f8edaa2c)
+
+commit ebba1d59b5e4bc11cbdfcda4e480639f7d9e1498
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 30 14:20:56 2015 +0100
+
+    osd/OSDMap: handle incrementals that modify+del pool
+    
+    Because new_pools was processed after old_pools, if something
+    was modified and then deleted in the same incremental
+    map, then the resulting state would be a pool in the map
+    that had no entry in pool_name.
+    
+    Fixes: #12429
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d4932e721e32a33c1bada54e11d4ecccd868618)
+
+commit 31ff7d14e801ef7b7dff3b050094f1eceacf75dd
+Author: renhwztetecs <rhwlyw@163.com>
+Date:   Tue Jul 21 18:55:00 2015 +0800
+
+    Update OSDMonitor.cc
+    
+    OSDMonitor::get_pools_health(), s/objects/bytes/
+    
+    Fixes: #12402
+    Signed-off-by: ren.huanwen@zte.com.cn
+    (cherry picked from commit 7fc13c9d6b9a4962d7640240416105d8f558d600)
+
+commit ea0d67c482659dfb8c08055a77f400d0689866e0
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Mon Jul 20 10:50:20 2015 +0800
+
+      mon/PGMonitor: avoid uint64_t overflow when checking pool 'target/max' status.
+      Fixes: #12401
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f84e6b8157e78fd05cd1c8ac957c92df17e6ba06)
+
+commit 6b42b06239426e5ec54dfaa82f48abca56f824ca
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Tue Jul 21 11:20:53 2015 +0100
+
+    mon: OSDMonitor: fix hex output on 'osd reweight'
+    
+    Fixes: #12251
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 787fa80c2746fde44ac0583ff7995ec5be9a672d)
+
+commit 5445461ba08fa5b2e12a951a1432916224d3f668
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Fri Jul 3 18:27:13 2015 +0800
+
+    mon/PGMonitor: bug fix pg monitor get crush rule
+    
+    when some rules have been deleted before, the index in array of crush->rules
+    is not always equals to crush_ruleset of pool.
+    
+    Fixes: #12210
+    Reported-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 498793393c81c0a8e37911237969fba495a3a183)
+
+commit c6f37807d04af66fbed24e75776aed1b44241150
+Author: Ketor Meng <d.ketor@gmail.com>
+Date:   Tue May 26 18:50:17 2015 +0800
+
+    Mutex: fix leak of pthread_mutexattr
+    
+    Need pthread_mutexattr_destroy after pthread_mutexattr_init
+    
+    Fixes: #111762
+    Signed-off-by: Ketor Meng <d.ketor@gmail.com>
+    (cherry picked from commit 2b23327b3aa8d96341d501a5555195ca1bc0de8f)
+
+commit 601eadbf707597d9a1572efbdf86b22ff6813602
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 11:51:13 2015 +0200
+
+    ceph.spec.in: drop SUSE-specific %py_requires macro
+    
+    %py_requires expands to
+    
+    BuildRequires: /usr/bin/python
+    PreReq: python = 2.7
+    
+    The BuildRequires: is already provided, and the PreReq is wrong because
+    e.g. SLE11-SP3 (a platform we are trying to support) has Python 2.6.
+    
+    http://tracker.ceph.com/issues/12351 Fixes: #12351
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit cbf6c7b46b42e163950bf7956e5b7565fca961c9)
+    
+    Conflicts:
+    	ceph.spec.in
+                in master, the %py_requires is in the python-rados package,
+                while in firefly it is in the python-ceph package
+
+commit a128ce37038ef6e44c8d7bd082ba196583974ef7
+Author: wuxingyi <wuxingyi2015@outlook.com>
+Date:   Wed Mar 11 17:34:40 2015 +0800
+
+    rgw/logrotate.conf: Rename service name
+    
+    The service name for ceph rados gateway was changed to "ceph-radosgw",
+    the previous version of service name "radosgw" would cause a failed reload,
+    and finally make it impossible to write any log data to the log file.
+    
+    Signed-off-by: wuxingyi <wuxingyi2015@outlook.com>
+    (cherry picked from commit 9df3f798179481fe8ae6ae873dcb793de7d8f367)
+
+commit 46e5deabebeeb047783cfc980aa28a3df46eeeca
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 15 13:05:40 2015 -0700
+
+    OSDMonitor: disallow ec pools as tiers
+    
+    Fixes: 11650
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 11b7801bb57cb25cd2d26d58722d49691747725b)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+               no "# make sure we can't clobber snapshot state" tests in firefly
+    	src/mon/OSDMonitor.cc
+               no tp->removed_snaps.empty() in firefly
+
+commit 37d16a9e572580eeae86a2bae6d4ddd0299fb833
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:32:03 2015 +0800
+
+    UnittestBuffer: Add bufferlist zero test case
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 577acf665948e531e50d36f5780ea19351a5bf65)
+
+commit 4443acdbef1148e0261bce25f7d7a3433e09cecc
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:42:42 2015 +0800
+
+    buffer: Fix bufferlist::zero bug with special case
+    
+    Fixes: #12252
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 43f583d160ccaf879eaf0f3020e77860cf8d1df0)
+
+commit d52187019d321fe8a2dc54fe8a67a5139c310db1
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed May 27 22:28:49 2015 +0100
+
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    get_next_key() had a bug in which we would always return the first key
+    from the iterator, regardless of whether its prefix had been specified
+    to the iterator.
+    
+    Fixes: #11786
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 2cc7aee1abe40453093306c8fef2312b650dff5d)
+
+commit 2f35a415dcd31683151defb0ddb1a6d5696c7051
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Jun 12 19:21:10 2015 +0100
+
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Whenever the monitor finishes committing a proposal, we call
+    Monitor::refresh_from_paxos() to nudge the services to refresh.  Once
+    all services have refreshed, we would then call each services
+    post_paxos_update().
+    
+    However, due to an unfortunate, non-critical bug, some services (mainly
+    the LogMonitor) could have messages pending in their
+    'waiting_for_finished_proposal' callback queue [1], and we need to nudge
+    those callbacks.
+    
+    This patch adds a new step during the refresh phase: instead of calling
+    directly the service's post_paxos_update(), we introduce a
+    PaxosService::post_refresh() which will call the services
+    post_paxos_update() function first and then nudge those callbacks when
+    appropriate.
+    
+    [1] - Given the monitor will send MLog messages to itself, and given the
+    service is not readable before its initial state is proposed and
+    committed, some of the initial MLog's would be stuck waiting for the
+    proposal to finish.  However, by design, we only nudge those message's
+    callbacks when an election finishes or, if the leader, when the proposal
+    finishes.  On peons, however, we would only nudge those callbacks if an
+    election happened to be triggered, hence the need for an alternate path
+    to retry any message waiting for the initial proposal to finish.
+    
+    Fixes: #11470
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 1551ebb63238073d2fd30201e6b656a8988e958c)
+
+commit b8e3f6e190809febf80af66415862e7c7e415214
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jul 24 15:38:18 2015 -0700
+
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Otherwise, _flush() might continue to write to m_fd after it's closed.
+    This might cause log data to go to a data object if the filestore then
+    reuses the fd during that time.
+    
+    Fixes: #12465
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8778ab3a1ced7fab07662248af0c773df759653d)
+    
+      - modified to drop m_flush_mutex_holder, which isn't present in firefly
+
+commit abe6c03d411f5f1d277f910ae87123309fa95d4a
+Merge: 5df7273 9614e4d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:51:43 2015 +0200
+
+    Merge pull request #4583 from SUSE/wip-11542-firefly
+    
+    rgw: Change variable length array of std::strings (not legal in C++) to std::vector<std::string>
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5df727379b63abb25421a32c95fdda9a0b62eeac
+Merge: 2a77154 8c573c8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:50:06 2015 +0200
+
+    Merge pull request #4584 from SUSE/wip-8911-firefly
+    
+    rgw: doesn't return 'x-timestamp' in header which is used by 'View Details' of OpenStack
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a77154b4abcd4eebc8d1a315f7beeeabaf2213b
+Merge: 64c3a64 d78e31f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:48:31 2015 +0200
+
+    Merge pull request #4642 from SUSE/wip-11323-firefly
+    
+    rgw: quota not respected in POST object
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 64c3a645c4dba1846532997758c5d7c3a42760b2
+Merge: f11df3c 177ef55
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:24:39 2015 +0200
+
+    Merge pull request #4641 from SUSE/wip-11149-firefly
+    
+    rgw: delete bucket does not remove .bucket.meta file
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit f11df3c7819a5f8dd5336be7dbe43e0f21dbb394
+Merge: 090bd12 2ee6e9a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:21:31 2015 +0200
+
+    Merge pull request #5170 from SUSE/wip-12039-firefly
+    
+    rgw: Swift API: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit 090bd126e15acacc46bce3271d2aa0cfa889b499
+Merge: 94060b7 2d5d8da
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:20:15 2015 +0200
+
+    Merge pull request #4762 from SUSE/wip-11754-firefly
+    
+    rgw: Object copy bug
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit 94060b78a2e9b38422a951664dd4f37a6f52316f
+Merge: b9f45fe 3223bd6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 24 11:42:58 2015 -0700
+
+    Merge pull request #5171 from SUSE/wip-12240-firefly
+    
+    librbd: internal.cc: 1967: FAILED assert(watchers.size() == 1)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b9f45fe7816da1ce1fcbfd2a60fe370abe0f1848
+Merge: 57bbc8e 9845258
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 24 11:26:23 2015 -0700
+
+    Merge pull request #4639 from SUSE/wip-7385-firefly
+    
+    librbd: Objectcacher setting max object counts too low
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8c02376bd58d463f742966b67fa075a59b5f4269
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jul 24 18:50:03 2015 +0200
+
+    TestPGLog: fix invalid proc_replica_log test case
+    
+    If 1'3 is non-divergent, no osd could correctly have created 2'3.  Also,
+    proc_replica_log does not add the extra entries from the auth log to the
+    missing set, that happens later on in activate.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1333b676dba4ed4b5db854fe935bdadbf9f21d08)
+    
+    Conflicts:
+        src/test/osd/TestPGLog.cc
+           "sjustm: set_hash() method just does hash = and then rebuilds a cache,
+            but the cache does not exist in firefly so hash = will do fine"
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4a709a450987cc90126b11a20c1c762d4fbfcea9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 2 10:33:35 2015 -0400
+
+    tests: verify librbd blocking aio code path
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4cf41486e9c9e1efcb863960a8f3e0326ffca7e5)
+
+commit 4501e90d9b667310a222c3616a7b1f1fb382f997
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 1 22:56:11 2015 -0400
+
+    librbd: new rbd_non_blocking_aio config option
+    
+    Setting this option to false reverts librbd to legacy behavior
+    where AIO operations could potentially block.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/librbd/librbd.cc: trivial resolution
+
+commit 68863968dd734fc45557dca100c8a0a5060cbd8b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 20:34:28 2015 -0400
+
+    PendingReleaseNotes: document changes to librbd's aio_read methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 861c20bb2cc05a13a45b0e2368c591212e4eda72
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:55:36 2015 -0400
+
+    tests: update librbd AIO tests to remove result code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    
+    Conflicts:
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 4e7cf6115798dd339efe41a528aac148be640b29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 13:33:09 2015 -0400
+
+    librbd: AioRequest::send no longer returns a result
+    
+    The librados calls used by AioRequest::send should always return
+    zero unless there is a bug.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c77bce3311ab62892eb8c1d883263ba7ed663b20)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/AioRequest.h: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit 0ccf47bf9f070ff7fe4e1dd3d3c3e3daa1f621c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:37:50 2015 -0400
+
+    librbd: internal AIO methods no longer return result
+    
+    All failures should be returned via the AioCompletion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ab42d613128ab08c688ddbea93df4c95068b9cd)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+    	src/librbd/internal.h: trivial resolution
+
+commit e48040413849c5d3a577addff644502858ac5a66
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:48:21 2015 -0400
+
+    Throttle: added pending_error method to SimpleThrottle
+    
+    Allow the client of SimpleThrottle to detect an async error
+    so that it can exit early.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b88b88c5df91325fb713c2031a56bffe421268e0)
+
+commit 8ec341ee8f256e8ab91a931f6005e1ebd4b1db60
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 20:18:50 2015 -0400
+
+    librbd: add new fail method to AioCompletion
+    
+    Helper method to handle passing fatal errors generated within
+    librbd (not from the OSDs) back to the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d1d0c867855a96bee4c13a0c0a39a0e002ccd12)
+    
+    Conflicts:
+    	src/librbd/AioCompletion.cc: trivial resolution
+    	src/librbd/AioCompletion.h: trivial resolution
+
+commit eed8ce258d116e837d8777f246f31c21a03ceb74
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 19:06:52 2015 -0400
+
+    librbd: avoid blocking AIO API methods
+    
+    Enqueue all AIO API methods within the new librbd thread pool to
+    reduce the possibility of any blocking operations. To maintain
+    backwards compatibility with the legacy return codes of the API's
+    AIO methods, it's still possible to block attempting to acquire
+    the snap_lock.
+    
+    Fixes: #11056
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a7b5e30efdb21aa1a0aeb68a5d02a1ac2a5faf3)
+    
+    Conflicts:
+    	src/librbd/librbd.cc: trivial resolution
+
+commit 220a23e8d99882838626d5647f9c055eefeb28c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add task pool / work queue for AIO requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afb896d91f886b647baf38f7ec94cc3739f6d2a9)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/librbd/ImageCtx.cc: trivial resolution
+    	src/librbd/ImageCtx.h: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit 3292bd26127f5beb69ac9c5fbcdea7efe002fd40
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 17:05:49 2015 -0400
+
+    WorkQueue: added virtual destructor
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3f5a75332c058816dc39b71e9d2b36e752159f4)
+
+commit 9bf970c1d956c94bc420d874eb83db1f16cf5d44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: add new ContextWQ work queue
+    
+    The queue holds a collection of Context pointers that will
+    be completed by the thread pool.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24a33e977f7b71962adeeb48f75d488a76e70fa9)
+    
+    Conflicts:
+    	src/common/WorkQueue.h: trivial resolution
+
+commit 94aa947b33083bcfe3cb61f41bfdaea7df605cbb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 14:21:08 2014 -0800
+
+    common/ceph_context: don't import std namespace
+    
+    This was broken by 7fed5dee4f96a83d1d6914f6fc0895bba2d15b99
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9029813b2784fe71879b1b7684855415b4c7566b)
+
+commit b91e594401ab75987a370f6dee0ee8d212189c9f
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Dec 1 23:54:16 2014 +0800
+
+    CephContext: Add AssociatedSingletonObject to allow CephContext's singleton
+    
+    If some objects associated to CephContext want to create a singleton object,
+    it can inherit AssociatedSingletonObject and implement destruction to get notified.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 7fed5dee4f96a83d1d6914f6fc0895bba2d15b99)
+
+commit f122788b3592da516cdb9f6abdc3383a2e7424a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 16 07:54:22 2015 -0800
+
+    mon/OSDMonitor: require mon_allow_pool_delete = true to remove pools
+    
+    This is a simple safety check.  Since we default to true it is currently
+    opt-in.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b982e4cc00f9f201d7fbffa0282f8f3295f2309)
+    
+    Conflicts:
+         src/test/mon/osd-pool-create.sh
+    
+         Changed $CEPH_MON to 127.0.0.1 -- the CEPH_MON was introduced after
+         firefly to allow tests to run in parallel. Back in firefly all tests
+         use the same port because 127.0.0.1 was hardcoded. We can't
+         conveniently backport all that's necessary for tests to run in
+         parallel, therefore we keep the 127.0.0.1 hardcoded.
+    
+    Conflicts:
+    	src/test/mon/osd-pool-create.sh
+                TEST_no_pool_delete() follows a different test than in master
+
+commit 57bbc8e630e6611d1ab74b9756c96daf0e91be6b
+Merge: c0a601f 3f2fb8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 24 10:49:50 2015 +0200
+
+    Merge pull request #5051 from SUSE/wip-12016-firefly
+    
+    osd: cache full mode still skips young objects
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c0a601ff5be45c0630a8659994107672558a982a
+Merge: 71174a5 ef648e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 22 11:36:47 2015 -0400
+
+    Merge pull request #5307 from dachary/wip-release-notes-firefly
+    
+    backport v0.80.10 release notes to firefly
+
+commit 5559a5fe0f593c0e7af47c6d8a8359dbcbfd1f08
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Sep 15 11:41:06 2014 +0000
+
+    For pgls OP, get/put budget on per list session basis, instead of per OP basis, which could lead to deadlock.
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    (cherry picked from commit 0f884fdb31a26f241401ce2a9329dc0f2c4eccc7)
+    
+    Conflicts:
+    	src/librados/IoCtxImpl.cc
+                In firefly, return value of objecter->pg_read() is not assigned to c->tid.
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+                There is no _op_submit_with_budget() function in firefly.
+                There is no Objecter::_finish_op() function in firefly.
+                In firefly, _take_op_budget() is called take_op_budget().
+
+commit 71174a53c9a3a2b54b947da7e1425ff36fd44e37
+Merge: f7a085f 989e047
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 22 10:20:32 2015 +0200
+
+    Merge pull request #4867 from SUSE/wip-11878-firefly
+    
+    mon: Clock skew causes missing summary and confuses Calamari
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 989e04799be567ee93369a87b90405833c3fe8d2
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Fri Feb 6 01:26:40 2015 +0100
+
+    Always provide summary for non-healthy cluster.
+    
+    This fixes a problem, wherein calamari does not provide
+    popup drill-downs for warnings or errors, should the summary
+    be missing.
+    
+    Calamari gets health info from /api/v1/cluster/$FSID/health.
+    If the data here has a summary field, this summary is provided
+    in a popup window:
+    
+      /api/v1/cluster/$FSID/health is populated (ultimately) with
+      status obtained via librados python bindings from the ceph
+      cluster. In the case where there's clock skew, the summary
+      field supplied by the ceph cluster is empty.
+    
+    No summary field, no popup window with more health details.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit eaf6e0cf48488fe604d0ef0db164d44948d4e8d4)
+    
+    Conflicts:
+    	src/mon/Monitor.cc
+                firefly needs ss
+
+commit f7a085f04c544895bab3d79569276574a96636eb
+Merge: 2f1dd10 d296120
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Jul 21 13:31:33 2015 -0700
+
+    Merge pull request #5062 from SUSE/wip-11699-firefly
+    
+    objecter: resend linger ops on split
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ef648e592404e8677df91f218c4400caad4034b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 21 12:54:19 2015 -0400
+
+    doc: v0.80.10 changelog
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3a3e0e509eb9530994723ed9d68f85b8434a22d)
+
+commit df837e5cc30cf08ade26df6e42ca402fc628a9bb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 21 10:55:18 2015 -0400
+
+    doc/release-notes: v0.80.10
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a61b611987d169d5a501d77fbd545068550c28c)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit 90cddd425ca8f1731fff0b8fb4b7e02a96048f92
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jul 14 16:15:44 2015 +0200
+
+    doc/release-notes: update notes for v0.80.10
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1231ae09d675f2ee9e76a04b91f771d73a2fb346)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit 2f1dd10b71d29a647aa2bab4a51fecc0501d6741
+Merge: 0caf6b3 1d2af55
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 13:03:07 2015 +0200
+
+    Merge pull request #5129 from SUSE/wip-12202-firefly
+    
+    mon: ceph fails to compile with boost 1.58
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0caf6b32849bd673d83e96f0eae19bb6d7e423ae
+Merge: 3010f23 6793d50
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:51:10 2015 +0200
+
+    Merge pull request #5044 from SUSE/wip-12011-firefly
+    
+    osd: PG stuck with remapped
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3010f23fae10d37ff78b8d7d5049ccba798bbe6b
+Merge: ad58a32 fcac7ab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:45:22 2015 +0200
+
+    Merge pull request #5039 from SUSE/wip-11961-firefly
+    
+    osd: Cleanup boost optionals for boost 1.56
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad58a32292eee39905ae7bacc8fc52a52c2d08fc
+Merge: 13946e9 a62c720
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:39:34 2015 +0200
+
+    Merge pull request #5037 from SUSE/wip-11875-firefly
+    
+    tools: ceph-post-file fails on rhel7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 13946e9c18bbeb209458a3faac6e687ff380ac6e
+Merge: bde35c1 90a1a75
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:34:51 2015 +0200
+
+    Merge pull request #4771 from SUSE/wip-11759-firefly
+    
+    tools: ceph-disk: support NVMe device partitions
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bde35c11acd34c76d704b41292d167c3fc4cea53
+Merge: 9e11564 10d85e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:22:52 2015 +0200
+
+    Merge pull request #4636 from SUSE/wip-11143-firefly
+    
+    tools:  src/ceph-disk : disk zap sgdisk invocation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9e11564b36a42b605f256943c7236b91a736fceb
+Merge: 8638443 424289d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:20:50 2015 +0200
+
+    Merge pull request #4635 from SUSE/wip-7387-firefly
+    
+    common: Malformed JSON command output when non-ASCII strings are present
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8638443eac92ac80b4c57271f7350d94e445abf5
+Merge: 4c37d3d 8372f1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:17:43 2015 +0200
+
+    Merge pull request #4632 from SUSE/wip-11507-firefly
+    
+    osd: object creation by write cannot use an offset on an erasure coded pool
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4c37d3d99f660c835d2f73e8f08b6ef170c0c7b7
+Merge: 8f11ab7 9090967
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:12:36 2015 +0200
+
+    Merge pull request #4630 from SUSE/wip-10983-firefly
+    
+    tools: use a new disk as journal disk,ceph-disk prepare fail
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8f11ab7ce27cab43145a2821f3996d3103cf7870
+Merge: 64a21b7 b1ff739
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:08:26 2015 +0200
+
+    Merge pull request #4597 from xinxinsh/wip-11026-firefly
+    
+    objecter: can get stuck in redirect loop if osdmap epoch == last_force_op_resend
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 64a21b71526c51f8e787b365039de2b0cb0a285b
+Merge: 82e8359 2cb2333
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:04:39 2015 +0200
+
+    Merge pull request #4582 from SUSE/wip-11543-firefly
+    
+    tools: ceph_argparse_flag has no regular 3rd parameter
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bba50ce8f227af29d559b486274871bb3999fb24
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:28:23 2015 -0700
+
+    TestPGLog: fix noop log proc_replica_log test case
+    
+    Need to fill in log.head, olog.head.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1e5b22053381580708af29a1536f1e106c9b479c)
+
+commit fdff8ce6c996cda7b3966d20c24b20ff545e468a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:41 2015 -0700
+
+    TestPGLog: add test for 11358
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b61e5aef15d7448d24b713d13dce77697bc0af74)
+    
+    Conflicts:
+    	src/test/osd/TestPGLog.cc
+                leave out all the other tests that have been
+                added to master since firefly
+
+commit 65028b6304235ba5fa54d14805028db1a032e5a0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:29 2015 -0700
+
+    PGLog::proc_replica_log: handle split out overlapping entries
+    
+    See the comment, if the overlapping entries between olog.head
+    and log.tail were split out, last_update should be log.tail.
+    
+    Fixes: 11358
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6561e0d955c4563a8505b717de41f5bd89119f63)
+
+commit 82e8359783f056cce5f7ede2dabd0cc61d278eab
+Merge: 741f0c2 b061140
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 17 16:10:43 2015 -0700
+
+    Merge pull request #4633 from SUSE/wip-11535-firefly
+    
+    Client admin socket leaks file descriptors
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da78889d76f641dfcbb80164be32b85e880bb97f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 18 13:49:20 2015 -0700
+
+    os/chain_xattr: handle read on chnk-aligned xattr
+    
+    If we wrote an xattr that was a multiple of a chunk, we will try to read
+    the next chunk and get ENODATA.  If that happens bail out of the loop and
+    assume we've read the whole thing.
+    
+    Backport: hammer, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8614dcebf384b74b283cd352001a65fa26ba925c)
+    
+    Conflicts:
+        src/test/objectstore/chain_xattr.cc
+            added #define FILENAME
+
+commit 4e7de5b5f0e32d1183e2a0490d65e4e01490d942
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 25 14:31:03 2015 -0700
+
+    rgw: error out if frontend did not send all data
+    
+    Fixes: #11851
+    The civetweb mg_write() doesn't return error when it can't flush all data
+    to the user, it just sends the total number of bytes written. Modified the
+    client io to return total number of bytes and return an error if didn't
+    send anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit daa679c3dd3770a6d5421e2cc9a36924f4550439)
+
+commit f0d2429f764aaa756fc8002309545b02dc45c199
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Wed Dec 10 11:53:43 2014 +0100
+
+    Unconditionally chown rados log file.
+    
+    This fixes bnc#905047 (in a somewhat ad-hoc way). Sadly the log
+    file gets created from several places, so its existence does not
+    mean init-radosgw had actually run.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit 41611ea27a42d6221c418613d24dce34d988f2e6)
+
+commit 68508e28adffae64a52ea4c0046fbe8537e0d093
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 22:37:52 2015 +0200
+
+    ceph.spec.in: use _udevrulesdir to eliminate conditionals
+    
+    The conditionals governing where 50-rbd.rules is installed were not doing the
+    right thing on SUSE distros.
+    
+    Start using the %_udevrulesdir RPM macro, while taking care that it is defined
+    and set to the right value. Use it to eliminate some conditionals around other
+    udev rules files as well.
+    
+    http://tracker.ceph.com/issues/12166 Fixes: #12166
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 8aa758ee7a14e93ded1d55b4aca7a54aa47d7174)
+    
+    Conflicts:
+    	ceph.spec.in
+                master spec file has diverged significantly from firefly
+
+commit bd25136d45d32b8d419fbe60e39572038e33cc59
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 13 18:12:01 2015 +0200
+
+    ceph.spec.in: do not run fdupes, even on SLE/openSUSE
+    
+    In openSUSE there is a policy to use %fdupes in the spec file if RPMLINT
+    complains about duplicate files wasting space in the filesystem.
+    
+    However, RPMLINT is not so complaining, so drop fdupes.
+    
+    http://tracker.ceph.com/issues/12301 Fixes: #12301
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 53072b9019caf72e0313b2804ea174237ed7da33)
+
+commit 2544a6fc59a86d3f02dcd67d3c84a72d7a1100a0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 16 18:27:20 2015 +0200
+
+    ceph.spec.in: python-argparse only in Python 2.6
+    
+    argparse is a widely-used Python module for parsing command-line arguments.
+    Ceph makes heavy use of Python scripts, both in the build environment and on
+    cluster nodes and clients.
+    
+    Until Python 2.6, argparse was distributed separately from Python proper.
+    As of 2.7 it is part of the Python standard library.
+    
+    Although the python package in a given distro may or may not Provide:
+    python-argparse, this cannot be relied upon.
+    
+    Therefore, this commit puts appropriate conditionals around Requires:
+    python-argparse and BuildRequires: python-argparse. It does so for Red
+    Hat/CentOS and SUSE only, because the last Fedora version with Python 2.6
+    was Fedora 13, which is EOL.
+    
+    argparse is required by both the ceph and ceph-common packages, but since ceph
+    requires ceph-common, the argparse Requires and BuildRequires need only appear
+    once, under ceph-common.
+    
+    http://tracker.ceph.com/issues/12034 Fixes: #12034
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23171c952997d104cfa6b5700ec0bb658a8e0915)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 54264210f4ebec23b08dd6712e09aea49543b52b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 12 15:24:26 2014 -0700
+
+    ReplicatedPG::maybe_handle_cache: do not skip promote for write_ordered
+    
+    We cannot redirect a RW ordered read.
+    
+    Fixes: #9064
+    Introduced: 0ed3adc1e0a74bf9548d1d956aece11f019afee0
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 24aeca9c439474b079d23600d8d42c0af5d9ce5c)
+
+commit 7e2526784203b0f1bce08869aa7b1fda9c5eedd9
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Mon Jul 28 14:06:06 2014 +0800
+
+    osd: promotion on 2nd read for cache tiering
+    
+    http://tracker.ceph.com/issues/8674 Fixes: #8674
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit 0ed3adc1e0a74bf9548d1d956aece11f019afee0)
+
+commit 66f61cd9ae105948f653fd888812df270ff1e832
+Author: Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Thu Jul 31 15:49:44 2014 -0700
+
+    ceph_test_rados_api_tier: test promote-on-second-read behavior
+    
+    http://tracker.ceph.com/issues/8674 References: #8674
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1417eded65f40bdb2a49c8252fcfffa383a7b965)
+
+commit a8f3d6e1f1f186cbe2299566a575bf5a40500227
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 11 11:31:22 2014 -0700
+
+    osd/osd_types: be pedantic about encoding last_force_op_resend without feature bit
+    
+    The addition of the value is completely backward compatible, but if the
+    mon feature bits don't match it can cause monitor scrub noice (due to the
+    parallel OSDMap encoding).  Avoid that by only adding the new field if the
+    feature (which was added 2 patches after the encoding, see
+    3152faf79f498a723ae0fe44301ccb21b15a96ab and
+    45e79a17a932192995f8328ae9f6e8a2a6348d10.
+    
+    Fixes: #8815
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c93da05bd3dd6b73e8cfa182c6349359b042717c)
+    
+    Conflicts:
+        src/osd/osd_types.cc
+            drop encode_compat variable which is otherwise unused in this context
+
+commit dc128758bb68a50992819f759b5684ff656406ba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 15 22:50:36 2015 +0800
+
+    mon: always reply mdsbeacon
+    
+    the MDS (Beacon) is always expecting the reply for the mdsbeacon messages from
+    the lead mon, and it uses the delay as a metric for the laggy-ness of the
+    Beacon. when it comes to the MDSMonitor on a peon, it will remove the route
+    session at seeing a reply (route message) from leader, so a reply to
+    mdsbeacon will stop the peon from resending the mdsbeacon request to the
+    leader.
+    
+    if the MDSMonitor re-forwards the unreplied requests after they are
+    outdated, there are chances that the requests reflecting old and even wrong
+    state of the MDSs mislead the lead monitor. for example, the MDSs which sent
+    the outdated messages could be dead.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3555e9c328633c9e1fbc27d652c004b30535e5b)
+
+commit da720d7d870a53faeacc169796083b4eba4b9406
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:20:21 2015 -0700
+
+    mon/MDSMonitor: rename labels to a better name
+    
+    * s/ignore/reply/
+    * s/out/ignore/
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f00ecb8b3df73ce6337985bc6d43bce5143ee537)
+    
+    Conflicts:
+        src/mon/MDSMonitor.cc
+            Do not compare known daemon health with m->get_health()
+
+commit 329da091d6e8444722b312ff94c7093f81939f13
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:55:06 2015 +0800
+
+    mon: send no_reply() to peon to drop ignored mdsbeacon
+    
+    so the peon can remove the ignored mdsbeacon request from the
+    routed_requets at seeing this reply, and hence no longer resend the
+    request.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 72a37b3a8e145d8522ea67fc14ce2c5510b6852b)
+
+commit 508332f53075a99295b9edb683ed343d8f59ca4f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:22:26 2015 +0800
+
+    mon: remove unnecessary error handling
+    
+    msg.get_session() should always return a non-zero pointer in
+    Monitor.dispatch()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 16e8e2cc82a90c49cd8aa3d0e3acc4694ba659a0)
+    
+    Conflicts:
+        src/mon/Monitor.cc
+            Monitor::_ms_dispatch(Message *m) is bool in firefly
+
+commit 741f0c235228b371486574ba0c3fbf55729e03bd
+Merge: 1623e6c 0a7f2b2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 9 17:24:04 2015 -0700
+
+    Merge pull request #4631 from SUSE/wip-11069-firefly
+    
+    PGLog: split divergent priors as well
+    
+    Reviewed-by: Samuel Just
+
+commit 3223bd604117b0b302b1e3e235ff9bced8e986bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 26 09:59:36 2015 -0400
+
+    librbd: assertion failure race condition if watch disconnected
+    
+    It's possible for librbd's watch of the header object to be reset by
+    connection issues just prior to the image being removed.  This will
+    causes an assertion failure which assumes at least one watcher on the
+    image.
+    
+    Fixes: #12176
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af276de4f299960e43761904c043924cec5fef11)
+
+commit 2ee6e9a0cc6513097d6c9f9e40787a1c60c09796
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Mar 8 22:45:34 2015 +0100
+
+    rgw: send Content-Length in response for HEAD on Swift account.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 29073d84640b61d0221b2d2ab73c271d60fd13ba)
+
+commit 1b4536aa47fd6d2440fbd30b64b5b46eaa21215a
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:18:19 2015 +0100
+
+    rgw: send Content-Length in response for DELETE on Swift container.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d260a93405a7a34a899f716bd9c4e731baa9ffd0)
+
+commit 177324ced968b254458a450f8e348c96cbd10931
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:14:26 2015 +0100
+
+    rgw: send Content-Length in response for PUT on Swift container.
+    
+    Fixes: #11036
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 10c1f1aa1e09df5edf5d2f9f9b0273ddbcb384fa)
+
+commit 0a580c4a23b985ede28cdfc751eed2a368473e09
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 27 15:23:16 2015 +0100
+
+    rgw: send Content-Length in response for GET on Swift container.
+    
+    Fixes: #10971
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5a64fb5430b6b42174bf53a5910d9435043c1380)
+
+commit 84525806a057054dca295f860e0a31d6936d4985
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 12:42:52 2015 +0100
+
+    rgw: enable end_header() to handle proposal of Content-Length.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f98fd4dfda7a105d621f99c2b48eb9ab9b45d979)
+
+commit 1d2af55f8c69b4a382eb516d8f90a8316fd3bc02
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 15:21:20 2015 +0800
+
+    mon: remove unused variable
+    
+    * as a side effect, this change silences
+      http://tracker.ceph.com/issues/11576
+    
+    Fixes: #11576
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7b196a4a091c0ea258866559ba06e7ed0cc4247)
+
+commit d296120c144a62794483338fd696ca8089778e84
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Oct 20 13:29:13 2014 -0700
+
+    Objecter: resend linger ops on any interval change
+    
+    Watch/notify ops need to be resent after a pg split occurs, as well as
+    a few other circumstances that the existing objecter checks did not
+    catch.
+    
+    Refactor the check the OSD uses for this to add a version taking the
+    more basic types instead of the whole OSD map, and stash the needed
+    info when an op is sent.
+    
+    Fixes: #9806
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit cb9262abd7fd5f0a9f583bd34e4c425a049e56ce)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    	src/osdc/Objecter.cc
+                Minor differences.
+
+commit a44b7b4bd870cc72e50a49900cd81fde6d40845b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed May 7 11:02:16 2014 -0700
+
+    osd_types: factor out is_new_interval from check_new_interval
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2ee355113ed4374c664909a8cd7dc5dfdc4c9ed7)
+
+commit 3f2fb8a996820843d0287a9337d38d80c7cb616b
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Nov 5 01:00:01 2014 +0000
+
+    osd: cache pool: flush object ignoring cache min flush age when cache pool is full
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    
+    (cherry picked from commit f4ee9496ed15e7734abd659f5a353a3cde0dcb23)
+
+commit 9969ae5710bc66e1e96125f22593ea1e0ba381af
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Fri Aug 1 16:09:50 2014 +0800
+
+    osd: add local_mtime to struct object_info_t
+    
+    This fixes a bug when the time of the OSDs and clients are not
+    synchronized (especially when client is ahead of OSD), and the cache
+    tier dirty ratio reaches the threshold, the agent skips the flush work
+    because it thinks the object is too young.
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit 13b9dc708426c35436ba58388fc54f04f066f03a)
+
+commit 6793d504b5b622a7a6f69ffb30d6822167b557a3
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Oct 13 04:18:45 2014 +0000
+
+    The fix for issue 9614 was not completed, as a result, for those erasure coded PGs with one OSD down, the state was wrongly marked as active+clean+degraded. This patch makes sure the clean flag is not set for such PG.
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    
+    (cherry picked from commit 35bbb2d6002b1b435ff0b1df653cc7ab271f9258)
+
+commit fcac7ab6305dc3215fdcb46951fa9ee4cbf68ce3
+Author: William A. Kennington III <william@wkennington.com>
+Date:   Sat Sep 20 22:52:31 2014 -0700
+
+    osd: Cleanup boost optionals
+    
+    Signed-off-by: William A. Kennington III <william@wkennington.com>
+    (cherry picked from commit a53ead14c113047567177630b4906136a2109b65)
+
+commit a62c720dc783b40377dcd34d4e53aa778ac1a84b
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 1 15:10:14 2015 -0700
+
+    Makefile: install ceph-post-file keys with mode 600
+    
+    Otherwise ssh (may) prompt for a password.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 106a1c3081d02446aa3d8e13865da0c3393bae90)
+
+commit fc59ca391f639b2ef1b5c9dda202a3d7d718cf25
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:05:32 2015 -0700
+
+    ceph-post-file: improve check for a source install
+    
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit ee170eadcdcb4b54d36a7d474558484de9d917eb)
+
+commit 50bfb354d0648162f94c2659e8b915d69f27bf73
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:00:39 2015 -0700
+
+    ceph-post-file: behave when sftp doesn't take -i
+    
+    Fixes: #11836
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit b84031ed5eaace1222e14d3c4076a3ab1155da96)
+
+commit 1623e6c6517ff6958b343493fce6a8e50a52a042
+Merge: ea6c958 254676b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 19 15:57:46 2015 -0700
+
+    Merge pull request #4960 from ceph/wip-10911-firefly
+    
+    osd,mon: explicitly specify OSD features in MOSDBoot
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 254676b63b559fa0117520edb5dcbfe067a76747 (refs/remotes/jashan/wip-10911-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 18 14:53:04 2015 -0800
+
+    osd,mon: explicitly specify OSD features in MOSDBoot
+    
+    We are using the connection features to populate the features field in the
+    OSDMap, but this is the *intersection* of mon and osd features, not the
+    osd features.  Fix this by explicitly specifying the features in
+    MOSDBoot.
+    
+    Fixes: #10911
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bee91548b07c6a28314dddc50fef8b6a2677e774)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    Minor difference in the MOSDBoot constructor
+
+commit b1ff7396c8e9668c0be1a4023e0ffa292067cf64
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Thu May 7 05:49:18 2015 +0800
+
+    osdc: add epoch_t last_force_resend in Op/LingerOp.
+    
+    Using this field record the pg_poo_t::last_force_op_resend to avoid op
+    endless when osd reply with redirect.
+    
+    Fixes: #11026
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit def4fc4a)
+
+commit 90a1a7587fcb567e2cc6e9188fdd56ec1255a881
+Author: islepnev <islepnev@gmail.com>
+Date:   Fri Apr 17 22:33:01 2015 +0300
+
+    ceph-disk: support NVMe device partitions
+    
+    Linux nvme kernel module v0.9 enumerate devices as following:
+    
+    /dev/nvme0 - characted revice
+    /dev/nvme0n1 - whole block device
+    /dev/nvme0n1p1 - first partition
+    /dev/nvme0n1p2 - second partition
+    
+    http://tracker.ceph.com/issues/11612 Fixes: #11612
+    
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit 9b62cf254d02d30609793be8b1cb8a94f38891f1)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit f93eadd793f9f4fded30df5589f98ccfc0e1839f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 16:37:56 2015 -0700
+
+    mon: prevent bucket deletion when referenced by a rule
+    
+    If a rule references a bucket with 'take', prevent deletion.
+    
+    Fixes: #11602
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3d591afef90b0601572c748f13faac029d05f5a0)
+
+commit d4ef5566e6b1b2ac73925660a1d627dbb6cc7632
+Author: Anton Aksola <anton.aksola@nebula.fi>
+Date:   Fri Apr 10 13:25:21 2015 +0300
+
+    rgw: always check if token is expired
+    
+    Fixes: #11367
+    
+    Currently token expiration is only checked by the token cache. With PKI
+    tokens no expiration check is done after decoding the token. This causes
+    PKI tokens to be valid indefinitely. UUID tokens are validated by
+    keystone after cache miss so they are not affected by this bug.
+    
+    This commit adds explicit token expiration check to
+    RGWSwift::validate_keystone_token()
+    
+    Signed-off-by: Anton Aksola <anton.aksola@nebula.fi>
+    Reported-by: Riku Lehto <riku.lehto@nexetic.com>
+    (cherry picked from commit 2df069390ea3bbcfbab5022750e89f51d197cc11)
+
+commit 2d5d8da35f627840ee57a8fa071147943b890d0c
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri May 15 14:22:29 2015 +0200
+
+    rgw: Use attrs from source bucket on copy
+    
+    On copy objects, when bucket source is the same as the destination, use attrs
+    from source bucket.
+    
+    Fixes: #11639
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 1dac80df1d4a2364154ed8b404d13609936c257b)
+
+commit d78e31f005756700e15d26d187c6cef6d5d24a7a
+Author: 9seconds <nineseconds@yandex.ru>
+Date:   Wed Apr 1 09:12:06 2015 +0300
+
+    rgw: quota not respected in POST object
+    
+    Signed-off-by: Sergey Arkhipov <nineseconds@yandex.ru>
+    Backport: hammer, firefly
+    Fixes: #11323
+    (cherry picked from commit e76f84e179d2ba8bfc0dc5abf4e620fef14bc8a0)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+            discard the whitespace modification hunks that were creating
+            conflicts and ignore the conflict due to an unrelated cast
+            modification in the context
+
+commit 177ef55d61c9b08783ae746aa521d2535324d6ba
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Apr 3 13:43:31 2015 +0200
+
+    rgw: remove meta file after deleting bucket
+    The meta file is deleted only if the bucket meta data is not synced
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    
+    Fixes: #11149
+    Backport: hammer, firefly
+    (cherry picked from commit dfdc7afb59cc8e32cf8bff55faa09076c853de06)
+
+commit 9845258f184c823d1e48da1dcb97bdde97220bb6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 22 11:27:35 2015 -0400
+
+    librbd: updated cache max objects calculation
+    
+    The previous calculation was based upon the image's object size.
+    Since the cache stores smaller bufferheads, the object size is not
+    a good indicator of cache usage and was resulting in objects being
+    evicted from the cache too often.  Instead, base the max number of
+    objects on the memory load required to store the extra metadata
+    for the objects.
+    
+    Fixes: #7385
+    Backport: firefly, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0b378942c4f1b79cb65967f2d3466728ca1c8d5b)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc
+            because firefly has cct->_conf->rbd_cache_size
+            instead of cache_size
+
+commit 0a7f2b2a859aefebc995ad5c7b699eb117d6fa12
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Sun May 10 17:13:22 2015 +0200
+
+    PGLog: split divergent priors as well
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bbe231a61fc06a1a668e6b831fb8224526721f93)
+    
+    Conflicts:
+        src/osd/PGLog.h
+    
+        The cherry-picked commit did not compile as-is, because the hobject_t
+        class in firefly lacks a get_hash() method, which was added in 6de83d4.
+        To get the patch to compile, I replaced i->second.get_hash() with
+        i->second.hash.
+
+commit 424289dc4ee8fb09dd00d507b5a9d1fecc786937
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 12:01:16 2015 +0800
+
+    json_sprit: fix the FTBFS on old gcc
+    
+    Fixes: #11574
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b68b27146852f057a3373cd04b08cd6917f3eea)
+
+commit 84b00f189554fc37413c990ac4011079bb5cdb60
+Author: Tim Serong <tserong@suse.com>
+Date:   Sat May 2 01:59:53 2015 +1000
+
+    json_spirit: use utf8 intenally when parsing \uHHHH
+    
+    When the python CLI is given non-ASCII characters, it converts them to
+    \uHHHH escapes in JSON.  json_spirit parses these internally into 16 bit
+    characters, which could only work if json_spirit were built to use
+    std::wstring, which it isn't; it's using std::string, so the high byte
+    ends up being zero'd, leaving the low byte which is effectively garbage.
+    
+    This hack^H^H^H^H change makes json_spirit convert to utf8 internally
+    instead, which can be stored just fine inside a std::string.
+    
+    Note that this implementation still assumes \uHHHH escapes are four hex
+    digits, so it'll only cope with characters in the Basic Multilingual
+    Plane.  Still, that's rather a lot more characters than it could cope
+    with before ;)
+    
+    (For characters outside the BMP, Python seems to generate escapes in the
+    form \uHHHHHHHH, i.e. 8 hex digits, which the current implementation
+    doesn't expect to see)
+    
+    Fixes: #7387
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 8add15b86e7aaef41397ab8fa9e77ee7957eb607)
+    
+    Conflicts:
+    	src/test/mon/osd-pool-create.sh
+    
+            Changed $CEPH_MON to 127.0.0.1 -- the CEPH_MON was introduced after
+            firefly to allow tests to run in parallel. Back in firefly all tests
+            use the same port because 127.0.0.1 was hardcoded. We can't
+            conveniently backport all that's necessary for tests to run in
+            parallel, therefore we keep the 127.0.0.1 hardcoded.
+
+commit 10d85e1126768f6c72f7d2a301ea7ca9ba59f8e0
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Mar 17 15:41:33 2015 +0100
+
+    Fix "disk zap" sgdisk invocation
+    
+    Fixes #11143
+    
+    If the metadata on the disk is truly invalid, sgdisk would fail to zero
+    it in one go, because --mbrtogpt apparently tried to operate on the
+    metadata it read before executing --zap-all.
+    
+    Splitting this up into two separate invocations to first zap everything
+    and then clear it properly fixes this issue.
+    
+    Based on patch by Lars Marowsky-Bree <lmb@suse.com> in ceph-deploy.
+    Created by Vincent Untz <vuntz@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b)
+
+commit b06114088a354f9dbcfc9182643733399c31f2a8
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Fri May 8 11:54:06 2015 -0400
+
+    common/admin_socket: close socket descriptor in destructor
+    
+    Long-running processes that do not reuse a single client connection will
+    see accumulating file descriptors as a result of not closing the
+    listening socket.  In this case, eventually the system will reach
+    file-max and subsequent connections will fail.
+    
+    Fixes: #11535
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit 88fabb1ee6cd3c7d211ca46919055195e32245db)
+
+commit 8372f1d53b847224bf4c61894d18851e84ccd357
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 15:54:07 2015 +0800
+
+    osd: refuse to write a new erasure coded object with an offset > 0
+    
+    Even if the offset is properly aligned.
+    
+    http://tracker.ceph.com/issues/11507 Fixes: #11507
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a4f1256c214ee0e7ebb91ac4ea8655f5d9642dc8)
+
+commit 909096713dcef5dc4fdbb22a9971609554688e23
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 8 15:15:35 2015 +0100
+
+    ceph-disk: more robust parted output parser
+    
+    In some cases, depending on the implementation or the operating system,
+    
+       parted --machine -- /dev/sdh print
+    
+    may contain empty lines. The current parsing code is fragile and highly
+    depends on output details. Replace it with code that basically does the
+    same sanity checks (output not empty, existence of units, existence of
+    the dev entry) but handles the entire output instead of checking line by
+    line.
+    
+    http://tracker.ceph.com/issues/10983 Fixes: #10983
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f5acf6bb6a342b05897605703d7d9cb7c09714fd)
+
+commit 8c573c8826096d90dc7dfb9fd0126b9983bc15eb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 30 14:15:47 2014 -0700
+
+    rgw: swift GET / HEAD object returns X-Timestamp field
+    
+    Fixes: #8911
+    Backport: giant, firefly, dumpling
+    Swift clients expect X-Timestamp header, dump it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5b41d80b7fb9ed96c26801fc42c044191bb18d84)
+
+commit 9614e4dc5be745dab367d17f29a07377e348b2ca
+Author: Daniel J. Hofmann <daniel@trvx.org>
+Date:   Mon May 5 16:06:45 2014 +0200
+
+    Variable length array of std::strings (not legal in C++) changed to std::vector<std::string>
+    
+    Signed-off-by: Daniel J. Hofmann <daniel@trvx.org>
+    (cherry picked from commit 08a4e8889783673436972a979c2fc780466893c8)
+
+commit 2cb2333ecdf2106925d74e024697f73ad947f378
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Thu Jul 3 00:15:03 2014 +0200
+
+    ceph_argparse_flag has no regular 3rd parameter.
+    
+    With clang warning: 'va_start' has undefined behavior with reference
+    types, noticing copy-paste mistake on ceph_argparse_flag.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit 0c7c7223a2beb4ffb953d89d316f87d350677063)
+
+commit f6022639758ec13b9a25b03cd831882db0b517b3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Dec 17 17:12:43 2014 -0800
+
+    rgw: use correct objv_tracker for bucket instance
+    
+    When trying to create a bucket that already existed, use the
+    objv_tracker of the newly created instance, and not of the original
+    bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe158ecc25feefcea8aea4133118e4a84900a8ec)
diff --git a/doc/changelog/v0.80.2.txt b/doc/changelog/v0.80.2.txt
new file mode 100644
index 000000000..1b4a857d3
--- /dev/null
+++ b/doc/changelog/v0.80.2.txt
@@ -0,0 +1,1187 @@
+commit 198236d68deceb16b6b590d807786a140209bfea (tag: refs/tags/v0.80.2, refs/remotes/gh/last)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Jul 9 19:32:26 2014 +0000
+
+    0.80.2
+
+commit 0b6bd2545925b5e8a80d41de1fda13ffe9d30e2b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 9 12:04:49 2014 -0700
+
+    Revert "rgw: if extra data pool name is empty, use data pool name instead"
+    
+    This breaks multipart uploads on firefly, though not on master.
+    Revert for now so we can release 0.80.2.
+    
+    This reverts commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1.
+
+commit 9426c4faa9ca327f18f20a57ba772b08d6f069e7
+Merge: b81a277 4733178
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 10:06:01 2014 -0700
+
+    Merge pull request #2083 from ceph/wip-8727.firefly
+    
+    messages: MForward: fix compat version
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4733178c3a8f57c54196f176b20154415783dba0
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 16:46:27 2014 +0100
+
+    messages: MForward: fix compat version
+    
+    A while ago we bumped the head version and reset the compat version to 0.
+    Doing this so happens to make the messenger assume that the message does
+    not support the compat versioning and sets the compat version to the head
+    version -- thus making compat = 2 when it should have been 1.
+    
+    The nasty side-effect of this is that upgrading from emperor to firefly
+    will have emperor-leaders being unable to decode forwarded messages from
+    firefly-peons.
+    
+    Fixes: #8727
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c92feebb81f90344a085f8256f0ed03cdc2f14e1)
+
+commit b81a2777aea636ffe544e737cd8e17dc892ba328
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:11:24 2014 -0700
+
+    ceph.spec.in: add bash completion file for radosgw-admin
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit b70096307130bcbac176704493a63c5d039d3edc)
+
+commit 4649d7e90ffec89b452aef69cf833522bdb81f88
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:10:55 2014 -0700
+
+    ceph.spec.in: rhel7-related changes:
+    
+    udev rules: /lib -> /usr/lib
+    /sbin binaries move to /usr/sbin or %{_sbindir}
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 235e4c7de8f8efe491edefbdde8e5da4dfc44034)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 38a9566e70d3a2de80c7547fd4e84c8372639f80
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:08:44 2014 -0700
+
+    Fix/add missing dependencies:
+    
+    - rbd-fuse depends on librados2/librbd1
+    - ceph-devel depends on specific releases of libs and libcephfs_jni1
+    - librbd1 depends on librados2
+    - python-ceph does not depend on libcephfs1
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 7cf81322391b629b241da90181800ca1f138ce78)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 733c89dc6af89321ee58ad78b30bc6a0c4e78291
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:05:00 2014 -0700
+
+    ceph.spec.in: whitespace fixes
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit ec8af52a5ede78511423a1455a496d46d580c644)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 21b03bc96ab1a0877235f1e3e4a96899cba2b4fa
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:04:10 2014 -0700
+
+    ceph.spec.in: split out ceph-common as in Debian
+    
+    Move files, postun scriptlet, and add dependencies on ceph-common
+    where appropriate
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit e131b9d5a5e90e87d8a8346cb96cb5a26135c144)
+
+commit dbee797d1344aec47fedc8d3fdd6c9460e0ffa16
+Merge: 6d6039a 9e81fc1
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 7 09:40:51 2014 -0700
+
+    Merge pull request #2057 from ceph/wip-8593-firefly
+    
+    mon: backport health check improvements
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 6d6039a5a56743c006a0d081157cb6ee9e3b7af6
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jul 3 16:30:49 2014 -0700
+
+    mon: check changes to the whole CRUSH map and to tunables against cluster features
+    
+    When we change the tunables, or set a new CRUSH map, we need to make sure it's
+    supported by all the monitors and OSDs currently participating in the cluster.
+    
+    Fixes: #8738
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 54af8104816ccc76aad251aa47a6efc1bf879e7d)
+
+commit 0a8d194f2c44a73c31ae8c52fabb430d66d4a1b1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jul 3 15:24:30 2014 -0700
+
+    OSDMonitor: fix quorum_features comparison in check_cluster_features
+    
+    We need to see if there's a feature which is not in the quorum_features,
+    not if there are no features in common!
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 2280c0eaa9f3f733062085a5e7f7dc23c3bbe291)
+
+commit a49c61222f8014db533d8f505ec44d10dc5478e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 12:57:36 2014 -0700
+
+    ceph_test_rados_api_tier: disable LibRadosTierECPP::HitSetWrite
+    
+    Disable this test until hitget-get reliably works on EC pools (currently
+    it does not, and this test usually passes only because we get the in-memory
+    HitSet).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c5b5ed672c039cc4fa04772e065a59d3b1df803a)
+
+commit 828ce4055c8580e42175159c02a9dfa76bc414ff
+Merge: b9b498e b1a4a7c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 4 21:18:32 2014 -0700
+
+    Merge pull request #2073 from ceph/wip-rgw-firefly-2
+    
+    rgw: more firefly backports
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit b1a4a7cb91e164d1f8af8ce9319e3b3c1949858d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 11 16:50:41 2014 -0700
+
+    rgw: set a default data extra pool name
+    
+    Fixes: #8585
+    Have a default name for the data extra pool, otherwise it would be empty
+    which means that it'd default to the data pool name (which is a problem
+    with ec backends).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b2542f8e880e45bb6d052f13eb3ef6a7c61e4cd4)
+
+commit b9b498e6a7794205af8aa99ffe5266e478c0e711
+Merge: f285266 053c261
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 3 07:00:52 2014 -0700
+
+    Merge pull request #2028 from ceph/wip-rgw-firefly
+    
+    rgw: a couple backports for firefly
+    
+    Passed the rgw suite, modulo a python bootstrap issue.
+    
+    sage-2014-07-01_09:50:22-rgw-wip-rgw-firefly-testing-basic-plana/337393
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 9f06e2c16e409a9a8284af9964f0cfff6e222164
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 12:55:45 2014 -0700
+
+    rgw: fix uninit ofs in RGWObjManifect::obj_iterator
+    
+    Valgrind picked this up:
+    
+      <kind>UninitCondition</kind>
+      <what>Conditional jump or move depends on uninitialised value(s)</what>
+      <stack>
+        <frame>
+          <ip>0x5145B8</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>RGWObjManifest::obj_iterator::seek(unsigned long)</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_rados.cc</file>
+          <line>562</line>
+        </frame>
+        <frame>
+          <ip>0x5672A4</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>list_multipart_parts(RGWRados*, req_state*, std::string const&amp;, std::string&amp;, int, int, std::map&lt;unsigned int, RGWUploadPartInfo, std::less&lt;unsigned int&gt;, std::allocator&lt;std::pair&lt;unsigned int const, RGWUploadPartInfo&gt; &gt; &gt;&amp;, int*, bool*, bool)</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_rados.h</file>
+          <line>217</line>
+        </frame>
+        <frame>
+          <ip>0x5688EE</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>RGWListMultipart::execute()</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_op.cc</file>
+          <line>2956</line>
+        </frame>
+    ...
+    
+    Fixes: #8699
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e0d3b78b6af523ae77b0cee385cd40f67f7a0ab6)
+
+commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 30 14:47:25 2014 -0700
+
+    rgw: if extra data pool name is empty, use data pool name instead
+    
+    Fixes: #8311
+    An empty pool name could be used as the extra data pool.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b300318113b162522759d4794b1cfa3d1d9398e4)
+
+commit cbad09e1907ceac4feb069f9cf257e580f36da91
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 15:35:20 2014 -0700
+
+    rgw: extend manifest to avoid old style manifest
+    
+    In case we hit issue #8269 we'd like to avoid creating an old style
+    manifest. Since we need to have parts that use different prefix we add a
+    new rule param that overrides the manifest prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 545d8ad1d2dfe53b967ab1bd17dfeb1780abbb5e)
+
+commit ea68b9372319fd0bab40856db26528d36359102e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 2 17:06:05 2014 -0700
+
+    rgw: don't allow multiple writers to same multiobject part
+    
+    Fixes: #8269
+    Backport: firefly, dumpling
+    
+    A client might need to retry a multipart part write. The original thread
+    might race with the new one, trying to clean up after it, clobbering the
+    part's data.
+    The fix is to detect whether an original part already existed, and if so
+    use a different part name for it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bd8e026f88b812cc70caf6232c247844df5d99bf)
+
+commit f285266a320bf22df8b09a669d1c1fd400b05765
+Merge: d43e711 6a0dd1b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 1 16:16:09 2014 -0700
+
+    Merge pull request #2056 from ceph/wip-8010
+    
+    Revert "mon: OSDMonitor: Refuse to delete CephFS pools"
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9e81fc180a3999ebec95949a2cb31fea970573fb
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 17:00:51 2014 -0700
+
+    mon: ensure HealthService warning(s) include a summary
+    
+    The low disk space check would change our status to HEALTH_WARN and include
+    a detail message, but no summary.  We need both.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3ed7f2dd4345633ff34017b201082f4c261ef387)
+
+commit e96637c2e33ec092f28af0ce96bfe2e4d7748ff8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:58:14 2014 -0700
+
+    mon: refactor check_health()
+    
+    Refactor the get_health() methods to always take both a summary and detail.
+    Eliminate the return value and pull that directly from the summary, as we
+    already do with the PaxosServices.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82e47db8073b622183a5e33f6e0b999a3a144804)
+
+commit 28584200df36dda5d434d4d7f58585933905de30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:40:05 2014 -0700
+
+    mon: fix typos, punctuation for mon disk space warning(s)
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98883f6308ce72f69a71feab29ef00e13f319cdb)
+    
+    Conflicts:
+    
+    	src/mon/DataHealthService.cc
+
+commit 4f0666085e2fd553ccc05cef8510aae820849aca
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:27:05 2014 -0700
+
+    mon/OSDMonitor: make down osd count sensible
+    
+    We currently log something like
+    
+     1/10 in osds are down
+    
+    in the health warning when there are down OSDs, but this is based on a
+    comparison of the number of up vs the number of in osds, and makes no sense
+    when there are up osds that are not in.
+    
+    Instead, count only the number OSDs that are both down and in (relative to
+    the total number of OSDs in) and warn about that.  This means that, if a
+    disk fails, and we mark it out, and the cluster fully repairs itself, it
+    will go back to a HEALTH_OK state.
+    
+    I think that is a good thing, and certainly preferable to the current
+    nonsense.  If we want to distinguish between down+out OSDs that were failed
+    vs those that have been "acknowledged" by an admin to be dead, we will
+    need to add some additional state (possibly reusing the AUTOOUT flag?), but
+    that will require more discussion.
+    
+    Backport: firefly (maybe)
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 55a97787088b79356c678a909b2410b3924e7f5b)
+
+commit 6a0dd1be909b423f52e2c7f11531b278dc8838ae
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jul 1 22:04:25 2014 +0100
+
+    Revert "mon: OSDMonitor: Refuse to delete CephFS pools"
+    
+    This reverts commit c7d265a9b276aad5caf9b35f3ce8bc7bdd169f0f.
+    
+    Because this pre-dates the `fs rm` command, this change was
+    preventing firefly users from ever deleting their filesystem pools.
+    
+    Fixes: #8010
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit 053c261e5bfcb4d39770048775515eb6a667f987
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 4 09:24:33 2014 -0700
+
+    rgw: set meta object in extra flag when initializing it
+    
+    As part of the fix for 8452 we moved the meta object initialization.
+    Missed moving the extra flag initialization that is needed. This breaks
+    setups where there's a separate extra pool (needed in ec backends).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 703d0eb9bffdf6c553b659728a3a07f068fb15ee)
+
+commit d43e7113dd501aea1db33fdae30d56e96e9c3897
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 07:05:04 2014 -0700
+
+    qa/workunits/suites/fsx.sh: don't use zero range
+    
+    Zero range is not supported by cephfs.
+    
+    Fixes: #8542
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2dec8a810060f65d022c06e82090b4aa5ccec0cb)
+
+commit ee0d322c0b1ebaba2f225b8bed9af2948fdcf4c3
+Merge: 3ceafcf a7119c3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 30 17:01:03 2014 +0200
+
+    Merge pull request #1991 from dachary/wip-8307-erasure-code-profile-implicit-creation
+    
+    erasure code profile implicit creation (firefly backport)
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit a7119c31e9a1b415f56426d2b4cd362357ad8b17
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 11 22:44:57 2014 +0200
+
+    erasure-code: pool create must not create profiles
+    
+    If a non existent profile is provided as an argument to osd pool create,
+    it must exit on error and not create the profile as a side effect.
+    
+    http://tracker.ceph.com/issues/8307 refs: #8307
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ff2eb234e63cd281b40405cb3397da5babda943f)
+
+commit a67c1c1c51b111f40819643530e07f8026227caa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 11 22:47:49 2014 +0200
+
+    erasure-code: OSDMonitor::get_erasure_code is a const
+    
+    If it is not, the non const version of OSDMap::get_erasure_code_profile
+    is called and a profile is created as a side effect, which is not
+    intended.
+    
+    http://tracker.ceph.com/issues/8307 refs: #8307
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 3c638111a4943758b6089c63a42aabbf281ac257)
+
+commit 3ceafcf3576efaef58dffbe70654baf1db4f3a1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue May 27 10:06:46 2014 +0200
+
+    mon: fix set cache_target_full_ratio
+    
+    It was a noop because it was incorrectly using the variable n. Add a
+    test to protect against regression.
+    
+    http://tracker.ceph.com/issues/8440 Fixes: #8440
+    
+    Reported-by: Geoffrey Hartz <hartz.geoffrey@gmail.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c2225f874dcf37222d831b65b5a319d598d2fcd9)
+
+commit 5f66a2c99fdef2c8d609126740ca790ce10e67a4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 16 11:48:24 2014 -0700
+
+    rgw: allocate enough space for bucket instance id
+    
+    Fixes: #8608
+    Backport: dumpling, firefly
+    Bucket instance id is a concatenation of zone name, rados instance id,
+    and a running counter. We need to allocate enough space to account zone
+    name length.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d2e86a66ca55685e04ffbfaa58452af59f381277)
+
+commit 84d7971c8e50e7d35d7a5fac9353e687f168a1a1
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Jun 20 11:14:25 2014 -0400
+
+    log the command that is being run with subprocess
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit e189a668285f9ab73116bc19f9df1cc515473541)
+
+commit 291d448478360a1be0a9826ada58c94ecd643ad0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 30 16:17:47 2014 -0700
+
+    rgw: fetch object attrs on multipart completion
+    
+    Fixes: #8452
+    Backport: firefly
+    This fixes a regression following a code cleanup.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 70afaaabf979d4cc1a14dbc2b772224cbafcac2f)
+
+commit 2fd808fe93fe0cbd8754eb81d281c83254411039
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 22 18:27:58 2014 -0700
+
+    rgw: check appropriate entity permission on put_metadata
+    
+    Fixes: #8428
+    Backport: firefly
+    
+    Cannot use verify_object_permission() to test acls, as the operation
+    here might either be on object or on bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2ceb13af34bc38b418bb88d81131d770a71159bf)
+
+commit d1afeb38b522baae1b15a7a3548002c830bb7567
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Thu Jun 5 10:08:42 2014 +0400
+
+    XfsFileStoreBackend: call ioctl(XFS_IOC_FSSETXATTR) less often
+    
+    No need to call ioctl(XFS_IOC_FSSETXATTR) if extsize is already set to
+    the value we want or if any extents are allocated - XFS will refuse to
+    change extsize in that's the case.
+    
+    Fixes: #8241
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit bc3b30ed09b8f3eb86b61e3a05ccacfd928faa95)
+
+commit 7b8e5c99a4a40ae788ad29e36b0d714f529b12eb
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:25:19 2014 +0100
+
+    mon: Fix default replicated pool ruleset choice
+    
+    Specifically, in the case where the configured
+    default ruleset is CEPH_DEFAULT_CRUSH_REPLICATED_RULESET,
+    instead of assuming ruleset 0 exists, choose the lowest
+    numbered ruleset.
+    
+    In the case where an explicit ruleset is passed to
+    OSDMonitor::prepare_pool_crush_ruleset, verify
+    that it really exists.
+    
+    The idea is to eliminate cases where a pool could
+    exist with its crush ruleset set to something
+    other than a value ruleset ID.
+    
+    Fixes: #8373
+    
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit 1d9e4ac2e2bedfd40ee2d91a4a6098150af9b5df)
+    
+    Conflicts:
+    
+    	src/crush/CrushWrapper.h
+
+commit 25a4a874c8e53cd644d6705478fbea4fdb232c4d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 23 14:58:54 2014 -0700
+
+    rgw: calc md5 and compare if user provided appropriate header
+    
+    Fixes: #8436
+    Backport: firefly
+    
+    This was broken in ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782. The fix
+    resurrects and old check that was dropped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9c56c86bdac6bcb8e76c3f04e7d393e4eaadd721)
+
+commit ad18a45a71be34899d9400c8bdef39d0548f9aaf
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:07:12 2014 -0700
+
+    rgw: calculate user manifest
+    
+    Fixes: #8169
+    Backport: firefly
+    We didn't calculate the user manifest's object etag at all. The etag
+    needs to be the md5 of the contantenation of all the parts' etags.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782)
+
+commit 6aa7f7ee4e082819decc48da1f0eaffc4e616302
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 3 11:45:20 2014 -0700
+
+    librados: simplify/fix rados_pool_list bounds checks
+    
+    We were not breaking out of the loop when we filled up the buffer unless
+    we happened to do so on a pool name boundary.  This means that len would
+    roll over (it was unsigned).  In my case, I was not able to reproduce
+    anything particularly bad since (I think) the strncpy was interpreting the
+    large unsigned value as signed, but in any case this fixes it, simplifies
+    the arithmetic, and adds a simple test.
+    
+    - use a single 'rl' value for the amount of buffer space we want to
+      consume
+    - use this to check that there is room and also as the strncat length
+    - rely on the initial memset to ensure that the trailing 0 is in place.
+    
+    Fixes: #8447
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3ec32a6bb11d92e36a0e6381b40ce2fd1fbb016a)
+
+commit 2ba03e08f1bce0a8af87014583d635eb5c0aa376
+Merge: 50dc246 3ad451e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 25 12:42:11 2014 -0700
+
+    Merge pull request #1982 from accelazh/firefly-fix-issue-8256
+    
+    Make <poolname> in "ceph osd tier --help" clearer (fix issue 8256).
+    
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+
+commit 50dc2462ab96c3457b08c50b4cc07676b50dbf72
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 3 16:14:15 2014 -0700
+
+    OSD::calc_priors_during: handle CRUSH_ITEM_NONE correctly
+    
+    Fixes: #8507
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0bd6f6797c69af9aff851033c57c42121671c684)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit a53bcf2357c7f351ad520039774d895b8c27987f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 3 16:11:32 2014 -0700
+
+    OSD::calc_priors_during: fix confusing for loop bracing (cosmetic)
+    
+    Confusing lack of braces is confusing.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d76936b92300be5cc403fd5a36616a2424c7877d)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 178e6f0f422c9f1b8a51d0e9c364773397a4555f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 24 10:11:21 2014 -0700
+
+    rados.cc: fix pool alignment check
+    
+    Only check pool alignment if io_ctx is initialized.
+    
+    Introduced in 304b08a23a3db57010078046955a786fe3589ef8
+    Fixes: #8652
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    
+    (cherry picked from commit d7350a3741bf4cdb270c6361e68090fe280cf36d)
+    
+    Conflicts:
+    	src/tools/rados/rados.cc
+
+commit a9f97f8cae5b9d0245e0b0842c5c581cdacbb2da
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 13:33:14 2014 -0700
+
+    osd: fix filestore perf stats update
+    
+    Update the struct we are about to send, not the (unlocked!) one we will
+    send the next time around.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4afffb4a10a0bbf7f2018ef3ed6b167c7921e46b)
+
+commit b758fd5f0ebcccdb1641a852d40ee880bad01107
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Apr 24 15:34:24 2014 -0700
+
+    FileStore: set XATTR_NO_SPILL_OUT when creating new files.
+    
+    Fixes: #8205
+    Backport: firefly
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e3b995e1567f3ccc6d00ae27ab2aa99ca157228a)
+
+commit d6f302314fc3580dfc746c9d82bd2f7f0bc87d42
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sat Jun 7 14:32:23 2014 +0800
+
+    FileStore: make _clone() copy spill out marker
+    
+    Previously we were not doing so, and that resulted in unpredictable loss
+    of xattrs from the client's perspective.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 239476a92849159d2a8966d90ca055c116bee91e)
+
+commit bb2ed23d8129c8db628475d744656b9b43f839c0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 18 17:01:54 2014 +0200
+
+    erasure-code: verify that rados put enforces alignment
+    
+    http://tracker.ceph.com/issues/8622 refs: #8622
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b46c4056014dd6de5e3bd736f2c41f096ea708b4)
+
+commit dece65064d949b5afcc359cd408615883b5e002a
+Author: Lluis Pamies-Juarez <lluis@pamies.cat>
+Date:   Wed Jun 18 10:00:09 2014 -0700
+
+    enforce rados put aligment
+    
+    Signed-off-by: Lluis Pamies-Juarez <lluis.pamies-juarez@hgst.com>
+    (cherry picked from commit 304b08a23a3db57010078046955a786fe3589ef8)
+
+commit 203318b8eb3753723277ec7a459239c81097f090
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 6 13:31:29 2014 -0700
+
+    osd/OSDMap: do not require ERASURE_CODE feature of clients
+    
+    Just because an EC pool exists in the cluster does not mean tha tthe client
+    has to support the feature:
+    
+    1) The way client IO is initiated is no different for EC pools than for
+       replicated pools.
+    2) People may add an EC pool to an existing cluster with old clients and
+       locking those old clients out is very rude when they are not using the
+       new pool.
+    3) The only direct client user of EC pools right now is rgw, and the new
+       versions already need to support various other features like CRUSH_V2
+       in order to work.  These features are present in new kernels.
+    
+    Fixes: #8556
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3fe1699f9620280d0070cfe6f01cfeb2332e7470)
+
+commit 6340b14b9c9a9c786b5448f2ea2a5556055a02e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 12 16:44:53 2014 -0700
+
+    osd/OSDMap: make get_features() take an entity type
+    
+    Make the helper that returns what features are required of the OSDMap take
+    an entity type argument, as the required features may vary between
+    components in the cluster.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 250677c965365edf3ecd24ef73700fc6d992ea42)
+
+commit 400fa88f6cf8b1e5b4b15d9e36b1de8c3d1c3970
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed May 21 18:12:22 2014 +0800
+
+    Avoid extra check for clean object
+    
+    We needn't to check clean object via buffer state, skip the clean object.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit f51e33bd9c5a8e1cfc7065b30785696dc45918bc)
+
+commit 3ad451ef57e2ab3fc644c94d44c2de7a17cced45
+Author: Accela Zhao <accelazh@gmail.com>
+Date:   Wed Jun 18 17:17:03 2014 +0800
+
+    Make <poolname> in "ceph osd tier --help" clearer.
+    
+    The ceph osd tier --help info on the left always says <poolname>.
+    It is unclear which one to put <tierpool> on the right.
+    
+    $ceph osd tier --help
+    osd tier add <poolname> <poolname> {--   add the tier <tierpool> to base pool
+    force-nonempty}                          <pool>
+    osd tier add-cache <poolname>            add a cache <tierpool> of size <size>
+    <poolname> <int[0-]>                     to existing pool <pool>
+    ...
+    
+    This patch modifies description on the right to tell which <poolname>:
+    
+    osd tier add <poolname> <poolname> {--   add the tier <tierpool> (the second
+     force-nonempty}                          one) to base pool <pool> (the first
+                                               one)
+    ...
+    
+    Fix: http://tracker.ceph.com/issues/8256
+    
+    Signed-off-by: Yilong Zhao <accelazh@gmail.com>
+
+commit f290a021bc8ec31c18b91cb37323345692f2f198
+Merge: 5722419 d02d46e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 09:25:32 2014 -0700
+
+    Merge pull request #1962 from dachary/wip-8599-ruleset-firefly
+    
+    mon: pool set <pool> crush_ruleset must not use rule_exists (firefly)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit d02d46e25080d5f7bb8ddd4874d9019a078b816b
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:50:18 2014 +0100
+
+    mon: pool set <pool> crush_ruleset must not use rule_exists
+    
+    Implement CrushWrapper::ruleset_exists that iterates over the existing
+    rulesets to find the one matching the ruleset argument.
+    
+    ceph osd pool set <pool> crush_ruleset must not use
+    CrushWrapper::rule_exists, which checks for a *rule* existing, whereas
+    the value being set is a *ruleset*. (cherry picked from commit
+    fb504baed98d57dca8ec141bcc3fd021f99d82b0)
+    
+    A test via ceph osd pool set data crush_ruleset verifies the ruleset
+    argument is accepted.
+    
+    http://tracker.ceph.com/issues/8599 fixes: #8599
+    
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 572241942706e2f845a147f19717bf6c487f23f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 20:18:49 2014 -0700
+
+    init-ceph: continue after failure doing osd data mount
+    
+    If we are starting many daemons and hit an error, we normally note it and
+    move on.  Do the same when doing the pre-mount step.
+    
+    Fixes: #8554
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6a7e20147cc39ed4689809ca7d674d3d408f2a17)
+
+commit e5a291ef69316519b105b57193753f0bf8ac8e62
+Author: Steve Taylor <steve@DES-U1404-STETAY.stc.local>
+Date:   Tue Jun 10 12:42:55 2014 -0600
+
+    Fix for bug #6700
+    
+    When preparing OSD disks with colocated journals, the intialization process
+    fails when using dmcrypt. The kernel fails to re-read the partition table after
+    the storage partition is created because the journal partition is already in use
+    by dmcrypt. This fix unmaps the journal partition from dmcrypt and allows the
+    partition table to be read.
+    
+    Signed-off-by: Stephen F Taylor <steveftaylor@gmail.com>
+    (cherry picked from commit 673394702b725ff3f26d13b54d909208daa56d89)
+
+commit 34d9b2bc659c4107b30e1ebe7a55315512a0222a
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:29:20 2014 -0700
+
+    doc: Added Disable requiretty commentary.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 54a89bab15a945d47b3f44448910521a2ebf41c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 16 16:56:33 2014 -0700
+
+    ReplicatedPG::start_flush: fix clone deletion case
+    
+    dsnapc.snaps will be non-empty most of the time if there
+    have been snaps before prev_snapc.  What we really want to
+    know is whether there are any snaps between oi.snaps.back()
+    and prev_snapc.
+    
+    Fixes: 8334
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 29f615b7ac9e92f77cdef9927070727fee9d5e33)
+
+commit 308e2ef5c107bf04d3c5623f3c6bdffa10cf8c21
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon May 12 15:08:07 2014 -0700
+
+    ReplicatedPG::start_flush: send delete even if there are no snaps
+    
+    Even if all snaps for the clone have been removed, we still have to
+    send the delete to ensure that when the object is recreated the
+    new snaps aren't included in the wrong clone.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2ec2182745fa7c22526a7cf3dedb25bc314c9db4)
+
+commit 8d04a7ece364c0a95b087990d6365401819b3efa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 15 20:53:27 2014 -0700
+
+    HashIndex: in cleanup, interpret missing dir as completed merge
+    
+    If we stop between unlinking the empty subdir and removing the root
+    merge marker, we get ENOENT on the get_info.  That's actually fine.
+    
+    Backport: firefly
+    Fixes: 8332
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5ff95dbdd2dbb533d344f37fea722ca4f140e670)
+
+commit b2bf8cb21761d88557fb2d5c0935c4a3678ee01f
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed May 28 11:48:12 2014 -0400
+
+    add backport of collections.Counter for python2.6
+    
+    Using Raymond Hettinger's MIT backport
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 23b75b550507438c79b3aa75e06721e5f7b134a4)
+
+commit 0b00443eec4c372f2ef245f85dc7c1f497ba3778
+Author: Ailing <ailzhang@users.noreply.github.com>
+Date:   Wed May 28 12:37:48 2014 -0700
+
+    rest-api: key missing for per "rx" and "rwx"
+    
+    commit 85a1cf31e6 and db266a3fb2 introduce new per "rx" and "rwx", but key missing for per "rx" and "rwx" in permmap
+    
+    Signed-off-by: Ailing Zhang <ailzhang@cisco.com>
+    (cherry picked from commit 0b5a67410793ec28cac47e6e44cbbcf5684d77e7)
+
+commit a898ee56e01f971c12c0f7bebc22e59f850f27c0
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed May 21 21:41:23 2014 -0700
+
+    cephfs-java: build against older jni headers
+    
+    Older versions of the JNI interface expected non-const parameters
+    to their memory move functions. It's unpleasant, but won't actually
+    change the memory in question, to do a cast_const in order to satisfy
+    those older headers. (And even if it *did* modify the memory, that
+    would be okay given our single user.)
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 4d4b77e5b6b923507ec4a0ad9d5c7018e4542a3c)
+
+commit 8a12a940817d957063c21ac71a6128f991801573
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Fri May 16 19:03:13 2014 +0400
+
+    OSDMonitor: set next commit in mon primary-affinity reply
+    
+    Commit 8c5c55c8b47e ("mon: set next commit in mon command replies")
+    fixed MMonCommand replies to include the right version, but the
+    primary-affinity handler was authored before that.  Fix it.
+    
+    Backport: firefly
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit a78b14ec1769ef37bef82bfda6faabb581b4cd7d)
+
+commit 6ba57da94a6e00040677d6e43abf10a36ed9a931
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 14:08:44 2014 +1000
+
+    prioritise use of `javac` executable (gcj provides it through alternatives).
+    
+     On Debian this fixes FTBFS when gcj-jdk and openjdk-7-jdk are installed at
+     the same time because build system will use default `javac` executable
+     provided by current JDK through `update-alternatives` instead of blindly
+     calling GCJ when it is present.
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 8b682d167e4535df582f1c77542e2b1ea0981228)
+
+commit 9e335a494c6eff209cff64bd663b5aa2a7bbc242
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 14:02:53 2014 +1000
+
+    pass '-classpath' option (gcj/javah ignores CLASSPATH environment variable).
+    
+     This should not affect OpenJDK which understands '-classpath' as well.
+    
+     With gcj-jdk we still get FTBFS later:
+    
+     ~~~~
+        java/native/libcephfs_jni.cc:2878:55: error: invalid conversion from 'const jbyte* {aka const signed char*}' to 'jbyte* {aka signed char*}' [-fpermissive]
+                     reinterpret_cast<const jbyte*>(rawAddress));
+                                                               ^
+        In file included from java/native/libcephfs_jni.cc:27:0:
+        /usr/lib/gcc/x86_64-linux-gnu/4.8/include/jni.h:1471:8: error:   initializing argument 4 of 'void _Jv_JNIEnv::SetByteArrayRegion(jbyteArray, jsize, jsize, jbyte*)' [-fpermissive]
+           void SetByteArrayRegion (jbyteArray val0, jsize val1, jsize val2, jbyte * val3)
+                ^
+        make[5] *** [java/native/libcephfs_jni_la-libcephfs_jni.lo] Error 1
+     ~~~~
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 89fe0353582bde7e2fbf32f1626d430a20002dd0)
+
+commit fcd30500790eac0266ffb70d83ee17b23aeaf209
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 13:57:20 2014 +1000
+
+    look for "jni.h" in gcj-jdk path, needed to find "jni.h" with gcj-jdk_4.9.0
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 0f4120c0115e7977ae7c03458addcc2b2916db07)
+
+commit de43f6d2d813d2c7e5cf915c1b9037fe662d6afc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 08:52:51 2014 -0700
+
+    ceph-disk: partprobe before settle when preparing dev
+    
+    Two users have reported this fixes a problem with using --dmcrypt.
+    
+    Fixes: #6966
+    Tested-by: Eric Eastman <eric0e@aol.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f196265f049d432e399197a3af3f90d2e916275)
+
+commit fe1fbfb7a43d6590efce99711e47f986249f0956
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue May 13 13:15:28 2014 -0700
+
+    test: fix some templates to match new output code
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 00225d739cefa1415524a3de45fb9a5a2db53018)
+
+commit 5c8afaa8861345efbcc5488e0336327a8a38d3bc
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu May 15 16:50:43 2014 -0700
+
+    OSD: fix an osdmap_subscribe interface misuse
+    
+    When calling osdmap_subscribe, you have to pass an epoch newer than the
+    current map's. _maybe_boot() was not doing this correctly -- we would
+    fail a check for being *in* the monitor's existing map range, and then
+    pass along the map prior to the monitor's range. But if we were exactly
+    one behind, that value would be our current epoch, and the request would
+    get dropped. So instead, make sure we are not *in contact* with the monitor's
+    existing map range.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 290ac818696414758978b78517b137c226110bb4)
+
+commit eb98f4ab3a27aa8e00e0eb7e504ca7bec7c48a06
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 19 10:32:12 2014 -0700
+
+    osd: skip out of order op checks on tiered pools
+    
+    When we send redirected ops, we do not assign a new tid, which means that
+    a given client's ops for a pool may not have strictly ordered tids.  Skip
+    this check if the pool is tiered to avoid false positives.
+    
+    Fixes: #8380
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf2b172c843da0599164901956b66c306a59e570)
+
+commit 29ee6faecb9e16c63acae8318a7c8f6b14367af7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 6 11:50:14 2014 -0700
+
+    ReplicatedPG: block scrub on blocked object contexts
+    
+    Fixes: #8011
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 7411477153219d66625a74c5886530029c516036)
+
+commit 6078726e0b89d0202c3a4bc594be11fc346bed9d
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri May 9 09:21:23 2014 +0000
+
+    msg: Fix inconsistent message sequence negotiation during connection reset
+    
+    Backport: firefly, emperor, dumpling
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit bdee119076dd0eb65334840d141ccdf06091e3c9)
+
+commit 02557f8bc05ff28013e241611cb046b28f97b4c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Apr 15 12:55:47 2014 -0700
+
+    Objecter::_op_submit: only replace the tid if it's 0
+    
+    Otherwise, redirected ops will suddenly have a different tid
+    and will become uncancelable.
+    
+    Fixes: #7588
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 76568aa0db4e16ac1af8fe6405edade1e61cbc81)
+
+commit 788b5fad515c48d4028f5a67d63052e373c81e66
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:42:42 2014 -0700
+
+    mon/OSDMonitor: force op resend when pool overlay changes
+    
+    If a client is sending a sequence of ops (say, a, b, c, d) and partway
+    through that sequence it receives an OSDMap update that changes the
+    overlay, the ops will get send to different pools, and the replies will
+    come back completely out of order.
+    
+    To fix this, force a resend of all outstanding ops any time the overlay
+    changes.
+    
+    Fixes: #8305
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 63d92ab0969357f78fdade749785136a509bc81b)
+
+commit 268df4862d95bf506ef6f594fdfa8294671babc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:50:51 2014 -0700
+
+    osd: discard client ops sent before last_force_op_resend
+    
+    If an op is sent before last_force_op_resend, and the client's feature is
+    present, drop the op because we know they will resend.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 45e79a17a932192995f8328ae9f6e8a2a6348d10)
+
+commit 8bc4992f434ed0b180d5e4a293d8191f4846358c
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:52:11 2014 -0700
+
+    osdc/Objecter: resend ops in the last_force_op_resend epoch
+    
+    If we are a client, and process a map that sets last_force_op_resend to
+    the current epoch, force a resend of this op.
+    
+    If the OSD expects us to do this, it will discard our previous op.  If the
+    OSD is old, it will process the old one, this will appear as a dup, and we
+    are no worse off than before.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd700bdf7115223cb3e517b851f462d75dd76a2b)
+
+commit 83d03f0e1c65385dcf465108d544da84f8bfad97
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:40:10 2014 -0700
+
+    osd/osd_types: add last_force_op_resend to pg_pool_t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3152faf79f498a723ae0fe44301ccb21b15a96ab)
+
+commit c94ac44ab41fc62add3fc93c5c7ce06950eed3f1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 9 09:20:34 2014 -0700
+
+    osd: handle race between osdmap and prepare_to_stop
+    
+    If we get a MOSDMarkMeDown message and set service.state == STOPPING, we
+    kick the prepare_to_stop() thread.  Normally, it will wake up and then
+    set osd.state == STOPPING, and when we process the map message next we
+    will not warn.  However, if dispatch() takes the lock instead and processes
+    the map, it will fail the preparing_to_stop check and issue a spurious
+    warning.
+    
+    Fix by checking for either preparing_to_stop or stopping.
+    
+    Fixes: #8319
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6b858be0676f937a99dbd51321497f30c3a0097f)
+
+commit 22e2e7cff42970b00dc352513f1f138132f78d04
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat May 10 10:29:11 2014 -0700
+
+    osd/ReplicatedPG: do not queue NULL dup_op
+    
+    We call start_flush() with a NULL op in a couple different places.  Do not
+    put a NULL pointer on the dup_ops list or we will crash later.
+    
+    Fixes: #8328
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0d67f9b0695765824bdc4a65fbed88edf8ea232e)
diff --git a/doc/changelog/v0.80.3.txt b/doc/changelog/v0.80.3.txt
new file mode 100644
index 000000000..d44f62834
--- /dev/null
+++ b/doc/changelog/v0.80.3.txt
@@ -0,0 +1,19 @@
+commit a129e85cdc3446a427fffffe4c31617d49024946 (tag: refs/tags/v0.80.3)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Jul 11 17:14:12 2014 +0000
+
+    0.80.3
+
+commit 8589c108dd5f00471986deae72d88811ccf0627f (refs/remotes/gh/firefly)
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 10 10:36:16 2014 -0700
+
+    rgw: fix RGWObjManifestRule decoder
+    
+    Only decode the new field if it is a new struct.
+    
+    Fixes: #8804
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c4afaf9dabd261853a44b2e08f0911c075c1cd3a)
diff --git a/doc/changelog/v0.80.4.txt b/doc/changelog/v0.80.4.txt
new file mode 100644
index 000000000..4b44638b1
--- /dev/null
+++ b/doc/changelog/v0.80.4.txt
@@ -0,0 +1,36 @@
+commit 7c241cfaa6c8c068bc9da8578ca00b9f4fc7567f (tag: refs/tags/v0.80.4)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jul 15 12:13:55 2014 -0700
+
+    0.80.4
+
+commit 7557a8139425d1705b481d7f010683169fd5e49b (refs/remotes/gh/firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 14 13:29:37 2014 -0700
+
+    XfsFileStoreBackend: default to disabling extsize on xfs
+    
+    This appears to be responsible for the deep scrub mismatches on some rbd
+    workloads.
+    
+    Fixes: 8830
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 01cd3cdc726a3e838bce05b355a021778b4e5db1)
+
+commit ee7016a046c09be674808efb093b6ff80d0df18c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 15:00:30 2014 -0700
+
+    Revert "Revert "rgw: if extra data pool name is empty, use data pool name instead""
+    
+    This reverts commit 0b6bd2545925b5e8a80d41de1fda13ffe9d30e2b.
+    
+    We confused commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1 with commit
+    b1a4a7cb91e164d1f8af8ce9319e3b3c1949858d in our tests.  We tested without
+    the latter, saw a failure, applied it and then reverted the former, and it
+    passed, but didn't actually resolve the problem.
+    
+    This puts them both back in place and all should be well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.80.5.txt b/doc/changelog/v0.80.5.txt
new file mode 100644
index 000000000..4c7a231d6
--- /dev/null
+++ b/doc/changelog/v0.80.5.txt
@@ -0,0 +1,354 @@
+commit 38b73c67d375a2552d8ed67843c8a65c2c0feba6 (tag: refs/tags/v0.80.5, refs/remotes/gh/last, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jul 29 05:24:39 2014 -0700
+
+    0.80.5
+
+commit b576d5a242c16bc9e38ba283a9784f838614882a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 9 08:41:33 2014 -0700
+
+    osd: cancel agent_timer events on shutdown
+    
+    We need to cancel all agent timer events on shutdown.  This also needs to
+    happen early so that any in-progress events will execute before we start
+    flushing and cleaning up PGs.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c0dc245b662f1f9c640d7dd15fdf4cf26e729782)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 6213130f76b1ee89947bae578a1fccf16b845956
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:11:44 2014 -0700
+
+    osd: s/applying repop/canceling repop/
+    
+    The 'applying' language dates back to when we would wait for acks from
+    replicas before applying writes locally.  We don't do any of that any more;
+    now, this loop just cancels the repops with remove_repop() and some other
+    cleanup.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ef40737eee4389faa7792661a0f9d15b3d0440f2)
+
+commit 2e7c91587d99c9eba85ab7bd407a5b722b25a81e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:11:27 2014 -0700
+
+    osd: clear PGBackend state on shutdown
+    
+    This was leaking state on shutdown whenever there were in-flight repops
+    that were canceled.
+    
+    Fixes: #7891
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cafceae6c243369d3616af8217884e7e029896b9)
+
+commit f03110f3889ce7340c64f19358840b2425c93f68
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:10:58 2014 -0700
+
+    osd: separate cleanup from PGBackend::on_change()
+    
+    The generic portion of on_change() cleaned up temporary on-disk objects
+    and requires a Transaction.  The rest is clearing out in-memory state and
+    does not.  Separate the two.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e299357e2a24474ab7a02e65ca0279336fd04b76)
+
+commit 42ad241efab751a1d4f50d5a748d91d0fe967844
+Merge: 5f4ceb2 5049fc4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 23 10:09:19 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8858-firefly' into firefly
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 5f4ceb208b55fab7fd1530dcf8fe03198b044148
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 22 00:25:37 2014 +0100
+
+    mon: AuthMonitor: always encode full regardless of keyserver having keys
+    
+    On clusters without cephx, assuming an admin never added a key to the
+    cluster, the monitors have empty key servers.  A previous patch had the
+    AuthMonitor not encoding an empty keyserver as a full version.
+    
+    As such, whenever the monitor restarts we will have to read the whole
+    state from disk in the form of incrementals.  This poses a problem upon
+    trimming, as we do every now and then: whenever we start the monitor, it
+    will start with an empty keyserver, waiting to be populated from whatever
+    we have on disk.  This is performed in update_from_paxos(), and the
+    AuthMonitor's will rely on the keyserver version to decide which
+    incrementals we care about -- basically, all versions > keyserver version.
+    
+    Although we started with an empty keyserver (version 0) and are expecting
+    to read state from disk, in this case it means we will attempt to read
+    version 1 first.  If the cluster has been running for a while now, and
+    even if no keys have been added, it's fair to assume that version is
+    greater than 0 (or even 1), as the AuthMonitor also deals and keeps track
+    of auth global ids.  As such, we expect to read version 1, then version 2,
+    and so on.  If we trim at some point however this will not be possible,
+    as version 1 will not exist -- and we will assert because of that.
+    
+    This is fixed by ensuring the AuthMonitor keeps track of full versions
+    of the key server, even if it's of an empty key server -- it will still
+    keep track of the key server's version, which is incremented each time
+    we update from paxos even if it is empty.
+    
+    Fixes: #8851
+    Backport: dumpling, firefly
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b551ae2bcea2dd17b37f5f5ab34251cc78de0e26)
+
+commit 5c1d74848863b45aa9e98d134cbe0cd1c8199557
+Merge: 442c3fd c390ec4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 21 13:10:28 2014 -0700
+
+    Merge pull request #2122 from ceph/wip-dencoder-firefly
+    
+    fix linking deps, cpeh-dencoder, etc., firefly backport
+
+commit 5049fc4c9d6513bc58c3c4464f3de18949ac0aa5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 18 14:52:48 2014 -0700
+
+    rgw: dump prefix unconditionally
+    
+    As part of issue #8858, and to be more in line with S3, dump the Prefix
+    field when listing bucket even if bucket is empty.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7209c11251d42227608bc54cc69232ef62ffe80)
+
+commit 7fdd6d94262d8e6c81d168f5b997b3a5f030e6fa
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 15:48:26 2014 -0700
+
+    rgw: list extra objects to set truncation flag correctly
+    
+    Otherwise we end up returning wrong truncated value, and no data on the
+    next iteration.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit dc417e477d4ad262885c6b5f5987cf06d63b159d)
+
+commit 1f8774e01a616739ab70a78c348ab986b64b8fce
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:45:44 2014 -0700
+
+    rgw: account common prefixes for MaxKeys in bucket listing
+    
+    To be more in line with the S3 api. Beforehand we didn't account the
+    common prefixes towards the MaxKeys (a single common prefix counts as a
+    single key). Also need to adjust the marker now if it is pointing at a
+    common prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 82d2d612e700f94a0bb2d9fb7555abf327be379b)
+
+commit cb0a7ab8eb1d2fc8799b97c5a387b5f4f93b515c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:24:51 2014 -0700
+
+    rgw: add NextMarker param for bucket listing
+    
+    Partially fixes #8858.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb)
+
+commit 34aa7086ba5293ee4d008016558dcdb1ada02e83
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 15:21:09 2014 -0700
+
+    rgw: improve delmited listing of bucket
+    
+    If found a prefix, calculate a string greater than that so that next
+    request we can skip to that. This is still not the most efficient way to
+    do it. It'll be better to push it down to the objclass, but that'll
+    require a much bigger change.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)
+
+commit d027ff1b432a265ad49ddfeafab45093bcd03e6c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 16:05:58 2014 -0700
+
+    utf8: export encode_utf8() and decode_utf8()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 49fc68cf8c3122c878ea9503c9c74d7046bc9c6f)
+
+commit c390ec4fa919e0a17e646e595b30cfa878c3454f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 22:44:51 2014 -0700
+
+    ceph-dencoder: don't link librgw.la (and rados, etc.)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34671108ce0b7597dba4f619732ecdb8933cda6e)
+
+commit 3d89380c90ab632eb76d3e429788fd81630e7a57
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 22:27:25 2014 -0700
+
+    rgw: move a bunch of stuff into rgw_dencoder
+    
+    This will help out ceph-dencoder ...
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b1a641f307942cbf43036f75ef67fb30441dfe95)
+
+commit dc6c9bf2b92d28bbd2bc822c267b0555f74c87fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 21:58:29 2014 -0700
+
+    libosd_types, libos_types, libmon_types
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1c170776cb8c0266f0f54f049ed07bbdb9c9ab5e)
+    
+    Conflicts:
+    
+    	src/os/Makefile.am
+    	src/os/ObjectStore.cc
+    	src/osd/Makefile.am
+
+commit 7be91e8c5d935f7094033842bde1da2486905c70
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 20:55:39 2014 -0700
+
+    Revert "ceph.spec: move ceph-dencoder to ceph from ceph-common"
+    
+    This reverts commit 95f5a448b52db545a2b9bbad47fdb287254f93ea.
+    (cherry picked from commit 58cc894b3252a848ebc2169bcc4980a0ae6cc375)
+
+commit 04ad08a18a773e104a826b2a77ab0b2ffaac1e5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 20:55:35 2014 -0700
+
+    Revert "debian: move ceph-dencoder to ceph from ceph-common"
+    
+    This reverts commit b37e3bde3bd31287b11c069062280258666df7c5.
+    (cherry picked from commit f181f78b7473260a717bc8ab4fc4d73a80e3b5ba)
+
+commit 442c3fd5a3b6a6c6a85d9bd7755923f14aa598e7
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 17 21:44:06 2014 -0700
+
+    configure: do not link leveldb with everything
+    
+    Detect leveldb, but do not let autoconf blindly link it with everything on the
+    planet.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Sighed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 06a8f7b99c5533f397b34f448138220384df60ac)
+
+commit c1073e430eafeb1f213f80636a3a24966b292698
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 17:02:05 2014 -0700
+
+    ceph.spec: move ceph-dencoder to ceph from ceph-common
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 95f5a448b52db545a2b9bbad47fdb287254f93ea)
+
+commit f4e9e0ef2eab3174584f1695122b114549feff57
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 14:16:43 2014 -0700
+
+    debian: move ceph-dencoder to ceph from ceph-common
+    
+    It links against the world currently (notably leveldb).  Not nice for the
+    client-side lib.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b37e3bde3bd31287b11c069062280258666df7c5)
+
+commit 681c399e83df4c6ebd7e0836203eb2466455f548
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 17 16:40:06 2014 -0700
+
+    logrotate.conf: fix osd log rotation under upstart
+    
+    In commit 7411c3c6a42bef5987bdd76b1812b01686303502 we generalized this
+    enumeration code by copying what was in the upstart scripts.  However,
+    while the mon and mds directories get a 'done' file, the OSDs get a 'ready'
+    file.  Bah!  Trigger off of either one.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14a9ca602fa9573b0b6b94709b384bca02d12fac)
+
+commit 794ec7fb43da4db930d28571a84c1c1cec1eb066
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 12:23:31 2014 -0700
+
+    rgw: don't try to wait for pending if list is empty
+    
+    Fixes: #8846
+    Backport: firefly, dumpling
+    
+    This was broken at ea68b9372319fd0bab40856db26528d36359102e. We ended
+    up calling wait_pending_front() when pending list was empty.
+    This commit also moves the need_to_wait check to a different place,
+    where we actually throttle (and not just drain completed IOs).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f9f2417d7db01ecf2425039539997901615816a9)
+
+commit e75dd2e4b7adb65c2de84e633efcd6c19a6e457b
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Jul 17 10:06:37 2014 -0400
+
+    remove suse service restarts
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 0f11aaeefd7e8b49f88607937aade6113ebda52c)
+
+commit 361c1f8554ce1fedfd0020cd306c41b0ba25f53e
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Jul 16 16:12:12 2014 -0400
+
+    remove ceph restarts on upgrades for RPMs
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit e3a5756f58ef5d07badf18ab08a26f47f7d232cb)
+
+commit 88f694dcd3d3342f764f3790572988b7808028ae
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Jul 16 11:07:13 2014 -0400
+
+    set the default log level to WARNING
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 420f0a471a31d1e56359ea642ed831e8a38b1e90)
+
+commit 67b5193f73a2c9ec9e503ad3431473998217375d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 15 18:11:41 2014 -0700
+
+    init-ceph: wrap daemon startup with systemd-run when running under systemd
+    
+    We want to make sure the daemon runs in its own systemd environment.  Check
+    for systemd as pid 1 and, when present, use systemd-run -r <cmd> to do
+    this.
+    
+    Probably fixes #7627
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Tested-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 3e0d9800767018625f0e7d797c812aa44c426dab)
diff --git a/doc/changelog/v0.80.6.txt b/doc/changelog/v0.80.6.txt
new file mode 100644
index 000000000..2e1435e5f
--- /dev/null
+++ b/doc/changelog/v0.80.6.txt
@@ -0,0 +1,3266 @@
+commit f93610a4421cb670b08e974c6550ee715ac528ae (tag: refs/tags/v0.80.6, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Oct 1 09:37:12 2014 -0700
+
+    0.80.6
+
+commit 2ecf362e15301ee07b511b2a84585ad691543569
+Merge: 1fafd6b 76341b0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:55:02 2014 -0700
+
+    Merge pull request #2603 from dachary/wip-9620-test-mon-thrash-firefly
+
+    qa/workunits/cephtool/test.sh: fix thrash (ultimate)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 76341b0b7581194273ac787df2b940221825d083
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Sep 29 13:47:06 2014 +0200
+
+    qa/workunits/cephtool/test.sh: fix thrash (ultimate)
+
+    Keep the osd trash test to ensure it is a valid command but make it a
+    noop by giving it a zero argument (meaning thrash 0 OSD maps).
+
+    Remove the loops that were added after the command in an attempt to wait
+    for the cluster to recover and not pollute the rest of the tests. Actual
+    testing of osd thrash would require a dedicated cluster because it the
+    side effects are random and it is unnecessarily difficult to ensure they
+    are finished.
+
+    http://tracker.ceph.com/issues/9620 Fixes: #9620
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit beade63a17db2e6fc68d1f55332d602f8f7cb93a)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit 1fafd6bf2ef03672dfa27ec7a201a274927040b7
+Merge: ebb5995 71005c1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Sep 25 10:08:03 2014 -0700
+
+    Merge pull request #2576 from ceph/wip-9593
+
+    osd/ReplicatedPG: fix objecter locking in start_flush
+
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 71005c1bdf38212ea2fde12de08c45598c815d37
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 25 09:59:29 2014 -0700
+
+    osd/ReplicatedPG: fix objecter locking in start_flush
+
+    Broken in backport fd96eb62ece27f5c660429584c2ff2e058bc6e94.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit ebb599545d70e4289e385f88b4f0594b4f23f3b7
+Merge: 2abd7a3 2e60c2f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 10:37:27 2014 -0700
+
+    Merge pull request #2548 from dachary/wip-9547-python-rados-truncate-firefly
+
+    python radio aio_read must not truncate on \000 (firefly)
+
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 2abd7a3fd018b52f45f9d6b7c9d847242aa97fe9
+Merge: 2675b0d 80f2f9c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 10:27:15 2014 -0700
+
+    Merge branch 'wip-sam-testing-firefly' into firefly
+
+commit 80f2f9cf687e25d6f2c4cee34fef55215ac6b027
+Merge: bb9e65b e6da732
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:04:30 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9240' into wip-sam-testing-firefly
+
+commit bb9e65bd9292e2ee03a260cc98aa6715008e6682
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 28 15:32:22 2014 -0700
+
+    PG: wait until we've build the missing set to discover_all_missing
+
+    Fixes: #9179
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 970d9830a3a6e8568337c660fb8b4c4a60a2b3bf)
+
+    Conflicts:
+	src/osd/PG.cc
+
+commit fa645b21b587b6a1d9adbb9cedce1af3af1e5a62
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 16:53:02 2014 -0700
+
+    PG: mark_log_for_rewrite on resurrection
+
+    Fixes: #8777
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8346e10755027e982f26bab4642334fd91cc31aa)
+
+commit fd96eb62ece27f5c660429584c2ff2e058bc6e94
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 7 20:13:41 2014 -0700
+
+    ReplicatedPG:start_flush send a second delete
+
+    Suppose we start with the following in the cache pool:
+
+    30:[29,21,20,15,10,4]:[22(21), 15(15,10), 4(4)]+head
+
+    The object doesn't exist at 29 or 20.
+
+    First, we flush 4 leaving the backing pool with:
+
+    3:[]+head
+
+    Then, we begin to flush 15 with a delete with snapc 4:[4] leaving the
+    backing pool with:
+
+    4:[4]:[4(4)]
+
+    Then, we finish flushing 15 with snapc 9:[4] with leaving the backing
+    pool with:
+
+    9:[4]:[4(4)]+head
+
+    Next, snaps 10 and 15 are removed causing clone 10 to be removed leaving
+    the cache with:
+
+    30:[29,21,20,4]:[22(21),4(4)]+head
+
+    We next begin to flush 22 by sending a delete with snapc 4(4) since
+    prev_snapc is 4 <---------- here is the bug
+
+    The backing pool ignores this request since 4 < 9 (ORDERSNAP) leaving it
+    with:
+
+    9:[4]:[4(4)]
+
+    Then, we complete flushing 22 with snapc 19:[4] leaving the backing pool
+    with:
+
+    19:[4]:[4(4)]+head
+
+    Then, we begin to flush head by deleting with snapc 22:[21,20,4] leaving
+    the backing pool with:
+
+    22[21,20,4]:[22(21,20), 4(4)]
+
+    Finally, we flush head leaving the backing pool with:
+
+    30:[29,21,20,4]:[22(21*,20*),4(4)]+head
+
+    When we go to flush clone 22, all we know is that 22 is dirty, has snaps
+    [21], and 4 is clean. As part of flushing 22, we need to do two things:
+    1) Ensure that the current head is cloned as cloneid 4 with snaps [4] by
+    sending a delete at snapc 4:[4].
+    2) Flush the data at snap sequence < 21 by sending a copyfrom with snapc
+    20:[20,4].
+
+    Unfortunately, it is possible that 1, 1&2, or 1 and part of the flush
+    process for some other now non-existent clone have already been
+    performed.  Because of that, between 1) and 2), we need to send
+    a second delete ensuring that the object does not exist at 20.
+
+    Fixes: #9054
+    Backport: firefly
+    Related: 66c7439ea0888777b5cfc08bcb0fbd7bfd8653c3
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4843fd510b33a71999cdf9c2cfa2b4c318fa80fd)
+
+commit 5aa35ac262c6e0910f99086f46dd792775ab6790
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 11 12:59:16 2014 -0700
+
+    ReplicatedPG::start_flush: remove superfluous loop
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 66c7439ea0888777b5cfc08bcb0fbd7bfd8653c3)
+
+commit 14fb643bbda51cc31919197541973c332b0c37bc
+Merge: c468507 3578b11
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:03:53 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9339' into wip-sam-testing-firefly
+
+commit 2e60c2f1ec8c2aaf56b73a64f55becd679dd2825
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 20 12:41:30 2014 +0200
+
+    test: check python rados aio_read with buffers containing null
+
+    http://tracker.ceph.com/issues/9547 Refs: #9547
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 226c0c7ac6ee95ff2c1665d4e7164e2962c0346e)
+
+commit 72a46dec211d5c08d1ca9f969013d939c4361ba9
+Author: Mohammad Salehe <salehe+dev@gmail.com>
+Date:   Sun May 25 10:42:11 2014 +0430
+
+    pybind: Fix aio_read handling of string buffer
+
+    Read data may contain \0, and buf.value interprerts them as string terminator.
+
+    Signed-off-by: Mohammad Salehe <salehe+dev@gmail.com>
+    (cherry picked from commit 8bda44ff37fd04a0fc9498fbbc22f0daf515d721)
+
+commit 2675b0d159ff0608bc6e607be3727ed23e7e2976
+Merge: a67f915 d535fab
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 19 11:30:17 2014 -0700
+
+    Merge pull request #2535 from dachary/wip-9470-pidfile-firefly
+
+    daemons: write pid file even when told not to daemonize (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d535fab6ac4a025541340ded0d3568f565acf1cd
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Wed Jul 30 23:08:43 2014 -0300
+
+    daemons: write pid file even when told not to daemonize
+
+    systemd wants to run daemons in foreground, but daemons wouldn't write
+    out the pid file with -f.  Fixed.
+
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    (cherry picked from commit bccb0eb64891f65fd475e96b6386494044cae8c1)
+
+commit c4685075f583ff58cc05220a3044a10b8392033c (refs/remotes/gh/wip-log-crash-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 15 15:44:11 2014 -0700
+
+    PGLog::claim_log_and_clear_rollback_info: fix rollback_info_trimmed_to
+
+    We have been setting it to the old head value.  This is usually
+    harmless since the new head will virtually always be ahead of the
+    old head for claim_log_and_clear_rollback_info, but can cause trouble
+    in some edge cases.
+
+    Fixes: #9481
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0769310ccd4e0dceebd8ea601e8eb5c0928e0603)
+
+commit 112f6f021bd063e2647169ba42729a447e5848e1
+Merge: a67f915 92cfd37 07f54f2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Sep 18 09:46:38 2014 -0700
+
+    Merge remote-tracking branches 'origin/wip-9497' and 'origin/wip-9482' into wip-log-crash-firefly
+
+commit 07f54f28df7429a88dd9243800281ecf23f10036
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 15 16:53:21 2014 -0700
+
+    PG::find_best_info: let history.last_epoch_started provide a lower bound
+
+    If we find a info.history.last_epoch_started above any
+    info.last_epoch_started, we must be missing updates and
+    min_last_update_acceptable should provisionally be max().
+
+    Fixes: #9482
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 92cfd370395385ca5537b5bc72220934c9f09026
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 16 20:36:51 2014 -0700
+
+    PG::choose_acting: let the pg go down if acting is smaller than min_size
+
+    Even if the backfill peer would bring us up to min_size, we can't go
+    active since build_prior will not consider the interval maybe_went_rw.
+
+    Fixes: #9497
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit a67f9152f128d693d160e5b08f2ac8b8bc83e8e0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Jul 24 15:29:40 2014 -0700
+
+    librbd: fix crash using clone of flattened image
+
+    The crash occurs due to ImageCtx->parent->parent being uninitialized,
+    since the inital open_parent() -> open_image(parent) ->
+    ictx_refresh(parent) occurs before ImageCtx->parent->snap_id is set,
+    so refresh_parent() is not called to open an ImageCtx for the parent
+    of the parent. This leaves the ImageCtx->parent->parent NULL, but the
+    rest of ImageCtx->parent updated to point at the correct parent snapshot.
+
+    Setting the parent->snap_id earlier has some unintended side effects
+    currently, so for now just call refresh_parent() during
+    open_parent(). This is the easily backportable version of the
+    fix. Further patches can clean up this whole initialization process.
+
+    Fixes: #8845
+    Backport: firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2545e80d274b23b6715f4d8b1f4c6b96182996fb)
+
+commit 2422f9fd634c239a1159c99aa4a49cfb5c5d097f
+Author: JuanJose 'JJ' Galvez <jgalvez@redhat.com>
+Date:   Sun Sep 14 20:38:20 2014 -0700
+
+    init-radosgw.sysv: Support systemd for starting the gateway
+
+    When using RHEL7 the radosgw daemon needs to start under systemd.
+
+    Check for systemd running on PID 1. If it is then start
+    the daemon using: systemd-run -r <cmd>. pidof returns null
+    as it is executed too quickly, adding one second of sleep and
+    script reports startup correctly.
+
+    Signed-off-by: JuanJose 'JJ' Galvez <jgalvez@redhat.com>
+    (cherry picked from commit ddd52e87b25a6861d3b758a40d8b3693a751dc4d)
+
+commit 6070383bc2c433e36f7ab5dc1ec09a3e7439d523
+Merge: 668cd13 7b25512
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 12 17:31:03 2014 -0700
+
+    Merge pull request #2479 from ceph/wip-9444
+
+    mds: fix root and mdsdir inodes' rsubdirs
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7b25512d65c6337cbddc7d6c6f55747ba1591be5
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri May 2 23:08:41 2014 +0800
+
+    mds: fix root and mdsdir inodes' rsubdirs
+
+    inode rstat accounts inode itself.
+
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit da17394941386dab88ddbfed4af2c8cb6b5eb72f)
+
+commit 668cd1359c94698bbb4f7b841e2f30264d4af937
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 14:03:50 2014 -0700
+
+    FileStore: report l_os_j_lat as commit latency
+
+    l_os_commit_lat is actually the commit cycle latency.
+
+    Fixes: #9269
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d165238b4ee7e925e06ca22890c1e9dac101a7da)
+
+commit 7686966677867d098141aa08f45364eaa97121dd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 12:58:07 2014 -0700
+
+    Objecter::_recalc_linger_op: resend for any acting set change
+
+    Fixes: #9220
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1349383ac416673cb6df2438729fd2182876a7d1)
+
+    Conflicts:
+
+	src/osdc/Objecter.cc
+	src/osdc/Objecter.h
+
+commit 3abf95456783346e6686919b803c4819d5e82ab7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:44:57 2014 -0700
+
+    osdc/Objecter: revoke rx_buffer on op_cancel
+
+    If we cancel a read, revoke the rx buffers to avoid a use-after-free and/or
+    other undefined badness by using user buffers that may no longer be
+    present.
+
+    Fixes: #9362
+    Backport: firefly, dumpling
+    Reported-by: Matthias Kiefer <matthias.kiefer@1und1.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2305b2897acba38384358c33ca3bbfcae6f1c74e)
+
+    (adjusted for op->con instead of s->con)
+
+commit bc4108443a17de47b7c582e50fb884396a287152
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:45:52 2014 -0700
+
+    ceph_test_rados_api_io: add read timeout test
+
+    Verify we don't receive data after a timeout.
+
+    Based on reproducer for #9362 written by
+    Matthias Kiefer <matthias.kiefer@1und1.de>.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f295c1fee4afb9447cdf46f05a44234274d23b6c)
+
+commit 19e7606545f829e5b9ea0255049d9130c51e42fd
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:42:43 2014 -0700
+
+    ceph_test_rados_api_*: expose nspace
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 977d289055d69ab8a7baaf7ef68c013019225833)
+
+commit 881c267fa4f8257a9205ec897c631183ef5c6388
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 12:40:51 2014 -0700
+
+    Revert "PG: mark_log_for_rewrite on resurrection"
+
+    Actually, we don't want to backport this one without the fix
+    for #9293.
+
+    This reverts commit 7ddf0a252bb887553b29fd93e58d01cac38835e6.
+
+commit 3578b1193d497373db14d040ff7f0fc9e06e5ac2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Sep 3 15:49:47 2014 -0700
+
+    ReplicatedPG: create max hitset size
+
+    Otherwise, hit_set_create could create an unbounded size hitset
+    object.
+
+    Fixes: #9339
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d73eb895cdc792ddef2626643e61c1521cf53434
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Aug 27 16:21:41 2014 -0700
+
+    PG::can_discard_op: do discard old subopreplies
+
+    Otherwise, a sub_op_reply from a previous interval can stick around
+    until we either one day go active again and get rid of it or delete the
+    pg which is holding it on its waiting_for_active list.  While it sticks
+    around futily waiting for the pg to once more go active, it will cause
+    harmless slow request warnings.
+
+    Fixes: #9259
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ae3d87348ca4e2dde809c9593b0d54ce0469f7a0)
+
+commit 7ddf0a252bb887553b29fd93e58d01cac38835e6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 16:53:02 2014 -0700
+
+    PG: mark_log_for_rewrite on resurrection
+
+    Fixes: #8777
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8346e10755027e982f26bab4642334fd91cc31aa)
+
+commit 49a08eb46ef277225fff0d9a0920c37427175f54
+Author: Thorsten Glaser <tg@mirbsd.de>
+Date:   Mon Sep 8 12:49:50 2014 -0700
+
+    debian: only B-R yasm on amd64
+
+    Make yasm dependency amd64 only, it isn?t used elsewhere
+    but breaks x32 (which is mis-detected as amd64)
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ab46dc5b49219aa6194861c393c938f23001c52)
+
+commit f19162adc241e627ce20140360352ac96f809a6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 17:43:10 2014 -0700
+
+    osd: fix osd_tp shutdown
+
+    We need to clear the queue, not just drain the currently executing jobs.
+
+    Fixes: #9218
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2f21c04207b9a2a65e514994a775632b36d6874)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit 598cde42e7f57fd0b294f70c0a930f94fd361fb3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 27 06:19:12 2014 -0700
+
+    osd/PG: fix crash from second backfill reservation rejection
+
+    If we get more than one reservation rejection we should ignore them; when
+    we got the first we already sent out cancellations.  More importantly, we
+    should not crash.
+
+    Fixes: #8863
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b13de16c522754e30a0a55fb9d072082dac455e)
+
+commit 0a9d49e5b5867c58ca1f5c6b9a4c5106639a8d01
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 06:58:45 2014 -0700
+
+    mon/Paxos: don't spam log with is_readable at dout level 1
+
+    Backport: firefly, dumpling
+    Reported-by: Aanchal Agrawal <Aanchal.Agrawal@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 62ca27d0b119b597ebad40dde64c4d86599e466d)
+
+commit 1660503ffafd69fac3722aea9915008113906e05
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Sep 4 13:58:14 2014 -0400
+
+    doc: add note on soft JS dependency for navigating docs
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 657be818375bea2d8b5998ea1e5505eedc2f294d)
+
+commit 2f075d442f58e9cfefdbc421b9b20f2034a56314
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 21:21:45 2014 -0400
+
+    doc: fix missing bracket
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 69638dfaeb0dcd96dac4b5f5c00ed08042432487)
+
+commit 4dacb593e830f42cd0394486333315f1709b502f
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 20:47:54 2014 -0400
+
+    doc: attempt to get the ayni JS into all head tags
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 35663fa55ac1579a3b0c8b67028a3a8dfea87b48)
+
+commit 275df4407b37923cb47510890a686ed370e2f39e
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Sat Aug 23 22:41:30 2014 +1000
+
+    Fix FTBFS on alpha due to incorrect check on BLKGETSIZE
+
+    Ceph FTBFS on Alpha with:
+
+    ~~~~
+    libtool: compile:  g++ -DHAVE_CONFIG_H -I. -D__CEPH__ -D_FILE_OFFSET_BITS=64 -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -D_GNU_SOURCE -DCEPH_LIBDIR=\"/usr/lib/alpha-linux-gnu\" -DCEPH_PKGLIBDIR=\"/usr/lib/alpha-linux-gnu/ceph\" -DGTEST_HAS_TR1_TUPLE=0 -D_FORTIFY_SOURCE=2 -I/usr/include/nss -I/usr/include/nspr -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -Werror=format-security -fno-strict-aliasing -fsigned-char -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-invalid-offsetof -Wstrict-null-sentinel -g -O2 -Wformat -Werror=format-security -c common/blkdev.cc  -fPIC -DPIC -o common/.libs/blkdev.o
+    In file included from /usr/include/alpha-linux-gnu/asm/ioctls.h:4:0,
+                     from /usr/include/alpha-linux-gnu/bits/ioctls.h:23,
+                     from /usr/include/alpha-linux-gnu/sys/ioctl.h:26,
+                     from common/blkdev.cc:3:
+    common/blkdev.cc:13:7: error: missing binary operator before token "int"
+     #elif BLKGETSIZE
+           ^
+    ~~~~
+
+    This error occurs because the value of BLKGETSIZE is tested in a
+    c-preprocessor conditional compilation test whereas the test should
+    be for existence.
+
+    From: Michael Cree <mcree@orcon.net.nz>
+    Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=756892
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 6ad8e61a428cfc9fc60ccdb9bce812e1f49822ac)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit fbbdc31a8cd4c6457e06570ddeb9b2021563498e
+Merge: 46d9611 895bddc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 29 17:31:29 2014 -0700
+
+    Merge pull request #2356 from dachary/wip-9273-mon-preload-erasure-code-firefly
+
+    erasure-code: preload the default plugins in the mon (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 46d9611aedd3194cfd55a338e5f94011e6e70668
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Aug 29 20:21:25 2014 +0100
+
+    osd: OSDMap: ordered blacklist on non-classic encode function
+
+    Fixes: #9211
+    Backport: firefly
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 81102044f417bd99ca570d9234b1df5195e9a8c9)
+
+commit 994a9e35e96984c0806261f7eb349d95c0fb4873
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 08:16:29 2014 -0700
+
+    osd/OSDMap: encode blacklist in deterministic order
+
+    When we use an unordered_map the encoding order is non-deterministic,
+    which is problematic for OSDMap.  Construct an ordered map<> on encode
+    and use that.  This lets us keep the hash table for lookups in the general
+    case.
+
+    Fixes: #9211
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4672e50922b75d642056020b9745a3a5844424d3)
+
+commit 895bddc5d4dabd2f05c13c343eb0d7457d6f132f
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Aug 29 18:13:08 2014 +0200
+
+    erasure-code: preload the default plugins in the mon
+
+    The commit 164f1a1959a863848319585fa752250c7b261381 preloads the
+    jerasure plugin in the OSD. They must also be preloaded in the mon for
+    the same reasons.
+
+    http://tracker.ceph.com/issues/9273 Fixes: #9273
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit db8d6e6e190de964ec3d5d1bbc87da2dcb3213f5
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 26 17:36:16 2014 +0100
+
+    mds: fix FP error in ROUND_UP_TO
+
+    Explicitly handle case where denominator is 0, instead of
+    passing into ROUND_UP_TO.
+
+    Regression from 9449520b121fc6ce0c64948386d4ff77f46f4f5f
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bf3e4835dabc057982def1b5c9a6499c04ac5312)
+
+commit 8e3120fcb379a00d370e4c04d34af35e596e2de9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 11:14:39 2014 -0700
+
+    mon: generate cluster_fingerprint if null
+
+    This triggers after an upgrade of a legacy cluster that has no fingerprint.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b245d600163f6337af15aedd1fea68f4e2a668a8)
+
+commit ebcdeb4cfe201dd0c630386226f9970650689ccf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 20 08:59:46 2014 -0700
+
+    mon: add a cluster fingerprint
+
+    Generate it on cluster creations with the initial monmap.  Include it in
+    the report.  Provide no way for this uuid to be fed in to the cluster
+    (intentionally or not) so that it can be assumed to be a truly unique
+    identifier for the cluster.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 675b0042eff0ad5e1453838410210b1206c39004)
+
+commit b3b029165d4ac0af3f2143fad205f4e8ab08a30f
+Merge: 21e1faa 07e1ceb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 13:13:08 2014 -0700
+
+    Merge pull request #2244 from dachary/wip-9044-use-ruleset-firefly
+
+    erasure-code: OSDMonitor::crush_ruleset_create_erasure needs ruleset (firefly)
+
+commit 21e1faa75da4e9cab8e139a8e2a1c4fc0ee30867
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 12 16:41:38 2014 -0700
+
+    ReplicatedPG::cancel_copy: clear cop->obc
+
+    Otherwise, an objecter callback might still be hanging
+    onto this reference until after the flush.
+
+    Fixes: #8894
+    Introduced: 589b639af7c8834a1e6293d58d77a9c440107bc3
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5040413054e923d6d5a2b4928162dba140d980e0)
+
+commit e6da7323655f3bfa4f669fb8361fcaa96f633456
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 12:02:52 2014 -0700
+
+    PG: recover from each osd at most once
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 8a1723f67f995253c9e6da95ab433743ef23f61c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 11:38:53 2014 -0700
+
+    PG: make the reservation sets more descriptively named
+
+    These sets won't precisely be the backfill_targets or actingbackfill
+    shortly.
+
+    %s/sorted_backfill_set/remote_shards_to_reserve_backfill/g
+    %s/acting_osd_it/remote_recovery_reservation_it/g
+    %s/sorted_actingbackfill_set/remote_shards_to_reserve_recovery/g
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 938d735697e2af0b01dd2613a9152b3caef5db57
+Merge: 2f11631 9ca451a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 10:30:14 2014 -0700
+
+    Merge pull request #2203 from ceph/wip-scrub-firefly
+
+    backport scrub throttling to firefly
+
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 2f11631f3144f2cc0e04d718e40e716540c8af19
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 16 12:42:33 2014 -0700
+
+    os/FileStore: fix mount/remount force_sync race
+
+    Consider:
+
+     - mount
+     - sync_entry is doing some work
+     - umount
+       - set force_sync = true
+       - set done = true
+     - sync_entry exits (due to done)
+       - ..but does not set force_sync = false
+     - mount
+     - journal replay starts
+     - sync_entry sees force_sync and does a commit while op_seq == 0
+     ...crash...
+
+    Fixes: #9144
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd11042f969b94f7a461d02e1475794031c79f61)
+
+    Conflicts:
+	src/os/FileStore.cc
+
+commit a38cf1bd503a09843d439f65937e7d60aed163e2
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 10 10:32:17 2014 +0800
+
+    Add random_cache.hpp to Makefile.am
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit a3e5c6d632119febd2150944a6f2cbce33cfda3a)
+
+commit a2c1532b5610207eccd24b8253643eec6bb05294
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:42:12 2014 -0700
+
+    os/KeyValueStore, MemStore: fix warning
+
+    os/MemStore.cc: In member function 'void MemStore::_do_transaction(ObjectStore::Transaction&)':
+    os/MemStore.cc:956:18: warning: unused variable 'expected_object_size' [-Wunused-variable]
+    os/MemStore.cc:957:18: warning: unused variable 'expected_write_size' [-Wunused-variable]
+    os/KeyValueStore.cc: In member function 'unsigned int KeyValueStore::_do_transaction(ObjectStore::Transaction&, KeyValueStore::BufferTransaction&, ThreadPool::TPHandle*)':
+    os/KeyValueStore.cc:1426:18: warning: unused variable 'expected_object_size' [-Wunused-variable]
+    os/KeyValueStore.cc:1427:18: warning: unused variable 'expected_write_size' [-Wunused-variable]
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 2b86bf4f4c1be2603368d5031ee7d706056e2d1e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 29 11:23:58 2014 -0700
+
+    osd: automatically scrub PGs with invalid stats
+
+    If a PG has recnetly split and has invalid stats, scrub it now, even if
+    it has scrubbed recently.  This helps the stats become valid again soon.
+
+    Fixes: #8147
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 68b440d66539e820c9ce86a6942c3188be4ee1ec)
+
+commit 271ca7d9bc41e3a2ac387b05ca9f30d8a250fb62
+Merge: 54333a4 87cd3a8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:38:34 2014 -0700
+
+    Merge pull request #2328 from dachary/wip-9209-round-up-to-firefly
+
+    common: ROUND_UP_TO accepts any rounding factor (firefly)
+
+commit 54333a49c9e99a398bf3695845b248ba76ee3930
+Merge: 4bff6f3 58d0871
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:09:17 2014 -0700
+
+    Merge pull request #2326 from yuyuyu101/wip-kvstore-firefly
+
+    Backport from master to Firefly(KeyValueStore)
+
+commit 87cd3a8f6e190aed06e361595afc4ec83148da98
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Aug 25 17:05:04 2014 +0200
+
+    common: ROUND_UP_TO accepts any rounding factor
+
+    The ROUND_UP_TO function was limited to rounding factors that are powers
+    of two. This saves a modulo but it is not used where it would make a
+    difference. The implementation is changed so it is generic.
+
+    http://tracker.ceph.com/issues/9209 Fixes: #9209
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9449520b121fc6ce0c64948386d4ff77f46f4f5f)
+
+commit 58d08714340049a5165ad682ec5b54292525b45d
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Mar 20 14:09:49 2014 +0800
+
+    Remove exclusive lock on GenericObjectMap
+
+    Now most of GenericObjectMap interfaces use header as argument not the union of
+    coll_t and ghobject_t. So caller should be responsible for maintain the
+    exclusive header.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit 98df982a6f830154ff6880f5ee00350ba3375b6b
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Jul 23 11:26:18 2014 +0800
+
+    common/RandomCache: Fix inconsistence between contents and count
+
+    The add/clear method may cause count inconsistent with the real size of
+    contents.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit c8ec7721526ccf4bfd91a220fee0dd75bef8fb21
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:41:28 2014 +0000
+
+    Add random cache and replace SharedLRU in KeyValueStore
+
+    SharedLRU plays pool performance in KeyValueStore with large header cache size,
+    so a performance optimized RandomCache could improve it.
+
+    RandomCache will record the lookup frequency of key. When evictint element,
+    it will randomly compare several elements's frequency and evict the least
+    one.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/common/config_opts.h
+	src/os/KeyValueStore.cc
+
+commit 33ca26a103a0d99627fddc30f2916c5aeb37c691
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:40:16 2014 +0000
+
+    Add Header cache to KeyValueStore
+
+    In the performance statistic recently, the header lookup becomes the main time
+    consuming for the read/write operations. Most of time it occur 50% to deal with
+    header lookup, decode/encode logics.
+
+    Now adding header cache using SharedLRU structure which will maintain the header
+    cache and caller will get the pointer to the real header. It also avoid too much
+    header copy operations overhead.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/os/KeyValueStore.cc
+	src/os/KeyValueStore.h
+
+commit e8dcb8cc603cff64bafc11ee42c5c04a9cf85d12
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Feb 26 17:46:07 2014 +0800
+
+    Fix write operation on a deleted object in the same transaction
+
+    If the following op happened:
+    touch obj
+    delete obj
+    write obj
+
+    KeyValueStore will fail at "write" operation.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit e0ae6f9de685b60eee5efa5ca960c7a15481e41a
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:35:57 2014 +0000
+
+    Remove SequencerPosition from KeyValueStore
+
+    Now KeyValueStore expects kv backend to ensure consistency and there is unusable
+    for KeyValueStore to store a SequencerPosition.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/os/KeyValueStore.cc
+	src/os/KeyValueStore.h
+
+commit 44721f71266e3098169ed00f1ca7e86496b76b07
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Jun 4 12:58:07 2014 +0800
+
+    Fix keyvaluestore fiemap bug
+
+    The result of fiemap is wrong and the offset get from
+    "StripObjectMap::file_to_extents" need to multiply by sequence number
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit 4bff6f3476e5df400db3dc1f0ad7b35a2a41830e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 13:05:35 2014 -0700
+
+    mon: fix occasional message leak after session reset
+
+    Consider:
+
+     - we get a message, put it on a wait list
+     - the client session resets
+     - we go back to process the message later and discard
+       - _ms_dispatch returns false, but nobody drops the msg ref
+
+    Since we call _ms_dispatch() a lot internally, we need to always return
+    true when we are an internal caller.
+
+    Fixes: #9176
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19df386b2d36d716be2e6d02de0386fac9e7bc1f)
+
+commit ca3ac907aa73c5d77760e12da6f1509ea860e894
+Merge: fdbab46 8d7e77b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 10:14:18 2014 -0700
+
+    Merge pull request #2298 from dachary/wip-9153-jerasure-upgrade-firefly
+
+    erasure-code: preload the jerasure plugin variant (sse4,sse3,generic)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8d7e77b9747f1b3dc3c10d1b0877a4b9c899ba86
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Aug 21 14:41:55 2014 +0200
+
+    erasure-code: preload the jerasure plugin variant (sse4,sse3,generic)
+
+    The preloading of the jerasure plugin ldopen the plugin that is in
+    charge of selecting the variant optimized for the
+    CPU (sse4,sse3,generic). The variant plugin itself is not loaded because
+    it does not happen at load() but when the factory() method is called.
+
+    The JerasurePlugin::preload method is modified to call the factory()
+    method to load jerasure_sse4 or jerasure_sse3 or jerasure_generic as a
+    side effect.
+
+    Indirectly loading another plugin in the factory() method is error prone
+    and should be moved to the load() method instead. This change should be
+    done in a separate commit.
+
+    http://tracker.ceph.com/issues/9153 Fixes: #9153
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit fdbab46852e74d405b5c747da98564a5866ec8a7
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue May 20 14:32:18 2014 +0800
+
+    Fix set_alloc_hint op cause KeyValueStore crash problem
+
+    Now KeyValueStore doesn't support set_alloc_hit op, the implementation of
+    _do_transaction need to consider decoding the arguments. Otherwise, the
+    arguments will be regarded as the next op.
+
+    Fix the same problem for MemStore.
+
+    Fix #8381
+
+    Reported-by: Xinxin Shu <xinxin.shu5040@gmail.com>
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit c08adbc98ff5f380ecd215f8bd9cf3cab214913c)
+
+commit 922e24e658f09a8a772309f18a8d54c560fad653
+Merge: ae787cf 164f1a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 20 10:10:08 2014 -0700
+
+    Merge pull request #2286 from dachary/wip-9153-jerasure-upgrade-firefly
+
+    erasure-code: preload the jerasure plugin (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 164f1a1959a863848319585fa752250c7b261381
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 19 01:30:15 2014 +0200
+
+    erasure-code: preload the jerasure plugin
+
+    Load the jerasure plugin when ceph-osd starts to avoid the following
+    scenario:
+
+    * ceph-osd-v1 is running but did not load jerasure
+
+    * ceph-osd-v2 is installed being installed but takes time : the files
+      are installed before ceph-osd is restarted
+
+    * ceph-osd-v1 is required to handle an erasure coded placement group and
+      loads jerasure (the v2 version which is not API compatible)
+
+    * ceph-osd-v1 calls the v2 jerasure plugin and does not reference the
+      expected part of the code and crashes
+
+    Although this problem shows in the context of teuthology, it is unlikely
+    to happen on a real cluster because it involves upgrading immediately
+    after installing and running an OSD. Once it is backported to firefly,
+    it will not even happen in teuthology tests because the upgrade from
+    firefly to master will use the firefly version including this fix.
+
+    While it would be possible to walk the plugin directory and preload
+    whatever it contains, that would not work for plugins such as jerasure
+    that load other plugins depending on the CPU features, or even plugins
+    such as isa which only work on specific CPU.
+
+    http://tracker.ceph.com/issues/9153 Fixes: #9153
+
+    Backport: firefly
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9b802701f78288ba4f706c65b853415c69002d27)
+
+    Conflicts:
+	src/test/erasure-code/test-erasure-code.sh
+	src/common/config_opts.h
+
+commit ae787cfa88dfd0f5add5932b297258c46af4e333
+Author: Matt Benjamin <matt@linuxbox.com>
+Date:   Thu May 29 10:34:20 2014 -0400
+
+    Work around an apparent binding bug (GCC 4.8).
+
+    A reference to h->seq passed to std::pair ostensibly could not bind
+    because the header structure is packed.  At first this looked like
+    a more general unaligned access problem, but the only location the
+    compiler rejects is a false positive.
+
+    Signed-off-by: Matt Benjamin <matt@linuxbox.com>
+    (cherry picked from commit c930a1f119069a424af28a618b0abff4947c221f)
+
+commit 486deefdc1496b72cc680e432a61d71e5fa265c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 17 20:54:28 2014 -0700
+
+    qa/workunits/rbd/qemu-iotests: touch common.env
+
+    This seems to be necessary on trusty.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 055be68cf8e1b84287ab3631a02e89a9f3ae6cca)
+
+commit cc4e6258d67fb16d4a92c25078a0822a9849cd77
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:52:08 2014 +0100
+
+    unittest_strtol: fix compilation warning
+
+    Was fixed in master by a4923f5bc373d530d1ffdf6c58a4d88139daedd2
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 7f9fe22a1c73d5f2783c3303fb9f3a7cfcea61c5
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Jun 17 13:12:58 2014 +0800
+
+      Fix EINVAL err when use "ceph tell osd.* bench"
+
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit 7dc93a9651f602d9c46311524fc6b54c2f1ac595)
+
+commit 14607fc045aa169f1e8fa6872b412fba8f09d645
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 16 17:48:34 2014 -0700
+
+    qa/workunits/cephtool/test.sh: fix get erasure_code_profile test
+
+    Manual backport of 4d6899c7560e990650959b442980a7249f0ba4c1
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 068b906ee396f1ad717968f214610ea86dd8d6b4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 27 21:41:18 2014 +0100
+
+    mon: OSDMonitor: add 'osd pool get-quota' command
+
+    Enables us to obtain current quotas for a given pool.
+
+    Fixes: #8523
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 714a9bb5a058b2553f3be3e4cfb7e7f30150e75a)
+
+commit 2ed0fa034aa8e49ab703bdb798bd6211a83e3a9d
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jun 3 10:12:41 2014 +0100
+
+    mon: name instead of id in "has tiers" message
+
+    Instead of "Pool foo has tiers 1 2" print
+    "Pool foo has tiers bar baz".
+
+    Signed-off-by: John Spray <jspray@redhat.com>
+    (cherry picked from commit 97772c2f53f726bd71710d0d3e34159d2679390a)
+
+commit 7e509b1b8b5a3679094f794cd9334598aef4441e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 17:01:38 2014 +0100
+
+    common/config.cc: allow integer values to be parsed as SI units
+
+    We are allowing this for all and any integer values; that is, OPT_INT,
+    OPT_LONGLONG, OPT_U32 and OPT_U64.
+
+    It's on the user to use appropriate units.  For instance, the user should
+    not use 'E(xabyte)' when setting a signed int, and use his best judgment
+    when setting options that, for instance, ought to receive seconds.
+
+    Fixes: 8265
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5500437e064cd6b4b45d63ee9396193df87f4d44)
+
+commit 4e34d83e337d0febf20c86b051721878feef49a8
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:52:08 2014 +0100
+
+    test/strtol.cc: Test 'strict_strtosi()'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 40587d4792fd55db72d33870aae8b6a806c9baaf)
+
+commit 3b06b8f4893bd18440ecb9705414831a827cef90
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:51:37 2014 +0100
+
+    common/strtol.cc: strict_strtosi() converts str with SI units to uint64_t
+
+    Accepts values with a suffix (B, K, M, G, T, P, E) and returns the
+    appropriate byte value.
+
+    E.g., 10B = 10, while 10K = 10240.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 67dc5751ba9a4e527ff12ea65000d1ba45d956f6)
+
+commit 92ab6ecf321ab68ea8fe3e8c2b310303a5726af3
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 13 15:50:20 2014 -0400
+
+    ceph-disk: linter cleanup
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit d74ed9d53fab95f27a9ad8e9f5dab7192993f6a3)
+
+commit 08772fd888fb0eca4570bd6b4bb25f8122691cb4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 12:00:50 2014 -0700
+
+    ceph-disk: warn about falling back to sgdisk (once)
+
+    This way the user knows something funny might be up if dmcrypt is in use.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f7798e37e098de38fbc73f86c4c6ee705abbe38)
+
+commit 50166efd205f46fa325dec9636d817387e5d4d3b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 11:40:34 2014 -0700
+
+    ceph-disk: only fall back to sgdisk for 'list' if blkid seems old
+
+    If the blkid doesn't show us any ID_PART_ENTRY_* fields but we know it is
+    a GPT partition, *then* fallback.  Otherwise, don't bother.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b1651afb34d9d2c324db3bf5f54ac9ce001c6af9)
+
+commit a45e815cb00f1a807691cbf687990839995f7629
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 11:39:47 2014 -0700
+
+    ceph-disk: add get_partition_base() helper
+
+    Return the base devices/disk for a partition device.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b75e8a340c49cbc067baa19790b994a5f904bb4f)
+
+commit 8c04e475efb4471c605e3c921ec8a03b67934899
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:26:07 2014 -0700
+
+    ceph-disk: display information about dmcrypted data and journal volumes
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c7a1ceba441fa99a82e19ed2cd3c6782a5d77636)
+
+commit b09e659226b26002b58c7697f8c600d06fe78c58
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:25:42 2014 -0700
+
+    ceph-disk: move fs mount probe into a helper
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f80ed26d2403ba12e80da6459fc45c22584f72de)
+
+commit 6775f46e4a5862ddbf81bef11016bc036222fda9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:25:10 2014 -0700
+
+    ceph-disk: use partition type UUIDs, and blkid
+
+    Use blkid to give us the GPT partition type.  This lets us distinguish
+    between dmcrypt and non-dmcrypt partitions.  Fake it if blkid doesn't
+    give us what we want and try with sgdisk.  This isn't perfect (it can't
+    tell between dmcrypt and not dmcrypt), but such is life, and we are better
+    off than before.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6c77f5f2f994c881232d76ce9c69af80d10772bd)
+
+commit 1804a83400b7c3dc07650e09da5224bb999d0940
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 13:53:16 2014 -0700
+
+    ceph-disk: fix log syntax error
+
+      File "/usr/sbin/ceph-disk", line 303, in command_check_call
+        LOG.info('Running command: %s' % ' '.join(arguments))
+    TypeError: sequence item 2: expected string, NoneType found
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1088d6cd11b476cd67ed30e07edd363c4057a003)
+
+commit 8a8a9c35349d3f58d899afa1a9e219b99586bc9f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 11 15:58:15 2014 -0700
+
+    Revert "Fix for bug #6700"
+
+    This reverts commit 673394702b725ff3f26d13b54d909208daa56d89.
+
+    This appears to break things when the journal and data disk are *not* the same.
+    And I can't seem to reproduce the original failure...
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2edf01ffa4a7425af2691b4e94bc5fd0bfab1e5b)
+
+commit 50b700fadf6ce6e0bf14b5daa2288786426d7359
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 11 15:57:52 2014 -0700
+
+    ceph-disk: fix verify_no_in_use check
+
+    We only need to verify that partitions aren't in use when we want to
+    consume the whole device (osd data), not when we want to create an
+    additional partition for ourselves (osd journal).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d6e6ba198efc4b3afff0c70af53497a70c6b3f19)
+
+commit c9847ef059f182ad15ef027c8bdfae6c99d91867
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu May 22 17:04:28 2014 -0400
+
+    better error reporting on incompatible device requirements
+
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 1ac3a503a15ddf7f7c1a33310a468fac10a1b7b6)
+
+commit 40649902891244242d5861ccd9538aec026cae3a
+Author: Stuart Longland <stuartl@vrt.com.au>
+Date:   Tue May 6 14:06:36 2014 -0700
+
+    ceph-disk: fix list for encrypted or corrupt volume
+
+    Continue gracefully if an fs type is not detected, either because it is
+    encrypted or because it is corrupted.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09beebe3f1fd1b179547743648049b891cb8bc56)
+
+commit 80896dcf2c3ccba0b9b0b1bd9f6e075585e3d502
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Jun 13 09:37:33 2014 -0400
+
+    support dmcrypt partitions when activating
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit ef8a1281512c4ee70a3764b28891da691a183804)
+
+commit 1b0da81d733a36de4a9dc80263058a6cdf05c745
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 16:41:43 2014 -0700
+
+    init-ceph: don't use bashism
+
+           -z STRING
+                  the length of STRING is zero
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d6d1aa7e0c5e0b5f99c9b548a1f890c511b4299)
+
+commit 9005f33d0d80d6d2b93493b38866b7f74f87d1eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 14:28:57 2014 -0700
+
+    osd: fix feature requirement for mons
+
+    These features should be set on the client_messenger, not
+    cluster_messenger.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ae0b9f17760eda9a7e09a6babac50bfe8ebb4b36)
+
+commit 9e46c5ad7dadd1c342c82629577b5a1bf037fe75
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 13:54:11 2014 -0700
+
+    unittest_osdmap: test EC rule and pool features
+
+    TODO: tiering feature bits.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f0e2951d773b6acce781b4b991d6d8e817ee2f9)
+
+commit dbbe0c627bcd4a60c1346bb120698b5a0c2192c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 14:04:05 2014 -0700
+
+    unittest_osdmap: create an ec pool in test osdmap
+
+    This is part of 7294e8c4df6df9d0898f82bb6e0839ed98149310.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c2aa74a22718d9a59d60201d71213aaff2492ff8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 08:55:10 2014 -0700
+
+    osd: only require crush features for rules that are actually used
+
+    Often there will be a CRUSH rule present for erasure coding that uses the
+    new CRUSH steps or indep mode.  If these rules are not referenced by any
+    pool, we do not need clients to support the mapping behavior.  This is true
+    because the encoding has not changed; only the expected CRUSH output.
+
+    Fixes: #8963
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 16dadb86e02108e11a970252411855d84ab0a4a2)
+
+commit 12430fed85b2cfeaeb33a443e6a4c6538d0833f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 08:52:37 2014 -0700
+
+    crush: add is_v[23]_rule(ruleid) methods
+
+    Add methods to check if a *specific* rule uses v2 or v3 features.  Refactor
+    the existing checks to use these.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d95486780a54c85a5c88936a4da4bdc3576a7b8)
+
+commit 2427ea8f153caf55b3bce77cd2fc88935563c7b3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 30 13:40:07 2014 -0700
+
+    PGLog: fix clear() to avoid the IndexLog::zero() asserts
+
+    Introduced in:
+      c5b8d8105d965da852c79add607b69d5ae79a4d4
+      ac11ca40b4f4525cbe9b1778b1c5d9472ecb9efa
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+    (cherry picked from commit 959f2b25910360b930183fbf469ce984a48542dd)
+
+commit 9ca451a6bf2f29e75f72cbd3dbcff0bf8986e462
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 19 12:34:36 2014 -0700
+
+    osd: allow io priority to be set for the disk_tp
+
+    The disk_tp covers scrubbing, pg deletion, and snap trimming
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d9073f486527ca13cdb2774745c4c63c218333ad)
+
+commit 11858d7e7a493c9493f7039b7e45dc03fc4feb90
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:09 2014 -0700
+
+    common/WorkQueue: allow io priority to be set for wq
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd6badcb5eedfec6748b3e6ca4d46e3b266038f6)
+
+    Conflicts:
+
+	src/common/WorkQueue.cc
+
+commit b75f85a2c4dd9807947862f7b89a5f25dfa1defe
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:42 2014 -0700
+
+    common/Thread: allow io priority to be set for a Thread
+
+    Ideally, set this before starting the thread.  If you set it after, we
+    could potentially race with create() itself.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1b8741022c5a2ebae38905215dadee696433e931)
+
+commit 9f29788fbc19aea341b4fe997a567aa3054b9d1c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:09 2014 -0700
+
+    common/io_priority: wrap ioprio_set() and gettid()
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2b49110ef65efd526c3430ad03c988ca9dde768)
+
+commit fac5bfafef1eb82f8cf23c369fb5f50502e2e6f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 10:47:24 2014 -0700
+
+    osd: introduce simple sleep during scrub
+
+    This option is similar to osd_snap_trim_sleep: simply inject an optional
+    sleep in the thread that is doing scrub work.  This is a very kludgey and
+    coarse knob for limiting the impact of scrub on the cluster, but can help
+    until we have a more robust and elegant solution.
+
+    Only sleep if we are in the NEW_CHUNK state to avoid delaying processing of
+    an in-progress chunk.  In this state nothing is blocked on anything.
+    Conveniently, chunky_scrub() requeues itself for each new chunk.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c4e8451cc5b4ec5ed07e09c08fb13221e31a7ac6)
+
+commit ad82f28122dc41095d5e703279a8a5c494d35913
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 14 10:30:50 2014 -0700
+
+    osd: add sanity check/warning on a few key configs
+
+    Warn when certain config values are set to bad values.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3ec7d0b23fdee39a34bda7595cd2a79c08daf8a)
+
+commit 289360c5dc57a4788289472f90d63781143539be
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 1 17:24:48 2014 -0700
+
+    osd: prevent pgs from getting too far ahead of the min pg epoch
+
+    Bound the range of PG epochs between the slowest and fastest pg
+    (epoch-wise) with 'osd map max advance'.  This value should be set to
+    something less than 'osd map cache size' so that the maps we are
+    processing will be in memory as many PGs advance forward in time in
+    loose synchrony.
+
+    This is part of the solution to #7576.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf25bdf6b0090379903981fe8cee5ea75efd7ba0)
+
+commit 662af7ac83b67af8c9edc554bbd505de717f9709
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 7 17:42:06 2014 -0700
+
+    osd: fix pg epoch floor tracking
+
+    If you call erase() on a multiset it will delete all instances of a value;
+    we only want to delete one of them.  Fix this by passing an iterator.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a52a855f6c92b03dd84cd0cc1759084f070a98c2)
+
+commit 229a346d824d84c361d7963a356af5251adc9e2e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 2 14:29:08 2014 -0700
+
+    osd: track per-pg epochs, min
+
+    Add some simple tracking so that we can quickly determine what the min
+    pg osdmap epoch is.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81e4c47722255ac3d46f701a80e104cc390e766c)
+
+commit 38c3a3c0b0d1903aea2fdc6224fc4f70bbc39329
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 13:31:10 2014 -0700
+
+    mon: fix divide by zero when pg_num adjusted and no osds
+
+    Fixes: #9052
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 239401db7b51541a57c59a261b89e0f05347c32d)
+
+commit 87bf00a2cb93ce2dc904a42c45cc9f43023725c3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 12:48:29 2014 -0700
+
+    ceph_test_rados_api_tier: fix cache cleanup (ec too)
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d7fb7bf5f2059f411633751e376c2270e6040fba)
+
+commit 3be0d731cbe158d837d05c907aefe16aa95977be
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 12:15:38 2014 -0700
+
+    ceph_test_rados_api: fix cleanup of cache pool
+
+    We can't simply try to delete everything in there because some items may
+    be whiteouts.  Instead, flush+evict everything, then remove overlay, and
+    *then* delete what remains.
+
+    Fixes: #9055
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ebbe8aab17cfa2281902f167a706639535da4010)
+
+commit 7f511ef7fd55cca8f1babaf696ca0faa0e1e411d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 11:41:23 2014 -0700
+
+    librados/TestCase: inheret cleanup_default_namespace
+
+    No need to duplicate this code.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d199fb1173dc9685dba96c22b83e4e9edf51a11)
+
+commit 4a5c93a90986de4510c50f0fba72ae7d5aed8a15
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 10:34:53 2014 -0700
+
+    osd/ReplicatedPG: only do agent mode calculations for positive values
+
+    After a split we can get negative values here.  Only do the arithmetic if
+    we have a valid (positive) value that won't through the floating point
+    unit for a loop.
+
+    Fixes: #9082
+    Tested-by: Karan Singh <karan.singh@csc.fi>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5be56ff86d9f3ab2407a258a5285d0b8f52f041e)
+
+commit 39bcafa6f2b14e4e11e1541cf01f24525e2b6449
+Merge: 7da121d cb48cd4
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 12 21:15:26 2014 -0700
+
+    Merge pull request #2231 from ceph/wip-8944-firefly
+
+    Wip 8944 firefly
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7da121d2aa1ea5c5c8accef92d7304912d4b2eb3
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 18:33:56 2014 -0700
+
+    OSD: add require_same_peer_inst(OpRequestRef&,OSDMap&) helper
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e99acf9810976b1fc74b84ad289773af43be973f)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit 8595e9bed3689933c03a8f3443052a36ff1d62f5
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 14:19:59 2014 -0700
+
+    OSD: introduce require_self_aliveness(OpRequestRef&,epoch_t) function
+
+    Take the self-aliveness checks out of require_same_or_newer_map() and use
+    the new function for that and for require_up_osd_peer().
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e179e9227b4a4482d8359682092fd7f426b9a919)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit d0f2c4891bbcc1938bee15b3fc0bfb796ce4c7de
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 14:08:30 2014 -0700
+
+    OSD: use OpRequestRef& for a few require_* functions
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit eb2f1ea2c33647934af878b504383829f5a198ce)
+
+commit 8d395f1cbbc63f4cd5d166fd0a667938c83f303c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Jul 22 16:57:00 2014 -0700
+
+    OSD: introduce require_up_osd_peer() function for gating replica ops
+
+    This checks both that a Message originates from an OSD, and that the OSD
+    is up in the given map epoch.
+    We use it in handle_replica_op so that we don't inadvertently add operations
+    from down peers, who might or might not know it.
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ccd0eec50103b919b3eb6eea96f7dc6438520ed3)
+
+commit 72c2474e05b918535047181d876678a6a077aa1d
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 8 10:16:20 2014 -0400
+
+    init-ceph: conditionally update  after argparsing
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 44ed1885fb588aad7e4729373e06c5933178b5e5)
+
+commit 8d7fb538e0163f828289d7da719692bd36a66f83
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Aug 7 11:11:57 2014 -0400
+
+    make ceph-disk use the new init flag for cluster
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit d0ccb1cbef86b8ccd8967af892e3582ee80edd55)
+
+commit 260a9c84630ca6956eb772d87abdc3bd4c4bb8fc
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Aug 7 10:48:09 2014 -0400
+
+    allow passing a --cluster flag to the init script
+
+    It will fallback to looking at /etc/ceph/$cluster.conf
+    otherwise.
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 23b4915fa252852cf02cc42a1b6143e2cd61b445)
+
+commit cb5ecdce68dcd4e4bc32b5c7c029cfd7ffe7c80e
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:30:30 2014 -0400
+
+    use cluster name when checking the host
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit bdf5f30010117c71a227855a47fe22ab55b3e0b8)
+
+commit 79452540826d0aed7b94d374cc61b5002e89e4a4
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:30:09 2014 -0400
+
+    use  name in /var/lib/ceph paths
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 3fb25c28e8e3cc62bc541538cc11f5ed41471611)
+
+commit 92755d1060ea06f13577cd2fffda6c7984a610b2
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:29:42 2014 -0400
+
+    when starting an OSD, pass in the cluster name
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 9e08e3a7794ce007c80c8fdb56e96900eb5240a3)
+
+commit 272b26f5aab71c22a1954ff92548b535a1f018d8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Aug 1 14:04:35 2014 -0700
+
+    osd_types: s/stashed/rollback_info_completed and set on create
+
+    Originally, this flag indicated that the object had already been stashed and
+    that therefore recording subsequent changes is unecessary.  We want to set it
+    on create() as well since operations like [create, writefull] should not need
+    to stash the object.
+
+    Fixes: #8625
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4260767973d96978e808cb87ef6ae991104b4f8d)
+
+commit 7b5bcb943578b99f5a2aeeae0bbc67f371c30233
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 19 18:27:49 2014 +0100
+
+    osd: use appropriate json types instead of stream when dumping info
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 363496ad878c685008ccd87f26f121d62a48c24c)
+
+commit daec3226e073b8d6047eb9111f3deb39b1001647
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 19 16:52:06 2014 +0100
+
+    osd: have 'tid' dumped as a json unsigned int where appropriate
+
+    Most of the places dumping it would dump it as a string. Others as an int.
+    Just a couple would dump it as an unsigned int.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3f0ea9512be78d10ac0bc605a38e83cfbab7ffee)
+
+commit 439bce34bc25cd7c8eba7ffc124185d810d3711f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Aug 2 13:01:05 2014 -0700
+
+    rgw: need to pass need_to_wait for throttle_data()
+
+    need_to_wait wasn't passed into processor->throttle_data(). This was
+    broken in fix for #8937.
+
+    CID 1229541:    (PW.PARAM_SET_BUT_NOT_USED)
+
+    Backport: firefly
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e93818df33286a2a7f73b593dc20da412db4e0a6)
+    (cherry picked from commit 8519e9ab06caf6999e1243514a93eaffb155ab2e)
+
+commit fc1a4b5fdce7b92ba1b321dec37c4c75501e3521
+Author: Sylvain Munaut <s.munaut@whatever-company.com>
+Date:   Thu Jun 5 11:28:27 2014 +0200
+
+    rgw: Don't send error body when it's a HEAD request
+
+    The main 'operation' know not to send any body for HEAD requests.
+
+    However for errors, this was not the case, the formatter would be
+    flushed and would send the error 'message' in the body in all cases.
+
+    For the FastCGI case it doesn't seem to be an issue, it's possible
+    that the webserver (apache/lighttpd/...) cleans up the response
+    into shape.
+
+    But when using the new civetweb frontend this cause invalid HTTP.
+
+    Backport: firefly
+    Fixes #8539
+
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    (cherry picked from commit 0a2b4c25541bbd15776d3d35986518e37166910f)
+
+commit e3bc15343720d0455e32e429e7cf865037bd90d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 13:57:34 2014 -0700
+
+    osd: prevent old clients from using tiered pools
+
+    If the client is old and doesn't understand tiering, don't let them use a
+    tiered pool.  Reply with EOPNOTSUPP.
+
+    Fixes: #8714
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0190df53056834f219e33ada2af3a79e8c4dfb77)
+
+commit f3113e9adde54fa50a64010381f5b6c113f5cda0
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 11:02:55 2014 -0700
+
+    mon/OSDMonitor: warn when cache pools do not have hit_sets configured
+
+    Give users a clue when cache pools are enabled but the hit_set is not
+    configured.  Note that technically this will work, but not well, so for
+    now let's just steer them away.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 383536a2ef2506c5591b25c0fd61dc8c181fdab9)
+
+    NOTE: removed readforward state for backport
+
+commit aaf732f7475c075f8bf8e79bc08b11a020e5fadf
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:28:12 2014 -0700
+
+    osd/ReplicatedPG: improve agent_choose_mode args
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit caf554bf2f752b50670d118c99843b7c4a2967b4)
+
+commit 76a519b215e591e0948d627fb1de826ed0bf415d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:26:03 2014 -0700
+
+    osd/ReplicatedPG: evict blindly if there is no hit_set
+
+    If there is no hit set for a PG, blindly evict objects.  This avoids an
+    assert(hit_set) in agent_estimate_atime_temp().
+
+    Fixes: #8982
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ea4996d3ef15c5d5a776d162b08d9fad6859c976)
+
+commit 8ff48d79697e98dca04799ee5982a5308a4efc92
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:13:11 2014 -0700
+
+    osd/ReplicatedPG: check agent_mode if agent is enabled but hit_sets aren't
+
+    It is probably not a good idea to try to run the tiering agent without a
+    hit_set to inform its actions, but it is technically possible.  For
+    example, one could simply blindly evict when we reach the full point.
+    However, this doesn't work because the agent mode is guarded by a hit_set
+    check, even though agent_setup() is not.  Fix that.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5d1c76f641310f5f65600f70ae76945b2aa472d7)
+
+commit f98a6194a9a9f5794c38814c1c33d13b7fcea538
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 07:05:34 2014 -0700
+
+    common/LogClient: fix sending dup log items
+
+    We need to skip even the most recently sent item in order to get to the
+    ones we haven't sent yet.
+
+    Fixes: #9080
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 057c6808be5bc61c3f1ac2b956c1522f18411245)
+
+commit bbb1e6a3a84b2d853f935687f01016e7c6a67951
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Aug 6 15:10:14 2014 +0530
+
+    RadosClient: Fixing potential lock leaks.
+
+    In lookup_pool and pool_delete, a lock is taken
+    before invoking wait_for_osdmap, but is not
+    released for the failure case of the call. Fixing the same.
+
+    Fixes: #9022
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit f1aad8bcfc53f982130dbb3243660c3c546c3523)
+
+commit e767254ce9a1610f3d640d8157fcb0d1fdf8edf2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 11 16:41:26 2014 -0700
+
+    librbd: fix error path cleanup for opening an image
+
+    If the image doesn't exist and caching is enabled, the ObjectCacher
+    was not being shutdown, and the ImageCtx was leaked. The IoCtx could
+    later be closed while the ObjectCacher was still running, resulting in
+    a segfault. Simply use the usual cleanup path in open_image(), which
+    works fine here.
+
+    Fixes: #8912
+    Backport: dumpling, firefly
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3dfa72d5b9a1f54934dc8289592556d30430959d)
+
+commit dec028d113ab4bcc7106fea0bf413b54829f67e7
+Merge: e087862 c7fbf27
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 14:38:33 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly-next' into firefly
+
+commit 07e1ceb55ac3cf0bea58cec27737e45391de9106
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 10 17:10:04 2014 +0200
+
+    erasure-code: ErasureCodeJerasure::create_ruleset must return a ruleset
+
+    CrushWrapper::add_simple_ruleset does not return a ruleset, it returns
+    a ruleid that must be converted into a ruleset before being returned.
+
+    http://tracker.ceph.com/issues/9044 Fixes: #9044
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 0029a35872d3fc15f9a0d60d095b2e111d6e98a6)
+
+commit 96047f1de4e0591a294e31ed6af6edfb3b2357e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 10 17:06:33 2014 +0200
+
+    erasure-code: OSDMonitor::crush_ruleset_create_erasure needs ruleset
+
+    When OSDMonitor::crush_ruleset_create_erasure checks the ruleset for
+    existence, it must convert the ruleid into a ruleset before assigning it
+    back to the *ruleset parameter.
+
+    http://tracker.ceph.com/issues/9044 Fixes: #9044
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 04a484a4d5800c9bcf9805d3e87ed0c8dee01c9a)
+
+commit e087862d5bd894572a6c223b45f4275cadd4b480
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Jul 14 14:27:17 2014 +0800
+
+    Add rbdcache max dirty object option
+
+    Librbd will calculate max dirty object according to rbd_cache_max_size, it
+    doesn't suitable for every case. If user set image order 24, the calculating
+    result is too small for reality. It will increase the overhead of trim call
+    which is called each read/write op.
+
+    Now we make it as option for tunning, by default this value is calculated.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3c7229a2fea98b30627878c86b1410c8eef2b5d7)
+
+commit 5f74fb294a0cd9b973aae45de17f38436bac6d61
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 4 23:22:18 2014 +0200
+
+    librbd/internal.cc: check earlier for null pointer
+
+    Fix potential null ponter deref, move check for 'order != NULL'
+    to the beginning of the function to prevent a) deref in ldout() call
+    and b) to leave function as early as possible if check fails.
+
+    [src/librbd/internal.cc:843] -> [src/librbd/internal.cc:865]: (warning)
+     Possible null pointer dereference: order - otherwise it is redundant
+     to check it against null.
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3ee3e66a9520a5fcafa7d8c632586642f7bdbd29)
+
+commit b2189465f36fc375b864f88a47d25c7808c08901
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:47:24 2014 -0700
+
+    librbd: add an interface to invalidate cached data
+
+    This is useful for qemu to guarantee live migration with caching is
+    safe, by invalidating the cache on the destination before starting it.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5d340d26dd70192eb0e4f3f240e3433fb9a24154)
+
+commit f7dc58552a9ee228b7fa2a10094e962ed66fe8fd
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:43:35 2014 -0700
+
+    librbd: check return code and error out if invalidate_cache fails
+
+    This will only happen when shrinking or rolling back an image is done
+    while other I/O is in flight to the same ImageCtx. This is unsafe, so
+    return an error before performing the resize or rollback.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e08b8b66c77be3a3d7f79d91c20b1619571149ee)
+
+commit c7fbf272a202cd8a72d91b7a55e53fbb5cbbbadd
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 9 13:23:06 2014 -0700
+
+    os/FileStore: dump open fds before asserting
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e8de1792731cf30f2744ab0659d036adc0565a3)
+
+commit 978f5749f3d08bb4c96529ca2d1b10760ee2606d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 13:51:45 2014 -0700
+
+    ceph_test_rados_api_tier: do fewer writes in HitSetWrite
+
+    We don't need to do quite so many writes.  It can be slow when we are
+    thrashing and aren't doing anything in parallel.
+
+    Fixes: #8932
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c5f766bb16c0ab3c3554e73791ad0b74077ad35c)
+
+commit 22aa81413e7db43418e613aea037e978031f5b8a
+Merge: b65cef6 7025075
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 8 19:05:37 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly-next' into firefly
+
+commit cb48cd47a26d27869884af83012d6a8a684abf25
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:50:37 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: verify ceph -c works with daemon
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit aa9ae1f270293778aa937e7f7e4bcaee3099b9b2)
+
+commit 1ac0e5436da68e441f42a1016b12481046147850
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:49:02 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: typo
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 22d20f39b7355966554319d5a1aa888967607569)
+
+commit a144950624208d8ed2901fd7421060a706ac2016
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:48:28 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: allow local ceph command
+
+    (cherry picked from commit 97a8d5a9fdbd3a25cc922c242ee57da58c57d0bc)
+
+commit 76b593f6268d77e6cf5a9931f6f3524b3f574c09
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 28 21:28:42 2014 -0700
+
+    ceph.in: Pass global args to ceph-conf for proper lookup
+
+    Fixes: #8944
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6d89a99648630f81b85ad115fe7662dba6b08a55)
+
+commit b65cef678777c1b87d25385595bf0df96168703e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 14 08:04:16 2014 -0700
+
+    ceph_test_rados_api_tier: fix [EC] HitSet{Read,Write,Trim} tests
+
+    The hit_set_ fields can only be set on tier pools as of
+    f131dfbaedf6f451572e7aa3a83f653912122953.
+
+    Fixes: #8823
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e17e9d857722ee478abda10adb32e15b11fff2ff)
+
+commit 70250755e4b2c114f1f0f463b42930dd99751d81
+Merge: 6f1a544 f8494c8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Aug 3 12:47:23 2014 -0700
+
+    Merge branch 'wip-8701-firefly' into firefly-next
+
+commit f8494c87e5276f0587c3341ddef9d7a933dfea11
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 06:53:41 2014 -0700
+
+    ceph_test_objectstore: clean up on finish of MoveRename
+
+    Otherwise, we leave collections around, and the next test fails.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4faf747b73e70dff9cb5c98ee6aaa4ecec215fc)
+
+commit 94fee997204d439abfd642be20a7e33df5979342
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 21 13:45:21 2014 -0700
+
+    os/LFNIndex: use FDCloser for fsync_dir
+
+    This prevents an fd leak when maybe_inject_failure() throws an exception.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3ec9a42b470422b1fe72b6294d82d9efcaca7f53)
+
+commit b213866eac422a6ac6aeacd3ab4741a6421ce20c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 23:16:09 2014 -0700
+
+    os/LFNIndex: only consider alt xattr if nlink > 1
+
+    If we are doing a lookup, the main xattr fails, we'll check if there is an
+    alt xattr.  If it exists, but the nlink on the inode is only 1, we will
+    kill the xattr.  This cleans up the mess left over by an incomplete
+    lfn_unlink operation.
+
+    This resolves the problem with an lfn_link to a second long name that
+    hashes to the same short_name: we will ignore the old name the moment the
+    old link goes away.
+
+    Fixes: #8701
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fb3260d59faab1e20ebf1e44f850f85f6b8342a)
+
+commit 4fc72d8d936ac499fbb53a1d73a3f7c05ebe596a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 17:28:18 2014 -0700
+
+    os/LFNIndex: remove alt xattr after unlink
+
+    After we unlink, if the nlink on the inode is still non-zero, remove the
+    alt xattr.  We can *only* do this after the rename or unlink operation
+    because we don't want to leave a file system link in place without the
+    matching xattr; hence the fsync_dir() call.
+
+    Note that this might leak an alt xattr if we happen to fail after the
+    rename/unlink but before the removexattr is committed.  We'll fix that
+    next.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ec36f0a130d67df6cbeefcc9c2d83eb703b6b28c)
+
+commit 816ed7e951b32bb9ac57466c0affaec9781ac7f4
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 21 13:43:42 2014 -0700
+
+    os/LFNIndex: FDCloser helper
+
+    Add a helper to close fd's when we leave scope.  This is important when
+    injecting failures by throwing exceptions.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a320c260a9e088ab0a4ea3d5298c06a2d077de37)
+
+commit cbfbe637851c7ebe4a9ec1fd6e429cdf85aef608
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 17:09:07 2014 -0700
+
+    os/LFNIndex: handle long object names with multiple links (i.e., rename)
+
+    When we rename an object (collection_move_rename) to a different name, and
+    the name is long, we run into problems because the lfn xattr can only track
+    a single long name linking to the inode.  For example, suppose we have
+
+    foobar -> foo_123_0 (attr: foobar) where foobar hashes to 123.
+
+    At first, collection_add could only link a file to another file in a
+    different collection with the same name. Allowing collection_move_rename
+    to rename the file, however, means that we have to convert:
+
+    col1/foobar -> foo_123_0 (attr: foobar)
+
+    to
+
+    col1/foobaz -> foo_234_0 (attr: foobaz)
+
+    This is a problem because if we link, reset xattr, unlink we end up with
+
+    col1/foobar -> foo_123_0 (attr: foobaz)
+
+    if we restart after we reset the attr.  This will cause the initial foobar
+    lookup to since the attr doesn't match, and the file won't be able to be
+    looked up.
+
+    Fix this by allow *two* (long) names to link to the same inode.  If we
+    lfn_link a second (different) name, move the previous name to the "alt"
+    xattr and set the new name.  (This works because link is always followed
+    by unlink.)  On lookup, check either xattr.
+
+    Don't even bother to remove the alt xattr on unlink.  This works as long
+    as the old name and new name don't hash to the same shortname and end up
+    in the same LFN chain.  (Don't worry, we'll fix that next.)
+
+    Fixes part of #8701
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+    (cherry picked from commit b2cdfce6461b81f4926602a8c63b54aa92684e6c)
+
+commit 5db6c12b61e00b0bc8084ead5976a912ece0fc65
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 15:46:58 2014 -0700
+
+    ceph_test_objectstore: fix warning
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf98805c09a38cce78ac08317899dc4152ae55a5)
+
+commit e18777ed6eb492e8421d53090b3c9af6e75bd792
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 15 14:50:33 2014 -0700
+
+    store_test: add long name collection_move_rename tests
+
+    Currently fails.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6aa48a485e03ca100f3d9ebec77cc06f99756cd7)
+
+    Conflicts:
+	src/test/objectstore/store_test.cc
+
+commit 4e03d5b512c8d2f7fa51dda95c6132e676529f9b
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 3 08:40:57 2014 -0700
+
+    Revert "enforce rados put aligment"
+
+    This reverts commit 7a58da53ebfcaaf385c21403b654d1d2f1508e1a.
+
+    This was alread backported in dece65064d949b5afcc359cd408615883b5e002a.
+
+    Fixes: #8996
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6f1a54436c63a4cb63712936fccade3267d03db8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 29 15:25:47 2014 -0700
+
+    rgw: fix crash in swift CORS preflight request
+
+    Fixes: #8586
+
+    This fixes error handling, in accordance with commit 6af5a537 that fixed
+    the same issue for the S3 case.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 18ea2a869791b4894f93fdafde140285f2e4fb65)
+
+commit dfcd99567e925bd6565ef4676c560f6c8c89e683
+Author: devicenull <dn@devicenull.org>
+Date:   Fri Jul 18 10:25:51 2014 -0400
+
+    rgw: fix decoding + characters in URL
+
+    Fixes: #8702
+    Backport: firefly
+
+    Only decode + characters to spaces if we're in a query argument. The +
+    query argument.  The + => ' ' translation is not correct for
+    file/directory names.
+
+    Resolves http://tracker.ceph.com/issues/8702
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Brian Rak <dn@devicenull.org>
+    (cherry picked from commit 4a63396ba1611ed36cccc8c6d0f5e6e3e13d83ee)
+
+commit f9716214a35e3cddd3006f3839f9688c41f0c7ca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 25 20:33:52 2014 -0700
+
+    rgw: call processor->handle_data() again if needed
+
+    Fixes: #8937
+
+    Following the fix to #8928 we end up accumulating pending data that
+    needs to be written. Beforehand it was working fine because we were
+    feeding it with the exact amount of bytes we were writing.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0553890e79b43414cc0ef97ceb694c1cb5f06bbb)
+
+    Conflicts:
+	src/rgw/rgw_rados.h
+
+commit 4cd1b60c043089358ea8e70a51310005d5b68383
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 24 15:30:27 2014 -0700
+
+    rgw: object write should not exceed part size
+
+    Fixes: #8928
+
+    This can happen if the stripe size is not a multiple of the chunk size.
+
+    Backport: firefly
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 14cad5ece7d1de9d93e72acca6d4c3b4a9cfcfa2)
+
+commit f68720a6a61364f72f4e0c62aabea65eabe33f98
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 22 15:30:11 2014 -0700
+
+    rgw: align object chunk size with pool alignment
+
+    Fixes: #8442
+    Backport: firefly
+    Data pools might have strict write alignment requirements. Use pool
+    alignment info when setting the max_chunk_size for the write.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fc83e197ab85355e385c13f2a64957cad7481298)
+
+    Conflicts:
+	src/rgw/rgw_rados.cc
+
+commit 67d73451c893b58634f06ed141792da1bf0fa77c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 11:53:16 2014 -0700
+
+    cls_rgw: fix object name of objects removed on object creation
+
+    Fixes: #8972
+    Backport: firefly, dumpling
+
+    Reported-by: Patrycja Szabłowska <szablowska.patrycja@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0f8929a68aed9bc3e50cf15765143a9c55826cd2)
+
+commit d90148da293b4a3e2a44d4486a6b48e27d35b79b
+Merge: 223a0a9 d48a737
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sat Aug 2 11:28:14 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8438' into firefly-next
+
+    Backport of c5b8d8105d965da852c79add607b69d5ae79a4d4
+
+commit 223a0a9e7ed76fbc114c2f058d3aff93556244e9
+Merge: 84bd3cd d72eec0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sat Aug 2 11:22:47 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-7999' into firefly-next
+
+    Backport of 830940bf242a73403ec1882a489e31f7694b7f7e
+
+commit 84bd3cd79961d1ddce12048f9ae980fc197e6be4
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 28 17:17:23 2014 -0700
+
+    unittest_crush_wrapper: fix build
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f36cffc986c973014c89aa37ca73740b2fc194ca)
+
+commit 4983aa32fe8c64c78957b921f5d31ab07be9f850
+Merge: a2681c9 b6d8fea
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 1 19:26:44 2014 -0700
+
+    Merge pull request #2178 from dachary/wip-erasure-code-profile-default-firefly
+
+    erasure-code: create default profile if necessary (firefly)
+
+commit d89a5a37f2515cc9af1322fd18357e445b94256f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 12:26:44 2014 -0700
+
+    mon: s/%%/%/
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d700076a42a5a5ebe769a8311fd3b52bf2e98cd2)
+
+commit da0de4ee973d9a9c8ee6f9a5c12eb37c83b4d453
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 14:52:06 2014 -0700
+
+    atomic: fix read() on i386, clean up types
+
+    Among other things, fixes #8969
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 96863128e6668257f435c6962263caae0d7d10dd)
+
+commit 1e3b927731104629cae9682b55b8d540ef8af12f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 5 10:43:16 2014 -0700
+
+    include/atomic: make 32-bit atomic64_t unsigned
+
+    This fixes
+
+    In file included from test/perf_counters.cc:19:0:
+    ./common/perf_counters.h: In member function ‘std::pair PerfCounters::perf_counter_data_any_d::read_avg() const’:
+    warning: ./common/perf_counters.h:156:36: comparison between signed and unsigned integer expressions [-Wsign-compare]
+    } while (avgcount2.read() != count);
+    ^
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2081c992bbe3a83d711f465634d19c011d28ea3e)
+
+commit 34ac3060d408604c7082c583641b90f330f12663
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Wed May 21 07:56:43 2014 +1000
+
+    Define AO_REQUIRE_CAS (fixes FTBFS on 'hppa')
+
+     to fix FTBFS due to undeclared atomic functions.
+
+    As reported
+
+        https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=748571
+
+    by John David Anglin <dave.anglin@bell.net>
+
+    ~~~~
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::inc()':
+    ./include/atomic.h:42:36: error: 'AO_fetch_and_add1' was not declared in this scope
+           return AO_fetch_and_add1(&val) + 1;
+                                        ^
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::dec()':
+    ./include/atomic.h:45:42: error: 'AO_fetch_and_sub1_write' was not declared in this scope
+           return AO_fetch_and_sub1_write(&val) - 1;
+                                              ^
+    ./include/atomic.h: In member function 'void ceph::atomic_t::add(size_t)':
+    ./include/atomic.h:48:36: error: 'AO_fetch_and_add' was not declared in this scope
+           AO_fetch_and_add(&val, add_me);
+                                        ^
+    ./include/atomic.h: In member function 'void ceph::atomic_t::sub(int)':
+    ./include/atomic.h:52:48: error: 'AO_fetch_and_add_write' was not declared in this scope
+           AO_fetch_and_add_write(&val, (AO_t)negsub);
+                                                    ^
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::dec()':
+    ./include/atomic.h:46:5: warning: control reaches end of non-void function [-Wreturn-type]
+         }
+         ^
+    make[5]: *** [cls/user/cls_user_client.o] Error 1
+    ~~~~
+
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 74218f3d6ca8ca9943ff9d08b7926e38fb13b329)
+
+commit a0c12b70d93604cde59fc21b4d42cae6bd0f4f26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Mar 31 14:49:50 2014 -0700
+
+    atomic_t: add atomic64_t
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bf3ba6001c7b4cf37edfe6551d3ef298ebcbf421)
+
+commit 5a8c8c01cac7f8ef8b898a88cac96af973350a3d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 13:40:33 2014 -0700
+
+    test/cli-integration/rbd: fix trailing space
+
+    Newer versions of json.tool remove the trailing ' ' after the comma.  Add
+    it back in with sed so that the .t works on both old and new versions, and
+    so that we don't have to remove the trailing spaces from all of the test
+    cases.
+
+    Backport: firefly
+    Fixes: #8920
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 605064dc685aa25cc7d58ec18b6449a3ce476d01)
+
+    Conflicts:
+	src/test/cli-integration/rbd/defaults.t
+
+commit 234b2d1933f7f403c094a3c65bf912fcee688181
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jul 7 15:45:54 2014 +0100
+
+    tests: don't depend on 'data' pool in rbd test
+
+    Since we removed the default 'data' and 'metadata' pools,
+    tests which need a pool should create it themselves.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a7a631d1e284f151e305f770cef2042a1b9f86c0)
+
+commit 751258db76d9f5288ff42e8e961a9ffea1d226de
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Mon Jul 28 16:54:48 2014 +0800
+
+    PGMonitor: fix bug in caculating pool avail space
+
+    Currently for pools with different rules, "ceph df" cannot report
+    right available space for them, respectively. For detail assisment
+    of the bug ,pls refer to bug report #8943
+
+    This patch fix this bug and make ceph df works correctlly.
+
+    Fixes Bug #8943
+
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 04d0526718ccfc220b4fe0c9046ac58899d9dafc)
+
+commit 7b1546cc5c361a1af7f724e3c3b7ca9436bad76f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 28 13:59:47 2014 -0700
+
+    mon: set min_size to data chunk count for erasure pools
+
+    Make the min_size value meaningful for erasure pools.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e06c58c9b8f585d2fe7c97d010aa0aa61c09d609)
+
+commit e859587911edddd3a5b7fd96ee2f6a7ed77478dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 11 13:36:03 2014 -0700
+
+    mon: include 'max avail' in df output
+
+    Include an estimate of the maximum writeable space for each pool.  Note
+    that this value is a conservative estimate for that pool based on the
+    most-full OSD.  It is also potentially misleading as it is the available
+    space if *all* new data were written to this pool; one cannot (generally)
+    add up the available space for all pools.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7a9652b58ea70f9a484a135bde20d872616c5947)
+
+commit a5e0ebc36ccbdf36e733e5eab926e4446a48fbd8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 11 13:31:14 2014 -0700
+
+    mon: right justify df values
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2f63a309df4b7086725949bc0a532595cf927edf)
+
+commit 3d578479420f4b0c153dffe96605bf413070ad45
+Author: John Spray <john.spray@inktank.com>
+Date:   Thu May 8 11:04:46 2014 +0100
+
+    mon: Fix % escaping (\% should be %%)
+
+    Clang's -Wpedantic points this out.
+
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit f0231ef364d531eb60351598c4a0f5fa6efad23c)
+
+    Conflicts:
+	src/mon/DataHealthService.cc
+
+commit 6164c3ef8d9ced38d878c191e4de3d84a8bda45e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 28 13:49:52 2014 -0700
+
+    crush: add get_rule_weight_map
+
+    Calculate a weight map of OSDs for a given rule.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 297f6169feecd20e121d102e1b63a505c8b3e74a)
+
+commit 2f90bfdb854750990881f2f53492687a07b19546
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Jul 9 11:20:36 2014 +0000
+
+    Fix the PG listing issue which could miss objects for EC pool (where there is object shard and generation).
+    Backport: firefly
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+
+    (cherry picked from commit 228760ce3a7109f50fc0f8e3c4a5697a423cb08f)
+
+commit 61c90252c3a496e09962cfda2eb11e6bd112af8a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 14:48:10 2014 -0700
+
+    osd/ReplicatedPG: requeue cache full waiters if no longer writeback
+
+    If the cache is full, we block some requests, and then we change the
+    cache_mode to something else (say, forward), the full waiters don't get
+    requeued until the cache becomes un-full.  In the meantime, however, later
+    requests will get processed and redirected, breaking the op ordering.
+
+    Fix this by requeueing any full waiters if we see that the cache_mode is
+    not writeback.
+
+    Fixes: #8931
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8fb761b660c268e2264d375a4db2f659a5c3a107)
+
+commit f701688a360fd977b5e1f59518622ab6d2299a08
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 14:43:48 2014 -0700
+
+    osd/ReplicatedPG: fix cache full -> not full requeueing when !active
+
+    We only want to do this if is_active().  Otherwise, the normal
+    requeueing code will do its thing, taking care to get the queue orders
+    correct.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 36aaab9eee7ed41a46a4ac27376d630a29de5eb9)
+
+commit a1ddce36ebf1ce479129def364328486fbcc5c86
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:50:37 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: verify ceph -c works with daemon
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit aa9ae1f270293778aa937e7f7e4bcaee3099b9b2)
+
+commit 50a1c08e46eaedd3d81a52f64923228bc8a81c26
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:49:02 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: typo
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 22d20f39b7355966554319d5a1aa888967607569)
+
+commit 4cfb7dbc5c8fe1a28fb945fa12d9795943db1db5
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:48:28 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: allow local ceph command
+
+    (cherry picked from commit 97a8d5a9fdbd3a25cc922c242ee57da58c57d0bc)
+
+commit 120b5d5ad51fa47a2bb0a144821635f5f81b5a2e
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 28 21:28:42 2014 -0700
+
+    ceph.in: Pass global args to ceph-conf for proper lookup
+
+    Fixes: #8944
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6d89a99648630f81b85ad115fe7662dba6b08a55)
+
+commit e0c27cd7fde84944fd350f7df89e79d9a39ace98
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 16 17:55:36 2014 -0700
+
+    qa/workunits/cephtool/test.sh: test osd pool get erasure_code_profile
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce9f12d7a2202948532fed9da4d763ed03f6b8fa)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit cf22759cdf1c0a6d258e5c0a2d32268ce051c742
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 16 17:48:34 2014 -0700
+
+    mon: OSDMonitor: add "osd pool get <pool> erasure_code_profile" command
+
+    Enable us to obtain the erasure-code-profile for a given erasure-pool.
+
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8ebcb79a462de29bcbabe40ac855634753bb2be)
+
+commit b94d2f142db63bdf395d8288f8dcef1558d30eaf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:25:53 2014 -0700
+
+    osd/ReplicatedPG: observe INCOMPLETE_CLONES in is_present_clone()
+
+    We cannot assume that just because cache_mode is NONE that we will have
+    all clones present; check for the absense of the INCOMPLETE_CLONES flag
+    here too.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 63abf11390bb9b8dd604aae2b3e90596f9ab65ac)
+
+commit 22cc1b643f83edbfc0475f6da89cf4bd4c89aae1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:24:51 2014 -0700
+
+    osd/ReplicatedPG: observed INCOMPLETE_CLONES when doing clone subsets
+
+    During recovery, we can clone subsets if we know that all clones will be
+    present.  We skip this on caching pools because they may not be; do the
+    same when INCOMPLETE_CLONES is set.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 41364711a66c89ce2e94435fe0d54eeda6092614)
+
+commit c42232e3539fc4682e8fec3e064b21acbcc996ed
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:23:56 2014 -0700
+
+    osd/ReplicatedPG: do not complain about missing clones when INCOMPLETE_CLONES is set
+
+    When scrubbing, do not complain about missing cloens when we are in a
+    caching mode *or* when the INCOMPLETE_CLONES flag is set.  Both are
+    indicators that we may be missing clones and that that is okay.
+
+    Fixes: #8882
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 956f28721dd98c5fb9eb410f4fe9e320b3f3eed3)
+
+commit bde3a9f0f610e6af30066dd77949a1249cd658bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:21:38 2014 -0700
+
+    osd/osd_types: add pg_pool_t FLAG_COMPLETE_CLONES
+
+    Set a flag on the pg_pool_t when we change cache_mode NONE.  This
+    is because object promotion may promote heads without all of the clones,
+    and when we switch the cache_mode back those objects may remain.  Do
+    this on any cache_mode change (to or from NONE) to capture legacy
+    pools that were set up before this flag existed.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 54bf055c5dadc55acf5731e08712d529b180ffc5)
+
+commit f19aa8ef0a2cbad7af3135defc1fe3485d377409
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jul 11 00:40:47 2014 +0100
+
+    qa/workunits: cephtool: adjust pool name where missing as it has changed
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 50e93c2138978f7f7c2fbafacc1611c8705a8eab)
+
+commit 163440a764a6fac8f2efbee6a8aee5cbc460dfe6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jul 11 00:41:01 2014 +0100
+
+    qa/workunits: cephtool: cleanup after pool creation
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 6cd345732b15e84de17d743e06bc4d85569b79d4)
+
+commit c71c99ff21063e26ebfef431f916463f5606aae2
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 19:22:49 2014 +0100
+
+    qa/workunits: cephtool: pool needs to be a tier to be used as such
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 704b0a33f2071eabeb8c5b000a6805ef6d498961)
+
+commit ba0a9bf013e8d056a4126b6840621e5a3a8b37fc
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 19:22:01 2014 +0100
+
+    qa/workunits: cephtool: test erroneous 'tier remove'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 49db6767152092d503ccf8ead6f7cb069e152a22)
+
+commit ffec081a551817288303cf3fa00c016bbffe114a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jul 3 15:32:46 2014 +0100
+
+    qa/workunits: cephtool: test get/set on both tier and non-tier pools
+
+    Make sure gets and sets of tiering-specific variables succeed on tier
+    pools and fail on non-tier pools.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9fea033f30aec44a3273c623ec6c93eb1d7dd26b)
+
+commit 55166c3a098d8751ed6dafb3541e45a6439c5491
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jul 3 15:24:51 2014 +0100
+
+    qa/workunits: cephtool: split get/set on tier pools from get/set tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit df5944955d96c041e65964a13b802028e9700904)
+
+commit 0425eb1dbd156d03a51e91c6f3079b8a687d62f3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 27 22:20:51 2014 +0100
+
+    qa/workunits: cephtool: test for 'osd pool {get,set}-quota'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b927c0de7d5c7a78bf3c133be52cbc1d769974bb)
+
+commit 68197bc28c8960db67c4246d50eb5146659593bd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Jun 30 17:51:47 2014 +0100
+
+    mon: OSDMonitor: 'osd pool' - if we can set it, we must be able to get it
+
+    Add support to get the values for the following variables:
+     - target_max_objects
+     - target_max_bytes
+     - cache_target_dirty_ratio
+     - cache_target_full_ratio
+     - cache_min_flush_age
+     - cache_min_evict_age
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit ddc04c83ff6842ca0b2f804b46099ea729b9fb6b)
+
+commit 01498ce8c897f7e3b66c8054a8b144a505a33010
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 1 08:42:58 2014 +0200
+
+    qa: support running under non privileged user
+
+    If the test is run against a cluster started with vstart.sh (which is
+    the case for make check), the --asok-does-not-need-root disables the use
+    of sudo and allows the test to run without requiring privileged user
+    permissions.
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 522174b066044e51a7019bd6cad81117e83c394e)
+
+commit 99259dff5bad93dce928c1ec07a16b34e3dac3dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 16:38:10 2014 -0700
+
+    qa/workunits/cephtool/test.sh: sudo ceph daemon
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcc09f93761d46202742ca85cce498a352edd494)
+
+commit 4f911d45632fd23c4124bf6b29287ece473d33b4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 21:25:29 2014 +0100
+
+    qa/workunits: cephtool: fix 'osd bench' test
+
+    Commit 7dc93a9651f602d9c46311524fc6b54c2f1ac595 fixed an incorrect
+    behavior with the OSD's 'osd bench' value hard-caps.  The test wasn't
+    appropriately modified unfortunately.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 48e38ac6323f4a0e06b0dedd37ecd10dc339b1e3)
+
+commit bad2f513c5bed4602005eaedff9384012103aaee
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:48:24 2014 +0100
+
+    qa/workunits: cephtool: only run heap profiler test if tcmalloc enabled
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4b0809a13eb363a15e52a6a57372a0a31a64cef8)
+
+commit 26f9860d19c38595bfced76fce864fe9ad9f3a27
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 23:16:17 2014 +0100
+
+    qa/workunits: cephtool: set +e for the tcmalloc tests
+
+    Avoids failing the tests when tcmalloc is not present
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5c4616e100297ba8639919aca7a9cb59e4bda54a)
+
+commit a0720b608db208ac766df8eea53bdbba31d79492
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:47:31 2014 +0100
+
+    qa/workunits: cephtool: delete unnecessary function calls
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 67255435151627314cc2fc38732d4fb5efddc3cc)
+
+commit 373224729e9ab2a46afb43b1388d37ff951cc42e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:46:44 2014 +0100
+
+    qa/workunits: cephtool: disable bash debug when not running tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 946bd0dad4b027326b03c13405782b99ef0f91b2)
+
+commit 52d733c7202b393b3b7da48e3aa6db275946757a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:39:34 2014 +0100
+
+    qa/workunits: cephtool: allow running individual tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5d26575ef2d31d745ec4aa69ca1501cd76e5e8db)
+
+commit 96e58d6d38969ba8b47b1924205eb34850a24ed6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 20:55:27 2014 +0100
+
+    qa/workunits: cephtool: cleanup state after erasure-code-profile test
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f4184086d0d647e064e34308f678ef889e13c373)
+
+commit 4cd5d179375c06a25f04ef8150e2a53aabf7427e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:17:58 2014 +0100
+
+    qa/workunits: cephtool: add/remove comments
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 780424df3a107c7da57fc28d64f9e7a4bb47f8e8)
+
+commit 5bcf690b1dbf5a2e4388661f9d280b5934db1c9c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 26 00:55:46 2014 +0100
+
+    qa/workunits: cephtool: split into properly indented functions
+
+    The test was a big sequence of commands being run and it has been growing
+    organically for a while, even though it has maintained a sense of
+    locality with regard to the portions being tested.
+
+    This patch intends to split the commands into functions, allowing for a
+    better semantic context and easier expansion.  On the other hand, this
+    will also allow us to implement mechanisms to run specific portions of
+    the test instead of always having to run the whole thing just to test a
+    couple of lines down at the bottom (or have to creatively edit the test).
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3d14a96f4b2b7094d05ead1dec7547d165857e31)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit 2af751807d1b51348302734c46d506db294303fd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:12:55 2014 +0100
+
+    qa/workunits: cephtool: move test line to where it's more appropriate
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 04658b7b2b5f98ae81ffb3f77303745e6d46eb81)
+
+commit 5bf9e3c7b59d7ce5996d4d05312b4a92f34bbcb6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:11:52 2014 +0100
+
+    qa/workunits: cephtool: split into functions
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit db6cc133ba4bb38b3c11eb835fd3983dc2f6b00e)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit c898c9736950437256b8c261106f2827fc160e01
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 20 09:38:39 2014 +0200
+
+    mon: test that pools used in tiers cannot be removed
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 39a4b78177cb9896ff35ab05bcf8774bfc934f3a)
+
+commit 5b72674238db80e62cf3d5cec1d94be5c0102388
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 17:02:23 2014 +0100
+
+    qa/workunits/cephtool: test setting options using SI units
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 38405d3554dfb0caf2b0a2c010b95a61bdb99349)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit aacf67d13f81fcc88881091d175fee9ecb0f8954
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 20:19:19 2014 +0100
+
+    mon: OSDMonitor: be scary about inconsistent pool tier ids
+
+    We may not crash your cluster, but you'll know that this is not something
+    that should have happened.  Big letters makes it obvious.  We'd make them
+    red too if we bothered to look for the ANSI code.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 8e5a8daf98052954f3880d2d3516841b5062466b)
+
+commit af59cbf5854bd0fabeb0ef394d7d5b1207bbd6d3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 19:47:52 2014 +0100
+
+    osd: pg_pool_t: clear tunables on clear_tier()
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 64bdf6c92bc44adad5a49b9dc4f674789cee80b0)
+
+commit 8f788e14d22c913479630bce9f9c502ac900328c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 19:46:48 2014 +0100
+
+    mon: OSDMonitor: limit tier-specific pool set/get on non-tier pools
+
+    Fixes: 8696
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f131dfbaedf6f451572e7aa3a83f653912122953)
+
+commit 4b768a8fe9146e6ec033d4f6780e0ef373fcfd7a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 24 10:06:31 2014 -0700
+
+    mon/OSDMonitor: improve no-op cache_mode set check
+
+    If we have a pending pool value but the cache_mode hasn't changed, this is
+    still a no-op (and we don't need to block).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 67d13d76f5692fa20649ea877f254c34094c11f6)
+
+commit c3f403293c7f8d946f66a871aa015a558120ce78
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sat May 3 03:42:19 2014 +0100
+
+    mon: OSDMonitor: disallow nonsensical cache-mode transitions
+
+    Fixes: 8155
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit fd970bbc95d89bf66c9551feca17ac0afbf4d1e2)
+
+commit d9aacd96acfe24b15c4e3b2441838945de1ab720
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Apr 29 18:03:56 2014 +0100
+
+    mon: OSDMonitor: return immediately if 'osd tier cache-mode' is a no-op
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit d01aa5bff30441eec1ffaa3e59a21187f8478475)
+
+commit 981add9cad5f2db06a4932be228eb0ff30254a9b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 18:01:14 2014 -0700
+
+    osd/ReplicatedPG: debug obc locks
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 356af4bf46d6387e2f1a59646548f9a77e49e5f8)
+
+commit e32e2559c8ff408fff81cce0f15d6601dcd4a2f7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 13:16:11 2014 -0700
+
+    osd/ReplicatedPG: greedily take write_lock for copyfrom finish, snapdir
+
+    In the cases where we are taking a write lock and are careful
+    enough that we know we should succeed (i.e, we assert(got)),
+    use the get_write_greedy() variant that skips the checks for
+    waiters (be they ops or backfill) that are normally necessary
+    to avoid starvation.  We don't care about staration here
+    because our op is already in-progress and can't easily be
+    aborted, and new ops won't start because they do make those
+    checks.
+
+    Fixes: #8889
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fe27823b8459271bf0c0e807493bb7cf1e4559b)
+
+commit 68651e9137df2b1034a5b470bbb2fc1d19d8debb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 13:11:42 2014 -0700
+
+    osd: allow greedy get_write() for ObjectContext locks
+
+    There are several lockers that need to take a write lock
+    because there is an operation that is already in progress and
+    know it is safe to do so.  In particular, they need to skip
+    the starvation checks (op waiters, backfill waiting).
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 09626501d7a0ff964027fd7a534465b76bad23cb)
+
+commit a2681c93fb30b912f653b596e27eb8b1bc5c6b78
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 2 10:38:43 2014 -0700
+
+    qa/workunits/rest/test.py: make osd create test idempotent
+
+    Avoid possibility that we create multiple OSDs do to retries by passing in
+    the optional uuid arg.  (A stray osd id will make the osd tell tests a
+    few lines down fail.)
+
+    Fixes: #8728
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bb3e1c92b6682ed39968dc5085b69c117f43cbb0)
+
+commit 7a58da53ebfcaaf385c21403b654d1d2f1508e1a
+Author: Lluis Pamies-Juarez <lluis@pamies.cat>
+Date:   Wed Jun 18 10:00:09 2014 -0700
+
+    enforce rados put aligment
+
+    Signed-off-by: Lluis Pamies-Juarez <lluis.pamies-juarez@hgst.com>
+    (cherry picked from commit 304b08a23a3db57010078046955a786fe3589ef8)
+    (cherry picked from commit dece65064d949b5afcc359cd408615883b5e002a)
+
+commit 7ae15549a795ffd9ffcd34ea1d5c101b90eee26d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 15:39:40 2014 -0700
+
+    use llrintl when converting double to micro
+
+    This avoids rounding error (noticeable on i386).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80911736bd61b6b88eac0974d24f21c15c5385a4)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc
+
+commit f5fc24368756c794d186a0990241e0dc0adfc03c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 24 18:22:22 2014 -0700
+
+    msg/SimpleMessenger: drop local_conneciton priv link on shutdwon
+
+    This breaks ref cycles between the local_connection and session, and let's
+    us drop the explicit set_priv() calls in OSD::shutdown().
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 63c1711a9e237c14d137131f56751970ad1567b1)
+
+commit b6d8feab85fda97bf3a68980249a71846778a157
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 19 00:49:13 2014 +0200
+
+    erasure-code: create default profile if necessary
+
+    After an upgrade to firefly, the existing Ceph clusters do not have the
+    default erasure code profile. Although it may be created with
+
+       ceph osd erasure-code-profile set default
+
+    it was not included in the release notes and is confusing for the
+    administrator.
+
+    The *osd pool create* and *osd crush rule create-erasure* commands are
+    modified to implicitly create the default erasure code profile if it is
+    not found.
+
+    In order to avoid code duplication, the default erasure code profile
+    code creation that happens when a new firefly ceph cluster is created is
+    encapsulated in the OSDMap::get_erasure_code_profile_default method.
+
+    Conversely, handling the pending change in OSDMonitor is not
+    encapsulated in a function but duplicated instead. If it was a function
+    the caller would need a switch to distinguish between the case when goto
+    wait is needed, or goto reply or proceed because nothing needs to be
+    done. It is unclear if having a function would lead to smaller or more
+    maintainable code.
+
+    http://tracker.ceph.com/issues/8601 Fixes: #8601
+
+    Backport: firefly
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 4e1405e7720eda71a872c991045ac8ead6f3e7d8)
+
+commit 2fff5f031b2cc00b55bd1ada4f83e4838fe59ddd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jan 13 17:54:08 2014 +0100
+
+    common: s/stringstream/ostream/ in str_map
+
+    There is no need to specialize more than ostream : it only makes it
+    impossible to use cerr or cout as a parameter to str_map.
+
+    Reviewed-By: Christophe Courtaut <christophe.courtaut@gmail.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 6aa45b133956b974a992b372496b90c908d94f12)
+
+commit d48a737a6f8daa463ddf82106455b2efe7715a33
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 2 13:44:49 2014 -0700
+
+    osd/: in scrub, check for and remove obsolete rollback objs
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 953c33edb640f6eb0c3f3a0d07c061c0d324242b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 16:53:37 2014 -0700
+
+    osd/: plumb rollback_trim_to through interfaces and messages
+
+    We simply trim the roll back info up to min_last_complete_ondisk
+    since we cannot correctly rollback past that point anyway!
+
+    ReplicatedPG
+    -> PGBackend::submit_transaction
+    -> Backend messages
+    -> PGBackend::Listener::log_operation
+    -> append_log
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit ac11ca40b4f4525cbe9b1778b1c5d9472ecb9efa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 16:27:32 2014 -0700
+
+    PGLog: add rollback_info_trimmed_to_riter to IndexedLog and use in PG
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 2207ed12fa544afc48d6d28f199381ccf0367292
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 15:40:15 2014 -0700
+
+    PGLog: fix logging whitespace in trim
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d72eec0341e02881229d0e4ffeafcae92eb33fb6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jun 27 10:51:13 2014 -0700
+
+    PG: block outgoing peering messages until the last interval is durable
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit f9f89be38333d3817a0eafc52a961365e21e1f3a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed May 21 13:09:33 2014 -0700
+
+    PG: add machinery to temporarily buffer outgoing recovery messages
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit f7498124325fe2269c33e45fae4c8181c8f7be14
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 9 11:00:32 2014 -0700
+
+    os/: add async flush_commit() method to Sequencer and implementations
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 669b60571d6db9a9f74020e6a43b12b82edee41e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 15:18:53 2014 -0700
+
+    PGLog: initialize complete_to
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 8679cdbeea6350fd860a9419f54578a90482895c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 29 12:10:31 2014 -0700
+
+    osd_types: add pg_log_t::rollback_info_trimmed_to
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 3bbe29eb5cca7cc42266a6c5561ce68feb4072de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 20 16:05:01 2014 -0700
+
+    common/Finisher: add queue(list<Context*>&)
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
diff --git a/doc/changelog/v0.80.7.txt b/doc/changelog/v0.80.7.txt
new file mode 100644
index 000000000..fd49a5bfb
--- /dev/null
+++ b/doc/changelog/v0.80.7.txt
@@ -0,0 +1,70 @@
+commit 6c0127fcb58008793d3c8b62d925bc91963672a3 (tag: refs/tags/v0.80.7, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Oct 14 12:10:38 2014 -0700
+
+    0.80.7
+
+commit fa599aad2d79e0bbc19e4f6a2ac6771b37bf6beb
+Merge: 805a4ff b8d2fc7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 13 10:34:47 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-7588-firefly' into firefly
+
+commit 805a4ff3f32c60798d01beef80e0702a4e9d82ee
+Merge: 4a3bc78 c5fd2d0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 13 10:34:34 2014 -0700
+
+    Merge remote-tracking branch 'upstream/wip-9696-firefly' into firefly
+
+commit b8d2fc72ea54eb17611d7ac90be4da6c4e4e7485
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 14:19:22 2014 -0700
+
+    osd/ReplicatedPG: carry CopyOpRef in copy_from completion
+
+    There is a race with copy_from cancellation.  The internal Objecter
+    completion decodes a bunch of data and copies it into pointers provided
+    when the op is queued.  When we cancel, we need to ensure that we can cope
+    until control passes back to our provided completion.
+
+    Once we *do* get into the (ReplicatedPG) callbacks, we will bail out
+    because the tid in the CopyOp or FlushOp no longer matches.
+
+    Fix this by carrying a ref to keep the copy-from targets alive, and
+    clearing out the tids that we cancel.
+
+    Note that previously, the trigger for this was that the tid changes when
+    we handle a redirect, which made the op_cancel() call fail.  With the
+    coming Objecter changes, this will no longer be the case.  However, there
+    are also locking and threading changes that will make cancellation racy,
+    so we will not be able to rely on it always preventing the callback.
+    Either way, this will avoid the problem.
+
+    Fixes: #7588
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 589b639af7c8834a1e6293d58d77a9c440107bc3)
+
+commit c5fd2d043ed4aa4fdb60fc19a284f51a86cef408 (refs/remotes/gh/wip-9696-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 9 16:21:18 2014 -0700
+
+    PG::choose_acting: in mixed cluster case, acting may include backfill
+
+    Fixes: 9696
+    Backport: firefly, giant
+    Introduced: 92cfd370395385ca5537b5bc72220934c9f09026
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 9b18d99817c8b54e30dff45047dfe1b29871d659)
+
+commit 4a3bc78c033ed68183f91824c7942dc54afb8797 (refs/remotes/gh/wip-9731-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 10 13:53:29 2014 -0700
+
+    PGLog::IndexedLog::trim(): rollback_info_trimmed_to_riter may be log.rend()
+
+    Fixes: #9731
+    Backport: giant, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d458b4f0f31161f561ff98e58ed979cf20c6f588)
diff --git a/doc/changelog/v0.80.8.txt b/doc/changelog/v0.80.8.txt
new file mode 100644
index 000000000..849e933b1
--- /dev/null
+++ b/doc/changelog/v0.80.8.txt
@@ -0,0 +1,2547 @@
+commit 69eaad7f8308f21573c604f121956e64679a52a7 (tag: refs/tags/v0.80.8)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jan 13 06:28:08 2015 -0800
+
+    0.80.8
+
+commit 3c7cacf00b66a0e17272cf67fe0823ee863dfa7c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 19 16:43:02 2014 -0700
+
+    mon/OSDMonitor: fix double-free on old MOSDBoot
+    
+    send_latest() does an m->put().
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 97f9b6df668315fba6a5924b79024c7a986f4110)
+
+commit 852d7b5b3c019c02c042b767fc88916088e1a94d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 11:17:03 2015 -0800
+
+    osd: requeue PG when we skip handling a peering event
+    
+    If we don't handle the event, we need to put the PG back into the peering
+    queue or else the event won't get processed until the next event is
+    queued, at which point we'll be processing events with a delay.
+    
+    The queue_null is not necessary (and is a waste of effort) because the
+    event is still in pg->peering_queue and the PG is queued.
+    
+    This is a firefly-specific patch; a (very) similar one will appear in master
+    in 492ccc900c3358f36b6b14a207beec071eb06707.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c26ebd38335bb361aade5aacd05ba3217e602b9c
+Merge: b67b7e7 96ba529
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 18:31:26 2015 -0800
+
+    Merge pull request #3217 from boydc2014/firefly
+    
+    clear data and payload after removed from ops_in_flight
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b67b7e7ad72a1af96f9fb26ade815e65f72b4cb0
+Merge: db92718 7faae89
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:32:17 2015 +0100
+
+    Merge pull request #3127 from ktdreyer/firefly-no-epoch
+    
+    Revert "ceph.spec.: add epoch"
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit db927186288cd4c63a3483b42e9eb9e016c96156
+Merge: 0d4abda 820dbfd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:31:29 2015 +0100
+
+    Merge pull request #3128 from dachary/wip-10281-make-check-fedora-20
+    
+    tests: fixes to run make check on fedora 20
+    
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d4abdaf80f1fedff7975d595abaac0a620c8035
+Merge: 1fdcb52 11995b3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:30:59 2015 +0100
+
+    Merge pull request #3169 from ceph/wip-8797-firefly
+    
+    Wip 8797 firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1fdcb524411a02b5627be66d9fd821a473223e9d
+Merge: 4b7b1b0 465eede
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:28:49 2015 +0100
+
+    Merge pull request #3179 from dachary/wip-9998-crush-underfloat-firefly
+    
+    crush: fix weight underfloat issue (firefly)
+    
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4b7b1b03bfcb7cb056783555884f211009ea5d46
+Merge: 4897ba4 f55b097
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:28:11 2015 +0100
+
+    Merge pull request #3220 from ceph/wip-mon-backports.firefly
+    
+    mon: backports for #9987 against firefly
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4897ba4a304bcac548b2121312cd7235c34dd5aa
+Merge: efe801f f20225c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:26:30 2015 +0100
+
+    Merge pull request #3258 from ceph/wip-10372-firefly
+    
+    osd: fix librados pool deletion race on firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit efe801ff3a0d25243da18937c07b89227edbaac4
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Dec 18 20:00:28 2014 -0800
+
+    If trusty, use older version of qemu
+    
+    Fixes #10319
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry-picked from 46a1a4cb670d30397979cd89808a2e420cef2c11)
+
+commit 96ba529ef8ce76e07b8444c94883afb3468d6762
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Tue Dec 30 21:02:45 2014 +0800
+
+    clear data and payload inside ops_in_flight_lock
+    
+    http://tracker.ceph.com/issues/9916 Fixes: #9916
+    Signed-off-by: Dong Lei <leidong@yahoo-inc.com>
+
+commit e0648e3d30de504b096c4ae3bbe7d9c17652bdb5
+Merge: 455f940 3624006
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 14:31:23 2014 -0800
+
+    Merge pull request #3264 from dachary/wip-jerasure-firefly
+    
+    erasure-code: update links to jerasure upstream
+
+commit 455f940908f242b1e34983af61351fd3045ce8ab
+Merge: b1ab685 aa95a2d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 10:55:39 2014 -0800
+
+    Merge pull request #3268 from ceph/firefly-10415
+    
+    libcephfs/test.cc: close fd before umount
+
+commit 362400667aad0b5098fbe8dbec1b0bde059f84a6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Dec 28 10:29:54 2014 +0100
+
+    erasure-code: update links to jerasure upstream
+    
+    It moved from bitbucket to jerasure.org
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e86f901939f16cc9c8ad7a4108ac4bcf3916d2c)
+
+commit aa95a2d20dbba2f3a775f709493c987d0d001e9c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Dec 23 10:22:00 2014 +0800
+
+    libcephfs/test.cc: close fd before umount
+    
+    Fixes: #10415
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3fb563cee4c4cf08ff4ee01782e52a100462429)
+
+commit f20225cb99a0d2d08fccfdf88dc89d758ecba077 (refs/remotes/gh/wip-10372-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 23 15:49:26 2014 -0800
+
+    osdc/Objecter: handle reply race with pool deletion
+    
+    We need to handle this scenario:
+    
+     - send request in epoch X
+     - osd replies
+     - pool is deleted in epoch X+1
+     - client gets map X+1, sends a map check
+     - client handles reply
+       -> asserts that no map checks are in flight
+    
+    This isn't the best solution.  We could infer that a map check isn't needed
+    since the pool existed earlier and doesn't now.  But this is firefly and
+    the fix is no more expensive than the old assert.
+    
+    Fixes: #10372
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 820dbfd9947455d07426981b7152861c3c216080
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 5 15:11:18 2014 -0700
+
+    test/ceph-disk.sh: mkdir -p
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2f58e6694a2457200ab3d59e037ad17b9c82028)
+
+commit 77d393024f9d867b574b8ec8e15ec48a1a291511
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Sep 4 12:23:27 2014 +0200
+
+    test/ceph-disk.sh: resolve symlinks before check
+    
+    Make sure symlinks are resolved in command_fixture()
+    before compare result of which command and the current
+    path.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8ea86dfa7c4a3d7e089cf9d4e49586657875f851)
+
+commit ed6ec2936513d7dd6c45bccd8edf69a12c71dc7b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jun 24 19:54:17 2014 +0200
+
+    test/ceph-disk.sh: fix for SUSE
+    
+    On SUSE 'which' returns always the full path of (shell) commands and
+    not e.g. './ceph-conf' as on Debian. Add check also for full
+    path returned by which.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 39530536ff923b91899f6303507c283b78040a20)
+
+commit 754363f4563e7dbda1ef23fadc8d6ef1a3fdd0af
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 13 14:41:39 2014 +0200
+
+    tests: prevent kill race condition
+    
+    When trying to kill a daemon, keep its pid in a variable instead of
+    retrieving it from the pidfile multiple times. It prevents the following
+    race condition:
+    
+      * try to kill ceph-mon
+      * ceph-mon is in the process of dying and removed its pidfile
+      * try to kill ceph-mon fails because the pidfile is not found
+      * another ceph-mon is spawned and fails to bind the port
+        because the previous ceph-mon is still holding it
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a1c13c57ba20fc329d943ea57523913e11067dc7)
+
+commit 5be6f2f60e3225bf3d214432044721fe474d55d7
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 25 00:31:48 2014 +0200
+
+    osd/OSD.cc: parse lsb release data via lsb_release
+    
+    Use lsb_release tool to be portable since parsing /etc/lsb-release
+    is not the same between different distributions. The old code failed
+    e.g. for SUSE products to parse LSB information.
+    
+    Fixes: #8654
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0b3a3987d382ff33fdf892f189b30df22be80e59)
+
+commit b62187c52324a4489c2fc1385b1d6574a058f7e8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 26 21:59:39 2014 +0200
+
+    tests: histogram prevent re-use of local variables
+    
+    By moving the tests to separate functions.
+    
+    http://tracker.ceph.com/issues/9235 Fixes: #9235
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 4b8b25ecd128c34a386ad7c4cc89f323c4d384e1)
+
+commit e2741c8f1d42cfe91b18201a6a49005d90d85d98
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 26 21:59:39 2014 +0200
+
+    tests: histogram prevent re-use of local variables
+    
+    By moving the test to a separate function.
+    
+    http://tracker.ceph.com/issues/9235 Fixes: #9235
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit ee02293ad2ef050672fa8c164ba17b10e8d4ceeb)
+
+commit cf102df7a3f3b38824c26b9e44a21664fcf979a7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 10 00:08:57 2014 +0100
+
+    tests: avoid tab interpretation problem on fedora 20
+    
+    Use . instead of tab in echo to avoid variations in how escape sequences
+    are interpreted by the shell.
+    
+    http://tracker.ceph.com/issues/10281 Fixes: #10281
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit f55b097764beb973c12866f2d7161c6bd870aa07 (refs/remotes/gh/wip-mon-backports.firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:50:59 2014 -0800
+
+    mon/PGMap and PGMonitor: update last_epoch_clean cache from new osd keys
+    
+    We were only invalidating the cached value from apply_incremental, which
+    is no longer called on modern clusters.
+    
+    Fix this by storing the update epoch in the key as well (it is not part
+    of osd_stat_t).
+    
+    Backport: giant, firefly, dumpling(?)
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 093c5f0cabeb552b90d944da2c50de48fcf6f564)
+
+commit 1d314e7a9ab7af5b693583cf2faa5db54f6beb69
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:49:48 2014 -0800
+
+    mon/PGMap: invalidate cached min_last_epoch_clean from new-style pg keys
+    
+    We were only invalidating the cache from the legacy apply_incremental(),
+    which is no longer called on modern clusters.
+    
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3fb731b722c50672a5a9de0c86a621f5f50f2d06)
+
+commit 465eedea9f7411b1e352dc3ccee60a3f1221541d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 23 18:50:51 2014 -0800
+
+    crush/CrushWrapper: fix create_or_move_item when name exists but item does not
+    
+    We were using item_exists(), which simply checks if we have a name defined
+    for the item.  Instead, use _search_item_exists(), which looks for an
+    instance of the item somewhere in the hierarchy.  This matches what
+    get_item_weightf() is doing, which ensures we get a non-negative weight
+    that converts properly to floating point.
+    
+    Backport: giant, firefly
+    Fixes: #9998
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9902383c690dca9ed5ba667800413daa8332157e)
+
+commit cee51af9c1dbde550177c95caf6c93f612442300
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:47:56 2014 -0800
+
+    crush/builder: prevent bucket weight underflow on item removal
+    
+    It is possible to set a bucket weight that is not the sum of the item
+    weights if you manually modify/build the CRUSH map.  Protect against any
+    underflow on the bucket weight when removing items.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8c87e9502142d5b4a282b94f929ae776a49be1dc)
+
+commit 7ccd5eec11e8cd945d24bf9f6390d6c3fa4a06de
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:37:03 2014 -0800
+
+    crush/CrushWrapper: fix _search_item_exists
+    
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eeadd60714d908a3a033aeb7fd542c511e63122b)
+
+commit b1ab685e00034751a161a3d5e0325c6581999c75
+Merge: dd7c8c2 ef3773a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 12 06:19:50 2014 -0800
+
+    Merge pull request #3124 from ceph/wip-10194-firefly
+    
+    rgw: optionally call FCGX_Free() on the fcgi connection
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11995b329045341c17553269267cfd3688a51b0f
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:53 2014 -0800
+
+    Call Rados.shutdown() explicitly before exit
+    
+    This is mostly a demonstration of good behavior, as the resources will
+    be reclaimed on exit anyway.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit b038e8fbf9103cc42a4cde734b3ee601af6019ea)
+
+commit e00270b51896f168d5013b7dc92ec7f8b9e19da3
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:16 2014 -0800
+
+    rados.py: remove Rados.__del__(); it just causes problems
+    
+    Recent versions of Python contain a change to thread shutdown that
+    causes ceph to hang on exit; see http://bugs.python.org/issue21963.
+    As it turns out, this is relatively easy to avoid by not spawning
+    threads on exit, as Rados.__del__() will certainly do by calling
+    shutdown(); I suspect, but haven't proven, that the problem is
+    that shutdown() tries to start() a threading.Thread() that never
+    makes it all the way back to signal start().
+    
+    Also add a PendingReleaseNote and extra doc comments to clarify.
+    
+    Fixes: #8797
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 5ba9b8f21f8010c59dd84a0ef2acfec99e4b048f)
+    
+    Conflicts:
+    	PendingReleaseNotes
+
+commit 7faae891aefa4c21c50430fa03d9204a86d082f8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Dec 9 14:52:19 2014 -0700
+
+    Revert "ceph.spec.: add epoch"
+    
+    If ICE ships 0.80.8, then it will be newer than what RHCEPH ships
+    (0.80.7), and users won't be able to seamlessly upgrade via Yum.
+    
+    We have three options:
+    A) Revert the "Epoch: 1" change on the Firefly branch.
+    B) Revert the "Epoch: 1" change in the ICE packages.
+    C) Bump the Epoch to "2" in Red Hat's packages.
+    
+    This commit does Option A.
+    
+    Option B may or may not be feasible - it would require a "downstream"
+    change in ICE, and we haven't done that sort of thing before.
+    
+    Due to the RHEL release schedule, Option C is not available to us at
+    this point.
+    
+    This reverts commit b890c1e4706d7cfef7ed24c9df65b439b4f7ff1d.
+
+commit ef3773ac93413c644e056babce1971f846bbc276
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 26 15:18:07 2014 -0800
+
+    rgw: optionally call FCGX_Free() on the fcgi connection
+    
+    Fixes: #10194
+    
+    A new configurable controls this behavior. This forces disconnection of
+    the fcgi connection when done with the request.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit dd7c8c2bc6b2810ff7f483af940fa09dbe74e83a
+Merge: d759467 a597096
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 8 15:02:52 2014 -0800
+
+    Merge pull request #3109 from ceph/firefly-10263
+    
+    mds: store backtrace for straydir
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d7594672b673796901961cc684c9e7de8dc2c95d
+Merge: adf9a75 73b47db
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Dec 8 13:19:44 2014 -0800
+
+    Merge pull request #3009 from dachary/wip-10018-primary-erasure-code-hinfo-firefly
+    
+    osd: deep scrub must not abort if hinfo is missing (firefly)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a5970963a2148697fc6da64facfbf6ab6686b9cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 7 11:38:37 2014 +0800
+
+    mds: store backtrace for straydir
+    
+    Backport: giant, firefly, emperor, dumpling
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0d89db5d3e5ae5d552d4058a88a4e186748ab1d2)
+
+commit adf9a758000182d27a6582d516356730d02e4099
+Merge: dea38a7 b4a4b75
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 11:06:02 2014 -0800
+
+    Merge pull request #3089 from dachary/wip-10063-hobject-shard-firefly
+    
+    common: do not omit shard when ghobject NO_GEN is set (firefly)
+
+commit dea38a7af638c833304272c324ed2bc386a40f8f
+Merge: ccc8b46 5138091
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 10:59:44 2014 -0800
+
+    Merge pull request #2480 from dachary/wip-9420-erasure-code-non-regression-firefly
+    
+    erasure-code: store and compare encoded contents (firefly)
+
+commit ccc8b46b2cdc55c1a861f092259ef36a1296f073
+Merge: bef363c cd3447d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:33:05 2014 -0800
+
+    Merge pull request #3096 from dachary/wip-9785-dmcrypt-keys-permissions-firefly
+    
+    ceph-disk: dmcrypt file permissions (firefly)
+
+commit cd3447d04cabf6745001afeef69f25a92400cd0e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 4 22:21:32 2014 +0100
+
+    ceph-disk: dmcrypt file permissions
+    
+    The directory in which key files are stored for dmcrypt must be 700 and
+    the file 600.
+    
+    http://tracker.ceph.com/issues/9785 Fixes: #9785
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 58682d1776ab1fd4daddd887d921ca9cc312bf50)
+
+commit bef363c1d1fc06fbb315024145a97a809a2471cd
+Merge: cb2c83b 9f3b21d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 09:04:00 2014 -0800
+
+    Merge pull request #3086 from dachary/wip-10125-radosgw-init-firefly
+    
+    rgw: run radosgw as apache with systemd (firefly)
+
+commit b4a4b75e6d4deb6818681902f85baa9f63acdb4f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 01:16:10 2014 +0100
+
+    common: do not omit shard when ghobject NO_GEN is set
+    
+    Do not silence the display of shard_id when generation is NO_GEN.
+    Erasure coded objects JSON representation used by ceph_objectstore_tool
+    need the shard_id to find the file containing the chunk.
+    
+    Minimal testing is added to ceph_objectstore_tool.py
+    
+    http://tracker.ceph.com/issues/10063 Fixes: #10063
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit dcf09aed121f566221f539106d10283a09f15cf5)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit 9f3b21d1b70be591d68bfa57c3393e8f9af8e7df
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 18:10:48 2014 +0100
+
+    rgw: run radosgw as apache with systemd
+    
+    Same as sysv.
+    
+    http://tracker.ceph.com/issues/10125 Fixes: #10125
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7b621f4abf63456272dec3449aa108c89504a7a5)
+    
+    Conflicts:
+    	src/init-radosgw.sysv
+
+commit cb2c83b2f216e503f7a52115f775bda1dbfe0c6a
+Merge: e2ec37b 02d4685
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Dec 4 11:32:18 2014 -0800
+
+    Merge pull request #3078 from ceph/wip-10030-firefly
+    
+    librbd: don't close an already closed parent image upon failure
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e2ec37bf45fa5e7f5e787db9b67dbb2a98f2fbb7
+Merge: c4c63e8 af12194
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 23:01:44 2014 -0800
+
+    Merge pull request #3063 from ceph/wip-10123-firefly
+    
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4c63e82cd7e4716315ca81208293a2567026c72
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 15:52:08 2014 -0700
+
+    ReplicatedPG: don't move on to the next snap immediately
+    
+    If we have a bunch of trimmed snaps for which we have no
+    objects, we'll spin for a long time.  Instead, requeue.
+    
+    Fixes: #9487
+    Backport: dumpling, firefly, giant
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c17ac03a50da523f250eb6394c89cc7e93cb4659)
+
+commit 1b656450ca75b12fb98dee82bace914ef5f45c44
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 23 16:21:33 2014 -0700
+
+    osd: initialize purged_snap on backfill start; restart backfill if change
+    
+    If we backfill a PG to a new OSD, we currently neglect to initialize
+    purged_snaps.  As a result, the first time the snaptrimmer runs it has to
+    churn through every deleted snap for all time, and to make matters worse
+    does so in one go with the PG lock held.  This leads to badness on any
+    cluster with a significant number of removed snaps that experiences
+    backfill.
+    
+    Resolve this by initializing purged_snaps when we finish backfill.  The
+    backfill itself will clear out any stray snaps and ensure the object set
+    is in sync with purged_snaps.  Note that purged_snaps on the primary
+    that is driving backfill will not change during this period as the
+    snaptrimmer is not scheduled unless the PG is clean (which it won't be
+    during backfill).
+    
+    If we by chance to interrupt backfill, go clean with other OSDs,
+    purge snaps, and then let this OSD rejoin, we will either restart
+    backfill (non-contiguous log) or the log will include the result of
+    the snap trim (the events that remove the trimmed snap).
+    
+    Fixes: #9487
+    Backfill: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 255b430a87201c7d0cf8f10a3c1e62cbe8dd2d93)
+
+commit 02d4685c56e129cb179a5ddfb8e87aefc2fce0b5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 6 05:01:38 2014 -0500
+
+    librbd: don't close an already closed parent image upon failure
+    
+    If librbd is not able to open a child's parent image, it will
+    incorrectly close the parent image twice, resulting in a crash.
+    
+    Fixes: #10030
+    Backport: firefly, giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61ebfebd59b61ffdc203dfeca01ee1a02315133e)
+
+commit af121942d7bdfc59fcfae0429ffb12993e7e019d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 17 21:49:26 2014 -0500
+
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    While listing child images, don't ignore error codes returned
+    from librados when creating an IoCtx. This will prevent seg
+    faults from occurring when an invalid IoCtx is used.
+    
+    Fixes: #10123
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d350b6817d7905908a4e432cd359ca1d36bab50)
+
+commit c982da44e0e9e0be3c3d4e8f5e0a186fb2fcebb3
+Merge: 4a148df c4e4a31
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 30 10:12:04 2014 -0800
+
+    Merge pull request #3014 from dachary/wip-9665-ceph-disk-partprobe-firefly
+    
+    ceph disk zap must call partprobe
+
+commit c4e4a310f14ca3049ac90422aea95051fe0d4b15
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:26:31 2014 +0200
+
+    ceph-disk: use update_partition in prepare_dev and main_prepare
+    
+    In the case of prepare_dev the partx alternative was missing and is not
+    added because update_partition does it.
+    
+    http://tracker.ceph.com/issues/9721 Fixes: #9721
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 23e71b1ee816c0ec8bd65891998657c46e364fbe)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit e70a81464b906b9a304c29f474e6726762b63a7c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 9 18:52:17 2014 +0200
+
+    ceph-disk: run partprobe after zap
+    
+    Not running partprobe after zapping a device can lead to the following:
+    
+    * ceph-disk prepare /dev/loop2
+    * links are created in /dev/disk/by-partuuid
+    * ceph-disk zap /dev/loop2
+    * links are not removed from /dev/disk/by-partuuid
+    * ceph-disk prepare /dev/loop2
+    * some links are not created in /dev/disk/by-partuuid
+    
+    This is assuming there is a bug in the way udev events are handled by
+    the operating system.
+    
+    http://tracker.ceph.com/issues/9665 Fixes: #9665
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fed3b06c47a5ef22cb3514c7647544120086d1e7)
+
+commit 5a5f427bc09076ef3fb13a710dede1b47bb232e0
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:23:34 2014 +0200
+
+    ceph-disk: encapsulate partprobe / partx calls
+    
+    Add the update_partition function to reduce code duplication.
+    The action is made an argument although it always is -a because it will
+    be -d when deleting a partition.
+    
+    Use the update_partition function in prepare_journal_dev
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 922a15ea6865ef915bbdec2597433da6792c1cb2)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit 73b47dbee8858f182fd2b4fd8eb5f3c786877bf4
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Nov 6 17:11:20 2014 +0100
+
+    osd: deep scrub must not abort if hinfo is missing
+    
+    Instead it should set read_error.
+    
+    http://tracker.ceph.com/issues/10018 Fixes: #10018
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9d84d2e8309d26e39ca849a75166d2d7f2dec9ea)
+
+commit 5138091a4073d966a65f537280f89372e801d019
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 11:38:09 2014 +0200
+
+    erasure-code: add corpus verification to make check
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 8d3d6bf59aec3877c0231e637270e371d9ed3b8c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 13:36:09 2014 +0200
+
+    erasure-code: workunit to check for encoding regression
+    
+    Clone the archive of encoded objects and decode all archived objects, up
+    to and including the current ceph version.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 4f4358708ed3c261ca4027cc9c3dc3f952a99470
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 10:16:31 2014 +0200
+
+    erasure-code: store and compare encoded contents
+    
+    Introduce ceph_erasure_code_non_regression to check and compare how an
+    erasure code plugin encodes and decodes content with a given set of
+    parameters. For instance:
+    
+    ./ceph_erasure_code_non_regression \
+          --plugin jerasure \
+          --parameter technique=reed_sol_van \
+          --parameter k=2 \
+          --parameter m=2 \
+          --stripe-width 3181 \
+          --create \
+          --check
+    
+    Will create an encoded object (--create) and store it into a directory
+    along with the chunks, one chunk per file. The directory name is derived
+    from the parameters. The content of the object is a random pattern of 31
+    bytes repeated to fill the object size specified with --stripe-width.
+    
+    The check function (--check) reads the object back from the file,
+    encodes it and compares the result with the content of the chunks read
+    from the files. It also attempts recover from one or two erasures.
+    
+    Chunks encoded by a given version of Ceph are expected to be encoded
+    exactly in the same way by all Ceph versions going forward.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 4a148df544978383c1ed7cd8b90f590adb563f3d
+Merge: c069bce 01faf13
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 19 02:45:26 2014 +0100
+
+    Merge pull request #2961 from ceph/wip-10114-firefly
+    
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 01faf1356f648ded9acda02e7cc67c1adb9e9ee3
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Fri Nov 14 17:59:57 2014 -0800
+
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    See discussion in http://tracker.ceph.com/issues/10114
+    
+    Building with these changes allows output from readelf like this:
+    
+     $ readelf -lW src/.libs/librados.so.2 | grep GNU_STACK
+      GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000
+    0x000000 RW  0x8
+    
+    (note the absence of 'X' in 'RW')
+    
+    Fixes: #10114
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 06a245a9845c0c126fb3106b41b2fd2bc4bc4df3)
+    (not-yet-present-in-firefly files in isa-l manually removed)
+
+commit c069bce4e8180da3c0ca4951365032a45df76468
+Merge: 0d8ad6a fac1654
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 13 10:36:12 2014 -0800
+
+    Merge pull request #2760 from ceph/wip-9835-firefly
+    
+    osd: fix erasure hung op bug (9835)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fac165475031efdebbb88898ca5c12cd307a5bc3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Nov 5 12:12:14 2014 -0800
+
+    osd: use OSDMap helper to tell if ops are misdirected
+    
+    calc_pg_role doesn't actually take into account primary affinity.
+    
+    Fixes: #9835
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 588602bf0095de5b59064123ca01345f1364bdde
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 20 13:55:33 2014 -0700
+
+    osd: discard rank > 0 ops on erasure pools
+    
+    Erasure pools do not support read from replica, so we should drop
+    any rank > 0 requests.
+    
+    This fixes a bug where an erasure pool maps to [1,2,3], temporarily maps
+    to [-1,2,3], sends a request to osd.2, and then remaps back to [1,2,3].
+    Because the 0 shard never appears on osd.2, the request sits in the
+    waiting_for_pg map indefinitely and cases slow request warnings.
+    This problem does not come up on replicated pools because all instances of
+    the PG are created equal.
+    
+    Fix by only considering role == 0 for erasure pools as a correct mapping.
+    
+    Fixes: #9835
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 0c1c4152e6f402af7612c8c8d4719ab0f4cc6ad9
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:04:35 2014 -0800
+
+    osd/OSDMap: add osd_is_valid_op_target()
+    
+    Helper to check whether an osd is a given op target for a pg.  This
+    assumes that for EC we always send ops to the primary, while for
+    replicated we may target any replica.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89c02637914ac7332e9dbdbfefc2049b2b6c127d)
+
+commit 0d8ad6ad3c376bcab981bea9a49e1924d7eddb68
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 11 18:16:02 2014 -0800
+
+    qa: allow small allocation diffs for exported rbds
+    
+    The local filesytem may behave slightly differently. This isn't
+    foolproof, but seems to be reliable enough on rhel7 rootfs, where
+    exact comparison was failing.
+    
+    Fixes: #10002
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e94d3c11edb9c9cbcf108463fdff8404df79be33)
+
+commit 0804deeab293e09123d1b58825051ccc4dddbc0e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 25 08:38:38 2014 -0700
+
+    osd: fix map advance limit to handle map gaps
+    
+    The recent change in cf25bdf6b0090379903981fe8cee5ea75efd7ba0 would stop
+    advancing after some number of epochs, but did not take into consideration
+    the possibilty that there are missing maps.  In that case, it is impossible
+    to advance past the gap.
+    
+    Fix this by increasing the max epoch as we go so that we can always get
+    beyond the gap.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1e0a82fd55dede473c0af32924f4bcb5bb697a2b)
+
+commit d30d6b986433eaef920b3703cf5af3c030f8dcf4
+Merge: d241aa7 d548431
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 14:10:18 2014 -0800
+
+    Merge pull request #2880 from ceph/wip-10025-firefly
+    
+    #10025/firefly -- tools: fix MDS journal import
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit d548431a388da1130564d710e1f006772934224b
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 7 11:34:43 2014 +0000
+
+    tools: fix MDS journal import
+    
+    Previously it only worked on fresh filesystems which
+    hadn't been trimmed yet, and resulted in an invalid
+    trimmed_pos when expire_pos wasn't on an object
+    boundary.
+    
+    Fixes: #10025
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fb29e71f9a97c12354045ad2e128156e503be696)
+
+commit d241aa7a347655242cc71b8fa3d778df6948c494
+Merge: 2c85b5d 4afb542
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:37:42 2014 -0800
+
+    Merge remote-tracking branch 'origin/wip-sam-firefly-backports' into firefly
+
+commit 2c85b5d72953d01296213185382707122e06415c
+Merge: 23cbffa 1228658
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:30:20 2014 -0800
+
+    Merge pull request #2737 from ceph/wip-9629-firefly
+    
+    osd: do not clone/preserve snapdir on cache-evict (firefly backport)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 23cbffaa2936dc2707b5b42f8c0e7ce804324ae2
+Merge: 3bba938 e296685
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:26:21 2014 -0800
+
+    Merge pull request #2657 from ceph/wip-9053-9301-firefly
+    
+    mon: backport two paxos fixes to firefly
+    
+    Reviewed-by: Joao Luis <joao@redhat.com>
+
+commit 3bba9387eb123c6cf055e874db2925b998dc406c
+Merge: 3f9bf73 835f8c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:21:12 2014 -0800
+
+    Merge pull request #2656 from ceph/wip-9502-firefly
+    
+    mon: backport mon disk full check to firefly
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3f9bf738daf47ff4ff56c9f76d1487a5afc5e30a
+Merge: a340e6d 3e17a08
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:18:27 2014 -0800
+
+    Merge pull request #2764 from ceph/wip-9851
+    
+    osd: bring FileJournal in sync with giant
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a340e6d0b166019f58dca0703faf30dd3178c14f
+Merge: b7d5f99 b9450b5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:12:17 2014 -0800
+
+    Merge pull request #2776 from ceph/wip-9675.firefly
+    
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit b7d5f99c8f4d751e83dc29305649d7a465c657b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 15 15:29:08 2014 -0700
+
+    ceph-disk: mount xfs with inode64 by default
+    
+    We did this forever ago with mkcephfs, but ceph-disk didn't.  Note that for
+    modern XFS this option is obsolete, but for older kernels it was not the
+    default.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11496399ef318498c11e551f139d96db52d3309c)
+
+commit 1a9d000bb679a7392b9dd115373c3827c9626694
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 9 10:20:27 2014 -0700
+
+    rgw: set length for keystone token validation request
+    
+    Fixes: #7796
+    Backport: giany, firefly
+    Need to set content length to this request, as the server might not
+    handle a chunked request (even though we don't send anything).
+    
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3dd4ccad7fe97fc16a3ee4130549b48600bc485c)
+
+commit 49d27efde2ce5d282c9ee6ca9c8ea9db8f609392
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 19 13:15:46 2014 -0700
+
+    rgw: subuser creation fixes
+    
+    Fixes: #8587
+    There were a couple of issues, one when trying to identify whether swift
+    user exists, we weren't using the correct swift id. The second problem
+    is that we relied on the gen_access flag in the swift case, where it
+    doesn't really need to apply.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1441ffe8103f03c6b2f625f37adbb2e1cfec66bb)
+
+commit 8db2f0969e1715f57088c311a33e7e3499933afb
+Merge: 9a15592 2a5d7f0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 08:35:50 2014 -0700
+
+    Merge pull request #2847 from dachary/wip-9752-past-intervals-firefly
+    
+    osd: past_interval display bug on acting
+
+commit 2a5d7f08303eb8b1687c5b58426443b3d40e415c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 31 00:49:21 2014 +0100
+
+    osd: past_interval display bug on acting
+    
+    The acting array was incorrectly including the primary and up_primary.
+    
+    http://tracker.ceph.com/issues/9752 Fixes: #9752
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c5f8d6eded52da451fdd1d807bd4700221e4c41c)
+
+commit 9a15592eae6739183049f8376392c5577145871e
+Merge: 6fd8879 c20a242
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Oct 30 17:01:24 2014 -0700
+
+    Merge pull request #2840 from ceph/firefly-9869
+    
+    Backport "client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid"
+
+commit c20a2421f4a5b33407d7f31806dd4587b4e9077c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Oct 22 17:16:31 2014 -0700
+
+    client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid
+    
+    m->get_client_tid() is 64 bits (as it should be), but Inode::flushing_cap_tid
+    is only 16 bits. 16 bits should be plenty to let the cap flush updates
+    pipeline appropriately, but we need to cast in the proper direction when
+    comparing these differently-sized versions. So downcast the 64-bit one
+    to 16 bits.
+    
+    Fixes: #9869
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a5184cf46a6e867287e24aeb731634828467cd98)
+
+commit 4afb54274bb2087da348103d0a7641b3a018d777
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Sep 11 13:46:51 2014 -0700
+
+    ReplicatedPG: cancel cb on blacklisted watcher
+    
+    Fixes: #8315
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 16bd45777166c29c433af3b59254a7169e512d98)
+
+commit 037aa342fa277351fc605da41489a6ceba81ab05
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:19:43 2014 -0700
+
+    ReplicatedPG::on_removal: clear rollback info
+    
+    Fixes: #9293
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 544b8c7ffb4af01765b87239f2d7ab88479ee779)
+
+commit 8978c8dc1f153a9052b5d967ceff11d4f08d51ea
+Merge: 8401e7f 4be53d5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 30 13:48:18 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9574' into wip-sam-firefly-backports
+
+commit 8401e7ffa1768770f451143b3c110d1deae1bd40
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 29 15:01:25 2014 -0700
+
+    PG: release backfill reservations if a backfill peer rejects
+    
+    Also, the full peer will wait until the rejection from the primary
+    to do a state transition.
+    
+    Fixes: #9626
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 624aaf2a4ea9950153a89ff921e2adce683a6f51)
+
+commit 5df09fa1b5a42992f9e41aca09e86db0d03d9fbd
+Merge: a0937ef a1aa06b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 30 13:47:22 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9113' into wip-sam-firefly-backports
+
+commit a0937ef214b9221e7a5e69e7b0f0697471d56293
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 12 10:05:51 2014 -0700
+
+    osd/osd_types: consider CRUSH_ITEM_NONE in check_new_interval() min_size check
+    
+    Fixes: #9718
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d947050c82a511f91c98e1c76e48ffa9e187eee7)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+
+commit 35e8e6b2c1733cd421bf6c3916553eea3786e76e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 20 14:10:58 2014 -0700
+
+    PG:: reset_interval_flush and in set_last_peering_reset
+    
+    If we have a change in the prior set, but not in the up/acting set, we go back
+    through Reset in order to reset peering state.  Previously, we would reset
+    last_peering_reset in the Reset constructor.  This did not, however, reset the
+    flush_interval, which caused the eventual flush event to be ignored and the
+    peering messages to not be sent.
+    
+    Instead, we will always reset_interval_flush if we are actually changing the
+    last_peering_reset value.
+    
+    Fixes: #9821
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d9ff3a6b789c5b9c77aefa3751bd808f5d7b8ca7)
+
+commit 3f35db42977704a12ac4b5bcad6261aaf6b6a88d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 23 09:11:28 2014 -0700
+
+    ReplicatedPG: writeout hit_set object with correct prior_version
+    
+    Fixes: #9875
+    Backport: giant, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1a3ad307f1a4c0a956d6fd31d13f01ffe411a09d)
+
+commit 6fd88792e77cdc7ad33ff0acf9b3189a7c525430
+Merge: 0975ec9 afe6bd8
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 26 20:37:52 2014 -0700
+
+    Merge pull request #2717 from dachary/wip-9747-ceph-spec-firefly
+    
+    rpm: 95-ceph-osd-alt.rules is not needed for centos7 / rhel7 (firefly)
+
+commit b9450b532ab7ad23ec6e2c22ed7cf55e6e1cc4c0
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Wed Aug 20 15:35:44 2014 +0800
+
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Originally in the add_simple_ruleset funtion, the ruleset_id
+    is not reused but rule_id is reused. So after some add/remove
+    against rules, the newly created rule likely to have
+    ruleset!=rule_id.
+    
+    We dont want this happen because we are trying to hold the constraint
+    that ruleset == rule_id.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 78e84f34da83abf5a62ae97bb84ab70774b164a6)
+    
+    Conflicts:
+    	src/test/erasure-code/TestErasureCodeIsa.cc
+    
+    Fixes: #9675
+
+commit 3e17a0872a3864cb6aee46959afd955ef0cbafeb
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Mon Jul 21 15:08:55 2014 +0800
+
+    os/FileJournal: When dump journal, using correctly seq avoid misjudging joural corrupt.
+    
+    In func FileJournal::dump, it always using seq=0 as last-seq and it can
+    misjudge the journal corrupt.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit 5f65b4db6d1dad7c2c5a09eab42af63a82ea9e9b)
+
+commit 350da8c98bc1e90cd392992aba290c7478280d88
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Sep 26 01:15:53 2014 +0200
+
+    os: io_event.res is the size written
+    
+    And not an error code to be converted with cpp_strerror()
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 7827e0035e3350ad2d9230f27a1629545f53af5c)
+
+commit ecff3761f3d15061a2cbf0a595ca249a4c424f4c
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 15:10:46 2014 +0800
+
+    os/FileJournal: For journal-aio-mode, don't use aio when closing journal.
+    
+    For jouranl-aio-mode when closing journal, the write_finish_thread_entry may exit before
+    write_thread_entry. This cause no one wait last aios to complete.
+    On some platform, after that the journal-header on journal corrupted.
+    To avoid this, when closing jouranl we don't use aio.
+    
+    Fixes: 9073
+    Reported-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit e870fd09ce846e5642db268c33bbe8e2e17ffef2)
+
+commit dbc33fbab4b35e2ce1e46a881f6714262502c243
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 21:07:51 2014 +0800
+
+    os/FileJournal: Only using aio then alloc the related resources.
+    
+    If define HAVE_LIBAIO, it alloc related resouces. But itt don't check whether
+    using aio mode. Only using aio it alloc the related resources.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit a66a4931d5be9ee26c0983b3154fdbe37261a51c)
+
+commit 3312c6eeca21dcd566df9bdd8de7b3fe33356b57
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 15:49:44 2014 +0800
+
+    os/FileJournal: Tune the judge logic for read_header.
+    
+    When reading journal-header, it should firstly check the result of
+    pread and then do decoce operation.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit c8e2b89cf6bc36a0ff29887b9e76cbbeceef9f8f)
+
+commit b42107584449e1f85cbee97bfa486ebeb310e6a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 19 20:50:13 2014 -0700
+
+    os/FileJournal: signal aio_cond even if seq is 0
+    
+    This can happen if we write a journal but no events.
+    
+    Reported-by: Somnath Roy <somnath.roy@sandisk.com>
+    Reported-by: Ma, Jianpeng <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 57778e2c577c1e1bbf9525232720a2994fa36abc)
+
+commit 4a43ea171d84a9f3a13064030eb386fcfbe3bbb8
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 23 10:10:38 2014 -0700
+
+    os/FileJournal: Update the journal header when closing journal
+    
+    When closing journal, it should check must_write_header and update
+    journal header if must_write_header alreay set.
+    It can reduce the nosense journal-replay after restarting osd.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5bf472aefb7360a1fe17601b42e551df120badfb)
+
+commit 0975ec9cec1c466f7b15f5173541a7eab02dae18
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 06:53:36 2014 -0700
+
+    Revert "os/FileJournal: stop aio completion thread *after* writer thread"
+    
+    This reverts commit 334631ae4641824b3df49245f36a8fd4b143bf3f.
+
+commit 5a10b95f7968ecac1f2af4abf9fb91347a290544
+Merge: cc69c16 d5bac46
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 17 10:47:22 2014 -0700
+
+    Merge pull request #2716 from ceph/wip-firefly-9419
+    
+    Backport fix for bug #9419
+
+commit cc69c16c2108cb231b2434a53e3eef51b597756b
+Merge: f9cdaab 334631a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 17 10:44:30 2014 -0700
+
+    Merge pull request #2724 from dachary/wip-9073-journal-aio-mode-firefly
+    
+    os/FileJournal: stop aio completion thread *after* writer thread
+
+commit f9cdaabe078415d7927e2618030996f2f24be2f1
+Merge: 0b4b34a 412c277
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 17 08:20:53 2014 -0700
+
+    Merge pull request #2742 from ceph/firefly-unknown-locktype
+    
+    mds: reply -EOPNOTSUPP for unknown lock type
+
+commit 412c2770c74abea73a94e10df7b83ebe11ac82ee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 14 22:02:41 2014 +0800
+
+    mds: reply -EOPNOTSUPP for unknown lock type
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 675392335c53ff7879031fb9184e4f35bcc90fe2)
+
+commit 1228658871e53e350bdab3e72cdefd1caf33c291
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 21 15:56:18 2014 -0700
+
+    osd/ReplicatedPG: do not clone or preserve snapdir on cache_evict
+    
+    If we cache_evict a head in a cache pool, we need to prevent
+    make_writeable() from cloning the head and finish_ctx() from
+    preserving the snapdir object.
+    
+    Fixes: #8629
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ce8eefca13008a9cce3aedd67b11537145e1fd77)
+
+commit 88e6014463e86e48d78ac419226644209f83f2a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 21 15:54:15 2014 -0700
+
+    ceph_test_rados_api_tier: add EvictSnap2 test case
+    
+    Verify an evict doesn't create a snapdir object.  Reproduces #8629
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 398c74eacb1ce4e573aef0d24718a5925d90272b)
+
+commit 0b4b34aac497d17a6474c35891aab2bde962524b
+Merge: 322958a 0a72235
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 16 06:09:51 2014 -0700
+
+    Merge pull request #2734 from ceph/wip-firefly-undump
+    
+    mds: fix --undump-journal
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0a72235b0556752fadebc3e155ad41b13a0a15e9
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 16 11:17:40 2014 +0100
+
+    mds: fix --undump-journal
+    
+    This hadn't worked for a long time.  This is a fix
+    for firefly only, as this code was refactored in giant.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit 835f8c6f6121f3ebdec3a0d2d5cb1376301dc03a
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Tue Sep 23 14:02:55 2014 +0100
+
+    ceph-mon: check fs stats just before preforking
+    
+    Otherwise statfs may fail if mkfs hasn't been run yet or if the monitor
+    data directory does not exist.  There are checks to account for the mon
+    data dir not existing and we should wait for them to clear before we go
+    ahead and check the fs stats.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 7f71c11666b25e91dd612c58b4eda9ac0d4752f8)
+    
+    Conflicts:
+    	src/ceph_mon.cc
+
+commit 1ddf435464562f70f63cdb0032da3187f34ce853
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:53:43 2014 +0100
+
+    ceph_mon: check available storage space for mon data dir on start
+    
+    error out if available storage space is below 'mon data avail crit'
+    
+    Fixes: #9502
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2da1a2914ac7df18ce842b0aac728fffb5bed2b6)
+    
+    Conflicts:
+    	src/ceph_mon.cc
+
+commit 112317791b744d9890a65adcc13554c85e90f3af
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:52:34 2014 +0100
+
+    mon: DataHealthService: use get_fs_stats() instead
+    
+    and relieve the DataStats struct from clutter by using
+    ceph_data_stats_t instead of multiple fields.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 9996d446988768658db751a7843b13cf3d194213)
+    
+    Conflicts:
+    	src/mon/DataHealthService.cc
+
+commit f0a92d72fd44542619338db7d6da98e147b6a9fc
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:32:20 2014 +0100
+
+    common: util: add get_fs_stats() function
+    
+    simplifies the task of obtaining available/used disk space, as well as
+    used available percentage.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 3d74230d1c0fbfa15487e2a90ac60b883476e840)
+
+commit a8fa009fbe5d5d4d9cfa134d5ecd05c92290a8eb
+Author: Joao Eduardo Luis <jeclui@gmail.com>
+Date:   Thu Sep 18 16:25:44 2014 +0100
+
+    include/util.h: prevent multiple inclusion of header
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 76eff9503493312cb97e4a2f9236f4dbcbf931df)
+
+commit e296685e8f3f5158238216eefb76482bd6d55134
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:23:36 2014 -0700
+
+    mon: re-bootstrap if we get probed by a mon that is way ahead
+    
+    During bootstrap we verify that our paxos commits overlap with the other
+    mons we will form a quorum with.  If they do not, we do a sync.
+    
+    However, it is possible we pass those checks, then fail to join a quorum
+    before the quorum moves ahead in time such that we no longer overlap.
+    Currently nothing kicks up back into a probing state to discover we need
+    to sync... we will just keep trying to call or join an election instead.
+    
+    Fix this by jumping back to bootstrap if we get a probe that is ahead of
+    us.  Only do this from non probe or sync states as these will be common;
+    it is only the active and electing states that matter (and probably just
+    electing!).
+    
+    Fixes: #9301
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c421b55e8e15ef04ca8aeb47f7d090375eaa8573)
+
+commit 0e57767d5fc524939e8968b506ce2fb3f4f80656
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:11:24 2014 -0700
+
+    mon/Paxos: fix off-by-one in last_ vs first_committed check
+    
+    peon last_committed + 1 == leader first_committed is okay.  Note that the
+    other check (where I clean up whitespace) gets this correct.
+    
+    Fixes: #9301 (partly)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d81cd7f86695185dce31df76c33c9a02123f0e4a)
+
+commit 1f4aaf648f4aa6f6056d0e8ce629eeea05c5424d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 16:17:02 2014 -0700
+
+    mon/Paxos: share state and verify contiguity early in collect phase
+    
+    We verify peons are contiguous and share new paxos states to catch peons
+    up at the end of the round.  Do this each time we (potentially) get new
+    states via a collect message.  This will allow peons to be pulled forward
+    and remain contiguous when they otherwise would not have been able to.
+    For example, if
+    
+      mon.0 (leader)  20..30
+      mon.1 (peon)    15..25
+      mon.2 (peon)    28..40
+    
+    If we got mon.1 first and then mon.2 second, we would store the new txns
+    and then boot mon.1 out at the end because 15..25 is not contiguous with
+    28..40.  However, with this change, we share 26..30 to mon.1 when we get
+    the collect, and then 31..40 when we get mon.2's collect, pulling them
+    both into the final quorum.
+    
+    It also breaks the 'catch-up' work into smaller pieces, which ought to
+    smooth out latency a bit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c54f1e4d66b22bad715ac17e9baa72ab93e48c46)
+
+commit 6c5b9a666fcd94e175a8b9771368b55246957efe
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 14 16:55:58 2014 -0700
+
+    mon/Paxos: verify all new peons are still contiguous at end of round
+    
+    During the collect phase we verify that each peon has overlapping or
+    contiguous versions as us (and can therefore be caught up with some
+    series of transactions).  However, we *also* assimilate any new states we
+    get from those peers, and that may move our own first_committed forward
+    in time.  This means that an early responder might have originally been
+    contiguous, but a later one moved us forward, and when the round finished
+    they were not contiguous any more.  This leads to a crash on the peon
+    when they get our first begin message.
+    
+    For example:
+    
+     - we have 10..20
+     - first peon has 5..15
+       - ok!
+     - second peon has 18..30
+       - we apply this state
+     - we are now 18..30
+     - we finish the round
+       - send commit to first peon (empty.. we aren't contiguous)
+       - send no commit to second peon (we match)
+     - we send a begin for state 31
+       - first peon crashes (it's lc is still 15)
+    
+    Prevent this by checking at the end of the round if we are still
+    contiguous.  If not, bootstrap.  This is similar to the check we do above,
+    but reverse to make sure *we* aren't too far ahead of *them*.
+    
+    Fixes: #9053
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e5ce5f0dcec9bbe9ed4a6b41758ab7802614810)
+
+commit 11d2c9dd4aeb835ca73bfb41fb15b1038547adf6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 16:01:01 2014 -0700
+
+    mon/Paxos: put source mon id in a temp variable
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bb046ed01ecf58b8c87eeeee2e00a476e6fba467)
+
+commit 322958a5aad82c031b54592b372aa053e8993be4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 15 12:26:00 2014 -0700
+
+    qa/workunits/rbd/import_export.sh: be case insensitive
+    
+    Stop tripping over this change (from dumpling).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5558afa03dbd1b20766b76e9410ef5bc3e73784f)
+
+commit 334631ae4641824b3df49245f36a8fd4b143bf3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 29 19:40:29 2014 -0700
+
+    os/FileJournal: stop aio completion thread *after* writer thread
+    
+    The writer thread may submit a new aio to update the header in its
+    final moments before shutting down.  Do not stop the aio thread until after
+    that has happened or else we may not wait for those aio completions.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c776a89880fdac270e6334ad8e49fa616d05d0d4)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+
+commit 111eec9ff6325a12fcbf066ae08f27919aeae5d8
+Merge: cf4e300 6c0127f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 14:57:42 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly' into firefly-next
+
+commit cf4e30095e8149d1df0f2c9b4c93c9df0779ec84
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Tue Aug 5 16:12:22 2014 +0800
+
+    mon/OSDMonitor : Use user provided ruleset for replicated pool
+    
+    When creating a replicated pool, currently ceph ignore the ruleset
+    name provided by user but use a global default ruleset.
+    
+    This patch fix this bug, so the rulset specified by
+    ceph osd pool create replicated
+    can be properly set.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit bf9726a294abd32b429170284ac328a592802648)
+
+commit bfd7da10e708a6eefc6d992b2b6337b7f06fd5ed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jun 3 13:05:19 2014 +0200
+
+    documentation: update osd pool create erasure
+    
+    The properties are replaced with erasure code profiles. Remove the
+    reference to properties and the documentation of each erasure-code
+    related property.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8ff4edda73abb920c91e1226a330e3659def1fbe)
+
+commit afe6bd89f8a1588fb67063d1a08a4be8c1ab2ce6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Oct 11 18:20:36 2014 +0200
+
+    rpm: 95-ceph-osd-alt.rules is not needed for centos7 / rhel7
+    
+    The || instead of && had it always installed. That was fixed in EPEL
+    already.
+    
+    http://tracker.ceph.com/issues/9747 Fixes: #9747
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 5ff4a850a0d809b3f25988c6cceb82c35095ef84)
+
+commit d5bac46e06c5420f29a021b294e391b2c6694cbd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 24 16:02:21 2014 -0700
+
+    osd: Return EOPNOTSUPP if a set-alloc-hint occurs with OSDs that don't support
+    
+    Add CEPH_FEATURE_OSD_SET_ALLOC_HINT feature bit
+    Collect the intersection of all peer feature bits during peering
+    When handling CEPH_OSD_OP_SETALLOCHINT check that all OSDs support it
+    by checking for CEPH_FEATURE_OSD_SET_ALLOC_HINT feature bit.
+    
+    Fixes: #9419
+    Backport: firefly
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9b39033f2b2bcdd2be0f6da4dff06023d0f77499)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+    	src/osd/PG.cc
+    	src/osd/PG.h
+    	src/osd/ReplicatedPG.cc
+
+commit de08802dcf35aea516d013d3d6116aaa7707b923
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 19 15:12:55 2014 -0700
+
+    osd: Remove unused PG functions queue_notify(), queue_info(), queue_log()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 70ef4c11cbae669799c30c7592073ad7aa11dcd6)
+
+commit 5b5aba73031e901457ca27cf15600ce1ca90e258
+Merge: 345714b a1ae7f4
+Author: Gregory Farnum <greg@inktank.com>
+Date:   Fri Oct 10 06:57:06 2014 -0700
+
+    Merge pull request #2691 from ceph/firefly-unused-variable
+    
+    Firefly unused variable
+
+commit a1ae7f471c809e69d363b9145e70160533bfa48c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 10 21:36:39 2014 +0800
+
+    mds: Locker: remove unused variable
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 345714b6b4d004ad03cc7952dc56c6db87664ee4
+Merge: fd20a1d 2afb6fe
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Fri Oct 10 09:37:53 2014 +0800
+
+    Merge pull request #2681 from ceph/firefly-locker-null
+    
+    mds: Locker: fix a NULL deref in _update_cap_fields
+
+commit 2afb6febdd8482b8fec5890d79944d656faf1382
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Oct 9 15:12:19 2014 -0700
+
+    mds: Locker: fix a NULL deref in _update_cap_fields
+    
+    The MClientCaps* is allowed to be NULL, so we can't deref it unless
+    the dirty param is non-zero. So don't do the ahead-of-time lookup;
+    just call it explicitly in the if block.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 3cd8a7fb9683577a7d6e934f18c29b7e84415be6)
+
+commit fd20a1d01bde67fb1edc6058e38435af9d5d6abc
+Merge: e1bd1b2 86926c6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Oct 8 08:44:46 2014 +0200
+
+    Merge pull request #2662 from dachary/wip-9677-ioprio-class-firefly
+    
+    common: ceph_ioprio_string_to_class always returns -EINVAL
+
+commit 86926c6089d63014dd770b4bb61fc7aca3998542
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:06:38 2014 +0200
+
+    common: ceph_ioprio_string_to_class always returns -EINVAL
+    
+    The l string is always empty because std::transform needs a
+    pre-allocated string. Replace with the in-place version. Add unit tests.
+    
+    http://tracker.ceph.com/issues/9677 Fixes: #9677
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 3535b7aba3df8b54fa5117b8a9c2f52b8f0f118b)
+    
+    Conflicts:
+    	src/test/Makefile.am
+
+commit 5f2eec5036a2910aca1e8ce2d94444d3ed0477df
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:05:08 2014 +0200
+
+    osd: log error if set_ioprio fails to parse class
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 5088e0d49332d579ba7e33c2c9baee3d5f701a3e)
+
+commit 2796d5151df4dcde324a4d09a83c9a779cece00e
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:03:39 2014 +0200
+
+    common: set_ioprio debug message including pid
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 33339c7754875eb7e513345ee6b26a9b2b4d2707)
+
+commit d5ed6b0587b9999b2fd41377b0426e3b09ef8ab9
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:02:09 2014 +0200
+
+    common: do not set ioprio if pid is not set
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c7e4c0bfe70bf29d3b8fe4df4e4b934853e33d26)
+
+commit e1bd1b2774f4eae5444b5f7b984193cb91a2dd98
+Merge: 726c6a1 459dca1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 7 09:58:03 2014 -0700
+
+    Merge pull request #2632 from ceph/wip-9039-firefly
+    
+    rgw: copy object data if target bucket is in a different pool
+
+commit 726c6a147a14c00cf12eb6c6561655475282419f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 15:50:51 2014 -0700
+
+    debian/control: fix python-ceph -> ceph file move to allow upgrades
+    
+    This is a backport of 5c6c366d2abe771c581690270c2d176ebb30c571 with the
+    version numbers changed, to compensate for the change in
+    fe3434f41cd09433975d7d0f9dbb2fae662e4a1b (backported in
+    bf1933e5c184476a354664c42fec834e9f59067c).
+    
+    Tested-by: Tamil Muthamizhan <tamil.muthamizhan@inktank.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 884f7c40c4a28d519847d3995c8d98e5837ceaf0
+Merge: 31d57c9 c8a8e47
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 07:01:50 2014 -0700
+
+    Merge remote-tracking branch 'gh/wip-rpm-epoch-firefly' into firefly
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 31d57c9a28502a4a72f8aa141f7ed63ffe1e0192
+Merge: 9a3bac0 548be0b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Oct 6 09:50:33 2014 +0200
+
+    Merge pull request #2643 from johnugeorge/wip-9492-crush-firefly
+    
+    Crush: Backporting fixes for #9492 to firefly
+    
+    Reviewed-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 548be0b2aea18ed3196ef8f0ab5f58a66e3a9af4
+Author: Johnu George <johnugeo@cisco.com>
+Date:   Mon Sep 29 10:07:44 2014 -0700
+
+    Crush: Ensuring at most num-rep osds are selected
+    
+    Crush temporary buffers are allocated as per replica size configured
+    by the user.When there are more final osds (to be selected as per
+    rule) than the replicas, buffer overlaps and it causes crash.Now, it
+    ensures that at most num-rep osds are selected even if more number of
+    osds are allowed by indep rule. The fix for firstn rules is already
+    merged as part of bug #9492. Required test files are added.
+    
+    Fixes: #9492
+    
+    Signed-off-by: Johnu George johnugeo@cisco.com
+    (cherry picked from commit 234b066ba04976783d15ff2abc3e81b6cc06fb10)
+
+commit e30c570ce691a994898b4a933c57e7ae014cdc30
+Author: Johnu George <johnugeo@cisco.com>
+Date:   Wed Sep 24 09:32:50 2014 -0700
+
+    Crush: Ensuring at most num-rep osds are selected
+    
+    Crush temporary buffers are allocated as per replica size configured
+    by the user.When there are more final osds (to be selected as per
+    rule) than the replicas, buffer overlaps and it causes crash.Now, it
+    ensures that at most num-rep osds are selected even if more number of
+    osds are allowed by the rule.
+    
+    Fixes: #9492
+    
+    Signed-off-by: Johnu George <johnugeo@cisco.com>
+    (cherry picked from commit 6b4d1aa99718e3b367496326c1e64551330fabc0)
+
+commit 9a3bac0c1a7a42cdf7ba846c9ad0a3ae0f15b4bb
+Merge: 0b978fb 5a5e7e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 3 11:58:41 2014 -0700
+
+    Merge pull request #2634 from dachary/wip-9653-ceph-disk-bootstrap-osd-firefly
+    
+    ceph-disk: bootstrap-osd keyring ignores --statedir (firefly)
+
+commit c8a8e4763a55dec44836bc679254ee7dcc448567
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 12:33:38 2014 -0700
+
+    ceph.spec: fix typo
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit da9ae5c92ecb4059e0ec6be5ce03af46430e76a5)
+
+commit b890c1e4706d7cfef7ed24c9df65b439b4f7ff1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 06:02:02 2014 -0700
+
+    ceph.spec.: add epoch
+    
+    This is done in fedora packaging.  Do it here too so that you can move
+    between upstream packages (from ceph.com) and fedora and other derivatives
+    will builds.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 83888362089346e473d6fd6e1d366b826d7bd739)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit d01db8080d97bfae34dcee3d153bf10e6f5327dd
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 13:41:35 2014 -0700
+
+    ceph.spec.in: remove BuildRoot
+    
+    Deprecated
+    
+    Fixes: #8143
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 401319a1527dd9cb5398916105d31e7ec065763d)
+
+commit 3c2b5c440863df548afc2bd8aa5440f15a44ac02
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 16:31:22 2014 -0700
+
+    ceph.spec.in: tests for rhel or centos need to not include _version
+    
+    rhel_version and centos_version are apparently the OpenSUSE Build
+    names; the native macros are just "rhel" and "centos" (and contain
+    a version number, should it be necessary).
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 7474f720c2418cf3d52b755f2b60c524e413570a)
+
+commit c82c29d6174022be45929fe9ba8a84993eef974a
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:46:52 2014 -0700
+
+    ceph.spec.in: Add a small comment on the empty %files section
+    
+    as suggested by Dan Mick.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit e37b262c7928934530c5bb09fe56f83eb61f4244)
+
+commit a4f748aa906fc65b14f65515721bc3a815c18fb8
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:39:18 2014 -0700
+
+    ceph.spec.in: Obsolete all older versions.
+    
+    Now this changeset can be used on all current ceph releases that already
+    have the package split.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 875a99e25f0ad2cb47149a3b5a28b4771a09125c)
+
+commit 74c7f3caec1cc7a5da89ef33de36a8b59249cfcd
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:09:43 2014 -0700
+
+    ceph.spec.in: No version on ceph-libs Obsoletes.
+    
+    If we are installing with the new package structure we don't ever want the
+    new package to co-exist with the old one; this includes the mistakenly-
+    released v0.81 on Fedora, which should be removed in favor of this
+    version.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 8f95daf66b5fdb2a8141988480f984c1249599c5)
+
+commit 561261b6efb7ea442686bb8fce387c4de2482067
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Fri Aug 8 18:01:30 2014 -0700
+
+    ceph.spec.in: Obselete ceph-libcephfs (not libcephfs)
+    
+    I am guessing that because it was a sub-package libcephfs was mistakenly
+    used instead of ceph-libcephfs.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 75985024bd30ca6fbe4c61aa7f7cbe5306c9a988)
+
+commit 107bfd9ee7dbf360561187b9e0946964d40b9b1c
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Fri Aug 1 14:20:18 2014 +0200
+
+    ceph.spec.in: We need those nice recent changes for rhel7 in Fedora too.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 00877ae502ac52613bcd5c5c834d72787d668dca)
+
+commit 7946c5e5de8d6cd25d20beee15f3489113e51539
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Aug 27 12:56:43 2014 -0700
+
+    Move fedora patch file (used by ceph.spec.in) to rpm/ subdir
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 06b92cee621cbe33a6f17e8c64169db4453a5160)
+
+commit cb2ae9afa611175226efb5544f7d2aa705d55ece
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Fri Aug 1 00:13:50 2014 +0200
+
+    ceph.spec.in, init-ceph.in: Don't autostart ceph service on Fedora.
+    
+    This patch is taken from the current Fedora package and makes the upstream
+    ceph.spec compliant with Fedora policy. The goal is to be fully compliant
+    upstream so that we can replace current Fedora package with upstream
+    package to fix many bugs in Fedora.
+    
+    Addition from Dan Mick <dan.mick@inktank.com>:
+    Do this for RHEL and Centos as well, since they surely will benefit
+    from the same policy.   Note: this requires changes to
+    autobuild-ceph and ceph-build scripts, which currently copy
+    only the dist tarball to the rpmbuild/SOURCES dir.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>:
+    (cherry picked from commit 461523b06cdf93e32f1d8b354ac3799e73162d33)
+
+commit 2b11376f1ee8925ab16065ebda912b11d3d7be59
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Thu Jul 31 23:54:03 2014 +0200
+
+    ceph.spec.in: Add obsoletes for libcephfs
+    
+    This fixes a bug for Fedora:
+    https://bugzilla.redhat.com/show_bug.cgi?id=1116614
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit e9da2d8f2142771f206ef67f19e7f194855275d0)
+
+commit eefc62e3b85f402a7e1ae31c272c8a432d979379
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Thu Jul 31 23:49:56 2014 +0200
+
+    ceph.spec.in: add ceph-libs-compat
+    
+    Added a ceph-libs-compat package in accordance with Fedora packaging
+    guidelines [1], to handle the recent package split more gracefully.
+    In Fedora this is necessary because there are already other packages
+    depending on ceph-libs, that need to be adjusted to depend on the new
+    split packages instead. In the mean time, ceph-libs-compat prevents
+    breakage.
+    
+    [1] http://fedoraproject.org/wiki/Upgrade_paths_%E2%80%94_renaming_or_splitting_packages
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 6c264f2204cbd54d90b02101e40ac9aa5aa72d7c)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit 0b978fb15a1307644aba3119419bb7386f98ee04
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 14:41:19 2014 -0700
+
+    mon/Paxos: add perfcounters for most paxos operations
+    
+    I'm focusing primarily on the ones that result in IO here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b09b8563d35dda23faed43afef2a983e93a879c5)
+
+commit 74aa7afc719e517dbed300f802c1bc2dafe43ee0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 14:00:11 2014 -0700
+
+    mon/MonitorDBStore: add get_{keys,bytes}() accounting to Transaction
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fd421b26748e872ddf8e0f068dda2106853edff1)
+
+commit 5a5e7e7bcedbccbe4ae8aab159af6d8615eb3887
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 3 14:08:57 2014 +0200
+
+    ceph-disk: bootstrap-osd keyring ignores --statedir
+    
+    The STATEDIR variable is used to initialize the bootstrap-osd keyring
+    before it gets a chance to be overriden by --statedir. Replace it with
+    {statedir} so that it can be substituted after all options have been
+    parsed.
+    
+    http://tracker.ceph.com/issues/9653 Fixes: #9653
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fa0bd06b4657e5b84e237b76033ac3d3478b6a1f)
+
+commit 459dca1613a14cfad8d3afd7e3c783d825573a42
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 13:36:11 2014 -0700
+
+    rgw: copy object data if target bucket is in a different pool
+    
+    Fixes: #9039
+    Backport: firefly
+    
+    The new manifest does not provide a way to put the head and the tail in
+    separate pools. In any case, if an object is copied between buckets in
+    different pools, we may really just want the object to be copied, rather
+    than reference counted.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d3a7e595f47455896304bf358e5251915d0f16f)
+
+commit 711a7e6f81983ff2091caa0f232af914a04a041c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 15 00:53:50 2014 -0400
+
+    rbd: ObjectCacher reads can hang when reading sparse files
+    
+    The pending read list was not properly flushed when empty objects
+    were read from a space file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cdb7675a21c9107e3596c90c2b1598def3c6899f)
+
+commit b7784dc1baa47560a733fe9dcd2acec51bc93165
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Sep 6 22:59:40 2014 -0400
+
+    Enforce cache size on read requests
+    
+    In-flight cache reads were not previously counted against
+    new cache read requests, which could result in very large
+    cache usage.  This effect is most noticeable when writing
+    small chunks to a cloned image since each write requires
+    a full object read from the parent.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fc9fffc494abedac0a9b1ce44706343f18466f1)
+
+commit ac4fca065a12f3b00a47bf5ec37983696255174b
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Sep 5 10:36:24 2014 -0700
+
+    rgw: add .log to default log path
+    
+    Fixes: #9353
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 46732420897a2619059050044f4980a4737df43e)
+
+commit f03ae7b00e5694f1670493396a1cee195fcc6b35
+Merge: 78c3ef9 ee02cfd
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Oct 2 15:28:40 2014 -0700
+
+    Merge pull request #2565 from ceph/wip-rgw-firefly-backports
+    
+    Wip rgw firefly backports
+
+commit 78c3ef90604ca117255cefe232771a7564fed8b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 25 13:16:52 2014 -0700
+
+    osdc/Objecter: only post_rx_buffer if no op timeout
+    
+    If we post an rx buffer and there is a timeout, the revocation can happen
+    while the reader has consumed the buffers but before it has decoded and
+    constructed the message.  In particular, we calculate a crc32c over the
+    data portion of the message after we've taken the buffers and dropped the
+    lock.
+    
+    Instead of fixing this race (for example, by reverifying rx_buffers under
+    the lock while calculating the crc.. bleh), just skip the rx buffer
+    optimization entirely when a timeout is present.
+    
+    Note that this doesn't cover the op_cancel() paths, but none of those users
+    provide static buffers to read into.
+    
+    Fixes: #9582
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    
+    backport of 126d0b30e990519b8f845f99ba893fdcd56de447
+
+commit a261b4952056aab7b067453930342960bbe55089
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 14:28:32 2014 -0700
+
+    debian: move ceph_rest_api.py into ceph
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fe3434f41cd09433975d7d0f9dbb2fae662e4a1b)
+
+commit eb0f6e347969b40c0655d3165a6c4531c6b595a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 14:24:01 2014 -0700
+
+    ceph.spec.in: move ceph_rest_api.py into ceph
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8cda623e0ba34a48a70e9ea988d619b15605c4fd)
+
+commit c4188e31f7bc8f3c337e637cd99c41d5ee4b6787
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:44:03 2014 -0700
+
+    ceph.spec: fix python-flask dependency
+    
+    This is needed by ceph-rest-api, which is in ceph.rpm; it's not related to
+    python-ceph (except that ceph-rest-api happens to require that too).
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b2416240b88b2e067dfc79a2723335f1584562d0)
+
+commit bf1933e5c184476a354664c42fec834e9f59067c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:40:18 2014 -0700
+
+    debian: python-flask is needed by ceph, not python-ceph
+    
+    It's used by ceph-rest-api which is in the 'ceph' (server) package.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e42424e777e4f7d8b03650482253734c1fa8709d)
+    
+    Conflicts:
+    
+    	debian/control
+
+commit 94a7fbaa11c51db294dce0dc1df728f69aef5bf8
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Sep 19 12:25:07 2014 +0200
+
+    rgw_main.cc: add missing virtual destructor for RGWRequest
+    
+    CID 1160858 (#1 of 1): Non-virtual destructor (VIRTUAL_DTOR)
+     nonvirtual_dtor: Class RGWLoadGenRequest has a destructor
+     and a pointer to it is upcast to class RGWRequest which doesn't
+     have a virtual destructor.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b82ceda7775ff85943d9143b73789eb37b09bfa9)
+
+commit 9fee8de25ab5c155cd6a3d32a71e45630a5ded15
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Sep 29 16:10:36 2014 -0700
+
+    Locker: accept ctime updates from clients without dirty write caps
+    
+    The ctime changes any time the inode does. That can happen even without
+    the file itself having changed, so we'd better accept the update whenever
+    the auth caps have dirtied, without worrying about the file caps!
+    
+    Fixes: #9514
+    Backport: firefly
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0ea20a668cf859881c49b33d1b6db4e636eda18a)
+
+commit 461ece5e9fb1d4994a6214a3b6bdae136773629d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 18:01:51 2014 -0700
+
+    doc/release-notes: fix attributions for 8702 fix
+    
+    Oops!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 188370a94353e29fcb8981699022803e23f3fedd)
+
+commit 917529a78e5046f621df5c48fe5d50d2f7e56560
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 17:48:12 2014 -0700
+
+    doc/release-notes: v0.80.6
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c0dc3a56974a469b61523b67cc032cc5726a3a5f)
+    
+    Conflicts:
+    
+    	doc/release-notes.rst
+
+commit 060a5b1422fcdfb8e84636579a2f0c2f1ec14300
+Author: Adam Crume <adamcrume@gmail.com>
+Date:   Thu Sep 18 16:57:27 2014 -0700
+
+    common: Add cctid meta variable
+    
+    Fixes: #6228
+    Signed-off-by: Adam Crume <adamcrume@gmail.com>
+    (cherry picked from commit bb45621cb117131707a85154292a3b3cdd1c662a)
+
+commit a1aa06b7fb30e509193e1b1bb7355b5f21aedc82
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 1 14:30:59 2014 -0700
+
+    ReplicatedPG: dump snap_trimq on pg query
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 34f38b68d89baf1dcbb4571d4f4d3076dc354538
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 29 16:26:54 2014 -0700
+
+    ReplicatedPG: do not queue the snap trimmer constantly
+    
+    Previously, we continuously requeued the snap trimmer while in
+    TrimmingObjects.  This is not a good idea now that we try to
+    limit the number of snap trimming repops in flight and requeue
+    the snap trimmer directly as those repops complete.
+    
+    Fixes: #9113
+    Backport: giant, dumpling, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit b29bf00f68cf133151c98db06e9498b3e8be22ed
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Sep 24 13:55:47 2014 -0700
+
+    ReplicatedPG: clean out completed trimmed objects as we go
+    
+    Also, explicitely maintain a max number of concurrently trimming
+    objects.
+    
+    Fixes: 9113
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit ee02cfd23facb3404fc377f643b213c2f498474d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 18 20:53:10 2014 -0700
+
+    rgw: calculate hash after writing data
+    
+    Since data is written asynchronously, we should do the hash calculation
+    while it's pending.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5bb94ede19a50543a02a8019ed6c9680b3852d4e)
+
+commit 216730221575d88a72b06ed3d71c9a54cffc5719
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 18 20:51:02 2014 -0700
+
+    crypto: don't hash zero sized buffer
+    
+    libnss returns an error and we assert in that case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7b137246b49a9f0b4d8b8d5cebfa78cc1ebd14e7)
+
+commit dab7a4f6ad054b53cedca76ee329a6395918b1ab
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 12 14:07:44 2014 -0700
+
+    rgw: push hash calculater deeper
+    
+    This might have been the culprit for #9307. Before we were calculating
+    the hash after the call to processor->handle_data(), however, that
+    method might have spliced the bufferlist, so we can't be sure that the
+    pointer that we were holding originally is still invalid. Instead, push
+    the hash calculation down. Added a new explicit complete_hash() call to
+    the processor, since when we're at complete() it's too late (we need to
+    have the hash at that point already).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d41c3e858c6f215792c67b8c2a42312cae07ece9)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.h
+
+commit bd0a91343ce70d71acced753688a502b7e8b552e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 21 16:30:10 2014 -0700
+
+    rgw: separate civetweb log from rgw log
+    
+    The civetweb log now is independent from the rgw log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 850242cad749e33e1e6bc008baa75c8ea7eda0c1)
+    
+    Conflicts:
+    	src/civetweb
+    	src/rgw/rgw_main.cc
+
+commit a777562b780e7ab312f881c38b1db26983a1ac47
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 13:40:39 2014 -0700
+
+    civetweb: update submodule
+    
+    Update submodule to include multiple fixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 1bdcc079d79d8211b44e2a46511cd2240f71744b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Aug 22 15:12:16 2014 -0700
+
+    rgw: convert header field underscores into dashes
+    
+    Fixes: 9206
+    Backport: firefly
+    
+    Certain web servers filter out underscores in the header field name.
+    Convert them into dashes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 11acb7097ce21c6218dd48d0c21e0e04a361eb9a)
+
+commit b8fa2ed60b6cce51701df972dbb6f5e02e0d84ba
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Aug 25 10:38:42 2014 -0700
+
+    rgw: fix test to identify whether object has tail
+    
+    Fixes: #9226
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Backport: firefly
+    
+    We need to identify whether an object is just composed of a head, or
+    also has a tail. Test for pre-firefly objects ("explicit objs") was
+    broken as it was just looking at the number of explicit objs in the
+    manifest. However, this is insufficient, as we might have empty head,
+    and in this case it wouldn't appear, so we need to check whether the
+    sole object is actually pointing at the head.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 751b3e26532932a42ca34f9c062a0a3e29a58cff)
+
+commit 6fee71154d838868807fd9824d829c8250d9d2eb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 27 17:44:18 2014 -0700
+
+    rgw: don't try to authenticate a CORS preflight request
+    
+    Fixes: #8718
+    Backport: firefly
+    
+    CORS preflight requests don't need to be authenticated. Treat them as
+    coming from anonymous user.
+    
+    Reported-by: Robert Hubbard <bobby.hubbard@garmin.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 848fcf7871e07fc689bdcd18943ace36b2f4906e)
+
+commit c75a79cbac30cd14d37d89217113824d98693572
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 14 13:35:12 2014 -0700
+
+    rgw: fix compilation
+    
+    RGWRadosPutObj couldn't refer to the ceph context.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6a555434ee3edaf742ee7e5910bcba8dd0de46dd)
+
+commit b0d08aab837808f18708a4f8ced0503c0fce2fec
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 11:17:47 2014 -0700
+
+    rgw: call throttle_data() even if renew_state() failed
+    
+    Otherwise we're going to leak the aio callback handle.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7998c322179dd69a1250937321c3c2bb023e0e57)
+
+commit a953b313f1e2f884be6ee2ce356780f4f70849dd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 21:32:48 2014 -0700
+
+    rgw: disable civetweb url decoding
+    
+    Fixes: #8621
+    
+    We want to have the raw request uri, as we do the decoding ourselves.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ffac52b316e7022796d44ae58804d9c20b9c3df9)
+
+commit ba5357714a19b8af989fef1c75ef775837c6a9d6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 14:23:46 2014 -0700
+
+    rgw: copy_obj_data() uses atomic processor
+    
+    Fixes: #9089
+    
+    copy_obj_data was not using the current object write infrastructure,
+    which means that the end objects weren't striped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 800eff24824c0083b8e2441fc34e0bdca5da36dc)
+
+commit d73dbc3a39117eddaaabb2c25d9238cd7c51711b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 21 21:53:38 2014 -0700
+
+    rgw: clear bufferlist if write_data() successful
+    
+    Fixes: #9201
+    Backport: firefly
+    
+    We sometimes need to call RGWPutObjProcessor::handle_data() again,
+    so that we send the pending data. However, we failed to clear the buffer
+    that was already sent, thus it was resent. This triggers when using non
+    default pool alignments.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9181114d6f6062c55ee4b351fc3495345e545c36)
+
+commit 4be53d5eebbc294878ba38050d841359b6c8e19e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 12:16:55 2014 -0700
+
+    PG: check full ratio again post-reservation
+    
+    Otherwise, we might queue 30 pgs for backfill at 0.80 fullness
+    and then never check again filling the osd after pg 11.
+    
+    Fixes: #9574
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
diff --git a/doc/changelog/v0.80.9.txt b/doc/changelog/v0.80.9.txt
new file mode 100644
index 000000000..aa274d137
--- /dev/null
+++ b/doc/changelog/v0.80.9.txt
@@ -0,0 +1,1148 @@
+commit b5a67f0e1d15385bc0d60a6da6e7fc810bde6047 (tag: refs/tags/v0.80.9, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Mar 9 10:42:08 2015 -0700
+
+    0.80.9
+
+commit 37901afd1556257151c029395caa1143e84860f2
+Merge: 00e5947 3dac68a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:53 2015 +0100
+
+    Merge pull request #3852 from dachary/wip-10965-rgw-firefly
+    
+    rgw: enable ipv6 in civetweb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 00e5947a28f7810a28329d2f4901aed09a289421
+Merge: 293222a b13f483
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:39 2015 +0100
+
+    Merge pull request #3853 from dachary/wip-10907-rgw-firefly
+    
+    rgw: pass civetweb configurables to civetweb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 293222a284c072a13950831205c106fec7a400df
+Merge: 6512b06 d57b38f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:25 2015 +0100
+
+    Merge pull request #3851 from dachary/wip-10978-rgw-firefly
+    
+    rgw: don't overwrite bucket / object owner when setting acls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d57b38f85b683dfb365c3cb98362d486594f9eb3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 15:32:50 2015 -0800
+
+    rgw: don't overwrite bucket / object owner when setting acls
+    
+    Fixes: #10978
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb13f2d4b60c031f16139f7cc4237c012644dd78)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc : trivial s/.empty()/== NULL/
+
+commit b13f483e51b37a768c7f4313b6933bf648950c7d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Feb 17 15:05:40 2015 -0800
+
+    rgw: pass civetweb configurables to civetweb
+    
+    Fixes: #10907
+    Backport: firefly
+    
+    Pass any configurables in the rgw frontends config line to civetweb.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 986d7554426764a149621ba733c5c075b94e0431)
+
+commit 3dac68a17a909b212a36b0a3c0ae2c47d323deee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 08:14:27 2015 -0800
+
+    civetweb: update submodule
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f28fe744285117a1715eac0d08911cdb37285103)
+
+commit 14aca3af18119a76c1cdfa6d71e6085d360e45e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 08:14:41 2015 -0800
+
+    rgw: update makefile to enable civetweb config
+    
+    Fixes: #10965
+    Backport: hammer, firefly
+    
+    Civetweb compilation now includes conf header to enable ipv6.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit caa90225bad9fe5e9c275e6189b3396b4d396e3f)
+    
+    Conflicts:
+    	src/rgw/Makefile.am
+               radosgw_CFLAGS = -Icivetweb/include
+            was not yet changed into
+               radosgw_CFLAGS = -I$(srcdir)/civetweb/include
+            in firefly
+
+commit 6512b06fc8a80e3befbe15b543f8850991f74e8a
+Merge: 7b748c6 555cc42
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:47:55 2015 +0100
+
+    Merge pull request #3820 from tchaikov/firefly-pg-leak-10421
+    
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7b748c62764311572342593820ce3f28f74fe9ca
+Merge: d971c95 da95149
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:46:42 2015 +0100
+
+    Merge pull request #3772 from ceph/wip-10883-firefly
+    
+    osd: Fix FileJournal wrap to get header out first
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d971c95c3d8b48494d05b68f40c4c9a4cc6f87a8
+Merge: 0820041 e539971
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:45:48 2015 +0100
+
+    Merge pull request #3521 from dzafman/wip-10676
+    
+    Backport doc fixes that appear to apply to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 555cc42fc826fd801f0d45187429079d4072d129
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 10 16:29:45 2015 +0800
+
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Fixes: #10421
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 01e154d592d6cdbf3f859cf1b4357e803536a6b4)
+
+commit 0820041e6515c82c41b81b9e6825e2dd5fcb8165
+Merge: 6565774 702dbc0
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Feb 26 15:59:24 2015 -0800
+
+    Merge pull request #3730 from ceph/wip-firefly-flock
+    
+    backport ceph-fuse file locking patches to Firefly
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 6565774d0356efc6225ad561bb13e7cf11da7b1e
+Merge: 7ab02ee 08c2fda
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 14:18:10 2015 -0800
+
+    Merge pull request #3406 from ceph/wip-10299-firefly
+    
+    librbd: complete all pending aio ops prior to closing image
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 08c2fda12cf46937a09a59bb032379c3c5321292
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 15 10:53:53 2014 -0500
+
+    librbd: complete all pending aio ops prior to closing image
+    
+    It was possible for an image to be closed while aio operations
+    were still outstanding.  Now all aio operations are tracked and
+    completed before the image is closed.
+    
+    Fixes: #10299
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7ab02ee5afb3d017b94d58c3dfc7731f7a3866d9
+Merge: 0c0a552 c23e42e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 14:12:29 2015 -0800
+
+    Merge pull request #3404 from ceph/wip-10270-firefly
+    
+    librbd: gracefully handle deleted/renamed pools
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0c0a5520b1b883bcdd5b865b217ba61e471e3ca2
+Merge: 9ef7743 836ab86
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 13:31:37 2015 -0800
+
+    Merge pull request #3410 from ceph/wip-9854-firefly
+    
+    osdc: Constrain max number of in-flight read requests
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da951497b7e5ec227aa6a5e459b0d04d8b88ef13 (refs/remotes/gh/wip-10883-firefly)
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 16:21:12 2015 -0800
+
+    osd: Fix FileJournal wrap to get header out first
+    
+    Correct and restore assert that was removed
+    
+    Cause by f46b1b473fce0322a672b16c7739e569a45054b6
+    Fixes: #10883
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 970bb4901f93575709421b5b25c3eff213de61b8)
+
+commit 702dbc0a247c149d53b52d1929f9880bc99d0522
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 15 12:00:58 2014 +0800
+
+    qa/workunits/fs/misc: Add a workunit for file lock interruption
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ac92c455a9aa19e4288acdf0c9a746e03a640efb)
+
+commit d2523b82c5c7b29293d70a66ba95493a1564a840
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 15 12:03:46 2014 +0800
+
+    mds: fix neighbor lock check
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b0e6e85aa08ea74cd209aad04f3f0bf991761e12)
+
+commit f9b6b66b05ddadef043d81676728bf40730ea16c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 13 11:34:18 2014 +0800
+
+    client: use finisher to abort MDS request
+    
+    When a request is interrupted, libfuse first locks an internal mutex,
+    then calls the interrupt callback. libfuse need to lock the same mutex
+    when unregistering interrupt callback. We unregister interrupt callback
+    while client_lock is locked, so we can't acquiring the client_lock in
+    the interrupt callback.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 09699454e729592d426aeff5b578697e850af12e)
+    
+    Conflicts:
+    	src/client/Client.cc
+    	src/client/Client.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ea355e9ca5203b77e6f74ca4a3e39ce23cc86f67
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 9 13:16:18 2014 +0800
+
+    client: use atomic variable to track reference of MetaRequeset
+    
+    this allow us to increase reference count of MetaRequest while not holding
+    the client_lock
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e464a7765230c504b3e3b88bcb1106f67b7c3eb9)
+
+commit ccbdf514717cb0e48d67b57197d6e538faeea415
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 13 10:44:46 2014 +0800
+
+    client: allow interrupting blocked file lock operation
+    
+    This commit introduce two new types of setfilelock request. Unlike
+    setfilelock (UNLOCK) request, these two new types of setfilelock request
+    do not drop locks that have alread been acquired, they only interrupt
+    blocked setfilelock request.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4134c149d3759dd6a3aaa1a353b77bbfe8e9491b)
+
+commit 875e2fcb060554941d94714ad48ebbc0cbbf8077
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 9 09:42:08 2014 +0800
+
+    client: register callback for fuse interrupt
+    
+    libfuse allows program to reigster a callback for interrupt. When a file
+    system operation is interrupted, the fuse kernel driver sends interupt
+    request to libfuse. libfuse calls the interrupt callback when receiving
+    interrupt request.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 289e8b4a7efa1ae6427115af9bbe541c9e1f0e90)
+
+commit c96aabbd54b24a0273af21a450cd6f517fe4ada0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Oct 4 09:14:44 2014 +0800
+
+    client: add helper function that updates lock state
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6a2303a6b6d97f2a6d1422e42d3d88991857618f)
+    
+    Conflicts:
+    	src/client/Client.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ebbd3ca66722cdc61b56d243baf8e63f7b8c1c1b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 20:21:36 2014 +0800
+
+    fuse: enable fuse_multithreaded by default
+    
+    GETFILELOCK MDS request may block for a long time, so we need to
+    use multithread event loop,
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b17b43a8660ed0db29fbecf44798265e47712f85)
+
+commit 924e6f2b6b03456a8b18d8b158b8f325051f0519
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 19:07:41 2014 +0800
+
+    client: posix file lock support
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a1b2c8ff955b30807ac53ce6bdc97cf61a7262ca)
+    
+    Conflicts:
+    	src/client/Client.cc
+    	src/client/Client.h
+    	src/client/Inode.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 82994946bdcb95867a61e0acf443e30b13925a34
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 18:02:50 2014 +0800
+
+    common: link mds/flock.o to libcommon
+    
+    later commit will use this code to track file locks held by cephfs
+    client.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e075c27c3554380c59dce0cc17ef0944eb415025)
+    
+    Conflicts:
+    	src/mds/Makefile.am
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9ef77430f3d46789b0ba1a2afa42729627734500
+Merge: 1205867 b668566
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Feb 13 18:34:11 2015 +0100
+
+    Merge pull request #3684 from ceph/wip-crush-straw-firefly
+    
+    osd: backport straw bucket fixes and all other recent crush goodness to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b668566fd8148414b8074f096b85b22c42ed3af9 (refs/remotes/gh/wip-crush-straw-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 16 09:02:28 2015 -0800
+
+    crush/builder: fix warnings
+    
+    crush/builder.c: In function 'crush_remove_list_bucket_item':
+    crush/builder.c:977:13: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+      if (weight < bucket->h.weight)
+                 ^
+    crush/builder.c: In function 'crush_remove_tree_bucket_item':
+    crush/builder.c:1031:14: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+       if (weight < bucket->h.weight)
+                  ^
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14eb1a73c71d81b7f193fce27c59cb3babf3e74a)
+
+commit 247afa4efe7d11b5ef06e094680b50ea30d7d5e3
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Oct 15 17:02:58 2014 -0700
+
+    crush: improve constness of CrushWrapper methods
+    
+    A number of CrushWrapper get methods or predicates were not const
+    because they need to maintain transparently the rmaps. Make the rmaps
+    mutable and update the constness of the methods to match what the caller
+    would expect.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 236895eea65f8706baa5fdef96fb00ad5b82218c)
+
+commit c59279a25b6c53e0ab3988b0f00ae3cce94f33d7
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Fri Sep 5 10:56:36 2014 +0800
+
+    Change CrushWrapper::crush to private
+    
+    Currently in CrushWrapper, the member "struct crush_map *crush"  is a public member,
+    so people can break the encapsulation and manipulate directly to the crush structure.
+    
+    This is not a good practice for encapsulation and will lead to inconsistent if code
+    mix use the CrushWrapper API and crush C API.A simple example could be:
+    1.some code use crush_add_rule(C-API) to add a rule, which will not set the have_rmap flag to false in CrushWrapper
+    2.another code using CrushWrapper trying to look up the newly added rule by name will get a -ENOENT.
+    
+    This patch move CrushWrapper::crush to private, together with three reverse map(type_rmap, name_rmap, rule_name_rmap)
+    and also change codes accessing the CrushWrapper::crush to make it compile.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit d734600f9251b52f525faa35441e2b5dd660161b)
+
+commit 9fc2fd50aec94233528870aaa371347238a6ea75
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 11 06:58:57 2014 -0700
+
+    crush: include CRUSH_V3, v2/v3 rules checks in dump_tunables()
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf94cf3531a349bbd1fc6ee56c3fc260110a252a)
+
+commit f6009a614a3689cc9a96f55b35e70362be4ec64b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 15:58:03 2014 -0800
+
+    mon: 'osd crush reweight-all'
+    
+    This corresponds to the crushtool --reweight command.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89b2feea8d53b9dc15ab5ae7f5920ad19c8bba18)
+
+commit d5d7495432dd023ed8c9ba2ac08222f06afee48f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 15:55:24 2014 -0800
+
+    crush: set straw_calc_version=1 for default+optimal; do not touch for presets
+    
+    When using the presets for compatibility (i.e., based on version), do not
+    touch the straw behavior, as it does not affect mapping or compatibility.
+    However, make a point of setting it by default and for optimal.
+    
+    For most users, this means that they will not see any change unless they
+    explicitly enable the new behavior, or switch to default or optimal
+    tunables.  The idea is that if they touched it, they shouldn't be
+    too surprised by the subsequent data movement.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd7b58f3b1aa1febfc6dc227937df93ee6e284eb)
+    
+    (Note: differs from original in that the hammer tunable profile is not
+    present)
+
+commit 5b7b7599123a9d7a837ded52946bd553b09b1ac7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 22:30:00 2014 -0800
+
+    crush/builder: a note about the original crush_calc_straw()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit adf5c6de0aca24a53d3c7b4e7eeb0a5dce9db0f1)
+
+commit 3ab835b059fd74a525cc2a8ebe8b6a1453e0cc87
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 16:43:16 2014 -0800
+
+    mon: add 'osd crush {get,set}-tunable <name> [value]' commands
+    
+    For now, just add the straw_calc_version tunable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9000068ae45a8b89315c152b7d5509ac873f2957)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+
+commit 884414c5788bac9a269f01b26cbc0c55850c34f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 16:33:11 2014 -0800
+
+    crush: fix crush_calc_straw() scalers when there are duplicate weights
+    
+    The straw bucket was originally tested with uniform weights and with a
+    few more complicated patterns, like a stair step (1,2,3,4,5,6,7,8,9).  And
+    it worked!
+    
+    However, it does not behave with a pattern like
+     1, 2, 2, 3, 3, 4, 4
+    
+    Strangely, it does behave with
+     1, 1, 2, 2, 3, 3, 4, 4
+    
+    and more usefully it does behave with
+     1, 2, 2.001, 3, 3.001, 4, 4.001
+    
+    That is, the logic that explicitly copes with weights that are duplicates
+    is broken.
+    
+    The fix is to simply remove the special handling for duplicate weights --
+    it isn't necessary and doesn't work correctly anyway.
+    
+    Add a test that compares the mapping result of  [1, 2, 2, 3, 3, ...] with
+    [1, 2, 2.001, 3, 3.001, ...] and verifies that the difference is small.
+    With the fix, we get .00012, whereas the original implementation gets
+    .015.
+    
+    Note that this changes the straw bucket scalar *precalculated* values that
+    are encoded with the map, and only when the admin opts into the new behavior.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 43d5c7caa7ce478477bde1bbd4f0649b5159cdcf)
+
+commit bf677093302f475a23de541471ddde6a7c7153b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:50:21 2014 -0800
+
+    crush: fix distortion of straw scalers by 0-weight items
+    
+    The presence of a 0-weight item in a straw bucket should have no effect
+    on the placement of other items.  Add a test validating that and fix
+    crush_calc_straw() to fix the distortion.
+    
+    Note that this effects the *precalculation* of the straw bucket inputs and
+    does not effect the actually mapping process given a compiled or encoded
+    CRUSH map, and only when straw_calc_version == 1 (i.e., the admin opted in
+    to the new behavior).
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 85498bc8f62ca56506b33f3c5ec4fc4b111ed73d)
+
+commit e02574ef5d3b66e73424a16460366de2d36eded3
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:49:42 2014 -0800
+
+    crush/builder: break out new version 1 of crush_calc_straw
+    
+    No change, yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7c1203635f2b5911f281ce3a441905df6e9bd103)
+
+commit c7312a47865c758e67852999803d8aa90ff809c1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:45:04 2014 -0800
+
+    crush: pass crush_map * to various builder methods
+    
+    In particular, we will need it for crush_calc_straw().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f35a3d88cb944c292e966f679ac7a8d7a1cd3093)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.cc
+
+commit 5137cc656548d942d0f21ba3ef28a5b7d4d21831
+Author: Rongze Zhu <zrzhit@gmail.com>
+Date:   Fri Oct 10 19:18:00 2014 +0800
+
+    crush: fix incorrect use of adjust_item_weight method
+    
+    adjust_item_weight method will adjust all buckets which the item
+    inside. If the osd.0 in host=fake01 and host=fake02, we execute
+    "ceph osd crush osd.0 10 host=fake01", it not only will adjust fake01's
+    weight, but also will adjust fake02's weight.
+    
+    the patch add adjust_item_weightf_in_loc method and fix remove_item,
+    _remove_item_under, update_item, insert_item, detach_bucket methods.
+    
+    Signed-off-by: Rongze Zhu <zrzhit@gmail.com>
+    (cherry picked from commit 9850227d2f0ca2f692a154de2c14a0a08e751f08)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.cc
+
+commit 80ec22014deb8536b9c34e8d57b286052898d05a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 10:59:22 2014 -0800
+
+    crush/CrushWrapper: fix detach_bucket
+    
+    In commit 9850227d2f0ca2f692a154de2c14a0a08e751f08 we changed the call that
+    changed the weight of all instances of item to one that explicitly
+    changes it in the parent bucket, but parent_id may not be valid at the
+    call site.  Move this into the conditional block to fix.
+    
+    Fixes: #10095
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f8b96a9fe4793906c74a571109a457aca7ca220)
+
+commit 5bc554ac5a7a4cd86b873bfc4f4313c91287b52c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:10:49 2014 -0800
+
+    crush: default to straw_calc_version 1
+    
+    Stick with bobtail tunables where it affects compatibility.  Use v1 of
+    straw_calc, though, since that does not, and we want the best for new
+    clusters.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9565621d6007302fdd68ba27b6aef22e487f0985)
+
+commit cfc718a5a1fec937cf00e6c2b55f66d4390088bb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 13 08:30:35 2015 -0800
+
+    crush: add straw_calc_version tunable
+    
+    It doesn't do anything, yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 50e2ceefee6a5dfbecbe54890139b1fa80a313c8)
+    
+    (Modified from original to not create the 'hammer' tunable profile, which
+    we will not backport in its entirety.)
+
+commit 1205867d3afe6d457483f9f51aaee79cca74e040
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 2 16:43:35 2015 +0100
+
+    qa: use correct binary path on rpm-based systems
+    
+    Fixes: #10715
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 05ce2aa1bf030ea225300b48e7914577a412b38c)
+
+commit da4146a5e03503083e6bc5c12984c06f41a3b4be
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Feb 5 21:12:17 2015 -0800
+
+    fsync-tester: print info about PATH and locations of lsof lookup
+    
+    We're seeing the lsof invocation fail (as not found) in testing and nobody can
+    identify why. Since attempting to reproduce the issue has not worked, this
+    patch will gather data from a genuinely in-vitro location.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a85051483874ff5b8b0fb50426a3577040457596)
+
+commit 61b4f09848796faeacf7eb78dd6dc06513beb737
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:04:34 2014 -0800
+
+    crush/CrushWrapper: dump chooseleaf_vary_r with other tunables
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c133a83fe7b77e2a7e7c711cb8ab943c59ff0885)
+
+commit e4939ed5535cc4678cf1d1ae80906290448e6590
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:12:25 2014 -0800
+
+    crush/CrushTester: add new --show-mappings option
+    
+    This makes --show-utilization and --show-statistics usable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3877f00877f53e9f86630c2d89e81ab9cff729ec)
+
+commit 8c48ebbf13e6bdb689b4b8ae58ac811653ad2acc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:11:57 2014 -0800
+
+    crushtool/CrushTester: output utilization even with 1 batch
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 294b06c6424f4cb69394976add826d9725073b50)
+
+commit e9faab9f296af9cf26b9475afd3562c3f3b9236b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:08:18 2014 -0800
+
+    crush: recalculate straw scalers during a reweight
+    
+    The crushtool --reweight function triggers a fresh calculation of bucket
+    weights so that they are always the sum of the item weights.  In the
+    straw bucket case, the weights were updated but the corresponding straw
+    scalers were not being recalculated.  The result is that there was not
+    effect on placement in adjusted buckets until the next time a bucket item's
+    weight was adjusted.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 35062937f174a45fb13d9c177eaa1fe4ed5ff4c2)
+
+commit 4c951e9dc1ca31429119de15755caf3d73f6ffce
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 08:36:41 2014 -0800
+
+    osdmaptool: --test-map-pgs-dump
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a99b489909b368bade36d8cc9722ad33d03a2e8)
+
+commit 3cb5d08feee4811624dd495f74912824203edf00
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 06:53:20 2014 -0800
+
+    crush: add dprintk's for crush_calc_straw
+    
+    These are compiled out by default.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 946bebd94b109642f95710802e892c59eb4f0b76)
+
+commit ca8471d65fb2cf7d2247d823c5436faa273efda5
+Author: Rongze Zhu <zrzhit@gmail.com>
+Date:   Tue Nov 11 00:13:42 2014 +0800
+
+    crush: fix tree bucket functions
+    
+    There are incorrect nodes' weight in tree bucket when construct tree
+    bucket. The tree bucket don't store item id in items array, so the tree
+    bucket will not work correctly. The patch fix above bugs and add a
+    simple test for tree bucket.
+    
+    Signed-off-by: Rongze Zhu <zrzhit@gmail.com>
+    (cherry picked from commit 13425488882d360fa740613dfcfd0d098c1b7616)
+
+commit af502f25b04cd0758b753941ecf0b1b59d33ea9e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 11 11:16:58 2014 -0800
+
+    crush/builder: replace printf with an empty dprintk macro
+    
+    This mirrors mapper.c.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e444b221733360cdea4f4bbdbbbfbf2b6ee30ff5)
+
+commit 26966c4aa9cd79cb53db052553a5fc5653f2591b
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Nov 6 17:48:01 2014 -0800
+
+    qa: use sudo even more when rsyncing /usr
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 3aa7797741f9cff06053a2f31550fe6929039692)
+
+commit 6af48421c0ba6195e9a1607053c42eeb62b14ccb
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Oct 21 10:55:06 2014 -0700
+
+    qa: use sudo when rsyncing /usr so we can read everything
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fa07c04231db2d130de54647957ffab4a7a53733)
+
+commit 861a18b1f7dd5bdcbb85bc3fa15d1719bb4b2813
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:31:22 2015 -0800
+
+    rgw: fix partial GET in swift
+    
+    Fixes: #10553
+    backport: firefly, giant
+    
+    Don't set the ret code to reflect partial download, just set the
+    response status when needed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7e1553cedff90fa0fefded65cde87ad068eb5f0c)
+
+commit e539971e2d528b4de6009ea44565f037acb2be66 (refs/remotes/gh/wip-10676)
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:13:27 2014 +0530
+
+    doc: Adds updated man page for ceph under man/
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8de9a0f437822c770600c19a9f61977745b7e530)
+
+commit 15596ffe388147b7984457041a38cbb9f472556c
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:27:45 2014 +0530
+
+    doc: Adds man page for ceph under man/.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit ffd6c7e49686f8f92ddb400ffdec62520708e64b)
+
+commit 893e5cd1f4fd5ea77d65f0e78cc8cff88eb19c1a
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:24:41 2014 +0530
+
+    doc: Adds man page for ceph.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 76da87a64ca6b3cc0ceeaf63e19a9f440d6f4161)
+
+commit 8d29a4a231666830914903b95599d80da7b97def
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 15 17:04:32 2014 -0800
+
+    osd: handle no-op write with snapshot case
+    
+    If we have a transaction that does something to the object but it !exists
+    both before and after, we will continue through the write path.  If the
+    snapdir object already exists, and we try to create it again, we will
+    leak a snapdir obc and lock and later crash on an assert when the obc
+    is destroyed:
+    
+    0> 2014-12-06 01:49:51.750163 7f08d6ade700 -1 osd/osd_types.h: In function 'ObjectContext::~ObjectContext()' thread 7f08d6ade700 time 2014-12-06 01:49:51.605411
+    osd/osd_types.h: 2944: FAILED assert(rwstate.empty())
+    
+    Fix is to not recreated the snapdir if it already exists.
+    
+    Fixes: #10262
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02fae9fc54c10b5a932102bac43f32199d4cb612)
+
+commit 8ba48d10c252b28cde5b4da1286421db12b57cc2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 19 18:28:20 2015 -0800
+
+    ceph_test_rados_api_misc: do not assert rbd feature match
+    
+    This test fails on upgrades when we (or the server) have new
+    features.  Make it less fragile.
+    
+    Fixes: #10576
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9147c62989871cea8b3a85b02c53017825efb55b)
+
+commit 836ab86d89ae15ed5b228ff656bc81c7cc3495aa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 27 14:47:19 2014 -0400
+
+    osdc: Constrain max number of in-flight read requests
+    
+    Constrain the number of in-flight RADOS read requests to the
+    cache size.  This reduces the chance of the cache memory
+    ballooning during certain scenarios like copy-up which can
+    invoke many concurrent read requests.
+    
+    Fixes: #9854
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 068d68850d09dfcaccc5a3ce85c80b2f6d808ea9)
+
+commit 2964efaf327cf798f892a6722eb4e24f2ffa0fde
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 12:46:00 2015 -0500
+
+    Revert "Enforce cache size on read requests"
+    
+    This reverts commit b7784dc1baa47560a733fe9dcd2acec51bc93165.
+
+commit e4c5b153735aba3f84554a50243f779f36568373
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 12:45:25 2015 -0500
+
+    Revert "rbd: ObjectCacher reads can hang when reading sparse files"
+    
+    This reverts commit 711a7e6f81983ff2091caa0f232af914a04a041c.
+
+commit c23e42e78ea3ba44706951a728e2ccb74cac7b33
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 10:28:56 2015 -0500
+
+    librbd: gracefully handle deleted/renamed pools
+    
+    snap_unprotect and list_children both attempt to scan all
+    pools. If a pool is deleted or renamed during the scan,
+    the methods would previously return -ENOENT. Both methods
+    have been modified to more gracefully handle this condition.
+    
+    Fixes: #10270
+    Backport: giant, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 436923c68b77c900b7774fbef918c0d6e1614a36)
+
+commit 24c13d87039d4f61df0bcabdb8862e0e94fe575d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 05:24:01 2014 -0800
+
+    rgw: change multipart upload id magic
+    
+    Fixes: #10271
+    Backport: firefly, giant
+    
+    Some clients can't sign requests correctly with the original magic
+    prefix.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5fc7a0be67a03ed63fcc8408f8d71a31a1841076)
+
+commit 617002d3ff469ef409a83e35d4f4fd6a0b5b1278
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Dec 11 09:07:10 2014 -0800
+
+    rgw: url decode http query params correctly
+    
+    Fixes: #10271
+    Backport: firefly
+    
+    This got broken by the fix for #8702. Since we now only url_decode if
+    we're in query, we need to specify that we're in query when decoding
+    these args.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 21e07eb6abacb085f81b65acd706b46af29ffc03)
+
+commit d7ccf71d962cec1571f53c9392f9b58350569062
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 14 15:01:38 2015 -0800
+
+    qa: ignore duplicates in rados ls
+    
+    These can happen with split or with state changes due to reordering
+    results within the hash range requested. It's easy enough to filter
+    them out at this stage.
+    
+    Backport: giant, firefly
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7cc6117adf653a4915fb7a75fac68f8fa0239ec)
+
+commit aef69572588a0dfad58df94cb0d0980d0590d8e4
+Merge: 69eaad7 534624b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:40:48 2015 -0800
+
+    Merge branch 'wip-firefly-rgw-backports' into firefly
+
+commit 534624b7e9decc880e88496355a6fbbe008ede5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 17:59:30 2014 -0700
+
+    init-radosgw.sysv: set ulimit -n before starting daemon
+    
+    If we do the ulimit inside the daemon command we will have already
+    dropped privs and will fail.
+    
+    Fixes: #9587
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9803cedf54a7baff45ccd0e0f65d2bc220958a46)
+
+commit fd49cbc535d0d7fa64ebfa458386b47c8ec8616e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 79bfffb55a0b3a302368e34417d62f74b82dc224
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:40:55 2014 -0800
+
+    rgw: remove swift user manifest (DLO) hash calculation
+    
+    Fixes: #9973
+    Backport: firefly, giant
+    
+    Previously we were iterating through the parts, creating hash of the
+    parts etags (as S3 does for multipart uploads). However, swift just
+    calculates the etag for the empty manifest object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef6d3ad964d34bc526dc4435486bd5c8cdc3b230)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+
+commit ac799f0834783590cbb6eb91784c8e0753cb1e03
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Mon Oct 27 10:29:48 2014 +0800
+
+    fix can not disable max_size quota
+    
+    Currently if we enable quota and set max_size = -1, it doesn’t
+    mean max_size is unlimited as expected. Instead, it means object
+    with any size is not allowed to upload because of “QuotaExceeded�.
+    The root cause is the function rgw_rounded_kb which convert max_size
+    to max_size_kb returns 0 for -1 because it takes an unsigned int
+    but we pass an int to it. A simple fix is check max_size before
+    it’s rounded to max_size_kb.
+    
+    Test case:
+    1 enable and set quota:
+    radosgw-admin quota enable --uid={user_id} --quota-scope=user
+    radosgw-admin quota set --quota-scope=user --uid={user_id}\
+     --max-objects=100 --max-size=-1
+    2 upload any object with non-zero length
+    it will return 403 with “QuotaExceeded� and return 200 if you apply the fix.
+    
+    Fixes: #9907
+    Backport: giant, firefly
+    Signed-off-by: Dong Lei leidong@yahoo-inc.com
+    (cherry picked from commit abd3fd3ef9ee9999b99811937af60b7a5e673e35)
+
+commit 30963fdc98f5650a68b5737729920d43459b5899
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:43:55 2014 -0700
+
+    rgw: rados->set_attrs() updates bucket index
+    
+    Fixes: #5595
+    Backport: dumpling, firefly
+    We need to update the bucket index when updating object attrs, otherwise
+    we're missing meta changes that need to be registered. It also
+    solves issue of bucket index not knowing about object acl changes,
+    although this one still requires some more work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f833f12a200ecc2c4f74ddb443d6fa61b7ad14db)
+
+commit 31f5e332c616dbb7f7338af3ab37ac65ff66f733
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 4 22:05:03 2014 -0800
+
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+    
+    Fixes: #9877
+    We only updated if len was > 0, should update it if r >= 0. This was the
+    culprit for issue #9877.
+    Backport: giant, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe7bf06366adaf787816d1e68f5e3f68e8c91134)
+
+commit 8682d1b15e097ececd927dfbd096dbb46403afca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:28:02 2014 -0800
+
+    rgw: send back ETag on S3 object copy
+    
+    Fixes: #9479
+    Backport: firefly, giant
+    We didn't send the etag back correctly. Original code assumed the etag
+    resided in the attrs, but attrs only contained request attrs.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b1bfc3a7e0c9088f01f8ff770ae14f569fbc570d)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 2d61fc76e97e5e28cf24cc6e341d49a8fe47059c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 12:35:42 2014 -0800
+
+    rgw: S3 object copy content type fix
+    
+    Fixes: #9478
+    Backport: firefly, giant
+    Content type for S3 object copy response should be set to
+    application/xml.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 31963459a0a869c4d32f55baa629f36df33eaa90)
+
+commit 011a8c3bd621337e5a1746f18882be20d3854b14
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 11:20:26 2014 -0800
+
+    rgw: http headers need to end with \r\n
+    
+    Fixes: #9254
+    Backport: firefly, giant
+    
+    Reported-by: Benedikt Fraunhofer <fraunhofer@traced.net>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7409ab3df18fb312dd6c9f79084f889c523afdce)
+    
+    Conflicts:
+    	src/rgw/rgw_civetweb.cc
+    	src/rgw/rgw_fcgi.cc
+
+commit 71d119f2952716aa1f75817e1daaf5fb67ecde94
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jan 7 13:56:14 2015 -0800
+
+    rgw: index swift keys appropriately
+    
+    Fixes: #10471
+    Backport: firefly, giant
+    
+    We need to index the swift keys by the full uid:subuser when decoding
+    the json representation, to keep it in line with how we store it when
+    creating it through other mechanism.
+    
+    Reported-by: hemant burman <hemant.burman@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 478629bd2f3f32afbe6e93eaebb8a8fa01af356f)
+
+commit 7b89a827a063acf78a848a34c1c3298db213950f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Nov 20 10:36:05 2014 -0800
+
+    rgw-admin: create subuser if needed when creating user
+    
+    Fixes: #10103
+    Backport: firefly, giant
+    This turned up after fixing #9973. Earlier we also didn't create the
+    subuser in this case, but we didn't really read the subuser info when it
+    was authenticating. Now we do that as required, so we end up failing the
+    authentication. This only applies to cases where a subuser was created
+    using 'user create', and not the 'subuser create' command.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9ba17a321db06d3d76c9295e411c76842194b25c)
+
+commit 9ee29b5355e9ffeac76707e8d4070bfff5dc99d5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 14:38:46 2014 -0800
+
+    rgw: update swift subuser perm masks when authenticating
+    
+    Fixes: #9918
+    Backport: firefly, giant
+    It seems that we weren't setting the swift perm mask correctly.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d9f36f757a7272c24d2c9adc31db1ed5e712992)
+
+commit d298fdda2aa177e1a4b43ca94a2292435a50dea4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 23 17:39:42 2014 -0700
+
+    rgw: send http status reason explicitly in fastcgi
+    
+    There are issues in certain versions of apache 2.4, where the reason is
+    not sent back. Instead, just provide the reason explicitly.
+    
+    Backport: firefly, giant
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9dd4af401328e8f9071dee52470a0685ceb296b)
diff --git a/doc/changelog/v0.87.1.txt b/doc/changelog/v0.87.1.txt
new file mode 100644
index 000000000..cf0eecac6
--- /dev/null
+++ b/doc/changelog/v0.87.1.txt
@@ -0,0 +1,2151 @@
+commit 283c2e7cfa2457799f534744d7d549f83ea1335e (tag: refs/tags/v0.87.1, refs/remotes/gh/giant)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Feb 23 12:02:04 2015 -0800
+
+    0.87.1
+
+commit 4178e32dd085adeead84fb168ab8a8a121256259
+Merge: ccb0914 734e9af
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 17 01:09:54 2015 +0100
+
+    Merge pull request #3731 from liewegas/wip-10834-giant
+    
+    osd: tolerate sessionless con in fast dispatch path
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 734e9af5df4ae419ded108f5036bee068a9bc2b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 1 18:15:59 2014 -0800
+
+    osd: tolerate sessionless con in fast dispatch path
+    
+    We can now get a session cleared from a Connection at any time.  Change
+    the assert to an if in ms_fast_dispatch to cope.  It's pretty rare, but it
+    can happen, especially with delay injection.  In particular, a racing
+    thread can call mark_down() on us.
+    
+    Fixes: #10209
+    Backport: giant
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 01df2227125abf94571b4b0c7bccca57098ed2dc)
+
+commit ccb0914f76da23acdd7374233cd1939ab80ef3c8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 2 16:43:35 2015 +0100
+
+    qa: use correct binary path on rpm-based systems
+    
+    Fixes: #10715
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 05ce2aa1bf030ea225300b48e7914577a412b38c)
+
+commit 78c71b9200da5e7d832ec58765478404d31ae6b5
+Merge: 222aa22 91515e7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 11 00:11:57 2015 +0100
+
+    Merge pull request #3407 from ceph/wip-9854-giant
+    
+    osdc: Constrain max number of in-flight read requests
+
+commit 222aa22ebc0ccb1b04156e0c9d05f4e4733ec290
+Merge: b9ff170 a5cb39c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 22:01:21 2015 +0100
+
+    Merge pull request #3568 from dachary/wip-10471-rgw-giant
+    
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+
+commit b9ff1708ad85ca5aeb10b4202bcbe197251e3bd8
+Merge: 34103b6 b1e4882
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:59:40 2015 +0100
+
+    Merge pull request #3263 from dachary/wip-jerasure-giant
+    
+    erasure-code: update links to jerasure upstream (giant)
+
+commit 34103b6355881820aa10b354c2427654bf229e8f
+Merge: 94889cf d125743
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:59:17 2015 +0100
+
+    Merge pull request #3191 from ceph/giant-10277
+    
+    Giant 10277
+
+commit 94889cf6bef5a542e51bf8434dbe7c68f64604ce
+Merge: d7b10d8 d28c8e0
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:52 2015 +0100
+
+    Merge pull request #3186 from ceph/wip-giant-mon-backports
+    
+    mon: backports for #9987 against giant
+
+commit d7b10d897e17bc3fa690c8484ad2d6f233896237
+Merge: 11f7d06 16c6d0d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:29 2015 +0100
+
+    Merge pull request #3185 from dachary/wip-10325-cauchy-giant
+    
+    erasure-code: relax cauchy w restrictions (giant)
+
+commit 11f7d064e5d93bc0ed8896750344c6cf6b37aeab
+Merge: 975be75 636b98f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:05 2015 +0100
+
+    Merge pull request #3178 from dachary/wip-9998-crush-underfloat-giant
+    
+    crush: fix weight underfloat issue (giant)
+
+commit 975be75f4bcea88b232ea76087b49e288d7c29f7
+Merge: 51fe79d d759e71
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:57:50 2015 +0100
+
+    Merge pull request #3579 from dachary/wip-9877-rgw-giant
+    
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+
+commit 51fe79d9e63c7df4da547a0ba7a12aa9c6cd7ab2
+Merge: fca9ead 319f9c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:57:18 2015 +0100
+
+    Merge pull request #3168 from ceph/wip-8797-giant
+    
+    Wip 8797 giant
+
+commit fca9eadaf3fcef77e269d4936d4eea86ab6c3faf
+Merge: 317532b 9886620
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:53 2015 +0100
+
+    Merge pull request #3582 from dachary/wip-10062-rgw-giant
+    
+    rgw: s3 keystone auth fixes
+
+commit 317532b70383762f473a910e043c889574eb6087
+Merge: 3e8f3e3 debc0c5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:37 2015 +0100
+
+    Merge pull request #3581 from dachary/wip-9918-rgw-giant
+    
+    rgw: update swift subuser perm masks when authenticating
+
+commit 3e8f3e38af76fa2cba86aedf962d3230d7979f63
+Merge: 1d77591 76f9de3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:17 2015 +0100
+
+    Merge pull request #3580 from dachary/wip-9907-rgw-giant
+    
+    fix can not disable max_size quota
+
+commit 1d7759149697242192be05decf7ffafb17b24cbe
+Merge: b016863 ad04a67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:55:56 2015 +0100
+
+    Merge pull request #3083 from dachary/wip-10211-erasure-code-buffer-alignement-giant
+    
+    erasure-code: enforce chunk size alignment (giant)
+
+commit b016863ad243388e7571da9ffca3013c8f99237a
+Merge: bdcc9dc d21f4e3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:55:23 2015 +0100
+
+    Merge pull request #3577 from dachary/wip-9587-rgw-giant
+    
+    init-radosgw.sysv: set ulimit -n before starting daemon
+
+commit bdcc9dcb8586c91b432c7087e33a2b52ef467b54
+Merge: df475f9 7b5f746
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:54:58 2015 +0100
+
+    Merge pull request #3576 from dachary/wip-5595-9576-rgw-giant
+    
+    update object content-length doesn't work correctly
+
+commit df475f92a41e3bd5a022335b2c9023ad40c3b47b
+Merge: db7adf8 b2f6f7f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:54:13 2015 +0100
+
+    Merge pull request #3575 from dachary/wip-9479-rgw-giant
+    
+    rgw: send back ETag on S3 object copy
+
+commit db7adf8d8ca225fea2d0277ced614e936df086c9
+Merge: 9b50db9 67ba4d3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:55 2015 +0100
+
+    Merge pull request #3574 from dachary/wip-9478-rgw-giant
+    
+    rgw: S3 object copy content type fix
+
+commit 9b50db97a9552841ed143588e2f63bab56d0aecb
+Merge: 583fe31 84e9b6c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:33 2015 +0100
+
+    Merge pull request #3573 from dachary/wip-9254-rgw-giant
+    
+    rgw: http headers need to end with \r\n
+
+commit 583fe31681c4eea8b85b413674074445e2b424a6
+Merge: 5240db5 1cb0955
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:15 2015 +0100
+
+    Merge pull request #3572 from dachary/wip-9973-rgw-giant
+    
+    rgw: remove swift user manifest (DLO) hash calculation
+
+commit 5240db588e9017dd8a487b7a9ee16f171fdda1ff
+Merge: 62e1552 e230fab
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:54 2015 +0100
+
+    Merge pull request #3571 from dachary/wip-8911-rgw-giant
+    
+    rgw: swift GET / HEAD object returns X-Timestamp field
+
+commit 62e15528dda20b5419e39744fa9e0c9c4cae053c
+Merge: 16cd892 c24fab3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:36 2015 +0100
+
+    Merge pull request #3570 from dachary/wip-10701-rgw-giant
+    
+    rgw: use strict_strtoll() for content length
+
+commit 16cd892aab4ffb1dc15b93a4101d9bc209591c94
+Merge: 028904c 6aef29e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:00 2015 +0100
+
+    Merge pull request #3569 from dachary/wip-10103-rgw-giant
+    
+    rgw-admin: create subuser if needed when creating user
+
+commit 028904cf7c36a1d5342cf29c115bc0437e9b2d74
+Merge: 520dcf8 425ee8a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:51:06 2015 +0100
+
+    Merge pull request #3567 from dachary/wip-10307-rgw-giant
+    
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+
+commit 520dcf8624eeafd694115b382616be83f9b344d3
+Merge: cae1de2 14cdb9b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:50:34 2015 +0100
+
+    Merge pull request #3443 from ceph/wip-10590-giant
+    
+    rbd: ensure aio_write buffer isn't invalidated during image import
+
+commit cae1de29922c2183eff021c6fe2b921a87b5f5b2
+Merge: b346ad3 83a0a2e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:50:14 2015 +0100
+
+    Merge pull request #3557 from dachary/wip-10688-boost-157-giant
+    
+    support Boost 1.57.0
+
+commit b346ad37a84b7cfd0bae84528f2533a8cc4a8e3d
+Merge: aacd51c 13bb880
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:49:47 2015 +0100
+
+    Merge pull request #2954 from sponce/giant
+    
+    Fixed trivial locking issue in the trunc method of libradosstriper - Giant branch
+
+commit aacd51c74c102d44982421b9bc384d12fc160e3c
+Merge: 1d97c7c 081f49b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:49:20 2015 +0100
+
+    Merge pull request #3405 from ceph/wip-10299-giant
+    
+    librbd: complete all pending aio ops prior to closing image
+
+commit 1d97c7c9a3087e7bc98774d9fe2882bdc4a84531
+Merge: 53dec0e 436923c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:48:49 2015 +0100
+
+    Merge pull request #3403 from ceph/wip-10270-giant
+    
+    librbd: gracefully handle deleted/renamed pools
+
+commit 53dec0eeee60b315e88acb4ba05666857ae3e0eb
+Merge: df8285c 1261bf2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:48:20 2015 +0100
+
+    Merge pull request #3356 from liewegas/wip-msgr-giant
+    
+    msgr: fast dispatch backports for giant
+
+commit df8285c5e5b14d9a4cd42fb9be8e18fe6cdf6f83
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Feb 5 21:12:17 2015 -0800
+
+    fsync-tester: print info about PATH and locations of lsof lookup
+    
+    We're seeing the lsof invocation fail (as not found) in testing and nobody can
+    identify why. Since attempting to reproduce the issue has not worked, this
+    patch will gather data from a genuinely in-vitro location.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a85051483874ff5b8b0fb50426a3577040457596)
+
+commit 91515e750bfe2453ce8ac9ec568b0e314823dd82
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 27 14:47:19 2014 -0400
+
+    osdc: Constrain max number of in-flight read requests
+    
+    Constrain the number of in-flight RADOS read requests to the
+    cache size.  This reduces the chance of the cache memory
+    ballooning during certain scenarios like copy-up which can
+    invoke many concurrent read requests.
+    
+    Fixes: #9854
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 98866208c64348ca885335d95a1c737071a17004
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Dec 22 19:57:19 2014 +0530
+
+    rgw: check keystone auth also for s3 post requests
+    
+    This patch adds keystone auth for s3 post requests, once a user fails in
+    cephx authentication, they are checked for keystone if configured.
+    
+    Fixes #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 8b3dfc9472022ea45ad24e02e0aa21dfdad798f8)
+
+commit 4e4372b8e551bb1b974f08dc69f5b27bdd22bb4b
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Nov 17 17:37:00 2014 +0530
+
+    rgw: check for timestamp for s3 keystone auth
+    
+    This commit ensures that we check for timestamp of s3 request is within
+    acceptable grace time of radosgw
+    Addresses some failures in #10062
+    Fixes: #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 4b35ae067fef9f97b886afe112d662c61c564365)
+
+commit debc0c593fb7401d07a34f7916380092ad7285f9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 14:38:46 2014 -0800
+
+    rgw: update swift subuser perm masks when authenticating
+    
+    Fixes: #9918
+    Backport: firefly, giant
+    It seems that we weren't setting the swift perm mask correctly.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d9f36f757a7272c24d2c9adc31db1ed5e712992)
+
+commit 76f9de330deaa4fafb86d3f6f2754f0d419306b1
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Mon Oct 27 10:29:48 2014 +0800
+
+    fix can not disable max_size quota
+    
+    Currently if we enable quota and set max_size = -1, it doesn’t
+    mean max_size is unlimited as expected. Instead, it means object
+    with any size is not allowed to upload because of “QuotaExceeded�.
+    The root cause is the function rgw_rounded_kb which convert max_size
+    to max_size_kb returns 0 for -1 because it takes an unsigned int
+    but we pass an int to it. A simple fix is check max_size before
+    it’s rounded to max_size_kb.
+    
+    Test case:
+    1 enable and set quota:
+    radosgw-admin quota enable --uid={user_id} --quota-scope=user
+    radosgw-admin quota set --quota-scope=user --uid={user_id}\
+     --max-objects=100 --max-size=-1
+    2 upload any object with non-zero length
+    it will return 403 with “QuotaExceeded� and return 200 if you apply the fix.
+    
+    Fixes: #9907
+    Backport: giant, firefly
+    Signed-off-by: Dong Lei leidong@yahoo-inc.com
+    (cherry picked from commit abd3fd3ef9ee9999b99811937af60b7a5e673e35)
+
+commit d759e71c8167ea29c8fda9483039a3e491083da5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 4 22:05:03 2014 -0800
+
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+    
+    Fixes: #9877
+    We only updated if len was > 0, should update it if r >= 0. This was the
+    culprit for issue #9877.
+    Backport: giant, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe7bf06366adaf787816d1e68f5e3f68e8c91134)
+
+commit d21f4e326eb4821cc9bd38a1b62a0210272277d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 17:59:30 2014 -0700
+
+    init-radosgw.sysv: set ulimit -n before starting daemon
+    
+    If we do the ulimit inside the daemon command we will have already
+    dropped privs and will fail.
+    
+    Fixes: #9587
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9803cedf54a7baff45ccd0e0f65d2bc220958a46)
+
+commit 7b5f746d2f97c7139f9c31962c107a074bfd1863
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:42:10 2014 -0700
+
+    rgw: PutObjMetadata, clear bufferlist before appending into it
+    
+    Fixes: #9576
+    Backport: firefly, dumpling
+    
+    We need to completely rewrite the bufferlist, not append into it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 44cfd88dc65d30f4295743c5834768bb13f7b805)
+
+commit e24f27b7b2e2aeb84b14788e8bf2757ecdf8f0c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:43:55 2014 -0700
+
+    rgw: rados->set_attrs() updates bucket index
+    
+    Fixes: #5595
+    Backport: dumpling, firefly
+    We need to update the bucket index when updating object attrs, otherwise
+    we're missing meta changes that need to be registered. It also
+    solves issue of bucket index not knowing about object acl changes,
+    although this one still requires some more work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f833f12a200ecc2c4f74ddb443d6fa61b7ad14db)
+
+commit b2f6f7f6205682aeb09c0785e373ddf5d89c6d04
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:28:02 2014 -0800
+
+    rgw: send back ETag on S3 object copy
+    
+    Fixes: #9479
+    Backport: firefly, giant
+    We didn't send the etag back correctly. Original code assumed the etag
+    resided in the attrs, but attrs only contained request attrs.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b1bfc3a7e0c9088f01f8ff770ae14f569fbc570d)
+
+commit 67ba4d3444f0f64dae6286be28276ba85376ecf6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 12:35:42 2014 -0800
+
+    rgw: S3 object copy content type fix
+    
+    Fixes: #9478
+    Backport: firefly, giant
+    Content type for S3 object copy response should be set to
+    application/xml.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 31963459a0a869c4d32f55baa629f36df33eaa90)
+
+commit 84e9b6c32c6fdc38e2c64f3360c185332e691bf4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 11:20:26 2014 -0800
+
+    rgw: http headers need to end with \r\n
+    
+    Fixes: #9254
+    Backport: firefly, giant
+    
+    Reported-by: Benedikt Fraunhofer <fraunhofer@traced.net>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7409ab3df18fb312dd6c9f79084f889c523afdce)
+
+commit 1cb09555d3fdb568296797cd83eb5557552f056c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:40:55 2014 -0800
+
+    rgw: remove swift user manifest (DLO) hash calculation
+    
+    Fixes: #9973
+    Backport: firefly, giant
+    
+    Previously we were iterating through the parts, creating hash of the
+    parts etags (as S3 does for multipart uploads). However, swift just
+    calculates the etag for the empty manifest object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef6d3ad964d34bc526dc4435486bd5c8cdc3b230)
+
+commit e230fabf29c4660594d19027af49810e57b82e35
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 30 14:15:47 2014 -0700
+
+    rgw: swift GET / HEAD object returns X-Timestamp field
+    
+    Fixes: #8911
+    Backport: giant, firefly, dumpling
+    Swift clients expect X-Timestamp header, dump it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5b41d80b7fb9ed96c26801fc42c044191bb18d84)
+
+commit c24fab3065ba6d81435981b609f2b69c3d98d21d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 10:51:52 2015 -0800
+
+    rgw: use strict_strtoll() for content length
+    
+    instead of strict_strtol().
+    
+    Backport: giant, firefly
+    Fixes: #10701
+    
+    Reported-by: Axel Dunkel <ad@dunkel.de>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c25dbafb45caf1b261cfcec15b868a2ba6b5fef)
+
+commit 6aef29e31e9c7c7ccf8e95d573700c08218b2b45
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Nov 20 10:36:05 2014 -0800
+
+    rgw-admin: create subuser if needed when creating user
+    
+    Fixes: #10103
+    Backport: firefly, giant
+    This turned up after fixing #9973. Earlier we also didn't create the
+    subuser in this case, but we didn't really read the subuser info when it
+    was authenticating. Now we do that as required, so we end up failing the
+    authentication. This only applies to cases where a subuser was created
+    using 'user create', and not the 'subuser create' command.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9ba17a321db06d3d76c9295e411c76842194b25c)
+
+commit a5cb39cbb6aee869b92ac20975b5c80a01210b63
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 425ee8a07bb8ce12eee124b3c374031f644aa32b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 83a0a2e5b52b18a25009faaf09fa4f48af3c52ee
+Author: William A. Kennington III <william@wkennington.com>
+Date:   Sat Sep 20 22:52:31 2014 -0700
+
+    osd: Cleanup boost optionals
+    
+    Signed-off-by: William A. Kennington III <william@wkennington.com>
+    (cherry picked from commit a53ead14c113047567177630b4906136a2109b65)
+
+commit eb30631ec3d081fd1bc2cdbd4812a334de9e1282
+Author: Petr Machata <pmachata@redhat.com>
+Date:   Thu Jan 29 10:15:02 2015 -0700
+
+    support Boost 1.57.0
+    
+    Sometime after 1.55, boost introduced a forward declaration of
+    operator<< in optional.hpp. In 1.55 and earlier, when << was used
+    without the _io having been included, what got dumped was an implicit
+    bool conversion.
+    
+    http://tracker.ceph.com/issues/10688 Refs: #10688
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 85717394c33137eb703a7b88608ec9cf3287f67a)
+    
+    Conflicts:
+    	src/include/encoding.h
+            trivial conflict
+
+commit 1ccd73a16e1829b5519ec5b83b2554af173ad052
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 23 17:39:42 2014 -0700
+
+    rgw: send http status reason explicitly in fastcgi
+    
+    There are issues in certain versions of apache 2.4, where the reason is
+    not sent back. Instead, just provide the reason explicitly.
+    
+    Backport: firefly, giant
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9dd4af401328e8f9071dee52470a0685ceb296b)
+
+commit 3bf42af2e932a473b19cb54637e8543a666a4a28
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:31:22 2015 -0800
+
+    rgw: fix partial GET in swift
+    
+    Fixes: #10553
+    backport: firefly, giant
+    
+    Don't set the ret code to reflect partial download, just set the
+    response status when needed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7e1553cedff90fa0fefded65cde87ad068eb5f0c)
+
+commit aa038684dce1964d5d23802d23f2bd772458ea11
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 15 17:04:32 2014 -0800
+
+    osd: handle no-op write with snapshot case
+    
+    If we have a transaction that does something to the object but it !exists
+    both before and after, we will continue through the write path.  If the
+    snapdir object already exists, and we try to create it again, we will
+    leak a snapdir obc and lock and later crash on an assert when the obc
+    is destroyed:
+    
+    0> 2014-12-06 01:49:51.750163 7f08d6ade700 -1 osd/osd_types.h: In function 'ObjectContext::~ObjectContext()' thread 7f08d6ade700 time 2014-12-06 01:49:51.605411
+    osd/osd_types.h: 2944: FAILED assert(rwstate.empty())
+    
+    Fix is to not recreated the snapdir if it already exists.
+    
+    Fixes: #10262
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02fae9fc54c10b5a932102bac43f32199d4cb612)
+
+commit e045ad4a39076547209ac1dc298df5ebffb76669
+Merge: a463b92 9f865fa
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Tue Jan 27 09:40:16 2015 -0800
+
+    Merge pull request #3502 from ceph/wip-10382-giant
+    
+    [giant backport] mds: handle heartbeat_reset during shutdown
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9f865fae095a1fe8a26acb50667f1d774d6020b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jan 14 10:35:53 2015 +0000
+
+    mds: handle heartbeat_reset during shutdown
+    
+    Because any thread might grab mds_lock and call heartbeat_reset
+    immediately after a call to suicide() completes, this needs
+    to be handled as a special case where we tolerate MDS::hb having
+    already been destroyed.
+    
+    Fixes: #10382
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit a463b92e475cd1f4cdb963e402033ebc9d37dbdc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 19 18:28:20 2015 -0800
+
+    ceph_test_rados_api_misc: do not assert rbd feature match
+    
+    This test fails on upgrades when we (or the server) have new
+    features.  Make it less fragile.
+    
+    Fixes: #10576
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9147c62989871cea8b3a85b02c53017825efb55b)
+
+commit 14cdb9bb6d27f2017a3a8e6c1f274b9f40fb7456
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 21 14:55:02 2015 -0500
+
+    rbd: ensure aio_write buffer isn't invalidated during image import
+    
+    The buffer provided to aio_write shouldn't be invalidated until
+    after aio_write has indicated that the operation has completed.
+    
+    Fixes: #10590
+    Backport: giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 4d3b49e9d62bc1040356ca3ebe7f90c181734eb6)
+
+commit 081f49b47ca8d7583211f546ab5699b14f773bfc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 15 10:53:53 2014 -0500
+
+    librbd: complete all pending aio ops prior to closing image
+    
+    It was possible for an image to be closed while aio operations
+    were still outstanding.  Now all aio operations are tracked and
+    completed before the image is closed.
+    
+    Fixes: #10299
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 436923c68b77c900b7774fbef918c0d6e1614a36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 10:28:56 2015 -0500
+
+    librbd: gracefully handle deleted/renamed pools
+    
+    snap_unprotect and list_children both attempt to scan all
+    pools. If a pool is deleted or renamed during the scan,
+    the methods would previously return -ENOENT. Both methods
+    have been modified to more gracefully handle this condition.
+    
+    Fixes: #10270
+    Backport: giant, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4c8a5cedcb7942e1e01ab4cedfbf03e4c56cc1e4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 05:24:01 2014 -0800
+
+    rgw: change multipart upload id magic
+    
+    Fixes: #10271
+    Backport: firefly, giant
+    
+    Some clients can't sign requests correctly with the original magic
+    prefix.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5fc7a0be67a03ed63fcc8408f8d71a31a1841076)
+
+commit b10c0d5110547586b2edac53c267391d3d42f974
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Dec 11 09:07:10 2014 -0800
+
+    rgw: url decode http query params correctly
+    
+    Fixes: #10271
+    Backport: firefly
+    
+    This got broken by the fix for #8702. Since we now only url_decode if
+    we're in query, we need to specify that we're in query when decoding
+    these args.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 21e07eb6abacb085f81b65acd706b46af29ffc03)
+
+commit 65bf3b08b572b9b25ad064fb784742e5d6456f06
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 14 15:01:38 2015 -0800
+
+    qa: ignore duplicates in rados ls
+    
+    These can happen with split or with state changes due to reordering
+    results within the hash range requested. It's easy enough to filter
+    them out at this stage.
+    
+    Backport: giant, firefly
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7cc6117adf653a4915fb7a75fac68f8fa0239ec)
+
+commit 1261bf24624f871672002ab0915e23f1c95b0aa5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 12:42:40 2014 -0700
+
+    Revert "Objecter: disable fast dispatch of CEPH_MSG_OSD_OPREPLY messages"
+    
+    This reverts commit 3f23709c474292f9239f77a6cce26309fc86ce29.
+    
+    We have changed mark_down() behavior so that it no longer blocks on
+    fast dispatch.  This makes the objecter reply handler safe again.
+    
+    Fixes: #9598
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c9f9e72e558521cb90f90538bc27f995f82d76c2)
+
+commit 300d4c6ff7e998dba0c67f6dde746dc23d681397
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 12:41:48 2014 -0700
+
+    msg/simple: do not stop_and_wait on mark_down
+    
+    We originally blocked in mark_down for fast dispatch threads
+    to complete to avoid various races in the code.  Most of these
+    were in the OSD itself, where we were not prepared to get
+    messges on connections that had no attached session.  Since
+    then, the OSD checks have been cleaned up to handle this.
+    There were other races we were worried about too, but the
+    details have been lost in the depths of time.
+    
+    Instead, take the other route: make mark_down never block on
+    dispatch.  This lets us remove the special case that
+    was added in order to cope with fast dispatch calling
+    mark_down on itself.
+    
+    Now, the only stop_and_wait() user is the shutdown sequence.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 00907e032011b9d2acd16ea588555cf379830814)
+
+commit c3335c7aa6c1e6c3f1879c0cd3cd2f13091221be
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 16:25:09 2014 -0700
+
+    msg/Pipe: inject delay in stop_and_wait
+    
+    Inject a delay in stop_and_wait.  This will mostly affect the connection
+    race Pipe takeover code which currently calls stop_and_wait while holding
+    the msgr->lock.  This should make it easier for a racing fast_dispatch
+    method to get stuck on a call that (indirectly) needs the msgr lock.
+    See #9921.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fe5c4c305218fdb1771857e4e0ef7c98a8d0fb6)
+
+commit 1dbe8f5a6f7bf2b7c86d24f27d569d71e0076ee9
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 28 16:45:43 2014 -0700
+
+    SimpleMessenger: Pipe: do not block on takeover while holding global lock
+    
+    We previously were able to cause deadlocks:
+    1) Existing pipe is fast_dispatching
+    2) Replacement incoming pipe is accepted
+      *) blocks on stop_and_wait() of existing Pipe
+    3) External things are blocked on SimpleMessenger::lock() while
+      blocking completion of the fast dispatch.
+    
+    To resolve this, if we detect that an existing Pipe we want to take over is
+    in the process of fast dispatching, we unlock our locks and wait on it to
+    finish. Then we go back to the lookup step and retry.
+    
+    The effect of this should be safe:
+    1) We are not making any changes to the existing Pipe in new ways
+    2) We have not registered the new Pipe anywhere
+    3) We have not sent back any replies based on Messenger state to
+       the remote endpoint.
+    
+    Backport: giant
+    Fixes: #9921
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2d6980570af2226fdee0edfcfe5a8e7f60fae615)
+
+commit 16c023d8fa5575d4dd138aeee4d4fd9b8f32c0f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 13:34:52 2015 -0800
+
+    osd: requeue PG when we skip handling a peering event
+    
+    If we don't handle the event, we need to put the PG back into the peering
+    queue or else the event won't get processed until the next event is
+    queued, at which point we'll be processing events with a delay.
+    
+    The queue_null is not necessary (and is a waste of effort) because the
+    event is still in pg->peering_queue and the PG is queued.
+    
+    Note that this only triggers when we exceeed osd_map_max_advance, usually
+    when there is a lot of peering and recovery activity going on.  A
+    workaround is to increase that value, but if you exceed osd_map_cache_size
+    you expose yourself to crache thrashing by the peering work queue, which
+    can cause serious problems with heavily degraded clusters and bit lots of
+    people on dumpling.
+    
+    Backport: giant, firefly
+    Fixes: #10431
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 492ccc900c3358f36b6b14a207beec071eb06707)
+
+commit 16c6d0d589d53aad7bb2cd0e104300fb920d5caf
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 16 13:31:30 2014 +0100
+
+    erasure-code: relax cauchy w restrictions
+    
+    A restriction that the w parameter of the cauchy technique is limited to
+    8, 16 or 32 was added incorrectly while refactoring parameter parsing in
+    the jerasure plugin and must be relaxed.
+    
+    http://tracker.ceph.com/issues/10325 Fixes: #10325
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit bb80437f247345502203ad87a7e7bbb5b5602b9a)
+
+commit 636b98faa6b1c9fd6de1b8653d1d282577b54684
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 23 18:50:51 2014 -0800
+
+    crush/CrushWrapper: fix create_or_move_item when name exists but item does not
+    
+    We were using item_exists(), which simply checks if we have a name defined
+    for the item.  Instead, use _search_item_exists(), which looks for an
+    instance of the item somewhere in the hierarchy.  This matches what
+    get_item_weightf() is doing, which ensures we get a non-negative weight
+    that converts properly to floating point.
+    
+    Backport: giant, firefly
+    Fixes: #9998
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9902383c690dca9ed5ba667800413daa8332157e)
+
+commit ced2472664fab06d03de03d7b23325f9319163b7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:47:56 2014 -0800
+
+    crush/builder: prevent bucket weight underflow on item removal
+    
+    It is possible to set a bucket weight that is not the sum of the item
+    weights if you manually modify/build the CRUSH map.  Protect against any
+    underflow on the bucket weight when removing items.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8c87e9502142d5b4a282b94f929ae776a49be1dc)
+
+commit adf8798dabb679110c6815af5d73ab6ff20a1af8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:37:03 2014 -0800
+
+    crush/CrushWrapper: fix _search_item_exists
+    
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eeadd60714d908a3a033aeb7fd542c511e63122b)
+
+commit 6caa4fa42c6eaa76b3b9caf37e4ee09844f017a7
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Dec 18 20:00:28 2014 -0800
+
+    If trusty, use older version of qemu
+    
+    Fixes #10319
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry-picked from 46a1a4cb670d30397979cd89808a2e420cef2c11)
+
+commit 44c944e96440bd338d22533779e0650b99115a16
+Merge: abdbbd6 910ec62
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 10:55:22 2014 -0800
+
+    Merge pull request #3266 from ceph/giant-10415
+    
+    libcephfs/test.cc: close fd before umount
+
+commit b1e48820785a1d3153fc926ad21355b3927b44e9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Dec 28 10:29:54 2014 +0100
+
+    erasure-code: update links to jerasure upstream
+    
+    It moved from bitbucket to jerasure.org
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e86f901939f16cc9c8ad7a4108ac4bcf3916d2c)
+
+commit 910ec624156d26a1830078161f47328a950a4eee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Dec 23 10:22:00 2014 +0800
+
+    libcephfs/test.cc: close fd before umount
+    
+    Fixes: #10415
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3fb563cee4c4cf08ff4ee01782e52a100462429)
+
+commit abdbbd6e846727385cf0a1412393bc9759dc0244
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue Dec 16 22:01:26 2014 -0800
+
+    Remove sepia dependency (use fqdn)
+    
+    Fixes: #10255
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit 19dafe164833705225e168a686696fb4e170aba7)
+
+commit d1257436fdf79bad5fe0719a6be71e2abb2d2462 (refs/remotes/gh/giant-10277)
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 17 15:59:44 2014 +0800
+
+    client: use remount to trim kernel dcache
+    
+    when remounting a file system, linux kernel trims all unused dentry
+    in the file system.
+    
+    Fixes: #10277
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 9de9901cacd2ed2c8c5f65a938fb6a996efab4cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 17 15:46:49 2014 +0800
+
+    client: cleanup client callback registration
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit d28c8e0fb924fbf36e4e15e19554ad30da3ff8f2 (refs/remotes/gh/wip-giant-mon-backports)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:50:59 2014 -0800
+
+    mon/PGMap and PGMonitor: update last_epoch_clean cache from new osd keys
+    
+    We were only invalidating the cached value from apply_incremental, which
+    is no longer called on modern clusters.
+    
+    Fix this by storing the update epoch in the key as well (it is not part
+    of osd_stat_t).
+    
+    Backport: giant, firefly, dumpling(?)
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 093c5f0cabeb552b90d944da2c50de48fcf6f564)
+
+commit 7646f239476609c96b6baf94dfd5f727fff49502
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:49:48 2014 -0800
+
+    mon/PGMap: invalidate cached min_last_epoch_clean from new-style pg keys
+    
+    We were only invalidating the cache from the legacy apply_incremental(),
+    which is no longer called on modern clusters.
+    
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3fb731b722c50672a5a9de0c86a621f5f50f2d06)
+
+commit 6ec14b07940ff64d6a121e21a730f691a1a71546
+Merge: 758d9cf 7bbf80f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Dec 11 17:03:07 2014 -0800
+
+    Merge pull request #3159 from ceph/wip-10229-giant
+    
+    osdc/Filer: use finisher to execute C_Probe and C_PurgeRange [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7bbf80ff7388f104cf318dd5ac61ca7d35274694
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Dec 4 12:18:47 2014 +0800
+
+    osdc/Filer: use finisher to execute C_Probe and C_PurgeRange
+    
+    Currently contexts C_Probe/C_PurgeRange are executed while holding
+    OSDSession::completion_lock. C_Probe and C_PurgeRange may call
+    Objecter::stat() and Objecter::remove() respectively, which acquire
+    Objecter::rwlock. This can cause deadlock because there is intermediate
+    dependency between Objecter::rwlock and OSDSession::completion_lock:
+    
+     Objecter::rwlock -> OSDSession::lock -> OSDSession::completion_lock
+    
+    The fix is exexcute C_Probe/C_PurgeRange in finisher thread.
+    
+    Fixes: #10229
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3ee89ace660161df7796affbf9a70f3d0dedce1)
+
+commit 758d9cf30bfc7736cc297ba3b047756f7eb8183e
+Merge: a8e5638 994dcbb
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Dec 11 10:47:38 2014 -0800
+
+    Merge pull request #3151 from ceph/wip-10288-giant
+    
+    mon: fix `fs ls` on peons [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 994dcbbef4bea532aea4143c3ac1372ca14d2aea
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 11 14:00:57 2014 +0000
+
+    mon: fix `fs ls` on peons
+    
+    This was incorrectly using pending_mdsmap instead
+    of mdsmap.  We didn't notice in test because of
+    single-mon configurations.
+    
+    Fixes: #10288
+    Backport: giant
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5559e6aea9e9374ecdac0351777dfd6f5f5d1e67)
+
+commit 319f9c9352bfd1b95bd685500922e6cee2199b34 (refs/remotes/gh/wip-8797-giant)
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:53 2014 -0800
+
+    Call Rados.shutdown() explicitly before exit
+    
+    This is mostly a demonstration of good behavior, as the resources will
+    be reclaimed on exit anyway.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit b038e8fbf9103cc42a4cde734b3ee601af6019ea)
+
+commit ed8c9af3376aeb6f245cbab694fdbc0ce95634a8
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:16 2014 -0800
+
+    rados.py: remove Rados.__del__(); it just causes problems
+    
+    Recent versions of Python contain a change to thread shutdown that
+    causes ceph to hang on exit; see http://bugs.python.org/issue21963.
+    As it turns out, this is relatively easy to avoid by not spawning
+    threads on exit, as Rados.__del__() will certainly do by calling
+    shutdown(); I suspect, but haven't proven, that the problem is
+    that shutdown() tries to start() a threading.Thread() that never
+    makes it all the way back to signal start().
+    
+    Also add a PendingReleaseNote and extra doc comments to clarify.
+    
+    Fixes: #8797
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 5ba9b8f21f8010c59dd84a0ef2acfec99e4b048f)
+    
+    Conflicts:
+    	PendingReleaseNotes
+
+commit a8e56380f08cd5940def4cc47cadba699a8ba45d
+Merge: 247a6fa e7faed5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Dec 8 13:19:20 2014 -0800
+
+    Merge pull request #3010 from dachary/wip-10018-primary-erasure-code-hinfo-giant
+    
+    osd: deep scrub must not abort if hinfo is missing (giant)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 247a6fac54854e92a7df0e651e248a262d3efa05
+Merge: 3372060 309fd5f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 8 12:36:48 2014 -0800
+
+    Merge pull request #3110 from ceph/giant-10263
+    
+    mds: store backtrace for straydir
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 309fd5f56ef5ea76ffd525fdde6e6fbbc9ef6ef1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 7 11:38:37 2014 +0800
+
+    mds: store backtrace for straydir
+    
+    Backport: giant, firefly, emperor, dumpling
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0d89db5d3e5ae5d552d4058a88a4e186748ab1d2)
+
+commit 3372060894a1da0adef6d36380a131902ca05c5f
+Merge: 1f00420 bff6747
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 11:06:20 2014 -0800
+
+    Merge pull request #3088 from dachary/wip-10063-hobject-shard-giant
+    
+    common: do not omit shard when ghobject NO_GEN is set (giant)
+
+commit 1f004209434570337a3f90d7f89741f80dcc7075
+Merge: 3b65226 1ec557c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:33:12 2014 -0800
+
+    Merge pull request #3095 from dachary/wip-9785-dmcrypt-keys-permissions-giant
+    
+    ceph-disk: dmcrypt file permissions (giant)
+
+commit 3b65226df806958f6a2f24df6099ee3a86d2a71f
+Merge: 691f011 36c7484
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:30:31 2014 -0800
+
+    Merge pull request #3006 from dachary/wip-9420-erasure-code-non-regression-giant
+    
+     erasure-code: store and compare encoded contents (giant)
+
+commit 1ec557c0eab94cb898ad3f5448482bd7afc53e09
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 4 22:21:32 2014 +0100
+
+    ceph-disk: dmcrypt file permissions
+    
+    The directory in which key files are stored for dmcrypt must be 700 and
+    the file 600.
+    
+    http://tracker.ceph.com/issues/9785 Fixes: #9785
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 58682d1776ab1fd4daddd887d921ca9cc312bf50)
+
+commit 691f0118ecd051d5f3f61fc696280e3c482de3de
+Merge: 81295c5 dabf6f5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 09:03:54 2014 -0800
+
+    Merge pull request #3085 from dachary/wip-10125-radosgw-init-giant
+    
+    rgw: run radosgw as apache with systemd (giant)
+
+commit bff67475c775914237604ed3374c8ccfe74d0ffd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 01:16:10 2014 +0100
+
+    common: do not omit shard when ghobject NO_GEN is set
+    
+    Do not silence the display of shard_id when generation is NO_GEN.
+    Erasure coded objects JSON representation used by ceph_objectstore_tool
+    need the shard_id to find the file containing the chunk.
+    
+    Minimal testing is added to ceph_objectstore_tool.py
+    
+    http://tracker.ceph.com/issues/10063 Fixes: #10063
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit dcf09aed121f566221f539106d10283a09f15cf5)
+
+commit dabf6f5f43b53a588bd9fa0cc5aa617ae8128735
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 18:10:48 2014 +0100
+
+    rgw: run radosgw as apache with systemd
+    
+    Same as sysv.
+    
+    http://tracker.ceph.com/issues/10125 Fixes: #10125
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7b621f4abf63456272dec3449aa108c89504a7a5)
+    
+    Conflicts:
+    	src/init-radosgw.sysv
+
+commit 81295c5ad2befced2e308c1cfb4e036cd5a825a9
+Merge: 8046359 3ff94ed
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Dec 4 11:32:01 2014 -0800
+
+    Merge pull request #3077 from ceph/wip-10030-giant
+    
+    librbd: don't close an already closed parent image upon failure
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ad04a677cefc1f0a02fbff0c68409fda6874fdc7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 00:59:08 2014 +0100
+
+    common: add bufferlist::rebuild_aligned_size_and_memory
+    
+    The function bufferlist::rebuild_aligned checks memory and size
+    alignment with the same variable. It is however useful to separate
+    memory alignment constraints from size alignment constraints. For
+    instance rebuild_aligned could be called to allocate an erasure coded
+    buffer where each 2048 bytes chunk needs to start on a memory address
+    aligned on 32 bytes.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9ade88e8dacc9b96c042bb668f4452447139a544)
+
+commit cc469b238f42ce989d0efa49154b95612e3d4111
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 01:07:34 2014 +0100
+
+    erasure-code: enforce chunk size alignment
+    
+    Let say the ErasureCode::encode function is given a 4096 bytes
+    bufferlist made of a 1249 bytes bufferptr followed by a 2847 bytes
+    bufferptr, both properly starting on SIMD_ALIGN address. As a result the
+    second 2048 had to be reallocated when bufferlist::substr_of gets the
+    second 2048 buffer, the address starts at 799 bytes after the beginning
+    of the 2847 buffer ptr and is not SIMD_ALIGN'ed.
+    
+    The ErasureCode::encode must enforce a size alignment based on the chunk
+    size in addition to the memory alignment required by SIMD operations,
+    using the bufferlist::rebuild_aligned_size_and_memory function instead of
+    bufferlist::rebuild_aligned.
+    
+    http://tracker.ceph.com/issues/10211 Fixes: #10211
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 4e955f41297283798236c505c3d21bdcabb5caa0)
+
+commit 5205d4dacf7ebe2e42d2294bc30cb27f226c8d22
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 02:04:14 2014 +0100
+
+    common: allow size alignment that is not a power of two
+    
+    Do not assume the alignment is a power of two in the is_n_align_sized()
+    predicate. When used in the context of erasure code it is common
+    for chunks to not be powers of two.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 73ad2d63d479b09037d50246106bbd4075fbce80)
+
+commit 80463596919d28f58010d16ad017b3c5ae6e558c
+Merge: 26e8cf1 3dc6298
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 23:02:43 2014 -0800
+
+    Merge pull request #3062 from ceph/wip-10123-giant
+    
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 26e8cf174b8e76b4282ce9d9c1af6ff12f5565a9
+Merge: aac7946 7cd8c3f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Dec 3 06:44:56 2014 -0800
+
+    Merge pull request #3055 from ceph/wip-10135-giant
+    
+    mon: OSDMonitor: allow adding tiers to FS pools
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7cd8c3f8a5afa9481b6f6a78d5fb8c04784ef4ca (refs/remotes/gh/wip-10135-giant)
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Nov 25 16:54:42 2014 +0000
+
+    mon: OSDMonitor: allow adding tiers to FS pools
+    
+    This was an overly-strict check.  In fact it is perfectly
+    fine to set an overlay on a pool that is already in use
+    as a filesystem data or metadata pool.
+    
+    Fixes: #10135
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 17b5fc9a40440e76dd1fa64f7fc19577ae3b58ce)
+
+commit 3ff94ed73ff27af2c8ea215ab693d815e285a27f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 6 05:01:38 2014 -0500
+
+    librbd: don't close an already closed parent image upon failure
+    
+    If librbd is not able to open a child's parent image, it will
+    incorrectly close the parent image twice, resulting in a crash.
+    
+    Fixes: #10030
+    Backport: firefly, giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61ebfebd59b61ffdc203dfeca01ee1a02315133e)
+
+commit aac794616580ed0bb00608c5867199b12d4e1d94
+Merge: 65f6814 c8b46d6
+Author: John Spray <jcspray@gmail.com>
+Date:   Tue Dec 2 11:35:59 2014 +0000
+
+    Merge pull request #2990 from ceph/wip-10151-giant
+    
+    mon: fix MDS health status from peons
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 3dc629822adeee961d78208b46b9bd7ef1200890
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 17 21:49:26 2014 -0500
+
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    While listing child images, don't ignore error codes returned
+    from librados when creating an IoCtx. This will prevent seg
+    faults from occurring when an invalid IoCtx is used.
+    
+    Fixes: #10123
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d350b6817d7905908a4e432cd359ca1d36bab50)
+
+commit 65f6814847fe8644f5d77a9021fbf13043b76dbe
+Merge: 28e2708 9158326
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 1 17:59:19 2014 -0800
+
+    Merge pull request #3047 from ceph/wip-10011-giant
+    
+    osdc: fix Journaler write error handling [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9158326eeb69312283a6e8174352f36ea30d0cbf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 6 11:46:29 2014 +0000
+
+    osdc: fix Journaler write error handling
+    
+    Since we started wrapping the write error
+    handler in a finisher, multiple calls to
+    handle_write_error would hit the assert()
+    on the second call before the actual
+    handler had been called (at the other end
+    of the finisher) from the first call.
+    
+    The symptom was that the MDS was intermittently
+    failing to respawn on blacklist, seen in #10011.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 762eda88a18ba707bd5410f38e21e95c4a6b3a46)
+
+commit 28e27080e25f95851039a0cc4e1c1d06b2cd597d
+Merge: 37ffccb cb1d681
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 25 21:18:59 2014 -0800
+
+    Merge pull request #3005 from dachary/wip-9665-ceph-disk-partprobe-giant
+    
+    ceph disk zap must call partprobe
+
+commit e7faed5d903cf7681d77a6af53cf8137eeb2fc69
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Nov 6 17:11:20 2014 +0100
+
+    osd: deep scrub must not abort if hinfo is missing
+    
+    Instead it should set read_error.
+    
+    http://tracker.ceph.com/issues/10018 Fixes: #10018
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9d84d2e8309d26e39ca849a75166d2d7f2dec9ea)
+
+commit 36c7484c18fd072ba2f7b176403414dd32fbe92b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Sep 25 14:46:07 2014 +0200
+
+    erasure-code: erasure_code_benchmark exhaustive erasure exploration
+    
+    Add the --erasure-generation exhaustive flag to try all combinations of
+    erasures, not just one at random.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 2d7adb23bc52e7c0753f4571fecd8eefa209ef02)
+    
+    Conflicts:
+    	src/test/erasure-code/ceph_erasure_code_benchmark.h
+
+commit 32daa9b0f4d39f8a49512b18d5c19437aca5fec6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Sep 29 11:17:13 2014 +0200
+
+    erasure-code: add erasure_code_benchmark --verbose
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 3ff2816b3eecfb7277295583387549dac5429628)
+    
+    Conflicts:
+    	src/test/erasure-code/ceph_erasure_code_benchmark.cc
+    	src/test/erasure-code/ceph_erasure_code_benchmark.h
+
+commit da9a7f07787d7f8c20b0c3e7a53fcaf95ed7ca20
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 14:37:57 2014 +0200
+
+    erasure_code: implement ceph_erasure_code to assert the existence of a plugin
+    
+    This is handy when scripting in the context of teuthology and only
+    conditionally run tests for the isa plugin, for instance.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit efe121d9f2028c312eef2650d32ccf0cbc828edb)
+
+commit c855f3958fb8c10bd824075c1739f40799f6d74b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 14:36:08 2014 +0200
+
+    erasure-code: ceph_erasure_code does not need to avoid dlclose
+    
+    The only reason for not dlclosing plugins at exit is for callgrind but
+    ceph_erasure_code has no workload that would require callgrind.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 49613cb2aab6e73e3ea50fa164735b55e80121cd)
+
+commit ba8ceb1f067e0f9f6419358435ed0008b61fa438
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 11:38:09 2014 +0200
+
+    erasure-code: add corpus verification to make check
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 6fdbdff2ad1b55d4a37dcb95cfbb06c4454cdaf2)
+
+commit ca4c2702139cc7fd8f2e3fa2ee5cda4094ecad79
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 12:58:27 2014 +0200
+
+    erasure-code: Makefile.am cosmetics
+    
+    Cluster benchmark related lines together.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 10c88c8f27080a8e25f128b7065cee5c2f68e91b)
+
+commit 208a5ee1c9975adaa8b09b1bf541aff0d8551c63
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 12:55:26 2014 +0200
+
+    erasure-code: s/alignement/alignment/ typos in jerasure
+    
+    The jerasure-per-chunk-alignment prameter was mispelled and while
+    useable that would lead to confusion.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 2c84d0b1db57d918840e669a17bbd8c5ddca9747)
+
+commit 1def82d530965bd0441e4f7f6aa032666984f17d
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 13:36:09 2014 +0200
+
+    erasure-code: workunit to check for encoding regression
+    
+    Clone the archive of encoded objects and decode all archived objects, up
+    to and including the current ceph version.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 7638b15f23976c3265cf766e16cf93af1a7e0091)
+
+commit 1b7fc7e49e6edf0d0f7d1d6d9f9447c42067d8b8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 10:16:31 2014 +0200
+
+    erasure-code: store and compare encoded contents
+    
+    Introduce ceph_erasure_code_non_regression to check and compare how an
+    erasure code plugin encodes and decodes content with a given set of
+    parameters. For instance:
+    
+    ./ceph_erasure_code_non_regression \
+          --plugin jerasure \
+          --parameter technique=reed_sol_van \
+          --parameter k=2 \
+          --parameter m=2 \
+          --stripe-width 3181 \
+          --create \
+          --check
+    
+    Will create an encoded object (--create) and store it into a directory
+    along with the chunks, one chunk per file. The directory name is derived
+    from the parameters. The content of the object is a random pattern of 31
+    bytes repeated to fill the object size specified with --stripe-width.
+    
+    The check function (--check) reads the object back from the file,
+    encodes it and compares the result with the content of the chunks read
+    from the files. It also attempts recover from one or two erasures.
+    
+    Chunks encoded by a given version of Ceph are expected to be encoded
+    exactly in the same way by all Ceph versions going forward.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit f5901303dbf50e9d08f2f1e510a1936a20037909)
+
+commit cb1d68113477cf9c2028a65372d2d4a3e6a8bdc1
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 9 18:52:17 2014 +0200
+
+    ceph-disk: run partprobe after zap
+    
+    Not running partprobe after zapping a device can lead to the following:
+    
+    * ceph-disk prepare /dev/loop2
+    * links are created in /dev/disk/by-partuuid
+    * ceph-disk zap /dev/loop2
+    * links are not removed from /dev/disk/by-partuuid
+    * ceph-disk prepare /dev/loop2
+    * some links are not created in /dev/disk/by-partuuid
+    
+    This is assuming there is a bug in the way udev events are handled by
+    the operating system.
+    
+    http://tracker.ceph.com/issues/9665 Fixes: #9665
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fed3b06c47a5ef22cb3514c7647544120086d1e7)
+
+commit d9c04b880d8bf867aa454132117119be5bd550ad
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:26:31 2014 +0200
+
+    ceph-disk: use update_partition in prepare_dev and main_prepare
+    
+    In the case of prepare_dev the partx alternative was missing and is not
+    added because update_partition does it.
+    
+    http://tracker.ceph.com/issues/9721 Fixes: #9721
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 23e71b1ee816c0ec8bd65891998657c46e364fbe)
+
+commit 5c9cd3c2a292ae540fd0a487defaf4d712e41e62
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:23:34 2014 +0200
+
+    ceph-disk: encapsulate partprobe / partx calls
+    
+    Add the update_partition function to reduce code duplication.
+    The action is made an argument although it always is -a because it will
+    be -d when deleting a partition.
+    
+    Use the update_partition function in prepare_journal_dev
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 922a15ea6865ef915bbdec2597433da6792c1cb2)
+
+commit c8b46d68c71f66d4abbda1230741cc4c7284193b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 24 11:00:25 2014 +0000
+
+    mon: fix MDS health status from peons
+    
+    The health data was there, but we were attempting
+    to enumerate MDS GIDs from pending_mdsmap (empty on
+    peons) instead of mdsmap (populated from paxos updates)
+    
+    Fixes: #10151
+    Backport: giant
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0c33930e3a90f3873b7c7b18ff70dec2894fce29)
+    
+    Conflicts:
+    	src/mon/MDSMonitor.cc
+
+commit 37ffccbd57e7e441e0eb1499e5c173aa9c375d35
+Merge: b13a56a 65c5657
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Nov 20 13:13:33 2014 -0800
+
+    Merge pull request #2975 from ceph/wip-9936-giant
+    
+    rbd: Fix the rbd export when image size more than 2G
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b13a56afe99c091600392a2fc15befa9438d59c9
+Merge: 828c1a2 46bd344
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 19 02:40:47 2014 +0100
+
+    Merge pull request #2963 from ceph/wip-10114-giant
+    
+    Wip 10114 giant
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 828c1a2bcd81a49264f8a81ca7f1070169037820
+Merge: ccfd241 6cb9a24
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Nov 18 15:48:16 2014 -0800
+
+    Merge pull request #2958 from ceph/wip-10128-giant
+    
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 46bd3441b00c22ba78f04617fd77f0231ccc698d
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Nov 18 15:21:30 2014 -0800
+
+    erasure-code isa-l: remove duplicated lines (fix warning)
+    
+    06a245a added a section def to assembly files; I added it twice to
+    this file.  There's no damage, but a compiler warning (on machines with
+    yasm installed)
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 10f6ef185a9d09e396e94036ec90bfe8a0738ce9)
+
+commit 1bba3887038aade137a808d751561cc02002f4bf
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Fri Nov 14 17:59:57 2014 -0800
+
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    See discussion in http://tracker.ceph.com/issues/10114
+    
+    Building with these changes allows output from readelf like this:
+    
+     $ readelf -lW src/.libs/librados.so.2 | grep GNU_STACK
+      GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000
+    0x000000 RW  0x8
+    
+    (note the absence of 'X' in 'RW')
+    
+    Fixes: #10114
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 06a245a9845c0c126fb3106b41b2fd2bc4bc4df3)
+
+commit 6cb9a2499cac2645e2cc6903ab29dfd95aac26c7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:02:50 2014 -0800
+
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Fixes: #10128
+    Caused by a2bd2aa7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0d5262ac2f69ed3996af76a72894b1722a27b37d)
+
+commit 13bb880b2a04ab354506eb183d2907b9054bf937
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue Nov 18 10:30:36 2014 +0100
+
+    Fixed locking issue in the trun method of libradosstriper leading to potential race conditions - Fixes: #10129
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 8613984373de946e1815cc84d50bbd4437a3f7a7)
+
+commit 65c565701eb6851f4ed4d2dbc1c7136dfaad6bcb
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Wed Oct 29 12:21:11 2014 +0800
+
+    rbd: Fix the rbd export when image size more than 2G
+    
+    When using export <image-name> <path> and the size of image is more
+    than 2G, the previous version about finish() could not handle in
+    seeking the offset in image and return error.
+    
+    This is caused by the incorrect variable type. Try to use the correct
+    variable type to fixed it.
+    
+    I use another variable which type is uint64_t for confirming seeking
+    and still use the previous r for return error.
+    
+    uint64_t is more better than type int for handle lseek64().
+    
+    Signed-off-by: Vicente Cheng <freeze.bilsted@gmail.com>
+    (cherry picked from commit 4b87a81c86db06f6fe2bee440c65fc05cd4c23ce)
+
+commit ccfd2414c68afda55bf4cefa2441ea6d53d87cc6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:11:10 2014 -0800
+
+    osd/OSD: use OSDMap helper to determine if we are correct op target
+    
+    Use the new helper.  This fixes our behavior for EC pools where targetting
+    a different shard is not correct, while for replicated pools it may be. In
+    the EC case, it leaves the op hanging indefinitely in the OpTracker because
+    the pgid exists but as a different shard.
+    
+    Fixes: #9835
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e05ba086a36ae9a04b347153b685c2b8adac2c3)
+
+commit 963947718a954f63f351ce4034bf97380421ab7c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:04:35 2014 -0800
+
+    osd/OSDMap: add osd_is_valid_op_target()
+    
+    Helper to check whether an osd is a given op target for a pg.  This
+    assumes that for EC we always send ops to the primary, while for
+    replicated we may target any replica.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89c02637914ac7332e9dbdbfefc2049b2b6c127d)
+
+commit 0cb32c157c6c11b26607521a20c6f320c5170516
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 11 18:16:02 2014 -0800
+
+    qa: allow small allocation diffs for exported rbds
+    
+    The local filesytem may behave slightly differently. This isn't
+    foolproof, but seems to be reliable enough on rhel7 rootfs, where
+    exact comparison was failing.
+    
+    Fixes: #10002
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e94d3c11edb9c9cbcf108463fdff8404df79be33)
+
+commit fe705c8fdef2371d3f5b11eb73f87a0cf6ef0f9e
+Author: Adam Crume <adamcrume@gmail.com>
+Date:   Thu Sep 18 16:57:27 2014 -0700
+
+    common: Add cctid meta variable
+    
+    Fixes: #6228
+    Signed-off-by: Adam Crume <adamcrume@gmail.com>
+    (cherry picked from commit bb45621cb117131707a85154292a3b3cdd1c662a)
+
+commit 5fc659a0d52094a4c595ca8b33b407ecdefc180a
+Merge: b27f5db a6c02a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 11 08:28:19 2014 -0800
+
+    Merge pull request #2804 from ceph/wip-9301-giant
+    
+    mon: backport paxos off-by-one bug (9301) to giant
+
+commit b27f5dba8677ca48c9819980e3c90b76f5f04267
+Merge: 97e423f fc5354d
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Nov 10 22:41:19 2014 -0800
+
+    Merge pull request #2887 from ceph/wip-9977-backport
+    
+    tools: skip up to expire_pos in journal-tool
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 97e423f52155e2902bf265bac0b1b9ed137f8aa0
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Sep 11 09:36:44 2014 +0800
+
+    client: trim unused inodes before reconnecting to recovering MDS
+    
+    So the recovering MDS does not need to fetch these ununsed inodes during
+    cache rejoin. This may reduce MDS recovery time.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2bd7ceeff53ad0f49d5825b6e7f378683616dffb)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 387efc5fe1fb148ec135a6d8585a3b8f8d97dbf8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 27 12:02:17 2014 +0000
+
+    client: allow xattr caps in inject_release_failure
+    
+    Because some test environments generate spurious
+    rmxattr operations, allow the client to release
+    'X' caps.  Allows xattr operations to proceed
+    while still preventing client releasing other caps.
+    
+    Fixes: #9800
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5691c68a0a44eb2cdf0afb3f39a540f5d42a5c0c)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit fc5354dec55248724f8f6b795e3a96882c33b490 (refs/remotes/gh/wip-9977-backport)
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 3 19:19:45 2014 +0000
+
+    tools: skip up to expire_pos in journal-tool
+    
+    Previously worked for journals starting from an
+    object boundary (i.e. freshly created filesystems)
+    
+    Fixes: #9977
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 65c33503c83ff8d88781c5c3ae81d88d84c8b3e4)
+    
+    Conflicts:
+    	src/tools/cephfs/JournalScanner.cc
+
+commit 9680613141b3eef62f35a6728e654efa5f6ba8e8
+Merge: fd4363d a5984ba
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 16:26:54 2014 -0800
+
+    Merge pull request #2876 from ceph/giant-readdir-fix
+    
+    Giant readdir fix
+
+commit fd4363d1bd49f73e1b3c22516686c7b7e1745b57
+Merge: f66bf31 7166ff8
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 14:10:40 2014 -0800
+
+    Merge pull request #2879 from ceph/wip-10025-giant
+    
+    #10025/giant -- tools: fix MDS journal import
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7166ff83f6343d31d52a58363e2767434554505c
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 7 11:34:43 2014 +0000
+
+    tools: fix MDS journal import
+    
+    Previously it only worked on fresh filesystems which
+    hadn't been trimmed yet, and resulted in an invalid
+    trimmed_pos when expire_pos wasn't on an object
+    boundary.
+    
+    Fixes: #10025
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fb29e71f9a97c12354045ad2e128156e503be696)
+
+commit a5984ba34cb684dae623df22e338f350c8765ba5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 27 13:57:16 2014 -0700
+
+    client: fix I_COMPLETE_ORDERED checking
+    
+    Current code marks a directory inode as complete and ordered when readdir
+    finishes, but it does not check if the directory was modified in the middle
+    of readdir. This is wrong, directory inode should not be marked as ordered
+    if it was modified during readddir
+    
+    The fix is introduce a new counter to the inode data struct, we increase
+    the counter each time the directory is modified. When readdir finishes, we
+    check the counter to decide if the directory should be marked as ordered.
+    
+    Fixes: #9894
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a4caed8a53d011b214ab516090676641f7c4699d)
+
+commit b5ff4e99c87958211e4b7716b59084fc3417ec17
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Tue Sep 9 17:34:46 2014 +0800
+
+    client: preserve ordering of readdir result in cache
+    
+    Preserve ordering of readdir result in a list, so that the result of cached
+    readdir is consistant with uncached readdir.
+    
+    As a side effect, this commit also removes the code that removes stale dentries.
+    This is OK because stale dentries does not have valid lease, they will be
+    filter out by the shared gen check in Client::_readdir_cache_cb()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 346c06c1647658768e927a47768a0bc74de17b53)
+
+commit 0671c1127015311d9894f15e2493805f93432910
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Tue Sep 9 14:06:06 2014 +0800
+
+    client: introduce a new flag indicating if dentries in directory are sorted
+    
+    When creating a file, Client::insert_dentry_inode() set the dentry's offset
+    based on directory's max offset. The offset does not reflect the real
+    postion of the dentry in directory. Later readdir reply from real postion
+    of the dentry in directory. Later readdir reply from MDS may change the
+    dentry's position/offset. This inconsistency can cause missing/duplicate
+    entries in readdir result if readdir is partly satisfied by dcache_readdir().
+    
+    The fix is introduce a new flag indicating if dentries in directory are
+    sorted. We use _readdir_cache_cb() to handle readdir only when the flag is
+    set, clear the flag after creating/deleting/renaming file.
+    
+    Fixes: #9178
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 600af25493947871c38214aa370e2544a7fea399)
+
+commit f66bf31b6743246fb1c88238cf18101238dee3a4
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Nov 6 17:48:01 2014 -0800
+
+    qa: use sudo even more when rsyncing /usr
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 3aa7797741f9cff06053a2f31550fe6929039692)
+
+commit f7ec448d6579f965eec184416a97b47ae27ab47a
+Merge: f410d76 f111bc8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 5 08:51:18 2014 +0100
+
+    Merge pull request #2858 from ceph/wip-9909
+    
+    tools: rados put /dev/null should write() and not create()
+    
+    Reviewed-by: Loic Dachary <loic-201408@dachary.org>
+
+commit f111bc8eac8a521b13340f4a75418d839725e010
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 2 09:23:55 2014 +0200
+
+    tools: rados put /dev/null should write() and not create()
+    
+    In the rados.cc special case to handle put an empty objects, use
+    write_full() instead of create().
+    
+    A special case was introduced 6843a0b81f10125842c90bc63eccc4fd873b58f2
+    to create() an object if the rados put file is empty. Prior to this fix
+    an attempt to rados put an empty file was a noop. The problem with this
+    fix is that it is not idempotent. rados put an empty file twice would
+    fail the second time and rados put a file with one byte would succeed as
+    expected.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 50e80407f3c2f74d77ba876d01e7313c3544ea4d)
+
+commit f410d764d2e6795389cb320b4436cff3607927bd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 9 10:20:27 2014 -0700
+
+    rgw: set length for keystone token validation request
+    
+    Fixes: #7796
+    Backport: giany, firefly
+    Need to set content length to this request, as the server might not
+    handle a chunked request (even though we don't send anything).
+    
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3dd4ccad7fe97fc16a3ee4130549b48600bc485c)
+
+commit dba7defc623474ad17263c9fccfec60fe7a439f0
+Merge: 6a201f8 e0b0441
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 08:35:42 2014 -0700
+
+    Merge pull request #2846 from dachary/wip-9752-past-intervals-giant
+    
+    osd: past_interval display bug on acting
+
+commit e0b04414b92018277a0d3b9d82e72ea7529f4ef5
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 31 00:49:21 2014 +0100
+
+    osd: past_interval display bug on acting
+    
+    The acting array was incorrectly including the primary and up_primary.
+    
+    http://tracker.ceph.com/issues/9752 Fixes: #9752
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c5f8d6eded52da451fdd1d807bd4700221e4c41c)
+
+commit 6a201f89b1aa6c2197383c29919cdeb4a8353d1b
+Merge: ebe1637 905aba2
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Oct 30 17:01:12 2014 -0700
+
+    Merge pull request #2841 from ceph/giant-9869
+    
+    Backport "client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid"
+
+commit 905aba2f3d847933f98124f3ea8d1d76d644edb4
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Oct 22 17:16:31 2014 -0700
+
+    client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid
+    
+    m->get_client_tid() is 64 bits (as it should be), but Inode::flushing_cap_tid
+    is only 16 bits. 16 bits should be plenty to let the cap flush updates
+    pipeline appropriately, but we need to cast in the proper direction when
+    comparing these differently-sized versions. So downcast the 64-bit one
+    to 16 bits.
+    
+    Fixes: #9869
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a5184cf46a6e867287e24aeb731634828467cd98)
+
+commit ebe16373e966917ca8cb03ebeac974bdff7b7685
+Merge: c51c8f9 b704f0d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 30 10:05:22 2014 -0700
+
+    Merge pull request #2838 from ceph/wip-9945-giant
+    
+    messages: fix COMPAT_VERSION on MClientSession
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b704f0dd888aacb10c32cdb63cdbf9f06296fc18
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 30 16:43:21 2014 +0000
+
+    messages: fix COMPAT_VERSION on MClientSession
+    
+    This was incorrectly incremented to 2 by omission
+    of an explicit COMPAT_VERSION value.
+    
+    Fixes: #9945
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1eb9bcb1d36014293efc687b4331be8c4d208d8e)
+
+commit a6c02a18d6249ea62cf7a74710c8d0192b6eecaa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:23:36 2014 -0700
+
+    mon: re-bootstrap if we get probed by a mon that is way ahead
+    
+    During bootstrap we verify that our paxos commits overlap with the other
+    mons we will form a quorum with.  If they do not, we do a sync.
+    
+    However, it is possible we pass those checks, then fail to join a quorum
+    before the quorum moves ahead in time such that we no longer overlap.
+    Currently nothing kicks up back into a probing state to discover we need
+    to sync... we will just keep trying to call or join an election instead.
+    
+    Fix this by jumping back to bootstrap if we get a probe that is ahead of
+    us.  Only do this from non probe or sync states as these will be common;
+    it is only the active and electing states that matter (and probably just
+    electing!).
+    
+    Fixes: #9301
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c421b55e8e15ef04ca8aeb47f7d090375eaa8573)
+
+commit 92d2a38efd458f9e8f4da228ea1e94df08dc8222
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:11:24 2014 -0700
+
+    mon/Paxos: fix off-by-one in last_ vs first_committed check
+    
+    peon last_committed + 1 == leader first_committed is okay.  Note that the
+    other check (where I clean up whitespace) gets this correct.
+    
+    Fixes: #9301 (partly)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d81cd7f86695185dce31df76c33c9a02123f0e4a)
diff --git a/doc/changelog/v0.87.2.txt b/doc/changelog/v0.87.2.txt
new file mode 100644
index 000000000..98cf11732
--- /dev/null
+++ b/doc/changelog/v0.87.2.txt
@@ -0,0 +1,1413 @@
+commit 87a7cec9ab11c677de2ab23a7668a77d2f5b955e (tag: refs/tags/v0.87.2, refs/remotes/gh/giant)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Apr 24 12:31:27 2015 -0700
+
+    0.87.2
+
+commit c1301e84aee0f399db85e2d37818a66147a0ce78
+Merge: 1a13e10 9e9c3c6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 7 21:08:24 2015 +0200
+
+    Merge pull request #4214 from dachary/wip-10430-giant
+    
+    osd/osd_types.cc: 456: FAILED assert(m_seed < old_pg_num)
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 1a13e1065829c59987c2f57a13eaa03de31df4ed
+Merge: 1fb08d3 5f4e62f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 7 16:39:28 2015 +0200
+
+    Merge pull request #4258 from ceph/wip-10643-v2
+    
+    mon: MDSMonitor: additional backports for #10643
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1fb08d3066b14b178a8912ffb3c9f50d2333738c
+Merge: 90b37d9 7684ee2
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Apr 2 08:55:48 2015 -0700
+
+    Merge pull request #4261 from ceph/wip-11303-giant
+    
+    allow -L to disable lttng. Enable it by default
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7684ee25ac21810153a44bdc4fc00b36e39eb12f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Apr 2 10:34:02 2015 -0400
+
+    allow -L to disable lttng. Enable it by default
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit 5f4e62f382767ee69e5b0c701b1a01d9e4132237
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 19:08:20 2014 +0100
+
+    mon: MDSMonitor: wait for osdmon to be writable when requesting proposal
+    
+    Otherwise we may end up requesting the osdmon to propose while it is
+    mid-proposal.  We can't simply return EAGAIN to the user either because
+    then we would have to expect the user to be able to successfully race
+    with the whole cluster in finding a window in which 'mds fs new' command
+    would succeed -- which is not a realistic expectation.  Having the
+    command to osdmon()->wait_for_writable() guarantees that the command
+    will be added to a queue and that we will, eventually, tend to it.
+    
+    Fixes: #9794
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2ae1cba595d9c56a0a4c534b34fe25250e7eb2d5)
+
+commit 257bd17db6470ca050403b1c8ff8daa94a4b80b5
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 18:59:51 2014 +0100
+
+    mon: MDSMonitor: have management_command() returning int instead of bool
+    
+    We can more easily differentiate between errors we get out of this
+    function, which makes the code a bit more versatile and readable.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 0dd473cbad4f9ea403fe60badffdc6da4dd3aa3c)
+
+commit 9e9c3c652339d85863af01cac621228f04eb4f18
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 9 11:20:13 2014 -0700
+
+    osd: Get pgid ancestor from last_map when building past intervals
+    
+    Fixed OSD::build_past_intervals_parallel() and PG::generate_past_intervals()
+    
+    Fixes: #10430
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c5b66da7a9ba516340d06d9e806beb9d1040d0e)
+
+commit 90b37d9bdcc044e26f978632cd68f19ece82d19a
+Merge: 2ccbc14 9f1f355
+Author: Loic Dachary <loic-redhat@dachary.org>
+Date:   Thu Mar 26 07:58:14 2015 +0100
+
+    Merge pull request #4175 from wonzhq/objecter-timer-2
+    
+    Objecter: failed assert(tick_event==NULL) at osdc/Objecter.cc
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9f1f35546e00e8f1ecbce0697d59b64f3537facf
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Wed Mar 25 16:32:44 2015 +0800
+
+    Objecter: failed assert(tick_event==NULL) at osdc/Objecter.cc
+    
+    When the Objecter timer erases the tick_event from its events queue and
+    calls tick() to dispatch it, if the Objecter::rwlock is held by shutdown(),
+    it waits there to get the rwlock. However, inside the shutdown function,
+    it checks the tick_event and tries to cancel it. The cancel_event function
+    returns false since tick_event is already removed from the events queue. Thus
+    tick_event is not set to NULL in shutdown(). Later the tick function return
+    ealier and doesn't set tick_event to NULL as well. This leads to the assertion
+    failure.
+    
+    This is a regression introduced by an incorrect conflict resolution when
+    d790833 was backported.
+    
+    Fixes: #11183
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+
+commit 2ccbc14d17b54ea4fd4126cb04a7b83cd64c7f1e
+Merge: 02f9cdb de4b087
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 20:39:26 2015 +0100
+
+    Merge pull request #4127 from dzafman/wip-11176-giant
+    
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 02f9cdbf889071ca6fe3811d9b9a92a0b630fa55
+Merge: 83bcc51 fc43d8c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:46 2015 +0100
+
+    Merge pull request #4097 from dachary/wip-10497-giant
+    
+    librados: c api does not translate op flag
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 83bcc516743e426c7a8c6f6401721bffbbec4fc0
+Merge: ebab2bd d790833
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:26 2015 +0100
+
+    Merge pull request #4096 from dachary/wip-9617-giant
+    
+    objecter shutdown races with msg dispatch
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit ebab2bd5f36205d666673600624aaa3e5e06c405
+Merge: e31c92d 970a797
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:03 2015 +0100
+
+    Merge pull request #4095 from dachary/wip-9675-giant
+    
+    splitting a pool doesn't start when rule_id != ruleset_id
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit e31c92d8dd814d276357e431ed87b93d75933f77
+Merge: f0ec5e3 7653511
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:10:42 2015 +0100
+
+    Merge pull request #4094 from dachary/wip-9891-giant
+    
+    Assertion: os/DBObjectMap.cc: 1214: FAILED assert(0)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit f0ec5e3b3ed58af65323bcc494e589935147aa45
+Merge: dd7c15b 13b0147
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:10:25 2015 +0100
+
+    Merge pull request #4093 from dachary/wip-9915-giant
+    
+    osd: eviction logic reversed
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit dd7c15b2b24027a7cc5fa4bff21222c5a4606e60
+Merge: 33b09e1 13b8364
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:09:28 2015 +0100
+
+    Merge pull request #4092 from dachary/wip-9985-giant
+    
+    osd: incorrect atime calculation
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 33b09e18aa78252d94cbec8bf94ec97ed5bb1573
+Merge: 950123e 5550cdd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:08:41 2015 +0100
+
+    Merge pull request #4091 from dachary/wip-9986-giant
+    
+    objecter: map epoch skipping broken
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 950123e4daa85562b2f52e0e12e0bae07f444095
+Merge: abdc065 21f81b8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:08:16 2015 +0100
+
+    Merge pull request #4090 from dachary/wip-10059-giant
+    
+    osd/ECBackend.cc: 876: FAILED assert(0)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit abdc065acd6cafa6439e9c1724cc87de02352bd7
+Merge: aee2825 1ccf583
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:07:52 2015 +0100
+
+    Merge pull request #4089 from dachary/wip-10080-giant
+    
+    Pipe::connect() cause osd crash when osd reconnect to its peer
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit aee28250fee66bca08d91e56ce47ec46c2e9fc24
+Merge: 6582253 3e875ab
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:07:20 2015 +0100
+
+    Merge pull request #4088 from dachary/wip-6003-giant
+    
+    journal Unable to read past sequence 406 ...
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 65822530ebfdea2feb9192c6eb6e3b8b9d60fe33
+Merge: 4b20f2d 96a5c67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:06:51 2015 +0100
+
+    Merge pull request #4082 from dachary/wip-10106-giant
+    
+    rgw acl response should start with <?xml version=1.0 ?>
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 4b20f2d2d1ee52deed33617f000fa342ebce2e49
+Merge: 7ff3a67 c7b02f5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:06:23 2015 +0100
+
+    Merge pull request #4078 from dachary/wip-11157-giant
+    
+    doc,tests: force checkout of submodules
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 7ff3a67c44ba3dc20a663a7dc6ba28c25714f063
+Merge: 440e706 4d4eb9f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:05:20 2015 +0100
+
+    Merge pull request #4077 from dachary/wip-10150-giant
+    
+    osd/ReplicatedPG.cc: 10853: FAILED assert(r >= 0) (in _scan_range)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 440e70607e7e3cd1d8ca33843c626109431caf8d
+Merge: 66f639b 499d94f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:04:51 2015 +0100
+
+    Merge pull request #4076 from dachary/wip-10153-giant
+    
+    Rados.shutdown() dies with Illegal instruction (core dumped)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 66f639b30ce6c74caae9397f20053761203f8e87
+Merge: 43b45df b79852f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:04:25 2015 +0100
+
+    Merge pull request #4074 from dachary/wip-10220-giant
+    
+    mon/Paxos.cc: 1033: FAILED assert(mon->is_leader())
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 43b45dfacd598bab51fa06c5d0e2d0605d6e83d6
+Merge: d282cfd b318e2f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:03:35 2015 +0100
+
+    Merge pull request #3548 from ceph/wip-10643
+    
+    mon: MDSMonitor: missing backports for giant
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d282cfd5ae3e65b74801cd27480ce8c0bd72d9a7
+Merge: 3f3b981 681c99f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:58 2015 +0100
+
+    Merge pull request #4053 from dachary/wip-8011-giant
+    
+    osd/ReplicatedPG.cc: 5244: FAILED assert(soid < scrubber.start || soid >= scrubber.end)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 3f3b98123cb46080068c1c73f4be41acbe18bd0d
+Merge: 6919eb1 4427358
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:40 2015 +0100
+
+    Merge pull request #4052 from dachary/wip-10844-giant
+    
+    mon: caps validation should rely on EntityName instead of entity_name_t
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 6919eb1684e34a6395963be6cc65215a51f5ba13
+Merge: 8876585 1d4ffbe
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:15 2015 +0100
+
+    Merge pull request #4050 from dachary/wip-10817-giant
+    
+    WorkQueue: make timeout when calling WaitInterval configurable
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 88765851bd56a6a526f6ab724920c8858ec4956c
+Merge: 6da3171 92c352d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:11:43 2015 +0100
+
+    Merge pull request #4049 from dachary/wip-10787-giant
+    
+    mon: OSDMonitor::map_cache is buggy, send_incremental is not conservative
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 6da3171f21b8e1a56fe941a5028f2ccfdccee18a
+Merge: 5a6eefc 25fcaca
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:09:28 2015 +0100
+
+    Merge pull request #4048 from dachary/wip-10770-giant
+    
+    rgw: pending bucket index operations are not cancelled correctly
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 5a6eefcb40c94da9a900d9893ecb6eaaf8fd0cea
+Merge: c67a7a5 2858327
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:09:06 2015 +0100
+
+    Merge pull request #4046 from dachary/wip-10723-giant
+    
+    rados python binding leaks Ioctx objects
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit c67a7a52dc96f176431125921e36e4a2b8a30f1c
+Merge: 41dcd2d d5b1b7e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:08:45 2015 +0100
+
+    Merge pull request #4044 from dachary/wip-10617-giant
+    
+    osd: pgs for deleted pools don't finish getting removed if osd restarts
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 41dcd2d9c307dd0b25d27bd6673943f3fdaaa28b
+Merge: 42e7413 c3d998e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:08:20 2015 +0100
+
+    Merge pull request #4034 from dachary/wip-10475-giant
+    
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 42e741339913990521c4509f4e266921f8dbe007
+Merge: ffb76d1 fa8d454
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:07:53 2015 +0100
+
+    Merge pull request #4033 from dachary/wip-10471-giant
+    
+    rgw: index swift keys appropriately
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit de4b08704172ac31b511dde50e5c11d58d811ca2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 16:57:40 2015 -0700
+
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Fixes: #11176
+    Backport: firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5b23f5b5892b36fb7d06efc0d77e64a24ef6e8c9)
+
+commit ffb76d16c5d9d65ac94d21b4e8fc1fdf86441977
+Merge: fea29b1 61d6006
+Author: John Spray <jcspray@gmail.com>
+Date:   Thu Mar 19 22:07:58 2015 +0000
+
+    Merge pull request #3971 from ceph/giant-11053
+    
+    mds: fix assertion caused by system clock backwards
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c3d998e4c3c9d90b50c3f6c0af7d48785616e032
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Tue Feb 3 17:54:38 2015 +0200
+
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Fixes: #10475
+    Backport: hammer, firefly
+    Reported-by: Josh Durgin <jdurgin@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit f67bfa24fd6f69c2fcc0987eba8b6b426dd78320)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+            trivial merge: prototype of an unrelated function changed
+            src/rgw/rgw_op.cc
+            s/is_object_op/!(s->object == NULL)/
+
+commit fc43d8c2c54ec9e9cb6ef4d19cca695eb2fb3aab
+Author: Matt Richards <mattjrichards@gmail.com>
+Date:   Thu Jan 8 13:16:17 2015 -0800
+
+    librados: Translate operation flags from C APIs
+    
+    The operation flags in the public C API are a distinct enum
+    and need to be translated to Ceph OSD flags, like as happens in
+    the C++ API. It seems like the C enum and the C++ enum consciously
+    use the same values, so I reused the C++ translation function.
+    
+    Signed-off-by: Matthew Richards <mattjrichards@gmail.com>
+    (cherry picked from commit 49d114f1fff90e5c0f206725a5eb82c0ba329376)
+
+commit d790833cb84d6f6349146e4f9abdcdffb4db2ee0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 29 18:17:29 2014 -0700
+
+    Objecter: check the 'initialized' atomic_t safely
+    
+    shutdown() resets initialized to 0, but we can still receive messages
+    after this point, so fix message handlers to skip messages in this
+    case instead of asserting.
+    
+    Also read initialized while holding Objecter::rwlock to avoid races
+    where e.g. handle_osd_map() checks initialized -> 1, continues,
+    shutdown() is called, sets initialized to 0, then handle_osd_map()
+    goes about its business and calls op_submit(), which would fail the
+    assert(initialized.read()) check. Similar races existed in other
+    message handlers which change Objecter state.
+    
+    The Objecter is not destroyed until after its Messenger in
+    the MDS, OSD, and librados, so this should be safe.
+    
+    Fixes: #9617
+    Backport: giant
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e506f896a9217324ab7a7865989f4454562aed5f)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            context changed: Objecter::tick() did not have
+            assert(initialized.read())
+
+commit ce436a33e0f720ea4b8cf0363bcac1126be3c28b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 29 18:12:50 2014 -0700
+
+    Objecter: init with a constant of the correct type
+    
+    Just a tiny cleanup.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1feba200aae7d9a042cda705c3de8fba2fc82331)
+
+commit 970a79753dc52d82d2abf29ffe6f88adac678eb0
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Wed Aug 20 15:35:44 2014 +0800
+
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Originally in the add_simple_ruleset funtion, the ruleset_id
+    is not reused but rule_id is reused. So after some add/remove
+    against rules, the newly created rule likely to have
+    ruleset!=rule_id.
+    
+    We dont want this happen because we are trying to hold the constraint
+    that ruleset == rule_id.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 78e84f34da83abf5a62ae97bb84ab70774b164a6)
+
+commit 76535116823f02f0392226e5725fbfef14c277ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 20 13:43:46 2015 -0800
+
+    DBObjectMap: lock header_lock on sync()
+    
+    Otherwise, we can race with another thread updating state.seq
+    resulting in the old, smaller value getting persisted.  If there
+    is a crash at that time, we will reuse a sequence number, resulting
+    in an inconsistent node tree and bug #9891.
+    
+    Fixes: 9891
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2b63dd25fc1c73fa42e52e9ea4ab5a45dd9422a0)
+    
+    Conflicts:
+    	src/os/DBObjectMap.cc
+            because we have state.v = 1; instead of state.v = 2;
+
+commit 13b01473c5c1a116d6cd6acbbb6cbe08ee6e9433
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Oct 28 09:37:11 2014 +0800
+
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reported-by: Xinze Chi <xmdxcxz@gmail.com>
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 622c5ac41707069ef8db92cb67c9185acf125d40)
+
+commit 13b8364dc869b2eefcb6646cff1e18c31126ce7d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 19:33:59 2014 -0700
+
+    osd/ReplicatedPG: fix compile error
+    
+    From 1fef4c3d541cba360738437420ebfa2447d5802e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4a9ad7dc2da6f4fa6a64235776a3f1d2799aef60)
+
+commit 55541b90db0a6d6c26c1fae6a4958fa5a320f82c
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Oct 29 07:11:11 2014 +0000
+
+    Get the currently atime of the object in cache pool for eviction
+    
+    Because if there are mutiple atime in agent_state for the same object, we should use the recently one.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 1fef4c3d541cba360738437420ebfa2447d5802e)
+
+commit 5550cdd876db913b152edad17a4de16bded31627
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:42 2014 +0800
+
+    osdc/Objecter: Fix a bug of dead looping in Objecter::handle_osd_map
+    
+    If current map epoch is less than oldest epoch, current map epoch
+    should step up to oldest epoch.
+    
+    Fixes: #9986
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit e0166a23c2cf655bfb4cf873be021a14d9b9be27)
+
+commit 7c5056f5cf77065e0e5a920f525f0a3be6b4b299
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:05 2014 +0800
+
+    osdc/Objecter: e shouldn't be zero in Objecter::handle_osd_map
+    
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit 31c584c8ba022cd44fe2872d221f3026618cefab)
+
+commit 21f81b85de652aede51e88b87bdff71f2e411da3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 19 08:20:16 2014 -0800
+
+    PG: always clear_primary_state on new interval, but only clear pg temp if not primary
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f692bfe076b8ddb679c6d1a6ea78cc47f0876326)
+
+commit 0c3f7637d8cf2146a4268330d8c0506ad38c354d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Nov 14 15:44:20 2014 -0800
+
+    PG: always clear_primary_state when leaving Primary
+    
+    Otherwise, entries from the log collection process might leak into the next
+    epoch, where we might end up choosing a different authoritative log.  In this
+    case, it resulted in us not rolling back to log entries on one of the replicas
+    prior to trying to recover from an affected object due to the peer_missing not
+    being cleared.
+    
+    Fixes: #10059
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c87bde64dfccb5d6ee2877cc74c66fc064b1bcd7)
+
+commit 1ccf58355d0796172935938572cf68ceb31a6801
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 2 15:17:57 2014 -0800
+
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    In the past (e229f8451d37913225c49481b2ce2896ca6788a2) we decided to disable
+    reset of lossless Pipes, because lossless peers resetting caused trouble and
+    they can't forget about each other. But they actually can: if mark_down()
+    is called.
+    
+    I can't figure out how else we could forget about a remote endpoint, so I think
+    it's okay if we tell them we reset in order to clean up state. That's desirable
+    so that we don't get into strange situations with out-of-whack counters.
+    
+    Fixes: #10080
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8cd1fdd7a778eb84cb4d7161f73bc621cc394261)
+
+commit 3e875ab108de8d2aa3717f76a3fe48ede286abb7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 6 09:52:29 2015 -0800
+
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Fixes: 6003
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bae1f3eaa09c4747b8bfc6fb5dc673aa6989b695)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+            because reinterpret_cast was added near two hunks after firefly
+
+commit 96a5c67121dde0d4d4cd13793bb131414b64cc28
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 18:42:40 2015 -0800
+
+    rgw: flush xml header on get acl request
+    
+    Fixes: #10106
+    Backport: firefly, giant
+    
+    dump_start() updates the formatter with the appropriate prefix, however,
+    we never flushed the formatter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb45f861343162e018968b8c56693a8c6f5b2cab)
+
+commit c7b02f5a7347ea8688c5214f85bb9f612925a586
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Mar 19 00:32:39 2015 +0100
+
+    doc,tests: force checkout of submodules
+    
+    When updating submodules, always checkout even if the HEAD is the
+    desired commit hash (update --force) to avoid the following:
+    
+        * a directory gmock exists in hammer
+        * a submodule gmock replaces the directory gmock in master
+        * checkout master + submodule update : gmock/.git is created
+        * checkout hammer : the gmock directory still contains the .git from
+          master because it did not exist at the time and checkout won't
+          remove untracked directories
+        * checkout master + submodule update : git rev-parse HEAD is
+          at the desired commit although the content of the gmock directory
+          is from hammer
+    
+    http://tracker.ceph.com/issues/11157 Fixes: #11157
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d4eb9faf1871c4469b78a7ee75d527ce5cc67ad
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 11 13:05:54 2014 -0800
+
+    ReplicatedPG::scan_range: an object can disappear between the list and the attr get
+    
+    The first item in the range is often last_backfill, upon which writes
+    can be occuring.  It's trimmed off on the primary side anyway.
+    
+    Fixes: 10150
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit dce6f288ad541fe7f0ef8374301cd712dd3bfa39)
+
+commit 499d94f573e78f6545bd5a12ade6b5ba5a75bbca
+Author: Federico Simoncelli <fsimonce@redhat.com>
+Date:   Sat Nov 15 14:14:04 2014 +0000
+
+    common: do not unlock rwlock on destruction
+    
+    According to pthread_rwlock_unlock(3p):
+    
+     Results are undefined if the read-write lock rwlock is not held
+     by the calling thread.
+    
+    and:
+    
+     https://sourceware.org/bugzilla/show_bug.cgi?id=17561
+    
+     Calling pthread_rwlock_unlock on an rwlock which is not locked
+     is undefined.
+    
+    calling pthread_rwlock_unlock on RWLock destruction could cause
+    an unknown behavior for two reasons:
+    
+    - the lock is acquired by another thread (undefined)
+    - the lock is not acquired (undefined)
+    
+    Moreover since glibc-2.20 calling pthread_rwlock_unlock on a
+    rwlock that is not locked results in a SIGILL that kills the
+    application.
+    
+    This patch removes the pthread_rwlock_unlock call on destruction
+    and replaces it with an assertion to check that the RWLock is
+    not in use.
+    
+    Any code that relied on the implicit release is now going to
+    break the assertion, e.g.:
+    
+     {
+       RWLock l;
+       l.get(for_write);
+     } // implicit release, wrong.
+    
+    Signed-off-by: Federico Simoncelli <fsimonce@redhat.com>
+    (cherry picked from commit cf2104d4d991361c53f6e2fea93b69de10cd654b)
+
+commit b79852f3ab0643bbb3f6b71a56b15e5a4b5fc1f5
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Dec 10 17:46:35 2014 +0000
+
+    mon: Paxos: reset accept timeout before submiting work to the store
+    
+    Otherwise we may trigger the timeout while waiting for the work to be
+    committed to the store -- and it would only take the write to take a bit
+    longer than 10 seconds (default accept timeout).
+    
+    We do wait for the work to be properly committed to the store before
+    extending the lease though.
+    
+    Fixes: #10220
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 18534615f184ba56b441fd1d4242eb06debdfe13)
+
+commit e997c9fed5feb3e877dfe07ffac1327b85d09ea2
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Tue Dec 9 17:35:47 2014 +0000
+
+    mon: MonitorDBStore: allow randomly injecting random delays on writes
+    
+    Adds two new config options:
+    
+    mon_inject_transaction_delay_probability : DOUBLE (0.0-1.0, default: 0.0)
+    mon_inject_transaction_delay_max : DOUBLE (seconds, default: 10.0)
+    
+    If probability is set to a value greater than 0, just before applying
+    the transaction, the store will decide whether to inject a delay,
+    randomly choosing a value between 0 and the max.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit beaa04e4119765d5775a6c48fd072dd95c984e3b)
+
+commit 1d4ffbe1d233de555c1ecb0a33eebe2391b29f33
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:41:19 2015 -0800
+
+    ShardedThreadPool: make wait timeout on empty queue configurable
+    
+    Fixes: 10818
+    Backport: giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7002f934e6664daa995ca0629c0ea3bae1c6bddf)
+
+commit 292c4339c064968c2aa05eec701fbc2a8d82dab0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:11:38 2015 -0800
+
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Fixes: 10817
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5aa6f910843e98a05bfcabe6f29d612cf335edbf)
+
+commit fea29b1bcbd17b3d1f642398ec70dbe258bbc98f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 15:15:08 2014 -0800
+
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Fixes: #10157
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1fe8b846641486cc294fe7e1d2450132c38d2dba)
+
+commit 4427358bb556d902b01df27fd097cc3eefa561da
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Feb 11 23:36:01 2015 +0000
+
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    entity_name_t is tightly coupled to the messenger, while EntityName is
+    tied to auth.  When expanding profiles we want to tie the profile
+    expansion to the entity that was authenticated.  Otherwise we may incur
+    in weird behavior such as having caps validation failing because a given
+    client messenger inst does not match the auth entity it used.
+    
+    e.g., running
+    
+    ceph --name osd.0 config-key exists foo daemon-private/osd.X/foo
+    
+    has entity_name_t 'client.12345' and EntityName 'osd.0'.  Using
+    entity_name_t during profile expansion would not allow the client access
+    to daemon-private/osd.X/foo (client.12345 != osd.X).
+    
+    Fixes: #10844
+    Backport: firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 87544f68b88fb3dd17c519de3119a9ad9ab21dfb)
+
+commit 8902279dd4b52516d59db712bd59e8d9372611a6
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Nov 14 21:03:54 2014 +0000
+
+    mon: Monitor: stash auth entity name in session
+    
+    Backport: giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit ca8e1efc0be9bffcfbdce5593526d257aa498062)
+
+commit 681c99fe12cfa6318f8cf06f4e825805635bcc07
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 14:27:39 2014 -0800
+
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Fixes: #8011
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 9b26de3f3653d38dcdfc5b97874089f19d2a59d7)
+
+commit 1f58a0adc3b23785fac00083d721b62f6a4c44a1
+Merge: 9fee7ba fe7d4ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:52:01 2015 -0700
+
+    Merge pull request #4042 from dachary/wip-10546-giant
+    
+    ceph time check start round bug in monitor.cc
+
+commit 9fee7ba9d7343bda29bc5113f8db5cdd9c09d71a
+Merge: ca635ce e7af52a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:50:26 2015 -0700
+
+    Merge pull request #4047 from dachary/wip-10762-giant
+    
+    mon: osd gets marked down twice
+
+commit ca635ce3eb21135b08613b97ce0c56977de44351
+Merge: 9094185 5771f57
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:49:53 2015 -0700
+
+    Merge pull request #4041 from dachary/wip-10512-giant
+    
+    osd: cancel_flush requeues blocked events after blocking event
+
+commit 90941850ceb2c080ac72c67fed991c951b13f449
+Merge: bc1b9d2 5fca232
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:47:26 2015 -0700
+
+    Merge pull request #4031 from dachary/wip-10353-giant
+    
+    crush: set_choose_tries = 100 for erasure code rulesets
+
+commit bc1b9d22d29a74e6853834fb0430e8803710bbbf
+Merge: bd69cfc 30a1744
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:47:08 2015 -0700
+
+    Merge pull request #4029 from dachary/wip-9910-giant
+    
+    msg/Pipe: discard delay queue before incoming queue
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit bd69cfcbd58103843fbc42c4e55d43c2bafe54ff
+Merge: 30f7df2 4bd5237
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:44:53 2015 -0700
+
+    Merge pull request #4030 from dachary/wip-10351-giant
+    
+    mount.ceph: avoid spurious error message
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 30f7df2b110f73477354fbfa607b4777317dddfa
+Merge: e5a50c3 8307318
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:44:00 2015 -0700
+
+    Merge pull request #4028 from dachary/wip-10259-giant
+    
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::o...
+
+commit e5a50c3ae42c8b9d16a19a13e10fc35b91149e73
+Merge: ba1d55d a5a76f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:42:10 2015 -0700
+
+    Merge pull request #4027 from dachary/wip-10257-giant
+    
+      mon: PGMonitor: several stats output error fixes
+
+commit ba1d55de1e92a0143702e9e3fbbb1820db17d6c3
+Merge: 530fce4 640986f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:41:56 2015 -0700
+
+    Merge pull request #3998 from dzafman/wip-10677
+    
+    Fix ceph command manpage to match ceph -h (giant)
+    
+    Reviewed-by: Xinxin Shu <xinxin.shu@intel.com>
+
+commit 530fce4d7cf51f4a4ac1e70ebe20b2795a30010e
+Merge: a2a3dad 8a5a3c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:40:02 2015 -0700
+
+    Merge pull request #3921 from sponce/wip-11078-giant
+    
+    Fix libstriprados::stat, use strtoll insdead of strtol
+
+commit a2a3dad8ec00434b99d6d538615bf8ff9eada217
+Merge: 0f57188 984df3b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:36:28 2015 -0700
+
+    Merge pull request #3819 from tchaikov/giant-pg-leak-10421
+    
+    osd: fix PG leak in SnapTrimWQ._clear()
+
+commit 0f5718822793074ffab45124afb3bf3160e92004
+Merge: 776c82c c5c6fcd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:35:37 2015 -0700
+
+    Merge pull request #3771 from ceph/wip-10883-giant
+    
+    osd: Fix FileJournal wrap to get header out first
+
+commit 776c82caac36de14a5f895688f81c88a883fca3e
+Merge: 938e036 37f196e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:35:29 2015 -0700
+
+    Merge pull request #3637 from sponce/wip-10758-giant
+    
+    Backport of pull request 3633 to giant : Fixed write_full behavior in libradosstriper
+
+commit 92c352d047c84035478e71828e32f554d9f507fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 12 13:49:50 2015 -0800
+
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    If the epoch cache says the osd has epoch 100 and the osd is asking for
+    epoch 200+, do not send it 100+.
+    
+    Fixes: #10787
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a5759e9b97107488a8508f36adf9ca1aba3fae07)
+
+commit 25fcaca19c76765284787c61584d25efd105a0ca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Feb 5 09:33:26 2015 -0800
+
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Fixes: #10770
+    Backport: firefly, giant
+    
+    Reported-by: baijiaruo <baijiaruo@126.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dfee96e3aebcaeef18c721ab73f0460eba69f1c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+            resolved by manual s/ADD/CANCEL/
+
+commit e7af52a147cc6e1a578d51193a2d5a425cee8a20
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 5 03:07:50 2015 -0800
+
+    mon: ignore osd failures from before up_from
+    
+    If the failure was generated for an instance of the OSD prior to when
+    it came up, ignore it.
+    
+    This probably causes a fair bit of unnecessary flapping in the wild...
+    
+    Backport: giant, firefly
+    Fixes: #10762
+    Reported-by: Dan van der Ster <dan@vanderster.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 400ac237d35d0d1d53f240fea87e8483c0e2a7f5)
+
+commit 28583276254ba6121b93688a6c6d8fab6588c864
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 9 20:50:23 2015 -0800
+
+    rados.py: keep reference to python callbacks
+    
+    If we don't keep a reference to these, the librados aio calls will
+    segfault since the python-level callbacks will have been garbage
+    collected. Passing them to aio_create_completion() does not take a
+    reference to them. Keep a reference in the python Completion object
+    associated with the request, since they need the same lifetime.
+    
+    This fixes a regression from 60b019f69aa0e39d276c669698c92fc890599f50.
+    
+    Fixes: #10775
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 36d37aadbbbece28d70e827511f1a473d851463d)
+    (cherry picked from commit 5f1245e131e33a98572408c8223deed2c7cf7b75)
+
+commit 1b5fb51a995d5d672a46176240d5887354c12967
+Author: Billy Olsen <billy.olsen@gmail.com>
+Date:   Mon Feb 2 16:24:59 2015 -0700
+
+    Fix memory leak in python rados bindings
+    
+    A circular reference was inadvertently created when using the
+    CFUNCTYPE binding for callbacks for the asynchronous i/o callbacks.
+    This commit refactors the usage of the callbacks such that the
+    Ioctx object does not have a class reference to the callbacks.
+    
+    Fixes: #10723
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Billy Olsen <billy.olsen@gmail.com>
+    Reviewed-by: Dan Mick <dmick@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 60b019f69aa0e39d276c669698c92fc890599f50)
+
+commit d5b1b7e52242e082820e0cdd76dc59e9aea7f284
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 23 10:47:44 2015 -0800
+
+    osd: do not ignore deleted pgs on startup
+    
+    These need to get instantiated so that we can complete the removal process.
+    
+    Fixes: #10617
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 879fd0c192f5d3c6afd36c2df359806ea95827b8)
+
+commit fe7d4ca17dacf4419be7bdc50010115cee36517b
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 30 11:37:28 2015 +0000
+
+    mon: Monitor: fix timecheck rounds period
+    
+    Fixes: #10546
+    Backports: dumpling?,firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2e749599ac6e1060cf553b521761a93fafbf65bb)
+
+commit 5771f57af731e61bf70f630cf85f5b94dcfd1edb
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 11 17:28:04 2015 -0800
+
+    osd: requeue blocked op before flush it was blocked on
+    
+    If we have request A (say, cache-flush) that blocks things, and then
+    request B that gets blocked on it, and we have an interval change, then we
+    need to requeue B first, then A, so that the resulting queue will keep
+    A before B and preserve the order.
+    
+    This was observed on this firefly run:
+    
+      ubuntu@teuthology:/a/sage-2015-01-09_21:43:43-rados-firefly-distro-basic-multi/694675
+    
+    Backport: giant, firefly
+    Fixes: #10512
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11bdfb4131ecac16d4a364d651c6cf5d1d28c702)
+
+commit fa8d45423e22f7b916a345024a9f70283fc3097d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jan 7 13:56:14 2015 -0800
+
+    rgw: index swift keys appropriately
+    
+    Fixes: #10471
+    Backport: firefly, giant
+    
+    We need to index the swift keys by the full uid:subuser when decoding
+    the json representation, to keep it in line with how we store it when
+    creating it through other mechanism.
+    
+    Reported-by: hemant burman <hemant.burman@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 478629bd2f3f32afbe6e93eaebb8a8fa01af356f)
+
+commit 5fca232080c0bde50b5c65c60c133bf81311e8d3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 17 16:06:55 2014 +0100
+
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    It is common for people to try to map 9 OSDs out of a 9 OSDs total ceph
+    cluster. The default tries (50) will frequently lead to bad mappings for
+    this use case. Changing it to 100 makes no significant CPU performance
+    difference, as tested manually by running crushtool on one million
+    mappings.
+    
+    http://tracker.ceph.com/issues/10353 Fixes: #10353
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 2f87ac807f3cc7ac55d9677d2051645bf5396a62)
+
+commit 4bd523744cefb4a5e6b0d4440b9829b0e19dc012
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jan 3 15:29:29 2015 +0800
+
+    mount.ceph: avoid spurious error message
+    
+    /etc/mtab in most modern distributions is a symbol link to
+    /proc/self/mounts.
+    
+    Fixes: #10351
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bdd0e3c4bda97fe18487a58dd173a7dff752e1a2)
+
+commit 30a17441f5030ac5c1227a9672811f88e580b4b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 29 14:45:11 2014 -0700
+
+    msg/Pipe: discard delay queue before incoming queue
+    
+    Shutdown the delayed delivery before the incoming queue in case the
+    DelayedDelivery thread is busy queuing messages.
+    
+    Fixes: #9910
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit f7431cc3c25878057482007beb874c9d4473883e)
+
+commit 8307318cdf9487dde3231aa4a00fca6079379738
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Dec 5 15:29:52 2014 -0800
+
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::operator==
+    
+    Fixes: 10259
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1ac17c0a662e6079c2c57edde2b4dc947f547f57)
+
+commit a5a76f81ae9242a54d6ef150fa0ac32b31e90c25
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Mon Jan 19 18:49:15 2015 +0000
+
+    mon: PGMonitor: skip zeroed osd stats on get_rule_avail()
+    
+    Fixes: #10257
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit b311e7c36273efae39aa2602c1f8bd90d39e5975)
+
+commit 95ec68505ce4a26ea707716791cfb1e46a75ed25
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:13:05 2015 +0000
+
+    mon: PGMonitor: available size 0 if no osds on pool's ruleset
+    
+    get_rule_avail() may return < 0, which we were using blindly assuming it
+    would always return an unsigned value.  We would end up with weird
+    values if the ruleset had no osds.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 8be6a6ab2aa5a000a39c73a98b11a0ab32fffa1c)
+
+commit 1c480b39acc184018d976beddc55187f86dfff1a
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:12:42 2015 +0000
+
+    mon: PGMonitor: fix division by zero on stats dump
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 50547dc3c00b7556e26b9a44ec68640c5c3a2384)
+
+commit 640986fdef3543c7469e43853900b63d7b2f6f36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 19:16:47 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (giant)
+    
+    Fixes: #10677
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit ec4c8997a67397e3ba5f335e444586f6f1a7864c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 16:50:13 2015 -0700
+
+    doc: Minor fixes to ceph command manpage
+    
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7e85722fd4c89715fc2ed79697c82d65d7ebf287)
+
+commit 15e1c6fb3a0e2e191025f96df33feaf115c1411d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 12 11:39:52 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (firefly)
+    
+    Improve synopsis section
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9ac488c1eb0e30511079ba05aaf11c79615b3940)
+    
+    Conflicts:
+    
+    	man/ceph.8 (took incoming version)
+
+commit 17d03e8e8b055ed50b1abb679a0dd1edf4ec3f30
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:11:22 2014 +0530
+
+    doc: Changes format style in ceph to improve readability as html.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8b796173063ac9af8c21364521fc5ee23d901196)
+
+commit 61d600687ae6887a2edea0e79d582b1353558a83
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Mar 10 19:55:57 2015 +0800
+
+    mds: fix assertion caused by system clock backwards
+    
+    Fixes: #11053
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit f1c9c71fd6d566687db05e1d8f15b04a9f793005
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:24:41 2014 +0530
+
+    doc: Adds man page for ceph.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 76da87a64ca6b3cc0ceeaf63e19a9f440d6f4161)
+
+commit 8a5a3c7a20517c7482dc9f5f2783a3da54bd4f23
+Author: Dongmao Zhang <deanraccoon@gmail.com>
+Date:   Fri Nov 14 18:48:58 2014 +0800
+
+    Fix libstriprados::stat, use strtoll insdead of strtol
+    
+    The return value(long int) of strict_strtol is too small for unstriped
+    object.
+    
+    Signed-off-by: Dongmao Zhang <deanraccoon@gmail.com>
+    (cherry picked from commit fe6679dca479fc24806d7e57ab0108a516cd6d55)
+
+commit 5d423a5188c62650eaa39077eb99a84085f5f3e2
+Author: Dongmao Zhang <deanraccoon@gmail.com>
+Date:   Wed Dec 10 18:55:28 2014 +0800
+
+    Fix libstriprados::remove, use strtoll insdead of strtol
+    
+    Signed-off-by: Dongmao Zhang <deanraccoon@gmail.com>
+    (cherry picked from commit 78a15ee4c61fdadccb1921e861748400cc651862)
+
+commit 938e03630e075af03780da139ae879b5b0377734
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 2 13:57:00 2015 -0800
+
+    Objecter::_op_submit_with_budget: add timeout before call
+    
+    Objecter::_send_op depends on the ontimeout field being filled in
+    to avoid 10340 and 9582.
+    
+    Fixes: 10340
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit cfcfafcb0f33994dbda1efe478ef3ab822ff50d4)
+
+commit 984df3b865f295ecf77b041a69d1d59384d80671
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 10 16:29:45 2015 +0800
+
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Fixes: #10421
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 01e154d592d6cdbf3f859cf1b4357e803536a6b4)
+
+commit c5c6fcd6ad7a0317301970bc25419727ee3192fa
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 16:21:12 2015 -0800
+
+    osd: Fix FileJournal wrap to get header out first
+    
+    Correct and restore assert that was removed
+    
+    Cause by f46b1b473fce0322a672b16c7739e569a45054b6
+    Fixes: #10883
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 970bb4901f93575709421b5b25c3eff213de61b8)
+
+commit 37f196e80906d6b855ca0a7ce7344d1f8f50ae7f
+Author: Sebastien Ponce <Sebastien.Ponce@cern.ch>
+Date:   Thu Feb 5 10:46:37 2015 +0100
+
+    libradosstriper: fixed write_full when ENOENT
+    
+    Fixes: #10758
+    write_full was returning ENOENT when the file did not exists, while it should just have created it without complaining.
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 6a91f2bb82035b3c8021a7dc7b23548ce3d61eb6)
+
+commit b318e2ffc3faedf6b02908429eb4ce79147d3471 (refs/remotes/gh/wip-10643)
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 18:01:05 2014 +0100
+
+    mon: MDSMonitor: don't return -EINVAL if function is bool
+    
+    Returning -EINVAL on a function that expects bool and the error code to
+    be in a variable 'r' can only achieve one thing: if this path is ever
+    touched, instead of returning an error as it was supposed to, we're
+    returning 'true' with 'r = 0' and, for no apparent reason, the user will
+    think everything went smoothly but with no new fs created.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 91b2a2b69b198567d42f45f75b7f7ea559f89150)
+
+commit a1a7d856f5409280b6345afbbd961bcd8ee8d046
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 17:43:33 2014 +0100
+
+    mon: MDSMonitor: check all conditions are met *before* osdmon proposal
+    
+    We should not allow ourselves to request the osdmon to propose before we
+    know for sure that we meet the required conditions to go through with
+    our own state change.  Even if we still can't guarantee that our
+    proposal is going to be committed, we shouldn't just change the osdmon's
+    state just because we can.  This way, at least, we make sure that our
+    checks hold up before doing anything with side-effects.
+    
+    Fixes: #10643
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 1d1ae413e9f0aafa314af903bcf6b815f6558082)
+
+commit 0df7cf1e58bc4d77c69354f2a7160e3d9e6d1fc1
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 17:37:03 2014 +0100
+
+    mon: MDSMonitor: return if fs exists on 'fs new'
+    
+    We were just setting return code to -EINVAL, while allowing the logic to
+    continue regardless.  If we are to return error, then we should abort
+    the operation as well and let the user know it went wrong instead of
+    continuing as if nothing had happened.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 07b7f101057a80d1356bd20040180e94cd20c211)
diff --git a/doc/changelog/v0.94.1.txt b/doc/changelog/v0.94.1.txt
new file mode 100644
index 000000000..227267ddb
--- /dev/null
+++ b/doc/changelog/v0.94.1.txt
@@ -0,0 +1,74 @@
+commit e4bfad3a3c51054df7e537a724c8d0bf9be972ff (tag: refs/tags/v0.94.1, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Apr 10 12:04:50 2015 -0700
+
+    0.94.1
+
+commit 733864738fa93979727e480e403293e079bb51e9
+Merge: b5921d5 5ca771a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Apr 10 17:51:38 2015 +0200
+
+    Merge pull request #4328 from dachary/wip-11364-hammer
+    
+    v4 bucket feature detection
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ca771a7d1df8e78ee503a7063068cf744d5efcc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 10 08:43:45 2015 -0700
+
+    crush: fix has_v4_buckets()
+    
+    alg, not type!
+    
+    This bug made us incorrectly think we were using v4 features when user type
+    5 was being used.  That's currently 'rack' with recent crush maps, but
+    was other types for clusters that were created with older versions.  This
+    is clearly problematic as it will lock out non-hammer clients incorrectly,
+    breaking deployments on upgrade.
+    
+    Fixes: #11364
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 38b35ab9d17eb84ac178c4cd3ebcf2ec0f66d8b6)
+
+commit 33e79ab7aa0b5428e8fb82a90eea17d31d363a88
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 17:17:59 2015 -0700
+
+    crush: fix dump of has_v4_buckets
+    
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d6e23413017fb8f5d7f18d74e993ceeedb82d8bc)
+
+commit b5921d55d16796e12d66ad2c4add7305f9ce2353
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 14:42:34 2015 -0700
+
+    crush/mapper: fix divide-by-0 in straw2
+    
+    If the item weight is 0 we don't want to divide; instead draw a minimal
+    value.
+    
+    Fixes: #11357
+    Reported-by: Yann Dupont <yd@objoo.org>
+    Tested-by: Yann Dupont <yd@objoo.org>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 64d1e900ec4f5831972ec80e8d0129604669f5bb)
+
+commit 93c8f436a4f84ac8088e1a1de82350dd33c68d64
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Thu Feb 26 08:13:12 2015 +0000
+
+    osd: fix negative degraded objects during backfilling
+    
+    When there is deleting requests during backfilling, the reported number of degraded
+    objects could be negative, as the primary's num_objects is the latest (locally) but
+    the number for replicas might not reflect the deletings. A simple fix is to ignore
+    the negative subtracted value.
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 14d7e36d3c978844da73d0e1c8a3a1ec863bac15)
diff --git a/doc/changelog/v0.94.10.txt b/doc/changelog/v0.94.10.txt
new file mode 100644
index 000000000..342e6a676
--- /dev/null
+++ b/doc/changelog/v0.94.10.txt
@@ -0,0 +1,2728 @@
+commit b1e0532418e4631af01acbc0cedd426f1905f4af
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Feb 21 15:12:34 2017 +0000
+
+    0.94.10
+
+commit 83af8cdaaa6d94404e6146b68e532a784e3cc99c
+Merge: 5855083 5485b49
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 23 17:49:54 2017 +0100
+
+    Merge pull request #13041 from SUSE/wip-18448-hammer
+
+    hammer: osd: filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 585508358f6412b179175a7812b76330ec5dda0d
+Merge: 4e4d3f0 eabe906
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 23 12:26:27 2017 +0100
+
+    Merge pull request #13022 from SUSE/wip-18602-hammer
+
+    hammer: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4e4d3f0fd86840609940f6b336f6aaf54a8695bc
+Merge: 9036fee cf9678c
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 23 12:25:43 2017 +0100
+
+    Merge pull request #13039 from SUSE/wip-fs-thrash-xfs
+
+    tests: run fs/thrash on xfs instead of btrfs
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit eabe9064aed3693599e3b6053b57eafb835d35da
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+
+commit 5485b49da39b9f07a4216ff26f8ec9e04590ddf8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:54:52 2017 +0100
+
+    os/filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE
+
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit 7bd95b595fddb8a4e618a2c7df1ba04eccf0829d)
+
+    Conflicts:
+	src/os/FileStore.cc (in master, this file is in a different location)
+
+commit cf9678c998600e76e94777b2904ba3047d051bb8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:24:25 2017 +0100
+
+    tests: run fs/thrash on xfs instead of btrfs
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9036feed9bd9f386dc013dee8326071ef0b4cbee
+Merge: 33c7ab7 fa5c625
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jan 21 10:00:04 2017 +0100
+
+    Merge pull request #12824 from SUSE/wip-18449-hammer
+
+    hammer: [teuthology] update rbd/singleton/all/formatted-output.yaml to support ceph-ci
+
+commit 33c7ab7480aa8d0ee80d6a47f91f4488515e07f8
+Merge: ef8ae1f ef513a7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 23:10:10 2017 +0100
+
+    Merge pull request #12906 from SUSE/wip-15943-hammer
+
+    hammer: osd/PG: fix cached_removed_snaps bug in PGPool::update after map gap
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ef8ae1ff13b297c68ed4f1d0e01aba8d20dc5007
+Merge: c51602f 8bab745
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 23:02:05 2017 +0100
+
+    Merge pull request #12805 from SUSE/wip-18432-hammer
+
+    hammer: ceph-create-keys loops forever
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c51602f0106e3681c143a2cfd94a08428e966caf
+Merge: 48687c7 83e7f17
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 08:39:14 2017 -0500
+
+    Merge pull request #12957 from SUSE/wip-18544-hammer
+
+    hammer: [teuthology] update Ubuntu image url after ceph.com refactor
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 83e7f178c9a0b70305644ca30598c1b1ff4d0474
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 16 22:12:51 2017 -0500
+
+    qa/tasks/qemu: update default image url after ceph.com redesign
+
+    Fixes: http://tracker.ceph.com/issues/18542
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d17befb3bbc3d83c9d23d763ad95e1e7b2e4be0)
+
+commit ef513a78062569de081311b50fffdf59d4031ea7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:35:38 2016 -0800
+
+    PG: fix cached_removed_snaps bug in PGPool::update after map gap
+
+    5798fb3bf6d726d14a9c5cb99dc5902eba5b878a actually made 15943 worse
+    by always creating an out-of-date cached_removed_snaps value after
+    a map gap rather than only in the case where the the first map after
+    the gap did not remove any snapshots.
+
+    Introduced: 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5642e7e1b3bb6ffceddacd2f4030eb13a17fcccc)
+
+commit fa5c6250125a51e61e34aa9586cc917df7458138
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:48:51 2017 +0100
+
+    tests: subst repo and branch in qemu test urls
+
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2484a5546160560a4050d35400a6edce37ae48ad)
+
+    Conflicts: (hammer doesn't have these files)
+	qa/suites/rbd/maintenance/qemu/xfstests.yaml
+	qa/suites/rbd/qemu/workloads/#qemu_xfstests.yaml#
+
+commit 572cd2660f5bf1cc7f980831d12bb3075816b96f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:45:10 2017 +0100
+
+    tests: subst branch and repo in qa/tasks/qemu.py
+
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 74689df754561e11a3377998840efcea9e780755)
+
+commit 237fb80f2b2874329aed25ba655a6789e1707114
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:06:11 2017 +0100
+
+    tests: subst repo name in qa/tasks/cram.py
+
+    Inspired by bcbe45d948f1c4da02e27b3be5f29a8b52745e28
+
+    Fixes: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 56e37e41f4dddd289dd3c1886b192cd328ed311b)
+
+commit 6045a83a438918f445504224b80809dfe715c9b0
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Fri Dec 9 15:11:49 2016 +0530
+
+    cram: support fetching from sha1 branch, tag, commit hash
+
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit d2f0d745987a2e2eee4e1822146aad8da5d42708)
+
+    Conflicts:
+	suites/krbd/unmap/tasks/unmap.yaml (hammer doesn't have this)
+	suites/rbd/singleton/all/formatted-output.yaml (moved to qa/,
+		trivial resolution)
+
+commit 48687c79adf489b27af15bb44205913acc89af7a
+Merge: 61917cb ac62d79
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 8 09:26:03 2017 +0200
+
+    Merge pull request #12227 from ceph/wip-orit-resharding-hammer
+
+    Wip orit resharding hammer
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8bab74537f5891f2b97f0fb6722e2e00416b723a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:28:49 2016 -0400
+
+    ceph-create-keys: wait 10 minutes to get or create the bootstrap key, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bea802bd13df883d06b7b67a83ac6540e44ab6c1)
+
+commit a7d919100e9e78b7f821da76b338bca24f374a0a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:25:32 2016 -0400
+
+    ceph-create-keys: wait 10 minutes to get or create a key, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8f84681a4d97a7ea6bc04e759eccfc40204a2fdb)
+
+    Conflicts:
+	src/ceph-create-keys ( hammer does not have
+		52e978e4b3660baa9f50b1bb8247909b672142e7 )
+
+commit 7b3bc5fbc55da3cb2ce036ef3df367832d7a59e2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:19:10 2016 -0400
+
+    ceph-create-keys: wait for quorum for ten minutes, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 32cedd2c18e48d5f9a1547e5fd69c24c7eac1638)
+
+commit 61917cbcb7a4e35aa0ae8718377526ae55b5b7cf
+Merge: 7b1f8fe 7106f42
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:15:13 2017 -0800
+
+    Merge pull request #11615 from mslovy/wip-17610-hammer
+
+    hammer: osd: fix fiemap issue in xfs when #extents > 1364
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7b1f8fe4c97b6c8ef81d899f0a24235946cc585a
+Merge: 5ba3c08 5f00e1d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:14:10 2017 -0800
+
+    Merge pull request #11936 from SUSE/wip-16432-hammer
+
+    hammer: librados: bad flags can crash the osd
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 5ba3c0873d10b247a45bfbc1cb483cfc2189a9fd
+Merge: b44e1db 8312dc6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:12:55 2017 -0800
+
+    Merge pull request #12121 from dzafman/wip-16225-min
+
+    hammer: common: SIGABRT in TrackedOp::dump() via dump_ops_in_flight()
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit b44e1db11b0d7310f2563fa7acfb1464db4b04fb
+Merge: 7bfb05b 92de2fd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:57 2017 -0800
+
+    Merge pull request #12266 from SUSE/wip-18109-hammer
+
+    hammer: msgr: simple/Pipe: error decoding addr
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7bfb05b00157fa40c053eeefb59cd3bb211fc2b3
+Merge: 715012b fe77c9a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:34 2017 -0800
+
+    Merge pull request #12417 from SUSE/wip-18132-hammer
+
+    hammer: osd: ReplicatedBackend::build_push_op: add a second config to limit omap entries/chunk independently of object data
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 715012bebe6e7838c702aae0283f1c3d37428762
+Merge: a20ad42 e9311dd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:13 2017 -0800
+
+    Merge pull request #12759 from SUSE/wip-18399-hammer
+
+    hammer: tests: objecter_requests workunit fails on wip branches
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a20ad42ecab39541107e01ac4516a0a71e379846
+Merge: 3e62a1d 67e2101
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:10:54 2017 -0800
+
+    Merge pull request #12762 from SUSE/wip-18405-hammer
+
+    hammer: Cannot reserve CentOS 7.2 smithi machines
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3e62a1d5dbd2ee6672388121be26afd5d0374f5f
+Merge: 77fa49f aa91bc7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:10:28 2017 -0800
+
+    Merge pull request #12685 from SUSE/wip-18273-hammer
+
+    hammer: tests: Workunits needlessly wget from git.ceph.com
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 77fa49f67d1c8c59d9131e6f52e595661ceda8f2
+Merge: 2c164b7 c28d5ea
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:09:59 2017 -0800
+
+    Merge pull request #12687 from SUSE/wip-18281-hammer
+
+    hammer: mon: osd flag health message is misleading
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2c164b73af9de4196821d9214fb7598ac88ef509
+Merge: dfefeda 996fc25
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jan 5 15:20:28 2017 +0100
+
+    Merge pull request #12423 from SUSE/wip-18222-hammer
+
+    hammer: tests: qemu/tests/qemu-iotests/077 fails in dumpling, hammer, and jewel
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dfefeda86e3e6761bc4a59cee960d807609344e4
+Merge: b0648f8 0c5ebf4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 20:15:21 2017 +0100
+
+    Merge pull request #12744 from SUSE/wip-18385-hammer
+
+    hammer: tests: Cannot clone ceph/s3-tests.git (missing branch)
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit b0648f85ee67da82ac8fb7e8f4c1546aa560bfda
+Merge: c45896a e97ceb1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 11:43:29 2017 +0100
+
+    Merge pull request #12418 from SUSE/wip-18218-hammer
+
+    hammer: rgw sends omap_getvals with (u64)-1 limit
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c45896a86810f02b86b3c355855eb70a998a3c9a
+Merge: 811213e db2d112
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 11:37:29 2017 +0100
+
+    Merge pull request #12619 from Werkov/wip-18317-hammer
+
+    hammer: rgw: TempURL in radosgw behaves now like its Swift's counterpart.
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit db2d112f22a525bfffb62e85747769a4b30f7375
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 4 16:30:50 2016 +0100
+
+    rgw: TempURL in radosgw behaves now like its Swift's counterpart.
+
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit a1ea7180969dcbaf88dfcc1bfafb17e82639df64)
+
+    Conflicts:
+	src/rgw/rgw_swift.cc
+	  - replace C++11 constructs,
+	    - s->info.method is actually unconditional member of
+	      allowed_methods, which was opaqued by C++11 ctors
+	  - drop temp_url_make_content_disp() which has no use with
+	    hammer's req_state
+
+commit 811213ef9df188c8cf21dec5aeab08093af483d7
+Merge: 9a8aaca 47e7202
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:31:27 2017 +0100
+
+    Merge pull request #12312 from YankunLi/fix_parameter_check
+
+    hammer: rbd: fix parameter check
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9a8aaca56e48aae78dff27a969e2398c332f2b09
+Merge: 78e3451 c4f1b42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:31:03 2017 +0100
+
+    Merge pull request #12446 from dillaman/wip-18111
+
+    hammer: librbd: diffs to clone's first snapshot should include parent diffs
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 78e34519f9e3c72aa91587593dcb23dac4ade3a3
+Merge: a9de35a 0f83bb7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:23:07 2017 +0100
+
+    Merge pull request #12398 from linuxbox2/hammer
+
+    hammer: rgw: do not abort when accept a CORS request with short origin
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a9de35a27cb7d57f8094de78f550291b1b45b944
+Merge: 3ef8d8c a602e2b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 09:17:35 2017 -0500
+
+    Merge pull request #12758 from SUSE/wip-18397-hammer
+
+    hammer: tests: OSDs commit suicide in rbd suite when testing on btrfs
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 67e2101ffa2b4defe07dec30b12cc37c9f2ed488
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 3 14:53:27 2017 +0100
+
+    qa/distros: add centos yaml; use that instead
+
+    Modeled after https://github.com/ceph/ceph/pull/12632
+
+    Fixes: http://tracker.ceph.com/issues/18405
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit e9311ddb60638c1dd39325e279b97c9a7d09b8a4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:35:02 2016 -0500
+
+    qa/tasks/admin_socket: subst in repo name
+
+    It is either ceph.git or ceph-ci.git.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bcbe45d948f1c4da02e27b3be5f29a8b52745e28)
+
+commit a602e2bf2e1cae8d21ae215c35a611cc99b6c7c9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 3 11:26:26 2017 +0100
+
+    rbd: only test on xfs
+
+    btrfs is not recommended for production
+
+    See 5864d56592ef3ffe765aa1ef0865610dbc7e8725 for precendent.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit aa91bc7df963728e6a4d4db1086214e7873fc2e7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 22:49:13 2017 +0100
+
+    tests: rbd/test_lock_fence.sh: fix rbdrw.py relative path
+
+    This commit fixes a regression introduced by
+    cf294777ea92f0911813a7132068584d4f73a65a
+
+    Fixes: http://tracker.ceph.com/issues/18388
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 91231de16dbe4d0e493ec617165a2b38078d122b)
+
+commit 0c5ebf4adf26274802ec4cff229b2272dc4a4266
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 4 10:03:08 2016 +0100
+
+    use ceph-master branch for s3tests
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 21160124028ab123b5c547499d911b0a1397214f)
+
+    Conflicts: (trivial resolution)
+	    qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml
+	qa/suites/rgw/verify/tasks/rgw_s3tests.yaml
+	qa/suites/rgw/verify/tasks/rgw_s3tests_multiregion.yaml
+
+commit 3ef8d8c6f0ef712dcff207fb6ef17887bd8719d7
+Merge: e038662 d18240f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 2 20:34:42 2017 +0100
+
+    Merge pull request #12743 from SUSE/wip-18383
+
+    hammer: tests: populate mnt_point in qa/tasks/ceph.py
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d18240f679840bdaeec1e3f7cdd9164b4406673c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 14:22:43 2017 +0100
+
+    qa/tasks/ceph.py: populate mnt_point in hammer
+
+    In hammer, there is no "mnt_point" variable, yet 53225d5 was cherry-picked to
+    hammer - see 87e0299448908775600c62130617bad503027768 and https://github.com/ceph/ceph/pull/12455
+
+    This caused lots of "NameError: global name 'mnt_point' is not defined" errors
+    in hammer integration testing.
+
+    This is not a cherry-pick, because the mnt_point variable was introduced by
+    a54ff597faa379c8fd369352fb924651be269f95 ("tasks/ceph: convert cluster creation
+    to work with multiple clusters") which was not backported to hammer.
+
+    Fixes: http://tracker.ceph.com/issues/18383
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ac62d7996b95d8fdd5665e490fc09506c8fe9e68
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Oct 31 13:59:55 2016 +0800
+
+    rgw: fix osd crashes when execute "radosgw-admin bi list --max-entries=1" command
+
+    Fixes: http://tracker.ceph.com/issues/17745
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 51a4405a1ef59671cbd51c530a84333359d3f3dd)
+
+commit 530d1e22920bd843d34ceffccad346fd5003c075
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 29 13:36:37 2016 +0100
+
+    rgw: use hammer rgw_obj_key api
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e396b37af147770434147c5bd0377e8624254cd9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 29 13:15:05 2016 +0100
+
+    Revert "rgw: rgw_obj encoding fixes"
+
+    This reverts commit c1b64c8f9c23473b758300958a94a172f1d39b64.
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a3e9865058e12fff18ee7674fcfd8435a1a37bf4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+
+    The same can happen if a previous workunit task already
+    checked out the repo.
+
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit a70e0d58e10184b0903ba36c047a237be3c7cc19
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit c28d5ea7107427175ef3f6089aaa36c97c2919bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 09:50:26 2016 -0500
+
+    mon/OSDMonitor: only show interesting flags in health warning
+
+    Also add PAUSEREC flag into the list.
+
+    Fixes: http://tracker.ceph.com/issues/18175
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26220f0608f5ed4a7d97fb8d10d0d12a0fcf5583)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (hammer lacks CEPH_OSDMAP_FULL and order of flags
+				   is slightly different)
+
+commit d1c654ea3f3b8c194f3fdc7295cb379c6270d6b3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 19 14:08:11 2016 -0500
+
+    qa/workunits: include extension for nose tests
+
+    When you have a relative path you have to include the extension.
+    Weird.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5666fd61d6dbd40be1d79354227cabd562e829ea)
+
+    Conflicts:
+	qa/workunits/fs/test_python.sh (not in hammer)
+	qa/workunits/rbd/test_librbd_python.sh (no valgrind in hammer)
+
+commit eaa51f6f907019727eb18fc4174fe4f2d04f396a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 15:10:28 2016 -0500
+
+    qa/workunits: use relative path instead of wget from git
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf294777ea92f0911813a7132068584d4f73a65a)
+
+    Conflicts: (all trivial)
+	qa/workunits/fs/test_python.sh
+	qa/workunits/rados/test_python.sh
+	qa/workunits/rbd/rbd_mirror.sh
+	qa/workunits/rbd/rbd_mirror_stress.sh
+	qa/workunits/rbd/test_librbd_python.sh
+	qa/workunits/rbd/test_lock_fence.sh
+
+commit 8eb4e8b6fd6c803fa9bea781db07c54d901e34cc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+
+    Root of git checkout
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit dccf904241f2a00ba0317135bd3cbe56c6dfdb80
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit e038662d80ec1c59edb3d6a1c63197ca387d26b2
+Merge: 12d6781 59878b3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:40:08 2016 -0600
+
+    Merge pull request #12455 from liewegas/qa-suite-hammer
+
+    hammer: merge ceph-qa-suite
+
+commit 59878b3bf775fabf0f4c168e67d6f473e69c15a1
+Merge: 12d6781 7ed0dce
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:30:00 2016 -0600
+
+    merge ceph-qa-suite
+
+commit 7ed0dce3cf7ded278e59a9b36d3fd713e3c42b8d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:55 2016 -0600
+
+    move ceph-qa-suite dirs into qa/
+
+commit 08ce5441dbad8754205717a20d6273c0a099777a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:27:47 2016 -0500
+
+    Revert "tasks/workunit.py: depth 1 clone"
+
+    This reverts commit 12a707cb0f7b162e15c31915eb677bbd49b5e2cf.
+
+commit 12a707cb0f7b162e15c31915eb677bbd49b5e2cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:19:44 2016 -0500
+
+    tasks/workunit.py: depth 1 clone
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4faf77a649cb3f8ddf497ca81937b3dbf63a18dc)
+
+commit 6bdc3bb6a00480b721b9795bd5e132b1dbf04ac0
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:18:29 2016 -0500
+
+    tasks/workunit: remove kludge to use git.ceph.com
+
+    This was hard-coded to ceph.git (almost) and breaks when
+    you specify --ceph-repo.  Remove it entirely.  We'll see if
+    github.com is better at handling our load than it used to
+    be!
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 159c455a0326eef2c017b3e3cf510f918b5ec76c)
+
+commit 87e0299448908775600c62130617bad503027768
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 10 02:36:52 2016 +0800
+
+    tasks/ceph: restore context of osd mount path before mkfs
+
+    all newly created files and directories under the mount dir inherit the
+    SELinux type of their parent directory. so we need to set it before
+    mkfs.
+
+    Fixes: http://tracker.ceph.com/issues/16800
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53225d5272a1d35d4183fcfa55a139f55f77e122)
+
+commit ca933160bec3532a5b2846fc7e684f11509c8298
+Merge: bbb2615 107e9c3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 12 16:42:55 2016 -0600
+
+    Merge pull request #1278 from ceph/wip-16225-min
+
+    hammer: SIGABRT in TrackedOp::dump() via dump_ops_in_flight()
+
+commit c4f1b42810339b5b45b48f74c3ff0ae661a8807a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 12 11:53:00 2016 -0500
+
+    librbd: diffs to clone's first snapshot should include parent diffs
+
+    If the clone has a backing object created after the snapshot that
+    overwrites an extent of the parent, the parent diffs within that
+    extent are not included in the result.
+
+    Hammer-specific implementation due to librbd refactoring during the
+    Infernalis release.
+
+    Fixes: http://tracker.ceph.com/issues/18111
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 996fc2575ef988e6b20fda9ca771b2f6f141cbd3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 09:59:39 2016 -0500
+
+    qa/workunits/rbd: removed qemu-iotest case 077
+
+    The test case is not stable due to racing console output. This
+    results in spurious failures.
+
+    Fixes: http://tracker.ceph.com/issues/10773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c70df978d605a45ff81971b86f5afbefbdaabb6)
+
+commit e97ceb13242fd73b05a20b60208fee7f032e3ef3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Nov 28 14:20:03 2016 -0800
+
+    rgw: omap_get_all() fixes
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 99e866feb3f8aa0ff768d6253979b6b4fc0fa452)
+
+commit 38ab892201ce9ca95010e5f33277be2b0213e96d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 18:37:39 2016 -0500
+
+    rgw/rgw_rados: do not omap_getvals with (u64)-1 max
+
+    Fixes: http://tracker.ceph.com/issues/17985
+
+    This will make the OSD fall over if the object is big
+    and the OSD doesn't impose it's own fail-safe limit.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7cd123ec0dd5951628e4f8ddd4e044a0fbb82f2f)
+
+commit fe77c9a906fb0fbd85762c8c079d85ca106613ce
+Author: Wanlong Gao <wanlong.gao@easystack.cn>
+Date:   Thu Jun 23 20:04:25 2016 +0800
+
+    osd: limit omap data in push op
+
+    We already have the config osd_recovery_max_chunk to limit the total
+    size of omap entries and omap data. But we need an individual config
+    to limit the number of omap entries independently. We call this config
+    osd_recovery_max_omap_entries_per_chunk here with the default number
+    of 64000.
+
+    Signed-off-by: Wanlong Gao <wanlong.gao@easystack.cn>
+    (cherry picked from commit 56064c5cb645254574dbad00e6c16b783115bb93)
+
+commit 0f83bb7dbc5f81ecef0108b746f5bca58e76ff31
+Author: LiuYang <yippeetry@gmail.com>
+Date:   Thu Dec 8 14:21:43 2016 +0800
+
+    rgw: do not abort when accept a CORS request with short origin
+
+    Fixed: #18187
+
+    when accept a CROS request, the request http origin shorter than the bucket's corsrule
+    (eg. origin: http://s.com corsrule: <AllowedOrigin>*.verylongdomain.com</AllowedOrigin>).
+    the rgw_cors.cc::is_string_in_set() will have a wrong index, the radosrgw server will
+    abort.
+
+    $ curl http://test.localhost:8000/app.data -H "Origin:http://s.com"
+
+     0> 2016-12-05 03:22:29.548138 7f6add05d700 -1 *** Caught signal (Aborted) **
+     in thread 7f6add05d700 thread_name:civetweb-worker
+
+     ceph version 11.0.2-2168-gd2f8fb4 (d2f8fb4a6ba75af7e6da0f5a7f1b49ec998b1631)
+     1: (()+0x50720a) [0x7f6b147c420a]
+     2: (()+0xf370) [0x7f6b09a33370]
+     3: (gsignal()+0x37) [0x7f6b081ca1d7]
+     4: (abort()+0x148) [0x7f6b081cb8c8]
+     5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7f6b08ace9d5]
+     6: (()+0x5e946) [0x7f6b08acc946]
+     7: (()+0x5e973) [0x7f6b08acc973]
+     8: (()+0x5eb93) [0x7f6b08accb93]
+     9: (std::__throw_out_of_range(char const*)+0x77) 0x7f6b08b21a17]
+     10: (()+0xbd97a) [0x7f6b08b2b97a]
+     11: (()+0x449c1e) [0x7f6b14706c1e]
+     12: (RGWCORSRule::is_origin_present(char const*)+0x48) [0x7f6b147073b8]
+     13: (RGWCORSConfiguration::host_name_rule(char const*)+0x37) [0x7f6b147074e7]
+     14: (RGWOp::generate_cors_headers(std::string&, std::string&, std::string&, std::string&, unsigned int*)+0xa3) [0x7f6b14593e63]
+     15: (dump_access_control(req_state*, RGWOp*)+0x61) [0x7f6b14653f91]
+
+    Signed-off-by: LiuYang <yippeetry@gmail.com>
+    (cherry picked from commit 67d4d9e64bc224e047cf333e673bb22cd6290789)
+
+commit 47e72024e091a55a5fcd31853e143c94d850b00a
+Author: YankunLi <lioveni99@gmail.com>
+Date:   Mon Dec 5 14:39:20 2016 +0800
+
+    rbd: fix parameter check
+
+    using import-diff command, the imagename must be specified.
+
+    Signed-off-by: Yankun Li <lioveni99@gmail.com>
+
+commit bbb261533455d37b4ef1e5bfc3c8f7765ab74e5a
+Merge: 8e5872b 4533623
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 2 15:09:32 2016 -0500
+
+    Merge pull request #1276 from ceph/no-btrfs-on-hammer
+
+    rados: remove btrfs tests; use xfs instead
+
+commit 7106f426c96978a0a41e9d1482239a4da505157e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 23 14:48:11 2016 +0800
+
+    FileStore::_do_fiemap: do not reference fiemap after it is freed
+
+    `last` points to fiemap::fm_extends[n], and if fiemap gets freed, we can
+    not reference any of its fieldis. so we could remember the check result before
+    freeing it.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c3748fa7737b6c43de2dae5d957fcff01322515c)
+
+    Conflicts:
+	src/os/FileStore.cc
+	put the parameter is_last in the right place
+
+commit 92de2fd9a8b19c3d83d2567720f4e7ad21e43a63
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 29 21:50:52 2016 -0500
+
+    msg/simple/Pipe: handle addr decode error
+
+    Fixes: http://tracker.ceph.com/issues/18072
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f5cfbc0de5464ab80581c416413a86c3d38dfb7)
+
+commit 12d67814912514bb34ba61ceb45a744299d65ff2
+Merge: 8406ee5 e1694d6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 29 22:16:34 2016 +0100
+
+    Merge pull request #11457 from tchaikov/wip-17558-hammer
+
+    hammer: mon: send updated monmap to its subscribers
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8406ee58ca5c4abbe962ba5a4025aabe220e3981
+Merge: bfda134 03af4aa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 29 22:12:46 2016 +0100
+
+    Merge pull request #11628 from ceph/wip-hammer-11567
+
+    hammer: rgw: Don't loop forever when reading data from 0 sized segment.
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5f00e1d4c04698c68248844aad31f4b36eff5735
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Feb 11 11:18:41 2015 +0800
+
+    osd: Add func has_flag in MOSDOp.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (manually cherry picked just one line from d9a2ca5c5ae50dc34e184d577c7b24c56b6cc6ad)
+
+commit be7184589de9208986a2054491b86782dac938e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:32 2016 -0400
+
+    osd: reject PARALLELEXEC ops with EINVAL
+
+    Errors are better than crashing.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 407495197dd878b62f1356f6d939e33ab50d78c6)
+
+    Conflicts:
+	src/osd/ReplicatedPG.cc (trivial resolution)
+
+commit 8d352358edc156f70b5eb68c37bb56cb2392fce2
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:14 2016 -0400
+
+    ceph_test_rados_api_misc: test rados op with bad flas
+
+    Pass the bad PARALLELEXEC flag to remove(), which takes a flags arg.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1aa807f4f29b4b016cc737ffa443e8f30c3b7693)
+
+    Conflicts:
+	src/test/librados/misc.cc (dropped ioctx.remove() test because there is no
+				  remove() in hammer librados)
+
+commit bfda134f6844d6e238052e4f1900f3d2e7497d5e
+Merge: 9bf08dd 0d2f431
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Nov 24 09:15:50 2016 +0100
+
+    Merge pull request #10437 from odivlad/fix-doc
+
+    hammer: docs: Bucket object versions
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 03af4aae8103b52b19737c951e432efc7135b7ca
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Oct 13 21:12:36 2016 -0400
+
+    Don't loop forever when reading data from 0 sized segment.
+
+    The 0 sized segment can arise depending on how
+    the client uploads the object in the first place.
+    The cpu loop then happens during a swift `GET'.
+
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46c5f9773246522e66bb2cca49345d0b62a16c42)
+
+commit 8312dc61ceaa39dca675fda2cb7c88ec3c5ca9f7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 14 13:28:39 2016 -0700
+
+    common/TrackedOp: Move tracking_enabled check into register_inflight_op()
+
+    Make tracking_enabled (and the lock private.)*
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (Partial cherry picked from commit 4e8e6172e8f6cc2adc63a429c32f5edd03982f5e)
+
+    * Backport: Exclude locking since tracking_enabled can't change after launch
+
+commit c7220ccc0ec80bb6788befefe71f676c75457b70
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 10 21:24:25 2016 -0800
+
+    common/TrackedOp: Handle dump racing with constructor
+
+    Use is_tracked to prevent TrackedOp::dump() from trying to call
+    virtual function while still in OpRequest constructor.
+
+    Fixes: #8885
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ad13e05499669a79bde9a219ba1089f929e0388e)
+
+    Conflicts:
+	src/common/TrackedOp.cc
+	src/common/TrackedOp.h
+    Hammer can't use atomic<bool> so use atomic_t instead
+
+commit e8e7664247c7112f1ba03ce4cab915fe70ea5b8e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 4 18:43:43 2016 -0800
+
+    common/TrackedOp: Missed locking when examining "events"
+
+    Another crash caused by a dump_ops_in_flight similar to #8885
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit be098c83143226e502a0587c9215cff941dbace1)
+
+    Conflicts:
+	src/common/TrackedOp.h (trivial)
+
+commit ce58688537d41e0662949f34b3595459581e74fc
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 11 17:58:22 2016 -0700
+
+    CLEANUP: Move locking into dump_ops_in_flight()/dump_historic_ops()
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 17c645f70708134f50374cadb5505b0babe2265a)
+
+    Conflicts:
+	src/mds/MDSRank.cc (variation of code exists in mds/MDS.cc)
+	src/mon/Monitor.cc (no equivalent code here)
+
+commit 5717eb001c6202837fe2e22039e3f855ea4aa065
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 14 15:47:02 2016 -0700
+
+    mds, osd: Fix missing locking for "dump_blocked_ops"
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit be5bed840f21da998be3b8374a8e33f208258e1b)
+
+    Conflicts:
+	src/mds/MDSRank.cc (missing dump_blocked_ops in mds)
+
+commit 6a7fbf42fdb253b7143cb9b3933990145358e06c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 10 21:29:13 2016 -0800
+
+    osd: cleanup: Specify both template types for create_request()
+
+    Code consistency
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cecdc49e6d2f38d5eb9f85218484f7a02bd2e1b7)
+
+commit 3e00ee5cfb0769e9b1c0e0a7540d0b14be880fed
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Jul 17 10:44:06 2015 +0800
+
+    osd: add dump_blocked_ops asok command.
+
+    From ceph health we can see there are blocked ops and from 'ceph health
+    detai' we can see which osd has blocked ops. Later we only in log of osd
+    to see the detail message of blocked ops.
+    Now via asok dump blocked ops.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit cf4e8c655c4a0e6c367c88338cd6e51a3474fa2f)
+
+    Conflicts:
+	src/common/TrackedOp.h (trivial)
+
+commit 656d0c9f307c1b3db072282be46decd72bc88810
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jul 8 13:12:49 2015 +0800
+
+    common/TrackedOp: Should lock ops_history_lock when access shutdown.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 7fe72a2e7d116093ccff6683666e6289f5f12421)
+
+commit 9bf08ddc9422484f5cf0f05c8ccdb4b440e9b641
+Merge: 9655228 2581953
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:50:58 2016 +0100
+
+    Merge pull request #9873 from odivlad/fix-init-el7-hammer
+
+    hammer: build/ops: remove SYSTEMD_RUN from initscript
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9655228d1900d56f4b439d41b2c4a532158f6c51
+Merge: e2c3e21 cb9cc4e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:49:29 2016 +0100
+
+    Merge pull request #10238 from aiicore/hammer
+
+    hammer: mon: fix memory leak in prepare_beacon
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e2c3e21aa4cae160c0136d8d1652b7e4a2c000f0
+Merge: 637d348 c5639e4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:44:19 2016 +0100
+
+    Merge pull request #10569 from jschmid1/wip-16918-hammer
+
+    hammer: build/ops: make stop.sh more portable
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 637d348a98676cc9bf13346d96e7f83219e914c2
+Merge: 84df525 2b87a55
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:40:28 2016 +0100
+
+    Merge pull request #10724 from Vicente-Cheng/wip-16584-hammer
+
+    hammer: crush: reset bucket->h.items[i] when removing tree item
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 84df525e60ea66a4a1808641a12eccd95e16dd2d
+Merge: a7c8c51 8568e57
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:20:29 2016 +0100
+
+    Merge pull request #10904 from tchaikov/wip-14470-hammer
+
+    hammer: mon: return size_t from MonitorDBStore::Transaction::size()
+
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a7c8c511ed061102e1b91858ca9772743d7e094e
+Merge: c8f54c8 a3ba985
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:18:42 2016 +0100
+
+    Merge pull request #10905 from SUSE/wip-11101-hammer
+
+    hammer: build/ops: improve ceph.in error message
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c8f54c84c33354ca27d277a25a2193a3fbd80880
+Merge: 48b6054 6c0b2b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:16:48 2016 +0100
+
+    Merge pull request #10987 from Vicente-Cheng/wip-16592-hammer
+
+    hammer: librbd: ceph 10.2.2 rbd status on image format 2 returns "(2) No such file or directory"
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 48b6054adf4a2d0c58a87bb2a9e8e8719febdc10
+Merge: a97c81c 10472f0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:13:05 2016 +0100
+
+    Merge pull request #11125 from tchaikov/wip-17179-hammer
+
+    hammer: tools: add a tool to rebuild mon store from OSD
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a97c81c3f00abbb79cef0ae069a6f151a089e30e
+Merge: c4cee49 4a36933
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:10:54 2016 +0100
+
+    Merge pull request #11273 from aiicore/wip-17403-hammer
+
+    hammer: mon: OSDMonitor: Missing nearfull flag set
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4cee499622c3c151dcb363193af21844a2f24f5
+Merge: 9495584 ece0ec6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:06:32 2016 +0100
+
+    Merge pull request #11618 from asheplyakov/hammer-16546
+
+    hammer: rbd: fix possible rbd data corruption
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 949558462317e16e1f517bda3ce3465fe30b931b
+Merge: 1fda4a0 5fb8fb4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:56:22 2016 +0100
+
+    Merge pull request #11676 from asheplyakov/hammer-16151
+
+    hammer: osd: update PGPool to detect map gaps and reset cached_removed_snaps
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 1fda4a0fcfe6a9d8f96fe0e054c5b9e8a6e2ecd8
+Merge: 11886bd 31c2594
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:54:39 2016 +0100
+
+    Merge pull request #11899 from SUSE/wip-17840-hammer
+
+    hammer: rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 11886bd5d20c8b2bf09dbabda1e49f14d2283260
+Merge: 21dc0ec f99a1d3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:53:40 2016 +0100
+
+    Merge pull request #11927 from SUSE/wip-17764-hammer
+
+    hammer: osd: fix collection_list shadow return value
+
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 21dc0ec953adeeabe934cd1090f959db7d11cefc
+Merge: 8f19eca 5711e79
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:50:04 2016 +0100
+
+    Merge pull request #12071 from SUSE/wip-17957-hammer
+
+    hammer: RWLock.h: 124: FAILED assert(r == 0) in rados-jewel-distro-basic-smithi
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8f19eca51863ca1c1dfde068479bca5606e4550d
+Merge: af6e208 76eed80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:47:24 2016 +0100
+
+    Merge pull request #11929 from SUSE/wip-17359-hammer
+
+    hammer: tools: ceph-objectstore-tool crashes if --journal-path <a-directory>
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit af6e2080f1ca56f1715ccb76d2372f86bdc272e9
+Merge: 655e904 fbf51ba
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:46:08 2016 +0100
+
+    Merge pull request #11930 from SUSE/wip-17346-hammer
+
+    hammer: cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 655e904142c295ef66c376a3aca148cf2bce60c0
+Merge: c118c0a 7ad7de6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:44:39 2016 +0100
+
+    Merge pull request #11931 from SUSE/wip-17333-hammer
+
+    hammer: tools: crushtool --compile generates output despite missing item
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c118c0af9da259330f5e5c3f3babdfca647a566b
+Merge: e4ea945 ad42da4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:43:19 2016 +0100
+
+    Merge pull request #11932 from SUSE/wip-17146-hammer
+
+    hammer: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (*this == hobject_t(hobject_t::get_max())))
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e4ea9452cf78e01413714b597f49d32d1d11a6dd
+Merge: 92e0a47 e302e63
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:42:24 2016 +0100
+
+    Merge pull request #11933 from SUSE/wip-17142-hammer
+
+    hammer: osd: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 92e0a471adffff1fa2027714116b3d0b5120b6bb
+Merge: d7fc569 64eeb8d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:40:47 2016 +0100
+
+    Merge pull request #11934 from SUSE/wip-17120-hammer
+
+    hammer: mon: %USED of ceph df is wrong
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d7fc569f70b98cf2fa5a842c753f43adc801cdf7
+Merge: 8a15ac7 fc46591
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:39:12 2016 +0100
+
+    Merge pull request #11935 from SUSE/wip-16870-hammer
+
+    hammer: osd: crash on EIO during deep-scrubbing
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8a15ac77c39cc3cc571c62bab68d327894f2db7a
+Merge: 44588f2 4c780bd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:33:27 2016 +0100
+
+    Merge pull request #11937 from SUSE/wip-16428-hammer
+
+    hammer: mon: prepare_pgtemp needs to only update up_thru if newer than the existing one
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 44588f2755429e65960eebf51b21ddb14f245bb8
+Merge: 8c90170 a56b93f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:31:15 2016 +0100
+
+    Merge pull request #11938 from SUSE/wip-14323-hammer
+
+    hammer: msg: OpTracker needs to release the message throttle in _unregistered
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8c90170685fa74fa338b4f22b349e62a01920220
+Merge: 49964b0 2332cf2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:29:45 2016 +0100
+
+    Merge pull request #11939 from SUSE/wip-13927-hammer
+
+    hammer: mds: fix cephfs-java ftruncate unit test failure
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0d2f431b2ecad03a6cda6ffbc9f9cbaac0956af9
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Mon Jul 20 15:37:09 2015 +0800
+
+    rgw: doc: Mark S3 object version API as supported
+
+    S3 object version is already in since Hammer.
+    Ref: http://marc.info/?l=ceph-devel&m=143715290627682
+
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit b62c3b9537e8ab33416a7fb0dd7b801a790d42ee)
+
+commit 8ca16052138bebe9f009104e62b147750dd6584d
+Author: shawn <chen.xiaowei@h3c.com>
+Date:   Thu Jul 30 10:09:12 2015 +0800
+
+    doc: add bucket object version description.
+    bucket object version has been supported, but do not have description in the docs,
+    so add this part.
+
+    Signed-off-by: shawn chen <cxwshawn@gmail.com>
+    (cherry picked from commit 5c395ff85627f0f4154af27aff03dd0a9c409fb4)
+
+commit 49964b01508c5f2f9b5edd620051c18fbea7dfd7
+Merge: 6e1ae86 d91c62d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:52:43 2016 +0100
+
+    Merge pull request #11946 from SUSE/wip-17883-hammer
+
+    hammer: mon: OSDs marked OUT wrongly after monitor failover
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6e1ae86e88661ce9de5aefb38390b4bfbc677a0a
+Merge: 2f9869c ab0786d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:51:51 2016 +0100
+
+    Merge pull request #11948 from SUSE/wip-17671-hammer
+
+    hammer: rbd: export diff should open image as read-only
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2f9869c1be5e03ca1d9daac9c541cfb226b2ae77
+Merge: 2dac884 815541f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:48:06 2016 +0100
+
+    Merge pull request #11949 from SUSE/wip-17374-hammer
+
+    hammer: librbd: image.stat() call in librbdpy fails sometimes
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2dac8848e94b9dcf1a217c6c70164edf669a8eee
+Merge: 1503c2b da653fc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:41:03 2016 +0100
+
+    Merge pull request #11950 from SUSE/wip-17123-hammer
+
+    hammer: rgw: COPY broke multipart files uploaded under dumpling
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1503c2ba753b8ee31e7c632d0463691a83a22b8b
+Merge: 118beb9 009c844
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:39:46 2016 +0100
+
+    Merge pull request #11951 from SUSE/wip-16594-hammer
+
+    hammer: rgw: swift: ranged request on a DLO provides wrong values in Content-Range HTTP header
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 118beb9a390bf481e520d6888453eecc805bb0d7
+Merge: 3b226b8 3404da8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:17:00 2016 +0100
+
+    Merge pull request #11952 from SUSE/wip-16318-hammer
+
+    hammer: rgw: fix inconsistent uid/email handling in radosgw-admin
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3b226b8282f1c994cf3fbb747131121ec9b0c7a0
+Merge: 9ad7369 d89b1f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:14:10 2016 +0100
+
+    Merge pull request #12018 from trociny/wip-17068-hammer
+
+    hammer: librbd: request exclusive lock if current owner cannot execute op
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9ad73698f57598ae1302aaf175cb96082eb64961
+Merge: 3ca7317 a7b182e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 06:42:06 2016 +0100
+
+    Merge pull request #11045 from SUSE/wip-17150-hammer
+
+    hammer: rgw: Anonymous user is able to read bucket with authenticated read ACL
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3ca73177fc28327960f0a8c1612de0358051d515
+Merge: 708c549 d50a501
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 06:32:09 2016 +0100
+
+    Merge pull request #11809 from SUSE/wip-17631-hammer
+
+    hammer: rgw: fix crash when client posts object with null condition
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 10472f0521f6aef516660a470be8c5fe453b1a76
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 22 11:53:57 2016 +0800
+
+    PendingReleaseNotes: document the feature to rebuild monstore
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 9473135aa65282499f67847edb52b7dfd8b374fb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 18:43:39 2016 +0800
+
+    doc: fill keyring with caps before passing it to ceph-monstore-tool
+
+    to make sure the recovered monitor store is ready for use.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af8e21163735377071b6832d8a81b035bb835257)
+
+commit 9c00ba17627c2b3526c558564954b8bb1d8472ef
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 16:32:27 2016 +0800
+
+    tools/ceph_monstore_tool: bail out if no caps found for a key
+
+    we take it as an error if no caps is granted to an entity in the
+    specified keyring file when rebuilding the monitor db.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b4bd4004a836121c11b0bb97d8123df54c271f04)
+
+commit 48696e9d6d89f6b6e99fb76ffa61665c026d7a0e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 30 17:58:14 2016 +0800
+
+    tools/ceph_monstore_tool: update pgmap_meta also when rebuilding store.db
+
+    we should rebuild pgmap_meta table from the collected osdmaps
+
+    Fixes: http://tracker.ceph.com/issues/17400
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cdfa7a69f63d047205dcfccd63b5d58ab0d4695b)
+    Conflicts:
+	src/tools/ceph_monstore_tool.cc: remove C++11-ism
+
+commit ca67e8e88c06a9a37e5dc9e8fe5a1dd1821f8452
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 11:40:56 2016 +0800
+
+    tools/rebuild_mondb: kill compiling warning
+
+    As follow:
+
+    [ 72%] Building CXX object src/tools/CMakeFiles/ceph-objectstore-tool.dir/RadosDump.cc.o
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc: In function ‘int update_mon_db(ObjectStore&, OSDSuperblock&, const string&, const string&)’:
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:289:22: warning: ‘crc’ may be used uninitialized in this function [-Wmaybe-uninitialized]
+	     if (have_crc && osdmap.get_crc() != crc) {
+			  ^
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:238:14: note: ‘crc’ was declared here
+	 uint32_t crc;
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit f16a31476a3f9b44a7c3dabf0dfd2a0d015b11b9)
+
+commit 94390a3296483323d1b2c57856ea2e8ecca99b64
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:43:10 2016 +0800
+
+    tools/rebuild_mondb: return error if ondisk version of pg_info is incompatible
+
+    In this case "r" will be >= 0 and caller will be able to catch
+    this kind of error.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit ac7c1dca0a4f6e92e50d362784122e2d3200e6b5)
+
+commit eaeb52057509b1fa45592ecc147ad694f57d33f3
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:33:56 2016 +0800
+
+    tools/rebuild_mondb: avoid unnecessary result code cast
+
+    In general we return negative codes for error cases, so there is
+    no need perform the cast here.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6a1c01d334fe65124043aa68a6e0cfaea43836b5)
+
+commit 828a4d7a07d178f2dcde30dfe96ca9f054bc1c0d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 00:59:58 2016 +0800
+
+    doc: add rados/operations/disaster-recovery.rst
+
+    document the process to recover from leveldb corruption.
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 79a9f299253e24d20547131b3c9c9e0667e3b869)
+
+commit 3ef5dcd2b37055994794f9cf6139dc2d8807d637
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:53:11 2016 +0800
+
+    tools/ceph_monstore_tool: add "rebuild" command
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d909fa035c8fbbded786b2ca072acc10ea6b6052)
+
+    Conflicts:
+	ceph_monstore_tool.cc: remove C++11ish
+
+commit 6695d1792df9ca7cf5afd5a8c25a04f1eb9ff0fa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:52:19 2016 +0800
+
+    tools/ceph-objectstore-tool: add "update-mon-db" command
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 24faea7ce446bbf09cbd4a9d3434dd5444a6c295)
+
+    Conflicts:
+	src/tools/Makefile-server.am: add rebuild_mondb.{cc,h}, as autotools
+	      support is dropped in master
+	ceph_objectstore_tool.cc, rebuild_mondb.cc: remove C++11ish
+	      adapt to hammer
+
+commit e7d9c15871efcd17b9b1064a5ee49edfe4a159e9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 13:11:24 2016 +0800
+
+    mon/AuthMonitor: make AuthMonitor::IncType public
+
+    so ceph-objectstore-tool is able to use it when rebuilding monitor
+    db.
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 19ef4f16b3aba04119ac647cd6261c74a57ce829)
+
+commit 107e9c358acbc75f212f0bc6fcf83075933035b3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 15 11:29:17 2016 -0700
+
+    thrasher: Add dump_ops_enable options
+
+    Add dump_ops_enable which continuously dumps ops using 3 commands
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (Partial cherry picked from commit 7a528763d126eaca90e40b02aad02ab9c3c92a32)
+
+    Excluded dynamic optracker testing with optrack_toggle_delay option
+
+    Conflicts:
+	tasks/ceph_manager.py (trivial)
+	tasks/thrashosds.py (trivial)
+
+commit b7c87ae74a7f3f3c0096fe6cfe36e1077f96ea1f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 12 09:27:33 2016 -0700
+
+    ceph_manager: Add timeout to admin_socket/osd_admin_socket
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4ad3b866041a3a5391f46cd3d96297834b95956b)
+
+commit f79027a229b2cbb16eaf877b06bf35ecff647bea
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Feb 25 16:10:51 2015 +0800
+
+    Thrasher: log backtrace of thrown exception
+
+    * add a wrapper to log uncaught exception to self.logger, greenlet also
+      prints the backtrace and exception to stderr, but teuthology.log does
+      not capture stderr. so we need to catch them by ourselves to reveal
+      more info to root-cause this issue.
+    * log uncaught exception thrown by Thrasher.do_thrash() to self.log.
+
+    See: #10630
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 64de3cd574b200d90cb3aea0d646a6236d8fec5e)
+
+commit 4533623b3df1e48b821a509013a91e8c3a7ec4a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 16:58:48 2016 -0500
+
+    rados: remove btrfs tests; use xfs instead
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit fb7f49658815d5427f6454b20560991a785a8399
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Mon Mar 30 15:25:34 2015 +0800
+
+    common/TrackedOp: checking in flight ops fix
+
+    Return earlier if tracking is not enabled when checking in flight ops.
+
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 7509a6caa4964d71625c43f2ddc10e6274b6a21b)
+
+commit b58873efafabdc9791f500cff77be9a0a7002227
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Mon Mar 30 14:41:21 2015 +0800
+
+    common/OpTracker: don't dump ops if tracking is not enabled
+
+    If tracking is disabled, there is nothing to dump. No need to go through
+    each sharded list. This is true for both in flight and historical ops.
+
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit ec3c40952bbc4b495875e08b57d764a7487c6c09)
+
+commit 24c92fea4bc68a27a240155f48ee5fa4ee3bbfa0
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Jul 6 13:49:42 2015 +0800
+
+    common/TrackedOp: check tracking_enabled for event "initiated/done".
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d7585c020de41f25e6a598661844169f2bdbb2b4)
+
+commit 0f37a397e35f105b5ea57c7a1a0463c95a2384e1
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Jul 6 10:48:36 2015 +0800
+
+    common/TrackedOp: clean up code make look good.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit edefd05edce6728774bb396ea4e6641ea53463cd)
+
+commit ece0ec60154d22bca7002cc308f3a0f4e36a053e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon May 23 15:14:21 2016 -0700
+
+    hammer: ObjectCacher: fix bh_read_finish offset logic
+
+    If we have an incoming read split across multiple BufferHeads, we want to
+    line up the BufferHead's bl with the incoming OSDOp's bl at the right offset. We
+    were erroneously using this nonsense calculation (always equal to zero!) when
+    a much simpler comparison of the BufferHead's logical object offset to the
+    incoming OSDOp's logical offset will do the trick nicely.
+
+    Fixes: http://tracker.ceph.com/issues/16002
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 9ec6e7f608608088d51e449c9d375844631dcdde)
+
+commit 33f277e601199003fc226c84f67d1bbebb36ad2c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Jun 10 17:01:09 2016 -0700
+
+    hammer: test: build a correctness test for the ObjectCacher
+
+    For now it's very specifically designed to reproduce
+    http://tracker.ceph.com/issues/16002, but it can
+    be extended to other patterns in the future.
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0fd55a9886dd8da344c23a5e9898ee5c5061e8f9)
+
+    Hammer specific changes:
+
+    * use utime_t instead of ceph::real_time
+    * ObjectCacher::prepare_write() has no journal_tid argument
+    * rewrite the code in C++98
+
+commit 439ce945dde196a4439e4468681e35eadd4fca5f
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue May 31 16:18:19 2016 -0700
+
+    hammer: test: split objectcacher test into 'stress' and 'correctness'
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cc9aab1b0a22c3f7320046b97f75dccf2b86cc6d)
+
+commit ff3635a5457ac89ea0fb02f40f0ca723fe34e87c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 13:41:46 2016 -0700
+
+    hammer: test: add a data-storing MemWriteback for testing ObjectCacher
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ed5801492bf2850e80a328ce5a61cb1ff2709215)
+
+    Hammer specific changes:
+
+    * use utime_t instead of ceph::real_time
+    * ObjectCacher::prepare_write() has no journal_tid argument
+    * rewrite code in C++98
+
+commit 7a2abfac759acaf668c6ace25df94619f27e829d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 28 17:06:59 2015 +0800
+
+    hammer: objectcacher: introduce ObjectCacher::flush_all()
+
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bd86c301a54a6e23cf2ce257b8024b6b92dfbe11)
+
+    Conflicts:
+	src/osdc/ObjectCacher.h - whitespace only changes
+
+    Semantic conflicts:
+	    ObjectCacher::flush_all(): dirty_or_tx_bh is a set<BufferHead *>
+	in Hammer
+
+commit 9df96606edfb9ff9932297b8253a297cf75c02f6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 14:23:51 2016 -0700
+
+    hammer: osd: provide some contents on ObjectExtent usage in testing
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a555d9a0c7d3b6c2206520cf78445234b0834639)
+
+commit c5639e4e276a8d5f6457ce9f63b3e77a230a669c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Dec 14 09:41:10 2015 +0000
+
+    stop.sh: make more portable
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27f2d806f905df1a51b11bd79f1af4c26471407a)
+
+commit 708c549b8d77b9a9ea4d46640287848a74898a40
+Merge: f776699 0ac7f2f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 09:03:44 2016 -0600
+
+    Merge pull request #12006 from SUSE/wip-17905-hammer
+
+    hammer: mon: MonmapMonitor should return success when MON will be removed
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d1e1207275eeaf2409b435236c579116900eb6b7
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Thu Oct 13 16:53:18 2016 +0800
+
+    FileStore:: fix fiemap issue in xfs when #extents > 1364
+
+    Fixes: #17610
+    Backport: jewel, hammer
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit 1a1c126d80b427d5230347fbc71a4edea5c0b6c8)
+
+    Conflicts:
+	    src/os/FileStore.cc
+		in hammer, there is no _do_seek_hole_data() function so remove it
+		in hammer, the logic is in FileStore::fiemap not in _do_fiemap()
+		so port the logic to the else branch in FileStore::fiemap
+
+commit a3ba985c38c07f85056f90d125e32ea277c65932
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 5 15:00:59 2016 +0800
+
+    ceph.in: improve the error message
+
+    Fixes: #11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e19e8f1553fdf8b6b642d3222d28f242fab696f0)
+
+commit 5711e794218cc83db023e1d6600ed75ef3ac3fcc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 29 15:24:34 2016 -0400
+
+    os/ObjectStore: fix _update_op for split dest_cid
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e3dc7c772f563f97bc68ebc6dc6e0d408e7c11f3)
+
+commit cb9cc4e57a2abd5c13f61599a7fe594c5f21c9c3
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Mon Jul 11 12:37:25 2016 +0200
+
+    mon/MDSMonitor: fix memory leak in prepare_beacon
+
+    prepare_beacon() case of prepare_update() should put()
+    message in two more cases, because is the last step
+    of dispatch()
+
+    Fixes: http://tracker.ceph.com/issues/17285
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+
+commit 4a36933df8b9338c0a32d582e005894442092d0f
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Wed Sep 14 07:36:43 2016 -0400
+
+    mon: OSDMonitor: Missing nearfull flag set
+
+    Output from 'ceph -s -f json-pretty' wans't showing nearfull flag set.
+
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+    (cherry picked from commit 754887b575795ef96cda095f7f0d8c898e20f00f)
+
+commit e1694d6e85694ee8750b2af211505b89b3d6f0c8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 13 13:19:27 2016 +0800
+
+    mon: send updated monmap to its subscribers
+
+    prior to this change, we sends monmap when serving the subscription
+    requests, but the updates are not sent to the subscribers anymore.
+    so we need to send latest monmap to subscribes and update the
+    subscription status accordingly when the monmap is updated.
+
+    http://tracker.ceph.com/issues/17558
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c2b348b12c47ba505f72228ae0ea9b1a1b674719)
+    Conflicts:
+	src/mon/Monitor.cc: resolve conflicts
+	src/mon/MonmapMonitor.cc: remove C++11-ism
+
+commit ab0786d6a023e391e61119bd309985dcf6e05a4d
+Author: YankunLi <lioveni99@gmail.com>
+Date:   Tue Sep 6 16:16:58 2016 +0800
+
+    rbd: this command should be EXPORT_DIFF
+
+    Fixes: http://tracker.ceph.com/issues/17671
+
+    Signed-off-by: liyankun <lioveni99@gmail.com>
+
+commit d91c62dcf5b1c98b4cfed4eebf29ff9a9d64289b
+Author: Dong Wu <archer.wudong@gmail.com>
+Date:   Thu Oct 27 11:02:36 2016 +0800
+
+    mon: update mon(peon)'s down_pending_out when osd up
+
+    Fixes: http://tracker.ceph.com/issues/17719
+    Signed-off-by: Dong Wu <archer.wudong@gmail.com>
+    (cherry picked from commit 664254a17afbad86983ea5b5b8d385662d89e65e)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (auto declaration for variable "found" is a C++11ism,
+	    replace with the full verbose C++98 type)
+
+commit d89b1f848b8c942d150076cef99491e4c5031bda
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Nov 16 12:01:00 2016 +0200
+
+    librbd: request exclusive lock if current owner cannot execute op
+
+    This is a direct commit to hammer due to librbd code has evolved
+    significantly in the master.
+
+    Fixes: http://tracker.ceph.com/issues/17068
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2b87a551c603291d6c018ed72b0c58e8e45bc334
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 1 20:44:35 2016 +0800
+
+    crush: reset bucket->h.items[i] when removing tree item
+
+    * crush: so we don't see the reference after the removing, this keeps
+      check_item_loc() happy, and move_bucket() use check_item_loc() to see if
+      the removed bucket disappears after the removal.
+    * test: also add unittest_crush_wrapper::CrushWrapper.insert_item
+
+    Fixes: http://tracker.ceph.com/issues/16525
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a7069c7aab6b3f605f3d8f909af96f87977e8698)
+
+    Backport change: modify some codes that only works on c++ 11
+
+commit 0ac7f2f6c87e0d454466927b8528b3f3dae812a2
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:33:52 2016 +0000
+
+    mon: MonmapMonitor: return success when monitor will be removed
+
+    Fixes: http://tracker.ceph.com/issues/17725
+
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit c9d46cfbf2512bc3495c6901de2b8f711bef9bae)
+
+    Conflicts:
+	src/mon/MonmapMonitor.cc (just inserted the "err = 0" assignment
+	    immediately after the error message line)
+
+commit 6c0b2b4ceba188397d5877a602b0e1996d29a4eb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 20:40:10 2016 -0400
+
+    librbd: prevent creation of v2 image ids that are too large
+
+    The librbd API is capped at 24 characters for expressing the
+    object prefix for data blocks (including trailing null byte).
+
+    Fixes: http://tracker.ceph.com/issues/16887
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4de7c8d0a7222c83268d03b99015c6b9d25f124d)
+
+commit 5fb8fb4e450949ebc4c724c102a5b2ae6f1d06c8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 19 16:00:35 2016 -0700
+
+    PG: update PGPool to detect map gaps and reset cached_removed_snaps
+
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a)
+
+    Conflicts:
+	src/osd/OSD.cc: trivial, fuzz is too big so git prefers to be safe
+
+commit 3404da8f1e491df86c8a1f12606546cd5a7d952c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:58:07 2016 -0500
+
+    rgw-admin: return error on email address conflict (add user)
+
+    Fixes the error return inconsistency documented in upstream
+    Ceph issue #13598
+
+    Revised after upstream review, permits changing email address as
+    originally (but use case-insensitive comp).
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 90c143efa2156e55dc017ebe81005315cacaf2c5)
+
+    Conflicts:
+	src/rgw/rgw_user.cc (there is no op_state.found_by_email
+	    conditional in hammer)
+
+commit f5bf5c0f9e9db788fd556f9e92f805fc215138f6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:06:19 2016 -0500
+
+    rgw-admin: convert user email addresses to lower case
+
+    Fixes the email address inconsistency documented in upstream
+    Ceph BUG #13598
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4c438dbbc0e6eda6b9a3018d60019a1a780d6f65)
+
+commit 009c844e34bcd3e4e34eace90f90a140484e2794
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Oct 12 18:43:27 2015 +0200
+
+    rgw: fix wrong length in Content-Range HTTP header of Swift's DLO.
+
+    Fixes: #13452
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit b48f3d774129e0b37531fccda513626bd8ef07e5)
+
+commit 2175b688358455472ac6320bb82ac6bef7133e59
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Oct 12 17:32:21 2015 +0200
+
+    rgw: fix wrong first byte pos in Content-Range HTTP header of Swift's DLO.
+
+    Fixes: #13452
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 66f47f35bbb4966cb289811672938b90ae6d44ae)
+
+commit da653fc3594d49bdac5dc00129c8aa5fd227d6c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 18:01:54 2016 -0700
+
+    rgw: adjust manifest head object
+
+    adjust the manifest head object:
+     - when reading manifest, set the head object to the object
+       we read the manifest from (and its size). Some manifests are
+       broken and point at a different object
+     - when copying multipart object, set the manifest head object to
+       point at the new head object
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 419264586fc46337322f0d60b0ffcdcce3bb5c5a)
+
+commit 8f1c915fa47a889cb50cf84e2d1230df7443752c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:49:07 2016 -0700
+
+    rgw: adjust objs when copying obj with explicit_objs set
+
+    If the head object contains data, need to set it in the list of
+    objs (in addition to being pointed at by head_obj).
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1e012112b99004745952accc6fb11ada5e8e0045)
+
+commit 43a5d5eb2835e1750ef4fbe6d8d3d85af02c6b30
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:36:33 2016 -0700
+
+    rgw: patch manifest to handle explicit objs copy issue
+
+    Fixes: http://tracker.ceph.com/issues/16435
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b51476466e5bb03bbaa6e428bb6bb189a259b9fd)
+
+commit 815541feb989442bdf4d30faf603ef2795bc1ade
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 07:25:36 2016 -0400
+
+    librbd: block name prefix might overflow fixed size C-string
+
+    The issue which resulted in too large v2 image ids was fixed
+    under #16887.
+
+    Fixes: http://tracker.ceph.com/issues/17310
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61734d266c6ee476c2f5fcfbbaefc7d0c7939617)
+
+commit a56b93faf2ae8db2ebe55aaf6003a48ac2bfb8b2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jan 5 16:24:15 2016 -0800
+
+    OpRequest: release the message throttle when unregistered
+
+    We don't want messages in the OpTracker history hanging on to
+    message throttle.
+
+    Fixes: #14248
+    Backport: hammer, firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 67be35cba7c384353b0b6d49284a4ead94c4152e)
+
+    Conflicts:
+	src/msg/Message.h (s/nullptr/0/ because C++98)
+
+commit 4c780bd7fc6e8a98334ddd049a4969d88a9de49f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jun 7 17:15:05 2016 -0700
+
+    OSDMonitor::prepare_pgtemp: only update up_thru if newer
+
+    Fixes: http://tracker.ceph.com/issues/16185
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5f2bd7b6b28aad96d68444b22c04b8b24564616b)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (master uses C++11 "auto" for new_up_thru
+	    iterator, replace with explicit type)
+	src/mon/OSDMonitor.h (trivial resolution)
+
+commit 7ad7de69631372ef364d19bf023e7dba3c88092c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 14:10:16 2016 +0800
+
+    crush/CrushCompiler: error out as long as parse fails
+
+    do not output compiled crush map if anything goes wrong when parsing
+    crush map.
+
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b2c0a079dc074c907e1bc429861230096534f597)
+
+    Conflicts:
+	src/test/cli/crushtool/compile-decompile-recompile.t (hammer needs $TESTDIR/ prefix)
+
+commit 2332cf22d23fd40ae238b98798e3290094548914
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Mar 30 17:35:59 2015 +0800
+
+    mds: fix out-of-order messages
+
+    When MDS is no longer laggy, it should process deferred messages
+    first, then process newly received messages.
+
+    Fix: #11258
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ccdeaf87df8b66e09f6b20950b57ac61bf213086)
+
+commit fc4659155406432404d6e7afb63de545e4abf1a6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Nov 12 12:37:00 2016 +0100
+
+    pg: restore correct behavior of read() callers
+
+    This patch is inspired by fabd6357e42e526d2704d7cb80375c12d731df8d but is not a
+    cherry-pick.
+
+    Fixes: http://tracker.ceph.com/issues/16870
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 64eeb8d96cf295c8a3739d5d993b51f77c35e466
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 5 14:27:43 2016 +0800
+
+    mon/PGMonitor: calc the %USED of pool using used/(used+avail)
+
+    we were using
+    "the raw space used by this pool" / "the raw space of the whole cluster"
+    as the %USED. but it's wrong:
+
+    - there is chance that not all OSDs are assigned to the pool in question
+    - we can not utilize all the raw space for the pool: there is overhead.
+      and the size of available space for a pool is capped by the assigned
+      OSD with minimal free space.
+
+    so we should use
+     USED / (USED + AVAIL)
+    as the %USED. so once we have no AVAIL space left, %USED will be 100%.
+
+    Fixes: http://tracker.ceph.com/issues/16933
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08532ea166dfd97c548d9f1fb478e163021cdda3)
+
+    Conflicts:
+	src/test/mon/CMakeLists.txt (file does not exist in hammer)
+
+commit d62258c03492fca5fca99650cc7514348251cb7b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Aug 7 00:58:04 2016 +0800
+
+    mon/PGMonitor: mark dump_object_stat_sum() as static
+
+    so we can test this method without create an instance of PGMonitor.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af3d3e25823ca9b40ddf09835edb82795ac68f33)
+
+    Conflicts:
+	src/mon/PGMonitor.cc ("const" suffix of function definition was removed
+		in master, but in hammer it wasn't there, so no change necessary)
+	src/mon/PGMonitor.h (added static prefix to function declaration; did
+		not remove "const" suffix because it wasn't there in hammer)
+
+commit e302e636b8fd8f3106a3971dfc44f2c91995e3f5
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:51 2016 -0700
+
+    PG: use upset rather than up for _update_calc_stats
+
+    Fixes: http://tracker.ceph.com/issues/16998
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 200cae8c9db99b3dede377f74413cc0b15033e1f)
+
+commit cfa2b2d3bcad8ebf0a94b89b3a27ce72445fc30a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:35 2016 -0700
+
+    PG: introduce and maintain upset
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 80a5c21d97116e7a66351a0e1f5d9b561f8663ac)
+
+commit ad42da47909f2ce6902c9f34e2e9e0c445bb1cc7
+Author: Tao Chang <changtao@hihuron.com>
+Date:   Mon May 9 09:50:17 2016 -0400
+
+    Cleanup: delete find_best_info again
+
+    After called find_best_info find a auth_pg_shard, it must be not incomplete,
+    so it will not enter find_best_info again.
+
+    Signed-off-by: Tao Chang <changtao@hihuron.com>
+    (cherry picked from commit a25cfc4d46c03d8d78e0254c728ea5c29e2246e1)
+
+    Conflicts:
+	src/osd/PG.cc (trivial resolution)
+
+commit fbf51ba878c7a00c045fc97bf309c39b0d9c4fcc
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Jun 14 17:34:44 2016 +1000
+
+    cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+    an empty secret and a null CryptoKeyHandler
+
+    Fixes: http://tracker.ceph.com/issues/16266
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 009e777fbd18602e5fd66f97bdad95e977e6fecc)
+
+commit 76eed80e6649fdd53a2a3ee3d7242a8e48e1168a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 17:39:24 2016 +0800
+
+    os/filestore/FileJournal: fail out if FileJournal is not block device or regular file
+
+    otherwise JournalingFileStore will assert when deleting FileJournal
+    which still has the non block/regular file opened.
+
+    Fixes: http://tracker.ceph.com/issues/17307
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7431eec6fd24cd08ca6c76a9893e3f6e8c63a916)
+
+commit f99a1d364ae8ec0f7bad220fdd6d665ecdc8029c
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Oct 9 18:31:54 2016 +0800
+
+    osd/PGBackend: fix collection_list shadow return value
+
+    Fixes: http://tracker.ceph.com/issues/17713
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 8a53a604f375eb568a6fc88a402a6afd83fa151a)
+
+    Conflicts:
+	src/osd/PGBackend.cc (master is using store->collection_list and ch;
+		hammer is using store->collection_list_partial and coll)
+
+commit a7b182e00639e635092fa40c39073859720215af
+Author: root <rahul.1aggarwal@gmail.com>
+Date:   Thu Sep 24 00:21:13 2015 +0530
+
+    13207: Rados Gateway: Anonymous user is able to read bucket with authenticated read ACL
+
+    Signed-off-by: root <rahul.1aggarwal@gmail.com>
+    (cherry picked from commit 99ba6610a8f437604cadf68cbe9969def893e870)
+
+commit 31c25941d551c61df4eabdb6c3e2226ef4b3cf02
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Oct 11 19:32:43 2016 +0800
+
+    rgw: fix the field 'total_time'  of log entry in log show opt
+
+    Fixes: http://tracker.ceph.com/issues/17598
+
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 8f4825c5f012c77a3a91e22706dcab9c6067d77e)
+
+commit d50a5018c43d200e4cd36daa7826b2b4c17434ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 20 10:17:36 2016 -0700
+
+    rgw: handle empty POST condition
+
+    Fixes: http://tracker.ceph.com/issues/17635
+
+    Before accessing json entity, need to check that iterator is valid.
+    If there is no entry return appropriate error code.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 23cb642243e09ca4a8e104f62a3bb7b2cbb6ea12)
+
+commit 258195373a6aaabe9a74805268d8094bf48fae0a
+Author: Vladislav Odintsov <odivlad@gmail.com>
+Date:   Mon Jul 4 15:29:22 2016 +0300
+
+    init-radosgw: do not use systemd-run in sysvinit
+
+    `systemd-run` logic in initscripts was introduced because of ticket
+    http://tracker.ceph.com/issues/7627.
+
+    If we have systemd-based operating system, we should use systemd unit files
+    from systemd directory to start/stop ceph daemons.
+    Otherwise, `daemon()` from `/etc/init.d/functions` on systemd distro starts service
+    in `system.slice` and everything works well.
+
+    `systemd-run` can not be run on non-systemd distros, so it's not needed
+    on SysV systems.
+
+    also, ceph-disk is now able to run the "systemctl"
+    to enable and start the ceph-osd, and ceph-deploy is also now
+    playing well with systemd when it comes to ceph-mon and ceph-mds
+
+    Fixes: http://tracker.ceph.com/issues/16440
+
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    (cherry picked from commit 1fd4f92a025e80092b8d08d9b7da2e0b73a52f0f)
+
+commit f776699fd00ff5f0deea15ec4f2efb991831b055
+Merge: 8e13e7c bfa7c14
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 29 10:24:18 2016 +0200
+
+    Merge pull request #11372 from ceph/wip-17386-hammer
+
+    hammer: doc: add "Upgrading to Hammer" section
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bfa7c14894ebab3ab58282782d8217438a3c4047
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 27 15:44:09 2016 +0800
+
+    doc: add "to Hammer" upgrade section
+
+    Fixes: http://tracker.ceph.com/issues/17534
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 8e5872bf6c18b410136fd76974ae11e645c09928
+Merge: 995d3a9 2c27626
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Oct 19 11:36:07 2016 -0600
+
+    Merge pull request #1215 from jcsp/wip-generate-caps-hammer
+
+    tasks/ceph: move generate_caps from teuthology
+
+commit 2c27626c1e5d9989ac6942a94b6d9198ccbd28fb
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 27 12:17:22 2016 +0100
+
+    tasks/ceph: move generate_caps from teuthology
+
+    This was only used in this task, and it is much too
+    ceph-specific to belong in teuthology.
+
+    Fixes: http://tracker.ceph.com/issues/17614
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 298cc8f932a2a327e07f5b73708404588a611322)
+
+commit e7253b8ecb49bf894dbc21d260a767ecd8d3eb19
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 16:43:44 2016 -0700
+
+    rgw_admin: add bi purge command
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 4f605ebf3f415943e8c151364dadfdce3b69cde5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 12:54:47 2016 -0700
+
+    rgw: bucket resharding, adjust logging
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6503430d1b47630b12164f46702bbb42dd7bb453
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 10:41:45 2016 -0700
+
+    cls/rgw: bi_list() fix is_truncated returned param
+
+    is_truncated was never set. Also, make sure that we don't return
+    more entries than requested.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f07e22ae7beab634068cc37a975356393c8b60bf
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 15:13:37 2016 -0700
+
+    rgw_admin: require --yes-i-really-mean-it for bucket reshard
+
+    in the case where num shards are less or equal to current bucket
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 55e3e7742b22fd0f015f7439e7aeabdce1845caa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:35:31 2016 -0700
+
+    rgw_admin: better bucket reshard logging
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 75f69da3ecba55992453f0a901f997e8a1737884
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:11:41 2016 -0700
+
+    rgw: limit bucket reshard num shards to max possible
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+	src/rgw/rgw_rados.h
+
+commit b6a15514e7cad6c8fd84f589fd5e4c7b505045e7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 11:55:32 2016 -0700
+
+    rgw_admin: fix bi list command
+
+    Changes scoping of json section, and push the --object param in so that
+    an object can be specified as a filter.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 6e91760e045b61a144d669f1caa5a55e10b13e58
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 15:49:37 2016 -0700
+
+    rgw_admin: use aio operations for bucket resharding
+
+    also created shards manager to make things slightly cleaner
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit a68d56febde154b3b92d3178d2e877e86b3e2dd9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 10:45:17 2016 -0700
+
+    rgw: bucket reshard updates stats
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/cls/rgw/cls_rgw_types.h
+	src/rgw/rgw_admin.cc
+
+commit 858bcf189afb9fd042c370bb1f53864447dcdc96
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 09:52:29 2016 -0700
+
+    cls/rgw: add bucket_update_stats method
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 37cda4ec1fee1d6e413d1ffe8c5f4f6558634740
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:46:36 2016 -0700
+
+    rgw_admin: reshard also links to new bucket instance
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6c3e734cd87138b55a8a65951d76f7f91c9f0d93
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:33:57 2016 -0700
+
+    rgw: rgw_link_bucket, use correct bucket structure for entry point
+
+    The bucket structure might be different than the one that we were using
+    before.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_bucket.cc
+
+commit 7cc1de860dce16e3956fe604b3bcb8ed1a2fe9ea
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:01:00 2016 -0700
+
+    radosgw-admin: bucket reshard needs --num-shards to be specified
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit b2cc000bbdb7e02e3fa9bdc7d7d71d3e979fee2b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 05:49:37 2016 -0700
+
+    cls/rgw: fix bi_list objclass command
+
+    was filtering entries, even if filter was not specified, and need to
+    set boundary for plain entries. Also, list_instance_entries() was not
+    working correctly, and added list_olh_entries().
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 53982a2e61614398de2e37705da12aa6874360a0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:34 2016 -0700
+
+    rgw_admin: bucket rehsrading, initial work
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 995d3a9c49c04d6ded62aa34cfaacc650b7d13a6
+Merge: c2a8f05 e4038a0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 21:34:49 2016 -0700
+
+    Merge pull request #1192 from ceph/wip-offline-split-hammer2
+
+    ceph_manager: test offline split via ceph-objectstore-tool
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8e13e7ce98e99d3ef1b2942f06b4c9ab35c12066
+Merge: e2d4f0a 99bb9f6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 21:34:26 2016 -0700
+
+    Merge pull request #11253 from ceph/wip-offline-split-hammer2
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e4038a0c0068569cc1c7bcaf1b590fba31291610
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 18:16:20 2016 -0700
+
+    ceph_manager: test offline split via ceph-objectstore-tool
+
+    When killing an osd, split all pools with a low threshold.
+    This will slow down tests, but should not impact correctness.
+
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit f28f881bda0ad426bfa9c0e22cc4658b2ce2ec5d)
+
+commit 99bb9f603f2123e81470a819c1cd64df13a22425
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 11:45:00 2016 -0700
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+
+    Use the usual split code, and split each dir that meets the
+    usual split criteria.
+
+    This can be run with lower than usual split settings, to avoid costly
+    online splits. To make sure the directories are not merged again, use
+    a load merge threshold (e.g. filestore merge threshold = 1), and
+    adjust the split multiplier accordingly.
+
+    Fixes: http://tracker.ceph.com/issues/17220
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7b0428e0e8d8f5459311dc698d94a3ac4f04684)
+
+    Conflicts:
+	src/tools/ceph_objectstore_tool.cc
+    * remove c++11 auto usage
+    * change HashIndex::list_subdirs() call to use set instead of vector
+    * adjust to hammer signature of coll_t::is_pg()
+
+commit c1b64c8f9c23473b758300958a94a172f1d39b64
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 10 12:52:43 2015 -0700
+
+    rgw: rgw_obj encoding fixes
+
+    Fix a few issues:
+     - conversion to and from rgw_obj_key
+     - strip_namespace_from_object() in the case where object starts
+       with underscore
+     - encoding if object has either namespace or instance and starts
+       with underscore
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c381cc5d69ebec4370ee18e350b9d003e15b2770
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:15 2016 -0700
+
+    rgw: utilities to support raw bucket index operations
+
+    and other related changes.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_rados.cc
+	src/rgw/rgw_rados.h
+
+commit d41cbb44c55cf2dca5ab25d5a1ec76e30db26150
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:06:17 2016 -0700
+
+    rgw: use bucket_info.bucket_id instead of marker where needed
+
+    We used to use these interchangeably, but they actually have diffent meaning.
+    The marker is the prefix we assign to the objects in this bucket, whereas
+    the bucket_id is the bucket instance's id. These used to hold the same
+    value, but with bucket resharding it's not going to be true anymore.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 228f52cca1866c40a3840e06bd260c1d92d59caa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:01:39 2016 -0700
+
+    cls/rgw: utilities to support raw bucket index operations
+
+    New flavour of bi_put() call, and a function to extract key off
+    a raw bi entry.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e2d4f0af387869569a8c143fa2d2873e6788d2a2
+Merge: 8f66abd 0b42082
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Sep 20 10:33:02 2016 -0400
+
+    Merge pull request #11140 from cbodley/wip-radosgw-admin-man-hammer
+
+    hammer: add orphan options to radosgw-admin --help and man page
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 0b420821fd3ed33d077ce76beef7a6f1a0e5edb7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 20 09:27:52 2016 -0400
+
+    man: add orphan options to radosgw-admin.8
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit 054e0d45c351525454517472df2c0b1dcecb5c39
+Author: tserlin <tserlin@redhat.com>
+Date:   Mon Sep 19 14:40:12 2016 -0400
+
+    Add two options to radosgw-admin.rst manpage
+
+    Add '--job-id' and '--max-concurrent-ios' to Orphan Search Options
+
+    Fixes: http://tracker.ceph.com/issues/17281
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit 697f30d86f49b73c981c06375ab2937570b1db01)
+
+commit 2920334210376bb34e59d6ecf08557732bcc3861
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 12:56:39 2016 -0600
+
+    radosgw-admin: add "--orphan-stale-secs" to --help
+
+    The radosgw-admin --help did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option
+    sets the number of seconds to wait before declaring an object to be an
+    orphan.
+
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 354059ae43b4f4cc797da1669715399cd96a4738)
+
+commit 34ecb3584230db8458f193ac01deff9264cd119a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Apr 1 13:46:30 2016 +0200
+
+    rgw_admin: improve the orphans find help
+
+    Adding  a few missing commandline switches and improve the help message
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 27e4c46f741a5f908d5c8ca1af227b3581c697f6)
+
+commit 3575fa8112ac39b1c6edf2aaee5b02aa67fa5766
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 11:40:14 2016 -0600
+
+    doc: add "--orphan-stale-secs" to radosgw-admin(8)
+
+    The radosgw-admin(8) manual page did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option sets
+    the number of seconds to wait before declaring an object to be an
+    orphan.
+
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit a676c516069cc448591018ecf4f7d1f7f7bc3bfd)
+
+commit 8f66abd02ed5f0bc697b80b3071f0c70886aab6e
+Merge: fe6d859 c7f6b47
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Wed Sep 7 13:06:33 2016 -0400
+
+    Merge pull request #10839 from oritwas/hammer-default-quota
+
+    Hammer default quota
+
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
diff --git a/doc/changelog/v0.94.2.txt b/doc/changelog/v0.94.2.txt
new file mode 100644
index 000000000..f8113af7f
--- /dev/null
+++ b/doc/changelog/v0.94.2.txt
@@ -0,0 +1,1563 @@
+commit 5fb85614ca8f354284c713a2f9c610860720bbf3 (tag: refs/tags/v0.94.2, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jun 9 12:32:34 2015 -0700
+
+    0.94.2
+
+commit d967cecf0a5d7fbf992a0195341cbd893a358264
+Merge: eb69cf7 968573b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 29 09:49:43 2015 +0200
+
+    Merge pull request #4795 from dachary/wip-11806-hammer
+    
+    ceph / ceph-dbg steal ceph-objecstore-tool from ceph-test / ceph-test-dbg
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 968573b8930a7c8485bf53e3a989ce2f7d0a2fff
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 28 10:35:51 2015 +0200
+
+    debian: ceph-dbg steals ceph-objectstore-tool from ceph-test-dbg
+    
+    When ceph-objectstore-tool was moved from ceph-test to
+    ceph by 61cf5da0b51e2d9578c7b4bca85184317e30f4ca, the ceph package in
+    debian/control was updated accordingly, as recommended by
+    https://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
+    
+    The same must be done for the ceph-dbg package because
+    /usr/lib/debug/usr/bin/ceph-objectstore-too is no longer in
+    ceph-test-dbg.
+    
+    Although the change was merged may 6th, 2015
+    8f23382064c189b657564d58c3f9d17720e891ed, teuthology jobs were not
+    always failing because packages were not systematically upgraded during
+    the installation. The missing dependencies that were responsible for
+    this upgrade problem were fixed by
+    f898ec1e4e3472b0202280f09653a769fc62c8d3 on may 18th, 2015 and all
+    upgrade tests relying on ceph-*-dbg packages started to fail
+    systematically after this date.
+    
+    http://tracker.ceph.com/issues/11546 Fixes: #11546
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 6f11fbf41fab10924b1e0e41fcf27864779d4073)
+
+commit eb69cf758eb25e7ac71e36c754b9b959edb67cee
+Merge: 63832d4 344328d
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue May 26 19:52:59 2015 -0600
+
+    Merge pull request #4773 from dachary/wip-11733-hammer
+    
+    Debian: ceph-test and rest-bench debug packages should require their respective binary packages
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 344328de584ac707b59ab857f1f3dd4165adfcf5
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon May 18 10:50:58 2015 -0600
+
+    debian: set rest-bench-dbg ceph-test-dbg dependencies
+    
+    Debian's debug packages ought to depend on their respective binary
+    packages. This was the case for many of our ceph packages, but it was
+    not the case for ceph-test-dbg or rest-bench-dbg.
+    
+    Add the dependencies on the relevant binary packages, pinned to
+    "= ${binary:Version}" per convention.
+    
+    http://tracker.ceph.com/issues/11673 Fixes: #11673
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f898ec1e4e3472b0202280f09653a769fc62c8d3)
+
+commit 63832d4039889b6b704b88b86eaba4aadcfceb2e
+Merge: 195884e 293affe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 18 14:26:16 2015 +0200
+
+    Merge pull request #4696 from ceph/wip-11622-hammer
+    
+    Wip 11622 hammer
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 195884e21760f4948f7d1df8b65788514f918054
+Merge: b69fb89 95818da
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 15 13:47:04 2015 -0700
+
+    Merge pull request #4649 from ceph/wip-hammer-package-perf-objectstore
+    
+    packaging: include ceph_perf_objectstore
+
+commit 293affe992118ed6e04f685030b2d83a794ca624 (refs/remotes/gh/wip-11622-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 13 17:05:22 2015 -0700
+
+    rgw: merge manifests correctly when there's prefix override
+    
+    Fixes: #11622
+    Backport: hammer, firefly
+    
+    Prefix override happens in a manifest when a rados object does not
+    conform to the generic prefix set on the manifest. When merging
+    manifests (specifically being used in multipart objects upload), we need
+    to check if the rule that we try to merge has a prefix that is the same
+    as the previous rule. Beforehand we checked if both had the same
+    override_prefix setting, but that might not apply as both manifests
+    might have different prefixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 389ae6739ddc6239a4dd7c5f7f9bfc9b645b8577)
+
+commit a43d24861089a02f3b42061e482e05016a0021f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 12 16:40:10 2015 -0700
+
+    rgw: restore buffer of multipart upload after EEXIST
+    
+    Fixes #11604
+    Backport: hammer, firefly
+    
+    When we need to restart a write of part data, we need to revert to
+    buffer to before the write, otherwise we're going to skip some data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 580ccaec12daae64c38a1616d0be907bdd70a888)
+
+commit 95818dac1522c218662ec12bd42c470d8394b3b9
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 13 16:08:35 2015 -0600
+
+    packaging: include ceph_perf_objectstore
+    
+    The /usr/bin/ceph_perf_objectstore file is installed by default. Prior
+    to this commit it was missing from the packaging. This caused the RPM to
+    fail to build in mock.
+    
+    Add ceph_perf_objectstore to the "ceph-test" RPM and Debian package.
+    
+    If we end up developing further ceph_perf_* utilities, it would make
+    sense to glob them all with a wildcard, similar to what we are doing
+    with all the ceph_test_* utilities in ceph-test.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 338b44bc74bc4597939c5c58f2a6f2cb08de7d9c)
+
+commit b69fb89122d6f989152a29124cc7ed54b5e4d43b
+Merge: 0f02512 efbfe6f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 11 10:27:02 2015 -0700
+
+    Merge pull request #4568 from dachary/wip-10662-10663-hammer
+    
+    RGW swift API: Response header of COPY request for object does not
+    
+    contain x-copied-from, custom metadata, x-copied-from-last-modified, X-Copied-From-Account headers
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0f025122fe574b99504a630e3d489c3449cbbe46
+Merge: 7df3eb5 6e2dd40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 11 14:08:53 2015 +0200
+
+    Merge pull request #4629 from ceph/hammer-uclient-checking
+    
+    Hammer uclient checking
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6e2dd408be95644ee5bceb556a90483f882fe51c
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 18:42:01 2015 +0100
+
+    client: fix error handling in check_pool_perm
+    
+    Previously, on an error such as a pool not existing,
+    the caller doing the check would error out, but
+    anyone waiting on waiting_for_pool_perm would
+    block indefinitely (symptom was that reads on a
+    file with a bogus layout would block forever).
+    
+    Fix by triggering the wait list on errors and
+    clear the CHECKING state so that the other callers
+    also perform the check and find the error.
+    
+    Additionally, don't return the RADOS error code
+    up to filesystem users, because it can be
+    misleading.  For example, nonexistent pool is
+    ENOENT, but we shouldn't give ENOENT on IO
+    to a file which does exist, we should give EIO.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e08cf25cafef5752877439c18cc584b0a75eca08)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 3c2e6ae97d6129cb8f5befb3e7bf4be16373f6a5
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 18:17:37 2015 +0100
+
+    client: use SaferCond in check_pool_perm
+    
+    Just because it's easier to read.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 289ee3b80ccce6bab2966f513a37332280d04a06)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 79b2ac215187402a99594424944db4169f2b2cdf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 24 15:23:21 2015 +0800
+
+    client: check OSD caps before read/write
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 3c4028ec21e3ef9e8801c4570420c88722651cc7)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7df3eb5e548f7b95ec53d3b9d0e43a863d6fe682
+Merge: 6a7fa83 2f86995
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:07:07 2015 -0700
+
+    Merge pull request #4567 from dachary/wip-10938-hammer
+    
+    RGW Swift API: response for GET/HEAD on container does not contain the X-Timestamp header
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6a7fa83b3e72b85f92d003a5bbb03a301354a657
+Merge: 553f0db 3edb196
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:59 2015 -0700
+
+    Merge pull request #4570 from dachary/wip-10243-hammer
+    
+    civetweb is hitting a limit (number of threads 1024)
+
+commit 553f0db9a1fcff2601a8791af1d2bb6975d2821d
+Merge: 3fe1f2b 3aef0f2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:41 2015 -0700
+
+    Merge pull request #4573 from dachary/wip-11125-hammer
+    
+    rgw: keystone token cache does not work correctly
+
+commit 3fe1f2b8ab3d0d3943a312e90f6a3de99c36beb4
+Merge: e0ed459 4d1f3f0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:22 2015 -0700
+
+    Merge pull request #4574 from dachary/wip-11160-hammer
+    
+    rgw: shouldn't need to disable rgw_socket_path if frontend is configured
+
+commit e0ed459442b1e9053e29e345cd0f30d1b4b4b994
+Merge: d6de3fa d2043a5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:04:36 2015 -0700
+
+    Merge pull request #4575 from dachary/wip-10650-hammer
+    
+    Response header of swift API PUT /container/object returned by RGW
+    
+    does not contain last-modified, content-length, x-trans-id headers. But Swift returns these headers.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit d6de3fa0b55918bc2ac2d65ee8308f04d3605dfd
+Merge: 96dc624 2cb5d60
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:03:37 2015 -0700
+
+    Merge pull request #4576 from dachary/wip-10661-hammer
+    
+    RGW swift API: Response header of POST request for object does not contain content-length and x-trans-id headers
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 96dc624ee1f593f817055d3426054ef2e05cbf92
+Merge: ae61aee f4a0dab
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:59:15 2015 -0700
+
+    Merge pull request #4579 from dachary/wip-11036-hammer
+    
+    RGW Swift API: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ae61aee99bee9185af22424ec8019e0308828bf5
+Merge: 593d07f 7f2a9ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:57:47 2015 -0700
+
+    Merge pull request #4580 from dachary/wip-11088-hammer
+    
+    RGW Swift API: wrong handling of empty metadata on Swift container
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 593d07f29df8584629936051be4458b00f8a8f1f
+Merge: 4f2b41c d164d80
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:57:12 2015 -0700
+
+    Merge pull request #4581 from dachary/wip-11323-hammer
+    
+    rgw: quota not respected in POST object
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 4f2b41c4c803786c49cd2d2806d82e9223ab96a9
+Merge: 92e7a7f 893ffd3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri May 8 09:15:41 2015 -0700
+
+    Merge pull request #4566 from dachary/wip-11478-hammer
+    
+    Queued AIO reference counters not properly updated
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 92e7a7f057dd9aabb4f66965c412135d05f6812f
+Merge: 2fbf171 0944051
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri May 8 09:15:19 2015 -0700
+
+    Merge pull request #4564 from dachary/wip-11369-hammer
+    
+    Periodic failure of TestLibRBD.DiffIterateStress
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2fbf1712d1e190774ada0af5094134369effb3ac
+Merge: 9d97946 02a3813
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 8 17:32:35 2015 +0200
+
+    Merge pull request #4585 from ceph/wip-11370-hammer
+    
+    A retransmit of proxied flatten request can result in -EINVAL
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9d97946a8908e48849355a410415e09914ef3948
+Merge: fb10594 c548d8d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri May 8 20:39:29 2015 +0800
+
+    Merge pull request #4618 from dachary/wip-11398-kill-daemons-hammer
+    
+    tests: ceph-helpers kill_daemons fails when kill fails
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c548d8d44baae78b868391e3c6fb7294f024b082
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed May 6 20:14:37 2015 +0200
+
+    tests: ceph-helpers kill_daemons fails when kill fails
+    
+    Instead of silently leaving the daemons running, it returns failure so
+    the caller can decide what to do with this situation. The timeout is
+    also extended to minutes instead of seconds to gracefully handle the
+    rare situations when a machine is extra slow for some reason.
+    
+    http://tracker.ceph.com/issues/11398 Fixes: #11398
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0e26e9f72bc14417266b87ac1159607e1b109303)
+
+commit fb10594f2ab2427a2bf1d2f2b164a3a0928e3335
+Author: Yuri Weinstein <yuri.weinstein@inktank.com>
+Date:   Thu May 7 17:12:35 2015 -0700
+
+    Added a "ceph hello world" for a simple check for ceph-deploy qa suite
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 13abae186357f4e9bb40990a7a212f93ec2e1e79)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 6cfae7f074462498e82cfeeddbc2fe8d302c7aa0
+Merge: 736cdf4 8d9f4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 13:29:57 2015 -0700
+
+    Merge pull request #4502 from dachary/wip-11026-hammer
+    
+    objecter: can get stuck in redirect loop if osdmap epoch == last_force_op_resend
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 736cdf450caa0b22cbfc54f4497717cf63d5bda7
+Merge: 3bd8e4f 46a4e8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 21:45:53 2015 +0200
+
+    Merge pull request #4562 from dachary/wip-11376-hammer
+    
+    ceph-objectstore-tool should be in the ceph server package
+    
+    
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 46a4e8af5be54d8348a920c4a3a58e24dbf35988
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Apr 30 15:53:22 2015 -0600
+
+    packaging: mv ceph-objectstore-tool to main ceph pkg
+    
+    This change ensures that the ceph-objectstore-tool utility is present on
+    all OSDs.  This makes it easier for users to run this tool to do manual
+    debugging/recovery in some scenarios.
+    
+    http://tracker.ceph.com/issues/11376 Refs: #11376
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 61cf5da0b51e2d9578c7b4bca85184317e30f4ca)
+        Conflicts:
+                debian/control
+                because file layout changes from ceph-test and ceph << 0.94.1-46
+
+commit 3bd8e4fa17d8acfd645b8a553bf58de48d59d648
+Merge: 76f6db2 6a04b55
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:05:48 2015 +0200
+
+    Merge pull request #4561 from dachary/wip-11143-hammer
+    
+    src/ceph-disk : disk zap sgdisk invocation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 76f6db24e09a4bfc55cbfb7075104f20653263cc
+Merge: e5a20f8 8996907
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:03:53 2015 +0200
+
+    Merge pull request #4560 from dachary/wip-11507-hammer
+    
+    object creation by write cannot use an offset on an erasure coded pool
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e5a20f83ed6d8d3d09827343d757318026f6a690
+Merge: e7671a5 8a6e6e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:03:16 2015 +0200
+
+    Merge pull request #4559 from dachary/wip-11429-hammer
+    
+    OSD::load_pgs: we need to handle the case where an upgrade from earlier versions which ignored non-existent pgs resurrects a pg with a prehistoric osdmap
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e7671a58939f6e2e37f9f6df0f039f485ad4ca16
+Merge: 126a372 113f3b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 18:48:56 2015 +0200
+
+    Merge pull request #4563 from tchaikov/wip-hammer-11534
+    
+    mon: Total size of OSDs is a maginitude less than it is supposed to be.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 126a37212550a4a59beaa80e0579098198f74db5
+Merge: 58b30d5 c87aa11
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 18:18:17 2015 +0200
+
+    Merge pull request #4577 from dachary/wip-10666-hammer
+    
+    RGW swift API: Response header of GET request for container does not contain X-Container-Object-Count, X-Container-Bytes-Used and x-trans-id headers
+    
+    Reviewed-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f4a0dabfe8debc7b54afa5da179d51db891f5bc0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Mar 8 22:45:34 2015 +0100
+
+    rgw: send Content-Length in response for HEAD on Swift account.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 29073d84640b61d0221b2d2ab73c271d60fd13ba)
+
+commit 32f4a7439d1e0ce6aab1c1e39407b14e124d03bf
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:18:19 2015 +0100
+
+    rgw: send Content-Length in response for DELETE on Swift container.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d260a93405a7a34a899f716bd9c4e731baa9ffd0)
+
+commit d39a66007fa7211c00a67f9cd898e55551f5ae62
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:14:26 2015 +0100
+
+    rgw: send Content-Length in response for PUT on Swift container.
+    
+    Fixes: #11036
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 10c1f1aa1e09df5edf5d2f9f9b0273ddbcb384fa)
+
+commit 02a3813a6a4f9c8ce14f64fc7f378030e7ea6f93
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 01:03:52 2015 -0400
+
+    librbd: flatten should return -EROFS if image is read-only
+    
+    Fixes: #11475
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f141e02ab719df830648318f4c1d9ca286071ed3)
+
+commit af8939be80310e234745fe81e67244ab52c6add5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 01:00:38 2015 -0400
+
+    librbd: allow snapshots to be created when snapshot is active
+    
+    The librbd API previously permitted the creation of snapshots while
+    the image context was associated to another snapshot.  A recent code
+    cleanup broke that ability, so this re-introduces it.  The code change
+    also allows minor cleanup with rebuild_object_map.
+    
+    Fixes: #11475
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 594a6610802f2cadb62200815bd8b9860809e759)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+
+commit d21c0c00d2bed282677d2063a3fb6f5346641286
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 21 12:59:33 2015 -0400
+
+    librbd: better handling for duplicate flatten requests
+    
+    A proxied flatten request could be replayed, resulting in a
+    -EINVAL error code being generated on the second attempt. Filter
+    out that error if it is known the parent did exist before the
+    op started.
+    
+    Fixes: #11370
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ef7e210c3f747bc4c3c8768c7b6407cc91c5c319)
+
+commit ec0bd1dea526e04333d8059421666dcd2a59044e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 18 11:51:47 2015 -0400
+
+    librbd: use generic helper for issuing async requests
+    
+    resize, flatten, and rebuild object map now use the same
+    bootstrap code for sending the request to the remote lock owner
+    or executing the request locally.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 18fd6ca7f59d5545f0bb0b0e899d0739639ce104)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+
+commit 8a6e6e4c107b03563b2e38aa24cc2067ce6a7350
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Apr 20 23:45:57 2015 -0700
+
+    OSD: handle the case where we resurrected an old, deleted pg
+    
+    Prior to giant, we would skip pgs in load_pgs which were not present in
+    the current osdmap.  Those pgs would eventually refer to very old
+    osdmaps, which we no longer have causing the assertion failure in 11429
+    once the osd is finally upgraded to a version which does not skip the
+    pgs.  Instead, if we do not have the map for the pg epoch, complain to
+    the osd log and skip the pg.
+    
+    Fixes: 11429
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit fbfd50de5b9b40d71d2e768418a8eca28b1afaca)
+
+commit efbfe6fced72d07309ccf1f1a219c037b7f535fa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat Mar 21 15:53:08 2015 +0100
+
+    rgw: improve metadata handling on copy operation of Swift API.
+    
+    Fixes: #10645
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit a00cb31cc52e91bfacdd15e0af60be74e66c1996)
+
+commit d164d8004feddb3e2238b26e3360a905e209d117
+Author: 9seconds <nineseconds@yandex.ru>
+Date:   Wed Apr 1 09:12:06 2015 +0300
+
+    rgw: quota not respected in POST object
+    
+    Signed-off-by: Sergey Arkhipov <nineseconds@yandex.ru>
+    Backport: hammer, firefly
+    Fixes: #11323
+    (cherry picked from commit e76f84e179d2ba8bfc0dc5abf4e620fef14bc8a0)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+            discard the whitespace modification hunk that were creating
+    	conflict and ignore the conflict due to an unrelated cast
+            modification in the context
+
+commit 7f2a9ed7a986145d4b34517a1a1bb44799ebf621
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 10 19:36:59 2015 +0100
+
+    rgw: fix handling empty metadata items on Swift container.
+    
+    Fixes: #11088
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1660d8625212f7b5fb30568ed640f467297e36cb)
+
+commit 8e6efdbcb0f820b2ab3728662efbfb4bc45495af
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 27 15:23:16 2015 +0100
+
+    rgw: send Content-Length in response for GET on Swift container.
+    
+    Fixes: #10971
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5a64fb5430b6b42174bf53a5910d9435043c1380)
+
+commit 54b62904a4cc3913be23803734fa68741a3c33cc
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 12:42:52 2015 +0100
+
+    rgw: enable end_header() to handle proposal of Content-Length.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f98fd4dfda7a105d621f99c2b48eb9ab9b45d979)
+
+commit c87aa110bd224ae3220c5486fbd2486ebdfb8b8e
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Mon Feb 16 18:47:59 2015 +0200
+
+    rgw: Swift API. Complement the response to "show container details"
+    
+    OpenStack Object Storage API v1 states that X-Container-Object-Count, X-Container-Bytes-Used and user-defined metadata headers should be included in a response.
+    
+    Fixes: #10666
+    Backport: hammer
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit d3a3d5ae5859cd7f2eed307e8f942f9a9fd75f35)
+
+commit 2cb5d600699085fe0e996b91cf85603be3da230e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Feb 26 19:21:03 2015 +0100
+
+    rgw: enforce Content-Length in response for POST on Swift cont/obj.
+    
+    Fixes: #10661
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit fd0c612cf867d2e99e74820130357e63305970fb)
+
+commit d2043a5f3b8d7a3ce50c4e84aa88481c8912ae25
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 21:41:45 2015 +0100
+
+    rgw: send Last-Modified header in response for PUT on Swift object.
+    
+    Fixes: #10650
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 9a22acc57684534a245f25848c23d1db21f16653)
+
+commit 4d1f3f03ef8809693c8ea2bcbc4d24fd41ae1842
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:49:13 2015 -0700
+
+    rgw: don't use rgw_socket_path if frontend is configured
+    
+    Fixes: #11160
+    Backport: hammer, firefly
+    
+    Previously if we wanted to use the tcp fcgi socket, we needed to clear
+    rgw_socket_path.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 36d6eea3caa79fcb4e08bdd340ccda2474b9e5ea)
+
+commit 3aef0f2bb6f88bb17c460a3cef0d3503550f716c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 3 11:03:35 2015 -0800
+
+    rgw: update keystone cache with token info
+    
+    Fixes: #11125
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6616294aa140ceb83cc61c6ab6f9947636f5e67d)
+
+commit 3edb1964fe9e8574aafcb758d170007f0e43a324
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 21 11:08:45 2015 -0700
+
+    civetweb: update max num of threads
+    
+    Fixes: #10243
+    cherry-picked upstream fixed into submodule
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7a432f7bdfbbc51518fb63d7f2ecab401e8a8d4f)
+
+commit bc6eb8d5f0a66aec3fbda0b794d008a3157a8154
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Apr 7 14:09:57 2015 +0200
+
+    rgw: improve code formatting ONLY.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 214c8b32b1b04885005e929a7ed2d4354b3ea20b)
+
+commit 7aa1ae60cea17e0bd140c0cf2313d82f2f64554f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 17 14:07:34 2015 +0100
+
+    rgw: send X-Copied-From-Last-Modified header of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e7724a1d8c0872362c19f578fe30ac2cf3dada90)
+
+commit 150b9e2b85a72dc247da4ba1ab770e6af053acb7
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 17 14:06:37 2015 +0100
+
+    rgw: dump object metadata in response for COPY request of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit c52b75edeaeef19471b9aca772bf08055bf04031)
+
+commit e749701be5368a22cad1630f8202e48f5d980409
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 17:59:01 2015 +0100
+
+    rgw: refactor dumping metadata of Swift objects.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ccf6eaac6f7068289c4a4ffd3f0481d497ba7c87)
+
+commit b034511fa79996415640b4aca3e8747340f2a127
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 18:37:39 2015 +0100
+
+    rgw: add support for X-Copied-From{-Account} headers of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 94f1375ccb9df02cdd8f6828153ae2e3a7ad36b1)
+
+commit c9e6a0bf8c601c4fd9065c1f3a8ea445bd652a52
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Mar 11 14:08:03 2015 +0100
+
+    rgw: send ETag, Last-Modified in response for copying Swift cobject.
+    
+    Fixes: #11087
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 385fe4b4bbbd4a9aab92abf2a813090deeaa037e)
+
+commit 7f41ff0a6577b0784c6719b9d705f32921b1c40e
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Fri Feb 20 18:31:03 2015 +0200
+
+    rgw: Swift API. Allows setting attributes with COPY object operation.
+    
+    http://developer.openstack.org/api-ref-objectstorage-v1.html says: "With COPY, you can add additional metadata to the object."
+    
+    Fixes: #10662
+    Backport: hammer
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit 1b722bbcd691e0a4a39ea77cd28e309fd723ec88)
+
+commit 2f869959ffe1adbcfef7d26ae2d022d23d982673
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 25 16:12:58 2015 +0100
+
+    rgw: improve format of X-Timestamp on Swift objects.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e54fef9542bc2b5db192308728a99df139a4b6cf)
+
+commit 48b19810a9860f6fccbf8d9b8a2fadfb37f598dd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 25 14:11:40 2015 +0100
+
+    rgw: add support for X-Timestamp on Swift containers.
+    
+    Fixes: #10938
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit b7e9bf6b98ee48d1977d907a9e5130c0ce073c54)
+
+commit 893ffd3767678ab881c4bc44ecfe1801cb9f9704
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 03:43:10 2015 -0400
+
+    tests: AioCompletion incorrectly freed
+    
+    The AioCompletion should be released instead of directly
+    deleted.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 387a09eeeaf0b66b3a2ddc36388da27d5804a4c7)
+
+commit 96b0db5decfad452964750cff92a63007433e519
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 03:42:24 2015 -0400
+
+    librbd: update ref count when queueing AioCompletion
+    
+    If the client releases the AioCompletion while librbd is waiting
+    to acquire the exclusive lock, the memory associated with the
+    completion will be freed too early.
+    
+    Fixes: #11478
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fd7723a1e62e682ac5a5279231a9fd6f5682bf94)
+
+commit 094405171bdead0ce09055d4acc6445274992a01
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 10 12:37:05 2015 -0400
+
+    librbd: failure to update the object map should always return success
+    
+    If an object map update fails, the object map will be flagged as
+    invalid.  However, if a subsequent update failure occurs, the error
+    code will propagate back to the caller.
+    
+    Fixes: #11369
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 99f5a7d595c653447b351898192410c9cb773770)
+
+commit 7ee7dcfd609731d3c7f51b74c1d99fb3fa51c413
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 6 15:40:48 2015 -0500
+
+    tests: librados_test_stub reads should deep-copy
+    
+    If a client of librados_test_stub modified a bufferlist
+    retrieved via a read call, the client will actually be
+    changing the contents of the file.  Therefore, read calls
+    should deep-copy the contents of the buffer::ptrs.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 76fe8d73ff79da2d734f70680208a2c188b58671)
+
+commit 113f3b14ec8631d57bbbc2931e242ac96cf9e6fc
+Author: Zhe Zhang <zzxuanyuan@gmail.com>
+Date:   Tue May 5 18:08:48 2015 -0500
+
+    mon: Total size of OSDs is a maginitude less than it is supposed to be.
+    
+    When dumping statistics of OSDs such as running command "ceph osd df",
+    the sum of OSDs' size is 2^10 times less than their real size.
+    
+    Fixes: #11534
+    Signed-off-by: Zhe Zhang <zzxuanyuan@gmail.com>
+    (cherry picked from commit 73d16f69d6f58fe8be262b0fb8db28c94605ea7d)
+
+commit 6a04b55df76faad9b1b0770fbe6038b3b8c7d645
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Mar 17 15:41:33 2015 +0100
+
+    Fix "disk zap" sgdisk invocation
+    
+    Fixes #11143
+    
+    If the metadata on the disk is truly invalid, sgdisk would fail to zero
+    it in one go, because --mbrtogpt apparently tried to operate on the
+    metadata it read before executing --zap-all.
+    
+    Splitting this up into two separate invocations to first zap everything
+    and then clear it properly fixes this issue.
+    
+    Based on patch by Lars Marowsky-Bree <lmb@suse.com> in ceph-deploy.
+    Created by Vincent Untz <vuntz@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b)
+
+commit 8996907e0a777320b505e74754f48a1a82308166
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 15:54:07 2015 +0800
+
+    osd: refuse to write a new erasure coded object with an offset > 0
+    
+    Even if the offset is properly aligned.
+    
+    http://tracker.ceph.com/issues/11507 Fixes: #11507
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a4f1256c214ee0e7ebb91ac4ea8655f5d9642dc8)
+
+commit 58b30d5426998bf7eab4a773f1a04e5bcfbf9b93
+Merge: bc51476 3a58e30
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Wed May 6 16:01:57 2015 +0800
+
+    Merge pull request #4481 from ceph/hammer-11482
+    
+    mds: remove caps from revoking list when caps are voluntarily released
+
+commit bc51476181429d7d95d2bba5f774d8b60c47fb1f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 13 16:33:17 2015 -0700
+
+    ceph_json: add decode / encoder for multimap
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 55594623e2a478c3c023336b924bfdef0017d97f)
+
+commit 7c7e651a0ae8bbcebe136da74b7dbe3a3e9edcc8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Mar 30 17:34:57 2015 -0700
+
+    cls_rgw: use multimap to keep pending operations in bucket index
+    
+    Fixes: #11256
+    Multiple concurrent requests might be sent using the same tag, need the
+    entry map to be able to hold multiple entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4e6a66b55e73c01347fc3330faa5c1307d29e9d3)
+
+commit cb7571375377295d0aff791a03b22da6eb26109d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Mar 27 16:32:48 2015 -0700
+
+    rgw: generate new tag for object when setting object attrs
+    
+    Fixes: #11256
+    Backport: firefly, hammer
+    
+    Beforehand we were reusing the object's tag, which is problematic as
+    this tag is used for bucket index updates, and we might be clobbering a
+    racing update (like object removal).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit edc0627a1dbeb66ea2f5f177f6ceca64559ff3d8)
+
+commit 7387c43382e7f114c43db3cc26ca77d081749d8e
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Fri Mar 27 19:34:12 2015 -0700
+
+    java: libcephfs_jni.so is in /usr/lib64 on rhel
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit aed3434dc7c5161c72c7d5655faa3bc693fc9777)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 5cc0f20ba00080881aff460fab2110cb7eaba279
+Merge: f7bcb2d 0e6a032
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:58:37 2015 +0200
+
+    Merge pull request #4498 from dachary/wip-11342-hammer
+    
+    librbd notification race condition on snap_create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7bcb2dc6043501d478c9d7664bf39f34d5ad6d8
+Merge: 4d95929 2864da8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:58:18 2015 +0200
+
+    Merge pull request #4497 from dachary/wip-7385-hammer
+    
+    Objectcacher setting max object counts too low
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d95929c3129028de2d48efdef71fba2b949edd9
+Merge: 16c2f4c c615972
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:57:48 2015 +0200
+
+    Merge pull request #4499 from dachary/wip-11363-hammer
+    
+    ImageWatcher should cancel in-flight ops on watch error
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 16c2f4c9e0428bda5d784c6c82929f748ec6cb4a
+Merge: addb0ec 0cdc93f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:57:17 2015 +0200
+
+    Merge pull request #4496 from dachary/wip-5488-hammer
+    
+    librbd: deadlock in image refresh
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit addb0ec950e3a3b1150f927f028d2260c1cd0615
+Merge: c0782ed 379ef71
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:31:24 2015 +0200
+
+    Merge pull request #4505 from dachary/wip-11322-hammer
+    
+    rgw - improve performance for large object (multiple chunks) GET
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c0782ed9ab1e59f39d3a30496214971d4cc509d9
+Merge: e074695 2f34d2e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:24:53 2015 +0200
+
+    Merge pull request #4501 from dachary/wip-11001-hammer
+    
+    Improve rgw HEAD request by avoiding read the body of the first chunk
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e074695e003f891e9c52e19a1679809155364d32
+Merge: 7f9d78d c6edc16
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:23:28 2015 +0200
+
+    Merge pull request #4500 from dachary/wip-11047-hammer
+    
+    rgw : make quota/gc thread configurable for starting
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7f9d78d4f12ff3feaee519dd319426650ca0ff88
+Merge: 11b1ccd 4789686
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:46:13 2015 +0200
+
+    Merge pull request #4504 from dachary/wip-10691-hammer
+    
+    ceph-dencoder links to libtcmalloc, and shouldn't
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11b1ccdb6502406456905d05624f0600ef9df893
+Merge: 7c050ec 8709e34
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:44:11 2015 +0200
+
+    Merge pull request #4503 from dachary/wip-10983-hammer
+    
+    use a new disk as journal disk,ceph-disk prepare fail
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7c050ecab2ccc29b03bab2c4ad67c22e9736bb9c
+Merge: c5e0b61 da7f683
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:43:23 2015 +0200
+
+    Merge pull request #4507 from dachary/wip-11432-hammer
+    
+    compilation error: No high-precision counter available (armhf, powerpc..)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c5e0b615d7c98ef700ebe8048ed038f1ff036ff4
+Merge: ee61a61 856b2fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 30 17:10:29 2015 -0700
+
+    Merge pull request #4515 from ceph/hammer-next
+    
+    rgw: critical fixes for hammer
+
+commit ee61a61face479d9895a5cd08ebc8aa93c8bb6ce
+Merge: abc0741 0ee022b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Apr 30 09:23:53 2015 -0700
+
+    Merge pull request #4462 from liewegas/wip-11211-hammer
+    
+    osd/ReplicatedPG: don't check order in finish_proxy_read
+
+commit 856b2fa1fc72916349e484bf3615860392b74100
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 22 15:39:05 2015 -0700
+
+    rgw-admin: a tool to fix object locator issue
+    
+    Objects that start with underscore need to have an object locator,
+    this is due to an old behavior that we need to retain. Some objects
+    might have been created without the locator. This tool creates a new
+    rados object with the appropriate locator.
+    
+    Syntax:
+    
+    $ ./radosgw-admin bucket check --check-head-obj-locator \
+                                   --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit be4355ad8ed622734172fdce77ca71fb2635b36c)
+
+commit 512ae4cb3e182ce79aca7354c66d2f2a662555da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 21 17:31:41 2015 -0700
+
+    rgw: set a special object locator if object starts with underscore
+    
+    Fixes: #11442
+    Backport: hammer
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3d4a1d20b86a5a00556df3d6a8dba096509274b7)
+
+commit da4d2274b5d83a116e767f3063752624d1719c32
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 23 15:33:03 2015 -0700
+
+    rgw: use correct oid when creating gc chains
+    
+    Fixes: #11447
+    Backport: hammer
+    
+    When creating gc chain, use the appropriate oid, otherwise objects will
+    leak.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cc5d34678c6d4bdcd552e6334a383c4df9797f46)
+
+commit 4e84f318de97e592e16493c67491ba0d7f8103a8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 23 17:36:47 2015 +0200
+
+    rgw: civetweb should use unique request id
+    
+    max_req_id was moved to RGWRados and changed to atomic64_t.
+    
+    The same request id resulted in gc giving the same idtag to all objects
+    resulting in a leakage of rados objects. It only kept the last deleted object in
+    it's queue, the previous objects were never freed.
+    
+    Fixes: 10295
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c26225980c2fd018e70033a453d635533fcdefec)
+
+commit abc0741d57f30a39a18106bf03576e980ad89177
+Merge: 74c2dc1 3001fad
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Apr 29 15:02:56 2015 -0700
+
+    Merge pull request #4506 from dachary/wip-11381-hammer
+    
+    messenger: double clear of pipe in reaper
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 478968670caecd67f7995a09b60f6208729e3de3
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:38:58 2015 +0200
+
+    Move ceph-dencoder build to client
+    
+    The patch simply moves the ceph-dencoder build from server part of the
+    Makefiles to client part of the Makefiles.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ef2164725f0b55ffa4b609d407eed5f3f3048b46)
+
+commit 7eabb70b906e50901551ab96453f05fe662a8876
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 15:07:03 2015 +0200
+
+    Rework mds/Makefile.am to support a dencoder client build
+    
+    The patch adds all the mds sources to DENCODER_SOURCES to allow a
+    dencoder client build. The patch also splits the Makefile.am file to
+    better accomodate the change.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c2b3a35fb3cbf28d46a5427f32fbaff142c85f2a)
+    
+    Conflicts:
+            src/mds/Makefile-server.am
+    	src/mds/Makefile.am
+            because the mds/StrayManager.h file was added after hammer
+
+commit da7f6835b15370ce0120a64f7ac3359f3ba4729b
+Author: James Page <james.page@ubuntu.com>
+Date:   Fri Mar 13 19:46:04 2015 +0000
+
+    Add support for PPC architecture, provide fallback
+    
+    Add high precision cpu cycles support for powerpc and powerpc64.
+    
+    Provide a fallback for other architectures and warn during
+    compilation.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit b2781fb5638afae7438b983a912ede126a8c5b85)
+
+commit 3001fad4b6d7e692f6070ef166ed4a3e4849760f
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Fri Apr 17 22:07:00 2015 +0800
+
+    Fix clear_pipe after reaping progress
+    
+    In pipe.cc:1353 we stop this connection and we will let reader and write threads stop. If now reader and writer quit ASAP and we call queue_reap to trigger the reap progress. Now we haven't call "connection_state->clear_pipe(this)" in pipe.cc:1379, so we may assert failure  here.
+    
+    Fixes: #11381
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 0ea0e011a6a6c6d6b40f5d97328bbad0e4568dd7)
+
+commit 379ef714f7149a748891dafd41db80c247d35975
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri Apr 3 12:27:04 2015 +0000
+
+    rgw : Issue AIO for next chunk first before flush the (cached) data.
+    
+    When handling GET request for large object (with multiple chunks), currently it will first flush the
+    cached data, and then issue AIO request for next chunk, this has the potential issue to make the retriving
+    from OSD and sending to client serialized. This patch switch the two operations.
+    
+    Fixes: 11322
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 366e8a85c0e9f00eed364eaebbfb6b672852eae9)
+
+commit b903ad28a68772fa0b7a88b4db2724f4d07565d5
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:33:00 2015 +0200
+
+    rgw/Makefile.am: Populate DENCODER_SOURCES properly
+    
+    Dencoder is built if ENABLE_CLIENT is set. However, the rgw/Makefile.am
+    populated DENCODER_SOURCES only if WITH_RADOSGW was set. The patch fixes
+    this and populates DENCODER_SOURES if ENABLE_CLIENT is set.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 0b264331f57b64880ce05fe3bd752e8df226d00c)
+
+commit f994483c0e3a60226c8fb6983380ef8400b0160e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:32:30 2015 +0200
+
+    Dencoder should never be built with tcmalloc
+    
+    The patch adds disabled perfglue stubs to DENCODER sources in order to
+    avoid tcmalloc-enabled ceph-dencoder builds.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit fb11c74d1dc3843f2f5b6dca9c76278c5ceeca1c)
+
+commit 8709e34f931809f7129cdac1203ec4e774e3eb4c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 8 15:15:35 2015 +0100
+
+    ceph-disk: more robust parted output parser
+    
+    In some cases, depending on the implementation or the operating system,
+    
+       parted --machine -- /dev/sdh print
+    
+    may contain empty lines. The current parsing code is fragile and highly
+    depends on output details. Replace it with code that basically does the
+    same sanity checks (output not empty, existence of units, existence of
+    the dev entry) but handles the entire output instead of checking line by
+    line.
+    
+    http://tracker.ceph.com/issues/10983 Fixes: #10983
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f5acf6bb6a342b05897605703d7d9cb7c09714fd)
+
+commit 8d9f4d4eb546e26eeb3911811bdeb166d06cb1d1
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 11:26:31 2015 +0800
+
+    osdc: add epoch_t last_force_resend in Op/LingerOp.
+    
+    Using this field record the pg_poo_t::last_force_op_resend to avoid op
+    endless when osd reply with redirect.
+    
+    Fixes: #11026
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit def4fc4ae51174ae92ac1fb606427f4f6f00743e)
+
+commit 2f34d2e73bb4bded4779af15a337c75eb2d1497f
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Tue Mar 3 09:46:52 2015 +0000
+
+    rgw: do not pre-fetch data for HEAD requests
+    
+    Backport: hammer
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit e32da3e7c880eaf7cb84d1c078447b28e1d8052b)
+
+commit c6edc16fee027f6cdefacba08f1edc436a7406c5
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Feb 4 03:21:52 2015 +0000
+
+    rgw - make starting quota/gc threads configurable
+    
+    Fixes: 11047
+    Backport: hammer
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit a88712aeb4e7cd4208b9a707aa3bd4d03340c3ff)
+
+commit c6159724f065731c41b2d29a48d0f0a3dc82340b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 20 11:56:55 2015 -0400
+
+    librbd: ImageWatcher should cancel in-flight ops on watch error
+    
+    Upon an watch error notification from librados, pending requests
+    should be canceled and in-flight IO should be flushed prior to
+    unlocking the image.
+    
+    Fixes: #11363
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fc2e511b2aed4d40eff5101a4c9e513b34e5e58e)
+
+commit 0e6a032c9e6ddae20be82df7500a0758d2cd8e74
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 7 15:39:13 2015 -0400
+
+    librbd: moved snap_create header update notification to initiator
+    
+    When handling a proxied snap_create operation, the client which
+    invoked the snap_create should send the header update notification
+    to avoid a possible race condition where snap_create completes but
+    the client doesn't see the new snapshot (since it didn't yet receive
+    the notification).
+    
+    Fixes: #11342
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6e20ed6834912ccd979d16e3f3b340c239e05288)
+
+commit 2864da86ca689472341e8a80f7ad6fcc5eb8321a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 22 11:27:35 2015 -0400
+
+    librbd: updated cache max objects calculation
+    
+    The previous calculation was based upon the image's object size.
+    Since the cache stores smaller bufferheads, the object size is not
+    a good indicator of cache usage and was resulting in objects being
+    evicted from the cache too often.  Instead, base the max number of
+    objects on the memory load required to store the extra metadata
+    for the objects.
+    
+    Fixes: #7385
+    Backport: firefly, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0b378942c4f1b79cb65967f2d3466728ca1c8d5b)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc
+            because hammer has cct->_conf->rbd_cache_size
+            instead of cache_size
+
+commit 0cdc93fbdcf68a31e6aada38b0cb9d66efdc512d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 11:04:22 2015 -0400
+
+    librbd: acquire cache_lock before refreshing parent
+    
+    cache_lock needs to be acquired before snap_lock to avoid
+    the potential for deadlock.
+    
+    Fixes: #5488
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 703ba377e3de4007920f2ed7d8a0780f68676fe2)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+            resolved by moving int r; in the scope of the block
+
+commit a1b4aeb8e8c3a8d5c8284dcee8e03f501a77928c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 13 18:08:47 2015 -0400
+
+    librados_test_stub: AIO operation callbacks should be via Finisher
+    
+    librados will execute all AIO callbacks via a single finisher to
+    prevent blocking the Objecter.  Reproduce this behavior to avoid
+    deadlocks that only exist when using the test stub.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b38c96f2c1747a6d864e7aaa2e9858139ce9d1fd)
+
+commit 3a58e30dc6563197f0effeabbd2fbf804403ad34
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 28 15:45:32 2015 +0800
+
+    mds: remove caps from revoking list when caps are voluntarily released
+    
+    Fixes: #11482
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 86788c4ea4155f0388b7ebaf475a3d3c37d39331)
+
+commit 74c2dc1f3924fa05e2c40f4cceb2ab060493bdfb
+Merge: 8a58d83 f30fa4a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 24 16:13:56 2015 -0700
+
+    Merge pull request #4463 from ceph/wip-11453-hammer-rgw-init-as-root
+    
+    rgw: init-radosgw: run RGW as root
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f30fa4a364602fb9412babf7319140eca4c64995
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Apr 22 16:36:42 2015 -0600
+
+    init-radosgw: run RGW as root
+    
+    The ceph-radosgw service fails to start if the httpd package is not
+    installed. This is because the init.d file attempts to start the RGW
+    process with the "apache" UID. If a user is running civetweb, there is
+    no reason for the httpd or apache2 package to be present on the system.
+    
+    Switch the init scripts to use "root" as is done on Ubuntu.
+    
+    http://tracker.ceph.com/issues/11453 Refs: #11453
+    
+    Reported-by: Vickey Singh <vickey.singh22693@gmail.com>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 47339c5ac352d305e68a58f3d744c3ce0fd3a2ac)
+
+commit 0ee022b1ae832c70a80e9d2cdf32403039f3f125 (refs/remotes/me/wip-11211-hammer)
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Mar 24 16:00:16 2015 +0800
+
+    osd/ReplicatedPG: don't check order in finish_proxy_read
+    
+    Read doesn't need to be ordered. So when proxy read comes back from base
+    tier, it's not necessarily at the front of the in progress list.
+    
+    Fixes: #11211
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 560a5839c0d1852b5816937b845b60390777636c)
+
+commit 8a58d83b0d039d2c2be353fee9c57c4e6181b662
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sun Mar 22 23:59:19 2015 +0800
+
+    Fix ceph_test_async_driver failed
+    
+    This test will create 10000 sockets which will failed because of limited system fd. Actually we only need to create several hundreds sockets and it's enough to get the test's goal.
+    
+    Fix bug #11198(cherry picked from commit cd11daa2d21b7b059df9877cad38432678bb6161)
+
+commit 85a68f9a8237f7e74f44a1d1fbbd6cb4ac50f8e8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Apr 3 13:43:31 2015 +0200
+
+    rgw: remove meta file after deleting bucket
+    The meta file is deleted only if the bucket meta data is not synced
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    
+    Fixes: #11149
+    Backport: hammer, firefly
+    (cherry picked from commit dfdc7afb59cc8e32cf8bff55faa09076c853de06)
+
+commit b01e68fae2f8235b7a813188b664d3ec7bd1fa09
+Merge: addc7e2 bd0ec49
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 14 20:58:40 2015 +0200
+
+    Merge pull request #4332 from dachary/wip-11217-hammer
+    
+    tests: TestFlatIndex.cc races with TestLFNIndex.cc
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit addc7e2c6a4973cbcb5f02635be01a446cb223f0
+Merge: e4bfad3 51f5763
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 14 09:05:53 2015 -0700
+
+    Merge pull request #4357 from dachary/wip-release-notes-hammer
+    
+    release-notes: backport Hammer release notes
+
+commit 51f57632f6d463e5f702bdb4e12c1914ec76d2b5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 14 11:30:43 2015 +0200
+
+    release-notes: backport Hammer release notes
+    
+    Include the Hammer release notes in Hammer.
+    
+    Cherry picking the Hammer release notes cannot be done cleanly, they are
+    copy/pasted instead. This will allow cherry-picking the release notes
+    for the next point releases. It should be undisturbed by the release
+    notes for other point releases because they modify parts of the file
+    that will not generate cherry-pick conflicts.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit bd0ec494739308dc33e6b042ae3e1aafd53c634c
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Tue Mar 24 18:42:04 2015 +0800
+
+    bug fix: test case for lfn index
+    
+    tests: TestFlatIndex.cc races with TestLFNIndex.cc
+    Both use the same PATH and when run in parallel they sometime conflict.
+    
+    Fixes: #11217
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 791c3879d62b848616972f9c6d921aac30ac4925)
diff --git a/doc/changelog/v0.94.3.txt b/doc/changelog/v0.94.3.txt
new file mode 100644
index 000000000..7c0a60d75
--- /dev/null
+++ b/doc/changelog/v0.94.3.txt
@@ -0,0 +1,2660 @@
+commit 95cefea9fd9ab740263bf8bb4796fd864d9afe2b (tag: refs/tags/v0.94.3, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Aug 26 10:39:37 2015 -0700
+
+    0.94.3
+
+commit 697101e4dfd9822050ce401b5f6212bfd81fea89
+Merge: 88e7ee7 81a311a
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Tue Aug 18 12:43:57 2015 +0100
+
+    Merge pull request #5589 from ceph/hammer-12709
+    
+    Workunits : fs/misc/chmod.sh : Include ACL characters in permission check
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 81a311a744987564b70852fdacfd915523c73b5d
+Author: Yazen Ghannam <yazen.ghannam@linaro.org>
+Date:   Mon May 4 12:33:16 2015 -0400
+
+    Workunits : fs/misc/chmod.sh : Include ACL characters in permission check.
+    
+    Signed-off-by: Yazen Ghannam <yazen.ghannam@linaro.org>
+    (cherry picked from commit d3dbfffefb0ae53583350f53258dc902670da659)
+
+commit 88e7ee716fdd7bcf81845087021a677de5a50da8
+Merge: bb12f92 1a32379
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 13:02:17 2015 +0200
+
+    Merge pull request #5160 from theanalyst/wip-11910-hammer
+    
+    mon: pg ls is broken
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bb12f925cf0d78f97eefc2e271e73596050b9919
+Merge: e801d4c e19f928
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 12:34:18 2015 +0200
+
+    Merge pull request #5384 from dachary/wip-12502-hammer
+    
+    rgw: need conversion tool to handle fixes following #11974
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit e801d4c943b8004ef613345505df91057913cd39
+Merge: 78a4024 154f18c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 11:22:08 2015 +0200
+
+    Merge pull request #5405 from ceph/wip-12465-hammer
+    
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 78a4024c14253503b770070aa36a090c6b8f1eaf
+Merge: a451e88 7034720
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 31 15:18:52 2015 -0400
+
+    Merge pull request #5121 from theanalyst/wip-11983-hammer
+    
+    FAILED assert(!old_value.deleted()) in upgrade:giant-x-hammer-distro-basic-multi run
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a451e882ab9a929d240747b4b09786ca4b4ce377
+Merge: 218f537 1063f52
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 31 15:17:00 2015 -0400
+
+    Merge pull request #5269 from dachary/wip-12362-hammer
+    
+    stuck incomplete
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 218f537491a46a0251bef7690a7f5a86b988ee63
+Merge: 07fa83a 8abc46a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:48 2015 +0200
+
+    Merge pull request #5117 from theanalyst/wip-12099-hammer
+    
+    rgw: rados objects wronly deleted
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 07fa83aefc9bd4ea1495fa1e117a438b2c460e46
+Merge: a69d431 56c2688
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:29 2015 +0200
+
+    Merge pull request #5118 from theanalyst/wip-12042-hammer
+    
+    DragonDisk fails to create directories via S3: MissingContentLength
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit a69d431ea52b5de7fc5cfe142ff3b69ff64e8048
+Merge: 5353480 c78cc00
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:14 2015 +0200
+
+    Merge pull request #5214 from SUSE/wip-12299-hammer
+    
+    RGW Swift API: support for 202 Accepted response code on container creation
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5353480f1df297ad9dd1bd3154887ed2564f0280
+Merge: fb9156f a5dbcbb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:57 2015 +0200
+
+    Merge pull request #5226 from SUSE/wip-12322-hammer
+    
+    rgw: keystone does not support chunked input
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit fb9156f412f83c0ce1fe4c5c9ccd57fd79f0c992
+Merge: 7193c16 b1618a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:41 2015 +0200
+
+    Merge pull request #5227 from SUSE/wip-12323-hammer
+    
+    RGW Swift API: XML document generated in response for GET on account does not contain account name
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7193c16b65fdc1694b968899d23eae0638d89f11
+Merge: ac86490 e39dce7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:26 2015 +0200
+
+    Merge pull request #5228 from theanalyst/wip-11872-hammer
+    
+    RGW does not send Date HTTP header when civetweb frontend is used
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ac86490821336ce024940d48d82f7a5ff7a302b1
+Merge: 33dbfc6 557865c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:02 2015 +0200
+
+    Merge pull request #5229 from theanalyst/wip-12242-hammer
+    
+    Fix tool for #11442 does not correctly fix objects created via multipart uploads
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 33dbfc6919840882c6cbc10dad2fc24cf0720bf9
+Merge: 99ca62f e50caab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:41 2015 +0200
+
+    Merge pull request #5237 from theanalyst/wip-12245-hammer
+    
+    rgw: empty json response when getting user quota
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 99ca62f2bf1e21a41cb7b6ecdb8a8731a18de195
+Merge: 1f5f319 2357b6c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:18 2015 +0200
+
+    Merge pull request #5284 from SUSE/wip-12398-hammer
+    
+    rgw: Properly respond to the Connection header with Civetweb
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 1f5f31905bb5f499a2db4a02993dbc6efa1c4251
+Merge: 5cbb6cf 9458b84
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:04 2015 +0200
+
+    Merge pull request #5285 from SUSE/wip-12399-hammer
+    
+    rgw: multipart list part response returns incorrect field
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5cbb6cfb69aad0db470f99e39e33f4b4b1abfb95
+Merge: 1df93e1 e4b55b3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:52:43 2015 +0200
+
+    Merge pull request #5286 from SUSE/wip-12400-hammer
+    
+    rgw: radosgw-admin dumps user info twice
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 1df93e19a0275ed218c8f83bc674f16d1856f241
+Merge: a48cbc0 2ecb3b7
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Fri Jul 31 09:50:22 2015 +0800
+
+    Merge pull request #5427 from dachary/wip-12088-hammer-part-2
+    
+    Fh ref count will leak if readahead does not need to do read from osd
+
+commit 2ecb3b7f4a49c574bc178a106c6bf0d8247f2a5e
+Author: Zhi Zhang <zhangz.david@outlook.com>
+Date:   Wed Jul 22 10:54:53 2015 +0800
+
+    Fh ref count will leak if readahead does not need to do read from osd
+    
+    The 3c8cdeacf46ae4031189d2ef6948aa3b6ab4ae43 backport introduced a leak.
+    
+    http://tracker.ceph.com/issues/12319 Fixes: #12319
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+
+commit a48cbc0a847f19ea613b76a479acc831e9316c62
+Merge: 06c27cd 5ef0846
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 21:43:48 2015 +0200
+
+    Merge pull request #5120 from theanalyst/wip-11999-hammer
+    
+    cephfs Dumper tries to load whole journal into memory at once
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 06c27cdd420598c497766ee5879335942a0acc09
+Merge: 19abe5e 408880b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 21:43:21 2015 +0200
+
+    Merge pull request #5119 from theanalyst/wip-12098-hammer
+    
+    kernel_untar_build fails on EL7
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 19abe5ee35c099c67b56ac268710fcd20bec60d3
+Merge: e3d17e4 4c199bf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 17:00:14 2015 +0200
+
+    Merge pull request #5417 from dachary/wip-11998-hammer
+    
+    debian/control: ceph-common (>> 0.94.2) must be >= 0.94.2-2
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4c199bf57dc54dc5e5f45cd9b34878a8459d434e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jul 30 09:43:20 2015 +0200
+
+    debian/control: ceph-common (>> 0.94.2) must be >= 0.94.2-2
+    
+    The d8733be2ef8874b9a858a7ffddfb81b9b656e9a6 backport introduced a
+    regression by adding an incorrect Depends / Break combo supposed to
+    reflect the fact that ceph_argparse moved from ceph to ceph-common after
+    v0.94.2. It assumed the package is released under the 0.94.2 version
+    where in reality it is released under the 0.94.2-1xxx version (where xxx
+    is trusty, jessie etc.).
+    
+    The Depends / Break combo is changed to use 0.94.2-2 instead.
+    
+    See also http://tracker.ceph.com/issues/12529 for a larger discussion.
+    
+    http://tracker.ceph.com/issues/11998 Fixes: #11998
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit e3d17e49731569ea92917f574d42d93258c77189
+Merge: cbba706 89aa8ff
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 29 16:22:36 2015 +0200
+
+    Merge pull request #5248 from ceph/wip-11833-hammer
+    
+    mon: add an "osd crush tree" command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit cbba7064c6cc4cde3e8a49c25ce671e91d31b9c7
+Merge: 8355bda 3c8cdea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 29 10:49:08 2015 +0200
+
+    Merge pull request #5222 from ceph/hammer-12088
+    
+    client: reference counting 'struct Fh'
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8355bdab56bc4e5ce4d20ba3486c082f06d8dcd1
+Merge: 52d0e5d ec70533
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Jul 28 23:33:18 2015 +0200
+
+    Merge pull request #5231 from theanalyst/wip-12243-hammer
+    
+    Civetweb RGW appears to report full size of object as downloaded when only partially downloaded
+
+commit 52d0e5da5ebad7fe42c2e469cea9773c7714c2b5
+Merge: 7fd31b1 03c07d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:40:23 2015 +0200
+
+    Merge pull request #5243 from theanalyst/wip-12239-hammer
+    
+    librbd/internal.cc: 1967: FAILED assert(watchers.size() == 1)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7fd31b1b3c2c8e9dd3d9e5464775422215f7a4bc
+Merge: 7230de3 5c812c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:40:03 2015 +0200
+
+    Merge pull request #5241 from theanalyst/wip-12238-hammer
+    
+    [  FAILED  ] TestLibRBD.ExclusiveLockTransition
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7230de317736a71a5764cf224bd1309da1c7b3c6
+Merge: 6b6228f 7132277
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:30:23 2015 +0200
+
+    Merge pull request #5265 from SUSE/wip-12368-hammer
+    
+    linking ceph to tcmalloc causes segfault on SUSE SLE11-SP3
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6b6228f8949e975cac763513898ea9704cb8baf1
+Merge: d62c3ea f99f312
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:27:40 2015 +0200
+
+    Merge pull request #5280 from ceph/wip-12384-hammer
+    
+    librbd: add valgrind memory checks for unit tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d62c3ea344d9e49e9586867e872e8d5b3f019948
+Merge: 7b57ff8 b872882
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:26:25 2015 +0200
+
+    Merge pull request #5279 from ceph/wip-12237-hammer
+    
+    A client opening an image mid-resize can result in the object map being invalidated
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7b57ff8a9ced6c2f22456ed034cc83d07f82fbb3
+Merge: 481728a f819332
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:10:03 2015 +0200
+
+    Merge pull request #5283 from SUSE/wip-12397-hammer
+    
+    ceph.spec.in: 95-ceph-osd.rules, mount.ceph, and mount.fuse.ceph not installed properly on SUSE
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 481728a04dd2c85096c3bc01cc37da9642b038ca
+Merge: 54bb924 d8733be
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:54:33 2015 +0200
+
+    Merge pull request #5206 from SUSE/wip-11998-hammer
+    
+    /usr/bin/ceph from ceph-common is broken without installing ceph
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 54bb924e68ae2b4df65576a5d788d593b9d9e722
+Merge: e099058 c5c627f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:47:29 2015 +0200
+
+    Merge pull request #5055 from SUSE/wip-12044-hammer
+    
+    rgw/logrotate.conf calls service with wrong init script name
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e0990583298277f1c631f7c2d2260d6c3fa64c9f
+Merge: 8b93978 e149916
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:46:11 2015 +0200
+
+    Merge pull request #5040 from SUSE/wip-11964-hammer
+    
+    systemd: Increase max files open limit for OSD daemon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b93978881375d063fe2df8f40406ea650dda766
+Merge: 5a7cab2 22f58ce
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:45:44 2015 +0200
+
+    Merge pull request #5038 from SUSE/wip-11876-hammer
+    
+    ceph-post-file fails on rhel7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5a7cab205bb1b3fdbf49a852cb978fc28eba8212
+Merge: 5218eff 38d36b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:45:25 2015 +0200
+
+    Merge pull request #5030 from SUSE/wip-12092-hammer
+    
+    packaging: add SuSEfirewall2 service files
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5218eff07c303fb2762ea9f38b9a9c23c24efcae
+Merge: 0b54d50 8acfb99
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:38:14 2015 +0200
+
+    Merge pull request #5028 from SUSE/wip-12090-hammer
+    
+    rcceph script is buggy
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0b54d50ecd1445dfc1a46552adb83b9dae9210d9
+Merge: 45beb86 37d77d3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:37:44 2015 +0200
+
+    Merge pull request #5026 from SUSE/wip-12087-hammer
+    
+    max files open limit for OSD daemon is too low
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e19f928bd770a37f2f631c4cd796e2e30a494234
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 26 16:56:28 2015 -0700
+
+    rgw: conversion tool to fix broken multipart objects
+    
+    Fixes: #12079
+    
+    Broken multipart objects: multipart objects that created on 0.94.2
+    and that start with underscore have a bad locator on their tail objects.
+    This extends the tool that was needed for older issue we've had with
+    hammer multipart objects (that start with underscore). The same usage
+    applies:
+    
+    $ radosgw-admin bucket check --check-head-obj-locator \
+                                 --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f02ca6107172cecd80a490df9f0d66204e62326c)
+
+commit 28d32f6090724d62b6168d64031454f44eb4cc88
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 26 13:49:55 2015 -0700
+
+    rgw: only scan for objects not in namespace
+    
+    Fixes: #11984
+    The tool should only work on the head objects, and these are not inside
+    any namespace.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8103908548bf7d6c9fa47fb181cd450670bae8d6)
+
+commit e22e2b43b4039a44f5f8fbbe59edc21fbe118bdc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 22 16:04:35 2015 -0700
+
+    rgw_admin: add --remove-bad flag to bucket check
+    
+    Add this flag so that the bad object will be removed (should be called
+    only after user has verified that objects content is correct).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 06d67d9139a95b704b80de527381fd1bbf7981ce)
+
+commit 154f18ce3e52094fe84b058565a865ed97b079d6 (refs/remotes/gh/wip-12465-hammer)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jul 24 15:38:18 2015 -0700
+
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Otherwise, _flush() might continue to write to m_fd after it's closed.
+    This might cause log data to go to a data object if the filestore then
+    reuses the fd during that time.
+    
+    Fixes: #12465
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8778ab3a1ced7fab07662248af0c773df759653d)
+
+commit b8728823493b9dfde0333fb41725002fc50e4d9b (refs/remotes/gh/wip-12237-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 10:47:38 2015 -0400
+
+    librados_test_stub: read op should return number of bytes read
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f8a7b507983e31399831e802e99429b95386ed41)
+
+commit 7d9fce3aa3832a1b8bd7f18abd4745dbc0033582
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 10:35:28 2015 -0400
+
+    tests: fixed TestObjectMap.InvalidateFlagInMemoryOnly
+    
+    librados and librados_test_stub return different result codes
+    for a read full object operation.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ace2b77f8ed83e753fe4a48bcc997f5d1dd465f)
+
+commit 4a77be0a65c8b4ec3dc437721f8c321737b260de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 11:09:09 2015 -0400
+
+    librbd: don't attempt to invalidate an object map in R/O mode
+    
+    The ImageWatcher is not initialized when in R/O mode, which
+    resulted in a NULL pointer dereference.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 64d740f8fa10ba872e324ec2580a4d8c3f99a9ce)
+
+commit 0aea70f68b299441e692efdce6d5e7ff18b78c39
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 11:17:12 2015 -0400
+
+    tests: add new unit tests for object map invalidation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0215e9753c09460f6fc84ded9397e36a209f2e32)
+
+commit c732cb889b4a61254d06703bf032082e56b196de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 25 09:41:13 2015 -0400
+
+    librbd: move object map codes to common location
+    
+    These codes will need to be accessible from cls_rbd and librbd.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4ac584c34d576b489ed4c4862703b8fb427b3bc2)
+
+commit 27c99ea972a7b218ea591b208d0d1dd51eef6f95
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 11:14:51 2015 -0400
+
+    librbd: only update image flags when holding exclusive lock
+    
+    It was possible for a client to open an image while another client
+    was shrinking an image.  This would result in the former invalidating
+    the object map on-disk if it openned the image between updating the
+    image header and resizing the object map.
+    
+    Fixes: #11791
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eb81a6a7e391327ac993fd406443b206a7f7bffc)
+
+commit ef453630200ab72373f08357ca6b5ac5c5bbb397
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 17 12:43:46 2015 -0400
+
+    librbd: new ImageWatcher::is_lock_supported method
+    
+    The new version does not attempt to acquire the snap_lock, to avoid
+    cases where a recursive lock would result.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e4b55b398e68e870a7cf21276e63da2c4c6e3faa
+Author: guce <guce@h3c.com>
+Date:   Sat Jul 11 14:08:33 2015 +0800
+
+    Fixes: #12286 radosgw-admin: after subuser modify print only once user info.
+    
+    remove rgw_admin.cc OPT_SUBUSER_MODIFY, show_user_info code block.
+    
+    switch (opt_cmd) {
+    ...
+    case OPT_SUBUSER_MODIFY:
+    show_user_info(info, formatter);       //show first time (remove this)
+    break;
+    ...
+    }
+    
+    // output the result of a user operation
+    if (output_user_info) {
+    ...
+    show_user_info(info, formatter);       //show second time
+    }
+    
+    test fix:
+    before: after subuser modify print twice user info.
+    after changes, do the same procedure, print only once user info.
+    
+    Signed-off-by: guce guce@h3c.com
+    (cherry picked from commit c604dd97fc179e5c2f640818c0f6e7cf99701947)
+
+commit 9458b845bf863ccf878873c4f0b089ddf84c7203
+Author: Henry Chang <henry@bigtera.com>
+Date:   Wed Apr 22 18:26:45 2015 +0800
+
+    rgw: fix ListParts response
+    
+    The response XML element name should be 'ListPartsResult'.
+    
+    Fixes: #11494
+    
+    Signed-off-by: Henry Chang <henry@bigtera.com>
+    (cherry picked from commit caa9f0e461f1eed526fc43ee74699a7243aef9b8)
+
+commit 2357b6c808f4f7c5997af48149585a6051c04b8f
+Author: Wido den Hollander <wido@42on.com>
+Date:   Sat Jul 11 00:01:52 2015 +0200
+
+    rgw: If the client sends a Connection: close header respond accordingly.
+    
+    HTTP/1.1 assumes Keep-Alive by default, but if a Connection: close header is send
+    the server should respond with it as well.
+    
+    This makes the client close the connection after the request.
+    
+    Fixes: #12298
+    (cherry picked from commit 79197d3711edc4b04a7ea4335b6e1b65754996d5)
+
+commit f819332e2826eae14849c5e68a380d1d87039d22
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 21:38:46 2015 +0200
+
+    ceph.spec.in: install 95-ceph-osd.rules, mount.ceph, and mount.fuse.ceph properly on SUSE
+    
+    http://tracker.ceph.com/issues/12261 Fixes: #12261
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 5ce38b9536efabf99a236c7a9d15c149fa4c16a6)
+
+commit d8733be2ef8874b9a858a7ffddfb81b9b656e9a6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Apr 14 07:58:17 2015 -0600
+
+    debian: move ceph_argparse into ceph-common
+    
+    Prior to this commit, if a user installed the "ceph-common" Debian
+    package without installing "ceph", then /usr/bin/ceph would crash
+    because it was missing the ceph_argparse library.
+    
+    Ship the ceph_argparse library in "ceph-common" instead of "ceph". (This
+    was the intention of the original commit that moved argparse to "ceph",
+    2a23eac54957e596d99985bb9e187a668251a9ec)
+    
+    http://tracker.ceph.com/issues/11388 Refs: #11388
+    
+    Reported-by: Jens Rosenboom <j.rosenboom@x-ion.de>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 110608e5bdd9e2f03020ad41f0c2d756684d4417)
+    
+    Conflicts:
+    	debian/ceph.install
+                There is no ceph_daemon.py in hammer
+            debian/control
+                Depends/Replaces/Breaks version adapted (from 9.0.0 to 0.94.2)
+                also adapted ceph-dbg Replaces/Breaks
+
+commit f99f3125ff76628e2525dca00bb7b983f941a08b (refs/remotes/gh/wip-12384-hammer)
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Fri Mar 20 16:15:42 2015 +0800
+
+    test: potential memory leak in FlushAioPP
+    
+    Should call the release function instead of deleting it to free
+    librbd::RBD::AioCompletion and librbd::AioCompletion. Otherwise there is
+    a potential memory leak.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit ada7ec860cb7901c560c12a5af36dc7c23051b76)
+
+commit a4fc63af630e77586e3ba2f17df3b6be4a1e2055
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 15:25:49 2015 -0400
+
+    pybind: fix valgrind warning on rbd_get_parent_info call
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2586e3ba1e20603a87c833513e09dae9281beb4d)
+
+commit aa3eb28f6be62991bc790de5c19cb7b6e30fa189
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 11:12:00 2015 -0400
+
+    osdc: invalid read of freed memory
+    
+    The bytes not in cache stat was potentially reading the bh length
+    from a deleted bufferhead.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5ccc4422d6172376bd6f1be8d3a99c0a54eab807)
+
+commit 18ede754388372cf210d7db87fa46f3536cf0e44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 10:56:15 2015 -0400
+
+    krbd: fix incorrect types in the krbd API
+    
+    The C API functions were referencing the C++ CephContext
+    instead of the C rados_config_t.  Additionally, the ceph
+    namespace was missing on the Formatter class.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 740fd275a60630e60b3bcf41637a2ca486885d9c)
+
+commit 488578c1d557ebec7e50d53e45ed46f42984f4f8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 10:54:47 2015 -0400
+
+    fsx: cleanup crypto library at exit
+    
+    Also made small tweaks so that it can be compiled under
+    a C++ compiler.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c44f8e7fbc19924a9453d8c032c624ebb6c0296f)
+
+commit 97ff6cb2f8fdd4d946eeab338ec225450e3ad8f3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 14:29:59 2015 -0400
+
+    tests: add run-rbd-valgrind-unit-tests.sh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5534faaa469b8a6a4c9687aad1a6723f3e859353)
+
+commit e690907cbb3b229f84f1e996d58636d00f823e8f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 00:23:03 2015 -0400
+
+    valgrind: update valgrind suppressions for lttng-ust
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d87bdf597aad3d6be47aedd216a673bd9093a24)
+
+commit fe013e0a813c5697e917da642143388de60e8528
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 00:21:15 2015 -0400
+
+    librbd: TaskFinisher should finish all queued tasks
+    
+    The destructor wasn't waiting for all Finisher tasks
+    to complete before stopping the thread.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8e20240e4155e2f0398e79f4c0095d2d6ba1d4cb)
+
+commit 43cd3ac923c9accfb81acf41f5bd12b8a05322c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:10:23 2015 -0400
+
+    tests: fix valgrind errors with librbd unit test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ed5472a10eb515e2a177a640c3f6ed929db9ee4f)
+
+commit 5d8d6a1a776f833847edc80d2a9b31ecb440ade5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:09:45 2015 -0400
+
+    tests: librbd should release global data before exit
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6ab1bb5614a5d257a82cf8ea280eef5c90cf765b)
+
+commit 13f926e4e96d0b7178a9762bbbf589961dba47b7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:08:51 2015 -0400
+
+    librados_test_stub: cleanup singleton memory allocation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 54c88255b74741d882b88f791497862635357634)
+
+commit 45beb86423c3bd74dbafd36c6822e71ad9680e17
+Merge: 5e399b0 582cf73
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 17 19:48:05 2015 +0200
+
+    Merge pull request #5046 from ceph/wip-12109-hammer
+    
+    librbd: new QA client upgrade tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1063f5275d1031812d564a1bd8ada64bed561026
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 20 12:08:15 2015 -0700
+
+    PG::find_best_info: ignore info.les for incomplete peer
+    
+    See included update to doc/dev/osd_internals/last_epoch_started.rst
+    
+    Fixes: 11687
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 371d9baa120dc0302e9e61d3bc0e309dfaa773a0)
+
+commit 5e399b035d7cf861cf66a8ead00b388c4857cbb6
+Merge: 706b1c7 ad5745b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 16 14:58:49 2015 -0700
+
+    Merge pull request #5159 from theanalyst/wip-11701-hammer
+    
+    make the all osd/filestore thread pool suicide timeouts separately configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 713227791ab28c5e09073acb7b2c3c83ca0f0d6a
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Mon Mar 16 00:13:38 2015 +0100
+
+    Conditional-compile against minimal tcmalloc.
+    
+    Certain older systems (SLE11 in this case) do not provide the full
+    tcmalloc functionality, due to e.g. incomplete libunwind
+    pieces. Use --with-tcmalloc-minimal to enable the cut-down
+    version.
+    
+    Here's how the various mem allocator switches interact now:
+    
+    --with-jemalloc: overrides --with-tcmalloc & --with-tcmalloc-minimal
+    --with-tcmalloc-minimal: overrides --with-tcmalloc
+    --with-tcmalloc: the default. use --without-tcmalloc to disable
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit c6f1c07113ca19547fdac10cd9b817a60142aee2)
+
+commit 706b1c7c5bcaaff96aa6950302b7aef097918d30
+Merge: daf5450 5e72479
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 16 11:04:52 2015 -0400
+
+    Merge pull request #5252 from ceph/wip-12021-hammer
+    
+    OSDMonitor: allow addition of cache pool with non-empty snaps with co…
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit daf5450765684b0b2ed049320d7463b637321e5a
+Merge: d20f513 bd91fb0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 16 01:15:51 2015 +0200
+
+    Merge pull request #4891 from theanalyst/wip-11740-hammer
+    
+    crush: take crashes due to invalid arg
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad5745bfd768b52ae6a766391232becad8587641
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:49:00 2015 -0700
+
+    OSD: add command_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit df4e5de819c30003cfbe50a071c49039cf534419)
+    
+    Conflicts:
+    	src/common/config_opts.h
+    Trivial merge conflict
+
+commit 059a579c02b312bbd32fa41485c361ae3847a3ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:54:31 2015 -0700
+
+    OSD: add remove_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f2fbfa32a16666be46359f0eab7b04ca80a753f5)
+
+commit b8826bc4e3da6fcb9338ad6c01af1a88e6585a4d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:52:40 2015 -0700
+
+    OSD: add scrub_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 547a7041edc833f3cc8e04d388574809e30a8af6)
+
+commit 878dd403930a2058656a99c14b465358e134843c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:51:28 2015 -0700
+
+    OSD: add snap_trim_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit e1073a4a577211672148a4112bd633831552d66f)
+
+commit 11575832a37ea247a8febe912b3058f51a464ab6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:50:19 2015 -0700
+
+    OSD: add recovery_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 85311b656852af75bfbbc6699f92fc6aa233c316)
+    
+    Conflicts: src/common/config_opts.h
+    
+    There was a merge conflict due to introduction of `osd_recovery_sleep`
+    which was introduced in #3829
+
+commit a82b4505848c09ad0094768c886f2015bdaa1148
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 11:02:19 2015 -0700
+
+    OSD: add op_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 89aa8ff9855ae868d59bd10fe3a3aab8517e90fc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:52:22 2015 +0800
+
+    mon: add an "osd crush tree" command
+    
+    * to print crush buckets/items in a tree
+    
+    Fixes: #11833
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5436c290f3622feb8d4b279ed6552b2510e0cee9)
+    
+    Conflicts:
+    	src/test/mon/osd-crush.sh:
+    		do not start mon as run() takes care of it already
+
+commit d20f513d9b185eff82bee2ca719b5453358e740b
+Merge: 8753b2b 3d74164
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 20:43:02 2015 +0200
+
+    Merge pull request #4899 from theanalyst/wip-11911-hammer
+    
+    start_flush: filter out removed snaps before determining snapc's
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8753b2b14536c34a7b6dec927c7a5b8100de7f68
+Merge: 3d72652 ecac1a4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 20:42:45 2015 +0200
+
+    Merge pull request #4868 from SUSE/wip-11879-hammer
+    
+    Clock skew causes missing summary and confuses Calamari
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3d72652d7ba6b2fff3d39ea7965c3c61d5fa0a04
+Merge: 9a79e8e fdb43eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 16:42:12 2015 +0200
+
+    Merge pull request #4883 from SUSE/wip-11638-hammer
+    
+    ceph.spec.in: ceph-common subpackage def needs tweaking for SUSE/openSUSE
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 03c07d76ac8361ddd302f5bc0575aee7fb5edc99
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 26 09:59:36 2015 -0400
+
+    librbd: assertion failure race condition if watch disconnected
+    
+    It's possible for librbd's watch of the header object to be reset by
+    connection issues just prior to the image being removed.  This will
+    causes an assertion failure which assumes at least one watcher on the
+    image.
+    
+    Fixes: #12176
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af276de4f299960e43761904c043924cec5fef11)
+
+commit 5c812c1552d954f2c91c000332ddc74c9e91825a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 25 16:51:31 2015 -0400
+
+    librbd: prevent object map updates from being interrupted
+    
+    Object map updates were being canceled in-flight when the exclusive lock
+    is released.  This resulted in an ERESTART error code bubbling up to
+    AioRequest.
+    
+    Fixes: 12165
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 590cdc90edaf4f4ff06c97eb2f43b92ab9b60084)
+    
+    Conflicts:
+    	src/librbd/ObjectMap.h
+    conflict due to a variable `m_snap_id' which was introduced in PR #4140
+    which is dropped as we are not backporting that feature
+
+commit e50caab2251bb68fea1adbd17acc43aa98ab1206
+Author: wuxingyi <wuxingyi@letv.com>
+Date:   Tue Jun 23 01:46:48 2015 +0000
+
+    rgw: fix empty json response when getting user quota
+    
+    Fixes: #12117
+    Signed-off-by: wuxingyi <wuxingyi@letv.com>
+    (cherry picked from commit 64fceed2202c94edf28b8315fe14c9affa8c0116)
+
+commit ec705336551436517c16bffdc6bf5467899ae4bb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 25 14:31:03 2015 -0700
+
+    rgw: error out if frontend did not send all data
+    
+    Fixes: #11851
+    The civetweb mg_write() doesn't return error when it can't flush all data
+    to the user, it just sends the total number of bytes written. Modified the
+    client io to return total number of bytes and return an error if didn't
+    send anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit daa679c3dd3770a6d5421e2cc9a36924f4550439)
+
+commit 557865c85bb907fe69248c4f1acb88320a7c1bb5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 17 15:11:28 2015 -0700
+
+    rgw: fix reset_loc()
+    
+    Fixes: #11974
+    
+    Only need to set locator for underscore if namespace is empty
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d3bd27f4855df6bb207b656527138026af1a36a2)
+
+commit b1618a97fef644dc3dced502d600de6a5d55d085
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Apr 20 14:55:00 2015 +0200
+
+    rgw: fix lack of account name in XML listing of Swift account.
+    
+    Fixes: #11431
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 837388bbc39a1bf9019302c3a4d3a3fe22caeeb4)
+
+commit e39dce7935dd513b77ce34bc79d70a2c23437cbb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 18 15:48:43 2015 +0100
+
+    rgw: generate the "Date" HTTP header for civetweb.
+    
+    Fixes: #10873
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ea384f83b601f60e135c3d3f960fdb75a919dd84)
+
+commit a5dbcbbdddce6cdeccb1e6f5641601d673cd1896
+Author: Hervé Rousseau <hroussea@cern.ch>
+Date:   Mon Apr 27 17:54:30 2015 +0200
+
+    Swift: Set Content-Length when requesting/checking Keystone tokens
+    
+    Running Keystone with WSGIChunkedRequest=On is not supported.
+    
+    We have to make sure that we set the Content-Length header when getting
+    an admin token and checking revoked tokens, otherwise Keystone returns
+    a HTTP 411 error.
+    
+    Same applies when checking revoked tickets.
+    
+    Fixes: #11473
+    Backport: Hammer, Firefly
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 24f477417fdac9d68902fa211c8edf92a2e8729f)
+
+commit 3c8cdeacf46ae4031189d2ef6948aa3b6ab4ae43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jul 8 10:11:43 2015 +0800
+
+    client: reference counting 'struct Fh'
+    
+    The async readahead finisher needs to reference 'struct Fh'. But
+    it's possible user closes FD and free the corresponding 'struct Fh'
+    before async readahead finishes.
+    
+    Fixes: #12088
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 34b939a81d38173b882c429b28dedce778504ba8)
+
+commit c78cc00afb6deb8022db60dbe8649335f61bd345
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Mar 19 14:52:18 2015 +0100
+
+    rgw: rectify 202 Accepted in response for PUT on existing bucket.
+    
+    Fixes: #11148
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 3998fe7e02a6c25a3302c80a9c9907357fd3a23e)
+
+commit 9a79e8e7da5f34f1adaf6137e01bcd42766ae677
+Merge: 5527720 7f1c0cc
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sun Jul 12 02:19:57 2015 +0800
+
+    Merge pull request #5208 from tchaikov/wip-11975-hammer
+    
+    tests: TEST_crush_reject_empty must not run a mon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7f1c0cc9cd3deab925440b56d82c3e61a8ba5ab1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jul 11 23:04:33 2015 +0800
+
+    crush/CrushTester: return EINVAL if crushtool returns non-zero
+    
+    this backports a tiny part of ec02441, otherwise
+    CrushTester will return 1, and "ceph" cli will take it
+    as EPERM, which is miss leading, and fails
+    osd-crush.sh:TEST_crush_reject_empty.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2aaeea145b24b972a0b98549c3527ccf98f4c96f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jul 10 16:23:47 2015 +0200
+
+    tests: TEST_crush_reject_empty must not run a mon
+    
+    * Back in Hammer, the osd-crush.sh individual tests did not run the
+      monitor, it was taken care of by the run() function. An attempt to run
+      another mon fails with:
+    
+      error: IO lock testdir/osd-crush/a/store.db/LOCK: Resource temporarily
+      unavailable
+    
+      This problem was introduced by cc1cc033930e8690a57674e842a003f6bbc7a242
+      from https://github.com/ceph/ceph/pull/4936
+    * replace test/mon/mon-test-helpers.sh with test/ceph-helpers.sh as
+      we need run_osd() in this newly added test
+    * update the run-dir of commands: ceph-helpers.sh use the different
+      convention for the run-dir of daemons.
+    
+    http://tracker.ceph.com/issues/11975 Refs: #11975
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 80afb81124a0d2ef25a23a12c86617ab1da3a4bd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 21 16:11:33 2015 +0200
+
+    ceph-helpers: implement test_expect_failure
+    
+    To display the output in case the command did not fail with the expected
+    output.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 5871781b10ff0b26c731b70d1898c474006cbee3)
+
+commit 6b5e9a1df7dfb3a971e40aec35119ec019515b69
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jun 10 23:16:01 2015 +0200
+
+    tests: display the output of failed make check runs
+    
+    After a make check fails, it shows a summary but not the output of the
+    failed tests although they contain information to diagnose the problem.
+    
+    Set the VERBOSE=true automake variable which is documented to collect
+    and display the failed script output at the end of a run (the content of
+    the test-suite.log file (valid from automake-1.11 up).
+    
+    http://www.gnu.org/software/automake/manual/automake.html#index-VERBOSE
+    
+    Also remove the run-make-check.sh that did the same in a way that is not
+    compatible with automake-1.11.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 3a55cb029bb7db9542d2b14f2deda90feb0ae0f6)
+
+commit 552772025cb8d5f51ffb3a069d1bd93bc73f1123
+Merge: f4d77c2 1440122
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:17:24 2015 +0200
+
+    Merge pull request #4889 from theanalyst/wip-11484-hammer
+    
+    OPT_INT option interprets 3221225472 as -1073741824, and crashes in Throttle::Throttle()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f4d77c22aa51edb45211e080f3fdf28a7a0cfdd4
+Merge: 5088105 a62c3aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:16:25 2015 +0200
+
+    Merge pull request #4776 from tchaikov/wip-11279-hammer
+    
+    ceph: cli interactive mode does not understand quotes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5088105300c013b1b804c938a30ac63ba710556d
+Merge: e3b1f7b 0b6d442
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:14:42 2015 +0200
+
+    Merge pull request #4657 from ceph/wip-hammer-11535-admin-socket
+    
+    common/admin_socket: close socket descriptor in destructor
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e3b1f7be9e8474fbec98076790ff683bccd44ce9
+Merge: dd29a86 558d639
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 16:07:48 2015 +0800
+
+    Merge pull request #4687 from SUSE/wip-7387-hammer
+    
+    utf8 and old gcc breakage on RHEL6.5
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dd29a869db5503fc9e2c6d1d44ee4311d95af20c
+Merge: 7f1fb57 0e5e7e1
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 16:00:00 2015 +0800
+
+    Merge pull request #5122 from theanalyst/wip-11982-hammer
+    
+    ceph fails to compile with boost 1.58
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7f1fb574608800c3e6aa12df6c7888acbf397a52
+Merge: adc7016 5141301
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 15:59:35 2015 +0800
+
+    Merge pull request #4936 from ceph/wip-11975-hammer
+    
+    mon crashes when "ceph osd tree 85 --format json"
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit adc70161d14fc2b51e6c6f38580f76ff0067717a
+Merge: 2d68db8 82988d6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:48:44 2015 +0200
+
+    Merge pull request #4892 from theanalyst/wip-11760-hammer
+    
+    ceph-disk: get_partition_type fails on /dev/cciss...
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2d68db8371263645642cf28473deea4456ca7021
+Merge: 1cffe8c ba1a016
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:43:57 2015 +0200
+
+    Merge pull request #4877 from SUSE/wip-11902-hammer
+    
+    admin/build-doc: script fails silently under certain circumstances
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 51413011417b76f5ad2830d9f93fbfe78c77e467
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 18:11:59 2015 +0800
+
+    mon: add "--check" to CrushTester::test_with_crushtool()
+    
+    so we don't need to call CrushTester::check_name_maps() in OSDMonitor.cc
+    anymore.
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c6e634875316cf17368d497e6dc4f6f4b5dd65d2)
+
+commit 5ec27cf589b4535f07e28a86bd304f7a46427ac4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 17:51:50 2015 +0800
+
+    crushtool: rename "--check-names" to "--check"
+    
+    * because "--check" also checks for the max_id
+    
+    Note: edited since we do not have the fix introduced in 46103b2 in
+          hammer.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9381d53acdce85fcbff828926b911e050ba36e51)
+
+commit 2a8fe8862a15342cc5716c146487d0b42af0fbf6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 16:58:23 2015 +0800
+
+    mon: check the new crush map against osdmap.max_osd
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 22e6bd6e01d5df3f3e897562597e22ca1737f8c8)
+
+commit c0b0f52ddbd4e22998a36addfb32f27614183e19
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 15:35:10 2015 +0800
+
+    crushtool: enable check against max_id
+    
+    add an argument "max_id" for "--check-names" to check if any item
+    has an id greater or equal to given "max_id" in crush map.
+    
+    Note: edited since we do not have the fix introduced in 46103b2 in
+          hammer.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d0658dd3cdf072b2a7c2a1986f8785a697c591ee)
+
+commit f041bbebf98a2aff2ad542e8d0c12c46af427573
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 15:34:33 2015 +0800
+
+    crush/CrushTester: check if any item id is too large
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e640d89240017956b8c7411babb86be0f1e2b172)
+
+commit cc1cc033930e8690a57674e842a003f6bbc7a242
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 25 20:14:32 2015 +0800
+
+    mon: validate new crush for unknown names
+    
+    * the "osd tree dump" command enumerates all buckets/osds found in either the
+      crush map or the osd map. but the newly set crushmap is not validated for
+      the dangling references, so we need to check to see if any item in new crush
+      map is referencing unknown type/name when a new crush map is sent to
+      monitor, reject it if any.
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a955f36a509e5412b1f72632a1a956d99e768e35)
+
+commit ff29a7f9dd21505c681881e609183aed9ac3250e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 12:08:36 2015 +0800
+
+    crushtool: add the "--check-names" option
+    
+    * so one is able to verify that the "ceph osd tree" won't chock on the
+      new crush map because of dangling name/type references
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d6b46d4c7b722945ce24ac2930381a109b1e3dda)
+
+commit 960ea49699f421ceb89c9e0c9430378a35f09a9a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 12:08:09 2015 +0800
+
+    crush/CrushTester: add check_name_maps() method
+    
+    * check for dangling bucket name or type names referenced by the
+      buckets/items in the crush map.
+    * also check for the references from Item(0, 0, 0) which does not
+      necessarily exist in the crush map under testing. the rationale
+      behind this is: the "ceph osd tree" will also print stray OSDs
+      whose id is greater or equal to 0. so it would be useful to
+      check if the crush map offers the type name indexed by "0"
+      (the name of OSDs is always "OSD.{id}", so we don't need to
+      look up the name of an OSD item in the crushmap).
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b75384d73958faf81d45847a7dfa56f4fa347e6f)
+
+commit 1cffe8c46a39142ee0da4e2279eda7276df262e1
+Merge: 6ffb1c4 ef6641c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:39:15 2015 +0200
+
+    Merge pull request #4667 from SUSE/wip-11611-hammer
+    
+    ceph.spec: update OpenSUSE BuildRequires
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5e72479b0882ac13597d7a613698e583dcb2b932
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 11:43:01 2015 -0700
+
+    OSDMonitor: allow addition of cache pool with non-empty snaps with config
+    
+    We need to be able to allow the version of ceph_test_* from earlier
+    versions of ceph to continue to work.  This patch also adjusts the
+    work unit to use a single rados snap to test the condition without
+    --force-nonempty to ensure that we don't need to be careful about
+    the config value when running that script.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 6ffb1c4ae43bcde9f5fde40dd97959399135ed86
+Merge: c7ebf96 524f4a5
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Jul 8 16:52:12 2015 +0100
+
+    Merge pull request #5123 from theanalyst/wip-11979-hammer
+    
+    MDSMonitor: handle MDSBeacon messages properly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ecac1a458bc7331ed8d667f20ba31995d74892d3
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Fri Feb 6 01:26:40 2015 +0100
+
+    Always provide summary for non-healthy cluster
+    
+    This fixes a problem, wherein calamari does not provide
+    popup drill-downs for warnings or errors, should the summary
+    be missing.
+    
+    Calamari gets health info from /api/v1/cluster/$FSID/health.
+    If the data here has a summary field, this summary is provided
+    in a popup window:
+    
+      /api/v1/cluster/$FSID/health is populated (ultimately) with
+      status obtained via librados python bindings from the ceph
+      cluster. In the case where there's clock skew, the summary
+      field supplied by the ceph cluster is empty.
+    
+    No summary field, no popup window with more health details.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit eaf6e0cf48488fe604d0ef0db164d44948d4e8d4)
+
+commit c7ebf96a9a4a6143b112c8606d5ee346fb800cec
+Merge: b163728 1a321e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:36:38 2015 +0200
+
+    Merge pull request #4862 from SUSE/wip-11874-hammer
+    
+    Bucket header is enclosed by quotes
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit b1637289ec4ff99d923457577893b4c4a8d2e9fe
+Merge: e33af22 54f4e7d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:35:53 2015 +0200
+
+    Merge pull request #4885 from theanalyst/wip-11755-hammer
+    
+    Object copy bug
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e33af22dec32467f229ca2dc1dc9668702a44ce8
+Merge: 5696b0f 9dfef60
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:34:45 2015 +0200
+
+    Merge pull request #4884 from theanalyst/wip-11722-hammer
+    
+    Keystone PKI token expiration is not enforced
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5696b0ff2a981ae8afe72df796ba7d7da47bb7d1
+Merge: 72ecd52 ed5442b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:30:48 2015 +0200
+
+    Merge pull request #4875 from ceph/wip-11770-hammer
+    
+    librbd: aio calls may block
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1a32379dd6cb56ed69b8f448d3612506c8131fbe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 19:30:30 2015 +0800
+
+    mon/PGMap: add more constness
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e1f1c56ce88ef3ad05e881d650fac637931ce3fe)
+
+commit 84ebc3d320bdf871ccf85e555951cea421b56021
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 19:29:13 2015 +0800
+
+    mon/PGMap: sort pg states by the states in "pg ls" spec
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 990dfb6cd45438bc8293ac37882daa413860a2f8)
+
+commit e310461aecc667cf26806bc5a3bbabb05696bdfc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 17:02:41 2015 +0800
+
+    mon: s/recovery/recoverying/ in "pg ls*" commands' spec
+    
+    * also translate "repair" if specified as "states"
+    * update test_mon_pg in cephtool-test-mon.sh
+    
+    Fixes: #11569
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 89f89ca3477eddcae11a05fbd58a8f3658eb1fc1)
+
+commit 524f4a52d115ecda8cd7793d0f8bea148eff92af
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 15 22:50:36 2015 +0800
+
+    mon: always reply mdsbeacon
+    
+    the MDS (Beacon) is always expecting the reply for the mdsbeacon messages from
+    the lead mon, and it uses the delay as a metric for the laggy-ness of the
+    Beacon. when it comes to the MDSMonitor on a peon, it will remove the route
+    session at seeing a reply (route message) from leader, so a reply to
+    mdsbeacon will stop the peon from resending the mdsbeacon request to the
+    leader.
+    
+    if the MDSMonitor re-forwards the unreplied requests after they are
+    outdated, there are chances that the requests reflecting old and even wrong
+    state of the MDSs mislead the lead monitor. for example, the MDSs which sent
+    the outdated messages could be dead.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3555e9c328633c9e1fbc27d652c004b30535e5b)
+
+commit 413e407dea446bebb9c36abb3025ada450dd4fe9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:20:21 2015 -0700
+
+    mon/MDSMonitor: rename labels to a better name
+    
+    * s/ignore/reply/
+    * s/out/ignore/
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f00ecb8b3df73ce6337985bc6d43bce5143ee537)
+
+commit a03968ad584a3ff8e351cc5dce053e535fcdc454
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:55:06 2015 +0800
+
+    mon: send no_reply() to peon to drop ignored mdsbeacon
+    
+    so the peon can remove the ignored mdsbeacon request from the
+    routed_requets at seeing this reply, and hence no longer resend the
+    request.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 72a37b3a8e145d8522ea67fc14ce2c5510b6852b)
+
+commit 39f34596b0ec6f769f507e2b372204f8551f7ee0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:22:26 2015 +0800
+
+    mon: remove unnecessary error handling
+    
+    msg.get_session() should always return a non-zero pointer in
+    Monitor.dispatch()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 16e8e2cc82a90c49cd8aa3d0e3acc4694ba659a0)
+
+commit 0e5e7e1d259579571c1fc05660f6af8e295e733b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 15:21:20 2015 +0800
+
+    mon: remove unused variable
+    
+    * as a side effect, this change silences
+      http://tracker.ceph.com/issues/11576
+    
+    Fixes: #11576
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7b196a4a091c0ea258866559ba06e7ed0cc4247)
+
+commit 70347209260688f1a067354c744569499adb6920
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Apr 1 16:37:51 2015 -0700
+
+    ReplicatedPG::finish_promote: handle results->snaps is empty case
+    
+    If results->snaps winds up empty after filtering removed snaps,
+    we need to treat the object as if we had gotten an ENOENT.
+    
+    PartialFix: #11296
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6150757dbe0fa11cceb14460865b859a7c8164c7)
+
+commit 3e44dc16ed3fbda053996e44826aa3d90042a234
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Apr 1 16:25:22 2015 -0700
+
+    ReplicatedPG::finish_promote: fix snap promote head snaps
+    
+    If the snaps vector is: 10=[9,5,2]:[4]+head, the backing pool's snaps
+    vector is 3=[2]:[]+head, and we request clone 4 from the backing pool,
+    the backing pool will send us head with an empty results->snaps vector.
+    Actually, clone 4 should be trimmed, but the cache pool does not know
+    that.  Thus, we should construct an empty snaps vector for that clone.
+    
+    PartialFix: #11296
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a45a698372def1623323470c6a1c4eb70e0bb79f)
+
+commit 5ef08466abf1b3934fcad0a0ca46f3a4380d6dbd
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 3 10:04:26 2015 +0100
+
+    tools: chunk reads in Dumper
+    
+    Previously tried to read entire journal
+    into memory in one go, which was problematic
+    for large journals.
+    
+    Fixes: #11746
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e3ddcb894ad09326698999d42de0ce3feb69f28e)
+
+commit 408880bed296e5cbf05864fa6744a5b00a245272
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 16 08:13:41 2015 -0700
+
+    qa: update to newer Linux tarball
+    
+    This should make newer gcc releases happier in their default configuration.
+    kernel.org is now distributing tarballs as .xz files so we change to that
+    as well when decompressing (it is supported by Ubuntu Precise so we should
+    be all good).
+    
+    Fixes: #11758
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 1ea3f47ab806d48ca7b045c2731d344eae3900e1)
+
+commit 56c2688b87d7d78831f8e147fc67cc0651ab644c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 24 14:45:40 2015 -0700
+
+    rgw: simplify content length handling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e97fd5052cab83c5f699531a8c960b93437a8f9f)
+
+commit d9bbef3e470c6406bb65dc40e7e9c08c5d599f73
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Fri Apr 24 10:49:16 2015 -0700
+
+    rgw: make compatability deconfliction optional.
+    
+    Per request from Yehuda, the deconfliction for having both
+    HTTP_CONTENT_LENGTH and CONTENT_LENGTH set is now optional, and
+    controlled by new configuration boolean, which defaults to false.
+    rgw content length compat
+    
+    X-URL: https://github.com/ceph/ceph/pull/4436#issuecomment-95994887
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 79d17af1a1ec0659884f768945a7bac6282b5e0b)
+
+commit 0260abd5d265bd63ea9c89f4082c31ba1c5ae8fa
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Wed Apr 22 12:53:06 2015 -0700
+
+    rgw: improve content-length env var handling
+    
+    The FastCGI specification, section 6.3 on Authorizers, describes a case
+    where HTTP_CONTENT_LENGTH will be set in the environment and
+    CONTENT_LENGTH will NOT be set.
+    
+    Further documention in the code.
+    
+    Fixes: #11419
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 3e38eab44bfb082fdd2b6f29b8b0357f8f5c11bb)
+
+commit 8abc46a157e4c1431a92a1e52ab694dccff5d514
+Author: wuxingyi <wuxingyi@letv.com>
+Date:   Wed Jun 10 06:57:57 2015 +0000
+
+    rgw: fix data corruption when race condition
+    
+    We should delete the object in the multipart namespace lastly to prevent a previous upload
+    wrongly deleting objects belong to the following upload.
+    
+    Fixes: #11749
+    Signed-off-by: wuxingyi <wuxingyi@letv.com>
+    (cherry picked from commit ac1e729a75b5d995028bbc223bcf5ecce0d112cc)
+
+commit 72ecd522941156c8a7e5303531944b0735dcbeb8
+Merge: 59f37a9 d723e11
+Author: Abhishek L <abhishekl.2006@gmail.com>
+Date:   Wed Jul 1 18:09:46 2015 +0530
+
+    Merge pull request #4886 from theanalyst/wip-11737-hammer
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 59f37a9bafc095181b3f41ec5d93ac58e2cda604
+Merge: 53a2143 89d0266
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sat Jun 27 17:11:11 2015 +0800
+
+    Merge pull request #5095 from ceph/wip-fix-doc-hammer
+    
+    doc: fix doc build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 89d0266a60729d5d9747867e8c30abf71a891231
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:44:55 2015 +0800
+
+    doc: add the corresponding @endcond command for @cond
+    
+    * they are used to applease asphyxiate, as it
+      is not able to handle "enum" sections
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2aa77b33a97e5a3ca134c9c555aa6e7a69ef50f7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:43:01 2015 +0800
+
+    doc: remove orphan: directive in ceph-create-keys.rst
+    
+    * it is used to silence the sphinx warning, but conf.py
+      does not like it.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit ad66e40e8bd598da7c9738cb44abb543008c90a3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:41:59 2015 +0800
+
+    doc: let doxygen ignore src/tracing
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 53a2143eb7e549de1185b01ed0bde841ffa5235a
+Merge: 6f7cd04 bfb1442
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jun 26 14:19:40 2015 -0700
+
+    Merge pull request #4902 from theanalyst/wip-11908-hammer
+    
+    Fixes for rados ops with snaps
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 6f7cd0460d5729c15966119e0177ddc56a361d8e
+Merge: 78d894a 356bd2c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 25 20:03:42 2015 -0400
+
+    Merge pull request #5069 from dachary/wip-11806-hammer
+    
+    ceph / ceph-dbg steal ceph-objecstore-tool from ceph-test / ceph-test-dbg
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 356bd2c68ca730e766d06c46a0364784f5d72275
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jun 24 14:58:47 2015 -0400
+
+    debian: ceph-dbg steals ceph-objectstore-tool from ceph-test-dbg (take 2)
+    
+    968573b8930a7c8485bf53e3a989ce2f7d0a2fff incorrectly backported
+    6f11fbf41fab10924b1e0e41fcf27864779d4073. It should instead reflect that
+    ceph-dbg in 0.94.2 and above will break ceph-test-dbg in all versions
+    prior to 0.94.2.
+    
+    In other words, 0.94-XXXX being lower than 0.94.1, upgrading from
+    0.94.1 to 0.94.2 will not notice that ceph-dbg breaks ceph-test-dbg.
+    
+    $ dpkg --compare-versions 0.94-XXXX lt 0.94.1 && echo yes || echo no
+    yes
+    $ dpkg --compare-versions 0.94.2 lt 0.94.1-xxx && echo yes || echo no
+    no
+    
+    http://tracker.ceph.com/issues/11806 Fixes: #11806
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit c5c627fdd3de669ee94ab7ecd6939c29a4a68011
+Author: wuxingyi <wuxingyi2015@outlook.com>
+Date:   Wed Mar 11 17:34:40 2015 +0800
+
+    rgw/logrotate.conf: Rename service name
+    
+    The service name for ceph rados gateway was changed to "ceph-radosgw",
+    the previous version of service name "radosgw" would cause a failed reload,
+    and finally make it impossible to write any log data to the log file.
+    
+    Signed-off-by: wuxingyi <wuxingyi2015@outlook.com>
+    (cherry picked from commit 9df3f798179481fe8ae6ae873dcb793de7d8f367)
+
+commit 582cf731e05baabd2cd79567af89d7f005c24925 (refs/remotes/gh/wip-12109-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 13:08:21 2015 -0400
+
+    tests: add librbd watch/notify version compatibility test
+    
+    Fixes: #11405
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 41e4cbe032e32762e3a9e8bc3eff8ece19f91a54)
+
+commit 43b9aef11c1281d8b83b659a523dba365d79add4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 11:22:55 2015 -0400
+
+    qa/workunits/rbd: add notify_master/slave bootstrap scripts
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 124b1d3d2d0dbd685bbd717856d29b316e62e660)
+
+commit f995fb50d3d7734161fa498db9e204eaded651b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 10:34:48 2015 -0400
+
+    qa/workunits/rbd: add new test_librbd_api workunit
+    
+    This only tests the public librbd API for use during upgrade tests.
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9039955f42d01044cfcf20c56ca2181e51c317ee)
+
+commit a09da2a2c6c12c6bf2f8e17f2096b3ab24587007
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 10:27:38 2015 -0400
+
+    tests: create librbd API-only integration test suite
+    
+    The QA client-upgrade test suite requires a librbd test
+    that is dynamically linked to librbd.  Since the current
+    ceph_test_librbd includes tests against the non-public API,
+    it is statically linked against librbd and thus cannot be
+    used to test a client upgrade scenario.
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6fe94c8cbb924c31c1007e2d0c76f28ce9dbca57)
+
+commit e149916cd40a58ce5db5f63a2b4efd82a8c4e1ba
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Apr 21 11:31:24 2015 +0200
+
+    Increase max files open limit for OSD daemon.
+    
+    Under heavy load the number of file descriptors opened
+    by the OSD can go beyond the 64K file limit. This patch
+    increases the default to 128K.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit ebda4ba1c67172852587e47a8e6fb538809a1b1c)
+
+commit 22f58ce2665f1ea6b38f6016dc585202a4595322
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 1 15:10:14 2015 -0700
+
+    Makefile: install ceph-post-file keys with mode 600
+    
+    Otherwise ssh (may) prompt for a password.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 106a1c3081d02446aa3d8e13865da0c3393bae90)
+
+commit 3e65a10bd2351744da199a4b076659191b4378a2
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:05:32 2015 -0700
+
+    ceph-post-file: improve check for a source install
+    
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit ee170eadcdcb4b54d36a7d474558484de9d917eb)
+
+commit c1f6743940250b04ae6dbea30d8805571add39b6
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:00:39 2015 -0700
+
+    ceph-post-file: behave when sftp doesn't take -i
+    
+    Fixes: #11836
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit b84031ed5eaace1222e14d3c4076a3ab1155da96)
+
+commit 38d36b1174160ad104704aaa7ff5290d1ec3e782
+Author: Tim Serong <tserong@suse.com>
+Date:   Thu Apr 30 10:55:38 2015 +1000
+
+    packaging: move SuSEfirewall2 templates out of src
+    
+    Better to have static config like this that needs to be installed in
+    /etc in a separate subdirectory of the project.
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 70292658d5febb6c5f94af4df9c3e93551772d12)
+
+commit 24bc9f2c58ca78c8023935b8b69f8b01adbc11af
+Author: Tim Serong <tserong@suse.com>
+Date:   Wed Apr 29 13:12:38 2015 +1000
+
+    packaging: add SuSEfirewall2 service files
+    
+    This adds SuSEfirewall2 service files for Ceph MON, OSD and MDS, for use
+    on SLES and openSUSE.  The MON template opens port 6789 and the OSD/MDS
+    template opens the range 6800-7300 as per
+    http://docs.ceph.com/docs/master/rados/configuration/network-config-ref/
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 77685f5b787c56bcb1c4d9f1e058e25312fa62fe)
+
+commit 8acfb994f22efa07beeecccda300cbd50d683566
+Author: Owen Synge <osynge@suse.com>
+Date:   Thu May 7 12:02:41 2015 +0200
+
+    Bug fix to ceph systemV compatability script.
+    
+    Was failing with more than one OSD / MON deamon on a single node.
+    Fixes suse bugzilla #927862
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit dfda3ff8741fcdbac3150456ca7614cf75ef1776)
+
+commit bd3fd929e6f95e6d9840317aa3ac02f52711c94b
+Author: Owen Synge <osynge@suse.com>
+Date:   Thu Jun 18 14:16:03 2015 +0200
+
+    Fixes to rcceph script
+    
+     - only start OSDs if mon daemons are also present
+     - adds support for mask and unmask
+     - removes support for cluster with non default cluster name,
+       as this was very limited and inconsistent
+     - Reapplied from a patch as could not cherry-pick
+       66cb46c411d874be009c225450eea5021cf1219b from Mon Jan 12
+       as this produced issues with src/gmock
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit bfa0c4a626fdbb2bf978ccfab783ac428156144b)
+
+commit 37d77d3680d2c8adda35eddf9625a331a45ece11
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Apr 21 11:31:24 2015 +0200
+
+    Increase max files open limit for OSD daemon.
+    
+    Under heavy load the number of file descriptors opened
+    by the OSD can go beyond the 64K file limit. This patch
+    increases the default to 128K.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit ebda4ba1c67172852587e47a8e6fb538809a1b1c)
+
+commit 78d894a634d727a9367f809a1f57234e5e6935be
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 17 09:35:28 2015 -0700
+
+    qa/workunits/rados/test-upgarde-v9.0.1: fix exclude syntax
+    
+    It's -, then a list of all exclusions separated by :.  There are just 2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 3e8d60a80ce31860eac76a1f6489a35e1795a0c0
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 16 21:05:29 2015 -0700
+
+    qa/workunits/rados/test-upgrade-v9.0.1: skip one more evict test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 348a3d3c9880e7d022e71a2faafe51c8f771406e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 15 15:12:43 2015 -0700
+
+    qa: add compatibility filtered rados api tests for upgrades
+    
+    Post-9.0.1, the evict op returns success when an object doesn't exist
+    in the cache tier. Skip the tests that are incompatible across
+    versions.
+    
+    Fixes: #11548
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d2b80966b8f74de818a671c90b4c821a4b0782db
+Merge: 70bba62 f68bf94
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 15 12:38:57 2015 -0700
+
+    Merge pull request #4961 from ceph/wip-11493-hammer
+    
+    backport 11493 fixes, and test, prevetning ec cache pools
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f68bf94e370fb11a3047ec2762a972a0b7a7c0bb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 15 13:05:40 2015 -0700
+
+    OSDMonitor: disallow ec pools as tiers
+    
+    Fixes: 11650
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 11b7801bb57cb25cd2d26d58722d49691747725b)
+
+commit 13c8d58da1303cc68d99da19f79d625f91f99d43
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 29 12:34:25 2015 -0700
+
+    mon: prevent pool with snapshot state from being used as a tier
+    
+    If we add a pool with snap state as a tier the snap state gets clobbered
+    by OSDMap::Incremental::propogate_snaps_to_tiers(), and may prevent OSDs
+    from starting.  Disallow this.
+    
+    Include a test.
+    
+    Fixes: #11493
+    Backport: hammer, giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbec53edf9e585af4e20bbc9ba9057d6fdfda342)
+
+commit 58e62662f6ef04ac76470090d1d958467e34194a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 8 10:26:48 2015 -0700
+
+    test/librados/tier.cc: destroy and recreate cache pool on every test
+    
+    Namespaces are not sufficient with the checks for 11493 in the mon.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bef09e0cdb274cb1c87335a2af9ee532d14a4596)
+
+commit 70bba6226a64090dcf41cd90b23fdf5aed8cd0ca (refs/remotes/jashan/hammer)
+Merge: 3b6977b 5a60a03
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 15 17:26:07 2015 +0200
+
+    Merge pull request #4846 from SUSE/wip-11862-hammer
+    
+    missing man pages for ceph-create-keys, ceph-disk-*
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 3b6977b706dbc99cac25bec1b71a628c398c6ff1
+Merge: 5fb8561 3db1026
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 11 13:54:35 2015 -0700
+
+    Merge pull request #4934 from dachary/wip-releases-hammer
+    
+    doc/release-notes: v0.94.2
+
+commit 3db1026f3706e6f5a5c25013cb6646a0298057d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 10 12:48:41 2015 -0700
+
+    doc/release-notes: v0.94.2
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 306345b29c259ab04a58ed5d40f801645485b29d)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit ed5442b1057dcc4fb1f9404c805dabe2bbde2252 (refs/remotes/jashan/wip-11770-hammer, refs/remotes/gh/wip-11770-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 2 10:33:35 2015 -0400
+
+    tests: verify librbd blocking aio code path
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4cf41486e9c9e1efcb863960a8f3e0326ffca7e5)
+    
+    Conflicts:
+    	src/test/librbd/test_librbd.cc: trival resolution
+
+commit 20e104869f3d17ce672438144700a4d984d487b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 1 22:56:11 2015 -0400
+
+    librbd: new rbd_non_blocking_aio config option
+    
+    Setting this option to false reverts librbd to legacy behavior
+    where AIO operations could potentially block.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915)
+
+commit b4571b3e238efc39767f753e0ec1622c8bd6d6e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 20:34:28 2015 -0400
+
+    PendingReleaseNotes: document changes to librbd's aio_read methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9ea1edd0ca9e385f823ad04b05bc887d77aa5136
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 13:33:09 2015 -0400
+
+    librbd: AioRequest::send no longer returns a result
+    
+    The librados calls used by AioRequest::send should always return
+    zero unless there is a bug.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c77bce3311ab62892eb8c1d883263ba7ed663b20)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/AsyncFlattenRequest.cc: trivial resolution
+
+commit 272df2aed79a95dd9c45db4e0953e9b321f7b0f5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:55:36 2015 -0400
+
+    tests: update librbd AIO tests to remove result code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 948b15eb52fd5d9ce842fa12ee0cecda17353b01)
+    
+    Conflicts:
+    	src/test/librbd/test_internal.cc: trivial resolution
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit dd2e4c13ff6d88edb25f90af62af16ba825c15c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:37:50 2015 -0400
+
+    librbd: internal AIO methods no longer return result
+    
+    All failures should be returned via the AioCompletion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ab42d613128ab08c688ddbea93df4c95068b9cd)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit dbd4e293d7124c89a22148e8fa5f425a995c900c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:48:21 2015 -0400
+
+    Throttle: added pending_error method to SimpleThrottle
+    
+    Allow the client of SimpleThrottle to detect an async error
+    so that it can exit early.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b88b88c5df91325fb713c2031a56bffe421268e0)
+
+commit 7df6091a30b1b94d764240262195e971175554b3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 20:18:50 2015 -0400
+
+    librbd: add new fail method to AioCompletion
+    
+    Helper method to handle passing fatal errors generated within
+    librbd (not from the OSDs) back to the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d1d0c867855a96bee4c13a0c0a39a0e002ccd12)
+
+commit cf6e1f50ea7b5c2fd6298be77c06ed4765d66611
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 19:06:52 2015 -0400
+
+    librbd: avoid blocking AIO API methods
+    
+    Enqueue all AIO API methods within the new librbd thread pool to
+    reduce the possibility of any blocking operations. To maintain
+    backwards compatibility with the legacy return codes of the API's
+    AIO methods, it's still possible to block attempting to acquire
+    the snap_lock.
+    
+    Fixes: #11056
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a7b5e30efdb21aa1a0aeb68a5d02a1ac2a5faf3)
+
+commit e61974aed09a3f81e1f65a4bbaed43e3f22b27b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add task pool / work queue for requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afb896d91f886b647baf38f7ec94cc3739f6d2a9)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc: trivial resolution
+    	src/librbd/ImageCtx.h: trivial resolution
+
+commit bfb144268b803340efad29cd6c627b170ea32402
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 20 16:10:02 2015 -0700
+
+    ReplicatedPG::release_op_ctx_locks: requeue in scrub queue if blocked
+    
+    Otherwise we can reorder an op around another op which got blocked by a
+    scrub which started after the first blocked on an obc.
+    
+    Fixes: #11691
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit be873eb8da7b29ecefaa5a99b88de7ddcca711ee)
+
+commit c7b6a6370a69149ea94f9e35d536aa90f06e7659
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 19 10:56:11 2015 -0700
+
+    ReplicatedPG::finish_ctx: take excl lock if operation is rw
+    
+    Fixes: #11677
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5c2b795724423ed484ab451de855ddcfc085342b)
+
+commit 1550a569dab120ce28396fe365565e8e4acd9801
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 21 12:13:43 2015 -0700
+
+    RadosModel: randomly prefix delete with assert_exists
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4fe7d2abdff2fce359e5e992206644cc03825ee0)
+
+commit 4cdc5f7d6b3ec488c79c09cb44a43d4d9398b74c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 21 11:36:42 2015 -0700
+
+    RadosModel: assert exists on subsequent writes
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 121aa3bc612b86281535ac3bcfe98bc99bc99ace)
+
+commit 25c730bda74b94f2c894c508ab09988dbd528c4e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 19 10:23:01 2015 -0700
+
+    test/librados/snapshots.cc: add test for 11677
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c2d17b927f8a222164b3bf2922a4ff337696f566)
+
+commit 3d74164d3d6caaa5099abd9a1d1920482d3e05c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 27 11:14:15 2015 -0700
+
+    ReplicatedPG::trim_object: write filtered snapset while we're at it
+    
+    If we trimmed an object, we might as well remove the obsolete snaps
+    as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 90eb7768f99ea249952df195a844a3a7c9a59b78)
+
+commit a1161540bc0094a951021d4ca651b95ec045213e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 27 11:00:54 2015 -0700
+
+    ReplicatedPG: start_flush: use filtered snapset
+    
+    Otherwise, we might send our deletes based on deleted snaps.  This is
+    problematic since we may have trimmed the clones to which those snaps
+    belong, causing us to send them at an earlier snap than we used before.
+    
+    The specific situation was
+    
+    78:[78, 70, 63, 5a, 58, 57]:[64(63), 58(58, 57)]
+    
+    with 58 already clean.  To flush 64, we send:
+    
+    delete@58
+    delete@59
+    copyfrom@62
+    
+    Then, snap 63 is trimmed leaving us with a snapset of:
+    
+    78:[78, 70, 63, 5a, 58, 57]:[58(58, 57)]
+    
+    since trim_object doesn't filter the head object snapset snaps.  This
+    isn't really a bug since in general all snapset users must be aware
+    that there may be trimmed snaps in snapset::snaps.  However, here
+    it becomes a problem when we go to flush head:
+    
+    delete@58 -- ignored due to snapc
+    delete@59 -- ignored due to snapc
+    copyfrom@78 -- not ignored
+    
+    The base pool head is at snap seq 62, so it clones that value into
+    clone 78(78, 70) instead of forgetting it.  What should have happened
+    is that we should have based our flushes on filtered snapset:
+    
+    78:[78, 70, 58, 57]:[58(58, 57)]
+    
+    Causing us to instead send:
+    
+    delete@58 -- ignored due to snapc
+    delete@69 -- not ignored, causes no clone to be made
+    copyfrom@78 -- not ignored, updates head such that a subsequent clone
+    will leave 70 out of the clone snaps vector.
+    
+    Fixes: 11787
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6051e255ac062985ada1989edb7f23cd750915e2)
+
+commit 82988d611bad6226138b94590275faadbca3554a
+Author: islepnev <islepnev@gmail.com>
+Date:   Fri Apr 17 22:33:01 2015 +0300
+
+    ceph-disk: support NVMe device partitions
+    
+    Linux nvme kernel module v0.9 enumerate devices as following:
+    
+    /dev/nvme0 - characted revice
+    /dev/nvme0n1 - whole block device
+    /dev/nvme0n1p1 - first partition
+    /dev/nvme0n1p2 - second partition
+    
+    http://tracker.ceph.com/issues/11612 Fixes: #11612
+    
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit 9b62cf254d02d30609793be8b1cb8a94f38891f1)
+
+commit bd91fb027ab91d487b1d61d25516c13590735d89
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 16:37:56 2015 -0700
+
+    mon: prevent bucket deletion when referenced by a rule
+    
+    If a rule references a bucket with 'take', prevent deletion.
+    
+    Fixes: #11602
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3d591afef90b0601572c748f13faac029d05f5a0)
+
+commit 56565ee1cdb06a7705d1c3f26f5592b10399324a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 14:03:49 2015 -0700
+
+    crush: fix crash from invalid 'take' argument
+    
+    Verify that the 'take' argument is a valid device or bucket.  Otherwise,
+    ignore it (do not add the value to the working vector).
+    
+    Backport: hammer, firefly
+    Fixes: #11602
+    Reported-by: shiva rkreddy <shiva.rkreddy@gmail.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9324d0a1af61e1c234cc48e2175b4e6320fff8f4)
+
+commit 1440122d61d5b0a3f8360f4e2101db1018109799
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 29 03:28:18 2015 -0700
+
+    common/config: detect overflow of float values
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 1ff409ef8d022a1a84d034bd3db976c4d769e993)
+
+commit 9b947fa320b77e0055a581005353c2561a12a198
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 29 15:41:08 2015 +0800
+
+    common/config: detect overflow of int values
+    
+    * #include "strtol.h" in strtol.cc, to ensure the function defintions
+      are consistent.
+    * add a test accordingly
+    * fix the testcase of converting 1024E.
+    * do not accept integers overflow after adding SI suffix
+    * do not accept integers underflow (i.e. negative values)
+    
+    Fixes: #11484
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d62f80dc7b25d312ff05b65b7be854aae15b66a8)
+
+commit d723e1156e70a492d633e43b86e7c373e5750065
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue May 12 14:52:30 2015 +0800
+
+    mds: clear CDir::STATE_REJOINUNDEF after fetching dirfrag
+    
+    Fixes: #11541
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ab1e5394dc778f6799472bd79a4d9ba7197107c2)
+
+commit 54f4e7d4a534448293c74612f8140cf34b9cf9f8
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri May 15 14:22:29 2015 +0200
+
+    rgw: Use attrs from source bucket on copy
+    
+    On copy objects, when bucket source is the same as the destination, use attrs
+    from source bucket.
+    
+    Fixes: #11639
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 1dac80df1d4a2364154ed8b404d13609936c257b)
+
+commit 9dfef6004d208af3730634796aad199391707826
+Author: Anton Aksola <anton.aksola@nebula.fi>
+Date:   Fri Apr 10 13:25:21 2015 +0300
+
+    rgw: always check if token is expired
+    
+    Fixes: #11367
+    
+    Currently token expiration is only checked by the token cache. With PKI
+    tokens no expiration check is done after decoding the token. This causes
+    PKI tokens to be valid indefinitely. UUID tokens are validated by
+    keystone after cache miss so they are not affected by this bug.
+    
+    This commit adds explicit token expiration check to
+    RGWSwift::validate_keystone_token()
+    
+    Signed-off-by: Anton Aksola <anton.aksola@nebula.fi>
+    Reported-by: Riku Lehto <riku.lehto@nexetic.com>
+    (cherry picked from commit 2df069390ea3bbcfbab5022750e89f51d197cc11)
+
+commit fdb43ebe2b1211acdb454836a64dbd589feeef45
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Fri May 15 21:43:34 2015 +0200
+
+    ceph.spec.in: tweak ceph-common for SUSE/openSUSE
+    
+    ceph-common needs python-argparse in SUSE/openSUSE and
+    needs redhat-lsb-core only in RHEL/CentOS/Fedora.
+    
+    http://tracker.ceph.com/issues/11638 Fixes: #11638
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 363d957d8fdd15a1674befbd8e485fd89b76d716)
+
+commit ba1a0167cc809081eda7e6cc9ecfb971e439d696
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 3 10:09:09 2015 +0100
+
+    admin/build-doc: fix dependency checks
+    
+    http://tracker.ceph.com/issues/11857 Fixes: #11857
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 539c1ba7211f579bad4f59ae824f1e68e620ecbd)
+    
+    Conflicts:
+    	admin/build-doc
+                Insert lines at appropriate spot
+
+commit 5a60a034bf3015eaf468e5e3f9d8feb08b8fdd95
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 3 10:41:27 2015 +0200
+
+    man/ceph-create-keys.8: add missing file
+    
+    This is the generated manpage, taken from a local build of master
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bcda61fcbe07ee36cd5172d80018f287591660ec)
+
+commit 19305b897ac147602049752a2dfbe1fd39ba562c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 21 14:59:32 2015 +0800
+
+    doc: add ceph-create-keys.8
+    
+    Fixes: #10725
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 27cee2f1f46a9f47cda9dfeb56ff1259e982960c)
+    
+    Conflicts:
+        doc/man/8/ceph-create-keys.rst
+        Includes fixes from https://github.com/ceph/ceph/pull/4855
+
+commit ffd0933dcc790d7cedc1048b664bf4e8c40464a3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 17:05:49 2015 -0400
+
+    WorkQueue: added virtual destructor
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3f5a75332c058816dc39b71e9d2b36e752159f4)
+
+commit a28adfbdd8abc86e7766c303bc610c0c252910f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: add new ContextWQ work queue
+    
+    The queue holds a collection of Context pointers that will
+    be completed by the thread pool.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24a33e977f7b71962adeeb48f75d488a76e70fa9)
+
+commit 1a321e477effa23bffbb6cb057aa955586197345
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Jun 3 13:13:33 2015 +0200
+
+    rgw: Do not enclose the Bucket header in quotes
+    
+    Fixes: #11860
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8af25faed93fe02d3dad585b8579ce8b041cc4e6)
+
+commit a62c3aa1df9e0f79ac75d94083d10b902dbba382
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 14:04:30 2015 +0800
+
+    ceph.in: handle unknown Exception correctly
+    
+    * in case parse_cmdargs() throws
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7789eefc01deb9ca7fe90f5521aece3e36c3c350)
+
+commit cc7f7441c1d0457e840bfdc3413f9bc6d5f5134a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 01:27:44 2015 +0800
+
+    ceph.in: improve the interactive mode
+    
+    * if ceph is not reading from a tty, expect EOF instead of "quit"
+      as the end of input.
+    * do not panic at seeing the EOF
+    * update the test case test_mon_injectargs_SI(). since we disables
+      "ceph injectargs <args,...>" in a458bd83, in which the arguments
+      of "injectargs" are supposed to be consumed by "tell" instead.
+      so "ceph injectargs ..." is taken as an incomplete command, and
+      this command will bring ceph cli into the interactive mode,
+      redirecting its stdin to /dev/null helps ceph cli quit the loop,
+      but in a way of throwing EOFError exception. this change handles
+      the EOF, so the "ceph injectargs ..." does not throws anymore.
+      but the side effect is that the test fails since it expects a
+      non-zero return code. so replace it with an equivalent "tell"
+      command which also fails but due to the non-SI postfix.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da9d2b4077ab1dceeed979ab71f0d9ed59b14266)
+
+commit eb26388cb985456e8f6e07a52a20fa912cf4efaa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 00:50:37 2015 +0800
+
+    ceph.in: parse quote correctly in interactive mode
+    
+    Fixes: #11279
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc7d8c99d2e16a141a8b575281ba12c67628dac3)
+
+commit 558d6391ecdfd5d716558341ce5d84ce1f7ec9c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 12:01:16 2015 +0800
+
+    json_sprit: fix the FTBFS on old gcc
+    
+    Fixes: #11574
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b68b27146852f057a3373cd04b08cd6917f3eea)
+
+commit 678b3e6082729698ce3575ba70313dd8399b1aed
+Author: Tim Serong <tserong@suse.com>
+Date:   Sat May 2 01:59:53 2015 +1000
+
+    json_spirit: use utf8 intenally when parsing \uHHHH
+    
+    When the python CLI is given non-ASCII characters, it converts them to
+    \uHHHH escapes in JSON.  json_spirit parses these internally into 16 bit
+    characters, which could only work if json_spirit were built to use
+    std::wstring, which it isn't; it's using std::string, so the high byte
+    ends up being zero'd, leaving the low byte which is effectively garbage.
+    
+    This hack^H^H^H^H change makes json_spirit convert to utf8 internally
+    instead, which can be stored just fine inside a std::string.
+    
+    Note that this implementation still assumes \uHHHH escapes are four hex
+    digits, so it'll only cope with characters in the Basic Multilingual
+    Plane.  Still, that's rather a lot more characters than it could cope
+    with before ;)
+    
+    (For characters outside the BMP, Python seems to generate escapes in the
+    form \uHHHHHHHH, i.e. 8 hex digits, which the current implementation
+    doesn't expect to see)
+    
+    Fixes: #7387
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 8add15b86e7aaef41397ab8fa9e77ee7957eb607)
+
+commit ef6641c0aa47c7f559aa56d7c35a5815afc2ba49
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 21:26:23 2015 +0100
+
+    ceph.spec: update OpenSUSE BuildRequires
+    
+    OpenSUSE 13.2 needs libsnappy-devel but not bzip2-devel.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit bdac3dc3fb189688af52f60b0b34339dd8fea251)
+
+commit 0b6d4427ac7234fd509a9541adf45e303e40cb16 (refs/remotes/jashan/wip-hammer-11535-admin-socket, refs/remotes/gh/wip-hammer-11535-admin-socket)
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Fri May 8 11:54:06 2015 -0400
+
+    common/admin_socket: close socket descriptor in destructor
+    
+    Long-running processes that do not reuse a single client connection will
+    see accumulating file descriptors as a result of not closing the
+    listening socket.  In this case, eventually the system will reach
+    file-max and subsequent connections will fail.
+    
+    Fixes: #11535
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit 88fabb1ee6cd3c7d211ca46919055195e32245db)
diff --git a/doc/changelog/v0.94.4.txt b/doc/changelog/v0.94.4.txt
new file mode 100644
index 000000000..a99f8713e
--- /dev/null
+++ b/doc/changelog/v0.94.4.txt
@@ -0,0 +1,3576 @@
+commit 95292699291242794510b39ffde3f4df67898d3a (tag: refs/tags/v0.94.4, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@ceph.com>
+Date:   Mon Oct 19 07:43:50 2015 -0700
+
+    0.94.4
+
+commit 5764e233e56be08a59ffe6292f6fba9a76288aee
+Merge: 7f485ed b203979
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 15 16:35:10 2015 -0400
+
+    Merge pull request #6287 from jcsp/wip-test-fixes
+    
+    git path backports for hammer
+
+commit b2039797638057dd74a8a47c99091c2c892b042e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 16 08:29:41 2015 -0700
+
+    use git://git.ceph.com
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34c467ba06345eacd2fd0e79c162dfd0b22f0486)
+
+commit 0f4ef19047c58aacfb359bdd9d104f059aa2f5b4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 24 18:10:46 2015 -0400
+
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4d65fbd6ad15320339751c604613ac63511e645)
+
+commit 7f485ed5aa620fe982561663bf64356b7e2c38f2
+Merge: 3dc1de2 294f016
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 13 00:28:52 2015 +0200
+
+    Merge pull request #6161 from dachary/wip-13227-hammer
+    
+    With root as default user, unable to have multiple RGW instances running
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 294f016ff84c7efef36ca94daef39fe2686df690
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:40:05 2015 -0700
+
+    init-radosgw.sysv: remove
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9d8c115b0ab05cfc1db6670d1d367737e61fe0ea)
+
+commit 3dc1de22552065bd930cc3e42a486e29639927ba
+Merge: aa74de9 7de65e7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:16:17 2015 -0700
+
+    Merge pull request #6166 from dachary/wip-13046-hammer
+    
+    RGW : setting max number of buckets for user via ceph.conf option
+
+commit aa74de9d34801552d7d3e0aced07b6bc5bd129b9
+Merge: d3e49eb 77cb503
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:15:21 2015 -0700
+
+    Merge pull request #6039 from ceph/wip-13015-hammer
+    
+    rgw: don't preserve acls when copying object
+
+commit d3e49ebce00d749c36fe8dc46a5c432b1b907126
+Merge: 417e948 424fc1c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:14:47 2015 -0700
+
+    Merge pull request #6042 from dreamhost/wip-hammer-crossftp
+    
+    rgw: remove trailing :port from host for purposes of subdomain matching
+
+commit 417e948898d260a1ad6354244f9967ccbd75d768
+Merge: b5badcb 9ab9c44
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:14:09 2015 -0700
+
+    Merge pull request #6160 from dachary/wip-13228-hammer
+    
+    rgw: segments are read during HEAD on Swift DLO
+
+commit b5badcb47e565cd17145adca49abf07e167963e0
+Merge: e3f8df9 6119b15
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:13:25 2015 -0700
+
+    Merge pull request #6162 from dachary/wip-13226-hammer
+    
+    Keystone Fernet tokens break auth
+
+commit e3f8df9e12b434714b67036838f37b75ed1f9a25
+Merge: 8078233 ad83304
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:12:26 2015 -0700
+
+    Merge pull request #6163 from dachary/wip-13054-hammer
+    
+    rgw: region data still exist in region-map after region-map update
+
+commit 8078233ca1dda8eef3c59d6a92760365634cf41a
+Merge: dd64c78 4b0686f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:12:07 2015 -0700
+
+    Merge pull request #6164 from dachary/wip-13053-hammer
+    
+    rgw: GWWatcher::handle_error -> common/Mutex.cc: 95: FAILED assert(r == 0)
+
+commit dd64c7813bbdd2a6a4ffde6767c5d060ee1b9290
+Merge: 135c112 e80bd0a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:11:16 2015 -0700
+
+    Merge pull request #5718 from dachary/wip-12859-hammer
+    
+    testGetContentType and testHead failed
+
+commit 135c112789d4925a735229394c0ec9e2b728d3ca
+Merge: fd17f3c 3b2affc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:10:55 2015 -0700
+
+    Merge pull request #5860 from lebauce/wip-12960-hammer
+    
+    rgw: add delimiter to prefix only when path is specified
+
+commit fd17f3ccbf98832098bd1cd35fe2a5d567b51f4c
+Merge: 036c718 297c04d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:08:59 2015 -0700
+
+    Merge pull request #6165 from dachary/wip-13052-hammer
+    
+    rgw: init_rados failed leads to repeated delete
+
+commit 036c7186f9304e4e48ecf50a4299cf74d55875fe
+Merge: 629b631 698d75c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 9 09:17:19 2015 +0200
+
+    Merge pull request #6201 from dachary/wip-13410-hammer
+    
+    TEST_crush_rule_create_erasure consistently fails on i386 builder
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 629b631488f044150422371ac77dfc005f3de1bc
+Merge: 70e612a 0a5b856
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 8 12:18:49 2015 -0400
+
+    Merge pull request #5885 from Abhishekvrshny/wip-13034-hammer
+    
+    osd: copy-from doesn't preserve truncate_{seq,size}
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 698d75c11fa116d6546b09d14484d094fae8f6d7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jul 22 10:42:32 2015 +0200
+
+    tests: robust test for the pool create crushmap test
+    
+    The test that goes with f1e86be589803596e86acc964ac5c5c03b4038d8 to
+    verify that a bugous crush ruleset will prevent the creation of a pool
+    trying to use it was fragile. I depends on the implementation of the
+    erasure code lrc plugin and turns out to not work on i386.
+    
+    The test is modified to use a fake crushtool that always returns false
+    and validate that it prevents the creation of a pool, which demonstrate
+    it is used for crushmap validation prior to the pool creation.
+    
+    http://tracker.ceph.com/issues/12419 Fixes: #12419
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e47903727b5d53f795014bf29b417d357f2bd65a)
+
+commit 70e612a395f7f5daf2fa289d8ae522824ef7a38f
+Merge: 0650862 2a28114
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 8 00:09:24 2015 +0200
+
+    Merge pull request #6192 from dachary/wip-13401-hammer
+    
+    mon: fix crush testing for new pools
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a28114053fbf4ceee09558790ad3cf755270d5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 21 16:40:34 2015 -0400
+
+    crush/CrushTester: test fewer inputs when running crushtool
+    
+    If there are a lot of crush rules (say, 100) then the test can
+    take a long time.  100 values per rule should be enough to catch
+    most issues.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f82f461b33d93d868e185912a2c7e4074d06900)
+    
+    Conflicts:
+    	src/crush/CrushTester.cc
+    	  in hammer the crushtool validation is via a shell
+        	  and not via an internal subprocess utility
+
+commit abc5b5f139b05da0be0fbaa99282be16386980db
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Sep 20 23:42:45 2015 +0200
+
+    tests: update to match crushmap validation message
+    
+    http://tracker.ceph.com/issues/13182 Fixes: #13182
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 21a1e75d8a7bad89a48cd9d36902c5d609be5015)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+    	  the test structure in hammer is different, adapt
+    	  the relevant test.
+
+commit 25bd2778f8b7f5eb5245efebca56a7348dd064eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:42:47 2015 -0400
+
+    mon/OSDMonitor: fix crush injection error message
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1b3090d50e5bd5ca3e6e396b23d2d9826896c718)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    	  hammer has a separate error message for EINTR
+    	  which is in the context of the fix for error message
+
+commit 6635530aa94b1f21c3052e64d4dfe3952f98fe83
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:41:25 2015 -0400
+
+    mon/OSDMonitor: only test crush ruleset for the newly created pool
+    
+    Otherwise, we test *all* crush rules.. which might be a lot, and
+    which is a big waste of time and effort.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 524b0bdcc45c2f4b95f2239c988e93250f337f3d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc : trivial resolution
+
+commit cc1fedda02fbf3c95d7a00f0b395db5439e96f90
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:40:13 2015 -0400
+
+    crush/CrushTester: allow testing by ruleset
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit aa238e5ed50f44a94caf84567267e4f6be8732a2)
+    
+    Conflicts:
+    	src/crush/CrushTester.cc
+              in hammer the crushtool validation is via a shell
+    	  and not via an internal subprocess utility
+    	src/tools/crushtool.cc
+    	  ceph_argparse_withint is preferred to ceph_argparse_witharg
+
+commit 065086246290cfa02f67c37c95a2ccdb05872f21
+Merge: 79385a8 3a50b90
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 6 11:11:14 2015 -0400
+
+    Merge pull request #5887 from Abhishekvrshny/wip-13044-hammer
+    
+    LibCephFS.GetPoolId failure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 79385a85beea9bccd82c99b6bda653f0224c4fcd
+Merge: 5b25fc9 3228161
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Oct 4 15:33:49 2015 +0200
+
+    Merge pull request #6172 from dachary/wip-13354-hammer
+    
+    qa/workunits/cephtool/test.sh: don't assume crash_replay_interval=45
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 322816116ea14a8fd79616b1c55545ae7c62e7f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 28 13:13:40 2015 -0400
+
+    qa/workunits/cephtool/test.sh: don't assume crash_replay_interval=45
+    
+    e.g., it's 5 in teuthology's ceph.conf.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80b7237e5e74d12b9d8b1f96ea535c96bdff9c6f)
+
+commit ad833044dee4208625213bc83c7847ce9d1a73aa
+Author: dwj192 <duanweijun@h3c.com>
+Date:   Wed Sep 9 09:48:16 2015 +0800
+
+    rgw:add --reset-regions for regionmap update
+    
+    Fixes: #12964
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit 95685c19d6f1eab50b903e61273b5351bedc2980)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc: usage is on cerr in hammer, not cout
+            src/test/cli/radosgw-admin/help.t: remove extra empty line
+
+commit 5b25fc9c8431423c6239da2eafcf7e8863d2262f
+Merge: e72bdc3 9026c4a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 3 09:39:12 2015 +0200
+
+    Merge pull request #6156 from dachary/wip-13170-hammer
+    
+    doc: update docs to point to download.ceph.com
+    
+    Reviewed-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+
+commit 7de65e7ede52f3e417c354f180cfd80f1407c6cb
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Sep 3 12:02:05 2015 +0530
+
+    rgw : setting max number of buckets for users via ceph.conf option
+    
+    This patch adds a new option "rgw_user_max_buckets" for setting
+    max number of buckets for users via ceph.conf.
+    
+    Fixes #12714
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit f65267c96cbd4cd25036b6bf399692e77bbb9436)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc: usage is on cerr in hammer, not cout
+
+commit 297c04d856c4864d21b2241ad01560afa61dde40
+Author: Xiaowei Chen <cxwshawn@gmail.com>
+Date:   Tue Sep 8 06:58:57 2015 -0400
+
+    rgw: init_rados failed leads to repeated delete
+    
+    Fixes: #12978
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit ab4232baa7bcc86e90746e13312ac9bda1772872)
+
+commit 4b0686f18f323b95adb1fc5cffe7c1e55e8f1c63
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 15:08:18 2015 -0700
+
+    rgw: delete finisher only after finalizing watches
+    
+    Fixes: #12208
+    
+    The watch error path might try to schedule a finisher work, delete finisher
+    only after watch destruction.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97aed59fb9e865d30d31d2b7f4e93fc9727c96fa)
+
+commit 6119b152412ddfa2a614c1862aad52a123a4fe8e
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+Date:   Mon Aug 24 23:11:35 2015 +0530
+
+    rgw: be more flexible with iso8601 timestamps
+    
+    make parsing 8601 more flexible by not restricting the length of seconds
+    to 5, this allows timestamp to be specified both as ms or us. Newer
+    keystone backends such as fernet token backend default to microseconds
+    when publishing iso8601 timestamps, so this allows these timestamps to
+    be allowed when specifying the token expiry time.
+    
+    Fixes: #12761
+    Reported-by: Ian Unruh <ianunruh@gmail.com>
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    (cherry picked from commit 136242b5612b8bbf260910b1678389361e86d22a)
+
+commit 607904e8d5616bd7df20e1e387a14d55e4ca43b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 14:12:02 2015 -0400
+
+    init-radosgw: specify pid file to start-stop-daemon
+    
+    This fixes restart when multiple instances are running.
+    
+    Fixes: #12407
+    Tested-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e17c8e1a7a06db0e2680dedc7e954021610320d1)
+
+commit f51ab26d8721bde6852e5c169869d7ae16bd0574
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Jul 23 18:10:06 2015 +0530
+
+    rgw: fix radosgw start-up script.
+    
+    radosgw init script is unable to start radosgw daemon.
+    as it is relies on requiretty being disabled.
+    once init script start this daemon with sudo it fails
+    to start the daemon.
+    
+    changing 'sudo' to 'su', it will fix this issue and
+    will also help running radosgw daemon with our new
+    "ceph" UID project.
+    
+    Fixes: #10927
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit c83542296aeb8f5dd96f5d9e09327e7a89370742)
+
+commit 544a98fa9dd47d02778f622de5f7ebe1acccc12f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:39:22 2015 -0700
+
+    init-radosgw: unify init-radosgw[.sysv]
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1c45f512a91ee49e9253ba6fd3857af9bd2ae57a)
+
+commit 2a733e9e720bcdb8ea342a96be679d6f9efa3b2b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:29:11 2015 -0700
+
+    init-radosgw: look in /var/lib/ceph/radosgw
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a4bb992426b39188e750fce9198052ca0d510117)
+
+commit d00c52b874ddc74aa5f4c2e07b70b881ce963114
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Sun May 10 18:17:53 2015 +0200
+
+    doc: rgw: fix typo in comments
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 05cf0db22dc3a9cb2bf4b86b8f240683c23f41ac)
+
+commit eb001d3a7275cc040bea716b8bdb0cf8b97efbc9
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Tue Mar 17 15:44:52 2015 -0700
+
+    rgw: init script waits until the radosgw stops
+    
+    Fixes: #11140
+    Init script waits in stop action until the radowgw daemons stop.
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit 1cca0c1e91a1e18f82f4d22855e96b2fc947f5ea)
+
+commit 9ab9c44583c96f1679db3dbcef322a706548a7dd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 17 09:10:39 2015 -0700
+
+    rgw: don't read actual data on user manifest HEAD
+    
+    Fixes: #12780
+    We unconditionally read all the data, which is not needed
+    when doing HEAD operation on user manifest objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 4a8b08d1dd2511e28e999f1625103fb919fb5465)
+
+commit 9026c4a7c7699a9a013d4fe207e6681403f934a6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:20:12 2015 -0400
+
+    doc: remove mention of ceph-extra as a requirement
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a6f07e9fa3bd5e50a2d8cfa387b44a39d383fa65)
+
+commit 45ed24da879fe12b921cf03e4a921ccc0582b8d2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:21 2015 -0400
+
+    doc: remove ceph-extras
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c0ef84fd22aac58b6ec8e58eb4f8dffed306d39e)
+
+commit faccdce79e003318c751dd9203773af7ed31d052
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:02 2015 -0400
+
+    doc: correct links to download.ceph.com
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 387d7800359154950431d0984c756f43f21dd9b4)
+
+commit e9f4aecf3f406b42b74f840edb398be9243da85a
+Author: ritz303 <ritz_303@yahoo.com>
+Date:   Tue Aug 25 10:02:49 2015 -0500
+
+    doc: Added "Hammer" in the list of major releases.
+    
+    Fixes: #12771
+    
+    Signed-off-by: ritz303 <ritz_303@yahoo.com>
+    (cherry picked from commit 582f0f64455079290ad3b4ae7338b716e170911f)
+
+commit 424fc1ccb244f5084f94904cb20cbf88a1bd4638
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 25 10:44:40 2015 -0700
+
+    rgw: set default value for env->get() call
+    
+    Fixes: #13239
+    
+    This fixes a regression introduced at commit abe4ec293d08b0314bf5c081ace2456073f3a22c.
+    The host var is a string, env->get() returns a char pointer, shouldn't
+    pass in NULL.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0d5730efc0054334897315dc23ba04f30548e36e)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit e72bdc3bf1541e445f74def70b5aa185f1757468
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 21 11:41:03 2015 -0400
+
+    osd/ReplicatedPG: tolerate promotion completion with stopped agent
+    
+    We may start a promotion, then get a pool update that disables the
+    agent, and then complete it.  We should not segfault in this case.
+    
+    Fixes: #13190
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fd9ce66e8b76787f69c5e337332d57aa72c8b8d5)
+
+commit a3afb3f59435050efa711436134b4abe63a8f5cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 9 14:15:10 2015 -0400
+
+    rgw: remove trailing :port from host for purposes of subdomain matching
+    
+    Some clients (ahem, CrossFTP) include the :port in the HTTP_HOST header.
+    Strip it out.
+    
+    Switch req_info field to a std::string and avoid copying it in preprocess.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit abe4ec293d08b0314bf5c081ace2456073f3a22c)
+
+commit fe0fc4f18f6f275b34e8bbfa868a7310ecc3842f
+Merge: e26f8bc 8ba6b2f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:01:26 2015 -0700
+
+    Merge pull request #5715 from dachary/wip-12592-hammer
+    
+    RGW returns requested bucket name raw in Bucket response header
+
+commit e26f8bc2dbc82b6ff31d2bd2d5c890aa12da9f4a
+Merge: 19e549e 6acf36f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:00:55 2015 -0700
+
+    Merge pull request #5719 from dachary/wip-12855-hammer
+    
+    segmentation fault when rgw_gc_max_objs > HASH_PRIME
+
+commit 19e549e9c2fbacdbf747344c1dafcc6b66902339
+Merge: 2b11a4c a13c7fd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:00:09 2015 -0700
+
+    Merge pull request #5720 from dachary/wip-12854-hammer
+    
+    the arguments 'domain' should not be assigned when return false
+
+commit 77cb503ba057f033825aa40d57dee42e838c67ac (refs/remotes/gh/wip-13015-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 9 15:41:02 2015 -0700
+
+    rgw: preserve all attrs if intra-zone copy
+    
+    Fixes: #13015
+    Intra zone copy requires that all objects' attributes are preserved.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit b9f2ed303eedfa0b747884f8e66fbe97cfeeb0d5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 13 10:59:32 2015 -0700
+
+    rgw: don't preserve acls when copying object
+    
+    Fixes: #12370
+    
+    When copying an object we need to use the acls we calculated earlier,
+    and not the source acls.
+    This was broken at e41d97c8e38bb60d7e09e9801c0179efe7af1734.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fb020247449d3daf033885bab5b3a10c0e5df4a5)
+
+commit 2b11a4c07a9416a2f75c8ce62a923994977d799c
+Merge: 0022c03 b3822f1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 15 10:23:49 2015 +0200
+
+    Merge pull request #5930 from liewegas/wip-11798-hammer
+    
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0022c035476eff471a777b23a619d024f632a1bf
+Merge: 23fb811 0d6a8c6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 14 23:57:13 2015 +0200
+
+    Merge pull request #5908 from liewegas/wip-simple-connect-hammer
+    
+    Pipe: Drop connect_seq increase line
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit b3822f113e07547194b844f647bcb7d45513b25f (refs/remotes/me/wip-11798-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 14:57:34 2015 -0400
+
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    It may take tens of seconds to restart each time, so 5 in 30s does not stop
+    the crash on startup respawn loop in many cases.  In particular, we'd like
+    to catch the case where the internal heartbeats fail.
+    
+    This should be enough for all but the most sluggish of OSDs and capture
+    many cases of failure shortly after startup.
+    
+    Fixes: #11798
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eaff6cb24ef052c54dfa2131811758e335f19939)
+
+commit 0d6a8c631f55367ac4f6f792a10ba62cd97e9fed (refs/remotes/me/wip-simple-connect-hammer)
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Feb 25 23:38:54 2015 +0800
+
+    Pipe: Drop connect_seq increase line
+    
+    Revert commit 0fc47e267b6f8dcd4511d887d5ad37d460374c25.
+    
+    When accepting and "connect.connect_seq == existing->connect_seq",
+    existing->state maybe STATE_OPEN, STATE_STANDBY or STANDY_CONNECTING.
+    This commit only fix partial problem and want to assert
+    "(existing->state == STATE_CONNECTING)".
+    
+    So later we added codes to catch
+    "(existing->state == STATE_OPEN || existing->state == STATE_STANDBY)"
+    before asserting.
+    
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 67225cb3ee1f6d274a02293724942bdb25cec8ca)
+
+commit 23fb811303971152f1c348e2a02de4e2bc6ed53d
+Merge: c1849ec 6c4ccc8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 13 14:10:23 2015 +0200
+
+    Merge pull request #5767 from dachary/wip-12597-hammer
+    
+    Crash during shutdown after writeback blocked by IO errors
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c1849ecfb2ae55b99b6db75ee49af19886c246b2
+Merge: f35c53d f028389
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 13 07:42:36 2015 -0400
+
+    Merge pull request #5687 from liewegas/wip-hammer-feature-hammer
+    
+    include/ceph_features: define HAMMER_0_94_4 feature
+
+commit f35c53df7c155d7ff4a316a0817b11bc018c57ca
+Merge: 6e22620 4be8a28
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 09:23:13 2015 -0400
+
+    Merge pull request #5892 from ceph/wip-13060-hammer
+    
+    osd: allow peek_map_epoch to return an error
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 4be8a28d5b5ba868d100300ab64ebb82b188222b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 11 11:00:38 2015 -0400
+
+    osd/PG: peek_map_epoch: skip legacy PGs if infos object is missing
+    
+    - pg is removed
+    - osd is stopped before pg is fully removed
+    - on restart, we ignore/skip the pg because its epoch is too old
+    - we upgrade to hammer and convert other pgs, skipping this one, and then
+      remove the legacy infos object
+    - hammer starts, tries to parse the legacy pg, and fails because the infos
+      object is gone, crashing.
+    
+    The fix is to continue ignoring the zombie pg.
+    
+    Fixes: #16030
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f237ed97228839a1b412ad213945f0343df05bf5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 11 17:41:01 2015 -0400
+
+    osd: allow peek_map_epoch to return an error
+    
+    Allow PG::peek_map_epoch to return an error indicating the PG
+    should be skipped.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f15d9585edc5a12ac2d076951076247253b897c2)
+    
+    [fixed *pepoch default of 0]
+    [fixed other return paths in peek_map_epoch]
+
+commit 3a50b904a2ffd613b695ead1b26c93278044d7df
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Aug 5 15:19:13 2015 +0800
+
+    crypto: fix unbalanced ceph::crypto::init/ceph::crypto:shutdown
+    
+    we may create a CephContext without calling common_init_finish(), then
+    delete the CephContext. In this case, ceph::crypto:init() is not called,
+    so CephContext::~CephContext() should not call ceph::crypto::shutdown().
+    
+    Fixes: #12598
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 98a85ec6464d8ec3fc7f0c647ac97c8cf002ebe2)
+
+commit 0a5b8569ecc6efa42b3d4fa7a58c155b3c2dbbe8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jul 30 16:59:32 2015 -0700
+
+    ReplicatedPG,Objecter: copy_get should include truncate_seq and size
+    
+    Otherwise, we break CephFS over cache tiers.
+    
+    Fixes: #12551
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6f9ee7961eee9ee3a61a07cbe0d8d289ee98fa9a)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit 6e22620165872a88ab4b75ed2fb93f61beb65d12
+Merge: 508733f 00e73ad
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 9 19:44:09 2015 -0400
+
+    Merge pull request #5769 from dachary/wip-12850-hammer
+    
+    Crash during TestInternal.MultipleResize
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 508733fd48f2c704b79ad36841346965f4d0429b
+Merge: 69a320e dc693fc
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 9 19:44:01 2015 -0400
+
+    Merge pull request #5768 from dachary/wip-12849-hammer
+    
+    [  FAILED  ] TestLibRBD.BlockingAIO
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 82ea02ab8d886c16c511c4bd7d03d826158f954b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 17 11:35:18 2015 -0700
+
+    rgw: fix assignment of copy obj attributes
+    
+    Fixes: #11563
+    Clarify the confusing usage of set_copy_attrs() by switching the source and
+    destinatiion params (attrs, src_attrs). Switch to use attrs instead of
+    src_attrs afterwards. In one of the cases we originally used the wrong
+    variable.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e41d97c8e38bb60d7e09e9801c0179efe7af1734)
+
+commit 3b2affce1a229cead3e0ffacfdfde69258a267cb
+Author: Sylvain Baubeau <sbaubeau@redhat.com>
+Date:   Fri Sep 4 22:51:44 2015 +0200
+
+    rgw: add delimiter to prefix only when path is specified
+    
+    http://tracker.ceph.com/issues/12960
+    Fixes: #12960
+    
+    Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
+    (cherry picked from commit 27cf257248ea55f8f4bc7851c3956611828bcae2)
+
+commit 69a320e185f20da0824d0a62d3de77578c431a7a
+Merge: 8039375 0bc909e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 9 16:58:56 2015 +0200
+
+    Merge pull request #5697 from tchaikov/wip-12638-hammer
+    
+    mon: add a cache layer over MonitorDBStore
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 8039375327b5074a700193aae2d8b1922ca85398
+Merge: 19ff928 256620e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 9 14:52:54 2015 +0200
+
+    Merge pull request #5381 from dachary/wip-12499-hammer
+    
+    ceph-fuse 0.94.2-1trusty segfaults / aborts
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 19ff92806fd1e0fb866737f58e379aa8078b8017
+Merge: 7614bf7 059bf98
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 14:58:16 2015 -0700
+
+    Merge pull request #5757 from dachary/wip-12836-hammer
+    
+    WBThrottle::clear_object: signal on cond when we reduce throttle values
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 7614bf7d117fdd476931cdd8825265321cef3c4c
+Merge: 2100631 bf72785
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:06:30 2015 +0200
+
+    Merge pull request #5759 from dachary/wip-12841-hammer
+    
+    recursive lock of md_config_t (0)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 210063120708a7d6502c2b8d3e8c61731e37de5b
+Merge: a2e1fe2 836f763
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:05:41 2015 +0200
+
+    Merge pull request #5761 from dachary/wip-12843-hammer
+    
+    long standing slow requests: connection->session->waiting_for_map->connection ref cycle
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a2e1fe26f602d0165999dfb72b97985f9bfe38ce
+Merge: 8ccb771 2348a5b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:04:51 2015 +0200
+
+    Merge pull request #5762 from dachary/wip-12844-hammer
+    
+    osd suicide timeout during peering - search for missing objects
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8ccb7711d68dd3b95d2522c6e46f5021362c9088
+Merge: e9db807 77624af
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:04:03 2015 +0200
+
+    Merge pull request #5763 from dachary/wip-12846-hammer
+    
+    osd/PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e9db8075034dc4ac0ffe5d8dd9dd2153509237fa
+Merge: 5ef999e aa00373
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:03:10 2015 +0200
+
+    Merge pull request #5764 from dachary/wip-12847-hammer
+    
+    common: do not insert emtpy ptr when rebuild emtpy bufferlist
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ef999e4a1dc04d7b7e58b1a61f85511099fd6d4
+Merge: c243f7a cd11b88
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:02:14 2015 +0200
+
+    Merge pull request #5373 from dachary/wip-12489-hammer
+    
+    pg_interval_t::check_new_interval - for ec pool, should not rely on min_size to determine if the PG was active at the interval
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c243f7a33b0fff261bb92595a3cc7abf12b7e56d
+Merge: e61fdc6 7bddf5d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:00:32 2015 +0200
+
+    Merge pull request #5383 from dachary/wip-12504-hammer
+    
+    rest-bench common/WorkQueue.cc: 54: FAILED assert(_threads.empty())
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e61fdc67031b0358800335b638c96d22ecb85bd6
+Merge: 62f543b 9f69660
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:54:07 2015 +0200
+
+    Merge pull request #5765 from dachary/wip-12883-hammer
+    
+    cache agent is idle although one object is left in the cache
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 62f543bbb092554746d3b89aab4b1ea75a5cdbf1
+Merge: f25315c e471c5d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:53:14 2015 +0200
+
+    Merge pull request #5754 from dachary/wip-12588-hammer
+    
+    Change radosgw pools default crush ruleset
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f25315c1e7e6e4700c2f39ba849dbcb25b0f1f89
+Merge: 021abe7 43a72e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:51:50 2015 +0200
+
+    Merge pull request #5377 from dachary/wip-12396-hammer
+    
+    register_new_pgs() should check ruleno instead of its index
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 021abe742ccec876497468ac556b9dfaf6c9503d
+Merge: 4125196 c94fd92
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 6 21:07:38 2015 -0400
+
+    Merge pull request #5758 from dachary/wip-12839-hammer
+    
+    Mutex Assert from PipeConnection::try_get_pipe
+
+commit 9f69660133f9534722589801a6fc827b74bead44
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 19:41:47 2015 +0200
+
+    tests: tiering agent and proxy read
+    
+    Verify that an object promoted to a cache tier because of a proxy read
+    is evicted as expected.
+    
+    http://tracker.ceph.com/issues/12673 Refs: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7924231930732bd297d3bd034c8295e96cb81088)
+
+commit 5656eec0000e95ea790c796a9f572c7261163cfa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 13:47:24 2015 +0200
+
+    osd: trigger the cache agent after a promotion
+    
+    When a proxy read happens, the object promotion is done in parallel. The
+    agent_choose_mode function must be called to reconsider the situation
+    to protect against the following scenario:
+    
+      * proxy read
+      * agent_choose_mode finds no object exists and the agent
+        goes idle
+      * object promotion happens
+      * the agent does not reconsider and eviction does not happen
+        although it should
+    
+    http://tracker.ceph.com/issues/12673 Fixes: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e1f58feb9b1d20b72f2eb2eefdea5982e0cddccd)
+
+commit 4125196d31de8933656561b2f860e16a546ec4b9
+Merge: 47af509 f47ba4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 23:17:22 2015 +0200
+
+    Merge pull request #5276 from dachary/wip-11824-hammer
+    
+    implicit erasure code crush ruleset is not validated
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 47af509003e551238702858ccd845b930e5ef1ed
+Merge: 2fd8ec8 1e05578
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 17:24:43 2015 +0200
+
+    Merge pull request #5382 from dachary/wip-12500-hammer
+    
+    segfault launching ceph-fuse with bad --name
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2fd8ec8385a90daaa1bdb9de189ec73c7ba49c6c
+Merge: e6bdae9 fa19474
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 17:23:19 2015 +0200
+
+    Merge pull request #5367 from dachary/wip-12311-hammer
+    
+    read on chunk-aligned xattr not handled
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit dc693fcaf2c596087b434f9e90fbc553394b9a14
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 29 12:46:24 2015 -0400
+
+    lockdep: allow lockdep to be dynamically enabled/disabled
+    
+    librbd test cases attempt to enable lockdep coverage via the librados
+    API.  Use a configuration observer to register/unregister lockdep
+    support.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af0cade0293be22e74a3293765676c7b227b06c4)
+
+commit 805732b143fd28603238969ba784adb57eff1a12
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 28 14:23:14 2015 -0400
+
+    tests: librbd API test cannot use private md_config_t struct
+    
+    Remove all depencencies on md_config_t and instead use librados API
+    methods to get/set configuration values.
+    
+    Fixes: #12479
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c51aadb447d3447b5dace19c5cb6268e94247c8)
+
+commit 7ac01731bcb0f1d46cf98ecf623c7e8e6cb1a38e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 15:06:27 2015 -0400
+
+    tests: ensure old-format RBD tests still work
+    
+    Override the RBD default image format back to version 1
+    to ensure tests properly cover the old format.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3b95edb5bd373bd05f83516eccfd30a6b5cf4693)
+
+commit b68d7574d12944589b246281474bcad01f5347be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 09:00:57 2015 -0400
+
+    librados_test_stub: implement conf get/set API methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4d03c664f2a9362009c7e3077ed65e2989f0f21b)
+
+commit f0fa637e4a91a93a326ba303bc22101fadcc787d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 28 13:14:29 2015 -0400
+
+    crypto: use NSS_InitContext/NSS_ShutdownContex to avoid memory leak
+    
+    Switched to context-aware NSS init/shutdown functions to avoid conflicts
+    with parent application.  Use a reference counter to properly shutdown the
+    NSS crypto library when the last CephContext is destroyed.  This avoids
+    memory leaks with the NSS library from users of librados.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9fa0112dd9286178da1d76020158503b2062d252)
+
+commit 3f542aa2410139bece0f4192079e4e34260da782
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Mar 21 07:13:51 2015 +0800
+
+    auth: use crypto_init_mutex to protect NSS_Shutdown()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit cf058174fe6be26129eb52f4fb24d0f7146ef7af)
+
+commit e487e8e3d84c7845ce7824ace3e375c6c389ba20
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Mar 21 01:02:42 2015 +0800
+
+    auth: reinitialize NSS modules after fork()
+    
+    Fixes: #11128
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4c24d0cc074462ae258b5cf901cd884bb4f50a53)
+
+commit 00e73adb03ef4453599268779fe354e4e8513e54
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 11 09:26:33 2015 -0400
+
+    librbd: prevent race condition between resize requests
+    
+    It was possible that the same resize request could be sent twice
+    if a completed resize op started a newly created resize op while
+    it was also being concurrently started by another thread.
+    
+    Fixes: #12664
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6ec431bac1cf9446f3af90209d193dfcf003f2c5)
+
+commit 6c4ccc854fa8a8403b03785b06cb35a7174f4f42
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 9 14:23:23 2015 +0800
+
+    librbd: Add a paramter:purge_on_error in ImageCtx::invalidate_cache().
+    
+    If bh_write met error, it will try again. For closing image, if met this
+    issue, it will trigger a assert:
+    >>2015-02-03 15:22:49.198292 7ff62d537800 -1 osdc/ObjectCacher.cc: In
+    function 'ObjectCacher::~ObjectCacher()' thread 7ff62d537800 time
+    >>2015-02-03 15:22:49.195927osdc/ObjectCacher.cc: 551: FAILED
+    >>assert(i->empty())
+    
+    Now add purge_on_error, when shutdown_cache it set true.
+    In ImageCtx::invalidate_cache, if met error and purge_on_error is true,
+    purge the dirty bh.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 35def5c81f7fc83d55d18320e4860c6a63d4c7f5)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc : trivial resolution
+
+commit 05734916ab119c6d3879c2ce2bc9f9581907861a
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 11:28:50 2015 +0800
+
+    librbd: Remvoe unused func ImageCtx::read_from_cache.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 101440a41253680770f94bc380af7072c7adaebf)
+
+commit 28838f20a722b72a6e926e53f104342d3b9f4791
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 11:03:10 2015 +0800
+
+    osdc: clean up code in ObjectCacher::Object::map_write
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 9f80c2909ace09cd51c24b49c98a093e0e864dca)
+
+commit 5c4f152efa8e8f57c59ea7decc05ae1a34f2a9ee
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 10:13:06 2015 +0800
+
+    osdc: Don't pass mutex into ObjectCacher::_wait_for_write.
+    
+    Because the mutex is the same as ObjectCacher::lock.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d7cf7aeea5cba1ffa8e51ff1ad2424b1ec161a12)
+
+commit 86e7698a19745c26b9d3e7a12a16c87ea9a5d565
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 2 11:23:44 2015 +0800
+
+    osdc: After write try merge bh.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 1a48a8a2b222e41236341cb1241f0885a1b0b9d8)
+
+commit c96541ad19fb142ed31ff3006f1dc25e0c1de86c
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 2 11:36:24 2015 +0800
+
+    osdc: Make last missing bh to wake up the reader.
+    
+    Avoid wakeup early and wait again.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d582bda090b3339d03e25b0e6d0971ad0267f476)
+
+commit 4135b9a2d199583685a1bae3713347dcc1b872e6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Feb 6 09:53:36 2015 +0800
+
+    osdc: For trust_enoent is true, there is only one extent.
+    
+    Now the judgement only in conditon which will return -ENOENT.
+    But o->exists don't depend on the extent size. It only depend on
+    trust_enoent. So move this judgement at the first of _readx().
+    Make this bug ASAP occur.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 2449fddc13b5ce8bbf1bb4ecaa5d6937f54e54d1)
+
+commit 81376b6a1a5db9f8b45c58ea318ec924e932d990
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Feb 3 14:46:39 2015 +0800
+
+    osdc: In _readx() only no error can tidy read result.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 540346d4a901d8041c9fd74641c98cdfd2e1ab32)
+
+commit e6bdae903cdd2f702829adc31fa9a31c239a3b87
+Merge: 317770d cdde626
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 15:38:43 2015 -0600
+
+    Merge pull request #5223 from SUSE/wip-12305-hammer
+    
+    ceph.spec.in: running fdupes unnecessarily
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit e80bd0a2a39afdabcb134100e0cc8fa74cbd14c2
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jun 25 18:28:50 2015 +0200
+
+    rgw: send Content-Length in response for GET on Swift account.
+    
+    Fixes: #12158
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5f47b11791cd46ce2771046e4888575869cc9685)
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+
+commit 2e542456dee6c30b3200e9830d4ccae4b0325388
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 22 13:51:49 2015 +0200
+
+    rgw: force content_type for swift bucket stats request
+    
+    Fixes: 12095
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit da259e4da1329e1e448d75c48c6ee3d1318e19b2)
+
+commit 5d57b63c32e43000ff95c2d8ce34347ed0579a8a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 13 15:02:03 2015 +0200
+
+    rgw: we should not overide Swift sent content type
+    
+    Fixes: #12363
+    backport: hammer
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 423cf136f15df3099c9266f55932542d303c2713)
+
+commit b8aafbc6f9f80f029606a577d73f8685a80225b8
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jun 23 11:54:09 2015 +0200
+
+    rgw: enforce Content-Type in Swift responses.
+    
+    Swift sends Content-Type HTTP header even if the response
+    doesn't contain body. We have this behaviour implemented
+    until applying some changes in end_header() function.
+    Unfortunately, lack of Content-Type causes early exits in
+    many Tempest's tests for Swift API verification.
+    
+    Fixes: #12157
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 106aeba206736d4080326f9bc191876bed63370b)
+
+commit 143cfc331efcc7bed8b1cf90d7e3188d0ac7604b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Apr 21 12:31:19 2015 +0200
+
+    rgw: force content-type header for swift account responses without body
+    
+    This regression was introduced in commit 4cfeca03ade21861ab70ca759f94eb244c16cb39
+    
+    Fixes: #11438
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 009664e560ee37ef6dc170a0155d6c749e151b06)
+
+commit b5420d6f1a6a1da34c7bba339e6763c4b369d349
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 9 20:04:59 2015 +0200
+
+    rgw: shouldn't return content-type: application/xml if content length is 0
+    
+    Fixed: #11091
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5cc92bb27cfbaee4af24dc312c9b98331f899794)
+
+commit 317770dd39510221cbc0fc7ee50960992137eaaf
+Merge: 6b02be7 6b36514
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Sep 3 12:20:38 2015 +0200
+
+    Merge pull request #5716 from dachary/wip-12851-hammer
+    
+    Ensure that swift keys don't include backslashes
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6b02be7395ad8f41f70458e7bd11630ddd498cc5
+Merge: fa74ea1 f1c7c62
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Sep 3 12:11:24 2015 +0200
+
+    Merge pull request #5717 from dachary/wip-12591-hammer
+    
+    rgw: create a tool for orphaned objects cleanup
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit fa74ea10ac25fe93f359ccf0fd036d567e9610fe
+Merge: 011e4c5 a478385
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:35:58 2015 +0200
+
+    Merge pull request #5755 from dachary/wip-12589-hammer
+    
+    ceph-disk zap should ensure block device
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 011e4c58ffb41aa5ae3240a40cbeba8266b89001
+Merge: e7c2269 b80859e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:22:59 2015 +0200
+
+    Merge pull request #5732 from ceph/wip-11455-hammer
+    
+    rgw: init some manifest fields when handling explicit objs
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e7c2269dbebd5337ae5aad127d6ddaa4260a8f53
+Merge: 92b641f da00bed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:19:02 2015 +0200
+
+    Merge pull request #5721 from dachary/wip-12853-hammer
+    
+    RGW Swift API: X-Trans-Id header is wrongly formatted
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 92b641f301df33a38ccfcf018c5f7657db8548ae
+Merge: 845d2ae 3ab5d82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:08:24 2015 +0200
+
+    Merge pull request #5498 from ceph/wip-12432-hammer
+    
+    rgw: HTTP return code is not being logged by CivetWeb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 836f7636f76b2778812e4fa055206a887fd27dad
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 21 11:31:12 2015 -0700
+
+    OSD: break connection->session->waiting message->connection cycle
+    
+    Fixes: #12338
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7f15e22be291df139c85909b96b2366f6dbadf37)
+
+commit 77624aff03a8df7805ac0c1c6a7a2085f41fb6a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 9 10:46:10 2015 -0400
+
+    osd/PGLog: dirty_to is inclusive
+    
+    There are only two callers of mark_dirty_to who do not pass max,
+    and they are both in the merge_log extending tail path.  In that
+    case, we want to include the last version specified in the log
+    writeout.  Fix the tail extending code to always specify the
+    last entry added, inclusive.
+    
+    Fixes: #12652
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f0ca14df0641daa04eee39d98d8bd0faf46e4e6d)
+
+commit aa003736d2aeea937ce0d8663b159cf06b2dda3e
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 23:02:10 2015 +0800
+
+    common: fix code format
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 15e5ebe14787bbbc6b906d72ccd2d418d762cfad)
+
+commit aab35da90d7d50750fda6355d5a124a45d66f1c5
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 22:59:40 2015 +0800
+
+    test: add test case for insert empty ptr when buffer rebuild
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 2d2f0eb338906742d516a2f8e6d5d05505be8f47)
+
+commit 2b0b7ae6443eeca271813415a91ab749c886fdf0
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 23:01:27 2015 +0800
+
+    common: fix insert empty ptr when bufferlist rebuild
+    
+    Fixes: #12775
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit fb1b6ddd7bfe40d4a38d5ec59b1c3f59f2a83f23)
+    
+    Conflicts:
+    	src/common/buffer.cc : because invalidate_crc did not exist
+            in hammer
+
+commit 845d2aed9e9a9893678226b8661fbd8ff87c090f
+Merge: a6517d2 3396a96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 12:00:11 2015 +0200
+
+    Merge pull request #5527 from SUSE/wip-12585-hammer
+    
+    OSD crash creating/deleting pools
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2348a5b5c96d065fbc2188bd9e8e7ff20c75ecaa
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Wed Jul 29 21:47:17 2015 +0000
+
+    osd: copy the RecoveryCtx::handle when creating a new RecoveryCtx instance from another one
+    
+    Fixes: 12523
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 6ab9efe799b947f2ef6113df8cc542c73760107d)
+
+commit bf72785e1e11b18df9a03ebfa7c284fe4f43eaa0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 24 15:40:39 2015 -0700
+
+    config: skip lockdep for intentionally recursive md_config_t lock
+    
+    lockdep can't handle recursive locks, resulting in false positive
+    reports for certain set_val_or_die() calls, like via
+    md_config_t::parse_argv() passed "-m".
+    
+    Fixes: #12614
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 415226995ebe841cc9e8639a3233150a75aecaed)
+
+commit c94fd926239a31dea1389a760f55f332a9b6814a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jul 23 16:36:19 2015 -0700
+
+    osd: Keep a reference count on Connection while calling send_message()
+    
+    Fixes: #12437
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a140085f467889f2743294a3c150f13b62fcdf51)
+
+commit 059bf98d40fa7e1eddb7aef2ecfdfb6bc96c14fb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 08:49:54 2015 -0700
+
+    WBThrottle::clear_object: signal if we cleared an object
+    
+    Fixes: #12223
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8447b08d60d5cf7dce20f8b839cc2cb5a93782de)
+
+commit a478385334323071b5cfa4d546eef5b0ef43dce7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun May 31 19:42:45 2015 +0200
+
+    ceph-disk: always check zap is applied on a full device
+    
+    zap only makes sense on a full device and this is verified for
+    
+        ceph-disk prepare --zap-disk
+    
+    but not for
+    
+        ceph-disk zap
+    
+    Unify the two and add a test to check that it fails as expected when
+    trying to zap a directory.
+    
+    http://tracker.ceph.com/issues/11272 Fixes: #11272
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e1bcf21f090fc98ac9b495d76614e3012f5872a)
+    
+    Conflicts:
+    	src/test/ceph-disk.sh : trivial resolution
+
+commit e471c5dc16847ba35d3c3c8c74da1bf7c4c946e0
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Wed Jul 8 10:35:49 2015 +0800
+
+    librados: Make librados pool_create respect default_crush_ruleset
+    
+    Make pool_create in librados use the osd_pool_default_crush_replicated_ruleset
+    
+    Fixes: #11640
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit da96a89033590277460aef1c80f385bd93d625e1)
+
+commit a6517d286b59eac283d7bfd04b620f5b859657b6
+Merge: a89c173 35fa47a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 02:19:26 2015 +0200
+
+    Merge pull request #5551 from ceph/wip-corpus-hammer
+    
+    ceph-object-corpus: add 0.94.2-207-g88e7ee7 hammer objects
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 35fa47a117a3e8d185f7ebe02eea5671f8229427 (refs/remotes/me/wip-corpus-hammer, refs/remotes/gh/wip-corpus-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 1 17:44:06 2015 -0400
+
+    ceph-object-corpus: add 0.94.2-207-g88e7ee7 hammer objects
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit b80859e2b2efc2402dc62aa319e40319863586f7 (refs/remotes/gh/wip-11455-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 14:34:30 2015 -0700
+
+    rgw: init some manifest fields when handling explicit objs
+    
+    Fixes: #11455
+    When dealing with old manifest that has explicit objs, we also
+    need to set the head size and head object correctly so that
+    code that relies on this info doesn't break.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7bf8cb594e5276d1c80544f5ec954d52b159750)
+
+commit f47ba4b1a1029a55f8bc4ab393a7fa3712cd4e00
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jul 16 18:02:02 2015 +0200
+
+    mon: test the crush ruleset when creating a pool
+    
+    We want to fix the following scenario:
+    
+     * an erasure code plugin (or another part of the code) creates a
+       ruleset
+     * the ruleset crashes during mapping (for whatever reason)
+     * ceph osd pool create uses the bugous ruleset
+     * the monitors try to do mapping a crash
+    
+    Having a bugous ruleset in the crush map is very difficult prevent. The
+    catastrophic event of using it with a newly created pool can however be
+    prevented by calling the CrushTester just before creating the pool and
+    after all implicit or explicit crush ruleset creation happened.
+    
+    http://tracker.ceph.com/issues/11814 Fixes: #11814
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f1e86be589803596e86acc964ac5c5c03b4038d8)
+    
+    Conflicts:
+    	src/test/mon/osd-crush.sh
+              removed the run_mon because in hammer it is shared
+              between all tests
+            src/mon/OSDMonitor.cc
+              prepare_new_pool changed stringstream to *ostream
+
+commit b58cbbab4f74e352c3d4a61190cea2731057b3c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat May 30 12:40:26 2015 +0200
+
+    erasure-code: set max_size to chunk_count() instead of 20 for shec
+    
+    See 8b64fe93b088a3a33d357869c47e6bf928c3f0e4 for a detailed explanation.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a44499f7e6609d9a32254dc6f4303122de74db31)
+
+commit 6f0af185ad7cf9640557efb7f61a7ea521871b5b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 26 21:22:31 2015 +0200
+
+    vstart.sh: set PATH to include pwd
+    
+    ceph-mon needs crushtool to be in PATH. Don't set if it is run
+    from ceph_vstart_wrapper, which already sets it as it needs.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e08bccf377292d72463796ae1dff5d8be40d88e9)
+
+commit a89c1731e0e327acdb876d39df574e27379aaa1e
+Merge: c3a2bf3 0fde3a2
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Aug 31 09:41:33 2015 +0100
+
+    Merge pull request #5365 from dachary/wip-12491-hammer
+    
+    buffer: critical bufferlist::zero bug
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit c3a2bf3f6b1d1bf416fa04df1629630abe2fe563
+Merge: 0a44eb1 4457d3e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:58:25 2015 +0200
+
+    Merge pull request #5369 from dachary/wip-12498-hammer
+    
+    get pools health'info have error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0a44eb12fc7d2ef902f1999e37abae0f11511d66
+Merge: dfcdba7 9916d37
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:52:52 2015 +0200
+
+    Merge pull request #5370 from dachary/wip-12496-hammer
+    
+    pgmonitor: wrong at/near target max“ reporting
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dfcdba75f310eaa2cfc5ae97ba3093afe1198737
+Merge: 69a9a4c b8176d0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:44:56 2015 +0200
+
+    Merge pull request #5378 from dachary/wip-12394-hammer
+    
+    Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 69a9a4cbf28c87fcd68f6c6c0c09f7a6d95ab003
+Merge: d3b160e 42bff0b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:44:26 2015 +0200
+
+    Merge pull request #5372 from dachary/wip-12493-hammer
+    
+    the output is wrong when runing ceph osd reweight
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit da00bedbc2f5d2facb9472078fd6cdd7260492ac
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Aug 6 15:52:58 2015 +0200
+
+    rgw: rework X-Trans-Id header to be conform with Swift API.
+    
+    Fixes: #12108
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d1735a4e8ea2056d333a5001a7a410fc03fe1b9e)
+
+commit 9937c818f1073bceffcf5811c7cc543b9043e646
+Author: Abhishek Dixit <dixitabhi@gmail.com>
+Date:   Mon Jun 8 22:59:54 2015 +0530
+
+    Transaction Id added in response
+    
+    Signed-off-by: Abhishek Dixit dixitabhi@gmail.com
+    
+    (cherry picked from commit b711e3124f8f73c17ebd19b38807a1b77f201e44)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+                 the prototype of the end_header( function
+                 in the context of the diff changed
+
+commit f1c7c629f2a7baf87a3d25d68b7a40fb3b2f6c49
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:35:04 2015 -0700
+
+    rgw: api adjustment following a rebase
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7cf1f376e7dbc98a44f4a8e78bfa11f84291a941)
+
+commit 85911df5226dccfd2f1806a204a23eae3966eee7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:34:44 2015 -0700
+
+    rgw: orphans, fix check on number of shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 47edec359be2a4d230a4a54c0c670d0aa711fa1e)
+
+commit c1cf7dfb672032012889840fb4462b1a076f34c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:34:11 2015 -0700
+
+    rgw: orphans, change default number of shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5528f21d0861d42378a8eb10759a1e55ddd66c09)
+
+commit bb1d4cc3a3e9af3bf8cd8fd80085fbbbaeb8b89f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 5 14:43:05 2015 -0700
+
+    rgw: change error output related to orphans
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cac57caf639236adc9a641a85352eb729b58725a)
+
+commit 2e0f6fe98787e1f3ea7d107a814caadce7ee5f23
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 17:02:29 2015 -0700
+
+    rgw: orphan, fix truncated detection
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d8ce04db8542dae0027423f3c50355e5de74dc9a)
+
+commit 1bfebefba2e09448d28853b34415d3f65e323c76
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 16:32:57 2015 -0700
+
+    radosgw-admin: simplify orphan command
+    
+    No need for --init-search
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4980cbd04ff650d79a033bc5298e6c1abe985133)
+
+commit f244b159ac8793c11a5707ea0b104b3180efb58f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 15:24:00 2015 -0700
+
+    radosgw-admin: stat orphan objects before reporting leakage
+    
+    We don't want to report new objects as leaked, because they might just
+    be written, and waiting to be linked to a new object. The number of
+    seconds prior to the job initialization can be cofigured through
+    --orphan-stale-secs (by default it's 24h).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66b0090023859d97da2bbaca698b6a50c225abd0)
+
+commit f80e2b2bc8879d1cc40338816196f51378aaa785
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 14:39:20 2015 -0700
+
+    radosgw-admin: orphans finish command
+    
+    A command to remove orphans data
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 55d6f5ecf18f532c7f056f8b60c101843594b00c)
+
+commit 88d32c641a06823df93f0e3fae640a880778e1e4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 17:28:30 2015 -0700
+
+    rgw: cannot re-init an orphan scan job
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f8993102731518153bb18eb74c3e6e4943a5dbac)
+
+commit 80a40342b6a0d9e7a47ca4b672901315246385df
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 16:38:08 2015 -0700
+
+    rgw: stat_async() sets the object locator appropriately
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7d117121858155acc8548908e9459a48c676dd1)
+
+commit 0082036348e0463e42520504466961eb2bd77602
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 16:34:09 2015 -0700
+
+    rgw: list_objects() sets namespace appropriately
+    
+    list_objects() now uses parse_raw_oid(), so that it can set the correct
+    namespace. It only affects users of the function that want to get all
+    objects in bucket, regardless to the namespace associated with it. This
+    makes it so that the orphan code actually works now with namespaced
+    objects, and with special named objects (namely, start with underscore).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20bd490bebf9be3553d4e25322c9544b0b090086)
+
+commit 1c3707291e83771ca1d5cd24ce445f723006b744
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 1 17:23:44 2015 -0700
+
+    rgw: modify orphan search fingerprints
+    
+    Now works with multipart uploads.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fa61ea2ccc94f04afb39868ac34abc264806d8d4)
+
+commit ef81367a1eaeb7778dc60a668b9fe2f6a77e06ff
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 1 15:17:10 2015 -0700
+
+    rgw: compare oids and dump leaked objects
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1bc63d98ff8a3aa180065153690b4f8a73658b79)
+
+commit f4d0544e63f68fa94b21fda71c0b78edd05038a1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 30 16:17:54 2015 -0700
+
+    rgw: keep accurate state for linked objects orphan scan
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f19b2f087cfb5a89edf3b19487c0df3a214e350f)
+
+commit 748ea57cfeca7b295afca5b8126d454eb85550c7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 17:12:34 2015 -0700
+
+    rgw: iterate over linked objects, store them
+    
+    only keep part of the oid name if it is in a namespace.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 75902fdb12ce0413d3beb455df47134f3d7386e7)
+
+commit 6c6aa5ddf38102012e61f79f5234b9647d42b461
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 17:12:00 2015 -0700
+
+    rgw: add rgw_obj::parse_raw_oid()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7c1aa8373a090d5c1b2aa121582547723120a079)
+
+commit 62d562d76e0456cc3c735c6708531c7deb2874da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 14:50:15 2015 -0700
+
+    rgw: iterate asynchronously over linked objects
+    
+    Read objects manifest. So that we could keep the relevant info later.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 05a953d965a0a6a0e9feddaa62f7429e154e828e)
+
+commit 00ecf2d0a895407bc8ec0e55e1b1b0f47a1aa96f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 14:15:33 2015 -0700
+
+    rgw: async object stat functionality
+    
+    An async functionality that stats object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 13adf3cf4f5fd8a0feb7cc1d7a4ee4ded9f573db)
+
+commit 7d1cc48ba32e8bc732c43ecb539312c6bfcc0f80
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 28 16:45:49 2015 -0700
+
+    rgw-admin: build index of bucket indexes
+    
+    for the orphan search tool
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 16a2dbd838b69fb445c06122a68fc65ec3a7b3de)
+
+commit c1b0e7a985cc9e742dcef4c8c427c57a2f58fa43
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Apr 25 09:37:53 2015 -0700
+
+    rgw: initial work of orphan detection tool implementation
+    
+    So far doesn't do much, iterate through all objects in a specific pool
+    data, store it in a sharded index.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 767fc29aa62a0a4e4dd8a78785576e8f1b44ee5b)
+
+commit b16129c3393d6e301b4af5203ffa1c6d8dfdda38
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Apr 29 13:35:29 2015 +0530
+
+    Avoid an extra read on the atomic variable
+    
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit 7476e944af049aefdb8871cce62883b86f6b6f8d)
+
+commit 1f6916d5a9b0ebe5181b801c4c381a699883bb09
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Apr 8 18:53:14 2015 +0530
+
+    RGW: Make RADOS handles in RGW to be a configurable option
+    
+    Having a configurable number of RADOS handles is resulting
+    in significant performance boost for all types of workloads.
+    Each RGW worker thread would now get to pick a RADOS handle
+    for its lifetime, from the available bunch.
+    
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit b3c3a24fe9c0fbb2bdbbccb08b40efc1dca207cc)
+
+commit d3b160ecbe181c2b969a14825cf314a056630188
+Merge: 7357659 695f782
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 15:50:03 2015 +0200
+
+    Merge pull request #5374 from dachary/wip-12487-hammer
+    
+    ceph osd crush reweight-subtree does not reweight parent node
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7357659b59d9bf3522bf664f1a3da7475a7c8a49
+Merge: 8f559d5 8a2ad05
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 15:49:41 2015 +0200
+
+    Merge pull request #5380 from dachary/wip-12390-hammer
+    
+    PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a13c7fd7c39e83072f2975477e1cdcfe079fb041
+Author: Ruifeng Yang <149233652@qq.com>
+Date:   Sat Aug 1 17:39:17 2015 +0800
+
+    rgw:the arguments 'domain' should not be assigned when return false
+    
+        Hostnames: [B.A]
+        Inputs: [X.BB.A]
+        Return: [false]
+        Output: [B.A] it is wrong.
+    
+    Fixes: #12629
+    Signed-off-by: Ruifeng Yang <149233652@qq.com>
+    (cherry picked from commit 9420d24c21b4e06d62f356637599c6ec8e7e3e4d)
+
+commit 6acf36f8851b8a369949f0fad9d0156af7dde20f
+Author: Ruifeng Yang <149233652@qq.com>
+Date:   Sun Aug 2 16:16:05 2015 +0800
+
+    rgw:segmentation fault when rgw_gc_max_objs > HASH_PRIME
+    
+    obj_names[] is index out of bounds when rgw_gc_max_objs > HASH_PRIME
+    
+    Fixes: #12630
+    Signed-off-by: Ruifeng Yang <149233652@qq.com>
+    (cherry picked from commit e3147b8ddcaafafde9ce2029f7af5062e22076d3)
+
+commit 6b365144862cbd387d8b931939a3dc2cfe17bbbd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jul 20 20:27:33 2015 -0700
+
+    rgw: avoid using slashes for generated secret keys
+    
+    Just use plain alphanumeric characterset.
+    
+    Fixes: #7647
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3f04a6126fdbfb93304f798da3775c0eec9b7d44)
+
+commit 8f559d58b42b91f070bfa590d9c33e9a156e0066
+Merge: d94e764 931ffe3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 30 08:36:37 2015 -0400
+
+    Merge pull request #5366 from dachary/wip-12303-hammer
+    
+    arm: all programs that link to librados2 hang forever on startup
+
+commit 8ba6b2f628a0f80964afbf4a83b61c4eee9c78ce
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 30 14:47:15 2015 -0700
+
+    rgw: url encode exposed bucket
+    
+    Fixes: #12537
+    Don't send the bucket name back without url encoding it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ad5507fe0bf72ed5bdf8353e315cc9092c740144)
+
+commit d94e7642d0b0213a4b650a5acffa050d042087eb
+Merge: 11a80d8 6417e8e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:41:01 2015 +0200
+
+    Merge pull request #5202 from SUSE/wip-12293-hammer
+    
+    ceph.spec.in: rpm: cephfs_java not fully conditionalized
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 11a80d8bec6ff704e794a2995d653cd5d5c1c979
+Merge: a792ae6 3728477
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:48 2015 +0200
+
+    Merge pull request #5203 from SUSE/wip-11997-hammer
+    
+     ceph.spec.in: rpm: not possible to turn off Java
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a792ae65e2e03e1ca8804380cdbbfa1aec7a2123
+Merge: f7e76d2 8f78001
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:30 2015 +0200
+
+    Merge pull request #5204 from SUSE/wip-11629-hammer
+    
+    ceph.spec.in: SUSE/openSUSE builds need libbz2-devel
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f7e76d2c9e681b3c5775f74c582830f546359703
+Merge: d1feb04 4eb58ad
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:09 2015 +0200
+
+    Merge pull request #5207 from SUSE/wip-12267-hammer
+    
+    ceph.spec.in: 50-rbd.rules conditional is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d1feb0415b3b3dc9c5b003d8528079ccccd0729f
+Merge: 2dca2ec 8b576bd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:57 2015 +0200
+
+    Merge pull request #5216 from SUSE/wip-12269-hammer
+    
+    ceph.spec.in: ceph-common needs python-argparse on older distros, but doesn't require it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2dca2ece0478972e89ed6611a15892ad9e449cc8
+Merge: 64fafc9 0818e9f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:31 2015 +0200
+
+    Merge pull request #5264 from SUSE/wip-12361-hammer
+    
+    ceph.spec.in: snappy-devel for all supported distros
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 64fafc95785024421cbb7ccae41ed4276eeb8fe9
+Merge: 4675971 add0f1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:14 2015 +0200
+
+    Merge pull request #5368 from dachary/wip-12331-hammer
+    
+    ceph: cli throws exception on unrecognized errno
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4675971ce3331f1285398d1556a594f671e489c0
+Merge: 53cdcf2 e004941
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:00 2015 +0200
+
+    Merge pull request #5371 from dachary/wip-12494-hammer
+    
+    ceph tell: broken error message / misleading hinting
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 53cdcf20482823cdb8682dde89706d6f21d8d0d2
+Merge: 362874c eccf369
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:43 2015 +0200
+
+    Merge pull request #5385 from dachary/wip-12501-hammer
+    
+    error in ext_mime_map_init() when /etc/mime.types is missing
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 362874ca39e32c3bc5ee13b13f905fd7693d9449
+Merge: 2044f3c 8804b3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:29 2015 +0200
+
+    Merge pull request #5411 from SUSE/wip-12446-hammer
+    
+    ceph.spec.in: radosgw requires apache for SUSE only -- makes no sense
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2044f3cd7b220cc642245e7ca93c21a17cc8c536
+Merge: 260e24b a785193
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:13 2015 +0200
+
+    Merge pull request #5412 from SUSE/wip-12448-hammer
+    
+    ceph.spec.in: useless %py_requires breaks SLE11-SP3 build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 260e24bba0e83dfa61d25ac0489f656244211db4
+Merge: dc944fb b575ecc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:30:18 2015 +0200
+
+    Merge pull request #5318 from ceph/wip-12236-hammer
+    
+    Possible crash while concurrently writing and shrinking an image
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit dc944fbbedcee56dfbd9d4f5ac3c38d15a6621ea
+Merge: a8c1b4e 92272dd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:30:01 2015 +0200
+
+    Merge pull request #5319 from ceph/wip-12235-hammer
+    
+    librbd: deadlock during cooperative exclusive lock transition
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a8c1b4e7436832ed3040d5a90df713ef6029cb69
+Merge: a39aeee e971820
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:29:26 2015 +0200
+
+    Merge pull request #5296 from ceph/wip-12345-hammer
+    
+    librbd: correct issues discovered via lockdep / helgrind
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a39aeeedf07518d65c9944476b98a0788585f5c1
+Merge: 6a949e1 153744d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:14:22 2015 +0200
+
+    Merge pull request #5559 from ceph/wip-12682-hammer
+    
+    object_map_update fails with -EINVAL return code
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6a949e10198a1787f2008b6c537b7060d191d236
+Merge: 1e423af 4faa8e0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:11:10 2015 +0200
+
+    Merge pull request #5468 from ceph/wip-12577-hammer
+    
+    osd: fix repair when recorded digest is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1e423afd3c7e534e302dbea0798a47f82acec141
+Merge: 574932b 0ca93db
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:06:59 2015 +0200
+
+    Merge pull request #5376 from dachary/wip-12433-hammer
+    
+    Show osd as NONE in ceph osd map <pool> <object>  output
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 574932b2f438787f1a330a271644d9787eecbb63
+Merge: ed162d4 d08db7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 11:54:55 2015 +0200
+
+    Merge pull request #5359 from ceph/wip-11470.hammer
+    
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ed162d498051ac237be33ac6a6764ae0c61d57a6
+Merge: 41a245a bee8666
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 28 14:43:31 2015 -0400
+
+    Merge pull request #5691 from jdurgin/wip-10399-hammer
+    
+    is_new_interval() fixes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 0bc909e54b0edd8a32a3e7db16877f1d875c3186
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 10 04:25:03 2015 -0700
+
+    mon: add a cache layer over MonitorDBStore
+    
+    the cache of of leveldb does not perform well under some condition,
+    so we need a cache in our own stack.
+    
+    * add an option "mon_osd_cache_size" to control the size of cache size
+      of MonitorDBStore.
+    
+    Fixes: #12638
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e99a578050976ca22b549812ac80d494fe7041d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.h
+    minor differences in OSDMonitor ctor
+
+commit bee86660377cfaa74f7ed668dd02492f25553ff9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 21 11:32:39 2015 -0400
+
+    Objecter: pg_interval_t::is_new_interval needs pgid from previous pool
+    
+    When increasing the pg_num of a pool, an assert would fail since the
+    calculated pgid seed would be for the pool's new pg_num value instead
+    of the previous pg_num value.
+    
+    Fixes: #10399
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f20f7a23e913d09cc7fc22fb3df07f9938ddc144)
+    
+    Conflicts: (hobject_t sort order not backported, trivial resolution)
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit b5418b9a7138b45b5fe4f24cfb679c2abb30ab71
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 26 12:35:11 2015 -0700
+
+    osd_types::is_new_interval: size change triggers new interval
+    
+    Fixes: 11771
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ff79959c037a7145f7104b06d9e6a64492fdb95f)
+
+commit f0283890d22e8d5c2210d565077ad3af1921f098 (refs/remotes/gh/liewegas-wip-hammer-feature-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 27 14:09:50 2015 -0400
+
+    include/ceph_features: define HAMMER_0_94_4 feature
+    
+    This is to constrain upgrades past hammer to version that include
+    the appropriate compatibility fixes (e.g., hobject_t encoding).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2868b49c0e39fdc7ae72af81962370c4f95a859e)
+    
+    # Conflicts:
+    #	src/include/ceph_features.h
+
+commit 41a245aff77dde8a2ab212f9c91a73a23a77c40d
+Merge: 95cefea 65b380f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 27 13:07:29 2015 -0400
+
+    Merge remote-tracking branch 'gh/wip-12536-hammer' into hammer
+    
+    This includes wip-osd-compat-hammer.
+
+commit 65b380f7f28a36bb51a6771a2fd9b657ea2848ba (refs/remotes/gh/wip-12536-hammer)
+Merge: 00d802d 9b91adc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 19 09:52:02 2015 -0400
+
+    Merge remote-tracking branch 'gh/wip-osd-compat-hammer' into wip-12536-hammer
+
+commit 153744d7c596705c4f92bee5e827846b46c80141
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 09:39:50 2015 -0400
+
+    tests: increase test coverage for partial encodes/decodes
+    
+    Multiple combinations of offsets/lengths are now tested when
+    performing partial encodes/decodes of the bit vector.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e145f714ac9b2d599b45a058c6b93595e38f424)
+
+commit fca78765366f3ac365cfa98224aca5fb79b2a7fe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 09:34:42 2015 -0400
+
+    common: bit_vector extent calculation incorrect for last page
+    
+    It's highly probable that the last page in the bit vector will not
+    be a full page size.  As a result, the computed extents will extend
+    beyond the data portion of the bit vector, resulting in a end_of_buffer
+    exception.
+    
+    Fixes: #12611
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6d98992691683524d3d96def83a90a6f5fe7f93)
+
+commit 3396a969741f74e5b1f55d8d7af258493ca26478
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 30 14:20:56 2015 +0100
+
+    osd/OSDMap: handle incrementals that modify+del pool
+    
+    Because new_pools was processed after old_pools, if something
+    was modified and then deleted in the same incremental
+    map, then the resulting state would be a pool in the map
+    that had no entry in pool_name.
+    
+    Fixes: #12429
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d4932e721e32a33c1bada54e11d4ecccd868618)
+
+commit 3ab5d82a1c09142267c27156b69a3117d30c5b7d (refs/remotes/gh/wip-12432-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 22 10:01:00 2015 -0700
+
+    rgw: set http status in civetweb
+    
+    Need to set the http status in civetweb so that we report it correctly.
+    Fixes: #12432
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b8e28ab9f914bf48c9ba4f0def9a0deb9dbb93bc)
+
+commit 10a03839853246c535e639b6dc6cea45b8673642
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 31 11:03:29 2015 -0700
+
+    civetweb: update submodule to support setting of http status
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 00d802d97ba5e1a8bde2414f8710494f15807fcd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 23 11:03:17 2014 -0800
+
+    hobject_t: fix get_boundary to work with new sorting regime
+    
+    The hash is no longer the most significant field; set everything that is
+    more significant, too.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 45281efad1ae6b2b5ddd5549c56dbdd46671e5c5)
+
+commit 9b91adc560c8dffc1f5f7470d28c25f448b2b619 (refs/remotes/gh/wip-osd-compat-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 4 13:21:50 2015 -0400
+
+    mon: disallow post-hammer OSDs if there are up pre-hammer OSDs
+    
+    Force *all* OSDs to upgrade to hammer before allowing post-hammer
+    OSDs to join.  This prevents any pre-hammer OSDs from running at
+    the same time as a post-hammer OSD.
+    
+    This commit, as well as the definition of the sentinal post-hammer
+    feature, should get backported to hammer stable series.
+    
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e38d60e94503c7e8511a60a8dceceb3cd39c2bd8)
+    
+    # Conflicts:
+    #	src/mon/OSDMonitor.cc
+
+commit 8a559c1e80bb7c70e13c46f1900896c9b8fcd9a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 4 13:16:11 2015 -0400
+
+    include/ceph_features: define MON_METADATA feature
+    
+    This was added in master in 40307748b9b8f84c98354a7141da645a3c556ba9.  We
+    define it but notably do not support it!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4faa8e04069417a6e2a6c4ea5c10b54d76aa00c4 (refs/remotes/gh/wip-12577-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:17:12 2015 -0700
+
+    osd: include newlines in scrub errors
+    
+    We may log more than one of these in a scrubmap; make sure they are
+    not concatenated.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b3dd1bb7ab360e8f467a47b7ce3a0e4d3a979b2)
+
+commit 455eb2a89197e9609c5bb510112d661d388b4c85
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:04:00 2015 -0700
+
+    osd: fix condition for loggin scrub errors
+    
+    We may have an error to log even though the authoritative map is empty:
+    when we have a digest that doesn't match all replicas.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d23766be902080ffb29528acb5a00d70beb97a9f)
+
+commit 67e7946ce8e28f00db9588bed670ef5141268f41
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:03:22 2015 -0700
+
+    osd: fix fallback logic; move into be_select_auth_object
+    
+    The fallback behavior was printing a promising message but then doing
+    a 'continue' and not actually scrubbing the object.  Instead, fall back to
+    a less-bad auth inside be_select_auth_object.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b597db59fedc9a96dbe4b37f03b819dcf8fdb1bb)
+
+commit 0f57c7017a98f7e416a160375b8b80355b8e0fc3
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 09:34:44 2015 -0700
+
+    osd: log a scrub error when we can't pick an auth object
+    
+    If we really can't find a suitable auth, log an error.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a476d8f3903ca78b5cf36d63059f13cb5268efe5)
+
+commit d4f4c5c4334add6acfc66407c2383d01094d384d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:20:23 2015 -0700
+
+    osd: repair record digest if all replicas match but do not match
+    
+    If the recorded digest does not match the replicas, and all replicas
+    match, then (on repair) fix the recorded digest.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f581fecd8a16b9250fa96beec5aa26074f401028)
+
+commit acfed6b5eae764d381c88ed7d3f4942c61191127
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:52:20 2015 -0700
+
+    osd: move recorded vs on disk digest warning into be_compare_scrubmaps
+    
+    This is a better place for it.  While we are here, change the wording to
+    clearly call out the 'recorded' digest (from object_info_t) vs the 'on
+    disk' digest (what we observed during scrub).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf349ff35e9111ca52db1b0aad3753d0680b486d)
+
+commit 674029bc6c519e82011d1429ab5eddff9720c2dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:46:22 2015 -0700
+
+    osd: be slightly paranoid about value of okseed
+    
+    The scrubber.seed value is set based on the peer feature; check for that
+    too explicitly, and assert the scrubber.seed value matches.
+    
+    No change in behavior here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e34d31b7a9a71132a17b95cf5ad60255ece60243)
+
+commit f2002b7fa2af2db52037da4c1cf66d405a4941e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:44:25 2015 -0700
+
+    osd: be precise about "known" vs "best guess"
+    
+    We cannot assume that the auth info has the "known" digest; all replicas
+    may have a matching digest that does not match the oi, or we may choose
+    a different auth info for some other reason.  Verify that the digest
+    matches the oi before calling it "known".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 546d1c7b8224d96d0cc93dc3c96f52787cd69413)
+
+commit 4e5d146ad36ef0380eb72e45dca651494658aeeb
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:24:11 2015 -0700
+
+    osd: record digest if object is clean (vs entire scrub chunk)
+    
+    If an individual object is clean, record it in the missing_digest map.
+    If not, leave it out.  This lets us drop the old condition that only
+    recorded any digests if the entire scrub chunk was clean.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c39e0e5f126028f552d410ab6924cac6e1f714fb)
+
+commit 1357ed1bd0e250b942bcba0346c97c24bb79a5d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 3 13:05:45 2015 -0400
+
+    hobject_t: decode future hobject_t::get_min() properly
+    
+    The post-hammer wip-temp branch changed hobject_t::get_min() so that pool
+    is INT64_MIN instead of -1 and neglected to deal with the encoding compat
+    with older versions.  Compensate on hammer by mapping INT64_MIN to -1
+    locally.  See commit ff99af38df830e215359bfb8837bf310a2023a4d.
+    
+    Note that this means upgrades from hammer to post-hammer *must* include
+    this fix prior to the upgrade.  This will need to be well-documented in the
+    release notes.
+    
+    Master gets a similar fix so that they map our min value to the new
+    INT64_MIN one on decode.
+    
+    Fixes: #12536 (for hammer)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6d01d6b700300841f49f288a89d514f26a4f94b1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jul 29 11:49:57 2015 -0700
+
+    OSDMonitor::preprocess_get_osdmap: send the last map as well
+    
+    Fixes: #12410
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1320e29dfaee9995409a6d99b9ccaa748dc67b90)
+
+commit a7851933033473d11e69fa1f237c0ad8bbbd4a13
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 11:51:13 2015 +0200
+
+    ceph.spec.in: drop SUSE-specific %py_requires macro
+    
+    %py_requires expands to
+    
+    BuildRequires: /usr/bin/python
+    PreReq: python = 2.7
+    
+    The BuildRequires: is already provided, and the PreReq is wrong because
+    e.g. SLE11-SP3 (a platform we are trying to support) has Python 2.6.
+    
+    http://tracker.ceph.com/issues/12351 Fixes: #12351
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit cbf6c7b46b42e163950bf7956e5b7565fca961c9)
+
+commit 8804b3f1ab5180c8a85b25b9b46b986fe7022868
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 16 15:16:22 2015 +0200
+
+    ceph.spec.in: remove SUSE-specific apache2-mod_fcgid dependency
+    
+    This package is no longer required for RGW to work in SUSE.
+    
+    http://tracker.ceph.com/issues/12358 Fixes: #12358
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f3f8d3660d0ece3139e5f3dc7881ccceadcf648d)
+    
+    Conflicts:
+    	ceph.spec.in
+                In master, the "%if %else %endif" has been changed
+                into two "%if %endif"s
+
+commit b575ecca9743ab08439bfe0b5283e483e2a799ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 14:20:16 2015 -0400
+
+    tests: verify that image shrink properly handles flush op
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8dfcc72071d63c7867a5a59abe871cf5db673a0a)
+
+commit d4eb7bd63a5e46b790bad7001d2873c5c238cc90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 14:18:20 2015 -0400
+
+    librbd: invalidate cache outside cache callback context
+    
+    When shrinking an image, it's possible that the op flush callback
+    will be from within the cache callback context.  This would result
+    in a deadlock when attempting to re-lock the cache lock in order to
+    invalidate the cache.
+    
+    Fixes: #11743
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 726d699b7790c7e371279281ab32cd3aeb8ece7b)
+
+commit 92272dd676ebb67a3095ce3933cac24809b054da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 11:18:29 2015 -0400
+
+    librbd: don't cancel request lock early
+    
+    It's possible that a stale notice is received and will
+    be discarded after the request lock has been canceled.
+    This will stale the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d9dd5c5890029107df40ed438f23fb9865d7de29)
+
+commit 58ae92f1a716e1ffac6408793e19d47f99d63520
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 14 16:13:38 2015 -0400
+
+    tests: new test for transitioning exclusive lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f97ce46ea4c9c7cdc6e37e3759871d5b41b25769)
+
+commit 7b21ccba7c2885027b1d345baaad08a978fbc72f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 13:31:50 2015 -0400
+
+    tests: verify that librbd will periodically resend lock request
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d2a1c226ab3f7e202f62896f0c80c4cf3607cdaf)
+
+commit c95b37f48c129ef6780f67b326e97957f3771472
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:49:36 2015 -0400
+
+    common: Mutex shouldn't register w/ lockdep if disabled
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 879b8a73e2452332b26b8f3428ff5e3e0af8ddad)
+
+commit 117205af89398457e197793505381622705488b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:47:04 2015 -0400
+
+    librbd: improve debugging output for ImageWatcher
+    
+    Include the instance pointer so that different images
+    can be differentiated in the logs.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b951a7398e3e749cb64a3f6a604212a5627e82a0)
+
+commit 08ae01231754d3010c1ede762579cf75c1c7460d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:45:04 2015 -0400
+
+    librados_test_stub: watcher id should be the instance id (gid)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e1e561cc29043eb64dff8afb3a2c1ed77788e93)
+
+commit 704c0e09479d435ecbb84693a786c60694732b19
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 12:51:49 2015 -0400
+
+    librbd: retry lock requests periodically until acquired
+    
+    If the exclusive lock owner acks the lock release request but crashes
+    before it actually releases the lock, the requestor will wait forever.
+    Therefore, after a certain timeout, retry the request again until it
+    succeeds.
+    
+    Fixes: #11537
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 37c74e6e5274208d1b6efaf315afec03ea7eaa82)
+
+commit dbaaed9cf99121d0c97c22a695a7a6e872a11f48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 12:35:36 2015 -0400
+
+    librbd: don't hold owner_lock for write during flush
+    
+    The various IO callback codepaths will attempt to take
+    the lock, which will result in deadlock since the flush
+    cannot complete.
+    
+    Backport: hammer
+    Fixes: #11537
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    
+    (cherry picked from commit 2b6d0633d5c89de3a557cdd72621dfc19f0540de)
+
+commit e9718201c6ed785c61077797a74ffda7e0438fb9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 22 15:30:02 2015 -0400
+
+    lockdep: do not automatically collect all backtraces
+    
+    It is expensive to collect backtraces every time a lock is
+    checked in order to provide cycle backtraces.  The backtraces
+    can be forced on for specific locks or globally via the new
+    config option "lockdep_force_backtrace".
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7354d25f56c5e004f288febdee2b6961c069163b)
+
+commit 27f7042e923366c31748c4cc9992c1a8cb37b457
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 9 13:20:54 2015 -0400
+
+    librbd: flush operations need to acquire owner lock
+    
+    Cache writeback operations will expect the owner lock to be held.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit adfa2e0070ccca7b6556d3bfc5fac7ce4d43a4d0)
+
+commit 5b39983025f510c3119ebb375870e7669be7ddd0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 21 00:13:31 2015 -0400
+
+    librbd: avoid infinite loop if copyup fails
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 43e0e3cd63f0067217ed0811d73f6c546f3027be)
+
+commit 88b583b249a74a71bc87409015f7a8d85200e1bf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 12 10:19:48 2015 -0400
+
+    librbd: flush pending ops while not holding lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3d5cef38c37e5dda6b23751ad560851f1304d86d)
+
+commit a88b180571cb481a31cb94c249a3b486220232fa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 12 10:07:21 2015 -0400
+
+    tests: fix possible deadlock in librbd ImageWatcher tests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 742a85d667b26b3490d96270b5c500b08f2a5283)
+
+commit 321eb8d1c3f7d553addb780928e201acf5091132
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 13:59:49 2015 -0400
+
+    tests: enable lockdep for librbd unit tests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45cb9cb5980e90235b8244d0d61ece40bde4784a)
+
+commit bfe5b90921a3faaad0533388cb150cea20f47ae9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 14:17:37 2015 -0400
+
+    librbd: owner_lock should be held during flush request
+    
+    Flush might result in the cache writing out dirty objects, which
+    would require that the owner_lock be held.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c9142fe35372cf69b7a56f334622a775a6b7c43f)
+
+commit 1e84fb05f48c146cc9d2d090be8c2d355326938b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 14:06:16 2015 -0400
+
+    osdc: ObjectCacher flusher might needs additional locks
+    
+    librbd requires the ObjectCacher flusher thread to acquire
+    an additional lock in order to maintain lock ordering
+    constraints.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a38f9e5104a6e08e130dc4f15ad19a06d9e63719)
+
+commit 506a45a906024d4bb5d3d4d6cc6cbb9eec39c5f2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 16:11:03 2015 -0400
+
+    librbd: fix recursive locking issues
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1b57cc1da7a51e6f8ffea689b94ef843732c20a4)
+
+commit acf5125fe27fb3b9de8b97c4e44fa1f5c61147fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 16:04:28 2015 -0400
+
+    librbd: simplify state machine handling of exclusive lock
+    
+    It is expected that all IO is flushed and all async ops are cancelled
+    prior to releasing the exclusive lock.  Therefore, replace handling of
+    lost exclusive locks in state machines with an assertion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d6b733dbdd0aeb5d1e136dcbf30c58c80952651e)
+
+commit 9454f04e62ece39fdcdbb4eb5a83945f76bcc0a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:32:38 2015 -0400
+
+    librbd: ObjectMap::aio_update can acquire snap_lock out-of-order
+    
+    Detected during an fsx run where a refresh and CoR were occurring
+    concurrently.  The refresh held the snap_lock and was waiting on
+    the object_map_lock, while the CoR held object_map_lock and was
+    waiting for snap_lock.
+    
+    Fixes: #11577
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8cbd92b1fe835b1eb3a898976f9507f51cc115b2)
+
+commit 3e0358e17e7ddb488109bce04bb08cd16fd5bb2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 16 14:15:10 2015 -0400
+
+    librbd: move copyup class method call to CopyupRequest
+    
+    Move AbstractWrite's invocation of copyup to the CopyupRequest
+    class.  The AioRequest write path will now always create a
+    CopyupRequest, which will now append the actual write ops to the
+    copyup.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7be3df67809925164237cc185f9f29e145f45768)
+
+commit 2ee64a80794297d4fc5aeafbb185818f7f6a77b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Mar 31 12:28:11 2015 -0400
+
+    librbd: simplify AioRequest constructor parameters
+    
+    Moved all parent overlap computation to within AioRequest so that
+    callers don't need to independently compute the overlap.  Also
+    removed the need to pass the snap_id for write operations since
+    it can only be CEPH_NOSNAP.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 46515971edab8888284b1a8307ffca88a9c75f5c)
+
+commit 3e71a7554da05aee06d5a2227808c321a3e3f0f1
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Mar 14 01:16:31 2015 +0100
+
+    librbd/AioRequest.h: fix UNINIT_CTOR
+    
+    Fix for:
+    
+    CID 1274319: Uninitialized scalar field (UNINIT_CTOR)
+     uninit_member: Non-static class member m_object_state is not
+     initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 48f18ea0e1c4c6de1921ea2359448deb761461e7)
+
+commit cb57fe581df7c14d174e6eaff382d6a28ab658d6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 17 15:04:10 2015 -0400
+
+    librbd: add object state accessor to ObjectMap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9249ab7dacd43e7a546656b99013a595381fd5fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:41:59 2015 -0400
+
+    librbd: AsyncObjectThrottle should always hold owner_lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c352bcdc0f63eea55677fe3c3b5f0c61347c0db3)
+
+commit 26902b94f97edb189ae620c86a8dda8166df471a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:34:43 2015 -0400
+
+    librbd: execute flush completion outside of cache_lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5f157f20980de7e9a05fb933fb57efdc759da78b)
+
+commit 571220d6f4642dd3cd78988882645fdf647c150e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:17:54 2015 -0400
+
+    librbd: add AsyncRequest task enqueue helper method
+    
+    In order to support the invariant that all state machine
+    callbacks occur without holding locks, transitions that
+    don't always involve a librados call should queue their
+    callback.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 218bc2d0f8d90c9c64408cb22e26680e88138844)
+
+commit 8e280f4cfef0d2fdc706d43bbee0c377d288a457
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 15:32:27 2015 -0400
+
+    librbd: disable lockdep on AioCompletion
+    
+    It is only used by clients and it causes a large slowdown
+    in performance due to the rate at which the lock is constructed/
+    destructed for each IO request.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 66e74641baeed9aadb7e474d6e6b142b7725722d)
+
+commit b38da48059792e9182fe2877786d9159007683d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:42:19 2015 -0400
+
+    librbd: AioCompletion shouldn't hold its lock during callback
+    
+    The callback routine most likely will attempt to retrieve the result
+    code, which will result in a recursive lock attempt.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ad19ae349ebb27ff6027e40883735240fa97a3c)
+
+commit 6fdd3f1ce69fa2e00c6f1bedd5f72352953e1e44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:51:03 2015 -0400
+
+    librbd: give locks unique names to prevent false lockdep failures
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c474ee42b87975c04eeb2b40b976deb5a5e2d518)
+
+commit 70041499f99cad551b3aab16f086f725b33b25e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:40:16 2015 -0400
+
+    librbd: complete cache read in a new thread context
+    
+    The ObjectCacher complete the read callback while still holding
+    the cache lock.  This introduces lock ordering issues which are
+    resolved by queuing the completion to execute in a clean (unlocked)
+    context.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0024677dc8adfd610418ca47599dd95d3a5a6612)
+
+commit 65ef695eba0f5b5d87347ffb3407bb5d6d75b402
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 19 15:35:59 2015 -0400
+
+    librbd: require callers to ObjectMap::aio_update to acquire lock
+    
+    This is needed to allow an atomic compare and update operation
+    from the rebuild object map utility.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2db758cb4cb6d88cbaf9842b1e5d0872a185d8df)
+
+commit 58b8fafb87b18f2a4d8c38f77ecf81ff58452409
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:38:29 2015 -0400
+
+    log: fix helgrind warnings regarding possible data race
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c1e14451775049bbd95bcb19a0b62ab5e2c0a7bb)
+
+commit a5203d32544d40cb51eb60fc906cc3ceaacbfe96
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:37:56 2015 -0400
+
+    librados_test_stub: fix helgrind warnings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b65ae4bc26f2dcaaa9518d5cce0e8b83ea310de8)
+
+commit b73e87eb307137786e0f0d89362ef0e92e2670b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:36:26 2015 -0400
+
+    librados_test_stub: add support for flushing watches
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6e400b9049ede5870e40e4dd2cb41874550eac25)
+
+commit 2fa35b1c5ca8e33959fff8c84eaa4feca0f67df3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:29:12 2015 -0400
+
+    common: lockdep now support unregistering once destructed
+    
+    librbd use of an image hierarchy resulted in lock names being
+    re-used and incorrectly analyzed.  librbd now uses unique lock
+    names per instance, but to prevent an unbounded growth of
+    tracked locks, we now remove lock tracking once a lock is
+    destructed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7c7df2ce9f837628535d21df61ae4f13d809c4fa)
+
+commit 7b85c7bfe599644eb29e6b1e03733da4774c2eac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:26:41 2015 -0400
+
+    common: add valgrind.h convenience wrapper
+    
+    Conditionally support helgrind annotations if valgrind support is
+    enabled during the build.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 650ad32d74a24d248dd8e19bff1bbd31f0bb224b)
+
+commit 6d3db5ffbef71c8927a9ffc68c5955dca0f5612c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add work queue for op completions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 21f990efe6c6d710376d5b1a49fc8cd53aa020e6)
+
+commit 64425e861400fde4703fb06529777c55650b6676
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: ContextWQ can now accept a return code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5ffae578e83c1e4fca3f328c937e7c9be8cc03c)
+
+commit eccf36980511de7ed122a38d426170496ffdea64
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Jun 23 13:41:53 2015 -0600
+
+    packaging: RGW depends on /etc/mime.types
+    
+    If the mimecap RPM or mime-support DEB is not installed, then the
+    /etc/mime.types file is not present on the system. RGW attempts to read
+    this file during startup, and if the file is not present, RGW logs an
+    error:
+    
+      ext_mime_map_init(): failed to open file=/etc/mime.types ret=-2
+    
+    Make the radosgw package depend on the mailcap/mime-support packages so
+    that /etc/mime.types is always available on RGW systems.
+    
+    http://tracker.ceph.com/issues/11864 Fixes: #11864
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit c74a2f86a1f9b15aae0e723b410bedfc1d01a79b)
+
+commit 7bddf5d4dad09b1f6d3b4e80e4bdd895191560b0
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Thu Jul 16 09:42:55 2015 +0800
+
+      rest_bench: bucketname is not mandatory as we have a default name
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit bc56a87ea3b69b4856a2b9f9fb2a63dbfb1211c3)
+
+commit 6e7358b3958b7ee11ef612302b300d07f3982d54
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Thu Jul 16 09:17:59 2015 +0800
+
+      rest_bench: drain the work queue to fix a crash
+      Fixes: #3896
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f3d34d8ff921dbd2ff21f6b72af7c73bb9c6940e)
+
+commit 1e055782f5ade94c19c2614be0d61d797490701e
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jul 21 16:09:32 2015 +0100
+
+    auth: check return value of keyring->get_secret
+    
+    get_secret can fail to populate the passed CryptoKey, for
+    example if the entity name is not found in the keyring.  In
+    this case, attempts to use the CryptoKey will lead to
+    segfaults.
+    
+    Fixes: #12417
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 64e5041008744362fdbb16e16bc3e049a2d426aa)
+
+commit 256620e37fd94ee4b3af338ea6955be55529d0d6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jul 16 04:45:05 2015 -0700
+
+    Client: check dir is still complete after dropping locks in _readdir_cache_cb
+    
+    We drop the lock when invoking the callback, which means the directory
+    we're looking at might get dentries trimmed out of memory. Make sure that
+    hasn't happened after we get the lock back. If it *has* happened, fall back
+    to requesting the directory contents from the MDS. Update the dirp location
+    pointers after each entry to facilitate this.
+    Because this requires we update the dirp->at_cache_name value on every loop,
+    we rework the updating scheme a bit: to dereference the dn->name before
+    unlocking, so we know it's filled in; and since we update it on every loop
+    we don't need to refer to the previous dentry explicitly like we did before.
+    
+    This should also handle racing file deletes: we get back a trace on
+    the removed dentry and that will clear the COMPLETE|ORDERED flags.
+    
+    Fixes #12297
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 62dd63761701a7e0f7ce39f4071dcabc19bb1cf4)
+
+commit 8a2ad0540994efe530a1775ebd949aeaf6609e57
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:29:45 2015 -0700
+
+    TestPGLog: fix invalid proc_replica_log test caes
+    
+    If 1'3 is non-divergent, no osd could correctly have created 2'3.  Also,
+    proc_replica_log does not add the extra entries from the auth log to the
+    missing set, that happens later on in activate.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1333b676dba4ed4b5db854fe935bdadbf9f21d08)
+
+commit df71e6be7ccbe18d99200b2a5844a9f19ac70141
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:28:23 2015 -0700
+
+    TestPGLog: fix noop log proc_replica_log test case
+    
+    Need to fill in log.head, olog.head.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1e5b22053381580708af29a1536f1e106c9b479c)
+
+commit 549ff9a938c5126d4dcb15535845d5354cb44bb2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:41 2015 -0700
+
+    TestPGLog: add test for 11358
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b61e5aef15d7448d24b713d13dce77697bc0af74)
+
+commit c224fc70e14bd76494bc437595c8b0b0d275694e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:29 2015 -0700
+
+    PGLog::proc_replica_log: handle split out overlapping entries
+    
+    See the comment, if the overlapping entries between olog.head
+    and log.tail were split out, last_update should be log.tail.
+    
+    Fixes: 11358
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6561e0d955c4563a8505b717de41f5bd89119f63)
+
+commit b8176d0f21ce80d2e5a92057ab85f7618504cd9f
+Author: Ketor Meng <d.ketor@gmail.com>
+Date:   Tue May 26 18:50:17 2015 +0800
+
+    Mutex: fix leak of pthread_mutexattr
+    
+    Need pthread_mutexattr_destroy after pthread_mutexattr_init
+    
+    Fixes: #111762
+    Signed-off-by: Ketor Meng <d.ketor@gmail.com>
+    (cherry picked from commit 2b23327b3aa8d96341d501a5555195ca1bc0de8f)
+
+commit 43a72e47c4adae674e02bb262645d88ac528e2be
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Fri Jul 3 18:27:13 2015 +0800
+
+    mon/PGMonitor: bug fix pg monitor get crush rule
+    
+    when some rules have been deleted before, the index in array of crush->rules
+    is not always equals to crush_ruleset of pool.
+    
+    Fixes: #12210
+    Reported-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 498793393c81c0a8e37911237969fba495a3a183)
+
+commit 0ca93db7d79f11e51af75e322941419b860bb2f8
+Author: Shylesh Kumar <shmohan@redhat.com>
+Date:   Wed Jul 22 18:47:20 2015 +0530
+
+    mon: ceph osd map shows NONE when an osd is missing
+    
+    Signed-off-by: Shylesh Kumar <shmohan@redhat.com>
+    (cherry picked from commit 4d030302ea3709eb021710e873f1ebe62f951cec)
+
+commit 695f782e59baffbe8429f9daa47a4f1476a02976
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jun 7 20:05:58 2015 -0400
+
+    crush/CrushWrapper: fix adjust_subtree_weight debug
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26be86f649a1e9846280204fe1e3f8344eb975ce)
+
+commit 0bd4c81ce41a22b22ca76e539de87260b140bbc4
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:15:45 2015 -0700
+
+    crush/CrushWrapper: return changed from adjust_subtree_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 551ab2d69dd50a078d62340740d357d85ab8dcbb)
+
+commit 05fc59bee9138d04227255a2f31bd82efe94b1e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:15:22 2015 -0700
+
+    crush/CrushWrapper: adjust subtree base in adjust_subtree_weight
+    
+    We were correctly adjusting all the children, but the subtree root was
+    not being updated.
+    
+    Fixes: #11855
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1da96ab8442522bda4fc39e05597f17b8ebddf89)
+
+commit d2f31adc49a7c6b8276dbf8773575e91ba5e18fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:12:41 2015 -0700
+
+    unittest_crush_wrapper: test adjust_subtree_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b461dc2e970f139dcb25d93b32be6b4bb9f1c8eb)
+
+commit 0ccdf342e7cf0072d3c38c6e1a6cf3a87f154a28
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:17:26 2015 -0700
+
+    unittest_crush_wrapper: attach buckets to root in adjust_item_weight test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a8351eb3de2b4069ffec9c4f561eb1519ea11aa2)
+
+commit 1e737532cfe1daf43bea3d9bd4e55faed560f0f2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:12:32 2015 -0700
+
+    unittest_crush_wrapper: parse env
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14c60b288b33e4cec9c9819d4668a5ff9855ccc1)
+
+commit cd11b887c6c586085a7014ba44123b115370a462
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Thu Jul 2 05:29:47 2015 +0000
+
+    osd: pg_interval_t::check_new_interval should not rely on pool.min_size to determine if the PG was active
+    
+    If the pool's min_size is set improperly, during peering, pg_interval_t::check_new_interval
+    might wrongly determine the PG's state and cause the PG to stuck at down+peering forever
+    
+    Fixes: #12162
+    Signed-off-by: Guang Yang yguang@yahoo-inc.com
+    (cherry picked from commit 684927442d81ea08f95878a8af69d08d3a14d973)
+    
+    Conflicts:
+    	src/osd/PG.cc
+               because PG::start_peering_interval has an assert
+               that is not found in hammer in the context
+    	src/test/osd/types.cc
+               because include/stringify.h is not included by
+               types.cc in hammer
+
+commit c5f0e2222cc55a92ebc46d12b618fadb3d2fb577
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Wed Jul 1 20:26:54 2015 +0000
+
+    osd: Move IsRecoverablePredicate/IsReadablePredicate to osd_types.h
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 466b083166231ec7e4c069fef8c9e07d38accab9)
+
+commit 42bff0b9e2f80e32001082a608d63719332677e3
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Tue Jul 21 11:20:53 2015 +0100
+
+    mon: OSDMonitor: fix hex output on 'osd reweight'
+    
+    Fixes: #12251
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 787fa80c2746fde44ac0583ff7995ec5be9a672d)
+
+commit e00494118ee42322784988fe56623cbc7aac4cc9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 15 19:03:33 2015 +0800
+
+    ceph.in: print more detailed warning for 'ceph <type> tell'
+    
+    put the full signature of "tell <target> <command> [options...]"
+    instead of "tell {0}.<id>", which could be misleading somehow.
+    
+    Fixes: 11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit aa1a5222eebd0256d4ecffb0eefeb0f78ecf0a1f)
+
+commit f18900f2eb724acd898e8b32fe75c0850d7cf94c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 15 18:01:52 2015 +0800
+
+    ceph.in: print more detailed error message for 'tell' command
+    
+    * we do not allow user specify a certain daemon when starting an
+      interactive session. the existing error message could lead to
+      some confusion. so put more details in it.
+    
+    Fixes: #11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 972dc9145235ff0f60663369ebcab8c16628b89f)
+
+commit 9916d37ead705d467b1d730adb15bb6974b3bf1b
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Mon Jul 20 10:50:20 2015 +0800
+
+      mon/PGMonitor: avoid uint64_t overflow when checking pool 'target/max' status.
+      Fixes: #12401
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f84e6b8157e78fd05cd1c8ac957c92df17e6ba06)
+
+commit 4457d3eb70712ec3b7d8e98af973e099c8c7d258
+Author: renhwztetecs <rhwlyw@163.com>
+Date:   Tue Jul 21 18:55:00 2015 +0800
+
+    Update OSDMonitor.cc
+    
+    OSDMonitor::get_pools_health(), s/objects/bytes/
+    
+    Fixes: #12402
+    Signed-off-by: ren.huanwen@zte.com.cn
+    (cherry picked from commit 7fc13c9d6b9a4962d7640240416105d8f558d600)
+
+commit add0f1e5e229c9ad66e8ef77ad59e0e390e20db6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Apr 16 18:29:00 2015 +0800
+
+    ceph.in: do not throw on unknown errno
+    
+    some of the errnos are not listed in errno.errorcode. if we happen
+    to run into them, print 'Unknown' instead.
+    
+    Fixes: #11354
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53514ad97f6f75746613f09d464f69a7c31bed55)
+
+commit fa194744fdaed13949bffb6f2b8a0fc420006e0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 18 13:49:20 2015 -0700
+
+    os/chain_xattr: handle read on chnk-aligned xattr
+    
+    If we wrote an xattr that was a multiple of a chunk, we will try to read
+    the next chunk and get ENODATA.  If that happens bail out of the loop and
+    assume we've read the whole thing.
+    
+    Backport: hammer, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8614dcebf384b74b283cd352001a65fa26ba925c)
+
+commit 931ffe3b3a43157798717e64de8f46fadbcd9e9b
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jun 3 12:24:48 2015 +0200
+
+    common/Cycles.cc: skip initialization if rdtsc is not implemented
+    
+    The Cycles initialization gets stuck in infinite loop if rdtsc is not
+    implemented. This patch fixes the issue by quitting the initialization
+    if rtdsc fails.
+    
+    The patch was cherry-picked from ubuntu patch by James Page, see
+    
+    https://bugzilla.redhat.com/show_bug.cgi?id=1222286
+    
+    for more details on the patch.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit 35c5fd0091fc4d63b21207fb94e46b343519fd56)
+
+commit 0fde3a2465e156ebf9a5cdc6adc45d66d6d647fc
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:42:42 2015 +0800
+
+    buffer: Fix bufferlist::zero bug with special case
+    
+    Fixes: #12252
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 43f583d160ccaf879eaf0f3020e77860cf8d1df0)
+
+commit dabc61110a5773262614fc77d3653674aac421f7
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:32:03 2015 +0800
+
+    UnittestBuffer: Add bufferlist zero test case
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 577acf665948e531e50d36f5780ea19351a5bf65)
+
+commit d08db7a0677412dbe590c850976fb4fe2f503a79 (refs/remotes/gh/wip-11470.hammer)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Jun 12 19:21:10 2015 +0100
+
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Whenever the monitor finishes committing a proposal, we call
+    Monitor::refresh_from_paxos() to nudge the services to refresh.  Once
+    all services have refreshed, we would then call each services
+    post_paxos_update().
+    
+    However, due to an unfortunate, non-critical bug, some services (mainly
+    the LogMonitor) could have messages pending in their
+    'waiting_for_finished_proposal' callback queue [1], and we need to nudge
+    those callbacks.
+    
+    This patch adds a new step during the refresh phase: instead of calling
+    directly the service's post_paxos_update(), we introduce a
+    PaxosService::post_refresh() which will call the services
+    post_paxos_update() function first and then nudge those callbacks when
+    appropriate.
+    
+    [1] - Given the monitor will send MLog messages to itself, and given the
+    service is not readable before its initial state is proposed and
+    committed, some of the initial MLog's would be stuck waiting for the
+    proposal to finish.  However, by design, we only nudge those message's
+    callbacks when an election finishes or, if the leader, when the proposal
+    finishes.  On peons, however, we would only nudge those callbacks if an
+    election happened to be triggered, hence the need for an alternate path
+    to retry any message waiting for the initial proposal to finish.
+    
+    Fixes: #11470
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 1551ebb63238073d2fd30201e6b656a8988e958c)
+
+commit 0818e9f78b11801aa619b9ec04eeffb2795c3f62
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 13:28:22 2015 +0200
+
+    ceph.spec.in: snappy-devel for all supported distros
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e4634ddcc1c0a2abe51679425d15e6f693811e17)
+    
+    Conflicts:
+    	ceph.spec.in
+                minor difference
+
+commit 8b576bd529d7c1604f0a4cdf97ece64fc5d69447
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 16 18:27:20 2015 +0200
+
+    ceph.spec.in: python-argparse only in Python 2.6
+    
+    argparse is a widely-used Python module for parsing command-line arguments.
+    Ceph makes heavy use of Python scripts, both in the build environment and on
+    cluster nodes and clients.
+    
+    Until Python 2.6, argparse was distributed separately from Python proper.
+    As of 2.7 it is part of the Python standard library.
+    
+    Although the python package in a given distro may or may not Provide:
+    python-argparse, this cannot be relied upon.
+    
+    Therefore, this commit puts appropriate conditionals around Requires:
+    python-argparse and BuildRequires: python-argparse. It does so for Red
+    Hat/CentOS and SUSE only, because the last Fedora version with Python 2.6
+    was Fedora 13, which is EOL.
+    
+    argparse is required by both the ceph and ceph-common packages, but since ceph
+    requires ceph-common, the argparse Requires and BuildRequires need only appear
+    once, under ceph-common.
+    
+    http://tracker.ceph.com/issues/12034 Fixes: #12034
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23171c952997d104cfa6b5700ec0bb658a8e0915)
+
+commit cdde626f6371e1da501306ed16bc450ee1ec0b91
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 13 18:12:01 2015 +0200
+
+    ceph.spec.in: do not run fdupes, even on SLE/openSUSE
+    
+    In openSUSE there is a policy to use %fdupes in the spec file if RPMLINT
+    complains about duplicate files wasting space in the filesystem.
+    
+    However, RPMLINT is not so complaining, so drop fdupes.
+    
+    http://tracker.ceph.com/issues/12301 Fixes: #12301
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 53072b9019caf72e0313b2804ea174237ed7da33)
+
+commit 6417e8eff942dbbf6cd231ffb73136b7bdcd837f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 12 17:50:58 2015 +0200
+
+    rpm: add missing Java conditionals
+    
+    http://tracker.ceph.com/issues/11991 Fixes: #11991
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 04e91bbd929801de7f1d6a05086d47a781c2e361)
+
+commit 37284773a754ef4b91d2b5436d46a5fd2be4f67e
+Author: Owen Synge <osynge@suse.com>
+Date:   Fri May 29 13:28:24 2015 +0200
+
+    Add rpm conditionals : cephfs_java
+    
+    Extracted conditionals from SUSE rpm spec file to forward port.
+    Original work done by Thorsten Behrens <tbehrens@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit c65538f489897821877421043300d9c008224f02)
+
+commit 8f78001c2e09d483dcbcd33c24e2ef8043119773
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Thu May 14 20:07:37 2015 +0200
+
+    ceph.spec.in: SUSE/openSUSE builds need libbz2-devel
+    
+    http://tracker.ceph.com/issues/11629 Fixes: #11629
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 584ed2e28b0a31821f29a99cf6078cfd76ee31fc)
+
+commit 4eb58ad2027148561d94bb43346b464b55d041a6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 22:37:52 2015 +0200
+
+    ceph.spec.in: use _udevrulesdir to eliminate conditionals
+    
+    The conditionals governing where 50-rbd.rules is installed were not doing the
+    right thing on SUSE distros.
+    
+    Start using the %_udevrulesdir RPM macro, while taking care that it is defined
+    and set to the right value. Use it to eliminate some conditionals around other
+    udev rules files as well.
+    
+    http://tracker.ceph.com/issues/12166 Fixes: #12166
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 8aa758ee7a14e93ded1d55b4aca7a54aa47d7174)
+    
+    Conflicts:
+    	ceph.spec.in
+                hammer does not yet have _with_systemd or ceph_daemon.py
diff --git a/doc/changelog/v0.94.5.txt b/doc/changelog/v0.94.5.txt
new file mode 100644
index 000000000..56c9e50bd
--- /dev/null
+++ b/doc/changelog/v0.94.5.txt
@@ -0,0 +1,86 @@
+commit 9764da52395923e0b32908d83a9f7304401fee43 (tag: refs/tags/v0.94.5, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@ceph.com>
+Date:   Mon Oct 26 07:05:32 2015 -0700
+
+    0.94.5
+
+commit 1107f29224e24211860b719b82f9b85a022c25e3
+Merge: d86eab5 d3abcbe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 20:41:33 2015 +0200
+
+    Merge pull request #6348 from dillaman/wip-13567-hammer
+    
+    librbd: potential assertion failure during cache read
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d86eab5b860186066c888208d92a42a748569d78
+Merge: 4ebfa1b 250dc07
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 15:20:32 2015 +0200
+
+    Merge pull request #6362 from liewegas/wip-shut-up-osd-hammer
+    
+    osd/ReplicatedPG: remove stray debug line
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 250dc0722b9156fe1b8be81e09fb4ead699065f3 (refs/remotes/me/wip-shut-up-osd-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 08:33:44 2015 -0400
+
+    osd/ReplicatedPG: remove stray debug line
+    
+    This snuck in
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ff9600a6cef613d40e875597b6392778df1bb04c)
+
+commit d3abcbea1fdb04f0994f19584b93f6f1b1ff37ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 21 13:12:48 2015 -0400
+
+    librbd: potential assertion failure during cache read
+    
+    It's possible for a cache read from a clone to trigger a writeback if a
+    previous read op determined the object doesn't exist in the clone,
+    followed by a cached write to the non-existent clone object, followed
+    by another read request to the same object.  This causes the cache to
+    flush the pending writeback ops while not holding the owner lock.
+    
+    Fixes: #13559
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4692c330bd992a06b97b5b8975ab71952b22477a)
+
+commit 991d0f0575411e2f2b53df35e36ff6170bcc9d8b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 21 13:09:54 2015 -0400
+
+    tests: reproduce crash during read-induced CoW
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 382219b6bba09156f8cf3d420c6348738e7ed4c6)
+
+commit 4ebfa1bcc7e3b57c3c12000633654d36a005d512
+Merge: 9529269 51f3d6a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 22 12:14:42 2015 -0400
+
+    Merge pull request #6330 from dachary/wip-13550-hammer
+    
+    qemu workunit refers to apt-mirror.front.sepia.ceph.com
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51f3d6a489760dbcb931676396eea92159ca2ad9
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Fri Jun 19 00:02:20 2015 +0800
+
+    qa: Use public qemu repo
+    
+    This would allow some external tests outside of sepia lab
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit 4731c1e35539c9506ff3fe3141553fad099d0eee)
diff --git a/doc/changelog/v0.94.6.txt b/doc/changelog/v0.94.6.txt
new file mode 100644
index 000000000..7fc155b7d
--- /dev/null
+++ b/doc/changelog/v0.94.6.txt
@@ -0,0 +1,4027 @@
+commit e832001feaf8c176593e0325c8298e3f16dfb403 (tag: refs/tags/v0.94.6, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Mon Feb 22 21:10:17 2016 +0000
+
+    0.94.6
+
+commit 7abb6ae8f3cba67009bd022aaeee0a87cdfc6477
+Merge: ceb6fcc a8fc6a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 14:19:57 2016 +0700
+
+    Merge pull request #7501 from SUSE/wip-14624-hammer
+    
+    hammer: fsx failed to compile
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ceb6fcc5f2828995bfe59bfc4e206010a4fe3230
+Merge: 2f4e1d1 f2ca42b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 5 21:10:46 2016 -0500
+
+    Merge pull request #7524 from ktdreyer/wip-14637-hammer-man-radosgw-admin-orphans
+    
+    hammer: doc: regenerate man pages, add orphans commands to radosgw-admin(8)
+
+commit 2f4e1d1ff8e91fc2ee9c23d5a17c3174d15a7103
+Merge: 2ca3c3e 9ab5fd9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Feb 5 10:30:22 2016 +0100
+
+    Merge pull request #7526 from ceph/wip-14516-hammer
+    
+    hammer: rgw-admin: document orphans commands in usage
+
+commit 2ca3c3e5683ef97902d0969e49980d69c81b4034
+Merge: 02353f6 5c8d1d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Feb 5 12:47:33 2016 +0700
+
+    Merge pull request #7441 from odivlad/backport-pr-14569
+    
+    [backport] hammer: rgw: Make RGW_MAX_PUT_SIZE configurable
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 02353f6baa460949d7dd2738346d8d748401bc5b
+Merge: f3bab8c 0e1378e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Feb 5 12:46:54 2016 +0700
+
+    Merge pull request #7442 from odivlad/backport-pr-14570
+    
+    [backport] rgw: fix wrong etag calculation during POST on S3 bucket.
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 9ab5fd9d67da43e986489e4f580a597dd8cb551e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 1 16:33:55 2016 -0800
+
+    rgw-admin: document orphans commands in usage
+    
+    Fixes: #14516
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 105a76bf542e05b739d5a03ca8ae55432350f107)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (trivial resolution)
+
+commit 0e1378effdd1d0d70d3de05c79b208e9f8b8e328
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Sep 21 20:32:29 2015 +0200
+
+    [backport] rgw: fix wrong etag calculation during POST on S3 bucket.
+    
+    Closes: #14570
+    (cherry picked from commit 742906a)
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    Tested-by: Vladislav Odintsov <odivlad@gmail.com>
+
+commit 5c8d1d74069f70b85bc4286e6d1136bce1dc593f
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Thu Jul 9 16:56:07 2015 +0800
+
+    [backport] rgw: Make RGW_MAX_PUT_SIZE configurable
+    
+    The 5GB limit of a single operation uploading was part of S3 spec.
+    However some private setups may have some special requirements
+    on this limit. It's more convinent to have a configurable value.
+    
+    Closes: http://tracker.ceph.com/issues/14569
+    (cherry picked from commit df97f28)
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    Tested-by: Vladislav Odintsov <odivlad@gmail.com>
+
+commit f2ca42b1218182f4bfa27718c9606705d8b9941f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Feb 3 19:51:58 2016 -0700
+
+    doc: add orphans commands to radosgw-admin(8)
+    
+    The --help text was added in 105a76bf542e05b739d5a03ca8ae55432350f107.
+    Add the corresponding entries to the man page.
+    
+    Fixes: #14637
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    
+    (cherry picked from commit ec162f068b40f594c321df5caa9fe2541551b89e)
+      Cherry-pick to hammer includes nroff source change (in master the
+      nroff sources are no longer present in Git.)
+
+commit e42ed6d6414ad55d671dd0f406b1dababd643af8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Feb 4 11:04:39 2016 -0700
+
+    man: rebuild manpages
+    
+    following the procedure in admin/manpage-howto.txt.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit a8fc6a9ffb2f8021657ad412fd0aaaaf7f98bd53
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Jan 13 13:17:53 2016 -0800
+
+    fsx: checkout old version until it compiles properly on miras
+    
+    I sent a patch to xfstests upstream at
+    http://article.gmane.org/gmane.comp.file-systems.fstests/1665, but
+    until that's fixed we need a version that works in our test lab.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 7d52372ae74878ebd001036ff0a7aad525eb15b6)
+
+commit f3bab8c7bc6bba3c79329d7106684596634e17a6
+Merge: 31d86b1 1b02859
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 3 12:41:56 2016 +0700
+
+    Merge pull request #7454 from dachary/wip-14584-hammer
+    
+    hammer: fsstress.sh fails
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 31d86b1580d59581d130e7a5e84905a5b4d67666
+Merge: 2c8e579 2817ffc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 3 11:38:57 2016 +0700
+
+    Merge pull request #6918 from asheplyakov/hammer-bug-12449
+    
+    osd: check for full before changing the cached obc (hammer)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1b02859480677abbd8708650764ed6815917e0cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 18 15:22:55 2015 +0800
+
+    qa/fsstress.sh: fix 'cp not writing through dangling symlink'
+    
+    On some test machines, /usr/lib/ltp/testcases/bin/fsstress is
+    dangling symlink. 'cp -f' is impotent in this case.
+    
+    Fixes: #12710
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 479f2a760baf6af4714d3391a366a6b3acf1bcaf)
+
+commit 2c8e57934284dae0ae92d1aa0839a87092ec7c51
+Merge: 1cab151 700be56
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:42:29 2016 -0500
+
+    Merge pull request #7236 from athanatos/wip-14376
+    
+    config_opts: increase suicide timeout to 300 to match recovery
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1cab151a6cdc8a1b9b2f11518b77ea149c19d067
+Merge: a134c44 5105d50
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:42:12 2016 -0500
+
+    Merge pull request #6450 from dachary/wip-13672-hammer
+    
+    tests: testprofile must be removed before it is re-created
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a134c44d626ead3308464474d67604363bac6f5d
+Merge: 520792d 3f1292e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:41:39 2016 -0500
+
+    Merge pull request #6680 from SUSE/wip-13859-hammer
+    
+    hammer: ceph.spec.in License line does not reflect COPYING
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+
+commit 520792d51e949c451767d116e267d86fee812ada
+Merge: 4d0fafb c2c6d02
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:41:18 2016 -0500
+
+    Merge pull request #6791 from branch-predictor/bp-5812-backport
+    
+    Race condition in rados bench
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+
+commit 4d0fafb289fc35f44e6e74bb974c402ba147d4d4
+Merge: 211a093 6379ff1
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:40:38 2016 -0500
+
+    Merge pull request #6973 from dreamhost/wip-configure-hammer
+    
+    configure.ac: no use to add "+" before ac_ext=c
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 211a093d4107c1806ad7e4876bc5c550a5b5b7d4
+Merge: 0c13656 53742bd
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:40:13 2016 -0500
+
+    Merge pull request #7206 from dzafman/wip-14292
+    
+    osd/PG.cc: 3837: FAILED assert(0 == "Running incompatible OSD")
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 0c136561600e295ec48dcf29a77aa2cd293a7236
+Merge: 1ea14ba ae56de0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:39:42 2016 -0500
+
+    Merge pull request #7207 from rldleblanc/recency_fix_for_hammer
+    
+    hammer: osd/ReplicatedPG: Recency fix for Hammer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1ea14bae10930ed3e66b7b0140e359009a44275e
+Merge: 1740d8c 8d9e08c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:39:11 2016 -0500
+
+    Merge pull request #7347 from tchaikov/wip-hammer-10093
+    
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1740d8ce8eb7e7debce1684a19736783489628c3
+Merge: 7848cdc c7252a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:38:35 2016 -0500
+
+    Merge pull request #7411 from dachary/wip-14467-hammer
+    
+    hammer: disable filestore_xfs_extsize by default
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7848cdc4c8c3598cd291d26a2dd4d76abc4bcda9
+Merge: 57abeab 70f1ba3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:38:13 2016 -0500
+
+    Merge pull request #7412 from dachary/wip-14470-hammer
+    
+    tools: tool for artificially inflate the leveldb of the mon store for testing purposes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 57abeab1f2610d89f0ab2a028c8e093afde5e403
+Merge: 4d7d7c3 9109304
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:37:46 2016 -0500
+
+    Merge pull request #7446 from liewegas/wip-14537-hammer
+    
+    mon: compact full epochs also
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4d7d7c33ff58ffeb27b47d379bf7dd4af90785b0
+Merge: 8360486 8c28f2f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:31 2016 -0800
+
+    Merge pull request #7182 from dachary/wip-14143-hammer
+    
+    hammer: Verify self-managed snapshot functionality on image create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8360486764c3ab4e10a34f5bdf7555e4c3385977
+Merge: 501e01a b2961ce
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:20 2016 -0800
+
+    Merge pull request #7183 from dachary/wip-14283-hammer
+    
+    hammer: rbd: fix bench-write
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 501e01acc55f846cd3ad85e8294a9598c1c90907
+Merge: 97d4f6a 24c0b27
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:05 2016 -0800
+
+    Merge pull request #7416 from dachary/wip-14466-hammer
+    
+    hammer: rbd-replay does not check for EOF and goes to endless loop
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 97d4f6a7bed5a0d47f18b3a223a916aef355eaf9
+Merge: 51cc015 46d626d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:50 2016 -0800
+
+    Merge pull request #7417 from dachary/wip-14553-hammer
+    
+    hammer: rbd: TaskFinisher::cancel should remove event from SafeTimer
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 51cc015c7e20d2ea9b9517238481fb80687be17c
+Merge: 23c24fc 73e03de
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:32 2016 -0800
+
+    Merge pull request #7407 from dillaman/wip-14543-hammer
+    
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 23c24fc50d07a07db12239ac372bfce5a9fe2255
+Merge: 9513391 26e832e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:12 2016 -0800
+
+    Merge pull request #6980 from dillaman/wip-14063-hammer
+    
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 91093041a86fda40de08a366b5118e5e3ae275f0 (refs/remotes/me/wip-14537-hammer)
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 28 02:09:53 2016 -0800
+
+    mon: compact full epochs also
+    
+    by compacting the ${prefix}.${start}..${prefix}..${end} does not
+    necessary compact the range of ${prefix}."full_"${start}..
+    ${prefix}."full_"${end}. so when more and more epochs get trimmed
+    with out a full range compaction, the size of monitor store could
+    be very large.
+    
+    Fixes: #14537
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 93d633a25ad8c36c972bb766c38187e2612041e1)
+
+commit 2817ffcf4e57f92551b86388681fc0fe70c386ec
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed Dec 16 15:31:52 2015 +0300
+
+    Check for full before changing the cached obc
+    
+    ReplicatedPG::prepare_transaction(): check if the pool is full before
+    updating the cached ObjectContext to avoid the discrepancy between
+    the cached and the actual object size (and other metadata).
+    While at it improve the check itself: consider cluster full flag,
+    not just the pool full flag, also consider object count changes too,
+    not just bytes.
+    
+    Based on commit a1eb380c3d5254f9f1fe34b4629e51d77fe010c1
+    
+    Fixes: #13335
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 951339103d35bc8ee2de880f77aada40d15b592a (refs/remotes/gh/wip-test-14716-4)
+Merge: e43aca5 5e5b512
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:31:47 2016 +0700
+
+    Merge pull request #6353 from theanalyst/wip-13513-hammer
+    
+    rgw: value of Swift API's X-Object-Manifest header is not url_decoded during segment look up
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit e43aca51d630274a57237b0f91a56df79ce8874a
+Merge: 7e20e6e 4420929
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:31:16 2016 +0700
+
+    Merge pull request #6620 from SUSE/wip-13820-hammer
+    
+    hammer: rgw: Setting ACL on Object removes ETag
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7e20e6e9d6626c5ac8b1f479011ab77a9e87da23
+Merge: f1d5570 cbb5c1f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:30:57 2016 +0700
+
+    Merge pull request #7186 from dachary/wip-13888-hammer
+    
+    hammer: rgw: orphans finish segfaults
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ae56de09fe1385958b5b600d1f0c91383989926f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 25 14:40:26 2015 -0500
+
+    osd: recency should look at newest (not oldest) hitsets
+    
+    Reported-by: xinxin shu <xinxin.shu@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 528eae92d010ea34ae8cf0e8b2290aaa5e058d24)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+                        code section was moved to ReplicatedPG::maybe_promote
+                        in master.
+    Signed-off-by: Robert LeBlanc <robert.leblanc@endurance.com>
+
+commit 5cefcb975771f0c2efb7dfc77ce14a93a4ee7f1b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 25 14:39:08 2015 -0500
+
+    osd/ReplicatedPG: fix promotion recency logic
+    
+    Recency is defined as how many of the last N hitsets an object
+    must appear in in order to be promoted.  The previous logic did
+    nothing of the sort... it checked for the object in any one of
+    the last N hitsets, which led to way to many promotions and killed
+    any chance of the cache performing properly.
+    
+    While we are here, we can simplify the code to drop the max_in_*
+    fields (no longer necessary).
+    
+    Note that we may still want a notion of 'temperature' that does
+    tolerate the object missing in one of the recent hitsets.. but
+    that would be different than recency, and should probably be
+    modeled after the eviction temperature model.
+    
+    Backport: infernalis, hammer
+    Reported-by: Nick Fisk <nick@fisk.me.uk>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 180c8743addc5ae2f1db9c58cd2996ca6e7ac18b)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+                        code section was moved to ReplicatedPG::maybe_promote
+                        in master.
+    Signed-off-by: Robert LeBlanc <robert.leblanc@endurance.com>
+
+commit f1d5570beab0769b925b917e402d441ff053794c
+Merge: c4bb343 50c82f2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:52:51 2016 -0500
+
+    Merge pull request #5789 from SUSE/wip-12928-hammer
+    
+    rpm: libcephfs_jni1 has no %post and %postun
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4bb34320df8bfa734512a400fe8664c131363ff
+Merge: 86ba6ca a5e4f70
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:50:56 2016 -0500
+
+    Merge pull request #7434 from tchaikov/wip-14441-hammer
+    
+    man: document listwatchers cmd in "rados" manpage
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a5e4f70af422b05ece53f245bc15491bb1dd540d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 23 11:23:38 2015 +0800
+
+    man: document listwatchers cmd in "rados" manpage
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c2e391301efc43f0b431e89737246b2c43bf10a9)
+    
+    man/rados.8: also added the rendered man.8 man page, as we don't
+                 put the generated man pages in master anymore. but
+                 they are still in the hammer's source repo.
+
+commit 46d626d92d7af218816d831cfcc1265b3ea31b72
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Fri Jan 22 11:18:40 2016 -0800
+
+    rbd: remove canceled tasks from timer thread
+    
+    When canceling scheduled tasks using the timer thread, TaskFinisher::cancel
+    does not call SafeTimer::cancel_event, so events fire anyway. Add this call.
+    
+    Fixes: #14476
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 2aa0f318c862dbe3027d74d345671506605778eb)
+
+commit 24c0b27c6f6a26c2b7bab5bcbc421a18592d026f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jan 21 13:45:42 2016 +0200
+
+    rbd-replay: handle EOF gracefully
+    
+    Fixes: #14452
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c59b84c3e2c9bbda68219e4d2288a889dd9ca6cb)
+    
+    Conflicts:
+    	src/rbd_replay/BufferReader.cc: trivial resolution
+            caused by s/CEPH_PAGE_SIZE/CEPH_BUFFER_APPEND_SIZE/
+            in the context of one hunk.
+
+commit 70f1ba33fe1a81d2631d54429749433a6cbfca44
+Author: Cilang Zhao <zhao.cilang@h3c.com>
+Date:   Tue Jan 5 14:34:05 2016 +0800
+
+    tools: monstore: add 'show-versions' command.
+    
+    Using this tool, the first/last committed version of maps will be shown.
+    
+    Signed-off-by: Cilang Zhao <zhao.cilang@h3c.com>
+    (cherry picked from commit 21e6ba0c18428caff45733e6b43d197be38af8bb)
+
+commit 926017187910c9e6a3fb8babf9b498cf07941819
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 16 18:28:52 2015 +0800
+
+    tools: ceph_monstore_tool: add inflate-pgmap command
+    
+    this command repeatly add the latest pgmap to the monstore in order
+    to inflate it. the command helps with the testing of some monstore
+    related performance issue of monitor
+    
+    Fixes: #14217
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b0f6af814a58dd374ca95e84a4e381f8ef1368de)
+
+commit a1d5728c928eb7e6b8c655741a0db3398ced7d56
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Tue Oct 20 15:23:49 2015 +0800
+
+    tools:support printing the crushmap in readable fashion.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit b7faf678ed0241abc7eb86b9baaa6db669a22d53)
+
+commit bd9484750f546974047dacd9176a8072be846596
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Mon Sep 14 19:50:47 2015 +0800
+
+    tools:print the map infomation in human readable format.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit 5ed8cdc19150382c946a373ec940d76f98e6ecb7)
+
+commit fba65832aad8a46d94a9256a56997e9df9e62297
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Mon Sep 14 19:19:05 2015 +0800
+
+    tools:remove the local file when get map failed.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit 0b03b32d8ba76fe9f6f1158e68eb440e3670393a)
+
+commit 1bb899a290b77188b44a53ef7c7a40910c9248b2
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Jul 13 12:35:13 2015 +0100
+
+    tools: ceph_monstore_tool: describe behavior of rewrite command
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit a881f9385feb0f5a61fa22357984d6f291c08177)
+
+commit 9035c69481f4aa4786414ac41cbc36fb4a3ca51d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 19 22:57:57 2015 +0800
+
+    tools/ceph-monstore-tools: add rewrite command
+    
+    "rewrite" command will
+     - add a new osdmap version to update current osdmap held by OSDMonitor
+     - add a new paxos version, as a proposal it will
+       * rewrite all osdmap epochs from specified epoch to  the last_committed
+         one with the specified crush map.
+       * add the new osdmap which is added just now
+    so the leader monitor can trigger a recovery process to apply the transaction
+    to all monitors in quorum, and hence bring them back to normal after being
+    injected with a faulty crushmap.
+    
+    Fixes: #11815
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 306373427836ca0c2418dbe6caab26d74d94d12e)
+
+commit 90aaed744243dfc7c620f91e19fc0bfa233f711f
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Apr 21 14:04:40 2015 +0800
+
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    this change fixes the "store-copy" command.
+    Like the bug reported in http://tracker.ceph.com/issues/10093.
+    
+    Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit d85e0f8c50fce62be012506f7ffcb1cdeb0c819c)
+
+commit c7252a3d54bae88f91c2b4e63fc9c27cfbb2423e
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Jan 18 08:24:46 2016 -0700
+
+    osd: disable filestore_xfs_extsize by default
+    
+    This option involves a tradeoff: When disabled, fragmentation is worse,
+    but large sequential writes are faster. When enabled, large sequential
+    writes are slower, but fragmentation is reduced.
+    
+    Fixes: #14397
+    (cherry picked from commit aed85775bf53c273786ce4999320134822722af5)
+
+commit 86ba6caf992d2544cdd174e3b3f26a6099c91fc5
+Merge: b6b8ee4 0325f8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 10:36:05 2016 +0700
+
+    Merge pull request #7316 from ceph/wip-deb-lttng-hammer
+    
+    deb: strip tracepoint libraries from Wheezy/Precise builds
+    
+    All other "modern" Debian-based OSes have a functional LTTng-UST. Since only hammer needs to build on these older distros, this fix only affects the deb building process for those two releases(since autoconf detects that LTTng is broken).
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b6b8ee41fc77679e82304e79da6dbd0c35f5c067
+Merge: d54840b f96c812
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 10:21:43 2016 +0700
+
+    Merge pull request #7187 from dachary/wip-13831-hammer
+    
+    hammer: init script reload doesn't work on EL7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 73e03def9271fb5d1739b195e428c3ebfcebd59b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:38:20 2016 -0500
+
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Blocking the notification thread will also result in librados async
+    callbacks becoming blocked (since they use the same thread).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7e2019a72733dff43e55c9b22df12939d584f87d)
+    
+     Conflicts:
+    	src/librbd/ImageWatcher.[cc|h]: fewer RPC messages
+
+commit 5617166f78c1995436b4e0794dab2d8254331815
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:35:54 2016 -0500
+
+    librados_test_stub: watch/notify now behaves similar to librados
+    
+    Notifications are executed via the same librados AIO callback
+    thread, so it's now possible to catch deadlock.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 43722571838a2d78ca8583b5a1ea381cd988de0b)
+
+commit 8fc82b23304ef327933723373cd4d1090d04bfbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 12:40:18 2016 -0500
+
+    tests: simulate writeback flush during snap create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e9570fcf4be9cd5c06937769d074fa52ffb32e4b)
+
+commit d54840bf4a70fc65285bbfdff0c7bf8f579643b1
+Merge: 4051bc2 9f30fe1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 26 10:12:29 2016 -0500
+
+    Merge pull request #7365 from liewegas/wip-tcmalloc-hammer
+    
+    osd: pass tcmalloc env through to ceph-osd
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9f30fe18e839f5846c90e3b5995171a0132d7f3a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 26 08:43:15 2016 -0500
+
+    upstart/ceph-osd.conf: pass TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES through
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a95a3d34cccb9f9f0782a96ca05e5fe9c2b01772)
+
+commit a58873e213d7423d89c95db4e1710dc9631e3313
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 20 18:36:08 2016 -0500
+
+    init-ceph: pass TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES through
+    
+    ..when set in the sysconfig/default file.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 03e01ae263ef207046933890821ae75d5e60d9b8)
+    
+    [source default/sysconfig file too]
+
+commit 8d9e08c44150a4f3ad06ec1abc130b9aeaf479d9
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Apr 21 14:04:40 2015 +0800
+
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    this change fixes the "store-copy" command.
+    Like the bug reported in http://tracker.ceph.com/issues/10093.
+    
+    Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit d85e0f8c50fce62be012506f7ffcb1cdeb0c819c)
+
+commit 0325f8af5cbee3d74e9f363f61c2e2ababf501d9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 21 15:36:37 2016 -0500
+
+    deb: strip tracepoint libraries from Wheezy/Precise builds
+    
+    These releases do not (by default) have a sane LTTng-UST environment,
+    which results in autoconf disabling support for tracing.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4051bc2a5e4313ac0f6236d7a34ed5fb4a1d9ea2
+Merge: 4e67418 e4d3e9b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 18 20:46:07 2016 -0500
+
+    Merge pull request #7252 from ceph/wip-13483-hammer
+    
+    qa: remove legacy OS support from rbd/qemu-iotests
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e4d3e9b29aa6a5f9efff8b787949a5bbf48ad7de (refs/remotes/gh/wip-13483-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 18 15:57:43 2016 -0500
+
+    qa: disable rbd/qemu-iotests test case 055 on RHEL/CentOS
+    
+    Fixes: #14385
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7acd44c26310242b69ee50322bd6b43fdc774b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 15 12:44:06 2015 -0400
+
+    qa: remove legacy OS support from rbd/qemu-iotests
+    
+    RHEL7 derivatives were failing test 002 since they were using
+    legacy test cases for now unsupported OSes.
+    
+    Fixes: #13483
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 8c28f2f28d960d823ffd632671edaf029c30fb0f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:41:49 2015 -0500
+
+    librbd: optionally validate RBD pool configuration (snapshot support)
+    
+    Fixes: #13633
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fea4dadc60e13518e9ee55d136fbc4e9d3a621e)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 700be56c530879a72a628c62265d18f0a5d8fb3b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 14 08:35:23 2016 -0800
+
+    config_opts: increase suicide timeout to 300 to match recovery
+    
+    Fixes: 14376
+    Backport: hammer, firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 4e67418958e5caf5e4f81c4ed566e8c7269930fa
+Merge: 28e99a9 1ab2b48
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:21:50 2016 -0500
+
+    Merge pull request #7179 from dachary/wip-14287-hammer
+    
+    hammer: ReplicatedPG: wrong result code checking logic during sparse_read
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 28e99a90b38ce8c0285c1aec9c5524365385be00
+Merge: 17a2965 86f5cf6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:53 2016 -0500
+
+    Merge pull request #7178 from dachary/wip-14285-hammer
+    
+    hammer: osd/OSD.cc: 2469: FAILED assert(pg_stat_queue.empty()) on shutdown
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 17a2965825494765249c244aef59ebd821711d42
+Merge: a1459ea b0856ee
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:34 2016 -0500
+
+    Merge pull request #7177 from dachary/wip-14043-hammer
+    
+    hammer: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a1459eacd7cc0d064493d01f963b8c1231cd3518
+Merge: c9a13a1 9cee89b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:10 2016 -0500
+
+    Merge pull request #7180 from dachary/wip-14288-hammer
+    
+    hammer: ceph osd pool stats broken in hammer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c9a13a12e88c9a18bfb3cc99d4a7c103b4330426
+Merge: 174a1a9 4d0b9a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:19:18 2016 -0500
+
+    Merge pull request #6994 from badone/wip-13993-hammer
+    
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 174a1a974725ce4025be4dfdf3b13df766dfac76
+Merge: cb167e9 6cf261c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:18:55 2016 -0500
+
+    Merge pull request #6839 from SUSE/wip-13789-hammer
+    
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cb167e998bbbdf1b9c273f6bc4f118796d396d37
+Merge: 1c4ba85 66ff0aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:18:23 2016 -0500
+
+    Merge pull request #6835 from SUSE/wip-13892-hammer
+    
+    hammer: auth/cephx: large amounts of log are produced by osd
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c4ba855b34290069c7ce0fa990fe72b7a1b381e
+Merge: 72b6b68 61da13b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:17:53 2016 -0500
+
+    Merge pull request #6834 from SUSE/wip-13930-hammer
+    
+    hammer: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 72b6b68f43ffc953ca7f3056574bd68428a5abe8
+Merge: d064f9f 53e81aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:17:25 2016 -0500
+
+    Merge pull request #6832 from SUSE/wip-13936-hammer
+    
+    hammer: Ceph daemon failed to start, because the service name was already used.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d064f9f6b3bc93fa5887304a9b9ab92d503ebd15
+Merge: 0ca6285 ac05617
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:16:54 2016 -0500
+
+    Merge pull request #6755 from SUSE/wip-13870-hammer
+    
+    hammer: OSD: race condition detected during send_failures
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0ca62855f22136c7042a5988366d46e59629f206
+Merge: e9f545b a81bcf7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:22:35 2016 +0100
+
+    Merge pull request #6415 from dillaman/wip-13541-hammer
+    
+    LTTng-UST tracing should be dynamically enabled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e9f545b4a44c91faf91a8ae3824b1b0d46909244
+Merge: 8521916 b2f1e76
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:15:02 2016 +0100
+
+    Merge pull request #7176 from dachary/wip-13440-hammer
+    
+    ceph-disk prepare fails if device is a symlink
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 85219163f9b78faa3c1fe96cad7129c6813550c0
+Merge: ebfb3fb 5264bc6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:14:31 2016 +0100
+
+    Merge pull request #7150 from jecluis/wip-14236
+    
+    mon: OSDMonitor: do not assume a session exists in send_incremental()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ebfb3fbe8a1920474941d4a95540cc4a53a747d4
+Merge: e1b9208 447ab1d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 13 17:15:32 2016 -0800
+
+    Merge pull request #7226 from dillaman/wip-13810-hammer
+    
+    tests: notification slave needs to wait for master
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 447ab1d70a63fceed39518d5714c138c54f6525e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 13 12:44:01 2016 -0500
+
+    tests: notification slave needs to wait for master
+    
+    If the slave instance starts before the master, race
+    conditions are possible.
+    
+    Fixes: #13810
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3992d6fe67bbf82322cedc1582406caaf6d4de60)
+
+commit 53742bdbc587747c93413f9db38f3eeb71487872
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 15 17:55:41 2015 -0700
+
+    ceph_osd: Add required feature bits related to this branch to osd_required mask
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0b2bab460c013ea5cece10ea028d26da3485eaf5)
+
+commit 3066231865e2fe56344de9db26024ac65e03053d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 4 18:47:42 2015 -0700
+
+    osd: CEPH_FEATURE_CHUNKY_SCRUB feature now required
+    
+    Feature present since at least the Dumpling release.
+    A later commit will add it to the osd_required mask
+    
+    Fixes: #11661
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7a10a7e196efd0f59fa7adb87d0a120655b227d8)
+
+commit e1b92081c9e4b21eb30cc873c239083a08fce12f
+Merge: 9708e59 3d3595f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jan 12 14:28:26 2016 +0100
+
+    Merge pull request #6530 from SUSE/wip-13760-hammer
+    
+    unknown argument --quiet in udevadm settle
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6379ff16dbc7fdecccaa8874d978d1ab58ce44cf
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 5 15:07:33 2015 +0800
+
+    configure.ac: no use to add "+" before ac_ext=c
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 899dd23766c5ae6fef39bf24ef0692127d17deb3)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit f96c8125d423f24e52df0f8420b0b2748e34ca90
+Author: Herve Rousseau <hroussea@cern.ch>
+Date:   Fri Nov 6 09:52:28 2015 +0100
+
+    rgw: fix reload on non Debian systems.
+    
+    When using reload in non-debian systems, /bin/sh's kill is used to send the HUP signal to the radosgw process.
+    This kill version doesn't understand -SIGHUP as a valid signal, using -HUP does work.
+    
+    Fix: #13709
+    Backport: hammer
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 1b000abac3a02d1e788bf25eead4b6873133f5d2)
+
+commit cbb5c1fc2386205277ad22474c1f696a07fcb972
+Author: Igor Fedotov <ifedotov@mirantis.com>
+Date:   Thu Nov 19 13:38:40 2015 +0300
+
+    Fixing NULL pointer dereference
+    
+    Signed-off-by: Igor Fedotov <ifedotov@mirantis.com>
+    (cherry picked from commit 93d3dfe0441be50a6990d458ee0ee3289af39b20)
+
+commit b2961cef3d8e84d2302815e33eb7dc9033d2bb78
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 18 16:05:29 2015 -0400
+
+    rbd: fix bench-write
+    
+    Make each IO get a unique offset!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 333f3a01a9916c781f266078391c580efb81a0fc)
+
+commit 9cee89bbdd1d89d2fe72c8d73fc0c18db515198a
+Author: BJ Lougee <almightybeeij@gmail.com>
+Date:   Mon Apr 13 11:24:38 2015 -0500
+
+    Check that delta_sum.stats.sum.num_object_copies and delta_sum.stats.sum.num_object are greater than zero
+    
+    This fixes division by zero.
+    
+    Signed-off-by BJ Lougee <almightybeeij@gmail.com>
+    
+    (cherry picked from commit 27ed729c1088133400aa072eeca9e125942f2d94)
+
+commit 1ab2b481596ad5296e168bf8027e59d41190176e
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Dec 22 17:05:06 2015 +0800
+
+    ReplicatedPG: fix sparse-read result code checking logic
+    
+    Move ahead the result code checking logic before we continue to verify the trailing hole, otherwise
+    the real result of non-hole reading may be overwritten and thus confuse caller.
+    
+    Fixes: #14151
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit b3aa29e324cf0a96a1f2f5dcf1ba998219457bcd)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc: trivial resolution
+
+commit 86f5cf6caa36760802775df19cdabe55bcafa33e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 1 09:32:04 2016 -0500
+
+    osd: clear pg_stat_queue after stopping pgs
+    
+    Fixes: #14212
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c25ff9927febe77cda31b534971b62f75c196ca2)
+
+commit b0856ee67c0e7cf6ab6095d3f657c18014859526
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 3 14:52:24 2015 -0800
+
+    osd: Test osd_find_best_info_ignore_history_les config in another assert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 02a9a41f151a3d968bf8066749658659dc6e3ac4)
+
+commit b2f1e76d8e95b6e2f88fa1c122ea8cd24532cd10
+Author: Joe Julian <jjulian@io.com>
+Date:   Fri Oct 9 12:57:06 2015 -0700
+
+    Compare parted output with the dereferenced path
+    
+    Compare parted output with the dereferenced path of the device as parted
+    prints that instead of the symlink we called it with.
+    
+    http://tracker.ceph.com/issues/13438 Fixes: #13438
+    
+    Signed-off-by: Joe Julian <jjulian@io.com>
+    (cherry picked from commit b3c7cb098195111b9c642e5a9b726b63717f2e0d)
+
+commit 9708e59a1fc2ae52cfae848ce585751bc9fbe572
+Merge: 9739d4d b62cac6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jan 11 09:01:12 2016 +0100
+
+    Merge pull request #6512 from SUSE/wip-13734-hammer
+    
+    rgw: swift API returns more than real object count and bytes used when retrieving account metadata
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5264bc677f0f612165987bf51fe7d7b4af32fa77
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 19:20:47 2016 +0000
+
+    mon: OSDMonitor: do not assume a session exists in send_incremental()
+    
+    We may not have an open session for a given osd. If we blindly assume we
+    do, we may end up trying to send incrementals we do not have to the osd.
+    
+    And then we will crash.
+    
+    This fixes a regression introduced by
+    
+      171fee1b82d2675e364da7f96dfb9dd286d9b6e6
+    
+    which is meant as a backport of
+    
+      de43a02e06650a552f048dc8acd17f255126fed9
+    
+    but so happens to intruduce a line that wasn't on the original patch. We
+    imagine it was meant to make the 's->osd_epoch' assignment work without
+    checking the session, as per the original patch, but the backporter must
+    have forgotten to also backport the assertion on the not-null session.
+    The unfortunate introduction of the check for a not-null session
+    triggered this regression.
+    
+    The regression itself is due to enforcing that a session exists for the
+    osd we are sending the incrementals to. However, if we come via the
+    OSDMonitor::process_failures() path, that may very well not be the case,
+    as we are handling potentially-old MOSDFailure messages that may no
+    longer have an associated session. By enforcing the not-null session, we
+    don't check whether we have the requested versions (i.e., if
+    our_earliest_version <= requested_version), and thus we end up on the
+    path that assumes that we DO HAVE all the necessary versions -- when we
+    may not, thus finally asserting because we are reading blank
+    incremental versions.
+    
+    Fixes: #14236
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 4d0b9a1bbbabe7b27279a7b6e0a45f5b0d920c66
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Dec 7 11:31:28 2015 +1000
+
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Fixes: #13993
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8e93f3f45db681f82633ca695a7dc4e7bd030584)
+
+commit 26e832e76de90e0a751868b044ea745a97a5af82
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 18 15:22:13 2015 -0500
+
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Fixes: #14063
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9739d4de49f8167866eda556b2f1581c068ec8a7
+Merge: d064636 a9d3f07
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 17 10:16:29 2015 -0500
+
+    Merge pull request #6544 from liewegas/wip-smaller-object-info
+    
+    osd: make encoded object_info_t smaller to fit inside the XFS inode
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a9d3f078cb6cdaf7f8778ba1a07d333548a9d1a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 13:13:33 2015 -0500
+
+    osd/osd_types: skip encoding newer object_info_t fields if they are unused
+    
+    This reduces the size of the encoded object_info_t in most cases,
+    enough to get us under the 255 byte limit for a single inline
+    xattr in XFS.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1548a3fda7dd63e28792140e7e8ad1ac9b706e49
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 13:00:27 2015 -0500
+
+    osd/ReplicatedPG: do not set local_mtime on non-tiered pool
+    
+    If a pool isn't tiered, don't bother with setting local_mtime.  The only
+    users are the tiering agent (which isn't needed if there is not tiering)
+    and scrub for deciding if an object should get its digest recorded (we can
+    use mtime instead).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 98bdb09bb8d632b2df0b252b3d9676fd9511a1c8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 12:59:29 2015 -0500
+
+    osd/PGBackend: use mtime for digest decision if local_mtime is empty
+    
+    If we don't have a local_mtime value, use mtime instead, for the purposes
+    of deciding if we should record a digest after scrub.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit fe1c28dea4e5607a0c72eab1f046074616cd55a7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 11:35:46 2015 -0500
+
+    osd/ReplicatedPG: do not set omap digest if there is no omap
+    
+    We want to avoid encoding it if we can.  And if the FLAG_OMAP is not set
+    we don't need to *also* store an empty crc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6cf261cd2e80adbaf2898cb354c8159a57749e9d
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Nov 2 21:46:11 2015 +0800
+
+    Objecter: remove redundant result-check of _calc_target in _map_session.
+    
+    Result-code check is currently redundant since _calc_target never returns a negative value.
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 5a6117e667024f51e65847f73f7589467b6cb762)
+
+commit 5d0b5c1389d8c0baddb40cc5ff8f1d9cc6a4d315
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 17:32:50 2015 +0800
+
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Objecter: potential null pointer access when do pool_snap_list. Shall check pool existence first.
+    Fixes: #13639
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 865541605b6c32f03e188ec33d079b44be42fa4a)
+
+commit 66ff0aa0130973aa2bea3a1a4ec8f4e3e6c19da8
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Thu Nov 5 07:29:49 2015 -0500
+
+    auth/cephx: large amounts of log are produced by osd
+    if the auth of osd is deleted when the osd is running, the osd will produce large amounts of log.
+    
+    Fixes:#13610
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 102f0b19326836e3b0754b4d32da89eb2bc0b03c)
+
+commit 61da13bbe271964c56116fb2bed51045290f951a
+Author: Chengyuan Li <chengyli@ebay.com>
+Date:   Thu Nov 19 22:29:39 2015 -0700
+
+    mon/PGMonitor: MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    In get_rule_avail(), even p->second is 0, it's possible to be used
+    as divisor and quotient is infinity, then is converted to an integer
+    which is negative value.
+    So we should check p->second value before calculation.
+    
+    It fixes BUG #13840.
+    
+    Signed-off-by: Chengyuan Li <chengyli@ebay.com>
+    (cherry picked from commit 18713e60edd1fe16ab571f7c83e6de026db483ca)
+
+commit 53e81aab442855b34ee6e922f181bae3bf33e292
+Author: wangchaunhong <root@A22832429.(none)>
+Date:   Tue Oct 20 18:40:23 2015 +0800
+
+    init-ceph: fix systemd-run cant't start ceph daemon sometimes
+    
+    Fixes: #13474
+    Signed-off-by: Chuanhong Wang <wang.chuanhong@zte.com.cn>
+    (cherry picked from commit 2f36909e1e08bac993e77d1781a777b386335669)
+    
+    Conflicts:
+    	src/init-ceph.in
+                different content of cmd variable
+
+commit c2c6d02591519dfd15ddcb397ac440322a964deb
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Mon Jul 6 09:56:11 2015 +0200
+
+    tools: fix race condition in seq/rand bench
+    
+    Under certain conditions (like bench seq/rand -b 1024 -t 128) it is
+    possible that aio_read reads data into destination buffers before or
+    during memcmp execution, resulting in "[..] is not correct!" errors
+    even if actual objects are perfectly fine.
+    Also, moved latencty calculation around, so it is no longer affeted
+    by memcmp.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+
+commit a619b621b0a7c670eeaf163d9e2b742d13c9f517
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Wed May 20 12:41:22 2015 +0200
+
+    tools: add --no-verify option to rados bench
+    
+    When doing seq and rand read benchmarks using rados bench, a quite large
+    portion of cpu time is consumed by doing object verification. This patch
+    adds an option to disable this verification when it's not needed, in turn
+    giving better cluster utilization. rados -p storage bench 600 rand scores
+    without --no-verification:
+    
+    Total time run:       600.228901
+    Total reads made:     144982
+    Read size:            4194304
+    Bandwidth (MB/sec):   966
+    Average IOPS:         241
+    Stddev IOPS:          38
+    Max IOPS:             909522486
+    Min IOPS:             0
+    Average Latency:      0.0662
+    Max latency:          1.51
+    Min latency:          0.004
+    
+    real    10m1.173s
+    user    5m41.162s
+    sys     11m42.961s
+    
+    Same command, but with --no-verify:
+    
+    Total time run:       600.161379
+    Total reads made:     174142
+    Read size:            4194304
+    Bandwidth (MB/sec):   1.16e+03
+    Average IOPS:         290
+    Stddev IOPS:          20
+    Max IOPS:             909522486
+    Min IOPS:             0
+    Average Latency:      0.0551
+    Max latency:          1.12
+    Min latency:          0.00343
+    
+    real    10m1.172s
+    user    4m13.792s
+    sys     13m38.556s
+    
+    Note the decreased latencies, increased bandwidth and more reads performed.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit ca6abca63de813c83a6960f83624be8e1a86a1f8)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    	src/common/obj_bencher.h
+
+commit d06463604cb8daeda288e824e8812352c0d6a7d9
+Merge: ec35347 609f256
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:48:00 2015 +0100
+
+    Merge pull request #6527 from theanalyst/wip-12856-hammer
+    
+    rgw: missing handling of encoding-type=url when listing keys in bucket
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ec35347d8fd7a0eb175710e6111a9196746e278c
+Merge: 487dc34 1b06f03
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:43 2015 +0100
+
+    Merge pull request #6491 from SUSE/wip-13716-hammer
+    
+    rgw:swift use Civetweb ssl can not get right url
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 487dc3436a8fabd1b1a31563071c907bdf18f525
+Merge: 18d3ba3 99b4d1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:27 2015 +0100
+
+    Merge pull request #6351 from theanalyst/wip-13538-hammer
+    
+    rgw: orphan tool should be careful about removing head objects
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 18d3ba30323a44c1cc45bf4e870b8d9aeb5c3b0b
+Merge: 36e4db3 6d89f4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:09 2015 +0100
+
+    Merge pull request #6349 from theanalyst/wip-13540-hammer
+    
+    rgw: get bucket location returns region name, not region api name
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 36e4db36f5ec9418672bbfe0dce6ae757f697efe
+Merge: 8c4145e db1cbe7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:46:49 2015 +0100
+
+    Merge pull request #5910 from kmroz/wip-hammer-backport-content-type
+    
+    rgw: backport content-type casing
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ac05617b246d3c2a329b1b99b0371e3f6b8541e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 17 21:42:53 2015 -0400
+
+    osd: fix send_failures() locking
+    
+    It is unsafe to check failure_queue.empty() without the lock.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3ca828ae8ebc9068073494c46faf3e8e1443ada)
+
+commit 8c4145ecc4a68accdb2120889fd933e8f6630dba
+Merge: 4804eec 112c686
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Nov 25 08:47:10 2015 -0800
+
+    Merge pull request #6587 from theanalyst/wip-13758-hammer
+    
+    common: pure virtual method called
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4804eec095959c3747b13d29224b2cfa5b7d198d
+Merge: 9b97e4f e693755
+Author: Daniel Gryniewicz <dang@fprintf.net>
+Date:   Wed Nov 25 10:49:15 2015 -0500
+
+    Merge pull request #6352 from theanalyst/wip-13536-hammer
+    
+    rgw: bucket listing hangs on versioned buckets
+
+commit 9b97e4f6a41bb3fe7bae92b71ae266361022cf5c
+Merge: 5a9e0a7 0378445
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 25 16:48:09 2015 +0100
+
+    Merge pull request #6589 from theanalyst/wip-13693-hammer
+    
+    osd: bug with cache/tiering and snapshot reads
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5a9e0a722c654b83fc483e083ddb3035281f5397
+Merge: 3047f2b a322317
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 25 16:22:10 2015 +0100
+
+    Merge pull request #6585 from theanalyst/wip-13753-hammer
+    
+    rbd: avoid re-writing old-format image header on resize
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3047f2bc50d5d26e70e832bd427f15073bf7733c
+Merge: 407bd02 9c33dcc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 19:57:03 2015 +0100
+
+    Merge pull request #6586 from theanalyst/wip-13755-hammer
+    
+    rbd: QEMU hangs after creating snapshot and stopping VM
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 407bd0281582ebb3b50ce51bdb48244e22463c8f
+Merge: c3c400f bddbda1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 17:32:21 2015 +0100
+
+    Merge pull request #6588 from theanalyst/wip-13770-hammer
+    
+    Objecter: pool op callback may hang forever.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c3c400f68d9c2593acd9485d1214d14af4e930d0
+Merge: d116959 a52f7cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 15:13:24 2015 +0100
+
+    Merge pull request #6430 from SUSE/wip-13654-hammer
+    
+    crush: crash if we see CRUSH_ITEM_NONE in early rule step
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3f1292e117ab1cdcaef3b24ee33854f7be142795
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 3 20:30:50 2015 +0200
+
+    ceph.spec.in: fix License line
+    
+    This is closer to my reading of
+    https://github.com/ceph/ceph/blob/master/COPYING than the previous version.
+    
+    http://tracker.ceph.com/issues/12935 Fixes: #12935
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ef7418421b3748c712019c8aedd02b8005c1e1ea)
+
+commit d116959442f67c8f36898ca989b490ca84a609c6
+Merge: 12be099 74203b8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 09:28:09 2015 +0100
+
+    Merge pull request #6420 from SUSE/wip-13637-hammer
+    
+    FileStore: potential memory leak if getattrs fails.
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 12be099a7b0f2692c167af96928e939b2227160c
+Merge: 39c70d4 2052187
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 09:10:27 2015 +0100
+
+    Merge pull request #6528 from theanalyst/wip-13695-hammer
+    
+    init-rbdmap uses distro-specific functions
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 39c70d4364a1be39f7c393847417f44279b4364c
+Merge: 65aeba0 9643ee6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 09:07:13 2015 +0100
+
+    Merge pull request #6499 from SUSE/wip-13692-hammer
+    
+    osd: do not cache unused memory in attrs
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 65aeba0b2f165d893d06df7e9bbe25f989edafe9
+Merge: 24d6698 394fbfc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 08:59:25 2015 +0100
+
+    Merge pull request #6143 from dachary/wip-13340-hammer
+    
+    small probability sigabrt when setting rados_osd_op_timeout
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 24d66981d351848c44ae9e0395dad392f852be2b
+Merge: 5d204db 171fee1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 08:58:11 2015 +0100
+
+    Merge pull request #5773 from tchaikov/wip-12835-hammer
+    
+    mon: map_cache can become inaccurate if osd does not receive the osdmaps
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a81bcf723c1099f2bea5daf8b01b7d9853de323a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 15 00:15:54 2015 -0400
+
+    ceph_context: remove unsafe cast for singletons
+    
+    It was previously assumed that a CephContext singleton would
+    inherit from CephContext::AssociatedSingletonObject, but it was
+    not enforced.  This could result in unknown behavior when the
+    singleton is destroyed due to the implied virtual destructor.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fb62c78637d7092f48871d943282f45029bd6d29)
+
+commit d50d7b2fa3751e8520694ee75eefe5ae56e57267
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:53:28 2015 -0400
+
+    osd: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0009f343a5daf28358a669836d95660c0d9068d8)
+
+commit bf34b3657339dc40c7939fcdddaf2b7ae78c82ad
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:37:05 2015 -0400
+
+    librados: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6368c281b53d6175a564725b85516de4b6ae54de)
+
+commit afc4f176f8371f0e61ce3463602f8a355b9283b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:10:10 2015 -0400
+
+    librbd: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1a6eea95bd1ccef53f01cf53e493e652faa203ee)
+    
+    Conflicts:
+    	src/librbd/librbd.cc: trivial resolution
+
+commit c82f93915875875b21ed779e7aaf297b8fbcdca1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 19:17:08 2015 -0400
+
+    common: new tracing config options
+    
+    Config options to enable LTTng-UST tracepoint providers for
+    the OSD, OSD objectstore, librados, and librbd.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ccef89badb1f7a63721a0bf48d05eb4c2d0a97a)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 2168c151066e0cea86284ffdf947a353f3b323d5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 18:49:32 2015 -0400
+
+    tracing: merge tracepoint provider makefiles into main makefile
+    
+    This results in the tracepoint provider shared libraries being
+    placed in the library path for unittests.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e11d8d68da27a1c224aa55fc40a72ac6aaa0441c)
+    
+    Conflicts:
+    	configure.ac: trivial resolution
+    	src/Makefile.am: trivial resolution
+
+commit d02beff1cf7650fe6e57cdafe64dcbee2631ed52
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 15:13:24 2015 -0400
+
+    tracing: dynamic tracepoint provider helper
+    
+    The TracepointProvider class is a configuration observer.  When
+    tracing is enabled, it will dynamically load the associated
+    tracepoint provider.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3d02cc21b4cc40a194f84a9bbbf82cf9e7956d1)
+    
+    Conflicts:
+    	src/common/Makefile.am: trivial resolution
+
+commit e53d66e42b1c3aea47832f7e8983284ec45d9efa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:33:48 2015 -0400
+
+    packaging: add new tracepoint probe shared libraries
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a7ed8e161b27c7852bf0f3a23a977ebd39137943)
+
+commit bb7c0f501918b75dee97052faeafb1d6db61bd0a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:30:43 2015 -0400
+
+    ceph.spec.in: add new tracepoint probe shared libraries
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4feee2f9dcd7b94476701b66d1f0bdf6fb6e0c2)
+    
+     Conflicts:
+    	ceph.spec.in: trivial resolution
+
+commit e1da27134e4c9b4a2881aca664818598e5b2125b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:13:46 2015 -0400
+
+    lttng: move tracepoint probes to dynamic libraries
+    
+    LTTng-UST initializes itself at program load, which means it is
+    currently always enabled.  This can lead to issues with SElinux
+    and AppArmor which might restrict access to the necessary device
+    files.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a5305e41677885d45b2b74299113a3d00189c66)
+    
+     Conflicts:
+    	src/test/Makefile-client.am: trivial resolution
+    	src/tracing/Makefile.am: trivial resolution
+
+commit 5d204db85b2ed84411cfabea9eb73aa04c6ce05a
+Merge: 1dbd4c6 8378aaf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:45:25 2015 +0100
+
+    Merge pull request #6580 from dachary/wip-13786-hammer
+    
+    rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1dbd4c66307e615670c7569f107418876f8b977f
+Merge: 24a5dc0 634d7f6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:45:03 2015 +0100
+
+    Merge pull request #6286 from dillaman/wip-13460-hammer
+    
+    rbd-replay-prep and rbd-replay improvements
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 24a5dc0a6b4f73c92c82e1933a1924db3f8e1090
+Merge: d27da2e 6a40e4f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:41:20 2015 +0100
+
+    Merge pull request #5790 from SUSE/wip-12932-hammer
+    
+    Miscellaneous spec file fixes
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d27da2ee8fa3e074fdd91d414df2d59978db9497
+Merge: 7d21127 8358fb8
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 19 19:13:11 2015 +0800
+
+    Merge pull request #6644 from dachary/wip-13812-upgrade
+    
+    revert: osd: use GMT time for hitsets
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8358fb8946d8809d695092baa4a6abf5d5b5e265
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 18 18:08:58 2015 +0100
+
+    revert: osd: use GMT time for hitsets
+    
+    "Merge pull request #5825 from tchaikov/wip-12848-hammer"
+    
+    This reverts commit 39544718dc2f09bcfdc632ac72fd2a3cda87687e, reversing
+    changes made to 4ad97162026e1eb6e6e948ddf3eb39f711431e45.
+    
+    http://tracker.ceph.com/issues/13812 Fixes: #13812
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 4420929eacb67d17635fed53f15471a28942f31d
+Author: brian <bjfelton@gmail.com>
+Date:   Wed Sep 23 09:49:36 2015 -0500
+
+    rgw: fix modification to index attrs when setting acls
+    
+    Fixes: #12955
+    
+     - add ACL change after initial population
+     - populate bufferlist with object attributes
+    
+    Signed-off-by: Brian Felton <bjfelton@gmail.com>
+    (cherry picked from commit 7496741ebbd75b74d5ffeca5341cccb2318176e6)
+
+commit 7d21127f433afa2d9172954e7b8ff47c40d2d62b
+Merge: 6930601 1448915
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 17 10:24:45 2015 +0100
+
+    Merge pull request #6402 from SUSE/wip-13621-hammer
+    
+    CephFS restriction on removing cache tiers is overly strict
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 693060102594a2c505bed1e7e274731b554e9179
+Merge: c2daf09 3f33ce6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 22:06:51 2015 +0100
+
+    Merge pull request #6354 from theanalyst/wip-13387-hammer
+    
+    librbd: reads larger than cache size hang
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c2daf09fc6b8802240e13367a0a2f3349a7cca56
+Merge: 1ca72fb af734e6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 19:48:07 2015 +0100
+
+    Merge pull request #6289 from dillaman/wip-13461-hammer
+    
+    librbd: invalidate object map on error even w/o holding lock
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1ca72fbe6be0d664a3cbbd4356a8547c3b749a03
+Merge: ca573fe 3e65730
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:25 2015 +0100
+
+    Merge pull request #6153 from dachary/wip-13205-hammer
+    
+    ReplicatedBackend: populate recovery_info.size for clone (bug symptom is size mismatch on replicated backend on a clone in scrub)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ca573fe7ac5a699b77cacb8fedfa50fffc77dd52
+Merge: fdb3446 7161a2c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:12 2015 +0100
+
+    Merge pull request #6158 from dachary/wip-13336-hammer
+    
+    osd: we do not ignore notify from down osds
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fdb3446194743135b78a65c30a8285d001e563b6
+Merge: f017566 0ad9521
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:01 2015 +0100
+
+    Merge pull request #6335 from Abhishekvrshny/wip-13488-hammer
+    
+    object_info_t::decode() has wrong version
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f017566823af70323b7d4cbe1f7b3b761378d07e
+Merge: a79acd4 9f3aebe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:05:39 2015 +0100
+
+    Merge pull request #6401 from SUSE/wip-13620-hammer
+    
+    osd: pg stuck in replay
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8378aaf3f9d5463fb61a19d601201dd63a884419
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 13 19:10:28 2015 +0100
+
+    build/ops: rbd-replay moved from ceph-test-dbg to ceph-common-dbg
+    
+    http://tracker.ceph.com/issues/13785 Fixes: #13785
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a0204dc47acc0c50223e449fd23a2fc97bfd111a)
+    
+        Conflicts:
+        	debian/control: << 0.94.5-2 is appropriate for hammer
+
+commit 03784457ab77617ddafc048b01044858b1e65bd3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 29 22:26:48 2015 +0800
+
+    osd: fix the snapshot reads of evicted tiering pool
+    
+    reset ssc->exsits in finish_ctx() if the ctx->cache_evict is true, and
+    the head is removed.
+    
+    Fixes: #12748
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b61f3e43f1c9d43daa0dedd7cbd0fe5787cbdfbb)
+
+commit bddbda121c8d52ba8e50abd0341f14e332c6af5a
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 20:04:11 2015 +0800
+
+    Objecter: pool_op callback may hang forever.
+    
+    pool_op callback may hang forever due to osdmap update during reply handling.
+    Fixes: #13642
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 00c6fa9e31975a935ed2bb33a099e2b4f02ad7f2)
+
+commit 112c686ffc98e3739c9944635f4044e2b34f210a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 7 12:11:13 2015 -0400
+
+    WorkQueue: new PointerWQ base class for ContextWQ
+    
+    The existing work queues do not properly function if added to a running
+    thread pool.  librbd uses a singleton thread pool which requires
+    dynamically adding/removing work queues as images are opened and closed.
+    
+    Fixes: #13636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e78b18b09d75626ca2599bac3b9f9c9889507a5)
+    
+    Conflicts:
+    	src/common/WorkQueue.h
+    Trivial merge conflict at class `ContextWQ` initialization
+
+commit 9c33dccaad9a4cdd46e9ecfa1e3ba6c03d95885a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 11:22:24 2015 -0500
+
+    librbd: fixed deadlock while attempting to flush AIO requests
+    
+    In-flight AIO requests might force a flush if a snapshot was created
+    out-of-band.  The flush completion was previously invoked asynchronously,
+    potentially via the same thread worker handling the AIO request. This
+    resulted in the flush operation deadlocking since it can't complete.
+    
+    Fixes: #13726
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfeb90e5fe24347648c72345881fd3d932243c98)
+
+commit b3b7877f9b4b3f43acab09d0dd6ee971b6aa1c29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 10:48:10 2015 -0500
+
+    tests: new test case to catch deadlock on RBD image refresh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a9729d9553e7fb925509cad8d388cf52a9fede9c)
+
+commit a3223173f85c42147ff4ced730beffe85146a4ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 2 16:50:19 2015 -0500
+
+    librbd: resize should only update image size within header
+    
+    Previously, the whole RBD image format 1 header struct was
+    re-written to disk on a resize operation.
+    
+    Fixes: #13674
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5be20b6d4646284571568ab28cbf45b0729390b)
+
+commit 6a40e4f19d7b4cd45a25161303c7363e96fe799e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Sep 29 10:39:16 2015 +0200
+
+    ceph.spec.in: lttng in SLES12 only
+    
+    Over in the SUSE sector, we are trying to enable the SLE_12 and openSUSE_13.2
+    build targets. The lttng/babeltrace stuff is currently available only in
+    SLE_12.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bf9ca1eb107c4462d8768faf3ff4db7972384ffd)
+    
+    Conflicts:
+    	ceph.spec.in
+                trivial resolution
+
+commit e508a44485366557ac8a280de35584f3b5edf720
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 15 12:54:20 2015 +0200
+
+    ceph.spec.in: fix lttng/babeltrace conditionals
+    
+    lttng and babeltrace are build dependencies for rbd-replay-prep. Make
+    sure the right package names are used. Enable for SLE12, as well as
+    for openSUSE 13.1 and higher.
+    
+    Move the BuildRequires out of the ceph-test subpackage and into the
+    distro-conditional dependencies section.
+    
+    Make ordering of BuildRequires a little more alphabetical.
+    
+    http://tracker.ceph.com/issues/12360 Fixes: #12360
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e267128ce22d8b8cd83f6d6d82f24f496600e678)
+    
+    Conflicts:
+    	ceph.spec.in
+                Dropped Requires: xmlstarlet in ceph-test that had inadvertently
+                been grabbed from d2cc2b1
+
+commit 19c9546b79f506d0b4ee005a138a77f120c629e8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Sep 8 10:41:02 2015 -0600
+
+    packaging: move rbd-replay* to ceph-common
+    
+    The rbd-replay* utilities are useful for Ceph users with RBD clients.
+    Currently the rbd-replay* utilities ship in the "ceph-test" package, and
+    we intend this ceph-test package for Ceph developers and contributors,
+    not normal users.
+    
+    Move the rbd-replay* utilities to "ceph-common".
+    
+    http://tracker.ceph.com/issues/12994 Fixes: #12994
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 96486fd6f284ca3ab67f4f94631896d41c529e85)
+    
+    Conflicts:
+    	ceph.spec.in
+                50a33dea has not been backported to hammer
+    	debian/ceph-test.install
+                50a33dea has not been backported to hammer
+    	debian/control
+                different ceph-test Replaces: and Breaks: version
+
+commit a79acd41187e6b049432bdc314f192e3fbb560a3
+Merge: 3d61493 31b7864
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 11:59:36 2015 +0100
+
+    Merge pull request #6213 from SUSE/wip-13425-hammer
+    
+    wrong conditional for boolean function KeyServer::get_auth()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3d61493337d481dec5139ccb6172710a118b2009
+Merge: 147f437 7ffd072
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:59:55 2015 +0100
+
+    Merge pull request #6336 from Abhishekvrshny/wip-13535-hammer
+    
+    LibRadosWatchNotify.WatchNotify2Timeout
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 147f437279e72c0766a126653e3db53abb73a4ac
+Merge: 1970e61 9085c82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:59:42 2015 +0100
+
+    Merge pull request #6391 from SUSE/wip-13590-hammer
+    
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1970e618209a242fc0bd3c707dfdfb1a886d6a2c
+Merge: fb83704 ade0f1a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:58:55 2015 +0100
+
+    Merge pull request #6398 from SUSE/wip-13588-hammer
+    
+    OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3d3595f86083a0f9847585eae4cffb8a82c816d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:13:27 2015 -0400
+
+    krbd: remove deprecated --quiet param from udevadm
+    
+    This parameter has been removed since systemd 213, so this
+    effects Fedora 21+, Debian Jessie, and potentially future
+    releases of RHEL 7.
+    
+    Fixes: #13560
+    Backport: hammer, infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4300f2a9fe29627eea580564ff2d576de3647467)
+
+commit 4d81cd19087e049bf3c2fe0d10de9215852ab51d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:12:34 2015 -0400
+
+    run_cmd: close parent process console file descriptors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f46f7dc94139c0bafe10361622416d7dc343d31f)
+
+commit fb83704d5db81aad95c61a61be52d07ce2d4d2b5
+Merge: 3954471 0742177
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 20:24:35 2015 +0100
+
+    Merge pull request #6199 from liewegas/wip-randomize-scrub-hammer
+    
+    osd: randomize scrub times
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 39544718dc2f09bcfdc632ac72fd2a3cda87687e
+Merge: 4ad9716 6a4734a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 20:23:23 2015 +0100
+
+    Merge pull request #5825 from tchaikov/wip-12848-hammer
+    
+    ReplicatedPG::hit_set_trim osd/ReplicatedPG.cc: 11006: FAILED assert(obc)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2052187929e059a25f6a3baf67329f7ce0bf6d8a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 23 15:31:27 2015 +0200
+
+    init-rbdmap: Rewrite to use logger + clean-up
+    
+    This patch rewrites the init-rbdmap init script so that it uses logger
+    instead of the log_* functions. The patch also fixes various smaller
+    bugs like:
+    * MAP_RV was undefined if mapping already existed
+    * UMNT_RV and UMAP_RV were almost always empty (if they succeeded) ->
+      removed them
+    * use of continue instead RET_OP in various places (RET_OP was not being
+      checked after the switch to logger messages)
+    * removed use of DESC (used only twice and only one occurrence actually
+      made sense)
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c567341e98fffbe39177f951511a7698f88abf5f)
+
+commit f972f38445bc7a6efe122a5f9fc2ba7658625e26
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 23 16:39:16 2015 +0200
+
+    ceph.spec.in: We no longer need redhat-lsb-core
+    
+    Drop the redhat-lsb-core dependency as it is no longer necessary on
+    fedora/rhel.
+    
+    The other two init scripts do not use redhat-lsb-core either. The
+    init-ceph.in conditionally requires /lib/lsb/init-functions and does not
+    use any of the functions defined in that file (at least not directly).
+    The init-radosgw file includes /etc/rc.d/init.d/functions on non-debian
+    platforms instead of /lib/lsb/init-functions file so it does not require
+    redhat-lsb-core either.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 15600572265bed397fbd80bdd2b7d83a0e9bd918)
+    
+    Conflicts:
+    	ceph.spec.in
+    Merge conflict because there were other commits that introduced systemd
+    & SUSE specific bits in the master which havent yet reached
+    hammer. Since this commit only removed redhat-lsb-core from *el distros
+    removing the specific lines
+
+commit 609f256a00f6ee18564ee055dd79f8293217940c
+Author: Jeff Weber <jweber@cofront.net>
+Date:   Wed Aug 5 19:06:46 2015 -0400
+
+    rgw: implement s3 encoding-type for get bucket
+    
+    This change introduces handling for the encoding-type request
+    parameter on the get bucket operation. An object key may contain
+    characters which are not supported in XML. Passing the value "url" for
+    the encoding-type parameter will cause the key to be urlencoded in the
+    response.
+    
+    Fixes: #12735
+    Signed-off-by: Jeff Weber <jweber@cofront.net>
+    (cherry picked from commit 180ca7b56ba91a3995c76dac698aa4ee31d9a2ce)
+
+commit 394fbfcc40d77ad6c8b3301c5ff2630c052107cd
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Fri Sep 25 10:18:11 2015 +0800
+
+    Objecter: maybe access wild pointer(op) in _op_submit_with_budget.
+    
+    look at "after giving up session lock it can be freed at any time by response handler" in _op_submit,
+    so the _op_submit_with_budget::op maybe is wild after call _op_submit.
+    
+    Fixes: #13208
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit 0635b1358354b19ae44105576f730381f3b5b963)
+
+commit 84068f8cea0452333f00f8b65230c00caa2e52ff
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Fri Sep 25 12:42:28 2015 +0800
+
+    Objecter: repeated free op->ontimeout.
+    
+    repeated free op->ontimeout in SafeTimer::timer_thread::callback->complete
+    
+    Fixes: #13208
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit f1d8a8f577cee6d66f4dcffac667675f18145ebb)
+
+commit 3e657304dc41facd40b8cab7531180083d14d22a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Aug 28 12:46:57 2015 -0700
+
+    ReplicatedBackend::prepare_pull: set recover_info.size for clones
+    
+    Fixes: #12828
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1d3e46bedb9a8c0d2200e39f62f4e2c3337619f3)
+
+commit 4ad97162026e1eb6e6e948ddf3eb39f711431e45
+Merge: 92b2153 f1271ea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:34:01 2015 +0100
+
+    Merge pull request #6157 from dachary/wip-13040-hammer
+    
+    common/Thread:pthread_attr_destroy(thread_attr) when done with it
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 92b21532f64792da023188d9aa620d900c0245ca
+Merge: 1f02886 a17f2a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:41 2015 +0100
+
+    Merge pull request #6155 from dachary/wip-13171-hammer
+    
+    objecter: cancellation bugs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1f02886d7edc10ff93074afc3d2bf4cf77c709d5
+Merge: 77df385 7d4b303
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:19 2015 +0100
+
+    Merge pull request #6152 from dachary/wip-13233-hammer
+    
+    mon: include min_last_epoch_clean as part of PGMap::print_summary and PGMap::dump
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77df385e23452f401f3c0c269e220446d288fb8e
+Merge: 469398f d8ca88d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:01 2015 +0100
+
+    Merge pull request #6146 from dachary/wip-13337-hammer
+    
+    segfault in agent_work
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 469398f14b62af01068e65bcc5cc7c585a04e878
+Merge: d87d136 397042a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:32:53 2015 +0100
+
+    Merge pull request #6145 from dachary/wip-13338-hammer
+    
+    filestore: fix peek_queue for OpSequencer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d87d1364fb8e345a1f85b65c2d4c7c80596ba5f9
+Merge: 717eff9 d4e4d85
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:32:42 2015 +0100
+
+    Merge pull request #6144 from dachary/wip-13339-hammer
+    
+    mon: check for store writeablility before participating in election
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 717eff9a61818fde71ec5220998e9b5c14bbb3a9
+Merge: e133ddc e8cce08
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:30:54 2015 +0100
+
+    Merge pull request #5891 from Abhishekvrshny/wip-13035-hammer
+    
+    requeue_scrub when kick_object_context_blocked
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e133ddc622d3532b0182c208f0ebb9956daab44f
+Merge: c72306a 8ee93d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:30:38 2015 +0100
+
+    Merge pull request #5890 from Abhishekvrshny/wip-13037-hammer
+    
+    hit set clear repops fired in same epoch as map change -- segfault since they fall into the new interval even though the repops are cleared
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c72306a1dd3b193948d5df3371471797d4131cfe
+Merge: 5e8c9d9 2bd5d0b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:21:07 2015 +0100
+
+    Merge pull request #6322 from sponce/WIP-13210-hammer
+    
+    tests : BACKPORT #13210 Fixed broken Makefiles after integration of ttng into rados
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b62cac6c345df53bbda1c42a5336cffc8f0f9652
+Author: Sangdi Xu <xu.sangdi@h3c.com>
+Date:   Wed Sep 23 19:31:32 2015 +0800
+
+    rgw: fix swift API returning incorrect account metadata
+    
+    Fixes: #13140
+    
+    Fix the bug that swift account stat command returns doubled object count and bytes used
+    
+    Signed-off-by: Sangdi Xu <xu.sangdi@h3c.com>
+    (cherry picked from commit 66d19c78ba75b79190c81c95f94e7ef3084fda9e)
+
+commit 9643ee65a6e968e479ca33a102f5f575924a7ff0
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Sun Aug 2 18:36:40 2015 +0800
+
+    bug fix: osd: do not cache unused buffer in attrs
+    
+    attrs only reference the origin bufferlist (decode from MOSDPGPush or
+    ECSubReadReply message) whose size is much greater than attrs in recovery.
+    If obc cache it (get_obc maybe cache the attr), this causes the whole origin
+    bufferlist would not be free until obc is evicted from obc cache. So rebuild
+    the bufferlist before cache it.
+    
+    Fixes: #12565
+    Signed-off-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit c5895d3fad9da0ab7f05f134c49e22795d5c61f3)
+
+commit 5e8c9d967e0ade393e36893965ca8ddfaa317b48
+Merge: db0366d dba8b5b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 9 08:38:31 2015 +0100
+
+    Merge pull request #5810 from Abhishekvrshny/wip-12948-hammer
+    
+    Heavy memory shuffling in rados bench
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1b06f03b541983461469f8f53ef2a271b08d2f78
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Thu Oct 29 21:46:02 2015 -0400
+
+    rgw:swift use Civetweb ssl can not get right url
+    
+    Fixes: #13628
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit e0fd540bf441e2f8276cbd96c601a0539892efe2)
+
+commit db0366dd979cc0d401b1974c233e38dfe5b1b5d1
+Merge: 22dce75 a65c398
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 5 13:12:11 2015 +0100
+
+    Merge pull request #5530 from SUSE/wip-12587-hammer
+    
+    FileStore calls syncfs(2) even it is not supported
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 22dce75181b254548226fc09decafac14dcf84a6
+Merge: ed153c1 d8ac510
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 19:38:53 2015 +0100
+
+    Merge pull request #5889 from Abhishekvrshny/wip-13042-hammer
+    
+    ThreadPool add/remove work queue methods not thread safe
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ed153c13226138eec71e472d1e3c8481db9b9009
+Merge: ca8802f 8610de8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 19:14:28 2015 +0100
+
+    Merge pull request #6151 from dachary/wip-13245-hammer
+    
+    client nonce collision due to unshared pid namespaces
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ca8802fb0cb8603267ecfc7320fee7a679617b8b
+Merge: 19450b9 6e29e90
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:36:39 2015 +0100
+
+    Merge pull request #6159 from dachary/wip-13341-hammer
+    
+    ceph upstart script rbdmap.conf incorrectly processes parameters
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 19450b90e4c20556580d276bab7dafb23db1758f
+Merge: 18882c8 6849288
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:34:07 2015 +0100
+
+    Merge pull request #6154 from dachary/wip-13195-hammer
+    
+    should recalc the min_last_epoch_clean when decode PGMap
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 18882c81a71c4b517cb0eaa5a884637ea5cbe7a6
+Merge: de4f37b 3c1f7cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:28:03 2015 +0100
+
+    Merge pull request #6132 from SUSE/wip-13307-hammer
+    
+    dumpling incrementals do not work properly on hammer and newer
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de4f37bb0abc932c813983792e170de1bb40241b
+Merge: 1c632a7 24268cf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:23:14 2015 +0100
+
+    Merge pull request #6097 from dillaman/wip-13045-hammer
+    
+    librbd: diff_iterate needs to handle holes in parent images
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1c632a7ea7057522fb9df61994b5b3cf65416689
+Merge: bedb334 71a42f7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:22:25 2015 +0100
+
+    Merge pull request #6060 from XinzeChi/wip-hammer-outdata-set
+    
+    bug fix: osd: avoid multi set osd_op.outdata in tier pool
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bedb3344f6c802ea04a171dabd4fb596284af34a
+Merge: 0d3d819 cecd68d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:20:50 2015 +0100
+
+    Merge pull request #5897 from tchaikov/wip-12940-hammer
+    
+    IO error on kvm/rbd with an erasure coded pool tier
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d3d81951db61a97f6cb26d9f75e80a3400a2a37
+Merge: 381cfcc 7475a8f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:11:00 2015 +0100
+
+    Merge pull request #5787 from SUSE/wip-12923-hammer
+    
+    logrotate reload error on Ubuntu 14.04
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 381cfccf38bade28a001f3f24392c61896b3c711
+Merge: cad1cfb 86f88df
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:10:26 2015 +0100
+
+    Merge pull request #5766 from dachary/wip-12590-hammer
+    
+    ceph mds add_data_pool check for EC pool is wrong
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit cad1cfb088b095e8333c4a7d79ccefc2b2c4d782
+Merge: 9764da5 e8d6d5a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:06:18 2015 +0100
+
+    Merge pull request #5361 from ceph/wip-11786.hammer
+    
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5105d50d1c20d465229189bbb24cdfb5d16cf7bc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Nov 3 00:21:51 2015 +0100
+
+    tests: test/librados/test.cc must create profile
+    
+    Now that the create_one_ec_pool function removes the testprofile each
+    time it is called, it must create the testprofile erasure code profile
+    again for the test to use.
+    
+    http://tracker.ceph.com/issues/13664 Refs: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a60342942b5a42ee04d59af77a6b904ce62eefc4)
+
+commit 302375068a59b86c50bda73ecefd99831ab52ea4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:24:51 2015 +0100
+
+    tests: destroy testprofile before creating one
+    
+    The testprofile erasure code profile is destroyed before creating a new
+    one so that it does not fail when another testprofile erasure code
+    profile already exists with different parameters.
+    
+    This must be done when creating erasure coded pools with the C++
+    interface, in the same way it's done with the C interface.
+    
+    http://tracker.ceph.com/issues/13664 Fixes: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 47abab9a6f182aa0abe5047c04402850379bcd6d)
+
+commit 3a7423dbe0a6af831d4c47acc98e6ced5ed5fe0c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:23:52 2015 +0100
+
+    tests: add destroy_ec_profile{,_pp} helpers
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ab46d79bc09fc711fa35302f49eecac81a98519b)
+
+commit a52f7cb372339dffbeed7dae8ce2680586760754
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 13 09:55:01 2015 -0400
+
+    crush/mapper: ensure take bucket value is valid
+    
+    Ensure that the take argument is a valid bucket ID before indexing the
+    buckets array.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 93ec538e8a667699876b72459b8ad78966d89c61)
+
+commit 81d8aa14f3f2b7bf4bdd0b4e53e3a653a600ef38
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 27 20:55:26 2015 -0400
+
+    crush/mapper: ensure bucket id is valid before indexing buckets array
+    
+    We were indexing the buckets array without verifying the index was within
+    the [0,max_buckets) range.  This could happen because a multistep rule
+    does not have enough buckets and has CRUSH_ITEM_NONE
+    for an intermediate result, which would feed in CRUSH_ITEM_NONE and
+    make us crash.
+    
+    Fixes: #13477
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 976a24a326da8931e689ee22fce35feab5b67b76)
+
+commit 74203b806e8c268aa3edac8cfc7c4b393f7b1f27
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Oct 26 18:38:01 2015 +0800
+
+    FileStore: potential memory leak if _fgetattrs fails
+    
+    Memory leak happens if _fgetattrs encounters some error and simply returns.
+    Fixes: #13597
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit ace7dd096b58a88e25ce16f011aed09269f2a2b4)
+
+commit db1cbe700e41ee7a2db5ef10173c429283bc5a2a
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri Sep 4 01:07:48 2015 +0000
+
+    Fix casing of Content-Type header
+    
+    It turns out, despite the HTTP spec declaring that header field names
+    should be case-insensitive, some clients treat them wrongly, and
+    consider "Content-type" to not match "Content-Type".
+    
+    CyberDuck was one of those clients, now fixed upstream in
+    https://trac.cyberduck.io/ticket/8999
+    
+    To reduce future occurances of this bug, fix the casing of the
+    Content-Type header, to strictly comply with the HTTP specification (be
+    strict about what you send, and generous about what you receive).
+    
+    Fixes: #12939
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 1b9fbffdc24160251b96cec820d62fb2a12b6eab)
+
+commit 1448915e49bb6c8abc192053e6f2e8a380c4e92a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 1 13:55:22 2015 +0100
+
+    qa: update cephtool test for CephFS tier cases
+    
+    1. Creating a filesystem using a
+       readonly tier on an EC pool (should be forbidden)
+    2. Removing a tier from a replicated base pool (should
+       be permitted)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f9b11b0abe7c2b0dcdfc5802336be8a0426ccb3a)
+
+commit 14c36f0ccf103dea39b63e0e1b03082b99c42cb1
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 1 13:54:25 2015 +0100
+
+    mon: forbid readonly cache tiers for CephFS
+    
+    For EC pools, we need a tier that is not just forwarding
+    writes, in order to provide the ops required
+    by CephFS.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 11e5faf3c1d4fa28a8b41e3a92f0f684a844e5f5)
+
+commit 31e59fbca0ba636fc57c208e612d7488a48a6287
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 14:23:37 2015 +0100
+
+    mon: refine check_remove_tier checks
+    
+    Fixes: #11504
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a50c8f1f2ad8845c7f77110868f9376f2d1ff883)
+
+commit 9f3aebee16e256888b149fa770df845787b06b6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 13:57:37 2015 -0400
+
+    osd: fix requeue of replay requests during activating
+    
+    If the replay period expires while we are still in the activating
+    state, we can simply insert our list of requests at the front of
+    the waiting_for_active list.
+    
+    Fixes: #13116
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d18cf51d9419819cdda3782b188b010969288911)
+
+commit ade0f1a4285348c20fbae8e2a366fc332a7b7fbe
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Oct 13 14:04:20 2015 +0800
+
+    OSD:shall reset primary and up_primary fields when beginning a new past_interval.
+    
+    Shall reset primary and up_primary fields when we start over a new past_interval in OSD::build_past_intervals_parallel().
+    Fixes: #13471
+    Signed-off-by: xie.xingguo@zte.com.cn
+    (cherry picked from commit 65064ca05bc7f8b6ef424806d1fd14b87add62a4)
+
+commit 9085c820491f25a04ec02accc5098c1ab9b57311
+Author: yangruifeng <yangruifeng.09209@h3c.com>
+Date:   Mon Oct 19 08:08:12 2015 -0400
+
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Fixes: #13525
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit c7f75b8f7c0a773148ec16141941efd00ee76626)
+
+commit 4875d05a3e265e80463a2a33a2c922affb0dad66
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Mar 16 10:02:35 2015 -0600
+
+    ceph.spec.in: rm EOL Fedoras; add OBS RHEL5 instead
+    
+    Fedora 12 has been EOL for a long time. Remove the reference in the
+    RPM .spec file.
+    
+    Since RHEL 5 support for Ceph is a work in progress, we won't remove
+    this entire python_sitelib / python_sitearch conditional for now, since
+    those are still needed on RHEL 5.
+    
+    Add the rhel_version macro to make the conditional compatible with
+    SUSE's OBS.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 353a3258968f76deaea4542d2165e9ab32b88de8)
+
+commit dfba6acce75eea02251e98e2d57e6fbb344241fa
+Author: Travis Rhoden <trhoden@redhat.com>
+Date:   Thu May 7 20:13:53 2015 -0400
+
+    packaging: Add rgw placeholder dirs in proper packages
+    
+    Automatically create /var/lib/ceph/radosgw with the
+    ceph-radosgw package, and /var/lib/ceph/bootstrap-rgw
+    via the ceph package.
+    
+    Signed-off-by: Travis Rhoden <trhoden@redhat.com>
+    (cherry picked from commit bf93128f664ac74383806530a1f974edde8df5b6)
+    
+    Conflicts:
+        ceph.spec.in
+            hammer uses %ghost
+
+commit 2387c8e0caef00e7181241e16549457d58a12192
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Fri May 15 21:39:39 2015 +0200
+
+    ceph.spec.in: consolidate centos/rhel macros
+    
+    0%{?rhel} || 0%{?centos} can be simplified to just 0%{?rhel}, since
+    CentOS defines that macro for compatibility with RHEL.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 46404dd381e240634d224e77adc69f0ba4807045)
+    
+    Conflicts:
+    	ceph.spec.in
+                https://github.com/ceph/ceph/pull/4068 has not been backported
+
+commit 7c58d312db2aa42088f419dbfa7e841d897fdad6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 16:55:10 2015 +0200
+
+    packaging: make logrotate a hard dependency across all distros
+    
+    Given ceph's reputation for voluminous logs, it makes sense for
+    logrotate to be a hard dependency for all distros.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0734cd1ec3ca3bd012686dc1bdda675909224a49)
+
+commit 61acc4e20ca58e11b9aba1efcdca78d1d7ddd2f7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 12:05:24 2015 +0200
+
+    ceph.spec.in: add missing -%{release}
+    
+    We have it everywhere else and I can't think of any reason why
+    it should be omitted here.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d952d59ae95d5f66aa4b98fd7aa4f4ecb36a4848)
+
+commit 20590d367226ea01c9d4d95de15cfd1aa354d02b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 26 20:21:59 2015 +0200
+
+    ceph.spec.in: remove duplicate BuildRequires: sharutils
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c418bd9615e6f23f3c14ae9e71003c66d31d4b78)
+
+commit 6b306eba829b1ec9822b24243ee7c021aab3d738
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 15 12:58:00 2015 +0200
+
+    ceph.spec.in: fix python-flask dependency for SUSE
+    
+    In SLE and openSUSE, the package is called python-Flask with an upper-case F.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 39c28b9faf0fb069d91ac069a4d8df7bf03fb490)
+    
+    Conflicts:
+    	ceph.spec.in
+                add missing SUSE-specific build deps
+
+commit 7a2cbcb0a4efa365b6d92ec58d7acf4a66e9d2b1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 18:55:39 2015 +0200
+
+    ceph.spec.in: clarify two important comments
+    
+    First, the terms "common" and "specific" are vague. Second,
+    "common" can easily be confused with the ceph-common subpackage.
+    
+    Fix this by rephrasing to "distro-unconditional dependencies" and
+    "distro-conditional dependencies", respectively.
+    
+    Third, move the "distro-unconditional dependencies" header so it
+    is above the part where these dependencies are actually defined.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit dd212fd1d367f70c61f65a97599959668b44aa53)
+
+commit dfe27c6ba05d49bb73854754b9d5c02cf1c7b1ef
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 24 16:37:58 2015 -0600
+
+    ceph.spec.in: rm reference to EOL Fedoras (< 20)
+    
+    Fedora 19 is EOL, so there's no need to refer to Fedoras less than 20.
+    "%{fedora}" effectively implies "Fedora 20 and above".
+    
+    Reported-by: Ira Cooper <ira@samba.org>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f3e4a91c458ca7b11706fe4608cb739499f5d3cc)
+    
+    Conflicts:
+    	ceph.spec.in
+               added xmlstarlet dependency to ceph-test package
+
+commit 371b64ef4aa786db3fd29bc84ff98e63000a859b
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 24 16:39:30 2015 -0600
+
+    ceph.spec.in: package rbd-replay-prep on all Fedoras
+    
+    This reverts the change in commit
+    85517d611b7bf4cb6cbffcd2c65303be0d038264. Since we BuildRequire:
+    libbabeltrace-devel, autoconf will see that babeltrace is available
+    during the build, and make will build/install the rbd-replay-prep
+    utility.
+    
+    This change also simplifies Fedora selection logic, because Fedora 19 is
+    EOL, so "%{fedora}" implies "Fedora 20 and above".
+    
+    Reported-by: Ira Cooper <ira@samba.org>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 2db9480fa2646de776f0076d82747e202e574ddd)
+
+commit 82425b33f712047442d9d29c5a4349818b44b40a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jun 16 23:07:04 2015 +0200
+
+    ceph.spec.in: rbd-replay-prep is not being built on f21+
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 85517d611b7bf4cb6cbffcd2c65303be0d038264)
+
+commit 53e2f702681dc08ddd0414b116e51de6488f42ca
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 8 17:48:55 2015 +0200
+
+    ceph.spec.in:BuildRequires sharutils
+    
+    The uudecode binary is used to build Java-related components, and
+    uudecode is provided by the sharutils package on all supported
+    RPM platforms. When building with "--without=cephfs_java",
+    sharutils is not needed.
+    
+    Thanks to Nathan Cutler <ncutler@suse.cz> for going into the
+    details with me.
+    
+    On OBS without this patch we get the error message:
+    
+    [  170s] -----------------------------------------------------------------
+    [  170s] ----- building ceph.spec (user abuild)
+    [  170s] -----------------------------------------------------------------
+    [  170s] -----------------------------------------------------------------
+    [  170s] + exec rpmbuild -ba --define '_srcdefattr (-,root,root)' --nosignature --define 'disturl obs://build.opensuse.org/home:osynge:ceph:wip:wip_obs_fedora/Fedora_20/c0bbbc1e62228ca956ac3d367edc4fba-master' /home/abuild/rpmbuild/SOURCES/ceph.spec
+    [  170s] error: Failed build dependencies:
+    [  170s]    sharutils is needed by ceph-1:2+git.1435043747.c1bd02c-1.1.x86_64
+    
+    With this patch we can build fedora 22 and fedora 20 rpms fine.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 43c1784640538b813f0575c50f721ac911a30b3e)
+    
+    Conflicts:
+    	ceph.spec.in
+                _with_systemd -> trivial resolution
+
+commit a84376c68499845de01f9e3b194b16a698125b6a
+Author: Owen Synge <osynge@suse.com>
+Date:   Wed Jun 10 12:11:50 2015 +0200
+
+    ceph.spec.in: fix:Add missing directories breaking build
+    
+    SUSE builds on OBS are failing with the missing dir entries:
+    
+        /usr/share/ceph
+        /usr/lib/ceph
+    
+    On suse these correspond to:
+    
+        %dir %{_datarootdir}/ceph/
+        %dir %{_libexecdir}/ceph/
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit f1ff3e4dcb5dd7eddd2c346a05cfa78d860a5ce6)
+
+commit 171fee1b82d2675e364da7f96dfb9dd286d9b6e6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 21 00:33:19 2015 +0800
+
+    mon: track osd_epoch of all MonSessions
+    
+    previously, we only track the osd_epoch for OSD peers. but other
+    MonClients who receives osdmaps can also benefit from osd_epoch.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit de43a02e06650a552f048dc8acd17f255126fed9)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc:
+    		do not assume that the MonSession is always available when
+            OSDMonitor::send_incremental() is called. as when the
+            connection to the mon client is dropped, the Monitor is
+            aware of this, but not the OSDMonitor.
+
+commit cc7da674965c3f758469e18788dc0e106e54d5bc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 21 00:24:52 2015 +0800
+
+    mon: track osd_epoch in MonSession
+    
+    * remove osd_epoch<osd, epoch> from OSDMonitor
+    * add osd_epoch to MonSession to track the latest osdmap epoch
+      OSDMonitor sends to a mon client
+    * do not remove osd_epoch entries if an OSD is down, or
+      max_osd > osd_id
+    
+    Fixes: #10930
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c05753eacc26e90b2e3b56e641a71bffd5b39bd0)
+
+commit 3f33ce6132d5ce8b3454f30fbe14471ece39951d
+Author: Lu Shi <shi.lu@h3c.com>
+Date:   Thu Sep 24 16:02:41 2015 +0800
+
+    librbd:reads larger than cache size hang.
+    
+    Fixes:#13164
+    
+    Signed-off-by: Lu Shi <shi.lu@h3c.com>
+    (cherry picked from commit 9c8200bb5d1ac9359803a182df03298b565b8479)
+
+commit 5e5b512189b809bef503c34b0a19e30ed81cfbfa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Aug 19 20:32:39 2015 +0200
+
+    rgw: url_decode values from X-Object-Manifest during GET on Swift DLO.
+    
+    Fixes: #12728
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 69cf089596a0e5d4da2826b5c276306316a00690)
+
+commit e693755a55e4da681a1947b1e1eb3bb537436cc0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 22 14:53:42 2015 -0700
+
+    cls_rgw: fix bucket listing when dealing with invisible entries
+    
+    Fixes: #12913
+    
+    The bucket listing logic was broken when dealing with invisible entries.
+    Need to read keys until we can fetch all the requested entries, or
+    determine that there are no more entries. Anything else requires
+    a change to the protocol.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef9730515e001d77637a77eb89950f70fd5e7f5a)
+
+commit 99b4d1df3f6867e5ff0dd74691203737f6de6974
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 22 15:49:27 2015 -0700
+
+    rgw: orphan tool shouldn't clean up head objects
+    
+    Fixes: #12958
+    
+    Head objects are mutable, so removing them can race with object removal
+    and a later recreation, so we might end up cleaning them up when we don't
+    need to.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 030f697d196df9963ecf4f05f727da4798c583dc)
+
+commit 6d89f4ba85063debe96734e9fdfcba247e770206
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 11:55:40 2015 -0700
+
+    rgw: location constraints should return api name
+    
+    Fixes: #13458
+    The S3 location constraints request should return the api name,
+    not the region name.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 03078bab6ee8c617025902dbe52ec19e64653c33)
+
+commit 2bd5d0b835b546c80b5af503d2b9538835c768fd
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue Oct 20 09:22:16 2015 +0200
+
+    radosstriper : Fixed broken Makefiles after integration of lttng
+    
+    Fixes: #13210, backport of #6322 into hammer
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+
+commit 7ffd072a8cc7e114d4d674d30333fea4db04dcdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 1 14:50:34 2015 -0400
+
+    osdc/Objecter: distinguish between multiple notify completions
+    
+    We may send a notify to the cluster multiple times due to OSDMap
+    changes.  In some cases, earlier notify attempts may complete with
+    an error, while later attempts succeed.  We need to only pay
+    attention to the most-recently send notify's completion.
+    
+    Do this by making note of the notify_id in the initial ACK (only
+    present when talking to newer OSDs).  When we get a notify
+    completion, match it against our expected notify_id (if we have
+    one) or else discard it.
+    
+    This is important because in some cases an early notify completion
+    may be an error while a later one succeeds.
+    
+    Note that if we are talking to an old cluster we will simply not record a
+    notify_id and our behavior will be the same as before (we will trust any
+    notify completion we get).
+    
+    Fixes: #13114
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e86d033854c76f344c678e92016c4e5c5e0385e2)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+    		In Objecter::handle_watch_notify, a conflict was there due to a modified comment by commit 47277c51db7bb2725ea117e4e8834869ae93e006, which was not backported
+
+commit 647c91098e8517912c73e3f7d27954e6f340472c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 1 14:50:00 2015 -0400
+
+    osd: reply to notify request with our unique notify_id
+    
+    The OSD assigns a unique ID to each notify it queues for
+    processing.  Include this in the reply to the notifier so that
+    they can match it up with the eventual completions they receive.
+    
+    This is necessary to distinguish between multiple completions
+    they may receive if there is PG peering and the notify is resent.
+    In particular, an earlier notify may return an error when a later
+    attempt succeeds.
+    
+    This is forwards and backwards compatible: new clients will make use of
+    this reply payload but older clients ignore it.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 049ea702b9bf4a8a62ae2770d9ba55c0730b3eef)
+
+commit 0ad9521944e0de5c135a066b0cda07f8976bdae1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 12 13:24:31 2015 -0700
+
+    osd: Correct the object_info_t::decode() version
+    
+    Caused by: 13b9dc70
+    
+    Fixes: #13462
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2b7ddde685213f3908f2dd9545b35f1d97f35a79)
+
+commit af734e63f28ac5af414b6507c945ffd9e736d5cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 12:31:59 2015 -0400
+
+    librbd: invalidate object map on error even w/o holding lock
+    
+    If there is a deep bug that prevents the object map from owning
+    the lock before objects are updated, flag the object map as invalid
+    regardless.
+    
+    Fixes: #13372
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e675400df7f8ba796d60e9bac4234857f0cb1392)
+    
+     Conflicts:
+    	src/librbd/ObjectMap.h/cc: hammer branch doesn't support rebuild
+
+commit 634d7f62486847846e3862091b8cb587d095cb52
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:28:04 2015 -0400
+
+    tests: removed obsolete rbd_replay test cases
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e0d8cb1283e4bdbfc770684b173e477c22558524)
+
+commit 5c63d3c614189381cc169c64a3138c7abcb368c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:26:42 2015 -0400
+
+    ceph-dencoder: new rbd_replay trace file types
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c2a83d0b518497dc8bcbbdea77b88febe52a3034)
+
+commit a145a1cd8e245505abae66f0497be0df0078cecb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:21:29 2015 -0400
+
+    rbd-replay: added version control to trace output file
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ecdae8388d69123b937a40ce614a0b795a757f1)
+
+commit a30ed76c34ecca6278112908871126e1730a22bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 15:39:18 2015 -0400
+
+    rbd-replay-prep: added --verbose command line option
+    
+    This new command-line will dump all the processed events as they
+    are extracted from the trace file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 646e50a771c3089121ec2f4369d1a8188001a3eb)
+
+commit adf31c88d3ed95d0c29c649e2642ee3ae758743b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 15:22:25 2015 -0400
+
+    rbd-replay-prep: stream events to the prep file
+    
+    Avoid building the entire prep event history in memory before
+    attempting to write the prepped trace file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98f513a113f4e8aff17c83991d2e6f0f3738a9c9)
+
+commit 8c9e67ad0b21629565a09f6b9b92723b98594f84
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 14:56:22 2015 -0400
+
+    rbd-replay-prep: simplify IO dependency calculation
+    
+    Only track read-after-write and write-after-write IO dependencies
+    via the associated write completions.  All IO events after a write
+    completion are considered to be dependent and can be pruned down
+    to at most the number of concurrent IOs.  This reduces the prep
+    time from a simple 'rbd bench-write' from over 4 hrs down to seconds.
+    
+    Fixes: #13378, #13384
+    Backport: hammer
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 65fb1b86cbab65023f6207798c9e189bce55dcf6)
+
+commit e7406fcba86646a4c3fbac66ee54b52ce3d5a33d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 13:16:51 2015 -0400
+
+    rbd-replay: improve error messages
+    
+    Fixes: #13221
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4f436e10802d7b3c3949b592fec238c189a02c50)
+
+commit 9f81d28107bd82b7118d1521ade982ab179c5a7a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 12:48:50 2015 -0400
+
+    rbd-replay-prep: support new read/write APIs
+    
+    Added support for read2, write2, aio_read2, and aio_write2 APIs.
+    
+    Fixes: 13325
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b5b4a9dd6bfe10844327a457849a0dbf7ef597ea)
+
+commit a856bee2aa7e22f2bedf54b684c4d55452766b01
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 5 17:30:54 2015 -0400
+
+    rbd-replay-prep: avoid using assert statements on user inputs
+    
+    Fixes: #13220
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1e99f06a78d7cf2ecc0c8e9fbc3331e4e4f18f7)
+
+commit 6a4734a7afecb359b8bc9c114332cd7321097c38
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 11:16:49 2015 -0700
+
+    ReplicatedPG::hit_set_setup: fix hit_set_remove_all call
+    
+    We only want to do it if the pool config changed AND we are
+    primary && active.
+    
+    fd38902dd4693b9b72f7532833e78f5df2c9efa5 partially fixed a related
+    bug.  This should be backported along with it.
+    
+    Fixes: 13192
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 36e4a80c39f7daab2a35efc04650ae8b7b11c4e0)
+
+commit ab3877dd566f3951e76abaa6b7deeb5835a512b1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Sep 22 11:13:44 2015 -0700
+
+    osd/: assert in HitSet constructor if type is TYPE_NONE as well
+    
+    Otherwise impl is unpopulated and we risk segfaults in several
+    methods.  It also seems like it would always indicate a bug.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8e5a801b43babf751b619ed9606d7be7e5b12837)
+
+commit 4fe07b7f69321535a985958fc761a54cb71136f6
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Thu Jul 23 09:26:44 2015 +0800
+
+    osd: only remove all the hit set objects on primary osd
+    
+    Only the primary PG is allowed to remove all the hit set objects. And
+    the PG should be in the active or peered states.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit fd38902dd4693b9b72f7532833e78f5df2c9efa5)
+
+commit 313a09b58b6aa6fe8999f4d5da07a04f00c64cd7
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Thu Jun 18 09:05:28 2015 +0800
+
+    osd: implement hit_set_remove_all
+    
+    When hit set is not configured on startup or on a change, remove all
+    previous hitsets.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit be28319bf3dc54b4b6f400d83405a29facfe3bd4)
+    Conflicts:
+            src/osd/ReplicatedPG.cc
+              pass the new argument of `p->using_gmt` to get_hit_set_archive_object()
+
+commit b632254252f56d3b2c0c32803593ba765b45b222
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 8 16:52:32 2015 +0800
+
+    mon: disable gmt_hitset if not supported
+    
+    the gmt_hitset is enabled by default in the ctor of pg_pool_t, this
+    is intentional. because we want to remove this setting and make
+    gmt_hitset=true as a default in future. but this forces us to
+    disable it explicitly when preparing a new pool if any OSD does
+    not support gmt hitset.
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e48cec3dc93b3988dcd8924933deb1b3a43e1d0f)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		remove unrelated hunk
+
+commit 27fc9a3b2bd34bec8d5bb3954b873ce6344771f4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 00:32:38 2015 +0800
+
+    osd: do not let OSD_HITSET_GMT reuse the feature bit
+    
+    * to ease the backport to hammer
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5a4f6a866bb675195ad83c38fd23238d078c78ed)
+    
+    Conflicts:
+    	src/include/ceph_features.h
+    		minor changes to resolve conflicts
+
+commit 7fcc61d26028eb417a0c224e8c9cd3cc95bc13d9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Aug 27 11:24:25 2015 -0700
+
+    osd: Decode use_gmt_hitset with a unique version
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2bc5a48f4c5d3667213be3a7b5a0e0f5ef9daf4f)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    		minor changes to resolve conflicts
+
+commit 26c7e968523ac09f608c4ed7c50681fbb741ca1f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 16:53:50 2015 +0800
+
+    mon: print use_gmt_hitset in "ceph osd pool get"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc2bcf760f2d2f20fc4a2fa78ba52475a23e55bf)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		use the old way to dump pool info
+
+commit e8e00dab1b064e77a997504f385b5d60ee8aca25
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 00:04:23 2015 +0800
+
+    mon: add "ceph osd pool set $pool use_gmt_hitset true" cmd
+    
+    allow "ceph osd pool set $pool use_gmt_hitset <true|1>" as long as
+    the cluster supports gmt hitset.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 03a1a3cf023a9aeb2fa26820e49e5efe3f3b3789)
+
+commit 040e390d29fc68dcefe48c59cccacf6258c6f690
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 5 21:06:48 2015 +0800
+
+    osd: use GMT time for the object name of hitsets
+    
+    * bump the encoding version of pg_hit_set_info_t to 2, so we can
+      tell if the corresponding hit_set is named using localtime or
+      GMT
+    * bump the encoding version of pg_pool_t to 20, so we can know
+      if a pool is using GMT to name the hit_set archive or not. and
+      we can tell if current cluster allows OSDs not support GMT
+      mode or not.
+    * add an option named `osd_pool_use_gmt_hitset`. if enabled,
+      the cluster will try to use GMT mode when creating a new pool
+      if all the the up OSDs support GMT mode. if any of the
+      pools in the cluster is using GMT mode, then only OSDs
+      supporting GMT mode are allowed to join the cluster.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42f8c5daad16aa849a0b99871d50161673c0c370)
+    
+     Conflicts:
+    	src/include/ceph_features.h
+    	src/osd/ReplicatedPG.cc
+    	src/osd/osd_types.cc
+    	src/osd/osd_types.h
+    		fill pg_pool_t with default settings in master branch.
+
+commit 31b7864b4161221f6027c6cbe1e42ed598d42b6a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Sep 27 21:55:00 2015 +0200
+
+    CephxServiceHandler.cc: fix get_auth conditional
+    
+    The KeyServer class has a public method get_auth() that returns a boolean
+    value. This value is being checked here - fix the conditional so it triggers
+    when get_auth() returns false.
+    
+    http://tracker.ceph.com/issues/9756 References: #9756
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 7250db62cb7ac1e9c0b1f5956d60b9d874496702)
+
+commit 0742177c767613d41bda0d260eebdd0087e3b392 (refs/remotes/me/wip-randomize-scrub-hammer)
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat May 16 00:07:27 2015 +0800
+
+    osd: use another name for randomize scrub option
+    
+    s/osd_scrub_interval_limit/osd_scrub_interval_randomize_ratio/
+    
+    Fixes: #10973
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit fad33861a87f2feb944ad5454775c95d09364b56
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 9 16:42:34 2015 +0800
+
+    osd: randomize scrub times to avoid scrub wave
+    
+    - to avoid the scrub wave when the osd_scrub_max_interval reaches in a
+      high-load OSD, the scrub time is randomized.
+    - extract scrub_load_below_threshold() out of scrub_should_schedule()
+    - schedule an automatic scrub job at a time which is uniformly distributed
+      over [now+osd_scrub_min_interval,
+            now+osd_scrub_min_interval*(1+osd_scrub_time_limit]. before
+      this change this sort of scrubs will be performed once the hard interval
+      is end or system load is below the threshold, but with this change, the
+      jobs will be performed as long as the load is low or the interval of
+      the scheduled scrubs is longer than conf.osd_scrub_max_interval. all
+      automatic jobs should be performed in the configured time period, otherwise
+      they are postponed.
+    - the requested scrub job will be scheduled right away, before this change
+      it is queued with the timestamp of `now` and postponed after
+      osd_scrub_min_interval.
+    
+    Fixes: #10973
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e44040e8528bff06cc0a5a3f3293ab146e0e4e1)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 9c130336a1a1dcf2338d202914aeacbc194af0b0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 9 16:47:30 2015 +0800
+
+    osd: use __func__ in log messages
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0f7f35670f03f3f58329c00d323963710e7e495d)
+
+commit 4a3dedff8693953944af3eeb40994f38befebf48
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 15 12:04:05 2015 +0800
+
+    osd: simplify OSD::scrub_load_below_threshold() a little bit
+    
+    avoid unnecessary comparison
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2ab0e606dfd2847e1918be9f21f572d91befbf7b)
+
+commit 24268cf4309267a0a1e32e28379cf7eafac00bca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 1 14:21:35 2015 -0400
+
+    librbd: invoking RBD::open twice will leak memory
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a5e88fcf11ebb16d4a1405b21cf6f895b37f185e)
+
+commit 8610de81c8eeae763354a4133e160ccb3faa8dad
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:46:42 2015 -0700
+
+    use simplifed messenger constructor for clients
+    
+    This is all mechanical except the calculation of the nonce, which is
+    now always randomized for clients.
+    
+    Fixes: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit a3a8c85b79afef67681c32c57b591c0e0a87a349)
+
+commit 9d059c3f1b3aa0b1830b6f84694c850d36e8a77a
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:20:53 2015 -0700
+
+    msg: add simplified messenger constructor
+    
+    Several callers create messengers using exactly the same parameters:
+    
+    - reading the ms type from cct that is also passed in
+    - a default entity_name_t::CLIENT
+    - the default features
+    
+    Additionally, the nonce should be randomized and not depend on
+    e.g. pid, as it does in several callers now. Clients running in
+    containers can easily have pid collisions, leading to hangs, so
+    randomize the nonce in this simplified constructor rather than
+    duplicating that logic in every caller.
+    
+    Daemons have meaningful entity_name_ts, and monitors currently depend
+    on using 0 as a nonce, so make this simple constructer
+    client-specific.
+    
+    Related: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e3785b0b5fbff870adbd5f121021b671912386cf)
+    
+    Conflicts:
+       src/msg/Messenger.cc: Messenger::create only has 5 arguments in
+       hammer. Since the last one is zero, it does not change the
+       semantic.
+
+commit 6e29e904ce8e7b3481fd3b318b070d326cd4a71a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 30 08:29:05 2015 -0400
+
+    init-rbdmap: fix CMDPARAMS
+    
+    Fixes: #13214
+    Reported-by: Wyllys Ingersoll <wyllys.ingersoll@keepertech.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02113ac449cd7631f1c9a3840c94bbf253c052bd)
+
+commit 7161a2c927a6ded0d6ffe7d7621b7abd7f18acc3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 12:00:49 2015 -0700
+
+    PG: ignore info from down osd
+    
+    Fixes: #12990
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 930d8eb1fb0daacd692d2e80b5ff473448bd4e8d)
+
+commit 9753551f9f4ea9a1638dd469ef536d895ca9bc3b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 12:01:03 2015 -0700
+
+    OSDMap.h: add has_been_up_since
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4b00350870e3a43aee5fff7982b3785fe1876634)
+
+commit f1271eae0decf4d1fd9cfd83e5e54d3be6abfb17
+Author: zqkkqz <zheng.qiankun@h3c.com>
+Date:   Fri Aug 7 10:49:45 2015 +0800
+
+    Common/Thread: pthread_attr_destroy(thread_attr) when done with it
+    When a thread attributes object is no longer required, it should be destroyed using the
+    pthread_attr_destroy() function. Destroying a thread attributes object has no effect on threads that were created using that object.
+    
+    Fixes: #12570
+    Signed-off-by: zqkqkz zheng.qiankun@h3c.com
+    (cherry picked from commit 9471bb838a420ef5f439191d87e5388fecccb9e6)
+
+commit 9581f9f4446ed5838bb3de6e1d7cf5faca85df90
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Jul 17 12:43:52 2015 +0200
+
+    Thread.cc: remove malloc/free pair
+    
+    There's no need for mallocing pthread_attr_t in Thread::try_create(),
+    it can be located on stack as it is freed in same function. This reduces
+    pressure put on memory manager.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 614ac0f4e375f3dd008c8937d3f89f6005d6e0a0)
+
+commit a17f2a9a6600803a8b7f6dd306f7bafb6acc1af5
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 21:35:19 2015 +0800
+
+    osdc/Objecter: optimize Objecter::tick.
+    
+    Set bool value is better than insert same value into set<>.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit ee204044d9e69ea44b533c05cec154974039264c)
+
+commit f80d237decac775d82f1e276a1ea600a9f352ffe
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 19:32:32 2015 +0800
+
+    osdc/Objecter: remove the unuseful code.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 80f10e3e59dadda2dca4eb62c68af972b701b316)
+
+commit 8f76f4c467dc69cd2af75d9cbb134c26f2d04a84
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 15:39:29 2015 +0800
+
+    osdc/Objecter: Don't forget call _op_cancel_map_check when cancel linger op.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 7cc963b1c6ab37bf33638dc6eca7848d93f7908f)
+
+commit ba36caba5226a44c41fdd2b19074916b38af5fdf
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 15:12:02 2015 +0800
+
+    osdc/Objecter: In _cancel_linger_op, it should make num_unacked/num_committed decrease.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 36b62710ddef0ffaee25837a92ca1ac9b353ff05)
+
+commit 6ff7b02dc23b42a99c7136d1a55ead05c12e2b97
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Aug 20 17:00:23 2015 +0800
+
+    osdc/Objeter: When cancel op, decrease num_unacked/num_uncommitted.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 064e8585a04edb3d87b38db6bed03e965cfcb359)
+
+commit d815e0aacb4a516f4bbbb59a2b2073b38759d9e8
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Aug 20 15:38:58 2015 +0800
+
+    osdc/Objecter: For func op_cancel_writes it can directly call op_cancel.
+    
+    Becasue we get write-lock of rwlock, so it is safe to call op_cancel
+    rather than _op_canchel(homeless_session for this case don't met).
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit e4ce619fe17a7a9dfc18e6af0b84928aa2d88c00)
+    
+    Conflicts:
+          src/osdc/Objecter.cc: the last if (found) was incorrectly
+          implemented as if (to_cancel.size()) after hammer. The
+          conflict is because the patch tries to revert an incorrect
+          if (to_cancel.size()) which is not in hammer.
+
+commit 53316f3694c2a219541dcbdb0269ce97e2d2e9d7
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Aug 19 14:54:21 2015 +0800
+
+    Objecter: Take RLocker when call is_active.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 89f0112e001a2561f9a5cd705898d43c8909501f)
+
+commit 684928883f5eaaec99a8cc8c43e81fae69f9fbfa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 16 15:08:17 2015 +0800
+
+    mon/PGMap: calc min_last_epoch_clean when decode
+    
+    Fixes: #13112
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d0ac68bf1785b330f3202d924c2203ace9393fe6)
+
+commit 7d4b303b2d9649e2722a9b8f0be5c505776124df
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Sep 23 15:58:02 2015 +0000
+
+    osd: print min_last_epoch_clean along with pg dump
+    
+    Fixes: 13198
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit f5359f2314ce3e76ed446c01889dd23550c3ac36)
+
+commit d8ca88dd4b47b4956211d206b0d42112e5842d3f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Sep 22 11:44:18 2015 -0700
+
+    OSDService::agent_entry: don't use PG::operator<< without pg lock
+    
+    Fixes: #13199
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit d422f2820d5beee15117e166546a6386120e0349)
+    
+    Conflicts:
+    	src/osd/OSD.cc: trivial resolution
+
+commit 397042a0cb2f4c628dd722192fee284359d2cbb3
+Author: Xinze Chi <xinze@xksy.com>
+Date:   Wed Sep 23 19:26:00 2015 +0800
+
+    filestore: fix peek_queue for OpSequencer
+    
+    Fixes: #13209
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 5f7b3f5cde779f1300a35a04df2cdb1299f55472)
+
+commit d4e4d85c005ff68cb0461e9d7f0cbe5e6cf2f073
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:01 2015 -0400
+
+    mon/Elector: do a trivial write on every election cycle
+    
+    Currently we already do a small write when the *first* election in
+    a round happens (to update the election epoch).  If the backend
+    happens to fail while we are already in the midst of elections,
+    however, we may continue to call elections without verifying we
+    are still writeable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef909ccbdc303cce8a39edef255325127832ff16)
+
+commit 20a4c0c4831a9dcd49d8ad34b99369effbda9ab8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:13 2015 -0400
+
+    mon/MonitorDBStore: assert/crash if there is a write error
+    
+    Do this globally intead of relying on teh zillion mon callers to
+    check the error code.  There are no cases where we want to
+    tolerate a commit failure.
+    
+    Fixes: #13089
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fb7b1f0e33ada7c9a1be3de2f7708eb0760fcef)
+
+commit 3c1f7cbc45c4744eea3c821f0343596756e05ab1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Sep 24 18:35:39 2015 -0700
+
+    OSDMap: fill in known encode_features where possible
+    
+    Otherwise, if we get an incremental from hammer (struct_v = 6) we will
+    encode the full map as if it were before CEPH_FEATURE_PGID64, which
+    was actually pre-argonaut.  Similarly, if struct_v >= 7, we know it
+    was encoded with CEPH_FEATURE_OSDMAP_ENC.
+    
+    Fixes: #13234
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 04679c5451e353c966f6ed00b33fa97be8072a79)
+
+commit 93c523c65c1f0b6f9a76b1d71acf9e8df57c0fc9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 8 15:47:37 2015 -0400
+
+    librbd: diff_iterate needs to handle holes in parent images
+    
+    If a clone's parent image snapshot includes a discarded extent,
+    this was previously causing an assert failure.  Instead, ignore
+    any discard holes in the parent image.
+    
+    Fixes: #12885
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ccc3bb4bd35e57209852d460633e371b4d004e2)
+    
+     Conflicts:
+    	src/librbd/DiffIterate.cc : file doesn't exist in hammer, logic is in internal.cc
+
+commit 359b68447393980bb7cdd4acc23582e8046b186c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 28 10:25:07 2015 -0400
+
+    tests: new test case for librbd diff_iterate over discard extents
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5650c9cf85188efa73b279c8f4e4723fa475308)
+    
+    Conflicts:
+        src/test/librbd/test_librbd.cc : diff_iterate2 doesn't exist in hammer
+
+commit 71a42f7c3aa55de688ef16ea7e392573fa5a7e62
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Jun 8 16:54:08 2015 +0800
+
+    bug fix: osd: avoid multi set osd_op.outdata in tier pool
+    
+    There are two read op on the same object for ec pool. First op read
+    miss happend, calling do_proxy_read and promote_object, The second op only
+    do_proxy_read. but before first op process_copy_chunk finish, the second op
+    finish_proxy_read. first op receive reply from base pool
+    first and then second received. so the second op set the field "outdata"
+    in m->ops first. And then first op requeue_ops in process_copy_chunk,
+    At last in do_osd_ops, it append outdata field.
+    
+    Fixes: 12540
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 855ae1fd6e4557adba1cbd8ab532488b867cee2a)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit e8cce089102abd8ec1b8e916c6f2c4a5b339cbd3
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Jul 29 16:28:33 2015 +0800
+
+    bug fix: osd: requeue_scrub when kick_object_context_blocked
+    
+    when read miss in writeback cache pool, osd do_proxy_read first
+    and maybe promote it. but in this case, the op is not added to
+    waiting_for_blocked_object. pg scrub maybe block by this object
+    (_range_available_for_scrub). so after promote it, we should
+    requeue_scrub.
+    
+    Fixes: 12515
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit f77949fedce3449befd74efeb5270579f5085b16)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+    		in ReplicatedPG::kick_object_context_blocked, removed hobject_t::BitwiseComparator from map p
+
+commit 8ee93d5f661b303cafcb0134a0e9a1fce7b8e6b2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 27 11:08:33 2015 -0700
+
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Otherwise, the is_active() checks in the hitset code can erroneously
+    return true firing off repops stamped with the new epoch which then get
+    cleared in the map change code.  The filestore callbacks then pass the
+    interval check and call into a destroyed repop structure.
+    
+    Fixes: 12809
+    Backport: hammer,firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 14e02bc90a463805f4c3e2de210892067a52514b)
+
+commit cecd68d06d4d114b0b735155da5c2bac736714c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 11:36:49 2015 +0800
+
+    ceph_test_rados_api_aio: add a test for aio_sparse_read
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 076bad955d374cbb37b77e2b0429f3c85f32abc0)
+
+commit 8849f9933fe0fc315da0981f9cd88ea108f686ed
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 27 22:57:16 2015 +0800
+
+    ceph_test_rados_api_io: add tests for sparse_read
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4d4920610ebfcb516630ed15678979c9e9292f5a)
+    
+     Conflicts:
+    	src/test/librados/test.cc
+    		minor changes and remove C++11 stuff
+
+commit 64b22dcba316bf9ac5598c249e4fba47c157f588
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 26 15:41:13 2015 +0800
+
+    ceph_test_rados: also send sparse_read in ReadOp
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5ae2e7a185b5f95753a09a89d7110fc38848a083)
+    
+    Conflicts:
+    	src/test/osd/RadosModel.h
+    		remove the piped-read test
+
+commit 6ca593026f81580ecbd02ad6140347f31086a87d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 14:27:53 2015 +0800
+
+    osd: should use ec_pool() when checking for an ecpool
+    
+    we were using pool.info.require_rollback() in do_osd_ops() when
+    handling OP_SPARSE_READ to tell if a pool is an ecpool. should
+    use pool.info.ec_pool() instead.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a5bfde69a9d14de67da1e3354173ec70ba089b37)
+
+commit 318049911445a3cc257db2b3beb3ab53e6f1d64e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 27 22:57:49 2015 +0800
+
+    osd: translate sparse_read to read for ecpool
+    
+    Fixes: #12012
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 700d42ef1c82f5602249b96690ae881c1d259d54)
+
+commit d8ac5103e433f6649e28392b365d2e02b4ba1ffe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 19:10:19 2015 -0400
+
+    WorkQueue: add/remove_work_queue methods now thread safe
+    
+    These methods were not acquiring the ThreadPool lock when
+    manipulating the work_queue collection.  This was causing
+    occasional crashes within librbd when opening and closing
+    images.
+    
+    Fixes: #12662
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e18449b01c1ab78d1bbfc1cf111aa9bdbef7b1f)
+
+commit e8d6d5a5f43b75f6aaec4c977221ad97d11e314c (refs/remotes/gh/wip-11786.hammer)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Sep 7 14:12:19 2015 +0100
+
+    mon: MonitorDBStore: make get_next_key() work properly
+    
+    We introduced a significant bug with 2cc7aee, when we fixed issue #11786.
+    Although that patch would fix the problem described in #11786, we
+    managed to not increment the iterator upon returning the current key.
+    This would have the iterator iterating over the same key, forever and
+    ever.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 70d31082fd3dc8c7857994104577f1a3631c678c)
+
+commit dba8b5b113f585586fcbbc0acf05543262486328
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Tue May 19 13:44:21 2015 +0200
+
+    tools: Don't delete, recreate and re-fill buffers in rados bench.
+    
+    Fixes the high CPU usage and corrects rados bench scores on fast SSDs
+    and ramdisks/memstore.
+    For bench run on SSD, on Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz
+    before this patch, times are:
+    write: real 5m0.169s, user 2m33.565s, sys 4m39.791s
+    seq: real 4m28.642s, user 1m35.250s, sys 6m42.948s
+    rand: real 5m0.258s, user 1m19.656s, sys 6m47.145s
+    
+    After this patch:
+    write: real 5m1.162s, user 0m27.788s, sys 3m11.707s
+    seq: real 5m1.149s, user 2m23.278s, sys 4m14.427s
+    rand: real 5m1.021s, user 2m30.514s, sys 4m20.347s
+    
+    Bench run: rados -p ssd bench 300 write|seq|read --no-cleanup
+    
+    Note the increase in user time cpu on seq/read tests,
+    along with decreased sys cpu time; this is because there's
+    additional memcmp() that compares read objects with expected
+    contents. With less time spent memory juggling, more time is
+    spent performing more reads per second, increasing memcmp call
+    count and increasing amount of user cpu time used.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit b894fc7acf7dee7f7ec8c5c280e7a6be41133328)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    		return -5 was replaced with return r
+
+commit fa72ecb509fa4ddd5f227b1e279eae390284a72c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri May 15 15:44:18 2015 +0200
+
+    bufferlist: implement bufferlist::invalidate_crc()
+    
+    This function iterates over all bufferlist internal buffers and calls
+    their invalidate_crc() method. Required for rados bench to work
+    correctly, because it modifies buffers outside buffer api, invalidating
+    internal CRC cache in the process - this method clears that cache, so
+    another call for buffer::get_crc() to return correct checksum.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 55a6f9efbee041d041742810ca4fa8874b6191a7)
+
+commit 50c82f2095add5ad75200a62081fd2f5fea8d8d9
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 1 15:57:03 2015 +0200
+
+    ceph.spec.in summary-ended-with-dot
+    
+    RPM spec files summary should not end with a dot.
+    This was inconsistent across the ceph rpm packages
+    and creates errors with rpm lint.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 042e8ff4dc71795d957bbfac12fade2bc0fc4674)
+
+commit 94699e724b45a9a6de84a943c5b834d2f20f85ef
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 1 14:58:31 2015 +0200
+
+    ceph.spec.in libcephfs_jni1 has no %post and %postun
+    
+    /usr/lib64/libcephfs_jni.so.1.0.0 requires /sbin/ldconfig to be
+    run after installing and after removing.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 0742d82b5561634a5f4ed73fc6cbe4d4f26b0d42)
+
+commit 7475a8fe3f31a46a949f051ded383cd6f396a0fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 21 16:08:09 2015 -0700
+
+    logrotate.conf: prefer service over invoke-rc.d
+    
+    On trusty 14.04, service works but invoke-rc.d does not (but is present).
+    
+    Fixes: #11330
+    Reported-by: Wim Heirman <wim@heirman.net>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6063a21f9edbbf80e44d45ff52a9f4ba1a1cb2c9)
+
+commit 86f88dfce3c0e785d0ff48ec8892e53e6a53b080
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 23 10:15:56 2015 +0100
+
+    mon: fix checks on mds add_data_pool
+    
+    Use same _check_pool helper as during fs creation.
+    
+    Fixes: #12426
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 329741ec238d6af4b69bc838ba667be232741b3d)
+
+commit a65c39852775b17323fa4efee54bde12ce22a512
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 29 15:52:19 2015 +0800
+
+    common/syncfs: fall back to sync(2) if syncfs(2) not available
+    
+    Fixes: #12512
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08210d6a85527b0759f6dc4411f742bc13a4939f)
+    
+    Conflicts:
+    	src/common/sync_filesystem.h
+              - subtle difference in #if/#else/#endif structure
+
+commit e6eaa914d685f2a8f1f8cf3b01112a1b720e1358
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed May 27 22:28:49 2015 +0100
+
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    get_next_key() had a bug in which we would always return the first key
+    from the iterator, regardless of whether its prefix had been specified
+    to the iterator.
+    
+    Fixes: #11786
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 2cc7aee1abe40453093306c8fef2312b650dff5d)
diff --git a/doc/changelog/v0.94.7.txt b/doc/changelog/v0.94.7.txt
new file mode 100644
index 000000000..b639752f6
--- /dev/null
+++ b/doc/changelog/v0.94.7.txt
@@ -0,0 +1,3179 @@
+commit d56bdf93ced6b80b07397d57e3fa68fe68304432 (tag: refs/tags/v0.94.7)
+Author: Jenkins Build Slave User <jenkins-build@trusty-small--296081f3-e7b1-46b1-96c2-d6b655cf71d3.localdomain>
+Date:   Tue May 10 14:47:39 2016 +0000
+
+    0.94.7
+
+commit 6e6c85a0d71df7cd6207febc3e22d30617648abc
+Merge: 934ad41 76c33de
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 9 13:55:24 2016 -0400
+
+    Merge pull request #8979 from athanatos/wip-15655-hammer
+    
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 934ad41dbf3f7004c93aa8f7660353329b7f3fad
+Merge: e219e85 3c03eee
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed May 4 11:37:33 2016 +0200
+
+    Merge pull request #8583 from Vicente-Cheng/wip-15089-hammer
+    
+    hammer: Bucket owner isn't changed after unlink/link
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 76c33decdeba83d4d977a9556e76e68fa964465f (refs/remotes/athanatos/wip-15655-hammer)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 28 13:47:48 2016 -0700
+
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Fixes: http://tracker.ceph.com/issues/15655
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c121bc51a5a877a49094cfbd3bd16af1f3ae5090)
+
+commit 3c03eee07e02b6b4d31f6a58a03253eff2d41324
+Author: Aran85 <zhangzengran@h3c.com>
+Date:   Mon Sep 28 08:23:22 2015 +0800
+
+    rgw:bucket link now set the bucket.instance acl
+    
+    Fixes: #11076
+    
+    Signed-off-by: Zengran Zhang <zhangzengran@h3c.com>
+    (cherry picked from commit 4d59b1d36f8924290c3ecb5b7608747191470188)
+    
+    Conflicts:
+    
+    	src/rgw/rgw_bucket.cc
+                1. Do not use the rgw_user structure and remove the tenant parameter that describes as below
+                2. user_id is not used so just remove the line
+                3. instead of system_obj_set_attr you can use the method set_attr
+    
+    Backport Change:
+    We do not use the rgw_user structure and remove the `tenant` parameter
+    because this feature is not introduced on hammer version.
+    The rgw multi-tenant feature is introduced on pr#6784 (https://github.com/ceph/ceph/pull/6784)
+    This feature is supported from v10.0.2 and later version.
+
+commit e219e85be00088eecde7b1f29d7699493a79bc4d
+Merge: eea2a77 f794ada
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:43:16 2016 -0400
+
+    Merge pull request #7922 from badone/wip-14961-hammer
+    
+    hammer: PK11_DestroyContext() is called twice if PK11_DigestFinal() fails
+
+commit eea2a77ef0b242c295f2480349d8608273dfb184
+Merge: 815760d 38b9c0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:42:44 2016 -0400
+
+    Merge pull request #7992 from SUSE/wip-13807-hammer
+    
+    hammer: segfault in Objecter::handle_watch_notify
+
+commit 815760dde45574f931e316f47c5990129375e9f2
+Merge: ba5c75a 01672b4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:41:41 2016 -0400
+
+    Merge pull request #8051 from xiaoxichen/wip-15024-hammer
+    
+    hammer: clock skew report is incorrect by ceph health detail command
+
+commit ba5c75a33ba10bbe7796715f2d8500ec1fe209fd
+Merge: dc09055 ac3569c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:40:51 2016 -0400
+
+    Merge pull request #8398 from asheplyakov/hammer-wip-15336
+    
+    hammer: monclient: avoid key renew storm on clock skew
+
+commit dc090558a3306feb2c50994950756e5599a92eaf
+Merge: 0f28a5b 744e9f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:24:55 2016 +0200
+
+    Merge pull request #8272 from Yan-waller/yj-wip-fixbl
+    
+    hammer: tests: bufferlist: do not expect !is_page_aligned() after unaligned rebuild
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0f28a5b033b47479ae6f1958d2f079bd6c57f1b4
+Merge: 374c3c1 f8d2abd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:20:15 2016 +0200
+
+    Merge pull request #7883 from tchaikov/wip-12848-hammer
+    
+    hammer: osd: use GMT time for the object name of hitsets
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 374c3c1d8cde50a4503cbd41ee2204aa6761557d
+Merge: add1674 0830275
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:18:11 2016 +0200
+
+    Merge pull request #7917 from ceph/wip-10794-hammer
+    
+    hammer: ceph-objectstore-tool, osd: Fix import handling
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit add1674c759f946c7c456e96e52b7ca1a45829ec
+Merge: d08bae2 be4a9fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:13:32 2016 +0200
+
+    Merge pull request #8026 from liewegas/wip-reweight-hammer
+    
+    hammer: mon: implement reweight-by-utilization feature
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d08bae217e50dfa19dce3bb58349149c4210361e
+Merge: f68697c d4cf190
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:12:55 2016 +0200
+
+    Merge pull request #8049 from xiaoxichen/wip-15022-hammer
+    
+    hammer: keyring permisions for mon deamon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f68697c71ad4b6f49dae12a50fad73cacbcaf6e1
+Merge: e5beeef 57fd7f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:09:42 2016 +0200
+
+    Merge pull request #8052 from xiaoxichen/wip-15051-hammer
+    
+    hammer: test/librados/tier.cc doesn't completely clean up EC pools
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e5beeef1b142a86140791470785c69f865e5df5a
+Merge: 77fbf58 77a4ed0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:03:07 2016 +0200
+
+    Merge pull request #8440 from dachary/wip-15382-hammer
+    
+    hammer: rpm package building fails if the build machine has lttng and babeltrace development packages installed locally
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 77fbf581cb2259146938a737c299d6cf762303d1
+Merge: 92a96e1 97f474f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 15:05:11 2016 +0200
+
+    Merge pull request #8401 from Vicente-Cheng/wip-15129-hammer
+    
+    hammer: Protect against excessively large object map sizes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 92a96e1b16d9cadf182d200dfef0de6c2b388202
+Merge: 7cfc739 281d63d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 13:44:09 2016 +0200
+
+    Merge pull request #8011 from dillaman/wip-15033-hammer
+    
+    hammer: librbd: possible QEMU deadlock after creating image snapshots
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 77a4ed0a8986b2529edd8059dd3efa38f778c6f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 5 11:53:54 2016 +0200
+
+    ceph.spec.in: disable lttng and babeltrace explicitly
+    
+    before this change, we do not pacakge tracepoint probe shared libraries
+    on rhel7. but "configure" script enables them if lttng is detected. and
+    rpm complains at seeing installed but not pacakged files. as EPEL-7 now
+    includes lttng-ust-devel and libbabeltrace-devel, we'd better
+    BuildRequire them, and build with them unless disabled otherwise. so in
+    this change
+    * make "lttng" an rpm build option enabled by default
+    * BuildRequire lttng-ust-devel and libbabeltrace-devel if the "lttng"
+     "lttng" option is enabled
+    * --without-lttng --without-babeltrace if the "lttng" option is disabled
+    
+    Fixes: #14844
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f1a44907a2f83cb8080cd8ab5980a30306a33fbd)
+    
+    Conflicts:
+    	ceph.spec.in: trivial resolution, the context changed
+            (other packages, files etc.) but the semantic did not
+
+commit 97f474fb6b26ae7b7e2fdd21de4f4bd98cdd56ab
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 14 13:57:28 2016 -0400
+
+    cls_rbd: protect against excessively large object maps
+    
+    Fixes: #15121
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4aff4ea3290dc7fb62c639bfc74fcfdde5fe9542)
+    
+      Conflicts:
+    	src/cls/rbd/cls_rbd.cc
+            `RBD_METADATA_KEY_PREFIX` is not introduced on hammer, so remove
+            it
+
+commit ac3569cd8bbb4b2b9a0edf298ce0b83bc18f1fea
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Mar 21 18:20:18 2016 +0300
+
+    hammer: monclient: avoid key renew storm on clock skew
+    
+    Refreshing rotating keys too often is a symptom of a clock skew, try to
+    detect it and don't cause extra problems:
+    
+    * MonClient::_check_auth_rotating:
+      - detect and report premature keys expiration due to a time skew
+      - rate limit refreshing the keys to avoid excessive RAM and CPU usage
+        (both by OSD in question and monitors which have to process a lot
+        of auth messages)
+    * MonClient::wait_auth_rotating: wait for valid (not expired) keys
+    * OSD::init(): bail out after 10 attempts to obtain the rotating keys
+    
+    Fixes: #15336
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 918c12c2ab5d014d0623b1accf959b041aac5128)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	  no loadavg checks in Hammer, hence the conflict. Move the
+    	  counter and max_attempts initialization to a suitable place.
+
+commit 7cfc739a513da84d76b959efc51dd21c1cc552d6
+Merge: 0418943 7eae05e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 31 13:50:23 2016 +0200
+
+    Merge pull request #8187 from liewegas/wip-15171
+    
+    hammer: osd/ReplicatedPG: do not proxy read *and* process op locally
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+
+commit f8d2abd2e41c5dd04977f85cc1d6e65853c9a1b2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 3 12:03:44 2016 +0800
+
+    osd/osd_types: encode pg_pool_t the old way
+    
+    as a workaround of #12410
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 720a090eb67b3955b0cadb7633c5a28a934171a4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 8 16:52:32 2015 +0800
+
+    mon: disable gmt_hitset if not supported
+    
+    the gmt_hitset is enabled by default in the ctor of pg_pool_t, this
+    is intentional. because we want to remove this setting and make
+    gmt_hitset=true as a default in future. but this forces us to
+    disable it explicitly when preparing a new pool if any OSD does
+    not support gmt hitset.
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e48cec3dc93b3988dcd8924933deb1b3a43e1d0f)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		remove unrelated hunk
+
+commit 7aec079f8a1bbe75625c438a17bb87e45398568e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 00:32:38 2015 +0800
+
+    osd: do not let OSD_HITSET_GMT reuse the feature bit
+    
+    * to ease the backport to hammer
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5a4f6a866bb675195ad83c38fd23238d078c78ed)
+    
+    Conflicts:
+    	src/include/ceph_features.h
+    		minor changes to resolve conflicts
+
+commit 370434136ef076c350db3db4fca6489f88f70453
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Aug 27 11:24:25 2015 -0700
+
+    osd: Decode use_gmt_hitset with a unique version
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2bc5a48f4c5d3667213be3a7b5a0e0f5ef9daf4f)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    		minor changes to resolve conflicts
+
+commit 64bca2a43b34b265621bad2ec1fb980217223847
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 16:53:50 2015 +0800
+
+    mon: print use_gmt_hitset in "ceph osd pool get"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc2bcf760f2d2f20fc4a2fa78ba52475a23e55bf)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		use the old way to dump pool info
+
+commit 87df212cfca33efbbee6376f528cb7d4895d1dc0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 00:04:23 2015 +0800
+
+    mon: add "ceph osd pool set $pool use_gmt_hitset true" cmd
+    
+    allow "ceph osd pool set $pool use_gmt_hitset <true|1>" as long as
+    the cluster supports gmt hitset.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 03a1a3cf023a9aeb2fa26820e49e5efe3f3b3789)
+
+commit 039240418060c9a49298dacc0478772334526dce
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 5 21:06:48 2015 +0800
+
+    osd: use GMT time for the object name of hitsets
+    
+    * bump the encoding version of pg_hit_set_info_t to 2, so we can
+      tell if the corresponding hit_set is named using localtime or
+      GMT
+    * bump the encoding version of pg_pool_t to 20, so we can know
+      if a pool is using GMT to name the hit_set archive or not. and
+      we can tell if current cluster allows OSDs not support GMT
+      mode or not.
+    * add an option named `osd_pool_use_gmt_hitset`. if enabled,
+      the cluster will try to use GMT mode when creating a new pool
+      if all the the up OSDs support GMT mode. if any of the
+      pools in the cluster is using GMT mode, then only OSDs
+      supporting GMT mode are allowed to join the cluster.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42f8c5daad16aa849a0b99871d50161673c0c370)
+    
+     Conflicts:
+    	src/include/ceph_features.h
+    	src/osd/ReplicatedPG.cc
+    	src/osd/osd_types.cc
+    	src/osd/osd_types.h
+    		fill pg_pool_t with default settings in master branch.
+
+commit 744e9f837a94f95c5e877f074fd1479f8da64e3c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 4 01:23:31 2015 +0800
+
+    test/bufferlist: do not expect !is_page_aligned() after unaligned rebuild
+    
+    if the size of a bufferlist is page aligned we allocate page aligned
+    memory chunk for it when rebuild() is called. otherwise we just call
+    the plain new() to allocate new memory chunk for holding the continuous
+    buffer. but we should not expect that `new` allocator always returns
+    unaligned memory chunks. instead, it *could* return page aligned
+    memory chunk as long as the allocator feels appropriate. so, the
+    `EXPECT_FALSE(bl.is_page_aligned())` after the `rebuild()` call is
+    removed.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8ed724222651812c2ee8cc3804dc1f54c973897d)
+
+commit 0830275b39afc408573c2c468fa3b0e7b1d5cc4e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 6 14:35:35 2015 -0400
+
+    osd/PG: fix generate_past_intervals
+    
+    We may be only calculating older past intervals and have a valid
+    history.same_interval_since value, in which case the local
+    same_interval_since value will end at the newest old interval we had to
+    generate.
+    
+    This was introduced by 70316541bbb115d9a35954bfba373cf1dc084b7e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 98cbf0380e9ee848725fc87d61c31a734305a84f)
+
+commit 0418943c6ef8c9649a58003444daeb4b6224fbab
+Merge: 35eae5f 2072a53
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 18 10:13:40 2016 -0400
+
+    Merge pull request #8123 from liewegas/wip-15136-hammer
+    
+    qa/workunits: remove 'mds setmap' from workunits
+
+commit 7eae05e14231cb3476d14bf74b15bf9646849d09 (refs/remotes/me/wip-15171)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 17 13:28:40 2016 -0400
+
+    osd/ReplicatedPG: do not proxy read *and* process op locally
+    
+    If we proxy the read, we cannot return false, even if we fail to also
+    trigger a promotion.
+    
+    Fixes: #15171
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit be4a9feae4c5f33685cceb386717a5733553df10 (refs/remotes/me/wip-reweight-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 14 08:07:19 2016 -0400
+
+    osd/OSDMap: fix typo in summarize_mapping_stats
+    
+    From ea9abe53d0e777b7dc3b22af71639f77c4de08c8
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cc7e2329046ef7c528f321092a464f6db6e7edb9)
+
+commit 2072a53dc920b79ed76265d6097e50d3d2994b1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 15 09:44:13 2016 -0400
+
+    qa/workunits: remove 'mds setmap' from workunits
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6a80e2df8b31802a39ef6f89136ed357dce50c20)
+
+commit 01672b4afd6f886d6bb8527427663e062ab9d100
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 11:20:36 2016 +0000
+
+    mon: Monitor: get rid of weighted clock skew reports
+    
+    By weighting the reports we were making it really hard to get rid of a
+    clock skew warning once the cause had been fixed.
+    
+    Instead, as soon as we get a clean bill of health, let's run a new round
+    and soon as possible and ascertain whether that was a transient fix or
+    for realsies. That should be better than the alternative of waiting for
+    an hour or something (for a large enough skew) for the warning to go
+    away - and with it, the admin's sanity ("WHAT AM I DOING WRONG???").
+    
+    Fixes: #14175
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    
+    (cherry pick from commit 17d8ff429c7dca8fc1ada6e7cc8a7c4924a22e28)
+
+commit f90b8bc1a2b3ffa5ba52c0992c1c5ff52e5ea104
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 11:05:44 2016 +0000
+
+    mon: Monitor: adaptative clock skew detection interval
+    
+    When in the presence of a clock skew, adjust the checking interval
+    according to how many rounds have gone by since the last clean check.
+    
+    If a skew is detected, instead of waiting an additional 300 seconds we
+    will perform the check more frequently, gradually backing off the
+    frequency if the skew is still in place (up to a maximum of
+    'mon_timecheck_interval', default: 300s). This will help with transient
+    skews.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    
+    (cherry pick from commit 45e16d0d7407cf4e886dde9321bb91c1a087776d)
+    
+    Conflicts:
+    	src/common/config_opts.h
+                Merge the change line.
+    	src/mon/Monitor.h
+                handle_timecheck_leader(MonOpRequestRef op) was replaced with handle_timecheck_leader(MTimeCheck *m)
+                also for handle_timecheck_peon and handle_timecheck.
+
+commit 57fd7f858c220af362b330d12db0e87de82b4346 (refs/remotes/gh/hammer-pr-8052)
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Nov 25 19:20:51 2015 -0800
+
+    test/librados/test.cc: clean up EC pools' crush rules too
+    
+    SetUp was adding an erasure-coded pool, which automatically adds
+    a new crush rule named after the pool, but only removing the
+    pool.  Remove the crush rule as well.
+    
+    http://tracker.ceph.com/issues/13878 Fixes: #13878
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 04b4795f81c15bfcb62ba5807745470ce0e5e949)
+
+commit d4cf19049349678065195659bb287b899f92a686
+Author: Owen Synge <osynge@suse.com>
+Date:   Wed Mar 2 12:15:48 2016 +0100
+
+    keyring permissions for mon daemon
+    
+    The command:
+    
+    sudo ceph-mon --cluster ceph --mkfs -i $NODE_NAME --keyring $PATH_MON_KEYRING
+    
+    Writes a new keyring with permissions set to "0644".
+    
+    Fixes: #14950
+    Signed-off-by: Owen Synge osynge@suse.com
+    (cherry picked from commit c2f91a8ce46974a72b960b7cb25af3d089fbb80d)
+
+commit 35eae5fbddd2b271a10db9a69d4c4e34029bca06
+Merge: c59b994 647723e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:05:42 2016 +0700
+
+    Merge pull request #7911 from dzafman/wip-meta-list-hammer
+    
+    hammer: tools, test: Add ceph-objectstore-tool to operate on the meta collection
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c59b9944a61174c7e29a26f4abfd7839020578af
+Merge: dd66eb9 d875620
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:03:43 2016 +0700
+
+    Merge pull request #7903 from branch-predictor/bp-hammer-force-bench-smoke
+    
+    hammer: tools/rados: add bench smoke tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit dd66eb91d0129163ecfc8105f043fd0f6926dad5
+Merge: de43aba 5726463
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:02:18 2016 +0700
+
+    Merge pull request #7876 from SUSE/wip-14946-hammer
+    
+    hammer: redhat-lsb-core dependency was dropped, but is still needed
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de43aba0015927cec5ebd8f98ad26a801b1353ff
+Merge: 48dfac6 6a6754f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:00:48 2016 +0700
+
+    Merge pull request #7817 from asheplyakov/hammer-bug-14873
+    
+    hammer: tools: fix race condition in seq/rand bench (part 2)
+    
+    Reviewed-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 48dfac64ba37e5d0ec262798fbec011c1016a4f0
+Merge: 7083829 3a5b102
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:00:18 2016 +0700
+
+    Merge pull request #7896 from asheplyakov/hammer-bug-14873-p1
+    
+    hammer: tools: fix race condition in seq/rand bench (part 1)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+
+commit 1b922e5431339d12e661de9fd4b16e8cf2c08ad5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:50:57 2016 -0500
+
+    test/pybind/test_ceph_argparse: fix reweight-by-utilization tests
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d0f87131962b26cef9e31fa11651771856c41164)
+
+commit 06a2a7596b876a5c2addb184a48222bcb6e23b9a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 10 21:41:15 2016 +0800
+
+    man/8/ceph.rst: remove invalid option for reweight-by-*
+    
+    --yes-i-really-mean-it is not valid option for reweight-by-*
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 797f1d41d98738411079a7a16d8b1d5352359d33)
+
+commit 241f76262273bafd72e91fee3d46c412f7418767
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 9 13:25:55 2016 -0500
+
+    mon: remove range=100 from reweight-by-* commands
+    
+    Specifying the range makes it skip instead of error when you give
+    it a bad value.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 661891aa28ca0600a62e25f6d10ce7fa94e32cce)
+
+commit 55ad2c7724e114728bb327a97726a6c9b660bb1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:46:12 2016 -0500
+
+    mon: make max_osds an optional arg
+    
+    We keep the config option with the default (4), but let you
+    specify the max # of osds to reweight as a third optional arg.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 651f05b6e6bec41b54c0a8f8ca6641ce3aa5dff3)
+
+commit f13cdeaf6657577c21a5bb2985d46971aa084c3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:38:57 2016 -0500
+
+    mon: make reweight max_change default configurable
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8a9429d85bc99874dc9871b43b2d0c7ad1086803)
+
+commit f4b4ef76295d40753c6acffa3d92128f7ebe8733
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:38:19 2016 -0500
+
+    mon/OSDMonitor: fix indentation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47b650c1497b8eaf37cfcb7e3b9fda12d8cf2d7a)
+
+commit 76eb3c8fbdb614a4cdde937a1dec680b4be82470
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 2 17:34:48 2016 -0500
+
+    qa/workunits/cephtool/test.sh: test reweight-by-x commands
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 28b33a5b57e3f898808bd8ccc5e314667f84912a)
+
+commit 9a9d147aa14a637bbf8edddf85f6c36a29874694
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:28:59 2016 -0500
+
+    osd/MonCommand: add/fix up 'osd [test-]reweight-by-{pg,utilization}'
+    
+    - show before/after pg placement stats
+    - add test- variants that don't do anything
+    - only allow --no-increasing on the -utilization versions (where
+      it won't conflict with the optional pool list and confuse the
+      arg parsing)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1de7e47833f2d14c6915d7d071fe0005408c8da0)
+    
+    - use non-C++11 sort
+    
+    # Conflicts:
+    #	src/mon/OSDMonitor.cc
+    #	src/mon/OSDMonitor.h
+
+commit 6ec676df083ec3078f5efd8058d6fb6a491c9b04
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 2 14:00:15 2016 -0500
+
+    mon: add 'osd utilization' command
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b757eff868dbce2393beadd9fe2522372288cc0)
+
+commit 94134d9148edfcc41d2d239db69d0b299fbf427f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:30:45 2016 -0500
+
+    osd/OSDMap: add summarize_mapping_stats
+    
+    Calculate some basic stats about the PG distribution.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ea9abe53d0e777b7dc3b22af71639f77c4de08c8)
+    
+    - remove c++11 auto and range-based for loops
+
+commit 932f75d1f2105f296681b5fb02757087947e01f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:49:03 2016 -0500
+
+    mon: make reweight-by-* max_change an argument
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a70eaabcfc1cc1dfb9b6ba2d7f71b3d76496eb55)
+
+commit d8372cea0c51db89cc48e5592236d53f7b01b6b3
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 22:29:46 2016 +0100
+
+    osd: add mon_reweight_max_osds to limit reweight-by-* commands
+    
+    Add configurable mon_reweight_max_osds which limits the number
+    of OSDs modified each time reweight-by-* is called (by default
+    to 4 OSDs).
+    
+    Also change the order in which we look at OSDs to go from most
+    to least utilized.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 3aa0ccefddc1790df5b808bfccd5da4857951220)
+
+commit 6a422b2626d48098c5a2c13de677814429263cba
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 22:28:46 2016 +0100
+
+    osd: add mon_reweight_max_change option which limits reweight-by-*
+    
+    Add a configurable mon_reweight_max_change which limits be default
+    the reweight-by-* commands to only change an OSDs weight by 0.05.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 6f0a1fb69da84ee18bdb9b151772cef8f32413c5)
+
+commit d3635b796ae28e1bc6598daf4109bf371a1f4ed8
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 21:58:00 2016 +0100
+
+    test: add simple test for new reweight-by-* options
+    
+    Test validity of --no-increasing and --yes-i-really-mean-it.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1a6ad5053e7fc4f8942855a17ce4540e340e9d4f)
+
+commit e993851b7abf80fd793aa5b7188f1af7dbe03041
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 21:52:41 2016 +0100
+
+    osd: add sure and no-increasing options to reweight-by-*
+    
+    Add a --no-increasing option to reweight-by-* which can be used to only decrease
+    OSD weights without increasing any. This is useful for example if you need to
+    urgently lower the weight of nearly full OSDs.
+    
+    Also add a --yes-i-really-mean-it confirmation to reweight-by-*.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit ddf5c2b62316bb69839f220fe2527d1467863421)
+
+commit 281d63df792c4cdf25cca5428969c65d2ae379e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 9 18:00:04 2016 -0500
+
+    librbd: complete cache reads on cache's dedicate thread
+    
+    If a snapshot is created out-of-band, the next IO will result in the
+    cache being flushed.  If pending writeback data performs a copy-on-write,
+    the read from the parent will be blocked.
+    
+    Fixes: #15032
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f9e5ae80e3b0799b5a7e1d3f93b2d85a4baff20f)
+
+commit 621e3aef85e4b720232ca9ee89a4620d58362047
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 9 17:31:06 2016 -0500
+
+    test: reproducer for writeback CoW deadlock
+    
+    Refs: #14988
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 16b6efdd24b25ba1f6bc658681afa3d0878eb397)
+    (cherry picked from commit 7ffbe4099c9367e7405e2196803a0d8b12253a06)
+
+commit 38b9c0b2fbbb056c29cd0485a9831edce020eaa6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 16 11:32:34 2015 -0500
+
+    osdc/Objecter: call notify completion only once
+    
+    If we race with a reconnect we could get a second notify message
+    before the notify linger op is torn down.  Ensure we only ever
+    call the notify completion once to prevent a segfault.
+    
+    Fixes: #13805
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 987f68a8df292668ad241f4769d82792644454dd)
+
+commit 7083829c227403a77fcf35a2376dc02e4c9693c8 (refs/remotes/markhpc/hammer)
+Merge: 651e49e c4ba93a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Mar 8 10:00:15 2016 +0700
+
+    Merge pull request #6604 from ukernel/hammer-13268
+    
+    hammer: client: added permission check based on getgrouplist
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 651e49ea2037834fbbf48e642a78075140e06782
+Merge: 522e3e8 b2393dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Mar 8 09:58:30 2016 +0700
+
+    Merge pull request #6551 from ceph/hammer-client-inoderef
+    
+    hammer: client inoderef
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f794adadd126bd77ae138794119725a046642712
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Fri Mar 4 13:06:47 2016 +1000
+
+    tests: Add TEST_no_segfault_for_bad_keyring to test/mon/misc.sh
+    
+    94da46b6e31cac206cb32fc5bd3159209ee25e8c adds
+    TEST_no_segfault_for_bad_keyring which requires changes to run
+    in hammer since test/mon/misc.sh is not written to run multiple tests in
+    succession in the hammer version.
+    
+    Refs: #14960
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 522e3e8d761a337796ed92868e00062b87e0f0a5
+Merge: d314849 e014ea8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 4 10:25:00 2016 +0700
+
+    Merge pull request #7797 from SUSE/wip-14861-hammer
+    
+    hammer: ceph init script unconditionally sources /lib/lsb/init-functions
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 94da46b6e31cac206cb32fc5bd3159209ee25e8c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Tue Dec 8 00:22:02 2015 +0800
+
+    tests: make sure no segfault occurs when using some bad keyring
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 2eb0bf32244fc174d56cc42e2298c67d8b5299ac)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit a371c0f1d69fea131b56100492d3045c15ae072c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Wed Nov 25 19:03:03 2015 +0800
+
+    auth: fix a crash issue due to CryptoHandler::create() failed
+    
+    In this case(e.g. user passes wrong key), attempts to call the CryptoKey.ckh will lead to a segfault.
+    
+    This patch fixes crash issue like following:
+    
+    Program received signal SIGSEGV, Segmentation fault.
+    [Switching to Thread 0x7fffed10e700 (LWP 25051)]
+    0x00007ffff59896c6 in CryptoKey::encrypt (this=0x7fffed10d4f0, cct=0x555555829c30, in=..., out=..., error=0x7fffed10d440) at auth/cephx/../Crypto.h:110
+    110	    return ckh->encrypt(in, out, error);
+    (gdb) bt
+        at auth/cephx/../Crypto.h:110
+        at auth/cephx/CephxProtocol.h:464
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit a7f520caabf997b3753f1200f19f21830531b3f5)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit af5da4f24d7c2ea5a1322450180099278dd0da7c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Wed Nov 25 18:24:30 2015 +0800
+
+    auth: fix double PK11_DestroyContext() if PK11_DigestFinal() failed
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit e9e05333ac7c64758bf14d80f6179e001c0fdbfd)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit c3f031a32df5d434721a2b00f2c8961759b1a916 (refs/remotes/markhpc/wip-10794-hammer)
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 20 14:09:02 2015 -0700
+
+    ceph-objectstore-tool, osd: Fix import handling
+    
+    Use same_interval_since set to zero to communicate that an import has happened
+    and the correct value needs to be computed.
+    
+    Fixes: #10794
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 70316541bbb115d9a35954bfba373cf1dc084b7e)
+    
+    Conflicts:
+    	src/osd/PG.cc (trivial)
+
+commit 647723ecd283273657bdb8b35dbb904e55c37add
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 4 10:37:42 2015 -0700
+
+    tools, test: Add ceph-objectstore-tool to operate on the meta collection
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 31fa89f24997c3e9b4f8795ea28938ab65493d10)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+    is_pg() still needs snap argument
+
+commit d8756208a3213e099574e57e9b654feab84b0497
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Thu Mar 3 11:30:53 2016 +0100
+
+    common/obj_bencher.cc: make verify error fatal
+    
+    When run without "--no-verify", all verification errors are noted,
+    but they are not forwarded/reported anywhere else but to cerr, which
+    will cause automated testing to ignore them. Make seq_read_bench and
+    rand_read_bench return -EIO on any verification error which will,
+    in turn, return it back to caller.
+    
+    Fixes: #14974
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 658bba85c488a38e25bc598637258c263f711631)
+
+commit 04fe9517cd049e97b845cef848f19a8b8dc3976c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Thu Mar 3 14:40:44 2016 +0100
+
+    test/test_rados_tool.sh: force rados bench rand and seq
+    
+    This forces rados bench rand and seq to execute, so if there's
+    a regression in any of above, it won't go unnoticed.
+    
+    Fixes: #14974
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+
+commit d314849ab9b59ae55c6a86b878d123eb1c13d88a
+Merge: 20b2a74 31a2fc4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 08:35:37 2016 -0500
+
+    Merge pull request #7672 from dachary/wip-14783-hammer
+    
+    hammer: test_bit_vector.cc uses magic numbers against #defines that vary
+
+commit 6a6754f8dc834dbb13144cebdeeea2ff29e09127
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Fri Feb 26 15:01:11 2016 +0300
+
+    hammer: tools: fix race condition in seq/rand bench (part 2)
+    
+    Commit c2c6d02591519dfd15ddcb397ac440322a964deb which is intended to
+    cherry-pick 9bcf5f065c4ed4b10d8f98961d1f99493bcdc9b8 incorrectly resolved
+    a conflict by adding code where it should have been removed. The incorrect
+    conflict resolution can been displayed with
+    
+       commit=c2c6d02591519dfd15ddcb397ac440322a964deb
+       picked_from=9bcf5f065c4ed4b10d8f98961d1f99493bcdc9b8
+       diff -u --ignore-matching-lines '^[^+-]' <(git show $picked_from) <(git show $commit)
+    
+    --- /dev/fd/63    2016-03-03 14:09:51.354329129 +0700
+    +++ /dev/fd/62    2016-03-03 14:09:51.358329122 +0700
+    @@ -76,20 +79,18 @@
+          ++data.in_flight;
+     -    if (!no_verify) {
+     -      snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index);
+    --      lock.Unlock();
+    +       lock.Unlock();
+     -      if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) {
+     -        cerr << name[slot] << " is not correct!" << std::endl;
+     -        ++errors;
+     -      }
+    --    } else {
+    --        lock.Unlock();
+    --    }
+    --
+    -+    lock.Unlock();
+    -     name[slot] = newName;
+    -   }
+    -
+    -@@ -789,11 +791,14 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
+    ++    if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) {
+    ++      cerr << name[slot] << " is not correct!" << std::endl;
+    ++      ++errors;
+    +     } else {
+    +       lock.Unlock();
+    +     }
+    +@@ -776,11 +785,14 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
+            }
+            lc.cond.Wait(lock);
+          }
+    
+    http://tracker.ceph.com/issues/14873 Fixes: #14873
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 3a5b1029f8f30bca569b12195329224208855096
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Mar 3 15:30:23 2016 +0300
+
+    hammer: tools: fix race condition in seq/rand bench (part 1)
+    
+    src/common/obj_bencher.cc:601: the lock should be taken before calling completion_ret,
+    not after. Also note that if r < 0 the lock will be unlocked twice in a row.
+    As a result rados bench seq fails with assertion in Mutex::Unlock().
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 0c8faf7c9982c564002771c3a41362a833ace9bb)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    	src/common/obj_bencher.h
+      Pick only the lock related part to unbreak seq bench. The failure due
+      to the missing (or wrong sized) objects can be easily worked around, and
+      the changes required to fix this problem are way too intrusive for hammer.
+    
+    http://tracker.ceph.com/issues/14873 Related: #14873
+
+commit c4ba93a6529ab549d42b601432ec88d8243909bf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 29 15:24:41 2015 +0800
+
+    client: use fuse_req_getgroups() to get group list
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0eb6d0ba1b65ac0aaffc72a82f07857cc65a6b88)
+
+commit a84ed87082076e1990fb79703aa5ac8eea5d34ea
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 29 15:11:24 2015 +0800
+
+    client: use thread local data to track fuse request
+    
+    When handling an operation, libcephfs code may want to access fuse
+    request for extra information. by tracking fuse request in thread
+    local data, we can avoid adding extra parameter to Client::ll_foo
+    functions.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a9f91d7d70893cdc5f832a26a5be298941d00315)
+
+commit e7f299ace762c7d02cad1f7253dba87404d944f5
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Aug 18 12:34:01 2015 +0200
+
+    client/Client.cc: remove only once used variable
+    
+    Fix for:
+    [src/client/Client.cc:4555]: (style) The scope of the variable
+     'initial_group_count' can be reduced.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a29dd45dd89f59ff15018f541601ac5ede162174)
+
+commit 16e3e2f0a0dc465ed9079df051215f4758f0ee4d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Aug 12 18:38:38 2015 +0200
+
+    client/Client.cc: fix realloc memory leak
+    
+    Fix handling of realloc. If realloc() fails it returns NULL, assigning
+    the return value of realloc() directly to the pointer without checking
+    for the result will lead to a memory leak.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4f98dab99c35663de89a06e2dfdbd874f56aed41)
+
+commit b13ddc0690742b11802e59f34b539987d632ac58
+Author: blinke <Burkhard.Linke@computational.bio.uni-giessen.de>
+Date:   Mon Aug 10 17:43:10 2015 +0200
+
+    client: added permission check based on getgrouplist
+    
+    Fixes: #13268
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f04c8da5432174874ca97d11a5b2fef56f95d73d)
+
+commit 562c0a962b291b90112ccea7d8c6f7c92a14851f
+Author: blinke <Burkhard.Linke@computational.bio.uni-giessen.de>
+Date:   Mon Aug 10 15:31:44 2015 +0200
+
+    configure.ac: added autoconf check for getgrouplist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 16b59c6cd68d532a47e8c0e7b1f088b41433f048)
+
+commit e014ea88861f4f27b4a5db14c32133a3b864411e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jul 23 15:07:45 2015 +0800
+
+    init-ceph: check if /lib/lsb/init-functions exists
+    
+    On OSX/FreeBSD, /lib/lsb/init-functions does not exist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bb1fa7f3993dfcd1243b2a9a0bb432c269df215f)
+
+commit 20b2a7413e8f7538ff0683313e7b19c41aaf69ad
+Merge: c1bc048 c63baeb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 3 12:05:58 2016 +0700
+
+    Merge pull request #7671 from tchaikov/wip-14583-hammer
+    
+    hammer: global/pidfile: do not start two daemons with a single pid-file
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c1bc048dd708a729137e1d28e0ccfedfe5d90933
+Merge: 546a7b9 28d42dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 3 11:51:33 2016 +0700
+
+    Merge pull request #7702 from ceph/wip-14077-hammer
+    
+    hammer: scrub bogus results when missing a clone
+    
+    Reviewed-by: Martin Palma <martin@palma.bz>
+    Reviewed-by: Wei-Chung Cheng <freeze.vicente.cheng@gmail.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5726463368ddcc688e6308d22992568ba4c0da32
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Feb 26 18:30:49 2016 +0100
+
+    packaging: lsb_release build and runtime dependency
+    
+    The lsb_release executable is being run in multiple places, not least in
+    src/common/util.cc, which calls it via shell in the collect_sys_info() code
+    path.
+    
+    This patch addresses this issue on SUSE- and Debian-derivatives, as well
+    as reinstating the dependency for RHEL/Fedora after it was dropped in
+    15600572265bed397fbd80bdd2b7d83a0e9bd918.
+    
+    http://tracker.ceph.com/issues/14906 Fixes: #14906
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b87f4efb5689a8db77e06baf4f469afcd9df3677)
+    
+    Conflicts:
+    	ceph.spec.in
+                 The jewel specfile has diverged considerably from hammer:
+                 systemd, package split, etc. This is more of a hand backport
+                 than a cherry-pick.
+
+commit c63baebbb9bb5e6564955e920d0f658e5a3092af
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Feb 1 19:32:13 2016 +0700
+
+    global: do not start two daemons with a single pid-file (part 2)
+    
+    Fixes the following bugs:
+    
+    * the fd is open(O_WRONLY) and cannot be read from, safe_read
+      always fails and never removes the pid file.
+    
+    * pidfile_open(g_conf) is close(STDOUT_FILENO) and there is a risk that
+      pidfile_open gets STDOUT_FILENO only to have it closed and redirected
+      to /dev/null.
+    
+    * Before writing the file, ftruncate it so that overriding a file
+      containing the pid 1234 with the pid 89 does not end up being
+      a file with 8934.
+    
+    * Before reading the file, lseek back to offset 0 otherwise it
+      will read nothing.
+    
+    * tests_pidfile was missing an argument when failing
+      TEST_without_pidfile and killed all process with ceph in their name,
+      leading to chaos and no useful error message.
+    
+    * lstat(fd) cannot possibly return a result different from the one
+      obtained right after the file was open, stat(path) must be used
+      instead.
+    
+    In addition to fixing the bugs above, refactor the pidfile.cc
+    implementation to:
+    
+    * be systematic about error reporting (using cerr for when removing
+      the pidfile because derr is not available at this point and derr
+      when creating the pidfile).
+    
+    * replace pidfile_open / pidfile_write with just pidfile_write since
+      there never is a case when they are not used together.
+    
+    More test cases are added to test_pidfile to verify the bugs above are
+    fixed.
+    
+    http://tracker.ceph.com/issues/13422 Fixes: #13422
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9828d49d6f3ccfc78d496153d263ea39b1722d4b)
+    
+    Conflicts:
+    	src/global/global_init.cc
+          - the `flag` argument of `global_init_prefork()` is not used, so
+            it was removed in master. but the cleanup commit was not
+            cherry-picked to hammer, thus the conflict. we can just keep it
+            around in hammer to minimize the code churn, although it may
+            stand in the way of future backports.)
+          - s/nullptr/NULL/ as hammer does not support c++11.
+
+commit 28d42dc8e0a0dd71d03c1b69fb7464b753bd4cc0
+Merge: 9144c38 9282c1d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:48 2016 -0800
+
+    Merge backport pull request #6545 into wip-14077-hammer
+    
+    A commit skipped because remove_coll() still needed in Hammer
+
+commit 9144c3837f284ac2c922d2438b0e0f0cc7304d7a
+Merge: fd74fbf 3226615
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:35 2016 -0800
+
+    Merge backport pull request #5783 into wip-14077-hammer
+
+commit fd74fbf04a965ca93dc9602f99e3cb7a3577340a
+Merge: 4309fa8 0fe3dfe
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:17 2016 -0800
+
+    Merge backport pull request #5031 into wip-14077-hammer
+
+commit 4309fa87e13a20368ca34f7699436b628c9d7876
+Merge: de5e11e 1bc8882
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:24:58 2016 -0800
+
+    Merge backport pull request #5842 into wip-14077-hammer
+
+commit de5e11ec69e7388a1a9c28614759b2e2d8b70ad8
+Merge: 7698e6c 6ff4217
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:24:31 2016 -0800
+
+    Merge backport pull request #5127 into wip-14077-hammer
+
+commit 7698e6c54aae1d06664d8e8324595400e6087f24
+Merge: c8057c6 de80bbf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:23:14 2016 -0800
+
+    Merge backport 6 commits from pull request #5197 into wip-14077-hammer
+    
+    Excludes the ceph-objectstore-tool meta-list op
+
+commit c8057c693bf0ae7bb01ef238f4ef9f23454a2e33
+Merge: ecbf6d2 df0e11e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:22:26 2016 -0800
+
+    Merge backport branch 'wip-temp' into wip-14077-hammer
+
+commit ecbf6d2882073363ca1ddc5209e1e9537f51fa77
+Merge: 3415312 6c8884b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:21:20 2016 -0800
+
+    Merge backport pull request #4932 into wip-14077-hammer
+
+commit 34153127963b5baec0c2b84f7aa19b29fc81d4fa
+Merge: e022b5b 87a7f99
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:20:41 2016 -0800
+
+    Merge backport pull request #4915 into wip-14077-hammer
+
+commit e022b5b55facb359f6fd27884e2408fda70db30a
+Merge: 4ed44e1 9ca2f35
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:20:00 2016 -0800
+
+    Merge backport 1 commit from pull request #4863 into wip-14077-hammer
+
+commit 4ed44e1e33df986468585d00a6dc401025263217
+Merge: 3f0fc84 78a59f8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:18:38 2016 -0800
+
+    Merge backport 8 commits from pull request #4784 into wip-14077-hammer
+    
+    Only excludes actual fix for #10794
+
+commit 3f0fc84c4180473a87db01cb40d0488bcaa2c202
+Merge: db68c6c eab0f24
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:15:05 2016 -0800
+
+    Merge backport 41 commits from pull request #4473 into wip-14077-hammer
+
+commit db68c6cb683870e68bf6143c6902b8ac89ecdc3a
+Merge: c3d938d 18c49b6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:13:37 2016 -0800
+
+    Merge backport branch 'wip-journal-header' of git://github.com/XinzeChi/ceph into wip-14077-hammer
+    
+    Includes a change to the code in the original merge commit itself
+
+commit c3d938d81020072c2d2eecf58f4639e773181b1a
+Merge: 7abb6ae a1fc101
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:13:02 2016 -0800
+
+    Merge backport 1 commit of pull request #3686 into wip-14077-hammer
+
+commit 9282c1d01c4f2d99fb3832a3cf495f99fdc6487e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 5 19:06:17 2015 -0800
+
+    ceph-objectstore-tool: Add dry-run checking to ops missing it
+    
+    Update test to check most operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e89f0e1c5e8a378809bec7c705ea2c23235d4b28)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit efc2183bafacfcd29fbb1b42231d4c147199e528
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 11 15:26:52 2015 -0800
+
+    test: Remove redundant test output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e6f1bdcd310bd6417f0f695fba3430fd0edba43d)
+
+commit 322661547ec437a9cdb893fcd280a48cd60ec990
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 28 21:34:44 2015 -0700
+
+    test: osd-scrub-snaps.sh uses ceph-helpers.sh and added to make check
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d712737be1aadb38ecc3882200999d79d748cdb8)
+    
+    ceph-helpers.sh in different location in Hammer
+
+commit 995a0042a1b8ed04b77e6bc4757a10552450dc7f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 10 18:20:46 2015 -0800
+
+    test: Verify replicated PG beyond just data after vstart
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9a29c5983cd392e4b479b6b685fb0c3797c10cd2)
+
+commit 6afb5d3be26931c670f22c892c925e2556c8e562
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 26 11:36:43 2015 -0700
+
+    osd: Use boost::optional instead of snap 0 for "all_clones"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 310bf784b7cd8d75f2ab7ee431bc764f5b312aa2)
+
+commit 750f81732bc803128a8a25c07e10e9f709dc2a1a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 10 17:57:59 2015 -0800
+
+    test: Fix verify() used after import to also check xattr and omap
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 92d00f22805599c519806c6ff6164847b5c33931)
+
+commit b8c9507f632510d7da8545d7afc49b618435ef3c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sun Oct 25 18:57:18 2015 -0700
+
+    osd, test: When head missing a snapset, clones not an error
+    
+    Fix log message for this case and make into "info"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f508ddc6e9b40fe5c1c54e5faa569f2bc7b78c4b)
+
+commit 59fee8a65c1a39d7e3806852aabbf68d7ca37feb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 9 20:02:21 2015 -0800
+
+    test: Add test cases for xattr and omap ceph-objectstore-tool operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cbaed6a8c737f77e68a0eb7f1fca32ec4155f045)
+
+commit 0988b1238d43630ff0b05d321952021ede73c2ae
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Oct 24 16:23:13 2015 -0700
+
+    osd, test: Keep missing count and log number of missing clones
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2e0bb0ae9498fd9a0c49f1b81bf0c86f7a7cac42)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h (trivial)
+
+commit 37be95996494f72d4fa3ecc62b54ee9478903c6e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 11 08:28:01 2015 -0800
+
+    rados: Minor output changes for consistency across operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 016ed34ce85fa111f54857a8ca1002ae32b52b8e)
+
+commit 6c51e4831f34a67308e3f8190d9aad2c3d029bf7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 20 11:14:42 2015 -0700
+
+    test: Eliminate check for bogus "obj13/head snaps empty" error
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8227b4b75c24b632e637decbe4b818606e8589ee)
+
+commit e92505bec093cb3070e4962bc86b82a257c64171
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 20:34:28 2015 -0700
+
+    ceph-objectstore-tool: Add new remove-clone-metadata object op
+    
+    Fixes: #12740
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6d283f0f16347a13418d8a0ed136913e1161893)
+    
+    No Sequencer in transactions in Hammer
+
+commit 8f88b442a26603cc321fccc7047cb8639b2a0f65
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 20:42:48 2015 -0700
+
+    osd: Fix trim_object() to not crash on corrupt snapset
+    
+    Add test case to check for trim_object() osd crash
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eb0ca424815e94c78a2d09dbf787d102172f4ddf)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (trivial)
+
+commit 78b13f5d591d9d9676939abc4febeb7d2eebb1a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 13:12:17 2015 -0700
+
+    ceph-objectstore-tool: Improve object spec error handling
+    
+    Add test cases
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit caf2d5970b0a3e3ce4ef941d1955b7821a0fc916)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 7b800b7c3efb5403d03fc0c4386821b433b85268
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 7 17:25:44 2015 -0700
+
+    ceph-objectstore-tool: Add undocumented clear-snapset command for testing
+    
+    Add test cases using new feature to corrupt SnapSet
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b6302acdd344997aabdb426104de9f24e9990437)
+    
+    No Sequencer in transactions in Hammer
+
+commit 7f398bd2a3264d1ac5bf40a5470c43454dbad571
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 21:59:50 2015 -0700
+
+    ceph-objectstore-tool: Add set-size command for objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 138a33b8946e6fea1e09527b8bc45b7aa4bee1d6)
+    
+    No Sequencer in transactions in Hammer
+
+commit 53dc87f82642de769a7ea11554552e6db06a3e65
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 22:04:05 2015 -0700
+
+    ceph-objectstore-tool: Enhanced dump command replaces dump-info
+    
+    Show object stat information from objectstore
+    Show SnapSet if present
+    Add --head for search by object name
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1688debf7892d36c773c12d7d0b1b9b4cddc5f98)
+
+commit a46fc6613e1f1f6709184d68f23bccab0fd03bd6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 2 13:39:29 2015 -0700
+
+    test: Add some clones to ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b4ba3e66548a5dee58553df071b916c85d701caf)
+
+commit fd518e704e0e5c01f3b7efa8905ddde1e64cc842
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 1 19:28:54 2015 -0700
+
+    ceph-objectstore-tool: For corrupt objectstores, don't abort listing on errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d276d3286504ea702ae561b91dff4121c99feb9a)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit ad7825a4beba879f6a38ef3eda0648f11f8783bd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 13:02:40 2015 -0700
+
+    ceph-objectstore-tool: Improve some error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b0c884ba8b51eedf9def093c61f0f06398101ecc)
+
+commit 26cbf14ef1e44936f48fdb2be40b59ae2117666f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 3 14:22:20 2015 -0700
+
+    ceph-objectstore-tool: White space fixes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0564f398269d920d90f6d6ecb008ebcd6609d611)
+
+commit 0f785646d2f52522706835a2f1c8e0101db9a622
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 15:33:26 2015 -0700
+
+    tools/rados: Improve xattr import handling so future internal xattrs ignored
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9222f56403cf52ec161a38fc437a3f04c24d1625)
+    
+    Conflicts:
+    	src/tools/rados/RadosImport.cc (Code in ceph_objectstore_tool.cc in Hammer)
+
+commit c8e27725249cf587e65af323e3ae0ba43c740c10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 29 19:57:43 2015 -0700
+
+    test: Test scrubbing of snapshot problems
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5ad33ec07ee9fffe862d84955a566261d101713)
+
+commit 113d5c75a5bd33b1b5fe399d83d94a671eb40658
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Sep 28 10:59:42 2015 -0700
+
+    osd: Don't crash if OI_ATTR attribute is missing or corrupt
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e0b39650028205e9d9e314b75c444cc8cf055a31)
+
+commit 3af8f9e8044fb1e126f208903c41cde065db304e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 18:02:32 2015 -0700
+
+    osd: Additional _scrub() check for snapset inconsistency
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9e48e1801ea1a10ce495b3947da06676c2f5f72f)
+
+commit 7103e74e335652f2a28b1dad0a2f2028398a600d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 3 19:43:35 2015 -0700
+
+    osd: Better SnapSet scrub checking (find issues instead of asserting)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3b381caaad20c683a330e8b7a4e1c017abcb60df)
+
+commit 18af852a2c71b733ec5c6bd397b2e712b676c693
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 26 13:58:09 2015 -0700
+
+    osd: Make the _scrub routine produce good output and detect errors properly
+    
+    Catch decode errors so osd doesn't crash on corrupt OI_ATTR or SS_ATTR
+    Use boost::optional<> to make current state clearer
+    Create next_clone as needed using head/curclone
+    Add equivalent logic after getting to end of scrubmap.objects
+    
+    Fixes: #12738
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a23036c6fd7de5d1dbc2bd30c967c0be51d94ca5)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (no num_objects_pinned in hammer)
+    	src/osd/ReplicatedPG.h (no get_temp_recovery_object() in hammer)
+
+commit 3a1b58838dba6bd482aa5786dc44962c4a977c17
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 30 17:17:15 2015 -0700
+
+    osd: Fix log message name of ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 545e4b2c582625f8b93502266ca59ee40e0d0e5f)
+
+commit 0fe3dfe8768b27743189add73cde9a44095f2a62
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 16 11:57:08 2015 +0300
+
+    ceph-objectstore-tool: add mark-complete operation
+    
+    It is supposed to be used as a last resort to fix a cluster that has
+    PGs in 'incomplete' state, using the following procedure:
+    
+    1) stop the osd that is primary for the incomplete PG;
+    2) run:
+      ceph-objectstore-tool --data-path ... --journal-path ... --pgid $PGID --op mark-complete
+    3) start the osd.
+    
+    Fixes: #10098
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 6907778d767ba08bb80c495785056ed122b023fe)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 1bc88827fc40f124f570dd8b7865c687f1b1f5a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 12:33:44 2015 -0700
+
+    test: Fix failure test to find message anywhere in stderr
+    
+    Consolidate test_failure() and test_failure_tty()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 95bd3c2c54312417b132ddb91c89fdbe63b01fd0)
+
+commit 6ff421738c93ff37521ccf67caf3d09d284e339c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 17 14:46:16 2015 +0800
+
+    test: add test for {get,set}-inc-osdmap commands.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 818de1a2c89272d98610328a958fea06d1cc41cc)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit de80bbf4d7c1b78017cf447a7f511ceefac39e50
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 23 16:51:27 2015 -0700
+
+    test: Add debug argument to the ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a3b14ede9e829ec85c49980ecbff11f584ca70ae)
+
+commit 0643797cb7a37b5601a7dd8da9a81f8194b8f25d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 13:41:31 2015 -0700
+
+    rados: Fix usage for "notify" command
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b968fb3bcefb0cdafb1e84d7c679b61a201a940f)
+
+commit 5ba8649423fd5a2d5c212273c901f9ddd5f44632
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 8 15:51:55 2015 +0800
+
+    test: add test for {get,set}-osdmap commands
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0257c150fe552300c0d9f5a22a27aaa49fb6cc64)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 3276258b1a9123caec8d635aa1a2de3ef02201e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 25 17:15:39 2015 -0700
+
+    tools, test: Some ceph-objectstore-tool error handling fixes
+    
+    Improve various error messages generated with invalid syntax
+    Add test cases for most of these error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 127a5f191d3d2442edd3a3fa820006bca3a08c93)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit cfabcc1066a06644b619fa027e27d338411120c9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 14 10:33:47 2015 +0800
+
+    tools/ceph-objectstore-tool: add get-inc-osdmap command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7801bb20d23ce83fc4093c23331bcb75e43c878f)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit c7d0fda01efc29d81723d4edfba2bfbbe6765760
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 20:11:10 2015 -0700
+
+    tools: Check for valid --op earlier so we can get a better error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d8460714fab61d0a3092b3e7b290c0c19ad58beb)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+    	src/tools/ceph_objectstore_tool.cc (meta-list op not in Hammer)
+
+commit be24c50bb74aa6cd7bb08519cdaf267d4107eef3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jul 11 16:27:47 2015 +0800
+
+    tools/ceph-objectstore-tool: add set-inc-osdmap command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 95344d07c3a1a9a6b1246e5eca3f7daeb062dd06)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 06dcf74374f57224536522ef46dba8a53a0d755d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jun 10 17:04:57 2015 -0700
+
+    tools: Fix newlines in output of --op list
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6aec38b51f0c14684912b5ee28fb5b1375cb8f8a)
+
+commit e44c042c3dffc4ca396dbdfac24fa65914c0b73b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 3 23:09:54 2015 +0800
+
+    tools/ceph-objectstore-tool: add "get-osdmap" command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f0c130d6b1bc2a8a0cce6d78a6e69995fe00a427)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 3f9e46779addaa07d04d501113eb95cea95b802b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 18:38:01 2015 -0700
+
+    tools: Fix dump-super which doesn't require pgid
+    
+    A valid pgid had to be specified in order for superblock to be displayed,
+    but it was otherwise ignored.
+    Fix message that didn't include dump-super
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fd1772e0ac4cb801d6f3b056645eeb4d3d6ef001)
+
+commit c60eee1dcd64d4452e96ed937f94a3ec41faf9fe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jul 2 19:14:55 2015 +0800
+
+    tools/ceph-objectstore-tool: add "set-osdmap" command
+    
+    Fixes: #12193
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3e30c1746fb8d90b04e4776849069db0b7737c87)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit cfe7d47d38fd103601c09eceb98b9c7365869fba
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 19:16:22 2015 -0700
+
+    tools: Check and specify commands that require the pgid specification
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5534bc864813d96a0d20791d79151c4f6695a013)
+
+commit df0e11e7783e239ae7be992b342df980d399ab95
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 11 16:25:49 2014 -0800
+
+    hobject_t: modify operator<<
+    
+    Put the most significant fields to the left so that it matches the sort
+    order.  Also use unambiguous separator when the nspace is present
+    (like we do with the key).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ecdc8f697acf1627b35a83663c219a3bb0ada453)
+
+commit 6c8884b11b8211642662bfd7d612872621ffd8ff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 11 12:57:10 2015 -0700
+
+    test, tools: Improve ceph-objectstore-tool import error handling and add tests
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ddc4d52782a04d99a4293c9fc278894dfe328515)
+
+commit 87a7f991d1f2f1a05fff18accc7e31ff393083f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 9 18:14:09 2015 -0700
+
+    tools: For ec pools list objects in all shards if the pgid doesn't specify
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1aa9655f10b0fcc42cb5114681b8b24a2ccb68f4)
+
+commit 9ca2f35d02169e6f82e3d45e00fe880c02cbd8ce
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Apr 20 17:28:15 2015 +0100
+
+    tools: clean up errors in ceph-objectstore-tool
+    
+    We use negative error codes everywhere else, should
+    use them here too.  Also add a couple of strategically
+    placed prints.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8e5594da29e782b6efd046ed03f313599dafa1c5)
+    
+    Conflicts:
+    	src/tools/RadosDump.cc (Excluded, not in Hammer)
+
+commit 78a59f8b0c43adbd9a7a1db68dafcbf7bd2ad74d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 1 17:22:18 2015 -0700
+
+    test/ceph-objectstore-tool: Don't need stderr noise
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d7858c66454093dd6f2fab60e33e5b2472bcd80a)
+
+commit eab0f2403ea025f4e659dc055323b227023448ab
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Apr 9 16:26:07 2015 -0700
+
+    osd: Show number of divergent_priors in log message
+    
+    Report dirty_divergent_priors as true or false
+    Use dout(5) so tests can verify divergent_priors created
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 67da8feb0ff8659f1ff812be5a866ef5e97facf2)
+
+commit d58793d10d682fba2ceafa4d6a1668c4917be77f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 6 18:14:06 2015 -0700
+
+    osd, tools: Always filter temp objects since not being exported
+    
+    Can't use object_locator_to_pg() directly with a temp object
+    because the pool is negative.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 557c6530a515cffacbaec439cabff8a079edbf5b)
+
+commit efc402ea9846d3995bf3d4c98188842cb69870cf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 1 17:15:21 2015 -0700
+
+    test/ceph-objectstore-tool: Show command that should have failed
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 23551deea66673c35d7183d18b63b5a8438ebf00)
+
+commit 88ac5198f542cdea87d0f600a2201132c47f158d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 12 15:28:07 2015 -0700
+
+    test: Add config changes to all tests to avoid order dependency
+    
+    ReplayCorrupt was crashing because journal_ignore_corruption wasn't set true
+    Improve ReplayCorrupt by checking setting of bool corrupt and use ASSERT_FALSE()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9016269a1c248da69a0e85df4ca3ba95011710c5)
+
+commit 3d99ecd6131f41ebe60c352266dca9193d61bbc9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 11:15:58 2015 -0700
+
+    tools: Don't export temporary objects until we have persistent-temp objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4fcf5dd370e8cfac75fadbf709a785053eb26ad9)
+    
+    Remove "assert(!i->hobj.is_meta());" since is_meta() is not in Hammer
+
+commit 13360d3a55818fc5b63a2bb9889a67f3fc98d286
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 17:41:40 2015 -0700
+
+    test/ceph_objectstore_tool: Improve dump-journal testing
+    
+    Fix false failure when journal has no entries
+    Test dump-journal on all osds in 2 places
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ce76f88777ec026521cce465095d6376efd460a8)
+
+commit 444ce0ac718e903b39c25cf58d0a503ead491790
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 11:52:00 2015 -0700
+
+    ceph-objectstore-tool: Allow --pgid specified on import (for post split)
+    
+    Revamp split import testing to use --pgid option post split
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 90202a70f8c32120625c99ef5131778b3c9f6303)
+
+commit aed1c49e7d3631d797776be1020f6df0f77ea1f3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 10:53:48 2015 -0700
+
+    ceph-objectstore-tool: Invalidate pg stats when objects were skipped during pg import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 51e27984ec131fd21bbc26411ae1540b9047eb37)
+
+commit af3f8ae1d9e210499fd0165e473f9b5d87666f75
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:22:03 2015 -0700
+
+    ceph-objectstore-tool: Add dump-super to show OSDSuperblock in format specified
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 99d7663f870abfa82711dbf0451334f7d6c883d8)
+
+commit 4dcf15bc71c815cfdb0f6bc6f90c9b603d18b37f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:25:23 2015 -0700
+
+    mds, include: Fix dump() numeric char array to include additional alpha chars
+    
+    Caused by: c87f55b5a8b3e003726e1be22b9cb6894ef2f37d
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f1520061bba1e052a9be3ae11161122c1f6c3425)
+
+commit feecacfb34dfd831291fc736ea39dbcf69e7d765
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:08:14 2015 -0700
+
+    ceph-objectstore-tool: Add dump-journal as not requiring --pgid in usage
+    
+    Additional comment describes --pgid requirement checking
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bd2f62445be7153a9600bcf4399e8fc56f1efd3f)
+
+commit 5e8fbb1858f7a8dd65d1b9f873aaca05f3ddb1eb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:37:46 2015 -0800
+
+    test: ceph_test_filejournal: Conform to test infrastructure requirements
+    
+    Only call RUN_ALL_TESTS() once
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e30efe6c15780a6239b9970297df1f2c1918c9b1)
+
+commit c161cbf81a29e523bb4ffb897f6d02112eb49b09
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:13:50 2015 -0800
+
+    test: ceph_test_filejournal need to force aio because testing with a file
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d540dde1faee47d01ee358ae384fa301fa59f00f)
+
+commit 06d3f512149dc9aa38d959ac31891b81634d0ea4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:04:36 2015 -0800
+
+    test: ceph_test_filejournal fix missing argument to FileJournal constructor
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c94fe8db36e7d54227edde94a99bc1aa236b8ec8)
+
+commit 2078f639d91fbaa9b89832811315c7c41da25a4b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 19 19:48:11 2015 -0800
+
+    test: ceph_test_filejournal  Add check of journalq in WriteTrim test
+    
+    Commit all journal entries and check for empty journalq
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8da4d9f6330c57d6b437ea44f54f147075e24195)
+
+commit ab893d769d2728368f94b035edde750a3fa5e617
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 29 17:14:39 2015 -0800
+
+    test: Fix ceph-objectstore-tool test missing fd.close()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ece630d5a741ba8c82f55f964f1887cb7fcaa493)
+
+commit b5f2ccd5bd77c7a473f38fdab183fd52577b7331
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 29 17:13:41 2015 -0800
+
+    test: Fix ceph-objectstore-tool test error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 11c13eb7c182870727fbcb25a83f3348ac0bad13)
+
+commit 848822db38e000a7e7bfe49ae0fe03996bb7e8c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Apr 1 10:32:32 2015 -0700
+
+    test: ceph-objectstore-tool: Remove duplicate debug messages, keep cmd/log/call together
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 94b3d7632f2548c19beca5a2e024c95f601b61f3)
+
+commit 771dcd945281a091b165c0b1a90c1f6058dbf923
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 27 19:47:01 2015 -0700
+
+    test: ceph-objectstore-tool import after split testing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d80ce6668c1aa3c9898ab49e847c4dadafe256f0)
+
+commit 4f387b1e1c8dbfe9ed2a70fc02ff5ccf93ffb905
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 31 08:16:03 2015 -0700
+
+    test: Use CEPH_DIR where appropriate
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1e18e8a88f88809081684b09f36691cd724720b5)
+
+commit b337d677daf5b6098a2dbcf2d8966703cf8db449
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 30 17:35:07 2015 -0700
+
+    test: Limit how long ceph-objectstore-tool test will wait for health
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 868cf457a3990ba9989aef55823630e3189acf4a)
+
+commit 09cb8a41ed6a9642978e399b426b21704c704f46
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 30 17:33:46 2015 -0700
+
+    test: Add optional arg to vstart() to provide additional args to vstart
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit dff17a800c9cc1cf0cc23d70479c031aa8e9cbfc)
+
+commit b4ac42be79bb11cb27282d97aec9e6ca8f658e5c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 19:35:27 2015 -0800
+
+    test: Test ceph-objectstore-tool --op dump-journal output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9b08bcf95caf77fb7f9bd293559534ed8ff3eded)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 729abf5a83a656ac2a95ad147909bf7c8fa18999
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 17 14:56:06 2015 -0700
+
+    test: Pep8 fixes for ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b0141108c0596b535d25792a9102787b1cd3eca3)
+
+commit 33813b66fa4402fe0d66d95d6f9b790b11c9f1f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 18:37:44 2015 -0800
+
+    test: Fix ceph-objectstore-tool test, overwrite OTHERFILE so second check is meaningful
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1b9f85befa1efff3b05b2af0e62d568eea37d790)
+
+commit f7ab316be9ce3592e0bc1c3a1012b0a83dfea2b4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 19:16:45 2015 -0800
+
+    osd: FileJournal: Add _fdump() that takes Formatter instead of ostream
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 130eba48495d0e6acd8a472cbf0c47ebbc7a038c)
+
+commit 99d3e17fe8c03eb30b18dee185623a9c16553d9e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 12:34:35 2015 -0800
+
+    osd: Add simple_dump() to FileJournal for unit testing
+    
+    Allow a dump when entries aren't ObjectStore::Transaction's
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8d5464c0f81aa9bb3be233a03bce64b505a265f4)
+
+commit 80fc57f79a9bd23e119be73fe01063e85c60ca83
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 12:29:38 2015 -0800
+
+    osd: FileJournal clean-up
+    
+    Move aio / directio adjustments to constructor
+    Indicate functions which only operate before journal is active
+    Make functions const when possible
+    Move push_back() of journalq to read_entry()
+    Change dump() to use do_read_entry() to minimize side effects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 12fdf4ced0b2c17770c21204c8eccca8e4d0d2c9)
+    
+    Conflicts:
+    	src/os/FileJournal.cc (trivial)
+
+commit b8f4ea17274f9094efa169a28440701319c14516
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:48:22 2015 -0800
+
+    osd: Dump header in FileJournal::dump()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3c97b32131e83c934b3d340eeb75284e898380c4)
+
+commit 21c3c18c7b25ddea664179797527ea9854c3f450
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:40:25 2015 -0800
+
+    osd: FileJournal::read_entry() can't use a zero seq to check for corruption
+    
+    Seen when using dump() on a journal which has all entries committed
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a7cda53d20eb6d35203fce6ec9f7ea1df432bf53)
+
+commit 288902fecd78b1926aa4a1d53587828fda63845d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:36:01 2015 -0800
+
+    osd: Fix flushing in FileJournal::dump()
+    
+    Use ostream arg since caller is specifying
+    Move flush after last section close
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f2d83520f67fdc1b5a89b4ea742493a3df392989)
+
+commit a935ce501692f51be004f8cb20da29f18e649d44
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Apr 30 10:20:24 2015 -0700
+
+    osd: Add admin socket feature set_recovery_delay
+    
+    Set the value of osd_recovery_delay_start to specified seconds
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 73b17319379d904ffdf9479ca476da8e8e34e935)
+    
+    Conflicts:
+    	src/osd/OSD.cc (trivial)
+
+commit 4ae3f88cc4493413c4aab54e21759507b0d572b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 27 15:37:14 2015 -0700
+
+    ceph-objectstore-tool: For import/export --debug dump the log
+    
+    Move log dumping code to common function dump_log()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 545c78c65f33fcac60b5aa7ee2b574ceb0211fd5)
+
+commit cc5fa68c184f66d3ca6ff736eb308cba0a0d10b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 30 16:56:10 2015 -0800
+
+    ceph-objectstore-tool: If object re-appears after removal, just skip it
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6e41663e7434b4dcbb1abbe62ac008388da268a)
+
+commit d8ae1a9b842b6871225f0cc2d8a8dbdcf01238e4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 30 16:11:13 2015 -0800
+
+    ceph-objectstore-tool: Add --no-overwrite flag for import-rados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 35cdcabbdad7dc7249029630c64022c822c67ddc)
+
+commit 2dbf843560de2b9b74d9b0d0baf2a303e6c4e86b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 2 16:23:18 2015 -0800
+
+    ceph-objectstore-tool: Remove list-lost because now we have --dry-run flag
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3e7fe7e85716f80e055ffbc5305f406b3e020608)
+
+commit 35991747c4cf5c4f9c150aae128f63c7b8c8a5cd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 23 07:34:11 2015 -0800
+
+    ceph-objectstore-tool: Add --dry-run option
+    
+    Fixes: #10290
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2795161f470ab87cf4a82f455773c9b1b82946ef)
+
+commit 05d3b73ac6ef841af5037336996d1ed6bfdac4b5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 22 10:07:09 2015 -0800
+
+    ceph-objectstore-tool: Add dump-info command to show object info
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 244092d034781d2e980f9de7c6f9eea227967c5a)
+
+commit 2d764c54d4b028455be8b592712c6424f5f9f19e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 27 14:32:42 2015 -0800
+
+    ceph-objectstore-tool: Use empty string for <object> to specify pgmeta object
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c00290b836adba12212e8a57f13b0c9193ac0ef8)
+
+commit 3a533d767bafc466e653bdd89d064b5f2b9aa257
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Apr 20 17:28:15 2015 +0100
+
+    ceph-objectstore-tool: Add a couple of strategically placed prints
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fdb1a4b676765c5d3ca5d436f34230dd6996d7a7)
+
+commit 7947f4f72f4bb6441ea5c353757d9ea3f464cff4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 20 16:10:18 2015 -0700
+
+    ceph-objectstore-tool: Clean up error handling
+    
+    Use negative errors throughout and positive error to specify exit status
+    cpp_strerror() handles negative errors
+    Clean up main() code to use local "ret"
+    Let end of main() decide how to treat the final value of "ret"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a50679a1a3313e4090bf24f6e4dda28e7d67d731)
+    
+    Adjust back cherry-pick f237ed97228839a1b412ad213945f0343df05bf5
+        Switched to use ret as in the original change from master
+
+commit 83de86e4e9dedc05fd65201715dadbd46074b7e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Apr 15 15:40:41 2015 -0700
+
+    ceph-objectstore-tool: Create section around log/missing/divergent_priors of --op log
+    
+    Remove extra new lines in the output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 548147b412eebe57861768e5fd66360c2582d2c6)
+
+commit ddfaa70ade3b6d7ff061083fcd610b588fbd4dd7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 14 13:49:26 2015 -0700
+
+    ceph-objectstore-tool: Add divergent_priors handling
+    
+    Add to --op log output
+    Add to metadata_section for export
+    Add filtering of divergent_priors in case of splits
+    
+    Fixes: #11237
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6ad7e9b362f437fe028fe59639489efe87b3525f)
+
+commit add937c437a4bd86c932ef49525aff039d9bee90
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Mar 18 10:05:24 2015 -0700
+
+    ceph-objectstore-tool: Add --force option which is used for import only
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2f4090daaea715b896e49f678933abb8a13b5c98)
+
+commit f33274894a83072a483b0646a91b5182a72c7c90
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 00:41:39 2015 -0700
+
+    ceph-objectstore-tool: Fix pgid scan to skip snapdirs
+    
+    Long standing bug where it wasn't skipping snapdirs if !debug
+    For debug output use stderr like all other cases
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 24f06598966319a62fa58a5831d9741fac1d471b)
+
+commit 3e68825e60a2e69a20198439347aadf8742d4ea6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 12:24:19 2015 -0800
+
+    ceph-objectstore-tool: Add dump-journal op
+    
+    Have dump-journal op directly dump a filestore journal without
+    mounting or try to dump after mounting.
+    
+    Fixes: #11135
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 381605e34a44edb9acfcfb5a56bd598b020e022f)
+
+commit aaff4d7bcbc3e48aa6c0342d66a9c30c93744064
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 12:46:20 2015 -0800
+
+    ceph-objectstore-tool: On any exit release CephContext so logging can flush
+    
+    Add myexit() which will put() the g_ceph_context if present before exit
+    Must umount() once the objectstore has been mounted
+    
+    Fixes: #10809
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 409328bace0ffb3a457c0d384377bbbf37a2962d)
+
+commit 7445cf5a04d396dbc36e21aecd44bc4aadaff397
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 17:28:04 2015 -0800
+
+    ceph-objectstore-tool: Check for keyvaluestore experimental feature
+    
+    Don't list keyvaluestore as a valid type if feature not enabled
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fb2f9d500e52aa0834666550a684fe991701c9dd)
+
+commit 9da6c01b5e59e59a80d00e3d17cf5da05dd84f72
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 17:14:50 2015 -0800
+
+    ceph-objectstore-tool: Eliminate obscure "Invalid params" error
+    
+    Will fail later with "No such file or directory" for an empty data-path
+    because we've already checked for vm.count("data-path")
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cfca6835bd53d535bde2793551211845b0facebd)
+
+commit c5ac7ce09bc5b96722738a1884c6046aba6518f2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 15:37:57 2015 -0800
+
+    ceph-objectstore-tool: Check pgid validity earlier like we did before
+    
+    This code was not doing the umount() even though it was moved after mount()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 565b413df9c562a4071b9372d6362a9105c3538e)
+
+commit 18c49b6474a563b5f7f7cef671050d46ac4c3c4f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 1 09:22:01 2015 -0700
+
+    Backport the merge commit of branch 'wip-journal-header' of git://github.com/XinzeChi/ceph
+    
+    (cherry picked from commit 8d596bfa24c9c3e2aadbb87494284cb5fd824e55)
+    
+    Conflicts:
+    	src/os/FileJournal.cc (trivial)
+    	src/os/FileJournal.h (trivial)
+    
+    Renamed write_header() to write_header_sync() in a merge
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 546a7b988dac56770730ddc9395ee0d52a666fa3
+Merge: 6b7660c 3352b14
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 22:39:06 2016 +0700
+
+    Merge pull request #7656 from dzafman/wip-14769-hammer
+    
+    ceph.in: Notify user that 'tell' can't be used in interactive mode
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 6b7660c91d1837b585b365068da835127752ebfc
+Merge: e34c209 14b5fea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 22:37:58 2016 +0700
+
+    Merge pull request #7648 from tchaikov/wip-13783-hammer
+    
+    mon/LogMonitor: use the configured facility if log to syslog
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cf433bac5b7bc6bdbb6fac56c420288544c79ef5
+Author: shun-s <songshun134@126.com>
+Date:   Wed Dec 2 10:36:35 2015 +0800
+
+    global/pidfile: do not start two daemons with a single pid-file
+    
+    add functions named pidfile_open and pidfile_verify to avoid starting two daemons by a single pid-file
+    
+    Fixes: #13422
+    Signed-off-by: shun song <song.shun3@zte.com.cn>
+    (cherry picked from commit f2c0ef40fd674fecc6e3e97cd6155b976e6759b4)
+
+commit e34c2094ef44e6e52896820f4dd90a1c5d5bbfe8
+Merge: ec33aec b43c5b2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 19:39:49 2016 +0700
+
+    Merge pull request #7801 from dachary/wip-14863-hammer
+    
+    hammer: unittest_crypto: benchmark 100,000 CryptoKey::encrypt() calls
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit b43c5b205f26198c79e21af3a6935a76f560a7fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:57:33 2015 -0800
+
+    unittest_crypto: benchmark 100,000 CryptoKey::encrypt() calls
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fb4b6c52d8e31e8dedfa8aecc16c389f0e7d86cf)
+    
+    Conflicts:
+    	src/test/crypto.cc : complements the incorrect cherry-pick
+               df3f971eafda9c54881c13dcf47f996f18e17028 see
+               http://tracker.ceph.com/issues/14863 for more information
+
+commit ec33aec31fcac3b09bf06256b95c841a8c20d56f
+Merge: 4b46182 17d1b0d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:09:54 2016 +0700
+
+    Merge pull request #7185 from dachary/wip-14325-hammer
+    
+    hammer: rgw: radosgw-admin bucket check --fix not work
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 4b461829c24cd35656e86f507a78b0fff860c0b5
+Merge: 722ca62 5c92d1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:09:28 2016 +0700
+
+    Merge pull request #7188 from dachary/wip-13799-hammer
+    
+    hammer: rgw: default quota params
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 722ca62e12ac977961d895dceaeb55f31d7b00ca
+Merge: 3c1e2d4 ad4df3b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:04:02 2016 +0700
+
+    Merge pull request #7414 from dachary/wip-14493-hammer
+    
+    hammer: make rgw_fronends more forgiving of whitespace
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 3c1e2d4216353c848f76b52d9c187e83574267a7
+Merge: b73a93d 3d84420
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:15:27 2016 -0800
+
+    Merge pull request #7415 from dachary/wip-14554-hammer
+    
+    hammer: Unable to bring up OSD's after dealing with FULL cluster (OSD assert with /include/interval_set.h: 386: FAILED assert(_size >= 0))
+
+commit b73a93d8cd8bc8757a7a8960b2a9d448c7041515
+Merge: b3eac8e 5a450e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:15:03 2016 -0800
+
+    Merge pull request #7576 from dachary/wip-14579-hammer
+    
+    hammer: mon sync does not copy config-key
+
+commit b3eac8e3dbb74ebcfdd22d10d6a6f2916d2bf127
+Merge: bf6dcc7 9e0a165
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:14:37 2016 -0800
+
+    Merge pull request #7577 from dachary/wip-14689-hammer
+    
+    hammer: consume_maps clearing of waiting_for_pg needs to check the spg_t shard for acting set membership
+
+commit bf6dcc7c6da5d06b21e4a4fad3f7a65e8ca84f50
+Merge: 293e8c7 d5ba063
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:03:55 2016 -0800
+
+    Merge pull request #7590 from SUSE/wip-14720-hammer
+    
+    hammer: OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from
+
+commit 293e8c7b038a22e2135e7bf288a8ff4b8ce32a14
+Merge: 5b724fc 10d29da
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:03:18 2016 -0800
+
+    Merge pull request #7645 from dachary/wip-14760-hammer
+    
+    hammer: OSD coredumps with leveldb compact on mount = true
+
+commit 5b724fcce0e260a04bd6889c3d4c6d9bf44971cf
+Merge: bb84692 1c8c708
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:56:13 2016 +0700
+
+    Merge pull request #7485 from dillaman/wip-14611-hammer
+    
+    librbd: flattening an rbd image with active IO can lead to hang
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bb84692e96569823e23fdd02356446577959f438
+Merge: ee014d8 df3f971
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:55:18 2016 +0700
+
+    Merge pull request #7488 from ceph/wip-auth-hammer
+    
+    hammer: auth: use libnss more safely
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ee014d8084c05ec3c876a9e46dcb6e558918a0e4
+Merge: 576225d 88ffcc2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:48:05 2016 +0700
+
+    Merge pull request #7475 from dachary/wip-14602-hammer
+    
+    hammer: ceph-disk: use blkid instead of sgdisk -i
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 576225d6d5a64f0f43971ccca4d74b5dc44c5c63
+Merge: a6f7a2f eb048a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:47:07 2016 +0700
+
+    Merge pull request #7456 from dachary/wip-14587-hammer
+    
+    hammer: qa/workunits/post-file.sh: sudo
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a6f7a2f1347393c1cf7564a2ba8d1ba6aff2122c
+Merge: 74cfe5c f04e007
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:38:47 2016 +0700
+
+    Merge pull request #6946 from dachary/wip-14087-shard-size
+    
+    osd: log inconsistent shard sizes
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 74cfe5c5f49cb566224a3ff0ef8ab515eadc78ac
+Merge: e832001 f209819
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:37:55 2016 +0700
+
+    Merge pull request #7110 from david-z/wip-hammer-fix-logrotate-ceph-fuse
+    
+    [ceph-fuse] fix ceph-fuse writing to stale log file after log rotation
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit a1fc101084c5aefeb64da344158783a186a28a93
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 8 07:47:23 2015 -0800
+
+    crushtool: send --tree to stdout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dca345280e4f6124a8b5aea73e6219488111c356)
+    
+    Conflicts:
+    	src/tools/crushtool.cc (trivial)
+
+commit 4fb688d15c40a9b9d5a54700b3c7695a97ffd03b
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Tue Mar 3 11:24:50 2015 +0000
+
+    osd: write journal header by force when journal write close
+    
+    osd update the journal header when ceph call FileStore::sync_entry(
+    ApplyManager::commit_finish -> FileJournal::committed_thru).
+    But, it doesnot write the journal head until the next transaction
+    (calling do_write or do_aio_write).
+    
+    So write jouranl header when journal write close, so that
+    committed_up_to in journal header is new enough. ceph may
+    not replay some transaction which has been writen to filestore.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 8f80ae3fbaba7b0e64b6e4df6c2dddd10ac50e5a)
+
+commit 31a2fc475a2de8a656048c85595049ec4548ec6a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 11 10:33:40 2016 -0500
+
+    common/bit_vector: use hard-coded value for block size
+    
+    The CEPH_PAGE_SIZE is not actually a constant. On aarch64 platforms
+    the page size is 64K instead of the assumed 4K.  The CRC block
+    computations require a true constant.
+    
+    Fixes: #14747
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ba2e6ce46fb5f781ffaf6483db8ebd99238419c6)
+
+commit 3352b144e0b1500651d8a194c3ebe2aae2f078d0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 20 18:10:39 2015 -0700
+
+    ceph.in: Notify user that 'tell' can't be used in interactive mode
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a341d977d42a3010267b64e15cebae3085507297)
+    
+    Conflicts:
+    	src/ceph.in (trivial resolution)
+
+commit 14b5feab80f07b4ed2b8f6d77ea1fa920e98be05
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jul 6 12:17:21 2015 +0800
+
+    mon/LogMonitor: use the configured facility if log to syslog
+    
+    Fixes: #13748
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4d2ca37d3b8f067d0cf0f7cfea24964ac181115d)
+
+commit 10d29dab3176ec945dc7272323df0028ef7f0274
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Thu Apr 16 13:42:42 2015 +0800
+
+    os/LevelDBStore:fix bug when compact_on_mount
+    
+    Initialize the performance counter(l_rocksdb_compact)
+    before use it in compact().
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 932d59d084090297c89cfa3ff7df0a952f807bab)
+
+commit d5ba063cdf3a8b92862ef6657001725d6192fe86
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Sun Feb 7 18:50:29 2016 +0800
+
+    OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from
+    
+    Base on current logic, OSDMonitor may call create_pending and
+    encode_pending twice for the some epoch.
+    
+    In encode_pending:
+    
+    tmp.deepish_copy_from(osdmap);
+    tmp.apply_incremental(pending_inc);
+    
+    This Op would change the tmp osd_primary_affinity, but the osd_primary_affinity
+    is declared as ceph::shared_ptr, so this would change the osdmap too. When this
+    round encode_pending is proposed fail. We may call encode_pending again, but the
+    osdmap is changed last round, so the pending_inc would be wrong.
+    
+    Fixes: #14686
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 990b437f4e616a87f4f7438e51945d531170ca83)
+
+commit 9e0a1656200c2f91386215924304b495dd705dae
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 7 08:09:46 2016 -0800
+
+    OSD::consume_map: correctly remove pg shards which are no longer acting
+    
+    A particular shard may be non-active even if the osd is in the acting
+    set.
+    
+    Fixes: 14278
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bd2fdf8c56c1aeed6cea3eb59046dbb6af470111)
+
+commit 5a450e6734b4ff9e5e483c74b67df385b9a3d640
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jan 26 05:56:12 2016 -0500
+
+    mon: add mon_config_key prefix when sync full
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit b77cd31b161e78467bbb6476776c0f1d43e615cf)
+
+commit eb048a393e7b54dcf5f2245e34a1aa0de281424d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 5 09:28:17 2015 -0400
+
+    qa/workunits/post-file.sh: sudo
+    
+    Only root can read the key in /usr/share/...
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 82b0243ec318457e376288a6b32487fbea059705)
+
+commit e9039f4fd3dfe9a3083fcfe64d8012568e8db96f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 28 17:40:20 2015 -0700
+
+    qa/workunits/post-file: pick a dir that's readable by world
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 04b00029e16971abf1937facbbff8da00d6a3452)
+
+commit 902abe7373c150000d3fc4d66d9c989ed208d0bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 09:58:39 2015 -0700
+
+    qa/workunits/post-file.sh: use /etc/default
+    
+    base-files doesn't existing on rhel.  /etc/default is everywhere and it's
+    small.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 52235e32561ef3e91f07c59c8fd89c4df244f461)
+
+commit 1c8c708e43e4c97af17932027ae6b5d26c124c43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 12:06:50 2015 -0500
+
+    librbd: ensure librados callbacks are flushed prior to destroying image
+    
+    Fixes: #14092
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98157ab3274bd960e4487e34f5a83e9c921a6ac8)
+
+commit f892566fc9b78ab38702b75422321a6d0a5bbb7b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 22:31:55 2015 -0400
+
+    librbd: simplify IO flush handling
+    
+    Add a new convenience method to ImageCtx for handling flush
+    requests and cleanup flush handling with dealing with the cache.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e8d8096babaf15fe8af717297d512f580aff6c18)
+
+commit e5dfd3dd7a53bf79f1bfc17b8a4a720aba08d7c3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 14 13:28:13 2015 -0400
+
+    WorkQueue: PointerWQ drain no longer waits for other queues
+    
+    If another (independent) queue was processing, drain could
+    block waiting.  Instead, allow drain to exit quickly if
+    no items are being processed and the queue is empty for
+    the current WQ.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b118d7df1e34387b6e5649a5b205cf061598d0d4)
+
+commit edf60b452de3c6eb22c75be2547f51812e6eaf1e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 2 10:54:53 2016 -0500
+
+    test: new librbd flatten test case
+    
+    AIO operations after a flatten operation were previously
+    hanging during the close of the parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b59158d9f0c3dbe4b47be1dfd27aaa5ad92e6063)
+
+commit 88ffcc2cbd73602819ad653695de7e8718f6707d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 18 17:03:21 2015 +0100
+
+    ceph-disk: use blkid instead of sgdisk -i
+    
+    sgdisk -i 1 /dev/vdb opens /dev/vdb in write mode which indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up) when
+    the device is closed (see below for the udev release note). The
+    implementation of this ioctl by the kernel (even old kernels) removes
+    all partitions and adds them again (similar to what partprobe does
+    explicitly).
+    
+    The side effects of partitions disappearing while ceph-disk is running
+    are devastating.
+    
+    sgdisk is replaced by blkid which only opens the device in read mode and
+    will not trigger this unexpected behavior.
+    
+    The problem does not show on Ubuntu 14.04 because it is running udev <
+    214 but shows on CentOS 7 which is running udev > 214.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+            * As an experimental feature, udev now tries to lock the
+              disk device node (flock(LOCK_SH|LOCK_NB)) while it
+              executes events for the disk or any of its partitions.
+              Applications like partitioning programs can lock the
+              disk device node (flock(LOCK_EX)) and claim temporary
+              device ownership that way; udev will entirely skip all event
+              handling for this disk and its partitions. If the disk
+              was opened for writing, the close will trigger a partition
+              table rescan in udev's "watch" facility, and if needed
+              synthesize "change" events for the disk and all its partitions.
+              This is now unconditionally enabled, and if it turns out to
+              cause major problems, we might turn it on only for specific
+              devices, or might need to disable it entirely. Device Mapper
+              devices are excluded from this logic.
+    
+    http://tracker.ceph.com/issues/14094 Fixes: #14094
+    
+    Signed-off-by: Ilya Dryomov <idryomov@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9dce05a8cdfc564c5162885bbb67a04ad7b95c5a)
+    
+    Conflicts:
+    	src/ceph-disk: keep get_partition_type as it is because
+            some hammer users may rely on the fact that it is able
+            to fallback to sgdisk if blkid is old. Chances are an
+            old blkid also means an old udev that does not have the
+            problem this fix is adressing. The get_partition_uuid
+            is modified to try blkid first, with the same rationale.
+
+commit f209819fb676eadf099cc764e3ed3b20fd17b688
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Feb 1 11:03:30 2016 +0800
+
+    [ceph-fuse] fix ceph-fuse writing to stale log file after log rotation
+    
+    This fix should be applied to hammer branch. It can't be directly applied to master branch, because logrotate.conf is changed on matser since ceph-osd, ceph-mon, etc, is controlled by systemd with user/group as 'ceph' by default, while ceph-fuse might be started as root privilege by external users.
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+
+commit 3d84420847642e1cb4212f943ecf3ed479112d96
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Jan 21 09:42:09 2016 +0300
+
+    PG::activate(): handle unexpected cached_removed_snaps more gracefully
+    
+    PGPool::update(): ditto
+    
+    Fixes: #14428
+    Backport: infernalis, hammer, firefly
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit aba6746b850e9397ff25570f08d0af8847a7162c)
+
+commit ad4df3be0f4adba5e8e14829be724fa3a92263c8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Dec 15 11:28:44 2015 -0500
+
+    rgw: warn on suspicious civetweb frontend parameters
+    
+    Warns about incorrect spacing near 'port' (e.g., 'port = 80').
+    
+    Fixes issue #12038.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 93497e576a946ed7e59f4a4299e447b65aa1774b)
+
+commit 5c92d1d2f11d59651eaa9c4ba6476b9f71990f1c
+Author: Daniel Gryniewicz <dang@redhat.com>
+Date:   Fri Sep 25 11:02:20 2015 -0400
+
+    rgw: Add default quota config
+    
+    Add a default quota config that is applied when no specific quota is
+    enabled.  There are both per-bucket and per-user defaults.  Defaults are
+    enabled if either max count or max size is enabled.
+    
+    This fixes issues 12912 and 12997.
+    
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 3047b5690eebb7fcd2d7a26acd20105e016a44c4)
+
+commit 17d1b0d859112dc9fb5b7a6e2140124790c5f260
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Sun Jan 3 20:12:04 2016 -0500
+
+    rgw: radosgw-admin bucket check --fix not work
+    
+    Fixed:#14215
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit a17f4e27d608ef29cf499fe76246929ec7962783)
+
+commit df3f971eafda9c54881c13dcf47f996f18e17028 (refs/remotes/markhpc/wip-auth-hammer, refs/remotes/gh/wip-auth-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 23 14:58:19 2015 -0800
+
+    auth: return error code from encrypt/decrypt; make error string optional
+    
+    This is simpler for a lot of callers.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 40203f711dd59c7f61f5f8eb56a97842c3206494)
+    
+    Conflicts:
+    	src/test/crypto.cc (trivial)
+
+commit 224bb3928de048b13a96ee0377b6d279ce84ac0a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 27 09:17:28 2015 -0800
+
+    auth: optimize crypto++ key context
+    
+    The cbc appears to be stateful, but the key is not.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7762f1886cab7f7b941851b0be4ec904723cb0e3)
+
+commit f11718de7873fe1fad1649358f208a4910837a7e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 1 15:07:35 2015 -0800
+
+    auth/Crypto: optimize libnss key
+    
+    Allocate the key and slot once per key instead of once per encrypt/
+    decrypt operation.  This improves things by a factor of almost 4x
+    for my microbenchmark.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 973cd1c00a7811e95ff0406a90386f6ead5491c4)
+
+commit d1b6096c79164f8a5830dee6b5f186c405cf73ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 1 15:05:54 2015 -0800
+
+    auth: refactor crypto key context
+    
+    Cache all of the crypto key context in a new CryptoKeyHandler struct that
+    is attached to CryptoKey.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 16b3515af3df439fba7d545587285b50e9814165)
+
+commit 3249f48a07391c98634aebd87f18fc2edbe95ca2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 22 15:49:25 2015 -0800
+
+    auth/cephx: optimize signature check
+    
+    The encode_encrypt() helper will generate a bufferlist with a simple
+    structure.  Profiles seem to indicate this is taking a large amount of time
+    in the message receive path.
+    
+    Avoid the encode overhead since we have a tiny buffer with a fixed and
+    known structure.
+    
+    Reported-by: Andreas Bluemle <andreas.bluemle@itxperts.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8d16d4ce14a82356007c14fb7535170933eb7812)
+
+commit 51eaf98e25287705c9c25d2d3a5df248ba066f2d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 22 15:40:23 2015 -0800
+
+    auth/cephx: move signature calc into helper
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e14a5fedbfacd164639f6a59a4d152404f3f9d1)
+    
+    Conflicts:
+    	src/auth/cephx/CephxSessionHandler.cc (trivial, const changes)
+
+commit c240da9e0693dc627162a0c70569dda519ec25e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:56:39 2015 -0800
+
+    auth/Crypto: avoid memcpy on libnss crypto operation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e972a6951142cbea3fe7c2e08933c808693d60c4)
+
+commit 86cc0f0c5e10262bb494ecee415789fd6815b1ff
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:49:10 2015 -0800
+
+    auth: make CryptoHandler implementations totally private
+    
+    There is no need to expose these implementations outside Crypto.cc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ad5a1549f401af3eae38f5c4de24e7b952e40f7f)
+
+commit f04e0075eabfb1b9d19c75761c321098546c4044
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 15 18:28:29 2015 +0100
+
+    osd: log inconsistent shard sizes
+    
+    When an OSD asserts because it cannot recover from an unexpected erasure
+    coded shard size, the object needs to be manually fixed (i.e. the shard
+    must be removed so it can be reconstructed). Unfortunately the assert
+    does not display the name of the object and it is not displayed unless
+    the OSD logs are >= 10, which is uncommon on an actual cluster because
+    it creates log files that are too big.
+    
+    The problem has been fixed in infernalis as part of
+    https://github.com/ceph/ceph/pull/5173 and backporting it is non
+    trivial.
+    
+    The error condition is modified to make it easy for the system
+    administrator to identify which object is causing the crash and what
+    steps must be taken to fix the issue.
+    
+    http://tracker.ceph.com/issues/14009 Refs: #14009
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit b2393dcea4ca02fe093481d6a2815b0c857fddc9 (refs/remotes/markhpc/hammer-client-inoderef)
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 20 15:42:05 2015 +0800
+
+    client: add InodeRef.h to make dist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit fa79bd978f646001365b14009efe9fa5c562f681
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:54:31 2015 +0800
+
+    client: use smart pointer to track 'cwd' and 'root_parents'
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 8c45ae1239c934b6727071398a1cfccb6277069e)
+
+commit fcafc65e042c0ed913b5b564812180cd29eefb43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:52:10 2015 +0800
+
+    client: convert Inode::snapdir_parent to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 89648e7ed56a1299a096b16080e4a52724893bdd)
+
+commit 78cca4e3b21308cc0a3f4838828a465482f619c7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:39:31 2015 +0800
+
+    client: use smart pointer to track temporary inode reference
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 217837b89f6adabcf686377a9b3d33d5008b1097)
+
+commit bad6f335ccd8d277f31a27f04d10683c73d0037a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:24:41 2015 +0800
+
+    client: convert CapSnap::in to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 4bb48ee4ae49355e3a4bcd7424545c04ce284f89
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:50:10 2015 +0800
+
+    client: convert Fh::inode to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dac11e6b4b1e8766ee5165366cf021d9ee91d3ae)
+
+commit 5bebb3acf1fa85ab301aa8ca4e99542c9d34d173
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:00:50 2015 +0800
+
+    client: use smart pointers in MetaRequest
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fd02f0f245ccdc1d33c6d24f57267c8fb102618b)
+
+commit e18f1aec94ec021a687317bab00c59d3d223c2bc
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 11:21:23 2015 +0800
+
+    client: convert Dentry::inode to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 07f580977319110cbdd60c78fff892c85711ef3e)
+
+commit a7f6d2ffb84dee5bec4e373c1db723805e39a1bb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat May 23 20:01:46 2015 +0800
+
+    client: hold reference for returned inode
+    
+    CLient::make_request() returns pointer of target inode, but it does not
+    increase reference count of the returned inode. The inode may get freed
+    when CLient::make_request() releases MetaRequest.
+    
+    he fix is hold a reference for the returned inode. Since there are many
+    of places use CLient::make_request() directly or indirectly, it's easy
+    to leak reference. This patch uses intrusive_ptr to track the reference.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ad9c22a4ca56d7ad3c1666795bd8c9785eaccf70)
diff --git a/doc/changelog/v0.94.8.txt b/doc/changelog/v0.94.8.txt
new file mode 100644
index 000000000..6191ef9f7
--- /dev/null
+++ b/doc/changelog/v0.94.8.txt
@@ -0,0 +1,2059 @@
+commit 838cd35201e4fe1339e16d987cc33e873524af90 (tag: refs/tags/v0.94.8, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--349f4788-0128-42d9-aca5-78c0aad008e6.localdomain>
+Date:   Fri Aug 26 00:02:49 2016 +0000
+
+    0.94.8
+
+commit eb27865f268eaa4abc4e4b46354f1ea1556d1a5c
+Merge: 08277b7 5248929
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 15:26:49 2016 +0200
+
+    Merge pull request #10750 from ceph/wip-14799-hammer
+    
+    hammer: rocksdb do not link against tcmalloc if it's disabled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5248929c296d6130ee99fe94fbe69151d241efb8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 16 23:45:18 2016 -0700
+
+    rocksdb: disable tcmalloc if disabled
+    
+    the rocksdb/configure.ac does not support --with-tcmalloc before,
+    and the updated rocksdb commit has the change for '--with-tcmalloc'
+    option, so let's pick it up.
+    
+    Fixes: http://tracker.ceph.com/issues/14799
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit fdfcd9b66cda1c637fbaad1c15d2ea04e426210d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 14:12:20 2015 -0400
+
+    ceph.spec: respect CEPH_EXTRA_CONFIGURE_ARGS
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 51abff11688f0201b8f4076ac515e4515929d4cb)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 08277b7bc7c0e533c3fd56a0040dc0ddc74637d6
+Merge: bcab369 d5274a3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sat Aug 13 19:19:37 2016 +0200
+
+    Merge pull request #10718 from oritwas/wip-14328-hammer
+    
+    hammer: rgw: the swift key remains after removing a subuser
+
+commit bcab36992580e5666afbf8e7bada4e850dfc84c6
+Merge: da2960f 699b7c8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sat Aug 13 18:22:19 2016 +0200
+
+    Merge pull request #10509 from yehudasa/wip-15557-hammer
+    
+    rgw: Have a flavor of bucket deletion to bypass GC and to trigger
+
+commit d5274a3aeb7cc40f5ac04ac1ffcd6e3fc3019be3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jan 20 17:24:38 2016 +0100
+
+    rgw: fix subuser rm command failure
+    
+    Fixes: 14375
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bbc06e70b49b2917ee63985727cb92724fb99209)
+
+commit f9637743efc7ba3164e3bb9ee7a768801e1599da
+Author: Sangdi Xu <xu.sangdi@h3c.com>
+Date:   Wed Dec 16 09:09:16 2015 +0800
+
+    rgw: add a method to purge all associate keys when removing a subuser
+    
+    Fixes: #12890
+    
+    When removing a subuser, make sure all of its keys, including the swift key and possible s3 keys, are also deleted.
+    
+    Signed-off-by: Sangdi Xu <xu.sangdi@h3c.com>
+    (cherry picked from commit e7b7e1afc7a81c3f97976f7442fbdc5118b532b5)
+
+commit da2960f974b9a64a683ac8b77faf08fda3a5d0ba
+Merge: 0f67539 3a30ffc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 12 12:23:41 2016 +0200
+
+    Merge pull request #10291 from badone/wip-13829-hammer
+    
+    hammer: config set with negative value results in "error setting 'filestore_merge_threshold' to '-40': (22) Invalid argument"
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0f67539cf5efc46a92d7ea3ac031e0ce1c573a7a
+Merge: 1e01986 0d4b601
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 12 09:46:30 2016 +0200
+
+    Merge pull request #10699 from ktdreyer/wip-15867-nroff
+    
+    jewel: doc: fix by-parttypeuuid in ceph-disk(8) nroff
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d4b6013f0f130c93e2e488d1cd83527ad91274f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Aug 11 17:11:41 2016 -0600
+
+    doc: fix by-parttypeuuid in ceph-disk(8) nroff
+    
+    Commit 221efb0b893adbfd7a19df171cf967fee87afcc7 altered the rST source
+    for the ceph-disk man page. In Hammer, we also have to modify the nroff
+    sources, because static copies of the generated man pages are stored in
+    Git.
+    
+    Fixes: http://tracker.ceph.com/issues/15867
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1e01986e9550700a11adfc1b175eca042c9cb188
+Merge: de28029 187d308
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 11 22:58:33 2016 +0200
+
+    Merge pull request #9741 from SUSE/wip-16343-hammer
+    
+    hammer: boost uuid makes valgrind complain
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit de28029c25e4ff527c6021488381022538331473
+Merge: cb0488c 64211fa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:58:51 2016 +0200
+
+    Merge pull request #10554 from Vicente-Cheng/wip-16815-hammer
+    
+    hammer: radosgw-admin region-map set is not reporting the bucket quota correctly
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cb0488c181a122de13d63686b2a64e1aa00791ca
+Merge: 0bc2d66 a3003f6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:56:39 2016 +0200
+
+    Merge pull request #10555 from Vicente-Cheng/wip-16317-hammer
+    
+    hammer: rgw: bucket listing following object delete is partial
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0bc2d66db5feba6cd887d7c297839b231012b880
+Merge: 5866fef 81aef60
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:55:01 2016 +0200
+
+    Merge pull request #10530 from yehudasa/wip-16412-hammer
+    
+    rgw: remove bucket index objects when deleting the bucket
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5866fefb32cc443549891754a97eb8a6b8dd2af0
+Merge: da4f735 300c111
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 11:14:53 2016 +0200
+
+    Merge pull request #8755 from kmroz/wip-hammer-convert-plain-object-to-versioned
+    
+    hammer: rgw: convert plain object to versioned (with null version) when removing
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit da4f735d7937aa623dfef41f921ad9381b1a07d7
+Merge: 4a12a88 d3eae0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:41:15 2016 +0200
+
+    Merge pull request #9271 from XinzeChi/wip-fix-digest-hammer
+    
+    osd: fix omap digest compare when scrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4a12a88d7098c1f67365a82f3cf9a174dd5ada32
+Merge: 6f107f1 ec02d8b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:38:57 2016 +0200
+
+    Merge pull request #8805 from SUSE/wip-15482-hammer
+    
+    hammer: is_split crash in handle_pg_create
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6f107f1a7a42c070ef2da7f2b0b07d8bd2826429
+Merge: 3b9e73f 3cb72dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:37:24 2016 +0200
+
+    Merge pull request #8593 from renhwsky/wip-15478-hammer
+    
+    hammer: Pool quota alarm is not in effect
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3b9e73f319b28c0403871c7d842304aca74f4c6a
+Merge: d34ac2a b9a4ad9
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:41 2016 +0100
+
+    Merge pull request #7542 from SUSE/wip-14668-hammer
+    
+    hammer: Wrong ceph get mdsmap assertion
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d34ac2a4d76eda4a371cc16dbc6bfdc0223c81d8
+Merge: 8f51f14 3ecdedd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:30 2016 +0100
+
+    Merge pull request #8042 from ukernel/hammer-15050
+    
+    hammer: mds: fix stray purging in 'stripe_count > 1' case
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8f51f146c98e281f065debdfdc4399624e0cce97
+Merge: 4773e9c 049bc8a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:22 2016 +0100
+
+    Merge pull request #8804 from SUSE/wip-15512-hammer
+    
+    hammer: Double decreased the count to trim caps which will cause failing to respond to cache pressure
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4773e9c40849c2796aa16b7cacddbfe99536919c
+Merge: 85a25e4 f6076dd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:09 2016 +0100
+
+    Merge pull request #9404 from SUSE/wip-16082-hammer
+    
+    hammer: mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 85a25e43d4451b999ad517f089593d45895a7c59
+Merge: 3c4d14f 2633ec3
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:30:58 2016 +0100
+
+    Merge pull request #10198 from SUSE/wip-16626-hammer
+    
+    hammer: Failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a3003f6f4f02e37cf96b6e243203d86ad2e39549
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Feb 25 16:02:15 2016 +0100
+
+    rgw: reset return code in when iterating over the bucket the objects
+    
+    Fixes: #14826
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23fdcbbce47aaf041b9f8a0812c8a729aa5014b)
+
+commit 64211fa78f872648aad2b2cec2b4b3cb8f2b9953
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Oct 13 17:04:57 2015 -0700
+
+    rgw: fix compilation
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9dfbae6d65bf6f8ab7cdf8597d556505b81cedd9)
+
+commit 3e45c6bb431f7763a12a48daac30fbe7aa3047cd
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Oct 12 11:14:19 2015 +0200
+
+    rgw: add bucket_quota to RGWRegionMap::encode_json
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e18b72f7e97a65af681e368d3706a465603df89d)
+
+commit 699b7c8d5f51d895f5ff0578de5bf610436ee33b
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Wed Apr 20 14:15:55 2016 +0530
+
+    rgw: Have a flavor of bucket deletion to bypass GC and to trigger
+    object deletions async.
+    
+    Fixes: http://tracker.ceph.com/issues/15557
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit b7a69fca248afeef1de1278890076693b16cf6d3)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_bucket.cc
+    	src/rgw/rgw_bucket.h
+    	src/test/cli/radosgw-admin/help.t
+
+commit 81aef600671f0fe523206cea16000e79bc6bb75c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 4 15:01:51 2016 +0200
+
+    rgw: remove bucket index objects when deleting the bucket
+    
+    Fixes: http://tracker.ceph.com/issues/16412
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3ae276390641ad5fc4fef0c03971db95948880b4)
+
+commit 3c4d14fe73a97c52345369e732e445c78fb656c8
+Merge: 79753ed 8a39e5e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 23:51:47 2016 +0200
+
+    Merge pull request #10182 from gaurav36/wip-configure-xfs-issue-hammer
+    
+    hammer: configure: Add -D_LARGEFILE64_SOURCE to Linux build.
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79753edf9c4763ae15d82502b53c6f3c9cd24349
+Merge: 75271a2 7dbace5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:40:46 2016 +0200
+
+    Merge pull request #7961 from dzafman/wip-15002
+    
+    hammer: objects unfound after repair (fixed by repeering the pg)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 75271a2529b1504d249b528fe3fafd4c5e90a9bc
+Merge: 76fa3b1 43d1b92
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:38:15 2016 +0200
+
+    Merge pull request #9125 from dzafman/wip-15635
+    
+    hammer: [MON] Fixed calculation of %USED
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 76fa3b1097b3711ef75bd36aa2a4074e7a8fa9e6
+Merge: cf5f111 ca0beef
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:36:05 2016 +0200
+
+    Merge pull request #8464 from dzafman/wip-13039-15315
+    
+    hammer: "FAILED assert(pg->info.history.same_interval_since == p.same_interval_since)" in upgrade:hammer-x-jewel-distro-basic-openstack
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit cf5f1117afe9e03c683fd40e5af964eaf5975e2b
+Merge: 76ce775 488a787
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:32:47 2016 +0200
+
+    Merge pull request #8573 from Vicente-Cheng/wip-15293-hammer
+    
+    hammer: ECBackend: send write and read sub ops on behalf of client ops at normal priority
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 76ce775761f755bd3085ba22cf088c3a46b78d3a
+Merge: 8629d37 e20df80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:31:16 2016 +0200
+
+    Merge pull request #8665 from Vicente-Cheng/wip-15525-hammer
+    
+    hammer: ReplicatedPG.cc: 430: FAILED assert(!actingbackfill.empty()): old watch timeout tries to queue repop on replica
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8629d378b05d4f0374436373a0c972360bae7273
+Merge: 79fc401 7b3f1da
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:29:58 2016 +0200
+
+    Merge pull request #8806 from SUSE/wip-15320-hammer
+    
+    hammer: LibRadosMiscConnectFailure.ConnectFailure (not so intermittent) failure in upgrade/hammer-x
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79fc401844c935283f43fbea83e113e298a830ba
+Merge: f4d56b0 cbd5aaf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:28:32 2016 +0200
+
+    Merge pull request #9001 from Vicente-Cheng/wip-15730-hammer
+    
+    hammer: osd: acting_primary not updated on split
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f4d56b03b1decf83bebbf6996909ee89783068f0
+Merge: f71c9e6 410ff15
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:26:55 2016 +0200
+
+    Merge pull request #9400 from Vicente-Cheng/wip-15796-hammer
+    
+    hammer: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f71c9e658d6f5af3cfa48a0dfb9a063f4965e7ce
+Merge: fdb730e 23498a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:24:42 2016 +0200
+
+    Merge pull request #9416 from chardan/wip-15770-hammer
+    
+    hammer: mon/OSDMonistor: improve reweight_by_utilization() logic
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit fdb730e169d8885ccd72e140c31797179c0a2c75
+Merge: f9e8dc8 2c18015
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:21:25 2016 +0200
+
+    Merge pull request #10189 from dreamhost/wip-16618-hammer
+    
+    rgw: fix multi-delete query param parsing.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f9e8dc824a5aa89e24ef12fc8fcfdbdd4f926aef
+Merge: f3cbf5f 24cc4f9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:17:57 2016 +0200
+
+    Merge pull request #8379 from Vicente-Cheng/wip-15283-hammer
+    
+    hammer: RGW shouldn't send Content-Type nor Content-Length for 304 responses
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3cbf5f891029975f4f5a915366fce28fcabd196
+Merge: ba8de3a f28477c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:14:39 2016 +0200
+
+    Merge pull request #8313 from SUSE/wip-15257-hammer
+    
+    hammer: rgw: radosgw server abort when user passed bad parameters to set quota
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ba8de3ab31567c8179b72fb1dc8dbf7ccd1f5f38
+Merge: 9562198 ed4ca7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:13:09 2016 +0200
+
+    Merge pull request #8528 from wido/hammer-issue-15348
+    
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 95621986d893029bc0a4a686c7da5cd6e0fa6f2b
+Merge: 28107d3 1ac920b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:10:57 2016 +0200
+
+    Merge pull request #9671 from ceph/hammer-rgw-copy-if-newer
+    
+    hammer: rgw: backport rgwx-copy-if-newer for radosgw-agent
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 28107d3ebfded3cd9046e5a23ab83d4e0252cce2
+Merge: 5e1a57a 850881c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:09:14 2016 +0200
+
+    Merge pull request #9961 from SUSE/wip-16190-hammer
+    
+    hammer: subuser rm fails with status 125
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5e1a57a6f9c2afbdfc6d3e73d438551202c68229
+Merge: 91889db ce313cd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:07:10 2016 +0200
+
+    Merge pull request #9402 from SUSE/wip-15966-hammer
+    
+    hammer: No Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 23498a9620f792cd099dba028c5bdf96b1a625be
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue May 31 15:40:05 2016 -0700
+
+    mon/OSDMonitor: avoid potential expensive grace calculation
+    
+    The grace calculation during check_failure() is now very complicated
+    and time-consuming. Therefore we shall skip this when it is possible.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 3557903d5d57642179b2ae137bedc389974b1956)
+    
+     Conflicts:
+    	src/mon/OSDMonitor.cc
+    
+    Resolved by choosing the move-to-top implementation. Removed unused vars.
+
+commit 1b6f6f27b77803727a523b4337cbad411e8321ed
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 11:13:32 2016 +0800
+
+    mon/OSDMonitor: improve reweight_by_utilization() logic
+    
+    By calling reweight_by_utilization() method, we are aiming at an evener result
+    of utilization among all osds. To achieve this, we shall decrease weights of
+    osds which are currently overloaded, and try to increase weights of osds which
+    are currently underloaded when it is possible.
+    However, we can't do this all at a time in order to avoid a massive pg migrations
+    between osds. Thus we introduce a max_osds limit to smooth the progress.
+    
+    The problem here is that we have sorted the utilization of all osds in a descending
+    manner and we always try to decrease the weights of the most overloaded osds
+    since they are most likely to encounter a nearfull/full transition soon, but
+    we won't increase the weights from the most underloaded(least utilized by contrast)
+    at the same time, which I think is not quite reasonable.
+    
+    Actually, the best thing would probably be to iterate over teh low and high osds
+    in parallel, and do the ones that are furthest from the average first.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit e7a32534ebc9e27f955ff2d7a8d1db511383301e)
+    
+     Conflicts:
+    	src/mon/OSDMonitor.cc
+    
+    Resolved by picking the lambda implemenation.
+    NOTE: Because hammer does not support C++11, the lambda functionality from the
+    current master has been moved into the "Sorter" function object.
+
+commit 91889db73ab097502615b208c6e9f3b40411e682
+Merge: f3c4f7b b78a1be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 27 08:28:38 2016 +0200
+
+    Merge pull request #10038 from tchaikov/wip-16297-hammer
+    
+    mon: Monitor: validate prefix on handle_command()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3c4f7b61ab519c532a2218a11c5472daaaf5fa9
+Merge: 2fe8edb b31ac2d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 26 19:08:23 2016 +0200
+
+    Merge pull request #9090 from tchaikov/wip-13990-hammer
+    
+    hammer: osd: remove all stale osdmaps in handle_osd_map()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2fe8edb87e7384f206414db8a49e909e51f48de3
+Merge: 67b7f11 0498969
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:44:14 2016 +0200
+
+    Merge pull request #10227 from SUSE/wip-15081-hammer
+    
+    hammer: When logging to a file fails, ceph logs excessively to stderr
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 67b7f11c63313a5dcdf34b904137ef7a4673f4ec
+Merge: bd59c96 474abb8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:43:49 2016 +0200
+
+    Merge pull request #9893 from Vicente-Cheng/wip-16430-hammer
+    
+    hammer: OSDMonitor: drop pg temps from not the current primary
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit bd59c9695fadedef89b77166f60f5a8cf85825ff
+Merge: 3794e3a 055427c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:42:54 2016 +0200
+
+    Merge pull request #9238 from SUSE/wip-15962-hammer
+    
+    hammer: ceph-osd valgrind invalid reads/writes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3794e3ad44d1e429f110abe71fb2d2319567bcd1
+Merge: 89233e3 d96086a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:42:06 2016 +0200
+
+    Merge pull request #8849 from SUSE/wip-14326-hammer
+    
+    hammer: osd/PGLog.h: 88: FAILED assert(rollback_info_trimmed_to == head)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 89233e32ffcdd7f795873771859caba1ce3ad23f
+Merge: 253143b fb1b40f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:40:57 2016 +0200
+
+    Merge pull request #8845 from SUSE/wip-15647-hammer
+    
+    hammer: osd: rados cppool omap to ec pool crashes osd
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 253143b559001eadb73ed9945abe2d0b8d946b65
+Merge: 81133dd 365f21b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jul 25 15:05:54 2016 +0200
+
+    Merge pull request #8470 from dachary/wip-15403-api-test-timeout-hammer
+    
+    hammer: tests: be more generous with test timeout
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 81133dd6e2724ee78cc69bda8e89ca35b20b6b23
+Merge: d71845c 3bb248b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 15:05:31 2016 +0200
+
+    Merge pull request #9977 from shun-s/hammer
+    
+    replcatedBackend: delete one of the repeated op->mark_started in ReplicatedBackend::sub_op_modify_impl
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d71845cec169c660c180a4c0313e0cdbc25329dc
+Merge: 387d5c1 5057c34
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 12:51:45 2016 +0200
+
+    Merge pull request #8960 from badone/wip-hammer-rados-bench-units
+    
+    hammer: Add units to rados bench output and clean up formatting
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 474abb8275cd7507d7c7eae0d0e5647612fc177e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 10:43:17 2016 -0700
+
+    OSDMonitor: drop pg temp from sources other than the current primary
+    
+    Fixes: http://tracker.ceph.com/issues/16127
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1a07123c38e3fecb3fb2e43bbbae962d8411d287)
+    
+    	change:
+                use NULL to replace the nullptr because we don't have C++11
+
+commit b31ac2d37ecb68ed9661923fbaeaedeac241ba4a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 12 20:28:11 2016 +0800
+
+    osd: reset session->osdmap if session is not waiting for a map anymore
+    
+    we should release the osdmap reference once we are done with it,
+    otherwise we might need to wait very long to update that reference with
+    a newer osdmap ref. this appears to be an OSDMap leak: it is held by an
+    quiet OSD::Session forever.
+    
+    the osdmap is not reset in OSD::session_notify_pg_create(), because its
+    only caller is wake_pg_waiters(), which will call
+    dispatch_session_waiting() later. and dispatch_session_waiting() will
+    check the session->osdmap, and will also reset the osdmap if
+    session->waiting_for_pg.empty().
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82b0af7cedc3071cd83ee53479f834c23c62b7d0)
+
+commit 3a30ffc21a5edd52726867302da6b358f93aa8dc
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Jan 27 11:34:47 2016 +1000
+
+    qa: Add test for #13829
+    
+    qa/workunits/cephtool/test.sh: add test for setting negative int options
+    src/test/daemon_config.cc: remove tests for failed negative values
+    
+    Fixes: #13829
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 994ac2942d04584d1617e6d4bbd5b880b1ea0448)
+
+commit f70e4adf8682ed82b267e21e5e63039048893243
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Jan 27 11:18:16 2016 +1000
+
+    common: Allow config set with negative value
+    
+    A recent commit disabled negative values but they are required for variables
+    such as filestore_merge_threshold.
+    
+    Modified patch to remove C++11 specific elements so it will build for hammer
+    
+    Fixes: #13829
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8b777a0c346bc70fd10d07e89368b3785b58f10e)
+
+commit 04989695915fd7ee5ef85ae34ede1fd680514f9d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 09:50:07 2016 -0500
+
+    log: do not repeat errors to stderr
+    
+    If we get an error writing to the log, log it only once to stderr.
+    This avoids generating, say, 72 GB of ENOSPC errors in
+    teuthology.log when /var/log fills up.
+    
+    Fixes: #14616
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d9ac0474b864afda58a44b9012cca4bbc6aaf509)
+    
+    Conflicts:
+    	src/log/Log.cc (drop m_uid and m_gid which are not used in hammer;
+    	order of do_stderr, do_syslog, do_fd conditional blocks is reversed in
+    	hammer; drop irrelevant speed optimization code from
+    	5bfe05aebfefdff9022f0eb990805758e0edb1dc)
+
+commit 2633ec3934ad8f7629c55b4345f426cacfb7b140
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 20:15:14 2016 +0800
+
+    mds: disallow 'open truncate' non-regular inode
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e4b6f2332bb4822cf324587a94144f1c98e4b97)
+    
+    Conflicts:
+    	src/mds/Server.cc (hammer has cur->inode.inline_version - in master
+    	this has been changed to cur->inode.inline_data.version)
+
+commit 3f0fb20ed08dfde614cff6c056d41b59af200c2c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 19:45:23 2016 +0800
+
+    mds: only open non-regular inode with mode FILE_MODE_PIN
+    
+    ceph_atomic_open() in kernel client does lookup and open at the same
+    time. So it can open a symlink inode with mode CEPH_FILE_MODE_WR.
+    Open a symlink inode with mode CEPH_FILE_MODE_WR triggers assertion
+    in Locker::check_inode_max_size();
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4d15eb12298e007744486e28924a6f0ae071bd06)
+
+commit 2c180159c00aa184e1fe8b051b1aa1e9f0f332fc
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Jul 7 14:14:36 2016 -0700
+
+    rgw: fix multi-delete query param parsing.
+    
+    Multi-delete is triggered by a query parameter on POST, but there are
+    multiple valid ways of representing it, and Ceph should accept ANY way
+    that has the query parameter set, regardless of what value or absence of
+    value.
+    
+    This caused the RubyGem aws-sdk-v1 to break, and has been present since
+    multi-delete was first added in commit 0a1f4a97da, for the bobtail
+    release.
+    
+    Fixes: http://tracker.ceph.com/issues/16618
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a7016e1b67e82641f0702fda4eae799e953063e6)
+
+commit 387d5c1ba836833a0cf11ddf9a4fb8690a532878
+Merge: 2fd75cc 4eea92b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 10:16:31 2016 -0700
+
+    Merge pull request #9407 from SUSE/wip-15833-hammer
+    
+    hammer: RGW :: Subset of uploaded objects via radosgw are unretrievable when using erasure coded pool
+
+commit 8a39e5e67bc9a93cadb6c13f25c7ccd57421af6b
+Author: Ira Cooper <ira@redhat.com>
+Date:   Thu Mar 31 12:30:45 2016 -0400
+
+    configure: Add -D_LARGEFILE64_SOURCE to Linux build.
+    
+    Fixes: http://tracker.ceph.com/issues/16612
+    
+    Without -D_LARGEFILE64_SOURCE we can't build against libxfs,
+    because off64_t must be defined.
+    
+    Signed-off-by: Ira Cooper <ira@redhat.com>
+    (cherry picked from commit 602425a)
+
+commit 2fd75cced9dbcfdfdcba979eae90a6f4fb384758
+Merge: dac65d0 2c97cb3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jul 6 20:25:20 2016 +0200
+
+    Merge pull request #8475 from Vicente-Cheng/wip-15343-hammer
+    
+    hammer: Multipart ListPartsResult has missing quotes on ETag
+    Reviewed-bu: Orit Wasserman <owasserm@redhat.com>
+
+commit 3bb248b7d530403b347e3ff68a8996b624fc1c98
+Author: shun-s <song.shun3@zte.com.cn>
+Date:   Tue Jun 28 15:30:16 2016 +0800
+
+    replcatedBackend: delete one useless op->mark_started as there are two in ReplicatedBackend::sub_op_modify_impl
+    delete one mark_start event as there are two same op->mark_started  in ReplicatedBackend::sub_op_modify_impl
+    Fixes: http://tracker.ceph.com/issues/16572
+    
+    Signed-off-by: shun-s <song.shun3@zte.com.cn>
+
+commit ed4ca7c92cdf7404754dab6f9d58c64b647136c5
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Apr 5 11:14:16 2016 +0200
+
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Before this patch the RGW would respond with the Origin send by the client in the request
+    if a wildcard/asterisk was specified as a valid Origin.
+    
+    This patch makes sure we respond with a header like this:
+    
+      Access-Control-Allow-Origin: *
+    
+    This way a resource can be used on different Origins by the same browser and that browser
+    will use the content as the asterisk.
+    
+    We also keep in mind that when Authorization is send by the client different rules apply.
+    In the case of Authorization we may not respond with an Asterisk, but we do have to
+    add the Vary header with 'Origin' as a value to let the browser know that for different
+    Origins it has to perform a new request.
+    
+    More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
+    
+    Fixes: #15348
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 0021e224480c7164330eaa7cc1078bb8795169bf)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+    		hammer still uses s->cio->print() where master uses STREAM_IO(s)->print()
+
+commit b78a1be835706e7dabc505be343945d0ac05697d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 30 13:24:22 2016 +0800
+
+    mon: Monitor: validate prefix on handle_command()
+    
+    Fixes: http://tracker.ceph.com/issues/16297
+    
+    Signed-off-by: You Ji <youji@ebay.com>
+    (cherry picked from commit 7cb3434fed03a5497abfd00bcec7276b70df0654)
+    
+    Conflicts:
+        src/mon/Monitor.cc (the signature of Monitor::reply_command()
+                            changed a little bit in master, so adapt the
+                            commit to work with the old method)
+
+commit dac65d048919f701877de96d3271131853e532ed (refs/remotes/gh/wip-optracker-4354)
+Merge: 2e156d7 a2e7ca1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 27 17:51:19 2016 +0200
+
+    Merge pull request #8538 from Vicente-Cheng/wip-15360-hammer
+    
+    hammer: Linking to -lrbd causes process startup times to balloon
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 850881cf787ccb23eca989861a6688f3a4de1a7c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jan 20 17:24:38 2016 +0100
+
+    rgw: fix subuser rm command failure
+    
+    Fixes: 14375
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bbc06e70b49b2917ee63985727cb92724fb99209)
+    
+    Conflicts:
+    	src/rgw/rgw_user.cc The "if (op_state.will_purge_keys())" block was
+    	later changed to "always purge all associated keys" by
+    	e7b7e1afc7a81c3f97976f7442fbdc5118b532b5 - keep the hammer version
+
+commit 055427cf58857eeadda11ed1b3381e84e384bc85
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 16:57:49 2016 -0700
+
+    Pipe: take a ref to existing while we are waiting
+    
+    Otherwise, if it is reaped while we are waiting, it'll be a
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15870
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b224912d249453d754fc0478d3680f8cfa1a5c22)
+    
+    Conflicts:
+        src/msg/simple/Pipe.cc: nullptr changed to 0 because hammer is not compiled
+        with -std=c++11
+
+commit 24cc4f9e67d9c61f68ccc8f7d80ac227d67aeda1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 15 16:53:16 2016 +0200
+
+    rgw: check for -ERR_NOT_MODIFIED in rgw_rest_s3.cc
+    
+    Fixes: http://tracker.ceph.com/issues/16327
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc38346c596d8b0bc156183970d6a327943cb577)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+    	  use ret instead of op_ret to check op result
+
+commit 7dbace59b83a059fcb364b78fc78d4e38164df6c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 28 22:35:08 2015 +0200
+
+    erasure-code: s/unreocvery/unfound/
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 28820d1cd630ec466ed1069a437d740a025fb9aa)
+
+commit e726f211107cd84cc722744c38213b96dc59cdc7
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Apr 22 09:45:04 2015 +0800
+
+    test: add test-case for repair unrecovery-ec pg.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 8f30db85219a6c57cb2fc03f4049a3f57446ca55)
+
+commit 40b1c2b231b62471d4f3cfba3ba2c3d596c9e170
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Apr 14 13:29:54 2015 +0800
+
+    osd: Remove the duplicated func MissingLoc::get_all_missing.
+    
+    It is the same as MissingLoc::get_needs_recovery.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit bdd6205408cf20e039496728052a36c1a7d758c6)
+
+commit 47d5dfc0323e8d7614ad5793d85c1476b63c18e6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Apr 14 09:11:58 2015 +0800
+
+    osd: Fix ec pg repair endless when met unrecover object.
+    
+    In repair_object, if bad_peer is replica, it don't add soid in
+    MissingLoc for ec pool. If there are more bad replica for ec pool
+    which cause object can't recover, the later recoverying will endless.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d51806f5b330d5f112281fbb95ea6addf994324e)
+
+commit 187d308b5b0cb0512b442bbad1bddfda8ef9203f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 24 14:51:47 2015 -0400
+
+    uuid: use boost::random:random_device
+    
+    The boost mt code uses uninitialized memory for extra randomness,
+    which is a bad idea in general but more importantly makes valgrind
+    unhappy.  Use /dev/urandom instead.
+    
+    Unfortunately this introduces a link time dependency.. meh!
+    
+    Fixes: #12736
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dbcaa544856fcc99ab912a101c4a28e1eb3fb94e)
+    
+    Conflicts:
+    	ceph.spec.in (trivial resolution)
+
+commit 174de7fce8080df6e02b363e7821e8cca6f8157f
+Author: Rohan Mars <code@rohanmars.com>
+Date:   Wed Aug 12 21:19:31 2015 -0700
+
+    moved to use boost uuid implementation, based on commit 4fe89a7b14c97b2ed7f357132901beb2bdcec551
+    
+    Signed-off-by: Rohan Mars <code@rohanmars.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 62bfc7a1ab1587e81ed3bff0ddfbb1aa69d1c299)
+    
+    Conflicts:
+    	debian/control (trivial resolution)
+    	src/common/Makefile.am (trivial resolution)
+    	src/common/blkdev.cc (no get_device_by_uuid() function in hammer)
+
+commit 2e156d7ad4b9f4ffd6028df3a460b50b30c8b0d3
+Merge: 054a90e 5cd922c
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Thu Jun 16 08:26:00 2016 +0300
+
+    Merge pull request #9722 from trociny/wip-16289-hammer
+    
+    hammer: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5cd922ca31d2eb987992b2afcbd45d5d92aeac69
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 14 15:28:06 2016 +0300
+
+    qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Fixes: http://tracker.ceph.com/issues/16289
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3048d3c7ce800a6174c54946b838e4e4079ec609)
+
+commit 1ac920bf291a58cf680108f8dc738197e9bae44d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 18 17:34:51 2015 -0700
+
+    rgw: fix identification of canceled operation
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9f63b15b3a7c4cc633ff90f529934f071aea7c62)
+
+commit a38f157a09097d1eaef64813fc947aac19e3844a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 25 17:35:40 2015 -0700
+
+    rgw: identify racing writes when using copy-if-newer
+    
+    When copying an object from a different zone, and copy-if-newer is
+    specified, if the final meta write is canceled check whether the
+    destinatioin that was created is actually newer than our mtime,
+    otherwise retry.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe9c64b75caf3f53ce2dfdea5e39d794528dfc33)
+
+commit 02f6d8aaa86f4fdb9912763883d079d02589e5be
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 22:48:03 2015 -0700
+
+    rgw: translate http error 304 to appropriate status
+    
+    So that we can return the appropriate response on a conditioinal multi
+    zone copy operation.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20d24e965884af440f94c7e3093587c7b8643cb9)
+
+commit 7319d76b269d541249124f46890f1b04279ba2ea
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 20:24:55 2015 -0700
+
+    rgw: fix if-modified-since boundary
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 27946e8ba4bc5dbfec2da319fa23266d5885d655)
+
+commit 5e4de5ad66dc24b059ef19664d1eef22be6890f7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 20:16:02 2015 -0700
+
+    rgw: add rgwx-copy-if-newer http header
+    
+    Similar to IF_MOD_SINCE, but does not take a time argument, and should
+    protected against races (not there yet).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8813a4401198636e0f4d3a220a0f726625a5c5d4)
+
+commit 054a90edb2812f78426b8cb1dac2e768b2e7fc51
+Merge: 785ffd1 c722d00
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Jun 7 15:48:00 2016 +0200
+
+    Merge pull request #8113 from Vicente-Cheng/wip-15023-hammer
+    
+    hammer: rgw: user quota may not adjust on bucket removal
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 785ffd1f616864909b24314c803d12566c885d14
+Merge: e0a3589 139691c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Jun 6 12:42:14 2016 -0700
+
+    Merge pull request #9277 from ceph/wip-split-lfn-hammer
+    
+    Hammer LFN collection split fix: backport of http://tracker.ceph.com/issues/14766
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e0a358972644d3718c80970fd7236a4df8e61afe
+Merge: 735c524 006ea56
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 6 08:19:23 2016 -0700
+
+    Merge pull request #9521 from ceph/revert-9496-wip-16149-hammer
+    
+    Revert "hammer: Scrub error: 0/1 pinned"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 006ea56f603e160c83c8b1fe35ee8022321bf3b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 6 11:15:54 2016 -0400
+
+    Revert "hammer: Scrub error: 0/1 pinned"
+
+commit 735c52423305d4ad548464e986ff88e57333b8ca
+Merge: a1a5f57 c294bd3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 6 08:46:45 2016 -0400
+
+    Merge pull request #9496 from Vicente-Cheng/wip-16149-hammer
+    
+    hammer: Scrub error: 0/1 pinned
+
+commit a1a5f57558e30fc0fd581756a75da5abee3e897b
+Merge: a336291 d1ab71f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 6 11:39:34 2016 +0200
+
+    Merge pull request #8987 from dreamhost/hammer-wip-pr7712
+    
+    hammer: rgw: S3: set EncodingType in ListBucketResult
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c294bd34e7e29f2c96e3277491c02ea26bf6da5c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 20 13:59:10 2016 -0700
+
+    ReplicatedPG: adjust num_pinned in _delete_oid
+    
+    Fixes: http://tracker.ceph.com/issues/15952
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 907d4e25c676fd3e1c2be90ce8ab5b64b362b0bc)
+
+commit 43d1b92b97e0bb1c419341ba3c740fa632cafef0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 17 17:14:07 2016 -0700
+
+    test: Fix grouping of mon_cephdf_commands by moving to MON_TESTS
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2fb0530a5650bcdb9c89d8bdd2504a7dd1642678)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh (trivial)
+
+commit 300c11179290d9b119ff6f310efd1cf5b559b3e1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 22 18:14:57 2016 -0700
+
+    rgw: convert plain object to versioned (with null version) when removing
+    
+    Fixes #15243
+    
+    When removing a plain null versioned object (was created prior to bucket versioning
+    enabled), need to convert the bucket index representation to a versioned one. This
+    is needed so that all the versioning mechanics play together.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit c6334d430b17739fed2df2b4481ae395ef6568d9)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc
+              - hammer is missing get_zone() API from which log_data can be
+                obtained. Needed to fall back to zone_public_config
+                structure in bucket_index_unlink_instance() definition.
+              - olh_tag string parameter added to
+                bucket_index_unlink_instance() definition.
+            src/rgw/rgw_rados.h
+              - olh_tag string parameter added to
+                bucket_index_unlink_instance() declaration.
+
+commit 4eea92b4a50e06d850c39009d122f8bbbd48c857
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 14:02:25 2016 -0700
+
+    rgw: handle stripe transition when flushing final pending_data_bl
+    
+    Fixes: http://tracker.ceph.com/issues/15745
+    
+    When complete_writing_data() is called, if pending_data_bl is not empty
+    we still need to handle stripe transition correctly. If pending_data_bl
+    has more data that we can allow in current stripe, move to the next one.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7a7d41839558b1a8786a66e27d9d80e606f61c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc (hammer write_data() takes rgw_obj reference as
+    	4th argument, hammer throttle_data() takes rgw_obj as 2nd argument)
+
+commit f6076dd0416dd4464955f208d8964ef136b6142b
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Tue May 3 17:12:26 2016 +0800
+
+    mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 88b6d669e37924536152a552db46ef43a7353562)
+    
+    Conflicts:
+    	src/mds/Server.cc (hammer has req->head.args.open.flags instead of just flags)
+
+commit ce313cd89e29c0cfc6b7c600a82bdea7b351cdbd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 10 17:45:20 2016 +0200
+
+    rgw: handle errors properly during GET on Swift's DLO.
+    
+    Backport: Jewel, Hammer
+    Fixes: http://tracker.ceph.com/issues/15812
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5ef0202a86a1e4cb56bd29d926555f114a1acdd8)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc (hammer uses ret instead of op_ret; enclosing if
+    			   block in hammer does not check skip_manifest)
+
+commit 410ff1528caafaec0cfd9ea6ee4b86ed6788b09d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:09:43 2016 -0400
+
+    osdc/Objecter: upper bound watch_check result
+    
+    This way we always return a safe upper bound on the amount of time
+    since we did a check.  Among other things, this prevents us from
+    returning a value of 0, which is confusing.
+    
+    Fixes: http://tracker.ceph.com/issues/15760
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e2ce365866a7569b55aff338a82c132aa927089)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+    	    By https://github.com/ceph/ceph/pull/8967, simply add 1 to return vaule.
+
+commit d3eae0a01ae8d33988ffccee1f0af145aa10fa33
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Mon May 23 21:38:39 2016 +0800
+
+    osd: fix omap digest compare when scrub
+    
+    Introduce by fe1c28dea4e5607a0c72eab1f046074616cd55a7.
+    
+    Fixes: http://tracker.ceph.com/issues/16000
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+
+commit a33629106ec64acdaba45bf9db18d2402a926b8e
+Merge: b9b5f96 dd29310
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat May 21 14:18:25 2016 -0400
+
+    Merge pull request #9240 from yehudasa/wip-15886-hammer
+    
+    rgw: keep track of written_objs correctly
+
+commit dd29310bd6273ec3643bf90e51f7cae4801629c7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 16 14:35:12 2016 -0700
+
+    rgw: keep track of written_objs correctly
+    
+    Fixes: http://tracker.ceph.com/issues/15886
+    
+    Only add a rados object to the written_objs list if the write
+    was successful. Otherwise if the write will be canceled for some
+    reason, we'd remove an object that we didn't write to. This was
+    a problem in a case where there's multiple writes that went to
+    the same part. The second writer should fail the write, since
+    we do an exclusive write. However, we added the object's name
+    to the written_objs list anyway, which was a real problem when
+    the old processor was disposed (as it was clearing the objects).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8fd74d11682f9d0c9085d2dc445fc3eb5631f6e0)
+
+commit c2ea6dbd652782bb227b7787c486660871b2d950
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 9 15:01:46 2016 +0800
+
+    osd: remove all stale osdmaps in handle_osd_map()
+    
+    in a large cluster, there are better chances that the OSD fails to trim
+    the cached osdmap in a timely manner. and sometimes, it is just unable
+    to keep up with the incoming osdmap if skip_maps, so the osdmap cache
+    can keep building up to over 250GB in size. in this change
+    
+    * publish_superblock() before trimming the osdmaps, so other osdmap
+      consumers of OSDService.superblock won't access the osdmaps being
+      removed.
+    * trim all stale osdmaps in batch of conf->osd_target_transaction_size
+      if skip_maps is true. in my test, it happens when the osd only
+      receives the osdmap from monitor occasionally because the osd happens
+      to be chosen when monitor wants to share a new osdmap with a random
+      osd.
+    * always use dedicated transaction(s) for trimming osdmaps. so even in
+      the normal case where we are able to trim all stale osdmaps in a
+      single batch, a separated transaction is used. we can piggy back
+      the commits for removing maps, but we keep it this way for simplicity.
+    * use std::min() instead MIN() for type safety
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 369db9930887d75b498927da9c97733bff4472b6)
+    
+    Conflicts:
+        remove the C++11ism
+
+commit ac0340ad30329df7919ce82938ad49d0a2e2d160
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Mar 16 21:15:35 2016 +0800
+
+    osd: populate the trim_thru epoch using MOSDMap.oldest_map
+    
+    instead of filling MOSDMap with the local oldest_map, we share
+    the maximum MOSDMap.oldest_map received so far with peers. That
+    way one OSD's failure to trim won't prevent it from sharing with
+    others that they are allowed to trim.
+    
+    Fixes: #13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9789c29fe0f82b236703f7ae4d356251fde3a46f)
+    
+    Conflict: use atomic_t instead of std::atomic<>
+
+commit b9b5f966ddfe7e706c038db204271ee662e0d825
+Merge: b90c097 bb5e015
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 18 17:02:26 2016 -0400
+
+    Merge pull request #9164 from liewegas/wip-osdmapcrc-hammer
+    
+    osd: dump full map bl at 20 when crc doesn't match
+
+commit bb5e015d50cb50361dd6ce258c720fe798b3835a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 6 13:56:38 2015 -0400
+
+    osd: dump full map bl at 20 when crc doesn't match
+    
+    This will help us debug cases where the encoding doesn't match due
+    to a bug.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bfe359af0b80f44ca04847f74d5a2d81097ce4e6)
+
+commit 5057c34ebe4f93792f058ea5f2e3c7e0a29b3d3d
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Mon Mar 23 11:05:44 2015 -0700
+
+    obj_bencher: cosmetic display fixes
+    
+    fix indents in output
+    change `snprintf' to `setprecision'
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit ddb422f40394ec2f4fc4a6a6cff49e43538121a9)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 6d8ad0edf6bae57a6e8ca5cc1a7add1659a92691
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Fri May 6 15:05:42 2016 +1000
+
+    common: Add space between timestamp and "min lat:" in bench output
+    
+    This change is taken from 069d95eaf49cadaa9a8fa1fa186455944a50ec7d
+    but I did not want to cherry-pick that patch since the rest of it
+    is purely cosmetic and would be unlikely to apply cleanly.
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit b90c097ddd3e172df4534509309a554cb3bc3917
+Merge: aefe2cb aab3a40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 16 13:02:44 2016 +0200
+
+    Merge pull request #8535 from asheplyakov/bug-14512-hammer
+    
+    hammer: rbd snap rollback: restore the link to parent
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 31849986896ba6b5fd2b0b13023b50f67842fd28
+Author: Adam Kupczyk <akupczyk@mirantis.com>
+Date:   Wed Mar 2 12:31:01 2016 +0100
+
+    [MON] Fixed calculation of %USED. Now it is shows (space used by all replicas)/(raw space available on OSDs). Before it was (size of pool)/(raw space available on OSDs).
+    
+    Signed-off-by: Adam Kupczyk akupczyk@mirantis.com
+    (cherry picked from commit 71c4e525f27b8efd2aa4f3b5e95f4a13f123d41a)
+
+commit fed256e40f5465959384735be957dec88733aeb7
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Tue Jan 12 22:08:41 2016 +0800
+
+    mon: add a column 'RAW USED' for ceph df detail
+    
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit 2b597ae6d4d53974f28a2da090232577daf09979)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh (trivial)
+    	src/mon/PGMonitor.cc (trivial)
+    	src/mon/PGMonitor.h (trivial)
+
+commit aefe2cb4f262112f0a028797733f868b83da7bd2
+Merge: d56bdf9 221efb0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 12 23:17:51 2016 +0200
+
+    Merge pull request #9107 from SUSE/wip-15867-hammer
+    
+    hammer: ceph-disk s/by-parttype-uuid/by-parttypeuuid/
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 139691c646809f6c25d72c8de064f26b3c036197
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 13:50:58 2016 -0700
+
+    src/test/objectstore/store_test.cc: fix shards for new tests
+    
+    This didn't matter on master/jewel, but in hammer, keyvaluestore
+    asserts that gen != NO_GEN => shard != NO_SHARD.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 221efb0b893adbfd7a19df171cf967fee87afcc7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu May 12 20:06:06 2016 +0200
+
+    doc: s/by-parttype-uuid/by-parttypeuuid/ in ceph-disk
+    
+    Fixes: http://tracker.ceph.com/issues/15867
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 62f4fbe66f8af79fb874125743df0433aa92b731
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 11 17:02:15 2016 -0800
+
+    store_test: improve synthetic coverage of the ec stash case
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f3a78e046d54260de01a4d1db8e104130e35cf42)
+    
+    Conflicts:
+            src/test/objectstore/store_test.cc
+
+commit b6bc9cbaef99670bc0fc55936ab02a3656ec8bfd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 11 14:05:37 2016 -0800
+
+    store_test: improve synthetic test coverage of long object names
+    
+    1/2 of objects will now have long names, and we'll shift the region
+    of the hash space where new objects are created every 1024 objects
+    created.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4b44720f752ebac63454ecd338e2b0c77c8fe501)
+    
+    Conflicts:
+    	src/test/objectstore/store_test.cc
+
+commit ec74c1281e641a45de039f2c168b50902e514729
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 15:38:26 2016 -0800
+
+    TestRados: make half the objects have long names
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8b64432d07a14df9dfd2ed1314783de16530b1d1)
+
+commit 9d1ee7cf390154c5939a01ff439e33d303a952c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 15:19:15 2016 -0800
+
+    LFNIndex::lfn_translate: consider alt attr as well
+    
+    If the file has an alt attr, there are two possible matching
+    ghobjects.  We want to make sure we choose the right one for
+    the short name we have.  If we don't, a split while there are
+    two objects linking to the same inode will result in one of
+    the links being orphaned in the source directory, resulting
+    in #14766.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 850ce11ea3c4dacc6ff1a3ce7ff37000818267c2)
+
+commit 6b821cc8f394e59760e3d58d27f382976bb69e13
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 11:06:06 2016 -0800
+
+    LFNIndex::created: fix return value with failure injection
+    
+    See comment for explanation.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit aec5560887a1c1dbbb5aba822280e2c1e05d5d8f)
+
+commit f500435da3ddf750cf0770c0f63e9cff0cebd7a7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 11:03:13 2016 -0800
+
+    store_test: add reproducer for #14766
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 0fb67c772561aaca9261ac9094b74d7524498f6f)
+    
+    Conflicts:
+            src/test/objectstore/store_test.cc
+
+commit cbd5aafe93a5c85392a460a183862f62af2836b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 15:24:09 2016 -0400
+
+    osd/PG: update info.stats.* mappings on split
+    
+    These are updated in the init and start_peering_interval paths, but not
+    on split.
+    
+    Fixes: http://tracker.ceph.com/issues/15523
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8b42294d98dd70715184991da5ba495fbe93d009)
+
+commit d1ab71ff6084719e86e07a447f3054af73e40868
+Author: Vitja Makarov <vitja.makarov@gmail.com>
+Date:   Wed Feb 17 13:46:18 2016 +0300
+
+    hammer: rgw: S3: set EncodingType in ListBucketResult
+    
+    Signed-off-by: Victor Makarov <vitja.makarov@gmail.com>
+    (cherry picked from commit d2e281d2beb0a49aae0fd939f9387cb2af2692c8)
+    X-Github-PR: 7712
+    Backport: hammer
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit df4eadc2376bb4a53210e2fe176520887b840f2f
+Author: Gu Zhongyan <guzhongyan@360.cn>
+Date:   Wed Dec 30 16:42:07 2015 +0800
+
+    rados: Add units to rados bench output
+    
+    Fixes: #12248
+    Signed-off-by: Gu Zhongyan <guzhongyan@360.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 669b932934d60bbfa751cebd9340f11bd95e2853)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit d96086af5d54a7450f3823210323b9fe08fc075a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 2 17:20:19 2015 -0800
+
+    PGLog::rewind_divergent_log: fix rollback_info_trimmed_to before index()
+    
+    Fixes: #13965
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 053ee919059b2671d40110e8dff8615dc13d34e0)
+
+commit e79162de4cb418576ca633d5c7ea6fb766706962
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 2 17:17:57 2015 -0800
+
+    TestPGLog: add test for 13965
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 66c7246491f4b6e5b95d80cc93c9157725a1a778)
+
+commit fb1b40f5e2da1b70d9ff026a75196c79d09cf0c6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Sun Mar 22 22:07:24 2015 +0800
+
+    osd/Replicated: For CEPH_OSD_OP_WRITE, set data digest.
+    
+    Add two cases which can add data digest for OP_WRITE:
+    a: offset = 0, and length > original size
+    b: offset = original size, and original has data_digest.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit c7702bf85d3617b3e1c6619b8ebeff34932fc3e4)
+
+commit f0242598ea7c990aef2d00f3dc6a0cd65656332c
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 23 09:54:52 2015 +0800
+
+    osd/ReplicatedPG: For obj has omap, it mean have omap data or omap header or have both.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit fc51ce2a837e8e878d46d8ca54531aa7bd5d01b2)
+
+commit 7b3f1da0ab7f60705b6bc802ddb1bfbce10c8738
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 28 10:33:53 2016 -0400
+
+    mon/MonClient: fix shutdown race
+    
+    While we are shutting down, we should drop all subsequent mon
+    messages.  Currently it is possible to finish the authentication
+    sequence *while* we are shutting down, such that a subsequent
+    attempt to authenticate succeeds.
+    
+    Fix this by resetting cur_con early in the sequence, as this
+    makes us drop all incoming messages.
+    
+    Fixes: http://tracker.ceph.com/issues/13992
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b44a348bfe4978446bbc7acf58eb8f60a47c965)
+
+commit ec02d8b7b1610bd7fd00c201fb01ef80d3b9f05b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 12 17:31:27 2016 +0800
+
+    PG: set epoch_created and parent_split_bits for child pg
+    
+    if we have a wrong "created", OSD receiving an osd_pg_create message
+    could fail to project its history and crash. a bad split_bits could
+    mislead the monitor and osd, and hence cause problems.
+    * set child.info.history.epoch_created to osdmap.get_epoch() instead of
+      using the parent's epoch_created
+    * set child.info.stats.parent_split_bits to the calculated split_bits
+      instead using the parent's split_bits
+    
+    Fixes: http://tracker.ceph.com/issues/15426
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8b985561ba08d4ed2d7751a8e09f9d9067c5fbd3)
+
+commit 049bc8a498d4ab516c2daded8ac44699aa72c982
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Thu Jan 14 11:04:42 2016 +0800
+
+    ceph-fuse: double decreased the count to trim caps
+    
+    Signed-off-by: Zhi Zhang zhangz.david@outlook.com
+    (cherry picked from commit d8a3f6d839649243b12cd6f73e6cb8aec1f955c1)
+
+commit e20df8020458a7b7679d0105183f870220ed0ce7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 8 10:35:29 2016 -0400
+
+    osd/ReplicatedPG: make handle_watch_timeout no-op if !active
+    
+    During on_change, we clean up old events on the obcs.  This can
+    include a queued watch timeout:
+    
+     3: (ReplicatedPG::handle_watch_timeout(std::shared_ptr<Watch>)+0x125) [0x7f1fc21fe375]
+     4: (HandleDelayedWatchTimeout::finish(int)+0xd3) [0x7f1fc213e2e3]
+     5: (Context::complete(int)+0x9) [0x7f1fc20ead29]
+     6: (ReplicatedPG::finish_degraded_object(hobject_t const&)+0x354) [0x7f1fc22429e4]
+     7: (ReplicatedPG::on_change(ObjectStore::Transaction*)+0x2ba) [0x7f1fc224353a]
+     8: (PG::start_peering_interval(std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, int, std::vector<int, std::allocator<int> > const&, int, ObjectStore::Transaction*)+0x7bd) [0x7f1fc219a0bd]
+    
+    In this case, handle_watch_timeout should not assume that we are
+    active and primary.
+    
+    Fixes: http://tracker.ceph.com/issues/15391
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b0e39ee11e7bd5079ff6704bc74627d3ba8ba44)
+
+commit 3cb72dd57f901f400d19022269fb098c06318d6b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jun 9 14:57:54 2015 +0200
+
+    mon/OSDMonitor.cc: fix UNINTENDED_INTEGER_DIVISION
+    
+    Fix for:
+    
+    CID 1297885 (#1 of 2): Result is not floating-point (UNINTENDED_INTEGER_DIVISION)
+     integer_division: Dividing integer expressions g_conf->mon_pool_quota_warn_threshold
+     and 100, and then converting the integer quotient to type float. Any remainder,
+     or fractional part of the quotient, is ignored.
+    
+    CID 1297885 (#2 of 2): Result is not floating-point (UNINTENDED_INTEGER_DIVISION)
+     integer_division: Dividing integer expressions g_conf->mon_pool_quota_crit_threshold
+     and 100, and then converting the integer quotient to type float. Any remainder,
+      or fractional part of the quotient, is ignored.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit be7e07a52871e16e8a46accde6171077cca8db5a)
+
+commit aab3a40f95dafab34a7eadc2159d142a5f0f88f1
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Apr 11 14:16:48 2016 +0300
+
+    hammer: rbd snap rollback: restore the link to parent
+    
+    So snapshot, flatten, rollback of a cloned image does not loose any data
+    
+    Fixes: #14512
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 488a78794d800312f79f7a8cd75dfda3bf3d6ed0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jan 8 13:26:39 2016 -0800
+
+    ECBackend: send subop writes and reads at normal priority
+    
+    We want these to be processed ahead of new client ops since
+    there are resources being held.
+    
+    Fixes: 14313
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b1151c4b8b7f22c7ac067da059d087007b0bc8ea)
+
+commit a2e7ca15d6fd0326526b77f443c15067eb1840ac
+Author: Richard W.M. Jones <rjones@redhat.com>
+Date:   Mon Mar 21 14:24:35 2016 +0000
+
+    common/Cycles: Do not initialize Cycles globally.
+    
+    Move initialization to the few tests that actually use it.
+    
+    Fixes: http://tracker.ceph.com/issues/15225
+    Signed-off-by: Richard W.M. Jones <rjones@redhat.com>
+    (cherry picked from commit 1c2831a2c1277c69f9649200d74a75c04a4b0296)
+    
+      Conflicts:
+    	src/test/msgr/perf_msgr_client.cc
+    	src/test/msgr/perf_msgr_server.cc
+    	src/test/perf_local.cc
+                These three files were not introduced on hammer, just remove
+
+commit ca0beef84812b09f1ab224e4796fd44c8fe37989
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 18 10:42:40 2016 -0700
+
+    osd/OSD: fix build_past_intervals_parallel
+    
+    We may be only calculating older past intervals and have a valid
+    history.same_interval_since value, in which case the local
+    same_interval_since value will end at the newest old interval we had to
+    generate.
+    
+    Same as 0830275b39afc408573c2c468fa3b0e7b1d5cc4e change in generate_past_intervals()
+    This was introduced by 70316541bbb115d9a35954bfba373cf1dc084b7e.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0fd674bbf0c17a673be40123645adee3d64375a0)
+
+commit fce79027034bef0cfedbecd9d2601589495f86e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 20 17:48:15 2015 -0700
+
+    osd: When generating past intervals due to an import end at pg epoch
+    
+    Add assert() to make sure same_interval_since isn't too far forward
+    
+    Fixes: #12387
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 65dcc2da76750d0b6dd2cf0031c44f32749f33e5)
+
+commit 2c97cb33c8aa3f53af8a331075a3f79e70113cea
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Mar 31 19:48:38 2016 +0800
+
+    rgw: fix compiling error
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 535eb0e608f3bf6b7c561f277e9977b91ab97b48)
+
+commit 2aa1ea6b505c768323abb99dcc8c8f24e594a898
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Mar 31 06:24:40 2016 +0000
+
+    rgw: Multipart ListPartsResult ETag quotes
+    
+    ListPartsResult output has always missed quotes on the ETag since it was
+    first committed.
+    
+    Fixes: #15334
+    Backports: hammer, infernalis
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a58b774e72cc1613d62e10b25322d6d15e9d2899)
+
+commit 365f21b1181b08e0bb89b916833301228e7c9efc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Apr 6 15:39:23 2016 +0200
+
+    tests: be more generous with test timeout
+    
+    When the thrasher is in action together with a validater (lockdep or
+    valgrind), a single test may hang for more than 360 seconds. Increase to
+    1200: it does not matter if the value is large, only that it prevents
+    the test from hanging forever.
+    
+    Fixes: http://tracker.ceph.com/issues/15403
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit af89474b3fb2c4aa63680aa6b30d71fad2fdd373)
+
+commit c722d00af5286785ad23c574359220e47fd15076
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Tue Feb 9 12:03:24 2016 -0800
+
+    rgw: user quota may not adjust on bucket removal
+    
+    Description:
+    If the user/admin removes a bucket using --force/--purge-objects options with s3cmd/radosgw-admin respectively, the user stats will continue to reflect the deleted objects for quota purposes, and there seems to be no way to reset them. User stats need to be sync'ed prior to bucket removal.
+    
+    Solution:
+    Sync user stats before removing a bucket.
+    
+    Fixes: #14507
+    Signed-off-by: Edward Yang <eyang@us.fujitsu.com>
+    (cherry picked from commit 06507992e3fad837c4667f467d59fc026a2ca0e8)
+    
+    Conflicts:
+    
+    	src/rgw/rgw_op.cc
+                reordering the check seqence and replace some op_ret to ret
+    
+    Backport Change:
+    We remove the `tenant` parameter because this feature is not introduced on hammer version.
+    The rgw multi-tenant feature is introduced on pr#6784 (https://github.com/ceph/ceph/pull/6784)
+    This feature is supported from v10.0.2 and later version.
+
+commit 20f300e2900ff123974312143c10dca59ee1a64a
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Mar 22 11:35:05 2016 +0100
+
+    rgw: Do not send a Content-Length header on a 304 response
+    
+    We tell the client that the content has not changed. If we
+    send a Content-Length header RFC2616 describes that the client
+    MUST use that new value:
+    
+    "If a cache uses a received 304 response to update a cache entry,
+    the cache MUST update the entry to reflect any new field values
+    given in the response."
+    
+    Therefor we should not send a Content-Length header
+    
+    Fixes: #15119
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit fb4e5cc5d6adc82423d294496855f56e43e8c11d)
+
+commit e53751d0e52f6fb611d9bd81fe2115cd7ce533b3
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Mar 21 16:50:49 2016 +0100
+
+    rgw: Do not send a Content-Type on a '304 Not Modified' response
+    
+    When we say the Content has not changed we should not respond
+    with a content type which defaults to binary/octet stream.
+    
+    Fixes: #15119
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 471fa29e643fcfd5e1383fb987f99b5b21038ed7)
+
+commit 19dbc2598d15d0676017abe4fb75f7ddc3248119
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 18 10:18:22 2015 -0800
+
+    rgw: dump_status() uses integer
+    
+    so that we can avoid atoi() later
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 744a29a34b72d2a709a58a5ea5fb6223a39c676a)
+    
+      Conflicts:
+    	src/rgw/rgw_rest.cc
+            rename the input parameter from `err` to `http_ret`
+
+commit c79b481311b3cd992794abc6e2f194f350d43531
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 18 10:07:27 2015 -0800
+
+    rgw: move status_num initialization into constructor
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2a12ffc3d7f98504ece8906a6fcb5dac8fd81aba)
+
+commit ceb8e19096f7d4ef35ed3ad8fad508aecbfa78ef
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Nov 11 19:01:15 2015 +0100
+
+    rgw: Do not send a Content-Length header on status 204
+    
+    RFC7230 says:
+      A server MUST NOT send a Content-Length header field in any response with a
+      status code of 1xx (Informational) or 204 (No Content).
+    
+    Fixes: #13582
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 4e5921dbc7d1c51feb4cc5c03aa59a432742765e)
+
+commit 3ecdedd5cfe67d85530123ed6684688b43910aa6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Mar 11 11:13:43 2016 +0800
+
+    mds: fix stray purging in 'stripe_count > 1' case
+    
+    Fixes: #15050
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 99ec183973e0f4686673d93df02e5381543be51f)
+
+commit f28477c9cf904e9b27e6ad6bb38fbeeb80db5017
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Mon Dec 28 18:55:37 2015 +0800
+
+    rgw: do not abort when user passed bad parameters to set quota
+    
+    when user passes bad parameter(e.g. pass "null" to bool parameter), the ceph_json.cc::decode_json_obj()
+    will raise a exception with type of JSONDecoder::err(). If we do not catch this exception, the radosgw
+    server will abort.
+    
+    Fixes: #14191
+    
+    $ curl -X PUT http://ceph1:9090/admin/user?quota&uid=test5&quota-type=user -d '{"enabled": "null"}'
+    
+     0> 2015-12-28 11:25:20.864091 7f0d46fb5700 -1 *** Caught signal (Aborted) **
+     in thread 7f0d46fb5700
+    
+     ceph version 10.0.1-787-gc485d29 (c485d29a53f6a2c1a3b561c54632dbac1c4e9473)
+     1: (ceph::BackTrace::BackTrace(int)+0x2d) [0x7f0e13ef88c5]
+     2: (()+0x8fbfb7) [0x7f0e13ef7fb7]
+     3: (()+0xf100) [0x7f0e082c3100]
+     4: (gsignal()+0x37) [0x7f0e072fd5f7]
+     5: (abort()+0x148) [0x7f0e072fece8]
+     6: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7f0e07c019b5]
+     7: (()+0x5e926) [0x7f0e07bff926]
+     8: (()+0x5e953) [0x7f0e07bff953]
+     9: (()+0x5eb73) [0x7f0e07bffb73]
+     10: (bool JSONDecoder::decode_json<bool>(char const*, bool&, JSONObj*, bool)+0x288) [0x7f0e13d74c2c]
+     11: (RGWQuotaInfo::decode_json(JSONObj*)+0x6c) [0x7f0e13d71128]
+     12: (void decode_json_obj<RGWQuotaInfo>(RGWQuotaInfo&, JSONObj*)+0x23) [0x7f0e13d21f16]
+     13: (int rgw_rest_get_json_input<RGWQuotaInfo>(CephContext*, req_state*, RGWQuotaInfo&, int, bool*)+0x11e) [0x7f0e13d21e48]
+     14: (RGWOp_Quota_Set::execute()+0x8c9) [0x7f0e13d1ed6d]
+     15: (()+0x7532d2) [0x7f0e13d4f2d2
+    
+    Reported-by: Coffee Chou <coffee.zyr@gmail.com>
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 59f9058467480d21bd98e0e90aa97313a92478a1)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+                hammer version lacks rgw_rest_get_json_input_keep_data()
+                template
+
+commit 9786394b2f75d883505125bc45c95fb685b96531
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Mon Dec 28 18:14:22 2015 +0800
+
+    rgw: do not abort when user passed bad parameters to set metadata
+    
+    Fixes: #14190
+    
+    when user passes bad parameter(e.g. pass "null" to bool parameter), the ceph_json.cc::decode_json_obj()
+    will raise a exception with type of JSONDecoder::err(). If we do not catch this exception, the radosgw
+    server will abort.
+    
+    $ curl -X PUT http://ceph1:9090/admin/metadata?user&format=json -d 'data = { "mtime": "null"}'
+    
+    terminate called after throwing an instance of 'JSONDecoder::err'
+    *** Caught signal (Aborted) **
+     in thread 7ff982f55700
+     ceph version 10.0.1-787-gc485d29 (c485d29a53f6a2c1a3b561c54632dbac1c4e9473)
+     1: (ceph::BackTrace::BackTrace(int)+0x2d) [0x7ffa814db8c5]
+     2: (()+0x8fbfb7) [0x7ffa814dafb7]
+     3: (()+0xf100) [0x7ffa758a6100]
+     4: (gsignal()+0x37) [0x7ffa748e05f7]
+     5: (abort()+0x148) [0x7ffa748e1ce8]
+     6: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7ffa751e49b5]
+     7: (()+0x5e926) [0x7ffa751e2926]
+     8: (()+0x5e953) [0x7ffa751e2953]
+     9: (()+0x5eb73) [0x7ffa751e2b73]
+     10: (bool JSONDecoder::decode_json<long>(char const*, long&, JSONObj*, bool)+0x28c) [0x7ffa8135920a]
+     11: (RGWMetadataManager::put(std::string&, ceph::buffer::list&, RGWMetadataHandler::sync_type_t, obj_version*)+0x173) [0x7ffa81468029]
+     12: (RGWOp_Metadata_Put::execute()+0x172) [0x7ffa81308dcc]
+     13: (()+0x7532d2) [0x7ffa813322d2]
+     14: (RGWFCGXProcess::handle_request(RGWRequest*)+0x90) [0x7ffa813328a4]
+    
+    Reported-by: Coffee Chou <coffee.zyr@gmail.com>
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 83503239c81573232429a5fe98381f1a8f1357e9)
+    
+    Conflicts:
+    	src/rgw/rgw_user.cc
+                hammer version uses RGWUserInfo instead of RGWUserCompleteInfo
+
+commit b9a4ad9da99d82bd247e5d39108840f19535fa5e
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Fri Mar 27 18:49:28 2015 +0800
+
+    Fixed the ceph get mdsmap assertion.
+    
+        When we want to get mdsmap, we try to get_version()
+        and the return value err = 0 means success.
+    
+        The assert verified r == 0. r would not change in this flow.
+        It always meet assert and lead mon failure.
+    
+        I think this verify should be:
+            assert(err == 0)
+        It will help to check return value of get_version().
+    
+    If you have any questions, feel free to let me know.
+    Thanks!
+    
+    Signed-off-by: Vicente Cheng <freeze.bilsted@gmail.com>
+    (cherry picked from commit f4398d2e6c245e3f81a6038425e1b8372b265b8c)
diff --git a/doc/changelog/v0.94.9.txt b/doc/changelog/v0.94.9.txt
new file mode 100644
index 000000000..52d883d44
--- /dev/null
+++ b/doc/changelog/v0.94.9.txt
@@ -0,0 +1,72 @@
+commit fe6d859066244b97b24f09d46552afc2071e6f90 (tag: refs/tags/v0.94.9, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--d6d4765c-3c95-4fe1-a8cb-776094e0d416.localdomain>
+Date:   Tue Aug 30 11:33:34 2016 +0000
+
+    0.94.9
+
+commit d51d10407459bd337f674c6fb0bab1bf55e30c1e
+Merge: b97a974 27d8055
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Aug 30 07:12:09 2016 -0400
+
+    Merge pull request #10913 from ceph/wip-revert-random-uuid-hammer
+    
+    hammer: revert use of boost for uuids
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b97a974878881dc5963c0c8a78a9f83fdd31d5f5
+Merge: 838cd35 a219cf5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 30 03:03:57 2016 +0200
+
+    Merge pull request #10898 from SUSE/wip-release-notes-hammer
+    
+    doc: release-notes: add missing hammer releases
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 27d805589552b63a2a152335e4932f996e979d86
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 29 15:29:29 2016 -0400
+
+    Revert "moved to use boost uuid implementation, based on commit 4fe89a7b14c97b2ed7f357132901beb2bdcec551"
+    
+    This reverts commit 174de7fce8080df6e02b363e7821e8cca6f8157f.
+
+commit 21f6f1da0764e1a003a43ef73d6788bdfc3ef848 (refs/remotes/gh/wip-revert-random-uuid-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 29 11:55:24 2016 -0400
+
+    Revert "uuid: use boost::random:random_device"
+    
+    This reverts commit 187d308b5b0cb0512b442bbad1bddfda8ef9203f.
+    
+    This breaks precise builds.
+
+commit a219cf5348b5a5d7b013f4dc1cb503d33d609f12
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Aug 28 09:27:00 2016 +0200
+
+    doc: release-notes.rst: add missing line to v0.94.8
+    
+    see 46264a3b6cb7f490680c79bf0957c36aeaea5f6c and
+    b83d25d2918cbd880fbaeec976bffcea8868d5d9 in master
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a6ba10174559d100341e3684e08d74c6de2e79c7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 27 12:50:09 2016 +0200
+
+    doc: add missing changelogs up to 0.94.8
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3dad334fa3147490c8aa94323bee16dd2449681
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 27 09:28:48 2016 +0200
+
+    doc: release-notes: add missing hammer releases
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
diff --git a/doc/changelog/v10.2.1.txt b/doc/changelog/v10.2.1.txt
new file mode 100644
index 000000000..5646e8a1a
--- /dev/null
+++ b/doc/changelog/v10.2.1.txt
@@ -0,0 +1,2249 @@
+commit 3a66dd4f30852819c1bdaa8ec23c795d4ad77269 (tag: refs/tags/v10.2.1, refs/remotes/gh/jewel)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--11a52675-9585-4db4-a514-798db40d6da2.localdomain>
+Date:   Thu May 12 21:48:00 2016 +0000
+
+    10.2.1
+
+commit d93a90ec2743c9aec229891c3b0da72b038cffad
+Merge: cb35d7c 466b7fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 12 17:30:14 2016 +0200
+
+    Merge pull request #8853 from ceph/wip-jewel-backports
+    
+    jewel: several backports
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cb35d7c25bb1df2a8eff01432d27c5d3bbb196af
+Merge: 0e1b633 b6ebb25
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu May 12 16:50:33 2016 +0200
+
+    Merge pull request #9041 from vshankar/jewel-15721-backport
+    
+    jewel: rbd: helpful error message on map failure
+    
+    Reviewed-by: Ilya Dryomov <idryomov@gmail.com>
+
+commit 0e1b6336b60af6bceaf9dbc8ce544fcb7af71961
+Merge: bfdf709 867f798
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 12:33:50 2016 -0700
+
+    Merge pull request #9081 from yehudasa/wip-rgw-period-commit-jewel
+    
+    rgw: period commit fix
+
+commit bfdf709dff9bd3d60d11950d4b2d8741ad0cb113
+Merge: bf1474f 9d5162f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:24:23 2016 -0700
+
+    Merge pull request #8969 from Abhishekvrshny/wip-15731-jewel
+    
+    jewel: MDSAuthCap parse no longer fails on paths with hyphens
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit bf1474f86ce1a102d15c54ee0e1b11005fe48191
+Merge: 7a37155 52ca195
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:23:28 2016 -0700
+
+    Merge pull request #8970 from Abhishekvrshny/wip-15732-jewel
+    
+    jewel: MDS incarnation no longer gets lost after remove filesystem
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7a37155d798aebc688fb4a06056dc4761062b098
+Merge: e83764c e6aa453
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:20:03 2016 -0700
+
+    Merge pull request #8971 from Abhishekvrshny/wip-15738-jewel
+    
+    jewel: handle standby-replay nodes properly in upgrades
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit e83764c03b9e88dd2a8ef848269768ffb8b6a703
+Merge: 4916950 dccb5ef
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:30 2016 -0700
+
+    Merge pull request #9054 from yehudasa/wip-rgw-admin-output-jewel
+    
+    jewel: rgw admin output
+
+commit 49169501a922034c2686dd88365d438e12605116
+Merge: aa24358 f071d8c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:17 2016 -0700
+
+    Merge pull request #9053 from yehudasa/wip-15745-jewel
+    
+    jewel: rgw: handle stripe transition when flushing final pending_data_bl
+
+commit aa243585508eedf87b2e9e558cc63c014f84b3b0
+Merge: f88e2df 9bb17db
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:02 2016 -0700
+
+    Merge pull request #9047 from ceph/wip-rgw-period-delete-jewel
+    
+    jewel: rgw: period delete fixes
+
+commit f88e2df881c045f52f78516aa08def8569be6ecf
+Merge: a614858 a496b70
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 11 12:30:18 2016 -0400
+
+    Merge pull request #9076 from dillaman/wip-cmake-test-rbd-mirror-jewel
+    
+    jewel: cmake: fix rbd compile errors
+
+commit a496b705ed48da187b341392530f7e8579637310
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 22:37:01 2016 -0400
+
+    cmake: fix rbd compile errors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d4c00d1bec431f8c721195c81684c4095079c1d)
+
+commit 9a46e13f3ca9915d71f01bb3506818db59c31a7b
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 28 10:22:07 2016 +0800
+
+    cmake: add library cls_journal for target unittest_librbd
+    
+    otherwise if we want to generate target unittest_librbd we need to
+    (re)generate cls_journal manually
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 7efb29416b789fcb7e68aab554d6033e6b5ebb43)
+
+commit a6148582432ff36110145ba642d2e0f38e5872b3
+Merge: 7a4810d 48b732f
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed May 11 16:50:41 2016 +0800
+
+    Merge pull request #9063 from dachary/wip-15838-jewel
+    
+    jewel: tests: make check fails on ext4
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 48b732fbfac1fa6af10bec0ef900029b71204c26
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon May 2 10:42:24 2016 +0200
+
+    tests: enable make check on ext4 (part 2)
+    
+    Followup of 475cc08c330e639b226f2ab57dedbe4495f63022
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit b0b3a068d4fcc7b739c5dd3d68297e2d517c378e)
+
+commit e1c67d4684c485711b2b9af02edabe161f3d7779
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Apr 30 17:12:05 2016 -0400
+
+    qa/workunits/ceph-helpers.sh: make ceph-osd behave on ext4
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 475cc08c330e639b226f2ab57dedbe4495f63022)
+
+commit 867f798d9c364ab5400a61d83cbaef0da9a57b27
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 10 15:51:27 2016 -0400
+
+    radosgw-admin: 'period commit' sends to new master zone by default
+    
+    if no --remote or --url is given, set --remote = master_zone_id to
+    override the rest_master_conn (which refers to the old master zone)
+    
+    Fixes: http://tracker.ceph.com/issues/15828
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fbe1a05355549dfe0c05750683eeb61e275950c2)
+
+commit a6a6c13c751bbb74ba4f12dc593cd054aabed87c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 10 15:48:33 2016 -0400
+
+    radosgw-admin: allow --remote to specify zone or zonegroup id
+    
+    --remote was only looking for zonegroup connections. extended to check
+    for zone connections as well
+    
+    updated the usage to clarify that it expects zone/zonegroup ids
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9fcff89ee1e60009fa367e33c915a90354c9a9cc)
+
+commit dccb5efb128233affaa544b43f93aa4b3d557259
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:59:49 2016 -0400
+
+    radosgw-admin: 'zonegroup add' prints updated zonegroup
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d355208023fe08a7e16f1078c872286048c4bcc6)
+
+commit bd6b5fc5d49c08f574b84a2705a8de4a1d5f8ed1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:59:32 2016 -0400
+
+    radosgw-admin: 'zonegroup modify' prints updated zonegroup
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a033f40caf60083111886aa804fcdbf3ee60aef8)
+
+commit c1e92df545d0a8db64c84879bc09023c98783fe4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:56:47 2016 -0400
+
+    radosgw-admin: 'zone modify' prints updated zone
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a639014a6f2dd950062aa405bbee1762ca23e09)
+
+commit f071d8c6885786ac5fd5400381369945704abaa8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 14:02:25 2016 -0700
+
+    rgw: handle stripe transition when flushing final pending_data_bl
+    
+    Fixes: http://tracker.ceph.com/issues/15745
+    
+    When complete_writing_data() is called, if pending_data_bl is not empty
+    we still need to handle stripe transition correctly. If pending_data_bl
+    has more data that we can allow in current stripe, move to the next one.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7a7d41839558b1a8786a66e27d9d80e606f61c7)
+
+commit 7a4810d1f51c072f6406a7306743ad7577476c76
+Merge: 67809b8 10b393b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 10 11:14:40 2016 -0700
+
+    Merge pull request #9052 from dillaman/wip-cmake-rbd-mirror-replay-status-jewel
+    
+    jewel: cmake: add missing source file to rbd_mirror/image_replayer
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 10b393bd90ed76ad41d2123aae5700e171a8e9bc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 29 17:33:03 2016 -0400
+
+    cmake: add missing source file to rbd_mirror/image_replayer
+    
+    fixes an undefined reference when linking librbd_mirror_internal.a
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2f3f56170ab47fc10d4232101ae2e85398a4c299)
+
+commit 67809b8e3d71b3cba5ce91e542e840c099c927cd
+Merge: e15854c 8bb4c7f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 10 14:10:05 2016 -0400
+
+    Merge pull request #9051 from athanatos/wip-15815-jewel
+    
+    Wip 15815 jewel
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8bb4c7f92ca784a116d0f7a9a22677b514e7e6ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 22 16:38:14 2016 -0400
+
+    ceph_test_keyvaluedb: clean up test dir
+    
+    Just like ceph_test_objectstore
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b075aa52d4475338c0f3cd3360e1ee4de8befa7)
+
+commit 108cdfdff65bcce98cb3b2da799fe46bed4421f4
+Author: Allen Samuels <allen.samuels@sandisk.com>
+Date:   Fri Apr 22 14:06:56 2016 -0400
+
+    ceph_test_keyvaluedb: fixup unit test for keyvaluedb
+    
+    Fixes: http://tracker.ceph.com/issues/15435
+    Signed-off-by: Allen Samuels <allen.samuels@sandisk.com>
+    (cherry picked from commit 85c14de20936ff9ee224f9b6569a73f9bae75f19)
+
+commit e15854ce98291b3186e9cfadb6cb47a401a4c8ae
+Merge: 7686c62 815967a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:52:54 2016 -0400
+
+    Merge pull request #9050 from dillaman/wip-15821
+    
+    jewel: rbd-mirror: image sync needs to handle snapshot size and protection status
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 815967ab97631f65c44cd200971cc8056e94d89e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 18:25:39 2016 -0400
+
+    test: add missing header dependency for MockJournaler
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fa91919c856ac8186545bc8f8da31a8f438a9b5d)
+
+commit 32189573c2da9e1922b6bfae232a62a909797e1a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 10:23:49 2016 -0400
+
+    rbd-mirror: image sync object copy was not properly mapping snapshots
+    
+    When the snapshot sequence is out-of-sync between remote and local
+    clusters (expected), the objects would not be written with the
+    correct snapshot context.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dd8f08039bf5354a0413c9da4263d1075243e9c9)
+
+commit d3dc919991a013eb45d55a8b6a04628d089d000a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 10:23:25 2016 -0400
+
+    rbd-mirror: clean up image sync debug messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit acca475fe5c9ff024c7f76397e7f49e4c4558b09)
+
+commit 58e0612a7bb66f6c50dcfc27ed0f1e49d6ec08fa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 00:12:05 2016 -0400
+
+    librbd: readahead should not read past end of image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit adb14303ccf8a26ed5536bb692ae63d92ac40a9f)
+
+commit 7dadecb96a4f856d18fcccffb323941b1149ceaa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:17:12 2016 -0400
+
+    test: randomize rbd-mirror test case IO
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d75f2ce61cee164da4e5b7ffb4e69c286e6036a)
+
+commit 7aef2976084b8be858e354a05dae12df30743737
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:15:58 2016 -0400
+
+    rbd-mirror: don't set object map to nonexistent during image sync
+    
+    The object maps are initialized to nonexistent at snapshot creation,
+    so this operation is unnecessary.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fe5d4a811f9d9d1160ff7104fe9243cc17ef1bef)
+
+commit 58e90f1bcb53a38be2491ac7797f170de324af9a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:01:30 2016 -0400
+
+    librbd: improve object map debug messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c657752a5067e11405dca59461ce9cbdf062602)
+
+commit 6c1af50b67e1a1769bb3d30d28e45fee8b13bb71
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 22:53:52 2016 -0400
+
+    test: snapshot stress test rbd-mirror image sync
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 468287bd182fa0cd02f7551ad6d729ff67ab1a22)
+
+commit 0f84aaf011f1a62250948647069cc7fbd6c45699
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 21:42:05 2016 -0400
+
+    rbd-mirror: allocate empty object map during snapshot create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57a7fe0f44c5542a72120d0beed8b38e9ccd66db)
+
+commit fa8a61df1570893e222c0d9431ba61e192a23c40
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 29 16:43:07 2016 -0400
+
+    librbd: optionally skip object map creation upon snapshot
+    
+    This feature is used by rbd-mirror during image bootstrap since
+    it will directly create an empty object map.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db16240c79be7e28d65edac2088352b9b146da5f)
+
+commit a140753b6b01215f832cc9524e3532e825e4c2f2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 25 15:11:28 2016 -0400
+
+    rbd-mirror: refactor snapshot create into its own state machine
+    
+    Initial implementation will resize the image (if needed) and create
+    the snapshot.  It is stubbed out for future state machines for handling
+    object map and clones.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8adc47fbf8efa3991bd247be43b786676fe5fff9)
+
+commit ef1e4c874adba51bc5a3acbbf3177174f25da088
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 14 21:47:19 2016 -0400
+
+    rbd-mirror: sync snapshot protection status during bootstrap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6fba2b2306c8f04ce088fb15ef70e1dac282ff3)
+
+commit c790da5aef0cb6a5573d86d0a8298d850b63ef25
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Apr 9 13:30:56 2016 -0400
+
+    test: stub test cases for rbd-mirror image bootstrap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d0f732877213ba9eec9d3158b839c5a4c796e7ab)
+
+commit 7a7889572789e1bec9cb72e08a0d66d8302d7639
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 5 00:03:12 2016 -0400
+
+    test: share journal::MockJournaler between librbd and rbd-mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 04db7ddec661ef195077c3fab06f248705b17220)
+    
+    Conflicts:
+    	src/test/librbd/test_mock_Journal.cc: MockJournaler moved to new file
+
+commit 7686c628b199dd2f3f58b70e7ab3bbb9c1f199c1
+Merge: 63d2d23 f221fed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:40:32 2016 -0400
+
+    Merge pull request #9040 from dillaman/wip-15818
+    
+    jewel: librbd: potential concurrent event processing during journal replay
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 63d2d2396d80fd7d9fd439488d08e8747846d181
+Merge: 1bb1a3a f18b14b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:39:46 2016 -0400
+
+    Merge pull request #9038 from dillaman/wip-15817
+    
+    jewel: journal: incorrectly computed object offset within set
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1bb1a3a4862e5bb5a99c7549d499564deecbdafe
+Merge: abbe69e 8605040
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:39:06 2016 -0400
+
+    Merge pull request #9039 from dillaman/wip-15820
+    
+    jewel: librbd: client-side handling for incompatible object map sizes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 86050402066d1f5d680ec777826386a5760e3456
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 17:00:59 2016 -0400
+
+    librbd: guard object map against incompatible image sizes
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 42d3d3ac2d33360aae071330cfa34b87c87eadc5)
+
+commit 7448d29027dafe1abd1e6ebff6f0fe5bb15e4b07
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 15:16:17 2016 -0400
+
+    librbd: properly handle object map open returning error code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 570de56bfc7fd2edd5cf1cf5b7b92084cd3352a3)
+
+commit 764b3bd413b81d2c98f8c325100661ee41a571bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 15:01:58 2016 -0400
+
+    librbd: add client-side memory constraint when loading object map
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2dd34b2c7b8c376817aafe5cfdda7aa9cfb60e1a)
+
+commit abbe69e19af6d453095e03adbaaaef680ed80958
+Merge: 05d5749 5ec1f79
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:30:49 2016 -0400
+
+    Merge pull request #9048 from dillaman/wip-15819
+    
+    jewel: librbd: constrain size of AioWriteEvent journal entries
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5ec1f79086e4841aa27c12531a1220f69ed3cccb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 00:08:38 2016 -0400
+
+    librbd: split large AioWrite journal events
+    
+    Ensure that AioWrite journal events fit within a single
+    journal object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 70b9fbf2745c795fe345f0298a2f30af80a12985)
+
+commit f1a2e6aeaa22233f13878d004cbe905a42bdc744
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 5 21:30:18 2016 -0400
+
+    journal: prohibit journal entries larger than the soft object size
+    
+    This avoids complications like a librbd write event of 128MB trying
+    to fit within a 24MB journal object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5f307dec10429a1ffc042919b71ac25466a1b72)
+
+commit 05d574926df94b4fa2e0186f31d5050bd0b39ed3
+Merge: 495313f 2d906e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:28:11 2016 -0400
+
+    Merge pull request #9036 from dillaman/wip-15814
+    
+    jewel: librbd: fix get/list mirror image status API
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2d906e600a8d81aae15bcf3f8b2fc694537890d4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 10 12:29:12 2016 +0300
+
+    rbd-mirror: fixup to get/list mirror image status API
+    
+    - embed name and rbd_mirror_image_info_t into rbd_mirror_image_status_t;
+    - index image status list by image id to make 'start' work correctly;
+    - provide rbd_mirror_image_status_list_cleanup function.
+    
+    Fixes: #15771
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e6c709b3a8be489a900178b8b0d962705d5687e9)
+
+commit 495313f355955463f747be9e6650fe8796c0d124
+Merge: 9750610 617004e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:15:23 2016 -0400
+
+    Merge pull request #9003 from dillaman/wip-15743
+    
+    jewel: librbd: reduce log level for image format 1 warning
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9750610ebf7f4d853dee23adaf6109c2c6b63ee0
+Merge: 0218f0b fb7eb4e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:13:27 2016 -0400
+
+    Merge pull request #9010 from dillaman/wip-15736
+    
+    jewel: rbd-mirror: admin socket commands to start/stop/restart mirroring
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fb7eb4e6548b6536fccb6fd38b18a5ced0f9ee4b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 28 09:32:33 2016 +0300
+
+    rbd-mirror: admin socket commands to start/stop/restart mirroring
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a22cf518f9cc0872f72a8a850ea01558094ebd00)
+
+commit 8e46e4383a15f022e9213c6710671e6241e7fb6c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun May 1 22:04:46 2016 +0300
+
+    qa/workunits/rbd: fix rbd-mirror log file name
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4c28fc2aa378c10ce6e78b1a2f98d5e14ac211e9)
+
+commit d73073a400807f2aa62d0eadc8ff6a8537ca04fa
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 28 09:23:35 2016 +0300
+
+    rbd-mirror: make image replayer asok commands available when not started
+    
+    Initially the asok commands were registered only after the image
+    replayer start (and unregistered on stop) because their names were
+    built using remote pool and image names, which became known only after
+    start.
+    
+    Now, the asok commands are registered on the image replayer construction
+    using the temporary name "remote_pool_name/global_image_id". They are
+    re-registered using "remote_pool_name/remote_image_name" when the
+    image replayer is started.  Also the commands are not unregistered on
+    the image replayer stop.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 21790484dc66e97e7625b5b2afabb7efe3a92b08)
+
+commit 24ea574a94c009d64d480bc3f887d4b0325d26d8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Apr 1 18:15:33 2016 -0700
+
+    qa/workunits/rbd: add env vars to adapt rbd_mirror.sh to teuthology
+    
+    With these set, this script can work in teuthology, where we want to
+    control the how daemons run and their log locations, valgrind
+    settings, etc. The workunit task always sets CEPH_ID, and allows
+    other env vars to be configured optionally.
+    
+    The RBD_MIRROR_USE_EXISTING_CLUSTER option can also speed up local testing.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit dcd11a1f262e0c56bb1b37d54283cda06abbf4c7)
+
+commit 21220a1fa668d4080eb7c4844cafeefffa77ecdc
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Apr 7 16:57:14 2016 -0700
+
+    qa: rbd_mirror.sh: change parameters to cluster rather than daemon name
+    
+    Daemon name is only useful to differentiate asok and pid filenames, so
+    just inline its use there. Everywhere else replace the 'daemon'
+    parameter with cluster or local_cluster as appropriate.
+    
+    Remove the CLUSTER{1,2}_DAEMON vars as well, since they are no longer
+    used.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 5fe64fa806f3bbdbdb950a5643238cc2508814b0)
+
+commit 0218f0b0750b88f308f3ee0837b4e4efc7aeea63
+Merge: a152b32 c1960de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:04:09 2016 -0400
+
+    Merge pull request #9005 from dillaman/wip-15716
+    
+    jewel: rbd-nbd: fix rbd-nbd aio callback error handling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a152b32016529633a810fc901a2c980ab1169b27
+Merge: 54177d0 0e95ee4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:03:38 2016 -0400
+
+    jewel: librbd: unlock image if journal error encountered during lock
+    
+    jewel: librbd: unlock image if journal error encountered during lock
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 54177d00f06ab3d712c8837832de080e452205de
+Merge: b6b3cdd a73b9dd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:02:24 2016 -0400
+
+    Merge pull request #9009 from dillaman/wip-15713
+    
+    jewel: librbd: possible race condition leads to use-after-free
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a73b9dd670bcd3a77611c7240d86963b29527983
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 07:41:30 2016 -0400
+
+    librbd: possible race condition leads to use-after-free
+    
+    The invoke async operation state machine can complete before the
+    owner lock is released.  Use a stack reference to prevent
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15690
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d9421a25b582e41550526714b71fb55f05abb1de)
+
+commit b6b3cdd40c60775c27b631cd8670e95bc84a5219
+Merge: a3e2e41 28ac027
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:53:30 2016 -0400
+
+    Merge pull request #9004 from dillaman/wip-15708
+    
+    jewel: rbd: help message distinction between commands and aliases
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a3e2e410f5ef6308fcfd59baf607c49112f69138
+Merge: ade4359 82838e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:53:00 2016 -0400
+
+    Merge pull request #8972 from Abhishekvrshny/wip-15697-jewel
+    
+    jewel: python: clone operation will fail if config overridden with "rbd default format = 1"
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ade4359b15d18ab51ab00168339d1aadcdffacbb
+Merge: 7f95348 451246d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:50:43 2016 -0400
+
+    Merge pull request #9008 from dillaman/wip-15698
+    
+    jewel: rbd-mirror: lockdep error during bootstrap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9bb17dbb1ce2ae910927e120cf35da9ee4c98225 (refs/remotes/gh/wip-rgw-period-delete-jewel)
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 15:57:22 2016 -0400
+
+    rgw: fix for duplicates in list_periods()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit abed30befd86c68922759a66db589285fe01e54b)
+
+commit 7ea6e78aa0aa3b86d663dab3f69998e5cd3a5177
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 15:18:46 2016 -0400
+
+    rgw: 'period delete' cleans up all period objects
+    
+    Fixes: http://tracker.ceph.com/issues/15469
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a286b32224d4a9757058415f98a2feff683ac520)
+
+commit 451246df64890e39205d20d40eb5676893a7e1db
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 10:33:50 2016 -0400
+
+    librbd: disable automatic refresh of image upon lock message
+    
+    There is no need to refresh the image upon request/release/acquire
+    of the exclusive lock.  The next IO or maintenance op will kick off
+    the refresh.  This is interfering with the refresh state machine
+    unit test case (since two concurrent refreshes shouldn't be possible).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ee617a0ca56d752daf62bfffa59219b2c540a428)
+
+commit 9e058fc852090ecb1b3a8daa3e0ee9df9d5eba3c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 10:26:29 2016 -0400
+
+    librbd: update_features should release lock if acquired
+    
+    There is a small window where requests are unblocked and before
+    the image is shut down where new requests can be accepted.  These
+    are correctly canceled during the shut down, but it can lead to
+    warning messages.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 608e09851396a9eeb7cf6dc1c7f687ef7a033601)
+
+commit e653a1541ca7df29d01e8524a3f3b597fa0fb67b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 09:27:29 2016 -0400
+
+    librbd: avoid applying refreshed image config within librados callback
+    
+    There is a potential that a synchronous API call could deadlock a
+    image refresh.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce5c701bc47b0959f8453b6b92dee4804d3b1d75)
+
+commit a5996e5ad64e01fcde50fc46bf9861b2d163f0c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 08:42:35 2016 -0400
+
+    rbd-mirror: don't use temporary string for snap operation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 06faf83bbfab000c8200a9cbe781adb0082b3f7b)
+
+commit a891919b85f44a7d81fb2fb84d1b8d7258b3fbdc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 08:31:54 2016 -0400
+
+    librbd: avoid recursive locking within operation state machine
+    
+    Fixes: http://tracker.ceph.com/issues/15664
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 91a4890ee78c25391c1548fdacb2b51c46a47415)
+
+commit 7f953488b46642ed3825349d41a34b0faeb6a76b
+Merge: 8963af7 895c975
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:47:01 2016 -0400
+
+    Merge pull request #8869 from Abhishekvrshny/wip-15669-jewel
+    
+    jewel: Errors encountered disabling object-map while flatten is in-progress
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8963af73823285e9261acf6305778903a8f4205d
+Merge: 978f30e 63738d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:46:12 2016 -0400
+
+    Merge pull request #8870 from Abhishekvrshny/wip-15668-jewel
+    
+    jewel: rbd disk-usage CLI command should support calculating full image usage
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 978f30e100fa91c148fd2aeaadad2f6e7bb94f9a
+Merge: 1dceb38 32c0901
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:35:41 2016 -0400
+
+    Merge pull request #8868 from Abhishekvrshny/wip-15666-jewel
+    
+    jewel: rbd CLI to retrieve rbd mirror state for a pool / specific image
+    
+    Conflicts:
+    	src/tools/rbd_mirror/ImageReplayer.cc: trivial resolution
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1dceb38538d9c8a13551ef8b9217cb5757089576
+Merge: 2495f08 357ad37
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:33:33 2016 -0400
+
+    Merge pull request #9046 from dillaman/wip-fix-snapc-validation-jewel
+    
+    jewel: librbd: put the validation of image snap context earlier
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 357ad37a4facc9917866c780892c7cd0e21653fd
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed Apr 20 09:55:39 2016 +0800
+
+    librbd: put the validation of image snap context earlier
+    
+    we'd better validate the snap context right after we got the mutable
+    metadata before we could go any further
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit ca8ae66ba42e7a598ad23ed911938589dc288456)
+
+commit 2495f08a64b352cbd21432b708fdd8e4090b67bb
+Merge: 5e9128d 125aab8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:31:10 2016 -0400
+
+    Merge pull request #8867 from Abhishekvrshny/wip-15661-jewel
+    
+    jewel: Parent image is closed twice if error encountered while opening
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5e9128dc8b4af16512f689d556baa7d6e08ae1f8
+Merge: c167514 af1c0bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:30:12 2016 -0400
+
+    Merge pull request #8866 from Abhishekvrshny/wip-15660-jewel
+    
+    jewel: rbd-mirror can crash if start up is interrupted
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c16751431baed0322765e502bf750b7f9be49e78
+Merge: 5e89e82 a2eb187
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:29:02 2016 -0400
+
+    Merge pull request #9044 from dillaman/wip-librbd-bad-header-jewel
+    
+    jewel: librbd: does not crash if image header is too short
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a2eb187658d045160fb85f6c1d9437eaa49f0830
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Apr 28 00:59:34 2016 +0800
+
+    librbd: does not crash if image header is too short
+    
+    if something goes wrong with the object, and returns a chunk shorter
+    than expected, don't panic
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7b52183addda83f98c8b1f37d3de20ca0fc4687b)
+
+commit 5e89e821f3e9b368b6d4963824e334acbd49cceb
+Merge: f99aba4 8cae07c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:26:48 2016 -0400
+
+    Merge pull request #8803 from SUSE/wip-15605-jewel
+    
+    jewel: Potential double free of SetSnapRequest instance
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f99aba4a869bbb6dda5353a6f5c9c65da32dcd02
+Merge: 9d40896 9b2c173
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:16:09 2016 -0400
+
+    Merge pull request #8865 from Abhishekvrshny/wip-15649-jewel
+    
+    jewel: possible double-free of object map invalidation request upon error
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b6ebb25afc3d75d834311a9931b08c84e858a5e4
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Thu May 5 15:28:16 2016 +0530
+
+    rbd: helpful error message on map failure
+    
+    Failure to map an rbd image provides pretty much nothing to
+    debug (or rectify) the nature of failure. This change tries
+    to improve the same by providing helpful error messages (by
+    pointing to dmesg, etc..).
+    
+    Fixes: http://http://tracker.ceph.com/issues/15816
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+
+commit f221fedb3b34bccd80383096d6ccfa19a4a72db9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 11:16:37 2016 -0400
+
+    librbd: assertion to ensure no concurrent processing of replay events
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 44827a3e8e02529be84c829a76ef01f70e0d20ee)
+
+commit 37f08e6e15b21eec2dedfd5e497619165cd3d117
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 11:05:50 2016 -0400
+
+    journal: suppress notifications if client still in try_pop_front loop
+    
+    One such example is popping the last entry from an object.  The next
+    object will be automatically prefetched.  When that object is received,
+    we do not want to alert the user that entries are available since
+    try_pop_front already indicated more records were available.
+    
+    Fixes: http://tracker.ceph.com/issues/15755
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3b8d21ac13865791d8fc4c519f47c1f7d6096e76)
+
+commit 82c04c5ad3476805d8c4f7569e1eba41244ad2c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 09:47:01 2016 -0400
+
+    librbd: delay processing of next journal entry until flush in-progress
+    
+    When replaying a journal flush event, do not start processing the next
+    journal entry until after the flush is in progress to ensure the barrier
+    is correctly guarding against future writes.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d205ef33cb8b5517f141d46a62947dd583500e0)
+
+commit 9d40896d280f44080623b1d9a0d9aba35ae410c5
+Merge: bc764b6 075ee03
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:31:02 2016 -0700
+
+    Merge pull request #8930 from theanalyst/wip-15707-jewel
+    
+    jewel: multisite: Issues with Deleting Buckets
+
+commit bc764b6af7fb7f5aea6bc5fd8c8f5b7174949a90
+Merge: 8bbe2a9 7029b9c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:30:52 2016 -0700
+
+    Merge pull request #9029 from theanalyst/wip-15680
+    
+    jewel: rgw: radosgw-admin zone set cuts pool names short if name starts with a period
+
+commit 8bbe2a924d9b27582c130f045e1fede29171f159
+Merge: 80beeff 5fdca84
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:30:03 2016 -0700
+
+    Merge pull request #9022 from yehudasa/wip-rgw-leak-3-jewel
+    
+    jewel: rgw: leak fixes
+
+commit 80beeff4b1bf89fba6c36b59ffcec61aa8fc975e
+Merge: b9cb6bd d93fa13
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:52 2016 -0700
+
+    Merge pull request #9021 from yehudasa/wip-rgw-xstorageurl-fix-jewel
+    
+    jewel: rgw: fix printing wrong X-Storage-Url in Swift's TempAuth.
+
+commit b9cb6bd73593da4312889c928fba8f02a50b1533
+Merge: ce2f028 f37318a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:27 2016 -0700
+
+    Merge pull request #9020 from yehudasa/wip-15597-jewel
+    
+    jewel: rgw: fix issue #15597
+
+commit ce2f028268509361700980d47dabe5429127910f
+Merge: 1cba195 a28fe02
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:18 2016 -0700
+
+    Merge pull request #9019 from yehudasa/wip-rgw-admin-zonegroup-jewel
+    
+    jewel: radosgw-admin: add missing --zonegroup-id to usage
+
+commit 1cba1951898e8d21dbfd6b8e354ebaa29f48b720
+Merge: 4ead0f9 1d28aac
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:03 2016 -0700
+
+    Merge pull request #9018 from yehudasa/wip-15626-jewel
+    
+    rgw: add AWS4 completion support for RGW_OP_SET_BUCKET_WEBSITE
+
+commit 4ead0f98dccf79f65878384ac0f45b9f3e2ad97c
+Merge: 6f23218 c08e90e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:28:50 2016 -0700
+
+    Merge pull request #9017 from yehudasa/wip-15625-jewel
+    
+    jewel: rgw: segfault at RGWAsyncGetSystemObj
+
+commit 6f23218bbcee8190acfdf0ae7b76e6e742015d91
+Merge: 2650806 e69bfa1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:28:37 2016 -0700
+
+    Merge pull request #9016 from yehudasa/wip-rgw-admin-modify-jewel
+    
+    jewel: radosgw-admin: update usage for zone[group] modify
+
+commit 26508062cbb13ba28252735449784dd4b63175af
+Merge: fdf8dcb 11d599c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:27:56 2016 -0700
+
+    Merge pull request #9015 from yehudasa/wip-rgw-zonegroup-remove-jewel
+    
+    jewel: radosgw-admin: zonegroup remove command
+
+commit f18b14b00fd3b8fbf3007f9eeb9424bbaf5ee14a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 11:22:48 2016 -0400
+
+    journal: incorrectly computed object offset within set
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f3372a1bcf81fcfd75e7ef5901dbcdc1daa062a3)
+
+commit 466b7fe3d5cd6da8de993169da4c0e4c0990e703
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Apr 22 09:45:30 2016 +0800
+
+    tools/crushtool: add straw2 support for help message
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6448ea283b4e1519682ecb29f18fbb767752d496)
+
+commit d6692ed70aebc5387997660e9e2b025e96eff96f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 20 16:23:55 2016 -0400
+
+    install-deps: remove distribute requirement
+    
+    Because it was meant for Ubuntu 12.04 and it is installing directly
+    from PyPI on newer releases that aren't being built for 12.04
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 64f14d43a829f998b9a2ad5f3d87095560914e2a)
+
+commit 3011edab65e78156346ff7de655707619589bd7a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 20 15:48:19 2016 -0400
+
+    tools: remove installation for distribute. It is no longer maintained and breaks builds
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d02bd9cb70479b7373c5b65e42e9c57f8eae55f1)
+
+commit 45a1f01aa4baa1913c335efb2f8b3941f3c0f791
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Fri Apr 29 03:30:05 2016 +0200
+
+    set 128MB tcmalloc cache size by bytes
+    
+    (cherry picked from commit d1d144c03bc9050931d86d635a16363bd2b73fd0)
+
+commit bb1aeb7c7bb296d9264f051519d100ecf1f50f1d
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Thu Apr 28 22:55:37 2016 +0200
+
+    etc/default/ceph: set 128MB tcmalloc cache size
+    
+    Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
+    (cherry picked from commit 9decbd1026c6ab0474c73a26a9c84c24557beb8a)
+
+commit 617004ef26b6c97621a85a44b78cc7b0708fe161
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 08:33:03 2016 -0400
+
+    test: update rbd integration cram test to remove format 1 warning
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 098038fad47b181a9ddd11767397082b05405be5)
+
+commit 7029b9c3b0c02c9f20f6079243521458d28b2ab4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 25 15:09:47 2016 -0700
+
+    rgw: fix suffix in RGWZoneParams::fix_pool_names()
+    
+    Fixes: http://tracker.ceph.com/issues/15598
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6f6b0c925994b0369eda390a6b3fd20005e70c9b)
+
+commit f37318af823f76ba7bf05e09da83141b569bf579
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 9 14:23:03 2016 -0700
+
+    rgw: code cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d53ffbfe65611425b08af71e0d24bfe2a0076f1e)
+
+commit a8b800b903f0f1144efe0cf3568b02b3e200c3b2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 11:31:43 2016 -0700
+
+    rgw: upgrade default zonegroup and set correct zone[group] id
+    
+    Fixes: http://tracker.ceph.com/issues/15597
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 27806c71922eef2f4cde360d7ba40dee9d5bd4bd)
+
+commit 5fdca846aaea9afd16828d33ccc279c4dcdbcd4c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 10:57:05 2016 -0700
+
+    rgw: RGWHTTPClient requests can unregister themselves early
+    
+    No need to wait for req_state to complete anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 91f61d68bf5fc39152d75fbc633f088e17d53d9e)
+    
+    Conflicts:
+    	src/rgw/rgw_http_client.cc
+
+commit 5609eb1bae0fc55b24a3025621597590f30aaad4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 4 18:08:38 2016 -0700
+
+    rgw: move around sync_env.init()
+    
+    call sync_env.init() even if we're initalized in RGWRemoteDataLog::init()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3d8442effc1c175e0b7c5064ed41bc642f6cdcb0)
+
+commit faf6b2d25e89b7a84ed262c08b0a858dbf4199da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 12:01:55 2016 -0700
+
+    rgw: rados crs, explicit cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c876eb3873692042c0af3644f8ee3ad15ffdbf5)
+
+commit b55514c390b22605fabdf3a8e5011415f57e6da8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 12:00:46 2016 -0700
+
+    rgw: RGWHTTPManager, avoid referring to req_data->client when completing
+    
+    this is past calling req_data->finish(), client can be destroyed by now
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7ada4df124a71eae413e7dec0e25264a8deffcad)
+
+commit 2d0cd8a08bb26f519d2b9b59bfe54f014cc36a1f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 11:59:11 2016 -0700
+
+    rgw: civetweb_callback() refer to store inside lock
+    
+    pe->store might change when reconfiguring
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2ca38dab071c7d20a224a4e1dacfd68548fe52dd)
+
+commit 2f65b313d08a5c9dfde220353eaa54204531b1ee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:03:14 2016 -0700
+
+    rgw: RGWPeriodPusher, stop http manager explicitly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 98433e589f2a8b519726ed23b1565c5eb331c68e)
+
+commit e5312b10aaa000d942468bf51d2164c689add279
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:02:02 2016 -0700
+
+    rgw: RGWHTTPManager, can call stop() more than once
+    
+    destructor calls stop(), but it can be also called explicitly, if caller
+    wants to ensure requests are not being handled anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit bdce81a2ae00f154420386e107a104cbf100d96d)
+
+commit cbea993562b03f75ceb0bdb7c02d72e5b51ae0fa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:01:33 2016 -0700
+
+    rgw: RGWReadRESTResourceCR, fix refcounting
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d1ec31def71505dc95139fbdd3b49c92fec1d296)
+
+commit 93a65f3306e21c1bc34f5a95d85deeba746adb62
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 09:59:47 2016 -0700
+
+    rgw: RGWReadRemoteDataLogShardCR fix destructor
+    
+    was missing destructor, constructor was broken
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 95162a4c1bf61f7329e363b7efd5686f7e1025f0)
+
+commit 2de16690b6a404a365af5d9f526de044873f9729
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 2 18:13:17 2016 -0700
+
+    rgw: RGWDataSyncStatusManager, cleanup if failing init
+    
+    init can be called more than once, ended up with leaking some resources
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 14cd44e0847fd914068f48dd0208ce7c1fe4eecb)
+
+commit 57266e6b78851f8e6167f15ad8dbf7195d561371
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 29 13:04:13 2016 -0700
+
+    rgw: rest crs, explicitly call cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a3a3f4f5883f3fc2bf47a7ebfa538206b7ab51d9)
+
+commit 6b86332ca3ee35eb522cbf88408b7fd5f3070772
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 28 17:03:21 2016 -0700
+
+    rgw: more leaks fixes
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cb797d4fd754b4a33eb34c249da3aabc65a7aba3)
+
+commit 0fb48546a9fe6eb497833a6566353229e0923d3f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 27 16:39:36 2016 -0700
+
+    rgw: drop a reference to http op
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4442d1c19acfc28c2dd4a011365ebebe08750963)
+
+commit d93fa132bba597e71c4dd483663d2069e7117bf1
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Apr 24 17:41:55 2016 +0200
+
+    rgw: fix printing wrong X-Storage-Url in Swift's TempAuth.
+    
+    If the option "rgw_swift_account_in_url" is being set to true,
+    both user and subuser IDs will be present in the X-Storage-Url
+    header generated by the implementation of Swift API's TempAuth.
+    This patch rectifies this behavior and makes that only the user
+    identifier will be placed there.
+    
+    Fixes: http://tracker.ceph.com/issues/15667
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 88f4c67d822388d2038454a25ead7e9a11f404b5)
+
+commit a28fe02c933e2ec7ea969a974f961769052e9d38
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 26 15:20:45 2016 -0400
+
+    radosgw-admin: add missing --zonegroup-id to usage
+    
+    also clarified that --rgw-zone takes the zone by name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    Reported-by: John Wilkins <jowilkin@redhat.com>
+    (cherry picked from commit 97fad754565b262529073e1b9f8d1799bfdfe0de)
+
+commit c08e90ea20b8ed96c7fe8638f725cf5abefc6783
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 27 15:09:55 2016 -0700
+
+    rgw: don't allow any concurrent sync requests on the same key
+    
+    We used to allow concurrent requests, as long as these had different op,
+    which wasn't correct.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit edea6d58dd25995bcc1ed4fc5be6f72ce4a6835a)
+
+commit 6c61341697b99db34466c9e7e90aa7867048d651
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 15:59:33 2016 -0700
+
+    rgw: collect children after waiting for them
+    
+    Otherwise we're going to busy loop there.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 806fb85d18653910863ba6b8cb56b8db1a0fa3d2)
+
+commit 4a5f33dd975eb44f908b8a90a7ad2e4beb17b354
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 15:29:41 2016 -0700
+
+    rgw: don't pass object context into async coroutines
+    
+    Fixes: http://tracker.ceph.com/issues/15625
+    
+    The async read data/attrs coroutines may outlive the callers. Instead
+    of introducing a complicated refcounting scheme, just don't pass it in.
+    Anyway, it was not thread safe, and the benefit of using it is not
+    clear.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b225d1758057efc30721aad25447d54fa36e4cb2)
+
+commit 1d12f82a24f78769bcebb32cb379445e04f35e46
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 21 17:24:16 2016 -0700
+
+    rgw: check for status >= 0
+    
+    Fixes: #15565
+    
+    check only verified that status was > 0, so we ended not updating mtime for
+    the target object. This resulted in a continuous sync of objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 255f19d9ef964536f2db17f70749f489330262a6)
+
+commit a288a054c18fedda9ac3230d392f5a41a140ca9a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 22 14:23:22 2016 -0700
+
+    rgw: don't try to delete object if does not exist
+    
+    If we know that the object does not exist, no need to remove it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8d59a30d56b5f197a293f7fba6ebb1a630a33a05)
+
+commit 308b9f54e9fa07d071335bb2d953a50a5095c464
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 22 11:55:28 2016 -0700
+
+    rgw, cls/rgw: store removed object mtime in bi log
+    
+    Store the mtime of the object that was removed in the bucket index log.
+    This will be used in data sync.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 80d64049937e1930ea8273fdad76a51493a5e14a)
+
+commit 1d28aac642d940bb1c63733406999a7b027e05f2
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Wed Apr 27 20:28:08 2016 +0200
+
+    rgw: add AWS4 completion support for RGW_OP_SET_BUCKET_WEBSITE
+    
+    Fixes: http://tracker.ceph.com/issues/15626
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 05373354e4725c3fc675df652fbf97da26160bcb)
+
+commit e69bfa14e184716c4ab6bf01746edf5c9663a8e1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 27 18:06:45 2016 -0400
+
+    radosgw-admin: update usage for zone[group] modify
+    
+    the help strings indicated that only --master could be changed
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 410d432d11541a29c3fd4608a070cecf8dc2766e)
+
+commit 11d599cda659a42ddde43f34307c5e3553596c54
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 14:26:46 2016 -0400
+
+    test/rgw: add test_zonegroup_remove
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e2b27c7266f0ace372dbf94e062598fd7a575795)
+
+commit 928ccf4952052ab19044de1a8f20246075d54832
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 14:26:23 2016 -0400
+
+    test/rgw: index zones by name instead of insertion order
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6327ade12bf0255c3b35bd6407f1dd2bcc6936fa)
+
+commit 5edacdccc61d565a0e91af452f9297d559a474e4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 2 11:38:24 2016 -0400
+
+    radosgw-admin: add missing 'zonegroup remove'
+    
+    Fixes: http://tracker.ceph.com/issues/15684
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit db0fa48ed5958d37928f893acd44cd66b6b9b990)
+
+commit 6b1d9ebb80478c1dfe0e974377bd4fb070a5d417
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 2 11:36:58 2016 -0400
+
+    rgw: RGWZoneGroup::remove_zone() takes zone id
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3a451511fb38a5f1cd83ca3610c91141e7279f63)
+
+commit 0e95ee4bd16f4d54d1da476b51fd0fd0829d69ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 10:15:08 2016 -0400
+
+    librbd: unlock image if journal error encountered during lock
+    
+    Explicitly unlock to prevent a client from accidentally blacklisting
+    itself when retrying the lock.
+    
+    Fixes: http://tracker.ceph.com/issues/15709
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a11f5e8e55fc448ed60616cbf66a3ea7db2247b9)
+
+commit fdf8dcb772d8313a3a801bf265f022505dba934d
+Merge: 030883f 0de6345
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 9 13:54:58 2016 -0400
+
+    Merge pull request #9006 from athanatos/wip-15655-jewel
+    
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c1960debf5c8696f365cd670dedeea01d2a3b9a8
+Author: cy.l@inwinstack.com <cy.l@inwinstack.com>
+Date:   Sat Apr 30 15:22:29 2016 +0800
+
+    rbd-nbd: Fix aio_callback error handling
+    
+    Change aio_callback return code handling for NBD_CMD_READ command.
+    
+    Tracker:
+    http://tracker.ceph.com/issues/15604
+    
+    Signed-off-by: Chang-Yi Lee <cy.l@inwinstack.com>
+    (cherry picked from commit ee211295441b126a995c07820176cfc966ab61c5)
+
+commit 4e87c1cd7e923f4254ce43e92e1cce4329fd86dd
+Author: cy.l@inwinstack.com <cy.l@inwinstack.com>
+Date:   Wed Apr 27 11:00:25 2016 +0800
+
+    Fix RBD-NBD aio_callback error handling
+    
+    Tracker-Id: #15604
+    
+    Signed-off-by: Chang-Yi Lee <cy.l@inwinstack.com>
+    (cherry picked from commit f8f6c1dc865b7ed642fec3f416e5eca5e91523d0)
+
+commit 28ac027f6f86635d320f0f5e25582452011d9bed
+Author: YongQiang <he.yongqiang@h3c.com>
+Date:   Fri Apr 22 11:20:31 2016 -0400
+
+    rbd:make a distinction of help message between ''rbd snap rollback" and "rbd snap revert"
+    
+    Fixes: https://tracker.ceph.com/issues/15521
+    
+    Signed-off-by: Yongqiang He <he.yongqiang@h3c.com>
+    (cherry picked from commit 1bf26509c6efd06a8facc0e45ab42255592ca74d)
+
+commit 039554d7ad8cfd82bcb6d4c6564f6dd46320a24f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 13:36:06 2016 -0400
+
+    librbd: reduce log level for image format 1 warning
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c87f383b65ccc2e91645394f1c75de1c2d52e1e1)
+
+commit 030883f41d3d2d8f297b3d3f6e7c7fa5a06ea4c0
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 15:12:17 2016 +0800
+
+    osdc/Objecter: fix race condition for sortbitwise flag detection
+    
+    During list_objects/list_nobjects(), we are checking osdmap's
+    sortbitwise flag setting without holding rwlock in hand, which
+    is a potential race condition.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 7fb6be706be2454cc2a831df29808997f0aa5ab3)
+
+commit 82838e340e0ab7c593618f5807ea5868ecd6bc48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 15:06:51 2016 -0400
+
+    librbd: default clone operation to image format 2
+    
+    Image format 1 doesn't support clones, so unless the user
+    overrode the setting, default to format 2.
+    
+    Fixes: http://tracker.ceph.com/issues/15685
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f24c70f54343aa6698af8ac1f0774282b0659968)
+
+commit ca13a9554d8a9d97428c2096b5a2ae8e25d08a9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 14:51:31 2016 -0400
+
+    librbd: add rbd_image_options_is_set helper method to API
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2633b045e0b57827cc10c2e7707bd5a5e344e59a)
+
+commit e6aa4533feb9c8e6d15c820f54dd719c378dd26e
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Apr 29 20:12:20 2016 +0100
+
+    mds: fix upgrades with replay daemons in map
+    
+    In the new world, a standby replay daemon which
+    is actively replaying has its rank set.  In the old
+    world it did not: map this during the upgrade/downgrade
+    paths in decode/encode.
+    
+    Fixes: http://tracker.ceph.com/issues/15591
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d94115aa2a4bd92c56552e9fd11205d391a1c673)
+
+commit 6c1b792b7b2fba729f21b0f10ec70d14b830ffa9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 30 12:06:27 2016 +0100
+
+    doc: add cephfs daemon management docs
+    
+    Principally to tell people how to configure
+    standby, but more generally as a place to define
+    our terms for GIDs etc.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 97dd6897efaa14d3f8859a791949f9d246769620)
+
+commit bf44c6d434a41b90c0739e493a022cb3d3df5554
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 30 12:05:43 2016 +0100
+
+    mds: omit fscid in single-filesystem status output
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ec8f3db3e5c26b60b9333e86453e1b17324a8cfa)
+
+commit be5274c3f47ecb0dca0facca37d4a90c68b37027
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 24 13:23:26 2016 +0000
+
+    mds: simplify standby/standby-replay logic
+    
+    This used to use an arcane set of constants
+    in standby_for_rank, combined with daemons sometimes
+    sending requests to enter state STANDBY_REPLAY.
+    
+    Simplify this so that there is only one place we
+    put daemons into standby replay, and that is
+    in tick->maybe_promote_staandby.
+    
+    There is a behavioural change in here, which is
+    that we used to sometimes promote standbys to
+    be standby-replay, even if they didn't have
+    "standby replay = true", when they did have
+    a standby_for_rank or standby_for_name set.
+    I'm calling that a bug, and making it so that
+    daemons will only go into standby-replay if
+    "standby replay = true" is set.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2a2a56b2d0e9e4224cae27f80d990b787e209bed)
+
+commit ae3ce5b8741eaf65d03aeb7b74925a75e5ef2598
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 24 13:13:55 2016 +0000
+
+    messages: add MMDSBeacon::standby_replay
+    
+    So that daemons can explicitly say that they
+    want to be standby replay, instead of using
+    magic standby_for_rank values.
+    
+    Because in some cases daemons would indicate this
+    with a beacon requesting STANDBY_REPLAY state, we
+    also munge these during decode to be normal STANDBY
+    requests with standby_replay=true.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit cdf8a29f81475a000824ce8942f02d9d570ec3bb)
+
+commit 02e3edd93c0f4ef6e0d11df1f35187f74c7ea2ff
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 22 17:44:22 2016 +0000
+
+    mds: remove ONESHOT_REPLAY mode
+    
+    This predated cephfs-journal-tool.  Don't bother
+    with any backwards-compatibility mechanisms here, as
+    it was a seriously niche undocumented thing.
+    
+    On the off chance that someone ran an old ceph-mds against
+    a newer ceph-mon with oneshot enabled, it shouldn't break
+    anything, it just won't do what they want.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit edfda9c6b0a0c8b75ebff5f06017aa475339b1fd)
+
+commit 52ca195221ffa86b778346e847aeec817329c651
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Apr 7 12:53:00 2016 +0100
+
+    mds: remove stale comments
+    
+    A little bit of detritus from the MDS->MDSRank refactor.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e9b58e7ff97646e557c5b3e2c86a067d2d136149)
+
+commit c1279d8ff8716a54622bf98bd29ce40faa2bf41d
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Apr 7 12:37:08 2016 +0100
+
+    mds: remove inc array from mdsmap
+    
+    Instead, use the MDSMap epochs to provide the `inc` that
+    we populate into mds_info_t for use by the daemons.
+    
+    It is not necessary to respect the current value of inc
+    in old maps, because the per-rank counters were necessarily
+    lower numbers than the overall map epoch (incrementing
+    the counters always corresponded to a map change).
+    
+    Fixes: http://tracker.ceph.com/issues/15399
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 30a375ebf814245fbbb5155fd39d8ff673f0a32a)
+
+commit 9d5162fe102301bc94065f99a20b3067395dde2a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 24 14:03:58 2015 +0100
+
+    test/mds: add test for symbols in paths
+    
+    Especially, hyphens.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4e9327f64e5965c57c0eeea397060dc2ab43cfbf)
+
+commit af3a4e4af36b5159721b634e216edcab9a171155
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Apr 12 02:49:32 2016 +0100
+
+    mds: fix auth caps with hyphen in path
+    
+    Quirk of the boost spirit char() fn, hyphens
+    are treated as range delimiters unless they're
+    the last character in the set.
+    
+    Fixes: #15465
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f82fa5f59a80851581573553be07e8f1620ee346)
+
+commit 075ee0310888d012c56c494291b873be47393e47
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 19 15:20:53 2016 +0200
+
+    rgw_op: pass delete bucket op to master zone first
+    
+    Basically first ask the master to process a delete bucket request before
+    we process it in the zone locally, otherwise we reach a situation where
+    we drop the bucket locally and master will still maintain an index
+    
+    Fixes: http://tracker.ceph.com/issues/15540
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit da6b44a6e47399493adae53ad3c36a029f1e2a23)
+
+commit 1527b56a888a65fa28223cd9c9cdb06d1575973c
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 19 15:20:07 2016 +0200
+
+    rgw: add errno entry for -ENOTEMPTY
+    
+    currently if master returns an -ENOTEMTPY, we return a -EIO as we dont
+    have an entry for this
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 40c3bfef8cc3bce7d15fbb0e135c1a909ee59368)
+
+commit 791eba81a5467dd5de4f1680ed0deb647eb3fb8b
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Thu Apr 28 22:48:52 2016 +0200
+
+    fix deb package /etc/default/ceph location
+    
+    commit 7384a14f243519547a61534d22e21e6069aae016 have introduce
+    the /etc/default/ceph file at a wrong location : /etc/default/ceph/ceph
+    
+    (cherry picked from commit 13804fafd4c33068d8ed6d0480b594daec0f9341)
+
+commit d9851351aeb6d45a2df1c107b23e77c992926d0a
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 11:13:32 2016 +0800
+
+    mon/OSDMonitor: improve reweight_by_utilization() logic
+    
+    By calling reweight_by_utilization() method, we are aiming at an evener result
+    of utilization among all osds. To achieve this, we shall decrease weights of
+    osds which are currently overloaded, and try to increase weights of osds which
+    are currently underloaded when it is possible.
+    However, we can't do this all at a time in order to avoid a massive pg migrations
+    between osds. Thus we introduce a max_osds limit to smooth the progress.
+    
+    The problem here is that we have sorted the utilization of all osds in a descending
+    manner and we always try to decrease the weights of the most overloaded osds
+    since they are most likely to encounter a nearfull/full transition soon, but
+    we won't increase the weights from the most underloaded(least utilized by contrast)
+    at the same time, which I think is not quite reasonable.
+    
+    Actually, the best thing would probably be to iterate over teh low and high osds
+    in parallel, and do the ones that are furthest from the average first.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit e7a32534ebc9e27f955ff2d7a8d1db511383301e)
+
+commit b0543fdfdebc6766fe927b4cc34a4369e7f86544
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Apr 25 15:43:03 2016 +0800
+
+    mon/OSDMonitor: avoid potential expensive grace calculation
+    
+    The grace calculation during check_failure() is now very complicated
+    and time-consuming. Therefore we shall skip this when it is possible.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 3557903d5d57642179b2ae137bedc389974b1956)
+
+commit 53686dfac0622a981374d448ce8229eb3984e5c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 22:01:56 2016 -0400
+
+    mon/OSDMonitor: max_osds must be > 0 for reweight-by-*
+    
+    Include a few test cases.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 84bd0dce18f6dd0c35d7e6cfee91ce8c5a0f4db7)
+
+commit ec416556e516b60ab490c424a4e45fa9ff8f8fd0
+Merge: 442c31b f4d63af
+Author: John Spray <jspray@redhat.com>
+Date:   Tue May 3 08:05:23 2016 -0500
+
+    Merge pull request #8787 from xiaoxichen/wip-15631
+    
+    jewel: CephFSVolumeClient should isolate volumes by RADOS namespace
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0de63455cbf662435ff57a35b9d98187091dcec7 (refs/remotes/gh/wip-sam-testing-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 28 13:47:48 2016 -0700
+
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Fixes: http://tracker.ceph.com/issues/15655
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c121bc51a5a877a49094cfbd3bd16af1f3ae5090)
+
+commit 63738d4d154d983d98021a09daa24607df64b979
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 15:53:59 2016 -0400
+
+    rbd: disk usage now includes all snapshots by default
+    
+    Added a new "--from-snap" optional to restrict the calculation to
+    specific subsets of snapshots and the HEAD revision.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1ccdcb5b6c1cfd176a86df4f115a88accc81b4d0)
+
+commit 895c975651b722832befeb829a43acc55823740b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:48:59 2016 -0400
+
+    librbd: block RPC requests while updating features
+    
+    Disabling the journal and object map require acquiring the exclusive
+    lock locally.  We don't want to start executing long-running ops
+    for this quick process.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db7aaff0f6d7a57e2b36d22b609f915f9b2b3893)
+
+commit 1f0056a427cd46793934c59c49aa225da2237628
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:43:24 2016 -0400
+
+    librbd: return -ESHUTDOWN when a refresh is requested on a closed image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ed132a1eb10e6b2eb4bc465658eb0379daca3d67)
+
+commit 6da1a84f16f4478ec17bdd4f50d9b0a02178c444
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:42:49 2016 -0400
+
+    librbd: synchronous block_writes should return possible error result
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 88e76cb3c9a74b4718ad39c827f2194666f5be2a)
+
+commit 04ef40e7652ff05146289a6e9e15ce2290a1ee32
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 11:46:20 2016 -0400
+
+    librbd: refresh image before executing maint op
+    
+    If a maint op is interrupted and restarted, it's possible that the
+    image will need to be refreshed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9cf6821bd9f58c181720a5c8156cec5ab33d35d4)
+
+commit 64cb6455f384f5ecc20abcf4d4ec6596569ac4c0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 11:10:25 2016 -0400
+
+    librbd: reduce log level when lock owner not detected
+    
+    This can happen under normal operating conditions and the
+    lock request is automatically retried.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a4893b7a7e50cc399696e1d0cb4d4f8785e723d)
+
+commit 49cfb0e48014610d04233a6c7549bbc3756dd3f6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 10:52:15 2016 -0400
+
+    librbd: reduce log level for interrupted maint ops
+    
+    Fixes: http://tracker.ceph.com/issues/15572
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1617328402345c68bf5d54828da6d687e51ef42b)
+
+commit 32c09015a16ef8dee535d2c81825a2590f7f15e7
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Apr 24 19:39:40 2016 +0300
+
+    qa/workunits/rbd: test mirror status in pool directory
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit f15cd51a1ddbd4079a8218bc2446a4bf68f95107)
+
+commit 9a30a89abc5749c37d26d39c175e51259f63caec
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 21 09:52:47 2016 +0300
+
+    rbd-mirror: in replay status store number of entries behind master
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e4229d1d4b163f1003e68fef1db63f796393ba23)
+
+commit 15a9131fd2d55191c3082c6692f5e8f714fcff71
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Apr 15 08:42:17 2016 +0300
+
+    librbd: API to retrieve image mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a2045d61874f9e418b5b3c3136af4482603c8e7f)
+
+commit 98ca7f3904a9c8ff9a661f153069a8747a51431b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Apr 6 14:57:29 2016 +0300
+
+    test: MockJournaler: add get_cached_client method
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 7633407ab584ce8c7777a525554773c2a358ce5f)
+
+commit e57c4d8f0832abcf431029b04da43e043a479d19
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:42:09 2016 +0300
+
+    rbd: CLI to retrieve rbd mirror state for a pool / specific image
+    
+    Fixes: #15144
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe5207278d884464c05d6808245a0c4df03c441c)
+
+commit 6111a25448fda1824ca6b3007fcdece1970674d6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:40:50 2016 +0300
+
+    rbd-mirror: store replay status in mirroring object
+    
+    Fixes: #14420
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 52b2fe14cfc932b2aa53ce76a3150ce87ce4d377)
+
+commit f22514201c3b49d93593fad27360deb7b715493d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:25:11 2016 +0300
+
+    test: cls_rbd: methods to store/retrieve mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8aefad59813489cee112f46b3fe10331bb039b57)
+
+commit 4a3f0d23ba4b96a28de661e219dcb52165bbbeac
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:24:07 2016 +0300
+
+    cls::rbd:: methods to store/retrieve mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 17dd40df4f4201cf0c0c6d58b1728177c5ee726b)
+
+commit 62c60144311518e7e7947791a17608ed1765f971
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Apr 1 08:27:03 2016 +0300
+
+    objclass: add method to list watchers
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 44a31d2b770ef016271f9285447c601921b9c838)
+
+commit 442c31bcf73899767c7fa198427712bbdf767067
+Merge: 6e5091e d302617
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon May 2 08:35:09 2016 +0200
+
+    Merge pull request #8843 from SUSE/wip-msd-typo-fix-jewel
+    
+    systemd: fix typo in preset file
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 125aab8dc4300dc9c67b2d4ca219cfbd60ab0924
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:56:11 2016 -0400
+
+    librbd: propagate any image refresh errors back to caller
+    
+    Previously, the saved error code was only propagated when an image
+    flush was required.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 105ef8a24aa6ac8719f68c5a9366129ddb41ee88)
+
+commit 5fe4beac0ed68ebf33fa69ba7a3082d2072bb9ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:45:25 2016 -0400
+
+    librbd: avoid second close attempt of invalid parent image
+    
+    Fixes: http://tracker.ceph.com/issues/15574
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e91e7f8d9ce10f0d19c48facfb6318d6b49798e8)
+
+commit af1c0bce9184c55aa88b4082db9302239009af36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:22:59 2016 -0400
+
+    rbd-mirror: interrupting image replay startup can cause crash
+    
+    Properly shut down the journaler if a shut down has been requested
+    while start up is in-progress.
+    
+    Fixes: http://tracker.ceph.com/issues/15630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a71052a5b14af34ed81f6ee77f862f576a7a5cc)
+
+commit 9b2c17392a2d9543f05d0cb8486183edf84eb2c3
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 28 09:05:33 2016 +0800
+
+    librbd: fix potential assertion for object map invalidation
+    
+    if update_flags fails we should not go any further, or the async request
+    will be removed from m_image_ctx.async_requests twice
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit dd0eb719a50e9fb24eca5a840c2e8adb53be41b2)
+
+commit 6e5091e454018e42ebe79bc4ec70cd24bfaeb36d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 30 16:21:13 2016 +0200
+
+    systemd: fix typo in preset file
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 80be4a8cbf1d42232ff61494d035e7abd90c2c14)
+
+commit d3026174f87ede746a0ab0aa96b0efbf5e77b161
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 30 16:21:13 2016 +0200
+
+    systemd: fix typo in preset file
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 80be4a8cbf1d42232ff61494d035e7abd90c2c14)
+
+commit 9cdad1f348a67e3f4daec617210340b14a2dd49a
+Merge: 96b3726 37ccacf
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 29 08:22:01 2016 +0200
+
+    Merge pull request #8801 from SUSE/wip-15633-jewel
+    
+    jewel: ceph-{mds,mon,osd} packages need scriptlets with systemd code
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 8cae07c10b9cb62af87d494724586c945f181127
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 21 14:48:55 2016 +0800
+
+    librbd: fix potential double free of SetSnapRequest instance
+    
+    if image feature EXCLUSIVE_LOCK is not enabled we should not try to
+    initialize the exclusive lock, or we may end with two async Contexts
+    to finish the same SetSnapRequest instance
+    
+    Fixes: http://tracker.ceph.com/issues/15571
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 0a6eaac12fe29762c3dd29068f9427840cfea100)
+
+commit 37ccacf7d1ce57291d6cbadd416fb96aed7db36a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Apr 27 18:10:51 2016 +0200
+
+    rpm: Add rpm scripts for ceph-rbd-mirror
+    
+    We are currently missing the systemd scripts for the new ceph-rbd-mirror
+    daemons. This patch introduces them.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 14652f32165f72efde569a939179fc26584ab155)
+
+commit c729bdd4bcb974081899c65934d3fdb9e0e96316
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Apr 27 18:00:08 2016 +0200
+
+    rpm: Start all the targets in %post
+    
+    We need to have the sub-targets active for the ceph.target to be able
+    to propagate its calls to the services. If the sub-target is inactive,
+    the main target won't propagate the stop/restart calls.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 73f457986753b504fd088c1fa064608aa9d340c9)
+
+commit c72f0bc3d4fb9a3f5abfefcf8d2775721d95b33a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 19 14:48:41 2016 +0200
+
+    rpm: implement scriptlets for the post-split daemon packages
+    
+    This patch gives each of the ceph-{mds,mon,osd,radosgw} packages its own
+    %post, %preun, and %postun scriptlets dealing with the package's unit files.
+    
+    The scriptlets of ceph-base are adapted to handle the ceph.target unit file
+    only.
+    
+    The scriptlets of ceph-mon handle ceph-create-keys services in addition to ceph-mon.
+    
+    The scriptlets of ceph-osd handle ceph-disk services in addition to ceph-osd.
+    
+    Fixes: http://tracker.ceph.com/issues/14941
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 644aba9270714e5e231ac7d7e3437477837531eb)
+
+commit 81f407337da18b5b555c6737e394a0e054d7c7da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 19 14:51:01 2016 +0200
+
+    systemd: enable all the ceph .target services by default
+    
+    Some distros, like Fedora and openSUSE, have a policy that all services are
+    disabled by default.
+    
+    This patch changes that default for the ceph.target and
+    ceph-{mds,mon,osd,radosgw}.target services.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 53b1a6799c2ce08acc8382e4aa996d1bde700316)
+
+commit f4d63af533b22f740d809272e2e47ebc9d8d6845
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Wed Apr 6 22:50:05 2016 +0800
+
+    python_cephfs: rule out empty/None volume_id
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 73368069dd466cdbde1ebeae97771d603ce431f5)
+
+commit b609017a52e3140e9ff6314b8d312a409af52a57
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Wed Apr 6 22:45:02 2016 +0800
+
+    python-cephfs: use rados namespace for data isolation.
+    
+    As cephfs dir layout supports rados namespace in jewel, it would be
+    good to use rados namespace instead of pool, since it is free,
+    we always enforce namespace isolation for volumes.
+    
+    The meaning of "data_isolated" flag changed a bit, it's no longer used
+    for security isolation, referring to physical isolation
+    (i.e. potentially using different OSDs).
+    
+    Also, (de)authorize based on namespace.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 0e1d013ea690c18f7fa6e2a19500f854a9949091)
+
+commit 96b3726d74cb7bbcacdd22e0ab0fe233e4c4d829
+Author: Tamil Muthamizhan <tmuthamizhan@MacBook-Air.local>
+Date:   Tue Apr 26 17:34:52 2016 -0700
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    
+    s/ceph-deploy/ceph-tests, hence removing ceph-deploy folder
+
+commit 77fdbf1747d73fd809607efd3325cc38b2fbafc0
+Author: Tamil Muthamizhan <tamil@magna002.ceph.redhat.com>
+Date:   Fri Apr 22 18:25:46 2016 -0400
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    
+      renamed the workunit and test to reflect reality and for better clarity
+    
+    (cherry picked from commit 0fd358b877c611bf67c94b4a2dacbe65e17deae9)
+
+commit 1fa533e83d8e541315aa4925a6192ab0401b688b
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Sun Apr 24 09:55:35 2016 +0200
+
+    fix ceph init script
+    
+    commit 65963739cd6815b8008282c8f64cd64365662e60 have introduce a bug
+    
+    test variables need to be quoted, or -n always return true, even if variables don't exist
+    
+    (cherry picked from commit b581dd56bea09520c154611ecea91ebe399058d3)
+
+commit 7acbefadace28412f395bb98aec5651d5ec70666
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 18 18:27:52 2016 -0700
+
+    test: Fix ceph-objectstore-tool test to run manually from src non-cmake
+    
+    Fix init-ceph to handle it when CEPH_* values set
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5e8e03b4d030db57c93047551d5b92db0b604063)
+
+commit af4b31cb9770d1710d038c557d547373f58a89a8
+Author: James Page <james.page@ubuntu.com>
+Date:   Tue Apr 5 16:58:58 2016 +0100
+
+    Drop --setuser/--setgroup from osd prestart
+    
+    These are not supported by /usr/lib/ceph/ceph-osd-prestart.sh,
+    resulting in warnings:
+    
+     ceph-osd-prestart.sh[23367]: getopt: unrecognized option '--setuser'
+     ceph-osd-prestart.sh[23367]: getopt: unrecognized option '--setgroup'
+    
+    --setuser and --setgroup are only needed for the ceph-osd process.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit 74977f78840fe95b67788f3aa135ac69a2353c43)
+
+commit 8a86d08e38a8b68ba5e04b2cc4f59a2983481ee7
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Apr 22 11:19:30 2016 +0200
+
+    osd: fix backwards min/max osd utilization
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 7529851f2a186501664461916f785da8456ad612)
diff --git a/doc/changelog/v10.2.10.txt b/doc/changelog/v10.2.10.txt
new file mode 100644
index 000000000..8493e7f4c
--- /dev/null
+++ b/doc/changelog/v10.2.10.txt
@@ -0,0 +1,4794 @@
+commit 5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Wed Oct 4 14:17:25 2017 +0000
+
+    10.2.10
+
+commit 951609bd3033f20560230d1d3514b535cffc0fde
+Merge: c4aa7c3df2 c6846df53c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 15:58:17 2017 -0400
+
+    Merge pull request #18100 from ceph/wip-yuriw-fix-clients-jewel
+    
+    tests: Added openstack.yaml bits to enable runs on ovh nodes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c6846df53c311ae5640a0bc28fbd752e38a745f3
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 3 12:50:24 2017 -0700
+
+    tests - Added openstack.yaml bits to enable runs on ovh nodes
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit c4aa7c3df2ead7d76aee9ed29d090c4433274af6
+Merge: 16110b985c add06614ab
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Oct 2 08:39:01 2017 -0700
+
+    Merge pull request #18044 from ceph/wip-yuriw-fix-clients-jewel
+    
+    qa: Changed distros symlink to point to supported OSs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit add06614ab1b8f4a99feece1225a9c2e1cde1279
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Fri Sep 29 12:29:32 2017 -0700
+
+    Changed distros symlink to point to supported OSs
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 16110b985c4278b12f6884537a176cd85cfd0060
+Merge: 750e67cab8 06e8f38d21
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 29 10:19:25 2017 -0700
+
+    Merge pull request #18027 from ceph/wip-yuriw-fix-clients-jewel
+    
+    tests - Added suit to test upgraded clients against jewel ceph clusters
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 06e8f38d216c049db4339bc291f7a495f2187d78
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Sep 28 14:37:03 2017 -0700
+
+    tests - Added suit to test upgraded clients against jewel ceph clusters
+    
+    Replaces https://github.com/ceph/ceph/pull/17981
+    We need to run this suite using suite-branch option in
+    order to use jewel workloads agains ceph cluster luminous+ branches
+    
+    Added 'libcephfs1' to exclude_packages in upgrade_workload
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 750e67cab8fd0498ca6d843f25007904041d49cd
+Merge: 189f0c6f27 8398a8a033
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Sep 22 22:23:51 2017 +0200
+
+    Merge pull request #17892 from smithfarm/wip-p2p-s3-test
+    
+    jewel: tests: fix upgrade/jewel-x/point-to-point-x
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 8398a8a0334718c68b9d9a04c820a875f7ec0056
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 21 22:50:06 2017 +0200
+
+    qa: point-to-point-x: upgrade client.1 to -x along with cluster nodes
+    
+    The client.1 rgw in workload_x had not been upgraded to -x.
+    
+    Fixes: http://tracker.ceph.com/issues/21499
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit afc5a7d3208071b65c0d76d124bfc47a099a446c)
+
+commit d377266df600586d8889ca9ab4a6b7b110fcf0dc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 21 17:50:28 2017 +0200
+
+    tests: use special branch of ceph/s3-tests with pre-10.2.10
+    
+    Jewel v10.2.10 introduces a fix for S3 ACL code, for which a new test was added
+    to ceph/s3-tests.git (ceph-jewel branch). Since the jewel point-to-point-x
+    upgrade test runs s3-tests on 10.2.7, modify the test to use a special
+    ceph/s3-tests branch (ceph-jewel-10-2-7) that omits the new test.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 189f0c6f2703758a6be917a3c4086f6a26e42366
+Merge: 0ffa38ee4d de00e2bb5b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Sep 19 18:03:29 2017 -0700
+
+    Merge pull request #17780 from smithfarm/wip-rh-74-jewel
+    
+    tests: CentOS 7.4 is now the latest
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 0ffa38ee4d074922737485cdd28d101449560107
+Merge: 94009032a8 0cd7df3649
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 19 14:22:52 2017 +0200
+
+    Merge pull request #16297 from smithfarm/wip-20518-jewel
+    
+    jewel: rbd: cli: map with cephx disabled results in error message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 94009032a87026099fa2cf781e421508a061eb76
+Merge: d8aa3c502e d7d988d1f3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 19 13:33:42 2017 +0200
+
+    Merge pull request #17781 from smithfarm/wip-drop-11429
+    
+    tests: Removed 11429 config
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d7d988d1f3befc4443415931ee3f5c6fcb03b11a
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Sep 21 09:14:09 2016 -0700
+
+    tests: Removed 11429 config
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from fde4d3bb96abc0c0ca5bca213a0a2423f95e91a2)
+    
+    Conflicts:
+        Cherry-picked manually because, nowadays, we have the entire ceph-qa-suite
+        under qa/ in ceph/ceph.git
+
+commit de00e2bb5b3c5f4026fd2d63c3799572655cd73a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 18 18:01:17 2017 +0200
+
+    tests: CentOS 7.4 is now the latest
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2311b64025cdb6131035aaf01e7c97486da12e15)
+    
+    Conflicts
+        qa/distros/supported/centos_latest.yaml (renamed from centos_7.3.yaml)
+
+commit d8aa3c502e63462ab6a18eaed103eddfdf753855
+Merge: 2a525df77b f86dc69844
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 11:05:28 2017 -0700
+
+    Merge pull request #16473 from smithfarm/wip-20723-jewel
+    
+    jewel: osd: rados ls on pool with no access returns no error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 2a525df77b91f5b9fdc62fa84677651e1b694bf4
+Merge: 955213635f 833c28e94f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 11:04:46 2017 -0700
+
+    Merge pull request #17626 from badone/wip-jewel-ceph-osd-flush-segfault
+    
+    jewel: core: kv: let ceph_logger destructed after db reset
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 955213635f1dd84b966a67e930e99a93d7880b51
+Merge: a536aa9878 c9b42c84e1
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Sep 12 15:50:42 2017 -0500
+
+    Merge pull request #17677 from mslovy/wip-disable-kstore-test
+    
+    jewel: qa: disable kstore to fix false positive case
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a536aa98789f1c22e7d5aad241d1090ceb62f9dc
+Merge: 5c5085c7fc 0215989814
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:12:13 2017 +0200
+
+    Merge pull request #17597 from smithfarm/wip-21186-jewel
+    
+    jewel: rgw: rgw_file:  incorrect lane lock behavior in evict_block()
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5c5085c7fc1309dc1678b520d88b6b0a9d246116
+Merge: 7429f50bb5 81810c1c7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:11:33 2017 +0200
+
+    Merge pull request #17287 from smithfarm/wip-21109-jewel
+    
+    jewel: rgw: send data-log list infinitely
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7429f50bb54c57356b98d103ae58aa85229c38c3
+Merge: adf2f8a5c9 9ab2e5fde0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:10:42 2017 +0200
+
+    Merge pull request #17285 from smithfarm/wip-20820-jewel
+    
+    jewel: rgw: Segmentation fault when exporting rgw bucket in nfs-ganesha
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit adf2f8a5c9bd9eb7ee30234e83893eea647753ff
+Merge: 694363a928 3c227e3f08
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:09:25 2017 +0200
+
+    Merge pull request #17281 from smithfarm/wip-20818-jewel
+    
+    jewel: rgw hangs in RGWRealmReloader::reload on SIGHUP
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 694363a928da9e0da3a284217a65c435fb44975d
+Merge: 9ac6c52bca cf65d63827
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:08:36 2017 +0200
+
+    Merge pull request #17280 from smithfarm/wip-20815-jewel
+    
+    jewel: rgw: uninitialized memory is accessed during creation of bucket's metadata
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9ac6c52bca7624ac9ec39b6f233ee86efe76fc11
+Merge: 6180494692 442911ebad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:07:48 2017 +0200
+
+    Merge pull request #17279 from smithfarm/wip-20813-jewel
+    
+    jewel: rgw: usage logging on tenated buckets causes invalid memory reads
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 618049469271eba8c819743688281e02e5009f08
+Merge: 4b37232d69 881bed7862
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:06:55 2017 +0200
+
+    Merge pull request #17277 from smithfarm/wip-20726-jewel
+    
+    jewel: rgw: user quota did not work well on multipart upload
+    
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+
+commit 4b37232d69083d506ba5b7d3fc00b6e91a5cde31
+Merge: c1449b9a34 443e167b91
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 12 16:05:16 2017 -0400
+
+    Merge pull request #17649 from dillaman/wip-21346
+    
+    jewel: qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit c1449b9a34687f873a6000f7787d047cfb3be447
+Merge: 9084ce8805 1516e24c85
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:04:20 2017 +0200
+
+    Merge pull request #17166 from smithfarm/wip-20719-jewel
+    
+    jewel: rgw: Truncated objects
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9084ce88053d5b256c175e0712f4a619002de742
+Merge: 89b00a8e16 c47e5abd0f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:03:30 2017 +0200
+
+    Merge pull request #17165 from smithfarm/wip-20715-jewel
+    
+    jewel: rgw: radosgw-admin data sync run crash
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 89b00a8e1699cebc016286a528173f04faa01573
+Merge: 568cc0c25b 9b5b21def2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:02:29 2017 +0200
+
+    Merge pull request #17164 from smithfarm/wip-20712-jewel
+    
+    jewel: rgw: not initialized pointer cause rgw crash with ec data pool
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 568cc0c25b1d5e047338e49aa7085c45ab0fbf3e
+Merge: 5b1e1cfae8 73c9d33ac0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:01:38 2017 +0200
+
+    Merge pull request #17159 from smithfarm/wip-20709-jewel
+    
+    jewel: rgw: radosgw-admin: bucket rm with --bypass-gc and without --purge-data doesn't throw error message
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5b1e1cfae89d31bf33b47ba79fb9b75d451e34cd
+Merge: fbae1127a6 e76198751c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:00:54 2017 +0200
+
+    Merge pull request #17156 from smithfarm/wip-20673-jewel
+    
+    jewel: rgw: multisite: RGWRadosRemoveOmapKeysCR::request_complete return val is wrong
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit fbae1127a6e7a9f4dd0822a91c8a06e5b95e3c52
+Merge: f64a82ad4d 26541c07b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:58:54 2017 +0200
+
+    Merge pull request #17148 from smithfarm/wip-20292-jewel
+    
+    jewel: rgw: multisite: log_meta on secondary zone causes continuous loop of metadata sync
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f64a82ad4d931978a8cf53ca81190434da8017f6
+Merge: bd918be29c 38718ace48
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:56:44 2017 +0200
+
+    Merge pull request #17147 from smithfarm/wip-20290-jewel
+    
+    jewel: rgw: rgw_file: prevent conflict of mkdir between restarts
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit bd918be29cd31f0ef6dac64edbfaf40f38ae2e07
+Merge: 37dec0227a ff67388e24
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:55:14 2017 +0200
+
+    Merge pull request #16856 from prallabh/wip-20895
+    
+    jewel: rgw: bucket index check in radosgw-admin removes valid index
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 37dec0227a9522e730baabb9c7428f007ba9c202
+Merge: f4734fada4 5a7ea2790c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:53:38 2017 +0200
+
+    Merge pull request #16767 from linuxbox2/jewel-rgw-mpart-race
+    
+    jewel: rgw : fix race in RGWCompleteMultipart
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c9b42c84e1f6850adea1cb9ba52f4a2a39fe8619
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Tue Sep 12 18:16:29 2017 +0000
+
+    qa: disable kstore to fix false positive case
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+
+commit f4734fada49ee6c2d4db9cd3e1e07d6bffe54071
+Merge: d3388d4601 3e70f3d662
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 18:18:40 2017 +0200
+
+    Merge pull request #17574 from smithfarm/wip-21297-jewel
+    
+    jewel: libradosstriper processes arbitrary printf placeholders in user input
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c47e5abd0f6583bb6cf2a676e6b1194d29a96780
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Jun 27 10:53:30 2017 +0800
+
+    rgw: fix radosgw-admin data sync run crash
+    
+    If sync thread have run before, then run data sync init. sync_status is still remain in rados pool. so no matter sync_status exists or not, if state is StateInit, sync_status.sync_info.num_shards should be updated.
+    
+    Fixes: http://tracker.ceph.com/issues/20423
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c307910d7131fc290f00bb8e33876e667afb72ec)
+    
+    Conflicts:
+        src/rgw/rgw_data_sync.cc (no data_sync_module or instance_id in jewel)
+
+commit d3388d4601d12c7c1e1187251c7221bbd8b58fbb
+Merge: 0277feb7d7 0a3227876c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:36:50 2017 +0200
+
+    Merge pull request #16299 from smithfarm/wip-20262-jewel
+    
+    jewel: rgw: datalog trim can't work as expected
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0277feb7d74a91ad7e32a7b42e3bf98d5072e72e
+Merge: a683b04589 c90969031f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:34:36 2017 +0200
+
+    Merge pull request #17167 from smithfarm/wip-20720-jewel
+    
+    jewel: multisite: RGWPeriodPuller does not call RGWPeriod::reflect() on new period
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f86dc698443bbaa6e2ad263875a17d0bccdff336
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Sep 12 09:30:37 2017 +0200
+
+    tests: use XFS explicitly in singleton-nomsgr/pool-access.yaml
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a7e1ee22b03abc40188f3745ad86534df78d757e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 24 16:46:00 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr: fix syntax
+    
+    This parsed out as
+    
+      tasks:
+      - install: null
+      - ceph:
+          conf:
+            osd: osd max object name len = 400 osd max object namespace len = 64
+      - workunit:
+          clients:
+            all:
+            - rados/test_health_warnings.sh
+    
+    which is clearly not correct.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from 85e2f3f31d25dbfd9770cc3b1f232025df53b628)
+    
+    Conflicts:
+        applied changes to pool-access.yaml instead of health-warnings.yaml
+
+commit a683b04589763681553078bc31e7a838614bb910
+Merge: 12fa3ec34e 7966b846a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:25:22 2017 +0200
+
+    Merge pull request #15556 from cbodley/wip-19847
+    
+    jewel: rgw: multisite: fixes for meta sync across periods
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 12fa3ec34e61e43525a013920ff89757a48bb0f2
+Merge: 151c9da33d 71262f1fae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:24:34 2017 +0200
+
+    Merge pull request #17155 from smithfarm/wip-20641-jewel
+    
+    jewel: rgw: multisite: lock is not released when RGWMetaSyncShardCR::full_sync() fails to write marker
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 151c9da33db8c59e5a9b5d7a0a97dd517322ceb1
+Merge: c049777c74 8b52105701
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:23:43 2017 +0200
+
+    Merge pull request #17278 from smithfarm/wip-20728-jewel
+    
+    jewel: rgw: multipart parts on versioned bucket create versioned bucket index entries
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 26541c07b4d107fa7b89e2ea7da821d1aacf22ca
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 22 13:06:13 2017 +0300
+
+    rgw: we no longer use log_meta
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac8b0077c1f53ff0037c4163489b838eebf8c247)
+
+commit 63df57807486a3b2fd6c4ec154f2d70fa9e88948
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 22 10:17:37 2017 +0300
+
+    rgw: is_single_zonegroup doesn't use store or cct
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 0b0090cee249c5380efa4bb416d724a7e2dbfc08)
+
+commit 3eb913e488c914d86eca2a4d31b050ea18914fa2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jun 21 12:37:21 2017 +0300
+
+    rgw: log_meta only for more than one zone
+    
+    Fixes: http://tracker.ceph.com/issues/20357
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 088dba03ccc65609ef1c51306389ebcf1126ec8c)
+
+commit 381d85d14dc48bcea83fd10010ea1080af0e5c55
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Jun 10 18:12:52 2017 -0400
+
+    rgw: only log metadata on metadata master zone
+    
+    Fixes: http://tracker.ceph.com/issues/20244
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b8272f3607074a2f7cbfd08f7bbc82f22cf120ba)
+
+commit c049777c74710bf342379e8995b31ee02282b61d
+Merge: 9afc62a7cc 787ba33e5d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:05:04 2017 +0200
+
+    Merge pull request #16296 from smithfarm/wip-20267-jewel
+    
+    jewel: rbd: api: is_exclusive_lock_owner shouldn't return -EBUSY
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9afc62a7cc124dc92872779b18356aec85736f61
+Merge: dd0ead4815 475dda114a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:04:09 2017 +0200
+
+    Merge pull request #17385 from dillaman/wip-19957
+    
+    jewel: librbd: reacquire lock should update lock owner client id
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit dd0ead48154bbe8c0775ab2d35dec00f22c15990
+Merge: 18085bd997 c755a0fccd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:03:29 2017 +0200
+
+    Merge pull request #17402 from dillaman/wip-20515
+    
+    jewel: librbd: fail IO request when exclusive lock cannot be obtained
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18085bd9973fb2f0d0eb2b2e2209e9e193270536
+Merge: ac27f23fa6 f9acf56dce
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:02:49 2017 +0200
+
+    Merge pull request #17412 from dillaman/wip-18704
+    
+    jewel: librbd: prevent self-blacklisting during break lock
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 443e167b91cec0d5c7ed51ce686d6bb3ccd8d036
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 7 08:55:27 2017 -0400
+
+    qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Fixes: http://tracker.ceph.com/issues/21181
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 273c84578b06905229a7c6eae7b1a12bbe60de95)
+    
+    Conflicts:
+            qa/workunits/rbd/import_export.sh: trivial resolution
+
+commit a0755a760a2d5017604564389240a9c975408b30
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 22 12:53:51 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr/pool-access: behave on ext4
+    
+    We may land on an ext4 root partition.
+    
+    Fixes: http://tracker.ceph.com/issues/20043
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from 657453d34914832f6e8012fbd69200e9680bd9ff)
+    
+    Conflicts:
+        applied the changes to pool-access.yaml instead of health-warnings.yaml
+        to address a specific test failure in the jewel branch
+
+commit a2fd5279992770f111710edd55e323afd5803445
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 11 12:13:35 2017 +0800
+
+    tasks/ceph: construct CephManager earlier
+    
+    Previously, if errors occurred during healthy(), then
+    the finally block would invoke osd_scrub_pgs, which relies
+    on CephManager being constructed, and it would die, hiding
+    the original exception.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c444db12d455a1901da8041e92f5eff1a9875170)
+    
+    Conflicts:
+            qa/tasks/ceph.py: the `tasks` directory was moved into
+                    `qa` after the cherry-picked change was merged.
+                    so apply the change manually to the ceph.py
+                    under `qa` directory.
+    (cherry picked from commit bc71dabb4e311bd2461489e98e9b95d5b635effc)
+
+commit 833c28e94f9a71ddfdc8afbdfacc9627a1e476d6
+Author: wumingqiao <wumingqiao@inspur.com>
+Date:   Tue Jul 25 14:45:09 2017 +0800
+
+    kv: let ceph_logger destructed after db reset
+    
+    if ceph_logger is deleted earlier than db, it may still be used by db, which cause a segment fault.
+    
+    Signed-off-by: wumingqiao <wumingqiao@inspur.com>
+    (cherry picked from commit a5cd03c643d6cb9074dfd2952cde83435de1b9dd)
+
+commit ac27f23fa697c7ad9be897efc04af95e3908b5b3
+Merge: bcc85d178e 72379a4930
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 10 21:38:15 2017 +0200
+
+    Merge pull request #15189 from mslovy/wip-19996-jewel
+    
+    jewel: osd: fix infinite loops in fiemap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bcc85d178e5900acbe1eaaf28de43ca53dadbb7e
+Merge: 2203d42ec6 314952c589
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 10 21:15:35 2017 +0200
+
+    Merge pull request #16711 from cbodley/wip-20346
+    
+    jewel: rgw: meta sync thread crash at RGWMetaSyncShardCR
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2203d42ec656b12c23de50874b36289354338aaa
+Merge: f212b361df 2d64d090c0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 14:05:18 2017 -0400
+
+    Merge pull request #17552 from dillaman/wip-21279
+    
+    jewel: test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 02159898147053fa492ec597bceb9096d160be08
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 25 17:55:11 2017 -0400
+
+    rgw_file:  fix LRU lane lock in evict_block()
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/21141
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3301596734adcbda1e8e76a742935efdeb7518fd)
+
+commit f212b361dfaed7f67e4b994b1f22b3c9897eeab2
+Merge: 26db2e7ff9 5adfc653dc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 18:54:17 2017 +0200
+
+    Merge pull request #17514 from smithfarm/wip-21232-jewel
+    
+    jewel: core: interval_set: optimize intersect_of insert operations
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 26db2e7ff93eb351e0963e8ae25548367e2f63ff
+Merge: f4a0d00032 1bd4df1d7f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 18:39:58 2017 +0200
+
+    Merge pull request #16293 from smithfarm/wip-20460-jewel
+    
+    jewel: core: test_envlibrados_for_rocksdb.yaml fails on crypto restart
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c755a0fccd007f95d709f124e931ab9758a6c93d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:20:26 2017 -0400
+
+    qa/suites/rbd: fixed cache override
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98061bb3d7ce6309ddb04ea4d7e9d44a7ecd09c6)
+
+commit 3e70f3d662e3c051352e342fe89cd3b2186f4cb5
+Author: Jesse Williamson <jwilliamson@suse.de>
+Date:   Tue Jun 13 12:31:16 2017 -0700
+
+    libradosstriper: remove format injection vulnerability
+    
+    Fixes: http://tracker.ceph.com/issues/20240
+    
+    Signed-off-by: Stan K <redrampage@selectel.ru>
+    (cherry picked from commit e94d3b9661d287e500cdf4e6c102d2e3bb04475e)
+
+commit f4a0d000322992b9c67f51830a8396269a5caa1e
+Merge: 00d09fce6a 1415ecbd7d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 17:41:58 2017 +0200
+
+    Merge pull request #17084 from badone/wip-ceph-disk-dmcrypt-key-jewel
+    
+    jewel: ceph-disk: Use stdin for 'config-key put' command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 00d09fce6a81bf7c8cd62caa5e0a3de44bd0ad84
+Merge: b014f3995d cd848c1e98
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 17:40:34 2017 +0200
+
+    Merge pull request #17008 from asheplyakov/19182-bp-jewel
+    
+    jewel: mon: fix force_pg_create pg stuck in creating bug
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a4c3ef5276eb7d232f5706e6fe07b0808ee581d4
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 29 19:25:43 2017 +1000
+
+    osd: Reverse order of op_has_sufficient_caps and do_pg_op
+    
+    Fixes: http://tracker.ceph.com/issues/19790
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit a921882e7c676d68ed67b54301b49fd9bed42945)
+    
+    Conflicts:
+            qa/suites/rados/singleton-nomsgr/all/pool-access.yaml - drop mgr.x
+                role, which is not needed in jewel and might even cause the test to
+                fail
+            src/osd/PrimaryLogPG.cc - this file doesn't exist in jewel, so apply
+                the change manually to ReplicatedPG.cc (i.e., by moving the
+                op->includes_pg_op() conditional below the
+                !op_has_sufficient_caps(op) conditional)
+
+commit b014f3995d7913fe5266c2e7d06f93f296e98991
+Merge: c3623149ef 53e97da166
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Sep 7 22:51:10 2017 +0800
+
+    Merge pull request #17133 from smithfarm/wip-21035-jewel
+    
+    jewel: ceph-disk: systemd unit timesout too quickly
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c3623149efc2b25a3509ec26b671d835a81b8c28
+Merge: 3aede767bd 713a88a5c9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 16:33:08 2017 +0200
+
+    Merge pull request #16316 from smithfarm/wip-20362-jewel
+    
+    jewel: rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 72379a4930edfc917a15ea1ce5ba67ec5093fd96
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 15 17:29:45 2016 -0400
+
+    ceph_test_objectstore: disable filestore_fiemap
+    
+    This very reliably triggers a test failure for
+    ObjectStore/StoreTest.Synthetic/1.
+    
+    FIEMAP is bad!  Do not use it!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 2d64d090c05be7792c934d8a3e060e48e68da129
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 21:37:16 2017 -0400
+
+    test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Fixes: http://tracker.ceph.com/issues/21217
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aeb8f29d21625f3570e2f3e6dd26594d1a6df810)
+
+commit 3aede767bdc391fe6521066aade80369081d23cc
+Merge: b2accb83ac e35c938208
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 10:21:10 2017 +0200
+
+    Merge pull request #16703 from ddiss/wip-20837-jewel
+    
+    jewel: ceph-disk: don't activate suppressed journal devices
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit b2accb83ac46a54669c4d0a603464ff117a66f06
+Merge: d7cc49d995 96de9d9078
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:47:41 2017 +0200
+
+    Merge pull request #17396 from smithfarm/wip-21113-jewel
+    
+    jewel: cephfs: get_quota_root sends lookupname op for every buffered write
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit d7cc49d9950917e7afb6c14b4b063b51e7e136b5
+Merge: ae480caf68 3adbc35205
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:46:00 2017 +0200
+
+    Merge pull request #17009 from dreamhost/wip-16463-jewel
+    
+    jewel: rgw: aws4: add rgw_s3_auth_aws4_force_boto2_compat conf option
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ae480caf68db8e6793c8f3db663e94215c9f75ea
+Merge: d4797bf193 3242b2b562
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:42:54 2017 +0200
+
+    Merge pull request #16963 from smithfarm/wip-20972-jewel
+    
+    jewel: cephfs: ceph-fuse segfaults at mount time, assert in ceph::log::Log::stop
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d4797bf193354199a3182cc2c07ac46c76d506ba
+Merge: 02fef5c436 788cfa89dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:41:19 2017 +0200
+
+    Merge pull request #16952 from dreamhost/wip-20966-jewel
+    
+    jewel: rgw: radosgw-admin: fix bucket limit check argparse, div(0)
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 02fef5c43687437a4b085b9f31d2e3eb982e637c
+Merge: d5e77009ac 6bd2dae193
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:34:52 2017 +0200
+
+    Merge pull request #16951 from linuxbox2/jewel-aws4-plus
+    
+    jewel: rgw: replace '+' with "%20" in canonical query string for s3 v4 auth
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d5e77009acf009f1280d352ef6d0f8b6731fd3e8
+Merge: 7bdbd075a1 37b37e78f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:33:25 2017 +0200
+
+    Merge pull request #16880 from linuxbox2/jewel-rgw-dir-suggest
+    
+    jewel: rgw: Fix up to 1000 entries at a time in check_bad_index_multipart
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 7bdbd075a11b9a8cd05eb0fd55d4787c554ac9a6
+Merge: 18e5db07f1 187e9eaa30
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:28:03 2017 +0200
+
+    Merge pull request #16720 from prallabh/wip-jewel-20821
+    
+    jewel: rgw: Do not decrement stats cache when the cache values are zero
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5adfc653dc201ec4faa60a5f78431ab68f0f4e19
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Sun Aug 27 05:25:01 2017 -0700
+
+    interval_set: optimize intersect_of for identical spans
+    
+    Optimize comparisons for identical spans of intervals.
+    When this patch is combined with the previous map insert
+    optimization, a benchmark using 400000 identical
+    intervals shows a 7 times performance improvement in
+    comparison to without the patches.
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit b6a035666c2765f8895ee9991348dbc025613ed7)
+
+commit 54dfe8733b32401a5a561bf544d91f44a41e378f
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Fri Aug 25 09:41:07 2017 -0700
+
+    interval_set: optimize intersect_of insert operations
+    
+    Use the std::map insert method with hint iterator to optimize
+    inserts. This increases performance more than 3.5 times for
+    large numbers of intervals. This will help performance
+    especially in the PGPool::update method, where profiling data
+    has shown that intersection operations are a hot spot. The
+    following benchmark data is for 400000 intervals:
+    
+        4 +-+--+----+----+----+----+----+----+----+----+--+-+
+    P     +    +    +    +    +    +    +    +  *************
+    E     |                             ********            |
+    R 3.5 +-+                       ****                  +-+
+    F     |                   ******                        |
+    O     |                 **                              |
+    R   3 +-+           ****                              +-+
+    M     |          ***                                    |
+    A     |        **                                       |
+    N 2.5 +-+     *                                       +-+
+    C     |     **                                          |
+    E     |     *                                           |
+        2 +-+ **                                          +-+
+    R     |  **                                             |
+    A     | **                                              |
+    T 1.5 +**                                             +-+
+    I     |**                                               |
+    O     +*   +    +    +    +    +    +    +    +    +    +
+        1 +*+--+----+----+----+----+----+----+----+----+--+-+
+          0   0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9
+    
+                            SET SIZE RATIO
+    
+    The above chart was generated using benchmark results
+    from the following program:
+    
+    #include <iostream>
+    #include <sys/timeb.h>
+    #include "include/interval_set.h"
+    
+    int main(int argc, char *argv[])
+    {
+      const int interval_count = std::stoi(argv[1]);
+      const int interval_distance = 4;
+      const int interval_size = 2;
+      const int sample_count = 8;
+      const int max_offset = interval_count * interval_distance;
+      interval_set<int> a, b, intersection;
+    
+      for (int i = 0; i < max_offset; i+=interval_distance) {
+        a.insert(i, interval_size);
+      }
+    
+      for (int m = 1; m < 100; m++) {
+        float ratio = 1 / float(m);
+    
+        for (int i = 0; i < max_offset; i+=interval_distance*m) {
+          b.insert(i, interval_size);
+        }
+    
+        struct timeb start, end;
+        int ms = 0;
+        for (int i = 0; i < sample_count; i++) {
+          ftime(&start);
+          intersection.intersection_of(a, b);
+          ftime(&end);
+          ms += (int) (1000.0 * (end.time - start.time)
+            + (end.millitm - start.millitm));
+          intersection.clear();
+        }
+        b.clear();
+    
+        std::cout << ratio << "\t" << ms << std::endl << std::flush;
+      }
+    }
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit 32bc0430f70b057d1bba623252e92ab9f279028d)
+
+commit 18e5db07f1e81eec8a2c8483c7bcc876be0d19c9
+Merge: 44cd2b19b5 178245d543
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 6 09:36:13 2017 +0200
+
+    Merge pull request #15966 from prallabh/jewel
+    
+    jewel: rgw: Custom data header support
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 44cd2b19b5ee603a96b53e8286cfc0cbd6e1fc76
+Merge: c93d2c6cd6 d90b1b3395
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 6 09:03:50 2017 +0200
+
+    Merge pull request #16144 from smithfarm/wip-20364-jewel
+    
+    jewel: mon: osd crush set crushmap need sanity check
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1bd4df1d7f16b6cca02abb791acc298a1e6ec592
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 24 00:50:40 2017 +0800
+
+    osdc/Objecter: release message if it is not handled
+    
+    Fixes: http://tracker.ceph.com/issues/19741
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5c2774234c20d809d56d005b4db34070f3029684)
+    
+    Conflicts:
+       src/osdc/Objecter.h (in master, ms_fast_dispatch function declaration has an
+                           "override" specifier; in jewel, this specifier is missing)
+
+commit 4e4b62b4c9ec8dc5eea025af31ae5761fdd127dd
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jun 21 14:25:01 2017 +0800
+
+    crypto: allow PK11 module to load even if it's already initialized
+    
+    there is chance that other pieces of application loads PK11 module
+    already and does not finalize it before calling common_init_finish().
+    
+    also, upon fork, PK11 module resets its entire status including `nsc_init`,
+    by which PK11 module tell if it is initialized or not. so the behavior
+    of NSS_InitContext() could be different before and after fork. that's
+    another reason to ignore CKR_CRYPTOKI_ALREADY_INITIALIZED error (see
+    NSS_GetError()).
+    
+    Fixes: http://tracker.ceph.com/issues/19741
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit fcc3effd8b447ef0c54b4c806b8f6e996d7467dd)
+
+commit c93d2c6cd65273eef4e9187269e5d44622af9a1c
+Merge: e9eda5aa28 92eb6836bf
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:13:53 2017 -0700
+
+    Merge pull request #17210 from dzafman/wip-20730-jewel
+    
+    jewel: core: disable skewed utilization warning by default
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e9eda5aa285ebf989c598b248c2c8d4fc660c6fa
+Merge: e4f02b2bef 153f775441
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:12:23 2017 -0700
+
+    Merge pull request #16405 from dzafman/wip-20041-jewel
+    
+    jewel: osd: scrub_to specifies clone ver, but transaction include head write ver
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e4f02b2befa9203a51c7f146c85539180953e451
+Merge: baa5183c52 97fdaa753e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:11:21 2017 -0700
+
+    Merge pull request #16169 from Vicente-Cheng/wip-20511-jewel
+    
+    jewel: osd: ReplicatedPG: solve cache tier osd high memory consumption
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit baa5183c524fc7af83dd46645a48e24464ab7c6e
+Merge: 81097b6fa1 f49c9c777f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 10:55:32 2017 -0700
+
+    Merge pull request #16167 from Vicente-Cheng/wip-20492-jewel
+    
+    jewel: osd: omap threadpool heartbeat is only reset every 100 values
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 81097b6fa1543e332f0da7cd9857cb4877c8eee1
+Merge: d639aa9136 932342c1e3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 21:10:57 2017 +0200
+
+    Merge pull request #16151 from Vicente-Cheng/wip-20349-jewel
+    
+    jewel: cephfs: df reports negative disk "used" value when quota exceed
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d639aa91369c6af4956a5186dd11bfb903f01421
+Merge: b6b08b72f0 6b6620ffef
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 21:09:55 2017 +0200
+
+    Merge pull request #16150 from Vicente-Cheng/wip-20403-jewel
+    
+    jewel: mds: save  projected path into inode_t::stray_prior_path
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b6b08b72f0516b74f502bcc98ad98d01b2012f7d
+Merge: bab324815d dac0ca797c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 12:54:34 2017 +0200
+
+    Merge pull request #16141 from smithfarm/wip-20270-jewel
+    
+    jewel: tests: LibRadosMiscConnectFailure.ConnectFailure hang
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bab324815da63ec395a4c036d451c7d3bb45fa99
+Merge: ecc5e960a0 3105f118bc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 17:16:57 2017 +0200
+
+    Merge pull request #16285 from smithfarm/wip-18137-jewel
+    
+    jewel: rbd-mirror: image sync should send NOCACHE advise flag
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit ecc5e960a0fdbcf9c0de7453626369938154d29c
+Merge: 0d53a36ea1 f9e87fef6a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 17:16:08 2017 +0200
+
+    Merge pull request #16295 from smithfarm/wip-20265-jewel
+    
+    jewel: rbd: cli: ensure positional arguments exist before casting
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0d53a36ea1b547f85172b79b84a61229af69d658
+Merge: 445833be4e 24bd60136e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:12:17 2017 +0200
+
+    Merge pull request #16276 from smithfarm/wip-19563-jewel
+    
+    jewel: rgw: folders starting with _ underscore are not in bucket index
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 445833be4e941e5eea106d558f02afa6dc201db2
+Merge: 182a578157 39b2b0b39a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:09:46 2017 +0200
+
+    Merge pull request #16268 from cbodley/wip-16072-jewel
+    
+    jewel: rgw: rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 182a5781575183fd74f6262b3d0f5137c356ec0c
+Merge: f61c562894 f16fa4b252
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:08:21 2017 +0200
+
+    Merge pull request #16266 from cbodley/wip-20408
+    
+    jewel: rgw: multipart copy-part remove '/' for s3 java sdk request header
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit f61c5628944f9b2b612f080d73296678a4a7da14
+Merge: 5d3c80eef4 00a7999552
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 12:50:50 2017 +0200
+
+    Merge pull request #15988 from cbodley/wip-jewel-15983
+    
+    jewel: rgw: fix infinite loop in rest api for log list
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5d3c80eef428b93edc7559709d4a2de7bd2750c1
+Merge: 6484a3bdab d3b34d3f9f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 12:48:00 2017 +0200
+
+    Merge pull request #15449 from smithfarm/wip-19720-jewel
+    
+    jewel: rgw: rgw_file: fix size and (c|m)time unix attrs in write_finish
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6484a3bdab941b077ae942ae207b11ac679aeedc
+Merge: bd2a0c80ab d0492ea07a
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 1 14:27:31 2017 -0500
+
+    Merge pull request #17351 from liewegas/wip-prune-past-intervals-jewel
+    
+    osd: add osd_hack_prune_past_intervals
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f9acf56dce26bb031cddb6676b2834b6bd426ae2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 21:30:47 2017 -0400
+
+    librbd: prevent self-blacklisting during break lock
+    
+    (derived from commit 5c590acaec4dd66a9a8c3aa0ec8ab904dd350216)
+    
+    Fixes: http://tracker.ceph.com/issues/18704
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d90b1b339542408fb053571b64e705be208cb3be
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 18 10:04:20 2017 +0100
+
+    mon: osd crush set crushmap need sanity check
+    
+    The sanity check verifying the new crushmap does not remove crush rules
+    that are in use is not exclusive to ceph setcrushmap.
+    
+    Fixes: http://tracker.ceph.com/issues/19302
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed760457bf154c10adf75c6df046eecab7eb8e4b)
+
+commit 2e7ddedf8acdd7a19753a0c376e9f29535329843
+Author: songbaisen <song.baisen@zte.com.cn>
+Date:   Thu Mar 31 11:39:44 2016 +0800
+
+     crush: when take place the crush map should consider the rule is in used
+    
+     Signed-off-by: song baisen song.baisen@zte.com.cn
+    
+    (cherry picked from commit 09bf6f2858d09c8c314a8c242a9c70d2834718dc)
+
+commit 26984b0b0506958da340b94e3a05eac348f20f2a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 08:03:03 2017 -0400
+
+    librbd: fix missing write block validation in IO work queue
+    
+    (derived from commit 8e76ebd93fd7525f6909a7da178e9b92c98e1ebe)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 79f220d08063077b08cda3acc447f30790ee9dfd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 27 14:05:58 2017 -0400
+
+    qa/suites/rbd: test dynamic features with cache disabled
+    
+    The journaling logic follows different paths depending on the
+    cache configuration.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57a507d845339e9052122755bd460eb6d61d6060)
+
+commit 412f75a8f8203f236c1eb9f52da8876e476a052f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 27 14:03:34 2017 -0400
+
+    qa/tasks/qemu: rbd cache is enabled by default
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fdc4c9221f2c54d5fb55e74491ebc5165f73e776)
+
+commit edf8b584f0856f6181579aa1b531963ad06ebbea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 09:48:47 2017 -0400
+
+    test: unit tests for librbd IO work queue failure path
+    
+    (derived from commit 6e23ef358f29ca65a2a6b08b133416415e117b06)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1c8bd5f44a0a006094235254504772ae8eb90547
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 09:31:02 2017 -0400
+
+    librbd: cleanup interface between IO work queue and IO requests
+    
+    This will help mock the testing of the IO work queue
+    
+    (derived from commit e0834d12e91bf085a5c9503f1c69a2bee2d21edf)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 75f0bb9e7bbdea0fa6d15d253139f0583be6abea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 22 13:31:10 2017 -0400
+
+    common: improve the ability to mock PointerWQ classes
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2d90bd8fdb30705d05569d242fc0cc4e36a2c7db)
+    
+    Conflicts:
+            src/common/WorkQueue.h: trivial resolution
+            src/librbd/AioImageRequestWQ.cc: file renamed
+
+commit fbfafb75c70a1455e102bf09623e1f1d8cf2074f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 07:51:37 2017 -0400
+
+    librbd: exclusive lock failures should bubble up to IO
+    
+    (derived from commit 048d475127b600b6a40bd5e0c3a0daf8133294f4)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8fb8eb61dd8b2e44cb10da9b28fa334267784e2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 22:08:15 2017 -0400
+
+    librbd: directly inform IO work queue when locks are required
+    
+    Due to lock dependency issues between librbd locks and the the thread
+    pool lock, it's impossible to directly determine if the lock is
+    required within the _void_dequeue method. Therefore, this state needs
+    to be internally tracked by the work queue.
+    
+    (derived from commit 4a525671b3541a0a208dd039ac96f42bc8fca2cc)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4509765fee20aab082dfbb2ba02cf76f796d6292
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 21:16:56 2017 -0400
+
+    librbd: clean up variable naming in IO work queue
+    
+    (derived from commit 9d8ae624858beb6fdc8ee2674b8e8d9b9cf14cbe)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 08c63b3d0c8a3c68f68ded0b9122c5e094d362ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 08:25:29 2017 -0400
+
+    librbd: convert ImageRequestWQ to template
+    
+    (derived from commit b9569585fab5654b6c29552c5eb009dcddfaaa3b)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 42780a8228da1440d7f3f44a09cddf3b88b90223
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Mon Aug 28 15:48:50 2017 +0000
+
+    os:kstore fix unittest for FiemapHole
+    
+    kstore always return [0, object_size] regardless of offset and length
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit dddae89fa7956fdfca1c09910b4e5b42ecbd22cf)
+
+commit 96de9d9078e5f638c74fdc68322ac4a199011597
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Thu Aug 10 11:33:46 2017 +0200
+
+    client: skip lookupname if writing to unlinked file
+    
+    When a client writes to an unlinked file, get_quota_root tries to
+    resolve the filename via an MDS lookupname op. The op always results in
+    -13 permission denied for path-restricted caps or -2 no such file or
+    directory otherwise. More importantly, the repeated lookupname ops slow
+    down buffered writes significantly.
+    
+    Don't do the lookupname for unlinked files; use the root_ancentor's
+    quota instead.
+    
+    Fixes: http://tracker.ceph.com/issues/20945
+    Backport: jewel, luminous
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1d617eee2e8c32f7300cb2cb8dd37b3714a4699f)
+
+commit bd2a0c80ab1827f1ecdb349b6ebc65c4ad519322
+Merge: 05df53404b e9e7850b51
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 31 10:24:21 2017 +0200
+
+    Merge pull request #16059 from vumrao/wip-vumrao-20353
+    
+    jewel: osd: Add heartbeat message for Jumbo Frames (MTU 9000)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 05df53404b0e05cefe46d30174df71b53714f7f2
+Merge: 1b3c3c7f65 0068d9f333
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 31 10:23:00 2017 +0200
+
+    Merge pull request #15726 from asheplyakov/20314-bp-jewel
+    
+    jewel: mon: fail to form large quorum; msg/async busy loop
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 475dda114a7e25b43dc9066b9808a64fc0c6dc89
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 20:59:20 2017 -0400
+
+    librbd: reacquire lock should update lock owner client id
+    
+    (derived from commit 21ce5e16703776cbae20ac981dff4ec7cc2bc9c8)
+    
+    Fixes: http://tracker.ceph.com/issues/19957
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d0492ea07abcc8652f9c713deaae792ef68dc491
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 24 18:08:49 2016 -0500
+
+    osd: add osd_hack_prune_past_intervals
+    
+    Last ditch (but dangerous) method of reducing memory usage for
+    past_intervals, which can help very very unhappy clusters recovery.
+    A properly implemented version of this is in luminous.  This hacky
+    version was used successfully to recover multiple jewel-based clusters,
+    but is still only recommended for use when the OSD is otherwise unable
+    to recover.
+    
+    This change is not cherry-picked from master because luminous implements
+    a more sophisticated version of this that changes the past intervals
+    representation entirely, but it is too invasive to backport.  This
+    workaround to prune just on startup should be sufficient for emergencies.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1b3c3c7f655b53b205a81d48d3fa5e6072d4cec5
+Merge: 4fcd7eea45 4be3db5973
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 18:51:27 2017 +0200
+
+    Merge pull request #15719 from Vicente-Cheng/wip-20146-jewel
+    
+    jewel: rgw: 'gc list --include-all' command infinite loop the first 1000 items
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 4fcd7eea45bc5158fd04b3b2bd8da717a59932b5
+Merge: ed19fdfc1c b786f7815b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 18:50:21 2017 +0200
+
+    Merge pull request #15477 from smithfarm/wip-19767-jewel
+    
+    jewel: rgw: Delete non-empty bucket in slave zonegroup
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ed19fdfc1ce39f9cf127748e607ed0439a209f7c
+Merge: da648d66eb 7bf8b854ab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:32:21 2017 +0200
+
+    Merge pull request #16015 from asheplyakov/19208-bp-jewel
+    
+    jewel: osd: osd_internal_types: wake snaptrimmer on put_read lock, too
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da648d66eb67b608d7812d6b75c77df403c21fae
+Merge: 879a650f20 52a00b3bff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:31:05 2017 +0200
+
+    Merge pull request #15762 from smithfarm/wip-17385-jewel
+    
+    jewel: ceph cli: Rados object in state configuring race
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 879a650f20ae74df389dfc58720c119bd6ee7330
+Merge: 2a8e0f6695 21b00c3ec0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:28:29 2017 +0200
+
+    Merge pull request #15474 from smithfarm/wip-19559-jewel
+    
+    jewel: osd: objecter full_try behavior not consistent with osd
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2a8e0f6695c3f32c6e2a4c1230dce1db037a355a
+Merge: c40316e9d3 3fa277b479
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 09:15:37 2017 +0200
+
+    Merge pull request #15947 from asheplyakov/20428-bp-jewel
+    
+    jewel: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Liu-Chunmei <chunmei.liu@intel.com>
+    Reviewed-by: Xie Xingguo <xie.xingguo@zte.com.cn>
+
+commit c40316e9d3c4c230da234ffdeac6e4a3307a9243
+Merge: 3ec5cd9c61 648dfa1e93
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 09:04:24 2017 +0200
+
+    Merge pull request #15475 from tchaikov/wip-16239-jewel
+    
+    qa: add a sleep after restarting osd before "tell"ing it
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3ec5cd9c61294a4b5ed4d17357f60b1544beb532
+Merge: 0916352610 5adc66bc2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 08:56:01 2017 +0200
+
+    Merge pull request #15473 from smithfarm/wip-20080-jewel
+    
+    jewel: build/ops: rpm: set subman cron attributes in spec file
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Thomas Serlin <tserlin@redhat.com>
+
+commit 0916352610200ddd2a9b6bbe2dbba5d8ca4d470a
+Merge: 988559b4e2 36b1a35a44
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:34:52 2017 +0200
+
+    Merge pull request #15460 from smithfarm/wip-17843-jewel
+    
+    jewel: rbd: object-map: batch updates during trim operation
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 988559b4e286360b3cc10ac0c6aa9238b98884dd
+Merge: 5d8ea26ebb b9ce1aa618
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:33:13 2017 +0200
+
+    Merge pull request #15488 from dillaman/wip-20023-jewel
+    
+    jewel: rbd: rbd-mirror: ensure missing images are re-synced when detected
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 5d8ea26ebb51656ef435b3ae932bb172af9407f7
+Merge: 9d1b08e619 5c53ae3f9a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:32:04 2017 +0200
+
+    Merge pull request #15602 from tchaikov/wip-20175-jewel
+    
+    jewel: tests: decouple ceph_test_librbd_api from libceph-common
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9d1b08e6191d55027e9573ab721a320a67453d24
+Merge: cccacd7e0f 61f132078a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:31:09 2017 +0200
+
+    Merge pull request #16124 from smithfarm/wip-19174-jewel
+    
+    jewel: rbd: rbd_clone_copy_on_read ineffective with exclusive-lock
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 178245d543d680bbdc7f2da07c431fc745da1143
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Fri Apr 14 21:42:45 2017 +0530
+
+    rgw: add a field to store generic user data in the bucket index,
+    that can be populated/fetched via a configurable custom http header
+    
+    Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
+    (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc
+                    Number of arguments to RGWPutObjProcessor_Multipart::do_complete has been changed in master with an extra field accounted_size added, which is not required in Jewel.
+                    Number of arguments for the processor->complete() has been changed in master, which is not required in Jewel.
+    
+            src/rgw/rgw_op.h
+                    A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master, which is not required in Jewel.
+    
+            src/rgw/rgw_rados.cc
+                    In the function RGWPutObjProcessor::complete() there is an extra argument to do_complete() in master, which is not required in Jewel.
+                    In the function RGWPutObjProcessor_Atomic::do_complete, a call to obj_op.write_meta has accounted_size, which is not required in Jewel.
+                    There is an extra argument accounted_size in RGWRados::Bucket::UpdateIndex::complete in master, which is not required in Jewel.
+                    In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Jewel.
+                    In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta structure.
+                    In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.
+    
+            src/rgw/rgw_rados.h
+                    In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Jewel instead of rgw_obj_index_key
+                    Corrected various function declarations as appropriate.
+                    RGWPutObjProcessor_Multipart is not part of this file in Jewel.
+    
+            src/rgw/rgw_rest_swift.cc
+                    In RGWListBucket_ObjStore_SWIFT::send_response(), accounted_size is not required in Jewel.
+    
+            src/rgw/rgw_common.h
+                    Add user_data field in RGWObjEnt structure.
+    
+            src/rgw/rgw_json_enc.cc
+                    Add user_data field while dumping RGWObjEnt.
+
+commit cccacd7e0fb7422e18020b919ad10bac8152b9e8
+Merge: 006804ab68 c2137f2f72
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 11:03:32 2017 +0200
+
+    Merge pull request #15760 from smithfarm/wip-20325-jewel
+    
+    libradosstriper silently fails to delete empty objects in jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 006804ab68818e73bdf822aa5994346b40cc6add
+Merge: 14a52865da 2efd7cd6f3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 10:50:27 2017 +0200
+
+    Merge pull request #16061 from weiqiaomiao/wqm-wip-ceph-disk-jewel
+    
+    jewel: ceph-disk: remove the special check to bcache devices
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+
+commit 14a52865da4de99b445560356533d7128df382d6
+Merge: d9bbf8c6e6 a3c487448b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 10:47:59 2017 +0200
+
+    Merge pull request #15503 from Vicente-Cheng/wip-20011-jewel
+    
+    jewel: tools: ceph-disk: separate ceph-osd --check-needs-* logs
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 187e9eaa30514c93cd8e982ff67ddb6502810af9
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Aug 21 17:04:45 2017 +0300
+
+    rgw: Prevent overflow of stats cached values
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Fixes: http://tracker.ceph.com/issues/20934
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 634215eea1ddd4e4f5dc0066c4a2e745cfc20475)
+    
+    Conflicts:
+            src/rgw/rgw_quota.cc
+                           Have the quota stats structures to reflect rounded values in KB and honor the signed integer values
+
+commit 59854a158e8df803013ba1266d414af67feebac1
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Tue Jul 18 14:40:04 2017 +0530
+
+    rgw: Do not decrement stats cache when the cache values are zero
+    
+    With RGWs configured in a load balancer, there is a possibility of
+    having the cached values going unbound, when PUT/DELETE operations
+    do not land up on the same RGW. To avoid such cases, make sure the
+    decrement of stats happen only when the cached values are sane.
+    
+    Fixes: http://tracker.ceph.com/issues/20661
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 3903e213c7ac7624e3452f5f3b1ca1c339bf2ca2)
+    
+    Conflicts:
+            src/rgw/rgw_quota.cc
+
+commit 81810c1c7aa03a353975db500abeb1163e38fde7
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Aug 9 11:48:53 2017 +0800
+
+    rgw: break sending data-log list infinitely
+    
+    radosgw send data-log list infinitely when opposite end trimmed the data-log and in quiescence.
+    Fixes: http://tracker.ceph.com/issues/20951
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 40db1fdbc7b24d56b2c7b463238e6b96f2d4bbd9)
+
+commit 9ab2e5fde02b705658d4e763519282cad3f0b173
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jul 20 08:18:22 2017 -0400
+
+    rgw_file: properly & |'d flags
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/20663
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f838c05c72ad46ce01884a916f81275ef43dacd)
+
+commit d3b34d3f9fece0d12c19132852dcb2c9b38e29d5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 18 09:19:13 2017 -0400
+
+    rgw_file:  pre-compute unix attrs in write_finish()
+    
+    New serialized Unix attrs need to reflect the change being made,
+    and should be reverted if the change fails.
+    
+    Fixes: http://tracker.ceph.com/issues/19653
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ed91d23ccaaac4e72a4c28a58e77485395949f04)
+    
+    Conflicts:
+        src/rgw/rgw_file.cc (processor->complete() takes different arguments in
+            jewel than in master)
+
+commit d9bbf8c6e6d6c2a8b978a5f31cdb0c5c52184d0f
+Merge: de27247717 ab2d3582f4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 18:10:39 2017 +0200
+
+    Merge pull request #14874 from ceph/wip-19805-jewel
+    
+    jewel: rbd: default features should be negotiated with the OSD
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit de2724771785b190cff88996bb09097413a08e84
+Merge: d3677e9f50 618a82e11e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 18:09:03 2017 +0200
+
+    Merge pull request #14977 from dillaman/wip-19873-jewel
+    
+    jewel: rbd: rbd-mirror: failover and failback of unmodified image results in split-brain
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d3677e9f5054362b70f9569a47967930b954a67c
+Merge: d47a2e05cf b63b7886dc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:53:15 2017 +0200
+
+    Merge pull request #15461 from smithfarm/wip-19795-jewel
+    
+    jewel: tests: test_notify.py: assert(not image.is_exclusive_lock_owner()) on line 147
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d47a2e05cf4004bb4d7ea2a1c20642d17e2f3b14
+Merge: 27fd5ec473 5c0986153e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:51:35 2017 +0200
+
+    Merge pull request #15463 from smithfarm/wip-20017-jewel
+    
+    jewel: rbd: rbd-nbd: kernel reported invalid device size (0, expected 1073741824)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 27fd5ec4739b87e260bd337ddfe9593943b5f802
+Merge: 2b759d2f74 bc8c5d6c77
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:49:36 2017 +0200
+
+    Merge pull request #15464 from smithfarm/wip-20153-jewel
+    
+    jewel: rbd: Potential IO hang if image is flattened while read request is in-flight
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 3c227e3f086cbfbfff23d11515d9261906c7219d
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Jul 19 18:49:11 2017 +0800
+
+    rgw: fix rgw hang when do RGWRealmReloader::reload after go SIGHUP
+    
+    Quota async processer reference count err when bucket has no explicit shard
+    
+    Fixes: http://tracker.ceph.com/issues/20686
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 957b9bdee2024723da240e488b17d777942ac77d)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - master has diverged from jewel; applied change
+            manually to the RGWGetBucketStatsContext ctor call
+
+commit cf65d638277b8cb36e70a0eae96dfbd47ca0bf37
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 08:16:11 2017 -0400
+
+    rgw: rgw_website.h doesn't assume inclusion of the std namespace anymore.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 06f18c163ec843f1fc852f078f539e1821644506)
+
+commit c348ec29947c034779937d6611f4d15545fd3e44
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 08:12:30 2017 -0400
+
+    rgw: never let http_redirect_code of RGWRedirectInfo to stay uninitialized.
+    
+    Fixes: http://tracker.ceph.com/issues/20774
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 6318b4990fe7fe8f25fff4755eee64fef069dee9)
+
+commit 442911ebad8a4b6f69fb9caa2e1f7e44486de166
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 09:25:36 2017 -0400
+
+    rgw: fix the UTF8 check on bucket entry name in rgw_log_op().
+    
+    Fixes: http://tracker.ceph.com/issues/20779
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit e9a787163080474a1d1a3b8db4db824161326982)
+
+commit 8b52105701e8bbc5e3eb2cc1b251e739376540df
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Thu Apr 13 17:14:48 2017 +0800
+
+    rgw: fix the bug that part's index can't be removed after completing
+    multipart upload when the bucket versioning is enabled.
+    
+    Fixes: http://tracker.ceph.com/issues/19604
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit decc01e2b1be9e4f809ff052149e1e6377c35047)
+
+commit 881bed78620a2c55c9530abefab433c7341895b9
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Nov 16 10:52:54 2016 +0800
+
+    rgw: Fix a bug that multipart upload may exceed quota ...
+    
+    ... because of improperly operation to the cache.
+    
+    Fixes: http://tracker.ceph.com/issues/19602
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit ffe95d3c11f05937109b72e8e54c0cfb7dc09a36)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - jewel does not have 885606c76cf8009570098cde48c0a23a21e6020d
+            so use state->size instead of state->accounted_size
+
+commit 2b759d2f74f3870c96c33de62d304b703141beb2
+Merge: f024855cae 89a2f6476d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:43:55 2017 +0200
+
+    Merge pull request #15455 from smithfarm/wip-19838-jewel
+    
+    jewel: rgw: reduce log level of 'storing entry at' in cls_log
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f024855cae7c41c3ef61365333672932369ff409
+Merge: ccf9d7109a 938e5b0728
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:42:23 2017 +0200
+
+    Merge pull request #15456 from smithfarm/wip-20003-jewel
+    
+    jewel: rgw: rgw_file:  v3 write timer does not close open handles
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit ccf9d7109a74875107736c017999437d233814d4
+Merge: 9302a44a69 8cf3f18c7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:38:29 2017 +0200
+
+    Merge pull request #15457 from smithfarm/wip-20018-jewel
+    
+    jewel: rgw: multisite: memory leak on failed lease in RGWDataSyncShardCR
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9302a44a69eb29d27b184eba76b9c9df613e6e3e
+Merge: dea17eaa1a 0897afe74d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:33:42 2017 +0200
+
+    Merge pull request #15459 from smithfarm/wip-20144-jewel
+    
+    jewel: rgw: rgw_file: cannot delete bucket w/uxattrs
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit dea17eaa1ab5cdc712b18567c1fed702ee655c2f
+Merge: e335808d26 2e278d821d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:32:40 2017 +0200
+
+    Merge pull request #15465 from smithfarm/wip-20155-jewel
+    
+    jewel: rgw: fix crash caused by shard id out of range when listing data log
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 36b1a35a449d7d1e0b3e506e021fc6bdb274750c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 26 10:06:25 2017 +0200
+
+    librbd: clean up object map update interface, revisited
+    
+    In master, the "batch update" change [1] was merged before the "order
+    concurrent updates" [2], while in jewel the latter is already
+    backported [3]. A backport of [1] to jewel was attempted, and was
+    necessarily applied on top of [3] - i.e. in the reverse order compared
+    to how the commits went into master. This reverse ordering caused the
+    automated cherry-pick to miss some parts of [1] which this commit is
+    adding manually.
+    
+    [1] https://github.com/ceph/ceph/pull/11510
+    [2] https://github.com/ceph/ceph/pull/12420
+    [3] https://github.com/ceph/ceph/pull/12909
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ca5577853cacb76ee46d50d644884102353caa21
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Sat Oct 15 17:18:30 2016 +0530
+
+    librbd: batch ObjectMap updations upon trim
+    
+    Shrinking a clone which has snapshots and does not share
+    majority of objects with its parent (i.e., there are less
+    objects to be copied up) involves huge number of object
+    map updates -- two (pre, post) per object. This results
+    in lots of requests to be send to OSDs especially when
+    trimming a gigantus image. This situation can be optimized
+    by sending batch ObjectMap updates for an object range
+    thereby significantly cutting down OSD traffic resulting
+    in faster trim times.
+    
+    Fixes: http://tracker.ceph.com/issues/17356
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 05653b7c512334533b801013f7e426363237301b)
+
+commit e335808d267435414dc1351e0f2e99e773fce805
+Merge: 47e3ff8408 244a2ae6a9
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Aug 25 14:26:21 2017 -0700
+
+    Merge PR #16248 into jewel
+    
+    * refs/remotes/upstream/pull/16248/head:
+            client: don't re-send interrupted flock request
+            mds/flock: properly remove item from global_waiting_locks
+            mds/flock: properly update ceph_lock_state_t::client_waiting_lock_counts
+            mds/flock: don't import ceph_lock_state_t::waiting_locks
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 47e3ff8408af027fe60f1260c3c46c7ac7f8135b
+Merge: aec54d8ecd 01890abd95
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:09:58 2017 +0200
+
+    Merge pull request #15454 from smithfarm/wip-19775-jewel
+    
+    jewel: rgw: multisite: realm rename does not propagate to other clusters
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit aec54d8ecdf3bfdb39b3f03a9e3971f78c13d4d4
+Merge: 494c634c23 a36211c84d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:09:15 2017 +0200
+
+    Merge pull request #15453 from smithfarm/wip-19768-jewel
+    
+    jewel: rgw: multisite: operating bucket's acl&cors is not restricted on slave zone
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 494c634c236dff709b1d6856b5f25c419a0ec067
+Merge: 0da1195f5f afd036cb67
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:08:26 2017 +0200
+
+    Merge pull request #15452 from smithfarm/wip-19765-jewel
+    
+    jewel: rgw: when uploading objects continuously into a versioned bucket, some objects will not sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0da1195f5f2f6d272c6da5ed4583f11246da07c4
+Merge: beb6e229b4 50f140768d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:06:37 2017 +0200
+
+    Merge pull request #15451 from smithfarm/wip-19764-jewel
+    
+    jewel: rgw: set latest object's acl failed
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit beb6e229b48d4d9f8c4feb18fa0868aa03109f0c
+Merge: 88126f86bf 61df8eaa02
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:05:54 2017 +0200
+
+    Merge pull request #15450 from smithfarm/wip-19758-jewel
+    
+    jewel: rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 92eb6836bfb3350f11fcc2d55a4710be45a34e4f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 23 15:31:50 2017 -0700
+
+    config: disable skewed utilization warning by default
+    
+    This has a few problems:
+    
+    1- It does not do it's analysis over CRUSH rule roots/classes, which
+    means that an innocent user of classes will see skewed usage (bc hdds are
+    more full than ssds, say)
+    
+    2- It does not take degraded clusters into account, which means the warning
+    will appear when a fresh OSD is added.
+    
+    See http://tracker.ceph.com/issues/20730
+    
+    Based on master commit 7832c53 but we've decided not to remove the code
+    in the older releases so it can be used if appropriate for a
+    particular cluster configuration.  Also, 7832c53 won't cleanly
+    cherry-pick so this is easier.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 88126f86bf240db8bd4573832a75709a8ae8ddba
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 10:27:22 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip ListObjectsError
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8199f3998f01ae31c4ae88fda7cd0b4704ae7c8e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 10:20:00 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip more tests
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 55cd3d90f8f3cd1d54df922badaf818acdcf140b
+Merge: b0814346f3 f965bdd013
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Aug 24 09:16:19 2017 -0500
+
+    Merge pull request #16870 from liewegas/wip-20929-jewel
+    
+    jewel: ceph-disk: dmcrypt cluster must default to ceph
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b0814346f3e7553654a999b27ba12ea7985a2ff3
+Merge: d79618b004 cd8c51569b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 10:15:57 2017 +0200
+
+    Merge pull request #14659 from asheplyakov/19473-jewel
+    
+    jewel: rgw: add the remove-x-delete feature to cancel swift object expiration
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d79618b004bdb9270cbd50b98e37d69721004bce
+Merge: 14804ad5c6 01abfed8f2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:13:23 2017 +0200
+
+    Merge pull request #15322 from gregsfortytwo/jewel-19931-snaptrim-shutdown
+    
+    jewel: osd: Reset() snaptrimmer on shutdown and do not default-abort on leaked pg refs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 14804ad5c6c4e4a8ea7951e2e7be562f2ac46d53
+Merge: 03fb91cfef 328e2cc71a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:11:52 2017 +0200
+
+    Merge pull request #15236 from asheplyakov/20036-bp-jewel
+    
+    jewel: mon: factor mon_osd_full_ratio into MAX AVAIL calc
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 03fb91cfef5b75baecc56d795f64a47793974c56
+Merge: a07f4d3a90 644bb1e81c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:07:08 2017 +0200
+
+    Merge pull request #14346 from ceph/wip-ceph-disk-fix-jewel
+    
+    jewel: build/ops: Add fix subcommand to ceph-disk, fix SELinux denials, and speed up upgrade from non-SELinux enabled ceph to an SELinux enabled one
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a07f4d3a90f0dd9a30801f813c91183f77e5eb6c
+Merge: 3e75beb162 e3cba06bd0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:04:49 2017 +0200
+
+    Merge pull request #15083 from asheplyakov/19926-bp-jewel
+    
+    jewel: mon: crash on shutdown, lease_ack_timeout event
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3e75beb1624046759694823b3406ce4e451bd5c5
+Merge: 6c51a81a82 e6582c43be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:56:38 2017 +0200
+
+    Merge pull request #14663 from smithfarm/wip-19228-jewel
+    
+    jewel: rbd: Enabling mirroring for a pool with clones may fail
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6c51a81a822408ac5b45706b888e22a02e3e99ca
+Merge: 9502460f7f 03ca042248
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:52:04 2017 +0200
+
+    Merge pull request #15050 from asheplyakov/19265-bp-jewel
+    
+    jewel: core: an OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 9502460f7f15e6ef9a79a09ccdb0463966aabb83
+Merge: a70259a441 9ec5d8b1a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:50:01 2017 +0200
+
+    Merge pull request #15065 from asheplyakov/19915-bp-jewel
+    
+    jewel: osd: PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit a70259a44173adeb1046a01e9c3d40b71052b3bd
+Merge: fd197dcbeb 09f076ed96
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:48:38 2017 +0200
+
+    Merge pull request #14943 from shinobu-x/wip-18293-jewel
+    
+    jewel: osd: leaked MOSDMap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fd197dcbeb4c416ad903a7795ce91ac3ffd0f296
+Merge: 5dc3b45b92 3776d960e0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:12:01 2017 +0200
+
+    Merge pull request #14699 from smithfarm/wip-19679-jewel
+    
+    jewel: mds: damage reporting by ino number is useless
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5dc3b45b92e9fb4b7025d3373dfd85c5867b6b60
+Merge: 8581349c8b a4c181fa7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:10:07 2017 +0200
+
+    Merge pull request #14673 from smithfarm/wip-19466-jewel
+    
+    jewel: mds: log rotation doesn't work if mds has respawned
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8581349c8b1b1e06522707d0b6a3bd2d0c3e35b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 23 13:53:48 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0.sh: more tests skipped
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c90969031f8f0c49a974c3e001fc44b1544547d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 12:16:45 2017 -0400
+
+    rgw: call update_latest_epoch() on all period updates
+    
+    when updating the period, callers use the atomic result of
+    update_latest_epoch() to determine whether they need to call
+    RGWPeriod::reflect() and RGWRealm::notify_new_period()
+    
+    this adds a missing call to RGWPeriod::reflect() to RGWPeriodPuller,
+    which was previously not safe to do without atomic updates to
+    latest_epoch
+    
+    Fixes: http://tracker.ceph.com/issues/19817
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d825b9d854864b815890cf9204d7e72e1e1a9ada)
+
+commit 6c8a7ca22bffc334032ae93953cc2b7ad47a363d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 11:45:39 2017 -0400
+
+    rgw: remove set_latest_epoch from RGWPeriod::store_info
+    
+    split the latest_epoch update out of RGWPeriod::store_info(), so callers
+    that need to call the atomic update_latest_epoch() can do so and
+    interpret its result separately
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c1731dd2d3de5a337903004f835045d609a731cf)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc (trivial resolution)
+
+commit 03cf10aefb9dcc253170472b3c95c829fb68cc40
+Merge: 1f9693be5e c49e249fb5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 08:41:55 2017 +0200
+
+    Merge pull request #15442 from Vicente-Cheng/wip-20025-jewel
+    
+    jewel: cephfs: osdc/Filer: truncate large file party by party
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1f9693be5ef8c64c7d58ccf08ef27cf2d7cb7cdb
+Merge: 198f7fac8f 0bb08832c0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 08:40:20 2017 +0200
+
+    Merge pull request #14691 from smithfarm/wip-19571-jewel
+    
+    jewel: tests: upgrade:client-upgrade/firefly-client-x: drop CentOS
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ab2d3582f42a45b5d3cf63e00007e0ca65df1c2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 28 12:52:18 2017 -0400
+
+    librbd: default features should be negotiated with the OSD
+    
+    Derived from f066ce8f80bce929edc209590efd47cce2196ae2
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5b5468189ea74da5db63bcba0ae6450d83cc99f7
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Nov 5 22:44:33 2016 +0200
+
+    cls/rbd: add get_all_features on client side
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 84f6d5c109911923c25414de639308921983e438)
+    
+    Conflicts:
+            src/cls/rbd/cls_rbd_client.h: trivial resolution
+
+commit ff312a4e6fb08dc93311e6969ef16d621388c395
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 10:44:20 2017 -0400
+
+    rgw: add atomic RGWPeriod::update_latest_epoch
+    
+    update_latest_epoch() uses RGWObjVersionTracker to implement atomic
+    updates to the period's latest_epoch, returning -EEXIST if we already
+    have an epoch >= the one given
+    
+    Fixes: http://tracker.ceph.com/issues/19816
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 55000d7fb3d4ab1ba4c19f3b3730823b5b056b67)
+
+commit e91a3945afa4ca27678c6adf8d1c792333b6b554
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 09:47:57 2017 -0400
+
+    rgw: period latest_epoch ops take optional objv tracker
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b855c39656d7556c7895850d12177ca6c1db01af)
+
+commit c864d5cac25d77307e9fa429c4d9a4f3f5f9df56
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 09:46:44 2017 -0400
+
+    rgw: remove unused RGWPeriod::use_next_epoch
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5770a01ebe8b5913cb5446bd4fd09d8029ac1381)
+
+commit 1516e24c8583237bc75c4185b416cd3fd4ec3df1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 13 11:23:37 2017 -0700
+
+    cls/refcount: store and use list of retired tags
+    
+    Fixes: http://tracker.ceph.com/issues/20107
+    
+    Keep around the list of retired tags, make sure we don't
+    drop a refcount using the same tag.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2c4fb020a13333bba6d3c29318c8a87099d915db)
+
+commit 9b5b21def271b1b85834fb32b762a0c0dc688d12
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Thu Jul 6 11:27:42 2017 +0300
+
+    rgw: fix not initialized pointer which cause rgw crash with ec data pool
+    
+    In RGWPutObjProcessor_Atomic::complete_writing_data()
+    with pending_data_bl.length() > 0 and next_part_ofs==data_ofs
+    not initialized void *handle leads to invalid pointer librados::AioCompletion::pc
+    which leads to rgw crash.
+    
+    Fixes: http://tracker.ceph.com/issues/20542
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 3e938dd9fe681fa8652dc4b0ec1dc56781d884c0)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc (trivial resolution)
+
+commit 7d199110fb0a46796bfe8422ee103b837deb964f
+Author: fang yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed May 3 13:46:50 2017 +0800
+
+    rgw: don't do unneccesary write if buffer with zero length
+    
+    Don't do unneccesary write if buffer with zero length,
+    or there will be one more shadow stripe rados object with size 0
+    
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit 4e21f7ea2167be7793273f31faeb0aa95f0fa30a)
+
+commit 73c9d33ac0d761ef0b616a4beae56dc15c7c3f5e
+Author: Abhishek Varshney <abhishek.varshney@flipkart.com>
+Date:   Wed Jul 19 11:57:08 2017 +0000
+
+    rgw: fix error message in removing bucket with --bypass-gc flag
+    
+    Fixes: http://tracker.ceph.com/issues/20688
+    
+    Signed-off-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+    (cherry picked from commit 596b4bc05f167baca3ae6e3ed9bed5b04d27409e)
+
+commit e76198751cf876ee1137194e6cdb1aa761b31d21
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Jul 7 11:09:02 2017 +0800
+
+    rgw:multisite: fix RGWRadosRemoveOmapKeysCR
+    
+    RGWRadosRemoveOmapKeysCR::request_complete return val is wrong. The return val should get from completion.  Some member variables is not used, clear up those variables.
+    
+    Fixes:http://tracker.ceph.com/issues/20539
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 82a0b6ac2130a7f326473e5f376c214f59f50829)
+    
+    Conflicts:
+            src/rgw/rgw_cr_rados.cc (trivial resolution)
+            src/rgw/rgw_cr_rados.h (master does not have IoCtx, jewel does not have override)
+
+commit 71262f1faeaa26481cec15e2fe3894a1ab2a3f3f
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Nov 29 08:57:30 2016 +0800
+
+    rgw: lock is not released when set sync marker is failed.
+    
+    Fixes: http://tracker.ceph.com/issues/18077
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 65ef596ce1d83515338367f69b1bf0ce0ab96ae5)
+
+commit 198f7fac8f67b5e2996f2b53420c6c58854e54b3
+Merge: ed53df2bb9 7e3db1519e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 16:25:14 2017 +0200
+
+    Merge pull request #15448 from smithfarm/wip-19615-jewel
+    
+    jewel: rgw: multisite: bucket zonegroup redirect not working
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 38718ace488d2f1ca93c50b2d6474d6095b1f544
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Tue Jun 13 14:39:14 2017 +0800
+
+    rgw_file: prevent conflict of mkdir between restarts
+    
+    Fixes: http://tracker.ceph.com/issues/20275
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 87cec5ff9a2f516321543076f536fcfff433c03b)
+
+commit 0bb08832c0ba20cc85a50d9dfbc4efc0f083b7ae
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 20 19:15:29 2017 +0200
+
+    tests: upgrade:client-upgrade/firefly-client-x: drop CentOS
+    
+    The RBD suite needs ceph-cm-ansible to install qemu-kvm on CentOS, but doing
+    that breaks the firefly install on CentOS because:
+    
+    1. the qemu-kvm that gets installed is from hammer (0.94.5)
+    2. qemu-kvm brings in librados2, librbd1 as dependencies
+    
+    As a result, the hammer librados2 and librbd1 are installed on the test nodes
+    even before the teuthology install task starts. When it does start and tries
+    to install firefly, it fails because firefly librados2 and librbd1 cannot be
+    installed over their hammer versions.
+    
+    Fixes: http://tracker.ceph.com/issues/19571
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ed53df2bb9d9fed42f47f73168f77c7346942601
+Merge: 7a599849d3 66c17ff10b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 13:47:21 2017 +0200
+
+    Merge pull request #15197 from smithfarm/wip-20007-jewel
+    
+    jewel: rgw: remove unnecessary 'error in read_id for object name: default'
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 53e97da166c6e36c6808063f90c4a2cef2d52a77
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 8 22:29:48 2017 +0200
+
+    ceph-disk: set the default systemd unit timeout to 3h
+    
+    There needs to be a timeout to prevent ceph-disk from hanging
+    forever. But there is no good reason to set it to a value that is less
+    than a few hours.
+    
+    Each OSD activation needs to happen in sequence and not in parallel,
+    reason why there is a global activation lock.
+    
+    It would be possible, when an OSD is using a device that is not
+    otherwise used by another OSD (i.e. they do not share an SSD journal
+    device etc.), to run all activations in parallel. It would however
+    require a more extensive modification of ceph-disk to avoid any chances
+    of races.
+    
+    Fixes: http://tracker.ceph.com/issues/20229
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a9eb52e0a4c06a80e5dbfaac394aac940edf4c68)
+
+commit 7a599849d3f62c4e6001e2c7e3973b5c368d85d1
+Merge: c79524153c 592293cdfb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 10:00:07 2017 +0200
+
+    Merge pull request #15051 from asheplyakov/19910-bp-jewel
+    
+    jewel: build/ops: extended, customizable systemd ceph-disk timeout
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c79524153cb52b1022dba0102aaef942f81d6fd9
+Merge: 161cba9cea 6b99bc9c60
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:54:12 2017 +0200
+
+    Merge pull request #15428 from smithfarm/wip-20162-jewel
+    
+    jewel: build/ops: deb: Fix logrotate packaging
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 161cba9cea5d83911e1ee7d42d58004ec327b9bb
+Merge: 1f0549ca49 9a5425d5de
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:52:49 2017 +0200
+
+    Merge pull request #15447 from smithfarm/wip-19613-jewel
+    
+    jewel: rgw: multisite: rest api fails to decode large period on "period commit"
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e6582c43bebac315ef2d76fded92373c7b1300be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 27 16:21:33 2017 -0400
+
+    test: remove hard-coded image name from RBD metadata test
+    
+    Fixes: http://tracker.ceph.com/issues/19798
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8f72e745e157cc12d76ca6babe956c5698ee297f)
+
+commit 8f23ea4fc0938c35998193ea76d7a029422b89b0
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Mar 2 17:18:18 2017 +0100
+
+    librbd: relax "is parent mirrored" check when enabling mirroring for pool
+    
+    If the parent is in the same pool and has the journaling feature enabled
+    we can assume the mirroring will eventually be enabled for it.
+    
+    Fixes: http://tracker.ceph.com/issues/19130
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe31bca22f90ce02f461d6421a4f66539db888d3)
+
+commit 1f0549ca49cc60ed62f52ae5ee22d1044f70aab0
+Merge: 648fabaa9f 00ce80b52f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:31:04 2017 +0200
+
+    Merge pull request #15196 from smithfarm/wip-20006-jewel
+    
+    jewel: build/ops: rpm: fix python-Sphinx package name for SUSE
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 1415ecbd7de21ed2ab0c8f6e20f707cbe4f1390c
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Aug 21 14:06:13 2017 +1000
+
+    ceph-disk: Use stdin for 'config-key put' command
+    
+    The 'osd new' command is not available in jewel so backport support for
+    stdin '-i -' and use it for the 'config-key put' command in order to
+    not log the dmcrypt key.
+    
+    Fixes: http://tracker.ceph.com/issues/21059
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 28cfb01dddab94b849a8d8bce8d8bd27c3818e25
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 17 09:38:52 2017 -0400
+
+    ceph: allow '-' with -i and -o for stdin/stdout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3a4931b0e467e3b64a2637b96d7a3cab9add1c80)
+    
+    Conflicts:
+            qa/workunits/cephtool/test.sh (post-jewel functions removed)
+            src/ceph.in
+
+commit a6da22fcf646ff9db45a01a43fa719982bf9bca4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 8 19:00:05 2017 +0200
+
+    ceph-disk: implement command_with_stdin
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 597393d5d09c52170b7db6f609ce142edfd4d8fe)
+
+commit 3adbc35205f663c8c312476f3694d2c14294c5d0
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Mon Aug 1 21:00:28 2016 +0200
+
+    rgw: aws4: add rgw_s3_auth_aws4_force_boto2_compat conf option
+    
+    Runtime bugfix to handle presigned urls computed with canonical requests using
+    the port number once.
+    
+    Boto2 computes canonical requests using the port number twice although it
+    should be used once only. This behaviour is a bug supported by AWS S3. Boto2 is
+    used in RGW S3 as reference implementation.
+    
+    The client-side tools not supporting this boto2 bug will fail although they
+    should work too.
+    
+    In order to support both presigned url implementations this patch adds a config
+    option to compute a second signature. With this option disabled, the code will
+    compute two signatures when the first signature is not valid. The aws4 auth
+    succeed if some of the two signatures is valid.
+    
+    The config option rgw_s3_auth_aws4_force_boto2_compat, is enabled by default so
+    one signature, working with boto2, is computed only.
+    
+    Fixes: http://tracker.ceph.com/issues/16463
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 078c513b6bc6b1d1da50db1d51fbbb65bddd44b9)
+
+commit cd848c1e9852c89a06e3baa669314cba2fdef965
+Author: Alexey Sheplyakov <asheplyakov@yandex.ru>
+Date:   Sun Aug 13 16:33:00 2017 +0400
+
+    jewel: mon: fix force_pg_create pg stuck in creating bug
+    
+    Register the creating PG through the common path to get the PG mapped to an OSD.
+    Adapted from 9a41a0b7289fa59f4b747a63e152e88af6e8abd5
+    
+    Fixes: http://tracker.ceph.com/issues/19182
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@yandex.ru>
+
+commit 648fabaa9fc1e78aa808a3a2c7c91dee464dbc76
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 12 09:00:22 2017 -0400
+
+    qa/workunits/rados/test/test-upgrade-v11.0.0: skip Quota test
+    
+    This makes a pool go full which confounds the wait for healthy check
+    during restarts.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 37b37e78f858a2d981698de2e9e1aafef7953642
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 31 14:45:25 2017 +0300
+
+    rgw: Fix up to 1000 entries at a time in check_bad_index_multipart
+    
+    Fixes: http://tracker.ceph.com/issues/20772
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 25cef9a09721e1a67d217535d4d624ab60adca02)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6bd2dae1934f5f2a752b2c0edaa937b5a4163c36
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Aug 9 13:51:40 2017 -0400
+
+    rgw: replace '+' with "%20" in canonical query string for s3 v4 auth.
+    
+    Fixes: http://tracker.ceph.com/issues/20501
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 416bc51101b4ae5da569c9bc3d8d738eeadc25a6)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3242b2b5629b951acfe90011a3c07533db1ab0a3
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 6 15:07:19 2016 -0800
+
+    ceph-fuse: start up log on parent process before shutdown
+    
+    Otherwise, we hit an assert in the Ceph context and logging teardown.
+    
+    Fixes: http://tracker.ceph.com/issues/18157
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cbf18b1d80d214e4203e88637acf4b0a0a201ee7)
+
+commit ff67388e24c93ca16553839c16f51030fa322917
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jan 10 16:36:13 2017 +0800
+
+    rgw: bucket index check in radosgw-admin removes valid index.
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit a786252ed9c0e1ba08a88a43902f7aa1e91c10cd)
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    
+    Conflicts:
+            src/rgw/rgw_bucket.cc
+                    Jewel has RGWObjEnt, honor the same while populating obj oid
+
+commit 788cfa89dd2abc3db8dc35803e275d4c2fbb6837
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri May 26 12:41:40 2017 -0400
+
+    rgw-admin: fix bucket limit check argparse, div(0)
+    
+    1. normalize arg parsing for "bucket limit check"
+    1.1 s/buckets/bucket/
+    
+    2. avoid dividing by num_shards when it is 0
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit fb8e2fa61f0b4621b5fdc83048467982a7ca2961)
+    Fixes: http://tracker.ceph.com/issues/20966
+
+commit e6f86ba31d3092dbb45bdd35f344aba7b2c5a61b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 9 09:56:21 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip api_io *CmpExt*
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5a7ea2790c391544a3a8ea4369ac8e46a0b3c4ec
+Author: Abhishek Varshney <abhishek.varshney@flipkart.com>
+Date:   Tue Aug 1 14:46:52 2017 +0000
+
+    rgw : fix race in RGWCompleteMultipart
+    
+    Fixes : http://tracker.ceph.com/issues/20861
+    
+    Signed-off-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+    (cherry picked from commit dd7dc62eaddb47447c01c03c87ad56cd7690693e)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6d5ef4b27d0fad62646ddf82b59d9c277179d293
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 7 14:01:58 2017 -0400
+
+    qa/workunits/rados/test-upgrade-11.0.0: skip RoundTrip3
+    
+    This does the crc op, which isn't in jewel.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit d4fa37323596394feadaed6168b49ccbaf6fc089
+Merge: 5776904626 d3e81588ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 7 10:07:13 2017 -0400
+
+    Merge pull request #16343 from dillaman/wip-20630-jewel
+    
+    jewel: tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit f965bdd0134cbc3d947ac83bf96e5ce08541a198
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 3 10:56:24 2017 +0200
+
+    ceph-disk: dmcrypt cluster must default to ceph
+    
+    If ceph_fsid is not found, which is the case for legacy dmcrypted OSD,
+    the cluster must default to ceph, as it was before support for non
+    standard cluster names was introduced.
+    
+    Fixes: http://tracker.ceph.com/issues/20893
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed22159fdd7071a110e81cd7b63d7e0f2296761b)
+
+commit 5776904626e17fd1ed73bcebeee742655523e833
+Merge: 8db71d796e 10636ef77f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:06:54 2017 +0200
+
+    Merge pull request #13507 from shinobu-x/wip-18468-jewel
+    
+    jewel: mon: Disallow enabling 'hashpspool' option to a pool without some kind of --i-understand-this-will-remap-all-pgs flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c49e249fb5a5232dc25b8250f4b7d2b4e3d47bef
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 25 16:21:24 2017 +0800
+
+    osdc/Filer: truncate large file party by party
+    
+    Fixes: http://tracker.ceph.com/issues/19755
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263)
+    
+    Conflicts:
+            src/osdc/Filer.h
+            src/osdc/Filer.cc
+                - add parameter to fit _modify (need onack parameter)
+            src/mds/MDCache.cc
+                - make truncate() consistency
+
+commit 314952c58993e6ea6a3fa1bb3b6f9a7118c52ea4
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 20:17:49 2017 +0200
+
+    rgw: lease_stack: use reset method instead of assignment
+    
+    It seems that the intent of 45877d38fd9a385b2f8b13e90be94d784898b0b3 was to
+    change all instances of "lease_stack = ..." to "lease_stack.reset(...)", but
+    this one was missed.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc425afeb2e2a6ba3c98c612b3977aea619c9f73)
+
+commit 0930eb07d1c983412e27f7c42836a319a3161c7a
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Tue Jun 13 16:40:16 2017 +0800
+
+    rgw: meta sync thread crash at RGWMetaSyncShardCR
+    
+    Fixes: http://tracker.ceph.com/issues/20251
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 45877d38fd9a385b2f8b13e90be94d784898b0b3)
+
+commit e35c938208a7e265387ba01cde9d84d08ec55ec0
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Wed Jul 5 16:23:37 2017 +0200
+
+    ceph-disk: don't activate suppressed journal devices
+    
+    Multipath device activation currently requires that devices representing
+    individual paths making up a multipath device are suppressed, to avoid
+    errors such as:
+      ceph_disk.main.Error: Error: /dev/sda1 is not a multipath block device
+    
+    Activation of suppressed OSD devices is currently handled by checks in
+    main_activate() and main_activate_space(). However, suppressed journal
+    and block devices are not detected.
+    
+    (Along with udev change) Fixes: http://tracker.ceph.com/issues/19489
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit c729627450d4032ede7932f4c1b57a6d0cff73b2)
+
+commit 153f77544118613e19d5e88c030c3901234cf950
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jul 18 15:08:14 2017 -0700
+
+    osd: scrub_to specifies clone ver, but transaction include head write ver
+    
+    Fixes: http://tracker.ceph.com/issues/20041
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fd598a0d23d61c645633ae774c3404a43d035e3c)
+    
+    Conflicts:
+            src/osd/ReplicatedPG.cc (trivial)
+
+commit 2efd7cd6f32aa780fea5d26c8314c48cae734793
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Jul 3 09:34:08 2017 +0800
+
+    jewel:ceph-disk:remove the special check to bcache devices
+    
+    if we skip partition when coming cross the bcache device in ceph-disk,
+    the udev can't detect the osd in such devices when we reboot the osd's host,
+    so the osd who deploy in bcache device can't auto start when the host restart.
+    
+    And partition is now supported by bcache devices (https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/commit/drivers/md/bcache?id=b8c0d911ac5285e6be8967713271a51bdc5a936a),
+    we can remove this special check now.
+    
+    This cannot be cherry-picked from master because 'accept bcache devices as data disks' [21f0216](https://github.com/ceph/ceph/commit/21f0216287485e6cce8811f324fee804ef733524)
+    is a jewel-only feature not present in master.
+    
+    Signed-off-by: Wei Qiaomiao <wei.qiaomiao@zte.com.cn>
+
+commit 03ca042248586d1e7612f4d026551891701048c3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jul 7 10:53:41 2017 -0700
+
+    jewel: mon: Fix status output warning for mon_warn_osd_usage_min_max_delta
+    
+    Fixes: http://tracker.ceph.com/issues/20544
+    
+    Caued by: 489e810c37ed6fb9d32d1015634041a577501ee4
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 56f9808016ab289bdc0ce7cfbb0503c78b509593)
+    
+    Conflicts:
+            apply changes to src/mon/PGMonitor.cc instead of src/mon/PGMap.cc
+
+commit 8db71d796e50bcdbb6bd711feba6e4fdbecfcd29
+Merge: 2ee413f771 8885f3ab6a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jul 20 15:00:32 2017 -0400
+
+    Merge pull request #16289 from cbodley/wip-20513
+    
+    jewel: rgw: cls: ceph::timespan tag_timeout wrong units
+
+commit e9e7850b51c824a50dde96332e9f47fd3a62e60b
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:34:19 2017 +0200
+
+    messages/MOSDPing: optimize encode and decode of dummy payload
+    
+    The dummy payload doesn't need to be processed, we can just skip over
+    it when decoding and we can use a single bufferptr instead of entire
+    bufferlist to encode it.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 15ce0772841f913d1eef1daebad0834e5f862383)
+
+commit 8acfd7189a4a2fe937e06d12132df55af5d90fcb
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:10:36 2017 +0200
+
+    messages/MOSDPing: fix the inflation amount calculation
+    
+    If user specifies a min_message_size small enough (or zero to disable
+    it altogether), OSDs will crash and burn while trying to allocate
+    almost 4GB of payload (both min_message_size and payload.length() are
+    unsigned, so it'll roll over back to 4GB and MAX(4GB, 0) will use 4GB).
+    If the size of dummy payload is 0, don't bother constructing bufferptr
+    and bufferlist, then encoding that.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit d959735a7d684018df564573a2ff88092d870fc1)
+
+commit 52f221b2103333facbc4ead30adc0eba8063983d
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:05:10 2017 +0200
+
+    OSD: mark two heartbeat config opts as observed
+    
+    "osd heartbeat min size" and "osd heartbeat interval" can be changed
+    at runtime, because their values, when used, are always taken from
+    global Ceph configuration. Mark them as observed, so the message
+    the user sees once they're changed doesn't confuse them.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 44b7839c788b20e0b9da76bc3838ae10d3ad2f89)
+
+commit f840864a5100dc65bb09a49e91fe4db776f69413
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 15 22:18:08 2017 -0400
+
+    messages/MOSDPing: initialize MOSDPing padding
+    
+    This memory must be initialized or else valgrind will be very unhappy.
+    
+    Avoid the cost of zeroing (or even allocating) the buffer for normal
+    padding values by (re)using a static zero buffer.
+    
+    Fixes: http://tracker.ceph.com/issues/20323
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9beaf5efd74daa8c15e50b42583264d1252a85f5)
+
+commit 71645604b6e07369f45034112710661167db052f
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jun 5 13:33:14 2017 -0700
+
+    osd: heartbeat with packets large enough to require working jumbo frames.
+    
+    We get periodic reports that users somehow misconfigure one of their switches
+    so that it drops jumbo frames, yet the servers are still passing them along. In
+    that case, MOSDOp messages generally don't get through because they are much
+    larger than the 1500-byte non-jumbo limit, but the MOSDPing messages have kept
+    going (as they are very small and dispatched independently, even when the
+    server is willing to make jumbo frames). This means peer OSDs won't mark down
+    the ones behind the broken switch, despite all IO hanging.
+    Push the MOSDPing message size over the 1500-byte limit so that anybody in
+    this scenario will see the OSDs stuck behind a bad switch get marked down.
+    
+    Fixes: http://tracker.ceph.com/issues/20087
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2096113d9e1589c571d96e34dd9cd841308a2567)
+    
+     Conflicts:
+            src/messages/MOSDPing.h
+             - Changed HEAD_VERSION to 3 and kept COMPAT_VERSION to 1.
+             - In class MOSDPing removed following line:
+                   if (header.version >= 2)
+             - To keep ::decode(stamp, p) without condition because HEAD_Version
+               is already 3 now and this condition is removed in the backport commit.
+
+commit d3e81588eef9a3745d2cc27382b3a26b12bc8005
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 14 10:32:28 2017 -0400
+
+    qa/tasks: rbd-mirror daemon not properly run in foreground mode
+    
+    Fixes: http://tracker.ceph.com/issues/20630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fa1918717b25a5ffafbf649eedcfe7d5ab829c2)
+
+commit 713a88a5c9ec8e1140227ddcf0df8badfd96706d
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue May 9 16:39:39 2017 +0800
+
+    rgw: VersionIdMarker and NextVersionIdMarker should be returned when listing
+    object versions if necessary.
+    
+    Fixes: http://tracker.ceph.com/issues/19886
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit f805c3e08948e379b7d2c4f2faf9e7f550e4cb23)
+
+commit 0a3227876c3142d60d137ef351cf91795227883a
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jun 6 15:43:20 2017 +0800
+
+    rgw: datalog trim and mdlog trim handles the result returned by osd
+    incorrectly.
+    
+    Fixes: http://tracker.ceph.com/issues/20190
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 7fd6e031e5b0b1f3eca70c5b459d50f6f214171f)
+
+commit 0cd7df3649d7486d444a61cab89c48a89ddd3e8d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 29 14:54:40 2017 -0400
+
+    rbd: do not attempt to load key if auth is disabled
+    
+    Fixes: http://tracker.ceph.com/issues/19035
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b9c8df6d7f0b75c5451953bb322bc1f9afb6299)
+
+commit 787ba33e5dba285dff874955a8f0d7aabd3f87fe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 08:17:05 2017 -0400
+
+    librbd: filter expected error codes from is_exclusive_lock_owner
+    
+    Fixes: http://tracker.ceph.com/issues/20182
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4daaf54e6bc42cd4fb2111ea20b2042941b0c31)
+
+commit f9e87fef6a4771545cc23fe94c9cb9656ca5b773
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 13:17:19 2017 -0400
+
+    rbd: properly decode features when using image name optional
+    
+    Fixes: http://tracker.ceph.com/issues/20185
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1b05a2b062a59ec1b6682f7683bfd816433a931)
+
+commit 9a5425d5de16045b57c1307cb3b8263c739909b1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 14:20:20 2017 -0400
+
+    rgw: allow larger payload for period commit
+    
+    testing with 3 zonegroups and 3 zones each, the period json grew larger
+    than 4k and caused decode failures on period commit
+    
+    updated to use the new config variable rgw_max_put_param_size
+    
+    Fixes: http://tracker.ceph.com/issues/19505
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f2871fe59d933b03f37fde40f1781b2320d0d50)
+    
+    Conflicts:
+        src/rgw/rgw_rest_realm.cc - in RGWOp_Period_Post::execute(), set max_size
+            explicitly instead of backporting rgw_max_put_param_size
+
+commit 61df8eaa02d3fcdc814568be20631933c34aa8a3
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue May 9 15:05:03 2017 +0800
+
+    rgw: when create_bucket use the same num_shards with info.num_shards
+    
+    pr #14388 only fix the num_shards in BucketInfo, "init_bucket_index" function still use local num_shards
+    
+    Fixes: http://tracker.ceph.com/issues/19745
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 4ce64a190b4ff36985e785e574c077d39796feea)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: init_bucket_index() called earlier
+
+commit 93799374a97e5acec00737bb7fd46dce314ec19a
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Apr 7 15:34:27 2017 +0800
+
+    rgw: using the same bucket num_shards as master zg when create bucket in secondary zg
+    
+    create bucket in secondary zonegroup will forward to master. The master may have different num_shards option.
+    So when create bucket in local, should use master's num_shards instead of local num_shards option.
+    
+    Fixes: http://tracker.ceph.com/issues/19745
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit a34c4b8fb13dd5590eb3c6ecb5e55207ed8e3ee8)
+    
+    Conflicts:
+        src/rgw/rgw_op.cc
+        - different argument list of store->create_bucket() in jewel, compared to
+          master
+        - no RGWBulkUploadOp:: in jewel; modifications to
+          RGWBulkUploadOp::handle_dir() omitted
+
+commit 8885f3ab6a529c451581046fb3c8f72fceec6e80
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jun 29 15:19:36 2017 -0400
+
+    rgw: cls: ceph::timespan tag_timeout wrong units
+    
+    In rgw_dir_suggest(), the ceph::timespan tag_timeout is intended
+    to be a value in seconds, but has been taken as something much
+    smaller (millis?).  The incorrect time scale likely induces a race
+    condition with object deletes.
+    
+    Fixes: http://tracker.ceph.com/issues/20380
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e6763ebd6124c1b4736c5c7850077df7644bc8f5)
+
+commit 3105f118bccb5ae9209cb289cec5ccdba6d9d7b6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 2 13:38:29 2016 +0200
+
+    rbd-mirror: set SEQUENTIAL and NOCACHE advise flags on image sync
+    
+    Fixes: http://tracker.ceph.com/issues/17127
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1167865e3b4c5284938cb1797ebeff15974f7260)
+    
+    Conflicts:
+        src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc (sparse reads are not
+            being backported to jewel)
+
+commit 24bd60136e033f14a556f8e6220cf4ea49520598
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 26 14:28:21 2017 +0300
+
+    rgw: fix next marker to pass test_bucket_list_prefix in s3test
+    
+    Fixes: http://tracker.ceph.com/issues/19432
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8fff351d7607c6183b77acc29057c9d55b11f9de)
+
+commit 28f0a8dba36a20b482a70e42241e4e5ee4771e5f
+Author: Giovani Rinaldi <gr.gigio@terra.com.br>
+Date:   Wed Apr 5 16:46:14 2017 -0300
+
+    rgw: fix listing of objects that start with underscore
+    
+    Current marker and prefix search utilized in rgw's function list_objects should respect index key name in order to correctly list objects which have names starting with underscore.
+    
+    Fixes: http://tracker.ceph.com/issues/19432
+    
+    Signed-off-by: Giovani Rinaldi <giovani.rinaldi@azion.com>
+    (cherry picked from commit c1a1539af596e97dc412a505d5411ee6cfdd3980)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - in jewel, the call to cur_marker.name.find() is in a
+            conditional block; not so in master
+
+commit 39b2b0b39aedea1c7ee5eab66ca7fb6c59a41e45
+Author: zhaokun <jswps2011@163.com>
+Date:   Mon May 16 09:30:48 2016 +0800
+
+    rgw/rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get
+    
+    Fixes: http://tracker.ceph.com/issues/16072
+    
+    Signed-off-by: zhao kun <jswps2011@163.com>
+    (cherry picked from commit 7e91c81702284f1eb68567fc5d9385ce22405e69)
+
+commit f16fa4b252b1601b00149f17ed1516bc90cbedf3
+Author: root <root@dlp-dev01.novalocal>
+Date:   Tue May 23 13:56:59 2017 +0800
+
+    rgw: multipart copy-part remove '/' for s3 java sdk request header.
+    
+    Fixes: http://tracker.ceph.com/issues/20075
+    Signed-off-by: donglinpeng@cmss.chinamobile.com
+    (cherry picked from commit d60ddddfaf6964364faed40f45b18eb88a841023)
+
+commit 244a2ae6a91e2f288cf1ef8c930dec712b588e37
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:33:36 2017 +0800
+
+    client: don't re-send interrupted flock request
+    
+    don't re-send interrupted flock request in cases of mds failover
+    and receiving request forward.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 044fabd669fdc8287f5cab0f38a82dfbfe41f0cd)
+    
+     Conflicts:
+            src/client/Client.cc: UserPerms not in jewel
+
+commit 93c8ca6450146cc2a3e50597d91f29455b42c742
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:06:10 2017 +0800
+
+    mds/flock: properly remove item from global_waiting_locks
+    
+    ceph_lock_state_t::remove_waiting() uses wrong key to search
+    global_waiting_locks. It should use item in waiting_locks as
+    key.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0d71c6120e61f31b803c3fb6488fc7e97134e348)
+
+commit e795086776e9dfc8d9068161b6e96e79e8c8f193
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:01:27 2017 +0800
+
+    mds/flock: properly update ceph_lock_state_t::client_waiting_lock_counts
+    
+    no item is added to waiting_locks when deadlock check fails
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit a6b01939a199833568961ae4d30d8540a25a7740)
+
+commit 917240a291c6366b55ddab1a381e23e77b3264c1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 11:42:12 2017 +0800
+
+    mds/flock: don't import ceph_lock_state_t::waiting_locks
+    
+    Item in waiting_locks is associated with flock mds request in
+    exporter mds. If client re-sends the flock mds request to the
+    importer, the importer starts a new MDRequest. The new one's
+    'flock_was_waiting' is false.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8d777c4855b0c8e58a8d6966371698b8b7721a42)
+
+commit 644bb1e81ca29e244cf2962bc1c06bdecaac427a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jul 7 12:37:55 2017 +0200
+
+    rpm: Fix undefined FIRST_ARG
+    
+    If FIRST_ARG is undefined, the rpms will show an error on upgrade
+    because the condition in the postun script gets broken.
+    
+    This was a regression introduced by commit for issue 20077 that moved
+    ceph-disk unit files to ceph-base.
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 562816914ccca8e4e7d9c31f333db2f0da6f7c99)
+
+commit e9608e0c4097caa3c2e344fc3456ac3d6c816d5e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 5 18:44:18 2017 +0200
+
+    selinux: Install ceph-base before ceph-selinux
+    
+    We need to have ceph-base installed before ceph-selinux to use ceph-disk
+    in %post script. The default ordering is random and so the installation
+    randomly failed to relabel the files.
+    
+    Fixes: http://tracker.ceph.com/issues/20184
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit e69086645d3411a2ed781609b670eb5f16ac4810)
+    Conflicts:
+            ceph.spec.in: No _epoch_prefix in jewel
+
+commit 5c53ae3f9ac03445a99e84796f1b50ae32a365f2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:55:36 2017 +0800
+
+    test/librbd: decouple ceph_test_librbd_api from libceph-common
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b7287fdc4d70c5ecedda78ae367b98e5d8f61c5b)
+    
+    Conflicts: master has switched from autotools to cmake, so this is
+      somewhat an equivalent in automake to the cherry-picked change on
+      CMakeLists.txt
+
+commit 14fcdb7b699185e6e4ca1f158d34b47c63f6563b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:55:04 2017 +0800
+
+    test/librados: extract functions using libcommon in test.cc into test_common.cc
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    (cherry picked from commit 82a848c2053ea69ebc6d3ec1003e18921e2d08d2)
+    
+    Conflicts: since master has switched to cmake. the changes in this
+      commit on .am files are ported from counterpart of CMakeLists.txt.
+
+commit 7d959e02f79ed3c268b03bf6fdf689cbe73063dc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:01:06 2017 +0800
+
+    test/librbd: replace libcommon classes using standard library
+    
+    so ceph_test_librbd_api has less dependencies on libcommon
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    (cherry picked from commit bc8fa0193c8b2fb4b749ce9976bf6efbea833e96)
+
+commit 61f132078a535d10d10d2feb0688853855319ece
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 18:01:33 2017 +0200
+
+    tests: librbd: adapt test_mock_RefreshRequest for jewel
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 97fdaa753e812dfeb5256ced3af1dbe5adfff95d
+Author: Peng Xie <peng.hse@xtaotech.com>
+Date:   Thu Jun 29 18:48:27 2017 +0800
+
+    osd/PrimaryLogPG solve cache tier osd high memory consumption
+    
+    during cache tier dirty data flushing, cache tier osd memroy consumption
+    will get increasing due to the accumulative pg log which will not be trimmed
+    
+    Fixes: http://tracker.ceph.com/issues/20464
+    
+    Signed-off-by: Peng Xie <peng.hse@xtaotech.com>
+    (cherry picked from commit da605aa35e2d0897597a0ab6e19be70d94e6da24)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+                - should modify on ReplicatedPG.cc
+
+commit f49c9c777ffca31fa4c338dd22154ae6ad4e2ecf
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jun 21 15:05:49 2017 -0700
+
+    osd/ReplicatedBackend: reset thread heartbeat after every omap entry in deep-scrub
+    
+    Doing this every 100 entries could be after 100MB of reads. There's
+    little cost to reset this, so remove the option for configuring it.
+    
+    This reduces the likelihood of crashing the osd due to too many omap
+    values on an object.
+    
+    Fixes: http://tracker.ceph.com/issues/20375
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 15ce60830aed7c4c2a0b10d2cb69d8daef418d20)
+    
+    Conflicts:
+            src/osd/ReplicatedBackend.cc
+                - remain the iterator checker (do not check status)
+
+commit 932342c1e3d704201dc4b4b42513c32a73ef0adf
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 5 11:30:45 2017 +0100
+
+    client: avoid returning negative space available
+    
+    ...when a quota is set and the used bytes exceed
+    the quota.
+    
+    Fixes: http://tracker.ceph.com/issues/20178
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0144fde1b1639a6917e487f78490f2b84dceeae6)
+
+commit 6b6620ffefc6481d629e19cd9a6c50b39dba222c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 21 15:48:54 2017 +0800
+
+    mds: save  projected path into inode_t::stray_prior_path
+    
+    Otherwise, path string like #10000000000/xxx may get saved into
+    inode_t::stray_prior_path.
+    
+    Fixes: http://tracker.ceph.com/issues/20340
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ba7472cbf2a3147136a31b611f7b88f883be5f6d)
+    
+    Conflicts:
+            src/mds/Server.cc
+              - remove counter `change_attr` because it does not involved
+
+commit dac0ca797cada404b4d5eeb29039b8eac84ce7d6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 6 15:13:50 2017 -0400
+
+    ceph_test_rados_api_misc: fix LibRadosMiscConnectFailure.ConnectFailure retry
+    
+    Fixes: http://tracker.ceph.com/issues/19901
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 087dff80cac707ee9bcb5bcfc98cb3ec047bd49f)
+
+commit ab70ffc815e23e4bffa1e563ade21a0e1e84b8bf
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Mon Feb 20 12:04:10 2017 +0530
+
+    librbd: acquire exclusive-lock during copy on read
+    
+    Fixes: http://tracker.ceph.com/issues/18888
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 7dba5311b12011a4a6e8564e68150e54c5af5ddd)
+    
+    Conflicts:
+        src/librbd/AioImageRequestWQ.h:
+          - in master this file has morphed into src/librbd/io/ImageRequestWQ.h
+          - jewel has AioImageRequest<ImageCtx> instead of ImageRequest<ImageCtx>
+        src/librbd/image/RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because jewel doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+        src/test/librbd/image/test_mock_RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because jewel doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+
+commit 7bf8b854ab573016bba10fe2904b4d4f4e9b1ee1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 2 15:03:57 2017 -0600
+
+    osd/osd_internal_types: wake snaptrimmer on put_read lock, too
+    
+    The snaptrimmer can block taking a write lock, which might happen due to
+    a conficting EC read.  When the EC read completes, we need to wake up the
+    snaptrimmer.
+    
+    Fixes: http://tracker.ceph.com/issues/19131
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 993179870ef7eece47cb9ccfb7275c29bf8536eb)
+    
+    Conflicts:
+            src/osd/osd_internal_types.h: ObjectContext::put_lock_type() is
+              defined in src/osd/osd_types.h in Jewel, patch that file
+
+commit 00a79995523ae3b9ed6753b645188cb6759c1d07
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 13:32:35 2017 -0400
+
+    rgw: rest handlers for mdlog and datalog list dont loop
+    
+    the do-while loops are unnecessary, because cls_log_list() will already
+    give us the requested number of entries, unless truncated. and these
+    rest operations return the truncated flag and last_marker, which the
+    client side is already using to run the same loop
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d75e6b967d2ddf84d6e82c78a53061578129294a)
+
+commit e89036abb20d33df60c9a7dc7db4c8cd9825c068
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 12:19:52 2017 -0400
+
+    rgw: fix RGWMetadataLog::list_entries() for null last_marker
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dd2893ba05c1095405b3a5edc16b3b1c7916cd7d)
+
+commit e5902f442bf87d7ed319f8e74b2f1e85791a39b9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 12:18:33 2017 -0400
+
+    rgw: RGWMetadataLog::list_entries() no longer splices
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 112464483ecb407ef1388ba8a3f8d29cb0725594)
+
+commit 6f597f073fa2b4ad0941351606405d79d0899b67
+Author: Xie Rui <jerry.xr86@gmail.com>
+Date:   Thu Jun 22 17:12:30 2017 +0800
+
+    fix infinite loop in rest api for log list
+    
+    as max_entries_str is empty, it will be infinite loop when log is more than MAX_ENTRIES(1000) from marker.
+    and max_entries maybe overflow if use it to count the left entries.
+    
+    Fixes: http://tracker.ceph.com/issues/20386
+    Signed-off-by: xierui <jerry.xr86@gmail.com>
+    (cherry picked from commit 9fbc5378d2737d38e2ac915f20b44d308e18bc68)
+
+commit 3fa277b479d69699bf5a6875cd4a5efcf9ae0788
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jun 27 16:07:01 2017 +0400
+
+    jewel: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+    
+    Based on commit bc683385819146f3f6f096ceec97e1226a3cd237. The OSD code has
+    been refactored a lot since Jewel, hence cherry-picking that patch introduces
+    a lot of unrelated changes, and is much more difficult than reusing the idea.
+    
+    Fixes: http://tracker.ceph.com/issues/20428
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit a36211c84d8c475c88820f33a467936e0b270420
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Wed May 24 16:06:07 2017 +0800
+
+    rgw: update bucket cors in secondary zonegroup should forward to master
+    
+    pr#14082 is incomplete, cors should be redirect to master
+    
+    Fixes: http://tracker.ceph.com/issues/16888
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 7dd099588a47024b410d513cb8ea919731718fa7)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc (jewel does not have 5fff6371d8e)
+            src/test/rgw/rgw_multi/tests.py (jewel does not have this file)
+
+commit fa419a87b64f7c88a302ac594af81e07eb0ea430
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Apr 6 13:13:54 2017 -0400
+
+    rgw: fix for EINVAL errors on forwarded bucket put_acl requests
+    
+    PutACL rejects request that include both a canned_acl and a request
+    body. when forwarding requests with canned_acls, we were also including
+    the generated policy in the body
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c863a78ed73cf1d89f05276627c7f0bde53ec9a)
+
+commit 1d7bd61d3f5f07a3f6d30faa9325f93c89800ade
+Author: Guo Zhandong <guozhandong@cmss.chinamobile.com>
+Date:   Wed Mar 22 18:00:37 2017 +0800
+
+    rgw: enable to update acl of bucket created in slave zonegroup
+    
+    Fixes: http://tracker.ceph.com/issues/16888
+    
+    Signed-off-by: Guo Zhandong <guozhandong@cmss.chinamobile.com>
+    (cherry picked from commit 5f89d37506adab0aa59e7a8a46eb962030565c72)
+
+commit a3c487448bbded31c98e752ff1c9518aa76c2e67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue May 9 12:32:51 2017 +0200
+
+    ceph-disk: separate ceph-osd --check-needs-* logs
+    
+    It is using the OSD id zero but have nothing to do with OSD zero and
+    this is confusing to the user. The log themselves do not need to be kept
+    around and are stored in the run directory so that they can be disposed
+    of after reboot.
+    
+    Fixes: http://tracker.ceph.com/issues/19888
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c7b3c46bd63b78475868e405bf20d9c142f0336a)
+
+commit a4c181fa7ab6ced676b9ff4f1381555215d77152
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Mar 16 21:45:46 2017 -0400
+
+    mds: set ceph-mds name uncond for external tools
+    
+    External tools like logrotate which use killall rely on the "ceph-mds" name to
+    find it in /proc/*/comm. This is normally the case but when ceph-mds respawns
+    using /proc/self/exe (on Linux), its name will change to "exe". This makes
+    logrotate fail to signal ceph-mds the log needs to be reopened and will lead to
+    the log growing until the disk is full.
+    
+    This patch unconditionally sets the name so on respawn the name is correct (and
+    in any other possible scenario, such as a misnamed executable).  Note, there is
+    still a very small race between execve and ceph_pthread_setname where the
+    process name is wrong.
+    
+    Problem was introduced by 66a122025f6cf023cf7b2f3d8fbe4964fb7568a7.
+    
+    Fixes: http://tracker.ceph.com/issues/19291
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 4f177bb6b72cf9c8eb363051b27496c026b345f0)
+    
+    Conflicts:
+        src/ceph_mds.cc - use pthread_setname_np() instead of
+            ceph_pthread_setname(), drop compat.h include
+
+commit 3776d960e07570727cea9c6ef8516ffa9953b54e
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 29 17:14:54 2016 +0100
+
+    suites: update log whitelist for scrub msg
+    
+    Fixes: http://tracker.ceph.com/issues/16016
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 795094628b65eb9c3f5d51c6c895fe1443c5f4cf)
+    
+    Conflicts:
+            suites/fs/recovery/tasks/forward-scrub.yaml (file does not exist in jewel)
+
+commit d9234d47a9abc290475a317bb3fd070192df1840
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 23 09:07:32 2017 -0400
+
+    mds: include advisory `path` field in damage
+    
+    This will just be whatever path we were looking
+    at at the point that damage was notified -- no
+    intention whatsoever of providing any up to date
+    path or resolution when there are multiple paths
+    to an inode.
+    
+    Fixes: http://tracker.ceph.com/issues/18509
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c0bff51ef409eb6e4b2fc248e06e5a7e43faf51e)
+    
+    Conflicts:
+        src/mds/CDir.cc - omit dout(10) because jewel does not have cb86740a
+        src/mds/ScrubStack.cc - jewel does not have 7b456109 which changed
+           in->make_path_string_projected() call to in->make_path_string() but
+           it's moot because that line is dropped
+
+commit 7675e41c88a38e107e9478bde32e0ece564f8878
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 20:25:58 2016 +0100
+
+    mds: populate DamageTable from scrub and log more quietly
+    
+    Fixes: http://tracker.ceph.com/issues/16016
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9c82040b1a762a3498c6794a29a43b3866f30dab)
+
+commit 88605fc76bc943697ca5b4e235d82f439766d14c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 18:26:42 2016 +0100
+
+    mds: tidy up ScrubHeader
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 111d2cf2d8504cd4486180a95e52f253018364b3)
+    
+    Conflicts:
+            src/mds/CInode.cc (jewel does not have 5259683e7819c22c14b21b1dd678a33e14574f21)
+
+commit c15fce6e50eb5775687feb121da5ed59df34e28b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 17:35:54 2016 +0100
+
+    mds: remove redundant checks for null ScrubHeader
+    
+    This was originally optional but now all the paths
+    that kick off a scrub should be going through
+    enqueue_scrub and thereby getting a header set.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0c890282699f877f42870408d674ec1e9f9322a3)
+    
+    Conflicts:
+            src/mds/CInode.cc (jewel does not have 5259683e7819c22c14b21b1dd678a33e14574f21)
+
+commit c17e3c1b7b6961f3fb866af76aa2d798cdb97c9f
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Tue Oct 11 21:23:57 2016 +0100
+
+    mds/DamageTable: move classes to .cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 96018b0a85d0bc7eec285398dd596ee01d16fae7)
+
+commit 52a00b3bffc2db22c15d0e5b1e9580f9bc593555
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Sep 16 10:56:51 2016 +0200
+
+    test: timeout verification that mon is unreachable
+    
+    Without a timeout on the command, it may hang for a very long time,
+    hunting for new mons. If it hangs for more than 60 seconds, it is
+    safe to assume the mon is indeed down.
+    
+    Fixes: http://tracker.ceph.com/issues/16477
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a5e5119bd138a97f35737c2b50d1e621fa8286d6)
+
+commit 3b5f9924b3db66fae952cf67331b69789045916f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Sep 15 10:42:24 2016 +0200
+
+    cli: retry when the mon is not configured
+    
+    The mon may return on error if a command is sent to it while it is
+    still in configuring state.
+    
+    Fixes: http://tracker.ceph.com/issues/16477
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d5a3c8d6844ba632767baf3c2c790fa4d947a95e)
+    
+    Conflicts:
+            src/pybind/ceph_argparse.py (trivial resolution; jewel does not have
+                8fc67075c6d7d4443747f53687e498439f80b57a)
+
+commit 00ce80b52f6f231125cc165ab477544238ed3a02
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue May 9 11:42:58 2017 +0200
+
+    build/ops: rpm: fix python-Sphinx package name for SUSE
+    
+    This commit moves "BuildRequires: python-sphinx" down to the RH/CentOS/Fedora
+    distro conditional and adds a "BuildRequires: python-Sphinx" to the SUSE
+    conditional.
+    
+    Signed-off-by: Jan Matejek <jmatejek@suse.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0688f57d1c35692cdddf219d9ac3571d03dbb8e0)
+
+commit 66c17ff10b1703d0d9098d0ddcaf34a644ff5fbe
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Jun 14 15:34:08 2016 +0800
+
+    rgw: remove unnecessary output
+    
+    a lot of radosgw-admin command will output
+    "error in read_id for object name: default : (2) No such file or directory"
+    when the zone named 'default' is not exist:
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 4e9c7be08b9828db2795536ce170a675d25bc1ed)
+
+commit b786f7815b94e7659a07778a6318ffe5e5079fc0
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Mon Mar 20 15:19:22 2017 +0800
+
+    rgw: delete non-empty buckets in slave zonegroup returns error but the
+    buckets have actually been deleted.
+    
+    Fixes: http://tracker.ceph.com/issues/19313
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 4714b17f53a9f30f7f155a69cfbfc682f809b4e4)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc - first argument to delete_bucket() and
+                check_bucket_empty() is "s->bucket" in jewel (master:
+                "s->bucket_info")
+            src/rgw/rgw_rados.cc - RGWRados::delete_bucket() takes bucket instead
+                of bucket_info; adapt RGWRados::check_bucket_empty() to take bucket
+                instead of bucket_info as well
+            src/rgw/rgw_rados.h - ditto
+
+commit 7e3db1519e40ecdde5d7e83798a35f3fca7e5cd8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:42:44 2017 -0400
+
+    rgw: fix for zonegroup redirect url
+    
+    local dest_url variable was shadowing the one in the enclosing scope, so
+    the changes were not applied and no Location header was written on redirect
+    
+    Fixes: http://tracker.ceph.com/issues/19488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 542e188a40f0495720b48308372366951ae41e62)
+
+commit 45f0396cffe528a752fb8330d2323e1de8d8b8ac
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:41:51 2017 -0400
+
+    rgw: use zonegroup's master zone endpoints for bucket redirect
+    
+    if no zonegroup endpoints are set, fall back to master zone endpoints
+    
+    Fixes: http://tracker.ceph.com/issues/19488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 92f63c6392bdc4633a2e57cb3867051bb1a3fd55)
+
+commit 50f140768d56d7813efac7ab2def3619e9639b4c
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jan 24 15:20:32 2017 +0800
+
+    rgw: put object's acl can't work well on the latest object when versioning
+    is enabled.
+    
+    Reported-by: Liu Hong <liuhong@cmss.chinamobile.com>
+    
+    Fixes: http://tracker.ceph.com/issues/18649
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit fe5f95fff30cc872484de6efb6cba1dd48b09316)
+    
+    Conflicts:
+         src/rgw/rgw_op.cc: read_op.prepare() takes no arguments in master; replace
+             this with read_op.prepare(NULL, NULL) to make jewel happy
+
+commit 01890abd953881ab1529652d148f7a19d171e18c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 21 15:04:48 2017 -0400
+
+    radosgw-admin: warn that 'realm rename' does not update other clusters
+    
+    Fixes: http://tracker.ceph.com/issues/19746
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a459c768ffecd689a53a79dfe33eb8f1bbc318f)
+
+commit 89a2f6476d81bd233d76771915e080fce66ff228
+Author: Willem Jan Withagen <wjw@digiware.nl>
+Date:   Sat Apr 29 13:36:07 2017 +0200
+
+    cls/log/cls_log.cc: reduce logging noise
+    
+     - The other reference in the source as already at 20.
+          ./src/cls/timeindex/cls_timeindex.cc:85:
+            CLS_LOG(20, "storing entry at %s", index.c_str());
+    
+       And we need not always know where in the log items are stored.
+       So it looks like a leftover debug feature.
+    
+    Fixes: http://tracker.ceph.com/issues/19835
+    Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>
+    (cherry picked from commit d76010900bf9012f2e66335787710531772766b7)
+
+commit 938e5b0728c07997abf09d30c670ffe1737a65ab
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 15 17:30:29 2017 -0400
+
+    rgw_file: v3: fix write-timer action
+    
+    For now, unify with v4 write-on-close path, by calling
+    RGWFileHandle::close() on write-timer expire, since it will
+    call write_finish() as a side-effect.
+    
+    Fixes: http://tracker.ceph.com/issues/19932
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ce6ecb553b85ea158af28c22827b93135a75d159)
+
+commit 0897afe74d381c19a9b1bc1c2b398f93568a86fe
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Mar 7 09:48:57 2017 -0500
+
+    rgw_file:  fix fs_inst progression
+    
+    Reported by Gui Hecheng<guimark@126.com>.  This change is a
+    variation on proposed fix by Dan Gryniewicz<dang@redhat.com>
+    to take root_fh.state.dev as fs_inst for new handles.
+    
+    Fixes: http://tracker.ceph.com/issues/19214
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e988edfb6ab9085d6d37cfc444d46e8a2841943)
+
+commit a54e6ccba31109d59abd8e7b9ba1f8fada30ecda
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 22 17:37:11 2017 -0400
+
+    rgw_file:  remove post-unlink lookup check
+    
+    This could induce asserts in multi-nfs-gateway scenarios.
+    
+    Fixes: http://tracker.ceph.com/issues/20047
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b3db617ddd6fa2477726b54828b6410b36989ac7)
+
+commit 7042733db61e8a773969d3ce5fc24a1df6ae417b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue May 23 15:05:45 2017 -0400
+
+    rgw_file: release rgw_fh lock and ref on ENOTEMPTY
+    
+    An early return in new unlink bucket num_entries check was
+    missing a conditional unlock and unref.
+    
+    Fixes: http://tracker.ceph.com/issues/20061
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 51c25593dd741b4df2e839fcef82b143f8c8cfca)
+
+commit 4a32ef856554921dab503d70257bd4922159831d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 22 14:51:19 2017 -0400
+
+    rgw_file: remove hidden uxattr objects from buckets on delete
+    
+    If a setattr (e.g., chown) has been performed on a bucket, then like
+    any directory it has a hidden object storing its attributes. This must
+    be deleted before attempting bucket delete, otherwise, actually empty
+    buckets will not be removable via NFS.
+    
+    Fixes: http://tracker.ceph.com/issues/20045
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 01e15c3ec6cf43a7a7c4e2d13aad5f385c9f9748)
+    
+    Conflicts:
+            src/rgw/rgw_file.cc (jewel does not have
+                    "inline int valid_fs_bucket_name()" and
+                    "inline int valid_fs_object_name()")
+
+commit b63b7886dc1fb704f8baa9749a20e09faa93c1d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Apr 24 16:23:21 2017 +0200
+
+    test/librbd/test_notify.py: don't disable feature in slave
+    
+    On jewel it will have stolen the exclusive lock. Instead, ensure that
+    object map and fast diff are already disabled on the clone before the
+    start of the test.
+    
+    Fixes: http://tracker.ceph.com/issues/19716
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e009e1bdd4b3997462feb9a050bd2eb201e028ba)
+
+commit 5c0986153e79d7ec4bc165e497689319bd95ada9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri May 5 15:59:44 2017 +0200
+
+    rbd-nbd: relax size check for newer kernel versions
+    
+    Fixes: http://tracker.ceph.com/issues/19871
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8e912b9a0564a57f1b761e9e567771197bd0fb98)
+
+commit bc8c5d6c77880d21248c683609c3814145008f36
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 23 12:07:45 2017 +0200
+
+    librbd: potential read IO hang when image is flattened
+    
+    Fixes: http://tracker.ceph.com/issues/19832
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from 10d58618e7c632ef01b9537492239e0a042dc17e)
+    
+    Conflicts:
+        src/librbd/AioObjectRequest.cc: manual backport because the master changes
+            are in a different file (src/librbd/io/ObjectRequest.cc)
+
+commit 2e278d821d1e5992f77e6ac1f0b3e1c7fcba7e12
+Author: redickwang <redickwang@tencent.com>
+Date:   Fri May 19 15:08:12 2017 +0800
+
+    rgw: segment fault when shard id out of range
+    
+    Fixes: http://tracker.ceph.com/issues/19732
+    
+    Signed-off-by: redickwang <redickwang@tencent.com>
+    (cherry picked from commit ff4c40fc2e3c092d17a218ae6132de4e6034c8ee)
+
+commit 5adc66bc2abe5b8d58111d52b23619edfbebbc7f
+Author: Thomas Serlin <tserlin@redhat.com>
+Date:   Wed May 24 12:07:41 2017 -0400
+
+    Set subman cron attributes in spec file
+    
+    Fixes: http://tracker.ceph.com/issues/20074
+    
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit a9eb6ecea7b9512722e2283e5acb4fc6a1b7b734)
+
+commit 21b00c3ec0ae60848f6677b5fe23e6321b0f8de8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 30 13:50:41 2017 -0400
+
+    osd/PrimaryLogPG: do not expect FULL_TRY ops to get resent
+    
+    The objecter will not resend FULL_TRY requests that were sent prior to
+    becoming full, so we should not discard them.
+    
+    Fixes: http://tracker.ceph.com/issues/19430
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f7acdbc9a942fd18937dbcf07fbc7b752c50ba3)
+
+commit afd036cb6725f9e91cfd38995d6536d92deb934d
+Author: lvshuhua <lvshuhua@cmss.chinamobile.com>
+Date:   Wed Dec 7 15:47:47 2016 +0800
+
+    rgw: fix versioned bucket data sync fail when upload is busy
+    
+    Fixes: http://tracker.ceph.com/issues/18208
+    
+    Signed-off-by: lvshuhua <lvshuhua@cmss.chinamobile.com>
+    (cherry picked from commit ce7d00ac1642d84c1d9111156a544c37801c5adf)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (in jewel, inc_marker is declared to be of
+                type rgw_bucket_shard_inc_sync_marker, while in master it is of
+                type rgw_bucket_shard_inc_sync_marker& - i.e. reference to
+                rgw_bucket_shard_inc_sync_marker)
+
+commit c2137f2f725d8536066b6b495f8565b126a720ed
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 19 13:51:50 2017 +0200
+
+    libradosstriper: delete striped objects of zero length
+    
+    This patch fixes the bug: "libradosstriper fails to delete striped objects of
+    zero length without returning any kind of error"
+    
+    N.B.: This is not cherry picked from master because
+    7a50ea479e7e5c2909d899d89d33d3fb082257f8, which fixes the issue in master, is
+    too complicated to backport.
+    
+    Fixes: http://tracker.ceph.com/issues/20325
+    Signed-off-by: Stan K <redrampage@selectel.ru>
+
+commit fb79932be69545b286497d0ee24c96d5e466344b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 18 13:54:56 2017 -0400
+
+    mon/PGMonitor: clean up min/max span warning
+    
+    Clean up option naming.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 489e810c37ed6fb9d32d1015634041a577501ee4)
+    
+    Conflicts:
+            src/common/config_opts.h: Jewel's options are quite different from ones of master,
+              s/mon_warn_osd_usage_percent/mon_warn_osd_usage_min_max_delta/
+
+commit 8180f1612a3dfac6ba5e8914f9e6a3faa8cf0297
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 22:23:06 2017 -0800
+
+    osd: Round fullness in message to correspond to df -h
+    
+    This really only works after journal drains because
+    we adjust for the journal.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 26dcb591f9af01ed444aa758c3d601e7f67261b2)
+
+commit 728097af06e1e49b5f02111887ce6716d3729213
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 17:25:12 2017 -0800
+
+    filestore: Account for dirty journal data in statfs
+    
+    Fixes: http://tracker.ceph.com/issues/16878
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 78adb70c21c6b8e6a9191b76917919b125a9490f)
+    
+    Conflicts:
+            src/os/filestore/FileStore.cc: adapt the code to use struct
+              statfs to keep changes minimal
+
+commit c6e4fff5e3f5c5a45b3c4fb647548bd2a2dfea0c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 16:37:07 2017 -0800
+
+    mon: Add warning if diff in OSD usage > config mon_warn_osd_usage_percent (10%)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8004e6558359fb542e45bb4b483a6c91afdc0b4)
+    
+    Conflicts:
+            src/common/config_opts.h: some options in the original patch hunk
+              are not supported in Jewel, manually add the new `mon_warn_osd_usage_percent'
+              option.
+
+commit 0412bf2adaf07e527ac6659c9bce0d406edeca93
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:40:05 2017 -0800
+
+    mon: Bump min in ratio to 75%
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 830cc7aa7be1ccd8f54f056b6a58e923cadd1c2b)
+
+commit 21e2a0337092d57f3e23d282b02b99146ed50a63
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:38:53 2017 -0800
+
+    osd: Fix ENOSPC crash message text
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50cfe03fcba253c8380b21043ed03879134d6836)
+    
+    Conflicts:
+            src/os/bluestore/BlueStore.cc: leave bluestore code as is,
+              bluestore users should use Kraken or Luminous
+
+commit 0068d9f3335fc8d6b7bfb5c29aa8cba87c0d01f2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:19:55 2017 +0800
+
+    msg/async: go to open new session when existing already closed
+    
+    Fixes: http://tracker.ceph.com/issues/20230
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 99f580a3959240f99061a9ad48ec591b39a9fd46)
+    
+    Changes: src/msg/async/AsyncConnection.cc: Mutex::Unlock() in Jewel
+               versus mutex::unlock() in master
+
+commit 28aa104c0c8f9d6c556aeab881231034191b55d1
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:16:47 2017 +0800
+
+    msg/async: fix accept_conn not remove entry in conns when lazy delete
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit bf98babb3289a7714543ff3cbd3872d80f0dc196)
+
+commit 9212d62007394d9219a1c629d875610aa9aeab4b
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sun Jun 19 16:01:31 2016 +0100
+
+    msg/AsyncMessenger.h:remove unneeded use of count
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit b9ccc756614870142ba878ffdf82c7c7c89e49a2)
+
+commit 8cf3f18c7af3a096da4f2e40f6c273c3e39c7e8f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed May 3 11:32:34 2017 -0400
+
+    rgw: RGWPeriodPusher spawns http thread before cr thread
+    
+    Fixes: http://tracker.ceph.com/issues/19834
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 21a943da45032a276997a0a885e869e2c2bc321d)
+
+commit 730c88e800ceedea51c92d9f3a59aeba709b692d
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Jun 27 19:40:13 2016 +0800
+
+    rgw: should delete in_stream_req if conn->get_obj(...) return not zero value
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 494fbe9ce595b0b59597bd426cc3bc4d14ba6644)
+
+commit 9b12fe08098a686afe96f7965b24ee331c7902ab
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Apr 20 16:33:32 2017 -0400
+
+    rgw: dont spawn error_repo until lease is acquired
+    
+    if RGWDataSyncShardCR fails to acquire its lease, it doesn't call
+    error_repo->finish() to stop the coroutine. wait until the lease
+    succeeds before spawning the error_repo
+    
+    Fixes: http://tracker.ceph.com/issues/19446
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1524a5eb249113fed8da1d47a4f92dec98db4566)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (different argument list of RGWOmapAppend()
+                function in jewel, compared to master)
+
+commit 6b99bc9c6083190e65825f2fd4a2d18e7f03a4ec
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 2 09:45:06 2017 +0200
+
+    build/ops: deb: fix logrotate packaging
+    
+    This minimal jewel-only fix is not cherry-picked from master because
+    the Debian packaging was refactored between jewel and master and the
+    master fix is totally different.
+    
+    Fixes: http://tracker.ceph.com/issues/20316
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4be3db597321fd61caa202af49f8c2d941db552d
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Jan 5 11:50:42 2017 +0800
+
+    rgw: fix 'gc list --include-all' command infinite loop the first 1000 items
+    
+    When the items to gc over 1000, 'gc list --include-all' command will infinite loop the first 1000 items.
+    Add next_marker to move to the next 1000 items.
+    
+    Fixes: http://tracker.ceph.com/issues/19978
+    
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit fc29f52ebca63104a05515484088ff136dfb0b15)
+
+commit 7966b846a875c60b5272f32baec648a719a10b9e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:58:05 2017 -0400
+
+    test/rgw: wait for realm reload after set_master_zone
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5cc99db52423f9ee98a549b215559c677f178f48)
+
+commit 97ffe49defb213c26288744ed572fbb5b158583a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:56:20 2017 -0400
+
+    test/rgw: fixes for test_multi_period_incremental_sync()
+    
+    test was only creating objects in subsequent periods, which wasn't
+    adding any entries to the mdlog. this wasn't correctly testing
+    incremental metadata sync across periods
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2976fd36cded95f68766f8ba485d43f932e97db2)
+
+commit 7c19e375be379cd8366097b57adea9f6010d783c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:00:15 2017 -0400
+
+    test/rgw: meta checkpoint compares realm epoch
+    
+    avoid marker comparisons between different periods
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 20df35ad94aea0c03be0c2c9739bd239354a46d8)
+
+commit cb957226083f7644db5b5826a3d7ab21c47def8e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 21 12:19:01 2017 -0400
+
+    rgw: remove rgw_realm_reconfigure_delay
+    
+    when the master zone is changed, this config variable was increasing the
+    window of time where the old master zone would continue to handle
+    requests to modify metadata. those changes would not be reflected by the
+    new metadata master zone, and would be lost to the cluster
+    
+    it was an attempt to optimize for the unlikely case of multiple period
+    changes in a short period of time, but the logic in reload() handles this
+    case correctly as is
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f422d4f1841e15a4ecf2d9304aa77021e8bd8626)
+
+commit 48d596df141f5014ae5ca9ab4e8955a6c6ba755e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 21 16:10:27 2017 -0400
+
+    rgw: require --yes-i-really-mean-it to promote zone with stale metadata
+    
+    if a zone is promoted to master before it has a chance to sync from the
+    previous master zone, any metadata entries after its sync position will
+    be lost
+    
+    print an error if 'period commit' is trying to promote a zone that is
+    more than one period behind the current master, and only allow the
+    commit to proceed if the --yes-i-really-mean-it flag is provided
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 721e3d6ee5917b19cfc15e3e9582d23623b8cca7)
+
+commit 81ced03d203ec00376fef5a143e4cac24aa65093
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 20 16:13:03 2017 -0400
+
+    rgw: period commit uses sync status markers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit bb49e2fbed3a87de837329cfa0c11f8d97633a94)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: RGWPeriod::update removed
+
+commit a9830dbdd43572995b3e8e157ecbc706d6481fac
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 1 15:21:52 2016 -0400
+
+    rgw: use RGWShardCollectCR for RGWReadSyncStatusCoroutine
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a4bf014b8642073f3eac226a93f6360cdd9cee25)
+    
+    Conflicts:
+            src/rgw/rgw_sync.cc: rgw_pool, rgw_raw_obj
+
+commit 0ab8e0ab5db988724bb0e26c88a279d099d3b8cd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 1 14:35:53 2016 -0400
+
+    rgw: change metadata read_sync_status interface
+    
+    makes the same change to read_sync_status() in RGWMetaSyncStatusManager,
+    needed to support multiple concurrent readers for the rest interface
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0372df20be86cb616600626ee4c755b31032f134)
+
+commit ac9045abba05818108655c75ecb57cdf065f56e8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 17 09:55:47 2017 -0400
+
+    rgw: store realm epoch with sync status markers
+    
+    sync status markers can't be compared between periods, so we need to
+    record the current period's realm epoch with its markers. when the
+    rgw_meta_sync_info.realm_epoch is more recent than the marker's
+    realm_epoch, we must treat the marker as empty
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 574ff5f5df62a0a1e9a8ff7e4a87cd65cc952d7e)
+    
+    Conflicts:
+            src/rgw/rgw_sync.cc: rgw_pool
+
+commit 013d349c791318a03cf7bd0bbbe6d4bec708359d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 27 15:19:54 2017 -0500
+
+    rgw: RGWBackoffControlCR only retries until success
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a898fb76a4179add68ccb526f2917768736ac52b)
+
+commit 237368752b013ad639581b506fdb46e2119f155a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 7 12:28:33 2017 -0500
+
+    rgw: clean up RGWInitDataSyncStatusCoroutine
+    
+    RGWInitDataSyncStatusCoroutine operates on a given rgw_data_sync_status
+    pointer, which saves us from having to read it back from rados
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 69be410d0e29cd8b4cf5171800e4a7fa938eb8c6)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc: rgw_pool, rgw_raw_obj
+
+commit 67011e51a7ba1538ad2c5f426311ae71bb5f8c34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 7 11:16:43 2017 -0500
+
+    rgw: fix marker comparison to detect end of mdlog period
+    
+    Fixes: http://tracker.ceph.com/issues/18639
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7c23713b1a3c14b135f8eb6fdb36d113ea860e4e)
+
+commit c19c72058e9a09ccfc316ebe0c53348be1e2ba32
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 22 22:06:15 2016 -0400
+
+    rgw: add == and != operators for period history cursor
+    
+    RGWMetaSyncCR was using operator== but it always returned true!
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 09c847ff790f55004871fb7304361ae6c9845b1a)
+
+commit 1c344e254efcd802e417cc45b6678989c871ad0a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 8 09:37:26 2016 -0400
+
+    rgw: add empty_on_enoent flag to RGWSimpleRadosReadCR
+    
+    RGWSimpleRadosReadCR won't currently fail with ENOENT, but instead
+    passes an empty object to handle_data(). add an empty_on_enoent flag to
+    the constructor, defaulting to true, to make this behavior optional for
+    callers that do want to fail on ENOENT
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c5c95e7f225d59a8bdd8eda3742053b77492c40c)
+
+commit bf973c8400db08dd541152812d42379cf113755d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 26 09:52:25 2017 +0200
+
+    rpm: Move ceph-disk to ceph-base
+    
+    The SELinux package now requires the ceph-disk binary but that one was
+    part of the ceph-osd package. The ceph-disk python library is already
+    packaged in ceph-base so moving ceph-disk to ceph-base seems like a
+    reasonable next step.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6991764f3bff7b8f6be699603927aff882377878)
+    
+    Conflicts:
+            ceph.spec.in: we have ceph-disk-udev in jewel
+            ceph.spec.in: few if conditions were different in jewel
+
+commit 6ca61b1b478f5dbbd33ddf87e0fc1af999ed75cb
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 25 14:36:13 2017 +0200
+
+    ceph-disk: Fix the file ownership, skip missing
+    
+    This commit fixes the file ownership for the /usr/bin/ and /etc/ceph
+    files and skips missing files as some of the files that we do specify
+    now can be missing from the system (not installed, e.f. radosgw).
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 077038b4393a28ccbd38ca4a90105dbd4c1ffcd5)
+
+commit 7d6eea0440f9856894570af80a14ba6c19a3c910
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 28 12:29:46 2017 +0200
+
+    selinux: Do parallel relabel on package install
+    
+    We can take advantage of ceph-disk fix subcommand when doing a package
+    install. We will keep using the differential fixfiles command otherwise.
+    
+    We also need to add relabel for /usr/bin/ daemons so that we could use
+    this.
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1cecddf031991f1c64ea203f173189624f11940e)
+
+commit b9ce1aa618d1f488f91f349bed8c87a421fcd5ba
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 3 21:36:21 2017 -0400
+
+    rbd-mirror: ensure missing images are re-synced when detected
+    
+    Fixes: http://tracker.ceph.com/issues/19811
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74bd4f230a0cb7b709f2cb5c6db3dc79f0d8dede)
+    
+    Conflicts:
+            src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc: trivial resolution
+            src/tools/rbd_mirror/image_replayer/BootstrapRequest.h: trivial resolution
+
+commit 648dfa1e939fa65ef629b7e4a31f2eda2b0b55c0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 5 15:40:10 2017 +0800
+
+    qa: add a sleep after restarting osd before "tell"ing it
+    
+    without the fast-fail feature, the monitor does not mark osd down after
+    a grace time. so we cannot truest the "healthy()" in ceph.restart task.
+    
+    also, "restart" task wait-for-healthy by default, so no need to do it
+    explicitly.
+    
+    Fixes: http://tracker.ceph.com/issues/16239
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts: this change is not cherry-picked from master. because, in
+               master we have the fast-fail feature, but we have'not
+               backport it to jewel. so adding a timeout would help to
+               make sure that the OSD is back online even "restart"
+               waits for "healthy" already.
+
+commit 01abfed8f26eab70dc3242aeaa4c44599f542af2
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 22:32:01 2017 -0700
+
+    osd: do not default-abort on leaked pg refs
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 4caf2df0c380a1281db9509b3feb342705512b58)
+    
+    Conflicts:
+            qa/clusters/fixed-4.yaml
+            src/osd/OSD.cc
+    
+    Fixes: http://tracker.ceph.com/issues/20084
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 967d6a54055b438d6730ab7b7c1fc1a1b980c5e2
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 21:52:49 2017 -0700
+
+    osd: Reset() the snaptrimmer on shutdown
+    
+    We were failing to exit various wait states which held PGRefs. Error!
+    
+    Fixes: http://tracker.ceph.com/issues/19931
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit b0e9deeea8a8e90f6d7e9d56b6b4aed890e01d7b)
+    
+    Conflicts:
+            src/osd/ReplicatedPG.cc
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 328e2cc71aaefa07a04381a7f752dac46b566da0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 10:08:33 2017 -0500
+
+    mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+    
+    If we only fill OSDs to 95%, we should factor that into
+    the MAX AVAIL calculation for the pool.
+    
+    Fixes: http://tracker.ceph.com/issues/18522
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f223ac92917f4bc18e5b9b3ad61afa155e4d088a)
+    
+    Conflicts:
+            src/mon/PGMap.cc: get_rule_avail() is a method of the PGMonitor
+              class in Jewel (and osd_stat is pg_map.osd_stat)
+
+commit b52bfe6b443f0ff88c8614441752102058063699
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Thu Apr 6 11:12:04 2017 +0000
+
+    os/filestore: fix infinit loops in fiemap()
+    
+    since fiemap can get extents based on offset --> len
+    but we should consider last extents is retrieved when len == 0
+    even though it is not last fiemap extents
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit 36f6b668a8910d76847674086cbc86910c78faee)
+
+commit e3cba06bd02b1fca07414c987cf147f1ded99b68
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 5 12:02:05 2017 +0800
+
+    mon: check is_shutdown() in timer callbacks
+    
+    introduce a helper class: C_MonContext, and initialize all timer events
+    using it, to ensure that they do check is_shutdown() before doing their
+    work.
+    
+    Fixes: http://tracker.ceph.com/issues/19825
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d)
+    
+    Conflicts:
+            src/mon/MgrMonitor.cc: no such service in Jewel
+
+commit ef5265f107858d37bbdeea165cec571db614b4a3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 4 22:49:04 2017 +0800
+
+    mon/Elector: call cancel_timer() in shutdown()
+    
+    instead of doing it manually.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 12139ae529a49b6caedea89f910d034ddca094b6)
+
+commit 9693b6bd1685f3ff1a3a44972b7e7f0e03d5da12
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed May 17 17:50:10 2017 +0400
+
+    jewel: mon: add override annotation to callback classes
+    
+    The only purpose of this patch is to avoid merge conflicts while
+    cherry-picking commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d.
+    Alternatively one could cherry-pick 1effdfe19bf9fd6d546620b96eaf452e889b15dc,
+    but that one brings a lot of unrelated changes.
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ea1b21d9604f19a48edeacc68ebabf8716143c2e
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Mon Aug 1 16:52:46 2016 +0100
+
+    mon/PaxosService: move classes to cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit a4e979a41c7d3649cf70f9d0768015d7ff4aca8a)
+
+commit 1869ed9b2d4225e2bf0f366baf75e0e14f202aea
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Fri Sep 23 19:43:56 2016 +0100
+
+    mon/Paxos: move classes to .cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit d21357a7e042424754d9ab41c134cae9e89945f9)
+
+commit 18cf302377aa7a09ad4922c07c64fefc047f2bdb
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sun Jul 24 10:51:54 2016 +0100
+
+    mon/Elector:move C_ElectionExpire class to cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit c819596e59fd5f66b3258fedd1d10b20cf70f0a9)
+
+commit fa33e7600b0205ca9113a1c5be535516886a0701
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sat Jul 30 09:58:30 2016 +0100
+
+    mon/Monitor: move C_Scrub, C_ScrubTimeout to .cc
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit e58750127cbe8728f4047411904af3890966490f)
+
+commit 9ec5d8b1a5ea7fa437b161a49d246a7f0ff653c8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 11 13:13:39 2017 +0800
+
+    osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+    
+    when a callback is called, it could be facing a PG already shut down by
+    OSD. but if that callback wants to shut that PG down. it should check
+    the PG's status first.
+    
+    Fixes: http://tracker.ceph.com/issues/19902
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3c44a0dfc859f6f625a92e727e0e521ed4a9207)
+
+commit 592293cdfb397337c2808117ae9b06d5dc77f7f2
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jan 31 13:54:38 2017 +0400
+
+    systemd/ceph-disk: make it possible to customize timeout
+    
+    When booting a server with 20+ HDDs udev has to process a *lot* of
+    events (especially if dm-crypt is used), and 2 minutes might be not
+    enough for that. Make it possible to override the timeout (via systemd
+    drop-in files), and use a longer timeout (5 minutes) by default.
+    
+    Fixes: http://tracker.ceph.com/issues/18740
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 22332f6bae57a6975a99523a115eb70608f26331)
+
+commit 3828c1443b0a580fa23680da6d0b624d76a6de2a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Mar 8 09:38:39 2017 +0100
+
+    ceph-disk: Add --system option for fix command
+    
+    This adds the ability to restore the labels of the underlying system
+    data in addition to ceph data.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 8d81af42fd507c7b92c8279eb114b0a733ac1da6)
+
+commit 5b4132266cc8a4d53f9e045bc4b21706a9eeabc1
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 16 11:34:27 2017 +0100
+
+    ceph-disk: Add more fix targets
+    
+    It turns out I forgot several more directories that needs to be fixed by
+    this script. We need to fix /var/log/ceph, /var/run/ceph and /etc/ceph
+    as well.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ae139307d6b2bfba47e21d29d6dbd3c8dc01b5b5)
+
+commit 7341ee445829114e3dfb57a61ca3a0e02b666a6e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 9 19:17:12 2017 +0100
+
+    ceph-disk: Add unit test for fix command
+    
+    This will simulate the command* functions to not actually run anything
+    thus excercising the python code directly. It also checks that the
+    proper (sub-strings) are in the output.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1ec53dee9a690134936bdc3a09c9a02fecf13a9d)
+
+commit 3122fa8ca37475fb1cbab296b89c5904ea7b6f76
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jan 31 13:19:33 2017 +0100
+
+    ceph-disk: Add fix subcommand
+    
+    This subcommand will fix the SELinux labels and/or file permissions on
+    ceph data (/var/lib/ceph).
+    
+    The command is also optimized to run the commands in parallel (per
+    sub-dir in /var/lib/ceph) and do restorecon and chown at the same time
+    to take advantage of the caching mechanisms.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6d5d30f6ed7538271579cc2ef4e2e364f01a4a6f)
+    
+    Conflicts:
+            doc/man/8/ceph-disk.rst: whitespacing changed
+            src/ceph-disk/ceph_disk/main.py: command_check_call changed type
+
+commit 618a82e11e53cf2f1d5d306a2eb330cdf5e459ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:56:22 2017 -0400
+
+    librbd: add no-op event when promoting an image
+    
+    The rbd-mirror process needs an event in the journal
+    to properly detect the transition between primary and
+    non-primary state between peers.
+    
+    Fixes: http://tracker.ceph.com/issues/19858
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4031555dda7597d24e9eb04b9ff29173909586f7)
+    
+    Conflicts:
+            src/librbd/journal/DemoteRequest.cc: logic exists in Journal.cc
+    
+    (cherry picked from commit 7970ec586bebd26b1ca4955136ad8f48bb833af6)
+    
+    Conflicts:
+            src/librbd/journal/PromoteRequest.[h|cc]: logic exists in Journal.cc
+            src/librbd/journal/Types.[h|cc]: trivial resolution
+            src/test/librbd/journal/test_mock_PromoteRequest.cc: does not exist
+
+commit 7ec6e8bc50b62b43f27cf572c9d267c1a5f9520e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:57:34 2017 -0400
+
+    rbd-mirror: prevent infinite loop when computing replay status
+    
+    If the image had a non-primary predecessor epoch whose tag tid
+    duplicates an epoch within its own journal, an infinite loop
+    would result.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f179bf86216540d8e25aad469c604f96f6aecd8)
+
+commit 09f076ed968209507b1f8a27388c3959cb91c82b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 14 13:21:38 2017 -0400
+
+    osd: fix occasional MOSDMap leak
+    
+    _committed_osd_maps() may return early (without putting
+    the ref) on shutdown.
+    
+    Fixes: http://tracker.ceph.com/issues/18293
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a74632f964e10a57fb8065aec90dc7b69bddba5b)
+
+commit cd8c51569baaccf3ea57507f427db373c415934e
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Apr 20 12:13:13 2017 +0400
+
+    rgw: add the remove-x-delete feature to cancel swift object expiration
+    
+    In openstack swift, it also support the feature to cancel the object expiration,
+    which could be found at last point in
+    https://docs.openstack.org/user-guide/cli-swift-set-object-expiration.html. we
+    can remove the object expiration by set 'X-Remove-Delete-At:'.
+    
+    This patch also could fix the bug that when we set the object expiration and
+    then upload the same object to the container again. The previous object expiration
+    also works, which is not compatible with the openstack swift.
+    
+    Fixes: http://tracker.ceph.com/issues/19074
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 230429ebc4ac9b5840bb93c7e0eeb5edbb949106)
+    
+    Conflicts:
+            src/rgw/rgw_file.cc:
+              - processor->complete() has no object_size argument
+                (compression is not supported in Jewel)
+            src/rgw/rgw_op.cc:
+              - processor->complete() has no object_size argument
+                (compression is not supported in Jewel)
+              - RGWPostObj::execute(): the code changed a lot in master,
+                adjust delete_at manually (patch(1) tries to apply
+                the corresponding hunk to a different function)
+            src/rgw/rgw_rest_swift.cc:
+              - trivial: STREAM_IO in Jewel versus dump_header in master
+
+commit 10636ef77f7327b85d9128c3ba0c0f05829ee821
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Sat Jan 21 01:44:31 2017 +0530
+
+    osd/Pool: Disallow enabling 'hashpspool' option to a pool without
+              '--yes-i-really-mean-it'
+    
+    Fixes: http://tracker.ceph.com/issues/18468
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 3715362883acf9b8a477eb058a9ad6ee6d81ae01)
diff --git a/doc/changelog/v10.2.2.txt b/doc/changelog/v10.2.2.txt
new file mode 100644
index 000000000..cda69548d
--- /dev/null
+++ b/doc/changelog/v10.2.2.txt
@@ -0,0 +1,3002 @@
+commit 45107e21c568dd033c2f0a3107dec8f0b0e58374 (tag: refs/tags/v10.2.2)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--abdffd45-81df-4aa8-a769-e169993c7a0f.localdomain>
+Date:   Tue Jun 14 11:43:24 2016 +0000
+
+    10.2.2
+
+commit 008c5af59b045f9b6b8046a4c6d5e915cbe18bb3
+Merge: b733244 dd1ea65
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 07:32:44 2016 -0400
+
+    Merge pull request #9692 from dillaman/wip-16292
+    
+    jewel: librbd: remove should ignore mirror errors from older OSDs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dd1ea65c2c8ae966c1f07ce78f0d1f16c9a315d9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 15:29:43 2016 -0400
+
+    librbd: remove should ignore mirror errors from older OSDs
+    
+    Fixes: http://tracker.ceph.com/issues/16268
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a97a9b7ef238c4ceb16225a0b937f9a0cdbb2b6)
+
+commit b733244b3dc46e11689ddfb46187877c01347d3a
+Merge: 1feddef 2656881
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 07:24:28 2016 -0400
+
+    Merge pull request #9691 from dillaman/wip-16260-jewel
+    
+    jewel: librbd: do not shut down exclusive lock while acquiring'
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1feddef07c182d5a50b3aa3c4a7ff87f21dbd3e1
+Merge: ffb64cd a38caf9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:57:38 2016 -0400
+
+    Merge pull request #9690 from dillaman/wip-16290
+    
+    jewel: rbd-mirror: FAILED assert(!m_status_watcher)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 26568819e331ef3adec6667a801f07058d94845b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 20:20:33 2016 -0400
+
+    librbd: track in-flight flush events for journaling
+    
+    The tracking for flush events was accidentally dropped. This
+    can result in the journal being closed too early since it
+    might miss in-flight flush ops.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1ea1edce9332fa1730625629e3b4e16f02caaf4)
+
+commit a85fbb4e9a9a409738e9f031c20a8e2beb1b514f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 12:00:28 2016 -0400
+
+    librbd: do not shut down exclusive lock while acquiring
+    
+    Fixes: http://tracker.ceph.com/issues/16260
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c5694fc6766fb8e213c4b65d2cd7b9d066b07ff7)
+
+commit a38caf90c98a04d57b81519803271f9d85fb5d56
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Jun 12 20:32:45 2016 +0300
+
+    rbd-mirror: FAILED assert(!m_status_watcher)
+    
+    Fixes: http://tracker.ceph.com/issues/16245
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8bfc0f2efb44d3eff7a5fd8bca24516c8815d62)
+
+commit ffb64cd0538cee05b4f360fa8334829da268d1b9
+Merge: 271406c edef63d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:54:51 2016 -0400
+
+    Merge pull request #9654 from dillaman/wip-16246
+    
+    jewel: librbd: recursive lock possible when disabling journaling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 271406c77a6176057ba73b55a110a193d6872a48
+Merge: b5e344e 0579a48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:54:31 2016 -0400
+
+    Merge pull request #9673 from dillaman/wip-16265
+    
+    jewel: rbd-mirror: do not re-use image id from mirror directory if creating image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b5e344e1e3670214da803ff6f7eb6e50bf52e46f
+Merge: 1720664 739f343
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 23:37:06 2016 -0400
+
+    Merge pull request #9674 from dillaman/wip-16267
+    
+    jewel: qa/workunits: ensure replay has started before checking position
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 739f343ec880eacd13b935d493b5dbfcc5efa387
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 00:19:37 2016 -0400
+
+    qa/workunits: ensure replay has started before checking position
+    
+    Fixes: http://tracker.ceph.com/issues/16248
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a68b68e0d2741e6694fd28d31766808ba881815)
+
+commit 0579a483bf7b0f1cc590fb1dbc6c740c17679dac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:09:00 2016 -0400
+
+    rbd-mirror: reset local image id when creating new image
+    
+    Fixes: http://tracker.ceph.com/issues/16253
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1948047acbc129de2982d53e130da1f577af224b)
+
+commit ec809870cc2902d2bdf6543d911e23e71567303c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:07:54 2016 -0400
+
+    rbd-mirror: handle an attempt to delete a non-mirrored image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45dda67b3ef3d5dc5abb327fc1f242140f004d90)
+
+commit 6c303007af20a317f02b5caf7d3cb9f2214d2ee5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:03:29 2016 -0400
+
+    cls_rbd: disallow setting an empty mirror image entry to disabling
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7b2f99a97ff64451f3ae72081186dff41c507271)
+
+commit edef63d56bc329638f79ce8017e72810dcd768f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 21:42:59 2016 -0400
+
+    librbd: avoid holding owner_lock waiting for exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/16235
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b96ab805eed49d4bdb494c8919d2e1ee7ad0a7e6)
+
+commit 1720664ddcb32dd300b2bd046ff15ed4916676a4
+Merge: f902309 2eaaf2e
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Jun 13 01:26:53 2016 -0700
+
+    Merge pull request #9656 from gregsfortytwo/wip-jewel-16024
+    
+    Jewel client: fix bugs accidentally disabling readahead
+
+commit f902309bc9c159c0d6a798c0398a95f64bb682ed
+Merge: 89d6545 d61e3dd
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Jun 13 01:22:22 2016 -0700
+
+    Merge pull request #9655 from gregsfortytwo/wip-jewel-15508
+    
+    Jewel mds: order directories by hash and fix simultaneous readdir races
+
+commit d61e3ddf6d99a4b9b72fd769d635bf9b2bffaf0e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 9 22:12:48 2016 +0800
+
+    client: fix simultaneous readdirs race
+    
+    Current readdir code uses list to track the order of the dentries
+    in readdir replies.  When handling a readdir reply, it pushes the
+    resulting dentries to the back of directory's dentry_list. After
+    readdir finishes, the dentry_list reflects how MDS sorts dentries.
+    
+    This method is racy when there are simultaneous readdirs. The fix
+    is use vector instead of list to trace how dentries are sorted in
+    its parent directory. As long as shared_gen doesn't change, each
+    dentry is at fixed position of the vector. So cocurrent readdirs
+    do not affect each other.
+    
+    Fixes: http://tracker.ceph.com/issues/15508
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9d297c5e98f814b282dadc379ab70dfa678db73e)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit ba9fa11a235dc6c55b3b03e3a98b17316d06cf27
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 23:24:46 2016 +0800
+
+    client: move dir_{release,ordered}_count into class Inode
+    
+    We close Inode::dir when it's empty. Once closing the dir, we lose
+    track of {release,ordered}_count. This causes direcotry to be wrongly
+    marked as complete. (dir is trimmed to empty in the middle of readdir)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 235fcf68f5d96a4d9d6cc260d12da912fa7ea4a8)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit f5db278c8cba873e580255dea6dba5227d508df7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 17:59:35 2016 +0800
+
+    ceph_test_libcephfs: check order of entries in readdir result
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9b17d145c15a469caa29853312b8dcb0b19f55f0)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 51a75063f59fcb47d3a4d5f06aa51cab83b76616
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 15:52:32 2016 +0800
+
+    mds: don't reset readdir offset if client supports hash order dentry
+    
+    Now the ordering of dentries is stable across directory fragmentation.
+    There is no need to reset readdir offset if directory get fragmented
+    in the middle of readdir.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 98a01af2cd8e01f14ab7be32c31a43571ef8bb87)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 3fe5a09f3e7aa9fc89cd47a261785110c2bf38a2
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 15:31:27 2016 +0800
+
+    client: using hash value to compose dentry offset
+    
+    If MDS sorts dentries in dirfrag in hash order, we use hash value to
+    compose dentry offset. dentry offset is:
+    
+      (0xff << 52) | ((24 bits hash) << 28) |
+      (the nth entry hash hash collision)
+    
+    This offset is stable across directory fragmentation.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 680766ec131b95271e320f54dfe6d69ea8d4fbb3)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit a65b3ef30123e5d9d3199cf2f5dafd7d9ab8a755
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 10:50:17 2016 +0800
+
+    client: record 'offset' for each entry of dir_result_t::buffer
+    
+    This is preparation for using hash value as dentry 'offset'
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bd6546e5e8bfb4ef8927807492d63d5285534618)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 8361b988441d073b82809dbafd82fba1c23b13be
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 9 11:51:36 2016 +0800
+
+    client: fix cached readdir after seekdir
+    
+    Client::seekdir doesn't reset dirp->at_cache_name for a forward seek
+    within same frag. So the dentry with name == at_cache_name may not be
+    the one prior to the readdir postion.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e32115bae2f1ac2c59b57852976b0de5587abac)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit cf261257f6ad9b0d51f39af5d26beb15e5759672
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Apr 21 17:31:10 2016 +0800
+
+    mds: define end/complete in readdir reply as single u16 flags
+    
+    so that we can introduce new flags for readdir reply.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 92cfbdf526d1030891da366299b635e589dd5f8e)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 9ce73cdeb6056060b299cf92e4c858d1ab2aef2c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 23 16:03:48 2016 +0800
+
+    mds: sort dentries in CDir in hash order
+    
+    This gives us stable ordering of dentries. (Previously ordering of
+    dentries changes after directory gets fragmented)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f483224a0bc70b9eb5f62cd6cbb8b97df5b65953)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 288160b7e4bec94238d3b24622b4d16808a273e1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 22 09:43:54 2016 +0800
+
+    client: save readdir result into dir_result_t directly
+    
+    Current code saves the readdir result into MedaRequest, then updates
+    dir_result_t according to MetaRequest. I can't see any reason why
+    we need to do this.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit db5d60d1b8d01afd43da09a063f00aa6ab312903)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 75edb5b1c8bcdbc24b4cd6cb811e9c07608f09f5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 22 09:31:00 2016 +0800
+
+    client: don't allocate dir_result_t::buffer dynamically
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit c41ceb9f1425432295a52965de8d4a23b15fe5e5)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 98e36d17d42d7b43720685745348317cab4a847f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Apr 21 20:53:55 2016 +0800
+
+    client: simplify 'offset in frag'
+    
+    don't distinguish leftmost frag from other frags. always use 2 as
+    first entry's offset.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6572c2a24ea2abec4c7eecef92a292c2cc72c762)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 2eaaf2e524c8b95b1b22b6bd7ba153b52746fd01
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jun 10 17:33:04 2016 -0400
+
+    client: move readahead ref get/put to obj constr
+    
+    The readahead context was incrementing the file handle reference count but only
+    decreasing in finish which is not called if readahead is a no-op.
+    
+    This fixes a bug caught in testing where readahead was causing inode to become
+    disconnected:
+    
+        2016-06-10 19:46:48.953018 7f2a4351be80  1 client.4125 dump_inode: DISCONNECTED inode 10000000502 #10000000502 ref 110000000502.head(faked_ino=2307 ref=1 ll_ref=0 cap_refs={1024=0,2048=0,4096=0,8192=0} open={1=0,2=0} mode=100666 size=4194304/0 mtime=2016-06-10 19:29:45.107417 caps=-(0=pAsLsXsFscr) objectset[10000000502 ts 2/4012653 objects 0 dirty_or_tx 0] 0x7f2a24300d00)
+        2016-06-10 19:46:48.953032 7f2a4351be80  2 client.4125 cache still has 0+155 items, waiting (for caps to release?)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b5a0773259ce9f2ef44e8e6925ab11dc1a5e55e6)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 1ec2ef876d0e5cfe595ebb4a98fabf4958e9e937
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu May 26 16:32:58 2016 -0400
+
+    client: use fh min readahead to enable readahead
+    
+    The client was using the configured maximum readahead of 0 which was changed in
+    95ee69989129750fddce6a3b5644238c4b88ed74. This prevented readahead from ever
+    running (without setting a different default from 0).
+    
+    Fixes: http://tracker.ceph.com/issues/16024
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 01c179bcce2dc8a269d78eb1be8198b12bd161f7)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit b0bccc3dc11731466f9809013d6deb09f020693b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri May 27 11:43:31 2016 -0400
+
+    client: update comment
+    
+    Default was changed to unlimited in 11e4680e7394e6e02612441c964e05207b97737b.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit bee02a073f4c551798bc490618a0e8d862893c25)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit e4104689b2698feccd9ea8432f1d57847effbd07
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu May 26 15:08:52 2016 -0400
+
+    client: use layout.get_period() helper
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b331d12a43a4177369a003c3ab9d525da6a89950)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit e5759fa8d016d3f8e2a1ba79edb0bf90ab24565b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 15:44:26 2016 -0400
+
+    librbd: recursive lock possible when disabling journaling
+    
+    If pool-level mirroring is enabled, deleting the journal
+    could cause a deadlock attempting to delete remote peer
+    sync-point snapshots.
+    
+    Fixes: http://tracker.ceph.com/issues/16235
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fb255e6c3cd44c8d24c53e3cd70395a11a712574)
+
+commit 89d65456ed1303b09e9bc054bbb62c0808db4565
+Merge: c526386 e15bfff
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:16:35 2016 -0400
+
+    Merge pull request #9553 from Abhishekvrshny/wip-16038-jewel
+    
+    jewel: pybind: rbd API should default features parameter to None
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c526386b4b087029e6fc00315b7726a2cf9caecb
+Merge: 6f9f830 6af7b22
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:13:16 2016 -0400
+
+    Merge pull request #9552 from Abhishekvrshny/wip-15981-jewel
+    
+    jewel: qa: dynamic_features.sh races with image deletion
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6f9f8307d7cff313bd2f476236591141d3c45c71
+Merge: 7edf393 dc38a26
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:08:39 2016 -0400
+
+    Merge pull request #9629 from dillaman/wip-16231
+    
+    jewel: rbd-mirror: fix deletion propagation edge cases
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7edf393743b2f7811d784e02a42fce13af19e8bc
+Merge: 39f530a bc658ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:08:23 2016 -0400
+
+    Merge pull request #9630 from dillaman/wip-16233
+    
+    jewel: rbd-mirror: do not propagate deletions when pool unavailable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 39f530a224d85cd12294ce125d8548cbe1e3e1c0
+Merge: bd454ff cb50679
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:07:42 2016 -0400
+
+    Merge pull request #9628 from dillaman/wip-16224
+    
+    jewel: rbd-mirror: fix journal shut down ordering
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit bd454ff3a95244d2be2177180814439667db73f1
+Merge: 3fe2f1b 3a2cd16
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:07:10 2016 -0400
+
+    Merge pull request #9627 from dillaman/wip-16217
+    
+    jewel: rbd-mirror: refresh image after creating sync point
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3fe2f1b652d7d6d80f9375174885b002db503f40
+Merge: 2a9fee9 99ff1fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 10 21:48:26 2016 -0400
+
+    Merge pull request #9579 from Abhishekvrshny/wip-16006-jewel
+    
+    jewel: BackoffThrottle spins unnecessarily with very small backoff while the throttle is full
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a9fee94f10cba7675f564edbeac35620e8a6602
+Merge: ac1b354 4a967eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 10 17:34:19 2016 -0400
+
+    Merge pull request #9638 from ceph/wip-16211-jewel
+    
+    Jewel: remove reliance on FLAG_OMAP for reads
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bc658ede5b2e68210852b66585bfa5560964f405
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 08:15:19 2016 -0400
+
+    rbd-mirror: do not propagate deletions when pool unavailable
+    
+    Fixes: http://tracker.ceph.com/issues/16229
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d321634414fa4b90c66474557ec990dfc935ae29)
+
+commit dc38a26e2e48f0a1fabdb51f1819e402e5b214ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 00:12:06 2016 -0400
+
+    rbd-mirror: image deleter properly handles multiple snapshots
+    
+    Fixes: http://tracker.ceph.com/issues/16226
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d72a8cb67fbdc7361f8f9e265c659ddb96dc0f74)
+
+commit 0a74552b5a4c297855f5e04336a9d5cf00441d9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 22:32:33 2016 -0400
+
+    rbd-mirror: propagate deletions even if image replayer is stopped
+    
+    If an image deletion which causes the replayer to stop (due to an error)
+    before the deletion is detected, the deletion should still occur.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a0c526a23783fe543737f6445a2d9438cc73b137)
+
+commit da6d409ab670e45e34c1dcf7de3f775632423d3f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 21:14:27 2016 -0400
+
+    qa/workunits/rbd: improve deletion propagation test case
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit de15b198cd7b5e94016605c8016a5cf38c07c6f6)
+
+commit cb50679bad91fa1a0e360dca0331996b543fdadd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 10:12:04 2016 -0400
+
+    librbd: quickly check for duplicate snap name upon create
+    
+    During journal replay, quickly check for a duplicate snap name
+    before allocating a snapshot sequence from the OSDs and
+    reverting due to the duplicate name.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8374d45c7644666103f88c410a2b1b781f3a0cb8)
+
+commit 43f3bbf130c9914f895bf4834305f951d0228954
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 14:45:16 2016 -0400
+
+    rbd-mirror: stop external replayer before closing remote journaler
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 88bd3002082184c045bb82547929b30da1526cfb)
+
+commit f0bf15e9d38e3de63c959bac0ae90b609dafc514
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 12:42:52 2016 -0400
+
+    librbd: stop journal recorder before starting external replay
+    
+    Fixes: http://tracker.ceph.com/issues/16165
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5aa6eb2782bdb5cde6a050a5f7a4e683800414c0)
+
+commit 3a2cd16e6c35b7b8ba0dd113b9a8cc586c1086b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 12:49:04 2016 -0400
+
+    rbd-mirror: refresh image after creating sync point snapshot
+    
+    Fixes: http://tracker.ceph.com/issues/16196
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c9a3e3be7ddc31992167d55de12bbe6849447b43)
+
+commit 6b77d2268e7b17911f0a6809045d356d97d1621a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 12:03:33 2016 -0400
+
+    rbd-mirror: consistently use dout/derr for debug logging
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03f5e4a917b3c18f8ccdaa5ce676ed9731dd2a8f)
+
+commit ac1b35478eea6bad574fac8641b47c0fcaee20f4
+Merge: 966186f 03466f0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:32:21 2016 -0400
+
+    Merge pull request #9612 from dillaman/wip-16208
+    
+    jewel: rbd-mirror: support bootstrap canceling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 966186f3c20080475d3e692e918cc363cac1ae47
+Merge: 49ed4ff bddf944
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:32:02 2016 -0400
+
+    Merge pull request #9610 from dillaman/wip-16209
+    
+    jewel: librbd: prevent error messages when journal externally disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49ed4ffeb2c17f266559137030ab32182750b321
+Merge: 8e533c8 0014a9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:19:47 2016 -0400
+
+    Merge pull request #9611 from dillaman/wip-16210
+    
+    jewel: librbd: journal IO error results in failed assertion in AioCompletion
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8e533c877e5d7bac84867d418aada1998425ef72
+Merge: a768a1c e424482
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:16:49 2016 -0400
+
+    Merge pull request #9580 from Abhishekvrshny/wip-15996-jewel
+    
+    jewel: librbd: reuse ImageCtx::finisher and SafeTimer for lots of images case
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a768a1cb7b0062963714361ce94f8a834db44bd4
+Merge: ba5d490 bb279f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:51 2016 -0400
+
+    Merge pull request #9609 from dillaman/wip-16216
+    
+    jewel: librbd: refresh image if needed in mirror functions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ba5d490824efa63c63713fee418631cd684293c4
+Merge: 41c5c27 3b8cb63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:32 2016 -0400
+
+    Merge pull request #9608 from dillaman/wip-16189
+    
+    jewel: cls_rbd: mirror image status summary should read full directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 41c5c273b1f52d0284c6b08f15db3edb17cd674c
+Merge: 7d217b9 cec6870
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:08 2016 -0400
+
+    Merge pull request #9556 from Abhishekvrshny/wip-16162-jewel
+    
+    jewel: ceph_test_librbd_fsx crashes during journal replay shut down
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7d217b9cc55778fa73f0ce4a0c2c5994f5f31c87
+Merge: 32f90a7 6c0e202
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:14:49 2016 -0400
+
+    Merge pull request #9554 from Abhishekvrshny/wip-16084-jewel
+    
+    jewel: librbd: validate image metadata configuration overrides
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 32f90a7c98ebd5a272ece71b1af7009be2742cb5
+Merge: 3715d20c 09200d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:13:06 2016 -0400
+
+    Merge pull request #9555 from Abhishekvrshny/wip-16147-jewel
+    
+    jewel: Object Map is showing as invalid, even when Object Map is disabled for that Image.
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3715d20c25c19ec1d1c58d275cef4e7ddb5b7778
+Merge: 7ea13df 73464af
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:12:47 2016 -0400
+
+    Merge pull request #9550 from Abhishekvrshny/wip-15969-jewel
+    
+    jewel: Initial python APIs to support mirroring
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7ea13dfb9d85613d5c4daf92811a786293ffc2be
+Merge: 2c22131 3084cf3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:12:37 2016 -0400
+
+    Merge pull request #9548 from Abhishekvrshny/wip-15963-jewel
+    
+    jewel: Disabling journaling feature results in "Transport endpoint is not connected" error
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2c221318e27fb4c4b68780c0037b9acae5862d6e
+Merge: 4d68ae0 69a9500
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 10 00:55:47 2016 +0200
+
+    Merge pull request #9424 from SUSE/wip-16107-jewel
+    
+    jewel: ceph gid mismatch on upgrade from hammer with pre-existing ceph user (SUSE)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d68ae062b90599c184282b807f7d16cfaf2f6ad
+Merge: 57b4542 b443fdf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 10 00:54:52 2016 +0200
+
+    Merge pull request #8802 from SUSE/wip-15606-jewel
+    
+    jewel: ceph-{mds,mon,osd,radosgw} systemd unit files need wants=time-sync.target
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 57b45426b5efa7ec4e293f9276f1f4299b5bf3ac
+Merge: 3cf6533 555cec9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:44:13 2016 -0700
+
+    Merge pull request #9320 from SUSE/wip-16017-jewel
+    
+    jewel: ceph: cli: exception when pool name has non-ascii characters
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3cf653328c988da5cf6be678cf3f0cb4c80c094d
+Merge: a422d03 663d7c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:43:48 2016 -0700
+
+    Merge pull request #9270 from Abhishekvrshny/wip-15852-jewel
+    
+    jewel: mon/Monitor: memory leak on Monitor::handle_ping()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a422d0372719841ea4cc63f87e77ddc1dee7d815
+Merge: 34b193b 8356021
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:43:04 2016 -0700
+
+    Merge pull request #9237 from SUSE/wip-15961-jewel
+    
+    jewel: ceph-osd valgrind invalid reads/writes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 34b193be043f0d88c42c5ae177a660b747e81486
+Merge: e2a035e 64f15b3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:42:32 2016 -0700
+
+    Merge pull request #9103 from Abhishekvrshny/wip-15797-jewel
+    
+    jewel: deadlock in OSD::_committed_osd_maps
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e2a035eeb19e9043c41e5132d2ea0b8a59ac1bed
+Merge: 2c35fe7 adfbe95
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:41:52 2016 -0700
+
+    Merge pull request #9101 from Abhishekvrshny/wip-15799-jewel
+    
+    jewel: osd boot race with noup being set
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 2c35fe7e739e99452cf4d039281185e8c4a40b1c
+Merge: 6528e43 f4306de
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:40:43 2016 -0700
+
+    Merge pull request #9104 from Abhishekvrshny/wip-15795-jewel
+    
+    jewel: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 6528e4371e0ab1c385e539646d08f4ea1dded893
+Merge: 50143ed b989084
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:37:43 2016 -0700
+
+    Merge pull request #9427 from SUSE/wip-16104-jewel
+    
+    jewel: ceph-disk: workaround gperftool hang
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 50143ed004ef198f2161b9f360bfeccd16503954
+Merge: b328e99 a32820d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 9 14:55:31 2016 -0400
+
+    Merge pull request #9614 from ceph/wip-16113-jewel
+    
+    osd: hobject_t::get_max() vs is_max() discrepancy
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 03466f096adcfe3322cad49c959fec976d887860
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:30:05 2016 +0300
+
+    rbd-mirror: don't return split-brain error if we still in MIRROR_PEER_STATE_SYNCING
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3ac620c5d66d8516e68264fcdb5e97b0e803d0ec)
+
+commit dfefd343e35b1c69fc2c938c7ebc30352e1320d4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 3 12:48:06 2016 +0300
+
+    rbd-mirror: refactor split-brain detection code
+    
+    to make it more readable and extendable.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 039cd2d2e3bb6a9c5042c125d2c67ec9adc02c47)
+
+commit c94e5b740628b98008e5342b8a3b2b888ff7b794
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 31 15:00:30 2016 +0300
+
+    rbd-mirror: support bootstrap canceling
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 21f895f48498e518c3675a9e851559f4d071f6dd)
+
+commit c8a3e759298c3a2adfa67d932178d284619d460e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 30 16:24:24 2016 +0300
+
+    rbd-mirror: return ECANCELED when ImageReplayer start is canceled
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit f581dbcb2eca108ce9440629b8f00f3229c07b95)
+
+commit 0014a9e284c6cfc7678e63b56d707a8ad9b93127
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 01:01:09 2016 -0400
+
+    librbd: flush journal commit positions before starting op
+    
+    Ensure all IO has been properly flushed and committed to the
+    journal before starting an op that could affect the IO path.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 636129653c7cac9665d0c423f6aafad5fd0c480a)
+
+commit 3cec45a17a3d8bdcd19c63c31dad6b9150a9fbfd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 00:11:34 2016 -0400
+
+    librbd: leaked AioCompletion if IO issued after image shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d3d4e068396e01c58832e0ea1224ca34bafb808e)
+
+commit ef12536830082194da52df3a0dc8e22a3f69cb54
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 19:00:47 2016 -0400
+
+    librbd: resize and snap create can hang on journal replay failure
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1032f19b3c9d9c7916669d04af909131574b3cab)
+
+commit c8b4cabea0e1a74a62fd9b25ccb5e0b15df38733
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 14:54:23 2016 -0400
+
+    librbd: missing journal state for flushing/restarting replay
+    
+    Fixes: http://tracker.ceph.com/issues/16077
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a7f5bc181c9cd677d39f95ad78134e0f85c22acb)
+
+commit bc7649be5867fca46c1e86dacb99c1f8b18ad2e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 12 21:22:56 2016 -0400
+
+    librbd: track IO in the order issued
+    
+    Fixes: http://tracker.ceph.com/issues/15034
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eb8dda5296eb5c19aa27e96d6e92eaa0c352cb62)
+
+commit 602ffd78f3de00d6655236dd914aa83006a8f862
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 12 15:15:04 2016 -0400
+
+    librbd: AioCompletion should retrieve CephContext from ImageCtx
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 133b1ffb21c982003c7aa9dafd585062f9b2eb51)
+
+commit 6e20662a9f5b9eabbd83eb91760532d8171d96b8
+Author: Hector Martin <marcan@marcan.st>
+Date:   Wed May 25 20:41:40 2016 +0900
+
+    librbd: only mark async completion as done after callback
+    
+    This change in behavior was introduced in fde9f78.
+    
+    Signed-off-by: Hector Martin <marcan@marcan.st>
+    (cherry picked from commit 77f7c1cf829e5417eb356ebf3bf1fbc8ff7a25a5)
+
+commit de29be73c043c0cd49b4d35a160ddd3910d423e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 18:17:49 2016 -0400
+
+    librbd: Journal no longer requires AioCompletion for IO events
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 82587a31f1d8ba1765f164391796753cf97a1878)
+
+commit 65556aa507e3e70352bb61c1549f340af0c4a9fb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 17:59:09 2016 -0400
+
+    librbd: bubble journal error up via existing AioObjectRequests
+    
+    The journal cannot directly fail the AioCompletion because it
+    might have one-or-more outstanding requests associated with
+    it.
+    
+    Fixes: http://tracker.ceph.com/issues/15791
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79c934742fb15c803780519b36070408599a74c5)
+
+commit bddf944797df5ffb93b587f8ac245846a48bd2e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 22:34:30 2016 -0400
+
+    librbd: potential duplicate snap removal can result in crash
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 276fed6b7078158c2cd04fba14a11531c27898e0)
+
+commit 4ce4130d0e6deef315dcbd5dc899faf5de9ab0e7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 00:43:53 2016 -0400
+
+    ObjectCacher: do not merge uncommitted journal bhs
+    
+    Avoid the possibility of an overwrite callback for regions
+    that were never associated with a journal write.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6060ade16205337de3941a10d05e234b738fa07)
+
+commit 6e645985e77bd0597eab6755b6e4071b79967584
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:54:34 2016 +0800
+
+    ObjectCacher: don't loop the extents of OSDRead in map_read
+    
+    Looping the extents of OSDRead is done in _readx. The looping in
+    map_read is not needed.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit fae912c3856cab6d03183411eecab90c6e6732d2)
+
+commit c5f281d64221c62e3736f9630204f85d67328670
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:49:56 2016 +0800
+
+    ObjectCacher: fix the bytes_written in writex
+    
+    The BufferHead returned after map_write may include other dirty data to
+    the left and/or right. The actual length of data been written is in the
+    ObjectExtent.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit ca6f1681e4baf61682e258c5761536d67bf6bdb8)
+
+commit 1c00d813791cdd8c6e685c1ebf6f93ef4aa0adbe
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:45:45 2016 +0800
+
+    osdc: don't loop the extents of OSDWrite in map_write
+    
+    We are looping all the extents of the OSDWrite in writex. The looping in
+    map_write is not needed. And if there are same object in these
+    extents,they will be mapped multiple times.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit 322f1059670898762a890a01af6d3b0dd3ccf826)
+
+commit b48cdb21da2ba02ae4ee5398c38b48e857dba8cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 00:42:30 2016 -0400
+
+    librbd: notify image feature update before releasing lock
+    
+    If a client is waiting for the lock, ensure it sees the latest
+    version of the image features.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f601ea835bb78554ac767ccb11789df960765bf9)
+
+commit 680efbdcc3d2e28b6de991e7ad5c18e0d71cc551
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 1 23:19:20 2016 -0400
+
+    librbd: refresh image after acquiring exclusive lock
+    
+    It's possible that the object map or journaling features have
+    been disabled while the lock was not owned.
+    
+    Fixes: http://tracker.ceph.com/issues/16114
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03c54f52d15d6283c630bac6f75427e6829f7d0a)
+
+commit bb279f1b288f1069b83d7661588f883716c80d9d
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Wed Jun 1 21:01:30 2016 -0400
+
+    librbd: refresh image if needed in mirror functions
+    
+    Fixes: http://tracker.ceph.com/issues/16096
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit a355c8f6d9e792517a8d0e096a99a64474369cfc)
+
+commit 3b8cb634b34dd7742bbdea2ca354a358cb0fe089
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 09:36:02 2016 -0400
+
+    cls_rbd: mirror image status summary should read full directory
+    
+    Previously only retrieved the status for the first 64 images in
+    the rbd_mirroring directory.
+    
+    Fixes: http://tracker.ceph.com/issues/16178
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61f86e7ec14233ea0dee5b3370595ce050ee28c3)
+
+commit 4a967eb42a7409b9d78d569585af2155414234d2 (refs/remotes/origin/wip-16211-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:44 2016 -0700
+
+    Revert "osd/ReplicatedPG: for copy_get get omap, firstly check ob whether has omap."
+    
+    This reverts commit 771c869651132013054599f3decfc1017092a25f.
+
+commit fd8f8af2456bc197877793a60d4401751f828032
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:39 2016 -0700
+
+    Revert "osd/ReplicatedPG: For omap read ops, it should check object wether has omap"
+    
+    This reverts commit 98caba1d0b4dd334748e3bcaf1bfb3f79f75eb65.
+
+commit d59ca31a4fcc472809b1f1bd3379fc3521c28fb0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:31 2016 -0700
+
+    Revert "osd/ReplicatedPG: When do omapclear, it should check object whether is omap."
+    
+    This reverts commit aeb8141c29cdd337961430bbf3e93a79e3e1bcd3.
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit b328e998b2aa066f5f42577abd0c74a5f3d42694
+Merge: b1c4c4c 188318b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 8 12:00:31 2016 +0200
+
+    Merge pull request #8938 from SUSE/wip-15739-jewel
+    
+    jewel: ceph.spec.in: drop support for RHEL<7 and SUSE<1210 in jewel and above
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit f4306de66d6c071cb240a0d77e8f877841f87033
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:09:43 2016 -0400
+
+    osdc/Objecter: upper bound watch_check result
+    
+    This way we always return a safe upper bound on the amount of time
+    since we did a check.  Among other things, this prevents us from
+    returning a value of 0, which is confusing.
+    
+    Fixes: http://tracker.ceph.com/issues/15760
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e2ce365866a7569b55aff338a82c132aa927089)
+
+commit 64f15b3cfd15faa65e7a36c9779713ea2436cf16
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Wed May 4 09:49:40 2016 +0800
+
+    OSD: fix deadlock in OSD::_committed_osd_maps
+    
+    Fixes: http://tracker.ceph.com/issues/15701
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 4fbf3891aab16347d02782f7c1c0400f9e0896af)
+
+commit adfbe9554be2cb79a6cbccc7c4a556139b4aed50
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 23:28:18 2016 -0400
+
+    osd: handle boot racing with NOUP set
+    
+    This is a follow-on to 7139a232d26beef441ffbc13bc087baab3505ea8,
+    which handled the NOUP set + clear case when the OSD found out
+    about the flag being cleared.  However, it's possible that the
+    flag will get cleared but the OSD won't get a map update (because
+    it hasn't subscribed and is not doing any work).
+    
+    This means that it is *more* likely than before that we will
+    restart the boot process even though the OSD did successfully
+    mark us up.  However, as before, it is unavoidable because there
+    is no notification of whether our boot request succeeds or not.
+    
+    And it is still mostly harmless (an extra mark down + up cycle).
+    
+    Fixes: http://tracker.ceph.com/issues/15678
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11e4242fbdb2f2f6f654d4cb3a7c95d5b38a88c2)
+
+commit e424482d0e79ad0a0c44b14f0977cfa69d1b7f72
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Feb 28 23:35:17 2016 +0800
+
+    TaskFinisher: use singleton SafeTimer and Finisher
+    
+    Fix #13938
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    
+    (cherry picked from commit dc6d6ce8b29de51b69242e569517f69acfe226f4)
+
+commit 99ff1fc34215da3339ccd227c4f17caf8294a32f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 18 12:09:10 2016 -0700
+
+    BackoffThrottle: wait() if past max
+    
+    Otherwise, we risk spinning on wait_for() with a small
+    delay.
+    
+    Fixes: http://tracker.ceph.com/issues/15953
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6f835726a7df5b29a96849fc1362186fd2116d9b)
+
+commit 5ce43eb7682535deeee149600f1226b87e41defd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 18 11:49:19 2016 -0700
+
+    BackoffThrottle: use wait_for instead of wait_until
+    
+    On some platforms, wait_until won't surrender the lock
+    with a negative argument.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 846808ff3f3837b1772f293ae71dba5eda609cc4)
+
+commit cec6870572621ed9abbe3791ba6e1a460333dfc0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 10:41:18 2016 -0400
+
+    test/rbd: fsx needs to shut down the journal
+    
+    Fixes: http://tracker.ceph.com/issues/16123
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d10ffaafbd2a5831154757b848ed8504dad99069)
+
+commit 09200d4b5002f8ef9e75a9218fe85aaa5da33a8b
+Author: xinxin shu <shuxinxin@chinac.com>
+Date:   Thu Jun 2 14:13:09 2016 +0800
+
+    remove invalid objectmap flag when objectmap is disabled
+    Fixes: http://tracker.ceph.com/issues/16076
+    Signed-off-by: xinxin shu <shuxinxin@chinac.com>
+    
+    (cherry picked from commit b2d475686ee7617bb2023d753941e3d6952f0878)
+
+commit 6c0e202573110de522d531e0aa8978e843b5167e
+Author: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+Date:   Sat Oct 7 16:52:07 2017 +0800
+
+    rbd: check value of config override when setting image-meta
+    
+    Fixes: http://tracker.ceph.com/issues/15522
+    
+    Signed-off-by: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+    (cherry picked from commit 4538f8152d0e73bdefc09874113c87467bdda622)
+
+commit e15bfff2e822e15e0a34d09825e04cee381d8737
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 23 09:18:34 2016 +0300
+
+    pybind/rbd: create/clone/copy: default to None for features param
+    
+    If None is specified don't set the features so that the defaults is
+    used.
+    
+    Fixes: http://tracker.ceph.com/issues/15982
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8b3a9757f7d517210a91f2893b0961b6b35afab)
+
+commit f9e32ac68a83932945165612dc64c307f0411b19
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 23 09:16:38 2016 +0300
+
+    librbd: clone: default to parent features
+    
+    Fixes: http://tracker.ceph.com/issues/15982
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit d305eaad8d57a6f64e6d80eebcd0839a65c9a668)
+
+commit 6af7b223f3d0a8d51cd472cb7024a5ec70d30f61
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 09:15:53 2016 +0300
+
+    qa: dynamic_features.sh: return error only if it failed on alive QEMU
+    
+    Fixes: #15500
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8a71a79d24f2e1f999a5fc0a4154a10401d5a261)
+
+commit 73464af844b45137049b0fff294449ed3a6f13bb
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:33:00 2016 +0300
+
+    test: initial python APIs to support mirroring
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit bfad0ca33b9d65dcec65bc1a18596d35fb3098f3)
+
+commit 5c312669ba2d208acaa8e819812e50ce23418208
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:29:17 2016 +0300
+
+    rbd: initial python APIs to support mirroring
+    
+    Fixes: http://tracker.ceph.com/issues/15656
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ef0ea8ee3c926a31b54e410c18e887415f6ea3cf)
+
+commit 3084cf32463a07cecec66befa1562a187bc1734f
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Mon May 16 16:18:59 2016 +0800
+
+    rbd: close journal before removing
+    
+    Otherwise there is some unwanted error message.
+    
+    Fixes: http://tracker.ceph.com/issues/15863
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit ecdad4acca635d4b9f0b407889dde9985ab3a506)
+
+commit a32820d15fd5d2f817bc5fc94b731f4e7adafbf3 (refs/remotes/origin/wip-16113-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:13:09 2016 -0700
+
+    src/: remove all direct comparisons to get_max()
+    
+    get_max() now returns a special singleton type from which hobject_t's
+    can be assigned and constructed, but which cannot be directly compared.
+    
+    This patch also cleans up all such uses to use is_max() instead.
+    
+    This should prevent some issues like 16113 by preventing us from
+    checking for max-ness by comparing against a sentinel value.  The more
+    complete fix will be to make all fields of hobject_t private and enforce
+    a canonical max() representation that way.  That patch will be hard to
+    backport, however, so we'll settle for this for now.
+    
+    Fixes: http://tracker.ceph.com/issues/16113
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7c5f2acaa57bd6caaf4b13b48154df3ad6fbe84d)
+
+commit f869594332fb8ffa4d29aae22573afd094c39f54
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:39:09 2016 -0700
+
+    PG::replica_scrub: don't adjust pool on max object
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8b7b7c7a2379ce77173bbdfd90c98c561d786155)
+
+commit 1737ff3f49080904ef07a7519d200ee608fab446
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:36:21 2016 -0700
+
+    hobject: compensate for non-canonical hobject_t::get_max() encodings
+    
+    This closes a loop-hole that could allow a non-canonical in memory
+    hobject_t::get_max() object which would return true for is_max(), but
+    false for *this == hobject_t::get_max().
+    
+    Fixes: http://tracker.ceph.com/issues/16113
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 42fad4b76def50a0984bf3de06f78ed434d56954)
+
+commit b1c4c4c456d1d20ccc91de4f762dbe46d8e339a3
+Merge: 24404c9 5d9ee88
+Author: John Spray <jspray@redhat.com>
+Date:   Fri Jun 3 12:37:00 2016 +0100
+
+    Merge pull request #9430 from ceph/wip-manila-backports
+    
+    jewel: backports needed for Manila
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 24404c928b878d28a34439b8adaf43854547d317
+Merge: 1aa81ce 208659f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 12:03:12 2016 -0400
+
+    Merge pull request #9423 from dillaman/wip-16101
+    
+    jewel: rbd-mirror: replicate cloned images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5d9ee88e2a0ee57455581350bafe613fef8e3da2
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Apr 26 17:51:48 2016 -0500
+
+    pybind: configurable cephfs_vol_client prefix and ns_prefix.
+    
+    Which is useful if we would like to have multiple backends
+    in manila and potentially set different layout based on prefix.
+    
+    And also make ns_prefix configurable also in order to support
+    multiple backend.
+    
+    Fixes #15417
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit d1a21b07ede77ab3f2be7a753426d76de0343086)
+
+commit 470605c38f772e5bcc466e71ac232294b41be276
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Apr 13 14:03:51 2016 +0530
+
+    ceph_volume_client: evict client also based on mount path
+    
+    Evict clients based on not just their auth ID, but also based on the
+    volume path mounted. This is needed for the Manila use-case, where
+    the clients using an auth ID are denied further access to a share.
+    
+    Fixes: http://tracker.ceph.com/issues/15855
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 68714b9dda8ed6010cad6730e0ddf54b3219150d)
+    (cherry picked from commit aaa0e9aea9fcf2f63e73e544bd40c10a0d694408)
+
+commit 726292eb305d58661993803dea613e9d98a64f4f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed May 11 13:18:23 2016 +0100
+
+    client: report root's quota in statfs
+    
+    When user is mounted a quota-restricted inode
+    as the root, report that inode's quota status
+    as the filesystem statistics in statfs.
+    
+    This allows us to have a fairly convincing illusion
+    that someone has a filesystem to themselves, when
+    they're really mounting a restricted part of
+    the larger global filesystem.
+    
+    Fixes: http://tracker.ceph.com/issues/15599
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit b6d2b6d1a51969c210ae75fef93c71ac21f511a6)
+
+commit 46c2bd0864f8f7ace80a8cad07b873d9aa57ead9
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Apr 1 14:27:31 2016 +0100
+
+    pybind: fix unicode handling in CephFSVolumeClient::purge
+    
+    os.path.join is sensitive to string encoding, but
+    just doing a straight substitution should not be.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e590f4d05fdb46747e83e35e66a26d9f4aa0314d)
+
+commit b989084dfc28659445fe08847537f8e5ad275cdd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 26 12:55:51 2016 +0200
+
+    ceph-disk: workaround gperftool hang
+    
+    Temporary workaround: if ceph-osd --mkfs does not
+    complete within 5 minutes, assume it is blocked
+    because of https://github.com/gperftools/gperftools/issues/786
+    
+    References http://tracker.ceph.com/issues/13522
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c092321c24b9ca493d90a9ccc8f0b4b9a38677f1)
+
+commit 69a9500d900b2d5b9bdb0e989f83fe60ddddeda0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu May 12 19:23:31 2016 +0200
+
+    rpm: unconditionally set ceph user's primary group to ceph (SUSE)
+    
+    This commit brings the user/group creation into greater semantic alignment
+    with the Debian packaging.
+    
+    Fixes: http://tracker.ceph.com/issues/15869
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b3dfa8d90962e60b6841555db830c8ea635f4cce)
+
+commit 208659fdebc888cab35239ec848f27320477420f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 12:49:45 2016 -0400
+
+    qa/workunits/rbd: basic cloned image test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 943ff829874f96f720d6c6299e1212e4da24c6c5)
+
+commit af6be1bc15dfc2757cf74772078dd8516ed889be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 08:22:16 2016 -0400
+
+    rbd-mirror: copy snapshot parent image settings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f5fd09373714ebe8886bcf51bb97732038fffe55)
+
+commit 8405c2c377c7f9e755a349bfc90494f7ee7cc477
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 12:01:15 2016 -0400
+
+    cls_rbd: asynchronous helper method to set parent
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fcfdebb471650a1730306f6a309f1775ba7a5eee)
+
+commit 26f4edcbb3b24eed98f111a2c77a5380cbf16927
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:38:06 2016 -0400
+
+    rbd-mirror: add support for cloning images from mirrored parent
+    
+    Fixes: http://tracker.ceph.com/issues/14937
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ae6e03dfb1fa3be61e6deac4fe39982e7e7c18aa)
+
+commit b8b01c57949955b1ff6a6a35dcae05108a2dda58
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:36:06 2016 -0400
+
+    librbd: extend internal API to clone from open parent image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfaa112c30da7091fd2d539b80a4a211176a6031)
+
+commit 9081c58a31aa8fa6f966f3b682b9870e2b6f8297
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:13:59 2016 -0400
+
+    cls_rbd: asynchronous helper methods to retrieve mirror image metadata
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a26080447cfa4b7bfdacdc13242569258a64076)
+
+commit 2c2f4b251d0c8b870728293d06e9c853856c8092
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 15:16:28 2016 -0400
+
+    rbd-mirror: helper state machine to open remote/read-only images
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3731c3c34c15f8dcfbab444bdbf3c05976b03431)
+
+commit 1aa81ce5ba492e8eeea2b20f08b77a18a94cd738
+Merge: 54bfefb 0bcc295
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 13:11:22 2016 -0400
+
+    Merge pull request #9409 from dillaman/wip-16068
+    
+    jewel: rbd-mirror: support multiple replicated pools
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0bcc295363d9ca20d43fd313084370158513bc62
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 27 11:40:58 2016 -0400
+
+    rbd-mirror: cluster-level asok commands need to support multiple pools
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4339ee8cb64d5f1a164e017fea942600ffb0ea9)
+
+commit e4547c036ef641d48ab03808f59e035b56136745
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 23:26:20 2016 -0400
+
+    rbd-mirror: replayer should only handle a single pool
+    
+    The pool watcher now only needs to scan a single pool for
+    its associated replayer since a peer is inherently tied to
+    a single pool.
+    
+    Fixes: http://tracker.ceph.com/issues/16045
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c5a8b780edeb9b62a601f0718459837f3d66e63d)
+
+commit 6fc5328e93b7d005479f925cca1268e7e7e10234
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 16:03:59 2016 -0400
+
+    rbd-mirror: group peers by local pools
+    
+    The peer structure is automatically unique per pool due to its
+    UUID, so grouping local pools by a single peer doesn't work.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 020f0282a0771c0d321732a62f53cee10a193715)
+    
+     Conflicts:
+    	src/tools/rbd_mirror/ClusterWatcher.cc: trivial resolution
+
+commit 0745cc3db35a349799891c8397249bf8d088ef83
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 14:00:34 2016 -0400
+
+    rbd-mirror: stop stale replayers before starting new replayers
+    
+    If the connection details are tweaked for a remote peer, stop
+    the existing replayer before potentially starting a new replayer
+    against the same remote.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a2adca2a47b384f64c20996b9fc77010a70bc22)
+
+commit 91f8373613a38d48d71f2762e75e21ec817305be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 13:29:49 2016 -0400
+
+    rbd-mirror: normalize debug log message prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d52843190d84d533ab50a1702cf4a829b71ad68b)
+
+commit 5f0d89a9258fdf2f7392c63ca5d6488c4e3f210d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 13:26:57 2016 -0400
+
+    qa/workunits/rbd: create secondary replicated pool
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 839544ee6446b959f01290667592e44fc5ee6991)
+
+commit 54bfefb7d108667058eda56fda24b0029a560746
+Merge: 522c7f9 61542c1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:51:05 2016 -0400
+
+    Merge pull request #9373 from dillaman/wip-16055
+    
+    jewel: journal: support asynchronous shutdown
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 522c7f9432e029f4db2b7455fb83b4aeb3ff4a4e
+Merge: f04972c 6375c78
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:49:39 2016 -0400
+
+    Merge pull request #9372 from dillaman/wip-15995
+    
+    jewel: rbd-mirror: Delete local image mirror when remote image mirroring is disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 61542c14d34c4192f1468e2d7865fe913cce21b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 02:31:11 2016 -0400
+
+    rbd-mirror: image replay now uses asynchronous journal shutdown
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1b0a1b1b9a237ae363a52c7b4038b19e87052a5)
+
+commit bf4c45830330b025bf818286e13693b837f1f2c5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 00:21:14 2016 -0400
+
+    librbd: integrate with async journaler shutdown API
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ad297850b1be8ed38f77b86913c6821748f3368b)
+
+commit 36cf42a3f47c23db82a8aca7a5560857cc3e0027
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 16:12:16 2016 -0400
+
+    journal: extend asynchronous shutdown to facade
+    
+    Fixes: http://tracker.ceph.com/issues/14530
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f40c595255a6828e27abfe63b4209ebae03e237b)
+
+commit 1b0422d7272788561253479a43f5b3dcdbb6282b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 15:53:12 2016 -0400
+
+    journal: trimmer now has asynchronous shutdown
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 59ca9916ba2e2b144eff5ee471a314b9c4391936)
+
+commit f913946ff74924fc14c7336fef6c73a4c6edfdfa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 15:36:17 2016 -0400
+
+    journal: metadata init and shutdown now asynchronous
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0543e1b29eb253bf1d9e9af7af6f22fa394dd854)
+
+commit 1a9d60f8244d6ba8697b2254d967907fdb618cdb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 12:06:26 2016 -0400
+
+    journal: player shutdown is now handled asynchronously
+    
+    Fixes: http://tracker.ceph.com/issues/15949
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79b41346678b3acdeb547fe07e44d8d0543d37a7)
+
+commit 4af5bb1d2d86de68be0cef3713e09dfe0f1a9030
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 22:26:05 2016 -0400
+
+    test: use randomized write sizes for rbd-mirror stress test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b9edff2174ff04bb8b6353dd2d2218c64757aae1)
+
+commit df2b8d9f0f399dcf9bb0e3eba3a39e14de504c49
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 22:21:33 2016 -0400
+
+    journal: eliminate watch delay for object refetches
+    
+    The randomized write sizes of the modified rbd-mirror stress
+    test results in a lot of journal object with few entries.
+    Immediately fetch objects when performing a refetch check prior
+    to closing an empty object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d7b511b854c8ab633dcdb1c935014218b83f4e09)
+
+commit c65614f155ceccebd1960d6042d55e13b31e62ba
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 14:57:03 2016 -0400
+
+    journal: keep active tag to assist with pruning watched objects
+    
+    It's possible that there might be additional entries to prune in
+    objects that haven't been prefetched yet. Keep the active tag
+    to allow these entries to be pruned after they have been loaded.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0fb730e6be632e25d76695e1f37eba542162d620)
+
+commit 30c7f4dbe0a7c515d14be98f3f59ba8a8cdb1dcf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 12:15:49 2016 -0400
+
+    journal: update commit entry object number upon overflow
+    
+    Otherwise the recorded object positions might point to an older
+    object that doesn't contain the actual entry.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5fbf5f82de8214c5c676d38feae7682a006fcab1)
+
+commit fa08330dadb9ce973db866ad641a79d5d2dd290c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 11:01:05 2016 -0400
+
+    journal: cleanup watch refetch flag handling
+    
+    Clear the refetch required flag while scheduling the watch
+    and remove the stale object after the watch completes if still
+    empty. Previously, it was possible for the flag to become
+    out-of-sync with whether or not it was actually refreshed
+    and pruned.
+    
+    Fixes: http://tracker.ceph.com/issues/15993
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ff2cc27ae592646b495bf1b614d35bd50c091a3d)
+
+commit 6375c7831de551c52f1dbf74bf9f65be3b6d8adf
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Fri May 13 16:47:25 2016 +0100
+
+    rbd-mirror: test: added image-deleter test case when mirroring is disabled
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit d85e312ead1da7c35d5ee4fe4be3d6e69a68b572)
+
+commit 1ba650539c1e90b82727a6bd9f1c16796b92b821
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Fri May 13 16:44:53 2016 +0100
+
+    rbd-mirror: Unregister clients from non-primary images journal
+    
+    A non-primary image may have registered clients on its journal
+    (for instance a primary image that was later demoted). We must
+    unregister the clients when disabling image mirroring with the
+    force option.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 9cb0b180bc8a12ea2ebb0d7e32279b696ddd127e)
+
+commit a53cf28ab95a704b78cfa46cf758c2eec8058619
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Apr 19 12:35:49 2016 +0100
+
+    rbd-mirror: test: Added unit test for testing image-deleter thread
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit ac9514941b0accb6c738ce4aa0953c36cdcf69a6)
+
+commit 78b13a504c32685090bb5b6512be9c050df2d8b6
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Apr 18 12:22:19 2016 +0100
+
+    rbd-mirror: test: Fixed test_PoolWatcher to support the new changes in PoolWatcher class
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit e3677f115e1f369a49227fba51a1d34505d487ac)
+
+commit fef7456f8bc85f32cbdb28ce81198a9eb7d56d3b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Apr 5 09:47:32 2016 +0100
+
+    rbd-mirror: Replayer: bootstrap existing mirrored images
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 3c2615aa6da837bfef2365f701f56a34c49b536f)
+
+commit 97eb85f971a74678e4075b16cb5a8e836a985b56
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Apr 28 11:49:43 2016 +0100
+
+    rbd-mirror: replayer: Added image-deleter thread to replayer
+    
+    Fixes: http://tracker.ceph.com/issues/14421
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit a6d698829d925894e4afcdec0ee42b1fc1205db0)
+
+commit 24406c5f8ea12d2c19573116ea84c90ba7446f8b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Apr 28 11:48:05 2016 +0100
+
+    rbd-mirror: Added image-deleter thread to rbd-mirror
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 011984d40bce448fe0febfc6a9fcf57d2c7b4768)
+
+commit 0780230b4af7e4ca923dab5f77735d0a11409202
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Apr 13 15:17:40 2016 +0100
+
+    rbd-mirror: Added implementation of image-deleter thread
+    
+    image-deleter thread will handle the deletion of local images that have
+    been disabled for mirroring by the primary site.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 6cc6e663ed2ec336a9475d93ab2cb605644e6ee0)
+
+commit 38514adc847a227dbd28318650931a8c2a7df7db
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed May 11 20:42:37 2016 +0800
+
+    rbd-mirror: librbd::mirror_peer_list never returns -ENOENT
+    
+    no need to check -ENOENT specially
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 81ff42253369a9f57118bd26b29115c86a1748c5)
+
+commit 4e7233ad43d368a5f6e812870cec77545e034186
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed May 11 20:52:38 2016 +0800
+
+    rbd-mirror: fix typo
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 25bfd4c786d0701fe6ae06e3d120098bd82f8aa0)
+
+commit c3b1bf1e530a4cf55585886a4b4038b9d2862471
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 18 10:07:55 2016 +0300
+
+    rbd-mirror: calculate behind_master only if mirror tag is not newer than master
+    
+    Fixes: http://tracker.ceph.com/issues/15916
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cbd8d526f94523c9de3c575d426063e63d7e1802)
+
+commit f04972c8be28fc6d3133363b5474f22ebac5fd15
+Merge: c194c5d cb950fc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:48:06 2016 -0400
+
+    Merge pull request #9377 from dillaman/wip-16063
+    
+    jewel: test: failure in journal.sh workunit test
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cb950fca12395a1c62421d4b0bce64b4a52b2fd4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 25 21:54:16 2016 +0300
+
+    test: workaround failure in journal.sh
+    
+    With the changes to ensure that the commit position of a new
+    client is initialized to the minimum position of other clients,
+    the 'journal inspect/export' commands return zero records because
+    the master client has committed all of its entries.
+    
+    Workaround this by restoring the initial commit position after
+    writing to the image.
+    
+    Fixes: http://tracker.ceph.com/issues/16011
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e9ed8ac33b1884c6ed5cdfb23023094ec790b116)
+
+commit f92c2a5604d4a8e83504b7283114d3d59bc45f38
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 25 14:51:43 2016 +0300
+
+    cls::journal: treat empty commit position as minimal
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 16846e8085b0c594f065322a768e6dfcbd5d2444)
+
+commit c194c5d8d454d0b387e191d03fec9d916271ff7f
+Merge: 4d3b668 8fbb555
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:46:30 2016 -0400
+
+    Merge pull request #9376 from dillaman/wip-15994
+    
+    jewel: cls_journal: initialize empty commit position upon client register
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d3b66842d4477ec4c4a418a91723fc9b800101c
+Merge: dafe375 e5ebb51
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:45:43 2016 -0400
+
+    jewel: librbd: write-after-write might result in an inconsistent replicated image
+    
+    jewel: librbd: write-after-write might result in an inconsistent replicated image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dafe375640967e10d1a94b02970d399c51980076
+Merge: 409fc64 305ebbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:30:47 2016 -0400
+
+    Merge pull request #9318 from Abhishekvrshny/wip-15957-jewel
+    
+    jewel: Metadata config overrides are applied synchronously
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 409fc64799aec36e183e6ba2bb25c3bf7e3d603b
+Merge: cd6ed4d 0c99028
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:29:45 2016 -0400
+
+    Merge pull request #9317 from Abhishekvrshny/wip-15956-jewel
+    
+    jewel: rbd-mirror should disable the rbd cache for local images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cd6ed4deb0f4b66a10ea7bd5e027daee3a93d1f4
+Merge: 9fa8596 be9e85d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 18:48:58 2016 +0800
+
+    Merge pull request #9403 from dachary/wip-16088-jewel
+    
+    jewel: tests: rm -fr /tmp/*virtualenv*
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit be9e85d83f3d025cec16b7da85d8a2ce1f64e8fa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri May 20 13:19:07 2016 +0200
+
+    tests: rm -fr /tmp/*virtualenv*
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8f0eefba00d1c6ad656302a608217fc3395d0401)
+
+commit 9fa85963a8accd669b0f0fbb17f4b721290432c6
+Merge: a046d2a 3674341
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 13:46:31 2016 +0800
+
+    Merge pull request #8968 from Abhishekvrshny/wip-15728-jewel
+    
+    jewel: osd: acting_primary not updated on split
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a046d2a8c37182652d5f300f58673f70c8485114
+Merge: 954af78 8bbb5ad
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 13:42:28 2016 +0800
+
+    Merge pull request #9100 from Abhishekvrshny/wip-15856-jewel
+    
+    jewel: Hammer (0.94.3) OSD does not delete old OSD Maps in a timely fashion (maybe at all?)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 954af787526a77b923fe85ed1282ba98277738e4
+Merge: e8e572c fae360f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 30 13:53:45 2016 +0300
+
+    Merge pull request #9245 from dreamhost/wip-15974-jewel
+    
+    rgw: fix manager selection when APIs customized
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e8e572c6454816119448240fa7813be7995fd250
+Merge: c11db6f ffd545b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon May 30 11:15:18 2016 +0200
+
+    Merge pull request #9239 from yehudasa/wip-15886-jewel
+    
+    rgw: keep track of written_objs correctly
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8fbb555befb1415ef5218c8aaa149ca44017d73d
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Thu May 19 22:45:34 2016 +0530
+
+    cls_journal: Select min commit position for new clients
+    
+    When a new client registers, its commit position is initialized
+    to the least commit position of all other clients.
+    
+    Fixes: http://tracker.ceph.com/issues/15757
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit dd2e3dae8adb3778e7e381db5dbfca0948571c55)
+
+commit 576ff0ca804779d1eb0b1c44d6ce38e0658b6a05
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Mon May 9 15:15:31 2016 +0800
+
+    cls_journal: remove duplicated key generation
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 715e99c83e1ebc01ceb9eff0f66da8e2b598766a)
+
+commit fae360f0e419266b80f001a25a6864668e772d08
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri May 20 16:00:33 2016 -0700
+
+    rgw: fix manager selection when APIs customized
+    
+    When modifying rgw_enable_apis per RGW instance, such as for staticsites, you
+    can end up with RESTManager instance being null in some cases, which returns a
+    HTTP 405 MethodNotAllowed to all requests.
+    
+    Example configuration to trigger the bug:
+    rgw_enable_apis = s3website
+    
+    Backport: jewel
+    X-Note: Patch from Yehuda in private IRC discussion, 2016/05/20.
+    Fixes: http://tracker.ceph.com/issues/15973
+    Fixes: http://tracker.ceph.com/issues/15974
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7c7a465b55f7100eab0f140bf54f9420abd1c776)
+
+commit 8bbb5add6208a552979270baf4623430cf45e1b3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 13 11:26:31 2016 +0800
+
+    osd/OpRequest: reset connection upon unregister
+    
+    this helps to free the resources referenced by the connection, among
+    other things, in the case of MOSDOp, the OSD::Session and OSDMap. this
+    helps to free the resource earlier and trim the osdmaps in time.
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 87850e96ea6fa45a8368bacabee50f9e95b40ae9)
+
+commit e97cc2d1e9bcee0a43f54ed592d3078182571874
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 12 20:28:11 2016 +0800
+
+    osd: reset session->osdmap if session is not waiting for a map anymore
+    
+    we should release the osdmap reference once we are done with it,
+    otherwise we might need to wait very long to update that reference with
+    a newer osdmap ref. this appears to be an OSDMap leak: it is held by an
+    quiet OSD::Session forever.
+    
+    the osdmap is not reset in OSD::session_notify_pg_create(), because its
+    only caller is wake_pg_waiters(), which will call
+    dispatch_session_waiting() later. and dispatch_session_waiting() will
+    check the session->osdmap, and will also reset the osdmap if
+    session->waiting_for_pg.empty().
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82b0af7cedc3071cd83ee53479f834c23c62b7d0)
+
+commit 555cec9a92568666880d70b9ae8f6e9748eca204
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue May 17 18:04:28 2016 +0100
+
+    ceph.in: fix exception when pool name has non-ascii characters
+    
+    When deleting a pool without the --i-really-really-mean-it option, if
+    the pool name has non-ascii characters, the format of the command
+    message raises a UnicodeEncodeError exception.
+    
+    Fixes: http://tracker.ceph.com/issues/15913
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 805873fed0135616a4cf8b499d1ba0b10dac004c)
+
+commit 305ebbc36473cf8bfd22513ffe55d3a4a99d5ef1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 19:19:24 2016 -0400
+
+    librbd: metadata retrieval added to open image state machine
+    
+    Fixes: http://tracker.ceph.com/issues/15928
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b64cb31f8e7796175b4709929c017b3236649462)
+
+commit 5c9ecea87d3eb90201a967d8da827018cd728f0d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 17:50:07 2016 -0400
+
+    cls_rbd: async version of metadata_list helper method
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 985cb38211c51c95d84479df231c4f53847cb2ec)
+
+commit 0c99028582326c4ac142a03f168ab1bcf0ddd3c2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 20:53:26 2016 -0400
+
+    rbd-mirror: disable librbd caching for replicated images
+    
+    Each image has its own cache and each cache uses its own thread. With
+    a large replicated cluster, this could result in thousands of extra
+    threads and gigabytes of extra memory.
+    
+    Fixes: http://tracker.ceph.com/issues/15930
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ea35f148257282fe3f3ae02fe7a26cf245cda952)
+
+commit 188318b7b567f206d60dfb754fedbb0bfc74ef91
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon May 23 17:06:19 2016 +0200
+
+    rpm: Restore /var/run/ceph context manually
+    
+    The fixfiles command won't fix label for /var/run/ceph (/run is probably
+    excluded from relabel), we need to restore the context manually.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6de39ce73737f79d4511dfc4e1d10d7f3733ab1b)
+
+commit ab1f65dd0429647c30fdae1dd97ff6f393dbc1a8
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 20 08:09:42 2016 +0200
+
+    rpm: Mute semodule in postun script
+    
+    Currently, we don't mute the semodule output in postun script. This
+    results in the following message when removing ceph-selinux package:
+    
+    libsemanage.semanage_direct_remove_key: Removing last ceph module (no
+    other ceph module exists at another priority).
+    
+    The fix is to simply mute the output of the command.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 3c104b106540fabdba43e0d04a3828d77f83e89d)
+
+commit 587fb3df704973071114a30bc55288768dbc2720
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 27 11:13:06 2016 +0200
+
+    install-deps.sh: systemd-rpm-macros must be installed on SUSE
+    
+    Otherwise the script breaks.
+    
+    Fixes: http://tracker.ceph.com/issues/15627
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a7f5ad5f6f9c01ef81f90a7071b6e2243b6da4e7)
+
+commit 69470d406d2b539e8021afd4def7de931ac91857
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue May 10 18:20:13 2016 +0200
+
+    rpm: Drop SELinux priority setting
+    
+    Older versions of semodule binary that are in Centos/RHEL do not
+    support priority setting, dropping it.
+    
+    Fixes: #15822
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7e606f29f499eac83ecb7053ea6874fcfd092924)
+
+commit 128dbc8af0095622d48b962ddd9476aa4140bc16
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed May 4 09:09:47 2016 +0200
+
+    rpm: Fix SELinux relabel on fedora
+    
+    The SELinux userspace utilities stopped providing versions when they
+    switched to CIL language. We need to use a different technique to
+    relabel the files.
+    
+    Fixes: #15725
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit bd26641d705a34a1148a781eea08d203e81b750d)
+
+commit 8182b9a4d2750bb8f0f035d84e9d8476abc62c3c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 12 15:33:45 2016 +0200
+
+    Test-enable XIO messenger and IB/RDMA support
+    
+    Fixes: http://tracker.ceph.com/issues/15472
+    
+    Signed-off-by: Lars Marowsky-Bree <lmb@suse.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d4b74c56d6cbb2c6852bf66f2f34b8a877d7a9ef)
+
+commit 50626cec7a55637764ba379d6cb71e0b485290a8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 28 19:29:00 2016 +0200
+
+    rpm: do a full make check when --with-tests is given
+    
+    The check-local target tests the CLI tools only.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e1514809ae3d2df08a0c47882f9b3bd0ed68d200)
+
+commit fb6858c283d96de15c70fbe8ffc33b5dada82739
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 26 17:03:28 2016 +0200
+
+    rpm: drop sysvinit bits from relabel_files function
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 20df58799a4853ed3798501176988ffaf888936b)
+
+commit db4f2d677df5c9fc00592fc9e003bccb44af6da6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 18:27:45 2015 +0200
+
+    rpm: replace literal paths with RPM macros
+    
+    The only place we should write out literal paths is in the RPM scriptlets,
+    and there only for things that are not installed by this package.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e0346b36fc0e6e17c80c047b13997448ccf1c76e)
+
+commit 3abf89517b0b2142698610debbaa3c6b535424f3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 18:05:12 2015 +0200
+
+    rpm: drop udev/95-ceph-osd-alt.rules
+    
+    This udev rules file was needed on older RHEL platforms, which are
+    unsupported as of jewel.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ee4acb31593dd33da99138998fd0f30ebc2f1a6e)
+
+commit a7e01dd4ff08607fc6d99e41189e3db1a3c1d4da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 17:20:46 2015 +0200
+
+    rpm: global replace $RPM_BUILD_ROOT with %{buildroot}
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bed92b6a86cdef784e89c38952fa779ce5fc5a91)
+
+commit d81cec99421e544e2cf7d212680b04d51ef675f3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:37:47 2015 +0200
+
+    rpm: put dependencies in alphabetical order
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 66042c03df012754215ed848e475f2195ee988a0)
+
+commit e8338a5844aea6dad8e61f00dbd94ca4e04df7be
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:32:46 2015 +0200
+
+    rpm: drop sysvinit-specific dependencies
+    
+    Obsolete in jewel and above.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c655cb896ad10e001b95848f4a4b64dd1ef6984f)
+
+commit b18d8ac0d65f91929fc956980d3b5977bb2970e6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:32:05 2015 +0200
+
+    rpm: move boost-random dependency to appropriate section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 46b16c19f73d82cc357b16e6201926a3479046f2)
+
+commit 94efaef9c28347baebc374e27d3e81f143e642e0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 19:57:49 2016 +0200
+
+    rpm: move unified python-sphinx build dependency
+    
+    Now that the python-sphinx build dependency is unified, move it
+    to the proper section of the spec file.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1d897da861c6134778136fa2b8b33e36c08faff9)
+
+commit 54dc8bf5703e2fb8962a97f7737a6114132b9b20
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 19:41:06 2016 +0200
+
+    rpm: drop init-ceph.in-fedora.patch
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e8cf501d0f22c757a0860a0c91dcb1d7a472ed34)
+
+commit 641e5bb0808a14d2b59cba141bad4acf5bbfb63a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:22:37 2015 +0200
+
+    rpm: drop dead conditionals
+    
+    This commit drops conditionals that no longer serve any purpose, since
+    jewel and above do not support the distro versions they are checking for.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1332fac70e537014b5e03810aa983e2c9210d8dc)
+
+commit 4317a17997f21019e1dec59606928a2a9f4d94b9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:18:33 2015 +0200
+
+    rpm: drop python-argparse dependency
+    
+    This was only necessary for older (now unsupported) distro versions.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6c42066d488d03142a56f92cff99d9c29d106214)
+
+commit 70466ffd386795082f02d718e804ad9e9c494401
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:02:40 2015 +0200
+
+    rpm: drop _with_systemd
+    
+    Ceph versions jewel and above only support systemd.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2ca6dd250b6a049f27fa17c5e8ca11674acb63f6)
+
+commit 928a8158aa71012db9f451a4302e8661783b5d37
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 09:43:06 2015 +0200
+
+    rpm: drop python_sitelib/sitearch conditional
+    
+    This conditional was required to support older versions of RHEL/CentOS that are
+    no longer supported in infernalis and above.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 98d52120f89af3e93896d0518dbea4c4f660af48)
+
+commit 9f76b9ff31525eac01f04450d72559ec99927496
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Apr 18 09:16:02 2016 -0400
+
+    udev: remove 60-ceph-partuuid-workaround-rules
+    
+    These were added to get /dev/disk/by-partuuid/ symlinks to work on
+    wheezy.  They are no longer needed for the supported distros (el7+,
+    jessie+, trusty+), and they apparently break dm by opening devices they
+    should not.
+    
+    Fixes: http://tracker.ceph.com/issues/15516
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9f77244b8e0782921663e52005b725cca58a8753)
+
+commit 1ad390197af21d8cd7e507e2959f803028d3d09d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 20 17:21:56 2016 +0200
+
+    rpm: refrain from installing systemd unit files twice
+    
+    These files are already installed by systemd/Makefile.am
+    
+    Fixes: http://tracker.ceph.com/issues/15549
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6d3cbaa509525f1807ac34bf5fed0fe99255f40f)
+
+commit 9c81f25b76ef2211910e453ba2bc4577fc921d7f
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Feb 23 05:57:14 2016 +0100
+
+    test/ec: build the libs only when 'make check'
+    
+    Currently, we are always building the erasure code libraries while we
+    need them only when 'make check' is run. Moving the test libraries to
+    check_LTLIBRARIES should fix this for us.
+    
+    We no longer need to remove the libec libs manually, remove the lines
+    that do that.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 0bbfea5cf13c138bbb2f0993607fa0dbf756281b)
+
+commit 663d7c2901ddf0b6e4184ec344bf0af00582eaf2
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 16:53:18 2016 +0800
+
+    mon/Monitor: use CEPH_MON_PORT for CRC output
+    
+    The default monitor port will be changed from 6789 to 3300, which has
+    been assigned to us by IANA. So update this accordingly for better
+    tracing and understanding.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 2b3d5686d76ee65fceca31d0d3343c2eaa4f2364)
+
+commit 64b948c21a4608a7680012e247f23dd2a3393f3f
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 15:03:55 2016 +0800
+
+    mon/Monitor: fix memory leak
+    
+    Fixes: http://tracker.ceph.com/issues/15793
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit ae5184ac07523a050584a3d621954baeb0dca01a)
+
+commit 9c3c6d57552a0a2cd448f0e60ded2f96ef9802ef
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 14:54:32 2016 +0800
+
+    mon/Monitor: use VOID_TEMP_FAILURE_RETRY wrapper for fd close().
+    
+    This is safer and much more robust.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit bfa40363b2d8fc25b4aed0164579d55137ac9e8b)
+
+commit 544c538d3720d7f18fca29ba15fd47200d3e05fc
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 10:02:01 2016 +0800
+
+    mon/Monitor: add "host" to tracked_conf_key
+    
+    The get_tracked_conf_keys() method gets the specified keys
+    registered and works as filters for configuration changing,
+    so we can track these key configuration changing and react
+    properly.
+    
+    Right now graylog prints host in its log entry, so
+    Monitor::update_log_clients() also reads from conf.host
+    and pass the host read from conf to graylog. That is
+    why we shall get "host" to the tracked_conf_key too.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit c79fc3305be42bd6126e8a60ae1f220d86754830)
+
+commit e5ebb51f22c43d49d716129ffcb5f87390b7acab
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 15:52:16 2016 -0400
+
+    journal: replay position might change after pruning stale tags
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f5f1c375d8ef7b301cfd20b6848bdc81a761746)
+
+commit 9ecc3dcb76cc8f6d2904118d6b246d77d10cd9cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 15:50:04 2016 -0400
+
+    librbd: delay commit of overwritten journal event
+    
+    With the cache enabled and write-after-write IOs to the same
+    object extents, it was possible for the overwritten journal event
+    to be committed before the overwriter journal event was written
+    to disk.  If a client crash occurs before the event is written,
+    the image will be inconsistent on replay.
+    
+    Fixes: http://tracker.ceph.com/issues/15938
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e8bf64cc85ffe3d2dda23eab1834f7a5f104f6fe)
+
+commit 070dc7c98daaacaa05f72f50561d47ce459984da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 14:13:46 2016 -0400
+
+    ObjectCacher: pass new journal tid when overwriting extent
+    
+    librbd needs to ensure the new journal event has been safely
+    committed to disk before it can mark an overwritten journal
+    event as committed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0051316effe0b25804378975b117691d7ad08a3b)
+
+commit 0dfc787ef58fe9aa8177742f7c6e294e3b5d4b89
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 13:48:22 2016 -0400
+
+    qa/workunits/rbd: record rbd CLI debug messages during mirror stress
+    
+    The debug messages from 'rbd bench-write' and 'rbd snap create',
+    in addition to the existing debug messages from rbd-mirror, make
+    it possible to determine the source of any image inconsistency.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 714ed5e64e65fed9ac886537becf3a85ae6ae7fd)
+
+commit ffd545bf173c5a076c47541cbe3889175188d73e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 16 14:35:12 2016 -0700
+
+    rgw: keep track of written_objs correctly
+    
+    Fixes: http://tracker.ceph.com/issues/15886
+    
+    Only add a rados object to the written_objs list if the write
+    was successful. Otherwise if the write will be canceled for some
+    reason, we'd remove an object that we didn't write to. This was
+    a problem in a case where there's multiple writes that went to
+    the same part. The second writer should fail the write, since
+    we do an exclusive write. However, we added the object's name
+    to the written_objs list anyway, which was a real problem when
+    the old processor was disposed (as it was clearing the objects).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8fd74d11682f9d0c9085d2dc445fc3eb5631f6e0)
+
+commit 8356021ef3d3d21fd632d3caa660ba9a03cf80d3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 16:57:49 2016 -0700
+
+    Pipe: take a ref to existing while we are waiting
+    
+    Otherwise, if it is reaped while we are waiting, it'll be a
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15870
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b224912d249453d754fc0478d3680f8cfa1a5c22)
+
+commit c11db6f103b2dd0049894a606916a84d66933833
+Merge: 7834070 e0dfc55
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 14:57:00 2016 -0400
+
+    Merge pull request #9232 from jdurgin/wip-mirror-workunit-jewel
+    
+    jewel: qa/workunits/rbd: fixed rbd_mirror teuthology runtime errors
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e0dfc55ac874f2531bd0b88aa1c81156d7f7aec9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 14:17:13 2016 -0400
+
+    qa/workunits/rbd: fixed rbd_mirror teuthology runtime errors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ef09c4d8d3c7c6fbc9b6f7ef17cca0179e73468)
+
+commit 7834070fd8ead19e49fb8c766ea3ce8bfe63f032
+Merge: 9873fc1 5f09b9a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:02:51 2016 -0400
+
+    Merge pull request #9226 from dillaman/wip-15950
+    
+    jewel: rbd-mirror: potential crash during image status update
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9873fc13d5647e94ec6f0152f903dd67b2882b0d
+Merge: aee6f7e 6c2973c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:02:06 2016 -0400
+
+    Merge pull request #9069 from dillaman/fixup-mirror_image_status_list-jewel
+    
+    jewel: cls::rbd: mirror_image_status_list returned max 64 items
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5f09b9aef8f30d9dc7e5ec934b228d885e6456e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 10:11:12 2016 -0400
+
+    journal: reset watch step after pruning expired tag
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 48c2f4e079000021b6ebe807bb1dba0dc1a6a11d)
+
+commit 2ee1e0a583d0ca851db4671b561eff9d493ca7d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 11:01:22 2016 -0400
+
+    rbd-mirror: additional debug messages during image replayer start/stop
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afc891f7fcb000d106b1d722386f6d81ddeddab6)
+
+commit f9e6bd8f1b6750d500fba3dd134257b7baac08c3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 00:55:01 2016 -0400
+
+    rbd-mirror: ensure proper handling of status updates during shutdown
+    
+    Previously, several shutdown race conditions could occur due to the
+    use of the async work queue for scheduling updates.
+    
+    Fixes: http://tracker.ceph.com/issues/15909
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ae4edc05660665c3a135536a7c51d4070d28a40)
+
+commit f789d83cbb9dbb1a6daa15585e2be2980ea19616
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 23:31:02 2016 -0400
+
+    rbd-mirror: track bootstrap state within image status
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5dd97b4bf4a993911febf34dce90470a2a91d59)
+
+commit e0b3879692d38bcc01d928cef09f74bc850cef86
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 23:25:25 2016 -0400
+
+    rbd-mirror: combine ImageReplayer stopped and uninitialized states
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a8f6dde4dbae8039380f31e026030a22a89fb4bd)
+
+commit 9c7977da45aef44160efadb48cb2d606568d4a7b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 16:14:42 2016 -0400
+
+    rbd-mirror: lock ordering issue in status update callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5c04c1361074ca9e9df1bc1c2dd82dfafc2a1304)
+
+commit 66f7f2d7bda7f2db6871870f5622594d6391b921
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:23:11 2016 +0300
+
+    rbd-mirror: don't unregister asok commands if image replayer start failed
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2fd6fdd3dc9ecb915362f5e3aa8a224210273540)
+
+commit 569f06f1241309a710a0570d4ede950c1dc6e5d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:25:04 2016 +0300
+
+    rbd-mirror: avoid potential deadlock
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 46bf727d43af4975beeb514630a99d59cfb0a406)
+
+commit aee6f7ed791abd63ff9d1158a258d51db6c5ad28
+Merge: 2d0669e 0d19390
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:01:50 2016 -0400
+
+    Merge pull request #9217 from dillaman/wip-15945
+    
+    jewel: journal: live replay might skip entries from previous object set
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0d193905b1d552600b542a466759af054c80e498
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 16 21:17:09 2016 -0400
+
+    qa/workunits/rbd: rbd-mirror daemon stress test
+    
+    This test repeatedly runs rbd bench-write, kills the process
+    randomly to create an unclean journal shutdown, and verifies
+    that the image content replicates correctly.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e2ab3128a37f4f36c9a1de6ddf2c0fe2e48aa7e2)
+
+commit d020b6ab1d10daa92fc99d1506222987b4306b26
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 16 18:08:35 2016 -0400
+
+    journal: helper method to detect newer tags
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a8a6126ea35344e85af7eb64ffc490938edba51)
+
+commit 56515070710a6723ae4adc3e3b166e9ae9a35fdf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun May 15 09:52:41 2016 -0400
+
+    journal: skip partially complete tag entries during playback
+    
+    If a journal client does not fully write out its buffered entries
+    before quiting, replay should skip over all remaining out-of-
+    sequence entries for the tag.
+    
+    Fixes: http://tracker.ceph.com/issues/15864
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9454f7e4c62437b1c288f371009feba1fd374584)
+
+commit 18f663d834edd03b42e2c08b9428e72fdc6bae9d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat May 14 18:58:41 2016 -0400
+
+    journal: close, advance, and open object set ordering
+    
+    Flush in-flight appends to open objects before advancing the
+    active object set.  Additionally, don't start recording to the
+    new objects until after advancing the active set.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit de830057d0f7286914e019540c6263423cb60428)
+
+commit 27fba6ad7e7cb566d32c6e63f35aff966fc60bd7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat May 14 18:13:38 2016 -0400
+
+    journal: new ObjectRecorder closed callback
+    
+    The callback will be invoked if there were in-flight appends
+    when the close was requested.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bba91437dbe3b7a9b6da8a61ccc4c597858c8efc)
+
+commit 870c21c738e42d2cf34478bfa0631d602175dadc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:34:44 2016 -0400
+
+    journal: do not flag append as full if already known to be full
+    
+    Once an object has overflowed or a close is in-progress, re-attempting
+    to close the object is not needed since the async process is already
+    underway.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45620dc312ca2aeaf084638144aaa4783b622c4a)
+
+commit 1cdebc7b896dd36bfee30f12aa3abd602c15af3e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:28:50 2016 -0400
+
+    journal: delay object overflow event until in-flight appends settled
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ee8d311a1fcdf7a85cdc168e1740fbaa252e87d3)
+
+commit 821d92af2fcc43ff560f01c78ff699ad7df33daa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:17:37 2016 -0400
+
+    journal: ignore flush on closed/overflowed object
+    
+    The journal would be in-progress on transitioning to a new
+    object recorder in a newer object set.  Once the records
+    re-attach to the new object player they will automatically
+    flush.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a13cb821f71b6b1b3378464bcc2d8318eb0da48a)
+
+commit 3d860d5db37f77c1e8c76d214529266062caaa75
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:10:11 2016 -0400
+
+    journal: implicitly detach future's flush handler on append
+    
+    If the future is already in-flight, there is no purpose served
+    by requesting the future be flushed to disk.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1cb9b1a1b25d0f4c2d9a9e22661636e195fc70a0)
+
+commit f98e1289ce10aeb059a0b7d7cc8d17d6f1d60219
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 15:22:30 2016 -0400
+
+    journal: async callback for advancing the active object set
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ca89abfd9e6f0e8edc3234afea387757ef694826)
+
+commit 86e51ebcb301d823fb5c8b30bdc07107704679b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 14:49:07 2016 -0400
+
+    journal: re-fetch active object before advancing set during replay
+    
+    During a live replay, it's possible that an append and and overflow
+    into the next object could race with the live playback of the same
+    object.  Re-fetch an "empty" object at least once before advancing
+    to next set to ensure all records have been read.
+    
+    Fixes: http://tracker.ceph.com/issues/15665
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6056f8c45c99bd37cb18933a37cc238c7e9a7c7d)
+
+commit 2d0669e172c37d38f118332574a474a534d27020
+Merge: f3979cd e885f1e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 13:08:00 2016 -0700
+
+    Merge pull request #9210 from yehudasa/wip-15926-jewel
+    
+    radosgw-admin: fix 'period push' handling of --url
+
+commit e885f1e3be0f9a0c6457c5387f1c38b6e8b6a492
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 16 09:37:05 2016 -0400
+
+    radosgw-admin: fix 'period push' handling of --url
+    
+    was calling send_to_remote_gateway(), but passing 'url' instead of
+    'remote'. now uses send_to_remote_or_url() to accept either
+    
+    Fixes: http://tracker.ceph.com/issues/15926
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit f3979cd1d2fe910185db6dc3894722dd385379bd
+Merge: 1356d29 37ecfa6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 19 12:56:15 2016 +0200
+
+    Merge pull request #9194 from ceph/wip-jewel-no-lttng-global-link
+    
+    Do not link lttng into libglobal
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 37ecfa6c754683cac1c6f4c66a1a491d2fd5db1d
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Mon May 2 14:01:27 2016 +0200
+
+    global: don't link lttng into libglobal
+    
+    Rely on dynamic initialization instead. Linking lttng in this way had
+    the unfortunate side effect of causing radosgw to segfault (when
+    daemonized) during sigterm processing (ie. during lttng_ust_exit()).
+    
+    This was originally removed in 638738f, but accidentally re-added via
+    5f61d36.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+
+commit 1356d29c70afedcbd55599d29aab93d2f79a02ad
+Merge: 15c4ad4 16ba13c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 13:23:39 2016 -0400
+
+    Merge pull request #9180 from dillaman/wip-15746-jewel
+    
+    jewel: doc: update mirroring guide to include pool/image status commands
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 16ba13c83d678656723173c061318dd83805768a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 11:15:34 2016 +0300
+
+    doc: update mirroring guide to include pool/image status commands
+    
+    Fixes: http://tracker.ceph.com/issues/15746
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4381e341c435c80793b847115eed616b8359c4e6)
+
+commit 818166e25d2ccac5d9bb3541f6166fcb8c25fd64
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 11:11:55 2016 +0300
+
+    doc: fixup: "rbd-mirror daemon" instead of "rbd-daemon"
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ddd6345f8f7edcff638ed435aef316ba3e0e40dd)
+
+commit 15c4ad44010c798af7804e287ba71dcc289f806f
+Merge: 3a66dd4 7be281d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 18 05:11:55 2016 -0400
+
+    Merge pull request #8815 from ktdreyer/wip-15646-jewel-system-targets
+    
+    jewel: debian: install systemd target files
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7be281dab96c15b53fcfc3c2ffe6d17aec84b315
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 28 09:13:50 2016 -0400
+
+    debian/control: dh_systemd_start is in the dh-systemd package
+    
+    Fixes: http://tracker.ceph.com/issues/15573
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89154d1e01dbc58dc92f37741031f7ddb0448d2e)
+
+commit e463aa8b92dabdc7c13e150585eb09790da638ba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 22 21:07:58 2016 +0800
+
+    debian: install systemd target files
+    
+    * enable it using dh_systemd_enable
+    * start the target using dh_systemd_start
+    * move the dh_installinit, dh_systemd_enable, dh_systemd_start calls
+      down, so they can identify the service files if they care about them.
+    
+    Fixes: http://tracker.ceph.com/issues/15573
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f950a0a7f5b35e13668379bc69dcb5d98a2dfc1f)
+
+commit 72c9b6f70752faf779b9272724f428e68d6499bb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 9 14:15:36 2016 +0800
+
+    osd: remove all stale osdmaps in handle_osd_map()
+    
+    in a large cluster, there are better chances that the OSD fails to trim
+    the cached osdmap in a timely manner. and sometimes, it is just unable
+    to keep up with the incoming osdmap if skip_maps, so the osdmap cache
+    can keep building up to over 250GB in size. in this change
+    
+    * publish_superblock() before trimming the osdmaps, so other osdmap
+      consumers of OSDService.superblock won't access the osdmaps being
+      removed.
+    * trim all stale osdmaps in batch of conf->osd_target_transaction_size
+      if skip_maps is true. in my test, it happens when the osd only
+      receives the osdmap from monitor occasionally because the osd happens
+      to be chosen when monitor wants to share a new osdmap with a random
+      osd.
+    * always use dedicated transaction(s) for trimming osdmaps. so even in
+      the normal case where we are able to trim all stale osdmaps in a
+      single batch, a separated transaction is used. we can piggy back
+      the commits for removing maps, but we keep it this way for simplicity.
+    * use std::min() instead MIN() for type safety
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 369db9930887d75b498927da9c97733bff4472b6)
+
+commit 6c2973cd08503a3d91eeb881732c79213e42599f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:14:46 2016 +0300
+
+    cls::rbd: mirror_image_status_list returned max 64 items
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e925ce86243c91e6d64ff8d7b45166e4e3f2650b)
+
+commit 36743419ba49df4f97cdbe9ac25d9b08b0685108
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 15:24:09 2016 -0400
+
+    osd/PG: update info.stats.* mappings on split
+    
+    These are updated in the init and start_peering_interval paths, but not
+    on split.
+    
+    Fixes: http://tracker.ceph.com/issues/15523
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8b42294d98dd70715184991da5ba495fbe93d009)
+
+commit b443fdf9402df621400ab7ca66799f597e8b2c23
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 11:42:25 2016 +0200
+
+    ceph.spec.in: recommend ntp-daemon on SUSE
+    
+    SUSE has settled on "ntp-daemon" as the generic package name. The "ntp" and
+    "chrony" etc. packages have "Provides: ntp-daemon" in their respective spec
+    files.
+    
+    References: http://tracker.ceph.com/issues/15419
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23108547454f53b95e193086d82d433570312ad4)
+
+commit 299f84ccdbf4d91412e216042fb248fd802af89d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 7 20:54:21 2016 +0200
+
+    deb: make ceph-base and radosgw recommend time-daemon
+    
+    References: http://tracker.ceph.com/issues/15419
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f27cda03b7a961e445cce4ad35db6417e9e55959)
+
+commit 2da7cb2c7daad3ce2f9d2614e019291f75d4737d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 7 20:17:44 2016 +0200
+
+    systemd: make Ceph daemon units "want" time-sync.target
+    
+    Fixes: http://tracker.ceph.com/issues/15419
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit df893f395e62c8b1db18a277b38b44cab8b2016f)
diff --git a/doc/changelog/v10.2.3.txt b/doc/changelog/v10.2.3.txt
new file mode 100644
index 000000000..a21d1b7a1
--- /dev/null
+++ b/doc/changelog/v10.2.3.txt
@@ -0,0 +1,5957 @@
+commit ecc23778eb545d8dd55e2e4735b53cc93f92e65b
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Sep 20 22:04:23 2016 +0000
+
+    10.2.3
+
+commit 9bfc0cf178dc21b0fe33e0ce3b90a18858abaf1b
+Merge: aeee80e 1cbc839
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 29 13:43:15 2016 +0200
+
+    Merge pull request #10804 from dachary/wip-17089-jewel
+    
+    jewel: OSD failed to subscribe skipped osdmaps after ceph osd pause
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit aeee80e64af2f685ac9785c3444f40b60d0877f2
+Merge: 3168a84 dc154cd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 29 13:43:00 2016 +0200
+
+    Merge pull request #10760 from dachary/wip-16865-jewel
+    
+    jewel: saw valgrind issues in ReplicatedPG::new_repop
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3168a8482211a2aaf9b4d715c0f7a920e896d9bb
+Merge: 12b374f ce986aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 26 22:12:39 2016 +0200
+
+    Merge pull request #10662 from dachary/wip-16901-jewel
+    
+    jewel: segfault in RGWOp_MDLog_Notify
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 12b374fccfb9792998ab88a4d7777f8b8134cb5c
+Merge: 91ee75f f298643
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:31:06 2016 +0200
+
+    Merge pull request #10763 from dachary/wip-17034-jewel
+    
+    jewel: rgw: object expirer's hints might be trimmed without processing in some circumstances
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 91ee75f39c05fe0c25aab2664f1fa71f858e7c6f
+Merge: d8bdc22 a865f26
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:28:10 2016 +0200
+
+    Merge pull request #10658 from dachary/wip-16862-jewel
+    
+    jewel: default zone and zonegroup cannot be added to a realm
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d8bdc2245ddec91c2f55d367045e4407112d2af1
+Merge: 6d270f6 fa2e42d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:26:53 2016 +0200
+
+    Merge pull request #10659 from dachary/wip-16863-jewel
+    
+    jewel: use zone endpoints instead of zonegroup endpoints
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6d270f6084a035040638638ff65c14f5d76eeef1
+Merge: fe23c2e 2485efc
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:24:20 2016 +0200
+
+    Merge pull request #10660 from dachary/wip-16864-jewel
+    
+    jewel: multisite segfault on ~RGWRealmWatcher if realm was deleted
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit fe23c2edc70d1ad15e640195570e624b31415aef
+Merge: a57c21a 577336e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 26 00:46:17 2016 +0200
+
+    Merge pull request #10847 from gregsfortytwo/jewel-fix-mds
+    
+    mds: fix double-unlock on shutdown
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit a57c21afa686967a3afc4ddabb97c665e5894b21
+Merge: 49022d4 ce5e250
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:37 2016 +0200
+
+    Merge pull request #10663 from dachary/wip-16934-jewel
+    
+    jewel: Add zone rename to radosgw_admin
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 49022d4b2e2c5506dd83f3d894ac0a4ba83f8c7b
+Merge: f7afbbf 22c9df1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:22 2016 +0200
+
+    Merge pull request #10664 from dachary/wip-16958-jewel
+    
+    jewel: Bug when using port 443s in rgw.
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f7afbbf094e5eabc1a651379f522d0b1544b01a3
+Merge: 7b672cb 73cc608
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:07 2016 +0200
+
+    Merge pull request #10764 from dachary/wip-17032-jewel
+    
+    jewel: multisite: RGWPeriodPuller tries to pull from itself
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7b672cbe343955228b2a701eba91a30f5cac7fe5
+Merge: 28bc826 55460e2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:03:40 2016 +0200
+
+    Merge pull request #10765 from dachary/wip-17006-jewel
+    
+    jewel: Increase log level for messages occuring while running rgw admin command
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 28bc826e9140520e86e2eb34470103979d10ae49
+Merge: 8b37c68 21da103
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 22:25:21 2016 +0200
+
+    Merge pull request #10845 from cbodley/wip-16702
+    
+    jewel: rgw: add tenant support to multisite sync
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b37c68f761d59025678c184a4f9b1755c90a628
+Merge: 94eb163 2266287
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 09:17:52 2016 +0200
+
+    Merge pull request #9405 from SUSE/wip-16083-jewel
+    
+    jewel: mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 94eb163c19e02ffae06659ced714140bd55584f0
+Merge: 9de2e93 f5db5a4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 09:02:17 2016 +0200
+
+    Merge pull request #10654 from dachary/wip-9577-jewel
+    
+    jewel: mon: "mon metadata" fails when only one monitor exists
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 577336e20f68aa289311103e54d4def793b0cc01
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Aug 24 13:34:43 2016 -0700
+
+    mds: fix double-unlock on shutdown
+    
+    We did a bad backport or something and accidentally ended up with two Unlock()
+    calls on mds_lock. Don't.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 21da1039fc57bcf4054c0e7a13bb2732781770f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 12 20:36:35 2016 -0700
+
+    rgw: collect skips a specific coroutine stack
+    
+    Fixes: http://tracker.ceph.com/issues/16665
+    
+    Instead of drain_all_but() that specifies number of stacks to leave behind,
+    added drain_all_but_stack() that has a specific stack specified. This is needed
+    so that we don't call wakeup() through lease_cr->go_down() on a cr stack that
+    was already collected.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5a2e8f0526db92a290c711f82627fc5042c290ea)
+
+commit 9de2e93d1b55c5f5f54b3dbf334d1513abb3f174
+Merge: 1d5dedf d1e0512
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 22:08:58 2016 +0200
+
+    Merge pull request #10791 from dachary/wip-17004-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_state == STATE_STOPPING)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1d5dedf0307f0fddf55f6487486a281c2625ef92
+Merge: e30c76e ecea6dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 22:08:30 2016 +0200
+
+    Merge pull request #10827 from dillaman/wip-16950-jewel
+    
+    jewel: librbd: delay acquiring lock if image watch has failed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 98779c32de98efc367e888b99fd154ad4e5274d9
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jul 11 17:38:43 2016 -0700
+
+    rgw: fix compilation
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 874de58024e12d893ae050cd421ef67c9521c753)
+
+commit f5d9537d3e69197c7eadb21a6d0d01373530e2de
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 7 14:42:00 2016 -0400
+
+    rgw: remove datalog keys from error repo on ENOENT
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 16976eedb70292e821193e39bb577a68df1bc95a)
+
+commit 32505b2c8d703d09224b6a2f7007b20a79e0fb3d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Jul 1 12:36:04 2016 -0400
+
+    test/rgw: add optional --tenant flag to test_multi.py
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6f65d192cd3917cdcf7d9deb5396750d2c641842)
+
+commit f68337aaa535ea182258eb4934af739fb703f556
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 15:50:00 2016 -0400
+
+    rgw: add tenant id to GetBucketInstanceInfo
+    
+    use the rgw_bucket overload of get_bucket_instance_info() so it can
+    supply the tenant id
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3cf715c6828d7d9732bf6a6fd9b46bbd8c08de5f)
+
+commit a0ffffa5aad423f363bf10a6b8dbaf19a47262a2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 29 11:58:55 2016 -0400
+
+    rgw: carry tenant id with data sync
+    
+    use rgw_bucket_shard to track buckets, which includes tenant id
+    
+    Fixes: http://tracker.ceph.com/issues/16469
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d945e2099525e39588b572e357b115df98c8cdca)
+
+commit bff626f7b32b3fe918510c147f88c49ead5007fa
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 17:43:40 2016 -0400
+
+    rgw: add tenant to url for RGWRESTStreamWriteRequest
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2a1e1a776c58f62fbcb57877d56b92eb1e8933e6)
+
+commit dc50687f9ee7291192b2291c3c5770c46c852677
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 17:42:54 2016 -0400
+
+    rgw: add tenant to rgw_bucket json format
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fc0df4802b48fa095587b93557c82da5446d9ca3)
+
+commit 06223adc51d2ccc1494c5e6891ad367edde69a91
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 00:13:47 2016 -0400
+
+    rgw: add rgw_bucket_parse_bucket_key()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d5ac140040ea2f17e33d2846870cb033c7ae6ffa)
+
+commit b4f687ee7fce682a41c707af7e2feb6b3a1cbe15
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jun 28 11:24:46 2016 -0400
+
+    rgw: add tenant name to datalog entries
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c8aa668fc3e34a7651ce6e2dda62902ced808541)
+
+commit dda0ee035fe9bb0c2fc4366df8355184db5034e3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 6 11:50:47 2016 -0400
+
+    rgw: convert bucket instance listings back to metadata key format
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3eae201c8a1513bae23233700c754ec076360664)
+
+commit 6225a6d45b8e76f9afaa741fa61d4b9742ef60fb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 5 09:00:19 2016 -0400
+
+    rgw: use tenant/ for bucket instance metadata keys
+    
+    to work around the ambiguity of parsing tenant: and :shard in the same
+    bucket instance metadata key, use tenant/ instead
+    
+    to preserve backward compatibility with existing objects, new helper
+    function rgw_bucket_instance_key_to_oid() converts this / back to a :
+    before being used as an object name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0befa1e7be3aeb37209bac2e2a7de0edf5d5a95)
+
+commit cac6612a8a4f344e910d0c880ee8d020fac1ec79
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 18:09:03 2016 -0400
+
+    rgw: add get_key() methods to format rgw_buckets
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 51ff8ef7c6bb7c9f55f3ca8ad059b029a4088cdf)
+
+commit ac557e04e2d104fe9fdeae1d1ca31e212cfc89f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 6 16:16:33 2016 -0700
+
+    rgw: data sync debug logging
+    
+    a high level debug logging of the data sync process.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97ef16c0cc0ad8345b5a897108717d83ab0aa9ab)
+
+commit 6bb8c15a38e1ebf4e2b03f43bb527d460cd82dc3
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Thu Jun 16 14:45:57 2016 +0530
+
+    rgw: modifying multi-site log messages.
+    
+    Increasing the log level and removing 'ERROR' from messages that appear during normal multi-site operation.
+    Fixes: http://tracker.ceph.com/issues/16121
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit e1737ddaa035f23757d021dd34b0c87d24ce372c)
+
+commit e30c76ee33ddcb1a58835637c5e506c6c4b61b12
+Merge: df71e57 308f514
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 18:57:23 2016 +0200
+
+    Merge pull request #10653 from dachary/wip-10495-jewel
+    
+    jewel : 60-ceph-partuuid-workaround-rules still needed by debian jessie (udev 215-17)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit df71e5740255ffc0a809b2c437ce37826aed11d3
+Merge: 286b30f f80d10c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 18:51:54 2016 +0200
+
+    Merge pull request #10357 from SUSE/wip-16748-jewel
+    
+    jewel: mount.ceph: move from ceph-base to ceph-common and add symlink in /sbin for SUSE
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 286b30fc54c1ea6a41c38a80c5227f9deeb13741
+Merge: eb28401 48ee3aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:48:00 2016 +0200
+
+    Merge pull request #10790 from dachary/wip-17005-jewel
+    
+    jewel: ImageReplayer::is_replaying does not include flush state
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit eb28401a59e2ca1a30d0f29aab7a2ea5eb704aea
+Merge: 4f3605b d336735
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:45 2016 +0200
+
+    Merge pull request #10792 from dachary/wip-16978-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_on_update_status_finish == nullptr)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4f3605b0c3cba436b362318487bec4358ea8c4d0
+Merge: 0c60d47 8316b6a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:36 2016 +0200
+
+    Merge pull request #10646 from dachary/wip-16576-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_local_image_ctx->object_map != nullptr)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0c60d47579591f07bc4e8d24a52c0355fbd17917
+Merge: 6dd0327 5968664
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:18 2016 +0200
+
+    Merge pull request #10647 from dachary/wip-16593-jewel
+    
+    jewel: FAILED assert(object_no < m_object_map.size())
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6dd0327b40b35e19cb14ce30dec0d75d5a7da086
+Merge: cb4cb4a e098fb1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:08 2016 +0200
+
+    Merge pull request #10786 from dachary/wip-17061-jewel
+    
+    jewel: bashism in src/rbdmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cb4cb4a02b160d9dd65e624cd762c12284eed241
+Merge: 089bb43 46246e3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 14:12:34 2016 +0200
+
+    Merge pull request #10421 from SUSE/wip-16659-jewel
+    
+    jewel: ReplicatedBackend doesn't increment stats on pull, only push
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 089bb43b146408e5e670c3788990b712d8d26543
+Merge: 59b6c20 28697fc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 15:42:11 2016 +0200
+
+    Merge pull request #10501 from Abhishekvrshny/wip-16621-jewel
+    
+    jewel: mds:  tell command blocks forever with async messenger (TestVolumeClient.test_evict_client failure)
+    
+    Reviewed-by: Douglas Fuller <dfuller@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 59b6c20ea5bfb2d7062f7119ffbecd312d342bc4
+Merge: e1ddce7 f4fb598
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 15:23:34 2016 +0200
+
+    Merge pull request #10816 from SUSE/wip-17092-jewel
+    
+    jewel: build/ops: need rocksdb commit 7ca731b12ce for ppc64le build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ecea6dcf1c36bc5d478cf030f7ba1e01ca35a2d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 13:24:54 2016 -0400
+
+    librbd: delay acquiring exclusive lock if watch has failed
+    
+    Fixes: http://tracker.ceph.com/issues/16923
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dfe9f3eac9cca3b83962e0e1b7eac38e6e76d7a5)
+
+commit 49a39ebf6f7c6b7b0b19e4486fc10e57637e143c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 13:47:33 2016 -0400
+
+    librbd: convert ImageWatcher class to template
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 814c305ce8c35b5ce01d7e29a912d5ef3978754b)
+    
+     Conflicts:
+            src/librbd/ImageWatcher.cc: no shrink guard
+            src/librbd/Operations.cc: no shrink guard
+
+commit e1ddce7654698f3e41d89406981b1eb26f8f72a3
+Merge: 2c387d5 a7987f0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:48:19 2016 +0200
+
+    Merge pull request #10103 from xiaoxichen/wip-16037-jewel
+    
+    jewel: MDSMonitor::check_subs() is very buggy
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 2c387d5cd2785d518982f1c0d209235c240cb7ae
+Merge: 128251c 64d99b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:48:01 2016 +0200
+
+    Merge pull request #10105 from xiaoxichen/wip-16515-jewel
+    
+    jewel: Session::check_access() is buggy
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 128251ceddf6129a1836a6d991a727adc93869df
+Merge: d27c6ac bce5646
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:37 2016 +0200
+
+    Merge pull request #10106 from xiaoxichen/wip-16215-jewel
+    
+    jewel: client: crash in unmount when fuse_use_invalidate_cb is enabled
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d27c6ac57b5cd2f5566c5ecf476b8fe37041ab27
+Merge: 63422d0 fd7ff96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:23 2016 +0200
+
+    Merge pull request #10108 from xiaoxichen/wip-16320-jewel
+    
+    jewel: fuse mounted file systems fails SAMBA CTDB ping_pong rw test with v9.0.2
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 63422d08ea6d401cefd6b438a7de48c1528da114
+Merge: 9fd5291 aa39361
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:10 2016 +0200
+
+    Merge pull request #10199 from SUSE/wip-16625-jewel
+    
+    jewel: Failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9fd5291345e4f7d08faf8110ba78df244d030db6
+Merge: 2823714 2beb56a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:53 2016 +0200
+
+    Merge pull request #10499 from Abhishekvrshny/wip-16299-jewel
+    
+    jewel: mds: fix SnapRealm::have_past_parents_open()
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 282371465476e37408f65fc33145f8a76cf35ac7
+Merge: e2f7de6 1d3a816
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:38 2016 +0200
+
+    Merge pull request #10500 from Abhishekvrshny/wip-16620-jewel
+    
+    jewel: Fix shutting down mds timed-out due to deadlock
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit e2f7de68393dcf008e366cfc40b199b5acfcd428
+Merge: 6f287bf 7c2eab1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:13 2016 +0200
+
+    Merge pull request #10502 from Abhishekvrshny/wip-16797-jewel
+    
+    jewel: MDS Deadlock on shutdown active rank while busy with metadata IO
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 6f287bf2b99b8f6b1ae21f017d905dfa4a569649
+Merge: d68d41e d244b7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:45:41 2016 +0200
+
+    Merge pull request #10104 from xiaoxichen/wip-16560-jewel
+    
+    jewel: mds: enforce a dirfrag limit on entries
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d68d41ebdae92666b4dcf6cc8455cf17ab1d903a
+Merge: bc6e329 a5f5513
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 01:19:33 2016 +0200
+
+    Merge pull request #10074 from ceph/jewel-16002
+    
+    jewel: ObjectCacher split BufferHead read fix
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f4fb59896ccb0d8ac01434cd4cf6ad67776fb0a0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Aug 22 20:56:38 2016 +0200
+
+    build/ops: bump rocksdb submodule
+    
+    Fixes a FTBFS on ppc64le
+    
+    Fixes: http://tracker.ceph.com/issues/17092
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bc6e3291b4960779fb84b2976079490d5f8f4935
+Merge: 46106f0 e825dd2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 22 10:37:14 2016 +0200
+
+    Merge pull request #10537 from theanalyst/wip-16778-jewel
+    
+    jewel: rgw multisite: preserve zone's extra pool
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 46106f0c0cfeb8746a521559bb466a5021a2c221
+Merge: 1bf76cd 96ad2d1
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:31:39 2016 +0200
+
+    Merge pull request #10655 from dachary/wip-16163-jewel
+    
+    jewel: rgw: can set negative max_buckets on RGWUserInfo
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1bf76cd58ec3635291e5c0080df47f0845088b77
+Merge: 3cef399 8b9954d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:29:40 2016 +0200
+
+    Merge pull request #10580 from dreamhost/wip-16928-jewel
+    
+    jewel: rgw: Fix civetweb IPv6
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3cef399857b5cb234e0f4cd54933d58744736f62
+Merge: e1bc847 9e5a3ae
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:28:14 2016 +0200
+
+    Merge pull request #10216 from SUSE/wip-16637-jewel
+    
+    jewel: add socket backlog setting for via ceph.conf
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e1bc84776e7a95062311fb6b00a85a5327926c92
+Merge: 089967f 429e9c0
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:27:03 2016 +0200
+
+    Merge pull request #10167 from jmunhoz/aws4-streaming-backport-jewel
+    
+    jewel: rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 089967f4374569495d177c7dea1e345ca08c772a
+Merge: 8e2de35 44decb4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:13:51 2016 +0200
+
+    Merge pull request #9544 from Abhishekvrshny/wip-16085-jewel
+    
+    jewel: A query on a static large object fails with 404 error
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8e2de35324258cdbe44dfe8e7ce757ca1b3ef99b
+Merge: 3f007a1 059ed62
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:10:25 2016 +0200
+
+    Merge pull request #10525 from Abhishekvrshny/wip-16732-jewel
+    
+    jewel: Bucket index shards orphaned after bucket delete
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3f007a192f8629b7a6000846db41550eb94cdcb2
+Merge: 1019325 8655803
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:09:21 2016 +0200
+
+    Merge pull request #10188 from dreamhost/wip-16618-jewel
+    
+    jewel: rgw: fix multi-delete query param parsing.
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1019325af6c7153893a74b126f2c5e6534841bc4
+Merge: 0cd574f 4a3c9f3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:06:51 2016 +0200
+
+    Merge pull request #9266 from Abhishekvrshny/wip-15964-jewel
+    
+    jewel: rgw: realm pull fails when using apache frontend
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0cd574f5a9a1db4ddfe262e2c4eb4643c5af9075
+Merge: cdd4ab2 dbf8cf0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 22 10:02:20 2016 +0200
+
+    Merge pull request #10710 from rzarzynski/wip-16393-jewel
+    
+    jewel: rgw: improve support for Swift's object versioning.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1cbc839ebd11acb2ed0ebf0b0117eab09b5141c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 15 15:42:29 2016 +0800
+
+    osd: set objecter.osdmap when starts
+    
+    if any OSD is full or the pause flag is set in cluster, objecter will
+    always try to subscribe the next osdmap using the mon client shared with
+    OSD. but if the OSD's osdmap is very far behind on osd maps due to some
+    reasons, `MonClient::sub_want_increment()` will reject the subscribe
+    request from OSD, because it's asking for older maps while the mon client
+    is about to send the request from objecter for the *next* osdmap.
+    
+    so we need to update objecter with the latest local osdmap, so it is
+    able to skip the new osdmaps if OSD needs to catch up with the cluster
+    first.
+    
+    Fixes: http://tracker.ceph.com/issues/17023
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 1515e08e7d425f01d2e9d02c34b2ae40cdf2611a)
+
+commit 4d37cfb6eed1737b21d9f3f18eee6169f48445b2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 12 19:19:51 2016 +0800
+
+    mon/MonClient: should update sub_sent with sub_new
+    
+    std::map::insert() does not overwrite existing items with the same key.
+    so we need to do this in a different way. and we will check sub_sent
+    aftewards, so we need to keep an updated sub_sent around.
+    
+    Fixes: http://tracker.ceph.com/issues/17023
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 550814900ebca9dd89e088b9abe24f66eb5e3e6d)
+
+commit cdd4ab2897a3d0004b1e50401754004f95559e18
+Merge: 38433ad 5498377
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 21 21:48:20 2016 +0200
+
+    Merge pull request #10797 from trociny/wip-17080-jewel
+    
+    jewel: the option 'rbd_cache_writethrough_until_flush=true' dosn't work
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 5498377205523052476ed81aebb2c2e6973f67ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 12:08:37 2016 -0400
+
+    librbd: cache was not switching to writeback after first flush
+    
+    Fixes: http://tracker.ceph.com/issues/16654
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2d9840af39555ce00246b50e4a5c186798bd88ff)
+
+commit 471871e752963bf8cc42dcc1ec7e44e34e7e0ca2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 23:28:43 2016 -0400
+
+    test: unit test cases for disabling librbd journal by policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a256f5044ac54a8a0a9598ee7fd151b8ea08f84)
+    
+    Conflicts:
+            src/test/librbd/CMakeLists.txt (test_ConsistencyGroups.cc)
+
+commit 062162f88f1bdaaaa96713429007757b72ec92f9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 10:56:48 2016 -0400
+
+    librbd: utilize factory methods to create AioObjectRequest objects
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2e5076eb19172919deeb0f4d11f2b2b6cc03fab3)
+
+commit d8eddc6b58fa2b0589719fab54beb42bc8e440bf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 09:07:00 2016 -0400
+
+    librbd: convert AioObjectRequest/AioObjectRead classes to templates
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 840a473e7fb94124ca7c571fc3dd2c787e0d265a)
+
+commit 1a7cb60ee4b898aaea43a9963e95ed575ec66f56
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 09:54:44 2016 -0400
+
+    librbd: move read callback helpers to narrowest scope
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 65b336f68596a430629692a8682c5cfe883c31f1)
+
+commit 026f6d2da25d41f21f5bb5f8ee367cc7601296f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 23:26:08 2016 -0400
+
+    librbd: convert AioImageRequest derived classes to templates
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a945c2c235d9259b1d4a8a579d7e6efc2ed35916)
+
+commit 61f0acb82ba344139bc2aa486fb096e5576e2553
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 08:49:44 2016 -0400
+
+    librbd: removed namespace indentation from legacy classes
+    
+    Better follows the Ceph C++ style guide
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1e391982b43ddfb363ff913260460368a6d5834)
+
+commit e83866bcf01f32c35af94a47ee614361b4a787e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 22:58:24 2016 -0400
+
+    librbd: do not record journal events if append is disabled by policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 405142c615352613e8bacee46e92484eb0c08f26)
+
+commit a9a84bad431f2d9916bdf88980b1cd2d0e36a1e5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 20:57:40 2016 -0400
+
+    librbd: remove unused journal replay canceled callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f453554edcb16ec72714b2456c08bab9e339b1eb)
+
+commit 942950007f9c424949eacf1d1cf3a74ef3b95725
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 20:48:13 2016 -0400
+
+    librbd: optionally support disabling journal appends via policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f7eeacd02f8f80c5d1d5d9cfed35bbd23f64b018)
+
+commit 1f63307a339fcfe3045cc001a72d1ecc197a7010
+Author: shawn <cxwshawn@gmail.com>
+Date:   Fri Jun 17 01:45:58 2016 -0400
+
+    librbd: optimize header file dependency & modify related file.
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 2185a6275c89d9462611e965da407ea0d504987d)
+
+commit c49398dcd92bab90ab852407427bcc2c23147779
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jun 7 03:12:55 2016 -0400
+
+    librbd: optimize operation header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 88afa408683238b803606f3d63de326ead8cefee)
+
+commit 5366973897c1103a1c291c2f886692da4170d901
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jun 7 01:59:05 2016 -0400
+
+    librbd: optimize journal header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 59397e27a04fb0f2b990a86dcaf3c418457d1b72)
+
+commit c95b83e7c0cae0b322af713045745fb1c8fa411a
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 23:36:06 2016 -0400
+
+    librbd: optimize image_watcher header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit aee1559f312e9493d6580e5ffcb606fe21ee3c59)
+
+commit e955496fb638163427c71b0734f6506aa6636d28
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 23:22:39 2016 -0400
+
+    librbd: optimize image header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 461958ebc7e8ccef76b3f08a979482762267c10b)
+
+commit f403abafd5da00e5afc13abba7514fa2bdce9760
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 22:56:49 2016 -0400
+
+    librbd: optimize exclusive_lock  header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 01282eb5cdeda7bbbb77438c7d1953d023ea54b6)
+
+commit 0ca8071e13492a9124be8322843cd96ca2bba531
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 03:37:22 2016 -0400
+
+    librbd: optimize object-map header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 1a2276927d5956918f7a4830b4a44048ac090328)
+
+commit 03314145d1bb5f4330a92d945af9df8856284724
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 14:31:23 2016 +0300
+
+    test: fix librbd tests for rbd_skip_partial_discard
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9717417b7b8aa1b24a82a0bfddbfc23748188641)
+    
+    Conflicts:
+            src/rocksdb this was a mistake in the original commit
+
+commit d0c0c2fce39091915428e815bc6aee265ac9e351
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 14:30:08 2016 +0300
+
+    librbd: discard hangs when 'rbd_skip_partial_discard' is enabled
+    
+    Fixes: http://tracker.ceph.com/issues/16386
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit dc41731fbfd73d9fbb63d3ff360d4c5dd62deaf1)
+
+commit 5bf4398e373ecc26958d8480171f4322341a9d82
+Author: xinxin shu <shuxinxin@chinac.com>
+Date:   Tue Jun 7 12:07:55 2016 +0800
+
+    librbd: object_may_exist always return true when you write an empty object
+    
+    if you write an empty object, object map is updated firstly
+    
+    Signed-off-by: xinxin shu <shuxinxin@chinac.com>
+    (cherry picked from commit a54073808146d205d54d6a932a6e0fd750f1dc38)
+
+commit 67861061fcd85f143e7d26077c71a582c6439745
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 25 12:43:13 2016 -0400
+
+    librbd: ensure that AIO ops are started prior to request
+    
+    Fixes: http://tracker.ceph.com/issues/16708
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6064f2346de0a8bf2878bf5bfac9a992cda7c4ca)
+
+commit 47279f8e0158d0483011bca01455ef9735453b34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 25 12:42:26 2016 -0400
+
+    librbd: helper method for creating and starting AioCompletions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3df7213c0a9f0186e3a37c9e4a10f1c8bc84446e)
+
+commit 38433ad90b81b4e398ae6f4b8e196af1a9ea20b8
+Merge: b98e27c 9fc5b41
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 19 20:35:19 2016 +0200
+
+    Merge pull request #10649 from dachary/wip-16867-jewel
+    
+    jewel: mkfs.xfs slow performance with discards and object map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d3367354b02be7ee231a50b9ad8bca098f840f46
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 14:41:00 2016 -0400
+
+    rbd-mirror: potential assertion failure during error-induced shutdown
+    
+    Fixes: http://tracker.ceph.com/issues/16956
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a465d9dad417e8b52909c5478f7e3e433748948)
+
+commit d1e05127b73c53a02944edc267548656732231fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 10 12:50:53 2016 -0400
+
+    rbd-mirror: potential race condition during failure shutdown
+    
+    Fixes: http://tracker.ceph.com/issues/16980
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74ec7c91f17630c77647cfc9813090d688b3410d)
+
+commit 48ee3aacef8b1628f41936bd13cc0327c44e59ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 9 08:13:30 2016 -0400
+
+    rbd-mirror: replaying state should include flush action
+    
+    Fixes: http://tracker.ceph.com/issues/16970
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7246f1b771b4d1f336ac11b9e881c9eb32bcd4e1)
+
+commit e098fb14f106bf294780abd5d580244e7ce3b20c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 14:10:27 2016 -0400
+
+    doc: added rbdmap man page to RBD restructured index
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4e05cbf7e6eda797a9b08e0081aead19523d5762)
+
+commit b7793d757fca99aa2eb371934e667b3cc555d666
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 14:07:43 2016 -0400
+
+    rbdmap: specify bash shell interpreter
+    
+    Fixes: http://tracker.ceph.com/issues/16608
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 51237c33d1ec4034c5c07f5d63d63838a76bce88)
+
+commit b98e27c42f29f4fa44cbe7f4b7d9ab3633404f0c
+Merge: eb706ab 92d7882
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 19 07:12:40 2016 +0200
+
+    Merge pull request #10684 from dillaman/wip-16904-jewel
+    
+    jewel: rbd-mirror: reduce memory footprint during journal replay
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 92d7882661d632af9e9f07817610297411c0248f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 16:18:56 2016 +0300
+
+    rbd-mirror: remove ceph_test_rbd_mirror_image_replay test case
+    
+    Fixes: http://tracker.ceph.com/issues/16539
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 06a333b116383de4d39c9e098e6e5fa195ceb370)
+
+commit 0d9d57385bcb544115ccdb00d6bbd67b987dbad0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 16:18:23 2016 -0400
+
+    qa/workunits/rbd: override rbd-mirror integration test poll frequency
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 574be7486ad737892422aed0322f80e5750a75a0)
+
+commit 96cdb11934e359024c1a2f7cbffd55a5c7f6715b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 21 07:28:54 2016 -0400
+
+    rbd-mirror: do not cancel maintenance ops with missing finish events
+    
+    librbd will replay these ops when opening an image, so rbd-mirror
+    should also ensure these ops are replayed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 862e581553fff510286b58135a1fd69705c06096)
+
+commit d47a23272765633df2303e142d67d78af5bd2bf4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 15:46:49 2016 -0400
+
+    rbd-mirror: potential memory leak when attempting to cancel image sync
+    
+    The cancel request could race with the actual scheduling of the image
+    sync operation.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6cdf955bad500561ddada2791641eba5fb27762)
+
+commit 1e4d98cdd4c5d671870735823a80e80751ba13ae
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 15:42:27 2016 -0400
+
+    rbd-mirror: fix issues detected when attempting clean shut down
+    
+    Fixed lockdep issue from status update callback and fixed the
+    potential for a stuck status state.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0275c7ca23b27dc5250cd33f317e2273470a9fe8)
+
+commit e4c43190b49e94ccfeda05ac7a7a8835d81b49a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 13:50:20 2016 -0400
+
+    rbd-mirror: shut down image replayers in parallel
+    
+    When multiple pools are being replicated, start the shut down
+    process concurrently across all pool replayers.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 73cdd08007c27d2c3c41fe644601e7a144f21c82)
+
+commit 1e3821268fbd17581a1352401af626a3e56ee3d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 00:50:14 2016 -0400
+
+    rbd-mirror: configuration options to control replay throttling
+    
+    Fixes: http://tracker.ceph.com/issues/16223
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24883e0605907d1f9bcd1206c8a95c3bde30d5dc)
+
+commit 549aada7f90b9f7d4a4447d2794c18d5f61807a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 08:11:53 2016 -0400
+
+    librbd: new configuration option to restrict journal payload size
+    
+    Ensure that, by default, IO journal events are broken up into manageable
+    sizes when factoring in that an rbd-mirror daemon might be replaying
+    events from thousands of images.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 11d7500b9bcda7b7c1d8756ade3373f404257f32)
+
+commit 99195e89e4d229922c6e88aed6a024619e939040
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 00:42:16 2016 -0400
+
+    librbd: wait for journal commit op event to be safely recorded
+    
+    Operation request op finish events should not be fire and forget.
+    Instead, ensure the event is committed to the journal before
+    completing the op. This will avoid several possible split-brain
+    events during mirroring.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47e0fbf231e52d00069c97b72c57c3158445bcf0)
+    
+    Conflicts:
+            src/test/librbd/operation/test_mock_ResizeRequest.cc: no shrink restriction
+
+commit 3ae52ebadb5ef0de63ba30f937dcbcad507a7048
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 16:17:41 2016 -0400
+
+    journal: optimize speed of live replay journal pruning
+    
+    When streaming playback, avoid the unnecessary watch delay when
+    one or more entries have been pruned.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 08a8ee98c03b5dfb30341c8d209f0c231b2c5d27)
+
+commit 3850ded99728d1d6acfcaa72cf3923e791dd8fed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 10:04:21 2016 -0400
+
+    journal: possible deadlock during flush of journal entries
+    
+    If a future flush is requested at the exact same moment that an
+    overflow is detected, the two threads will deadlock since locks
+    are not taken in a consistent order.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c65471de4b0f54b8ed722f5deaf51ba62632e37)
+
+commit b4b984325f67a617c890ce6ccfbea5f42322cec5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 09:15:26 2016 -0400
+
+    journal: improve debug log messages
+    
+    rbd-mirror debugging involved potentially thousands of journals
+    concurrently running. The instance address will correlate log
+    messages between journals.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 11475f4fe740cccdfea459ebeabdca8cb94dc911)
+
+commit 196de276d5f1102a1cb4c7600252c9f1e783acc6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 15:34:53 2016 -0400
+
+    journal: support streaming entry playback
+    
+    Now that it's possible for the ObjectPlayer to only read a
+    partial subset of available entries, the JournalPlayer needs
+    to detect that more entries might be available.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 28d5ca16cbcb445f985469413b2a8a3048ab66b7)
+
+commit b08335d052ba04765690397f70d0d7f9b54898ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 15:15:58 2016 -0400
+
+    journal: replay should only read from a single object set
+    
+    Previously it was prefetching up to 2 object sets worth of journal
+    data objects which consumed too much memory.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2666d366645b22a5db2a2bcbfce466726bf0b3c0)
+
+commit c6f5303fb2d12830d0c9ac8605884b0c89d16539
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 08:06:13 2016 -0400
+
+    journal: optionally restrict maximum entry payload size
+    
+    Journal playback will need to read at least a full entry which was
+    currently limited to the maximum object size. In memory constrained
+    environment, this new optional limit will set a fix upper bound on
+    memory usage.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c1877b82fee0db1dba76252b32ff348226d41a7)
+
+commit 6dc609f24e54c8363a5acecef982aa67beb6130e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 11:01:26 2016 -0400
+
+    journal: optionally fetch entries in small chunks during replay
+    
+    Support fetching the full object or incremental chunks (with a
+    minimum of at least a single decoded entry if available).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f7362e9a57e484fffd840ca0eef01778dcacb65b)
+
+commit 91c70f34edabfb3983da643478d746568ed6b738
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 09:31:40 2016 -0400
+
+    journal: helper class for organizing optional settings
+    
+    Additional runtime configuration settings will be needed. The
+    new class will avoid the need to expand the constructor.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dad8328f2d502d18923c35f7b86a0cc2ccec133a)
+
+commit d68fe79eedc475c11858c9da5cb3fd2b66ed2b2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 11 15:32:45 2016 -0400
+
+    rbd-mirror: preprocess journal events prior to applying
+    
+    Fixes: http://tracker.ceph.com/issues/16622
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4df913d10b2dd0444db806fccb2812547efa1b56)
+
+commit 01f5f3f15c0da44588b4644905953e234a551def
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 16:19:52 2016 -0400
+
+    rbd-mirror: event preprocessor to handle snap rename operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fdfca557370c9d86acb81d50edb6aafc42044747)
+
+commit f71dc87e509ada03b40b2bec5ea82579709e0d2d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 15:16:04 2016 -0400
+
+    librbd: improve journaling debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 270cb74bc276bfb1f828a6a6933fa827f6cdce42)
+    
+    Conflicts:
+            src/librbd/journal/Replay.cc: no snap limit restriction
+
+commit 14db53587aa9a918174b616f006c38d1b4c99e58
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 14:37:14 2016 -0400
+
+    librbd: separate journal event decoding and processing
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57cd75e8058b84b5dce38f3d8f4b7b4138ac6c9a)
+
+commit 0c4a73fb422ea78e4c5db0763866206cbb54f120
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 09:14:58 2016 -0400
+
+    librbd: record original snap name in snap rename journal event
+    
+    Remote peers need a key to map snapshot ids between clusters.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f70b90c48d9520bbb4bb29058375e8205cf63771)
+
+commit 1a25490367343d7d4083961163c62f1c32cac105
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 09:13:07 2016 -0400
+
+    librbd: simple duplicate op checks for all maintenance operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 77699bfe749bc7a898024638fb8347c53fe12123)
+    
+    Conflicts:
+            src/test/librbd/mock/MockOperations.h: no shrink restriction
+
+commit 45a0b74a3eb50395a8e5ebb0f2a87add5090ddc1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 7 22:16:51 2016 -0400
+
+    qa/workunits/rbd: exercise snapshot renames within rbd-mirror test
+    
+    Snapshot rename operations utilize the (cluster) unique snapshot
+    sequence to prevent attempts at replays. When mirroring to a
+    different cluster, these sequences will not align.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f4cb26d8bbd1457bc261547103b56ad40b3c464)
+
+commit c7ab24e7ed63dd1754753c7b0e5c8981a8454a0b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 15 11:31:14 2016 +0300
+
+    librbd: re-register watch on old format image rename
+    
+    The watching object name is changed when renaming an old format image,
+    so unregister the watcher before the rename, and register back after,
+    to avoid "Transport endpoint is not connected" error.
+    
+    Fixes: http://tracker.ceph.com/issues/16321
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1a3973c8814200dc449a557a4fc8212911633985)
+
+commit 46fdba458750a13d217708ae3209b0f8bd058072
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 20:28:33 2016 -0400
+
+    rbd-mirror: gracefully restart pool replayer when blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/16349
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f55aa5e33b2fe242ebb9702ba9ff6f8d5cef96c)
+
+commit a806cdbb057e6bf14322ec6c3da8670ea2e9cfdc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 20:28:02 2016 -0400
+
+    rbd-mirror: do not clear image replayer error code after shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fc27545c2092c178293ed636b6aadb11bc8cbd3)
+
+commit b88a851d96b224d8332aa76a61d17a659dd5d14e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 16:57:50 2016 -0400
+
+    rbd-mirror: image deleter should use provided librados connection
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03c2aec4c613f11fcfed5bcd47855005f42abd79)
+
+commit 4b05677581ce9910e356e84203000251c92d8405
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 16:03:03 2016 -0400
+
+    rbd-mirror: each pool replayer should use its own librados connection
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 48f301decbf1f27937bb77a3b47e54933f272d7d)
+
+commit 569fbee99d1367bc0d51e96e42dcf64eeb930648
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:05:33 2016 -0400
+
+    rbd-mirror: clean up ImageReplayer before stopping state machine
+    
+    Fixes: http://tracker.ceph.com/issues/16489
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c97f7245a015dbdda25584134840d023fd65cdd1)
+
+commit eb706abd814fbd5cc704f21f3f9839ff09aaea6e
+Merge: 3167918 fcc00f7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 19:17:56 2016 +0200
+
+    Merge pull request #10679 from dillaman/wip-16735-jewel
+    
+    jewel: rbd-nbd does not properly handle resize notifications
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3167918679b519ae7d2625ea08b96419111c9992
+Merge: e6f9f28 2afc176
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 19:10:04 2016 +0200
+
+    Merge pull request #10148 from SUSE/wip-16599-jewel
+    
+    jewel: rgw: Swift API returns double space usage and objects of account metadata
+    
+    Reviewed-by: Pritha Srivastava <prsrivas@redhat.com>
+
+commit 55460e2e7b0f1298796b37b38a64820a8f23f6e2
+Author: Shilpa Jagannath <smanjara@redhat.com>
+Date:   Sat Aug 6 14:25:48 2016 +0530
+
+    rgw: raise log levels for common radosgw-admin errors
+    
+    Currently while running radosgw-admin command we see these messages thrown at
+    the endusers since debug level is set to 0:
+    
+    2016-08-02 14:48:28.687251 7f9e630639c0  0 Cannot find zone
+    id=7a2a89b9-4f5e-4f60-a29e-451ac9acc5a8 (name=us-west), switching to local
+    zonegroup configuration
+    2016-08-02 14:48:35.742185 7f9e630639c0  0 error read_lastest_epoch
+    .rgw.root:periods.a3db7884-2445-4bab-a165-6730f8573b8f:staging.latest_epoch
+    
+    Raising the log level to 1.
+    
+    Fixes: http://tracker.ceph.com/issues/16935
+    Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
+    (cherry picked from commit 8ff2b8e71abd34648c8eefb6b158ae5de46bd709)
+
+commit 73cc60862891cba8581370e3d5726a9f75ba1651
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Aug 5 11:28:13 2016 -0400
+
+    rgw: fixes for period puller
+    
+    * reset epoch=0 before RGWPeriod::init() so we get the latest_epoch
+    * if we're the metadata master, don't try to pull periods from ourself
+    
+    Fixes: http://tracker.ceph.com/issues/16939
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 06c384f1c5aa5b4b6404a3caf8a5f88ee40b0b03)
+
+commit f29864302648f38050626086bb87d47448a72aaf
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:18:23 2016 +0200
+
+    rgw: ONLY improve code formatting in rgw_object_expirer_core.{cc,h}.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit bc1ecdfb42bf9c9c5825af035409359536ed5b1e)
+
+commit e7be5a544a712142742323d573586dc9e98cbd77
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:11:04 2016 +0200
+
+    rgw: fix skipping some objects to delete by RadosGW's object expirer.
+    
+    Wei Qiaomiao has found that expired objects, which should be removed by
+    the object expirer of RadosGW, might be left unprocessed till next
+    restart of a RadosGW's instance. This happens when process_single_shard
+    method of RGWObjectExpirer class exhaust a time slot for a single round
+    and finishes without informing caller about the situation.
+    
+    Fixes: http://tracker.ceph.com/issues/16705
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 99f7d6eb565cee116c662305411df2adef002cae)
+
+commit af7e1e0dde953a56f33197c80da499e89f3dfc93
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:05:37 2016 +0200
+
+    rgw: fix trimming object expirer's hints.
+    
+    Fixes: http://tracker.ceph.com/issues/16684
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 83cd7852ed1fd8350d7c95a1b8811ffd34d7789f)
+
+commit dc154cd1e497b02386b91fde28a658d79ec6659e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jul 25 11:12:14 2016 +0800
+
+    osd: fix the mem leak of RepGather
+    
+    ReplicatedPG::new_repop() returns a pointer to RepGather with two refcounts,
+    one is held by ReplicatedPG::repop_queue, the other is supposed to be
+    held by the caller of this function. but it's caller
+    ReplicatedPG::submit_log_entries() assigns it to a
+    boost::intrusive_ptr<RepGather>() directly, why by default add_ref() in
+    its constructor. this makes the refcount 3. that's why we have a leak of
+    RepGather in `ReplicatedPG::new_repop(ObcLockManager&&,
+    boost::optional<std::function<void ()>>&&)`.
+    
+    Fixes: http://tracker.ceph.com/issues/16801
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d3a28465fc7b7002f38cff364fdf601f7486add3)
+
+commit e6f9f287e226e44035920827be2a45753f7452d1
+Merge: 84e3810 90d29fc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:41:12 2016 +0200
+
+    Merge pull request #10678 from dillaman/wip-16701-jewel
+    
+    jewel: rbd-mirror: include local pool id in resync throttle unique key
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 84e38109472b4a2a169eb8286c6632da4b1edb6e
+Merge: c2defb0 0ff0960
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:39:02 2016 +0200
+
+    Merge pull request #10614 from SUSE/wip-16959-jewel
+    
+    jewel: rpm: OBS needs ExclusiveArch
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c2defb0cea4ba9eed88af3f234c248c57ba48f4f
+Merge: 6d7ebb1 461782e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:38:27 2016 +0200
+
+    Merge pull request #10519 from Abhishekvrshny/wip-16312-jewel
+    
+    jewel: selinux denials in RGW
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6d7ebb10e6b20f7c3656fd763d96e64ac247e59f
+Merge: 338a58c b26acc0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:37:51 2016 +0200
+
+    Merge pull request #10364 from SUSE/wip-16750-jewel
+    
+    jewel: ceph-osd-prestart.sh contains Upstart-specific code
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 338a58ca2ed9ba27f9d66c22e725322ce15466af
+Merge: c7a1c81 c1a47c7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:36:47 2016 +0200
+
+    Merge pull request #10008 from dachary/wip-16099-jewel
+    
+    jewel: ceph-base requires parted
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c7a1c812c4aab478ee38db60947e6f686e8a7407
+Merge: 2d5ada4 ad3b788
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:35:54 2016 +0200
+
+    Merge pull request #9917 from SUSE/wip-16461-jewel
+    
+    jewel: ceph Resource Agent does not work with systemd
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 96ad2d16df6ec43cd730a4c21b866524a188c532
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 2 04:28:30 2016 -0700
+
+    rgw: can set negative max_buckets on RGWUserInfo
+    
+    This can be used to disable user's bucket creation
+    
+    Fixes: http://tracker.ceph.com/issues/14534
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 626d795139a6c5104be098780b70500c2de9b8f4)
+
+commit 2d5ada4ef8ee7d6803a903b7395e0c2b19b781ea
+Merge: aa03e1d 00f3fd4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 16:39:22 2016 +0200
+
+    Merge pull request #10303 from ukernel/jewel-16655
+    
+    jewel: ceph-fuse: link to libtcmalloc or jemalloc
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit aa03e1dbe6b243c56ed40eee8734051f434c2ed2
+Merge: 0280773 3da251f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 16:21:01 2016 +0200
+
+    Merge pull request #10420 from SUSE/wip-16798-jewel
+    
+    jewel: ceph command line tool chokes on ceph –w (the dash is unicode 'en dash' &ndash, copy-paste to reproduce)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dbf8cf0b5d46cc49eaf0a14bcc05a4fb215d98fb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri May 13 18:23:12 2016 +0200
+
+    rgw: improve support for Swift's object versioning.
+    
+    This patch allows RadosGW to pass the RefStack with an accuracy
+    to the RFC7230 violation issue which is clearly a Tempest bug.
+    
+    Fixes: http://tracker.ceph.com/issues/15925
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 237ad1210f5a6e4f191293ce532ca15869612a93)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc
+              In contrast to master, Jewel doesn't support
+              container quota of Swift API. All tracks of this
+              feature have been eradicated from the patch.
+            src/rgw/rgw_rest_swift.cc
+              Jewel doesn't include boost/optional.hpp but has
+              boost/utility/in_place_factory.hpp. The conflict
+              has been resolved to pull in both headers.
+
+commit 02807732b0f66826c2d110fe06ece15751924760
+Merge: 1e04a35 67eb961
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 10:40:05 2016 +0200
+
+    Merge pull request #10520 from Abhishekvrshny/wip-16319-jewel
+    
+    jewel: radosgw-admin: inconsistency in uid/email handling
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1e04a357a6e9e67c81b5104f57ab61e5ce07a5e3
+Merge: f6e927c 3f76e4a
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:18:36 2016 -0400
+
+    Merge pull request #10073 from SUSE/wip-16565-jewel
+    
+    jewel: rgw: data sync stops after getting error in all data log sync shards
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f6e927cfe3e296ebd5e3b22c3f05d717f87cd4ed
+Merge: d2e8692 93d7875
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:18:06 2016 -0400
+
+    Merge pull request #10524 from Abhishekvrshny/wip-16731-jewel
+    
+    jewel: failed to create bucket after upgrade from hammer to jewel
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d2e869255869ae68ca419d5f0a3bbfbcf3a91710
+Merge: 87e71aa dc96383
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:16:41 2016 -0400
+
+    Merge pull request #10523 from Abhishekvrshny/wip-16700-jewel
+    
+    jewel: rgw: segmentation fault on error_repo in data sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 87e71aae7a2d73b7b5444e1a5ab283ece570e3fb
+Merge: cc60230 fe57ace
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:16:06 2016 -0400
+
+    Merge pull request #10518 from Abhishekvrshny/wip-16272-jewel
+    
+    jewel: rgw ldap: fix ldap bindpw parsing
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cc60230eef82173eafa52df3d2015c8c0473d236
+Merge: 31264f4 489f8ce
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:59:51 2016 +0200
+
+    Merge pull request #10144 from dillaman/wip-fix-missing-return-jewel
+    
+    jewel: librbd: fix missing return statement if failed to get mirror image state
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 31264f4126f79c26cff73b2a4d2bea77426c5cd9
+Merge: 4ee9117 c6546bb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:55:42 2016 +0200
+
+    Merge pull request #10552 from ceph/wip-16507-jewel
+    
+    jewel: expose buffer const_iterator symbols
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4ee9117d52d41a72444722afe81ce2074dbf58d3
+Merge: 5b7899e 3d6d36a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:53:40 2016 +0200
+
+    Merge pull request #10497 from Abhishekvrshny/wip-16586-jewel
+    
+    jewel: partprobe intermittent issues during ceph-disk prepare
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5b7899e89154eceb53625cd6f7ae33984298857f
+Merge: c65536c bd70d6d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 15 08:35:57 2016 -0700
+
+    Merge pull request #10716 from ceph/wip-16975
+    
+    qa: add rados test script for upgrades
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c65536cf4f3433fea2cc4683da930676f2def223
+Merge: 40e3e2e 5ffdc34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 08:28:32 2016 -0400
+
+    Merge pull request #10732 from dillaman/wip-rbd-default-format-jewel
+    
+    doc: format 2 now is the default image format
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5ffdc344401e91c4db89f9ed017adb15e535ebb4
+Author: Chengwei Yang <yangchengwei@qiyi.com>
+Date:   Fri Aug 12 17:44:16 2016 +0800
+
+    doc: format 2 now is the default image format
+    
+    Signed-off-by: Chengwei Yang <yangchengwei@qiyi.com>
+    (cherry picked from commit c8c92f8a2c2106d11b18c70c4c183c3724c2f3c2)
+
+commit 40e3e2e075f21c2e4d36219747d6fdba530d5ed8
+Merge: f58ca19 2c39d36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 11:14:58 2016 +0200
+
+    Merge pull request #10217 from SUSE/wip-16636-jewel
+    
+    jewel: rgw: document multi tenancy
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f58ca19f36179a64e50e2d14555b1efbb8fbfa79
+Merge: 40fc75e 546141c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 15 10:02:12 2016 +0200
+
+    Merge pull request #9453 from wido/jewel-issue-15348
+    
+    jewel: rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 40fc75ef4fafada6aa3fac47963da060f50646b0
+Merge: ec14cf5 3ff6e8f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 12 17:41:07 2016 +0200
+
+    Merge pull request #10693 from dreamhost/wip-15975-jewel
+    
+    rgw: Fallback to Host header for bucket name.
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit bd70d6ddba1bc237a6ffafa0dadea16b2c65a39c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 11 20:39:20 2016 -0700
+
+    qa: remove tmap_migrate tests from upgrade testing
+    
+    tmap_upgrade only exists in jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 49db733bad375645214ddb8f9b95b8817f9b41e1
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 11 16:05:04 2016 -0700
+
+    qa: add rados test script for upgrades
+    
+    Disable master tests that are not meant to work on jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3ff6e8fa852346f4b69cd8c10b2f14ae4983d527
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu May 26 15:41:20 2016 -0700
+
+    rgw: Fallback to Host header for bucket name.
+    
+    RGW should fallback to using the Host header as the bucket name if valid &
+    possible even when it is NOT a suffix match against the DNS names, or a match
+    against the CNAME rule.
+    
+    This mirrors AWS S3 behavior for these cases (The AWS S3 does not do any DNS
+    lookups regardless).
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15975
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 46aae19eeb91bf3ac78a94c9d4812a788d9042a8)
+
+commit 90d29fc94a826b1f56ff99e5b5d81a735abbc4cd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 11 21:58:45 2016 -0400
+
+    rbd-mirror: include local pool id in resync throttle unique key
+    
+    Fixes: http://tracker.ceph.com/issues/16536
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ad36cab7cbf3492bfa972b43e4a5f75a110bfe6)
+
+commit 03c76148ef9e10fad649a407374f4fc855529377
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 15:18:51 2016 -0400
+
+    test: ensure unique template specializations per test case
+    
+    With optimizations disabled, there can be cross translation unit
+    symbol leaking that is not present when inlining is enabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 995a16ebcbe79fd14e6de3b25087373744238294)
+
+commit ea9e031e2144ee32b3776cfe2237681f7065861b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 14:59:17 2016 -0400
+
+    test: missing template specialization for unittest_rbd_mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 187e2c87b50d64e93736a42fcf449c55e8550125)
+
+commit c2e6d08469ea733dac984770c30d8ab6356f3eca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 17:49:54 2016 -0400
+
+    qa/workunits/rbd: increase writes in large image count test
+    
+    This will help to test edge cases where the remote image does
+    or does not own the exclusive lock when the sync starts.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d16698f96c65b9627cc1f64ef80b7f1e39d69d45)
+
+commit 88a7c5c1b28c6a077d35577787757b0a98b7b2a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 17:42:59 2016 -0400
+
+    rbd-mirror: prune sync points referencing missing snapshots
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 378914f6b8a8a8c5517fee8fa186268078fbcbfb)
+
+commit 9a6bad3bd7eaa733042c0a23aaf48be4fa6f84ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 16:34:44 2016 -0400
+
+    rbd-mirror: ignore empty snapshot sequence mapping
+    
+    This invalid condition will be gracefully detected and handled
+    when the snapshot mappings are computed.
+    
+    Fixes: http://tracker.ceph.com/issues/16329
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 58ed8a18e91401333bc3f3f957ce5d715b6687b6)
+
+commit c8089dea98fd40651664ae80b0e16834bf727540
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 13:01:13 2016 -0400
+
+    qa/workunits/rbd: remove temporary image exports
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45498d04990861a034e2aad69a93658e018c991a)
+
+commit bdb2189c94badb6e389a3a05794ad8af3eceda1c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 11:13:29 2016 -0400
+
+    rbd-mirror: fix potential image replayer state transition race
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e224c6c03a6a5de0e179bd788387cfa3ff49e9d)
+
+commit e7d7990958bdd8add7cfdcf037bcda5dd05c59d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 11:06:57 2016 -0400
+
+    rbd-mirror: cancel image deletion callback on shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c3f1cb3a34e1b4d258877d519e683e25bf65c65a)
+
+commit bba7811489170e01471ef7c086066a006f57fb6b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 10:58:09 2016 -0400
+
+    rbd-mirror: fixed potential leaking image deletion context callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9d1cd90c220f95d23d2694b4f1fb6b2fed073ced)
+
+commit b71efb090708e30225502d0cbf974b880def1eb8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:45:25 2016 -0400
+
+    rbd-mirror: ImageReplayer doesn't need image deleted callback
+    
+    The Replayer will detect that the ImageReplayer is stopped and will
+    schedule the "wait for deleted" callback before attempting to
+    restart the ImageReplayer.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7e5afc71dfd3ec3e7b0ddaca96f92ac6e0414006)
+
+commit b657d1849a37bc4461edb18337078041a5460818
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:21:05 2016 -0400
+
+    rbd-mirror: use async callback when deletion not in-progress
+    
+    Fixes: http://tracker.ceph.com/issues/16491
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c136f443366fe804057a9c1f8c01c3f0a33a4d63)
+
+commit c9c1216b3c06e49d3f0addeb7a4f6f5796c00d30
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Jun 9 10:40:23 2016 +0100
+
+    rbd-mirror: tests: ImageSyncThrottler unit tests
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 8ca9a84f36efa73ad17fd27f545dff3716cd798c)
+
+commit c56d6ec4c1898e710aad307b5d3696b9b159ba0c
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Jun 8 16:38:01 2016 +0100
+
+    rbd-mirror: Usage of image-sync throttler in BootstrapRequest
+    
+    Fixes: http://tracker.ceph.com/issues/15239
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 6a91146255d84229688bd8b378732be5975a778b)
+
+commit 5323ebd40a96484f5341574ed8a783c2e696f5dc
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Jun 8 16:37:20 2016 +0100
+
+    rbd-mirror: Implementation of image-sync throttler
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c4f926d6980d1efd95771885a11d8cc4ebd2e4c3)
+
+commit 783bd686c451ae816ea9c7001b3c6a0ef1b5f554
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jun 14 16:43:19 2016 +0100
+
+    rbd-mirror: tests: Support for inflight image sync point update
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit cee543bd96bc99f219024aedbfafc5dcd99abb5a)
+
+commit b878f5ffc75e70b1f09eee461f93b1cb5031586e
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Jun 2 10:04:41 2016 +0100
+
+    rbd-mirror: image-sync: Periodically update sync point object number
+    
+    Fixes: http://tracker.ceph.com/issues/15108
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c2eedf4d27b73d7eaf3fda54f9b0ee74e455bc1d)
+
+commit 2ba61a73b1c616fab5acb1b92fe0a0b7ce7b64cb
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Jun 27 11:07:41 2016 +0100
+
+    rbd-mirror: image-replayer: Fix bug in resync listener remotion
+    
+    Fixes: http://tracker.ceph.com/issues/16488
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 11a5851e90bca2c1813156bf12dda0192965db37)
+
+commit cbd1ab461d5d75baf6e8387788740a6bcb3520de
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 19 12:06:08 2016 +0100
+
+    rbd-mirror: resync: Added unit tests
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 8953825956f2a560d17c7b651d1724a95f2f7d1d)
+
+commit 5fe93a4383521b7b67d7f168e4f20735440bb8ca
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:12:33 2016 +0100
+
+    rbd-mirror: image-replayer: Implementation of resync operation
+    
+    Fixes: http://tracker.ceph.com/issues/15670
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit f2b114238103315197a9eefc8490786c484c8520)
+
+commit c2b786ca92728ef6ceeb836133598a837cd18a2c
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:10:38 2016 +0100
+
+    rbd: journal: Support for listening updates on client metadata
+    
+    Currently we only support listening for image resync requests.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 0dd85739e9034912b86250ced2834dd7fb3d92fd)
+
+commit 21d2ba5eefd1ef904360ce09b4335df77c952d95
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:09:09 2016 +0100
+
+    journal: Support for registering metadata listeners in the Journaler
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c535eb0399cd649371db233ee1ed01cf5441a4b3)
+
+commit fcc00f7ac2c9ed517d12ef9ebab735bdd0d1a162
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jun 27 10:53:09 2016 +0300
+
+    test: fsx: fix rbd-nbd daemon logging
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 09710c46d3811e602250f858fe7292925c1e3c06)
+
+commit 63fbed0f39da22374f6818030674c6b5e73f41dd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Jun 3 21:11:21 2016 +0300
+
+    qa/workunits/rbd: image resize test for rbd-nbd
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fa58acbb6fcc79e5a9bfd4b86250a0e2b9c8efa1)
+
+commit 3db1a3c8ea5a6829e1e44ecd0c96316c759bf048
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 24 15:52:00 2016 +0300
+
+    rbd-nbd: use librbd API method to watch image size update
+    
+    Fixes: http://tracker.ceph.com/issues/15715
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1e1d3b1609432ea80d6fd1c5b3df49f38ad579f1)
+
+commit 8f94f5e3f7ba0cb3578952ffbca85c0dc6a772a0
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 24 15:50:22 2016 +0300
+
+    librbd: API: methods to watch image stat update
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9952b75a0c8c9bed9792cb77671fd5588f32630f)
+    
+    Conflicts:
+            src/include/rbd/librbd.h: removed CG references
+            src/tracing/librbd.tp: removed CG references
+
+commit ec14cf5ce3e8d04eee3f13aadcfc2d7f22c5a633
+Merge: c587e9e 393bf7e
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 11:31:40 2016 +0200
+
+    Merge pull request #9790 from SUSE/wip-16381-jewel
+    
+    jewel: comparing return code to ERR_NOT_MODIFIED in rgw_rest_s3.cc (needs minus sign)
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 22c9df1d9bef3f48ab2a12357035fc54fbde141d
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Fri Jul 1 14:15:42 2016 +0530
+
+    rgw: Fix for using port 443 with pre-signed urls.
+    
+    Fixes http://tracker.ceph.com/issues/16548
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 44b9ed37ac659d83fbd4bb99c69da6b9dec06b8d)
+
+commit ce5e250405449c6035b805bbcdcca73ad4399d68
+Author: Shilpa Jagannath <smanjara@redhat.com>
+Date:   Fri Jun 17 18:29:42 2016 +0530
+
+    rgw: added zone rename to radosgw_admin
+    
+    Added zone rename option to radosgw-admin help
+    
+    Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
+    (cherry picked from commit da3235234eaf41f9dbd378fb0d9994d2c4096947)
+
+commit ce986aa6e8b3ee54697802a361246c256e8097b4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 14 13:38:44 2016 -0400
+
+    rgw: RGWMetaSyncCR holds refs to stacks for wakeup
+    
+    because RGWCoroutine::wakeup() calls RGWCoroutinesStack::wakeup(), the
+    stack must also stay alive
+    
+    Fixes: http://tracker.ceph.com/issues/16666
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e4bc16044e1b80636855dbc39da1d121a3508308)
+
+commit 2485efc26d68b6848cda02d66b54dd58769fcbd8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 26 12:50:13 2016 -0400
+
+    rgw: clear realm watch on failed watch_restart
+    
+    if the realm is deleted while the gateway has a watch, it's disconnected
+    and watch_restart() will fail. this results in a watch handle of 0,
+    which leads to a segfault on ~RGWRealmWatcher when we pass it to
+    unwatch()
+    
+    this commit cleans up the watch when watch_restart() fails, so we don't
+    try to unwatch() on destruction
+    
+    Fixes: http://tracker.ceph.com/issues/16817
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e5b8b5b27502fdacdac38effd7a55fd719911e85)
+
+commit fa2e42d4614619f761a5cf4c86e3db7d7e101e11
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 27 12:16:57 2016 -0400
+
+    rgw: use endpoints from master zone instead of zonegroup
+    
+    Fixes: http://tracker.ceph.com/issues/16834
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d4872ec9f7554ff49f4be336fe59cdd2051ee2a5)
+
+commit a865f26e68a991eaf55dc5f105b7fd7cb4634bc3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 28 09:55:05 2016 -0400
+
+    radosgw-admin: zone[group] modify can change realm id
+    
+    allows the default zone and zonegroup (created with empty realm_id) to
+    be later added to a realm. the 'modify' command now accepts either
+    --realm_id=id or --rgw-realm=name
+    
+    Fixes: http://tracker.ceph.com/issues/16839
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 064b7e953dd6ace2c32b94150e70959e95a01761)
+
+commit c587e9e5c73cd8a990106708a43222fcb5a2a709
+Merge: aacb793 3250c4d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 10 13:33:12 2016 +0200
+
+    Merge pull request #10026 from SUSE/wip-16392-jewel
+    
+    jewel: master: build failures with boost > 1.58
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f5db5a4b0bb52fed544f277c28ab5088d1c3fc79
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue May 17 17:53:56 2016 +0100
+
+    mon: tolerate missing osd metadata
+    
+    Just because one OSD's metadata is missing,
+    don't give up on outputting all the other
+    OSDs' metadata.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2c7dd5f22288c4aee814573baa6af131421f36a4)
+
+commit 36e5c86469eda59f7d5d15b16e15d0aab81a34fa
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue May 17 14:03:09 2016 +0100
+
+    mon: fix metadata dumps for empty lists
+    
+    Because 'r' was set inside loop, these would
+    fail to set it (leaving ENOSYS) if no
+    metadata is present.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fe4fe4d92f5b2d67896beac9b5b3b48b309a6b4)
+    
+    Conflicts:
+            src/mon/Monitor.cc and src/mon/MDSMonitor.cc
+            changes relate to features that are not backported to jewel
+
+commit de99bd53f5efe84647e1a1d79a8db330887b7704
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun May 15 10:32:28 2016 +0800
+
+    mon: 'std::move` Metadata when updating it
+    
+    to avoid copying of the std::map
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dd67eaab00698d747bf8669a3065c72ea68f7784)
+
+commit 828df3995430ce89fe9be40c6c19c44fb6f14752
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 12 15:49:23 2016 +0100
+
+    mon: fix 'mon metadata' for lone monitors
+    
+    Previously, writing to the store was only
+    triggered when MMonMetadata was received
+    from peers, so if you had a single mon then
+    you would always get ENOENT from "mon metadata"
+    
+    Fixes: http://tracker.ceph.com/issues/15866
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 93ab00c77059e93b52ac4caa703e259944be1d1e)
+
+commit 308f5143a9e9b121fb5c4df0a227ba30bda4636e
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Mon Jul 4 13:52:03 2016 +0800
+
+    ceph.spec.in: fix rpm package building error as follows:
+    
+          error: Installed (but unpackaged) file(s) found:
+          /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 49dfad1f53d7abe0e9efc7817aaaf1735df5a9c9)
+
+commit 727a704e79995c8a74f2b5768c63e90eecf9a883
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 23 09:23:09 2016 +0200
+
+    udev: always populate /dev/disk/by-parttypeuuid
+    
+    ceph-disk activate-all walks /dev/disk/by-parttypeuuid at boot time. It
+    is not necessary when udev fires ADD event for each partition and
+    95-ceph-osd.rules gets a chance to activate a ceph disk or journal.
+    
+    There are various reasons why udev ADD events may not be fired at
+    boot (for instance Debian Jessi 8.5 never does it and CentOS 7.2 seems
+    to be racy in that regard when a LVM root is being used).
+    
+    Populating /dev/disk/by-parttypeuuid fixes ceph-disk activate-all that
+    would not work without it. And it guarantees disks are activated at boot
+    time regardless of wether udev fires ADD events at the right time (or at
+    all).
+    
+    The new udev file is a partial resurection of the
+    60-ceph-partuuid-workaround-rules that was removed by
+    9f77244b8e0782921663e52005b725cca58a8753. It is given a name that
+    reflects its new purpose.
+    
+    Fixes http://tracker.ceph.com/issues/16351
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35004a628b2969d8b2f1c02155bb235165a1d809)
+
+commit 9fc5b41598f09414572e89572e8980fd3d3c76e0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 14:03:01 2016 -0400
+
+    librbd: optimize away unnecessary object map updates
+    
+    Fixes: http://tracker.ceph.com/issues/16689
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5b4188635c4ee1ee0c4353cfc5ecd6e887d536b)
+
+commit 81a2534742e5051fa92549029303cef3f3d187a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 13:37:37 2016 -0400
+
+    rbd-replay: decode and replay discard IO operations
+    
+    Fixes: http://tracker.ceph.com/issues/16707
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b7a4db213d0d4812aa6cd9c54e9646ff57d10411)
+
+commit 5968664fd4ddf8e26989360ebbc048faa8431251
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 30 15:00:32 2016 -0400
+
+    librbd: failed assertion after shrinking a clone image twice
+    
+    Fixes: http://tracker.ceph.com/issues/16561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 34d2297eed040f694b4d1c5e2606af04752c9e1a)
+
+commit 8316b6a6bad28343f8bade63f05b5fb93f4fe415
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 30 10:24:01 2016 -0400
+
+    rbd-mirror: gracefully fail if object map is unavailable
+    
+    If the exclusive lock was lost due to a watch failure from an
+    overloaded cluster, gracefully abort the image sync.
+    
+    Fixes: http://tracker.ceph.com/issues/16558
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6f573ca2d63784e4a2b1e7a410044afdf58f6801)
+
+commit aacb793948a69294e518f8458be6e670df23fe53
+Merge: 954e978 5ae0e43
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 17:09:42 2016 +0200
+
+    Merge pull request #10625 from dachary/wip-16969-jewel
+    
+    jewel: src/script/subman fails with KeyError: 'nband'
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5ae0e43e7f75a3a6d96d9fc5f03c4b0a86c7481b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Aug 9 09:13:17 2016 +0200
+
+    subman: use replace instead of format
+    
+    Otherwise all {} are assumed to be substituted. Add a test.
+    
+    Fixes: http://tracker.ceph.com/issues/16961
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit b006c136761746e12704e349a672325b8b6e51f5)
+
+commit 0ca27727173501856e9f17785b9f555684661709
+Author: Ali Maredia <amaredia@redhat.com>
+Date:   Thu Apr 21 20:32:47 2016 -0400
+
+    cmake: script that sets env vars for unit tests
+    
+    Signed-off-by: Ali Maredia <amaredia@redhat.com>
+    (cherry picked from commit 15a669435aaa78562746f6728bb185b1e99a4274)
+
+commit 954e978bc8cd233ac149a2e490d8730d9782583a
+Merge: d387bf4 cbc9636
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:41:30 2016 +0200
+
+    Merge pull request #10054 from dachary/wip-16484-jewel
+    
+    jewel: ExclusiveLock object leaked when switching to snapshot
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d387bf44d405a3a8658697134e6bfc9ae077dfef
+Merge: 14a4484 cf65ed9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:28:44 2016 +0200
+
+    Merge pull request #10041 from dachary/wip-16315-jewel
+    
+    jewel: When journaling is enabled, a flush request shouldn't flush the cache
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 14a448466e9bda6b8d565f34b248a0ab6f9f90b3
+Merge: fc39bd8 757babb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:50 2016 +0200
+
+    Merge pull request #10055 from dachary/wip-16485-jewel
+    
+    jewel: Whitelist EBUSY error from snap unprotect for journal replay
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit fc39bd804e479fbaefa6f81befd899b8a1a82581
+Merge: fc084a3 d9c3f28
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:22 2016 +0200
+
+    Merge pull request #10053 from dachary/wip-16483-jewel
+    
+    jewel: Close journal and object map before flagging exclusive lock as released
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit fc084a32ecf7c1868cbbbd808b273f2d095746f0
+Merge: 543dd04 b1d9698
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:08 2016 +0200
+
+    Merge pull request #10052 from dachary/wip-16482-jewel
+    
+    jewel: Timeout sending mirroring notification shouldn't result in failure
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 543dd0460228f778ca10e8ff238c8db3f1b239fe
+Merge: 8fb4121 51defea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:50 2016 +0200
+
+    Merge pull request #10051 from dachary/wip-16460-jewel
+    
+    jewel: Crash when utilizing advisory locking API functions
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 8fb4121c53850f0f60eadb6e2a3bfc8c750e5935
+Merge: 64d5ff9 db28ddc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:40 2016 +0200
+
+    Merge pull request #10050 from dachary/wip-16459-jewel
+    
+    jewel: rbd-mirror should disable proxied maintenance ops for non-primary image
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 64d5ff97ced5cc0cfa4f15b66b54cc2fb11b2c22
+Merge: 4fe02a6 70bf746
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:21 2016 +0200
+
+    Merge pull request #10047 from dachary/wip-16426-jewel
+    
+    jewel: Possible race condition during journal transition from replay to ready
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 4fe02a62b7ad2c6b4a32962741a73d8b1db9fb11
+Merge: b86aec9 f3f4a4a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:11 2016 +0200
+
+    Merge pull request #10046 from dachary/wip-16425-jewel
+    
+    jewel: rbd-mirror: potential race condition accessing local image journal
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit b86aec99e8375eb2de1d495338ab212e1880a0f7
+Merge: 4a6e48e 0399958
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:00 2016 +0200
+
+    Merge pull request #10045 from dachary/wip-16424-jewel
+    
+    jewel: Journal needs to handle duplicate maintenance op tids
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 4a6e48ed85914413b919e4da5a67ee4f95f34ae2
+Merge: adcf5a7 6a28d63
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:48 2016 +0200
+
+    Merge pull request #10044 from dachary/wip-16423-jewel
+    
+    jewel: Journal duplicate op detection can cause lockdep error
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit adcf5a7fbf181505c6319c4399572cd3e9259a47
+Merge: eab8994 1e85da9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:35 2016 +0200
+
+    Merge pull request #10043 from dachary/wip-16371-jewel
+    
+    jewel: rbd-mirror: ensure replay status formatter has completed before stopping replay
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit eab89942c53ef5bff15cd837dea8ba42a228dc8f
+Merge: 289c2d2 bf58eab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:11 2016 +0200
+
+    Merge pull request #10042 from dachary/wip-16372-jewel
+    
+    jewel: Unable to disable journaling feature if in unexpected mirror state
+
+commit 289c2d2a512723e039bc1465353dbb39084edb80
+Merge: e01ebb9 dd635e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:24:55 2016 +0200
+
+    Merge pull request #10010 from dachary/wip-16486-jewel
+    
+    jewel: Object map/fast-diff invalidated if journal replays the same snap remove event
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e01ebb9319d17fd71b83bd0ccf9cb0f231a1783e
+Merge: 28575db db7ce96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:24:38 2016 +0200
+
+    Merge pull request #10009 from dachary/wip-16514-jewel
+    
+    jewel: Image removal doesn't necessarily clean up all rbd_mirroring entries
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 0ff096073f4c26bbccd9125a345372f090754d63
+Author: Michel Normand <normand@linux.vnet.ibm.com>
+Date:   Fri Aug 5 16:34:48 2016 +0200
+
+    ExclusiveArch for suse_version
+    
+    for SLES supports only x86_64 and aarch64 targets
+    for openSUSE (Tumbleweed and Leap) add ppc64/ppc64le targets.
+    
+    fixes: http://tracker.ceph.com/issues/16936
+    Signed-off-by: Michel Normand <normand@linux.vnet.ibm.com>
+    (cherry picked from commit a8db0f4d96d71223083bf6c2275acd892666a13e)
+
+commit 28575db3fb1579cdfa85b14b0484363cc0634a2e
+Merge: 3e36f8c 1f19dbd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 8 08:47:34 2016 +0200
+
+    Merge pull request #9952 from dillaman/wip-fix-use-after-free-jewel
+    
+    jewel: librbd: potential use after free on refresh error
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3e36f8c66326d3de97a79859eab363e509dfc8c2
+Merge: 5c98730 6b41d76
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 8 08:45:58 2016 +0200
+
+    Merge pull request #9752 from dillaman/wip-fix-task-finisher-jewel
+    
+    jewel: librbd: cancel all tasks should wait until finisher is done
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5c98730854f11b0efb3b3e03be426ce2b7a999af
+Merge: f014619 7e1ba28
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Aug 5 12:46:10 2016 -0700
+
+    Merge pull request #10561 from dzafman/wip-16672-jewel
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f0146196ccfbcfd923191f63d93e4e81219523b1
+Merge: e400999 c025010
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:30:52 2016 +0200
+
+    Merge pull request #9562 from Abhishekvrshny/wip-16152-jewel
+    
+    jewel: client: fstat cap release
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e400999a2cb0972919e35dd8510f8d85f48ceace
+Merge: 0cb5ca6 c3f6d82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:30:41 2016 +0200
+
+    Merge pull request #9561 from Abhishekvrshny/wip-16136-jewel
+    
+    jewel: MDSMonitor fixes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0cb5ca6db018461c6275636f0d9cc1b1c8f56e53
+Merge: 67f2eb0 19c1366
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:33 2016 +0200
+
+    Merge pull request #9557 from Abhishekvrshny/wip-15898-jewel
+    
+    jewel: Confusing MDS log message when shut down with stalled journaler reads
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 67f2eb01deff699016c16a312448d665cbef3922
+Merge: 7ec15f6 eea0e91
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:23 2016 +0200
+
+    Merge pull request #9560 from Abhishekvrshny/wip-16135-jewel
+    
+    jewel: MDS: fix getattr starve setattr
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 7ec15f69f83e78ac862515c1068a64769caaff91
+Merge: e51060a 2c7fc95
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:08 2016 +0200
+
+    Merge pull request #9559 from Abhishekvrshny/wip-16041-jewel
+    
+    jewel: mds/StrayManager.cc: 520: FAILED assert(dnl->is_primary())
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e51060ad8190bddf2b9b367c4d0a67e8cce7daef
+Merge: fda98f7 edd3f79
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:17:22 2016 +0200
+
+    Merge pull request #9996 from dachary/wip-16437-jewel
+    
+    jewel: async messenger mon crash
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit fda98f780134c26e491ce15fca0890dc240d67cf
+Merge: 56bbcb1 18fdc1c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:10:27 2016 +0200
+
+    Merge pull request #9997 from dachary/wip-16431-jewel
+    
+    jewel: librados,osd: bad flags can crash the osd
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 56bbcb1aa11a2beb951de396b0de9e3373d91c57
+Merge: 7586d69 a826bb8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:10:08 2016 +0200
+
+    Merge pull request #9998 from dachary/wip-16429-jewel
+    
+    jewel: OSDMonitor: drop pg temps from not the current primary
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7586d693e1c4f08a17e60f89fda5abcb31d2f80d
+Merge: 3b28428 6554d46
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:09:52 2016 +0200
+
+    Merge pull request #10001 from dachary/wip-16427-jewel
+    
+    jewel: prepare_pgtemp needs to only update up_thru if newer than the existing one
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3b28428f901c1a6ed08b6c23b4f9991b87fe0960
+Merge: 6996b76 762db30
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Aug 5 09:09:23 2016 -0400
+
+    Merge pull request #9743 from vumrao/wip-vumrao-16339
+    
+    jewel : rgw: support size suffixes for --max-size in radosgw-admin command
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 8b9954d0ccab9694c7519002209be8cb9dc03362
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Aug 4 17:15:18 2016 -0700
+
+    rgw: Fix civetweb IPv6
+    
+    Commit c38e3cbb6f7c6221209f2b512cba18c564c895a4 introduced a second
+    compiled version of src/civetweb/src/civetweb.c, but did not pass the
+    configuration header (civetweb/include/civetweb_conf.h).
+    
+    As a result, USE_IPV6 was not defined when it was compiled, and that
+    copy was included into the radosgw binary. This caused breakage for the
+    civetweb frontend when used with IPv6:
+      rgw frontends = civetweb port=[::]:7480
+    
+    Reintroduce the header so that civetweb is compiled correctly again.
+    
+    Fixes: http://tracker.ceph.com/issues/16928
+    Backport: jewel
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 921c556b65eafe2136f2afcc8234681711aa7348)
+
+commit 6996b76d35b20a558d13d35fafaaad47fe331434
+Merge: 8586e9b 74dd035
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:45:05 2016 +0200
+
+    Merge pull request #10007 from dachary/wip-15806-jewel
+    
+    jewel: New pools have bogus stuck inactive/unclean HEALTH_ERR messages until they are first active and clean
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 8586e9bd49d1fc083a7c37a2b1865744bbcce26a
+Merge: e85b58b a2e8ae6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:43:36 2016 +0200
+
+    Merge pull request #10006 from dachary/wip-16249-jewel
+    
+    jewel: sparse_read on ec pool should return extends with correct offset
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e85b58bbe6aa217e4f9c5cef052aabedf728b9fe
+Merge: f05eaae f8e4911
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:43:22 2016 +0200
+
+    Merge pull request #9740 from vumrao/wip-vumrao-16338
+    
+    jewel : rados: Add cleanup message with time to rados bench output
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f05eaaea3f39159cc5e1a127a7d9913f479a7bf4
+Merge: 093f30d b8f7aa2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:42:53 2016 +0200
+
+    Merge pull request #10004 from dachary/wip-16374-jewel
+    
+    jewel: AsyncConnection::lockmsg/async lockdep cycle: AsyncMessenger::lock, MDSDaemon::mds_lock, AsyncConnection::lock
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 093f30d1788b4a7225e3ab1c3345db14e40a5237
+Merge: 648442b 3dbb08a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:42:32 2016 +0200
+
+    Merge pull request #10003 from dachary/wip-16380-jewel
+    
+    jewel: msg/async: connection race hang
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 648442bfea4ea72c04fad441fc933d03d980f80c
+Merge: a35e2fa caad884
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 12:35:14 2016 +0200
+
+    Merge pull request #10036 from liewegas/wip-16297-jewel
+    
+    mon: Monitor: validate prefix on handle_command()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a35e2fa5f1a22ea5798c559196d50d1d38d7679e
+Merge: e499d1c 7b0318a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 09:53:10 2016 +0200
+
+    Merge pull request #9547 from jcsp/wip-jewel-15705
+    
+    jewel backport: mds: fix mdsmap print_summary with standby replays
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7e1ba28b1e218deb7d54ad6bef8853c9d19411a0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 22:32:02 2016 -0700
+
+    os: Fix HashIndex::recursive_remove() to remove everything but original path
+    
+    Fixes: http://tracker.ceph.com/issues/16672
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ba88a3aac0d3b620986e32ed718237513d6592f6)
+
+commit cb1f17f05422ac802378df0de698ce20a1f2976e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 23:26:41 2016 -0700
+
+    filestore: Clear objects before calling collection_list() again
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 892303cd02b64b622669183c2acffcdd7a8f3547)
+
+commit bdcfcaf821add71c518b01d6fb3dccb7c2e074a7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 23:24:12 2016 -0700
+
+    filestore: Improve logging
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b78c457785553e3a6e4f901bfebb41fc9c4d9bd5)
+
+commit e499d1c8fb216b490a739f2712b3c733623d45d1
+Merge: 269d742 8981f3b
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Aug 2 10:28:00 2016 -0400
+
+    Merge pull request #10293 from theanalyst/wip-16589-jewel
+    
+    jewel: multisite sync races with deletes
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 269d7427dfa9de7b1a288da0532505810bdf283b
+Merge: 894a5f8 e1eb8af
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Aug 2 12:31:18 2016 +0100
+
+    Merge pull request #10453 from ajarr/wip-manila-backports-jewel
+    
+    jewel: essential backports for OpenStack Manila
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e1eb8afea9f202947eef33e8361a0aac0e955eea
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jul 7 17:15:13 2016 +0530
+
+    ceph_volume_client: version on-disk metadata
+    
+    Version on-disk metadata with two attributes,
+    'compat version', the minimum CephFSVolume Client
+    version that can decode the metadata, and
+    'version', the version that encoded the metadata.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 1c1d65a45f4574ca5f33cc9d949089a5c956e363)
+
+commit e8dd1eeec7ee2fd11a2985843825a3ec0f522b0c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jul 6 15:33:06 2016 +0530
+
+    ceph_volume_client: add versioning
+    
+    Add class attributes to CephFSVolumeClient to version
+    its capabilities.
+    
+    'version' attribute stores the current version number
+    of CephFSVolumeClient.
+    
+    'compat_version' attribute stores the earliest version
+    number of CephFSVolumeClient that the current version is
+    compatible with.
+    
+    Fixes: http://tracker.ceph.com/issues/15406
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 46876fb2ceb22082c0a1703fe77ad1694b508ad8)
+
+commit a0ffc859f5763850342a165ec4c49b3b1e83bb95
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 23:22:12 2016 +0530
+
+    ceph_volume_client: disallow tenants to share auth IDs
+    
+    Restrict an auth ID to a single OpenStack tenant to enforce
+    strong tenant isolation of shares.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 82445a20a258a4c8800f273dc5f2484aace0e413)
+
+commit d3e22db9841404d6c305012032427d6bb5311253
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 17:11:33 2016 +0530
+
+    ceph_volume_client: cleanup auth meta files
+    
+    Remove auth meta files on last rule for an auth ID deletion
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit ec2e6e37d01c961d269d83661d0b95ada6a8449e)
+
+commit 2e9c37adfb7b9f7e67801ffc2e1fa6c85674c9c1
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:55:30 2016 +0530
+
+    ceph_volume_client: fix log messages
+    
+    Log the path of the volume during creation and deletion of volumes.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7731287761f91e1fdee0e6306d9ecf9b04ad363c)
+
+commit 7e93d3d0e631f375a496673174667934d5ecc08e
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:31:23 2016 +0530
+
+    ceph_volume_client: create/delete VMeta for create/delete volume
+    
+    Create and delete volume meta files during creation and deletion of
+    volumes.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 37fbfc7aa8f10d37f5202603a906425507271ff5)
+
+commit 7cea0eee455c83fbdd64fff192c6aa10235a2c40
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:06:53 2016 +0530
+
+    ceph_volume_client: modify locking of meta files
+    
+    File locks are applied on meta files before updating the meta
+    file contents. These meta files would need to be cleaned up
+    sometime, which could lead to locks being held on unlinked meta
+    files. Prevent this by checking whether the file had been deleted
+    after lock was acquired on it.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f7c037229bcf3f5a3d06897ec7fe0c5419dd7143)
+
+commit e77684fc68cfd3dfb33a80e7c5604356921a40b0
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 17:39:32 2016 +0530
+
+    cephfs.pyx: implement python bindings for fstat
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f58403f3d19e22edeb8f91b6f87a0b7947b0ff21)
+
+commit d2792ef1babd452d76528ce5ca511106d8c836b4
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 16:57:01 2016 +0530
+
+    ceph_volume_client: restrict volume group names
+    
+    Prevent craftily-named volume groups from colliding with meta files.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7f7d2a76ae9b556c1de418f0eab8461c538f91d9)
+
+commit 4f874dd9a9c4ce58f623c635c31bb67012a72199
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 16:34:56 2016 +0530
+
+    ceph_volume_client: use fsync instead of syncfs
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 27eb51baab4cda6b385aef53fc7c3962a9debae5)
+
+commit bde2c8f111d8276b026a3b1e37a5c9081a498ec9
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Fri Apr 29 14:47:42 2016 -0500
+
+    pybind: expose fsync in cephfs binding.
+    
+    So we don't necessary to syncfs when want to persistent
+    some file.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 1c952fbaf0fd393ef2dcb83a3db721a077b4274e)
+
+commit 0c13bf2d028eab4cd062209be7ed985cdf28960c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 00:42:18 2016 +0530
+
+    ceph_volume_client: recover from dirty auth and auth meta updates
+    
+    Check dirty flag after locking something and call recover() if we are
+    opening something dirty (racing with another instance of the driver
+    restarting after failure) -- only required if someone running multiple
+    manila-share instances with Ceph loaded.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 647a2447f0c4354dc21d1083043591d2b6f6f94f)
+
+commit 8f7defb5d084f8e781fa44e9c21edf6c472cfb4a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Jun 21 12:14:56 2016 +0530
+
+    ceph_volume_client: modify data layout in meta files
+    
+    Notable changes to data layout in auth meta and volume meta files:
+    
+    In the auth meta files, add a 'dirty' flag to track the status of auth
+    updates to a single volume.
+    
+    In the volume meta file, make the 'dirty' flag track the status of
+    auth updates for a single ID.
+    
+    Optimize the recovery of partial auth update changes to auth meta,
+    volume meta, and the Ceph backend, facilitated by changes in the
+    data layout in the meta files.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7c8a28a7e817d030a4d419b0cc627c30c1064270)
+
+commit 748a5a08ecc1e212e4bd10e0c8eff0a805fc931f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 2 12:30:45 2016 +0000
+
+    pybind: ceph_volume_client authentication metadata
+    
+    Store a two-way mapping between auth IDs and volumes.
+    
+    Enables us to record some metadata on auth ids (which
+    openstack tenant created it) so that we can avoid exposing
+    keys to other tenants who try to use the same ceph
+    auth id.
+    
+    Enables us to expose the list of which auth ids have access
+    to a volume, so that Manila's update_access() can be
+    implemented efficiently.
+    
+    DNM: see TODOs inline.
+    
+    Fixes: http://tracker.ceph.com/issues/15615
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d2e9eb55ca6ed5daa094cf323faf143615b9380b)
+
+commit e14dc25e1fca7fd74971cbda74d1fcadff5be3b4
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 7 13:06:41 2016 +0000
+
+    pybind: enable integer flags to libcephfs open
+    
+    The 'rw+' style flags are handy and convenient, but
+    they don't capture all possibilities.  Change to
+    optionally accept an integer here for advance users
+    who want to specify arbitrary combinations of
+    flags.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5678584f4176d07301acd7f57acc4efd7fb20e43)
+
+commit c6546bb29b25fb96f270257b959426f949eb9418
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Wed Mar 16 14:12:05 2016 -0700
+
+    buffer: fix iterator_impl visibility through typedef
+    
+    The following program doesn't compile because of symbol visibility issues.
+    While bufferlist::iterator is a class implementation with visibility specified,
+    it is unclear after google-fu how to do the same through typedef.
+    
+    int main()
+    {
+     ceph::bufferlist bl;
+     ceph::bufferlist::const_iterator it = bl.begin();
+     (void)it;
+     return 0;
+    }
+    
+    [nwatkins@bender ~]$ g++ -Wall -std=c++11 -Iinstall/include -Linstall/lib -o test test.cc -lrados
+    /tmp/cciR9MUj.o: In function `main':
+    test.cc:(.text+0x43): undefined reference to `ceph::buffer::list::iterator_impl<true>::iterator_impl(ceph::buffer::list::iterator const&)'
+    /usr/bin/ld: test: hidden symbol `_ZN4ceph6buffer4list13iterator_implILb1EEC1ERKNS1_8iteratorE' isn't defined
+    /usr/bin/ld: final link failed: Bad value
+    collect2: error: ld returned 1 exit status
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit 16bc3e59325d4057d36cd63a57111ab36fbc50fd)
+
+commit e825dd2f3fc9048995062bfe96e32e6df7ef37d9
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jul 22 10:57:45 2016 +0200
+
+    rgw multisite: preserve zone's extra pool
+    
+    In current multisite scenarios,if a bucket is created in master, we end
+    up storing multipart metadata in `$source-zone.rgw.buckets.non-ec` pool
+    instead of the zone's own non-ec pool, so we end up additionally
+    creating this pool and storing multipart metadata entries in it. Also if
+    a bucket is created in a secondary zone, and we initiate a multipart
+    upload, before mdlog sync with master, we end up getting errors during
+    complete multipart requests as omap entries are partly stored in the
+    `$zone.rgw.buckets.non-ec` as well as `$source-zone.rgw.buckets.non-ec`
+    pools which leads to total number of parts mismatch.
+    
+    Fixes: http://tracker.ceph.com/issues/16712
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit d4ecc956f603a567bd4043c2be39b3e7e3402c75)
+
+commit 059ed62ff937ce63b7255d72bf9b3cf687c4b532
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 4 15:01:51 2016 +0200
+
+    rgw: remove bucket index objects when deleting the bucket
+    
+    Fixes: http://tracker.ceph.com/issues/16412
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3ae276390641ad5fc4fef0c03971db95948880b4)
+
+commit 93d787557939d77c6a15854520a42f3d0f358370
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Jul 8 10:41:59 2016 +0200
+
+    rgw: add missing master_zone when running with old default region config
+    
+    Fixes: http://tracker.ceph.com/issues/16627
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit a0420741d446341a4979d78db7e2e58f396fdc4b)
+
+commit dc963833af2d81984c7fd023f718f5b7e58a37dc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 6 09:15:29 2016 -0400
+
+    rgw: fix error_repo segfault in data sync
+    
+    RGWDataSyncShardCR will only allocate an error_repo if it's doing
+    incremental sync, so RGWDataSyncSingleEntryCR needs to guard against a
+    null error_repo
+    
+    also, RGWDataSyncShardCR::stop_spawned_services() was dropping the last
+    reference to the error_repo before calling drain_all(), which meant that
+    RGWDataSyncSingleEntryCR could still be holding a pointer. now uses a
+    boost::intrusive_ptr in RGWDataSyncSingleEntryCR to account for its
+    reference
+    
+    Fixes: http://tracker.ceph.com/issues/16603
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 28609029cf1be2fc9f8c8e3f47320636db29014a)
+
+commit 67eb961927664cbded6b01c5f76bfd9db8481f8c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 16:06:15 2016 -0500
+
+    rgw: add line space between inl. member function defns (rgw_user.h)
+    
+    This is a cosmetic change only.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5a7f7f5560e57f8dc64a3c84c1e668be834d81e0)
+
+commit 3f8298a049e255f542a012688b4148456d465d8d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:58:07 2016 -0500
+
+    rgw-admin: return error on email address conflict (add user)
+    
+    Fixes the error return inconsistency documented in upstream
+    Ceph issue #13598
+    
+    Revised after upstream review, permits changing email address as
+    originally (but use case-insensitive comp).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 90c143efa2156e55dc017ebe81005315cacaf2c5)
+
+commit 962e7dc47dcaa64f4774a197ae21d0ee59f7854f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:06:19 2016 -0500
+
+    rgw-admin: convert user email addresses to lower case
+    
+    Fixes the email address inconsistency documented in upstream
+    Ceph BUG #13598
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4c438dbbc0e6eda6b9a3018d60019a1a780d6f65)
+
+commit 461782ee6b755b14f6ae90ea56ebcb656e305b94
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 13 12:34:39 2016 +0200
+
+    selinux: allow chown for self and setattr for /var/run/ceph
+    
+    Fixes: http://tracker.ceph.com/issues/16126
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 2a6c738abda35f540af6f9398406d4e49337c34d)
+
+commit fe57aceeb02ad9163feb2d196589b5927cedfa0f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jun 6 16:19:17 2016 -0400
+
+    rgw ldap: fix ldap bindpw parsing
+    
+    Also add additional LDAP debugging output at 0, 10, and 15 to make
+    troubleshooting easier.
+    
+    Fixes DN search issue using QE configuration of MS AD.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 72b2f3e8dcb879be3dac7ac03142fd4a13ff28ac)
+
+commit 429e9c03f3f47e3804f54cac380db0add18584f2
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jun 14 11:04:23 2016 +0200
+
+    rgw: aws4: fix buffer sharing issue with chunked uploads
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit dd9f53115a452e1e4268a1036f8fb69f9c1b86f7)
+
+commit 7c2eab19e2c088e97de924f7f1a633a9f291439d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jul 1 21:06:17 2016 -0400
+
+    mds: move Finisher to unlocked shutdown
+    
+    This commit resolves a deadlock reported in i16042 where the thread calling
+    MDSRankDispatcher::shutdown would hold the mds_lock while asynchronous
+    callbacks in the Finisher would attempt to lock mds_lock.
+    
+    For simplicity, I merged the finisher stop with the messenger shutdown as both
+    need the mds_lock dropped.
+    
+    Fixes: http://tracker.ceph.com/issues/16042
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b1d6689b6cae579d5f1b2047fd3c653cda422454)
+    
+    Conflicts:
+            src/mds/MDSRank.cc
+                    trivial conflict in MDSRankDispatcher::shutdown()
+
+commit 28697fc5101395be2a1bd778445983e61a2dc089
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Mon Jun 27 12:30:11 2016 -0700
+
+    mds: Kill C_SaferCond in evict_sessions()
+    
+    MDSRankDispatcher::evict_sessions waits on a C_SaferCond for
+    kill_session to complete on each of its victims. Change the
+    command handling flow to pass command messages all the way down
+    to MDSRankDispatcher. Extract the MDSDaemon's reply path into a
+    static function callable from a new context in the MDSRankDispatcher.
+    
+    See: http://tracker.ceph.com/issues/16288
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 3a4d63ee67765010a8e53af5a89aef4f49fafd56)
+
+commit 1d3a816814c6cb87f645f205362de800cc84d484
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Jun 27 13:00:26 2016 +0800
+
+    mds: fix shutting down mds timed-out due to deadlock
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit ca069149654ecd10778cd5327bf59b9643e23967)
+
+commit 49a1ce936d76b4bbf7a3b46be300ad5ebb6d301e
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Jun 27 12:59:08 2016 +0800
+
+    msg/async: remove the unnecessary checking to wakup event_wait
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 560dc32771be00664c4b22a0ebca5190cd246e0a)
+
+commit 2beb56a10a0319b6d061caf1da769bb3c9f5f90b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 2 17:03:05 2016 +0800
+
+    mds: fix SnapRealm::have_past_parents_open()
+    
+    the '!' got delete accidentally in commit f7fb2cb52c (mds: fix open
+    snap parents tracking)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f28f72635c7d11f0db4f156a6d108a480295056d)
+
+commit 3d6d36a12bd4823352dc58e2135d03f261d18dbe
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 26 09:38:47 2016 +0200
+
+    ceph-disk: partprobe should block udev induced BLKRRPART
+    
+    Wrap partprobe with flock to stop udev from issuing BLKRRPART because
+    this is racy and frequently fails with a message like:
+    
+        Error: Error informing the kernel about modifications to partition
+        /dev/vdc1 -- Device or resource busy.  This means Linux won't know about
+        any changes you made to /dev/vdc1 until you reboot -- so you shouldn't
+        mount it or use it in any way before rebooting.
+    
+    Opening a device (/dev/vdc for instance) in write mode indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up)
+    when the device is closed (see below for the udev release note).
+    
+    However, if udev fails to acquire an exclusive lock (with
+    flock(fd, LOCK_EX|LOCK_NB); ) the BLKRRPART ioctl is not issued.
+    
+    https://github.com/systemd/systemd/blob/045e00cf16c47bc516c0823d059b7548f3ce9c7c/src/udev/udevd.c#L1042
+    
+    Acquiring an exclusive lock before running the process that opens the
+    device in write mode is therefore an effective way to control this
+    behavior.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+      * As an experimental feature, udev now tries to lock the
+           disk device node (flock(LOCK_SH|LOCK_NB)) while it
+           executes events for the disk or any of its partitions.
+           Applications like partitioning programs can lock the
+           disk device node (flock(LOCK_EX)) and claim temporary
+           device ownership that way; udev will entirely skip all event
+           handling for this disk and its partitions. If the disk
+           was opened for writing, the close will trigger a partition
+           table rescan in udev's "watch" facility, and if needed
+           synthesize "change" events for the disk and all its partitions.
+           This is now unconditionally enabled, and if it turns out to
+           cause major problems, we might turn it on only for specific
+           devices, or might need to disable it entirely. Device Mapper
+           devices are excluded from this logic.
+    
+    Fixes: http://tracker.ceph.com/issues/15176
+    
+    Signed-off-by: Marius Vollmer <marius.vollmer@redhat com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8519481b72365701d01ee58a0ef57ad1bea2c66c)
+
+commit 8cd7f44d8066a796cccd872dd2593582ac061331
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri Jun 3 17:34:10 2016 +0200
+
+    rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support
+    
+    When authenticating requests using the Authorization header in AWS4, you have
+    the option of uploading the payload in chunks. You can send data in fixed size
+    or variable size chunks.
+    
+    This patch enables streaming mode and signed headers support with chunked
+    uploads.
+    
+    Fixes: http://tracker.ceph.com/issues/16146
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 5de5876a535537f7878615898bb9cf7887204cb1)
+    
+    Conflicts:
+            src/rgw/rgw_rest_s3.cc
+    
+    No change required to resolve the conflict. Manual merge was enough.
+
+commit 276ec72e85d6650782aef2c96f4b1fbc47918f42
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 11 17:34:33 2016 +0100
+
+    rgw: use std::unique_ptr for rgw_aws4_auth management.
+    
+    This is purely for code maintainability.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 6f273eb52c07b27c7aa2d69be3e5ff5d6578558c)
+
+commit 2c422e306989f65d9112f95a131ff29479b2d0f0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 11 17:33:47 2016 +0100
+
+    rgw: add handling of memory allocation failure in AWS4 auth.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ddbb0ab9d5046672795ec876221de9ebd466364f)
+
+commit 2cd3ed8a59786be28b55a0983a438af1aab226ea
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Apr 27 23:26:44 2016 +0530
+
+    ceph_volume_client: allow read-only authorization for volumes
+    
+    Allow clients to be restricted to read-only mount of the volume
+    by restricting their ceph auth ID's MDS and OSD caps to read-only.
+    
+    Fixes: http://tracker.ceph.com/issues/15614
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 011ea5e7fb35ee07848e0c3abac24702a778ad63)
+
+commit 46246e39b8cadc14aede930559eddd09059301fc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 6 19:41:54 2016 +0800
+
+    osd: increment stas on recovery pull also
+    
+    PGMap::recovery_rate_summary() summaries the recovery progress from
+    a pool's
+    pool_stat_t.stats.sum.num_{objects_recovered,bytes_recovered,keys_recovered},
+    now we only increment stats on completion of recovery push, but there
+    are chances that a PG recovers by pulling data from replicas to primary.
+    in that case, teuthology will erroneously consider recovery hung: a zero
+    recovering_keys_per_sec or recovering_bytes_per_sec or
+    recovering_objects_per_sec. so we should increment stats on pull as
+    well.
+    
+    Fixes: http://tracker.ceph.com/issues/16277
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9c6c4d8fef67380e2a6d93f6c101744c926d8b9b)
+
+commit 3da251fe7f3bafa9a936df209b1906824764e5f6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 22 15:55:16 2016 +0800
+
+    pybind/ceph_argparse: handle non ascii unicode args
+    
+    we raise UnicodeDecodeError at seeing non-ascii args if we fail to match
+    it with any command signatures. instead, we should use a unicode string
+    for representing the error in that case. please note, the exception is
+    not printed at all in real-world. =)
+    
+    Fixes: http://tracker.ceph.com/issues/12287
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5864626d275c423cdb8d3e52a91fe4cc6b8e6f90)
+
+commit b01af21ceeb83e3b2fb7cee608f2a9229e6eed7e
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Wed Jun 15 00:50:20 2016 +0300
+
+    Fix tabs->whitespace in ceph_argparse
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit bef2c83c1da7dcc32fc2ff98cf214fc93bef75b8)
+
+commit bb2dc9537612b56a79f1237618126faecbeccc9e
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Wed Jun 15 00:54:08 2016 +0300
+
+    Make usage of builtins in ceph_argparse compatible with Python 3
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit ddf06041ef80ac9606da8d57f048d2e23233d122)
+
+commit f80d10c82034fcd9d53b3890afe62a4dbfeb1799
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 20 10:23:48 2016 +0200
+
+    rpm: move libatomic_ops-devel to non-distro-specific section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 94e4f2190b6ab3ff7b57b0b6095fea6d9228d917)
+
+commit a28810c69b356d1865d1ce9e989bcc9e8a66017a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 20 10:22:37 2016 +0200
+
+    rpm: move gperftools-devel to non-distro-specific section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 06940e29b702c7437dc9f8df365fcc383cb882e1)
+
+commit e6b7a4ba8a1a8ea13d6f7178bd69432a3705754b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 9 22:55:54 2016 +0200
+
+    rpm: use new name of libatomic_ops-devel
+    
+    The previous form, libatomic-ops-devel, has been deprecated since May 23, 2012.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c835f6ddf5839c418d29db1c2f46da32fbf05d2c)
+
+commit 9bbf2e8d874201460c3cf1dfd14cfd17752d397d
+Author: Dan Horák <dan@danny.cz>
+Date:   Fri May 6 13:29:03 2016 +0200
+
+    fix tcmalloc handling in spec file
+    
+    - there is no gperftools/tcmalloc on s390(x) yet
+    - explicitly disable tcmalloc when built without
+    
+    Signed-off-by: Dan Horák <dan@danny.cz>
+    (cherry picked from commit efa7f7b365d27797573bf4e5a9878f94f41aede2)
+
+commit 894a5f8d878d4b267f80b90a4bffce157f2b4ba7
+Merge: b15cf42 2538b77
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jul 21 13:58:16 2016 +0100
+
+    Merge pull request #10373 from ceph/jewel-mds-snap-failover
+    
+    Jewel mds snap failover
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b26acc03ab42ba7eacb804ef8dc08c03a6e64e2d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 13 10:27:18 2016 +0200
+
+    ceph-osd-prestart.sh: drop Upstart-specific code
+    
+    Before this patch, the ceph-osd-prestart.sh script would do Upstart-specific
+    things if the journal device failed to appear after a 5-second timeout.
+    
+    Fixes: http://tracker.ceph.com/issues/15984
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 47a06e608b7f686252d42c5a879373d8ebb65d9d)
+
+commit 1e622a5d3b79a89a6affb56f282e1305c48ec835
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jul 19 15:40:36 2016 +0100
+
+    rpm: Fix creation of mount.ceph symbolic link for SUSE distros
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 976c9d4f31793712b2af334e4a7a58745af8c099)
+
+commit 89cb116ca14abf5764edf07373d045bb127ca7eb
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 18 19:09:54 2016 +0200
+
+    build/ops: build mount.ceph and mount.fuse.ceph as client binaries
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b2675ff82445cda9873292ac2be8186e054098ed)
+
+commit 84b45b75c47e6234aac9d3ee1cf1c6844b898ae9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 18 16:16:11 2016 +0200
+
+    rpm: move mount.ceph from ceph-base to ceph-common
+    
+    Ceph clients use mount.ceph to mount CephFS filesystems, and
+    ceph-base is not expected to be installed on client systems.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b8c24bf2f8c6a6b125778ca92d68d8e64ccf9ae0)
+
+commit e028cd140840b15a8f99962dcf8ec646da3f1f60
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 6 10:13:03 2016 +0200
+
+    rpm: create mount.ceph symlink in /sbin (SUSE only)
+    
+    Fixes: http://tracker.ceph.com/issues/16598
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b090e9da326a72068260682599c886c2d11f06b3)
+
+commit ad67d1c15f49233fe1108258d9b968c8b2301330
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 2 02:08:45 2016 +0800
+
+    makefile: install mount.fuse.ceph,mount.ceph into /usr/sbin
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 52021ff9f0b23c36a012f19c7388dcfb2281b523)
+    
+    Conflicts:
+            src/CMakeLists.txt (Install mount.ceph to ${CMAKE_INSTALL_SBINDIR}
+            instead of sbin; install mount.fuse.ceph to ${CMAKE_INSTALL_SBINDIR} as
+            well)
+
+commit 2538b77a2f219a76c19289ab3e675927f6914149
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 15:32:41 2016 +0800
+
+    client: fix MetaRequest::set_other_inode()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 265f96bda7e106883063f074133450ce39ee262c)
+
+commit ffcfe69ef7abaaa3c663de86fb04ea986dc90b15
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 11:25:37 2016 +0800
+
+    client: close directory's snapdir when deleting directory
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 3099cabd11a16d22c6707631861bef0aa176ed02)
+
+commit b9007025f3dd49532a45267d7478dcdc5308ce3a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 11:07:01 2016 +0800
+
+    client: invalidate snap inodes after removing snapshot
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f180ad149ab510626fcd6cbd8221f550f9858126)
+
+commit d9f957a5cda55adcdba5db28689aeb344d10e413
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 10:51:13 2016 +0800
+
+    mds: fix incorrect "unconnected snaprealm xxx" warning
+    
+    If a snaprealm has no child/parent snaprelam, and the snaprealm inode
+    is not in the cache while client reconnects. The snaprealm does not
+    get properly removed from MDCache::reconnected_snaplrealm. This causes
+    incorrect "unconnected snaprealm xxx" warning
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 85e687de876ca7e0174734ba81130949c4ab6a40)
+
+commit 3e745ef3172aeef07038c9156d4668ca08078f18
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 30 16:05:57 2016 +0800
+
+    qa/workunits/fs: fix expect_failure function in test scripts
+    
+    The origin expect_failure function return 0 regardness of command's
+    return value.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 16f7d7c71e65cd68cecde3e5b89d189ab5950548)
+
+commit 37157d146f1983b94588135e4b0468a9c59c3ead
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 20:49:40 2016 +0800
+
+    client: make sure snapflush is sent before normal cap message
+    
+    MDS does null snapflush when it receives normal cap message. So client
+    must send snapflush first.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3916717e2edc8000400f678fa7134ca1406a074)
+
+commit 326d46b54280ff5612ee571671cc4956fcb7e8eb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 28 20:39:08 2016 +0800
+
+    client: unify cap flush and snapcap flush
+    
+    This patch includes following changes
+    - assign flush tid to snapcap flush
+    - remove session's flushing_capsnaps list. add inode with snapcap
+      flushes to session's flushing_caps list instead.
+    - when reconnecting to MDS, re-send one inode's snapcap flushes and
+      cap flushes at the same time.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a05e996b2a2c36496abd8538829ac4897392f6eb)
+
+commit 5c2ff04061cc686c8ece37cee3393365769d2bf1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 17:15:01 2016 +0800
+
+    mds: handle partly purged directory
+    
+    For a snapshoted direcotry whose snaprealm parents are being opened,
+    MDS does not know if the directory is purgeable. So MDS can't skip
+    committing dirfrags of the directory. But if the direcotry is purgeale,
+    some dirfrags could have already been deleted during MDS failover.
+    Committing them could return -ENOENT.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bc50e0309280c08c3ca79dfa5514ac3a15f81a23)
+
+commit 57b39f04087f3ba855248950723c949e3b7dcd7c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 11:42:42 2016 +0800
+
+    mds: do files recovery after processing cap flushes
+    
+    File recovery may update inode and trigger inode COW. MDS relies on
+    client caps to setup CInode::client_need_snapflush. But for a given
+    client, the reconnected caps may not include the flushing caps.
+    (Before MDS failover, client released and flushed some caps at the
+    same time. When MDS recovers, client re-send the cap flush and send
+    cap reconnect to the MDS.) This may cause later snapflush to get
+    dropped.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dd98448d3d0bbb7d338f03f7ad1e40f217eebe0d)
+
+commit bace1c8c7ce7d29676b9ed6925cdee41af8a8425
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 11:25:12 2016 +0800
+
+    mds: combine MDCache::{reconnected_caps,cap_imports_dirty}
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 57067e032e84bbdb69c499aa7ea93ca68082569b)
+
+commit 365bda18e3fe960be69d81882e11dcb9932a002c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jun 24 17:09:34 2016 +0800
+
+    mds: remove CEPH_LOCK_IFLOCKL from cinode_lock_info
+    
+    Currently we don't support dirty CEPH_CAP_FLOCK_EXCL
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit cfc3ec17b6f245e6d8a0be4fdf6cfa64d2fb725f)
+
+commit 55367c5cda09167778839eb8474f86903857f53a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 23 16:59:46 2016 +0800
+
+    mds: rebuild the internal states that tracking pending snapflush
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 1b7d198f633a8608b704f82f01a3f4a4a3a4892b)
+
+commit 0897fc4f15804a25a667bf7e495dc4ff1209127b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 22 20:34:41 2016 +0800
+
+    mds: using cap_reconnect_t to track cap recconect
+    
+    Previous commit extended cap_reconnect_t to include snapflush
+    related information. This information are needed in various
+    places
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 25b6011dcf07c1fc663cbe29ffd119b66545a0ac)
+
+commit ac508dc9202ebdb8f39bf1304bb459637cae1cb9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 22 15:56:29 2016 +0800
+
+    mds: add 'follows' of first pending snapflush to cap reconnect message
+    
+    This helps the recovering MDS to reconstruct the internal states that
+    tracking pending snapflush.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dd3963a878d4bb2fb5992278ccbc9f7633fb8786)
+
+commit c258f52afa68bf991b4d6c76d36271333c3e49bf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 21 17:17:56 2016 +0800
+
+    mds: journal snap inodes that need flush when expiring log segment
+    
+    Treat snap inodes that need flush in the same way as open files.
+    When MDS recovers, this make sure that journal replay bring snap
+    inodes that need flush into the cache
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 306153b3d012832bdfa20402077fa60a9a5d626c)
+
+commit 42dd72df1a514175be05d5d613d9b00296cb1b1e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 21 16:20:58 2016 +0800
+
+    mds: properly update client_snap_caps when splitting snap inode
+    
+    update the new snap inode's client_snap_caps according to the old
+    snap inode.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4883779935219817c7e391940a025be1679daeb5)
+
+commit b15cf42a4be7bb290e095cd5027d7f9ac604a97d
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Thu Jul 14 01:06:51 2016 +0300
+
+    install-deps: Get the latest virtualenv in a temporary folder
+    to work around a bug in old virtualenv
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit 2699b6d89b8c1c193fd86b5233d1ea86458753a0)
+
+commit 00f3fd4a39f35780e602acfa4023bf59031b3617
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jul 12 16:07:53 2016 +0800
+
+    ceph-fuse: link to libtcmalloc or jemalloc
+    
+    Fixes: http://tracker.ceph.com/issues/16655
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e644f6eb64037b1e21ce55d4dcffa43571ac1327)
+
+commit d794e295786599102d5ea1a4c10002f4f8d85649
+Merge: bb94997 7178f23
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 14 12:10:11 2016 -0400
+
+    Merge remote-tracking branch 'upstream/pull/10298/head' into jewel
+    
+    * upstream/pull/10298/head:
+      doc: fix standby replay config
+
+commit 7178f23c1bcb800bec2c7bec138ac02b22dbad2b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jul 12 15:43:23 2016 -0400
+
+    doc: fix standby replay config
+    
+    I tried using these settings in tests without success. The correct config names
+    are prefixed with "mds".
+    
+    Fixes: http://tracker.ceph.com/issues/16664
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8981f3beb5732f51197a5be84df18edde64217ca
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jun 21 15:09:53 2016 -0400
+
+    rgw: add pg_ver to tombstone_cache
+    
+    a tombstone cache was added to remember the mtime of deleted objects for
+    use with the HTTP_IF_MODIFIED_SINCE header, but the comparison was still
+    failing because of a missing pg_ver. added pg_ver to the tombstone cache
+    so it can be passed with HTTP_DEST_PG_VER
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit adb529f2fb983df671a1db58a0b17862a29762f0)
+
+commit 6c32fe8a75f535f21d3bec089bf06a82db5ec876
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 10 14:35:01 2016 -0700
+
+    rgw: add obj tombstone cache
+    
+    The obj tombstone cache is used in multi-zone environmet to keep
+    track of removed objects' mtime. This is then used to fetch remote
+    object only if its newer than the object that was removed, otherwise
+    we're just fetching ghost of the past.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb10214920c23b24edd94ca53d0f36c85404644d)
+
+commit f651a8f0cc2a11eb1a037338e35de3cf165f5ac2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 10 14:34:41 2016 -0700
+
+    rgw: rgw_obj comparison also accounts for bucket id
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a7f32c4b1bd951b1a18b7d13013dc8e2822d6ffc)
+
+commit 354e81c1800066e4e8f525706214c41d90816b9d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 14:30:42 2016 -0700
+
+    cls/rgw: reduce tag_timeout significantly
+    
+    The tag timeout is used for applying dir_suggest changes. Shorten it
+    so that changes will be reported early and can be used in data sync.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 99efdc673b7aed439eebdaa92ff117ba9646dd7c)
+
+commit 24d483119035a78973a1ee3827f5f7c8cb20ce5a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 14:22:07 2016 -0700
+
+    cls_rgw: dir_suggest entries bucket index logging
+    
+    Fixes: http://tracker.ceph.com/issues/16222
+    
+    Log entries that were set through the dir_suggest mechanism in
+    the bucket index log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ae00c5529219d81987425160dbb2a4e937661a6c)
+
+commit bdef8b23714eaff933992ed8c02fb35cd8d11818
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 12:59:35 2016 -0700
+
+    cls/rgw: fix timespan comparison
+    
+    Fixes: http://tracker.ceph.com/issues/16220
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e0a07f70f7740a66ffa2646e0a57e8bdc0285373)
+
+commit c34dbd4b19dcc35483306f95932b3ed6ed8332fd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Jun 4 05:47:50 2016 -0700
+
+    rgw: data sync squash, prioritize object removal
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2fcd8b1d49aae2fd03b986dd10bb3f98d3b8f32e)
+
+commit 09eee3be8fcd79ef46ecfbd277e8cc2bf4f28d93
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Jun 4 03:29:12 2016 -0700
+
+    rgw: squash bi complete bi log entries
+    
+    only process the newest complete operation in a bilog listing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f69db8e455f07c5594363c5beac329cb964be1ff)
+
+commit 571a13255aff4ca072ff88bb4ce54da086fbad86
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 3 02:03:13 2016 -0700
+
+    rgw: stop bucket lease only after draining object sync operations
+    
+    Fixes: http://tracker.ceph.com/issues/16143
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit be6ad9a76c3008ea25a737c0de718faab8fca611)
+
+commit 7af0306de5778340fc7c9395b237bf4e73716d0a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 12:57:19 2016 -0400
+
+    rgw: don't leak RGWRados::cr_registry on error
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 53a7f1a9951fa21cb46b5fb3914db3547b568aa5)
+
+commit 9591e505c1d2c7b66b4c17421f94b6fc7e68913f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 10:40:25 2016 -0400
+
+    rgw: dont need atomic RGWRados::next_rados_handle
+    
+    next_rados_handle is only accessed under an exclusive handle_lock
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1adff94b720d01240040fdffebdbf53efdd528a5)
+
+commit 047379c2521434f2dbbe67ebbf53e59ed92654f0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 10:23:26 2016 -0400
+
+    rgw: remove unneeded RGWRados::num_rados_handles
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 72d5a485e41ac5824c30556b6cfe659094cd303c)
+
+commit 7848482988711406c9cde48d828a0d118f764ad1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 3 10:35:41 2016 -0400
+
+    rgw: use vector for librados handles
+    
+    using a vector instead of an array of pointers cleans up our
+    initialization/shutdown logic
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 02fb39d7f5835ada4d6304f318203444dc7eedc9)
+
+commit bb94997bf4548a430b686a7ceb98052fdb051223
+Merge: 6ae4d13 a40cfe4
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Jul 12 10:09:34 2016 -0400
+
+    Merge pull request #9268 from theanalyst/wip-15992-jewel
+    
+    jewel: radosgw-admin: EEXIST messages for create operations
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6ae4d13e81d4b98e34d7094e939a8eab5820f608
+Merge: bf73c9a 21f0216
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Jul 10 23:03:16 2016 +0200
+
+    Merge pull request #8497 from sabaini/jewel
+    
+    ceph-disk: Accept bcache devices as data disks
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2c39d3652acbab77e8da50fa1b7ef537374ac109
+Author: Pete Zaitcev <zaitcev@kotori.zaitcev.us>
+Date:   Tue Jun 7 17:44:20 2016 -0600
+
+    rgw: Add documentation for the Multi-tenancy feature
+    
+    Signed-off-by: Pete Zaitcev <zaitcev@redhat.com>
+    (cherry picked from commit 0e622020fa2a185eaf0546d93a20b06d44e7f691)
+
+commit 9e5a3ae3cd35ccbd7a9f48c555fb93d5beee57cb
+Author: Feng Guo <diluga@gmail.com>
+Date:   Fri Jun 24 09:26:16 2016 +0800
+
+    RGW:add socket backlog setting for via ceph.conf
+    http://tracker.ceph.com/issues/16406
+    
+    Signed-off-by: Feng Guo <diluga@gmail.com>
+    (cherry picked from commit 3e4df832d34c782795a32b5a30f5c7414a1c3fa9)
+
+commit aa3936131d6525e656cd56315947bbf9ddc508ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 20:15:14 2016 +0800
+
+    mds: disallow 'open truncate' non-regular inode
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e4b6f2332bb4822cf324587a94144f1c98e4b97)
+
+commit 3af7b422a3a97c0e89dbff757c5ed0f4557a6602
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 19:45:23 2016 +0800
+
+    mds: only open non-regular inode with mode FILE_MODE_PIN
+    
+    ceph_atomic_open() in kernel client does lookup and open at the same
+    time. So it can open a symlink inode with mode CEPH_FILE_MODE_WR.
+    Open a symlink inode with mode CEPH_FILE_MODE_WR triggers assertion
+    in Locker::check_inode_max_size();
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4d15eb12298e007744486e28924a6f0ae071bd06)
+
+commit 8655803d8fb9ac21f75b636fb6d666f387462934
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Jul 7 14:14:36 2016 -0700
+
+    rgw: fix multi-delete query param parsing.
+    
+    Multi-delete is triggered by a query parameter on POST, but there are
+    multiple valid ways of representing it, and Ceph should accept ANY way
+    that has the query parameter set, regardless of what value or absence of
+    value.
+    
+    This caused the RubyGem aws-sdk-v1 to break, and has been present since
+    multi-delete was first added in commit 0a1f4a97da, for the bobtail
+    release.
+    
+    Fixes: http://tracker.ceph.com/issues/16618
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a7016e1b67e82641f0702fda4eae799e953063e6)
+
+commit bf73c9ad08ed2cc8db821c08694b11461549fb26
+Merge: f672ddf f858f94
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Jul 6 14:18:44 2016 -0400
+
+    Merge pull request #9545 from Abhishekvrshny/wip-16117-jewel
+    
+    jewel: rgw: aws4 parsing issue
+
+commit 546141c94a1c5e45dcb70e2d5fd06fe1ac0b1599
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Apr 5 11:14:16 2016 +0200
+
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Before this patch the RGW would respond with the Origin send by the client in the request
+    if a wildcard/asterisk was specified as a valid Origin.
+    
+    This patch makes sure we respond with a header like this:
+    
+      Access-Control-Allow-Origin: *
+    
+    This way a resource can be used on different Origins by the same browser and that browser
+    will use the content as the asterisk.
+    
+    We also keep in mind that when Authorization is send by the client different rules apply.
+    In the case of Authorization we may not respond with an Asterisk, but we do have to
+    add the Vary header with 'Origin' as a value to let the browser know that for different
+    Origins it has to perform a new request.
+    
+    More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
+    
+    Fixes: #15348
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 0021e224480c7164330eaa7cc1078bb8795169bf)
+
+commit f672ddfc5ad6ac86051987985067318436f0667b
+Merge: aa211cb ec884a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 6 11:55:13 2016 +0200
+
+    Merge pull request #9568 from dreamhost/wip-16182
+    
+    jewel: rgw: backport merge of static sites fixes
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 2afc176a2792a9ef389f34c7de1aba697fca6a31
+Author: Albert Tu <albert.t@inwinstack.com>
+Date:   Tue Jun 28 10:38:07 2016 +0800
+
+    rgw: fix double counting in RGWRados::update_containers_stats()
+    
+    Fixes: http://tracker.ceph.com/issues/16188
+    Signed-off-by: Albert Tu <albert.t@inwinstack.com>
+    (cherry picked from commit 5dd825aed25588843dc4834be3f5fdf10d93bc68)
+
+commit 489f8ce48b30c708879a002a55aecc080421b5a1
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Tue Jul 5 16:08:40 2016 +0800
+
+    librbd: fix missing return statement if failed to get mirror image state
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit ea775178b61ba38237343b07a90d19802f0b7dac)
+
+commit a7987f060479f81d6181f8949e8ed2c8b3029dfd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 25 18:33:49 2016 +0800
+
+    MDSMonitor.cc: fix mdsmap.<namespace> subscriptions
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 8f09dd15cb07597c57d0a6ae981f15a47de11bb9)
+
+commit d244b7a0c6eb4a57a424297d4293184dff28b94c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jun 17 11:53:32 2016 -0400
+
+    mds: add maximum fragment size constraint
+    
+    This commit adds a new config option
+    
+        mds_bal_fragment_size_max = 10000*10
+    
+    which is an order of magnitude larger than mds_bal_split_size.
+    
+    This limit prevents a fragment from getting too large which results in large
+    omap directories.
+    
+    Right now the limit is enforced only in the RPC paths and in stray directory
+    entry creation.
+    
+    Fixes http://tracker.ceph.com/issues/16164
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 60af83c80910070d8fb10ac7a4f6f24d49521c1b)
+
+commit 64d99b160d9d6a8758b7a9b3783bd1d153e15c9a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jun 17 11:58:13 2016 +0800
+
+    mds: fix Session::check_access()
+    
+    It calls CInode::make_path_string(...) with the second argument is
+    false. The second argument makes the third argument useless. For
+    newly created inode, the path string is something like #1xxxxxxxxx.
+    This can cause the access check to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/16358
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a94ef56523a383c44c7a52e473e37a43fa0cb6a2)
+
+commit bce5646cd7c9bc9c753bfcefeff37991763b9066
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jun 6 16:41:49 2016 +0800
+
+    client: skip executing async invalidates while umounting
+    
+    Fixes: http://tracker.ceph.com/issues/16137
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b95e603a3f9568debeb76fc49aae9a6ee4b75c6b)
+
+commit fd7ff96cbc4a2e9b38d805f36cd4e72a32f04925
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 10 15:15:48 2015 +0800
+
+    ceph-fuse: add option to disable kernel pagecache
+    
+    When multiple clients read/write to a file at the same time, we need
+    to disable cache and let reads/writes go to OSDs. In ceph-fuse case,
+    there are two caches, one is in ceph-fuse, another one is in kernel.
+    We can't disable/enable the kernel cache dynamically. So adding an
+    config option to completely disable kernel cache. Disabling kernel
+    cache may reduce performance, so option is disabled by default.
+    
+    Fixes: 12653
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0f11ec237d4692d313a038ed61aa07a3f40555ba)
+    
+    Conflicts:
+            src/common/config_opts.h
+               follow the content in patchset, set fuse_use_invalidate_cb
+               to True. Which is also the backport
+               for http://tracker.ceph.com/issues/15634
+
+commit 3f76e4acf243dda26c44ae95919b3d4444bd7d88
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 29 14:50:12 2016 -0700
+
+    rgw: finish error_repo cr in stop_spawned_services()
+    
+    Fixes: http://tracker.ceph.com/issues/16530
+    
+    Need to call finish, otherwise drain_all() wouldn't be able to
+    drain it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 492bb608733c83a5d85319cf47c4d86402344376)
+
+commit a5f5513ede87d7d77d83a9327e8d178767f734ea
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 28 14:39:46 2016 -0700
+
+    test: fix CMake build of ceph_test_objectcacher_stress
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2ee3d02c8f5de8d2a0ae3e22f3029b269fe4a212)
+
+commit 17f1bffdb891a155532d966b7d3ab4983c9016cf
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon May 23 15:14:21 2016 -0700
+
+    ObjectCacher: fix bh_read_finish offset logic
+    
+    If we have an incoming read split across multiple BufferHeads, we want to
+    line up the BufferHead's bl with the incoming OSDOp's bl at the right offset. We
+    were erroneously using this nonsense calculation (always equal to zero!) when
+    a much simpler comparison of the BufferHead's logical object offset to the
+    incoming OSDOp's logical offset will do the trick nicely.
+    
+    Fixes: http://tracker.ceph.com/issues/16002
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 9ec6e7f608608088d51e449c9d375844631dcdde)
+
+commit 73bc6d11745246df45ea6c4b5fbf9be3b9e91c09
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 14:23:51 2016 -0700
+
+    osd: provide some contents on ObjectExtent usage in testing
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a555d9a0c7d3b6c2206520cf78445234b0834639)
+
+commit 3446fa498266ec2902a1d4d9215de4e4b1d9c455
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Jun 10 17:01:09 2016 -0700
+
+    test: build a correctness test for the ObjectCacher
+    
+    For now it's very specifically designed to reproduce
+    http://tracker.ceph.com/issues/16002, but it can
+    be extended to other patterns in the future.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0fd55a9886dd8da344c23a5e9898ee5c5061e8f9)
+
+commit b6684914e29fdc375d91be7c80cdf9615d637c8c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue May 31 16:18:19 2016 -0700
+
+    test: split objectcacher test into 'stress' and 'correctness'
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cc9aab1b0a22c3f7320046b97f75dccf2b86cc6d)
+
+commit 74f59203a13da4ec5ca673a921ed3d07e8d5fc9b
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 13:41:46 2016 -0700
+
+    test: add a data-storing MemWriteback for testing ObjectCacher
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ed5801492bf2850e80a328ce5a61cb1ff2709215)
+
+commit aa211cb4ff0a7e9cac4779187848b3ba00c39bf9
+Merge: e3ccf0f d48a1ed
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:36:20 2016 +0200
+
+    Merge pull request #9099 from Abhishekvrshny/wip-15841-jewel
+    
+    jewel: s3website: x-amz-website-redirect-location header returns malformed HTTP response
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e3ccf0f9d31c9e35edb551dc24da7af7265799c1
+Merge: 448deed a08caa6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:35:23 2016 +0200
+
+    Merge pull request #9265 from Abhishekvrshny/wip-15965-jewel
+    
+    jewel: No Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 448deed8484b06bdbbdad62cbe34cfab7958bbc1
+Merge: ad182de ff9c29a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:34:42 2016 +0200
+
+    Merge pull request #9267 from Abhishekvrshny/wip-15960-jewel
+    
+    jewel: rgw: custom metadata aren't camelcased in Swift's responses
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ad182de1ca849d90fbbbf2ec057b476f32e86d8b
+Merge: 3ccfac6 108638f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:28:45 2016 +0200
+
+    Merge pull request #9316 from Abhishekvrshny/wip-15954-jewel
+    
+    jewel: rgw: initial slashes are not properly handled in Swift's BulkDelete
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3ccfac6b8e98afef19e6c113bd38ae7ab0c51c24
+Merge: 4e1ebac 0bdc8fd
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:26:09 2016 +0200
+
+    Merge pull request #9390 from vumrao/wip-vumrao-16071
+    
+    jewel: rgw : cleanup radosgw-admin temp command as it was deprecated
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4e1ebac3048ee1750978e11115144df074923e22
+Merge: 4957014 4eded9a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:25:51 2016 +0200
+
+    Merge pull request #9542 from Abhishekvrshny/wip-15967-jewel
+    
+    jewel: rgw: account/container metadata not actually present in a request are deleted during POST through Swift API
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4957014a0e7a744af6c782e6d4e9e5a89c934ace
+Merge: bb02d3e e8b7dd4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:25:30 2016 +0200
+
+    Merge pull request #9543 from Abhishekvrshny/wip-16040-jewel
+    
+    jewel: rgw: updating CORS/ACLs might not work in some circumstances
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 757babb03a9024d6aa42fb327244fd983c066545
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 09:03:24 2016 -0400
+
+    librbd: memory leak possible if journal op event failed
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f5069750849c2c43caa4bff766301169a4bfc6ca)
+
+commit e7ec20e6c54f913dd4bf52a949488ac9d258c150
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 08:45:59 2016 -0400
+
+    librbd: ignore snap unprotect -EBUSY errors during journal replay
+    
+    Fixes: http://tracker.ceph.com/issues/16445
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit beef0b4b024e9692305f4b413e6c6b520fdaa7f9)
+
+commit cbc963684def355b9c27932fdb4c605bf373ed6e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 09:23:32 2016 -0400
+
+    librbd: delete ExclusiveLock instance when switching to snapshot
+    
+    Fixes: http://tracker.ceph.com/issues/16446
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 677832ceb24096ddbeaf2d60e0ac72d28c399f02)
+
+commit d9c3f28f1200e99c8ecf92f50ec4e8840d74a0f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 13:34:56 2016 -0400
+
+    librbd: mark exclusive lock as released after journal is closed
+    
+    Fixes: http://tracker.ceph.com/issues/16450
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1811e62aa0ba3bab85c536eaab8f3adec6d1fffa)
+
+commit b1d969868bdd898958236212ee847f7a401c6406
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 24 10:03:53 2016 -0400
+
+    librbd: do not propagate mirror status notification failures
+    
+    These should be treated as a broadcast since no ACK is required. The
+    peer will eventually see the notification or its watch will have timed
+    out and it will re-sync.
+    
+    Fixes: http://tracker.ceph.com/issues/16470
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 97bade9f76af62765d7aa8c2154e51a7b231e40f)
+
+commit 51defeada069c0e33b03b96be962f155ce3df295
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 18:19:52 2016 -0400
+
+    librbd: fix crash while using advisory locks with R/O image
+    
+    Fixes: http://tracker.ceph.com/issues/16364
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1007aea3d78ca5aead8a11f932da0c1d9d67259e)
+
+commit db28ddcf88c13aef80e5a7131db463b305102abe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 10:14:21 2016 -0400
+
+    rbd-mirror: block proxied ops with -EROFS return code
+    
+    When replicating to a local image, the daemon will own the
+    exclusive lock and will receive any proxied maintenance ops
+    from other clients. Since the image is non-primary, respond
+    with -EROFS.
+    
+    Fixes: http://tracker.ceph.com/issues/16411
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 07b49df24e5f30460ce3ab584a89370ea3ff7cc8)
+
+commit ebce8ceb9353052d1d43d18e2bb76c68e581272e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 10:13:45 2016 -0400
+
+    librbd: optionally block proxied requests with an error code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 93e2faf38e866fb3e32a7b3f3527d97215c60d31)
+
+commit bb02d3e7a4db89d693ed1555d2ac82c5452978e1
+Merge: 8542898 b4a80cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 30 09:41:20 2016 +0200
+
+    Merge pull request #9631 from dillaman/wip-16232
+    
+    jewel: test: improve rbd-mirror test case coverage
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 70bf7465ad7c75c9c7623a446218ab501f329bd3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 09:39:24 2016 -0400
+
+    librbd: potential race when replaying journal ops
+    
+    Fixes: http://tracker.ceph.com/issues/16198
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 75b0f58e37b0ede5e8cf8dcaea980cf71a5ca908)
+
+commit f3f4a4a20ab3039f8dfeda23c773141bf6d95792
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:27:50 2016 -0400
+
+    librbd: journal callback to interrupt replay
+    
+    If the exclusive lock is lost while the journal is in the
+    replay state, the journal close will block until the replay
+    completes. The new callback will facilitate stopping replay
+    in a timely fashion so that the journal can be closed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9687e5e34aca98934fcf04089ead2794629455a1)
+
+commit b203168e21c0cfba9df55e9ff43e73a5905119d6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 23:04:48 2016 -0400
+
+    rbd-mirror: keep local pointer to image journal
+    
+    Fixes: http://tracker.ceph.com/issues/16230
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47e25e59a8ed4c4e64ae58b41c03125c6d3c4104)
+
+commit 0399958f6cc2a16487f0962b5d5a317dc1d9ff21
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 11:41:31 2016 -0400
+
+    rbd-mirror: keep events from different epochs independent
+    
+    Fixes: http://tracker.ceph.com/issues/16362
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a5b491ec61134dc2e18cbe4e27a54e64b17f7d2)
+
+commit 6a28d63ac609c4e3eb43a31cabe36f2c79c875d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 10:32:04 2016 -0400
+
+    librbd: fix lockdep issue when duplicate event detected
+    
+    Fixes: http://tracker.ceph.com/issues/16363
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 86ef725c34ae950c0e41e89c1aa0c6a15e40f369)
+
+commit 1e85da9cb97f6262eb25d8de234d45e8daccd461
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 17:32:20 2016 -0400
+
+    rbd-mirror: ensure replay status formatter has completed before stopping
+    
+    Fixes: http://tracker.ceph.com/issues/16352
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cccdca460b4be310f99877cf43c48300713150df)
+
+commit bf58eabf561fdb041ed170e54e5d35c46cbe3258
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 10:37:56 2016 -0400
+
+    journal: do not log watch errors against deleted journal
+    
+    The peer rbd-mirror process will eventually notice that the journal has
+    been removed and clean itself up. This avoids flooding the log with
+    expected error messages when journaling is disabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8317ce1611c39ad6a58bf2d760a010587d91ec60)
+
+commit deb6ca84e4083b1cf569d22b84f3194b1fb27a4b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:38:28 2016 -0400
+
+    librbd: force-remove journal when disabling feature and removing image
+    
+    If mirroring is enabled, it's nearly certain that the peer rbd-mirror
+    daemon will be a registered client within the journal. Without
+    forcing the removal, this would prevent the removal from occurring.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d83aa1e0e44784e1f38698a839f8a353c87027eb)
+
+commit 415ea77fdad016faf7da150630e933930c5ac4b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:31:03 2016 -0400
+
+    librbd: ignore ENOENT error when removing image from mirror directory
+    
+    Fixes: http://tracker.ceph.com/issues/16348
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4255afd34927ad6c303074aa6814538a8b5ae96e)
+
+commit cf65ed998b175e5d766364ff18741554b775b632
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Thu May 12 18:22:12 2016 +0800
+
+    rbd: Skip rbd cache flush if journaling is enabled under aio_flush
+    
+    With journaling rbd writes will be persisteted on rbd journal objects.
+    The journal will be replayed if crash happen. So it's not necessary to
+    flush rbd_cache in this case. This will improve the flush latency.
+    
+    This patch adds checking on handling aio_flush: if journaling is
+    enabled, rbd cache flushing is skipped.
+    In a system flush(ImageCtx::flush) the cache is flushed even w/ journaling
+    where we truly do need to flush all IO out to disk.
+    
+    Fixes: http://tracker.ceph.com/issues/15761
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit f2def83b7a4a98bc60db5ba8936d78a49abace88)
+
+commit caad884704ce22ae0a860a12693d7529a5837212
+Author: youji <youji@ebay.com>
+Date:   Tue Jun 14 11:12:16 2016 -0700
+
+    mon: Monitor: validate prefix on handle_command()
+    
+    Fixes: http://tracker.ceph.com/issues/16297
+    
+    Signed-off-by: You Ji <youji@ebay.com>
+    (cherry picked from commit 7cb3434fed03a5497abfd00bcec7276b70df0654)
+
+commit 3250c4d5a0bd9fc32eb6b1831a38363581f7c1bd
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon Jun 20 15:56:50 2016 +0200
+
+    rgw_swift: newer versions of boost/utility no longer include in_place
+    
+    boost > 1.58 no longer includes in_place in boost/utility, we need to
+    include in_place_factory explicitly. This causes build failures in
+    distros that ship with a higher version of boost. Since the only call is for
+    swift_ver_location, another possibility is to use emplace()
+    instead (though this requires boost ~ 1.56)
+    
+    Fixes: http://tracker.ceph.com/issues/16391
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a14f0f3869bb7defa7587ad9ccb18c3f086b2c3d)
+    
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc
+                jewel does not have #include <boost/optional.hpp>
+
+commit 85428983070c8df663056404a7c63959bb8ef693
+Merge: 2817f99 e9f9916
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 16:47:15 2016 +0200
+
+    Merge pull request #9294 from theanalyst/wip-16009-jewel
+    
+    jewel: radosgw-admin: failure for user create after upgrade from hammer to jewel
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit dd635e46ee2425335264ae493cb6b8437cd712fc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 10:25:54 2016 -0400
+
+    librbd: ignore missing object map during snap remove
+    
+    Fixes: http://tracker.ceph.com/issues/16350
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68b296b0f107db39b16fa90c186fdc920d97170d)
+
+commit db7ce96a3093e661f535efdd5e7e37177ba0732b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 24 10:52:16 2016 -0400
+
+    librbd: removal of partially deleted image needs id lookup
+    
+    Several operations depend on the image id but if the image cannot be
+    opened to retrieve the id, these cleanup operations cannot be executed.
+    
+    Fixes: http://tracker.ceph.com/issues/16471
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a9e7da6c3da40c7b25ff3d5ac84027e94beb287)
+
+commit c1a47c7728a17dbfc71280270b0c3079e5961ffb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue May 31 17:50:05 2016 -0600
+
+    packaging: move parted requirement to -osd subpkg
+    
+    Prior to this change, ceph-base required the "parted" package, which
+    meant that any installation of ceph-osd, ceph-mon, or ceph-mds would
+    pull in the parted package.
+    
+    Move the parted dependency to ceph-osd, since ceph-disk is the only
+    thing that uses parted.
+    
+    The advantage of this change is that Mon and MDS systems will not need
+    to install the parted package.
+    
+    Fixes: http://tracker.ceph.com/issues/16095
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 115349680bad520a0aa56ce3a346d93f541a905c)
+
+commit 74dd0359d4d6db3c6dac4fd41703270e5020aad7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 14:28:55 2016 -0400
+
+    osd/PG: set last_* PG fields on a creating PG
+    
+    Use the value from pg_history_t that ultimately came from the
+    mon if last_epoch_started == 0.  This establishes a sane lower
+    bound on these timestamps, and prevents a mon health error from
+    coming up about how a PG is stuck in whatever state while it is
+    being created.
+    
+    (We addressed half of this problem in commit 6ca6aeac, but the
+    warning comes back as soon as the OSD reports with a creating
+    state.)
+    
+    Fixes: http://tracker.ceph.com/issues/14952
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3be3bc60c12448a36f607c8d4fbf3300c7bbdbee)
+
+commit 2c03e02a04c217b8461b858fd3b46b73c4a370d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 14:27:16 2016 -0400
+
+    osd: set pg history last_clean_scrub_stamp on create
+    
+    We were setting the other two; set this one as well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47f540d5b7ecc4ae193057df429db24ca6b3de8d)
+
+commit a2e8ae68ea452d03519359cb0cd344e71e603fa9
+Author: kofiliu <liu.xuan@h3c.com>
+Date:   Fri May 27 03:45:06 2016 -0400
+
+    osd: sparse_read offset may not be zero for ecpool
+    
+    Signed-off-by: kofiliu <liu.xuan@h3c.com>
+    (cherry picked from commit 65e8738611cde0090619b3566a2e25f83b4c8468)
+
+commit 2817f9978f73014b1b1363cecacabdd98e499c72
+Merge: 38a2542 8b82bb8
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 29 11:42:43 2016 +0300
+
+    Merge pull request #9721 from trociny/wip-16289-jewel
+    
+    jewel: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b8f7aa28c394830bac9aa73429131e92f7499aa0
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sat Jun 11 13:39:23 2016 +0800
+
+    msg/async/AsyncConnection: make verify auth callback without connection lock
+    
+    Fixes: http://tracker.ceph.com/issues/16237
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 2b7776545c3f87d7f54a53190e65ec48378eaa05)
+
+commit 3dbb08a069c8e8238d2884a78b04f0a0d01a36e3
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Thu May 12 12:01:35 2016 +0800
+
+    AsyncConnection: create writable event for in progress connection
+    
+    Previously we use a tricky with ceph msgr protocol, if initiator side got
+    in progress connection state, it will wait until read event. Because if
+    tcp session built successfully server side will send the banner firstly
+    and initiator side will get read event. Otherwise, if connection failed,
+    read event also be called.
+    
+    But actually man(2)[http://man7.org/linux/man-pages/man2/connect.2.html]
+    specify if we want to get notification whether connection built, we need
+    to listen writable event. It means when connection built, send buffer
+    is ready to be written.
+    
+    This patch follow the strict nonblock connect process. Not fully sure fix
+    http://tracker.ceph.com/issues/15849
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit a74ce419133881ff8618733a0501c4a47e1368e3)
+
+commit 6554d462059b68ab983c0c8355c465e98ca45440
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jun 7 17:15:05 2016 -0700
+
+    OSDMonitor::prepare_pgtemp: only update up_thru if newer
+    
+    Fixes: http://tracker.ceph.com/issues/16185
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5f2bd7b6b28aad96d68444b22c04b8b24564616b)
+
+commit a826bb818b02edac71b69fdd97e9318bb42ebc36
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 10:43:17 2016 -0700
+
+    OSDMonitor: drop pg temp from sources other than the current primary
+    
+    Fixes: http://tracker.ceph.com/issues/16127
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1a07123c38e3fecb3fb2e43bbbae962d8411d287)
+
+commit 18fdc1c9f061b396df8095907c5b0ffb0591e76d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:32 2016 -0400
+
+    osd: reject PARALLELEXEC ops with EINVAL
+    
+    Errors are better than crashing.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 407495197dd878b62f1356f6d939e33ab50d78c6)
+
+commit f91da931bcca9fd9f055a8d42d86ba3c3d3ea25f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:14 2016 -0400
+
+    ceph_test_rados_api_misc: test rados op with bad flas
+    
+    Pass the bad PARALLELEXEC flag to remove(), which takes a flags arg.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1aa807f4f29b4b016cc737ffa443e8f30c3b7693)
+
+commit edd3f799fb1e5d70244412c5a1be17f74397aae2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:42:36 2016 +0800
+
+    msg/async: close STATE_WAIT connection in short period
+    
+    1. in practice, STATE_WAIT connection caused by racing connect should be
+    resolved in milliseconds level. we don't need to keep this connection
+    forever.
+    2. it will avoid unexpected osd peering hang because of outside network
+    problem.
+    
+    Fixes: http://tracker.ceph.com/issues/16378
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 4de5407ac96686748497253e4daf51177f809a95)
+
+commit af8e86c20e9b1124abe33f0dff58817fc8aad7c9
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:12:10 2016 +0800
+
+    test_msgr: add assert if wait for shutdown hang
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit ca22e0bc09c1a13c8dac14b25c00501e912b8006)
+
+commit 42ef435554a21dfd5daec53002440a38f4d9f705
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:04:46 2016 +0800
+
+    test/test_msgr: add shutdown hang debug info
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 37b53d69d3d8c848d4627475fd63c80e15e118e7)
+
+commit d4c531dca281560b904ec14b2749bc2924f470fb
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun May 22 23:43:47 2016 +0800
+
+    test_msgr: add verbose info for pending connections
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 39515b0f7c49d0cedbec0cf0dc2196b9d6d6339c)
+
+commit 90ce35c4f449bfd48398b8a164e423f3d72609b2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:03:07 2016 +0800
+
+    msg/async/AsyncConnection: lower racing case debug log level
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit d8cc985fe0ebc18d4ff725f50304a5ff36a77cd0)
+
+commit 38a25420a0120288029d65cf924f1cbe5f3eb3fd
+Merge: 980abcf 75d3261
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:56:58 2016 +0200
+
+    Merge pull request #9425 from cbodley/wip-rgw-sync-retry-jewel
+    
+    jewel: rgw: retry on bucket sync errors
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 980abcfd3966c29b4871d5a0913051a8312d866c
+Merge: e230023 933fdef
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:55:26 2016 +0200
+
+    Merge pull request #9327 from cbodley/wip-rgw-zone-modify-master-jewel
+    
+    jewel: rgw: add_zone only clears master_zone if --master=false
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e2300232b07175a12b92f23352deb34210d8ae41
+Merge: e63dcb0 8163c4d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:50:04 2016 +0200
+
+    Merge pull request #9242 from yehudasa/wip-15911-jewel
+    
+    jewel: rgw: don't unregister request if request is not connected to manager
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e63dcb08083dff3a8382900f90df9f3311c9ec99
+Merge: c9f2fda b751d48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 15:53:03 2016 -0400
+
+    Merge pull request #9883 from dillaman/wip-16422
+    
+    jewel: librbd: flag image as updated after proxying maintenance op
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c9f2fda32c55f85f717ec26dfcf9140b16fa8cc6
+Merge: 9aafefe 70018bf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Jun 27 12:43:11 2016 -0700
+
+    Merge pull request #9105 from Abhishekvrshny/wip-15768-jewel
+    
+    jewel: FileStore: umount hang because sync thread doesn't exit
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 9aafefeab6b0f01d7467f70cb2f1b16ae88340e8
+Merge: 25c807b cb9e9e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 17:50:39 2016 +0200
+
+    Merge pull request #9633 from liewegas/wip-async-jewel
+    
+    msg/async: backports of all asyncmsgr fixes to jewel
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 25c807b1919fe11b8d8183b06cdfc5465357d509
+Merge: ae34e37 6619bd9
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:12:14 2016 -0400
+
+    Merge pull request #9576 from Abhishekvrshny/wip-16148-jewel
+    
+    jewel: Scrub error: 0/1 pinned
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ae34e3742651f7b679e4b524676d26d4fb906562
+Merge: e16830c b7f0e73
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:11:43 2016 -0400
+
+    Merge pull request #9575 from Abhishekvrshny/wip-16150-jewel
+    
+    jewel: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e16830c6287323aae4fdddd9f4192793a6c48e74
+Merge: 2cf8d58 6c1163c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:10:36 2016 -0400
+
+    Merge pull request #8754 from kmroz/wip-backport-fix-run-dir-chown
+    
+    jewel: global-init: fixup chown of the run directory along with log and asok files
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1f19dbd0f092390b44acfb5fe9dc3936c6732fdd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jun 27 14:45:02 2016 +0300
+
+    librbd: potential use after free on refresh error
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e92b3950cd261d36de422ccaa4a2441d41c80d0c)
+
+commit 2cf8d58beb164017d065e91fdbf8c7a380a4dddb
+Merge: b861c30 3f8642d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:12:35 2016 +0200
+
+    Merge pull request #9578 from Abhishekvrshny/wip-16080-jewel
+    
+    jewel: osd:sched_time not actually randomized
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b861c30bdf86378845379bd3e66db32f264b23ee
+Merge: eea4851 1c7abcb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:08:25 2016 +0200
+
+    Merge pull request #9574 from Abhishekvrshny/wip-16153-jewel
+    
+    jewel: Missing export for rados_aio_get_version in src/include/rados/librados.h
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit eea48519d76e53ebe94947165c9404259397a37e
+Merge: 057ff74 9415d38
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:06:54 2016 +0200
+
+    Merge pull request #9565 from dzafman/wip-15767-jewel
+    
+    jewel: Fixes for list-inconsistent-*
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 057ff74ede5510e1b28b19e3f99d823781d5da39
+Merge: 50e242c b2d1df1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 09:42:28 2016 +0200
+
+    Merge pull request #8904 from SUSE/wip-15700-jewel
+    
+    jewel: rados/test.sh workunit timesout on OpenStack
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad3b788b0c83ffb1339d940cd86555dbc3b1b55f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 20:33:17 2016 +0200
+
+    Drop ceph Resource Agent
+    
+    This RA wraps the ceph sysvinit script. As of Jewel, none of the supported
+    distros are using sysvinit anymore. So, drop it.
+    
+    Incidentally, Pacemaker can control systemd units without any wrappers.
+    
+    References: http://tracker.ceph.com/issues/14828
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bb624c7334ee4241ea3bf892f88e25d165dc3477)
+
+commit b751d4854aa8629f46f0e3dd835aff27b8be709d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 21 14:56:43 2016 -0400
+
+    librbd: flag image as updated after proxying maintenance op
+    
+    Fixes: http://tracker.ceph.com/issues/16404
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 478dd5f173591914b41d87b8c6d035c32cac8d3e)
+
+commit 50e242c28b766bd35c924f0f01c0298ed166d20a
+Merge: a34b227 9b75275
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Jun 21 16:43:20 2016 -0700
+
+    Merge branch 'wip-dmick-install-deps-jewel' into jewel
+    
+    install-deps.sh: use mk-build-deps instead of processing control
+    
+    Reviewed-by: Erwan Velu <erwan@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9b7527537547b388d1cd75cb70a522712420a455
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Mar 23 17:43:53 2016 -0700
+
+    install-deps.sh: use mk-build-deps instead of processing control
+    
+    mk-build-deps is designed to handle installing build dependencies;
+    use that instead, so '|' indications are handled properly.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 99bca09b8d09c41077a9d2141ff556b74328e57a)
+
+commit a34b227c1ed11f51d830ea48a6fbfc0c74a981a6
+Author: Roi Dayan <roid@mellanox.com>
+Date:   Wed Apr 20 17:56:16 2016 +0300
+
+    xio: add MNop.h to dist tarball
+    
+    This file is needed when we want to build a package with xio messenger
+    but was not added to dist tarball.
+    
+    Signed-off-by: Roi Dayan <roid@mellanox.com>
+    (cherry picked from commit ac50842d81d271569d4149737834ac53b2f51db2)
+
+commit 393bf7ef409253123e0721cdfe3b78aa4fd55148
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 15 16:53:16 2016 +0200
+
+    rgw: check for -ERR_NOT_MODIFIED in rgw_rest_s3.cc
+    
+    Fixes: http://tracker.ceph.com/issues/16327
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc38346c596d8b0bc156183970d6a327943cb577)
+
+commit 6b41d76f53337a696523cddb47b53b703cdfa571
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 14 11:03:19 2016 +0800
+
+    TaskFinisher: cancel all tasks wait until finisher done
+    
+    Otherwise, caller may think pending task won't be executed but actually
+    finisher may execute callback which may cause refer to freed object.
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 40c5679ef038375e2bde982f401d78c1f3e05c6c)
+
+commit 762db30a3c2c43e56b227017b993802369c07219
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Tue May 24 09:29:32 2016 +0530
+
+    rgw: support size suffixes for --max-size in
+    radosgw-admin command
+    
+    Fixes: http://tracker.ceph.com/issues/16339
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 62eec902c70463173e758e55fa38594c594f868f)
+
+commit e3a99c082e3ebd56d5b40d7d94d98e35629df81e
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 23:30:25 2016 +0530
+
+    common: add int64_t template for strict_si_cast()
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 8e429d05370fbe7935212d0ae9608e7547f39860)
+
+commit f8e491168d4802ab071e30e4fdd654dca018bfe6
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon May 9 18:00:28 2016 +0530
+
+    rados: Add cleanup message with time to rados bench output
+    
+    Fixes: http://tracker.ceph.com/issues/16338
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 885ebb1650bb4a355600d85ca436c1ecd4916dce)
+
+commit 8b82bb85e4f2d6fb81c3c4ccc6789856c78f4984
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 14 15:28:06 2016 +0300
+
+    qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Fixes: http://tracker.ceph.com/issues/16289
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3048d3c7ce800a6174c54946b838e4e4079ec609)
+
+commit 9415d38139a66b86ff40e0521b769665f31fd36b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu May 5 23:20:58 2016 -0700
+
+    rados: Improve list-inconsistent json format
+    
+    Use array "errors" of strings for the error bits
+    Change snapshot number to integer instead of hex
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 07dc844d6804db93055573ec83e6753773a4c049)
+
+commit 2fd6e7cf78f2eabbec669f3f9c4e60f1aa3ee8bd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 6 19:54:11 2016 -0700
+
+    test: Fix test to not use jq -S which isn't avail in all distributions
+    
+    Fixes: http://tracker.ceph.com/issues/15766
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a2147ad3cb256c34541ff7dc285594a22a24dfc6)
+
+commit c789aa39f27dc83f52465ba2d5b513a06b99c9d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 3 12:10:38 2016 -0700
+
+    test: Add testing of new scrub commands in rados
+    
+    Test list-inconsistent-pg
+    Test list-inconsistent-obj
+    Test inconsistent-snapset
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit db517ba9176f1ee829453b016b1bd43c6054a555)
+
+commit a6f3f762505849994c91efea98c345da16561a22
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 11:25:18 2016 -0700
+
+    rados: Don't bother showing list-inconsistent-* errors that aren't set
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a551b6bd5c4eba11c170afe63994258e7598c3d2)
+
+commit 4c72195a6c5cb9eeca952f12edf62a10ec666f10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 17:09:13 2016 -0700
+
+    osd, rados: Fixes for list-inconsistent-snapset
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fdca113fc24adbb4f60bfdc55fbbd5044b711b5d)
+
+commit 4e4e562926d38371f70ba2cf2f7003daa251b9f0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 11:26:39 2016 -0700
+
+    include, rados: Fixes for list-inconsistent-obj and librados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4fc4326b36aa00092043514746aa39aedad06b0f)
+
+commit 80f0dd348f32ff6e1f98ca29ea2ed38b5b7f5854
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 26 18:16:22 2016 -0700
+
+    rados: Balance format sections in same do_get_inconsistent_cmd()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de4681b80a5d24e72bbc7ac6dfee4775987fd834)
+
+commit d248aba10ed232df116729591fea5b195a245735
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 21 20:41:03 2016 -0700
+
+    rados: Include epoch in the list-inconsistent-* command output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit df4bc16c4d49489f7398eb1ecc7b9aef93258414)
+
+commit 43141e383ebac822221b3f22392f02b95b015ef5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 21 20:39:53 2016 -0700
+
+    rados: Improve error messages for list-inconsistent commands
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cea7cf56ee3b9593c935d0c74ce6e4b28b14c2da)
+
+commit 502540faf67308fa595e03f9f446b4ba67df731d
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--dfeed223-b008-47e6-b689-a0d96992b62c.localdomain>
+Date:   Fri May 13 17:50:34 2016 +0000
+
+    10.2.1
+
+commit b4a80cb1e30f8da8e2b0fc2b18fc846609282879
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Jun 11 20:01:31 2016 +0300
+
+    qa/workunits/rbd: specify source path
+    
+    otherwise it looks in $PATH for the script
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 698242e7435c9c4d1db7cb6d5aab3faf57eda840)
+
+commit 70c97bd07e6764e1c6ff83225f6a2a9dcdfb989e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 16:23:52 2016 -0400
+
+    qa/workunits/rbd: additional rbd-mirror stress tests
+    
+    Fixes: http://tracker.ceph.com/issues/16197
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db3e583a5fe4a7985b1e7f1740114da414835af5)
+
+commit c7cfb4825fbcc82e74b5b1461fc86591098a8599
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 17:14:36 2016 -0400
+
+    vstart: add --nolockdep option
+    
+    rbd-mirror stress test cases need a way to disable lockdep when
+    initiated in a vstart environment.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5eec7b2f4222dde0cfdaa9e0e4b43be015c3692)
+
+commit d48a1ed7845843cfc7598d97dce6b10992079ec3
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sun Apr 17 08:23:23 2016 -0700
+
+    rgw/s3website: Fix x-amz-website-redirect-location support.
+    
+    Support for the x-amz-website-redirect-location header was broken in the
+    feature merging of Jewel, as it previously depended on the error handler to
+    redirect, which was overkill. Simplify it to work.
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15531
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7cbb63bb748c4c78c02926acb4ad9bcea5593695)
+
+commit 3f8642d134c473be331a53f420d122b4f390dee6
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat May 14 11:28:04 2016 +0800
+
+    osd: fix sched_time not actually randomized
+    
+    The test program:
+    
+    main()
+    {
+      for (int i = 0; i < 1000; i++) {
+        double before = rand() / RAND_MAX;
+        double after = rand() / (double)RAND_MAX;
+        cout << "before: " << before << " after: " << after << endl;
+      }
+    }
+    
+    And the output(partial and with "-std=c++11" option):
+    
+    before: 0 after: 0.394383
+    before: 0 after: 0.79844
+    before: 0 after: 0.197551
+    before: 0 after: 0.76823
+    before: 0 after: 0.55397
+    before: 0 after: 0.628871
+    before: 0 after: 0.513401
+    before: 0 after: 0.916195
+    before: 0 after: 0.717297
+    before: 0 after: 0.606969
+    before: 0 after: 0.242887
+    before: 0 after: 0.804177
+    before: 0 after: 0.400944
+    before: 0 after: 0.108809
+    before: 0 after: 0.218257
+    before: 0 after: 0.839112
+    before: 0 after: 0.296032
+    before: 0 after: 0.524287
+    
+    Fixes: http://tracker.ceph.com/issues/15890
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit d3b7767a64af0cd57d17b888e4ec3bdae6fdab87)
+
+commit 6619bd9dbe257bfeb19931cf7f94c56520ae1ebf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 20 13:59:10 2016 -0700
+
+    ReplicatedPG: adjust num_pinned in _delete_oid
+    
+    Fixes: http://tracker.ceph.com/issues/15952
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 907d4e25c676fd3e1c2be90ce8ab5b64b362b0bc)
+
+commit b7f0e73223687035e470fcd7ffc7b851c04aba00
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 19 16:00:35 2016 -0700
+
+    PG: update PGPool to detect map gaps and reset cached_removed_snaps
+    
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a)
+
+commit 1c7abcbfe58f77d9d915dd2fc2fc4c75883d46c1
+Author: Jim Wright <jim@quadraturecapital.com>
+Date:   Thu Jun 2 15:12:30 2016 +0100
+
+    librados: Added declaration for rados_aio_get_version
+    
+    Was missing from librados.h and hence had an incorrect
+    visability.
+    
+    Fixes: #15535
+    (cherry picked from commit a5a1c1732e50be7d27f8742e794eba88d35c1c7f)
+
+commit ec884a3406568ccb4c997b4da8924d25d3e2473b
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Tue May 17 17:38:38 2016 -0700
+
+    rgw/s3website: whitespace style fixes
+    
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit fd35ad001c9430753cc2b1f91a7d48f591c6754e)
+
+commit bf26b6eeb7fb806bff0ee306467277a273dcc887
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Tue May 17 17:35:41 2016 -0700
+
+    rgw/s3website: Fix ErrocDoc memory leak.
+    
+    Yehuda pointed [1] out a memory leak with the RGWGetObj not being handed back, and
+    suggested that shared_ptr was used to handle it, rather than the prior approach
+    of explicitly calling handler->put_op.
+    
+    This commit implements just that change, in isolation from other changes, for
+    ease of comparision testing.
+    
+    X-Reference-1: https://github.com/ceph/ceph/pull/8669#discussion_r63445516
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 2fd7bc8cbe4a2e23553e716ce0be75937853e07b)
+
+commit 36672c61be8289ba00dd694c92cd9057dbf5cf29
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sun Apr 17 08:23:23 2016 -0700
+
+    rgw/s3website: Fix x-amz-website-redirect-location support.
+    
+    Support for the x-amz-website-redirect-location header was broken in the
+    feature merging of Jewel, as it previously depended on the error handler to
+    redirect, which was overkill. Simplify it to work.
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15531
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7cbb63bb748c4c78c02926acb4ad9bcea5593695)
+
+commit 3c0ac8eb9c9d19a6418e3d84df36e4a87f24fb23
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Wed Apr 20 15:52:51 2016 -0700
+
+    rgw/s3website: Implement ErrorDoc & fix Double-Fault handler
+    
+    Fix more last minute breakage from merges, now has has a working ErrorDoc as
+    well as working double-fault. Also moves some s3website-specific code out of
+    the main S3 codepath.
+    
+    Fixes: #15532
+    Fixes: #15555
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit fcb3cf169f1fa7cf878eb154dc3f1ff78e278056)
+
+commit cb9e9e1c322cb7fb2150b15b17bde4371fd1e703
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Feb 26 13:54:20 2016 +0100
+
+    msg/async: Implement smarter worker thread selection
+    
+    This changeset makes AsyncMessenger a bit smarter when it comes
+    to assigning worker threads to AsyncConnections. Each time a worker
+    is assigned, its reference count is increased. Next time when Async
+    Messenger needs to assign another worker to new AsyncConnection, it
+    picks the one with the lowest reference count. If it cannot find an
+    idle one, and number of currently instantiated workers is less than
+    specified with "ms async op max threads", the new worker is created
+    and returned.
+    Once AsyncConnection goes away, the reference count on assigned
+    worker is decreased.
+    This does not prevent, but greatly reduces chances of having a single
+    async worker thread doing most (or even all) of the ops, and also
+    removes the need to manually tune the "ms async op threads" option.
+    
+    Signed-off-by: Piotr Dałek <git@predictor.org.pl>
+    (cherry picked from commit 3e80f8d74a535e14d4092b27ea5417bacff8394e)
+
+commit 578ac8aacfdd0ecfecb3ae3ebeb8a0b3ff53b67d
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Wed Apr 20 14:23:20 2016 -0400
+
+    Event: fix delete_time_event while in processing list
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 1ddeede83200566666fce80867eb7cb5a61a4f62)
+
+commit 8c7a13fdfa46095621b96a7da8d3b9ce09439509
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Fri Apr 15 13:33:35 2016 +0800
+
+    test_msgr: add delay inject test
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 0a2392919f9ad7286ae4b5924566197c1069474f)
+
+commit 846992f025586fa83c69eaec3ed09b6ab6677fcc
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Fri Apr 15 11:43:42 2016 +0800
+
+    AsyncConnection: make delay message happen within original thread
+    
+    Fixes: http://tracker.ceph.com/issues/15503
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 83f7db58aad2509e1a8742e862d4e8bbfd85c37c)
+
+commit 9b199d0b19220bf4adb9b0754f493e7c1ad78a4e
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Tue Apr 5 09:37:23 2016 +0200
+
+    msg/async: add missing DelayedDelivery and delay injection
+    
+    Delay injection was missing from a few spots, also, DelayedDelivery
+    was added.
+    
+    Fixes: http://tracker.ceph.com/issues/15372
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 49a0c9981bd4bf61b520ece8fb8adfdf7439185b)
+
+commit 0e6324a726e85e498946a49393e0f7c228c2f913
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun May 1 08:32:24 2016 +0800
+
+    Event: replace ceph_clock_now with coarse_real_clock
+    
+    reduce cpu utilization on real clock latency
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 79343a8a093630f1c0696f135c6e3bef0c23da28)
+
+commit 6597fab4cc27de6d6f1dcfa070ed401612bfed76
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Mon Apr 25 09:45:20 2016 +0800
+
+    msg/async: fix some return values and misspellings.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 0519f938e13d0adf53c1328fd636b9fa5e6c5b93)
+
+commit 18f18c7beee3c7a072009838c02d5ba1f97fef2c
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Mon Mar 28 15:33:30 2016 +0800
+
+    msg/async: delete the confused comments.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 69a587b139815812433f2b651db6bd723353605d)
+
+commit 79354f4223191c9d8ce6f92aaadf3481f9abf72d
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Fri Mar 25 17:34:12 2016 +0800
+
+    msg/async: add numevents statistics for external_events
+    
+    Maybe we need to add the statistics for external_events.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit f08ca0a0892767a8c40e06e336297109aa6142a1)
+
+commit 4005a5193a1cb3e3a0ac1e6f019bead0837ea552
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Mar 13 12:24:35 2016 +0800
+
+    AsyncConnection: remove unnecessary "send" flag
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 5c0a689d720f1016846ff945ca75b9f91756170d)
+
+commit 441847d9951e230b48776fa0fbe639d7b23a595b
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sat Mar 12 21:02:51 2016 +0800
+
+    async: skip unnecessary steps when parsing simple messages
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit a9ac1c4f88ffb5f66c43527ea0ae1deef1be3a96)
+
+commit c02501005b0316ba9f0ae1cd316a92c567b37c3b
+Author: Noah Watkins <nwatkins@redhat.com>
+Date:   Wed May 4 17:22:14 2016 +0800
+
+    client: fstat should take CEPH_STAT_CAP_INODE_ALL
+    
+    Fixes: http://tracker.ceph.com/issues/15723
+    Signed-off-by: Noah Watkins <nwatkins@redhat.com>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4b1a5d4ef9c3a116bb08100edd576819930047e3)
+
+commit c3f6d82d86f9720a373ec9792b927292f01897c0
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Jun 1 14:28:17 2016 +0800
+
+    mon/MDSMonitor: fix wrongly set expiration time of blacklist
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 0c3a92bb999e90212a9f38f9f4dc3bf89bd20acb)
+
+commit d4017ae915e8355f9146844a443942c0dce32476
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Jun 1 11:17:32 2016 +0800
+
+    mon/MDSMonitor: fix wrong positive of jewel flag check
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 24b82bafffced97384135e55ab2a97091e9a7b4b)
+
+commit eea0e916640c3ac2d69ffb9c335dde6332b03938
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 6 19:07:07 2016 +0800
+
+    mds: finish lock waiters in the same order that they were added.
+    
+    Current code first processes lock waiters who have smaller wait mask.
+    Lock waiters who have large wait mask can starve if client keeps
+    sending requests that add waiter with small mask.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d463107473382170c07d9250bb7ace0e5a2a7de2)
+
+commit 2c7fc95e06f6b1b52bef89904399de4088efaff1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 23 17:40:05 2016 +0800
+
+    mds: fix race between StrayManager::{eval_stray,reintegrate_stray}
+    
+    StrayManager::eval_stray() is called after Server::respond_to_request()
+    drops locks. So it can race with StrayManager::reintegrate_stray()
+    
+    Fixes: http://tracker.ceph.com/issues/15920
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 65827a0fd4d4ecb6d5e2eccf3a1818026ce1f10c)
+
+commit 19c13669ecb1bc9788b6741753d3eedbde713620
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon May 2 22:23:04 2016 +0100
+
+    osdc: send error to recovery waiters on shutdown
+    
+    ...instead of sending them '0', which gets things
+    confused.
+    
+    Fixes: http://tracker.ceph.com/issues/15689
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 62c7a3c59ce479d5240eb5fbef01edf11388c7bb)
+
+commit 7b0318aad79a08f1549bbf9054519a18c9e8c379
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon May 2 22:14:07 2016 +0100
+
+    mds: fix mdsmap print_summary with standby replays
+    
+    Fixes: http://tracker.ceph.com/issues/15705
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 899a16e50a6c22359e7a5c5ac720a605f6a4b67e)
+
+commit f858f94c9dcd2b2845fe59ff9d213b59574144c5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 1 04:24:34 2016 -0700
+
+    rgw: reduce string copy
+    
+    As suggested by Casey Bodley.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f8f1f217314c32cf65ac1fa4e8e0132b501ee184)
+
+commit 67e3be38fdfcec8fa4b00dfc703a45ffa101679b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 15:02:21 2016 -0700
+
+    rgw: rework aws4 header parsing
+    
+    Fixes: #15940
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e3618c87026b5ced8ef81adbcafc7f9b34f2d48d)
+
+commit 5bdd13e6f2bc8c52f9a4829a5cc80691947233bb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 12:52:54 2016 -0700
+
+    rgw: don't add port to aws4 canonical string if using default port
+    
+    Fixes: #15939
+    
+    When either port 80 is used, or if it's a secure connection and port 443 is used,
+    and when going through the presigned url auth, don't add the port to the signed
+    string.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 033888bbd0e4d8d81358bf61a099276dddb5692b)
+
+commit 474739e87231dc1fc3e3a805584c3f15e1dd1f94
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 11:30:44 2016 -0700
+
+    rgw: use correct method to get current epoch
+    
+    Fixes: #15939
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 310f5bdf56a9deb09347aadc158da25750fb6735)
+
+commit fc34fbd469dd1c35804305a96d4e650828049d51
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 18 17:21:28 2016 -0700
+
+    rgw: check for aws4 headers size where needed
+    
+    Fixes: #15940
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 493cc5d1241693f3ea52f4d7f3a194d9e0ec1905)
+
+commit 44decb4ea1d8e60ba929500e0ccbdac3417c3647
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed May 25 14:23:29 2016 +0200
+
+    rgw: properly handle initial slashes in SLO's segment path.
+    
+    Fixes: http://tracker.ceph.com/issues/16015
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d384b2b8e0ed670f229eb889a14f521fa8d194fc)
+
+commit e8b7dd47018ab115ffa27b7e72470956de738fd3
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 23 13:27:24 2016 +0200
+
+    rgw: remove unnecessary data copying in RGWPutMetadataBucket.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 19c12bbc233a118496f8ad5d640d19bb0e2c5d05)
+
+commit 63e0993e33b10adc4d9e1f80c4fe4c5ee9c5f4ff
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun May 22 15:32:19 2016 +0200
+
+    rgw: Fix updating CORS/ACLs during POST on Swift's container.
+    
+    Introduced in: 7a7de9b75265b978ba4e53966f614fac033972cb
+    Fixes: http://tracker.ceph.com/issues/15976
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    
+    (cherry picked from commit d9e4727b7e6ffa1cb2918b610381d41439a056e8)
+
+commit 4eded9aa94384e60e765accb4c9f093bd2534970
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat May 21 02:47:12 2016 +0200
+
+    rgw: fix update of already existing account/bucket's custom attributes.
+    
+    Introduced in: 3f3b18bff16f6a5b36987f888ba3f2a0d1ea3155.
+    Fixes: http://tracker.ceph.com/issues/15977
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    
+    (cherry picked from commit d6129e664fc8d25e70bfaf83e340703005f8f73f)
+
+commit 30ee18018ae890a058ae40a6006e1045258d36d5
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 9 17:22:45 2016 +0200
+
+    rgw: fix updating account/container metadata of Swift API.
+    
+    This patch rectifies an issue with handling of user metadata that are
+    actually stored by an updated resource (account or container).
+    The expected behaviour is they will be merged with new medadata coming
+    from an HTTP client.
+    
+    Backport: Jewel
+    Fixes: http://tracker.ceph.com/issues/15779
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 3f3b18bff16f6a5b36987f888ba3f2a0d1ea3155)
+
+commit 75d326117dd57acdae355cec5ac25112e70ff2ba
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 14:13:27 2016 -0700
+
+    rgw: back off if error repo is empty
+    
+    Don't check it every time
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dbf6dcb29faa853c2439457c767d550d5fcdc0f2)
+
+commit 8dcd2a1d93f08db6ede05b3028c3fc601defe932
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 11:13:48 2016 -0700
+
+    rgw: data sync retries sync on prevously failed bucket shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f1ccc4cd973d16e7676b2374eeefe4ee6f6a4630)
+
+commit 3e5210ddedd1d98473e0bcff04a03958b0b850fc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 15:59:27 2016 -0700
+
+    rgw: store failed data sync entries in separate omap
+    
+    so that we can reiterate over them
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7deb7cb09304f1b0963139296bdb3abb22895ff)
+
+commit d08ca528ea869cf5a899fd684caecf030b96f5fe
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 11:20:49 2016 -0700
+
+    rgw: configurable window size to RGWOmapAppend
+    
+    We want to be able to disable buffering for certain operations.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 94ff675e2e7763650d14f62aaf1ff9ddb05cc380)
+
+commit 368e88409e57d6827c7b10f72761d0320cdb4dc2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 12 09:14:29 2016 -0700
+
+    rgw: add a cr for omap keys removal
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20c1b4b8a3695b818c5c64d61d5ea4f84c48c782)
+
+commit 933fdefb56ce1ab883e6d3ee09c34fb2aa14808e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 16 11:03:45 2016 -0400
+
+    rgw: add_zone only clears master_zone if --master=false
+    
+    Fixes: http://tracker.ceph.com/issues/15901
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 40c4d150757cba4b05938c6c760ab9f0296c2ff6)
+
+commit e9f99166f7fc1b4b468e146a47ec9ac71541bec9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri May 20 09:52:01 2016 +0200
+
+    rgw: add missing metadata_heap pool to old zones
+    
+    Fixes: http://tracker.ceph.com/issues/15937
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit b79856bd3f7db8498231d7f35b53b4bcb44a0422)
+
+commit 22662875320717d1ee939732f9cf7939db697106
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Tue May 3 17:12:26 2016 +0800
+
+    mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 88b6d669e37924536152a552db46ef43a7353562)
+
+commit 0bdc8fd58ce790638c62bbe5264294ed5e1f710a
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 15:43:41 2016 +0530
+
+    rgw : cleanup radosgw-admin temp command as it was deprecated
+    and also implementation code for this command was removed in commit
+    8d7c8828b02c46e119adc4b9e8f655551512fc2d
+    
+    Fixes: http://tracker.ceph.com/issues/16071
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 949f917af80ae0bd9a5448129d3ce8979acf7e0b)
+
+commit 108638f366a70ca634a21b7fe4c9206368791514
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 15 16:48:26 2016 +0100
+
+    rgw: handle initial slashes properly in BulkDelete of Swift API.
+    
+    Fixes: http://tracker.ceph.com/issues/15948
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1dde062c21c3d6fa3cc01e8c922d7f89c6973bfa)
+
+commit a40cfe4df81c03b197dc31888f5d77927eeae7c5
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue May 3 14:56:38 2016 +0200
+
+    rgw: remove -EEXIST error msg for ZoneCreate
+    
+    currently for any admin operations like user create etc. you would
+    always see:
+    
+    `RGWZoneParams::create(): error creating default zone params: (17) File
+    Exists`
+    
+    in stdout as the debug level is set to 0, which doesn't make much sense
+    for an end user, so skip the error message, callers of the function handle
+    the error message anyway, so we skip it here
+    
+    Fixes: http://tracker.ceph.com/issues/15720
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 2ace9d7f349ef09d3ed87fb216cda3e305ef706f)
+
+commit ff9c29a0e7a3719858ad94f632b6d1af3f3ec73e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 16 18:24:55 2016 +0200
+
+    rgw: camelcase names of custom attributes in Swift's responses.
+    
+    Fixes: http://tracker.ceph.com/issues/15902
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit cfde4c42c0248b52ef1b855f7de79ee1e229e73f)
+
+commit 4a3c9f338fb0f2cee71f31652cf31a8ba61d5e4e
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri May 13 15:39:32 2016 +0200
+
+    rgw: fix realm pull and period pull for apache frontend
+    
+    add missing content-type and content-length headers
+    
+    Fixes: http://tracker.ceph.com/issues/15846
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8454ff1a91322697224332f0e6f0c7a9c856ec9a)
+
+commit a08caa6d3cbca0395a9ed5f487f9de33b615f59f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 10 17:45:20 2016 +0200
+
+    rgw: handle errors properly during GET on Swift's DLO.
+    
+    Backport: Jewel, Hammer
+    Fixes: http://tracker.ceph.com/issues/15812
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5ef0202a86a1e4cb56bd29d926555f114a1acdd8)
+
+commit 8163c4dd34d24249bf662c1110c49136ff335bf9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 15:34:19 2016 -0700
+
+    rgw: don't unregister request if request is not connected to manager
+    
+    That means that request is already complete and done with the manager.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 16c0605bf276b245fbf41cb3e000ecdf7b049e15)
+
+commit 70018bfe6f5e298791c34ae4ad6eac2140d6f761
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 3 17:20:04 2016 +0800
+
+    os/FileStore::sync_entry check for stop in after wait
+    
+    there is chance that the sync_entry() gets signaled in the
+    WaitInterval(max_interval) call because of sync_cond.Signal() call after
+    "stop = true" in umount(), so without this change, sync_entry() will
+    continue wait until min_interval is reached. this forces umount() to
+    wait even it has called d_force_sync(), and hence slows down the
+    progress of umount(). so we need to check for `stop` if we are not
+    signalled because of `force_sync`.
+    
+    Fixes: http://tracker.ceph.com/issues/15695
+    Reported-by: Haomai Wang <haomaiwang@gmail.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 65426a522d9d052fd7c38964f143087f277816c5)
+
+commit b2d1df129bc4403d98b66873ee6bca64ba1ea264
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Apr 6 15:39:23 2016 +0200
+
+    tests: be more generous with test timeout
+    
+    When the thrasher is in action together with a validater (lockdep or
+    valgrind), a single test may hang for more than 360 seconds. Increase to
+    1200: it does not matter if the value is large, only that it prevents
+    the test from hanging forever.
+    
+    Fixes: http://tracker.ceph.com/issues/15403
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit af89474b3fb2c4aa63680aa6b30d71fad2fdd373)
+
+commit 6c1163cd6e39ba293be1be4a3f82e60fd054a348
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Mon Apr 25 17:24:07 2016 +0200
+
+    admin-socket: use chown instead of fchown
+    
+    fchown() returns success, but does not change asok file
+    ownership. chown() does.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 3d051a58cc117ff79d4cdd768603d1dec8ca7f60)
+
+commit 3963de78635d760924a36eeb1e8b7782dd5a276e
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:55:18 2016 +0200
+
+    global-init: fixup inconsistent use of ceph ctx and conf
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 088cc66a94886130df647976050c9d98fcd7572e)
+
+commit 2bc41945ca6a8e8e8e852ae0b7e888bc223dd7a3
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:54:45 2016 +0200
+
+    global-init: chown pid files
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 9c82d8466a264c0f07fd3ec4cc58c1ddb88cc84f)
+
+commit 5681b78ac74728db70a89bc86845a74ff7d81217
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:47:58 2016 +0200
+
+    global-init: chown run dir
+    
+    Also performs log file chown() operations if permission drop has been
+    deferred. Documents that admin socket chown() happen in the common init
+    path.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 9d513793b3fe4b7e1402e36a4a675553edd6f317)
+
+commit c2075448f052ef7e49bc95c1f8240c2ff559b5ae
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:40:08 2016 +0200
+
+    common-init: chown admin socket after service thread started
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 206585c1d0fc6826c91f7830cde8ded86288abef)
+
+commit f7e6b3cf9c363a3a65da4e176c7324d027352420
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:36:01 2016 +0200
+
+    global-init: check init flags and set accordingly
+    
+    If init flags have changed between the time ceph context was
+    instantiated and now, update them. This can happen, for example, if
+    between manual global_pre_init() and global_init() calls, a daemon adds
+    modifies it's init flags.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit a0ad13db7aba95537808a570ad4c90f6ac4ca0b8)
+
+commit d4afe947578a7ae4956a99fbc9350920e2239e0b
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:35:02 2016 +0200
+
+    global-init: add a path chown wrapper function
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 860977b105595dbb50320a30dc0edb922279cb00)
+
+commit 770ae9eecf5c8e064529efec29020f38a3675a63
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:32:45 2016 +0200
+
+    ceph-context: add function to set init flags
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit aecab14089bcfd33d6a18adabede6a54040460d4)
+
+commit 21f0216287485e6cce8811f324fee804ef733524
+Author: Peter Sabaini <peter@sabaini.at>
+Date:   Fri Apr 8 00:30:55 2016 +0200
+
+    ceph-disk: Accept bcache devices as data disks
+    
+    Fixes: http://tracker.ceph.com/issues/13278
+    Signed-off-by: Peter Sabaini <peter@sabaini.at>
diff --git a/doc/changelog/v10.2.4.txt b/doc/changelog/v10.2.4.txt
new file mode 100644
index 000000000..f9b388f6c
--- /dev/null
+++ b/doc/changelog/v10.2.4.txt
@@ -0,0 +1,4586 @@
+commit 9411351cc8ce9ee03fbd46225102fe3d28ddf611
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Dec 5 22:15:20 2016 +0000
+
+    10.2.4
+
+commit a9444915a10038c35d5726485e110e56349fe3df
+Merge: 4d9a28d 4d6f848
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 5 14:50:23 2016 +0100
+
+    Merge pull request #12167 from liewegas/wip-osdmap-encoding-jewel
+    
+    jewel: osd: condition OSDMap encoding on features
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d9a28d26d84a1dab90d22a09b1a044c193a41ad
+Merge: 6d17188 d194db8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Dec 3 10:57:18 2016 +0100
+
+    Merge pull request #12067 from SUSE/wip-17953-jewel
+    
+    jewel: mon: OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit d194db826bb26483f847235bf062ba7b225be147
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Nov 14 11:50:23 2016 -0800
+
+    OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    If the osd actually restarts, there is no guarrantee that the epoch will
+    advance past up_from.  If the inst is different, it can't really be a
+    dup.  At worst, it might be a queued MOSDBoot from a previous inst, but
+    in that case, the real inst would see itself marked up, and then back
+    down causing it to try booting again.
+    
+    Fixes: http://tracker.ceph.com/issues/17899
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 033ad5b46c0492134e72a8372e44e3ef1358d2df)
+
+commit 6d17188786ef7bd574dd8336d1e97e069db9de74
+Merge: b168c26 79be070
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Dec 2 08:16:27 2016 -0800
+
+    Merge pull request #12207 from jdurgin/wip-librados-setxattr-overload-jewel
+    
+    librados: remove new setxattr overload to avoid breaking the C++ ABI
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b168c262f4c04d748b8a71aa6289e13385398764
+Merge: be5c828 00de014
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Dec 2 10:01:39 2016 +0100
+
+    Merge pull request #12267 from dachary/wip-17904-jewel
+    
+     jewel: Error EINVAL: removing mon.a at 172.21.15.16:6789/0, there will be 1 monitors
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 00de0140410ec8457ca25920866f8409f0d54a10
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:38:36 2016 +0000
+
+    mon: MonmapMonitor: drop unnecessary 'goto' statements
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 20dcb597e35e6961db81831facefbe22cecddec3)
+
+commit 25f1b390525d975f90d0c7232b785415431d0f3e
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:33:52 2016 +0000
+
+    mon: MonmapMonitor: return success when monitor will be removed
+    
+    Fixes: http://tracker.ceph.com/issues/17725
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit c9d46cfbf2512bc3495c6901de2b8f711bef9bae)
+
+commit be5c82829916c581765f7c0130b738395a27720c
+Merge: 427f357 3cc29c6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 1 11:08:04 2016 -0800
+
+    Merge pull request #12001 from dachary/wip-17915-jewel
+    
+    jewel: filestore: can get stuck in an unbounded loop during scrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 79be070a4d442229d62b168ab87c95b662df1a9c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Nov 28 22:06:56 2016 -0800
+
+    librados: remove new setxattr overload to avoid breaking the C++ ABI
+    
+    Fixes: http://tracker.ceph.com/issues/18058
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit b8ff781ddcf737882163cf56d7c9b11e815fb699)
+    
+    Conflicts:
+    	src/include/rados/librados.hpp (trivial namespace change in removed line)
+
+commit 4d6f8481b22ecfa5e55b2a6f8f8660f2d0445030
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 14:15:50 2016 -0500
+
+    crush: condition latest tunable encoding on features
+    
+    This avoids throwing hammer OSDMap encodings off.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e5ff86487bd1f5979866b5e16300dd4a3979f97)
+
+commit bf96b30eaf552148249953ed4fb654cbb101c3d0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 28 14:35:53 2016 -0500
+
+    crush/CrushWrapper: encode with features
+    
+    No behavior change yet; just fixing callers.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b7c9e055848c8aa951bc48c957cff3ef323ea441)
+    
+    [Updated write_file to use all feaetures]
+    [Updated OSDMonitor.cc to use mon->quorum_features instead of the
+     mon->get_quorum_con_featuers() helper]
+    [trivial conflict from removed write_file and read_file]
+    
+    Conflicts:
+    	src/crush/CrushWrapper.h
+    	src/mgr/PyModules.cc
+    	src/mon/OSDMonitor.cc
+    	src/tools/ceph_monstore_tool.cc
+
+commit c5f5b94fc2e18b38d05f00fec04d2d7e3c35c54c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 28 14:35:24 2016 -0500
+
+    crush/CrushWrapper: drop unused 'lean' encode() argument
+    
+    No callers, no users.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 638a38bbb55c07ad0358a35a56418e66874d1c26)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.h
+    
+    [trivial conflict due to removal of write_file and read_file]
+
+commit c66c556852b08e18d409e769eb7bd945c35e43cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 13:51:59 2016 -0500
+
+    osd/osd_types: encode pg_pool_t like hammer if features indicate hammer
+    
+    If the target features are missing the new OSDOp encoding, the
+    first feature we added post-hammer, encode like hammer.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f8cfb632823ba4e63eaff394392d6af7979d7c8)
+
+commit 85caf34924bbfb0e872abd096ec571fff73035ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 13:48:35 2016 -0500
+
+    osd/osd_types: conditional pg_pool_t encoding
+    
+    Align this with decode.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 01d9e8a20bbc3c039f67b040da95018e2c7b00b6)
+
+commit 3cc29c6736007c97f58ba3a77ae149225e96d42a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 10 13:56:24 2016 -0500
+
+    os/filestore/HashIndex: fix list_by_hash_* termination on reaching end
+    
+    If we set *next to max, then the caller (a few lines up) doesn't terminate
+    the loop and will keep trying to list objects in every following hash
+    dir until it reaches the end of the collection.  In fact, if we have an
+    end bound we will never to an efficient listing unless we hit the max
+    first.
+    
+    For one user, this was causing OSD suicides when scrub ran because it
+    wasn't able to list all objects before the timeout.  In general, this would
+    cause scrub to stall a PG for a long time and slow down requests.
+    
+    Broken by refactor in 921c4586f165ce39c17ef8b579c548dc8f6f4500.
+    
+    Fixes: http://tracker.ceph.com/issues/17859
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c5180262a086c2d3895aff4bf0fb0ff9a6666149)
+
+commit 427f357f0eed32c9ce17590ae9303a94e8b710e7
+Merge: 0c38c46 8b595f5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 9 20:53:18 2016 +0100
+
+    Merge pull request #11822 from SUSE/wip-17816-jewel
+    
+    jewel: Missing comma in ceph-create-keys causes concatenation of arguments
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b595f567407be2a759e4987f33ce79e3763ea49
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Sep 18 16:26:29 2016 -0400
+
+    ceph-create-keys: add missing argument comma
+    
+    The arguments "get" and "client.admin" were being concatenated into
+    "getclient.admin".
+    
+    Found using ceph-ansible + strace:
+    
+        13031 execve("/usr/bin/ceph", ["ceph", "--cluster=ceph", "--name=mon.", "--keyring=/var/lib/ceph/mon/ceph-ceph-mon0/keyring", "auth", "getclient.admin"], ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "LANG=en_US.UTF-8", "CLUSTER=ceph", "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728", "CEPH_AUTO_RESTART_ON_UPGRADE=no"] <unfinished ...>
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 482022233d845b75876b04ca23fb137281a9f6ab)
+
+commit 0c38c464fff2280a6345b470f1c83aa6229776cc
+Merge: eb67259 eea546f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 7 14:39:48 2016 +0100
+
+    Merge pull request #11679 from dachary/wip-17734-jewel
+    
+    jewel: Upgrading 0.94.6 -> 0.94.9 saturating mon node networking
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eea546fafcf22573e33332ce91e3d112202ac207
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat May 21 14:11:55 2016 +0800
+
+    mon: expose require_jewel_osds flag to user
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 83ffc2b761742d563777e50959faa6a6010edae0)
+
+commit f8ee076ac4559dc9dbf828121618e78ad11687fd
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 21 12:25:08 2016 -0400
+
+    mon/OSDMonitor: encode OSDMap::Incremental with same features as OSDMap
+    
+    The Incremental encode stashes encode_features, which is
+    what we use later to reencode the updated OSDMap.  Use
+    the same features so that the encoding will match!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 916ca6a0aaa32bd9c2b449e0d7fbd312c29f06e5)
+
+commit 1f629b2ba91c793db34614f319d12594646f5ce2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 13 12:16:40 2016 -0400
+
+    mon/OSDMonitor: health warn if require_{jewel,kraken} flags aren't set
+    
+    We want to prompt users to set these flags as soon as their
+    upgrades complete.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 12e508313dbd5d1d38c76859cb7de2ce22404e12)
+    
+    Conflicts:
+       src/mon/OSDMonitor.cc: remove references to kraken
+    
+        if ((osdmap.get_up_osd_features() & CEPH_FEATURE_SERVER_KRAKEN) &&
+    	!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+          string msg = "all OSDs are running kraken or later but the"
+    	" 'require_kraken_osds' osdmap flag is not set";
+          summary.push_back(make_pair(HEALTH_WARN, msg));
+          if (detail) {
+    	detail->push_back(make_pair(HEALTH_WARN, msg));
+          }
+        } else
+
+commit 34555f11b068eb335866d4b536c9e10fe1de62e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 30 18:02:39 2016 -0400
+
+    mon/OSDMonitor: encode canonical full osdmap based on osdmap flags
+    
+    If the JEWEL or KRAKEN flags aren't set, encode the full map without
+    those features.  This ensure that older OSDs in the cluster will be able
+    to correctly encode the full map with a matching CRC.  At least, that is
+    true as long as the encoding changes are guarded by those feature bits.
+    That appears to be true currently, and we plan to ensure that it is true
+    in the future as well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5e0daf6642011bf1222c4dc20aa284966fa5df9f)
+    
+    Conflicts:
+       src/mon/OSDMonitor.cc: removed reference to kraken
+    
+        if (!tmp.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+          dout(10) << __func__ << " encoding without feature SERVER_KRAKEN" << dendl;
+          features &= ~CEPH_FEATURE_SERVER_KRAKEN;
+        }
+
+commit eb672598ad8209dcb7b7454fea31f921d255af1f
+Merge: 5e079cf d48e603
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Nov 4 15:31:05 2016 +0100
+
+    Merge pull request #11742 from tchaikov/wip-17728-jewel
+    
+    jewel: test/ceph_test_msgr: do not use Message::middle for holding transient…
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5e079cfd46e4649e0eb24b15cb2c45887f5762fe
+Merge: 7b3ec11 eb30cc5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 3 15:54:28 2016 +0100
+
+    Merge pull request #11746 from liewegas/wip-post-file-key-jewel
+    
+    jewel: ceph-post-file: use new ssh key
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit eb30cc5ad6b439417298b7c615ff8ae15bf00fa2
+Author: David Galloway <dgallowa@redhat.com>
+Date:   Fri Aug 19 16:11:32 2016 -0400
+
+    ceph-post-file: Ignore keys offered by ssh-agent
+    
+    In my case, I had multiple private keys in ssh-agent which resulted in
+    the sftp connection failing despite explicitly specifying the private
+    key to use
+    
+    Signed-off-by: David Galloway <dgallowa@redhat.com>
+    (cherry picked from commit a61fcb2eac35a149b49efdc9b2ffa675afb968e8)
+
+commit 43282b0657ff19060dad25df79981ce17a76900f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 2 09:37:41 2016 -0400
+
+    ceph-post-file: migrate to RSA SSH keys
+    
+    DSA keys are being deprecated: http://www.openssh.com/legacy.html
+    
+    drop.ceph.com will continue to allow the old DSA key but eventually,
+    users submitting logs using ceph-post-file will run into issues when
+    OpenSSH completely drops support for the algorithm.
+    
+    Fixes: http://tracker.ceph.com/issues/14267
+    
+    Signed-off-by: David Galloway <dgallowa@redhat.com>
+    (cherry picked from commit ecd02bf3f1c7a07a3271b2736a9e12dd6e897821)
+    
+    # Conflicts:
+    #	src/CMakeLists.txt
+
+commit d48e603d6c6715fbc127003226d327b79f30713a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 23 18:40:57 2016 -0500
+
+    msg: adjust byte_throttler from Message::encode
+    
+    Normally we never call encode on a message that has a byte_throttler set
+    because we only use it for messages we received.  However, for forwarded
+    messages that we clear_payload() before resending, we *do* reencode, and in
+    that case we need to retake the appropriate number of bytes from the
+    throttler--just like we release them in clear_payload().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a9651282f7c16df872757b82d3d2995d92458d5c)
+
+commit 1bc616138fea897f36c1c25851f87df91404011c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 23 18:10:00 2016 -0500
+
+    msg/Message: fix set_middle vs throttler
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e7bf50b27a495ed75def67025d1ceca83861ba35)
+
+commit 5d5b5952322bb6a571c9cd58fd3e683eb32a2509
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 22 14:01:34 2016 -0400
+
+    messages/MForward: reencode forwarded message if target has differing features
+    
+    This ensures we reencode the payload with the
+    appropriate set of features if the client, us, or the
+    target do not have identical features.  Otherwise we
+    may forward an encoding with more features than the
+    target can handle.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a433455e59067a844c3df4a0d6080db2ceb4ec59)
+
+commit e068c9206a3d618b3b04975d03f61ca64a92c4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 28 11:44:28 2016 -0400
+
+    messages/MForward: fix encoding features
+    
+    We were encoding the message with the sending client's
+    features, which makes no sense: we need to encode with
+    the recipient's features so that it can decode the
+    message.
+    
+    The simplest way to fix this is to rip out the bizarre
+    msg_bl handling code and simply keep a decoded Message
+    reference, and encode it when we send.
+    
+    We encode the encapsulated message with the intersection
+    of the target mon's features and the sending client's
+    features.  This probably doesn't matter, but it's
+    conceivable that there is some feature-dependent
+    behavior in the message encode/decode that is important.
+    
+    Fixes: http://tracker.ceph.com/issues/17365
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4f5e88f36e5388ae9e062c4bc49ac1c684a3f3c)
+
+commit 158b003751d8bedafdca60d859aef67e69d9a732
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sat Jun 4 23:24:06 2016 +0100
+
+    all: add const to operator<< param
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 0a157e088b2e5eb66177421f19f559ca427240eb)
+
+commit 3e1edde98610b11b94c59d23de979d6cd79dd8fe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 29 01:54:58 2016 +0800
+
+    test/ceph_test_msgr: do not use Message::middle for holding transient data
+    
+    Message::middle is used for holding encoded data, so we we can not stuff
+    it with payload and leave the "payload" field empty. this change
+    refactors the ceph_test_msgr by introducing a Payload class which
+    encodes all test data in it.
+    
+    Fixes: http://tracker.ceph.com/issues/17728
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 56896a7ed20869ce91ade4c77c1d6cbab8d50de1)
+    Conflicts:
+    	src/test/msgr/test_msgr.cc: do not use the new-style DENC()
+    framework for implementing the encoder of Payload class. DENC() was
+    introduced after jewel was released.
+
+commit 8f75bd64768b6984c537dd15b5d9159070b86d91
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 8 23:20:58 2016 +0800
+
+    test/ceph_test_msgr: fix circular locking dependency
+    
+    * do not acquire lock when sending message
+    * remove lock in session
+    * reduce the scope guarded by locks for better performance.
+    
+    Fixes: http://tracker.ceph.com/issues/16955
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cf1801c260c42aa93850538eea7a194440ebe350)
+
+commit f960db4646a9535bcee6d53740811b84e0678c93
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Thu Jul 7 14:59:51 2016 +0800
+
+    ceph_test_msgr: use ceph log infrastructure to output
+    
+    because we want to get the right log sequence which mixes ceph logginer and
+    cerr. Otherwise, cerr output make the logs a little disordered.
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit d1268a6aa895ee93cd9fee6fc6c759317e681a85)
+
+commit 7b3ec119cb968a26526ad95355c5bf7525fb5346
+Merge: 5efb6b1 779af22
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 1 17:05:27 2016 -0700
+
+    Merge pull request #11728 from ceph/wip-librados-upgrade-jewel
+    
+    qa: remove EnumerateObjects from librados upgrade tests
+
+commit 5efb6b1c2c9eb68f479446e7b42cd8945a18dd53
+Merge: 3c9fe54 39b8e78
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 28 14:29:20 2016 +0200
+
+    Merge pull request #11669 from ceph/wip-jewel-11566
+    
+    rgw: fix put_acls for objects starting and ending with underscore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3c9fe545dccf13413bb491098c089d3e4a76bcd2
+Merge: 5ca8791 6356664
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 27 21:40:26 2016 +0200
+
+    Merge pull request #11472 from dachary/wip-17510-jewel
+    
+    jewel: ERROR: got unexpected error when trying to read object: -2
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 39b8e783defb930b1dd8eeecdfee68d0d886d03b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Oct 19 21:51:01 2016 +0200
+
+    rgw: fix put_acls for objects starting and ending with underscore
+    
+    Fixes: http://tracker.ceph.com/issues/17625
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 14d4d912c7b47c56b16ae0bdc6bc08d208de3461)
+
+commit 5ca879114fcc98b906cac64a5ef5cb2a8568cb60
+Merge: ed9a824 dc2ffda
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Oct 27 13:52:22 2016 -0400
+
+    Merge pull request #11662 from linuxbox2/jewel-17635
+    
+    jewel: rgw: handle empty POST condition
+
+commit ed9a824f2075e15ac245b9aac683cb28183ecd3a
+Merge: e08b0f3 f5e37ab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 27 16:48:43 2016 +0200
+
+    Merge pull request #11634 from dillaman/wip-17590-jewel
+    
+    jewel: journal: do not prematurely flag object recorder as closed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 779af22fc920f1fdfdd3fa2b01d8587088372bd4
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Oct 26 16:33:53 2016 -0700
+
+    qa: remove EnumerateObjects from librados upgrade tests
+    
+    These rely on new rados functionality not present in jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e08b0f308f5af45c1b1867ab5b757486bba51333
+Merge: 0aee633 4cb83c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 23:00:35 2016 +0200
+
+    Merge pull request #11644 from ceph/wip-17695
+    
+    jewel: librbd: discard after write can result in assertion failure
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4cb83c14dbe09d4b371f7b728d9b5c0549e59f1a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 25 09:43:06 2016 -0400
+
+    librbd: discard after write can result in assertion failure
+    
+    With journaling enabled, the proper lock is not held when handling
+    a discard after write to overlapping extents. This issue is only present
+    on the jewel branch due to design changes on the master branch.
+    
+    Fixes: http://tracker.ceph.com/issues/17695
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dc2ffda7819d2ebeed3526d9e6da8f53221818de
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 20 10:17:36 2016 -0700
+
+    rgw: handle empty POST condition
+    
+    Fixes: http://tracker.ceph.com/issues/17635
+    
+    Before accessing json entity, need to check that iterator is valid.
+    If there is no entry return appropriate error code.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 23cb642243e09ca4a8e104f62a3bb7b2cbb6ea12)
+
+commit 0aee6331ad93a3caf212b84412bb648171758fff
+Merge: 06f7d7a cd99a64
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 16:47:03 2016 +0200
+
+    Merge pull request #11657 from dachary/wip-17707-jewel
+    
+    jewel: ceph-disk: using a regular file as a journal fails
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 06f7d7a9b2403cf8b7af5301dae575d6f21c71ce
+Merge: 7ff2c10 7964187
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 14:04:20 2016 +0200
+
+    Merge pull request #11321 from linuxbox2/jewel-11051
+    
+    jewel: rgw: Do not archive metadata by default
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 7ff2c108ea12451b6dcb25dab4574f2de68162d1
+Merge: 0fb486d 2f9a5be
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 11:55:49 2016 +0200
+
+    Merge pull request #11626 from ceph/wip-jewel-11567
+    
+    jewel: don't loop forever when reading data from 0 sized segment.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0fb486d59e1574e936564e4b048a089dee8b267b
+Merge: bf5c47a 86eef27
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 11:39:40 2016 +0200
+
+    Merge pull request #11478 from dachary/wip-17312-jewel
+    
+    jewel: build/ops: allow building RGW with LDAP disabled
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cd99a642a088c7cd010e45a81d6680cdfca16ca6
+Author: Jayashree Candadai <jayaajay@indiana.edu>
+Date:   Fri Oct 21 13:52:05 2016 -0400
+
+    ceph-disk: allow using a regular file as a journal
+    
+    Because of a missing return, ceph-disk prepare would fail if given a
+    regular file as a journal. If the journal file does not exist, ceph-disk
+    will create it but fail to ensure that the ceph user owns it. The
+    symlink to the journal file is not set when the journal file is
+    specified on the command line and the journal file does not exist at
+    all. The ceph-osd daemon will silently create it as a file but it will
+    not be the file given in argument.
+    
+    Add a test case to verify using a regular file as a journal works as
+    expected.
+    
+    Fixes: http://tracker.ceph.com/issues/17662
+    
+    Signed-off-by: Jayashree Candadai <jayaajay@indiana.edu>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit db917d50eb5f86a07a5487e130f46a6b1d27672a)
+
+commit a80040230cddc395809d2323392c87a4a1fef923
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:19:33 2016 +0530
+
+    ceph-disk: PEP8ify
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit f7a15ccfa2cc214c3688a2b41ac45be1abfb7700)
+    
+    Conflicts:
+    	src/ceph-disk/setup.py: python 2.7 restriction not backported
+
+commit e200b17016a1a7cd103affde2d2b265916755498
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:15:25 2016 +0530
+
+    ceph-disk: Set space_symlink to the path, not file object
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit d290454cf5f660c1681eefd70a38e79da683525f)
+
+commit 9191825c06b3d92e2caa26c9efdb69a79cc3964b
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:13:53 2016 +0530
+
+    ceph-disk: Use os.path.isabs instead of .startswith('/')
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit 6f795356de00dd7c33282183b11a03da800fde7b)
+
+commit f1c2de768535ce18259d4fc187cd370766c1e6cf
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Sun Jun 19 05:10:27 2016 +0530
+
+    ceph-disk: Use context manager with FileLock
+    
+    acquire and release methods of FileLock are dropped
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit c1011d514ce0c7d340a5acd6f9c640165e169156)
+
+commit 26cb5b6bfbd6eee46bfa3cf005b1cd9f06b0e262
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Sun Jun 19 04:05:42 2016 +0530
+
+    ceph-disk: Fix bug in FileLock
+    
+    Python fcntl.lockf() accepts a file descriptor, not a file object
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit df9cc2ce938a969f4044b63fd80030d00f64f060)
+
+commit 91a9ca374f646966e31b015928a2cc60bc5225aa
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Thu Jun 16 16:37:00 2016 +0530
+
+    ceph-disk: Use true integer division in get_dev_size
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit f1bb72c82806cc03e85e0b19c83d61409c6b2d51)
+
+commit 2373ccb7a991069406e344b91fa70c0b2d1aa5e0
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Tue Jun 14 21:42:02 2016 +0530
+
+    ceph-disk: Compatibility fixes for Python 3
+    
+    ceph-disk: Misc cleanups
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit d0e29c74f84a2ed3014a516c0106172619314bdc)
+    
+    Conflicts:
+         src/ceph-disk/tox.ini: python3 is not supported in jewel
+
+commit 79cf6330d67ad52af5690f4d4efd29aa2722acb2
+Author: Shylesh Kumar <shmohan@redhat.com>
+Date:   Thu Jul 7 20:45:57 2016 +0530
+
+    ceph-disk: change ownership of init file to ceph:ceph
+    
+    Fixes: http://tracker.ceph.com/issues/16280
+    Signed-off-by: Shylesh Kumar <shmohan@redhat.com>
+    (cherry picked from commit aab9d03e1b50ba10a383663088400b9fabe306cb)
+
+commit 796418746ecd2a4971e365499c6952e131e74913
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Mon Sep 12 14:30:43 2016 +0530
+
+    rgw: Do not archive metadata by default
+    
+    Fixes: http://tracker.ceph.com/issues/17256
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    
+    (cherry picked from commit c617ea83e0cca0061af18e0811c7ef8b4e836519)
+
+commit bf5c47a28abe6d91dfdace8d6803a7016bbf02fb
+Merge: 7714689 1eedf18
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 25 15:49:55 2016 +0200
+
+    Merge pull request #11642 from tchaikov/wip-17685-jewel
+    
+    jewel: mon: fix missing osd metadata (again)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 77146891ab254bc796e38e2a45b8500c0119745f
+Merge: 3ccc17b e8e1acb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 25 15:40:01 2016 +0200
+
+    Merge pull request #11467 from dachary/wip-17262-jewel
+    
+    jewel: rbd-nbd IO hang
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1eedf181176ce75212a8cfbb87ac927350a826ed
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed May 25 17:56:51 2016 +0100
+
+    mon: fix missing osd metadata (again)
+    
+    The JSON output was getting broken by continuing
+    in the wrong place.
+    
+    Fixes: http://tracker.ceph.com/issues/17685
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c5700ce4b45b3a385fe4c2111da852bea7d86da2)
+
+commit f5e37abefcb015b2cb58295cfdf109fd6f5833b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 17 09:48:20 2016 -0400
+
+    journal: do not prematurely flag object recorder as closed
+    
+    Fixes: http://tracker.ceph.com/issues/17590
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2be6367cb0119d86dfecfa94eb4a3a102c40162a)
+
+commit 2f9a5beb9a903220b70a780b30564d9741e9f0e5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Oct 13 21:12:36 2016 -0400
+
+    Don't loop forever when reading data from 0 sized segment.
+    
+    The 0 sized segment can arise depending on how
+    the client uploads the object in the first place.
+    The cpu loop then happens during a swift `GET'.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46c5f9773246522e66bb2cca49345d0b62a16c42)
+
+commit 3ccc17b81d2794406d803ff4210e930a1fa67455
+Merge: 112b89d 0b30a1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 12:03:13 2016 +0200
+
+    Merge pull request #10757 from dachary/wip-17056-jewel
+    
+    jewel: mon/osdmonitor: decouple adjust_heartbeat_grace and min_down_reporters
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 112b89dc315dd1240d68eb50fba1438057002327
+Merge: 7ab1e02 0dcefd2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 12:03:01 2016 +0200
+
+    Merge pull request #10759 from dachary/wip-16866-jewel
+    
+    jewel: OSD: ceph osd df does not show summarized info correctly if one or more OSDs are out
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7ab1e0275101a30c2d318bcd55787f61c687328f
+Merge: e053b20 4f9e02c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 11:55:41 2016 +0200
+
+    Merge pull request #11590 from dachary/wip-17642-jewel
+    
+    jewel: TestJournalReplay: sporadic assert(m_state == STATE_READY || m_state == STATE_STOPPING) failure
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4f9e02c40ac5359f344d29f20332ec6ed8575b8e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 13 10:04:24 2016 -0400
+
+    test: TestJournalReplay test cases need to wait for committed journal event
+    
+    Fixes: http://tracker.ceph.com/issues/17566
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1bdf7a425eb7436838c9445ea2c8ab42dfd3a1b6)
+
+commit e8e1acb1d5154b749d251efa88b45e8ad3edb2bb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 8 11:51:34 2016 -0400
+
+    librbd: ignore cache busy errors when shrinking an image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4ce663845679dc35f2f15b893c6f988c4a60b25b)
+    
+    Conflicts:
+         src/test/librbd/operation/test_mock_ResizeRequest.cc:
+         when_resize does not have the allow_shrink argument because
+         d1f2c557b2c039730baca9efa3f5244bc19dcb1a has not been
+         backported
+
+commit ba2e87e0443069b0552b698cc7a508898a3f585d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 8 09:59:37 2016 -0400
+
+    librbd: invalidate cache before trimming image
+    
+    Any potential writeback outside the extents of a shrunk image
+    would result in orphaned objects.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f93a1917437ba7b69b306e4ff971b79e8b79c89)
+
+commit d7c0873ea77234b7b736080c3de4012d2f6adaee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 7 11:49:26 2016 -0400
+
+    rbd-nbd: mask out-of-bounds IO errors caused by image shrink
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6cfb616a846959a2cd1c0e540b14668c61a2afd)
+
+commit 0ce342d39f359d23c92ab94efb910b84a634094a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 5 10:01:45 2016 -0400
+
+    rbd-nbd: fix kernel deadlock during teuthology testing
+    
+    Fixes: http://tracker.ceph.com/issues/16921
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce7c1520a8019f011fa34dd898af317f78974577)
+
+commit e053b2085b0d0be0e8c9912c82c7142bb0871783
+Merge: f6d7290 caf08d7
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:17:49 2016 -0400
+
+    Merge pull request #11519 from dachary/wip-17576-jewel
+    
+    jewel: RGW loses realm/period/zonegroup/zone data: period overwritten if somewhere in the cluster is still running Hammer
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f6d729082359b450950513080c9c08461d6f557c
+Merge: 9e9a38d 40689d1
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:17:22 2016 -0400
+
+    Merge pull request #11471 from dachary/wip-17511-jewel
+    
+    jewel: s3tests-test-readwrite failing with 500
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9e9a38df303749147a638f0df42ad875c829d294
+Merge: 9b0fa2b 5a53ffa
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:16:38 2016 -0400
+
+    Merge pull request #11469 from dachary/wip-17538-jewel
+    
+    jewel: rgw:user email can modify to empty when it has values
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9b0fa2b27386dd0c09aead832c9f049ef10b26fb
+Merge: 0eac635 4babd3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:31:10 2016 +0200
+
+    Merge pull request #11466 from dachary/wip-17290-jewel
+    
+    jewel: ImageWatcher: use after free within C_UnwatchAndFlush
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0eac63545da063a8bbab4b1ea6c9a765023617fc
+Merge: 51d5ed0 b73356b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:53 2016 +0200
+
+    Merge pull request #11464 from dachary/wip-17373-jewel
+    
+    jewel: image.stat() call in librbdpy fails sometimes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51d5ed0ddbe7e1018be1fa46c912d2e0ca07a984
+Merge: d2150fb b410c8e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:44 2016 +0200
+
+    Merge pull request #11463 from dachary/wip-17384-jewel
+    
+    jewel: helgrind: TestLibRBD.TestIOPP potential deadlock closing an image with read-ahead enabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d2150fb54e894039e03855352708fad6b08d75d0
+Merge: f8989ef 32d84e0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:32 2016 +0200
+
+    Merge pull request #11462 from dachary/wip-17404-jewel
+    
+    jewel: update_features API needs to support backwards/forward compatibility
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f8989ef7a79dd37f58fc9bddbd1e3fcd83e14f42
+Merge: ac2ce6d 7d6801b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:28:18 2016 +0200
+
+    Merge pull request #11459 from dachary/wip-17483-jewel
+    
+    jewel: RBD should restrict mirror enable/disable actions on parents/clones
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ac2ce6dcda27a70a45e17270cab4d5e9a856abc7
+Merge: cd48521 391936a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:27:36 2016 +0200
+
+    Merge pull request #11460 from dachary/wip-17482-jewel
+    
+    jewel: Enable/Disable of features is allowed even the features are already enabled/disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cd48521cde8bb69b6879977f51404a3014571a84
+Merge: 410a368 dd93e00
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:26:53 2016 +0200
+
+    Merge pull request #11461 from dachary/wip-17481-jewel
+    
+    jewel: Proxied operations shouldn't result in error messages if replayed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 410a368e9bdf31f6512be5e9a783c7990ba9003f
+Merge: 07a8d8a d66bb7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 12:42:01 2016 +0200
+
+    Merge pull request #11431 from ceph/wip-jewel-tarball
+    
+    jewel: build/ops: include more files in "make dist" tarball
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 07a8d8aedfa101a52e4764cc211fd2a1d8f36582
+Merge: 62f7f64 50fd48f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:48:49 2016 +0200
+
+    Merge pull request #11474 from dachary/wip-17350-jewel
+    
+    jewel: rgw:response information is error when geting token of swift account
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 62f7f646b85f81c0c812f3d5adc1926d4cee3f7d
+Merge: 638590c 91bd342
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:47:43 2016 +0200
+
+    Merge pull request #11492 from SUSE/wip-17575-jewel
+    
+    jewel: aarch64: Compiler-based detection of crc32 extended CPU type is broken
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 638590c97a5365df118c72aab2e510ffcd2a9470
+Merge: c58d626 6575545
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:28:29 2016 +0200
+
+    Merge pull request #11475 from dachary/wip-17349-jewel
+    
+    jewel: Modification for TEST S3 ACCESS section in INSTALL CEPH OBJECT GATEWAY page
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c58d62677449b08023b761762a52cc4ceaf69def
+Merge: 3cb4bf4 7423661
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:27:54 2016 +0200
+
+    Merge pull request #11473 from dachary/wip-17509-jewel
+    
+    jewel: Config parameter rgw keystone make new tenants in radosgw multitenancy does not work
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3cb4bf43b923a71872525b0222b4b0fc73664fb0
+Merge: fb74b16 de672a0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:27:18 2016 +0200
+
+    Merge pull request #11201 from ceph/wip-backport-logrotate-jewel
+    
+    jewel: build/ops: backport 'logrotate: Run as root/ceph'
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit fb74b1688603c0351e80ed504be4faad320a67e6
+Merge: 9d3373d 25a35d4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:22:45 2016 +0200
+
+    Merge pull request #11126 from tchaikov/wip-17179-jewel
+    
+    jewel: add a tool to rebuild mon store from OSD
+    
+    Reviewed-by: huanwen ren <ren.huanwen@zte.com.cn>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9d3373d71f8af16a8f0220aa0d0682e2f3bb5a8c
+Merge: 0c83eb3 eb6c3cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 09:39:15 2016 +0200
+
+    Merge pull request #11563 from ceph/wip-jewel-acl-underscore
+    
+    rgw: fix regression with handling double underscore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit eb6c3cbcc6cadd4eff9de0d6332f42c785486f7b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 18:35:59 2016 -0700
+
+    rgw: set correct instance on the object
+    
+    Fixes: http://tracker.ceph.com/issues/17443
+    
+    This was broken by commit bc840afafdfe5e528e5c1b711e71420ac3cb5a67
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d2ada35c8c0d938a704ecf9974988ea30a9c0105)
+
+commit 0c83eb355e989fb6ed38a3b82f9705fd5d700e89
+Merge: 7496388 f400ff2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:14:23 2016 +0200
+
+    Merge pull request #11548 from dachary/wip-17609-jewel
+    
+    jewel: tests: ceph-disk must ignore debug monc
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7496388468688763cab7fab9cc2f61cfaeeb4e7c
+Merge: f8055a9 d205b74
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:11:24 2016 +0200
+
+    Merge pull request #11411 from dachary/wip-17245-jewel
+    
+    jewel: tests: scsi_debug fails /dev/disk/by-partuuid
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f8055a9453e228079a8c61ba398e93e9fbdcf55a
+Merge: 3b2d360 dea93dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:11:04 2016 +0200
+
+    Merge pull request #10884 from dachary/wip-17149-jewel
+    
+    jewel: ceph-disk: expected systemd unit failures are confusing
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 084108e440b7b9b8f0d36282dd4eba64a4b96f4b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Aug 31 22:25:57 2016 +0200
+
+    rgw: fix regression with handling double underscore
+    
+    Fixes: http://tracker.ceph.com/issues/16856
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bc840afafdfe5e528e5c1b711e71420ac3cb5a67)
+
+commit 3b2d36039305bc070ac1eaf5142d5b5bfa7accb1
+Merge: d15d6dc 483d8c4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 16:50:44 2016 +0200
+
+    Merge pull request #11408 from dachary/wip-17345-jewel
+    
+    jewel: Ceph Status - Segmentation Fault
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d15d6dc896d0ac30ddff4354600d7c2ecb61b54a
+Merge: d4197dc 5ae4f31
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 16:50:22 2016 +0200
+
+    Merge pull request #11407 from dachary/wip-17360-jewel
+    
+    jewel: ceph-objectstore-tool crashes if --journal-path <a-directory>
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d4197dcae43b073d121c4b12ff7cca2f619745e1
+Merge: 54bb909 c94244d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 09:47:00 2016 +0200
+
+    Merge pull request #11158 from dillaman/wip-rbdmap-jewel
+    
+    jewel: systemd: add install section to rbdmap.service file
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 54bb9092a8d658ab2f352ed8c850d719b2468d83
+Merge: dc59575 c3c2910
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 09:46:19 2016 +0200
+
+    Merge pull request #10862 from dachary/wip-17095-jewel
+    
+    jewel: rpm: ceph installs stuff in %_udevrulesdir but does not own that directory
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f400ff222e531feb00de3ebf1860a0d4bcca736e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Oct 18 17:33:23 2016 +0200
+
+    tests: ceph-disk: force debug monc = 0
+    
+    The sh function will collect both stderr and stdout and debug
+    will mess the json parsing.
+    
+    Fixes: http://tracker.ceph.com/issues/17607
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 469a53a4adc335ea4ff8e34a958b12f0f222a6c1)
+
+commit dc59575b2fec477bdcf152f91a02532fcdfd1a10
+Merge: 778d884 c2cf21d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 08:56:46 2016 +0200
+
+    Merge pull request #10860 from dachary/wip-17121-jewel
+    
+    jewel: the %USED of ceph df is wrong
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 778d884512d2cf298c60634cda9a6f66dc2b948b
+Merge: 82edc5b a25a483
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 01:35:13 2016 +0200
+
+    Merge pull request #10886 from dachary/wip-17144-jewel
+    
+    jewel: mark_all_unfound_lost() leaves unapplied changes
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 82edc5bc526e02d7c535d01642d8ef7f00d19939
+Merge: c4f4f8a e6ac214
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 18 13:05:43 2016 +0200
+
+    Merge pull request #10784 from dachary/wip-17067-jewel
+    
+    jewel: Request exclusive lock if owner sends -ENOTSUPP for proxied maintenance op
+    
+    Reviewed-by: Ilya Dryomov <idryomov@redhat.com>
+
+commit 25a35d43a8bf7e1a7d78d5d1d2a5556dff98f5e8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 18:43:39 2016 +0800
+
+    doc: fill keyring with caps before passing it to ceph-monstore-tool
+    
+    to make sure the recovered monitor store is ready for use.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af8e21163735377071b6832d8a81b035bb835257)
+
+commit 73ea9264803bdf53a5da64992c1b91e94633f5e3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 16:32:27 2016 +0800
+
+    tools/ceph_monstore_tool: bail out if no caps found for a key
+    
+    we take it as an error if no caps is granted to an entity in the
+    specified keyring file when rebuilding the monitor db.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b4bd4004a836121c11b0bb97d8123df54c271f04)
+
+commit 4ebf87bccdb17bd04b2c615e6278a7816ae1ff43
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 30 17:58:14 2016 +0800
+
+    tools/ceph_monstore_tool: update pgmap_meta also when rebuilding store.db
+    
+    we should rebuild pgmap_meta table from the collected osdmaps
+    
+    Fixes: http://tracker.ceph.com/issues/17400
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cdfa7a69f63d047205dcfccd63b5d58ab0d4695b)
+
+commit d2deed6bfbc5693e3bd8a10a2f9be8df8ef1b0e5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 11:40:56 2016 +0800
+
+    tools/rebuild_mondb: kill compiling warning
+    
+    As follow:
+    
+    [ 72%] Building CXX object src/tools/CMakeFiles/ceph-objectstore-tool.dir/RadosDump.cc.o
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc: In function ‘int update_mon_db(ObjectStore&, OSDSuperblock&, const string&, const string&)’:
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:289:22: warning: ‘crc’ may be used uninitialized in this function [-Wmaybe-uninitialized]
+             if (have_crc && osdmap.get_crc() != crc) {
+                          ^
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:238:14: note: ‘crc’ was declared here
+         uint32_t crc;
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit f16a31476a3f9b44a7c3dabf0dfd2a0d015b11b9)
+
+commit 09701269de225e556099b9e5c511faa44acae024
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:33:56 2016 +0800
+
+    tools/rebuild_mondb: avoid unnecessary result code cast
+    
+    In general we return negative codes for error cases, so there is
+    no need perform the cast here.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6a1c01d334fe65124043aa68a6e0cfaea43836b5)
+
+commit 5191b06294cf505716d0c64ac36528e9ea57b0a8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 1 14:18:55 2016 +0800
+
+    doc: add rados/operations/disaster-recovery.rst
+    
+    document the process to recover from leveldb corruption.
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 79a9f299253e24d20547131b3c9c9e0667e3b869)
+    Conflicts:
+            src/tools/rebuild_mondb.cc:
+    		remove the code change in this file from this commit.
+    		and the code gets removed is added in anther commit.
+
+commit 8c8d5ce529fa826bc0d453edf5fb5e98e29294d3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:53:11 2016 +0800
+
+    tools/ceph_monstore_tool: add "rebuild" command
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d909fa035c8fbbded786b2ca072acc10ea6b6052)
+
+commit 1fcb0bb81a8989b31a79bc565354f6817ceb12d7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:52:19 2016 +0800
+
+    tools/ceph-objectstore-tool: add "update-mon-db" command
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+    	src/tools/CMakeLists.txt: this file was added in master, so
+    		update src/CMakeLists.txt instead
+    	src/tools/Makefile-server.am: jewel is still using autotools,
+    		so update this file also.
+            src/tools/rebuild_mondb.cc: move the code spilled into
+                    doc/rados/troubleshooting/troubleshooting-mon.rst
+                    by accident back to this commit.
+    (cherry picked from commit 24faea7ce446bbf09cbd4a9d3434dd5444a6c295)
+
+commit 416750258fb63064a36eaf53f586a51fc3ea63e0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 13:11:24 2016 +0800
+
+    mon/AuthMonitor: make AuthMonitor::IncType public
+    
+    so ceph-objectstore-tool is able to use it when rebuilding monitor
+    db.
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 19ef4f16b3aba04119ac647cd6261c74a57ce829)
+
+commit c4f4f8a72421e9c43c09eb9dfb5a3c6fe2123886
+Merge: ded7c27 5ef9085
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 17 11:56:30 2016 -0700
+
+    Merge pull request #10885 from dachary/wip-17145-jewel
+    
+    jewel: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (*this == hobject_t(hobject_t::get_max())))
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ded7c27a2d0fc0ae307321de2bf419e065c36ba1
+Merge: bdcdedd 58b7c52
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 17 11:54:29 2016 -0700
+
+    Merge pull request #10883 from dachary/wip-17141-jewel
+    
+    jewel: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit bdcdedd32f9e8a71735ae1d4d3ca18b41bde3378
+Merge: 757f8e8 3bb2a9e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 18:08:46 2016 +0200
+
+    Merge pull request #11420 from dachary/wip-17556-jewel
+    
+    jewel: librbd::Operations: update notification failed: (2) No such file or directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 757f8e87e0aa39af05f0bc7b8146c826191e8032
+Merge: 4d2bc33 bd63666
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 18:08:32 2016 +0200
+
+    Merge pull request #10857 from dachary/wip-16984-jewel
+    
+    jewel: Disabling pool mirror mode with registered peers results orphaned mirrored images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d2bc33101aaadbab73cdae3327fc0888a15e25a
+Merge: 23d91cf 18a66cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:55:10 2016 +0200
+
+    Merge pull request #10645 from dachary/wip-16458-jewel
+    
+    jewel: Potential crash during journal::Replay shut down
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 23d91cfcd1c77766972577639ef72db353076e37
+Merge: 51c2794 775c78d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:54:55 2016 +0200
+
+    Merge pull request #10652 from dachary/wip-16951-jewel
+    
+    jewel: ceph 10.2.2 rbd status on image format 2 returns (2) No such file or directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51c279425e26626884ec54add6043c7ed3873920
+Merge: ee32be2 bb81f9d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:37:41 2016 +0200
+
+    Merge pull request #11337 from SUSE/wip-17060-jewel
+    
+    jewel: librbd: cannot disable journaling or remove non-mirrored, non-primary image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ee32be2ae3c328238fd2bbb953c18d3b9f0a51b1
+Merge: 7d0dd1c 2b27212
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:37:06 2016 +0200
+
+    Merge pull request #10650 from dachary/wip-16868-jewel
+    
+    jewel: Prevent the creation of a clone from a non-primary mirrored image
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7d0dd1cf6f6568f86fa593de0a7845462a510d31
+Merge: f1c21c6 1c76ef4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:06:51 2016 +0200
+
+    Merge pull request #11433 from dillaman/wip-17416-jewel
+    
+    jewel: rbd-mirror: improve resiliency of stress test case
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f1c21c6f01e20949753198e5f3ad160e0213cc5b
+Merge: 986a8b5 80e25b0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:04:11 2016 +0200
+
+    Merge pull request #10796 from dillaman/wip-17059-jewel
+    
+    jewel: rbd: bench io-size should not be larger than image size
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 986a8b5eacbf0d6084121eef8407088a24d27ffd
+Merge: c7b7f2e 10e603b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:28:06 2016 +0200
+
+    Merge pull request #11414 from dachary/wip-17477-jewel
+    
+    jewel: Crash in Client::_invalidate_kernel_dcache when reconnecting during unmount
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c7b7f2e8c0de596255e651ae5e499c586f1e4493
+Merge: ada7edd 3320da0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:53 2016 +0200
+
+    Merge pull request #11412 from dachary/wip-17479-jewel
+    
+    jewel: Duplicate damage table entries
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ada7edd1d21073b71c1d4538ab66b15b978cf418
+Merge: 1412b17 3a79db1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:43 2016 +0200
+
+    Merge pull request #11415 from dachary/wip-17476-jewel
+    
+    jewel: Failure in snaptest-git-ceph.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1412b171c057fabd05abaf9adcf31aa29a4f5b00
+Merge: 368c96c 0a17741
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:30 2016 +0200
+
+    Merge pull request #11416 from dachary/wip-17474-jewel
+    
+    jewel: Failure in dirfrag.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 368c96c513cd5424d7935fc02edd6297fb680bb7
+Merge: 4486e3b 2c4e1c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:17 2016 +0200
+
+    Merge pull request #11418 from dachary/wip-17246-jewel
+    
+    jewel: Log path as well as ino when detecting metadata damage
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4486e3b5e00e809bc3391613cc5160810ce5bf1e
+Merge: b174220 5173563
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:04 2016 +0200
+
+    Merge pull request #11419 from dachary/wip-17244-jewel
+    
+    jewel: Failure in snaptest-git-ceph.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b1742204e10d563573bbda82917b843c01b20420
+Merge: 54240c4 c0db9fb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:26:30 2016 +0200
+
+    Merge pull request #10877 from dachary/wip-16946-jewel
+    
+    jewel: client: nlink count is not maintained correctly
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 54240c4d3dfeef97c6fba343fe501f0303c73f1b
+Merge: 8b5aa5d 430ab1b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 13:40:13 2016 +0200
+
+    Merge pull request #10758 from dachary/wip-17007-jewel
+    
+    jewel: ceph-disk should timeout when a lock cannot be acquired
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit caf08d731c7cec40fe6635189de877d1f047e8b4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 14:18:29 2016 +0200
+
+    rgw: avoid corruption when running old radosgw-admin on a newer rgw
+    
+    Fixes:http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8eab9454b4d45ed8092a1292bd1904ef928c2120)
+
+commit e5f7854ddfc2a6c123d77b1d33e659b4f870fbe6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 11:19:01 2016 +0200
+
+    rgw: Fix missing master zone for default region conversion
+    
+    Fixes:http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit cd3bccdef2ecd1d294ab1192ab3b9e731b5592e0)
+
+commit ec2fb022307eea93a802f80307b957bfb2581249
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 11:00:00 2016 +0200
+
+    rgw: mark configuration as converted to avoid corruption when running older admins
+    
+    Fixes: http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ceafa2863dd9c02da8a30c5a5324b472ed5d3404)
+
+commit 8b5aa5d94fd24793c160372f02b02735632b5fd0
+Merge: 6baac41 2311ab2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 18:13:09 2016 +0200
+
+    Merge pull request #10864 from dachary/wip-17131-jewel
+    
+    jewel: Jewel: segfault in ObjectCacher::FlusherThread
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 6baac417eb95df880b97cdc720e6b00f6f8f27d8
+Merge: 19922c5 ca8fc6f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:26:32 2016 -0500
+
+    Merge pull request #11417 from dachary/wip-17347-jewel
+    
+    jewel: ceph-create-keys: sometimes blocks forever if mds allow is set
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 19922c5b818e82c297dfc0d2eb914276945e9a2e
+Merge: 952a81a 35660d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:25:42 2016 -0500
+
+    Merge pull request #11272 from aiicore/wip-17402-jewel
+    
+    jewel: OSDMonitor: Missing nearfull flag set
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 952a81aed3bae88239e448fcae1ce73e725d9966
+Merge: e34b7da 165e5ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:24:42 2016 -0500
+
+    Merge pull request #11193 from SUSE/wip-17377-jewel
+    
+    jewel: LIBRADOS modify Pipe::connect() to return the error code
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e34b7dad7e11f1db45f2bd636b1f3ed73742540a
+Merge: f9c969e 7d92e2e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:23:30 2016 -0500
+
+    Merge pull request #10861 from dachary/wip-17135-jewel
+    
+    jewel: ceph mon Segmentation fault after set crush_ruleset ceph 10.2.2
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f9c969eb54fef6f474bc8f784735123039ffeccc
+Merge: 4bcc21b 117aa35
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:22:05 2016 -0500
+
+    Merge pull request #10855 from dachary/wip-16657-jewel
+    
+    jewel: i386 tarball gitbuilder failure on master
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4bcc21b9288ed046a2781e95e781bfc71636b9c1
+Merge: 92a3538 5d0e2f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 15:37:25 2016 +0200
+
+    Merge pull request #11409 from dachary/wip-17341-jewel
+    
+    jewel: librados memory leaks from ceph::crypto (WITH_NSS)
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 92a353847852dbec003e4795e3322d2a92752b9d
+Merge: 67d0ded 8833c64
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:45:46 2016 +0200
+
+    Merge pull request #11311 from dzafman/wip-scrub-boundary-jewel
+    
+    jewel: osd: adjust scrub boundary to object without SnapSet
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 67d0ded8d3ceda56ab7c0aaca25f3c1c51731c72
+Merge: 52e596f 9cb45e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:43:27 2016 +0200
+
+    Merge pull request #11231 from badone/wip-17376
+    
+    jewel: common: Log.cc: Assign LOG_INFO priority to syslog calls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 52e596ffe0b8a83bfeae0ad836227a73d7d6facb
+Merge: 65e8bbc cca589f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:30:05 2016 +0200
+
+    Merge pull request #9872 from odivlad/fix-init-el7-jewel
+    
+    jewel: remove SYSTEMD_RUN from initscript
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 91bd342df6de15263004f3a41c285f6658a16d45
+Author: Alexander Graf <agraf@suse.de>
+Date:   Mon Sep 26 10:26:30 2016 +0200
+
+    AArch64: Detect crc32 extension support from assembler
+    
+    The used compiler may or may not be recent enough to recognize the
+    crc32 extended cpu type. However, it does not really have to know about
+    them either, since all we do is pass inline assembly instructions to
+    the assembler.
+    
+    This patch moves the crc cpu extension detection from compiler based
+    to assembler based, so that we can build optimized code even when the
+    compiler does not know about the cpu type yet.
+    
+    Signed-off-by: Alexander Graf <agraf@suse.de>
+    
+    (manual backport of e70ab48b7f6d39a281b3ec65098535a55018b681 - manual backport
+    was undertaken because jewel uses autotools)
+
+commit 65e8bbccdb73cc6696736367a3df59ea3dcfda1f
+Merge: 7c9f1c7 086f6e0
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:52:30 2016 -0400
+
+    Merge pull request #10891 from dachary/wip-16793-jewel
+    
+    jewel: rgw: upgrade from old multisite to new multisite fails
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7c9f1c7940c77451b2d44ef22a278fdfb2938cea
+Merge: e3839ab 27626ba
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:59 2016 -0400
+
+    Merge pull request #10889 from dachary/wip-17143-jewel
+    
+    jewel: rgw: rgw file uses too much CPU in gc/idle thread
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e3839ab5ce63abd248d886dcf362f3a6c49f382e
+Merge: 0561550 416ec6f
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:38 2016 -0400
+
+    Merge pull request #10868 from dachary/wip-17064-jewel
+    
+    jewel: rgw: radosgw daemon core when reopen logs
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 05615509d946c340d03af7e407a31e3e53447ac8
+Merge: abc8e3d f034fd0
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:17 2016 -0400
+
+    Merge pull request #10867 from dachary/wip-17118-jewel
+    
+    jewel: rgw： period commit return error when the current period has a zonegroup which doesn't have a master zone
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit abc8e3dd0388778d01a19f3de15a1ffcb662549f
+Merge: 8adc55d 83a91ba
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 17:43:52 2016 +0200
+
+    Merge pull request #11367 from linuxbox2/jewel-s3-versioning
+    
+    jewel: rgw: S3 object versioning fails when applied on a non-master zone
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8adc55d0d3f98acde4d91ba6f8ab9fb379ec10d9
+Merge: 50404ae 77ea511
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:43:04 2016 +0200
+
+    Merge pull request #11342 from SUSE/wip-17505-jewel
+    
+    jewel: rgw: doc: description of multipart part entity is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 50404aef1f36cd2035c4384ecdf608910e6cb215
+Merge: f1c55ad de0c4e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:42:45 2016 +0200
+
+    Merge pull request #11330 from cbodley/wip-17073
+    
+    jewel: rgw: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f1c55ad7eeeb633e5ad3414e37ebb5dacccb930b
+Merge: a336110 e2ce857
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:39:06 2016 +0200
+
+    Merge pull request #11139 from cbodley/wip-radosgw-admin-man-jewel
+    
+    jewel: add orphan options to radosgw-admin --help and man page
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a336110785ea06f4404f136dddcce61660d85b2f
+Merge: 79885ae cf47628
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:35:57 2016 +0200
+
+    Merge pull request #10832 from ceph/jewel-default-quota
+    
+    jewel: rgw - default quota fixes
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 79885ae86be1e444985446ab8315a21ad6d24634
+Merge: 9d1f756 54b6b5e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 14:58:33 2016 +0200
+
+    Merge pull request #10107 from xiaoxichen/wip-16313-jewel
+    
+    jewel: client: FAILED assert(root_ancestor->qtree == __null)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit d205b74e07ab364378afb4fc63cb08814a8a1f4e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Aug 23 12:17:00 2016 +0200
+
+    tests: populate /dev/disk/by-partuuid for scsi_debug
+    
+    The scsi_debug SCSI devices do not have a symlink in /dev/disk/by-partuuid
+    because they are filtered out by 60-persistent-storage.rules. That was
+    worked around by 60-ceph-partuuid-workaround-rules which has been
+    removed by 9f76b9ff31525eac01f04450d72559ec99927496.
+    
+    Add create rules targetting this specific case, only for tests since the
+    problem does not show in real use cases.
+    
+    Fixes: http://tracker.ceph.com/issues/17100
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7cbf1f0a5eced402e6c7242015550668e3b568f3)
+
+commit 86eef272304f3896a9cda5cde7e882d09d0269c0
+Author: Daniel Gryniewicz <dang@redhat.com>
+Date:   Fri Jun 10 11:33:56 2016 -0400
+
+    Allow building RGW with LDAP disabled
+    
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 38abfcb5d036c050a363533a2c7b658d528d3612)
+
+commit 657554574414dcece6126bfa1b4008b038c3f729
+Author: la-sguduru <SGuduru@walmartlabs.com>
+Date:   Thu May 12 11:53:25 2016 +0530
+
+    doc: Modification for "TEST S3 ACCESS" section
+    
+    Signed-off-by: SirishaGuduru sirishaguduru99@gmail.com
+    (cherry picked from commit 27d6cf0c6958de9f0533bedb90a74c4020b53675)
+
+commit 50fd48fbdc8f4f0e9151eb405eb112b710304bb9
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Fri Mar 18 13:58:22 2016 -0400
+
+    rgw:response information is error when geting token of swift account
+    The header is X-Trans-Id instead of x-amz-request-id in the response header
+    
+    Fixes:#15195
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 08e909d0b844e8be3cb82d6add3e87d38e19e80e)
+
+commit 742366116d368d3adff4b7941a4751d362073b35
+Author: SirishaGuduru <SGuduru@walmartlabs.com>
+Date:   Mon Sep 19 10:21:06 2016 +0530
+
+    doc: Radosgw multitenancy config paramater change
+    
+    Radosgw multitenancy configuration parameter
+    "rgw keystone make new tenants" never works even
+    applied. When gone through the code, itseems this
+    parameter is not used. But "rgw keystone implicit
+    tenants" works as the code looks for this.
+    
+    Modified the configuration parameter in two files
+    mentioned below from "rgw keystone make new tenants"
+    to "rgw keystone implicit tenants"
+    
+    Fixes: http://tracker.ceph.com/issues/17293
+    
+    Signed-off-by: SirishaGuduru <SGuduru@walmartlabs.com>
+    (cherry picked from commit bd9695d50cfd472508a8a704f1ea5fc62dc08faf)
+
+commit 635666412c464228edb014b525abb0af3a55ac50
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Tue Oct 4 09:18:09 2016 +0800
+
+    rgw: fix versioned object IO error
+    
+        When accessing a copied destination object, its source object's instance ID
+        information is needed, however it's missing now in the destination object's
+        manifest.
+    
+        In order to fix this problem, we can record source object's version_id/instance
+        into dest object's manifest(a new filed 'tail_instance' is added). When creating
+        a new object(not copy), 'tail_instance' should be equal to its instance value.
+        When copy/get a object, 'tail_instance' should always be used to get the right
+        tail objects.
+    
+        Fixes: http://tracker.ceph.com/issues/17111
+        Signed-off-by: Yang Honggang <joseph.yang@xtaotech.com>
+    
+    (cherry picked from commit d43b69e529328f73da6c29cd746557788a989ae0)
+
+commit 40689d1acab4da963d05540eb837fd12a4411269
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 30 16:13:00 2016 -0700
+
+    rgw: don't fail if lost race when setting acls
+    
+    Fixes: http://tracker.ceph.com/issues/16930
+    
+    When concurrently setting acls on object/bucket, we could lose in a race.
+    Instead of retry, just return success (same effect as if we won and then
+    other writer overwrote us).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9a915b565923081f609048072b8d75716a74ea)
+
+commit 5a53ffa8a9e8590ea0d1992fad9a30cde135cbac
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Oct 5 11:41:42 2016 -0700
+
+    rgw: remove user email index using helper function
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 10dbfc540aef5c3d45b7e3aabe61090c302f47ab)
+
+commit ad54bf9a0dc541cb6d664edec9ca7f3ec6ffbc93
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Sat Feb 27 04:28:14 2016 -0500
+
+    rgw:user email can modify to empty
+    
+    Fixes: http://tracker.ceph.com/issues/13286
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit ebfd713d9a40e1cb33dfdf2198c20d621e387e36)
+
+commit 4babd3fa335351bd4327cded47a1ffe5a8cfd897
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Sep 17 08:29:15 2016 -0400
+
+    librbd: corrected use-after-free in ImageWatcher
+    
+    Fixes: http://tracker.ceph.com/issues/17289
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f4d4868e3b721c932c35ae7e8f0dd96b36a37fc)
+
+commit 1ca4dc6d1bd6aebece500c6e91f6a9871af0e1f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 9 10:42:50 2016 -0400
+
+    librbd: possible deadlock if cluster connection closed after image
+    
+    Fixes: http://tracker.ceph.com/issues/17254
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 818c2f2abc972f689acb7d783c2a684bcc9e6e51)
+
+commit b73356b500f9ff364c09e6d78e62fc0e79fbfbac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 07:25:36 2016 -0400
+
+    librbd: block name prefix might overflow fixed size C-string
+    
+    The issue which resulted in too large v2 image ids was fixed
+    under #16887.
+    
+    Fixes: http://tracker.ceph.com/issues/17310
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61734d266c6ee476c2f5fcfbbaefc7d0c7939617)
+
+commit b410c8ea2164eb53ee7882859f0a977b67cf8b80
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 10:19:45 2016 -0400
+
+    librbd: potential deadlock closing image with in-flight readahead
+    
+    Fixes: http://tracker.ceph.com/issues/17198
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c971d58f8a5550b19374b74bb89d69143423479b)
+
+commit 32d84e0eed671f0cfe37057d59f0afe1b63f3c4d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 14:47:36 2016 -0400
+
+    librbd: update_features should handle Infernalis OSDs
+    
+    Fixes: http://tracker.ceph.com/issues/17330
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8cb2ccda1b847e0f33c0d34bf57e9ec29bbbb43b)
+
+commit b4e13808b004fd10d342e3daa17a3d4e830d4d00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 14:34:16 2016 -0400
+
+    cls/rbd: set_features should adjust the mask to known features
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a26c7a5cacebf8814e20a2f3d5b69c20d8798871)
+
+commit dd93e009a222e8e86041661742b4df9c92b097f7
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon Jun 27 23:36:09 2016 +0530
+
+    rbd: cleanup - Proxied operations shouldn't result
+    in error messages if replayed
+    
+    Fixes: http://tracker.ceph.com/issues/16130
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit d09c9c471f40f15c14f392a93a04353ca30b1c5e)
+
+commit 391936a30cc6022f067cb11c2e39bd47b9e58d61
+Author: Lu Shi <shi.lu@h3c.com>
+Date:   Wed Jun 15 09:24:43 2016 +0800
+
+    librbd: enable/disable of features is not allowed when already enabled/disabled
+    
+    Fixes: http://tracker.ceph.com/issues/16079
+    
+    Signed-off-by: Lu Shi <shi.lu@h3c.com>
+    (cherry picked from commit a8a633396a4105991c9643c2b39391621934c26d)
+
+commit 7d6801bdd376f8eb5e1fbfa8f3a9f4de5dc710a1
+Author: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+Date:   Sat Jun 25 10:21:25 2016 +0800
+
+    rbd: restrict mirror enable/disable actions on parents/clones
+    
+    Fixes: http://tracker.ceph.com/issues/16056
+    
+    Signed-off-by: zhuangzeqiang zhuang.zeqiang@h3c.com
+    (cherry picked from commit 11dee0bbf0a85f2c197192d0560bd486bc2ad6fc)
+
+commit cf476284ae9089e73e5fdf5ddfefdd5842246190
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Sun Jun 19 13:06:06 2016 +0530
+
+    rgw: Add documentation for RGW default quota
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    This was modified to use size in kB, rather than size in bytes, since
+    Jewel has not yet been converted to use size in bytes.
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit e3ab3476146cf0fe604783e9e5a0d63237271735)
+
+commit 22348154ad51c20d73aa7715695244b8d85fac37
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Sun Jun 19 12:48:42 2016 +0530
+
+    rgw: Do not bother to check against conf values during quota handling
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    This was modified to use size in kB, rather than size in bytes, since
+    Jewel has not yet been converted to use size in bytes.
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 17d2c1712a5b72315a47ab0f8380331bfd478c0b)
+
+commit 0b8ecce6ac5abafe0175c941dbf55862b1515b1d
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Mon May 23 15:34:58 2016 +0530
+
+    rgw: Let the default quota settings take effect during user creation
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 400d7c982f18efd7cf96acfc3a63078791d1ec0a)
+
+commit 1c76ef4e3ea7cb401345af0938d6b76652061d05
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Oct 1 11:21:21 2016 +0300
+
+    journal: ensure in-flight ops are complete destroying journaler
+    
+    Fixes: http://tracker.ceph.com/issues/17446
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a25b1d7cbebb7b19cebee4cfb362cd744bbb4607)
+
+commit 9bc5e444543fade1f6d78788cadeb7bfdd29710b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Sep 29 16:55:22 2016 +0300
+
+    journal: complete action only after notification completed
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 55762cca2bf586d037cb9f32775ec158dc3287c1)
+
+commit 74873322da011d0e6221c90ff39d137de80720ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 4 21:29:31 2016 -0400
+
+    qa/workunits: reduce amount of client IO for rbd-mirror stress
+    
+    Journal IO is faster now, resulting in OSDs quickly filling up
+    under the stress test case.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a6dd6b50d632aa6995a0f4b6fc5e1a1bda23e2a0)
+
+commit d51a2f6689391274bba41f9359acb9395574e629
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 4 12:22:06 2016 -0400
+
+    rbd-mirror: potential race allocating tag during shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c2ff9bce61a8af024150b8be9dee484f339f7df)
+
+commit cea000c6225c809b35b6c643919a01d940522c55
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 30 12:32:37 2016 -0400
+
+    librbd: ignore attempts to request lock if already lock owner
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 355c79cb7c955e62e3153cf28046a9b8a6d0d25c)
+
+commit 912ce3728c09b2bf613da287f5013b97920cc27c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 30 12:59:16 2016 -0400
+
+    journal: clean up object recorder closed/overflow callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24faead086a50ea1b9614268d4dd5f3ea7bbe445)
+
+commit 310f3f79cf54daeefa8f00aba0d7b692261e4a33
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 29 08:36:53 2016 -0400
+
+    journal: delay moving overflowed buffers until quiesced
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4483531aa3aa3d0f298778062e2b9a339ad05075)
+
+commit 9f3614377043efb56c606905a4b4f5c86b3e074e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 29 08:19:52 2016 -0400
+
+    journal: use reverse iterator search to find flush record
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a96065815558e50361af4c701c23e5248962dfe0)
+
+commit 21502d90ecc72e70faa19b1202f9c7095341aaec
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 28 08:35:36 2016 -0400
+
+    journal: avoid holding lock while sending journal append
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dc77a629ed353d586b63f0bd8e20f54a7595afba)
+
+commit e0de824f7e728b06025a572a0c8213ef9fb1f112
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 21 15:41:55 2016 -0400
+
+    rbd-mirror: wait for in-flight event commit notifications
+    
+    Fixes: http://tracker.ceph.com/issues/17355
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f810c106ad4e9ae94494801fea6c580d81b9156b)
+
+commit efa12f7187b6e037c22ba62563030eb883dab5f9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 11 13:55:21 2016 -0400
+
+    journal: fix AsyncOpTracker compiler error
+    
+    The master branch has an async JournalTrimmer which shouldn't be
+    backported to the Jewel branch yet. This change addresses the
+    missing sync AsyncOpTracker::wait_for_ops method from the backported
+    class.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2460a3dbe7d04f1a4daa44d7367d24d47fa070fb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 21 13:02:34 2016 -0400
+
+    common: move AsyncOpTracker to common library
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 72d8992f054a7e36f92fdd2e01278ce3b9ede2eb)
+    
+    Conflicts:
+    	src/journal/CMakeLists.txt: doesn't exist in Jewel
+
+commit 1748b38da48beb8e25dc5ca831a540d389a963ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 28 16:35:48 2016 -0400
+
+    librbd: support deleting image by id instead of name
+    
+    The rbd-mirror daemon will use this API to delete images instead
+    of attempting to use the local image name.
+    
+    Fixes: http://tracker.ceph.com/issues/16227
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 87b32d15914207f61595c1c943817d983faceacd)
+
+commit bb5f6b6872e81291176543e954bf3654eb88120f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 13 14:49:06 2016 -0400
+
+    rbd-mirror: use pool id + global image id as deletion primary key
+    
+    Fixes: http://tracker.ceph.com/issues/16538
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 25203a8a9d59ff025d223ec1afaeb14946d54993)
+
+commit 96d551d7e8f10f98cc0a6ae5d90d33e6bca968a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 15 18:14:15 2016 -0400
+
+    rbd-mirror: concurrent access of event might result in heap corruption
+    
+    Fixes: http://tracker.ceph.com/issues/17283
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ac9ad37ab80db8913f9f1149707bd0009f8235c4)
+
+commit 8070d6a560044e5b45af7bd6995345ab30489d84
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Sep 21 23:08:18 2016 +0100
+
+    rbd-mirror: test: Fixed timeout problem in rbd_mirror_stress.sh
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 5c737038dd6d44bd76605587486ddf9457bc2d96)
+
+commit 6ea4cde78ac7db6081e83d2530552946368021ef
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Sep 6 17:28:22 2016 +0100
+
+    rbd: Fix race between journal flush and append events
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit aa959e71fe5a8cec43de75007fc9cef8de5ee3a5)
+
+commit 180a86b61498e6a27a59c7673f74e32ce1ae02cf
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Sep 7 15:26:34 2016 +0100
+
+    journal: make librados call async in ObjectRecorder
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 7b740f5b4ac1c66ac3c80782d2d34e846d00fddd)
+
+commit 5edbfe278532225aaaa6bb3fe6bef84a4c693d83
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Jul 25 17:00:50 2016 +0100
+
+    journal: increase concurrency of journal recorder
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 5c88edd68a1ee7c77f11e4113251fbe5768b8d99)
+
+commit 9ad132e670564c738e448bf719503cc73525fd8a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 19 11:24:17 2016 -0400
+
+    journal: send update notification when tag allocated
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1cd613e3d8ccb6d05b9adfe1956927991a4f4fe)
+
+commit f9aae06152e281c271f50201a8dd1852a132447f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 21:54:46 2016 -0400
+
+    librbd: new journal listener event for force promotion
+    
+    Fixes: http://tracker.ceph.com/issues/16974
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fd005490e95d7fca85be4cad34344a58986f64d6)
+
+commit 4f5ce86349e246e12abe8e0c55380f400bf05ebb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 16:38:51 2016 -0400
+
+    librbd: helper class for quiescing in-flight async ops
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 39d9e5cc9b38de2ee9ad2faf8e04253314160811)
+
+commit 829ff8c5fa9d9470f1e5370cf601509809b39674
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 12:37:53 2016 -0400
+
+    librbd: unify journal event callbacks into single interface
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dbbcecf4a289ca36b734b7bda9530cc0a59f84ac)
+
+commit 41cf3d93356bde801c32dec4b7a21ae065295aa7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 16:28:50 2016 -0400
+
+    journal: expose ability to retrieve partial tag list
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 277b6dd9f13a8390cbf7c9ac7a313813ecad4d27)
+
+commit 2f9e6412982826fe0712a0b98e004c405b60fac3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 14 08:49:39 2016 -0400
+
+    qa/workunits/rbd: fix remove mirrored image race conditions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a0f65b968b350629dfad978c191fc878ca26d093)
+
+commit 2ec5e93806f8487bdc5f8df23740aab5738cde1f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 12:32:45 2016 -0400
+
+    qa/workunits/rbd: new mirroring forced promotion test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a43268a4a30c06233152d531cbf2550224fb8a15)
+
+commit 5d1d898e1132325cae7045dc764a533878d56e00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 13:31:36 2016 -0400
+
+    test/rbd: fix possible mock journal race conditions
+    
+    Fixes: http://tracker.ceph.com/issues/17317
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 471898392372ba4c404376410fb56f3af5287c80)
+
+commit 775a999e8bac9d0dc02cb40f0206b96c5864b8d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 6 12:59:25 2016 +0300
+
+    qa/workunits/rbd: before removing image make sure it is not bootstrapped
+    
+    If an image is being bootstrapped, it implies that the rbd-mirror
+    daemon currently has the image open. The removal API will prevent the
+    removal of any image that is opened by another client.
+    
+    Works-around: http://tracker.ceph.com/issues/16555
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9737a8d6cbaf1b79bbc6008249d39acbae883941)
+
+commit 3bbd8ba4282ea7f3fa286d0a1944e9e93d321365
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 27 14:47:02 2016 -0400
+
+    librbd: fix journal create compilation error
+    
+    The synchronous journal create method no longer exists on the master
+    branch and the associated change to introduce an asynchronous journal
+    creation state machine should be delayed to provide more testing time
+    on the master branch before being backported to jewel.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 89c6618df89dc0d5d3eb1f855f6f93c72be75939
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 21:11:07 2016 -0400
+
+    test: fixed unused function warnings in unittest_rbd_mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3545d9ed19be8d3956f0db901ea9d3bb8b10d13d)
+
+commit ca94f25aa960bb352043a9d53eee361071d537da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 20:48:27 2016 -0400
+
+    rbd-mirror: prevent syncing to non-primary image after failover
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit beaef377d69f555277b706afff944a15086da28e)
+
+commit 7a434842ac2a2799b611aa87422009c244418922
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 23:45:46 2016 -0400
+
+    rbd-mirror: demote/promote in same cluster results in split-brain
+    
+    Fixes: http://tracker.ceph.com/issues/16855
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a6901ca1a065419426b3ad704e27e43ba8d591b8)
+
+commit f8f3bbd7246d9a1d5a82f6b0b112f185323a5fb3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 19:39:21 2016 -0400
+
+    librbd: normalize journal promotion/demotion events
+    
+    A non-primary image's commit possition won't accurately reflect
+    the current demotion/promotion chain. Therefore, directly specify
+    the predecessor for promotion events.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ac590e85a472533005cad73c82b2d61db161ab7a)
+
+commit 4baea6ace14bd834a3cb2c1d9d5202e94546e264
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 19:09:09 2016 -0400
+
+    librbd: block RPC requests when demoting the image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b195e1fc8fe70a0e5417934302d5831b1f8dfb3)
+
+commit 8ccdad6d9d09e0f09431df6d3ebb2fbed47040c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 9 12:24:19 2016 -0400
+
+    librbd: separate journal::TagData predecessor data into new struct
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 718befdae711141ef4a1e2e9f5e9aca97f1b5513)
+
+commit 13daaffb451e7187d5492a82bb85488c1bfe55d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 20:57:45 2016 -0400
+
+    rbd-mirror: include tag tid in bootstrap debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47d1e625006d554164f020e496a847735240ec95)
+
+commit d775680dc4eb53c08a692e746dd65c57a560496d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 10:39:01 2016 -0400
+
+    qa/workunits/rbd: demote/promote image on same cluster
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6aa18ea0df6dc4d1add597bc2d972c79699bf4c)
+
+commit f7ffbfa71d6f8c417b140ce434b7714cd0053b09
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 10 13:46:46 2016 +0300
+
+    rbd-mirror: option to automatically resync after journal client disconnect
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 77fd6a1c2016262d734b0bb5387e6b6a41232e8b)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 0b402390381dfbfcf4d3810f3179f90c4aa995ef
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 13:45:32 2016 +0300
+
+    rbd-mirror: stop replay when client is disconnected
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 330dba00ba3153ba2862eef52714e0dceae05192)
+
+commit 80aa7e0b4fae6f71a34e28ad1225e3540f9606ed
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 10 11:22:16 2016 +0300
+
+    rbd-mirror: resync was possible only when image replayer start had succeeded
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4bf6912f3c75560b89324fc29286028750f122c1)
+
+commit 170476e408308fad622063eddbd282a38a28ed89
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jul 26 16:10:30 2016 +0300
+
+    rbd-mirror: decode_client_meta should return false on error
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cd5eb36e98f46a1d84bddeafa4e7dcad415aa4a2)
+
+commit 7382e1a5a2b801cdfcbf7cda109343c365d005f3
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 3 14:19:51 2016 +0300
+
+    rbd: new command to disconnect journal client
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fc3ba54b3c101498a08a3f34ac8f7eab0152ad7c)
+
+commit d3ad2ff9dff192d93d2fbef82ec0ccd37809d2ca
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 14:06:42 2016 +0300
+
+    librbd: optionally flag "laggy" journal clients disconnected
+    
+    Fixes: http://tracker.ceph.com/issues/14738
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8eafefba9f2221a0ce927568795cb1c3ac0fa9c)
+
+commit 4056e360117b3aacfba2ae98cd4ecb60e356730c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 13 15:49:40 2016 +0300
+
+    journal: allow to trim journal for "laggy" clients
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0b8b1aaedc10f7f46e91bf6ad809414feb770c8d)
+
+commit 3aec576572066a4329488c0b4420fe863cbbeeb2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 13:42:19 2016 +0300
+
+    cls/journal: add async client_update_state method
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 58b8c66d5bfa60e6dd3ad2ec79360c2eca165c58)
+
+commit d66bb7a991affb8b77eb9440d7c8874cc0b3bbf4
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Oct 3 14:43:25 2016 -0600
+
+    build: include more files in "make dist" tarball
+    
+    Include all the documentation, qa, and CMakeLists.txt files in the "make
+    dist" tarball.
+    
+    Kraken and newer releases will already do this. This change is
+    restricted to releases that still use autotools.
+    
+    The purpose of this change is to make it easier to apply downstream
+    patches on an unpacked upstream tarball, because often those patches are
+    cherry-picks of commits that touch files under doc/, qa/, or
+    CMakeLists.txt.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9d1f756dfa1775338c2fa1cf6bfafd45b842b75f
+Merge: 59f0f4c 57f08fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:44 2016 +0200
+
+    Merge pull request #11389 from jcsp/wip-17466-jewel
+    
+    jewel: mon: don't crash on invalid standby_for_fscid
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 59f0f4c7cc81b20fc584b8e6fff84887ec9c26dd
+Merge: b747903 c2d4239
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:25 2016 +0200
+
+    Merge pull request #10997 from batrick/i17105-backport
+    
+    jewel: multimds: allow_multimds not required when max_mds is set in ceph.conf at startup
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b7479037472c7ce633e2130fb6dbc302dbb3affb
+Merge: db2e822 cf211d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:03 2016 +0200
+
+    Merge pull request #10958 from ukernel/jewel-16764
+    
+    jewel: client: fix shutdown with open inodes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit db2e8224853dc344b71871317204f13a0a53d808
+Merge: 4769027 1bc047b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:29:36 2016 +0200
+
+    Merge pull request #10921 from jcsp/wip-client-lock-backport
+    
+    jewel: client: add missing client_lock for get_root
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4769027f0c83cb22f25186356b56e86b49579b8b
+Merge: f49bac2 494687b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:29:05 2016 +0200
+
+    Merge pull request #11400 from ceph/wip-rbd-cli-jewel
+    
+    jewel: krbd-related CLI patches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f49bac2cbbda6aefef676b03f6656a98ec390b87
+Merge: 2a348d2 5ab5e82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 11:03:40 2016 +0200
+
+    Merge pull request #11171 from ceph/wip-rasize-doc-jewel
+    
+    jewel: doc: fix description for rsize and rasize
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3bb2a9ed9602b0d280a357f53bc90c6cf83ffe32
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 10 11:57:03 2016 -0400
+
+    librbd: ignore notify errors on missing image header
+    
+    The rename op on v1 images fails since the header no longer exists. In
+    the general case, the removal of the header object will also fail the
+    watcher which has its own recovery path.
+    
+    Fixes: http://tracker.ceph.com/issues/17549
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fe3e2eafa087f81c9ab59f3fbc39600d6adaa9c2)
+
+commit 5173563fbfc2799cc2328468bb197d65a94b4d7a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Sep 2 16:19:29 2016 +0800
+
+    client: properly set inode number of created inode in replay request
+    
+    Fixes: http://tracker.ceph.com/issues/17172
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e59385f16afc607ec700397b0bea5229ce69df30)
+
+commit 2c4e1c1cd1d01e51d5e4ad500d6b31832591d201
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 6 13:16:04 2016 +0100
+
+    mds: log path with CDir damage messages
+    
+    Previously you just got the inode number, which
+    wasn't terribly useful for e.g. a missing fragment
+    object, as you couldn't readily resolve the parent
+    path.
+    
+    Fixes: http://tracker.ceph.com/issues/16973
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 439cd5e1b4725e594786785189a37184243828d9)
+    
+    Conflicts:
+    	src/mds/CDir.cc: the go_bad() prototype which is part of the
+            context of the patch has changed.
+
+commit d52f190d449a2e68baed0659367795ebfb0dcb3d
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 6 13:07:38 2016 +0100
+
+    mds: s/used_ions/used_inos/
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bd3645fb07714d43fb7bd3d66b7e34b33768a8e8)
+
+commit 494687b7f11d0c9ca56d96b6b4480de96847bec2
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Oct 7 12:32:43 2016 +0200
+
+    rbd: expose rbd unmap options
+    
+    Reuse rbd map -o infrastructure to expose rbd unmap options in
+    a similar fashion.  Currently it's just one bool option, but we may
+    need more in the future.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 620f5e1455fdcb05cd3873c1e260141849829e35)
+    
+    Conflicts:
+    	doc/man/8/rbd.rst [ PR #9151 ("rbd: add methods to set and get
+    	  snapshot limits") not in jewel ]
+
+commit 6b0226774e2f09f98751e6fa3b0125a960417cc0
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Oct 7 11:14:16 2016 +0200
+
+    rbd: fix up terminology in help texts
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ba8f9ee791d55c4cf25ad751213ea6ba770b7434)
+
+commit 071bd4778547930d9aab9340dcabbb6d9ffdd890
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Mon Oct 3 14:09:11 2016 +0200
+
+    rbd: recognize lock_on_read option
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit e857b7896527b676155d6e01c78567337dc33b1c)
+
+commit 7d0714e17557ad281bd92907e10462894a482ce9
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Oct 4 10:21:36 2016 +0200
+
+    doc: clarify rbd size units
+    
+    It wasn't clear from the man page that --size defaults to M.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit d38dc29ed588b8933da3b66e46e1f0dd4cbb2cf4)
+    
+    Conflicts:
+    	doc/man/8/rbd.rst [ PR #9066 ("rbd: introduce a simple bench
+    	  for read") not in jewel ]
+
+commit ca8fc6fc8baae2e4da842e16e25b91d304702cb5
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Jul 23 22:56:45 2016 +0100
+
+    ceph-create-keys: fix existing-but-different case
+    
+    We just have to refrain from calling get-or-create
+    if the named key already exists, to avoid potentially
+    having an error when the default creation args don't
+    match the key as it has already been created, such
+    as on certain upgrades.
+    
+    Fixes: http://tracker.ceph.com/issues/16255
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 278196d86c52b0be3cb9c17ed7f8f3e3502a217a)
+
+commit 0a17741861faa1643bc338459397072268e469bd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 20 18:07:56 2016 +0800
+
+    client: fix readdir vs fragmentation race
+    
+    following sequence of events tigger the race
+    
+    client readdir frag 0* -> got item 'A'
+    MDS merges frag 0* and frag 1*
+    client send readdir request (frag 1*, offset 2, readdir_start 'A')
+    MDS reply items (that are after item 'A') in frag *
+    
+    Fixes: http://tracker.ceph.com/issues/17286
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit feb63776d4f2b03ece189715fdd75da5cae2afc2)
+
+commit 3a79db10f259d66ca17c2152cf639a63f245505d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 14 18:22:01 2016 +0800
+
+    mds: remove fail-safe queueing replay request
+    
+    MDSRank::queue_one_replay() does not create active request
+    immediately, it just queues corresponding C_MDS_RetryMessage
+    for execution. So the fail-safe code can queue an extra replay
+    request. This can cause replay requests be processed out-of-order
+    
+    Fixes: http://tracker.ceph.com/issues/17271
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d755a4a78242293dd3103bdd0748292df3034563)
+
+commit 10e603b1e3024c760cd5567dbbc73fbb3c1281f3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 21 21:57:25 2016 +0800
+
+    client: fix segment fault in Client::_invalidate_kernel_dcache().
+    
+    when umounting, root can be NULL
+    
+    Fixes: http://tracker.ceph.com/issues/17253
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ddea9a5b78151c0abe834e4a9872f7cf5f918ebc)
+
+commit 3320da0adb346af15a5d50a45e0aea5ca234eb79
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 15:18:24 2016 +0100
+
+    mds: catch duplicates in DamageTable
+    
+    There was an implicit assumption in the code that callers
+    wouldn't hit the notify_*damaged paths twice because they would
+    have checked is_*_damaged paths first.
+    
+    However, that's not really true in all cases, e.g. scrub
+    code isn't required to respect existing damage entries
+    when trying to load a CDir.
+    
+    Simply fix this by having the DamageTable notify* functions
+    check the key they're inserting doesn't already exist.
+    
+    Fixes: http://tracker.ceph.com/issues/17173
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c9cfaef104e9aaefad55583d7e54f8b4665904b3)
+
+commit 5d0e2f8690b08418fc404dc43df7a3a16ac83309
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 2 14:13:19 2016 -0400
+
+    common: only call crypto::init once per CephContext
+    
+    Fixes: http://tracker.ceph.com/issues/17205
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9dfc1537564c4ce65e4d0230a83f0d8d47503b16)
+
+commit 483d8c41a241aa917878284765aef2ba2ced0bb3
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Jun 14 17:34:44 2016 +1000
+
+    cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+    an empty secret and a null CryptoKeyHandler
+    
+    Fixes: http://tracker.ceph.com/issues/16266
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 009e777fbd18602e5fd66f97bdad95e977e6fecc)
+
+commit 5ae4f316b2cf37ce5d13b1ced4b65962c5583882
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 17:39:24 2016 +0800
+
+    os/filestore/FileJournal: fail out if FileJournal is not block device or regular file
+    
+    otherwise JournalingFileStore will assert when deleting FileJournal
+    which still has the non block/regular file opened.
+    
+    Fixes: http://tracker.ceph.com/issues/17307
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7431eec6fd24cd08ca6c76a9893e3f6e8c63a916)
+
+commit 2a348d23e5a047bf6a9c80140cf31ba2d6d4113f
+Merge: da04ac0 29a8701
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 20:30:54 2016 +0200
+
+    Merge pull request #10812 from SUSE/wip-17094-jewel
+    
+    jewel: build/ops: ceph-osd-prestart.sh fails confusingly when data directory does not exist
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit da04ac055188e2e84d21b29feee3069bb95e5ebe
+Merge: a80aacf 1a7faefd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Oct 10 10:46:33 2016 -0400
+
+    Merge pull request #11335 from linuxbox2/jewel-nfs
+    
+    jewel: rgw: nfs backports
+
+commit a80aacf048c2809bc2b7fd7e2f41e12b5470c5ab
+Merge: e795450 a10d700
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Oct 10 10:45:07 2016 -0400
+
+    Merge pull request #11332 from linuxbox2/jewel-ldap
+    
+    jewel: rgw: combined LDAP backports
+
+commit c2d4239331e397e6a48b1ee7a4c69dad0c951f94
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 29 16:25:10 2016 -0400
+
+    mds: remove max_mds config option
+    
+    It is now required that all changes to max_mds use the run-time `ceph fs
+    set max_mds` command. The rationale for this change is that it is
+    confusing to have a configuration for max_mds which is only observed at
+    file system creation.
+    
+    Fixes: http://tracker.ceph.com/issues/17105
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 52f4e545e0ce4782d5f2239d31080d392c12eba2)
+
+commit e7954500dab4f3e3bead824ba0da66105940e36a
+Merge: eac384a 0b93a9a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 10 08:54:02 2016 -0500
+
+    Merge pull request #10761 from dachary/wip-16377-jewel
+    
+    jewel: msgr/async: Messenger thread long time lock hold risk
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit eac384a5a158191e3276940d340ee2fe0b4b4b73
+Merge: c21e1de d94e5bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 10 08:52:59 2016 -0500
+
+    Merge pull request #10278 from SUSE/wip-16667-jewel
+    
+    jewel: incorrect value of CINIT_FLAG_DEFER_DROP_PRIVILEGES
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 57f08fe72269b573d95791b44f25de044d40fba6
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 30 21:57:35 2016 +0100
+
+    mds: make mds_role_t member order match constructor
+    
+    Previously this was mildy confusing.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 63d711d4e9decd2940c55aa697e6a588593da23a)
+
+commit c19326cef28ca3764340434cacc1f77e5899b887
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 3 07:33:27 2016 +0100
+
+    mon: don't crash on invalid standby_for_fscid
+    
+    Fixes: http://tracker.ceph.com/issues/17466
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 92fdcf36bd7d57b97f5d887d0370a181860e388c)
+
+commit cbd9a39e83258a7fa98d3d33aea652def17d3906
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 3 07:40:12 2016 +0100
+
+    messages: fix MMDSBeacon constructor
+    
+    This was leaving garbage in some fields during decode
+    when decoding messages with an older version.
+    
+    Fixes: http://tracker.ceph.com/issues/17466
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f1fb667dfd21cd241f4f45ce9ef3f8393d114c60)
+
+commit c21e1de83294fea247bd2a458143639544f91f6d
+Merge: 8a84877 f7d23d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:26:02 2016 +0200
+
+    Merge pull request #10794 from SUSE/wip-17082-jewel
+    
+    jewel: disable LTTng-UST in openSUSE builds
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8a8487711339596f2f22da572e59a194c6166c69
+Merge: 4c526a4 47605a2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:21:59 2016 +0200
+
+    Merge pull request #9739 from vumrao/wip-vumrao-16337
+    
+    jewel: osd: add peer_addr in heartbeat_check log message
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 4c526a4211f29a5151c5d79d3f0aa221f3d24221
+Merge: c362c71 5ffee35
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:20:30 2016 +0200
+
+    Merge pull request #9388 from vumrao/wip-vumrao-16069
+    
+    jewel: mon: Display full flag in ceph status if full flag is set
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c362c71ef34b136112852cce7a769a8bac30626c
+Merge: f707d1d 79e2acb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:03:51 2016 +0200
+
+    Merge pull request #10496 from Abhishekvrshny/wip-16583-jewel
+    
+    jewel: mon crash: crush/CrushWrapper.h: 940: FAILED assert(successful_detach)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f707d1d4f1cae66927e630da9a4524d2af6c9a11
+Merge: 84adafe f337a07
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 09:31:29 2016 +0200
+
+    Merge pull request #11018 from ceph/wip-17223-jewel
+    
+    jewel: krbd qa scripts and concurrent.sh test fix
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cca589fdb18147ea61e79b8d98b886ef30cd65d5
+Author: Vladislav Odintsov <odivlad@gmail.com>
+Date:   Mon Jul 4 15:29:22 2016 +0300
+
+    init-radosgw: do not use systemd-run in sysvinit
+    
+    `systemd-run` logic in initscripts was introduced because of ticket
+    http://tracker.ceph.com/issues/7627.
+    
+    If we have systemd-based operating system, we should use systemd unit files
+    from systemd directory to start/stop ceph daemons.
+    Otherwise, `daemon()` from `/etc/init.d/functions` on systemd distro starts service
+    in `system.slice` and everything works well.
+    
+    `systemd-run` can not be run on non-systemd distros, so it's not needed
+    on SysV systems.
+    
+    also, ceph-disk is now able to run the "systemctl"
+    to enable and start the ceph-osd, and ceph-deploy is also now
+    playing well with systemd when it comes to ceph-mon and ceph-mds
+    
+    Fixes: http://tracker.ceph.com/issues/16440
+    
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    (cherry picked from commit 1fd4f92a025e80092b8d08d9b7da2e0b73a52f0f)
+
+commit 83a91baff05f03f907fb80702ae1c68194b5308b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 30 17:36:16 2016 -0700
+
+    rgw: forward input data when forwarding set_bucket_version to master
+    
+    Fixes: http://tracker.ceph.com/issues/16494
+    
+    Needed to keep input data around to be forwarded correctly. Also, master
+    does not send any data back, so don't try to parse anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1eec00bef1e5540bf3e31d1f8fb1645eb64b1e62)
+
+commit 0986aff2aafdf64847a7e9608abb84edaea59eb1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 30 17:34:38 2016 -0700
+
+    rgw: set args when rebuilding req_info
+    
+    Was missing args, so it didn't sign requests with subresources
+    correctly when forwarding.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dd1068c54e50671d8904b99189823418a5a5bc07)
+
+commit bb81f9dd83da76704a5b6ba3e718fef3ac918290
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 12:39:45 2016 -0400
+
+    librbd: permit disabling journaling if in corrupt state
+    
+    Fixes: http://tracker.ceph.com/issues/16740
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6afb884d755697528684fad54dbb69b15d4386f1)
+
+commit b14d7f148c8f64d2e2da533fe2b35661536810c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 12:21:02 2016 -0400
+
+    librbd: new journal policy to disable initializing the journal
+    
+    This will be used in the case where the journal is being disabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 73e4c65c809a1e4161229f49285b21b2cfc623ca)
+    
+    Conflicts:
+    	src/librbd/Makefile.am (no CreateRequest.h or RemoveRequest.h in jewel)
+
+commit 77ea51138f05b4fa1c603ade0e62f961ec708441
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Aug 10 14:11:43 2016 +0800
+
+    doc/radosgw: fix description of response elements 'Part'
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 9c3fe46a7eb89aedc52582737c200e58d63738a5)
+
+commit 1a7faefd035aa8f31ab11ddfe1046989d0e69aac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Sep 21 17:18:20 2016 -0400
+
+    rgw_file:  pre-assign times
+    
+    Set unix timestamps based on RGW values for creation, modification
+    time for buckets, objects in stat requests.  Regard any saved
+    value of these in unix attributes as an overlay.
+    
+    Fixes: http://tracker.ceph.com/issues/17367
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 969750e96380859a350e25015bc8cbf9bdf88512)
+    
+    Fixes: http://tracker.ceph.com/issues/17394
+
+commit 0c932b9078a249bcb9dbb8c425788787f1cd48f2
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Sep 20 17:32:03 2016 -0400
+
+    rgw_file: fix set_attrs operation
+    
+    The effective part of this change is to always set a value
+    for RGW_ATTR_UNIX_KEY1 (because it is expected later).
+    
+    Secondarily, do not pass the address of the to-set attributes
+    buffer::list as remove attrs--this is confusing.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 08afb40b2812270a6af3cd1965e8c03c10392ace)
+    
+    Fixes: http://tracker.ceph.com/issues/17393
+
+commit 0de33909012f2a8305ff2038906669c78070cbcd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Apr 14 19:18:37 2016 -0400
+
+    rgw_file: implement rgw_setattr
+    
+    Introduce a new RGWSetattrs RGWOp descendant, to create or replace
+    sets of attrs on buckets or objects.
+    
+    This version of the change uses the standard RGWRADOS::set_attrs op
+    (we want attribute changes to (e.g.) sync with other changes).
+    
+    Previous versions of this changed incorrectly masked the values
+    of st->st_ino in RGWFileHandle::stat(), now fixed.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4de1c3c260265f821ebee842d49cb35bf49d8e4e)
+    
+    Fixes: http://tracker.ceph.com/issues/17311
+    Fixes: http://tracker.ceph.com/issues/17332
+
+commit 00f3fbe7dc940d8c9176a8da35fccd42e6ddc10f
+Author: zhangweibing <zhangweibing@unitedstack.com>
+Date:   Fri Aug 26 10:49:48 2016 +0800
+
+    rgw: remove duplicated calls to getattr
+    
+    Signed-off-by: Weibing Zhang <zhangweibing@unitedstack.com>
+    (cherry picked from commit 87b550ed2fba472d4ef2e1f771bb5f9fd248804f)
+    
+    Fixes: http://tracker.ceph.com/issues/17327
+
+commit c53992856d6058b71bed55337c3830dbaefb6823
+Author: Min Chen <chenmin@xsky.com>
+Date:   Fri Aug 19 01:02:50 2016 -0400
+
+    rgw: rgw file fix bug of rgw_lookup can not exact match file name
+    
+    bug reproduce steps:
+    in nfs-client (connect to nfs-ganesha server):
+    1. mv file file.rename
+    2. stat file // get file attrs, not return with -ENOENT
+    
+    the reason is that:
+    RGWStatLeafRequest does not exact match the file name,
+    just take file name as a prefix filter
+    
+    Signed-off-by: Min Chen <chenmin@xsky.com>
+    (cherry picked from commit 9d813bafc8e197507457c58ab4f365ccdb7f3589)
+    
+    Fixes: http://tracker.ceph.com/issues/17326
+
+commit 9f5c7377143c56b169a62a96dba5c6ff800e4d5c
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Aug 23 12:51:10 2016 +0800
+
+    rgw: fix the wrong return value
+    
+    Here `parent->readdir` will return negative numbers When there is an error.
+    so we should just return it.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit c979be7810793534715c24c6a9f7c8b0ed31de8d)
+    
+    Fixes: http://tracker.ceph.com/issues/17325
+
+commit cf1353c24917ecb9056272da7f6b3e73bba11fa8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:28:48 2016 -0400
+
+    rgw_file: restore local definition of RGWLibFS gc interval
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 31936caa09b46e86700faad969adfa9d43176206)
+    
+    Fixes: http://tracker.ceph.com/issues/17323
+
+commit 7a648ed991630541b5875a329820b67b269af570
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 16:50:42 2016 -0400
+
+    rgw_file:  explain semantics of RGWFileHandle::commit
+    
+    Add a comment explaining why the method currently returns 0
+    unconditionally.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b1da1354320cc5f69277b225a293a03cc2a5054f)
+
+commit 0135da74e2a08fccc2e2da9f1b7e55478bf79157
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 15:57:06 2016 -0400
+
+    ceph_timer: prefer using directive to typedef
+    
+    This change replaces the existing member-hook typedef as well as
+    the new set-type typedefs, so committed separately.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit f7d4d418658d2091652033045abdb4bfbeb637aa)
+
+commit 2a9233f1fbff38c12d378d5b6528a7e576a62e3e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 14:18:43 2016 -0400
+
+    librgw: add API version defines for librgw and rgw_file
+    
+    This change borrows the major, minor+extra format used by libcephfs.
+    The version numbering is starting at 1,1,0 on the theory that the
+    implicit version at Jewel is 1,0,0.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 355ccc5ded7f0b459bca24bc8b504b41807c583d)
+
+commit 2d6966f7e1f4c6dab12f7500746f1021b52d91af
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 11:24:01 2016 -0400
+
+    rgw file: allow RGWLibFS::write_completion_interval_s to be set in conf
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70dad0fba561edaf0bb4e746e29b92a4e9b014b2)
+
+commit 7071845f2e477648c322f0072e54490c47ed4154
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 11:00:22 2016 -0400
+
+    rgw_file:  log on write completion events at level 10
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d1e512d8c0e20df5a96ccb5f775a7a2ca123efbe)
+
+commit 95310c7f960eead2e22bc02c0e9d94ec863cafd6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:52:58 2016 -0400
+
+    rgw_file: add RGW_OPEN_FLAG_STATELESS as a synonym for V3
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit da596eb3af69c2cdf191893eb44a8947662634cf)
+
+commit 697d4ef7837061bb6b5ebaad07ae86836cdbe126
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 16 18:17:53 2016 -0400
+
+    rgw_file: implement stateless write completion timer
+    
+    Implements a temporal mechanism to enforce write completion for setups
+    which lack open state tracking (e.g., NFS3).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2c83ed4bd43fcb5d5497151a157b1dc08022fed9)
+
+commit 4a31509c5ae7e40dd1538b2075287d8cc202f159
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 12 16:52:51 2016 -0400
+
+    rgw_file: fix/update rgw_create flag arguments
+    
+    Allow passing POSIX open flags as well as api call flags.  Needed
+    for NFS3 support.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2d0ea2b4649e76cbbb87f44c57754d7ea9c46dce)
+
+commit 73f9a0b147914f3ef4f7b69a3b57899034406bca
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 12 10:39:21 2016 -0400
+
+    rgw_file: fix/update rgw_open flag arguments
+    
+    Allow passing POSIX open flags as well as api call flags.  Needed
+    for NFS3 support.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ada29f7f8323be2301588941bcd5c60190b32a4b)
+
+commit 4ceb70eba4d777ecc34cce9d1d10002807a6e770
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 16:49:41 2016 -0400
+
+    rgw file: remove busy-wait in RGWLibFS::gc()
+    
+    This is a background thread.  However, CPU is wasted.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ca33241286f52d849dbde8092507131b8b1108cc)
+    
+    Fixes: http://tracker.ceph.com/issues/17321
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 8e1515b068991567136042fb11384fb0fd0620fe
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 14:23:00 2016 -0400
+
+    rgw_file: unlock() must precede out label
+    
+    In lookup_handle(...).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 9f64f34e481278a683e962b17c7aa811005783a8)
+
+commit 88eeaa967494ba6b536d493d06bc4bdbe92e452f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Aug 7 18:39:33 2016 -0400
+
+    rgw nfs: fix write continuations
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b4d079f2b5a4d49b2ba7576b272a2a3f3b0d66c8)
+
+commit f0b1799910564706766fccb8ec2b1c2c27a10467
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 5 10:03:33 2016 -0400
+
+    rgw nfs: don't leak fh->mtx in lookup_handle()
+    
+    This change fixes a serious latent locking problem, noticed after
+    updating the ganesha/rgw driver invalidation after renames.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d74d46170d7104a6553674f111bbdbe3a116cf54)
+
+commit f16ac90f9c097352de2763215f0c83812503e815
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Aug 8 10:18:35 2016 -0400
+
+    rgw file: refuse to rename directories
+    
+    The representation of paths as an aggregate involving any
+    number of objects argues against ever permitting such
+    operations.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 297220fd2a96c0cf34701127b8cf724e24e6865f)
+
+commit 23b749f93ed9a7de93aabc871df0adeeebac4e0f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 3 13:53:15 2016 -0400
+
+    rgw_file: refuse ops on deleted targets
+    
+    Detect these illegal cases, as they are indicate various incorrect
+    behaviors/bugs.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 34224ecd8360e03603a1a5dd1c1c9f0a31f1c1d5)
+
+commit 345de45cbf5932174222b733bf71820e818fbfd5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Apr 14 19:18:37 2016 -0400
+
+    rgw_file: implement rgw_setattr
+    
+    Introduce a new RGWSetattrs RGWOp descendant, to create or replace
+    sets of attrs on buckets or objects.
+    
+    This version of the change uses the standard RGWRADOS::set_attrs op
+    (we want attribute changes to (e.g.) sync with other changes).
+    
+    Previous versions of this changed incorrectly masked the values
+    of st->st_ino in RGWFileHandle::stat(), now fixed.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4de1c3c260265f821ebee842d49cb35bf49d8e4e)
+
+commit 88b5027850618f14c562d07ba25a664fd4c22771
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Jul 13 10:16:59 2016 -0400
+
+    rgw_file: refuse partial, out-of-order writes
+    
+    A single file object may be opened only once per gateway
+    instance, and writes to that object must be complete, and in-order.
+    Enforce this.
+    
+    If an invalid write is seen, deletes the current write transaction.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 12aded803e24539266ce9698c678088e2158a82a)
+
+commit be243695c1309acfc293f7080cf213edbc558d14
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jul 12 17:43:43 2016 -0400
+
+    rgw_file: fix rename cases and unify unlink
+    
+    Rather ordinary rename cases failed in the unlink step.  Fix this,
+    unifying the unlink path while at it.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 05af1fbb870a905ea8903594dfc607d346c9dd31)
+
+commit a10d700697d8f432aca2b1bff6766e2478ac060d
+Author: Harald Klein <hari@vt100.at>
+Date:   Wed Aug 31 17:41:26 2016 +0200
+
+    add ldap auth custom search filter feature patch - http://tracker.ceph.com/issues/17185
+    
+    Signed-off-by: Harald Klein <hari@vt100.at>
+    (cherry picked from commit c935885ae9d5fd413495448a0b0e5fce899c9b73)
+    
+    Fixes: http://tracker.ceph.com/issues/17185
+
+commit 7486638563c1eeda7781dcf58b0c536d11f17a0b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Aug 18 10:54:16 2016 -0400
+
+    rgw ldap:  protect rgw::from_base64 from non-base64 input
+    
+    Also adds unit tests for:
+    1. empty output from from_base64 (turns out to be harmless)
+    2. random and specific non-base64 and sort strings
+    (modified from upstream to avoid alteration of src/test/test_rgw_token.cc)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0a4c91ec7652d02673a9b156cd16144d778a3844)
+    
+    Fixes: http://tracker.ceph.com/issues/17324
+
+commit 68788fb898118c826b136b7cd7f60265a6dfef79
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 5 10:02:03 2016 -0400
+
+    rgw ldap: enforce simple_bind w/LDAPv3
+    
+    Found by Harald Klein <hklein@redhat.com>.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 08d54291435e4d1cb5e02cda3951bc6e8510b0e2)
+
+commit 871caeb5011b06d54bd41370ee20ba38e84c9ac0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Aug 1 17:36:17 2016 -0400
+
+    rgw: add reinit/rebind logic (ldap)
+    
+    Gracefully handle stale LDAP connections by rebinding--verified
+    with MS AD.  Rebind is attempted once per request until rebound--
+    not worse than misconfiguring the environment.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 9b8fccf68789ba5c5986766c544b23aeada0e87e)
+
+commit de0c4e1281b5b67af1459aa11fbb292a0cae2ab2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 16 16:58:51 2016 -0400
+
+    rgw: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR
+    
+    Fixes: http://tracker.ceph.com/issues/17073
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e77a523f1d74768f4fef58c05cc167705d219036)
+
+commit 8833c64459edb77fa0d394b2eda2f79cd0f6dba9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 3 10:34:51 2016 -0700
+
+    PG: block writes and scan log for scrub using inclusive upper bound
+    
+    See comment in commit.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 0e2338e5ec5441e5fc173a7af69343c775b02a13
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 3 14:34:19 2016 -0700
+
+    osd_types,PG: force the map used in _scrub to be sorted correctly
+    
+    Previously, ScrubMap::objects was always sorted bitwise (nibblewise
+    before the comparator change was made.  It didn't matter because we
+    always scrubbed whole hash values.  Now, we need the objects in the
+    objectstore ordering because we may be missing objects at the tail of
+    the scanned range and need them to show up at the tail of the
+    ScrubMap::objects mapping.  We don't need to do anything else to handle
+    the upgrade process since the actual objects *in* the map were
+    determined by the objectstore ordering.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6d410e97232afdad4f226151333b209d8b50f4ed)
+
+commit 16f9d95204f3eca6908ae241e5bacc48b006cb23
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Sep 28 13:24:56 2016 -0700
+
+    src/osd: relax the requirement that we scrub a whole hash value
+    
+    Previously, we needed to scrub all objects in clones in a single
+    hash value mainly to ensure that _scrub had access to all clones
+    of a single object at the same time.  Instead, just avoid letting
+    head or snapdir be a boundary (see the comment in the commit
+    for details).
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 27bdc8ce6d0a7d8ae47f29540f281ba417e16b4c)
+
+commit 2176c847d588fbec654653db48c4ad156a87aac2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Sep 28 13:25:42 2016 -0700
+
+    hobject: clarify is_snap and has_snapset for max
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a2c27c9423b43b8c5633fb0af95f28b0de94b365)
+
+commit 84adafe3fe6f57d49b4bdf1e036e1b686e6b0f55
+Merge: 5246f81 fb36dd0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Sep 30 14:02:43 2016 -0400
+
+    Merge pull request #10656 from dachary/wip-16564-jewel
+    
+    jewel: cors auto memleak
+
+commit 35660d1aabd8b27bf3ba71e50d00ba1cf1cecf00
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Wed Sep 14 07:36:43 2016 -0400
+
+    mon: OSDMonitor: Missing nearfull flag set
+    
+    Output from 'ceph -s -f json-pretty' wans't showing nearfull flag set.
+    
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+    (cherry picked from commit 754887b575795ef96cda095f7f0d8c898e20f00f)
+
+commit 5246f8135124dd49466268ead3c8c10d241c94f9
+Merge: 9c464a5 3cb0a5e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 15:54:05 2016 -0700
+
+    Merge pull request #11252 from ceph/wip-offline-split-jewel
+    
+    ceph-objectstore-tool: add a way to split filestore directories offline
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9c464a55c8ffe7278450c88cc208f2f2e7bce1a0
+Merge: 085e9e3 457d78f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 11:34:44 2016 -0700
+
+    Merge pull request #10866 from dachary/wip-17122-jewel
+    
+    jewel: COPY broke multipart files uploaded under dumpling
+
+commit 085e9e369c5c6fbe48f888009e3997d86dcaf968
+Merge: cb46489 23d73dc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 11:32:09 2016 -0700
+
+    Merge pull request #10890 from dachary/wip-17140-jewel
+    
+    jewel: rgw: period commit loses zonegroup changes: region_map converted repeatedly
+
+commit cb46489b92b14c5722368e88a4c66712ecc1623d
+Merge: a0fce63 cefd6f5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 10:51:09 2016 -0700
+
+    Merge pull request #11021 from cbodley/wip-16794
+    
+    jewel: rgw: back off bucket sync on failures, don't store marker
+
+commit a0fce6302013b2e9ddad8466d7ab34c370d3786e
+Merge: 4e61538 92581a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 10:49:28 2016 -0700
+
+    Merge pull request #10657 from dachary/wip-16792-jewel
+    
+    jewel: metadata sync can skip markers for failed/incomplete entries
+
+commit 3cb0a5e0b0517701ad3b44bdf6d4b60d6a51e8ee
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 11:45:00 2016 -0700
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+    
+    Use the usual split code, and split each dir that meets the
+    usual split criteria.
+    
+    This can be run with lower than usual split settings, to avoid costly
+    online splits. To make sure the directories are not merged again, use
+    a load merge threshold (e.g. filestore merge threshold = 1), and
+    adjust the split multiplier accordingly.
+    
+    Fixes: http://tracker.ceph.com/issues/17220
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7b0428e0e8d8f5459311dc698d94a3ac4f04684)
+    
+    Conflicts:
+    	src/os/filestore/FileStore.cc
+    	src/tools/ceph_objectstore_tool.cc
+    (trivial resolution)
+
+commit de672a09f1e2b29de1e0e6e15d26be99315c41b9
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Aug 5 11:14:53 2016 +0200
+
+    logrotate: Run as root/ceph
+    
+    Currently, we run the logrotate scripts as ceph/ceph but that way we
+    cannot rotate the scripts created by qemu (they are root/ceph and 644).
+    The original su line was introduced in commit 73d7bed9 because logrotate
+    complained that the directory was writable by a non-root group and it
+    needed a su line to supress that error. This way, we keep the group
+    settings and we can access and rotate the qemu logs as well.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+
+commit 4e61538d9f1b372f28e61ad94c368c782be09d92
+Merge: ecc2377 e72e7a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 27 15:37:17 2016 -0500
+
+    Merge pull request #10888 from dachary/wip-17062-jewel
+    
+    jewel: tests: fix TestClsRbd.mirror_image failure in upgrade:jewel-x-master-distro-basic-vps
+
+commit 9cb45e11ec392da7c22578539b99619bdbf37e35
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue May 10 16:44:44 2016 +1000
+
+    log: Log.cc: Assign LOG_INFO priority to syslog calls
+    
+    LOG_DEBUG prio messages are not logged by a default syslog
+    configuration so log at LOG_INFO instead.
+    
+    Fixes: http://tracker.ceph.com/issues/15808
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 3ab5a660a45506d6c4c41bfeb5babdf188e62b3d)
+
+commit 165e5abdbf6311974d4001e43982b83d06f9e0cc
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri Apr 1 16:55:36 2016 +0530
+
+    librados: modify Pipe::connect() to return the error code
+    
+    Fixes: http://tracker.ceph.com/issues/15308
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit cad38dca0c90fb2ee4b589f336f8272122f50ad6)
+
+commit 5ab5e8268174bd34a095a1e96a1a969ad9e3531b
+Author: Andreas Gerstmayr <andreas.gerstmayr@gmail.com>
+Date:   Thu Sep 15 22:02:23 2016 +0200
+
+    doc: fix description for rsize and rasize
+    
+    Signed-off-by: Andreas Gerstmayr <andreas.gerstmayr@gmail.com>
+    (cherry picked from commit a004254a02cafdee848a382f893f89c3c5a9845a)
+
+commit c94244d3b132b6cf1f8ad309644cf48739282f04
+Author: Jelle vd Kooij <vdkooij.jelle@gmail.com>
+Date:   Thu Sep 1 00:42:34 2016 +0200
+
+    Add Install section to systemd rbdmap.service file
+    
+    Signed-off-by: Jelle vd Kooij <vdkooij.jelle@gmail.com>
+    (cherry picked from commit 57b6f656e17124a5ab4cd1400840d9c7c87a3cc3)
+
+commit e2ce8576f547b1e073c169ef331c7063319d9783
+Author: tserlin <tserlin@redhat.com>
+Date:   Mon Sep 19 14:40:12 2016 -0400
+
+    Add two options to radosgw-admin.rst manpage
+    
+    Add '--job-id' and '--max-concurrent-ios' to Orphan Search Options
+    
+    Fixes: http://tracker.ceph.com/issues/17281
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit 697f30d86f49b73c981c06375ab2937570b1db01)
+
+commit 4e66f9e9d1179204a6fdf08e43608a852d2b7fd3
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 12:56:39 2016 -0600
+
+    radosgw-admin: add "--orphan-stale-secs" to --help
+    
+    The radosgw-admin --help did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option
+    sets the number of seconds to wait before declaring an object to be an
+    orphan.
+    
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 354059ae43b4f4cc797da1669715399cd96a4738)
+
+commit bfa90a152867e0277ae05c64b6bdc429b659b9a5
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 11:40:14 2016 -0600
+
+    doc: add "--orphan-stale-secs" to radosgw-admin(8)
+    
+    The radosgw-admin(8) manual page did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option sets
+    the number of seconds to wait before declaring an object to be an
+    orphan.
+    
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit a676c516069cc448591018ecf4f7d1f7f7bc3bfd)
+
+commit cefd6f51068df3ed04c755b1167dd146b85a0741
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:59:50 2016 -0700
+
+    rgw: fix collection of object sync errors
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit aacc31f72cef6f151459e4a0543850edeeaf1938)
+
+commit aa369817c9bf10cffa420bfb88a27957407fb625
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:43:48 2016 -0700
+
+    rgw: fix marker tracker completion handling
+    
+    Was not tracking high markers correctly. Could only work if there was a single
+    hole in the completion range. Just keep a map of all the complete entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f5801c726efdc2f3067a071e6bb5ac83fd0cd147)
+
+commit bce19a39b6f6ff23b6373f82ed2e6e12d65a77ef
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:42:52 2016 -0700
+
+    rgw: collect() stops if error encoutered
+    
+    and returns true if needed to be called again
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5528932e4c327a0fb0ce34a676eb59707a575325)
+
+commit f82b59363d12e38446b4f06f554b932a3cbdb1fb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 19 15:32:03 2016 -0700
+
+    rgw: back off bucket sync on failures, don't store marker
+    
+    Fixes: http://tracker.ceph.com/issues/16742
+    
+    If we fail on any single entry in bucket, skip updating the marker tracker
+    so that next time we'll go over that entry, and back off. This will trigger
+    a report to the data sync error repo and eventually a retry on the failing
+    object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1f3fec807043fd313ef7c66ff48f18b82c8bfa66)
+
+commit f337a0720713329b5305ddd9e5c18e8d2b11b475
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Jun 3 17:24:48 2016 +0200
+
+    krbd: don't segfault if images are unmapped concurrently
+    
+    "rbd map c" can die from a NULL dereference on any of this_pool,
+    this_image or this_snap in wait_for_udev_add():
+    
+        <image a is mapped>
+        rbd map c
+                                        rbd map b
+                                        rbd unmap a
+                                        rbd unmap b
+    
+    However unlikely, this segfault is triggered by the rbd/concurrent.sh
+    workunit on a regular basis.
+    
+    Similarly, "rbd showmapped" can die if an image to be listed is
+    unmapped.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 2bfecb1c0a0b2a314a5d137e6ca2cccd9ddc9b54)
+
+commit 2399d4ae54780cb18c313a8da79b697a70a87146
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri May 13 17:57:12 2016 +0200
+
+    qa: rbd/simple_big.sh: drop extraneous sudo
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6643f4d9723e5d9c5195680ac9abb9915ef42a56)
+
+commit 472cb29c5712c4fbbeaef26cdfddc1afd5f2945d
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri May 13 15:05:53 2016 +0200
+
+    qa: rbd/concurrent.sh: suppress rbd map output
+    
+    rbd_map_image() echoes the id of the image and is used in command
+    substitution:
+    
+        id=$(rbd_map_image "${image}")
+    
+    Output from rbd map isn't consumed and clobbers the return.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a7d4189587e343a581660f1a118f6667c26e5bd4)
+
+commit 667d42aad7e19e36200ee7cd979efdd19154876a
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu May 12 15:07:12 2016 +0200
+
+    qa: rbd: don't modprobe, chown sysfs files or udevadm settle
+    
+    This is a followup to 38a572011dbb ("qa: rbd/map-snapshot-io.sh: don't
+    chown sysfs files") for the rest of the workunits:
+    
+    - /sys/bus/rbd/add_single_major is used if present, so chown'ing
+      /sys/bus/rbd/add doesn't help.  Use sudo on rbd map instead.
+    
+    - Don't modprobe or udevadm settle - rbd CLI tool takes care of that.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6bddef8ecff97f732bf707d9ba6d2386bc147f5a)
+
+commit 866c3e594b754d10bf960375f8498ba573c531dd
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed May 11 17:26:39 2016 +0200
+
+    qa: rbd/map-snapshot-io.sh: don't chown sysfs files
+    
+    /sys/bus/rbd/add_single_major is used if present, so chown'ing
+    /sys/bus/rbd/add doesn't help.  Use sudo on rbd map instead.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 38a572011dbbf8298cb7ba8ebf43909967eec689)
+
+commit cf211d7dc2eef46cb409b425caa72aac1b24bd7c
+Author: John Spray <john.spray@redhat.com>
+Date:   Sun Jul 24 17:19:55 2016 +0100
+
+    client: fix shutdown with open inodes
+    
+    This piece of code was dereferencing an invalid
+    iterator (invalidated by call to erase())
+    
+    Fixes:  http://tracker.ceph.com/issues/16764
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d642b4faec3266f609e4871ccdccdcd73707dc23)
+
+commit 1bc047b1c6ccc1a1757c97e52da5e4db5bc8f801
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Jun 29 15:45:30 2016 -0400
+
+    client: add missing client_lock for get_root
+    
+    A segmentation fault was observed in a recent test failure
+    (test_client_cache_size) [1]. An analysis of the core dump revealed that the
+    client->root inode was NULL. The client log [2] showed that the root inode was
+    deleted prior to the segfault:
+    
+        2016-06-27 14:24:13.358500 7fe75cff9700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 0
+        2016-06-27 14:24:13.358502 7fe75cff9700 10 client.4161 put_inode on 100000003e9.head(faked_ino=0 ref=1 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40)
+        2016-06-27 14:24:13.358513 7fe75cff9700 15 inode.put on 0x7fe76000ba40 100000003e9.head now 0
+        2016-06-27 14:24:13.358515 7fe75cff9700 10 client.4161 remove_cap mds.0 on 100000003e9.head(faked_ino=0 ref=0 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40)
+        2016-06-27 14:24:13.358538 7fe75cff9700 15 client.4161 remove_cap last one, closing snaprealm 0x7fe76000c1b0
+        2016-06-27 14:24:13.358544 7fe75cff9700 20 client.4161 put_snap_realm 1 0x7fe76000c1b0 1002 -> 1001
+        2016-06-27 14:24:13.358546 7fe75cff9700 10 client.4161 put_inode deleting 100000003e9.head(faked_ino=0 ref=0 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=- 0x7fe76000ba40)
+    
+    After looking through the reference count inc/dec, I noticed this mismatched inc/dec:
+    
+        2016-06-27 14:24:13.352622 7fe75dffb700  3 client.4161 ll_forget 100000003e9 1
+        2016-06-27 14:24:13.352601 7fe7767fc700  1 -- 172.21.15.58:0/3762258427 >> 172.21.15.57:6804/11121 conn(0x7fe789bf3b60 sd=20 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=7 cs=1 l=0). == rx == mds.0 seq 1473 0x7fe764037a80 client_reply(???:470 = 0 (0) Success safe) v1
+        2016-06-27 14:24:13.352665 7fe75dffb700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 1
+        2016-06-27 14:24:13.352687 7fe75d7fa700  3 client.4161 ll_lookup 0x7fe76000ba40 testdir
+        2016-06-27 14:24:13.352699 7fe75d7fa700 20 client.4161 _lookup have dn testdir mds.-1 ttl 0.000000 seq 0
+        2016-06-27 14:24:13.352705 7fe75d7fa700 15 inode.get on 0x7fe760011da0 100000003ea.head now 537
+        2016-06-27 14:24:13.352693 7fe76effd700  1 -- 172.21.15.58:0/3762258427 <== mds.0 172.21.15.57:6804/11121 1473 ==== client_reply(???:470 = 0 (0) Success safe) v1 ==== 27+0+0 (3458149698 0 0) 0x7fe764037a80 con 0x7fe789bf3b60
+        2016-06-27 14:24:13.352708 7fe75d7fa700 10 client.4161 _lookup 100000003e9.head(faked_ino=0 ref=3 ll_ref=1 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40) testdir = 100000003ea.head(faked_ino=0 ref=537 ll_ref=999 cap_refs={} open={} mode=40755 size=0/0 mtime=2016-06-27 14:24:13.325297 caps=pAsLsXsFsx(0=pAsLsXsFsx) COMPLETE parents=0x7fe750001f50 0x7fe760011da0)
+    
+    This sequence of ll_forget/ll_lookup is repeated in this test several hundred times. This is the prior sequence (in one thread):
+    
+        2016-06-27 14:24:13.324896 7fe75dffb700  3 client.4161 ll_forget 100000003e9 1
+        2016-06-27 14:24:13.324904 7fe75dffb700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 1
+        2016-06-27 14:24:13.324915 7fe75dffb700  3 client.4161 ll_lookup 0x7fe76000ba40 testdir
+        2016-06-27 14:24:13.324921 7fe75dffb700 20 client.4161 _lookup have dn testdir mds.-1 ttl 0.000000 seq 0
+        2016-06-27 14:24:13.324931 7fe75dffb700 15 inode.get on 0x7fe760011da0 100000003ea.head now 621
+        2016-06-27 14:24:13.324904 7fe7767fc700  1 -- 172.21.15.58:0/3762258427 >> 172.21.15.57:6804/11121 conn(0x7fe789bf3b60 sd=20 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=7 cs=1 l=0). == rx == mds.0 seq 1387 0x7fe764026720 client_reply(???:385 = 0 (0) Success safe) v1
+        2016-06-27 14:24:13.324934 7fe75dffb700 10 client.4161 _lookup 100000003e9.head(faked_ino=0 ref=3 ll_ref=2 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40) testdir = 100000003ea.head(faked_ino=0 ref=621 ll_ref=998 cap_refs={} open={} mode=40755 size=0/0 mtime=2016-06-27 14:24:13.321021 caps=pAsLsXsFsx(0=pAsLsXsFsx) COMPLETE parents=0x7fe750001f50 0x7fe760011da0)
+    
+    The key line is `_lookup 100000003e9.head(...)`. ll_ref=2 is the expected count
+    after this sequence but, as we see at 14:24:13.352708, ll_ref=1.
+    
+    This racing behavior is not serialized by client_lock because
+    Client::get_root() does not lock the client_lock. Additionally, this race is
+    not easy to identify in the logs because Client::get_root() does not use
+    Client::_ll_get to increment the reference, which hides the increments in the
+    logs. Instead it directly increments the reference using Inode:ll_get().
+    
+    This commit adds the client_lock for Client::get_root() and
+    Client::get_root_ino() (which may not strictly be necessary but can't hurt).
+    
+    [1] http://pulpito.ceph.com/pdonnell-2016-06-27_06:54:23-fs-wip-pdonnell-i16164---basic-smithi/280361/
+    [2] http://qa-proxy.ceph.com/teuthology/pdonnell-2016-06-27_06:54:23-fs-wip-pdonnell-i16164---basic-smithi/280361/remote/smithi058/log/ceph-client.0.23448.log.gz
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 208e3b6fd58237b309aae15b18dcd82672b133af)
+
+commit 086f6e0c07c807dc5001e40d18a72d53c5a10f66
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jul 20 13:02:03 2016 +0200
+
+    rgw: fix upgrade from old multisite to new multisite configuration
+    
+    We need to store the updated current period after adding the old converted regions
+    
+    Fixes: http://tracker.ceph.com/issues/16751
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 6d7841c6f050f6bd813d54df65d38e7f6934d22d)
+
+commit 23d73dca4e8fd9472b8e39d9219fda90323478fb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 23 15:10:44 2016 -0400
+
+    rgw: delete region map after upgrade to zonegroup map
+    
+    convert_regionmap() reads the region map and uses it to initialize the
+    zonegroup map.  but it doesn't remove the region_map afterwards, so
+    radosgw (and some radosgw-admin commands) will keep doing this on
+    startup, overwriting any changes made to the period/zonegroup map
+    
+    Fixes: http://tracker.ceph.com/issues/17051
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 215cd6f6ff7b8cb74df6dc3b94a5928f56ab9540)
+
+commit 27626ba9cc7495e312bf5caba6be8d6ee2f1ea64
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:28:48 2016 -0400
+
+    rgw_file: restore local definition of RGWLibFS gc interval
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 31936caa09b46e86700faad969adfa9d43176206)
+
+commit 91b65e7b4976103ea45c923bac9d21321d962773
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 16:49:41 2016 -0400
+
+    rgw file: remove busy-wait in RGWLibFS::gc()
+    
+    This is a background thread.  However, CPU is wasted.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ca33241286f52d849dbde8092507131b8b1108cc)
+
+commit e72e7a26f22022e6520232481f66f509367073a1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 14:44:09 2016 -0400
+
+    qa/workunits: support filtering cls_rbd unit test cases
+    
+    Fixes: http://tracker.ceph.com/issues/16529
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0891ede57cd254aa75c66301fde4a2eb6957a3d6)
+
+commit a25a4835ef91f2c249107a3f402c299d8c5c40da
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jun 3 14:41:13 2016 -0700
+
+    ReplicatedPG: call op_applied for submit_log_entries based repops
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8bde0ae122f384817f4e522604379d1e9b21d8d9)
+
+commit 5ef90856ce26761b31cfd4eb1bea4dfbe88cfb13
+Author: Tao Chang <changtao@hihuron.com>
+Date:   Mon May 9 09:50:17 2016 -0400
+
+    Cleanup: delete find_best_info again
+    
+    After called find_best_info find a auth_pg_shard, it must be not incomplete,
+    so it will not enter find_best_info again.
+    
+    Signed-off-by: Tao Chang <changtao@hihuron.com>
+    (cherry picked from commit a25cfc4d46c03d8d78e0254c728ea5c29e2246e1)
+
+commit dea93dc35859ebf31366dfe5f2b8c2ebebc4f6a6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 27 11:48:17 2016 +0200
+
+    ceph-disk: do not activate device that is not ready
+    
+    If the journal (or data) device is not ready when we are activating the
+    data (or journal) device, just print an info message and exit with 0 so
+    that the ceph-disk systemd service won't fail in this case.
+    
+    Fixes: http://tracker.ceph.com/issues/15990
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 73a7a65f65b1f6e90c4cae101bd43fabaeac3045)
+
+commit 58b7c52db2df0105aea3fbc36a0b2bc0d299415f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:51 2016 -0700
+
+    PG: use upset rather than up for _update_calc_stats
+    
+    Fixes: http://tracker.ceph.com/issues/16998
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 200cae8c9db99b3dede377f74413cc0b15033e1f)
+
+commit 4e2b4917c30b431f87b0da0d090664f2210f7ad2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:35 2016 -0700
+
+    PG: introduce and maintain upset
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 80a5c21d97116e7a66351a0e1f5d9b561f8663ac)
+
+commit c0db9fb831dc67ae138056c2f03c31dcc7f6adbf
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:15 2016 -0400
+
+    ceph: don't fudge the ctime in stat() unless it's really older than the mtime
+    
+    The current code just looks at the sec field, but the difference may
+    only be evident in the nsec field.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 14ee7bcbf0bdd6553dc1b78898ae429aae644100)
+
+commit 66cd43bfedddbbc82e3c303554f089f20eabf0f6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:15 2016 -0400
+
+    client: only skip querying the MDS in _lookup when we have the necessary caps
+    
+    If we don't have the caps we'll need later, then we must reissue the
+    lookup to get them regardless of whether we have a lease on the
+    dentry.
+    
+    Fixes: http://tracker.ceph.com/issues/16668
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 9e8476743eef195a0a9bc6ef82392a8781e82549)
+
+commit fb4a9398a8d60f40ed4c8f57965de54dac1df983
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:14 2016 -0400
+
+    client: plumb a mask argument into _lookup
+    
+    ...and attempt to pass in a sane value there, based on what we'll do
+    with the resulting inode.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f3605d39e53b3ff777eb64538abfa62a5f98a4f2)
+
+commit b5cbd5766fbf2a9ff5ee0e58b4ea42c706f4505a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:14 2016 -0400
+
+    client: add mask parameter to _do_lookup
+    
+    We need to allow callers to specify caps to acquire during a lookup, as
+    they may need to scrape certain info out of the inode later. Allow them
+    to pass in a mask. For now, _lookup just passes in 0 for the mask,
+    but verify_reply_trace passes in the regetattr_mask to match what we'd
+    request in the _getattr request if there were a traceless reply.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit a2ce16f8bfdb16ac485b8c4ad9a51ade5c256a5b)
+
+commit 416ec6ff91c602f385385acd8ac458ac58f13222
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Aug 16 14:19:20 2016 +0800
+
+    rgw: fix radosgw daemon core when reopen logs
+    
+    the rgw_user_st_syn thread use the pointer meta_mgr in RGWUserStatsCache::sync_all_users() fuction, so we should close the thread before delete the pointer in RGWRados::finalize() function when close storage before reopen_logs.
+    
+    Fixes: http://tracker.ceph.com/issues/17036
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit ea5d7fdd7fa978deadeee42bc0ea33fb11662e59)
+
+commit f034fd0a7b959dc3553fb0708da989ba6ac77db3
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Aug 24 09:38:03 2016 +0800
+
+    rgw: fix period update --commit return error
+    
+    period commit return error when the current period has a zonegroup which doesn't have a master zone
+    
+    Fixes: http://tracker.ceph.com/issues/17110
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit fcedda6e5dfbaff3975b4474d8546cf77f78cc83)
+
+commit 457d78f4f497fdaa0f0f116228ebf135a1ce7fb6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 18:01:54 2016 -0700
+
+    rgw: adjust manifest head object
+    
+    adjust the manifest head object:
+     - when reading manifest, set the head object to the object
+       we read the manifest from (and its size). Some manifests are
+       broken and point at a different object
+     - when copying multipart object, set the manifest head object to
+       point at the new head object
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 419264586fc46337322f0d60b0ffcdcce3bb5c5a)
+
+commit 77e497da5c618a30fb908cb3915aa1ca7279ce7f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:49:07 2016 -0700
+
+    rgw: adjust objs when copying obj with explicit_objs set
+    
+    If the head object contains data, need to set it in the list of
+    objs (in addition to being pointed at by head_obj).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1e012112b99004745952accc6fb11ada5e8e0045)
+
+commit c1250ee6cbc4a7710399e603ff3ee0e46aa92d49
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:36:33 2016 -0700
+
+    rgw: patch manifest to handle explicit objs copy issue
+    
+    Fixes: http://tracker.ceph.com/issues/16435
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b51476466e5bb03bbaa6e428bb6bb189a259b9fd)
+
+commit 2311ab24049805ea274451607949ba9b54028470
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jul 15 11:32:18 2016 +0800
+
+    ObjectCacher: fix last_write check in bh_write_adjacencies()
+    
+    Make the last_write check in bh_write_adjacencies() match corresponding
+    check in flush().
+    
+    Fixes: http://tracker.ceph.com/issues/16610
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ce166c2cd706b8573deabd331d29544e75e85972)
+
+commit c3c2910fbe8c6cc206e1cd70d267503f8ab36e74
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Aug 7 23:08:48 2016 +0200
+
+    rpm: proper packaging of udev directories
+    
+    The issue here is that ceph.spec.in does not package the directories
+    /usr/lib/udev and /usr/lib/udev/rules.d. The problem was not showing because
+    hdparm, which is brought in as a build dependency, packages these directories.
+    However, in SUSE a recent update to hdparm changes that and the problem
+    manifests.
+    
+    This PR addresses the issue by adding udev as a build dependency, which should
+    cover /usr/lib/udev, and by adding explicit "%dir %{_udevrulesdir}" to the
+    packages that put files in this directory.
+    
+    Fixes: http://tracker.ceph.com/issues/16949
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1a164bc94d661b9d83b4a80e957712ed6f5cd122)
+
+commit 7d92e2ea1855a21f9a444c716c590e90dcb9f978
+Author: songbaisen <song.baisen@zte.com.cn>
+Date:   Mon Apr 25 10:34:42 2016 +0800
+
+    crush: When consider rule size use the correct method.
+    
+    Signed-off-by: song baisen <song.baisen@zte.com.cn>
+    (cherry picked from commit 4cf4791ea1200528ccc0d65b3d4a0fbc234c5df0)
+
+commit c2cf21dcce9f481a53c8309800707c80c77754e1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 5 14:27:43 2016 +0800
+
+    mon/PGMonitor: calc the %USED of pool using used/(used+avail)
+    
+    we were using
+    "the raw space used by this pool" / "the raw space of the whole cluster"
+    as the %USED. but it's wrong:
+    
+    - there is chance that not all OSDs are assigned to the pool in question
+    - we can not utilize all the raw space for the pool: there is overhead.
+      and the size of available space for a pool is capped by the assigned
+      OSD with minimal free space.
+    
+    so we should use
+     USED / (USED + AVAIL)
+    as the %USED. so once we have no AVAIL space left, %USED will be 100%.
+    
+    Fixes: http://tracker.ceph.com/issues/16933
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08532ea166dfd97c548d9f1fb478e163021cdda3)
+
+commit 109da6fd2fa8a6f8e56a1fe2daa6a738bb2a1fde
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Aug 7 00:58:04 2016 +0800
+
+    mon/PGMonitor: mark dump_object_stat_sum() as static
+    
+    so we can test this method without create an instance of PGMonitor.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af3d3e25823ca9b40ddf09835edb82795ac68f33)
+
+commit bd636662a2ff26d9af13955b08f9ab20f29de771
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 09:52:21 2016 -0400
+
+    librbd: fix possible inconsistent state when disabling mirroring
+    
+    Fixes: http://tracker.ceph.com/issues/16984
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7cfedb54ea0cf496cc4b55d08a787abc2d6a4bbe)
+
+commit 117aa35094c059dbf5770b01ac13a583471e54aa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Jun 26 01:02:03 2016 +0800
+
+    common: instantiate strict_si_cast<long> not strict_si_cast<int64_t>
+    
+    this fixes the build on armf.
+    
+    on 32bit platforms, cstdint is very likely to
+    
+     typedef long long int int64_t;
+    
+    this results in compilation error like
+    
+     `common/strtol.cc:190:75: error: duplicate explicit instantiation of 'T
+     strict_si_cast(const char, std::string) [with T = long long int;
+     std::string = std::basic_string]'
+    
+     [-fpermissive]
+     template int64_t strict_si_cast(const char *str, std::string *err);
+     ^`
+    
+    we can address this by instantiate the primitive type of `long long`
+    instead of `in64_t`.
+    
+    Fixes: http://tracker.ceph.com/issues/16398
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 31db4c5f9f725e13e38f3c90744e299e023d02a4)
+
+commit d93eda88048d2bcefe4be3ea0aaa6ca0289eabbf
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 23:30:25 2016 +0530
+
+    common: add int64_t template for strict_si_cast()
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 8e429d05370fbe7935212d0ae9608e7547f39860)
+
+commit f7cd28460147530cfd265a593b32d02adb93abe6
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sat Apr 30 18:31:37 2016 +0800
+
+    common/config: cast OPT_U32 options using uint32_t
+    
+    the OPT_U32 options was translated using strict_si_cast<int>(), and then
+    cast the converted result to uint32_t. this could cause integer
+    underflow. we could have lifted the burden of checking invalid input
+    from the user of this option to the strict_si_cast<>() function. so in
+    this change, we use strict_si_cast<uint32_t>() instead, before casting
+    the converted value into `uint32_t`.
+    
+    Signed-off-by: Kefu Chai <tchaikov@gmail.com>
+    (cherry picked from commit b7babd6aa671d688eef0af61ca17fd11eec22773)
+
+commit 518883d939f34ec0afa03aea1bac35960fb579f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 25 09:09:40 2016 +0200
+
+    Revert "common: add int64_t template for strict_si_cast()"
+    
+    This reverts commit e3a99c082e3ebd56d5b40d7d94d98e35629df81e.
+
+commit 29a87012f560c213e34b893cd8dd9dedfd0b11da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Aug 22 11:52:45 2016 +0200
+
+    ceph-osd-prestart.sh: check existence of OSD data directory
+    
+    Fixes: http://tracker.ceph.com/issues/17091
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d6ffa770784f0efb88827ee7c65ff2e5f9029434)
+
+commit 80e25b00554e3955f7bee2e03eba6c55a955999b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 08:26:49 2016 -0400
+
+    rbd: bench io-size should not be larger than image size
+    
+    Fixes: http://tracker.ceph.com/issues/16967
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 113775eb583fa28a669c6dc1f8dbf47a8ecd789c)
+    
+    Conflicts:
+    	src/tools/rbd/action/BenchWrite.cc: trivial resolution
+
+commit f7d23d59535dee004686c5baa79c7b0815ab610a
+Author: Michel Normand <normand@linux.vnet.ibm.com>
+Date:   Fri Aug 5 15:19:50 2016 +0200
+
+    LTTng-UST disabled for openSUSE
+    
+    LTTng-UST not yet supported in openSUSE so do not enable lltng for it.
+    The (1) is where is defined "is_opensuse"
+    
+    Remove value for test of suse_version in spec file
+    and change related comment from SLES12 to SLE
+    as per comment in https://github.com/ceph/ceph/pull/10592
+    
+    (1) https://en.opensuse.org/openSUSE:Build_Service_cross_distribution_howto#Detect_a_distribution_flavor_for_special_code
+    
+    Fixes: http://tracker.ceph.com/issues/16937
+    Signed-off-by: Michel Normand <normand@linux.vnet.ibm.com>
+    (cherry picked from commit 7da19b6db7eaefb5fe159375cc5a465a722d3897)
+
+commit e6ac214f8a6d8773de032c83f5fdd7c65b68d53f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 28 15:09:53 2016 -0400
+
+    librbd: request exclusive lock if current owner cannot execute op
+    
+    The initial krbd implementation will not support executing maintenance
+    ops and instead will return -EOPNOTSUPP. In this case, librbd can take
+    the lock and execute the operation.
+    
+    Fixes: http://tracker.ceph.com/issues/16171
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d2d2d90d64663905c2b81f7809f1d636db6b7fb1)
+
+commit 0b93a9a1e7929bc0709ec139fbc6686783f99044
+Author: Wei Jin <wjin.cn@gmail.com>
+Date:   Thu Apr 28 19:00:48 2016 +0800
+
+    fast dispatch heartbeat message
+    
+    Signed-off-by: Wei Jin <wjin.cn@gmail.com>
+    (cherry picked from commit 6f1f717446314618db7a165c75dfd6a76a01f55e)
+
+commit 0dcefd2352c06b719960a2df970ac3cbdcce8f52
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Jul 18 13:57:08 2016 +0800
+
+    mon/osdmonitor: fix incorrect output of "osd df" due to osd out
+    
+    If an osd is automatically marked as out, the output of "osd df"
+    is not right, as follow:
+    
+    -5 10.00999        -  5586G  2989G  2596G     0    0     host ceph192-9-9-8
+    11  0.90999  1.00000   931G   542G   388G 58.25 0.99         osd.11
+    14  0.90999  1.00000   931G   530G   400G 56.97 0.97         osd.14
+    20  0.90999  1.00000   931G   716G   214G 76.99 1.31         osd.20
+    22  0.90999  1.00000   931G   477G   453G 51.29 0.87         osd.22
+    26  0.90999        0      0      0      0     0    0         osd.26
+    28  0.90999  1.00000   931G   587G   343G 63.09 1.07         osd.28
+    30  0.90999  1.00000   931G   602G   328G 64.75 1.10         osd.30
+    16  0.90999  1.00000   931G   589G   341G 63.34 1.08         osd.16
+    18  0.90999  1.00000   931G   530G   400G 56.93 0.97         osd.18
+    24  0.90999  1.00000   931G   202G   728G 21.77 0.37         osd.24
+    32  0.90999  1.00000   931G   477G   454G 51.23 0.87         osd.32
+    
+    Two problems are identified from the above output:
+    
+    1. the total capacity(total, total used, total avial)
+    only includes osd.32, osd.24, osd.18, osd.16, osd.30, osd.28, and other
+    healthy osds such as osd.11, osd.14 etc. are excluded.
+    
+    2. the average utilization/deviation are forced resetted.
+    
+    Fixes: http://tracker.ceph.com/issues/16706
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 1e4735440ca329f4b95d9bd6f58d9efab7d28d20)
+
+commit ac27352743a4ac1e4a4bec422d41a0ad8f17e41a
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Jul 15 16:08:33 2016 +0800
+
+    mon/osdmonitor: initialize local variable "kb_avail_i"
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 98f50cebe1d1708adeeb6fc6d0aa2cdc85ac942c)
+
+commit 430ab1b83e67dfb697b034e669b06b7a600bcc6b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jul 12 16:56:52 2016 +0200
+
+    ceph-disk: timeout ceph-disk to avoid blocking forever
+    
+    When ceph-disk runs from udev or init script, it is in the background
+    and should it block for any reason, it may keep a lock forever. All
+    calls to ceph-disk in these context are changed to timeout.
+    
+    The TimeoutStartSec= and TimeoutStopSec= which are both set via
+    TimeoutSec= do not apply to Type=oneshot services.
+    
+    https://www.freedesktop.org/software/systemd/man/systemd.service.html
+    
+    Fixes: http://tracker.ceph.com/issues/16580
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit bed1a5cc05a9880b91fc9ac8d8a959efe3b3d512)
+
+commit 0b30a1d210710593678462b287ff33f22de52323
+Author: Zengran Zhang <zhangzengran@h3c.com>
+Date:   Mon Apr 25 22:45:56 2016 -0400
+
+    mon/osdmonitor: decouple adjust_heartbeat_grace and min_down_reporters
+    
+    if cancel mon_osd_adjust_heartbeat_grace, we must set mon_osd_min_down_reporters to zero.
+    otherwise the next checking for
+    	reporters_by_subtree.size() >= g_conf->mon_osd_min_down_reporters
+    will allways be fail.the two options should take effect respectively...
+    
+    Signed-off-by: Zengran Zhang <zhangzengran@h3c.com>
+    (cherry picked from commit 7ac5ca5698cfdaab019bf282b537bcc897f9444c)
+
+commit 92581a388462039a7f4dc748e8318c4226f068d3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:43:48 2016 -0700
+
+    rgw: fix marker tracker completion handling
+    
+    Was not tracking high markers correctly. Could only work if there was a single
+    hole in the completion range. Just keep a map of all the complete entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f5801c726efdc2f3067a071e6bb5ac83fd0cd147)
+
+commit fb36dd0c7a72ce5f3affd69d1f77e6222e26de94
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Jun 28 16:17:58 2016 +0800
+
+    rgw: fix potential memory leaks in RGWPutCORS_ObjStore_S3::get_params
+    
+    We should free `data` before return or jump to `done_err` label.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 69c6cf551754e51debb4e645716a89e29517f4ee)
+
+commit 775c78d4bb31640a006d255ec0cb3818b6a5d23a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 20:40:10 2016 -0400
+
+    librbd: prevent creation of v2 image ids that are too large
+    
+    The librbd API is capped at 24 characters for expressing the
+    object prefix for data blocks (including trailing null byte).
+    
+    Fixes: http://tracker.ceph.com/issues/16887
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4de7c8d0a7222c83268d03b99015c6b9d25f124d)
+
+commit 2b272126de6ea6e75b9dcf4fedac5c216093e36d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jul 4 13:54:32 2016 +0300
+
+    librbd: prevent creation of clone from non-primary mirrored image
+    
+    Fixes: http://tracker.ceph.com/issues/16449
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ba849e3b04a5c513849d40a7fe4151375265302a)
+
+commit 18a66cb375bf6540a3908f843b004c652eacc2dc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 18:09:29 2016 -0400
+
+    librbd: journal::Replay no longer holds lock while completing callback
+    
+    Fixes: http://tracker.ceph.com/issues/16433
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3112a93b49c24f9ae101a7f252c8e708e0d3e260)
+
+commit 79e2acb9a585ede5b97d804550b5394f277e7cc4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 1 20:44:35 2016 +0800
+
+    crush: reset bucket->h.items[i] when removing tree item
+    
+    * crush: so we don't see the reference after the removing, this keeps
+      check_item_loc() happy, and move_bucket() use check_item_loc() to see if
+      the removed bucket disappears after the removal.
+    * test: also add unittest_crush_wrapper::CrushWrapper.insert_item
+    
+    Fixes: http://tracker.ceph.com/issues/16525
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a7069c7aab6b3f605f3d8f909af96f87977e8698)
+
+commit d94e5bc0903a40d8ea353327ddaa7b8567b95f5f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 27 16:53:26 2016 -0400
+
+    common: fix value of CINIT_FLAG_DEFER_DROP_PRIVILEGES
+    
+    0x16 (binary 10110) was overlapping two other flags
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dfb897ad576e4861a3cb75d85dffbbeaad5e980e)
+
+commit 54b6b5e42bca2b7f05b9ff77315de61d3ba14daf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 8 17:47:58 2016 +0800
+
+    client: kill QuotaTree
+    
+    Multiple clients can modify cephfs at the same time. It is
+    very tricky to keep QuotaTree consistant with the global FS
+    hiberarchy. This patch kills the quota tree.
+    
+    After removing the quota tree, we traverse inode's path to
+    find quota root.
+    
+    Fixes: http://tracker.ceph.com/issues/16066
+    Fixes: http://tracker.ceph.com/issues/16067
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 161954bbfeb13d5588668637d5258221948128ea)
+
+commit 47605a2fdde65265e3c4dc60aac206c6ae712be5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:46:06 2016 -0400
+
+    msg/msg_types: update sockaddr, sockaddr_storage accessors
+    
+    Return sockaddr_storage by value.  New sockaddr-based accessors.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 0038d362d38181bd348881bc0fbe717211eb8d15)
+
+commit d5c77c8ff6ad0e4b4242669e4f218f607f433310
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri May 20 15:47:23 2016 +0530
+
+    osd: add peer_addr in heartbeat_check log message
+    
+    Fixes: http://tracker.ceph.com/issues/16337
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit f235b9cc1541d33c210e2d56e8061e8908b91ac9)
+
+commit 5ffee353cb947bfb79925ca4d62daec248c3ba4a
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri May 20 23:09:46 2016 +0530
+
+    mon : Display full flag in ceph status if
+    full flag is set
+    
+    Fixes: http://tracker.ceph.com/issues/16069
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 6b1c894b2b083bf6cead21e9f96d304b2eb7887d)
diff --git a/doc/changelog/v10.2.5.txt b/doc/changelog/v10.2.5.txt
new file mode 100644
index 000000000..53d3f1006
--- /dev/null
+++ b/doc/changelog/v10.2.5.txt
@@ -0,0 +1,37 @@
+commit c461ee19ecbc0c5c330aca20f7392c9a00730367
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Fri Dec 9 20:08:24 2016 +0000
+
+    10.2.5
+
+commit 2c7d2b99d3be6f23188af4bf95f2d6f4a4fb516a
+Merge: 9411351 f7abffe
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 8 07:55:27 2016 -0800
+
+    Merge pull request #12376 from liewegas/wip-msgr-eagain-loop-jewel
+    
+    msg/simple/Pipe: avoid returning 0 on poll timeout
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f7abffec751e454d119df273dc6e49e5f7106078
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 18:25:55 2016 -0600
+
+    msg/simple/Pipe: avoid returning 0 on poll timeout
+    
+    If poll times out it will return 0 (no data to read on socket).  In
+    165e5abdbf6311974d4001e43982b83d06f9e0cc we changed tcp_read_wait from
+    returning -1 to returning -errno, which means we return 0 instead of -1
+    in this case.
+    
+    This makes tcp_read() get into an infinite loop by repeatedly trying to
+    read from the socket and getting EAGAIN.
+    
+    Fix by explicitly checking for a 0 return from poll(2) and returning
+    EAGAIN in that case.
+    
+    Fixes: http://tracker.ceph.com/issues/18184
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6c3d015c6854a12cda40673848813d968ff6afae)
diff --git a/doc/changelog/v10.2.6.txt b/doc/changelog/v10.2.6.txt
new file mode 100644
index 000000000..4628c7cad
--- /dev/null
+++ b/doc/changelog/v10.2.6.txt
@@ -0,0 +1,5165 @@
+commit 656b5b63ed7c43bd014bcafd81b001959d5f089f
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Mar 7 13:29:40 2017 +0000
+
+    10.2.6
+
+commit 420a9a0796e327676cb704bc18b2020f9941d7b6
+Merge: 9267dec 88f4895
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Mar 2 14:00:13 2017 -0700
+
+    Merge pull request #13749 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/upgrade/hammer-x: Add some volumes
+
+commit 88f48955900e9401680f5434f9a162907b146e74
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Mar 2 10:57:54 2017 -0700
+
+    qa/suites/upgrade/hammer-x: Add some volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit 9267dec7e794afb67d0904bf9795e9ac7a0b2a78
+Merge: c66683d 0c242d1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Mar 1 16:35:37 2017 -0700
+
+    Merge pull request #13708 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+commit 0c242d18c4f4b2009467d6cbc0ef26a08a9d9865
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Feb 28 13:07:15 2017 -0700
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+    
+    Looks like we only need two per node, since there is only one OSD per
+    node, and ceph-deploy wants two disks per OSD to account for the
+    journal.
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 87072e277c9ef259c9ee2ae1f761e252aa216713)
+
+commit c66683d908de4a747de9ddffee353206f50bf50a
+Merge: d5a778c ccd0265
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Mar 1 07:26:30 2017 -0800
+
+    Merge pull request #13710 from tmuthamizhan/wip-mv-centos-version
+    
+    qa: replace centos 7.2 with centos 7.3
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit d5a778ced47a7601cfcea79b660ba6367e740e01
+Merge: 7187b61 2cbec5b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 28 16:05:32 2017 -0800
+
+    Merge pull request #13705 from ceph/wip_remove_dumpling_jewel
+    
+    qa: Removed dumplin test 13234.yaml as not needed anymore
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ccd0265a0540e4c4e9423b177b04fae60716bc1c
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Tue Feb 28 16:02:57 2017 -0800
+
+    qa: replace centos 7.2 with centos 7.3
+    
+    s/centos_7.2/centos_7.3
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 2cbec5b909920a05afd34e83c6fa00c5ece4a991
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Feb 28 15:17:54 2017 -0800
+
+    Removed dumplin test 13234.yaml as not needed anymore
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 7187b617ccdc7ba70134ba468eb4c3848a555a41
+Merge: ac3ba2a 771e1d9
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Feb 27 12:23:18 2017 -0700
+
+    Merge pull request #13674 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/{ceph-ansible,rest}: OpenStack volumes
+
+commit 771e1d98a2b5cd437dcf68e03d504f3dec4f2e06
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:14:41 2017 -0700
+
+    qa/suites/rest: Openstack volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 99d942145f4206c00aca30c0bb74f0edc4bac798)
+
+commit a18640fcd7df45eea0baf2b0cfb1d717f982490b
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:06:26 2017 -0700
+
+    qa/suites/ceph-ansible: Openstack volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 964b983bdbd412311bce56184d12d1b6d43c7f28)
+
+commit ac3ba2adcd21ac011ad556ac4506623e61fbe696
+Merge: 016238b 841688b
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 17:22:17 2017 -0700
+
+    Merge pull request #13642 from zmc/wip-fs-openstack-jewel
+    
+    qa/suites/fs: Add openstack volume configuration
+
+commit 841688bb2e46bade3ede4a902c0b42a99e6a62f6
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 15:25:19 2017 -0700
+
+    qa/suites/fs: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit b076d89a3f1fbad7d477913812b2e17529abeacf)
+
+commit 016238b06938064afb6debb2c6dd753037c05279
+Merge: 48bc625 9778743
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:44:10 2017 -0700
+
+    Merge pull request #13639 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/{knfs,hadoop,samba}: OpenStack volume configuration
+
+commit 48bc625c721f426b29eed4f19f030f1243048649
+Merge: f5643f8 ba35859
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:43:40 2017 -0700
+
+    Merge pull request #13636 from zmc/wip-kcephfs-openstack-jewel
+    
+    qa/suites/kcephfs: Openstack volume configuration
+
+commit 9778743547fb9337e98e636fb7ad801fe1ff39ca
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:46:54 2017 -0700
+
+    qa/suites/samba: Openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit e0296d706422ea4dc01d84f8786f6f7104c3d996)
+
+commit cd1e8ef4fc2ccfe5abc11b6282a496185af49455
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:45:18 2017 -0700
+
+    qa/suites/hadoop: Openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 3fef0a49da2ccfdceba7b98e9096be8305da1111)
+
+commit ac7add155f0b787d6a7a55b597da84278b785ae8
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:44:33 2017 -0700
+
+    qa/suites/knfs: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 62c6fd3371adf0f420c12d9c7e2b3a2a0c69256b)
+
+commit ba35859ef26b49ecab750cd36c87a98315e8c023
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:37:23 2017 -0700
+
+    qa/suites/kcephfs: Openstack volume configuration
+    
+    (cherry picked from commit ec6fb28eaf8e2db327e4afc115879a40c7664e07)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit f5643f8818772a4778e2c553d483c7c6ab67ac18
+Merge: bfac1be aced718
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:41 2017 -0700
+
+    Merge pull request #13632 from zmc/wip-krbd-openstack-jewel
+    
+    qa/suites/krbd: Add openstack volume configuration
+
+commit bfac1bee09c90ec81b6d969ecaccbbe047a82604
+Merge: d9eaab4 94d5888
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:22 2017 -0700
+
+    Merge pull request #13612 from ceph/wip-rgw-openstack-jewel
+    
+    qa/suites/rgw: Add openstack volume configuration
+
+commit aced718032fdfedfdfad441a5761f26f04028af3
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 11:17:45 2017 -0700
+
+    qa/suites/krbd: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 201b4d0d1e92bf95ac6a8f2951b664763030f12d)
+
+commit 94d5888cea0ddac8fa03b32225c937955d83dd86
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Feb 23 10:14:05 2017 -0700
+
+    qa/suites/rgw: Add openstack volume configuration
+    
+    Without this, OSDs will fail to create on instances whose root fs isn't
+    xfs.
+    
+    (cherry picked from commit 8af4c35f9577ef5a88307ea5cbbe2561a473926c)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit d9eaab456ff45ae88e83bd633f0c4efb5902bf07
+Merge: ecdfb7c bf3400f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Feb 23 16:32:20 2017 +0100
+
+    Merge pull request #13184 from smithfarm/wip-18720-jewel
+    
+    jewel: build/ops: systemd restarts Ceph Mon to quickly after failing to start
+
+commit ecdfb7ce0c29aa5eb954b3644e5be1544e018515
+Merge: 96b4cb8 5c328f0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:32:01 2017 +0100
+
+    Merge pull request #13240 from smithfarm/wip-18804-jewel
+    
+    jewel: tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 96b4cb8eab5e7531eeaf412cf4b10db5947bef70
+Merge: 27095f3 20e7502
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:30:30 2017 +0100
+
+    Merge pull request #13058 from wido/issue-18635-jewel
+    
+    jewel: systemd: Restart Mon after 10s in case of failure
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 27095f36b51a91ff5575bd42d12a16d2af11b731
+Merge: 0766a8b 67e7a90
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:28:16 2017 +0100
+
+    Merge pull request #13048 from SUSE/wip-18457-jewel
+    
+    jewel: selinux: Allow ceph to manage tmp files
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0766a8bc604d0c4ee7a7d93427022105f5eb6425
+Merge: 9f36610 463e88e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 22 22:59:41 2017 +0100
+
+    Merge pull request #13131 from ovh/bp-osdmap-hammer-compat
+    
+    jewel: mon: OSDMonitor: clear jewel+ feature bits when talking to Hammer OSD
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9f366100bb7c9882146def8b5ed9ce9eddd4db21
+Merge: 083e0f4 78d296b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 22 22:36:07 2017 +0100
+
+    Merge pull request #13255 from liewegas/wip-enxio-jewel
+    
+    jewel: osd: do not send ENXIO on misdirected op by default
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 083e0f4ac06f922c32b806d6d027f738c5057cae
+Merge: 04a8492 8a75f98
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Feb 21 21:40:14 2017 +0100
+
+    Merge pull request #13050 from SUSE/wip-18406-jewel
+    
+    jewel: tests: Cannot reserve CentOS 7.2 smithi machines
+
+commit 04a8492dfed7ec6f235cb201ab6de14201e27bbd
+Merge: d10a8e6 ebb2f73
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Feb 21 10:27:51 2017 +0100
+
+    Merge pull request #13459 from ukernel/jewel-18708
+    
+    jewel: mds: fix incorrect assertion in Server::_dir_is_nonempty()
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d10a8e6d927ea8171e29b7a7c10afcc2c10c5392
+Merge: a009942 cb7bb0c
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 20 10:20:58 2017 -0600
+
+    Merge pull request #13533 from smithfarm/wip-stress-split-ec
+    
+    jewel: qa/suites/upgrade/hammer-x: break stress split ec symlinks
+
+commit a00994299fb360c1c086baad3b65e6889579a4ba
+Merge: 40eedcc 1e75e23
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 20 10:20:51 2017 -0600
+
+    Merge pull request #13222 from liewegas/wip-hammer-x-jewel
+    
+    jewel: tests: qa/suites/upgrade/hammer-x/stress-split: finish thrashing before final upgrade
+
+commit 40eedcc9428abc8bc988594b6e624bbc3c732405
+Merge: 786b318 e9a6dec
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:42:59 2017 +0100
+
+    Merge pull request #13273 from asheplyakov/jewel-bp-18773
+    
+    jewel: rgw: fix period update crash
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 786b31877fcf1c13bfb5034ca92e5044d5d6a78f
+Merge: 7566139 b06d6f5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:41:56 2017 +0100
+
+    Merge pull request #13341 from dreamhost/wip-17550-jewel
+    
+    jewel: rgw: bucket resharding
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7566139391f43616e7847e806c657c527e7a098d
+Merge: b3bb9cc b1d6c2e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:41:29 2017 +0100
+
+    Merge pull request #13004 from asheplyakov/jewel-bp-18563
+    
+    jewel: rgw: leak from RGWMetaSyncShardCR::incremental_sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cb7bb0c49315cc66130336625ede92c70beaec74
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 20 11:44:49 2017 +0100
+
+    tests: reduce stress-split-erasure-code-x86_64 dependency on stress-split
+    
+    This patch breaks the symlinks for two files from stress-split that
+    are being changed by https://github.com/ceph/ceph/pull/13222 in a way
+    that would break this test otherwise.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit b3bb9cc382e11f535a874675d65015829d1a3254
+Merge: 7d6e915 7cf37f6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 11:48:26 2017 +0100
+
+    Merge pull request #12754 from Abhishekvrshny/wip-18285-jewel
+    
+    jewel: rbd: partition func should be enabled When load nbd.ko for rbd-nbd
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 7d6e915a2f7d78741d8fcdfdc7d44e71236ac48d
+Merge: 7e844b6 401271e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 11:45:29 2017 +0100
+
+    Merge pull request #13183 from smithfarm/wip-17313-jewel
+    
+    jewel: build/ops: add ldap lib to rgw lib deps based on build config
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8ae18863d013655e05a46e53de53f9f97fc4d42d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 20 11:44:49 2017 +0100
+
+    tests: reduce stress-split-erasure-code dependency on stress-split
+    
+    This patch breaks the symlinks for two files from stress-split that
+    are being changed by https://github.com/ceph/ceph/pull/13222 in a way
+    that would break this test otherwise.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7e844b66848ba6e911aa357ebfca0ebf20aa46d3
+Merge: 5cda572 4d4b38e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 17 14:43:00 2017 -0800
+
+    Merge pull request #13502 from tmuthamizhan/wip-ansible-rm-trusty
+    
+    qa: drop ubuntu trusty support
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 4d4b38eca81f7b57e3d3b31e1c13e7ab0ba5b30f
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 17 21:06:43 2017 +0000
+
+    qa: drop ubuntu trusty support
+    
+    ceph-ansible dropped support for OS that doesnt support systemd
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 5cda5722e05784163190b6110dcc2dd164142e90
+Merge: 87f8341 260801f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 22:27:53 2017 +0100
+
+    Merge pull request #13276 from rzarzynski/wip-rgw-18364-2-jewel
+    
+    jewel: rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 87f8341909e791e420a4b9344810e3840c1e83cc
+Merge: 3d9a584 a0ee8b9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 16:30:56 2017 +0100
+
+    Merge pull request #12729 from liupan1111/wip-fix-disable-rgw-compile-error
+    
+    jewel: build/ops: fixed compilation error when --with-radowsgw=no
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3d9a584bf0f335caf65265345faf2a940f901519
+Merge: 9f1e1ce 89248e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 15:43:32 2017 +0100
+
+    Merge pull request #12239 from liupan1111/wip-fix-makefile-bug
+    
+    jewel: build/ops: fixed the issue when --disable-server, compilation fails.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ebb2f73e48092c8d2ecf18c75ce70f1a88c9bb22
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 18 16:23:49 2017 +0800
+
+    mds: fix incorrect assertion in Server::_dir_is_nonempty()
+    
+    when filelock is in XLOCKDONE state. client of xlocker can rdlock
+    the filelock. In that case, only client of xlocker can read the lock.
+    
+    Fixes: http://tracker.ceph.com/issues/18708
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fe4ab52b30079a785be053a9fd0197d6990737fe)
+
+commit 9f1e1ce394c7779420ef27bda793bc719cdb9096
+Merge: 792a0c8 36ff758
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 23:39:11 2017 +0100
+
+    Merge pull request #12380 from dachary/wip-18183-jewel
+    
+    jewel: cephfs metadata pool: deep-scrub error omap_digest != best guess omap_digest
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 792a0c8160a1e239f7388cb1ca5be412d850aa82
+Merge: f95cbe0 3eff1ac
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 17:08:26 2017 +0100
+
+    Merge pull request #13130 from rjfd/wip-18608-jewel
+    
+    jewel: librbd: allow to open an image without opening parent image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f95cbe064639d31c218079d93c5831821e7cd13c
+Merge: 49b16ee 214ce1b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 17:06:53 2017 +0100
+
+    Merge pull request #13233 from smithfarm/wip-18556-jewel
+    
+    jewel: rbd: Potential race when removing two-way mirroring image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49b16eed6bf5303667df6657238735b47f573ebb
+Merge: cd9e5df 8742203
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:48:21 2017 +0100
+
+    Merge pull request #13113 from jcsp/wip-18679
+    
+    jewel: mds: finish clientreplay requests before requesting active state
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cd9e5df1593f999c943376b82de49a3eba291f8c
+Merge: 3192ef6 21622c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:47:59 2017 +0100
+
+    Merge pull request #13139 from jcsp/wip-18100
+    
+    jewel: mds: fix MDSMap upgrade decoding
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3192ef6a034bf39becead5f87a0e48651fcab705
+Merge: e84b3f8 384e5c0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:18:04 2017 +0100
+
+    Merge pull request #12490 from linuxbox2/jewel-rgw-header
+    
+    jewel: rgw: add option to log custom HTTP headers (rgw_log_http_headers)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e84b3f8e073ec02f7a45387768490b90d6d777b4
+Merge: 9b4175e d7a479c4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:14:23 2017 +0100
+
+    Merge pull request #12079 from rzarzynski/wip-rgw-17961
+    
+    jewel: rgw: TempURL properly handles accounts created with the implicit tenant
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 9b4175e491286f0578bcba5152a72e0e3b934051
+Merge: 2420d47 732405e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:13:32 2017 +0100
+
+    Merge pull request #12044 from Abhishekvrshny/wip-17886-jewel
+    
+    jewel: rgw: multisite: ECANCELED & 500 error on bucket delete
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2420d474a8d22f3237b9fb28a78f0ff3f168ba34
+Merge: 81efc40 256b850
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 15 00:39:48 2017 +0100
+
+    Merge pull request #13422 from smithfarm/wip-fix-hammer-x-fix
+    
+    tests: remove extra indentation in exec block
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 81efc40b11fc0f146f5d2388033333aeb9ad9d56
+Merge: 994df97 4824ad2
+Author: Tamilarasi Muthamizhan <tmuthami@redhat.com>
+Date:   Tue Feb 14 13:59:58 2017 -0800
+
+    Merge pull request #13364 from tmuthamizhan/wip-ansible-jewel
+    
+    qa: ceph-ansible smoke suite modified for jewel
+
+commit 256b850fc1a80f71b0b8365c6d66abea84482f9e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Feb 14 22:13:37 2017 +0100
+
+    tests: remove extra indentation in exec block
+    
+    The exec block was indented by an extra space, causing
+    
+    line 439, in parse_block_mapping_key
+        "expected <block end>, but found %r" % token.id, token.start_mark)
+    yaml.parser.ParserError: while parsing a block mapping
+      in "<string>", line 111, column 3:
+          sequential:
+          ^
+    expected <block end>, but found '<block sequence start>'
+      in "<string>", line 126, column 4:
+           - exec:
+           ^
+    
+    when trying to run upgrade/hammer-x
+    
+    Reported-by: Yuri Weinstein <yweins@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 994df972dd64fc39ef7acce1acfff0e9e8440e1f
+Merge: 9bf8dcf 898702d
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 13 18:44:37 2017 -0600
+
+    Merge pull request #13404 from smithfarm/wip-fix-hammer-x
+    
+    tests: add require_jewel_osds to upgrade/hammer-x/tiering
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 898702d58f3e26c41713e44625fcf357841f52af
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 13 22:04:17 2017 +0100
+
+    tests: add require_jewel_osds to upgrade/hammer-x/tiering
+    
+    Without this, the test tends to fail, but sometimes passes (apparently because
+    the "wait-for-healthy: true" in the last ceph.restart is racy - HEALTH_OK is
+    received before the MONs notice that all OSDs are running jewel without
+    require_jewel_osds.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 214ce1b96c112c193614aeeabaa278bd1004f6b2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 31 22:49:36 2016 +0300
+
+    librbd: async method to check journal tag owner
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 953561f84140efcc870ec4a8f1ecb732b8e99409)
+    
+    Conflicts:
+    	src/librbd/Journal.cc - trivial context difference (master does not
+                                    have "class SafeTimerSingleton"),
+                                    added "typedef ::journal::Journaler Journaler;" in
+                                    C_IsTagOwner template to fix build failure
+
+commit 5723b932cfb12ed67cb54347a127c5f3539d0bcf
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 10:40:46 2017 +0100
+
+    rbd-mirror: check image mirroring state when bootstrapping
+    
+    Fixes: http://tracker.ceph.com/issues/18447
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5fc5a8ac895524f05eed6e7db20b0dda3a8cb60f)
+
+commit 8361a60c3cb272d2d4895840dadbbc6c50294f51
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 09:23:19 2017 +0100
+
+    rbd-mirror: async request to test if image is primary
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0a1cb35caacdf85029f31a0364dc07a5d7462f5f)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/CMakeLists.txt - file doesn't exist in jewel
+                (ported the patch to src/tools/Makefile-client.am)
+    	src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc - Journal is
+                 implemented by a typedef in jewel
+    	src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc -
+                 no compat.h include in jewel, Journal is implemented by a typedef
+                 in jewel
+
+commit 9bf8dcf914e76a340fed29d2b46346f8ada6d0c7
+Merge: 748f75c 7515a77
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 13 08:20:53 2017 +0100
+
+    Merge pull request #13331 from smithfarm/wip-18869-jewel
+    
+    jewel: tests: SUSE yaml facets in qa/distros/all are out of date
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 748f75cd845e3b33687d9d235508f9bc7ebdcefb
+Merge: c27aba8 21ded74
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Mon Feb 13 14:52:59 2017 +0800
+
+    Merge pull request #11410 from dachary/wip-17334-jewel
+    
+    jewel: crushtool --compile is create output despite of missing item
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 21ded748e7bbf2c02b3bf381cf796afe99a62ce0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 21 22:58:49 2016 +0800
+
+    test/cli/crushtool: fix the test of compile-decompile-recompile.t
+    
+    should read the map from $TESTDIR,
+    it's a regression introduced by b2c0a07
+    
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 404043980d7882f320f48fbaed4b18f31bb15eb7)
+
+commit d92738c0c9491c97fd189ff49ca57427316b9eb7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 14:10:16 2016 +0800
+
+    crush/CrushCompiler: error out as long as parse fails
+    
+    do not output compiled crush map if anything goes wrong when parsing
+    crush map.
+    
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b2c0a079dc074c907e1bc429861230096534f597)
+
+commit 4824ad27345f32ed0043c2cb20fe5fc7e019555a
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 10 12:37:51 2017 -0800
+
+    qa: ceph-ansible smoke suite modified for jewel
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit c27aba8f723b013baeb0704fe84c6956d1cb005d
+Merge: 8c87d09 5ed454d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 15:04:02 2017 -0500
+
+    Merge pull request #13358 from linuxbox2/wip-jewel-marker
+    
+    jewel: rgw: fix use of marker in List::list_objects()
+
+commit 5ed454dd8f470749ef661782a8f4e866ae8193e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 26 16:26:42 2017 -0800
+
+    rgw: fix use of marker in List::list_objects()
+    
+    Fixes: http://tracker.ceph.com/issues/18331
+    
+    List marker is an index key, so treat it as such. This
+    fixes infinite loop in orphans find command.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a5d1fa0587184f43c69d8e03114b58d43f320781)
+
+commit b06d6f5f8fb6693dcf4f480d1f80b1081cd63305
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Feb 9 14:38:02 2017 -0800
+
+    Backport bucket reshard to jewel.
+    
+    Fixes: http://tracker.ceph.com/issues/17756
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 25af6e5c2448d2478ac3d8def6141fcbfb086f3c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 17:00:41 2016 -0700
+
+    rgw_admin: add a few admin commands to the usage
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 91ed1216d7465a42d11546cb967c70997ea5b1d8)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit b429331f4fb22e845edcd526b618b10c82db6286
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 16:43:44 2016 -0700
+
+    rgw_admin: add bi purge command
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit aaf0d213eb39192ceb252c9c7db68c1a48ba1272)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit eb65394363676629726aed2aac8182a745ba96e6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 12:54:47 2016 -0700
+
+    rgw: bucket resharding, adjust logging
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e2b8dc6113e2625bdf65ea6f2c42510229d04c87)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit e197ec0312cb07d576c0bf045b803454702ccc81
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 10:41:45 2016 -0700
+
+    cls/rgw: bi_list() fix is_truncated returned param
+    
+    is_truncated was never set. Also, make sure that we don't return
+    more entries than requested.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 47f422a4e0382d53023af6f651433011606b8625)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 81daefa473a4e6826be5afa4dae5ea63d66bd1be
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 15:13:37 2016 -0700
+
+    rgw_admin: require --yes-i-really-mean-it for bucket reshard
+    
+    in the case where num shards are less or equal to current bucket
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 75da4375ee0e36c58f96cbf2920c3b9aadf12733)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 4adb247e56e8e77cbdfecca8f90240edbd04742f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:35:31 2016 -0700
+
+    rgw_admin: better bucket reshard logging
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ac88b5d9e6bc3c8b59c500bf79a0e3daa923a47c)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 139842d7451ef44cb0548676a818d3056e2e4ac6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:11:41 2016 -0700
+
+    rgw: limit bucket reshard num shards to max possible
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d9c88afec4b52848f9ad8957bab5a86fdafecded)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7e4493a4cca99bfa817803c0255755e1cd330cbf
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 11:55:32 2016 -0700
+
+    rgw_admin: fix bi list command
+    
+    Changes scoping of json section, and push the --object param in so that
+    an object can be specified as a filter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 716d096bbb8e836aefa6a451b799389d3bd85620)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 1604b521730577fc6aa5345127381b1bfc5625b7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 15:49:37 2016 -0700
+
+    rgw_admin: use aio operations for bucket resharding
+    
+    also created shards manager to make things slightly cleaner
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97e7ee9ca213ccf4b8f537e02125bd0c4ef24103)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit cd2e35e84599d25e94f28fad8e4a743883fec94d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 10:45:17 2016 -0700
+
+    rgw: bucket reshard updates stats
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dd712384ffe72ee23cbe0a20d7400aff7fb779a8)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 678dac9289ce52f1e5f13e603b5858a3b867c216
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 09:52:29 2016 -0700
+
+    cls/rgw: add bucket_update_stats method
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4cc7d3a33a28602b45ec04ff5384e6bc62b376cb)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit d39eac038266a61a6081c820360c2165279bd8c1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:46:36 2016 -0700
+
+    rgw_admin: reshard also links to new bucket instance
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 094fe3f0cfeb27b32abfc93b07054b60de363a20)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 29c9e9872c2cef6b6e874bf95c797a72a580e84c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:33:57 2016 -0700
+
+    rgw: rgw_link_bucket, use correct bucket structure for entry point
+    
+    The bucket structure might be different than the one that we were using
+    before.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f2d9fc015556d6d70e596f75c382cc7157add411)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit b40ce2738964c6fc1c9c390af18758b16820cdb9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:01:00 2016 -0700
+
+    radosgw-admin: bucket reshard needs --num-shards to be specified
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d0569f913340cb251c1a49f1e470b176d8b34346)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit d9df7451a70a4fe64443e7f4669e995c54d74361
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 05:49:37 2016 -0700
+
+    cls/rgw: fix bi_list objclass command
+    
+    was filtering entries, even if filter was not specified, and need to
+    set boundary for plain entries. Also, list_instance_entries() was not
+    working correctly, and added list_olh_entries().
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f87c4b2c2a6ecadaf9f0e0cfef4b6061878a023b)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 4892def3643555720619b987bc1ccb508cdd8ac7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:34 2016 -0700
+
+    rgw_admin: bucket rehsrading, initial work
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d8c5931bceaa42ad47ae5ad9b2c32bb6c321484a)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+
+commit c1cf61f3612844fc644a832f7c61f716f28ecb46
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:15 2016 -0700
+
+    rgw: utilities to support raw bucket index operations
+    
+    and other related changes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b01e732f3d597670f4f781be3db81786e63d5053)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 0ce2a125f4d400abaf2a65ff90d1d40ee000aea9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:06:17 2016 -0700
+
+    rgw: use bucket_info.bucket_id instead of marker where needed
+    
+    We used to use these interchangeably, but they actually have diffent meaning.
+    The marker is the prefix we assign to the objects in this bucket, whereas
+    the bucket_id is the bucket instance's id. These used to hold the same
+    value, but with bucket resharding it's not going to be true anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 94afaf722fc8289af6e75025ba7d783f11c4b7d0)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 5fe58c5bf22ad593d650d68b059d462be49214ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:01:39 2016 -0700
+
+    cls/rgw: utilities to support raw bucket index operations
+    
+    New flavour of bi_put() call, and a function to extract key off
+    a raw bi entry.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9b3a2a40a5732689be98a940f7e6c3c4a6e73c53)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7515a77bf1d06ec7cea448adecc56e50c417540f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 21:23:54 2017 +0100
+
+    tests: drop buildpackages.py
+    
+    The buildpackages suite has been moved to teuthology. This cleans up a file
+    that was left behind by https://github.com/ceph/ceph/pull/13297
+    
+    Fixes: http://tracker.ceph.com/issues/18846
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6b7443fb50c117ee7f20d53bbc7530bb0eb7ebd5)
+
+commit 36f96f40e7ce9abf1fc21878c91365d300dee281
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 15:27:00 2017 +0100
+
+    tests: update SUSE yaml facets in qa/distros/all
+    
+    Fixes: http://tracker.ceph.com/issues/18856
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0bd56e871a8549d4b0b1211f09dad2d1120fb606)
+
+commit 8c87d0927447233697d623c995163fde25524fa5
+Merge: a00efd8 bb3c594
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 8 13:16:25 2017 +0100
+
+    Merge pull request #13299 from dachary/wip-18848-jewel
+    
+    jewel: tests: remove qa/suites/buildpackages
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 7cf37f68e14935d6d6c3d540ab5fd6b7a2ebb605
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Tue Dec 6 21:04:03 2016 +0800
+
+    rbd-nbd: support partition for rbd-nbd mapped raw block device.
+    
+    Fixes: http://tracker.ceph.com/issues/18115
+    Signed-off-by: Pan Liu pan.liu@istuary.com
+    (cherry picked from commit 42645a301869b08b4be860fcac491ae4189b313a)
+    
+    Conflicts:
+    	src/tools/rbd_nbd/rbd-nbd.cc
+    		Removed exclusive option
+
+commit bb3c5941d15f9e6ae35c51334f0774dc3ef17f64
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 7 18:33:29 2017 +0100
+
+    buildpackages: remove because it does not belong
+    
+    It should live in teuthology, not in Ceph. And it is currently broken:
+    there is no need to keep it around.
+    
+    Fixes: http://tracker.ceph.com/issues/18846
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5a43f8d57925da227c95480501ceec10a29395d8)
+    
+    Conflicts:
+    	qa/tasks/buildpackages/common.sh (remove)
+    	qa/tasks/buildpackages/make-rpm.sh (remove)
+
+commit 5c328f0ec51f435e51357f015bb088a450277ece
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 6 18:43:49 2017 +0100
+
+    tests: fix regression in qa/tasks/ceph_master.py
+    
+    https://github.com/ceph/ceph/pull/13194 introduced a regression:
+    
+    2017-02-06T16:14:23.162 INFO:tasks.thrashosds.thrasher:Traceback (most recent call last):
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 722, in wrapper
+        return func(self)
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 839, in do_thrash
+        self.choose_action()()
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 305, in kill_osd
+        output = proc.stderr.getvalue()
+    AttributeError: 'NoneType' object has no attribute 'getvalue'
+    
+    This is because the original patch failed to pass "stderr=StringIO()" to run().
+    
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit db2582e25e390fcaf75952eb59a73dcff643f49c)
+
+commit a00efd8d2d22aaca89747f4db818866b9d5fc982
+Merge: 84a0fdb 0cf7a61
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 6 16:31:06 2017 -0800
+
+    Merge pull request #13280 from athanatos/wip-revert-jewel-18581
+    
+    Revert "Merge pull request #12978 from asheplyakov/jewel-18581"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0cf7a6133ee0d4609242d94088dd77e83665aa93
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 6 10:20:55 2017 -0800
+
+    Revert "Merge pull request #12978 from asheplyakov/jewel-18581"
+    
+    See: http://tracker.ceph.com/issues/18809
+    
+    This reverts commit 8e69580c97622abfcbda73f92d9b6b6780be031f, reversing
+    changes made to c05730ceb3387fb43c35937f0506297a34a44452.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 260801f9bb78169648c3f2052ebc3e1328113367
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 1 19:05:50 2017 +0100
+
+    rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion.
+    
+    Fixes: http://tracker.ceph.com/issues/18364
+    Fixes: http://tracker.ceph.com/issues/16355
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 871e1f51afe9d6c8b88debc07460b4316121f999)
+
+commit e9a6dec55b894b34e3278dbf6b226063fd461cd5
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 15:05:30 2017 +0200
+
+    rgw: add check for update return value
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 125955e0625461065dc4755b900e51c3598cadb4)
+
+commit a27f6a72ec836af1672e8aa344fb80d067ed20a8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:42:14 2017 +0200
+
+    rgw: we need to reinit the zonegroup after assignment to avoid invalid cct and store
+    
+    Fixes: http://tracker.ceph.com/issues/18631
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac9a7565ddf801121f22476cf3f66668f311833e)
+
+commit 006140adf7413ec3d482971bb44bfc3ffada2fbf
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:40:16 2017 +0200
+
+    rgw: fix init_zg_from_period when default zone is not set as default
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5393077e07bed45b9fc007591d365f1229d3e815)
+
+commit 78d296b076a284e35ef4bae931756e7d17a5c09d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 17:40:29 2017 -0500
+
+    osd: do not send ENXIO on misdirected op by default
+    
+    In practice this tends to get bubbled up the stack as an error on
+    the caller, and they usually do not handle it properly.  For example,
+    with librbd, this turns into EIO and break the VM.
+    
+    Instead, this will manifest as a hung op on the client.  That is
+    also not ideal, but given that the root cause here is generally a
+    bug, it's not clear what else would be better.
+    
+    We already log an error in the cluster log, so teuthology runs will
+    continue to fail.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 923e7f5ce5ed437af15e178299a61029ff48e4a2)
+    
+    # Conflicts:
+    #	PendingReleaseNotes
+    #	src/common/config_opts.h
+
+commit 30fb615c3686154275cf10a567a0c3ad961cbb50
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 31 00:46:22 2017 +0100
+
+    tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+    
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 046e873026c59e733f1844b28ffdc030cbe57b36)
+
+commit 3eff1ac2d680d2f6ae1f7ff0d8fe6b6329a17522
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jan 10 15:11:19 2017 +0000
+
+    librbd: allow to open an image without opening parent image
+    
+    Fixes: http://tracker.ceph.com/issues/18325
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 61af1c25015de087a2423811548d975dd7d430b4)
+    
+    Conflicts:
+      src/librbd/ImageState.cc - added missing arg to RefreshRequest::create
+      src/librbd/exclusive_lock/PostAcquireRequest.cc -
+                                            deleted, does not exist in jewel
+      src/librbd/image/OpenRequest.cc - added missing arg to
+                                        RefreshRequest::create
+      src/librbd/internal.cc - added missing arg to ImageState::open
+      src/librbd/librbd.cc - added missing arg to ImageState::open
+      src/test/librbd/exclusive_lock/test_mock_PostAcquireRequest.cc -
+                                            deleted, does not exist in jewel
+      src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc -
+                                            added missing arg to ImageState::open
+      src/test/rbd_mirror/test_PoolWatcher.cc - added missing arg to
+                                                ImageState::open
+
+commit bee1d2c24ab463a0f5c1e28cb70f6b89e0278d22
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 15:02:02 2017 +0100
+
+    rbd-mirror: hold owner lock when testing if lock owner
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 82aa89668d666f434cd19ff444223017b5512c6a)
+
+commit 463e88e24378470ef405bdfb2eb823b04e017e5e
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jan 20 16:07:10 2017 +0100
+
+    OSDMonitor: clear jewel+ feature bits when talking to Hammer OSD
+    
+    During upgrade from Hammer to Jewel, when upgrading MONs first and OSDs
+    last, Jewel MONs send OSDMaps with components in encoding version not
+    encodable by Hammer OSDs, generating extra load on MONs due to requests
+    for full OSDMap after failing the CRC check.
+    Fix this by not including CEPH_FEATURE_NEW_OSDOP_ENCODING (which
+    is responsible for encoding pg_pool_t in version 24 instead of 21) and
+    CEPH_FEATURE_CRUSH_TUNABLES5 (responsible for adding chooseleaf_stable
+    field into encoded CRUSH map) when CEPH_OSDMAP_REQUIRE_JEWEL flag
+    is not present.
+    Note that this issue applies only to upgrade from Hammer to Jewel,
+    because direct upgrade from Hammer to any other later release is not
+    supported. For that reason, there is no need to have this patch in any
+    release other than Jewel.
+    
+    Fixes: http://tracker.ceph.com/issues/18582
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+
+commit b1d6c2ebee2e6216cb6aa2afaa9e32ce5e571aa9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 13:10:50 2017 -0500
+
+    rgw: RGWCloneMetaLogCoroutine uses RGWMetadataLogInfoCompletion
+    
+    Fixes: http://tracker.ceph.com/issues/18412
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 34a2edbf53f26a21e3206027dc61efc70d9c6aad)
+    
+    Minor changes:
+    	RGWCloneMetaLogCoroutine::state_read_shard_status(): use
+    	boost::intrusive_ptr ctor instead of reset() to be compatible
+    	with older boost versions (there's no bundled boost in jewel)
+
+commit 7ca400b9bd7b6fd382a9143b450de5a2dbcf057a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 13:10:18 2017 -0500
+
+    rgw: expose completion for RGWMetadataLog::get_info_async()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7b3532e2ad0f478a1670ceb61fd68209df87aa8b)
+
+commit 335a7326a9e8416b114467096d938fafe6b09605
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Dec 21 14:32:04 2016 -0500
+
+    rgw: RGWMetaSyncShardCR drops stack refs on destruction
+    
+    if the coroutine is canceled before collect_children() can clean up
+    all of its child stacks, those stack refs will leak. store these
+    stacks as boost::intrusive_ptr so the ref is dropped automatically on
+    destruction
+    
+    Fixes: http://tracker.ceph.com/issues/18300
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 060fe72faf6a483a36d481207c6624c46a414231)
+
+commit e5646a0192c1968a542b42dbfe907565b604971e
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 26 11:19:50 2016 -0400
+
+    rgw: librados aio wait_for_safe, not wait_for_complete
+    
+    We want to wait for the op to be durable, not ordered.
+    
+    Note that wait_for_safe works for read ops as well, despite a
+    name that implies an update.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e447eafb34a7019ca15189d73a3ad3341640dc5)
+
+commit 84a0fdb8b464734a58bfc98edb010f1278fa25f2
+Merge: 96626c2 73cef9f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Feb 2 00:08:29 2017 +0100
+
+    Merge pull request #11872 from dachary/wip-17838-jewel
+    
+    jewel: leak in RGWFetchAllMetaCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 96626c2555b4bab7c293ee8a6f3382614c6306d1
+Merge: 3322556 73d2114
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:58:29 2017 +0100
+
+    Merge pull request #13182 from smithfarm/wip-18498-jewel
+    
+    jewel: rgw: Realm set does not create a new period
+
+commit 3322556f3ebabcd82be703bcffbd539454b191e0
+Merge: bc704b1 33c6ef5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:57:40 2017 +0100
+
+    Merge pull request #13180 from smithfarm/wip-18547-jewel
+    
+    jewel: rgw: multisite: segfault after changing value of rgw_data_log_num_shards
+
+commit bc704b1beb1897286e6eaa62f03ba8ddabb17ee9
+Merge: f46c125 8b124c8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:56:21 2017 +0100
+
+    Merge pull request #13177 from smithfarm/wip-18676-jewel
+    
+    jewel: rgw: librgw: objects created from s3 apis are not visible from nfs mount point
+
+commit f46c1259dcd63ce335715817e3aeb18def59c503
+Merge: a3cb5e6 2cb0307
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:55:25 2017 +0100
+
+    Merge pull request #13175 from smithfarm/wip-18684-jewel
+    
+    jewel: rgw: multisite: sync status reports master is on a different period
+
+commit a3cb5e61d18cc48d176b0630fb21ebc603472b4c
+Merge: 73ccbdc 0a47342
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:54:33 2017 +0100
+
+    Merge pull request #13173 from smithfarm/wip-18710-jewel
+    
+    jewel: rgw: slave zonegroup cannot enable the bucket versioning
+
+commit 73ccbdce630350a4601255483469959f00829c45
+Merge: 5d0f2f3 7e51bec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:53:39 2017 +0100
+
+    Merge pull request #13171 from smithfarm/wip-18712-jewel
+    
+    jewel: rgw: radosgw-admin period update reverts deleted zonegroup
+
+commit 5d0f2f3ec893f5990860f901f40b870a6fd2e737
+Merge: 2f4990d e8f55f6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:13:11 2017 +0100
+
+    Merge pull request #13001 from asheplyakov/jewel-bp/18559
+    
+    jewel: rgw multisite: fix ref counting of completions
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2f4990d7a6cb355b059e5b8cfb97b6e371155dde
+Merge: a379c01 c21622d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:10:37 2017 +0100
+
+    Merge pull request #12997 from asheplyakov/jewel-bp-18569
+    
+    jewel: radosgw valgrind "invalid read size 4" RGWGetObj
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a379c01ee916e73c12da4ba8ef1c7620be81e3b8
+Merge: 5834732 9a59ce9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:02:14 2017 +0100
+
+    Merge pull request #12764 from SUSE/wip-17342-jewel
+    
+    jewel: tests: assertion failure in a radosgw-admin related task
+
+commit 5834732ad44d67b9414d782799f41e320d66bcf2
+Merge: 9486ccd 9a1258d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:01:02 2017 +0100
+
+    Merge pull request #12738 from SUSE/wip-18286-jewel
+    
+    jewel: rgw: multisite: coroutine deadlock in RGWMetaSyncCR after ECANCELED errors
+
+commit 9486ccd9bd13e3020947a0d669c2e70248f24bfb
+Merge: edf2e6e 5aa9387
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:00:12 2017 +0100
+
+    Merge pull request #12678 from Abhishekvrshny/wip-18348-jewel
+    
+    jewel: rgw ldap: enforce simple_bind w/LDAPv3 redux
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit edf2e6eaccf5d0fa848d56c0920d44c980a86298
+Merge: 2f087a3 d584f9e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:45:54 2017 +0100
+
+    Merge pull request #12622 from wido/jewel-15776
+    
+    jewel: rgw: log name instead of id for SystemMetaObj on failure
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 2f087a37bc3d9ed86c4a6268bd21fe42c843cbb7
+Merge: 2d5f8fd 87a2a95
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:39:52 2017 +0100
+
+    Merge pull request #12542 from dreamhost/wip-17076-jewel
+    
+    jewel: rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth.
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2d5f8fda98cba07ce48a3e85896c83b3459a5973
+Merge: 9897021 ddb5403
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:38:17 2017 +0100
+
+    Merge pull request #12428 from SUSE/wip-18216-jewel
+    
+    jewel: rgw-admin: missing command to modify placement targets
+
+commit 9897021fd29716974ba547814c35f8a606dce88f
+Merge: bf1e63b 26c87fd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:36:19 2017 +0100
+
+    Merge pull request #12426 from SUSE/wip-18214-jewel
+    
+    jewel: add max_part and nbds_max options in rbd nbd map, in order to keep consistent with
+    
+    http://tracker.ceph.com/issues/17851#note-59
+
+commit bf1e63b6441af02404ad680462de5380e8714c43
+Merge: 9c83af8 8b10d3b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:34:15 2017 +0100
+
+    Merge pull request #12419 from SUSE/wip-18217-jewel
+    
+    jewel: rgw sends omap_getvals with (u64)-1 limit
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9c83af87ee8716dd7785c2e21fcc8007a92f9c92
+Merge: c0e845c fe753db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:32:24 2017 +0100
+
+    Merge pull request #12405 from jan--f/wip-18199-jewel
+    
+    jewel: build/ops: install-deps.sh based on /etc/os-release
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c0e845c4f6b03f3c28c31ba2278be5b20e5be13c
+Merge: 45281fb 4eb7c73
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:29:24 2017 +0100
+
+    Merge pull request #12397 from linuxbox2/jewel-next
+    
+    jewel: rgw: do not abort when accept a CORS request with short origin
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 45281fb4d3ddca81d051cd776269cb111cc26bc1
+Merge: 1f85f84 7db6d1d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:15:35 2017 +0100
+
+    Merge pull request #12316 from dachary/wip-18101-jewel
+    
+    jewel: Add workaround for upgrade issues for older jewel versions
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1f85f847b6a8413047891d6d46e838ad83d5a8b7
+Merge: 735be97 3839727
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 21:37:45 2017 +0100
+
+    Merge pull request #12890 from dillaman/wip-18453-jewel
+    
+    jewel: librbd: new API method to force break a peer's exclusive lock
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1e75e23b9973c9c5203b3c3ed2cec35333b40d77
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 1 15:24:50 2017 -0500
+
+    qa/suites/upgrade/hammer-x: wrap thrash and workloads
+    
+    We need the thrashing to stop before we do the final upgrade step.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 735be9772f0029eb6c7f9aef7d0469107897521a
+Merge: ce309b8 5400673
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 20:23:36 2017 +0100
+
+    Merge pull request #12302 from SUSE/wip-18135-jewel
+    
+    jewel: build/ops: add hostname sanity check to run-{c}make-check.sh
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ce309b8493a3e69498d3495bbbdfa7b2c2e31cf5
+Merge: 4cbe0e3 478e40a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:18:04 2017 +0100
+
+    Merge pull request #12315 from dachary/wip-18102-jewel
+    
+    jewel: rgw: Unable to commit period zonegroup change
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4cbe0e398ad9b46eefafbfd2d0202d0cbb65402a
+Merge: e371af6 b502b96
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:14:15 2017 +0100
+
+    Merge pull request #12314 from dachary/wip-18107-jewel
+    
+    jewel: multisite: failed assertion in 'radosgw-admin bucket sync status'
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e371af6b7d17020c1bda09f0de28036a920303fd
+Merge: 056f8ac ece622d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:12:40 2017 +0100
+
+    Merge pull request #12313 from dachary/wip-18112-jewel
+    
+    jewel: rgw: multisite requests failing with '400 Bad Request' with civetweb 1.8
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 056f8ac054c82358ec4ea20782a7da327e6ad19e
+Merge: 41fcf74 d9c1d86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:02:05 2017 +0100
+
+    Merge pull request #12258 from rzarzynski/wip-rgw-17931-jewel
+    
+    jewel: rgw: add support for the prefix parameter in account listing of Swift API
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 41fcf740ba9205c08a2995fd69dc29051ed81b6a
+Merge: 8a86bf5 a0b4e60
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:56:48 2017 +0100
+
+    Merge pull request #12156 from dachary/wip-17969-jewel
+    
+    jewel: rgw: multisite upgrade from hammer -> jewel ignores rgw_region_root_pool
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8a86bf508503f7b22b47c6663b56f6fd7012df4c
+Merge: d71c75a d1df8f7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:34:45 2017 +0100
+
+    Merge pull request #11990 from asheplyakov/jewel-17908
+    
+    jewel: rgw: for the create_bucket api, if the input creation_time is zero, we …
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d71c75ac6bc2ed576344e0f75fe99ce860b8914d
+Merge: 7d0c6de 6a3c10f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:31:21 2017 +0100
+
+    Merge pull request #11876 from dachary/wip-17839-jewel
+    
+    jewel: rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7d0c6dee032c1254a2c78b589bb48bea335ca0f2
+Merge: a695e8d 3239ce8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:26:21 2017 +0100
+
+    Merge pull request #11868 from dachary/wip-17512-jewel
+    
+    jewel: multisite: metadata master can get the wrong value for 'oldest_log_period'
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a695e8d167ada37260b4ef8ed4595f758fc4421b
+Merge: 4395560 0b7577e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:24:11 2017 +0100
+
+    Merge pull request #11866 from dachary/wip-17709-jewel
+    
+    jewel: rgw: multisite: coroutine deadlock assertion on error in FetchAllMetaCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4395560d686cc7ccdc28eaba1f6cbadf57e1bba5
+Merge: 1bc9432 8d3b0e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:21:33 2017 +0100
+
+    Merge pull request #11497 from rzarzynski/wip-rgw-slashinfo-jewel
+    
+    jewel: rgw: add suport for Swift-at-root dependent features of Swift API
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1bc9432e744c42fae98c570f09ddc3f4606d2297
+Merge: 92d385e 6c1edcd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:18:42 2017 +0100
+
+    Merge pull request #11477 from dachary/wip-17119-jewel
+    
+    jewel: rgw: multisite: assert(next) failed in RGWMetaSyncCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 92d385e8cb4cff91051464a8552ab5aed4cbfae4
+Merge: a61be36 d54b354
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:17:03 2017 +0100
+
+    Merge pull request #11476 from dachary/wip-17162-jewel
+    
+    jewel: rgw: multisite doesn't retry RGWFetchAllMetaCR on failed lease
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a61be367615f7ef5f8101d0fbbe40615782d8c5c
+Merge: 5821e8d aa24a8f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:15:01 2017 +0100
+
+    Merge pull request #11470 from dachary/wip-17514-jewel
+    
+    jewel: rgw：bucket check remove _multipart_ prefix
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 5821e8d9c448f95f9a4b73e6353db8f7d2cfb534
+Merge: bb702bc 19a836b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:12:51 2017 +0100
+
+    Merge pull request #10661 from dachary/wip-16871-jewel
+    
+    jewel: rgw: Have a flavor of bucket deletion in radosgw-admin to bypass garbage collection
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit bb702bc4b9b720af86f759d0e040d57a9a91b3da
+Merge: c5461bc 18545a2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 14:02:56 2017 +0100
+
+    Merge pull request #13187 from asheplyakov/jewel-bp-18729
+    
+    jewel: cli: ceph-disk: convert none str to str before printing it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c5461bc9703a9062c3c2ea8022b9f4d4842e0b65
+Merge: 1c6eacb 1481c8f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:29:50 2017 +0100
+
+    Merge pull request #13153 from smithfarm/wip-no-firefly-on-centos
+    
+    jewel: tests: upgrade:hammer-x: install firefly only on Ubuntu 14.04
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1c6eacb6f905cd39248663383da8f797aaef9ebe
+Merge: 5784855 5c4fffa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:17:48 2017 +0100
+
+    Merge pull request #13118 from smithfarm/wip-18274-jewel
+    
+    jewel: Memory leaks in object_list_begin and object_list_end
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5784855782385186a40696c8bcec151403bfae0c
+Merge: cc867e1 173ea7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 13:15:09 2017 +0100
+
+    Merge pull request #13025 from SUSE/wip-18605-jewel
+    
+    jewel: ceph-disk prepare writes osd log 0 with root owner
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cc867e1ed2b631c6dd380a575c4475973b5a2a4d
+Merge: 864f915 3a02868
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:12:57 2017 +0100
+
+    Merge pull request #13106 from ceph/wip-cd-dev-option
+    
+    qa/tasks: jewel backport -  ceph-deploy use of dev option
+    
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 864f9159a97b671f4332b917b72a29600c79aa4d
+Merge: 8e69580 ff91688
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:02:54 2017 +0100
+
+    Merge pull request #13062 from asheplyakov/jewel-bp-18379
+    
+    jewel: msg/simple: clear_pipe when wait() is mopping up pipes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8e69580c97622abfcbda73f92d9b6b6780be031f
+Merge: c05730c 509de4d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:53:06 2017 +0100
+
+    Merge pull request #12978 from asheplyakov/jewel-18581
+    
+    jewel: ReplicatedBackend: take read locks for clone sources during recovery
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c05730ceb3387fb43c35937f0506297a34a44452
+Merge: 36005e9 fcdd5e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:33:19 2017 +0100
+
+    Merge pull request #12755 from Abhishekvrshny/wip-18284-jewel
+    
+    jewel: Need CLI ability to add, edit and remove omap values with binary keys
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 36005e9efa4a5b75efecfd0eec6effc17639d7db
+Merge: 615549a dcc9483
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:31:13 2017 +0100
+
+    Merge pull request #12677 from SUSE/wip-18104-jewel
+    
+    jewel: mon: ceph osd down detection behaviour
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 615549abbf7eed157289dde4603e3c0b25a40d2e
+Merge: 90c9ad7 8a774cc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:28:50 2017 +0100
+
+    Merge pull request #12291 from asheplyakov/jewel-18108
+    
+    jewel: msg/simple/Pipe: error decoding addr
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 90c9ad7fe60b89a87f53b2b224100befc334dbe1
+Merge: ef7d826 a9da605
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:27:46 2017 +0100
+
+    Merge pull request #11998 from Abhishekvrshny/wip-17877-jewel
+    
+    jewel: FileStore: fiemap cannot be totally retrieved in xfs when the number of extents > 1364
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ef7d82679e91d1e2364e3fb433542940511598ee
+Merge: d1a019a fff2127
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:26:28 2017 +0100
+
+    Merge pull request #11991 from asheplyakov/jewel-17909
+    
+    jewel: osd: limit omap data in push op
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d1a019ad3b0a35bd8f339f0ceefd5f8617ee18fa
+Merge: 0b33f2c b044361
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 11:40:41 2017 -0500
+
+    Merge pull request #12045 from Abhishekvrshny/wip-17875-jewel
+    
+    jewel: rgw: file: remove spurious mount entries for RGW buckets
+
+commit 0b33f2caf7bf2762ac66779fba0032868e9488e5
+Merge: 92827ae 75d0580
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jan 31 16:42:14 2017 +0100
+
+    Merge pull request #13161 from smithfarm/wip-lfn-upgrade-hammer
+    
+    jewel: tests: add require_jewel_osds before upgrading last hammer node
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 92827aeb4ea557f204f850a86d69eeed74812c93
+Merge: 790f2a1 077290b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 16:36:17 2017 +0100
+
+    Merge pull request #12425 from SUSE/wip-18190-jewel
+    
+    jewel: rbd-mirror: gmock warnings in bootstrap request unit tests
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 790f2a11860378785bab7fb0195a455ec1d70857
+Merge: 32890c1 649b1d4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:16:45 2017 +0100
+
+    Merge pull request #13129 from smithfarm/wip-18558-jewel
+    
+    jewel: rbd: bench-write will crash if --io-size is 4G
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 32890c153dcd8abb3e1ea942f16718582bd0c836
+Merge: c0e3ae3 dd1f425
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:15:30 2017 +0100
+
+    Merge pull request #13155 from smithfarm/wip-18494-jewel
+    
+    jewel: rbd: [rbd-mirror] sporadic image replayer shut down failure
+
+commit c0e3ae3332a51dab3a17f1546b6e8258eaea09eb
+Merge: 7849ea5 c2f86a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:14:31 2017 +0100
+
+    Merge pull request #13128 from smithfarm/wip-18633-jewel
+    
+    jewel: rbd: [qa] crash in journal-enabled fsx run
+    
+    http://tracker.ceph.com/issues/13512
+
+commit 7849ea5166030e7f0b331ef21d0867b235fedb2d
+Merge: 833c58e 61e1b0c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:11:43 2017 +0100
+
+    Merge pull request #13104 from idryomov/wip-fio-unmap-devices-jewel
+    
+    qa/tasks: backport rbd_fio fixes to jewel
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 833c58edc0445fc54364cc5ae7a58e1c17a96422
+Merge: 26798a1 16a2fec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:10:03 2017 +0100
+
+    Merge pull request #13156 from smithfarm/wip-18455-jewel
+    
+    jewel: rbd: Attempting to remove an image w/ incompatible features results in partial removal
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 26798a1850c2ee6de2e6ec2fd9b9597e447f63dd
+Merge: dff349c 99bafc1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:59:56 2017 +0100
+
+    Merge pull request #13157 from smithfarm/wip-18434-jewel
+    
+    jewel: rbd: Improve error reporting from rbd feature enable/disable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dff349c8249388e5b31314dc6297446bf9f0f9e3
+Merge: f9b56db 1555638
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:59:04 2017 +0100
+
+    Merge pull request #13168 from trociny/wip-18550-jewel
+    
+    jewel: librbd: metadata_set API operation should not change global config setting
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f9b56dbdb28cd02ea665f0dfde630f8419a1067c
+Merge: 586f4ea 9c84a65
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:55:58 2017 +0100
+
+    Merge pull request #12739 from SUSE/wip-18323-jewel
+    
+    jewel: JournalMetadata flooding with errors when being blacklisted
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 586f4ead083518e95115d31dbecfbf4d5995e7ed
+Merge: 9af8b21 ad869de
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:54:19 2017 +0100
+
+    Merge pull request #12416 from SUSE/wip-18219-jewel
+    
+    jewel: msg: don't truncate message sequence to 32-bits
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9af8b2191268d1b2b4e4d4ba234ba12d265e3731
+Merge: 1a9eb84 336c351
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:52:00 2017 +0100
+
+    Merge pull request #13115 from smithfarm/wip-18404-jewel
+    
+    jewel: mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1a9eb8454368160be35e966b5264c296db87ac0a
+Merge: b03280d 35e10a0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:50:52 2017 +0100
+
+    Merge pull request #13045 from SUSE/wip-18553-jewel
+    
+    jewel: mon: peon wrongly delete routed pg stats op before receive pg stats ack
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b03280d67c2464599289da659152a6877da65827
+Merge: e524035 7bbb5a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:49:56 2017 +0100
+
+    Merge pull request #13117 from smithfarm/wip-18280-jewel
+    
+    jewel: mon: osd flag health message is misleading
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e5240357a6901a7e37440f01883ecac56fbc771f
+Merge: c4f8684 820ab7d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 08:34:22 2017 -0500
+
+    Merge pull request #12320 from dachary/wip-18061-jewel
+    
+    jewel: rgw:fix for deleting objects name beginning and ending with underscores of one bucket using POST method of js sdk.
+
+commit c4f868406274881a8b5bbe97bc19e32146efe861
+Merge: e47969e 13fa5db
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 08:31:34 2017 -0500
+
+    Merge pull request #11759 from Abhishekvrshny/wip-17783-jewel
+    
+    jewel: rgw: json encode/decode of RGWBucketInfo missing index_type field
+
+commit e47969e5d5cdf51149c23da4eec04dd0a2faec6f
+Merge: b278ece edac06f
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:10:46 2017 +0100
+
+    Merge pull request #12153 from dachary/wip-18010-jewel
+    
+    jewel: Cleanly reject session evict command when in replay
+
+commit b278ece85be7ed4f57dec6f524f187d1d10f4616
+Merge: 2d2e4af e1af490
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:10:23 2017 +0100
+
+    Merge pull request #12324 from dachary/wip-18103-jewel
+    
+    jewel: truncate can cause unflushed snapshot data lose
+
+commit 2d2e4afa3eef29ee82ec8057d6172f5706f4b872
+Merge: 916e95d 7819adb
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:09:59 2017 +0100
+
+    Merge pull request #12783 from SUSE/wip-18413-jewel
+    
+    jewel: cephfs: lookup of /.. in jewel returns -ENOENT
+
+commit 916e95d13692e9a025e47b11724aa65d51860411
+Merge: 384a425 8404426
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:56 2017 +0100
+
+    Merge pull request #12921 from xiaoxichen/wip-18520-jewel
+    
+    Jewel: speed up readdir by skipping unwanted dn
+
+commit 384a425cf69b7554e460d528e7f788c3b0c2eff1
+Merge: 4fd62b6 b147022
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:30 2017 +0100
+
+    Merge pull request #13119 from smithfarm/wip-18565-jewel
+    
+    jewel: MDS crashes on missing metadata object
+
+commit 4fd62b65c2097052378043ee82c937bb702263c1
+Merge: 01a6b1f bc9b779
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:20 2017 +0100
+
+    Merge pull request #13120 from smithfarm/wip-18551-jewel
+    
+    jewel: ceph-fuse crash during snapshot tests
+
+commit 01a6b1f31eafc66cea06702984a77391562e0ad1
+Merge: 831c1d8 fef3de8
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:01 2017 +0100
+
+    Merge pull request #13123 from smithfarm/wip-18282-jewel
+    
+    jewel: monitor cannot start because of FAILED assert(info.state == MDSMap::STATE_STANDBY)
+
+commit 831c1d8c8fc3e8777559da8a154aff104f1c831d
+Merge: 7fbe164 df4558c
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:07:52 2017 +0100
+
+    Merge pull request #13125 from smithfarm/wip-18195-jewel
+    
+    jewel: cephfs: fix missing ll_get for ll_walk
+
+commit 7fbe164962d488a6b2c2b921619794fea254b848
+Merge: 3793798 5eda4aa
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:07:41 2017 +0100
+
+    Merge pull request #13126 from smithfarm/wip-18192-jewel
+    
+    jewel: standby-replay daemons can sometimes miss events
+
+commit 75d05809a66bee219031a7ccb64d414a2d6c8775
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 22:27:18 2017 +0100
+
+    tests: add require_jewel_osds before upgrading last hammer node
+    
+    Note: this commit was inspired by
+    http://github.com/ceph/ceph-qa-suite/commit/50758a4810794d265c5d36a71d1e16799251a00d
+    
+    As of 10.2.4, when upgrading a cluster from hammer to jewel, after the last
+    node is upgraded the MON will put the cluster into HEALTH_WARN and say: "all
+    OSDs are running jewel or later but the 'require_jewel_osds' osdmap flag is not
+    set". The release notes say:
+    
+        This is a signal for the admin to do "ceph osd set require_jewel_osds" – by
+        doing this, the upgrade path is complete and no more pre-Jewel OSDs may be
+        added to the cluster.
+    
+    Fixes: http://tracker.ceph.com/issues/18719
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3793798049ba74f8f3a5742c50398c8e89bbac0b
+Merge: a1a209e 23680e0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 11:19:39 2017 +0100
+
+    Merge pull request #13049 from SUSE/wip-18433-jewel
+    
+    jewel: rados bench seq must verify the hostname
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a1a209ed1b33fbb78a4335f69f1f2d5d9085bda4
+Merge: 25bd0be c2bbf7f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:40:52 2017 +0100
+
+    Merge pull request #13040 from SUSE/wip-fs-thrash-jewel
+    
+    jewel: tests: run fs/thrash on xfs instead of btrfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 25bd0befb0749bb2759059559405f5bafe933d20
+Merge: bdb5e36 7a341a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:39:35 2017 +0100
+
+    Merge pull request #13029 from SUSE/wip-18611-jewel
+    
+    jewel: cephfs: client segfault on ceph_rmdir path /
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit bdb5e362dfec1debf1de5db6c14b23a63437a0d6
+Merge: 1bc16a4 f24c3ff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:34:46 2017 +0100
+
+    Merge pull request #12875 from asheplyakov/jewel-bp-18485
+    
+    jewel: osd/PG: publish PG stats when backfill-related states change
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1bc16a4333e27c9c3b603af6c52722183ce10a51
+Merge: 1722b46 2296c87
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:32:58 2017 +0100
+
+    Merge pull request #12789 from SUSE/wip-18417-jewel
+    
+    jewel: osd: leveldb corruption leads to Operation not permitted not handled and assert
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1722b4684808eb968b0997b8584477a51cd8940a
+Merge: 345bcdc bbf4c27
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:28:22 2017 +0100
+
+    Merge pull request #12761 from SUSE/wip-18402-jewel
+    
+    jewel: tests: objecter_requests workunit fails on wip branches
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 345bcdc0a41ac362ae813a64756fcd3f1d2ff2f4
+Merge: 85ae998 ef2709a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:10:20 2017 +0100
+
+    Merge pull request #11947 from SUSE/wip-17884-jewel
+    
+    jewel: mon: OSDs marked OUT wrongly after monitor failover
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 85ae998886c5f8b429b20a8a7d80b2ed4ce57e37
+Merge: 0ddbaf3 4f8287f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:07:40 2017 +0100
+
+    Merge pull request #11508 from SUSE/wip-17583-jewel
+    
+    jewel: utime.h: fix timezone issue in round_to_* funcs.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3a02868be40f3431d2bfedf16737b37ebf1a7e89
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jan 26 13:21:30 2017 -0800
+
+    Revert "use the create option during instantiation"
+    
+    jewel cephfs still uses old Filesystem initializtion method
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit ebdc0d2ce82ba67aecfaa25e66270c9529238700
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 15 14:11:00 2016 -0800
+
+    use dev option instead of dev-commit
+    
+    Fixes: http://tracker.ceph.com/issues/18736
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 0ddbaf384a76e87f218af1babb99f4a08da5abce
+Merge: 4ef0a63 016b059
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 30 13:25:23 2017 -0500
+
+    Merge pull request #13169 from linuxbox2/jewel-mg-errortrans
+    
+    jewel:rgw: fix interface compliance of RGWCivetWeb::write_data()
+
+commit 3839727f2969337a6d70e948db3de034a1346e90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 30 11:38:21 2017 -0500
+
+    librbd: ensure owner lock is held before purging cache
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d0c12edc52d45a0e429e0d4cca78f8724e39e926
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 11:17:10 2017 -0500
+
+    librados: blacklist_add should wait for latest OSD map
+    
+    This ensures that future operations against the OSDs force
+    a OSD map update to notice the blacklisted client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9242a2e4e1a5a9fcea48d8135b1589493fc28242)
+
+commit bf8b78c3a6fb8a6b516793493c4169ceee15d9f8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 13:31:57 2017 -0500
+
+    librbd: prevent assertion failure when journal IO is blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18429
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c720f6e3704ed7e8cf41dffdb931dbb05d59a003)
+
+commit 2ca703073fd7563c06a310b66191fb4a99d4fa63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 4 12:12:27 2017 -0500
+
+    librbd: ignore blacklist error when releasing exclusive lock
+    
+    This ensures the journal and object map are properly closed so that the
+    image can be properly closed w/o failing any assertions.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 418dcf29cb8c450049047e09a4dad2941af87018)
+
+commit 7aa424ab450afb6ff308272cedc854a33420d081
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 15:29:17 2017 -0500
+
+    librbd: fail immediately if the exclusive lock cannot be acquired
+    
+    Fixes: http://tracker.ceph.com/issues/16988
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47b89f4d48a0b99876462167389df28893a8d418)
+
+commit 5d96332700cbb05f541c211c375ca97506b41aad
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 14:51:14 2017 -0500
+
+    librbd: add new lock_get_owners / lock_break_lock API methods
+    
+    If the client application supports failover, let the application
+    force break the current lock and blacklist the owner. This is
+    required in case the current lock owner is alive from the point-of-view
+    of librbd but failover was required due to a higher level reason.
+    
+    Fixes: http://tracker.ceph.com/issues/18327
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a5a8c75a025143cee6f92f3dbc3a12f2b6a9ad7)
+    
+    Conflicts:
+    	src/pybind/rbd/rbd.pyx: trivial resolution
+    	src/test/pybind/test_rbd.py: trivial resolution
+
+commit 245898aa9ae3d6fe03668f3a88b2fac84d11cb29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 22 15:00:23 2016 -0500
+
+    librbd: separate break lock logic into standalone state machine
+    
+    The current lockers are now queried before the lock is attempted to
+    prevent any possible race conditions when one or more clients attempt
+    to break the lock of a dead client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 23f60fee86d1ff9b261fbb6411746a2a9479cf19)
+
+commit 968a10b2adc56a45f292ac9faf9b671f2f66b996
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 22 17:24:47 2016 -0500
+
+    librbd: separate locker query into standalone state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03533b912c59d5e433d0f006e1a063e014468ca5)
+    
+    Conflicts:
+    	src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc: trivial resolution
+
+commit 652e65a2143f9fa2a69822fe9924e59a83ba5c21
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Feb 22 15:17:20 2016 +0100
+
+    librbd/exclusive_lock/AcquireRequest.cc: init lock_type
+    
+    Fixup for:
+    
+    CID 1351687 (#1 of 1): Uninitialized scalar variable (UNINIT)
+     var_decl: Declaring variable lock_type without initializer
+     uninit_use: Using uninitialized value lock_type.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit da9ede90cd257ff605ab3ebfcc1aa995f655f1cc)
+
+commit d4085d34f31fa9eed73d69f241184b99e5198b22
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 8 15:11:02 2016 +0300
+
+    librbd: API methods to directly acquire and release the exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/15632
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8f1b47fd5da021ec320fd0b5fc0fd68ffff8a706)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/include/rbd/librbd.h: trivial resolution
+    	src/librbd/CMakeLists.txt: trivial resolution
+    	src/librbd/Makefile.am: trivial resolution
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 476e2b14364daa4363425b392199e9cc23c2f02b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Jun 10 14:04:04 2016 +0300
+
+    rbd-mirror: fix error messages formatting
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2af72480d94b2b90ed6eac7b3e131437864eada7)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc: trivial resolution
+
+commit 374d89f3706aea019f053350c9d790cd8c2c1423
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Sep 11 09:08:41 2016 -0400
+
+    librbd: ignore partial refresh error when acquiring exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/17227
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24396dcba77a97342d19916fdd285bae0c38fd19)
+
+commit 0bd843aa7bcf212bf370800a4c0c3176af5d6e9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 9 08:31:52 2016 -0400
+
+    librbd: potential seg fault when blacklisting an image client
+    
+    Fixes: http://tracker.ceph.com/issues/17251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7025fe8976f8672a2fce05ea693c06a8e9faed19)
+
+commit 273fd99085f13fb5f76e9e60a81012c22c49418d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Sep 4 10:48:48 2016 -0400
+
+    librbd: potential double-unwatch of watch handle upon error
+    
+    Fixes: http://tracker.ceph.com/issues/17210
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1068ded0cba59831a0712f347946731689a68553)
+
+commit 33e037a089337b5c96a03befac44549e680db13d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 31 21:33:54 2016 -0400
+
+    librbd: deadlock when replaying journal during image open
+    
+    Fixes: http://tracker.ceph.com/issues/17188
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3dc13067f5f0d140ee76b0166eb4cec568610211)
+
+commit bca65c46fdac4829132308f7fcf2ac59e2aaea6c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 31 20:56:54 2016 -0400
+
+    librbd: improve image state machine debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c71182a73146c65dfe7bf955ad67ebeebcf7b1fd)
+
+commit a12f435041860f3329e2465586805a168dcc87c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 15:16:37 2016 -0400
+
+    librbd: remove unused refresh request logic
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5156b438d5ad69803b4d4529083039db825d6c8c)
+
+commit a475bfb4d2c8906cfad52a7d81196a87c4c2f0bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 14:58:22 2016 -0400
+
+    librbd: interlock image refresh and lock operations
+    
+    Fixes: http://tracker.ceph.com/issues/16773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 583ac91872859e81d68c9d346516522c6aa1614c)
+
+commit 3d61b69b11ef67719e78d77fec000403d43d70d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 14:02:32 2016 -0400
+
+    librbd: image state machine now has hooks for lock requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a5b8c9c1d81ed609e71a5a8abe4d0e90194408cc)
+
+commit 4ea8d55c95304e07a90a6c4b8890787ccaeecedf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 16:23:57 2016 -0400
+
+    librbd: integrate asynchronous image rewatch state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d0d97fcca2cb427adbdf5c32f0e07830c38aaf4b)
+
+commit 62f265b30531141dfda8a7490d18b1d0d787fe13
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 15:46:23 2016 -0400
+
+    librbd: helper state machine for asynchronous watch recovery
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 32180aaf42050a01981c33f84edd95eff931ee6c)
+    
+    Conflicts:
+    	src/librbd/CMakeLists.txt: trivial resolution
+    	src/librbd/Makefile.am: trivial resolution
+
+commit ead7201874ed8617d86dc548798aa4bde7168fdb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 14:20:20 2016 -0400
+
+    librbd: exclusive lock now supports reacquiring a lost lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 66c605573f840c0db8b3630315ea50e9fc987509)
+
+commit 38ca4ff1d8d21db17f382a6821d098e472be4657
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 13:11:19 2016 -0400
+
+    librbd: store exclusive lock cookie instead of recalculating
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aa53f74ad261f453e971bf3cef0b96bba4932b7b)
+
+commit 73a445572e8fab135c7d87b80e374355e77d2e28
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 12:28:09 2016 -0400
+
+    librbd: helper state machine to update lock cookie
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d523df8dafac472f95233805d3a82edb3b3b02ea)
+
+commit 98a5e116c28ec9096ce68308be06e794883ce29e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 16:50:43 2016 -0400
+
+    cls_lock: support updating the lock cookie without releasing the lock
+    
+    Fixes: http://tracker.ceph.com/issues/17015
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 377f57652f8ddae53f44c59f21e89c51b2bf0f7b)
+
+commit 8a75f980d3cb00cc31c479f5e4253a6385563663
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 30 14:19:05 2017 +0100
+
+    tests: make distros/all/centos.yaml be a symlink to centos_7.3
+    
+    Before this commit, tests that used distros/all/centos.yaml would
+    fail on VPS because VPS defaults to centos 7.0 when os_version is not
+    specified.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4ef0a63e7a29eafad6dda52ea3c92846f03f268d
+Merge: efad365 900f2ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:26:37 2017 +0100
+
+    Merge pull request #12323 from dachary/wip-18024-jewel
+    
+    jewel: rbd: FAILED assert(m_processing == 0) while running test_lock_fence.sh
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit efad365753b7847ecce1280fe267d97d2ccf2888
+Merge: a1e7c06 64c0cae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:25:48 2017 +0100
+
+    Merge pull request #12649 from xiaoxichen/wip-18278-jewel
+    
+    jewel: rbd: librbd: use proper snapshot when computing diff parent overlap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a1e7c06dcb1b47eceea273e9b6ae5044c5958a07
+Merge: ed50d14 8877ee4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:24:22 2017 +0100
+
+    Merge pull request #12741 from SUSE/wip-18320-jewel
+    
+    jewel: rbd status: json format has duplicated/overwritten key
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ed50d14b4fa4acd13b4197e881666b2120525601
+Merge: b7481a8 d313e42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:23:44 2017 +0100
+
+    Merge pull request #12753 from Abhishekvrshny/wip-18288-jewel
+    
+    jewel: rbd-mirror: image sync object map reload logs message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b7481a85453ed059b7ab40932e57ac5354b7581f
+Merge: 2ebb894 944ec03
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:23:09 2017 +0100
+
+    Merge pull request #12756 from Abhishekvrshny/wip-18276-jewel
+    
+    jewel: rbd-nbd: invalid error code for "failed to read nbd request" messages
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2ebb8948858a33c077f6a1481a1ce3e3f23d5d74
+Merge: 60a2037 b359935
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:22:37 2017 +0100
+
+    Merge pull request #12822 from SUSE/wip-18450-jewel
+    
+    jewel: tests: update rbd/singleton/all/formatted-output.yaml to support ceph-ci
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 60a2037ef52a6fc75171260067cee9c9bcc66bd9
+Merge: 60bc353 cdd6cbf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:21:25 2017 +0100
+
+    Merge pull request #12909 from dillaman/wip-18290-jewel
+    
+    jewel: librbd: properly order concurrent updates to the object map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 60bc35357cdbe39e8c85ebd217526c1b9ad4fc76
+Merge: 18cb72c aa8e57d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:20:07 2017 +0100
+
+    Merge pull request #12529 from SUSE/wip-18270-jewel
+    
+    jewel: rbd: add image id block name prefix APIs
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18cb72c129d6c90afa489fd1cda6713fe8a9e1ed
+Merge: 20a480d 4a157ea
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:18:32 2017 +0100
+
+    Merge pull request #12322 from dachary/wip-18110-jewel
+    
+    jewel: diff calculate can hide parent extents when examining first snapshot in clone
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18545a27eccfa0b22b1782bb52e3f47afef8ec39
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 3 20:40:00 2017 +0800
+
+    ceph-disk: convert none str to str before printing it
+    
+    Error('somethings goes wrong', e) is thrown if exception `e` is caught
+    in ceph-disk, where e is not a string. so we can not just concat it in
+    Error's __str__(). so cast it to str before doing so.
+    
+    introduced by d0e29c7
+    
+    Fixes: http://tracker.ceph.com/issues/18371
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e0dd1e7df43a3be589d17878714756a22052d8e)
+
+commit 077290b873e2cd991a995de14a116d85d83ba66e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Dec 6 23:19:54 2016 +0200
+
+    rbd-mirror: fix gmock warnings in bootstrap request unit tests
+    
+    Fixes: http://tracker.ceph.com/issues/18156
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 376026d7b24c77a59ef95d0f66686494caf0e9d0)
+
+commit ce32297c880779532ba3482edb93fc4e4340d94b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 2 10:10:52 2016 +0200
+
+    qa/workunits/rbd: test_status_in_pool_dir: explicitly check grep return value
+    
+    Otherwise, it does not work as supposed to work in statements like below:
+    
+     set -e
+     test_status_in_pool_dir ... && ...
+    
+    (e.g. in wait_for_status_in_pool_dir)
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 7c078eda0fc260f6a885fa43f377b47844867d12)
+
+commit 6d729d231401f9253aa1cbde06e57cd8bd066a90
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Nov 29 11:44:05 2016 +0200
+
+    rbd-mirror: make 'rbd mirror image resync' work after split-brain
+    
+    Fixes: http://tracker.ceph.com/issues/18051
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 6cb1ed485f89f30fe6dda31e77e16e23f9b5b2ab)
+
+commit e34a403df5e247b0937abee617758e0838fc0c8e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Nov 29 11:40:14 2016 +0200
+
+    rbd-mirror: split-brain issues should be clearly visible in mirror status
+    
+    Fixed: http://tracker.ceph.com/issues/16991
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cccca67d5f898c43207a19a6e029a1abb86efbcb)
+
+commit 8f9a93c68947c05636a633c92dbbd69d2ed70c3d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Nov 27 16:27:51 2016 +0200
+
+    qa/workunits/rbd: use image id when probing for image presence
+    
+    This fixes a race in resync tests leading to false negative results.
+    
+    Fixes: http://tracker.ceph.com/issues/18048
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c23f56b9835f4e59d72f20a2218de8236574e65f)
+
+commit 1f2d30cbb735d2c7fd3eea6431f1a1e9555d5942
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Oct 3 10:10:33 2016 +0300
+
+    qa/workunits/rbd: check status also in pool dir after asok commands
+    
+    wait_for_image_replay_stopped returns not when the state is stopped,
+    but when the state is not replaying. So a race was possible when an
+    asok command was running when the previos stop command was still in
+    progress, leading to unexpected results.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2882f8c01a42ee3cfe22a838b64f21c93cf84d16)
+    
+    Conflicts:
+    	qa/workunits/rbd/rbd_mirror.sh (tests for not backported features)
+
+commit 5d0fba44501130165416a57a10490ff8ba292d48
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Aug 2 20:06:17 2016 +0300
+
+    qa/workunits/rbd: wait for image deleted before checking health
+    
+    This is a fixup to the previous commit.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a5f63f726146c2c230d172f6909d8ca1ad46895a)
+
+commit 38e06fbfcd8ba3a529af0a902c1af6168cf7b0f6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jul 28 14:45:56 2016 +0300
+
+    qa/workunits/rbd: wait for image deleted before checking health
+    
+    When a primiry image is being deleted, the mirrored image might
+    temporary be reported in error state, before deletion is propagated.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8dbe42d34520d0b44b189ae4d8b96559752a57da)
+
+commit 735e32bb53fd370058ed8dfb0e29e4a9cfc6e86f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jul 28 14:49:37 2016 +0300
+
+    qa/workunits/rbd: small fixup and improvements for rbd-mirror tests
+    
+    - log to stderr;
+    - log status if a `wait_for` function failed;
+    - don't needlessly sleep in `wait_for` functions after the last
+      unsuccessful iteration;
+    - make `wait_for_pool_images` work for image removal case;
+    - fix `wait_for_pool_images` reset timeout (last_count set).
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fdb971a2660111d35f0a3077f17a9ca85ca2ef54)
+
+commit bf3400f7f1dbc14917f3392c8f9221b6b02df185
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+    
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+    
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+    
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+    
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+    
+    Fixes: #18635
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit 20a480defb607e6e5a72f2bcc3868fb14f9e56de
+Merge: 6b4bcd3 bcd4698
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Jan 29 13:22:49 2017 +0100
+
+    Merge pull request #13043 from SUSE/wip-18570-jewel
+    
+    jewel: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 401271e353fcfd59bf0285c53643b667281954d0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jan 29 13:12:43 2017 +0100
+
+    build/ops: add libldap dependency for RGW
+    
+    This is a minimal manual backport of the relevant parts of
+    a4c7e13d17ceff3ee15fc311c2a344cd4573821d and
+    b3b3185008a0a2149dcba59813e0f0400d2e47de
+    
+    Fixes: http://tracker.ceph.com/issues/17313
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 73d2114ed60e8297d26b7f31f0500034b3d784e7
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:56:44 2017 +0200
+
+    radosgw-admin: check for name mistmatch in realm set
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 4facc5f4f3e842d371115a9a04d86257280014f0)
+
+commit e4b6cf633a88dc7fcd48f9a4a54d10fea7bef7e2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:40:37 2017 +0200
+
+    radosgw-admin: relam set can use input redirection
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit b8b3ae3be3f8e4c05cb23062d25c701b15900475)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc ("return EINVAL" in master, "return -EINVAL" in
+                  jewel; but it's irrelevant because the whole block is deleted)
+
+commit 3cd42f4b75c9ea9d253c276e61789d5a2b2395b4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:36:04 2017 +0200
+
+    radosgw-admin: realm set should create a new realm
+    
+    Fixes: http://tracker.ceph.com/issues/18333
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23339c9ef34f6b9df90b1ab64b550af9b541d9e)
+
+commit 33c6ef53413ef83f84be3bc3825125030aa2281b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 11 09:32:59 2017 -0500
+
+    rgw: fix off-by-one in RGWDataChangesLog::get_info
+    
+    Fixes: http://tracker.ceph.com/issues/18488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0974fdcf62e60cf31bc15588e7b718da6f6ade3)
+
+commit 8b124c84749e0fdd6663aa8302e0bd93a52626e7
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jan 19 18:14:30 2017 -0500
+
+    rgw_file: add timed namespace invalidation
+    
+    With change, librgw/rgw_file consumers can provide an invalidation
+    callback, which is used by the library to invalidate directories
+    whose contents should be forgotten.
+    
+    The existing RGWLib GC mechanism is being used to drive this.  New
+    configuration params have been added.  The main configurable is
+    rgw_nfs_namespace_expire_secs, the expire timeout.
+    
+    Updated post Yehuda review.
+    
+    Fixes: http://tracker.ceph.com/issues/18651
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit deb2c1ea985fcb906e47b93fd3d0117794e2d0a1)
+    
+    Conflicts:
+    	src/rgw/rgw_lib_frontend.h - in class RGWLibProcess : public RGWProcess
+                           there was no public method stop() in jewel (now there is)
+
+commit 2cb0307e7522a8dac9e6a400d987e22b9c09b56a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:32:55 2017 +0100
+
+    rgw_admin: read master log shards from master's current period
+    
+    Also make the sync output look similar to the output of data sync
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    
+    (cherry picked from commit cc306c506ca6607223cb89cd388f8e18673c4fe2)
+
+commit b2fedddd3daf544d935ba4f80a5feef35400f2e3
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:29:47 2017 +0100
+
+    rgw: allow getting master log shards info on specified period
+    
+    This is needed for rgw admin's sync status or else we end up always
+    publishing that we're behind since we are always checking against
+    master's first period to sync from
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 063c949d4409a18a22b64791d497e20f7473bc01)
+
+commit a0e08935b342ff3713ab9172a541e284fd1bb006
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jan 12 22:09:01 2017 +0100
+
+    rgw_admin: get master's period from store's current period info
+    
+    This ensures that we get the current period in contrast to the admin log
+    which gets the master's earliest period.
+    
+    Fixes: http://tracker.ceph.com/issues/18064
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4ca18df7198a9f0ded8b0100a70b5db7187c3de4)
+
+commit 0a4734261bd2f02f8b1acfae8ae65daf6b54d7d7
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Dec 12 14:00:05 2016 +0100
+
+    rgw: complete versioning enablement after sending it to meta master
+    
+    Fixes: http://tracker.ceph.com/issues/18003
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 2d8aafb9dbe64bd9dd2b7d5ed50c6e9550cbe1ab)
+
+commit 7e51bec0dd77bedc248a595e1ab63f87d7a30458
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Dec 25 12:36:34 2016 +0200
+
+    rgw: clear master_zonegroup when reseting RGWPeriodMap
+    
+    Fixes: http://tracker.ceph.com/issues/17239
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit d8f42fe6be659c1d48bf04b30aa54ad616936145)
+
+commit 509de4d9a2d3ba95e9f9e0375bd13239280b0e66
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jan 18 10:24:13 2017 -0800
+
+    PrimaryLogPG::try_lock_for_read: give up if missing
+    
+    The only users calc_*_subsets might try to read_lock an object which is
+    missing on the primary.  Returning false in those cases is perfectly
+    reasonable and avoids the problem.
+    
+    Fixes: http://tracker.ceph.com/issues/18583
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3833440adea6f8bcb0093603c3a9d16360ed57ec)
+
+commit cedaecf88efd3e4807e764d023bb956f806051da
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 23 15:41:13 2016 -0800
+
+    ReplicatedBackend: take read locks for clone sources during recovery
+    
+    Otherwise, we run the risk of a clone source which hasn't actually
+    come into existence yet being used if we grab a clone which *just*
+    got added the the ssc, but has not yet actually had time to be
+    created (can't rely on message ordering here since recovery messages
+    don't necessarily order with client IO!).
+    
+    Fixes: http://tracker.ceph.com/issues/17831
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8)
+    
+    Conflicts:
+    	src/osd/ReplicatedBackend.cc:
+    	  PGBackend::failed_push() takes a single pg_shard_t in jewel
+    	src/osd/ReplicatedPG.h:
+    	  trivial: get_obc() is not declared as override in jewel,
+    	           no pgb_{set,clear}_object_snap_mapping() in jewel
+
+commit 6b4bcd388829d6c5b78a7acb6d75d6f905a60f53
+Merge: c9ece04 7b74238
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Sat Jan 28 17:58:45 2017 -0800
+
+    Merge pull request #13166 from smithfarm/wip-drop-ext4-test
+    
+    tests: Remove ext4 option from rados:thrash tests
+
+commit 016b059fcf2993f721e97212d50b5e6da8180a03
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat Jan 28 14:17:10 2017 -0500
+
+    rgw: fix interface compliance of RGWCivetWeb::write_data()
+    
+    Backport of (portions of) civet web error handling bugfixes.
+    
+    Adapted from 3a9f50c55e0be6733893a7ae1a5b4f504a3b0f61.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 15556389c3c3bf9d6855aaa4699545a787d9fe7a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 18:31:21 2017 +0100
+
+    librbd: metadata_set API operation should not change global config setting
+    
+    Fixes: http://tracker.ceph.com/issues/18465
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27465b5916b55ac3c2846c74b89f4362ad17ff1e)
+    
+    Conflicts:
+        src/librbd/Operations.cc (after jewel set_metadata migrated to Operations)
+
+commit 7b7423837ea4f400507cf775f609c676a4cf3c8e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 28 12:11:08 2017 +0100
+
+    tests: Remove ext4 option from rados:thrash tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (manual cherry-pick from ceph-qa-suite 1fcc4457144278f77dd8462ecf34948a11dcb2a9)
+
+commit c9ece0444611994ddc1292a83e6c091f7773212f
+Merge: efcbcac 4e60be5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 27 12:57:32 2017 -0800
+
+    Merge pull request #13160 from ceph/revert-13068-jewel-rgw-multipart-upload-copy
+    
+    Revert "jewel: rgw: multipart upload copy"
+    
+    Reverting, will require further testing before merging. Some tests fail.
+
+commit 4e60be5867975b4ae5fe6f53aacd6abe128a18c3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 27 15:53:47 2017 -0500
+
+    Revert "jewel: rgw: multipart upload copy"
+
+commit efcbcac34e034cd3f3bc542fb71d527d64e66688
+Merge: a632cc4 514e2ba
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 27 13:04:46 2017 -0500
+
+    Merge pull request #13068 from linuxbox2/jewel-rgw-multipart-upload-copy
+    
+    DNM: jewel:rgw: multipart upload copy
+
+commit 99bafc1bab8eb8349e44b601f5971af97c131bd9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 18:41:25 2017 +0100
+
+    rbd: enabling/disabling rbd feature should report missing dependency
+    
+    Currently while enabling or disabling rbd feature command does not
+    give missing dependency for eg: attempting to enable the journaling
+    feature on an image that doesn't have the exclusive-lock feature
+    enabled should give missing dependency error message.
+    
+    Fixes: http://tracker.ceph.com/issues/16985
+    
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (manual cherry pick of bd023cfec8e9aaa8fb0095a8d9534c21b3209020)
+
+commit 16a2feca3523a4b0495769c2750dd1f5d70de25c
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Thu Dec 22 21:00:41 2016 -0500
+
+    librbd: don't remove an image w/ incompatible features
+    
+    Fixes: http://tracker.ceph.com/issues/18315
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit f76127b5e617923d14adb62bfb836a635c14f209)
+
+commit dd1f4252ed3a89407063cd283eebdbdc7cf5653c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 15:59:22 2017 -0500
+
+    rbd-mirror: avoid processing new events after stop requested
+    
+    Fixes: http://tracker.ceph.com/issues/18441
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d30873b269441815b5fc7de14c7d9a1077a17d8d)
+
+commit de1ebc36943208057e1aa427f9fda631d0bd717d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 16:59:44 2017 +0100
+
+    tests: explicitly use centos 7.3 in distros/supported
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0133d6316fc83373977c132cdbaa6f99d4f1140e
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Jan 3 15:59:18 2017 -0800
+
+    qa: fixed distros links
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit fbb560c90101937317380a6621a70564f10e0ae3)
+
+commit 1481c8fac899e395757ef81ae5b7baffe8e492ab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 15:29:27 2017 +0100
+
+    tests: upgrade: install firefly only on Ubuntu 14.04
+    
+    The upgrade:hammer-x/f-h-x-offline test installs firefly, but firefly does not
+    build on CentOS anymore, just Ubuntu 14.04.
+    
+    References: http://tracker.ceph.com/issues/18089
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 514e2ba64f2babdf6f58c57a7e8e1263abbf187c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:44:08 2016 -0700
+
+    rgw: minor optimization
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4919dc9987c6376d3d4e143702c26417449524c5)
+
+commit 7f76bb1229f40c4b535d97f163f26ee642798e2b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:00 2016 -0700
+
+    rgw: rgw_obj_key use adjustment in multipart copy part
+    
+    This fixes a case where objects start with double underscore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 29fece3545cc1df404a25eec46706b16f893a5df)
+
+commit f99ead1570a23a9050e396383006deb2e7e1d9f8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:35 2016 -0700
+
+    rgw: multipart copy-part handle versionId
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 53521efffb1cb92e5f5ce992d4127bf9498d7c33)
+
+commit 679348947910a92bf9bb7920c071142536272b19
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:24:13 2016 -0700
+
+    rgw: multipart copy part minor parsing cleanup
+    
+    no need for range.size()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9b824d5d4017239d58b4752ebc43bfad8f698d)
+
+commit 2ca1bcdacc37e74cb61c2ed9855150b4cc9537e9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:11:43 2016 -0700
+
+    rgw: multipart copy, check for empty bucket, improve logging
+    
+    also reduce log level for non critical user errors.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2bcb3d286b230ef917d5ba96c8276a942f544689)
+
+commit e5ac1204c26a26b29b146d350ff5034bfd9fc798
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:07:14 2016 -0700
+
+    rgw: multipart copy part, chunked read
+    
+    Don't read the entire range from source object, read it in parts.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4049e47a0cfc1eef6efd502590b68ba7234589d3)
+
+commit a54a7ada65fafc88251897b2739b364bf59be128
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:56:50 2016 +0200
+
+    rgw: doc: add multipart uploads copy part feature as supported
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 8a7ebeee2ff3f10ceb23b7fa43e43c3c450efe22)
+
+commit 987b42561e4c98c54963055f357955d243881238
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:53:44 2016 +0200
+
+    rgw: multipart uploads copy part support
+    
+    Add multipart uploads copy part feature.
+    
+    Fixes: http://tracker.ceph.com/issues/12790
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 949480c2e9760855ed6a0501d364d5e766c8207d)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 21622c13468d85f017d139a376097267943ea328
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 21 12:10:05 2016 +0000
+
+    src/mds: fix MDSMap upgrade decoding
+    
+    Hammer MDSMonitors did not validate the state in beacons
+    and would apply anything to the mds_info_t for a standby,
+    such as setting it to down:dne.  We must handle this
+    case during upgrade.
+    
+    Fixes: http://tracker.ceph.com/issues/17837
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a977c9029ff80b4eb4f3b128965693694b729425)
+
+commit be8bc1195270b87bdc0840b36684863d990404df
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jul 15 23:40:17 2016 -0400
+
+    mds: use FSMap::insert to add to standby_daemons
+    
+    This reduces the number of code sites which modify standby_daemons.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 9566ae27b33dd4d266ee9223dc73738922b6eceb)
+
+commit 649b1d4253a1753c3f4d783a8e2d7b48f861db08
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Tue Jan 10 15:25:13 2017 +0100
+
+    rbd: bench-write should return error if io-size >= 4G
+    
+    Currently if user perform bench-write with io-size > 4G
+    then its crashing because currently during memory allocation
+    bufferptr taking size of buffer as a unsigned and io-size > 4G
+    will overflow with unsigned. so during memset operation it will
+    try to set io_size size of memory area pointed by bufferptr,
+    (bufferptr area is:  (4G - io_size)), so it will cause
+    segmentation fault.
+    
+    Fix is to return error if io-size >= 4G
+    
+    Fixes: http://tracker.ceph.com/issues/18422
+    
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 6ab73e5f420e89e19b52e39dab28fa4c94e00197)
+    
+    Conflicts:
+    	src/tools/rbd/action/BenchWrite.cc - jewel has rbd_bencher b(&image)
+                while in master this has become rbd_bencher b(&image, io_type,
+                io_size)
+
+commit c2f86a461ea556cc5f9d93a7143cd80fb3e7dc96
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 20 14:26:43 2017 -0500
+
+    journal: don't hold future lock during assignment
+    
+    It's possible that the future raced with its owner and reaches
+    an empty reference count. This was resulting in the future being
+    destructed while its lock was still held.
+    
+    Fixes: http://tracker.ceph.com/issues/18618
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f21ceef8336e35ca16148a9d58f511037911418)
+
+commit 5eda4aac77200b858371ebab50d8779f70fe0dca
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 18 21:11:54 2016 +0000
+
+    mds: fix dropping events in standby replay
+    
+    Ensure that we never drop the last segment during
+    standby replay -- this avoids the case where we
+    start ignoring events because we think we're
+    still waiting to see a subtreemap.
+    
+    Fixes: http://tracker.ceph.com/issues/17954
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e3f2fa356f419cbac8d72bd068313f64769ef3af)
+
+commit df4558c9d5779ac8916811ea63d1106c2390638f
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Nov 17 18:01:22 2016 +0800
+
+    cephfs: fix missing ll_get for ll_walk
+    
+    Fixs: http://tracker.ceph.com/issues/18086
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ad846d10d5fbab2ded5fddc47475b95a788c223c)
+    
+    Conflicts:
+    	src/client/Client.cc - jewel has fill_stat(in, attr) instead of
+                                   fill_statx(in, mask, stx) in master
+
+commit fef3de88d0381322d49ce7842f777454b0b40b63
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 8 16:49:04 2016 +0000
+
+    mon/MDSMonitor: fix iterating over mutated map
+    
+    If a standby is promoted, this was modifying the
+    standby_daemons map that we were iterating over
+    in maybe_promote_standby.
+    
+    Fixes: http://tracker.ceph.com/issues/18166
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit db3deb80d6e4a8e8ca7a2cf6278d5d5cb24eb616)
+
+commit 05916271c17050fd4c1ad2351b6da17d3b89fc15
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 7 19:09:56 2016 -0400
+
+    mon: use clearer code structure
+    
+    The failed map is not changed in the first if, so the second if should be an
+    else.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit adba77e34c3fbf983d221729dd340afcc942778b)
+
+commit bc9b7792e4d44c199b526c2d5b17916d948b0354
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 17:16:40 2017 +0800
+
+    client: fix Client::handle_cap_flushsnap_ack() crash
+    
+    Struct CapSnap holds a reference to its parent inode. So erasing
+    struct CapSnap from Inode::cap_snaps may drop inode's last reference.
+    The inode gets freed in the middle of erasing struct CapSnap
+    
+    Fixes: http://tracker.ceph.com/issues/18460
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 525c52fd491ed1ced385c8047872e3f557f8423f)
+    
+    Conflicts:
+    	src/client/Client.cc (jewel does in->cap_snaps.erase(follows), master
+                                 does not; put it after the tmp_ref assignment)
+
+commit b14702209f5bad7d3aa65d3acba89b7757f725dd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 11 15:50:52 2017 +0800
+
+    qa/tasks: add test_open_ino_errors
+    
+    Validate that errors encountered during opening inos are properly
+    propagated
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6526ecc084733b34129aa1f21085fa41fb53b785)
+
+commit 3385419e8f41d0af986dc088e1c9ce83b569eb19
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 3 11:11:12 2017 +0800
+
+    mds: propagate error encountered during opening inode by number
+    
+    Fixes: http://tracker.ceph.com/issues/18179
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2213cc2dcc0e8fb01bcae3863d0d8a4a1fd8873f)
+
+commit 5c4fffa878c6ea881cc29b0ee4f99cde5e0e4ea8
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Dec 14 16:29:08 2016 +1000
+
+    librados: Memory leaks in object_list_begin and object_list_end
+    
+    We allocate a cursor in the constructor but simply reassign it in these
+    functions without cleaning up the original. We have a utility setter that
+    handles this exact case so we should use it.
+    
+    Fixes: http://tracker.ceph.com/issues/18252
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8d9d84bce923a009054ad2b223a97d7eb00e6774)
+
+commit 7bbb5a8918c7828d925f4796d2c25a583c8323e3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 09:50:26 2016 -0500
+
+    mon/OSDMonitor: only show interesting flags in health warning
+    
+    Also add PAUSEREC flag into the list.
+    
+    Fixes: http://tracker.ceph.com/issues/18175
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26220f0608f5ed4a7d97fb8d10d0d12a0fcf5583)
+
+commit 336c3515043f89494e452266069f778d7d8fe821
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 29 12:08:28 2016 -0500
+
+    mon/OSDMonitor: set last_force_op_resend on overlay pool too
+    
+    We currently set the last_force_op_resend field on the
+    base pool when we set or clear the overlay.  Set it on
+    the cache/overlay pool too.  The Objecter should resend
+    even with a change only to the base pool, but the OSD
+    needs to see the change on the overlay pool to correctly
+    discard the op.
+    
+    Fixes: http://tracker.ceph.com/issues/18366
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08c3618771b852840aa88cff1ca98d980d802941)
+
+commit 874220371890a59b8459bb6306a5db047e32563e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 11:16:47 2017 +0800
+
+    mds: finish clientreplay requests before requesting active state
+    
+    All clientreplay requests' finish contexts should be executed
+    before MDCache::export_remaining_imported_caps(). Otherwise
+    MDCache::try_reconnect_cap() may fail to reconnect client caps.
+    
+    Fixes: http://tracker.ceph.com/issues/18461
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 80dae314ee90e79d60e2cfee301e43a435c10801)
+
+commit b3599351225bd8f6299121e84b62a582f698199b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:48:51 2017 +0100
+
+    tests: subst repo and branch in qemu test urls
+    
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2484a5546160560a4050d35400a6edce37ae48ad)
+    
+    Conflicts:
+    	qa/suites/rbd/qemu/workloads/#qemu_xfstests.yaml#
+                (file does not exist in jewel)
+
+commit f66bd81d7082af3c1e28cfee19e46906a35835ee
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:45:10 2017 +0100
+
+    tests: subst branch and repo in qa/tasks/qemu.py
+    
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 74689df754561e11a3377998840efcea9e780755)
+
+commit 69a0efa7a689b5ec80d44119858cdcf86ca85588
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:09:23 2017 +0100
+
+    tests: subst repo name in krbd/unmap/tasks/unmap.yaml
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ed0e8be0b2c4d7a3e6e0716a0211d19e8b93f125)
+
+commit 2931aefc952ca0592eca26606864cbfa724b4c07
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:06:11 2017 +0100
+
+    tests: subst repo name in qa/tasks/cram.py
+    
+    Inspired by bcbe45d948f1c4da02e27b3be5f29a8b52745e28
+    
+    Fixes: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 56e37e41f4dddd289dd3c1886b192cd328ed311b)
+
+commit 205403b02156e4ed6b2b53a2b21189b5122162f8
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Fri Dec 9 15:11:49 2016 +0530
+
+    cram: support fetching from sha1 branch, tag, commit hash
+    
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit d2f0d745987a2e2eee4e1822146aad8da5d42708)
+    
+    Conflicts:
+    	suites/rbd/singleton/all/formatted-output.yaml (has been moved
+               under qa/)
+
+commit a632cc41755be4393a12dd768726ade86a49bbc9
+Merge: e55b84e ea65450
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 25 10:40:25 2017 -0500
+
+    Merge pull request #13103 from dillaman/wip-18672
+    
+    jewel: qa/workunits/rbd: use more recent qemu-iotests that support Xenial
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ea65450c49083f186781a3cbffe6cd24ed3c7a00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 5 13:46:02 2016 -0500
+
+    qa/workunits/rbd: use more recent qemu-iotests that support Xenial
+    
+    Fixes: http://tracker.ceph.com/issues/18149
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4314cb945a9c2296e2f7cd357b09015777f233c0)
+
+commit f449e3d1745ffd09e32058ea66342b486090d394
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 09:59:39 2016 -0500
+
+    qa/workunits/rbd: removed qemu-iotest case 077
+    
+    The test case is not stable due to racing console output. This
+    results in spurious failures.
+    
+    Fixes: http://tracker.ceph.com/issues/10773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c70df978d605a45ff81971b86f5afbefbdaabb6)
+
+commit 61e1b0caddfbc3721cd9fa2de5b90b2f65740f79
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 31 19:30:19 2016 +0200
+
+    tasks/rbd_fio: unmap rbd devices on cleanup
+    
+    Not doing so leads to issues and can interfere with subsequent jobs.
+    One example is the invocation of vgs(8) during the inital test setup:
+    it will issue a read to the left-behind rbd device(s) whose backing
+    cluster is long gone, locking up the job.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 15be2d29be3e42644b0c541f5c70461f1874f24f)
+
+commit 5fcfa32f21709054f054bcda719df090777f52d5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 31 19:05:25 2016 +0200
+
+    tasks/rbd_fio: don't use sudo unnecessarily
+    
+    Creating and cloning images doesn't require sudo.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ec97445740bbf7848488fbb6213e74d50a355547)
+
+commit e55b84e4cdb13839d8466cf162adb001d4134687
+Merge: ce3e387 ea9665b
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:57:17 2017 +0100
+
+    Merge pull request #12137 from jcsp/wip-17974
+    
+    jewel: client: fix stale entries in command table
+
+commit ce3e387766f2d6839798dfac96eacbf6334e4674
+Merge: 9dd1251 d0b0d41
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:56:24 2017 +0100
+
+    Merge pull request #12686 from SUSE/wip-18272-jewel
+    
+    jewel: tests: Workunits needlessly wget from git.ceph.com
+
+commit 9dd1251efdcd20ec9ce3467118b16b4d10a5360e
+Merge: a9c1fe0 85fbddd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:56:03 2017 +0100
+
+    Merge pull request #12836 from SUSE/wip-18462-jewel
+    
+    jewel: Decode errors on backtrace will crash MDS
+
+commit a9c1fe0f693020d5e01aad5f7c2264939591cbf4
+Merge: 0bcd904 bf873a7
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:46 2017 +0100
+
+    Merge pull request #13023 from SUSE/wip-18603-jewel
+    
+    jewel: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+commit 0bcd9049b469d4afcc547169ee1f9585557d3e0d
+Merge: 966bdbd 05e5a5a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:28 2017 +0100
+
+    Merge pull request #12155 from dachary/wip-17956-jewel
+    
+    jewel: Clients without pool-changing caps shouldn't be allowed to change pool_namespace
+
+commit 966bdbd7a1ef7fbe1023b6e7bde0f78c2eb4ab7a
+Merge: d276861 e725605
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:11 2017 +0100
+
+    Merge pull request #12325 from dachary/wip-18026-jewel
+    
+    jewel: ceph_volume_client.py : Error: Can't handle arrays of non-strings
+
+commit d276861915979e6a82662e7b4eba0b399c6ddca8
+Merge: dd703bc 50a3fa1
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:51 2017 +0100
+
+    Merge pull request #13060 from asheplyakov/jewel-bp-18615
+    
+    jewel: mds: fix null pointer dereference in Locker::handle_client_caps
+
+commit dd703bc028eeb9c58dc1ef39ae506c73642ea948
+Merge: 347f71d 36186d0
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:35 2017 +0100
+
+    Merge pull request #11656 from ajarr/wip-17705-jewel
+    
+    jewel: ceph_volume_client: fix recovery from partial auth update
+
+commit 347f71dc21c51d61ea42269236568756c7d0742a
+Merge: e90396f 6efad69
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:06 2017 +0100
+
+    Merge pull request #12154 from dachary/wip-18008-jewel
+    
+    jewel: Cannot create deep directories when caps contain path=/somepath
+
+commit e90396f16c495df0ac74acff841703b919e596b9
+Merge: 72b24f0 89dcd8b
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:53:45 2017 +0100
+
+    Merge pull request #13085 from jcsp/wip-18361-jewel
+    
+    jewel: client: populate metadata during mount
+
+commit 85fbddd4f68ab159425c9198740c0d5787adc739
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 24 15:49:24 2017 +0100
+
+    qa/tasks/cephfs/filesystem.py: backport _write_data_xattr() function
+    
+    This is a partial manual backport of 5f77f09b019b607b84e6a8f89ce19065383ca108
+    
+    It is needed by test_corrupt_backtrace() in qa/tasks/cephfs/test_damage.py
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 89dcd8b6287da7d97af0e7cad45ee023a5a39811
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Jan 13 00:30:28 2017 +0000
+
+    client: populate metadata during mount
+    
+    This way we avoid having to over-write the "root"
+    metadata during mount, and any user-set overrides (such
+    as bad values injected by tests) will survive.
+    
+    Because Client instances may also open sessions without
+    mounting to send commands, add a call into populate_metadata
+    from mds_command as well.
+    
+    Fixes: http://tracker.ceph.com/issues/18361
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1dbff09ad553f9ff07f4f4217ba7ece6c2cdc5d2)
+
+commit ff9168803ebb2e3e0c4b42268765cd4b53e50d68
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:18:08 2016 -0500
+
+    msg/simple: clear_pipe when wait() is mopping up pipes
+    
+    When wait is mopping up connections it may hit one that
+    is in the process of accepting.  It will unregister it
+    whilst the accept() thread is trying to set it up,
+    aborting the accept and getting it reaped.  However,
+    the pipe mop-up does not clear_pipe() the way that
+    mark_down(), mark_down_all(), and fault() do, which
+    leads to this assert.
+    
+    Pipe is accepting...
+    
+      -161> 2016-12-22 17:31:45.460613 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept:  setting up session_security.
+      -160> 2016-12-22 17:31:45.460733 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept new session
+      -159> 2016-12-22 17:31:45.460846 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept success, connect_seq = 1, sending READY
+      -158> 2016-12-22 17:31:45.460959 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept features 1152921504336314367
+    
+    wait() is shutting down...
+    
+      -156> 2016-12-22 17:31:45.461882 9506ac0 20 -- 172.21.15.14:6804/20738 wait: stopping accepter thread
+      -155> 2016-12-22 17:31:45.462994 9506ac0 10 accepter.stop accept listening on: 15
+    ...
+      -116> 2016-12-22 17:31:45.482137 9506ac0 10 -- 172.21.15.14:6804/20738 wait: closing pipes
+      -115> 2016-12-22 17:31:45.482850 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).unregister_pipe
+      -114> 2016-12-22 17:31:45.483421 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).stop
+    
+    ...which interrupts the accept()...
+    
+      -113> 2016-12-22 17:31:45.484164 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept fault after register
+    
+    and makes accept() return failure, and reader() to exit
+    and reap...
+    
+      -110> 2016-12-22 17:31:45.486103 9506ac0 10 -- 172.21.15.14:6804/20738 wait: waiting for pipes 0x3e2a5c20 to close
+      -109> 2016-12-22 17:31:45.487146 37353700 10 -- 172.21.15.14:6804/20738 queue_reap 0x3e2a5c20
+      -108> 2016-12-22 17:31:45.487658 9506ac0 10 -- 172.21.15.14:6804/20738 reaper
+      -107> 2016-12-22 17:31:45.487722 9506ac0 10 -- 172.21.15.14:6804/20738 reaper reaping pipe 0x3e2a5c20 172.21.15.35:0/146098963
+      -106> 2016-12-22 17:31:45.487816 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).discard_queue
+      -105> 2016-12-22 17:31:45.494742 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).reader done
+    ...
+       -92> 2016-12-22 17:31:45.527589 9506ac0 -1 /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: In function 'void SimpleMessenger::reaper()' thread 9506ac0 time 2016-12-22 17:31:45.488264
+    /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: 235: FAILED assert(!cleared)
+    
+    Fixes: http://tracker.ceph.com/issues/15784
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 948f97b3bdd39269a38277238a61f24e5fec6196)
+
+commit 50a3fa1ba1ab94ab736abf73830762afc0d05352
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jan 6 15:42:52 2017 +0800
+
+    mds: fix null pointer dereference in Locker::handle_client_caps
+    
+    Locker::handle_client_caps delays processing cap message if the
+    corresponding inode is freezing or frozen. When the message gets
+    processed, client can have already closed the session.
+    
+    Fixes: http://tracker.ceph.com/issues/18306
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e281a0b9c1fdeaf09f1b01f34cecd62e4f49d02e)
+
+commit 20e75023d6c9498ac19d3ee55e556063e94ea6e6
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+    
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+    
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+    
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+    
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+    
+    Fixes: #18635
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit aa0e45040f8b02c38a97670c41b8993742171cfb
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:19:42 2016 -0500
+
+    qa/distros: centos_7.yaml -> centos.yaml
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 004ef2c648c0c70a2efcd9b5f211369a2eb14f1d)
+
+commit b0c1e8863d84cbd04c74c54e6141a6a8355a3dee
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:41:25 2016 -0500
+
+    qa/suites: centos_7.2.yaml -> centos_7.yaml
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e9f119bda70554b79ca7f344571902c936c92f49)
+    
+    Conflicts:
+    	qa/suites/upgrade/jewel-x/point-to-point-x/distros (this directory used
+                to be a symlink to distros/supported)
+
+commit 8a98f06f25951a0113f2044ce2c062500eaea2e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:39:41 2016 -0500
+
+    qa/distros: add centos 7.3
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c168ce92aa1c1d834adecfe8fe32fe442ea7d21c)
+
+commit a45ad83c8f057da7054892d8ebc9767fc9238611
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:39:35 2016 -0500
+
+    qa/distros: add centos 7 yaml; use that instead
+    
+    No need to be picky about the point release here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6e7db2329b917cf179390bd87937098f153aa513)
+
+commit 23680e080758be00a950e15594235b88b1e55b01
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 6 19:39:20 2016 +0200
+
+    doc: document hostname constraints for rados bench
+    
+    Fixes: http://tracker.ceph.com/issues/17526
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1d7e8188b9067f4f51958a13a23da1e105d89c63)
+
+commit 67e7a904373ee0863ca21eaa7da6edd559bb463d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Sep 29 12:08:39 2016 +0200
+
+    selinux: Allow ceph to manage tmp files
+    
+    Two new denials showed up in testing that relate to ceph trying to
+    manage (rename and unlink) tmp files. This commit allows ceph to manage
+    the files.
+    
+    Fixes: http://tracker.ceph.com/issues/17436
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit f8a0e201ee54759695ef44f7ed98b3b9705fafe3)
+
+commit 35e10a010f58fe50d9f236ae57db46e96b0ffed3
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Jan 2 13:20:10 2017 +0800
+
+    mon: do not send duplicated osdmap msg to not sync'ed osd
+    
+    prior to this change:
+    a peon may forward the pgstats to leader, and record it locally, but leader will
+    check if osd has the latest map before process, if not, will use a route op to
+    indicate peon to send it, then poen will delete routed op when fininaly send
+    out which make peon cannot send pgstatack when leader has processed the
+    pgstat update. so osd will always track it util reach a threshold block pgstats
+    sending, at worst, reopen mon session.
+    also, both leader and peon will send out the osdmap message to the osd.
+    
+    after this change:
+    only the peon will send out the osdmap message. and the pgstatack message
+    will be routed to the osd as expected. so the osd will not keep track of the
+    "acked" pg stats in its queue forever before times out.
+    
+    Fixes: http://tracker.ceph.com/issues/18458
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit 57274488c072ec6912b700288ce5b1ea8372d162)
+
+commit bcd4698115011810e7a8370ef04a5c3f0b9a3fb0
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Fri Jan 13 16:57:45 2017 +0530
+
+    Doc: Fixes Python Swift client commands
+    
+    Fixes: http://tracker.ceph.com/issues/17746
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 8c79959557d60f619adf1a3ed1b5bd1112ceaabb)
+
+commit c2bbf7ff4f929a63e0c7d6f70bb66e906a6c6cab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:24:25 2017 +0100
+
+    tests: run fs/thrash on xfs instead of btrfs
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7a341a8886c93a8c4f266f8035784dd2e9528e3f
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Thu Jan 12 21:22:20 2017 +0000
+
+    client/Client.cc: prevent segfaulting
+    
+    The segfaulting in the rmdir function is caused by calling
+    filepath::last_dentry() function.
+    last_dentry() function assumes that the bits vector has always at
+    least one element, which is not the case for the the filepath object
+    created with "/" input.
+    This commit also fixes other functions affected by this bug:
+    link, unlink, rename, mkdir, mknod and symlink.
+    
+    Fixes: http://tracker.ceph.com/issues/9935
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 6ed7f2364ae5507bab14c60b582929aa7b0ba400)
+    
+    Conflicts:
+    	src/client/Client.cc (Client.cc - path_walk(), may_create(), and
+                                  _link() take fewer parameters in jewel)
+    	src/test/libcephfs/test.cc (preceding tests are missing in jewel)
+
+commit 173ea7f9856e37ee4febc5d1c13709bb81a4f37b
+Author: Samuel Matzek <smatzek@us.ibm.com>
+Date:   Mon Jan 16 11:11:31 2017 -0600
+
+    Ceph-disk to use correct user in check_journal_req
+    
+    The ceph-disk tool calls ceph-osd to check the journal requirements
+    using OSD id 0. This creates a log file for osd-0 on the system
+    using the current user/group for file permissions.  When ceph-disk
+    is run as root this makes the file owned by root which makes
+    the osd daemon for osd.0 unable to write to its own log file.
+    
+    This commit changes the journal reqs calls of ceph-osd to pass
+    the ceph user and group so ceph-osd creates the log file with the
+    appropriate permissions.
+    
+    Fixes: http://tracker.ceph.com/issues/18538
+    
+    Signed-off-by: Samuel Matzek <smatzek@us.ibm.com>
+    (cherry picked from commit bcf7514bf53693ec61e482341787c80494589faf)
+
+commit bf873a73cc3ee7e0fa5d7a1d2280fc3e3dd26ab0
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+    
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+    
+    Conflicts:
+    	qa/tasks/cram.py (trivial resolution)
+
+commit 72b24f0f47bebaed793d7b75ce0eacc3c1533ab5
+Merge: 174ed80 95edad2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 15:50:16 2017 +0100
+
+    Merge pull request #12766 from jtlayton/wip-18408-jewel
+    
+    client: Fix lookup of "/.." in jewel
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 174ed80fb77b172b6a7d3edb7c97cb49afe98ae8
+Merge: 01ff675 bee7e3a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 20 12:31:26 2017 +0100
+
+    Merge pull request #12147 from dachary/wip-18007-jewel
+    
+    jewel: ceph-disk: ceph-disk@.service races with ceph-osd@.service
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit e8f55f65cc908cb43fe44c271866ed7672773f1f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 16:06:45 2017 -0500
+
+    rgw: RGWAsyncRadosRequest drops notifier ref on cancel
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1d586f76a11fed937fc7bb0f7cf6a44ca0506881)
+
+commit 3f509aac28fc81c464d3fc2a0b4c011260533e93
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Nov 7 10:58:50 2016 -0500
+
+    rgw: remove circular reference in RGWAsyncRadosRequest
+    
+    RGWAsyncRadosRequest does not need a reference to its calling coroutine,
+    it only needs the completion notifier. this eliminates a circular
+    reference between RGWAsyncRadosRequest and the coroutines that create
+    them
+    
+    Fixes: http://tracker.ceph.com/issues/17792
+    Fixes: http://tracker.ceph.com/issues/17793
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f670c0ea5de7c6969e1d332824d80c55c8d6af0)
+
+commit 0ef1bdf973ae94e982c1e678862645f87ea85bdd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 4 11:36:00 2017 -0500
+
+    rgw: release RGWAioCompletionNotifier refs on destruction
+    
+    Fixes: http://tracker.ceph.com/issues/18407
+    Fixes: http://tracker.ceph.com/issues/18414
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7a4975adba8b890e211d47662896e445a9822970)
+
+commit c21622d31d830d59278808805238e11888d39350
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 6 12:30:42 2017 -0500
+
+    rgw_rados: add guard assert in add_io()
+    
+    Use the iterator-returning insert operation in std::map, check
+    assert the insert case.  As a side effect, this makes use of the
+    inserted object record more clear.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d10c37adf925d8b34daa81b419463ea376ec27aa)
+
+commit 585eb484956c1666bfb870e67a9c506a8154294e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 6 15:16:32 2017 -0500
+
+    rgw_rados: sanitize dout print in GWRados::get_obj_iterate_cb(...)
+    
+    We cannot assume pbl may be deferenced.  Per review, move the dout
+    print into the r < 0 condition--since it's now an error, make it's
+    trace level 0.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0233c4abb9f99304e3d82272bbb4385086bbfc0)
+
+commit 01ff6751e1705094fe1c8057685fa7216bd56118
+Merge: 25a9e5f e2eaae4
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Wed Jan 18 12:43:34 2017 -0800
+
+    Merge pull request #12983 from ceph/wip-cherry-pick-4vasu
+    
+    qa: Wip cherry pick https://github.com/ceph/ceph/pull/12969
+
+commit e2eaae4b4260df473cb5b7d9a467d5e4ca82d417
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 9 16:45:01 2017 -0800
+
+    Add ceph-create-keys to explicitly create admin/bootstrap keys
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 68f9b7eb3c0548c88650f67fb72c6ff9bc0f3ead)
+
+commit 2adc0ee858062a646380c8f594fa6a6207da29a3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 9 17:59:20 2017 -0800
+
+    Remove debug overrides
+    
+    the high level of debug for mon/osd is causing remoto to hang during get key
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f7dcc74cd3f119a2f65584fdb544c08d115f8c39)
+
+commit 8f36e23b49191f5a37184f9ee9b670025eecc4eb
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Jan 10 15:43:12 2017 -0800
+
+    use the create option during instantiation
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit be836bb30960000468c79e08fb416ceefd79d7db)
+
+commit 25a9e5f526d2758543357d7072dbdc2ec9031e5f
+Merge: 601fb96 0ab5b7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 18 17:12:54 2017 +0100
+
+    Merge pull request #12210 from ddiss/tracker18049_ceph_disk_trigger_flock_timeout_jewel
+    
+    jewel: systemd/ceph-disk: reduce ceph-disk flock contention
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 601fb96599ebcb07017951542ddc58b2ab2abfd4
+Merge: de70003 1ea9de2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 08:41:25 2017 -0500
+
+    Merge pull request #12959 from SUSE/wip-18545-jewel
+    
+    jewel: [teuthology] update Ubuntu image url after ceph.com refactor
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1ea9de2c14ebf6a4683cb0b796e92f86a58f3d59
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 16 22:12:51 2017 -0500
+
+    qa/tasks/qemu: update default image url after ceph.com redesign
+    
+    Fixes: http://tracker.ceph.com/issues/18542
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d17befb3bbc3d83c9d23d763ad95e1e7b2e4be0)
+
+commit 36186d03ebd5c6eb14a720a0fca3a5bbeb49c0a9
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 11 14:18:29 2016 +0530
+
+    test_volume_client: remove superfluous arguments
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit bb60e01904187db417e8c7d6e57401823a0072fd)
+
+commit 75496812dc3d35a1737bc091b588ea7ad533ca54
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 11 13:40:43 2016 +0530
+
+    test_volume_client: check volume size
+    
+    Check that the total size shown by the df output of a mounted volume
+    is same as the volume size and the quota set on the volume.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 91c74f4778ce5433968226345ffe26e876eb56a7)
+
+commit 3e3ffcf7362a9a6c8fc8eebcc527e92330cf5a5e
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Sep 6 17:31:04 2016 +0530
+
+    tasks/cephfs: test recovery of partial auth update
+    
+    ... in ceph_volume_client.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f0134a3db576282ed05d4b94b969b9593297669d)
+
+commit 3320ef1944a10cc5835510ee752750237cbefc7a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 4 13:55:46 2016 +0530
+
+    ceph_volume_client: fix partial auth recovery
+    
+    ... for volumes whose group_id is None.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 0ab8badcf3ffe685135af17dc28b238f6e686922)
+
+commit 5115c217bb9070aa9d0f0ea15a84cfb7862345b3
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 28 14:06:54 2016 +0530
+
+    ceph_volume_client: check if volume metadata is empty
+    
+    ... when recovering from partial auth updates.
+    
+    Auth update happens in the following order:
+    auth metadata update, volume metadata update, and then Ceph auth
+    update.
+    
+    A partial auth update can happen such that auth metadata is updated,
+    but the volume metadata isn't updated and is empty, and the auth
+    update did not propogate to Ceph. When recovering from such a
+    scenario, check if volume metadata is empty and if so remove the
+    partial auth update info in auth metadata.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit a95de7882cdf70e04e3c918ff41fc690d0d9bda3)
+
+commit bf33cd52a395b99453f72af016ea8f559ad3a13a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 4 16:50:13 2016 +0530
+
+    ceph_volume_client: fix _recover_auth_meta() method
+    
+    It needs to be an instance method.
+    
+    Fixes: http://tracker.ceph.com/issues/17216
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 675cb91b68c1b54698708d604253ab9d1b2abdec)
+
+commit 84044262635cd8674dbda79b9b6a7ab29cb554c8
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Jan 10 19:11:08 2017 -0700
+
+    mds/server: skip unwanted dn in handle_client_readdir
+    
+    We can skip unwanted dn which  < (offset_key, snap) via map.lower_bound, rather than
+    iterate across them.
+    
+    Previously we iterate and skip dn which < (offset_key, dn->last), as dn->last >= snap
+     means (offset_key, dn->last) >= (offset_key, snap), and such iterate_and_skip logic
+    still keep, so this commit doesnt change code logic but an optimization.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 52fe52baf920c672ac7f63a3087dcd31137891b6)
+
+commit de70003fda092d3fdf86b3323ab5a42038139c11
+Merge: 15c081c 06e40eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 13 11:10:39 2017 +0100
+
+    Merge pull request #12745 from SUSE/wip-18386-jewel
+    
+    jewel: tests: use ceph-jewel branch for s3tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a0ee8b930a41b56cd964b232e7230859d8d3b4aa
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Sun Jan 1 17:32:39 2017 +0800
+
+    jewel: fix compile error for dencode test case when --with-radosgw=no
+    
+    If cannot disable radosgw, the user has to always compile radosgw part,
+    even only want to use block device or file storage. Cherry-pick cannot
+    be done because ceph master doesn't have Makefile.am any more.
+    
+    Fixes: http://tracker.ceph.com/issues/18512
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+
+commit 5e6c72946374895dca65f25d590615428ab3d8ef
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Sun Jan 1 17:19:35 2017 +0800
+
+    jewel: fixed compile error when --with-radosgw=no
+    
+    If cannot disable radosgw, the user has to always compile radosgw part,
+    even only want to use block device or file storage. Cherry-pick cannot
+    be done because ceph master doesn't have Makefile.am any more.
+    
+    Fixes: http://tracker.ceph.com/issues/18512
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+
+commit cdd6cbfdfe5a77008ba298667bb7add8c236027a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 9 09:39:39 2016 -0500
+
+    librbd: block concurrent in-flight object map updates for the same object
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7d743bfce61c6ede0a34fc0982e52be1d367d772)
+
+commit 15c081c6433e9ee4b6dd7c145e8e6aaddf334e69
+Merge: 988cd95 04cee05
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 12 13:58:42 2017 -0800
+
+    Merge pull request #12912 from liewegas/wip-workunits-jewel
+    
+    qa/tasks/workunits: backport misc fixes to jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 04cee0521681a49eca733d196011be3213c34682
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+    
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+    
+    The same can happen if a previous workunit task already
+    checked out the repo.
+    
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+    
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit 1a98850578577dfe8a20df84be5e1638dc682e8b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit c101fba58b97015e77e5362d2db4112f8a8586fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+    
+    Root of git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit c7b74cda893e651750dc744185f009e46d115c0e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit 384e5c05b4986e19a0ca45173d23da7d573d894a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Dec 8 10:11:42 2016 -0500
+
+    rgw: add 'rgw log http headers' gloss to config-ref.rst
+    
+    Explain the option for upstream doc.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 04bf0f8ca1dfb949bc912b93a4ff84bf8bac57aa)
+
+commit 9fd29b440990c3458e4e65f22733d9da0c2606bc
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Dec 14 16:31:19 2016 -0500
+
+    use std::map
+    
+    (adaptation commit for #7639)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 8838bcbabcd2c9b3d7aec2403cad0bad4ad9a057
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 12 17:02:53 2016 -0500
+
+    rgw: add rgw_log_http_headers option
+    
+    Tracks headers that should be handled conditionally (currently,
+    can only log, so using minimal structure to represent the
+    mapping).
+    
+    Adds map of custom headers to rgw_log_entry, and populate it with
+    headers pre-selected for custom logging in RGWREST.  Added to encoder
+    and Formatter output.
+    
+    Some additional optimization possible.
+    
+    (cherry picked from commit b82919a152217b3cd49afdc28bb890f329c2742a)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 87557754f591607a1667fecfc8dabc7e52bbbca0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 22:38:47 2016 -0500
+
+    librbd: new block guard helper to prevent concurrent IO to blocks
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b1d624b43ec159d4a07616e86557ea48f089b7a1)
+    
+    Conflicts:
+    	src/librbd/BlockGuard.h: fixed compile issue
+    	src/librbd/Makefile.am: added BlockGuard
+    	src/test/Makefile-client.am: added BlockGuard test
+    	src/test/librbd/CMakeLists.txt: trivial resolution
+
+commit 5d306fd015d3e6d0aa35f368d3ca6cde6e7ca77b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 8 18:27:33 2016 -0500
+
+    librbd: convert ObjectMap to template for unit testing
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ea7b30a4fb105e052603c55ac2dc2aca11e66545)
+    
+    Conflicts:
+    	src/librbd/image/CreateRequest.cc: not in jewel
+    	src/librbd/internal.cc: trivial resolution
+    	src/librbd/object_map/CreateRequest.cc: not in jewel
+    	src/librbd/object_map/RemoveRequest.cc: not in jewel
+    	src/test/librbd/test_ObjectMap.cc: trivial resolution
+
+commit c53df3780dd9221cfe602c09651eeee06046ebeb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 22:41:56 2016 -0500
+
+    librbd: clean up object map update interface
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 477ae54a568bd1081fd2c77b570b04dd1b983cd9)
+    
+    Conflicts:
+    	src/librbd/AioObjectRequest.cc: trivial resolution
+    	src/librbd/ObjectMap.cc: trivial resolution
+    	src/librbd/operation/TrimRequest.cc: removed optimizations
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc: trivial resolution
+
+commit 6fe9be8de391aa239fc82f5483508d762a44cb99
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 16:28:22 2016 -0500
+
+    librbd: update in-memory object map after on-disk update committed
+    
+    Concurrent IO to the same object would previously result in the first
+    IO pausing to update the object map while the other IO would proceed
+    to directly update the object before the object map state was properly
+    updated.
+    
+    Fixes: http://tracker.ceph.com/issues/16176
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 378b810cbaadb1a12a7f0d21ed9a33e2a9640f55)
+    
+    Conflicts:
+    	test/librbd/object_map/test_mock_UpdateRequest.cc: resize op signature
+
+commit 988cd95d84be7973f774a4dae2d09faf4dbc30c3
+Merge: d4b1341 895ab24
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 12 07:29:34 2017 +0100
+
+    Merge pull request #12791 from athanatos/wip-15943-jewel
+    
+    jewel: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas (part 2)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f24c3fffc0ee26fddbc416620bbb9c0b353a0cf2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 30 17:28:59 2016 -0500
+
+    osd/PG: publish PG stats when backfill-related states change
+    
+    These frequently get flushed because other updates
+    happen, but we should explicitly ensure that the mon
+    sees these state changes.
+    
+    Fixes: http://tracker.ceph.com/issues/18369
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4adeb7500a113097fdd717ada0231f68badafbb)
+    
+    Conflicts:
+    	src/osd/PG.cc: trivial - PG::RecoveryState::Recovering::Recovering()
+    	hunk fails to apply since there's no PG::queue_recovery(),
+    	it's pg->osd->queue_for_recovery(pg) in jewel
+    
+    Fixes: http://tracker.ceph.com/issues/18485
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit d4b1341cf38f430f047f7263d713c0161b44730f
+Merge: 5b402f8 182babf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jan 10 16:25:18 2017 -0800
+
+    Merge pull request #12868 from athanatos/wip-17899-jewel
+    
+    OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fe753dbc9b4bd03d7a6063724edc59c2f860fccf
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Tue Jan 10 09:42:18 2017 +0100
+
+    install-deps.sh: unify indentation in case statement
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+
+commit 94ab8d85b59ff8bd8919930b9d912d1bf55eb9ed
+Author: Nitin A Kamble <Nitin.Kamble@Teradata.com>
+Date:   Mon Oct 31 11:46:13 2016 -0700
+
+    install-deps.sh: allow building on SLES systems
+    
+    Avoids this error on SLES systems:
+    
+    > ./install-deps.sh
+    sles is unknown, dependencies will have to be installed manually.
+    
+    Signed-off-by: Nitin A Kamble <Nitin.Kamble@Teradata.com>
+    (cherry picked from commit 266f4d4f20a1756e825ee54d79fe5f8d931d4b77)
+    
+        Conflict in install-deps.sh due to indentation. Trivial resolution.
+
+commit 7159265ac154ab798f64272b97aa5d7154d0b303
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 5 13:40:41 2017 +0000
+
+    qa/tasks: add test_corrupt_backtrace
+    
+    Validate that we get EIO and a damage table entry
+    when seeing a decode error on a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5f6cdab80f6e2f09af5783c8f616d8ddd6d9f428)
+
+commit 893d4ab72fec1529550b28ee930be8de5a74ee2b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Dec 20 18:04:47 2016 +0000
+
+    mds: check for errors decoding backtraces
+    
+    Fixes: http://tracker.ceph.com/issues/18311
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6f489c74ac0040631fde0ceb0926cbab24d3ad55)
+
+commit 895ab243fe5d3bd07bff0bd422d1dc9cb561f0a0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:35:38 2016 -0800
+
+    PG: fix cached_removed_snaps bug in PGPool::update after map gap
+    
+    5798fb3bf6d726d14a9c5cb99dc5902eba5b878a actually made 15943 worse
+    by always creating an out-of-date cached_removed_snaps value after
+    a map gap rather than only in the case where the the first map after
+    the gap did not remove any snapshots.
+    
+    Introduced: 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5642e7e1b3bb6ffceddacd2f4030eb13a17fcccc)
+
+commit 299478a1e69c9da8dc902e2f102d5d272412fdc8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 14 15:48:59 2016 -0800
+
+    qa/config/rados.yaml: enable osd_debug_verify_cached_snaps
+    
+    Also, make map gaps more likely.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit d4b6615a49e4635113f9ba900e9c57147b224357)
+
+commit b492c0089de858cfe2d7030eddd046acac6872ab
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:33:13 2016 -0800
+
+    PG::handle_advance_map: add debugging option to verify cached_removed_snaps
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit aeb8fef92469831d94f06db457a4ba15b5b0e3c5)
+
+commit 2296c87e2e8c885c5c6bee1ea1807aa7862f88fc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Nov 24 18:23:43 2016 +0100
+
+    osd: improve error message when FileStore op fails due to EPERM
+    
+    References: http://tracker.ceph.com/issues/18037
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f7723e59b6e10e1aef35543639e0ea1fcac65574)
+
+commit 7819adb3b5f9af813d4df05d3483175ee54e10df
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 12 18:48:12 2016 -0400
+
+    client: don't use special faked-up inode for /..
+    
+    The CEPH_INO_DOTDOT thing is quite strange. Under most OS (Linux
+    included), the parent of the root is itself. IOW, at the root, '.' and
+    '..' refer to the same inode.
+    
+    Change the ceph client to do the same, as this allows users to get
+    valid stat info for '..', as well as elimnating some special-casing.
+    
+    Also in several places, we're checking dn_set.empty as an indicator
+    of being the root. While that is true for the root, it's also true
+    for unlinked directories.
+    
+    This patch has treats them the same. An unlinked directory will
+    be reparented to itself, effectively acting as a root of its own.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 30d4ca01db0de9a1e12658793ba9bf9faf0331dd)
+
+commit 95edad2d058575231bb9b5468559f7b42667f40e
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Jan 3 12:56:51 2017 -0500
+
+    client: don't use special faked-up inode for /..
+    
+    The CEPH_INO_DOTDOT thing is quite strange. Under most OS (Linux
+    included), the parent of the root is itself. IOW, at the root, '.' and
+    '..' refer to the same inode.
+    
+    Change the ceph client to do the same, as this allows users to get
+    valid stat info for '..', as well as elimnating some special-casing.
+    
+    Also in several places, we're checking dn_set.empty as an indicator
+    of being the root. While that is true for the root, it's also true
+    for unlinked directories.
+    
+    This patch has treats them the same. An unlinked directory will
+    be reparented to itself, effectively acting as a root of its own.
+    
+    Fixes: http://tracker.ceph.com/issues/18408
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 30d4ca01db0de9a1e12658793ba9bf9faf0331dd)
+
+commit 9a59ce946c4f73d200969d9352b7049c6690eaf4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Sep 19 15:05:22 2016 +0200
+
+    rgw: fix decoding of creation_time and last_update.
+    
+    Fixes: http://tracker.ceph.com/issues/17167
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3afe715381ca63539bb20609b614f5e60561711a)
+
+commit bbf4c27cab6476e284feed1fa151cca8763cd442
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:35:02 2016 -0500
+
+    qa/tasks/admin_socket: subst in repo name
+    
+    It is either ceph.git or ceph-ci.git.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bcbe45d948f1c4da02e27b3be5f29a8b52745e28)
+
+commit 944ec0348e5b4cd69c8aedda833cf5e468a98dc9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Dec 13 18:54:17 2016 +0200
+
+    rbd-nbd: invalid error code for "failed to read nbd request" messages
+    
+    Fixes: http://tracker.ceph.com/issues/18242
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 42c2514fe22d34e1bd5a5d2686e06f7e01b7c0c3)
+
+commit fcdd5e75c3a3107e4b8c8562036fd54e8ea40ddf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 2 13:36:35 2016 -0500
+
+    rados: optionally support reading omap key from file
+    
+    Fixes: http://tracker.ceph.com/issues/18123
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 286ceb1e035f060cc564c1ef7400382331893101)
+
+commit d313e42db210e3ab04b5e7e513f7f2660ba1aa1d
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed Dec 14 15:18:18 2016 +0800
+
+    librbd: ignore error when object map is already locked by current client
+    
+    otherwise when using rbd cli to rollback image with object-map feature
+    enabled, the following error message will be printed out on the screen,
+    which is confusing to users:
+      librbd::object_map::LockRequest: failed to lock object map: (17) File exists
+    
+    Fixes: http://tracker.ceph.com/issues/16179
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 711da4e2396a8967cd457133a847d40cb46b0a5a)
+
+commit d0b0d4114bb5933bbc6b707e781f1a9262b56fd5
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 22:49:13 2017 +0100
+
+    tests: rbd/test_lock_fence.sh: fix rbdrw.py relative path
+    
+    This commit fixes a regression introduced by
+    cf294777ea92f0911813a7132068584d4f73a65a
+    
+    Fixes: http://tracker.ceph.com/issues/18388
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 91231de16dbe4d0e493ec617165a2b38078d122b)
+
+commit 06e40eb27f65df861b8f9992f0706fbc464d3f39
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 4 10:03:08 2016 +0100
+
+    tests: use ceph-jewel branch for s3tests
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8877ee4d48b025c6dc171f3ba6064c72b9171ddc
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 16 12:50:50 2016 +0200
+
+    rbd: fix json formatting for image and journal status output
+    
+    Fixes: http://tracker.ceph.com/issues/18261
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4c2659a0f49ff5f709a8275d6adaa817daf8f76a)
+
+commit 9c84a657e25ccb32be7f775fef3d4c563d4e95b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 14 20:49:46 2016 -0500
+
+    journal: prevent repetitive error messages after being blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18243
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 60f1b1a2f2a96cfe72acfc6855b9cc26b480732d)
+
+commit d069464be70c3a6286cb7918eb83d555e1d52759
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 13 14:40:23 2016 -0500
+
+    journal: avoid logging an error when a watch is blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18243
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b82a1c1d1fc43e13aa6e76f5468f43a1f779f7fa)
+
+commit 9a1258de21d4ee2b965b3cdeb65bd71ceac573c1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 12 16:42:15 2016 -0500
+
+    rgw: use explicit flag to cancel RGWCoroutinesManager::run()
+    
+    RGWCoroutinesManager::run() was setting ret = -ECANCELED to break out of
+    the loop when it sees going_down. coroutines that failed with -ECANCELED
+    were confusing this logic and leading to coroutine deadlock assertions
+    below. when we hit the going_down case, set a 'canceled' flag, and check
+    that flag when deciding whether to break out of the loop
+    
+    Fixes: http://tracker.ceph.com/issues/17465
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 73cd8df887fb5e45f2d49275cedfeab31809ddc8)
+
+commit a67dca4a45cf68625b6f673b251e101fe713e5b5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+    
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+    
+    The same can happen if a previous workunit task already
+    checked out the repo.
+    
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+    
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit e5c81c34bb766d94d2ef21b407d70d2d089c425b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit efaedb355dbc4697a3f9c3147053a5c92642e785
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 19 14:08:11 2016 -0500
+
+    qa/workunits: include extension for nose tests
+    
+    When you have a relative path you have to include the extension.
+    Weird.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5666fd61d6dbd40be1d79354227cabd562e829ea)
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    Conflicts:
+    	qa/workunits/rados/test_python.sh (nosetests instead of nose)
+
+commit de15912c3bb714f9ff48ad8e10b8b4dd6961069e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 15:10:28 2016 -0500
+
+    qa/workunits: use relative path instead of wget from git
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf294777ea92f0911813a7132068584d4f73a65a)
+    
+    Conflicts:
+    	qa/workunits/rados/test_python.sh (nosetests instead of nose)
+
+commit 74aac99ce20462de0aca042b6128ea479d17bb4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+    
+    Root of git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit 117d38e2382bc0016729387aba1dfaae9bb7a684
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit 5aa9387555c98666a83f909646b62c8688ac24d7
+Author: Weibing Zhang <atheism.zhang@gmail.com>
+Date:   Thu Dec 22 14:38:27 2016 +0800
+
+    rgw: ldap: simple_bind() should set ldap option on tldap
+    
+        ldap_set_option() should set option for "tldap" other than "ldap".
+        The current code will lead to "Protocol error" as the ldap version
+        of tldap is not set.
+    
+    Signed-off-by: Weibing Zhang <zhangweibing@unitedstack.com>
+    (cherry picked from commit 96cf7fa06a1192293b24c823ad9d08456f81ac8a)
+
+commit dcc9483cbd605997876983ec65f6fb80d5da8a52
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Apr 7 16:43:17 2016 +0800
+
+    mon: OSDMonitor: trigger an immediate propose if any newly down osd is detected during tick()
+    
+        Currently we rely on OSDs to watch over each other and perform failure detection
+        and report to OSDMonitor. Before we can safely and undoubtedly mark an OSD as down,
+        enough reports from a certain number of different reporters must have been collected.
+        Also, the victimed OSD has to be declared failed long enough before we make any final
+        decision in order to avoid temperary problems such as network failure, network traffic jam etc.,
+        which if handled carelessly, may cause even serious problem such as flapping.
+    
+        Form the above analysis, even if we have gathered enough witnesses, we have to wait long
+        enough to sentence the guilty OSD to death. Therefore we rely on the tick() thread to
+        do such an hourglass job. However, the problem here is currently the tick() thread is
+        unable to trigger a propose even if it has witnessed such a memont, and this is our goal
+        to solve such an embrassing situation.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit a80d6c500ab247013a1c068c457c1b9bfbc750b2)
+
+commit 64c0caeab21f1f764bb48c0355b24f24dda2cfe2
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Fri Dec 9 02:40:03 2016 +0800
+
+    librbd/diff_iterator: use proper snap to query parent overlap
+    
+    If no snapshot has been made, we will use CEPH_NOSNAP instead of 0,
+    to query the parent overlap.
+    
+    Fixes bug http://tracker.ceph.com/issues/18200
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit a88ceff2767761483aee73590767c412116a7489)
+
+commit d584f9e124cfd1da1b8d8dbfa1a0d370185c6835
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon May 9 11:41:19 2016 +0200
+
+    rgw: log name instead of id for SystemMetaObj on failure
+    
+    Currently if we fail to read a SystemMetaObj we try to log the
+    MetaObject id, however this will not be set mostly as read_id has
+    failed, so we end up logging an empty id, changing this to log
+    the object name instead
+    
+    Fixes: http://tracker.ceph.com/issues/15776
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e9f896a9efea74c42d56dd5a7feb8b8710d6becc)
+
+commit 1a0becf4e45e753503d27719540b966d74fdafff
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon May 9 14:05:06 2016 +0200
+
+    rgw: drop unnecessary spacing in rgw zg init log
+    
+    Dropping unneeded space when we're printing the failed reading zg info
+    message
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit a43ac6e56857ac9b2bf7d6aa2956b7bcc045d5ce)
+
+commit 5b402f8a7b5a763852e93cd0a5decd34572f4518
+Merge: e565d89 3dbf0c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 22 00:18:11 2016 +0100
+
+    Merge remote-tracking branch 'ceph/jewel-next' into jewel
+
+commit e565d8924241ba0520a836fc96040de41582906f
+Merge: f97b399 5799d5f
+Author: jtlayton <jlayton@redhat.com>
+Date:   Wed Dec 21 09:18:18 2016 -0500
+
+    Merge pull request #12591 from jtlayton/wip-18308-jewel
+    
+    Clear setuid bits on ownership changes
+
+commit 5799d5fdba9b890634a9b3d6fa038dfde3818779
+Merge: a0ae9a8 f97b399
+Author: jtlayton <jlayton@redhat.com>
+Date:   Tue Dec 20 15:36:39 2016 -0500
+
+    Merge branch 'jewel' into wip-18308-jewel
+
+commit f97b399896261612aa0b8bab2c720446d908a644
+Merge: c067b58 f15c8da
+Author: jtlayton <jlayton@redhat.com>
+Date:   Tue Dec 20 15:35:54 2016 -0500
+
+    Merge pull request #12592 from jtlayton/wip-18307-jewel
+    
+    Fix mount root for ceph_mount users and change tarball format
+
+commit f15c8dafbcc15982265f072aeddb9d5dfc7166d6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 14:44:04 2016 -0500
+
+    ceph_disk: fix a jewel checkin test break
+    
+    Silly python:
+    
+        ceph_disk/main.py:173:1: E305 expected 2 blank lines after class or function definition, found 1
+        ceph_disk/main.py:5011:1: E305 expected 2 blank lines after class or function definition, found 1
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+
+commit 8e0cffdd435eeb24f4ea3658cf1db9973759dd29
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 11:54:25 2016 -0500
+
+    automake: convert to tar-pax
+    
+    We hit some recent build issues with the merge of ceph-qa-suite into
+    the main repo. The ustar format barfs on >100 character symlink
+    paths.
+    
+    Convert to using "tar-pax" which should make it use the posix format.
+    Any build machine that we're reasonably targeting should support it.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+
+commit a0ae9a83df222fce75f41d3b5a3e3d0396dd748a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:17:21 2016 -0500
+
+    client: drop setuid/setgid bits on ownership change
+    
+    When we hold exclusive auth caps, then the client is responsible for
+    handling changes to the mode. Make sure we remove any setuid/setgid
+    bits on an ownership change.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 18d2499d6c85a10b4b54f3b8c335cddf86c4588f)
+
+commit d49e628361a17cca65f31f85350468228a0ec2d7
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:16:43 2016 -0500
+
+    mds: clear setuid/setgid bits on ownership changes
+    
+    If we get a ownership change, POSIX mandates that you clear the
+    setuid and setgid bits unless you are "appropriately privileged", in
+    which case the OS is allowed to leave them intact.
+    
+    Linux however always clears those bits, regardless of the process
+    privileges, as that makes it simpler to close some potential races.
+    Have ceph do the same.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 6da72500882d9749cb2be6eaa2568e6fe6e5ff4d)
+
+commit f2dfc200dbf93927ca1059ae203c8242b23eeb40
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:07:23 2016 -0500
+
+    client: set metadata["root"] from mount method when it's called with a pathname
+    
+    Currently, we only set the root properly config file or the
+    --client_metadata command line option. If a userland client program
+    tries to call ceph_mount with a pathname, it's not being properly
+    set.
+    
+    Since we already hold the mutex, we can just update it directly.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 9f8810008c82eebe6e354e7e321e33a3dcba8407)
+
+commit 87a2a95ad1103cecf4b710776d35182920e72823
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Mon Aug 29 14:32:42 2016 +0530
+
+    rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth.
+    
+    s3cmd encodes space as "%20" while signature computation and
+    encodes space as '+' while sending the canonical uri. This
+    results in a SignatureMismatch Error in rgw, since rgw
+    computes the signature based on the request received from
+    the client (s3cmd in this case).
+    
+    Fixes http://tracker.ceph.com/issues/17076.
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 20e5ff023ebad89c386a520d07613547d4836399)
+
+commit aa8e57d0d4fa855ec6b2201274dd78504ab9e5c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 6 12:56:31 2016 -0400
+
+    rbd: utilize new API methods for image id and block name prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a0a88c71552aa5858384fa802a3161da90e7c86)
+    
+     Conflicts:
+    	src/tools/rbd/action/Info.cc (jewel does not have
+                    653bc453e3c8f1062cdbc4d0d8f77f623f48915b)
+
+commit 05295eff568aec5ff9f5d01bb73b7c7bd1dee581
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 6 12:48:22 2016 -0400
+
+    librbd: new API methods to retrieve image id and block name prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 366e6075cab2748efab395cd23882eaee4ba402f)
+
+commit 240431bea219412edf9588490f6d72d16b1f01b6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 5 15:40:33 2016 +0200
+
+    build/ops: fix /etc/os-release parsing in install-deps.sh
+    
+    85a370e35fc42031a7f7e24dea9e50a649c0f309 introduced the DISTRO variable whose
+    value was obtained by parsing /etc/os-release like this:
+    
+    DISTRO=$(grep  "^ID=" /etc/os-release | sed "s/ID=//")
+    
+    This unfortunately picks up the double-quotes, so on a CentOS system DISTRO
+    will be equal to '"centos"'.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fd37e87f6de0b8e272a2859ec52d6ceab277db7a)
+
+commit f6519509049362ec3f378e2178e577f761a72c42
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Jun 29 09:40:07 2016 +0200
+
+    install-deps.sh: initial distro detection based on /etc/os-release
+    
+    This avoids initial distribution detection (before lsb_release is available)
+    based on the presence of a specific package manager. This caused an
+    issue with openSUSE Tumbleweed before since both zypper and apt-get are
+    available.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 85a370e35fc42031a7f7e24dea9e50a649c0f309)
+
+commit c067b589128fac3fb382f64ac91ade3be3c69a51
+Merge: c461ee1 d2380d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:39:56 2016 -0600
+
+    Merge pull request #12454 from liewegas/qa-suite-jewel
+    
+    jewel: merge ceph-qa-suite
+
+commit d2380d7d1d4fd568b286e0fb90d8d192f361f7c9
+Merge: c461ee1 1c28e7f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:59 2016 -0600
+
+    merge ceph-qa-suite
+
+commit 1c28e7f08d0e42a14e28b78e12b57a219ce8796c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:55 2016 -0600
+
+    move ceph-qa-suite dirs into qa/
+
+commit 282451d8cf8d98ca90502e29081aaf7c4a4c9508
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:27:58 2016 -0500
+
+    Revert "tasks/workunit.py: depth 1 clone"
+    
+    This reverts commit e6f61ea9f19d0f1fad4a6547775fa80616eeeb89.
+
+commit e6f61ea9f19d0f1fad4a6547775fa80616eeeb89
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:19:44 2016 -0500
+
+    tasks/workunit.py: depth 1 clone
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4faf77a649cb3f8ddf497ca81937b3dbf63a18dc)
+
+commit 426f7cf2d151a1122cb603a2c9f224a25c9702a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:18:29 2016 -0500
+
+    tasks/workunit: remove kludge to use git.ceph.com
+    
+    This was hard-coded to ceph.git (almost) and breaks when
+    you specify --ceph-repo.  Remove it entirely.  We'll see if
+    github.com is better at handling our load than it used to
+    be!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 159c455a0326eef2c017b3e3cf510f918b5ec76c)
+
+commit 1ba5995b7d9816bab936bc1ba557504821e94c89
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 10 02:36:52 2016 +0800
+
+    tasks/ceph: restore context of osd mount path before mkfs
+    
+    all newly created files and directories under the mount dir inherit the
+    SELinux type of their parent directory. so we need to set it before
+    mkfs.
+    
+    Fixes: http://tracker.ceph.com/issues/16800
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53225d5272a1d35d4183fcfa55a139f55f77e122)
+
+commit 26c87fd1f1d105e5aa1c4c32c73f75414ea3d04f
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Dec 8 20:03:30 2016 +0800
+
+    rbd: --max_part and --nbds_max options for nbd map
+    
+    Fixes: http://tracker.ceph.com/issues/18186
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+    (cherry picked from commit 45ceb389b685a5ec7bc0b8ef9e180ce851646082)
+    
+    Conflicts:
+    	src/test/cli/rbd/help.t (no --exclusive in jewel-next)
+    	src/tools/rbd/action/Nbd.cc (no --exclusive in jewel-next)
+
+commit ddb5403605f843b4bc5abd1093b034e087c07f8b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 14:36:38 2016 -0500
+
+    radosgw-admin: 'zone placement modify' doesnt require pool names
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b59afea4dbb06454b1e14f03dd65ba2449674a15)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (jewel-next lacks --tier* options)
+
+commit 7cfc346cf1427e37be38ebb92ad9518afa982b83
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 13:43:28 2016 -0500
+
+    radosgw-admin: add 'zonegroup placement default' command
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 711cc132528f2dfeb6c1f8defb19493ffe7cb9be)
+
+commit dbc1b614add263fe80ef6fc0c09c0597fb9cddff
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 13:08:38 2016 -0500
+
+    radosgw-admin: fix 'placment' typos
+    
+    Fixes: http://tracker.ceph.com/issues/18078
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0cf08985683820cd11492a9ad684d06565a04f34)
+
+commit 4f7147cb79aa0637e156a3a29739fcbadc37e62d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:55:52 2016 -0800
+
+    rgw_admin: commands to manage placement targets
+    
+    Fixes: http://tracker.ceph.com/issues/18078
+    
+    added the following commands:
+    
+     radosgw-admin zone placement add
+     radosgw-admin zone placement modify
+     radosgw-admin zone placement rm
+     radosgw-admin zone placement list
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ead132ae84bf3b2738fa0443f4f9666d59edbb43)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (jewel-next lacks the --tier* and --sync-from* options)
+    	src/test/cli/radosgw-admin/help.t (jewel-next lacks the --tier* and --sync-from* options)
+
+commit 155641f9ce6a16ee1fdb3231d76bea0eafc89ff8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:15:09 2016 -0800
+
+    rgw-admin: add commands to manage zonegroup placement fields
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a91c7b5cc699b2c37d449974d2ebf2eece9506a0)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (chose raw_storage_ops_list from master)
+
+commit bb9678bf44b8ed0dd8bb1f895da3c78edc792eeb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:05:27 2016 -0800
+
+    rgw: use set for zonegroup placement target tags
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 12162d654ce24db0617e4bc3de24aebc28b50996)
diff --git a/doc/changelog/v10.2.7.txt b/doc/changelog/v10.2.7.txt
new file mode 100644
index 000000000..473f45e1d
--- /dev/null
+++ b/doc/changelog/v10.2.7.txt
@@ -0,0 +1,1530 @@
+commit 50e863e0f4bc8f4b9e31156de690d765af245185
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Apr 10 11:43:44 2017 +0000
+
+    10.2.7
+
+commit a64d3e4b33e904aa6585464df8ffff6aafdec10c
+Merge: c92640a 6c2a40a
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:27:07 2017 -0600
+
+    Merge pull request #14230 from linuxbox2/wip-jewel-expand-argv
+    
+    jewel: rgw_file: expand argv
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c92640a113843a491678c11319d2352f14486f17
+Merge: 7c35658 fcc3ada
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:17:10 2017 -0600
+
+    Merge pull request #14233 from ktdreyer/wip-19421-jewel
+    
+    jewel: librbd: possible race in ExclusiveLock handle_peer_notification
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7c3565899af5f1f474d82d8802d15c6c216ea894
+Merge: 12467f6 eedb9f7
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:16:48 2017 -0600
+
+    Merge pull request #14231 from linuxbox2/wip-jewel-fhcache
+    
+    jewel: rgw_fh: RGWFileHandle dtor must also cond-unlink from FHCache
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 12467f6c3f346072d81c444ed68eb206402d8c33
+Merge: 2d2521c 78c8be7
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:09:14 2017 -0600
+
+    Merge pull request #14215 from mdw-at-linuxbox/wip-jewel-rgw-openssl-fix1
+    
+    rgw: fix openssl
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2d2521c0300000421740c611a5133c82d2e4d0f6
+Merge: 0840cf8 7e4e290
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:07:48 2017 -0600
+
+    Merge pull request #14206 from linuxbox2/wip-gui-jewel
+    
+    jewel: rgw_file: various fixes
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0840cf86214c793794c6f913b63cc67e0ab3fc94
+Merge: 0bf6360 b24a8c2
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:07:03 2017 -0600
+
+    Merge pull request #14205 from linuxbox2/wip-dir-orphan-jewel
+    
+    jewel:  rgw_file: wip dir orphan
+    
+     Conflicts:
+    	src/rgw/rgw_file.cc (whitespace, trivial resolution)
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0bf636057ba853d1d1a777147014a740d5149b17
+Merge: 83b5a7f fb85c68
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:05:40 2017 -0600
+
+    Merge pull request #14169 from linuxbox2/wip-18650-jewel
+    
+    jewel: rgw: rgw_file: FHCache residence check should be exhaustive
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 83b5a7f1c589546645e0f377d5d5f7b1f7363a59
+Merge: 6a175f2 a969c44
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:02:58 2017 -0600
+
+    Merge pull request #13869 from smithfarm/wip-19161-jewel
+    
+    jewel: rgw: rgw_file: fix marker computation
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fb85c68dfbfcfb299cac3a7e6723067f216483ef
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 15:49:06 2017 -0500
+
+    rgw_file:  use fh_hook::is_linked() to check residence
+    
+    Previously we assumed that !deleted handles were resident--there
+    is an observed case where a !deleted handle is !linked.  Since
+    we currently use safe_link mode, an is_linked() check is
+    available, and exhaustive.
+    
+    Fixes: http://tracker.ceph.com/issues/19111
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0aa515f8d8c57ec5ee09e3b60df3cac60453c40)
+
+commit fcc3ada733bac74a642ccd0f1ed6301ad7318a30
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 28 15:55:11 2017 +0200
+
+    jewel: librbd: possible race in ExclusiveLock handle_peer_notification
+    
+    This is a cherry-pick from kraken -- the master diverged after
+    ManagedLock refactoring and is not affected.
+    
+    Fix: http://tracker.ceph.com/issues/19368
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit df59d6d5f7deb586cf14a6ef6984e6847db08852)
+
+commit eedb9f7d2d2a4047b473f23c5a6956c40a3da126
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 20:24:12 2017 -0500
+
+    rgw_file: RGWFileHandle dtor must also cond-unlink from FHCache
+    
+    Formerly masked in part by the reclaim() action, direct-delete now
+    substitutes for reclaim() iff its LRU lane is over its high-water
+    mark, and in particular, like reclaim() the destructor is certain
+    to see handles still interned on the FHcache when nfs-ganesha is
+    recycling objects from its own LRU.
+    
+    Fixes: http://tracker.ceph.com/issues/19112
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d51a3b1224ba62bb53c6c2c7751fcf7853c35a4b)
+
+commit 6c2a40a800277ded302f3183ac6c68b01ca3ed41
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 17 11:23:45 2017 -0500
+
+    rgw_file:  split last argv on ws, if provided
+    
+    This is intended to allow an "extra" unparsed argument string
+    containing various cmdline options to be passed as the last argument
+    in the argv array of librgw_create(), which nfs-ganesha is
+    expecting to happen.
+    
+    While at it, hook env_args() too.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit fbc19e4effc736c98cc1cc283e5c7b131a0fa766)
+
+commit d2ca03b391340c6944ba453b106f3e1c45b75f1d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 20:18:26 2017 -0500
+
+    rgw_file: fix hiwat behavior
+    
+    Removed logic to skip reclaim processing conditionally on hiwat,
+    this probably meant to be related to a lowat value, which does
+    not exist.
+    
+    Having exercised the hiwat reclaim behavior, noticed that the
+    path which moves unreachable objects to LRU, could and probably
+    should remove them altogether when q.size exceeds hiwat.  Now
+    the max unreachable float is lane hiwat, for all lanes.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b8791b2217e9ca87b2d17b51f283fa14bd68b581)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 166cb7f85c240eeaffc70968abf5352d9cd45bd9
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 18:20:43 2017 -0500
+
+    rgw_file: refcnt bugfixes
+    
+    This change includes 3 related changes:
+    
+    1. add required lock flags for FHCache updates--this is a crash
+       bug under concurrent update/lookup
+    
+    2. omit to inc/dec refcnt on root filehandles in 2 places--the
+       root handle current is not on the lru list, so it's not
+       valid to do so
+    
+    3. based on observation of LRU behavior during creates/deletes,
+       update (cohort) LRU unref to move objects to LRU when their
+       refcount falls to SENTINEL_REFCNT--this cheaply primes the
+       current reclaim() mechanism, so very significanty improves
+       space use (e.g., after deletes) in the absence of scans
+       (which is common due to nfs-ganesha caching)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit beaeff059375b44188160dbde8a81dd4f4f8c6eb)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 31a71be9c9f24e22cf7e6eb390d3b39811ee3577
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Feb 11 16:38:05 2017 -0500
+
+    rgw_file:  add refcount dout traces at debuglevel 17
+    
+    These are helpful for checking RGWFileHandle refcnt invariants.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 462034e17f919fb783ee33e2c9fa8089f93fd97d)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 1d6c72fe9a59561fb8f33889895b6708342b2856
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 17:14:16 2017 -0500
+
+    rgw_file: add pretty-print for RGWFileHandle
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ef330f385d3407af5f470b5093145f59cc4dcc79)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit a969c449272ce88ddcbbd509432134f19520c8c3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+    
+    Fixes: http://tracker.ceph.com/issues/19018
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+
+commit a70e83c90eb42439ba194ca197f507f972a18c3c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+    
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+    
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+    
+    Fixes: http://tracker.ceph.com/issues/18991
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+
+commit 209987e1c764cac7574bf0bb969e93fcf55b4361
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+    
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+    
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+    
+    Fixes: http://tracker.ceph.com/issues/18989
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+
+commit 6a175f27961bd4f0fda8f94c200458f17865c9f9
+Merge: d32ae21 f3face6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:36:35 2017 -0600
+
+    Merge pull request #14140 from smithfarm/wip-19341-jewel
+    
+    jewel: 'period update' does not remove short_zone_ids of deleted zones
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d32ae21128d664370f7d07ed14427b75e10da7f8
+Merge: 4dfeed9 0be4e89
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:35:16 2017 -0600
+
+    Merge pull request #13886 from cbodley/wip-17786
+    
+    jewel: rgw: clear data_sync_cr if RGWDataSyncControlCR fails
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4dfeed949655a5b5041f612b868225c234e86bba
+Merge: d69c54a 1985662
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:34:57 2017 -0600
+
+    Merge pull request #13867 from smithfarm/wip-19159-jewel
+    
+    jewel: rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d69c54aca9dde3d64b56e5ee72f1cd29057a80c6
+Merge: 0830135 dfaaec0
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:34:24 2017 -0600
+
+    Merge pull request #13858 from smithfarm/wip-19152-jewel
+    
+    jewel: rgw_file:  restore (corrected) fix for dir partial match (return of FLAG_EXACT_MATCH)
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0830135f40ac37ac9027bece65b8aac64c2c7ec9
+Merge: 4dd3a9f 97fed01
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:33:18 2017 -0600
+
+    Merge pull request #13848 from smithfarm/wip-19150-jewel
+    
+    jewel: rgw_file:  avoid interning .. in FHCache table and don't ref for them
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 4dd3a9f628053784cff2aa5849ab3c4cdb2acf4e
+Merge: 5eeba9c d333add
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:29:30 2017 -0600
+
+    Merge pull request #13844 from smithfarm/wip-19148-jewel
+    
+    jewel: rgw: DUMPABLE flag is cleared by setuid preventing coredumps
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 5eeba9cb85e9111ef124acd36dccf77c6f00b3e4
+Merge: ae632b0 f7ce5df
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:26:48 2017 -0600
+
+    Merge pull request #13823 from asheplyakov/19176-bp-jewel
+    
+    jewel: rgw: fix swift cannot disable object versioning with empty X-Versions-Location
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ae632b0da6cc90a993fcae31b56f97c834d15aa1
+Merge: 874120f e8041f6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:22:23 2017 -0600
+
+    Merge pull request #13778 from smithfarm/wip-18811-jewel
+    
+    jewel: librgw: RGWLibFS::setattr fails on directories
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 874120f627f509c75b28d1be699812fce881f4bb
+Merge: b0108ea e475bfa
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:21:44 2017 -0600
+
+    Merge pull request #13717 from asheplyakov/19115-bp-jewel
+    
+    jewel: rgw_file: ensure valid_s3_object_name for directories, too
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit b0108ea3bae1662e90c7fb9a911257f45193798c
+Merge: 6518d70 db928d6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:19:03 2017 -0600
+
+    Merge pull request #13596 from dillaman/wip-19038-jewel
+    
+    jewel: rbd-mirror: deleting a snapshot during sync can result in read errors
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6518d70b89686b9a4e8854a4c6290381fbcdd8c3
+Merge: db50938 8941881
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:10:43 2017 -0600
+
+    Merge pull request #13583 from asheplyakov/jewel-bp-18901
+    
+    jewel: rgw_file:  interned RGWFileHandle objects need parent refs
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit db50938013f0b1c1fe8a23247c91bbfc38d36a8c
+Merge: 7536871 d44263f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:02:43 2017 -0600
+
+    Merge pull request #13503 from linuxbox2/wip-jewel-10156
+    
+    jewel: rgw: make sending Content-Length in 204 and 304 controllable
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 75368712aafc51b062b674b263354cca2dd4b49a
+Merge: 250071e 2f20328
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:53:49 2017 -0600
+
+    Merge pull request #13232 from ovh/wip-rework-recovery-priorities-jewel
+    
+    jewel: osd: Increase priority for inactive PGs backfill
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 250071eb8dc744237acfed01df2cf04f2d88bc85
+Merge: 6f57a77 9910eac
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:51:46 2017 -0600
+
+    Merge pull request #13219 from linuxbox2/jewel-rgw-multipart-upload-copy-4
+    
+    jewel: rgw: multipart uploads copy part support
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6f57a775301edce7a1039a4aa4479bbd8883cbf6
+Merge: 8c02e54 3fce77a
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:51:24 2017 -0600
+
+    Merge pull request #13108 from vumrao/wip-vumrao-jewel-18657
+    
+    jewel: osd: fix OSD network address in OSD heartbeat_check log message
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b24a8c2f3890b6fcc60f3b52cba93d573b9d45a2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Mar 2 17:21:57 2017 +0800
+
+    rgw_file: posix style atime,ctime,mtime
+    
+    As an ganesha FSAL backend, rgw_file should properly maintain
+    the atime,ctime,mtime properly against operations such as:
+    	(read,write) for file
+    	(create,unlink,mkdir,rmdir,rename) for dir
+    	(setattr) for file and dir
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ac25da2479b9be876cbdb820560ac46a6e2b17d7)
+
+commit b6181833be925b7eb11afffff7f03486bdde2d25
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+    
+    Fixes: http://tracker.ceph.com/issues/19018
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit ed9308cba2af227991a50a9535df30cf9bb18a82
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+    
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+    
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+    
+    Fixes: http://tracker.ceph.com/issues/18991
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 26a38e2e1525714978feda018c03698d7af65129
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+    
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+    
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+    
+    Fixes: http://tracker.ceph.com/issues/18989
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 7e4e29038bd820e171d007360cf383c85f67879b
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 8 16:23:11 2017 +0800
+
+    rgw_file: fix reversed return value of getattr
+    
+    When ::getattr returns -ESTALE, rgw_getattr returns ESTALE,
+    which is a not expected postive.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 39203cf872b8f4af86eb0e4a0f96dffd9cc92b41)
+
+commit 3c02ee4fe9ae70f30328a7b015f5cac4a1804c67
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Mon Mar 20 10:53:46 2017 +0800
+
+    rgw_file: fix non-negative return code for open operation
+    
+    The nfs-ganesha expects a negative retcode for errors.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit b5f70ef7b066f9ff44770cc8a50ccadaa02f4691)
+
+commit e9a4903c6e77caae6f161056e8aaa5ba10ae5ef2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 15 15:01:05 2017 +0800
+
+    rgw_file: fix double unref on rgw_fh for rename
+    
+    Skip unref after unlink to fix the problem.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit bff228734c73b536d2482e2e2fa4ad38b206ebff)
+
+commit 9910eac98474930369d694b236c06ffd627fee04
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 22:20:10 2016 -0700
+
+    rgw: multipart part copy, fix regression
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66fbe9384703c004c01783eb664f55895d310439)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit a3fdf0e246637ef4145b6b5ba1f4b114ec7ebc62
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:44:08 2016 -0700
+
+    rgw: minor optimization
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4919dc9987c6376d3d4e143702c26417449524c5)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2161376baf413acfbf02df07e404d2918729bfcc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:00 2016 -0700
+
+    rgw: rgw_obj_key use adjustment in multipart copy part
+    
+    This fixes a case where objects start with double underscore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 29fece3545cc1df404a25eec46706b16f893a5df)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 135f84d860d10a2961f430708983113a87ddf899
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:35 2016 -0700
+
+    rgw: multipart copy-part handle versionId
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 53521efffb1cb92e5f5ce992d4127bf9498d7c33)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 9ab65f5501fb59aac17cfdde57371b00af03d84b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:24:13 2016 -0700
+
+    rgw: multipart copy part minor parsing cleanup
+    
+    no need for range.size()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9b824d5d4017239d58b4752ebc43bfad8f698d)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3eb9dc96ff8e655415500a3595a78cab80739826
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:11:43 2016 -0700
+
+    rgw: multipart copy, check for empty bucket, improve logging
+    
+    also reduce log level for non critical user errors.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2bcb3d286b230ef917d5ba96c8276a942f544689)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2588e95d9d422d3b33fd710de1f5884873465483
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:07:14 2016 -0700
+
+    rgw: multipart copy part, chunked read
+    
+    Don't read the entire range from source object, read it in parts.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4049e47a0cfc1eef6efd502590b68ba7234589d3)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit bd118b556562176ce2390a935b3bb8c25a62d0c4
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:56:50 2016 +0200
+
+    rgw: doc: add multipart uploads copy part feature as supported
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 8a7ebeee2ff3f10ceb23b7fa43e43c3c450efe22)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit b56b719299becc38ec36d427a36b8c29f2416c08
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:53:44 2016 +0200
+
+    rgw: multipart uploads copy part support
+    
+    Add multipart uploads copy part feature.
+    
+    Fixes: http://tracker.ceph.com/issues/12790
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 949480c2e9760855ed6a0501d364d5e766c8207d)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit d44263fc91da12ea0ad4fec0cd2877b36ddb9e9f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 17 00:56:34 2017 +0100
+
+    rgw: make sending Content-Length in 204 and 304 controllable
+    
+    This commit introduces a new configurable "rgw print prohibited
+    content length" to let operator decide whether RadosGW complies
+    to RFC 7230 (a part of the HTTP specification) or violates it
+    but follows the Swift's behavior.
+    
+    Fixes: http://tracker.ceph.com/issues/16602
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d8e3e64ec97a3c222a56bb6f510e5e23d7858615)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit f3face61351a61f3b71dfb8268a4b645c4d92da2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Mar 9 15:24:08 2017 -0500
+
+    rgw: clear old zone short ids on period update
+    
+    the short ids of old, removed zones were being kept in the period to
+    guard against hash collisions with new zones
+    
+    but for a hash collision to cause a wrong object to sync, that object
+    would have to be uploaded simultaneously to two different zones that had
+    the same short id
+    
+    to avoid this, we just have to prevent the period from containing two
+    colliding zones at the same time - we don't have to remember old zone
+    short ids forever
+    
+    Fixes: http://tracker.ceph.com/issues/15618
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9c45633c836c966ab1f75ea2b1ad3fa0a4886600)
+
+commit 8c02e54a747644d24feb98dbc15a15cdd55d1afd
+Merge: 9d222b9 952f34f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 24 08:13:02 2017 -0700
+
+    Merge pull request #13146 from dzafman/wip-18502-jewel-zafman
+    
+    jewel: osd: Scrub improvements and other fixes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9d222b943217a2ec022678e50b34766d231004ce
+Merge: ce2e989 cb9fae5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Mar 23 16:49:13 2017 +0100
+
+    Merge pull request #14100 from smithfarm/wip-18781
+    
+    Manually merge PR#13143 which was mistakenly merged to jewel-next
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit cb9fae56ebdfda28b0606ee89631deb60e5f7de7
+Merge: ce2e989 ee3c67c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 23 09:46:53 2017 +0100
+
+    Merge branch 'jewel-next' into jewel
+    
+    Branch "jewel-next" was merged to jewel on Dec 22 (5b402f8a7b), and then PR
+    https://github.com/ceph/ceph/pull/13143 merged to jewel-next much later (Feb
+    1st), and then jewel-next was never merged to jewel again before we tagged
+    v10.2.6.
+    
+    This merge brings in a single commit, i.e. the one from
+    https://github.com/ceph/ceph/pull/13143
+    
+    Reported-by: Ken Dreyer <kdreyer@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ce2e9897c08775cfbe318f637d61e07eb5433df5
+Merge: 656b5b6 d0a0d2f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Mar 14 09:19:28 2017 -0600
+
+    Merge pull request #13952 from smithfarm/wip-fix-merge-error
+    
+    tests: fix merge error in rgw/singleton/all/radosgw-admin.yaml
+
+commit 0be4e89419f2083d081ac784891e4653290cd530
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 11 15:21:42 2016 -0400
+
+    rgw: hold a reference on data_sync_cr over run()
+    
+    run() will drop its reference to data_sync_cr, so we need to hold a
+    reference until we can reacquire the lock
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4cf0d2a768e7402e71280ca16b47353ca2a68505)
+
+commit 18ffdb7baf4aff1fd7f894af1054081f3ee61d28
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 11 15:19:37 2016 -0400
+
+    rgw: clear data_sync_cr if RGWDataSyncControlCR fails
+    
+    async notifications will still try to call wakeup() on RGWDataSyncControlCR
+    if it fails, leading to segfault
+    
+    Fixes: http://tracker.ceph.com/issues/17569
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5cc599b9bf2dde31de16a5b2831baf06851d69d5)
+    
+     Conflicts:
+    	src/rgw/rgw_data_sync.cc: declaration of 'int r'
+
+commit d0a0d2f4bf1dacf7c1f38b968be84a5e905554a8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Mar 14 01:26:23 2017 +0100
+
+    tests: fix merge error in rgw/singleton/all/radosgw-admin.yaml
+    
+    Introduced by 94d5888c
+    
+    Reported-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 19856624cd5ea8cb82bc5b46e062beb39674bd34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 20 16:00:01 2017 -0500
+
+    rgw: RGWMetaSyncShardControlCR retries with backoff on all error codes
+    
+    RGWBackoffControlCR only treats EBUSY and EAGAIN as 'temporary' error
+    codes, with all other errors being fatal when exit_on_error is set
+    
+    to RGWMetaSyncShardControlCR, a 'fatal' error means that no further sync
+    is possible on that shard until the gateway restarts
+    
+    this changes RGWMetaSyncShardControlCR to set exit_on_error to false, so
+    that it will continue to retry with backoff no matter what error code it
+    gets
+    
+    Fixes: http://tracker.ceph.com/issues/19019
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3e4059557fd6cad5d31014327f60832b36d04a6c)
+
+commit dfaaec0446c3db458bffe1d725518ab4f7cc2fa8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 10:21:38 2017 -0500
+
+    rgw_file:  return of RGWFileHandle::FLAG_EXACT_MATCH
+    
+    Allow callers of rgw_lookup() on objects attested in an
+    rgw_readdir() callback the ability to bypass exact match in
+    RGWLibFS::stat_leaf() case 2, but restore exact match enforcement
+    for general lookups.
+    
+    This preserves required common_prefix namespace behavior, but
+    prevents clients from eerily permitting things like "cd sara0" via
+    partial name match on "sara01."
+    
+    Fixes: http://tracker.ceph.com/issues/19059
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70ef7d45e0abf2661bd4e23161d4e70cf5178079)
+
+commit 9b3784d924112d9ba42b2088d5fb2656ef74fadc
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 17:43:17 2017 -0500
+
+    rgw_file: invalid use of RGWFileHandle::FLAG_EXACT_MATCH
+    
+    The change which introduced this flag also caused it to be
+    given as the flags argument to RGWLibFS::stat_leaf() when called
+    from rgw_lookup().
+    
+    This was incorrect:  in particular, when a directory is known only
+    as a common prefix of other objects, the AWS namespace mapping
+    convention requires lookup("foo") to match a non-materialized
+    instance of "foo/" (case 2 in RGWLibFS::stat_leaf's stat loop).
+    
+    Fixes: http://tracker.ceph.com/issues/18992
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e31e9eb980f958640150e8d7f17de1b9e5478b1e)
+
+commit 7761376c92eb38a2dd3d19f0b3d81895a4b1167c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 14:57:59 2017 -0500
+
+    rgw_file: rgw_lookup: don't ref for "/" or ".."
+    
+    These refs won't be returned by nfs-ganesha, and are sufficiently
+    magical that other consumers should be persuaded to understand
+    their specialness.
+    
+    Fixes: http://tracker.ceph.com/issues/19060
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit dea8d1ee373399a21851690a9753388b659b8ede)
+
+commit 97fed013f66a89c404bac5145080e5d556ff5c42
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 10:24:29 2017 -0500
+
+    rgw_file: avoid stranding invalid-name bucket handles in fhcache
+    
+    To avoid a string copy in the common mkdir path, handles for
+    proposed buckets currently are staged in the handle table, before
+    being rejected.  They need to be destaged, not just marked deleted
+    (because deleted objects are now assumed not to be linked, as of
+    beaeff059375b44188160dbde8a81dd4f4f8c6eb).
+    
+    This triggered an unhandled Boost assert when deleting staged
+    handles, as current safe_link mode requires first removing from
+    the FHCache.
+    
+    Fixes: http://tracker.ceph.com/issues/19036
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6cde812c92e5bba9f85fbf8486ebe69b55952370)
+
+commit d333addb71a0a92fbac5a7b922dbd69fc9e0604e
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Feb 27 13:06:59 2017 +1000
+
+    rgw: set dumpable flag after setuid post ff0e521
+    
+    ff0e521 resolved the issue for the other daemons but not for rgw since
+    it calls setuid (via civetweb) after the new code sets PR_SET_DUMPABLE.
+    Add another prctl call before wait_shutdown.
+    
+    Fixes: http://tracker.ceph.com/issues/19089
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit bc458d39630b599e0e1ca9fe25ad7455fcffdd10)
+
+commit f7ce5df4064e5538156cb44f9525d7552a0dd098
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Feb 8 15:07:43 2017 +0800
+
+    rgw: fix swift cannot disable object versioning with empty X-Versions-Location
+    
+    we should be able to disable object verioning by removing its X-Versions-Location
+    metadata header by sending an empty key value. this description can be found at
+    No.8 in http://docs.openstack.org/user-guide/cli-swift-set-object-versions.html.
+    
+    Fixes: http://tracker.ceph.com/issues/18852
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 17c5a0edd2227703cec867f0f588d4eae36dfe1c)
+
+commit 2f2032814189a4ecbf8dc01b59bebfae8ab3f524
+Author: Bartłomiej Święcki <bartlomiej.swiecki@corp.ovh.com>
+Date:   Fri Dec 2 16:54:46 2016 +0100
+
+    osd: Increase priority for inactive PGs backfill
+    
+    This change does prioritize backfill of PGs which don't
+    have min_size active copies. Such PGs would cause IO stalls
+    for clients and would increase throttlers usage.
+    
+    This change also fixes few subtlle out-of-bounds bugs.
+    
+    Signed-off-by: Bartłomiej Święcki <bartlomiej.swiecki@corp.ovh.com>
+    (cherry picked from commit 6a76adcdb1f92c136841d960aa7cd4e5b94addec)
+    
+    Conflicts:
+    	PendingReleaseNotes (removed version number, merged conflicts)
+
+commit e8041f627c6a05a59dba29819ed610ea0896f4dd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 3 13:44:45 2017 -0500
+
+    rgw_file:  fix RGWLibFS::setattr for directory objects
+    
+    Fixes:  http://tracker.ceph.com/issues/18808
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4ad5a9226852d6d564baf2e63278ed6c4c185ecb)
+
+commit e475bfaf7d3a1b0e54172083a92546560219665a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 16:02:07 2017 -0500
+
+    rgw_file: ensure valid_s3_object_name for directories, too
+    
+    The logic in RGWLibFS::mkdir() validated bucket names, but not
+    object names (though RGWLibFS::create() did so).
+    
+    The negative side effect of this was not creating illegal objects
+    (we won't), but in a) failing with -EIO and b) more importantly,
+    not removing up the proposed object from FHCache, which produced a
+    boost assert when recycled.
+    
+    Fixes: http://tracker.ceph.com/issues/19066
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit eb1cd3b30c0504385f05bf2d2dd5e2251b7efed7)
+
+commit 78c8be7a0df3d1c669f8a2a8fd7a5676d0823209
+Author: Marcus Watts <mdw@linuxbox.com>
+Date:   Thu Feb 23 02:30:52 2017 -0500
+
+    rgw/openssl fix: xenial autoconf logic problem: gcc/ld got too smart...
+    
+    On xenial, cc -o foo -lssl -lcrypto doesn't always
+    record libssl.so libcrypto.so as runtime library dependencies.
+    It is necessary to actually *use* a function from the library before
+    it gets recorded.  The ld(1) options "--as-needed" and "no-as-needed"
+    control this.  Evidently the default has changed in xenial.
+    That caused my smart "soname" detecting logic for openssl to
+    stop working.  To make it work, the test program has to
+    actually reference routines or variables inside the library.
+    
+    This is a quick fix for xenial / autoconf.  There needs to be
+    a better fix for cmake and master.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+
+commit db928d6b3f983b3a1ccc07440fcd3680700a1188
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 15:33:01 2017 -0500
+
+    rbd-mirror: retry object copy after -ENOENT error
+    
+    Fixes: http://tracker.ceph.com/issues/18990
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b4f36d5dc3f4f3cbb23f61cbb945b222248a50df)
+    
+    Conflicts:
+    	src/test/librados_test_stub/MockTestMemIoCtxImpl.h: sparse reads not supported
+    	src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc: sparse reads not supported
+
+commit a643fa80e03b41edcc720ff77b69ebaf24a23f3e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 13:09:39 2017 -0500
+
+    rbd-mirror: object copy should always reference valid snapshots
+    
+    If a remote snapshot is deleted while an image sync is in-progress,
+    associate the read request against the most recent, valid remote
+    snapshot for a given snapshot object clone.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a91efc3047963364944f8be91cee8e8f6afc49a)
+
+commit 37bbc95d56f73e4ec9c6e13ddbae199b14b104b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 11:52:00 2017 -0500
+
+    rbd-mirror: replace complex object op tuple with struct
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0c181527c0e151784a0f7c466aaa70b0772f91b1)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc: sparse reads not supported
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h: sparse reads not supported
+
+commit 8941881e8986af0b2871c15d188d94d49e95dc02
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Dec 30 23:30:16 2016 -0500
+
+    rgw_file:  interned RGWFileHandle objects need parent refs
+    
+    RGW NFS fhcache/RGWFileHandle operators assume existence of the
+    full chain of parents from any object to the its fs_root--this is
+    a consequence of the weakly-connected namespace design goal, and
+    not a defect.
+    
+    This change ensures the invariant by taking a parent ref when
+    objects are interned (when a parent ref is guaranteed).  Parent
+    refs are returned when objects are destroyed--essentially by the
+    invariant, such a ref must exist.
+    
+    The extra ref is omitted when parent->is_root(), as that node is
+    not in the LRU cache.
+    
+    Fixes: http://tracker.ceph.com/issues/18650
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e5299f3f43e633a5d8a9360893b4b11f6217d81)
+
+commit 952f34f39cdd0438e4a4fb369ea8ca20b26488a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 12 15:53:25 2016 -0800
+
+    test: Update for new error message when doing scrub with deep-scrub errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 85e0774a7dded4aa6c67f237416041e25a7680bc)
+
+commit b655b98e48989ae954dc150fcb3d8976978cd90d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 9 12:14:06 2016 -0800
+
+    osd: Add "trigger_scrub" admin socket command
+    
+    This allows testing to fake a schedule scrub (must_scrub not set)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8dc945260ee2ee841aca00fbc802d41036033d6)
+
+commit 94c958d2c9570e55305384ac86185e328746d2ff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 8 23:00:13 2016 -0800
+
+    test: Add test for keeping deep-scrub information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 64a7012e986ec88994c073b738fd08e8958395c3)
+
+commit 42eb24f18d5114410bd2a3e84e9219584d9e165e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 1 16:39:42 2016 -0800
+
+    osd: When deep-scrub errors present upgrade regular scrubs
+    
+    Previously, if a weekly deep-scrub found errors the next daily scrub
+    would cause the deep-scrub information to be removed.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 532a759dca466181f37a329f86045c34f1a2506f)
+
+commit 76a457aa7c411dc5ed22f171954b8ed3c4661845
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 19 17:10:29 2016 -0700
+
+    tasks/scrub_test.py: Make test deterministic by updating digests
+    
+    ceph_test_rados: Get coverage of digest updates in deep-scrub/repair
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e000ab2f46b7b98a223176a2a25bb7195e601af6)
+
+commit bd1f1983129d6e1787b8b5eb4884c5e908f7b274
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 19 17:10:07 2016 -0700
+
+    repair_test, scrub_test: Fix whitelists for scrub changes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fe8522df47f56842d227e08b2912623504afe24)
+
+commit 28106f06404407df7d05b35f9f570a80e785e635
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 31 12:02:31 2016 -0700
+
+    scrub_test: Fix for list-inconsistent-obj output changes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5ae0f5c75a8236d4a026f9ddcb5ff59964d90579)
+
+commit abcefc1395f76e5b05a988b970457a7f6dac6e8e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:11:44 2016 -0700
+
+    doc, test: Add schemas for list-inconsistent-* rados command output
+    
+    If jsonschema cmd is available use it to test output against schema
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eb73dd473600fbbb45fad00194f7c46b565d6b81)
+
+commit 2fa2b64676555a91883a5aabb4e4237124308629
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:22:55 2016 -0700
+
+    test: Update testing for new list-inconsistent-obj output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b7bacd219b000869b9c93e21edba4f8a3ace60d9)
+    
+    Adapted for difference in Jewel object_info
+
+commit 304f697ff1106695188b572e0da2415437040a6c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:22:29 2016 -0700
+
+    rados, osd: Improve attrs output of list-inconsistent-obj
+    
+    Persist the user_version and shard id of scrubbed obj
+    Rados command dump inconsistent obj's version and shard-id
+        so they can be passed to repair command
+    Rados list-inconsistent-obj output of attrs
+        Make attrs an array since there are more than one
+        Use base64 encode for values with non-printable chars
+        Add indication if base64 encoding used
+    Add checking for ss_attr_missing and ss_attr_corrupted
+        Rename attr errors to attr_key_mismatch and attr_value_mismatch
+    Add missing size_mismatch_oi scrub checking
+    For erasure coded pools add ec_size_error and ec_hash_error not just read_error
+    Use oi_attr_missing and oi_attr_corrupted just like list-inconsistent-snap does
+    Pick an object info based on version and use that to find specific shards in error
+        Check for object info inconsistency which should be rare
+    Make all errors based on comparing shards to each other object errors
+        We don't want give the impression that we've picked the correct one
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit df3ff6dafeadb3822b35c424a890db9a14d7f60f)
+
+commit ea52f527e530b111b1bb26d10824c0230d662c4a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 1 14:45:01 2016 -0700
+
+    osd: Fix logging to help with diagnostics
+    
+    These messages were not outputing the intended information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b39001ec6ff0996860bcb5a3578bc2c66355c781)
+
+commit 5e8f8a2288aa1a9f1de86d3f7d0f9c66c795113e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 21 18:05:25 2016 -0700
+
+    test: Fix use of wait_for_clean()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d0503a52d0fe5505bdb38dfd613c03a20500c05d)
+
+commit 2230e6adc66ca3e5dbe81aecccdb4a435e93e256
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 10:56:06 2016 -0700
+
+    common: Change cleanbin() to use base64 encoding, update ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 92e982c71995b863466d83671468f84761cb1793)
+
+commit 83ea077ee560d31a5c302a62b55451a2571fda8d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 11:05:16 2016 -0700
+
+    common: Move cleanbin() function to common/util.cc
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5c79074ffaee34b2956d9dfc67b1eff9f39b47f3)
+    
+    Conflicts:
+    	src/tools/CMakeLists.txt (changes goes in src/CMakeLists.txt)
+
+commit ba84ca9fa391d963d1d4e46fbf27ce3dbff397be
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jun 8 09:48:00 2016 -0700
+
+    test: Add test support for deep-scrub
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 907e79e2b77835d1aca9e8ba2fae5fa2fd437e5a)
+
+commit 6a421d19d0be4d59beed5c69bb0aa4477d65a14e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Jun 18 17:58:36 2016 -0700
+
+    common: Fix indentation
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a74418a29e4e469117fc0c4edd80f78b62944c98)
+
+commit c7c3e070336dc898460d4338b4d78b8963467c8a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Aug 12 12:06:31 2016 -0700
+
+    osd: Handle corrupt attributes in get_object_context()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 30f8b0d6593718dd10bcf6ff459c95b4bb68c05c)
+
+commit 8006ba7e86ebb8201a040c427cc95197901064be
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 11 21:37:50 2016 +0800
+
+    ReplicatedPG::failed_push: release read lock on failure
+    
+    and requeue the blocked ops.
+    
+    Fixes: http://tracker.ceph.com/issues/17857
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3224a18f6acc7ed54c2162b140a33b6146a16be)
+
+commit 5ca69d57dbe2ee7acc64d28ca35bb390bf463199
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 14 16:17:55 2016 -0700
+
+    test.sh: Make check for flags more robust
+    
+    Low space broke test, saw "flags nearfull,pauserd,pausewr...."
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b4c080b1b4378d659c1ca8a17811cd6f84595166)
+
+commit 1be38e5ebd2d01d5d527b05e64b026df955ffe69
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jul 21 17:36:34 2016 -0700
+
+    test: Remove extra objectstore_tool call which causes a recovery
+    
+    Caused by: 70e000a9a42c50eda48f5d3b9e432ffc3a70f75b
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6904529d1b53993304de2927500937ba0d493e9e)
+
+commit 026f181e72f2e09e769a8821dfe2c99f6213e6a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 14 20:09:15 2016 -0700
+
+    test: Handle object removals in a non-racey way
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bfe3ebd94fdc1ef36ebe6e8f9d38acea322eca22)
+
+commit c5ef385c25086dc1582ebe5343481d05283b0cc6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu May 26 22:09:42 2016 -0700
+
+    osd: Fix hang on unfound object after mark_unfound_lost is done
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 73a27533beba3587f8447b4d41d200427c45042b)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (trivial)
+
+commit 6284f745157458439699c76e8616455c50d1eb71
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 15:20:18 2016 -0700
+
+    osd: Handle recovery read errors
+    
+    Fixes: http://tracker.ceph.com/issues/13937
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c51d70e1e837c972e42ddd5fa66f7ca4477b95cc)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h (trivial)
+
+commit 27393a8c05d4656f342ecd32817307e558a2e400
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 18:19:42 2016 -0700
+
+    osd: Fix log messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b40ec3fe890679b4c73778815e1bedd492cb264b)
+
+commit cbf66f3b16f194dd0c67b46e0fec247e02e84134
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 13:58:32 2016 -0700
+
+    osd: CLEANUP: Remove unused pending_read member
+    
+    Remove unused struct
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 36fd68c96653e83f87767feb08530a9fc6e841b0)
+
+commit 228b91d72a15b62adc10591604c4e8a849df53d5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 11 18:08:12 2016 -0700
+
+    test/osd-scrub-repair.sh: Use test case specific object names to help with diagnostics
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0bf4da589155ee50969812492cfbc66368efb54c)
+
+commit ee3c67c9cbfdeeb3e628bac34b708cf150b3862e
+Merge: 3dbf0c9 39848e4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:28:22 2017 +0100
+
+    Merge pull request #13143 from linuxbox2/jewel-mdw-rgw-lf
+    
+    jewel: rgw: radosgw/swift: clean up flush / newline behavior.
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 39848e41b7c517cc5faab1ccf77c2804fd7d2628
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Jan 11 00:06:15 2017 -0500
+
+    radosgw/swift: clean up flush / newline behavior.
+    
+    The current code emits a newline after swift errors, but fails
+    to account for it when it calculates 'content-length'.  This results in
+    some clients (go github.com/ncw/swift) producing complaints about the
+    unsolicited newline such as this,
+    	Unsolicited response received on idle HTTP channel starting with "\n"; err=<nil>
+    
+    This logic eliminates the newline on flush.  This makes the content length
+    calculation correct and eliminates the stray newline.
+    
+    There was already existing separator logic in the rgw plain formatter
+    that can emit a newline at the correct point.  It had been checking
+    "len" to decide if previous data had been emitted, but that's reset to 0
+    by flush().  So, this logic adds a new per-instance variable to separately
+    track state that it emitted a previous item (and should emit a newline).
+    
+    Fixes: http://tracker.ceph.com/issues/18473
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f229d6a33eae4906f22cdb90941835e47ee9f02)
+
+commit 3fce77ab3662496368e25cbbf6d1b37d3c140db9
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Wed Jan 25 21:54:27 2017 +0530
+
+    osd: jewel fix OSD network address in OSD heartbeat_check log message
+    
+    Fixes: http://tracker.ceph.com/issues/18657
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+
+commit 14a6aabe22f68436ea3297ce0851700f86ee5b12
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Aug 3 17:36:55 2016 -0400
+
+    rgw: Handle multiple listening addreses w/ optional ssl "correctly" with civetweb.
+    
+    For civetweb: accept a range of port numbers joined with '+'.
+    Port numbers may include an ipaddress: prefix and 's' suffix.
+    Additionally, use "mg_get_local_addr" to correctly deduce host port per
+    incoming connection.
+    
+    civetweb can accept connections on multiple ports, some of which
+    might have SSL turned on and some not.  Both s3 and swift have various
+    authorization protocols in which the port number matters.  In the generic
+    radosgw frontend process, each frontend only has one port number, but
+    we should want to have both ssl and non-ssl connections managed within
+    one rgw frontend, because the thread pool is also per front-end, and
+    that *is* a scarce resource.
+    
+    So, this patch enables the use of multiple ports with a single civetweb
+    frontend.  To indicate https: append an 's' to portno.  To use multiple
+    ports, use +.  So 80+443s indicates use of the usual default http ports.
+    The parsed port is not stored in the frontend structure,
+    
+    So instead, this patch adds logic to use the results of
+    mg_get_local_addr() on a per-connection basis insetad of the generic
+    front-end port number.  This will affect "v4" s3 authorization, and also
+    affect swift pre-signed URLs.
+    
+    mg_get_local_addr() is a new customization to civetweb; that submodule
+    was updated (in a temporary repository) by the previous commit to this.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 8bc6decc0774fae9ac881f91e73da55deebe3360)
+
+commit 698250563ccc4c69e5ca5aebf65dc352d80a8bdd
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Tue Dec 20 00:22:02 2016 -0500
+
+    rgw: s3: secure_port should override port, also apply ssl default right.
+    
+    Without https, only port is set.  With https, secure_port and port are
+    both set to the same value.  The previous logic looked at port first and
+    had overly simplified conditional logic which was liable to try to apply
+    both non-default cases.  The correct behavior is: look secure_port first,
+    and if secure_port is set, then only check to see if it's a non-default
+    port.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit a113cf5ff5a642d2ee4cc83f5c7001b4bfe0a5df)
+
+commit 28f2841c8b9a832e486b9f89c574a4591bf3d448
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Nov 2 00:18:48 2016 -0400
+
+    rgw: Get civetweb ssl enhancement: wip-listen3 = mg_get_local_addr
+    
+    The logic inside of radosgw that computes aws v4 signatures wants to know
+    what server port the client connected.  The patch to civetweb patch adds a
+    call mg_get_local_addr() which will permit that code to actually find out
+    on what address a connection was received, rather than merely guessing
+    based on configuration as it previously did.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46ced9ddd2795f00f014e22e5637070b49e7a6d5)
+
+commit 8d83dfb4176ede7490d0cab589e9566bc7d4e387
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Mar 23 17:32:14 2016 -0400
+
+    rgw: Document that radosgw now supports SSL.
+    
+    This includes information on file format and configuration file syntax.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit d4e72dfed30274b3cfbad4ac58c0746a98c0148b)
+
+commit e3f80c9d0ed6762a39fc242561f5ea26f0f26546
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Jan 13 03:39:57 2017 -0500
+
+    rgw: civetweb/openssl: automagic: load libssl.so and libcrypto.so by soname.
+    
+    If building with radosgw, always look for openssl library (even when
+    building with nss).  Then, use objdump to fetch SONAME from the copies
+    of libssl and libcrypto that were found.  When building civetweb; pass
+    the library soname values in as the libraries to load with "dlopen".
+    
+    The issue reported here against master
+    http://tracker.ceph.com/issues/16535
+    reflects stuff that doesn't seem to have made it into jewel
+    	(which had: -lssl -lcrypto hard-wired into it.)
+    Still, since people were pretty riled about making that change,
+    this puts things in congruence to the final result in master.
+    	(which is: runtime shared object load of ssl crypto by soname.)
+    
+    Fixes: http://tracker.ceph.com/issues/11239
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (inspired by commit 7caa0bd002110b62514da83a37a2a3deb841267a)
+
+commit 045551485415309ba9bad77e1aee28a0008881ca
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Mon Jul 11 17:54:07 2016 -0400
+
+    rgw: civetweb/openssl: Load libssl.so and libcrypto.so by soname.
+    
+    If building with radosgw, always look for openssl library (even when
+    building with nss).  Then, use objdump to fetch SONAME from the copies
+    of libssl and libcrypto that were found.  When building civetweb; pass
+    the library soname values in as the libraries to load with "dlopen".
+    
+    This is a problem that went away for a bit, but came back with some
+    changes for
+    http://tracker.ceph.com/issues/16535
+    
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1341775
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1258961
+    
+    Fixes: http://tracker.ceph.com/issues/11239
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 7caa0bd002110b62514da83a37a2a3deb841267a)
+
+commit 386640865dee30d38f17e55fc87535e419bc3cb5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Nov 3 23:33:44 2016 -0400
+
+    rgw: cmake: remove useless civetweb include path side effect.
+    
+    For 'target_include_directories" for the cmake object library
+    'civetweb_common_objs', change from PUBLIC to PRIVATE.  This doesn't
+    break anything, so it wasn't doing anything useful.  If it has it
+    any effect, it would be to cause everything that linked against this
+    "library" to also use the indictated include path.  Which would be great
+    except everything in ceph wants to include "civetweb/civetweb.h" and
+    not "civetweb.h".  We already make separate arrangements elsewhere for
+    that to work.  Additionally, static object libraries in cmake aren't
+    really libraries, so I'm not entirely sure this even does anything.
+    So: making this public is not useful, and could be harmful.  Making it
+    private makes this only take effect for building civetweb.c itself,
+    exactly the effect we we require, and no more.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 8308a13b0257c9460fd2a721c20b0c37cb9e7c57)
diff --git a/doc/changelog/v10.2.8.txt b/doc/changelog/v10.2.8.txt
new file mode 100644
index 000000000..575a7b062
--- /dev/null
+++ b/doc/changelog/v10.2.8.txt
@@ -0,0 +1,5435 @@
+commit f5b1f1fd7c0be0506ba73502a675de9d048b744e
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Jul 6 14:56:18 2017 +0000
+
+    10.2.8
+
+commit 66dbf9beef04988dbd3653591e51afa6d84e3990
+Merge: 2f491b2e5e f46ccf2cb4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 17:43:57 2017 +0200
+
+    Merge pull request #14710 from smithfarm/wip-start-race
+    
+    tests: rados: sleep before ceph tell osd.0 flush_pg_stats after restart
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 2f491b2e5e8b1f340b28415c3bd3d9628603c377
+Merge: 552a573f84 a372b4eca1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jul 4 07:31:50 2017 -0700
+
+    Merge pull request #16089 from ceph/wip_fix_point_jewel
+    
+    qa/Fixed upgrade sequence to 10.2.0 -> 10.2.7 -> latest -x (10.2.8)
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 552a573f8426ecfec1a0df21a6c3941afd4e460c
+Merge: 53a3be7261 55eeaadfc4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jul 3 17:23:43 2017 -0700
+
+    Merge pull request #16088 from smithfarm/wip-fix-client-upgrade-centos
+    
+    tests: run upgrade/client-upgrade on latest CentOS 7.3
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit a372b4eca1f25647541943918ae737f20783db11
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Jul 3 14:18:14 2017 -0700
+
+    Fixed upgrade sequence to 10.2.0 -> 10.2.7 -> latest -x (10.2.8)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 55eeaadfc4025c83cb63c951265710868df0325f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 3 22:55:21 2017 +0200
+
+    tests: run upgrade/client-upgrade on latest CentOS 7.3
+    
+    Before this patch, all centos jobs were failing because there are no longer any
+    CentOS 7.2 machines in Sepia.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 53a3be7261cfeb12445fbdba8238eefa40ed09f5
+Merge: 84bd162978 d33b30cdb0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jun 30 16:43:38 2017 +0200
+
+    Merge pull request #15504 from Vicente-Cheng/wip-20151-jewel
+    
+    jewel: ceph-disk: do not setup_statedir on trigger
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 84bd162978e48eead40335bcbd92e4ab18e9c590
+Merge: c710689109 8e0e4a0ce7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 10:20:23 2017 +0200
+
+    Merge pull request #15904 from smithfarm/wip-20413-jewel
+    
+    jewel: tests: upgrade:hammer-x/stress-split-erasure-code-x86_64 fails in 10.2.8 integration testing
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit c7106891096c895f0cc5c2cef438078ea48de95d
+Merge: 5c6cb14806 9d3110c276
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 08:16:33 2017 +0200
+
+    Merge pull request #14930 from smithfarm/wip-19829-jewel
+    
+    jewel: tests: New upgrade test for #14930
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5c6cb1480699f5ce464e25b9cacdda770ce3660d
+Merge: bdc085d02a d43e19d886
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 08:16:04 2017 +0200
+
+    Merge pull request #14392 from asheplyakov/19508-jewel
+    
+    jewel: osd: pg_pool_t::encode(): be compatible with Hammer <= 0.94.6
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9d3110c276917055b078cd14c181b2bda2625821
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 10:32:16 2017 +0200
+
+    tests: upgrade/hammer-x/v0-94-6-mon-overload: tweak packages list
+    
+    Include some hammer dependencies that aren't in the jewel default packages
+    list, and exclude some java packages that may not be in the hammer repo and are
+    not needed for the upgrade test in any case.
+    
+    N.B.: This cannot be cherry-picked from master because upgrade/hammer-x was
+    dropped in master.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 6a64f8901bb3b218a8dc58b11d6c13033d45f067
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed May 3 11:39:27 2017 +0200
+
+    tests: upgrade/hammer-x: new v0-94-6-mon-overload subsuite
+    
+    This is not a cherry-pick from master because direct upgrades
+    from hammer to kraken+ are not supported.
+    
+    Fixes: http://tracker.ceph.com/issues/19829
+    References: http://tracker.ceph.com/issues/19508
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bdc085d02ab9723f6b90b6a3047bc51cf224b930
+Merge: e41ae4a7b1 d2d4b7202d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 27 12:41:59 2017 +0200
+
+    Merge pull request #15936 from batrick/i20412
+    
+    qa: enable quotas for pre-luminous quota tests
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d2d4b7202d77e5696eb18c4da4f7d614116ced36
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Jun 26 19:04:48 2017 -0700
+
+    qa: enable quotas for pre-luminous quota tests
+    
+    This cannot be cherry-picked from master because the config option is removed
+    since 0f250a889dba2100d3afcea0a18e4f6a8d086b86.
+    
+    Fixes: http://tracker.ceph.com/issues/20412
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit e41ae4a7b1b9cc4394473e21f6e6d6ef9cab1d59
+Merge: a21af3b7a1 682b4d717c
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jun 26 21:24:11 2017 -0500
+
+    Merge pull request #15933 from smithfarm/wip-hammer-jewel-x
+    
+    jewel: tests: drop upgrade/hammer-jewel-x
+
+commit 682b4d717c96b516c315a01b1174af3503dedba6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 27 02:27:22 2017 +0200
+
+    tests: drop upgrade/hammer-jewel-x
+    
+    This suite doesn't have any test logic in it. Its existence in the jewel branch
+    appears to be an oversight.
+    
+    This cannot be cherry-picked from master because the upgrade/hammer-jewel-x
+    suite is present (and justified) in master and is not currently being dropped
+    there.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a21af3b7a1ea5abfa4f344800e413d6249824204
+Merge: 615a6ab9b7 a744340790
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jun 26 16:25:12 2017 -0400
+
+    Merge pull request #15438 from Vicente-Cheng/wip-20027-jewel
+    
+    jewel: mds: issue new caps when sending reply to client
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 615a6ab9b723d204c6de567750fe9450742fcedb
+Merge: 9b13b48b3e 8ac0e5c363
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jun 26 16:23:48 2017 -0400
+
+    Merge pull request #15000 from jan--f/wip-19846-jewel
+    
+    jewel: cephfs: normalize file open flags internally used by cephfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9b13b48b3ed919340789a41d065eb4a9a27110de
+Merge: d217da1742 b429fa1807
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 26 18:25:03 2017 +0200
+
+    Merge pull request #15383 from asheplyakov/20014-bp-jewel
+    
+    jewel: cls/rgw: list_plain_entries() stops before bi_log entries
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d217da174252f9126d530868aa472230ecba31ca
+Merge: e520040ece 4028774122
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Jun 26 10:17:03 2017 -0600
+
+    Merge pull request #15870 from smithfarm/wip-swift-task-move-jewel
+    
+    tests: move swift.py task from teuthology to ceph, phase one (jewel)
+
+commit e520040ecec756ce181f716dad1c0bad41c77a7d
+Merge: dde8656e6b 1c0c9093ab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 26 09:24:04 2017 +0200
+
+    Merge pull request #15842 from smithfarm/wip-sortbitwise-jewel
+    
+    qa/suites/upgrade/hammer-x: set "sortbitwise" for jewel clusters
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dde8656e6b5e7ffe66a6fd695cbc17dfb18fb43e
+Merge: 498c96e66a 06cf9f3650
+Author: John Spray <jspray@redhat.com>
+Date:   Sun Jun 25 19:59:40 2017 -0400
+
+    Merge pull request #15468 from smithfarm/wip-20140-jewel
+    
+    jewel: cephfs: Journaler may execute on_safe contexts prematurely
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4028774122954023265d7825fbf9e91dc526fdee
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 12:42:36 2017 +0200
+
+    tests: swift.py: tweak imports
+    
+    The ".." form only works within the teuthology repo. With swift.py now in the
+    Ceph repo, we have to be explicit.
+    
+    Error message was: "ValueError: Attempted relative import beyond toplevel
+    package
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8e0e4a0ce7489542f47522e0a5161a5bf123c744
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 10:27:58 2017 +0200
+
+    tests: upgrade/hammer-x/stress-split: tweak packages list
+    
+    Include some hammer dependencies that aren't in the jewel default packages
+    list, and exclude some java packages that may not be in the hammer repo and are
+    not needed for the upgrade test in any case.
+    
+    N.B.: This cannot be cherry-picked from master because upgrade/hammer-x was
+    dropped in master.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a86ce728954a765797ce634025d43650d990e480
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 09:27:47 2017 +0200
+
+    tests: swift.py: clone the ceph-jewel branch
+    
+    The master branch of ceph/swift.git contains tests that are incompatible with
+    Jewel and Hammer. The ceph-jewel branch omits these tests.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 498c96e66a91edc8bd614cfc8fc5a14b3d210a76
+Merge: 38af498f9f cda721bbbf
+Author: John Spray <jspray@redhat.com>
+Date:   Fri Jun 23 08:02:48 2017 -0400
+
+    Merge pull request #15472 from smithfarm/wip-20148-jewel
+    
+    jewel: mds: Too many stat ops when trying to probe a large file
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 3d5b489369bb2cecccb1f36347654c0a37069d1c
+Merge: 38af498f9f 7b58ac97e9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:35:27 2017 +0200
+
+    Merge branch 'master' of /home/smithfarm/src/ceph/upstream/teuthology into wip-swift-task-move-jewel
+
+commit 7b58ac97e9dd195f4170e9e0ea00bae76d1f3ccd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:27:42 2017 +0200
+
+    tests: move swift.py task to qa/tasks
+    
+    In preparation for moving this task from ceph/teuthology.git into ceph/ceph.git
+    
+    The move is necessary because jewel-specific changes are needed, yet teuthology
+    does not maintain a separate branch for jewel. Also, swift.py is a
+    Ceph-specific task so it makes more sense to have it in Ceph.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 38af498f9f7b62f9f851364ae7f2691832423198
+Merge: d0ae1de51f aa0cd461df
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jun 22 22:07:23 2017 +0200
+
+    Merge pull request #15529 from badone/wip-async-sleep-timer-fix-jewel
+    
+    jewel:  osd: Implement asynchronous scrub sleep
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1c0c9093ab913a82c1dc5656a54b4009bdc35c9c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 22 11:32:42 2017 +0200
+
+    qa/suites/upgrade/hammer-x: set "sortbitwise" for jewel clusters
+    
+    Inspired by 3734280522a913ca8340ebc98b80978f63bade6f
+    
+    This cannot be cherry-picked from master because master does not have
+    qa/suites/upgrade/hammer-x
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit d0ae1de51f5faf26a2f4b0d5b7f494a4923f870d
+Merge: 64c011a8c4 de76fdbb9f
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Jun 22 11:59:51 2017 +0800
+
+    Merge pull request #15824 from tchaikov/jewel
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests the right way
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit de76fdbb9f435652e2c15326d00b01d26ab007a7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 22 08:06:43 2017 +0800
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests the right way
+    
+    --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS], so we cannot add
+    multiple exclusive patterns using -pattern:-pattern, instead, we should
+    use: -pattern:pattern
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+            qa/workunits/rados/test-upgrade-v11.0.0.sh: this change is not
+        cherry-picked from master, because the clone-range op was removed
+        from master. and only supported in pre-luminous releases.
+
+commit 64c011a8c4af27dc095b1a9190ccf1ca76d2cc8f
+Merge: e8da5e376f ab78cd040f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:54:00 2017 +0200
+
+    Merge pull request #14661 from smithfarm/wip-19575-jewel
+    
+    jewel: rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response()
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e8da5e376fc426f85fcab84a5ae71b3c17ed0068
+Merge: 5a1e849ecf 1af6781d3c
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Jun 20 22:42:47 2017 +0800
+
+    Merge pull request #15778 from tchaikov/wip-upgrade-without-clone-range-jewel
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d43e19d88692bd318f0569559867df919c26d8db
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Fri Apr 7 12:34:20 2017 +0400
+
+    jewel: osd: pg_pool_t::encode(): be compatible with Hammer <= 0.94.6
+    
+    This patch is necessary for Jewel only since direct upgrades from Hammer
+    to Kraken and newer are not supported.
+    
+    Fixes: http://tracker.ceph.com/issues/19508
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 1af6781d3c60421930087d31124e62cae530ca24
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 20 19:49:14 2017 +0800
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    The jewel-x upgrade test now runs this script against a mixed cluster on
+    a machine with code from master installed.  That means we have to
+    skip any new tests that will fail on a mixed cluster. CloneRange was
+    removed in 0d7b0b7.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+        qa/workunits/rados/test-upgrade-v11.0.0.sh: this change is not
+    cherry-picked from master, because the clone-range op was removed from
+    master. and only supported in pre-luminous releases.
+
+commit a74434079088129244b7aae6ccc6df7094282eba
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 22 12:27:12 2017 +0800
+
+    mds: issue new caps when sending reply to client
+    
+    After Locker::issue_new_caps() adds new Capability data struct,
+    do not issue caps immediately. Let CInode::encode_inodestate()
+    do the job instead. This can avoid various races that early reply
+    is not allowed, caps that haven't been sent to client gets revoked.
+    
+    Fixes: http://tracker.ceph.com/issues/19635
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 799703a4acb49db0b6cc99a23e4326767e694c3a)
+
+commit d33b30cdb019937ff88f9724599f52f4e00d37cf
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 1 11:37:20 2017 +0200
+
+    ceph-disk: do not setup_statedir on trigger
+    
+    trigger may run when statedir is unavailable and does not use it.
+    
+    Fixes: http://tracker.ceph.com/issues/19941
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 16bfbdd3d9988523bba31aace516c303057daa58)
+
+commit f46ccf2cb4701cd93cd9b15a4e57b5b97798b947
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 21 11:05:05 2017 +0200
+
+    tests: rados: sleep before ceph tell osd.0 flush_pg_stats after restart
+    
+    Even though we wait for HEALTH_OK after restarting the daemons, they are not
+    ready to respond to flush_pg_stats.
+    
+    The reason why the osd is not ready for "tell" command after "ceph health"
+    shows that the cluster is "HEALTH_OK" is that the monitor fails to be notified
+    that the osd in question is not up in "heatbeat_interval". Because infernalis
+    does not have the osd_fast_fail_on_connection_refused support, the monitor
+    needs longer to detect that an osd is down, and osd_heartbeat_grace is used to
+    determine if an osd is down.
+    
+    References: http://tracker.ceph.com/issues/16239
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit ab78cd040f6d3946ed40b6638ebcf52969a7cbb6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 8 14:52:34 2017 -0800
+
+    rgw: fix crash when listing objects via swift
+    
+    Fixes: http://tracker.ceph.com/issues/19249
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9ec5e8ce184e19c009863db4d3519f9d8af91bd)
+    
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc ("key" element of RGWObjEnt struct
+               is not a reference; fix)
+    
+    (cherry picked from commit 92b35155ff7b7492f3c50bf4f2ff0ffef2ca1c55)
+
+commit 5a1e849ecf215d82e31b9bdd0970cb04200de2c9
+Merge: 2469085d57 66c3db7aee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 19 22:46:47 2017 +0200
+
+    Merge pull request #14752 from cbodley/wip-19474
+    
+    jewel: rgw: allow system users to read SLO parts
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cda721bbbfae00ec4244718ae20cbd9ae914c630
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 19 09:37:15 2017 +0800
+
+    client: update the 'approaching max_size' code
+    
+    The old 'approaching max_size' code expects MDS set max_size to
+    '2 x reported_size'. This is no longer true. The new code reports
+    file size when half of previous max_size increment has been used.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 9316b0442c6f828dcf8da952e4c7a63c4db1398d)
+    
+    Conflicts:
+            src/client/Client.cc - in jewel, second argument to check_caps() is
+                a bool (see 0df562a8e13 which is not in jewel)
+
+commit 439f39128ec278ce78139d4b96ed098c68efa3f5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 17 19:08:37 2017 +0800
+
+    mds: limit client writable range increment
+    
+    For very large file, setting the writable range to '2 * file_size'
+    causes file recovery to run a long time. To recover a 1T file, Filer
+    needs to probe 2T~1T range.
+    
+    Fixes: http://tracker.ceph.com/issues/19955
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 538f35bef944b18e9bca2b15ed7f4e8807ef0554)
+    
+    Conflicts:
+            src/mds/Locker.h - in jewel, file_update_finish() has different
+                arguments than it does in master
+
+commit 06cf9f365033f7913051bdf4060f0bc6fc0444d7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue May 23 21:46:54 2017 +0800
+
+    osdc/Journaler: avoid executing on_safe contexts prematurely
+    
+    Journaler::_do_flush() can skip flushing some data when prezered
+    journal space isn't enough. Before updating Journaler::next_safe_pos,
+    we need to check if Journaler::_do_flush() has flushed enough data.
+    
+    Fixes: http://tracker.ceph.com/issues/20055
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 6511e7a9e35a14216c03cd6921ca4d232274f953)
+
+commit 2e299b50de4a297fee2aec21290632336d239857
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 12 16:00:18 2017 +0800
+
+    osdc/Journaler: make header write_pos align to boundary of flushed entry
+    
+    This can speed up the process that detects and drops partial written
+    entry in the log tail.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8ae2962b79903e217fda83cea4140af64b5d6883)
+    
+    Conflicts:
+            src/osdc/Journaler.cc - 8d4f6b92cba is not being backported to jewel
+            src/osdc/Journaler.h - Journaler::Journaler initializer list is different in jewel, compared to master
+
+commit 2469085d57a05933589165f6f99a67b2e28c7022
+Merge: 42c3fbc129 d57437e338
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 10:01:27 2017 -0400
+
+    Merge pull request #14672 from smithfarm/wip-19334-jewel
+    
+    jewel: MDS heartbeat timeout during rejoin, when working with large amount of caps/inodes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8ac0e5c363bd6439071d26874b6714cf2376736f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 12 10:38:51 2017 +0800
+
+    pybind: fix cephfs.OSError initialization
+    
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "cephfs.pyx", line 672, in cephfs.LibCephFS.open (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:10160)
+      File "cephfs.pyx", line 155, in cephfs.OSError.__init__ (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:1889)
+    TypeError: __init__() takes exactly 3 positional arguments (2 given)
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e6493f64ba4592b8dca54ece4464efa6c7f331a7)
+
+commit 09b9410c2e69a466b001d92fc14eb44d768009f1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 10 08:13:52 2017 +0800
+
+    pybind: fix open flags calculation
+    
+    (O_WRONLY | O_RDWR) is invaild open flags
+    
+    Fixes: http://tracker.ceph.com/issues/19890
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 2c25c99cb4572ffae97555a56b24a4c4097dcdec)
+
+commit 42c3fbc129cbb60d447c1a6b0402a9def1656446
+Merge: 71d45e1905 7347f11939
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:43:05 2017 -0400
+
+    Merge pull request #14677 from smithfarm/wip-19665-jewel
+    
+    jewel: mds: C_MDSInternalNoop::complete doesn't free itself
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 71d45e190528124a8ff7e4674f7bfb7c340f80ee
+Merge: 27c915f5d2 e6daee8a9f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:35:08 2017 -0400
+
+    Merge pull request #15466 from smithfarm/wip-19762-jewel
+    
+    jewel: cephfs: non-local quota changes not visible until some IO is done
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 27c915f5d21bd84502e35eb269d955fafc47de0b
+Merge: a76357622a db053da618
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:32:47 2017 -0400
+
+    Merge pull request #14700 from smithfarm/wip-19709-jewel
+    
+    jewel: mds: enable start when session ino info is corrupt
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a76357622a2773850153aa1f6ea02b1737942c4b
+Merge: 85aab833f7 db86a24e79
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:31:22 2017 -0400
+
+    Merge pull request #14685 from smithfarm/wip-19675-jewel
+    
+    jewel: cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 85aab833f7f66ac81d8b12f4203fa215787f8d2a
+Merge: f6b395115e 7b9283beec
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:30:53 2017 -0400
+
+    Merge pull request #14684 from smithfarm/wip-19673-jewel
+    
+    jewel: cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit f6b395115e4de15d73269ff6b96f2ee0fd0ea9c3
+Merge: ff60fceb5f b52c508861
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:30:13 2017 -0400
+
+    Merge pull request #14683 from smithfarm/wip-19671-jewel
+    
+    jewel: cephfs: MDS assert failed when shutting down
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ff60fceb5f5f7f9df4a48da4ad8bd863b9a04e2d
+Merge: 6a6d57d2de 96e801fb53
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:29:50 2017 -0400
+
+    Merge pull request #14682 from smithfarm/wip-19668-jewel
+    
+    jewel: cephfs: MDS goes readonly writing backtrace for a file whose data pool has been removed
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 6a6d57d2de674c5a971e999cb2731b2d8ae1b523
+Merge: 8260669efa f34489dd52
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:27:13 2017 -0400
+
+    Merge pull request #14679 from smithfarm/wip-19666-jewel
+    
+    jewel: cephfs: The mount point break off when mds switch hanppened.
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8260669efa74f41d3b1b1039fb7b34e070951c34
+Merge: 013529b61f 824b19a9a6
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:26:40 2017 -0400
+
+    Merge pull request #14676 from smithfarm/wip-19619-jewel
+    
+    jewel: cephfs: MDS server crashes due to inconsistent metadata.
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 013529b61fc4fbf02656f7c6cb0baa1bc6004758
+Merge: b518522f64 eab56dae67
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:24:38 2017 -0400
+
+    Merge pull request #14674 from smithfarm/wip-19482-jewel
+    
+    jewel: cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b518522f64b66f4a14618a9345b6314ca0f2c54c
+Merge: 388e0d1bc3 63f41d543f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:23:27 2017 -0400
+
+    Merge pull request #14671 from smithfarm/wip-19044-jewel
+    
+    jewel: tests: buffer overflow in test LibCephFS.DirLs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 388e0d1bc35985a6916d3eb1ca5184a0907b6e8b
+Merge: d7c7ce7ebb 7146816065
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:23:13 2017 -0400
+
+    Merge pull request #14670 from smithfarm/wip-18949-jewel
+    
+    jewel: mds: avoid reusing deleted inode in StrayManager::_purge_stray_logged
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d7c7ce7ebbb663dab0dfa8058c845d494d7615cc
+Merge: d717ef73bc d8b139b584
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:22:28 2017 -0400
+
+    Merge pull request #14669 from smithfarm/wip-18900-jewel
+    
+    jewel: cephfs: Test failure: test_open_inode
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d717ef73bc0d5ef24551ec9157385c8b0521380b
+Merge: c2a3b7567f 36c86f71ef
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:21:43 2017 -0400
+
+    Merge pull request #14668 from smithfarm/wip-18705-jewel
+    
+    jewel: mds: fragment space check can cause replayed request fail
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c2a3b7567fbe0b2f62bcd38cd9bb9a5a2a238743
+Merge: fd9256b770 5b56214519
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jun 13 19:05:55 2017 +0100
+
+    Merge pull request #14698 from smithfarm/wip-19677-jewel
+    
+    jewel: cephfs: ceph-fuse does not recover after lost connection to MDS
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit fd9256b77010066c934fd0016eb6d3f9c1fb54e1
+Merge: 26ada59c81 c49b114e8d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:39:17 2017 +0200
+
+    Merge pull request #14766 from smithfarm/wip-19757-jewel
+    
+    jewel: rgw: fix failed to create bucket if a non-master zonegroup has a single zone
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 26ada59c810bfda5c16f2d935a2b98711e7b2c76
+Merge: fdd25c2bdb e552d91f73
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:38:46 2017 +0200
+
+    Merge pull request #14787 from linuxbox2/jewel-rgw-shard-limit-ck
+    
+    jewel: rgw: add bucket size limit check to radosgw-admin
+    
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fdd25c2bdb9ec6bb8e3061088b8782bfb2331bc4
+Merge: cfd6750416 fb3ee2efcc
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:38:23 2017 +0200
+
+    Merge pull request #14789 from mdw-at-linuxbox/wip-jewel-rgw-rvk
+    
+    jewel: rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+    
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cfd6750416baeb2bbd3836bc7dc55d83fc214c20
+Merge: ce1fc3492e 86980a045b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:36:02 2017 +0200
+
+    Merge pull request #14815 from smithfarm/wip-19786-jewel
+    
+    jewel: rgw: failure to create s3 type subuser from admin rest api
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ce1fc3492e87c669f7059c2047a3bed077418a89
+Merge: 7ca0252560 aa99558934
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 7 20:13:15 2017 +0200
+
+    Merge pull request #15312 from theanalyst/wip-20078
+    
+    jewel: rgw: only append zonegroups to rest params if not empty
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7ca0252560d1ff23384afb50a9c2ae2aad2ce85c
+Merge: 62c500f522 59bd6711a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 7 13:48:06 2017 +0200
+
+    Merge pull request #15382 from theanalyst/wip-mem-leak2
+    
+    jewel: rgw:fix memory leaks in data/md sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit aa0cd461df5fee6d143bc07440ec6de829761cef
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 22 13:21:25 2017 +1000
+
+    osd: Move scrub sleep timer to osdservice
+    
+    PR 14886 erroneously creates a scrub sleep timer for every pg resulting
+    in a proliferation of threads. Move the timer to the osd service so
+    there can be only one.
+    
+    Fixes: http://tracker.ceph.com/issues/19986
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit f110a82437df79dc20207d296e8229fc0e9ce18b)
+    
+    Conflicts:
+            src/osd/PG.cc - ceph_clock_now requires a CephContext argmunent
+            in Jewel
+
+commit c47bd0562b1187ffb0b1b2c1ef5f105aa7951d10
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 24 14:10:47 2017 +1000
+
+    osd: Implement asynchronous scrub sleep
+    
+    Rather than blocking the main op queue just do an async sleep.
+    
+    Fixes: http://tracker.ceph.com/issues/19497
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7af3e86c2e4992db35637864b83832535c94d0e6)
+
+commit e6daee8a9fbc576da2a03550a81056d093a516c9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 4 20:39:58 2017 +0200
+
+    Client.cc: adjust Client::_getattr calls
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a2c7a2262ac8ecbea78f09e6e8e6a37498568d57
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:36:08 2017 +0000
+
+    qa/cephfs: use getfattr/setfattr helpers
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dd43d3bc646aeab88486b0963fc83de0b18800c4)
+    
+    Conflicts:
+            qa/tasks/cephfs/test_data_scan.py: difference in the
+                self._mount.run_shell() call in NonDefaultLayout::write (which is
+                being dropped by this commit) - in jewel it has "sudo", and in
+                master it doesn't
+
+commit 12aa35a6d50f612df77199ac4f35c7baeed0583e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 22 13:00:44 2016 +0100
+
+    tasks/cephfs: fix race while mounting
+    
+    This could fail if the mount hadn't finished
+    coming up.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit adfb757c898a80f18c15dafd02e29840c5931c87)
+
+commit a7b699269b65c76361fcb1d10593812be40c7612
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:26:30 2017 +0000
+
+    qa: add test for reading quotas from different clients
+    
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61617f8f10a6322603a9add77980865cd972ef97)
+
+commit 8b8ee392b8093b9b140dbbe895691f69ae40440f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 17:51:44 2017 +0000
+
+    client: _getattr on quota_root before using in statfs
+    
+    ...so that after someone adjusts the quota settings
+    on an inode that another client is using as its mount root,
+    the change is visible immediately on the other client.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3d25941aadd223669448d0f5d3c0bd1fefa72308)
+
+commit dd7d59a08141d6a24b172c22f5e27c8962e25fb9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 15:32:47 2017 +0000
+
+    client: getattr before read on ceph.* xattrs
+    
+    Previously we were returning values for quota, layout
+    xattrs without any kind of update -- the user just got
+    whatever happened to be in our cache.
+    
+    Clearly this extra round trip has a cost, but reads of
+    these xattrs are fairly rare, happening on admin
+    intervention rather than in normal operation.
+    
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 532dc4b68e538c189ef828f67cecd0d647a62250)
+
+commit 62c500f52240eaa5faadd3795bd9ec84bdcbc6c7
+Merge: 2badc2416c 8dd93cabd5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 2 09:54:22 2017 -0700
+
+    Merge pull request #15416 from dzafman/wip-20126
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8dd93cabd52cbafc29a47862f343431eb6f1cfe3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 31 15:39:19 2017 -0700
+
+    osd: Object level shard errors are tracked and used if no auth available
+    
+    Shards with object mismatch are tracked to mark them inconsistent
+    Fix test because storing omap_digest in object_info not behaving as before
+    
+    Fixes: http://tracker.ceph.com/issues/20089
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    
+    (cherry picked from commit 1cacbea763c7aabfeaaf4bd5e878301044184117)
+    
+    Conflicts:
+            src/test/osd/osd-scrub-repair.sh (no alloc_hint in object_info)
+
+commit 59bd6711a47c354117a612dd4ef033d70d449383
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Jun 1 17:20:49 2016 +0800
+
+    rgw:fix memory leaks
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 73e5be2b6133cf4caa0e5e5c8c9eae748b785dbf)
+
+commit b429fa1807062716c9705ddcf316ed9b2741cc43
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 5 14:56:40 2017 -0400
+
+    cls/rgw: list_plain_entries() stops before bi_log entries
+    
+    list_plain_entries() was using encode_obj_versioned_data_key() to set
+    its end_key, which gives a prefix of BI_BUCKET_OBJ_INSTANCE_INDEX[=2]
+    
+    that range between start_key and end_key would not only span the
+    BI_BUCKET_OBJS_INDEX[=0] prefixes, but BI_BUCKET_LOG_INDEX[=1] prefixes
+    as well. this can result in list_plain_entries() trying and failing to
+    decode a rgw_bi_log_entry as a rgw_bucket_dir_entry
+    
+    Fixes: http://tracker.ceph.com/issues/19876
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b29a1633a57abf443d5790c13d680d2917f86037)
+
+commit 2badc2416c9e16babbc91364502dcb03877e88bf
+Merge: 5d2a68eb90 1f895c2403
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue May 30 09:39:47 2017 -0500
+
+    Merge pull request #15360 from liewegas/wip-jewel-master-mixed
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1f895c24030b79ae9f2eae16b09582fc72928b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 30 09:58:09 2017 -0400
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    The jewel-x upgrade test now runs this script against a mixed cluster on
+    a machine with code from master installed.  That means we have to skip
+    any new tests that will fail on a mixed cluster.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5d2a68eb903f533689d1a1cbc410ca940baeaff3
+Merge: 998d1ee4f5 81e35b9416
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat May 27 09:21:44 2017 +0200
+
+    Merge pull request #15208 from liewegas/wip-sortbitwise-jewel
+    
+    mon: fix 'sortbitwise' warning on jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 998d1ee4f503cb4875283fe8d2140f3b10a26643
+Merge: 54bc1e13ea 99c65bbc18
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat May 27 02:46:55 2017 +0200
+
+    Merge pull request #14851 from yehudasa/wip-rgw-support-ragweed-jewel
+    
+    jewel: rgw: add apis to support ragweed suite
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit aa99558934c6143cb35e722148b87f35ccecc397
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 23 10:22:42 2016 -0700
+
+    rgw: rest conn functions cleanup, only append zonegroup if not empty
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4e41af19846db75081cb0ddb7b33dc2bb9321ace)
+
+commit 5d90798432a6446636699fc03b2f548010b1212f
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Thu Mar 17 10:32:14 2016 +0100
+
+    rgw: rest and http client code to use param vectors
+    
+    Replaces param/header lists with vectors. In these cases, we're only ever
+    adding to the back of the list, so a vector should be more efficient.
+    Also moves param_pair_t/param_vec_t higher up the include chain for
+    cleaner function signatures.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit d4a2527872e0f5c3ae2874bb7d0ff459ae83cfd4)
+    
+    Conflicts:
+            src/rgw/rgw_http_client.cc
+    trivial ws conflict on rebase
+
+commit 54bc1e13ea19642c8d5893f29e96ee5c053f9c59
+Merge: 966f222917 43327f83ef
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu May 25 12:45:30 2017 -0400
+
+    Merge pull request #14765 from smithfarm/wip-18972-jewel
+    
+    jewel: ceph-disk does not support cluster names different than 'ceph'
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 81e35b941659a0f6dff8a935c27c7dd6d5cc4213
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 24 09:48:11 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr/*: set sortbitwise after upgrade
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f2814e4dbb2821a415310559fce405c2eab23947
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Tue Dec 27 10:54:45 2016 +0000
+
+    mon/OSDMonitor: fixup sortbitwise flag warning
+    
+    "ceph -s" does not report warning when using
+    command "ceph osd unset sortbitwise" to drop
+    sortbitwise flag.
+    we should use "osdmap.get_up_osd_features() &
+    CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT"
+    instead of "(osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+    CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT)",
+    because osdmap.get_features only get local "features"
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit c25ee187e28724846d0011cd8145e16956d3636e)
+
+commit 02617188688eebde759c375a2257e076e4538491
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 1 12:09:57 2016 -0400
+
+    mon: remove config option to disable no sortbitwise warning
+    
+    We'll require this soon.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 04e3319e8aecde9ca58ccb7c89016f9079c7d657)
+
+commit 966f2229178c5d72722982d0f6a40f4d01210d9a
+Merge: 6537fc741c 4ceaa7cce9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 17 09:29:21 2017 +0200
+
+    Merge pull request #13450 from dreamhost/wip-18887-jewel
+    
+    jewel: msg: IPv6 Heartbeat packets are not marked with DSCP QoS - simple messenger
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb79663490468b4ac2832aa9e3cbac019c1f712d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Apr 20 18:38:43 2017 +0200
+
+    fs: normalize file open flags internally used by cephfs
+    
+    The file open flags (O_foo) are platform specific. Normalize these flags
+    before they are send to the MDS. For processing of client messages the
+    MDS should only compare to these normalized flags.
+    Otherwise this can lead to bogus flags getting transmitted on ppc64.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 88d2da5e93198e69435e288ce00d216d5fe27f80)
+    
+    Conflicts:
+            src/client/Client.cc
+              Conflicts can be resolved by choosing changes from HEAD and
+              adding a call to ceph_flags_sys2wire where flags are logged.
+            src/mds/Server.cc
+              Conflicts can be resolved by choosing changes from HEAD and
+              while making sure that the MDS compares request O_ flags the
+              the CEPH_O_ flags, since all wire O_ flags are normalized.
+
+commit 4ceaa7cce9c9132d47564d79204b48b1d02e531c
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Wed May 3 22:31:40 2017 -0700
+
+    msg/simple/Pipe: manual backport of fix in PR#14795
+    
+    Manual backport of errno fixup from PR#14795
+    (6f1037e22c2a304795895498cdc955e0ef80f8e3), as noted by
+    https://github.com/ceph/ceph/pull/13450#discussion_r114696885.
+    
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 6537fc741c3267472e607e581c002854d8eafd46
+Merge: af31b453f4 82ea0971b3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 13:04:27 2017 +0200
+
+    Merge pull request #14667 from smithfarm/wip-18699-jewel
+    
+    jewel: client: fix the cross-quota rename boundary check conditions
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit af31b453f404cb73ee7bdb8b6b02d71ad8aab7e7
+Merge: 472034cbe0 335258f975
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:33:24 2017 +0200
+
+    Merge pull request #14332 from shinobu-x/wip-19396-jewel
+    
+     jewel: Objecter::epoch_barrier isn't respected in _op_submit()
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 472034cbe0735131f3419205e9f7e6530377bfba
+Merge: afe98ba074 acf608a903
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:32:37 2017 +0200
+
+    Merge pull request #14204 from dzafman/wip-18533-jewel
+    
+    jewel: core: two instances of omap_digest mismatch
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit afe98ba07436ea105f4c9c42a52795ce3757419e
+Merge: 0353a91bfd 043d70461c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:31:38 2017 +0200
+
+    Merge pull request #13884 from shinobu-x/wip-19119-jewel
+    
+    jewel: pre-jewel "osd rm" incrementals are misinterpreted
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0353a91bfd799077ddd87de901e8a07f5da6a13d
+Merge: 460b12c259 608785a007
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:30:32 2017 +0200
+
+    Merge pull request #13647 from mslovy/wip-19083-jewel
+    
+    jewel: osd: preserve allocation hint attribute during recovery
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 460b12c259f5563d9d1b2477149fe79486ba5bcd
+Merge: 630cfca36c 905c4acb99
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 28 10:24:35 2017 +0200
+
+    Merge pull request #14791 from smithfarm/wip-19774-jewel
+    
+    jewel: osd: promote throttle parameters are reversed
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 630cfca36c5a42a58882966c5598752aac9ff54e
+Merge: 013e781b45 3ec1a9bf16
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 28 10:23:45 2017 +0200
+
+    Merge pull request #14763 from smithfarm/wip-19562-jewel
+    
+    jewel: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 99c65bbc1899663d0b23e2cec3d6e516d0e95ad7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 9 13:04:43 2017 -0800
+
+    rgw: new rest api to retrieve object layout
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2768583dc486109e49d209243675b99fdd39e92c)
+
+commit 33745a342433716ad8a07ef846dbea9b57de5017
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 5 13:47:24 2017 -0800
+
+    rgw: rest api to read zone config params
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a220a2efbfd675e6abf14ae33c21005bcbf6dadf)
+
+commit 013e781b45e28af15da4ddc32025aad713f128e8
+Merge: d144d99f00 b698d1fa4c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:46:41 2017 +0200
+
+    Merge pull request #14481 from dillaman/wip-19468-jewel
+    
+    jewel: librbd: is_exclusive_lock_owner API should ping OSD
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d144d99f008776dc02c5838feb5a9eb9b84a400b
+Merge: 0f0cd82eda dbe90c79b8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:05:11 2017 +0200
+
+    Merge pull request #14666 from smithfarm/wip-19612-jewel
+    
+    jewel: librbd: Issues with C API image metadata retrieval functions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0f0cd82edabcb2feb0b29793e5b45eb424f1c42d
+Merge: eba821ce9c 216156b5d4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:04:27 2017 +0200
+
+    Merge pull request #14664 from smithfarm/wip-19325-jewel
+    
+    jewel: rbd: [api] temporarily restrict (rbd_)mirror_peer_add from adding multiple peers
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit eba821ce9c61bcb197895e6949ce30a5c07097fa
+Merge: 25e29c9215 b8fd297eb6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 20:46:08 2017 +0200
+
+    Merge pull request #14694 from ceph/wip-bp-systemd
+    
+    [backport] qa/tasks: systemd test backport to jewel
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 25e29c921557883da4bf48ff2496bcec6448e6ec
+Merge: d7b63e21ca 7c6c3c753c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 08:44:26 2017 +0200
+
+    Merge pull request #13544 from shinobu-x/wip-18932-jewel
+    
+    jewel: tests: 'ceph auth import -i' overwrites caps, should alert user before overwrite
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit acf608a9034e915e38ccea6002ee808c46620433
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 20 17:28:45 2017 -0700
+
+    filestore, tools: Fix logging of DBObjectMap check() repairs
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1704f62c0831e6b07138f7dd14a89fef3c9ed2c1)
+
+commit fecc52338b2a58bf1730f7b7a3e4a293e45160d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 3 15:04:02 2017 -0800
+
+    osd: Simplify DBObjectMap by no longer creating complete tables
+    
+    Bump the version for new maps to 3
+    Make clone less efficient but simpler
+    Add rename operation (use instead of clone/unlink)
+    For now keep code that understands version 2 maps
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 738156a99ed1caf61e5a8230eb8048360056c08e)
+    
+    No ghobject_t::operator>() so use Kraken cmp_bitwise() instead
+    Need to use MIN_GHOBJ/MAX_GHOBJ instead of std::min/std::max
+
+commit 6902c3141eeaefaacd92f33877cf319872f626c7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 16:17:32 2017 -0800
+
+    ceph-osdomap-tool: Fix seg fault with large amount of check error output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1dda0411f4fbb14ce1e0062da9f14ec3af505d39)
+
+commit 4a3e4bcf40cd004fc53e7be467a29084dedc3e1c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 15:02:33 2017 -0800
+
+    osd: Add automatic repair for DBObjectMap bug
+    
+    Add repair command to ceph-osdomap-tool too
+    
+    Under some situations the previous rm_keys() code would
+    generated a corrupt complete table.  There is no way to
+    figure out what the table should look like now.  By removing
+    the entries we fix the corruption and aren't much worse off
+    because the corruption caused some deleted keys to re-appear.
+    
+    This doesn't breaking the parent/child relationship during
+    repair because some of the keys may still be contained
+    in the parent.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4cd3c74c928a32e065ed9543d6c91d8718a6ae3d)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit d4f0ac0a405266f638b25ec475c0110741e3c431
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 14:59:40 2017 -0800
+
+    ceph-osdomap-tool: Fix tool exit status
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 666f14ed90655a2d1bedde8561949625db7a9e6c)
+
+commit 5f36c319cd7fa4be15efd27b8aabbebc99d8999c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:51:42 2017 -0800
+
+    DBObjectMap: rewrite rm_keys and merge_new_complete
+    
+    Leverage the updated in_complete_region and needs_parent to simplify
+    these methods.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c4dffb68eaafe724f7fdae93a4285a7f8003ea29)
+
+commit 1fe4b856a37b29c85a9317e514c7f15f8e4905d5
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:50:57 2017 -0800
+
+    DBObjectMap: strengthen in_complete_region post condition
+    
+    Previously, in_complete_region didn't guarantee anything about
+    where it left complete_iter pointing.  It will be handy for
+    complete_iter to be pointing at the lowest interval which ends
+    after to_test.  Make it so.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 97b35f4d7d4862da4b6f50ecaef0d292a671fd04)
+
+commit 85f2151fec991e5db13d8e6f44b27e092605fb35
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:48:57 2017 -0800
+
+    DBObjectMap: fix next_parent()
+    
+    The previous implementation assumed that
+    lower_bound(parent_iter->key()) always leaves the iterator
+    on_parent().  There isn't any guarantee, however, that that
+    key isn't present on the child as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 74a7631d0938d7b44894f022224eab10a90d5cec)
+
+commit 484ccda5c53a8ee151b9f97687edd160ad7ebbd7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Feb 9 10:47:59 2017 -0800
+
+    test_object_map: add tests to trigger some bugs related to 18533
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f131dbcf5bb17107c029f942a57e9bf4432a26ee)
+
+commit cdeb690869bf6dd52226476c21514cf03ff37d6e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 12:40:33 2017 -0800
+
+    test: Add ceph_test_object_map to make check tests
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0e97a01bd7291458881ee53cece2d887f6333669)
+
+commit cf5d588d39498c8d65bc64bd2935aecac8546e40
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:56:27 2017 -0800
+
+    ceph-osdomap-tool: Add --debug and only show internal logging if enabled
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fb2b2d13953979e5da9f571ab8c4b0b510b8368)
+
+commit 4c4a06ff525d9fa2271099db73701c7994054d36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:55:48 2017 -0800
+
+    osd: DBOjectMap::check: Dump complete mapping when inconsistency found
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fcf1e17c645e8fad5216c3e59627c817e5c858c7)
+
+commit 6c128ff8c6944e57059008959f49bd03635c5417
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 15:38:51 2017 -0800
+
+    test_object_map: Use ASSERT_EQ() for check() so failure doesn't stop testing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 053a273cbc02d6902a4bb1f11db1ea946498df3a)
+
+commit aa769a9d08ac7490f5063bb44c3a44c77a5d7232
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 10:02:40 2017 -0800
+
+    tools: Check for overlaps in internal "complete" table for DBObjectMap
+    
+    Changed check to return an error count and fix tool error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5e8eb962db6187ea19b96ba29ac83469c90b4ea)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 761ee7c6af8802ab6b668a0b7ccaa819b2764456
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 09:40:49 2017 -0800
+
+    tools: Add dump-headers command to ceph-osdomap-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f4101591ad701a62fe027c4744ca8ea505f44bdc)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 117db1c6d6952d181614f4fe22f0b1866eed6f10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:09:42 2017 -0800
+
+    tools: Add --oid option to ceph-osdomap-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2d94889e9ee3359017b1efd560f3557ce03ccee6)
+
+commit 4d8120d3b3975ce101f6272240e5e43bcfe0e742
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:31:18 2017 -0800
+
+    osd: Remove unnecessary assert and assignment in DBObjectMap
+    
+    Fix and add comment(s)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 937e6a03ea4692cc44d53faa0615f8e808c9eb03)
+
+commit 86980a045b7176428062a19758d145e38e515b40
+Author: snakeAngel2015 <li.jieA@h3c.com>
+Date:   Mon Jul 18 14:51:37 2016 +0800
+
+    rgw: add suport for creating S3 type subuser of admin rest api
+    
+    Fixes: http://tracker.ceph.com/issues/16682
+    
+    The original code cannot support create s3 type subuser of admin rest api as when i execute the following command:
+    
+      ./s3curl.pl --id=personal --put -- http://radosgw.h3c.com:8000/admin/user?subuser\&uid=yrf2\&subuser=yrf2:yrf1\&key-type=s3 -v
+    
+    it would return msg as follows :
+    
+             < HTTP/1.1 403 Forbidden
+            < Date: Thu, 14 Jul 2016 07:04:40 GMT
+            * Server Apache/2.4.7 (Ubuntu) is not blacklisted
+            < Server: Apache/2.4.7 (Ubuntu)
+            < x-amz-request-id: tx00000000000000006608f-0057873988-8551-slave
+            < Accept-Ranges: bytes
+            < Content-Length: 114
+            < Content-Type: application/json
+            <
+            * Connection #0 to host slave.com left intact
+            {"Code":"InvalidAccessKeyId","RequestId":"tx00000000000000006608f-0057873988-8551-slave","HostId":"8551-slave-us"}
+    
+     But i have modified the codes for support it ,and it will return actual msg as follows :
+    
+      "subusers": [
+            {
+                "id": "yrf2:yrf1",
+                "permissions": "<none>"
+            }
+        ],
+        "keys": [
+            {
+                "user": "yrf2",
+                "access_key": "B46PXYFEWUX0IMHGKP8C",
+                "secret_key": "2JYxywXizqwiiMd74UXrJdSJMPNlBtYwF5z8rNvh"
+            },
+            {
+                "user": "yrf2:yrf1",
+                "access_key": "INO55WXJ7JQ1ZZGSAB6B",
+                "secret_key": "GgCKEfF9hArV2hglunbO7KtvKZnbhmsDpqjSj5DL"
+            }
+        ],
+    
+    Please check it ,thanks .
+    
+    Signed-off-by: snakeAngel2015 <li.jieA@h3c.com>
+    (cherry picked from commit 6535f6ad2137ee55bf5531e865c05aa10bd39bd0)
+
+commit d7b63e21ca9818e21afeb2945b882e24b7a9b10b
+Merge: a3fae531cd f32b5c613a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 21:32:31 2017 +0200
+
+    Merge pull request #14809 from tchaikov/wip-18193-jewel
+    
+    jewel: tests: test/librados/tmap_migrate: g_ceph_context->put() upon return
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3fae531cdb8c2c64c66d38c947ca139f4ede83d
+Merge: 89d6ddb1f5 6c4826606d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 21:32:09 2017 +0200
+
+    Merge pull request #14701 from smithfarm/wip-18193-jewel
+    
+    jewel: core: transient jerasure unit test failures
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 89d6ddb1f5ee08b0f019dfaadc5857a788c92a09
+Merge: c5f1fce21d be9e83281b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Apr 26 14:28:29 2017 -0400
+
+    Merge pull request #14776 from linuxbox2/jewel-pullup-civet-chunked
+    
+    [DNM] jewel: pullup civet chunked
+
+commit c5f1fce21d0996cce751b6bcca5e57da2cafc135
+Merge: 4d97e0ba8e 97cd21afc5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:32:07 2017 +0200
+
+    Merge pull request #14416 from smithfarm/wip-19557-jewel
+    
+    jewel: tests: upgrade/hammer-x failing with OSD has the store locked when Thrasher runs ceph-objectstore-tool on down PG
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit be9e83281b8c765cd111d5687a516fcd3ca521a5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 25 09:44:39 2017 -0400
+
+    civetweb:  pullup chunked encoding by Marcus
+    
+    Fixes: http://tracker.ceph.com/issues/19736
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 608785a0079f807ff860c56d96b1b67bf6a2ed74
+Author: yaoning <yaoning@unitedstack.com>
+Date:   Fri Jun 24 09:51:07 2016 +0800
+
+    os: make zero values noops for set_alloc_hint() in FileStore
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+    (cherry picked from commit e2ec24f61b55457caccefecd56f9f08b98264802)
+
+commit f32b5c613a8acd50e32747c3581131d28d209efa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 26 22:58:30 2017 +0800
+
+    test/librados/tmap_migrate: g_ceph_context->put() upon return
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflict: test/librados/tmap_migrate.cc
+      this change is not cherry-picked from master, because tmap_migrate was
+    removed in master. so we are applying the same change in
+    cb1cda96713b2ec0f6418c4cbe3d964c2020729c to this test.
+
+commit 905c4acb99f9ea78ff615034dae969ab089bda06
+Author: Mark Nelson <mnelson@redhat.com>
+Date:   Tue May 3 09:56:47 2016 -0500
+
+    Fix reveresed promote throttle default parameters.
+    
+    Signed-off-by: Mark Nelson <mnelson@redhat.com>
+    (cherry picked from commit 793ceac2f3d5a2c404ac50569c44a21de6001b62)
+
+commit fb3ee2efcc13c37db90faa1bc8bad584bab22efa
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Apr 13 05:33:55 2017 -0400
+
+    rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+    
+    Keystone tokens can be revoked.  This causes them to fail
+    validation.  However, in ceph, we cache them.  As long as
+    they're in the cache we trust them.  To find revoked tokens
+    there's a call OSI-PKI/revoked but that's only useful for
+    pki tokens.  Installations using fernet/uuid may not even
+    have the proper credentials to support the call, in which
+    case the call blows up in various ways filling up logs
+    with complaints.
+    
+    This code makes the revocation thread optional; by disabling it,
+    the complaints go away.  A further fix is in the works
+    to use other more modern calls available in modern keystone
+    installations to properly deal with non-PKI/PKIZ tokens.
+    
+    (NB: jewel has this logic in src/rgw/rgw_swift.cc not in src/rgw/rgw_keystone.h)
+    
+    To disable the revocation thread, use at least one of these:
+            rgw_keystone_token_cache_size = 0
+                    using this will cause tokens to be validated on every call.
+    You may instead want to set
+            rgw_keystone_revocation_interval = 0
+                    using just this will disable the revocation thread,
+                    but leaves the cache in use.  That avoids the extra
+                    validation overhead, but means token revocation won't
+                    work very well.
+    
+    Fixes: http://tracker.ceph.com/issues/9493
+    Fixes: http://tracker.ceph.com/issues/19499
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 003291a8cbca455c0e8731f66759395a0bb1f555)
+
+commit 4d97e0ba8ebc89b7797b0936a9e046ef59cc3899
+Merge: cc820a0d4c c2efeb4b62
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 21:06:26 2017 +0200
+
+    Merge pull request #14686 from smithfarm/wip-19686-jewel
+    
+    jewel: osd: Give requested scrubs a higher priority
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit e552d91f73d996c44821f5fbfb28cfc3e5cddc9a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Nov 5 13:13:47 2016 -0400
+
+    rgw: add bucket size limit check to radosgw-admin
+    
+    The change adds a new list of all buckets x all users, with
+    fields for bucket name, tenant name, current num_objects,
+    current num_shards, current objects per shard, and the
+    corresponding fill_status--the latter consisting of 'OK',
+    'WARN <n>%', or 'OVER <n>%.'
+    
+    The warning check is relative to two new tunables.  The threshold
+    max objects per shard is set as rgw_bucket_safe_max_objects_per_shard,
+    which defaults to 100K.  The value rgw_bucket_warning_threshold is
+    a percent of the current safe max at which to warn (defaults to
+    90% of full).
+    
+    From review:
+    
+    * fix indentation (rgw_admin)
+    * if user a user_id is provided, check only buckets for that user
+    * update shard warn pct to be pct-of-fill (not 100 - pct-of-fill)
+    * print only buckets near or over per-shard limit, if --warnings-only
+    * s/bucket limitcheck/bucket limit check */
+    * sanity shard limit should be 90, not 10 (because that changed)
+    * fixes for memleaks and other points found by cbodley
+    
+    Fixes: http://tracker.ceph.com/issues/17925
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 7bc144ce36fedc16a3dedc54598b0d75fb8c68bc)
+
+commit cc820a0d4c2676799383aeaf49a9269e104853ce
+Merge: 5ee54cfe8f d079b91479
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:38:21 2017 +0200
+
+    Merge pull request #14605 from asheplyakov/19476-jewel
+    
+    jewel: rgw: don't return skew time in pre-signed url
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 5ee54cfe8f0a955b11edb4819a652ee420f43e5d
+Merge: 37254aee6c c05bd1cb1f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:37:16 2017 +0200
+
+    Merge pull request #14660 from smithfarm/wip-19478-jewel
+    
+    jewel: rgw: zonegroupmap set does not work
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 37254aee6c3f725cf60deb5c7a3fc28405c51216
+Merge: 09919f938f faeb8088ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:36:23 2017 +0200
+
+    Merge pull request #14607 from asheplyakov/19607-jewel
+    
+    jewel: rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 09919f938f3c1e9e81963f676a9dd13114d54e73
+Merge: d5e1345a2a 527911fab7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 15:50:26 2017 +0200
+
+    Merge pull request #14587 from asheplyakov/19617-jewel
+    
+    jewel: mon/MonClient: make get_mon_log_message() atomic
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 6c4826606dc81fdd847959a49454c69a958bb1d8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Dec 5 20:23:21 2016 +0800
+
+    test/ceph_crypto: do not read ceph.conf in global_init()
+    
+    ForkDeathTest.MD5 expect an empty output while global_init() complains
+    if ceph.conf is missing if 0 is passed in as the `flags`. this test
+    passes if ceph.conf is in current working directory, but jenkins does
+    not prepare this file for this test.
+    
+    Fixes: http://tracker.ceph.com/issues/18128
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c72a2271a8012a66d7bbccf5766a73da5bb878d6)
+    
+    Conflicts:
+            src/test/ceph_crypto.cc - jewel does not have
+                 5af29540675b674c1985ff98b28a783ed124acf6
+
+commit aca2659952528aa1b40b02828293342128657194
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 8 12:40:42 2016 +0100
+
+    tests: fix erasure-code premature deallocation of cct
+    
+    The setup function returns before the run function, the cct variable
+    must be a data member, not a local variable that gets de-allocated
+    before run() starts.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit efa1e54362423d4cfd1541fb8c68237b7b9ebbe3)
+
+commit 8bddd427d9a4bca75a352ca333847dbd6d2c369c
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Feb 16 22:17:52 2017 +0800
+
+    rbd-nbd: no need create asok file for unmap and list-mapped commands.
+    
+    Fixes: http://tracker.ceph.com/issues/17951
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 72352653d585ef89043a4ece371b5c0cb3f6f32a)
+
+commit 328bfbd25c63b1b0e253865abedada7a9e5858e4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 29 12:36:00 2016 -0500
+
+    rbd-nbd: restart parent process logger after forking
+    
+    Fixes: http://tracker.ceph.com/issues/18070
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 29baf254d72cc593572b5a6215360ba51e3be198)
+
+commit 192e7bcdcd1bc02b40b29eff86a335cc8919f663
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 23 19:45:57 2016 +0800
+
+    crushtool: do not release g_ceph_context at exit
+    
+    it is but a work around of occasionally timeout.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d305cc51b18cbf4b2757bbacb5d43324461306a9)
+
+commit 8a2f27cc632c26d7c2b8e8528b4d459b1d78705b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 15 14:21:03 2016 +0800
+
+    common,test: g_ceph_context->put() upon return
+    
+    prior to this change, global_init() could create a new CephContext
+    and assign it to g_ceph_context. it's our responsibilty to release
+    the CephContext explicitly using cct->put() before the application
+    quits. but sometimes, we fail to do so.
+    
+    in this change, global_init() will return an intrusive_ptr<CephContext>,
+    which calls `g_ceph_context->put()` in its dtor. this ensures that
+    the CephContext is always destroyed before main() returns. so the
+    log is flushed before _log_exp_length is destroyed.
+    
+    there are two cases where global_pre_init() is called directly.
+    - ceph_conf.cc: g_ceph_context->put() will be called by an intrusive_ptr<>
+      deleter.
+    - rgw_main.cc: global_init() is called later on on the success code
+      path, so it will be taken care of.
+    
+    Fixes: http://tracker.ceph.com/issues/17762
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cb1cda96713b2ec0f6418c4cbe3d964c2020729c)
+    
+    Conflicts:
+            src/ceph_fuse.cc
+            src/ceph_mgr.cc
+            src/global/global_init.cc
+            src/rgw/rgw_main.cc
+            src/test/compressor/test_compression.cc
+            src/test/compressor/test_compression_plugin.cc
+            src/test/compressor/test_compression_plugin_snappy.cc
+            src/test/compressor/test_compression_plugin_zlib.cc
+            src/test/compressor/test_compression_snappy.cc
+            src/test/compressor/test_compression_zlib.cc
+            src/test/erasure-code/TestErasureCode.cc
+            src/test/erasure-code/TestErasureCodeExample.cc
+            src/test/erasure-code/TestErasureCodeIsa.cc
+            src/test/erasure-code/TestErasureCodeJerasure.cc
+            src/test/erasure-code/TestErasureCodeLrc.cc
+            src/test/erasure-code/TestErasureCodePlugin.cc
+            src/test/erasure-code/TestErasureCodePluginIsa.cc
+            src/test/erasure-code/TestErasureCodePluginJerasure.cc
+            src/test/erasure-code/TestErasureCodePluginLrc.cc
+            src/test/erasure-code/TestErasureCodePluginShec.cc
+            src/test/erasure-code/TestErasureCodeShec.cc
+            src/test/erasure-code/TestErasureCodeShec_thread.cc
+            src/test/fio/fio_ceph_objectstore.cc
+            src/test/librados/misc.cc
+            src/test/mon/PGMap.cc
+            src/test/msgr/test_async_networkstack.cc
+            src/test/msgr/test_userspace_event.cc
+            src/test/objectstore/Allocator_test.cc
+            src/test/objectstore/BitAllocator_test.cc
+            src/test/objectstore/test_bluefs.cc
+            src/test/objectstore/test_bluestore_types.cc
+            src/test/objectstore/test_memstore_clone.cc
+            src/test/osd/TestPGLog.cc
+            src/test/rgw/test_http_manager.cc
+            src/test/rgw/test_rgw_compression.cc
+            src/test/test_mempool.cc
+            src/tools/rados/rados.cc
+
+commit d5e1345a2a6bd8456417db6bb60d61871165e6bd
+Merge: 256f48f463 5096fc9c7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 15:42:43 2017 +0200
+
+    Merge pull request #14665 from smithfarm/wip-19610-jewel
+    
+    jewel: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 256f48f46354f11a2f238f24fa9d890b55f4f4fc
+Merge: 4f67da1b9d 21a83e1276
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 09:20:19 2017 +0200
+
+    Merge pull request #14653 from smithfarm/wip-19662-jewel
+    
+    jewel: rgw_file: fix event expire check, don't expire directories being read
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 4f67da1b9d554a1a5371665573b1f56d3f903de3
+Merge: c2452c5364 33af18e592
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 25 09:17:33 2017 +0200
+
+    Merge pull request #14635 from smithfarm/wip-19690-jewel
+    
+    jewel: doc: Improvements to crushtool manpage
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 040ff013d5c8aded8beedc59bbc4f5afccc64e46
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 16 11:56:09 2016 +0800
+
+    crushtool: s/exit(EXIT_FAILURE)/return EXIT_FAILURE/
+    
+    so the destructor(s) can be called.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e01b89ed6be6b99fec5c725f4bc5769b42468cac)
+    
+    Conflicts:
+            src/tools/crushtool.cc - jewel does not have 17feefbcb3105553b763cb7ce123b20b77b95857
+
+commit 8e993e6e43ecd6e2b444961d1e6c945081b64207
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 16 11:19:04 2016 +0800
+
+    global/signal_handler: reset injected segv after test
+    
+    ~CephContext() => ~TypedSingletonWrapper() => ~MempoolObs =>
+    unregister_command() => ldout() << "unregister_command" =>
+    Log::submit_entry() => *(volatile int *)(0) = 0xdead;
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d932c8f2f23263924103a900714db82ee87f6eef)
+    
+    Conflicts:
+            src/log/Log.cc - jewel has "namespace log" instead of "namespace
+                             logging" (trivial resolution)
+
+commit f1c0042b831826e5bd72ada79a4918a9c26bda24
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 15 14:42:35 2016 +0800
+
+    test_cors.cc: fix the mem leak
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c8a3777203482cabf6739a8ba69b127df8697628)
+
+commit c49b114e8dde6c3f3c5a5b663bd8299b4f2feafb
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Sep 6 16:34:52 2016 +0800
+
+    rgw: fix failed to create bucket if a non-master zonegroup has a single zone
+    
+    If a non-master zonegroup has a single zone, the metadata sync thread not running and
+    the non-master zonegroup can't sync user from master zonegroup,
+    so we can't create bucket(or other metadata update) in it
+    because the authenticated user not found in the zone of non-master zonegroup.
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 949af79b21098e6410bc29274cf36eae2d89faea)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc - retain d32654b7cd60ccc4e23d3f05b9e4385a697bacd6
+                which was merged after this commit
+
+commit 43327f83efcbc5ce54be866a30da8dd1e58d6707
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 22 01:49:12 2017 +0100
+
+    ceph-disk: dmcrypt activate must use the same cluster as prepare
+    
+    When dmcrypt is used, the fsid cannot be retrieved from the data
+    partition because it is encrypted. Store the fsid in the lockbox to
+    enable dmcrypt activation using the same logic as regular activation.
+    
+    The fsid is used to retrive the cluster name that was used during
+    prepare, reason why activation does not and must not have a --cluster
+    argument.
+    
+    Fixes: http://tracker.ceph.com/issues/17821
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7f66672b675abbc0262769d32a38112c781fefac)
+    
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py - in master, self.create_key() takes an
+                argument (self.args.cluster) but in jewel it takes no argument
+
+commit c2452c53641fca416268c2e31c774b8b7e609c88
+Merge: 82b8c89e47 01d04e28db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 07:20:41 2017 +0200
+
+    Merge pull request #13608 from smithfarm/wip-19063-jewel
+    
+    jewel: tests: eliminate race condition in Thrasher constructor
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 3ec1a9bf16e2c305096e11223aaa8db94dc4084d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 31 10:06:42 2017 -0400
+
+    ceph_test_librados_api_misc: fix stupid LibRadosMiscConnectFailure.ConnectFailure test
+    
+    Sometimes the cond doesn't time out and it wakes up instead.  Just repeat
+    the test many times to ensure that at least once it times out (usually
+    it doesn't; it's pretty infrequent that it doesn't).
+    
+    Fixes: http://tracker.ceph.com/issues/15368
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8bc197400d94ee2716d3f2fa454247379a676cf9)
+
+commit 82b8c89e477610641a6e21123b7c929e9c560729
+Merge: 28c7ce595a 30c9527353
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:27:33 2017 +0200
+
+    Merge pull request #14195 from cbodley/wip-19353
+    
+    jewel: rgw: use separate http_manager for read_sync_status
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 28c7ce595ab8fef7c0076f091f249f6b3548ad54
+Merge: c05ecff3f0 2e50fe1684
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:26:50 2017 +0200
+
+    Merge pull request #14066 from asheplyakov/19321-bp-jewel
+    
+    jewel: rgw: fix break inside of yield in RGWFetchAllMetaCR
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c05ecff3f0cb5af2f3aa52b68188742796335ddb
+Merge: 3240cbf4ef dc4e7a1a86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:26:14 2017 +0200
+
+    Merge pull request #14064 from asheplyakov/19211-bp-jewel
+    
+    jewel: rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 3240cbf4ef60733f55800039051e85f2a7ace61d
+Merge: d4672acbb8 85fbb00f6e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:25:21 2017 +0200
+
+    Merge pull request #13842 from smithfarm/wip-19145-jewel
+    
+    jewel: rgw: a few cases where rgw_obj is incorrectly initialized
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit d4672acbb82daf81eaf259e659dd627ec9f3bc79
+Merge: 61ed719d75 ec0668c201
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:24:29 2017 +0200
+
+    Merge pull request #13837 from smithfarm/wip-19048-jewel
+    
+    jewel: rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync()
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 61ed719d75bf3606fa9e3ae8c8ed776aa2ee4313
+Merge: f5e51db564 ced799f9c6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:23:24 2017 +0200
+
+    Merge pull request #13724 from asheplyakov/18626-bp-jewel
+    
+    jewel: rgw: Use decoded URI when verifying TempURL
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 66c3db7aee5b53f83e87ee8c8f081ab9b0336177
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Mar 8 16:31:34 2017 -0500
+
+    rgw: data sync skips slo data when syncing the manifest object
+    
+    Fixes: http://tracker.ceph.com/issues/19027
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8b69847d7b3e92c70090d1dddf7cea5c44fb6b20)
+    
+    Conflicts: bucket cleanup, overrides
+
+commit 303a62f7f15c69413165ed604869909587714a94
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 13 11:33:02 2017 -0400
+
+    rgw: RGWGetObj applies skip_manifest flag to SLO
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 987377ae34382e107e1d54f0bfc1121fcedb4513)
+
+commit f3d99ae8aafaa65e91cd233b5ce8054678d8fa11
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Feb 21 10:27:13 2017 -0500
+
+    rgw: allow system users to read SLO parts
+    
+    multisite data sync relies on fetching the object as the system user
+    
+    Fixes: http://tracker.ceph.com/issues/19027
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d50d18c500fd5dd89e7cada1162cf453b36df370)
+    
+    Conflicts: auth rework
+
+commit f5e51db5644d4eafdbe3b2d541582a17febed139
+Merge: c90cfb7a32 eac0e27193
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 22:51:59 2017 +0200
+
+    Merge pull request #14643 from smithfarm/wip-revert-14427
+    
+    Wip revert 14427
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit db053da61807e26876d97786550a769295c7955a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:56:31 2017 +0100
+
+    mds: validate prealloc_inos on sessions after load
+    
+    Mitigates http://tracker.ceph.com/issues/16842
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c39aaf90ed1b23343eba2b341bb8ee6a50a4ea74)
+    
+    Conflicts:
+            src/mds/InoTable.cc - no 5259683e7819c22c14b21b1dd678a33e14574f21 in jewel
+            src/mds/InoTable.h - no 5259683e7819c22c14b21b1dd678a33e14574f21 in jewel
+
+commit 2b5eb8fa141fa8bd9173dee206ec3530d702fc3a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:33:59 2017 +0100
+
+    mds: operator<< for Session
+    
+    Use this to get a nice human readable name
+    when available (also including the session id in
+    parentheses)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f89787d8312f132ebb621f16c44e950b17a395a)
+
+commit 5b562145197833c6a6f48f006dfb13b6e38f9084
+Author: Henrik Korkuc <henrik@uber.com>
+Date:   Sun Feb 19 11:44:20 2017 +0200
+
+    client/Client.cc: add feature to reconnect client after MDS reset
+    
+    Client.cc marks session as stale instead of reconecting after received
+    reset from MDS. On MDS side session is closed so MDS is ignoring cap
+    renew. This adds option to reconnect stale client sessions instead of
+    just marking sessions stale.
+    
+    Fixes: http://tracker.ceph.com/issues/18757
+    
+    Signed-off-by: Henrik Korkuc <henrik@kirneh.eu>
+    (cherry picked from commit e0bbc704676ef4aed510daff075ef63c9e73b7b3)
+
+commit 8f21038d30097622e319ab986631b03f87f5d907
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Mar 21 12:49:45 2017 +0800
+
+    doc: cephfs: fix the unexpected indent warning
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e423f0b59711422b40c4b3de0bdc73b0947c04d3)
+
+commit f9a1954e113e807b5f9ee6fd56351adc2841730a
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Thu Feb 16 10:45:36 2017 +0100
+
+    doc: additional edits in FUSE client config
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit b6cad3364c020abd3acf906643fa0b6cbb862a0a)
+
+commit 018649f8a4628881c706a6df9b3eef48403fe91e
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Thu Jan 26 12:23:34 2017 +0100
+
+    doc: Dirty data are not the same as corrupted data
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit 80db40f8559128baadad42b925ae813e51a31409)
+
+commit 1d8a5b6d64a26a36fb3a227c52908631d95bef79
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Mon Jan 23 16:34:55 2017 +0100
+
+    doc: minor changes in fuse client config reference
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit e57605681f10436f4b2c85e95179a2904b8c80da)
+
+commit 1ae46b2b94280b8b4a1db89b2cbed8f12bcbcc53
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Jul 24 23:21:29 2016 -0400
+
+    doc: add client config ref
+    
+    Fixes: http://tracker.ceph.com/issues/16743
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 9ad2ccf29830d5309336fc7de877b6926e5dbacd)
+
+commit b8fd297eb64cd29c58596d0a5b0be8fbe9d94a72
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Apr 11 13:51:47 2017 -0700
+
+    use sudo to check check health
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7af157ad4ce7f7e2b8de97ee10eeaf64b9099bc0)
+
+commit 1b91ffc0fbe76c5475f17d4e15ea295ee3680688
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:27:20 2017 -0700
+
+    Add reboot case for systemd test
+    
+    test systemd units restart after reboot
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7b587304a54d9b21041ffdfbc85fad8d87859c49)
+
+commit 3d8d1dad8a698a9fb45be7b31c92f8b71ee67720
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:56:11 2017 -0700
+
+    Fix distro's, point to latest version
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1947648669971c1bd1ca189870ed9b25bbd48d3a)
+
+commit c90cfb7a327fcbb9508e617c2353becb7e5cb45f
+Merge: 327276cf3d 9b77b16b88
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 22:05:57 2017 +0200
+
+    Merge pull request #14602 from asheplyakov/19646-jewel
+    
+    jewel: ceph-disk: enable directory backed OSD at boot time
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 327276cf3d4fb7a2558620a8a7d9cc90e5d6e5c5
+Merge: 7008c64c51 25e43ac256
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 22:04:41 2017 +0200
+
+    Merge pull request #14449 from smithfarm/wip-test-doc-oversight
+    
+    tests: fix oversight in yaml comment
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f34489dd52f07aeb88e1dbf361ead63f09bb9c65
+Author: YunfeiGuan <yunfei.guan@xtaotech.com>
+Date:   Mon Apr 10 05:48:47 2017 +0000
+
+    cephfs: fix mount point break off problem after mds switch occured
+    
+    The hot-standby become active as we expected but the mount piont broken strangely
+    when the active mds is down. The root reason is the new mds use last_cap_renews
+    decoded from ESesson::replay in find_idle_sessions and wrongly killed the session.
+    Maybe we should reset session->last_cap_renew to the current time when server send
+    OPEN to client in reconnect stage.
+    
+    Fixes: http://tracker.ceph.com/issues/19437
+    Signed-off-by: Guan yunfei <yunfei.guan@xtaotech.com>
+    (cherry picked from commit 4ef830c5d6f22bf0d4f82a8624c772ecbbda44a6)
+    
+    Conflicts:
+            src/mds/Server.cc (leave '\n' in because jewel does not have
+            693132eb00b1803d5e97a79908521d5a6903e9f8; jewel ceph_clock_now takes a
+            CephContext object)
+
+commit 7008c64c5143a3ea19d6679a3e521897f74ff69a
+Merge: d62644abcd f8aa6be06c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:26:52 2017 +0200
+
+    Merge pull request #13606 from smithfarm/wip-19062-jewel
+    
+    jewel: build/ops: enable build of ceph-resource-agents package on rpm-based os
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit d62644abcde0fd99da24922ce7638d173952ceb4
+Merge: 8df8960553 482bd1adab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:18:05 2017 +0200
+
+    Merge pull request #14654 from smithfarm/wip-19461-jewel
+    
+    jewel: admin ops: fix the quota section
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c2efeb4b62e6dc4e6975561eb8b20cfca66f7237
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 17 14:58:02 2017 -0700
+
+    osd: Give requested scrub work a higher priority
+    
+    Once started we now queue scrub work at higher priority than
+    scheduled scrubs.
+    
+    Fixes: http://tracker.ceph.com/issues/15789
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ebab8b1f4f67fbdec1e147c580329c1e2b5cf7cd)
+    
+    Conflicts:
+            src/osd/OSD.h - in jewel, the PGScrub() call is enclosed within
+                op_wq.queue(make_pair(...)) instead of enqueue_back()
+
+commit db86a24e7906ca7c70c60b1752f1230d56361bcd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 22 17:33:05 2017 +0800
+
+    client: wait for lastest osdmap when handling set file/dir layout
+    
+    Fixes: http://tracker.ceph.com/issues/18914
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 76f5eb86cdd61dde4e6c7cfeb5cf34f0c0334f21)
+    
+    Conflicts:
+            src/client/Client.cc (jewel does not have 201c56039)
+            src/client/Client.h (jewel does not have 201c56039)
+
+commit 7b9283beec0c4bf8e2067558fc4e03e336849b77
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Thu Apr 13 20:09:07 2017 +0800
+
+    cephfs: fix write_buf's _len overflow problem
+    
+    After I have set about 400 64KB xattr kv pair to a file,
+    mds is crashed. Every time I try to start mds, it will crash again.
+    The root reason is write_buf._len overflowed when doing
+    Journaler::append_entry().
+    
+    This patch try to fix this problem through the following changes:
+    
+     1. limit file/dir's xattr size
+     2. throttle journal entry append operations
+    
+    Fixes: http://tracker.ceph.com/issues/19033
+    Signed-off-by: Yang Honggang joseph.yang@xtaotech.com
+    (cherry picked from commit eb915d0eeccbe523f8f70f6571880003ff459459)
+
+commit b52c5088618011a569f157616c5c667c2fc1e9fe
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 8 12:13:46 2017 +0000
+
+    mds: shut down finisher before objecter
+    
+    Some of the finisher contexts would try to call into Objecter.
+    We mostly are protected from this by mds_lock+the stopping
+    flag, but at the Filer level there's no mds_lock, so in the
+    case of file size probing we have a problem.
+    
+    Fixes: http://tracker.ceph.com/issues/19204
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 177a97d5c55ee6a2d5dcd3cf0893546190b10f7a)
+    
+    Conflicts:
+            src/mds/MDSRank.cc (no purge_queue.shutdown() in jewel because jewel
+            does not have 8ebf7d95a9071de24bb1e56a6423c505169cb4de)
+
+commit 96e801fb53941214c669ac01c1804a4d1f27086c
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 28 14:13:33 2017 -0400
+
+    mds: ignore ENOENT on writing backtrace
+    
+    We get ENOENT when a pool doesn't exist.  This can
+    happen because we don't prevent people deleting
+    former cephfs data pools whose files may not have
+    had their metadata flushed yet.
+    
+    http://tracker.ceph.com/issues/19401
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3fccc2372f2715d075b05e459140360cf6e7ca96)
+
+commit 8df896055366d777ca309dd4a7840f2a2a5a72cc
+Merge: da888fa871 7468689314
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:53:30 2017 +0200
+
+    Merge pull request #14680 from smithfarm/wip-19711-jewel
+    
+    jewel: [test] test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 74686893147f2d7e1fd1287de37587554fea96c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:26:31 2017 -0400
+
+    test: rbd master/slave notify test should test active features
+    
+    Fixes: http://tracker.ceph.com/issues/19692
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0dcba41cba96566d0b8da54cf0316d523b88ded2)
+
+commit 7347f1193963666ce07ff9163064aeb0cf343d38
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 5 21:29:10 2017 +0800
+
+    mds: make C_MDSInternalNoop::complete() delete 'this'
+    
+    Fixes: http://tracker.ceph.com/issues/19501
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 424e0c6744e7f63459ca0ff7deab751726aa30cd)
+    
+    Conflicts:
+            src/mds/MDSContext.h (omit "override" because jewel does not have
+            1a91aeab987870b3ccbcf2f1e476fac8b534d449)
+
+commit 824b19a9a66261a90c4e4de64e877a42438febd2
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 29 19:38:37 2017 +0100
+
+    tools/cephfs: set dir_layout when injecting inodes
+    
+    When we left this as zero, the MDS would interpret it was HASH_LINUX
+    rather than the default HASH_RJENKINS.  Potentially that
+    could cause problems if there perhaps were already dirfrags in
+    the metadata pool that were set up using rjenkins.  Mainly
+    it just seems more appropriate to explicitly set this field
+    rather than hit the fallback behaviour.
+    
+    Related: http://tracker.ceph.com/issues/19406
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d6d542885bd29b71214f9ca52bd26e9183c5d01)
+
+commit eab56dae6714706cf9650dff172b20a764f5481c
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 9 13:15:46 2017 +0000
+
+    mon: fix hiding mdsmonitor informative strings
+    
+    Local `stringstream ss` declarations were hiding
+    the real variable used to feed back to the user.
+    
+    Fixes: http://tracker.ceph.com/issues/16709
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 00404ae9bd4cce0518a44d36d2d6a5612f4f9d04)
+
+commit da888fa8713178cc23c1069ce2651e02c98aad05
+Merge: 8068b546af 2271cd8128
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Apr 20 19:10:33 2017 +0800
+
+    Merge pull request #14402 from shinobu-x/wip-17331-jewel
+    
+     jewel: ceph-disk: ceph-disk list reports mount error for OSD having mount options with SELinux context
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8068b546afbe8e774c9d699205a32215e38c5d4f
+Merge: 118ccad0ba 754b4a482c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 13:06:33 2017 +0200
+
+    Merge pull request #13865 from smithfarm/wip-19158-jewel
+    
+    jewel: rgw: health check errors out incorrectly
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit d57437e338984c5db84f9d16387e082ada5a0a33
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:51:31 2017 +0000
+
+    mds: reset heartbeat in export_remaining_imported_caps
+    
+    This loop can be very long.
+    
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85071f1509beba4a390730e6a3a4332484646d63)
+
+commit 6adf1904ed7209b70328b20f248a701fbdd3127c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:24:50 2017 +0000
+
+    mds: heartbeat_reset in dispatch
+    
+    Previously we only heartbeated in tick.  However, our locking is
+    not guaranteed to be fair, so on a super-busy dispatch queue it may be
+    possible for the heartbeat to time out while the tick() function
+    is waiting for mds_lock.
+    
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 819394549af10532419d88742fae3a69d2ea487d)
+
+commit 63f41d543f8a5f1f55a12612d39c6a2a1cf9c114
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 15 11:45:26 2017 +0800
+
+    test/libcephfs: avoid buffer overflow when testing ceph_getdents()
+    
+    The buffer size should be at least "2 * sizeof(struct dirent)".
+    Otherwise, the code that checks dentry '..' overflow.
+    
+    Fixes: http://tracker.ceph.com/issues/18941
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fa6671345b8f3a82dcd232f99e55a982b0a641f1)
+
+commit 7146816065bea55b4e3fec59048a459bcff50f55
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Fri Feb 10 10:56:46 2017 +0800
+
+    mds/StrayManager: aviod reusing deleted inode in StrayManager::_purge_stray_logged
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 4978e57419482384279d7e784a625f5e5c10961a)
+
+commit d8b139b5847cb46aa13486ee85e26f26421f36d6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Feb 1 00:38:08 2017 +0000
+
+    tasks/cephfs: switch open vs. write in test_open_inode
+    
+    Do the write after opening the file, so that we get good
+    behaviour wrt the change in Mount.open_background that uses
+    file existence to confirm that the open happened.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a027dba78fc8bc84ae39d7998b386ce21c01e1bf)
+
+commit e8ae80fc8721e500782c7e87a7cb89128c2a9687
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 26 16:48:58 2017 +0000
+
+    qa: fix race in Mount.open_background
+    
+    Previously a later remote call could end up executing
+    before the remote python program in open_background
+    had actually got as far as opening the file.
+    
+    Fixes: http://tracker.ceph.com/issues/18661
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c6d91dd91252e703d08b8ac62ac6a47ee82c0bed)
+
+commit 36c86f71efc10f9a651aedf01dc6fb9d1bfed703
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 15:28:23 2017 +0800
+
+    mds: don't purge strays when mds is in clientreplay state
+    
+    MDS does not trim log when it's in clientreplay state. If mds hang
+    at clientreplay state (due to bug), purging strays can submit lots
+    of log events and create very large mds log.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 86bbc7fff02668077f27d0924ba3efe6544b77f6)
+
+commit 8b01cf33575783661a1e1151c8214d327a08b6f9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 11:03:45 2017 +0800
+
+    mds: skip fragment space check for replayed request
+    
+    when handling replayed request, stray directory can be different
+    from the stray directory used by the original request. The fragment
+    space check for stray directory can fail.
+    
+    Fixes: http://tracker.ceph.com/issues/18660
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit afe889cbc5baab196567c2aad01f49fe90901fda)
+
+commit 82ea0971b3cb07c32ec837cb85de63d4068a70d7
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Dec 14 12:09:44 2016 -0800
+
+    client: fix the cross-quota rename boundary check conditions
+    
+    We were previously rejecting a rename if either of the involved directories
+    was a quota root, even if the other directory was part of the same quota
+    "tree". What we really want to do is identify the correct quota root
+    (whether local or ancestral) for each directory and compare them. So
+    now we do.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8e8892aa46accb519faa4bb9fecf66618f1b11b2)
+    
+    Conflicts:
+            src/client/Client.cc (do not pass perm to get_quota_root() because
+            jewel does not have 3caa4d233633fb7a67747f2c79c4a0ab89112294)
+
+commit dbe90c79b86743c7d143d33eb5389fcee0ac76ef
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 11 22:31:43 2017 +0200
+
+    librbd: fix rbd_metadata_list and rbd_metadata_get
+    
+    - properly check for val_len in rbd_metadata_list
+    - don't expect output buffers are zero pre-filled
+    
+    Fixes: http://tracker.ceph.com/issues/19588
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 75afc74ea681402e22b6dec8b83276d145fc786b)
+
+commit 5096fc9c7c62e3043a9a0638eb2516792526fdd7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 12 10:47:28 2017 -0400
+
+    test/librados_test_stub: fixed cls_cxx_map_get_keys/vals return value
+    
+    Fixes: http://tracker.ceph.com/issues/19597
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ffd464dac102f684d6dfa78e58d2cb45e165ed6)
+
+commit 216156b5d4e2666ca592eaab2211940028422bc0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 10 10:56:38 2017 -0500
+
+    rbd: prevent adding multiple mirror peers to a single pool
+    
+    The rbd-mirror daemon does not currently support replication
+    from multiple peers. Until that is supported, add a temporary
+    restriction to prevent confusion.
+    
+    Fixes: http://tracker.ceph.com/issues/19256
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c0c9d1014d57b3d5b95e7513fcc38d04b9ea5165)
+
+commit c05bd1cb1f570c42b1999fbc70c86b0a72a7fcb3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Apr 5 13:31:08 2017 +0300
+
+    radosgw-admin: use zone id when creating a zone
+    
+    Fixes: http://tracker.ceph.com/issues/19498
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3fea36d635fcba8ca584a1c0ec9f07840009402c)
+
+commit ba81cbbdee1ccf95ceff56eef0a1b2b06be4024a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 15:43:13 2017 -0400
+
+    qa: rgw task uses period instead of region-map
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e3e3a71d1f1fb43bb4172ce2dfac9a28ca89df0f)
+
+commit a755c95262e10fac774716e460e2ba2eaee9df70
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 14:18:15 2017 -0400
+
+    rgw-admin: remove deprecated regionmap commands
+    
+    Fixes: http://tracker.ceph.com/issues/18725
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5830c1849a0c0110d17c37784808e456e6dcb7b3)
+    
+    Conflicts:
+            src/rgw/rgw_admin.cc (trivial resolution)
+
+commit 482bd1adabf6f54c8f2bd73090d2e4c02d07f8a3
+Author: hrchu <petertc@gmail.com>
+Date:   Wed Mar 29 02:17:04 2017 +0000
+
+    doc: rgw: correct the quota section
+    
+    Add the missing option and fix typo.
+    
+    Fixes: http://tracker.ceph.com/issues/19397
+    
+    Signed-off-by: Chu, Hua-Rong <hrchu@cht.com.tw>
+    (cherry picked from commit 51a88267f0d7f51aeb62092949b66b9f6c062e15)
+
+commit 21a83e1276e415e98a3780d0374bb9d4feb191d9
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:35:16 2017 -0400
+
+    rgw_file:  remove unused rgw_key variable
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 1100a1c26e76485569cfebcf863b18cf908f6161)
+
+commit ebad040b3ab62e702afc52edd9d99d984b24cdc1
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Apr 14 15:56:37 2017 -0400
+
+    rgw_file: fix readdir after dirent-change
+    
+    Also, fixes link count computation off-by-one, update of state.nlink
+    after computation, link computation reset at start, and a time print
+    in debug log.
+    
+    Fixes: http://tracker.ceph.com/issues/19634
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    
+    link count
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0f80266ecd424bf9466579b3edc03911a7c5719)
+
+commit dd9833cacaec20e5bd1a70ec46a427a7352eb5d6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 06:42:07 2017 -0400
+
+    rgw_file: don't expire directories being read
+    
+    If a readdir expire event turns out to be older than last_readdir,
+    just reschedule it (but actually, we should just discard it, as
+    another expire event must be in queue.
+    
+    Fixes: http://tracker.ceph.com/issues/19625
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 007b7451c26716c51207c161dc347e9a00da53f1)
+
+commit 30a5e857aade2474c5bd621c57938dfe333c3ae5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:40:35 2017 -0400
+
+    rgw_file:  rgw_readdir:  return dot-dirs only when *offset is 0
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 61482c2b85a07519f2256b1a3f2b6d8aa99d5f06)
+
+commit fe836bfb7a286c87c57576d29d6862be2514ada5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 05:56:13 2017 -0400
+
+    rgw_file:  chunked readdir
+    
+    Adjust readdir callback path for new nfs-ganesha chunked readdir,
+    including changes to respect the result of callback to not
+    continue.
+    
+    Pending introduction of offset name hint, our caller will just be
+    completely enumerating, so it is possible to remove the offset map
+    and just keep a last offset.
+    
+    Fixes: http://tracker.ceph.com/issues/19624
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0191d74e3aef06bf300df045a53a3952a71f651)
+
+commit 16eeb8c2acc8a1f843fa87967c2ee4c0ed2cd8bd
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri Mar 31 10:42:40 2017 +0800
+
+    rgw_file: fix missing unlock in unlink
+    
+    Fixes: http://tracker.ceph.com/issues/19435
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit cb6808a6366a70f54d0cc16437d16aa1b7819c84)
+
+commit 8c7cb8227c39b09060cdca4e11dec75cf7f2336c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Mar 13 21:52:08 2017 -0400
+
+    rgw_file: implement reliable has-children check (unlink dir)
+    
+    Bug report and discussion provided by
+    Gui Hecheng <guihecheng@cmss.chinamobile.com> in nfs-ganesha upstream
+    github.  Briefly, while a reliable check is potentially costly,
+    it is necessary.
+    
+    Fixes: http://tracker.ceph.com/issues/19270
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b05f1c6d61aa4501a971e87de6dcaf3e58c3d9b4)
+
+commit 18f14dd86400b50f46930a9ef56666d82035507e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 4 20:16:13 2017 -0400
+
+    rgw_file: introduce rgw_lookup type hints
+    
+    The new type hints optimize object type deduction, when the
+    rgw_lookup is called from an rgw_readdir callback.
+    
+    Fixes: http://tracker.ceph.com/issues/19623
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2e66c7a7cc763c5c0d6f5db04855f60f2b2ceed3)
+
+commit 118ccad0ba763b98c9ee3de225a947be5372dc0d
+Merge: bf30ecd7cb 65465356b5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:08:45 2017 +0200
+
+    Merge pull request #14383 from smithfarm/wip-19547-jewel
+    
+    jewel: build/ops: rbdmap.service not included in debian packaging (jewel-only)
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit bf30ecd7cb8dea43a9d55cb1b02c72bfda6bad09
+Merge: 376c5e4753 9e123e6d6c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:02:39 2017 +0200
+
+    Merge pull request #14143 from smithfarm/wip-19355-jewel
+    
+    jewel: rgw: when converting region_map we need to use rgw_zone_root_pool
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 376c5e47531a2803ac9b7ea236b989c528268a9f
+Merge: da306df4de e2ee70a8ad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:01:49 2017 +0200
+
+    Merge pull request #14136 from smithfarm/wip-19330-jewel
+    
+    jewel: rgw: upgrade to multisite v2 fails if there is a zone without zone info
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit da306df4deaaa782c0bad72539affd3ee1ec7efc
+Merge: 57b210da41 5ee8feaba4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:00:35 2017 +0200
+
+    Merge pull request #13872 from smithfarm/wip-19163-jewel
+    
+    jewel: doc: radosgw-admin: add the 'object stat' command to usage
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 57b210da4185d624524d05bcd9ad01c2df16ca76
+Merge: db92019e9a 9cd7dd8490
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 10:59:33 2017 +0200
+
+    Merge pull request #13863 from smithfarm/wip-19155-jewel
+    
+    jewel: rgw: typo in rgw_admin.cc
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit db92019e9af43c2367fdf7865c0fab8cb38453e6
+Merge: 6dcd5fa847 6add2a457e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 10:56:35 2017 +0200
+
+    Merge pull request #13779 from smithfarm/wip-18866-jewel
+    
+    jewel: rgw: 'radosgw-admin sync status' on master zone of non-master zonegroup
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit eac0e27193c67ffca60b5e7c61e7769ad8ace6aa
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 19 16:53:57 2017 +0200
+
+    Revert "osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag."
+    
+    This reverts commit 2d68822c784eb4d62d3b0198ed4ec04404dbffb3.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0efe16d2566f0d6040f61fafd38c6661f08da1cd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 19 16:53:49 2017 +0200
+
+    Revert "osdc/Objecter: resend RWORDERED ops on full"
+    
+    This reverts commit f2474042ecd6560323673170c13f2cb964406e70.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 33af18e59298926d9636023faf871a191d582c1a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 18 08:06:01 2017 +0200
+
+    doc: mention --show-mappings in crushtool manpage
+    
+    Fixes: http://tracker.ceph.com/issues/19649
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b48b6f4ed8b5f5b5852cbbfd5b3d5b650efb7f1b)
+
+commit 6dcd5fa8474c3a43ffca2394557b11f4906ee1ff
+Merge: 7c006fc164 68fcb01211
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Apr 19 02:47:27 2017 -0400
+
+    Merge pull request #14596 from gregsfortytwo/wip-17916-divergent
+    
+    Wip 17916 divergent
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 97cd21afc54efe3afb482b041f9c34ab6cdc682e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Apr 9 20:11:27 2017 +0200
+
+    tests: Thrasher: handle "OSD has the store locked" gracefully
+    
+    On slower machines (VPS, OVH) it takes time for the OSD to go down.
+    
+    Fixes: http://tracker.ceph.com/issues/19556
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a5b19d2d73540b730392f8001c8601f2cecc1b51)
+
+commit faeb8088ac3bd16bc04a1e5a55fec50285a7253f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 16:19:57 2017 -0400
+
+    rgw: fix for null version_id in fetch_remote_obj()
+    
+    commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f fixed the wrong use of
+    the dest_obj's version, but removed the check for "null" version
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 915370776df5b964c2ee8d9f9329562919eef8d5)
+
+commit 6180fcb4ec33bdade37f5693193712f8016b2560
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Fri Mar 17 16:26:56 2017 +0800
+
+    rgw: version id doesn't work in fetch_remote_obj
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: trivial: dest_obj.key.instance in master
+              versus dest_obj.get_instance() in Jewel
+
+commit d079b91479abfb474fad4fafe54c119f009ea572
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+    
+    Fixes: http://tracker.ceph.com/issues/18828
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 9b77b16b888b8efbf7d50d333e1880a6ec70d87a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Apr 13 23:49:50 2017 +0200
+
+    ceph-disk: enable directory backed OSD at boot time
+    
+    https://github.com/ceph/ceph/commit/539385b143feee3905dceaf7a8faaced42f2d3c6
+    introduced a regression preventing directory backed OSD from starting at
+    boot time.
+    
+    For device backed OSD the boot sequence starts with ceph-disk@.service
+    and proceeds to
+    
+        systemctl enable --runtime ceph-osd@.service
+    
+    where the --runtime ensure ceph-osd@12 is removed when the machine
+    reboots so that it does not compete with the ceph-disk@/dev/sdb1 unit at
+    boot time.
+    
+    However directory backed OSD solely rely on the ceph-osd@.service unit
+    to start at boot time and will therefore fail to boot.
+    
+    The --runtime flag is selectively set for device backed OSD only.
+    
+    Fixes: http://tracker.ceph.com/issues/19628
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f425a127b7487d2093c8c943f0bcdec3d673d601)
+    
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py: trivial: Jewel does not support
+               OpenRC and other inits, hence no corresponding 'elif'
+
+commit 68fcb01211e064f0d200cc9c9576254e9a6b949c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 18:09:55 2017 -0700
+
+    pglog: require users set a config option before ignoring divergent_priors
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit b9477303b010b3653934f77fa533df01aeff1c3c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 14:33:20 2017 -0700
+
+    osd: pglog: clean up divergent_priors off disk when running; don't assert on startup
+    
+    Fixes: http://tracker.ceph.com/issues/17916
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7c006fc1640d2f6b9dc9002bfd994ecdf25510ee
+Merge: e31a540dce 721b2083cd
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Apr 17 17:41:19 2017 -0400
+
+    Merge pull request #14492 from gregsfortytwo/wip-jewel-snaptrim
+    
+    Backport snap trimming improvements to Jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 721b2083cd2ed8567d41aba6b17caf30b678e6b3
+Merge: a84dc8fe74 e31a540dce
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Apr 17 17:39:59 2017 -0400
+
+    Merge branch 'jewel' into wip-jewel-snaptrim
+
+commit a84dc8fe747d1e17c7910c9857f7f5d3d56e2dff
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 14:32:38 2017 -0700
+
+    PendingReleaseNotes: discuss snap trim improvements
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 360a9d9af003c650cdf00534909d6488c702c413
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Apr 12 16:30:55 2017 -0700
+
+    PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine
+    
+    Rather than blocking the main op queue, just pause for that amount of
+    time between state machine cycles.
+    
+    Also, add osd_snap_trim_sleep to a few of the thrasher yamls.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2ed7759cfeb03e71f0fbd98fe7c2db2bb741861c)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 67336454a4cee66522bc0ca01b2c58b8960f75ec)
+    
+    Conflicts:
+            qa/suites/rados/thrash/thrashers/default.yaml
+            qa/suites/rados/thrash/thrashers/pggrow.yaml
+            src/osd/OSD.h
+            src/osd/ReplicatedPG.cc
+            src/osd/ReplicatedPG.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 18dbf6a0245e35dbbdb5ddb760182795b37983c0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:41:21 2017 -0800
+
+    rados: check that pool is done trimming before removing it
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4aebf59d906fa3e03d21bdac182f89fe3cd4c802)
+    (cherry picked from commit 34398c29b3c57f00d932cf96570f882dce64a82b)
+
+commit 7f78450bd184335ba0098f3ded0fefa2c79a5dd5
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Apr 11 14:04:19 2017 -0700
+
+    osd/ReplicatedPG: limit the number of concurrently trimming pgs
+    
+    This patch introduces an AsyncReserver for snap trimming to limit the
+    number of pgs on any single OSD which can be trimming, as with backfill.
+    Unlike backfill, we don't take remote reservations on the assumption
+    that the set of pgs with trimming work to do is already well
+    distributed, so it doesn't seem worth the implementation overhead to get
+    reservations from the peers as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 21cc515adfb225ba70f1d80b1b76f0345c214c22)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+            src/osd/PrimaryLogPG.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    
+    (cherry picked from commit 68ea24396ca6450d4d8042a7c5f51306b7d199fa)
+    (cherry picked from commit c7176b869898c870b56b1762958652d801af4c4c)
+    
+    Conflicts: Many. As evidenced by involving two distinct patches
+    in this one commit, it wasn't a clean backport.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 527911fab78b4752313a4a2a5d3ab0ae736bc50f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Apr 10 14:53:46 2017 +0800
+
+    mon/MonClient: make get_mon_log_message() atomic
+    
+    * LogClient: move reset_session() into get_mon_log_message() and add a
+      "flush" param to the latter. so it can get_mon_log_message()
+      atomically. otherwise another call changing the log queue could sneak
+      in between reset_session() and get_mon_log_message().
+    * MonClient: add a "flush" param to do_send() so we can reset the
+      LogClient session once we are connected to a monitor.
+    
+    Fixes: http://tracker.ceph.com/issues/19427
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5215e291da2b527d85e129eda86043490843178e)
+    
+    Conflicts:
+            src/mon/MonClient.cc: handle_auth: replaced 'log_client->reset_session();
+            send_log();' sequence with newly introduced 'send_log(true);' like
+            the original patch does
+
+commit e31a540dcea96b3d5b4f7ecd20e2d54e81a68e2b
+Merge: 7c36d1650f 06916a8798
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 14 22:28:51 2017 +0200
+
+    Merge pull request #13834 from smithfarm/wip-18969-jewel
+    
+    jewel: rgw: Change loglevel to 20 for 'System already converted' message
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7c36d1650fdb7a357ff724b37cf59b4da413f76b
+Merge: 0e3aa2cb01 4c1f302f7d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 14 22:26:51 2017 +0200
+
+    Merge pull request #13833 from smithfarm/wip-18908-jewel
+    
+    jewel: rgw: the swift container acl does not support field .ref
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit b698d1fa4ce4aca5e392eeec600e3357c3cf71a9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:28:41 2017 -0400
+
+    librbd: is_exclusive_lock_owner API should ping OSD
+    
+    This is required to detect if a peer has been silently blacklisted
+    and is therefore no longer the lock owner.
+    
+    Fixes: http://tracker.ceph.com/issues/19287
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e15db05960a284bdf3701256722299d553cfd5aa)
+    
+    Conflicts:
+            src/librbd/ManagedLock.[h|cc]: logic moved to ExclusiveLock
+    
+    (cherry picked from commit 7e30b630e2806c73ea503871599f958b58df7934)
+
+commit 0e3aa2cb011be8a2af69040b94ee240ea6c1e663
+Merge: 8d5a5ddfec 419c9926d9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 11:09:27 2017 +0200
+
+    Merge pull request #13214 from ovh/bp-osd-updateable-throttles-jewel
+    
+    jewel: osd: allow client throttler to be adjusted on-fly, without restart
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8d5a5ddfecc7c020eeb8aeb927e007bc04885534
+Merge: 091aaa2ab7 bcd3c906e5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:44:23 2017 +0200
+
+    Merge pull request #14326 from shinobu-x/wip-15025-jewel
+    
+    jewel: osd: new added OSD always down when full flag is set
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d311eea6bc5b2c88984cfc16340970ea1459b74a
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sat Feb 11 10:32:53 2017 -0800
+
+    msg/simple/Pipe: support IPv6 QoS.
+    
+    Extend DSCP marking for heartbeat packets to IPv6, as commit
+    9b9a682fe035c985e416ee1c112fa58f9045a27c only implemented
+    support for IPv4.
+    
+    Conflicts: Cherry-picked 91a29bc490fdfbbef0875fa620c7ba1a1a6492ae from master to avoid conflict.
+    Backport: jewel, luminious
+    Fixes: http://tracker.ceph.com/issues/18887
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 2d6021fbf7a728f73c2998be17e9224f14b83a30)
+
+commit 332b5174c769f395074255e075de8d2cc1ee4021
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:27:08 2017 -0400
+
+    pybind: fix incorrect exception format strings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68617455f534a612ade1331f43b032ab524704ae)
+
+commit 091aaa2ab768858e840e2d05e0896c229ce69984
+Merge: 3f2e4cd2d5 d30c4d55ad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:38:48 2017 +0200
+
+    Merge pull request #13874 from smithfarm/wip-19171-jewel
+    
+    jewel: doc: rgw S3 create bucket should not do response in json
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 3f2e4cd2d53f1cbdbdb1fc8687c5ffe0d46b346b
+Merge: ea0bc6c553 0e11a938c5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:06:40 2017 +0200
+
+    Merge pull request #13492 from shinobu-x/wip-18516-jewel
+    
+    jewel: build/ops: "osd marked itself down" will not recognised if host runs mon + osd on shutdown/reboot
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit ea0bc6c553454f7641a2594013412aee142fbd11
+Merge: 845972f4de d012c381e8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:01:23 2017 +0200
+
+    Merge pull request #13254 from shinobu-x/wip-14609-jewel
+    
+    jewel: common: radosstriper: protect aio_write API from calls with 0 bytes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 845972f4de1d803aa4dab0e1afaed693bbba088f
+Merge: a3deef997f cfa37d6a16
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:52:10 2017 +0200
+
+    Merge pull request #13489 from shinobu-x/wip-18955-jewel
+    
+    jewel: ceph-disk: bluestore --setgroup incorrectly set with user
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3deef997ff0800b9e3e2d141cfc6fdc73cac837
+Merge: 702edb5519 39aab763a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:51:42 2017 +0200
+
+    Merge pull request #14070 from smithfarm/wip-19339-jewel
+    
+    jewel: tests: dummy suite fails in OpenStack
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 702edb5519e67bc5f8c5b65c6f63c9635cd758cf
+Merge: f509ccc4b7 a20d2b89ee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:51:12 2017 +0200
+
+    Merge pull request #14329 from smithfarm/wip-19493-jewel
+    
+    jewel: ceph-disk: Racing between partition creation & device node creation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Sébastien Han <seb@redhat.com>
+
+commit f509ccc4b734697267cec35ff57dd7f1c5aaaddb
+Merge: c8c4bff2af f2474042ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:48:29 2017 +0200
+
+    Merge pull request #14427 from smithfarm/wip-19140-jewel
+    
+    jewel: osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c8c4bff2afa7fe0339a97f32e69bb9d0546f1318
+Merge: 349baea1a4 b5b441abaa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:48:08 2017 +0200
+
+    Merge pull request #14324 from shinobu-x/wip-19371-jewel
+    
+    jewel: common: monitor creation with IPv6 public network segfaults
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 349baea1a4486e475e0381a6b316d64a6ce0139c
+Merge: dd466b7d9a 72e2476a13
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:45:35 2017 +0200
+
+    Merge pull request #14112 from shinobu-x/wip-19192-jewel
+    
+    jewel: tools: brag fails to count "in" mds
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dd466b7d9acb03c8830bdd83b3b73602c68083c2
+Merge: b8f8bd0a94 ee06517547
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:45:11 2017 +0200
+
+    Merge pull request #14150 from smithfarm/wip-18823-jewel
+    
+    jewel: tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b8f8bd0a949c917b119eca91aec95c6a971a1fb4
+Merge: 222916a375 1cc8d0d085
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:47 2017 +0200
+
+    Merge pull request #14152 from smithfarm/wip-18893-jewel
+    
+    jewel: librbd: Incomplete declaration for ContextWQ in librbd/Journal.h
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 222916a3758b0253bcb851b5e955f6970f171306
+Merge: 49f84b1a14 b85677397e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:25 2017 +0200
+
+    Merge pull request #14154 from smithfarm/wip-18948-jewel
+    
+    jewel: tests: additional rbd-mirror test stability improvements
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49f84b1a14295e80ef35204ed691b3131c5f744c
+Merge: 2a232d43a1 f6489d01ca
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:04 2017 +0200
+
+    Merge pull request #14148 from smithfarm/wip-18778-jewel
+    
+    jewel: rbd: rbd --pool=x rename y z does not work
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2a232d43a18b47399f96abeac3ac800f204a9eaf
+Merge: 413ac584d6 8bed107b84
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:43:41 2017 +0200
+
+    Merge pull request #14083 from smithfarm/wip-19357-jewel
+    
+    jewel: rbd: systemctl stop rbdmap unmaps all rbds and not just the ones in /etc/ceph/rbdmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 413ac584d63fd469ea28defa68c39538444d01b1
+Merge: 23d595b1b0 547e867628
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:43:21 2017 +0200
+
+    Merge pull request #13154 from smithfarm/wip-18496-jewel
+    
+    jewel: librbd: Possible deadlock performing a synchronous API action while refresh in-progress
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 23d595b1b0fb874adfd1507d86db261cf6a193a9
+Merge: 4add6f5580 915dbace5d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:42:54 2017 +0200
+
+    Merge pull request #13244 from smithfarm/wip-18775-jewel
+    
+    jewel: rbd: qemu crash triggered by network issues
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4add6f5580b7d65571bc426e95fac35be7150ff4
+Merge: 37ab19cc29 1a4e1e09b1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:42:23 2017 +0200
+
+    Merge pull request #13809 from asheplyakov/18321-bp-jewel
+    
+    jewel: librbd: remove image header lock assertions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 37ab19cc29d61970e08af1b4627137cfcaa99474
+Merge: f7c04e3ca6 8d0140a9ed
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:41:56 2017 +0200
+
+    Merge pull request #13107 from smithfarm/wip-18669-jewel
+    
+    jewel: tests: [  FAILED  ] TestLibRBD.ImagePollIO in upgrade:client-upgrade-kraken-distro-basic-smithi
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7c04e3ca69c7fc134b267e825277eccea228c9b
+Merge: d2909bdb4c a18a2dd108
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:39:26 2017 +0200
+
+    Merge pull request #13585 from asheplyakov/jewel-bp-16585
+    
+    jewel: msg: set close on exec flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d2909bdb4cf135de850cc865f735cc61eb3d4ea6
+Merge: cd748603da 6d47615c11
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:56:22 2017 +0200
+
+    Merge pull request #14371 from tchaikov/wip-19429-jewel
+    
+    jewel: tests: clone workunit using the branch specified by task
+    
+    Nathan Cutler <ncutler@suse.com>
+
+commit cd748603da43c0b7377b5ef07aff618711b4879c
+Merge: 1a20c12355 dd25a8f36b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:54:11 2017 +0200
+
+    Merge pull request #14325 from shinobu-x/wip-18619-jewel
+    
+    jewel: osd: degraded and misplaced status output inaccurate
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1a20c123556e2f566af7e87ab3da6fc2ca328bb9
+Merge: 4838c4db4c 7fdf4d41c5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:53:25 2017 +0200
+
+    Merge pull request #14236 from smithfarm/wip-19392-jewel
+    
+    jewel: mon: remove bad rocksdb option
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4838c4db4c610926edc599677cb7e74f3bdc3077
+Merge: e26b7033eb 3860ccf16d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:52:41 2017 +0200
+
+    Merge pull request #14181 from mslovy/wip-19394-jewel
+    
+    jewel: osd: bypass readonly ops when osd full
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e26b7033ebd4a09210d482a54677806fe2c03505
+Merge: 389150bcc3 819af9e413
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:49:49 2017 +0200
+
+    Merge pull request #14113 from shinobu-x/wip-19319-jewel
+    
+     jewel: cli: RadosImport::import should return an error if Rados::connect fails
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 389150bcc376702022cf15c0c79dbe856a97dffa
+Merge: a8b10082f5 90de64bd81
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:45:33 2017 +0200
+
+    Merge pull request #14047 from asheplyakov/reindex-on-pg-split
+    
+    jewel: osd: reindex properly on pg log split
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a8b10082f5761ea1132b85bd916a84f87e0276e2
+Merge: 32ed9b7897 ae498e84ff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:45:15 2017 +0200
+
+    Merge pull request #14044 from mslovy/wip-19311-jewel
+    
+    jewel: core: os/filestore: fix clang static check warn use-after-free
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 32ed9b789733e21a2d597c7016eabf95763e50ae
+Merge: 6705e911a2 335b5fa4a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:44:25 2017 +0200
+
+    Merge pull request #13932 from asheplyakov/18911-bp-jewel
+    
+    jewel: rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6705e911a229cee50d5ac992c7dd9af834f2400d
+Merge: 3d21a0080d 714eb863c3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:42:13 2017 +0200
+
+    Merge pull request #13831 from jan--f/wip-19206-jewel
+    
+    jewel: fs: Invalid error code returned by MDS is causing a kernel client WARNING
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 3d21a0080dfd00c0b89ed23fd8049e46c3041af4
+Merge: 8a6d64331b 6b5322c5f6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:41:36 2017 +0200
+
+    Merge pull request #13827 from tchaikov/wip-19185-jewel
+    
+    jewel: osd: ReplicatedPG: try with pool's use-gmt setting if hitset archive not found
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8a6d64331b73a4a24a59f4dcdb6771f1dc638879
+Merge: f96392a589 cebba011e5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:40:09 2017 +0200
+
+    Merge pull request #13788 from shinobu-x/wip-18235-jewel
+    
+    jewel: core: os/filestore/HashIndex: be loud about splits
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f96392a58970ab1e81653d5657d3d680120a02b6
+Merge: 8fe6ffcfef 1d054c3856
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:39:27 2017 +0200
+
+    Merge pull request #13786 from shinobu-x/wip-19129-jewel
+    
+    jewel: build/ops: ceph-base package missing dependency for psmisc
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8fe6ffcfef8918fd8634e87255ad3fe7a544aa16
+Merge: 6f589a159e b249fd5bd8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:38:34 2017 +0200
+
+    Merge pull request #13732 from liewegas/wip-19119-jewel
+    
+    jewel: doc: PendingReleaseNotes: note about 19119
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 6f589a159e4eb5f6c96634a742acaf6cd6e174c2
+Merge: b8f2d35884 8c7a1df251
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:36:21 2017 +0200
+
+    Merge pull request #13541 from shinobu-x/wip-18929-jewel
+    
+     jewel: osd: restrict want_acting to up+acting on recovery completion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit b8f2d35884a52586a28f1ff4eaf99c8c3ba1c43f
+Merge: 40d1443d68 2d17092fab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:34:54 2017 +0200
+
+    Merge pull request #13477 from asheplyakov/jewel-bp-18951
+    
+    jewel: osd: --flush-journal: sporadic segfaults on exit
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 40d1443d68b788737f83c10db3811bc37dbe8dda
+Merge: 50e863e0f4 07501dec6f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:33:25 2017 +0200
+
+    Merge pull request #13261 from shinobu-x/wip-18587-jewel
+    
+    jewel: mon: OSDMonitor: make 'osd crush move ...' work on osds
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 25e43ac25619d883d5a04e2df1cf6f57fea73fd0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 11 08:23:23 2017 +0200
+
+    tests: fix oversight in yaml comment
+    
+    When the file was copied from the hammer version, the word "hammer"
+    was not changed to "infernalis".
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    This cannot be cherry-picked from master because the test has been dropped.
+
+commit f2474042ecd6560323673170c13f2cb964406e70
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 2 21:20:08 2017 -0600
+
+    osdc/Objecter: resend RWORDERED ops on full
+    
+    Our condition for respecting the FULL flag is complex, and involves
+    the WRITE | RWORDERED flags vs the FULL_FORCE | FULL_TRY flags.  Previously,
+    we could block a read bc of RWORDRED but not resend it later.
+    
+    Fix by capturing the complex condition in a respects_full() bool and using
+    it both for the blocking-on-send and resending-on-possibly-notfull-later
+    checks.
+    
+    Fixes: http://tracker.ceph.com/issues/19133
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c4b73f19a7be13ff412eef804efcd8c18ed4dae6)
+
+commit 2d68822c784eb4d62d3b0198ed4ec04404dbffb3
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu May 5 23:44:57 2016 +0800
+
+    osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 07b2a22210e26eac1b2825c30629788da05e5e12)
+
+commit 2271cd81282f3f026316134cbab630f3e4f47782
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 3 13:37:17 2017 +1000
+
+    ceph-disk: Populate mount options when running "list"
+    
+    Also tidy up by moving duplicated code into a function
+    
+    Fixes: http://tracker.ceph.com/issues/17331
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7943ab2e01e24f2dfc5b6f1d3ffdc8a49e01af45)
+
+commit 65465356b5e7cb6cba67f8cbb81259d21e888dfb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Feb 16 12:56:34 2016 -0700
+
+    debian: replace SysV rbdmap with systemd service
+    
+    Stop shipping /etc/init.d/rbdmap in the Debian packages. Ship the
+    rbdmap.service systemd unit file instead.
+    
+    The corresponding change has already been made for RPMs, in
+    9224ac2ad25f7d017916f58b642c0ea25305c3e5.
+    
+    For Upstart-based systems (eg Ubuntu Trusty), the Debian packages
+    already contain rbdmap.conf.
+    
+    (This gets us a tiny bit closer to being able to remove the rbdmap SysV
+    script from our tree entirely.)
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 839807118dda2fb4d57ed9d50ec46e3ee0e2820a)
+    
+    Conflicts:
+        debian/ceph-common.install (retain /etc/init.d/rbdmap so jewel users can choose sysvinit or systemd)
+        debian/rules (retain /etc/init.d/rbdmap so jewel users can choose sysvinit or systemd)
+
+commit 6d47615c11a216733ba368f0dbd9a0a1b9c8fd35
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Apr 1 23:04:22 2017 +0800
+
+    qa/tasks/workunit.py: use "overrides" as the default settings of workunit
+    
+    otherwise the settings in "workunit" tasks are always overridden by the
+    settings in template config. so we'd better follow the way of how
+    "install" task updates itself with the "overrides" settings: it uses the
+    "overrides" as the *defaults*.
+    
+    Fixes: http://tracker.ceph.com/issues/19429
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 47080150a17d238f38d9da824d227393ad767aad)
+
+commit fdc71e75cd6361be49c4c20e77d3bdff017b38bf
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 30 12:37:01 2017 +0800
+
+    tasks/workunit.py: specify the branch name when cloning a branch
+    
+    c1309fb failed to specify a branch when cloning using --depth=1, which
+    by default clones the HEAD. and we can not "git checkout" a specific
+    sha1 if it is not HEAD, after cloning using '--depth=1', so in this
+    change, we dispatch "tag", "branch", "HEAD" using three Refspec classes.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 9ca7ccf5f1739f731da8bf31260594aea3a2932d)
+    
+    Conflicts:
+            qa/tasks/workunit.py (trivial resolution)
+
+commit 6c14a803894d71bae858705855551a44bdf9bf67
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Mar 28 20:08:13 2017 -0700
+
+    tasks/workunit.py: when cloning, use --depth=1
+    
+    Help avoid killing git.ceph.com.  A depth 1 clone takes about
+    7 seconds, whereas a full one takes about 3:40 (much of it
+    waiting for the server to create a huge compressed pack)
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit c1309fbef300a062138ac40eb5d3e5081b833072)
+
+commit f8aa6be06cc628b1cf64c9196c30045d020d657e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 7 06:59:13 2017 +0200
+
+    build/ops: rpm: move $CEPH_EXTRA_CONFIGURE_ARGS to right place
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bb3eb4284fe74cbf78e4a406c7b5f67a8e3c84b3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 7 06:48:51 2017 +0200
+
+    build/ops: rpm: explicitly provide --with-ocf to configure
+    
+    Fixes: http://tracker.ceph.com/issues/19546
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    (Note: This cannot be cherry-picked because master uses cmake, but
+    the fix does bring the jewel spec file into better alignment its master
+    counterpart, at least as far as this one little bit is concerned.)
+
+commit 30c952735327d05b6049e9364c1a053ebf651751
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Mon Feb 27 15:52:43 2017 +0800
+
+    rgw: use separate http_manager for read_sync_status
+    
+    concurrent users of read_sync_status() use different cr managers, when get_resource must
+    use http_manager related to the cr manager.
+    
+    Fixes: http://tracker.ceph.com/issues/19236
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c412024889f8995d98096ac863bafee71624bd70)
+
+commit 87cb8474f523be1b281882c46a8a3597977a51c9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 28 20:20:29 2016 -0400
+
+    rgw: pass cr registry to managers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ef4d7eab11fb5d2a41c9c28b9eb8b075aaff0d05)
+    
+     Conflicts:
+            src/rgw/rgw_rados.cc: removed ref to RGWSyncLogTrimThread (not backported)
+
+commit 1a6d7c0506d4bcda775dda05bb357d4d5695dabb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Jul 22 11:00:16 2016 -0400
+
+    rgw: use separate cr manager for read_sync_status
+    
+    RGWCoroutinesManager::run() is not reentrant, so concurrent users of
+    read_sync_status() must use different managers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6b1e40d7a21c549b55e6576ec56973c8d3c092d1)
+
+commit c466adee9c8f7a97ff7e99bee56da8ce51bf0f00
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 21 23:46:20 2016 -0400
+
+    rgw: change read_sync_status interface
+    
+    RGWDataSyncStatusManager::read_sync_status() now operates on the given
+    parameter, rather than its internal member variable. this allows
+    multiple concurrent readers, which is needed for the rest interface
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b7cd4e0e8b879b5e528de75bea3307585b96cbf2)
+
+commit 36921a3f3d01547508dc6270f6b19f2576e067d9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 8 11:24:11 2016 -0400
+
+    rgw: don't ignore ENOENT in RGWRemoteDataLog::read_sync_status()
+    
+    rest handlers for sync status need to return ENOENT errors. the only
+    other callers are in radosgw-admin, so the ENOENT errors are ignored at
+    those call sites instead
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2cc533b30c0f23c0750ea8d02c51b3b3d3b4821a)
+
+commit b249fd5bd816a63b445db12c8f846cfda199c8b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:18:44 2017 -0600
+
+    PendingReleaseNotes: warning about 'osd rm ...' and #19119
+    
+    See http://tracker.ceph.com/issues/19119
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit be96003c464481d8e84825178d600234a0d64d22)
+
+commit 335258f975a8e8539774e8cb22690d746ec90d9f
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Mar 28 11:49:08 2017 +0200
+
+    osdc/Objecter: respect epoch barrier in _op_submit()
+    
+    Epoch barrier instructs us to avoid sending (i.e. pause) any OSD ops
+    until we see a barrier epoch.  The only thing epoch_barrier check in
+    target_should_be_paused() does is keep already paused ops paused.  We
+    need to actually pause incoming OSD ops in _op_submit().
+    
+    Fixes: http://tracker.ceph.com/issues/19396
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit f8e8efc0a53d7bd807cc0c2178aef7c4bed62ab7)
+
+commit a20d2b89ee13e311cf1038c54ecadae79b68abd5
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Fri Mar 31 14:54:33 2017 +0200
+
+    ceph-disk: Adding retry loop in get_partition_dev()
+    
+    There is very rare cases where get_partition_dev() is called before the actual partition is available in /sys/block/<device>.
+    
+    It appear that waiting a very short is usually enough to get the partition beein populated.
+    
+    Analysis:
+    update_partition() is supposed to be enough to avoid any racing between events sent by parted/sgdisk/partprobe and
+    the actual creation on the /sys/block/<device>/* entrypoint.
+    On our CI that race occurs pretty often but trying to reproduce it locally never been possible.
+    
+    This patch is almost a workaround rather than a fix to the real problem.
+    It offer retrying after a very short to be make a chance the device to appear.
+    This approach have been succesful on the CI.
+    
+    Note his patch is not changing the timing when the device is perfectly created on time and just differ by a 1/5th up to 2 seconds when the bug occurs.
+    
+    A typical output from the build running on a CI with that code.
+            command_check_call: Running command: /usr/bin/udevadm settle --timeout=600
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_partition_dev: Try 1/10 : partition 2 for /dev/sda does not in /sys/block/sda
+            get_partition_dev: Found partition 2 for /dev/sda after 1 tries
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_dm_uuid: get_dm_uuid /dev/sda2 uuid path is /sys/dev/block/8:2/dm/uuid
+    
+    fixes: #19428
+    
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 93e7b95ed8b4c78daebf7866bb1f0826d7199075)
+
+commit 2d5d0aec60ec9689d44a53233268e9b9dd25df95
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Wed Mar 22 10:11:44 2017 +0100
+
+    ceph-disk: Reporting /sys directory in get_partition_dev()
+    
+    When get_partition_dev() fails, it reports the following message :
+        ceph_disk.main.Error: Error: partition 2 for /dev/sdb does not appear to exist
+    The code search for a directory inside the /sys/block/get_dev_name(os.path.realpath(dev)).
+    
+    The issue here is the error message doesn't report that path when failing while it might be involved in.
+    
+    This patch is about reporting where the code was looking at when trying to estimate if the partition was available.
+    
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 413c9fcfbe8e6ab33d73b8428090ccacc33c5d15)
+
+commit bcd3c906e5b57e7f44df1963c6e11b78ff89482c
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Mar 13 23:41:58 2017 +0800
+
+    osd: don't share osdmap with objecter when preboot
+    
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit a5a3644eecc49b4eea890c6999fe87536495dcbe)
+
+commit dd25a8f36bef1901f3ce6193cfcbdaf7ab2424a1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 18 08:33:40 2017 -0800
+
+    osd: Calculate degraded and misplaced more accurately
+    
+    Calculate num_object_copies based on the larger of pool size,
+    up set size and acting set size.
+    
+    Calculate num_objects_degraded as the difference between num_object_copies
+    and all copies found on acting set and backfilling up set OSDs.
+    
+    Calculate num_objects_misplaced as all copies on acting set OSDs not in up set
+    less copies that have been backfilled to up set OSDs.
+    
+    Fixes: http://tracker.ceph.com/issues/18619
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8423bc40759cca137f61e7b755411719a84369d4)
+
+commit b5b441abaa852e85ddefd8b22835c9b85898cc06
+Author: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+Date:   Wed Mar 22 16:13:50 2017 +0100
+
+    common: fix segfault in public IPv6 addr picking
+    
+    sockaddr is only 16 bytes big, so declaring net as sockaddr
+    and then casting to sockaddr_in6 in case of IPv6 cannot
+    work.
+    
+    using sockaddr_storage works for both IPv4 and IPv6, and is
+    used in other code parts as well.
+    
+    note that the tests did not find this issue as they declared
+    the bigger structs and casted the references to (sockaddr *)
+    
+    Fixes: http://tracker.ceph.com/issues/19371
+    Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+    (cherry picked from commit ae2ee3d3835fe25b35eeb1a841ee5234cd69eb65)
+
+commit 7fdf4d41c5bef14269cb302301a08d5a3a57a768
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 15:29:12 2016 -0400
+
+    mon: remove bad rocksdb option
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0ac671ece258e509f71a05253e62a9878e279840)
+
+commit 3860ccf16d7dfb137face9886c3d7f29cd527835
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu May 5 23:07:06 2016 +0800
+
+    osd: bypass readonly ops when osd full.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit e2a0ae8e88e6b7354b14adb503fd8ba8525bee39)
+    See: http://tracker.ceph.com/issues/19394
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit b85677397ef9a3fe16c087e67d3f752851bbe070
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 21 15:31:43 2016 -0500
+
+    qa/workunits/rbd: resolve potential rbd-mirror race conditions
+    
+    Fixes: http://tracker.ceph.com/issues/18935
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 63eae97afc1a92412525468263fb8696a243ebac)
+
+commit 1cc8d0d08560af4e8785d7c4c6a925b1fc9f988e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Feb 8 23:47:57 2017 +0100
+
+    librbd: Include WorkQueue.h since we use it
+    
+    We use m_work_queue of type ContextWQ in handle_update function but we
+    do not include common/WorkQueue.h that defines ContextWQ. This results
+    in dereference of an incomplete type and causes build error in latest
+    Fedora rawhide (future 26).
+    
+    Fixes: http://tracker.ceph.com/issues/18862
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 480f82847ad1fc7959f1fe5a90761a5a24550993)
+
+commit ee06517547ae174472d739f966c0a27d3a97d742
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 18 20:54:22 2017 -0500
+
+    librbd: avoid possible recursive lock when racing acquire lock
+    
+    Fixes: http://tracker.ceph.com/issues/17447
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e46e8eb664f573bd70ae7c96a6d9a98b0deb09e)
+
+commit f6489d01ca41d6979b5de28e3cde6b43fcaa8edb
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Mon Jan 30 13:03:20 2017 +0100
+
+    rbd: destination pool should be source pool if it is not specified
+    
+    Currently if user perform image rename operation and user give pool
+    name as a optional parameter (--pool=<pool_name>) then currently
+    its taking this optional pool name for source pool and making
+    destination pool name default pool name.
+    With this fix if user provide pool name as a optional pool name
+    parameter then it  will consider both soruce and destination pool
+    name as optional parameter pool name.
+    
+    Fixes: http://tracker.ceph.com/issues/18326
+    
+    Reported-by: М�РК КОРЕ�БЕРГ <socketpair@gmail.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 01f23aa99fb694da326ab408e75b33c640ce660b)
+
+commit 9e123e6d6c8bbd54514b498df5a22d961f0cefbb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Mar 12 12:11:28 2017 +0200
+
+    rgw: use rgw_zone_root_pool for region_map like is done in hammer
+    
+    Fixes: http://tracker.ceph.com/issues/19195
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c91dd6d9efd148e0fe0f027dde537e977de9aa26)
+
+commit e2ee70a8ad51992bbd763d2465f6d8a01dad6a31
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 13:03:24 2017 +0200
+
+    rgw: skip conversion of zones without any zoneparams
+    
+    Fixes: http://tracker.ceph.com/issues/19231
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 36cf5a5c8179c6313346b2e29286c537c6fefce8)
+
+commit c7d292bf6714d7aaf10412e5109badb90f5dc208
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 11:16:26 2017 +0200
+
+    rgw: better debug information for upgrade
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e9f3bf8eab1dd46a92f54b0f7afe1f4c0e4204db)
+
+commit 11f5c841c7698e6239017478fa05f742b7c0ab1c
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jan 31 18:01:32 2017 +0100
+
+    rgw/rgw_rados.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 7086cf9a73f2ec1eb96c0e752beb1b74fca18570)
+
+commit 819af9e4139997cd845dc24a137d43218d8a40a8
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Mar 21 12:22:20 2017 +1000
+
+    tools/rados: Check return value of connect
+    
+    Fail gracefully if Rados::connect returns an error.
+    
+    Fixes: http://tracker.ceph.com/issues/19319
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit c119091ef0844e4a1ddd790a8bfef8f06bb57d58)
+
+commit 72e2476a130e14abcd541ff61328454cb69ad9c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 11:33:27 2017 +0800
+
+    brag: count the number of mds in fsmap not in mdsmap
+    
+    this change was introduced in 4e9b953
+    
+    Fixes: http://tracker.ceph.com/issues/19192
+    Signed-off-by: Peng Zhang <zphj1987@gmail.com>
+    (cherry picked from commit 2d25a9c0c760664d3de33ecca0e0272c1031cd46)
+
+commit 3cb192779a1c3662d27bba7715eb31c5f7b6a5b7
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Thu Jun 30 00:51:50 2016 +0300
+
+    brag: Assume there are 0 MDS instead of crashing when data is missing
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+
+commit 8bed107b84efdc8c735245cdfb51bfd8d07da13b
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Fri Feb 10 19:19:46 2017 +0100
+
+    doc: update description of rbdmap unmap[-all] behaviour
+    
+    Fixes: http://tracker.ceph.com/issues/18884
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit f987396e126d5e61240a6645ffed439f79b072b4)
+
+commit da4e0b56c60f4bc2c67daa5dfe4d5255ab8bfc03
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Dec 15 18:23:41 2016 +0100
+
+    doc: add verbiage to rbdmap manpage
+    
+    Fixes: http://tracker.ceph.com/issues/18262
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fbac4a081547d83bb2436cd60b0b7ee7250f8a6c)
+
+commit 167d4fd7ccf0cdac536f95250bbfa3e9879ab769
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Fri Feb 10 17:50:12 2017 +0100
+
+    rbdmap: unmap RBDMAPFILE images unless called with unmap-all
+    
+    When called with a "map" parameter, the rbdmap script iterates the list
+    of images present in RBDMAPFILE (/etc/ceph/rbdmap), and maps each entry.
+    When called with "unmap", rbdmap currently iterates *all* mapped RBD
+    images and unmaps each one, regardless of whether it's listed in the
+    RBDMAPFILE or not.
+    
+    This commit adds functionality such that only RBD images listed in the
+    configuration file are unmapped. This behaviour is the new default for
+    "rbdmap unmap". A new "unmap-all" parameter is added to offer the old
+    unmap-all-rbd-images behaviour, which is used by the systemd service.
+    
+    Fixes: http://tracker.ceph.com/issues/18884
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit e58413abf408cbe254232e563f3e30d2dc0d707c)
+
+commit 39aab763a44e45e025c311cdfff95116df11a4c4
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Dec 15 13:01:02 2016 +0100
+
+    Revert "dummy: reduce run time, run user.yaml playbook"
+    
+    This reverts commit d4e3cec1851ae35889127b90912e133178085bc6.
+    
+    Fixes: http://tracker.ceph.com/issues/18259
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a8a2a8c3e68b910dbaeb3186576898bf9f89f9fd)
+
+commit 2e50fe1684f73ebe96969c341242b6f20c8470a0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 20 15:01:01 2016 -0400
+
+    rgw: fix break inside of yield in RGWFetchAllMetaCR
+    
+    the yield macro is implemented with for/switch, so the breaks in
+    RGWFetchAllMetaCR weren't being applied to the for loop as expected -
+    so any of these breaks send RGWFetchAllMetaCR into an infinite loop
+    
+    removed the yield {} block, so that breaks will apply to the for loop as
+    intended, then added a single yield; statement to allow the
+    entries_index consumer to run one per iteration
+    
+    Fixes: http://tracker.ceph.com/issues/17655
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 190bd385a7be52867d65740c410884f5c8cbc21f)
+
+commit dc4e7a1a865ea0ae7362c1b6a7a542aa5f72107d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 12:10:40 2017 -0500
+
+    rgw: delete_system_obj() fails on empty object name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 67401193f871db95a6045915fa59dce8c5dd1012)
+
+commit e9a577c8c535702d1eb285429978bdbb395e2d5c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 11:42:45 2017 -0500
+
+    rgw: if user.email is empty, dont try to delete
+    
+    Fixes: http://tracker.ceph.com/issues/18980
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 022ecf0fcc8e44912c8758ee1d9a452dc23cbbce)
+
+commit 90de64bd81fedcb9540e40d50420e169a4a81248
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Mar 20 14:05:17 2017 +0400
+
+    jewel: osd/PGLog: reindex properly on pg log split
+    
+    When PGLog::IndexedLog::split_into runs it builds the list, which means
+    the old indices are wrong (point to bad memory), but index() will not
+    rebuild them because ever since b858e86 we won't rebuild them if they
+    are already built. Fix that by calling unindex() before the split.
+    
+    Based on 643ae42cf27f16dd6ed4e1402acc0483bb9fca74. Notice that both
+    the child and the parent log are re-indexed in Jewel, so the only
+    problem is missing unindex().
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ae498e84ffcff7424721f0d2704ec739d1cc092d
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Tue Dec 20 13:21:40 2016 +0800
+
+    os/filestore: fix clang static check warn use-after-free
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 1d359455b3dd6abb383542ba596a03f14ac54dbd)
+    See: http://tracker.ceph.com/issues/19311
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit 335b5fa4a9694620546422f9a02bdcc16549d7cc
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 2 11:11:35 2017 +0100
+
+    rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+    
+    Fixes: http://tracker.ceph.com/issues/18335
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 596e5ea8a5df72002672eef0a6d20572ca6f60f0)
+    
+    Conflicts:
+            qa/workunits/rbd/rbd-nbd.sh: the original commit removes
+              TOO_LARGE_IMAGE test, do the same thing
+            src/tools/rbd_nbd/rbd-nbd.cc: help git to add
+              "include/stringify.h"
+    
+    Other changes:
+            src/tools/rbd_nbd/rbd-nbd.cc: #include <fstream> so
+              the code compiles
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ced799f9c6558482d538f8dec854c62162685ad0
+Author: Michal Koutný <mkoutny@suse.com>
+Date:   Thu Jan 26 16:08:09 2017 -0500
+
+    rgw: Use decoded URI when verifying TempURL
+    
+    Instead of calliing url_decode directly, we reuse s->decoded_uri that is
+    initialized in RGWREST::preprocess().
+    
+    Fixes: http://tracker.ceph.com/issues/18590
+    
+    Adapted from 4e1318f4dcbfd64c3ec94f4addf6e38ddd6c013a. Cherry-picking
+    that patch requires a quite a number of unrelated changes, hence this
+    patch does s/s->info.request_uri/s->decoded_uri/ to keep the fix as
+    minimal as possible.
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 043d70461c1eb874d9185f9bd671930fad05ff65
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Mar 1 17:19:04 2017 +0100
+
+    osd/OSDMap: don't set weight to IN when OSD is destroyed
+    
+    Since commit 4e28f9e63644 ("osd/OSDMap: clear osd_info, osd_xinfo on
+    osd deletion"), weight is set to IN when OSD is deleted.  This changes
+    the result of applying an incremental for clients, not just OSDs.
+    Because CRUSH computations are obviously affected, pre-4e28f9e63644
+    servers disagree with post-4e28f9e63644 clients on object placement,
+    resulting in misdirected requests.
+    
+    Fixes: http://tracker.ceph.com/issues/19119
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a6009d1039a55e2c77f431662b3d6cc5a8e8e63f)
+
+commit 6b5322c5f62f4b90c4206c6ddcc70d090fa7eeb9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Mar 7 18:49:46 2017 +0800
+
+    osd/ReplicatedPG: try with pool's use-gmt setting if hitset archive not found
+    
+    due to http://tracker.ceph.com/issues/19185, a hammer OSD could store
+    pg_hit_set_info_t with "use-gmt = true" even the pool setting is false.
+    so we use the pool setting as a fallback if the hitset archive is not
+    found locally and the pool.use_gmt does not match with hitset.use_gmt.
+    
+    Fixes: http://tracker.ceph.com/issues/19185
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts:
+      osd/ReplicatedPG.cc:  This cannot be cherry-picked from master
+        because: hammer should upgrade to jewel first before moving to a
+        higher version. so there is no necessary to include this workaround
+        in releases later than jewel.
+
+commit d30c4d55ad52e2b63fdbd06ce256d92fc3fd36c9
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Feb 16 17:40:50 2017 +0100
+
+    doc: rgw: make a note abt system users vs normal users
+    
+    Mention that system users don't behave like normal users in context of
+    normal rest operations
+    
+    Fixes: http://tracker.ceph.com/issues/18889
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a47bcf70c9f51a6601b809cba219f5615b204d34)
+    
+    Conflicts:
+            doc/radosgw/multisite.rst (trivial whitespace difference)
+
+commit 5ee8feaba469886f9e3bd3909475ffef62ba261d
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 7 14:37:36 2017 +0530
+
+    rgw: Let the object stat command be shown in the usage
+    
+    Fixes: http://tracker.ceph.com/issues/19013
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 0fe76f83d19be098ef54fb0492a376fef3aa9e23)
+
+commit 754b4a482cb0369215beed58103a1e241231cf77
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 21 16:33:29 2017 +0530
+
+    rgw: Correct the return codes for the health check feature
+    Fixes: http://tracker.ceph.com/issues/19025
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    
+    (cherry picked from commit 4da2bf310f6d43423554c32e43ebf90ad2c3f3a9)
+
+commit 9cd7dd84909abdb9e603ff3aeb9958cdab8c70ad
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Wed Feb 22 12:03:46 2017 +0530
+
+    rgw: Fixes typo in rgw_admin.cc
+    
+    Issue: http://tracker.ceph.com/issues/19026
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 58837ef6ce8cbcfc2cac29d5f833b2cf62d8737a)
+
+commit 85fbb00f6ef5f11bc5d615ccd8e2202ce3896fd1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 27 10:35:01 2017 -0800
+
+    rgw: don't init rgw_obj from rgw_obj_key when it's incorrect to do so
+    
+    Fixes: http://tracker.ceph.com/issues/19096
+    
+    rgw_obj_key currently deals with the bucket index key, and not
+    representing a (name, instance, ns) tupple. Need to initialize
+    it in two steps.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 392c5d9dae6ba699014ffe6e1e67818fa62d7e41)
+
+commit ec0668c201a71b4a17ef0ab3c5908f57229aa6ef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Nov 29 11:29:41 2016 -0500
+
+    rgw: fix for broken yields in RGWMetaSyncShardCR
+    
+    Fixes: http://tracker.ceph.com/issues/18076
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e62d48a9bf2e309eab1a863f167af5267ebcc371)
+
+commit 6afe3efa4b636ede1cd77086cb2e70ed09fa2e95
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon Jul 25 11:21:11 2016 +0200
+
+    rgw: kill a compile warning for rgw_sync
+    
+    killing the compile warning for
+    
+    ```
+    /ceph/src/rgw/rgw_sync.cc:1462:12:
+    warning: suggest explicit braces to avoid ambiguous ‘else’ [-Wparentheses]
+             if (can_adjust_marker) yield {
+                ^
+    ```
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 831640bb46621a6f003ad562cef7928ffa9a7ad3)
+
+commit 06916a8798439ec033294d791749ce7381d92f51
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon Feb 13 23:07:25 2017 +0530
+
+    rgw: change log level to 20 for 'System already converted' message
+    
+    Fixes: http://tracker.ceph.com/issues/18919
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 55b567c767830170d04de4cdc8f10aba30a3f379)
+
+commit 4c1f302f7d71bedb0dead220f17eeb84e7e3f737
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Jan 11 05:28:43 2017 +0800
+
+    rgw: the swift container acl should support field .ref
+    
+    On the openstack-swift. The container acl supports .ref, which is ignored on ceph swift.
+    
+    Fixes: http://tracker.ceph.com/issues/18484
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit b06f9cd9f0900db7b0d0fbcaea69cdd0d4b10132)
+    
+    Conflicts:
+            src/rgw/rgw_acl_swift.cc - no extract_referer_urlspec() in jewel
+                see https://github.com/ceph/ceph/pull/8657
+
+commit 714eb863c30df4e653068e6ea16630504e58b704
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Jun 14 19:32:01 2016 +0800
+
+    server: negative error code when responding to client
+    
+    As the comment suggests. Also a zero or positive return code
+    shall indicates a success, which does not match our intention here.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 26931f888ce4661765cca106b3a3dc66702266df)
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+
+commit 1a4e1e09b1e562bf97cfe96f5cb9f937b6987165
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 13 14:10:58 2016 -0500
+
+    librbd: remove image header lock assertions
+    
+    This assertions can sporadically fail if the watch is lost and
+    recovered in the background. Upon a true loss of the lock, the
+    client would either be blacklisted or it would have completed
+    all in-flight ops before releasing.
+    
+    Fixes: http://tracker.ceph.com/issues/18244
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce4f2a52ec0a794d89e7576b59c9b9aefe3db288)
+    
+    Conflicts:
+            src/librbd/operation/SnapshotCreateRequest.cc: rbd class
+               does not support the snapshot namespaces in Jewel, skip
+               the corresponding argument
+
+commit cebba011e502f7009208bbddc295eb17f88f1bb9
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Dec 9 22:06:26 2016 +0100
+
+    os/filestore/HashIndex: be loud about splits
+    
+    Filestore splits are a rare yet important enough event that an
+    OSD should visibly report when they happen.
+    
+    Without this reporting an operator could spend hours trying to
+    understand the cause of any split-induced slow requests.
+    
+    Fixes: http://tracker.ceph.com/issues/18235
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 61c47acd3a1f3e01f0106d4a541bb7f28a1301d8)
+
+commit 1d054c3856a63ceebe44f66ff83fda691c374f71
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 2 12:41:07 2017 +0100
+
+    build/ops: add psmisc dependency to ceph-base (deb and rpm)
+    
+    Fixes: http://tracker.ceph.com/issues/19129
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 769b695465162bc8424abf8e2f259e6765b5bbff)
+    
+    Conflicts:
+            debian/control (jewel does not have f11acf2b 7e71cd2c)
+
+commit 6add2a457e2826b71c0e9e82c6f6686cecbc4584
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Nov 25 21:31:22 2016 +0800
+
+    rgw: metadata sync info should be shown at master zone of slave zonegroup
+    
+    When executing 'radosgw-admin sync status', the metadata sync info should be shown on the srceen at master zone of slave zonegroup.
+    
+    Using the function store->is_meta_master() instead of 'zonegroup.is_master && zone.id == zonegroup.master_zone'
+    
+    Fixes: http://tracker.ceph.com/issues/18091
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit c12d0af2f98b246a73cc3ee027449a22192795b3)
+
+commit 0e11a938c5c9acd8a50efa9a154ea3bf21bcafc5
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jan 25 12:39:40 2017 +0100
+
+    systemd: Start OSDs after MONs
+    
+    Currently, we start/stop OSDs and MONs simultaneously. This may cause
+    problems especially when we are shutting down the system. Once the mon
+    goes down it causes a re-election and the MONs can miss the message
+    from the OSD that is going down.
+    
+    Resolves: http://tracker.ceph.com/issues/18516
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7f4acf45dd0d86e7d9992a8c30e5876fb57b1914)
+    
+    Conflicts:
+            systemd/ceph-osd@.service (jewel does not have 4179aa8d)
+
+commit 3bdd4398f1dcad0b7e22f1750ca524b97feca15a
+Author: yaoning <yaoning@unitedstack.com>
+Date:   Mon Jun 6 13:31:52 2016 +0800
+
+    osd: preserve allocation hint attribute during recovery
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+    (cherry picked from commit e15be792960da6bac2bd469acf7d30007be61781)
+    
+    Conflicts:
+        src/osd/ReplicatedBackend.cc (in master, it contains alloc_hint_flags for set_alloc_hint)
+        src/osd/ReplicatedPG.cc (in master, it contains alloc_hint_flags in object_info_t struct)
+        src/osd/osd_types.cc (in master, it contains alloc_hint_flags in message serialization)
+        alloc_hint_flags is used in master bluestore, filestore does not use alloc_hint_flags.
+        therefore, remove alloc_hint_flags here in jewel
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit 8d0140a9eda814beadf1f59c9b4205f30a1d2e35
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 24 09:24:52 2017 -0500
+
+    librbd: improve debug logging for lock / watch state machines
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cc046597983bd491cc66081cc33d9046264fe24b)
+    
+    Conflicts:
+        NOTE: cherry-picked from kraken commit to avoid conflicts
+
+commit 62ce3461c3b205eaa9062113526cf572184d0a27
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 23 21:24:41 2017 -0500
+
+    test: use librados API to retrieve config params
+    
+    The CephContext object is not ABI-stable, so it is necessary to
+    use the "conf_get" librados methods to safely retrieve a setting.
+    
+    Fixes: http://tracker.ceph.com/issues/18617
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ed02f3cd56bf89984c3538ac3f21ec2321cd3b7)
+    
+    Conflicts:
+            src/test/librbd/test_librbd.cc (jewel does not have
+                006138e2d80b779d8c15b141002bb4b3852f6c4a or
+                cb3712e08cdc2c37a983b479f4692bbdfe83b220)
+
+commit 01d04e28db7c2969b86df5b38a20b9eb156cf393
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Feb 2 23:23:54 2017 +0100
+
+    tests: Thrasher: eliminate a race between kill_osd and __init__
+    
+    If Thrasher.__init__() spawns the do_thrash thread before initializing the
+    ceph_objectstore_tool property, do_thrash races with the rest
+    of Thrasher.__init__() and in some cases do_thrash can call kill_osd() before
+    Trasher.__init__() progresses much further. This can lead to an exception
+    ("AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'")
+    being thrown in kill_osd().
+    
+    This commit eliminates the race by making sure the ceph_objectstore_tool
+    attribute is initialized before the do_thrash thread is spawned.
+    
+    Fixes: http://tracker.ceph.com/issues/18799
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b519d38fb1967628ad8a1c46fcfb3f984de58790)
+    
+    Conflicts:
+        qa/tasks/ceph_manager.py (jewel has only one if statement after
+            "self.thread = gevent.spawn(self.do_thrash)" while master has four;
+            jewel lacks 66836c957ffd974dec136997e23261ec7de2f0aa which disables
+            ceph-objectstore-tool testing in master)
+
+commit 08a667883b68ccc72e3a4bc3013856deef1df93d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Feb 18 14:33:25 2017 +0100
+
+    rpm: build ceph-resource-agents by default
+    
+    To align with debian build
+    
+    Fixes: http://tracker.ceph.com/issues/17613
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 3e157bf16c3020ac11cb26df5df3ed331faf3c25)
+
+commit d22becab0f2c541584ce891d392760a5c4f1d153
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Thu Jul 14 19:10:29 2016 +0800
+
+    msg/simple: cleanups
+    
+    should save the `errno` which may be changed by `ldout` and/or `<<` operator
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 91a29bc490fdfbbef0875fa620c7ba1a1a6492ae)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit a18a2dd108678d2e4b57e08b559c1f9a262d6923
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 17 13:58:55 2016 +0800
+
+    msg/simple: set close on exec on server sockets
+    
+    mds execv() when handling the "respawn" command, to avoid fd leakage,
+    and enormous CLOSE_WAIT connections after respawning, we need to set
+    FD_CLOEXEC flag for the socket fds.
+    
+    Fixes: http://tracker.ceph.com/issues/16390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f019ad563ce90f5aea0d8dd8b7b98688441596e0)
+
+commit 91a968b8fc7b363cae351b8648259211a1e71d18
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 17 01:17:05 2016 +0800
+
+    msg/async: set close on exec on server sockets
+    
+    mds execv() when handling the "respawn" command, to avoid fd leakage,
+    and enormous CLOSE_WAIT connections after respawning, we need to set
+    FD_CLOEXEC flag for the socket fds.
+    
+    Fixes: http://tracker.ceph.com/issues/16390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit eaf68c724144d07f9506037a14d9192cb9f16d70)
+    
+    Conflicts:
+            src/msg/async/AsyncMessenger.cc: Processor::accept(): applied
+                    the hunk manually (invoke set_close_on_exec on a socket
+                    returned by accept)
+
+commit 547e867628975c7144590e9332aa62b0ef82a433
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 12:12:57 2017 -0500
+
+    librbd: possible deadlock with flush if refresh in-progress
+    
+    Fixes: http://tracker.ceph.com/issues/18419
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b95f92a5572d3035c20eba07e76d2c825a9853f7)
+    
+    Conflicts:
+            src/librbd/ImageState.h (master commit just adds a function
+                          declaration, so just add it to jewel as well)
+
+commit 07501dec6f1c70afd4e4c2a50d7f874c39f2220b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 18 17:02:54 2017 -0600
+
+    mon/OSDMonitor: make 'osd crush move ...' work on osds
+    
+    Currently it only allows you to move buckets, which is annoying and much
+    less useful.  To move an OSD you need to use create-or-move, which is
+    harder to use.
+    
+    Fixes: http://tracker.ceph.com/issues/18587
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47956475dea8bb8e07331dd76344a60b776b5158)
+    
+    Conflicts:
+        qa/workunits/mon/crush_ops.sh: adapted "ceph osd find" to jewel syntax
+
+commit 7c6c3c753ccdd3baea834338e1a761f05b4e0a12
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Feb 16 23:51:11 2017 +0530
+
+    auth: 'ceph auth import -i' overwrites caps, if caps are not specified
+    in given keyring file, should alert user and should not allow this import.
+    Because in 'ceph auth list' we keep all the keyrings with caps and importing
+    'client.admin' user keyring without caps locks the cluster with error[1]
+    because admin keyring caps are missing in 'ceph auth'.
+    
+    [1] Error connecting to cluster: PermissionDeniedError
+    
+    Fixes: http://tracker.ceph.com/issues/18932
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 90144aa64c11a685b6a7cb3aafea75d427f569be)
+
+commit 8c7a1df251e8289e7cf2df5b3096b91d8640695d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Feb 14 15:00:09 2017 -0500
+
+    osd/PG: restrict want_acting to up+acting on recovery completion
+    
+    On recovery completion we recalculate want_acting to see if we
+    should add recently backfilled osds into acting.  However, at
+    this point we may have gotten infos from others OSDs outside
+    of up/acting that could be used for want_acting.  We currently
+    assert that only up/acting osds are used in
+    PG::RecoveryState::Active::react(const AdvMap&), so we must
+    restrict want_acting to up/acting here.
+    
+    We could remove this restriction, but it would mean
+    
+    1) checking on every map change that want_acting hasn't been
+    invalidated, and if so, recalculating want_acting and requesting
+    a new pg_temp.  Also, presumably
+    
+    2) on each new info, checking whether we can construct a better
+    want_acting, and if so, doing it.
+    
+    That would be a good thing, but is a more complicated change.  In
+    reality this case comes up very rarely, so simply make our
+    post-recovery want_acting calculation limit itself to up+acting.
+    
+    See 1db67c443d84dc5d1ff53cc820fdfd4a2128b680 for the assertion.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f2dee9aa48a00a7f2f809cd4d20e98df771da81)
+
+commit cfa37d6a1674e3f6f8eef4d8519823a7af70df01
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+    
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+    
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 2d17092fab8080f819369d74d4c76d8ae58d899b
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Feb 7 16:47:45 2017 +0400
+
+    ceph-osd: --flush-journal: sporadic segfaults on exit
+    
+    FileStore holds a number of recources like op thread pool and work
+    queue, key/value DB threads, etc. These should be properly stopped
+    (released) before exiting to avoid segfaults on exit.
+    
+    Note: more code paths (mkfs, dump_journal, etc) need similar fixes,
+    these will be submitted as separate patches.
+    
+    Fixes: http://tracker.ceph.com/issues/18820
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 00184814c156f6194a6ba4b696073ca1c18a3f8f)
+    
+    Adjustments:
+     - release g_ceph_context in the same way as the main code path does
+
+commit d012c381e8c59994ea9a40dc006d23f1bdd6a026
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue May 10 11:27:59 2016 +0200
+
+    radosstriper : protect aio_write API from calls with 0 bytes
+    
+    an assertion was failing so far, while we only have to return without doing anything
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 7cce1e8c51640f466d8bb37a21c0d5f1b00db8ab)
+
+commit 915dbace5ddea69fff29f7965f213229b6fbc0ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 11:55:00 2017 -0500
+
+    osdc: cache should ignore error bhs during trim
+    
+    A read error (such as injecting a timeout into an OSD op) might result
+    in a bh in an error state. These should be trimable by the cache.
+    
+    Fixes: http://tracker.ceph.com/issues/18436
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5910ed9de9856b5821488a1836487bbbd3d6460e)
+
+commit 419c9926d9ed57cb60228bc95956a9a1471b92cb
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Jan 31 16:07:18 2017 +0100
+
+    OSD: allow client throttler to be adjusted on-fly, without restart
+    
+    This patch allows the osd_client_message_cap and
+    osd_client_message_size_cap to be adjusted on-fly, using admin socket
+    functionality.
+    
+    Fixes: http://tracker.ceph.com/issues/18791
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 64c309d7e18a975931b526e6f5d6f610c3a0d632)
+    
+    Conflicts:
+            src/osd/OSD.cc (suppressed post-jewel option)
+
+commit 957c19b844fb44cde78ad59f872815f82bbf23b8
+Author: Ali Maredia <amaredia@redhat.com>
+Date:   Thu Nov 10 13:58:35 2016 -0500
+
+    swift: added "--cluster" to rgw-admin command for multisite support
+    
+    Signed-off-by: Ali Maredia <amaredia@redhat.com>
+
+commit 8423bc1eefa45366bdd215a17c61701c9b05dfdd
+Merge: 6dc30c4140 c078534376
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Apr 23 14:49:30 2015 -0500
+
+    Merge pull request #470 from ceph/wip-remote
+    
+    Add timeouts to Remote connection functions
+
+commit 6dc30c4140a833fd6cd126f8b5c1eceebad90510
+Merge: d55484f9e5 c078534376
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Apr 20 12:38:29 2015 -0500
+
+    Merge pull request #466 from ceph/wip-11426
+    
+    Log stderr in get_latest_image_version_deb()
+
+commit d55484f9e562779e7d47a8f63ff029337dc01eef
+Merge: c078534376 cd72cf2b31
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Apr 15 10:30:46 2015 -0700
+
+    Merge pull request #462 from ceph/wip-ssh-keys
+    
+    When modifying authorized_keys, store a backup
+
+commit c078534376d594aa3bf70d1d2e5dfc09ca8ae248
+Merge: cd72cf2b31 fce2ed683f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Apr 14 11:38:15 2015 -0600
+
+    Merge pull request #460 from zhouyuan/mkdir_p
+    
+    Make parent directories as needed
+
+commit cd72cf2b3132e7c6371aa4dbfe7564ad3ad0509e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Nov 24 09:59:30 2014 -0800
+
+    swift: set full access to subusers creation
+    
+    Default subuser permissions are 'none'.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit fce2ed683fcd3798db968a40200f8e8f215595fa
+Author: Zack Cerza <zack.cerza@inktank.com>
+Date:   Wed Aug 6 10:06:34 2014 -0600
+
+    Remove most ceph-specific tasks. They are in ceph-qa-suite now.
+    
+    Signed-off-by: Zack Cerza <zack.cerza@inktank.com>
+
+commit fac452ae55594aea482db5c13a0bd0207b6ecff6
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Mar 27 11:35:28 2014 -0500
+
+    Revert "Lines formerly of the form '(remote,) = ctx.cluster.only(role).remotes.keys()'"
+    
+    This reverts commit d693b3f8950ffd1f2492a4db0f8234fee31f00f0.
+
+commit e98b107302e06fa5c3c628a7ab0e7455de9ab568
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Fri Feb 28 19:13:40 2014 -0800
+
+    Lines formerly of the form '(remote,) = ctx.cluster.only(role).remotes.keys()'
+    and '(remote,) = ctx.cluster.only(role).remotes.iterkeys()' would fail with
+    ValueError and no message if there were less than 0 or more than 1 key.
+    Now a new function, get_single_remote_value() is called which prints out
+    more understandable messages.
+    
+    Fixes: 7510
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit e5fe884edfec66cc7e520938bd520aa8f1344f85
+Merge: 8e2cdbf5ed f8bf53c4fe
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 19 16:40:17 2014 -0500
+
+    Merge pull request #186 from ceph/wip-7369
+    
+    Fix #7369: "sed expression must be raw string"
+
+commit 8e2cdbf5ede871ebde260e6bdaec13daae03cfc5
+Merge: 132f3e8ae6 f8bf53c4fe
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 14:59:04 2014 -0600
+
+    Merge pull request #188 from ceph/wip-calamari-onefile
+    
+    Add Calamari test tasks, test script
+
+commit 132f3e8ae664a35cad89896b7300be13b5c604ec
+Merge: 24363351c3 7b63876676
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:50:55 2014 -0600
+
+    Merge pull request #192 from ceph/wip-6537-wusui
+    
+    Readjust the indentation of mon_clock_skew_check.py and mon_thrash.py.
+
+commit 24363351c3b1d23a8d6ef4e3f952855415f97ba8
+Merge: 4eb147291f 7b63876676
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:45:51 2014 -0600
+
+    Merge pull request #194 from ceph/wip-6534-wusui
+    
+    Add docstrings to internal.py
+
+commit 4eb147291fccdb2164a46e977ee43dbf50894e5a
+Merge: ad9aaf8fa3 f4284b520a
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:45:19 2014 -0600
+
+    Merge pull request #193 from ceph/wip-6538-wusui
+    
+    Add doc strings to Swift tests
+
+commit ad9aaf8fa35d49c33373fa69df7b38d3aca6abc5
+Merge: 7b63876676 f8bf53c4fe
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:43:15 2014 -0600
+
+    Merge pull request #187 from ceph/wip-better-debug
+    
+    Debug output improvements
+
+commit f4284b520a554b1cbe130731741e53a7fcf4e35f
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Feb 13 21:11:34 2014 -0800
+
+    Add doc strings to Swift tests
+    
+    Fixes: 6538
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit 7b63876676f1a6845ba3b9147cf7bb2348ef2468
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue Feb 11 20:21:06 2014 -0800
+
+    Add docstrings to s3 related tasks.
+    
+    Fixes: 6536
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit f8bf53c4fe52009abdf730b05e2cb2ddbb412dea
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Sep 24 14:19:24 2013 -0500
+
+    Fix namespace collision
+
+commit db6efe3e0ba7446ca42baf2a50eef18a10cf4a10
+Merge: 611733c8b5 66555a4039
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Sep 24 08:17:22 2013 -0700
+
+    Merge pull request #106 from ceph/wip-mirror
+    
+    Remove lots of ceph.com hardcoding; default to upstream sources
+
+commit 66555a4039c61db9b96d6eecf8d2f298c98b6bad
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Sep 20 15:53:58 2013 -0500
+
+    Don't hardcode the git://ceph.com/git/ mirror
+    
+    Default to https://github.com/ceph/ but add a ~/teuthology.yaml option
+
+commit 611733c8b5ea55206df50c122efac612cb146c2f
+Merge: 2346f1d735 6e8a3807c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Sep 6 13:24:34 2013 -0700
+
+    Merge pull request #78 from ceph/wip-6247
+    
+    Move helper scripts to /usr/local/bin to clean up logs.
+
+commit 6e8a3807c766f728027c3099eebfa24cdc645bd1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Sep 6 15:08:01 2013 -0500
+
+    Helper scripts live in /usr/local/bin now!
+
+commit 2346f1d735ccb40d00b8ff61d4acb446f684b3b2
+Author: Joe Buck <jbbuck@gmail.com>
+Date:   Fri Aug 23 19:54:53 2013 -0700
+
+    s3tests: extend for multi-region tests
+    
+    Added code to the s3tests task to extract
+    multi-region info so that that data
+    can be added to the S3TEST_CONF file
+    used to run S3 tests.
+    
+    Signed-off-by: Joe Buck <jbbuck@gmail.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit d3b6d633e161bc422f4781c7be5011f8a7ed8a32
+Merge: 442a36c57c 09b01b27a3
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Aug 19 14:10:18 2013 -0700
+
+    Merge pull request #41 from ceph/wip-3791
+    
+    Various usability and documentation fixes
+
+commit 442a36c57c3173b36ce17e5f85a49abf7e80ea93
+Merge: 09b01b27a3 9b2c4fa4ad
+Author: wusui <warren.usui@inktank.com>
+Date:   Fri Aug 16 14:47:59 2013 -0700
+
+    Merge pull request #40 from ceph/wip-teutharm-wusui
+    
+    Wip teutharm wusui
+
+commit 09b01b27a3a1310d4257133def60896ad37fb575
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Aug 15 08:49:35 2013 -0500
+
+    Fix some instances where print is being used instead of log
+
+commit 9b2c4fa4ad4c4258b26526afb0c16c71ce47f593
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 31 13:32:58 2013 -0700
+
+    s3/swift tests: call radosgw-admin as the right client
+    
+    This allows the right region and zone info to be read from ceph.conf
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3b3816df58a3ba1f2f850faf8969ad070aa0046e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 25 16:47:34 2013 -0700
+
+    s3tests: clone correct branch
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit f5170fb460a2183aed33122b4e6d0117a4220fec
+Merge: 7207a31e58 bd56af707a
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Jul 25 19:50:39 2013 -0700
+
+    Merge branch 'master' of github.com:ceph/teuthology
+
+commit 7207a31e5812dd0b29f2d6378360015622ddf4aa
+Merge: bd56af707a 3da945512e
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Jul 25 19:50:02 2013 -0700
+
+    Merge remote-tracking branch 'origin/wip-sandon-vm'
+    
+    Conflicts:
+            teuthology/lock.py
+            teuthology/misc.py
+            teuthology/task/install.py
+
+commit bd56af707ae50c98ec46344cf57eb333061847b1
+Merge: 3da945512e 343a42c0d8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Jul 19 14:44:51 2013 -0700
+
+    Merge branch 'wip-centos-rgw'
+
+commit 343a42c0d86af5b8630a30716c03fc84ba22f944
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jul 9 18:50:52 2013 -0700
+
+    s3tests: fix client configurations that aren't dictionaries
+    
+    They're always used as dictionaries later on.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3da945512ed78081be5dbe9ba59c836a311e1973
+Merge: 2c34d1971f 253cc98d98
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 16:18:36 2013 -0700
+
+    Merge pull request #15 from ceph/wip-ulimits
+    
+    Reviewed-by: Warren Usui <warren.usui@inktank.com>
+
+commit 253cc98d98855d65be7ebcdd46a39aa1004f8e67
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 23 09:15:28 2013 -0700
+
+    enable-coredump -> adjust-ulimits
+    
+    and set max_files to be big, too!
+
+commit 2c34d1971f1e82311b364bf8efe60b223158d676
+Merge: 61dba20d1f b366ad334a
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue May 7 19:27:51 2013 -0700
+
+    Merge branch 'wip-teuth4768a-wusui'
+    
+    Conflicts:
+            teuthology/task/install.py
+
+commit 61dba20d1fce774eac5b56f0d61d4229460875c0
+Merge: a9f3eb6310 5a7267f85c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 21:31:36 2013 -0700
+
+    Merge branch 'next'
+
+commit a9f3eb631064931cbdde7ef218c16e154bdb9991
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 2 13:47:46 2013 -0700
+
+    s3tests: add force-branch with higher precdence than 'branch'
+    
+    This way we can force a branch despite something in overrides.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b366ad334af55867ba781e22c8f87b6ac7775bf2
+Merge: 2a51e32891 5a7267f85c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed May 1 09:52:02 2013 -0700
+
+    Merge remote branch 'origin/next'
+
+commit 5a7267f85c80f88aca1b0081b07de1de3909f2e7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 17:07:53 2013 -0700
+
+    fix some errors found by pyflakes
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit f866037f045887ccc5da15404935ce3361a74a08
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 13:23:22 2013 -0700
+
+    s3tests: revert useless portion of 1c50db6a4630d07e72144dafd985c397f8a42dc5
+    
+    Perhaps it was attempting to debug something, but it shouldn't have been committed.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2dcce57c00264b85cfb906223dfb89db9cc61ba5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 16:49:04 2013 -0700
+
+    rgw tests: remove users after each test
+    
+    These should all be cleanup up at some point. They're
+    almost all the same code.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3c604251d9014dba45f3adbb2008bac16ff6346d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 16:47:34 2013 -0700
+
+    rgw tests: clean up immediately after the test
+    
+    There's no need for an explicit cleanup function, so move it back
+    to where it came from (except in s3roundtrip, which did not have it).
+    
+    Instead, since these use a nested contextmanager, pass through
+    and yield to the top-level run_tasks after the nested
+    contextmanager has finished (and thus run all the cleanup steps
+    in the subtasks for this test).
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 022bd4aa42312b80917169282cbfba655bbad6f1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Apr 30 07:06:03 2013 -0700
+
+    swift, s3readwrite: add missing yield
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 820c72b8d0177b01f69887ec64d98702db37077c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 29 11:24:04 2013 -0700
+
+    s3tests, s3readwrite, swift: cleanup explicitly
+    
+    Cleaning up test dir explicitly after run, so that
+    consecutive runs don't fail.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2a51e328913f16917a881129b475a4aeeab24ed0
+Merge: cccadb9b03 617534e769
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 20 14:10:50 2013 -0800
+
+    Merge remote-tracking branch 'origin/wip-3634'
+
+commit cccadb9b03ca4421a5fd841a61bf252c329e3649
+Merge: 3eb19c8107 fa1f89478a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 19 21:04:24 2013 -0800
+
+    Merge branch 'unstable'
+    
+    Conflicts:
+            teuthology/task/workunit.py
+
+commit fa1f89478a76373cb33cf2524e2ebf68b3cd622c
+Author: Sander Pool <sander.pool@inktank.com>
+Date:   Wed Feb 6 19:16:52 2013 +0000
+
+    Install ceph debs and use installed debs
+    
+    The ceph task installs ceph using the debian
+    packages now, and all invocations of binaries installed
+    in {tmpdir}/binary/usr/local/bin/ are replace with
+    the use of the binaries installed in standard locations
+    by the debs.
+    
+    Author:    Sander Pool <sander.pool@inktank.com>
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+
+commit 3eb19c810725b011baacdb8a6f5b172f4720a39a
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Wed Jan 23 14:37:39 2013 -0600
+
+    Replace /tmp/cephtest/ with configurable path
+    
+    Teuthology uses /tmp/cephtest/ as the scratch test directory for
+    a run.  This patch replaces /tmp/cephtest/ everywhere with a
+    per-run directory: {basedir}/{rundir} where {basedir} is a directory
+    configured in .teuthology.yaml (/tmp/cephtest if not specified),
+    and {rundir} is the name of the run, as given in --name.  If no name
+    is specified, {user}-{timestamp} is used.
+    
+    To get the old behavior (/tmp/cephtest), set test_path: /tmp/cephtest
+    in .teuthology.yaml.
+    
+    This change was modivated by #3782, which requires a test dir that
+    survives across reboots, but also resolves #3767.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2f829870e140c87b30e5b7aa3ad237a90dcb2179
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Dec 21 10:20:02 2012 -0800
+
+    task/swift: change upstream repository url
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 334d6386753510c312898552c6f92313942786ef
+Merge: b8e6ce4db9 26df886d82
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Thu Nov 29 00:53:59 2012 +0000
+
+    Merge branch 'wip-mon-thrasher'
+
+commit b8e6ce4db9a603ce3523b1759c65eeadee55daa7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Nov 22 13:59:58 2012 -0800
+
+    s3tests: fix typo
+
+commit 26df886d825e28c25b630887b8dcc1c8c6d687d8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 16:19:06 2012 -0800
+
+    rgw-logsocket: a task to verify opslog socket works
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 617534e76978acb09a9f925f18bba475a65a7dd2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 10 11:08:57 2012 -0700
+
+    s3tests: run against arbitrary branch/sha1 of s3-tests.git
+
+commit 7d5c7ee8c6f2bfedd193a8d3b7102b4cfe0bf74f
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Jun 6 16:00:55 2012 -0700
+
+    pull s3-tests.git using git, not http
+
+commit ce951cf4caffd5d6883cc8dcd24372bcdf03690a
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat May 5 09:30:41 2012 -0700
+
+    ceph.newdream.net -> ceph.com
+
+commit 2b879905fcfd660e242ed1a804d1c8301d17ab84
+Merge: 1ac4bb10fc 1970713a2f
+Author: Mark Nelson <nhm@clusterfaq.org>
+Date:   Wed Mar 14 15:32:23 2012 -0500
+
+    Merge branch 'master' of github.com:ceph/teuthology
+
+commit 1970713a2fc43e3afae376712356ca93a65d9e1f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Mar 2 10:55:19 2012 -0800
+
+    github.com/NewDreamNetwork -> github.com/ceph
+
+commit 1ac4bb10fc4b7d8d07c44b0e92b1627c721ab925
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Tue Feb 21 14:54:33 2012 -0800
+
+    Add necessary imports for s3 tasks, and keep them alphabetical.
+
+commit 92110e5a4460281139233dcea3f629d01182d398
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Tue Feb 21 12:12:03 2012 -0800
+
+    rgw: access key uses url safe chars
+    
+    Signed-off-by: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+
+commit 709d9441127fec93da74c7702cafa54a47e10e8f
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Jan 15 22:48:33 2012 -0800
+
+    use local mirrors for (most) github urls
+    
+    A cronjob on ceph.newdream.net updates these every 15 minutes.  Sigh.
+
+commit 9598e47949ba65030c722947dc433e38875b1bd6
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Mon Dec 5 10:07:25 2011 -0800
+
+    Rename "testrados" and "testswift" tasks to not begin with "test".
+    
+    Anything "test*" looks like a unit test, and shouldn't be used for
+    actual code.
+
+commit 6236e7db22edac7b51fc6329188b6afa74f3fc78
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Thu Nov 17 16:53:21 2011 -0800
+
+    testswift: fix config
+
+commit 1dd607cabb07126769b4beb1ba6677e21c448719
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Wed Nov 16 16:00:01 2011 -0800
+
+    rgw: add swift task
+    
+    still not completely working (for some reason it skips all the tests)
+
+commit cb425c158085568cd92c239a071e282c74eddf1a
+Author: Greg Farnum <gregory.farnum@dreamhost.com>
+Date:   Fri Sep 30 09:26:42 2011 -0700
+
+    s3-tests: use radosgw-admin instead of radosgw_admin
+    
+    Signed-off-by: Greg Farnum <gregory.farnum@dreamhost.com>
+
+commit 37d7d515345ab04c333d6fada722e432e5816eb3
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Fri Sep 16 11:09:45 2011 -0700
+
+    s3tests: Clone repository from github.
+    
+    Signed-off-by: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+
+commit 29a242d97dd4a5a9110710027cdddb244b8b0e29
+Author: Tommi Virtanen <tv@eagain.net>
+Date:   Tue Sep 13 14:53:02 2011 -0700
+
+    Move orchestra to teuthology.orchestra so there's just one top-level package.
+
+commit ec49a5f263f71aa473257e3fd49d86e475fe9456
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Fri Sep 9 13:22:03 2011 -0700
+
+    Callers of task s3tests.create_users don't need to provide dummy "fixtures" dict.
+
+commit d7d995e82b45e6077040b467c8ef9a82a573faf7
+Author: Stephon Striplin <stephon.striplin@dreamhost.com>
+Date:   Tue Aug 9 13:43:46 2011 -0700
+
+    allow s3tests.create_users defaults be overridden
+
+commit 0086109767d5bfbbc370ca13d3fe91895b207821
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Thu Jul 14 16:47:29 2011 -0700
+
+    Make targets a dictionary mapping hosts to ssh host keys.
+
+commit 1b2c96416f554c0890b2690291b9a2dc8a6dc17a
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Wed Jul 6 14:17:24 2011 -0700
+
+    Skip s3-tests marked fails_on_rgw, they will fail anyway.
+
+commit 06fb9b95e39985630d89e1635dcd12510686d9cd
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Tue Jul 5 09:27:28 2011 -0700
+
+    The shell exits after the command, hence there is no need for pushd/popd.
+
+commit cd524a6904bf8254edc73a9148308f642638e33d
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Fri Jun 24 17:09:47 2011 -0700
+
+    Add s3tests task.
diff --git a/doc/changelog/v10.2.9.txt b/doc/changelog/v10.2.9.txt
new file mode 100644
index 000000000..551f268ee
--- /dev/null
+++ b/doc/changelog/v10.2.9.txt
@@ -0,0 +1,63 @@
+commit 2ee413f77150c0f375ff6f10edd6c8f9c7d060d0
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Jul 13 13:04:57 2017 +0000
+
+    10.2.9
+
+commit 9295f588535c45431d19b9601b4063c8de88752d
+Merge: 7b10d629ae fef1c8718f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 13 10:31:31 2017 +0200
+
+    Merge pull request #16282 from smithfarm/wip-20599-jewel
+    
+    jewel: cephfs: Damaged MDS with 10.2.8
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit fef1c8718f77c190a0908d353f38b16b7c3832ab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 12 08:40:20 2017 +0200
+
+    Revert "osdc/Journaler: make header write_pos align to boundary of flushed entry"
+    
+    This reverts commit 2e299b50de4a297fee2aec21290632336d239857.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3f89971e9edb88e313e1c190f7d05a83b52e6d91
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 12 08:40:13 2017 +0200
+
+    Revert "osdc/Journaler: avoid executing on_safe contexts prematurely"
+    
+    This reverts commit 06cf9f365033f7913051bdf4060f0bc6fc0444d7.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7b10d629ae32122b267486fe9e176f5cd0e330cf
+Merge: f5b1f1fd7c 6b479c275a
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Jul 11 15:58:04 2017 -0500
+
+    Merge pull request #16273 from smithfarm/wip-jewel-pending-release-notes
+    
+    jewel: doc: clarify status of jewel PendingReleaseNotes
+
+commit 6b479c275a24451c278074d81758a385eca12869
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jul 11 22:53:56 2017 +0200
+
+    doc: zero PendingReleaseNotes in preparation for v10.2.9
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 55de93f9d711e13980168cc884dcb04d8849708e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jul 11 22:27:33 2017 +0200
+
+    doc: clarify status of jewel PendingReleaseNotes
+    
+    Status as of 10.2.8 release
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
diff --git a/doc/changelog/v11.2.1.txt b/doc/changelog/v11.2.1.txt
new file mode 100644
index 000000000..0decf8f65
--- /dev/null
+++ b/doc/changelog/v11.2.1.txt
@@ -0,0 +1,8020 @@
+commit e0354f9d3b1eea1d75a7dd487ba8098311be38a7
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Aug 8 19:07:05 2017 +0000
+
+    11.2.1
+
+commit 11d5c2b0aae372f9c993a7dea5a914ee74e7480c
+Merge: 397b553932 6b428b77f2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 2 15:08:48 2017 +0200
+
+    Merge pull request #14702 from ceph/wip-bp-kraken-systemd
+
+    kraken: qa/tasks: misc systemd updates
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com
+
+commit 397b5539320236bffc66caab579422b1bd4570f7
+Merge: 0bab3f3a10 d24edde072
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 2 07:55:48 2017 +0200
+
+    Merge pull request #15509 from ceph/wip-ceph-disk-fix-kraken
+
+    kraken: selinux: Do parallel relabel on package install
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0bab3f3a1026c41736e06f72f8ede68b3386d772
+Merge: 646f2b3c26 a91253ec50
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 20:19:15 2017 +0200
+
+    Merge pull request #14612 from smithfarm/wip-19651
+
+    tests: backport Sage's fixes to qa/suites/upgrade/jewel-x
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 646f2b3c260ea5af3657d5f289753b114ba0d71e
+Merge: e9c255689f cf06edd36b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:39:37 2017 +0200
+
+    Merge pull request #16298 from smithfarm/wip-20517-kraken
+
+    kraken: rbd: cli: map with cephx disabled results in error message
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e9c255689fcf2a7afe1deda3b987591162767daa
+Merge: 143e431d76 1fc14857a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:17:40 2017 +0200
+
+    Merge pull request #16175 from smithfarm/wip-20263-kraken
+
+    kraken: rgw: datalog trim can't work as expected
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 143e431d769ad68101df436735154c1230f3732b
+Merge: c9a545dba6 b758348447
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:16:54 2017 +0200
+
+    Merge pull request #15985 from prallabh/kraken
+
+    kraken: rgw: Custom data header support
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c9a545dba68fe09e7aa971aec00c7e5aa1f217d2
+Merge: 245bf9205d 45b4c86452
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:15:33 2017 +0200
+
+    Merge pull request #16186 from smithfarm/wip-20264-kraken
+
+    kraken: rbd: [cli] ensure positional arguments exist before casting
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 245bf9205dc811b4bbe99e53bf20b32daeb85f8d
+Merge: 3d19b8acf0 f2d61c199d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:12:58 2017 +0200
+
+    Merge pull request #16342 from dillaman/wip-20630-kraken
+
+    kraken: tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3d19b8acf0f76794c2b9c48f6ab386b6e5d14139
+Merge: a57772db99 f72ea68186
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:14:09 2017 +0200
+
+    Merge pull request #16290 from smithfarm/wip-19759-kraken
+
+    kraken: rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a57772db99068c5e05b7b8cd1e346bdfb1fba292
+Merge: a1ff4e61af af0a6df270
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:12:37 2017 +0200
+
+    Merge pull request #16180 from smithfarm/wip-20347-kraken
+
+    kraken: rgw: meta sync thread crash at RGWMetaSyncShardCR
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a1ff4e61aff4497e84d07070de984e34c6539bf0
+Merge: d0d98362e7 17fd055a4f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 12:47:45 2017 +0200
+
+    Merge pull request #16190 from smithfarm/wip-20026-kraken
+
+    kraken: mds: unresponsive when truncating a very large file
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d0d98362e7363fd84334941085203aa3bfd7a2d2
+Merge: 72e5d6116c 719ed0101b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 11:08:47 2017 +0200
+
+    Merge pull request #15526 from badone/wip-async-sleep-timer-fix-kraken
+
+    kraken: osd: Implement asynchronous scrub sleep
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 72e5d6116cc1afd83f96e26e32c5949548c1dcbc
+Merge: 16ca3c7f40 533ff8a540
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 16:47:45 2017 +0200
+
+    Merge pull request #16137 from smithfarm/wip-20024-kraken
+
+    kraken: tests: HEALTH_WARN pool rbd pg_num 244 > pgp_num 224 during upgrade
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 16ca3c7f40d37f1fb6e38665dd0c33fe3ab5a060
+Merge: a1c2cbd074 4d1272e5c6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 15:31:12 2017 +0200
+
+    Merge pull request #14734 from smithfarm/wip-19670-kraken
+
+    kraken: build/ops: logrotate is missing from debian package (kraken, master)
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a1c2cbd074dbc2954fea0d2808a80958f17fe9c3
+Merge: fd6816bb72 ccb33bab37
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 15:21:02 2017 +0200
+
+    Merge pull request #16166 from smithfarm/wip-19840-kraken
+
+    kraken: rgw: civetweb frontend segfaults in Luminous
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit a91253ec50dd5c2ab85b00d52361fd22580e1ed3
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 7 22:35:02 2017 -0500
+
+    qa/suites/upgrade/jewel-x: do not thrash cluster full during upgrade
+
+    The mon command varies.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 39fdc53fe5f33678fbbd00cf8810b6d523d0040c)
+
+commit afb5f4b0b6eb0bbbfbdd628ff466e1fad6e6e4ef
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 15:16:13 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: expand workload matrix
+
+    These should run independently against a racing upgrade.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1a0ad2b48839fea75d4d3339f7d198404ff1ac37)
+
+commit a88bd845df01b9fa8d1b0482b1c4d6bfae301f40
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 14:11:53 2017 -0500
+
+    qa/suites/upgrade/jewel-x/stress-split-erasure-code: box thrashosds
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b2d354d56359d08b35cbea58f811c7bafb700d31)
+
+commit d4521fc2b9b13cee58abb4b578e8ba00c6aca0ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 14:07:53 2017 -0500
+
+    qa/suites/upgrade/jewel-x/stress-split: finish client.0 upgrade too
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 56f9387736eed136b38c087a4805821063e9f8ab)
+
+commit 907888f881cc7224f9f9c17f38cc98773ca33906
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Mar 5 14:14:40 2017 -0500
+
+    qa/suites/upgrade/jewel-x: remove kraken references
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 468285b65d6bef2d86c80ebdfecf0920294ca5cd)
+
+commit 70415451e90d5aba473dbb3b1bd00ce4de0d077b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 19:01:20 2017 -0600
+
+    qa/suite/upgrade/jewel-x: drop x86 ec test
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1e0e53c80d8fa88970cee1d61aaa8514004547ac)
+
+commit fa7934b3abe395d52f127c6e170a54296d936211
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 20 14:38:49 2017 -0500
+
+    qa/suites/upgrade/jewel-x: fix upgrade order; fix split grouping
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 269eafb2027ea6dfbad730f6fb1481a1cabc5e60)
+
+    Conflicts:
+        stress-split/6-kraken.yaml (do not delete)
+        stress-split/6-luminous.yaml (do not add)
+        stress-split-erasure-code/6-kraken.yaml (do not delete)
+        stress-split-erasure-code/6-luminous.yaml (do not add)
+
+commit b574cb341c9ce0f9d986c79c03951c7c9eafb0b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 19:01:12 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: upgrade mons before osds
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3c80e15c3b34ac2adc4e70f09929e3bc01785594)
+
+commit 4eb3c2dbf321cf7a110b2ac7022d5d9cc80b8f3a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:58:07 2017 -0600
+
+    qa/suites/upgrade/jewel-x/parallel: fix upgrade to luminous
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e3ea53b41cacc4b4bbc4ffe7342fb00233009cf)
+
+    Conflicts:
+       4-kraken.yaml (do not delete)
+       4-luminous.yaml (do not add)
+
+commit fd6816bb725f7699537276c13f68d815a7406228
+Merge: a6fdfccd11 b7503d3a80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:32:45 2017 +0200
+
+    Merge pull request #13871 from smithfarm/wip-19162-kraken
+
+    kraken: rgw: rgw_file: fix marker computation
+
+commit a6fdfccd117180cf3906e1349b7af19462eae0e1
+Merge: b50909c861 a7af766aa8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:26:47 2017 +0200
+
+    Merge pull request #16133 from smithfarm/wip-18378-kraken
+
+    kraken: msg/simple/SimpleMessenger.cc: 239: FAILED assert(!cleared)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b50909c861450f1aeb40d3fa1a0ca4d997d09a0c
+Merge: 938b723717 c6542ac7e6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:23:08 2017 +0200
+
+    Merge pull request #16134 from smithfarm/wip-19340-kraken
+
+    kraken: An OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed
+
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 17fd055a4f68cb3c51baa1ac1fb089d15b818312
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 25 16:21:24 2017 +0800
+
+    osdc/Filer: truncate large file party by party
+
+    Fixes: http://tracker.ceph.com/issues/19755
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263)
+
+    Conflicts:
+            src/osdc/Filer.h
+            src/osdc/Filer.cc
+            src/mds/MDCache.cc
+
+commit 938b723717745eff4e3ab18b9a0f11cfc5190876
+Merge: e56d4c481d b8dfa2f73a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 11:58:20 2017 +0200
+
+    Merge pull request #16131 from smithfarm/wip-swift-kraken
+
+    tests: swift.py: clone the ceph-jewel branch
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 719ed0101b1bfdd4b71ef84101515492597153f9
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 22 13:21:25 2017 +1000
+
+    osd: Move scrub sleep timer to osdservice
+
+    PR 14886 erroneously creates a scrub sleep timer for every pg resulting
+    in a proliferation of threads. Move the timer to the osd service so
+    there can be only one.
+
+    Fixes: http://tracker.ceph.com/issues/19986
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit f110a82437df79dc20207d296e8229fc0e9ce18b)
+
+commit 460a820a3b2fbd48c8a7966502b235aae8d5d298
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 24 14:10:47 2017 +1000
+
+    osd: Implement asynchronous scrub sleep
+
+    Rather than blocking the main op queue just do an async sleep.
+
+    Fixes: http://tracker.ceph.com/issues/19497
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7af3e86c2e4992db35637864b83832535c94d0e6)
+
+commit e56d4c481dead2f8ccb7baaae80db2f4acfc7bf0
+Merge: 024272160d bdb16fd0f5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 25 07:14:37 2017 +0200
+
+    Merge pull request #16493 from smithfarm/wip-suppress-upgrade-fail
+
+    tests: run certain upgrade/jewel-x tests on Xenial only
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 024272160de37c3ed2a2df701fc1e8f4c11c2da5
+Merge: d8e0ddc57b 838d7840f0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 25 07:13:48 2017 +0200
+
+    Merge pull request #14597 from gregsfortytwo/wip-kraken-snaptrim
+
+    kraken: core: improve control and throttling of the snap trimmer
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bdb16fd0f533691b7b7a01b176764ad907143814
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 22 10:13:22 2017 +0200
+
+    tests: upgrade/jewel-x/parallel: drop duplicate kraken.yaml
+
+    This yaml file has the effect of re-running "ceph osd set require_kraken_osds"
+    at the very end of the test. Drop it.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit dc8c2231cefdbc8811cdea380d7c856b57c62251
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 22 00:06:48 2017 +0200
+
+    tests: run certain upgrade/jewel-x tests on Xenial only
+
+    This PR drops two upgrade/jewel-x test cases that are not compatible with
+    https://github.com/ceph/ceph/pull/14597
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit b8dfa2f73aff0998a4aebf6f61865983f476b8d1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 09:27:47 2017 +0200
+
+    tests: swift.py: clone the ceph-jewel branch
+
+    The master branch of ceph/swift.git contains tests that are incompatible with
+    Jewel and Hammer. The ceph-jewel branch omits these tests.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a86ce728954a765797ce634025d43650d990e480)
+
+    Conflicts:
+        qa/tasks/swift.py: clone ceph-kraken branch instead of ceph-jewel
+
+commit d8e0ddc57b0e2632baf70bebf409d52808c202fa
+Merge: c539ea49a8 19c7524ace
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 21 08:22:25 2017 +0200
+
+    Merge pull request #16111 from smithfarm/wip-20497-kraken
+
+    kraken: tests: insufficient timeout in radosbench task
+
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit af0a6df270f3c469280cae877c177086ddf10245
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 20:17:49 2017 +0200
+
+    rgw: lease_stack: use reset method instead of assignment
+
+    It seems that the intent of 45877d38fd9a385b2f8b13e90be94d784898b0b3 was to
+    change all instances of "lease_stack = ..." to "lease_stack.reset(...)", but
+    this one was missed.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc425afeb2e2a6ba3c98c612b3977aea619c9f73)
+
+    Conflicts:
+            src/rgw/rgw_sync.cc (trivial)
+
+commit c539ea49a8dab85bade53140d80f2a05f3f1860c
+Merge: 0a9badb3e2 640a7a2629
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 20 16:38:27 2017 +0200
+
+    Merge pull request #16178 from smithfarm/wip-20268-kraken
+
+    kraken: rgw: get wrong content when download object with specific range when compression was enabled
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d24edde072ec63d35965e30a58aaaf02547f9c57
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jul 7 12:37:55 2017 +0200
+
+    rpm: Fix undefined FIRST_ARG
+
+    If FIRST_ARG is undefined, the rpms will show an error on upgrade
+    because the condition in the postun script gets broken.
+
+    This was a regression introduced by commit for issue 20077 that moved
+    ceph-disk unit files to ceph-base.
+
+    Fixes: http://tracker.ceph.com/issues/20077
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 562816914ccca8e4e7d9c31f333db2f0da6f7c99)
+
+commit eac6a0dd475e22512cc24bb3b8fe018c70cf7bc8
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 5 18:44:18 2017 +0200
+
+    selinux: Install ceph-base before ceph-selinux
+
+    We need to have ceph-base installed before ceph-selinux to use ceph-disk
+    in %post script. The default ordering is random and so the installation
+    randomly failed to relabel the files.
+
+    Fixes: http://tracker.ceph.com/issues/20184
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit e69086645d3411a2ed781609b670eb5f16ac4810)
+    Conflicts:
+            ceph.spec.in: No _epoch_prefix in kraken.
+
+commit 0a9badb3e2be41be5e7de6d865a8e323b3706fa8
+Merge: 13b04089ce baa772372c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:09:32 2017 +0200
+
+    Merge pull request #16114 from smithfarm/wip-20500-kraken
+
+    kraken: cephfs: src/test/pybind/test_cephfs.py fails
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 13b04089ceff7136785c37595f526841574cdcd9
+Merge: 34c32b9d07 b941ca7eac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:03:19 2017 +0200
+
+    Merge pull request #16108 from smithfarm/wip-19763-kraken
+
+    kraken: cephfs: non-local quota changes not visible until some IO is done
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 34c32b9d0721407511a8420ad3888cbee2fd8f5f
+Merge: bb95e12790 4241a6eef0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:02:09 2017 +0200
+
+    Merge pull request #16107 from smithfarm/wip-19710-kraken
+
+    kraken: mds: enable daemon to start when session ino info is corrupt
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit bb95e127909052679a883375feb55935ccb127d5
+Merge: 8202ddcf11 0b1dee0edc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:01:01 2017 +0200
+
+    Merge pull request #16106 from smithfarm/wip-19680-kraken
+
+    kraken: mds: damage reporting by ino number is useless
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8202ddcf1156093c58a0c161d6cdc3455fc440b5
+Merge: e2efa6e9e7 f458d60838
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:00:13 2017 +0200
+
+    Merge pull request #16105 from smithfarm/wip-19678-kraken
+
+    kraken: cephfs: ceph-fuse does not recover after lost connection to MDS
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit e2efa6e9e742dba2b9779e8b7382c173abfea0af
+Merge: 055319bfe7 7674f84ddf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:58:56 2017 +0200
+
+    Merge pull request #16104 from smithfarm/wip-19676-kraken
+
+    kraken: cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 055319bfe7e6dd15417e30b708590a4040d2a884
+Merge: 5c6a770308 bee73d2429
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:56:04 2017 +0200
+
+    Merge pull request #16103 from smithfarm/wip-19674-kraken
+
+    kraken: cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5c6a770308200788231173c270833a15d04765ed
+Merge: 8d86192e34 32c7ebe3e6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:55:17 2017 +0200
+
+    Merge pull request #16102 from smithfarm/wip-19672-kraken
+
+    kraken: mds: assert fail when shutting down
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8d86192e34dfd3d91a99a354545fb3e5b7b94f79
+Merge: 99d0afda7f b267a1a8be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:54:07 2017 +0200
+
+    Merge pull request #16101 from smithfarm/wip-19669-kraken
+
+    kraken: mds: daemon goes readonly writing backtrace for a file whose data pool has been removed
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 99d0afda7f8748b13940267c3d5286b8a98d5a8f
+Merge: 818ecc66c7 93e81a87ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:53:11 2017 +0200
+
+    Merge pull request #16100 from smithfarm/wip-19667-kraken
+
+    kraken: cephfs: mount point break off problem after mds switch
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 818ecc66c71e3ccb351353a36b32b3dd0c6c2123
+Merge: c8ccfc72b3 e72d6362c1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:05:12 2017 +0200
+
+    Merge pull request #16099 from smithfarm/wip-19664-kraken
+
+    kraken: mds: C_MDSInternalNoop::complete doesn't free itself
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit c8ccfc72b3556969722c24eacdd78828a3194b7f
+Merge: fbcef7d42a 3c4a5ea385
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:03:52 2017 +0200
+
+    Merge pull request #14998 from jan--f/wip-19845-kraken
+
+    kraken: cephfs: normalize file open flags internally used by cephfs
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit fbcef7d42a3832c2e404aa0d141238252045d3d9
+Merge: dbaae75029 f32e26e909
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 16:51:05 2017 +0200
+
+    Merge pull request #16168 from smithfarm/wip-20031-kraken
+
+    kraken: rgw: Swift's at-root features (/crossdomain.xml, /info, /healthcheck) are broken
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit dbaae750299d9169e2e1dda9d0af98521c8abb5e
+Merge: e26575be67 e773b304ee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 16:15:21 2017 +0200
+
+    Merge pull request #16164 from smithfarm/wip-19777-kraken
+
+    kraken: rgw: swift: disable revocation thread under certain circumstances
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit e26575be67e00dbf674ff77ef6e7729f22750d8b
+Merge: 415174d29e 53024570e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:43:16 2017 +0200
+
+    Merge pull request #16191 from smithfarm/wip-20028-kraken
+
+    kraken: cephfs: Deadlock on two ceph-fuse clients accessing the same file
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 415174d29e20f73884e81773fd0b4d1055d439b4
+Merge: 2879b80281 b27595ae9c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:32:52 2017 +0200
+
+    Merge pull request #16183 from smithfarm/wip-20405-kraken
+
+    kraken: rgw: Lifecycle thread will still handle the bucket even if it has been removed
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+
+commit 2879b80281c1d4bd8b8daeb5662879a296274fca
+Merge: eed9a165bd 3105e2327e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:31:50 2017 +0200
+
+    Merge pull request #16181 from smithfarm/wip-20363-kraken
+
+    kraken: rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit eed9a165bd1dd95f5afe8ccd78a742fbf16a4503
+Merge: b759d4362b d0e742cafa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:31:14 2017 +0200
+
+    Merge pull request #16179 from smithfarm/wip-20269-kraken
+
+    kraken: rgw: wrong object size after copy of uncompressed multipart objects
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit b759d4362b7a3ae2bc4174eb0ef328f919644e6c
+Merge: 82e9c73fe7 90288afc10
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:30:33 2017 +0200
+
+    Merge pull request #16174 from smithfarm/wip-20261-kraken
+
+    kraken: rgw: 'radosgw-admin usage show' listing 0 bytes_sent/received
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 82e9c73fe70b5e78bc170ad566fa7900191ece70
+Merge: 2c5b063f43 dda4d912cd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:29:57 2017 +0200
+
+    Merge pull request #16173 from smithfarm/wip-20156-kraken
+
+    kraken: fix: rgw crashed caused by shard id out of range when listing data log
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2c5b063f4386663f00ff897beee0523fa6e95acd
+Merge: 423d6c319f f8235c5c0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:29:22 2017 +0200
+
+    Merge pull request #16165 from smithfarm/wip-19839-kraken
+
+    kraken: rgw: reduce log level of 'storing entry at' in cls_log
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 423d6c319f55fb3eddc0602954645b3b1f1d7bd8
+Merge: 0af35a358e 68a853a14d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:28:30 2017 +0200
+
+    Merge pull request #16163 from smithfarm/wip-19766-kraken
+
+    kraken: rgw: when uploading the objects continuesly in the versioned bucket, some objects will not sync
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0af35a358ea8a8b3376faf4d77bd75edb1e808fe
+Merge: 872a8b53ea 9a7a73edf1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:27:38 2017 +0200
+
+    Merge pull request #16162 from smithfarm/wip-19725-kraken
+
+    kraken: rgw: S3 v4 authentication issue with X-Amz-Expires
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 872a8b53ea022923e9c442011432df5e403520ae
+Merge: 129345c319 d52aeec031
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:25:48 2017 +0200
+
+    Merge pull request #16161 from smithfarm/wip-19614-kraken
+
+    kraken: multisite: rest api fails to decode large period on 'period commit'
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 129345c3196effdaea08f9678d1a9f1cd8265d81
+Merge: 72c3a04f23 91569f6385
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:24:59 2017 +0200
+
+    Merge pull request #16139 from smithfarm/wip-20147-kraken
+
+    kraken: rgw: 'gc list --include-all' command infinite loop the first 1000 items
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 72c3a04f23a227713f50527b8b8651580556d45a
+Merge: 762901080c 6d2f959735
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:29:31 2017 +0200
+
+    Merge pull request #16096 from smithfarm/wip-19336-kraken
+
+    kraken: rbd: refuse to use an ec pool that doesn't support overwrites
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 762901080c3e7982a0f38705cd09578be1c6d940
+Merge: ac8bd5ea97 5dab0825d7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:28:08 2017 +0200
+
+    Merge pull request #16097 from smithfarm/wip-19609-kraken
+
+    kraken: tests: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ac8bd5ea97b7f87533d0320048335f80dcc66224
+Merge: c5d53bd299 ac706f2b4a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:27:11 2017 +0200
+
+    Merge pull request #16184 from smithfarm/wip-20154-kraken
+
+    kraken: rbd: Potential IO hang if image is flattened while read request is in-flight
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c5d53bd299807d90d3311625554e055696898e71
+Merge: f73412843d 18f1830e8a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:24:05 2017 +0200
+
+    Merge pull request #16187 from smithfarm/wip-20266-kraken
+
+    kraken: rbd: [api] is_exclusive_lock_owner shouldn't return -EBUSY
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f73412843db6cb9e3416d66a2ddf92e1b8d7038f
+Merge: 0d01813a8c ce874ab676
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:22:57 2017 +0200
+
+    Merge pull request #16195 from dillaman/wip-20351-kraken
+
+    kraken: tests: test/librbd: decouple ceph_test_librbd_api from libceph-common
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0d01813a8c72e0d797d100e8e45d99f1ff99ba5a
+Merge: 771ebef418 379309b552
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 12:56:47 2017 +0200
+
+    Merge pull request #15486 from dillaman/wip-20022-kraken
+
+    kraken: rbd-mirror: ensure missing images are re-synced when detected
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 771ebef418eb0d585ca9bd9dd565ea9f1caf3766
+Merge: 9d6e0f2ef8 3173da6e3b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 12:50:22 2017 +0200
+
+    Merge pull request #14540 from smithfarm/wip-18910-kraken
+
+    kraken: rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 9d6e0f2ef86b34b0bd2443f8c8a747bb15732e7c
+Merge: bb1b7c3b10 9a26882289
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 08:05:11 2017 +0200
+
+    Merge pull request #16140 from smithfarm/wip-20271-kraken
+
+    kraken: tests: LibRadosMiscConnectFailure.ConnectFailure hang
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bb1b7c3b1060b983eeb8e3a3f9607306b47ead77
+Merge: 5ea47f79c9 76fedab576
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 23:14:59 2017 +0200
+
+    Merge pull request #16112 from smithfarm/wip-20499-kraken
+
+    kraken: tests: ObjectStore/StoreTest.OnodeSizeTracking/2 fails on bluestore
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ea47f79c91229b57981bac77472102c868cfa89
+Merge: 1913303660 47f751f48b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 18 13:58:14 2017 -0700
+
+    Merge pull request #14960 from yehudasa/wip-19704
+
+    kraken: civetweb: move to post 1.8 version
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 838d7840f023c5802507727231983311182a1c05
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jul 10 13:33:26 2017 -0700
+
+    osd: do not default-abort on leaked pg refs
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 4caf2df0c380a1281db9509b3feb342705512b58)
+
+commit 4bf34200896ce120bba2e2d974c4f3dadb342821
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 22:14:38 2017 -0700
+
+    osd: shutdown our reserver_finisher earlier in the process
+
+    This finisher thread has a lot of callbacks which can hold PGRefs. Make
+    sure we drain them out before checking that all the PGs have finished
+    and have no outstanding references.
+
+    Moving this should be safe; we've already stopped the op thread et al
+    and the only things still running are the OSDService's objecter_finisher,
+    recovery_request_timer, and snap_sleep_timer (which has definitely been emptied
+    by the time we get here as it's synchronously cleared out on PG shutdown).
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 66ea9c1f66ae31035e62bd4335b08948b1e8e5e2)
+
+commit 2d5cafdf6461c213aa2642210a3c4efabce61efd
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 21:52:49 2017 -0700
+
+    osd: Reset() the snaptrimmer on shutdown
+
+    We were failing to exit various wait states which held PGRefs. Error!
+
+    Fixes: http://tracker.ceph.com/issues/19931
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit b0e9deeea8a8e90f6d7e9d56b6b4aed890e01d7b)
+
+commit cc0046a999600b2e684c17fddaebcf451c259443
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed May 24 18:43:34 2017 -0700
+
+    osd: print out pg debug ref counts on acquire/drop
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a3b028a6d1ba74ed975ebd665b1b50fb7e5039a4)
+
+commit c6542ac7e6408f26ce7376d3a58eed0661a17099
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jul 7 10:53:41 2017 -0700
+
+    mon: Fix status output warning for mon_warn_osd_usage_min_max_delta
+
+    Fixes: http://tracker.ceph.com/issues/20544
+
+    Caued by: 489e810c37ed6fb9d32d1015634041a577501ee4
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 56f9808016ab289bdc0ce7cfbb0503c78b509593)
+
+    Conflicts:
+        src/mon/PGMap.cc:
+            kraken has no daa0793c393a21bd2dd2ec52a0efd181e1032400 so we manually
+            apply the changes from 56f9808016ab289bdc0ce7cfbb0503c78b509593 to
+            src/mon/PGMonitor.cc instead of to src/mon/PGMap.cc
+
+commit 2f26f8cbd3e81700c42202f8d933f1576772e212
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 18 13:54:56 2017 -0400
+
+    mon/PGMonitor: clean up min/max span warning
+
+    Clean up option naming.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 489e810c37ed6fb9d32d1015634041a577501ee4)
+
+commit 90b7fd8a0accc2f8f458d35199361344652d8859
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Feb 17 12:27:36 2017 -0800
+
+    bluestore: Fix BlueStore::statfs available amount to not include bluefs min reserved
+
+    This fixes OSD crashes because checking osd_failsafe_full_ratio won't work
+    without accurate statfs information.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 72d83f848a35e8831d66e8529c4e26f51e845da6)
+
+commit d69e4e5e6ad633e38fc62ca0859cea1f146744b1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 22:23:06 2017 -0800
+
+    osd: Round fullness in message to correspond to df -h
+
+    This really only works after journal drains because
+    we adjust for the journal.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 26dcb591f9af01ed444aa758c3d601e7f67261b2)
+
+commit eb887e3cb1f55f016ca70ac3d4441aa03478d934
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 17:25:12 2017 -0800
+
+    filestore: Account for dirty journal data in statfs
+
+    Fixes: http://tracker.ceph.com/issues/16878
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 78adb70c21c6b8e6a9191b76917919b125a9490f)
+
+commit bc324e0446d5fd6b35651b024926c5a3099a0aff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 16:37:07 2017 -0800
+
+    mon: Add warning if diff in OSD usage > config mon_warn_osd_usage_percent (10%)
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8004e6558359fb542e45bb4b483a6c91afdc0b4)
+
+commit 2c2e0a353bd518f1e44435376ad1ad9a0ee6fe67
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:40:05 2017 -0800
+
+    mon: Bump min in ratio to 75%
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 830cc7aa7be1ccd8f54f056b6a58e923cadd1c2b)
+
+commit 899f3fca7f89ab37d20cb1ec06b911ba039c52a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:38:53 2017 -0800
+
+    osd: Fix ENOSPC crash message text
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50cfe03fcba253c8380b21043ed03879134d6836)
+
+commit 19133036602d9dc9eb234b5e4dec6b845b4ade4b
+Merge: 7224ebc1f1 85cefa113b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:50:46 2017 +0200
+
+    Merge pull request #16143 from smithfarm/wip-20365-kraken
+
+    kraken: mon: osd crush set crushmap need sanity check
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7224ebc1f18fec1f472688f6494ed10d2b574f88
+Merge: 47359c12e6 ea186807c3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:48:51 2017 +0200
+
+    Merge pull request #16138 from smithfarm/wip-20034-kraken
+
+    kraken: ceph-disk: Racing between partition creation & device node creation
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 47359c12e69fbde4c36403ccd7992fe4ecf9223b
+Merge: 9860e65818 f04eb66d36
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:47:34 2017 +0200
+
+    Merge pull request #16135 from smithfarm/wip-20010-kraken
+
+    kraken: ceph-disk: separate ceph-osd --check-needs-* logs
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f2d61c199d7ba103290178573984db98e8c8ed21
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 14 10:32:28 2017 -0400
+
+    qa/tasks: rbd-mirror daemon not properly run in foreground mode
+
+    Fixes: http://tracker.ceph.com/issues/20630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fa1918717b25a5ffafbf649eedcfe7d5ab829c2)
+
+commit cf06edd36b176a10e6ce3d62eb957bfdb0146703
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 29 14:54:40 2017 -0400
+
+    rbd: do not attempt to load key if auth is disabled
+
+    Fixes: http://tracker.ceph.com/issues/19035
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b9c8df6d7f0b75c5451953bb322bc1f9afb6299)
+
+commit f72ea6818691e1824d4c22630e856d3b280a008c
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue May 9 15:05:03 2017 +0800
+
+    rgw: when create_bucket use the same num_shards with info.num_shards
+
+    pr #14388 only fix the num_shards in BucketInfo, "init_bucket_index" function still use local num_shards
+
+    Fixes: http://tracker.ceph.com/issues/19745
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 4ce64a190b4ff36985e785e574c077d39796feea)
+
+    Conflicts:
+            src/rgw/rgw_rados.cc - init_bucket_index() called earlier
+
+commit 9182c279cf2f2228471d506113c61566cbfea4bb
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Apr 7 15:34:27 2017 +0800
+
+    rgw: using the same bucket num_shards as master zg when create bucket in secondary zg
+
+    create bucket in secondary zonegroup will forward to master. The master may have different num_shards option.
+    So when create bucket in local, should use master's num_shards instead of local num_shards option.
+
+    Fixes: http://tracker.ceph.com/issues/19745
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit a34c4b8fb13dd5590eb3c6ecb5e55207ed8e3ee8)
+
+    Conflicts:
+        src/rgw/rgw_op.cc - no RGWBulkUploadOp:: methods in kraken; modifications
+            to RGWBulkUploadOp::handle_dir() omitted
+
+commit b758348447e60af23b114aa47c28f151ffd97792
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Fri Apr 14 21:42:45 2017 +0530
+
+    rgw: add a field to store generic user data in the bucket index,
+    that can be populated/fetched via a configurable custom http header
+
+    Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
+    (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)
+
+    Conflicts:
+            src/rgw/rgw_op.cc
+                    Signature fixes for RGWPutObjProcessor_Multipart::do_complete().
+
+            src/rgw/rgw_op.h
+                    A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master version, which is not required in Kraken.
+
+            src/rgw/rgw_rados.cc
+                    In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Kraken.
+                    In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta
+                    In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.
+
+            src/rgw/rgw_rados.h
+                    In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Kraken instead of rgw_obj_index_key
+                    RGWPutObjProcessor_Multipart is not part of this file in Kraken.
+
+            src/rgw/rgw_rest_swift.cc
+                    In RGWListBucket_ObjStore_SWIFT::send_response(), there is no meta struct in Kraken.
+
+            src/rgw/rgw_common.h
+                    Add user_data field in RGWObjEnt structure.
+
+            src/rgw/rgw_json_enc.cc
+                    Add user_data field while dumping RGWObjEnt.
+
+commit 4fa19ce669338bb2d4c3233c70da8a3829a468f6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jul 10 13:31:21 2017 -0700
+
+    osd: repair the PG_DEBUG_REFS build
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ec4185d1980b48b01687a177248d0894f326dc37)
+
+    Conflicts:
+            src/osd/OSD.cc
+            src/osd/Session.h
+
+commit 2a3e66c388892efcecebd8eb3d7bf082be485bab
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed May 24 00:19:51 2017 -0700
+
+    osd: use PGRef instead of boost::intrusive_ptr<PG> everywhere
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 306ad85f0fb4597ba3bb3c04a67abdaba556ba74)
+
+commit 9860e658189e3ac775102b3f66676b0a77a80ba7
+Merge: 6e1023ad24 c65eaaafb1
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Jul 8 21:39:59 2017 -0500
+
+    Merge pull request #16215 from liewegas/wip-17743
+
+    kraken: CMakeLists.txt: disable memstore make check test
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c65eaaafb174ac1f62fe8fc08ae63deadeffd0ad
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 7 11:38:25 2017 -0400
+
+    CMakeLists.txt: disable memstore make check test
+
+    This fails frequently on kraken, but is probably a wontfix.
+
+    See http://tracker.ceph.com/issues/17743
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 379309b552db51dcab9b08ba5437c1dae5cd45a7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 3 21:36:21 2017 -0400
+
+    rbd-mirror: ensure missing images are re-synced when detected
+
+    Fixes: http://tracker.ceph.com/issues/19811
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74bd4f230a0cb7b709f2cb5c6db3dc79f0d8dede)
+
+    Conflicts:
+            src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc: trivial resolution
+            src/tools/rbd_mirror/image_replayer/BootstrapRequest.h: trivial resolution
+
+commit 6e1023ad2482f5829c06a54adb4271643b1e53c4
+Merge: 6353508134 076e4b7991
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:49:51 2017 +0200
+
+    Merge pull request #14067 from asheplyakov/19322-bp-kraken
+
+    kraken: rgw: fix break inside of yield in RGWFetchAllMetaCR
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6353508134bd4d2bcbca7d0f9a42788895fe5c27
+Merge: 425992f92d 87811ff161
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:48:57 2017 +0200
+
+    Merge pull request #14509 from smithfarm/wip-18499-kraken
+
+    kraken: rgw: Realm set does not create a new period
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 425992f92d30c8503053311471322a9849a37746
+Merge: 39bf493984 6860fce424
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:48:01 2017 +0200
+
+    Merge pull request #14511 from smithfarm/wip-18772-kraken
+
+    kraken: rgw: crash when updating period with placement group
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 39bf493984cb98c5a3f7691458188fea7a246c70
+Merge: eb397c1780 1cb9868456
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:45:15 2017 +0200
+
+    Merge pull request #14513 from smithfarm/wip-18843-kraken
+
+    kraken: rgw: usage stats and quota are not operational for multi-tenant users
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit eb397c178028530d061e9edc32f84ddd6d826a89
+Merge: 1ed342d75d 5f4c38fa6b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:44:21 2017 +0200
+
+    Merge pull request #14515 from smithfarm/wip-18904-kraken
+
+    kraken: rgw: first write also tries to read object
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1ed342d75d0befb602597385b5f49d27a244d450
+Merge: d3a2417cf1 935cc11595
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:42:36 2017 +0200
+
+    Merge pull request #14516 from smithfarm/wip-18909-kraken
+
+    kraken: rgw: the swift container acl does not support field .ref
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit d3a2417cf11e6ccebf25de8fe12242c1e8eda626
+Merge: ce95f0f7d8 ffe5bbfc6e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:25:43 2017 +0200
+
+    Merge pull request #14517 from smithfarm/wip-19047-kraken
+
+    kraken: rgw: fix use of marker in List::list_objects()
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ce95f0f7d81af4bfbad7afb4f429b9cfa2e37726
+Merge: aab812c517 03e5be4498
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:24:13 2017 +0200
+
+    Merge pull request #14524 from smithfarm/wip-19477-kraken
+
+    kraken: rgw: S3 v4 authentication issue with X-Amz-Expires
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit aab812c51706fc3395af3c34b0b3720e84c30b61
+Merge: a441042e00 a5fe812483
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:22:26 2017 +0200
+
+    Merge pull request #14525 from smithfarm/wip-19479-kraken
+
+    kraken: rgw: zonegroupmap set does not work
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a441042e00cb752737e303c4c38322fadf13a2f6
+Merge: 3b128e458f cac2647189
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:21:22 2017 +0200
+
+    Merge pull request #14528 from smithfarm/wip-19534-kraken
+
+    kraken: rgw: Error parsing xml when get bucket lifecycle
+
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3b128e458f86a23e8b6ff987b0f0774e7d5597bb
+Merge: 6366b2382b 8f81bb33c7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:19:08 2017 +0200
+
+    Merge pull request #14606 from asheplyakov/19608-kraken
+
+    kraken: rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6366b2382b8d457ab9e544cc889aa77f024a626f
+Merge: 19280ae91e 727653623f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:18:10 2017 +0200
+
+    Merge pull request #15384 from asheplyakov/20015-bp-kraken
+
+    kraken: cls/rgw: list_plain_entries() stops before bi_log entries
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 19280ae91e710cbb9569fc1dc3a6a24b6ea05ca4
+Merge: eb56af6c0d 595c31f390
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:03:16 2017 +0200
+
+    Merge pull request #14522 from smithfarm/wip-19472-kraken
+
+    kraken: rgw: add the remove-x-delete feature to cancel swift object expiration
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit eb56af6c0d8eea0e19a16da4e186e8edbfe13148
+Merge: 4ccd32619f f8db63f24d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:00:43 2017 +0200
+
+    Merge pull request #14523 from smithfarm/wip-19475-kraken
+
+    kraken: rgw: multisite: EPERM when trying to read SLO objects as system/admin user
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 4ccd32619f39a4f6fa7a6700c8c90c622c83d67a
+Merge: e59eba0ff3 c3d8444890
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:58:44 2017 +0200
+
+    Merge pull request #14519 from smithfarm/wip-19175-kraken
+
+    kraken: swift API: cannot disable object versioning with empty X-Versions-Location
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit e59eba0ff375011c18b3a8dcdb228c71c9085fe9
+Merge: aaa7992ee5 faf80bc1dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:56:47 2017 +0200
+
+    Merge pull request #14526 from smithfarm/wip-19524-kraken
+
+    kraken: rgw: 'radosgw-admin zone create' command with specified zone-id creates a zone with different id
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit aaa7992ee5fd2767d8a144c47255489080f5fb47
+Merge: 6c3daeb297 e5719234b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:55:30 2017 +0200
+
+    Merge pull request #14529 from smithfarm/wip-19573-kraken
+
+    kraken: rgw: Response header of swift API returned by radosgw does not contain x-openstack-request-id. But Swift returns it.
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit ce874ab676e8b5a09e32db19f077c650cff9979b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:59:19 2017 +0800
+
+    test/librbd: decouple ceph_test_librbd_api from libceph-common
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b7287fdc4d70c5ecedda78ae367b98e5d8f61c5b)
+
+    Conflicts:
+            src/test/librbd/CMakeLists.txt: trivial resolution
+
+commit 57f7213d9fd5a9a82e46c4ca3f3dda87c3dde717
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:31:53 2017 +0800
+
+    test/librbd: replace libcommon classes using standard library
+
+    so ceph_test_librbd_api has less dependencies on libcommon
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc8fa0193c8b2fb4b749ce9976bf6efbea833e96)
+
+    Conflicts:
+            src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 7d044c6fbdcabc92d4dae7d9759dc63f87b2e4f4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:58:47 2017 +0800
+
+    test/librados: extract functions using libcommon in test.cc into test_common.cc
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82a848c2053ea69ebc6d3ec1003e18921e2d08d2)
+
+    Conflicts:
+            src/test/librados/CMakeLists.txt: trivial resolution
+            src/test/librados/test.cc: trivial resolution
+
+commit 53024570e75f2963da84f3a045f1ac1f9ee4081e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 22 12:27:12 2017 +0800
+
+    mds: issue new caps when sending reply to client
+
+    After Locker::issue_new_caps() adds new Capability data struct,
+    do not issue caps immediately. Let CInode::encode_inodestate()
+    do the job instead. This can avoid various races that early reply
+    is not allowed, caps that haven't been sent to client gets revoked.
+
+    Fixes: http://tracker.ceph.com/issues/19635
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 799703a4acb49db0b6cc99a23e4326767e694c3a)
+
+commit 18f1830e8acaa386d0113171ec649b5b33698dd9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 08:17:05 2017 -0400
+
+    librbd: filter expected error codes from is_exclusive_lock_owner
+
+    Fixes: http://tracker.ceph.com/issues/20182
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4daaf54e6bc42cd4fb2111ea20b2042941b0c31)
+
+commit 45b4c86452d19eaf3078704ba951e2fdb2e49a30
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 13:17:19 2017 -0400
+
+    rbd: properly decode features when using image name optional
+
+    Fixes: http://tracker.ceph.com/issues/20185
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1b05a2b062a59ec1b6682f7683bfd816433a931)
+
+commit d463de9b8ae357a4edbfcf63e65740ecd4845b26
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Tue Jun 13 16:40:16 2017 +0800
+
+    rgw: meta sync thread crash at RGWMetaSyncShardCR
+
+    Fixes: http://tracker.ceph.com/issues/20251
+
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 45877d38fd9a385b2f8b13e90be94d784898b0b3)
+
+    Conflicts:
+            src/rgw/rgw_data_sync.cc
+              - kraken RGWContinuousLeaseCR() has slightly different options
+            src/rgw/rgw_sync.cc
+              - kraken RGWContinuousLeaseCR() has slightly different options
+              - added "override" qualifier to ~RGWFetchAllMetaCR() definition
+
+commit d52aeec0310d3f1dab986a43bee2f4fc768efc06
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Jun 10 18:12:52 2017 -0400
+
+    rgw: only log metadata on metadata master zone
+
+    Fixes: http://tracker.ceph.com/issues/20244
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b8272f3607074a2f7cbfd08f7bbc82f22cf120ba)
+
+commit 9651ed95894707d6d5517979b18a8acdbcc1b004
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 21 15:04:48 2017 -0400
+
+    radosgw-admin: warn that 'realm rename' does not update other clusters
+
+    Fixes: http://tracker.ceph.com/issues/19746
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a459c768ffecd689a53a79dfe33eb8f1bbc318f)
+
+commit 4e4cd5453670af8d87b217c60dc84dd39e192952
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:42:44 2017 -0400
+
+    rgw: fix for zonegroup redirect url
+
+    local dest_url variable was shadowing the one in the enclosing scope, so
+    the changes were not applied and no Location header was written on redirect
+
+    Fixes: http://tracker.ceph.com/issues/19488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 542e188a40f0495720b48308372366951ae41e62)
+
+commit 693d88081358f2059a65de5ddb4d35272ef1f3fe
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:41:51 2017 -0400
+
+    rgw: use zonegroup's master zone endpoints for bucket redirect
+
+    if no zonegroup endpoints are set, fall back to master zone endpoints
+
+    Fixes: http://tracker.ceph.com/issues/19488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 92f63c6392bdc4633a2e57cb3867051bb1a3fd55)
+
+commit f23df08430c6b62314898a4c95f81d8cb01f93c3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 14:20:20 2017 -0400
+
+    rgw: allow larger payload for period commit
+
+    testing with 3 zonegroups and 3 zones each, the period json grew larger
+    than 4k and caused decode failures on period commit
+
+    updated to use the new config variable rgw_max_put_param_size
+
+    Fixes: http://tracker.ceph.com/issues/19505
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f2871fe59d933b03f37fde40f1781b2320d0d50)
+
+commit dea9a69c3f85504070a56f8a2dbf1a4c129b9ef4
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Mar 3 16:50:33 2017 +0100
+
+    rgw: make a configurable size for requests with xml params
+
+    We currently read all user input for a few apis accepting xml, avoid
+    doing this and error out early, most s3 apis limit to about 1000 xml
+    entries, make this configurable via a new config param,
+    `rgw_max_put_param_size` defaulting to 1MB. Also modified
+    `rgw_rest_read_all_input` with an additional param to not allow for
+    chunked uploads, though we error out in a better way by responding with
+    content length required rather than S3's 501 if chunked is set.
+
+    This also adds the same behavior in RGWPutCORS reusing
+    `rgw_rest_read_all_input`, and using a data_deleter to manage the char*
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+    rgw: rgw_rest_s3: make PUT CORS accept only a configured max input
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 35375b75160dc93beeb04c7fe010dc0d21952534)
+
+commit ac706f2b4a4aee7b813a0b5ba0589431b84f3dcd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 23 12:07:45 2017 +0200
+
+    librbd: potential read IO hang when image is flattened
+
+    Fixes: http://tracker.ceph.com/issues/19832
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 10d58618e7c632ef01b9537492239e0a042dc17e)
+
+commit b27595ae9c3490a4ff1d39b0d864a4a47a941981
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Jun 14 10:29:53 2017 +0800
+
+    rgw: lifecycle thread shouldn't process the bucket which has been deleted.
+
+    Fixes: http://tracker.ceph.com/issues/20285
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit a2b042fe9f7e6503273fa4c2a4c56e399d7c338f)
+
+commit 3105e2327ea29558fdd97dfb6da2c7ee9ec08f8f
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue May 9 16:39:39 2017 +0800
+
+    rgw: VersionIdMarker and NextVersionIdMarker should be returned when listing
+    object versions if necessary.
+
+    Fixes: http://tracker.ceph.com/issues/19886
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit f805c3e08948e379b7d2c4f2faf9e7f550e4cb23)
+
+commit d0e742cafae325cf794a2160b6df19296ac398a4
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Thu May 4 15:58:37 2017 +0800
+
+    rgw: set object accounted size correctly
+
+    sometimes, object accounted size is set wrong,
+    because we don't konw the object size if don't resort to the compression info or manifest.
+    e.g, when i use s3cmd do copy object(bucket_A/obj_A -> bucket_B/obj_B, assume the size of obj_A is 4M).
+    then i use s3cmd do list bucket, I got obj_B size is 512K, it is the head size apparently.
+
+    Fixes: http://tracker.ceph.com/issues/20071
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit 539985a99eebdc72c8d2446acc1108664a162f68)
+
+commit 640a7a26296e652ffb8b8bfab87839d7bfd66d1b
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Sat May 27 15:20:30 2017 +0800
+
+    rgw: get wrong content when download object with specific range when
+    compression was enabled
+
+    look at the prototype:
+    RGWGetObj_Decompress::handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+    we should trim the bl using bl_ofs and bl_len.
+
+    Fixes: http://tracker.ceph.com/issues/20100
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit afe9d99570059b88837690950a4f4525e0e60262)
+
+commit 6c3daeb2970f52970145eb31d4bef56b33df8b40
+Merge: 08b8a736e2 bd9aec00ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 17:46:06 2017 +0200
+
+    Merge pull request #16113 from smithfarm/wip-19807-kraken
+
+    kraken: tests: remove hard-coded image name from TestLibRBD.Mirror
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 1fc14857a9dd81b1b4726428f115ef5cfb495781
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jun 6 15:43:20 2017 +0800
+
+    rgw: datalog trim and mdlog trim handles the result returned by osd
+    incorrectly.
+
+    Fixes: http://tracker.ceph.com/issues/20190
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 7fd6e031e5b0b1f3eca70c5b459d50f6f214171f)
+
+commit 90288afc10d878b59dacb99383ada309377e4bb4
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Mon Mar 20 11:41:48 2017 +0530
+
+    rgw: Added code to correctly account for bytes sent/ received during a 'PUT' operation.
+
+    Currently, the bytes sent/ received are both set to zero after
+    an object is uploaded to a bucket. Added code to correct the logic.
+
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 85735c972f5db1a110f55e01a4f6249202152553)
+
+commit dda4d912cd753e1443beac4e7f92d97f24b57992
+Author: redickwang <redickwang@tencent.com>
+Date:   Fri May 19 15:08:12 2017 +0800
+
+    rgw: segment fault when shard id out of range
+
+    Fixes: http://tracker.ceph.com/issues/19732
+
+    Signed-off-by: redickwang <redickwang@tencent.com>
+    (cherry picked from commit ff4c40fc2e3c092d17a218ae6132de4e6034c8ee)
+
+commit f32e26e909c3d56160e9a33d051d690fa4366bbd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Apr 6 17:08:23 2017 +0200
+
+    rgw: fix broken /crossdomain.xml, /info and /healthcheck of Swift API.
+
+    Fixes: http://tracker.ceph.com/issues/19520
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit fe2afd42aaf706d38d6ed611796a181ae9e02ae8)
+
+commit ccb33bab37e45bf47dad5577602e9be9d1d683c6
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 25 14:46:09 2017 +0200
+
+    rgw: use a vector for options passed to civetweb
+
+    Since the array we used needs additional check to ensure that the size
+    is correct, and causes undefined behaviour in a few platforms, using a
+    vector and passing the c array back to mg_start so that we don't go past
+    the end of array.
+
+    Fixes: http://tracker.ceph.com/issues/19749
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Jesse Williamson <jwilliamson@suse.de>
+    (cherry picked from commit 3959a8b52c2910e4060f7d273c8939bbdc83e48a)
+
+commit f8235c5c0a64eb409957d80d4c4a22a87280d0b2
+Author: Willem Jan Withagen <wjw@digiware.nl>
+Date:   Sat Apr 29 13:36:07 2017 +0200
+
+    cls/log/cls_log.cc: reduce logging noise
+
+     - The other reference in the source as already at 20.
+          ./src/cls/timeindex/cls_timeindex.cc:85:
+            CLS_LOG(20, "storing entry at %s", index.c_str());
+
+       And we need not always know where in the log items are stored.
+       So it looks like a leftover debug feature.
+
+    Fixes: http://tracker.ceph.com/issues/19835
+    Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>
+    (cherry picked from commit d76010900bf9012f2e66335787710531772766b7)
+
+commit e773b304eefa3d2ca7c1fe0817c89082bf574a38
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Apr 13 05:33:55 2017 -0400
+
+    rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+
+    Keystone tokens can be revoked.  This causes them to fail
+    validation.  However, in ceph, we cache them.  As long as
+    they're in the cache we trust them.  To find revoked tokens
+    there's a call OSI-PKI/revoked but that's only useful for
+    pki tokens.  Installations using fernet/uuid may not even
+    have the proper credentials to support the call, in which
+    case the call blows up in various ways filling up logs
+    with complaints.
+
+    This code makes the revocation thread optional; by disabling it,
+    the complaints go away.  A further fix is in the works
+    to use other more modern calls available in modern keystone
+    installations to properly deal with non-PKI/PKIZ tokens.
+
+    To disable the revocation thread, use at least one of these:
+            rgw_keystone_token_cache_size = 0
+                    using this will cause tokens to be validated on every call.
+    You may instead want to set
+            rgw_keystone_revocation_interval = 0
+                    using just this will disable the revocation thread,
+                    but leaves the cache in use.  That avoids the extra
+                    validation overhead, but means token revocation won't
+                    work very well.
+
+    Fixes: http://tracker.ceph.com/issues/9493
+    Fixes: http://tracker.ceph.com/issues/19499
+
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 003291a8cbca455c0e8731f66759395a0bb1f555)
+
+commit 68a853a14d89261fcd0a132a1a7a7151721b84c8
+Author: lvshuhua <lvshuhua@cmss.chinamobile.com>
+Date:   Wed Dec 7 15:47:47 2016 +0800
+
+    rgw: fix versioned bucket data sync fail when upload is busy
+
+    Fixes: http://tracker.ceph.com/issues/18208
+
+    Signed-off-by: lvshuhua <lvshuhua@cmss.chinamobile.com>
+    (cherry picked from commit ce7d00ac1642d84c1d9111156a544c37801c5adf)
+
+commit 9a7a73edf1f432df98984ab7ad60306aa6aca6e3
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+
+    Fixes: http://tracker.ceph.com/issues/18828
+
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 08b8a736e2b49bba39cf8fe51fe3435a7d381e72
+Merge: f45de0f39d 405503472e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:28:26 2017 +0200
+
+    Merge pull request #16132 from smithfarm/wip-20522-kraken
+
+    kraken: FAILED assert(object_contexts.empty()) (live on master only from Jan-Feb 2017, all other instances are different)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f45de0f39db316219207d674198ecc578ed76f7e
+Merge: 4ca8f1f4a9 ff19977fbc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:27:37 2017 +0200
+
+    Merge pull request #15962 from asheplyakov/20443-bp-kraken
+
+    kraken: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4ca8f1f4a9c5cdb15bd273f74df7ac6cda218f66
+Merge: d45c64d1b9 2aec591cfe
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:26:40 2017 +0200
+
+    Merge pull request #15066 from asheplyakov/19916-bp-kraken
+
+    kraken: osd: osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d45c64d1b94fb6a91c78e8333f924a2be08fcddb
+Merge: 4890ba0df0 62bb2086af
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:24:26 2017 +0200
+
+    Merge pull request #14942 from shinobu-x/wip-18293-kraken
+
+    kraken: osd: leaked MOSDMap
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4890ba0df02403b8bc5348235d1aadb76ab8f4c1
+Merge: c45570b4a6 d5aea7a68b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:20:56 2017 +0200
+
+    Merge pull request #14736 from smithfarm/wip-19326-kraken
+
+    kraken: bluestore bdev: flush no-op optimization is racy
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c45570b4a61243839e7d4e64c5e1f1b017bb3948
+Merge: 5a1116c0d5 e76725f4e9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:19:50 2017 +0200
+
+    Merge pull request #14732 from smithfarm/wip-19560-kraken
+
+    kraken: objecter: full_try behavior not consistent with osd
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5a1116c0d5c1fa8810031104cd093ed1b1e8cb3a
+Merge: 7133d4db9b 1bc9cfff42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:18:24 2017 +0200
+
+    Merge pull request #13542 from shinobu-x/wip-18929-kraken
+
+     kraken: osd: bogus assert when checking acting set on recovery completion in rados/upgrade
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7133d4db9b31b0627c6278f2f72b690ea366551e
+Merge: 6db580b6cf 56ca3b0190
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:11:09 2017 +0200
+
+    Merge pull request #14939 from rzarzynski/wip-rgw-19754-kraken
+
+    kraken: rgw: fix RadosGW hang during multi-chunk upload of AWSv4
+
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6db580b6cfdf3ef14403adbbcd27a2f16cbf15fc
+Merge: be8b7610d1 59391c40db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 08:54:16 2017 +0200
+
+    Merge pull request #14530 from smithfarm/wip-19574-kraken
+
+    kraken: rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response()
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit be8b7610d164bf2798bb4b9676db330ef5fb6c27
+Merge: e12eae92b0 1f86be6e19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 08:40:16 2017 +0200
+
+    Merge pull request #13868 from smithfarm/wip-19160-kraken
+
+    kraken: rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 85cefa113b31c9dc6b8a7a88e3343de923278c7d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 18 10:04:20 2017 +0100
+
+    mon: osd crush set crushmap need sanity check
+
+    The sanity check verifying the new crushmap does not remove crush rules
+    that are in use is not exclusive to ceph setcrushmap.
+
+    Fixes: http://tracker.ceph.com/issues/19302
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed760457bf154c10adf75c6df046eecab7eb8e4b)
+
+commit 9a26882289e1746fe8ca1443666ed0a583e07021
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 6 15:13:50 2017 -0400
+
+    ceph_test_rados_api_misc: fix LibRadosMiscConnectFailure.ConnectFailure retry
+
+    Fixes: http://tracker.ceph.com/issues/19901
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 087dff80cac707ee9bcb5bcfc98cb3ec047bd49f)
+
+commit 91569f63852f12d958175bf970967fa9d25b2cfc
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Jan 5 11:50:42 2017 +0800
+
+    rgw: fix 'gc list --include-all' command infinite loop the first 1000 items
+
+    When the items to gc over 1000, 'gc list --include-all' command will infinite loop the first 1000 items.
+    Add next_marker to move to the next 1000 items.
+
+    Fixes: http://tracker.ceph.com/issues/19978
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit fc29f52ebca63104a05515484088ff136dfb0b15)
+
+commit ea186807c3fb7ac79f8ef1a8ae874ced69aec6e1
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Fri Mar 31 14:54:33 2017 +0200
+
+    ceph-disk: Adding retry loop in get_partition_dev()
+
+    There is very rare cases where get_partition_dev() is called before the actual partition is available in /sys/block/<device>.
+
+    It appear that waiting a very short is usually enough to get the partition beein populated.
+
+    Analysis:
+    update_partition() is supposed to be enough to avoid any racing between events sent by parted/sgdisk/partprobe and
+    the actual creation on the /sys/block/<device>/* entrypoint.
+    On our CI that race occurs pretty often but trying to reproduce it locally never been possible.
+
+    This patch is almost a workaround rather than a fix to the real problem.
+    It offer retrying after a very short to be make a chance the device to appear.
+    This approach have been succesful on the CI.
+
+    Note his patch is not changing the timing when the device is perfectly created on time and just differ by a 1/5th up to 2 seconds when the bug occurs.
+
+    A typical output from the build running on a CI with that code.
+            command_check_call: Running command: /usr/bin/udevadm settle --timeout=600
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_partition_dev: Try 1/10 : partition 2 for /dev/sda does not in /sys/block/sda
+            get_partition_dev: Found partition 2 for /dev/sda after 1 tries
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_dm_uuid: get_dm_uuid /dev/sda2 uuid path is /sys/dev/block/8:2/dm/uuid
+
+    fixes: #19428
+
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 93e7b95ed8b4c78daebf7866bb1f0826d7199075)
+
+commit f7f6375b54233f1085ba8d64df1053c038ff3991
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Wed Mar 22 10:11:44 2017 +0100
+
+    ceph-disk: Reporting /sys directory in get_partition_dev()
+
+    When get_partition_dev() fails, it reports the following message :
+        ceph_disk.main.Error: Error: partition 2 for /dev/sdb does not appear to exist
+    The code search for a directory inside the /sys/block/get_dev_name(os.path.realpath(dev)).
+
+    The issue here is the error message doesn't report that path when failing while it might be involved in.
+
+    This patch is about reporting where the code was looking at when trying to estimate if the partition was available.
+
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 413c9fcfbe8e6ab33d73b8428090ccacc33c5d15)
+
+commit 533ff8a540b0b30378a9562953f1da46291e9832
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed May 3 18:28:01 2017 +0800
+
+    qa/tasks/ceph_manager: always fix pgp_num when done with thrashosd task
+
+    Fixes: http://tracker.ceph.com/issues/19771
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da1161cbd8b50b8a980e8a3b48154a97f988426c)
+
+commit f04eb66d365c088e0e6985864d54bdc4fc3d3bfd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue May 9 12:32:51 2017 +0200
+
+    ceph-disk: separate ceph-osd --check-needs-* logs
+
+    It is using the OSD id zero but have nothing to do with OSD zero and
+    this is confusing to the user. The log themselves do not need to be kept
+    around and are stored in the run directory so that they can be disposed
+    of after reboot.
+
+    Fixes: http://tracker.ceph.com/issues/19888
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c7b3c46bd63b78475868e405bf20d9c142f0336a)
+
+commit a7af766aa89b9435c72cdb93e5d67a9e9331c635
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:18:08 2016 -0500
+
+    msg/simple: clear_pipe when wait() is mopping up pipes
+
+    When wait is mopping up connections it may hit one that
+    is in the process of accepting.  It will unregister it
+    whilst the accept() thread is trying to set it up,
+    aborting the accept and getting it reaped.  However,
+    the pipe mop-up does not clear_pipe() the way that
+    mark_down(), mark_down_all(), and fault() do, which
+    leads to this assert.
+
+    Pipe is accepting...
+
+      -161> 2016-12-22 17:31:45.460613 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept:  setting up session_security.
+      -160> 2016-12-22 17:31:45.460733 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept new session
+      -159> 2016-12-22 17:31:45.460846 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept success, connect_seq = 1, sending READY
+      -158> 2016-12-22 17:31:45.460959 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept features 1152921504336314367
+
+    wait() is shutting down...
+
+      -156> 2016-12-22 17:31:45.461882 9506ac0 20 -- 172.21.15.14:6804/20738 wait: stopping accepter thread
+      -155> 2016-12-22 17:31:45.462994 9506ac0 10 accepter.stop accept listening on: 15
+    ...
+      -116> 2016-12-22 17:31:45.482137 9506ac0 10 -- 172.21.15.14:6804/20738 wait: closing pipes
+      -115> 2016-12-22 17:31:45.482850 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).unregister_pipe
+      -114> 2016-12-22 17:31:45.483421 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).stop
+
+    ...which interrupts the accept()...
+
+      -113> 2016-12-22 17:31:45.484164 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept fault after register
+
+    and makes accept() return failure, and reader() to exit
+    and reap...
+
+      -110> 2016-12-22 17:31:45.486103 9506ac0 10 -- 172.21.15.14:6804/20738 wait: waiting for pipes 0x3e2a5c20 to close
+      -109> 2016-12-22 17:31:45.487146 37353700 10 -- 172.21.15.14:6804/20738 queue_reap 0x3e2a5c20
+      -108> 2016-12-22 17:31:45.487658 9506ac0 10 -- 172.21.15.14:6804/20738 reaper
+      -107> 2016-12-22 17:31:45.487722 9506ac0 10 -- 172.21.15.14:6804/20738 reaper reaping pipe 0x3e2a5c20 172.21.15.35:0/146098963
+      -106> 2016-12-22 17:31:45.487816 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).discard_queue
+      -105> 2016-12-22 17:31:45.494742 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).reader done
+    ...
+       -92> 2016-12-22 17:31:45.527589 9506ac0 -1 /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: In function 'void SimpleMessenger::reaper()' thread 9506ac0 time 2016-12-22 17:31:45.488264
+    /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: 235: FAILED assert(!cleared)
+
+    Fixes: http://tracker.ceph.com/issues/15784
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 948f97b3bdd39269a38277238a61f24e5fec6196)
+
+commit 6d2f95973561c9f0911fce04dff2036f73a77442
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 28 13:36:01 2016 -0500
+
+    test: added missing IoCtx copy/assignment methods in librados_test_stub
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c35d307c62961072b4c00f94e37486e3371c1a21)
+
+commit 405503472ea6b3a2965455c8c064dc0e4d713ad4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Feb 14 12:47:37 2017 -0800
+
+    ReplicatedBackend: don't queue Context outside of ObjectStore with obc
+
+    We only flush the ObjectStore callbacks, not everything else.  Thus,
+    there isn't a guarrantee that the obc held by pull_complete_info will
+    be cleaned up before the Flush callback is triggered.  Instead, just
+    defer clearing the pull state until the callback (it'll be cleaned up
+    during the interval change) and remove the ObjectContext from
+    pull_complete_info.
+
+    Introduced: 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8
+    Fixes: http://tracker.ceph.com/issues/18927
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 51eee55c475a3d931844831e040ed3d66ee59af4)
+
+commit 5257d1a643fb1af36b80650ea2b014b4887dc10f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 3 13:12:47 2017 -0800
+
+    osd/: don't leak context for Blessed*Context or RecoveryQueueAsync
+
+    This has always been a bug, but until
+    68defc2b0561414711d4dd0a76bc5d0f46f8a3f8, nothing deleted those contexts
+    without calling complete().
+
+    Fixes: http://tracker.ceph.com/issues/18809
+    Bug shadowed until: 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 91b74235027c8a4872dcab6b37767b12c3267061)
+
+commit e12eae92b04d30d322740a6c7a33d86e448bf9c5
+Merge: 7bdc3ef238 73fe367bc4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:04:41 2017 +0200
+
+    Merge pull request #15792 from liewegas/wip-bluestore-leaks-kraken
+
+    kraken: os/bluestore: deep decode onode value
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7bdc3ef238228bb156f8f97867a66138fcc93807
+Merge: 0a71e1899b 27e9644ceb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:01:58 2017 +0200
+
+    Merge pull request #15729 from asheplyakov/20315-bp-kraken
+
+    kraken: mon: fail to form large quorum; msg/async busy loop
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0a71e1899bd8ee84640059e18126461d3a1748a3
+Merge: d7598df528 d38a752999
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:00:17 2017 +0200
+
+    Merge pull request #15084 from asheplyakov/19928-bp-kraken
+
+    kraken: mon crash on shutdown, lease_ack_timeout event
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d7598df5281e1648c5890a4b18b7b64d74b8dc95
+Merge: 0dbbd5001e 47a4544176
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:59:03 2017 +0200
+
+    Merge pull request #14735 from smithfarm/wip-19685-kraken
+
+    kraken: osd: Give requested scrubs a higher priority
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0dbbd5001ea84906d15bb27c99b73493d20eab75
+Merge: 0d177d5c99 7a275769b6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:57:53 2017 +0200
+
+    Merge pull request #14733 from smithfarm/wip-19561-kraken
+
+    kraken: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0d177d5c992ca8cd99afa8e9be3f429a933d490a
+Merge: 11ed483791 0227920438
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:51:26 2017 +0200
+
+    Merge pull request #13883 from shinobu-x/wip-19119-kraken
+
+    kraken: pre-jewel "osd rm" incrementals are misinterpreted
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 11ed483791849905910c879152d1eea41d95604e
+Merge: 74b0c9da46 fa2753245d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 19:39:54 2017 +0200
+
+    Merge pull request #14646 from gregsfortytwo/wip-17916-kraken
+
+    kraken: osd: pglog: with config, don't assert in the presence of stale diverg…
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 74b0c9da461825ef53ddf79ec00d08168e33025c
+Merge: 7a87735243 648c6adf52
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 18:41:59 2017 +0200
+
+    Merge pull request #15237 from asheplyakov/20035-bp-kraken
+
+    kraken: mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7a877352430e49f94dff614dc0a697a4697f6a77
+Merge: 4a7c26083f 5c7a07677a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 18:19:05 2017 +0200
+
+    Merge pull request #14331 from shinobu-x/wip-19396-kraken
+
+    kraken: Objecter::epoch_barrier isn't respected in _op_submit()
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4a7c26083feebca273044691c7f7ef0dbf8305eb
+Merge: f978b2514c 071f111abb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:54:30 2017 +0200
+
+    Merge pull request #16083 from smithfarm/wip-18677-kraken
+
+    kraken: osd: metadata reports filestore when using bluestore
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f978b2514cb17dabd341b5c801bd541a489071a7
+Merge: 56c65b120f 7054ff69d0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:51:36 2017 +0200
+
+    Merge pull request #16098 from smithfarm/wip-19621-kraken
+
+    kraken: rbd-nbd: add signal handler
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 56c65b120f1ef7323109889c6a82287e5e4f5371
+Merge: e13f5467fa 27cbff1114
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:50:21 2017 +0200
+
+    Merge pull request #15612 from smithfarm/wip-19611-kraken
+
+    kraken: rbd: Issues with C API image metadata retrieval functions
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e13f5467fac1e296d2d56b8ed4dc89d5cdfa0f5d
+Merge: 19deb31d39 b19d6eb311
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:48:24 2017 +0200
+
+    Merge pull request #14833 from smithfarm/wip-19794-kraken
+
+    kraken: tests: test_notify.py: assert(not image.is_exclusive_lock_owner()) on line 147
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 19deb31d399715716b02c8fdf58ea2845b42d58c
+Merge: cc5241e4aa c79d45ff52
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:26:58 2017 +0200
+
+    Merge pull request #14543 from smithfarm/wip-19173-kraken
+
+    kraken: rbd: rbd_clone_copy_on_read ineffective with exclusive-lock
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cc5241e4aab227b8b272774bb5d4ff1f37ed3446
+Merge: e705528331 9efeb6bc2d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:37:01 2017 +0200
+
+    Merge pull request #14974 from dillaman/wip-19872-kraken
+
+    kraken: rbd-mirror: failover and failback of unmodified image results in split-brain
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e70552833137eed1f9ede6c3eb59d345a50ac9ce
+Merge: 2b73b570e0 d51b755391
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:36:14 2017 +0200
+
+    Merge pull request #14934 from dillaman/wip-19833-kraken
+
+    kraken: cls_rbd: default initialize snapshot namespace for legacy clients
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2b73b570e060d2c77b6c86579cecc84221b166f9
+Merge: dbe8a8ba84 e0f90f03dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:30:07 2017 +0200
+
+    Merge pull request #14539 from smithfarm/wip-18771-kraken
+
+    kraken: rbd: Improve compatibility between librbd + krbd for the data pool
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit baa772372c52ca2f5b27bad59798734def8656f1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 12 10:38:51 2017 +0800
+
+    pybind: fix cephfs.OSError initialization
+
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "cephfs.pyx", line 672, in cephfs.LibCephFS.open (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:10160)
+      File "cephfs.pyx", line 155, in cephfs.OSError.__init__ (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:1889)
+    TypeError: __init__() takes exactly 3 positional arguments (2 given)
+
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e6493f64ba4592b8dca54ece4464efa6c7f331a7)
+
+commit ac9aed9c2c46bd64384ae047f461b990cb3f6ffe
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 10 08:13:52 2017 +0800
+
+    pybind: fix open flags calculation
+
+    (O_WRONLY | O_RDWR) is invaild open flags
+
+    Fixes: http://tracker.ceph.com/issues/19890
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 2c25c99cb4572ffae97555a56b24a4c4097dcdec)
+
+commit bd9aec00ec284e2e828625b5e92165465aee38b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 27 16:21:33 2017 -0400
+
+    test: remove hard-coded image name from RBD metadata test
+
+    Fixes: http://tracker.ceph.com/issues/19798
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8f72e745e157cc12d76ca6babe956c5698ee297f)
+
+commit 474d504dd67378588d2dd243b06b844737ab64d5
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Mar 2 17:18:18 2017 +0100
+
+    librbd: relax "is parent mirrored" check when enabling mirroring for pool
+
+    If the parent is in the same pool and has the journaling feature enabled
+    we can assume the mirroring will eventually be enabled for it.
+
+    Fixes: http://tracker.ceph.com/issues/19130
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe31bca22f90ce02f461d6421a4f66539db888d3)
+
+commit 44c116dfd6f519538b6c1401e3c3bc283d31b78a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 10:40:46 2017 +0100
+
+    rbd-mirror: check image mirroring state when bootstrapping
+
+    Fixes: http://tracker.ceph.com/issues/18447
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5fc5a8ac895524f05eed6e7db20b0dda3a8cb60f)
+
+commit d5fa6a65b52c3f2cf5363ebb55af7e96d8be90ab
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 09:23:19 2017 +0100
+
+    rbd-mirror: async request to test if image is primary
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0a1cb35caacdf85029f31a0364dc07a5d7462f5f)
+
+commit 4035dc6fdaf9df06ffeb8b86813046a1b5e0cff2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 15:02:02 2017 +0100
+
+    rbd-mirror: hold owner lock when testing if lock owner
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 82aa89668d666f434cd19ff444223017b5512c6a)
+
+commit 76fedab57612ea0e15d18729e779ec43bf830704
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Dec 27 19:39:51 2016 +0800
+
+    os/bluestore: fix OnodeSizeTracking testing
+
+    [  FAILED  ] 1 test, listed below:
+    [  FAILED  ] ObjectStore/StoreTest.OnodeSizeTracking/2, where GetParam() = "bluestore"
+
+     1 FAILED TEST
+
+    The above test failure happens as the bluestore mount() process
+    will try to load all collections and put them into the coll_map,
+    which will be also tracked as mempool::bluestore_meta_other.
+    So total_bytes from mempool won't be equal to zero.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 1d97862f716ce24bfe227b192ecc5e1eb3f76757)
+
+commit 19c7524ace7aeb4f1bc7986162191f4a88bcb4d2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 23 11:12:01 2017 -0400
+
+    qa/tasks/radosbench: increase timeout
+
+    The current timeout isn't enough in some cases (powercycle thrashing leaves
+    osds down for a long time because rebooting is so slow).
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dfa95db57d2c73923918ecf7014c801a925ddfb7)
+
+commit fa88fc77b4df0c548cda8783334862e33867abac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 14:49:13 2017 -0400
+
+    librbd: image create validates that pool supports overwrites
+
+    Fixes: http://tracker.ceph.com/issues/19081
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a16beba7843539bef4e5035148253a7b594a5cfd)
+
+    Conflicts:
+        src/librbd/image/CreateRequest.cc - kraken uses create_rados_ack_callback;
+            in master, this has been renamed ceph_rados_callback
+
+commit d5aea7a68b8b2603cb9fa57df3a3ae5a44ec450a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 24 12:15:50 2017 -0400
+
+    os/bluestore/KernelDevice: fix uninit value
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 42db0c70bc7ef595f0925657c043ce081799b2b9)
+
+    Conflicts:
+        src/os/bluestore/KernelDevice.h
+          - flush_lock mutex is still used in kraken, so keep it
+
+commit c08720553f6ae787fe3b0edbdd1497859cdfe0d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 9 16:51:21 2017 -0500
+
+    os/bluestore/BlueFS: fix flush_bdev placement
+
+    We need to flush any new writes on any fsync().  Notably, this includes
+    the rocksdb log. However, previously _fsync was only doing a bdev flush if
+    we also had a dirty bluefs journal and called into _sync_and_flush_journal.
+    If we didn't, we weren't doing a flush() at all, which could lead to
+    corrupted data.
+
+    Fix this by moving the first flush_bdev *out* of _sync_and_flush_log.  (The
+    second one is there to flush the bluefs journal; the first one was to
+    ensure prior writes are stable.)  Instead, flush prior writes in all of the
+    callers prior to calling _sync_and_flush_log.  This includes _fsync (and
+    fixes the bug by covering the non-journal-flush path) as well as several
+    other callers.
+
+    Fixes: http://tracker.ceph.com/issues/19250
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2924a96493d8570317e55854a25fc64911ecf151)
+
+commit 594332631e442f3da1b7e6aa49248a6e3c46a30f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 9 16:51:05 2017 -0500
+
+    os/bluestore/KernelDevice: make flush() thread safe
+
+    flush() may be called from multiple racing threads (notably, rocksdb can call fsync via
+    bluefs at any time), and we need to make sure that if one thread sees the io_since_flush
+    command and does an actual flush, that other racing threads also wait until that flush is
+    complete.  This is accomplished with a simple mutex!
+
+    Also, set the flag on IO *completion*, since flush is only a promise about
+    completed IOs, not submitted IOs.
+
+    Document.
+
+    Fixes: http://tracker.ceph.com/issues/19251
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b3c52643c8e5fa820c53d96608862b7649c3fd0)
+
+commit dbe8a8ba848d5567db935d7cc945fdd85d8e9d3f
+Merge: ccde4da15c 5dcf60702c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 13:32:46 2017 +0200
+
+    Merge pull request #14065 from asheplyakov/19212-bp-kraken
+
+    kraken: rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit b941ca7eac3090bf219b6ced4a9239e315f663de
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:36:08 2017 +0000
+
+    qa/cephfs: use getfattr/setfattr helpers
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dd43d3bc646aeab88486b0963fc83de0b18800c4)
+
+    Conflicts:
+        qa/tasks/cephfs/test_data_scan.py - the master commit drops p =
+            self._mount.run_shell(...) assignment which isn't present in kraken
+            (trivial resolution)
+
+commit f8843445aa1b027c7a2ab5729b715d0da616bfb7
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:26:30 2017 +0000
+
+    qa: add test for reading quotas from different clients
+
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61617f8f10a6322603a9add77980865cd972ef97)
+
+commit b7f46973b03bc9e113ccd55ede0f766a1b1eb0d3
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 17:51:44 2017 +0000
+
+    client: _getattr on quota_root before using in statfs
+
+    ...so that after someone adjusts the quota settings
+    on an inode that another client is using as its mount root,
+    the change is visible immediately on the other client.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3d25941aadd223669448d0f5d3c0bd1fefa72308)
+
+commit 5a69e33057982d78157b75bce3fc458e03260d9e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 15:32:47 2017 +0000
+
+    client: getattr before read on ceph.* xattrs
+
+    Previously we were returning values for quota, layout
+    xattrs without any kind of update -- the user just got
+    whatever happened to be in our cache.
+
+    Clearly this extra round trip has a cost, but reads of
+    these xattrs are fairly rare, happening on admin
+    intervention rather than in normal operation.
+
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 532dc4b68e538c189ef828f67cecd0d647a62250)
+
+commit 4241a6eef03e93918d18e4a4c2ac38d4a55b82b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:56:31 2017 +0100
+
+    mds: validate prealloc_inos on sessions after load
+
+    Mitigates http://tracker.ceph.com/issues/16842
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c39aaf90ed1b23343eba2b341bb8ee6a50a4ea74)
+
+commit 41be67e80909962a004c66491815195e9a6fbf39
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:33:59 2017 +0100
+
+    mds: operator<< for Session
+
+    Use this to get a nice human readable name
+    when available (also including the session id in
+    parentheses)
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f89787d8312f132ebb621f16c44e950b17a395a)
+
+commit ccde4da15c9ea50a4663b0697d0cef827c26f9c7
+Merge: a1ed9bac52 52ee6a4c49
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 13:18:48 2017 +0200
+
+    Merge pull request #16092 from smithfarm/wip-20150-kraken
+
+    kraken: ceph-disk: fails if OSD udev rule triggers prior to mount of /var
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0b1dee0edcf3fe38289c10262655875d61a0dbbf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 23 09:07:32 2017 -0400
+
+    mds: include advisory `path` field in damage
+
+    This will just be whatever path we were looking
+    at at the point that damage was notified -- no
+    intention whatsoever of providing any up to date
+    path or resolution when there are multiple paths
+    to an inode.
+
+    Fixes: http://tracker.ceph.com/issues/18509
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c0bff51ef409eb6e4b2fc248e06e5a7e43faf51e)
+
+    Conflicts:
+        src/mds/CDir.cc - trivial resolution (kraken does not have
+            cb86740a5f4aa3eed43c7f09ac5e7e525a5c1d67)
+
+commit f458d60838628c4ed08448998956bc6ce7228f9b
+Author: Henrik Korkuc <henrik@uber.com>
+Date:   Sun Feb 19 11:44:20 2017 +0200
+
+    client/Client.cc: add feature to reconnect client after MDS reset
+
+    Client.cc marks session as stale instead of reconecting after received
+    reset from MDS. On MDS side session is closed so MDS is ignoring cap
+    renew. This adds option to reconnect stale client sessions instead of
+    just marking sessions stale.
+
+    Fixes: http://tracker.ceph.com/issues/18757
+
+    Signed-off-by: Henrik Korkuc <henrik@kirneh.eu>
+    (cherry picked from commit e0bbc704676ef4aed510daff075ef63c9e73b7b3)
+
+commit 7674f84ddf8f8f8f0a3fe8c3b336bacc0718eeed
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 22 17:33:05 2017 +0800
+
+    client: wait for lastest osdmap when handling set file/dir layout
+
+    Fixes: http://tracker.ceph.com/issues/18914
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 76f5eb86cdd61dde4e6c7cfeb5cf34f0c0334f21)
+
+commit bee73d2429628e7d27cf9b1ca67eb5d5f049f285
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Thu Apr 13 20:09:07 2017 +0800
+
+    cephfs: fix write_buf's _len overflow problem
+
+    After I have set about 400 64KB xattr kv pair to a file,
+    mds is crashed. Every time I try to start mds, it will crash again.
+    The root reason is write_buf._len overflowed when doing
+    Journaler::append_entry().
+
+    This patch try to fix this problem through the following changes:
+
+     1. limit file/dir's xattr size
+     2. throttle journal entry append operations
+
+    Fixes: http://tracker.ceph.com/issues/19033
+    Signed-off-by: Yang Honggang joseph.yang@xtaotech.com
+    (cherry picked from commit eb915d0eeccbe523f8f70f6571880003ff459459)
+
+commit 32c7ebe3e6d19d37f198a8ae6bdc46d68fb3ed05
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 8 12:13:46 2017 +0000
+
+    mds: shut down finisher before objecter
+
+    Some of the finisher contexts would try to call into Objecter.
+    We mostly are protected from this by mds_lock+the stopping
+    flag, but at the Filer level there's no mds_lock, so in the
+    case of file size probing we have a problem.
+
+    Fixes: http://tracker.ceph.com/issues/19204
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 177a97d5c55ee6a2d5dcd3cf0893546190b10f7a)
+
+    Conflicts:
+        src/mds/MDSRank.cc - kraken has no 7189b53b410424d4a662486c1081181b2bd662ff
+            so we omit the purge_queue.shutdown() call
+
+commit b267a1a8be0ea4893abb77695f1d1b2a0dac2c90
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 28 14:13:33 2017 -0400
+
+    mds: ignore ENOENT on writing backtrace
+
+    We get ENOENT when a pool doesn't exist.  This can
+    happen because we don't prevent people deleting
+    former cephfs data pools whose files may not have
+    had their metadata flushed yet.
+
+    http://tracker.ceph.com/issues/19401
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3fccc2372f2715d075b05e459140360cf6e7ca96)
+
+commit 93e81a87ac99080a8670b7485dc9c8c801477518
+Author: YunfeiGuan <yunfei.guan@xtaotech.com>
+Date:   Mon Apr 10 05:48:47 2017 +0000
+
+    cephfs: fix mount point break off problem after mds switch occured
+
+    The hot-standby become active as we expected but the mount piont broken strangely
+    when the active mds is down. The root reason is the new mds use last_cap_renews
+    decoded from ESesson::replay in find_idle_sessions and wrongly killed the session.
+    Maybe we should reset session->last_cap_renew to the current time when server send
+    OPEN to client in reconnect stage.
+
+    Fixes: http://tracker.ceph.com/issues/19437
+    Signed-off-by: Guan yunfei <yunfei.guan@xtaotech.com>
+    (cherry picked from commit 4ef830c5d6f22bf0d4f82a8624c772ecbbda44a6)
+
+commit 78ccba49d3a1f53c3fd7ffe0d6b6bb45eb66c801
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Mar 5 21:59:45 2017 -0500
+
+    common: remove \n on clog messages
+
+    Confirmed these aren't needed, and are inconsistently included.
+    Remove them where they do appear.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 693132eb00b1803d5e97a79908521d5a6903e9f8)
+
+    Conflicts:
+            src/mon/OSDMonitor.cc - trivial resolution
+            src/osd/PrimaryLogPG.cc - trivial resolution
+
+commit e72d6362c15668e2340212dcdeaa07fc94d12ba7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 5 21:29:10 2017 +0800
+
+    mds: make C_MDSInternalNoop::complete() delete 'this'
+
+    Fixes: http://tracker.ceph.com/issues/19501
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 424e0c6744e7f63459ca0ff7deab751726aa30cd)
+
+    Conflicts:
+        src/mds/MDSContext.h - omit "override" because kraken does not have
+            1a91aeab987870b3ccbcf2f1e476fac8b534d449
+
+commit a1ed9bac523d3115fe5120b2f74f450a739d89b1
+Merge: 4c8d3ed4dc 8d64dd4f29
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 11:37:29 2017 +0200
+
+    Merge pull request #14604 from asheplyakov/19647-kraken
+
+    kraken: ceph-disk: enable directory backed OSD at boot time
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7054ff69d0450ecd86aff5c9c78a0a93f536d179
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Fri Mar 31 00:23:12 2017 +0800
+
+    rbd-nbd: remove debug message from do_unmap
+
+    Global context is not initialized when do_unmap is called.
+
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit cd748f9b260a3c388b45091ff0bfc041257acd0c)
+
+commit 36e2199f0bfd869054506476f64af7f603ee6541
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Mar 29 19:07:34 2017 +0800
+
+    rbd-nbd: s/cpp_error/cpp_strerror/ to fix FTBFS
+
+    the build failure was introduced by ff4dcf0
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c12ecd984df0f9ac0d7de81b9d689f6182a3646a)
+
+commit 4d310c24af51bb0539426a5a7ca6f2b1ca31a1d0
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Tue Mar 28 16:48:21 2017 +0800
+
+    rbd-nbd: polish the output info before and after ioctl NBD_DISCONNECT.
+
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit ff4dcf029028e8a3636ac71a6c5ac5380bf274e4)
+
+commit e6a01244e6ae2862b1c6e78be67b3c2be6899364
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Tue Mar 28 16:33:25 2017 +0800
+
+    rbd-nbd: support signal handle for SIGHUP, SIGINT, and SIGTERM.
+
+    Fixes: http://tracker.ceph.com/issues/19349
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 3ba01aa6ce052d1afa42132feffc2353d73caae6)
+
+commit 5dab0825d705d84926fd5f2f106fa945492bda34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 12 10:47:28 2017 -0400
+
+    test/librados_test_stub: fixed cls_cxx_map_get_keys/vals return value
+
+    Fixes: http://tracker.ceph.com/issues/19597
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ffd464dac102f684d6dfa78e58d2cb45e165ed6)
+
+commit 8ed81b033a7a9120a24611c9e193d85e109ad2e5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 16:33:58 2017 -0400
+
+    librbd: clean up debug messages within image create state machine
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4e27c48aa3f96f650d6cbfe103be5aa980e0f56)
+
+commit fbd296933bb64239f49909a123fcd654000f910f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 14:12:35 2017 -0400
+
+    librbd: removed legacy state machine callbacks from CreateRequest
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d79bf2bcba1d09d8e5589c5a8ee6dca1d27f800)
+
+    Conflicts:
+        src/librbd/image/CreateRequest.cc - kraken does not have 5f2689b62ae
+
+commit f1cf9465984801b17ce536a6a94b2789b482d1a7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 2 10:29:36 2017 -0500
+
+    librbd: avoid duplicating librados IoCtx objects if not needed
+
+    This introduces the potential for shutdown race conditions within
+    the unit tests.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cc5ac6aa66f4c97cbe1c7d6334b3f710610f6742)
+
+commit 4c8d3ed4dc7fc51d7116bb65915ed5cf8ac468bd
+Merge: 8452c0c134 d186951730
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 10:26:43 2017 +0200
+
+    Merge pull request #14616 from smithfarm/wip-18723-kraken
+
+    kraken: osd: calc_clone_subsets misuses try_read_lock vs missing
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b7503d3a80b14dd99f6e46fe63414815afcff814
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Apr 14 15:56:37 2017 -0400
+
+    rgw_file: fix readdir after dirent-change
+
+    Also, fixes link count computation off-by-one, update of state.nlink
+    after computation, link computation reset at start, and a time print
+    in debug log.
+
+    Fixes: http://tracker.ceph.com/issues/19634
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+    link count
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0f80266ecd424bf9466579b3edc03911a7c5719)
+
+commit ea3aec3334871c5952986fcf14aac61348f1cc00
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 06:42:07 2017 -0400
+
+    rgw_file: don't expire directories being read
+
+    If a readdir expire event turns out to be older than last_readdir,
+    just reschedule it (but actually, we should just discard it, as
+    another expire event must be in queue.
+
+    Fixes: http://tracker.ceph.com/issues/19625
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 007b7451c26716c51207c161dc347e9a00da53f1)
+
+commit 247f897c31f15c48fa0521311b1afc557e5a10ae
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 05:56:13 2017 -0400
+
+    rgw_file:  chunked readdir
+
+    Adjust readdir callback path for new nfs-ganesha chunked readdir,
+    including changes to respect the result of callback to not
+    continue.
+
+    Pending introduction of offset name hint, our caller will just be
+    completely enumerating, so it is possible to remove the offset map
+    and just keep a last offset.
+
+    Fixes: http://tracker.ceph.com/issues/19624
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0191d74e3aef06bf300df045a53a3952a71f651)
+
+commit ac1dd8d323c5d90e20fd88e56031441f704ee95a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 20:24:12 2017 -0500
+
+    rgw_file: RGWFileHandle dtor must also cond-unlink from FHCache
+
+    Formerly masked in part by the reclaim() action, direct-delete now
+    substitutes for reclaim() iff its LRU lane is over its high-water
+    mark, and in particular, like reclaim() the destructor is certain
+    to see handles still interned on the FHcache when nfs-ganesha is
+    recycling objects from its own LRU.
+
+    Fixes: http://tracker.ceph.com/issues/19112
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d51a3b1224ba62bb53c6c2c7751fcf7853c35a4b)
+
+commit 78e251632dd3103c5476544ea7386203bd5ff7a2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Mar 2 17:21:57 2017 +0800
+
+    rgw_file: posix style atime,ctime,mtime
+
+    As an ganesha FSAL backend, rgw_file should properly maintain
+    the atime,ctime,mtime properly against operations such as:
+            (read,write) for file
+            (create,unlink,mkdir,rmdir,rename) for dir
+            (setattr) for file and dir
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ac25da2479b9be876cbdb820560ac46a6e2b17d7)
+
+commit 59a965a8e017da3dc6c79a25d55441b945cd50bc
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 8 16:23:11 2017 +0800
+
+    rgw_file: fix reversed return value of getattr
+
+    When ::getattr returns -ESTALE, rgw_getattr returns ESTALE,
+    which is a not expected postive.
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 39203cf872b8f4af86eb0e4a0f96dffd9cc92b41)
+
+commit e5c78f8bf2733b803f4e8e051884c8130809e506
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 15 15:01:05 2017 +0800
+
+    rgw_file: fix double unref on rgw_fh for rename
+
+    Skip unref after unlink to fix the problem.
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit bff228734c73b536d2482e2e2fa4ad38b206ebff)
+
+commit f97ab40539385494f9d4a66806c2bfc08d3654bb
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 14:57:59 2017 -0500
+
+    rgw_file: rgw_lookup: don't ref for "/" or ".."
+
+    These refs won't be returned by nfs-ganesha, and are sufficiently
+    magical that other consumers should be persuaded to understand
+    their specialness.
+
+    Fixes: http://tracker.ceph.com/issues/19060
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit dea8d1ee373399a21851690a9753388b659b8ede)
+
+commit 6e0b260154328b14c45ecbea00007e537ce220ac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 16:02:07 2017 -0500
+
+    rgw_file: ensure valid_s3_object_name for directories, too
+
+    The logic in RGWLibFS::mkdir() validated bucket names, but not
+    object names (though RGWLibFS::create() did so).
+
+    The negative side effect of this was not creating illegal objects
+    (we won't), but in a) failing with -EIO and b) more importantly,
+    not removing up the proposed object from FHCache, which produced a
+    boost assert when recycled.
+
+    Fixes: http://tracker.ceph.com/issues/19066
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit eb1cd3b30c0504385f05bf2d2dd5e2251b7efed7)
+
+commit 349de8a5b30691b0579708ce95f91ce8b929a393
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 3 13:44:45 2017 -0500
+
+    rgw_file:  fix RGWLibFS::setattr for directory objects
+
+    Fixes:  http://tracker.ceph.com/issues/18808
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4ad5a9226852d6d564baf2e63278ed6c4c185ecb)
+
+commit 2b9a77edf203ef39049677a94e13db0fdf0fed8d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 4 20:16:13 2017 -0400
+
+    rgw_file: introduce rgw_lookup type hints
+
+    The new type hints optimize object type deduction, when the
+    rgw_lookup is called from an rgw_readdir callback.
+
+    Fixes: http://tracker.ceph.com/issues/19623
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2e66c7a7cc763c5c0d6f5db04855f60f2b2ceed3)
+
+commit 64cce346c0fcc1402d4dab548f6f7536430703e7
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 10:21:38 2017 -0500
+
+    rgw_file:  return of RGWFileHandle::FLAG_EXACT_MATCH
+
+    Allow callers of rgw_lookup() on objects attested in an
+    rgw_readdir() callback the ability to bypass exact match in
+    RGWLibFS::stat_leaf() case 2, but restore exact match enforcement
+    for general lookups.
+
+    This preserves required common_prefix namespace behavior, but
+    prevents clients from eerily permitting things like "cd sara0" via
+    partial name match on "sara01."
+
+    Fixes: http://tracker.ceph.com/issues/19059
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70ef7d45e0abf2661bd4e23161d4e70cf5178079)
+
+commit fa208008dad8cb165ee72a8cf96ead36d855ba30
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 15:49:06 2017 -0500
+
+    rgw_file:  use fh_hook::is_linked() to check residence
+
+    Previously we assumed that !deleted handles were resident--there
+    is an observed case where a !deleted handle is !linked.  Since
+    we currently use safe_link mode, an is_linked() check is
+    available, and exhaustive.
+
+    Fixes: http://tracker.ceph.com/issues/19111
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0aa515f8d8c57ec5ee09e3b60df3cac60453c40)
+
+commit 29fe4cfeb71a1b0a74fb7a073c2348cf2d1d143d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 18:20:43 2017 -0500
+
+    rgw_file: refcnt bugfixes
+
+    This change includes 3 related changes:
+
+    1. add required lock flags for FHCache updates--this is a crash
+       bug under concurrent update/lookup
+
+    2. omit to inc/dec refcnt on root filehandles in 2 places--the
+       root handle current is not on the lru list, so it's not
+       valid to do so
+
+    3. based on observation of LRU behavior during creates/deletes,
+       update (cohort) LRU unref to move objects to LRU when their
+       refcount falls to SENTINEL_REFCNT--this cheaply primes the
+       current reclaim() mechanism, so very significanty improves
+       space use (e.g., after deletes) in the absence of scans
+       (which is common due to nfs-ganesha caching)
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit beaeff059375b44188160dbde8a81dd4f4f8c6eb)
+
+commit 99941105a982f77513c8df576176f4a707daf26a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Feb 11 16:38:05 2017 -0500
+
+    rgw_file:  add refcount dout traces at debuglevel 17
+
+    These are helpful for checking RGWFileHandle refcnt invariants.
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 462034e17f919fb783ee33e2c9fa8089f93fd97d)
+
+commit a446c724aa2404f4f4ff8efc2da3e12ca2bf6892
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 17:14:16 2017 -0500
+
+    rgw_file: add pretty-print for RGWFileHandle
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ef330f385d3407af5f470b5093145f59cc4dcc79)
+
+commit 1553877b1673b21694a4a3b3ef9811800e67144e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Dec 30 23:30:16 2016 -0500
+
+    rgw_file:  interned RGWFileHandle objects need parent refs
+
+    RGW NFS fhcache/RGWFileHandle operators assume existence of the
+    full chain of parents from any object to the its fs_root--this is
+    a consequence of the weakly-connected namespace design goal, and
+    not a defect.
+
+    This change ensures the invariant by taking a parent ref when
+    objects are interned (when a parent ref is guaranteed).  Parent
+    refs are returned when objects are destroyed--essentially by the
+    invariant, such a ref must exist.
+
+    The extra ref is omitted when parent->is_root(), as that node is
+    not in the LRU cache.
+
+    Fixes: http://tracker.ceph.com/issues/18650
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e5299f3f43e633a5d8a9360893b4b11f6217d81)
+
+commit 4af9f6fef77597a826cfdf5d0b3a814ce4629bc3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 10:24:29 2017 -0500
+
+    rgw_file: avoid stranding invalid-name bucket handles in fhcache
+
+    To avoid a string copy in the common mkdir path, handles for
+    proposed buckets currently are staged in the handle table, before
+    being rejected.  They need to be destaged, not just marked deleted
+    (because deleted objects are now assumed not to be linked, as of
+    beaeff059375b44188160dbde8a81dd4f4f8c6eb).
+
+    This triggered an unhandled Boost assert when deleting staged
+    handles, as current safe_link mode requires first removing from
+    the FHCache.
+
+    Fixes: http://tracker.ceph.com/issues/19036
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6cde812c92e5bba9f85fbf8486ebe69b55952370)
+
+commit e03a5cba8ff4a2ceebf85a2697e90d525185b2d2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri Mar 31 10:42:40 2017 +0800
+
+    rgw_file: fix missing unlock in unlink
+
+    Fixes: http://tracker.ceph.com/issues/19435
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit cb6808a6366a70f54d0cc16437d16aa1b7819c84)
+
+commit 34553f919ccf0324138b990ff928b32dc0e4baf2
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:35:16 2017 -0400
+
+    rgw_file:  remove unused rgw_key variable
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 1100a1c26e76485569cfebcf863b18cf908f6161)
+
+commit b04d051fd765a88a270041c5238a65481180528c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:40:35 2017 -0400
+
+    rgw_file:  rgw_readdir:  return dot-dirs only when *offset is 0
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 61482c2b85a07519f2256b1a3f2b6d8aa99d5f06)
+
+commit b350b61bd2c75bc176f854bb3a39a453a2faec97
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Mar 13 21:52:08 2017 -0400
+
+    rgw_file: implement reliable has-children check (unlink dir)
+
+    Bug report and discussion provided by
+    Gui Hecheng <guihecheng@cmss.chinamobile.com> in nfs-ganesha upstream
+    github.  Briefly, while a reliable check is potentially costly,
+    it is necessary.
+
+    Fixes: http://tracker.ceph.com/issues/19270
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b05f1c6d61aa4501a971e87de6dcaf3e58c3d9b4)
+
+commit 7faa917cdb1c484018dd47559c5f25b6b33b5529
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+
+    Fixes: http://tracker.ceph.com/issues/19018
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+
+commit 50955a5aef10f18f050735af03734189f519f585
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jan 19 18:14:30 2017 -0500
+
+    rgw_file: add timed namespace invalidation
+
+    With change, librgw/rgw_file consumers can provide an invalidation
+    callback, which is used by the library to invalidate directories
+    whose contents should be forgotten.
+
+    The existing RGWLib GC mechanism is being used to drive this.  New
+    configuration params have been added.  The main configurable is
+    rgw_nfs_namespace_expire_secs, the expire timeout.
+
+    Updated post Yehuda review.
+
+    Fixes: http://tracker.ceph.com/issues/18651
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit deb2c1ea985fcb906e47b93fd3d0117794e2d0a1)
+
+    Conflicts:
+        src/rgw/rgw_lib_frontend.h - in class RGWLibProcess : public RGWProcess
+                               there was no public method stop() in kraken (now there is)
+
+commit c613ee7e5853c40c89215efdc231c9e5274aec58
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+
+    Fixes: http://tracker.ceph.com/issues/18991
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+
+commit 2e27aa9bb2484512620d885c0a19e35b1de7b652
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+
+    Fixes: http://tracker.ceph.com/issues/18989
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+
+commit 70ebe961de881734b637c7d13e7dd63bc09066c6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 17:43:17 2017 -0500
+
+    rgw_file: invalid use of RGWFileHandle::FLAG_EXACT_MATCH
+
+    The change which introduced this flag also caused it to be
+    given as the flags argument to RGWLibFS::stat_leaf() when called
+    from rgw_lookup().
+
+    This was incorrect:  in particular, when a directory is known only
+    as a common prefix of other objects, the AWS namespace mapping
+    convention requires lookup("foo") to match a non-materialized
+    instance of "foo/" (case 2 in RGWLibFS::stat_leaf's stat loop).
+
+    Fixes: http://tracker.ceph.com/issues/18992
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e31e9eb980f958640150e8d7f17de1b9e5478b1e)
+
+commit 52ee6a4c49e5749463811b57f4af4a0824875b70
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 1 11:37:20 2017 +0200
+
+    ceph-disk: do not setup_statedir on trigger
+
+    trigger may run when statedir is unavailable and does not use it.
+
+    Fixes: http://tracker.ceph.com/issues/19941
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 16bfbdd3d9988523bba31aace516c303057daa58)
+
+commit 8452c0c134a1cae30ac9e80eb7e08c6e54b16239
+Merge: e8312132ff 75cdc3f3a1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 00:02:05 2017 +0200
+
+    Merge pull request #14852 from yehudasa/wip-rgw-support-ragweed-kraken
+
+    kraken: rgw: add apis to support ragweed suite
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e8312132ff8caee9b0f83b641bfc0a1875bacf3f
+Merge: 7540396a34 6e5ac14ab6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 22:28:08 2017 +0200
+
+    Merge pull request #13514 from rzarzynski/wip-rgw-controllable-204-kraken
+
+    kraken: rgw: make sending Content-Length in 204 and 304 controllable
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7540396a34147fbb34558356fde18f0a11798d8f
+Merge: f31033cb5b 910b989970
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 17:05:34 2017 +0200
+
+    Merge pull request #14403 from shinobu-x/wip-17331-kraken
+
+    kraken: ceph-disk list reports mount error for OSD having mount options with SELinux context
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 071f111abbd7d6c7a90fc58f52c84907cf4d9860
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 21:06:05 2017 +0100
+
+    osd: Return correct osd_objectstore in OSD metadata
+
+    Do not simply read the configuration value as it might have changed
+    during OSD startup by reading the type from disk.
+
+    Fixes: http://tracker.ceph.com/issues/18638
+
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8fe6a0303b02ac1033f5bfced9f94350fe3e33de)
+
+    Conflicts:
+        src/osd/OSD.cc
+          - g_conf->osd_objectstore was changed to cct->_conf->osd_objectstore by
+            1d5e967a05ddbcceb10efe3b57e242b3b6b7eb8c which is not in kraken
+
+commit ffe5bbfc6e09a47009dd281a38b35a8a3528812e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 26 16:26:42 2017 -0800
+
+    rgw: fix use of marker in List::list_objects()
+
+    Fixes: http://tracker.ceph.com/issues/18331
+
+    List marker is an index key, so treat it as such. This
+    fixes infinite loop in orphans find command.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a5d1fa0587184f43c69d8e03114b58d43f320781)
+
+commit 56ca3b01906ad7ee25b4bc2db0b6df51d2b162cb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Apr 25 12:22:54 2017 +0200
+
+    rgw: fix RadosGW hang during multi-chunk upload of AWSv4.
+
+    Fixes: http://tracker.ceph.com/issues/19754
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 72c1e2e351d984d0425a20f2c772951cbc36f13e)
+
+commit cac26471897b6126a9b95e02f1a93a3a5386dae1
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Mon Mar 27 13:08:12 2017 +0800
+
+    rgw/lifecycle: do not send lifecycle rules when GetLifeCycle failed
+
+    Now, RGW will send two HTTP responses when GetLifeCycle failed. The first one is
+    Error Respnse like 404, and the second is lifecycle rules. It will breaks s3 sdk
+    and s3 utilities.
+
+    Fixes: http://tracker.ceph.com/issues/19363
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit c3c0c828da5a64ca896475c1b0c369fde1bbd76a)
+
+commit e5719234b48a90bbe59a693c96da3634884a0578
+Author: tone-zhang <tone.zhang@linaro.org>
+Date:   Thu Apr 6 17:56:05 2017 +0800
+
+      rgw: fix response header of Swift API
+
+    Response header of Swift API returned by radosgw does not contain
+    "x-openstack-request-id", but Swift returns it. Enhance the
+    compatibility of radosgw.
+
+    Fixes: http://tracker.ceph.com/issues/19443
+
+    Signed-off-by: tone-zhang <tone.zhang@linaro.org>
+    (cherry picked from commit e96db213079ab5e026156ab4b38418d1d4c23d27)
+
+commit 59391c40db969f2a9e97505cfcd2997d02ea86a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 8 14:52:34 2017 -0800
+
+    rgw: fix crash when listing objects via swift
+
+    Fixes: http://tracker.ceph.com/issues/19249
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9ec5e8ce184e19c009863db4d3519f9d8af91bd)
+
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc ("key" element of RGWObjEnt struct
+               is not a reference; fix)
+
+commit e0f90f03ddf2824b448f6affc34ed40d4cead5d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 26 14:02:11 2017 -0500
+
+    librbd: introduce new constants for tracking max block name prefix
+
+    Fixes: http://tracker.ceph.com/issues/18653
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c08629c99d90aa7676b59263c055c9f1f577039)
+
+commit 3173da6e3bc7083ed2bdeea3a25bb3fa790bbc86
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Feb 16 22:17:52 2017 +0800
+
+    rbd-nbd: no need create asok file for unmap and list-mapped commands.
+
+    Fixes: http://tracker.ceph.com/issues/17951
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 72352653d585ef89043a4ece371b5c0cb3f6f32a)
+
+commit 61c5957b30f807f5eab4c072deb00f19433f0a46
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 2 11:11:35 2017 +0100
+
+    rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+
+    Fixes: http://tracker.ceph.com/issues/18335
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 596e5ea8a5df72002672eef0a6d20572ca6f60f0)
+
+commit c79d45ff52a636c136e0c5f8aec7911c04601b6f
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Mon Feb 20 12:04:10 2017 +0530
+
+    librbd: acquire exclusive-lock during copy on read
+
+    Fixes: http://tracker.ceph.com/issues/18888
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 7dba5311b12011a4a6e8564e68150e54c5af5ddd)
+
+    Conflicts:
+        src/librbd/AioImageRequestWQ.h:
+          - in master this file has morphed into src/librbd/io/ImageRequestWQ.h
+          - kraken has AioImageRequest<ImageCtx> instead of ImageRequest<ImageCtx>
+        src/librbd/image/RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because kraken doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+        src/test/librbd/image/test_mock_RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because kraken doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+
+commit d186951730d947881fea0e638a6e112718820ae0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jan 18 10:24:13 2017 -0800
+
+    PrimaryLogPG::try_lock_for_read: give up if missing
+
+    The only users calc_*_subsets might try to read_lock an object which is
+    missing on the primary.  Returning false in those cases is perfectly
+    reasonable and avoids the problem.
+
+    Fixes: http://tracker.ceph.com/issues/18583
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3833440adea6f8bcb0093603c3a9d16360ed57ec)
+
+commit 30ff11c84fd6d53c6439aaed8f7742b98a5b6218
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 23 15:41:13 2016 -0800
+
+    ReplicatedBackend: take read locks for clone sources during recovery
+
+    Otherwise, we run the risk of a clone source which hasn't actually
+    come into existence yet being used if we grab a clone which *just*
+    got added the the ssc, but has not yet actually had time to be
+    created (can't rely on message ordering here since recovery messages
+    don't necessarily order with client IO!).
+
+    Fixes: http://tracker.ceph.com/issues/17831
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8)
+
+commit e76725f4e9fb4a6cd7c76480d0313a8e0264eb73
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 30 13:50:41 2017 -0400
+
+    osd/PrimaryLogPG: do not expect FULL_TRY ops to get resent
+
+    The objecter will not resend FULL_TRY requests that were sent prior to
+    becoming full, so we should not discard them.
+
+    Fixes: http://tracker.ceph.com/issues/19430
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f7acdbc9a942fd18937dbcf07fbc7b752c50ba3)
+
+commit 7a275769b6a281eaec1a659975ec50bfe847e679
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 31 10:06:42 2017 -0400
+
+    ceph_test_librados_api_misc: fix stupid LibRadosMiscConnectFailure.ConnectFailure test
+
+    Sometimes the cond doesn't time out and it wakes up instead.  Just repeat
+    the test many times to ensure that at least once it times out (usually
+    it doesn't; it's pretty infrequent that it doesn't).
+
+    Fixes: http://tracker.ceph.com/issues/15368
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8bc197400d94ee2716d3f2fa454247379a676cf9)
+
+commit 4d1272e5c654c32a844a791b08a97ea8c28c5e5d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 18 14:07:04 2017 +0800
+
+    debian: package ceph.logroate properly
+
+    see also "man dh_installlogrotate"
+
+    Fixes: http://tracker.ceph.com/issues/19390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 17ca501fe8927d541d50a502da53a4d7628d2b4a)
+
+commit 47a45441762e610eef548273c719cf70643bfeac
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 17 14:58:02 2017 -0700
+
+    osd: Give requested scrub work a higher priority
+
+    Once started we now queue scrub work at higher priority than
+    scheduled scrubs.
+
+    Fixes: http://tracker.ceph.com/issues/15789
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ebab8b1f4f67fbdec1e147c580329c1e2b5cf7cd)
+
+    Conflicts:
+            src/osd/OSD.h - in kraken, the PGScrub() call is enclosed within
+                       op_wq.queue(make_pair(...)) instead of enqueue_back()
+
+commit f31033cb5bc7764b44871c15717aed5a34921881
+Merge: b7a2e67460 795538210c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 11:39:42 2017 +0200
+
+    Merge pull request #16069 from smithfarm/wip-20345-kraken
+
+    kraken: make check fails with Error EIO: load dlopen(build/lib/libec_FAKE.so): build/lib/libec_FAKE.so: cannot open shared object file: No such file or directory
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 795538210ce76adc7521fac2caf5de27856aee39
+Author: Kyr Shatskyy <kyrylo.shatskyy@gmail.com>
+Date:   Wed Feb 8 16:02:52 2017 +0100
+
+    ceph.spec.in, debian/control: Add bc to build dependencies
+
+    The bc is missing for ceph-helpers.sh
+
+    Fixes: http://tracker.ceph.com/issues/18876
+    Signed-off-by: Kyr Shatskyy <kyrylo.shatskyy@gmail.com>
+    (cherry picked from commit 3ff4be6d6896346b7b9ec7f158fcde8866faeb38)
+
+commit 977770d828252c25004cb6344c43d85abd8ee6f5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 12 17:31:52 2017 +0800
+
+    qa/workunits/ceph-helpers: do not error out if is_clean
+
+    it would be a race otherwise, because we cannot be sure that the cluster
+    pgs are not all clean or not when run_osd() returns, but we can be sure
+    that they are expected to active+clean after a while. that's what
+    wait_for_clean() does.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6cb4503a40ae4ebee5690fda024cae8d1a506bce)
+
+commit b1d3c1a322dd611b3608732e7ef31cf15e671845
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 12 12:33:53 2017 +0800
+
+    qa/workunits/ceph-helpers: display rejected string
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0196e154ed2e164cf55b0d7ed9f9cdd1f4f50100)
+
+commit 75cdc3f3a16d3799ae417862b9b5ee41cfcaee3e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 9 13:04:43 2017 -0800
+
+    rgw: new rest api to retrieve object layout
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2768583dc486109e49d209243675b99fdd39e92c)
+
+commit df30ac30112a0c15798c868dc1fb6bd58bcc05a2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 5 13:47:24 2017 -0800
+
+    rgw: rest api to read zone config params
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a220a2efbfd675e6abf14ae33c21005bcbf6dadf)
+
+commit 47f751f48ba972f2c93ecad919c66ee03b4aec70
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 4 10:59:07 2017 -0700
+
+    civetweb: move to post 1.8 version
+
+    Fixes: http://tracker.ceph.com/issues/19704
+
+    Version that went into kraken was pre 1.8, and had a few issues.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ff19977fbc4cf4a787af80b6c0457df932004fd3
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jun 27 16:07:01 2017 +0400
+
+    kraken: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+
+    Based on commit bc683385819146f3f6f096ceec97e1226a3cd237. The OSD code has
+    been refactored a lot since Kraken, hence cherry-picking that patch introduces
+    a lot of unrelated changes, and is much more difficult than reusing the idea.
+
+    Fixes: http://tracker.ceph.com/issues/20443
+
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 9efeb6bc2df559f8d91c480c396b4dd97b981d34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:56:22 2017 -0400
+
+    librbd: add no-op event when promoting an image
+
+    The rbd-mirror process needs an event in the journal
+    to properly detect the transition between primary and
+    non-primary state between peers.
+
+    Fixes: http://tracker.ceph.com/issues/19858
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4031555dda7597d24e9eb04b9ff29173909586f7)
+
+    Conflicts:
+            src/librbd/journal/DemoteRequest.cc: logic exists in Journal.cc
+
+commit 1d71475ff561c7f7d4936774d0ff821786b15fe6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:57:34 2017 -0400
+
+    rbd-mirror: prevent infinite loop when computing replay status
+
+    If the image had a non-primary predecessor epoch whose tag tid
+    duplicates an epoch within its own journal, an infinite loop
+    would result.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f179bf86216540d8e25aad469c604f96f6aecd8)
+
+commit b7a2e674601133f481449d3f8cdd3512ccbad51c
+Merge: 2f5c65bfc2 9fd233c2d4
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Jun 26 14:11:56 2017 -0600
+
+    Merge pull request #15869 from smithfarm/wip-swift-task-move-kraken
+
+    tests: move swift.py task from teuthology to ceph, phase one (kraken)
+
+commit 9fd233c2d406a31bfc35993539d60e0b41a53cc0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 12:42:36 2017 +0200
+
+    tests: swift.py: tweak imports
+
+    The ".." form only works within the teuthology repo. With swift.py now in the
+    Ceph repo, we have to be explicit.
+
+    Error message was: "ValueError: Attempted relative import beyond toplevel
+    package
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 5d8ffee4d07d21f12a748af2a86820b1307f2e7c
+Merge: 2f5c65bfc2 7b58ac97e9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:34:53 2017 +0200
+
+    Merge branch 'master' of /home/smithfarm/src/ceph/upstream/teuthology into wip-swift-task-move-kraken
+
+commit 7b58ac97e9dd195f4170e9e0ea00bae76d1f3ccd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:27:42 2017 +0200
+
+    tests: move swift.py task to qa/tasks
+
+    In preparation for moving this task from ceph/teuthology.git into ceph/ceph.git
+
+    The move is necessary because jewel-specific changes are needed, yet teuthology
+    does not maintain a separate branch for jewel. Also, swift.py is a
+    Ceph-specific task so it makes more sense to have it in Ceph.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 73fe367bc4bf27730479bdc46931dd7c3c710c39
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 29 21:55:33 2017 -0400
+
+    os/bluestore: deep decode onode value
+
+    In particular, we want the attrs (map<string,bufferptr>) to be a deep
+    decode so that we do not pin this buffer, and so that any changed attr
+    will free the previous memory.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f53f56350b893bfdd47fe730a4339ca5793791a0)
+
+commit 2f5c65bfc229cb43ac5c193fed7c9f51bd20cf79
+Merge: 4b1e8bcf4c f96262fc9c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:59:27 2017 +0200
+
+    Merge pull request #13181 from smithfarm/wip-18548-kraken
+
+    kraken: rgw: multisite: segfault after changing value of rgw_data_log_num_shards
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4b1e8bcf4c86168ec66808a9a88a6fa556e97445
+Merge: 7706a386e2 291ffe1801
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:56:59 2017 +0200
+
+    Merge pull request #13838 from smithfarm/wip-19049-kraken
+
+    kraken: rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync()
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7706a386e255eec46fafa58ea1435293581218cc
+Merge: 2f4a775d5f d1ec59544b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:52:13 2017 +0200
+
+    Merge pull request #13224 from smithfarm/wip-18780-kraken
+
+    kraken: rgw: Swift API: spurious newline after http body causes weird errors
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d38a7529995cd13b1a3d13a8e00d22ce0172aa5c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 5 12:02:05 2017 +0800
+
+    mon: check is_shutdown() in timer callbacks
+
+    introduce a helper class: C_MonContext, and initialize all timer events
+    using it, to ensure that they do check is_shutdown() before doing their
+    work.
+
+    Fixes: http://tracker.ceph.com/issues/19825
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d)
+
+commit e7361176f4ad1d4b43a393e6e34c50233a73e829
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 4 22:49:04 2017 +0800
+
+    mon/Elector: call cancel_timer() in shutdown()
+
+    instead of doing it manually.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 12139ae529a49b6caedea89f910d034ddca094b6)
+
+commit bf347e92a2d67032f4365a125f3bb9975930e912
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed May 17 17:50:10 2017 +0400
+
+    kraken: mon: add override annotation to callback classes
+
+    The only purpose of this patch is to avoid merge conflicts while
+    cherry-picking commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d.
+    Alternatively one could cherry-pick 1effdfe19bf9fd6d546620b96eaf452e889b15dc,
+    but that one brings a lot of unrelated changes.
+
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 3c4a5ea385c7d2f2dfe88328e2c5f7778928d92d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Apr 20 18:38:43 2017 +0200
+
+    fs: normalize file open flags internally used by cephfs
+
+    The file open flags (O_foo) are platform specific. Normalize these flags
+    before they are send to the MDS. For processing of client messages the
+    MDS should only compare to these normalized flags.
+    Otherwise this can lead to bogus flags getting transmitted on ppc64.
+
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 88d2da5e93198e69435e288ce00d216d5fe27f80)
+
+    Conflicts:
+            src/client/Client.cc - trivial resolution, add
+              ceph_flags_sys2wire when flags are logged
+
+commit b19d6eb3110cdf121c086a754176005a0e62797f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Apr 24 16:23:21 2017 +0200
+
+    test/librbd/test_notify.py: don't disable feature in slave
+
+    On jewel it will have stolen the exclusive lock. Instead, ensure that
+    object map and fast diff are already disabled on the clone before the
+    start of the test.
+
+    Fixes: http://tracker.ceph.com/issues/19716
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e009e1bdd4b3997462feb9a050bd2eb201e028ba)
+
+commit 2f4a775d5ffd170903c4412e4274ab2a188922ef
+Merge: 358081d27a 4776067797
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 08:46:28 2017 +0200
+
+    Merge pull request #13174 from smithfarm/wip-18711-kraken
+
+    kraken: rgw: slave zonegroup cannot enable the bucket versioning
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit f8db63f24dcb0d3aa93eba84e80a3450fb4bd666
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Mar 8 16:31:34 2017 -0500
+
+    rgw: data sync skips slo data when syncing the manifest object
+
+    Fixes: http://tracker.ceph.com/issues/19027
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8b69847d7b3e92c70090d1dddf7cea5c44fb6b20)
+
+    Conflicts:
+            src/rgw/rgw_rest_conn.cc (kraken is missing
+                50c522ea89a756123bf74ab615138cf8478b2cee)
+            src/rgw/rgw_rest_s3.h (omitted override because kraken is missing
+                3b247475a2c35526c129535021adfa621ecb5327)
+
+commit e3d4d5791867009c8c4b4bc867b65a23116653ae
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 13 11:33:02 2017 -0400
+
+    rgw: RGWGetObj applies skip_manifest flag to SLO
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 987377ae34382e107e1d54f0bfc1121fcedb4513)
+
+commit 6c1ba33a7bdffb7fdaf288596acf628ea8ed471c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Feb 21 10:27:13 2017 -0500
+
+    rgw: allow system users to read SLO parts
+
+    multisite data sync relies on fetching the object as the system user
+
+    Fixes: http://tracker.ceph.com/issues/19027
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d50d18c500fd5dd89e7cada1162cf453b36df370)
+
+commit 6860fce4243f0a02bf424a477bf554da608cf90c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 15:05:30 2017 +0200
+
+    rgw: add check for update return value
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 125955e0625461065dc4755b900e51c3598cadb4)
+
+commit b9c7c6503eff88bccea3bb19b4edeba89cb73718
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:42:14 2017 +0200
+
+    rgw: we need to reinit the zonegroup after assignment to avoid invalid cct and store
+
+    Fixes: http://tracker.ceph.com/issues/18631
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac9a7565ddf801121f22476cf3f66668f311833e)
+
+commit db6fb3c005e05f0c6f503a367919e31943044e6b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:40:16 2017 +0200
+
+    rgw: fix init_zg_from_period when default zone is not set as default
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5393077e07bed45b9fc007591d365f1229d3e815)
+
+commit 5f4c38fa6b2162cb97162089a8a22095a7cde99d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 17:05:24 2017 -0800
+
+    rgw: don't update bucket index multiple times in overwrite
+
+    Instead of this for overwrites:
+     prepare (index),
+       write (head) [-EEXIST]
+     cancel (index)
+     read (head)
+     prepare (index)
+       write (head)
+     complete (index)
+
+    We now do:
+     prepare (index),
+       write (head) [-EEXIST]
+       read (head)
+       write (head)
+     complete (index)
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7f4818f9611ea8f7b3fac7df8c5c314964f657a2)
+
+commit 96c8161a786783b84a164afef162dd24c46abdc4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 16:17:49 2017 -0800
+
+    rgw: assume obj write is a first write
+
+    if fails and object already exists then retry. This improves first obj
+    write performance on the expense of overwrites.
+
+    Fixes: http://tracker.ceph.com/issues/18622
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6f27f6089238d2a20e7a0f13066eddfc31192dc8)
+
+commit f2e39fec64df7592bc3b6191f122cdfb5d407d6a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 14:40:58 2017 -0800
+
+    rgw: configurable write obj window size
+
+    Fixes: http://tracker.ceph.com/issues/18623
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66a82b4266acedfdd71c64394d68d9e50ed11b20)
+
+commit 8c67dd10c63009e5bc67c20493de732f7bf4abf7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 12:03:54 2017 -0800
+
+    rgw: change default chunk size to 4MB
+
+    Fixes: http://tracker.ceph.com/issues/18621
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 54ef04bc957a3fb034dce903d62a42d77950dd72)
+
+commit f96262fc9c3805aa5fa04348f520dba8be1e4ae8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 11 09:32:59 2017 -0500
+
+    rgw: fix off-by-one in RGWDataChangesLog::get_info
+
+    Fixes: http://tracker.ceph.com/issues/18488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0974fdcf62e60cf31bc15588e7b718da6f6ade3)
+
+commit 935cc11595f4a0b56313c8c98aae0c7684bfff05
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 3 14:41:50 2017 +0100
+
+    rgw: improve handling of illformed Swift's container ACLs.
+
+    Fixes: http://tracker.ceph.com/issues/18796
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f780fc6ec40395ad0941d4e0309d464fe33836b1)
+
+commit 500eac722d3e59ca8408d1bf9a23cc7368164c58
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Jan 11 05:28:43 2017 +0800
+
+    rgw: the swift container acl should support field .ref
+
+    On the openstack-swift. The container acl supports .ref, which is ignored on ceph swift.
+
+    Fixes: http://tracker.ceph.com/issues/18484
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit b06f9cd9f0900db7b0d0fbcaea69cdd0d4b10132)
+
+commit d1ec59544b3894f8ce7a9d00b5e8fb937fb20b7a
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Jan 11 00:06:15 2017 -0500
+
+    radosgw/swift: clean up flush / newline behavior.
+
+    The current code emits a newline after swift errors, but fails
+    to account for it when it calculates 'content-length'.  This results in
+    some clients (go github.com/ncw/swift) producing complaints about the
+    unsolicited newline such as this,
+            Unsolicited response received on idle HTTP channel starting with "\n"; err=<nil>
+
+    This logic eliminates the newline on flush.  This makes the content length
+    calculation correct and eliminates the stray newline.
+
+    There was already existing separator logic in the rgw plain formatter
+    that can emit a newline at the correct point.  It had been checking
+    "len" to decide if previous data had been emitted, but that's reset to 0
+    by flush().  So, this logic adds a new per-instance variable to separately
+    track state that it emitted a previous item (and should emit a newline).
+
+    Fixes: http://tracker.ceph.com/issues/18473
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f229d6a33eae4906f22cdb90941835e47ee9f02)
+
+commit 1cb98684565427a28b286dcaaa6cce48626ba79b
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 1 19:05:50 2017 +0100
+
+    rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion.
+
+    Fixes: http://tracker.ceph.com/issues/18364
+    Fixes: http://tracker.ceph.com/issues/16355
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 871e1f51afe9d6c8b88debc07460b4316121f999)
+
+commit 87811ff161d3c6273050a6be5d7eb7582ecca23c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:56:44 2017 +0200
+
+    radosgw-admin: check for name mistmatch in realm set
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 4facc5f4f3e842d371115a9a04d86257280014f0)
+
+commit ff888f8f9a4b81347b78ae6681a868cc9a7910e9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:40:37 2017 +0200
+
+    radosgw-admin: relam set can use input redirection
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit b8b3ae3be3f8e4c05cb23062d25c701b15900475)
+
+commit 68d2d4bb91d6c508c0e1c154aff3b5de3dc347eb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:36:04 2017 +0200
+
+    radosgw-admin: realm set should create a new realm
+
+    Fixes: http://tracker.ceph.com/issues/18333
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23339c9ef34f6b9df90b1ab64b550af9b541d9e)
+
+commit 291ffe18018029f1418afb2dc038c605a1e34157
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Nov 29 11:29:41 2016 -0500
+
+    rgw: fix for broken yields in RGWMetaSyncShardCR
+
+    Fixes: http://tracker.ceph.com/issues/18076
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e62d48a9bf2e309eab1a863f167af5267ebcc371)
+
+commit 4776067797b46b4d38d987caa5da4e6954d392aa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Dec 12 14:00:05 2016 +0100
+
+    rgw: complete versioning enablement after sending it to meta master
+
+    Fixes: http://tracker.ceph.com/issues/18003
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 2d8aafb9dbe64bd9dd2b7d5ed50c6e9550cbe1ab)
+
+commit 27e9644ceb469fb2f54cea468d873ad13c1fe6c4
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:19:55 2017 +0800
+
+    msg/async: go to open new session when existing already closed
+
+    Fixes: http://tracker.ceph.com/issues/20230
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 99f580a3959240f99061a9ad48ec591b39a9fd46)
+
+commit 67ee45bbed3975f338b798caf81cd89f80ceb785
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:16:47 2017 +0800
+
+    msg/async: fix accept_conn not remove entry in conns when lazy delete
+
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit bf98babb3289a7714543ff3cbd3872d80f0dc196)
+
+commit 27cbff111458e77baae617c7a222610c8ff0abee
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 11 22:31:43 2017 +0200
+
+    librbd: fix rbd_metadata_list and rbd_metadata_get
+
+    - properly check for val_len in rbd_metadata_list
+    - don't expect output buffers are zero pre-filled
+
+    Fixes: http://tracker.ceph.com/issues/19588
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 75afc74ea681402e22b6dec8b83276d145fc786b)
+
+commit 30e3ae27e0b99ae9b7f3234dbb93260ebda91466
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 26 09:52:25 2017 +0200
+
+    rpm: Move ceph-disk to ceph-base
+
+    The SELinux package now requires the ceph-disk binary but that one was
+    part of the ceph-osd package. The ceph-disk python library is already
+    packaged in ceph-base so moving ceph-disk to ceph-base seems like a
+    reasonable next step.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6991764f3bff7b8f6be699603927aff882377878)
+
+    Conflicts:
+            ceph.spec.in: ceph-disk-udev is present in kraken
+
+commit fa100ed57c3002b357d56398c3589f33cc651fcf
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 25 14:36:13 2017 +0200
+
+    ceph-disk: Fix the file ownership, skip missing
+
+    This commit fixes the file ownership for the /usr/bin/ and /etc/ceph
+    files and skips missing files as some of the files that we do specify
+    now can be missing from the system (not installed, e.f. radosgw).
+
+    Fixes: http://tracker.ceph.com/issues/20077
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 077038b4393a28ccbd38ca4a90105dbd4c1ffcd5)
+
+commit 4e0cfecd593b267811e6bf92e81ba98dbede5317
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 28 12:29:46 2017 +0200
+
+    selinux: Do parallel relabel on package install
+
+    We can take advantage of ceph-disk fix subcommand when doing a package
+    install. We will keep using the differential fixfiles command otherwise.
+
+    We also need to add relabel for /usr/bin/ daemons so that we could use
+    this.
+
+    Fixes: http://tracker.ceph.com/issues/20077
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1cecddf031991f1c64ea203f173189624f11940e)
+
+commit 358081d27a9b2c39e9daed10244819df5291da48
+Merge: ae0eab53f0 bf26c56555
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 2 09:54:45 2017 -0700
+
+    Merge pull request #15421 from dzafman/wip-20125
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bf26c5655505369753142646ac357df2b8897ab7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 31 15:39:19 2017 -0700
+
+    osd: Object level shard errors are tracked and used if no auth available
+
+    Shards with object mismatch are tracked to mark them inconsistent
+    Fix test because storing omap_digest in object_info not behaving as before
+
+    Fixes: http://tracker.ceph.com/issues/20089
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1cacbea763c7aabfeaaf4bd5e878301044184117)
+
+commit 727653623f6cc616bdeeab2f0ce98f41a649ceee
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 5 14:56:40 2017 -0400
+
+    cls/rgw: list_plain_entries() stops before bi_log entries
+
+    list_plain_entries() was using encode_obj_versioned_data_key() to set
+    its end_key, which gives a prefix of BI_BUCKET_OBJ_INSTANCE_INDEX[=2]
+
+    that range between start_key and end_key would not only span the
+    BI_BUCKET_OBJS_INDEX[=0] prefixes, but BI_BUCKET_LOG_INDEX[=1] prefixes
+    as well. this can result in list_plain_entries() trying and failing to
+    decode a rgw_bi_log_entry as a rgw_bucket_dir_entry
+
+    Fixes: http://tracker.ceph.com/issues/19876
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b29a1633a57abf443d5790c13d680d2917f86037)
+
+commit 648c6adf52a7847a5b46d1523da45a29504a68da
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 10:08:33 2017 -0500
+
+    mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+
+    If we only fill OSDs to 95%, we should factor that into
+    the MAX AVAIL calculation for the pool.
+
+    Fixes: http://tracker.ceph.com/issues/18522
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f223ac92917f4bc18e5b9b3ad61afa155e4d088a)
+
+commit 2aec591cfe1ba69631860b4886d34de80f752bd7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 11 13:13:39 2017 +0800
+
+    osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+
+    when a callback is called, it could be facing a PG already shut down by
+    OSD. but if that callback wants to shut that PG down. it should check
+    the PG's status first.
+
+    Fixes: http://tracker.ceph.com/issues/19902
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3c44a0dfc859f6f625a92e727e0e521ed4a9207)
+
+commit ae0eab53f0bb05c954fc98e019e2d5a054dbba45
+Merge: 9df2d772ff 28e1fd0f5d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 11 08:06:47 2017 +0200
+
+    Merge pull request #14345 from ceph/wip-ceph-disk-fix-kraken
+
+    ceph-disk: Add fix subcommand kraken back-port
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 28e1fd0f5d1442d4b56194082014853db551def4
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Mar 8 09:38:39 2017 +0100
+
+    ceph-disk: Add --system option for fix command
+
+    This adds the ability to restore the labels of the underlying system
+    data in addition to ceph data.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 8d81af42fd507c7b92c8279eb114b0a733ac1da6)
+
+commit dfa721f48df5eac565d27388adc17c40991502c6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 16 11:34:27 2017 +0100
+
+    ceph-disk: Add more fix targets
+
+    It turns out I forgot several more directories that needs to be fixed by
+    this script. We need to fix /var/log/ceph, /var/run/ceph and /etc/ceph
+    as well.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ae139307d6b2bfba47e21d29d6dbd3c8dc01b5b5)
+
+commit 2db2fc81835ea8d0b35f563dd09691b030c920e9
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 9 19:17:12 2017 +0100
+
+    ceph-disk: Add unit test for fix command
+
+    This will simulate the command* functions to not actually run anything
+    thus excercising the python code directly. It also checks that the
+    proper (sub-strings) are in the output.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1ec53dee9a690134936bdc3a09c9a02fecf13a9d)
+
+commit 82764ca8e986c8efc31488f2b30d2a2706f9826c
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jan 31 13:19:33 2017 +0100
+
+    ceph-disk: Add fix subcommand
+
+    This subcommand will fix the SELinux labels and/or file permissions on
+    ceph data (/var/lib/ceph).
+
+    The command is also optimized to run the commands in parallel (per
+    sub-dir in /var/lib/ceph) and do restorecon and chown at the same time
+    to take advantage of the caching mechanisms.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6d5d30f6ed7538271579cc2ef4e2e364f01a4a6f)
+
+commit 9df2d772ff272386bff6b1657f3f1528fcaf4a8a
+Merge: 6b1782959e 04f8186f37
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon May 8 11:33:59 2017 -0500
+
+    Merge pull request #14983 from liewegas/wip-denc-dump-kraken
+
+    ceph-object-corpus: kraken objects (kraken)
+
+commit 04f8186f37d5add0e68ff47a025f0296377e96d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 27 16:10:34 2017 -0500
+
+    test/encoding/readable.sh: join children at finish
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c8a349eb4603be295a6f0e1683105cb93bfcd066)
+
+commit ba249885d1a37e703f37a08958af2f85eb227bd6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 27 16:10:24 2017 -0500
+
+    test/encoding/readable: fix passing of results to parent
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1325ba672d5b5c8a12722a395ad029c98d1ff236)
+
+commit d97154e61bb615faf566a42a878b2e7a303d99ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:48:25 2017 -0400
+
+    messages/MCommand: fix type on decode
+
+    Wow, this has been broken since v0.38, but apparently
+    the message never made it into the object corpus so
+    we never noticed!
+
+    In reality the bug is harmless: decode_message() will
+    set_header which clobbers whatever version the default
+    ctor fills in, so this only affects ceph-dencoder's
+    test.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3018b907c1d0375b5ae20802811591c0a46b61be)
+
+commit b05fa6cd2ff07349d6cc76be0732e627f0290308
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:32:48 2017 -0400
+
+    ceph-object-corpus: kraken objects
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4ff0cb953e190806f7b9e6791d300bd22ad281d9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:31:02 2017 -0400
+
+    test/encoding/generate-corpus-objects: fix typo
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b89dff472c86c435fcee767ec9d992b308c4d816)
+
+commit 34e04de108004eccab82d6f791fbffc9ec97ab77
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 09:19:23 2017 -0400
+
+    test/encoding/generate-corpus-objects.sh: simplify object corpus population
+
+    Script that generates corpus objects.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b38b663a71fdc37a2d5c3341836b6716d576cecc)
+
+commit 260352405e82009132755105d78a885ae327a418
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 15:19:05 2017 -0400
+
+    qa/workunits/rgw/run-s3tests.sh: run s3-tests
+
+    This works out of the box with a vstart environment and
+
+     RGW=1 ../src/vstart.sh -n -l
+     PATH=bin:$PATH ../qa/workunits/rgw/run-s3tests.sh
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 40fe1d181cf8d31b80db6f8f99394bd66d2affd5)
+
+commit 2b107f91cefbdc6ab173890c3ccf3693768bc76c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 4 18:33:21 2017 -0400
+
+    include/denc: support ENCODE_DUMP
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 97987b4ace737b86a4dc2dce94ce9d330fbe903c)
+
+commit 62bb2086af2f0d63c081a8a0b9d1de8d1c1179a7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 14 13:21:38 2017 -0400
+
+    osd: fix occasional MOSDMap leak
+
+    _committed_osd_maps() may return early (without putting
+    the ref) on shutdown.
+
+    Fixes: http://tracker.ceph.com/issues/18293
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a74632f964e10a57fb8065aec90dc7b69bddba5b)
+
+commit d51b7553919233d2f18d5fa79a0ff3c09e71ba8a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 1 21:06:19 2017 -0400
+
+    cls_rbd: default initialize snapshot namespace for legacy clients
+
+    Creating a snapshot on >=Kraken OSDs using <=Jewel clients can result
+    in an improperly initialized snapshot namespace. As a result, attempting
+    to remove the snapshot using a >=Kraken client will result in an -EINVAL
+    error.
+
+    Fixes: http://tracker.ceph.com/issues/19413
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03b0b03071f3e04754896664c69f73759ddb907a)
+
+    Conflicts:
+            src/cls/rbd/cls_rbd.h: trivial resolution
+
+commit 6b1782959ee29de6eff9a2d2c81a108485347bbd
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 28 18:03:59 2017 -0400
+
+    mgr/DaemonServer: do not crash on bad authorizer
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 33c211bf9868156e41d7196cbac5dc6bff418248
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 14:32:38 2017 -0700
+
+    PendingReleaseNotes: discuss snap trim improvements
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 911a894d64616b2cb5bfe92e3366e8033fb0f59e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 15:45:12 2017 -0700
+
+    PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine
+
+    Rather than blocking the main op queue, just pause for that amount of
+    time between state machine cycles.
+
+    Also, add osd_snap_trim_sleep to a few of the thrasher yamls.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2ed7759cfeb03e71f0fbd98fe7c2db2bb741861c)
+
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 02a2ef41ae54dac7d903bc5753e6702582d55dd4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:41:21 2017 -0800
+
+    rados: check that pool is done trimming before removing it
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4aebf59d906fa3e03d21bdac182f89fe3cd4c802)
+
+commit 95306559db139ad3cda3ee2eac36195582ff9329
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:05:14 2017 -0800
+
+    osd/: add PG_STATE_SNAPTRIM[_WAIT] to expose snap trim state to user
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c2eac34c86517e410eb4842d8b8085da7d8d7973)
+
+commit 995ef4a6b01635ef09532487f7a4ea5fc90e721d
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 15:41:18 2017 -0700
+
+    osd/PrimaryLogPG: limit the number of concurrently trimming pgs
+
+    This patch introduces an AsyncReserver for snap trimming to limit the
+    number of pgs on any single OSD which can be trimming, as with backfill.
+    Unlike backfill, we don't take remote reservations on the assumption
+    that the set of pgs with trimming work to do is already well
+    distributed, so it doesn't seem worth the implementation overhead to get
+    reservations from the peers as well.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 21cc515adfb225ba70f1d80b1b76f0345c214c22)
+
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+            src/osd/PrimaryLogPG.h
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 49f99c3b4d9051bccb34117075348e3ea58804db
+Merge: 0d6953ec50 e9a10eaccd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:53:49 2017 +0200
+
+    Merge pull request #14506 from smithfarm/wip-19119-kraken
+
+    kraken: doc: PendingReleaseNotes: warning about 'osd rm ...' and #13733
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0d6953ec501f0ec6c9a5a72b532e99554dfe5d09
+Merge: 557fee8d0d 8d0c2297f1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:28:58 2017 +0200
+
+    Merge pull request #14692 from smithfarm/wip-fix-hammer-jewel-x
+
+    qa/suites/upgrade: add tiering test to hammer-jewel-x
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 557fee8d0dad8847728820b1d345e2d9ed9ab433
+Merge: e85b09f415 543e4b9060
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:22:14 2017 +0200
+
+    Merge pull request #14425 from smithfarm/wip-19564-kraken
+
+    kraken: build/ops: ceph-base missing dependency for psmisc in Ubuntu Xenial
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit e85b09f4158296dbc559d3b2e0816364ca1c448c
+Merge: d2c17adc14 8c2f590338
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 22 10:55:16 2017 +0200
+
+    Merge pull request #13494 from shinobu-x/wip-18516-kraken
+
+    kraken: build/ops: systemd: Start OSDs after MONs
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d2c17adc140d2358512c4c00e079299ae7f31f74
+Merge: 9aa180b7f1 02afe9855a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 17:36:42 2017 +0200
+
+    Merge pull request #14588 from asheplyakov/19618-kraken
+
+    kraken: mon/MonClient: make get_mon_log_message() atomic
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9aa180b7f13674a3de28115347534540ecf6ae38
+Merge: 8e2e3d8292 914d8a6109
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 17:12:29 2017 +0200
+
+    Merge pull request #13543 from shinobu-x/wip-18954-kraken
+
+    kraken: ceph-disk prepare get wrong group name in bluestore
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8e2e3d8292f93812933c049f9489871f249c9dfa
+Merge: e68d3e14b8 d8946abf07
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 08:53:25 2017 +0200
+
+    Merge pull request #12746 from SUSE/wip-18387-kraken
+
+    kraken: tests: use ceph-kraken branch for s3tests
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6b428b77f287b3a2b9b7061e3da3279381f7d3aa
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Apr 11 13:51:47 2017 -0700
+
+    use sudo to check check health
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7af157ad4ce7f7e2b8de97ee10eeaf64b9099bc0)
+
+commit 80c47c87e5c060d45ea704ac31709bb061d79db6
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:27:20 2017 -0700
+
+    Add reboot case for systemd test
+
+    test systemd units restart after reboot
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7b587304a54d9b21041ffdfbc85fad8d87859c49)
+
+commit a6c2d7feb3248a0bbe0399323bb2fe11e23d15e1
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:56:11 2017 -0700
+
+    Fix distro's, point to latest version
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1947648669971c1bd1ca189870ed9b25bbd48d3a)
+
+commit 8d0c2297f1dbe95c236357861026263677aea0c5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 15:50:59 2017 +0800
+
+    qa/suites/upgrade: add tiering test to hammer-jewel-x
+
+    Related: http://tracker.ceph.com/issues/19185
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e8c04a027b5d108b30e63ab759760add57d43198)
+
+commit e68d3e14b8ea42baa30ca4dd836c1651deb99dcd
+Merge: 766a55ac87 1067764418
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 21:12:42 2017 +0200
+
+    Merge pull request #14620 from smithfarm/wip-19659-kraken
+
+    kraken: librbd: corrected resize RPC message backwards compatibility
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 766a55ac87cb7c08f1316cf7f0fd6bca834ad269
+Merge: 01804c78e0 55a15b18e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:06:35 2017 +0200
+
+    Merge pull request #14574 from smithfarm/wip-19620-kraken
+
+    kraken: cephfs: MDS server crashes due to inconsistent metadata.
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 01804c78e0af6fd518e8ad5de2bf39e6743e3039
+Merge: 1866332928 93efc1e948
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:05:44 2017 +0200
+
+    Merge pull request #14573 from smithfarm/wip-19483-kraken
+
+    kraken: cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 18663329283c9df32bd42e1dc41cff10ba2e7826
+Merge: 9ff3560ab4 941a356fa7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:04:34 2017 +0200
+
+    Merge pull request #14572 from smithfarm/wip-19335-kraken
+
+    kraken: cephfs: MDS heartbeat timeout during rejoin, when working with large amount of caps/inodes
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9ff3560ab4c4dfa35983beb845331f8fb261270b
+Merge: 5554f5a82a 473ad4ea2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:03:42 2017 +0200
+
+    Merge pull request #14571 from smithfarm/wip-19045-kraken
+
+    kraken: cephfs: buffer overflow in test LibCephFS.DirLs
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 5554f5a82ad2c78b93651aeef5f37e4ffb4586f1
+Merge: cc49802d76 64616bb2a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:02:54 2017 +0200
+
+    Merge pull request #14570 from smithfarm/wip-18950-kraken
+
+    kraken: cephfs: mds/StrayManager: avoid reusing deleted inode in StrayManager::_purge_stray_logged
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit cc49802d769e3a59664fc5ef80d20171c07693df
+Merge: 0c51569478 2d5d8f2767
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:02:02 2017 +0200
+
+    Merge pull request #14569 from smithfarm/wip-18899-kraken
+
+    kraken: cephfs: test_open_inode fails
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0c51569478ebb75defb6ba059f60007db597016a
+Merge: 63fd2b9884 a7831ccf79
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:00:51 2017 +0200
+
+    Merge pull request #14568 from smithfarm/wip-18706-kraken
+
+    kraken: cephfs: fragment space check can cause replayed request fail
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 63fd2b9884eeb7551e08875cb6421a79d22e9674
+Merge: f1affa93d1 eb12e3a752
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:59:25 2017 +0200
+
+    Merge pull request #14567 from smithfarm/wip-18700-kraken
+
+    kraken: cephfs: client: fix the cross-quota rename boundary check conditions
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit f1affa93d15f5f37910debe73eee2b4c8774fe2c
+Merge: 2887fb06aa 95335098d5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:58:08 2017 +0200
+
+    Merge pull request #14566 from smithfarm/wip-18616-kraken
+
+    kraken: cephfs: segfault in handle_client_caps
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2887fb06aa25aeb1b7a845b288ba4a556cab74bc
+Merge: 655ab2a76f 91bec618f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:57:08 2017 +0200
+
+    Merge pull request #14565 from smithfarm/wip-18566-kraken
+
+    kraken: cephfs: MDS crashes on missing metadata object
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 655ab2a76f29c3a06e7d7e9bacd3b51b2725952e
+Merge: bb08c04f26 a51e9b5af9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:56:04 2017 +0200
+
+    Merge pull request #14564 from smithfarm/wip-18562-kraken
+
+    kraken: cephfs: test_client_recovery.TestClientRecovery fails
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit bb08c04f26f1d05eedbbdc7cd23f51314edc13cf
+Merge: e3db135268 ba7b6ced0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:55:01 2017 +0200
+
+    Merge pull request #14563 from smithfarm/wip-18552-kraken
+
+    kraken: cephfs: ceph-fuse crash during snapshot tests
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e3db1352688fc68063707882cf0c26f34db4fabf
+Merge: 7fda19ee7e 3bbd3c5dc3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:18:59 2017 +0200
+
+    Merge pull request #14521 from smithfarm/wip-19462-kraken
+
+    kraken: doc: rgw: admin ops: fix the quota section
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7fda19ee7e8a80f0b34e9d539746c973c8c45707
+Merge: d1f066b7b5 d5398effcf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:52:45 2017 +0200
+
+    Merge pull request #14641 from smithfarm/wip-19693-kraken
+
+    kraken: [test] test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d1f066b7b58343e48ddaff398be7c799323fac4d
+Merge: d82f0cd250 a0ad2d11ce
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:51:45 2017 +0200
+
+    Merge pull request #14533 from smithfarm/wip-18501-kraken
+
+    kraken: rbd-mirror: potential race mirroring cloned image
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d82f0cd2505dd06507e1a08c30fa20fdbb024847
+Merge: 7638d3d0cd d826600ee2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:50:56 2017 +0200
+
+    Merge pull request #14534 from smithfarm/wip-18549-kraken
+
+    kraken: rbd: 'metadata_set' API operation should not change global config setting
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7638d3d0cd0b50bc7ae9cb537c7465152cf9e476
+Merge: 2c1d5db604 b9258ccfd8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:50:11 2017 +0200
+
+    Merge pull request #14536 from smithfarm/wip-18557-kraken
+
+    kraken: rbd: 'rbd bench-write' will crash if --io-size is 4G
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2c1d5db6048ed6dd239af0bdbfa79cbc14432fcc
+Merge: 8794053597 e3a230cb19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:49:24 2017 +0200
+
+    Merge pull request #14537 from smithfarm/wip-18601-kraken
+
+    kraken: rbd: Add missing parameter feedback to 'rbd snap limit'
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 87940535979da874b9b44e037b27d869972efae2
+Merge: b457cbd768 bafa2b0c83
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:23:08 2017 +0200
+
+    Merge pull request #14538 from smithfarm/wip-18632-kraken
+
+    kraken: rbd: [qa] crash in journal-enabled fsx run
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b457cbd768b12701532bfa57304e5c25e6b49528
+Merge: 24e6d0e440 3112ab5726
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:19:59 2017 +0200
+
+    Merge pull request #14622 from smithfarm/wip-19037-kraken
+
+    kraken: rbd-mirror: deleting a snapshot during sync can result in read errors
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 24e6d0e44080558f956d2c8446cf60f7137649bc
+Merge: 13de280cfe 551ce273a1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:18:48 2017 +0200
+
+    Merge pull request #14545 from smithfarm/wip-19324-kraken
+
+    kraken: rbd: [api] temporarily restrict (rbd_)mirror_peer_add from adding multiple peers
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fa2753245d056f39f0f6996988e193bf4917bc52
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 14:33:20 2017 -0700
+
+    osd: pglog: with config, don't assert in the presence of stale divergent_priors
+
+    Fixes: http://tracker.ceph.com/issues/17916
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d5398effcfacb6d5436ae32e1ee40872efd5b504
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:26:31 2017 -0400
+
+    test: rbd master/slave notify test should test active features
+
+    Fixes: http://tracker.ceph.com/issues/19692
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0dcba41cba96566d0b8da54cf0316d523b88ded2)
+
+commit d02d0e5d97fd7d9b3eac1e2ba55776d2c9f7f8ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:12:04 2017 -0400
+
+    qa/suites: client-upgrade/jewel-client-x should test multiple features
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7ab93b7779b293ceda325a6d1cdb446cf624a6e3)
+
+commit 13de280cfe7379440185fefcb71a6d7d4a30e65f
+Merge: 5d604d058b b8d988f7f1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Apr 18 14:17:02 2017 -0600
+
+    Merge pull request #14487 from zmc/wip-kraken-workunit
+
+    qa/tasks/workunit: Backport repo fixes from master
+
+commit 5d604d058b0b1f7b7659390093437f109fcb9af7
+Merge: 959248d249 c2232643d0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 22:09:19 2017 +0200
+
+    Merge pull request #14531 from smithfarm/wip-18493-kraken
+
+    kraken: rbd-mirror: sporadic image replayer shut down failure
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 959248d2497a48f98c0a4fd818fcce79c994ca61
+Merge: c195418b4f e91de89b0d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 22:08:34 2017 +0200
+
+    Merge pull request #14532 from smithfarm/wip-18495-kraken
+
+    kraken: rbd: Possible deadlock performing a synchronous API action while refresh in-progress
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 10677644181356f561cc10a19d76833d55917a43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 18 10:05:53 2017 -0400
+
+    librbd: corrected resize RPC message backwards compatibility
+
+    Commit d1f2c557 incorrectly changed the order of variables within
+    the payload. This resulted in breaking the resize RPC message
+    with older versions of Ceph.
+
+    Fixes: http://tracker.ceph.com/issues/19636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9db305a99fd945059a173f5ae8ae61744dd28615)
+
+commit c195418b4f240a8f4bf1b1739af9a64ba046c2d2
+Merge: 432bc8b416 81f3d86032
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 16:58:39 2017 +0200
+
+    Merge pull request #14095 from shinobu-x/wip-19319-kraken
+
+    kraken: RadosImport::import should return an error if Rados::connect fails
+
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 8f81bb33c7791c46db5e4db1ee45ab02a70580d9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 16:19:57 2017 -0400
+
+    rgw: fix for null version_id in fetch_remote_obj()
+
+    commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f fixed the wrong use of
+    the dest_obj's version, but removed the check for "null" version
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 915370776df5b964c2ee8d9f9329562919eef8d5)
+
+commit ad2b140e01462d3dfe58fd03879f3868e561f165
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Fri Mar 17 16:26:56 2017 +0800
+
+    rgw: version id doesn't work in fetch_remote_obj
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f)
+
+    Conflicts:
+            src/rgw/rgw_rados.cc: trivial: dest_obj.key.instance in master
+              versus dest_obj.get_instance() in Jewel
+
+commit 8d64dd4f29e1b0af51e7938d1aa363e4ff710154
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Apr 13 23:49:50 2017 +0200
+
+    ceph-disk: enable directory backed OSD at boot time
+
+    https://github.com/ceph/ceph/commit/539385b143feee3905dceaf7a8faaced42f2d3c6
+    introduced a regression preventing directory backed OSD from starting at
+    boot time.
+
+    For device backed OSD the boot sequence starts with ceph-disk@.service
+    and proceeds to
+
+        systemctl enable --runtime ceph-osd@.service
+
+    where the --runtime ensure ceph-osd@12 is removed when the machine
+    reboots so that it does not compete with the ceph-disk@/dev/sdb1 unit at
+    boot time.
+
+    However directory backed OSD solely rely on the ceph-osd@.service unit
+    to start at boot time and will therefore fail to boot.
+
+    The --runtime flag is selectively set for device backed OSD only.
+
+    Fixes: http://tracker.ceph.com/issues/19628
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f425a127b7487d2093c8c943f0bcdec3d673d601)
+
+commit 02afe9855a4d04fe56951a9e9ffd3e32537ff77e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Apr 10 14:53:46 2017 +0800
+
+    mon/MonClient: make get_mon_log_message() atomic
+
+    * LogClient: move reset_session() into get_mon_log_message() and add a
+      "flush" param to the latter. so it can get_mon_log_message()
+      atomically. otherwise another call changing the log queue could sneak
+      in between reset_session() and get_mon_log_message().
+    * MonClient: add a "flush" param to do_send() so we can reset the
+      LogClient session once we are connected to a monitor.
+
+    Fixes: http://tracker.ceph.com/issues/19427
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5215e291da2b527d85e129eda86043490843178e)
+
+    Conflicts:
+            src/mon/MonClient.cc: handle_auth: replaced 'log_client->reset_session();
+            send_log();' sequence with newly introduced 'send_log(true);' like
+            the original patch does
+
+commit 432bc8b4168e70a68e53795dea538882f83a9a8e
+Merge: 8eb10440ce c2cc96e6c1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Apr 16 04:13:12 2017 +0200
+
+    Merge pull request #14323 from shinobu-x/wip-19371-kraken
+
+    kraken: common: monitor creation with IPv6 public network segfaults
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8eb10440ce144ddab2997ac0b8a868d28de932fe
+Merge: 74907feb3f 1d25327b00
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 10:08:15 2017 +0200
+
+    Merge pull request #13284 from shinobu-x/wip-18599-kraken
+
+    kraken: osd: os/bluestore: fix statfs to not include DB partition in free space
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 74907feb3fda64f9e9a1f7410ce71683a40f417c
+Merge: fc5c4cd2c0 0e0d149895
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 09:57:21 2017 +0200
+
+    Merge pull request #13046 from SUSE/wip-18554-kraken
+
+    kraken: mon: peon wrongly delete routed pg stats op before receive pg stats ack
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit fc5c4cd2c01fa97f65b4bbad0d2cfe6dc7e3f9a9
+Merge: 19b8a9f1f8 7e30b630e2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:30:39 2017 +0200
+
+    Merge pull request #14480 from dillaman/wip-19467-kraken
+
+    kraken: librbd: is_exclusive_lock_owner API should ping OSD
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 19b8a9f1f87b2b9ff69eb67b38e1f2e161ea1c1d
+Merge: 91fbfaf3bd d4a43e1fd5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:28:19 2017 +0200
+
+    Merge pull request #13877 from smithfarm/wip-19178-kraken
+
+    kraken: rgw: anonymous user error code of getting object is not consistent with SWIFT
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 91fbfaf3bdbb9dfcf6704effc271d5038263676d
+Merge: abcf1b1e28 b39bef0961
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:25:51 2017 +0200
+
+    Merge pull request #13866 from smithfarm/wip-19157-kraken
+
+    kraken: rgw: health check errors out incorrectly
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit abcf1b1e28c75feb702b100bbc0ca0ff9c4d14f6
+Merge: cbcb932d3d 47cf8f6699
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:22:22 2017 +0200
+
+    Merge pull request #13843 from smithfarm/wip-19146-kraken
+
+    kraken: rgw: a few cases where rgw_obj is incorrectly initialized
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cbcb932d3d40118101df350694eacbe2cfb22232
+Merge: 6702056568 03b1ab4b13
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:20:29 2017 +0200
+
+    Merge pull request #13829 from smithfarm/wip-18898-kraken
+
+    kraken: no http referer info in container metadata dump in swift API
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 670205656847742f1e9d46b152c66f725dfba5dc
+Merge: 841216b781 c8d70ee012
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:19:00 2017 +0200
+
+    Merge pull request #13780 from smithfarm/wip-18896-kraken
+
+    kraken: should parse the url to http host to compare with the container referer acl
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 841216b781e229205476f633eae75355f87c27ae
+Merge: eb84959bac bfc058d8f9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:15:15 2017 +0200
+
+    Merge pull request #12986 from Werkov/rgw-fix-tempurl-url-encoding
+
+    kraken: rgw: Use decoded URI when verifying TempURL
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 55a15b18e7a2188f5ff74b5fa26ef0316ec9c005
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 29 19:38:37 2017 +0100
+
+    tools/cephfs: set dir_layout when injecting inodes
+
+    When we left this as zero, the MDS would interpret it was HASH_LINUX
+    rather than the default HASH_RJENKINS.  Potentially that
+    could cause problems if there perhaps were already dirfrags in
+    the metadata pool that were set up using rjenkins.  Mainly
+    it just seems more appropriate to explicitly set this field
+    rather than hit the fallback behaviour.
+
+    Related: http://tracker.ceph.com/issues/19406
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d6d542885bd29b71214f9ca52bd26e9183c5d01)
+
+commit 93efc1e9484ea41f3eb831dcb3d19d63b044ef70
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 9 13:15:46 2017 +0000
+
+    mon: fix hiding mdsmonitor informative strings
+
+    Local `stringstream ss` declarations were hiding
+    the real variable used to feed back to the user.
+
+    Fixes: http://tracker.ceph.com/issues/16709
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 00404ae9bd4cce0518a44d36d2d6a5612f4f9d04)
+
+commit 941a356fa77ffc2f41d5054a8e3f43b46523008e
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:51:31 2017 +0000
+
+    mds: reset heartbeat in export_remaining_imported_caps
+
+    This loop can be very long.
+
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85071f1509beba4a390730e6a3a4332484646d63)
+
+commit 5e19692bb68049ea8fed0c39534fe8abb10a5378
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:24:50 2017 +0000
+
+    mds: heartbeat_reset in dispatch
+
+    Previously we only heartbeated in tick.  However, our locking is
+    not guaranteed to be fair, so on a super-busy dispatch queue it may be
+    possible for the heartbeat to time out while the tick() function
+    is waiting for mds_lock.
+
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 819394549af10532419d88742fae3a69d2ea487d)
+
+commit 473ad4ea2a35c74d89605b5579a4ff3866134c2d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 15 11:45:26 2017 +0800
+
+    test/libcephfs: avoid buffer overflow when testing ceph_getdents()
+
+    The buffer size should be at least "2 * sizeof(struct dirent)".
+    Otherwise, the code that checks dentry '..' overflow.
+
+    Fixes: http://tracker.ceph.com/issues/18941
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fa6671345b8f3a82dcd232f99e55a982b0a641f1)
+
+commit 64616bb2a5b84f8518ecbb9878a5ba3abaaf87ed
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Fri Feb 10 10:56:46 2017 +0800
+
+    mds/StrayManager: aviod reusing deleted inode in StrayManager::_purge_stray_logged
+
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 4978e57419482384279d7e784a625f5e5c10961a)
+
+commit 2d5d8f27679d40ff962c8329026b2d75a3fa7c78
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Feb 1 00:38:08 2017 +0000
+
+    tasks/cephfs: switch open vs. write in test_open_inode
+
+    Do the write after opening the file, so that we get good
+    behaviour wrt the change in Mount.open_background that uses
+    file existence to confirm that the open happened.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a027dba78fc8bc84ae39d7998b386ce21c01e1bf)
+
+commit 284cd5065f871fc7aeb134d0f9dd48ba9d1f12bf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 26 16:48:58 2017 +0000
+
+    qa: fix race in Mount.open_background
+
+    Previously a later remote call could end up executing
+    before the remote python program in open_background
+    had actually got as far as opening the file.
+
+    Fixes: http://tracker.ceph.com/issues/18661
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c6d91dd91252e703d08b8ac62ac6a47ee82c0bed)
+
+commit a7831ccf795b31705a66be89beef63b0bbe817d3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 15:28:23 2017 +0800
+
+    mds: don't purge strays when mds is in clientreplay state
+
+    MDS does not trim log when it's in clientreplay state. If mds hang
+    at clientreplay state (due to bug), purging strays can submit lots
+    of log events and create very large mds log.
+
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 86bbc7fff02668077f27d0924ba3efe6544b77f6)
+
+commit 224745491e5c9e495adce5abbf7731f7fae1ddc5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 11:03:45 2017 +0800
+
+    mds: skip fragment space check for replayed request
+
+    when handling replayed request, stray directory can be different
+    from the stray directory used by the original request. The fragment
+    space check for stray directory can fail.
+
+    Fixes: http://tracker.ceph.com/issues/18660
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit afe889cbc5baab196567c2aad01f49fe90901fda)
+
+commit eb12e3a7524fcbc009cabda333a6a958390743bd
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Dec 14 12:09:44 2016 -0800
+
+    client: fix the cross-quota rename boundary check conditions
+
+    We were previously rejecting a rename if either of the involved directories
+    was a quota root, even if the other directory was part of the same quota
+    "tree". What we really want to do is identify the correct quota root
+    (whether local or ancestral) for each directory and compare them. So
+    now we do.
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8e8892aa46accb519faa4bb9fecf66618f1b11b2)
+
+commit 95335098d5fa1a8e308f697a6bcef6ce3e090ac9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jan 6 15:42:52 2017 +0800
+
+    mds: fix null pointer dereference in Locker::handle_client_caps
+
+    Locker::handle_client_caps delays processing cap message if the
+    corresponding inode is freezing or frozen. When the message gets
+    processed, client can have already closed the session.
+
+    Fixes: http://tracker.ceph.com/issues/18306
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e281a0b9c1fdeaf09f1b01f34cecd62e4f49d02e)
+
+commit 91bec618f87d4fc5dc0619d1b980c14ec7527d48
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 11 15:50:52 2017 +0800
+
+    qa/tasks: add test_open_ino_errors
+
+    Validate that errors encountered during opening inos are properly
+    propagated
+
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6526ecc084733b34129aa1f21085fa41fb53b785)
+
+commit 4d5775092d29c2eb3c2d88d1fff93ce5af4779dd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 3 11:11:12 2017 +0800
+
+    mds: propagate error encountered during opening inode by number
+
+    Fixes: http://tracker.ceph.com/issues/18179
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2213cc2dcc0e8fb01bcae3863d0d8a4a1fd8873f)
+
+commit a51e9b5af9fbdb5b81ec20e30dfda2222d6ac418
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jan 9 20:47:37 2017 +0800
+
+    qa/tasks/cephfs: fix kernel force umount
+
+    Fixes: http://tracker.ceph.com/issues/18396
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4cdeeaac105d7824452a8b578632b7371275b45c)
+
+commit ba7b6ced0a69991bf4f095172575dc536736d40d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 17:16:40 2017 +0800
+
+    client: fix Client::handle_cap_flushsnap_ack() crash
+
+    Struct CapSnap holds a reference to its parent inode. So erasing
+    struct CapSnap from Inode::cap_snaps may drop inode's last reference.
+    The inode gets freed in the middle of erasing struct CapSnap
+
+    Fixes: http://tracker.ceph.com/issues/18460
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 525c52fd491ed1ced385c8047872e3f557f8423f)
+
+commit 551ce273a132d4e7b59b818377568d40cf597f85
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 10 10:56:38 2017 -0500
+
+    rbd: prevent adding multiple mirror peers to a single pool
+
+    The rbd-mirror daemon does not currently support replication
+    from multiple peers. Until that is supported, add a temporary
+    restriction to prevent confusion.
+
+    Fixes: http://tracker.ceph.com/issues/19256
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c0c9d1014d57b3d5b95e7513fcc38d04b9ea5165)
+
+commit 3112ab57269847233c1e006ca05ba02071d579b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 15:33:01 2017 -0500
+
+    rbd-mirror: retry object copy after -ENOENT error
+
+    Fixes: http://tracker.ceph.com/issues/18990
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b4f36d5dc3f4f3cbb23f61cbb945b222248a50df)
+
+commit 46ba56025c283a70b0f384ef38f47542f0885c5b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 13:09:39 2017 -0500
+
+    rbd-mirror: object copy should always reference valid snapshots
+
+    If a remote snapshot is deleted while an image sync is in-progress,
+    associate the read request against the most recent, valid remote
+    snapshot for a given snapshot object clone.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a91efc3047963364944f8be91cee8e8f6afc49a)
+
+commit d1ac70a4d06ed07a66e7886c00c712a26fd72e9d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 11:52:00 2017 -0500
+
+    rbd-mirror: replace complex object op tuple with struct
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0c181527c0e151784a0f7c466aaa70b0772f91b1)
+
+commit bafa2b0c83d32b97a6dd85e681f2344384c539ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 20 14:26:43 2017 -0500
+
+    journal: don't hold future lock during assignment
+
+    It's possible that the future raced with its owner and reaches
+    an empty reference count. This was resulting in the future being
+    destructed while its lock was still held.
+
+    Fixes: http://tracker.ceph.com/issues/18618
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f21ceef8336e35ca16148a9d58f511037911418)
+
+commit e3a230cb19d62bdcd18f3685d384d76e83b42ec5
+Author: tang.jin <tang.jin@istuary.com>
+Date:   Mon Jan 16 22:28:23 2017 +0800
+
+    rbd: add error prompt when input command 'snap set limit' is incomplete
+
+    Signed-off-by: Tang Jin <tang.jin@istuary.com>
+    (cherry picked from commit 8860028f508a9be5a08f512022cfb042021fd19f)
+
+commit b9258ccfd836d49d64c8de3cb614a4f0ce9806aa
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Tue Jan 10 15:25:13 2017 +0100
+
+    rbd: bench-write should return error if io-size >= 4G
+
+    Currently if user perform bench-write with io-size > 4G
+    then its crashing because currently during memory allocation
+    bufferptr taking size of buffer as a unsigned and io-size > 4G
+    will overflow with unsigned. so during memset operation it will
+    try to set io_size size of memory area pointed by bufferptr,
+    (bufferptr area is:  (4G - io_size)), so it will cause
+    segmentation fault.
+
+    Fix is to return error if io-size >= 4G
+
+    Fixes: http://tracker.ceph.com/issues/18422
+
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 6ab73e5f420e89e19b52e39dab28fa4c94e00197)
+
+commit d826600ee2c1b50675a0c8642c70ad803bca02a9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 18:31:21 2017 +0100
+
+    librbd: metadata_set API operation should not change global config setting
+
+    Fixes: http://tracker.ceph.com/issues/18465
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27465b5916b55ac3c2846c74b89f4362ad17ff1e)
+
+commit a0ad2d11cea867eb19d40a121907ec43f37e39a3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 6 15:51:51 2016 -0500
+
+    librbd: delay mirror registration when creating clones
+
+    Fixes: http://tracker.ceph.com/issues/17993
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 778e1126a0343e2221446b8e13b48df5ccac263c)
+
+commit e91de89b0d62ee4869992254ff291c6a23bcafcc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 12:12:57 2017 -0500
+
+    librbd: possible deadlock with flush if refresh in-progress
+
+    Fixes: http://tracker.ceph.com/issues/18419
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b95f92a5572d3035c20eba07e76d2c825a9853f7)
+
+commit c2232643d0ee38c8148f13e69065a90c003f46e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 15:59:22 2017 -0500
+
+    rbd-mirror: avoid processing new events after stop requested
+
+    Fixes: http://tracker.ceph.com/issues/18441
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d30873b269441815b5fc7de14c7d9a1077a17d8d)
+
+commit faf80bc1dd332f0ccaaa1caf2679c4b2be02288d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Apr 5 13:31:08 2017 +0300
+
+    radosgw-admin: use zone id when creating a zone
+
+    Fixes: http://tracker.ceph.com/issues/19498
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3fea36d635fcba8ca584a1c0ec9f07840009402c)
+
+commit a5fe8124831daaaa67eb89555058d87600d3fff0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 15:43:13 2017 -0400
+
+    qa: rgw task uses period instead of region-map
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e3e3a71d1f1fb43bb4172ce2dfac9a28ca89df0f)
+
+commit 0e964e477ac1e1a2f6f8c28b8fd0da8e9225360c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 14:18:15 2017 -0400
+
+    rgw-admin: remove deprecated regionmap commands
+
+    Fixes: http://tracker.ceph.com/issues/18725
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5830c1849a0c0110d17c37784808e456e6dcb7b3)
+
+commit 03e5be4498d31c738f08d534bf8f27381fb8c12b
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+
+    Fixes: http://tracker.ceph.com/issues/18828
+
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 595c31f390fda700e6e346c208849d23c74bdf5f
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Feb 24 04:45:04 2017 +0800
+
+    rgw: add the remove-x-delete feature to cancel swift object expiration
+
+    In openstack swift, it also support the feature to cancel the object expiration,
+    which could be found at last point in
+    https://docs.openstack.org/user-guide/cli-swift-set-object-expiration.html. we
+    can remove the object expiration by set 'X-Remove-Delete-At:'.
+
+    This patch also could fix the bug that when we set the object expiration and
+    then upload the same object to the container again. The previous object expiration
+    also works, which is not compatible with the openstack swift.
+
+    Fixes: http://tracker.ceph.com/issues/19074
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 230429ebc4ac9b5840bb93c7e0eeb5edbb949106)
+
+commit 3bbd3c5dc316ebd0fffc1a4cf22f44bd01330f6e
+Author: hrchu <petertc@gmail.com>
+Date:   Wed Mar 29 02:17:04 2017 +0000
+
+    doc: rgw: correct the quota section
+
+    Add the missing option and fix typo.
+
+    Fixes: http://tracker.ceph.com/issues/19397
+
+    Signed-off-by: Chu, Hua-Rong <hrchu@cht.com.tw>
+    (cherry picked from commit 51a88267f0d7f51aeb62092949b66b9f6c062e15)
+
+commit c3d8444890c394077177c555908ed78dc824a587
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Feb 8 15:07:43 2017 +0800
+
+    rgw: fix swift cannot disable object versioning
+
+    we should be able to disable object verioning by removing its X-Versions-Location
+    metadata header by sending an empty key value. this description can be found at
+    No.8 in http://docs.openstack.org/user-guide/cli-swift-set-object-versions.html.
+
+    Fixes: http://tracker.ceph.com/issues/18852
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 17c5a0edd2227703cec867f0f588d4eae36dfe1c)
+
+commit eb84959bac369dbb86674f9e4225ba1c836c104c
+Merge: 85b7bc82e2 dc8ef3508b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 18:20:05 2017 +0200
+
+    Merge pull request #13609 from smithfarm/wip-19064-kraken
+
+    kraken: AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e9a10eaccd1d36e74969127fea7a595a21645b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:18:44 2017 -0600
+
+    PendingReleaseNotes: warning about 'osd rm ...' and #19119
+
+    See http://tracker.ceph.com/issues/19119
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit be96003c464481d8e84825178d600234a0d64d22)
+
+    Conflicts:
+            PendingReleaseNotes
+            - drop "Calculation of recovery priorities has been updated" because
+              that was included in 11.2.0 release notes
+            - do not backport >=12.0.0 release notes
+            - change heading to 11.2.1
+
+commit 85b7bc82e28c6f1da08c6f6829555724d43255d2
+Merge: dd44cfa736 46548c5125
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:43:50 2017 +0200
+
+    Merge pull request #13790 from shinobu-x/wip-18298-kraken
+
+    kraken: mon: force_create_pg could leave pg stuck in creating state
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit dd44cfa7368b193997bc59114e187c7f51e66cb1
+Merge: d477944262 177141ff17
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:41:16 2017 +0200
+
+    Merge pull request #13500 from shinobu-x/wip-18587-kraken
+
+     kraken: mon: 'osd crush move ...' doesnt work on osds
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d4779442620a8f6da5484be6188eeb9f636319c0
+Merge: 37bb087113 33d9a50dec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:39:51 2017 +0200
+
+    Merge pull request #13490 from shinobu-x/wip-18820-kraken
+
+    kraken: osd: --flush-journal: sporadic segfaults on exit
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 37bb087113e3c8f8f7a79ee256ae7651a5165bc2
+Merge: 8a6e631b29 d8c0141a75
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:35:45 2017 +0200
+
+    Merge pull request #13295 from asheplyakov/kraken-bp-18497
+
+    kraken: osd: publish PG stats when backfill-related states change
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8a6e631b292c56b4a577fd58cd604d31da8ba0bf
+Merge: b7877d1bb5 af2af7015a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:30:00 2017 +0200
+
+    Merge pull request #13253 from liewegas/wip-enxio-kraken
+
+    kraken: osd: do not send ENXIO on misdirected op by default
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b7877d1bb5e5a38ad2cae4b0833cbefeed2547cc
+Merge: e1f548e38e b49a71daf5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:28:08 2017 +0200
+
+    Merge pull request #14322 from shinobu-x/wip-18619-kraken
+
+    kraken: osd: ceph degraded and misplaced status output inaccurate
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7e30b630e2806c73ea503871599f958b58df7934
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:28:41 2017 -0400
+
+    librbd: is_exclusive_lock_owner API should ping OSD
+
+    This is required to detect if a peer has been silently blacklisted
+    and is therefore no longer the lock owner.
+
+    Fixes: http://tracker.ceph.com/issues/19287
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e15db05960a284bdf3701256722299d553cfd5aa)
+
+    Conflicts:
+            src/librbd/ManagedLock.[h|cc]: logic moved to ExclusiveLock
+
+commit e1f548e38ef38ad7dc9631ba61e35b52781dfc2f
+Merge: 086a4c8de1 baa42b6d7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 11:11:44 2017 +0200
+
+    Merge pull request #13216 from ovh/bp-osd-updateable-throttles-kraken
+
+    kraken: osd: allow client throttler to be adjusted on-fly, without restart
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 086a4c8de1243c2324d18cbe2d67704218065f83
+Merge: 95d4b4cefd 6d719558eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:45:10 2017 +0200
+
+    Merge pull request #14321 from shinobu-x/wip-kraken-15025
+
+    kraken: osd: New added OSD always down when full flag is set
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 95d4b4cefd0aff88e9c8fe25e44a4b33dfe66451
+Merge: c4e1e706dc 2b7083f8dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:42:31 2017 +0200
+
+    Merge pull request #13091 from dzafman/wip-18624-kraken
+
+    kraken: osd: Revert "PrimaryLogPG::failed_push: update missing as well"
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c4e1e706dc8b7efd723bffd9b770b8b09c9a2300
+Merge: 4ea954d75e 06093bcb91
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:52:06 2017 +0200
+
+    Merge pull request #12790 from SUSE/wip-18418-kraken
+
+    kraken: osd: leveldb corruption leads to Operation not permitted not handled and assert
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4ea954d75e0ff4969a5a35c77a229d43c66b9b28
+Merge: a4b0d2eeb5 478d0315cb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:03:40 2017 +0200
+
+    Merge pull request #13172 from smithfarm/wip-18713-kraken
+
+    kraken: rgw: radosgw-admin period update reverts deleted zonegroup
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a4b0d2eeb579b5a347d5a09f97eff8ca5a428101
+Merge: ce6423ac2c 29f55d7dd3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:02:35 2017 +0200
+
+    Merge pull request #13176 from smithfarm/wip-18709-kraken
+
+    kraken: rgw: multisite: sync status reports master is on a different period
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ce6423ac2c3d92011ab5e2434a3daec82195e085
+Merge: 577e3b4f23 5e15025ffb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:01:42 2017 +0200
+
+    Merge pull request #13864 from smithfarm/wip-19156-kraken
+
+    kraken: rgw: typo in rgw_admin.cc
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 577e3b4f23cca525d49cc02b6873a69ddf7f98e3
+Merge: fca689ec2f c56a3134eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:00:37 2017 +0200
+
+    Merge pull request #14137 from smithfarm/wip-19331-kraken
+
+    kraken: rgw: upgrade to multisite v2 fails if there is a zone without zone info
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fca689ec2f98ea40e3bc2ef7d338e73cb551934b
+Merge: 16f9611e05 84447a42e8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:59:18 2017 +0200
+
+    Merge pull request #14141 from smithfarm/wip-19342-kraken
+
+    kraken: rgw: "period update" does not remove short_zone_ids of deleted zones
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 16f9611e054e24f0e9324079b07ab2571a699caa
+Merge: 2fec93964c 76ce6841f7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:57:16 2017 +0200
+
+    Merge pull request #14142 from smithfarm/wip-19354-kraken
+
+    kraken: rgw: multisite: some 'radosgw-admin data sync' commands hang
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2fec93964cccd43cedda95f724392f3f22bd47cb
+Merge: 41770bef0d 51e8ba0a06
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:56:22 2017 +0200
+
+    Merge pull request #14144 from smithfarm/wip-19356-kraken
+
+    kraken: rgw: when converting region_map we need to use rgw_zone_root_pool
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 41770bef0d640c5cee48ec117e2c90bd3e13f442
+Merge: 278ad230f3 f9e3e9f9ff
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Apr 12 10:08:40 2017 -0700
+
+    Merge pull request #14485 from zmc/wip-upgrade-openstack
+
+    suites/upgrade/hammer-jewel-x: Add volumes
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit b8d988f7f19112c3cc2a8379a4ed20418414abba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Apr 1 23:04:22 2017 +0800
+
+    qa/tasks/workunit.py: use "overrides" as the default settings of workunit
+
+    otherwise the settings in "workunit" tasks are always overridden by the
+    settings in template config. so we'd better follow the way of how
+    "install" task updates itself with the "overrides" settings: it uses the
+    "overrides" as the *defaults*.
+
+    Fixes: http://tracker.ceph.com/issues/19429
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 47080150a17d238f38d9da824d227393ad767aad)
+
+commit 9212c1ec702be1b927a5a10dc1514803a13d85d5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 30 12:37:01 2017 +0800
+
+    tasks/workunit.py: specify the branch name when cloning a branch
+
+    c1309fb failed to specify a branch when cloning using --depth=1, which
+    by default clones the HEAD. and we can not "git checkout" a specific
+    sha1 if it is not HEAD, after cloning using '--depth=1', so in this
+    change, we dispatch "tag", "branch", "HEAD" using three Refspec classes.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 9ca7ccf5f1739f731da8bf31260594aea3a2932d)
+
+commit 665f03a467c1b59eb80782e4ba3bbeb48b5d5efc
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Mar 28 20:08:13 2017 -0700
+
+    tasks/workunit.py: when cloning, use --depth=1
+
+    Help avoid killing git.ceph.com.  A depth 1 clone takes about
+    7 seconds, whereas a full one takes about 3:40 (much of it
+    waiting for the server to create a huge compressed pack)
+
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit c1309fbef300a062138ac40eb5d3e5081b833072)
+
+commit 63153e957862ac0b46b8a205ed81d319b66ea6f5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Feb 27 16:29:40 2017 +0800
+
+    qa/tasks/workunit: use ceph.git as an alternative of ceph-ci.git for workunit repo
+
+    if we run upgrade test, where, for example, "jewel" is not in
+    ceph-ci.git repo, we should check ceph.git to clone the workunits.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit edceabbd47697cb4e7d21798e57a58969a92ce79)
+
+commit 471303c79a552bbd3765a6c47507fc2f2926a29d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Feb 24 16:43:07 2017 +0800
+
+    qa/tasks/workunit: use the suite repo for cloning workunit
+
+    as "workunits" reside in ceph/qa/workunits, it's more intuitive to
+    respect suite-repo option when cloning workunits.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4cf28de4c9a60a4618d101f555f5a2e59b7ab109)
+
+commit 278ad230f35c7742b6b066e693335dceaf291f3d
+Merge: baeac21515 c4f8114c6b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:42:37 2017 +0200
+
+    Merge pull request #14098 from shinobu-x/wip-19192-kraken
+
+    kraken: tools: ceph-brag fails to count "in" mds
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit baeac21515b9f472330f72fab6762e4e2535e6ba
+Merge: 29eb1ec09a d91697237a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:11:30 2017 +0200
+
+    Merge pull request #14155 from smithfarm/wip-18947-kraken
+
+    kraken: rbd-mirror: additional test stability improvements
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 29eb1ec09ab66cf8d1cee411809ab44a5a5d13b6
+Merge: c2e63ada80 6269f15223
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:10:47 2017 +0200
+
+    Merge pull request #14153 from smithfarm/wip-18892-kraken
+
+    kraken: librbd: Incomplete declaration for ContextWQ in librbd/Journal.h
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c2e63ada80e3baecbaa8f567695ab860257e5cc5
+Merge: 14caf24f1f 3b0a7a8d2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:09:55 2017 +0200
+
+    Merge pull request #14151 from smithfarm/wip-18822-kraken
+
+    kraken: tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 14caf24f1f3bc94d1a2a081957376474e8c3d66f
+Merge: 041b553a14 85b5817567
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:06:58 2017 +0200
+
+    Merge pull request #14149 from smithfarm/wip-18777-kraken
+
+    kraken: rbd: rbd --pool=x rename y z does not work
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 041b553a144cf798679caebcdb16d2ce2104c7ab
+Merge: 1d4617e1f2 24ecaed808
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:50:51 2017 +0200
+
+    Merge pull request #14315 from smithfarm/wip-19460-kraken
+
+    kraken: build/ops: spec file mentions non-existent ceph-create-keys systemd unit file, causing ceph-mon units to not be enabled via preset
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1d4617e1f25498a6213cf94c3ff0701b61283ed2
+Merge: c860645bfa fcb86696eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:42:27 2017 +0200
+
+    Merge pull request #13875 from smithfarm/wip-19172-kraken
+
+    kraken: rgw: S3 create bucket should not do response in json
+
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit c860645bfaa89ae48c4343208be8613fdc2538ce
+Merge: 2b7c762223 86a9450380
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:34:55 2017 +0200
+
+    Merge pull request #13873 from smithfarm/wip-19164-kraken
+
+    kraken: rgw: radosgw-admin: add the 'object stat' command to usage
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2b7c762223ee628a8e29e216fee4417b5b570edc
+Merge: 2869d35262 d96ae9ead2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:32:54 2017 +0200
+
+    Merge pull request #13245 from smithfarm/wip-18776-kraken
+
+    kraken: rbd: qemu crash triggered by network issues
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2869d35262bc261783b3418649458d2d6e263f18
+Merge: 9b17d88cf2 85bda6e263
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:32:17 2017 +0200
+
+    Merge pull request #13247 from smithfarm/wip-18456-kraken
+
+    kraken: rbd: attempting to remove an image with incompatible features results in partial removal
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9b17d88cf2ef4df40d26622ffa627d2fb70a557f
+Merge: 4a3f60f04f deb6d2c64c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:31:24 2017 +0200
+
+    Merge pull request #13132 from rjfd/wip-18609-kraken
+
+    kraken: librbd: allow to open an image without opening parent image
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4a3f60f04fde783fdfbf7ce1ea592ee348be34d3
+Merge: cfed98ae29 16b2fd00ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:17:09 2017 +0200
+
+    Merge pull request #13185 from smithfarm/wip-18721-kraken
+
+    kraken: build/ops: systemd restarts Ceph Mon to quickly after failing to start
+
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+
+commit 57f3656231b7780ff11f95a9feaaa28c3d573b35
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:27:08 2017 -0400
+
+    pybind: fix incorrect exception format strings
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68617455f534a612ade1331f43b032ab524704ae)
+
+commit cfed98ae291f12c119083ba08fe813b9f0bc9eb9
+Merge: 5abdffe950 e40f48dbfc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:55:16 2017 +0200
+
+    Merge pull request #13845 from smithfarm/wip-19147-kraken
+
+    kraken: rgw: DUMPABLE flag is cleared by setuid preventing coredumps
+
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 6e5ac14ab6f53424831e4cf08a3349a12b1ca947
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 17 00:56:34 2017 +0100
+
+    rgw: make sending Content-Length in 204 and 304 controllable
+
+    This commit introduces a new configurable "rgw print prohibited
+    content length" to let operator decide whether RadosGW complies
+    to RFC 7230 (a part of the HTTP specification) or violates it
+    but follows the Swift's behavior.
+
+    Fixes: http://tracker.ceph.com/issues/16602
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d8e3e64ec97a3c222a56bb6f510e5e23d7858615)
+
+commit 5abdffe950b51d5aff3500fd8dd7454a279da2fc
+Merge: 0517890f58 6f06cf80a7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:43:09 2017 +0200
+
+    Merge pull request #13555 from smithfarm/wip-18707-kraken
+
+    kraken: mds: failed filelock.can_read(-1) assertion in Server::_dir_is_nonempty
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0517890f58106eed3c0348a0a18b24bde7b340ab
+Merge: a48a897710 b464d1a571
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:37:49 2017 +0200
+
+    Merge pull request #13488 from shinobu-x/wip-18955-kraken
+
+    kraken: ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a48a8977100838fc4140d9d59511a6cc21ec0075
+Merge: 59c5511cb1 29c8c3fb86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 12:46:00 2017 +0200
+
+    Merge pull request #13112 from jcsp/wip-18678
+
+    kraken: mds: finish clientreplay requests before requesting active state
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 59c5511cb18286db1b032db8f3459763cc8dbb7c
+Merge: 8ded39b989 dab9fe761e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:52:42 2017 +0200
+
+    Merge pull request #14048 from asheplyakov/reindex-on-pg-split-kraken
+
+    kraken: osd: pg log split does not rebuild index for parent or child
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8ded39b9899ea461756650c17e1446682690c6f0
+Merge: 2a2bcc6051 35b13c6687
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:47:12 2017 +0200
+
+    Merge pull request #13116 from smithfarm/wip-18403-kraken
+
+    kraken: mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version)
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2a2bcc6051ad0d4badfdbabb81bd3d9cfb449234
+Merge: 005e803abe 7475e44214
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:41:21 2017 +0200
+
+    Merge pull request #13487 from shinobu-x/wip-18819-kraken
+
+    kraken: common: possible lockdep false alarm for ThreadPool lock
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 005e803abe5dbe652bbee1c3e353fb38db446800
+Merge: 42e93ae1b5 a379430d19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:36:12 2017 +0200
+
+    Merge pull request #14147 from smithfarm/wip-18769-kraken
+
+    kraken: rbd: [  FAILED  ] TestJournalTrimmer.RemoveObjectsWithOtherClient
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 42e93ae1b5e1e65a8abb3f37469030ed4145fbda
+Merge: fb29415450 ecb459f083
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 02:53:06 2017 +0200
+
+    Merge pull request #13026 from SUSE/wip-18606-kraken
+
+    kraken: tools: ceph-disk prepare writes osd log 0 with root owner
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f9e3e9f9ff22901d90e783a28efdfafa12be650f
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Apr 11 09:25:50 2017 -0600
+
+    suites/upgrade-hammer-jewel-x: Add volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit 543e4b90603b5992ea5d1c66d89e077375faf14e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 2 12:41:07 2017 +0100
+
+    build/ops: add psmisc dependency to ceph-base (deb and rpm)
+
+    Fixes: http://tracker.ceph.com/issues/19129
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 769b695465162bc8424abf8e2f259e6765b5bbff)
+
+commit fb29415450eda1ec480600132a937efbe988a1f4
+Merge: 10b441aac7 6b3655fe4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Apr 9 22:40:21 2017 +0200
+
+    Merge pull request #13497 from shinobu-x/wip-17821-kraken
+
+    kraken: ceph-disk: does not support cluster names different than 'ceph'
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 910b9899702cfc08b74df6d988e1981efdb33f88
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 3 13:37:17 2017 +1000
+
+    ceph-disk: Populate mount options when running "list"
+
+    Also tidy up by moving duplicated code into a function
+
+    Fixes: http://tracker.ceph.com/issues/17331
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7943ab2e01e24f2dfc5b6f1d3ffdc8a49e01af45)
+
+commit 5c7a07677aa142fadbab36d0c1b24aea32ce9ad5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Mar 28 11:49:08 2017 +0200
+
+    osdc/Objecter: respect epoch barrier in _op_submit()
+
+    Epoch barrier instructs us to avoid sending (i.e. pause) any OSD ops
+    until we see a barrier epoch.  The only thing epoch_barrier check in
+    target_should_be_paused() does is keep already paused ops paused.  We
+    need to actually pause incoming OSD ops in _op_submit().
+
+    Fixes: http://tracker.ceph.com/issues/19396
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit f8e8efc0a53d7bd807cc0c2178aef7c4bed62ab7)
+
+commit c2cc96e6c19c77d0ea13e8a015d0dfa66ba1b304
+Author: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+Date:   Wed Mar 22 16:13:50 2017 +0100
+
+    common: fix segfault in public IPv6 addr picking
+
+    sockaddr is only 16 bytes big, so declaring net as sockaddr
+    and then casting to sockaddr_in6 in case of IPv6 cannot
+    work.
+
+    using sockaddr_storage works for both IPv4 and IPv6, and is
+    used in other code parts as well.
+
+    note that the tests did not find this issue as they declared
+    the bigger structs and casted the references to (sockaddr *)
+
+    Fixes: http://tracker.ceph.com/issues/19371
+    Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+    (cherry picked from commit ae2ee3d3835fe25b35eeb1a841ee5234cd69eb65)
+
+commit b49a71daf51bf2fdd892cbfc033c0760cbce0464
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 18 08:33:40 2017 -0800
+
+    osd: Calculate degraded and misplaced more accurately
+
+    Calculate num_object_copies based on the larger of pool size,
+    up set size and acting set size.
+
+    Calculate num_objects_degraded as the difference between num_object_copies
+    and all copies found on acting set and backfilling up set OSDs.
+
+    Calculate num_objects_misplaced as all copies on acting set OSDs not in up set
+    less copies that have been backfilled to up set OSDs.
+
+    Fixes: http://tracker.ceph.com/issues/18619
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8423bc40759cca137f61e7b755411719a84369d4)
+
+commit 6d719558eb8135b3104132e7e70c163169e93876
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Mar 13 23:41:58 2017 +0800
+
+    osd: don't share osdmap with objecter when preboot
+
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit a5a3644eecc49b4eea890c6999fe87536495dcbe)
+
+commit 24ecaed8081f0bfec3324f86423f80e5d29c99b3
+Author: Sébastien Han <seb@redhat.com>
+Date:   Thu Mar 30 09:32:31 2017 +0200
+
+    systemd: remove all occurence of ceph-create-keys
+
+    ceph-create-keys unit file does not exist anymore so there is no need to
+    call it.
+
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit 205cff12b3fe577a02db119ed3084d5c46ba76ce)
+
+commit b77d780c91f96ce9bb63fa1c09d507d02e555178
+Author: Sébastien Han <seb@redhat.com>
+Date:   Wed Mar 29 15:47:56 2017 +0200
+
+    systemd: remove ceph-create-keys from presets
+
+    ceph-create-keys unit file was removed here:
+
+    * https://github.com/ceph/ceph/commit/8bcb4646b6b9846bb965cdec3ca2a21eb3b26bab
+    * https://github.com/ceph/ceph/commit/dc5fe8d415858358bd0baf5d8dce0a753f5e0cea
+
+    As a consequence the systemctl preset command now fails to run since the
+    unit does not exist anymore. Due to the redirection in /dev/null we
+    don't know what's happening.
+
+    Ultimately the mon unit doesn't get enabled and the mon service won't
+    start after reboot.
+    Removing the old/non-existent unit makes the command succeed now.
+
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit 151c0ce213186b2196531ce74478ed7671eb9819)
+
+commit 10b441aac71ad2d7a7ee6181c958a8bb0f710213
+Merge: 55444ce3ae df59d6d5f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Mar 28 21:00:31 2017 -0400
+
+    Merge pull request #14163 from trociny/wip-19368
+
+    kraken: librbd: possible race in ExclusiveLock handle_peer_notification
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 55444ce3aea473f5cd7c8f6eeb770b819318a6a4
+Merge: 98a87fa97c 2d668468f6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 28 15:25:39 2017 -0700
+
+    Merge pull request #14200 from dzafman/wip-18533-again
+
+    Fixes: http://tracker.ceph.com/issues/19391
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2d668468f60dd55a5591aa74bda491545368eee0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 20 17:28:45 2017 -0700
+
+    filestore, tools: Fix logging of DBObjectMap check() repairs
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1704f62c0831e6b07138f7dd14a89fef3c9ed2c1)
+
+commit f4945116b70b2ca457565ed313d2e0b52cf13383
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 3 15:04:02 2017 -0800
+
+    osd: Simplify DBObjectMap by no longer creating complete tables
+
+    Bump the version for new maps to 3
+    Make clone less efficient but simpler
+    Add rename operation (use instead of clone/unlink)
+    For now keep code that understands version 2 maps
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 738156a99ed1caf61e5a8230eb8048360056c08e)
+
+    No ghobject_t::operator>() so use Kraken cmp_bitwise() instead
+    Need to use MIN_GHOBJ/MAX_GHOBJ instead of std::min/std::max
+
+commit 06c27d7ec0fa31000232b8f45a0fbeb6702f0180
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 16:17:32 2017 -0800
+
+    ceph-osdomap-tool: Fix seg fault with large amount of check error output
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1dda0411f4fbb14ce1e0062da9f14ec3af505d39)
+
+commit 4810c544b10972e6b276517a2bb4ba2da0b06697
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 15:02:33 2017 -0800
+
+    osd: Add automatic repair for DBObjectMap bug
+
+    Add repair command to ceph-osdomap-tool too
+
+    Under some situations the previous rm_keys() code would
+    generated a corrupt complete table.  There is no way to
+    figure out what the table should look like now.  By removing
+    the entries we fix the corruption and aren't much worse off
+    because the corruption caused some deleted keys to re-appear.
+
+    This doesn't breaking the parent/child relationship during
+    repair because some of the keys may still be contained
+    in the parent.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4cd3c74c928a32e065ed9543d6c91d8718a6ae3d)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 61dad6c15dc7ac94c4bea1fb57823d78ae2db5a1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 14:59:40 2017 -0800
+
+    ceph-osdomap-tool: Fix tool exit status
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 666f14ed90655a2d1bedde8561949625db7a9e6c)
+
+commit a72f923186ac490fe780f7168a6a34c7d933cf96
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:51:42 2017 -0800
+
+    DBObjectMap: rewrite rm_keys and merge_new_complete
+
+    Leverage the updated in_complete_region and needs_parent to simplify
+    these methods.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c4dffb68eaafe724f7fdae93a4285a7f8003ea29)
+
+commit 960431a6b8e79f6832a85e462889dd1f26f12624
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:50:57 2017 -0800
+
+    DBObjectMap: strengthen in_complete_region post condition
+
+    Previously, in_complete_region didn't guarantee anything about
+    where it left complete_iter pointing.  It will be handy for
+    complete_iter to be pointing at the lowest interval which ends
+    after to_test.  Make it so.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 97b35f4d7d4862da4b6f50ecaef0d292a671fd04)
+
+commit a3c3e3ef4e3d20baf28000f11b4ea5bb45e161f9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:48:57 2017 -0800
+
+    DBObjectMap: fix next_parent()
+
+    The previous implementation assumed that
+    lower_bound(parent_iter->key()) always leaves the iterator
+    on_parent().  There isn't any guarantee, however, that that
+    key isn't present on the child as well.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 74a7631d0938d7b44894f022224eab10a90d5cec)
+
+commit 86881dd361e21e3e4553b416331ca393e88b7a3e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Feb 9 10:47:59 2017 -0800
+
+    test_object_map: add tests to trigger some bugs related to 18533
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f131dbcf5bb17107c029f942a57e9bf4432a26ee)
+
+commit 4c7fbea9e4c171ab21f3dcd6ca0b17abb3420979
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 12:40:33 2017 -0800
+
+    test: Add ceph_test_object_map to make check tests
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0e97a01bd7291458881ee53cece2d887f6333669)
+
+commit ef5aeab9b22292c366489668ee21ce4b9d2d8572
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:56:27 2017 -0800
+
+    ceph-osdomap-tool: Add --debug and only show internal logging if enabled
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fb2b2d13953979e5da9f571ab8c4b0b510b8368)
+
+commit c5d462f3524f0e5eff1594d7ddf97c3c9c0882a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:55:48 2017 -0800
+
+    osd: DBOjectMap::check: Dump complete mapping when inconsistency found
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fcf1e17c645e8fad5216c3e59627c817e5c858c7)
+
+commit b37b0298ca9c002033b3ae8ebcaeb7a24690447d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 15:38:51 2017 -0800
+
+    test_object_map: Use ASSERT_EQ() for check() so failure doesn't stop testing
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 053a273cbc02d6902a4bb1f11db1ea946498df3a)
+
+commit 8f297204a428b48d2119df0e1a8da41e4f15e4fb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 10:02:40 2017 -0800
+
+    tools: Check for overlaps in internal "complete" table for DBObjectMap
+
+    Changed check to return an error count and fix tool error message
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5e8eb962db6187ea19b96ba29ac83469c90b4ea)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit bc2228a903bbb89a11d7863f9307d6c3b5d4928c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 09:40:49 2017 -0800
+
+    tools: Add dump-headers command to ceph-osdomap-tool
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f4101591ad701a62fe027c4744ca8ea505f44bdc)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 97186735783a9758a1695a9938b22c2b4b46f530
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:09:42 2017 -0800
+
+    tools: Add --oid option to ceph-osdomap-tool
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2d94889e9ee3359017b1efd560f3557ce03ccee6)
+
+commit c208c91c7f845896b4fe0befd62cc66bee69018f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:31:18 2017 -0800
+
+    osd: Remove unnecessary assert and assignment in DBObjectMap
+
+    Fix and add comment(s)
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 937e6a03ea4692cc44d53faa0615f8e808c9eb03)
+
+commit df59d6d5f7deb586cf14a6ef6984e6847db08852
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 28 15:55:11 2017 +0200
+
+    kraken: librbd: possible race in ExclusiveLock handle_peer_notification
+
+    This is a direct commit to kraken -- the master diverged after
+    ManagedLock refactoring and is not affected.
+
+    Fix: http://tracker.ceph.com/issues/19368
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d91697237ad77eaf60098536a2cd3538ea9aeabf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 21 15:31:43 2016 -0500
+
+    qa/workunits/rbd: resolve potential rbd-mirror race conditions
+
+    Fixes: http://tracker.ceph.com/issues/18935
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 63eae97afc1a92412525468263fb8696a243ebac)
+
+commit 6269f152238b77b0d02e4ad865164a531541d6db
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Feb 8 23:47:57 2017 +0100
+
+    librbd: Include WorkQueue.h since we use it
+
+    We use m_work_queue of type ContextWQ in handle_update function but we
+    do not include common/WorkQueue.h that defines ContextWQ. This results
+    in dereference of an incomplete type and causes build error in latest
+    Fedora rawhide (future 26).
+
+    Fixes: http://tracker.ceph.com/issues/18862
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 480f82847ad1fc7959f1fe5a90761a5a24550993)
+
+commit 3b0a7a8d2aa0dbc7cfce317bd327f88afce2469e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 18 20:54:22 2017 -0500
+
+    librbd: avoid possible recursive lock when racing acquire lock
+
+    Fixes: http://tracker.ceph.com/issues/17447
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e46e8eb664f573bd70ae7c96a6d9a98b0deb09e)
+
+commit 85b5817567c22acf63719a65bd65e77709ffce0c
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Mon Jan 30 13:03:20 2017 +0100
+
+    rbd: destination pool should be source pool if it is not specified
+
+    Currently if user perform image rename operation and user give pool
+    name as a optional parameter (--pool=<pool_name>) then currently
+    its taking this optional pool name for source pool and making
+    destination pool name default pool name.
+    With this fix if user provide pool name as a optional pool name
+    parameter then it  will consider both soruce and destination pool
+    name as optional parameter pool name.
+
+    Fixes: http://tracker.ceph.com/issues/18326
+
+    Reported-by: М�РК КОРЕ�БЕРГ <socketpair@gmail.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 01f23aa99fb694da326ab408e75b33c640ce660b)
+
+commit a379430d1921a1e5a330b4bfe0189a9a21dc5f81
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 30 17:41:51 2017 -0500
+
+    journal: stop processing removal after error
+
+    Fixed: http://tracker.ceph.com/issues/18738
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 040004f53f7d334fc9452e3f5fd80169cad5f65e)
+
+commit 51e8ba0a06f2b5574f82f0ca63a9e9b73c291b51
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Mar 12 12:11:28 2017 +0200
+
+    rgw: use rgw_zone_root_pool for region_map like is done in hammer
+
+    Fixes: http://tracker.ceph.com/issues/19195
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c91dd6d9efd148e0fe0f027dde537e977de9aa26)
+
+commit 76ce6841f7066b4902665dcf752568d0b5417652
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Mon Feb 27 15:52:43 2017 +0800
+
+    rgw: use separate http_manager for read_sync_status
+
+    concurrent users of read_sync_status() use different cr managers, when get_resource must
+    use http_manager related to the cr manager.
+
+    Fixes: http://tracker.ceph.com/issues/19236
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c412024889f8995d98096ac863bafee71624bd70)
+
+commit 84447a42e82a21e08f71ea6113216d9a2cb2ab09
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Mar 9 15:24:08 2017 -0500
+
+    rgw: clear old zone short ids on period update
+
+    the short ids of old, removed zones were being kept in the period to
+    guard against hash collisions with new zones
+
+    but for a hash collision to cause a wrong object to sync, that object
+    would have to be uploaded simultaneously to two different zones that had
+    the same short id
+
+    to avoid this, we just have to prevent the period from containing two
+    colliding zones at the same time - we don't have to remember old zone
+    short ids forever
+
+    Fixes: http://tracker.ceph.com/issues/15618
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9c45633c836c966ab1f75ea2b1ad3fa0a4886600)
+
+commit c56a3134ebe1b531af09b19214a16b37e0e8a6d8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 13:03:24 2017 +0200
+
+    rgw: skip conversion of zones without any zoneparams
+
+    Fixes: http://tracker.ceph.com/issues/19231
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 36cf5a5c8179c6313346b2e29286c537c6fefce8)
+
+commit d0d70359811c306116921cf8f3768a97da16a813
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 11:16:26 2017 +0200
+
+    rgw: better debug information for upgrade
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e9f3bf8eab1dd46a92f54b0f7afe1f4c0e4204db)
+
+commit 01f5d8899b8d16e419e1bfdd4e8bbd80a058df93
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jan 31 18:01:32 2017 +0100
+
+    rgw/rgw_rados.cc: prefer ++operator for non-primitive iterators
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 7086cf9a73f2ec1eb96c0e752beb1b74fca18570)
+
+commit c4f8114c6b86bc1e0afabecf9424dadd30b37006
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 11:33:27 2017 +0800
+
+    brag: count the number of mds in fsmap not in mdsmap
+
+    this change was introduced in 4e9b953
+
+    Fixes: http://tracker.ceph.com/issues/19192
+    Signed-off-by: Peng Zhang <zphj1987@gmail.com>
+    (cherry picked from commit 2d25a9c0c760664d3de33ecca0e0272c1031cd46)
+
+commit 81f3d860324da49f71b757e6cc8f32966272cf5c
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Mar 21 12:22:20 2017 +1000
+
+    tools/rados: Check return value of connect
+
+    Fail gracefully if Rados::connect returns an error.
+
+    Fixes: http://tracker.ceph.com/issues/19319
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit c119091ef0844e4a1ddd790a8bfef8f06bb57d58)
+
+commit 076e4b7991e3bd734a41604fa7cb00c7b03d7749
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 20 15:01:01 2016 -0400
+
+    rgw: fix break inside of yield in RGWFetchAllMetaCR
+
+    the yield macro is implemented with for/switch, so the breaks in
+    RGWFetchAllMetaCR weren't being applied to the for loop as expected -
+    so any of these breaks send RGWFetchAllMetaCR into an infinite loop
+
+    removed the yield {} block, so that breaks will apply to the for loop as
+    intended, then added a single yield; statement to allow the
+    entries_index consumer to run one per iteration
+
+    Fixes: http://tracker.ceph.com/issues/17655
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 190bd385a7be52867d65740c410884f5c8cbc21f)
+
+commit 5dcf60702cd1733dbe7f2d1d7dc68a096684bf77
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 12:10:40 2017 -0500
+
+    rgw: delete_system_obj() fails on empty object name
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 67401193f871db95a6045915fa59dce8c5dd1012)
+
+commit dea53b3bf267709d2a8daacd961997892ad2d0f4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 11:42:45 2017 -0500
+
+    rgw: if user.email is empty, dont try to delete
+
+    Fixes: http://tracker.ceph.com/issues/18980
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 022ecf0fcc8e44912c8758ee1d9a452dc23cbbce)
+
+commit dab9fe761ec7f46b7da035f710f824516363119c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 17 12:46:38 2017 -0500
+
+    osd/PGLog: avoid return by value on ginormous log
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a6ead998771753d95382abd082f451e6f67744e4)
+
+commit eac5610896e59bc8e711ed48446aa6967ebb61c8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 17 14:50:38 2017 -0500
+
+    osd/PGLog: reindex properly on pg log split
+
+    When pg_log_t::split_out_child() runs it builds the list, which means the
+    old indexes are wrong (the point to bad memory), but index() will not
+    rebuild them because ever since b858e869e78927dccebaa350d246bd74af7f1de9
+    we won't rebuild them if they are already built.
+
+    Fix that by calling unindex() before the split.
+
+    Further, the new child log also needs to be indexed.  Fix that too.
+
+    Fixes: http://tracker.ceph.com/issues/18975
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 643ae42cf27f16dd6ed4e1402acc0483bb9fca74)
+
+commit 0227920438d98a8c1d74f0ec7a95bc63fa066acf
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Mar 1 17:19:04 2017 +0100
+
+    osd/OSDMap: don't set weight to IN when OSD is destroyed
+
+    Since commit 4e28f9e63644 ("osd/OSDMap: clear osd_info, osd_xinfo on
+    osd deletion"), weight is set to IN when OSD is deleted.  This changes
+    the result of applying an incremental for clients, not just OSDs.
+    Because CRUSH computations are obviously affected, pre-4e28f9e63644
+    servers disagree with post-4e28f9e63644 clients on object placement,
+    resulting in misdirected requests.
+
+    Fixes: http://tracker.ceph.com/issues/19119
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a6009d1039a55e2c77f431662b3d6cc5a8e8e63f)
+
+commit d4a43e1fd56cfce1e1b64827a3d98c14ec69e193
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Feb 3 19:06:31 2017 +0800
+
+    rgw: swift: fix anonymous user's error code of getting object
+
+    The openstack swift will return 401 rather than 403 when
+    the anon user has no permission to get objects.
+
+    Fixes: http://tracker.ceph.com/issues/18806
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 318de28e1011914dc7f3701549eb3ef227abbc3c)
+
+commit fcb86696eb4999689c7da71d565e1262beba9dec
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Feb 16 17:40:50 2017 +0100
+
+    doc: rgw: make a note abt system users vs normal users
+
+    Mention that system users don't behave like normal users in context of
+    normal rest operations
+
+    Fixes: http://tracker.ceph.com/issues/18889
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a47bcf70c9f51a6601b809cba219f5615b204d34)
+
+commit 86a945038059d976e2f7227d9ee327ec2899d15c
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 7 14:37:36 2017 +0530
+
+    rgw: Let the object stat command be shown in the usage
+
+    Fixes: http://tracker.ceph.com/issues/19013
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 0fe76f83d19be098ef54fb0492a376fef3aa9e23)
+
+commit 1f86be6e19dc3e26f62bf57c55d662414247b480
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 20 16:00:01 2017 -0500
+
+    rgw: RGWMetaSyncShardControlCR retries with backoff on all error codes
+
+    RGWBackoffControlCR only treats EBUSY and EAGAIN as 'temporary' error
+    codes, with all other errors being fatal when exit_on_error is set
+
+    to RGWMetaSyncShardControlCR, a 'fatal' error means that no further sync
+    is possible on that shard until the gateway restarts
+
+    this changes RGWMetaSyncShardControlCR to set exit_on_error to false, so
+    that it will continue to retry with backoff no matter what error code it
+    gets
+
+    Fixes: http://tracker.ceph.com/issues/19019
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3e4059557fd6cad5d31014327f60832b36d04a6c)
+
+commit b39bef09611653723a29c989a6012064f17bd586
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 21 16:33:29 2017 +0530
+
+    rgw: Correct the return codes for the health check feature
+    Fixes: http://tracker.ceph.com/issues/19025
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+
+    (cherry picked from commit 4da2bf310f6d43423554c32e43ebf90ad2c3f3a9)
+
+commit 5e15025ffbf80f65cde891ad47fe1f1b913465cd
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Wed Feb 22 12:03:46 2017 +0530
+
+    rgw: Fixes typo in rgw_admin.cc
+
+    Issue: http://tracker.ceph.com/issues/19026
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 58837ef6ce8cbcfc2cac29d5f833b2cf62d8737a)
+
+commit e40f48dbfce23cbc2164f8789db4688fbec9c3bb
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Feb 27 13:06:59 2017 +1000
+
+    rgw: set dumpable flag after setuid post ff0e521
+
+    ff0e521 resolved the issue for the other daemons but not for rgw since
+    it calls setuid (via civetweb) after the new code sets PR_SET_DUMPABLE.
+    Add another prctl call before wait_shutdown.
+
+    Fixes: http://tracker.ceph.com/issues/19089
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit bc458d39630b599e0e1ca9fe25ad7455fcffdd10)
+
+commit 47cf8f6699ec0690e2a296b3abe289ce5d212e1c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 27 10:35:01 2017 -0800
+
+    rgw: don't init rgw_obj from rgw_obj_key when it's incorrect to do so
+
+    Fixes: http://tracker.ceph.com/issues/19096
+
+    rgw_obj_key currently deals with the bucket index key, and not
+    representing a (name, instance, ns) tupple. Need to initialize
+    it in two steps.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 392c5d9dae6ba699014ffe6e1e67818fa62d7e41)
+
+commit 03b1ab4b13780a53c7b9017f3a6207dab2ffff5b
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Thu Jan 19 21:00:26 2017 +0800
+
+    rgw: swift: the http referer acl in swift API should be shown
+    The container acl about http referer set should be shown in container metadata dump.
+
+    Fixes: http://tracker.ceph.com/issues/18665
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 3860ddad084c5b8d9d5ec83aeb3220207cd0e5ad)
+
+commit 98a87fa97c9b23e21a05130c72730f5034691310
+Merge: 7e16ea218d e2074cef71
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Mar 4 11:22:44 2017 -0600
+
+    Merge pull request #13501 from shinobu-x/wip-18371-kraken
+
+     kraken: ceph-disk: error on _bytes2str
+
+commit 46548c51255518e422d544f0d0776068051fdacc
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Mon Nov 14 19:33:56 2016 -0500
+
+    common: Unskew clock
+
+    In preparation to deglobalizing CephContext, remove the CephContext*
+    parameter to ceph_clock_now() and ceph::real_clock::now() that carries
+    a configurable offset.
+
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 750ad8340c827d2f8896e1251e45f921dddb9f30)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 1120cd74d88b163f7cd1c81ae5670bf6d6153715
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 31 12:31:42 2017 -0500
+
+    mon/PGMonitor: fix force_create_pg
+
+    We weren't carefully registering the creating PG.  In
+    particular, the current osd mappings (acting, up, etc.)
+    weren't getting filled in, which meant the PG wasn't
+    (necessarily) mapped to an OSD until the OSDMap updated
+    and we recalculated mappings.
+
+    Fix by sending us through the common code path so that
+    all fields get updated properly.
+
+    Fixes: http://tracker.ceph.com/issues/18298
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9a41a0b7289fa59f4b747a63e152e88af6e8abd5)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 396f3325507cf899433da0a5f4964b81e0c2caa8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 31 12:31:12 2017 -0500
+
+    mon/PGMonitor: clean up some send_pg_creates cruft
+
+    ff1fd4b4b69b52ef5ae734115b1d733aad03b03e already removed most of
+    this; remove the now useless stub and clean up the unneede con
+    check.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b79a85e80a1a660c5a5746270edb86d3f09cd983)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 6196b6edd5ea87c9fabe598a309516e1c0ba73e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 17:42:26 2016 -0500
+
+    mon/MonCommands: remove send_pg_creates from cmd table
+
+    Forgot this in 6cbdd6750cf330047d52817b9ee9af31a7d318ae
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 15f17bcca0bde2ed15bb45d67659f4ebbd4cb516)
+
+commit c8d70ee0129ce2cc459a2be386531aeee5c6c422
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Thu Jan 19 21:26:17 2017 +0800
+
+    rgw: swift: The http referer should be parsed to compare in swift API
+    The http referer should be parsed to compare with the url set on the container read acl. If we set .r:www.example.com on container read acl, we should parse the hostname 'www.example.com' of the http referer like 'http://www.example.com' from the http request.
+
+    Fixes: http://tracker.ceph.com/issues/18685
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 941dfad67174ae3ec517e76bf4028c50fb46fe82)
+
+commit 7e16ea218d38703f1903dca002905f040a821590
+Merge: ccc1982472 6adf39b983
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Mar 3 11:14:48 2017 +0800
+
+    Merge pull request #13757 from liewegas/wip-pgp-kraken
+
+    qa/tasks: set pgp = pg num on thrashing finish
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6adf39b98305b6af8d0192b35ec22c5007293d32
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Feb 19 13:10:44 2017 +0800
+
+    test: Thrasher: do not update pools_to_fix_pgp_num if nothing happens
+
+    we should not update pools_to_fix_pgp_num if the pool is not expanded or
+    the pg_num is not increased due to pgs being created. this prevent us
+    from fixing the pgp_num after done with thrashing if we actually did
+    nothing when fixing the pgp_num when thrashing, but we removed the pool
+    from pools_to_fix_pgp_num after set_pool_pgpnum() returns.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c0f0cde399a80cea617c115417e5390a63ec674e)
+
+commit ab5538fd7e3b727616272edb565e3782d004632a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Feb 12 13:16:57 2017 +0800
+
+    test: Thrasher: update pgp_num of all expanded pools if not yet
+
+    otherwise wait_until_healthy will fail after timeout as seeing warning
+    like:
+
+    HEALTH_WARN pool cephfs_data pg_num 182 > pgp_num 172
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 136483a8f940710cbe40804afa7af2eac6728d50)
+
+commit ccc1982472898efe2fcbd6b1cd3cad7afbf2c015
+Merge: ae8eff5f2d 2c9889bb76
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Mar 1 15:16:17 2017 -0700
+
+    Merge pull request #13707 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+commit 2c9889bb768198fe292de0c3edb076707e8a50e0
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Feb 28 13:07:15 2017 -0700
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+    Looks like we only need two per node, since there is only one OSD per
+    node, and ceph-deploy wants two disks per OSD to account for the
+    journal.
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 87072e277c9ef259c9ee2ae1f761e252aa216713)
+
+commit ae8eff5f2dddcc064485ab69221c6d716e63925f
+Merge: 9fa9248e92 f9a282bc9d
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Feb 27 12:23:10 2017 -0700
+
+    Merge pull request #13673 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/{ceph-ansible,rest}: OpenStack volumes
+
+commit f9a282bc9dda98a103db2c7ebe63450dad23cc5b
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:14:41 2017 -0700
+
+    qa/suites/rest: Openstack volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 99d942145f4206c00aca30c0bb74f0edc4bac798)
+
+commit e0332944c7647cc269d6b18ca867896d5bea2fa7
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:06:26 2017 -0700
+
+    qa/suites/ceph-ansible: Openstack volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 964b983bdbd412311bce56184d12d1b6d43c7f28)
+
+commit 9fa9248e92f34127ca8b0d3b021b5ac0da0f6609
+Merge: f159208832 b2e3692347
+Author: Brad Hubbard <badone@users.noreply.github.com>
+Date:   Sun Feb 26 09:23:43 2017 +1000
+
+    Merge pull request #13485 from shinobu-x/wip-18644-kraken
+
+    kraken: osd: condition object_info_t encoding on required (not up) features
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f1592088328480c6545435a7031d8f06277a9d0c
+Merge: c59d13d162 2ab8239b09
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 17:22:11 2017 -0700
+
+    Merge pull request #13641 from zmc/wip-fs-openstack-kraken
+
+    qa/suites/fs: Add openstack volume configuration
+
+commit 2ab8239b09f5d1215ad828eae7c64fb0f199d663
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 15:25:19 2017 -0700
+
+    qa/suites/fs: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit b076d89a3f1fbad7d477913812b2e17529abeacf)
+
+commit c59d13d162a2b2d51fe7c65a56a2a17b7d94aa8f
+Merge: 8f1f2018fb 596cfbbd0f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:44:04 2017 -0700
+
+    Merge pull request #13638 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/{knfs,hadoop,samba}: OpenStack volume configuration
+
+commit 8f1f2018fb06bd56b63cb661a852ba6b44c5e749
+Merge: 1e71356ee4 cec66c4dac
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:43:34 2017 -0700
+
+    Merge pull request #13635 from zmc/wip-kcephfs-openstack-kraken
+
+    qa/suites/kcephfs: Openstack volume configuration
+
+commit 6b3655fe4bd3c493e6b10dd597d791a5b572df2b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 22 01:49:12 2017 +0100
+
+    ceph-disk: dmcrypt activate must use the same cluster as prepare
+
+    When dmcrypt is used, the fsid cannot be retrieved from the data
+    partition because it is encrypted. Store the fsid in the lockbox to
+    enable dmcrypt activation using the same logic as regular activation.
+
+    The fsid is used to retrive the cluster name that was used during
+    prepare, reason why activation does not and must not have a --cluster
+    argument.
+
+    Fixes: http://tracker.ceph.com/issues/17821
+
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7f66672b675abbc0262769d32a38112c781fefac)
+
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py
+
+commit 596cfbbd0fc521ef66e7e11a7098ea78adeb2565
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:46:54 2017 -0700
+
+    qa/suites/samba: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit e0296d706422ea4dc01d84f8786f6f7104c3d996)
+
+commit 3359050ce87426f3b41afe85c117de2d47ece395
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:45:18 2017 -0700
+
+    qa/suites/hadoop: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 3fef0a49da2ccfdceba7b98e9096be8305da1111)
+
+commit a2ff395cb66fc20fd9a18e74a6cba5510d144a61
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:44:33 2017 -0700
+
+    qa/suites/knfs: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 62c6fd3371adf0f420c12d9c7e2b3a2a0c69256b)
+
+commit cec66c4dacb39600f74927c9f8890805d3f7acdc
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:37:23 2017 -0700
+
+    qa/suites/kcephfs: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit ec6fb28eaf8e2db327e4afc115879a40c7664e07)
+
+commit 1e71356ee44828a0624d77f77b6a22580d1df28b
+Merge: 90c61976e8 7439d4cff2
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:16:02 2017 -0700
+
+    Merge pull request #13633 from zmc/wip-krbd-openstack-kraken
+
+    qa/suites/krbd: Add openstack volume configuration
+
+commit 90c61976e82c5e23d4d3cbae00eddb47e649bb65
+Merge: 91a2ab03e2 4c55a78a52
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:29 2017 -0700
+
+    Merge pull request #13613 from ceph/wip-rgw-openstack-kraken
+
+    qa/suites/rgw: Add openstack volume configuration
+
+commit 91a2ab03e268227164ffec37b693d370485cc004
+Merge: 6c6b185bab 19ca0db392
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 24 10:37:12 2017 -0800
+
+    Merge pull request #13618 from tmuthamizhan/wip-rm-trusty-kraken
+
+    qa: drop ubuntu trusty support
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 7439d4cff231e31dbf7c19591560bc0aa778f68a
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 11:17:45 2017 -0700
+
+    qa/suites/krbd: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 201b4d0d1e92bf95ac6a8f2951b664763030f12d)
+
+commit 4c55a78a52f9e089b97ab8a2feb9876ebdba0516
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Feb 23 10:14:05 2017 -0700
+
+    qa/suites/rgw: Add openstack volume configuration
+
+    Without this, OSDs will fail to create on instances whose root fs isn't
+    xfs.
+
+    (cherry picked from commit 8af4c35f9577ef5a88307ea5cbbe2561a473926c)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit b2e369234705ae78774261c62081729e6a50e8e5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Mon Feb 6 12:51:05 2017 +0100
+
+    osd/OSDMap: require OSD features only of OSDs
+
+    Fix up commit 1a5cc32f0a3b ("osd/OSDMap: reflect REQUIRE_*_OSDS flag in
+    required features") -- require_*_osds shouldn't affect older clients.
+
+    Fixes: http://tracker.ceph.com/issues/18831
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ab558bbf2956157cbde59c155a5180a62ae73d07)
+
+    Conflicts:
+            src/osd/OSDMap.cc
+
+commit 19ca0db392b7bdf9758aae4fb6bb6cb927411568
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 17 21:06:43 2017 +0000
+
+    qa: drop ubuntu trusty support
+
+    ceph-ansible dropped support for OS that doesnt support systemd
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    (cherry picked from commit 4d4b38eca81f7b57e3d3b31e1c13e7ab0ba5b30f)
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 6c6b185bab1e0b7d7446b97d5d314b4dd60360ff
+Merge: 7d9ef63d65 8ea14ce9ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 23 14:55:56 2017 -0800
+
+    Merge pull request #13283 from zmc/wip-smoke-openstack-kraken
+
+    qa/suites/smoke: add openstack requirements
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit dc8ef3508b0b3f902772eed947e5861720036dab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Feb 2 23:23:54 2017 +0100
+
+    tests: Thrasher: eliminate a race between kill_osd and __init__
+
+    If Thrasher.__init__() spawns the do_thrash thread before initializing the
+    ceph_objectstore_tool property, do_thrash races with the rest
+    of Thrasher.__init__() and in some cases do_thrash can call kill_osd() before
+    Trasher.__init__() progresses much further. This can lead to an exception
+    ("AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'")
+    being thrown in kill_osd().
+
+    This commit eliminates the race by making sure the ceph_objectstore_tool
+    attribute is initialized before the do_thrash thread is spawned.
+
+    Fixes: http://tracker.ceph.com/issues/18799
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b519d38fb1967628ad8a1c46fcfb3f984de58790)
+
+commit 7d9ef63d65aad40ce1f2f9f324d68c5fa9d77369
+Merge: 033fdb1406 ab8558e05e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Feb 21 16:16:26 2017 +0200
+
+    Merge pull request #13201 from dillaman/wip-18703-kraken
+
+    kraken: librbd: prevent self-blacklisting during break lock
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 033fdb14064d2a5ae9078944ab5ae8938a583d81
+Merge: f65754e312 cc04659798
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Feb 21 16:15:35 2017 +0200
+
+    Merge pull request #13102 from dillaman/wip-18668
+
+    kraken: test: use librados API to retrieve config params
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6f06cf80a7ff7407af40f46a0aefb4b88f926390
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 18 16:23:49 2017 +0800
+
+    mds: fix incorrect assertion in Server::_dir_is_nonempty()
+
+    when filelock is in XLOCKDONE state. client of xlocker can rdlock
+    the filelock. In that case, only client of xlocker can read the lock.
+
+    Fixes: http://tracker.ceph.com/issues/18578
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fe4ab52b30079a785be053a9fd0197d6990737fe)
+
+commit 914d8a6109f7c2125080104bd4bf463c45833c3c
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 1bc9cfff42237b572e90e976697b1ac5faafad4a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Feb 14 15:00:09 2017 -0500
+
+    osd/PG: restrict want_acting to up+acting on recovery completion
+
+    On recovery completion we recalculate want_acting to see if we
+    should add recently backfilled osds into acting.  However, at
+    this point we may have gotten infos from others OSDs outside
+    of up/acting that could be used for want_acting.  We currently
+    assert that only up/acting osds are used in
+    PG::RecoveryState::Active::react(const AdvMap&), so we must
+    restrict want_acting to up/acting here.
+
+    We could remove this restriction, but it would mean
+
+    1) checking on every map change that want_acting hasn't been
+    invalidated, and if so, recalculating want_acting and requesting
+    a new pg_temp.  Also, presumably
+
+    2) on each new info, checking whether we can construct a better
+    want_acting, and if so, doing it.
+
+    That would be a good thing, but is a more complicated change.  In
+    reality this case comes up very rarely, so simply make our
+    post-recovery want_acting calculation limit itself to up+acting.
+
+    See 1db67c443d84dc5d1ff53cc820fdfd4a2128b680 for the assertion.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f2dee9aa48a00a7f2f809cd4d20e98df771da81)
+
+commit e2074cef713a973f7e8fe23055e6fa27cae50e42
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 3 20:40:00 2017 +0800
+
+    ceph-disk: convert none str to str before printing it
+
+    Error('somethings goes wrong', e) is thrown if exception `e` is caught
+    in ceph-disk, where e is not a string. so we can not just concat it in
+    Error's __str__(). so cast it to str before doing so.
+
+    introduced by d0e29c7
+
+    Fixes: http://tracker.ceph.com/issues/18371
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e0dd1e7df43a3be589d17878714756a22052d8e)
+
+commit 177141ff1745e93996875b7d8d72af6f00307731
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 18 17:02:54 2017 -0600
+
+    mon/OSDMonitor: make 'osd crush move ...' work on osds
+
+    Currently it only allows you to move buckets, which is annoying and much
+    less useful.  To move an OSD you need to use create-or-move, which is
+    harder to use.
+
+    Fixes: http://tracker.ceph.com/issues/18587
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47956475dea8bb8e07331dd76344a60b776b5158)
+
+commit 8c2f590338dec75dabfb1e9ae0d2c516c91b2636
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jan 25 12:39:40 2017 +0100
+
+    systemd: Start OSDs after MONs
+
+    Currently, we start/stop OSDs and MONs simultaneously. This may cause
+    problems especially when we are shutting down the system. Once the mon
+    goes down it causes a re-election and the MONs can miss the message
+    from the OSD that is going down.
+
+    Resolves: http://tracker.ceph.com/issues/18516
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7f4acf45dd0d86e7d9992a8c30e5876fb57b1914)
+
+commit 33d9a50dec40346511f8480061dadab2a30174cc
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Feb 7 16:47:45 2017 +0400
+
+    ceph-osd: --flush-journal: sporadic segfaults on exit
+
+    FileStore holds a number of recources like op thread pool and work
+    queue, key/value DB threads, etc. These should be properly stopped
+    (released) before exiting to avoid segfaults on exit.
+
+    Note: more code paths (mkfs, dump_journal, etc) need similar fixes,
+    these will be submitted as separate patches.
+
+    Fixes: http://tracker.ceph.com/issues/18820
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 00184814c156f6194a6ba4b696073ca1c18a3f8f)
+
+commit b464d1a5719ef36410a38cb31e8c4aab802732a6
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 7475e442143de0f0b1919ec8dab8b6d2446eb12b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Feb 4 15:27:11 2017 +0100
+
+    common: possible lockdep false alarm for ThreadPool lock
+
+    Fixes: http://tracker.ceph.com/issues/18819
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8677dea4cd462d2141da28623a82b208cbc926f6)
+
+commit f65754e312f11823c5bb7e9fe24655f3fd68bcd2
+Merge: 21b2aa49b3 476b535f85
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 13 08:19:36 2017 +0100
+
+    Merge pull request #13330 from smithfarm/wip-18870-kraken
+
+    kraken: tests: SUSE yaml facets in qa/distros/all are out of date
+
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 21b2aa49b35bbbbd87ad3b3f27920e79c569f17a
+Merge: 958a4c9cc1 d819ec7198
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 10 15:41:06 2017 -0800
+
+    Merge pull request #12984 from ceph/wip-cherry-pick-4tamil
+
+    qa: Wip cherry pick 4tamil ceph-ansible
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 958a4c9cc135c34f18aa83a62b726bcefbb511b7
+Merge: b2955f4cc5 c968ed34b1
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Feb 9 22:48:01 2017 +0800
+
+    Merge pull request #13239 from smithfarm/wip-18805-kraken
+
+    kraken: tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+
+    Reviewed-by: Kefu Chai <kefu@redhat.com>
+
+commit 476b535f85c5b9a8bcd294e1b1629849efe9e391
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 21:23:54 2017 +0100
+
+    tests: drop buildpackages.py
+
+    The buildpackages suite has been moved to teuthology. This cleans up a file
+    that was left behind by https://github.com/ceph/ceph/pull/13297
+
+    Fixes: http://tracker.ceph.com/issues/18846
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6b7443fb50c117ee7f20d53bbc7530bb0eb7ebd5)
+
+commit ad456bfa2bab14032593c76b5a7f7434381bd1e8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 15:27:00 2017 +0100
+
+    tests: update SUSE yaml facets in qa/distros/all
+
+    Fixes: http://tracker.ceph.com/issues/18856
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0bd56e871a8549d4b0b1211f09dad2d1120fb606)
+
+commit b2955f4cc5506224d75e64c202392ee9a39fb7bf
+Merge: ce8edcfed6 d24ddec58e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 8 18:50:52 2017 +0100
+
+    Merge pull request #13298 from dachary/wip-18849-kraken
+
+    kraken: tests: remove qa/suites/buildpackages
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit d24ddec58e0086ce6a5f08feb6a96c4022f13a4a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 7 18:33:29 2017 +0100
+
+    buildpackages: remove because it does not belong
+
+    It should live in teuthology, not in Ceph. And it is currently broken:
+    there is no need to keep it around.
+
+    Fixes: http://tracker.ceph.com/issues/18846
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5a43f8d57925da227c95480501ceec10a29395d8)
+
+commit c968ed34b100baeb3a6a3245ad39c2a2d6fab04e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 6 18:43:49 2017 +0100
+
+    tests: fix regression in qa/tasks/ceph_master.py
+
+    https://github.com/ceph/ceph/pull/13194 introduced a regression:
+
+    2017-02-06T16:14:23.162 INFO:tasks.thrashosds.thrasher:Traceback (most recent call last):
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 722, in wrapper
+        return func(self)
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 839, in do_thrash
+        self.choose_action()()
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 305, in kill_osd
+        output = proc.stderr.getvalue()
+    AttributeError: 'NoneType' object has no attribute 'getvalue'
+
+    This is because the original patch failed to pass "stderr=StringIO()" to run().
+
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit db2582e25e390fcaf75952eb59a73dcff643f49c)
+
+commit d8c0141a7506cd38d385270f5a1cf315d808974e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 30 17:28:59 2016 -0500
+
+    osd/PG: publish PG stats when backfill-related states change
+
+    These frequently get flushed because other updates
+    happen, but we should explicitly ensure that the mon
+    sees these state changes.
+
+    Fixes: http://tracker.ceph.com/issues/18369
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4adeb7500a113097fdd717ada0231f68badafbb)
+
+commit 1d25327b00f01e987dec022412edb95040d69a42
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 26 14:22:53 2017 -0500
+
+    os/bluestore: fix statfs to not include DB partition in free space
+
+    If we report the DB space as vailable, ceph thinks the OSD can store more
+    data and will not mark the cluster as full as easily.  And in reality, we
+    can't actually store data in this space--only metadata.  Avoid the problem
+    by not reporting it as available.
+
+    Fixes: http://tracker.ceph.com/issues/18599
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c66d5babb1e283869ba0f1f59029bead5ca5f37d)
+
+commit 8ea14ce9ce185204f6906b3919f7fac1419a26a9
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Jan 12 14:55:26 2017 -0700
+
+    Add openstack requirements to smoke suite
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit fe9b7552d116b150f178c4cc778fa27cd8d956d1)
+
+commit af2af7015adb0e6617da619ffe89e70e0904fcec
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 17:38:05 2017 -0500
+
+    osd: do not send ENXIO on misdirected op by default
+
+    In practice this tends to get bubbled up the stack as an error on
+    the caller, and they usually do not handle it properly.  For example,
+    with librbd, this turns into EIO and break the VM.
+
+    Instead, this will manifest as a hung op on the client.  That is
+    also not ideal, but given that the root cause here is generally a
+    bug, it's not clear what else would be better.
+
+    We already log an error in the cluster log, so teuthology runs will
+    continue to fail.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 923e7f5ce5ed437af15e178299a61029ff48e4a2)
+
+    # Conflicts:
+    #       PendingReleaseNotes
+
+commit 85bda6e263400118830aa40a889bdb695d321b58
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Thu Dec 22 21:00:41 2016 -0500
+
+    librbd: don't remove an image w/ incompatible features
+
+    Fixes: http://tracker.ceph.com/issues/18315
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit f76127b5e617923d14adb62bfb836a635c14f209)
+
+commit d96ae9ead2d2f58180e533a63c640098d5134047
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 11:55:00 2017 -0500
+
+    osdc: cache should ignore error bhs during trim
+
+    A read error (such as injecting a timeout into an OSD op) might result
+    in a bh in an error state. These should be trimable by the cache.
+
+    Fixes: http://tracker.ceph.com/issues/18436
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5910ed9de9856b5821488a1836487bbbd3d6460e)
+
+commit 9620088542f80efb97960b4d9d51876ff1784aa9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 31 00:46:22 2017 +0100
+
+    tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 046e873026c59e733f1844b28ffdc030cbe57b36)
+
+commit baa42b6d7c616e4d3518416fcb86fef5c12a82db
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Jan 31 16:07:18 2017 +0100
+
+    OSD: allow client throttler to be adjusted on-fly, without restart
+
+    This patch allows the osd_client_message_cap and
+    osd_client_message_size_cap to be adjusted on-fly, using admin socket
+    functionality.
+
+    Fixes: http://tracker.ceph.com/issues/18791
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 64c309d7e18a975931b526e6f5d6f610c3a0d632)
+
+commit ab8558e05e765683691799acc4cfb2280cd537b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 25 14:45:56 2017 -0500
+
+    librbd: prevent self-blacklisting during break lock
+
+    Fixes: http://tracker.ceph.com/issues/18666
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5c590acaec4dd66a9a8c3aa0ec8ab904dd350216)
+
+    Conflicts:
+            src/librbd/exclusive_lock/BreakRequest.cc: managed lock refactor
+            src/test/librbd/exclusive_lock/test_mock_BreakRequest.cc: managed lock refactor
+
+commit 16b2fd00ecf7d7fcc7590f0ca030ff33a32f2196
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+
+    Fixes: #18635
+
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit 29f55d7dd3f35423294fa63cd1e3ccb18e6f7449
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:32:55 2017 +0100
+
+    rgw_admin: read master log shards from master's current period
+
+    Also make the sync output look similar to the output of data sync
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+    (cherry picked from commit cc306c506ca6607223cb89cd388f8e18673c4fe2)
+
+commit 0aebe210079e456c23e857e9c684495813151726
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:29:47 2017 +0100
+
+    rgw: allow getting master log shards info on specified period
+
+    This is needed for rgw admin's sync status or else we end up always
+    publishing that we're behind since we are always checking against
+    master's first period to sync from
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 063c949d4409a18a22b64791d497e20f7473bc01)
+
+commit 17d68c22da7c3eac336f5ea8ecac13b5e8d75568
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jan 12 22:09:01 2017 +0100
+
+    rgw_admin: get master's period from store's current period info
+
+    This ensures that we get the current period in contrast to the admin log
+    which gets the master's earliest period.
+
+    Fixes: http://tracker.ceph.com/issues/18064
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4ca18df7198a9f0ded8b0100a70b5db7187c3de4)
+
+commit 478d0315cb8457ecfd5cb6f96c82d12e066c7c2f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Dec 25 12:36:34 2016 +0200
+
+    rgw: clear master_zonegroup when reseting RGWPeriodMap
+
+    Fixes: http://tracker.ceph.com/issues/17239
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit d8f42fe6be659c1d48bf04b30aa54ad616936145)
+
+commit bfc058d8f951d7694104433f709fc73940511b0a
+Author: Michal Koutný <mkoutny@suse.com>
+Date:   Wed Jan 18 20:15:29 2017 +0100
+
+    rgw: Use decoded URI when verifying TempURL
+
+    Instead of calliing url_decode directly, we reuse s->decoded_uri that is
+    initialized in RGWREST::preprocess().
+
+    Fixes: http://tracker.ceph.com/issues/18590
+    Signed-off-by: Michal Koutný <mkoutny@suse.com>
+    (cherry picked from commit 4e1318f4dcbfd64c3ec94f4addf6e38ddd6c013a)
+
+commit deb6d2c64c432869caf8747a75ed2e555acc772b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jan 10 15:11:19 2017 +0000
+
+    librbd: allow to open an image without opening parent image
+
+    Fixes: http://tracker.ceph.com/issues/18325
+
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 61af1c25015de087a2423811548d975dd7d430b4)
+
+    Conflicts:
+      src/librbd/exclusive_lock/PostAcquireRequest.cc - deleted, does not
+                                                        exist in kraken
+      src/librbd/librbd.cc - removed rbd_group_* functions that don't exist
+                             in kraken
+
+commit 35b13c6687026f08c100a28f2775d6e772a0e734
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 29 12:08:28 2016 -0500
+
+    mon/OSDMonitor: set last_force_op_resend on overlay pool too
+
+    We currently set the last_force_op_resend field on the
+    base pool when we set or clear the overlay.  Set it on
+    the cache/overlay pool too.  The Objecter should resend
+    even with a change only to the base pool, but the OSD
+    needs to see the change on the overlay pool to correctly
+    discard the op.
+
+    Fixes: http://tracker.ceph.com/issues/18366
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08c3618771b852840aa88cff1ca98d980d802941)
+
+commit ce8edcfed6cd908779efd229202eab1232d16f1c
+Merge: 16fc6d8a2a 7db1bf762c
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:33:46 2017 +0100
+
+    Merge pull request #13024 from SUSE/wip-18604-kraken
+
+    kraken: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+commit 16fc6d8a2af89bb8a620298729d3951ef32945f7
+Merge: f36efa76d8 ed82de11f5
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:31:53 2017 +0100
+
+    Merge pull request #12813 from SUSE/wip-18439-kraken
+
+    kraken: fuse: TestVolumeClient.test_evict_client failure creating pidfile
+
+commit f36efa76d83205783ac5931ea678357ebddc62f6
+Merge: 08b560c51b 61ca8fc351
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:29:55 2017 +0100
+
+    Merge pull request #12951 from jcsp/wip-18361-kraken
+
+    kraken: client: populate metadata during mount
+
+commit 08b560c51b4218697b922cb208da2708f8c50021
+Merge: d50af9e948 8d0eb6e287
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:29:00 2017 +0100
+
+    Merge pull request #13030 from SUSE/wip-18612-kraken
+
+    kraken: client: segfault on ceph_rmdir path /
+
+commit d50af9e9488501778d1a82d32ba4042ecb5fca54
+Merge: 0fbf923290 8b79964696
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:27:57 2017 +0100
+
+    Merge pull request #13028 from SUSE/wip-18531-kraken
+
+    kraken: speed up readdir by skipping unwanted dn
+
+commit 0fbf9232903b8e7d7d93c91ea0dafcb4ff8f3c15
+Merge: e4d348b3d8 1ac9886896
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:25:34 2017 +0100
+
+    Merge pull request #12835 from SUSE/wip-18463-kraken
+
+    kraken: Decode errors on backtrace will crash MDS
+
+commit 29c8c3fb861a7153a13e3f56aaa7772c9116c751
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 11:16:47 2017 +0800
+
+    mds: finish clientreplay requests before requesting active state
+
+    All clientreplay requests' finish contexts should be executed
+    before MDCache::export_remaining_imported_caps(). Otherwise
+    MDCache::try_reconnect_cap() may fail to reconnect client caps.
+
+    Fixes: http://tracker.ceph.com/issues/18461
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 80dae314ee90e79d60e2cfee301e43a435c10801)
+
+commit 2b7083f8dd4c8f281f6f8ccbd13f06f08de6bcc1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 24 16:35:23 2017 -0800
+
+    Revert "PrimaryLogPG::failed_push: update missing as well"
+
+    This reverts commit dd48b972afde2dfa9ab1a6942c7961750222986d.
+
+    Fixes: http://tracker.ceph.com/issues/18659
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit cc046597983bd491cc66081cc33d9046264fe24b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 24 09:24:52 2017 -0500
+
+    librbd: improve debug logging for lock / watch state machines
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c5976ac36751e15fd81c67945a2c6d049dc7b316
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 23 21:24:41 2017 -0500
+
+    test: use librados API to retrieve config params
+
+    The CephContext object is not ABI-stable, so it is necessary to
+    use the "conf_get" librados methods to safely retrieve a setting.
+
+    Fixes: http://tracker.ceph.com/issues/18617
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ad40645ac3948f8341e9a80ce8aff8ac5b9ad11)
+
+    Conflicts:
+            src/test/librbd/test_librbd.cc: trivial resolution
+
+commit e4d348b3d850465ae25d7b3cbc8f21fac20ae478
+Merge: 03458fae64 f583485a4e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Jan 23 20:46:55 2017 +0100
+
+    Merge pull request #13006 from rzarzynski/wip-rgw-18476-kraken
+
+    kraken: rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT.
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f583485a4eacdf489ce00f93cd49dc147bfdb5f9
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jan 10 12:09:50 2017 +0100
+
+    rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT.
+
+    Before this patch the flag was wrongly handled in the Swift API
+    implementation. In rare conditions this might result in setting
+    req_state::system_request.
+
+    This may happen only if both of those conditions are fulfilled:
+     * RadosGW is running in a multi-site configuration (at least
+       one user with the system flag turned on is present),
+     * the "rgw_swift_account_in_url" configurable has been switched
+       to true. The value is false by default and our documentation
+       doesn't actually mention about the option.
+
+    The issue doesn't affect Jewel nor any previous release.
+
+    Fixes: http://tracker.ceph.com/issues/18476
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 8dac93392b6679c3ad9bb28ea66f6bb8c7be511d)
+
+commit 03458fae64df24d58426e5b3274dd1981ef42fc6
+Merge: 61b1beef1d dca2265c41
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 23 19:29:57 2017 +0100
+
+    Merge pull request #13044 from SUSE/wip-18571-kraken
+
+    kraken: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7db1bf762c5503933bbbb266e14fc1bd69120dd8
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+
+commit 0e0d149895198ee74cff85353eabf19aa4677258
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Jan 2 13:20:10 2017 +0800
+
+    mon: do not send duplicated osdmap msg to not sync'ed osd
+
+    prior to this change:
+    a peon may forward the pgstats to leader, and record it locally, but leader will
+    check if osd has the latest map before process, if not, will use a route op to
+    indicate peon to send it, then poen will delete routed op when fininaly send
+    out which make peon cannot send pgstatack when leader has processed the
+    pgstat update. so osd will always track it util reach a threshold block pgstats
+    sending, at worst, reopen mon session.
+    also, both leader and peon will send out the osdmap message to the osd.
+
+    after this change:
+    only the peon will send out the osdmap message. and the pgstatack message
+    will be routed to the osd as expected. so the osd will not keep track of the
+    "acked" pg stats in its queue forever before times out.
+
+    Fixes: http://tracker.ceph.com/issues/18458
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit 57274488c072ec6912b700288ce5b1ea8372d162)
+
+commit dca2265c41bf035855b30d3279e1ec5726d74ffc
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Fri Jan 13 16:57:45 2017 +0530
+
+    Doc: Fixes Python Swift client commands
+
+    Fixes: http://tracker.ceph.com/issues/17746
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 8c79959557d60f619adf1a3ed1b5bd1112ceaabb)
+
+commit 8d0eb6e2870593b94cb71f29bcac7a1f422cd101
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Thu Jan 12 21:22:20 2017 +0000
+
+    client/Client.cc: prevent segfaulting
+
+    The segfaulting in the rmdir function is caused by calling
+    filepath::last_dentry() function.
+    last_dentry() function assumes that the bits vector has always at
+    least one element, which is not the case for the the filepath object
+    created with "/" input.
+    This commit also fixes other functions affected by this bug:
+    link, unlink, rename, mkdir, mknod and symlink.
+
+    Fixes: http://tracker.ceph.com/issues/9935
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 6ed7f2364ae5507bab14c60b582929aa7b0ba400)
+
+commit 61b1beef1dc4802c32367fc71968101a09042c15
+Merge: f223e27eeb 6206e1998a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 20 11:14:24 2017 -0600
+
+    Merge pull request #13011 from liewegas/wip-18595-kraken
+
+    os/bluestore: fix Allocator::allocate() int truncation
+
+    Reviewed-by: Igor Fedotov <ifedotov@mirantis.com>
+
+commit 8b7996469652d01143806e1e815894afde3f426b
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Jan 10 19:11:08 2017 -0700
+
+    mds/server: skip unwanted dn in handle_client_readdir
+
+    We can skip unwanted dn which  < (offset_key, snap) via map.lower_bound, rather than
+    iterate across them.
+
+    Previously we iterate and skip dn which < (offset_key, dn->last), as dn->last >= snap
+     means (offset_key, dn->last) >= (offset_key, snap), and such iterate_and_skip logic
+    still keep, so this commit doesnt change code logic but an optimization.
+
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 52fe52baf920c672ac7f63a3087dcd31137891b6)
+
+commit ecb459f0837dd05d2802320e7fac3246025e6c13
+Author: Samuel Matzek <smatzek@us.ibm.com>
+Date:   Mon Jan 16 11:11:31 2017 -0600
+
+    Ceph-disk to use correct user in check_journal_req
+
+    The ceph-disk tool calls ceph-osd to check the journal requirements
+    using OSD id 0. This creates a log file for osd-0 on the system
+    using the current user/group for file permissions.  When ceph-disk
+    is run as root this makes the file owned by root which makes
+    the osd daemon for osd.0 unable to write to its own log file.
+
+    This commit changes the journal reqs calls of ceph-osd to pass
+    the ceph user and group so ceph-osd creates the log file with the
+    appropriate permissions.
+
+    Fixes: http://tracker.ceph.com/issues/18538
+
+    Signed-off-by: Samuel Matzek <smatzek@us.ibm.com>
+    (cherry picked from commit bcf7514bf53693ec61e482341787c80494589faf)
+
+commit 6206e1998a1636f6339ef2f1bd2d67a69cd3abf0
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 19 19:51:45 2017 -0600
+
+    os/bluestore: fix Allocator::allocate() int truncation
+
+    An allocation of size 0x80000000 gets truncated to 0.  Larger values can
+    break things in similar ways.
+
+    Introduced in 5ab034345d7320fbc86a2133c0c29ec1aca4b71a.
+
+    Fixes: http://tracker.ceph.com/issues/18595
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e19aa8484632ac7d83aa5dc868a1fe4dc167d9b9)
diff --git a/doc/changelog/v12.2.1.txt b/doc/changelog/v12.2.1.txt
new file mode 100644
index 000000000..dda1d4b55
--- /dev/null
+++ b/doc/changelog/v12.2.1.txt
@@ -0,0 +1,3737 @@
+commit 3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Sep 26 16:27:07 2017 +0000
+
+    12.2.1
+
+commit bf5f5ec7cf0e06125515866acedcc04c393f90b9
+Merge: 63ce514631 d1a6631fc4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 11:53:56 2017 -0400
+
+    Merge pull request #17970 from idryomov/wip-pr-17346-luminous
+    
+    luminous: qa/suites/krbd: rbd_xfstests job overhaul
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d1a6631fc4f7eaf563377f52dcc6fdb5833c0dd1
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Aug 18 17:49:03 2017 +0200
+
+    qa/suites/krbd: new rbd_xfstests job definition
+    
+    ext4 seems to be a better choice for our purposes -- less test churn,
+    rather small and reliable exclude list.
+    
+    All excluded tests but generic/050 fail with no krbd in the mix, most
+    have popped up on linux-ext4 list at least once.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6647c65ebe0688283cc8b9da9f638a4d6ffb75de)
+
+commit 902422c71d2e2322fa05dca2cf8d71e49033005c
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu Aug 17 15:35:42 2017 +0200
+
+    qa/tasks/rbd.xfstests: take exclude list from yaml
+    
+    Different filesystems (and further, different configurations of the
+    same filesystem) need different exclude lists.  Hard coding the list in
+    a wrapper script is inflexible.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 01914be5a3b35479037054293b85d96596cb870b)
+    
+    Conflicts:
+            qa/run_xfstests_krbd.sh [ trivial, caused by commit
+              3aae5ca6fda8 ("scripts: fix bash path in shebangs") ]
+
+commit 49284c671b335887244fcd06a3667952068cf5f2
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: quit building xfstests on test nodes
+    
+    xfstests is a pain to build on trusty, xenial and centos7 with a single
+    script.  It is also very sensitive to dependencies, which again need to
+    be managed on all those distros -- different sets of supported commands
+    and switches, some versions have known bugs, etc.
+    
+    Download a pre-built, statically linked tarball and use it instead.
+    The tarball was generated using xfstests-bld by Ted Ts'o, with a number
+    of tweaks by myself (mostly concerning the build environment).
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 1a982fea53c30fb246c2ab28822af6a69165ffea)
+
+commit e318dbd19883118885e8eb6c7375d0ca8a3699f6
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: drop *_MKFS_OPTIONS variables
+    
+    AFAICT ./check doesn't query EXT4_MKFS_OPTIONS or BTRFS_MKFS_OPTIONS,
+    We don't need anything special for xfs, so remove all of them to avoid
+    confusion.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 7a0e45046d556c6624e0225ff006c31d8dbae2a8)
+
+commit b5aeb7921d86452b5b19ac144685b3a199707751
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: kill FS_TYPE variable
+    
+    ./check expects FSTYP, so use that throughout.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ca99f1e8406c14e5899d393ab0bb503061c47b9b)
+
+commit 63ce514631ab0106ebd242f0ece43409dc83f479
+Merge: 9f8d66ebc5 b335066f81
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 22:40:05 2017 +0200
+
+    Merge pull request #17945 from batrick/i21540
+    
+    luminous: qa whitelist fixes
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9f8d66ebc5fb9ad48bdca4c56d54618a7a592415
+Merge: 35a23b861a 1dda4227a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 22:32:17 2017 +0200
+
+    Merge pull request #17821 from smithfarm/wip-21472-luminous
+    
+    luminous: tests: kcephfs: ignorable MDS cache too large warning
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1dda4227a5c8e2953c645c634e0b77dc6a0515a3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 28 11:47:29 2017 -0700
+
+    qa: whitelist expected MDS_CACHE_OVERSIZED
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 71f0066f6ec32be18abc0c220a49e203ac3b9786)
+    
+    Conflicts:
+       qa/suites/kcephfs/recovery/tasks/client-limits.yaml - fix
+          already applied by some other commit
+
+commit b335066f81a0f990fb87b84f69c34dda037ba9cf
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 28 11:52:36 2017 -0700
+
+    qa: whitelist expected rstat warning
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit c53d091e5110011866aa85495ff4e5abe0442d71)
+
+commit 35a23b861a037ca0e0bc77952c413b46e95141d4
+Merge: b6f38c4fcd d94b3ca2ae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 15:43:06 2017 +0200
+
+    Merge pull request #17922 from batrick/kcephfs-backports
+    
+    luminous: qa: kcephfs whitelist fixes
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit b6f38c4fcd0643fd24644bf0d0ad249112727a5e
+Merge: 793b89dc80 16c5e4f0c3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 13:25:59 2017 -0700
+
+    Merge pull request #17855 from smithfarm/wip-21490-luminous
+    
+    luminous: tests: test_rebuild_simple_altpool triggers MDS assertion
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 16c5e4f0c3f365f22e5cd79021f55c2927347a26
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 20 15:44:40 2017 -0700
+
+    qa: get config only on running MDS
+    
+    Fixes: http://tracker.ceph.com/issues/21466
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8a535d9c72965c79692dccc6ff418b46fb86c3eb)
+
+commit d94b3ca2ae04ff05937cb48a81901187212370a6
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:44:43 2017 -0700
+
+    qa: whitelist cache too large in client-limits
+    
+    Fixes: http://tracker.ceph.com/issues/21510
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 87039eb30a25158a192ab7099fa81fbb9ce02953)
+
+commit 018331f26c079c5d728a7009db24560e00407feb
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:42:06 2017 -0700
+
+    qa: whitelist failover log warnings
+    
+    Fixes: http://tracker.ceph.com/issues/21509
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 37a7737712f36422b5f035a0e02bab25ae2da890)
+
+commit dbad9455f956a6e7ebd731b68c50cbb96038542f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:38:59 2017 -0700
+
+    qa: ignore evicted client in client-recovery
+    
+    Fixes: http://tracker.ceph.com/issues/21508
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8ecef57bb10a83587f02ce087b657d303ae7c589)
+
+commit 793b89dc80c82a0f3027210d894dd1b4be6cc12b
+Merge: e8a6c0cfdf 4bb1d89373
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:43 2017 -0700
+
+    Merge PR #17878 into luminous
+    
+    * refs/remotes/upstream/pull/17878/head:
+            qa/cephfs: Fix error in test_filtered_df
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit e8a6c0cfdf1a5ac2505afa7ca1814a9f356647d5
+Merge: 6f6a98619e b0cacf2684
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:41 2017 -0700
+
+    Merge PR #17877 into luminous
+    
+    * refs/remotes/upstream/pull/17877/head:
+            client: Delete onsafe
+            client: add mountedness check inside client_lock
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 6f6a98619e9e121fea20d1604fa97122209a6a72
+Merge: 71b3f31cad 7a84f943ed
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:39 2017 -0700
+
+    Merge PR #17870 into luminous
+    
+    * refs/remotes/upstream/pull/17870/head:
+            mds: reset Formatter on error
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 71b3f31cad17a735ee0c278de04c88338b78b746
+Merge: 26beaa6667 69f0422310
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:37 2017 -0700
+
+    Merge PR #17854 into luminous
+    
+    * refs/remotes/upstream/pull/17854/head:
+            mds: void sending cap import message when inode is frozen
+            client: fix message order check in handle_cap_export()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 26beaa6667fbf95a371a2cf0d5130d8ff30327d9
+Merge: fc9043a2f5 97ab78548d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:35 2017 -0700
+
+    Merge PR #17852 into luminous
+    
+    * refs/remotes/upstream/pull/17852/head:
+            mds: check ongoing catter-gather process before capping log
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit fc9043a2f5ee3d641cb5710675fa43c44784fe87
+Merge: 5ae7723f80 13f0c9f534
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:33 2017 -0700
+
+    Merge PR #17822 into luminous
+    
+    * refs/remotes/upstream/pull/17822/head:
+            qa/cephfs: kill mount if it gets evicted by mds
+            qa/cephfs: fix test_evict_client
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 5ae7723f8028a888df93b71252ce3b31fc77da8d
+Merge: 653c332507 8d18150560
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:44:54 2017 -0700
+
+    Merge pull request #17817 from smithfarm/wip-21464-luminous
+    
+    luminous: qa: ignorable MDS_READ_ONLY warning
+    
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 653c332507c6642a4f8167d022ba452c91315c92
+Merge: 8681be5725 105f82d550
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:41:18 2017 -0700
+
+    Merge pull request #17879 from smithfarm/wip-21449-luminous
+    
+    luminous: tests: test_misc creates metadata pool with dummy object resulting in WRN: POOL_APP_NOT_ENABLED
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 8681be5725b7a9a1256e338600dd2050e1ddf2f7
+Merge: 09451bc47b f4b4467f8e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:40:46 2017 -0700
+
+    Merge pull request #17886 from smithfarm/wip-21486-luminous
+    
+    luminous: tests: [kcephfs] test_client_pin times out waiting for dentry release from kernel
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 09451bc47bd1d061c022d098d462206f1dac83ac
+Merge: 1700dc4f14 c902ebe7e8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:39:26 2017 -0700
+
+    Merge pull request #17887 from smithfarm/wip-21487-luminous
+    
+    luminous: cephfs: MDS rank add/remove log messages say wrong number of ranks
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 1700dc4f1409bbc8202b420deadfc494aaaaed74
+Merge: a931cf6812 f09786d294
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:37:42 2017 -0700
+
+    Merge pull request #17888 from smithfarm/wip-21488-luminous
+    
+    luminous: tests: failures from pjd fstest
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit a931cf68120d99661b785cc794e2b6965d621f06
+Merge: 0132a275aa 875f1c2519
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Sep 21 10:56:16 2017 -0500
+
+    Merge pull request #17812 from liewegas/wip-21410-luminous
+    
+    Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit f09786d2940ca5532f6f59464be4fdf29a4d75b3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sun Sep 17 16:51:04 2017 +0800
+
+    ceph: do link/rename semantic checks after srcdn is readable
+    
+    For hard link, source inode must not be directory. For rename,
+    types of source/destination inodes must match. If srcdn is replica
+    and we do these checks while it's not readble, it's possible that
+    wrong source inode is used in these checks.
+    
+    Fixes: http://tracker.ceph.com/issues/21383
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 46962b253563a867707e7c5d7887abf2060cc4d7)
+
+commit c902ebe7e89130f7aceaaf664074d74760b30926
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 13:16:40 2017 -0400
+
+    mon/MDSMonitor: fix clog msgs on MDS rank add/rm
+    
+    Because these were generated before the actual
+    change, the get_num_in values did not make sense.
+    
+    Fixes: http://tracker.ceph.com/issues/21421
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 511439b515414ae93d791f54a187e44d5fdffa7b)
+
+commit f4b4467f8edc30f33c56e2bc22fb509061d814b6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 19 14:38:13 2017 +0800
+
+    client: set client_try_dentry_invalidate to false by default
+    
+    By default, ceph-fuse uses side effect of 'dentry invalidation' to
+    trim kernel dcache if it runs on kernel < 3.18. The implemention of
+    kernel function d_invalidate() changed in 3.18 kernel, the method no
+    longer works for upstream kernel >= 3.18.
+    
+    RHEL 3.10 kernel includes backport of patches that change implemention
+    of d_invalidate(). So checking kernel version to decide if 'dentry
+    invalidation' method works is unreliable.
+    
+    Fixes: http://tracker.ceph.com/issues/21423
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 69adaabface27880dd6c8dbfdeeb06cf3f3d346a)
+
+commit 105f82d5502deb3cabb87d8fdd83a98e880fd6e2
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Tue Sep 12 13:22:09 2017 -0400
+
+    qa/tasks/cephfs: Whitelist POOL_APP_NOT_ENABLED for test_misc
+    
+    test_misc verifies that ceph fs new will not create a filesystem
+    on a pool that already contains objects. As part of the test, it
+    inserts a dummy object into a pool and then attempts to use it for
+    CephFS. This triggers POOL_APP_NOT_ENABLED. Setting the application
+    metadata for the pool (and having ceph fs new fail because of the
+    existing metadata) would then exercise a different failure case.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit eba01c48ca8b3113b1faf353990533a5ae39232c)
+
+commit 4bb1d893739e33fc35ae57ac0290606b5cf8b918
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Sep 13 14:00:20 2017 -0400
+
+    qa/cephfs: Fix error in test_filtered_df
+    
+    ceph df accounts for pool size, so there is no need to do it in the test.
+    
+    Fixes: http://tracker.ceph.com/issues/21381
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit b059cb6290d81df07bd8ec4e1ec9ef0be24892a2)
+
+commit b0cacf2684bfe8c55282bb7defd54053dc7fa2e8
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Thu Sep 7 12:41:40 2017 +0530
+
+    client: Delete onsafe
+    
+    Variable "onsafe" going out of scope leaks the storage it points to. This fixes the Coverity
+    Scan CID 1417473.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit e24a91e9189587c800ef5709c78f1bc0e0ac8419)
+
+commit f146b9f51da11e5698c7c4c9021635961b2790ad
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: add mountedness check inside client_lock
+    
+    Currently we check for mountedness in the high level wrappers, but those
+    checks are lockless. It's possible to have a call that races with
+    ceph_unmount(). It could pass one of the is_mounted() checks in the
+    wrapper, and then block on the client_lock while the unmount is actually
+    running. Eventually it picks up and runs after the unmount returns, with
+    questionable results -- possibly even a crash in some cases.
+    
+    For now, we can explain this away with a simple admonition that
+    applications should ensure that no calls are running when ceph_unmount
+    is called. In the future though, we may need to forcibly shut down the
+    mount when certain events occur (not returning a lease or delegation in
+    time, for instance).
+    
+    Sprinkle in a bunch of "unmounting" checks after taking the client_lock,
+    and simply have the functions return errors (or sensible values in some
+    cases) when the Client is being downed. With that, we ensure that this
+    sort of race can't occur, even when the unmount is not being driven by
+    userland. Note too that in some places I've replaced assertions in the
+    code with error returns, as that's nicer behavior for libraries.
+    
+    Note that this can't replace the ->is_mounted() checks in the lockless
+    wrappers as those are needed to determine whether the client pointer in
+    the ceph_mount_info is still valid. The admonition not to allow
+    ceph_unmount to race with other calls is therefore still necessary.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit efca340596ef4da2b254ff1c64ec4c462d7b95a5)
+
+commit 7a84f943edcdabd013b8ee4eb701e04062ad931a
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 5 14:44:49 2017 -0700
+
+    mds: reset Formatter on error
+    
+    This prevents partial results from concatenating with the error message.
+    
+    Fixes: http://tracker.ceph.com/issues/21252
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 2b2b3d2035a700a65c3c469eedae69ad31023cf6)
+
+commit 847cb76968bb225e4f3713cdfc0a5c38c4047e4e
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:48:42 2017 -0500
+
+    qa/cephfs: Refactor alternate pool test
+    
+    Remove the alternate pool recovery test from test_data_scan. Newer
+    commits will place the test in its own file.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 47318f8ac4391148a1394846dd77c8452b7cf5ae)
+
+commit 3342ba0c6853fda90cf2c8c9bec9d523c9269855
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:45:13 2017 -0500
+
+    qa/cephfs: test CephFS recovery pools
+    
+    Test recovering metadata in to a separate RADOS pool with
+    cephfs_data_scan and friends.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 6af2ae80d327e564ad17f2caeded9c8142becf29)
+
+commit 5ac721394770de88a6218dd3f9bedbdbc9982232
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:43:39 2017 -0500
+
+    qa/cephfs: support CephFS recovery pools
+    
+    Add support for testing recovery of CephFS metadata into an alternate
+    RADOS pool, useful as a disaster recovery mechanism that avoids
+    modifying the metadata in-place.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 8f9a25202093339afb5308051d354d3ae79c6b2d)
+
+commit 0f5f99c1de5484e7563b5c5c9fa481200e1d00c6
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 26 13:05:59 2017 -0400
+
+    qa/cephfs: Allow deferred fs creation
+    
+    Permit Filesystem objects to be created and settings modified before
+    calling Filesystem.create().
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 5fafc03cb9e6d5f39ce494375ea28c89b896e2f5)
+
+commit 47e485e5ccedce3977b30503e05b743e09fc7c77
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:41:11 2017 -0500
+
+    qa/ceph_test_case: support CephFS recovery pools
+    
+    Add support for testing recovery of CephFS metadata into an alternate
+    RADOS pool, useful as a disaster recovery mechanism that avoids
+    modifying the metadata in-place.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit c85562c94a80b8a18975b8d0ee6a7fbd932cf024)
+
+commit 69f042231056071dee16ed49aad95eaa3b0447f6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 29 11:35:56 2017 +0800
+
+    mds: void sending cap import message when inode is frozen
+    
+    To export an inode to other mds, mds need to:
+    
+    - Freeze the inode (stop issuing caps to clients)
+    - Flush client sessions (ensure client have received all cap messages)
+    - Send cap export message
+    
+    These steps guarantee that clients receive cap import/export messages
+    in proper order (In the case that inode gets exported servel times
+    within a short time)
+    
+    When inode is frozen, mds may have already flushed client sessions.
+    So mds shouldn't send cap import messages.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 48a9e695e58ac15f1e494977e0db7a164bb2fe98)
+
+commit 25def78ceb4cb17c97ae911203a62fb410cd9399
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 28 17:13:31 2017 +0800
+
+    client: fix message order check in handle_cap_export()
+    
+    If importer mds' cap already exists, but cap ID mismatches, client
+    should have received corresponding import message (the imported caps
+    got released later). Because cap ID does not change as long as client
+    holds the caps.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 3c31256a4addcc4eca30c62109e33f77224258c2)
+
+commit 97ab78548d26f980ead8dc5c986304d2cfd3568d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Sep 21 10:56:07 2017 +0800
+
+    mds: check ongoing catter-gather process before capping log
+    
+    When deactivating mds, MDLog::trim() may start scatter-gather
+    process on mdsdir inode. Locker::scatter_writebehind() submits
+    log entry. So mds should make sure there is no scatter-gather
+    before capping log.
+    
+    Fixes: http://tracker.ceph.com/issues/21467
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 0132a275aa86ca80bf1dcacd9905243257317e6c
+Merge: 1befcbebcb bc4095ee44
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 20 08:08:14 2017 -0700
+
+    Merge pull request #17829 from jdurgin/wip-21428-luminous
+    
+    osd/PrimaryLogPG: request osdmap update in the right block
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bc4095ee44f8c060a43470c29c08767517b4d749
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 20 01:18:35 2017 -0400
+
+    osd/PrimaryLogPG: request osdmap update in the right block
+    
+    Fixes: http://tracker.ceph.com/issues/21428
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit afc6624f768ea3c6e2d155122797db5cce8836f7)
+
+commit 13f0c9f534d0a9df362bb86ab546648fa046c104
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 16:34:40 2017 +0800
+
+    qa/cephfs: kill mount if it gets evicted by mds
+    
+    otherwise, teardown() hange at umount
+    
+    Fixes: http://tracker.ceph.com/issues/21275
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 98d86a075242f0c4576814db15cc3f8fd8e06017)
+
+commit 651280281ab1347fb7d0e3114cf87b74398dbc87
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 16:15:31 2017 +0800
+
+    qa/cephfs: fix test_evict_client
+    
+    executing mount_a.kill() twice, then executing mount_b.kill_cleanup()
+    twice do not make sense.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8433ced84733da86d545ccacc06ddea3a0282071)
+
+commit 1befcbebcbcedf398b782adfcc165038d13fc3da
+Merge: ddf84249fa a6ce7bfcf3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 19 16:45:40 2017 -0700
+
+    Merge pull request #17796 from jdurgin/wip-21428-luminous
+    
+    osd: request new map from PG when needed
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8d18150560bbf0c41f4d62b17b642449b3610660
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 5 17:09:32 2017 +0800
+
+    qa: whitelist expected MDS_READ_ONLY for kcephfs test
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 584241285f52fc420ebc4a6a6455497a3ed4d45a)
+
+commit a6ce7bfcf3ab00046f220a00ef2c283c3aa0bb31
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 22:31:10 2017 -0400
+
+    osd: request new map from PG when needed
+    
+    The fast dispatch refactor in 3cc48278bf0ee5c9535d04b60a661f988c50063b
+    eliminated the osdmap subscription in the ms_fast_dispatch path, which
+    meant ops could reach a PG without having the latest map. In a cluster
+    with few osdmap updates, where the monitor fails to send a new map to
+    an osd (it tries one random osd), this can result in indefinitely
+    blocked requests.
+    
+    Fix this by adding an OSDService mechanism for scheduling a new osdmap
+    subscription request.
+    
+    Fixes: http://tracker.ceph.com/issues/21428
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit dd33360e5a4ed06228af552edadee88e66c5aba9)
+
+commit 875f1c2519c15a94b4f4ccfa3ae0b0b191cfb1dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 16:52:28 2017 -0400
+
+    Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items"
+    
+    This reverts commit 09af9b8afb40cc8aa629501582a75e03edf0bf2e.
+    
+    We need to prevent duplicates in the final result.  For example, we
+    can currently take
+     [1,2,3] and apply [(1,2)] and get [2,2,3]
+    or
+     [1,2,3] and apply [(3,2)] and get [1,2,2]
+    
+    The rest of the system is not prepared to handle duplicates in the
+    result set like this.
+    
+    The reverted commit was intended to allow
+    
+     [1,2,3] and [(1,2),(2,1)] to get [2,1,3]
+    
+    to reorder primaries.  First, this bidirectional swap is hard to implement
+    in a way that also prevents dups.  For example,
+     [1,2,3] and [(1,4),(2,3),(3,4)] would give [4,3,4]
+    but would we just drop the last step we'd have [4,3,3] which
+    is also invalid, etc.  Simpler to just not handle bidirectional
+    swaps.  In practice, they are not needed: if you just want to choose
+    a different primary then use primary_affinity, or pg_upmap
+    (not pg_upmap_items).
+    
+    Fixes: http://tracker.ceph.com/issues/21410
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9c73305e3ad11177d58632eba6ece5d2c0e701da)
+
+commit ddf84249fa8a8ec3655c39bac5331ab81c0307b1
+Merge: d324411744 4c16e1b36b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 22:46:27 2017 +0200
+
+    Merge pull request #17724 from theanalyst/wip-21374-luminous
+    
+    luminous: incorrect erasure-code space in command ceph df
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit d32441174440b7e639658a9218d7004ebe604b62
+Merge: df73cdcdfe 381ed45b49
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Sep 15 13:35:36 2017 -0700
+
+    Merge pull request #17745 from tchaikov/wip-luminous-20529
+    
+    luminous: rocksdb,cmake:  build portable binaries
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit df73cdcdfec8263d5f1cd0af98d51928aa7659f3
+Merge: 24c5b9bfb7 e7158e8e41
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:58:05 2017 +0200
+
+    Merge pull request #17635 from smithfarm/wip-21342-luminous
+    
+    luminous: ceph mgr versions shows active mgr as Unknown
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 24c5b9bfb72461005585329f82a14b994c8546e4
+Merge: 57e798bc55 b10311dc73
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:57:54 2017 +0200
+
+    Merge pull request #17634 from smithfarm/wip-21341-luminous
+    
+    luminous: mon/OSDMonitor: deleting pool while pgs are being created leads to assert(p != pools.end) in update_creating_pgs()
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 57e798bc55ed5d1ae44d2279e2bba349abc88e76
+Merge: 9fc9577df0 fe2fe20100
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:57:13 2017 +0200
+
+    Merge pull request #17624 from tchaikov/wip-luminous-pr-17599
+    
+    luminous: encoding: reset optional<> if it is uninitialized
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 9fc9577df0c91f8ee3c55464e57f31fb20060853
+Merge: 0c28640ba0 91dd0fe479
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:55:41 2017 +0200
+
+    Merge pull request #17487 from smithfarm/wip-21231-luminous
+    
+    luminous: core: interval_set: optimize intersect_of insert operations
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 0c28640ba09cd1035ae5b4103833ef6bd172ae6c
+Merge: f52c6dcd60 24fe250bb5
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:49:37 2017 +0200
+
+    Merge pull request #17712 from batrick/bp21221
+    
+    luminous: MDCache::try_subtree_merge() may print N^2 lines of debug message
+    
+    Reviewed-By: Yan Zhen <zyan@redhat.com>
+
+commit f52c6dcd60ea352f8762d98e2b2c245abed5c0fc
+Merge: c9f0a6bd2e 9f65be2feb
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:45:04 2017 +0200
+
+    Merge pull request #17623 from liewegas/wip-pr-17572-luminous
+    
+    tools/ceph_objectstore_tool: fix 'dup' unable to duplicate meta PG
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit c9f0a6bd2ec55b308de28b8ddf1d1658887d263b
+Merge: ece893d1e8 80d848f069
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:43:22 2017 +0200
+
+    Merge pull request #17584 from jdurgin/wip-recovery-delete-encoding-luminous
+    
+    osd/PGBackend, MOSDPGRecoveryDelete[Reply]: handle incompatible encoding
+    
+    Reviewed-By: Sage Weil <sage@redhat.com>
+
+commit ece893d1e8ee7df9f0ee6413628aa2aeda311364
+Merge: dfc8a0fe47 eb6a0bf22e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:39:49 2017 +0200
+
+    Merge pull request #17686 from ukernel/luminous-21070
+    
+    luminous: mds: fix 'dirfrag end' check in Server::handle_client_readdir
+    
+    Reviewed-By: Patrick Donnelly <pdonelly@redhat.com>
+
+commit dfc8a0fe47b950155eab2bdbe98f42645c3f6140
+Merge: 0be1e868f1 a1be6c9647
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:38:36 2017 +0200
+
+    Merge pull request #17711 from batrick/bp20594
+    
+    luminous: mds: support limiting cache by memory
+    
+    Reviewed-By: Yan Zheng <zyan@redhat.com>
+
+commit 0be1e868f10601df3f293af58a173702db6efed0
+Merge: b0b4d07663 e7e3d8977b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:36:00 2017 +0200
+
+    Merge pull request #17714 from batrick/bp21222
+    
+    luminous: MDS: standby-replay mds should avoid initiating subtree export
+    
+    Reviewed-By: Yan, Zheng <zyan@redhat.com>
+
+commit b0b4d0766345ffcc552a841be16101f4b80dcf55
+Merge: c69673d955 73ef8de0bf
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:25:56 2017 +0200
+
+    Merge pull request #17570 from theanalyst/wip-luminous-21233
+    
+    luminous: rgw: memory leak in MetadataHandlers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit c69673d955d784ca580fc283228b881b30065ff6
+Merge: 671740533e ccb3c1edc0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:25:31 2017 +0200
+
+    Merge pull request #17642 from theanalyst/wip-21350-luminous
+    
+    luminous: rgw: data encryption sometimes fails to follow AWS settings
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 671740533e71ac742259d32b86102fa807ea7fee
+Merge: 87d3d3419e 665c3a6397
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:23:01 2017 +0200
+
+    Merge pull request #17674 from theanalyst/luminous-rgw-reshard-enable
+    
+    luminous: rgw: reshard enable
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 87d3d3419ebb9370aeb14496901c55c2a01fdf1e
+Merge: 4bb174db3a 2be6c63f1b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:22:05 2017 +0200
+
+    Merge pull request #17479 from smithfarm/wip-21139-luminous
+    
+    luminous: rgw: put lifecycle configuration fails if Prefix is not set
+    
+    Reviewed-By: Daniel Gryniewicz <dang@redhat.com>
+
+commit 381ed45b491fecd9254c2b8ee3da89064cfe8c1a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 15 10:30:16 2017 +0800
+
+    cmake: do not pass $SIMD_COMPILE_FLAGS to rocksdb cmake
+    
+    which enables SSE42 globally in rocksdb. and we will end up with a
+    binary not portable on non-SSE42 enabled machines.
+    
+    Fixes: http://tracker.ceph.com/issues/20529
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts:
+            this change is not cherry-picked from master. because the
+    PR targeting master (https://github.com/ceph/ceph/pull/17388) is
+    still pending on review. and the cmake changes is different if
+    we want to use a recent commit of rocksdb, as it's doing differently
+    in cmake to address the portability issues.
+
+commit 9216086795a6a9c7b33000d44bea8ec9c62d41bb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 15 10:23:47 2017 +0800
+
+    rocksdb: pick up portable fixes
+    
+    to pick up the the fix to disable SSE42 globally, and only enable it on
+    crc32c. this change is pushed to ceph/rocksdb:luminous.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+            this change is not cherry-picked from master, as the master PR
+    (https://github.com/ceph/ceph/pull/17388) is still pending on review.
+    and the latest rocksdb's cmake was revised to address the portability
+    issues. so the fix on ceph side is different if we want to use a
+    recent rocksdb's commit.
+
+commit 73ef8de0bf147a3ca670ab8a4b04ddaaf0ed5bfd
+Author: Luo Kexue <luo.kexue@zte.com.cn>
+Date:   Wed Aug 30 09:32:18 2017 +0800
+
+    rgw: fix resource leak in rgw_bucket.cc and rgw_user.cc
+    
+    Fixes: http://tracker.ceph.com/issues/21214
+    
+    Signed-off-by: Luo Kexue <luo.kexue@zte.com.cn>
+    (cherry picked from commit 1f23976adab74e25ce06941692a881521885c6ee)
+
+commit 34548611d2d0394499e907ff7dd4b43cf53cd2cc
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Aug 25 15:32:56 2017 +0530
+
+    rgw: add missing override in list_keys_init()
+    
+    Added missing override in list_keys_init().
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit bfc999f1db96293e6a2188922074031fa0b40ecc)
+
+commit 4bb174db3a3cc18b544d7282516289075763a1f3
+Merge: e8e4f1fd6d 98a53f929e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:32:28 2017 +0200
+
+    Merge pull request #17612 from smithfarm/wip-qemu-cleanup
+    
+    build/ops: rpm: do not create librbd.so.1 symlink in /usr/lib64/qemu
+    
+    Reviewed-By: Ken Dreyer <kdreyer@redhat.com>
+
+commit e8e4f1fd6d82540451be2896626e959e1b1bdaa8
+Merge: ad534c0d7a ce2a181834
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:31:40 2017 +0200
+
+    Merge pull request #17478 from smithfarm/wip-21138-luminous
+    
+    luminous: rgw: object copied from remote src acl permission become full-control issue
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit ad534c0d7adaa83b3b219592de5a7b1213cb6317
+Merge: f05328733a 07a15984c6
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:31:23 2017 +0200
+
+    Merge pull request #17445 from smithfarm/wip-21110-luminous
+    
+    luminous: rgw: send data-log list infinitely
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 4c16e1b36b60028e9b6877dea7abcfe872b2e753
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Wed Sep 6 11:50:17 2017 +0800
+
+    mon: incorrect MAX AVAIL in "ceph df"
+    
+    Fixes: http://tracker.ceph.com/issues/21243
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 365558571c59dd42cf0934e6c31c7b4bf2c65026)
+
+commit f05328733a04acf454d229f73d79015292bd5fdf
+Merge: 8d292af09f cac29c9618
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:33:29 2017 +0200
+
+    Merge pull request #17488 from smithfarm/wip-21241-luminous
+    
+    luminous: rgw: usage of --inconsistent-index should require user confirmation and print a warning
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 8d292af09f663a49d935011a7bebec096c77b04e
+Merge: 038dc170c7 24803b76c8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:28:16 2017 +0200
+
+    Merge pull request #17484 from smithfarm/wip-21184-luminous
+    
+    luminous: tests: NameError: global name 'name' is not defined
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 038dc170c7928a415b0d08536be64b9eebd17511
+Merge: 67e87bff24 39b3b132f2
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:26:12 2017 +0200
+
+    Merge pull request #17474 from smithfarm/wip-21114-luminous
+    
+    luminous: cephfs: tests: FS_DEGRADED spurious health warnings in some sub-suites
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 67e87bff24424a3e233a9a89bbdef1e1545c0df4
+Merge: ba746cd14d 25a651e100
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:25:51 2017 +0200
+
+    Merge pull request #17471 from smithfarm/wip-21107-luminous
+    
+    luminous: cephfs: client/mds has wrong check to clear S_ISGID on chown
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit e7e3d8977b4e46db7a370880d54094e7b7915826
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 11:27:18 2017 +0800
+
+    ceph: fix "stopping mds can't export subtree"
+    
+    caused by commit
+    790c34138f "Make standby-replay mds avoid initiating subtree export"
+    
+    Fixes: http://tracker.ceph.com/issues/21378
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit c294051987de1ff79540060a0563fa02c8f03f91)
+
+commit a0693c8ca4ea54950d44e9b5b2da0219d46ea5a6
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Mon Sep 4 21:49:26 2017 +0800
+
+    Make standby-replay mds not migrate empty subtree when trimming cache
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 178ca58fec2d820f61724d223addac1c4fa67086)
+
+commit 70e814cf60c533fe2e850e451a7607ed5ffe7ed9
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Mon Sep 4 18:03:30 2017 +0800
+
+    Make standby-replay mds avoid initiating subtree export
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 790c34138fbdf995c72e558057b9e64c882b522b)
+
+commit 24fe250bb5fbb8546f5fb280cca10162e3aabb2f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Sep 4 11:36:00 2017 -0700
+
+    mds: print subtrees only after merge
+    
+    During up:resolve, the MDS tries to merge each subtree with its parent. During
+    testing, QE found that many thousands of subtrees in a directory (made possible
+    using pins) would cause the MDS to spend minutes printing out subtree maps to
+    the debug log. This causes the heartbeat code to consider the MDS as stalled so
+    beacons are no longer sent to the mons resulting in the MDS being removed from
+    the rank.
+    
+    A more complete solution to this problem is to selectively print subtrees
+    relating to the operation (e.g. the subtree and its parents).
+    
+    Fixes: http://tracker.ceph.com/issues/21221
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1485783
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d0747a37fd06053b2206bb9a952f7ab77f0db2f0)
+
+commit a1be6c96470b843d46754ffda08c524a3c7b9a59
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 15:55:46 2017 +0800
+
+    mds: optimize MDCache::rejoin_scour_survivor_replicas()
+    
+    avoid iterating dentries if dirfrag is non-auth
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit d32a2376047bc1363b9f7a2ed09060ec84c78b97)
+
+commit fb3afeb1e7572cda45966fa925102767dcd2f1b7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 15:54:45 2017 +0800
+
+    mds: fix MDSCacheObject::clear_replica_map
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fd447405bef347da80ed5611331b211021b1628b)
+
+commit 179bf1831c6c4e88eb13ed689293a677ef5d2983
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Sep 11 15:21:52 2017 -0700
+
+    mds: support limiting cache by memory
+    
+    This introduces two config parameters:
+    
+        mds_cache_memory_limit: Sets the soft maximum of the cache to the given
+        byte count. (Like mds_cache_size, this doesn't actually limit the maximum
+        size of the cache. It just dictates the steady-state size.)
+    
+        mds_cache_reservation: This replaces mds_health_cache_threshold everywhere
+        except the Beacon heartbeat sent to the mons. The idea here is to specify a
+        reservation of memory (5% by default) for operations and the MDS tries to
+        always maintain that reservation. So, the MDS will recall caps from clients
+        when it begins dipping into its reservation of memory.
+    
+    mds_cache_size still limits the cache by Inode count but is now by-default 0
+    (i.e. unlimited). The new preferred way of specifying cache limits is by memory
+    size. The default is 1GB.
+    
+    Fixes: http://tracker.ceph.com/issues/20594
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1464976
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 06c94de584e6cd7d347bcdfb79d9fef4fed0d277)
+    
+     Conflicts:
+            PendingReleaseNotes
+            src/mds/MDCache.cc
+
+commit c4d393e6d0b4497e94ab88a8f61436c6271e2ab8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 7 21:01:31 2017 -0700
+
+    common: refactor of lru
+    
+    Avoids an unnecessary "max" size of the LRU which was used to calculate the
+    midpoint. Instead, just dynamically move the LRUObjects between top and bottom
+    on-the-fly.
+    
+    This change is necessary for a cache which which does not limit by the number
+    of objects but by some other metric. (In this case, memory.)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 12d615b3c546211b48ee75921f0e06371dd62dae)
+
+commit 009d3ab0bc3cb5248701a4383e58a86337ee4982
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 13:40:17 2017 -0700
+
+    mds: resolve unsigned coercion compiler warning
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0c2032c2876c3e2cd0d23ab883fc72375731cadf)
+
+commit e25881b1e8c02dd1262c5b2b41eb353b293e1e37
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 7 20:51:56 2017 -0700
+
+    common: use safer uint64_t for list size
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0ddd260a32e781ff6bfa49c1a01d6b2d80e5dcd3)
+
+commit f21d2fa7e57e18f368c2c62534eba2cc9d0c80fb
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Jul 30 16:09:32 2017 -0700
+
+    common: add bytes2str pretty print function
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 7fff24e10e9cacc94ec320b87e60c7735ff028b6)
+
+commit 97fdc68e029ed1567eff7c5d13a6c1c29bfe5d33
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 14:31:18 2017 -0700
+
+    mds: check if waiting is allocated before use
+    
+    This prevents accidental allocation of the map.
+    
+    Also, privatize the variable to protect from this in child classes.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 055020ce80b1f08d258b4c023bf5465ed7a46034)
+
+commit d50f2c735ac40b9ccfbd8d3423fd817f76de9c3c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 14:29:49 2017 -0700
+
+    mds: go back to compact_map for replicas
+    
+    Zheng observed that an alloc_ptr doesn't really work in this case since any
+    call to get_replicas() will cause the map to be allocated, nullifying the
+    benefit. Use a compact_map until a better solution can be written. (This means
+    that the map will be allocated outside the mempool.)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5d67b5cc57ac803e60c2921f554cd0507ed4848e)
+
+commit 2ef222a58c3801eaac5a6d52dda2de1ffe37407b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 17:21:54 2017 -0700
+
+    mds: use mempool for cache objects
+    
+    The purpose of this is to allow us to track memory usage by cached objects so
+    we can limit cache size based on memory available/allocated to the MDS.
+    
+    This commit is a first step: it adds CInode, CDir, and CDentry to the mempool
+    but not all of the containers in these classes (e.g. std::map). However,
+    MDSCacheObject has been changed to allocate its containers through the mempool
+    by converting compact_* containers to the std versions offered through mempool
+    via the new alloc_ptr.
+    
+    (A compact_* class simply wraps a pointer to the std:: version to reduce memory
+    usage of an object when the container is only occasionally used. The alloc_ptr
+    allows us to achieve the same thing explicitly with only a little handholding:
+    when all entries in the wrapped container are deleted, the caller must call
+    alloc_ptr.release().)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e035b64fcb0482c3318656e1680d683814f494fe)
+
+commit f26412883c390b78df132e32d99afa5a14c525f3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:06:23 2017 -0700
+
+    mds: cleanup replica_map access
+    
+    The gymnastics protecting the map failed as the code evolved. Just expose it
+    normally with a getter.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d1b6cadd6c1514aeacfbacec253305e1a715c387)
+
+commit 1bf3b1939ccdf30c4645009887c93fcb03d029e8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:10:14 2017 -0700
+
+    common: add alloc_ptr smart pointer
+    
+    This ptr is like a unique_ptr except it allocates the underlying object on
+    access. The idea being that we can save memory if the object is only needed
+    sometimes.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5fa557d2713558038af7579de8a3ca56d58b8d90)
+
+commit 44e206f351297fc9092577a9280fabdadd993cc6
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:04:04 2017 -0700
+
+    common: add warning on base class use of mempool
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit c0d0fa804ee0fa775ae18f9b7bde2317bb6af52d)
+
+commit 8c82de6a32bcbc8df0f6dadb7aacd7c41bcf77d8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:03:15 2017 -0700
+
+    common: use atomic uin64_t for counter
+    
+    Making this interface thread-safe...
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 59b5931a2f47f90fced779fb89e06868c739c688)
+
+commit ba746cd14ddd70a4f24a734f83ff9d276dd327d1
+Merge: 138acd38a6 21b9659cdd
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Sep 13 23:13:03 2017 +0200
+
+    Merge pull request #17646 from theanalyst/wip-luminous-multifixes
+    
+    luminous: rgw fixes for multisite tests in luminous
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit eb6a0bf22e5777b3977226d831380f5e68ae3739
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 28 11:24:13 2017 +0800
+
+    mds: fix 'dirfrag end' check in Server::handle_client_readdir
+    
+    The 'dirfrag end' check can be wrong when not reading from beginning
+    of dirfrag.
+    
+    Fixes: http://tracker.ceph.com/issues/21070
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 88332c0f2f0a7a99639d1de237d5bb68edddbe70)
+
+commit 138acd38a67bad303fb6804c1f5be0b4e5c4c47f
+Merge: 6753d98d90 e2dda02e02
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 12 16:04:54 2017 -0400
+
+    Merge pull request #17647 from dillaman/wip-21345
+    
+    luminous: qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2be6c63f1bf910562ecca8ffd70206b605f1fe07
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Sep 12 11:19:44 2017 +0800
+
+    rgw: LCFilter_S3::to_xml the second "prefix" missed "/"
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit dcef8555e1bf2656aeb5316c4b78f1facf2ae172)
+
+commit 21b9659cdd6c0dae63609ee58915fe1166487bbc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 30 11:16:25 2017 -0400
+
+    test/rgw: fix test_encrypted_object_sync for 3+ zones
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8762c4b29051b2b49bfd8c1e0ceaa1aee6417a74)
+
+commit 665c3a6397ed43d4dde0a0ad22483fc964ab61c8
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Sep 12 18:07:01 2017 +0200
+
+    luminous: rgw: re enable dynamic resharding
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 6753d98d90c65961e4abbea43f70acb938753f7a
+Merge: 4102a50c0d 64fc7cf105
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Sep 12 08:01:43 2017 -0400
+
+    Merge pull request #17653 from ceph/backport-wip-bz1478598
+    
+    luminous: ceph-volume allow filtering by `uuid`, do not require osd id
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 4102a50c0ddc969b7f58642f2f67b67e0f671b23
+Merge: 7ca6156003 63c9382eaf
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 13:02:35 2017 +0200
+
+    Merge pull request #17485 from smithfarm/wip-21185-luminous
+    
+    luminous: rgw: rgw_file:  incorrect lane lock behavior in evict_block()
+    
+    Reviewed-By: Orit Wasserman <oritwas@redhat.com>
+
+commit 7ca615600393a8337426818fbe0390117752f0ef
+Merge: ab9650b929 e171fa6a6d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 12:40:44 2017 +0200
+
+    Merge pull request #17476 from smithfarm/wip-21135-luminous
+    
+    luminous: rgw: bucket index sporadically reshards to 65521 shards
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit ab9650b929320320b6f88f31a39575e16d69c05d
+Merge: 31dbc795cd 18442e1585
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 11:06:05 2017 +0200
+
+    Merge pull request #17470 from smithfarm/wip-21104-luminous
+    
+    luminous: build/ops: ceph-fuse RPM should require fusermount
+    
+    Reviewed-By: Ken Dreyer <ktdreyer@redhat.com>
+    Reviewed-By: Kefu Chai <kchai@redhat.com>
+
+commit 31dbc795cd1b71f572f99ba8bf5936de19fa9fb8
+Merge: 39385a7512 17626e01d8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 11:05:14 2017 +0200
+
+    Merge pull request #17477 from smithfarm/wip-21137-luminous
+    
+    luminous: mgr: 500 error when attempting to view filesystem data
+    
+    Reviewed-By: John Spray <jcsp@redhat.com>
+
+commit 39385a7512112570c98c95b40c1215f6d4aa19d9
+Merge: 75db027e25 d57399c9da
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:09:01 2017 -0500
+
+    Merge pull request #17495 from smithfarm/wip-21235-luminous
+    
+    luminous: tests: thrashosds read error injection doesn't take live_osds into account
+
+commit 75db027e25b07e93f8cdf62316dd51ed6e305df2
+Merge: 975af9c2a4 ca7e48c62e
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:07:22 2017 -0500
+
+    Merge pull request #17483 from smithfarm/wip-21183-luminous
+    
+    luminous: mgr: Crash in MonCommandCompletion
+
+commit 975af9c2a48394c04852b142ea9d5ffa98b12db1
+Merge: b755445f84 8e9b52ee88
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:06:42 2017 -0500
+
+    Merge pull request #17486 from smithfarm/wip-21188-luminous
+    
+    luminous: pybind: dashboard usage graph getting bigger and bigger
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b755445f8443b2cd7bb31027996be6a6daef96a4
+Merge: e763746bbf 2a1ae18fa7
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:05:24 2017 -0500
+
+    Merge pull request #17350 from liewegas/wip-pr-17334-luminous
+    
+    os/bluestore: reshard-fix wrong shard length
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e763746bbffd6139e49bb035b2c5763bc28dcd9c
+Merge: 69552ed435 00ed756ae8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:31:14 2017 +0200
+
+    Merge pull request #17473 from smithfarm/wip-21112-luminous
+    
+    luminous: cephfs: get_quota_root sends lookupname op for every buffered write
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 69552ed4352e25d90574da8e802f2c9572ea2529
+Merge: 71814348c2 a74b3267ee
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:29:32 2017 +0200
+
+    Merge pull request #17465 from smithfarm/wip-21045-luminous
+    
+    luminous: rbd: TestMirroringWatcher.ModeUpdated: periodic failure due to injected message failures
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 71814348c28071e14ebda8ac6bf257e9cbf08a79
+Merge: 2142746fe1 1e9983ac83
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:28:59 2017 +0200
+
+    Merge pull request #17565 from theanalyst/wip-21278-luminous
+    
+    luminous: the standbys are not updated via ceph tell mds.* command
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 2142746fe14f5717062b2e1f9931b36e12896336
+Merge: be9331252a 0ec531240a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:28:06 2017 +0200
+
+    Merge pull request #17566 from theanalyst/wip-21267-luminous
+    
+    luminous: Incorrect grammar in FS message 1 filesystem is have a failed mds daemon
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit be9331252ae966a9d7ec1e5ce9321d8806061d6a
+Merge: cfbd752f6d 22208c8237
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:23:50 2017 +0200
+
+    Merge pull request #17557 from theanalyst/wip-21269-luminous
+    
+    luminous: some generic options can not be passed by rbd-nbd
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit cfbd752f6d46fa96307d3529022524279c4896fe
+Merge: b9ec9193d7 3f0ee2aa39
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:43:48 2017 +0200
+
+    Merge pull request #17448 from smithfarm/wip-21118-luminous
+    
+    luminous: rgw: need to stream metadata full sync init
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit b9ec9193d7b49e44de3d34674f9dc2b8815f99df
+Merge: 5f0ff20aca 5f0ba1353b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:37:26 2017 +0200
+
+    Merge pull request #17464 from smithfarm/wip-20964-luminous
+    
+    luminous: rbd: switch to new config option getter methods
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 64fc7cf105ec4cdeacb6e2af3060b914ba47abeb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Sep 11 12:32:08 2017 -0400
+
+    ceph-volume lvm.trigger just split the osd_id once
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ecd5af88da2815603283742c661d27e2a48f9f7e)
+
+commit 7cdf1772d69e85646a55202c4faac1d0aaace5c4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Sep 11 12:30:34 2017 -0400
+
+    ceph-volume tests catch improper parsing of osd uuid
+    
+    When the osd ID existed in the UUID in the same format (e.g. '1-') the
+    parsing would break returning a bogus UUID that is impossible to find
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5392697121ef150f0edac05df14d116cf0148593)
+
+commit 944e69863347bb11450874980e4f64807d9e6e87
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Sep 8 15:32:15 2017 -0500
+
+    tests: remove 'ceph-' prefix from test node hostnames
+    
+    This updates our Vagrantfile to match the changes in ceph-ansible
+    introduced by https://github.com/ceph/ceph-ansible/commit/298a63c43
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f41d359ebaec8762ed1a7b8cfd85c23f351a5070)
+
+commit f1b92868fa775ab4c210b3fd02e9a4913485db3a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 12:25:57 2017 -0400
+
+    ceph-volume lvm.api PVolumes should filter by all matching tags, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b8613c1d7bf56c49d1b7f1c41a37073b8739bbc3)
+
+commit 54cdf1805a12fd691bd9da0441dab186446aef76
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 12:25:31 2017 -0400
+
+    ceph-volume tests pv API should also match all pv_tags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 120fd6a196045c8511c3eaafec37de43003bf450)
+
+commit 6a1189d1eb74d9c3d7653f4f32c480158d260901
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 11:47:09 2017 -0400
+
+    ceph-volume tests prove all volume tags have to match in vgs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fe2a7106decb586e83f5b60d6c18c454afe4883e)
+
+commit f4930e0b8327e5dcd6aafaac7cd51c1933b95456
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 11:46:34 2017 -0400
+
+    ceph-volume lvm.api all vg tags should match, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6d679855aa077e2de3f20316625d478a3697c78f)
+
+commit fafc161e1f6c825930c0848c32fa06b1d5edc849
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 17:00:22 2017 -0400
+
+    ceph-volume tests prove all tags have to match in lvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80e401ca970066541ae7255be1caf42883048df4)
+
+commit cdba58264732a648425dd1f78e75a91ea6a5198d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:59:53 2017 -0400
+
+    ceph-volume lvm.api all lv tags should match, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dbaf30c3e4d921df2a92812383c8805a77eb8f5b)
+
+commit 12d638afea087189a1d3f350dd87185b0ade6617
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:56:55 2017 -0400
+
+    ceph-volume lvm.activate allow not using osd_id
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 99a8a719695591558da29b3d16a347d5bff8bf3d)
+
+commit a50dc9647282c7b7455774502532f5281ae2aece
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:54:19 2017 -0400
+
+    ceph-volume tests prove missing args.osd_id is fine
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d1a1a885fd65073308c205949bc70b41805f92f)
+
+commit 834d4c0eabf18fe1219f52a70e3bf78235f51f62
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:51:43 2017 -0400
+
+    ceph-volume tests add is_root and lv fixtures to conftest
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d18d5e139be82de36e1a60340525adbf79fdedd9)
+
+commit 5f0ff20aca1bb25f8758d6533f0aa38b3614c954
+Merge: 90fbef808e 5d229d18ec
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:22:44 2017 +0200
+
+    Merge pull request #17446 from smithfarm/wip-21115-luminous
+    
+    luminous: rgw multisite: objects encrypted with SSE-KMS are stored unencrypted in target zone
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 90fbef808eb84f3cdc72172a8e4690f8857c960c
+Merge: a05e676a6a a1a3c07ba4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:22:05 2017 +0200
+
+    Merge pull request #17468 from smithfarm/wip-21052-luminous
+    
+    luminous: build/ops: RHEL 7.3 Selinux denials at OSD start
+    
+    Reviewed-By: Ken Dreyer <ktdreyer@redhat.com>
+
+commit a05e676a6a619327929d1000e22a557e77c1dc6f
+Merge: 16ccc2084c 7c3a82225e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:19:17 2017 +0200
+
+    Merge pull request #17444 from smithfarm/wip-20968-luminous
+    
+    luminous: rgw: bytes_send and bytes_recv in the msg of usage show returning is 0 in master branch
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 16ccc2084c7d477af362e2d99a409f7c929268ac
+Merge: c0c79fd74d 4f7c155c2e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:16:40 2017 +0200
+
+    Merge pull request #17555 from theanalyst/wip-21270-luminous
+    
+    luminous: rgw: shadow objects are sometimes not removed
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit c0c79fd74df79c0746ee31ef0590dd60d586946c
+Merge: 7b69aefc30 40d46c01b7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:00:36 2017 +0200
+
+    Merge pull request #17558 from theanalyst/wip-21277-luminous
+    
+    luminous: [cls] metadata_list API function does not honor `max_return` parameter.
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 7b69aefc30d11fd4cc3ea2bb9911761a5d0b7770
+Merge: 893ca2493a 3ac5343841
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:59:53 2017 +0200
+
+    Merge pull request #17556 from theanalyst/wip-21265-luminous
+    
+    luminous: [cli] rename of non-existent image results in seg fault
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 893ca2493aa19b18859064a25034b77642f18fe6
+Merge: 8d180e4386 6468ddf8fc
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:59:28 2017 +0200
+
+    Merge pull request #17560 from theanalyst/wip-21288-luminous
+    
+    luminous: [test] various teuthology errors
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8d180e4386f6385b2c76976de8e73ca4d93a5c08
+Merge: 06f566fd07 7989da7da6
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:58:52 2017 +0200
+
+    Merge pull request #17561 from theanalyst/wip-21289-luminous
+    
+    luminous: [rbd] image-meta list does not return all entries
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit e2dda02e02d25b3785c7355abffdad98a1242482
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 7 08:55:27 2017 -0400
+
+    qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Fixes: http://tracker.ceph.com/issues/21181
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 273c84578b06905229a7c6eae7b1a12bbe60de95)
+    
+    Conflicts:
+            qa/workunits/rbd/import_export.sh: trivial resolution
+
+commit 06f566fd073c25c674f0441f83e3ce8f4396e1d5
+Merge: 5e5002d993 9cd6899346
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:08:14 2017 +0200
+
+    Merge pull request #17336 from tchaikov/wip-luminous-pr-16774
+    
+    luminous: mgr/dashboard: fix rbd's pool sub menu
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit ccb3c1edc0a45dfaadb5c4a4aa6700d0b8c8d92f
+Author: hechuang <hechuang@xsky.com>
+Date:   Thu Jun 29 10:38:23 2017 +0800
+
+    rgw: Data encryption is not follow the AWS agreement
+    
+    Encryption request headers should not be sent for GET requests and HEAD
+    requests if your object uses SSE-KMS/SSE-S3 or you’ll get an HTTP 400
+    BadRequest error.
+    
+    Signed-off-by: hechuang <hechuang@xsky.com>
+    (cherry picked from commit dfe1a7c1e6c470d623c70d83c32cc5b7a921bbc5)
+
+commit e7158e8e41e89eb845ecd7a888a5bfc6a008cc75
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 7 09:44:36 2017 -0400
+
+    mon: fix dropping mgr metadata for active mgr
+    
+    drop_standby() was killing it and it was only getting added
+    back in certain locations.  Instead, make the metadata
+    drop conditional and only do it in the places we're
+    really dropping the daemon, not when we're promoting
+    it to active.
+    
+    Fixes: http://tracker.ceph.com/issues/21260
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29c6f9adf178f6611a625740f395e397cad9147b)
+
+commit f5ff82d8ff4c45aee345b271ef4a3d1b95f83237
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 7 09:42:29 2017 -0400
+
+    qa/tasks/mgr: regression test for 21260
+    
+    (aka http://tracker.ceph.com/issues/21260)
+    
+    Fixes: http://tracker.ceph.com/issues/21260
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 28cb4bb2e68298a8a307eb2de6121b60db43cb86)
+
+commit b10311dc730edfbd622a5a2ee7fe45293887b61a
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Sep 8 12:48:12 2017 +0100
+
+    mon/OSDMonitor: don't create pgs if pool was deleted
+    
+    A pool may be deleted while in the process of creating pgs. If that
+    happens, we should not queue additional pgs from that pool to be
+    created.
+    
+    Fixes: http://tracker.ceph.com/issues/21309
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 7a8a88032ec0bf7f1b485419a11646d7441faf2c)
+
+commit f847b164fda0e54a725e4a805124b78a61e545bf
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Sep 7 18:10:18 2017 +0100
+
+    mon/OSDMonitor: fix typo
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit bc5fe2e9099dbb560c2153d3ac85f38b46593a77)
+
+commit fe2fe20100f7f5349d7101bc718f040f5e45bde1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 8 19:28:09 2017 +0800
+
+    encoding: reset optional<> if it is uninitialized
+    
+    * should reset it, in case we reuse it after initializing it.
+    * initialize the value of `p` using the C++11 style initializer, so it
+      is zero-initialized.
+    * revert 2a83ef3c which disables a warning of:
+     ./include/encoding.h:317:7: warning: 't' may be used uninitialized in
+        this function [-Wmaybe-uninitialized]
+     where the `t` is the temporary variable for initializing the value of
+     `p`.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0363236dc754180a0215a818d58587943f84fb53)
+
+commit 9f65be2feb0186276e2b91fc58e6573779921970
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Sep 6 17:31:16 2017 +0800
+
+    tools/ceph_objectstore_tool: fix 'dup' unable to duplicate meta PG
+    
+    Recently we plan to bring a Jewel cluster into Luminous.
+    After that is done, which turns out to be a big success,
+    we then try to transform all FileStore osds into BlueStore ones
+    offline but with no luck. The ceph_objectstore_tool keeps complaining:
+    --------------------------------------------------------------------
+    dup from filestore: /var/lib/ceph/osd/ceph-20.old
+          to bluestore: /var/lib/ceph/osd/ceph-20
+    fsid d444b253-337d-4d15-9d63-86ae134ec9ac
+    65 collections
+    1/65 meta
+    cannot get bit count for collection meta: (61) No data available
+    --------------------------------------------------------------------
+    
+    The root cause is that for FileStore Luminous will always try to rewrite
+    pg "bits" as a file attribute on "Load" if that is not available.
+    But since meta pg is never loaded (we skip it during OSD::load_pgs()),
+    we actually never get the chance to do so; hence making the
+    dup method from ceph_objectstore_tool very unhappy since it always
+    expects to see such a attribute from underlying store.
+    
+    Fix the above problem by manually skipping loading the "bits" attribute
+    if underlying OS is FileStore for dup.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit af2c4847e42945a878e2864319185d795b0664a5)
+
+commit 5e5002d99360cc2a3c956aeee43ab4fb25a6eef9
+Merge: ec8fe46855 68f48c7af8
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 14:16:20 2017 -0500
+
+    Merge pull request #17622 from liewegas/wip-more-lum-pi
+    
+    osd: update info only if new_interval
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ec8fe46855979e999f0e518f73ddd18ba40a23c6
+Merge: 6957c220ed 4683108903
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 14:01:27 2017 -0500
+
+    Merge pull request #17564 from theanalyst/wip-21283-luminous
+    
+    qa/objectstore/bluestore*: less debug output
+
+commit 68f48c7af89fd5d8b224d52c8f60cfd2cdaf2df8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 4 13:51:34 2017 +0800
+
+    osd: update info only if new_interval
+    
+    refactor OSD::build_initial_pg_history() so we update the info only if a
+    new interval is identified. also, this is consistent with
+    OSD::build_past_intervals_parallel().
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4ae053439e526fab0451d1502576dce89eb6b44f)
+
+commit 6957c220edc5c1bab7d41fa0d3ca06f02ec268d5
+Merge: 462fc906e1 e2e81403f8
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 13:59:45 2017 -0500
+
+    Merge pull request #17496 from smithfarm/wip-21236-luminous
+    
+    luminous: osd: build_initial_pg_history doesn't update up/acting/etc
+
+commit 462fc906e14bc994667f015f8dd1ece923861c0d
+Merge: 0832adaaa7 a32f6a2e3a
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 13:56:51 2017 -0500
+
+    Merge pull request #17601 from liewegas/wip-21171-luminous
+    
+    os/bluestore: fix deferred write deadlock, aio short return handling
+
+commit 98a53f929ef92ac6c68f34e30df929eca3f323bc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Aug 29 00:19:24 2017 +0200
+
+    build/ops: rpm: do not create librbd.so.1 symlink in /usr/lib64/qemu
+    
+    As reported by Ademar de Souza Reis Jr <areis@redhat.com>:
+    
+    This symlink should not be necessary anymore. QEMU is properly linked to
+    ceph/librbd these days (look at the qemu-block-rbd sub-package in Fedora and
+    the respective package in RHEL).
+    
+    The symlink was a hack from a time when librbd was distributed and supported by
+    ceph and the rbd driver would be enabled at runtime only when/if the symlink
+    was present.
+    
+    So the right fix is to actually get rid of the symlink altogether and never
+    touch (much less own) /usr/lib64/qemu.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit be4d6b1a74a0612621c3ad2cedebaa1fa40ffbab)
+
+commit 0832adaaa7e7faaf60ff7a51d18229bfc6eebf76
+Merge: aba1e7cd34 d769b077b1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 14:04:14 2017 -0400
+
+    Merge pull request #17551 from dillaman/wip-21280
+    
+    luminous: test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 80d848f069b9c5c97a720972f62bbb8eda1d3573
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 14:16:57 2017 -0700
+
+    osd/PGBackend, MOSDPGRecoveryDelete[Reply]: handle incompatible encoding
+    
+    Handle a pre-luminous encoding without min_epoch for both these
+    messages.  Bump the header version, so the simple path of inline
+    decoding works for the future.  For 12.2.0 and pre-luminous, we can
+    check the SERVER_LUMINOUS flag on the connection to tell which
+    encoding to use.
+    
+    This is not a cherry-pick from master since all upgrades must stop at
+    luminous, and will use the existing encoding before continuing.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit aba1e7cd34bebca60187bed3fc8fcd75a18642b4
+Merge: 28e5f64ec3 ce434ac24f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:58:39 2017 -0500
+
+    Merge pull request #17467 from smithfarm/wip-21046-luminous
+    
+    luminous: common: adding line break at end of some cli results
+
+commit 28e5f64ec3fc946bad8472c342c73c5d3bafb17f
+Merge: 3a270224a5 e2cc9ef48a
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:57:33 2017 -0500
+
+    Merge pull request #17472 from smithfarm/wip-21108-luminous
+    
+    luminous: mon: OSDMonitor: osd pool application get support
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3a270224a5a1e75eddcc74adc8e6effc39674e2d
+Merge: 79750eab64 a656a38d28
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:56:56 2017 -0500
+
+    Merge pull request #17475 from smithfarm/wip-21133-luminous
+    
+    luminous: osd: PrimaryLogPG: sparse read won't trigger repair correctly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 79750eab642eca2b4d6227dfdf28932edcae1256
+Merge: fb1a23d974 0c75157a91
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:55:55 2017 -0500
+
+    Merge pull request #17481 from smithfarm/wip-21182-luminous
+    
+    luminous: core: osd crush rule rename not idempotent
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fb1a23d974cae62ea9effc10f25a03e616f863ac
+Merge: e5885fae2f 927b19ff77
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:55:37 2017 -0500
+
+    Merge pull request #17494 from smithfarm/wip-21234-luminous
+    
+    luminous: bluestore: asyn cdeferred_try_submit deadlock
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e5885fae2ff8215d6c06acf23edb69d02e59ce46
+Merge: e22b3a9410 daacbfd90f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 8 08:51:33 2017 -0700
+
+    Merge pull request #17469 from smithfarm/wip-21103-luminous
+    
+    luminous: cephfs: client: missing space in some client debug log messages
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e22b3a941098cd54f4986b8bea8019483fefb214
+Merge: bcd492a062 dd5f9e5333
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 8 08:48:05 2017 -0700
+
+    Merge pull request #17562 from theanalyst/wip-21276-luminous
+    
+    luminous: os/bluestore/BlueFS.cc: 1255: FAILED assert(!log_file->fnode.extents.empty())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bcd492a062875c71047faaae58e77568dbaa4c41
+Merge: d48762a858 2c7fbc463c
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:28:35 2017 -0500
+
+    Merge pull request #17345 from liewegas/wip-pr-17311-luminous
+    
+    mon: use ceph_clock_now if message is self-generated
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d48762a858f87177a7834f6ebab6a4be8f8a3d82
+Merge: 7c347e82e9 43ffb7c897
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:28:17 2017 -0500
+
+    Merge pull request #17344 from liewegas/wip-pr-17330-luminous
+    
+    mon/OSDMonitor: make 'osd crush class rename' idempotent
+
+commit 7c347e82e9919bc8c72022b3b101e197089428de
+Merge: 92ea7eb529 7b3ab3252f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:27:55 2017 -0500
+
+    Merge pull request #17527 from liewegas/wip-pr-17510-luminous
+    
+    crush: fix fast rule lookup when uniform
+
+commit 92ea7eb529d915a17826dc69f0a266112ad2d0f3
+Merge: 2e60c1711d 81bf6cbfd4
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:27:31 2017 -0500
+
+    Merge pull request #17511 from liewegas/wip-stupid-luminous
+    
+    os/bluestore: default to stupid allocator
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 2e60c1711da79e687b5002e40bfe4183ccee5f0a
+Merge: 70c0d2ebf7 ac4cb6dd41
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:26:27 2017 -0500
+
+    Merge pull request #17501 from smithfarm/wip-21242-luminous
+    
+    luminous: osd: osd/PG: discard msgs from down peers
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 70c0d2ebf79307ab4e42555672579dc5dfca4592
+Merge: 3ade14114d 8a0d7aa073
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:25:37 2017 -0500
+
+    Merge pull request #17500 from smithfarm/wip-21240-luminous
+    
+    luminous: mon: rate limit on health check update logging
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3ade14114da0798b05b59842a31c9630f0393269
+Merge: 9437427b3e 4c0073d20b
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:24:50 2017 -0500
+
+    Merge pull request #17498 from smithfarm/wip-21238-luminous
+    
+    luminous: osd: subscribe to new osdmap while waiting_for_healthy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9437427b3e7f544bbd02eeb698198ed6154983f0
+Merge: a06f09ea3f 3806293b1e
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:22:49 2017 -0500
+
+    Merge pull request #17497 from smithfarm/wip-21237-luminous
+    
+    luminous: bluestore: osd crash when change option bluestore_csum_type from none to CRC32
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a06f09ea3f63ffc7874f12a29539201abfce8be8
+Merge: 0fd1035a6f 33dc4e65b2
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:22:23 2017 -0500
+
+    Merge pull request #17378 from jan--f/wip-21187-luminous
+    
+    luminous: core: osd/PGLog: write only changed dup entries
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a32f6a2e3a46c1709e5a3e0443d8b45df4bd9c99
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 16:28:59 2017 -0400
+
+    os/bluestore/aio: handle short return from io_submit
+    
+    io_submit may return a value less than nr, indicating that only some of
+    the provided iocbs were queued.  If that happens we should loop, not
+    return and silently drop those aios on the floor.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dc17dfd9ff05b5676488c2b1bca53026b2ca6244)
+
+commit accd582d59efa2ba8349e6ad58775e1ca619d821
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 30 17:18:18 2017 -0400
+
+    os/bluestore: more deferred debugging
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11ac691dcc86aeed4ccb692a79ade4be04c5ef2f)
+
+commit e9b89c8b011fbbeeb128553b1b12708155a52628
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 16:15:52 2017 -0400
+
+    os/bluestore: drop deferred_submit_lock
+    
+    This lock serves no purpose.  It only protects the call to the aio_submit
+    on the OpSequencer running batch, but we are the only caller who will do
+    that submission, and a new batch won't be moved into place until ours is
+    completed and retired (by the completion for the aio we are queueing).
+    
+    More importantly, this fixes a deadlock:
+    
+    - thread A submits aio, but the queue is full, and blocks
+    - thread B blocks taking deferred_submit_lock while holding deferred_lock
+    - aio completion thread blocks trying to take deferred_lock, and thus
+    no further aios are drained.
+    
+    Fixes: http://tracker.ceph.com/issues/21171
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a5ef62af194373c3a8c307b05ae6d467e49c9f0)
+
+commit 0fd1035a6f280ebd38e6aabc925afb3d49fd53ea
+Merge: 823585311c c9c168b0ea
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Fri Sep 8 09:34:36 2017 +0800
+
+    Merge pull request #17534 from liewegas/wip-pr-17459-luminous
+    
+    os/bluestore: add bluestore_prefer_deferred_size_hdd/ssd to tracked keys
+
+commit 0ec531240ad47dc2d11db68a50d7e9debfe67efe
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 13:50:46 2017 +0100
+
+    mds: fix "1 filesystem is have a..." message
+    
+    Like "Health check failed: 1 filesystem is have a failed mds daemon (FS_WITH_FAILED_MDS)"
+    
+    Fixes: http://tracker.ceph.com/issues/21153
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d21119a47212140d121df99efdb838a3f078a82)
+
+commit 1e9983ac833e8bd74fa82c6a03ad8fbba68747ac
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:33:24 2017 +0800
+
+    ceph: collect all mds in mdsids()
+    
+    otherwise, only the active mds are returned.
+    
+    Fixes: http://tracker.ceph.com/issues/21230
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dd702cc94e32f533f0f2e76f9d202d1fa7c6a57f)
+
+commit dabbe79e17269e8061aae60198008292601335f3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:05:44 2017 +0800
+
+    ceph: always populate `targets` with ids_by_service()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b682e61ddc190b91da4c6cbaeba3c69d54652afb)
+
+commit 428c9934fdf78d378b2c75ac43759581710472fb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:00:59 2017 +0800
+
+    ceph: extract ids_by_service() so it can be reused
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 25639f66910aa8a188d5b4b8fff8803e169da62a)
+
+commit 468310890376a9d6b055e01d1743a8c5753dce35
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 17:43:28 2017 -0400
+
+    qa/objectstore/bluestore*: less debug output
+    
+    Let's see if this makes the spurious MON_DOWN failures go away?  (See
+    http://tracker.ceph.com/issues/20910)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 444f5aa08539cde827e7d93a514367a8ba39b122)
+    
+    Conflicts:
+     qa/objectstore/bluestore-bitmap.yaml
+     Added bluestore-bitmap.yaml to Luminous as well
+
+commit dd5f9e53335892fa04a1d489de1020d5655e1e3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 15:01:02 2017 -0400
+
+    os/bluestore/BlueFS: prevent _compact_log_async reentry
+    
+    _should_compact_log uses new_log != nullptr to tell whether compaction is
+    already in progress, but we don't set it until we are midway through the
+    process.  Set it at the top of the method to prevent reentry.
+    
+    See 455cc6cea2e5e8bc1126b5047b6003fa583af1b5, which failed to implement
+    this properly.
+    
+    Fixes: http://tracker.ceph.com/issues/21250
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6bd9db304f906e9e4a07ca5ce07ffffdbc1678f0)
+
+commit 7989da7da6b9c2fe1d0e183d8c6d67b9bcb7c097
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 6 16:14:58 2017 -0400
+
+    rbd: support iterating over metadata items when listing
+    
+    Fixes: http://tracker.ceph.com/issues/21179
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9c4f9856b034061aed052192b0ee09b9a9d09bc0)
+
+commit 6468ddf8fc17155229edd6e579db7f1c6405e27d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:26:52 2017 -0400
+
+    qa/suites/rbd: reduce monitor requirements for mirroring tests
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 27194c30c50009eaf6391a21837f73b312fe5de6)
+
+commit 0cef781d74bf387a8b29f2197e492579aba42389
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:20:26 2017 -0400
+
+    qa/suites/rbd: fixed cache override
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98061bb3d7ce6309ddb04ea4d7e9d44a7ecd09c6)
+
+commit 8d6ec22730d6538f8bfce3fdc076d360f7cc0e6b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 15:59:47 2017 -0400
+
+    qa/suites/rbd: reduce OSD debug log levels
+    
+    The teuthology machines are periodically running out of space
+    due to the aggressive log settings.
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 763f1877ecda680e62a2b7eff5815bb6b7dd18f2)
+
+commit 40d46c01b77891f64129ad18575f82cb6cf00534
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 10:53:47 2017 -0400
+
+    cls/rbd: metadata_list not honoring max_return parameter
+    
+    Fixes: http://tracker.ceph.com/issues/21247
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0130b85732849da8bb3e96f4afb7c023c958aef9)
+
+commit 22208c8237913c5865b4d4cf40ddb083f2ffe48c
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Sun Sep 3 12:12:15 2017 +0800
+
+    rbd-nbd: fix generic option issue
+    
+    Fixes: http://tracker.ceph.com/issues/20426
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit 16f64a4c01068603736af476f640268d0baa338d)
+
+commit c34be1841d041d318ded0b63e95eab79d63bd0ef
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Tue Aug 29 23:30:15 2017 +0800
+
+    rbd-nbd: lower down the log level when return -EINVAL in aio_callback.
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit dcd7e5bb82524bf111a6d80d9e0c050ae55b26a1)
+
+commit 3ac5343841805daa21a86be144da94c60a58590a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 12:11:45 2017 -0400
+
+    librbd: rename of non-existent image results in seg fault
+    
+    Fixes: http://tracker.ceph.com/issues/21248
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a75ee43d3b108a9593a010160479fd62884eda8)
+
+commit 4f7c155c2eb241e87160db30de4e24e7d8931188
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 5 16:10:38 2017 -0700
+
+    rgw: add tail tag to track tail instance
+    
+    Use tail_tag instead of idtag. The tail_tag is modified when either
+    creating a new tail, or adding a refcount to the tail (e.g., when
+    copying object). When just modifying the object's head, tail_tag
+    should remain the same. The gc will then use the tail_tag (if
+    exists, idtag if not) as the refcount id.
+    
+    Fixes: http://tracker.ceph.com/issues/20234
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f71a3e525d9a69fe537428f25930fb207a602afe)
+
+commit d769b077b1a6fa34bfddbef857ae543872df2f75
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 21:37:16 2017 -0400
+
+    test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Fixes: http://tracker.ceph.com/issues/21217
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aeb8f29d21625f3570e2f3e6dd26594d1a6df810)
+
+commit c9c168b0ea550fe13181e2e64b53cd8511794d22
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Sep 5 11:45:53 2017 +0800
+
+    os/bluestore: add bluestore_prefer_deferred_size_hdd/ssd to tracked keys
+    
+    To make these two keys can be changed online:
+    
+    ./bin/ceph tell osd.* injectargs "--bluestore_prefer_deferred_size_hdd 0"
+    osd.0: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    osd.1: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    osd.2: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit cd1225634cf4d20e485f7c30d5a0438febc7ddcc)
+
+commit 7b3ab3252f9010590da1cc695919b70e09a3d5e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 22:25:03 2017 -0400
+
+    crush: fix fast rule lookup when uniform
+    
+    Older clients will search for the first rule with a matching ruleset,
+    type, and size.  The has_uniform_rules bool is only set if we have rule
+    ids and rulesets that line up, but we must also verify that the rest of the
+    mask matches or else we can get a different CRUSH mapping result because
+    the mask might not match and old clients will fail to find a rule and we
+    will find one.  We also can't just check the ruleset as the legacy clients
+    find the *first* (of potentially many) matching rules; hence we only do
+    the fast check if all rulesets == rule id.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f24095e0e9734531dbdbcd4bff1392c463188e8e)
+
+commit 81bf6cbfd4cdea04937bb04eb4902762114e5913
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 23:04:15 2017 -0400
+
+    os/bluestore: mark bluestore_allocator as DEV
+    
+    We don't want users changing this without a good reason.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4b6a4a7ff3e82e015fc8268142d8c5c6437d089f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 11:58:40 2017 -0400
+
+    os/bluestore: put StupidAllocator btree into bluestore_alloc mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f189d8ab3c549a5c5e01616310c6eb39119fbc1)
+
+commit b943e7c5844d818b0cbc51e0d88fa147c31a44e2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 11:21:32 2017 -0400
+
+    os/bluestore: default 'bluefs_allocator = stupid'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b6eea80eacd65cbb1aa3702a3a3b642304c04f5)
+
+commit 3e06813fe92c0ad07ac033aab14d088136b996f9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 12:14:07 2017 -0400
+
+    os/bluestore: default to bluestore_allocator = stupid
+    
+    Performance looks better, and code is simpler.
+    
+    Also fix config option annotations.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3750cf9cafabd3296c22b8b8dff35e44d18b9dd8)
+
+commit ac4cb6dd41adfc549d064756215421c33368dc86
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 24 16:04:54 2017 +0800
+
+    osd/PG: discard msgs from down peers
+    
+    if a repop is replied after a replica goes down in a new osdmap, and
+    before the pg advances to this new osdmap, the repop replies before this
+    repop can be discarded by that replica OSD, because the primary resets the
+    connection to it when handling the new osdmap marking it down, and also
+    resets the messenger sesssion when the replica reconnects. to avoid the
+    out-of-order replies, the messages from that replica should be discarded.
+    
+    Fixes: http://tracker.ceph.com/issues/19605
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e3fce6be44506168a7a138aab93f6a4d6776397b)
+
+commit 8a0d7aa073eb14fa3324594449968d2755535d6c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 7 07:52:42 2017 -0400
+
+    mon: rate limit on health check update logging
+    
+    Fixes: http://tracker.ceph.com/issues/20888
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5b903c7b428c2c7024fdfc1dffacad0b1521b750)
+
+commit 2484a728e2694357de5e50dae2345ce6894d873b
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 05:33:16 2017 -0400
+
+    mon: add helper for emitting logs by health status
+    
+    We were starting to repeat our little "if(status==HEALTH_ERR"
+    blocks in too many places.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 618a27868d65aa152ac67cce8057a75c4d61def9)
+
+commit 4c0073d20b72a30e67c97b1527d091d43c67a358
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 17:56:13 2017 -0400
+
+    osd: subscribe to new osdmap while waiting_for_healthy
+    
+    If we are sitting around waiting until we are able to ping our "up" peers,
+    we need to be sure that our notion of "up" is still correct and we're not
+    just stuck on an old, stale OSDMap.
+    
+    Fixes: http://tracker.ceph.com/issues/21121
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fbafa659dae94faba435ca449ee5e77b51108b4b)
+
+commit 3806293b1ecc888f92fdc7f15ef3f25ac37ac29b
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 31 12:42:36 2017 +0800
+
+    os/bluestore: formatting nits
+    
+    So we don't output anything if csum is currently off:
+    
+    _do_alloc_write blob Blob(0x7fad7cc25950 blob([0x42a80000~10000] none/0x1)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 22740cc9b3e489ae82b4b3071a17793c11abce79)
+
+commit ca667063b77445a2f45cdce04110b3affbdf6bed
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 31 11:42:37 2017 +0800
+
+    os/bluestore: don't re-initialize csum-setting for existing blobs
+    
+    The global checksum setting may change, e.g., from NONE to CRC32,
+    which can cause improper re-initialization of the csum-settings of
+    existing blobs(e.g., partial write/overwrite may turn out to shrink
+    'csum_data').
+    
+    We could develop some complicated solutions but for now let's not
+    bother since the above scenario is rare.
+    
+    Fixes: http://tracker.ceph.com/issues/21175
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 8ec4b4960f9b93ff6d85a02308ccda0bccadb599)
+
+commit e2e81403f8dd2329c0e99c8c8abbfa8351f0401e
+Author: w11979 <wang.wenfeng@h3c.com>
+Date:   Fri Sep 1 10:02:34 2017 -0400
+
+    osd: fix build_initial_pg_history
+    
+    We need to update our info about the previous interval in order to
+    detect interval changes properly.
+    
+    Fixes: http://tracker.ceph.com/issues/21203
+    
+    Signed-off-by: w11979 <wang.wenfeng@h3c.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit afdacc6ed76dd288fdbc0f99857d9ffba23c5132)
+
+commit d57399c9dab0be6905a90b743d6b87ca93700cc6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 31 14:55:46 2017 -0400
+
+    qa/tasks/ceph_manager: revive osds before doing final rerr reset
+    
+    We assume below that rerrosd is up, but it may not be when we exit the
+    loop.
+    
+    Fixes: http://tracker.ceph.com/issues/21206
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 21027233b2a31b587e7caf95b1d85f5bcf0ae8ec)
+
+commit 927b19ff77b667d6d5b73b5024fd3007196d0002
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 31 16:43:39 2017 -0400
+
+    os/bluestore: separate finisher for deferred_try_submit
+    
+    Reusing finishers[0], which is used for completions back into the OSD,
+    is deadlock-prone: the OSD code might block trying to submit new IO or
+    while waiting for some other bluestore work to complete.
+    
+    Fixes: http://tracker.ceph.com/issues/21207
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b6a9d1ec0778f66b6e9d9f216df21a00caeba3f)
+
+commit cac29c9618fc5ca8bfcc6a5b4b83ff51443654fa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Aug 23 12:13:24 2017 +0300
+
+    rgw-admin: require --yes-i-really-mean-it when using --inconsistent_index
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Fixes: http://tracker.ceph.com/issues/20777
+    (cherry picked from commit 1cd7ef0eb9bde5a08f8c758ce57c9e298fabad8d)
+
+commit 91dd0fe479c63e0b907ce218fa9f262fa950364f
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Sun Aug 27 05:25:01 2017 -0700
+
+    interval_set: optimize intersect_of for identical spans
+    
+    Optimize comparisons for identical spans of intervals.
+    When this patch is combined with the previous map insert
+    optimization, a benchmark using 400000 identical
+    intervals shows a 7 times performance improvement in
+    comparison to without the patches.
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit b6a035666c2765f8895ee9991348dbc025613ed7)
+
+commit 83236f0c476bc7e36c11e934787e69c11283ec13
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Fri Aug 25 09:41:07 2017 -0700
+
+    interval_set: optimize intersect_of insert operations
+    
+    Use the std::map insert method with hint iterator to optimize
+    inserts. This increases performance more than 3.5 times for
+    large numbers of intervals. This will help performance
+    especially in the PGPool::update method, where profiling data
+    has shown that intersection operations are a hot spot. The
+    following benchmark data is for 400000 intervals:
+    
+        4 +-+--+----+----+----+----+----+----+----+----+--+-+
+    P     +    +    +    +    +    +    +    +  *************
+    E     |                             ********            |
+    R 3.5 +-+                       ****                  +-+
+    F     |                   ******                        |
+    O     |                 **                              |
+    R   3 +-+           ****                              +-+
+    M     |          ***                                    |
+    A     |        **                                       |
+    N 2.5 +-+     *                                       +-+
+    C     |     **                                          |
+    E     |     *                                           |
+        2 +-+ **                                          +-+
+    R     |  **                                             |
+    A     | **                                              |
+    T 1.5 +**                                             +-+
+    I     |**                                               |
+    O     +*   +    +    +    +    +    +    +    +    +    +
+        1 +*+--+----+----+----+----+----+----+----+----+--+-+
+          0   0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9
+    
+                            SET SIZE RATIO
+    
+    The above chart was generated using benchmark results
+    from the following program:
+    
+    #include <iostream>
+    #include <sys/timeb.h>
+    #include "include/interval_set.h"
+    
+    int main(int argc, char *argv[])
+    {
+      const int interval_count = std::stoi(argv[1]);
+      const int interval_distance = 4;
+      const int interval_size = 2;
+      const int sample_count = 8;
+      const int max_offset = interval_count * interval_distance;
+      interval_set<int> a, b, intersection;
+    
+      for (int i = 0; i < max_offset; i+=interval_distance) {
+        a.insert(i, interval_size);
+      }
+    
+      for (int m = 1; m < 100; m++) {
+        float ratio = 1 / float(m);
+    
+        for (int i = 0; i < max_offset; i+=interval_distance*m) {
+          b.insert(i, interval_size);
+        }
+    
+        struct timeb start, end;
+        int ms = 0;
+        for (int i = 0; i < sample_count; i++) {
+          ftime(&start);
+          intersection.intersection_of(a, b);
+          ftime(&end);
+          ms += (int) (1000.0 * (end.time - start.time)
+            + (end.millitm - start.millitm));
+          intersection.clear();
+        }
+        b.clear();
+    
+        std::cout << ratio << "\t" << ms << std::endl << std::flush;
+      }
+    }
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit 32bc0430f70b057d1bba623252e92ab9f279028d)
+
+commit 8e9b52ee884dce49385855eed571ece6cdc9e0d7
+Author: yanyx <yanyx@umcloud.com>
+Date:   Mon Aug 7 18:14:14 2017 +0800
+
+    pybind: fix chart becomes bigger when refresh
+    
+    Fixes: http://tracker.ceph.com/issues/20746
+    Signed-off-by: Yixing Yan <yanyx@umcloud.com>
+    (cherry picked from commit edf93427734ce156f5eaac7a25f4c16a92852b98)
+
+commit 63c9382eafb5d40441ce487c2430c98c214ce84a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 25 17:55:11 2017 -0400
+
+    rgw_file:  fix LRU lane lock in evict_block()
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/21141
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3301596734adcbda1e8e76a742935efdeb7518fd)
+
+commit 24803b76c804fb1717a4c6ed4f93c63fbd4a8f19
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 28 10:14:03 2017 -0400
+
+    qa: create_cache_pool no longer runs 'pool application enable'
+    
+    Fixes: http://tracker.ceph.com/issues/21155
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 054297450484ff99f95470905cd6d8986c26e2c4)
+
+commit ca7e48c62e6a18c5a627d5e71ed8016e1e41b9a1
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 10:08:12 2017 -0400
+
+    mgr: fix crash in MonCommandCompletion
+    
+    Fixes: http://tracker.ceph.com/issues/21157
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fa61b2ed078a3cf333a3111d4f43584ce27ba9d)
+
+commit 0c75157a91506dc5edd3c4ae5eb95ac1a0c38aee
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Aug 29 08:21:07 2017 +0800
+
+    mon/OSDMonitor: make 'osd crush rule rename' idempotent
+    
+    Fixes: http://tracker.ceph.com/issues/21162
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit cd2191c3f192986e20beaab84b643dfaf49901f7)
+
+commit b575d00073ccf3f84d84a220f542058471a58472
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Aug 3 15:14:05 2017 +0200
+
+    rgw_lc: add support for optional filter argument and make ID optional
+    
+    Support Filter tag in Lifecycle XML similar to AWS S3, while S3 docs
+    mention that this tag is mandatory, older clients still default to
+    Prefix, and S3 itself seems to relaxed in enforcing the rule, this
+    implementation also follows a similar pattern. Filter optionally
+    supports filtering via (multiple) Object Tags, this is still a TODO. The
+    current implementation of object tags is still as an object xattr, and
+    since the LC processing still iterates over the bucket index which
+    currently doesn't have any info. on tags, this requires some thought
+    into for implementing without a larger performance penalty
+    
+    Fixes: http://tracker.ceph.com/issues/20872
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 93a858392a2a3bc8c16369dd8c3f6845e99af404)
+
+commit ce2a181834cf60f9d03bbcf849ae72a740bb10a9
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Thu Aug 10 10:57:13 2017 +0800
+
+    rgw: fix obj copied from remote gateway acl full_control issue
+    
+    This part of the code is commit in
+    ea3efca
+    
+    When copy a object to a remote gateway, the check processing which is
+    done by
+    "Bitwise And" between source object's acl elements and
+    grants_headers_def array elements
+    will be done to produce the target object's acl elements.
+    
+    So when the full_control is the first element of grants_headers_def, no
+    matter the
+    source object's acl element permission field is write or read, the
+    result of
+    bitwise and will always be true, then call grants_by_type_add_one_grant
+    with
+    check_perm which is full_control, all of the permission field of the
+    target object's
+    acl elements will be full_control.
+    
+    Fixes: http://tracker.ceph.com/issues/20658
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 6bb2ed24ad175005e7ebd187166bb8735761493c)
+
+commit 17626e01d832bac0068697947f32202d2f8cc13a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 10 11:10:09 2017 +0100
+
+    mgr/dashboard: fix FS status on old MDS daemons
+    
+    These may not have the version string.  We already handled
+    this in the equivalent code in the status module, so just
+    do the same thing in dashboard.
+    
+    Fixes: http://tracker.ceph.com/issues/20692
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 800662c2fa3f040992987dc5c635dfbc14f0a0ea)
+
+commit e171fa6a6dd77b2d3490e2d656de149beb5daaab
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Aug 21 17:04:45 2017 +0300
+
+    rgw: Prevent overflow of stats cached values
+    
+    Fixes: http://tracker.ceph.com/issues/20934
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 634215eea1ddd4e4f5dc0066c4a2e745cfc20475)
+
+commit a656a38d285acb413d07c5b8b3d0ea90d491ed53
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 24 15:53:23 2017 +0800
+
+    osd/PrimaryLogPG: fix sparse read won't trigger repair correctly
+    
+    Normal reads instead of hole-reads should trigger auto-repair.
+    Also we shall quit on successfully triggering a repair, or else
+    we are at risk of doing the op multiple times.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 4ad25c346215dc2ddc4b1a43948b6fe01316edf2)
+
+commit 39b3b132f24c2e66b7d862ef5b553992dd109cf4
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 21 16:56:20 2017 +0800
+
+    qa: add health whitelist for kcephfs suite
+    
+    Fixes: http://tracker.ceph.com/issues/20892
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit b10989209f41ddb5abe16f816ffd8a3ce118fd02)
+
+commit 00ed756ae866f2f4777c3994f75032824f3ee5ac
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Thu Aug 10 11:33:46 2017 +0200
+
+    client: skip lookupname if writing to unlinked file
+    
+    When a client writes to an unlinked file, get_quota_root tries to
+    resolve the filename via an MDS lookupname op. The op always results in
+    -13 permission denied for path-restricted caps or -2 no such file or
+    directory otherwise. More importantly, the repeated lookupname ops slow
+    down buffered writes significantly.
+    
+    Don't do the lookupname for unlinked files; use the root_ancentor's
+    quota instead.
+    
+    Fixes: http://tracker.ceph.com/issues/20945
+    Backport: jewel, luminous
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1d617eee2e8c32f7300cb2cb8dd37b3714a4699f)
+
+commit e2cc9ef48a7f7a2014c211a3baa10e7ac1a1a8a7
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Aug 9 21:24:49 2017 +0800
+
+    mon/OSDMonitor: "osd pool application ls" support
+    
+    It would be a pain if we have to call 'ceph osd dump --format=json-pretty'
+    to find out these each time...
+    
+    Demo output:
+    (1) ceph osd pool application get
+    {
+        "cephfs_data_b": {
+            "cephfs": {}
+        },
+        "cephfs_metadata_a": {
+            "cephfs": {}
+        },
+        "test_pool": {
+            "rbd": {
+                "test": "me"
+            }
+        }
+    }
+    
+    (2) ceph osd pool application get test_pool
+    {
+        "rbd": {
+            "test": "me"
+        }
+    }
+    
+    (3) ceph osd pool application get test_pool rbd
+    {
+        "test": "me"
+    }
+    
+    (4) ceph osd pool application get test_pool rbd test
+    me
+    
+    Fixes: http://tracker.ceph.com/issues/20976
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 46586b22882c0c08c099243e189f5a165063da0c)
+
+commit 25a651e100d5854d730f1218154838fa37868686
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 21 13:14:40 2017 -0700
+
+    qa: add chown test clearing S_ISGID and S_ISUID
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 7fd94abd0a2b682c1d7f01699962e27809c061cd)
+
+commit d9a8b3e797252114c930117e6311f924ef0588b3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:48:21 2017 -0700
+
+    ceph-fuse: load supplementary groups by default
+    
+    Partially addresses POSIX test failures in [1] due to the config setting being
+    false by default.
+    
+    [1] https://bugzilla.redhat.com/show_bug.cgi?id=1480182#c6
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 08b077338cba1250c36232f91c708d189326bbb6)
+
+commit b9514d48471b40574a47ad433e3573674075e466
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:20:11 2017 -0700
+
+    client: clear suid/sgid bits on non-zero write
+    
+    According to [1], these bits should be cleared regardless of any exe bits on
+    the file. Also, add the required non-zero write check.
+    
+    [1] http://pubs.opengroup.org/onlinepubs/9699919799/functions/pwrite.html
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 24c9de5d11d5d1145da96a2af52dd22d81e0c89d)
+
+commit e41a8c25345020f47513c43269eb91e3ca6d0852
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:07:43 2017 -0700
+
+    client: add missing space in debug msg
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 3f806c72dd3b9b28761ee32f5e32001de8362028)
+
+commit f9eb24e74f948d85729432dbb440aa177789df85
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 09:52:36 2017 -0700
+
+    cephfs: clear suid/sgid if regular file is exe
+    
+    According to [1], the suid/sgid should be cleared if any of the executable bits
+    are set.
+    
+    Found this while experimenting for [2].
+    
+    [1] http://pubs.opengroup.org/onlinepubs/9699919799/functions/chown.html
+    [2] https://bugzilla.redhat.com/show_bug.cgi?id=1480182
+    
+    Fixes: http://tracker.ceph.com/issues/21004
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d306d74a60e6d2c79aa4e9a840f3fc12ee8461bc)
+
+commit 469e50646c7d83e1f4e59ea90aeeefbd437976e8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 09:48:09 2017 -0700
+
+    client: refactor clear set uid/gid if -1
+    
+    libfuse already does not set FUSE_SET_ATTR_UID if the chown uid is -1. However,
+    another libcephfs client could call setattr directly with -1 as the uid/gid so we
+    should handle that potential case.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 6448d4927fb4cfc29392ff58da8775d68e3e875b)
+
+commit 18442e15854f7973c1b478befaff42f08b549505
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Aug 21 13:02:44 2017 -0600
+
+    rpm: ceph-fuse requires fuse package
+    
+    The ceph-fuse(8) man page describes using the "fusermount" tool to mount
+    CephFS. This utility is in the "fuse" package. Prior to this change, the
+    ceph-fuse RPM only depends on "fuse-libs", not "fuse".
+    
+    The Debian packaging has always depended on the full "fuse" (previously
+    "fuse-utils") package, so we should make the RPM packaging do the same.
+    
+    Fixes: http://tracker.ceph.com/issues/21057
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 6fba9cc3f910442d3d09b41f01f454a39bfdbc48)
+
+commit daacbfd90fb4e127dc73142674eacf63d618cdaf
+Author: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+Date:   Wed Aug 23 10:20:17 2017 +0800
+
+    client: add blank space between function name and inode number for log
+    
+    Fixes: http://tracker.ceph.com/issues/21069
+    
+    Signed-off-by: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+    (cherry picked from commit 0cb4d379fdd2b869bd765e1fcb424a8f8a8f6a04)
+
+commit a1a3c07ba41c2edea2499d1b31baba310e6417e2
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jun 9 14:38:28 2017 +0200
+
+    selinux: Allow nvme devices
+    
+    This commit allows nvme devices which use a different label than
+    standard block devices.
+    
+    Fixes: http://tracker.ceph.com/issues/19200
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit a8af61c8da9028adc13f7aea611d5a5fee9dbfd1)
+
+commit ce434ac24f89e3cd3f51fbbe63b37f500913bc7d
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Thu Aug 17 11:44:43 2017 +0800
+
+    common: add new line break when get command descriptions
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 19dd4203eb930ffb7a321cb617c580d99ceca9c1)
+
+commit 59a327e0a9a71802c61f8dddf8d217a5a7bfe2f6
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Wed Aug 9 14:26:32 2017 +0800
+
+    rbd: adding newline operation when only '--format' used
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 992829d38cb89732f6994642c1bdaa2448e610b7)
+
+commit 94fcad155dd2b1d35264e628309b67977146dfff
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Tue Aug 1 19:48:04 2017 +0800
+
+    rbd: '--pretty-format' should return error if '--format' not used
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 47e7549826b55905e7f298717fedf0e8f220940a)
+    
+    Conflicts:
+        src/test/cli/rbd/help.t (master has several new commands compared to
+                                luminous: omit these)
+
+commit a74b3267eef23f17eb90fff180db03c0f2fa61a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 17 18:39:05 2017 -0400
+
+    test/librbd: TestMirroringWatcher unit tests should ignore duplicates
+    
+    Fixes: http://tracker.ceph.com/issues/21029
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d3d5ea5aca5807b09050fe8ad2245c4963a0f34)
+
+commit 5f0ba1353b6fe0eec5ed67489a9d1f7f87466d23
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 1 15:03:57 2017 -0400
+
+    rbd: switched from legacy to new-style configuration options
+    
+    Fixes: http://tracker.ceph.com/issues/20737
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7dc82bc08cbfabd424dcd0b83ed618d28db5fe8e)
+
+commit 3f0ee2aa39c7b48380617fe33871c7db5989abcb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 16:23:59 2016 -0800
+
+    rgw: metadata full sync streams metadata keys
+    
+    We don't read the entire list of all metadata entries in a single
+    request.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4a509a669125a9a2d74de11cce555df2547103d1)
+
+commit a8c156b4761edea882c7cabb981108d2d127f442
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 15:43:16 2016 -0800
+
+    rgw: rest list metadata supports max-entries and marker
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a45fd78d08f4d862c273a13abf787a93ce8e0906)
+
+commit ebd09a4c7aaa8788fe6fbc5ba80b0ea3308ca3e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:34:46 2016 -0800
+
+    rgw-admin: add marker to metadata list
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ae600c2c3f765f409cb16557870ceb91d6714760)
+
+commit 8d1b349f860a7eaaae0ea281e2f5451ce8fd06f3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:30:57 2016 -0800
+
+    rgw: add marker to metadata api keys listing
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 84ece81bc3590b08e5565c5b96bcbb4cb97a4100)
+
+commit 298366deb4473370467e3f345dddadc2d5a87b71
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:22:57 2016 -0800
+
+    rgw: marker for raw pool listing
+    
+    We now have a marker (cursor) that can be used for raw pool
+    listing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a24b8aeb1e376ce9e5e50535d8ce1a287c99fa90)
+
+commit 5d229d18ec89fbe49a95cb5004be1a18b6376c65
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 16 09:41:43 2017 -0400
+
+    test/rgw: add kms encryption key for teuthology
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f27ebabe551f62241d1f5b55add9ec6b5d782073)
+
+commit d0c2adc1b71cea08a40805a65cb397b0dc5bedad
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 16 09:01:28 2017 -0400
+
+    test/rgw: add multisite test_encrypted_object_sync
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit acb953a9ed83e5e760d9222c8f9ede6c8c73a341)
+
+commit cc9b83c2a759356142436eae2fc005cc8ce684a9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:26:25 2017 -0400
+
+    rgw: rgwx-skip-decrypt header bypasses decryption filter on GetObj
+    
+    Fixes: http://tracker.ceph.com/issues/20668
+    Fixes: http://tracker.ceph.com/issues/20671
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit adf7a333017f52682c7319420804433b7d3851a0)
+
+commit b13a1d5ffa00030170bfb0bb944d94f1b011afb3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:24:13 2017 -0400
+
+    rgw: add skip_decrypt to RGWRESTConn::get_obj()
+    
+    this adds a rgwx-skip-decrypt header to requests from
+    RGWRados::fetch_remote_obj()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2a7a5be4b41641cb17fb9127b84ce717ca55d3fe)
+
+commit df4131c7a9bc6b61aadf315d33b0258b52b0fbed
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:23:26 2017 -0400
+
+    rgw: name arguments to RGWRESTConn::get_obj()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9bd1aa7facf94eed41c4235722490c91c9dbf399)
+
+commit 07a15984c665b5f4720aa6c69ce9f44a8216d859
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Aug 9 11:48:53 2017 +0800
+
+    rgw: break sending data-log list infinitely
+    
+    radosgw send data-log list infinitely when opposite end trimmed the data-log and in quiescence.
+    Fixes: http://tracker.ceph.com/issues/20951
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 40db1fdbc7b24d56b2c7b463238e6b96f2d4bbd9)
+
+commit 7c3a82225e44305fcc5648b0f78719c0331ecca5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Aug 4 20:01:32 2017 -0400
+
+    Test bytes_sent bugs.
+    
+    Rearrange logic to make it easier to measure accumulation.
+    Instrument the boto request/response loop to count bytes in and out.
+    Accumulate byte counts in usage like structure.
+    Compare actual usage reported by ceph against local usage measured.
+    Report and assert if there are any short-comings.
+    Remove zone placement rule that was newly added at end: tests should be rerunable.
+    
+    Nit: the logic to wait for "delete_obj" is not quite right.
+    
+    Fixes: http://tracker.ceph.com/issues/19870
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit a45ab45f74042617ec382f3d50f4d45d883042f5)
+
+commit 8c9805921edac7257a135873e663aea5c4316d1e
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Sun Jun 18 18:18:39 2017 -0400
+
+    Fix bytes_sent bugs.
+    
+    log bytes sent/received.
+    add cct to bufferingfilter
+    add cct to RGWRestfulIO
+    AccountingFilter - save cct for debugging output
+    implement AccountingFilter::complete_request() - account for bytes reported here.
+    BufferingFilter<T>::complete_request() - ignore counts from send_content_length() complete_header();
+    
+    Code quality note:
+    this patch makes "cct" available for a lot of newly added debug
+    statements.  The debug statements are mostly not very useful (and should
+    go away in the future) - *But* the "cct" logic should be redone and
+    incorporated into some base class (such RestfulClient) so that it is
+    possible to easily add in debug statements such as these in the future.
+    
+    Fixes: http://tracker.ceph.com/issues/19870
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 0172aab860abe324923a392baf51f956a65b64a5)
+
+commit 823585311c41729309809690a7df5cb83c4ece12
+Merge: 85bc024bd9 e1253e2c24
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Sep 2 13:47:29 2017 -0500
+
+    Merge pull request #17383 from liewegas/wip-luminous-mon-bits-luminous
+    
+    mon: final luminous mon bits
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e1253e2c248446cb1ef57777e1796d376f2ae97e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 2 14:46:47 2017 -0400
+
+    PendingReleaseNotes: must upgrade to 12.2.1 before mimic
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 85bc024bd9496c3cdfd64c5facb841a048179c50
+Merge: 3e35d292bd 6979dd31ed
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Sep 1 14:37:38 2017 -0700
+
+    Merge pull request #17429 from ceph/wip-luminous-x-luminous
+    
+    qa: add upgrade/luminous-x suite
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6979dd31ede42d7ab204b13b9346662a049b343a
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Fri Sep 1 14:29:41 2017 -0700
+
+    Initial check in luminous-x suite
+    
+    same from master https://github.com/ceph/ceph/pull/17160
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 3e35d292bdd7d931deb204f5878c0a5dea02b27d
+Merge: fa996769fc 55a519056c
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Sep 1 07:14:15 2017 -0400
+
+    Merge pull request #17408 from ceph/luminous-bz1485011
+    
+    luminous: ceph-volume Persist non-lv devices for journals
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 55a519056ce2dd2c7e155f6f8a06ee95c428ad72
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 31 10:27:39 2017 -0400
+
+    ceph-volume tests add new ceph_* ansible dev variables required (centos)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d3abfd42ef8e558a1bbb31e24d0ab3c44ee7f4c4)
+
+commit 00c28d7b3fbc4cf3d04e8de76abe284ed98451aa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:46:54 2017 -0400
+
+    ceph-volume tests add new ceph_* ansible dev variables required (xenial)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ebb48c58123a1cfcfc6eccdf9f6c2b3a720f1233)
+
+commit 22fbc9d8d0ef3803166c7fcf627f7b41847ad7e3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:12:33 2017 -0400
+
+    ceph-volume util create a disk utility for blkid operations
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a19e1288e8ae044f41dff9a1770126aa540cc729)
+
+commit 5ce9597d3579e5f4ee6523b59c42f6b3ef2ece82
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:11:52 2017 -0400
+
+    ceph-volume lvm.prepare store the blkid uuid of a partition journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5160c5398bbf107bcc826696664c503aa52d0020)
+
+commit 3e8f2cc2a57351c990836792256af899bc233b1a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:05:22 2017 -0400
+
+    ceph-volume lvm.activate use the partuuid of a partition to link the journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a1080f833b35c5de30d7b577deabf0a0e82fbbf2)
+
+commit a971b0e0c694da4c2cd5eda8d540f4a3965cd80c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 13:34:21 2017 -0400
+
+    ceph-volume tests update the ansible version for functional/tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 007655afa3def6788e07d2db10c7cd9cb4ab45b7)
+
+commit 9b20da46761c92f3e39e44383bc0b3e3625ee26c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 13:25:28 2017 -0400
+
+    ceph-volume tests add pv* related unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7c22743c9aa1306242fc09ccae5dedf94e025cc4)
+
+commit 459ed5ad1efe0af070e92267657ac4d6a6a15c28
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 11:31:56 2017 -0400
+
+    ceph-volume lvm.api include lv_uuid as output fields
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d85656da2531471e11cb90fcc74570c9c2b804e0)
+
+commit 9b04a4cff09e7aaab6fde5d456854847acc4c061
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 09:38:08 2017 -0400
+
+    ceph-volume lvm.activate always update the link to the journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6a1de41838cb9dbbdc4a8f59eb07e4340ccb9680)
+
+commit 9c1e84ab1fdaad47cb0e27453b19de8c899a9700
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 09:26:35 2017 -0400
+
+    ceph-volume lvm.activate retrieve the journal uuid if journal is a device
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a368fa1ad6d6265c5094eacbe7e72f642b43559d)
+
+commit 7da501a250fb551edb396dde1f19d9737a5812c7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 17:33:39 2017 -0400
+
+    ceph-volume exceptions create a specifc error for multiple pvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 73315c49115dbb45f3bb6c9e46955b333602cec2)
+
+commit 38e0caa1bbb2471808b2895604ede20d8a6615fe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 16:52:33 2017 -0400
+
+    ceph-volume lvm.prepare make a journal a pv, use uuids always
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 913cef2d01b91e12c16e42d268043886b62c72fa)
+
+commit 237e42c4ea21a48f04bc092f9c3f9c0d407ec298
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 16:51:11 2017 -0400
+
+    ceph-volume lvm.api create the PVolumes class and helpers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f2bc125d980485c7b456e0193ec9c66ea0acbe54)
+
+commit 337edc3cd1145cf85dfa3745a8e0ab7c5d26cbf3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 24 19:28:12 2017 -0400
+
+    ceph-volume lvm.api create the PVolume class
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1725a5f3589ba2faddc9ce633997e5563fc66a01)
+
+commit eda292b40a2c221d98b8ddf740752cc8dd857cf9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 24 16:37:48 2017 -0400
+
+    ceph-volume lvm.api add a helper to create pvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00f3fa401871f3108558fea71f6ac03660d00ba1)
+
+commit 4f021d396271c59492f38fe20f7d739be862c15d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 30 17:22:33 2017 -0400
+
+    qa/suites/rados/thrash/d-luminous/at-end: check for purged_snapdirs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 807c4c834136e5a3d20b31d50e770430695473ff
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 00:01:19 2017 -0400
+
+    mon: set purged_snapdirs OSDMap flag once snapsets have all converted
+    
+    This makes it easier to test whether the upgrade + conversion has
+    completed.  In particular, mimic+ will be able to simply test for this
+    flag without waiting for complete PG stats to know whether it is safe to
+    upgrade beyond luminous.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c3cbaad2a1a4df68fe39394e07c2f3ac9818b9c8)
+
+commit 2f3e2e5578089c6c7498358c166689c697d82020
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 28 15:21:44 2017 -0400
+
+    mon: CEPH_MON_FEATURE_INCOMPAT_LUMINOUS
+    
+    - set this once we have a luminous quorum
+    - require luminous if we have this set locally
+    - include this on mkfs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b6c451e3fdaf0785b197fdfacf1c782b932bec9)
+
+commit fa996769fcf1d1eb135c00d98b7b48e370f210d0
+Merge: 935faacb34 dc76083ee4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 16:39:09 2017 -0400
+
+    Merge pull request #17381 from dillaman/ritz303-wip-doc-20437
+    
+    luminous: doc/rbd: iSCSI Gateway Documentation
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dc76083ee451bbaef3e67720d46a6e268b877b60
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 8 19:14:57 2017 -0400
+
+    doc: tweaks for the RBD iSCSI configuration documentation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b0abab17e3fa96ca973791a3afe077406d066035)
+
+commit 3717fd060b815f8b4f0c2ec3225b28b6bca33601
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Wed Jul 12 17:11:25 2017 -0500
+
+    doc/rbd: Replaced numbered steps with a # for dynamic numbering
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit f109031cb5c8150f042f9754f16bfa5216411eda)
+
+commit 846093201207f1544b386d69c38573b21c69f4db
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:22:38 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 2a59c82b3d4bbb13bc82cf62fce1d9cb5ba7e6be)
+
+commit 08fffcd3562d299cdb7ba499012a4ea19250433f
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:22:02 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 1d4f5b458d127065d92652c323c73704cf775754)
+
+commit cee1621875257782d1077a513baf7d264a782795
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:20:40 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 441820d95890d95ea79bae55afb3ca3b1a8a36e3)
+
+commit bf6086377298ca762d0d901934c6c4ec88b0f456
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:19:01 2017 -0500
+
+    doc/rbd: Updated the requirements based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit c57266bf82c554b93653489a4b355f53b4b86424)
+
+commit 75a3c6d4f11dedee222dc7d07ab1f7a8fe18db14
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:35:38 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway requirements content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 9335742140a2b4e8b408560c600a490b2749bd9d)
+
+commit e05635e22894baf37e115a263b483bf251285aad
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:34:49 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway overview content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit d022034bb3a6e1b00cf185cae0b5fa3f4b617a38)
+
+commit 0cd40caa68355fa55b6b9d0fd57c45ae804cfd36
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:34:10 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway 'gwtop' monitoring content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 7e8233429ba158c1a4cfe6e58a7ce24ffe8759c9)
+
+commit 3ef7639370939e1d087693c7dda5c819fd707e23
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:32:53 2017 -0500
+
+    doc/rbd: Added the iSCSI targets, Ansible and CLI docs
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit cbf654c702e7b7a8e7023c4707c4ea38d584be3e)
+
+commit 1d277117d3aa347177927ff85ce314c2d8233498
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:31:39 2017 -0500
+
+    doc/rbd: Added the iSCSI initiators, RHEL, Windows, and ESX docs
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit f0e177fa3d8807cfa6ae17f8a0dc759a045623f9)
+
+commit ad0b8eb0944275e023ddfc9140bd32886545f78c
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:28:06 2017 -0500
+
+    doc/rbd: Updated the "toctree" adding the iSCSI gateway overview doc
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 4ef8426bff9a32e00093f7c5d2b6b9620252293b)
+
+commit 745f1748769bdef44635c4918deae106bbc28a1f
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:23:50 2017 -0500
+
+    doc/rbd: Added the unmodified iSCSI initiator images for ESX and Windows 2016
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 2fd61f8de1587d2bfbbb7e15c952a29566b390a1)
+
+commit 33dc4e65b2862fbadd7eba504247144d4c710a7c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 24 18:03:36 2017 -0400
+
+    osd/PGLog: write only changed dup entries
+    
+    Writing all of the dup entries whenever one is changed causes a large
+    regression in performance.
+    
+    Instead, keep track of ranges that need to be cleared and written
+    after log merging (dirty_{to,from}_dups) and the earliest dup entry we
+    haven't written yet during normal operation (write_from_dups). This
+    parallels the way we track unpersisted log entries.
+    
+    Check that the correct set of dups is persisted by adding a roundtrip
+    through memstore to each merge_log unit test.
+    
+    Fixes: http://tracker.ceph.com/issues/21026
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 75af72f758aacc1260b8d828d0b8bb7786597fdf)
+
+commit 4c5db08e6600b47cb4e2a3b940ab008bc9b1cb72
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 24 18:34:48 2017 -0400
+
+    Revert "osd: adjust osd_min_pg_log_entries"
+    
+    This reverts commit 1c4df03394312fe67f36448613d8b54cb1a0e2c9.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 4c8fd9978369448e28355d62b4d1eef3685ea63b)
+
+commit 935faacb342ca2fc264d5a48085e7779757ab667
+Merge: bcf89707d6 8f0945a0f3
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Aug 30 08:28:33 2017 -0500
+
+    Merge pull request #17359 from tchaikov/wip-luminous-ceph-release-stable
+    
+    luminous: ceph_release: luminous is now 'stable' (12.2.x)
+
+commit 8f0945a0f39e7da629fa94f406bc1314e1b39fc8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 30 12:23:42 2017 +0800
+
+    ceph_release: luminous is now 'stable' (12.2.x)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2a1ae18fa71fa7463389b65245eee3279afade54
+Author: chenliuzhong <liuzhong.chen@easystack.cn>
+Date:   Tue Aug 29 02:28:01 2017 -0400
+
+    os/bluestore:reshard-fix wrong shard length
+    
+    change fault_range parameter from 'needs_reshard_end' to 'needs_reshard_end-needs_reshard_begin'
+    be given needs_reshard_end may be out of bounds
+    
+    Signed-off-by: chenliuzhong <liuzhong.chen@easystack.cn>
+    (cherry picked from commit 23bce6a9504955d7faf352242d88d26d4fe7ac96)
+
+commit bcf89707d6c3c3086f001d08c754095b9e7a85f1
+Merge: f726fee338 32ce2a3ae5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 10:48:19 2017 -0400
+
+    Merge tag 'v12.2.0' into luminous
+    
+    v12.2.0
+
+commit 2c7fbc463cf5a981e5966a4a8b7d2a0d1eff3e95
+Author: huangjun <huangjun@xsky.com>
+Date:   Mon Aug 28 22:52:37 2017 +0800
+
+    mon: use ceph_clock_now if message is self-generated
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 56e8031dac7ee65d54e831e7e372e8771fa0f94a)
+
+commit 43ffb7c897f0fc3b407d09f19954865fad168e03
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Aug 29 09:33:51 2017 +0800
+
+    mon/OSDMonitor: make 'osd crush class rename' idempotent
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 2ee80aead88c90388871ee65d4ed31a2fa47f532)
+
+commit f726fee338e66356f62f9e6d0435df8c94b00cfd
+Merge: 10a5e1911b b713cc831d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Aug 29 21:33:58 2017 +0800
+
+    Merge pull request #17332 from tchaikov/wip-luminous-pr-17269
+    
+    luminous: mon: fix two stray legacy get_health() callers
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 10a5e1911bf0114a7563ad634b7387fefd90caad
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Aug 28 16:30:20 2017 +0000
+
+    12.2.0
+
+commit 9cd689934677b2b28b67a864a1a9dbe482824adb
+Author: yanyx <yanyx@umcloud.com>
+Date:   Thu Aug 3 14:13:07 2017 +0800
+
+    mgr/dashboard: fix rbd's pool sub menu position
+    
+    Signed-off-by: Yixing Yan <yanyx@umcloud.com>
+    (cherry picked from commit 4e1d539f7cbebdbb0fb29175694413deaa2aaddb)
+
+commit b713cc831da238844a6e42e80e93cb31ee060aad
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 25 18:30:12 2017 -0400
+
+    mon: fix two stray legacy get_health() callers
+    
+    These need to call the new method when appropriate.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 69038a18070d0cd47dc857020cce115514beaa3d)
+
+commit 09e661dcb02db872d79427dfab9a9868bbb93750
+Merge: 9df9e82338 34bafa6134
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Aug 28 21:41:56 2017 -0500
+
+    Merge pull request #17327 from tchaikov/wip-luminous-pr-17326
+    
+    luminous: doc: fix order of options in osd new
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 34bafa6134e1fc6b0f1e7b0eb43192bc29917769
+Author: Neha Ojha <nojha@redhat.com>
+Date:   Mon Aug 28 15:59:19 2017 -0700
+
+    doc: fix order of options in osd new
+    
+    Signed-off-by: Neha Ojha <nojha@redhat.com>
+    (cherry picked from commit e47463028921f0414e2f64c467f74cf37a1de085)
diff --git a/doc/changelog/v12.2.2.txt b/doc/changelog/v12.2.2.txt
new file mode 100644
index 000000000..7f63ca6e3
--- /dev/null
+++ b/doc/changelog/v12.2.2.txt
@@ -0,0 +1,9540 @@
+commit cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Nov 30 14:59:27 2017 +0000
+
+    12.2.2
+
+commit 83684b91a3c6b31419114b83fc22106146885fb6
+Merge: 6d9f2161b2 b5d02ac0fd
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 30 22:26:46 2017 +0800
+
+    Merge pull request #19240 from tchaikov/wip-22266-luminous
+    
+    luminous: tools/ceph_monstore_tool: rebuild initial mgrmap also
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 6d9f2161b269b19b0301348dd9b17660420624d5
+Merge: 0b4c3b8f1e b02075a963
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Nov 29 16:25:14 2017 -0500
+
+    Merge pull request #19230 from tchaikov/wip-22247-luminous
+    
+    luminous: mon/PGMap: Fix %USED calculation
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 0b4c3b8f1ed72c3a0053ed372e1c5c9b0d48de59
+Merge: a57edd4f1a eb99a7ee36
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Nov 29 10:33:34 2017 -0600
+
+    Merge pull request #19239 from ceph/luminous-bz1518264
+    
+    luminous: ceph-volume correctly fallback to bluestore when no objectstore is specified
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit b5d02ac0fd60c26417ad33dc6efcf80bff503fdd
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 29 20:21:33 2017 +0800
+
+    tools/ceph_monstore_tool: rebuild initial mgrmap also
+    
+    Fixes: http://tracker.ceph.com/issues/22266
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f63d1da4afa3463730ecbc0be29df6375b79fa8f)
+
+commit eb99a7ee36d541939f589a674d9890b14e017dbe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 28 09:22:19 2017 -0500
+
+    ceph-volume lvm.activate correct detection and fallback of objectstore flags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f0ac2dfdbc0a05cff0a5edbedca91274885b7870)
+
+commit 10dbbb0a6e09c6181e81b04765cd368963753bb8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 28 09:21:34 2017 -0500
+
+    ceph-volume tests.devices.lvm verify objectstore flags in activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d22856d6c95ac3b45918765b382cae507a8d1a4)
+
+commit a57edd4f1a47f08c0f93d7a3ff57c376b72c9a8a
+Merge: 1b0a5660f7 ee02b5c037
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Nov 29 08:43:07 2017 -0600
+
+    Merge pull request #19217 from liewegas/wip-p2p-app-warning-luminous
+    
+    upgrade/jewel-x/parallel: debug fuse
+
+commit b02075a963c24c6e1b24d1b076275dcce5350a03
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Sun Nov 26 22:51:58 2017 -0700
+
+    mon/PGMap: Fix %USED calculation bug.
+    
+    Previous code forgot to multiple raw_used_ratio to calculate
+    used byte.
+    
+    Fixes: http://tracker.ceph.com/issues/22247
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit d10c6c26f9ef7372e2c95da79d23b07ce5f4e0e5)
+
+commit ee02b5c037c67c708335745898328e990ecacc90
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 16:48:36 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: whitelist more stuff
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit d2a284c782fed3bec4ed38c898351826bca0a8bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 15:02:18 2017 -0600
+
+    qa/suites/upgrade/jewel-x/parallel: only mount ceph-fuse once
+    
+    Otherwise we do 4 mounts for no good reason.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit b7c6d9799cbfba3cff7bee5fd1de309a8299d911
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 15:00:05 2017 -0600
+
+    upgrade/jewel-x/parallel: debug fuse
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1b0a5660f793714e8cb470c35e084a141135cd8c
+Merge: 4ed747dccf e50af8bb04
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Nov 28 14:48:08 2017 -0600
+
+    Merge pull request #19216 from liewegas/wip-p2p-app-warning-luminous
+    
+    qa/suites/upgrade/jewel-x/point-to-point: fix suppression of pool app warning
+
+commit e50af8bb04bd37bd0e28ddc7935e29bcdc106b05
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 14:46:02 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: fix suppression of pool app warning
+    
+    It's generated on the mgr.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4ed747dccfaae15a9a807d3f3db2fc44b8b21fda
+Merge: acb02717f6 6e1e33a88d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Nov 28 23:47:17 2017 +0800
+
+    Merge pull request #19205 from tchaikov/wip-22136-luminous
+    
+    luminous: qa/ceph-disk: enlarge the simulated SCSI disk
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit acb02717f6e96f96d4128bbebd946238d3c79291
+Merge: 55f5d7d1f1 bb03f6b20b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 28 14:03:01 2017 +0100
+
+    Merge pull request #19152 from smithfarm/wip-22235-luminous
+    
+    luminous: ceph-disk flake8 test fails on very old, and very new, versions of flake8
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6e1e33a88d038d5b6a469d7fb4e52182b888010f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 28 14:42:31 2017 +0800
+
+    qa/ceph-disk: enlarge the simulated SCSI disk
+    
+    100MB will be allocated for journal, and the remaining 100MB is for data
+    device. taking the inode into consideration, there will be approximately
+    87988 kB available for the activated OSD. and it will complain with a
+    "nearfull" state.
+    
+    Fixes: http://tracker.ceph.com/issues/22136
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3c159e9fb8c31d0acd75f0702080f18959f672e)
+
+commit 55f5d7d1f19696af981007c63a7503b2e5d0a5a8
+Merge: 3bd0007d5f ad2393b8b8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 28 03:23:09 2017 +0100
+
+    Merge pull request #19173 from smithfarm/wip-22251-luminous
+    
+    luminous: build/ops: macros expanding in spec file comment
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 3bd0007d5f1c991a80ff80fc2b7bb983ceee971e
+Merge: 7ce2013dfd 5633a58824
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 27 10:23:57 2017 -0800
+
+    Merge pull request #19179 from ceph/wip-yuriw-BP-19175-luminous
+    
+    qa/suites/rados/rest: move rest_test from qa/suites/rest/
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 5633a58824f6248b79af74780411746cb7518813
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 10:11:51 2017 -0600
+
+    qa/suites/rados/rest: move rest_test from qa/suites/rest/
+    
+    ...and add more health whitelists.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ddb274669ca16b95c452b256e58a2830af10c73d)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 7ce2013dfda6f06ac47878fd4635ac0862bfbbbd
+Merge: ecec65906a 61e4de133d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 27 09:20:18 2017 -0800
+
+    Merge pull request #19177 from liewegas/wip-jewel-x-p2p
+    
+    qa/suites/upgrade/jewel-x/point-to-point: skip ec tests when mons may be old
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 61e4de133d9fa1eddcd858152b7828515e0b280f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 10:28:16 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: skip ec tests when mons may be old
+    
+    Early point release mons don't handle legacy ruleset-* ec profiles, new
+    ones do.  Skip the ec tests that may trigger this when we are doing a
+    workload that races with mon upgrades.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit ad2393b8b8dc71a8099fa87924dcb6936a27fd84
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Aug 17 09:19:38 2017 -0600
+
+    rpm: rm macros in comments
+    
+    rpm expands all macros in a .spec file, even those in comments. Drop the
+    percent signs so rpm will not expand these.
+    
+    This change silences rpmlint's warning about macros in comments.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 5c1e9f55d3a144c93c9c18b031db3cdc320561ef)
+
+commit bb03f6b20b4963ad5d51c1c6f45f19bc7eaf220b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Nov 21 11:36:02 2017 +0100
+
+    tests: ceph-disk: ignore E722 in flake8 test
+    
+    Very old, and very new, versions of flake8 treat E722 as an error:
+    
+    flake8 runtests: commands[0] | flake8 --ignore=H105,H405,E127 ceph_disk tests
+    ceph_disk/main.py:1575:9: E722 do not use bare except'
+    ceph_disk/main.py:1582:9: E722 do not use bare except'
+    ceph_disk/main.py:3252:5: E722 do not use bare except'
+    ceph_disk/main.py:3288:21: E722 do not use bare except'
+    ceph_disk/main.py:3296:17: E722 do not use bare except'
+    ceph_disk/main.py:4358:5: E722 do not use bare except'
+    tests/test_main.py:26:1: E722 do not use bare except'
+    ERROR: InvocationError: '/opt/j/ws/mkck/src/ceph-disk/.tox/flake8/bin/flake8 --ignore=H105,H405,E127 ceph_disk tests'
+    
+    Fixes: http://tracker.ceph.com/issues/22207
+    References: https://gitlab.com/pycqa/flake8/issues/361
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 3600cd7bfdca76485d2998c5da1a0dd25816d1ce)
+
+commit ecec65906af60c16fc99f669c1591bb821305a64
+Merge: 613634c1da aac7a85a3a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 22 13:36:58 2017 +0800
+
+    Merge pull request #19080 from smithfarm/wip-22212-luminous
+    
+    luminous: tests: ceph-disk: silence deprecation warnings
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit aac7a85a3ae9cce38987ce3f51be4c6cb63c8dd2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 21 21:47:30 2017 +0800
+
+    qa/workunits: silence py warnings for ceph-disk tests
+    
+    ceph-disk now prints "depreacted" warning message when it starts. but
+    the tests parses its stdout and stderr for a json string. so we need to
+    silence the warnings for the tests.
+    
+    Fixes: http://tracker.ceph.com/issues/22154
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d44334f31704487ec3574738e75145872d9932cf)
+
+commit 613634c1da4cea975d6e1da7d258600c454c9400
+Merge: 5a469c0376 94839deb2f
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Nov 20 09:24:41 2017 -0600
+
+    Merge pull request #18989 from ceph/wip-luminous-rm22154
+    
+    luminous: ceph-disk create deprecation warnings
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5a469c0376e299e5c60d7d9b3c16640515b11f26
+Merge: 3532f233f6 37d2726751
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Mon Nov 20 22:55:06 2017 +0800
+
+    Merge pull request #19025 from tchaikov/wip-pr-19024-luminous
+    
+    luminous: qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3532f233f691ac22c32c4533a677b95d0e4c6273
+Merge: 5e519aebe7 0f7332e925
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Nov 20 07:23:30 2017 -0600
+
+    Merge pull request #18945 from liewegas/wip-22128
+    
+    mon/OSDMonitor: add option to fix up ruleset-* to crush-* for ec profiles
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 37d2726751c05a0b5a3c1b739bce49995cc72445
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 20 13:40:56 2017 +0800
+
+    qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    see also #17902
+    
+    Fixes: http://tracker.ceph.com/issues/21474
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 749bbda07522114b99b17a4a01cfcf019520285d)
+
+commit 5e519aebe718bd16d52e974beba8e3bf97b04ae8
+Merge: 9abf091336 96c981ce6b
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 20:34:50 2017 -0600
+
+    Merge pull request #18983 from pdvian/wip-22150-luminous
+    
+    luminous: os/bluestore: fix SharedBlob unregistration
+
+commit 0f7332e9259b320777770f0312233a31103bfeda
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 15 08:55:33 2017 -0600
+
+    mon/OSDMonitor: add option to fix up ruleset-* to crush-* for ec profiles
+    
+    The jewel->luminous upgrade test will fail if we finish the upgrade while
+    a workload setting old-style ec profiles is running.  Add option to
+    automatically fix them up.  Warn to the cluster log when this happens.
+    
+    For now, enable this option to ease upgrades and whitelist the warning.
+    
+    Only include this option in luminous so that we implicitly sunset this
+    compatibility kludge immediately.
+    
+    Fixes: http://tracker.ceph.com/issues/22128
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 9abf0913367a1ad8cf076c79bf3adace8c526853
+Merge: c0e032c153 975229c367
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:44:46 2017 -0600
+
+    Merge pull request #18751 from shinobu-x/wip-21701-luminous
+    
+    luminous: ceph-kvstore-tool does not call bluestore's umount when exit
+
+commit c0e032c153a36c00109970395fe42cc2013fdb9a
+Merge: 81b768ab03 4d4aee14a4
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:44:19 2017 -0600
+
+    Merge pull request #18750 from shinobu-x/wip-21702-luminous
+    
+    luminous: BlueStore::umount will crash when the BlueStore is opened by start_kv_only()
+
+commit 81b768ab036e66d3f4d36ddc31febc4bbcb143fb
+Merge: f02d7109c5 0fe463b5e1
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:41:37 2017 -0600
+
+    Merge pull request #18860 from liewegas/wip-filestore-rocksdb-compression-luminous
+    
+    luminous: os/filestore: disable rocksdb compression
+    
+    Reviewed-by: Douglas Fuller <dfuller@redhat.com>
+
+commit 94839deb2f32e8059206ad48ad711dd2baf2fef5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 13:10:29 2017 -0500
+
+    doc/ceph-volume create a migration section to help users coming from ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e433efe51481ec63bc9753a0f446fe2b12b5bb6a)
+
+commit b8bf0d047868054135592188c7ebe186181310c5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 13:01:14 2017 -0500
+
+    ceph-disk add deprecation warnings in favor of ceph-volume
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    
+    Fixes: http://tracker.ceph.com/issues/22154
+    (cherry picked from commit c9da92e1ec8d60dc40d86630dab95e4fec4d0d30)
+
+commit f02d7109c5f00cd2165082c4336b3623352be7c5
+Merge: 9515da2777 ebbe033936
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Nov 17 12:19:36 2017 -0500
+
+    Merge pull request #18973 from ceph/backport-18924
+    
+    luminous: ceph-volume: allow using a device or partition for `lvm --data`
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit ebbe033936c424a0f6ba2d102fc271d0b0e9578d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 08:41:24 2017 -0500
+
+    ceph-volume tests.devices.lvm prepare isn't bluestore specific anymore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2f40dbac386be631ac97e2450117c218d9bf02f9)
+
+commit 96c981ce6b00520a94385228bcf70a90c5ae8ff9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 7 22:05:10 2017 -0600
+
+    os/bluestore: fix SharedBlob unregistration
+    
+    We use the SharedBlobSet remove() in three cases:
+    
+    - from SharedBlob::put(), we try to remove ourselves from the set, but
+      have to deal with a racing lookup, so the removal is conditional on
+      nref still being 0.
+    - from split_cache(), we move the SharedBlob to another collection
+    - from make_blob_unshared(), we remove the entry when we clear the sbid.
+    
+    The problem is that the condtiional remove() (for the first case) was being
+    used for all three cases, and in the second two cases nref is always != 0,
+    so it doesn't actually happen.  This can lead to a crash during cache
+    shutdown.
+    
+    Fix by making two variants: remove() that is unconditional, and
+    try_remove() that is conditional.
+    
+    Set the sb->coll pointer after because remove() asserts the parent matches
+    where we are unregistering.
+    
+    Fixes: http://tracker.ceph.com/issues/22039
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78a465b90b1c5187cfec9f25eb58b3bf617ca39b)
+
+commit 9515da277737ad81d48f5c032f627f3cb919f746
+Merge: 5c9b93dadd d19b740359
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Nov 16 13:40:32 2017 -0800
+
+    Merge pull request #18957 from tchaikov/wip-pr-18808-luminous
+    
+    luminous: qa: do not wait for down/out osd for pg convergence
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ad4668b65cb3e32f1b3afc5ec767004f166856e6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 16 08:50:34 2017 -0500
+
+    ceph-volume tests.functional create a common playbook directory for setups
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit def89a411f2f4bdb23c4bbd3b71c1310a564f07d)
+
+commit d25a04114c98aa1449809248e8b87ecd1124af3d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 11:01:48 2017 -0500
+
+    doc/ceph-volume lvm prepare can accept devices for --data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9bfab348b2ff59c0b939a19c22d4a06bc5236b96)
+    
+    Conflicts:
+            doc/ceph-volume/lvm/prepare.rst
+
+commit 7f7d62a1848e297153f88a8e476585596114e1d5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:42:27 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for xenial filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 210c1e1d4770dde47f0008cadaaec35deb729272)
+
+commit 210e1b3bf507654167349411c5cc85641b1d8680
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:42:02 2017 -0500
+
+    ceph-volume tests.functional add data device and journal to xenial filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d0754e25177b9bf3bce695b7991e99f49312d91c)
+
+commit 211895ac0906747bc448b2326ae5c62f3e6ee4a1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:40:37 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for xenial bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d0f5623963f58fc4d5727a79d8377b2191b0f9dc)
+
+commit 398a1365a12de95bc9810b89a9aeab9040bb07b8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:40:18 2017 -0500
+
+    ceph-volume tests.functional add raw data device to xenial bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fbcfb522981cd4039cfe617609c5f160f0b05c52)
+
+commit 0d082d0dbb43cc418bf0786b1f708968050ae114
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:38:23 2017 -0500
+
+    ceph-volume tests.functional tox.ini addition of setup playbook for partitions
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d074b534643dde37d732b34c36c5089e41848d6d)
+
+commit a167556eaacb444c41299cd28f1efd4327da4747
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:37:17 2017 -0500
+
+    ceph-volume tests.functional add raw data device to centos7 bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2d416a849ad7b33dde1386da09c81249e6c2299d)
+
+commit 12e180a84f1456dfe7ab940255c6c47ee8ebddf0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:36:36 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for centos7 bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 161fd373d2f11f64466927e5d615d039822e2ab5)
+
+commit f06ea77590336d439235bd9acc6028b854e8c181
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:35:59 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for centos7 filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b3bb147298f8816651f231f77bc91a3a1b50d96f)
+
+commit f10bfbeefe06e59629a4e9d904e8035bf060920b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:35:42 2017 -0500
+
+    ceph-volume tests.functional add raw device to centos7 filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c5b8d72570b24dfd0d2f8c1cc8f13812c6c11a69)
+
+commit 968006a271dc82f3dcada4e5de93ecb0e23238c6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:29:32 2017 -0500
+
+    ceph-volume lvm.common update --data flag to reflect device acceptance
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4df7ea8d83cfa02c8864d043b5d5fb2f5409dc96)
+
+commit c8f64af027689a1fb1bb916e6d479519a12b764e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:28:32 2017 -0500
+
+    ceph-volume lvm.prepare add example of raw device or partition in help menu
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6fb8d566aac0e0fa081e6500852855ad177f784f)
+
+commit f930934159edd947024eef1a33e15a30467cb332
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:26:44 2017 -0500
+
+    ceph-volume lvm.prepare allow partition or raw device as input for --data in filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 89236ee3f81e50b9b059c10f6d6e7ce048c06474)
+
+commit d19b7403595bef1c3ba73f4a5fb83f5ef2dd3f81
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 8 14:13:54 2017 +0800
+
+    qa: do not wait for down/out osd for pg convergence
+    
+    that osd is not invovlved in the PG state changes.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7f549af459b40c5bb51a24a3a443ed7a7116915e)
+
+commit 5c9b93dadd8f05b1a146e3a19f7480ae4019815f
+Merge: cc41c393b3 2f380b32da
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Nov 14 08:38:35 2017 -0800
+
+    Merge pull request #18908 from smithfarm/wip-luminous-fixup
+    
+    qa/suites/fs/basic_functional/clusters: more osds
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit cc41c393b3a76a5f1a18cbda45acc01e2d6fdfc2
+Merge: d69f42e1af 806e49aadb
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Nov 14 11:06:00 2017 -0500
+
+    Merge pull request #18910 from ceph/backport-18882
+    
+    luminous: ceph-volume: add functional tests for simple, rearrange lvm tests
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 806e49aadb0d832a4bf46fcd681d889d93158232
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:18:07 2017 -0500
+
+    ceph-volume tests.functional update lvm/tox.ini paths after moving it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7d787849556788961155534039886aedfcdb2a88)
+
+commit 724ad01afc0af79fc7f808aae819ff9dc0971b33
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:14:55 2017 -0500
+
+    ceph-volume tests.functional move top level tox.ini to lvm/tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9b14b96d3bf6c8326cfc91e657161ffab3b8c2b4)
+
+commit 754c237597cc91af4bfe7199f891a9c2a6938ad4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:12:51 2017 -0500
+
+    ceph-volume tests.functional move xenial to lvm/xenial
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00d576d266b63080129e58482bd6fba62534ee97)
+
+commit 762c5f1cc10266d5efa62628c0686b3f8d696bff
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:11:58 2017 -0500
+
+    ceph-volume tests.functional lvm move xenial/bluestore to xenial/bluestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d5ec7d1c58141f4593ed9d1e4501c9f3be6d9755)
+
+commit 5b1bde338fbb1e689e3e1305e91f5450ef175edd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:09:48 2017 -0500
+
+    ceph-volume tests.functional lvm move xenial/create to xenial/filestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c504b8781b3a31475280a032cd75c9e1eaa6d8c9)
+
+commit 5a00604f27acad8cb368495f841ce76cce383414
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:02:05 2017 -0500
+
+    ceph-volume tests.functional move centos7 to lvm/centos7
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc1da3fd0aeafe59a82343fb16fe1fcea66ae3eb)
+
+commit 57d64a86d0758d174c1ca0b432a43194cbe6162c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:58:27 2017 -0500
+
+    ceph-volume tests.functional lvm move bluestore to bluestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit aec1a80a3e1e61720f494980711808bfee783a03)
+
+commit 94a9f3f43d9c3a2ac08be4025e65c47c3767d2c1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:54:01 2017 -0500
+
+    ceph-volume tests.functional lvm move create to filestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d6905f3f620d4c733835e21a275d4127cdcb769b)
+
+commit bc134c20a5409bd6f966d205dd7ba850ff28fc91
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:42:19 2017 -0500
+
+    ceph-volume tests.functional simple xenial bluestore scenario
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8f0fdc975951a2a8af31bbaa3abd2b5d807a0add)
+
+commit 004efbfbe4d1eb56190e5e7497a9613776494fae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:40:18 2017 -0500
+
+    ceph-volume tests.functional simple xenial filestore scenario
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7452e1b6f9b82c2ef62bae19c632e81039638ba5)
+
+commit b51744f26f3c6db4d2927a2ac1733f0b80e44010
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:30:05 2017 -0500
+
+    ceph-volume tests.functional simple tox.ini update for xenial scnearios
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 76ed6a8f2ecab7c58def0453ffed9d8a997e0d5b)
+
+commit 19983b9ba5ed45c8e6fac967fb702fd586132c35
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 09:12:09 2017 -0500
+
+    ceph-volume tests.functional simple tox.ini addition for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 02154e445b87a8fdcabbd699a2532833764140b5)
+
+commit 040e7913b028c54aeb9acd854610f6a4f112465d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 09:11:39 2017 -0500
+
+    ceph-volume tests.functional add initial files for simple bluestore activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3320f28e799e1d77d1ea69be9ced603bb69cc5fa)
+
+commit 23f6c3e779c743bc1302d1ad542201434ff9431f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 16:55:30 2017 -0500
+
+    ceph-volume tests.functional add initial files for simple filestore activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9d18bae52ad77be0e64ef105c33a019446b6f1cf)
+
+commit 8f425b678f6c947a8c83f121be4fa46151fd6664
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 16:07:53 2017 -0500
+
+    ceph-volume tests.functional create a specific test.yml for simple activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 37bde0d3b37a87c456438b0edc0cb9e2dcd39120)
+
+commit 6f1fff00f5210840fe143aac7506ce3773600f05
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 15:40:51 2017 -0500
+
+    ceph-volume tests.functional create a separate tox.ini environ for
+    `simple`
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 89757ad12e6938fe2bf997e0f21a5ffac082e30b)
+
+commit d69f42e1afaf8759de1544b3d381671a01cfb55a
+Merge: a7c8c8101d 90976ef6b9
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Nov 13 10:55:24 2017 -0600
+
+    Merge pull request #18907 from ceph/luminous-wip-bz1498200
+    
+    luminous: ceph-volume: support GPT and other deployed OSDs
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2f380b32da1b6294b6b0f60622b09b6517f9afdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 9 13:44:51 2017 -0500
+
+    qa/suites/fs/basic_functional/clusters: more osds
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 998122c324047c96c16d60d7fbb3a21f4dd63c56)
+
+commit 90976ef6b9815b68df620aac46f2c04117766f99
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 10:50:43 2017 -0500
+
+    doc/ceph-volume include the top level systemd.rst in the toctree
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e2f281c115c15e820588b478214c3600ff97470f)
+
+commit 9a8a00d3a17f03d2b1b3a92662f7288f7d18fe7a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 06:49:49 2017 -0500
+
+    ceph-volume simple.activate add --file in help docstring as an example
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ff60592897b63581ebce372b3f6ffa0f10d2fe5a)
+
+commit 34194b8d788d91b4f7c969be5f0099a279bc61bf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 06:46:59 2017 -0500
+
+    doc/ceph-volume activate has a --file option for json files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 29913951b71d68b3f3f9b9fa076f309c0252da36)
+
+commit 582262b57bcc959d1698636000d7a804cab9c611
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 17:23:41 2017 -0500
+
+    ceph-volume simple.activate enable the osd when activating
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 366fb8c43af54e9c92f4e95d9a0582c1fc378dbc)
+
+commit a32e72cdb0efb55d185488fb02f44055f9d3dffe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 15:25:05 2017 -0500
+
+    ceph-volume systemd.systemctl always force symlink when masking ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cb470622fd4f13fafe9cb2f38fb25fd1fbd8d89c)
+
+commit 3af40c35f3cf42be722f9d905cb2d0ba2baf1b1b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 15:21:10 2017 -0500
+
+    ceph-volume simple.activate use the contents of the JSON file, fallback to CLI args
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c954c913395557bcd89f25b0496c4428e6109494)
+
+commit 161e9657d985aa4ceffad2f56e18edd78ca2a81f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 14:50:16 2017 -0500
+
+    ceph-volume simple.activate only check for missing id and uuid if file is not passed in
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 826c2709dd5ab0fbf6eb419a735d15e718a7712c)
+
+commit 585072afc814c6ac87fd64f15c561c51c1e55070
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 14:28:12 2017 -0500
+
+    ceph-volume simple.activate allow to pass a file to activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bd3498f3eabf07637cb936e6c036d1d8d75415b0)
+
+commit 01266c193588066425b44fea4c1729692f999692
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 12:49:31 2017 -0500
+
+    ceph-volume tests.util add tests for OSDPath validator
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7fc26ab4e2cebe485647510119ea300d47db184)
+
+commit 56fdc1c973b41d855563e459d23682990e8940bf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 11:56:36 2017 -0500
+
+    ceph-volume systemd.systemctl masking ceph-disk needs to be done by directly symlinking to dev/null
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cda47c247b929401a7dce3f1ef70371ea80eeceb)
+
+commit 0a0c289de605b6b74b762b2a0a4ed0f3a04de8ad
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:47:12 2017 -0500
+
+    ceph-volume systemd.systemctl masking needs to use systemctl mask, not disable
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bad2f62b34243addc854abf691854b886aecdc19)
+
+commit 32913844b36285aae40f9786e5cb035bfca22f2d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:46:20 2017 -0500
+
+    ceph-volume simple.scan if mounted device cannot be detected, raise an error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6b2fc4c0ba63e4bd51774ea09cc6542738128621)
+
+commit fe73229fb123fdb45f4ef743d8584f6748318868
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:17:52 2017 -0500
+
+    ceph-volume simple.activate remove str formatting from logging
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 230faf04a185c0b95c8ad497ddd5501dc70c1409)
+
+commit 50c19ef335305215a636595506eda1b5d16f3687
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:11:41 2017 -0500
+
+    ceph-volume util.arg_validators always convert the path to an absolute path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b5acf4d720b5395ae4ccbf007c94c99610e40089)
+
+commit d27c35c972abba2b230e1cff1fa35437500dcdf6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:10:47 2017 -0500
+
+    ceph-volume util.arg_validators require superuser privs to check paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 376dad74159532f901d0044f95330fc02f93bd25)
+
+commit 0832ddc52496cd81aa7c2e1e85252b50d2f33044
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:09:06 2017 -0500
+
+    ceph-volume simple.scan skip directories to scan
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6bc1e9ee0b8d7bc475f218ae3dd8c6c7fe5a983a)
+
+commit d6071cb75ba1a702c551d073f032336560654e97
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 15:08:13 2017 -0500
+
+    ceph-volume tests.devices.simple add activate CLI unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cde0d517563b537319d06c758b93e421cdabe59d)
+
+commit 2bff2e7fdbba37280791d3bc9a363e47aa26ab53
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 15:07:44 2017 -0500
+
+    ceph-volume tests.devices add help menu unit test
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dcc7c72196a327f9501045f025761b228b5379df)
+
+commit ec8624005082913faf89a9b001d4abeb7c27a589
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:42:45 2017 -0500
+
+    doc/ceph-volume systemd describe the interaction with 'simple' activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7071eeaeb5d77c6ab6e7a24f1b630cb0cf8a8e7c)
+
+commit aacd22681d1e7f5a604025b2019b195041151b3f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:42:11 2017 -0500
+
+    doc/ceph-volume index initial description of simple
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f9ee51f9feadb6d0f91b76a2b2ba8a24f1523a77)
+
+commit e9527e65d5da503107e21dc5b695f287c8529049
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:41:12 2017 -0500
+
+    doc/ceph-volume lvm add back a system.rst with just the lvm portion of it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f872cd29b93e02b77172f7b382c4746af9ba845d)
+
+commit ef027a737989dca83c0e5810935bad2280146356
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:40:44 2017 -0500
+
+    doc/ceph-volume make systemd.rst generic enough to be related to other sub-commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cf4e32a0dc3ab4f55fe7857b51afbc63a638ec66)
+
+commit a83b746eaeece5896770b40949804316c29f6bc1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:39:52 2017 -0500
+
+    doc/ceph-volume move lvm/systemd.rst to top level, making it generic
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0672dd608d4601a936d703ea5ba6edc14a161d8e)
+
+commit bc492490e15d3ea91654cd4dabfb3115f835b9ac
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 11:52:17 2017 -0500
+
+    doc/ceph-volume simple.activate initial take on documenting activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80c796bd1c43125f5678710b494b03c486b3fce6)
+
+commit 6301c83097729749fc712846954a3c0f89647f07
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 08:35:57 2017 -0500
+
+    doc/ceph-volume lvm update systemd anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 53420a4756a5e4fa6b0f2aa74d6ad1e2919f1954)
+
+commit 5fe4122bcfdc9c8e4f69079e45c907148eccdb7d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 08:35:43 2017 -0500
+
+    doc/ceph-volume lvm.activate update systemd anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7fc0e7ebcae06f28f6ed08c0db14075ada34e8a7)
+
+commit c98886a99914748d73d0f695808a74850bc9e66e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 07:56:35 2017 -0500
+
+    doc/ceph-volume lvm.systemd update anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 01a8b7e6a45f6fd85bf53d3cab15acc81d0bae1b)
+
+commit 68f89b52d608c3946be9ad92e733fe9472bd35f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 15:53:39 2017 -0500
+
+    doc/ceph-volume document the scan functionality
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b30a3eaf17a20a1273724b2cfcaf0a7b36eb08f6)
+
+commit 19d0ca22908fcd3fbe83d39c1b708c5818b34f47
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 14:17:20 2017 -0500
+
+    doc/ceph-volume update the index to include the simple sub command and files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7f85c64e4b409ea95e30ccb5a1388278843c478)
+
+commit 61c90bacc562e1e2d65b767dca97b08973832131
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 11:15:28 2017 -0500
+
+    ceph-volume tests.simple add tests for trigger parsing from systemd
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e2620234e82abf85274b3c08b39cdf76430ce48)
+
+commit 80ae5b1fb024827cceb0594a60e3361fd4a1e7a3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 10:36:14 2017 -0500
+
+    ceph-volume tests.simple create etc_path scan tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 90ebc4e1ffa7b3273787d708bb2c05180530b5d1)
+
+commit b9ad514b45e0233a17834e19a232bca4467a8140
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 10:06:02 2017 -0500
+
+    ceph-volume tests.simple add checks for get_contents in scan
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 647bfe0883d2bb05b4ff7f99894075df3e73d2d5)
+
+commit c9308af91a322f92ab7d1f45a423b2a94f64c429
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 09:43:58 2017 -0500
+
+    ceph-volume tests.util add binary verification tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 827aa57557561355dd4068e26801b5f1ed62f295)
+
+commit c94f52dffe0ea4b48c9fb316e8c53fd06a81b870
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 09:42:53 2017 -0500
+
+    ceph-volume tests create a fixture for temporary files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 23da3445718c01f71e2a5748fb4fc1c5fdc85ca5)
+
+commit bbb25df806609083efd377befb44d6d3bef3338b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:36:38 2017 -0400
+
+    ceph-volume systemd.systemctl helpers to mask ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3da23fa4caea8c9dc91da3eee5b5acc1860bf242)
+
+commit c971321430fd1a4de7c6fa2687ecf35a5b3ec71c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:36:13 2017 -0400
+
+    ceph-volume devices include simple at the module import level
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 027a09b3ea6bac416ff632a69a53393badd51708)
+
+commit 1615d947db50ac37e4ef554acc6debe8fd870962
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:35:45 2017 -0400
+
+    ceph-volume main add simple to the sub-commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 94579c77ffe4d5ba2d457ad3981a2b9ed232a90f)
+
+commit abb77a8d9824eef3bd9c75d17f8df101440c25d1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:35:03 2017 -0400
+
+    ceph-volume util add an OSDPath argument validator
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a826fdf4a2ed9ff704fb823d05d3cb706ba89e38)
+
+commit ff2a1833da385d6c0c9087e98cb2b93925b06cab
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:34:39 2017 -0400
+
+    ceph-volume simple.system add a helper to check if a file is a binary or not
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7bf8676045b4f966b60cdadf4d5af1c89fca69ae)
+
+commit 84d2a15ce9f1a045ed67f79fa7d1dc99dc1e4b8f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:34:06 2017 -0400
+
+    ceph-volume util.system add a context manager for temporary mounting/unmounting
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6394cdd41943e3cda6867ecc59ab835cb2151707)
+
+commit 3d8ff08c2d46017749e7ecaa4e4d49dda4d55d74
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:32:42 2017 -0400
+
+    ceph-volume simple.scan initial take on directory/device scanning
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e54b9be621cc0969a3bbc972d1870080ccdae77)
+
+commit d701569a6486d7ccccf14b5eadf702ad07db4821
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:27:23 2017 -0400
+
+    ceph-volume simple.trigger enable systemd triggering for simple setups
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cbc0c0f775e96a5b0ab5d5bc141acebe4cc48373)
+
+commit 76fe659a19a8a450b4c61f6d2b56e03dc46746ae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:26:13 2017 -0400
+
+    ceph-volume simple.activate initial take on activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5a97374605408d18e9be29bcc051ae7b913d1675)
+
+commit 206c7551435983b4cbdaece61b737e4a1f265dbd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:23:47 2017 -0400
+
+    ceph-volume simple add __init__ for the module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 285e5bdd28a42d98418ca90ad0cdec6bd87849db)
+
+commit 8a5edb9a2acc3f0b5477f09edae4f97f5ec7f5b5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:23:29 2017 -0400
+
+    ceph-volume simple create the initial, main module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f050502425a1993ae8edfdb5785b24129cffeeba)
+
+commit 0fe463b5e1fe62e03b498c32aa708be8c10eb0b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 9 20:10:59 2017 -0600
+
+    os/filestore: disable rocksdb compression
+    
+    Experience working with customer escalations suggests that disabling
+    compression improves performance, and the storage overhead is generally
+    not a concern for the metadata and omap data we are storing.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b878ead071b328e5fe7309a2368383e67679e9f7)
+    
+    
+    # Conflicts:
+    #       src/common/options.cc
+
+commit a7c8c8101d4b78b4d6e437620b2c1a38cd752c3f
+Merge: 1071fdcf73 f56de67980
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 18:08:27 2017 +0100
+
+    Merge pull request #18801 from linuxbox2/luminous-rgw-readdir2
+    
+    rgw_file: implement variant offset readdir processing -- pullup 2 of 2
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 1071fdcf73faa387d0df18489ab7b0359a0c0afb
+Merge: 4db48ca836 8cacd2a568
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:36:27 2017 +0800
+
+    Merge pull request #18734 from shinobu-x/wip-21648-luminous
+    
+    luminous: mgr[zabbix] float division by zero
+    
+    Reviewed-by: Wido den Hollander <wido@widodh.nl>
+
+commit 4db48ca8369d1dae021131c93a1c8484512aa99b
+Merge: 9cd31820f2 d858570755
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:35:44 2017 +0800
+
+    Merge pull request #18851 from tchaikov/wip-18759-pr-luminous
+    
+    mon/LogMonitor: "log last" should return up to n entries
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9cd31820f22d746e680f4ce3b0c780157d24194f
+Merge: 60ff142c8e 8a7f983f33
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:27:47 2017 +0800
+
+    Merge pull request #18741 from shinobu-x/wip-22019-luminous
+    
+    luminous: tests: "ceph osd create" is not idempotent
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d858570755d88e9a3708952827a99cc24f09f9ab
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 6 16:01:23 2017 +0800
+
+    mon/LogMonitor: "log last" should return up to n entries
+    
+    limit the # of returned entries to "num", and backoff the start iterator
+    by one if it reaches the rend().
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit de14103f5201da69b360d9b01e249896d59376b3)
+
+commit f56de6798077399f472c8d2df25fb1c3ccaf8090
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 21:48:19 2017 -0400
+
+    rgw_file: implement variant offset readdir processing
+    
+    Introduce new rgw_readdir2(...), which in which continues
+    from an arbitrary dirent name, which presumably has been
+    seen in a prior partial enumeration.
+    
+    Add single-file unit test for READDIR cases, librgw_file_marker.cc.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit a2c34c597371809bd8fd40ee1fc6b8e6a5145609)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 60ff142c8e8ed8fb294f7d13a5461f070aa1ff4f
+Merge: f1776bc788 7efa91c7ce
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 10:10:53 2017 +0100
+
+    Merge pull request #18800 from linuxbox2/luminous-rgw_file-fsid-mount
+    
+    Luminous rgw file fsid mount -- pullup 1 of 2
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit f1776bc7887d2245e632d5a89189294a823ff3b2
+Merge: 2d3c14e5b8 f7dd3be6b7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 10:04:08 2017 +0100
+
+    Merge pull request #18767 from smithfarm/wip-22020-luminous
+    
+    luminous: multisite: race between sync of bucket and bucket instance metadata
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2d3c14e5b88c773548af61312438a69dcb454726
+Merge: 2bad0ac00c 36e214c67b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 09:56:16 2017 +0100
+
+    Merge pull request #18794 from theanalyst/wip-18709
+    
+    luminous: rgw: Stale bucket index entry remains after object deletion
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2bad0ac00c898d50c9c036400b4fba09918fb331
+Merge: 9787509155 f81e6dc8ab
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:19:27 2017 +0800
+
+    Merge pull request #18726 from shinobu-x/wip-luminous-22035
+    
+    luminous: Spurious ceph-mgr failovers during mon elections
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9787509155df02bd9df21f389968f77d3de2d988
+Merge: 0c3e9e6aab 7e08cdf539
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:18:39 2017 +0800
+
+    Merge pull request #18738 from shinobu-x/wip-21547-luminous
+    
+    luminous: ceph-mgr gets process called "exe" after respawn
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0c3e9e6aab5cfbd3d8e998b148e19bab30f3fa0e
+Merge: 628e76fe2a ed4a692254
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:17:54 2017 +0800
+
+    Merge pull request #18727 from shinobu-x/wip-luminous-22034
+    
+    luminous: key mismatch for mgr after upgrade from jewel to luminous(dev)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 628e76fe2a939aae4996f43e9e9c9f7d1269e0bb
+Merge: ef5264afc9 be5363a771
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:44:41 2017 +0800
+
+    Merge pull request #18723 from shinobu-x/wip-luminous-22023
+    
+    luminous: osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL
+
+commit ef5264afc992d78c1b34c8a96a43216191fa5406
+Merge: a0a2d27c1e 31c56c6e6b
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:38:10 2017 +0800
+
+    Merge pull request #18737 from shinobu-x/wip-21549-luminous
+    
+    luminous: the dashboard uses absolute links for filesystems and clients
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a0a2d27c1e40280ae214696dbf5c25567c28ba38
+Merge: 947cae47db 76ab4eba99
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:37:19 2017 +0800
+
+    Merge pull request #18736 from shinobu-x/wip-21638-luminous
+    
+    luminous: dashboard OSD list has servers and osds in arbitrary order
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 947cae47db9f5dc6d77a85834515670e20b5faf8
+Merge: 2e11e6662a e14de28f9e
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:36:40 2017 +0800
+
+    Merge pull request #18728 from shinobu-x/wip-luminous-22032
+    
+    luminous: dashboard barfs on nulls where it expects numbers
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2e11e6662ae167276ece7a96783f02a24fc1f89f
+Merge: 31afe858a9 cd18f84304
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:34:35 2017 +0800
+
+    Merge pull request #18699 from shinobu-x/wip-luminous-21538
+    
+    luminous: upmap does not respect osd reweights
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 31afe858a96f1d5c67124b92f19cf4542df7e647
+Merge: d9df23cf3a 388b5016d9
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:30:42 2017 +0800
+
+    Merge pull request #18719 from tchaikov/wip-pr-18373-luminous
+    
+    luminous: osd,mon: add max-pg-per-osd limit
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 388b5016d981c9dd98b6505b2d75b0220cfee772
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 18 16:07:50 2017 +0800
+
+    doc: fix the desc of "osd max pg per osd hard ratio"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d940f6220346ad21dfc4f351c57389e8430c0257)
+
+commit 04852a400483b6459985be80a7773ef153c04b1e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 12:17:09 2017 +0800
+
+    mon: change "mon_pg_warn_min_per_osd" to uint64_t
+    
+    * this silences a warning from -Wsign-compare.
+    * also switch all its user to the new-style option
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c3cd0c13a1585388df0f17ab53c170e39c711615)
+
+commit d9df23cf3aa002315e15db46d195be7d5657f9d0
+Merge: 567c29f964 53096755c0
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Nov 7 22:27:15 2017 -0600
+
+    Merge pull request #18702 from liewegas/wip-pr-18670-luminous
+    
+    luminous: qa/tasks/thrashosds: set min_in default to 4
+
+commit 567c29f96461901bd86d4c2d4f84f7b84342837f
+Merge: 77c2b0de60 8cbb2eb937
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 22:28:58 2017 +0100
+
+    Merge pull request #18596 from smithfarm/wip-21955-luminous
+    
+    luminous: tests: add EC data pool to testing
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 7efa91c7ced61f0fde81adfc0540a36fafafb10d
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Sat Jul 22 13:12:06 2017 +0800
+
+    rgw_file: introduce rgw_mount2 with a bucket name parameter
+    
+    This explicitly allow mount a specified bucket in librgw.
+    Originally, mounting a bucket is implemented like a sub-directory
+    mount in nfs-ganesha FSAL_RGW with lookup_path.
+    With this change, we allow root_fh to points exactly to the root
+    of a mounted fs instance, which is a bucket or "/" rather than
+    always let root_fh points to "/".
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 860716e17e840df11a7e1d8b6ca0c1ee694b038b)
+
+commit b53c3948a5e48623bb437b2d36718ce3a799aa54
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri May 26 17:45:29 2017 +0800
+
+    rgw_file: new fsid for fs instance
+    
+    We use an in-memory fs_inst_counter for fsid currently, but it
+    inevitably cause nfs automount problem across a nfs-ganesha crash.
+    So here we relate the fsid to the s3 uid with a hash.
+    With this, fsid remains consistent across all fs instances and
+    across crashes.
+    
+    We should be able to upgrade from old fsid to new fsid with
+    update_fh().
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 178b547e83a2c0eeb024bbe4ca2ff8d8e7d50572)
+
+commit 77c2b0de6042707b42d6e717e90a525fda8ead76
+Merge: 14e0530f50 555746a09e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 22:07:45 2017 +0100
+
+    Merge pull request #18628 from batrick/i21953
+    
+    luminous: mds: sanitize mdsmap of removed pools
+    
+    Reviewed-By: Sage Weil <sage@redhat.com>
+
+commit 14e0530f50cd5fba46da914cdd09489f13adb2ec
+Merge: bc9f508477 c2237c7c6d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:33:21 2017 +0100
+
+    Merge pull request #18626 from dillaman/wip-21968
+    
+    luminous: cls/journal: fixed possible infinite loop in expire_tags
+    
+    Reviewed-By: Mykola Golub <trociny@gmail.com>
+
+commit bc9f5084778607c95dcbc93ea60ce296e8e8ee88
+Merge: 6bc121ddee 1aeec0c032
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:32:20 2017 +0100
+
+    Merge pull request #18688 from liewegas/wip-balancer-luminous
+    
+    luminous: mgr/balancer: backport crush-compat work from master
+    
+    Reviewed-By: John Spray <jspray@redhat.com>
+
+commit 36e214c67b6982cc7a85e08dedc83ea61e56d6a4
+Author: J. Eric Ivancich <ivancich@redhat.com>
+Date:   Fri Nov 3 09:15:13 2017 -0400
+
+    rgw: fix BZ 1500904, Stale bucket index entry remains after object deletion
+    
+    We have a race condition:
+    
+     1. RGW client #1: requests an object be deleted.
+     2. RGW client #1: sends a prepare op to bucket index OSD #1.
+     3. OSD #1:        prepares the op, adding pending ops to the bucket dir entry
+     4. RGW client #2: sends a list bucket to OSD #1
+     5. RGW client #2: sees that there are pending operations on bucket
+                       dir entry, and calls check_disk_state
+     6. RGW client #2: check_disk_state sees that the object still exists, so it
+                       sends CEPH_RGW_UPDATE to bucket index OSD (#1)
+     7. RGW client #1: sends a delete object to object OSD (#2)
+     8. OSD #2:        deletes the object
+     9. RGW client #2: sends a complete op to bucket index OSD (#1)
+    10. OSD #1:        completes the op
+    11. OSD #1:        receives the CEPH_RGW_UPDATE and updates the bucket index
+                       entry, thereby **RECREATING** it
+    
+    Solution implemented:
+    
+    At step #5 the object's dir entry exists. If we get to beginning of
+    step #11 and the object's dir entry no longer exists, we know that the
+    dir entry was just actively being modified, and ignore the
+    CEPH_RGW_UPDATE operation, thereby NOT recreating it.
+    
+    Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
+    (cherry picked from commit b33f529e79b74314a2030231e1308ee225717743)
+
+commit 6bc121ddeec4b12c91ae5ac982ad113bda14d2bf
+Merge: ee27efad8b f26f3dc842
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:21:41 2017 +0100
+
+    Merge pull request #18569 from kmroz/wip-21939-luminous
+    
+    luminous: list bucket which enable versioning get wrong result when user marker
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit ee27efad8ba4d751828dd73012de12ab16c2f78d
+Merge: 601249c6aa a340f1acab
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:21:03 2017 +0100
+
+    Merge pull request #18591 from cbodley/wip-21938
+    
+    luminous: rgw: fix error handling in ListBucketIndexesCR
+    (approved in irc & standups)
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 601249c6aa154d0f9e53332480ac45b5730fb503
+Merge: d2226d5adb 15a52ba832
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:17:57 2017 +0100
+
+    Merge pull request #18539 from linuxbox2/luminous-baixuyeyu-refcnt
+    
+    luminous: baixuyeyu refcnt
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit d2226d5adbb4752e0180dfa3406e188bd3b37d6e
+Merge: 925c68e9e9 902d467a0e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:15:38 2017 +0100
+
+    Merge pull request #18516 from ceph/wip-bp-pr18193-luminous
+    
+    qa/rgw: ignore errors from 'pool application enable'
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 925c68e9e9538a97f98bef1e5a884d31811aa229
+Merge: 2f35da4e60 add8c3db19
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:14:21 2017 +0100
+
+    Merge pull request #18599 from linuxbox2/lum-rgwfile-21940
+    
+    rgw_file:  set s->obj_size from bytes_written
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2f35da4e60c2b5fd2f45348587a5b175868d04d7
+Merge: b2961db15d 26c523c3be
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 13:07:09 2017 +0100
+
+    Merge pull request #18566 from kmroz/wip-21914-luminous
+    
+    luminous: [rbd-mirror] peer cluster connections should filter out command line optionals
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8e3227ad51fa4909a39eb0d05798b48200a03fc5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 12:10:55 2017 +0800
+
+    osd,mon: change "mon_max_pg_per_osd" to uint64_t
+    
+    as it should never be a negative number. if this option is disabled,
+    it's 0.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0d68197376b6cf48c6679d77ec9f28f279f3da68)
+
+commit ae7823187186310bac117fac437d991398cffd5e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:54:14 2017 +0800
+
+    osd: add max-pg-per-osd limit
+    
+    osd will refused to create new pgs, until its pg number is lower
+    than the max-pg-per-osd upper bound setting.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4c7df944c7f28232873ba681eedce72cdb062ea5)
+
+commit 1db44c5bef752b3a6ce59ea21548b54b77b587d2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Oct 15 20:46:28 2017 +0800
+
+    qa/tasks/ceph: add "create_rbd_pool" option
+    
+    True by default, but we can opt not to create the "rbd" pool.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ac1e13ce63a6f32ae9ab0db8196c8cc8100d94ed)
+
+commit ed34b00566f9b1b65c09989af44ad24cdb27fbdc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 12 20:07:46 2017 +0800
+
+    osd: print osd as "osd.$osdid" in log message
+    
+    easier to read this way
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c6b71ebf6b7b7c54560f331d5f8e4e24f5a60633)
+
+commit df611786d2882e346075f5a32aef46bea3c36d47
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:50:08 2017 +0800
+
+    mon/OSDMonitor: bypass checks if pg_temp.forced
+    
+    originally, monitor ignores the pg_temp messages sent from replica osds,
+    but to re-trigger peering if replica osd's pg number drops down below
+    the setting, we need to bypass the checks.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da6f7a810055a0dc9e364aca8e6efbfbc32881b3)
+
+commit aefc9315ed6ebbf209d4496b0fb33c72dc107961
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:49:11 2017 +0800
+
+    messages/MOSDPGTemp: add `forced` field
+    
+    the monitor is supposed to bypass some checks if `forced` is true.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6335bf791003e9e33b2f738ab3da06f3c626537a)
+
+commit b2961db15dd7528019df7990f8f7690ae3a95aa9
+Merge: a5899a57e4 4cac43679a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Nov 7 17:48:34 2017 +0800
+
+    Merge pull request #18775 from tchaikov/wip-upgrade-luminous-x
+    
+    qa: add 1-ceph-install to upgrade:luminous-x
+    
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 4cac43679a7688cb4765728e1a75e812d0df22de
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Aug 29 09:50:51 2017 -0700
+
+    qa: add 1-ceph-install to upgrade:luminous-x
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit aff342627cf77e2faae6904e626d9fb2c897f155)
+
+commit 1aeec0c0326678bcdba7ee0a6a1204f460e3ddf7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 6 21:17:22 2017 -0600
+
+    qa/suites/rados/thrash/d-*/*balancer*: debug osd in mgr
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f7dd3be6b7804035b47d2e6fb94463f0bc772bbe
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:56:01 2017 -0400
+
+    rgw: remove placement_rule from cls_user_bucket_entry
+    
+    placement_rule is no longer needed in cls_user_bucket_entry, because the
+    only time that it's needed, we can read it from the bucket instance in
+    RGWRados::update_containers_stats()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8e62e3526643da67f5af7daa687120feed469785)
+
+commit 17cc13b87b5378bf03e2b0b52c274772ebaf08ef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:29:31 2017 -0400
+
+    rgw: remove placement_rule from rgw_link_bucket()
+    
+    with the fallback in RGWRados::update_containers_stats(), we no longer
+    need to pass placement_rule into the cls_user_bucket_entry
+    
+    this removes the dependency between bucket entrypoint metadata and its
+    bucket instance metadata during multisite sync
+    
+    Fixes: http://tracker.ceph.com/issues/21990
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dd5d278a4f9b080234cbb77e448208803ec0cf93)
+
+commit 338248a8b376074c0f43b626e9f69fc59ee26487
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:26:54 2017 -0400
+
+    rgw: take placement_rule from bucket info in update_containers_stats
+    
+    in case the cls_user_bucket_entry doesn't contain a placement_rule, take
+    it from the bucket instance info
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 531195d80d5bac774d58b210307eb19b57d26345)
+
+commit 975229c3674ce99704aef399ecfb770cc8bcf1da
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Oct 3 00:12:43 2017 +0800
+
+    tool: ceph-kvstore-tool doesn't umount BlueStore properly
+    
+    Fixes: http://tracker.ceph.com/issues/21625
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 81e4560781c4e5a5a033bfd4a4fe4094fc27d964)
+
+commit 4d4aee14a46ed36cdb40c0ff67193c8650b106da
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Oct 3 00:01:43 2017 +0800
+
+    os/bluestore: make BlueStore opened by start_kv_only umountable
+    
+    ceph-kvstore-tool use start_kv_only to debug the kvstore. we
+    will get a crash when we try to umount bluestore in kvstore-tool.
+    
+    Fixes: http://tracker.ceph.com/issues/21624
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 2754c5acad442e50d97c8daf648cfcadd38222a6)
+
+commit 3f7eadb79f4bb0a2fe25b229c24b652e7dcbaf00
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Nov 4 09:32:51 2017 -0500
+
+    qa/suites/rados/thrash: combine balancer with require-luminous
+    
+    We can only do the balancer stuff if we've done the mkfs-time
+    require luminous because otherwise the mgr commands aren't proxied via
+    the mon.
+    
+    Only do balancer for thrash tests; not for basic, monthrash, verify.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8a7f983f3322b62cb18421c088e652341125eb1f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 1 10:45:09 2017 +0800
+
+    qa: stop testing deprecated "ceph osd create"
+    
+    "ceph osd create" is not idempotent, and is considered deprecated.
+    
+    Fixes: http://tracker.ceph.com/issues/21993
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 371d3f40911917704f4992c48236b4c60905bf4e)
+
+commit 7e08cdf53992570d27b47d0028c698b78908ba83
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 15 12:23:39 2017 -0400
+
+    mgr: set explicit thread name
+    
+    This gets used as our process name in some situations
+    when respawning.  This is the same as what commit 4f177bb6b
+    did for the MDS.
+    
+    Fixes: http://tracker.ceph.com/issues/21404
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 44dce0a5664639b2ae0949cc434f1be1176d872d)
+
+commit 31c56c6e6b50b44fef56852e58ab97520ee8614a
+Author: Nick Erdmann <n@nirf.de>
+Date:   Tue Aug 22 16:55:11 2017 +0000
+
+    pybind/mgr/dashboard: updated favicon (old one was 404)
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 7a54520f5c3d4db27fe7e2a12760dcd838ba89aa)
+
+commit d822c15af9b838dc2eea08b5f13f1dba785d776e
+Author: Nick Erdmann <n@nirf.de>
+Date:   Tue Aug 22 08:08:45 2017 +0000
+
+    pybind/mgr/dashboard: clean up
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit f512ac3d7712850f274cd01bb589aa9fbc8c8bef)
+
+commit 1f47cc0b53f7e20e156f74b28502c7705ddb4700
+Author: Nick Erdmann <n@nirf.de>
+Date:   Mon Aug 21 17:21:10 2017 +0000
+
+    pybind/mgr/dashboard: add url_prefix
+    
+    This adds a configuration variable url_prefix to the dashboard that
+    that is prepended to all URLs so you can access the dashboard at
+    http://$IP:$PORT/$PREFIX/. This is necessary if you wish to use a
+    reverse http proxy that forwards to the dashboard under a sub-path.
+    
+    Fixes: http://tracker.ceph.com/issues/20568
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 6d285fff0df598d66f46d38bf0a6a2cc9c0dc62f)
+
+commit 76ab4eba99785edf01c8ff9464d997b2ec691b31
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 15:13:48 2017 +0100
+
+    mgr/dashboard: sort servers and OSDs in OSD list
+    
+    Fixes: http://tracker.ceph.com/issues/21572
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f409099fa25b73fe580ee4662aff51636f118ed2)
+
+commit 8cacd2a568fca023a51f3be6e20f90b4ec08c82b
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 15:22:34 2017 +0100
+
+    mgr/zabbix: log exceptions with backtrace
+    
+    Logging just the exception object gave a sometimes-enigmatic
+    single line.  Let's use the logger exception method so that
+    we get a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 316dcc8f0b42b85503632d5472c45c9828882f2c)
+
+commit 06706bea9339e26d1a0ab395e9d42d391ee8c03c
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 15:18:18 2017 +0100
+
+    mgr/zabbix: fix div by zero
+    
+    Fixes: http://tracker.ceph.com/issues/21518
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9c02738b4ea0303f5b3cbfc4748d6791007be834)
+
+commit a5899a57e4f077f4061fc2aa33a61377de7ac025
+Merge: 52f66ef5ba 35de92b259
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 4 10:52:27 2017 -0500
+
+    Merge pull request #18673 from dzafman/wip-21833
+    
+    osd: build_past_intervals_parallel: Ignore new partially created PGs
+
+commit e14de28f9e524575c633dc98c6ebb9a234c8d1c5
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 14:33:58 2017 +0100
+
+    mgr/dashboard: handle null in format_number
+    
+    Fixes: http://tracker.ceph.com/issues/21570
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f43859d4dca696ba4cac224e0c7e87e4b2aad40f)
+
+commit ed4a692254e71ce7b30566dfd6d16e8a070b3461
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 19 09:28:18 2017 -0400
+
+    mon: don't blow away bootstrap-mgr on upgrades
+    
+    Fixes: http://tracker.ceph.com/issues/20950
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 57229ea2a4369518c7a16b7a09b045b7896f5a70)
+
+commit f81e6dc8abe3535a54cbf683992c95c4eaf4e186
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 12 11:57:50 2017 +0100
+
+    mon: handle monitor lag when killing mgrs
+    
+    Fixes: http://tracker.ceph.com/issues/20629
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3c3776b30a2da3f5d2ece15c57713c0ce191c778)
+
+commit be5363a771172d3260049b0012519b485e4d728e
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Tue Oct 31 13:23:49 2017 +0800
+
+    osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL
+    
+    These two counters should be reported to mgr because the dashboard
+    plugin needing them to display osd detailed information.
+    
+    Fixes: http://tracker.ceph.com/issues/21981
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 40bb3c5f05b6f2a917a7e400f88677f41a9629a3)
+
+commit 77766609d808a974aafdff359dadc46af5bd26f4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 2 16:18:03 2017 -0500
+
+    pybind/mgr/mgr_module: fix calc_pg_upmaps
+    
+    This was fixed in the object cleanup.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ed442164d52c803916ca43a5ed76047089d3da66)
+
+commit 56c27e6be813e6a8b972733468eb099596ff7381
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 2 16:11:26 2017 -0500
+
+    mgr/balancer: enable module by default
+    
+    It will still be "off".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26710f0a9b176289237a52e5fa5894342ad163dc)
+
+commit 53096755c02d953867d0fff3e08bc3a62d872ec6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 1 08:31:31 2017 -0500
+
+    qa/tasks/thrashosds: set min_in default to 4
+    
+    We have EC tests with k=2,m=2, so we need a min of 4.
+    
+    Fixes: http://tracker.ceph.com/issues/21997
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d21809b14ea58dc1f44df844e407ebab5a315062)
+
+commit 52f66ef5ba291fccc265fd7a34a4eae201667153
+Merge: 1b00bdfeef e58656005d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Nov 3 10:26:44 2017 +0100
+
+    Merge pull request #18446 from theanalyst/wip-21854-luminous
+    
+    luminous: rgw_file: explicit NFSv3 open() emulation
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit cd18f84304124e1341516194fb73c8bde48db445
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:17:47 2017 +0200
+
+    osd: upmap should respect osd reweights
+    
+    Modify OSDMap::calc_pg_upmaps to take the osd reweight into account when
+    computing the size of each OSD.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/21538
+    (cherry picked from commit b8561cefa705d31954d47723125376a3c087f4f4)
+
+commit 03a11694feee0d78dcc1f296a77200635a651707
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:15:45 2017 +0200
+
+    test/cli/osdmaptool: test upmap with an out osd
+    
+    Test upmap with one osd marked out.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit ff9e024a59d87eda6cbf3be9464ee7ae6f1140d7)
+
+commit 2c7972d3a4928cec039e8637048c26e94c6278d2
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:14:09 2017 +0200
+
+    osdmaptool: add --mark-out option
+    
+    Add a --mark-down <id> option which sets the reweight for a given
+    osd id to 0.0.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit bd75a78187143f750f90fb937984eaa6333159d3)
+
+commit 1b00bdfeefd7b724f14634383ae4baaf7a42c996
+Merge: c6052f2706 92d0545781
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 2 21:23:50 2017 +0100
+
+    Merge pull request #18439 from theanalyst/wip-21655-luminous
+    
+    luminous: expose --sync-stats via admin api
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 282e21bd07c3dfb5d18f338abd3ced4f4eeaa247
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 27 12:53:52 2017 -0500
+
+    mgr/balancer: simplify pool_info tracking
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a59a1df85382d559362c51bb45284b61caaca1d4)
+
+commit e2346f539f477a4b7a4685828876eb89c84ad00d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 25 22:47:02 2017 -0500
+
+    mgr/balancer: less verbose on 'eval' by default; add 'eval-verbose'
+    
+    The verbose output is helpful for debugging and understanding what is
+    being measured, but most of the time all that you care about is the score.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2c300bc8d7189b464f59c93deb5eb56915df62b8)
+
+commit 64d75fa26d3ceb6ccdbee3b31b3705c4db2fc59f
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 10:46:27 2017 -0500
+
+    mgr/balancer: fix pg vs object terminology
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6513e8b0fc2ebcbbbda937ec6d533ea084ef278f)
+
+commit 6082f53391a46076bda1b5d56685d928acd2cd4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 17:12:54 2017 -0500
+
+    mgr/balancer: restrict to time of day
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f0b50b50aeb34ed39767289b8475ec5b877cc27)
+
+commit 91b84cb24466f84ce3e189e08f8cba1e2900a4cc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:31:06 2017 -0500
+
+    mgr/module: adjust osd_weight min step to .005
+    
+    That should be ~1 PG or less on average; no real sense it taking a
+    step smaller than that!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 297e2d65abcebe45c6be69a768cf3ea98e1b354d)
+
+commit acbf539d326403b6249ed0e2b0ca5ffc4a52152d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:26:16 2017 -0500
+
+    mgr/balancer: if score regresses, take a few more steps
+    
+    To avoid getting stuck in a local minima, take a few additional
+    steps if our score drops and see if it gets better.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 70b503495a0ceb7d1aa4f5fce4403a8eff9a03b5)
+
+commit b9cc962adfd95304cbd56833bb84b827e5eed4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:25:19 2017 -0500
+
+    mgr/balancer: allow 5% misplaced
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1ad3101818d1855c8eea8386d7682fab2fe9c8b8)
+
+commit 452962767b4f899c8c2589d01a6b8acc3cac9a69
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:25:06 2017 -0500
+
+    mgr/balancer: more aggressive steps
+    
+    We are smart enough to back off if we misplace too much.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0dbcf737eaed02b5aa94bc6f27231b4b87f09ce5)
+
+commit 901ae5920c590a2a83f30897b05017e001572e5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 16:00:06 2017 -0500
+
+    qa/suites/rados/thrash/d-balancer: enable balancer in various modes
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2c9c18d1ec3d33d385adf2a30dc1d3b8e987f9a5)
+
+commit a1ebe97886f7a578bc56a109f10a3d6149954087
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 17:05:05 2017 -0400
+
+    mgr/balancer: crush-compat: phase out osd_weights
+    
+    Phase out the osd_weight values as we optimize the crush compat
+    weight set.  Allow a small regression in overall score if it means
+    we are taking a step to remove the osd_weight value.
+    
+    Fix the 'osd reweightn' invocation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 861c37fd99a76d5f1665a610224778fcbb41e9e1)
+
+commit 72716f058cd132d836c3251b4e6d4b056bb4c0a8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 15:12:54 2017 -0400
+
+    mgr/balancer: crush_compat: cope with 'out' osds
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2bba5820a0bf2225dfc9ba39cc67c78083aba11)
+
+commit c982848f32ed85639346931e86d331fb52458355
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 15:12:36 2017 -0400
+
+    mgr/balancer: stop if we get a perfect score
+    
+    ..and fix the 'worse' threshold (we're just working around floating
+    point imprecision; 1.01 was too coarse).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f1f3014ffcc67200c20e6fb2b2e6e8f1141a0a1d)
+
+commit 37ed6f218ba4fafd5360b8f62feb321a1feec56f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:40:36 2017 -0400
+
+    mgr/balancer: more dead code
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 37bd64405b8a578329b9a903c9a53322ea29caaf)
+
+commit b3dd7abc79fa3dff7768898caf814bcdfb8f8690
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:27:42 2017 -0400
+
+    mgr/balancer: crush-compat: throttle changes based on max_misplaced
+    
+    Take smaller steps if we overshoot max_misplaced.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit df2e008d3388b657d62ea4547e6b1b7a0251aa04)
+
+commit 25bfd12cf252dbba81f482fe9abe53959f992b4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:19:30 2017 -0400
+
+    mgr/balancer: remove dead code
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 020be37fd06a75ec86e05af966be318c2199da9f)
+
+commit 1aff9dfadc148fb1cf9b60400c35bd11011772b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:19:23 2017 -0400
+
+    mgr/balancer: include pg up mapping in MappingState
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f6f2d253c379a6235e810808e7ce3b83cc4e212c)
+
+commit 9121e73f639bd47310ee29dce69a4c105582a3ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:36:06 2017 -0400
+
+    mgr/balancer: normalize weight-set weights to sum to target weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 99fcd161bd222a4f0d2ddde0f1a4caecd1a78882)
+
+commit 619ae53354aa5682aeb7d77d3795a323188dad1a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:35:42 2017 -0400
+
+    mgr/balancer: note root id in Eval
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26a7ec0c8a4c215791369d8650cd17d41317aefe)
+
+commit afcce93591c7e5c53596cec03165033f049a0c63
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:00:08 2017 -0400
+
+    mgr/balancer: make crush-compat mode work!
+    
+    - it does multiple iterations, like the upmap optimizer.
+    - it decreases the step size if it isn't improving, in the hope
+    that it is overshooting the minimum
+    - debug output is cleaned up a bit (the info level should be
+    genuinely useful)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d9a31595ba15de5fda104a0154778e3200fc46a0)
+
+commit c6052f2706ad6b7c92f9d6fa381527e8922b30c6
+Merge: 4e1eec777c 56359c367d
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:54:50 2017 -0500
+
+    Merge pull request #18590 from liewegas/wip-prime-fixes-luminous
+    
+    luminous: ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+
+commit 4e1eec777cf194cdf9bd4db1e7c4672d5b2adac7
+Merge: 79736e781f 338af16881
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:54:18 2017 -0500
+
+    Merge pull request #18620 from tchaikov/wip-21527-luminous
+    
+    luminous: mon/mgr: sync "mgr_command_descs","osd_metadata" and "mgr_metadata" prefixes to new mons
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79736e781fbd36083c639d3f4c22f8f3ae426cce
+Merge: 815d8ab573 9d8e5d763b
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:53:16 2017 -0500
+
+    Merge pull request #18621 from tchaikov/wip-21534-luminous
+    
+    luminous: mon: update get_store_prefixes implementations
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 815d8ab573215277a5283af1ecabaac84eaf6200
+Merge: c23865f07e 6b6eceee02
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:52:32 2017 -0500
+
+    Merge pull request #18623 from liewegas/wip-pr-18585-luminous
+    
+    luminous: ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c23865f07e9147f8ac8a8d3837f735558af30dba
+Merge: 24bd1de6af 796e336470
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:58 2017 -0500
+
+    Merge pull request #18625 from tchaikov/wip-ceph-disk-unlock-dmcrypted-partitions-luminous
+    
+    luminous: ceph-disk: unlocks dmcrypted partitions when activating them
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 24bd1de6afaa0d3dab763ae946b03ffb7b333430
+Merge: 240edcfb18 3c3609b7e3
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:32 2017 -0500
+
+    Merge pull request #18650 from b-ranto/wip-allow-getattr-luminous
+    
+    luminous: selinux: Allow getattr on lnk sysfs files
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 240edcfb18475bef84bd5d6345dd365411398773
+Merge: 98edc3b9f2 deb78542ed
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:11 2017 -0500
+
+    Merge pull request #18675 from jcsp/wip-luminous-mgr
+    
+    luminous: bulk backport of ceph-mgr improvements
+
+commit 98edc3b9f2012db277c5796055223f6e69b67e66
+Merge: 96102df540 0c0d8e6f40
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Nov 2 13:39:05 2017 -0400
+
+    Merge pull request #18687 from ceph/backport-18656
+    
+    luminous: "ceph-volume: adds functional testing for bluestore"
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 96102df5405fc470696df84996853e254c29fd31
+Merge: 172e6c1e0c 2e6b29084a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Nov 3 01:02:26 2017 +0800
+
+    Merge pull request #18293 from smithfarm/wip-21795-luminous
+    
+    luminous: Ubuntu amd64 client can not discover the ubuntu arm64 ceph cluster
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0c0d8e6f402ae78ae8110a717188f8b366a67696
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:25:23 2017 -0500
+
+    ceph-volume tests: adds objecstore as a testing factor
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f8b114be61d033c853a2e8295b966e30c6a5f721)
+
+commit 9d970edddc1ab453a683bff1dfd19cab46e8cf9a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:10:28 2017 -0500
+
+    ceph-volume: enable the centos7-bluestore and xenial-bluestore tests
+    
+    This also updates the ansible and testinfra version to match the current
+    ceph-ansible master branch.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b49a19a9773599de295197979b498d06ca500cf8)
+
+commit f9554c4e3ef0eea1f4f09828f98c58838390d4fa
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:09:52 2017 -0500
+
+    ceph-volume: adds the xenial-bluestore testing scenario
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 38372890c9c40e6f536026f61c5ef6e6d23d7111)
+
+commit b7fef119d0730a3cb8f19a461f773e66ce97af0f
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:05:46 2017 -0500
+
+    ceph-volume: adds the centos7-bluestore testing scenario
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 45e4b0c9d7d4fea1d7bd3481d46edf3b42c40d26)
+
+commit deb78542ed45ac402b4492d5b9ae4a9cdf182724
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 2 07:42:56 2017 -0400
+
+    qa: fix mgr _load_module helper
+    
+    I inadvertently broke this with the latest change
+    to the module ls output.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4fb3025682d812f86c50fa36532290fc0f8857ae)
+
+commit d4fcb97839a501094db2e1704c9799a107ad100e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 19:10:19 2017 +0000
+
+    mgr: fix up make_unique usage for backport
+    
+    This was getting the definition some other way in master,
+    but in luminous we need to include the backport14 header.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit f41cbb72f39359d7efc9feb0825cbad61e51c07c
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 20 13:54:29 2017 +0100
+
+    mon: fix up mgr_inactive_grace for backport
+    
+    This was converted to options.cc only, but we cannot
+    backport another commit that removes this legacy health
+    reporting code, so this commit updates the legacy health
+    reporting code for the options.cc style config opt.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit a80a7a4d7f9ac73b4d3b86f9787dd9de9865530a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:35:14 2017 -0400
+
+    mon: include disabled modules in `mgr module ls`
+    
+    Otherwise, when someone wants to see what's possible
+    to do with `mgr module enable` they have to trawl
+    through the whole mgr map dump.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5861c4c022e825fe28347ba7e61ae86a6f260f24)
+
+commit fad998cb11628db970099cc192277c1824be884a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:34:36 2017 -0400
+
+    doc: describe using `mgr module ...` commands
+    
+    ...including the new "mgr services" command.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d220e1adc337daeac418563dee125f161e85afdf)
+
+commit a46c5733881403f65872ca36d300ab6a4ca9d2cc
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:28:30 2017 -0400
+
+    doc: describe how to implement standby modules
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ceb2a91190441b19171e5b5726748e43ee4d1e92)
+
+commit c704b344a4a739e5ba18535a0fd4f05308c89019
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 23 05:37:49 2017 -0400
+
+    qa: fix mgr caps
+    
+    This was still using Kraken era settings
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9988ebed9530718b523a23d729d499dedab5eb6d)
+
+commit f58cb7035cf8992b8772f3358f2d67787ab585c3
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 19 07:50:19 2017 -0400
+
+    qa: expand mgr testing
+    
+    Some extra coverage of the dashboard, including its standby
+    redirect mode and the publishing of URIs.
+    
+    Also invoking the command_spam mode of the selftest module.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 05e648be6a51d3efa110ad9826bbdd0adcc4dd4d)
+
+commit a067468b3717dbab20f84996e76e2336d5fb5f30
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:39:17 2017 -0400
+
+    mgr/selftest: extend test and add background spam mode
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a382c3f1ca98b9fb7300e2d410bb2a1bb10b35ae)
+
+commit 1430e448ff93f73b8e8fba61f591ce4ec8c02380
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:16:22 2017 -0400
+
+    mgr: drop GIL around set_uri, set_health_checks
+    
+    These didn't need to keep the GIL to go and do their
+    pure C++ parts, and by keeping it they could deadlock
+    while trying to take ActiveMgrModules::lock.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 27ee148e040ebaf512f8e11f814b3a7c8cf21f8b)
+
+commit 2a94381b53241f9ecb98c4cc605ef6547b910ca2
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:14:43 2017 -0400
+
+    mgr: fix ~MonCommandCompletion
+    
+    This was doing a Py_DECREF outside of the Gil.
+    
+    Fixes: http://tracker.ceph.com/issues/21593
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 58dfa97ba88882fb3540d15e31bcac48a1aef5ef)
+
+commit 045ed0e023ae2d1a567b3426ef2b48980a851b7e
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 16 10:51:34 2017 -0400
+
+    mgr: update for SafeThreadState
+    
+    A bunch of the previous commits were done
+    before this class existed, so updating in
+    one go instead of trying to edit history
+    in fine detail.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29193a47e6cf8297d9b1ceecc7695f2c85434999)
+
+commit ef67307966575b618614225d6206019781d71647
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 13 11:31:22 2017 -0400
+
+    mgr: refactor PyOSDMap etc implementation
+    
+    Implement real python classes from the C side,
+    rather than exposing only module methods.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e61f79f5d56b568103a067d9a1eb87af997ad61)
+
+commit d85f91cf3e2f747cc23218a52d9271dd6e27febb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:35:29 2017 -0400
+
+    mgr/PyOSDMap: add CRUSH get_item_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eacc9021459b31e42232bb958536d594d03b07b3)
+
+commit b1307e074b5fb708420686bf814bd4a7aa76f6d8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 16 06:33:48 2017 -0400
+
+    mgr: fix py_module_registry shutdown
+    
+    Was calling way too early, which did a
+    Py_Finalize before the modules had been
+    joined.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0d5b1d70e616d7d1c2d6360375770f5c4754649d)
+
+commit 642a26dfcd9bef06fceb119008f5c3078bbf10e7
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 12 13:14:02 2017 -0400
+
+    mgr: fix thread naming
+    
+    Was passing a reference to a local stringstream into
+    Thread::create, not realising that it was taking a char*
+    reference instead of a copy.  Result was garbage (or usually,
+    all threads having the name of the last one created)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bb4e71ed2ebdee1ac5e4b3eee390060e19fea0d8)
+
+commit f9223ad56d8e01c1f6e8690a6a186d02eed8d96d
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 6 11:02:44 2017 -0400
+
+    mgr: cut down duplication between active+standby
+    
+    ...by using PyModuleRunner class from ActivePyModule too.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit df8797320bed7ad9f121477e35d7e3862efd89bd)
+
+commit 728e1e4e136aab5b4d185c738de8df7bcdce18ee
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 4 13:13:25 2017 -0400
+
+    mgr: fix os._exit overrides
+    
+    These would throw an exception when passed
+    a status code.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e2442c1e20bf4ff12d58af500b34a18cc60d2de1)
+
+commit ca51a1b9d2cf0a3bf0fd7b69b833fa6696e61888
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 24 14:07:37 2017 -0400
+
+    mon/MgrMonitor: reset services map on drop_active
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7b629ae46599d79ca1929cfc6637b367c6bb9029)
+
+commit 8d4512b5636eb41b2cbb50a42d071580cff728c6
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 14:47:10 2017 -0400
+
+    mgr/dashboard: implement standby mode
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4f7007d1b0226af3f0cc33627ebf5051975657ac)
+
+commit 3953c0b3b7d0012507ac047085a3f863dc4929d3
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 11:41:26 2017 -0400
+
+    pybind/mgr: add MgrStandbyModule
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3048e85cd712b7da77cf6ac55dd6a689d00e47e5)
+
+commit e37df9459d6d9abdbe310153bd53d1c1c4384a6b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 14:42:11 2017 -0400
+
+    mgr: standby modules come up and run now
+    
+    ...they still don't have access to any config though.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c1471c7501948004096581ee415ab4a1fa2d9379)
+
+commit 977583eba266df834a7e27516cacb750bf7121ec
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 16 10:23:59 2017 -0400
+
+    mgr: enable running modules in standby mode
+    
+    Modules can implement a second, separate class
+    that has access to very little state about the
+    system and can't implement commands.
+    
+    They have just enough information to redirect
+    or forward incoming requests/traffic to the
+    active instance of the module on the active mgr.
+    
+    This enables module authors to create modules
+    that end users can access via any (running) mgr node
+    at any time, rather than having to first work out
+    which mgr node is active.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 25566d1edca638bd15b3ba3326ee7e4d3e573cbb)
+
+commit 6a35a96ea838760945073b756ea7ae13e9a68ccf
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 15 06:53:18 2017 -0400
+
+    mgr: clean up python source file naming
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 70d45a6b93c92baf8d6a3b15765110a5384c5e60)
+
+commit d37bf83f419fd5b34f0cbc8ee5313425e5177bc8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 14 06:31:18 2017 -0400
+
+    mgr: refactor python module management
+    
+    Separate out the *loading* of modules from
+    the *running* of modules.
+    
+    This is a precursor to enabling modules to run
+    in standby mode.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9718896c8b844db2f3c07df1d344636da4605e61)
+
+commit b563555edb51b576fcbf94a3b92598a7300aaf09
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 13:49:27 2017 -0400
+
+    pybind/mgr: use set_uri hook from dashboard+restful modules
+    
+    No more guessing the URL!
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 089e105dd7ec762572ac06794caa7f5543075001)
+
+commit 6dd4d0504de9806fcf7c398a8d66b956b45986d9
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:50:23 2017 -0400
+
+    mgr: enable python modules to advertise their service URI
+    
+    Fixes: http://tracker.ceph.com/issues/17460
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a0183a63fa791954d14c57632e184858cefe893d)
+
+commit 778322d0913d4d9c70609a3c40d809f62a366020
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:49:45 2017 -0400
+
+    mon/MgrMonitor: store services in map and expose with command
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c3c3e4e90ba6b09e29879b500f211d607ebabb53)
+
+commit a6b87a8e7c859607d4554833c5f9ad4cb2fea187
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:46:40 2017 -0400
+
+    messages: `services` in MMgrBeacon
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 236841b3b62af92ce0c4852045327fcfbc5c1651)
+
+commit b44cf5f70d0286c9dcdcda8dbb9f014a3386813a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:45:53 2017 -0400
+
+    mon/MgrMap: store list of services
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3f703bd91f07b2fe43a16df0083d7b7c23803fd5)
+
+commit 5b677f3f02500b87757b9965d20151e345b8ba3f
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 06:31:01 2017 -0400
+
+    mgr: carry PyModules ref in MonCommandCompletion
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e938bf9b9d27e192765c805e5f532c9dd4808b21)
+
+commit b69a656ac76fe38431bcd66c626883e324be1ba4
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 12:31:13 2017 -0400
+
+    pybind: update MgrModule for ceph_state->ceph_module
+    
+    & tidy up the places where ceph_state was getting
+    used outside of MgrModule.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 62cb512e4740f1f78f516b4f2179c1123fae1b36)
+
+commit 8b5f302706e91c5c6a88e918a9d4baafd726c211
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 07:44:00 2017 -0400
+
+    mgr: refactor python interface
+    
+    Expose a python class instead of a module,
+    so that we have a place to carry our reference
+    to our MgrPyModule* and to PyModules*, rather than
+    passing a handle for the former and using
+    a global pointer for the latter.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 563878ba217491dd0a6fbd588cd56d09e3456c14)
+
+commit 35b4518c4a0c477f0190bfc006434093fe720c05
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 3 06:22:35 2017 -0400
+
+    mgr/dashboard: remove blue highlight on scrubbing pg states
+    
+    This was kind of unnecessary, highlighting a completely normal
+    and healthy situation in a different colour.  The blue was
+    also really hard to read against a grey background.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 99fa1fdf4e1be57792f50907147781d12009b32b)
+
+commit 764e7011d0e69e08aa5f56a5c8c56b8875820b7e
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:42:16 2017 -0400
+
+    mgr/dashboard: clean up fs standby list when empty
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5e64787c0ae0ac2a365c89bf89dfea425adc17d4)
+
+commit 1d1dce056664a40ea4dc9ebd1dc3826f0e449555
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 13:56:39 2017 +0100
+
+    mgr: remove old-style config opt usage
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ec09a7abc515f802451bf7ef3d22ce8ee6c6c7b3)
+
+commit a0131144036a9ea64c6f8e3289562bf7db18c746
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 12:12:40 2017 +0100
+
+    mon: remove old-style mgr config opt usage
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6af4120d63324150ba19022c41fe4fa8a38cacbb)
+
+commit 526425c63b6a8f2d3f79fdd75c360d7c2ecfeec6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 11:48:25 2017 +0100
+
+    common: populate manager config option metadata
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit eba4c3f2762ae40ba746091e32364c2d68e780d9)
+
+commit 1eee973b90418979c33ca42b0720045f2de670c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jul 13 14:49:48 2017 +0800
+
+    common,mds,mgr,mon,osd: store event only if it's added
+    
+    otherwise
+    * we will try to cancel it even it's never been added
+    * we will keep a dangling pointer around. which is, well,
+      scaring.
+    * static analyzer will yell at us:
+      Memory - illegal accesses  (USE_AFTER_FREE)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2449b3a5c365987746ada095fde30e3dc63ee0c7)
+
+commit 66aa481be983fdd60beddc7ce621dad3edbf5490
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 3 08:16:10 2017 -0400
+
+    mgr: safety checks on pyThreadState usage
+    
+    Previously relied on the caller of Gil() to
+    pass new_thread=true if they would be
+    calling from a different thread.
+    
+    Enforce this with an assertion, by wrapping
+    PyThreadState in a SafeThreadState class
+    that remembers which POSIX thread
+    it's meant to be used in.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 625e1b5cfb9b8a5843dfe75e97826f70a57d6ebe)
+
+commit 385a6a0e8e6a6dd315ab358dcea7f3c069d9ad4d
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 11:38:25 2017 -0400
+
+    mgr: move Gil implementation into .cc
+    
+    The inclusion of Python.h in the .h was awkward
+    for other files including Gil.h.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 23c3a075ee1a27e1b57fcb452a4d6ce53080264e)
+
+commit 241d655e127c71e9d9012d9205005c97d61def8a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 07:21:40 2017 -0400
+
+    mgr: reduce Gil verbosity at level 20
+    
+    Even at 20, it's pretty heavy to be logging
+    every lock acquire/release.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 987612a97529be7e67b89977c4a0cf47906a5ecb)
+
+commit 30db4f5a54eac28e546c43a2ce099aa89d179495
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 12:28:19 2017 +0200
+
+    pybind/mgr/prometheus: no ports in osd_metadata
+    
+    Ports might change on a OSD restart and this would create a new metadata
+    metric for this osd.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 48fec7db4b214fe8ef6a04f8cb53fb8a2fb9c2ca)
+
+commit 80fc65cafaad3bb5499a15edbd97ecf483b4ad82
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 10:59:33 2017 +0200
+
+    pybind/mgr/prometheus: add osd_in/out metric; make osd_weight a metric
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit e4c44c1d702ce242f2cb9a58ca7ce1c31fe0a498)
+
+commit db09ea1c75c80dc189042c59e09ebc1c1f8d1249
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 20:07:19 2017 +0200
+
+    pybind/mgr_module: move PRIO_* and PERFCOUNTER_* to MgrModule class
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit f69484debade5f4fa2bd3a0d1badc9291cc9d7b7)
+
+commit bbe8fbe9ca98c1b0c8b5b43c0296fd9cc4d09c77
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 9 12:10:22 2017 +0100
+
+    qa/mgr: fix influx/prometheus test names
+    
+    This was a typo: they were swapped around.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d96a59e74b6984b77c9f3b15f702e3bf45053590)
+
+commit be207ede6028e515a0a632e12330848c93d04a34
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 28 10:50:53 2017 -0400
+
+    doc: flesh out prometheus docs
+    
+    Explain ceph_disk_occupation, importance
+    of instance labels and honor_labels, provide
+    example prometheus configuration yaml.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5227afed5f33fa9487e1bfa3fd8ce0d82eb4a20f)
+
+commit 8a9e1cb00110ed73f492f1f6cf23b3595756efc4
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 28 10:10:14 2017 -0400
+
+    mgr/prometheus: add ceph_disk_occupation series
+    
+    This is the magic series that enables consumers to
+    easily get the drive stats that go with their
+    OSD stats.
+    
+    Fixes: http://tracker.ceph.com/issues/21594
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 284be75524f7125dc1409b9c05fe47b37484964e)
+
+commit 51d74e41223397abcde006a4ce53e693a2125852
+Author: Benjeman Meekhof <bmeekhof@users.noreply.github.com>
+Date:   Wed Oct 4 10:05:17 2017 -0400
+
+    mgr/influx: Correct name of daemon stat measurement to 'ceph_daemon_stats'
+    
+    Signed-off-by: Benjeman Meekhof <bmeekhof@umich.edu>
+    (cherry picked from commit f9014a1c75c6a3adf414b48a707fd444e65b3024)
+
+commit 2934dda9133a3a8876ab2701b11d4379440a5e41
+Author: Benjeman Meekhof <bmeekhof@users.noreply.github.com>
+Date:   Tue Oct 3 16:30:43 2017 -0400
+
+    mgr/influx: modify module database check to not require admin privileges
+    
+    - existing check tried to list all DB and fails even if DB exists if user is not admin level
+    - still tries to create database if not found and user has privs
+    
+    Signed-off-by: Benjeman Meekhof <bmeekhof@umich.edu>
+    (cherry picked from commit 06d7d37c7b9a8c3f4435eff04b6f4934be5e676f)
+
+commit 8c816b8e0fc1fd3fe227690287bcc6e6fce7c54d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Tue Oct 10 08:40:31 2017 +0200
+
+    pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 6306392492d103200b21ea91bce10a315d7c4e16)
+
+commit 2ebab2f19cdd44f4c568eb96388fc4296f0f1814
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 11:14:57 2017 -0400
+
+    mgr: respect perf counter prio_adjust in MgrClient
+    
+    This awkwardly involves re-ordering some definitions
+    in perf_counters.h in order to refer to the prio
+    names defined in PerfCountersBuilder.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 88163749b572ffd2bfe0850136fad5dbed2a9180)
+
+commit f8e9c37286d37d43bb768531f9cfbf70f2cb06b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 09:06:13 2017 -0400
+
+    test: update perfcounters test for priority in output
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f531f7871a68db96b2fb66ffdf6fae6935e6107)
+
+commit f073fc4663db17ea44e9c36831d21df1597d193a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 17:16:54 2017 -0400
+
+    qa: add mgr module selftest task
+    
+    The module self test commands give us a chance to
+    catch any other ceph changes that change something
+    that a module was relying on reading.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 99352ceced9d0fe92ddad6b97b1393b41de75d50)
+
+commit 4c22f0f2669d343a8e7e83f0bc2a2dacbe194f34
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 10:46:56 2017 -0400
+
+    mgr/prometheus: remove explicit counter list
+    
+    These have had their priorities bumped up to
+    USEFUL, so they'll appear in the default
+    get_all_counters output.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ad5a31efbea8081f03dd73669e891d03857ef9cc)
+
+commit fec2b3abf9d9abdc12bfeedaf2b22e8b0cc8c9f4
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 10:45:21 2017 -0400
+
+    mon: elevate priority of many perf counters
+    
+    We can be quite liberal here, because mons are
+    small in number.  However, we don't want to expose
+    KV database counters at this database from OSDs, so
+    use the prio_adjust mechanism for that.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ac8320f23dd4c00eb80da0d9837c29744e38bd57)
+
+commit 8b9a18d3a0c40cb0ef10e32018ea1b23ff53f51e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 07:07:50 2017 -0400
+
+    osd: upgrade a bunch of perf counters to PRIO_USEFUL
+    
+    These are broadly the OSD-wide IO stats, which happen
+    to also be the ones that were named in the
+    prometheus plugin until I changed it to be
+    priority-based.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a1cc4ba2993de62b60fd1e58a9704877a6da5fe4)
+
+commit 278188c76ebdbaaa36b3d185b1378abf7665f3fc
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 07:06:24 2017 -0400
+
+    common: PerfCountersBuilder helper for priorities
+    
+    Let the caller set a priority as the defaul, to enable them
+    to create a bunch at a given priority.  This is just a
+    convenience.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 66f61eeda6a2465b5fc0e40a4f1300913db065dc)
+
+commit 3dff5c0f39289e02e882bf86bf3e23e94ee33aa8
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 10:27:12 2017 -0400
+
+    mgr/prometheus: add a self-test command
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 76e1ba52b1b95d417cdd04b8fe985acee648f0e9)
+
+commit a6bc96dfe93f7cbcefa8030a3b6830117516931e
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 08:05:28 2017 -0400
+
+    mgr/influx: remove file-based config
+    
+    ...and also trim down the configuration to what's really
+    needed.  In general users don't need to pick and choose
+    metrics.  We could add it back if there was a strong
+    motivation.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6776d4645afc49a4bfb4b62673c91384239037f4)
+
+commit bda26fe0e707ec9e609be613805340dadd17e8f3
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 06:51:21 2017 -0400
+
+    mgr/influx: enable self-test without dependencies
+    
+    The idea of self-test commands is that they're self
+    contained and just exercise the module's calls
+    to the Ceph-side.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 125294ab9d6e99aa4c960fea147a4e86624b869e)
+
+commit 9abd779d3321d1cda9c8677600e1e35af46109e7
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 06:18:15 2017 -0400
+
+    mgr/influx: revise perf counter handling
+    
+    - Use new get_all_perf_counters path
+    - Consequently get counters for all daemons, not just OSD
+    - Tag stats with ceph_daemon rather than osd_id, as some
+      stats appear from more than one daemon type
+    - Remove summing of perf counters, external TSDB and/or queries
+      can do this.
+    - Remove mgr_id tag: this would change depending on which
+      mgr was active, which is certainly not desirable.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 59b48e7660f4b757804974835027cd08a59843c2)
+
+commit 32f5f0fe515e6890e8aefad6ad8cdc2574ae8a34
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 3 13:00:56 2017 -0400
+
+    mgr: omit module list in beacon logging
+    
+    This is useful in itself, but awkward when dealing
+    with logs generally, because it means that when you
+    grep on the name of a module, you get mostly beacon
+    messages rather than the log messages from the
+    module.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d1277fa5c578ce0ea23a70cc58c6cf99921ee25)
+
+commit 10ab4f8b6821e2d6593bc09161c23a9163b93611
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 05:42:23 2017 -0400
+
+    mgr: define perf counter constants in mgr_module
+    
+    So that modules can consume perf counter data
+    intelligently without having to hunt around
+    in C land for these constants and redefine them.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 39ab28ed47e869e1466cb3a316a2cb11bdedd23a)
+
+commit 290d15ed912285803a40d02308b921b415c580a2
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 11 09:12:25 2017 -0400
+
+    ceph.in: use PRIO_INTERESTING as daemonperf threshold
+    
+    Using PRIO_USEFUL as the threshold for what goes into
+    time series databases.  I'm claiming that we have
+    more "useful" counters than fit on the screen,
+    so daemonperf's "a screen's worth" threshold
+    should be at the "interesting" level.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 30a74ce343caec2a433cb532ba697fe7013ed05c)
+
+commit 5ee9e15858d5aa2a71a89c13dea9a5db72579726
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 11 09:12:01 2017 -0400
+
+    mon: set some priorities on perf counters
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29a71c35c39fbe1d4887e3f5ebb93232daab3487)
+
+commit 3bd478794e2626251eda91850974188da208b591
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 4 05:39:11 2017 -0400
+
+    mgr/prometheus: tag stats by daemon name
+    
+    Using osd=0 or similar tags was problematic because
+    daemons of different types have some same-named
+    counters (e.g. MDS and OSD both have objecter
+    perf counters).
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit eb524c272c89f8f99f22969b78caa016db7c671e)
+
+commit af92c011b20eaef073730be2f8d7112a0f7e99f8
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:02:37 2017 -0400
+
+    mgr/prometheus: use new get_all_perf_counters interface
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 11137aa269271ad15dcf19a8d51ce6f4acb7a98e)
+
+commit 3ce4a950975188051b6a3e6cb366c2fb23f0e88f
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:01:35 2017 -0400
+
+    common: used fixed size int for perf counter prio
+    
+    ...to avoid any ambiguity in allowed range and
+    make clear how to encode it down the wire.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ba08fc1008d17aa7a5f285ea2705705ce1a0bda0)
+
+commit d4a0c778747970c231a55baea32f59a1009af4dd
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:00:59 2017 -0400
+
+    mgr: transmit perf counter prio to the mgr
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f304f84cfbc22c1a54d152cc38227077bc564a7e)
+
+commit cdcac6e92ff57f1a4e5f5cac676049c6584452c4
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 10:46:56 2017 -0400
+
+    common: always include priority in perf counter dump
+    
+    JSON output with inconsistent sets of members is
+    annoying to use on the receiving side.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e631f1a72735ec618e2f3012ad7b9c5830d6c0eb)
+
+commit 802cd49bae2cce6c55eddd68feb617df893e1046
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 29 11:55:28 2017 -0400
+
+    mgr: add get_all_perf_counters to MgrModule interface
+    
+    This is for use by modules that dump counters
+    in bulk, e.g. to a TSDB.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9a42d4255d9d968d6162b53b71db292d9d3de2e4)
+
+commit 85a93dcdddc4cd82c8935bfe123cb20f13f8928a
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 13:09:24 2017 +0200
+
+    pybind/mgr/prometheus: export cluster-wide pg stats, not per osd
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 13b1236b96d4563e0985cad40d3009b60cc475e7)
+
+commit 1d89c0809f64009d646b929fec675e1aaa6c2b1a
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:51:47 2017 +0200
+
+    pybind/mgr/prometheus: add more osd metadata
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit e7704fa9cc35549dba526212c2830df589670416)
+
+commit 6a65408c1364669916643329494825aae394b200
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:05:09 2017 +0200
+
+    pybind/mgr/prometheus: don't get perf counters that are not in schema
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit d4ba07d04477ccae3a89dcdcafbb7e76149dfd1c)
+
+commit 62092680ad966eabcdc6b557e9858fe0562147ee
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:04:28 2017 +0200
+
+    pybind/mgr/prometheus: add mon and osd perf counters to export
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit fa25d31263a26074225e2a00cb82448066b54069)
+
+commit 752c888533aaa9c7ae5781fafe4a6aa3302059fc
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 19:46:07 2017 +0200
+
+    pybind/mgr/prometheus: add index page, export metrics under metrics/
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit d99a506ed37c2d0991d68ecd34ac5fb213a3eea4)
+
+commit b279ae25d95f6c4b8290042fc9f33c4a6f7afac2
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:19:42 2017 +0200
+
+    pybind/mgr/prometheus: export selected perf_counters
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit f6e2e36ba72caf6347f3bb6a985925d0e35077a2)
+
+commit 4c092524929db048f6863acf1a9f12281bdc1646
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:18:36 2017 +0200
+
+    pybind/mgr/prometheus: export osd and pool metadata
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 2bea3814699c27baa8f633b56a8800d697685898)
+
+commit e3dafc3dd682a10e2310efdac4615d834933e7d6
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:15:56 2017 +0200
+
+    pybind/mgr/prometheus: actually emit reported pg counts
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit c288624eed862559b2c86c5dfc85c837716739ab)
+
+commit 1ef5c88ef08ab9950ee8d7d4acda032b5a54f2f5
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:09:17 2017 +0200
+
+    pybind/mgr/prometheus: no need to wait for notify event
+    
+    If stats or perf counters are not available they won't be emitted.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit ead0973d7dd12fe985390891c80f1bc15f7b9aec)
+
+commit d0ef1cd2570644451205e5306feca0da94462d1f
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:07:14 2017 +0200
+
+    pybind/mgr/prometheus: no need to convert perf_schema to ordered_dict
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 5e4b4b5ea2a217731691c1c391c252b08452798a)
+
+commit 7f191ff9dff33cac551ae5f64027d566b17b6d98
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Aug 9 17:22:49 2017 +0200
+
+    pybind/mgr/prometheus: add device_class label to osd metrics
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 76d1918724320b7d6b1120b57b3002bb24099001)
+
+commit b24263387e46ac537a8ac189f9e0e80699518e0b
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Aug 9 16:19:38 2017 +0200
+
+    pybind/mgr/prometheus: add cluster wide metrics; no perf counters for now
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 49b3ff83cd231066d2a8f1809fadbdeb2c0c1f88)
+
+commit 5f0ef3a73a55a3f24997e415a848308afa0263ba
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 4 10:23:11 2017 +0200
+
+    pybind/mgr/prometheus: prefix metrics with 'ceph'; replace :: with _
+    
+    Both follow prometheus best practices. While : is a legal metric
+    character, "Exposed metrics should not contain colons, these are for
+    users to use when aggregating."
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 177afcc7886aa3898d092ebd1e101697bc6539fd)
+
+commit a303218c85498ccf472d2f1b35fd391376fb0faf
+Author: mhdo2 <mhdo2@users.noreply.github.com>
+Date:   Mon Aug 21 12:13:01 2017 -0400
+
+    doc/mgr: add influx plugin docs
+    
+    Signed-off-by: My Do <mhdo@umich.edu>
+    (cherry picked from commit e345fe3c5780976a4e33488b3a75cd24bb2c96c5)
+
+commit de89603e54f550ffc4ff994026f32c5b7d5f6529
+Author: mhdo2 <mhdo2@users.noreply.github.com>
+Date:   Tue Jul 18 18:33:55 2017 -0400
+
+    mgr/influx: added influx plugin
+    
+    Signed-off-by: My Do <mhdo@umich.edu>
+    (cherry picked from commit 68ae26c014d0471cc3f2f979dc8d822b2e50740f)
+
+commit 2fea47d9710ec1e388db044065bc42b8046dae22
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 11:55:55 2017 -0400
+
+    mgr: store declared_types in MgrSession
+    
+    Because we don't (yet) properly prevent multiple sessions
+    from daemons reporting the same name (e.g. rgws), storing
+    it in the DaemonPerfCounters meant that one daemon's report
+    was referring to another daemon's set of reported types.
+    
+    This should always have been a property of the session.
+    
+    The behaviour will still be ugly when multiple daemons
+    are using the same name (stomping on each other's stats/statsu)
+    but it shouldn't crash.
+    
+    Fixes: http://tracker.ceph.com/issues/21197
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dc415f1ae09a308bd448614934a4c168eb9cf07b)
+
+commit 951d67fc9fee23f3f8f7d7df8fbfb8c29e60f82f
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 10:12:00 2017 +0100
+
+    mgr: make pgmap_ready atomic to avoid taking lock
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d20915741d985e080a723cd6563bc6f4a657276f)
+
+commit 946d1541bd2b14d6f8abbaa201c3521dd117f01b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 07:29:36 2017 -0400
+
+    mgr/DaemonServer: handle MMgrReports in parallel
+    
+    The DaemonStateIndex locking is sufficient to make all
+    the report processing safe: holding DaemonServer::lock
+    through all ms_dispatch was unnecessarily serializing
+    dispatch.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 64af9d3da0fceff9ad0ff668f60d272c46912f34)
+
+commit d70fae092db920e2db77c7c8b044cfb4d9687992
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 24 12:53:24 2017 -0400
+
+    mgr: clean up DaemonStateIndex locking
+    
+    Various things here were dangerously operating
+    outside locks.
+    
+    Additionally switch to a RWLock because this lock
+    will be relatively read-hot when it's taken every time
+    a MMgrReport is handled, to look up the DaemonState
+    for the sender.
+    
+    Fixes: http://tracker.ceph.com/issues/21158
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 806f10847cefe5c7a78fc319b1b130d372197dd3)
+
+commit b86cc9a2e49a4d1b53e83f47876a42b42819efcb
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 31 12:13:23 2017 -0400
+
+    mgr: runtime adjustment of perf counter threshold
+    
+    ceph-mgr has missed out on the `config set` command
+    that the other daemons got recently: add it here
+    and hook it all up to the stats period and threshold
+    settings.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 057b73d641decb9403aba50caae9d139f3a34dd4)
+
+commit c278580d207f0b99daed48afdc689ae77fe73e9b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jul 31 09:24:09 2017 -0400
+
+    mgr: apply a threshold to perf counter prios
+    
+    ...so that we can control the level of load
+    we're putting on ceph-mgr with perf counters.  Don't collect
+    anything below PRIO_USEFUL by default.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bdc775fdd8acdad5c58ff3065a21396f80ce5db4)
+
+commit 11720b96ec8f9ef683710dffa3da4cda2ecf096d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 16:36:23 2017 -0400
+
+    pybind/mgr/balancer: make auto mode work
+    
+    (with upmap at least)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef1a3be05671ad31907cf8c4beb64a766359bc66)
+
+commit b6a000f0efef3cd6a143b82ae33cacf660cd8e4c
+Author: Spandan Kumar Sahu <spandankumarsahu@gmail.com>
+Date:   Mon Aug 7 04:01:57 2017 +0530
+
+    src/pybind/mgr/balancer/module.py: improve scoring method
+    
+    * score lies in [0, 1), 0 being perfect distribution
+    * use shifted and scaled cdf of normal distribution
+      to prioritize highly over-weighted device.
+    * consider only over-weighted devices to calculate score
+    
+    Signed-off-by: Spandan Kumar Sahu <spandankumarsahu@gmail.com>
+    (cherry picked from commit c09308c49ca087fb8c5e7d4261b0234190f863d9)
+
+commit 6090ae6c224904516736c5f1a4ae5bcb6d7e6caa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 4 17:59:20 2017 -0400
+
+    pybind/mgr/balancer: make 'crush-compat' sort of work
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a00e02acd1b2ff21dac829de30f80fd69eae602)
+
+commit add3cd36db7c02d52c2fa429c034c114cac526d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 3 16:23:08 2017 -0400
+
+    pybind/mgr/balancer: rough framework
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d5e5c68c374e7d5514f89aac2d3df6008d103a76)
+
+commit 0c73e433ab6583fca6eea7678c23b469d643ae04
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 23:33:06 2017 -0400
+
+    mgr/PyOSDMap: OSDMap.map_pool_pgs_up, CRUSHMap.get_item_name
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a928bf62316c32f37dd1791192fd9a2ddaef0d33)
+
+commit 423947563c8e88f89bef9d71b3116c3302bd40c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jul 23 00:10:56 2017 -0400
+
+    mgr/PyOSDMap: get_crush, find_takes, get_take_weight_osd_map
+    
+    These let us identify distinct CRUSH hierarchies that rules distribute
+    data over, and create relative weight maps for the OSDs they map to.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3b8a276c437cfd599c55a935d141375afda676ff)
+
+commit 073f23734c8058e3efcb440df89adab818510695
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 10:07:31 2017 -0400
+
+    crush/CrushWrapper: rule_has_take
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef140de639078b40c05971fb219f7b8c12d83228)
+
+commit df426b5c24e12b1156bccdd8948cbed5977c348a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:50:27 2017 -0400
+
+    crush/CrushWrapper: refactor get_rule_weight_osd_map to work with roots too
+    
+    Allow us to specify a root node in the hierarchy instead of a rule.
+    This way we can use it in conjunction with find_takes().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 69454e0570274ff7f252e7f081965dcc9bb04459)
+
+commit 89cac2d5176300838c23a28814257d0f395e39c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:17:18 2017 -0400
+
+    pybind/mgr/balancer: do upmap by pool, in random order
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 028a66d43244c15a77e71f3d3e4f41773837ab02)
+
+commit bfdc955ab6ab424b6a46dc1b0022fb3dc13ce157
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:27:08 2017 -0400
+
+    pybind/mgr/balancer: add balancer module
+    
+    - wake up every minute
+    - back off when unknown, inactive, degraded
+    - throttle against misplaced ratio
+    - apply some optimization step
+      - initially implement 'upmap' only
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d9685c50f79fbb53dbc8bd98c95900ef6e902b8)
+
+commit c57a55e8d426e6f87d4649a799d9ddee665618e0
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:26:16 2017 -0400
+
+    pybind/mgr/mgr_module: add default arg to get_config
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 39c42ddb9339c1950a3a474e8083db8b24e775a6)
+
+commit 4d686ee875d5bc0125426d348e044988aaca9f91
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 10 23:23:19 2017 -0400
+
+    mgr: add trivial OSDMap wrapper class
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2ef005196ba2eb49c34c32def624938c7a8beb03)
+
+commit 1b721632aa8271d74b319714db0a9fedacb2629b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 10:06:45 2017 -0400
+
+    mgr/PyModules: add 'pg_dump' get
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bfb9286f4212947183c46543d609b664ea13b489)
+
+commit 68d411cf7b934c724f1f8ebeaa7148daebd79599
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:25:42 2017 -0400
+
+    mgr/PyModules: add 'pg_status' dump
+    
+    This is summary info, same as what's in 'ceph status'.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 85b5b80906d00e098d4b1af1354c60a357022dd2)
+
+commit 35de92b259f50b52e51ab1daf57041553170406d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 1 09:53:34 2017 -0700
+
+    osd: build_past_intervals_parallel: Ignore new partially created PGs
+    
+    Fixes: http://tracker.ceph.com/issues/21833
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 172e6c1e0c5bb86db7cc0017acaa674aa893e3a0
+Merge: 2988a39abe 966683fde6
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Wed Nov 1 16:28:25 2017 -0400
+
+    Merge pull request #18674 from ceph/wip-rgw-s3-branch
+    
+    qa/tests: use ceph-luminous branch for s3tests
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 966683fde600837eb9a0f82ca6fd00f52e721809
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Nov 1 10:32:07 2017 -0700
+
+    qa: use ceph-luminous branch for s3tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 2988a39abedc6a092b8b3c2f0b33a3559a55461b
+Merge: 4dc0a0e38a 8f87fa2d3a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Nov 1 15:24:55 2017 +0100
+
+    Merge pull request #18412 from kmroz/wip-21659-luminous
+    
+    luminous: mgr: fix crashable DaemonStateIndex::get calls
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4dc0a0e38a3f1a988c180d47970102df4b326b9e
+Merge: 273e035dbe 72c8583107
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Nov 1 12:33:45 2017 +0100
+
+    Merge pull request #18113 from jcsp/wip-prometheus-port-backport
+    
+    luminous: ceph-mgr: can not change prometheus port for mgr
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 273e035dbe158f162c2ee559744c456c942f1737
+Merge: a7f519913d f463cd84ae
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Oct 31 14:45:27 2017 -0700
+
+    Merge pull request #18655 from ceph/wip-yuriw-bp-PR18634-luminous
+    
+    qa: add "restful" to ceph_mgr_modules in ceph-ansible suite
+
+commit f463cd84ae1fb38817078967a4a40ba4b600f0e5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 31 11:21:58 2017 +0800
+
+    qa: add "restful" to ceph_mgr_modules in ceph-ansible suite
+    
+    backport of https://github.com/ceph/ceph/pull/18634
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit caf9ee5c60d999951979d0b67afda8d56e1cd91d)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit a7f519913df3aba83aa3ead9eee6b94aa8f90ffd
+Merge: f2749114c5 b84803d609
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 10:26:42 2017 -0700
+
+    Merge pull request #17889 from smithfarm/wip-21372-luminous
+    
+    luminous: core: Improve OSD startup time by only scanning for omap corruption once
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit f2749114c596d4016eb14e4777cb93a11b14b705
+Merge: 528c3b6d32 1236e0f2e9
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 10:18:07 2017 -0700
+
+    Merge pull request #18004 from linuxbox2/luminous-rgwf-wr
+    
+    luminous: rgw_file: fix write error when the write offset overlaps.
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 528c3b6d32f9c3319fd429de37bbca578538ed7c
+Merge: ec2d294684 d800747fa1
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 09:03:48 2017 -0700
+
+    Merge pull request #18438 from theanalyst/wip-21696-luminous
+    
+    luminous: fix a bug about inconsistent unit of comparison
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 3c3609b7e3ce7339a845f9191a709d12025fbbb2
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Sep 21 17:24:07 2017 +0200
+
+    selinux: Allow getattr on lnk sysfs files
+    
+    This showed up during downstream testing for luminous. We are doing
+    getattr on the sysfs lnk files and the current policy does not allow
+    this.
+    
+    Fixes: http://tracker.ceph.com/issues/21523
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 394c26adb97cd150233fe8760355f486d03624a4)
+
+commit ec2d2946840fac65d9588f225adaa6f51c32f6b0
+Merge: 90017e7d93 def3d55eb7
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 31 13:04:00 2017 +0800
+
+    Merge pull request #18410 from kmroz/wip-21732-luminous
+    
+    luminous: qa/suites/rest/basic/tasks/rest_test: whitelisting
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 90017e7d933b52794d32fef69d97d5ba529d3936
+Merge: 4f024c6831 c603faaf17
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Oct 30 20:23:39 2017 -0700
+
+    Merge pull request #18629 from ceph/wip-yuriw-21978-luminous
+    
+    qa/suites/upgrade/jewel-x: Changed typo ('hammer' to 'jewel')
+
+commit c603faaf171fd3074ffe5e1342e8b96cd261e097
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Oct 30 14:52:52 2017 -0700
+
+    Changed typo ('hammer' to 'jewel')
+    
+    Fixes http://tracker.ceph.com/issues/21978
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 555746a09e8bb7bcf99da91f9d00dfab9791d80c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Oct 27 13:20:07 2017 -0700
+
+    MDSMonitor: wait for readable OSDMap before sanitizing
+    
+    Fixes: http://tracker.ceph.com/issues/21945
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit ca52f3bd93e6c743aa05171108527d877807b426)
+
+commit be13cdc0d4e54ce23c6ecf658100ddd860b1b96f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 3 12:25:12 2017 -0700
+
+    mds: clean up non-existent data pools in MDSMap
+    
+    Older versions of Ceph weren't strict about preventing pool deletion when the
+    MDSMap referred to to-be-deleted pool. If we are dealing with a cluster
+    upgrade, we should try to gracefully handle that by cleaning out data pools
+    that have been removed.
+    
+    Reproduced this by allowing CephFS pools to be deleted:
+    
+    diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
+    index 85c47c13da6..694b240cb9f 100644
+    --- a/src/mon/OSDMonitor.cc
+    +++ b/src/mon/OSDMonitor.cc
+    @@ -10962,7 +10962,7 @@ int OSDMonitor::_check_remove_pool(int64_t pool_id, const pg_pool_t& pool,
+       FSMap const &pending_fsmap = mon->mdsmon()->get_pending();
+       if (pending_fsmap.pool_in_use(pool_id)) {
+         *ss << "pool '" << poolstr << "' is in use by CephFS";
+    -    return -EBUSY;
+    +    //return -EBUSY;
+       }
+    
+       if (pool.tier_of >= 0) {
+    
+    pdonnell@icewind ~/ceph/build$ bin/ceph osd pool create derp 4 4
+    pool 'derp' created
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs add_data_pool cephfs_a derp
+    added data pool 3 to fsmap
+    pdonnell@icewind ~/ceph/build$ bin/ceph osd pool rm derp derp --yes-i-really-really-mean-it
+    pool 'derp' is in use by CephFSpool 'derp' removed
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs ls
+    ...
+    2017-10-03 12:50:48.409561 7f9e2e05b700 -1 /home/pdonnell/ceph/src/osd/OSDMap.h: In function 'const string& OSDMap::get_pool_name(int64_t) const' thread 7f9e2e05b700 time 2017-10-03 12:50:48.407897
+    /home/pdonnell/ceph/src/osd/OSDMap.h: 1184: FAILED assert(i != pool_name.end())
+    
+     ceph version 12.1.2-2624-g37884a41964 (37884a419640b446fffc1fa4d6074c97339fdd96) mimic (dev)
+      1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xf5) [0x564ebb5420f5]
+      2: (()+0x41dade) [0x564ebb3cbade]
+      3: (MDSMonitor::preprocess_command(boost::intrusive_ptr<MonOpRequest>)+0x1fb9) [0x564ebb4cd119]
+    
+    Note when testing this fix, use something like this after removing the data pool:
+    
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs set cephfs_a max_mds 2
+    
+    Setting max_mds will cause a new FSMap to be created where MDSMap::sanitize is
+    called; this is simulating the initial load+sanitize of a Hammer legacy MDSMap
+    by the mons.
+    
+    Fixes: http://tracker.ceph.com/issues/21568
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    
+    (cherry picked from commit 7adf0fb819cc98702cd97214192770472eab5d27)
+
+commit 23fa3b726368f036b31e53a0bec8ad4bc654d993
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 3 12:23:03 2017 -0700
+
+    mds: reduce variable scope
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 37884a419640b446fffc1fa4d6074c97339fdd96)
+
+commit 4f024c68319a79bef9d937d7f2363a5b69575773
+Merge: 6166148078 5a0016131b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Oct 30 14:05:00 2017 -0400
+
+    Merge pull request #18627 from ceph/backport-18513
+    
+    ceph-volume lvm zap backport
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 5a0016131b1ec900255057876c98fc70e00542f0
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Oct 27 11:29:50 2017 -0500
+
+    ceph-volume: set journal_uuid and journal_device when using a partition
+    
+    This correctly sets the tags when a partition is used for a filestore
+    journal.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 1a8561d38dd7363dc920ae82ec1343b2b75a1ca2)
+
+commit 05522e75f8d98658b5440d3c2f70e93ce65c65ce
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 24 10:13:40 2017 -0500
+
+    docs for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3e93a31deda96ed6fb23fa19ce3e273de05f7d88)
+
+commit cb1e81447f5c0c4eb8c0fd9fb11d0ceae3e26199
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Oct 23 09:51:43 2017 -0500
+
+    ceph-volume: add tests for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3d5e391693fc64747a4774287c968f842294eaa6)
+
+commit 33927c10e78842bb4c4ce4c0e4051ccceb8bb9e8
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Oct 23 09:44:07 2017 -0500
+
+    ceph-volume: print success message if zap succeeds
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 6d70762dee0cd9047c291cf9869f666a375e956b)
+
+commit d8381ab509d9f60f6fe62c5692a9374b3187d5c9
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Oct 20 09:58:48 2017 -0500
+
+    ceph-volume: terminal.write only takes one argument
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit a5454eba241abd8cc1b1a660b12a1aec7c3c16e0)
+
+commit 6f2bd88278f2c7dc4cfd156969f98d7cc8c3e763
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 16:29:39 2017 -0500
+
+    ceph-volume: the metadata field for the lv path is lv_path not path
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit e3a9113e7025f1e3a3130ec1e2d565f37bf3d2dc)
+
+commit 1dfe7e9854b00850b86168049821447d33298fff
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:59:26 2017 -0500
+
+    ceph-volume: update help text for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cbc38c4e4a7dcc24b31bcfa6af73eb8cf04f56ad)
+
+commit c5206435a9bae9d93e6cd9a3c2b98127871ad94e
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:55:44 2017 -0500
+
+    ceph-volume: remove lvm metadata when zapping an lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5f57fc87205c2d35da9a1f28c72e233ffb3fe4d9)
+
+commit eb53ef67b4e3bf5afb6e414fd42caab67c4cfdf2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:55:02 2017 -0500
+
+    ceph-volume: adds Volume.clear_tags
+    
+    Will remove all tags for the current lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3c9401f1618f7fcbea827fe7279e8dfde617d957)
+
+commit 4ee13e0587c5e152fdac18004bc4336f8b90e4af
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 12:01:57 2017 -0500
+
+    ceph-volume: zap should leave lvs and partitions intact
+    
+    This will remove filesystems and wipe data from any lv or partition
+    given to 'ceph-volume lvm zap' but still leave it intact for further
+    use.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit caba9fb80b358222192f736bd1b0ab31dca25cec)
+
+commit 8c3ab3459abfebac20bb4ce3ddecf1a7a58cf401
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 11:44:11 2017 -0500
+
+    ceph-volume: adds utilites to wipe the fs and clear data when zapping
+    
+    These should eventually move to a disk api.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2f64d4a0abd41afbcb9eba6a237642502d9a9135)
+
+commit 8a0e7fad57e41e10d51e0e167b671668ee1aa286
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 10:35:51 2017 -0500
+
+    ceph-volume: adds tests for api.lvm.remove_lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 937b57438147681c2c6e32c6db38d8bea68d4731)
+
+commit c6dd47ab6d1ec6784ec3cef8f65139baeaf4fa3a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 10:35:10 2017 -0500
+
+    ceph-volume: api.lvm.remove_lv should return True when successful
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7f055aa6deb3904cf0334e214c13a26098b08aa8)
+
+commit 71ae0b8cabc0a696e6762d7f377f9f4b28d2a963
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Oct 18 10:19:25 2017 -0500
+
+    ceph-volume: zap logical volumes
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 1d083a2191315ee94301c2f5f102a0906dd05fa8)
+
+commit 7110922c42ee30fa25dc9dacbfc8d0f9ae2f2a79
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Oct 18 10:19:00 2017 -0500
+
+    ceph-volume: adds a remove_lv command to the lvm api
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cf98b6971ced0caa29ff4f7ac346df652177fa29)
+
+commit 731610f18b9e4109ce72a4467af209144e4a127d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 17 14:15:18 2017 -0500
+
+    ceph-volume: stubs out the ceph-volume lvm zap command
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2e64b797ef6ae91623ffba8ae28d3f8ccc7d7b93)
+
+commit c2237c7c6dbcd5420d8ed87c6f2648c3f4e6edae
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 09:58:20 2017 -0400
+
+    qa/suites/rbd: run cls tests for all dependencies
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 27613a63d1f1ee6fa7327cd1c63b875c0e5247f8)
+
+commit 6b75efab390b3019e404f007090e9031328e1301
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 09:57:43 2017 -0400
+
+    cls/journal: fixed possible infinite loop in expire_tags
+    
+    Fixes: http://tracker.ceph.com/issues/21956
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9e66dca49591e50b9cab5df311f1dc217eb58fcc)
+
+commit 796e33647022de1fd831b31045242fb8e9d0e4d2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 30 18:46:49 2017 +0800
+
+    ceph-disk: unlock all partitions when activate
+    
+    should unlock all dmcrypted partitions when activating a device.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ba2aa0cee9e077d8439ba31228b41beb2d827a04)
+
+commit 3e9aad1a116e02bb126010f310ca2267177b5162
+Author: Felix Winterhalter <felix@audiofair.de>
+Date:   Mon Jul 17 02:04:39 2017 +0200
+
+    ceph-disk activate unlocks bluestore data partition
+    
+    Signed-off-by: Felix Winterhalter <felix@audiofair.de>
+    (cherry picked from commit 1287caf2dbb5ef6e5f243fe1d23633946aef26f9)
+
+commit 6b6eceee027e571d8d3da4af144391f8da85b53e
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Oct 28 18:23:30 2017 +0800
+
+    ceph-bluestore-tool: the link target should not ending with new line
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 39c68d128cf29f8e0a617009d16c5edf471ead47)
+
+commit cc84812507607d0a91233a513088094533a2d6a7
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Oct 28 18:22:27 2017 +0800
+
+    ceph-bluestore-tool: the right action is prime-osd-dir not prime-osd-dev
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit d418a04e9fed6bf91cc9acc5ac777eadf664c974)
+
+commit 9d8e5d763b173247f95f0769fd52372a8a7a5371
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 10:44:28 2017 +0100
+
+    mon: implement MDSMonitor::get_store_prefixes
+    
+    Fixes: http://tracker.ceph.com/issues/21534
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a3c317b406c69b9d6a2d7df94806f4c308e6ee6a)
+
+commit 338af1688147babb4e92523972175fe09ebb34b6
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 15:01:02 2017 +0800
+
+    mon/mgr: sync mgr_command_descs table and mgr_metadata table
+    
+    sync mgr_command_descs table and mgr_metadata table to new mons
+    
+    Fixes: http://tracker.ceph.com/issues/21527
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 13f6aa3aaa6de0aeccec67d7c6f3effe43dcae49)
+    
+    Conflict: in master PaxosService::get_store_prefixes(..) is marked const
+    while in luminous, the cleanup commit which added the `const` is not
+    backported yet, so drop the `const` in the backported commit.
+
+commit de07d2954afb012e51d2eaca12c1763e29a19eca
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 10:37:18 2017 +0100
+
+    mon/OSDMonitor: tidy prefix definitions
+    
+    We should define them in one place to make it easy
+    when updating get_store_prefixes.
+    
+    Fixes: http://tracker.ceph.com/issues/21534
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 889ac5166ad8fd30678325ddc2da59f45db53f06)
+
+commit 981e552d72c151b6bf2c9bb64b969f9446ef4789
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 14:55:55 2017 +0800
+
+    mon/osd_metadata: sync osd_metadata table
+    
+    sync osd_metadata table to new mons when add new mons
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 755ec735107c75156ae4935f7255bbfe5dc384d9)
+
+commit 3f0ad5f453a259cc53f50b1a9515e8a720da4a1d
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 14:55:55 2017 +0800
+
+    mon/osd_metadata: sync osd_metadata table
+    
+    sync osd_metadata table to new mons when add new mons
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 6c20433bfb049ac6c69f6f7a979006e8b9ea0e29)
+
+commit 61661480780e555fc501aec7c32163596e1e18d3
+Merge: c0ec364ad6 410434b3d2
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:28 2017 -0700
+
+    Merge PR #18385 into luminous
+    
+    * refs/pull/18385/head:
+            mds: fix race in PurgeQueue::wait_for_recovery()
+            mds: open purge queue when transitioning out of standby replay
+            mds: always re-probe mds log when standby replay done
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c0ec364ad6f54da7efbbb6b84ad34ea9782e3df2
+Merge: 5ee57e1b42 695d7ad511
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:24 2017 -0700
+
+    Merge PR #18316 into luminous
+    
+    * refs/pull/18316/head:
+            mds: prevent trim count from underflowing
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 5ee57e1b42d93fc0dc77b1221c362753684b280e
+Merge: 84f4bf40da b99acfd601
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:20 2017 -0700
+
+    Merge PR #18300 into luminous
+    
+    * refs/pull/18300/head:
+            mds: keep CInode::STATE_QUEUEDEXPORTPIN state when exporting inode
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 84f4bf40dade5ef55eba139d63187959b98e84e5
+Merge: 2638d5039e 460268b5e9
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:16 2017 -0700
+
+    Merge PR #17729 into luminous
+    
+    * refs/pull/17729/head:
+            ceph.in: validate service glob
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 2638d5039e615035563338cd428b44fb23d879ff
+Merge: f6f88c2a9b 2973b6d418
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:18 2017 -0700
+
+    Merge PR #18299 into luminous
+    
+    * refs/pull/18299/head:
+            mds: update client metadata for already open session
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit f6f88c2a9b40cf84658319a07a2bc26858763093
+Merge: 388901f091 f353a1e805
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:14 2017 -0700
+
+    Merge PR #18298 into luminous
+    
+    * refs/pull/18298/head:
+            osdc/ObjectCacher: limit memory usage of BufferHead
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 388901f0911b2d45f5b5ca2d653da5c605d99e2d
+Merge: 45cfeae6eb c5ccbf1108
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:10 2017 -0700
+
+    Merge PR #18085 into luminous
+    
+    * refs/pull/18085/head:
+            ceph_volume_client: fix setting caps for IDs
+    
+    Reviewed-by: Ramana Raja <rraja@redhat.com>
+
+commit 45cfeae6ebd5a0910c03ff323ecdd5740e569b41
+Merge: ecd5b0066f 1021fe235c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:06 2017 -0700
+
+    Merge PR #18030 into luminous
+    
+    * refs/pull/18030/head:
+            qa: relax cap expected value check
+            mds: improve cap min/max ratio descriptions
+            mds: fix whitespace
+            mds: cap client recall to min caps per client
+            mds: fix conf types
+            mds: fix whitespace
+            doc/cephfs: add client min cache and max cache ratio describe
+            mds: adding tunable features for caps_per_client
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit ecd5b0066fc091f34a166989b2e68e165c390dd0
+Merge: 04acd559db d5e583490d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 10:55:32 2017 -0700
+
+    Merge PR #17921 into luminous
+    
+    * refs/pull/17921/head:
+            ceph_volume_client: perform snapshot operations in
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Ramana Raja <rraja@redhat.com>
+
+commit add8c3db195acfda75a1628196e839b0147e7c3f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Oct 26 18:28:10 2017 -0400
+
+    rgw_file:  set s->obj_size from bytes_written
+    
+    Required to store correct final accounted_size of objects in
+    RGWWriteWriteRequest::write_finish.
+    
+    Fixes: http://tracker.ceph.com/issues/21940
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b1f528d35aeccabb4e5818aec6feb8e53e562500)
+
+commit 8cbb2eb937cae4e136ef77bc93e6d5346e21c8da
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 9 08:15:21 2017 -0500
+
+    qa/cephfs: test ec data pool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d0732fc96fbc6849dd51b391d85f765c74cfb593)
+
+commit 04acd559db55c6e8040a2ba65a850e72a04ad2ab
+Merge: d294493c42 8a87d43db2
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Fri Oct 27 11:34:15 2017 -0500
+
+    Merge pull request #18593 from ceph/luminous-wip-bz1499840
+    
+    luminous ceph-volume lvm bluestore support
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d294493c42ea21e85a1fd36eb8623229a2ff6ea6
+Merge: 58c1b8c4f2 34cd96d342
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:30:35 2017 +0200
+
+    Merge pull request #18334 from kmroz/wip-21816-luminous
+    
+    luminous: rgw: fix bilog entries on multipart complete
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 58c1b8c4f28b73d9b7db1cf3fe79ae5062832c88
+Merge: 8742c619b0 d7f6b93af4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:30:07 2017 +0200
+
+    Merge pull request #17861 from smithfarm/wip-21441-luminous
+    
+    luminous: rbd: [cli] mirror getter commands will fail if mirroring has never been enabled
+
+commit 8742c619b03cc5f048dc08f76d67ea64c109b4b3
+Merge: 7c03693d0b f8bfd9458d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:55 2017 +0200
+
+    Merge pull request #18416 from kmroz/wip-21855-luminous
+    
+    luminous: librbd: object map batch update might cause OSD suicide timeout
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 7c03693d0bd1a7108bd2d55eda7b7bd1e4e7f904
+Merge: fc486a26cb 8b3e9917a7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:43 2017 +0200
+
+    Merge pull request #18337 from kmroz/wip-21640-luminous
+    
+    luminous: rbd-mirror: forced promotion can result in incorrect status
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit fc486a26cb12466df6269ec4c7717332be2ed92c
+Merge: c944c688e5 4ebd4b3928
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:22 2017 +0200
+
+    Merge pull request #18336 from kmroz/wip-21639-luminous
+    
+    luminous: librbd: snapshots should be created/removed against data pool
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit c944c688e5a4a7935233e7fb3824b97d9afe44d9
+Merge: bc7e648b18 34af07f97a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:03 2017 +0200
+
+    Merge pull request #17860 from smithfarm/wip-21299-luminous
+    
+    luminous: rbd: [rbd-mirror] asok hook names not updated when image is renamed
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8a87d43db25a364841d969636bd74b7590c3d563
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 26 15:09:18 2017 -0400
+
+    ceph-volume lvm.activate remove links on activate before priming
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 634b5caaed7c9188e426727e83a0768bdbc51f0d)
+
+commit 869d13037dbb2b8882c4a4a91ec20c9814bcc438
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 26 07:48:24 2017 -0400
+
+    ceph-volume lvm.activate ceph-bluestore-tool needs --cluster too
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fb36087bc3c7e7bb8358cbf8ad84409fa23fac68)
+
+commit 3baab9f4e286c3b54c179b0f471435c2dbc859b2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 18:57:28 2017 -0400
+
+    ceph-volume lvm.activate consume cluster_name from lvm metadata
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2b7ff497fe9e0e167ab01bf0d8ce5e26f0da836f)
+
+commit 8f41a983356ee9d182f8247f2da8db15848afae0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 18:57:03 2017 -0400
+
+    ceph-volume lvm.prepare persist cluster_name on lvm metadata
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6d38ad987e08791be379f6c006c1eb5bf9464415)
+
+commit 1ae30d3130af845ebf563a74a1e2642f5236f1ed
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 15:30:43 2017 -0400
+
+    ceph-volume util.prepare bluestore db and wal paths are to devices, not osd dir paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fd616acd6ae0f184c1757e7de38fc746d526908c)
+
+commit fdfbfb8140fc1be3123d1254f4463613dc7c7cb0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 15:30:06 2017 -0400
+
+    ceph-volume lvm.prepare bluestore wal and db links get created with mkfs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80698d2309a221d2318b11be9b8f8a95b51416ea)
+
+commit 6ac58f3cd5e292153c82d881a30db1f6c4bfd936
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 10:28:43 2017 -0400
+
+    ceph-volume lvm.prepare use wal and db for mkfs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4970ab5c804111ce4b2b57817f1e29d58b18a893)
+
+commit c9ecd14ca220e45960650715703cae3e4376e3c0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 16:28:33 2017 -0400
+
+    ceph-volume lvm.prepare ensure tags are set for ceph.type correctly for block and data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 58dbefd3be7bad51e6c56c50fbfb450e205b3036)
+
+commit 76fb5fec94a9bbfed1163a23faf87e19d5617bd4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 16:26:03 2017 -0400
+
+    ceph-volume lvm.activate check if lv device is mounted at path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1ead135bef801280e7cec21283c95c7ba440d84b)
+
+commit 343362027368c4170bd785f1633afeb5ed90f3a2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:55:21 2017 -0400
+
+    ceph-volume lvm.prepare default to using bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 39dd0d1d10aba08c3e8299eb1702260274a756a7)
+
+commit 0a27fe88266458eead5a011c27cc323f5b7173a5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:54:57 2017 -0400
+
+    ceph-volume lvm.create default to using bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d3145da6e643dedb2fbed72e7d4d0cddf397c5e3)
+
+commit bd2e0aa78ad1a500935c3134f4d1e6c3413a26ae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:54:11 2017 -0400
+
+    ceph-volume lvm.common remove boolean defaults for CLI flags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7d5965ab532a621f402bad55d45dfed61d90cc5)
+
+commit c3d474af562ec5b8658bed6362046b0208a2c0eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 13:29:18 2017 -0400
+
+    ceph-volume lvm.prepare update to use create_osd_path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bafb72144666f891773bed95e9a0f757673af2e0)
+
+commit bc7e648b182569865d1c2a2aea6da1e52a3d3cb1
+Merge: 690be9c711 b015ce1161
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:57:48 2017 +0200
+
+    Merge pull request #18138 from kmroz/wip-21684-luminous
+    
+    luminous: rgw: stop/join TokenCache revoke thread only if started.
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 690be9c711b34be1ec4254ba0cedf7d6755f59f6
+Merge: 5727e4a33e 06c31a6cee
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:55:22 2017 +0200
+
+    Merge pull request #17994 from ukernel/luminous-21337
+    
+    luminous: mds: make sure snap inode's last matches its parent dentry's last
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 5727e4a33e0b82e0f9591251572b61458ec2cfae
+Merge: d973d6d7ae 624b3ac505
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:52:43 2017 +0200
+
+    Merge pull request #18431 from theanalyst/wip-21857-luminous
+    
+    luminous: rgw: We cant't get torrents if  objects are encrypted using SSE-C
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 02013a714ec5fa6c2ff1073e432b8f6b9ff41305
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 23 09:44:27 2017 -0400
+
+    ceph-volume lvm.activate only prime-osd-dir when directory is empty
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9bb6cfd867c2d8e972093d023622f8c2e5b440d4)
+
+commit 28593dbf1caeeec8f7c69dbd06e2ff9759afd072
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 23 09:20:26 2017 -0400
+
+    ceph-volume lvm.activate safeguard against auto detect objectstore flag not being set
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c9b5f352a8f9e2743e5dca3a0479c8099a59fd12)
+
+commit 8df51c8af5d95d67effece720ec2ff82517155b9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 15:18:20 2017 -0400
+
+    ceph-volume util.system use strings for comparison in py3
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1b671a8602b036cb8d53bc40eb70198c13143051)
+
+commit 1f09ee754fec40c4b17ea33829e6449d8749690a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 15:10:08 2017 -0400
+
+    f ceph-volume tests update changes from the help menu in lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3ddcd3c2f19cc848a11a2882f150462ad2f26c65)
+
+commit ec7c42bd59dc05cb8bad082d64ffdc28f13d3dc8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:12:02 2017 -0400
+
+    ceph-volume lvm.common --journal-size doesn't accept a volume group
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a8282e9a580bcfa74abdd33032e2ce2851b5cf29)
+
+commit 243098815d1d49570c193767d008cbc4fa0f174b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:04:31 2017 -0400
+
+    ceph-volume lvm.prepare filestore does not require a volume group
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3653f105927d455b8df143797af0cfe72f8d6834)
+
+commit da5f764a083613e5d74d1401da65c2ba3aff5fb4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:04:05 2017 -0400
+
+    ceph-volume lvm.common volume groups are no longer required
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 526c0aa81ac8eed95c2b4664a00ded2118372fff)
+
+commit eaa9150d4a6bacf62fcf485e8d36430743ac06cf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 12:07:06 2017 -0400
+
+    doc/ceph-volume activate workflow mention tmpfs in osd dirs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8788fe3721034f390240c01493bc1d27d65a73a5)
+
+commit d65af56eaaab728d8c81ddb69b967cb13f177766
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 12:02:10 2017 -0400
+
+    doc/ceph-volume update activation to include bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 288a3124793d11d71b540555fd03c2f61a79dd16)
+
+commit 6545b1c14e84af5f25fd4bbd405b56f7b5797113
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 11:58:47 2017 -0400
+
+    doc/ceph-volume update create to indicate bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 61ce8b4b844fcb64fc95373c5515de902a52fc8c)
+
+commit d973d6d7aefae1c34b664f98955baed4c841579b
+Merge: cf621f8b4e 5b5f0b5a33
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:49:03 2017 +0200
+
+    Merge pull request #17858 from smithfarm/wip-21448-luminous
+    
+    luminous: rgw: string_view instance points to expired memory in PrefixableSignatureHelper
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit cf621f8b4e8138284843f6d979aca61651a854bd
+Merge: e0c3a05959 0b3a974be7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:54 2017 +0200
+
+    Merge pull request #18442 from theanalyst/wip-21637-luminous
+    
+    luminous: encryption: PutObj response does not include sse-kms headers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit e0c3a05959a70b4cf4213320ba7c990578bf8c19
+Merge: dd7aa70301 14a1dcb1ff
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:26 2017 +0200
+
+    Merge pull request #18437 from theanalyst/wip-21698-luminous
+    
+    luminous: radosgw-admin usage show loops indefinitly
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit dd7aa7030144d42958f92af5c7f6804a402f3b6d
+Merge: 3a8f046716 c3a719da27
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:12 2017 +0200
+
+    Merge pull request #18434 from theanalyst/wip-21817-luminous
+    
+    luminous: rgw: zone compression type is not validated
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3a8f046716456eb0f51db79f152f3dc67d84c319
+Merge: 0d9b3a01d0 4c18ac1240
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:48 2017 +0200
+
+    Merge pull request #18440 from theanalyst/wip-21652-luminous
+    
+    luminous: policy checks missing from Get/SetRequestPayment operations
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 0d9b3a01d0972a2fc5ae9550ba80c38acd9edbeb
+Merge: aa71baeabe a3e2ae4925
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:33 2017 +0200
+
+    Merge pull request #18441 from theanalyst/wip-21651-luminous
+    
+    luminous: rgw: avoid logging keystone revocation failures when no keystone is configured
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit aa71baeabef5aa7eb6903593b3dfcc8878227c99
+Merge: 1067293a7e 0b84dd07c4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:13 2017 +0200
+
+    Merge pull request #18443 from theanalyst/wip-21634-luminous
+    
+    luminous:  s3:GetBucketLocation bucket policy fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 1067293a7e21fc835447c9e5e1d8416902593ba1
+Merge: f2f8aef89c abefb578c3
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:59 2017 +0200
+
+    Merge pull request #18444 from theanalyst/wip-21635-luminous
+    
+    luminous: s3:GetBucketCORS/s3:PutBucketCORS policy fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit f2f8aef89c3144ac5d859ea0fe8a7db57cfa8ad6
+Merge: 385d84b8eb 0e65ea466f
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:44 2017 +0200
+
+    Merge pull request #18430 from theanalyst/wip-21695-luminous
+    
+    luminous: failed CompleteMultipartUpload request does not release lock
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 385d84b8ebb3a4192ce4817aaf6c99b27af0190d
+Merge: 4a74a41c47 f9850d2148
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:13 2017 +0200
+
+    Merge pull request #18445 from theanalyst/wip-21633-luminous
+    
+    luminous: s3:GetBucketWebsite/PutBucketWebsite fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 4a74a41c473fcda61f156d8bf720bf0e4cf0aec0
+Merge: 14cb83cd07 8b420c4627
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:54 2017 +0200
+
+    Merge pull request #18436 from theanalyst/wip-21789-luminous
+    
+    luminous: user creation can overwrite existing user even if different uid is given
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 14cb83cd077258e061cfa09733cf0b9a82f7507a
+Merge: 0339b8b599 fe45086a54
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:36 2017 +0200
+
+    Merge pull request #18435 from theanalyst/wip-21790-luminous
+    
+    luminous: RGW: Multipart upload may double the quota
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 0339b8b599f313e6a7b176f0f8b94b8101ab50c3
+Merge: 68cc868d1f 248bd94dab
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:19 2017 +0200
+
+    Merge pull request #18432 from theanalyst/wip-21856-luminous
+    
+    disable dynamic resharding in multisite enviorment
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 68cc868d1fdca55308fa1aba0bb708d07170e668
+Merge: f337bef88a 17df0ba8f0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:44:40 2017 +0200
+
+    Merge pull request #18429 from theanalyst/wip-21792-luminous
+    
+    luminous: encryption: reject requests that don't provide all expected headers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 29470eac4ca939c824b56f4fbc03c8b7814f9d02
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 11:51:55 2017 -0400
+
+    doc/ceph-volume update prepare with bluestore workflow
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9f1a5627399e2589fe706b158a32a9fb8642ac23)
+
+commit c90e473ff35d15d309eb92dbdc6b9df2e8bd5ecc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 10:15:09 2017 -0400
+
+    doc/ceph-volume update filestore to indicate it is not the only/default objectstore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c93603e5764e021305cfeb1c297ee16318126fe1)
+
+commit 3890c47dddd0d028b0c003cb46a9bec670c445f1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 10:00:17 2017 -0400
+
+    doc/ceph-volume elaborate on env vars for interval and tries in activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1dc24d1a7065b4e23b1bfa0e7c6d263b5ebc6906)
+
+commit 88240acf19176ae5c139836b7d46848e15ef5454
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:33:55 2017 -0400
+
+    ceph-volume lvm.common update cli flags help for bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c9693fb3f8386016aba5d7c8f53f5f71f948c88d)
+
+commit a945e7776309d39955918cec7e0c5f1895abc39b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:23:35 2017 -0400
+
+    ceph-volume util.disk st_mode is needed for stat()
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e3b5663f7968e44ec0cd6db2bfc43acdb233e314)
+
+commit e833916afe22e9c4765344229b5b89cfc9af1b14
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:23:02 2017 -0400
+
+    ceph-volume decorators always log to the file with exceptions
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cb3533a72571251d9f1b6171f783246462726857)
+
+commit 53e9cf6fcd7549b4bfa78d7568d56b30b38e81fd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:49:32 2017 -0400
+
+    ceph-volume tests verify behavior of get_lv_from_argument
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit adabbd462bcb19a6da65211e24dfb2b9b5043ef4)
+
+commit 40cfec281cc2584b2e2987f356109c6c160d4597
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:48:01 2017 -0400
+
+    ceph-volume api.lvm return result of get_lv when using arguments
+    
+    If the argument is an absolute path it is fine to just return whatever
+    get_lv finds since it is a "safe" call, it will return a None if nothing
+    is found
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f209c9d4d50bd7abe0f7f6b9fb29359cb7196718)
+
+commit 7139077aab892c3213815d1dcf71d93df80a32c8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:12:01 2017 -0400
+
+    ceph-volume lvm.listing support bluestore, not only filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6a5b89a475c2a36022ca09dbb7f6b94cc3b44881)
+
+commit 654e06f1202596d5cecef9d03134861eed053882
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:10:51 2017 -0400
+
+    ceph-volume api.lvm allow full paths to detect lv from arguments
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d257101b1556ec5ce805622e9b7b6b74f6b000d)
+
+commit 454d48eb29f4429370b831645a29e0450b5b46b1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:10:26 2017 -0400
+
+    ceph-volume tests add support for bluestore list verification
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4674da9c07f871bf1f25bb782c136fe082095a14)
+
+commit 8d72737f90b0cd1db860678ebb66596e233fd646
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 13:20:55 2017 -0400
+
+    ceph-volume tests activate with fsid works for bluestore and filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5b9f0853f780db3b0af6b4b7c1ad1d59184069c9)
+
+commit 846aa2492f6db342d9fb4c477c2e13e61e414aaf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 13:14:05 2017 -0400
+
+    ceph-volume tests arg validator can accept absolute paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 15586715f506505a57ec5122416541479df74bc8)
+
+commit 2c9ee51a0a7ed5edc900316c19ba27facb9163eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 12:03:17 2017 -0400
+
+    ceph-volume tests update changes from the help menu in lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 39bdc2e3f621a32bcbb3dfe64f156bbfd7e4f5ee)
+
+commit 5724bb04bb74e17b9f7e96b3a3e434cac720c4ed
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 11:46:57 2017 -0400
+
+    ceph-volume tests update to get_lv from get_journal_lv
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9be1012072f36e21a3f3cef42a42f7f1eb83c64e)
+
+commit 43e25933701605ac8cd7cb3d6fbd4cbf6705565e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 11:30:29 2017 -0400
+
+    ceph-volume tests.util new path/device mount interface
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d5984bad97347e58318793fc05da84832d323113)
+
+commit 9b36a8bf8d471047dc94cdfcac1f84a9802b9fc5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 09:49:25 2017 -0400
+
+    ceph-volume tests update api.lvm tests
+    
+    The create_lv signature changed to require full size description and
+    tags need to be an actual dictionary (vs. keyword args)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e8a18637531cde59d73a97d106958489c1d12117)
+
+commit ea57e70e6f826ac17eae0226b9d916e8bff4d021
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 16:06:55 2017 -0400
+
+    ceph-volume lvm.activate ceph-bluestore-tool should populate the osd dir
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f3bb616bcf8a4737a3701ebf812de831785d6341)
+
+commit fe3b61ac0da67d9c80ead24c6ccf79e1507fa132
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 15:06:01 2017 -0400
+
+    ceph-volume lvm.activate auto detect objectstore type
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 472aa0b1997a0b1301b699b7ce30dce728aaea0c)
+
+commit 5da9775184e4bc9bf1b83d18eae602c9a510a89f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 15:00:19 2017 -0400
+
+    ceph-volume lvm.trigger enable objectstore auto detection for activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f579ef07026564f1a94b2923a2e4640ade2da503)
+
+commit 307c99e4382895e6b42c193e9b98b9206fd57f9a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 12:00:34 2017 -0400
+
+    ceph-volume lvm.activate add bluestore support for activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5696fed09b65044142b34f89492a6d91d3243e14)
+
+commit 70632b0fac2fa7e411418c90a50b99391379f436
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 11:40:38 2017 -0400
+
+    ceph-volume util.system create mappings of mounts
+    
+    This makes it easier for consumers to detect a path, or a device that
+    might be mounted or might have more than one mount, including tmpfs and
+    devtmpfs support, which was just not possible before.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f61007cc650f6964ef2ff3ea98a1debf02713198)
+
+commit 839dc1f554a12fb109ca1f992784dd4f4933898a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 16 06:51:05 2017 -0400
+
+    ceph-volume util.prepare separate filestore vs. bluestore prepare utils
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 85cdc783008ff0e0566c34aaab7cd5f12db1cbc7)
+
+commit deae88983d80cedcdc842d192f8d659b7533d3dc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 16 06:50:27 2017 -0400
+
+    ceph-volume process allow obfuscating a key or index when logging commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bbcca70d962855d991f0aa8e86a19f1855345668)
+
+commit 18336242b85065d8a2b0a92f22b2e91a81b997ba
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 13 09:22:53 2017 -0400
+
+    ceph-volume util.prepare add other ln helpers for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 555b664882db2cd247087f907bdf4d426fb652f8)
+
+commit 37a1564a41ba8ae6a89eb0903f0fdf6b463d90f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:08:52 2017 -0400
+
+    ceph-volume util.arg_validators allow to pass a group as --data for lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 211c38f591945dee53196068533784c385bae102)
+
+commit afd1588fe06a80f2277082fd7bd15f43d8e601b7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:08:16 2017 -0400
+
+    ceph-volume util.disk add utilities for is_device and is_partition
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6b23fc72240ced158cf5335f6e815dbfd804dfab)
+
+commit 5e8f798d2b8f80b21f3ce4701d22e47992dd0938
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:05:22 2017 -0400
+
+    ceph-volume api.lvm udpate create_lv for bluestore
+    
+    Require full size notation for LVM (e.g. 50G or 100M) and do not
+    "translate" tags by pre-fixing them with 'ceph'. That is already how the
+    rest of the API works, no need to make that translation in one place
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ac0d1555d3d9caeef9d4fc3e46bc11b7e437413b)
+
+commit f337bef88a31a5d23420665406a06f761e8d4e6e
+Merge: 3db3ca9938 d2be1e0ccb
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:43:43 2017 +0200
+
+    Merge pull request #17857 from smithfarm/wip-21446-luminous
+    
+    luminous: rgw:multisite: Get bucket location which is located in another zonegroup, will return 301 Moved Permanently
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 3db3ca99380521c79596b06b695eefbb81d57908
+Merge: ced01a5b85 c4d6539374
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:43:00 2017 +0200
+
+    Merge pull request #17856 from smithfarm/wip-21444-luminous
+    
+    luminous: rgw: setxattrs call leads to different mtimes for bucket index and object
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 3bbd840fe0d34d42c04fac523aee5fc023d8bd3b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 15:42:22 2017 -0400
+
+    ceph-volume lvm.prepare update filestore workflow to use new helpers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1e8a79b8f4b9e6360e8e3ace580e46caeedca8c7)
+
+commit ced01a5b85c9ec5657d8f09a447b6308ea597754
+Merge: 2c9a9af424 fe0a41b4a9
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:41:15 2017 +0200
+
+    Merge pull request #18417 from kmroz/wip-21782-luminous
+    
+    luminous: cls/journal: possible infinite loop within tag_list class method
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 2c9a9af424c618c50334c0efa7f5a5f807aeb4fd
+Merge: a1d286252a 7d2b7e8e31
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:38:04 2017 +0200
+
+    Merge pull request #18287 from smithfarm/wip-retract-assertions-luminous
+    
+    luminous: rgw: Remove assertions in IAM Policy
+    
+    Reviewed-By: Adam Emerson <aemerson@redhat.com>
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit fb95f3f9fd6122f9b8c3c2215463fc8a1ef62ba2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 14:19:07 2017 -0400
+
+    ceph-volume lvm.prepare initial take on bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e4fc3464af472a8dbdf049917eed73519ff82c3b)
+
+commit a1d286252a75c3632a7ecc63a366e5262f9e16cd
+Merge: b151619a45 fd207447d1
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:29:02 2017 +0200
+
+    Merge pull request #17859 from smithfarm/wip-21451-luminous
+    
+    luminous: rgw: lc process only schdule the first item of lc objects
+    
+    Reviewed-By: Daniel Gryniewicz <dgryniewicz@redhat.com>
+
+commit 26ef33efc37cdd626c6a7577812bb87947417e01
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 14:16:25 2017 -0400
+
+    ceph-volume lvm.common update help flags for bluestore usage
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 17028f3a3c1f9394fa82dc6f2bbd596d96acb037)
+
+commit 005359b92a910b72712edbcaa7b2e27a54d3a093
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 11 14:30:41 2017 -0400
+
+    ceph-volume api.lvm allow to create a vg
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 16d7feb2b5d5eb5fa0bf23a95b60623ae2e1676f)
+
+commit 795e69673f452227afc12d76ed5aa0d6a8af8665
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 10 12:30:06 2017 -0400
+
+    ceph-volume lvm add block CLI flags for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 63d2afbc605ac1531bc9f00a1891017bfd42fbce)
+
+commit cbfb082c180e93cd147e164afd1d32ef9234924e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 10 12:15:14 2017 -0400
+
+    ceph-volume lvm make bluestore the default option
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc3ced30747312dedd0e895a75ec15b1925ef1ea)
+
+commit a340f1acab04587f5b2c323abc1a14b8dd7de915
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:46:47 2017 -0400
+
+    rgw: RGWDataSyncControlCR retries on all errors
+    
+    similar to RGWMetaSyncShardControlCR, we don't want to exit and
+    stop the data sync processor thread on failures. we want to keep
+    retrying with backoff
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 065e67bbd312240f5128c31f5525098c62f3afce)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (sync tracing)
+
+commit fdec63a28e24e056938139591e5799a0e3693c20
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:09:40 2017 -0400
+
+    rgw: fix error handling in ListBucketIndexesCR
+    
+    the call to set_state() returns 0, when we want operate() to return the
+    error code instead. use set_cr_error() to do this
+    
+    Fixes: http://tracker.ceph.com/issues/21735
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ed6340a47f32472bb1f650407dbe739c44bf4bd6)
+
+commit e711cf319fa12ec7b03edda769685353e24cbfcb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:08:21 2017 -0400
+
+    rgw: ListBucketIndexesCR spawns entries_index after listing metadata
+    
+    if the metadata listing fails, we won't have to clean up entries_index
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f127f54406ef0e08a1265b9bc20433fe3f21523)
+
+commit 56359c367d8267281f714dcecefdb50c598718a6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 26 13:51:40 2017 -0500
+
+    ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+    
+    If the symlink points to the right location, do nothing.  If it doesn't,
+    replace it.  If it's not a symlink, bail with EEXIST.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit de8dc42d42218bc1a1779e1bcc5831c567853c8d)
+
+commit b151619a4523b148026e613860a05add6aff3831
+Merge: 84fc30873a 352373ff4a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 15:11:00 2017 +0200
+
+    Merge pull request #18433 from theanalyst/wip-21545-luminous
+    
+    luminous: rgw file write error
+    
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 84fc30873aa6ed3c1bf0c91307c362d27e9bc72a
+Merge: 2bc24caa00 bb8586b259
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Fri Oct 27 08:03:16 2017 -0500
+
+    Merge pull request #18587 from ceph/luminous-wip-volume-api-change
+    
+    luminous ceph-volume lvm api refactor/move
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2bc24caa003ea3c4898202a887a5864078c0f3fd
+Merge: a5548a8247 5f96edfb41
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 27 06:39:58 2017 -0500
+
+    Merge pull request #18568 from liewegas/wip-fix-rop-leak-luminous-2
+    
+    luminous: osd/PG: fix recovery op leak due to recovery preemption
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit bb8586b259726eb0ae72db7c6a238b2d2432fc9b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:13:33 2017 -0400
+
+    ceph-volume create an api module to hold common api modules
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f72368641d7f38b6395afb7ed70a0a9d0794a03a)
+
+commit 3d32fb30193437b57535c79657ee726704f4b037
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:44:54 2017 -0400
+
+    ceph-volume tests move lvm api tests into its new test module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit eec0c8a5c27adc9b3fc5e999cbee8165bdb45736)
+
+commit e6145af90c3f1c8c520bb7671194e6d8a13bb5d2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:43:36 2017 -0400
+
+    ceph-volume tests refactor lvm api import for test_listing
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 66fd41fd4b35e275196bfb94794f3fda9fd77f76)
+
+commit c63d3ba89489cc42aaa364c99b3d3b4c1bba226c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:43:18 2017 -0400
+
+    ceph-volume tests refactor lvm api import for api tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2a9a36bd24bddb35a7618127a0ec57f661a71566)
+
+commit 6efd25344230c0abb51cd82f99310a4b94f2fcd3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:42:47 2017 -0400
+
+    ceph-volume tests refactor lvm api import for activate tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d81aa8c112312f2fb49d94ce3512b3356b0b8814)
+
+commit 47cd005b672828aedf11aeb67a6099477783f7f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:42:01 2017 -0400
+
+    ceph-volume tests refactor lvm api import in conftest
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a00aff7066fe6a679498ac8a02ec22ef29feced3)
+
+commit 4a6dfa9b6949bdcdba22a7003c451390105d287e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:17:08 2017 -0400
+
+    ceph-volume devices.lvm refactor importing api to come from api/lvm.py
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f4bee6cb42a017abce269ed3926deff04038a47e)
+
+commit 9a20f184676973222c931cecff26ca313e636e1d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:14:16 2017 -0400
+
+    ceph-volume move lvm/api.py to api/lvm.py so disk can consume it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bb72480d2bf44ff04ea93c98f4a4e59032a28896)
+
+commit f26f3dc842c04ac70e309266aaedbae9d45eeae8
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Sun Sep 24 08:41:04 2017 +0800
+
+    rgw:fix list objects with marker when bucket is enable versioning
+    
+    fix: http://tracker.ceph.com/issues/21500
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit bc16b162cf757b9c6ceae30912eb647ef9304f75)
+
+commit a5548a8247cd9b072a2e0357cf6d0358550e7565
+Merge: 724609a8a5 1751a094d7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:52:28 2017 -0500
+
+    Merge pull request #18498 from liewegas/wip-recovery-fixes-luminous
+    
+    luminous: osd: fix recovery priority and pg state on recovery->backfill transition
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com.
+
+commit 724609a8a54a7619b48f4608790d81a2a43d4683
+Merge: 50c4cc0610 565b5cd1d6
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Oct 26 16:50:20 2017 -0500
+
+    Merge pull request #18567 from ceph/backport-wip-bz1491250
+    
+    luminous: ceph-volume lvm list
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5f96edfb41c4b028732cf12da5ab44322e251b2a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 26 16:23:41 2017 -0500
+
+    osd/PG: fix recovery op leak due to recovery preemption
+    
+    This was fixed in master in a different patch, but are not yet ready to
+    backport the bits there that came before this.  For now, fix it
+    specifically for luminous.  We can either sort out the conflicts later
+    or revert this and backport the master parts conflict-free.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 50c4cc06101fd1c8a9323cd3a04c92c8a9ab2580
+Merge: a175c0321e 33aa167331
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:14:43 2017 -0500
+
+    Merge pull request #17805 from tchaikov/wip-luminous-pr-17508
+    
+    luminous: mon: more aggressively convert crush rulesets -> distinct rules
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a175c0321e745ac095707d5a2d3cd25232272bfc
+Merge: 72b057187a b382db7cfc
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:14:07 2017 -0500
+
+    Merge pull request #18364 from kmroz/wip-21544-luminous
+    
+    luminous: mon: osd feature checks with 0 up osds
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 72b057187aed9b4e5f3a4ecf82fafa6e9cd2d3fd
+Merge: f6362e3110 b60d8cf0cb
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:13:31 2017 -0500
+
+    Merge pull request #18413 from kmroz/wip-21693-luminous
+    
+    luminous: osd: additional protection for out-of-bounds EC reads
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f6362e3110e4e482ff5a676035b862b1b0b666df
+Merge: f36670e190 4c6b1b65f7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:11:42 2017 -0500
+
+    Merge pull request #18529 from smithfarm/wip-21918-luminous
+    
+    luminous: Disable messenger logging (debug ms = 0/0) for clients unless overridden.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f36670e190ea96589c373d87d30ee8f4b3c6b20b
+Merge: 9a28a5c9d7 cec24f99ab
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:11:21 2017 -0500
+
+    Merge pull request #18540 from liewegas/wip-pr-17894-luminous
+    
+    luminous: ceph.spec.in,debian/rules: change aio-max-nr to 1048576
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 565b5cd1d68437096e283ccb942d8f108c26d3d4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Sep 13 08:23:40 2017 -0400
+
+    ceph-volume lvm add listing module to main
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ff72fd0ed52ef2d166007c7769c0140424982dc7)
+
+commit 58095350953140a1fa4b968c2e0a5774a960e739
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:41:10 2017 -0400
+
+    doc/ceph-volume add lvm/list to the toctree
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d11c80c917f44b33077715b19564aeea30c78e42)
+
+commit 3893947d3e04f049a85245750b841adbbcefeac8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:36:06 2017 -0400
+
+    doc/ceph-volume create the reference for the list sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9a52939ef8fc4abe720d2525c59997f4f9760902)
+
+commit 0992c3d7c136a58ff38f521fc0978ff036843d27
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:35:39 2017 -0400
+
+    doc/ceph-volume add create and list to the index
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ed5fa519aca1335039bfc32bc060af5181265517)
+
+commit 148537cc333d88b75640506c15a1bc221c9e09d6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 08:59:37 2017 -0400
+
+    ceph-volume tests create a unit test module for listing features
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2c2cf547b07f9510146b86d0746190b6d3fead3d)
+
+commit 285bc06569c8ed06890d23e9537a9d1106825b7f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 08:59:01 2017 -0400
+
+    ceph-volume tests create a small factory fixture
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5a0b17de81770ef3b851a1ff98232d93c78f49f9)
+
+commit 1e4e241d9c77796874a003799a0546fc185c5241
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 14:40:41 2017 -0400
+
+    ceph-volume tests ensure Volume.as_dict gets populated correctly
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c5b933a96d067fb8c7f753a67123e5662398f964)
+
+commit a0ab94e8d70ae9fc5d741c47a8f0dcfe033f9a5e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 14:31:57 2017 -0400
+
+    ceph-volume lvm.listing initial take on listing command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2d80190a039798a637383513f292e6aab62cd3f8)
+
+commit 454c655f2eb0b1e79ca8e1e1d87ff9c0fe4886da
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 12:35:13 2017 -0400
+
+    ceph-volume lvm.api be able to return a dictionary repr of a Volume object
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0cae22750f396d1c1664875edb2742c50a41bffd)
+
+commit 1390fb76d76b76428e6a9210005dbf01fd3b3a1a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 12:12:57 2017 -0400
+
+    ceph-volume process allows to fully mute terminal output on commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 233f64a4fd90a19c50d57b849ffaa6a25c835f34)
+
+commit 00e9ec1fa25ac3950f43ec3cf84f8321b04507eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 14 07:35:47 2017 -0400
+
+    ceph-volume lvm.api use a helper to get an lv from an arg
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3c5bbbc5822f02a038f093647c5de58bef2fe8f5)
+
+commit 26c523c3beeb5da7e855be64a053ac9dca7d10e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 23 14:53:30 2017 -0400
+
+    rbd-mirror: strip environment/CLI overrides for remote cluster
+    
+    Fixes: http://tracker.ceph.com/issues/21894
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 70dc22e03a115b130b9c214030d5996276a1634b)
+
+commit 9a28a5c9d7ae618f5b79d3727ba155b5066a2653
+Merge: 42172a4431 18a99f5f6b
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Oct 25 11:45:38 2017 -0700
+
+    Merge pull request #18456 from liewegas/wip-21882-luminous
+    
+    luminous: messages/MOSDMap: do compat reencode of crush map, too
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 18a99f5f6b4976f87dcd0d4fe7e34fddd90de22b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 25 07:00:14 2017 -0500
+
+    qa/suites/upgrade/jewel-x: make sure min compat client is hammer for straw2
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit cec24f99ab5e368d2dae7d53628fee09515bd873
+Author: chenliuzhong <liuzhong.chen@easystack.cn>
+Date:   Tue Oct 24 10:54:33 2017 +0800
+
+    ceph.spec.in,debian/rules: change aio-max-nr to 1048576
+    
+    when osd is more than 14 in one host,it report error that aio is not enough.
+    As the default aio-max-nr is 65536, one OSD needs 4096 aios and other programs may use aios.
+    This patch change aio-max-nr to 1048576 when install ceph-osd rpm package and debian package
+    
+    Signed-off-by: chenliuzhong <liuzhong.chen@easystack.cn>
+    (cherry picked from commit 36326dc7104fc2f20f19d51b6f618a029ba072d7)
+
+commit 15a52ba8329831ddf350664e6e6612523f98dcf4
+Author: baixueyu <baixueyu@inspur.com>
+Date:   Wed Oct 18 11:16:20 2017 +0800
+
+    You can find the problem do like this:
+    
+    upload obj to bucket1
+    s3cmd put obj s3://bucket1
+    cp obj from bucket1 to bucket2
+    s3cmd cp s3://bucket1/obj s3://bucket2
+    del obj from bucket1 and bucket2
+    s3cmd del s3://bucket1/obj
+    s3cmd del s3://bucket2/obj
+    you can see the pool 'default.rgw.buckets.data', the data of obj can not deleted.
+    ceph df | grep default.rgw.buckets.data
+    
+    Fixes: http://tracker.ceph.com/issues/21819
+    
+    Signed-off-by: Xueyu Bai baixueyu@inspur.com
+    (cherry picked from commit 2e11545077c485d5c155ab932b6274a8ac863e61)
+
+commit 4c6b1b65f7337809682bf907be94fe946909959a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 22:13:36 2017 -0400
+
+    common/common_init: disable ms subsystem log gathering for clients
+    
+    The log gathering causes large performance degradation to clients
+    with high message throughputs. This is hopefully a short-term
+    workaround until per-message logging can be replaced with an
+    efficient data recording system for post-incident analysis
+    use-cases.
+    
+    Fixes: http://tracker.ceph.com/issues/21860
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a3a40413f7908b08c40dec4020034cca4a0c4798)
+
+commit 42172a443183ffe6b36e85770e53fe678db293bf
+Merge: f2afb7a0d5 9f49698fab
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:36:18 2017 -0500
+
+    Merge pull request #18503 from liewegas/wip-21878-luminous
+    
+    luminous: os/bluestore/BlueFS: fix race with log flush during async log compaction
+    
+    Reviewed-by: Varada Kari <varada.kari@gmail.com>
+
+commit f2afb7a0d590ff55ae11f5bf3b13264a56380ce6
+Merge: d0be2106f9 3ad21109ed
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:35:29 2017 -0500
+
+    Merge pull request #18501 from liewegas/wip-21766-luminous
+    
+    luminous: os/bluestore: handle compressed extents in blob unsharing checks
+
+commit d0be2106f96377f059ec08800e32e492ec6f82a3
+Merge: d52e59bad7 e6eeb796b4
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:34:52 2017 -0500
+
+    Merge pull request #18459 from liewegas/wip-pr-18457-luminous
+    
+    luminous: qa/suites/rados/rest/mgr-restful: whitelist more health
+
+commit 902d467a0ec23baebf28a2419a4c0c78e0b1b662
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 10:11:14 2017 -0400
+
+    qa/rgw: ignore errors from 'pool application enable'
+    
+    Fixes: http://tracker.ceph.com/issues/21715
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 160240e0c1c216c950b7210a561b27436198de1f)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 1751a094d76ac84fbcecfc9816904cbb496bd765
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 17:11:59 2017 -0500
+
+    osd/PrimaryLogPG: clear DEGRADED at recovery completion even if more backfill
+    
+    We may have log recovery *and* backfill to do, but cease to be degraded
+    as soon as the log recovery portion is done.  If that's the case, clear
+    the DEGRADED bit so that the PG state is not misleading.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e91fda79d3be771f46394312a6b72ab75660012)
+
+commit d52e59bad700e8add6cd4cf895817aa8fd7a601a
+Merge: 724a51cd44 0048e6a58c
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 24 11:51:19 2017 +0800
+
+    Merge pull request #18491 from liewegas/wip-21573-luminous
+    
+    luminous: buffer: fix ABI breakage by removing list _mempool member
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9f49698fab5de9ade79d3aba957afe82daa4fecc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 08:51:17 2017 -0500
+
+    os/bluestore/BlueFS: fix race with log flush during async log compaction
+    
+    During async log compaction we rely on _flush-and_sync_log to update the
+    log_writer to jump_to.  However, if racing threads are also trying to flush
+    the log and manage to flush our new log events for us, then our flush will
+    turn into a no-op, and we won't update jump_to correctly at all.  This
+    results in a corrupted log size a bit later one.
+    
+    Fix by ensuring that there are no in-progress flushes before we add our
+    log entries.  Also, add asserts to _flush_and_sync_log to make sure we
+    never bail out early if jump_to is set (which would indicate this or
+    another similar bug is still present).
+    
+    Fixes: http://tracker.ceph.com/issues/21878
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4324c8bc7e66633035c15995e3f82ef91d3a5e8c)
+
+commit 3ad21109ed2df068bb113e8551505b850fbfcb30
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:48:41 2017 -0500
+
+    os/bluestore: handle compressed extents in blob unsharing checks
+    
+    If the blob is compressed, we aren't mapping to a range within
+    the allocated extents, but rather referencing the entire blob.
+    
+    Fixes: http://tracker.ceph.com/issues/21766
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eb26cfbfbb2b9b9fe84dfe9c999e2d67fd2661c0)
+
+commit 8f176f31d091cb53e1baa8c317df9e64bef4d95e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 22 22:46:00 2017 -0500
+
+    osd/PG: on recovery done, requeue for backfill
+    
+    We were keeping our existing recovery reservation slot (with a high
+    priority) and going straight to waiting for backfill reservations on
+    the peers.  This is a problem because the reserver thinks we're doing
+    high priority work when we're actually doing lower-priority backfill.
+    
+    Fix by closing out our recovery reservation and going to the
+    WaitLocalBackfillReserved state, where we'll re-request backfill at the
+    appropriate priority.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 09c50e2fe4fc3e8951a483e4b2693f0ffc008dcf)
+
+commit cc707155ea475836f97791c08c31994fb40e4374
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 22 22:43:18 2017 -0500
+
+    osd/PG: move local_reserver recovery cancel to Recovering state transition
+    
+    This is easier to follow than canceling the reservation in the next state.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dde007749bb960122f399b4cb8608a8c1ea4eb9c)
+
+commit def29079dc623926fed26a4946d6b9a074dfa2ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:55:29 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: run some jewel after completed upgrade
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8697eca5ea24c4724f637e4d4f75d3e0d2b95278
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:52:45 2017 -0500
+
+    qa/suites/upgrade/jewel-x/: set up compat weight-set after cluster upgrade
+    
+    ...but before we run some client workloads.  This will hopefully ensure
+    that older clients will behave.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 89ee508ac51839c3d1f71c5e21443e8f2679c59e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 13:03:17 2017 -0500
+
+    mon/OSDMonitor: add 'osd crush set-all-straw-buckets-to-straw2'
+    
+    This is a common and recommended step that users should take once they
+    have clients that support straw2 (hammer!).  Let's make it easy.
+    
+    This may result in a small amount of data movement, but usually it is
+    very little.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 928770513cbe3a244095e36dcb41c83c0724cfe9)
+
+commit 94a4ca6c8e4d26465cbefd1492ee05f21b08ff1e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 12:59:16 2017 -0500
+
+    crush/CrushWrapper: add bucket_set_alg()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit edc411ee4cb7b88a970ccfc9d29b3a3ba15d066c)
+
+commit 0048e6a58c7cdf3b3d98df575bc47db8397cd5a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 19 16:19:35 2017 -0500
+
+    buffer: remove list _mempool member
+    
+    This broke the C++ ABI by changing the list structure size.  Also, it's
+    not necessary as we can infer the mempool by looking at the other list
+    contents.  We don't (currently) have a need to map an empty list to a
+    particular mempool and have that state stick.
+    
+    Fixes: http://tracker.ceph.com/issues/21573
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9b92d87d37e3892b0096cd728b46154aed1e2d86)
+
+commit 3cd334b6264998490e6fb8321fb7242ff82b36e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 19 16:09:10 2017 -0500
+
+    buffer: allow mempool to be passed into raw* ctors and create methods
+    
+    This is more convenient, and also faster than initializing it in
+    buffer_anon and the immediately moving it elsewhere.
+    
+    Drop the optionality of the alignment argument.
+    
+    No users yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 481277b8c92368b8cac8fdafe1848720ec1035c2)
+
+commit 724a51cd442d35c950414b86608c48936b01b78b
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 12:05:39 2017 -0500
+
+    /etc/sysconfig/ceph: remove jemalloc option
+    
+    This breaks when used with rocksdb, which is now the default.
+    
+    See http://tracker.ceph.com/issues/20557
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19efccef248c2d66fe3d7fdac8f56d455f2c7b27)
+
+commit 95b6bc9ac41c4985ad97a515c4105351a0e6b94d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 11:16:26 2017 -0500
+
+    etc/default/ceph: remove jemalloc option
+    
+    This breaks when used with rocksdb, which is now the default.
+    
+    See http://tracker.ceph.com/issues/20557
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fba2055b4e54128db1e9a83f27bb3526d65d4db7)
+
+commit e6eeb796b4d24fcf6797519be06aca9771e73028
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 23:14:36 2017 -0500
+
+    qa/suites/rados/rest/mgr-restful: whitelist more health
+    
+    The test is fiddling with OSDs.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ddf3e9162da542af0c5f025957b8304e7359c924)
+
+commit 33aa1673316b77b71e473d1ab71cf61587e425a6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 22:46:48 2017 -0400
+
+    mon/OSDMonitor: improve crush map validation
+    
+    - move into OSDMap method
+    - ensure that rules exist for each pool
+    - ensure pool type matches rule type
+    - ensure rule mask min/max size cover the pool size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8f8f0b25058ff7319ec9d023f0eceb5832d0ecff)
+
+commit d825de7681ba1cff257ba4108d6c22c81e3a6aab
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Aug 25 11:06:21 2017 +0100
+
+    mon: more forceful renumbering of legacy ruleset IDs
+    
+    Previously, the rules were only modified in the trivial case,
+    so we continued to potentially have CRUSH maps with the
+    legacy ruleset functionality in use.
+    
+    In order to ultimately remove rulesets entirely, we need
+    to do this more aggressively, renumbering all the rules
+    and then updating any pools as needed.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 71d4b2bed54371657693cfb999ade44449be0efd)
+    
+    Conflicts:
+            src/mon/OSDMonitor.cc: the check for multiple rules was removed
+    in master, but not in luminous. once we renumber the legacy ruleset IDs,
+    it's not need to check for and to warn the user at seeing the case where
+    1-to-n mapping from ruleset to rule IDs.
+
+commit f52d1ad23d97e1ab8c70f9b17b75daa0773698b7
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Aug 25 10:42:58 2017 +0100
+
+    osd: s/crush_ruleset_in_use/crush_rule_in_use/
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 675483ecca06686fd4c626bfea33748092aef0c6)
+
+commit 17d73e5a2e592719df8099450c9cd5f46d6fd582
+Merge: d4158133f7 e25a583323
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:10:49 2017 -0500
+
+    Merge pull request #17773 from xiexingguo/wip-luminous-object-legacy-flag
+    
+    luminous: osd/PrimaryLogPG: _delete_oid - fix incorrect 'legacy' flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d4158133f7586472d01d9c6e669db92ddcd2e73d
+Merge: 84fbb0ff70 8b4fe14ba0
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:10:06 2017 -0500
+
+    Merge pull request #18362 from kmroz/wip-21543-luminous
+    
+    luminous: qa/tasks: prolong revive_osd() timeout to 6 min
+
+commit 84fbb0ff705df5bc9f164dd5a02bd5a82e6478d7
+Merge: 87ab8d5c80 993d098505
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:09:52 2017 -0500
+
+    Merge pull request #18388 from tchaikov/wip-pr-18346-luminous
+    
+    os/filestore: print out the error if do_read_entry() fails
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 87ab8d5c80374a73ad95f9ed9dd8826c54e64e9f
+Merge: 00daa8462a fd42f2a9cc
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:09:17 2017 -0500
+
+    Merge pull request #18398 from smithfarm/wip-21783-luminous
+    
+    luminous: cli/crushtools/build.t sometimes fails in jenkins' make check run
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 00daa8462a5a337ecc57bd8664bfcc748befeead
+Merge: 5a4850c8c6 112069c70d
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:08:58 2017 -0500
+
+    Merge pull request #18401 from liewegas/wip-bluestore-label-luminous
+    
+    luminous: os/bluestore: ceph-bluestore-tool repair updates and prime-osd-dir support
+
+commit dbddf21bd3aac284122fd5fc84abdd4c89b12b3e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:32:33 2017 -0500
+
+    messages/MOSDMap: do compat reencode of crush map, too
+    
+    If we are reencoding an incremental, and it embeds a crush map, we need
+    to reencode that in a compatible way too.  This is especially true now
+    because we have the compat crush weight-sets.  Otherwise, a client may
+    learn the crush map through an incremental but not understand choose_args,
+    and not see/understand the alternate weight set.  It will send requests
+    to the wrong OSDs where they will just get dropped.
+    
+    Fixes: http://tracker.ceph.com/issues/21882
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9eaafc66278d59447b29ce6cc378d0cd3ad14ad4)
+
+commit c5ccbf1108532547f54f084b8f06c7f5ddf8a4e8
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Sun Sep 24 20:23:12 2017 +0530
+
+    ceph_volume_client: fix setting caps for IDs
+    
+    ... that have empty OSD and MDS caps. Don't add a ',' at the
+    start of OSD and MDS caps.
+    
+    Fixes: http://tracker.ceph.com/issues/21501
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit baf3b88800c63ef7467abbc7b54e40c0da669f38)
+
+commit b382db7cfce84adc897cd68e4cd7a4eb1b40a9f1
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Sep 20 13:15:30 2017 +1000
+
+    mon/OSDMonitor: mon osd feature checks with 0 up osds
+    
+    get_up_osd_features() returns 0 if no osds are up which causes feature
+    checks to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/21471
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 26ba0ba044846d18f9bdfdbee6e2a9c46882c742)
+    
+    Conflicts:
+            src/mon/MonCommands.h
+              - removed mimic reference
+            src/mon/OSDMonitor.cc
+              - removed mimic-only command path
+
+commit c845db3f8c2a0c2494fbdc9d6cc63c8e0c3b0563
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 18:25:56 2017 -0400
+
+    osd/OSDMap: ignore xinfo if features == 0
+    
+    Some old bug (e.g., http://tracker.ceph.com/issues/20751) could
+    result in an UP+EXISTS osd having features==0.  If that happens,
+    we shouldn't crash the mon, which (reasonably) does
+    
+       if (osdmap.get_epoch()) {
+         if (osdmap.get_num_up_osds() > 0) {
+           assert(osdmap.get_up_osd_features() & CEPH_FEATURE_MON_STATEFUL_SUB);
+           check_subs();
+         }
+       }
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8399833f3d221364a8ededb97cd4e1c5a258d924)
+
+commit e58656005d681ee0e35a25b324321cfeea8c7b3a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 17 17:45:37 2017 -0400
+
+    rgw_file: explicit NFSv3 open() emulation
+    
+    Expect RGW_OPEN_FLAG_V3 in rgw_write(), and if present, attempt
+    a stateless open inline.
+    
+    Required by Ganesha v2.5 and later
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6ed23b4a0cea7e41b4743e27319737af693947ec)
+
+commit f9850d2148b396032634f7ffa18ae458bd208195
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Sep 28 13:54:32 2017 -0400
+
+    rgw: Check bucket Website operations in policy
+    
+    Add code to check s3:GetBucketWebsite and s3:PutBucketWebsite
+    operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21597
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1493896
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit ceed535957ac186e241fcff26b103cf7efa959b1)
+
+commit abefb578c345a0ddd7100cae987e582b530d3729
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 15:42:27 2017 -0400
+
+    rgw: Check bucket CORS operations in policy
+    
+    Add code to check s3:GetCORS and s3:PutCORS operations against bucket
+    policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21578
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1494140
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 27eb13fe568cc802feaf69131a21db076bcb6746)
+
+commit 0b84dd07c41a6d69d99672218c72a4f69c21b040
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 16:08:56 2017 -0400
+
+    rgw: Check bucket GetBucketLocation in policy
+    
+    Add code to check s3:GetBucketLocation against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21582
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1493934
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 79188d679edeb6e2f7ca852fdc4224368412cb72)
+
+commit 0b3a974be7bb9cd1d25ee9c2b861f38ea721c66b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 27 14:26:25 2017 -0400
+
+    rgw: include SSE-KMS headers in encrypted upload response
+    
+    Fixes: http://tracker.ceph.com/issues/21576
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a5b3db7acb4150d9d27ba42f72658e418dd444c0)
+
+commit a3e2ae4925e7e25c213ef37072d576d1f1ffdc13
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Sep 22 16:11:55 2017 +0200
+
+    rgw: defer constructing keystone engine unless url is configured
+    
+    currently we create a keystone revocation thread even when keystone url
+    is empty, lets defer the construction of keystone unless the urls are
+    configured
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 104c2f59011e6b515e3845cce529ee44334e76c0)
+
+commit 4c18ac124048f72a10d92aecfe1eadabab0a80b4
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 14:35:59 2017 -0400
+
+    rgw: Check bucket versioning operations in policy
+    
+    Add code to check s3:GetBucketVersioning and s3:PutBucketVersioning
+    operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21389
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1490278
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 16de0fc1c5ede961ebd481f13753214a971c206c)
+
+commit be1ded107ac4e1a1b1515d40553aed3581e0afac
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Sep 14 18:07:15 2017 -0400
+
+    rgw: Check payment operations in policy
+    
+    Add code to check s3:GetBucketRequestPayment and
+    s3:PutBucketRequestPayment operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21389
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1490278
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit f9d1ae1d153319e870c3ccaf7afdc92786cdaa3b)
+
+commit 92d0545781ca2bff39f7bd8e9d63bb8c2d8cb997
+Author: Nathan Johnson <njohnson@ena.com>
+Date:   Thu Sep 7 23:27:40 2017 -0500
+
+    rgw: admin api - add ability to sync user stats from admin api
+    
+    Fixes: http://tracker.ceph.com/issues/21301
+    Signed-off-by: Nathan Johnson <nathan@nathanjohnson.org>
+    (cherry picked from commit 828412d573a0b31387fc9f6a879f9a6083535730)
+
+commit d800747fa18a1872f1d748119bfcc7eff34ae6cf
+Author: gaosibei <gaosb@inspur.com>
+Date:   Tue Sep 26 10:19:22 2017 +0800
+
+    RGW: fix a bug about inconsistent unit of comparison
+    
+    Fixes：http://tracker.ceph.com/issues/21590
+    
+    Signed-off-by: gaosibei <gaosb@inspur.com>
+    (cherry picked from commit cb39f065d6eba87cf1fc2e99334322bf63092df3)
+
+commit 14a1dcb1ff70b2a468cb47e7f239b98e45c63a88
+Author: Mark Kogan <mkogan@redhat.com>
+Date:   Mon Sep 25 09:53:00 2017 +0300
+
+    rgw: update the usage read iterator in truncated scenario
+    Fixes: http://tracker.ceph.com/issues/21196
+    
+    Signed-off-by: Mark Kogan <mkogan@redhat.com>
+    (cherry picked from commit 7306514a813661b77bfcbfc6f534dbabbdd3aa78)
+
+commit 8b420c4627faca27f61f472597c6f67cca2efa2f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 5 16:39:30 2017 -0400
+
+    rgw: RGWUser::init no longer overwrites user_id
+    
+    if an admin op specifies a user_id and does not find a user with that
+    id, but does find a user based on a later field (email, access key,
+    etc), RGWUser::user_id will be overwritten with the existing user's id
+    
+    when this happens on 'radosgw-admin user create', RGWUser::execute_add()
+    will modify that existing user, instead of trying to create a new user
+    with the given user_id (and failing due to the conflicting email,
+    access key, etc)
+    
+    by preserving the original user_id (when specified), this uid conflict
+    is detected in RGWUser::check_op() and a "user id mismatch" error is
+    returned
+    
+    Fixes: http://tracker.ceph.com/issues/21685
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4e65c9e9af86e832cb96d57d487771aa28251e7c)
+
+commit fe45086a54d76cd9b913663d4537b36a8a276ed4
+Author: gaosibei <gaosb@inspur.com>
+Date:   Tue Sep 26 10:44:09 2017 +0800
+
+    RGW: Multipart upload may double the quota
+    
+    Fixes: http://tracker.ceph.com/issues/21586
+    
+    Signed-off-by: Sibei Gao <gaosb@inspur.com>
+    (cherry picked from commit 97f95e457fb7f9e36031f41e0f2dd3955daedd87)
+
+commit c3a719da278a830cc1b8101e3d8ef25316da33d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 12 13:26:29 2017 -0400
+
+    rgw: 'zone placement' commands validate compression type
+    
+    Fixes: http://tracker.ceph.com/issues/21775
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 33f8ee8b6fa5605edbbfe12d8e097123e509ad9f)
+
+commit 352373ff4a333c9b3869c9f168ee14530067907a
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sun Sep 24 22:11:22 2017 +0800
+
+    rgw_file: fix write error when the write offset overlaps.
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 872d73f945364002f0fa31762e6976db5b4b3c19)
+
+commit 248bd94dab4359dfea34bfa0cbdd16d9a9b76a02
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Oct 9 13:25:36 2017 +0300
+
+    rgw: disable dynamic resharding in multisite environment
+    
+    Fixes:http://tracker.ceph.com/issues/21725
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit f1ec7a49bde054a19a0a8fd80f37866e1e40379f)
+
+commit 624b3ac5053dbef008d2e9238c6958d07206f6c3
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Sep 26 10:00:59 2017 +0800
+
+    rgw: Torrents are not supported for objects encrypted using SSE-C
+    
+    Fixes: http://tracker.ceph.com/issues/21720
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 9a8ae664da54eb74e74a12cc16a52b0df7df70a0)
+
+commit 0e65ea466f05a2bc8825e51b32901d74082c1b65
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 3 17:48:29 2017 -0400
+
+    rgw: release cls lock if taken in RGWCompleteMultipart
+    
+    Follows Casey's proposal to conditionally release the lock in
+    ::complete(), in order to avoid duplicated code in various early
+    return cases.
+    
+    Fixes: http://tracker.ceph.com/issues/21596
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 704f793f08a02760d23eb5778b738bb07be0e7cf)
+
+commit 17df0ba8f00abbf4b30e0370090251b0c7f6b643
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Sat Aug 26 04:46:35 2017 +0800
+
+    rgw: encryption add exception handling for from_base64 on bad input
+    
+    If calling from_base64 with charactors that is not base64 encoded,
+    rgw will crash.
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit f0b88e51be86d84793b794f6fe87fdda62004a4d)
+
+commit 46f09e3803ba3c4b463bfa6a2eeb40a0bd7cc0ef
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:48:53 2017 +0800
+
+    rgw: encryption fix the issue when not provide encryption mode
+    
+    Now, in RGW, if someone want to upload an object using server-side
+    encryption with providing customer key or kms key id, but not
+    specify the encryption mode in the
+    "x-amz-server-side-encryption-customer-algorithm" or
+    "x-amz-server-side-encryption", the object will be uploaded
+    successfully without encryption.
+    
+    This is not a correct way to deal with it. It is better to
+    return error.
+    
+    Fixes: http://tracker.ceph.com/issues/21581
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit f8c4be8a8943096217d89f7c5abd129fa2414b36)
+
+commit fefbc9fec47cc6858f74b7ea372d39efeb95c61f
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:47:21 2017 +0800
+
+    rgw: encryption SSE-KMS add the details of error msg in response
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 3f16785daf4b3609f2bff7c107a35f908103e32a)
+
+commit 2e493e1f7a1f20d2f33e66816cc72d6b5f193b61
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:37:52 2017 +0800
+
+    rgw: encryption SSE-C add the details of error msg in response
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 9502549ac2133e969a5f268601b92fd4063c1bd9)
+
+commit fe0a41b4a921ac3510b344a6edf6e930a2fa2465
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 12 08:58:03 2017 -0400
+
+    cls/journal: fixed possible infinite loop which could kill the OSD
+    
+    Fixes: http://tracker.ceph.com/issues/21771
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9e7475f8fb08b0ef30cad2c0d680cecf331b44ef)
+
+commit e0d57c544d55f3693f858464f7995b4b30cfa88d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 12 08:56:57 2017 -0400
+
+    test: ceph_test_cls_journal was dropped when converting to cmake
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d9cfebabac33b645c6199bda63ff4619e6f538f)
+
+commit f8bfd9458d44d96d4ef56a28195a9e36fc0190c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Oct 15 17:26:25 2017 -0400
+
+    librbd: batch large object map updates into multiple chunks
+    
+    Fixes: http://tracker.ceph.com/issues/21797
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 04c5d12acc045731fbf1d0ddce276b5743d9fd89)
+
+commit 6866abc01be5fd92acc918472a604a3af9894b44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 18:55:47 2017 -0400
+
+    test/librbd: initial test cases for trim state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4674b578d8233da3b3c145606ed9c0e4e88a540a)
+
+commit 3b87dea6584010ae05e540825867c9961708ff36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 15:49:00 2017 -0400
+
+    librbd: tweaks to support testing of trim state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3220480d4a22c81cb74f25e528f2bcf3986342c1)
+
+commit ce628329641ac6322d5c59055e8db9eecafbfef4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 14:44:40 2017 -0400
+
+    librbd: combine trim state machine object map batch update states
+    
+    The PRE/POST states were previously divided into two halves for handling
+    the copy-up batch and the direct removal batch. This can be simplified by
+    just using a single PRE/POST for the entire deletion region.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 72ce4576fa2b562799a5bc78bd423cfabe097d67)
+    
+    Conflicts:
+            src/librbd/operation/TrimRequest.h
+
+commit 80ff76db5495ae41baf6d4292b6b10c2f789c155
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:06:38 2017 -0400
+
+    cls/rbd: object map update now utilizes constant-time bit vector operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fc99603ea2b5b268181d38507b8b6fb55ae0a6d0)
+
+commit 5dc46378370da8eede9958a88d6764737d55b09b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:05:48 2017 -0400
+
+    common/bit_vector: provide constant time iteration of underlying bufferlist
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit daa29f7d2b50bc4b44a108a0f91d45ce90fc16e5)
+
+commit ae4ebaca231e4ae94efc8d445fee792f76e656e1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:03:25 2017 -0400
+
+    common/buffer: expose hidden const deference operator
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f35947e3c6661dfe9f951896bc54bc8da158b490)
+
+commit b60d8cf0cbe56500af98b3d3857245e94d904b19
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 2 17:28:04 2017 -0400
+
+    osd: additional protection for out-of-bounds EC reads
+    
+    Fixes: http://tracker.ceph.com/issues/21629
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 881f2a02aecef0cd6bf8ec3e5045b76c8441e832)
+
+commit 8f87fa2d3af44840a3044cd736a2f8da0b602b07
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 13:21:47 2017 -0400
+
+    mgr: fix crashable DaemonStateIndex::get calls
+    
+    This function was recently fixed to return null
+    on missing entries: handle that properly.
+    
+    Fixes: http://tracker.ceph.com/issues/17737
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ade4827d86bb2bc79466a2da040475651c2aae0d)
+
+commit def3d55eb71e0c62b7fd523865c2ba196342aa1a
+Author: huangjun <huangjun@xsky.com>
+Date:   Fri Oct 6 11:58:50 2017 +0800
+
+    qa/suites/rest/basic/tasks/rest_test: whiltelist OSD_DOWN
+    
+    Fixes: http://tracker.ceph.com/issues/21425
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 838e12cfc84b386fd4eae09e526b078286f0771e)
+
+commit 20c9e641f7096bf3e208b4d6d28494603eeba891
+Author: huangjun <huangjun@xsky.com>
+Date:   Tue Sep 19 16:13:34 2017 +0800
+
+    qa/suites/rest/basic/tasks/rest_test: more whitelisting
+    
+    Fixes: http://tracker.ceph.com/issues/21425
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 4d30d02e522039461762d51a3a826a337214ca7e)
+
+commit 5a4850c8c66b75d02fc2597e1ca744c6b0303836
+Merge: 17c6f35703 03e2a68e43
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:43:39 2017 -0500
+
+    Merge pull request #17692 from smithfarm/wip-21176-luminous
+    
+    build/ops: python-numpy-devel build dependency for SUSE
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 17c6f357030315040f8af95e751921c475b32289
+Merge: 97b30154f8 5aa445897c
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:42:45 2017 -0500
+
+    Merge pull request #17730 from xiexingguo/wip-pr-17371
+    
+    luminous: mon, osd: per pool space-full flag support
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 97b30154f884f4ea987e3330799bf412b08ba3e4
+Merge: 0a8f1540ba fb7e6a581d
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:42:07 2017 -0500
+
+    Merge pull request #17779 from smithfarm/wip-rh-74-luminous
+    
+    tests: CentOS 7.4 is now the latest
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 0a8f1540ba08bb79e9ec8f3358490a31be734b2b
+Merge: 5ca7af785a d7cde5f316
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:41:34 2017 -0500
+
+    Merge pull request #17863 from smithfarm/wip-21343-luminous
+    
+    luminous: mon: DNS SRV default service name not used anymore
+
+commit 5ca7af785a6695121f841909a6118a5103795d99
+Merge: 37d7cfc9f1 49e030da55
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:40:45 2017 -0500
+
+    Merge pull request #17864 from smithfarm/wip-21438-luminous
+    
+    luminous: core: Daemons(OSD, Mon...) exit abnormally at injectargs command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 37d7cfc9f136c9604a1e53604a8e0d939eee60db
+Merge: 223b0956b9 077b3601d6
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:40:18 2017 -0500
+
+    Merge pull request #17866 from smithfarm/wip-21320-luminous
+    
+    luminous: mgr: Quieten scary RuntimeError from restful module on startup
+
+commit 223b0956b93504881a5f06d2589a0b138abf37a9
+Merge: 4f1a7a82a1 82651da2d5
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:39:53 2017 -0500
+
+    Merge pull request #17951 from smithfarm/wip-21548-luminous
+    
+    luminous: tests: ceph_manager: bad AssertionError: failed to recover before timeout expired
+
+commit 4f1a7a82a1134d6688537653ac07d924275e1f3a
+Merge: 37b13acabe a670921f4f
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:39:32 2017 -0500
+
+    Merge pull request #18342 from xiexingguo/wip-pr-18318
+    
+    luminous: bluestore: using bluestore_compression_min_blob_size instead of bluestore_compression_max_blob_size to config comp_min_blob_size
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 112069c70df5c7719af9291a8de2b4fcf48d049a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:46:03 2017 -0500
+
+    os/bluestore: fsync on _write_bdev_label
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7aca8af6625156eeaa936629f83ede33d3bbe866)
+
+commit 1e9aa60e3f6acca6649f5d5d51421cc2bd51e55a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:45:54 2017 -0500
+
+    ceph-bluestore-tool: implement 'set-label-key' and 'rm-label-key'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b468c989f9e8609e7f640cfa4385c454ae6dc17)
+
+commit 170499a1e570bc110be08ca2a38063229fee75dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:42:29 2017 -0500
+
+    os/bluestore: make _write_bdev_label static and public
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0b0550c967ae0d6ae49641c6859f0b63267e2cd9)
+
+commit 895a277568f902f1c00b9df475ea56b00b115841
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 11:40:11 2017 -0500
+
+    doc/man/8/ceph-bluestore-tool: add man page
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7b91e50dbd289e65f10cb98c5eea9f3171c2d1f1)
+
+commit 24b3b7960e5c5d118aacb63753c794466e801f84
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 13 09:20:53 2017 -0500
+
+    common/options: document 'key', 'keyfile', 'keyring'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cdee8b26b9d93189dcd2875038361fb59ed21ba2)
+
+commit 508889d4a7aa93f3bbe13ad8bfc181f3fc9ed2a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 13:54:06 2017 -0500
+
+    ceph-bluestore-tool: fix show-label to use dev name as key
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4785345ace7b675a0e1c34c457aad1103aab5a1d)
+
+commit 6e17557f288aa7686b8b68c1df31a9a8c26912ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 08:48:59 2017 -0500
+
+    ceph-bluestore-tool: drop "action ..." print
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5e486c2bd3414f4ee6519574d4d4584618144867)
+
+commit a128aad320e99c396ff17bcaadb48866667b87df
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 08:48:46 2017 -0500
+
+    ceph-bluestore-tool: only infer devs if devs.empty()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2d217367af0df5e18394b1c6179d92db3c0b4448)
+
+commit 24d86603faf4a826684a5e124ed9b447de0b50e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 17:36:40 2017 -0500
+
+    ceph-bluestore-tool: implement prime-osd-dir
+    
+    This populates an osd dir based on the metadata in a bluestore
+    device label.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72ee7e29fb77f581a537b1ff2b274c9f3a7de53f)
+
+commit 0f43e6907cfd27f2d6791b896480f2c537115212
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:33:54 2017 -0500
+
+    os/bluestore: keep meta items for extra bdev paths
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8d07fa5ad0772b4d979dc10695d230074d1f1e69)
+
+commit bb9b70c33a3e013ef78d931fd865ab44c255d772
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 17:33:00 2017 -0500
+
+    vstart.sh: specify block files
+    
+    This is mostly just to test ceph-bluestore-tool prime-osd-dir.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4e0c46d89c2ebe6ae5f9af1c909a09daaea9687)
+
+commit ce83461a60cea6026a7abdd51b2a7a35302392c6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:32:16 2017 -0500
+
+    osd: store osd cephx key in "osd_key" meta slot
+    
+    Note that this *only* works with --mkkey (deprecated except for
+    vstart.sh) and --key <key>.  If you write out a full keyring before
+    running --mkfs it doesn't get populated.  Users should pass --key
+    instead of --keyring in that case.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 65b707358e5442eab0fd6ba4d05d9f0a7adc969f)
+
+commit 60d43cb51d7b05cc778cda6b5fcd368db618e3db
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:17:09 2017 -0500
+
+    ceph-osd: mkkey before mkfs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 75400ead3061ba1e2820878783fe7b1ddcd9a5fb)
+
+commit faad6afa3441b5ce9455a4de0535baef78b8e7de
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 15:53:56 2017 -0500
+
+    os/bluestore: store meta values in main device label
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 33b8e8e86f6bdbee120729561c6cf433d286e2f8)
+
+commit 1027e57d9af374f1fa1dee2a585dfe539dc4f5eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 10 15:37:10 2017 -0400
+
+    os/bluestore: fsck: fix error prefixes
+    
+    __func__ is now _fsck instead of fsck
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7d381af7aed14e08823eca4d0b12de14456ef6a1)
+
+commit 68642c43384d5a30155cd67129267d38d6cc3a06
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 12:29:32 2017 -0400
+
+    os/bluestore: allow repair invocation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 659fcd279ae82c90172fd6ca07a032d75fe50c96)
+
+commit fd42f2a9ccc5016660b9c205f6aa0104ffdd0b10
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 11 17:45:19 2017 +0800
+
+    crushtool: print error message to stderr not dout(1)
+    
+    in hope to fix the mysterious test failure where cli/crushtool/build.t
+    prints nothing when error message is expected.
+    
+    Fixes: http://tracker.ceph.com/issues/21758
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dc78d40af77ff8d5e4f3d39497e6e3c98c309306)
+
+commit 6902627cb3cb41547be5134aba5c10a39b78163a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:51:47 2017 -0400
+
+    crush/CrushWrapper: fix output arg for find_{takes,roots}()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 60b9cfafc3128cc0cb1f89137221fcc46fcd3802)
+
+commit 993d09850580ca7a697c1a221a8e258f4cd64237
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 17:08:59 2017 +0800
+
+    os/filestore: print out the error if do_read_entry() fails
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42e85fe35d2fe8f3e99bd110021fd5157cf589d7)
+
+commit 410434b3d23c54971f3e2c4df179b139979a33c0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 19 11:32:38 2017 +0800
+
+    mds: fix race in PurgeQueue::wait_for_recovery()
+    
+    After PurgeQueue::wait_for_recovery() locks the mutex, purge queue
+    may have already been recovered.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit ee96c34a4d1fc48dec6a1afb4fe98634805c02ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 19:59:47 2017 +0800
+
+    mds: open purge queue when transitioning out of standby replay
+    
+    MDS opens the purge queue when it starts standby replay. This is
+    wrong because purge queue may change during standby replay.
+    
+    Fixes: http://tracker.ceph.com/issues/19593
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e515e84f69eeab90ea5c5831f7d3e684e48fb62e)
+
+commit 681cddfa420a0f5f4335564843cdfdb1a4f2371d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 18:49:55 2017 +0800
+
+    mds: always re-probe mds log when standby replay done
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 82c1f5fcfd151aa1e8c39c7f198abec9bf807b65)
+
+commit 37b13acabe3a0cbdcd88c06f49548146d2c69c14
+Merge: 90b6067f6f f30e9a5e6b
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Tue Oct 17 15:47:31 2017 -0700
+
+    Merge pull request #18363 from ceph/wip-yuriw-21822-luminous
+    
+    tests - ceph-ansible vars additions
+
+commit f30e9a5e6bd47129f8a35c1f9067216cfe8a6a70
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 17 13:14:36 2017 -0700
+
+    tests - ceph-ansible vars additions
+    added symlinks for distros
+    
+    Fixes http://tracker.ceph.com/issues/21822
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 90b6067f6f5cbfaf48afb08bd17cb9e5f46a5af9
+Merge: 1718fd969c ab53002186
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:31:25 2017 -0500
+
+    Merge pull request #17975 from dillaman/wip-systemd-rbd-mirror-luminous
+    
+    luminous: systemd: rbd-mirror does not start on reboot
+    
+    Reviewed-by: Sébastien Han <seb@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 1718fd969c32cc09cfc208a18455291ab2886e93
+Merge: 345286b775 9d12fa5e6d
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:30:46 2017 -0500
+
+    Merge pull request #18234 from jdurgin/wip-filestore-rocksdb-luminous
+    
+    filestore: set default readahead and compaction threads for rocksdb
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 345286b775fd269eb9f13cfcb8d0aab533dbfbc6
+Merge: 9223ace029 ab644b89e0
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:29:38 2017 -0500
+
+    Merge pull request #18236 from tchaikov/wip-21609-luminous
+    
+    luminous: mon,osd: fix "pg ls {forced_backfill, backfilling}"
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9223ace02909d6551f6126181d11bf00bab1cfe3
+Merge: fdefc8094e 099801e616
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:29:08 2017 -0500
+
+    Merge pull request #18257 from smithfarm/wip-21699-luminous
+    
+    luminous: mgr status module uses base 10 units
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8b4fe14ba0e9f8e1b09ae4112dbebd370136bd3f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 22 10:58:40 2017 +0800
+
+    qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    bluestore_fsck_on_mount and bluestore_fsck_on_mount_deep are enabled by
+    default. and bluestore is used as the default store backend. it takes
+    longer to perform the deep fsck with verbose log. so prolong the
+    revive_osd()'s timeout from 150 sec to 360 sec.
+    
+    Fixes: http://tracker.ceph.com/issues/21474
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42be200c564184feff1056b7771ce3c1b2cc5527)
+
+commit a670921f4fb2e15f5ea44e9dda238c2eef96897c
+Author: linbing <hawkerous@gmail.com>
+Date:   Mon Oct 16 09:57:23 2017 +0800
+
+    bluestore: using bluestore_compression_min_blob_size instead of bluestore_compression_max_blob_size to config comp_min_blob_size
+    
+    Signed-off-by: linbing <linbing@t2cloud.net>
+    (cherry picked from commit da8dc4d4c60f5fd66f6eb2ba7257c52948f24681)
+
+commit fdefc8094ed639c859db5b4eba858cfe05329e5c
+Merge: bac56cccf9 3a94d69264
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 17 15:19:01 2017 +0800
+
+    Merge pull request #18339 from tchaikov/wip-gen-state-diagram-luminous
+    
+    doc: build with multiple-line state transition function signature
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+
+commit bac56cccf9e3fb44a0ed85c151a8b1a0fb5fa4b0
+Merge: 61aca44033 37ddfc61c3
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 17 12:44:04 2017 +0800
+
+    Merge pull request #17835 from rzarzynski/wip-rgw-per-storage-policy-stats-luminous
+    
+    luminous: rgw: add support for Swift's per storage policy statistics
+    
+    Reviewed-by: Matt Benjamin <mbenjami@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 3a94d6926403ac1f55a9d5723f6b07486f35060b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 6 08:57:01 2017 -0400
+
+    doc parse two lines for getting the context
+    
+    So that signatures can get parsed when they are split like:
+    
+    PG::RecoveryState::RepWaitBackfillReserved::react(
+      const RemoteReservationCanceled &evt)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9e2a52ccec4a5c833dd861ae528d52efbc0f9e5f)
+
+commit ddd88ff365c5399d88890abe1608f12c4a960488
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 6 08:03:36 2017 -0400
+
+    doc raise exceptions with a base class
+    
+    Although valid in very old Python, it is no longer possible to raise
+    "bare" exceptions without a class.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 347b7b1f193f97a3577a971e074a23366b0c8617)
+
+commit 8b3e9917a70452cc4182aae469aac331314df4e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 15:46:28 2017 -0400
+
+    rbd-mirror: ensure forced-failover cannot result in sync state
+    
+    If the local image already exists and the local image isn't
+    registered in the remote journal, it should be assumed that
+    it's already in the replaying state so it can verify the
+    ancestry to detect split-brains.
+    
+    Fixes: http://tracker.ceph.com/issues/21559
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bc96c53d947625d12f7b30a45da68a96c09faf70)
+
+commit d7da395e5196eae13b31c018bf503b9e0c169bda
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 14:21:42 2017 -0400
+
+    rbd-mirror: forced-promotion should interrupt replay delay to shut down
+    
+    Fixes: http://tracker.ceph.com/issues/21559
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e3b58312572ceb7abc675128dfb231c9b888655e)
+
+commit 4ebd4b39282f715a30a849c01510a281faec2ca6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 29 15:11:38 2017 -0400
+
+    librbd: snapshots should be created/removed against data pool
+    
+    Fixes: http://tracker.ceph.com/issues/21567
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a3baf1bd852d6c5d0de10a33839658789edc4eb)
+
+commit 34cd96d3425dcea4053ce0514632194dd7b2eaab
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 12 11:25:13 2017 -0400
+
+    cls/rgw: increment header version to avoid overwriting bilog entries
+    
+    Fixes: http://tracker.ceph.com/issues/21772
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 081ba106a910583851b9acdcaf5971e37a439329)
+
+commit 972916bf69192a2c504bf855e2af0480815ecc91
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 11 19:27:55 2017 -0400
+
+    test/rgw: add test_multipart_object_sync
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c14d4daf9201d58b1d0038bdd733a16275e037a)
+
+commit 695d7ad51179d4f1f5e4168dec3758e3ffa4fe4d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sat Oct 14 13:47:53 2017 -0700
+
+    mds: prevent trim count from underflowing
+    
+    Fixes: http://tracker.ceph.com/issues/21807
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 452bdd8b35643ed82d8614efcd4ca22688392eb6)
+
+commit 61aca440334219d5d04df632812dbdf164de3fbc
+Merge: 462b93ecc5 bb92cc93ac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 14:30:57 2017 -0400
+
+    Merge pull request #17834 from rzarzynski/wip-rgw-21148-luminous
+    
+    luminous: rgw: add support for Swift's reversed account listings
+
+commit 462b93ecc5e8caf7b0c35dd83dbbe7e33cc015d9
+Merge: c4ac0d64a9 119995a8f0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 12:48:02 2017 -0400
+
+    Merge pull request #17811 from smithfarm/wip-21456-luminous
+    
+    luminous: rgw: wrong error message is returned when putting container with a name that is too long
+
+commit c4ac0d64a95a6f9ec974446a6971f9d60ab67b9f
+Merge: 5695829ffd 957e221f1f
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Oct 13 20:42:16 2017 -0400
+
+    Merge pull request #17832 from rzarzynski/wip-rgw-18977-luminous
+    
+    luminous: rgw: list_objects() honors end_marker regardless of namespace.
+
+commit b99acfd6018178ad8cd7db187d6892780f82a276
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 12 12:01:35 2017 +0800
+
+    mds: keep CInode::STATE_QUEUEDEXPORTPIN state when exporting inode
+    
+    Fixes: http://tracker.ceph.com/issues/21768
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 1d160098a93092904026b62326fd9890de48044a)
+
+commit 2973b6d4184cb5c36f8823010ab83a2b8ebb12d0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 10 17:47:33 2017 +0800
+
+    mds: update client metadata for already open session
+    
+    session opened by Server::prepare_force_open_sessions() has no
+    client metadata.
+    
+    Fixes: http://tracker.ceph.com/issues/21746
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 9d5c92a3367369789836d8669aa421074c69e4f3)
+
+commit f353a1e805f4be3e31d8c3346f0f41fa2619c770
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 9 18:00:38 2017 +0800
+
+    osdc/ObjectCacher: limit memory usage of BufferHead
+    
+    when doing small size random writes, size of data structures that used
+    for tracking dirty data can be larger than the dirty data size.
+    
+    Fixes: http://tracker.ceph.com/issues/21726
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 27db0255992354e15b4285891a33fede6849cf62)
+
+commit 5695829ffd72a03f5611393c6037caf73ff1e813
+Merge: d3804d216d 6d032e6ce3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Oct 13 15:29:45 2017 -0700
+
+    Merge pull request #18275 from ceph/wip-yuriw-21776-luminous
+    
+    tests - Removed `distors` and added `supported` to run on all support…
+    
+    Reviewed-by: Vasu Kulkarni <vakulkar@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6d032e6ce33fe153958a445066a2a7ba00ffd260
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Oct 12 13:36:20 2017 -0700
+
+    tests - Removed `distors` and added `supported` to run on all supported OSs
+    Added `openstack` fragment to run on vps
+    Forced/hard-coded `machine_type=vps`
+    Added `print`s
+    Added only centos and ubuntu latest to support systemd (in distros dir)
+    Added `ceph osd set-require-min-compat-client luminous`
+    
+    Fixes http://tracker.ceph.com/issues/21776
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit d3804d216d21d7e2b12e69eae1c1ef293229a6ca
+Merge: c9b493d2db 28e7d5645e
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 13 14:34:56 2017 -0500
+
+    Merge pull request #18050 from xiexingguo/wip-pr-17610
+    
+    luminous: os/bluestore: set bitmap freelist resolution to min_alloc_size
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c9b493d2db39765c6ecfb31a3c9e8a38465e6ba0
+Merge: 09d597d34a b435af1137
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 13 14:34:30 2017 -0500
+
+    Merge pull request #18247 from liewegas/wip-seesaw-warning-luminous
+    
+    qa/suites/rados/singleton/all/mon-seesaw: whitelist MON_DOWN
+
+commit 2e6b29084ae505098fe6889d69133a8527c773be
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 1 17:32:22 2017 +0800
+
+    arch/arm: set ceph_arch_aarch64_crc32 only if the build host supports crc32cx
+    
+    HWCAP_CRC32 is defined by the linux kernel source. so it's defined as
+    long as the linux kernel source is new enough. but the compiler on the
+    building host is not necessarily able to build the `crc32cx`
+    instruction. if we happen to have an incapable compiler on a machine with
+    recent linux kernel source, the dummy "ceph_crc32c_aarch64()" will be
+    selected by `ceph_choose_crc32()`. and it always return 0.
+    
+    See-also: http://tracker.ceph.com/issues/19705
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8a077bd2a93d3e12e20013e76e57a35240f5f481)
+
+commit 7d2b7e8e3113daf40fd56d8258294d45b854ca55
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Oct 10 16:21:48 2017 -0400
+
+    rgw: Remove assertions in IAM Policy
+    
+    A couple of them could be triggered by user input.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit b3118cabb8060a8cc6a01c4e8264cb18e7b1745a)
+
+commit 09d597d34a1d567dc5350570f044eaa3d74a024f
+Merge: 7fdd9a6dac 37ea1976b3
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 12 14:57:38 2017 -0500
+
+    Merge pull request #18253 from ceph/wip-yuriw-21660_2-luminous_1
+    
+    tests - Added yaml fragmet to cover testing...
+
+commit 7fdd9a6dac4e047ae710dfaad38d5dbe3caa2634
+Merge: 8ad4617f3d 2b7bd94f13
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 12 14:57:23 2017 -0500
+
+    Merge pull request #18254 from ceph/wip-yuriw-21660_3-luminous_1
+    
+    tests - Added yaml fragmet to cover testing...
+
+commit ab644b89e0aad7b0c1a26ed3b9a2f3ea685c7a15
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 11 11:45:33 2017 +0800
+
+    qa: s/backfill/backfilling/
+    
+    it's renamed "backfilling" in 4015343f .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e21114274f6a2742748a5a9b965d415241c80f3c)
+
+commit 099801e616c3c9e8501f153501b74d5220383f33
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Wed Oct 11 09:31:14 2017 +0800
+
+    mgr/status: fix ceph fs status returns error
+    
+    Fixes: http://tracker.ceph.com/issues/21752
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit 4a90daa700142ed1f58f5f2a2cc85f3049bccd64)
+
+commit 0618f19128d0eb04004e65e6eab477b7b1b7fb6a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 30 20:32:11 2017 +0100
+
+    mgr/status: format byte quantities in base 2 multiples
+    
+    Fixes: http://tracker.ceph.com/issues/21189
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2a5ae210928f8c9f1e3b72863c277ae5655bb771)
+
+commit 2b7bd94f139251b4373a9662e5add86456566486
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Oct 11 13:12:45 2017 -0700
+
+    tests - Added yaml fragmet to cover testing...
+    ... for http://tracker.ceph.com/issues/21660#note-17
+    
+    for luminous-x (see similar #18200, #18226)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit e7fdda31803cd37604a9296a876320bfcb849d34)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 37ea1976b30e4e03ac535ee353c3a5277c53dbd3
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 10 13:48:33 2017 -0700
+
+    tests - Added yaml fragmet to cover testing...
+    ... for http://tracker.ceph.com/issues/21660#note-17
+    for jewel-x (see similar https://github.com/ceph/ceph/pull/18200)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit b552e636ddd8ce2cfb605b4c124440d8dd1e8e47)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 8ad4617f3dea71984d0d2a6119e9e51ef76779fd
+Merge: edf28e6912 59a5a10904
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:20:04 2017 -0500
+
+    Merge pull request #18025 from liewegas/wip-recovery-preemption-luminous
+    
+    luminous: osd: allow recovery preemption
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit edf28e6912c183de4d955f125ef18d8a52594f59
+Merge: 91cb216c01 3354937698
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:19:45 2017 -0500
+
+    Merge pull request #18154 from ktdreyer/luminous-gitignore-debian-patches
+    
+    luminous: .gitignore: allow debian .patch files
+
+commit 91cb216c01eca89ff7e338328d64216998effd16
+Merge: 27d3f8c748 b3f9439cfd
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:19:08 2017 -0500
+
+    Merge pull request #18189 from tchaikov/wip-retry-oserror-luminous
+    
+    luminous: ceph-disk: retry on OSError
+
+commit 27d3f8c748de33a35c5006513d198bbcbee19b45
+Merge: a33dd01164 f60a942023
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:18:27 2017 -0500
+
+    Merge pull request #18227 from liewegas/wip-bluestore-mempool-luminous
+    
+    luminous: os/bluestore: several mempool accounting fixes
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit b435af1137ede418740d2dcf39114d8476775fd8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 08:24:50 2017 -0500
+
+    qa/suites/rados/singleton/all/mon-seesaw: whitelist MON_DOWN
+    
+    Mgr can get marked down when mon weirdness is happening.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0a886bc9f3e9805de04c8c60bd3d78b937737b96)
+
+commit 51c6f0def379ba013a82204398a021663f1b5d56
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 10 20:32:08 2017 +0800
+
+    mon/PGMap: let pg_string_state() return boost::optional<>
+    
+    better to be specific, so we don't run into the same problem even we
+    are using the MSB of uint64_t for a pg state bit in future. we could,
+    however use uint64_t(-1) to indicate the pg_string_state()'s failure to
+    parse the state string, because pg_string_state() only translate a
+    single state a time. but it's always better to be explicit than
+    implicit.
+    
+    Fixes: http://tracker.ceph.com/issues/21609
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 260f87a9f1c4e32ff3a596ea341ad07a901f44a9)
+    
+    Conflicts:
+            src/mon/PGMap.cc
+            src/osd/osd_types.cc
+            src/osd/osd_types.h: trivial resolution
+
+commit 9d12fa5e6dad5492491f4718ba7c593077d984a6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Oct 10 19:54:31 2017 -0400
+
+    common/options: enable multiple rocksdb compaction threads for filestore
+    
+    One of the major benefits of rocksdb over leveldb is multithreaded
+    compaction. The default of 1 thread does not provide much benefit, and
+    is insufficient for heavy rgw workloads.
+    
+    For high-write and delete omap workloads I've seen up to 8 compaction
+    threads be used.  There's little overhead to having a higher max than
+    are needed, so set the default to 8.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 023fa810aa6b3af305e9027e3f717e54d1bb2712)
+    
+    Conflicts:
+        src/common/options.cc (trivial)
+
+commit 6f2f8cdc6a6fa64928fd2d5459b4974e281d857e
+Author: Mark Nelson <mnelson@redhat.com>
+Date:   Thu Sep 21 17:21:16 2017 -0500
+
+    common/options.cc: Set Filestore rocksdb compaction readahead option.
+    
+    fixes: http://tracker.ceph.com/issues/21505
+    
+    Signed-off-by: Mark Nelson <mnelson@redhat.com>
+    (cherry picked from commit 66567e573836c570040c8d8148c34b0cf9dc9ce2)
+
+commit f60a942023088cbba53a816e6ef846994921cab3
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 08:02:50 2017 -0400
+
+    os/bluestore: move several buffer{ptr,list}s into cache_other mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80c60fcde22cf1269ada45d0914543b8f7d49b3e)
+
+commit 37dd32f449bf2b0aaa3dbcba5cc06da076adbb1e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 08:14:37 2017 -0400
+
+    os/bluestore: put new attrs in correct mempool too
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e7762ea1477e861ebc738aa0588a01c0da883d62)
+
+commit dea2c1e6704c40de11c041e9982e9a9f7d9d4cb3
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 22 08:09:37 2017 -0400
+
+    os/bluestore: put attrs in mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef6524436fe095f94fce421ce26ed4c64e1424fe)
+
+commit c2ed1f9b1801e43cf1d345da4d4876de5334cc40
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 22 08:08:52 2017 -0400
+
+    buffer: add ptr::[try_]reassign_to_mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 937379d57a0f8097ae713adf714ba0414d36dffe)
+
+commit a33dd01164bb18b24a0909c9419e15fcd2e79446
+Merge: 9e48df69de c0a1168ec2
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Oct 10 13:17:46 2017 -0700
+
+    Merge pull request #18200 from ceph/wip-yuriw-21660-luminous
+    
+    tests - Added yaml fragmet to cover testing
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c0a1168ec2bacf6d78041f3ae49bfe6fff669913
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Oct 9 12:51:59 2017 -0700
+
+    tests - Added yaml fragmet to cover testing on standanone client.4
+    for http://tracker.ceph.com/issues/21660#note-17
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit b3f9439cfd9141ddde9874c06fd43d346c10c5da
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 7 22:15:11 2017 +0800
+
+    ceph-disk: retry on OSError
+    
+    we are likely to
+    1) create partition, for instance, sdc1
+    2) partprobe sdc
+    3) udevadm settle
+    4) check the device by its path: /dev/sdc1
+    
+    but there is chance that the uevent sent from kernel fails to reach udev
+    before we call "udevadm", hence "/dev/sdc1" does not exist even after
+    "udevadm settle" returns. so we retry in case of OSError here.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4f82dfb9e761a09484e6ba3bd027da535162783e)
+
+commit 83b3377c6b04d78b20b7c23d145e015d63e471a4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 7 21:13:47 2017 +0800
+
+    ceph-disk: factor out the retry logic into a decorator
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 02a8620de7f200736900aafc944b2c1ac47d9910)
+
+commit 9e48df69de14d8415eed008b824204313774e2ba
+Merge: a81b0da8e4 41af4ca8f3
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Sun Oct 8 12:05:25 2017 -0400
+
+    Merge pull request #18167 from tchaikov/wip-luminous-sphinx
+    
+    luminous: admin: bump sphinx to 1.6
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 41af4ca8f3cd3db8dc076c07e1a082a053b91b26
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 11 21:54:33 2017 +0800
+
+    doc: switch to upstream sphinx-ditaa
+    
+    it works with setuptools and is now compatible with py3
+    
+    the py3 branch is created to track the upstream's master branch
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit fda079d5ed4c8a80c677f714a680233a568d512e)
+
+commit e49cae8b03340834e0b44491a4a903c0d61e1f74
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 15 09:12:27 2017 -0400
+
+    admin: bump sphinx to 1.6
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 88a1114cb3f90e52ba718dc832617fa004d63b3c)
+
+commit a81b0da8e4aca275cc3195c04f3e0d485e4a2f31
+Merge: 1e521b5fb4 e4164c43f5
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Oct 6 17:42:30 2017 -0400
+
+    Merge pull request #17914 from liewegas/wip-21498-luminous
+    
+    ceph-disk: fix '--runtime' omission for ceph-osd service
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 1e521b5fb47de74720d6aab39750c53aa2b2d8e7
+Merge: d550610512 ee5b9ea46c
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 6 15:35:06 2017 -0500
+
+    Merge pull request #18135 from liewegas/wip-vta-luminous
+    
+    luminous: cmake: disable VTA on options.cc
+
+commit 33549376987b18a9bdcf5f56e62268c5791124bc
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 7 11:07:59 2017 -0600
+
+    .gitignore: allow debian .patch files
+    
+    The Ubuntu packaging layout with git-buildpackage assumes a
+    "debian/patches/" directory with several .patch files in it.
+    
+    When upstream's .gitignore tells Git to ignore .patch files, we have to
+    edit that line out downstream. When we forget to do that downstream, it
+    can lead to missing patches and broken downstream builds.
+    
+    Allow patches in the /debian/patches directory so it's easier to
+    maintain an Ubuntu package based on upstream's Git repo.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit c734b0c0296152721b658af7b699a64b3a49d251)
+
+commit d5506105120479c1831ab559b555e2d4ea4c357c
+Merge: d46675a1ec 0dc73f1210
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 6 07:38:53 2017 -0500
+
+    Merge pull request #18140 from liewegas/wip-21660-luminous
+    
+    luminous: src/messages/MOSDMap: reencode OSDMap for older clients
+
+commit d46675a1ecb10f7176265cb1fc5ca971209aecb1
+Merge: 39ad203313 8b2bd38785
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 22:12:22 2017 -0500
+
+    Merge pull request #18037 from ajarr/wip-21602-luminous
+    
+    luminous: ceph_volume_client: add get, put, and delete object interfaces
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 39ad2033136e32d76e89e86d3f00bd1b928242ce
+Merge: 473f9d1317 80f6508fc2
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:03:42 2017 -0500
+
+    Merge pull request #18103 from liewegas/wip-21259-luminous
+    
+    luminous: osd: dump bluestore debug on shutdown if debug option is set
+
+commit 473f9d13173aa751d54b9a322060111d627f71f1
+Merge: c50df6511f 49294965f9
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:03:28 2017 -0500
+
+    Merge pull request #18112 from liewegas/wip-localpool-min-size-luminous
+    
+    luminous: mgr/localpool: fix min_size, 3x default, crush rule
+    
+    Reviewed-by: Alexander Marangone <amarango@redhat.com>
+
+commit c50df6511fda7d03c3cfa828acd5e56f2ef45d0f
+Merge: 4ff03e5192 16c6dd5129
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:02:53 2017 -0500
+
+    Merge pull request #18126 from smithfarm/wip-sigpipe-porting-luminous
+    
+    luminous: msg: reimplement sigpipe blocking
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 4ff03e51928c0d7da9a90d50ec504b30eb9e4b32
+Merge: 766c6f3647 7986d4566a
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:02:09 2017 -0500
+
+    Merge pull request #18127 from liewegas/wip-21470-luminous
+    
+    luminous: os/bluestore: fix another aio stall/deadlock
+
+commit 766c6f3647b57b482f76ef254c95a651741d9ad5
+Merge: f972702229 3850f8e0b7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:53 2017 -0500
+
+    Merge pull request #18128 from liewegas/wip-bluefs-allocation-luminous
+    
+    luminous: os/bluestore: make bluefs behave better near enospc
+
+commit f97270222935d1c43a41102c3ada586a7deca190
+Merge: e4c51f62ff f99fe83a37
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:27 2017 -0500
+
+    Merge pull request #18131 from liewegas/wip-ifname-luminous
+    
+    luminous: common/pick_address: add {public,cluster}_network_interface option
+
+commit e4c51f62ffc0f2fcc9214f6b7895210aa79c288b
+Merge: 66af04cd9b f0a6f616c6
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:06 2017 -0500
+
+    Merge pull request #18132 from gregsfortytwo/wip-20416-bitwise-assert-luminous
+    
+    osd: make the PG's SORTBITWISE assert a more generous shutdown
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0dc73f12105ec271a43fe176c2f2a22849e8475c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 15:26:16 2017 -0500
+
+    src/messages/MOSDMap: reencode OSDMap for older clients
+    
+    We explicitly select which missing bits trigger a reencode.  We
+    already had jewel and earlier covered, but kraken includes all of
+    the previously mentioned bits but not SERVER_LUMINOUS.  This
+    prevents kraken clients from decoding luminous maps.
+    
+    Fixes: http://tracker.ceph.com/issues/21660
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d2664aa34dc4c13da64d1ad187019737080069d7)
+
+commit 3850f8e0b791f2d05906647866b14a390ab2f5da
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 11:18:17 2017 -0500
+
+    os/bluestore: add bluestore_bluefs_min_free
+    
+    We need at least ~1GB free so we can write out new SSTs (which are 256MB
+    each).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 65be614f2bfd65a1f13205a3075c87acc72c4836)
+    
+    (avoided 1_G syntax that luminous doesn't support)
+
+commit b015ce116159ec57d20bcd29306e450a816610f9
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Mon Oct 2 14:01:47 2017 -0700
+
+    rgw: stop/join TokenCache revoke thread only if started.
+    
+    Thread::join triggers an assert otherwise.
+    
+    Fixes http://tracker.ceph.com/issues/21666
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.de>
+    (cherry picked from commit 26f2da083c7dd21b89c1c1e6c498b14e034364a6)
+
+commit ee5b9ea46c8b1e831c60f9cc0b10819b14d5efb8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 31 18:15:28 2017 +0800
+
+    cmake: disable VTA on options.cc
+    
+    to silence following warning and to avoid compiling this file twice:
+    
+    ceph/src/common/options.cc: In function ‘std::vector<Option> get_global_options()’:
+    ceph/src/common/options.cc:151:21: note: variable tracking
+    size limit exceeded with -fvar-tracking-assignments, retrying without
+     std::vector<Option> get_global_options() {
+                         ^~~~~~~~~~~~~~~~~~
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4bb56402a6d6f642d54f329aa1e809cbe044e0c6)
+
+commit 59a5a109040884dd2c7c77f3bc84378b54048f1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 15:28:26 2017 -0500
+
+    osd/PG: separate event for RemoteReservationCanceled
+    
+    Right now we transparently map a RemoteReservationRejected into a
+    *Canceled event because this what peers send over the wire.  Even
+    once new peers start sending and explicit CANCEL, old peers will
+    still do so, so we'll maintain this mapping for a while.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 84d71e6a10b02591b5d5e126b346771871eb1575)
+
+commit 206400997daaec97fe9af7e2e735e1b609006107
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 14:55:15 2017 -0500
+
+    osd/PG: separate verb 'Reject' from passive 'rejected'
+    
+    This reduces pg->reject_reservation() callsites from 2 to 1 and
+    makes the state transitions a bit more explicit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bf7f101a787652644c27aae4e752fd21f265e866)
+
+commit 8c1716a42def8547f3af446a143715fad24addb5
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:05:40 2017 -0500
+
+    osd: make note about when we get MBackfillReserve REJECT messages
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6e829a3a7c7aaff78142514fde7703cad48953fd)
+
+commit 260ab6009cdce0d846685e4c7789336db22885f1
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:04:34 2017 -0500
+
+    osd/PG: handle RecoveryReservationRejected in RepWaitRecoveryReserved
+    
+    This state is analogous to RepWaitBackfillReserved; just like we do there
+    we want to handle the REJECT from the primary by canceling our local
+    remote_reservation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ab8f1d291216c07fed7b661896b0a0c1279f50eb)
+
+commit 479b05a8c1659e80028190c7fe90e762084c4c24
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:03:22 2017 -0500
+
+    osd/PG: ignore RemoteReservationRejected if we are RepNotRecoverying
+    
+    The primary may send us a REJECT (meaning cancel) if recovery/backfill is
+    preempted there.  That can happen even if the recovery isn't reserved or
+    requested here (e.g., because the primary is still waiting for the local
+    reservation).  Just ignore it and remain in RepNotRecovering.
+    
+    Fixes: http://tracker.ceph.com/issues/21613
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1ce235c5dd0b777223f6465e71fb7ee4befd13d8)
+
+commit 2a95156d7be9e6c796d5f0ce7bc0eab89c42230d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:01:05 2017 -0500
+
+    osd/PG: cancel local reservation in RemoteReservationRejected handler
+    
+    We can get a RemoteReservationRejected event either because *we* decide
+    to reject, or because we get a REJECT from the primary that means "cancel"
+    (e.g., because recovery/backfill was preempted there).  In both cases we
+    want to cancel our remote_reservation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f5809afb0f2ca2f1187609e645d6c4c5bd73e39d)
+
+commit 92ff2cbd69c444c6648889a2da7e01be6b00311e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 14:59:31 2017 -0500
+
+    osd/PG: move reject_reservation out of RemoteReservationRejected reaction
+    
+    The RemoteReservationRejected event is also submitted when we are a
+    replica or backfill target and get a MBackfillReserve REJECT message
+    because the primary canceled or was preempted.  In that case, we don't
+    want to send a REJECT back to the primary; we only need to send it in the
+    cases where *we*, locally, decide to reject.  Move the call to those call
+    sites.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 57d18f0e8d9b0428a193c63112a28ddd046337b4)
+
+commit f0a6f616c6c0358c2a7b9eb24fd6ca9c4290be0a
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Sep 29 15:18:26 2017 -0700
+
+    osd: make the PG's SORTBITWISE assert a more generous shutdown
+    
+    We want to stop working if we get activated while sortbitwise is not set
+    on the cluster, but we might have old maps where it wasn't if the flag
+    was changed recently. And doing it in the PG code was a bit silly anyway.
+    
+    Instead check SORTBITWISE in the main OSDMap handling code prior to
+    prepublishing it. Let it go through if we aren't active at the time.
+    
+    Fixes: http://tracker.ceph.com/issues/20416
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0a691b2b1c19fbc30af5c4046736dacb2fb8bfa4)
+
+commit f99fe83a37241fe1c616ecb854b5196112985861
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 17:47:17 2017 -0400
+
+    common/pick_address: add {public,cluster}_network_interface option
+    
+    Add _interfaces option to constrain the choice of IPs in the network
+    list to those on interfaces matching the provided list of interface names.
+    The _interfaces options only work in concert with the _network options,
+    so you must also specify a list of networks if you want to use a specific
+    interface, e.g., by specifying a broad network like "::" or "0.0.0.0/0".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 95cc790acddb597d6fef2d9e444f0b6e0436f16f)
+
+commit b84803d6092667631527b8b37000cac6447e0b84
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 17:17:13 2017 -0700
+
+    osd: Only scan for omap corruption once
+    
+    Before
+            state 2: Can have complete tables (some may be bad)
+            state 3: Never had complete tables
+    After
+            state 2: Can have complete tables (some may be bad)
+            state 3 with legacy: Can have complete tables (bad ones are cleared)
+            state 3: Never had complete tables
+    
+    Once OSDs boot with this change you can't downgrade to a previous release.
+    If someone does downgrade they could have unstable OSDs that hit assert(state.v < 3).
+    The following command run after shutting down the cluster but before downgrading
+    ceph packages would be a way to fix this.
+    
+    ceph-osdomap-tool --omap-path ... --command resetv2
+    
+    Fixes: http://tracker.ceph.com/issues/21328
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8805ef53424e30fd3f24ee38f5a6bdd9e6dd8641)
+
+commit d0ea152865eeff63cf49e137f5ac342ec6725a6a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 18:06:10 2017 -0700
+
+    tools: Add --backend option to ceph-osdomap-tool default to rocksdb
+    
+    Fix hard-coded "leveldb" backend.  The command is broken in Luminous
+    now that "rocksdb" is the default.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de43493990923bcdd20f88e9d04647e5ba709feb)
+
+commit 94090379a13639ce0a8619dfe178cc62acdd8fc4
+Author: Luo Kexue <luo.kexue@zte.com.cn>
+Date:   Mon Aug 28 09:24:10 2017 +0800
+
+    osd, mds, tools: drop the invalid comment and some unused variables
+    
+    Signed-off-by: Luo Kexue <luo.kexue@zte.com.cn>
+    (cherry picked from commit 2e93424167de15e91394169f4395f5f446e710e2)
+
+commit 382ff40fb17db2c657318038495ae7140f95b7e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 23:14:15 2017 -0700
+
+    tools: Add the ability to reset state to v2
+    
+    Available for testing and user downgrade.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3e4d68640cc43efc0cf10ea8119b3aa583b7f36b)
+
+commit dc2c1d9d2fdd5449dd7c3213e1d2b5cbd928d4d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 22:12:52 2017 -0700
+
+    tools: Show DB state information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3214882a95f9f70d9f6d28f4e403ee16324530f9)
+
+commit 36675e34b610ed6afa90bdb93da6e49025e4d9e1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 10:41:33 2017 -0500
+
+    os/bluestore/BlueFS: crash on enospc
+    
+    We were passing this error back to rocksdb, which would then crash (much)
+    later with a corrupted SST file!  No good.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 569e924812bd7cc61e6e755f265890dbd5c95c31)
+
+commit 16c6dd5129f54a6f5eec7fe16719a179c176548e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 3 15:54:06 2017 -0700
+
+    msgr: add a mechanism for Solaris to avoid dying on SIGPIPE
+    
+    This is fairly clean: we define an RAII object in the Messenger.h on
+    Solaris, and "declare" it with a macro in the implementations. There's
+    no code duplication and on Linux it's just entirely compiled out.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cba20a95e816aaf9f0971b29b14b0be8c524b59d)
+    
+    Conflicts:
+        src/msg/async/PosixStack.cc - luminous #include "common/simple_spin.h" is
+            missing in master (trivial resolution)
+
+commit 7986d4566a1f6d8e22a9acd19608530ce64cbfa1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 16:48:37 2017 -0500
+
+    os/bluestore: use normal Context for async deferred_try_submit
+    
+    I'm not quite sure why the FunctionContext did not ever execute on the
+    finisher thread (perhaps the [&] captured some state on the stack that it
+    shouldn't have?).  In any case, using a traditional Context here appears
+    to resolve the problem (of the async deferred_try_submit() never executing,
+    leading to a bluestore stall/deadlock).
+    
+    Fixes: http://tracker.ceph.com/issues/21470
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 67ec75805787ed63b35f8d70478a7a2cd785df06)
+
+commit 9ab0f64bcf74f74838f6066d554f728902c31715
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 29 13:47:19 2017 -0500
+
+    os/bluestore: wake kv thread when blocking on deferred_bytes
+    
+    We need to wake the kv thread whenever setting deferred_aggressive to
+    ensure that txns with deferred io that have committed but haven't submitted
+    their deferred writes get submitted.  This aligns us with the other
+    users of deferred_aggressive (e.g., _osr_drain_all).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 633b17007701d87903fe1d10b19c44210a3326eb)
+
+commit 0251cfcd17111334d3f13dc1e9d52a9aa9302354
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 3 15:06:25 2017 -0700
+
+    Revert "SIGPIPE suppression for platforms without SO_NOSIGPIPE or MSG_NOSIGNAL"
+    
+    This reverts commit 131deb39769c1187c334ee84f552d3be01f1751b. It added
+    code that shouldn't have entered the project repo.
+    
+    Conflicts:
+            src/msg/async/AsyncConnection.cc
+            src/msg/async/AsyncConnection.h
+            src/msg/simple/Pipe.cc
+            src/msg/simple/Pipe.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 21bdb730932196a48407be0effab8813a78c4b1b)
+
+commit 72c858310796848128bbccfa328794d504471e5a
+Author: wujian <wujian3659@163.com>
+Date:   Fri Sep 15 11:23:44 2017 +0800
+
+    ceph-mgr: can not change prometheus port for mgr
+    
+    Signed-off-by: wujian <wujian3659@163.com>
+    (cherry picked from commit 6cc0338ed15a0402471cfd05acfcf00d07517d1a)
+
+commit 49294965f9f92946382949264344c2aa0b6d70bd
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 08:25:38 2017 -0500
+
+    mgr/localpool: fix rule selection
+    
+    The 'osd pool create' arg parsing is broken; the rule name for
+    'ceph osd pool create $name $numpgs replicated $rulename' is passed
+    via the erasure_code_profile param.  Too many req=false options
+    without a way to disambiguate them.
+    
+    Work around it by passing both 'rule' and 'erasure_code_profile'
+    keys, so that if/when the hack in OSDMonitor.cc is removed it will
+    still work.  Blech.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7623513935525498640defa2064c291fd69a2b76)
+
+commit cf6f90c5e1e17a1f44f6ed80422b5f1c543362ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 2 17:11:46 2017 -0500
+
+    mgr/localpool: optionally adjust min_size too
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit af72a8932c60a52aad76f1cc94bfbb31048215cc)
+
+commit d18b7b8652e38c679993241ac232bb7310db7e22
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 2 17:11:38 2017 -0500
+
+    mgr/localpool: default to 3x
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9624923f19dc649a8f182f28e9580a3045964d9e)
+
+commit 80f6508fc25018dd9c70d05f99c40325062d129e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 15:19:47 2017 -0400
+
+    osd: make shutdown debug conditional (and off by default)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6169cf64423c7554dc0630d90d8dee27437bf05f)
+
+commit ca177e93d4e52d50a49b76af516a8009153197e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 17:12:42 2017 -0400
+
+    osd: debug_bluestore on shutdown
+    
+    ...just like we do with filestore etc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 354dccda322b7ac8844b321f603e049128bf4779)
+
+commit 4d931ad0d349a6974f9f7053792e6f826b67169a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 17:11:30 2017 -0400
+
+    os/bluestore: dump stray cache content on shutdown
+    
+    Tracking down http://tracker.ceph.com/issues/21259.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 28d9b6b0e92cf51996a12a43c81f7ac2abcaecaa)
+
+commit 66af04cd9b61d482362361cb3b12af2d8ffa462c
+Merge: 57b5f58f5e b064ed172a
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 3 13:41:21 2017 -0500
+
+    Merge pull request #17734 from liewegas/wip-21089-onmount
+    
+    luminous: os/bluestore: replace 21089 repair with something online (instead of fsck)
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 57b5f58f5e8be116deed59c5c97636eb730c3dc8
+Merge: ae9de1673a a069e8ae61
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 3 13:40:56 2017 -0500
+
+    Merge pull request #17814 from liewegas/wip-pg-limits-luminous
+    
+    luminous: mon: cluster limit on pgs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a069e8ae61a452cefe5673f0035251c00857fee5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 16:01:14 2017 -0400
+
+    doc/rados/operations/health-checks: fix TOO_MANY_PGS discussion
+    
+    Fiddling with pgp_num doesn't help with TOO_MANY_PGS.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 027672b777402381f6736e517ed287b38bb17abb)
+
+commit 7320ee7293a34e4ea0ab5254373f51b93249b91d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 16:00:31 2017 -0400
+
+    mon: rename mon_pg_warn_max_per_osd -> mon_max_pg_per_osd
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 986b86fbebf9e06f9f841da8ded0bedb310fc69b)
+
+commit 37ddfc61c3376afeccfe7a8029412b9a38401bfc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 22 12:03:57 2017 -0400
+
+    rgw: dont reuse stale RGWObjectCtx for get_bucket_info()
+    
+    if the earlier call to store->get_bucket_entrypoint_info() failed with
+    ENOENT, the obj_ctx will cache exists=false.
+    put_bucket_entrypoint_info() doesn't invalidate that, so this call to
+    get_bucket_info() was reading from a stale cache and failing with ENOENT
+    
+    Fixes: http://tracker.ceph.com/issues/21506
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3f4f760a85f162262c5b419e9cf78966c7299f0a)
+
+commit 28e7d5645e5fdf5471a2a809bd232521f2d33814
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 20 12:38:46 2017 -0400
+
+    os/bluestore: ignore 0x2000~2000 extent oddity from luminous upgrade
+    
+    Luminous does a block_size granularity freelist, and assumes that
+    0~ROUND_UP_TO(SUPER_RESERVED,block_size) is used.  Current master uses
+    min_alloc_size granularity and changes that assumption to
+    0~ROUND_UP_TO(SUPER_RESERVED,min_alloc_size).  That means if master
+    fsck's a luminous-created bluestore, it will think 0x2000~2000 is used
+    (current baked-in min_alloc_size-based assumption) but the old freelist
+    says it is free (old mkfs assumption).  The disparity is harmless since
+    the extent is below min_alloc_size, so ignore it.
+    
+    Fixes: http://tracker.ceph.com/issues/21408
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4959ad310a96208565b47c035ab5d5dded1b8ff5)
+
+commit ae9de1673a7b4aa032cbde7c5646c4d945ff618a
+Merge: eec0edaaa0 3a037dda42
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:56:10 2017 -0500
+
+    Merge pull request #17867 from smithfarm/wip-21443-luminous
+    
+    luminous: mgr: Prometheus crash when update
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit eec0edaaa031400e7728081c306a1901cb981486
+Merge: 98e729a977 ddca5dbe04
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:59 2017 -0500
+
+    Merge pull request #17868 from smithfarm/wip-21452-luminous
+    
+    luminous: mgr: prometheus module generates invalid output when counter names contain non-alphanum characters
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 98e729a9773acb5200357b3178cf04883383cf4a
+Merge: 87e8829605 faf3fec1a6
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:30 2017 -0500
+
+    Merge pull request #17896 from liewegas/wip-bluestore-tool-luminous
+    
+    ceph-objectstore-tool and ceph-bluestore-tool: backports from master
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 87e8829605adb2609cd4867f3d23489d84e657fb
+Merge: a48b2e41e7 1fc6a51a41
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:11 2017 -0500
+
+    Merge pull request #17930 from jcsp/wip-luminous-health-warn
+    
+    mon: show legacy health warning in `status` output
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a48b2e41e78f9190b7f94d11af2a04c919380aae
+Merge: ced74c00d6 4f28ef0016
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:54:47 2017 -0500
+
+    Merge pull request #18031 from xiexingguo/wip-pr-17987
+    
+    luminous: osd/PrimaryLogPG: kick off recovery on backoffing a degraded object
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ced74c00d6de9fbf84ecb4d3064fbaec01ec827f
+Merge: 242635d938 f9d0442ca5
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:54:17 2017 -0500
+
+    Merge pull request #18033 from xiexingguo/wip-pr-17698
+    
+    luminous: os/bluestore: allocate entire write in one go
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 242635d938fcd46d337b7727f0f3f578122b41ad
+Merge: 9d8acf6f3f 86d143722e
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:53:56 2017 -0500
+
+    Merge pull request #18038 from jecluis/wip-21300-luminous
+    
+    luminous: mon/MgrMonitor: handle cmd descs to/from disk in the absence of active mgr
+
+commit 1021fe235ca756337e3a3ff4804680c1e93b0270
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 29 08:48:14 2017 -0700
+
+    qa: relax cap expected value check
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b37c7f7db7f0d76e524aabdad6dd0f16548268a6)
+
+commit 633b114aca65548b91a898db7bbbef784b76180b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:08:51 2017 -0400
+
+    os/bluestore: use min_alloc_size for freelist resolution
+    
+    For HDD with min_alloc_size=64k, this is a 16x reduction in allocation
+    metadata!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b8e4d512604095fb8a209229d4633ac19b499de)
+    
+    Conflicts:
+    Slightly conflict with 834542c4027a8cc4d23eff089827d328919372d2, which
+    drops the literal description of apply().
+
+commit 3decf1c0de9f54b09afc954b13c2c8fac1011de9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:08:07 2017 -0400
+
+    os/bluestore: align bluefs_extents to min_alloc_size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0c777efdcb2ee5a6322f0eb277e681d0f086e0b6)
+
+commit 529c02db4da9b45cc13bedaa20509a896217fd83
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:07:38 2017 -0400
+
+    os/bluestore/FreelistManager: create: accept min alloc size
+    
+    Accept a block size other than bdev_block_size.  Let's call it, oh, I don't
+    know, min_alloc_size.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 52453d4ca223c8819f8e35f2c0b691803e74537f)
+
+commit 37929849863c60531bcdbf80124e751504ac710f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:06:05 2017 -0400
+
+    os/bluestore: mkfs: choose min_alloc_size earlier
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3efde01586776b23bbac1e663ae5baf6500acee4)
+
+commit efcf874295db0a68c43a9fd0a3a20cc3dcdf050a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:05:29 2017 -0400
+
+    os/bluestore: require that bluefs_alloc_size be multiple of min_alloc_size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b47ac59816894e983a98d3da8b5415d569c6663)
+
+commit 58c52baeaed4a0a6dd5aed076db60988845431d0
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 17:42:07 2017 -0400
+
+    qa/suites/rados/singleton/all/recovery-preemption: add test
+    
+    This mirrors what I was testing locally.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d7b29acb1952d0b3dfd19326fd3418cfbd66ef3c)
+
+commit 2e28f3e5ae3a4140e802f915b8db5dc434b6a3c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 12:37:13 2017 -0400
+
+    osd/PG: handle racy preemption
+    
+    If we finish recovery/backfill and go active, but also get
+    preempted at the same time, we can ignore the event.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d8c3756d26abbaf326f634a494dcb12fe373f68d)
+
+commit ba106b6d32fb3138b7a61fe613b211e02f1e44aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:26:40 2017 -0500
+
+    osd/PG: allow local recovery reservations to be preempted
+    
+    If a PG has a higher recovery priority and a lower-priority item is in
+    progress, allow it to be preempted.  This triggers the RecoveryCancel
+    or BackfillCancel event with a 0 delay, which means it will immediately
+    re-request a reservation (and presumably wait).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a8534cce1a1661375a93aff2314633bce63695ed)
+
+commit 33ea286d0cedf44b7b3c70bc9f35c7f88fcc0361
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:25:05 2017 -0500
+
+    common/AsyncReserver: support preemption
+    
+    If an (optional) preemption context is provided, use that to preempt
+    and existing reservation and grant a higher-priority one.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dbc002eaa90e952df1acf295a630443ac3ada418)
+
+commit dad61658d1f48f1089e265d7441bf1349fd42f1f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 12:37:49 2017 -0400
+
+    common/AsyncReserver: get a cct
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08d2c8875bba7c41d1f9eeec586f6880f8be99b2)
+
+commit 1b0875ca6d9ab91f59ffe2356113ed2cc643a276
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:30:57 2017 -0400
+
+    osd: PG_STATE_BACKFILL -> PG_STATE_BACKFILLING
+    
+    Match user-facing string
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 31a34946f7aa0ad2b9848cbc808d209394a9ad44)
+    
+    - add update to mon/PGMap.cc
+
+commit 9d8acf6f3f85dd1410629cc680474e38f3be9838
+Merge: 62ff5e4c98 b54c7ba225
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:11:31 2017 -0500
+
+    Merge pull request #17862 from smithfarm/wip-21307-luminous
+    
+    luminous: mon: Client client.admin marked osd.2 out, after it was down for 1504627577 seconds
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 62ff5e4c981a9e0d9a92bf3ac85210d98b0db038
+Merge: 453858d4b7 4c9d86bd86
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:55 2017 -0500
+
+    Merge pull request #17865 from smithfarm/wip-21465-luminous
+    
+    luminous: OSD metadata 'backend_filestore_dev_node' is unknown even for simple deployment
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 453858d4b777ee81e5af4f443afc07e6af8b1a60
+Merge: 2e85e4f0c4 430a140d38
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:26 2017 -0500
+
+    Merge pull request #17936 from liewegas/wip-ruleset-errors-luminous
+    
+    mon/OSDMonitor: error out if setting ruleset-* ec profile property
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2e85e4f0c4f196dce8115e0a3aecc506caf1c7d1
+Merge: 1c228c95dc 481da57cca
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:06 2017 -0500
+
+    Merge pull request #17946 from liewegas/wip-mgr-localpool-luminous
+    
+    luminous: mgr/localpool: module to automagically create localized pools
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1c228c95dc142c3ca142f039e606e25cb500e53a
+Merge: 9915a2f600 9aafc53ed4
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:09:27 2017 -0500
+
+    Merge pull request #17998 from liewegas/wip-pr-17978-luminous
+    
+    rbdmap: fix umount when multiple mounts use the same RBD
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 86d143722e1f16a19a56ed7a163d1dc9327c0c90
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Sep 20 17:53:40 2017 +0100
+
+    mon/MgrMonitor: read cmd descs if empty on update_from_paxos()
+    
+    If the MgrMonitor's `command_descs` is empty, the monitor will not send
+    the mgr commands to clients on `get_descriptions`. This, in turn, has
+    the clients sending the commands to the monitors, which will have no
+    idea how to handle them.
+    
+    Therefore, make sure to read the `command_descs` from disk if the vector
+    is empty.
+    
+    Fixes: http://tracker.ceph.com/issues/21300
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 3d06079bae0fbc096d6c3639807d9be3597e841a)
+
+commit 8b2bd387859bdc2287ba6dcaba27d8b234984a5a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 13 19:53:43 2017 +0530
+
+    pybind/ceph_volume_client: add get, put, and delete object interfaces
+    
+    Wrap low-level rados APIs to allow ceph_volume_client to get, put, and
+    delete objects. The interfaces would allow OpenStack Manila's
+    cephfs driver to store config data in a shared storage to implement
+    highly available Manila deployments. Restrict  write(put) and
+    read(get) object sizes to 'osd_max_size' config setting.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit d1bd171d6b6eb00c47168f38cec1a30f9c9f02bd)
+
+commit 9421b40f1748c50b7f92abe8615c0cd9208fdbdc
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Mon Sep 18 20:16:30 2017 +0530
+
+    pybind/ceph_volume_client: remove 'compat_version'
+    
+    ... class attribute of the 'CephFSVolumeClient' class. It was supposed
+    to record the earliest version of CephFSVolumeClient that the current
+    version is compatible with. It's not useful data to be stored as a
+    class attribute.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 894a734aa5a56b20d3f68bb3ad644b370f193934)
+
+commit dd23023329b2382afef23ee02d7ef9b14baaf07b
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Mon Sep 18 20:09:55 2017 +0530
+
+    pybind/ceph_volume_client: set the version
+    
+    ... of on-disk structures to be same as the class attribute 'version'
+    of the CephFSVolumeClient class.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 8267c2ba09841095f20a8833c155185529e64f46)
+
+commit 9ee3f7ad21049d0692804d4405b4bf0f8c795ae3
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Sep 27 17:55:17 2017 +0100
+
+    mon/MgrMonitor: populate on-disk cmd descs if empty on upgrade
+    
+    During kraken, when we first introduced the mgrs, we wouldn't populate
+    the on-disk command descriptions on create_initial(). Therefore, if we
+    are upgrading from a cluster that never had a mgr, we may end up
+    crashing because we have no cmd descs to load from disk.
+    
+    Fixes: http://tracker.ceph.com/issues/21300
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+
+commit f9d0442ca50078224b72f9876d2fed41fd409be6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 16:44:59 2017 -0400
+
+    os/bluestore: allocate entire write in one go
+    
+    On the first pass through the writes, compress data and calculate a final
+    amount of space we need to allocate.  On the second pass, assign the
+    extents to blobs and queue the writes.
+    
+    This allows us to do a single allocation for all blobs, which will lead
+    to less fragmentation and a much better write pattern.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e200f358499af8e3acb6ac4f675cc167433b53ec)
+
+commit 4f28ef00162692557654d84bc1c0988998a6f2c4
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Sep 27 16:05:56 2017 +0800
+
+    osd/PrimaryLogPG: kick off recovery on backoffing a degraded object
+    
+    As we are now blocking frontend ops from accessing that very object!
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit db20328b456d70d6728fd27f17da6f2f3546e84b)
+
+commit 711892dceaa7e4598bffff0edcff45554bc06ffc
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 28 16:01:00 2017 -0700
+
+    mds: improve cap min/max ratio descriptions
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit dcf97d17356f59c8870a36a99a77f879a20df348)
+
+commit bef8ded7b82fc6896398bd334bdbd4fc63ebfe51
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 28 15:56:26 2017 -0700
+
+    mds: fix whitespace
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit ab69d682937a0a13246d28cfa2bbbf8c1b7ced38)
+
+commit a5c96d4f7a5081242c6abf6c722adfa131030a55
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 10:39:01 2017 -0700
+
+    mds: cap client recall to min caps per client
+    
+    Fixes: http://tracker.ceph.com/issues/21575
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 538834171fe4524b4bb7cffdcb08c5b13fe7689f)
+
+commit 913c72d96a6c405b4e4fe57abcfbf58d53a66e63
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 09:29:39 2017 -0700
+
+    mds: fix conf types
+    
+    This correct an assertion failure.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit bfc490249566356cff785f2f89dc03d69af322da)
+
+commit 91084538cdfe80230bf346ee9cb6e8bfe080d6d8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 09:29:23 2017 -0700
+
+    mds: fix whitespace
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8a5d71bf4517e6a3c194c3e63e0283747b75d4b1)
+
+commit 5b1306142feb820a6c4f6e180ba68e428985b3f1
+Author: ren.huanwen <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 18 15:51:29 2017 +0800
+
+    doc/cephfs: add client min cache and max cache ratio describe
+    
+    Signed-off-by: ren.huanwen <ren.huanwen@zte.com.cn>
+    (cherry picked from commit d879ff91e76f66f90364038920fc2c62dd18c449)
+
+commit a39ab633cb97fd4fa0e1ef3e0e2e56709f980857
+Author: ren.huanwen <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 18 15:14:19 2017 +0800
+
+    mds: adding tunable features for caps_per_client
+    
+    Sometimes we need to raise or lower the value of "max_caps_per_client" and
+    "min_caps_per_client" to improve recall_client_state efficiency
+    
+    Signed-off-by: ren.huanwen <ren.huanwen@zte.com.cn>
+    (cherry picked from commit a07b376628505832f5528bd7f2b0faeabe9a5f5d)
+
+commit 4ff28ab144b3a0adfea8ad8ddee750243c824d14
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:28:59 2017 -0400
+
+    osd/osd_types: make BACKFILL <-> "backfilling" for parser
+    
+    We render BACKFILL as "backfilling"; make sure parse works that
+    way too.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fa40e44dc579a009edff7be95d3fd37defbc7f6)
+
+commit 2a9cfa2ad256df11bc1c3d95720ef89d11813846
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:28:08 2017 -0400
+
+    osd/osd_types: remove weird BACKFILL state hack
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5bcfaf4bd40fee9ea4bfd5ed086b51facc5fe78e)
+
+commit 429804b6bd193611f7474174e45ce94d784fd184
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:47:23 2017 -0500
+
+    osd/PG: Cancel{Recovery,Backfill} -> Defer{Recovery,Backfill}
+    
+    "Defer" is more accurate here; we aren't canceling anything, just
+    rescheduling the work.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e45497a20a0b61975fbf0cf851d417f31b35489)
+
+commit 4cbd62ecdc42aa91045f7f5c7ce5dab0cb59c16e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:53:31 2017 -0500
+
+    osd/PG: specify delay in Cancel{Recovery,Backfill}
+    
+    For now it is always the retry interval, but later perhaps not!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 597dfd11728b55ad181316ac10be7155e37a6ba9)
+
+commit 86778de01166902612b3173f1d0687f13027a21f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:49:05 2017 -0500
+
+    osd/PG: make some trivial events TrivialEvent
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3eadfa087f3ff5338f87306fd384d1f4749b8005)
+
+commit 6c529092e8f4bd1e2a4a4069bf9f319d49430ccb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:44:50 2017 -0500
+
+    osd/PG: set {backfill,recovery}_wait when canceling backfill/recovery
+    
+    The only caller currently is when we get as far as we can with backfill
+    or recovery but still have unfound objects.  In this case, go back into
+    the *_wait state instead of appearing as though we are still doing
+    something.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b216771bd31e5712471e13ad32ee8a1e519eb30)
+
+commit 06c31a6ceecf1c33e1456c746169961802903001
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 22:19:06 2017 +0800
+
+    mds: make sure snap inode's last matches its parent dentry's last
+    
+    This patch is for luminous only. The issue has been fixed in master
+    branch in another way (change is much bigger)
+    
+    Fixes: http://tracker.ceph.com/issues/21337
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 1236e0f2e9e8724569cf5fc7d7d525c7fca6ad77
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sun Sep 24 22:11:22 2017 +0800
+
+    rgw_file: fix write error when the write offset overlaps.
+    
+    Fixes: http://tracker.ceph.com/issues/21455
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 872d73f945364002f0fa31762e6976db5b4b3c19)
+
+commit 9aafc53ed4edca8bebb96bc36a0b37ce32c79a75
+Author: Alexandre Marangone <a.marangone@gmail.com>
+Date:   Tue Sep 26 11:35:04 2017 -0700
+
+    rbdmap: fix umount when multiple mounts use the same RBD
+    
+    When a Kubernetes Pod consumes a RBD it is mounted two
+    times on the same host. When the host shutdown umount will
+    fail leading to a hung system
+    
+    Signed-off-by: Alexandre Marangone <amarango@redhat.com>
+    (cherry picked from commit 40825daecedb2a3481021e4d36a367c339eb9b62)
+
+commit 9915a2f600788c37fb42f39a766ce93b9ae4d816
+Merge: 3e7492b9ad 6b5f212b91
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Wed Sep 27 09:50:44 2017 -0700
+
+    Merge pull request #17953 from ceph/wip-ceph-ansible-bport
+    
+    tests - ceph-ansible backports
+
+commit 6b5f212b9170dbd99779527b41be233b9d40b5e4
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 21 15:34:31 2017 -0700
+
+    use ovh instead of vps
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 56d1511bdfaee5a2a0d85bd606348632bfcbf09e)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit ab5300218699c947f380b0437f931d559bdbf074
+Author: Sébastien Han <seb@redhat.com>
+Date:   Tue Sep 26 14:05:37 2017 +0200
+
+    rbd-mirorr: does not start on reboot
+    
+    The current systemd unit file misses 'PartOf=ceph-rbd-mirror.target',
+    which results in the unit not starting after reboot.
+    If you have ceph-rbd-mirror@rbd-mirror.ceph-rbd-mirror0, it won't start
+    after reboot even if enabled.
+    Adding 'PartOf=ceph-rbd-mirror.target' will enable
+    ceph-rbd-mirror.target when ceph-rbd-mirror@rbd-mirror.ceph-rbd-mirror0
+    gets enabled.
+    
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit e6cd9570ba210c5f4501b6f4fa443245a22c4d6c)
+
+commit 481da57cca2f43f3cb14beaaa22eff3b2f62c875
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 12:10:28 2017 +0800
+
+    doc/mgr: add "local pool" plugin to toc
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0b831e7886daad5836ca3cb58c01c123f99a1cde)
+
+commit 315e336f38d1bb0e821dbc1943447cc61f203e2c
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:09:08 2017 -0700
+
+    qa/tests: skip test that use public bucket landsat-pds from AWS
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e11a1f773d8795dd9bd9819aaf229c6598e22ca6)
+
+commit fc7db58fc5ed071436d5f0f6cc9dcd61e3276aea
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:06:03 2017 -0700
+
+    qa/tests: update required ceph-ansbile vars
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 184d5e6bad46451b3a8c5787f56148566bb42f20)
+
+commit 4342d80354c941d47598897ca7ec6a6b4928edf6
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:03:46 2017 -0700
+
+    qa/tests: Fix ceph-ansible upstream vars
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 077904a0b5dffe9d64feade94cf30ffc92f1c0e5)
+
+commit a3d43ad8f3e685a028e943b5aee9ec4720cba60b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Sun Sep 10 12:42:16 2017 -0700
+
+    Fix get_system_type failure due to invalid remote name
+    
+    recent changes caused the remote name to be invalid, fix the
+    arg passed to get_system_type
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9fe58d5ed6bd2338271a9e003f51d80c2e440e23)
+
+commit ea8ddc95a8336f37065edfd7f061601c5c4cac4b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Sep 8 10:00:49 2017 -0700
+
+    Stop the mgr on node which is not client, this will ensure
+    the client.0's mgr is active mgr for workunit to work.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit b1fd615e9f507d4243744b34887366086c1890ac)
+
+commit 69bbaf5173c09afbf02b1894843570d403ad4235
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 7 17:10:04 2017 -0700
+
+    Test ceph-mgr RESTful api
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e5b5a1b8c8d0e750d29fe41a16252dcb0adf449f)
+
+commit 581656e3874822f958400c235241cddb07c4aca3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 7 16:54:51 2017 -0700
+
+    Add additional variables as required by ceph-ansible for upstream
+    installation.
+    
+    possible revert if those are made default
+    issue: https://github.com/ceph/ceph-ansible/issues/1834
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e963bfd6ca0ea1525e37dc35bf3309548c7a3831)
+
+commit 559a47997c7f6c1e8e86e2fd06759ded0e7cccd9
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:35:54 2017 -0700
+
+    Rename folders to fix task order
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9cc00c5c1a6e99323a4655aad5c093b8d87609f6)
+
+commit 655b0733970c6798af087fdb886bf914711fef1b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:09:50 2017 -0700
+
+    use bluestore with dmcrypt option
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1041c803f1f8f7be0edd0e78df352152fa3c4e6f)
+
+commit 244689b36a2beb6dbd9beed429e5b46f4b714bc3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:08:00 2017 -0700
+
+    Add dmcrypt option
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f6de5d9f9e2a3666116311c146a8f2a45f3b8dbb)
+
+commit d2ddcc0324a66b1cb67a640df987e5bc1a6f4887
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:05:01 2017 -0700
+
+    Separate the main task from options
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 60d00e0eadda3a7a7cfbac2acdc81a4d55a27570)
+
+commit 05f7ce97a8b94b5abcaddd7d125398c7d0118bbc
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 10:47:22 2017 -0700
+
+    Catchup with recent changes with ceph-ansible
+    
+    Adds osd_scenario and ceph_stable_release variables
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 0395b844884a63a2cb7480dcc40762df6915b403)
+
+commit 58ffed6fb9e8695deb53edd7754aed56c73403bd
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Aug 24 11:02:20 2017 -0700
+
+    Add workaround for http://tracker.ceph.com/issues/20950
+    
+    mgr bootstrap key differs on disk, rewrite the new key
+    using auth get.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 5c43c5972ddb04724c8659a1b8d71cda1e0acdfb)
+
+commit c2284be89df724bbf1446b28cdf712722d7581a1
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Aug 24 09:24:52 2017 -0700
+
+    Add kraken to luminous upgrade case
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 71ea4987e0d55cc49e90dfbaf9722769cc892f3f)
+
+commit 6cc150023f1dc2450bbf83ebdba8360c5693645c
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Aug 7 12:19:41 2017 -0700
+
+    Add kernel tar/untar and systemd task
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 526d4aef1cb73ffa6f73e1353dc06eed6cccb9f9)
+
+commit 39eb7cb0bd77925d422c68aaed86fbd39bdfc80e
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Aug 1 16:32:42 2017 -0700
+
+    Restart services after upgrade
+    
+    This seems to be an issue and should be fixed in our systemd scripts,
+    need to discuss more on this.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 2116f76a1f1fbb32ca5844c1d051e22b1b657e0f)
+
+commit 566dada1f6a73b8be128167a425becea7ba110fe
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Jul 28 23:11:41 2017 -0700
+
+    since the roles are mapped inside ceph-deploy, store the roles that
+    are mapped and use the new mapped role for upgrades during later
+    stage.
+    
+    eg: mon.a is mapped to mon.mira002 during install, store this mapping
+    and durig upgrade map it back to appropriate name to find the hostname
+    with that role
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 8a2b9a08095dc97e0e440cc900c650d5efb55a4f)
+
+commit 093a13268dff5b1114294456e83e0e24bfd7be6d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Jul 28 17:48:09 2017 -0700
+
+    Add doc string for missing ceph-deploy-branch
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e7a0a4f87f2fbea0c9e27341c952245f1ad34ab8)
+
+commit df415e81887b06918731d7d3b7677ecab1ee2db5
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jul 27 15:24:25 2017 -0700
+
+    use mon.a as ceph-admin node, the previous get_first_mon returns
+    a sorted(mons)[0] as first mon.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 407ce2613d37252e48c2a648b94b065ab070ec46)
+
+commit 1128ae099cc350f29b243321bc2815aaf7bbfe80
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jul 27 11:17:31 2017 -0700
+
+    use elif instead of else to handle cases for jewel install
+    
+    jewel needs neither filestore or bluestore as an option, so provide none
+    when running with jewel branch.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f3ec7cabf66c76d712f77a4dfefe96f374507982)
+
+commit 7db1550a55002d28d899aee6ec695604045edabb
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 19:18:11 2017 -0700
+
+    Jewel to luminous upgrade scenario using ceph-deploy
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9c85c76a90dcf46ec2bcc6840f4fc57ed86583b5)
+
+commit ab219d2e985cd1c6a7fd17b5ced4db25cb0b7427
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 18:56:08 2017 -0700
+
+    Add upgrade functionality using ceph-deploy
+    
+    This is to test for customer like upgrade scenarios and to find
+    any issues that may be related to systemd, packaging etc
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9a73127c16d3395934d003024964ed1373d81083)
+
+commit a145127bde2e2fb96a1fd2b821ed2b647e8ae223
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 15:01:15 2017 -0700
+
+    Fix old flake8 issues
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit b409c3d163194b0b65e26b0fbc93def8e17a10b4)
+
+commit 28b3c242bdbfc655534cdd037966ed1dcddc978a
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 14:58:50 2017 -0700
+
+    Add option to skip mgr install for old release
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit a9be0c387ed2ae09227dad833c8647af110bd2de)
+
+commit 82651da2d504913f2145085df8a89484308f3eba
+Author: huangjun <huangjun@xsky.com>
+Date:   Wed Sep 20 00:04:04 2017 +0800
+
+    qa/ceph_manager: check pg state again before timedout
+    
+    Pg state maybe all in active+clean when no recovering going on,
+    so check it again before timedout.
+    
+    Fixes: http://tracker.ceph.com/issues/21294
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit fa40add7f0123dfeac30986f3d53cdfa77736a87)
+
+commit 4017a26162706f1b6bed3f7b37174c8ca47b54ad
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 24 14:05:14 2017 -0400
+
+    qa/workunits/mgr/test_localpool: simple test for localpool mode
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6383fa5b3004a606c15efd2be1b7ce5d57f05086)
+
+commit 45a804f25bc42f03b0596b1ba394fb6be575d392
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 6 15:34:50 2017 -0400
+
+    pybind/mgr/localpool: module to automagically create localized pools
+    
+    By default, this will create a pool per rack, 3x replication, with a host
+    failure domain.  Those parameters can be customized via mgr config-key
+    options.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1a0f42b70a4c9fa68dc47f2f521d0f1e8f5bb220)
+
+commit 430a140d38ba6b9e07c53d5a4a5c6232bef4fb0e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 20 16:42:01 2017 -0400
+
+    mon/OSDMonitor: error out if setting ruleset-* ec profile property
+    
+    We change ruleset -> crush back in dc7a2aaf7a34b1e6af0c7b79dc44a69974c1da23.
+    If someone tries to use the old property, error out early, instead of
+    silently not doing the thing they thought they told us to do.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6376d75eda648789b5e316a1ac2883708db7128e)
+
+commit 1fc6a51a416761d9ccd172937cda1a6b1c11faad
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 13:48:36 2017 +0100
+
+    mon: show legacy health warning in `status` output
+    
+    Previously you only got the text of this if you were
+    either looking at "health detail" or if you had
+    already set the preluminous_compat setting (in which
+    case you presumably were already aware so the message
+    isn't doing much).
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit d5e583490d946242bdfd26e863dd0aac27c12e3c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 20 20:27:22 2017 +0530
+
+    ceph_volume_client: perform snapshot operations in
+    
+    ... client configured snap directory name, instead of in hard-coded
+    '.snap' directory.
+    
+    Fixes: http://tracker.ceph.com/issues/21476
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f4fc1722594ed007706b54901fb07a2a443d1b96)
+
+commit e4164c43f5ec808c0420036d2a647a5a7d38a436
+Author: Carl Xiong <xiongc05@gmail.com>
+Date:   Fri Sep 22 11:55:33 2017 +0800
+
+    ceph-disk: fix '--runtime' omission for ceph-osd service
+    
+    f425a127b introduces a regression that ceph-disk omits "--runtime" when
+    enabling ceph-osd@$ID.service units for device-backed OSDs.
+    
+    Fixes: http://tracker.ceph.com/issues/21498
+    
+    Signed-off-by: Carl Xiong <cxiong@suse.com>
+    (cherry picked from commit a385b5b0c35106c9b44d81655983b2f7566b21cd)
+
+commit faf3fec1a628c9c4f4791e33d7795ef719267cbc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 13 18:33:21 2017 -0400
+
+    ceph-bluestore-tool: better default logging; --log-file and --log-level options
+    
+    - keep derr going to stderr (so we see 'fsck error' messages)
+    - hide the rest
+    - add friendly --log-file option
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9bd98b42fea1791935280f5ad19e5420d5303e03)
+
+commit 2184e3077caa9de5f21cc901d26f6ecfb76de9e1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:27:20 2017 -0400
+
+    ceph-bluestore-tool: add 'bluefs-bdev-expand' to expand wal or db usage
+    
+    If you are using the wal or db devices, this will expand bluefs's usage to
+    include the entire block device.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit aa4baad529835c1999ff9cc1a2f509c52a0cc699)
+
+commit af9a43a47b1dd0d8bb9946d4b4a0ab9b5c39e427
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:20:27 2017 -0400
+
+    ceph-bluestore-tool: add 'bluefs-bdev-sizes' command
+    
+    Show bdev sizes vs owned extents.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e492f2a2217ee94670e9e0d6cf28e4b7bff5ad0)
+
+commit 6a6678c52ac4aba7fd313ddc2cb3c013819fe28f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:12:21 2017 -0400
+
+    ceph-bluestore-tool: factor out bluefs mount
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 86db2d7b09de4362d90fba834d61978860d73309)
+
+commit 53f9941f28ac1435bf14f48d369b8ea03336c9d2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 12:27:35 2017 -0400
+
+    os/ObjectStore: add repair interface
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c7b7a1f04f78fa62890c567d0ca53874c8d75eb7)
+
+commit edd4d98fefe63ad31bf96fa61716b4ee079c0498
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 8 17:53:07 2017 -0700
+
+    ceph-objectstore-tool: Make pg removal require --force
+    
+    Add new export-remove to combine the 2 operations
+    
+    Fixes: http://tracker.ceph.com/issues/21272
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3bb20f6d750915d176c7a34367dac966a20faa76)
+
+commit bbe0f34cc81a8db873ef23cca0cdea8651411b86
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 8 17:09:48 2017 -0700
+
+    ceph-objectstore-tool: Better messages for bad --journal-path
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 49ca1fff7fc4360d2f3a9cac60c0ba651cbc4750)
+
+commit fd7d53bacb10d5b2094f1f579c96899d8a0a4b4f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 6 20:41:50 2017 -0700
+
+    test: Fix ceph-objectstore-tool test for standalone and latest code
+    
+    vstart.sh now defaults to bluestore, so specify filestore
+    Set environment for run-standalone.sh and cmake build
+    Create td/cot_dir as test directory
+    Crush output format change
+    Change dir into test directory
+    Give a little time after pool creation
+    Check for core files as ceph-helpers.sh does
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3ac219df2d635a08be52df3ad44cf0683de46af2)
+
+commit 3e20e0f3071188f52b7a7c3f9e36b09305431ed0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 6 20:40:37 2017 -0700
+
+    test: Move ceph-objectstore-tool test to standalone
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 495c32fd31b311d22f0d4509e896916bd2671615)
+
+commit ddca5dbe04a2647218fabff6098691edd6b7a157
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 19:55:06 2017 +0100
+
+    mgr/prometheus: only turn - into _minus at end
+    
+    ...of perf counter name.
+    
+    So that mds_mem_dir- becomes mds_mem_dir_minus, but
+    throttle-filestore_bytes becomes throttle_filestore_bytes.
+    
+    At some point once this is all settled we should
+    probably just change Ceph's internal perf counter
+    naming to satisfy the major TSDB naming rules.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29ac9270d43a225bb2336adaaf813c12e12a715a)
+
+commit 8375251d150397c0097718f6e93ed38e9e89cfbb
+Author: jermudgeon <jhaustin@gmail.com>
+Date:   Sun Aug 27 21:26:28 2017 -0800
+
+    mgr/prometheus: Fix for MDS metrics
+    
+    MDS metrics come in these forms:
+    
+    mds_mem_dir #Directories
+    mds_mem_dir+ #Directories opened
+    mds_mem_dir- #Directories closed
+    
+    In this case, continuing the trend of replacing all illegal characters with '_' results in…
+    
+    mds_mem_dir #Directories
+    mds_mem_dir_ #Directories opened
+    mds_mem_dir_ #Directories closed
+    
+    which is palpably a bad idea.
+    
+    Suggested replacement for '+' = '_plus' seems fine, and a perusal of all metrics indicate that only MDS metrics end in '-' or '+' at this time.
+    
+    Replacing '-' with '_minus' is probably less good for the general case, if anyone has a better idea…
+    
+    I suppose another alternative would be to change MDS metrics so they don't use 'illegal' characters, but this also seems cumbersome and would break more third parties.
+    
+    Fixes: http://tracker.ceph.com/issues/20899
+    Signed-off-by: Jeremy H Austin <jhaustin@gmail.com>
+    (cherry picked from commit d719cd04b294e90ab9d440ba7d033826c069a2de)
+
+commit 3a037dda4273cea289641e93a006b2857147ee73
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 8 11:33:02 2017 -0400
+
+    mgr: fix py calls for dne service perf counters
+    
+    Fixes: http://tracker.ceph.com/issues/21253
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f9a4ca07acecd15986cbce61a6e118a6cb05af29)
+
+commit 077b3601d6bf68167d1857b79bb3782e23bc8ccc
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:48:42 2017 -0400
+
+    mgr/restful: cleaner message when not configured
+    
+    RuntimeError+backtrace prints should be for
+    genuinely exceptional exceptions.  For these
+    known cases, we should just print the message cleanly.
+    
+    http://tracker.ceph.com/issues/21292
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f7a1d57dd0d6c29b8c57a5de624da33777403fe5)
+
+commit 4c9d86bd86954cbee25a3ea2a60af063114f7790
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 13:59:41 2017 -0400
+
+    os/filestore: fix device/partition metadata detection
+    
+    The UUID thing (a) relies on partition labels to work, which isn't
+    always true (and won't be true for ceph-volume going forward), and
+    (b) reportedly doesn't work anyway.  The fd-based helper works
+    just fine (even for vstart).
+    
+    Fixes: http://tracker.ceph.com/issues/20944
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e62862be0b68a17763e19ed8453436c457294505)
+
+commit 49e030da55030068c35790f751ea6631a324e5c2
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Wed Sep 13 18:15:33 2017 +0800
+
+    test/config: regulate parameter order of ASSERT_EQ
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit ccaba817dcd2c84c75cf0f784d90b78527c9f92b)
+
+commit 85ab5c86aafa92a36078133d7da07db54f064f4b
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Sep 12 11:20:34 2017 +0800
+
+    common: fix daemon abnormal exit at parsing invalid arguments
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 3f075372d1400806ae32e0e9210b9c15316dfe66)
+
+commit d7cde5f316242caa86ef62690104228aafc58a42
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Sep 7 13:12:15 2017 +0800
+
+    mon,monmap: use new style config opts
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3124eb2148f5c6703c75b9e556ddb773898a6e55)
+
+commit fe039e41d78f7d74c39cdc9034cadccf632e9ac2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Sep 7 12:47:36 2017 +0800
+
+    mon,option: set default value for mon_dns_srv_name
+    
+    Fixes: http://tracker.ceph.com/issues/21204
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 50207769f0c187cc8caf2bed3a1c94df409718e5)
+
+commit b54c7ba225314cc3e67707ec8dbac4a7f8381929
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 6 12:57:40 2017 -0400
+
+    mon: fix `osd out` clog message
+    
+    This was printing the absolute time instead of the period.
+    
+    Fixes: http://tracker.ceph.com/issues/21249
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f07480fec729d66f4cdc19a1c3bde3d185df1289)
+
+commit d7f6b93af4ce1115a1bde5883421c8ad860aa53f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 16:43:58 2017 -0400
+
+    rbd: mirror "get" actions now have cleaner error messages
+    
+    Fixes: http://tracker.ceph.com/issues/21319
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 053be9847ff0052348aa259520d641923e57537d)
+
+commit bb0874c9db50bbfbf84396a4f1308282102a1694
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 15:41:36 2017 -0400
+
+    cls/rbd: avoid recursively listing the watchers on rbd_mirroring object
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79be496ac098859304efcabc7b5eb8afca6eca91)
+
+commit 34af07f97a67ead7d00b4352aea6300ac737ad9b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Aug 18 20:08:12 2017 +0200
+
+    rbd-mirror: potential lockdep issue
+    
+    (cycle between ImageReplayerAdminSocketHook and ImageReplayer locks)
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 226b1a3be16915c79e16106d42260687683e7a92)
+
+commit 925f223b7a57eae382c168d4bc4f05f50dd7bf6b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Aug 11 14:27:41 2017 +0200
+
+    rbd-mirror: update asok hook name on image rename
+    
+    Fixes: http://tracker.ceph.com/issues/20860
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9ddb5da3377029d11cb1089102d7146e9ee2537a)
+
+commit fd207447d1acebbf7a62c0d03b3ec9025ac19fbd
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Aug 17 16:02:02 2017 +0800
+
+    rgw: fix lc process only schdule the first item of lc objects
+    
+    If more than two bucket hash to the same lc object, only the first bucket will do lifecycle operation. Other buckets lifecycle will never be schduled.
+    
+    Fixes: http://tracker.ceph.com/issues/21022
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 9561dc0a903d9021f64857762e73887ffe46c28e)
+
+commit 5b5f0b5a33c638ec6f52d6936c60e5efd76eacab
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Aug 23 21:31:11 2017 +0200
+
+    rgw: fix accessing expired memory in PrefixableSignatureHelper.
+    
+    Fixes: http://tracker.ceph.com/issues/21085
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 9260d39ceb1ef39a32c8f1742c3069bb83d628f8)
+
+commit d2be1e0ccb64021f770f28a6097f5468d375dbec
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Aug 25 15:05:52 2017 +0800
+
+    rgw: return bucket's location no matter which zonegroup it located in.
+    
+    Get bucket location which is created in another zonegroup, will return "301 Moved Permanently".
+    
+    Fixes: http://tracker.ceph.com/issues/21125
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 71d2eca3e00de19419fb07bf9346f5863b20d40f)
+
+commit c4d6539374a02378209688dfa829caa39af0a214
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Aug 31 13:39:13 2017 +0200
+
+    rgw: rgw_rados: set_attrs now sets the same time for BI & object
+    
+    `RGWRados::set_attrs()` used to set slightly different mtimes for the
+    object and in the bucket dir entry as we do an object write and set the
+    time at bucket index as the time at that point. Fix this by using the
+    setting the same mtime for the object as well.
+    
+    Fixes: http://tracker.ceph.com/issues/21200
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 87e8e89fe11fa805739a2b85d1b009424641fa4e)
+
+commit dfe42384fbce90ec4c315e10ebd6b4fc4234827b
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 19:22:32 2016 +0100
+
+    rgw: calculate and print Swift's X-Account-Storage-Policy-* headers.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5e5c0677ba69f0d162df1da9a22b1fb56028baa4)
+
+commit 6e4c63fb136d958e0b4fe279419978a322043201
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 17:14:35 2016 +0100
+
+    rgw: bucket linking stores also the info about a placement rule.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit de7a8679af93457c10bf514c17a7cfc5327e745c)
+
+commit 933f840094c3614466904f0fce32c2346a8f1709
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 17:12:50 2016 +0100
+
+    rgw: convey placement rule in RGWBucketEnt and cls_user_bucket_entry.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5eca89f08d29ac6aa7b55a3b4fb5b4183bdc0496)
+
+commit 5a1d0c42854fb35d44e63859d81074867f1a3295
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 15:54:42 2016 +0100
+
+    rgw: clean-up around and implement the move semantics in RGWBucketEnt.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 545dabe31a9ac231db4b8e3ce674ddfd9c91ee20)
+
+commit bb92cc93ac32c1f98d7cfae778d684b94dd2de58
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 19:16:20 2017 +0200
+
+    rgw: enforce the std::move semantic across the path of RGWUserBuckets.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 4a7a4344840c873189331f22c8143ba031622d20)
+
+commit 395e9a48d32db7dfe0419316772886b939a44233
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 19:19:59 2017 +0200
+
+    rgw: {end_}marker params are handled during Swift's reversed account listing.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 2f1f7a179d6a031c77929359313fc1fcc9f75bef)
+
+commit 8582664267b2e72a836e27df859910481371dee0
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Mon Aug 28 21:54:56 2017 +0200
+
+    rgw: add basic support for Swift's reversed account listings.
+    
+    Fixes: http://tracker.ceph.com/issues/21148
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 870f54806b7c1f570d2e82d3d2aa308b2178c6f0)
+
+commit d61de49cf270fedb206530a9d3cc1b4970f2b484
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Mon Aug 28 21:53:28 2017 +0200
+
+    rgw: abstract partial data processing in RGWListBuckets.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit a842dd09d584ff7636ec972646caff218df59c4b)
+
+commit 957e221f1ffc73726187aba99e9e103291b0ba6f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed May 24 16:50:26 2017 +0200
+
+    rgw: list_objects() honors end_marker regardless of namespace.
+    
+    This patch fixes a regression related to handling of the end_marker
+    parameter during Swift's container listing operation. It has been
+    introduced in a5d1fa0587184f43c69d8e03114b58d43f320781 and causes
+    Tempest's test_list_container_contents_with_end_marker to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/18977
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 09db1e412ef0853e7f345f813c8d452ec8354c36)
+
+commit 004240eea79b02eb830f3c9e72de8e0216ff7557
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 9 22:47:06 2017 -0400
+
+    qa/standalong/mon/osd-pool-create: fewer pgs in test
+    
+    This runs afoul of the new max pg per osd limit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c9ffeeebebe84111ce3be51c04170804bf2dbc74)
+
+commit 54ba2dda2034dd80594f17261451832744d2c7bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 08:08:30 2017 -0400
+
+    mon/OSDMonitor: assume a minimum cluster size of 3
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1010761e6ad392fcaa647ec0d1b3d071535adfb3)
+
+commit f581731eaeb0f0c625f62928bf3724f8e705132c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 1 14:45:12 2017 -0400
+
+    mon/OSDMonitor: prevent pg_num from exceeding mon_pg_warn_max_per_osd
+    
+    Check total pg count for the cluster vs osd count and max pgs per osd
+    before allowing pool creation, pg_num change, or pool size change.
+    
+    "in" OSDs are the ones we distribute data too, so this should be the right
+    count to use.  (Whether they happen to be up or down at the moment is
+    incidental.)
+    
+    If the user really wants to create the pool, they can change the
+    configurable limit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3ea2e518d27e6c06182c2cb3d9c0b9a0dab8dd22)
+
+commit acb2f280f03324e2ddb9d359e18ff2f0a3f897e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 6 11:41:07 2017 -0400
+
+    common/options: reduce mon_pg_warn_max_per_osd to 200
+    
+    This is 2x the recommended target (100 per OSD).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b7fa440a9802005c167f854e1d19f9dd4fa8dfff)
+
+commit 119995a8f0ed70fb3e9efb5511576c44cfd6dbaa
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Sep 19 14:46:53 2017 +0200
+
+    rgw: rename the configurables for metadata limits to start with rgw_.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 9b06985caec27fc102769e46799ca6608e80eb85)
+
+commit 6080248f12e05a7478af1f8f31747da1aa9b881c
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 12:16:30 2017 +0200
+
+    rgw: return proper message when deleting non-empty Swift's container.
+    
+    The bug that is fixed in this patch has been responsible for failing
+    the Tempest's test_delete_non_empty_container test case.
+    
+    The investigation has been made by: Marcus Watts <mwatts@redhat.com>.
+    
+    Fixes: http://tracker.ceph.com/issues/21169
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 308c8d3ab7e3e4978beccba2c27beb95e75cde22)
+
+commit 71e37b1a10796ddfe786a2d389b0497ce1dc1bf4
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Nov 20 21:49:57 2016 +0100
+
+    rgw: seed::get_torrent_file returns errors in the usual way.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 21ad80a1c237f8d243efaacd8218b29719a0a6b0)
+
+commit 429f5e1d8b7ec114289ba45e7c896e264b15ec7c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 16:36:06 2017 +0200
+
+    rgw: add support for max_meta_count of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17934
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 9e53c14a3398dfe55f6b816bb57c4b1b6d01c8ec)
+
+commit 70e150cdfa108f469d85a27ad1a93952f71347e0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 22:19:33 2017 +0200
+
+    rgw: Swift API returns 400 Bad Request on too long container names.
+    
+    Fixes: http://tracker.ceph.com/issues/17935
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1845e41292696da9e20ecd87b9260b44ee312aed)
+
+commit 02c208a6b1ae335ab15c0fab3d9d6dba803037fa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 29 19:08:21 2017 +0200
+
+    rgw: honor custom rgw_err::message in Swift's error handling.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e8516147d2926631fed98a793606bcbb0e6db9e1)
+
+commit 912132b272dbf534e7594b40a78c5efcd51c9466
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 29 18:02:23 2017 +0200
+
+    rgw: add support for max_meta_value_length of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17936
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 06b1f25021924e0c55da0bc22216f31344735dfe)
+
+commit cd219153978974a3711093912ca329ae5d1d997c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Nov 21 19:34:48 2016 +0100
+
+    rgw: refactor rgw_get_request_metadata to reduce the number of dynallocs.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 64b792326350c4c09ed5f829d082d4050bfb0408)
+
+commit 4b82d1e25bf8f7be1915e7467cb822d5beea96a1
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 22:19:58 2017 +0200
+
+    rgw: add support for max_meta_name_length of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17938
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 593d656dffd477943b8bd15153004d905b4b0d73)
+
+commit fb7e6a581d063db10bb7e0cce4ae358068c7a2bd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 18 18:01:17 2017 +0200
+
+    tests: CentOS 7.4 is now the latest
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2311b64025cdb6131035aaf01e7c97486da12e15)
+
+commit e25a583323a225dcfab0567c872247b33bf36f8f
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Sep 18 14:44:36 2017 +0800
+
+    osd/PrimaryLogPG: _delete_oid - fix incorrect 'legacy' flag
+    
+    For pre-Luminous created objects, we shall default 'legacy' flag
+    to true, so we can still create a snapdir object properly if necessary
+    for Luminous backward compatibility.
+    
+    Note that this patch is not going to land on master first
+    (and hence can not be cherry-picked from master) because it will
+    finally be deprecated by https://github.com/ceph/ceph/pull/17579,
+    in which we are going to kill the snapdir object completely for Mimic.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 5aa445897c293cae407454fadbbc47116a2cbe76
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat Sep 16 10:04:56 2017 +0800
+
+    qa: fix pool-quota related tests
+    
+    https://github.com/ceph/ceph/pull/17371 introduces support of
+    per-pool space-full flag, which turns out to set both
+    full and full_no_quota flags now if a pool is currently running out
+    of quota.
+    
+    Actually this test is fragile as long as we keep appending new flags
+    at pool granularity, but let's not bother with that complexity now.
+    
+    Fixes: http://tracker.ceph.com/issues/21409
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 5dd3458eec7d6c56a71ee6f0fcdd45aa15a58ec7)
+
+commit 00b44ecf4415710150e0646174174fbde0167f42
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Aug 28 15:51:28 2017 +0800
+
+    mon, osd: per pool space-full flag support
+    
+    The newly introduced 'device-class' can be used to separate
+    different type of devices into different pools, e.g, hdd-pool
+    for backup data and all-flash-pool for DB applications.
+    
+    However, if any osd of the cluster is currently running out
+    of space (exceeding the predefined 'full' threshold), Ceph
+    will mark the whole cluster as full and prevent writes to all pools,
+    which turns out to be very wrong.
+    
+    This patch instead makes the space 'full' control at pool granularity,
+    which exactly leverages the pool quota logic but shall solve
+    the above problem.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit b4ca5ae462c6f12ca48b787529938862646282cd)
+    
+    Conflicts:
+    slight confilicts in src/osd/OSDMap.h because
+    e71626732452951ed9e6c489f15618424d15acf2 removed get_full_osd_util().
+
+commit b064ed172a309aad4e8850110be4311466bf5894
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 10:02:33 2017 -0400
+
+    os/bluestore: repair 21089 on freelist init
+    
+    Fix up the size inconsistency on freelist init.  This way it will always
+    happen after an upgrade... and before the user moves to something
+    post-luminous.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1bf89aa3ce2f1fdb7979e536f6788c8dae275d47
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 10:01:25 2017 -0400
+
+    os/bluestore: fsck: remove fsck repair for 21089
+    
+    This requires the user to run fsck, which they likely won't do.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit e9f143c681ab1a79b9c726cab55282d71ecfe1c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 13 23:03:07 2017 -0400
+
+    os/bluestore/KernelDevice: hack to inject bad device size
+    
+    This conditionally reintroduces bug http://tracker.ceph.com/issues/21089,
+    fixed by f6f1ae3724d593d3709d982c973ec18a25a47b6e, to aid in testing
+    repair.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 460268b5e9c9a77aae134049aa320977f02fc911
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 30 15:28:11 2017 -0700
+
+    ceph.in: validate service glob
+    
+    Fixes: http://tracker.ceph.com/issues/21191
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 59bee1e7274934cebe0dcefd9fcedb6886a5e2d0)
+
+commit 03e2a68e431eaa737b9e66c4d3b289e58d6cf19b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Aug 30 10:17:31 2017 +0200
+
+    build/ops: python-numpy-devel build dependency for SUSE
+    
+    Fixes: http://tracker.ceph.com/issues/21176
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1586f2ca9ab94af85682945a3c7c7ebbd82c6e03)
diff --git a/doc/changelog/v12.2.3.txt b/doc/changelog/v12.2.3.txt
new file mode 100644
index 000000000..c8974f7db
--- /dev/null
+++ b/doc/changelog/v12.2.3.txt
@@ -0,0 +1,10247 @@
+commit 2dab17a455c09584f2a85e6b10888337d1ec8949
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Feb 19 23:14:46 2018 +0000
+
+    12.2.3
+
+commit c0429254cec6fdfa1089f3dc205a7a6199d2e40a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 20 06:40:14 2018 +0800
+
+    cmake: disable DOWNLOAD_NO_PROGRESS if cmake ver is lower than 3.1
+    
+    see https://cmake.org/cmake/help/v3.1/module/ExternalProject.html and
+    https://cmake.org/cmake/help/v3.0/module/ExternalProject.html. the
+    former has DOWNLOAD_NO_PROGRESS, while the latter does not.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d051bf0441c933e27fb0f54bc6ed999d1e8eca89)
+
+commit 74c89b975f786ca2bf6de7310f191e92faeaca6c
+Merge: 76df814ac4 7e33d67f1b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Feb 16 08:06:00 2018 -0500
+
+    Merge pull request #20452 from ceph/backport-wip-rm22785
+    
+    luminous ceph-volume consume mount/format options from ceph.conf
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 7e33d67f1b143eff9c63af5d44d2d2dd480ed2f3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 16 06:51:12 2018 -0500
+
+    ceph-volume tests remove unused import
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ad49283b2ee1e4a1a8a54d0c71039a2afc778b29)
+
+commit 76df814ac485df9f72e58081f8d6bb17151cc18b
+Merge: 3e86319b8f cdc27208aa
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Feb 16 11:15:57 2018 +0100
+
+    Merge pull request #20451 from ceph/wip-yuri-PR20053-luminous
+    
+    qa/tests: Applied PR 20053 to stress-split tests
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit cdc27208aa170061ef798b9457d6e85294e9238b
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Feb 15 10:26:46 2018 -0800
+
+    qa/tests: Applied PR 20053 to stress-split tests
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 18a0ee570ffb456053e1712881ee61f8b2072f87
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:47:36 2018 -0500
+
+    ceph-volume util.prepare extend flags when found
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 287c952bc8c3422c6eed72a1e0b22612d0728314)
+
+commit cb5da5fffe625aa6b930bdf4a77869716b77caff
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:47:17 2018 -0500
+
+    ceph-volume util.constants update mount flags to include rw, normalize on list
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 457a5173d2e08192273c22f1abe52a64773188b4)
+
+commit 33f7da3f165be7bbc4a56613078d95d6fa6e79cb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:46:47 2018 -0500
+
+    ceph-volume configuration normalize underscore vs. whitespace named options
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b80e3bf1175a65181d1f2ed38b6a416c6b730d82)
+
+commit 93e41ef6434daea37df3322e87e0c6e367de1741
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:46:06 2018 -0500
+
+    ceph-volume tests check for mount/format usage from ceph.conf
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 676c93fcf3041de07e8125d36d8dc7ad474db9b1)
+
+commit c586a24e91ca627449452103d562b6be1fbfcb4d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:44:45 2018 -0500
+
+    ceph-volume tests add ceph.conf stub for custom conf files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a8eecc64716564479308fafba5159e271c42feac)
+
+commit 3e86319b8fa5f85a078441b88935f812c8039580
+Merge: dbe215d69d 599869d7f2
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 14 16:59:11 2018 +0100
+
+    Merge pull request #20429 from ceph/backport-wip-rm22988
+    
+    luminous ceph-volume use realpath when checking mounts
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 599869d7f2df5d207ba9afa8915ce4980ff65137
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Feb 13 13:22:41 2018 -0500
+
+    ceph-volume tests verify realpath usage in mounts
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 16eabbc3abb7d03a0808963a9aae6b2ffb3a875b)
+
+commit 565c044d556eaf741d79e7abbf5f472ee4366b44
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Feb 13 12:59:48 2018 -0500
+
+    ceph-volume util.system optionally use realpath on devices
+    
+    To check a mounted device it is needed to verify on a combination of
+    realpath and plain devices against realpath and plain paths. In LVM, two
+    different paths might refer to the same devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 53e691e1d5122ff533e69ae08c73e18c1ed74765)
+
+commit dbe215d69d8ccf0956f197cef88f50f6518bbfa0
+Merge: 2fc8d6c4ad dd94ac120e
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 14 14:13:59 2018 +0100
+
+    Merge pull request #20438 from ceph/backport-wip-rm22876
+    
+    luminous: ceph-volume: lvm zap will unmount osd paths used by zapped devices
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit dd94ac120ed337f0080dc776f3234eeaade46b54
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 12 15:23:08 2018 -0600
+
+    ceph-volume: use terminal.MultiLogger in `lvm zap`
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3f2e7074ee42ff8248afaed2aa9bfd919096fc38)
+
+commit ec84f39a0573887e005fdb0bfe7d4e5e9457c51b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 09:06:45 2018 -0600
+
+    ceph-volume: use api helper methods for pvs and lvs in lvm zap
+    
+    This is just a cleaner way of doing this instead of using PVolumes
+    and Volumes directly.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit c8a6da4e1bb645ca5ee8cb8f5d861584fe70210d)
+
+commit 19e02e6c433f718ce638624a803105525168cb52
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 08:35:47 2018 -0600
+
+    ceph-volume: fix documentation typos for lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5552cefc4ce6d6d819f43671cec83ca23b119665)
+
+commit 1de5592d4a13a0c0475f723fc02fb8e1c70b482e
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 1 14:52:46 2018 -0600
+
+    ceph-volume: when zapping unmount osd directories
+    
+    If you zap an lv, device or partition and it's currently mounted as a
+    ceph osd directory then it will be unmounted so the zap can complete.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d4639ac116407f71ba6bff06a3202c1490d57dee)
+
+commit 13f42da7ae449dde84a94f93ac22f8bf0a8c7f79
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 1 14:29:30 2018 -0600
+
+    ceph-volume: adds a util.system.unmount function
+    
+    This function will unmount the given path
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 6909baef542f3cd9b7d97113a6152280e72013f6)
+
+commit 2fc8d6c4adfe32586b9d177ed6ab7b46c6b2b110
+Merge: 22c6b1d3d6 24f423fa0b
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Feb 13 16:43:14 2018 -0500
+
+    Merge pull request #20407 from cbodley/wip-qa-multisite-trim-luminous
+    
+    luminous: qa/rgw: disable log trim in multisite suite
+    
+    Reviewed-by: Yuri Weinstein <yuri.weinstein@gmail.com>
+
+commit 24f423fa0b937fc8df8f7c48fdc1d83510d0a44f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 11 16:14:05 2017 -0500
+
+    qa/rgw: disable log trim in multisite suite
+    
+    the multisite tests run manual trim operations with radosgw-admin, which
+    can race with internal log trimming to produce tests failures
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f8909bb6a6320822da7a4c4dab242f84732ebf5d)
+
+commit 22c6b1d3d6bbc7ee466fddd3af13bcdb7f0c9a55
+Merge: fda9e4a3e2 01f79c086a
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Feb 8 18:56:12 2018 +0100
+
+    Merge pull request #20368 from ceph/backport-20367
+    
+    luminous: ceph-volume: adds custom cluster name support to simple
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fda9e4a3e2ab4dd91430d9197792ed15e8ad256d
+Merge: faefe0b68f 29c89c4964
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Feb 8 18:08:49 2018 +0100
+
+    Merge pull request #20357 from pdvian/wip-22938-luminous
+    
+    luminous: rgw: unlink deleted bucket from bucket's owner
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 01f79c086a2d6e1fd10f3d6283c6f12ef03991a2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 8 07:09:57 2018 -0600
+
+    ceph-volume: use a custom cluster name in simple functional tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7f1dc6b3ab885253d28c223bda7e6e1232a84609)
+
+commit 8d50005d3efddde777916df214ace36178f4741f
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 14:42:24 2018 -0600
+
+    ceph-volume: set conf.cluster name during simple activate
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b1cee74fa2225fe10a024c7291bd23aadcc23441)
+
+commit faefe0b68fa748e8cb50ffb97b34d39ace65f35c
+Merge: 13f400f801 bf3ee8b22b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Feb 8 12:43:44 2018 +0200
+
+    Merge pull request #20338 from cbodley/wip-22930
+    
+    luminous: rgw: fix for empty query string in beast frontend
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 29c89c496424dbaf66ab8771d1e9c9578eb092a2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 18 14:53:35 2018 -0500
+
+    rgw: unlink deleted bucket from bucket's owner
+    
+    if a bucket is deleted by an admin/system user instead of its
+    owner, the unlink would fail and the deleted bucket remained
+    visible to the original owner
+    
+    Fixes: http://tracker.ceph.com/issues/22248
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4701e85a3ea72852a61a106724fa22497f3d5789)
+
+commit 13f400f80182c4fe0ebe8f9be19fa63898003f0b
+Merge: 609f2a88e0 b8b3eee73d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:49:36 2018 -0800
+
+    Merge pull request #20204 from tchaikov/wip-pr-18614-luminous
+    
+    luminous: osd,mgr: report pending creating pgs to mgr
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 609f2a88e0e8b0cb40ac7853d474a175f4c9699e
+Merge: 2cef7159fe d93eef3127
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:45:33 2018 -0800
+
+    Merge pull request #20249 from pdvian/wip-22864-luminous
+    
+    luminous: mds: fix scrub crash
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 2cef7159fe89a428256910d90dbfb0d42924d0cf
+Merge: 8bca8fb027 7756a44529
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:45:07 2018 -0800
+
+    Merge pull request #20256 from pdvian/wip-22860-luminous
+    
+    luminous: osdc: "FAILED assert(bh->last_write_tid > tid)" in powercycle-wip-yuri-master-1.19.18-distro-basic-smithi
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8bca8fb0274c5f3690262400a92eb23bb87a4012
+Merge: 16d85f9673 042144a281
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:44:39 2018 -0800
+
+    Merge pull request #20299 from pdvian/wip-22859-luminous
+    
+    luminous: mds: set higher priority for some perf counters
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 16d85f9673931596017aeb3eb23132b5f3ef3500
+Merge: 43a4e40916 5282a60c8f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:44:03 2018 -0800
+
+    Merge pull request #20300 from pdvian/wip-22867-luminous
+    
+    luminous: MDS : Avoid the assert failure when the inode for the cap_export from other…
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 43a4e40916159f347bca12ff02d2b64947d2a4e4
+Merge: 1133c9bba1 c63fc964ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:41:45 2018 -0800
+
+    Merge pull request #20339 from batrick/i22242
+    
+    luminous: mds: limit size of subtree migration
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 1133c9bba1750c8f861293f44ed9e99f679fd2cb
+Merge: a8cdfea4b3 87991643b3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:41:14 2018 -0800
+
+    Merge pull request #20340 from batrick/i22240
+    
+    luminous: mds: prevent filelock from being stuck at XSYN state
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit a8cdfea4b34e46a59fdf023cbeb57b9dfc0db2ee
+Merge: e2e6d30f3f 655d266944
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:40:33 2018 -0800
+
+    Merge pull request #20341 from batrick/i22089
+    
+    luminous: mds: don't report repaired backtraces in damagetable, write back after repair, clean up scrub log
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit e2e6d30f3f00af38c49aa9845eaac940680f37cd
+Merge: 33ebb80c91 6caae86b7c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:38:58 2018 -0800
+
+    Merge pull request #20337 from liewegas/wip-bluestore-aio-read
+    
+    luminous: os/bluestore: propagate read EIO errors up the stack
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 33ebb80c915c66dfc4d64b2ce85f6fc690f4326a
+Merge: b99521accd fc5807003b
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 7 19:02:31 2018 +0100
+
+    Merge pull request #20350 from ceph/backport-20264
+    
+    luminous: ceph-volume dmcrypt support for simple
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fc5807003b01e6d009c392e0577765582db53d64
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Feb 6 11:36:03 2018 -0600
+
+    ceph-volume: do not test custom cluster names with simple
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 63046e0d7bd05a42477dd1fbd4af11f81046b08d)
+
+commit c1eb7b9915a61d93a5b09ff588a5906e35a2b208
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Feb 6 10:46:44 2018 -0600
+
+    ceph-volume: sleep 2 minutes after reboot in simple tests
+    
+    This will allow for some time for OSD services to start back up after
+    reboot. If not we see race conditions where tests fail because they
+    were run before the OSD finished startup.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 85b319a2c577c1fa86c6ccbbac61f015dd24ad21)
+
+commit a7ecc1c91a9492337736c9674b4242317a4e526a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 5 14:18:28 2018 -0600
+
+    ceph-volume: adds the simple dmcrypt_plain and dmcrypt_luks tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3052010a80a1783b6ebd629a603638ab5806f8fd)
+
+commit aab4961b5e597614480d23efcc88008cb0d600c4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 5 13:53:13 2018 -0600
+
+    ceph-volume: fix ceph-volume simple scan help menu test
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2ee70bf2d1958c0655ddb5b7cc4d2ed3f4bc58b8)
+
+commit c1035af7a6a669c5f88ef7f30942a2b9ed347c1c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:11:09 2018 -0500
+
+    ceph-volume tests.functional add simple xenial filestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00b14177bdbe6d66424001b1c35ca8fa7b46977f)
+
+commit 55def6eb6c463c08cd1c24839b6891395425372a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:56 2018 -0500
+
+    ceph-volume tests.functional add simple xenial filestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0b2f869ccbb30c27bd19f2a05bb893957912142e)
+
+commit 591796d073d6032a39a7e7b70cd56965fc8320f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:36 2018 -0500
+
+    ceph-volume tests.functional add simple xenial bluestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0174f9e1646f4b51c712506f629b48c329f20ea1)
+
+commit 31665eac381cf78cd4fdc8993b75dc291121a095
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:12 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 filestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4c26162002709ccdd1b48c05126befdab7f269da)
+
+commit 4de4d69bf0e260c38177b32ba4f199dd13b9983f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:57 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 filestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a911d7c6a99e2ea235400614568d9cd362470cac)
+
+commit eb97511c3fb9a33df50bf35ace515a068211df7c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:41 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 bluestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8e0213ab053e80038bdae8d1b06d448fd397e3bc)
+
+commit badd73df5d30f034c5ceee37fdb479bba17fffa6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:21 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 bluestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 76151c461bfe1e5ae4dc277c9d9357efb81e8c92)
+
+commit 5d6aa430a51a481cae400b8fa9d8ee3fbc5fa392
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:08:23 2018 -0500
+
+    ceph-volume tests.functional add simple bluestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d6e24022e8dfe845823fa0e87f3fba028c4b25b5)
+
+commit 8fcd52e88a54959dde41333ed440b88d78a021a3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:36:45 2018 -0500
+
+    doc/ceph-volume lvm prepare fully supports encryption now
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0b2075759e8788b2c4ae46f94fdee953cc82b21e)
+
+commit 476eac136a7aee3485b91b81108bc1d05d0d6698
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:36:02 2018 -0500
+
+    doc/ceph-volume scan update for encryption support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e0d3bb82255a937cd937d33f966979fb541d1443)
+
+commit 141279aeff3892f1778fddb373d2e2180e402950
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:28:40 2018 -0500
+
+    doc/ceph-volume remove notice that dmcrypt is not supported
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7d11ca27df1d22c8109cad5bf7be8fd23395c71)
+
+commit 2f7d03418783b33eaf0cab1fd22ff95f94861d3e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:26:58 2018 -0500
+
+    doc/man/ceph-volume add simple documentation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9524021aa15280d98f2b04d1fd66552733adf8a2)
+
+commit 1adafb88b6e458527759a13afe2362aa79c4adaa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:15:36 2018 -0500
+
+    ceph-volume simple.scan update help menu to indicate device support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 09f35cb249674eda044b8fceea6f402ffac7731d)
+
+commit d015b1933920a855a6a2cafdda863162b804ecbe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:13:12 2018 -0500
+
+    ceph-volume tests for keyring parsing
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b90c1a86843d8f0c8c7fb68fe059bdd72f3730fd)
+
+commit 77d40d0713385dcb54b8aa1d0d5a2e1f8386609c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:12:51 2018 -0500
+
+    ceph-volume tests for validate_devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bece7af0523b04d39c26011990cb1fe323aecdd8)
+
+commit 70b5a325ffe93cb55eaf0f6debfc4f5dd305c24b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:12:05 2018 -0500
+
+    ceph-volume simple.activate b64decode keys for activation as well
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 636ebc127797c8020cfb7aff09955b72beac8bdf)
+
+commit baf12acff49e0e2129e0a7b63d10a497929f4992
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 16:17:19 2018 -0500
+
+    ceph-volume tests add validation for lsblk parsers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 03102e4417194e16deb8b738fcdaf87e699e6073)
+
+commit 44cbcdee151adc302bd77ca2377032ed1bdb1bfd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:51:41 2018 -0500
+
+    ceph-volume tests validate parsing of cryptsetup
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cce6a12f4a0d8d9a3d7dcb509b42ed2536d9257f)
+
+commit 941fbbf29ba893e0fe7657669e969c7d784e08d3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:51:12 2018 -0500
+
+    ceph-volume tests add a stub for process.call
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b80b61b90f8539b363c03c96e63f31c38c707fc8)
+
+commit 2ce659749487f60c4012207aaae6665e889f52dc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:35:11 2018 -0500
+
+    ceph-volume util.encryption add notes about extra b64decode call for ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c2367ef4a4db1e9c62e33dc211474f652d6bc6af)
+
+commit 36d81bdf2ab24b52db2808ca8cf91ebd864eddb1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:34:09 2018 -0500
+
+    ceph-volume util.encryption parse legacy encrypted with dirs too
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 32cb810f8ab7e4a710bfb2590a065dea41369fce)
+
+commit de3013287fe9c6d879cc4fbca6c06437e5989e21
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:32:54 2018 -0500
+
+    ceph-volume terminal create a logger to get terminal+log messages in one call
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f6dd0ff15f0aeda9f660e923153f1d3dae68e583)
+
+commit c25ba9528e66ab7522d2452fc14c935924767839
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:32:16 2018 -0500
+
+    ceph-volume simple.activate support dmcrypted devices for both plain and luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5ece73b533ee4b577c0ce61e3685a880e1de4851)
+
+commit 8a19b31c162ae08913136895144303d8999600dd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:31:28 2018 -0500
+
+    ceph-volume simple.scan parse the keyring out of the keyring file
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a1124362816cbaaad7255b7ffc3698b68601ffe5)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/devices/simple/scan.py
+
+commit 1d94a5f8e2d9d5e314463cbbb7cf7fc704d3ffc2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 16:23:06 2018 -0500
+
+    ceph-volume simple.scan support dmcrypt OSDs when scanning
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6c87d66d76ee1fc70f8dc6211d89dbd489c6ed3a)
+
+commit c3aa6e6b90a73d4754ae28270b650ca9a79809eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:35:53 2018 -0500
+
+    ceph-volume util.encryption add a utilty to open plain encrypted devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e5759e4fc9485e92a2faec277c8a920295751844)
+
+commit b2bd65505df108420ee79c93d8da6ffdfa7f7675
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:34:53 2018 -0500
+
+    ceph-volume util.system tmp mounts can now remove a dmcrypt mapper
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 60e8699650a491013d51239b9dc58160946d3bf9)
+
+commit 520200ceea0070a1a14976d06befc72217ed438f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:22:19 2018 -0500
+
+    ceph-volume util.system do not 'translate' using realpath
+    
+    Using realpath always means that device mapper paths, like for dmcrypt
+    or LVM will get mangled and will return something like `/dev/dm-1` which
+    is not useful for anything in ceph-volume
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 469d01dc1aee8d6528944ef0acec58df868a9da7)
+
+commit abf21f28612c03ce82465b1bb816d477b07da702
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:19:06 2018 -0500
+
+    ceph-volume util.disk add a PART_ENTRY_TYPE detection utility
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cdb812438d6064089bac964cf38817d877afca38)
+
+commit c177b491758db2e7d81de100a9e52821f38b748a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 26 10:44:14 2018 -0500
+
+    ceph-volume util.encryption add helpers for legacy devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d59b087e50e4e2076c59f6316cc89eb2a070aa52)
+
+commit cc9840140e7a7fe385fcd2cbaeef7351cfd79ba8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 26 10:25:31 2018 -0500
+
+    ceph-volume util.constants add ceph-disk partition labels
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 548a74bd239abe6b1b7a5a27fb2779800cfa7ac1)
+
+commit 69d142adb8d70f1b5c9f55d72395313fd1e80048
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 25 11:04:15 2018 -0500
+
+    ceph-volume util.disk support PKNAME and absolute paths in lsblk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b90044c2ffff38d0dc36d598d237fcb066efe400)
+
+commit b99521accd04a9761fea8d9a927442ae51bf18a5
+Merge: 62337485fa 40a16ac6cc
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 7 08:19:29 2018 -0500
+
+    Merge pull request #20323 from ceph/backport-20203
+    
+    luminous: ceph-volume: fix usage of the --osd-id flag
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 62337485faf13bd1caa02adec877fd9a5de1bd78
+Merge: 3d84e35378 b47837100f
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 7 08:18:32 2018 -0500
+
+    Merge pull request #20322 from ceph/backport-20059
+    
+    luminous: doc/ceph-volume OSD use the fsid file, not the osd_fsid
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 3d84e3537826d1dcffe8a82195b93a8488d71b8b
+Merge: 1c847a1fc8 a26b4ae765
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:21:16 2018 -0800
+
+    Merge pull request #20162 from tchaikov/wip-pr-20130-luminous
+    
+    mgr: balancer: fixed mistype "AttributeError: 'Logger' object has no ...
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c847a1fc88bb23126c77ac09a38dde29a5500e4
+Merge: 6b5ae18b13 03569abb40
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:20:18 2018 -0800
+
+    Merge pull request #20325 from jcsp/wip-22851-luminous
+    
+    luminous: pybind/mgr/dashboard: fix duplicated slash in html href
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6b5ae18b133a260ef50f39652b64fd8ea208722a
+Merge: ac0e193f40 91673b1edf
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:19:14 2018 -0800
+
+    Merge pull request #20326 from theanalyst/wip-22892-luminous
+    
+     luminous: _read_bdev_label unable to decode label at offset
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 655d266944374eea2a846f182cc00193ef4cf4b8
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sun Nov 19 20:25:22 2017 +0800
+
+    mds: properly execute scrub finish context
+    
+    Bug was introduced by commit 7e52729699 (mds: flush after scrub repairs)
+    
+    Fixes: http://tracker.ceph.com/issues/22058
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ff71fbdcbb87ffc9baa02319ff0f336c173c5616)
+
+commit 798cc37c7a2acb4af182bdc7b214c4c7445645d0
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 09:39:15 2017 -0400
+
+    mds: clean up clog damage message when repairing
+    
+    If we've repaired then we don't need to be at WRN
+    and we don't need to advise the user to look up
+    the damage detail.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3160ef2c0a66324621a95926bed0416bbd537353)
+
+commit 6e544ac967b533f47ce2708b1757cd048ea2354c
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 06:30:57 2017 -0400
+
+    mds: flush after scrub repairs
+    
+    Otherwise, if we restart the MDS right after the scrub,
+    then scrub again, it will see the same inconsistency when
+    it looks at the on-disk state.
+    
+    This involves adapting the use of ScrubHeader to be something
+    writeable during scrub to record whether repair happened.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e52729699e87dd75022433153450fd4ba71b40e)
+
+commit f19f1e1cbc5b23eeb028c015c77ffe32306aba64
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 05:24:52 2017 -0400
+
+    mds: don't report repaired backtraces in damagetable
+    
+    Fixes: http://tracker.ceph.com/issues/18743
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 444382c3370bafa21564bb1e6a5c80f3e4825f6a)
+
+commit 162e25ec04e5288608718dcbd9d57b8234c17fb9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 05:24:03 2017 -0400
+
+    mds: cleaner scrub complete log msg
+    
+    People often are not using a tag with their scrub,
+    let's not have the ugly "with tag ''" in that case.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fd00a6e1b914880ab9f401faf591e73024f18ef)
+
+commit 87991643b38b1a0e526b599b65bb010e66c08374
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 12:39:49 2017 +0800
+
+    mds: prevent filelock from being stuck at XSYN state
+    
+    Lock:file_eval() does not properly handle the condition: filelock
+    is in XSYN state, both loner_cap and want_loner_cap are >= 0, and
+    loner_cap != want_loner_cap.
+    
+    To set loner to the wanted one, mds need to revoke caps from the old
+    loner. Changing lock state to MIX can do the job.
+    
+    Fixes: http://tracker.ceph.com/issues/22008
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com
+    (cherry picked from commit 67a428741fc82eb5f4120042197acf0cec34213a)
+
+commit 4fbde822b85fb96e79f800df0ab24d508887665b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 10:50:14 2017 +0800
+
+    mds: implement LOCK_XSYN_LOCK and LOCK_XSYN_MIX
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit c7261e1a6d95d1ed98a1d9066905aa75af4c2886)
+
+commit f354347debffec93910e0db0be4e165c00778d83
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 10:37:44 2017 +0800
+
+    mds: cleanup set/drop loner code
+    
+    integrate functionalities of try_drop_loner and try_set_loner into
+    CInode::choose_ideal_loner
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit d4cbf3aeeada57a88b794d3d6d5fb1cd891e31cd)
+
+commit ac0e193f402a90202fafa3f1bfff4f99697babba
+Merge: bd75db92c6 06a1c3d1e4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 09:40:01 2018 -0800
+
+    Merge pull request #19157 from pdvian/wip-22237-luminous
+    
+    luminous: mds: don't delay processing completed requests in replay queue
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit bd75db92c630fc6636a357697f01784c9b2aa98e
+Merge: 4974d42eea 024e11dbfc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 09:37:50 2018 -0800
+
+    Merge pull request #20327 from theanalyst/wip-22922-luminous
+    
+    luminous: rgw: bucket resharding should not update bucket ACL or user stats
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c63fc964cef1d45fd0a4de9c6715bcd16badc597
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 14:14:07 2017 +0800
+
+    mds: limit size of MExportDir message
+    
+    Only export port of subtree if whole subtree tree is too large for
+    sinlge MExportDir message. The un-exported portion are added to
+    export queue.
+    
+    Fixes: http://tracker.ceph.com/issues/21892
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ca5c987860821a7784db1c6a74d1c661cdf427cc)
+
+commit a7fd4f95d2d4462f4e382cd85f0b011786319ff9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 16:13:32 2017 +0800
+
+    mds: optimize MDCache::try_subtree_merge
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 1f27ebbef95b39f7fa3c94d5dc9f191a58c9f9b4)
+
+commit 45f4b1a141fe3c3caafaece0e82f2f35d0586030
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 12:03:44 2017 +0800
+
+    mds: optimize import/export state access
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fb9277c14314aaa6a97ee23da9057ddc3c750408)
+
+commit bf3ee8b22bf1c2242189107cc662a28f5ea7c219
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 25 12:09:20 2018 -0500
+
+    rgw: fix for empty query string in beast frontend
+    
+    when the target does not contain a ?, the QUERY_STRING was being set to
+    the same value as REQUEST_URI. this QUERY_STRING is included in the
+    signature, and caused SignatureDoesNotMatch failures
+    
+    Fixes: http://tracker.ceph.com/issues/22797
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ab9e79684ac7ae33b78522df6732b99271d10016)
+
+commit 6caae86b7c7591b1e2902a9002b7649fe99022e5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 22:28:40 2017 +0800
+
+    os/bluestore: propagate read-EIO for aio
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 301912603789d78e3560da84df1e337edd046e46)
+
+commit e57274ee2a51e100579dadae88eb3c61e4fb1642
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Wed Sep 6 18:09:55 2017 +0800
+
+    os/bluestore: report error and quit correctly when disk error happens.
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit ed3175895fa92929089f7507e7d173f557d61e27)
+
+commit 68180fafd60681a2ea385f0785463792586f01d5
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Wed Sep 6 18:06:06 2017 +0800
+
+    os/filestore: use __func__ instead of hardcode function name
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit fdb4048a92fad40beab1350a68b29681bd4b6131)
+
+commit 9b93ff70a329e3a66b918d8194f851a2c8a5e3d1
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 14:30:47 2017 +0800
+
+    os/bluestore: add "bluestore_read_eio" counter
+    
+    To track down how many EIO errors have been propagated to
+    high level callers.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit d2d989fcc6a194b9374aeea64e45969b08990262)
+
+commit 902c01792228beb2a539095fec5f24bd293814b5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 10:30:04 2017 +0800
+
+    os/bluestore: propagate read-EIO to high level callers
+    
+    E.g., we can let auto-repair to properly handle this
+    instead of crashing the whole osd.
+    
+    Observe this once on one of our test cluster:
+    
+    /clove/vm/clove/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: 6604: FAILED assert(r == 0)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit a51d9e979fbbe49917712889efb102806479fad2)
+
+commit 024e11dbfcb7077ce2956969f2242bb9bc814d9d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 21 10:12:43 2018 +0200
+
+    rgw: bucket resharding should not update bucket ACL or user stats
+    
+    bucket link op resets bucket ACL using rgw_link_bucket instead
+    
+    Fixes: http://tracker.ceph.com/issues/22124
+    Fixes: http://tracker.ceph.com/issues/22742
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 1223baf76b45559f171c67c5e737f8e5f7d843e0)
+
+commit 91673b1edfd3d4023d77eb95801db6acf7f900f9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 23 21:07:08 2018 -0600
+
+    os/bluestore: change bdev parse error to ENOENT
+    
+    If there is not a valid label, then the label is not found.  This is a
+    more reasonable error code than "Invalid argumnet".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5cd98b0dfdc7758761ff789b12faca2266a7e128)
+
+commit 9c083ea16b02c4daef444629439247a415c557c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 23 21:06:27 2018 -0600
+
+    os/bluestore: make bdev label parse error less noisy
+    
+    This happens during the normal initialization of a new bluestore osd and it
+    is confusing for users.  Make it less noisy.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14c498dd22d1e89d30a0896e89fe83e8adf6ac76)
+
+commit 40a16ac6cc1550a7d8631c10cbc523a089d143a2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 31 15:45:44 2018 -0600
+
+    ceph-volume: adds tests for zap --destroy and --osd-id
+    
+    These are not added for dmcrypt right now because there is a bug related
+    to zapping encrypted disks that needs fixed first.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 8d38932c211c589b6171d9e17b100869c23890ac)
+
+commit 97592c255a6df1caeb97e41f88160e17c6e3878b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 31 09:04:09 2018 -0600
+
+    ceph-volume: no need to return osd_id from util.prepare.check_id
+    
+    Now that osd_id is passed to util.prepare.create_id it doesn't
+    make any sense to return osd_id from check_id anymore as it's
+    not being used.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f88f2d3bc1f5dcae1e9b2be62dab20942374fc29)
+
+commit c01a2bf515c08ae0a058fdf0fd2ca5f469ed5880
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 15:00:04 2018 -0600
+
+    ceph-volume: when reusing an osd ID you must give that id to 'osd new'
+    
+    If you do not then auth will not be created for the new OSD and the
+    daemon will not be able to start.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 43f699fcbc9f30875f70295e086a4e423008b266)
+
+commit 7d0c8833a9b91a6a9da41f4b97938c8dbd244a99
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 11:39:05 2018 -0600
+
+    ceph-volume: prepare_* methods do not need to recreate fsid or osd_id
+    
+    The prepare_filestore and prepare_bluestore methods do not need to
+    attempt to recreate osd_id or fsid because this happens in prepare
+    already.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7c809166aed82298b1fe1581f0e28174a535574f)
+
+commit eab0d1e6cd760e3ec854febae3988a8024d136d4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 11:32:53 2018 -0600
+
+    ceph_volume: modify util.prepare.check_id to handle stdout as a list
+    
+    process.call returns stdout as a list, not a string
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit a08fc639613b3d9bcb3a09875e535287dd58870a)
+
+commit d670b7b3fcb191b8e9f415f261b029689087d116
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 09:22:58 2018 -0600
+
+    ceph-volume: tests for util.prepare.check_id
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0d27fff319f1c4705648fdaff18ee47441f26a24)
+
+commit 55c0eda3f31f1327578daa11435720d1e513a250
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 09:21:47 2018 -0600
+
+    ceph-volume: return osd_id from util.prepare.check_id if it exists
+    
+    This also changes this so the osd_id is returned as a string so
+    an ID of 0 would evaluate to True.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 699aa65878ca18f2e2cd70d7444cad18b4a3fd98)
+
+commit 5b04509b61dcfa4b920ef16d62a93934df1c363b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 29 10:48:09 2018 -0600
+
+    ceph-volume: check to see if an OSD id exists before reusing it
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 548eadeec6fb42a18c3e84dbbd8164862f8836e6)
+
+commit 7e8dbc50129b7729036974bf3f20cfbfad2794bb
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 29 10:43:04 2018 -0600
+
+    ceph-volume: adds a prepare util for checking OSD ID existance
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 4ac99de6d98b646bf160230584f6532775cdc9cb)
+
+commit 03569abb408e603e4799f97eac333d118ddf23c7
+Author: Shengjing Zhu <zhsj@umcloud.com>
+Date:   Thu Feb 1 16:47:26 2018 +0800
+
+    pybind/mgr/dashboard: fix duplicated slash in html href
+    
+    The url_prefix val in html templates is expected not containing '/'
+    in the end. Otherwise the href will be rendered as '//static/', which
+    causes browser treating it as http://static/.
+    
+    Fixes: http://tracker.ceph.com/issues/22851
+    Signed-off-by: Shengjing Zhu <zhsj@umcloud.com>
+    (cherry picked from commit 24fe1cae636bf00e3d5923c7e826992d42d92946)
+
+commit b47837100f3eae46a2c983f0299e40bf7944c9f8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 15:16:40 2018 -0500
+
+    doc/ceph-volume OSD use the fsid file, not the osd_fsid
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a5f26c622b0f11b7d8179da22d18f719b9febe0a)
+
+commit 4974d42eea073bd1f2f27cdd3b5a35be789450a7
+Merge: 239b1ae5e1 f4d5757fb5
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Feb 5 16:08:27 2018 +0100
+
+    Merge pull request #20241 from ceph/backport-wip-rm22619
+    
+    luminous ceph-volume: dmcrypt support for lvm
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5282a60c8fa50db6fff957673dbd73ec1c22a59a
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Fri Jan 5 00:54:30 2018 +0800
+
+    Avoid the assert failure when the inode for the cap_export from other MDS happened not in MDCache
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 82b9b838ae0bd17247f6296f9d7001331f50663c)
+
+commit 042144a281073c4cd3e80e969f7eaf7f2b289de5
+Author: Shangzhong Zhu <zhu.shangzhong@zte.com.cn>
+Date:   Tue Jan 23 20:35:36 2018 +0800
+
+    mds: set higher priority for some perf counters
+    Fixes: http://tracker.ceph.com/issues/22776
+    
+    Signed-off-by: Shangzhong Zhu <zhu.shangzhong@zte.com.cn>
+    (cherry picked from commit 05c496d444b388ec505df4598abd3fc6309ed99f)
+
+commit 239b1ae5e19c16e976c2045fef5ad65f1f727278
+Merge: 982bd23865 7a8e915483
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sat Feb 3 11:31:34 2018 -0800
+
+    Merge PR #18930 into luminous
+    
+    * refs/pull/18930/head:
+            mds: additional damage handling case in EImportStart
+
+commit 982bd2386543be41eb3d67f9207971d285b08807
+Merge: b8392683a1 127236f676
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Feb 2 15:47:11 2018 -0800
+
+    Merge pull request #20098 from ovh/bp-luminous-snaptrimq-visibility
+    
+    luminous: osd, pg, mgr: make snap trim queue problems visible
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b8392683a1cc9ab489d3c77f8edac395469b2ce8
+Merge: 9083f7db62 5cec46600d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:10:18 2018 -0800
+
+    Merge pull request #19267 from tchaikov/wip-ceph-disk-fsid-luminous
+    
+    luminous: ceph_disk: allow "no fsid" on activate
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 9083f7db62b49037d1b6d8c1a092f0028b5882fb
+Merge: 119c9f0363 bd558d7e72
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:09:49 2018 -0800
+
+    Merge pull request #19387 from ceph/wip-cd-luminous-upgrade
+    
+    luminous: tests: ceph-deploy: use wip branch for final upgrade step
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 119c9f03637bcb2e7236b8010fb038f0c3235bad
+Merge: 5b9c671928 5a40ef2244
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:08:49 2018 -0800
+
+    Merge pull request #19610 from ukernel/luminous-22306
+    
+    luminous: osdc/Journaler: add 'stopping' check to various finish callbacks
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5b9c6719287aa9e2783cdbec9be95955f877f3bb
+Merge: faf8e1cc56 44643cae8d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:08:10 2018 -0800
+
+    Merge pull request #19967 from smithfarm/wip-22707-luminous
+    
+    luminous: ceph_objectstore_tool: no flush before collection_empty() calls; ObjectStore/StoreTest.SimpleAttrTest/2 fails
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit faf8e1cc5654abea8efcc7d05804fdd2db8f5803
+Merge: b7728d23be 654678ab2d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:04:29 2018 -0800
+
+    Merge pull request #19047 from shinobu-x/wip-22193-luminous
+    
+    luminous: bluestore: OSD crash on boot with assert caused by Bluefs on flush write
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b7728d23bec0fb8af5d10d8af6c48d325331818c
+Merge: f185e67496 e56106cfc1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:01:50 2018 -0800
+
+    Merge pull request #19388 from ceph/wip-cv-luminous-bp
+    
+    luminous: qa: ceph-volume updates
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit f185e67496721fe04cef4d99d097242a8277b2bd
+Merge: 9ed49bdb3e e621e9d7f1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:01:04 2018 -0800
+
+    Merge pull request #19487 from smithfarm/wip-22389-luminous
+    
+    luminous: ceph-objectstore-tool: Add option dump-import to examine an export
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 9ed49bdb3ed0953e348bcde7cc73d39022b5a759
+Merge: 2d92e6a4b8 5e5d5c8196
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:00:26 2018 -0800
+
+    Merge pull request #19501 from shinobu-x/wip-22399-luminous
+    
+    luminous: Manager daemon x is unresponsive. No standby daemons available
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2d92e6a4b89d5bd12aef5f7049f94a63ca1228af
+Merge: e579ed377d 6cc4a33933
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:59:49 2018 -0800
+
+    Merge pull request #19865 from smithfarm/wip-22634-luminous
+    
+    luminous: build/ops: ceph-mgr dashboard has dependency on python-jinja2
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e579ed377daf35679c551a254ea0770806499e03
+Merge: 595a5984db d2a68571a8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:59:08 2018 -0800
+
+    Merge pull request #19966 from smithfarm/wip-22706-luminous
+    
+    luminous: tests: force backfill test can conflict with pool removal
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 595a5984db75025018ab0f740e1e2de7eebf7931
+Merge: 241573d06c cd9e6788cb
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:58:23 2018 -0800
+
+    Merge pull request #20116 from smithfarm/wip-22266-luminous
+    
+    tools/ceph_monstore_tool: include mgrmap in initial paxos epoch
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 241573d06ca41144e1674f418ab7ae923485e20d
+Merge: 2d6afdca94 f454c293d1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:57:20 2018 -0800
+
+    Merge pull request #20215 from smithfarm/wip-22701-luminous
+    
+    luminous: build/ops: ceph-volume fails when centos7 image doesn't have lvm2 installed
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 2d6afdca9449c94d81aa1c10266894c94f192f2b
+Merge: fcfd31a5eb f2f00cac71
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:56:30 2018 -0800
+
+    Merge pull request #20219 from tchaikov/wip-pr-20128-luminous
+    
+    luminous: common/pick_address: wrong prefix_len in pick_iface()
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit fcfd31a5ebc7cad1a1a4304a007cf48374a4f90c
+Merge: 77c4502c8d 9d617b4ecc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:38:23 2018 -0800
+
+    Merge pull request #19849 from liewegas/wip-monlog-luminous
+    
+    mon: allow cluster and debug logs to both go to stderr with different prefixes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 77c4502c8d566cb1e63d8f42b0088f790922e61e
+Merge: f24fb875a8 f59380e159
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:36:55 2018 -0800
+
+    Merge pull request #19888 from pdvian/wip-22633-luminous
+    
+    luminous: OSD crushes with FAILED assert(used_blocks.size() > count) during the first start after upgrade 12.2.1 -> 12.2.2
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f24fb875a8313d0965dce7dad05d988f6df118d7
+Merge: db33f288c0 8b13643b32
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:36:03 2018 -0800
+
+    Merge pull request #19928 from jdurgin/wip-luminous-omap-recovery
+    
+    config: lower default omap entries recovered at once
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit db33f288c036bf0ebdf540eb607745d53ad4589c
+Merge: 9379ed806a d5e2e43de8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:35:38 2018 -0800
+
+    Merge pull request #19938 from liewegas/wip-list-size-luminous
+    
+    luminous: common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9379ed806ae419871cd535ce067e7e7adc3eec8a
+Merge: 5879a59eb6 e9f5612bd7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:34:27 2018 -0800
+
+    Merge pull request #19947 from tchaikov/wip-pr-18191-luminous
+    
+    qa/standalone/osd/osd-mark-down: create pool to get updated osdmap faster
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5879a59eb65b6d5277658a8a130d57be00c60d4b
+Merge: 8f212ed733 8e3edae0c8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:33:48 2018 -0800
+
+    Merge pull request #19969 from smithfarm/wip-22691-luminous
+    
+    luminous: ceph-base symbols not stripped in debs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8f212ed733baa63f0d971f00e23b8b92cd416fe8
+Merge: d9542bcf02 dc96e3c1f8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:33:13 2018 -0800
+
+    Merge pull request #19976 from pdvian/wip-22690-luminous
+    
+    luminous: tests: increase osd count for ec testing
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d9542bcf02dd3574d27eca04fb7ab799d06b8452
+Merge: 29ffc82d0b 91be5317da
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:32:39 2018 -0800
+
+    Merge pull request #19977 from pdvian/wip-22692-luminous
+    
+    luminous: common: compute SimpleLRU's size with contents.size() instead of lru.…
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 29ffc82d0bdb34a3c3e6b89df87b95b3bc6361e5
+Merge: 6b80c14af1 07a122a3fd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:31:56 2018 -0800
+
+    Merge pull request #19995 from pdvian/wip-22698-luminous
+    
+    luminous: common/throttle: start using 64-bit values
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f4d5757fb574411e8e47055f2aaaf54bce076501
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 14:21:37 2018 -0500
+
+    doc/ceph-volume add lvm/encryption to the index
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 163ee6a61235e97c341b2817114d98b576697096)
+
+commit 4d237d7418a7c1a6f7bce80d978ee5fae8e2d22a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:11:32 2018 -0500
+
+    doc/man/ceph-volume add dmcrypt support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit babcdf2fa59baab6bc2cd8a1ac190b8d02fa3fce)
+
+commit 5a7b50f121aee3fc13474b42e11b5d19a36d7764
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:06:25 2018 -0500
+
+    doc/man/ceph-volume add docs for zap sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ad73be2c2a6ee273d37b706626974d6900742256)
+
+commit 3c8abd24b2801a8e76f4b327a1b029cb0eecfdd7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:03:18 2018 -0500
+
+    doc/man/ceph-volume add docs for list sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 498d7a06a1f17c769c11da7ba9f83bcf2c0e433d)
+
+commit e51d7dd4da3a156250489f33deddcbe8c208d2aa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 11:41:32 2018 -0500
+
+    doc/dev/ceph-volume update internal LVM metadata information
+    
+    Includes dmcrypt information, and updated key names to reflect current
+    code
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 46babe80b638eb607e2af6f0c8bb77e675d4d63b)
+
+commit 8b938c42f76b95425476c7ee2df275fa721d8de9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:12:52 2018 -0500
+
+    ceph-volume tests.functional add xenial filestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f0f0c1f8c4dc433d8978de134c120328842a5d7f)
+
+commit 5c405a05fcb07b8abd81a11c152f197e3596db78
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:12:40 2018 -0500
+
+    ceph-volume tests.functional add xenial bluestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit be0dfe76535fa135168763275513d7fafba6fe4c)
+
+commit 558b5c62fcc1dd6ac7627e11523c60b0bfab3787
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:08:25 2018 -0500
+
+    ceph-volume tests.functional add dmcrypt support in tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d1ef5c487ba1ea868fd85b8c425986a2cf435519)
+
+commit 95c279b4363a3630422f15a3bd5ad33bd10204e3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:03:58 2018 -0500
+
+    ceph-volume tests.functional add centos7 filestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2477ac805e172a10ae3214151a78b4601cf88021)
+
+commit 9b9ed02de83fa4397f776427fa37a21fdb16f4b9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 10:45:32 2018 -0500
+
+    ceph-volume tests.functional add centos7 bluestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 18972322567e158a56d0e71ca4f2afbed413c855)
+
+commit 1c560cd8053ab5e6b921b3c1365cb70a1c71e811
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 09:58:36 2018 -0500
+
+    ceph-volume tests add util.prepare unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a774eae43a089798f895507f18b0c5d68a49fbb4)
+
+commit 43fb4561d8c44ddd5d246e601671d91606a09ca3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 09:58:18 2018 -0500
+
+    ceph-volume tests add fixture helpers for call and run
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d51973a64d9f18e0dfde7601b3ffbc57571ea653)
+
+commit 22c10490150426a45c832a59cf6613d331f7cb19
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:37:00 2018 -0500
+
+    ceph-volume util allow prepare module to work with encryption keys
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a36dcdf9cfbc3ca012c5cb022e0d7dbcd95984f4)
+
+commit b5f674daff7113a7c17b10cd213734140545b018
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:22:06 2018 -0500
+
+    ceph-volume lvm.activate allow encryption setups for luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3476e87cec353b100cf8c117d74708f51dd6de2c)
+
+commit a26321213b8d77084345af306d7ee8a73b238255
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:19:26 2018 -0500
+
+    ceph-volume lvm.prepare allow encryption setups for luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit be3300455e479b03ac072d28bb05ccc7eaddb2e9)
+
+commit fde4817d87132d5dade973fba36743f8b3c37230
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 14:46:42 2018 -0500
+
+    ceph-volume process allow silencing output on file logging for sensitive data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e2190c7a22b61aeecdb60b6c8bfe5ba6aa873a4)
+
+commit f357d5db65d427c1184169e588c475742ac1a45d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 14:29:53 2018 -0500
+
+    ceph-volume util create a helper module for encryption
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc34a32675a870da98931adffacb2d3db4f74c6c)
+
+commit 38c8a384b22da1de36b567d2b3816ba936083216
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 16 09:02:31 2018 -0500
+
+    ceph-volume lvm.common add dmcrypt flag
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 88ff873e9a60bd811356f7e15c8ef7f826d09337)
+
+commit 65a38c7855f3e1ce7788d25d02ac7ca779b917a0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 11 09:02:03 2018 -0500
+
+    doc/ceph-volume create an encryption doc explaining dmcrypt workflows
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 477670a0912c9e01fe59155bf135eaf92bdc3318)
+
+commit 6b80c14af1f36dbb0bd5c73d9935d41bf7d53350
+Merge: a219bf4727 0aa31d29d5
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Feb 2 13:19:51 2018 -0600
+
+    Merge pull request #20244 from ceph/backport-19276
+    
+    luminous: ceph-volume: do not use --key during mkfs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7756a44529dd27e0960393e098104c7c1808bb36
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jan 25 12:08:47 2018 +0800
+
+    osdc/ObjectCacher: don't merge TX buffer heads
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0b1e515d1c5dddf3c11eddbe6d53167d2fb6aaab)
+
+commit 43b37b83df897b57c10cea2ae307897f1f7fb96c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jan 25 10:25:25 2018 +0800
+
+    osdc/ObjectCacher: fix off-by-one error in bh_write_commit()
+    
+    Fixes: https://tracker.ceph.com/issues/22741
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit cc4c6dcc2ffdc482f1915db569949e674691c372)
+
+commit d93eef31277d30020b016d74e8d6b7e93e850fd9
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Fri Jan 19 00:08:44 2018 +0800
+
+    mds: fix scrub crash
+    
+    Fixes: http://tracker.ceph.com/issues/22730
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit e741c1a6e16cb5179d623ef491f66d31b0ffdfd6)
+
+commit a219bf47277d923db18dd221660a04bb6bac3580
+Merge: e62f21ca34 f57a80f0a8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 1 15:40:51 2018 -0800
+
+    Merge pull request #20247 from ceph/revert-19552-wip-22452-luminous
+    
+    Revert " luminous: msg/async: unregister connection failed when racing happened"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f57a80f0a8f4ad4d7d482c05ac8b25fd160f55df
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Feb 1 17:28:06 2018 -0600
+
+    Revert " luminous: msg/async: unregister connection failed when racing happened"
+
+commit e62f21ca347750d3289bc1f66f5066a09c3445f8
+Merge: 419ed62db3 82c9e3dbe1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:10:13 2018 -0800
+
+    Merge pull request #19187 from liewegas/wip-trim-mgrmap-luminous
+    
+    mon/MgrMonitor: limit mgrmap history
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 419ed62db3d8a6aed94c1598e6dadaddbac7b9ab
+Merge: 11c3f8f3dc 3bfb493fa0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:09:29 2018 -0800
+
+    Merge pull request #19269 from tchaikov/wip-pr-19202-luminous
+    
+    luminous: cmake,common/RWLock: check for libpthread extensions
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 11c3f8f3dc179d34f3197ce7ffd07731b5e652c9
+Merge: 4106dbddea c22f06b3c5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:08:45 2018 -0800
+
+    Merge pull request #19499 from shinobu-x/wip-22402-luminous
+    
+    luminous: osd: replica read can trigger cache promotion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 4106dbddeab069e816219c1fc775852eab8fd1ff
+Merge: 6dcfa33858 9a9ed272a7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:08:07 2018 -0800
+
+    Merge pull request #19552 from shinobu-x/wip-22452-luminous
+    
+     luminous: msg/async: unregister connection failed when racing happened
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6dcfa3385836357f556e5d45e97a0b05eda5df58
+Merge: 589342d932 47f74bcd13
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:07:36 2018 -0800
+
+    Merge pull request #19628 from shinobu-x/wip-22501-luminous
+    
+     luminous: tests: do not configure ec data pool with memstore
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 589342d932d3e7310167842dd22c1303bf1156d5
+Merge: 37ce36b6d0 ce54fd9b99
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:06:56 2018 -0800
+
+    Merge pull request #19630 from shinobu-x/wip-22507-luminous
+    
+     luminous: bluestore: do not crash on over-large objects
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 37ce36b6d0e630b0c527ead826d9761fd4cb9622
+Merge: bb9964ae21 6791e67faf
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:06:17 2018 -0800
+
+    Merge pull request #19742 from tchaikov/wip-luminous-pr-19649-and-19650
+    
+    luminous: common/dns, erasure-code: fix mem leaks
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb9964ae2163eeff35538e4df897317c1e477fa8
+Merge: 6f0167b181 ac064e1fc8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:02:15 2018 -0800
+
+    Merge pull request #20049 from liewegas/wip-zero-length-luminous
+    
+    os: fix 0-length zero semantics, test
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6f0167b181a5850b9c40128b682ab210b217824a
+Merge: 742b7ba11f a0c0e637fe
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:01:35 2018 -0800
+
+    Merge pull request #20068 from smithfarm/wip-22761-luminous
+    
+    luminous: osd: do not check out-of-date osdmap for DESTROYED flag on start
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 742b7ba11fae67fe806306284b9ae76fcb8160a0
+Merge: c8082bda93 238ddccbbc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:01:11 2018 -0800
+
+    Merge pull request #20069 from smithfarm/wip-22770-luminous
+    
+    luminous: tools: ceph-objectstore-tool set-size should clear data-digest
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c8082bda938d6db1c4def4617875ab99429d78aa
+Merge: df3088dbfd 8950736b8e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:00:41 2018 -0800
+
+    Merge pull request #20150 from pdvian/wip-22807-luminous
+    
+    luminous: mon: do not use per_pool_sum_delta to show recovery summary
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0aa31d29d5eb3b2071dbdb678fc3f1dc303a64ea
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 2 00:43:21 2017 +0800
+
+    ceph-volume: process: disable stdin param of run()
+    
+    we cannot use process.communicate() to feed the Popen with input,
+    because, upon return of process.communicate() the stdout,stderr are
+    closed. see https://docs.python.org/2/library/subprocess.html#subprocess.Popen.communicate .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6df444cf33a251f5dcb536606b44af696cdb4ed5)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/process.py
+
+commit a67d46b63228517cd9ce37617a7e0f5f79ab16f1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 30 08:33:21 2017 -0600
+
+    ceph-volume: use --keyfile instead of --key
+    
+    We do not want the key to show up on the command line (it may appear in
+    the process list or sudo log file).
+    
+    Fixes: http://tracker.ceph.com/issues/22283
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 46abd50f8bcbd347d577581ae42156b718556158)
+
+commit 5717860f38cc79e165bd5bf791064177f8a4200d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 30 08:32:47 2017 -0600
+
+    osd: accept 'keyfile' as well as 'key' during mkfs
+    
+    Also, don't print the raw key to the log.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Kefu Chai <kefu@redhat.com>
+    (cherry picked from commit 18aff37ee3a2c2d3016f08c98161a5acc41ff8e5)
+
+commit df3088dbfd021393106a5fba93bd722a5e0f540a
+Merge: d43ec551af cf6799e4b5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:15:14 2018 -0800
+
+    Merge pull request #19370 from shinobu-x/wip-22339-luminous
+    
+    luminous: client: quit on failed remount during dentry invalidate test #19370
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d43ec551af92b7c205f80c1c6617a9b258a98697
+Merge: 685e526cc5 4943899005
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:14:47 2018 -0800
+
+    Merge pull request #19585 from ukernel/luminous-22460
+    
+    luminous: mds: handle client session messages when mds is stopping
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 685e526cc5d601f21292f13f56cfdcafd5c3c9b4
+Merge: 5ca708cd9a d0d66c73b0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:14:18 2018 -0800
+
+    Merge pull request #19626 from shinobu-x/wip-22499-luminous
+    
+     luminous: cephfs-journal-tool: tool would miss to report some invalid range
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5ca708cd9ac00e63557648ba14b4ef6b369dbaa1
+Merge: b5c189b84e 6eea75200f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:13:48 2018 -0800
+
+    Merge pull request #19627 from shinobu-x/wip-22500-luminous
+    
+     luminous: cephfs: potential adjust failure in lru_expire
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b5c189b84e7a98ece7c0d485ee7ec87bb2cdc0cf
+Merge: a6189cc37f 08edb16311
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:13:16 2018 -0800
+
+    Merge pull request #19776 from ukernel/luminous-22492
+    
+    luminous: mds: respect mds_client_writeable_range_max_inc_objs config
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit a6189cc37f169ecdce05576fd88362f7b2871927
+Merge: b4bc724549 dd6b6626cc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:12:48 2018 -0800
+
+    Merge pull request #19829 from pdvian/wip-22573-luminous
+    
+    luminous: vstart_runner: fixes for recent cephfs changes
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b4bc7245497d2550c9fc2fd047ac9a185c334b7b
+Merge: 66189e1656 98e3e2a7c8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:12:20 2018 -0800
+
+    Merge pull request #19830 from shinobu-x/wip-22579-luminous
+    
+    luminous: mds: check for CEPH_OSDMAP_FULL is now wrong; cluster full flag is obsolete
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 66189e16560485cdea9b97b46e47371cf134b249
+Merge: b35c39f6db 5dcd2a5977
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:52 2018 -0800
+
+    Merge pull request #19871 from smithfarm/wip-21948-luminous
+    
+    luminous: mon: MDSMonitor: reject misconfigured mds_blacklist_interval
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b35c39f6db246463045a5eb6b5c477b99fdeab77
+Merge: 243f187c02 5fd01ca888
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:28 2018 -0800
+
+    Merge pull request #19959 from smithfarm/wip-22694-luminous
+    
+    luminous: mds: fix dump last_sent
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 243f187c023962f17fa7715c196c45e712ea51a1
+Merge: b1c27aedb9 80ea9ed3b3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:02 2018 -0800
+
+    Merge pull request #19962 from smithfarm/wip-22580-luminous
+    
+    luminous: tests: full flag not set on osdmap for tasks.cephfs.test_full
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b1c27aedb92d79f507b4fa0bc514b2d173d87296
+Merge: f60c2df92d cc76ab1eef
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:10:29 2018 -0800
+
+    Merge pull request #19968 from ukernel/luminous-22699
+    
+    luminous: ceph-fuse: ::rmdir() uses a deleted memory structure of dentry leads …
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit f60c2df92d021f4eb82da5e74de06464818e30c8
+Merge: 69c87bedcc 5353569eea
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:09:07 2018 -0800
+
+    Merge pull request #19982 from ukernel/luminous-22719
+    
+    luminous: mds: handle 'inode gets queued for recovery multiple times'
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 69c87bedcc0814e10db6d38b50f836be6c0528c9
+Merge: d81ce5c0ed 2260f03b36
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:05:39 2018 -0800
+
+    Merge pull request #20042 from tchaikov/wip-pr-19998-luminous
+    
+    luminous: fix broken use of streamstream::rdbuf()
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d81ce5c0ed368d2ec01d539b282966e25b4138e7
+Merge: 2b0599d308 3ccf7e0bc6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:04:16 2018 -0800
+
+    Merge pull request #20106 from smithfarm/wip-22502-luminous
+    
+    luminous: osd: Pool Compression type option doesn't apply to new OSDs
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2b0599d308ecf7e7d637caf1932eef8020ff49fc
+Merge: 2ac9bbdd2e b12e00097b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 15:07:48 2018 -0500
+
+    Merge pull request #20240 from ceph/backport-20010
+    
+    luminous: ceph-volume: adds a --destroy flag to ceph-volume lvm zap
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b12e00097b7dc9ad322abc5f4e53d7367b3ee37c
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:54:55 2018 -0600
+
+    ceph-volume: clean up docstrings for lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 57471fe7e74c94351e713ff99cbcbaa7dc6a0f52)
+
+commit d06e7d52d27f19ec1ab9d39af4f45733fbe21246
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:41:53 2018 -0600
+
+    ceph-volume: print a message to terminal if --destroy is skipped
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d2529ee5dd9a4dc6a81eeb2d5e54c0bdd6beeb2f)
+
+commit aba99e70d9f3f555b7d7e183d2ea9e506ca2ca40
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:34:22 2018 -0600
+
+    ceph-volume: no need to set PVolume.vg_name explicitly
+    
+    This will be set automatically in __init__ by looping
+    through kw.items and creating properties on PVolume
+    with them.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 129182645c4a7e5e42500d3ef2083cfdfb0137e4)
+
+commit 039d1875e4e5ca52e2d54c35ff2dbfea25202d49
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:31:20 2018 -0600
+
+    ceph-volume: switch remove_vg and remove_pv to process.run
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5a1be9a73919e13fe603861588bcc20872a4c133)
+
+commit 5f98bc964b2ef05110363c70507048c85e33fa8c
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:25:46 2018 -0600
+
+    ceph-volume: allow passing a fail_msg param to process.run
+    
+    This will allow the user to provide a more detailed error message
+    on why the command failed.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7204851042fa42799e1003ef57d57a512d4a7f28)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/process.py
+
+commit 2ac9bbdd2ee98f0b07f61d507fbf56f5bfbe213e
+Merge: 3f0d1f7166 ccc6b55463
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 15:01:04 2018 -0500
+
+    Merge pull request #20239 from ceph/backport-19949
+    
+    luminous: ceph-volume: adds --crush-device-class flag for lvm prepare and create #19949
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 8d7552bfff302728669b51df41c2334e2520e2e8
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 12:04:38 2018 -0600
+
+    ceph-volume: tests for lvm zap --destroy
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit fafa183d8f13fa769eaab5f865f6592b89225901)
+
+commit 4f3ebfe4e1f008230bad5df7b42e6c103c75e79d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:10:48 2018 -0600
+
+    ceph-volume: also remove pvs with lvm zap --destroy
+    
+    Leaving the pv around doesn't keep us from redeploying on
+    that device because we also wipefs, but explicitly destroying
+    it makes the output read better.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cb748ccb2264aea93dd4717a46d7f833c5b1eccd)
+
+commit 2a42491c65bc49bbcd7ecd86ca61878dde5dc73d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:10:15 2018 -0600
+
+    ceph-volume: adds an api.lvm.remove_pv method
+    
+    Used to remove physical volumes
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0811c02b118c270b2bec87cd26a877b26bc780b1)
+
+commit 8fd6f29ee3988d3158467e9cf447e63c7d3485e5
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:05:32 2018 -0600
+
+    ceph-volume: remove the -a flag from api.lvm.get_pvs
+    
+    When we run `pvs -a` it returns all pvs, even ones that
+    have been recently deleted. Because of this pvs that have just
+    been zapped with `lvm zap` show up and can cause issues if a device
+    is zapped more than once.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 10a349742cae909d1ebf0a0b3b49c88774929f53)
+
+commit 4d23258e19b56d04f2d6f108ad808b01bc38b831
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 08:26:07 2018 -0600
+
+    ceph-volume: docs for the --destroy flag of lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 708cb43c503e962304211041570d6ca660bca1fb)
+
+commit 89a0aa94c9b5becee205a1803541ceb2aad961ad
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 15:11:19 2018 -0600
+
+    ceph-volume: adds the --destroy flag to ceph-volume lvm zap
+    
+    If you use the --destroy flag and are zapping a raw device
+    or parition then zap will destroy any vgs or lvs it finds on that
+    device.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit aea326b772395c076e0d75f7c2f591a46aef7f57)
+
+commit 3002953e4f506850339c676a85dc5402def7423d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 13:12:17 2018 -0600
+
+    ceph-volume: adds an api.lvm.remove_vg method
+    
+    This method can be used to remove volume groups
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 50a6a543a956d133b428cf36ad629c2131be5490)
+
+commit 9146305907b1c8d9f0162b44c76fd6133b7cb094
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 12:58:50 2018 -0600
+
+    ceph-volume: expose vg_name in api.lvm.get_api_pvs
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b7f24ec451e167d81a83c1e21597a9fece4f0684)
+
+commit 3f0d1f7166e20defae5ed8783f7f49f7a8b461d1
+Merge: 5250230360 4bd607b2ae
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 14:55:57 2018 -0500
+
+    Merge pull request #20238 from ceph/backport-19875
+    
+    luminous: ceph-volume: adds success messages for lvm prepare/activate/create
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit ccc6b55463335895b5896f4e772f6c0e412d7522
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 11:44:10 2018 -0600
+
+    ceph-volume: print the lvm activate success message correctly
+    
+    Move the terminal.success to the methods that have access to the
+    correct osd_id so that they do not print None for the osd_id
+    in the log message.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit c9f7689a7a71a6b2321301b84599f319e8ae8eba)
+
+commit 0e47c46fef88b3dc4ee9c7849da67f24db375044
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 15 11:55:44 2018 -0600
+
+    ceph-volume: docs for --crush-device-class
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 21f459437bd1208812504a20e5a50f7a47bfa979)
+
+commit 73e714357d898a06b7bb172691a0dbe360f0f535
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 11 13:11:31 2018 -0600
+
+    ceph-volume: adds crush_device_class to json metadata for 'osd new'
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3404d8bba503f3f74f411b11108d39cb626d3bed)
+
+commit 5d9b454522128c0ba0ee1c93485accc48e91e7a4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 11 13:04:07 2018 -0600
+
+    ceph-volume: adds crush_device_class to the functional tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 36a388d72acb049b5ee25ceea6a8d341aa58e31f)
+
+commit 00efa1a1beac442fb008f2ccb2e26939eab2cc10
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 10 12:19:54 2018 -0600
+
+    ceph-volume: add crush_device_class to lvm metadata
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 11da2185635ce14103471d851f1e8c84199d3745)
+
+commit 48591335c3903cb5446208da9940752fb8b390c7
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 10 09:29:15 2018 -0600
+
+    ceph-volume: adds a --crush-device-class flag to lvm create and prepare
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit bf468bc3737c15a8f15ec19ebf81c42672e28aaf)
+
+commit 4bd607b2ae7519414e8dd5c92e7522c8c5024060
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 12:21:35 2018 -0600
+
+    ceph-volume: adds success message to ceph-volume simple activate
+    
+    This used to only show the message if systemd was not being used, now
+    it will show this always if the command is successful.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 87c6ed14e7e0b528a1fe107e2e531908cdea0f47)
+
+commit ad73312ff0d376b8e055936e2f9c6d02ec90d0f4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:52:23 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm create
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0185d93cc1be1e7a6d3a803631ff92df9787754a)
+
+commit 4709380f6b23d7bb5e6c0b98f7745d2495f2af07
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:48:09 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm activate
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d30928d7c7d7cfb7e465ec2e7e46dc48407436f0)
+
+commit ce5df11e06c85e2d00ee1bdd98a0386212effce6
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:45:40 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm prepare
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 75c460f6e1864f15e1d23146c5740ce43c000e42)
+
+commit 5250230360a7d91d6cee1a425e183c8220684889
+Merge: 4e9092b194 25fbed91ec
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Feb 1 20:43:41 2018 +0100
+
+    Merge pull request #20237 from ceph/backport-wip-rm22281
+    
+    luminous: ceph-volume rollback on failed OSD prepare/create
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 25fbed91ec23260cba944411929055a75d151f96
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 15:31:55 2017 -0500
+
+    ceph-volume lvm.create rollback osd creation when prepare or activate fails
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 07be6fe0ab7981ebdc34779a8ccd6220d640a549)
+
+commit d6598635b89eb3dd11389c4fae322b8f3934b5c8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:22:05 2017 -0500
+
+    ceph-volume lvm.create fallback to bluestore when objectstore is not specified
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6781d6498e01bfc89189c235540e5c2a2e6bc8ca)
+
+commit c49473556746a0e7f0fe9b7a25e0a2cd056281d5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:21:20 2017 -0500
+
+    ceph-volume lvm.prepare fallback to bluestore when objectstore is not specified
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9b17ad2f8fe617d219fdb2f46dd2675669f32f08)
+
+commit 14e16a236ae6c5fad0f9e25a70a2c2d74eee5546
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:11:03 2017 -0500
+
+    ceph-volume lvm.prepare rollback osd when prepare fails and an osd id was generated
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 579d12806b2ffc693a4e70ba3eaf663fa281d35e)
+
+commit 2d23836024ee796f9212bb44ef9f1fb18d77eef4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:08:35 2017 -0500
+
+    ceph-volume lvm.common create a rollback_osd utility to cleanup failed osd prepare/create calls
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e533792e0dbafc160238c0f5783f4430e2705571)
+
+commit 4e9092b1949d635599a1d47ab8ccedfa69d77db5
+Merge: 041dd60315 26b7337583
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 12:30:26 2018 -0500
+
+    Merge pull request #19483 from smithfarm/wip-22375-luminous
+    
+    luminous: build/ops: luminous build fails with --without-radosgw
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 041dd603153c5270ebeb4091b537ae88b5941f2d
+Merge: c727db5bb9 445583a922
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 12:10:34 2018 -0500
+
+    Merge pull request #20211 from dillaman/wip-22198-luminous
+    
+    luminous: librbd: compare and write against a clone can result in failure
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 445583a922c9eecf1eba60df4f9da49fb7bc8926
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 08:24:50 2017 -0500
+
+    test/librbd: disable compare and write if skipping partial discard
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4e8b0b60a72fe1e1ac10a2beb46b77d4b58c91c4)
+
+commit e41f815e6b4e96ba7155202a2ce3e046370c0d56
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 12:10:30 2017 -0500
+
+    librbd: refactor object write request state machines
+    
+    Fixes: http://tracker.ceph.com/issues/20789
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0e643fb926f0484bb4f860740c7f85b692de6737)
+
+commit 11f21becff13e1f27f758500816cb9643cd010d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 10:24:08 2017 -0500
+
+    librbd: consolidate all object discard-related logic into single state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 318797f59adcedcc386e01f1975011e0086434ac)
+
+commit 4f77368ab08df27417cc5c5f3c3200a5f68733ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 8 12:31:28 2017 -0500
+
+    librbd: simplify interface between object IO and copyup state machines
+    
+    The initial copyup was not receiving a write hint and the
+    code for hints was duplicated multiple times. Additionally,
+    the object map state should match the last executed write op.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7117aa4ecaf0b619b527b2783fa1d09b11f3dd55)
+    
+    Conflicts:
+            src/librbd/io/ObjectRequest.h: trivial resolution
+
+commit 7807083ea27ccb4b60dbc050e9d135464756fdb1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 15:15:58 2017 -0500
+
+    librbd: copyup state machine needs to handle empty write ops
+    
+    The compare-and-write object operation cannot be executed
+    concurrently within a copyup operation since the object might not
+    exist yet (if not performing a deep-copy).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f6db9b8027b6978b4d28fcf9f0389c88f9109e75)
+
+commit 487c41e71460610ff3833a6cc6f3c21ab345a2d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 14:36:10 2017 -0500
+
+    librbd: object cacher should re-use read state machine
+    
+    This adds support for sparse-reads and ensures all object reads
+    utilize a single, tested code path.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e79c0cf2009c69681ed4c70f52ade2f5fd570567)
+
+commit b12810056dd74d0b6367123a4a7be6599b93f727
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 12:24:44 2017 -0500
+
+    librbd: refactor io::ObjectReadRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 62c3282b875f7345d7b71228d1e35e0cc87d44fa)
+
+commit f7b21684671b0d3df6db2bca9507921181a008e4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 14:17:25 2017 -0500
+
+    librbd: default template parameter for C_SparseReadRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ab0e558961ed9b63c00ad1d09ee0f9f79fd62295)
+
+commit 7b20cc9b519707e047bee6e6166daea232b35f5c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 13:42:19 2017 -0500
+
+    librbd: reduce lock scope when assembling read results
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 284feb6a32795c64b1c5ec3b3bf0d959e72dd940)
+
+commit ba3b6099945ae6e4d3781eee13027a6bcea2e61f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 13:33:12 2017 -0500
+
+    librbd: ObjectReadRequest does not require buffer extents
+    
+    The ReadResult wrapper requires the buffer extents to know
+    how to properly reconstruct the out buffer.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 35ce1c2d39302011adbf494ada6248495681a436)
+
+commit b77b7218c4087555997872661fb0d08f737406a4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 6 18:35:42 2017 -0500
+
+    librbd: templatize IO object request state machines
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 654a78fa54520370f5466f9d282d4e9f8365ad48)
+
+commit 1fe2fe8d899930feecf6deee01d594691cab3f90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 10 21:45:13 2017 -0500
+
+    test/librados_test_stub: mocked remainder of IO ops used by librbd
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c4e5613e4a74316ba9667654e887a835b240609a)
+
+commit 36f7a543252761f356aa60acc8d499142fed4966
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 10 16:26:19 2017 -0500
+
+    test/librados_test_stub: simulate set_alloc_hint creating a non-existent file
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4b851ce0fa534ca043aca835057b9d68660971a4)
+
+commit 1581b9e3d3cd5f3c90e3883709d5db947501bd40
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 23 09:46:42 2017 +0200
+
+    test/librados_test_stub: pass snap context to zero op
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 78ca0fde22076e8b156feb8dfd60aaee316a1895)
+
+commit c727db5bb921bd68cbc7e23c9c49c50c401a8669
+Merge: a3647582de 092e6cf29a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:15:26 2018 -0500
+
+    Merge pull request #20209 from dillaman/wip-22033-luminous
+    
+    luminous: mgr/dashboard: added iSCSI IOPS/throughput metrics
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit a3647582deb09ead251562afa6e92d970473f564
+Merge: 2f887e044e 436d89b9d5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:14:48 2018 -0500
+
+    Merge pull request #20210 from dillaman/wip-22169-luminous
+    
+    luminous: librbd: set deleted parent pointer to null
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 2f887e044ec10bb018ab8d6284be1a367769fbaf
+Merge: 402b8e0885 c1c43110ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:13:36 2018 -0500
+
+    Merge pull request #20208 from dillaman/wip-21920-luminous
+    
+    luminous: librbd: default to sparse-reads for any IO operation over 64K
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 402b8e08854c3436f92b9c8af01a429caee79da8
+Merge: d2dbebf90b c22e49f321
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:12:42 2018 -0500
+
+    Merge pull request #20207 from dillaman/wip-21793-luminous
+    
+    luminous: rbd-mirror: primary image should register in remote, non-primary image's journal
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit d2dbebf90b52ec4b23bcb1dd7bcd49e49c6414d9
+Merge: eaedbcd0e1 1a0e9498a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:12:11 2018 -0500
+
+    Merge pull request #20206 from dillaman/wip-21694-luminous
+    
+    luminous: librbd: journal should ignore -EILSEQ errors from compare-and-write
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit eaedbcd0e11ac833c833840a8a132d16f6f39cba
+Merge: 672ef6752b 2691ada5ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:11:39 2018 -0500
+
+    Merge pull request #20205 from dillaman/wip-22577-luminous
+    
+    luminous: qa/workunits/rbd: simplify split-brain test to avoid potential race
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 672ef6752bc1dd754cc74dabb0dd64b0ac5096f6
+Merge: 381c044d77 cfd05be6ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:11:04 2018 -0500
+
+    Merge pull request #20153 from pdvian/wip-22809-luminous
+    
+    luminous: librbd: fix snap create/rm may taking long time
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 381c044d7726983a459103bcd0b29f08ea0944c5
+Merge: b272904f57 37067a6f12
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:09:15 2018 -0500
+
+    Merge pull request #20135 from dillaman/wip-22806-luminous
+    
+    luminous: librbd: force removal of a snapshot cannot ignore dependent children
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit b272904f57eb1daa8769737bed268fccb4c9070f
+Merge: cf8fd69d16 2e69cd92d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:00:25 2018 -0500
+
+    Merge pull request #19123 from pdvian/wip-22174-luminous
+    
+    luminous: possible deadlock in various maintenance operations
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f2f00cac71c5ae1b6b19706b5a77dfa1bbf11ff7
+Author: Gu Zhongyan <guzhongyan@360.cn>
+Date:   Fri Jan 26 16:19:50 2018 +0800
+
+    common/pick_address: wrong prefix_len in pick_iface()
+    With prefix_len initialized as zero, mask in netmask_ipv4/6()
+    will always be zero, so find_ip_in_subnet() always return the
+    first interface.
+    Set prefix_len to the right value to fetch the right interface.
+    
+    Signed-off-by: Gu Zhongyan guzhongyan@360.cn
+    Signed-off-by: yupeng chen chenyupeng-it@360.cn
+    (cherry picked from commit b0d8043a72af22648c058382c46a132264736aaa)
+
+commit 06a1c3d1e4cdb0f267dfc1189534ef747ac38521
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Nov 20 08:29:12 2017 +0800
+
+    mds: don't delay processing completed requests in replay queue
+    
+    Completed requests can also be in replay queue, they need to be processed
+    (treated as lookup request) in clientreplay stage. MDS will be stuck at
+    clientreplay stage if it delays processing requests of this type.
+    
+    the bug was introduced by commit 0afbc033
+    (mds: don't rdlock locks in replica object while auth mds is recovering)
+    
+    Fixes: http://tracker.ceph.com/issues/22163
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 959fad28b2a1b6ad1e5f8597d04ceb9b1761b98a)
+
+commit f454c293d178f108fa0d795d8f2d4487942c71b7
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Thu Dec 14 17:43:59 2017 +0100
+
+    ceph-volume: Require lvm2, move to osd package
+    
+    Fixes: http://tracker.ceph.com/issues/22443
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit 02bc369e052125f50c7d3a7fe9b311215291c84d)
+
+commit 93f13ab24cf2c54a183e90e23a4bddb4fe028da3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Nov 21 21:32:57 2017 +0100
+
+    build/ops: rpm: fix systemd macros for ceph-volume@.service
+    
+    Fixes: http://tracker.ceph.com/issues/22217
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d1f2df37dd03d77132e711423aacda46768d1e02)
+
+commit e621e9d7f1447f796b3f80475e38bc7de3f9b6a6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 8 18:48:21 2017 -0800
+
+    test: ceph_objectstore_tool.py: Perform dump-import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c4602c9ac8f7819abc89c8668fbdc4572341ac5e)
+
+commit 24b0482add051f0e0bf5e40490a001540b29e074
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 20:57:47 2017 -0800
+
+    ceph-objectstore-tool: Add option "dump-import" to examine an export
+    
+    Fixes: http://tracker.ceph.com/issues/22086
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a8b8d541ddcc8085a9e7e301948665f3d6c17f0c)
+    
+    Conflicts: (trivial resolution: list of subcommands is slightly different in
+                luminous, compared to master)
+            qa/standalone/special/ceph_objectstore_tool.py
+            src/tools/ceph_objectstore_tool.cc
+
+commit a5282f55df1fcc55ad7ec140bde87001f2145d58
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 21:04:16 2017 -0800
+
+    ceph-objectstore-tool: Remove unused map section from log dump
+    
+    Caused by: 44cb6d30640954c66fc03d4ce58b583ffd614951
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b20eeada312c95713cf2ea032fd7437d2b282558)
+
+commit 99d740322e231b3c1ddc8a8c6517baca71c14a01
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 18:25:19 2017 -0800
+
+    ceph-objectstore-tool: Fix output of section type number
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 508328784516136fe189581bfbfe5a791db8e9fd)
+
+commit faef2777ffe0c6548616f1691c72470096e96fb4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 17:53:01 2017 -0800
+
+    ceph-objectstore-tool: Improve ceph-objectstore-tool usage output
+    
+    Remove unused argument test-align
+    
+    Caused by: 1c8731c31eb53e17d8f363906438963ab5688fe0
+    
+    Positional arguments are described by usage() because
+    the parsing code creates descriptions like "--object" for the
+    object positional argument.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6896cccc35bc9a159bbe8c3bbb6a0c9ab08fcac)
+
+commit cf8fd69d16b786caa178ca412c509425766856b4
+Merge: 427bad3aae dbf54df879
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 31 14:10:45 2018 -0800
+
+    Merge pull request #17875 from smithfarm/wip-21359-luminous
+    
+    luminous: cephfs: racy is_mounted() checks in libcephfs
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit cf6799e4b5323641502e6da9f546c9a6cee3cb21
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Wed Jan 31 22:57:42 2018 +0100
+
+    client: quit on failed remount during dentry invalidate test
+    
+    Fixes: http://tracker.ceph.com/issues/22269
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5b2b1d14468c290c56ee6c95ea557c99464e0098)
+    
+    Conflicts:
+      PendingReleaseNotes: trivial conflict
+            src/ceph_fuse.cc
+            src/client/Client.cc: i chosed to pick code from 5b2b1d1 because, to fixthe issue, we need to call _do_remount which was introduced in 5b2b1d1.
+
+commit 427bad3aae9b427fe7308e91e965e57a29964918
+Merge: 9e0eea2be2 d4f5dc2780
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 31 13:21:50 2018 -0800
+
+    Merge pull request #20121 from batrick/i21252p2
+    
+    luminous: mds: fix return value of MDCache::dump_cache
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5dcd2a59770887a3560d24712904c57f39dd3742
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 18 11:34:53 2017 +0100
+
+    mon: s/mds_blacklist_interval/mon_mds_blacklist_interval/
+    
+    We can rename LEVEL_DEV options at will.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ed7fc5370c45deb4c80a36e9166e6dd84d4a2b99)
+
+commit d8f9aec74259620bafcde7fa673eea52777d8348
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 23:06:07 2017 +0100
+
+    mon: tidy up mds_blacklist_interval
+    
+    Remove the misleading reference to this from the client
+    eviction page, it was never the right option to mention
+    there (my mistake).
+    
+    Demote the option from LEVEL_ADVANCED to LEVEL_DEV as it
+    is hard to imagine a good reason for the user to change it.
+    
+    Set a hard minimum of one hour, to make it harder to
+    corrupt` a system by setting it close to zero.
+    
+    Remove the legacy definition of the field while we're at it.
+    
+    Fixes: http://tracker.ceph.com/issues/21821
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0d68d3513e03cc2400facf421e08c8a92cb2886a)
+    
+    Conflicts:
+        doc/cephfs/eviction.rst (trivial resolution of conflict resulting from
+            0e43f0f01cd800fee4cd800f1545405b449fa55b being merged to luminous)
+
+commit 9e0eea2be2d135c3c55fb402c64db171a70022a1
+Merge: 6ef48aeb55 075e3b7506
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 31 13:27:40 2018 -0500
+
+    Merge pull request #20213 from dillaman/wip-21868-luminous
+    
+    luminous: doc/rbd: tweaks for the LIO iSCSI gateway
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 075e3b75063b0aaf04adb9f451651ab2290c0d15
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 3 10:09:38 2017 -0400
+
+    doc/rbd: tweaks to the Windows iSCSI initiator directions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 997bb80ba231c4a21237485ac6780d573be992c2)
+
+commit ba26353aae43cf01d9ee66e2d23d8c0ba9a0c9d9
+Author: Ashish Singh <assingh@redhat.com>
+Date:   Fri Oct 20 20:45:35 2017 +0530
+
+    doc: Added CHAP cofiguration instructions for iSCSI
+    
+    Added details to specify the CHAP username and password while
+    discovering/login the iSCSI target.
+    
+    Signed-off-by: Ashish Singh <assingh@redhat.com>
+    (cherry picked from commit 2a5755e0767108aefa6ccfb8a85e5d63e8c04fe8)
+
+commit a3101dd478baedbc701d548c2752bdba0556f4a2
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Mon Jan 22 17:08:55 2018 -0600
+
+    doc: fix ceph-iscsi-config version number
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3ab71020a5e420a333cb4a6b0be67ade185c1593)
+
+commit 3560c6de1892998cf848f799f312fea258dc8b01
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Sat Jan 20 16:06:42 2018 -0600
+
+    doc: add ceph iscsi test kernel info
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 5c30318660eebc882326d87eec39b4fbe4cc85d7)
+
+commit 4bc150c8280fe94f1c4525d2e58ba55cd1229e82
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Tue Jan 16 12:46:09 2018 -0600
+
+    doc: add rbd pool requirement for gwcli
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit b56d994f3addcd9c6a7848dec9be93568cdb2d0e)
+
+commit 30c93cdb409e64baa9f11f4e23717c5529f09b20
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Mon Jan 15 12:59:37 2018 -0600
+
+    doc: add daemon-reload for rbd-target-api setup
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 4ca0d508bb9b68c791abc5799d0195f4baace455)
+
+commit 46bf788c310031739016cf07a5be8c23170e6f90
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Tue Jan 16 12:04:28 2018 -0600
+
+    doc: add gwcli manual install instructions
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f45aafed1c7a5393e712f9fbfd7479fc444ad3d8)
+
+commit efdb585141bfebfaab38dc0049905c6c3a5baae1
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 14:54:06 2018 -0600
+
+    doc: separate gwcli install from post install setup steps
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f80f083d0a982d7e89c77e3a6f8d71c2871a8f2c)
+
+commit f399a3399cb210ef76fa17c8a31178a2327aaa25
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 14:53:16 2018 -0600
+
+    doc: add note to clarify trusted_ip_list use
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3cc367dc6ebca18e884d7c75d2cd1fa7907b7139)
+
+commit 2eda81b9245c340be76decd4728ccb5a3d950ac9
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 22:14:18 2018 -0600
+
+    doc: update esx instructions
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3806f9750d29c6d930a7c8a003902e79b99766f6)
+    
+    Conflicts:
+            doc/rbd/iscsi-initiator-esx.rst: trivial resolution
+
+commit f4b1f3974b26b52d0cfd54f6c1237bccf5cb9921
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 20:33:03 2018 -0600
+
+    doc: add warning about SCSI PGRs/reservations
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 62661a2e23ce98432f3c037fa9a8155a633ee182)
+
+commit 723d5fabfabfff2ad517f67a8fd1bdf1c2380bfc
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 18:51:51 2018 -0600
+
+    doc: add esx web based client images
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 78e12135eeb5829cad65cebd3c2be9df4d582b1f)
+
+commit 0f937ed3d67103e5874bd8862d36ab53177f4677
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 19:36:06 2018 -0600
+
+    doc: remove windows based vsphere client images
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 426b562fdc1b373b72de2d69c4851be22ec12008)
+
+commit f8ab982b72f2a8f27bc034945685dc55ef9222bf
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 19:35:22 2018 -0600
+
+    doc: fix gwcli examples
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 480f6a82a2b9dcd0179951ea16432d8d32f2c96d)
+
+commit 335fa90e0db06a2b6b74ac756f0d935ea4e67648
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 16:38:13 2018 -0600
+
+    doc: Update ceph iscsi kernel/distro requirements.
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f436e993d359dfdad26b7d569d555a3b3a5b798d)
+
+commit 0363593523c1b95233a23529ae1a8910088f0fd2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 11 12:02:55 2017 -0400
+
+    doc/rbd: tweaks for the LIO iSCSI gateway
+    
+    Fixes: http://tracker.ceph.com/issues/21763
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4f227e108d40271d5d6b2b1e88659d86cbc6c914)
+
+commit 092e6cf29a33d49ed1abddb2558695bd6de4ae5b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 31 11:05:47 2017 -0400
+
+    mgr/dashboard: added iSCSI IOPS/throughput metrics
+    
+    Fixes: http://tracker.ceph.com/issues/21391
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e55ef24c9140c23c832a2205a2154e68452545a3)
+
+commit 451ae9ed812481aedb372ddb97d0e01e1c711ca7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 31 10:41:36 2017 -0400
+
+    mgr/dashboard: include A/O start relative timestamp for iSCSI
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e62f186613a1de64a155a852b21ec9033c6e73ea)
+
+commit 076ee1a247a6a65e88f2b62b1742be3197f81387
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:40:55 2017 -0400
+
+    librbd: track image open and lock acquire time via perf counter
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8583270fd2d0930c1fccd2285378de0844b1d928)
+
+commit 30ff1fbec70cb7140ffc66a4cb25060795550fea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:21:42 2017 -0400
+
+    librbd: export read and writes performance counters for daemons
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7a9d10a4e61ed168678f38f6a224859e0df44099)
+
+commit fad78966d49f01b6c3d3fa3aba20f314d1e758b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:49:22 2017 -0400
+
+    librbd: moved performance counter enums to common location
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2224e5ca1dacef1e3f8ec4854f08b4abe86f5d75)
+    
+    Conflicts:
+            src/librbd/internal.h: trivial resolution
+
+commit 436d89b9d538b7521993b0fd6d0aeca253827775
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Nov 18 08:30:27 2017 -0500
+
+    librbd: set deleted parent pointer to null
+    
+    Fixes: http://tracker.ceph.com/issues/22158
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7b53256bb825383fc604a96d71bcd51c36668cc4)
+
+commit b8b3eee73d79345e28404281040f7bdd3c36d171
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 23 12:56:31 2017 +0800
+
+    qa: silence SLOW_OPS,PENDING_CREATING_PGS warnings
+    
+    this is an intermediate step to deprecate REQUEST_SLOW warnings.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4a1f2a5c78cc77eafe99f087d10c0d3daa7fca7b)
+    
+    Conflicts:
+            qa/suites/rados/singleton/all/thrash-eio.yaml
+            qa/suites/smoke/basic/tasks/mon_thrash.yaml
+            qa/suites/smoke/basic/tasks/rados_bench.yaml
+            qa/suites/smoke/basic/tasks/rados_cache_snaps.yaml
+            qa/suites/smoke/basic/tasks/rados_ec_snaps.yaml
+            qa/suites/smoke/basic/tasks/rbd_fsx.yaml: do not cherry-pick the
+    SLOW_OPS related changes.
+
+commit 030d7e8c840386c366ca794ec86e14ddab44d6d5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 1 23:17:57 2017 +0800
+
+    mgr: summarize osd metrics in MMgrReport and sent it to mon
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7e7978732d20c506eca581f2b153ede70ceefa3d)
+    
+    Conflicts:
+            src/mgr/OSDHealthMetricCollector.cc: only the change reporting
+    pending-creating-pgs is cherry-picked.
+
+commit 687967459b0a929fb33cc689401475c6354ab391
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 9 21:37:11 2017 +0800
+
+    mon/health_check: s/std::list/list/
+    
+    so it's self-contained, and .cc files including it are not obliged to `using namespace std`.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 416082c1ce6c357943f4a6b9e081a3211b1110c0)
+
+commit bb5ee06baac34aff59578575e14fe2a6bb6ae0ab
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 12:29:31 2017 +0800
+
+    osd: send health-checks to mgr
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f4b74125e44fe78154fb377fa06fc08b3325859d)
+    
+    Conflicts:
+            src/osd/OSD.cc
+            src/osd/OSD.h: only the changes related to reporting
+    pending-creating-pgs are cherry-picked. because we want to minimize the
+    impact to luminous. and to remove the slow ops from cluster log is not
+    in the scope of this backport. also, with_unique_lock was introduced
+    after luminous is branched, so use the plain lock_guard<> instead.
+            src/osd/OSDHealthMetric.h: because denc.h does not support
+    denc of enum types in luminous. so we need to case from/to osd_metric
+    when necessary.
+
+commit 2472ec3b956b104717fa23ca5d0dc29cfbc43955
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 14:57:08 2017 -0400
+
+    librbd: track the child of an image in-memory
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce2ae1d7c0f8818c86cbc746921e8eb6b0f89913)
+
+commit c22e49f3216c44bc26e029b05602a4b06e92c49d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:02:47 2017 -0400
+
+    rbd-mirror: journal debug message should not be logged as error
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 625222edbce66c98b86fc87fa3bd12ea40db17a2)
+
+commit 08ed96a7ddc1e587854a7da7edd6c350cc0a1fe5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:02:20 2017 -0400
+
+    rbd-mirror: avoid attempting to decode an empty client data buffer
+    
+    Fixes: http://tracker.ceph.com/issues/21961
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1e1d15eb0c14a6be82a6d75ab0088a6f0f4f8125)
+
+commit c1c43110ef683b7a1f8dbb7136ad44b31943fe5a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 13:52:56 2017 -0400
+
+    librbd: default to sparse-reads for any IO operation over 64K
+    
+    Testing BlueStore against both HDDs and OSDs with fully allocated
+    and sparse-allocated objects shows a performance improvement with
+    sparse-read between 32K and 64K.
+    
+    Fixes: http://tracker.ceph.com/issues/21849
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 251658471eab6d8cd968d678922bab437f72a9c7)
+
+commit cec71d94ffd0b86caee5524d2346b1d66f19869e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 14:32:05 2017 -0400
+
+    librbd: templatize io::CopyupRequest and io::ObjectRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dd5452004047c31e0f3eb813d5dd0b50bd68e76f)
+
+commit ba74ae952cbc032689833ab60bb8c59df99a675e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 5 16:38:32 2017 -0400
+
+    rbd-mirror: removed duplicate client registration logic
+    
+    The client will now be registered before the bootstrap state machine
+    is invoked.
+    
+    Fixes: http://tracker.ceph.com/issues/21561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d774d4702fbb9ff0b88797392b6673964d6b8c7c)
+
+commit f7d43cb0dc690dd0210090276514eaf257aa22d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 5 15:56:19 2017 -0400
+
+    rbd-mirror: local primary images should still register to remote images
+    
+    Fixes: http://tracker.ceph.com/issues/21561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d74b95dc10a5073ee7980583d94628ded7850ebb)
+
+commit 1a0e9498a6bd32987051fb02b3264bdb212a6310
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 15:42:50 2017 -0400
+
+    librbd: journal should ignore -EILSEQ errors from compare-and-write
+    
+    Fixes: http://tracker.ceph.com/issues/21628
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c7a9b045808ba741f6aadd2c2e65bd84135347c4)
+
+commit 2691ada5ac3179d874381b44c1482c3dc85cce31
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 19 16:29:18 2017 -0500
+
+    qa/workunits/rbd: simplify split-brain test to avoid potential race
+    
+    Fixes: http://tracker.ceph.com/issues/22485
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c2ee6dd0f4c77b9fbbc24a673841b04d3e175d6)
+    
+    Conflicts:
+            qa/workunits/rbd/rbd_mirror.sh: trivial resolution
+
+commit 69e8a0ab28c53303030423ac795577564827dbaa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Oct 29 19:07:54 2017 +0800
+
+    osd: remember is_primary when adding pending creating pgs
+    
+    so mgr can avoid calculating the same PG from different OSDs.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 486515ae884b91571a9066296680fff78f93f66d)
+
+commit 530283690eb8b5016ad5ed92f9f44cdcb977e847
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 24 22:41:30 2017 +0800
+
+    mgr/MgrClient: send health_checks using MMgrReport
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6fed1d21be1adf1464b5834787c95fae205a91b5)
+
+commit 6ef48aeb555790a6d5a8ef025a7ecf84dfe07de3
+Merge: 0489bddb5d bb3d091b46
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:09:08 2018 +0000
+
+    Merge pull request #19555 from shinobu-x/wip-22455-luminous
+    
+     luminous: balancer crush-compat sends "foo" command
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0489bddb5db768d4034bbcfdbd75151cc1df8278
+Merge: 8bbf1653ff f2a202775f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:06:59 2018 +0000
+
+    Merge pull request #20156 from pdvian/wip-22811-luminous
+    
+    luminous: mon: fix mgr using auth_client_required policy
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8bbf1653ff4fabdac6bc8e79c9913fed5c2727ab
+Merge: 1e3229418a 0aa3c5c577
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:01:45 2018 +0000
+
+    Merge pull request #19481 from jan--f/wip-22421-luminous
+    
+    luminous: mon: reenable timer to send digest when paxos is temporarily inactive
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1e3229418aa47cc1648121134e18d8f491c6cdac
+Merge: 2d64d41a39 c5cb0cab9f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:59:04 2018 +0000
+
+    Merge pull request #19553 from shinobu-x/wip-22453-luminous
+    
+     luminous: mgr/balancer/upmap_max_iterations must be cast to integer
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2d64d41a394b8dea69436c9b9c06d2a8aa3466f1
+Merge: d351505965 40954a9c27
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:49:41 2018 +0000
+
+    Merge pull request #19624 from shinobu-x/wip-22496-luminous
+    
+     luminous: KeyError: ('name',) in balancer rm
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d351505965654e1f8c4525ca7e9cfebd308828fc
+Merge: ea85e979b2 b5892a13b9
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:48:50 2018 +0000
+
+    Merge pull request #19834 from pdvian/wip-22585-luminous
+    
+    luminous: pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ea85e979b239127ee80373949dfa88a9c317d03f
+Merge: 81712618b4 1b2b589a12
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:47:05 2018 +0000
+
+    Merge pull request #20084 from pdvian/wip-22760-luminous
+    
+    luminous: mgr: prometheus: added osd commit/apply latency metrics (#22718).
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 81712618b4ce36511674bdfb0720b4d7bb454c06
+Merge: fab5aa92cb bb21f23731
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:46:03 2018 +0000
+
+    Merge pull request #20089 from pdvian/wip-22768-luminous
+    
+    luminous: mgr: disconnect unregistered service daemon when report received
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit fab5aa92cb2883b2aab88ffec769a57cc36fe115
+Merge: 7eb2423bef 91bfcc73f6
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:42:58 2018 +0000
+
+    Merge pull request #20182 from jcsp/wip-22831
+    
+    luminous: pybind/mgr/dashboard: fix reverse proxy support
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 7eb2423befadb1bd04ba68ebdbb7a4734daff9e2
+Merge: 3512fc40cb 52e59854c2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:47:21 2018 -0800
+
+    Merge pull request #17869 from smithfarm/wip-21479-luminous
+    
+    luminous: mgr: Services reported with blank hostname
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 3512fc40cb75f97ca66df1682c5873c5110a3b78
+Merge: 25cf00a740 4db6b35548
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:46:34 2018 -0800
+
+    Merge pull request #18832 from shinobu-x/wip-22075-luminous
+    
+    luminous: mgr tests don't indicate failure if exception thrown from serve()
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 25cf00a740573d329a35cd050b2723070d8e0bad
+Merge: ef1038691d eeb12253d4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:46:04 2018 -0800
+
+    Merge pull request #18842 from shinobu-x/wip-21863-luminous
+    
+    luminous: ceph-conf: dump parsed config in plain text or as json
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ef1038691da028df20b4e22a6a628bf3e6a72586
+Merge: a3091bdd20 de1d25c2dd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:45:40 2018 -0800
+
+    Merge pull request #19002 from tchaikov/wip-pr-18976-luminous
+    
+    luminous: udev: Fix typo in udev OSD rules file
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3091bdd20a64c02dc423052015e46d42240adf3
+Merge: 5563a65058 1bdd81e095
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:44:46 2018 -0800
+
+    Merge pull request #19023 from pdvian/wip-22164-luminous
+    
+    luminous: cluster [ERR] Unhandled exception from module 'balancer' while running on mgr.x: 'NoneType' object has no attribute 'iteritems'" in cluster log
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 5563a650587375640ecefca6643570d5366ce420
+Merge: b0b09df8aa c21e1c7831
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:44:16 2018 -0800
+
+    Merge pull request #19031 from pdvian/wip-22167-luminous
+    
+    luminous: Various odd clog messages for mons
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b0b09df8aa5150fcad9bfd2f200bd74b71f5faaf
+Merge: a35101135f 6f983c14d9
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:42:52 2018 -0800
+
+    Merge pull request #19039 from jan--f/wip-22199-luminous
+    
+    luminous: tools/crushtool: skip device id if no name exists
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a35101135f6fd8bb83e4a44baf7abe9e2c5b83d6
+Merge: 5f175b5bea 488228e919
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:41:50 2018 -0800
+
+    Merge pull request #19046 from shinobu-x/wip-22189-luminous
+    
+    luminous: osdc/Objecter: objecter op_send_bytes perf counter always 0
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5f175b5bea82c848a7c2bc9aef4860e5e8672310
+Merge: a1636c348e d5f2e16600
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:41:25 2018 -0800
+
+    Merge pull request #19059 from pdvian/wip-22176-luminous
+    
+    luminous: osd: subscribe osdmaps if any pending pgs
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a1636c348ebb1e8ec8a79e60cc22193701da84c2
+Merge: 5c79793917 540f425f1a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:40:44 2018 -0800
+
+    Merge pull request #19083 from smithfarm/wip-22213-luminous
+    
+    luminous: On pg repair the primary is not favored as was intended
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 5c7979391775529b5650afe18b25315bfc1e2c4a
+Merge: 525fe577b8 ed24c8ce8f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:40:04 2018 -0800
+
+    Merge pull request #19084 from shinobu-x/wip-22216-luminous
+    
+     luminous: "osd status" command exception if OSD not in pgmap stats
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 525fe577b803552eae568d765f34b4e9c98ea8c7
+Merge: 91d1a7bed8 8f67e4553a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:39:36 2018 -0800
+
+    Merge pull request #19118 from tangwenjun3/wip-backport-19030
+    
+    luminous: os/bluestore: fix the allocate in bluefs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+    Reviewed-by: Shinobu Kinjo <skinjo@redhat.com>
+
+commit 91d1a7bed8254f54290ca089ccc39a41a5da962e
+Merge: 2c2e60681d bdb9d385a5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:33:43 2018 -0800
+
+    Merge pull request #19133 from smithfarm/wip-22194-luminous
+    
+    luminous: Default kernel.pid_max is easily exceeded during recovery on high OSD-count system
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 2c2e60681d93b0aa3b1b8d8da8d1f02b12bdc8ed
+Merge: 3937ae421a 7ffe8c41d3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:30:53 2018 -0800
+
+    Merge pull request #19631 from shinobu-x/wip-22509-luminous
+    
+     luminous: osd: "sudo cp /var/lib/ceph/osd/ceph-0/fsid ..." fails
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3937ae421a47d781af11860788f999fcaaeeade4
+Merge: 74064cb9e4 0a347506af
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:30:11 2018 -0800
+
+    Merge pull request #19839 from tchaikov/wip-22558-luminous
+    
+    luminous: rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 74064cb9e4d2731919374a28c030cda034d24907
+Merge: f5dcf47a10 c86675dadc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:28:05 2018 -0800
+
+    Merge pull request #18838 from shinobu-x/wip-21973-luminous
+    
+    luminous: [test] UpdateFeatures RPC message should be included in test_notify.py
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f5dcf47a109c52bb4db7385d9ae6fd9ae53a5102
+Merge: 40ca9d22de 8cb21e6ea6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:27:34 2018 -0800
+
+    Merge pull request #18840 from shinobu-x/wip-21970-luminous
+    
+    luminous: [journal] tags are not being expired if no other clients are registered
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 40ca9d22de212ac111cd1d744106e21aa13d5615
+Merge: f7778d4700 74a6592d7f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:26:21 2018 -0800
+
+    Merge pull request #18949 from jcsp/wip-luminous-rgw-config-desc
+    
+    luminous: options.cc: document rgw config options
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f7778d4700e654c62b49158b30c541d6bf879523
+Merge: c421d533a2 2d9aafe1bc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:25:18 2018 -0800
+
+    Merge pull request #19051 from shinobu-x/wip-22185-luminous
+    
+    luminous: rbd: abort in listing mapped nbd devices when running in a container
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 91bfcc73f68a869703eb2f6a5ee5f5962b9b727b
+Author: Nick Erdmann <n@nirf.de>
+Date:   Mon Jan 22 12:33:32 2018 +0000
+
+    pybind/mgr/dashboard: fix reverse proxy support
+    
+    This fixes http redirection for reverse http proxies
+    
+    Fixes: http://tracker.ceph.com/issues/22557
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 95e1963cb5327f0699081c4c4b0b355d109ff0e3)
+
+commit f0cae0bffced632fcc9f0dfec5d1c85424d9a42c
+Author: Nick Erdmann <n@nirf.de>
+Date:   Fri Jan 26 13:40:27 2018 +0000
+
+    pybind/mgr: fix typo
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit fdd64bb2dac33085b47c737032a525d89319c866)
+
+commit ba2c6015ebe0d2fd55e38f1a1472810551e8d0c1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 15 11:49:08 2018 +0800
+
+    pybind/mgr/mgr_module: add default param for MgrStandbyModule.get_config()
+    
+    this matches its counterpart of MgrModule.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc7de8b124c32a2ee6910a3bac6a06f1c566a2ac)
+
+commit c421d533a258c8950205f59a032a3bdf9fd298fb
+Merge: d146d83076 660abda211
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Jan 30 09:29:15 2018 -0500
+
+    Merge pull request #20062 from cbodley/wip-luminous-rgw-bilog-trim
+    
+    luminous: rgw multisite: automated trimming for bucket index logs
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2e69cd92d07ca7f7957f67d40d46a04b94ccb0b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 14 12:07:36 2017 -0500
+
+    librbd: added missing locks for snap rollback and rename
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0ccd26f9989b892be2ab620bcf7f4fbf14980e8e)
+
+commit 7df6f0602cef58ee9185814ca3363d5448bb739d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 13 13:28:06 2017 -0500
+
+    librbd: possible deadlock with synchronous maintenance operations
+    
+    Fixes: http://tracker.ceph.com/issues/22120
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 90b7ecd8a8aac9a5c282d44004752ade0c195331)
+
+commit d146d830762414934375afe257c69a25f4e533f3
+Merge: 20cfb9bda3 67fbd3830f
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 29 10:22:14 2018 -0600
+
+    Merge pull request #19848 from cbodley/wip-luminous-boost-166
+    
+    luminous: backport beast frontend and boost 1.66 update
+
+commit 20cfb9bda35dea6ccd83f526a88a38edef210412
+Merge: 9981b51339 7dabbe9131
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 29 10:21:07 2018 -0600
+
+    Merge pull request #19500 from shinobu-x/wip-22401-luminous
+    
+    luminous: rgw: make HTTP dechunking compatible with Amazon S3
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9981b513395e261b472ff516b17df419cfec714d
+Merge: abeb3c56c0 fc791b980d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:12:39 2018 -0800
+
+    Merge pull request #17816 from smithfarm/wip-hadoop-s3a
+    
+    luminous: tests: qa/rgw: hadoop-s3a suite targets centos_latest
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit abeb3c56c074f9024565ed8d48dc0d89b4337084
+Merge: 8fe88c45e3 0d18c24fce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:10:27 2018 -0800
+
+    Merge pull request #18972 from linuxbox2/luminous-22084
+    
+    luminous: rgw: Fix swift object expiry not deleting objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 8fe88c45e3eb9d985ee20cf49679f5db08faf3db
+Merge: d55eda8267 fe5c8b34a7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:09:56 2018 -0800
+
+    Merge pull request #19086 from shinobu-x/wip-22210-luminous
+    
+    luminous: radosgw-admin zonegroup get and zone get should return defaults when there is no realm
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d55eda826741653537f64c6cccd7464d7b82efa2
+Merge: 7e84974d2d 14434beda6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:09:25 2018 -0800
+
+    Merge pull request #19088 from pdvian/wip-22177-luminous
+    
+    luminous: rgw: check going_down() when lifecycle processing
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 7e84974d2d5b653776eb89c623130ef651108a0d
+Merge: 0f1f98ad44 e3bb21529b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:08:15 2018 -0800
+
+    Merge pull request #19636 from shinobu-x/wip-22506-luminous
+    
+    luminous: rgw usage trim only trims a few entries
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0f1f98ad449b73568077513196f0d8dc43a9b07f
+Merge: d51dbcaf03 ad0ea9fa53
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:06:43 2018 -0800
+
+    Merge pull request #19784 from linuxbox2/luminous-20201
+    
+    luminous: RGW: S3 POST policy should not require Content-Type
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit d51dbcaf03a8c8a6c7a8330fb2f23a03d75a411a
+Merge: 1708c17f28 84fff7aca5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:05:58 2018 -0800
+
+    Merge pull request #19799 from smithfarm/wip-22581-luminous
+    
+    luminous: rgw: multisite: 'radosgw-admin sync error list' contains temporary EBUSY errors
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 1708c17f28dfc57462f9b05c1c9b6c73312193e5
+Merge: 7c3e468476 2adc133b86
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:03:10 2018 -0800
+
+    Merge pull request #19867 from smithfarm/wip-22591-luminous
+    
+    luminous: rgw: refuses upload when Content-Type missing from POST policy
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7c3e46847667ad104b3b718b7f4460b2fc9f763f
+Merge: 915a8e1454 4318ac5e3f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:02:23 2018 -0800
+
+    Merge pull request #20051 from ceph/revert-19538-wip-22434-luminous
+    
+    Revert "luminous: rgw: user stats increased after bucket reshard"
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 915a8e1454982e6b996e6757d13523fb4083cd2d
+Merge: 9882489f8a e0b8512419
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:01:56 2018 -0800
+
+    Merge pull request #20056 from linuxbox2/luminous-rgw-latchfix
+    
+    luminous: rgw_file: alternate fix deadlock on lru eviction
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9882489f8ae81104ae621cf9633f61b469640a8e
+Merge: 440fdfa438 35d2baf47b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:57:29 2018 -0800
+
+    Merge pull request #20073 from smithfarm/wip-22767-luminous
+    
+    luminous: rgw: librgw: fix shutdown error with resources uncleaned
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 440fdfa438bf881a3e5e2c8d4379284dc70eb673
+Merge: f83a246ba5 e3fbaa92ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:56:53 2018 -0800
+
+    Merge pull request #20075 from smithfarm/wip-22773-luminous
+    
+    luminous: rgw: file deadlock on lru evicting
+    
+    Reviewed-by: Matt Benjamin <mbenjami@redhat.com>
+
+commit f83a246ba550a7725438823c296cc18e56dea7ff
+Merge: e50835f80c b219b854b7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:56:04 2018 -0800
+
+    Merge pull request #20107 from dreamhost/wip-22792
+    
+    luminous: rgw: When a system object is created exclusively, do not distribute the
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a26b4ae76521e4cd12a65e69c735676454ce6e4c
+Author: Konstantin Shalygin <k0ste@k0ste.ru>
+Date:   Fri Jan 26 18:31:20 2018 +0700
+
+    mgr: balancer: fixed mistype "AttributeError: 'Logger' object has no attribute 'err'"
+    
+    Signed-off-by: Konstantin Shalygin <k0ste@k0ste.ru>
+    (cherry picked from commit 2062e84c7a33fc5170740f2d60d07ddf62085457)
+
+commit f2a202775fdf1132891221fa58700b276da7429b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 23 07:43:12 2018 -0500
+
+    mgr: apply auth_service_required to client conns
+    
+    Previously was using auth_cluster_required for all
+    connections, which meant that if someone had
+    disabled client cephx, they'd get BADAUTHORIZER
+    from their CLI when it tried to load mgr
+    command descriptions.
+    
+    Disabling cephx on the admin CLI is odd, but the mon
+    tolerates it so the mgr should too.
+    
+    Fixes: https://tracker.ceph.com/issues/22096
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 86ee30c33a06a13e6bb360b80dd1e41ba928147b)
+
+commit 2852a5f533a5cf044517fa288e8076ac493b1da0
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jan 22 13:42:20 2018 +0000
+
+    mon: fix mgr using auth_client_required policy
+    
+    This caused mgr daemons to fail to authenticate
+    when auth_client_required was set to something
+    different to auth_cluster_required.
+    
+    Fixes: https://tracker.ceph.com/issues/22096
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1e06fe003e50d360b5c5ce9824bba044b6406ec5)
+
+commit cfd05be6ace8fb3145068c464bc0a73d487836a1
+Author: Song Shun <song.shun3@zte.com.cn>
+Date:   Wed Jan 17 09:32:45 2018 +0800
+
+    librbd: fix snap create/rm may taking long time
+    
+      fix snap create/rm may taking long time
+      http://tracker.ceph.com/issues/22716
+    
+    Signed-off-by: Song Shun <song.shun3@zte.com.cn>
+    (cherry picked from commit d04ed348a1b20e5ea5bedada2462cb41f0f1d85a)
+    
+    Conflicts:
+            test_mock_SnapshotRemoveRequest.cc: define om as a ObjectMap<ImageCtx> obj
+
+commit 8950736b8e3c2837f201b4eba4fdebcc97fbd9be
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Jan 23 15:39:56 2018 +0800
+
+    mon: use pool_sum as param name in PGMapDigest::recovery_summary
+    
+    the param name of delta_sum of PGMapDigest::recovery_summary() is misleading too.
+    delta_sum fits the recovery_rate_summary() well, but not recovery_summary().
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 73ff298b898ca76e3f9b615449b4e4c0f2d0c017)
+
+commit e94900c7caa2bd920684073704bc688249bc0591
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Thu Jan 18 22:50:25 2018 +0800
+
+    mon: do not use per_pool_sum_delta to show recovery summary
+    
+    Fixes: http://tracker.ceph.com/issues/22727
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 5981ddc1992190431743ce1f0b834fb8899b6811)
+
+commit 540f425f1a86dabf746ac4bdd14c54b4ec76f664
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 20 19:50:32 2017 -0700
+
+    osd: Scrub of object with only object error(s) are not logged
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1ad05b1068ddd5d3312af45af1a60587200ddcd7)
+
+commit 4c2a8f1dd08882eca83512152c8570f497081229
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 20 17:34:49 2017 -0700
+
+    osd: Primary shard needs to be at the front to be preferred
+    
+    Caused by: cd0d8b0714d8684cf61b4650e170027ef46f489b
+    
+    Fixes: http://tracker.ceph.com/issues/21907
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b7d2f27da1a68046a688749f95093a88e6362b28)
+
+commit 17c64595a2f3481ab60ffe6e453a6dc879f1e01e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 19 12:56:58 2017 -0700
+
+    ceph-osdomap-tool: Add compact comand option
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b1f397f4fc3313ee0325722156759bfe32286c6b)
+
+commit 152272ce49cf2b32e4b3dbba5aa28019e9a183e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 18:07:41 2017 -0700
+
+    osd: Fixes for mark_unfound_lost
+    
+    Let recovery requeue operations and release backoff
+    For "delete" mark object context, if present, to non-existent
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c106f1a46815bb769e7a503b1adb265480b6779)
+
+commit 339e99005691ebdcdac45ba249add4ce4d6a8125
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 16:07:16 2017 -0700
+
+    ceph-objectstore-tool: Fix debug output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 090c773fc9b7f3316ed88edd24b85a703224ddc5)
+
+commit f23035fb17fb816c1b0f41ccf4072a14f50b1fa1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 16:04:59 2017 -0700
+
+    test: Remove bogus check in ceph_objectstore_tool.py
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f918b1fac1f3cdb4cf316fb4c7afc90de49ce06e)
+
+commit dbf54df87941f538e73b672131018ce4baba338c
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: reset unmounting flag to false when starting a new mount
+    
+    Once the "unmounting" flag goes to true, it never flips back to false.
+    We don't have a lot of checks for "unmounting" in the code, but they're
+    in some subtle places today and I plan to add more later.
+    
+    It's not clear to me whether it's possible (or advisable) to reuse a
+    Client that has previously been unmounted. It most certainly won't work
+    as expected today, and the checks I'll be adding will break it for sure.
+    
+    Make sure we clear the unmounting flag when we go to re-mount the
+    Client.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 3bc61f6c23a9ada5e2250f66e4df04a66dcfbacf)
+
+commit 20164df443795c4bde6bfdbb9eeda34e915c7557
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: rework Client::get_local_osd() return codes
+    
+    It currently returns -1 when there isn't one, but the C wrapper can
+    also return -ENOTCONN if the cmount isn't mounted. Change it to return
+    -ENXIO (No such device or address) in the case of "no local osd".
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 4285b970cb57c88deae0317be24afcf21a95fdc0)
+
+commit 12e93809aa1adbb7c3b112b4ba6b3a4374429163
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: remove misleading comment in get_cap_ref
+    
+    That may have been true at one time, but it's not always a single bit now.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 93f8cc1f71ffdc8dddca7efb56d82548a66ee3fe)
+
+commit e50835f80c63563a419ae47b05f72885e61e528e
+Merge: bba828b2d7 4efada35fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 26 08:15:17 2018 -0500
+
+    Merge pull request #20136 from dillaman/wip-22815-luminous
+    
+    luminous: qa/workunits/rbd: switch devstack to pike release
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 4efada35fd6b2d9b24cd89ac1dc3aeb7c823cfbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 24 11:28:19 2018 -0500
+
+    qa/workunits/rbd: switch devstack to pike release
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7ff1e0bc2193acd009a74db81cf8e8cc8639ad12)
+
+commit 37067a6f1275ea62d40f398c7b5d8e76f08f527a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 24 14:40:56 2018 -0500
+
+    librbd: force removal of a snapshot cannot ignore dependent children
+    
+    Fixes: http://tracker.ceph.com/issues/22791
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fcc58ecfeba6bbc72588e69dba35779f94d28ba5)
+
+commit bba828b2d761b1459b32588accb6838ceeb0cb40
+Merge: f473d3e444 4dda1b6ead
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:03:21 2018 -0800
+
+    Merge pull request #19105 from batrick/i22228
+    
+    luminous: client: anchor Inode while trimming caps
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit f473d3e4446110f219f69a577a4e5392ec4b422d
+Merge: 646a2c4dd4 ee06a16ee8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:02:32 2018 -0800
+
+    Merge pull request #19360 from ukernel/luminous-22219
+    
+    luminous: mds: ignore export pin for unlinked directory
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 646a2c4dd4e02a2f995b9b59d514cdef19065478
+Merge: 1b1ce1326e 1643d5a4f8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:01:35 2018 -0800
+
+    Merge pull request #19646 from pdvian/wip-22503-luminous
+    
+    luminous: mds: properly eval locks after importing inode
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 1b1ce1326e991b2e2d65826848fb594d238b6815
+Merge: 09baff4bbb 80de33b1ca
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 25 11:46:32 2018 -0800
+
+    Merge pull request #20055 from dzafman/wip-22724
+    
+    luminous: miscounting degraded objects and PG stuck in recovery_unfound
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 80de33b1ca215046290160fcc2781f3b29cb5c35
+Merge: 7522ddd978 09baff4bbb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 25 11:44:18 2018 -0800
+
+    Merge branch 'luminous' into wip-22724
+
+commit 09baff4bbbaa80d57ecb130b7921f19e9df03e8f
+Merge: d7874dc154 0040c7bfe1
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 25 11:36:22 2018 -0800
+
+    Merge pull request #20081 from dzafman/wip-recovering-luminous
+    
+    luminous: osd/ReplicatedPG.cc: recover_replicas: object added to missing set for backfill, but is not in recovering, error!
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d7874dc1543b26ea0ac86f93d70588ec2e394fd9
+Merge: 2ecc4c326e a170e67760
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:12:20 2018 -0800
+
+    Merge pull request #18008 from ukernel/luminous-21584
+    
+    luminous: mds: fix CDir::log_mark_dirty()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2ecc4c326edaef0ab95ee43e69ddbc37b428840a
+Merge: 6097c4f10e ddba907279
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:11:17 2018 -0800
+
+    Merge pull request #18782 from ukernel/luminous-21985
+    
+    luminous: mds: fix MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 definition
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 6097c4f10e5353ee73b40b7defb4c828aeb7b071
+Merge: b18280dbd5 ab103b9fe3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:09:39 2018 -0800
+
+    Merge pull request #18783 from ukernel/luminous-21975
+    
+    luminous: mds: trim 'N' log segments according to how many log segments are there
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b18280dbd572768b0b84edce034a2419ae181a0f
+Merge: 443bfb698f d73deb017c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:08:27 2018 -0800
+
+    Merge pull request #18831 from shinobu-x/wip-22076-luminous
+    
+    luminous: cephfs: "ceph tell mds" commands result in "File exists" errors on client admin socket
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 443bfb698f96c2e339a793a2950aab7dcd03b232
+Merge: 0e762253c6 efd4147fa8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:07:27 2018 -0800
+
+    Merge pull request #18835 from shinobu-x/wip-22074-luminous
+    
+    luminous: mds: don't check gid when none specified in auth caps
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 0e762253c6709631c984cfddd2a066bc87034255
+Merge: 1412d885aa 1410b5a5f5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:06:52 2018 -0800
+
+    Merge pull request #18869 from shinobu-x/wip-21952-luminous
+    
+    luminous: mds: no assertion on inode being purging in find_ino_peers()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 1412d885aab6a0c76566960e22109920f7580803
+Merge: dbc7cf62ea e485b89ffd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:06:17 2018 -0800
+
+    Merge pull request #18871 from shinobu-x/wip-21947-luminous
+    
+    luminous: mds: preserve order of requests during recovery of multimds cluster
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit dbc7cf62ea5b46775c1b142523e9cb01d65a9dad
+Merge: 4a0835d0a4 1ec93753a3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:05:28 2018 -0800
+
+    Merge pull request #18912 from shinobu-x/wip-22077-luminous
+    
+    luminous: src/mds/MDCache.cc: 7421: FAILED assert(CInode::count() == inode_map.size() + snap_inode_map.size())
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 4a0835d0a476aa7686ec449f2d8c7cb0b2ced4c3
+Merge: 6fa7c82bca bd3e7795ca
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:04:50 2018 -0800
+
+    Merge pull request #19055 from shinobu-x/wip-22192-luminous
+    
+    luminous: MDSMonitor: monitor gives constant "is now active in filesystem cephfs as rank" cluster log info messages
+    
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 6fa7c82bca25a0d28fd2104850aac3770d0243c7
+Merge: cdfe41dde0 c514d3d427
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:02:59 2018 -0800
+
+    Merge pull request #19326 from ukernel/luminous-22263
+    
+    luminous: mds: handle client reconnect gather race
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit cdfe41dde0e4ae2bc5d8d46a27595532db299a62
+Merge: dc68871f08 35bb3d4d17
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:00:27 2018 -0800
+
+    Merge pull request #19480 from smithfarm/wip-22407-luminous
+    
+    luminous: cephfs: client: implement delegation support in userland cephfs
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Jeff Layton <jlayton@redhat.com>
+
+commit dc68871f0850971c41434b1618e7662b436e4d18
+Merge: 75d7ce3cb3 8d6c2c2dea
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:58:46 2018 -0800
+
+    Merge pull request #19775 from ukernel/luminous-19578
+    
+    luminous: mds: track dirty dentries in separate list
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 75d7ce3cb3cea53340bd7738d1b9567f3c5e88d1
+Merge: b2e38487d7 e97399d2ee
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:58:18 2018 -0800
+
+    Merge pull request #19827 from pdvian/wip-22587-luminous
+    
+    luminous: mds: reduce debugging level for balancer messages
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit b2e38487d78327c2c092e786842028e61177edbc
+Merge: af9499dba0 55f4642be2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:57:05 2018 -0800
+
+    Merge pull request #20082 from smithfarm/wip-21525-luminous
+    
+    luminous: cephfs: client: dual client segfault with racing ceph_shutdown
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit af9499dba0e1822b8cd56f3d533fdc0be88e0065
+Merge: 832f228dc8 dc0e0250d1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:55:53 2018 -0800
+
+    Merge pull request #20085 from pdvian/wip-22763-luminous
+    
+    luminous: cephfs-journal-tool: add "set pool_id" option
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 832f228dc845129c95bbb688f70322ee636f7e74
+Merge: e7113bca11 4c211d0d6c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:55:19 2018 -0800
+
+    Merge pull request #20086 from pdvian/wip-22765-luminous
+    
+    luminous: client: avoid recursive lock in ll_get_vino
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 26b733758360f144008a4b48272bad45a7ac7673
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 5 09:58:04 2017 -0500
+
+    ceph-dencoder: moved RBD types outside of RGW preprocessor guard
+    
+    Fixes: http://tracker.ceph.com/issues/22321
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b00ac12f4045cd39ac9a436cf32d233d16bc3485)
+
+commit d4f5dc27803d0f356c15ee2a4de4c22dbf943fe3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 6 17:06:44 2017 +0800
+
+    mds: fix return value of MDCache::dump_cache
+    
+    previous commit "mds: track snap inodes through sorted map" makes
+    MDCache::dump_cache return 1 on success.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit f519fca9dd958121a289676edf5175fb8be9894f)
+    
+    Backport note: the original cause was commit
+    46c829d23aa5379a973d2330b0551a5b92598133. The fix was part of snapshot fixes
+    for Mimic in f519fca9dd958121a289676edf5175fb8be9894f which was not backported.
+
+commit fc791b980d2e9835e66fec8b39264be370d6a740
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 18 13:29:40 2017 -0400
+
+    qa/rgw: hadoop suite targets centos_latest
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a97d5bea42a40909aa4e9672505b9ad994cec184)
+    
+    Conflicts:
+        qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml (changes already backported
+            via 6b5f212b9170dbd99779527b41be233b9d40b5e4)
+
+commit 470c5f23a0b85fb8a71c98566ecb0f95e376ceef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 18 13:28:22 2017 -0400
+
+    qa/rgw: use overrides for hadoop version
+    
+    instead of having two separate tasks, add v27.yaml and v28.yaml with
+    overrides for hadoop_version
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4a953a16d7b80d343899de39a8cf55dc761847de)
+
+commit e7113bca11803206dd81b8c3a5e457c4941c845a
+Merge: 60c74451c3 22ff147734
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 09:11:10 2018 -0800
+
+    Merge pull request #18019 from ukernel/luminous-21091
+    
+    luminous: mds: fix StrayManager::truncate()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 60c74451c36d1e20a79bcb25edc58a070c74f934
+Merge: cea6b6e392 de05d2c819
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 12:03:54 2018 -0500
+
+    Merge pull request #19485 from dillaman/wip-21788-luminous
+    
+    luminous: librbd: refresh image after applying new/removing old metadata
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit cea6b6e392dccd3793d6db2eeed50dd9969c80e8
+Merge: fa97a4793b 369dcb36c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 11:44:39 2018 -0500
+
+    Merge pull request #18834 from shinobu-x/wip-22073-luminous
+    
+    luminous: [api] compare-and-write methods not properly advertised
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fa97a4793bfb8ee092fc10b4961652465f5d686b
+Merge: 2d0a2df9b8 818c354a43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 11:27:18 2018 -0500
+
+    Merge pull request #19058 from shinobu-x/wip-22190-luminous
+    
+    luminous: class rbd.Image discard----OSError: [errno 2147483648] error discarding region
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2d0a2df9b82412b1fd64ee8a6ce13a60ea59c56e
+Merge: 5bfd5cb595 2070fcf7d4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:18:13 2018 -0800
+
+    Merge pull request #19066 from pdvian/wip-22172-luminous
+    
+    luminous: [rbd-nbd] Fedora does not register resize events
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 5bfd5cb5953d2cc0524632b79ee07821b01b7fae
+Merge: 043a7a19e0 17ff1f1fd2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:17:38 2018 -0800
+
+    Merge pull request #19107 from pdvian/wip-22208-luminous
+    
+    luminous: rbd: disk usage on empty pool no longer returns an error message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 043a7a19e07b821693fce5d42317dbf9d825993e
+Merge: 588e575936 c3c3bb0bc9
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:17:01 2018 -0800
+
+    Merge pull request #19305 from pdvian/wip-21700-luminous
+    
+    luminous: rbd-mirror: Allow a different data-pool to be used on the secondary cluster
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 588e5759360fdeaf00ac6c3c901cf3d20a269191
+Merge: ab6b45d687 08a2358f85
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:16:26 2018 -0800
+
+    Merge pull request #19447 from smithfarm/wip-21646-luminous
+    
+    luminous: librbd: Image-meta should be dynamically refreshed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ab6b45d687b4a66e25833875c81d2f2dab8d2d95
+Merge: 43333e0d5f 5e8c4f83f4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:15:50 2018 -0800
+
+    Merge pull request #19479 from smithfarm/wip-22376-luminous
+    
+    luminous: rbd: Python RBD metadata_get does not work
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 43333e0d5fd2949441d99e9683e6399afd67e5e3
+Merge: 3ac673215f 235032ec64
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:14:53 2018 -0800
+
+    Merge pull request #19484 from dillaman/wip-21644-luminous
+    
+    luminous: rbd-mirror: sync image metadata when transfering remote image
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 3ac673215f3310c12762ad1bc3e32948b220e741
+Merge: 9560a8b2dc 1507015169
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:13:37 2018 -0800
+
+    Merge pull request #19503 from shinobu-x/wip-22395-luminous
+    
+    luminous: librbd: cannot clone all image-metas if we have more than 64 key/value pairs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9560a8b2dcb13cf774a94d97b7d3bc274f77c22c
+Merge: 9f48a65b50 feb4d7ba37
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:12:33 2018 -0800
+
+    Merge pull request #19504 from shinobu-x/wip-22393-luminous
+    
+    luminous: librbd: cannot copy all image-metas if we have more than 64 key/value pairs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9f48a65b50538c3925fc162f5e282c0a414c0a14
+Merge: 49d5128b05 2b9bbf605e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:11:57 2018 -0800
+
+    Merge pull request #19554 from shinobu-x/wip-22454-luminous
+    
+     luminous: cluster resource agent ocf:ceph:rbd - wrong permissions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49d5128b055930859d4d8716ea94100e5cbc3878
+Merge: 5051aa8646 3d0f4fa752
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:11:22 2018 -0800
+
+    Merge pull request #19625 from shinobu-x/wip-22497-luminous
+    
+     luminous: [rbd-mirror] new pools might not be detected
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5051aa8646f9240043261e184baa5fd05bcedd23
+Merge: 147d57b3a5 a15eb7df1e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:10:37 2018 -0800
+
+    Merge pull request #19800 from smithfarm/wip-21641-luminous
+    
+    luminous: rbd ls -l crashes with SIGABRT
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 52e59854c21d4e29e49a16c6d2f73470e68c73cd
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Sat Oct 21 23:35:47 2017 +0800
+
+    mgr: request daemon's metadata when receiving a report message from an unknown server
+    
+    Fixes: http://tracker.ceph.com/issues/21687
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit a39813837ca84434e57afa47138973268ff366d3)
+
+commit e97ad14cdc22c0da907fabc48d62698c94f6ba66
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Thu Aug 24 10:56:29 2017 +0800
+
+    mgr: kill MgrSession when MMgrReport come from daemon without metadata info
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 29080134339e5e64d50af1db9fe50df7ea55c1d0)
+
+commit 147d57b3a5e7c9dde57d8f7ccd4f947c92d31ad7
+Merge: a69f543b70 88c987cdb5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:09:49 2018 -0800
+
+    Merge pull request #19802 from smithfarm/wip-21690-luminous
+    
+    luminous: tests: rbd_mirror_helpers.sh request_resync_image function saves image id to wrong variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a69f543b70e554c88022dec020bd78d48da0445d
+Merge: 7ba84945e7 17aa16dc6b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:09:02 2018 -0800
+
+    Merge pull request #19853 from pdvian/wip-22593-luminous
+    
+    luminous: rbd: librbd: filter out potential race with image rename
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4318ac5e3f36b9885953f03d30c6b6a72b62d7e6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 22 17:18:43 2018 +0200
+
+    Revert "luminous: rgw: user stats increased after bucket reshard"
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cd9e6788cbb8ead60c82d15c574c5a00f4c1d222
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 4 13:06:52 2018 +0800
+
+    tools/ceph_monstore_tool: include mgrmap in initial paxos epoch
+    
+    before this change, the "mgr" map is not included paxos, thus why the
+    peon mon fails to have the mgr map after it is sync'ed with its leader.
+    and hence there is chance that ceph-mgr is unable to get a mgr map from
+    the monitor it is connected to.
+    
+    Fixes: http://tracker.ceph.com/issues/22266
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3f2a8bad4da7fda485eba7412d40b7b5758452d)
+
+commit b219b854b708c532bacb37ee8c8253c80586fef6
+Author: J. Eric Ivancich <ivancich@redhat.com>
+Date:   Wed Dec 6 16:36:28 2017 -0500
+
+    When a system object is created exclusively, do not distribute the
+    cache information to the other rados gateways and instead let them
+    lazily retrieve the information when they need it.
+    
+    Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
+    (cherry picked from commit 647ce3387312fc683660c1f3c7571c577379be1c)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7ba84945e7598333dca5f75dcbb7771f6b85fa13
+Merge: 9297805666 9a7e1f6a44
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:19:46 2018 -0800
+
+    Merge pull request #18722 from shinobu-x/wip-luminous-21946
+    
+    luminous: mds: set PRIO_USEFUL on num_sessions counter
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 92978056660f60e878a4a2aca377e2721c4d08e5
+Merge: ae19fa733f 948ca443fc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:18:41 2018 -0800
+
+    Merge pull request #18730 from shinobu-x/wip-luminous-22030
+    
+    luminous: List of filesystems does not get refreshed after a filesystem deletion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit ae19fa733f2fe6dc1feab1650851d559abb48791
+Merge: fbb14a0e7b fd956acfd6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:18:04 2018 -0800
+
+    Merge pull request #18754 from shinobu-x/wip-21636-luminous
+    
+    luminous: ceph-monstore-tool --readable mode doesn't understand FSMap, MgrMap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3ccf7e0bc6ca51d0d1e54f5bb7558f9f7aa199f3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 22 13:02:40 2018 +0800
+
+    osd/PG: print a space after __func__ in log message
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7bdf589db2010ea82aa33d60e9c54c14c7a78f0)
+
+commit 32c0eab8749e813b44ffe5ecefa3e59c61cb90a7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 22 13:08:48 2018 +0800
+
+    osd: update store with options after pg is created
+    
+    Fixes: http://tracker.ceph.com/issues/22419
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 43b2e512e61319f14f7390d063d66f7a2a568b4d)
+    
+    Conflicts:
+        src/osd/PG.cc (omitted immediately preceding write_if_dirty() call
+            because fa7818aefa92b4e9e067945c33f96d520e885f4e not backported to
+            luminous)
+
+commit fbb14a0e7bd76053f7347afb1c0460e4e984b768
+Merge: 06b9f828cf 7d6984b554
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 24 17:07:06 2018 +0100
+
+    Merge pull request #20053 from dillaman/wip-22740-luminous
+    
+    luminous: qa/suites/upgrade: disable broken kraken upgrade cls_rbd test
+    
+    Reviewed-By: Yuri Weinstein <yuriw@redhat.com>
+
+commit 06b9f828cfef1843e43852e2e95efcb6d0b6c4d7
+Merge: 1898fb6f38 767ad8518d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:05:13 2018 -0800
+
+    Merge pull request #18564 from kmroz/wip-21688-luminous
+    
+    luminous: Possible deadlock in 'list_children' when refresh is required
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1898fb6f38e5eaea16ff0b1d6007bfc605337136
+Merge: b0a841fd7c 1d4435f0cd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:04:38 2018 -0800
+
+    Merge pull request #18698 from shinobu-x/wip-luminous-21808
+    
+    luminous: rbd: fix crash during map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b0a841fd7cc356b073bc38f4b66d55e741e5de68
+Merge: 61ec339e6c c6bc756304
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:03:43 2018 -0800
+
+    Merge pull request #19038 from pdvian/wip-22183-luminous
+    
+    luminous: rgw: set sync_from_all as true when no value is seen
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 61ec339e6cba3935f531886684d25cd5708fb67a
+Merge: 2d93e37609 34450ed6c0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 24 15:23:54 2018 +0100
+
+    Merge pull request #19446 from smithfarm/wip-22404-luminous
+    
+    luminous: doc: crush_ruleset is invalid command in luminous
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 2d93e37609a757f2d3e6e4870a8054eb36626ed8
+Merge: c93fa099ee acccae56c4
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Jan 24 08:46:21 2018 -0500
+
+    Merge pull request #18747 from shinobu-x/wip-21794-luminous
+    
+    luminous: backoff causes out of order op
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 127236f6766c8539dbcf6a7957b4f3a04e7807db
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 16:13:33 2017 +0100
+
+    options, Mon: monitor the snap trim queues
+    
+    If new option "mon osd snap trim queue warn on" is set to value larger
+    than 0 (32768 by default), cluster will go into HEALTH_WARN state
+    once any pg has a snap trim queue larger than that value. This can
+    be used as an indicator of snaptrimmer not keeping up and disk space
+    not being reclaimed fast enough. Warning message will tell how many
+    pgs are affected.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 8412a65e0e6f610fb39430d8f65e561b6dbda13f)
+
+commit ca4413dee9a148c5ffda235793b02499a3975e88
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 15:23:55 2017 +0100
+
+    osd, PG: share the snaptrimq.size() within pg_stat_t
+    
+    That way it will be unnecessary to go through all pgs separately
+    to find pgs with excessively long snap trim queues. And we don't need
+    to share snap trim queues itself, which may be large by itself.
+    As snap trim queues tend to be short and anything above 50 000
+    I consider absurdly large, the snaptrimq_len is capped at 2^32 to
+    save space in pg_stat_t.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 022d246f17b31d309525395831c84e9856533adb)
+    
+    Conflicts:
+            src/osd/osd_types.cc
+         - needed a fix for snaptrimq_len that was placed after two other
+    new fields
+
+commit faeb81cd4dc24dc087877644ee27b937d50b4108
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 10:14:30 2017 +0100
+
+    osd/PrimaryLogPG: dump snap_trimq size
+    
+    One can just parse the snap_trimq string, but that's much more
+    expensive than just reading an unsigned int.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit dc7781cf17d11cb09067656cb25c0d710ab60d71)
+
+commit c93fa099ee173a0fea14bc8b2c36c2161fa0124b
+Merge: 6c5684c880 b97227d34e
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Jan 24 11:23:49 2018 +0800
+
+    Merge pull request #20087 from dzafman/wip-misc-luminous
+    
+    qa/standalone: Minor fixes for ceph_objectstore_tool.py test
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0040c7bfe1a47b73954c046cf460e2e96c66263a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 11 20:16:44 2017 -0700
+
+    test: Cleanup test-erasure-eio.sh code
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 69b5fc54feb59f8b0a26a3ca3e925980c91b5b75)
+
+commit 325eec1ba9c9b03d3ad6bcf56572be64d6c16c98
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 14:15:51 2017 -0700
+
+    test: Add replicated recovery/backfill test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c2572bee3c6256419e0b265e9e2829e7f3afb76d)
+
+commit 3068ea1e93f1806b74b78c636693f7afcff95aea
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 14:03:23 2017 -0700
+
+    osd: Better recovery/backfill restart for mark_unfound_lost delete/revert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 689bff354a6bfa1cf47b0d0a04fa9f1b2ef68f75)
+
+commit 25a901baf7f5d8130b7ed3d6913608105ccd4de6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 08:19:21 2017 -0700
+
+    osd, mon: Add new pg states recovery_unfound and backfill_unfound
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7f8b0ce9e681f727d8217e3ed74a1a3355f364f3)
+    
+    Conflicts:
+            src/osd/PG.h (trivial)
+            src/osd/osd_types.h (trivial)
+
+commit e40e2aa53b192a86abaa95fc74f05b5575e16d62
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 08:17:29 2017 -0700
+
+    osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bb2bcb95f51abc206e005e44ef383ee45b8f2209)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial, no MBackfillReserve::CANCEL)
+            src/osd/PG.h (trivial)
+    
+    fixup! osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions
+
+commit bb21f23731027df30844236573648bca300b523e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 30 10:05:14 2017 -0500
+
+    mgr: disconnect unregistered service daemon when report received
+    
+    This will allow the service daemon to reconnect and re-register
+    itself as a service daemon without requiring the mgr client to
+    subscribe to MgrStats and detect its removal.
+    
+    Fixes: http://tracker.ceph.com/issues/22286
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c97cca13ab066ed2306708986ff387d903b2486a)
+
+commit b97227d34e097e282aa51fe88a7549fe12caaad6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Sep 18 14:07:09 2017 -0700
+
+    test: Fix ceph-objectstore-tool usage check
+    
+    Caused by: c7b7a1f04f78fa62890c567d0ca53874c8d75eb7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0364ae104afc15effadfe968b5f60221704d933c)
+
+commit fe956597f2a3f2e9e0b4c2eb80547d4e575f6ee3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 23 18:16:32 2018 -0800
+
+    test: ceph_objectstore_tool.py bad --filestore option to vstart
+    
+    Luminous doesn't have --filestore option
+    
+    Caused by: fd7d53bacb10d5b2094f1f579c96899d8a0a4b4f
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 4c211d0d6ce76e592eb9b70445fde6f572fdb4f2
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Mon Jan 8 16:00:31 2018 +0800
+
+    client: avoid recursive lock in ll_get_vino
+    
+    Fixes: http://tracker.ceph.com/issues/22629
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit fdfbe40c3f16c14dc9c3352f3eea66d34138ee9d)
+
+commit dc0e0250d1d5a261b2ee4a29eaa33246d7a7ebcb
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 17:07:11 2018 +0800
+
+    cephfs-journal-tool: add usage help info for header set <field>
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 2b06bff950a78b15aca083b20eaa812b4214e8d4)
+
+commit 61a64511f99c2d8f30ab046fa72e518fba727058
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 16:54:40 2018 +0800
+
+    mds: add error log info
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit a7c2ecd6391d22c5312a9e568ff742cff252749e)
+
+commit 4d07eb8ba724c5bfcc7f8f12e6213282f497fb58
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 16:31:07 2018 +0800
+
+    cephfs-journal-tool: add "set pool_id" option
+    
+    Fixes: http://tracker.ceph.com/issues/22631
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit b3a5e313117028b23133f3e0d72e0b6b7159a178)
+
+commit 1b2b589a12c2e70c09c44e9f490819c833e1e69d
+Author: Konstantin Shalygin <k0ste@k0ste.ru>
+Date:   Wed Jan 17 13:40:05 2018 +0700
+
+    mgr: prometheus: added osd commit/apply latency metrics (#22718).
+    
+    Signed-off-by: Konstantin Shalygin <k0ste@k0ste.ru>
+    (cherry picked from commit 1adf4325455ccdbf03da63046c510bb041ac8520)
+
+commit 6c5684c880882e71e262aec4c8d7d4478edb6c98
+Merge: f7bfda8fce 7012cf4b60
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:10:49 2018 -0800
+
+    Merge pull request #18742 from shinobu-x/wip-21924-luminous
+    
+    luminous: ceph_test_objectstore fails ObjectStore/StoreTest.Synthetic/1 (filestore) buffer content mismatch
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f7bfda8fce31ebf186907400c50f8a13eb64088a
+Merge: 6bd8b907ae dfebcce5a4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:10:19 2018 -0800
+
+    Merge pull request #18744 from shinobu-x/wip-21922-luminous
+    
+    luminous: Objecter::C_ObjectOperation_sparse_read throws/catches exceptions on -ENOENT
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6bd8b907aec6414fb04cbfd24ed1d01d66331109
+Merge: f5e1d2b160 c281456346
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:09:41 2018 -0800
+
+    Merge pull request #18745 from shinobu-x/wip-21921-luminous
+    
+    luminous: Objecter::_send_op unnecessarily constructs costly hobject_t
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 55f4642be2bbdc758378cb58a2d369c14e37bfb1
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:39 2017 -0400
+
+    test: make the LibCephFS.ShutdownRacer test even more thrashy
+    
+    Have each thread do the startup and shutdown in a loop for a specified
+    number of times.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f877e365a42f8f34af6fc3382593ef09101a50d0)
+
+commit 963d6fb26565799a08e01577cf15adc0a2f1a956
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    lockdep: free_ids and lock_ref hashes must be truly global
+    
+    It's possible for the teardown of g_lockdep_ceph_ctx to occur, followed
+    by a new context being registered as the lockdep context. When that
+    occurs, we can end up reusing lock id's that were previously handed out
+    to consumers. We need for those IDs to be persistent across lockdep
+    enablement and disablement.
+    
+    Make both the free_ids table, and the lock_refs map persistent across
+    lockdep_unregister_ceph_context and lockdep_register_ceph_context cycles.
+    Entries in those tables will only be deleted by the destruction of the
+    associated mutex.
+    
+    When lockdep_unregister is called, do the refcounting like we normally
+    would, but only clear out the state when the lockid is registered
+    in the lock_names hash.
+    
+    Finally, we do still need to handle the case where g_lockdep has gone
+    false even when there are outstanding references after the decrement.
+    Only log the message if that's not the case.
+    
+    With this, we can deal with the case of multiple clients enabling and
+    disabling lockdep in an unsynchronized way.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 82fe4e94bcb706434f9215bc8405ff60770cf14f)
+
+commit 42bc9058a77ff427dd1dc8a4ba8098c54782a918
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    common: add a clear_g_str_vec() function to clear g_str_vec
+    
+    Prefix str_vec and str_vec_lock with "g_" to make it clear that they are
+    truly global values. Add a new clear_g_str_vec function to allow it to
+    be explicitly cleaned out by callers that need that functionality
+    (mostly testcase for now).
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f72ace0a5666db49bb390ff4ec6808e6e55a779f)
+
+commit f5e1d2b1608c17449002f9ab61e0021cf25b256f
+Merge: 2e0800bd68 5234ef2bca
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:08:59 2018 -0800
+
+    Merge pull request #18746 from shinobu-x/wip-21916-luminous
+    
+    luminous: msg/async/AsyncConnection.cc: 1835: FAILED assert(state == STATE_CLOSED)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8dec6175ab1167bab01af2594ac5d687072f3af2
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    common: make it safe to call env_to_vec multiple times
+    
+    After it has been called once and we have outstanding CephContexts with
+    pointers into str_vec, we can't call get_str_vec on it again.
+    
+    Add a static local mutex to protect access to str_vec.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 90e898de5f4b8d22f1a6d0e2aedf9e8c50cf72d5)
+
+commit eeb62a53c057bdb55e2da014dc7633312ab648b5
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: fix Mutex tests to disable lockdep properly
+    
+    ...and make g_lockdep a bool.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 0cd0bd778a6149d3e2fe657060e223be3675aed7)
+
+commit 864a2c5a2a40806067fa09a1390c3af14c8dd0ef
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:33 2017 -0400
+
+    client: test shutdown race
+    
+    Spawn threads that bring up a bunch of ceph_mounts with individual
+    CephContext objects, and then tear them down in parallel.
+    
+    Tracker: http://tracker.ceph.com/issues/20988
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 8252f3168a1bc90add7f7515c549e9eecffa54b7)
+
+commit 8db3ecd6c759251fa8105aeb676e74f9c9302027
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 12:22:52 2017 -0400
+
+    client: fix signed/unsigned comparison compiler warning
+    
+    The build says:
+    
+    src/client/Client.cc: In member function ‘void Client::trim_caps(MetaSession*, int)’:
+    src/client/Client.cc:4121:22: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+       if (s->caps.size() > max)
+           ~~~~~~~~~~~~~~~^~~~~
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit e057b6770b5545f327990cbaf17e7a391e3e8a50)
+
+commit 49ac68c3f95a0bdccb7a6c2f1706a3784ffc706a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: fix races with concurrent lockdep teardown
+    
+    If the cct is unregistered while other threads are flogging mutexes,
+    then we can hit all sorts of bugs. Ensure that we handle that
+    situation sanely, by checking that g_lockdep is still set after
+    we take the lockdep_mutex.
+    
+    Also, remove an assertion from lockdep_unregister, and just turn it into
+    an immediate return. It's possible to have a call to
+    lockdep_unregister_ceph_context, and then a call to
+    lockdep_register_ceph_context while a mutex is being held by another
+    task.
+    
+    In that case, it's possible the lock does not exist in the map
+    when we go to unregister it. That's not a bug though, just a natural
+    consequence of that series of actions.
+    
+    Tracker: http://tracker.ceph.com/issues/20988
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 75f41a95782a7ee83a243d91963e8d591402f8a6)
+
+commit b66ba62822bc7935d60a4424cedbfe7a845631c6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: don't take lockdep_mutex twice for new lock registrations
+    
+    We can do it under the same mutex, which should be more efficient.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 01863bb6fb62ea89aa3e21e43bf4dc4f3da9cfcb)
+
+commit 2e0800bd68234a32edb1f23c33402252fa6d200a
+Merge: ea771cbd92 8941606f5e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:07:07 2018 -0800
+
+    Merge pull request #18749 from shinobu-x/wip-21785-luminous
+    
+    luminous: OSDMap cache assert on shutdown
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ea771cbd926656db371ca6fa08ed172c76a88195
+Merge: b03323c80c b36bd48204
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:06:19 2018 -0800
+
+    Merge pull request #18753 from shinobu-x/wip-21697-luminous
+    
+    luminous: OSDService::recovery_need_sleep read+updated without locking
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b03323c80c4759bece12748268f16b8abe1197a6
+Merge: a0b489712d 5c9d212f94
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:52:13 2018 -0800
+
+    Merge pull request #18715 from shinobu-x/wip-luminous-21964
+    
+    luminous: ceph_test_cls_log failures related to cls_cxx_subop_version()
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a0b489712dbcc95c2f8cec8f8988acb532947385
+Merge: c8d526b8a8 b6f50a463a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:51:37 2018 -0800
+
+    Merge pull request #18732 from shinobu-x/wip-luminous-21875
+    
+    luminous: ceph-mgr spuriously reloading OSD metadata on map changes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c8d526b8a8b0eb5fa545c25210e98625232cb8a8
+Merge: 029594a080 f0914619a5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:50:34 2018 -0800
+
+    Merge pull request #18735 from shinobu-x/wip-22029-luminous
+    
+    luminous: restarting active ceph-mgr cause glitches in bps and iops metrics
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a09a42a4a850ee6f5e64fc0dc79995dbbfbdf13d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 2 13:51:17 2017 -0700
+
+    test: Test case that reproduces tracker 18162
+    
+    recover_replicas: object added to missing set for backfill, but is not in recovering, error!
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b9de5eec267627c8bc5ff0759ddea6c4a8aa7bce)
+
+commit 23f7c556d9ad903ad928e6193b50f5a61106429a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 3 18:32:20 2017 -0700
+
+    osd: Better handle failure to get enough EC shards to backfill
+    
+    Fixes: http://tracker.ceph.com/issues/18162
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6a02bfef3d44a13589c1a90bec29ff0ac64f97aa)
+
+commit 081fe72398633b66b93bc961cfd7d8599d5d0cba
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 20:35:27 2017 -0700
+
+    osd: For recovery get all possible shards to read on errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 390d12f71a1bd6e07f3516b1c73e467e9960725d)
+
+commit 5bed56a620f8c1c333f900f72d818cdec0f6381a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 27 14:29:25 2017 -0700
+
+    test: Use feature to get last array element
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 43e3206de2a20bf17ad9f2b16b7e486804097bf2)
+
+commit 62e9c05af9c7b1de44ebe4e055e84a419e0542b8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 17:48:28 2017 -0700
+
+    osd: Allow recovery to send additional reads
+    
+    For now it doesn't include non-acting OSDs
+    Added test for this case
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1235810c2ad08ccb7ef5946686eb2b85798f5bca)
+
+commit bfd0029910e33840e6ef5d87bb48b222c57bad03
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 14:49:17 2017 -0700
+
+    test: Allow modified options to existing setup functions
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f92aa6c82449152901a4663c523e9ba43363eca3)
+
+commit 39bb3cf8e688a4fd4bc66295c39c66554cb8e089
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 15:09:14 2017 -0700
+
+    osd: CLEANUP: Change getattr_maybe_cache() to reflect how it is being used
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 143111a112ec64724226f83ed285f66ee52204b2)
+
+commit a7a2de03ffcbb829c6ce6f95c5e20fb14ad1a3af
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 15:03:33 2017 -0700
+
+    test: Add a removal test for erasure code read
+    
+    Test feature: http://tracker.ceph.com/issues/14513
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50e08b0a5d7dc30a164313c1480b6cc829b6ec65)
+
+commit 7522ddd97845b8c2a15ad81a30b46a69e6f1444c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 16:07:48 2017 -0500
+
+    mon/PGMap: 'unclean' does not imply damaged
+    
+    Everything (that I can think of) that would lead to a PG being unclean is
+    already reported via another health message.  And there are cases where a
+    PG is unclean (e.g., because it is backfilling) where we are not degraded.
+    
+    Fix by ignoring this flag in the health checks.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89e4cb90af3c193fe436390b5e3288701e6fc4af)
+
+commit 67fbd3830f2681c49b65b0ac93eccd58c219114c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jan 23 10:49:31 2018 -0500
+
+    boost: include more specific asio headers
+    
+    when building with gcc5 and boost 1.66, this #include <boost/asio.hpp> fails to compile:
+    
+    In file included from build/boost/include/boost/asio/impl/write.hpp:25:0,
+                     from build/boost/include/boost/asio/write.hpp:927,
+                     from build/boost/include/boost/asio/buffered_write_stream.hpp:29,
+                     from build/boost/include/boost/asio/buffered_stream.hpp:22,
+                     from build/boost/include/boost/asio.hpp:41,
+                     from src/common/Graylog.h:7,
+                     from src/common/LogClient.cc:20:
+    build/boost/include/boost/asio/detail/consuming_buffers.hpp: In member function ‘boost::asio::detail::consuming_buffers<Buffer, Buffers, Buffer_Iterator>::prepared_buffers_type boost::asio::detail::consuming_buffers<Buffer, Buffers, Buffer_Iterator>::prepare(std::size_t)’:
+    build/boost/include/boost/asio/detail/consuming_buffers.hpp:105:50: error: parse error in template argument list
+         while (next != end && max_size > 0 && result.count < result.max_buffers)
+                                                      ^
+    src/CMakeFiles/common-objs.dir/build.make:135: recipe for target 'src/CMakeFiles/common-objs.dir/common/LogClient.cc.o' failed
+    
+    see also: https://svn.boost.org/trac10/ticket/13368
+    
+    this commit targets the luminous branch specifically because the issues
+    doesn't reproduce on master (as we require gcc7+)
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit 660abda21163bf4297b325ea2ac42a28661d5f62
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 17:55:34 2017 -0400
+
+    rgw: more documentation for bilog trim config
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4d4442168c15775f8346ebc6295a6de912250b8f)
+
+commit 2ed35c9f922409ad5e5e1e199687d25977b35cf6
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 16:54:59 2017 -0400
+
+    rgw: fix notify timeout for BucketTrimWatcher
+    
+    from seconds to msec
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ada2d108e627d519794e1a6c08af23cecbc5be79)
+
+commit 3f06276fc80696f693cc754b49351d5f16b76818
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 16:51:39 2017 -0400
+
+    rgw: curly brace style for bilog trim classes
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0c438eefcf0c99094efa0d98902cc4fca5957f0)
+
+commit 6fc4c7a12345b59cf3e14546006bbe7941eb98a3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 14:45:12 2017 -0400
+
+    rgw: BucketTrimWatcher checks handle in stop
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6e4d9735036a68575696e53b6732e4680a0226d6)
+
+commit 24f7fd2cf9efedf86b214c4d5f45113eed0a2f80
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 12:23:41 2017 -0400
+
+    rgw: hold cr reference in MetadataListCR callback
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3a1474031e4266100c5ccaf082756bdf0607a2be)
+
+commit 32ebc0e546903a3129defcee76541ed613bba195
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 12:23:08 2017 -0400
+
+    rgw: add comment for bucket in BucketCounter
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c6d946b3e1fbb10da43273e62c8584ffe3afbcdf)
+
+commit d1620fef15d5dff8ebdfe26af0b37f0f24a6046a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 11:08:24 2017 -0400
+
+    rgw: move shard marker helper into BucketIndexShardsManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4be7039964a2d17069b3604a4c994743145a013e)
+
+commit 668c2833ace4af4cb805e4d5ae38b22b6e03473a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 10 15:05:01 2017 -0400
+
+    qa/rgw: add kwargs for debug output
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fa8a0713de6b5b8ea9c9400f3ed467d87940612a)
+
+commit f1bcf84000c43772cc9e89040356fef6c3b464cc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 20 10:01:17 2017 -0400
+
+    test/rgw: add test_bucket_index_log_trim()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d9dc58cf2c6f64562843cc1d00df82d99379e640)
+
+commit 96baf2d00fe111b131f4fd9fa96e56442ee8b359
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 15:54:44 2017 -0400
+
+    rgw: add TrimComplete to watch/notify api
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 916f5995c92acd0c5ba66fde6031fed7cb91cb95)
+
+commit 7c430a08afaf6700cf4e64b7f7333e6ceafab2be
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 14:48:43 2017 -0400
+
+    rgw: RGWDataSyncSingleEntryCR calls BucketChangeObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c50d727b5df574e28d90cd99abe15db1742e4b1)
+    
+    Conflicts: sync tracing not backported
+            src/rgw/rgw_data_sync.cc
+            src/rgw/rgw_data_sync.h
+
+commit 07a122a3fda26c7b65ecf0879f24a8380568b47f
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Wed Jan 3 14:16:19 2018 +0300
+
+    common/throttle: start using 64-bit values
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit fa37ed1a48fd804ac199509bd78c470480ecbb22)
+    
+    Conflicts:
+            src/common/Throttle.cc : Resolved in _reset_max and put func
+            src/common/Throttle.h : Retained only count var changes
+
+commit e3fbaa92cedb101f79c7e0712017662f6d2ec1cf
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 19 13:05:27 2018 -0500
+
+    rgw_file: alternate fix deadlock on lru eviction
+    
+    This change is an alternate fix for two problems found and fixed
+    by Yao Zongyou <yaozongyou@vip.qq.com>.
+    
+    The deadlock can be avoided just by not taking it in the recycle
+    case, which invariantly holds the lock.
+    
+    The invalidation of the insert iterator by the recyle-path unlink
+    we'd like to handle as a condition in order to preserve the cached
+    insertion point optimization we get in the common case.  (The
+    original behavior was, indeed, incorrect.)
+    
+    Based on feedback from Yao, removed the RGWFileHandle dtor version
+    of the unlink check, which I think happened twice.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3cf0880f86b8f7911139c4e3d672cf47420c8f49)
+
+commit 35d2baf47b2ad29c961da2f861b4d71b65835b00
+Author: Tao Chen <chentao@umcloud.com>
+Date:   Fri Dec 1 16:42:44 2017 +0800
+
+    librgw: fix shutdown err with resources uncleaned
+    
+    Fixed: http://tracker.ceph.com/issues/22296
+    
+    Signed-off-by: Tao Chen <chentao@umcloud.com>
+    (cherry picked from commit 0fa2be059e649684a4fe15a456d564acde8e7831)
+
+commit 238ddccbbc4ae83e1cd49d2522bf1d43d62ae4f9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 9 16:11:14 2017 -0800
+
+    ceph-objectstore-tool: Fix set-size to clear data_digest if changing disk size
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7ca3ce93099de9412012b7a88121f9fced34f4b8)
+
+commit a0c0e637fe276aa1b6223522ca00a810e83d5084
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 10:38:29 2018 -0600
+
+    osd: only exit if *latest* map(s) say we are destroyed
+    
+    It's possible our current map is older, we were destroyed then, but in
+    newer maps our osd was recreated.  This happens when the oldest map after
+    a recreated osd happens to land on an epoch where the osd was marked
+    destroyed.
+    
+    Fix by only exiting if one of the newest maps says we are (still)
+    destroyed.
+    
+    Fixes: http://tracker.ceph.com/issues/22673
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e4d6466aab259eb0436989401b80b6c3cf87a7c)
+
+commit 21aef40b727ddfd7fba4e4c341ddd1cf2e21c404
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 14:09:14 2017 -0400
+
+    rgw: start BucketTrimManager in RGWRados
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 74aedaef0757f56a77671d6de59628538cfa0bbb)
+
+commit 0d86081fcd74c85d2834aa7e7279a1fdf26e0c04
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:27:53 2017 -0400
+
+    rgw: add BucketTrimInstanceCR
+    
+    fetches bucket sync status from each peer, calculates the min markers
+    for each shard, and trims the bilog shards. calls the TrimObserver on
+    success
+    
+    Fixes: http://tracker.ceph.com/issues/18229
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 35cf54e1059bd0a46e99fecc191c117f656c7b02)
+
+commit da126d8fe2637f09688e343edb7ddaf4d0f60b37
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:24:13 2017 -0400
+
+    rgw: add HTTPManager to BucketTrimManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 04400bc60b1cf9efbc49761cba8311e0a95dfc9e)
+
+commit 6b41f30976c292dce92a5a19f0949778c55aed56
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:17:46 2017 -0400
+
+    rgw: RGWGetBucketInstanceInfoCR takes rgw_bucket or metadata key
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 689badbe7fb468d4aec14e2f86bfc4cce9c103a2)
+
+commit aba2e0b64c0bc9e8dde20296a258b0d9850b5f72
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 14 16:50:47 2017 -0400
+
+    rgw: add RGWRadosBILogTrimCR
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0dea44dad3a572cd966d22dd345c1e226c073f6e)
+
+commit 1efe55f46ea366233a8aaeb73ea58add944dfea0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 14 16:51:13 2017 -0400
+
+    rgw: add RGWBucketInfo overload for BucketShard::init
+    
+    for use by callers that have already read the bucket instance info
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f0caa1ca8a1d502128fd45237c1e7c13118a9711)
+
+commit 8dc99bbd6931dc47a6197c07e6a6dde383edc0cb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 15:26:35 2017 -0400
+
+    rgw: add /admin/log rest api for bucket sync status
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fab595ab9bfe7f477ab03efee66d1a335d91f7cb)
+
+commit 001f55774cdef304f1d4ec5a7b4a424ad697d0ae
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:17:00 2017 -0400
+
+    rgw: add json decoders for bucket sync status
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 28c6ac62b877f0eafc94facb77255bfe81f48c38)
+
+commit 2c71e5cdda0eb2bfe871653a40b95627546f7612
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 12 12:48:56 2017 -0400
+
+    rgw: add rgw_bucket_sync_status() to bypass manager
+    
+    RGWBucketSyncStatusManager::init() is doing a lot of extra work that's
+    not needed to serve the rest api (spawning an http manager thread,
+    fetching the bucket instance info, etc)
+    
+    uses RGWShardCollectCR to limit the number of concurrent reads to 16
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 403d139c673180ce00eab420a2fdebdacf366dac)
+
+commit b26e997d0a57f9845ed1bc192e0b92c4307da07f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 6 12:59:03 2017 -0400
+
+    rgw: add 'radosgw-admin bilog autotrim'
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4f736f1ec74f74e8dc1ae4ffb5b71274351c66f0)
+
+commit f65ad4a3fa9637c5eb69422d95386dd7ed2a3888
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 12:48:47 2017 -0400
+
+    rgw: add configure_bucket_trim()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d29f96ae3e1b7d65b2f513340e8c2c42cf6de9f1)
+
+commit b72f8aefc277b607d6207eac1586de85e02a8f4d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:06:30 2017 -0400
+
+    rgw: BucketTrimManager implements BucketTrimObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7be4eab8a339e9e083352a44ad09272da717c73e)
+
+commit e7b3a636a76daa60dc12e1aee02b6a7ccfb9a779
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:40:58 2017 -0400
+
+    rgw: collect cold buckets for trim
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 06a22a134f9af92753fa206eb64025472ec94f40)
+
+commit d13f6a10fa3d5a92ed1a676cece453537af1a374
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:37:56 2017 -0400
+
+    rgw: add BucketTrimStatus
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2c07d7dd0e6c358bcdba409747ebf13f846a77e1)
+
+commit fbc356a2df53368c2fb72d34f6d59faae2eda7a0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 12:12:43 2017 -0400
+
+    rgw: add MetadataListCR to loop over bucket instances
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c9d50860b735809e343a0831ba3d346b0b37973c)
+
+commit f746e86bcade6edf1374578b3b64d2f5b9594b88
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:31:44 2017 -0400
+
+    rgw: add BucketTrimCR to spawn trim for active buckets
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 82c059530871af054c48de0e6a091b1f74f2eb12)
+
+commit 8434b7bd3e85352b24f8dff08f805556115c58e8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:22:47 2017 -0400
+
+    rgw: add BucketTrimPollCR for interval and lease logic
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 129fc99d5208279029ff1722d21f0ad24c37db62)
+
+commit 1d8dbaebca536cbf0341d4d0284dcbad191e64ad
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:29:55 2017 -0400
+
+    rgw: add TrimCounters api to BucketTrimWatcher
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5bcf109eac30780cfa9ae5d524d2bde638651f40)
+
+commit e34da7e6b35ed29560189e8ccf7d2c8f46861df5
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:26:01 2017 -0400
+
+    rgw: add BucketTrimWatcher to serve watch/notify apis
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f96d9a8e22195bfb3347b5add7a4385895d36f9c)
+
+commit fb7e2c0e6377a1f3bd3f32a0fe81a76e7ac53ea2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 10:57:41 2017 -0400
+
+    rgw: BucketTrimManager implements BucketChangeObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b4249cc432b5b74e5c8f545bdc7daddbc9e60d23)
+
+commit 92c9fd9521aac1fed4f2bf6d991a62280351c9a6
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Aug 24 10:01:36 2017 -0400
+
+    common: introduce BoundedKeyCounter and unit test
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e9a5ec9f64dd6cd163a855335b846181c3ac83d2)
+    
+    Conflicts:
+            src/test/common/CMakeLists.txt (add_ceph_unittest)
+
+commit 3aba3f5975c7abaac104732155f567e2ec2685b7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 10:51:57 2017 -0400
+
+    rgw: add skeleton for BucketTrimManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4309adb36be8dff737ab2196f59743c316b12bca)
+    
+    Conflicts:
+            src/rgw/CMakeLists.txt (sync tracing not backported)
+
+commit 82c1b7d75a2b4486e8c41440deafa6326641e73e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 30 16:19:36 2017 -0400
+
+    rgw: introduce RGWRadosNotifyCR for aio_notify
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 964d966969bf757570bffb650ebb4ef515a6592e)
+
+commit 7308044ac932ada012b22ce9b0ab65ce70d9bec9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 29 15:51:56 2017 -0400
+
+    rgw: MetadataManager interface takes const string refs
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b3a5c5a0569f95a5a755ea0dc03ec2c1ea4bdc4d)
+
+commit 7d6984b554158f8d5c3bae0223bfdf740ab9926d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 22 10:34:57 2018 -0500
+
+    qa/suites/upgrade: disable broken kraken upgrade cls_rbd test
+    
+    Fixes: https://tracker.ceph.com/issues/22740
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e0b85124193882fb1eb04fec0ab2d907523a2dde
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 19 13:05:27 2018 -0500
+
+    rgw_file: alternate fix deadlock on lru eviction
+    
+    This change is an alternate fix for two problems found and fixed
+    by Yao Zongyou <yaozongyou@vip.qq.com>.
+    
+    The deadlock can be avoided just by not taking it in the recycle
+    case, which invariantly holds the lock.
+    
+    The invalidation of the insert iterator by the recyle-path unlink
+    we'd like to handle as a condition in order to preserve the cached
+    insertion point optimization we get in the common case.  (The
+    original behavior was, indeed, incorrect.)
+    
+    Based on feedback from Yao, removed the RGWFileHandle dtor version
+    of the unlink check, which I think happened twice.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3cf0880f86b8f7911139c4e3d672cf47420c8f49)
+
+commit e8822fe18f68eefd8043e36dd803bda1c2aa4f0c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 16 10:44:29 2018 -0800
+
+    tests: recovery-unfound-found test needs to account for correct misplaced calculations
+    
+    The test expected HEALTH_OK when in a state with misplaced objects therefore HEALTH_WARN
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9f103f013c08bb215f58247f75d33f4f95bb3719)
+
+commit 40ecfe282e3f6584cb7f61ffbe99ea7cde7dfa18
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 10 13:30:41 2018 -0800
+
+    osd: Don't start recovery for missing until active pg state set
+    
+    I was seeing recovery hang when it is started before _activate_committed()
+    The state machine passes into "Active" but this transitions to activating
+    pg state and after commmitted into "active" pg state.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 64047e1bac2e775a06423a03cfab69b88462538c)
+
+commit 02660d92390b4e4ba3ce46ccc1eba7bf0644460b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 14:26:27 2017 -0600
+
+    osd/PG: restart recovery if NotRecovering and unfound found
+    
+    If we are in recovery_unfound state waiting for unfound objects, and we
+    find them, we need to restart the recovery reservation process so that we
+    can recover.  Do this by queueing DoRecover() event instead of calling
+    queue_recovery() (which won't do anything since we're not in
+    recoverying|backfilling pg states).
+    
+    Make the parent Active state ignore DoRecovery so that if we are already
+    in some phase of recovery/backfill the event gets ignored.  It is already
+    handled by the other important substates that care, like Clean (for
+    repair's benefit).
+    
+    I'm not sure why states like Activating are paying attention tot his vevent...
+    
+    Fixes: http://tracker.ceph.com/issues/22145
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4cfe31c63b519f2dce22f061c9951c302f6efb1e)
+    
+    Conflicts:
+            src/osd/PG.h (trivial, not all events exist in luminous)
+
+commit 673df31fe2fa6244b4327e646e47605aa482aa43
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 15:20:01 2017 -0600
+
+    qa/suites/rados: test for recovery_unfound bug
+    
+    See http://tracker.ceph.com/issues/22145
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 25b7965f8803ae68759973f4afe515d3da9c0f3f)
+
+commit f4f25becde3d83ea10a02e2fb19ec8bfe1d792c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 15:11:45 2017 -0600
+
+    osd/PG: document state hierarchy
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e2a75c91dd21f362d0b7fb6acf6a360beefd4168)
+
+commit 23a08fce7db373cfed8e69b379f82654110e436a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 12 09:22:37 2017 -0600
+
+    osd/PG: include primary in PG operator<< for ec pools
+    
+    Otherwise it is confusing!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d98f262c0894b11b77deb3a4bf494f20be486c3)
+
+commit 029594a080b5f02c47da4f08ab4b99c77ddb75a5
+Merge: 1a69f3ed7c 747d05c298
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 22 09:02:32 2018 -0600
+
+    Merge pull request #19257 from ifed01/wip-ifed-rocksdb-fix
+    
+    luminous: rocksdb: fixes early metadata spill over to slow device in
+
+commit ac064e1fc846a5a0db0062e6acfca3e0c623137c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 15:20:49 2017 -0500
+
+    ceph_test_objectstore: do not change model for 0-length zero
+    
+    We almost fixed this in the pr merged at 2dbbb351e2e0e3703880023bf51f55790f763e04
+    but missed this piece.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 390779d1aa9b46648e4f5390dc431c255c70385d)
+
+commit 6a691bb79db67ebe58e2d5179deaa1d7f17e391d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:32 2017 -0500
+
+    os/bluestore: 0-length zero should not change object size
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b588eaf2b0fdf06c94104d5a542bd571499f2b85)
+
+commit 445f9b52face0d44fb386bbb9a9704ad803322fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:15 2017 -0500
+
+    os/filestore: make 0-length zero avoid touching object length
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 979b7c9cd2e4919b8c7d5ed2418019310959f4bf)
+
+commit a146e49c51d3ed0d82a293bd5b809e4e97961732
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:28:49 2017 -0500
+
+    os/ObjectStore: 0-length zero does not change length of object
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ad1f4f10ff7bfe32d0a37361640fe5c65e56699)
+
+commit 2260f03b360e8226bf5875c5840dabc07080d174
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 11:23:39 2018 -0600
+
+    mon/OSDMonitor: squelch 'id' arg errors
+    
+    Several commands use 'id' but for other types (e.g., vector<int64_t>).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0580f0125bc014e5d97b8317a1bd53a3495f03bf)
+    
+    Conflicts:
+            src/mon/OSDMonitor.cc: pass g_ceph_context instead of cct to
+    cmd_getval() to be consistent with other callsites of this function. we
+    can do the s/g_ceph_context/cct/ in another commit, if it turns out to
+    be a critical fix.
+
+commit e07090345249bba6d3720bff62ab13641b1a9c38
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 11:22:55 2018 -0600
+
+    use stringsream::str(), not rdbuf()
+    
+    Fixes: http://tracker.ceph.com/issues/22715
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 918361e92518056bf89c3d19a03e9398c7a74b97)
+
+commit 1a69f3ed7c4b13940848a32bbf474f8b6b77f3bf
+Merge: 7a4b42185b 58d6d043b2
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Jan 19 15:10:19 2018 -0600
+
+    Merge pull request #19071 from smithfarm/wip-22181-luminous
+    
+    luminous: rgw segfaults after running radosgw-admin data sync init
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 7a4b42185b8948fc3d078ace811cb3fc9178a396
+Merge: 6d56c2fdef 23b3c9de3d
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Jan 19 15:09:45 2018 -0600
+
+    Merge pull request #19077 from smithfarm/wip-22171-luminous
+    
+    luminous: rgw: log keystone errors at a higher level
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6d56c2fdef336f8ce1034a67cd18758690055827
+Merge: 63a05442c3 5b9f1281a3
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:08:03 2018 -0500
+
+    Merge pull request #19810 from adamemerson/wip-luminous-21901
+    
+    luminous: rgw: bucket policy evaluation logical error
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 63a05442c33d191786ff781dba8fd47b5297b0c2
+Merge: e24edea8d9 9db61c8b14
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:06:08 2018 -0500
+
+    Merge pull request #19847 from adamemerson/wip-22541-luminous
+    
+    luminous: rgw: put bucket policy panics RGW process
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e24edea8d94639f6fb2da5c75dea397b2e06e258
+Merge: ca5ae981f6 c49417d8b0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:03:58 2018 -0800
+
+    Merge pull request #19785 from linuxbox2/luminous-22410
+    
+    luminous: rgw: fix chained cache invalidation to prevent cache size growth
+    
+    Reviewed-by: Adam Emerson <aemerson@redhat.com>
+
+commit ca5ae981f65da0a2f8213cd3ea52cb37db432e96
+Merge: f2c5146f40 34864fe7ba
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:02:34 2018 -0800
+
+    Merge pull request #19788 from adamemerson/wip-luminous-cache
+    
+    luminous: rgw: Random 500 errors in Swift PutObject (needs cache fixes)
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit f2c5146f409ab4f51b17ae311ad8090ea3ccf5b7
+Merge: e002a82db8 b9bf9b1255
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:01:40 2018 -0500
+
+    Merge pull request #19053 from shinobu-x/wip-22187-luminous
+    
+    luminous: rgw: add cors header rule check in cors option request
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e002a82db83f20812f6aa1c70165f1275c4efff7
+Merge: ccd4b53800 5bf8d71001
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:01:10 2018 -0800
+
+    Merge pull request #19819 from adamemerson/wip-backport-22601
+    
+    luminous: rgw: S3 API Policy Conditions IpAddress and NotIpAddress do not work
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit ccd4b538003acee14dfb9f72f5c39a38ab6ec12b
+Merge: 21ff007c65 274fef45c2
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:00:56 2018 -0500
+
+    Merge pull request #19050 from shinobu-x/wip-22184-luminous
+    
+    luminous: Dynamic bucket indexing, resharding and tenants seems to be broken
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 21ff007c656327fe4bfcdb723bb97fdb64e26985
+Merge: 599d6a1681 90b56a2903
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:00:38 2018 -0500
+
+    Merge pull request #19085 from shinobu-x/wip-22215-luminous
+    
+    luminous: rgw: bucket index object not deleted after radosgw-admin bucket rm --purge-objects --bypass-gc
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 599d6a16814965d721259bac94ed92fdb1964ce5
+Merge: 7a686ddb09 a88e48e7d9
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:54:24 2018 -0500
+
+    Merge pull request #18870 from shinobu-x/wip-21949-luminous
+    
+    luminous: rgw: null instance mtime incorrect when enable versioning
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7a686ddb093b4198ba44501ab0d5993d868c907f
+Merge: c143de468a 860625046d
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:54:05 2018 -0500
+
+    Merge pull request #18764 from smithfarm/wip-22017-luminous
+    
+    luminous: rgw: Segmentation fault when starting radosgw after reverting .rgw.root
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c143de468a9a279e95e37d65354733bdb3895a20
+Merge: 92a1908c88 cb38378e5d
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:53 2018 -0500
+
+    Merge pull request #18765 from smithfarm/wip-22024-luminous
+    
+    luminous: RGWCrashError: RGW will crash if a putting lc config request does not include an ID tag in the request xml
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 92a1908c88bb862c13363122851acc5739ea6bc1
+Merge: 11a3dae295 ed87de0622
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:40 2018 -0500
+
+    Merge pull request #18766 from smithfarm/wip-22021-luminous
+    
+    luminous: rgw: modify s3 type subuser access permission fail
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 11a3dae29509ceb00bcc93da94b9873cc457229f
+Merge: 7ae70b8b6a dbd70f0b82
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:28 2018 -0500
+
+    Merge pull request #18867 from shinobu-x/wip-22027-luminous
+    
+    luminous: multisite: destination zone does not compress synced objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7ae70b8b6a3efc89895e8fdf74e929a1d41eaf1c
+Merge: 262bd71520 2dc009e68e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:48:23 2018 -0800
+
+    Merge pull request #18868 from shinobu-x/wip-22026-luminous
+    
+    luminous: Policy parser may or may not dereference uninitialized boost::optional sometimes
+    
+    Reviewed-by: Adam Emerson <aemerson@redhat.com>
+
+commit 262bd71520169871cbf5cf050bf4dd6eeee4f7da
+Merge: 14ca29d5cf eb0c60f7d7
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:46:58 2018 -0500
+
+    Merge pull request #19538 from shinobu-x/wip-22434-luminous
+    
+    luminous: rgw: user stats increased after bucket reshard
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 14ca29d5cfc6c088c39bb47b26d2a9533f552f48
+Merge: b26aeac2c4 4085e87b84
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:46:21 2018 -0500
+
+    Merge pull request #19489 from dplyakin/luminous-get-website-error-code
+    
+    luminous: rgw: fix GET website response error code
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit b26aeac2c462e47d91f9847a25e79ae5e6b9b0e2
+Merge: 370abb546d 05b60db8ef
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:44:55 2018 -0800
+
+    Merge pull request #19090 from pdvian/wip-22179-luminous
+    
+    luminous: rgw: Fix swift object expiry not deleting objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 370abb546d871d8cb72f61bc7d74a547b90e4112
+Merge: 85558065d4 b013f7fdb5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:43:07 2018 -0800
+
+    Merge pull request #19434 from cbodley/wip-luminous-pr-16807
+    
+    luminous: rgw: revert PR #16807
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 85558065d46dde2d1d26b2b159b09e77f73de432
+Merge: eaedb0f06c 27a1a7f6fa
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:41:22 2018 -0800
+
+    Merge pull request #19502 from shinobu-x/wip-22397-luminous
+    
+    luminous: rgw: radosgw-admin reshard command argument error.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit eaedb0f06cbf1287c636cb0a3394179edd049506
+Merge: ad11a8e9b2 72de713bf6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:40:45 2018 -0800
+
+    Merge pull request #19506 from shinobu-x/wip-22388-luminous
+    
+    luminous: rgw: 501 is returned When init multipart is using V4 signature and chunk encoding
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzyns@redhat.com
+
+commit e066b69178b7b9a98d8e88bed9bd41cdec08c4c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 28 17:51:52 2017 -0800
+
+    test: Verify stat calculations during backfill
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 88ce0c1a91178c25c75e9b627c2a2b64b0969648)
+
+commit 98cf928941303ed96b9b8e51e16c3e99b12812e3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Nov 17 17:01:56 2017 -0800
+
+    test: Verify stat calculations during recovery
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f5af1af6d3727f915dfcf7ea8a243b4a52b43859)
+
+commit f7bc7475b072724104003f12f439fa7502754598
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 11 14:22:54 2018 -0800
+
+    ceph-helpers.sh: Add flush_pg_stats() to wait_for_clean() to make it reliable
+    
+    osd-scrub-repair.sh: Fixes for omap keys landing on different OSDs due to flush
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit aeba36a660cd4ab59669ee15bf92cb9640df0371)
+
+commit 82b1a41e10641dc9e8ccc90f7dd2f00e366eec92
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 4 13:02:04 2017 -0800
+
+    qa: Ignore degraded PGs when injecting random eio errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c77941f593755f79e4a800545d7ff437bbe0562d)
+
+commit 7065a781a42c0ab94155f92b0719e429105a8072
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 18 10:16:53 2017 -0800
+
+    osd: Improve the way insufficient targets is handled to be compatible with EC
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 91d1487ecfb497905a20556baf699ce8dde1cc30)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial)
+
+commit 6a652944e376e9f319996a87d0c7a253eed019c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 31 18:15:53 2017 -0700
+
+    osd: Improve pg degraded state setting based on _update_calc_stats() degraded count
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 74f9e70903723bdad190bc3f71a2ca2109bfe4f7)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial, ignore change to code not present)
+            src/osd/PG.h (trivial, white space diff)
+
+commit 1afeb8bc5e3291d517beed781b8698bba3280b2c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 26 15:36:54 2017 -0700
+
+    osd: Handling when recovery sources have missing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b769b98686e98b16a74318346ddf0cee88ac7476)
+
+commit b683ba9fe9d01b1010c2fbbb48a4d654a5831f5b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 25 21:44:14 2017 -0700
+
+    osd: Base pg degraded state on num_degraded_objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5ffde0002f605401fb99535ff86afa0eb0404c67)
+
+commit 15bd330eac78e00d20b0dab6a5ac3dbc1fc9897d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 25 09:37:00 2017 -0700
+
+    osd: Rewrite _update_calc_stats() to make it cleaner and more accurate
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e1075f107372c97580b984c74666027fc2e71daf)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial)
+
+commit 9142833bec8adf3d4e0355c796d5c5f5c5fa115c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 14 13:00:06 2017 -0800
+
+    osd: cleanup: Remove unused const vars
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9a66a6f517ea0c9c7963b2f3127f5efb78aea346)
+
+commit 50cdacb9b9bfc658b8825b269cafdb48210b5003
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 26 10:36:32 2017 -0700
+
+    osd: cleanup: Fix log message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 01ac714aa3522dde130b48d5901e72533aa423b1)
+
+commit ad11a8e9b2d4e53389fe91748e75baacd47dbbf5
+Merge: abb91dc4f1 ad9db7eea8
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 18 10:02:00 2018 +0000
+
+    Merge pull request #19929 from zmc/wip-luminous-prom-pg_deep
+    
+    luminous: mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr pro…
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit abb91dc4f194696ea2b91a37d159ece5ccbbc9ce
+Merge: 2b7410003e 124b4e3465
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 17 20:13:36 2018 -0500
+
+    Merge pull request #19943 from dillaman/wip-22676-luminous
+    
+    luminous: cls/rbd: remove incompatible group features from partial implementation
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 124b4e3465ddf8751011dbd920851e251e1d2c57
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 17 17:21:47 2018 -0500
+
+    qa/suites: filter out cls_rbd tests against unsupported methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2b7410003e332e8dd87ee5a367e712439d0744be
+Merge: cc6a3561cc 9f9db135a0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 17 23:12:37 2018 +0100
+
+    Merge pull request #18865 from shinobu-x/wip-21631-luminous
+    
+    luminous: doc: remove region from "INSTALL CEPH OBJECT GATEWAY"
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit cadd8426b358647505e950017d9a146e6f7f6696
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Jan 13 16:15:48 2018 -0500
+
+    cls/rbd: remove incompatible group features from partial implementation
+    
+    Fixes: https://tracker.ceph.com/issues/22676
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5353569eea13e1ef6ddd752c8ebfdc367c6e7245
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 10 13:04:55 2018 +0800
+
+    mds: handle 'inode gets queued for recovery multiple times'
+    
+    Fixes: http://tracker.ceph.com/issues/22647
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 06bbf939ac7af9969765d50fad275c24b204a188)
+
+commit 4063b807daa9e02be7fc9829f2338d13364baeee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 10 11:52:35 2018 +0800
+
+    mds: use elist to track recover queue items
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 74f2a87887e81dc1d5895976982efb51b0a5e97e)
+    
+     Conflicts:
+            src/mds/CInode.cc
+
+commit 91be5317da05dea8f8cf9695b2059a72c4c151c4
+Author: Xuehan Xu <xuxuehan@360.cn>
+Date:   Sat Jan 6 10:40:33 2018 +0800
+
+    common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    As libstdc++ earlier than version 5 implement the list::size() as a O(n) operation,
+    this should be needed to avoid regression of various ceph component's performance.
+    
+    Signed-off-by: Xuehan Xu <xuxuehan@360.cn>
+    (cherry picked from commit 7e0a27a5c8b7d12d378de4d700ed7a95af7860c3)
+
+commit dc96e3c1f876ecb7fccb3238f8641d79b0dcc51d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jan 9 16:48:18 2018 -0800
+
+    qa: increase osd count for ec testing
+    
+    Missing in d0732fc96fbc6849dd51b391d85f765c74cfb593.
+    
+    Fixes: http://tracker.ceph.com/issues/22646
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit f8c275b8b6c59c2f269990ec81e503a54c0dc7c1)
+
+commit 1142dacc6467af2643c317fa74fdeed5064c0565
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jan 9 16:47:25 2018 -0800
+
+    qa: add missing openstack configs
+    
+    First introduced in: ec6fb28eaf8e2db327e4afc115879a40c7664e07
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e8a0f1c554cad7529e49dcacfd8506c7e33f949a)
+
+commit 8e3edae0c8b472a5fc9db119173de21a3374dffd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 9 09:40:06 2018 -0600
+
+    debian: add -dbg packages for python-{rados,rgw,rbd,cephfs}
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b5bd43323fd96aa975e0500261b65a46067b68fa)
+
+commit 98eaf4435c92ae158a45df2dfef5c70506548fed
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 9 08:37:15 2018 -0600
+
+    debian/rules: strip ceph-base libraries
+    
+    This includes rados classes and ec plugins.
+    
+    Fixes: http://tracker.ceph.com/issues/22640
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c3a7895178751051f8a0f2d0afae577cfe7badbd)
+
+commit cc76ab1eefab661b794d2852836441fe6e7a761b
+Author: YunfeiGuan <yunfeiguan@xtaotech.com>
+Date:   Wed Jan 3 11:43:07 2018 +0800
+
+    ceph-fuse: ::rmdir() uses a deleted memory structure of dentry leads a core
+    
+    we should add the dentry ref immediately after "get_or_create" in
+    case of the ref be put to zero.
+    
+    Fixes: http://tracker.ceph.com/issues/22536
+    Signed-off-by: YunfeiGuan <yunfeiguan@xtaotech.com>
+    (cherry picked from commit 65fcccc04a6239a9c7cbf0192de14f963ebf1079)
+
+commit 44643cae8da8068c3b9ab086b9dfc367f653e043
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Wed Jan 3 16:48:51 2018 +0300
+
+    objectstore/store_test: fix lack of flush prior to collection_empty() call
+    
+    Fixes: http://tracker.ceph.com/issues/22409
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit f669fcdfd5e5facd1b702ed113cac6f1d56bef5b)
+
+commit d2a68571a834e0c8ee8495fd82cde5d85b872098
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 3 08:37:12 2018 -0600
+
+    mgr/DaemonServer: fix error string
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78b7f10d298070515418f5a71931d6235498c87f)
+
+commit adc47b95958b96322a649e6e2f5d6d4e2d58701e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 3 08:37:02 2018 -0600
+
+    qa/tasks/ceph_manager: tolerate failure to force backfill/recoery
+    
+    The pool may have been deleted out from underneath us.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8651e15c93ce419529b82c4c19feef1ab2b647c8)
+
+commit 80ea9ed3b30bcea3e1b561a160525280238af4bf
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Dec 18 21:29:11 2017 -0800
+
+    qa: check pool full flags
+    
+    Cluster-wide flag removed in b4ca5ae462c6f12ca48b787529938862646282cd.
+    
+    Fixes: http://tracker.ceph.com/issues/22475
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 6e046dfc90e0a119ceb13935dc6d15efb2845184)
+
+commit 5fd01ca888b9f8b0077d07f18e62e8a4d108a38e
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Wed Jan 3 21:35:16 2018 +0800
+
+    mds: fix dump last_sent
+    
+    issue: http://tracker.ceph.com/issues/22562
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 3d3df18bdb21aff25d1f1111718eccbb6640b5a6)
+
+commit e56106cfc1056e2479919ab8cc721cbf75791ca5
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 15 13:57:51 2018 -0800
+
+    create 4 lv's by default for ceph-volume tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit ad9db7eea86f3518faa54820ac62c6e34dc7cc7c
+Author: Peter Woodman <peter@shortbus.org>
+Date:   Sat Nov 11 16:32:37 2017 -0800
+
+    mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr prometheus plugin
+    
+    without this the endpoint throws 500s when any PG is doing a deep scrub.
+    
+    Signed-off-by: Peter Woodman <peter@shortbus.org>
+    Fixes: http://tracker.ceph.com/issues/22116
+    (cherry picked from commit 8c93c0eeaff406af0878916ef5b9ce958896d586)
+
+commit a1ca7f7d93acd5f7fa58b25371b9bcfb7855ef25
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Mon Nov 13 14:42:56 2017 +0100
+
+    pybind/mgr/prometheus: don't crash when encountering an unknown PG state
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 67a1b66b06bf74274c2cf1d4b5c900f2c56a990d)
+
+commit cc6a3561cc8ec65e05451073d0dfdd6efb4b9700
+Merge: 905b73442b 1961e32ce9
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 15 14:42:55 2018 +0000
+
+    Merge pull request #19925 from jcsp/wip-dashboard-trim-luminous2
+    
+    luminous: mgr: remove unused static files from dashboard module
+    
+    Reviewed-by: Sebastien Han <shan@redhat.com>
+
+commit e9f5612bd789ca17a01c6e51794e2f65e40ecaed
+Author: huangjun <huangjun@xsky.com>
+Date:   Mon Oct 9 22:05:21 2017 +0800
+
+    qa/standalone/osd/osd-mark-down: create pool to get updated osdmap faster
+    
+    Mon send osdmap to random osds after we mark osd down, the down osd
+    may use more than $sleep time to get updated osdmap if there is no
+    osd ping between osds. So create pool after setup cluster.
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit ee618a38a9ed06b3ea4d02e46cdeae6afb376b82)
+
+commit d5e2e43de82cb1bf02b32bfea5a8e4c9bb495494
+Author: Xuehan Xu <xuxuehan@360.cn>
+Date:   Sat Jan 6 10:40:33 2018 +0800
+
+    common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    As libstdc++ earlier than version 5 implement the list::size() as a O(n) operation,
+    this should be needed to avoid regression of various ceph component's performance.
+    
+    Signed-off-by: Xuehan Xu <xuxuehan@360.cn>
+    (cherry picked from commit 7e0a27a5c8b7d12d378de4d700ed7a95af7860c3)
+
+commit 8b13643b324fe1be39cb45777c11652da2a80c61
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 10 21:39:28 2018 -0500
+
+    config: lower default omap entries recovered at once
+    
+    For large omap DBs, reading 64k leads to heartbeat timeouts.  There
+    are numerous callchains leading to this recovery step, many of which
+    do not have heartbeat handles, so for an easily backported version
+    just change the default number of entries read. DBs approaching 100GB
+    may require an even lower setting, but this should be good enough for
+    most clusters, without sacrificing recovery speed.
+    
+    Fixes: http://tracker.ceph.com/issues/21897
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 72c2076f2c14778982fb944ffade3f071a727d1a)
+
+commit 1961e32ce997917e819b40b7f8ec21fd14d7d174
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:11:07 2017 +0100
+
+    mgr/dashboard: remove unneeded bits of AdminLTE
+    
+    AdminLTE is delivered in a handy all-in format,
+    but we don't need all this stuff at runtime.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 38df5150ee026c6dc9ec763f757713c7e4d99cdb)
+
+commit 492c170a6874f0fe12bef21ae702f5f6314b2107
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:15:04 2017 +0100
+
+    mgr/dashboard: remove non-minified bootstrap
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5ce51ebbf047e373eac624e49e2d2bcfb46f09ff)
+
+commit f081fde52e523f10f2e57133c3a86e6fa9596714
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:54:17 2017 +0100
+
+    mgr/dashboard: remove un-minified datatables source
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 888f0f934e59fe28fdd97bca044702ce445bfe2e)
+
+commit 47194cdcca4e065616f74415de62e7f562eb7eb0
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 06:10:31 2017 -0400
+
+    mgr/dashboard: don't include remote js/css
+    
+    This will help anyone running away from the internet.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 199d7ff26959961f200b86062f6290c9c17cba9a)
+
+commit 8e4cc4f9991bc2ee5fcf01d0b1f6ff9334c6c853
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 06:13:45 2017 -0400
+
+    mgr/dashboard: re-arrange static files
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85a87da7aa46266abd8d4c42218bcbccce386316)
+
+commit 905b73442b8b16a898fcc8168a8ef4da792424bc
+Merge: 0e57572c31 723b0f2050
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Jan 11 14:31:26 2018 +0800
+
+    Merge pull request #19880 from jcsp/wip-doc-dashboard-standby-luminous
+    
+    doc: update mgr/dashboard doc about standbys
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f59380e1598981468cdafe9f575bace41c9a61b3
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Sat Dec 30 02:05:13 2017 +0300
+
+    os/bluestore: add asserts for fsck's used_blocks bitmap access as
+    boost doesn't always handle out-of-range access properly.
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit 121c40286a60a63cda32c7111aaade2043ff18df)
+
+commit f919061a53ef8a7e4e01e841428f4bbb02e640d7
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Fri Dec 29 20:59:16 2017 +0300
+
+    os/bluestore: refactor FreeListManager to get clearer view on the number
+    of alloc units it tracks.
+    This also fixes out-of-range access for fsck's used_blocks bitmap that
+    might happen when checking stores created prior to v12.2.2
+    Fixes http://tracker.ceph.com/issues/22535
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit f64c236f278732b0fa211cd9e93c4f9d5a77a356)
+    
+    Conflicts:
+            src/os/bluestore/BlueStore.cc: Removed argument 'what' from apply.
+
+commit 723b0f2050dcb06f2772c7b1f3e5f94e68a4b2e0
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 9 22:23:33 2018 +0000
+
+    doc: update mgr/dashboard doc about standbys
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fa14812cbb584b61cb2ce687ae4c14225eecd9fe)
+
+commit 2adc133b86a50a6c79d59f81b245b6bfd5cf1e14
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 31 18:31:13 2017 -0400
+
+    RGW: S3 POST policy should not require Content-Type
+    
+    In the current implementation, putting a key in the acting
+    RGWPolicyEnv makes it required in RGWPolicyEnv::match_policy_vars.
+    
+    I'm not sure this is the intent, but in any case, add it to the env
+    only if sent.
+    
+    Fixes: http://tracker.ceph.com/issues/20201
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit af29276c32f7009a78bd8b90d8f7d19026859c74)
+
+commit 6cc4a33933109bd9cd333034a538b2cc3621f56a
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Dec 19 11:41:08 2017 +0000
+
+    packaging: explicit jinja2 dependency for dashboard
+    
+    We were getting this via python-flask, when mgr was
+    installed on nodes with ceph-mon, but for correctness
+    in general we should depend on it from ceph-mgr (it
+    is imported by the dashboard module).
+    
+    Fixes: http://tracker.ceph.com/issues/22457
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 87399bea8321a7a9ab554085be3a3d5e0563f27f)
+
+commit 0e57572c311df7597c0e7c235dfe195ae04b7cfd
+Merge: 0706e9b842 2042881c67
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 9 11:13:04 2018 +0000
+
+    Merge pull request #19844 from jcsp/wip-22615
+    
+    luminous: mgr/dashboard: Fix PG status coloring
+    
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0706e9b842cb6027f70adea8d8cc996763fe642b
+Merge: 3de49aa945 0f0df7db74
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 9 09:16:58 2018 +0000
+
+    Merge pull request #19858 from joscollin/wip-luminous-doc-misc-fixes
+    
+    luminous: doc: misc fixes for CephFS best practices
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 0f0df7db74069fef8b526c849002f82f9d8087dd
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 11:51:00 2018 +0530
+
+    doc: misc fixes
+    
+    misc fixes for best-practices.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit ecacd1078db5020e6d65436f9d56cc9ed0016666)
+
+commit 2e7c40232ff12393a1afe10b8ff1669f13047b14
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 11:24:23 2018 +0530
+
+    doc: fix heading
+    
+    Fixed 'Which kernel version?' heading not having brown color background.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit a068fd842f431a54f9db2d437a71cdc11907a8e1)
+
+commit 17aa16dc6b993373539cecc2a4e110c0455dcbb6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 20 16:55:06 2017 -0500
+
+    librbd: filter out potential race with image rename
+    
+    Fixes: http://tracker.ceph.com/issues/18435
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 001d2fc35c170f17e706aeae541fb86dac960319)
+
+commit 9d617b4ecceda094040f4fe4db490588e6684a18
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 7 16:40:04 2017 -0600
+
+    mon/LogMonitor: add mon_cluster_log_to_stderr
+    
+    Optionally send cluster log messages to stderr (prefixed by the
+    channel).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 49b327e8657695a359ea63e427b5de2e4525f480)
+
+commit acc76d713f11f52c39f6ba5868ca1e502c1985b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 7 16:39:30 2017 -0600
+
+    log: add log_stderr_prefix option
+    
+    Allows you to set a prefix for debug log messages send to stderr (e.g.,
+    "debug ").
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c142ae3f1530067f35b8a1e1bac9ddeea4d81c09)
+
+commit 473f12d4fd4c0ee747f7a1822f513ad1f87cdfb0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sun Nov 26 15:46:26 2017 -0500
+
+    cmake: add WITH_BOOST_CONTEXT option
+    
+    adds a more specific option for this boost::context dependency, which was
+    previously only used by the radosgw beast frontend. see
+    http://tracker.ceph.com/issues/20048 for more background
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b589b73c70bc4c644b7a040d9aee5083881254b9)
+    
+    Conflicts:
+            src/include/config-h.in.cmake (missing HAVE_GETENTROPY from
+                      https://github.com/ceph/ceph/pull/17972)
+
+commit b1ad4b1e9e27d2b6e0cfe2b05e7693aa4c298eb5
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 22:23:53 2017 -0500
+
+    cmake: remove Beast submodule and include path
+    
+    the beast library is included in boost 1.66
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d85f42dd697365ad6c6be80af2e31a6415f55d5e)
+    
+    Conflicts:
+            src/rgw/CMakeLists.txt (missing SYSTEM flag from
+                     https://github.com/ceph/ceph/pull/18711)
+
+commit eb6f089631225ab4779184a3d20c7a1cbde44853
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Dec 5 14:43:06 2017 -0500
+
+    rgw: update beast frontend for boost 1.66
+    
+    Fixes: http://tracker.ceph.com/issues/22600
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a66a6449296d39e57724fa733ac3069fecfdcdaa)
+
+commit b1bfa824b096189c8e5ee8e42f9bf400c44059d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 13:03:35 2017 -0500
+
+    cmake: update minimum boost version to 1.66
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0e47a339dface3807ab0061f9ce62fe1d2654d29)
+
+commit 1d2e15ac08f43ad2125ba0892f3beb1c632eda20
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Nov 27 14:16:33 2017 -0500
+
+    submodule: update Beast to ceph/ceph-master branch
+    
+    pulls beast submodule up from v116 tag to v124, with an additional bug
+    fix for async_read_some()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit efe90765957371a78b36305727dc3eb34252ad91)
+
+commit acb49cfc8f2cc79b0bf38ab0a95c07398a9c5061
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Nov 9 23:20:40 2017 -0500
+
+    rgw: RGWEnv::set() takes std::string
+    
+    the beast frontend will either pass a string_ref or a string_view,
+    depending on the boost version. we can't overload RGWEnv::set() for
+    both, because a call to env.set("literal") would be ambiguous
+    
+    both string_ref and string_view have a to_string() member function, so
+    RGWEnv::set() now takes a std::string by value and moves it into the
+    map. this involes a single string copy, whether we pass a temporary
+    std::string (in beast) or a const char* (in civetweb)
+    
+    Fixes: http://tracker.ceph.com/issues/22101
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6fbd3f358a17debea8c04f976946d5f245576f31)
+
+commit 0f08d3d9a53f1cb1f1ea13181a3c96a14f7b65dd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 18 20:22:11 2017 -0400
+
+    rgw: fix for pause in beast frontend
+    
+    pause_for_new_config() was only stopping the listener on pause, but
+    existing keepalive connections would keep trying to read. this prevented
+    the frontend thread calls to io_service::run() from returning to
+    complete the pause
+    
+    Fixes: http://tracker.ceph.com/issues/21831
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c64ac823adfb6d717570d3d3e82b89ea73dc3a77)
+
+commit 3d79dadf4aba3f15352fbf9750ff69afe0adaf9e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 26 14:35:56 2017 -0400
+
+    qa/rgw: add beast frontend to some rgw suites
+    
+    added a qa/rgw_frontend directory for civetweb.yaml and the new
+    beast.yaml. the rgw suites for multifs and singleton now symlink
+    rgw_frontend/civetweb.yaml. the multisite, tempest and verify suites
+    symlink rgw_frontend to test both. this doubles the number of jobs in
+    those suites
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 607b72f553b3d4c1ad6feba9ee7199a4d91272f1)
+
+commit 53ff61f95db75a892da0c085e9eaf9e3425e94d2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 11 09:01:35 2017 -0400
+
+    rgw: ask beast parser about keepalive/close
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5d7f154a77eb2cf82b6e4e49d20687c8ed839113)
+
+commit c3bb0bae2a9878ec8c3302168f82a22aed5a983d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 25 11:13:08 2017 -0400
+
+    rgw: remove boost::coroutine and context deps
+    
+    the beast frontend no longer uses stackful coroutines, so these
+    dependencies aren't necessary
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b904b575042d6598a9cf74d23beecd4884e097d2)
+
+commit 30b4d2bb9385dfc7837f7941df5a7a3c507c8b72
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Jul 17 10:12:07 2017 -0400
+
+    rgw: beast frontend discards unread body before next header
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c6bc7e1b0cbaf553f000aaa2893fa1486fc02a75)
+
+commit c897636ff13750acd446cd7a37b4b5c83e4d4dde
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 12 11:39:26 2017 -0400
+
+    rgw: beast frontend uses callbacks instead of coroutines
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7767d8d88aecac8e88ed4c87a187c7a2ed67cf48)
+
+commit 0a024c978b72402e4f8792c60c017d3ccbc60050
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 11 16:12:02 2017 -0400
+
+    rgw: set header/body size limits on beast parser
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1f72684eb4c6fb9fd021240ae55812894afc19e9)
+
+commit 1bc2728da84992c2a737202f04ad358470140a34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 6 16:31:23 2017 -0400
+
+    rgw: update beast frontend/submodule to v116
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 746c218c620d7681f6c9d769631ee1ac0d2b5987)
+
+commit 2dd6512cdd9ed2548ceae42d5785c140b1ac2a35
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Thu Aug 10 15:21:32 2017 +0800
+
+    rgw: fix opslog uri as per Amazon s3
+    
+    According to s3[1], current Request-URI opslog entry are missing:
+    
+    + request method
+    + query string
+    + http version number
+    
+    [1] http://docs.aws.amazon.com/AmazonS3/latest/dev/LogFormat.html
+    
+    Fixes: http://tracker.ceph.com/issues/20971
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 96bb12a158ab899ec219d88e140499a4e27b4ba5)
+
+commit e7bd12a38716d17a7d6b752d6730d5913916cded
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Mon Aug 7 17:30:27 2017 +0800
+
+    rgw: fix opslog can't record referrer when using curl as client
+    
+    Fixes: http://tracker.ceph.com/issues/20935
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 23323b7f725dc1e99e4a81512b57d342dab9a3fe)
+
+commit 9db61c8b14a915c8f99dfe896610a2431f277f68
+Author: Bingyin Zhang <zhangbingyin@cloudin.cn>
+Date:   Tue Dec 26 17:06:44 2017 +0800
+
+    rgw: put bucket policy panics RGW process
+    
+    Fixes: http://tracker.ceph.com/issues/22541
+    Signed-off-by: Bingyin Zhang <zhangbingyin@cloudin.cn>
+    (cherry picked from commit f05a044cad18c16ebc2c9a177f38b4bdc76cfd66)
+
+commit 3de49aa94560625909794153b4ab965aeeba4b37
+Merge: 907a72465e ca980a63c3
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Mon Jan 8 17:22:30 2018 +0000
+
+    Merge pull request #19505 from shinobu-x/wip-22392-luminous
+    
+    luminous: mds: tell session ls returns vanila EINVAL when MDS is not active
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+
+commit 907a72465e0f3546724d9686b3cafe5d7208cf9f
+Merge: 70755a02e6 567bd5acad
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 8 16:18:12 2018 +0000
+
+    Merge pull request #19831 from pdvian/wip-22576-luminous
+    
+    luminous: qa: configure zabbix properly before selftest
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+
+commit 2042881c6702a9d660b57385a908e9da7df4cd0f
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Dec 11 11:33:16 2017 +0100
+
+    mgr/dashboard: Fix PG status coloring
+    
+    This was broken in the 12.2.2 release and PGs would not get a
+    color anymore.
+    
+    This commit also makes sure pools/PGs are marked as red when
+    one or more PGs are inactive, inconsistent, incomplete, down, etc.
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 450aa2133d05916822083969260ee07622591e33)
+
+commit 0a347506af4b45decf604bea4144ac1eafc2d34e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 4 16:08:25 2018 +0800
+
+    rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    this is the rpm's counterpart of debian/control changes related to the
+    ceph-{osdomap,kvstore,monstore}-tool feature move. see #19328 and #19356.
+    the commit introducing this move is 6dba25e. and
+    
+    $ git describe 6dba25e
+    v12.2.2-8-g6dba25e39d
+    
+    so the first release that have this change is 12.2.2-8. in other words,
+    ceph-{base,osd,mon} >= 12.2.2.8 cannot co-exist with ceph-test < 12.2.2-8
+    in the same system. so we let ceph-test Requires ceph-common with the
+    same version. and since ceph-{osd,mon} Requires ceph-base with the same
+    version, and ceph-base Requires ceph-common with the same version, so by
+    tiering ceph-test with ceph-common with the same version, we enforce
+    this restriction.
+    
+    Fixes: http://tracker.ceph.com/issues/22558
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b07aa210aa0ede54ffc3dbe49e334bd51a8f6342)
+
+commit 70755a02e66eb82fefc7d3a0114fe5a98a7f1dc2
+Merge: c859483fee 4a79dd9065
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 8 12:15:39 2018 +0200
+
+    Merge pull request #19787 from linuxbox2/luminous-21984
+    
+    rgw: fix rewrite a versioning object create a new object bug
+
+commit b5892a13b9e10f1d5cc5e3c4e869bac06c3926ca
+Author: Ilya Margolin <ilya@ulani.de>
+Date:   Thu Dec 14 17:07:01 2017 +0100
+
+    pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Fixes "Prometheus exporter can't get metrics after update to 12.2.2"
+    
+    Fixes: http://tracker.ceph.com/issues/22313
+    Signed-off-by: Ilya Margolin <listen@ulani.de>
+    (cherry picked from commit 58669bedf530e1ccaf4d6c4bf018eeec43b78647)
+
+commit 567bd5acad2d1d5598a63835b9ce874e697cf2a8
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 21 08:27:45 2017 -0500
+
+    qa: configure zabbix properly before selftest
+    
+    Even though the selftest routine doesn't care about
+    the settings, we should set them to avoid emitting
+    nasty log/health messages when enabling the module.
+    
+    Fixes: http://tracker.ceph.com/issues/22514
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c64c9ff00d2df2177135bcd2735bd7aeac625643)
+
+commit 98e3e2a7c840f985baa4ae82a9ce943e85b99657
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Dec 19 11:37:14 2017 -0800
+
+    mds: check metadata pool not cluster is full
+    
+    CEPH_OSDMAP_FULL flag was obsoleted by
+    b4ca5ae462c6f12ca48b787529938862646282cd. So, check if the metadata pool is
+    full instead which is a decent proxy (as metadata operations can still
+    proceed). However, the data pool may still be full which would result in some
+    operations still not completing (like inode backtrace updates).
+    
+    Fixes: http://tracker.ceph.com/issues/22483
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d678415bb03afd1a67edaa0eac031a6a9cf3fbf9)
+
+commit dd6b6626ccc97666a9de24fcc770f9b7ee9b3a31
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Dec 13 18:07:00 2017 -0800
+
+    vstart_runner: ignore failed dentry invalidation
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit cd1e5f53f0a732e4398efadbbf5b6ce914ddd647)
+
+commit c5c0435efcdce7e2e87cf95a0e13642b40b9a42f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Dec 13 17:50:32 2017 -0800
+
+    vstart_runner: set ec_profile attribute
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0f507d88559e7060e86a4a3ea9fcffb02c013bec)
+
+commit e97399d2eefb13ff8083ce2d9c9686e69c30e488
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Dec 15 14:23:01 2017 -0800
+
+    mds: reduce debugging level for balancer messages
+    
+    Fixes: http://tracker.ceph.com/issues/21853
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 435babe4a44c73d094eeddb5e621ebd29a62aff8)
+
+commit 5bf8d71001bf929813fd5e3bb101c60630ab6c5d
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Sep 19 09:17:22 2017 -0400
+
+    rgw: Policies now properly evaluate the X-Forwarded-For header.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 5f7d9c4ff6c78f65d074dbdf8a181cb9ae09851e)
+
+commit ac3e81b8ad887cc84e553d7dd24aa002ea536269
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Sep 19 10:55:12 2017 -0400
+
+    rgw: Test of proper parsing of the X-Forwarded-For header for policies.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit e02facbf5c7b06b4d1d628ddb83ee74935493def)
+
+commit 65b515b9d0c119dde57f57d1d7c75d81e175113b
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:49:50 2017 -0500
+
+    rgw: Bucket IP address policy evaluation now uses rgw_remote_addr_param.
+    
+    Previously bucket policy ip address restrictions were only being evaluated
+    against the REMOTE_ADDR environment variable and ignoring the header
+    specified by the rgw_remote_addr_param configuration option. This rendered
+    ip-based bucket policies worthless when running behind a reverse proxy.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit c4c24ca986f17c68b75f76fc48ad489002fcf87e)
+
+commit a75908b8098ef39ed0193616e9d4fb723d73883c
+Author: John Gibson <jgibson@mitre.org>
+Date:   Thu Sep 14 10:12:59 2017 -0400
+
+    rgw: rgw_iam test harness can now stringify() the fake identity.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 557bdb617e6eb78f9f0e6d6ba35b79b318a6c275)
+
+commit 1d8f684a3cf11c5fc422462790b5a87d5ab5b587
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:48:00 2017 -0500
+
+    rgw: Fixed several bugs in policies related to IPv6 addresses.
+    
+    The IPv6 conversion was not properly converting the address to host byte
+    order.
+    The text conversion of IPv6 addresses was using raw byte values instead of
+    the converted number. The portions of the addresses were grouped by bytes
+    instead of 16-bit words. The prefix length was erroneously being rendered
+    in hex.
+    
+    http://tracker.ceph.com/issues/20991
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit a25ca37401d1e8dc4349201b9f64aa6990bea0d5)
+
+commit 9f7aeb4bd52bbdaef4ffd0127b38c3084e95a706
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:44:54 2017 -0500
+
+    rgw: Fixed several bugs in policies related to IP Addresses.
+    
+    Comparisons of two individual IP addresses caused an assertion error.
+    The text conversion of IPv4 addresses was using raw byte values instead of
+    the converted number.
+    NotIpAddress condition now works with multiple values.
+    
+    http://tracker.ceph.com/issues/20991
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit ca21596c95aa3871d14ac9112840d3b1101a8635)
+
+commit 5fbb50c1880b0dafc64892e0c23d188c2eccbec7
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Dec 26 16:40:31 2017 -0500
+
+    rgw: Added tests for S3 Policy IP Address feature.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 3d260f26e6826182b4de520878b548a28a96e81b)
+
+commit e4a5cf95e645d118a781b65c4ca6e5c37dd7eb6c
+Author: John Gibson <jgibson@mitre.org>
+Date:   Thu Aug 10 13:39:35 2017 -0400
+
+    rgw: Added support for testing the wildcard principal in policies.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit e83b647d44153a7775647693fca1848e592ee107)
+
+commit 7cca4a6c1034ce32c41b73bace538965c5e77a8c
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Mon Nov 27 14:32:44 2017 +0800
+
+    rgw: implement ipv4 aws:SourceIp condition for bucket policy
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit 2fb445b6f7c1e997e83b1c7da2a1fecdde164d35)
+
+commit e49bf0889e6144b6683d8cb4554e38efe29b7a8a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Nov 3 10:42:37 2017 -0400
+
+    rgw: simplify use of map::emplace in iam
+    
+    the piecewise_construct overloads of map::emplace() are only needed when
+    there's ambiguity around which arguments go to which constructor
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9787fe6b8dc545419b0f3375c06a57a414ae8aa6)
+
+commit 8eb36333bc8e709cfb9e4958821548f813e105da
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jul 27 17:36:37 2017 +0200
+
+    rgw: policy: support for s3 conditionals in ListBucket
+    
+    This adds support for s3:prefix,delimeter & maxkeys identifiers when
+    specified as conditionals in policy.
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a7184ca8c507b66ef01687bd85528342ea3bf934)
+
+commit 3f8ab0b8940c332b95c517d02e4aa658f394cd6d
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Mon Aug 7 15:55:19 2017 +0800
+
+    rgw: fix opslog can not record remote_addr
+    
+    Fixes: http://tracker.ceph.com/issues/20931
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 650d30d64e084df12f4cfcb3d1edaefb5d475bb4)
+
+commit 5b9f1281a3bfe3e75bd4ecd2d3983543396d119d
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 22:48:18 2017 -0400
+
+    rgw: Fix evaluation of bucket management permissions
+    
+    I spent some more time looking through the documentation of how work
+    is evaluated, and the examples on
+    
+    <http://docs.aws.amazon.com/AmazonS3/latest/dev/
+    access-control-auth-workflow-bucket-operation.html>
+    
+    Have convinced me that the behavior that was requested is more correct
+    than what we were doing before.
+    
+    Fixes: http://tracker.ceph.com/issues/21901
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 343a25aa2134b6fdddeca6c9dfbaefde2dc9c66a)
+
+commit febd39cf67eb026cf577d72440f5d5b61c98c3ad
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 20:17:52 2017 -0400
+
+    rgw: Refactor checking of some ops
+    
+    Since some operations check the user against the bucket owner in the
+    absence of a policy, rather than open-coding that everywhere, act like
+    a proper computer scientist and abstract it.
+    
+    Fixes: http://tracker.ceph.com/issues/21896
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 8818a0cb5e699135976e057061fb8e9d99850cd3)
+
+commit c859483fee526656b6c6cd4d9d0c975cff57b565
+Merge: bccb4839ee ce1fe2480c
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 17:07:00 2018 +0000
+
+    Merge pull request #19449 from smithfarm/wip-22398-luminous
+    
+    luminous: doc: man page for mount.fuse.ceph
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+
+commit 34864fe7ba839e0d593437b6e62f6812f1c1a375
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Dec 20 17:06:32 2017 -0500
+
+    rgw: Plumb refresh logic into object cache
+    
+    Now when we force a refetch of bucket info it will actually go to the
+    OSD rather than simply using the objects in the object cache.
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit d997f657750faf920170843e62deacab70008d8b)
+
+commit 5ceb7cb998b505d2fe0cd98b534a918424e4d809
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Dec 19 16:47:09 2017 -0500
+
+    rgw: Add expiration in the object cache
+    
+    We had it in the chained caches, but it doesn't do much good if
+    they just fetch objects out of the object cache.
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 82a7e6ca31b416a7f0e41b5fda4c403d1d6be947)
+
+commit 767fec1e22247ec1c6010c308d51ba77dd31cb39
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Dec 19 12:53:05 2017 -0500
+
+    rgw: retry CORS put/delete operations on ECANCELLED
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit bff7e61ca5a66b301ec49c1cf9054d1b74535832)
+
+commit bfd4091f7faf43ebc4eb81c8215e8b3ec2d4a8c4
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 17:15:26 2017 -0500
+
+    rgw: Expire entries in bucket info cache
+    
+    To bound the degree to which an RGW instance can go out to lunch if
+    the watch/notify breaks down, force refresh of any cache entry over a
+    certain age.
+    
+    Fifteen minutes by default, and expiration can be turned off entirely.
+    
+    This is separate from the LRU. The LRU removes entries based on the
+    last time of access. This expiration patch forces refresh based on the
+    last time they were updated.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 4489cb58a15647a31ac0546d70400af5668404cb)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit ce1fe2480c0cbef7337f870400bf441ef13e9544
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 19:37:31 2018 +0530
+
+    doc: fix typo
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit a925bb4520797376b4b169bbcfe613cd1fa36429)
+
+commit fa86f78983e990b10a24d6c25526014e47370a71
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 12:41:37 2018 +0530
+
+    doc: add mount.fuse.ceph to index
+    
+    Added mount.fuse.ceph to index.rst
+    
+    Fixes: http://tracker.ceph.com/issues/22595
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit db23f4ce30cd521b5b8909edc8c26dc87aeecd98)
+
+commit e21b3af6a378cfffe58df3d56a9f9884adba8a36
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Mon Nov 27 16:16:35 2017 +0530
+
+    doc: doc for mount.fuse.ceph
+    
+    Created doc for mount.fuse.ceph.
+    
+    Fixes: http://tracker.ceph.com/issues/21539
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 6c39818eaf39358cab8dd3dce579c932fba0d05d)
+
+commit 88c987cdb5f02ddc445389b40f9473e8fc14b8af
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 13:25:33 2017 -0400
+
+    qa/workunits/rbd: fixed variable name for resync image id
+    
+    Fixes: http://tracker.ceph.com/issues/21663
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 22816ed3f48b0a08a6d4e0cf0b0e14dabbb9ce69)
+
+commit a15eb7df1e3b6d273b4366e7afefd3c2ade4775e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 27 09:40:08 2017 -0400
+
+    librbd: hold cache_lock while clearing cache nonexistence flags
+    
+    When transitioning from a snapshot that had an associated parent
+    to a snapshot where the parent was flattened and removed, the cache
+    was being referenced without holding the required lock.
+    
+    Fixes: http://tracker.ceph.com/issues/21558
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 16ef97830cde30efb96f7aee69834b3a5c2d5248)
+
+commit 84fff7aca5d7387566304206eb4cfbb934c2d536
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 11:42:21 2017 -0500
+
+    rgw: dont log EBUSY errors in 'sync error list'
+    
+    these temporary errors get retried automatically, so no admin
+    intervention is required. logging them only serves to waste space in
+    omap and obscure the more serious sync errors
+    
+    Fixes: http://tracker.ceph.com/issues/22473
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ca4510bc76ad7beed1128539aa9e424d29dd8585)
+    
+    Conflicts:
+        src/rgw/rgw_data_sync.cc ("multisite log tracing" feature - see
+            https://github.com/ceph/ceph/pull/16492 - is not being backported to
+            luminous)
+
+commit e51e1fa45e6a1740b94b56977bc927d302474f5c
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:16:38 2017 -0500
+
+    rgw: Handle stale bucket info in RGWDeleteBucketPolicy
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit e397b7e6d0c49d625fb2b2363311e6486f2045fe)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 08bf9c07e012a365d40901088feede4c4c200f84
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:15:04 2017 -0500
+
+    rgw: Handle stale bucket info in RGWPutBucketPolicy
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 1738b4f6b726b462abb436f78026c1577b55f05e)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 3f2fa364fb7f8140d20ad7de13a69cd2cc31ec77
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:05:06 2017 -0500
+
+    rgw: Handle stale bucket info in RGWDeleteBucketWebsite
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit f4d274248e43cb38ff2b27782c010b2c35b12b2b)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit b266755161b6769af0a08bf647c771a00a7aa50d
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:03:13 2017 -0500
+
+    rgw: Handle stale bucket info in RGWSetBucketWebsite
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit b2b7385f194def1025a8947bab876c9856b06400)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 672eea5d88099a590ddf847ad080214944c780cb
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:59:44 2017 -0500
+
+    rgw: Handle stale bucket info in RGWSetBucketVersioning
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit a0a1e7c2ef992b8758bcfb20d893730c1b202475)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit fc271e4e3c44e520d8a530ebfc758a438c110737
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:53:05 2017 -0500
+
+    rgw: Handle stale bucket info in RGWPutMetadataBucket
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit ebb86301b20098e15824f469001f6153b27965f5)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit dea29460ed513f965de2dcac9e29180b06d56b58
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:51:42 2017 -0500
+
+    rgw: Add retry_raced_bucket_write
+    
+    If the OSD informs us that our bucket info is out of date when we need
+    to write, we should have a way to update it.
+    
+    This template function allows us to wrap relevant sections of code so
+    they'll be retried against new bucket info on -ECANCELED.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 1a3fcc70c0747791aa423cd0aa7d2596eaf3d73c)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 4a79dd90657c6635f39de555fcc33daa584ecbd7
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Tue Oct 31 15:21:21 2017 +0800
+
+    rgw: fix rewrite a versioning object create a new object bug
+    
+    Fixes: http://tracker.ceph.com/issues/21984
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 700a0292362128cb29586a64ef8215a07d96736b)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2e3a89f019a776ad5731e49cd16b51b3f8e2e807
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Nov 16 14:42:58 2017 -0500
+
+    rgw: Add try_refresh_bucket_info function
+    
+    Sometimes operations fail with -ECANCELED. This means we got raced. If
+    this happens we should update our bucket info from cache and try again.
+    
+    Some user reports suggest that our cache may be getting and staying
+    out of sync. This is a bug and should be fixed, but it would also be
+    nice if we were robust enough to notice the problem and refresh.
+    
+    So in that case, we invalidate the cache and fetch direct from the
+    OSD, putting a warning in the log.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 9114e5e50995f0c7d2be5c24aa4712d89cd89f48)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit c49417d8b0f614a895bdc6f19b00868d694f8ef8
+Author: Mark Kogan <mkogan@redhat.com>
+Date:   Tue Dec 12 10:34:05 2017 -0500
+
+    rgw: fix chained cache invalidation to prevent cache size growth
+    above the rgw_cache_lru_size limit
+    
+    Fixes: http://tracker.ceph.com/issues/22410
+    
+    Signed-off-by: Mark Kogan <mkogan@redhat.com>
+    (cherry picked from commit a6a1b664d313a54ad9d2f64b859296b1352b1ce4)
+
+commit ad0ea9fa530d641fdadcae29059660446b428588
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 31 18:31:13 2017 -0400
+
+    RGW: S3 POST policy should not require Content-Type
+    
+    In the current implementation, putting a key in the acting
+    RGWPolicyEnv makes it required in RGWPolicyEnv::match_policy_vars.
+    
+    I'm not sure this is the intent, but in any case, add it to the env
+    only if sent.
+    
+    Fixes: http://tracker.ceph.com/issues/20201
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit af29276c32f7009a78bd8b90d8f7d19026859c74)
+
+commit 08edb1631191ad67266a1a8a66b1d147ed0abf9f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 20 09:56:50 2017 +0800
+
+    mds: respect mds_client_writeable_range_max_inc_objs config
+    
+    get_layout_size_increment() return layout.get_period(). A period
+    contain layout.stripe_count objects.
+    
+    The config is for limiting number of objects need to probe when
+    recovering a file.
+    
+    Fixes: http://tracker.ceph.com/issues/22492
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ee11b6cd9ce97820c4f53ed98380bbd63fab891f)
+    
+     Conflicts:
+            src/mds/Locker.cc
+
+commit 8d6c2c2dea78ed485d3d63fd39085ca1e82c7d71
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 18 16:48:51 2017 +0800
+
+    mds: track dirty dentries in separate list
+    
+    this should improve performance of large directory
+    
+    Fixes: http://tracker.ceph.com/issues/19578
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 49559663825ff742bde386355be864e03e646ffa)
+
+commit bccb4839ee8fde4636001976df1890b7a0f1f0a0
+Merge: 840cc7b174 c1c539f1ef
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Jan 3 11:07:39 2018 -0600
+
+    Merge pull request #19243 from liewegas/wip-fix-statfs-luminous
+    
+    mon/Monitor: fix statfs handling before luminous switchover happens
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 840cc7b1747fe12e42e7e182afd8298d1f8684b0
+Merge: d94942cdfe 2928e2cf3a
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Tue Jan 2 14:23:35 2018 -0800
+
+    Merge pull request #19746 from ceph/wip-use-installer0-luminous2
+    
+    qa/tests - Added options to use both cases: mon.a and installer.0
+
+commit 2928e2cf3ab02357b0eea1ae9afe332bc240af06
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Jan 2 10:12:54 2018 -0800
+
+    qa/tests - Added options to use both cases: mon.a and installer.0
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 10fc85089c3bb64ced8c3a0ea17987e9ec5f46a2)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 322966a68940637a8575f9e06f07b0475ed271ca
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Dec 20 13:05:22 2017 -0800
+
+    qa/tests: run ceph-ansible task on installer.0 role/node
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 3b2a26d919c173c887fd193f186ea56c33fcd9ae)
+
+commit 6791e67faf352b31c9e6d4e191b82ba8fac7adda
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Fri Dec 22 16:44:31 2017 +0800
+
+    erasure-code: use jerasure_free_schedule to properly free a schedule
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 8113fa522d223a2e6708c345a5f216cbe4a5e758)
+
+commit e950f5dbf07ed8d7fb37047be797f3cfe60a1fe1
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Dec 23 12:48:47 2017 +0800
+
+    common/dns_resolve: fix memory leak
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit d7c3b5940fae5a474a27b7eee72582223e1b0504)
+
+commit d94942cdfe8fc9fccf9fd79ece8bc828d608e713
+Merge: c4905daa94 eb9b1c0413
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Jan 2 09:17:16 2018 -0600
+
+    Merge pull request #19741 from joscollin/luminous
+    
+    luminous: doc: update Blacklisting and OSD epoch barrier
+
+commit eb9b1c0413aadd7f81d55a0917e840de62af67ff
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 2 16:17:39 2018 +0530
+
+    doc: fix grammar mistake
+    
+    Fixed grammar mistake in a sentence.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+
+commit 0e43f0f01cd800fee4cd800f1545405b449fa55b
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 2 16:11:30 2018 +0530
+
+    doc: update Blacklisting and OSD epoch barrier
+    
+    Updated missing OSD epoch barrier section, which is referenced by http://docs.ceph.com/docs/luminous/cephfs/full/#hammer-and-later.
+    
+    Fixes: http://tracker.ceph.com/issues/22552
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+
+commit c4905daa94d8b2fbad32a4e39c3e3076380363c5
+Merge: 7aa78f01c9 5ec37f1f49
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Dec 30 14:12:19 2017 -0600
+
+    Merge pull request #19721 from tchaikov/wip-luminous-boost-un-submodule
+    
+    luminous: boost un-submodule
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ec37f1f49acf8fbfcb7667be43b139a68d3aa96
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 24 13:56:02 2017 +0800
+
+    make-dist: exclude unused bits in boost
+    
+    the docs, examples and tests are not used. so drop them. we could go
+    further by removing unused components in boost. but that'd be an issue
+    if somebody added a component in CMakeLists but forgets to update this
+    script. also, we need to remove boost/$component and lib/$component to
+    achieve this goal. this also introduces extra complicity. so leave it
+    for another change.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 65f91227a6b052ad60b01276d7d72dea07cbb1a4)
+
+commit 5eb42af8ce5b96b41b7d8c80cd2cc95b7eff541d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 24 10:27:19 2017 +0800
+
+    make-dist: repackage boost in the correct path
+    
+    before this change, boost is put into ./src. after this change, it is
+    put into $outfile/src. i.e. something like
+    ceph-12.1.2-4592-gf5f2ced624/src .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2218efa005ab871f26402b5dd33b2e599897c11f)
+
+commit 9e49711c2293761bd8b501ef1bc948bbf1e08066
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Thu Sep 28 15:28:53 2017 +1000
+
+    make-dist,cmake: Try multiple URLs to download boost before failing
+    
+    Remove SPOF during boost download for make-dist and later cmake > 3.7
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 598556f98b65a03ad40ad076623b9cc8b507810e)
+
+commit 7929ff7f7edeff6df27adfdf6da5b4a89e36c4e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 17:15:17 2017 -0400
+
+    make-dist,cmake: move boost tarball location to download.ceph.com
+    
+    Sourceforge is down.  Also, we can hammer our servers instead of
+    theirs.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19987549e0ce526e353e24adddc7df8b1f165aab)
+
+commit b798225cf69da9d156e46ebe4fec463f79430d70
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 1 01:00:04 2017 +0800
+
+    make-dist: download and repackage boost
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f04436b189376cf7b53d710ab2f2bf01d47482ad)
+
+commit 3c6f83bb7c4b544118626fa92f7e9e5a407a1973
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 30 23:58:44 2017 +0800
+
+    boost: remove boost submodule
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b23056cf8d0d1bbd269bc96eef33dc27c25fd9e)
+
+commit 7aa78f01c95a3eac32e27c2c90ffe6d867cb89dc
+Merge: f519145585 84d60ae507
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Dec 29 18:27:43 2017 +0100
+
+    Merge pull request #19680 from tchaikov/wip-luminous-22220
+    
+    luminous: install-deps.sh: revert gcc to the one shipped by distro
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f519145585040612801416b3f23097494d46667b
+Merge: 88b4534a0f 96b924fc75
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Dec 26 01:27:04 2017 +0800
+
+    Merge pull request #19048 from shinobu-x/wip-22196-luminous
+    
+    luminous: mgr[zabbix] float division by zero (osd['kb'] = 0)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 84d60ae507db8c290f9b8fab302bdaaacce247e1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Dec 22 22:42:16 2017 +0800
+
+    install-deps.sh: update g++ symlink also
+    
+    we need to update g++ symlink also, if it points to the wrong version
+    
+    http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts: the libboost issue does not affect master. as master builds
+         boost from source. so, it's not cherry-picked from master.
+    (cherry picked from commit 248a157635b46d3cf23e37ae263c62b0dc4e0e59)
+
+commit 54abba8cf87414f0d5444b1de78fd35fa379a461
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Dec 14 21:01:43 2017 +0800
+
+    install-deps.sh: readlink /usr/bin/gcc not /usr/bin/x86_64-linux-gnu-gcc
+    
+    See: http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5be6ff11e536cb492dd50dedf8a04fb9acc1222e)
+
+commit 2d377d351120d06382ae303e10a330de143f0a85
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 13 13:36:54 2017 +0800
+
+    install-deps.sh: point gcc to the one shipped by distro
+    
+    to define a struct in a method is legal in C++11, but it causes internal
+    compiler error due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82155
+    if we are using GCC-7. so we need to either workaround in our source
+    code by moving the struct definition out of the member method or revert
+    to a GCC without this bug. but if we go with the first route, the jewel
+    build still fails, because GCC-7 starts to use the new CXX11 ABI, which
+    is not compatible with the libboost we use in jewel. the libboost was
+    still built with the old ABI for backward compatibility. so let's just
+    fix the install-deps.sh to point gcc to the origin one.
+    
+    See: http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts: the libboost issue does not affect master. as master builds
+     boost from source. so, it's not cherry-picked from master.
+    (cherry picked from commit ccc4dea90e483ea8bf6bee0721ef929e7f48ff5a)
+
+commit 1643d5a4f8f726ce27ac9f5cc8160c34c07c09bb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Dec 9 14:09:24 2017 +0800
+
+    mds: properly eval locks after importing inode
+    
+    We should call Locker:eval() for all imported inodes who have non-zero
+    'wanted caps'. MDS does not properly handle following case.
+    
+    - client open a inode for read, it send a cap message to MDS.a (the cap
+      message updates 'wanted caps')
+    - MDS.a receive the cap message, the inode is non-auth and is ambiguous
+      auth. MDS.a can not request 'wanted caps' from auth mds.
+    - MDS.a finishes importing the inode from. But no caps are imported and
+      mds_caps_wanted map is empty.
+    
+    The bug can cause read hang.
+    
+    Fixes: http://tracker.ceph.com/issues/22357
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit debb556076c5685f0d8bb8029a95684b9e552eb8)
+
+commit e3bb21529b8e5a8f483cf8dab4d0012279e0373b
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 28 14:09:31 2017 +0100
+
+    test/cls: add a basic test for rgw usage log
+    
+    A basic test for cls_rgw for usage that sets 512 usage log entries,
+    reads them and deletes them
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit ca5be9fdf88ae2f35122ee0bafb5575731b872f6)
+
+commit 7810567fa37308bfeadf63e65f08adc36bfb886e
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 21 16:38:27 2017 +0100
+
+    cls/rgw: trim all usage entries in cls_rgw
+    
+    Currently trim usage will only trim upto 128 omap entries, since we need
+    to run this in a loop until we're done, actually make the cls return
+    -ENODATA so that we know when to stop the loop (inspired by a similar
+    call in cls_log) this involves the following changes
+    
+    * return -ENODATA when iterate entries goes through and the value of
+      iter (which is set as the last value of key when succeeded)
+    * use IoCtx for calling the loop from within cls rather than in rgw
+    * drop the goto call in rgw_rados since we can return once we're done
+      processing
+    
+    Fixes: http://tracker.ceph.com/issues/22234
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit b548a3f3443452210d92cad574bcb73ba6d2ce42)
+
+commit 7ffe8c41d3c095ebaf635aceaf2ee39d8eb58001
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Dec 12 19:58:01 2017 -0800
+
+    memstore: write fsid to fsid not fs_fsid
+    
+    Fixes: http://tracker.ceph.com/issues/20736
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e8c6882cc9c28b07ae1d5dc1169920c3ee821ed5)
+
+commit ce54fd9b995a5e28fbba462e5843c17ee58fea29
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 09:30:50 2017 -0600
+
+    os/bluestore: prevent mount if osd_max_object_size >= 4G
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f284bbb0593eafc25013c01c8aa7b8bdfc39d906)
+
+commit 47f74bcd136c3f9d0034ed1414442fdb03824bf8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Dec 18 18:41:31 2017 -0800
+
+    qa: don't configure ec data pool with memstore
+    
+    Fixes: http://tracker.ceph.com/issues/22436
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b2284f23b813399613f8540efd2d426b3c6f9839)
+
+commit 6eea75200fed82a2c5513795a79834527090957c
+Author: root <tdd108128@163.com>
+Date:   Fri Dec 1 15:26:00 2017 +0800
+
+    cephfs: potential adjust failure in lru_expire
+    
+    Fix: the first adjust is no needed,it will never take real effect.
+         the second 'adjust' may never get the chance to be executed
+         suppose we can reach the second 'adjust', it will crash because the bottom list is empty now.
+    
+    Fixes: http://tracker.ceph.com/issues/22458
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 590c39eab02e64de7393c35ae7a9efb6ce626770)
+
+commit d0d66c73b02065dcdaf70905a69fc37c0f7a6e6e
+Author: root <tdd108128@163.com>
+Date:   Sun Dec 10 15:35:21 2017 +0800
+
+    cephfs-journal-tool: tool would miss to report some invalid range
+    
+    Fixes: http://tracker.ceph.com/issues/22459
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 7708bff40cdf059a362902156a6d0660ba21fb14)
+
+commit 3d0f4fa752751a54ee2a88faa0abbb96a55fea11
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 15 14:51:46 2017 -0500
+
+    rbd-mirror: cluster watcher should ensure it has latest OSD map
+    
+    Fixes: http://tracker.ceph.com/issues/22461
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9eb247961ae8bfe63c4025032ad532b7c3e3a1e7)
+
+commit 40954a9c27cb3813c8f15c2f89d7223069b9031c
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Mon Dec 18 13:53:20 2017 +0100
+
+    mgr/balancer: fix KeyError in balancer rm
+    
+    Fix the typo in the plan name which leads to a KeyError in balancer
+    rm.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/22470
+    (cherry picked from commit 4f581e738ee7c9d519fc4113f6bb0d0d1ce827ff)
+
+commit 5a40ef22449481bca0223f48be8823a3c9c2b198
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 11 09:06:07 2017 +0800
+
+    osdc/Journaler: introduce STATE_STOPPING state
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 05397e405f552bb720e12b9c0557b4c9234ffaa0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 11 08:43:57 2017 +0800
+
+    osdc/Journaler: add 'stopping' check to various finish callbacks
+    
+    These callbacks are executed by finisher. When they are being executed,
+    Journaler can be in stopping state.
+    
+    Fixes: http://tracker.ceph.com/issues/22360
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7ffa87e6a2ab8fb2c64588411c6c2ebff2f91f93)
+
+commit 4943899005267250f42909e2a91a8a54e9479da4
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 29 18:44:30 2017 +0800
+
+    mds: handle client session messages when mds is stopping
+    
+    handle session messages except CEPH_SESSION_REQUEST_OPEN. The problem
+    I found is that mds ignores CEPH_SESSION_REQUEST_RENEWCAPS, which
+    causes client sessions to become stale. Locker::revoke_stale_caps()
+    increases client caps' sequence number. This causes clients to warn
+    about caps' sequence number mismatch when handle caps import/export
+    message.
+    
+    mds should handle CEPH_SESSION_FLUSHMSG_ACK message too. Because
+    one step of exporting subtree is flushing session messages.
+    
+    Fixes:  http://tracker.ceph.com/issues/22460
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 559fdcaec28c8459c5d801efc3842e604324f91f)
+
+commit 88b4534a0fab6f7020874ff8903b2f2eb9d06eb2
+Merge: 5bef6fa349 d223a8f1b9
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Dec 18 13:25:15 2017 -0800
+
+    Merge pull request #18771 from ceph/wip-yuriw-22048-luminous
+    
+    tests - Initial checkin for luminous point-to-point upgrade
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb3d091b463dce78f147e9f1333bd667cf2836ed
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Dec 13 13:08:46 2017 +0000
+
+    mgr/balancer: don't use 'foo' tags on commands
+    
+    This looks weird in logs when code elsewhere logs
+    that it didn't handle a command with tag 'foo'
+    
+    Fixes: http://tracker.ceph.com/issues/22361
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 432d07a0fe3e9992c21539898ce5be6c6b12a745)
+
+commit 2b9bbf605e776640b07ec52d15606f6c04fe39f8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Dec 13 18:15:27 2017 +0100
+
+    build/ops: rpm: set permissions 0755 on rbd resource agent
+    
+    Fixes: http://tracker.ceph.com/issues/22362
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 37e4bc3141ee36a4152834bd10e570bad7389807)
+
+commit c5cb0cab9f307f5bd1b58ce94b0a9e1fb510db9e
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Wed Dec 13 17:02:44 2017 +0100
+
+    mgr/balancer: cast config vals to int or float
+    
+    upmap_max_iterations and other config vals need to be numeric.
+    Cast them appropriately.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/22429
+    (cherry picked from commit d7713e6f823dd62a33a27996e1da8e29f3f7b7c5)
+
+commit 9a9ed272a76a60d9dafebbcc8ff1356ff4d7a9a6
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Wed Nov 15 18:33:17 2017 +0800
+
+    msg/async/AsyncConnection: unregister connection when racing happened
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 5216309c25522e9e4a3c3a03ceb927079de91e9b)
+
+commit 5bef6fa349207cfc88356a8a24c9ab2557179132
+Merge: d268faf20b e134317b81
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 15 20:52:14 2017 +0800
+
+    Merge pull request #19522 from ceph/wip-ceph-disk-deprecation-revert
+    
+    Revert "ceph-disk add deprecation warnings in favor of ceph-volume"
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eb0c60f7d73a89dd7e77f4edcda83b81fa20e784
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Nov 30 12:09:10 2017 +0200
+
+    rgw: reshard should not update stats when linking new bucket instance
+    
+    Fixes: http://tracker.ceph.com/issues/22124
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit fc95210f05367d08c679ec77d23bee6467d01980)
+    
+    Conflicts:
+            src/rgw/rgw_bucket.h: i chosed to pick code from commit fc95210 because conflicts were caused by exsiting parameter and newly added parameter which is used in a method.
+
+commit d223a8f1b91c73ee558a2d28f8af812e9583353b
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Nov 6 12:22:27 2017 -0800
+
+    tests - Initial checkin for luminous point-to-point upgrade
+    
+    Fixes http://tracker.ceph.com/issues/22048
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit d268faf20bc30748d6efe949dc81c12812955f39
+Merge: d242b1c767 fb3b176a80
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 22:03:23 2017 +0100
+
+    Merge pull request #19532 from ceph/luminous-rm22297
+    
+    luminous ceph-volume handle inline comments in the ceph.conf file
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d242b1c7677df993eeee64fbd2a9b128ff4ddcd4
+Merge: dd24747b44 1018f803bd
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 21:50:22 2017 +0100
+
+    Merge pull request #19530 from ceph/luminous-rm22326
+    
+    luminous ceph-volume: warn on missing ceph.conf file
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fb3b176a80b0ada18445bd725d7670c9492509fc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 08:22:23 2017 -0500
+
+    ceph-volume configuration allow inlined comments for # and ;
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit aec5f5042a684952c2b07922695e7675e13645ec)
+
+commit 8fad755dd5ab1de78c636cb93dce5864cc9f07a2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 08:21:29 2017 -0500
+
+    ceph-volume tests verify that INI comments can be inlined
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fe6c4c0d6055c46baadd89723c0a2b78b5ffb7f8)
+
+commit dd24747b44bd70ef82ff81efe835d7eceefb4d22
+Merge: e3f0c349b0 8748c8fb84
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 14 13:48:53 2017 -0500
+
+    Merge pull request #19525 from ceph/backport-19363
+    
+    luminous: ceph-volume: removed the explicit use of sudo
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 1018f803bde2f748ea5b15928d587f80f9bc938d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 13:59:22 2017 -0500
+
+    ceph-volume test logging ignored ceph.conf file
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 69482d1d8059c3b264532f7623c9e6592c57b7c3)
+
+commit 72dfb268fab89bd2fa93e7d9af0894498df6c45f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 13:58:53 2017 -0500
+
+    ceph-volume main warn on inability to load ceph.conf, don't raise
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 44c768db0da0163340b52643122b66ecebbefe23)
+
+commit e3f0c349b0ea34378bcbc183c1ea366aa484c5c4
+Merge: 4909437230 2b06a7f413
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:30:52 2017 +0100
+
+    Merge pull request #19528 from ceph/luminous-rm22305
+    
+    luminous ceph-volume should be able to handle multiple LVM (VG/LV) tags
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 4909437230d8ba9f38205c4ee6ed94a4b0b549e8
+Merge: d87dbedc8f 21924133a9
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:26:30 2017 +0100
+
+    Merge pull request #19527 from ceph/luminous-rm22299
+    
+    luminous ceph-volume Format correctly when vg/lv cannot be used
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d87dbedc8f83fa4c1d4ec5698e53b27e852de788
+Merge: c7383d20c4 775369d5c8
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:22:22 2017 +0100
+
+    Merge pull request #19526 from ceph/luminous-rm22280
+    
+    luminous ceph-volume: handle leading whitespace/tabs in ceph.conf
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2b06a7f413796e2b42faca1172ee97f60adf2f1b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 09:56:00 2017 -0500
+
+    ceph-volume tests.api ensure that we can handle non-ceph tags in lvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c98731ecede09d8d15dca99c57e331212747060e)
+
+commit 47d46678449c13a1aa45f0ae08cecac6397950a1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 09:54:35 2017 -0500
+
+    ceph-volume api.lvm only consider 'ceph.' tags in logical volumes
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7e7df14602a10190a2830262a67202e7ad38f49)
+
+commit 21924133a9c8f954d86ca9da531867d09dd6298b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 13:53:39 2017 -0500
+
+    ceph-volume tests verify proper formatting of RuntimeError on vg/lv error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7320313a15ad8bb948e501382269c8f33a8cb70)
+
+commit 9119839d6ae002fa602901340ef28271833ce08d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 13:53:06 2017 -0500
+
+    ceph-volume lvm.prepare correctly format argument for vg/lv error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7c0b00615d46a9451cb3a7731594cd7864eabc07)
+
+commit 775369d5c8832cbbb9815aeb0de8c36f33aabfd8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 30 08:58:37 2017 -0500
+
+    ceph-volume test leading whitespace is handled in configurations
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ec7f8a27b8c51e1e1d02a2a13d9219bbb1a58c26)
+
+commit 89c9183aad044f2811dafa7e072a1db2f3412d31
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 30 08:58:04 2017 -0500
+
+    ceph-volume trim tabbed/whitespaced configuration files when loading them
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9706e8cc9eb53d0f01fc78978625c1d18bf4667e)
+
+commit 8748c8fb84c23ab747aaed8acae5e5f8ce88f659
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Dec 6 10:40:55 2017 -0600
+
+    ceph-volume: removed the explicit use of sudo
+    
+    This will allow users to run ceph-volume on systems that do
+    not have sudo installed.
+    
+    Fixes: http://tracker.ceph.com/issues/22282
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit aee71a3f5169043f750fd768e7fea9c74309e12d)
+
+commit e134317b8162efd3a6fa25c1c803e43228535f38
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Dec 14 08:28:58 2017 -0500
+
+    Revert "ceph-disk add deprecation warnings in favor of ceph-volume"
+    
+    This reverts commit b8bf0d047868054135592188c7ebe186181310c5.
+    
+    Deprecation warnings for ceph-disk will no longer be present in any
+    Luminous release beyond 12.2.2 - but are still present in master and any
+    newer release.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit 7dabbe91310cb0f97b9d6d09f697f74be2a8ac80
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Thu Aug 17 15:05:41 2017 -0400
+
+    rgw: incorporate the Transfer-Encoding fix for CivetWeb.
+    
+    This commit updates the version of CivetWeb used for RadosGW to
+    get the fix for handling clients that send Content-Length together
+    with Transfer-Encoding. The current handling differs from S3 and
+    thus affects AWSv4.
+    
+    Fixes: http://tracker.ceph.com/issues/21015
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit c9e1daeffb10f36a191960328977f09ef2c05c19)
+
+commit 72de713bf697c4e444adf80fe76df87873119f1a
+Author: Jeegn Chen <jeegnchen@gmail.com>
+Date:   Thu Nov 16 09:12:24 2017 +0800
+
+    rgw: AWS v4 authorization work when INIT_MULTIPART is chunked
+    
+    Add RGW_OP_INIT_MULTIPART as a the single chunk special case
+    like RGW_OP_COMPLETE_MULTIPART.
+    
+    Fixes: http://tracker.ceph.com/issues/22129
+    Signed-off-by: Jeegn Chen <jeegnchen@gmail.com>
+    (cherry picked from commit c8827e5017ce81377cb174ad74cc2f44c3368457)
+
+commit ca980a63c3125a356cff79eba9df06fd2b37d731
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Wed Nov 22 17:20:58 2017 +0530
+
+    pybind: return error message when ceph_mds_command() returns error
+    
+    Returned the error message when ceph_mds_command() returns error.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 941b58c968f6b0e359f279c8bc9e7decf51e75d1)
+
+commit 42bb91a258512c34ea402f34f130518069db3835
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Wed Nov 22 17:12:43 2017 +0530
+
+    mds: Fix error message when mds not active
+    
+    Fix error message for mds not active state. Also fixed the 'unrecognized command' logic by avoiding the new stringstream local object.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 10f93cf5828afbc8ff25ef81b2437ef5c0362396)
+
+commit feb4d7ba37260599952c076c23033288e671f30e
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Fri Oct 20 17:07:41 2017 +0800
+
+    test:add a test case in test_librbd
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit be5ad8764f65f43b6844eaa99c591a31c4d3af82)
+
+commit 13ef77c450a7f108233a6fa5f15d72393db33feb
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Mon Oct 16 22:46:40 2017 +0800
+
+    rbd:can not copy all image-metas if we have more than 64 key/value pairs
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 71178643b2361557f31094e2490d4124b27cd7ff)
+
+commit 1507015169f062cbaa9b3b1e652bb34ba5d586b5
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Sat Oct 28 14:29:45 2017 +0800
+
+    test_librbd:add a test case of 70 key/val pairs in TestClone2
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 85713bdcc3870728dce02f87ee3cfd36257ae533)
+
+commit c4715235c2b3d28c3aaa4584f58d3a49a89daf79
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Sat Oct 21 11:31:02 2017 +0800
+
+    test_librbd:add a test case of 70 key/val pairs in TestClone
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 3e08577a296bc4ce2c1592849108926b45e9b541)
+
+commit fecfd013a8d2e251057840ee1fd112e20af45a5e
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Fri Oct 20 14:21:09 2017 +0800
+
+    librbd: cannot clone all image-metas if we have more than 64 key/value pairs
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit ccc56384032be7f6bd48e28b6825b3ce589c7cf7)
+
+commit 27a1a7f6fa6e5368fe1b3b56acfd81dbc49f85c6
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Mon Oct 9 17:19:45 2017 +0800
+
+    rgw: fix command argument error for radosgw-admin.
+    
+    Fixes: http://tracker.ceph.com/issues/21723
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit bba3a89591683e6129a514af3144f1c0e8e1cc05)
+
+commit 5e5d5c8196cf99137009be049cca159aca9396ea
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 29 10:29:52 2017 -0600
+
+    qa/suites/rados/thrash: extend mgr beacon grace when many msgr failures injected
+    
+    Fixes: http://tracker.ceph.com/issues/21147
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 431d1482ffef80f9e52826a0425cae899c5dfbda)
+
+commit c22f06b3c5ee51f7bc6f4d1a0340ed7f63c106a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 10:17:07 2017 -0400
+
+    osd/PrimaryLogPG: move cache_mode==none check to top
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5a26ac4d0f643f90d1f513ee33081faa9b0a7e7d)
+
+commit 21880df6bf27465637c97e504b5a6dcf2ea31d9f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 7 18:42:57 2017 -0400
+
+    osd/PrimaryLogPG: send requests to primary on cache miss
+    
+    If a client has {BALANCE,LOCALIZE}_READS and sends a request to a
+    replica, but the object isn't in the cache, send them back to the
+    primary.  Otherwise we might do something rash (like trigger a
+    promotion from a replica).
+    
+    Fixes: http://tracker.ceph.com/issues/20919
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 741a8720996b74434a036b143209111ce5203cbc)
+
+commit 4085e87b84a788a4285b5f18f1cd64556c08038d
+Author: Dmitry Plyakin <dplyakin@gmail.com>
+Date:   Wed Nov 29 16:03:02 2017 +0300
+
+    rgw: fix GET website response error code
+    
+    Change NoSuchKey error code to NoSuchWebsiteConfiguration, when bucket doesn't have website configuration.
+    
+    Fixes: http://tracker.ceph.com/issues/22272
+    Signed-off-by: Dmitry Plyakin <dplyakin@gmail.com>
+    (cherry picked from commit 56344f0e147e1781bb359bfde6878511b077487f)
+
+commit de05d2c81913e608ca9089b40df971f339ed0ba9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 6 15:03:34 2017 -0400
+
+    librbd: refresh image after applying new/removing old metadata
+    
+    Fixes: http://tracker.ceph.com/issues/21711
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ad01a883c5d9139d3ad3176a39a538bb8b247388)
+
+commit 235032ec642a7da077b04b26147cfd86e3ce3d86
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 28 14:00:29 2017 -0400
+
+    rbd-mirror: sync image metadata when transfering remote image
+    
+    Fixes: http://tracker.ceph.com/issues/21535
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9aa5dfb8ec46e8babd0f167dcc0e234a7c60a50c)
+
+commit 35bb3d4d17db84eac03496d22969b69ee6111b2b
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Oct 24 08:49:27 2017 -0400
+
+    mds: fold mds_revoke_cap_timeout into mds_session_timeout
+    
+    Right now, we have two different timeout settings -- one for when the
+    client is just not responding at all (mds_session_timeout), and one for
+    when the client is otherwise responding but isn't returning caps in a
+    timely fashion (mds_cap_revoke_timeout).
+    
+    The default settings on them are equivalent (60s), but only the
+    mds_session_timeout is communicated via the mdsmap. The
+    mds_cap_revoke_timeout is known only to the MDS. Neither timeout results
+    in anything other than warnings in the current codebase.
+    
+    There is also a third setting (mds_session_autoclose) that is also
+    communicated via the MDSmap. Exceeding that value (default of 300s)
+    could eventually result in the client being blacklisted from the
+    cluster. The code to implement that doesn't exist yet, however.
+    
+    The current codebase doesn't do any real sanity checking of these
+    timeouts, so the potential for admins to get them wrong is rather high.
+    It's hard to concoct a use-case where we'd want to warn about these
+    events at different intervals.
+    
+    Simplify this by just removing the mds_cap_revoke_timeout setting, and
+    replace its use in the code with the mds_session_timeout. With that, the
+    client can at least determine when warnings might start showing up in
+    the MDS' logs.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 3321cc7b375a5e0ea1da4ab197ab447639ca4db3)
+
+commit 9981a1db90984b18431017d348f0a3e7ffb76f61
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 18 07:27:49 2017 -0400
+
+    client: add new delegation testcases
+    
+    Test basic acquire/break functionality from both other clients
+    and the same client, for different conflicting opens, as well as
+    changes to the namespace.
+    
+    Then test delegation timeout behavior. Open file, take delegation in
+    main thread. Spawn another thread to open the file again, breaking
+    delegation. Have main thread ignore it, and wait for the spawned thread
+    to be joined. Once it is, ensure that subsequent access of the cmount
+    returns -ENOTCONN.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit d9c6a9129eb65031eb1fc6f769cfe59bf3bb1cff)
+
+commit 0aa3c5c577ce4011b727123b45e89648bba32596
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Dec 8 16:13:19 2017 +0100
+
+    mon: reenable timer to send digest when paxos is temporarily inactive
+    
+    Fixes: http://tracker.ceph.com/issues/22142
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 0457043d2f0040629ec425a356b1bb1f96cf0332)
+
+commit 9490a1551dcb09b9d05e5e49632dfb3277f13981
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Nov 14 07:26:56 2017 -0500
+
+    client: add delegation support for cephfs
+    
+    Add the ability for ceph userland clients to request a delegation for a
+    cephfs open file. With this, userland will get a callback if there is a
+    request for access that conflicts with the delegation. The client is
+    then expected to return the delegation at which point the conflicting
+    access will succeed.
+    
+    Handing out a delegation means that we're trusting the application to
+    give back the caps when they are needed elsewhere. If it fails to
+    uphold its end of the bargain, we need to take steps to forcibly revoke
+    them, so that other clients can make progress.
+    
+    When that occurs, shut down the mount such that further calls return
+    -ENOTCONN. That should prevent the application from holding on to caps
+    indefinitely, and ensure that it will no longer do damage. It should
+    also prevent the client from ending up blacklisted.
+    
+    Since this is an implicit agreement between ceph and the application,
+    we require that the application call ceph_set_deleg_timeout() to set
+    the delegation return timeout before we hand out any delegations. This
+    helps ensure that the various application and ceph cluster timeouts
+    are all in agreement.
+    
+    An open call on ceph can easily return without any caps being granted to
+    the client. This is not generally a problem on ceph since it will wait
+    on caps before doing any operations on that open Fh, but it's less than
+    ideal when there is a delegation outstanding.
+    
+    Both NFS and SMB servers can grant deny locks, and at least in the case
+    of NFS, those can be cached such that the server is unaware of them if
+    the client holds a delegation.
+    
+    Most applications that use delegations will also want to ensure that
+    delegations are broken before allowing an open to proceed. A non-zero
+    delegation return timeout also cues the client to wait on a minimal set
+    of caps after an open return to ensure that this is the case.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit fad99776195e299b37071892e459ad8abf2fddfc)
+    
+    Conflicts:
+        src/client/Inode.cc (luminous does not have
+            a5d97ad257d6fae8f66200513805e1778cb0c8ba so we preserve the following
+            lines at the end of the Inode dtor:
+              delete fcntl_locks;
+              delete flock_locks;
+            )
+
+commit 3a613f6966ef3f32eb5f26ec5ebc5772a4dc2bf0
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Oct 12 08:29:28 2017 -0400
+
+    common: remove data_dir_option from common_preinit and global_pre_init
+    
+    No one ever passes anything in there.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 85f2315bc59106b8fba6099c5e79209ea9b96efa)
+
+commit 5e8c4f83f4f161a595e75340cecbdd41b1803f39
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Tue Dec 5 15:48:58 2017 +0200
+
+    pybind/rbd: raise KeyError when metadata does not exist
+    
+    Signed-off-by: Mykola Golub <to.my.trociny@gmail.com>
+    (cherry picked from commit 7a1b13f8849ce7bd325316bafad5bab660c77825)
+
+commit 2d7bea6148756fd8f3f7e124148d296d4290691f
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Tue Dec 5 15:48:23 2017 +0200
+
+    pybind/rbd: fix metadata functions error handling
+    
+    Fixes: http://tracker.ceph.com/issues/22306
+    Signed-off-by: Mykola Golub <to.my.trociny@gmail.com>
+    (cherry picked from commit 2e6872b28eccd10b8bafdadaf3e6049123792022)
+
+commit c7383d20c4258984de2fb524f2751207aa150d78
+Merge: b3fd46810c 5bd076ef41
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Dec 12 09:04:20 2017 -0600
+
+    Merge pull request #19042 from liewegas/wip-22128
+    
+    mon/OSDMonitor: fix ruleset-* to crush-* fixup
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 08a2358f8520efe03eac435d1f8b2b30e936a021
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 29 12:22:57 2017 -0400
+
+    librbd: avoid dynamically refreshing non-atomic configuration settings
+    
+    Fixes: http://tracker.ceph.com/issues/21529
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ede691323d94dc04a30f81aca5576a3d6d1930af)
+
+commit caf17a876e75cd448947b7b729a8e39689a304df
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Fri Aug 11 17:44:19 2017 +0800
+
+    librbd: notify watcher when updating image metadata
+    
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit b10d26dfa84627b2622d405d272b1133bb773245)
+    
+    Conflicts:
+        src/librbd/image/RefreshRequest.h (luminous does not have
+            233482cf708aa7b4723c529d387c4605bdf0722f so we leave the following variables
+            uninitialized:
+              uint8_t m_order;
+              uint64_t m_size;
+              uint64_t m_features;
+              uint64_t m_incompatible_features;
+              uint64_t m_flags;
+            )
+
+commit 34450ed6c0a324ed86fd47770662554e7cdfeae2
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Dec 11 17:09:25 2017 +0100
+
+    doc: globally change CRUSH ruleset to CRUSH rule
+    
+    Since kraken, Ceph enforces a 1:1 correspondence between CRUSH ruleset and
+    CRUSH rule, so effectively ruleset and rule are the same thing, although
+    the term "ruleset" still survives - notably in the CRUSH rule itself, where it
+    effectively denotes the number of the rule.
+    
+    This commit updates the documentation to more faithfully reflect the current
+    state of the code.
+    
+    Fixes: http://tracker.ceph.com/issues/20559
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit aea9fa01ae630f4ffd2744f577723f5305c718b4)
+    
+    Conflicts: (trivial resolution)
+            doc/man/8/ceph.rst
+            doc/rados/configuration/pool-pg-config-ref.rst
+            doc/rados/operations/pools.rst
+
+commit b013f7fdb5b57725f01fc5305d94d32d7599f732
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Fri Aug 4 11:19:00 2017 +0800
+
+    rgw: revert PR #8765
+    
+    As talk with cbodley in PR(#16716), we shouldn't not mix such dangerous
+    configuration reload with periodically SIGHUP for logs zip.
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 2f30bbb492db3df0522888905d497174e36d0596)
+
+commit b3fd46810c01be00ebf26b9893d8bab107ee93f2
+Merge: 092ea5174c f6fb266699
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 8 10:54:12 2017 +0800
+
+    Merge pull request #19355 from tchaikov/wip-18589-luminous
+    
+    luminous: build/ops: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1d6644d091ab6c3aa64879e0ae68aa697877f32d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Dec 6 12:13:40 2017 -0800
+
+    qa/tests: Add debug info when creating ceph volumes
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9f1a1e0543da6be15b75c3168603cc84a62bd157)
+
+commit 0616bcba07b24ff6aced3048753f8e6535aedf06
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Dec 1 14:16:45 2017 -0800
+
+    qa/tests: add tests for ceph-volume
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 95fb43b54d843e4a7bedaad72b4a71c3489f0dea)
+
+commit bccaa6e627f32725698b544499cab44f3ed72807
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Dec 1 14:11:55 2017 -0800
+
+    qa/tests: update tests to use new ceph-volume syntax
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 3ecaebd0d884033c55a501d218ab426385d92357)
+
+commit bd558d7e729005aece735fedadeb3325efae235d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 7 17:56:55 2017 -0800
+
+    qa/tests: default to wip branch for final upgrade.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 3f28b195d3f391f1a9e36073209f2c895dee89f9
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 7 17:47:01 2017 -0800
+
+    qa/tests: when no branch is specified, use wip branch under test for upgrade
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 307e52109411e237e34b824da78fa5b175b16a9d)
+
+commit f6fb266699f7a39005cb76a2c878bdff5b3a7ff0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 6 10:19:09 2017 +0800
+
+    debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    this is a follow-up of #19328. we need to get this change into 12.2.3.
+    so better off do the switch somewhere after 12.2.2 which has been
+    tagged, and before 12.2.3, which is not tagged yet.
+    
+    please note, this is not targetting master, because i want to make
+    sure the change number (the <num> in << 12.2.2-<num>) is correct. it
+    does not hurt if it's not, as long as it is ">> 12.2.2", so the replace
+    machinery in 12.2.3 works, and it covers the releases where the
+    ceph-{osdomap,kvstore,monstore}-tool are not move yet. but why don't
+    make it more right?
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 57bb57f76d733d6e6dea4fc0682058e4e6abe7df)
+
+commit ee06a16ee8c8434f75adf05c04f94607a5ec090f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 22 12:33:09 2017 +0800
+
+    mds: remove useless check in Migrator::export_dir
+    
+    There is an assert(dest != mds->get_nodeid()) at very beginning of
+    the function. There also is a check for if 'dest' mds is active.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7afe92f644c97ae1282837bc75cde0b4a8054d0f)
+
+commit 443fa3546593f667b01cca7466e3692223c4f226
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 22 11:49:43 2017 +0800
+
+    mds: ignore export pin for unlinked directory
+    
+    Otherwise, stray directory inode may have pinned subtree dirfrag.
+    The subtree dirfrag prevents stray inode from getting purged.
+    
+    Fixes: http://tracker.ceph.com/issues/22219
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit b7cee60467582d7398bbe85e051a722e30c0897e)
+
+commit d57d0945dbb6ec4b46e1749fe7989b48794c3544
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 4 21:25:16 2017 -0600
+
+    debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    The backport didn't make 12.2.2, but it will be in 12.2.3.
+    
+    Fixes: http://tracker.ceph.com/issues/22319
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e0c814266fe653311673e07bd7c7dbb51d866f82)
+
+commit 00503d1880e829024cd1d5e10cca0d388718dedb
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Aug 18 21:52:52 2017 +0800
+
+    doc: add ceph-kvstore-tool's man
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 0667db4eb61338887743d17c93a61c0e89868b08)
+
+commit 92093824d5deddad4dfe4219fee3c95fe4c38f55
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 20:15:16 2017 +0800
+
+    debian: fix package relationships after d3ac8d18
+    
+    d3ac8d18 moves ceph-client-debug from ceph-test to ceph-base without
+    updating the package relationships between the two involved packages.
+    which results in:
+    
+    dpkg: error processing archive /var/cache/apt/archives/ceph-test_12.2.1-241-g43e027b-1trusty_amd64.deb (--unpack):
+     trying to overwrite '/usr/bin/ceph-client-debug', which is also in package ceph-base 10.2.10-14-gcbaddae-1trusty
+    dpkg-deb: error: subprocess paste was killed by signal (Broken pipe)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ed988fc660c5da3cb97e48e82ac5b7ad8f3bcd41)
+
+commit a566ab942b35af11f2cd0fd269f48a4e9fdc3a66
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 15:26:35 2017 +0800
+
+    debian: fix package relationships after 40caf6a6
+    
+    we have issues when running upgrade tests:
+    
+    dpkg: error processing archive /var/cache/apt/archives/ceph-osd_13.0.0-2201-g6cc0b41-1trusty_amd64.deb (--unpack):
+    trying to overwrite '/usr/bin/ceph-osdomap-tool', which is also in package ceph-test 10.2.10-14-gcbaddae-1trusty
+    
+    in 40caf6a6, we moves some tools from ceph-test out into ceph-osd,
+    ceph-mon and ceph-base respectively. but didn't update the relationships
+    between these packages accordingly. this causes the upgrade failure.
+    
+    see https://www.debian.org/doc/debian-policy/#document-ch-relationships
+    for more details on "Breaks" and "Conflicts".
+    
+    the reason why the package version to be replaced/conflicted is 12.2.2
+    is that: i assume that this change will be backported to luminous, and
+    the next release of it will be 12.2.2 .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 50dad805d9f635f2d8085237e94ee8fd0948dd3c)
+
+commit 09fd3c730804e094624cbbbd3e6c4481d1a3a86e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Oct 13 10:34:52 2017 +0200
+
+    build/ops: deb: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    ceph-osdomap-tool into ceph-osd subpackage
+    ceph-monstore-tool into ceph-mon subpackage
+    ceph-kvstore-tool into the ceph-base subpackage
+    
+    Fixes: http://tracker.ceph.com/issues/21762
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 40caf6a6d85fbde930b74df57a62381f6225c3fd)
+
+commit 6dba25e39d07aeb7fe073fc77f3af054c3481ce0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Oct 13 10:34:52 2017 +0200
+
+    build/ops: rpm: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    ceph-osdomap-tool into ceph-osd subpackage
+    ceph-monstore-tool into ceph-mon subpackage
+    ceph-kvstore-tool into the ceph-base subpackage
+    
+    Fixes: http://tracker.ceph.com/issues/21762
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d7b493a7108a68302bc0f48337bf3c253a720266)
+
+commit 092ea5174c069f060bcccc1fa044d5e67da3938c
+Merge: 117e91ef29 753279aac6
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Dec 5 07:01:35 2017 -0500
+
+    Merge pull request #19299 from tchaikov/wip-pr-19196-luminous
+    
+    luminous: ceph-disk: fix signed integer is greater than maximum when call major
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit c514d3d427176eb957c3cd57b99b7ed590468ed7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 28 17:06:47 2017 +0800
+
+    mds: handle client reconnect gather race
+    
+    Fixes: http://tracker.ceph.com/issues/22263
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e4ecf26a8ef622f8290129a682afb1c64b3e7e00)
+
+commit c3c3bb0bc90c980214ff3247823dfbf06046ff95
+Author: Adam Wolfe Gordon <awg@digitalocean.com>
+Date:   Fri Sep 29 15:32:38 2017 +0000
+
+    doc: Update rbd-mirror docs to reflect data pool selection changes
+    
+    Signed-off-by: Adam Wolfe Gordon <awg@digitalocean.com>
+    (cherry picked from commit 57745b94394ac65436a17249733a22c463d51c0c)
+
+commit b634fdaaf0b0f998a62c52ec59a40a4709a41331
+Author: Adam Wolfe Gordon <awg@digitalocean.com>
+Date:   Tue Sep 26 20:30:06 2017 +0000
+
+    rbd-mirror: Improve data pool selection when creating images
+    
+    Previously we used the source image's data pool name
+    unconditionally. There were two problems with that:
+    
+    1. If a pool with the same name didn't exist locally, creation of the
+       local image would fail.
+    2. If the local pool had a default data pool configured it would be
+       ignored.
+    
+    Change local image creation so it uses the default pool if configured,
+    and uses the remote pool name only if a pool with that name exists
+    locally. If neither of those is true, leave the data pool unset.
+    
+    Signed-off-by: Adam Wolfe Gordon <awg@digitalocean.com>
+    (cherry picked from commit 2e239c05518e67b3db80500dbdd3fc5dde30e443)
+
+commit 753279aac6beeacda7fb7c7d15129dff1a27d214
+Author: Song Shun <song.shun3@zte.com.cn>
+Date:   Tue Nov 28 11:28:43 2017 +0800
+
+    ceph-disk: fix signed integer is greater than maximum when call major
+      fix signed integer is greater than maximum when call os.major
+      using python 2.7.5 in Centos 7
+    
+    Signed-off-by: Song Shun <song.shun3@zte.com.cn>
+    (cherry picked from commit f77934b19939796d7ab52daf4dac44846a2ad162)
+
+commit 117e91ef2986f65b059203e23d6efe689a182fb5
+Merge: cf0baeeeeb ce90e0b764
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 09:57:44 2017 -0500
+
+    Merge remote-tracking branch 'gh/luminous' into luminous
+
+commit ce90e0b7648587c4dc8bf354abbfaa15b63b50b1
+Merge: 9c23617d6d 6d02b2a1a2
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 1 13:48:29 2017 +0800
+
+    Merge pull request #19001 from tchaikov/wip-pr-18848-luminous
+    
+    luminous: mgr/dashboard: fix audit log loading
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9c23617d6d3e3f6d6d21de0156e17f9b7cb5f0e9
+Merge: 83684b91a3 77258ed181
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 1 13:47:17 2017 +0800
+
+    Merge pull request #19270 from tchaikov/wip-pr-19225-luminous
+    
+    luminous: qa/suites/rados/singleton: more whitelist
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77258ed181dbe04e56da249a71edd3220baf954a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 29 13:51:49 2017 +0800
+
+    qa/suites/rados/singleton: more whitelist
+    
+    * SLOW_OPS is normal in a cluster with flattering OSDs
+    * so is OBJECT_MISPLACED.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0e987665fe480530a0d1664392604713828bfa5c)
+
+commit 3bfb493fa0c61f33ac930320f7b69ce4a96c2eb6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 28 15:21:55 2017 +0800
+
+    cmake,common/RWLock: check for libpthread extensions
+    
+    pthread_rwlockattr_setkind_np() is a GNU extension of libpthread. and
+    Tianshan Qu pointed out, we cannot use
+    ifdef(PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) to detect the
+    availability of this function, because it's an enum not a macro. so,
+    like other *_np() extensions, we check this one also using cmake at
+    the configure phase.
+    
+    Reported-by: Tianshan Qu <tianshan@xsky.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 5cec46600de8b60385b43cba1e7dc4d1f9a380ac
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Nov 29 15:29:17 2017 -0800
+
+    ceph_disk: allow "no fsid" on activate
+    
+    The intent was to allow "no fsid" configurations when only one
+    conf file named 'ceph.conf' was present, but the code has a bug
+    in that ceph-osd --show-config-value will return a default all-0
+    uuid.  Treat 'all-0' as we were treating 'None'.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 4d010fb83d3cad50953cdf010d1bb20c62588b3c)
+
+commit 7e8dc9763039bf784030480f60b2554f6ff7ffd5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 30 19:32:05 2017 +0800
+
+    ceph-disk: silence deprecate warnings while testing
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ecd4f69c12f57b0d2eacd18bd4bef30f1ce24dd2)
+
+commit d0b0e03ef37a20760e702805fdc95862ec3daeba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 30 19:31:09 2017 +0800
+
+    ceph-disk/tests: mock get_fsid
+    
+    should offer a valid fsid for the ceph-disk under testing.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc49d1d2b99dcc49c60db8ea1ee90caf58650aa4)
+
+commit 747d05c298b13c1d4350b90beb8cec8828262332
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Thu Nov 30 16:05:20 2017 +0300
+
+    luminous: rocksdb: fixes early metadata spill over to slow device in
+    bluefs.
+    
+    Fixes http://tracker.ceph.com/issues/22264
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+
+commit c1c539f1efc8f2fe448bf3c4cbbed8f6401ab24d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 29 15:20:59 2017 -0600
+
+    mon/Monitor: fix statfs handling before luminous switchover happens
+    
+    After the mons are luminous but before we switch over to using the
+    MgrStatMonitor's new info, the version on mgrstat will generally be <<
+    than that of pgmon, and the client will send that version with the
+    request.  This means that the statfs message will perpetually appear to be
+    in the future and fail the is_readable() check.
+    
+    Fix this with any ugly hack that resets the version to 1 if we haven't
+    completed the luminous upgrade yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 82c9e3dbe1b6ff04feae76a089d1a821c3ed819f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 16:45:25 2017 -0600
+
+    mon/MgrMonitor: limit mgrmap history
+    
+    Keep 500 by default (like we do osdmaps and mdsmaps).
+    
+    Fixes: http://tracker.ceph.com/issues/22257
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 00fa8b10990f945f3e875c9850eedeb65af3fd2e)
+    
+    [removed const from get_trim_to() for luminous]
+
+commit bdb9d385a5e1675ac2832300e3cfa1ba6b746ce2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 20 15:42:09 2017 +0800
+
+    cmake: only create sysctl file on linux
+    
+    and check 64bit platform by using the sizeof(void*)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 89a48189ea7b9bd58db1bc95ccd7b3d9fb9b1bb3)
+
+commit acccd2332cd0bc0f8b8f00276ce25a9582c083d7
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Tue Nov 14 16:32:39 2017 +0100
+
+    sysctl.d: set kernel.pid_max=4194304 on 64-bit systems
+    
+    For CONFIG_BASE_FULL Linux kernels, the maximum number of proc/thread
+    IDs is set to 32768 by default. This default limit can be quite easily
+    hit during recovery on nodes with high OSD counts.
+    To avoid hitting the pid_max default limit, attempt to configure it to
+    4194304, which corresponds to the maximum limit possible on 64-bit
+    CONFIG_BASE_FULL kernels.
+    
+    Fixes: http://tracker.ceph.com/issues/21929
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit 14a0c2a727dd71560c3cf18171378ccde6e14699)
+
+commit 718528304063c7ea03d7777c3eaf6e26a0900f59
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Tue Nov 14 15:55:39 2017 +0100
+
+    sysctl: relocate 90-ceph-osd.conf under etc/sysctl/
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit 4d0b70068894af2d9c2c4b6fea1451a47864bfd8)
+
+commit 8f67e4553ac67ff062f3d9d1187a3bd67f02f020
+Author: tangwenjun <tang.wenjun3@zte.com.cn>
+Date:   Mon Nov 20 15:28:29 2017 +0800
+
+    os/bluestore: fix the allocate in bluefs
+    
+    when bluefs succeed to reserve but failed to alloc in db space,
+    
+    it will cause a assert, just because of the space fragmentation.
+    
+    in this situation, it could not use slow device space,
+    
+    and it would happen in stupid or avl allocator.
+    
+    Signed-off-by: tangwenjun <tang.wenjun3@zte.com.cn>
+    (cherry picked from commit d4f868ae7b9df723a9f785cbe6caee1f718e5e33)
+
+commit 17ff1f1fd2bfa744c141741af9dc81761218da63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 20 11:03:17 2017 -0500
+
+    rbd: disk usage on empty pool no longer returns an error message
+    
+    Fixes: http://tracker.ceph.com/issues/22200
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4528122d66d1bc7dd07a64d1af7b65fe53f7b0b)
+
+commit 4dda1b6ead6b1f04f996403881f61e9b7d94dba0
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Nov 19 15:08:18 2017 -0800
+
+    client: anchor Inode while trimming caps
+    
+    This prevents the Inode from being deleted until after cap trimming is
+    finished. In particular, this prevents remove_all_caps from being called which
+    screws up the traversal of caps in trim_caps.
+    
+    Fixes: http://tracker.ceph.com/issues/22157
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 1439337e949c9fcb7d15eb38c22d19eb57d3d0f2)
+
+commit 05b60db8ef1999bb7a75f7cbbc721f64e67cec48
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Wed Nov 8 21:35:54 2017 +0530
+
+    rgw: Fix swift object expiry not deleting objects
+    
+    In cls_timeindex_list() though `to_index` has expired for a timespan, the marker is set for a subsequent index during the time boundary check.
+    This marker is further returned to RGWObjectExpirer::process_single_shard(), where this out_marker is trimmed from the respective shard,
+    resulting in a lost removal hint and a leaked object.
+    
+    Fixes: http://tracker.ceph.com/issues/22084
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 70adfaae5073d2680a9722526a6a19795dd18780)
+
+commit fe5c8b34a7dbaa1585de54668abeae005dc56388
+Author: lvshanchun <lvshanchun@gmail.com>
+Date:   Wed Nov 1 15:52:48 2017 -0400
+
+    radosgw-admin zonegroup get and zone get return defaults when there is no realm
+    
+    Fixs: http://tracker.ceph.com/issues/21615
+    Signed-off-by: lvshanchun <lvshanchun@gmail.com>
+    (cherry picked from commit 2c1653ca379d0bf78dfcfe931d895a16b9bfd21f)
+
+commit 14434beda69510108a37ae14d8d5318c93d78da4
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Thu Nov 9 20:40:15 2017 +0800
+
+    rgw: check going_down() when lifecycle processing
+    
+    Fixes: http://tracker.ceph.com/issues/22099
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit b761989033ac0722a1504238648a806c8004b8e3)
+
+commit 90b56a2903db892bcd736d9f9eeca3a8847b353b
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Tue Nov 14 15:45:39 2017 +0300
+
+    rgw: fix radosgw-admin bucket rm with --purge-objects and --bypass-gc
+    
+    Call RGWRados::delete_bucket() from rgw_remove_bucket_bypass_gc()
+    instead of partial copy of code of RGWRados::delete_bucket().
+    
+    Fix updating user stats after radosgw-admin bucket rm --purge-objects and --bypass-gc
+    Due to rgw_user(const std::string& s) was called incorrect version of rgw_bucket_sync_user_stats().
+    
+    Fixes: http://tracker.ceph.com/issues/22122
+    Fixes: http://tracker.ceph.com/issues/19959
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit db42e385d26ee4d1ef94b900102b705d6a794029)
+
+commit ed24c8ce8f74a9b66d3da3dc2ae3ab5c4a4c86e6
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Fri Oct 27 13:28:03 2017 +0800
+
+    mgr: 'osd status' command return add state
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit f3475c9fdf5b32443df01c1a735c7289066c7508)
+
+commit 85667ad225882cdf4af58166b0d302e69b2d17c0
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Mon Oct 9 13:47:11 2017 +0800
+
+    mgr: fix "osd status" command exception if OSD not in pgmap stats
+    
+    Fixes: http://tracker.ceph.com/issues/21707
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit ce3953248a04c065643938898029cfbf9c843ea3)
+
+commit 23b3c9de3d3a950eb64ff9a4b77da145f7d1d865
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Nov 17 14:48:17 2017 +0100
+
+    rgw: keystone: bump up logging when error is received
+    
+    When keystone engine is used and there is an error (often due to
+    misconfigured urls etc) these do not show up at the final logs as we
+    just return  -1 at the final stage. Adding logs at level 5 for these
+    cases, as they can help identify a possible misconfigured url or admin
+    token. Eg:
+    
+    ```
+    5 Failed keystone auth from http://localhost:5000/v3/v3/auth/tokens with 404
+    ```
+    
+    Fixes: http://tracker.ceph.com/issues/22151
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 347c5148a57de9007d6c1293add6bf27004fce41)
+
+commit 58d6d043b20bd2ce185c560814315f82fc33c399
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Nov 10 14:27:59 2017 -0500
+
+    rgw: set num_shards on 'radosgw-admin data sync init'
+    
+    Fixes: http://tracker.ceph.com/issues/22083
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 90e860c25b87670810426e94b04b515af1e4f154)
+
+commit 239a2f8eb0ad5ba3869d0168cfa5bed81479a9d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 8 10:21:55 2017 -0500
+
+    dencoder/rgw: expose rgw sync status types
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 662b02489c9394a359834cd3d3527bce5b903100)
+
+commit 442120898b8abf22fcd66e06eb8607481653c890
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Nov 9 15:50:56 2017 +0100
+
+    rgw: data sync: set num_shards when building full maps
+    
+    When radosgw-admin data sync init is called on a cluster, the next run
+    of rgw crashes as when it processes ListBucketIndexesCR, num_shards
+    isn't set which is later referenced in ListBucketIndexesCR. Setting the
+    n sync_info.num_shards correctly to handle this case
+    
+    Fixes: http://tracker.ceph.com/issues/22083
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4015a57c473d896164b0617be93777d0947a9576)
+
+commit 2070fcf7d4260e3a13219ce708a46b2c550ccc14
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 10:35:16 2017 -0500
+
+    librbd: invalidating the cache shouldn't hold write lock
+    
+    This can cause deadlock when readahead is in-progress since neither
+    can make forward progress.
+    
+    Fixes: http://tracker.ceph.com/issues/22131
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a335481d20c6a765c84d561a01fb52172eccba4)
+
+commit 3e91197ccd0df2bb0560e057861a2f0639da02bd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 10:34:32 2017 -0500
+
+    rbd-nbd: rescan partition table after image resize event
+    
+    Fixes: http://tracker.ceph.com/issues/22131
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db13e4757451b463e7fb65c43247e033f24d45e5)
+
+commit 818c354a43ca83be10daa1f0cd251635b57371b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 09:09:15 2017 -0500
+
+    librbd: prevent overflow of discard API result code
+    
+    Prevent discard/writesame lengths larger than 2GB.
+    
+    Fixes: http://tracker.ceph.com/issues/21966
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3effd324db181e625665be33b5c6529dca723cc5)
+    
+    Conflicts:
+            PendingReleaseNotes
+            src/librbd/librbd.cc: i chosed to pick code from 3effd32 in order to prevent discard/writesame against lengths larger than 2GB. `len` used in picked code was already in luminous. so it's safe to pick.
+
+commit 5bd076ef41fd8bdd366654507f2e7c287d1f95dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 14:35:48 2017 -0600
+
+    qa/suites/upgrade/jewel-x: move mon_warn_on_pool_no_app to global
+    
+    Check runs on mgr.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit bd3e7795ca9315aa4d550eeef42884f6f0516029
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Oct 27 12:36:06 2017 -0700
+
+    MDSMonitor: only clog changes to active
+    
+    Otherwise we get constant INFO messages that an MDS is active.
+    
+    Fixes: http://tracker.ceph.com/issues/21959
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 997a688d0809221500e0e3dd30e12ccec6c19780)
+
+commit b9bf9b1255d3b3f8d581fcc54e7c39912517f6d0
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Thu Oct 26 14:35:56 2017 +0800
+
+    rgw: add cors header rule check in cors option request
+    
+    fix http://tracker.ceph.com/issues/22002
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit 72e8fc52fb078fa1443e1ba5321718882969aa78)
+
+commit 2d9aafe1bc07cfa65e1bd3d5ea28b575a15eaf53
+Author: Li Wang <laurence.liwang@gmail.com>
+Date:   Wed Nov 1 09:21:29 2017 +0000
+
+    rbd-nbd: fix unused nbd device search bug in container
+    
+    In some container scenarios, the host may choose to
+    map a specific nbd device, for example, /dev/nbd6 into the
+    container, in that case, the nbd device available in the
+    container is not numbered from 0. The current unused
+    nbd device search function will return no result.
+    This patch fixes it.
+    
+    Fixes: http://tracker.ceph.com/issues/22012
+    
+    Signed-off-by: Li Wang <laurence.liwang@gmail.com>
+    Reviewed-by: Yunchuan Wen <yunchuan.wen@kylin-cloud.com>
+    (cherry picked from commit be0f9581f9727187ca03232e0b368e7da7a60609)
+
+commit 210b826ff06fd145f730e2b5303e6472d82fd8e5
+Author: Li Wang <laurence.liwang@gmail.com>
+Date:   Wed Nov 1 06:31:10 2017 +0000
+
+    rbd-nbd: not abort in listing mapped nbd devices
+    
+    In some container scenarios, the file '/sys/block/nbd0/pid'
+    in the container records the host pid of the process
+    which opened the device, therefore the file '/proc/pid/cmdline'
+    corresponding to the pid in the container does not exist,
+    in that case, 'rbd-nbd list-mapped' will cause the
+    following assertion failure, this patch fixes it.
+    
+    src/tools/rbd_nbd/rbd-nbd.cc: In function 'int get_mapped_info(int, Config*)'
+    /src/tools/rbd_nbd/rbd-nbd.cc: 834: FAILED assert(ifs.is_open())
+     ceph version 13.0.0-1632-gf9cfe84 (f9cfe843a1c749d145b6c3b81a0519cc6536eb28) mimic (dev)
+     1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x7fc6b1054ba2]
+     2: (()+0x15c58) [0x560cd4697c58]
+     3: (main()+0x9) [0x560cd46937f9]
+     4: (__libc_start_main()+0xf0) [0x7fc6b003f830]
+     5: (_start()+0x29) [0x560cd46938f9]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    Aborted (core dumped)
+    
+    Fixes: http://tracker.ceph.com/issues/22011
+    
+    Signed-off-by: Li Wang <laurence.liwang@gmail.com>
+    (cherry picked from commit 07d5b81b57f17696f4f693225fcb5a1f7060e9d9)
+
+commit 274fef45c2ca619aaad3ec14e8c6d80d22832a85
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 8 12:01:38 2017 +0200
+
+    rgw: allow tenant without user id for reshard commands
+    
+    Fixes: http://tracker.ceph.com/issues/22046
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 0a2142e83b58fa8e238bcb748d1cb97bdba674c5)
+
+commit 58870ed9111fe3f3e05eadde1fecc8eae7b0eda2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 7 15:37:34 2017 +0200
+
+    rgw: use tenant when updating the reshard log
+    
+    Fixes: http://tracker.ceph.com/issues/22046
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 055c0d34394b8274a7e48d4f2e3913c059099898)
+
+commit 96b924fc750008e66e1cea29a67f1e8c7008a3ed
+Author: Ilja Slepnev <islepnev@gmail.com>
+Date:   Tue Oct 24 20:27:59 2017 +0300
+
+    mgr/zabbix: ignore osd with 0 kb capacity
+    
+    Fixes: http://tracker.ceph.com/issues/21904
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit e532949556453668158d0e261e59e0fae48e461f)
+
+commit 654678ab2de4b4909e1ad12384e790105709ec1a
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Nov 10 22:47:41 2017 +0800
+
+    common/buffers: add function parameter to limit buffers size.
+    
+    Fixes: http://tracker.ceph.com/issues/21932
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit fba63cebba215b36ce850f47bd8e706edaa8023e)
+
+commit 488228e919980d1aba3313bf5f1601e30bfdc03d
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Nov 9 01:00:52 2017 +0800
+
+    osdc/Objecter: record correctly value for l_osdc_op_send_bytes.
+    
+    Fixes: http://tracker.ceph.com/issues/21982
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit c5c7ad56c0d5c758874de0f5ea66504963bbb75e)
+
+commit 134818228b5c0fa54e934c842038de0b9ad63620
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 08:55:59 2017 -0600
+
+    mon/OSDMonitor: fix mon_fixup_legacy_erasure_code_profiles
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6f983c14d94f51b458730a8ef1d1723d8aa8c1e2
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Nov 8 10:30:09 2017 +0100
+
+    tools/crushtool: skip device id if no name exists
+    
+    When an OSD with an id < max_id is removed, i.e. the osd ids are not
+    continuous, crushtool decompile prints bogus info. Skip any device ids
+    without a name.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 85737f94571208f21c972c11530ab56bfaededa2)
+
+commit d5f2e1660055d296640eebcfcaf13435dff212ef
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 14 18:42:02 2017 +0800
+
+    osd: subscribe osdmaps if any pending pgs
+    
+    subscribe from monitor continously for new osdmaps so osd is able to get
+    the osdmap instructing it to delete PGs even if nobody is poking it.
+    
+    Fixes: http://tracker.ceph.com/issues/22113
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4037ab44b30ab23b2e464b24a82fee6fff9a2d08)
+
+commit c6bc756304ff567f39e163e2140d6bb00e8adc6a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 14 17:39:16 2017 +0100
+
+    rgw: set sync_from_all as true when no value is seen
+    
+    In order to support jewel-luminous multisite scenarios where when a
+    jewel cluster is the master, the sync-from-all json field wouldn't be
+    set leading to the secondary not syncing data as this value can't be
+    overriden to true, so defaulting the value of sync_from_all to true when
+    we don't encounter it in the json
+    
+    Fixes: http://tracker.ceph.com/issues/22062
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit e0116fa42020ff5c0d78101b9b3b15e62236b4b8)
+
+commit c21e1c7831e43ed5d07924589b24ce826bb86467
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 8 11:54:44 2017 -0500
+
+    mon: clean up cluster logging on mon events
+    
+    These changes come from observing the output
+    when killing a mon and watching the survivors
+    form a new quorum.
+    
+    Fixes: http://tracker.ceph.com/issues/22082
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 294fe62feeb155e1ec06d508d92435de98cce90a)
+
+commit 6875bdaf5fcc000c4a4615e3f6a03a530bdd74da
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 8 11:53:45 2017 -0500
+
+    mon: monmap log on active should be debug
+    
+    This isn't even a change to the map, just something
+    that got echoed every time a new mon was the leader.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c9eb2793e6074521d161632e20954295527d3653)
+
+commit 1bdd81e095a4bd628c278a3e6abd7faf62ba5399
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 12 19:49:02 2017 -0600
+
+    mgr/balancer: skip CRUSH_ITEM_NONE
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e2cbb4bc7a2badc3ff89fb4bf4d678d8e9f94652)
+
+commit 3efc4c8d542060cd43fa509022063323d1eecaac
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 8 17:37:29 2017 -0600
+
+    qa/suites/rados: stop testing firefly tunables
+    
+    We can't mix the balancer compat-set testing with firefly tunables because
+    it requires that all buckets be straw2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6455954d29e180e24dcadfabce685699477e2736)
+
+commit 5f8a59a48e2399bf41c7a0b6ce9662f6b150ba44
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 8 17:35:35 2017 -0600
+
+    mgr/balancer: fail a bit more gracefully if we can't create a compat weight-set
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 476b2335fd08568d142d17848bfc58dc6e5c86d2)
+
+commit 6d02b2a1a276c98dd3f9881d3e113f1db14fee04
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 9 07:33:10 2017 -0500
+
+    mgr/dashboard: fix audit log loading
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85e1dadb287ee5216985b16b4a144dfe148d6a60)
+
+commit de1d25c2dd31ac731225ee2a3c3d2805513881b4
+Author: Mitch Birti <yahooguntu@users.noreply.github.com>
+Date:   Thu Nov 16 16:27:20 2017 -0600
+
+    udev: Fix typo in udev OSD rules file
+    
+    The rule for lockbox partitions had an invalid UUID for the "change" action.
+    
+    Signed-off-by: Mitch Birti <yahooguntu@gmail.com>
+    (cherry picked from commit 515417294c2548694c2101639c8bbf7f88e716aa)
+
+commit a170e67760ee3e36d6f5c0393710b0d58fd94005
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 22 09:59:12 2017 +0800
+
+    mds: fix CDir::log_mark_dirty()
+    
+    the 'will dirty' check is wrong because we don't always project fnode.
+    
+    Fixes: http://tracker.ceph.com/issues/21584
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 461bbc7e89a9ed440478f30145158b4989c137d0)
+
+commit 0d18c24fcedd82fa0203ce37bac9099bdd455f35
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Wed Nov 8 21:35:54 2017 +0530
+
+    rgw: Fix swift object expiry not deleting objects
+    
+    In cls_timeindex_list() though `to_index` has expired for a timespan, the marker is set for a subsequent index during the time boundary check.
+    This marker is further returned to RGWObjectExpirer::process_single_shard(), where this out_marker is trimmed from the respective shard,
+    resulting in a lost removal hint and a leaked object.
+    
+    Fixes: http://tracker.ceph.com/issues/22084
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 70adfaae5073d2680a9722526a6a19795dd18780)
+
+commit 74a6592d7fc8a4e1d4000025615b72e7297285a1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 26 13:04:26 2017 -0700
+
+    options.cc: document rgw config options
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 79e58a346e743b95b4435fdc72f2fd7ac01e3f1e)
+
+commit 7acb9a8da2c843ac4fb577c3e5eb6c699604da9e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 29 10:51:54 2017 +0800
+
+    common/options: use user-defined literals for sizes
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6788ea0e9a51217cd56b282c40533a8971a36530)
+
+commit ecf102618ec34afee96093812e68382e6b9c4872
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 23 14:35:19 2017 +0800
+
+    common/options: use user-defined literals for default intervals
+    
+    for better readablility
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3704fe283b606737533d62a082337381b95bdbd9)
+
+commit bc94035a30360a8c2353afb5f0355e0675abbdeb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 24 17:08:00 2017 +0800
+
+    common/options: use user-defined literals for default sizes
+    
+    for better readablity.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d9b38a1d6172a43e7ebb755be5cf044f19d7035d)
+
+commit 7a8e915483397d740e3be3815ece23f4cd3c6b7a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 11 11:36:36 2017 +0100
+
+    mds: additional damage handling case in EImportStart
+    
+    Fixes: http://tracker.ceph.com/issues/21759
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit b4c662c2cf47f7b6f952e776d0212156861eaffb)
+
+commit 1ec93753a37a24a7e72566848ccc0baab1df84ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Aug 2 17:26:56 2017 +0800
+
+    mds: track snap inodes through sorted map
+    
+    Current mds track both head inodes and snap inodes through unsorted
+    map. The unsorted map makes finding snap inode that follows a given
+    snapid difficult. Currnt MDCache::pick_inode_snap() use snap set to
+    guess snap inode's last. The method isn't reliable because snap set
+    may change after creating the snap inode. For example:
+    
+    MDS cows inode[2,head] with snap set[5,6], which results inode[2,6]
+    and inode[7,head].
+    
+    Later mds wants to find snap inode that follows snapid 2. But the
+    snap set become [5], mds can't find snap inode [2,5].
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7b9eae62c8c654ff82684451c222257d2c93be64)
+    
+    Conflicts:
+            src/mds/MDCache.cc: when i cherry-picked 3ca602e, there were conflicts in same file. to fix conflicts, i need to introduce snap_inode_map because it's not in luminous. to intoroduce this, i need to cherry-pick 7b9eae62. After cherry-picking 7b9eae62, there were conflicts. i picked code from head because it's introduced by 3ca602e which is required to fix http://tracker.ceph.com/issues/21928.
+
+commit 85791328fc563932673e2d6ad8701179fee0c3a3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 26 16:21:41 2017 +0800
+
+    mds: fix inode count check in MDCache::check_memory_usage()
+    
+    Fixes: http://tracker.ceph.com/issues/21928
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 3ca602e3bdaa211ae2446d4bbf62431e14d41bef)
+    
+    Conflicts:
+            src/mds/MDCache.cc
+
+commit e485b89ffd5754b5c6356ccd018fe602d7e55e07
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 20 17:03:10 2017 +0800
+
+    mds: disable early reply for dir layout and quota related requests
+    
+    These requests impacts whole subtree tree, replaying them when
+    mds recovers may break order of requests in multimds cluster.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit edd76fc88377b398b5f2d274684d10f6ed3314cd)
+
+commit cb8eff43b1abd8c268df9e57906d677ff4be8d95
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 18 20:58:15 2017 +0800
+
+    mds: don't rdlock locks in replica object while auth mds is recovering
+    
+    Auth mds may take xlock on the lock and change the object when replaying
+    unsafe requests. To guarantee new requests and replayed unsafe requests
+    (on auth mds) get processed in proper order, we shouldn't rdlock locks in
+    replica object while auth mds of the object is recovering
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0afbc0338e1b9f32340eaa74899d8d43ac8608fe)
+
+commit f1be92ee34399617cad31a489ec49e993c6eaff6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 18 20:35:33 2017 +0800
+
+    mds: move ScatterLock::state_flags into SimpleLock
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit da6e50890de8b682eb5756d0e4e08583908b3778)
+
+commit ebd7a6670fbb9724bf72f7b0587e5c18ade03a48
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Aug 11 17:52:19 2017 +0800
+
+    mds: make mksnap/setlayout wait for unsafe requests on other mds
+    
+    This guarantees replayed unsafe requests (on other mds) and mksnap/setlayout
+    get processed in proper order.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit dee3711bd745704fba224dc6f90e88c04cacff58)
+
+commit a88e48e7d9cba77db8cb5aeb9512733d62a08382
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Oct 10 15:51:04 2017 +0800
+
+    rgw: don't change rados object's mtime when update olh
+    
+    null instance and olh share rados object. Null instance using the rados object mtime as its mtime. When olh updated, the rados object mtime will change.
+    Thus the null instance will be incorrect. When list objects in bucket, the mtime is correct, down the null instance the last modified time is incorrect.
+    So when update olh, using the previous mtime, thus the null instance mtime will be correct.
+    
+    Fixes: http://tracker.ceph.com/issues/21743
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 3189edc28ea6d3c117c2e2f0664f45c69f67ab50)
+
+commit 1410b5a5f5505ba7df0fdf313eefee4306807f3a
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Oct 9 14:33:05 2017 +0800
+
+    mds: no assertion on inode being purging in find_ino_peers()
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit e55b2dbfcacb3b5afddba2d2b24c0386a0a48ebb)
+
+commit 2dc009e68ef9e850aebcc55634a1505212786309
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 15:57:18 2017 -0400
+
+    rgw: Fix dereference of empty optional
+    
+    Due to the lack of a return, there was a case where an invalid ARN
+    could cause a dereference of an uninitialized boost::optional.
+    
+    As a bit of defensive programming, restructure a couple functions to
+    make that kind of error impossible by ensuring the optional is only in
+    scope when it is initialized and relying less in early return on
+    error.
+    
+    Fixes: http://tracker.ceph.com/issues/21962
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 5249139be7a2748eabbf898cf340989875bfa509)
+
+commit dbd70f0b82a8eb512614b68d8de0c861da9e9d47
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 23 14:23:32 2017 -0400
+
+    rgw: fix extra_data_len handling in PutObj filters
+    
+    the RGWPutObj_Compress filter relies on a starting offset of 0 to ensure
+    that we only compress entire objects
+    
+    for RGWRados::fetch_remote_obj(), we also fetch object metadata. the
+    replies come back with a 'Rgwx-Embedded-Metadata-Len' header, which
+    specifies how many bytes of object metadata are at the front of the
+    request body. when this is present, the offsets passed from
+    RGWRadosPutObj down to the RGWPutObjDataProcessor filters are offsets
+    into the http response body, rather than logical offsets into the object
+    data itself
+    
+    this commit adds a transformation to RGWRadosPutObj so that only that
+    logical offset is visible to the RGWPutObjDataProcessor
+    
+    Fixes: http://tracker.ceph.com/issues/21895
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ac7ffe6ef7faef4e251c970ce2efd04616c0459c)
+
+commit 9f9db135a058c3134a96a845900bb795d8e708fc
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Oct 1 08:40:27 2017 +0300
+
+    doc: replace region with zonegroup in configure bucket sharding section
+    
+    Fixes: http://tracker.ceph.com/issues/21610
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8a1034bccdd514a3eaa6abfdc62c5e3d1e46d5cf)
+
+commit eeb12253d4caec2194970a84a9440d7ccf372d3d
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Oct 17 09:48:15 2017 +0200
+
+    tools/ceph-conf: dump parsed config in plain text or as json
+    
+    This is useful for finding differences between ceph.conf on disk
+    and in osd/mon memory.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 951434f3a40fd697e5eae8fbcbf779abceb3ba34)
+
+commit 8cb21e6ea6fa0abe50d0c5d2b2070963c3ee3de2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:45:54 2017 -0400
+
+    cls/journal: ensure tags are properly expired
+    
+    Previously, if only the local image was using the journal or if
+    a disconnected peer was attached, the tag entries could not be
+    expired even if unreferenced.
+    
+    Fixes: http://tracker.ceph.com/issues/21960
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 19fa1c7f5b2809e9a223b7b196dfc031e97a5dcd)
+
+commit c86675dadc8d97c60cb94f32713c9819941607b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 26 10:57:20 2017 -0400
+
+    test/librbd: added update_features RPC message to test_notify
+    
+    Fixes: http://tracker.ceph.com/issues/21936
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b003ff320036437b42bd8f31c0096162a3c47fca)
+
+commit efd4147fa895ea86b65c807aee7991c016c4d7aa
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Thu Nov 2 16:30:17 2017 -0400
+
+    cephfs: Do not check auth gid when not specified
+    
+    For auth caps that omit the gid, do not check for a gid match.
+    
+    Fixes: http://tracker.ceph.com/issues/22009
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 0e2cfdf507ab3ec8459bc6ea9b73b7a1285274d0)
+
+commit 369dcb36c70ec6c9ab974eb6b7542068ed9fcbda
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 3 12:03:49 2017 -0400
+
+    librbd: added preprocessor macro for detecting compare-and-write support
+    
+    Fixes: http://tracker.ceph.com/issues/22036
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 95d716cc7098cb04063649584e0774f49a580e1d)
+
+commit 4db6b35548fdbf5cd2a84413933170547abf9a01
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 12:08:03 2017 -0400
+
+    mgr: emit cluster log message on serve() exception
+    
+    Fixes: http://tracker.ceph.com/issues/21999
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f4763c32fa84f57c93c559d742203fb3fd2985a8)
+
+commit d73deb017ce9ad2525a2bda54553c01f386f0b35
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 31 11:55:29 2017 -0700
+
+    ceph.in: pass RADOS inst to LibCephFS
+    
+    This avoids multiple instances of the admin socket and other redundancies.
+    
+    Fixes: http://tracker.ceph.com/issues/21967
+    Fixes: http://tracker.ceph.com/issues/21406
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 76a950b2ec473741ed3cffca5c198e8c56f32a1c)
+
+commit ab103b9fe36ad0dd7e470c4c2cf701963cc8bf3c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 30 20:03:29 2017 +0800
+
+    mds: trim 'N' log segments according to how many log segments are there
+    
+    Config 'mds_log_max_expiring' is 20 by default. It means that at most
+    20 log segments get trimmed in each tick. For busy cluster, this can
+    cause mds behind on trimming log segments.
+    
+    Fixes: fixes: http://tracker.ceph.com/issues/21975
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7de37382ace36b6b724b07dcd58178137c49e6b8)
+
+commit ddba907279719631903e3a20543056d81d176a1b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 31 16:56:51 2017 +0800
+
+    mds: fix MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 definition
+    
+    Fixes: http://tracker.ceph.com/issues/21985
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 6c1543dfc55d6db8493535b9b62a30236cf8c638)
+
+commit e0186c9e259898286b720e0e671e708f580cbf25
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 31 16:37:07 2017 +0800
+
+    mds: remove useless incompat feature definination in FSMap.h
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 990088ef16f901588169cd20ed99d0415ec1f7f4)
+
+commit cb38378e5dc4821c232b15a99b30aeeb7b9f9744
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 21:10:56 2017 +0800
+
+    rgw: loadgen fix rgw crash issue
+    
+    When generating random bucket name and object name during
+    loadgen processing, RGW will crash. The reason is calling
+    "gen_rand_alphanumeric" with "cct = NULL".
+    
+    Fixes: http://tracker.ceph.com/issues/22006
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 7d021782e29c74eca36a9bb6ca59930f87093115)
+
+commit ed87de0622cdb1ac93915967949d581ed62e31d9
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Tue Oct 31 14:20:14 2017 +0800
+
+    rgw: modify_s3_type_subuser_access_permissions_fail_through_admin_rest_api
+    
+    fix: http://tracker.ceph.com/issues/21983
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit bd1385d4766ee496bf87587dd40086ee3d510672)
+
+commit 03d1cd996cf189d10a23c839aead4075cbdb888f
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 18:56:08 2017 +0800
+
+    rgw: lc fix rgw crash when lc configuration xml not including ID
+    
+    When a putting lc config request does not include an ID tag in lc
+    configuration xml, RGW should generate a random ID for the lc
+    configuration.
+    
+    At present RGW will crash when generate a random ID for
+    lc configuration.
+    
+    Fixes: http://tracker.ceph.com/issues/21980
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 3d2f63de494f8cd065d3af3cfa49e111a294622c)
+
+commit 5e44bedf293beffb6823ae866373314a3675c431
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 02:36:47 2017 +0800
+
+    rgw: lc support Content-MD5 in request headers
+    
+    According to AWS S3, this header must be used
+    as a message integrity check to verify that
+    the request body was not corrupted in transit.
+    
+    Content-MD5 is the base64-encoded 128-bit MD5
+    digest of the data
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 9c0abe6d638c92f1f29798afa846f8a80fa64814)
+
+commit 860625046d4a0b79e01b0bbcc5027fbbbfbad07a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 1 17:49:46 2017 +0200
+
+    rgw: add missing current_history initialization
+    
+    Fixes: http://tracker.ceph.com/issues/21996
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit cc3246cf46ba26604e4b3bef1f8fe0c9990bdeb6)
+
+commit 32ca69b0d9fec21f1ecb94707850eb9d3322c435
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 1 12:15:35 2017 +0200
+
+    rgw: init oldest period after setting run_sync_thread
+    
+    Fixes: http://tracker.ceph.com/issues/21996
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8b301679161d1405fdae379ff93c33d4d637698d)
+
+commit f0914619a5cc0d9108c4c6efd0f3464ec4fd0fff
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 31 18:42:51 2017 +0800
+
+    mon/PGMap: use new-style options
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7a23097c34c35c1bf6ec09e86ed3acbb0807068c)
+    
+    Conflicts:
+            src/common/legacy_config_opts.h
+            src/mon/PGMap.cc: in 729a089, PGMap::get_health() is removed
+    from master branch, but in luminous, this function is still around for
+    backward compatibility. but that function is still using the old-style
+    options. in this change, i choose to keep some of the old-style options
+    removed in the cherry-picked cleanup to appease PGMap::get_health(),
+    instead of migrating all of them to the new-style because of less
+    efforts.
+
+commit fd956acfd674b32a7518cb4699a500fc4c845172
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 17:11:24 2017 -0400
+
+    tools: update monstore tool for fsmap, mgrmap
+    
+    Fixes: http://tracker.ceph.com/issues/21577
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0ad7bd0b43b473211de30ed9f2050059e069e4d3)
+
+commit 0ebc4cc18f974975a27c77212ea0c38bcbdcb67c
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 14:59:26 2017 -0400
+
+    tools: handle decode errors in monstore tool
+    
+    Print a single line message instead of dumping
+    a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61ceafc8bba3747b492f8b0f779793813a682ecb)
+
+commit b36bd48204f80b47e3d90189fce366caad474650
+Author: Neha Ojha <nojha@redhat.com>
+Date:   Thu Sep 28 09:50:24 2017 -0700
+
+    osd: hold lock while accessing recovery_needs_sleep
+    
+    Signed-off-by: Neha Ojha <nojha@redhat.com>
+    (cherry picked from commit 4bf8d13273b66cc89a94e55259b6fcb106061bd6)
+
+commit 8941606f5e559850dd968ea2966c2364c09b976a
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Oct 9 14:16:05 2017 -0700
+
+    common: by default, do not assert on leaks in the shared_cache code
+    
+    Update the standard qa suite cluster configs so that we continue
+    asserting in our nightlies, but users don't hit this.
+    
+    Fixes: http://tracker.ceph.com/issues/21737
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 165b61a7df2731602e9e28c5107a873444bf0507)
+
+commit acccae56c4b67bbb0a788e8ca7b248e9b8826285
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 16:03:38 2017 -0400
+
+    osd: fix waiting_for_peered vs flushing
+    
+    on_flush() requeues waiting_for_peered, but we flush twice on the
+    primary during peering, and we don't want to requeue the first one
+    (when we have the master pg log merged).
+    
+    Fix by moving waiting_for_peered to waiting_for_flush if we aren't
+    already flush on _activate_committed.  If we get an op and are
+    peered but not flushed, queue ops there.  (We can simplify this
+    check a bit since pgbackend inactive message handling doesn't care
+    about flushed or not flushed.)  When flushed, we requeue
+    waiting_for_flush.
+    
+    The waiting_for_flush, waiting_for_peered, and waiting_for_active
+    lists are all mutually exclusive, so this mostly serves to
+    clarify what we are waiting for (not to keep items separate). And
+    it means that on_flushed() will only requeue things that were
+    waiting for it specifically.
+    
+    Fixes: http://tracker.ceph.com/issues/21407
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8f7dc8b0d8135b20322c09a5909ad43d4c83b0ea)
+
+commit 5234ef2bcaa905861e7c15a9774b82467b2e84e7
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Mon Oct 23 12:38:56 2017 +0800
+
+    msg/async/AsyncConnection: state will be NONE if replacing by another one
+    
+    Fixes: http://tracker.ceph.com/issues/21883
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit f2eb981377f4061debfa67cbd88ca2921a9dfb27)
+
+commit c281456346f0c65e3fa4bb8ca0baf42f4d16448e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 20 09:32:14 2017 -0400
+
+    osdc/Objecter: delay initialization of hobject_t in _send_op
+    
+    Fixes: http://tracker.ceph.com/issues/21845
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eca4fafbcfa922d9e2e98564e69596f6c0b1b93c)
+
+commit dfebcce5a445793267ec8979bcb56ec04954d154
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 22:24:31 2017 -0400
+
+    osdc/Objecter: skip sparse-read result decode if bufferlist is empty
+    
+    If the OSD does not execute sub-ops due to errors encountered prior to
+    the sub-op, the sub-op result remains zeroed with empty out data.
+    Attempting to decode the empty bufferlist results in large exception
+    handling CPU overhead.
+    
+    Fixes: http://tracker.ceph.com/issues/21844
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dc9b309d03074862daad9ef05ef643da870f6722)
+
+commit 7012cf4b60c34dc7f41db9acbdeeeab20e8cd80a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 15:20:49 2017 -0500
+
+    ceph_test_objectstore: do not change model for 0-length zero
+    
+    We almost fixed this in the pr merged at 2dbbb351e2e0e3703880023bf51f55790f763e04
+    but missed this piece.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 390779d1aa9b46648e4f5390dc431c255c70385d)
+
+commit 51b380a0a74f0335a04d1121042449909549c976
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:32 2017 -0500
+
+    os/bluestore: 0-length zero should not change object size
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b588eaf2b0fdf06c94104d5a542bd571499f2b85)
+
+commit 45d70b1329a8c635d2ac0ea5ec171cbea0f6ce02
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:15 2017 -0500
+
+    os/filestore: make 0-length zero avoid touching object length
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 979b7c9cd2e4919b8c7d5ed2418019310959f4bf)
+
+commit 0d921686080d0d1ce81c55a0e8926951fd9ffa58
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:28:49 2017 -0500
+
+    os/ObjectStore: 0-length zero does not change length of object
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ad1f4f10ff7bfe32d0a37361640fe5c65e56699)
+
+commit bacebf0fd7f841185236c1dc6754955da1dfb9b2
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Oct 16 17:31:12 2017 +0300
+
+    mgr: skip first non-zero incremental in PGMap::apply_incremental()
+    
+    After initialization of PGMap instance PGMap::stamp is zero
+    and this cause huge first delta.
+    Also after mgr restart first non-zero value coming to PGMap::apply_incremental()
+    is current pg_sum value so it produces unreasonably huge pg_sum_delta.
+    This patch introduces a workaround to save pg_sum and not update pg_sum_delta
+    by first non-zero incremental.
+    
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    Fixes: http://tracker.ceph.com/issues/21773
+    (cherry picked from commit 013a3803c931ac2e5c390dcd209e1dd0a58418b9)
+    
+    Conflicts:
+            src/mon/PGMap.cc
+
+commit b6f50a463a2757b5822ad5143c4ba8eb86e8d27e
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Wed Aug 9 10:22:19 2017 +0800
+
+    mgr/Mgr: implement completion of osd MetadataUpdate
+    
+    Fixes: http://tracker.ceph.com/issues/21159
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit cb26eead3f195dd51ae79ec9e41700cdde674648)
+
+commit 948ca443fcdf876436a66cc0188cb7e99e1cb068
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 07:02:21 2017 -0400
+
+    mgr/dashboard: redirect away if viewed filesystem is removed
+    
+    Previously this would sit there with stale data in the browser
+    if you were viewing a filesystem and then deleted it in the
+    background.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 83800cd7c11e229e291c699ed45bc65c7fcd9357)
+
+commit 2ed0f0918f4d2d02c57e0ed65c6d790aa185ad8d
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 07:01:48 2017 -0400
+
+    mgr/dashboard: 404 instead of 500 on missing filesystem
+    
+    Still not the most beautiful but lets reserve 500s
+    for real errors.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e62cb703c64dbe177cc69d904ed4888b74cdb86)
+
+commit ed2dcad99b9d2ebb7d8d3b836b4761b0c10555f5
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 06:43:24 2017 -0400
+
+    mds: fix FSMap copy constructor
+    
+    This was confusing ceph-mgr, which did a fsmap = new_fsmap
+    and found that old filesystems were never disappearing
+    after being removed.
+    
+    Fixes: http://tracker.ceph.com/issues/21599
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 516896d3c93ace4e610960f0f2b7a9824df3f7f2)
+
+commit 9a7e1f6a448e57f5ae24844d8a7edb0fd4963933
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 09:41:56 2017 -0400
+
+    mds: set PRIO_USEFUL on num_sessions counter
+    
+    This is used by dashboard and status modules,
+    so let's make sure we're sending it to the mgr.
+    
+    Fixes: http://tracker.ceph.com/issues/21927
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1d0a7d1f35e1c1be472b0555323a9d2ac2d2d777)
+
+commit 5c9d212f94160715a44dca79f18be1c8b2f7ce55
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 12:26:07 2017 -0400
+
+    cmake/cls: add install() for ceph_test_cls_log
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9e94244db2be48bc6ed198a093d7bbb8ae8a72b1)
+
+commit b228d35e1df2302215d751ed35988a34f1238bef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 30 15:31:03 2017 -0400
+
+    qa: add ceph_test_cls_log to cls workunit
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 83d6b493f71db9417c7b2b7db5f1da9c2d71eec3)
+
+commit 9f09b1bb1f032d626992fa9b63ac613ee0969f3d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Oct 28 16:28:27 2017 -0400
+
+    osd: add processed_subop_count for cls_cxx_subop_version()
+    
+    cls_log_add() relies on cls_cxx_subop_version() to generate unique keys
+    for log entries with the same timestamp. because cls calls back into
+    do_osd_ops(), resetting current_osd_subop_num means that cls_log_add()
+    will keep seeing the same subop version and generating the same keys.
+    this causes the following failure in ceph_test_cls_log:
+    
+    [ RUN      ] cls_rgw.test_log_add_same_time
+    /home/cbodley/ceph/src/test/cls_log/test_cls_log.cc:144: Failure
+          Expected: 10
+    To be equal to: (int)entries.size()
+          Which is: 1
+    [  FAILED  ] cls_rgw.test_log_add_same_time (1180 ms)
+    
+    Fixes: http://tracker.ceph.com/issues/21964
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 95faac9ae2cd38b1ca25c90f90bd4a53fe6af4d4)
+
+commit 1d4435f0cda503b9742f1cad72930bf09bfce265
+Author: Peter Keresztes Schmidt <carbenium@outlook.com>
+Date:   Sun Oct 15 06:36:54 2017 +0200
+
+    rbd: fix crash during map
+    
+    Currently the iterator isn't advanced after the erase call leading to a
+    second call on the iterator, which crashes due to a double free.
+    Since C++11 the map::erase function returns an iterator pointing to the
+    next element. Use the return value to set the iterator after erasing.
+    
+    Fixes: http://tracker.ceph.com/issues/21808
+    
+    Signed-off-by: Peter Keresztes Schmidt <carbenium@outlook.com>
+    (cherry picked from commit 9e49c4124422e58dd40dfb6038425430d3845412)
+
+commit 767ad8518d670e396b1a6a9d9574c5b8479673d1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 4 10:46:46 2017 -0400
+
+    librbd: list_children should not attempt to refresh image
+    
+    The snap_lock is being held when this method is invoked, which can
+    result in a deadlock.
+    
+    Fixes: http://tracker.ceph.com/issues/21670
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4c585d826f38ff97d3a484a30eca0588c79396b4)
+
+commit 22ff1477348f2fbfe94ef6a08b09687916734aa6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Aug 24 17:06:30 2017 +0800
+
+    mds: fix StrayManager::truncate()
+    
+    old code does not set PurgeItem::action
+    
+    Fixes: http://tracker.ceph.com/issues/21091
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 45fd51254523f5707e5a67dd7c6ba6011f80e179)
diff --git a/doc/changelog/v9.2.1.txt b/doc/changelog/v9.2.1.txt
new file mode 100644
index 000000000..cdd9e95d5
--- /dev/null
+++ b/doc/changelog/v9.2.1.txt
@@ -0,0 +1,1637 @@
+commit 752b6a3020c3de74e07d2a8b4c5e48dab5a6b6fd (tag: refs/tags/v9.2.1)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Wed Feb 24 22:07:26 2016 +0000
+
+    9.2.1
+
+commit 71f380a81c6870466e11a74a597f847494ba23e9
+Merge: 8350215 b0c9176
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:53 2016 +0700
+
+    Merge pull request #7484 from dillaman/wip-14610-infernalis
+    
+    librbd: flattening an rbd image with active IO can lead to hang
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 83502159f5b94139784b908dd4e4bdf1d420e62a
+Merge: f5e33cd 46ed443
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:41 2016 +0700
+
+    Merge pull request #7406 from dillaman/wip-14542-infernalis
+    
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f5e33cd348532b4ed911f49240a5da0c6859a9ad
+Merge: a474c09 389ed34
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:25 2016 +0700
+
+    Merge pull request #6981 from dillaman/wip-14062-infernalis
+    
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b0c91768d4ac81baaf9b5edec92adad91ade9b64
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 12:06:50 2015 -0500
+
+    librbd: ensure librados callbacks are flushed prior to destroying image
+    
+    Fixes: #14092
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98157ab3274bd960e4487e34f5a83e9c921a6ac8)
+
+commit f3987367920368cbce865533cf11a1eb207fb9c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 22:31:55 2015 -0400
+
+    librbd: simplify IO flush handling
+    
+    Add a new convenience method to ImageCtx for handling flush
+    requests and cleanup flush handling with dealing with the cache.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (based on commit ee7c6f73992d3b09c6b401fbb782b2151f2399c7)
+
+commit da772fa88db47703d491ebaa9bd709e62858b3b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 14 13:28:13 2015 -0400
+
+    WorkQueue: PointerWQ drain no longer waits for other queues
+    
+    If another (independent) queue was processing, drain could
+    block waiting.  Instead, allow drain to exit quickly if
+    no items are being processed and the queue is empty for
+    the current WQ.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b118d7df1e34387b6e5649a5b205cf061598d0d4)
+
+commit 5593e57c1b1c198e95a636ffdda4922259582a5f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 2 10:54:53 2016 -0500
+
+    test: new librbd flatten test case
+    
+    AIO operations after a flatten operation were previously
+    hanging during the close of the parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5b3a4d2cbca51e5c6795ba7d1189920c7d9af806)
+
+commit 46ed443bf674c50a04705597f1caa4882c1ca5b3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:38:20 2016 -0500
+
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Blocking the notification thread will also result in librados async
+    callbacks becoming blocked (since they use the same thread).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6f94bde44500cc4592ac9a842cbb150b8cabf96b)
+    
+    Conflicts:
+        src/librbd/ImageWatcher.[cc|h]: fewer RPC messages and synchronous
+                                        snapshot actions
+
+commit 2be722a3e944436deff7a8ae513e22b02fd31237
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:35:54 2016 -0500
+
+    librados_test_stub: watch/notify now behaves similar to librados
+    
+    Notifications are executed via the same librados AIO callback
+    thread, so it's now possible to catch deadlock.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a3822f1559ba3fe3def6a65883b9c6c7c5a33fe)
+
+commit 8990ee87c9d943f01de68e772da213dc7aad9011
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 12:40:18 2016 -0500
+
+    tests: simulate writeback flush during snap create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e564ea9f869b987f3ada2465edfbe5edf9f6435)
+
+commit 389ed3462f6377d48d87a0e0952061585b0ea106
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 18 15:22:13 2015 -0500
+
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Fixes: #14062
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (derived from commit 68125dd01349edf93cfa1af5028c2d438b5ae089)
+
+commit a474c096a3084ef9f8453e76cebff49e4649d4ea
+Merge: 4c04050 0f50439
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 13:59:30 2016 +0700
+
+    Merge pull request #6629 from Abhishekvrshny/wip-13733-infernalis
+    
+    rbd: misdirected op in rbd balance-reads test
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4c04050379e5d8d8512dc51df6b0ecbd1e37dcd7
+Merge: 1aa60f6 29d30ec
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 13:00:23 2016 +0700
+
+    Merge pull request #7431 from Abhishekvrshny/wip-14067-infernalis
+    
+    infernalis : Ceph file system is not freeing space
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1aa60f6680195a557ce81eeabc6794b4c84d87b1
+Merge: 4dc9791 7fd56ea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:59:30 2016 +0700
+
+    Merge pull request #7429 from Abhishekvrshny/wip-14490-infernalis
+    
+    infernalis: fsx failed to compile
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4dc97912b6003069bc1582caf18bbc702a8864be
+Merge: 2bfc75a a2644ed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:57:48 2016 +0700
+
+    Merge pull request #6853 from Abhishekvrshny/wip-13889-infernalis
+    
+    infernalis: Segmentation fault accessing file using fuse mount
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 2bfc75a0c32685f1151c923a2d84a29549b89ec9
+Merge: e71d062 6c68971
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:57:09 2016 +0700
+
+    Merge pull request #6752 from ukernel/infernalis-11482
+    
+    mds: fix client capabilities during reconnect (client.XXXX isn't responding to mclientcaps warning)
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit e71d0629caaf0d639efdc9f081bcc2aa8ce156e1
+Merge: 6552eeb 58b43e6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:53:34 2016 +0700
+
+    Merge pull request #6628 from Abhishekvrshny/wip-13792-infernalis
+    
+    rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6552eeb2c687e5c7eb8ffbed942f6d99b1c969f9
+Merge: e507a5f 7a44714
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:58:19 2016 +0700
+
+    Merge pull request #7079 from Abhishekvrshny/wip-14199-infernalis
+    
+    infernalis: [ FAILED ] TestLibRBD.SnapRemoveViaLockOwner
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e507a5f90f5ec857359a985b28c64d3cb6366bc4
+Merge: 930ef94 83bba36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:58:01 2016 +0700
+
+    Merge pull request #7080 from Abhishekvrshny/wip-14142-infernalis
+    
+    infernalis: Verify self-managed snapshot functionality on image create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 930ef9462ad1d5a92225f2b83ef797b37e22aeba
+Merge: 1061db6 8175e16
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:57:37 2016 +0700
+
+    Merge pull request #7428 from Abhishekvrshny/wip-14321-infernalis
+    
+    infernalis: cls_rbd: object_map_save should enable checksums
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1061db6ba16955ef2f3fa49dfed461872b970614
+Merge: 2de6016 2c5b90a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 22:23:18 2016 +0700
+
+    Merge pull request #7427 from Abhishekvrshny/wip-14465-infernalis
+    
+    infernalis: rbd-replay does not check for EOF and goes to endless loop
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2de60165dd76ece7dc84326e2721d61cde05e004
+Merge: a8d147e f870ba9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 22:23:01 2016 +0700
+
+    Merge pull request #7426 from Abhishekvrshny/wip-14552-infernalis
+    
+    infernalis: rbd: TaskFinisher::cancel should remove event from SafeTimer
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a8d147e813d157855f65e271569cedb0ac95a72d
+Merge: 77a26d8 fde4ec6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:49:41 2016 -0500
+
+    Merge pull request #6397 from SUSE/wip-13615-infernalis
+    
+    OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77a26d8e7bdb4f351c90a61e8ac87b251294a461
+Merge: cea356a af2ced5
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:49:17 2016 -0500
+
+    Merge pull request #6840 from SUSE/wip-13791-infernalis
+    
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cea356a959ed410a6094140c9d03852e968ab7a8
+Merge: 5469e99 5ef86d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:49 2016 -0500
+
+    Merge pull request #6851 from Abhishekvrshny/wip-14018-infernalis
+    
+    infernalis: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5469e9904ccc170d282d061308f6694e20ef8946
+Merge: ba4583f 7ac5b15
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:25 2016 -0500
+
+    Merge pull request #6849 from Abhishekvrshny/wip-13979-infernalis
+    
+    osd: call on_new_interval on newly split child PG
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ba4583f3a25a0d01d31e82cdfcd2be414a5648ae
+Merge: fbd96c8 b211686
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:03 2016 -0500
+
+    Merge pull request #6907 from Abhishekvrshny/wip-13929-infernalis
+    
+    infernalis: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fbd96c8a49548a4cd6598c700111ac8b1be7b1d3
+Merge: cef8fc3 73564ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:47:36 2016 -0500
+
+    Merge pull request #7421 from Abhishekvrshny/wip-14494-infernalis
+    
+    infernalis: pgs stuck inconsistent after infernalis upgrade
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cef8fc305e2c52a09b4af9e5a54c3c4117325453
+Merge: 15c27ce 2647c20
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:46:25 2016 -0500
+
+    Merge pull request #6627 from Abhishekvrshny/wip-13771-infernalis
+    
+    Objecter: pool op callback may hang forever.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 15c27ce7328baf92ca904e1b92aa8e21b8452263
+Merge: d1da031 2b6d817
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 18:18:07 2016 +0700
+
+    Merge pull request #7543 from SUSE/wip-14676-infernalis
+    
+    infernalis: rgw: radosgw-admin --help doesn't show the orphans find command
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d1da03131acf735946c3a0deab72d8f7da9e44d8
+Merge: 22e0c6a 1665f81
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:22:28 2016 +0700
+
+    Merge pull request #6993 from badone/wip-13993-infernalis
+    
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 22e0c6a7388af5e3b3a085ef7f3e2ad162d2c0f8
+Merge: 489a13d e102e5a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:21:11 2016 +0700
+
+    Merge pull request #6882 from dachary/wip-13988-reuse-osd-id-infernalis
+    
+    tests: verify it is possible to reuse an OSD id
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 489a13d4dd4483937dd7bd7798cb6e7d1dbd240c
+Merge: de01ba4 7d90c01
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:18:35 2016 +0700
+
+    Merge pull request #6852 from Abhishekvrshny/wip-14013-infernalis
+    
+    infernalis: systemd/ceph-disk@.service assumes /bin/flock
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de01ba4ea2df67e5b732d6e71cdf324642becbd2
+Merge: b5427f4 5d1e6ae
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:16:23 2016 +0700
+
+    Merge pull request #6846 from Abhishekvrshny/wip-13638-infernalis
+    
+    FileStore: potential memory leak if getattrs fails.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b5427f466eb8507dc865d80d0e3293652452f558
+Merge: 5dbd998 ac31f7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:14:18 2016 +0700
+
+    Merge pull request #6836 from SUSE/wip-13891-infernalis
+    
+    infernalis: auth/cephx: large amounts of log are produced by osd
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5dbd9984e09f3c520c69a406f98577bfdcd9b220
+Merge: 2016468 c968a40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:12:47 2016 +0700
+
+    Merge pull request #6833 from SUSE/wip-13935-infernalis
+    
+    infernalis: Ceph daemon failed to start, because the service name was already used.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 201646882e2c05d2e2436f62b7bfb50a6466aa54
+Merge: b09670f edde13c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:12:00 2016 +0700
+
+    Merge pull request #6694 from xiexingguo/xxg-wip-13869
+    
+    osd: fix race condition during send_failures
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b09670fea2ff844333c654fb73046dd1596fb2e3
+Merge: 59df52d 4d2a135
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:09:30 2016 +0700
+
+    Merge pull request #6626 from Abhishekvrshny/wip-13655-infernalis
+    
+    crush: crash if we see CRUSH_ITEM_NONE in early rule step
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 59df52df1697ee8dccb39430940e683a7bdab275
+Merge: f8f36b6 0eb2ddb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:06:41 2016 +0700
+
+    Merge pull request #6449 from dachary/wip-13671-infernalis
+    
+    tests: testprofile must be removed before it is re-created
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2b6d817187265686d21de76db14870ff22d549de
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 1 16:33:55 2016 -0800
+
+    rgw-admin: document orphans commands in usage
+    
+    Fixes: #14516
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 105a76bf542e05b739d5a03ca8ae55432350f107)
+
+commit f8f36b6877c0d0d356d1c53a171d5a1a0be423b0
+Merge: 7ffe6ff 399b7c5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 4 16:23:51 2016 -0500
+
+    Merge pull request #6880 from dachary/wip-14044-infernalis
+    
+    infernalis: ceph-disk list fails on /dev/cciss!c0d0
+
+commit 7ffe6ff4e26db538335b8069ce4fd14fe4cdd76c
+Merge: a8fd5ba 8707b0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 09:05:14 2016 -0500
+
+    Merge pull request #6392 from SUSE/wip-13589-infernalis
+    
+    mon: should not set isvalid = true when cephx_verify_authorizer retur…
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a8fd5ba7764b941aeb8a8ec2867a7e212159c84d
+Merge: 1296c2b 1bfb6f7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:55:45 2016 -0500
+
+    Merge pull request #6500 from SUSE/wip-13678-infernalis
+    
+    systemd: no rbdmap systemd unit file
+
+commit 29d30ecd4667567a58a7f92641a8a7618e88fd44
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Nov 12 21:57:27 2015 +0800
+
+    mds: properly set STATE_STRAY/STATE_ORPHAN for stray dentry/inode
+    
+    Fixes: #13777
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 460c74a0b872336a7279f0b40b17ed672b6e15a1)
+
+commit 5f54671e0715781146044e2d38ba79ad03634a64
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 16 10:57:56 2015 +0000
+
+    mon: don't require OSD W for MRemoveSnaps
+    
+    Use ability to execute "osd pool rmsnap" command
+    as a signal that the client should be permitted
+    to send MRemoveSnaps too.
+    
+    Note that we don't also require the W ability,
+    unlike Monitor::_allowed_command -- this is slightly
+    more permissive handling, but anyone crafting caps
+    that explicitly permit "osd pool rmsnap" needs to
+    know what they are doing.
+    
+    Fixes: #13777
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0b474c52abd3d528c041544f73b1d27d7d1b1320)
+
+commit 7fd56eaf215753156bdb6481835f399ad303faf0
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Jan 13 13:17:53 2016 -0800
+
+    fsx: checkout old version until it compiles properly on miras
+    
+    I sent a patch to xfstests upstream at
+    http://article.gmane.org/gmane.comp.file-systems.fstests/1665, but
+    until that's fixed we need a version that works in our test lab.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 7d52372ae74878ebd001036ff0a7aad525eb15b6)
+
+commit 8175e163fceb2bdc49ca291e183c08575e6c5d18
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Thu Jan 7 11:01:19 2016 -0800
+
+    cls_rbd: enable object map checksums for object_map_save
+    
+    object_map_save disables CRCs when an object map footer isn't provided.
+    Unconditionally re-enable object map CRCs before re-encoding the new object
+    map.
+    
+    Fixes: #14280
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit d5c02f3ed26edec095d45d7a7f26ff26d1b5aacc)
+
+commit 2c5b90a268ebcdcd8c806ef72c08184cc4e0417a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jan 21 13:45:42 2016 +0200
+
+    rbd-replay: handle EOF gracefully
+    
+    Fixes: #14452
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c59b84c3e2c9bbda68219e4d2288a889dd9ca6cb)
+
+commit f870ba9ab41cac127d182a52f435c7adc628de71
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Fri Jan 22 11:18:40 2016 -0800
+
+    rbd: remove canceled tasks from timer thread
+    
+    When canceling scheduled tasks using the timer thread, TaskFinisher::cancel
+    does not call SafeTimer::cancel_event, so events fire anyway. Add this call.
+    
+    Fixes: #14476
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 2aa0f318c862dbe3027d74d345671506605778eb)
+
+commit 73564ac22319d158d4bade9ab1a3832a9669d099
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 24 11:38:41 2015 -0400
+
+    osd/PG: For performance start scrub scan at pool to skip temp objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 05d79faa512210b0f0a91640d18db33b887a6e73)
+
+commit 2480fca93beb700aa9a963f5b568ab6bf8915e0f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 18 09:08:19 2015 -0800
+
+    osd/OSD: clear_temp_objects() include removal of Hammer temp objects
+    
+    Fixes: #13862
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 10b4a0825d9917b6fdd0d6450640238b78ba05d4)
+
+commit 9ff90acc3efdb1c5bdd23c8658e8692f2879231e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 17 18:04:08 2015 -0800
+
+    osd: Improve log message which isn't about a particular shard
+    
+    Remove redundant dout()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e85907fcc582922925609f595f68c597a88c39dc)
+
+commit 1296c2baef3412f462ee2124af747a892ea8b7a9
+Merge: 9dc5e6c d4c41d0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 13 17:15:41 2016 -0800
+
+    Merge pull request #7225 from dillaman/wip-13810-infernalis
+    
+    tests: notification slave needs to wait for master
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d4c41d041b7b176723ccd548102a9d2edf91956e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 13 12:44:01 2016 -0500
+
+    tests: notification slave needs to wait for master
+    
+    If the slave instance starts before the master, race
+    conditions are possible.
+    
+    Fixes: #13810
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3992d6fe67bbf82322cedc1582406caaf6d4de60)
+
+commit e102e5a0e261f407913c5d47991c2549f6664ffc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 10 15:20:32 2015 +0100
+
+    tests: verify it is possible to reuse an OSD id
+    
+    When an OSD id is removed via ceph osd rm, it will be reused by the next
+    ceph osd create command. Verify that and OSD reusing such an id
+    successfully comes up.
+    
+    http://tracker.ceph.com/issues/13988 Refs: #13988
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7324615bdb829f77928fa10d4e988c6422945937)
+
+commit 399b7c5409d9801aa7f38533dd8b7c19c726a594
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:33:45 2016 +0100
+
+    ceph-disk: list accepts absolute dev names
+    
+    The ceph-disk list subcommand now accepts /dev/sda as well as sda.
+    The filtering is done on the full list of devices instead of restricting
+    the number of devices explored. Always obtaining the full list of
+    devices makes things simpler when trying to match a dmcrypted device to
+    the corresponding raw device.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 591d581c84cfd72d7c655ac88b0911a318b96e95)
+    
+    Conflicts:
+    	src/ceph-disk: as part of the implementation of deactivate /
+    	destroy in master, the prototype of list_device was changed
+            to take a list of paths instead of the all arguments (args).
+
+commit e222d7bce218b732025ce094ed2d1bb31af0d6f6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 14:25:51 2016 +0100
+
+    ceph-disk: display OSD details when listing dmcrypt devices
+    
+    The details about a device that mapped via dmcrypt are directly
+    available. Do not try to fetch them from the device entry describing the
+    devicemapper entry.
+    
+    http://tracker.ceph.com/issues/14230 Fixes: #14230
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7aab4ed6f108ddc7bc90300f1999a38f30da3a57)
+    
+    Conflicts:
+    	src/ceph-disk: an incorrect attempt was made to fix the same
+                           problem. It was not backported and does not
+                           need to be. It is entirely contained in the
+                           code block removed and is the reason for the
+                           conflict.
+
+commit 87db693eb7d52ad297e8d265ffffa0d7e1125478
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 9 16:52:10 2015 +0100
+
+    tests: limit ceph-disk unit tests to test dir
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 499c80db606fe3926a8a603e03fdba6967d66003)
+
+commit 2c0d7ef2c72b7570a1304cad927a0b612ea3d4a8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:38:59 2016 +0100
+
+    ceph-disk: factorize duplicated dmcrypt mapping
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35a0c94c4cd3a57cfc382c64eaa9cfb9306dd2e6)
+
+commit 22881b1686dc040d8af01c66f59a935065bb680a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:42:11 2016 +0100
+
+    ceph-disk: fix regression in cciss devices names
+    
+    The cciss driver has device paths such as /dev/cciss/c0d1 with a
+    matching /sys/block/cciss!c0d1. The general case is that whenever a
+    device name is found in /sys/block, the / is replaced by the !.
+    
+    When refactoring the ceph-disk list subcommand, this conversion was
+    overlooked in a few places. All explicit concatenation of /dev with a
+    device name are replaced with a call to get_dev_name which does the same
+    but also converts all ! in /.
+    
+    http://tracker.ceph.com/issues/13970 Fixes: #13970
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a2fd3a535e66b3a2b694cda9c6add33383ccfa4a)
+    
+    Conflicts:
+    	src/ceph-disk : trivial resolution
+
+commit 9dc5e6cbf85681d0fb9ab249c571672456eb6556
+Merge: 64fab4f 5798d1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 7 15:06:32 2016 +0100
+
+    Merge pull request #7001 from dachary/wip-14145-infernalis
+    
+    infernalis: ceph-disk: use blkid instead of sgdisk -i
+    
+    
+    On CentOS 7.1 and other operating systems with a version of udev greater or equal to 214,
+    running ceph-disk prepare triggered unexpected removal and addition of partitions on
+    the disk being prepared. That created problems ranging from the OSD not being activated
+    to failures because /dev/sdb1 does not exist although it should.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5798d1e76c380182472d08ec6f62ebe0f1a9897e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jan 6 23:36:57 2016 +0100
+
+    tests: ceph-disk cryptsetup close must try harder
+    
+    Similar to how it's done in dmcrpyt_unmap in master (
+    132e56615805cba0395898cf165b32b88600d633 ), the infernalis tests helper
+    that were deprecated by the addition of the deactivate / destroy
+    ceph-disk subcommand must try cryptsetup close a few times in some
+    contexts.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 379bbe0613e8dbe2f4d38f4f9ee23a10ed39a64d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Dec 19 00:53:03 2015 +0100
+
+    ceph-disk: protect deactivate with activate lock
+    
+    When ceph-disk prepares the disk, it triggers udev events and each of
+    them ceph-disk activate. If systemctl stop ceph-osd@2 happens while
+    there still are ceph-disk activate in flight, the systemctl stop may be
+    cancelled by the systemctl enable issued by one of the pending ceph-disk
+    activate.
+    
+    This only matters in a test environment where disks are destroyed
+    shortly after they are activated.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 6395bf856b4d4511f0758174ef915ebcafbe3777)
+    
+    Conflicts:
+    
+            src/ceph-disk: ceph-disk deactivate does not exist in ceph-disk
+                on infernalis. But the same feature is implemented in
+                ceph-test-disk.py for test purposes and has the same
+                problem. The patch is adapted to ceph-test-disk.py.
+
+commit 4da2f9494dbd72e84d381cc12125c61931a27628
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jan 6 11:15:19 2016 +0100
+
+    ceph-disk: retry cryptsetup remove
+    
+    Retry a cryptsetup remove ten times. After the ceph-osd terminates, the
+    device is released asyncrhonously and an attempt to cryptsetup remove
+    will may fail because it is considered busy. Although a few attempts are
+    made before giving up, the number of attempts / the duration of the
+    attempts cannot be controlled with a cryptsetup option. The workaround
+    is to increase this by trying a few times.
+    
+    If cryptsetup remove fails for a reason that is unrelated to timeout,
+    the error will be repeated a few times. There is no undesirable side
+    effect. It will not hide a problem.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 132e56615805cba0395898cf165b32b88600d633)
+
+commit 676a4b42267133a9242a81503f7dd0eaeb0afd33
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 18 17:03:21 2015 +0100
+
+    ceph-disk: use blkid instead of sgdisk -i
+    
+    sgdisk -i 1 /dev/vdb opens /dev/vdb in write mode which indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up) when
+    the device is closed (see below for the udev release note). The
+    implementation of this ioctl by the kernel (even old kernels) removes
+    all partitions and adds them again (similar to what partprobe does
+    explicitly).
+    
+    The side effects of partitions disappearing while ceph-disk is running
+    are devastating.
+    
+    sgdisk is replaced by blkid which only opens the device in read mode and
+    will not trigger this unexpected behavior.
+    
+    The problem does not show on Ubuntu 14.04 because it is running udev <
+    214 but shows on CentOS 7 which is running udev > 214.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+            * As an experimental feature, udev now tries to lock the
+              disk device node (flock(LOCK_SH|LOCK_NB)) while it
+              executes events for the disk or any of its partitions.
+              Applications like partitioning programs can lock the
+              disk device node (flock(LOCK_EX)) and claim temporary
+              device ownership that way; udev will entirely skip all event
+              handling for this disk and its partitions. If the disk
+              was opened for writing, the close will trigger a partition
+              table rescan in udev's "watch" facility, and if needed
+              synthesize "change" events for the disk and all its partitions.
+              This is now unconditionally enabled, and if it turns out to
+              cause major problems, we might turn it on only for specific
+              devices, or might need to disable it entirely. Device Mapper
+              devices are excluded from this logic.
+    
+    http://tracker.ceph.com/issues/14080 Fixes: #14080
+    
+    Signed-off-by: Ilya Dryomov <idryomov@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9dce05a8cdfc564c5162885bbb67a04ad7b95c5a)
+
+commit bb8d53636d3a433bb7752e3a291a86a8af058215
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 15:57:03 2015 +0100
+
+    ceph-disk: dereference symlinks in destroy and zap
+    
+    The behavior of partprobe or sgdisk may be subtly different if given a
+    symbolic link to a device instead of an actual device. The debug output
+    is also more confusing when the symlink shows instead of the device it
+    points to.
+    
+    Always dereference the symlink before running destroy and zap.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit fe71647bc9bd0f9ddc6d470ee7bee1e6b0983e2b)
+    
+    Conflicts:
+            src/ceph-disk
+              trivial, because destroy is not implemented
+              in infernalis
+
+commit 74c27423f4925c7a31d6675851ffa3d2f5265ef1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:33:25 2015 +0100
+
+    ceph-disk: increase partprobe / udevadm settle timeouts
+    
+    The default of 120 seconds may be exceeded when the disk is very slow
+    which can happen in cloud environments. Increase it to 600 seconds
+    instead.
+    
+    The partprobe command may fail for the same reason but it does not have
+    a timeout parameter. Instead, try a few times before failing.
+    
+    The udevadm settle guarding partprobe are not necessary because
+    partprobe already does the same. However, partprobe does not provide a
+    way to control the timeout. Having a udevadm settle after another is
+    going to be a noop most of the time and not add any delay. It matters
+    when the udevadm settle run by partprobe fails with a timeout because
+    partprobe will silentely ignores the failure.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 730b5d62d3cda7de4076bafa6e9e35f1eb8e2190)
+
+commit 263eb579ac7179a19157004ed2de294085dc4a63
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:36:47 2015 +0100
+
+    tests: ceph-disk workunit increase verbosity
+    
+    So that reading the teuthology log is enough in most cases to figure out
+    the cause of the error.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit fd7fe8c4977658f66651dad5efb0d816ae71b38b)
+    
+    Conflicts:
+    	qa/workunits/ceph-disk/ceph-disk-test.py:
+              trivial, because destroy/deactivate are not implemented
+              in infernalis. The existing destroy_osd function
+              has to be modified so the id returned by sh() does
+              not have a trailing newline.
+
+commit 936dda76ce11baa9f2b7cce882b655550188a731
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:30:20 2015 +0100
+
+    ceph-disk: log parted output
+    
+    Should parted output fail to parse, it is useful to get the full output
+    when running in verbose mode.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f5d36b9ac299e9f6d52cc32d540cc1c3342de6e7)
+
+commit db26cc4b7fbd5b6f2a3da58fb7ac7736fc058065
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:29:17 2015 +0100
+
+    ceph-disk: do not discard stderr
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5fa35ba10e10b56262757afc43929ab8ee4164f2)
+    
+    Conflicts:
+    	src/ceph-disk : trivial, because destroy/deactivate
+            are not implemented in infernalis
+
+commit 83bba36afbae58ebe7cea2b0fb8ff51edb475fff
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:49:55 2015 -0500
+
+    tests: new integration test for validating new RBD pools
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 00cfe4efacd664032f700afe9701d41bacf8700a)
+
+commit 96cb5e2888e698879f3d3663dfa6f556e983a555
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:41:49 2015 -0500
+
+    librbd: optionally validate RBD pool configuration (snapshot support)
+    
+    Fixes: #13633
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fea4dadc60e13518e9ee55d136fbc4e9d3a621e)
+
+commit 7a44714b66218cf39aa6e1673059a13729c7c39e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 13:26:39 2015 -0500
+
+    librbd: properly handle replay of snap remove RPC message
+    
+    Fixes: #14164
+    Backport: infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bc309d9d7612f005a3d50ecf099ddf9b706a1bf6)
+
+commit 64fab4f080c5b7d79b51b8db8c71428b507efec5
+Merge: f480cea 7676343
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Dec 23 10:47:30 2015 -0800
+
+    Merge pull request #7038 from dillaman/wip-14121-infernalis
+    
+    tests: rebuild exclusive lock test should acquire exclusive lock
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7676343c781e9c15e18c5960d221a29840835332
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 10:31:07 2015 -0500
+
+    tests: rebuild exclusive lock test should acquire exclusive lock
+    
+    Starting with Jewel, the object map will not be loaded until the
+    exclusive lock is acquired since it might be updated by the
+    lock owner.
+    
+    Fixes: #14121
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1665f8187b16c9b47238743a7a94f0850160eb71
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Dec 7 11:31:28 2015 +1000
+
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Fixes: #13993
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8e93f3f45db681f82633ca695a7dc4e7bd030584)
+
+commit b211686dddf35d990c07ea50b868b98ac3abf8d3
+Author: Chengyuan Li <chengyli@ebay.com>
+Date:   Thu Nov 19 22:29:39 2015 -0700
+
+    mon/PGMonitor: MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    In get_rule_avail(), even p->second is 0, it's possible to be used
+    as divisor and quotient is infinity, then is converted to an integer
+    which is negative value.
+    So we should check p->second value before calculation.
+    
+    It fixes BUG #13840.
+    
+    Signed-off-by: Chengyuan Li <chengyli@ebay.com>
+    (cherry picked from commit 18713e60edd1fe16ab571f7c83e6de026db483ca)
+
+commit f480cea217008fa7b1e476d30dcb13023e6431d1
+Merge: d3aaeae 34d0b69
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Wed Dec 9 11:22:26 2015 +0530
+
+    Merge pull request #6395 from SUSE/wip-13593-infernalis
+    
+    Ceph-fuse won't start correctly when the option log_max_new in ceph.conf set to zero
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d3aaeae1884a26b6a069c6926ebe9cea62625850
+Merge: 476af49 ebc5e4e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 9 00:06:33 2015 +0100
+
+    Merge pull request #6828 from dachary/wip-ceph-disk-augeas
+    
+    tests: ceph-disk workunit uses configobj
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ebc5e4ee46dce9ab17208cdebc31ae5cc8c7e828
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 22 01:48:31 2015 +0200
+
+    tests: ceph-disk workunit uses the ceph task
+    
+    The ceph-disk workunit deploy keys that are not deployed by default by
+    the ceph teuthology task.
+    
+    The OSD created by the ceph task are removed from the default
+    bucket (via osd rm) so they do not interfere with the tests.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 163de5b0f8f46695ab41b3f2288e9b5c1feaedab)
+
+commit 0afaeb243448819dfce0de2778c27015a46e195a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 22 00:21:49 2015 +0200
+
+    tests: ceph-disk workunit uses configobj
+    
+    Instead of using augtool to modify the configuration file, use
+    configobj. It is also used by the install teuthology task. The .ini
+    lens (puppet lens really) is unable to read ini files created by
+    configobj.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f4906a124cc194dccd855679a04a5c7ffc125a44)
+
+commit a2644ed52c4b8a075919866af9d333fa1f703644
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Nov 9 11:37:02 2015 +0800
+
+    client: use null snapc to check pool permission
+    
+    snap inodes' ->snaprealm can be NULL, so dereferencing it in
+    check_pool_perm() can cause segment fault. The pool permission
+    check does not write any data, so it's safe to use null snapc.
+    
+    Fixes: #13714
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fad3772fb7731272d47cbfd9e81f22f5df3701a2)
+
+commit 476af491ba03843e953dd257ad972f09c64dd816
+Merge: af86215 54c8af8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Dec 8 09:34:39 2015 +0100
+
+    Merge pull request #6845 from dachary/wip-14019-infernalis
+    
+    infernalis: libunwind package missing on CentOS 7
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 7d90c018e5cfec982f96ce6dd45b6f89218f441d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 4 21:11:09 2015 +0100
+
+    build/ops: systemd ceph-disk unit must not assume /bin/flock
+    
+    The flock command may be installed elsewhere, depending on the
+    system. Let the PATH search figure that out.
+    
+    http://tracker.ceph.com/issues/13975 Fixes: #13975
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c8f7d44c935bd097db7d131b785bdab78a7a650c)
+
+commit 5ef86d78f86c8feeae5bade408ce03bd3b86573c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 3 14:52:24 2015 -0800
+
+    osd: Test osd_find_best_info_ignore_history_les config in another assert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 02a9a41f151a3d968bf8066749658659dc6e3ac4)
+
+commit 7ac5b151224f618e2f81387f74b1b7c3159c2b43
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 2 14:50:28 2015 -0500
+
+    osd: call on_new_interval on newly split child PG
+    
+    We must call on_new_interval() on any interval change *and* on the
+    creation of the PG.  Currently we call it from PG::init() and
+    PG::start_peering_interval().  However, PG::split_into() did not
+    do so for the child PG, which meant that the new child feature
+    bits were not properly initialized and the bitwise/nibblewise
+    debug bit was not correctly set.  That, in turn, could lead to
+    various misbehaviors, the most obvious of which is scrub errors
+    due to the sort order mismatch.
+    
+    Fixes: #13962
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fb120d7b2da5715e7f7d1baa65bfa70d2e5d807a)
+
+commit 5d1e6ae408ee0d9e2972b3de9f81b6a95e86a0a3
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Oct 26 18:38:01 2015 +0800
+
+    FileStore: potential memory leak if _fgetattrs fails
+    
+    Memory leak happens if _fgetattrs encounters some error and simply returns.
+    Fixes: #13597
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit ace7dd096b58a88e25ce16f011aed09269f2a2b4)
+
+commit 54c8af80c6e253d13993931d55720c8da99a2b2c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 8 08:02:56 2015 +0100
+
+    build/ops: enable CR in CentOS 7
+    
+    To get libunwind from the CR repositories until CentOS 7.2.1511 is released.
+    
+    http://tracker.ceph.com/issues/13997 Fixes: #13997
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 247ee6084b58861da601d349bdba739b252d96de)
+
+commit af2ced594699f95f2dcafd82fa15ca480fa844eb
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Nov 2 21:46:11 2015 +0800
+
+    Objecter: remove redundant result-check of _calc_target in _map_session.
+    
+    Result-code check is currently redundant since _calc_target never returns a negative value.
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 5a6117e667024f51e65847f73f7589467b6cb762)
+
+commit 43dede3144f98de16f8373f376e716f2f73b6f82
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 17:32:50 2015 +0800
+
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Objecter: potential null pointer access when do pool_snap_list. Shall check pool existence first.
+    Fixes: #13639
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 865541605b6c32f03e188ec33d079b44be42fa4a)
+
+commit ac31f7f8ac168e27b5a33093832aab3bfc97f667
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Thu Nov 5 07:29:49 2015 -0500
+
+    auth/cephx: large amounts of log are produced by osd
+    if the auth of osd is deleted when the osd is running, the osd will produce large amounts of log.
+    
+    Fixes:#13610
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 102f0b19326836e3b0754b4d32da89eb2bc0b03c)
+
+commit c968a40dfb0a0412d1feb7f8f085c8293b42bf4d
+Author: wangchaunhong <root@A22832429.(none)>
+Date:   Tue Oct 20 18:40:23 2015 +0800
+
+    init-ceph: fix systemd-run cant't start ceph daemon sometimes
+    
+    Fixes: #13474
+    Signed-off-by: Chuanhong Wang <wang.chuanhong@zte.com.cn>
+    (cherry picked from commit 2f36909e1e08bac993e77d1781a777b386335669)
+
+commit 0eb2ddb0adf39ef167012e85317e3cfdddf34598
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Nov 3 00:21:51 2015 +0100
+
+    tests: test/librados/test.cc must create profile
+    
+    Now that the create_one_ec_pool function removes the testprofile each
+    time it is called, it must create the testprofile erasure code profile
+    again for the test to use.
+    
+    http://tracker.ceph.com/issues/13664 Refs: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a60342942b5a42ee04d59af77a6b904ce62eefc4)
+
+commit 26adf7a3536d7fe5b3a04866e37a4e6afac08509
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:24:51 2015 +0100
+
+    tests: destroy testprofile before creating one
+    
+    The testprofile erasure code profile is destroyed before creating a new
+    one so that it does not fail when another testprofile erasure code
+    profile already exists with different parameters.
+    
+    This must be done when creating erasure coded pools with the C++
+    interface, in the same way it's done with the C interface.
+    
+    http://tracker.ceph.com/issues/13664 Fixes: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 47abab9a6f182aa0abe5047c04402850379bcd6d)
+
+commit c3619f4c1ba16e3d0959c78865ee4218e911da39
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:23:52 2015 +0100
+
+    tests: add destroy_ec_profile{,_pp} helpers
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ab46d79bc09fc711fa35302f49eecac81a98519b)
+
+commit 1bfb6f767357fcd7ea424fc17b799fb2c171cf80
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Nov 2 14:07:47 2015 +0100
+
+    rbdmap: systemd support
+    
+    Fixes: #13374
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 9224ac2ad25f7d017916f58b642c0ea25305c3e5)
+
+commit 4eded389eea931fd9cdb0b9700c022088e8825ec
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 30 18:33:36 2015 +0100
+
+    rbdmap: Move do_map and do_unmap shell functions to rbdmap script
+    
+    This patch creates rbdmap shell script that is called from init-rbdmap
+    init script. The patch also renames src/rbdmap configuration file to
+    src/etc-rbdmap so that rbdmap shell script can be installed via build
+    system directly. Finally, the patch accomodates these changes in spec
+    file and build system.
+    
+    Fixes: #13374
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c0980af3c72f01e6f99fd1e7e91c446934d6d856)
+    
+    Conflicts:
+    	src/init-rbdmap
+                Trivial resolution.
+
+commit 6c68971d1cfc27d30f92a83119e922180b389b43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 21:04:04 2015 +0800
+
+    mds: consider client's flushing caps when choosing lock states
+    
+    Client may flush and drop caps at the same time. If client need to
+    send cap reconnect before the caps get flushed. The issued caps in
+    the cap reconnect does not include the flushing caps. When choosing
+    lock states, MDS only consider the issued caps in cap reconnect, it
+    may choose wrong states.
+    
+    Fixes: #11482
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ce9a596dcaf95dd4af0a3a9e28871462a6bcb930)
+
+commit 3fcf905dd65d6db514d403e2829b039df7bd9645
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 17:28:02 2015 +0800
+
+    client: cancel revoking caps when reconnecting the mds
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9ab61b2779740d9038b169da0de616b89b07b813)
+
+commit 7e8706bada034ce428716963a5e3151723f83126
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 17:21:11 2015 +0800
+
+    mds: choose EXCL state for filelock when client has Fb capability
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 655ae79dd266dfd488aff67642aceab4ab383d01)
+
+commit af86215e0b178340055ff6ec7c8ec3b3c0bd1acf
+Merge: 899ff61 497c1f0
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Tue Dec 1 17:44:24 2015 +0530
+
+    Merge pull request #6634 from Abhishekvrshny/wip-13761-infernalis
+    
+    unknown argument --quiet in udevadm settle
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 899ff6160a289dfeb5c287afa8d9b168e11382ed
+Merge: f2b323a fdb5dbd
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:56:40 2015 +0530
+
+    Merge pull request #6650 from Abhishekvrshny/wip-13830-infernalis
+    
+    init script reload doesn't work on EL7
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit f2b323a69bb54ee99743bf53043d6a4e21e2a0bb
+Merge: 5c02e2b effaf26
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:55:55 2015 +0530
+
+    Merge pull request #6477 from SUSE/wip-13705-infernalis
+    
+    rbd : enable feature objectmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5c02e2bd78782f1ea19836806d3773748be66ed1
+Merge: ed8ac03 c2e7fe9
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:55:22 2015 +0530
+
+    Merge pull request #6474 from SUSE/wip-13619-infernalis
+    
+    rbd clone issue
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ed8ac030a979ca9fb2cbf340c316c4432e54cd0d
+Merge: 1551c1b ad84753
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:54:44 2015 +0530
+
+    Merge pull request #6633 from Abhishekvrshny/wip-13759-infernalis
+    
+    rbd: pure virtual method called
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1551c1be297cfc73457819c7c983b113ac9c86f6
+Merge: b8000d6 83c3880
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:54:18 2015 +0530
+
+    Merge pull request #6632 from Abhishekvrshny/wip-13756-infernalis
+    
+    QEMU hangs after creating snapshot and stopping VM
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b8000d6f489af3e13c4996b46363c3884ccfd0de
+Merge: fc6f21a 74cb593
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:50:40 2015 +0530
+
+    Merge pull request #6630 from Abhishekvrshny/wip-13754-infernalis
+    
+    Avoid re-writing old-format image header on resize
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit fc6f21a1d3ff34b4ff522ef03b368b3ad4570a1c
+Merge: bb2ecea a2c6a21
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 30 15:01:23 2015 +0100
+
+    Merge pull request #6396 from SUSE/wip-13342-infernalis
+    
+    ceph upstart script rbdmap.conf incorrectly processes parameters
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit edde13ce55d0789523a3101e5f435fc61bef5d69
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 17 21:42:53 2015 -0400
+
+    osd: fix send_failures() locking
+    
+    It is unsafe to check failure_queue.empty() without the lock.
+    Fixes: #13869
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3ca828ae8ebc9068073494c46faf3e8e1443ada)
+
+commit 0f5043975dbde75036b9f9e8be757494b8cdc73b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 23 14:17:06 2015 -0400
+
+    osd: move misdirected op check from OSD thread to PG thread
+    
+    The op message isn't fully decoded until after the start of the PG
+    op handling.  Therefore, checking for misdirected ops needs to wait
+    until after the RWM flags have been initialized.
+    
+    Fixes: #13491
+    Backport: infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 56ba90f5e49d499ea9f6bda9665031ce90abdc32)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+    		In ReplicatedPG::do_op, resolved conflicts caused by afcfb050930e2e4086d631119022fd2dee9feb21 and 818d790f7d424520bc96c651571f2c86e94caf1e
+
+commit a2c6a212fbb8002fa95270c0f37c02496447624f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 30 08:29:05 2015 -0400
+
+    init-rbdmap: fix CMDPARAMS
+    
+    Fixes: #13214
+    Reported-by: Wyllys Ingersoll <wyllys.ingersoll@keepertech.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02113ac449cd7631f1c9a3840c94bbf253c052bd)
+
+commit fdb5dbd744a6cfa4f7497e3a98aed13ab8daf4d9
+Author: Herve Rousseau <hroussea@cern.ch>
+Date:   Fri Nov 6 09:52:28 2015 +0100
+
+    rgw: fix reload on non Debian systems.
+    
+    When using reload in non-debian systems, /bin/sh's kill is used to send the HUP signal to the radosgw process.
+    This kill version doesn't understand -SIGHUP as a valid signal, using -HUP does work.
+    
+    Fix: #13709
+    Backport: hammer
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 1b000abac3a02d1e788bf25eead4b6873133f5d2)
+
+commit 497c1f06a2e221631626ffb66bece310406a7491
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:13:27 2015 -0400
+
+    krbd: remove deprecated --quiet param from udevadm
+    
+    This parameter has been removed since systemd 213, so this
+    effects Fedora 21+, Debian Jessie, and potentially future
+    releases of RHEL 7.
+    
+    Fixes: #13560
+    Backport: hammer, infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4300f2a9fe29627eea580564ff2d576de3647467)
+
+commit d0536f9d2f5f54791daa476a9b951fcef9488a50
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:12:34 2015 -0400
+
+    run_cmd: close parent process console file descriptors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f46f7dc94139c0bafe10361622416d7dc343d31f)
+
+commit ad84753ad2d2bfb9ef572a7d21cacb47be6eec1b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 7 12:11:13 2015 -0400
+
+    WorkQueue: new PointerWQ base class for ContextWQ
+    
+    The existing work queues do not properly function if added to a running
+    thread pool.  librbd uses a singleton thread pool which requires
+    dynamically adding/removing work queues as images are opened and closed.
+    
+    Fixes: #13636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e78b18b09d75626ca2599bac3b9f9c9889507a5)
+
+commit 83c38802a2be16faf7be4ac5ac3c589de8b2dbb0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 11:22:24 2015 -0500
+
+    librbd: fixed deadlock while attempting to flush AIO requests
+    
+    In-flight AIO requests might force a flush if a snapshot was created
+    out-of-band.  The flush completion was previously invoked asynchronously,
+    potentially via the same thread worker handling the AIO request. This
+    resulted in the flush operation deadlocking since it can't complete.
+    
+    Fixes: #13726
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfeb90e5fe24347648c72345881fd3d932243c98)
+
+commit 0b21b3f0dc2a9676f3fb2b19141ce75324f53c72
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 10:48:10 2015 -0500
+
+    tests: new test case to catch deadlock on RBD image refresh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a9729d9553e7fb925509cad8d388cf52a9fede9c)
+
+commit 74cb593a3d29a5549cbe89a974aaa4814d35b8e0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 2 16:50:19 2015 -0500
+
+    librbd: resize should only update image size within header
+    
+    Previously, the whole RBD image format 1 header struct was
+    re-written to disk on a resize operation.
+    
+    Fixes: #13674
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5be20b6d4646284571568ab28cbf45b0729390b)
+
+commit 6750cab2d6f2c5f928f9c2d2c356047f93e9ce30
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 23 14:15:54 2015 -0400
+
+    osd: ensure op rwm flags are checked before they are initialized
+    
+    These flags are now lazy-initialized within the PG thread at the
+    start of op processing instead of once the message is received.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 652856321b0f14293cd885c865552d5b7e787f9a)
+
+commit 58b43e656cd9278667af2b7ac4fee72f50a03caa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 13 19:10:28 2015 +0100
+
+    build/ops: rbd-replay moved from ceph-test-dbg to ceph-common-dbg
+    
+    http://tracker.ceph.com/issues/13785 Fixes: #13785
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a0204dc47acc0c50223e449fd23a2fc97bfd111a)
+
+commit 2647c20cee7ed5738c711a0c435e35938c7739bd
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 20:04:11 2015 +0800
+
+    Objecter: pool_op callback may hang forever.
+    
+    pool_op callback may hang forever due to osdmap update during reply handling.
+    Fixes: #13642
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 00c6fa9e31975a935ed2bb33a099e2b4f02ad7f2)
+
+commit 4d2a135c042039e0c48302ffd667dc88d024eed9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 13 09:55:01 2015 -0400
+
+    crush/mapper: ensure take bucket value is valid
+    
+    Ensure that the take argument is a valid bucket ID before indexing the
+    buckets array.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 93ec538e8a667699876b72459b8ad78966d89c61)
+
+commit ecb6aa232180e3226d56bc8fe0cb643c90f5101b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 27 20:55:26 2015 -0400
+
+    crush/mapper: ensure bucket id is valid before indexing buckets array
+    
+    We were indexing the buckets array without verifying the index was within
+    the [0,max_buckets) range.  This could happen because a multistep rule
+    does not have enough buckets and has CRUSH_ITEM_NONE
+    for an intermediate result, which would feed in CRUSH_ITEM_NONE and
+    make us crash.
+    
+    Fixes: #13477
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 976a24a326da8931e689ee22fce35feab5b67b76)
+
+commit effaf26acccf81675b74277eb0a75368668b0677
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Wed Oct 21 19:01:21 2015 +0800
+
+    librbd : fix enable objectmap feature issue
+    
+    Fixes: #13558
+    
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit b0536ebab4e1f34e1ed87fe5efbb00d0f7b48abb)
+
+commit c2e7fe98738cd31c0886d594920c50a53cbf9bf4
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Wed Oct 21 14:56:17 2015 +0800
+
+    rbd: fix clone issue when we specify image feature
+    
+    Fixes: #13553
+    
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit da48dbb8f8c9417343d2ca7819c58b7c46ef7ad0)
+
+commit fde4ec62888ba0aee97ce714bb15d191625b736f
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Oct 13 14:04:20 2015 +0800
+
+    OSD:shall reset primary and up_primary fields when beginning a new past_interval.
+    
+    Shall reset primary and up_primary fields when we start over a new past_interval in OSD::build_past_intervals_parallel().
+    Fixes: #13471
+    Signed-off-by: xie.xingguo@zte.com.cn
+    (cherry picked from commit 65064ca05bc7f8b6ef424806d1fd14b87add62a4)
+
+commit 34d0b692cf1b9c0c2ea508b81d252b901fe1f52b
+Author: wenjunhuang <wenjunhuang@tencent.com>
+Date:   Sat Oct 10 14:30:56 2015 +0800
+
+    ceph-fuse.cc: While starting ceph-fuse, start the log thread first
+    
+    http://tracker.ceph.com/issues/13443 Fixes: #13443
+    
+    Signed-off-by: Wenjun Huang <wenjunhuang@tencent.com>
+    (cherry picked from commit f2763085754462610730a23bb5652237714abc2a)
+
+commit 8707b0b083e7a7732d1e3563b00d3adf3e09807b
+Author: yangruifeng <yangruifeng.09209@h3c.com>
+Date:   Mon Oct 19 08:08:12 2015 -0400
+
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Fixes: #13525
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit c7f75b8f7c0a773148ec16141941efd00ee76626)
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 000000000..2a083fae0
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,261 @@
+import fileinput
+import glob
+import logging
+import os
+import shutil
+import sys
+import yaml
+import sphinx.util
+
+
+top_level = \
+    os.path.dirname(
+        os.path.dirname(
+            os.path.abspath(__file__)))
+
+pybind_rgw_mod = __import__('rgw', globals(), locals(), [], 0)
+sys.modules['pybind_rgw_mod'] = pybind_rgw_mod
+
+
+def parse_ceph_release():
+    with open(os.path.join(top_level, 'src/ceph_release')) as f:
+        lines = f.readlines()
+        assert(len(lines) == 3)
+        # 16, pacific, dev
+        version, codename, status = [line.strip() for line in lines]
+        return version, codename, status
+
+
+def latest_stable_release():
+    with open(os.path.join(top_level, 'doc/releases/releases.yml')) as input:
+        releases = yaml.safe_load(input)['releases']
+        # get the first release
+        return next(iter(releases.keys()))
+
+
+def is_release_eol(codename):
+    with open(os.path.join(top_level, 'doc/releases/releases.yml')) as input:
+        releases = yaml.safe_load(input)['releases']
+        return 'actual_eol' in releases.get(codename, {})
+
+
+# project information
+project = 'Ceph'
+copyright = ('2016, Ceph authors and contributors. '
+             'Licensed under Creative Commons Attribution Share Alike 3.0 '
+             '(CC-BY-SA-3.0)')
+version, codename, release = parse_ceph_release()
+pygments_style = 'sphinx'
+
+# HTML output options
+html_theme = 'ceph'
+html_theme_options = {
+    'logo_only': True,
+    'display_version': False,
+    'prev_next_buttons_location': 'bottom',
+    'style_external_links': False,
+    'vcs_pageview_mode': 'edit',
+    'style_nav_header_background': '#eee',
+    # Toc options
+    'collapse_navigation': True,
+    'sticky_navigation': True,
+    'navigation_depth': 4,
+    'includehidden': True,
+    'titles_only': False
+}
+html_theme_path = ['_themes']
+html_title = "Ceph Documentation"
+html_logo = 'logo.png'
+html_context = {'is_release_eol': is_release_eol(codename)}
+html_favicon = 'favicon.ico'
+html_show_sphinx = False
+html_static_path = ["_static"]
+html_sidebars = {
+    '**': ['smarttoc.html', 'searchbox.html']
+    }
+
+html_css_files = ['css/custom.css']
+
+# general configuration
+templates_path = ['_templates']
+source_suffix = '.rst'
+exclude_patterns = ['**/.#*',
+                    '**/*~',
+                    'start/quick-common.rst',
+                    '**/*.inc.rst']
+if tags.has('man'):             # noqa: F821
+    master_doc = 'man_index'
+    exclude_patterns += ['index.rst',
+                         'architecture.rst',
+                         'glossary.rst',
+                         'release*.rst',
+                         'api/*',
+                         'cephadm/*',
+                         'cephfs/*',
+                         'dev/*',
+                         'governance.rst',
+                         'foundation.rst',
+                         'install/*',
+                         'mon/*',
+                         'rados/*',
+                         'mgr/*',
+                         'ceph-volume/*',
+                         'radosgw/*',
+                         'rbd/*',
+                         'start/*',
+                         'releases/*']
+else:
+    master_doc = 'index'
+    exclude_patterns += ['man_index.rst']
+
+build_with_rtd = os.environ.get('READTHEDOCS') == 'True'
+
+sys.path.insert(0, os.path.abspath('_ext'))
+
+smartquotes_action = "qe"
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.graphviz',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.todo',
+    'sphinx-prompt',
+    'sphinx_autodoc_typehints',
+    'sphinx_substitution_extensions',
+    'breathe',
+    'ceph_commands',
+    'ceph_releases',
+    'ceph_confval',
+    'sphinxcontrib.mermaid',
+    'sphinxcontrib.openapi',
+    'sphinxcontrib.seqdiag',
+    ]
+
+ditaa = shutil.which("ditaa")
+if ditaa is not None:
+    # in case we don't have binfmt_misc enabled or jar is not registered
+    ditaa_args = ['-jar', ditaa]
+    ditaa = 'java'
+    extensions += ['sphinxcontrib.ditaa']
+else:
+    extensions += ['plantweb.directive']
+    plantweb_defaults = {
+        'engine': 'ditaa'
+    }
+
+if build_with_rtd:
+    extensions += ['sphinx_search.extension']
+
+# sphinx.ext.todo options
+todo_include_todos = True
+
+# sphinx_substitution_extensions options
+rst_prolog = f"""
+.. |stable-release| replace:: {latest_stable_release()}
+"""
+
+# breath options
+breathe_default_project = "Ceph"
+# see $(top_srcdir)/Doxyfile
+
+breathe_build_directory = os.path.join(top_level, "build-doc")
+breathe_projects = {"Ceph": os.path.join(top_level, breathe_build_directory)}
+breathe_projects_source = {
+    "Ceph": (os.path.join(top_level, "src/include/rados"),
+             ["rados_types.h", "librados.h"])
+}
+breathe_domain_by_extension = {'py': 'py',
+                               'c': 'c', 'h': 'c',
+                               'cc': 'cxx', 'hpp': 'cxx'}
+breathe_doxygen_config_options = {
+    'EXPAND_ONLY_PREDEF': 'YES',
+    'MACRO_EXPANSION': 'YES',
+    'PREDEFINED': 'CEPH_RADOS_API= ',
+    'WARN_IF_UNDOCUMENTED': 'NO',
+}
+
+# graphviz options
+graphviz_output_format = 'svg'
+
+def generate_state_diagram(input_paths, output_path):
+    sys.path.append(os.path.join(top_level, 'doc', 'scripts'))
+    from gen_state_diagram import do_filter, StateMachineRenderer
+    inputs = [os.path.join(top_level, fn) for fn in input_paths]
+    output = os.path.join(top_level, output_path)
+
+    def process(app):
+        with fileinput.input(files=inputs) as f:
+            input = do_filter(f)
+            render = StateMachineRenderer()
+            render.read_input(input)
+            with open(output, 'w') as dot_output:
+                render.emit_dot(dot_output)
+
+    return process
+
+
+# mocking ceph_module offered by ceph-mgr. `ceph_module` is required by
+# mgr.mgr_module
+class Dummy(object):
+    def __getattr__(self, _):
+        return lambda *args, **kwargs: None
+
+
+class Mock(object):
+    __all__ = []
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        return Mock()
+
+    @classmethod
+    def __getattr__(cls, name):
+        mock = type(name, (Dummy,), {})
+        mock.__module__ = __name__
+        return mock
+
+
+# autodoc options
+sys.modules['ceph_module'] = Mock()
+
+if build_with_rtd:
+    autodoc_mock_imports = ['ceph']
+    pybinds = ['pybind/mgr',
+               'python-common']
+else:
+    pybinds = ['pybind',
+               'pybind/mgr',
+               'python-common']
+
+for c in pybinds:
+    pybind = os.path.join(top_level, 'src', c)
+    if pybind not in sys.path:
+        sys.path.insert(0, pybind)
+
+# openapi
+openapi_logger = sphinx.util.logging.getLogger('sphinxcontrib.openapi.openapi30')
+openapi_logger.setLevel(logging.WARNING)
+
+# seqdiag
+seqdiag_antialias = True
+seqdiag_html_image_format = 'SVG'
+
+# ceph_confval
+ceph_confval_imports = glob.glob(os.path.join(top_level,
+                                              'src/common/options',
+                                              '*.yaml.in'))
+ceph_confval_mgr_module_path = 'src/pybind/mgr'
+ceph_confval_mgr_python_path = 'src/pybind'
+
+# handles edit-on-github and old version warning display
+def setup(app):
+    if ditaa is None:
+        # add "ditaa" as an alias of "diagram"
+        from plantweb.directive import DiagramDirective
+        app.add_directive('ditaa', DiagramDirective)
+    app.connect('builder-inited',
+                generate_state_diagram(['src/osd/PeeringState.h',
+                                        'src/osd/PeeringState.cc'],
+                                       'doc/dev/peering_graph.generated.dot'))
diff --git a/doc/dev/PlanningImplementation.txt b/doc/dev/PlanningImplementation.txt
new file mode 100644
index 000000000..871eb5f37
--- /dev/null
+++ b/doc/dev/PlanningImplementation.txt
@@ -0,0 +1,43 @@
+ <big>About this Document</big>
+This document contains planning and implementation procedures for Ceph.  The audience for this document includes technical support personnel, installation engineers, system administrators, and quality assurance.  
+<B>Prerequisites<b>
+Users of this document must be familiar with Linux command line options.  They must also be familiar with the overall Ceph product. 
+Before You Begin
+Before implementing a new Ceph System, first answer the questions in the Ceph Getting Started Guide to determine your configuration needs.  Once you have determined your hardware and configuration needs, the following decisions must be made:
+•	Determine what level of technical support you need.  Pick from the Ceph Technical Support options in the next section.
+•	Determine how much and what level of training your organization needs.
+Ceph Technical Support Options
+The Ceph Technical support model provides 4 tiers of technical support options:
+1st – This option is for brand new customers that need installation, configuration, and setup on their production environment.  
+2nd – This level of support requires a trouble ticket to be generated on a case by case basis as customer difficulties arise.  Customers can choose between two maintenance options; they can either purchase a yearly maintenance contract, or pay for each trouble resolution as it occurs.
+3rd – This option comes with our bundled packages for customers who have also purchased our hosting plans.  In this case, the customer is a service provider. The Help Desk can generally provide this level of incident resolution. (NEED MORE INFO)
+4th – This level of support requires a Service Level Agreement (SLA) between the customer and Dreamhost.  This level is used for handling the most difficult or advanced problems.
+Planning a Ceph Cluster Configuration
+The following section contains guidelines for planning the deployment for a Ceph cluster configuration.  A Ceph cluster consists of the following core components:
+•	Monitors – These must be an odd number, such as one, three, or five.  Three is the preferred configuration.
+•	Object Storage Devices (OSD) – used as storage nodes
+•	Metadata Servers (MDS)
+For redundancy, you should employ several of these components. 
+Monitors 
+The monitors handle central cluster management, configuration, and state.  
+Hardware Requirements: 
+•	A few gigs of local disk space 
+•	A fixed network address 
+ Warning: Never configure 2 monitors per cluster.  If you do, they will both have to be up all of the time, which will greatly degrade system performance. 
+Object Storage Devices 
+The OSDs store the actual data on the disks. A minimum of two is required. 
+Hardware Requirements: 
+•	As many disks as possible for faster performance and scalability
+•	An SSD or NVRAM for a journal, or a RAID controller with a battery-backed NVRAM. 
+•	Ample RAM for better file system caching 
+•	Fast network 
+ Metadata Servers 
+The metadata server daemon commands act as a distributed, coherent cache of file system metadata. They do not store data locally; all metadata is stored on disk via the storage nodes. 
+Metadata servers can be added into the cluster on an as-needed basis.  The load is automatically balanced. The max_mds parameter controls how many cmds instances are active. Any additional running instances are put in standby mode and can be activated if one of the active daemons becomes unresponsive. 
+Hardware Requirements: 
+•	Large amount of  RAM 
+•	Fast CPU 
+•	Fast (low latency) network 
+•	At least two servers for redundancy and load balancing
+TIPS: If you have just a few nodes, put cmon, cmds, and cosd on the same node.  For moderate node configurations, put cmon and cmds together, and cosd on the disk nodes.  For large node configurations, put cmon, cmds, and cosd each on their own dedicated machine. 
+
diff --git a/doc/dev/balancer-design.rst b/doc/dev/balancer-design.rst
new file mode 100644
index 000000000..684d16352
--- /dev/null
+++ b/doc/dev/balancer-design.rst
@@ -0,0 +1,58 @@
+============================
+Balancing in Ceph
+============================
+
+Introduction
+============
+
+In distributed storage systems like Ceph, it is important to balance write and read requests for optimal performance. Write balancing ensures fast storage
+and replication of data in a cluster, while read balancing ensures quick access and retrieval of data in a cluster. Both types of balancing are important
+in distributed systems for different reasons.
+
+Upmap Balancing
+==========================
+
+Importance in a Cluster
+-----------------------
+
+Capacity balancing is a functional requirement. A system like Ceph is as full as its fullest device: When one device is full, the system can not serve write
+requests anymore, and Ceph loses its function. To avoid filling up devices, we want to balance capacity across the devices in a fair way. Each device should
+get a capacity proportional to its size so all devices have the same fullness level. From a performance perspective, capacity balancing creates fair share
+workloads on the OSDs for write requests.
+
+Capacity balancing is expensive. The operation (changing the mapping of pgs) requires data movement by definition, which takes time. During this time, the
+performance of the system is reduced.
+
+In Ceph, we can balance the write performance if all devices are homogeneous (same size and performance).
+
+How to Balance Capacity in Ceph
+-------------------------------
+
+See :ref:`upmap` for more information.
+
+Read Balancing
+==============
+
+Unlike capacity balancing, read balancing is not a strict requirement for Ceph’s functionality. Instead, it is a performance requirement, as it helps the system
+“work� better. The overall goal is to ensure each device gets its fair share of primary OSDs so read requests are distributed evenly across OSDs in the cluster.
+Unbalanced read requests lead to bad performance because of reduced overall cluster bandwidth.
+
+Read balancing is cheap. Unlike capacity balancing, there is no data movement involved. It is just a metadata operation, where the osdmap is updated to change
+which participating OSD in a pg is primary. This operation is fast and has no impact on the cluster performance (except improved performance when the operation
+completes – almost immediately).
+
+In Ceph, we can balance the read performance if all devices are homogeneous (same size and performance). However, in future versions, the read balancer can be improved
+to achieve overall cluster performance in heterogeneous systems.
+
+How to Balance Reads in Ceph
+----------------------------
+See :ref:`read_balancer` for more information.
+
+Also, see the Cephalocon 2023 talk `New Read Balancer in Ceph <https://www.youtube.com/watch?v=AT_cKYaQzcU/>`_ for a demonstration of the offline version
+of the read balancer.
+
+Plans for the Next Version
+--------------------------
+
+1. Improve behavior for heterogeneous OSDs in a pool
+2. Offer read balancing as an online option to the balancer manager module
diff --git a/doc/dev/blkin.rst b/doc/dev/blkin.rst
new file mode 100644
index 000000000..989cddcd7
--- /dev/null
+++ b/doc/dev/blkin.rst
@@ -0,0 +1,215 @@
+=========================
+ Tracing Ceph With LTTng
+=========================
+
+Configuring Ceph with LTTng
+===========================
+
+Use -DWITH_LTTNG option (default: ON)::
+
+  ./do_cmake -DWITH_LTTNG=ON
+
+Config option for tracing must be set to true in ceph.conf.
+Following options are currently available::
+
+  bluestore_tracing
+  event_tracing (-DWITH_EVENTTRACE)
+  osd_function_tracing (-DWITH_OSD_INSTRUMENT_FUNCTIONS)
+  osd_objectstore_tracing (actually filestore tracing)
+  rbd_tracing
+  osd_tracing
+  rados_tracing
+  rgw_op_tracing
+  rgw_rados_tracing
+
+Testing Trace
+=============
+
+Start LTTng daemon::
+
+  lttng-sessiond --daemonize
+
+Run vstart cluster with enabling trace options::
+
+  ../src/vstart.sh -d -n -l -e -o "osd_tracing = true"
+
+List available tracepoints::
+
+  lttng list --userspace
+
+You will get something like::
+
+  UST events:
+  -------------
+  PID: 100859 - Name: /path/to/ceph-osd
+      pg:queue_op (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+      osd:do_osd_op_post (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+      osd:do_osd_op_pre_unknown (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+      osd:do_osd_op_pre_copy_from (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+      osd:do_osd_op_pre_copy_get (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+      ...
+
+Create tracing session, enable tracepoints and start trace::
+
+  lttng create trace-test
+  lttng enable-event --userspace osd:*
+  lttng start
+
+Perform some Ceph operation::
+
+  rados bench -p ec 5 write
+
+Stop tracing and view result::
+
+  lttng stop
+  lttng view
+
+Destroy tracing session::
+
+  lttng destroy
+
+=========================
+ Tracing Ceph With Blkin
+=========================
+
+Ceph can use Blkin, a library created by Marios Kogias and others,
+which enables tracking a specific request from the time it enters
+the system at higher levels till it is finally served by RADOS.
+
+In general, Blkin implements the Dapper_ tracing semantics
+in order to show the causal relationships between the different
+processing phases that an IO request may trigger. The goal is an
+end-to-end visualisation of the request's route in the system,
+accompanied by information concerning latencies in each processing
+phase. Thanks to LTTng this can happen with a minimal overhead and
+in realtime. The LTTng traces can then be visualized with Twitter's
+Zipkin_.
+
+.. _Dapper: http://static.googleusercontent.com/media/research.google.com/el//pubs/archive/36356.pdf
+.. _Zipkin: https://zipkin.io/
+
+
+Configuring Ceph with Blkin
+===========================
+
+Use -DWITH_BLKIN option (which requires -DWITH_LTTNG)::
+
+  ./do_cmake -DWITH_LTTNG=ON -DWITH_BLKIN=ON
+
+Config option for blkin must be set to true in ceph.conf.
+Following options are currently available::
+
+  rbd_blkin_trace_all
+  osd_blkin_trace_all
+  osdc_blkin_trace_all
+
+Testing Blkin
+=============
+
+It's easy to test Ceph's Blkin tracing. Let's assume you don't have
+Ceph already running, and you compiled Ceph with Blkin support but
+you didn't install it. Then launch Ceph with the ``vstart.sh`` script
+in Ceph's src directory so you can see the possible tracepoints.::
+
+  OSD=3 MON=3 RGW=1 ../src/vstart.sh -n -o "rbd_blkin_trace_all"
+  lttng list --userspace
+
+You'll see something like the following:::
+
+  UST events:
+  -------------
+  PID: 8987 - Name: ./ceph-osd
+        zipkin:timestamp (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_integer (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_string (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        lttng_ust_tracelog:TRACE_DEBUG (loglevel: TRACE_DEBUG (14)) (type: tracepoint)
+
+  PID: 8407 - Name: ./ceph-mon
+        zipkin:timestamp (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_integer (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_string (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        lttng_ust_tracelog:TRACE_DEBUG (loglevel: TRACE_DEBUG (14)) (type: tracepoint)
+
+  ...
+
+Next, stop Ceph so that the tracepoints can be enabled.::
+
+  ../src/stop.sh
+
+Start up an LTTng session and enable the tracepoints.::
+
+  lttng create blkin-test
+  lttng enable-event --userspace zipkin:timestamp
+  lttng enable-event --userspace zipkin:keyval_integer
+  lttng enable-event --userspace zipkin:keyval_string
+  lttng start
+
+Then start up Ceph again.::
+
+  OSD=3 MON=3 RGW=1 ../src/vstart.sh -n -o "rbd_blkin_trace_all"
+
+You may want to check that ceph is up.::
+
+  ceph status
+
+Now put something in using rados, check that it made it, get it back, and remove it.::
+
+  ceph osd pool create test-blkin
+  rados put test-object-1 ../src/vstart.sh --pool=test-blkin
+  rados -p test-blkin ls
+  ceph osd map test-blkin test-object-1
+  rados get test-object-1 ./vstart-copy.sh --pool=test-blkin
+  md5sum vstart*
+  rados rm test-object-1 --pool=test-blkin
+
+You could also use the example in ``examples/librados/`` or ``rados bench``.
+
+Then stop the LTTng session and see what was collected.::
+
+  lttng stop
+  lttng view
+
+You'll see something like:::
+
+  [15:33:08.884275486] (+0.000225472) ubuntu zipkin:timestamp: { cpu_id = 53 }, { trace_name = "op", service_name = "Objecter", port_no = 0, ip = "0.0.0.0", trace_id = 5485970765435202833, span_id = 5485970765435202833, parent_span_id = 0, event = "osd op reply" }
+  [15:33:08.884614135] (+0.000002839) ubuntu zipkin:keyval_integer: { cpu_id = 10 }, { trace_name = "", service_name = "Messenger", port_no = 6805, ip = "0.0.0.0", trace_id = 7381732770245808782, span_id = 7387710183742669839, parent_span_id = 1205040135881905799, key = "tid", val = 2 }
+  [15:33:08.884616431] (+0.000002296) ubuntu zipkin:keyval_string: { cpu_id = 10 }, { trace_name = "", service_name = "Messenger", port_no = 6805, ip = "0.0.0.0", trace_id = 7381732770245808782, span_id = 7387710183742669839, parent_span_id = 1205040135881905799, key = "entity type", val = "client" }
+
+
+Install  Zipkin
+===============
+One of the points of using Blkin is so that you can look at the traces
+using Zipkin. Users should run Zipkin as a tracepoints collector and
+also a web service. The executable jar runs a collector on port 9410 and
+the web interface on port 9411
+
+Download Zipkin Package::
+
+  git clone https://github.com/openzipkin/zipkin && cd zipkin
+  wget -O zipkin.jar 'https://search.maven.org/remote_content?g=io.zipkin.java&a=zipkin-server&v=LATEST&c=exec'
+  java -jar zipkin.jar
+
+Or, launch docker image::
+
+  docker run -d -p 9411:9411 openzipkin/Zipkin
+
+Show Ceph's Blkin Traces in Zipkin-web
+======================================
+Download babeltrace-zipkin project. This project takes the traces
+generated with blkin and sends them to a Zipkin collector using scribe::
+
+  git clone https://github.com/vears91/babeltrace-zipkin
+  cd babeltrace-zipkin
+
+Send lttng data to Zipkin::
+
+  python3 babeltrace_zipkin.py ${lttng-traces-dir}/${blkin-test}/ust/uid/0/64-bit/ -p ${zipkin-collector-port(9410 by default)} -s ${zipkin-collector-ip}
+
+Example::
+
+  python3 babeltrace_zipkin.py ~/lttng-traces-dir/blkin-test-20150225-160222/ust/uid/0/64-bit/ -p 9410 -s 127.0.0.1
+
+Check Ceph traces on webpage::
+
+  Browse http://${zipkin-collector-ip}:9411
+  Click "Find traces"
diff --git a/doc/dev/bluestore.rst b/doc/dev/bluestore.rst
new file mode 100644
index 000000000..91d71d037
--- /dev/null
+++ b/doc/dev/bluestore.rst
@@ -0,0 +1,85 @@
+===================
+BlueStore Internals
+===================
+
+
+Small write strategies
+----------------------
+
+* *U*: Uncompressed write of a complete, new blob.
+
+  - write to new blob
+  - kv commit
+
+* *P*: Uncompressed partial write to unused region of an existing
+  blob.
+
+  - write to unused chunk(s) of existing blob
+  - kv commit
+
+* *W*: WAL overwrite: commit intent to overwrite, then overwrite
+  async.  Must be chunk_size = MAX(block_size, csum_block_size)
+  aligned.
+
+  - kv commit
+  - wal overwrite (chunk-aligned) of existing blob
+
+* *N*: Uncompressed partial write to a new blob.  Initially sparsely
+  utilized.  Future writes will either be *P* or *W*.
+
+  - write into a new (sparse) blob
+  - kv commit
+
+* *R+W*: Read partial chunk, then to WAL overwrite.
+
+  - read (out to chunk boundaries)
+  - kv commit
+  - wal overwrite (chunk-aligned) of existing blob
+
+* *C*: Compress data, write to new blob.
+
+  - compress and write to new blob
+  - kv commit
+
+Possible future modes
+---------------------
+
+* *F*: Fragment lextent space by writing small piece of data into a
+  piecemeal blob (that collects random, noncontiguous bits of data we
+  need to write).
+
+  - write to a piecemeal blob (min_alloc_size or larger, but we use just one block of it)
+  - kv commit
+
+* *X*: WAL read/modify/write on a single block (like legacy
+  bluestore).  No checksum.
+
+  - kv commit
+  - wal read/modify/write
+
+Mapping
+-------
+
+This very roughly maps the type of write onto what we do when we
+encounter a given blob.  In practice it's a bit more complicated since there
+might be several blobs to consider (e.g., we might be able to *W* into one or
+*P* into another), but it should communicate a rough idea of strategy.
+
++--------------------------+--------+--------------+-------------+--------------+---------------+
+|                          | raw    | raw (cached) | csum (4 KB) | csum (16 KB) | comp (128 KB) |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 128+ KB (over)write      | U      | U            | U           | U            | C             |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 64 KB (over)write        | U      | U            | U           | U            | U or C        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 4 KB overwrite           | W      | P | W        | P | W       | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte overwrite       | R+W    | P | W        | P | R+W     | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte append          | R+W    | P | W        | P | R+W     | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 4 KB clone overwrite     | P | N  | P | N        | P | N       | P | N        | N (F?)        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte clone overwrite | P | N  | P | N        | P | N       | P | N        | N (F?)        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
diff --git a/doc/dev/ceph-volume/index.rst b/doc/dev/ceph-volume/index.rst
new file mode 100644
index 000000000..5feef8089
--- /dev/null
+++ b/doc/dev/ceph-volume/index.rst
@@ -0,0 +1,14 @@
+===================================
+ceph-volume developer documentation
+===================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1
+
+
+   plugins
+   lvm
+   zfs
+   systemd
diff --git a/doc/dev/ceph-volume/lvm.rst b/doc/dev/ceph-volume/lvm.rst
new file mode 100644
index 000000000..f2df6d850
--- /dev/null
+++ b/doc/dev/ceph-volume/lvm.rst
@@ -0,0 +1,179 @@
+
+.. _ceph-volume-lvm-api:
+
+LVM
+===
+The backend of ``ceph-volume lvm`` is LVM, it relies heavily on the usage of
+tags, which is a way for LVM to allow extending its volume metadata. These
+values can later be queried against devices and it is how they get discovered
+later.
+
+.. warning:: These APIs are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
+
+
+.. _ceph-volume-lvm-tag-api:
+
+Tag API
+-------
+The process of identifying logical volumes as part of Ceph relies on applying
+tags on all volumes. It follows a naming convention for the namespace that
+looks like::
+
+    ceph.<tag name>=<tag value>
+
+All tags are prefixed by the ``ceph`` keyword to claim ownership of that
+namespace and make it easily identifiable. This is how the OSD ID would be used
+in the context of lvm tags::
+
+    ceph.osd_id=0
+
+
+.. _ceph-volume-lvm-tags:
+
+Metadata
+--------
+The following describes all the metadata from Ceph OSDs that is stored on an
+LVM volume:
+
+
+``type``
+--------
+Describes if the device is an OSD or Journal, with the ability to expand to
+other types when supported (for example a lockbox)
+
+Example::
+
+    ceph.type=osd
+
+
+``cluster_fsid``
+----------------
+Example::
+
+    ceph.cluster_fsid=7146B649-AE00-4157-9F5D-1DBFF1D52C26
+
+
+``data_device``
+---------------
+Example::
+
+    ceph.data_device=/dev/ceph/data-0
+
+
+``data_uuid``
+-------------
+Example::
+
+    ceph.data_uuid=B76418EB-0024-401C-8955-AE6919D45CC3
+
+
+``journal_device``
+------------------
+Example::
+
+    ceph.journal_device=/dev/ceph/journal-0
+
+
+``journal_uuid``
+----------------
+Example::
+
+    ceph.journal_uuid=2070E121-C544-4F40-9571-0B7F35C6CB2B
+
+
+``encrypted``
+-------------
+Example for enabled encryption with ``luks``::
+
+    ceph.encrypted=1
+
+When encryption is not supported or simply disabled::
+
+    ceph.encrypted=0
+
+
+``osd_fsid``
+------------
+Example::
+
+    ceph.osd_fsid=88ab9018-f84b-4d62-90b4-ce7c076728ff
+
+
+``osd_id``
+----------
+Example::
+
+    ceph.osd_id=1
+
+
+``block_device``
+----------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.block_device=/dev/mapper/vg-block-0
+
+
+``block_uuid``
+--------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.block_uuid=E5F041BB-AAD4-48A8-B3BF-31F7AFD7D73E
+
+
+``db_device``
+-------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.db_device=/dev/mapper/vg-db-0
+
+
+``db_uuid``
+-----------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.db_uuid=F9D02CF1-31AB-4910-90A3-6A6302375525
+
+
+``wal_device``
+--------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.wal_device=/dev/mapper/vg-wal-0
+
+
+``wal_uuid``
+------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.wal_uuid=A58D1C68-0D6E-4CB3-8E99-B261AD47CC39
+
+
+``vdo``
+-------
+A VDO-enabled device is detected when device is getting prepared, and then
+stored for later checks when activating. This affects mount options by
+appending the ``discard`` mount flag, regardless of mount flags being used.
+
+Example for an enabled VDO device::
+
+    ceph.vdo=1
diff --git a/doc/dev/ceph-volume/plugins.rst b/doc/dev/ceph-volume/plugins.rst
new file mode 100644
index 000000000..95bc761e2
--- /dev/null
+++ b/doc/dev/ceph-volume/plugins.rst
@@ -0,0 +1,65 @@
+.. _ceph-volume-plugins:
+
+Plugins
+=======
+``ceph-volume`` started initially to provide support for using ``lvm`` as
+the underlying system for an OSD. It is included as part of the tool but it is
+treated like a plugin.
+
+This modularity, allows for other device or device-like technologies to be able
+to consume and re-use the utilities and workflows provided.
+
+Adding Plugins
+--------------
+As a Python tool, plugins ``setuptools`` entry points. For a new plugin to be
+available, it should have an entry similar to this in its ``setup.py`` file:
+
+.. code-block:: python
+
+    setup(
+        ...
+        entry_points = dict(
+            ceph_volume_handlers = [
+                'my_command = my_package.my_module:MyClass',
+            ],
+        ),
+
+The ``MyClass`` should be a class that accepts ``sys.argv`` as its argument,
+``ceph-volume`` will pass that in at instantiation and call them ``main``
+method.
+
+This is how a plugin for ``ZFS`` could look like for example:
+
+.. code-block:: python
+
+    class ZFS(object):
+
+        help_menu = 'Deploy OSDs with ZFS'
+        _help = """
+    Use ZFS as the underlying technology for OSDs
+
+    --verbose   Increase the verbosity level
+        """
+
+        def __init__(self, argv):
+            self.argv = argv
+
+        def main(self):
+            parser = argparse.ArgumentParser()
+            args = parser.parse_args(self.argv)
+            ...
+
+And its entry point (via ``setuptools``) in ``setup.py`` would looke like:
+
+.. code-block:: python
+
+        entry_points = {
+            'ceph_volume_handlers': [
+                'zfs = ceph_volume_zfs.zfs:ZFS',
+            ],
+        },
+
+After installation, the ``zfs`` subcommand would be listed and could be used
+as::
+
+    ceph-volume zfs
diff --git a/doc/dev/ceph-volume/systemd.rst b/doc/dev/ceph-volume/systemd.rst
new file mode 100644
index 000000000..8553430ee
--- /dev/null
+++ b/doc/dev/ceph-volume/systemd.rst
@@ -0,0 +1,37 @@
+.. _ceph-volume-systemd-api:
+
+systemd
+=======
+The workflow to *"activate"* an OSD is by relying on systemd unit files and its
+ability to persist information as a suffix to the instance name.
+
+``ceph-volume`` exposes the following convention for unit files::
+
+    ceph-volume@<sub command>-<extra metadata>
+
+For example, this is how enabling an OSD could look like for the 
+:ref:`ceph-volume-lvm` sub command::
+
+    systemctl enable ceph-volume@lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+
+These 3 pieces of persisted information are needed by the sub-command so that
+it understands what OSD it needs to activate.
+
+Since ``lvm`` is not the only subcommand that will be supported, this
+is how it will allow other device types to be defined.
+
+At some point for example, for plain disks, it could be::
+
+    systemctl enable ceph-volume@disk-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+At startup, the systemd unit will execute a helper script that will parse the
+suffix and will end up calling ``ceph-volume`` back. Using the previous
+example for lvm, that call will look like::
+
+    ceph-volume lvm activate 0 8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+
+.. warning:: These workflows are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
diff --git a/doc/dev/ceph-volume/zfs.rst b/doc/dev/ceph-volume/zfs.rst
new file mode 100644
index 000000000..18de7652a
--- /dev/null
+++ b/doc/dev/ceph-volume/zfs.rst
@@ -0,0 +1,176 @@
+
+.. _ceph-volume-zfs-api:
+
+ZFS
+===
+The backend of ``ceph-volume zfs`` is ZFS, it relies heavily on the usage of
+tags, which is a way for ZFS to allow extending its volume metadata. These
+values can later be queried against devices and it is how they get discovered
+later.
+
+Currently this interface is only usable when running on FreeBSD.
+
+.. warning:: These APIs are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
+
+
+.. _ceph-volume-zfs-tag-api:
+
+Tag API
+-------
+The process of identifying filesystems, volumes and pools as part of Ceph relies
+on applying tags on all volumes. It follows a naming convention for the
+namespace that looks like::
+
+    ceph.<tag name>=<tag value>
+
+All tags are prefixed by the ``ceph`` keyword to claim ownership of that
+namespace and make it easily identifiable. This is how the OSD ID would be used
+in the context of zfs tags::
+
+    ceph.osd_id=0
+
+Tags on filesystems are stored as property.
+Tags on a zpool are stored in the comment property as a concatenated list 
+separated by ``;`` 
+
+.. _ceph-volume-zfs-tags:
+
+Metadata
+--------
+The following describes all the metadata from Ceph OSDs that is stored on a
+ZFS filesystem, volume, pool:
+
+
+``type``
+--------
+Describes if the device is an OSD or Journal, with the ability to expand to
+other types when supported 
+
+Example::
+
+    ceph.type=osd
+
+
+``cluster_fsid``
+----------------
+Example::
+
+    ceph.cluster_fsid=7146B649-AE00-4157-9F5D-1DBFF1D52C26
+
+
+``data_device``
+---------------
+Example::
+
+    ceph.data_device=/dev/ceph/data-0
+
+
+``data_uuid``
+-------------
+Example::
+
+    ceph.data_uuid=B76418EB-0024-401C-8955-AE6919D45CC3
+
+
+``journal_device``
+------------------
+Example::
+
+    ceph.journal_device=/dev/ceph/journal-0
+
+
+``journal_uuid``
+----------------
+Example::
+
+    ceph.journal_uuid=2070E121-C544-4F40-9571-0B7F35C6CB2B
+
+
+``osd_fsid``
+------------
+Example::
+
+    ceph.osd_fsid=88ab9018-f84b-4d62-90b4-ce7c076728ff
+
+
+``osd_id``
+----------
+Example::
+
+    ceph.osd_id=1
+
+
+``block_device``
+----------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.block_device=/dev/gpt/block-0
+
+
+``block_uuid``
+--------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.block_uuid=E5F041BB-AAD4-48A8-B3BF-31F7AFD7D73E
+
+
+``db_device``
+-------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.db_device=/dev/gpt/db-0
+
+
+``db_uuid``
+-----------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.db_uuid=F9D02CF1-31AB-4910-90A3-6A6302375525
+
+
+``wal_device``
+--------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.wal_device=/dev/gpt/wal-0
+
+
+``wal_uuid``
+------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.wal_uuid=A58D1C68-0D6E-4CB3-8E99-B261AD47CC39
+
+
+``compression``
+---------------
+A compression-enabled device can always be set using the native zfs settings on
+a volume or filesystem. This will/can be activated during creation of the volume
+of filesystem. 
+When activated by ``ceph-volume zfs`` this tag will be created.
+Compression manually set AFTER ``ceph-volume`` will go unnoticed, unless this 
+tag is also manually set.
+
+Example for an enabled compression device::
+
+    ceph.vdo=1
diff --git a/doc/dev/ceph_krb_auth.rst b/doc/dev/ceph_krb_auth.rst
new file mode 100644
index 000000000..92416e928
--- /dev/null
+++ b/doc/dev/ceph_krb_auth.rst
@@ -0,0 +1,1096 @@
+===============================================================================
+A Detailed Documentation on How to Set up Ceph Kerberos Authentication
+===============================================================================
+
+This document provides details on the Kerberos authorization protocol. This is
+the 1st draft and we will try to keep it updated along with code changes that
+might take place.
+
+Several free implementations of this protocol are available (MIT, Heimdal,
+MS...), covering a wide range of operating systems. The Massachusetts
+Institute of Technology (MIT), where Kerberos was originally developed,
+continues to develop their Kerberos package and it is the implementation we
+chose to work with. `MIT Kerberos <http://web.mit.edu/Kerberos/>`_.
+
+Please, provide feedback to Daniel Oliveira (doliveira@suse.com)
+
+*Last update: Dec 3, 2018*
+
+|
+
+Background
+----------
+
+Before we get into *Kerberos details*, let us define a few terms so we can
+understand what to expect from it, *what it can and can't do*:
+
+Directory Services
+    A directory service is a customizable information store that functions as
+    a single point from which users can locate resources and services
+    distributed throughout the network. This customizable information store
+    also gives administrators a single point for managing its objects and their
+    attributes. Although this information store appears as a single point to
+    the users of the network, it is actually most often stored in a distributed
+    form. A directory service consists of at least one *Directory Server and a
+    Directory Client* and are implemented based on *X.500 standards*.
+
+    *OpenLDAP, 389 Directory Server, MS Active Directory, NetIQ eDirectory* are
+    some good examples.
+
+    A directory service is often characterized as a *write-once-read-many-times
+    service*, meaning the data that would normally be stored in an directory
+    service would not be expected to change on every access.
+
+    The database that forms a directory service *is not designed for
+    transactional data*.
+
+|
+
+LDAP (Lightweight Directory Access Protocol v3)
+    LDAP is a set of LDAP Protocol Exchanges *(not an implementation of a
+    server)* that defines the method by which data is accessed. LDAPv3 is a
+    standard defined by the IETF in RFC 2251 and describes how data is
+    represented in the Directory Service (the Data Model or DIT).
+
+    Finally, it defines how data is loaded into (imported) and saved from
+    (exported) a directory service (using LDIF). LDAP does not define how data
+    is stored or manipulated. Data Store is an 'automagic' process as far as
+    the standard is concerned and is generally handled by back-end modules.
+
+    No Directory Service implementation has all the features of LDAP v3
+    protocol implemented. All Directory Server implementations have their
+    different problems and/or anomalies, and features that may not return
+    results as another Directory Server implementation would.
+
+|
+
+Authentication
+    Authentication is about validating credentials (like User Name/ID and
+    password) to verify the identity. The system determines whether one is what
+    they say they are using their credentials.
+
+    Usually, authentication is done by a username and password, and sometimes
+    in conjunction with *(single, two, or multi) factors of authentication*,
+    which refers to the various ways to be authenticated.
+
+|
+
+Authorization
+    Authorization occurs after the identity is successfully authenticated by
+    the system, which ultimately gives one full permission to access the
+    resources such as information, files, databases, and so forth, almost
+    anything. It determines the ability to access the system and up to what
+    extent (what kind of permissions/rights are given and to where/what).
+
+|
+
+Auditing
+    Auditing takes the results from both *authentication and authorization* and
+    records them into an audit log. The audit log records all actions
+    taking by/during the authentication and authorization for later review by
+    the administrators. While authentication and authorization are preventive
+    systems (in which unauthorized access is prevented), auditing is a reactive
+    system (in which it gives detailed log of how/when/where someone accessed
+    the environment).
+
+|
+
+Kerberos (KRB v5)
+    Kerberos is a network *authentication protocol*. It is designed to provide
+    strong authentication for client/server applications by using secret-key
+    cryptography (symmetric key). A free implementation of this protocol is
+    available from the MIT. However, Kerberos is available in many commercial
+    products as well.
+
+    It was designed to provide secure authentication to services over an
+    insecure network. Kerberos uses tickets to authenticate a user, or service
+    application and never transmits passwords over the network in the clear.
+    So both client and server can prove their identity without sending any
+    unencrypted secrets over the network.
+
+    Kerberos can be used for single sign-on (SSO). The idea behind SSO is
+    simple, we want to login just once and be able to use any service that we
+    are entitled to, without having to login on each of those services.
+
+|
+
+Simple Authentication and Security Layer (SASL)
+    SASL **(RFC 4422)** is a framework that helps developers to implement
+    different authentication mechanisms (implementing a series of challenges
+    and responses), allowing both clients and servers to negotiate a mutually
+    acceptable mechanism for each connection, instead of hard-coding them.
+
+    Examples of SASL mechanisms:
+
+        * ANONYMOUS **(RFC 4505)**
+
+            - For guest access, meaning *unauthenticated*
+
+        * CRAM-MD5 **(RFC 2195)**
+
+            - Simple challenge-response scheme based on *HMAC-MD5*.
+              It does not establish any security layer. *Less secure than
+              DIGEST-MD5 and GSSAPI.*
+
+        * DIGEST-MD5 **(RFC 2831)**
+
+            - HTTP Digest compatible *(partially)* challenge-response scheme
+              based upon MD5, offering a *data security layer*. It is preferred
+              over PLAIN text passwords, protecting against plain text attacks.
+              It is a mandatory authentication method for LDAPv3 servers.
+
+        * EXTERNAL **(RFCs 4422, 5246, 4301, 2119)**
+
+            - Where *authentication is implicit* in the context (i.e; for
+              protocols using IPsec or TLS [TLS/SSL to performing certificate-
+              based authentication] already). This method uses public keys for
+              strong authentication.
+
+        * GS2 **(RFC 5801)**
+
+            - Family of mechanisms supports arbitrary GSS-API mechanisms in
+              SASL
+
+        * NTLM (MS Proprietary)
+
+            - MS Windows NT LAN Manager authentication mechanism
+
+        * OAuth 1.0/2.0 **(RFCs 5849, 6749, 7628)**
+
+            - Authentication protocol for delegated resource access
+
+        * OTP **(RFC 2444)**
+
+            - One-time password mechanism *(obsoletes the SKEY mechanism)*
+
+        * PLAIN **(RFC 4616)**
+
+            - Simple Cleartext password mechanism **(RFC 4616)**. This is not a
+              preferred mechanism for most applications because of its relative
+              lack of strength.
+
+        * SCRAM **(RFCs 5802, 7677)**
+
+            - Modern challenge-response scheme based mechanism with channel
+              binding support
+
+|
+
+Generic Security Services Application Program Interface (GSSAPI)
+    GSSAPI **(RFCs 2078, 2743, 2744, 4121, 4752)** is widely used by protocol
+    implementers as a way to implement Kerberos v5 support in their
+    applications. It provides a generic interface and message format that can
+    encapsulate authentication exchanges from any authentication method that
+    has a GSSAPI-compliant library.
+
+    It does not define a protocol, authentication, or security mechanism
+    itself; it instead makes it easier for application programmers to support
+    multiple authentication mechanisms by providing a uniform, generic API for
+    security services. It is a set of functions that include both an API and a
+    methodology for approaching authentication, aiming to insulate application
+    protocols from the specifics of security protocols as much as possible.
+
+    *Microsoft Windows Kerberos* implementation does not include GSSAPI support
+    but instead includes a *Microsoft-specific API*, the *Security Support
+    Provider Interface (SSPI)*. In Windows, an SSPI client can communicate with
+    a *GSSAPI server*.
+
+    *Most applications that support GSSAPI also support Kerberos v5.*
+
+|
+
+Simple and Protected GSSAPI Negotiation Mechanism (SPNEGO)
+    As we can see, GSSAPI solves the problem of providing a single API to
+    different authentication mechanisms. However, it does not solve the problem
+    of negotiating which mechanism to use. In fact for GSSAPI to work, the two
+    applications communicating with each other must know in advance what
+    authentication mechanism they plan to use, which usually is not a problem
+    if only one mechanism is supported (meaning Kerberos v5).
+
+    However, if there are multiple mechanisms to choose from, a method is
+    needed to securely negotiate an authentication mechanism that is mutually
+    supported between both client and server; which is where
+    *SPNEGO (RFC 2478, 4178)* makes a difference.
+
+    *SPNEGO* provides a framework for two parties that are engaged in
+    authentication to select from a set of possible authentication mechanisms,
+    in a manner that preserves the opaque nature of the security protocols to
+    the application protocol that uses it.
+
+    It is a security protocol that uses a *GSSAPI authentication mechanism* and
+    negotiates among several available authentication mechanisms in an
+    implementation, selecting one for use to satisfy the authentication needs
+    of the application protocol.
+
+    It is a *meta protocol* that travels entirely in other application
+    protocols; it is never used directly without an application protocol.
+
+|
+
+*Why is this important and why do we care? Like, at all?*
+
+    Having this background information in mind, we can easily describe things
+    like:
+
+        1. *Ceph Kerberos authentication* is based totally on MIT *Kerberos*
+        implementation using *GSSAPI*.
+
+        2. At the moment we are still using *Kerberos default backend
+        database*, however we plan on adding LDAP as a backend which would
+        provide us with *authentication with GSSAPI (KRB5)* and *authorization
+        with LDAP (LDAPv3)*, via *SASL mechanism*.
+
+|
+
+Before We Start
+---------------
+
+We assume the environment already has some external services up and running
+properly:
+
+    * Kerberos needs to be properly configured, which also means (for both
+      every server and KDC):
+
+        - Time Synchronization (either using `NTP <http://www.ntp.org/>`_  or `chrony <https://chrony.tuxfamily.org/>`_).
+
+            + Not only Kerberos, but also Ceph depends and relies on time
+              synchronization.
+
+        - DNS resolution
+
+            + Both *(forward and reverse)* zones, with *fully qualified domain
+              name (fqdn)* ``(hostname + domain.name)``
+
+            + KDC discover can be set up to use DNS ``(srv resources)`` as
+              service location protocol *(RFCs 2052, 2782)*, as well as *host
+              or domain* to the *appropriate realm* ``(txt record)``.
+
+            + Even though these DNS entries/settings are not required to run a
+              ``Kerberos realm``, they certainly help to eliminate the need for
+              manual configuration on all clients.
+
+            + This is extremely important, once most of the Kerberos issues are
+              usually related to name resolution. Kerberos is very picky when
+              checking on systems names and host lookups.
+
+    * Whenever possible, in order to avoid a *single point of failure*, set up
+      a *backup, secondary, or slave*, for every piece/part in the
+      infrastructure ``(ntp, dns, and kdc servers)``.
+
+
+Also, the following *Kerberos terminology* is important:
+
+    * Ticket
+
+        - Tickets or Credentials, are a set of information that can be used to
+          verify the client's identity. Kerberos tickets may be stored in a
+          file, or they may exist only in memory.
+
+        - The first ticket obtained is a ticket-granting ticket (TGT), which
+          allows the clients to obtain additional tickets. These additional
+          tickets give the client permission for specific services. The
+          requesting and granting of these additional tickets happens
+          transparently.
+
+            + The TGT, which expires at a specified time, permits the client to
+              obtain additional tickets, which give permission for specific
+              services. The requesting and granting of these additional tickets
+              is user-transparent.
+
+    * Key Distribution Center (KDC).
+
+        - The KDC creates a ticket-granting ticket (TGT) for the client,
+          encrypts it using the client's password as the key, and sends the
+          encrypted TGT back to the client. The client then attempts to decrypt
+          the TGT, using its password. If the client successfully decrypts the
+          TGT (i.e., if the client gave the correct password), it keeps the
+          decrypted TGT, which indicates proof of the client's identity.
+
+        -  The KDC is comprised of three components:
+
+            + Kerberos database, which stores all the information about the
+              principals and the realm they belong to, among other things.
+            + Authentication service (AS)
+            + Ticket-granting service (TGS)
+
+    * Client
+
+        - Either a *user, host or a service* who sends a request for a ticket.
+
+    * Principal
+
+        - It is a unique identity to which Kerberos can assign tickets.
+          Principals can have an arbitrary number of components. Each component
+          is separated by a component separator, generally ``/``. The last
+          component is the *realm*, separated from the rest of the principal by
+          the realm separator, generally ``@``.
+
+        - If there is no realm component in the principal, then it will be
+          assumed that the principal is in the default realm for the context in
+          which it is being used.
+
+        - Usually, a principal is divided into three parts:
+
+            + The ``primary``, the ``instance``, and the ``realm``
+
+            + The format of a typical Kerberos V5 principal is
+              ``primary/instance@REALM``.
+
+            + The ``primary`` is the first part of the principal. In the case
+              of a user, it's the same as the ``username``. For a host, the
+              primary is the word ``host``. For Ceph, will use ``ceph`` as a
+              primary name which makes it easier to organize and identify Ceph
+              related principals.
+
+            + The ``instance`` is an optional string that qualifies the
+              primary. The instance is separated from the primary by a slash
+              ``/``. In the case of a user, the instance is usually ``null``,
+              but a user might also have an additional principal, with an
+              instance called ``admin``, which one uses to administrate a
+              database.
+
+              The principal ``johndoe@MYDOMAIN.COM`` is completely separate
+              from the principal ``johndoe/admin@MYDOMAIN.COM``, with a
+              separate password, and separate permissions. In the case of a
+              host, the instance is the fully qualified hostname,
+              i.e., ``osd1.MYDOMAIN.COM``.
+
+            + The ``realm`` is the Kerberos realm. Usually, the Kerberos realm
+              is the domain name, in *upper-case letters*. For example, the
+              machine ``osd1.MYDOMAIN.COM`` would be in the realm
+              ``MYDOMAIN.COM``.
+
+    * Keytab
+
+        - A keytab file stores the actual encryption key that can be used in
+          lieu of a password challenge for a given principal. Creating keytab
+          files are useful for noninteractive principals, such as *Service
+          Principal Names*, which are often associated with long-running
+          processes like Ceph daemons. A keytab file does not have to be a
+          "1:1 mapping" to a single principal. Multiple different principal
+          keys can be stored in a single keytab file:
+
+            + The keytab file allows a user/service to authenticate without
+              knowledge of the password. Due to this, *keytabs should be
+              protected* with appropriate controls to prevent unauthorized
+              users from authenticating with it.
+
+            + The default client keytab file is ``/etc/krb5.keytab``
+
+|
+
+The 'Ceph side' of the things
+------------------------------
+
+In order to configure connections (from Ceph nodes) to the KDC:
+
+1. Login to the Kerberos client (Ceph server nodes) and confirm it is properly
+   configured, by checking and editing ``/etc/krb5.conf`` file properly:  ::
+
+    /etc/krb5.conf
+    [libdefaults]
+        dns_canonicalize_hostname = false
+        rdns = false
+        forwardable = true
+        dns_lookup_realm = true
+        dns_lookup_kdc = true
+        allow_weak_crypto = false
+        default_realm = MYDOMAIN.COM
+        default_ccache_name = KEYRING:persistent:%{uid}
+    [realms]
+        MYDOMAIN.COM = {
+            kdc = kerberos.mydomain.com
+            admin_server = kerberos.mydomain.com
+            ...
+        }
+    ...
+
+
+2. Login to the *KDC Server* and confirm it is properly configured to
+   authenticate to the Kerberos realm in question:
+
+    a. Kerberos related DNS RRs:  ::
+
+        /var/lib/named/master/mydomain.com
+        kerberos                IN A        192.168.10.21
+        kerberos-slave          IN A        192.168.10.22
+        _kerberos               IN TXT      "MYDOMAIN.COM"
+        _kerberos._udp          IN SRV      1 0 88 kerberos
+        _kerberos._tcp          IN SRV      1 0 88 kerberos
+        _kerberos._udp          IN SRV      20 0 88 kerberos-slave
+        _kerberos-master._udp   IN SRV      0 0 88 kerberos
+        _kerberos-adm._tcp      IN SRV      0 0 749 kerberos
+        _kpasswd._udp           IN SRV      0 0 464 kerberos
+        ...
+
+
+    b. KDC configuration file:  ::
+
+        /var/lib/kerberos/krb5kdc/kdc.conf
+        [kdcdefaults]
+                kdc_ports = 750,88
+        [realms]
+                MYDOMAIN.COM = {
+                    acl_file = /var/lib/kerberos/krb5kdc/kadm5.acl
+                    admin_keytab = FILE:/var/lib/kerberos/krb5kdc/kadm5.keytab
+                    default_principal_flags = +postdateable +forwardable +renewable +proxiable
+                                                            +dup-skey -preauth -hwauth +service
+                                                            +tgt-based +allow-tickets -pwchange
+                                                            -pwservice
+                    dict_file = /var/lib/kerberos/krb5kdc/kadm5.dict
+                    key_stash_file = /var/lib/kerberos/krb5kdc/.k5.MYDOMAIN.COM
+                    kdc_ports = 750,88
+                    max_life = 0d 10h 0m 0s
+                    max_renewable_life = 7d 0h 0m 0s
+                }
+        ...
+
+
+3. Still on the KDC Server, run the Kerberos administration utility;
+   ``kadmin.local`` so we can list all the principals already created.  ::
+
+    kadmin.local:  listprincs
+    K/M@MYDOMAIN.COM
+    krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+    kadmin/admin@MYDOMAIN.COM
+    kadmin/changepw@MYDOMAIN.COM
+    kadmin/history@MYDOMAIN.COM
+    kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+    root/admin@MYDOMAIN.COM
+    ...
+
+
+4. Add a *principal for each Ceph cluster node* we want to be authenticated by
+   Kerberos:
+
+    a. Adding principals:  ::
+
+        kadmin.local:  addprinc -randkey ceph/ceph-mon1
+        Principal "ceph/ceph-mon1@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd1
+        Principal "ceph/ceph-osd1@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd2
+        Principal "ceph/ceph-osd2@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd3
+        Principal "ceph/ceph-osd3@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd4
+        Principal "ceph/ceph-osd4@MYDOMAIN.COM" created.
+        kadmin.local:  listprincs
+        K/M@MYDOMAIN.COM
+        krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+        kadmin/admin@MYDOMAIN.COM
+        kadmin/changepw@MYDOMAIN.COM
+        kadmin/history@MYDOMAIN.COM
+        kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+        root/admin@MYDOMAIN.COM
+        ceph/ceph-mon1@MYDOMAIN.COM
+        ceph/ceph-osd1@MYDOMAIN.COM
+        ceph/ceph-osd2@MYDOMAIN.COM
+        ceph/ceph-osd3@MYDOMAIN.COM
+        ceph/ceph-osd4@MYDOMAIN.COM
+        ...
+
+
+    b. This follows the same idea if we are creating a *user principal*  ::
+
+        kadmin.local:  addprinc johndoe
+        WARNING: no policy specified for johndoe@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "johndoe@MYDOMAIN.COM":
+        Re-enter password for principal "johndoe@MYDOMAIN.COM":
+        Principal "johndoe@MYDOMAIN.COM" created.
+        ...
+
+
+5. Create a *keytab file* for each Ceph cluster node:
+
+    As the default client keytab file is ``/etc/krb5.keytab``, we will want to
+    use a different file name, so we especify which *keytab file to create* and
+    which *principal to export keys* from:  ::
+
+        kadmin.local:  ktadd -k /etc/gss_client_mon1.ktab ceph/ceph-mon1
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd1.ktab ceph/ceph-osd1
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd2.ktab ceph/ceph-osd2
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd3.ktab ceph/ceph-osd3
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd4.ktab ceph/ceph-osd4
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+
+        # ls -1 /etc/gss_client_*
+        /etc/gss_client_mon1.ktab
+        /etc/gss_client_osd1.ktab
+        /etc/gss_client_osd2.ktab
+        /etc/gss_client_osd3.ktab
+        /etc/gss_client_osd4.ktab
+
+
+    We can also check these newly created keytab client files by:  ::
+
+        # klist -kte /etc/gss_client_mon1.ktab
+        Keytab name: FILE:/etc/gss_client_mon1.ktab
+        KVNO Timestamp           Principal
+        ---- ------------------- ------------------------------------------------------
+           2 10/8/2018 14:35:30 ceph/ceph-mon1@MYDOMAIN.COM (aes256-cts-hmac-sha1-96)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (aes128-cts-hmac-sha1-96)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (des3-cbc-sha1)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (arcfour-hmac)
+        ...
+
+
+6. A new *set parameter* was added in Ceph, ``gss_ktab_client_file`` which
+   points to the keytab file related to the Ceph node *(or principal)* in
+   question.
+
+    By default it points to ``/var/lib/ceph/$name/gss_client_$name.ktab``. So,
+    in the case of a Ceph server ``osd1.mydomain.com``, the location and name
+    of the keytab file should be: ``/var/lib/ceph/osd1/gss_client_osd1.ktab``
+
+    Therefore, we need to ``scp`` each of these newly created keytab files from
+    the KDC to their respective Ceph cluster nodes (i.e):
+    ``# for node in mon1 osd1 osd2 osd3 osd4; do scp /etc/gss_client_$node*.ktab root@ceph-$node:/var/lib/ceph/$node/; done``
+
+    Or whatever other way one feels comfortable with, as long as each keytab
+    client file gets copied over to the proper location.
+
+    At this point, even *without using any keytab client file* we should be
+    already able to authenticate a *user principal*:  ::
+
+        # kdestroy -A && kinit -f johndoe && klist -f
+        Password for johndoe@MYDOMAIN.COM:
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: johndoe@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        10/10/2018 15:32:01  10/11/2018 07:32:01  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+            renew until 10/11/2018 15:32:01, Flags: FRI
+        ...
+
+
+    Given that the *keytab client file* is/should already be copied and available at the
+    Kerberos client (Ceph cluster node), we should be able to authenticate using it before
+    continuing:  ::
+
+        # kdestroy -A && kinit -k -t /etc/gss_client_mon1.ktab -f 'ceph/ceph-mon1@MYDOMAIN.COM' && klist -f
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: ceph/ceph-mon1@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        10/10/2018 15:54:25  10/11/2018 07:54:25  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+            renew until 10/11/2018 15:54:25, Flags: FRI
+        ...
+
+
+7. The default client keytab is used, if it is present and readable, to
+   automatically obtain initial credentials for GSSAPI client applications. The
+   principal name of the first entry in the client keytab is used by default
+   when obtaining initial credentials:
+
+    a. The ``KRB5_CLIENT_KTNAME environment`` variable.
+    b. The ``default_client_keytab_name`` profile variable in ``[libdefaults]``.
+    c. The hardcoded default, ``DEFCKTNAME``.
+
+    So, what we do is to internally, set the environment variable
+    ``KRB5_CLIENT_KTNAME`` to the same location as ``gss_ktab_client_file``,
+    so ``/var/lib/ceph/osd1/gss_client_osd1.ktab``, and change the ``ceph.conf``
+    file to add the new authentication method.  ::
+
+        /etc/ceph/ceph.conf
+        [global]
+            ...
+            auth_cluster_required = gss
+            auth_service_required = gss
+            auth_client_required = gss
+            gss_ktab_client_file = /{$my_new_location}/{$my_new_ktab_client_file.keytab}
+            ...
+
+
+8. With that the GSSAPIs will then be able to read the keytab file and using
+   the process of name and service resolution *(provided by the DNS)*, able to
+   request a *TGT* as follows:
+
+    a. User/Client sends principal identity and credentials to the KDC Server
+       (TGT request).
+    b. KDC checks its internal database for the principal in question.
+    c. a TGT is created and wrapped by the KDC, using the principal's key
+       (TGT + Key).
+    d. The newly created TGT, is decrypted and stored in the credentials
+       cache.
+    e. At this point, Kerberos/GSSAPI aware applications (and/or services) are
+       able to check the list of active TGT in the keytab file.
+
+|
+|
+
+** *For Ceph Developers Only* **
+=================================
+
+We certainly could have used straight native ``KRB5 APIs`` (instead of
+``GSSAPIs``), but we wanted a more portable option as regards network security,
+which is the hallmark of the ``GSS`` *(Generic Security Standard)* ``-API``.
+It does not actually provide security services itself.
+
+Rather, it is a framework that provides security services to callers in a
+generic way.
+
+.. ditaa::
+
+    +---------------------------------+
+    |        Application              |
+    +---------------------------------+
+    | Protocol (RPC, Etc. [Optional]) |
+    +---------------------------------+
+    |         GSS-API                 |
+    +---------------------------------+
+    |   Security Mechs (Krb v5, Etc)  |
+    +---------------------------------+
+
+
+The GSS-API does two main things:
+
+    1. It creates a security context in which data can be passed between
+       applications. A context can be thought of as a sort of *"state of trust"*
+       between two applications.
+
+       Applications that share a context know who each other are and thus can
+       permit data transfers between them as long as the context lasts.
+
+    2. It applies one or more types of protection, known as *"security services"*,
+       to the data to be transmitted.
+
+
+GSS-API provides several types of portability for applications:
+
+    a. **Mechanism independence.** GSS-API provides a generic interface to the
+       mechanisms for which it has been implemented. By specifying a default
+       security mechanism, an application does not need to know which mechanism
+       it is using (for example, Kerberos v5), or even what type of mechanism
+       it uses. As an example, when an application forwards a user's credential
+       to a server, it does not need to know if that credential has a Kerberos
+       format or the format used by some other mechanism, nor how the
+       credentials are stored by the mechanism and accessed by the application.
+       (If necessary, an application can specify a particular mechanism to use)
+
+    b. **Protocol independence.** The GSS-API is independent of any
+       communications protocol or protocol suite. It can be used with
+       applications that use, for example, sockets, RCP, or TCP/IP.
+       RPCSEC_GSS "RPCSEC_GSS Layer" is an additional layer that smoothly
+       integrates GSS-API with RPC.
+
+    c. **Platform independence.** The GSS-API is completely oblivious to the
+       type of operating system on which an application is running.
+
+    d. **Quality of Protection independence.** Quality of Protection (QOP) is
+       the name given to the type of algorithm used in encrypting data or
+       generating cryptographic tags; the GSS-API allows a programmer to ignore
+       QOP, using a default provided by the GSS-API.
+       (On the other hand, an application can specify the QOP if necessary.)
+
+    The basic security offered by the GSS-API is authentication. Authentication
+    is the verification of an identity: *if you are authenticated, it means
+    that you are recognized to be who you say you are.*
+
+    The GSS-API provides for two additional security services, if supported by the
+    underlying mechanisms:
+
+    1. **Integrity:** It's not always sufficient to know that an application
+       sending you data is who it claims to be. The data itself could have
+       become corrupted or compromised.
+
+       The GSS-API provides for data to be accompanied by a cryptographic tag,
+       known as an ``Message Integrity Code (MIC)``, to prove that the data
+       that arrives at your doorstep is the same as the data that the sender
+       transmitted. This verification of the data's validity is known as
+       *"integrity"*.
+
+    2. **Confidentiality:** Both authentication and integrity, however, leave
+       the data itself alone, so if it's somehow intercepted, others can read
+       it.
+
+       The GSS-API therefore allows data to be encrypted, if underlying
+       mechanisms support it. This encryption of data is known as *"confidentiality"*.
+
+|
+
+Mechanisms Available With GSS-API:
+
+    The current implementation of the GSS-API works only with the Kerberos v5 security
+    mechanism.  ::
+
+        Mechanism Name          Object Identifier       Shared Library  Kernel Module
+        ----------------------  ----------------------  --------------  --------------
+        diffie_hellman_640_0    1.3.6.4.1.42.2.26.2.4   dh640-0.so.1
+        diffie_hellman_1024_0   1.3.6.4.1.42.2.26.2.5   dh1024-0.so.1
+        SPNEGO                  1.3.6.1.5.5.2
+        iakerb                  1.3.6.1.5.2.5
+        SCRAM-SHA-1             1.3.6.1.5.5.14
+        SCRAM-SHA-256           1.3.6.1.5.5.18
+        GSS-EAP (arc)           1.3.6.1.5.5.15.1.1.*
+        kerberos_v5             1.2.840.113554.1.2.2    gl/mech_krb5.so gl_kmech_krb5
+
+        Therefore:
+            Kerberos Version 5 GSS-API Mechanism
+            OID {1.2.840.113554.1.2.2}
+
+            Kerberos Version 5 GSS-API Mechanism
+            Simple and Protected GSS-API Negotiation Mechanism
+            OID {1.3.6.1.5.5.2}
+
+
+    There are two different formats:
+
+        1. The first, ``{ 1 2 3 4 }``, is officially mandated by the GSS-API
+           specs. ``gss_str_to_oid()`` expects this first format.
+
+        2. The second, ``1.2.3.4``, is more widely used but is not an official
+           standard format.
+
+    Although the GSS-API makes protecting data simple, it does not do certain
+    things, in order to maximize its generic nature. These include:
+
+        a. Provide security credentials for a user or application. These must
+           be provided by the underlying security mechanism(s). The GSS-API
+           does allow applications to acquire credentials, either automatically
+           or explicitly.
+
+        b. Transfer data between applications. It is the application's
+           responsibility to handle the transfer of all data between peers,
+           whether it is security-related or "plain" data.
+
+        c. Distinguish between different types of transmitted data (for
+           example, to know or determine that a data packet is plain data and
+           not GSS-API related).
+
+        d. Indicate status due to remote (asynchronous) errors.
+
+        e. Automatically protect information sent between processes of a
+           multiprocess program.
+
+        f. Allocate string buffers ("Strings and Similar Data") to be passed to
+           GSS-API functions.
+
+        g. Deallocate GSS-API data spaces. These must be explicitly deallocated
+           with functions such as ``gss_release_buffer()`` and
+           ``gss_delete_name()``.
+
+|
+
+These are the basic steps in using the GSS-API:
+
+    1. Each application, sender and recipient, acquires credentials explicitly,
+       if credentials have not been acquired automatically.
+
+    2. The sender initiates a security context and the recipient accepts it.
+
+    3. The sender applies security protection to the message (data) it wants to
+       transmit. This means that it either encrypts the message or stamps it
+       with an identification tag. The sender transmits the protected message.
+       (The sender can choose not to apply either security protection, in which
+       case the message has only the default GSS-API security service
+       associated with it. That is authentication, in which the recipient knows
+       that the sender is who it claims to be.)
+
+    4. The recipient decrypts the message (if needed) and verifies it
+       (if appropriate).
+
+    5. (Optional) The recipient returns an identification tag to the sender for
+       confirmation.
+
+    6. Both applications destroy the shared security context. If necessary,
+       they can also deallocate any *"leftover"* GSS-API data.
+
+    Applications that use the GSS-API should include the file ``gssapi.h``.
+
+    Good References:
+        - `rfc1964 <https://tools.ietf.org/html/rfc1964>`_.
+        - `rfc2743 <https://tools.ietf.org/html/rfc2743>`_.
+        - `rfc2744 <https://tools.ietf.org/html/rfc2744>`_.
+        - `rfc4178 <https://tools.ietf.org/html/rfc4178>`_.
+        - `rfc6649 <https://tools.ietf.org/html/rfc6649>`_.
+        - `MIT Kerberos Documentation <https://web.mit.edu/kerberos/krb5-latest/doc/appdev/gssapi.html>`_.
+
+|
+
+** *Kerberos Server Setup* **
+------------------------------
+
+First and foremost, ``this is not a recommendation for a production
+environment``. We are not covering ``Master/Slave replication cluster`` or
+anything production environment related (*ntp/chrony, dns, pam/nss, sssd, etc*).
+
+Also, on the server side there might be different dependencies and/or
+configuration steps needed, depending on which backend database will be used.
+``LDAP as a backend database`` is a good example of that.
+
+On the client side there are different steps depending on which client backend
+configuration will be used. For example ``PAM/NSS`` or ``SSSD`` (along with
+LDAP for identity service, [and Kerberos for authentication service]) which is
+the best suited option for joining ``MS Active Directory domains``, and doing
+``User Logon Management``.
+
+By no means we intend to cover every possible scenario/combination here. These
+steps are for a simple *get a (MIT) Kerberos Server up and running*.
+
+Please, note that *rpm packages might have slightly different names*, as well
+as the locations for the binaries and/or configuration files, depending on
+which Linux distro we are referring to.
+
+Finally, keep in mind that some Linux distros will have their own ``wizards``,
+which can perform the basic needed configuration:  ::
+
+    SUSE:
+        Kerberos server:
+            yast2 auth-server
+
+        Kerberos client:
+            pam/nss: yast2 ldapkrb
+            sssd: yast2 auth-client
+
+
+However, we are going through the ``manual configuration``.
+
+
+In order to get a new MIT KDC Server running:
+
+1. Install the KDC server by:
+
+    a. Install the needed packages:  ::
+
+        SUSE: zypper install krb5 krb5-server krb5-client
+            Additionally:
+                for development: krb5-devel
+                if using 'sssd': sssd-krb5 sssd-krb5-common
+
+        REDHAT: yum install krb5-server krb5-libs krb5-workstation
+            Additionally: 'Needs to be checked'
+
+
+    b. Edit the KDC Server configuration file:  ::
+
+        /var/lib/kerberos/krb5kdc/kdc.conf
+        [kdcdefaults]
+                kdc_ports = 750,88
+        [realms]
+                MYDOMAIN.COM = {
+                    acl_file = /var/lib/kerberos/krb5kdc/kadm5.acl
+                    admin_keytab = FILE:/var/lib/kerberos/krb5kdc/kadm5.keytab
+                    default_principal_flags = +postdateable +forwardable +renewable +proxiable
+                                                            +dup-skey -preauth -hwauth +service
+                                                            +tgt-based +allow-tickets -pwchange
+                                                            -pwservice
+                    dict_file = /var/lib/kerberos/krb5kdc/kadm5.dict
+                    key_stash_file = /var/lib/kerberos/krb5kdc/.k5.MYDOMAIN.COM
+                    kdc_ports = 750,88
+                    max_life = 0d 10h 0m 0s
+                    max_renewable_life = 7d 0h 0m 0s
+                }
+        ...
+
+
+    c. Edit the Kerberos Client configuration file:  ::
+
+        /etc/krb5.conf
+        [libdefaults]
+            dns_canonicalize_hostname = false
+            rdns = false
+            forwardable = true
+            dns_lookup_realm = true     //--> if using DNS/DNSMasq
+            dns_lookup_kdc = true       //--> if using DNS/DNSMasq
+            allow_weak_crypto = false
+            default_realm = MYDOMAIN.COM
+            default_ccache_name = KEYRING:persistent:%{uid}
+
+        [realms]
+            MYDOMAIN.COM = {
+                kdc = kerberos.mydomain.com
+                admin_server = kerberos.mydomain.com
+                ...
+            }
+        ...
+
+
+2. Create the Kerberos database:  ::
+
+    SUSE: kdb5_util create -s
+
+    REDHAT: kdb5_util create -s
+
+
+3. Enable and Start both 'KDC and KDC admin' servers:  ::
+
+    SUSE: systemctl enable/start krb5kdc
+          systemctl enable/start kadmind
+
+    REDHAT: systemctl enable/start krb5kdc
+            systemctl enable/start kadmin
+
+
+4. Create a Kerberos Administrator
+    Kerberos principals can be created either locally on the KDC server itself
+    or through the network, using an 'admin principal'. On the KDC server,
+    using ``kadmin.local``:
+
+    a. List the existing principals:  ::
+
+        kadmin.local:  listprincs
+        K/M@MYDOMAIN.COM
+        krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+        kadmin/admin@MYDOMAIN.COM
+        kadmin/changepw@MYDOMAIN.COM
+        kadmin/history@MYDOMAIN.COM
+        kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+        root/admin@MYDOMAIN.COM
+        ...
+
+
+    b. In case we don't have a built-in 'admin principal', we then create one
+    (whatever ``principal name``, we are using ``root``, once by default
+    ``kinit`` tries to authenticate using the same system login user name,
+    unless a ``principal`` is passed as an argument ``kinit principal``):  ::
+
+        # kadmin.local -q "addprinc root/admin"
+        Authenticating as principal root/admin@MYDOMAIN.COM with password.
+        WARNING: no policy specified for root/admin@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "root/admin@MYDOMAIN.COM":
+
+
+    c. Confirm the newly created 'admin principal' has the needed permissions
+       in the KDC ACL (if ACLs are changed, ``kadmind`` needs to be restarted):  ::
+
+        SUSE: /var/lib/kerberos/krb5kdc/kadm5.acl
+        REDHAT: /var/kerberos/krb5kdc/kadm5.acl
+
+        ###############################################################################
+        #Kerberos_principal      permissions     [target_principal]      [restrictions]
+        ###############################################################################
+        #
+        */admin@MYDOMAIN.COM     *
+
+
+    d. Create a simple 'user principal' (same steps as by *The 'Ceph side' of
+       the things*; 4a):  ::
+
+        kadmin.local:  addprinc johndoe
+        WARNING: no policy specified for johndoe@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "johndoe@MYDOMAIN.COM":
+        Re-enter password for principal "johndoe@MYDOMAIN.COM":
+        Principal "johndoe@MYDOMAIN.COM" created.
+
+
+    e. Confirm the newly created 'user principal' is able to authenticate (same
+       steps as by *The 'Ceph side' of the things*; 6):  ::
+
+        # kdestroy -A && kinit -f johndoe && klist -f
+        Password for johndoe@MYDOMAIN.COM:
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: johndoe@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        11/16/2018 13:11:16  11/16/2018 23:11:16  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+                renew until 11/17/2018 13:11:16, Flags: FRI
+        ...
+
+
+5. At this point, we should have a *simple (MIT) Kerberos Server up and running*:
+
+     a. Considering we will want to work with keytab files, for both 'user and
+        service' principals, refer to The *'Ceph side' of the things* starting
+        at step 4.
+
+     b. Make sure you are comfortable with following and their ``manpages``:  ::
+
+            krb5.conf       -> Krb client config file
+            kdc.conf        -> KDC server config file
+
+            krb5kdc         -> KDC server daemon
+            kadmind         -> KDC administration daemon
+
+            kadmin          -> Krb administration tool
+            kdb5_util       -> Krb low-level database administration tool
+
+            kinit           -> Obtain and cache Kerberos ticket-granting ticket tool
+            klist           -> List cached Kerberos tickets tool
+            kdestroy        -> Destroy Kerberos tickets tool
+
+
+6. Name Resolution
+    As mentioned earlier, Kerberos *relies heavily on name resolution*. Most of
+    the Kerberos issues are usually related to name resolution, since Kerberos
+    is *very picky* on both *systems names* and *host lookups*.
+
+    a. As described in *The 'Ceph side' of the things*; step 2a, DNS RRs
+       greatly improves service location and host/domain resolution, by using
+       ``(srv resources)`` and ``(txt record)`` respectively (as per
+       *Before We Start*; *DNS resolution*).  ::
+
+        /var/lib/named/master/mydomain.com
+        kerberos                IN A        192.168.10.21
+        kerberos-slave          IN A        192.168.10.22
+        _kerberos               IN TXT      "MYDOMAIN.COM"
+        _kerberos._udp          IN SRV      1 0 88 kerberos
+        _kerberos._tcp          IN SRV      1 0 88 kerberos
+        _kerberos._udp          IN SRV      20 0 88 kerberos-slave
+        _kerberos-master._udp   IN SRV      0 0 88 kerberos
+        _kerberos-adm._tcp      IN SRV      0 0 749 kerberos
+        _kpasswd._udp           IN SRV      0 0 464 kerberos
+        ...
+
+
+    b. For a small network or development environment, where a *DNS server is
+       not available*, we have the option to use ``DNSMasq``, an
+       ease-to-configure lightweight DNS server (along with some other
+       capabilities).
+
+       These records can be added to ``/etc/dnsmasq.conf`` (in addition to the
+       needed 'host records'):  ::
+
+        /etc/dnsmasq.conf
+        ...
+        txt-record=_kerberos.mydomain.com,"MYDOMAIN.COM"
+        srv-host=_kerberos._udp.mydomain.com,kerberos.mydomain.com,88,1
+        srv-host=_kerberos._udp.mydomain.com,kerberos-2.mydomain.com,88,20
+        srv-host=_kerberos-master._udp.mydomain.com,kerberos.mydomain.com,88,0
+        srv-host=_kerberos-adm._tcp.mydomain.com,kerberos.mydomain.com,749,0
+        srv-host=_kpasswd._udp.mydomain.com,kerberos.mydomain.com,464,0
+        srv-host=_kerberos._tcp.mydomain.com,kerberos.mydomain.com,88,1
+        ...
+
+
+    c. After 'b)' is all set, and ``dnsmasq`` service up and running, we can
+       test it using:  ::
+
+        # nslookup kerberos
+        Server:     192.168.10.1
+        Address:    192.168.10.1#53
+
+        Name:   kerberos.mydomain.com
+        Address: 192.168.10.21
+
+        # host -t SRV _kerberos._tcp.mydomain.com
+        _kerberos._tcp.mydomain.com has SRV record 1 0 88 kerberos.mydomain.com.
+
+        # host -t SRV {each srv-host record}
+        # host -t TXT _kerberos.mydomain.com
+        _kerberos.mydomain.com descriptive text "MYDOMAIN.COM"
+        ...
+
+
+    f. As long as ``name resolution`` is working properly, either ``dnsmasq``
+       or ``named``, Kerberos should be able to find the needed service
+       records.
diff --git a/doc/dev/cephadm/compliance-check.rst b/doc/dev/cephadm/compliance-check.rst
new file mode 100644
index 000000000..eea462445
--- /dev/null
+++ b/doc/dev/cephadm/compliance-check.rst
@@ -0,0 +1,121 @@
+================
+Compliance Check
+================
+
+The stability and reliability of a Ceph cluster is dependent not just upon the Ceph daemons, but
+also the OS and hardware that Ceph is installed on. This document is intended to promote a design 
+discussion for providing a "compliance" feature within mgr/cephadm, which would be responsible for
+identifying common platform-related issues that could impact Ceph stability and operation.
+
+The ultimate goal of these checks is to identify issues early and raise a healthcheck WARN
+event, to alert the Administrator to the issue.
+
+Prerequisites
+=============
+In order to effectively analyse the hosts that Ceph is deployed to, this feature requires a cache
+of host-related metadata. The metadata is already available from cephadm's HostFacts class and the
+``gather-facts`` cephadm command. For the purposes of this document, we will assume that this
+data is available within the mgr/cephadm "cache" structure.
+
+Some checks will require that the host status is also populated e.g. ONLINE, OFFLINE, MAINTENANCE
+
+Administrator Interaction
+=========================
+Not all users will require this feature, and must be able to 'opt out'. For this reason,
+mgr/cephadm must provide controls, such as the following;
+
+.. code-block::
+
+   ceph cephadm compliance enable | disable | status [--format json]
+   ceph cephadm compliance ls [--format json]
+   ceph cephadm compliance enable-check <name>
+   ceph cephadm compliance disable-check <name>
+   ceph cephadm compliance set-check-interval <int>
+   ceph cephadm compliance get-check-interval
+
+The status option would show the enabled/disabled state of the feature, along with the
+check-interval.
+
+The ``ls`` subcommand would show all checks in the following format;
+
+``check-name status description``
+
+Proposed Integration
+====================
+The compliance checks are not required to run all the time, but instead should run at discrete
+intervals. The interval would be configurable under via the :code:`set-check-interval`
+subcommand (default would be every 12 hours)
+
+
+mgr/cephadm currently executes an event driven (time based) serve loop to act on deploy/remove and
+reconcile activity. In order to execute the compliance checks, the compliance check code would be 
+called from this main serve loop - when the :code:`set-check-interval` is met.
+
+
+Proposed Checks
+===============
+All checks would push any errors to a list, so multiple issues can be escalated to the Admin at
+the same time. The list below provides a description of each check, with the text following the 
+name indicating a shortname version *(the shortname is the reference for command Interaction
+when enabling or disabling a check)*
+
+
+OS Consistency (OS)
+___________________
+* all hosts must use same vendor
+* all hosts must be on the same major release (this check would only be applicable to distributions that
+  offer a long-term-support strategy (RHEL, CentOS, SLES, Ubuntu etc)
+
+
+*src: gather-facts output*
+
+Linux Kernel Security Mode (LSM)
+________________________________
+* All hosts should have a consistent SELINUX/AppArmor configuration
+
+*src: gather-facts output*
+
+Services Check (SERVICES)
+_________________________
+Hosts that are in an ONLINE state should adhere to the following;
+
+* all daemons (systemd units) should be enabled
+* all daemons should be running (not dead)
+
+*src: list_daemons output*
+
+Support Status (SUPPORT)
+________________________
+If support status has been detected, it should be consistent across all hosts. At this point
+support status is available only for Red Hat machines.
+
+*src: gather-facts output*
+
+Network : MTU (MTU)
+________________________________
+All network interfaces on the same Ceph network (public/cluster) should have the same MTU
+
+*src: gather-facts output*
+
+Network : LinkSpeed (LINKSPEED)
+____________________________________________
+All network interfaces on the same Ceph network (public/cluster) should have the same Linkspeed
+
+*src: gather-facts output*
+
+Network : Consistency (INTERFACE)
+______________________________________________
+All hosts with OSDs should have consistent network configuration - eg. if some hosts do
+not separate cluster/public traffic but others do, that is an anomaly that would generate a
+compliance check warning.
+
+*src: gather-facts output*
+
+Notification Strategy
+=====================
+If any of the checks fail, mgr/cephadm would raise a WARN level alert
+
+Futures
+=======
+The checks highlighted here serve only as a starting point, and we should expect to expand
+on the checks over time.
diff --git a/doc/dev/cephadm/design/mockups/OSD_Creation_device_mode.svg b/doc/dev/cephadm/design/mockups/OSD_Creation_device_mode.svg
new file mode 100644
index 000000000..6c0f6933a
--- /dev/null
+++ b/doc/dev/cephadm/design/mockups/OSD_Creation_device_mode.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1272" height="1316"><g transform="translate(-240 -44)" lucid:page-tab-id="0_0"><path d="M0 0h1760v1360H0z" fill="#fff"/><path d="M555 207.8c0-4.4 3.58-8 8-8h889c4.42 0 8 3.6 8 8v130.32c0 4.42-3.58 8-8 8H563c-4.42 0-8-3.58-8-8zM301.25 206.7c0-4.43 3.58-8 8-8H542c4.42 0 8 3.57 8 8V337c0 4.42-3.58 8-8 8H309.25c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M453.15 77c0-4.42 3.58-8 8-8h836.75c4.4 0 8 3.58 8 8v25.16c0 4.4-3.6 8-8 8H461.15c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" stroke-width="2" fill="#fff" fill-opacity="0"/><use xlink:href="#a" transform="matrix(1,0,0,1,457.14655278534315,73) translate(246.2241975308642 25.782777777777778)"/><use xlink:href="#b" transform="matrix(1,0,0,1,457.14655278534315,73) translate(310.2501234567901 25.782777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,457.14655278534315,73) translate(419.2908641975308 25.782777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,457.14655278534315,73) translate(519.6637037037037 25.782777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,457.14655278534315,73) translate(596.7279012345679 25.782777777777778)"/><path d="M725.1 108.8h315.85M725.12 108.8h-1.02M1040.93 108.8h1.02" stroke="#a9afb8" stroke-width="2" fill="none"/><path d="M453.15 118.16c0-4.42 3.58-8 8-8h837.7c4.42 0 8 3.58 8 8V121c0 4.42-3.58 8-8 8h-837.7c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" stroke-width="2" fill="#fff" fill-opacity="0"/><use xlink:href="#f" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(263.9143209876543 14.917037037037037)"/><use xlink:href="#g" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(301.3052674897119 14.917037037037037)"/><use xlink:href="#h" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(350.4658641975309 14.917037037037037)"/><use xlink:href="#i" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(391.1710082304527 14.917037037037037)"/><use xlink:href="#j" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(458.17728395061727 14.917037037037037)"/><use xlink:href="#k" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(470.6267695473251 14.917037037037037)"/><use xlink:href="#l" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(529.2200823045268 14.917037037037037)"/><use xlink:href="#m" transform="matrix(1,0,0,1,457.1465527853436,114.15522395689881) translate(554.7139094650206 14.917037037037037)"/><path d="M879.52 68.5V17.38" stroke="#333" fill="none"/><path d="M880.02 69h-1v-.5h1z" fill="#333"/><path d="M879.52 2.62l4.64 14.26h-9.27z" stroke="#333" fill="#333"/><path d="M301.25 361.45c0-4.42 3.58-8 8-8H1412c4.42 0 8 3.58 8 8v169c0 4.42-3.58 8-8 8H309.25c-4.42 0-8-3.58-8-8z" fill="#fff"/><path d="M301.25 353.45H1420v36H301.25z" fill="#ccc"/><path d="M301.25 425.45H1420v36H301.25zM301.25 497.45H1420v36H301.25z" fill="#f0f0f0"/><path d="M481.06 353.45v185M625.8 353.45v185M760.4 353.45v185M859.62 353.45v185M905.05 353.45v185M1083.94 353.45v185M301.25 361.45c0-4.42 3.58-8 8-8H1412c4.42 0 8 3.58 8 8v169c0 4.42-3.58 8-8 8H309.25c-4.42 0-8-3.58-8-8z" stroke="#333" fill="none"/><use xlink:href="#n" transform="matrix(1,0,0,1,307.25,353.4522590030593) translate(36.574999999999996 21.6)"/><use xlink:href="#o" transform="matrix(1,0,0,1,487.0611274067389,353.4522590030593) translate(41.55 21.6)"/><use xlink:href="#p" transform="matrix(1,0,0,1,631.7851045427196,353.4522590030593) translate(38.9 21.6)"/><use xlink:href="#q" transform="matrix(1,0,0,1,766.4120600180506,353.4522590030593) translate(20.525 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,865.615297833935,353.4522590030593) translate(9.5 21.6)"/><use xlink:href="#s" transform="matrix(1,0,0,1,911.0518953068593,353.4522590030593) translate(36.35 21.6)"/><use xlink:href="#t" transform="matrix(1,0,0,1,1089.937462394705,353.4522590030593) translate(68.64999999999999 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,1089.937462394705,353.4522590030593) translate(168.45 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,307.25,389.4522590030593) translate(79 21.6)"/><use xlink:href="#w" transform="matrix(1,0,0,1,487.0611274067389,389.4522590030593) translate(44.5 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,631.7851045427196,389.4522590030593) translate(27.5 21.6)"/><use xlink:href="#y" transform="matrix(1,0,0,1,766.4120600180506,389.4522590030593) translate(29 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,865.615297833935,389.4522590030593) translate(9.5 21.6)"/><use xlink:href="#z" transform="matrix(1,0,0,1,911.0518953068593,389.4522590030593) translate(68.825 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1089.937462394705,389.4522590030593) translate(159.525 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,307.25,425.4522590030593) translate(79.5 21.6)"/><use xlink:href="#C" transform="matrix(1,0,0,1,487.0611274067389,425.4522590030593) translate(44.5 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,631.7851045427196,425.4522590030593) translate(27.5 21.6)"/><use xlink:href="#D" transform="matrix(1,0,0,1,766.4120600180506,425.4522590030593) translate(22.025 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,865.615297833935,425.4522590030593) translate(9.5 21.6)"/><use xlink:href="#z" transform="matrix(1,0,0,1,911.0518953068593,425.4522590030593) translate(68.825 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1089.937462394705,425.4522590030593) translate(159.525 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,307.25,461.4522590030593) translate(79 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,487.0611274067389,461.4522590030593) translate(59 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,631.7851045427196,461.4522590030593) translate(54 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,766.4120600180506,461.4522590030593) translate(36.5 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,865.615297833935,461.4522590030593) translate(9.5 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,307.25,497.4522590030593) translate(79 21.6)"/><use xlink:href="#E" transform="matrix(1,0,0,1,487.0611274067389,497.4522590030593) translate(29.5 21.6)"/><use xlink:href="#F" transform="matrix(1,0,0,1,631.7851045427196,497.4522590030593) translate(28 21.6)"/><use xlink:href="#G" transform="matrix(1,0,0,1,766.4120600180506,497.4522590030593) translate(29.5 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,865.615297833935,497.4522590030593) translate(9.5 21.6)"/><use xlink:href="#z" transform="matrix(1,0,0,1,911.0518953068593,497.4522590030593) translate(68.825 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1089.937462394705,497.4522590030593) translate(159.525 21.6)"/><path d="M1183.54 299.87c-6.23 0-11.3 5.05-11.3 11.3v15.05c0 6.23 5.07 11.3 11.3 11.3h93.67c6.24 0 11.3-5.07 11.3-11.3v-15.06c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#H" transform="matrix(1,0,0,1,1182.25,303.6320403898173) translate(12.075000000000003 19.1)"/><use xlink:href="#I" transform="matrix(1,0,0,1,1182.25,303.6320403898173) translate(67.025 19.1)"/><path d="M381.25 397.63c0-4.42 3.58-8 8-8h64c4.42 0 8 3.58 8 8v14.62c0 4.42-3.58 8-8 8h-64c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M381.25 394.94h20v20h-20z" stroke="#333" fill="#fff"/><path d="M385.25 402.94l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M381.25 428.25c0-4.42 3.58-8 8-8h64c4.42 0 8 3.58 8 8v14.62c0 4.42-3.58 8-8 8h-64c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M381.25 425.56h20v20h-20z" stroke="#333" fill="#fff"/><path d="M381.25 463c0-4.42 3.58-8 8-8h64c4.42 0 8 3.58 8 8v14.62c0 4.42-3.58 8-8 8h-64c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M381.25 460.3h20v20h-20z" stroke="#333" fill="#fff"/><path d="M381.25 504.05c0-4.42 3.58-8 8-8h64c4.42 0 8 3.58 8 8v14.62c0 4.42-3.58 8-8 8h-64c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M381.25 501.36h20v20h-20z" stroke="#333" fill="#fff"/><path d="M385.25 509.36l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M1305.8 299.87c-6.24 0-11.3 5.05-11.3 11.3v15.05c0 6.23 5.06 11.3 11.3 11.3h131.16c6.23 0 11.3-5.07 11.3-11.3v-15.06c0-6.24-5.07-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#J" transform="matrix(1,0,0,1,1304.5,303.6320403898173) translate(5.599999999999994 19.1)"/><use xlink:href="#K" transform="matrix(1,0,0,1,1304.5,303.6320403898173) translate(57.5 19.1)"/><path d="M312 261.7c0-4.43 3.58-8 8-8h212.75c4.42 0 8 3.57 8 8v24c0 4.4-3.58 8-8 8H320c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><path d="M502.75 263.7c0-4.43 3.58-8 8-8h20c4.42 0 8 3.57 8 8v20c0 4.4-3.58 8-8 8h-20c-4.42 0-8-3.6-8-8z" stroke="#333" fill="none"/><path d="M509.95 266.5h21.6l-10.8 14.4z" stroke="#333" fill="#333"/><use xlink:href="#H" transform="matrix(1,0,0,1,324,253.68805152498527) translate(0 23.6)"/><use xlink:href="#L" transform="matrix(1,0,0,1,324,253.68805152498527) translate(54.95 23.6)"/><use xlink:href="#M" transform="matrix(1,0,0,1,324,253.68805152498527) translate(101.85 23.6)"/><use xlink:href="#N" transform="matrix(1,0,0,1,324,253.68805152498527) translate(121.85 23.6)"/><path d="M312 307.87c0-4.42 3.58-8 8-8h212.75c4.42 0 8 3.58 8 8v21.64c0 4.43-3.58 8-8 8H320c-4.42 0-8-3.57-8-8z" stroke="#333" fill="#fff"/><path d="M512.52 299.87v37.64m0-18.8h28.23" stroke="#333" fill="none"/><path d="M518.17 313.04l8.46-9.4 8.47 9.4zm0 11.3l8.46 9.4 8.47-9.4z" fill="#333"/><use xlink:href="#O" transform="matrix(1,0,0,1,312,307.3960431736093) translate(50.575 15.1)"/><use xlink:href="#P" transform="matrix(1,0,0,1,312,307.3960431736093) translate(97.525 15.1)"/><use xlink:href="#Q" transform="matrix(1,0,0,1,312,307.3960431736093) translate(112.525 15.1)"/><path d="M580 262.74c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8H588c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><use xlink:href="#R" transform="matrix(1,0,0,1,590,258.5000779958426) translate(0 19.1)"/><use xlink:href="#S" transform="matrix(1,0,0,1,590,258.5000779958426) translate(39 19.1)"/><use xlink:href="#T" transform="matrix(1,0,0,1,590,258.5000779958426) translate(54.5 19.1)"/><use xlink:href="#U" transform="matrix(1,0,0,1,590,258.5000779958426) translate(116.2 19.1)"/><use xlink:href="#V" transform="matrix(1,0,0,1,590,258.5000779958426) translate(137.15 19.1)"/><use xlink:href="#U" transform="matrix(1,0,0,1,590,258.5000779958426) translate(243.3 19.1)"/><use xlink:href="#W" transform="matrix(1,0,0,1,590,258.5000779958426) translate(264.25 19.1)"/><path d="M580 213c0-4.42 3.58-8 8-8h556.5c4.42 0 8 3.58 8 8v33.8c0 4.42-3.58 8-8 8H588c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#X" transform="matrix(1,0,0,1,585,210) translate(0 24.4)"/><use xlink:href="#Y" transform="matrix(1,0,0,1,585,210) translate(59.85000000000001 24.4)"/><use xlink:href="#Z" transform="matrix(1,0,0,1,585,210) translate(119.70000000000002 24.4)"/><use xlink:href="#aa" transform="matrix(1,0,0,1,585,210) translate(134.70000000000002 24.4)"/><use xlink:href="#ab" transform="matrix(1,0,0,1,585,210) translate(196.60000000000002 24.4)"/><use xlink:href="#ac" transform="matrix(1,0,0,1,585,210) translate(286.45000000000005 24.4)"/><use xlink:href="#ad" transform="matrix(1,0,0,1,585,210) translate(308.35 24.4)"/><use xlink:href="#ae" transform="matrix(1,0,0,1,585,210) translate(364.3 24.4)"/><path d="M1183.54 254.74c-6.23 0-11.3 5.05-11.3 11.3v15.04c0 6.24 5.07 11.3 11.3 11.3h93.67c6.24 0 11.3-5.06 11.3-11.3v-15.05c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#af" transform="matrix(1,0,0,1,1182.25,258.5000779958426) translate(6.599999999999994 19.1)"/><use xlink:href="#ag" transform="matrix(1,0,0,1,1182.25,258.5000779958426) translate(56.55 19.1)"/><path d="M312 217.9c0-4.42 3.58-8 8-8h182.75c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8H320c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#ah" transform="matrix(1,0,0,1,317,214.89999999999998) translate(0 18.15)"/><use xlink:href="#ai" transform="matrix(1,0,0,1,317,214.89999999999998) translate(55.550000000000004 18.15)"/><path d="M1305.8 254.74c-6.24 0-11.3 5.05-11.3 11.3v15.04c0 6.24 5.06 11.3 11.3 11.3h131.66c6.23 0 11.3-5.06 11.3-11.3v-15.05c0-6.24-5.07-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#J" transform="matrix(1,0,0,1,1304.5,258.5000779958426) translate(24.625 19.1)"/><use xlink:href="#ag" transform="matrix(1,0,0,1,1304.5,258.5000779958426) translate(76.525 19.1)"/><path d="M580 307.87c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.43-3.58 8-8 8H588c-4.42 0-8-3.57-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aj" transform="matrix(1,0,0,1,585,304.8680376060253) translate(184.55 17.9)"/><use xlink:href="#ak" transform="matrix(1,0,0,1,585,304.8680376060253) translate(199.55 17.9)"/><use xlink:href="#al" transform="matrix(1,0,0,1,585,304.8680376060253) translate(265.5 17.9)"/><use xlink:href="#am" transform="matrix(1,0,0,1,585,304.8680376060253) translate(315.5 17.9)"/><use xlink:href="#an" transform="matrix(1,0,0,1,585,304.8680376060253) translate(334.45 17.9)"/><use xlink:href="#ao" transform="matrix(1,0,0,1,585,304.8680376060253) translate(349.45 17.9)"/><path d="M1425 361.45c0-4.42 3.58-8 8-8h24c4.42 0 8 3.58 8 8v169c0 4.42-3.58 8-8 8h-24c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M1427 363.45c0-4.42 3.58-8 8-8h20c4.42 0 8 3.58 8 8v20c0 4.42-3.58 8-8 8h-20c-4.42 0-8-3.58-8-8z" stroke="#333" fill="none"/><path d="M1434.2 380.65h21.6l-10.8-14.4z" stroke="#333" fill="#333"/><path d="M1427 401.45c0-4.42 3.58-8 8-8h20c4.42 0 8 3.58 8 8v38.5c0 4.42-3.58 8-8 8h-20c-4.42 0-8-3.58-8-8zM1429 418.7h32m-32 2h32m-32 2h32M1427 528.45c0 4.42 3.58 8 8 8h20c4.42 0 8-3.58 8-8v-20c0-4.42-3.58-8-8-8h-20c-4.42 0-8 3.58-8 8z" stroke="#333" fill="none"/><path d="M1434.2 511.25h21.6l-10.8 14.4z" stroke="#333" fill="#333"/><path d="M301.25 560.5c0-4.42 3.58-8 8-8H1452c4.42 0 8 3.58 8 8V1347c0 4.42-3.58 8-8 8H309.25c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M1249.48 1305c-6.54 0-11.83 5.3-11.83 11.83v15.78c0 6.55 5.3 11.84 11.83 11.84h187.44c6.53 0 11.83-5.3 11.83-11.83v-15.77c0-6.53-5.3-11.83-11.83-11.83z" stroke="#333" fill="#fff"/><use xlink:href="#ap" transform="matrix(1,0,0,1,1247.6465527853431,1308.9443079618625) translate(40.875 19.6)"/><use xlink:href="#aq" transform="matrix(1,0,0,1,1247.6465527853431,1308.9443079618625) translate(99.775 19.6)"/><path d="M348 1187.66c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H356c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#ar" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(0 17.9)"/><use xlink:href="#as" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(10.95 17.9)"/><use xlink:href="#ak" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(25.95 17.9)"/><use xlink:href="#at" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(91.9 17.9)"/><use xlink:href="#am" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(163.85000000000002 17.9)"/><use xlink:href="#as" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(182.8 17.9)"/><use xlink:href="#ao" transform="matrix(1,0,0,1,353.00000000000006,1184.6596159481924) translate(197.8 17.9)"/><path d="M348 1219.3c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H356c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#au" transform="matrix(1,0,0,1,353.00000000000006,1216.2996437861125) translate(0 17.9)"/><use xlink:href="#av" transform="matrix(1,0,0,1,353.00000000000006,1216.2996437861125) translate(10.95 17.9)"/><use xlink:href="#aw" transform="matrix(1,0,0,1,353.00000000000006,1216.2996437861125) translate(51.85000000000001 17.9)"/><use xlink:href="#ax" transform="matrix(1,0,0,1,353.00000000000006,1216.2996437861125) translate(122.80000000000001 17.9)"/><use xlink:href="#ay" transform="matrix(1,0,0,1,353.00000000000006,1216.2996437861125) translate(167.8 17.9)"/><path d="M348 1253.94c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H356c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#ar" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(0 17.9)"/><use xlink:href="#as" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(10.95 17.9)"/><use xlink:href="#az" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(25.95 17.9)"/><use xlink:href="#aA" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(82.3 17.9)"/><use xlink:href="#aB" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(112.1 17.9)"/><use xlink:href="#aC" transform="matrix(1,0,0,1,353.00000000000006,1250.9396716240321) translate(137.1 17.9)"/><path d="M320 1158.22c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8H328c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aD" transform="matrix(1,0,0,1,325.00000000000006,1155.2165363295667) translate(0 17.9)"/></g><path d="M312 570.43c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H320c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aE" transform="matrix(1,0,0,1,317.00000000000006,567.4299860810156) translate(0 17.9)"/><use xlink:href="#aF" transform="matrix(1,0,0,1,317.00000000000006,567.4299860810156) translate(66.85000000000001 17.9)"/><use xlink:href="#aG" transform="matrix(1,0,0,1,317.00000000000006,567.4299860810156) translate(123.75000000000001 17.9)"/><use xlink:href="#aH" transform="matrix(1,0,0,1,317.00000000000006,567.4299860810156) translate(173.65 17.9)"/></g><path d="M680 649.25c0-4.42 3.58-8 8-8h29c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8h-29c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><g><use xlink:href="#aI" transform="matrix(1,0,0,1,690,645.0140027837674) translate(7.5 19.1)"/></g><path d="M320 887c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H328c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aJ" transform="matrix(1,0,0,1,325.00000000000006,884) translate(0 17.9)"/><use xlink:href="#aK" transform="matrix(1,0,0,1,325.00000000000006,884) translate(84.85000000000001 17.9)"/></g><path d="M533.13 932c0-4.42 3.58-8 8-8h187.3c4.4 0 8 3.58 8 8v16.36c0 4.42-3.6 8-8 8h-187.3c-4.42 0-8-3.58-8-8zM744.52 932.1h32.36m-32.36 8.08h32.36m-32.36 8.1h32.36m-8.1-16.2v24.28m-8.08-24.27v24.26m-8.1-24.27v24.26M744.53 924h32.36v32.36H744.5z" stroke="#333" fill="#fff"/><g><use xlink:href="#aL" transform="matrix(1,0,0,1,553.45500347974,927.235997216208) translate(1.5250000000000057 16.6)"/><use xlink:href="#aM" transform="matrix(1,0,0,1,553.45500347974,927.235997216208) translate(51.525000000000006 16.6)"/><use xlink:href="#aN" transform="matrix(1,0,0,1,553.45500347974,927.235997216208) translate(81.47500000000001 16.6)"/></g><path d="M348 925.82c0-4.42 3.58-8 8-8h169.13c4.4 0 8 3.58 8 8V951c0 4.42-3.6 8-8 8H356c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aO" transform="matrix(1,0,0,1,353.00000000000006,922.8200139189844) translate(0 20.4)"/><use xlink:href="#aP" transform="matrix(1,0,0,1,353.00000000000006,922.8200139189844) translate(42.95 20.4)"/><use xlink:href="#aQ" transform="matrix(1,0,0,1,353.00000000000006,922.8200139189844) translate(86.9 20.4)"/><use xlink:href="#aR" transform="matrix(1,0,0,1,353.00000000000006,922.8200139189844) translate(155.8 20.4)"/></g><path d="M320 1025.23c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8H328c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aE" transform="matrix(1,0,0,1,325.00000000000006,1022.2265432890589) translate(0 17.9)"/><use xlink:href="#aS" transform="matrix(1,0,0,1,325.00000000000006,1022.2265432890589) translate(66.85000000000001 17.9)"/><use xlink:href="#aT" transform="matrix(1,0,0,1,325.00000000000006,1022.2265432890589) translate(103.75000000000001 17.9)"/><use xlink:href="#aU" transform="matrix(1,0,0,1,325.00000000000006,1022.2265432890589) translate(156.60000000000002 17.9)"/></g><path d="M355 1052.24c0-4.42 3.58-8 8-8h449c4.42 0 8 3.58 8 8v37c0 4.4-3.58 8-8 8H363c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aV" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(0 25.15)"/><use xlink:href="#aW" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(53.900000000000006 25.15)"/><use xlink:href="#aX" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(121.80000000000001 25.15)"/><use xlink:href="#aY" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(208.70000000000002 25.15)"/><use xlink:href="#aZ" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(226.65 25.15)"/><use xlink:href="#ba" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(263.6 25.15)"/><use xlink:href="#bb" transform="matrix(1,0,0,1,360,1049.2365502485516) translate(358.6 25.15)"/></g><path d="M665 1059.9c0-4.4 3.58-8 8-8h29c4.42 0 8 3.6 8 8v21.65c0 4.42-3.58 8-8 8h-29c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><g><use xlink:href="#bc" transform="matrix(1,0,0,1,675,1055.675535633637) translate(2.5 19.1)"/></g><path d="M355 1105.23c0-4.42 3.58-8 8-8h449c4.42 0 8 3.58 8 8v37c0 4.4-3.58 8-8 8H363c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#bd" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(0 25.15)"/><use xlink:href="#be" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(41.95 25.15)"/><use xlink:href="#bf" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(75.95 25.15)"/><use xlink:href="#aY" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(129.95 25.15)"/><use xlink:href="#aZ" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(147.89999999999998 25.15)"/><use xlink:href="#ba" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(184.84999999999997 25.15)"/><use xlink:href="#bb" transform="matrix(1,0,0,1,360,1102.2265432890592) translate(279.84999999999997 25.15)"/></g><path d="M590.63 1111.44c0-4.42 3.58-8 8-8h29c4.4 0 8 3.58 8 8v21.64c0 4.42-3.6 8-8 8h-29c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><g><use xlink:href="#bg" transform="matrix(1,0,0,1,600.625,1107.2070824024177) translate(2.5 19.1)"/></g><path d="M357.13 974.34c0-4.42 3.58-8 8-8h480c4.4 0 8 3.58 8 8v24c0 4.42-3.6 8-8 8h-480c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M357.13 972.34h20v20h-20z" stroke="#333" fill="#fff"/><path d="M361.13 980.34l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><g><use xlink:href="#bh" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(5.17500000000004 19.6)"/><use xlink:href="#bi" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(25.17500000000004 19.6)"/><use xlink:href="#bj" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(68.17500000000004 19.6)"/><use xlink:href="#bk" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(88.17500000000004 19.6)"/><use xlink:href="#aP" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(157.12500000000003 19.6)"/><use xlink:href="#bl" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(201.07500000000002 19.6)"/><use xlink:href="#bm" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(278.975 19.6)"/><use xlink:href="#bn" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(371.875 19.6)"/><use xlink:href="#bo" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(397.82499999999993 19.6)"/><use xlink:href="#bp" transform="matrix(1,0,0,1,389.125,966.342494780393) translate(436.82499999999993 19.6)"/></g><path d="M348 1288c0-4.42 3.58-8 8-8h570.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H356c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#ar" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(0 17.9)"/><use xlink:href="#bq" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(10.95 17.9)"/><use xlink:href="#br" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(95.85000000000001 17.9)"/><use xlink:href="#aY" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(165.75 17.9)"/><use xlink:href="#an" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(183.7 17.9)"/><use xlink:href="#bs" transform="matrix(1,0,0,1,353.00000000000006,1285) translate(198.7 17.9)"/></g><path d="M356.13 609.07c0-4.42 3.58-8 8-8h1099c4.4 0 8 3.58 8 8v104c0 4.42-3.6 8-8 8h-1099c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M376.13 621.07c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><path d="M371.13 621.07c0 2.76-2.24 5-5 5-2.77 0-5-2.24-5-5s2.23-5 5-5c2.76 0 5 2.24 5 5z" stroke="#333" fill="#333"/><path d="M376.13 661.07c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10zM376.13 701.07c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><g><use xlink:href="#ap" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(0 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(58.900000000000006 23.6)"/><use xlink:href="#bt" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(77.85000000000001 23.6)"/><use xlink:href="#bu" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(140.65000000000003 23.6)"/><use xlink:href="#bv" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(151.60000000000002 23.6)"/><use xlink:href="#bw" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(185.60000000000002 23.6)"/><use xlink:href="#bx" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(215.60000000000002 23.6)"/><use xlink:href="#by" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(249.60000000000002 23.6)"/><use xlink:href="#bz" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(298.25 23.6)"/><use xlink:href="#bA" transform="matrix(1,0,0,1,388.125,601.0700139189355) translate(404.1 23.6)"/></g><g><use xlink:href="#bB" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(0 23.6)"/><use xlink:href="#bC" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(36.95 23.6)"/><use xlink:href="#aQ" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(123.85000000000001 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(192.75 23.6)"/><use xlink:href="#bD" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(211.7 23.6)"/><use xlink:href="#bE" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(271.65 23.6)"/><use xlink:href="#bF" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(341.65 23.6)"/><use xlink:href="#bG" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(394.59999999999997 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(414.59999999999997 23.6)"/><use xlink:href="#bI" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(463.54999999999995 23.6)"/><use xlink:href="#bu" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(502.44999999999993 23.6)"/><use xlink:href="#bJ" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(513.4 23.6)"/><use xlink:href="#bw" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(554.3 23.6)"/><use xlink:href="#bK" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(584.3 23.6)"/><use xlink:href="#bL" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(653.25 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(711.15 23.6)"/><use xlink:href="#bz" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(730.1 23.6)"/><use xlink:href="#bA" transform="matrix(1,0,0,1,388.125,641.0700139189355) translate(835.95 23.6)"/></g><g><use xlink:href="#ap" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(0 23.6)"/><use xlink:href="#bM" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(58.900000000000006 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(81.80000000000001 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(138.15 23.6)"/><use xlink:href="#bN" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(157.1 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(187.1 23.6)"/><use xlink:href="#bO" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(236.05 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(275.05 23.6)"/><use xlink:href="#bP" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(294 23.6)"/><use xlink:href="#bQ" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(366.8 23.6)"/><use xlink:href="#bR" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(410.8 23.6)"/><use xlink:href="#bS" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(477.70000000000005 23.6)"/><use xlink:href="#bT" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(550.6500000000001 23.6)"/><use xlink:href="#bU" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(587.5500000000001 23.6)"/><use xlink:href="#bV" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(633.5000000000001 23.6)"/><use xlink:href="#bu" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(681.5000000000001 23.6)"/><use xlink:href="#bv" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(692.4500000000002 23.6)"/><use xlink:href="#bw" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(726.4500000000002 23.6)"/><use xlink:href="#bx" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(756.4500000000002 23.6)"/><use xlink:href="#by" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(790.4500000000002 23.6)"/><use xlink:href="#bz" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(839.1000000000001 23.6)"/><use xlink:href="#bA" transform="matrix(1,0,0,1,388.125,681.0700139189355) translate(944.9500000000002 23.6)"/></g><path d="M505 890c0-4.42 3.58-8 8-8h107.33c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8H513c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M505 888h20v20h-20z" stroke="#333" fill="#fff"/><path d="M509 896l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M320.12 732.36c0-4.42 3.6-8 8-8h570.25c4.42 0 8 3.58 8 8V754c0 4.42-3.58 8-8 8H328.12c-4.4 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aE" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(0 17.9)"/><use xlink:href="#aF" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(66.85000000000001 17.9)"/><use xlink:href="#bW" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(123.75000000000001 17.9)"/><use xlink:href="#bX" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(210.5 17.9)"/><use xlink:href="#bY" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(250.45 17.9)"/><use xlink:href="#bZ" transform="matrix(1,0,0,1,325.12499999999983,729.3599721620801) translate(320.29999999999995 17.9)"/></g><path d="M356.13 770c0-4.42 3.58-8 8-8h1099c4.4 0 8 3.58 8 8v104c0 4.42-3.6 8-8 8h-1099c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M376.13 782c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10zM376.13 822c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10zM376.13 862c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><path d="M371.13 862c0 2.76-2.24 5-5 5-2.77 0-5-2.24-5-5s2.23-5 5-5c2.76 0 5 2.24 5 5z" stroke="#333" fill="#333"/><g><use xlink:href="#ca" transform="matrix(1,0,0,1,388.125,762) translate(0 23.6)"/><use xlink:href="#cb" transform="matrix(1,0,0,1,388.125,762) translate(66 23.6)"/><use xlink:href="#bM" transform="matrix(1,0,0,1,388.125,762) translate(114.95 23.6)"/><use xlink:href="#bF" transform="matrix(1,0,0,1,388.125,762) translate(137.85 23.6)"/><use xlink:href="#cc" transform="matrix(1,0,0,1,388.125,762) translate(190.8 23.6)"/><use xlink:href="#cd" transform="matrix(1,0,0,1,388.125,762) translate(245.8 23.6)"/><use xlink:href="#bG" transform="matrix(1,0,0,1,388.125,762) translate(276.75 23.6)"/><use xlink:href="#ce" transform="matrix(1,0,0,1,388.125,762) translate(296.75 23.6)"/><use xlink:href="#cf" transform="matrix(1,0,0,1,388.125,762) translate(353.65 23.6)"/><use xlink:href="#cg" transform="matrix(1,0,0,1,388.125,762) translate(373.65 23.6)"/><use xlink:href="#bM" transform="matrix(1,0,0,1,388.125,762) translate(421.65 23.6)"/><use xlink:href="#bF" transform="matrix(1,0,0,1,388.125,762) translate(444.54999999999995 23.6)"/><use xlink:href="#cd" transform="matrix(1,0,0,1,388.125,762) translate(497.49999999999994 23.6)"/><use xlink:href="#ch" transform="matrix(1,0,0,1,388.125,762) translate(528.4499999999999 23.6)"/><use xlink:href="#ci" transform="matrix(1,0,0,1,388.125,762) translate(598.4 23.6)"/><use xlink:href="#cd" transform="matrix(1,0,0,1,388.125,762) translate(643.35 23.6)"/><use xlink:href="#cj" transform="matrix(1,0,0,1,388.125,762) translate(674.3000000000001 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,762) translate(767.1500000000001 23.6)"/><use xlink:href="#ck" transform="matrix(1,0,0,1,388.125,762) translate(786.1000000000001 23.6)"/><use xlink:href="#M" transform="matrix(1,0,0,1,388.125,762) translate(822.0500000000002 23.6)"/><use xlink:href="#cl" transform="matrix(1,0,0,1,388.125,762) translate(842.0500000000002 23.6)"/><use xlink:href="#bN" transform="matrix(1,0,0,1,388.125,762) translate(885.9500000000002 23.6)"/><use xlink:href="#cm" transform="matrix(1,0,0,1,388.125,762) translate(915.9500000000002 23.6)"/></g><g><use xlink:href="#ca" transform="matrix(1,0,0,1,388.125,802) translate(0 23.6)"/><use xlink:href="#cn" transform="matrix(1,0,0,1,388.125,802) translate(66 23.6)"/><use xlink:href="#co" transform="matrix(1,0,0,1,388.125,802) translate(90 23.6)"/><use xlink:href="#bM" transform="matrix(1,0,0,1,388.125,802) translate(134 23.6)"/><use xlink:href="#bF" transform="matrix(1,0,0,1,388.125,802) translate(156.9 23.6)"/><use xlink:href="#cp" transform="matrix(1,0,0,1,388.125,802) translate(209.85000000000002 23.6)"/><use xlink:href="#cq" transform="matrix(1,0,0,1,388.125,802) translate(234.85000000000002 23.6)"/><use xlink:href="#cr" transform="matrix(1,0,0,1,388.125,802) translate(249.85000000000002 23.6)"/><use xlink:href="#bO" transform="matrix(1,0,0,1,388.125,802) translate(297.8 23.6)"/><use xlink:href="#cd" transform="matrix(1,0,0,1,388.125,802) translate(336.8 23.6)"/><use xlink:href="#cj" transform="matrix(1,0,0,1,388.125,802) translate(367.75 23.6)"/><use xlink:href="#bG" transform="matrix(1,0,0,1,388.125,802) translate(460.6 23.6)"/><use xlink:href="#bN" transform="matrix(1,0,0,1,388.125,802) translate(480.6 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,388.125,802) translate(510.6 23.6)"/><use xlink:href="#cs" transform="matrix(1,0,0,1,388.125,802) translate(559.5500000000001 23.6)"/></g><g><use xlink:href="#ct" transform="matrix(1,0,0,1,388.125,842) translate(0 23.6)"/><use xlink:href="#cu" transform="matrix(1,0,0,1,388.125,842) translate(98.85000000000001 23.6)"/><use xlink:href="#bN" transform="matrix(1,0,0,1,388.125,842) translate(145.75 23.6)"/><use xlink:href="#bF" transform="matrix(1,0,0,1,388.125,842) translate(175.75 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,388.125,842) translate(228.7 23.6)"/><use xlink:href="#cw" transform="matrix(1,0,0,1,388.125,842) translate(272.65 23.6)"/><use xlink:href="#cx" transform="matrix(1,0,0,1,388.125,842) translate(336.54999999999995 23.6)"/><use xlink:href="#cy" transform="matrix(1,0,0,1,388.125,842) translate(383.49999999999994 23.6)"/><use xlink:href="#cz" transform="matrix(1,0,0,1,388.125,842) translate(448.4 23.6)"/><use xlink:href="#am" transform="matrix(1,0,0,1,388.125,842) translate(502.34999999999997 23.6)"/><use xlink:href="#cA" transform="matrix(1,0,0,1,388.125,842) translate(521.3 23.6)"/><use xlink:href="#cB" transform="matrix(1,0,0,1,388.125,842) translate(559.25 23.6)"/></g><defs><path fill="#a9afb8" d="M140-251c80 0 125 45 125 126S219 4 139 4C58 4 15-44 15-125s44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="cC"/><path fill="#a9afb8" d="M169-182c-1-43-94-46-97-3 18 66 151 10 154 114 3 95-165 93-204 36-6-8-10-19-12-30l50-8c3 46 112 56 116 5-17-69-150-10-154-114-4-87 153-88 188-35 5 8 8 18 10 28" id="cD"/><path fill="#a9afb8" d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="cE"/><g id="a"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#cC"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,20.39506172839506,0)" xlink:href="#cD"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,37.876543209876544,0)" xlink:href="#cE"/></g><path fill="#a9afb8" d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="cF"/><path fill="#a9afb8" d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="cG"/><path fill="#a9afb8" d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="cH"/><path fill="#a9afb8" d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="cI"/><path fill="#a9afb8" d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="cJ"/><path fill="#a9afb8" d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="cK"/><path fill="#a9afb8" d="M110-194c64 0 96 36 96 99 0 64-35 99-97 99-61 0-95-36-95-99 0-62 34-99 96-99zm-1 164c35 0 45-28 45-65 0-40-10-65-43-65-34 0-45 26-45 65 0 36 10 65 43 65" id="cL"/><path fill="#a9afb8" d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="cM"/><g id="b"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#cF"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,14.5679012345679,0)" xlink:href="#cG"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,24.76543209876543,0)" xlink:href="#cH"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,39.333333333333336,0)" xlink:href="#cI"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,53.90123456790123,0)" xlink:href="#cJ"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,62.56913580246913,0)" xlink:href="#cK"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,69.85308641975308,0)" xlink:href="#cL"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,85.80493827160494,0)" xlink:href="#cM"/></g><path fill="#a9afb8" d="M67-93c0 74 22 123 53 168H70C40 30 18-18 18-93s22-123 52-168h50c-32 44-53 94-53 168" id="cN"/><path fill="#a9afb8" d="M128 0H69L1-190h53L99-40l48-150h52" id="cO"/><g id="c"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#cN"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,8.667901234567902,0)" xlink:href="#cE"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,27.53333333333333,0)" xlink:href="#cH"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,42.10123456790124,0)" xlink:href="#cO"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,56.66913580246913,0)" xlink:href="#cK"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,63.95308641975308,0)" xlink:href="#cF"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,78.52098765432098,0)" xlink:href="#cH"/></g><path fill="#a9afb8" d="M220-157c-53 9-28 100-34 157h-49v-107c1-27-5-49-29-50C55-147 81-57 75 0H25l-1-190h47c2 12-1 28 3 38 10-53 101-56 108 0 13-22 24-43 59-42 82 1 51 116 57 194h-49v-107c-1-25-5-48-29-50" id="cP"/><path fill="#a9afb8" d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="cQ"/><g id="d"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#cP"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,23.308641975308642,0)" xlink:href="#cL"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,39.260493827160495,0)" xlink:href="#cQ"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,55.212345679012344,0)" xlink:href="#cH"/></g><path fill="#a9afb8" d="M102-93c0 74-22 123-52 168H0C30 29 54-18 53-93c0-74-22-123-53-168h50c30 45 52 94 52 168" id="cR"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#cR" id="e"/><path fill="#a9afb8" d="M153-248C145-148 188 4 80 4 36 3 13-21 6-62l32-5c4 25 16 42 43 43 27 0 39-20 39-49v-147H72v-28h81" id="cS"/><path fill="#a9afb8" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="cT"/><path fill="#a9afb8" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="cU"/><path fill="#a9afb8" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="cV"/><g id="f"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#cS"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,7.648148148148148,0)" xlink:href="#cT"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.14609053497942,0)" xlink:href="#cU"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,24.644032921810698,0)" xlink:href="#cV"/></g><path fill="#a9afb8" d="M240 0l2-218c-23 76-54 145-80 218h-23L58-218 59 0H30v-248h44l77 211c21-75 51-140 76-211h43V0h-30" id="cW"/><path fill="#a9afb8" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="cX"/><path fill="#a9afb8" d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="cY"/><path fill="#a9afb8" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="cZ"/><path fill="#a9afb8" d="M24 0v-261h32V0H24" id="da"/><g id="g"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#cW"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,12.704423868312757,0)" xlink:href="#cX"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.06111111111111,0)" xlink:href="#cY"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,24.559053497942386,0)" xlink:href="#cT"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,33.05699588477366,0)" xlink:href="#cZ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,41.55493827160494,0)" xlink:href="#da"/></g><path fill="#a9afb8" d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="db"/><path fill="#a9afb8" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="dc"/><path fill="#a9afb8" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="dd"/><g id="h"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#db"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,11.897119341563785,0)" xlink:href="#da"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,15.25380658436214,0)" xlink:href="#dc"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,27.958230452674897,0)" xlink:href="#dd"/></g><path fill="#a9afb8" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="de"/><path fill="#a9afb8" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="df"/><path fill="#a9afb8" d="M9 0v-24l116-142H16v-24h144v24L44-24h123V0H9" id="dg"/><g id="i"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#cW"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,12.704423868312757,0)" xlink:href="#cU"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,21.20236625514403,0)" xlink:href="#de"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,26.25864197530864,0)" xlink:href="#df"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,30.50761316872428,0)" xlink:href="#cX"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,33.864300411522635,0)" xlink:href="#cV"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,42.36224279835391,0)" xlink:href="#cZ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,50.86018518518518,0)" xlink:href="#dg"/></g><path fill="#a9afb8" d="M32 76v-337h29V76H32" id="dh"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dh" id="j"/><path fill="#a9afb8" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="di"/><path fill="#a9afb8" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="dj"/><g id="k"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#db"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,11.897119341563785,0)" xlink:href="#di"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,19.545267489711932,0)" xlink:href="#df"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,23.79423868312757,0)" xlink:href="#dd"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,32.29218106995884,0)" xlink:href="#dj"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,40.79012345679012,0)" xlink:href="#cZ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,49.288065843621396,0)" xlink:href="#de"/></g><path fill="#a9afb8" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="dk"/><path fill="#a9afb8" d="M155-56V0h-30v-56H8v-25l114-167h33v167h35v25h-35zm-30-156c-27 46-58 90-88 131h88v-131" id="dl"/><path fill="#a9afb8" d="M68-38c1 34 0 65-14 84H32c9-13 17-26 17-46H33v-38h35" id="dm"/><g id="l"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dk"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,8.497942386831275,0)" xlink:href="#dl"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.99588477366255,0)" xlink:href="#dm"/></g><path fill="#a9afb8" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="dn"/><path fill="#a9afb8" d="M101-251c68 0 85 55 85 127S166 4 100 4C33 4 14-52 14-124c0-73 17-127 87-127zm-1 229c47 0 54-49 54-102s-4-102-53-102c-51 0-55 48-55 102 0 53 5 102 54 102" id="do"/><g id="m"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dn"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,8.497942386831275,0)" xlink:href="#do"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.99588477366255,0)" xlink:href="#dn"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,25.493827160493822,0)" xlink:href="#do"/></g><path fill="#333" d="M185-189c-5-48-123-54-124 2 14 75 158 14 163 119 3 78-121 87-175 55-17-10-28-26-33-46l33-7c5 56 141 63 141-1 0-78-155-14-162-118-5-82 145-84 179-34 5 7 8 16 11 25" id="dp"/><path fill="#333" d="M30 0v-248h187v28H63v79h144v27H63v87h162V0H30" id="dq"/><path fill="#333" d="M30 0v-248h33v221h125V0H30" id="dr"/><path fill="#333" d="M212-179c-10-28-35-45-73-45-59 0-87 40-87 99 0 60 29 101 89 101 43 0 62-24 78-52l27 14C228-24 195 4 139 4 59 4 22-46 18-125c-6-104 99-153 187-111 19 9 31 26 39 46" id="ds"/><path fill="#333" d="M127-220V0H93v-220H8v-28h204v28h-85" id="dt"/><path fill="#333" d="M30-248c118-7 216 8 213 122C240-48 200 0 122 0H30v-248zM63-27c89 8 146-16 146-99s-60-101-146-95v194" id="du"/><g id="n"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#dr"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,46,0)" xlink:href="#ds"/><use transform="matrix(0.05,0,0,0.05,58.95,0)" xlink:href="#dt"/><use transform="matrix(0.05,0,0,0.05,69.9,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,81.9,0)" xlink:href="#du"/></g><path fill="#333" d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="dv"/><path fill="#333" d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="dw"/><g id="o"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dv"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,26.950000000000003,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#dt"/></g><path fill="#333" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="dx"/><path fill="#333" d="M205 0l-28-72H64L36 0H1l101-248h38L239 0h-34zm-38-99l-47-123c-12 45-31 82-46 123h93" id="dy"/><g id="p"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dx"/><use transform="matrix(0.05,0,0,0.05,10.65,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,21.3,0)" xlink:href="#dt"/><use transform="matrix(0.05,0,0,0.05,32.25,0)" xlink:href="#dv"/></g><path fill="#333" d="M137-103V0h-34v-103L8-248h37l75 118 75-118h37" id="dz"/><g id="q"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dt"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#dz"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#dx"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#dq"/></g><path fill="#333" d="M33 0v-38h34V0H33" id="dA"/><g id="r"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dA"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dA"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dA"/></g><path fill="#333" d="M137 0h-34L2-248h35l83 218 83-218h36" id="dB"/><path fill="#333" d="M33 0v-248h34V0H33" id="dC"/><path fill="#333" d="M160-131c35 5 61 23 61 61C221 17 115-2 30 0v-248c76 3 177-17 177 60 0 33-19 50-47 57zm-97-11c50-1 110 9 110-42 0-47-63-36-110-37v79zm0 115c55-2 124 14 124-45 0-56-70-42-124-44v89" id="dD"/><g id="s"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,10.65,0)" xlink:href="#dB"/><use transform="matrix(0.05,0,0,0.05,21.3,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,33.3,0)" xlink:href="#dC"/><use transform="matrix(0.05,0,0,0.05,38.3,0)" xlink:href="#dr"/><use transform="matrix(0.05,0,0,0.05,48.3,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,60.29999999999999,0)" xlink:href="#dD"/><use transform="matrix(0.05,0,0,0.05,72.3,0)" xlink:href="#dr"/><use transform="matrix(0.05,0,0,0.05,82.3,0)" xlink:href="#dq"/></g><path fill="#333" d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="dE"/><path fill="#333" d="M153-248C145-148 188 4 80 4 36 3 13-21 6-62l32-5c4 25 16 42 43 43 27 0 39-20 39-49v-147H72v-28h81" id="dF"/><g id="t"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dE"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,24.950000000000003,0)" xlink:href="#dF"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,45.95,0)" xlink:href="#ds"/><use transform="matrix(0.05,0,0,0.05,58.900000000000006,0)" xlink:href="#dt"/><use transform="matrix(0.05,0,0,0.05,69.85000000000001,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,81.85000000000001,0)" xlink:href="#du"/></g><path fill="#333" d="M190 0L58-211 59 0H30v-248h39L202-35l-2-213h31V0h-41" id="dG"/><g id="u"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dE"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,24.950000000000003,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,36.95,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,48.95,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,62.95,0)" xlink:href="#dG"/><use transform="matrix(0.05,0,0,0.05,75.9,0)" xlink:href="#dp"/></g><path fill="#333" d="M-5 72V49h209v23H-5" id="dH"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dH" id="v"/><path fill="#333" d="M106-169C34-169 62-67 57 0H25v-261h32l-1 103c12-21 28-36 61-36 89 0 53 116 60 194h-32v-121c2-32-8-49-39-48" id="dI"/><path fill="#333" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="dJ"/><path fill="#333" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="dK"/><path fill="#333" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="dL"/><path fill="#333" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="dM"/><g id="w"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dM"/></g><path fill="#333" d="M0 4l72-265h28L28 4H0" id="dN"/><path fill="#333" d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="dO"/><path fill="#333" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="dP"/><path fill="#333" d="M108 0H70L1-190h34L89-25l56-165h34" id="dQ"/><path fill="#333" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="dR"/><g id="x"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#dR"/></g><g id="y"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dO"/></g><g id="z"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dz"/><use transform="matrix(0.05,0,0,0.05,10.35,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20.35,0)" xlink:href="#dK"/></g><path fill="#333" d="M16-82v-28h88v28H16" id="dS"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dS" id="A"/><path fill="#333" d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="dT"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dT" id="B"/><path fill="#333" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="dU"/><g id="C"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dU"/></g><path fill="#333" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="dV"/><path fill="#333" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="dW"/><g id="D"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#dP"/></g><path fill="#333" d="M155-56V0h-30v-56H8v-25l114-167h33v167h35v25h-35zm-30-156c-27 46-58 90-88 131h88v-131" id="dX"/><path fill="#333" d="M101-251c68 0 85 55 85 127S166 4 100 4C33 4 14-52 14-124c0-73 17-127 87-127zm-1 229c47 0 54-49 54-102s-4-102-53-102c-51 0-55 48-55 102 0 53 5 102 54 102" id="dY"/><g id="E"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dM"/><use transform="matrix(0.05,0,0,0.05,44,0)" xlink:href="#dX"/><use transform="matrix(0.05,0,0,0.05,54,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,64,0)" xlink:href="#dY"/></g><g id="F"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#dT"/></g><g id="G"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dO"/></g><path fill="#333" d="M24 0v-261h32V0H24" id="dZ"/><path fill="#333" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="ea"/><g id="H"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,44.95,0)" xlink:href="#dL"/></g><path fill="#333" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="eb"/><g id="I"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#dZ"/></g><g id="J"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ds"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,36.900000000000006,0)" xlink:href="#dV"/></g><path fill="#333" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="ec"/><g id="K"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,50.900000000000006,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,60.900000000000006,0)" xlink:href="#dV"/></g><path fill="#333" d="M101-234c-31-9-42 10-38 44h38v23H63V0H32v-167H5v-23h27c-7-52 17-82 69-68v24" id="ed"/><g id="L"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,18.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,22.9,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,32.9,0)" xlink:href="#dK"/></g><g id="M"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dJ"/></g><path fill="#333" d="M206 0h-36l-40-164L89 0H53L-1-190h32L70-26l43-164h34l41 164 42-164h31" id="ee"/><g id="N"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#ee"/></g><g id="O"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#ee"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dX" id="P"/><path fill="#333" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="ef"/><g id="Q"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#dK"/></g><g id="R"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/></g><path fill="#333" d="M18-150v-26h174v26H18zm0 90v-26h174v26H18" id="eg"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg" id="S"/><path fill="#333" d="M109-170H84l-4-78h32zm-65 0H19l-4-78h33" id="eh"/><g id="T"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,6.3500000000000005,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,16.35,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,26.35,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,35.35,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,40.35,0)" xlink:href="#dM"/><use transform="matrix(0.05,0,0,0.05,50.35,0)" xlink:href="#eh"/></g><g id="U"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ef"/></g><path fill="#333" d="M179-190L93 31C79 59 56 82 12 73V49c39 6 53-20 64-50L1-190h34L92-34l54-156h33" id="ei"/><path fill="#333" d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="ej"/><g id="V"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ei"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,44.5,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,50.85,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,60.85,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,69.85,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,84.8,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,94.8,0)" xlink:href="#eh"/></g><g id="W"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ei"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,44.5,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,50.85,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,59.85,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,69.85,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,79.85,0)" xlink:href="#eh"/></g><path fill="#333" d="M76-208v77h127v40H76V0H24v-248h183v40H76" id="ek"/><path fill="#333" d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="el"/><path fill="#333" d="M25 0v-261h50V0H25" id="em"/><path fill="#333" d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="en"/><path fill="#333" d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="eo"/><path fill="#333" d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="ep"/><path fill="#333" d="M35-132v-50h50v50H35zM35 0v-49h50V0H35" id="eq"/><g id="X"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ek"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,15.950000000000001,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,36.900000000000006,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#eq"/></g><path fill="#333" d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="er"/><path fill="#333" d="M121-226c-27-7-43 5-38 36h38v33H83V0H34v-157H6v-33h28c-9-59 32-81 87-68v32" id="es"/><path fill="#333" d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="et"/><g id="Y"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#er"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#es"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,33.900000000000006,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,44.85000000000001,0)" xlink:href="#eo"/></g><path fill="#333" d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="eu"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eu" id="Z"/><path fill="#333" d="M110-194c64 0 96 36 96 99 0 64-35 99-97 99-61 0-95-36-95-99 0-62 34-99 96-99zm-1 164c35 0 45-28 45-65 0-40-10-65-43-65-34 0-45 26-45 65 0 36 10 65 43 65" id="ev"/><path fill="#333" d="M195-6C206 82 75 100 31 46c-4-6-6-13-8-21l49-6c3 16 16 24 34 25 40 0 42-37 40-79-11 22-30 35-61 35-53 0-70-43-70-97 0-56 18-96 73-97 30 0 46 14 59 34l2-30h47zm-90-29c32 0 41-27 41-63 0-35-9-62-40-62-32 0-39 29-40 63 0 36 9 62 39 62" id="ew"/><path fill="#333" d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="ex"/><g id="aa"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,15.950000000000001,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,51.900000000000006,0)" xlink:href="#em"/></g><path fill="#333" d="M144 0l-44-69L55 0H2l70-98-66-92h53l41 62 40-62h54l-67 91 71 99h-54" id="ey"/><path fill="#333" d="M135-194c53 0 70 44 70 98 0 56-19 98-73 100-31 1-45-17-59-34 3 33 2 69 2 105H25l-1-265h48c2 10 0 23 3 31 11-24 29-35 60-35zM114-30c33 0 39-31 40-66 0-38-9-64-40-64-56 0-55 130 0 130" id="ez"/><path fill="#333" d="M137-138c1-29-70-34-71-4 15 46 118 7 119 86 1 83-164 76-172 9l43-7c4 19 20 25 44 25 33 8 57-30 24-41C81-84 22-81 20-136c-2-80 154-74 161-7" id="eA"/><g id="ab"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,37.95,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,57.95,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,62.95,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,73.9,0)" xlink:href="#et"/></g><g id="ac"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ev"/></g><g id="ad"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,45,0)" xlink:href="#en"/></g><path fill="#333" d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="eB"/><path fill="#333" d="M128 0H69L1-190h53L99-40l48-150h52" id="eC"/><g id="ae"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,45.95,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,55.95,0)" xlink:href="#eA"/></g><g id="af"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dy"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ei"/></g><g id="ag"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,12.899999999999999,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,17.9,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,27.899999999999995,0)" xlink:href="#ef"/></g><path fill="#333" d="M136-208V0H84v-208H4v-40h212v40h-80" id="eD"/><path fill="#333" d="M135-194c52 0 70 43 70 98 0 56-19 99-73 100-30 1-46-15-58-35L72 0H24l1-261h50v104c11-23 29-37 60-37zM114-30c31 0 40-27 40-66 0-37-7-63-39-63s-41 28-41 65c0 36 8 64 40 64" id="eE"/><g id="ah"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,9.600000000000001,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,19.6,0)" xlink:href="#eE"/><use transform="matrix(0.05,0,0,0.05,30.55,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,35.550000000000004,0)" xlink:href="#eo"/></g><path fill="#333" d="M85 4C-2 5 27-109 22-190h50c7 57-23 150 33 157 60-5 35-97 40-157h50l1 190h-47c-2-12 1-28-3-38-12 25-28 42-61 42" id="eF"/><g id="ai"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,31.900000000000006,0)" xlink:href="#es"/><use transform="matrix(0.05,0,0,0.05,37.85000000000001,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,42.85000000000001,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,53.80000000000001,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,64.75000000000001,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,71.75000000000001,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,81.75000000000001,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,87.70000000000002,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,92.70000000000002,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,103.65000000000003,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,114.60000000000002,0)" xlink:href="#eq"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dX" id="aj"/><g id="ak"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,51.95,0)" xlink:href="#dK"/></g><path fill="#333" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="eG"/><g id="al"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#dO"/></g><g id="am"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#dV"/></g><path fill="#333" d="M126-127c33 6 58 20 58 59 0 88-139 92-164 29-3-8-5-16-6-25l32-3c6 27 21 44 54 44 32 0 52-15 52-46 0-38-36-46-79-43v-28c39 1 72-4 72-42 0-27-17-43-46-43-28 0-47 15-49 41l-32-3c6-42 35-63 81-64 48-1 79 21 79 65 0 36-21 52-52 59" id="eH"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eH" id="an"/><g id="ao"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dK"/></g><g id="ap"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ds"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#dP"/></g><path fill="#333" d="M47-170H22l-4-78h33" id="eI"/><g id="aq"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#du"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,42.35,0)" xlink:href="#dK"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dS" id="ar"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dU" id="as"/><g id="at"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,56.95,0)" xlink:href="#dO"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dS" id="au"/><g id="av"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dE"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#ee"/></g><g id="aw"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,51.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,56.95,0)" xlink:href="#ei"/></g><path fill="#333" d="M54-142c48-35 137-8 131 61C196 18 31 33 14-55l32-4c7 23 22 37 52 37 35-1 51-22 54-58 4-55-73-65-99-34H22l8-134h141v27H59" id="eJ"/><g id="ax"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dM"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#dY"/></g><path fill="#333" d="M143 4C61 4 22-44 18-125c-5-107 100-154 193-111 17 8 29 25 37 43l-32 9c-13-25-37-40-76-40-61 0-88 39-88 99 0 61 29 100 91 101 35 0 62-11 79-27v-45h-74v-28h105v86C228-13 192 4 143 4" id="eK"/><g id="ay"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dD"/></g><g id="az"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#du"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,42.35,0)" xlink:href="#dK"/></g><g id="aA"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,20.85,0)" xlink:href="#dZ"/></g><g id="aB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/></g><g id="aC"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,49.95,0)" xlink:href="#dO"/></g><path fill="#333" d="M169-182c-1-43-94-46-97-3 18 66 151 10 154 114 3 95-165 93-204 36-6-8-10-19-12-30l50-8c3 46 112 56 116 5-17-69-150-10-154-114-4-87 153-88 188-35 5 8 8 18 10 28" id="eL"/><path fill="#333" d="M220-157c-53 9-28 100-34 157h-49v-107c1-27-5-49-29-50C55-147 81-57 75 0H25l-1-190h47c2 12-1 28 3 38 10-53 101-56 108 0 13-22 24-43 59-42 82 1 51 116 57 194h-49v-107c-1-25-5-48-29-50" id="eM"/><path fill="#333" d="M123 10C108 53 80 86 19 72V37c35 8 53-11 59-39L3-190h52l48 148c12-52 28-100 44-148h51" id="eN"/><g id="aD"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,54.95,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,64.95,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,71.95,0)" xlink:href="#eN"/><use transform="matrix(0.05,0,0,0.05,81.95,0)" xlink:href="#eq"/></g><path fill="#333" d="M67-125c0 53 21 87 73 88 37 1 54-22 65-47l45 17C233-25 199 4 140 4 58 4 20-42 15-125 8-235 124-281 211-232c18 10 29 29 36 50l-46 12c-8-25-30-41-62-41-52 0-71 34-72 86" id="eO"/><g id="aE"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,17.950000000000003,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,28.900000000000006,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,44.85000000000001,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#ep"/></g><g id="aF"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#eN"/></g><path fill="#333" d="M67-93c0 74 22 123 53 168H70C40 30 18-18 18-93s22-123 52-168h50c-32 44-53 94-53 168" id="eP"/><path fill="#333" d="M140-251c80 0 125 45 125 126S219 4 139 4C58 4 15-44 15-125s44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="eQ"/><g id="aG"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eP"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,19.95,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#er"/></g><path fill="#333" d="M102-93c0 74-22 123-52 168H0C30 29 54-18 53-93c0-74-22-123-53-168h50c30 45 52 94 52 168" id="eR"/><g id="aH"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,37,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,42.95,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,58.900000000000006,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,69.85000000000001,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,75.80000000000001,0)" xlink:href="#eq"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dU" id="aI"/><path fill="#333" d="M114-157C55-157 80-60 75 0H25v-261h50l-1 109c12-26 28-41 61-42 86-1 58 113 63 194h-50c-7-57 23-157-34-157" id="eS"/><g id="aJ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eS"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,42.95,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,64.85000000000001,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,69.85000000000001,0)" xlink:href="#eo"/></g><g id="aK"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,48.900000000000006,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,59.85,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,70.80000000000001,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,81.75000000000001,0)" xlink:href="#eq"/></g><path fill="#333" d="M33-154v-36h34v36H33zM33 0v-36h34V0H33" id="eT"/><g id="aL"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eH"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#dY"/></g><g id="aM"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dW"/></g><path fill="#333" d="M99-251c64 0 84 50 84 122C183-37 130 33 47-8c-14-7-20-23-25-40l30-5c6 39 69 39 84 7 9-19 16-44 16-74-10 22-31 35-62 35-49 0-73-33-73-83 0-54 28-83 82-83zm-1 141c31-1 51-18 51-49 0-36-14-67-51-67-34 0-49 23-49 58 0 34 15 58 49 58" id="eU"/><g id="aN"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dU"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#dU"/><use transform="matrix(0.05,0,0,0.05,50,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,60,0)" xlink:href="#dU"/><use transform="matrix(0.05,0,0,0.05,70,0)" xlink:href="#dY"/></g><g id="aO"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#dL"/></g><g id="aP"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#du"/></g><g id="aQ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#dV"/></g><g id="aR"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dL"/></g><g id="aS"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#eB"/></g><g id="aT"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eS"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,37.85,0)" xlink:href="#eA"/></g><g id="aU"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,41.9,0)" xlink:href="#eq"/></g><g id="aV"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dx"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#ea"/></g><path fill="#333" d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="eV"/><g id="aW"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,37.95,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eV"/></g><g id="aX"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,52.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,56.900000000000006,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,66.9,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,76.9,0)" xlink:href="#dL"/></g><g id="aY"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#dK"/></g><g id="aZ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#dK"/></g><g id="ba"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dV"/></g><path fill="#333" d="M252-156c43 0 55 33 55 80 0 46-13 78-56 78-42 0-55-33-55-78 0-48 12-80 56-80zM93 0H65l162-248h28zM13-171c-1-47 13-79 56-79s55 33 55 79-13 79-55 79c-43 0-56-33-56-79zM251-17c28 0 30-29 30-59 0-31-1-60-29-60-29 0-31 29-31 60 0 29 2 59 30 59zM69-112c27 0 28-30 29-59 0-31-1-60-29-60-29 0-30 30-30 60s2 59 30 59" id="eW"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eW" id="bb"/><path fill="#333" d="M64 0c3-98 48-159 88-221H18v-27h164v26C143-157 98-101 97 0H64" id="eX"/><g id="bc"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dY"/></g><g id="bd"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dv"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#dL"/></g><g id="be"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eG"/></g><path fill="#333" d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="eY"/><g id="bf"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#dP"/></g><path fill="#333" d="M134-131c28 9 52 24 51 62-1 50-34 73-85 73S17-19 16-69c0-36 21-54 49-61-75-25-45-126 34-121 46 3 78 18 79 63 0 33-17 51-44 57zm-34-11c31 1 46-15 46-44 0-28-17-43-47-42-29 0-46 13-45 42 1 28 16 44 46 44zm1 122c35 0 51-18 51-52 0-30-18-46-53-46-33 0-51 17-51 47 0 34 19 51 53 51" id="eZ"/><g id="bg"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dY"/></g><g id="bh"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dC"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dV"/></g><g id="bi"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,28,0)" xlink:href="#dP"/></g><g id="bj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ed"/></g><g id="bk"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,43.95,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,53.95,0)" xlink:href="#eY"/></g><g id="bl"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,63.900000000000006,0)" xlink:href="#dK"/></g><g id="bm"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,43.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,53.95,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,63.95,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,73.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,77.9,0)" xlink:href="#dP"/></g><g id="bn"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ef"/></g><g id="bo"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dT"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/></g><g id="bp"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ei"/></g><g id="bq"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dq"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,21,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,29.950000000000003,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,44.9,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,59.900000000000006,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,69.9,0)" xlink:href="#dO"/></g><g id="br"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,44.900000000000006,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#dV"/></g><g id="bs"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,14.950000000000001,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#dK"/></g><g id="bt"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,43.85000000000001,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,53.85,0)" xlink:href="#dZ"/></g><path fill="#333" d="M87 75C49 33 22-17 22-94c0-76 28-126 65-167h31c-38 41-64 92-64 168S80 34 118 75H87" id="fa"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fa" id="bu"/><g id="bv"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#dL"/></g><g id="bw"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dL"/></g><g id="bx"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dV"/></g><g id="by"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,14.65,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,19.65,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,29.649999999999995,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,38.65,0)" xlink:href="#dL"/></g><g id="bz"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,46.900000000000006,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,61.85,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,71.85000000000001,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,81.85000000000001,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,90.85000000000001,0)" xlink:href="#dP"/></g><path fill="#333" d="M33-261c38 41 65 92 65 168S71 34 33 75H2C39 34 66-17 66-93S39-220 2-261h31" id="fb"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb" id="bA"/><path fill="#333" d="M232-93c-1 65-40 97-104 97C67 4 28-28 28-90v-158h33c8 89-33 224 67 224 102 0 64-133 71-224h33v155" id="fc"/><g id="bB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fc"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,21.950000000000003,0)" xlink:href="#dP"/></g><path fill="#333" d="M145-31C134-9 116 4 85 4 32 4 16-35 15-94c0-59 17-99 70-100 32-1 48 14 60 33 0-11-1-24 2-32h30l-1 268h-32zM93-21c41 0 51-33 51-76s-8-73-50-73c-40 0-46 35-46 75s5 74 45 74" id="fd"/><g id="bC"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#fd"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,49,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,59,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,64,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,67.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,77.95,0)" xlink:href="#dZ"/></g><g id="bD"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,45.95,0)" xlink:href="#dK"/></g><g id="bE"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ed"/></g><g id="bF"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#du"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#dK"/></g><g id="bG"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dL"/></g><g id="bH"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#dP"/></g><g id="bI"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,23.9,0)" xlink:href="#dP"/></g><g id="bJ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,12.949999999999998,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#ee"/></g><g id="bK"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,44.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,54.95,0)" xlink:href="#dK"/></g><g id="bL"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,47.900000000000006,0)" xlink:href="#dL"/></g><g id="bM"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#dZ"/></g><g id="bN"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dP"/></g><g id="bO"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/></g><path fill="#333" d="M68-38c1 34 0 65-14 84H32c9-13 17-26 17-46H33v-38h35" id="fe"/><g id="bP"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,43.85000000000001,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,53.85,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,57.80000000000001,0)" xlink:href="#fe"/></g><g id="bQ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dP"/></g><g id="bR"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,18.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,22.9,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,41.9,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,51.900000000000006,0)" xlink:href="#dO"/></g><g id="bS"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,37.95,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,57.95,0)" xlink:href="#dP"/></g><g id="bT"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#dI"/></g><g id="bU"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#ef"/></g><g id="bV"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#dK"/></g><path fill="#333" d="M24 0v-248h195v40H76v63h132v40H76v65h150V0H24" id="ff"/><g id="bW"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eP"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,17.95,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,38.9,0)" xlink:href="#eE"/><use transform="matrix(0.05,0,0,0.05,49.85,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,59.85,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,70.8,0)" xlink:href="#ew"/></g><path fill="#333" d="M231 0h-52l-39-155L100 0H48L-1-190h46L77-45c9-52 24-97 36-145h53l37 145 32-145h46" id="fg"/><g id="bX"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fg"/></g><g id="bY"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,26.899999999999995,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,33.9,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,43.9,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#eo"/></g><g id="bZ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,50.95,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,55.95,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,61.900000000000006,0)" xlink:href="#eN"/><use transform="matrix(0.05,0,0,0.05,71.9,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,77.85000000000001,0)" xlink:href="#eq"/></g><g id="ca"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dx"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,41,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,51,0)" xlink:href="#dO"/></g><g id="cb"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dp"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#du"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eT"/></g><g id="cc"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#dO"/></g><g id="cd"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dP"/></g><g id="ce"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,36.900000000000006,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,46.900000000000006,0)" xlink:href="#dL"/></g><g id="cf"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dY"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dA"/></g><g id="cg"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dw"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,33,0)" xlink:href="#dP"/></g><g id="ch"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,49.95,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,59.95,0)" xlink:href="#dA"/></g><g id="ci"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dt"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#ei"/></g><g id="cj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,38.9,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,42.85,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,52.85,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,62.85,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,67.85,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,77.85,0)" xlink:href="#dO"/></g><g id="ck"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,20.95,0)" xlink:href="#dV"/></g><g id="cl"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,19.9,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#dP"/></g><g id="cm"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,49.900000000000006,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,59.900000000000006,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,68.9,0)" xlink:href="#dP"/></g><g id="cn"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ei"/></g><g id="co"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eT"/></g><g id="cp"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dV"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb" id="cq"/><g id="cr"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#dQ"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#dV"/></g><g id="cs"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,23.9,0)" xlink:href="#dP"/></g><g id="ct"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dC"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,19.950000000000003,0)" xlink:href="#dW"/><use transform="matrix(0.05,0,0,0.05,34.900000000000006,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,44.900000000000006,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,58.85,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,68.85000000000001,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,73.85000000000001,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,83.85000000000001,0)" xlink:href="#eT"/></g><g id="cu"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dD"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,17.95,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,21.9,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#eY"/></g><g id="cv"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dN"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,28.950000000000003,0)" xlink:href="#dV"/></g><g id="cw"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,19.95,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,29.950000000000003,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,33.9,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,43.9,0)" xlink:href="#dI"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#dL"/></g><g id="cx"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ee"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#ei"/></g><g id="cy"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#dO"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#ed"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#eb"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,50.95,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#dL"/></g><g id="cz"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#ec"/><use transform="matrix(0.05,0,0,0.05,28.950000000000003,0)" xlink:href="#dJ"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#dV"/></g><g id="cA"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#dV"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ee"/></g><g id="cB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ea"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#dZ"/><use transform="matrix(0.05,0,0,0.05,12.949999999999998,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#dL"/><use transform="matrix(0.05,0,0,0.05,36.95,0)" xlink:href="#dP"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#ef"/><use transform="matrix(0.05,0,0,0.05,52.900000000000006,0)" xlink:href="#dK"/><use transform="matrix(0.05,0,0,0.05,61.900000000000006,0)" xlink:href="#fb"/></g></defs></g></svg>
\ No newline at end of file
diff --git a/doc/dev/cephadm/design/mockups/OSD_Creation_host_mode.svg b/doc/dev/cephadm/design/mockups/OSD_Creation_host_mode.svg
new file mode 100644
index 000000000..b15b3e942
--- /dev/null
+++ b/doc/dev/cephadm/design/mockups/OSD_Creation_host_mode.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1323.2" height="1870.4"><g transform="translate(0 0)" lucid:page-tab-id="0_0"><path d="M0 0h1323.2v1870.4H0z" fill="#fff"/><path d="M323.75 229.3c0-4.4 3.58-8 8-8h889c4.42 0 8 3.6 8 8v130.32c0 4.42-3.58 8-8 8h-889c-4.42 0-8-3.58-8-8zM70 228.2c0-4.43 3.58-8 8-8h232.75c4.42 0 8 3.57 8 8v130.3c0 4.42-3.58 8-8 8H78c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M221.9 98.5c0-4.42 3.58-8 8-8h836.75c4.4 0 8 3.58 8 8v25.16c0 4.4-3.6 8-8 8H229.9c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" stroke-width="2" fill="#fff" fill-opacity="0"/><use xlink:href="#a" transform="matrix(1,0,0,1,225.89655278534315,94.5) translate(259.40814814814814 25.782777777777778)"/><use xlink:href="#b" transform="matrix(1,0,0,1,225.89655278534315,94.5) translate(323.4340740740741 25.782777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,225.89655278534315,94.5) translate(432.4748148148148 25.782777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,225.89655278534315,94.5) translate(506.47975308641975 25.782777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,225.89655278534315,94.5) translate(583.5439506172839 25.782777777777778)"/><path d="M493.85 130.3H809.7M493.87 130.3h-1.02M809.68 130.3h1.02" stroke="#a9afb8" stroke-width="2" fill="none"/><path d="M221.9 139.66c0-4.42 3.58-8 8-8h837.7c4.42 0 8 3.58 8 8v2.84c0 4.42-3.58 8-8 8H229.9c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" stroke-width="2" fill="#fff" fill-opacity="0"/><use xlink:href="#f" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(263.9143209876543 14.917037037037037)"/><use xlink:href="#g" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(301.3052674897119 14.917037037037037)"/><use xlink:href="#h" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(350.4658641975309 14.917037037037037)"/><use xlink:href="#i" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(391.1710082304527 14.917037037037037)"/><use xlink:href="#j" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(458.17728395061727 14.917037037037037)"/><use xlink:href="#k" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(470.6267695473251 14.917037037037037)"/><use xlink:href="#l" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(529.2200823045268 14.917037037037037)"/><use xlink:href="#m" transform="matrix(1,0,0,1,225.8965527853436,135.6552239568988) translate(554.7139094650206 14.917037037037037)"/><path d="M648.27 90V38.88" stroke="#333" fill="none"/><path d="M648.77 90.5h-1V90h1z" fill="#333"/><path d="M648.27 24.12l4.64 14.26h-9.26z" stroke="#333" fill="#333"/><path d="M70 382.95c0-4.42 3.58-8 8-8h1102.75c4.42 0 8 3.58 8 8v279.93c0 4.42-3.58 8-8 8H78c-4.42 0-8-3.58-8-8z" fill="#fff"/><path d="M70 374.95h1118.75v36H70z" fill="#ccc"/><path d="M70 446.95h1118.75v36H70zM70 518.95h1118.75v36H70zM70 590.95h1118.75v36H70zM70 662.95h1118.75v7.93H70z" fill="#f0f0f0"/><path d="M413.6 374.95v295.93M597.14 374.95v295.93M732.4 374.95v295.93M906.54 374.95v295.93M1025.85 374.95v295.93M70 382.95c0-4.42 3.58-8 8-8h1102.75c4.42 0 8 3.58 8 8v279.93c0 4.42-3.58 8-8 8H78c-4.42 0-8-3.58-8-8z" stroke="#333" fill="none"/><use xlink:href="#n" transform="matrix(1,0,0,1,76,374.9522590030592) translate(97.125 21.6)"/><use xlink:href="#o" transform="matrix(1,0,0,1,76,374.9522590030592) translate(196.97500000000002 21.6)"/><use xlink:href="#p" transform="matrix(1,0,0,1,419.58657198523383,374.9522590030592) translate(63.4 21.6)"/><use xlink:href="#q" transform="matrix(1,0,0,1,603.1386638638228,374.9522590030592) translate(38.525 21.6)"/><use xlink:href="#r" transform="matrix(1,0,0,1,738.3936115668581,374.9522590030592) translate(49.55 21.6)"/><use xlink:href="#s" transform="matrix(1,0,0,1,912.5386587366694,374.9522590030592) translate(45.025 21.6)"/><use xlink:href="#t" transform="matrix(1,0,0,1,1031.8475184577524,374.9522590030592) translate(55.525 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,410.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,410.9522590030592) translate(211 21.6)"/><use xlink:href="#w" transform="matrix(1,0,0,1,419.58657198523383,410.9522590030592) translate(52 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,603.1386638638228,410.9522590030592) translate(47 21.6)"/><use xlink:href="#y" transform="matrix(1,0,0,1,738.3936115668581,410.9522590030592) translate(60.5 21.6)"/><use xlink:href="#z" transform="matrix(1,0,0,1,912.5386587366694,410.9522590030592) translate(34 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1031.8475184577524,410.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,410.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,446.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,446.9522590030592) translate(211 21.6)"/><use xlink:href="#C" transform="matrix(1,0,0,1,419.58657198523383,446.9522590030592) translate(52.5 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,603.1386638638228,446.9522590030592) translate(47 21.6)"/><use xlink:href="#y" transform="matrix(1,0,0,1,738.3936115668581,446.9522590030592) translate(60.5 21.6)"/><use xlink:href="#D" transform="matrix(1,0,0,1,912.5386587366694,446.9522590030592) translate(34 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1031.8475184577524,446.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,446.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,482.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,482.9522590030592) translate(211 21.6)"/><use xlink:href="#E" transform="matrix(1,0,0,1,419.58657198523383,482.9522590030592) translate(52 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,603.1386638638228,482.9522590030592) translate(47 21.6)"/><use xlink:href="#y" transform="matrix(1,0,0,1,738.3936115668581,482.9522590030592) translate(60.5 21.6)"/><use xlink:href="#F" transform="matrix(1,0,0,1,912.5386587366694,482.9522590030592) translate(34 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1031.8475184577524,482.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,482.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,518.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,518.9522590030592) translate(211 21.6)"/><use xlink:href="#G" transform="matrix(1,0,0,1,419.58657198523383,518.9522590030592) translate(52 21.6)"/><use xlink:href="#x" transform="matrix(1,0,0,1,603.1386638638228,518.9522590030592) translate(47 21.6)"/><use xlink:href="#y" transform="matrix(1,0,0,1,738.3936115668581,518.9522590030592) translate(60.5 21.6)"/><use xlink:href="#H" transform="matrix(1,0,0,1,912.5386587366694,518.9522590030592) translate(34 21.6)"/><use xlink:href="#A" transform="matrix(1,0,0,1,1031.8475184577524,518.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,518.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,554.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,554.9522590030592) translate(211 21.6)"/><use xlink:href="#I" transform="matrix(1,0,0,1,419.58657198523383,554.9522590030592) translate(54.5 21.6)"/><use xlink:href="#J" transform="matrix(1,0,0,1,603.1386638638228,554.9522590030592) translate(47.5 21.6)"/><use xlink:href="#K" transform="matrix(1,0,0,1,738.3936115668581,554.9522590030592) translate(62 21.6)"/><use xlink:href="#L" transform="matrix(1,0,0,1,912.5386587366694,554.9522590030592) translate(34 21.6)"/><use xlink:href="#M" transform="matrix(1,0,0,1,1031.8475184577524,554.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,554.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,590.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,590.9522590030592) translate(211 21.6)"/><use xlink:href="#N" transform="matrix(1,0,0,1,419.58657198523383,590.9522590030592) translate(52 21.6)"/><use xlink:href="#J" transform="matrix(1,0,0,1,603.1386638638228,590.9522590030592) translate(47.5 21.6)"/><use xlink:href="#K" transform="matrix(1,0,0,1,738.3936115668581,590.9522590030592) translate(62 21.6)"/><use xlink:href="#O" transform="matrix(1,0,0,1,912.5386587366694,590.9522590030592) translate(34 21.6)"/><use xlink:href="#M" transform="matrix(1,0,0,1,1031.8475184577524,590.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,590.9522590030592) translate(81 21.6)"/><use xlink:href="#u" transform="matrix(1,0,0,1,76,626.9522590030592) translate(116 21.6)"/><use xlink:href="#v" transform="matrix(1,0,0,1,76,626.9522590030592) translate(211 21.6)"/><use xlink:href="#P" transform="matrix(1,0,0,1,419.58657198523383,626.9522590030592) translate(52 21.6)"/><use xlink:href="#Q" transform="matrix(1,0,0,1,603.1386638638228,626.9522590030592) translate(40.025 21.6)"/><use xlink:href="#R" transform="matrix(1,0,0,1,738.3936115668581,626.9522590030592) translate(60.025 21.6)"/><use xlink:href="#S" transform="matrix(1,0,0,1,912.5386587366694,626.9522590030592) translate(34 21.6)"/><use xlink:href="#T" transform="matrix(1,0,0,1,1031.8475184577524,626.9522590030592) translate(46 21.6)"/><use xlink:href="#B" transform="matrix(1,0,0,1,1031.8475184577524,626.9522590030592) translate(81 21.6)"/><path d="M80.75 283.2c0-4.43 3.58-8 8-8H301.5c4.42 0 8 3.57 8 8v24c0 4.4-3.58 8-8 8H88.75c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><path d="M271.5 285.2c0-4.43 3.58-8 8-8h20c4.42 0 8 3.57 8 8v20c0 4.4-3.58 8-8 8h-20c-4.42 0-8-3.6-8-8z" stroke="#333" fill="none"/><path d="M278.7 288h21.6l-10.8 14.4z" stroke="#333" fill="#333"/><use xlink:href="#U" transform="matrix(1,0,0,1,92.75000000000003,275.18805152498527) translate(0 23.6)"/><use xlink:href="#V" transform="matrix(1,0,0,1,92.75000000000003,275.18805152498527) translate(54.95 23.6)"/><use xlink:href="#W" transform="matrix(1,0,0,1,92.75000000000003,275.18805152498527) translate(101.85 23.6)"/><use xlink:href="#X" transform="matrix(1,0,0,1,92.75000000000003,275.18805152498527) translate(121.85 23.6)"/><path d="M80.75 329.37c0-4.42 3.58-8 8-8H301.5c4.42 0 8 3.58 8 8V351c0 4.43-3.58 8-8 8H88.75c-4.42 0-8-3.57-8-8z" stroke="#333" fill="#fff"/><path d="M281.27 321.37V359m0-18.8h28.23" stroke="#333" fill="none"/><path d="M286.92 334.54l8.46-9.4 8.47 9.4zm0 11.3l8.46 9.4 8.47-9.4z" fill="#333"/><use xlink:href="#Y" transform="matrix(1,0,0,1,80.75,328.8960431736093) translate(50.575 15.1)"/><use xlink:href="#Z" transform="matrix(1,0,0,1,80.75,328.8960431736093) translate(97.525 15.1)"/><use xlink:href="#aa" transform="matrix(1,0,0,1,80.75,328.8960431736093) translate(112.525 15.1)"/><path d="M348.75 284.24c0-4.42 3.58-8 8-8H927c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8H356.75c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><use xlink:href="#ab" transform="matrix(1,0,0,1,358.74999999999994,280.0000779958426) translate(0 19.1)"/><use xlink:href="#ac" transform="matrix(1,0,0,1,358.74999999999994,280.0000779958426) translate(39 19.1)"/><use xlink:href="#ad" transform="matrix(1,0,0,1,358.74999999999994,280.0000779958426) translate(54.5 19.1)"/><path d="M348.75 234.5c0-4.42 3.58-8 8-8h556.5c4.42 0 8 3.58 8 8v33.8c0 4.42-3.58 8-8 8h-556.5c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#ae" transform="matrix(1,0,0,1,353.75,231.5) translate(0 24.4)"/><use xlink:href="#af" transform="matrix(1,0,0,1,353.75,231.5) translate(57.95 24.4)"/><use xlink:href="#ag" transform="matrix(1,0,0,1,353.75,231.5) translate(89.85000000000001 24.4)"/><use xlink:href="#ah" transform="matrix(1,0,0,1,353.75,231.5) translate(132.70000000000002 24.4)"/><use xlink:href="#ai" transform="matrix(1,0,0,1,353.75,231.5) translate(154.60000000000002 24.4)"/><use xlink:href="#aj" transform="matrix(1,0,0,1,353.75,231.5) translate(180.55 24.4)"/><use xlink:href="#ak" transform="matrix(1,0,0,1,353.75,231.5) translate(227.45000000000002 24.4)"/><use xlink:href="#al" transform="matrix(1,0,0,1,353.75,231.5) translate(252.45000000000002 24.4)"/><use xlink:href="#am" transform="matrix(1,0,0,1,353.75,231.5) translate(310.35 24.4)"/><use xlink:href="#an" transform="matrix(1,0,0,1,353.75,231.5) translate(330.35 24.4)"/><use xlink:href="#ao" transform="matrix(1,0,0,1,353.75,231.5) translate(374.3 24.4)"/><path d="M952.3 276.24c-6.24 0-11.3 5.05-11.3 11.3v15.04c0 6.24 5.06 11.3 11.3 11.3h93.66c6.23 0 11.3-5.06 11.3-11.3v-15.05c0-6.24-5.07-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#ap" transform="matrix(1,0,0,1,951,280.0000779958426) translate(6.599999999999994 19.1)"/><use xlink:href="#aq" transform="matrix(1,0,0,1,951,280.0000779958426) translate(56.55 19.1)"/><path d="M80.75 239.4c0-4.42 3.58-8 8-8H271.5c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8H88.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#ar" transform="matrix(1,0,0,1,85.75,236.39999999999998) translate(0 18.15)"/><use xlink:href="#as" transform="matrix(1,0,0,1,85.75,236.39999999999998) translate(55.550000000000004 18.15)"/><path d="M1074.54 276.24c-6.23 0-11.3 5.05-11.3 11.3v15.04c0 6.24 5.07 11.3 11.3 11.3h131.67c6.24 0 11.3-5.06 11.3-11.3v-15.05c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><use xlink:href="#at" transform="matrix(1,0,0,1,1073.25,280.0000779958426) translate(24.625 19.1)"/><use xlink:href="#aq" transform="matrix(1,0,0,1,1073.25,280.0000779958426) translate(76.525 19.1)"/><path d="M1193.75 382.95c0-4.42 3.58-8 8-8h19c4.42 0 8 3.58 8 8v279.93c0 4.42-3.58 8-8 8h-19c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M1195.75 384.95c0-4.42 3.58-8 8-8h15c4.42 0 8 3.58 8 8v15c0 4.42-3.58 8-8 8h-15c-4.42 0-8-3.58-8-8z" stroke="#333" fill="none"/><path d="M1201.95 398.65h18.6l-9.3-12.4z" stroke="#333" fill="#333"/><path d="M1195.75 417.95c0-4.42 3.58-8 8-8h15c4.42 0 8 3.58 8 8v98.96c0 4.43-3.58 8-8 8h-15c-4.42 0-8-3.57-8-8zM1197.75 465.43h27m-27 2h27m-27 2h27M1195.75 660.88c0 4.42 3.58 8 8 8h15c4.42 0 8-3.58 8-8v-15c0-4.42-3.58-8-8-8h-15c-4.42 0-8 3.58-8 8z" stroke="#333" fill="none"/><path d="M1201.95 647.18h18.6l-9.3 12.4z" stroke="#333" fill="#333"/><path d="M70 1063c0-4.42 3.58-8 8-8h1142.75c4.42 0 8 3.58 8 8v789c0 4.42-3.58 8-8 8H78c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M1016.83 1810c-6.53 0-11.83 5.3-11.83 11.83v15.78c0 6.55 5.3 11.84 11.83 11.84h187.44c6.54 0 11.83-5.3 11.83-11.83v-15.77c0-6.53-5.3-11.83-11.83-11.83z" stroke="#333" fill="#fff"/><use xlink:href="#au" transform="matrix(1,0,0,1,1015,1813.9443079618625) translate(40.875 19.6)"/><use xlink:href="#av" transform="matrix(1,0,0,1,1015,1813.9443079618625) translate(99.775 19.6)"/><path d="M116.75 1667.16c0-4.42 3.58-8 8-8H695c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H124.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aw" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(0 17.9)"/><use xlink:href="#Z" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(10.95 17.9)"/><use xlink:href="#ax" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(25.95 17.9)"/><use xlink:href="#ay" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(91.9 17.9)"/><use xlink:href="#az" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(163.85000000000002 17.9)"/><use xlink:href="#aA" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(182.8 17.9)"/><use xlink:href="#aB" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(226.8 17.9)"/><use xlink:href="#aC" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(270.8 17.9)"/><use xlink:href="#aD" transform="matrix(1,0,0,1,121.75000000000011,1664.1596159481924) translate(305.8 17.9)"/><path d="M116.75 1704.8c0-4.42 3.58-8 8-8H695c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H124.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aE" transform="matrix(1,0,0,1,121.75000000000011,1701.7996437861125) translate(0 17.9)"/><use xlink:href="#aF" transform="matrix(1,0,0,1,121.75000000000011,1701.7996437861125) translate(10.95 17.9)"/><use xlink:href="#aG" transform="matrix(1,0,0,1,121.75000000000011,1701.7996437861125) translate(51.85000000000001 17.9)"/><use xlink:href="#aH" transform="matrix(1,0,0,1,121.75000000000011,1701.7996437861125) translate(122.80000000000001 17.9)"/><use xlink:href="#aI" transform="matrix(1,0,0,1,121.75000000000011,1701.7996437861125) translate(192.8 17.9)"/><path d="M116.75 1742.44c0-4.42 3.58-8 8-8H695c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H124.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aw" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(0 17.9)"/><use xlink:href="#aJ" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(10.95 17.9)"/><use xlink:href="#aK" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(45.95 17.9)"/><use xlink:href="#aL" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(102.30000000000001 17.9)"/><use xlink:href="#aM" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(132.10000000000002 17.9)"/><use xlink:href="#aN" transform="matrix(1,0,0,1,121.75000000000011,1739.4396716240321) translate(157.10000000000002 17.9)"/><path d="M88.75 1637.72c0-4.42 3.58-8 8-8H667c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8H96.75c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aO" transform="matrix(1,0,0,1,93.75000000000006,1634.7165363295667) translate(0 17.9)"/><path d="M80.75 1072.93c0-4.42 3.58-8 8-8H659c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H88.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aP" transform="matrix(1,0,0,1,85.75000000000006,1069.9299860810156) translate(0 17.9)"/><use xlink:href="#aQ" transform="matrix(1,0,0,1,85.75000000000006,1069.9299860810156) translate(66.85000000000001 17.9)"/><use xlink:href="#aR" transform="matrix(1,0,0,1,85.75000000000006,1069.9299860810156) translate(123.75000000000001 17.9)"/><use xlink:href="#aS" transform="matrix(1,0,0,1,85.75000000000006,1069.9299860810156) translate(173.65 17.9)"/><path d="M448.75 1151.75c0-4.42 3.58-8 8-8h29c4.42 0 8 3.58 8 8v21.64c0 4.4-3.58 8-8 8h-29c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><use xlink:href="#aT" transform="matrix(1,0,0,1,458.75,1147.5140027837674) translate(7.5 19.1)"/><path d="M88.75 1401.5c0-4.42 3.58-8 8-8H667c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H96.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aU" transform="matrix(1,0,0,1,93.75000000000006,1398.5) translate(0 17.9)"/><use xlink:href="#aV" transform="matrix(1,0,0,1,93.75000000000006,1398.5) translate(84.85000000000001 17.9)"/><path d="M301.88 1446.5c0-4.42 3.58-8 8-8h187.3c4.4 0 8 3.58 8 8v16.36c0 4.42-3.6 8-8 8h-187.3c-4.42 0-8-3.58-8-8zM513.27 1446.6h32.36m-32.36 8.08h32.36m-32.36 8.1h32.36m-8.1-16.2v24.28m-8.08-24.27v24.26m-8.1-24.27v24.26m-8.08-32.36h32.36v32.36h-32.36z" stroke="#333" fill="#fff"/><use xlink:href="#aW" transform="matrix(1,0,0,1,322.20500347973996,1441.735997216208) translate(1.5250000000000057 16.6)"/><use xlink:href="#aX" transform="matrix(1,0,0,1,322.20500347973996,1441.735997216208) translate(51.525000000000006 16.6)"/><use xlink:href="#aY" transform="matrix(1,0,0,1,322.20500347973996,1441.735997216208) translate(81.47500000000001 16.6)"/><path d="M116.75 1440.32c0-4.42 3.58-8 8-8h169.12c4.42 0 8 3.58 8 8v25.18c0 4.42-3.58 8-8 8H124.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aZ" transform="matrix(1,0,0,1,121.75000000000006,1437.3200139189844) translate(0 20.4)"/><use xlink:href="#ba" transform="matrix(1,0,0,1,121.75000000000006,1437.3200139189844) translate(42.95 20.4)"/><use xlink:href="#bb" transform="matrix(1,0,0,1,121.75000000000006,1437.3200139189844) translate(86.9 20.4)"/><use xlink:href="#bc" transform="matrix(1,0,0,1,121.75000000000006,1437.3200139189844) translate(155.8 20.4)"/><path d="M88.75 1521.5c0-4.42 3.58-8 8-8H667c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H96.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aP" transform="matrix(1,0,0,1,93.75000000000006,1518.5) translate(0 17.9)"/><use xlink:href="#bd" transform="matrix(1,0,0,1,93.75000000000006,1518.5) translate(66.85000000000001 17.9)"/><use xlink:href="#be" transform="matrix(1,0,0,1,93.75000000000006,1518.5) translate(103.75000000000001 17.9)"/><use xlink:href="#bf" transform="matrix(1,0,0,1,93.75000000000006,1518.5) translate(156.60000000000002 17.9)"/><path d="M123.75 1548.5c0-4.4 3.58-8 8-8h449c4.42 0 8 3.6 8 8v37c0 4.42-3.58 8-8 8h-449c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#bg" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(0 25.15)"/><use xlink:href="#bh" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(53.900000000000006 25.15)"/><use xlink:href="#bi" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(121.80000000000001 25.15)"/><use xlink:href="#bj" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(208.70000000000002 25.15)"/><use xlink:href="#bk" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(226.65 25.15)"/><use xlink:href="#bl" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(263.6 25.15)"/><use xlink:href="#bm" transform="matrix(1,0,0,1,128.75,1545.5100069594923) translate(358.6 25.15)"/><path d="M433.75 1556.18c0-4.4 3.58-8 8-8h29c4.42 0 8 3.6 8 8v21.65c0 4.4-3.58 8-8 8h-29c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><use xlink:href="#bn" transform="matrix(1,0,0,1,443.7499999999999,1551.948992344578) translate(2.5 19.1)"/><path d="M123.75 1601.5c0-4.42 3.58-8 8-8h449c4.42 0 8 3.58 8 8v37c0 4.4-3.58 8-8 8h-449c-4.42 0-8-3.6-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#bo" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(0 25.15)"/><use xlink:href="#bp" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(41.95 25.15)"/><use xlink:href="#bq" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(75.95 25.15)"/><use xlink:href="#bj" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(129.95 25.15)"/><use xlink:href="#bk" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(147.89999999999998 25.15)"/><use xlink:href="#bl" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(184.84999999999997 25.15)"/><use xlink:href="#bm" transform="matrix(1,0,0,1,128.75,1598.5000000000002) translate(279.84999999999997 25.15)"/><path d="M359.38 1607.72c0-4.42 3.58-8 8-8h29c4.4 0 8 3.58 8 8v21.64c0 4.4-3.6 8-8 8h-29c-4.42 0-8-3.6-8-8z" stroke="#333" fill="#fff"/><use xlink:href="#br" transform="matrix(1,0,0,1,369.375,1603.4805391133586) translate(2.5 19.1)"/><path d="M125.87 1488.84c0-4.42 3.6-8 8-8h480c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8h-480c-4.4 0-8-3.58-8-8z" fill="none"/><path d="M125.87 1486.84h20v20h-20z" stroke="#333" fill="#fff"/><path d="M129.87 1494.84l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><use xlink:href="#bs" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(5.17500000000004 19.6)"/><use xlink:href="#bt" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(25.17500000000004 19.6)"/><use xlink:href="#bu" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(68.17500000000004 19.6)"/><use xlink:href="#bv" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(88.17500000000004 19.6)"/><use xlink:href="#ba" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(157.12500000000003 19.6)"/><use xlink:href="#bw" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(201.07500000000002 19.6)"/><use xlink:href="#bx" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(278.975 19.6)"/><use xlink:href="#by" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(371.875 19.6)"/><use xlink:href="#bz" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(397.82499999999993 19.6)"/><use xlink:href="#bA" transform="matrix(1,0,0,1,157.87499999999997,1480.842494780393) translate(436.82499999999993 19.6)"/><path d="M116.75 1786.5c0-4.42 3.58-8 8-8H695c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H124.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><use xlink:href="#aw" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(0 17.9)"/><use xlink:href="#bB" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(10.95 17.9)"/><use xlink:href="#bC" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(95.85000000000001 17.9)"/><use xlink:href="#bj" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(165.75 17.9)"/><use xlink:href="#bD" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(183.7 17.9)"/><use xlink:href="#bE" transform="matrix(1,0,0,1,121.75000000000011,1783.5) translate(208.7 17.9)"/><path d="M608.75 332.4c0-4.4 3.58-8 8-8H933c4.42 0 8 3.6 8 8v24c0 4.43-3.58 8-8 8H616.75c-4.42 0-8-3.57-8-8z" stroke="#333" fill="#fff"/><path d="M903 334.4c0-4.4 3.58-8 8-8h20c4.42 0 8 3.6 8 8v20c0 4.43-3.58 8-8 8h-20c-4.42 0-8-3.57-8-8z" stroke="#333" fill="none"/><path d="M910.2 337.2h21.6L921 351.6z" stroke="#333" fill="#333"/><use xlink:href="#bF" transform="matrix(1,0,0,1,620.75,324.4141810265149) translate(132.5 23.6)"/><path d="M80.75 418.77c0-4.42 3.58-8 8-8h354c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8h-354c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M100.75 426.77c0 5.53-4.48 10-10 10s-10-4.47-10-10c0-5.52 4.48-10 10-10s10 4.48 10 10z" stroke="#333" fill="#fff"/><path d="M95.75 426.77c0 2.76-2.24 5-5 5s-5-2.24-5-5 2.24-5 5-5 5 2.24 5 5z" stroke="#333" fill="#333"/><path d="M224.08 426.77c0 5.53-4.47 10-10 10-5.52 0-10-4.47-10-10 0-5.52 4.48-10 10-10 5.53 0 10 4.48 10 10zM347.42 426.77c0 5.53-4.48 10-10 10-5.53 0-10-4.47-10-10 0-5.52 4.47-10 10-10 5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75,410.7731329741161) translate(5.024999999999999 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333331,410.7731329741161) translate(5.024999999999999 23.6)"/><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666663,410.7731329741161) translate(4.850000000000001 23.6)"/><path d="M80.75 454.9c0-4.4 3.58-8 8-8h354c4.42 0 8 3.6 8 8v24c0 4.43-3.58 8-8 8h-354c-4.42 0-8-3.57-8-8z" fill="none"/><path d="M100.75 462.9c0 5.54-4.48 10-10 10s-10-4.46-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10z" stroke="#333" fill="#fff"/><path d="M95.75 462.9c0 2.78-2.24 5-5 5s-5-2.22-5-5c0-2.75 2.24-5 5-5s5 2.25 5 5z" stroke="#333" fill="#333"/><path d="M224.08 462.9c0 5.54-4.47 10-10 10-5.52 0-10-4.46-10-10 0-5.5 4.48-10 10-10 5.53 0 10 4.5 10 10zM347.42 462.9c0 5.54-4.48 10-10 10-5.53 0-10-4.46-10-10 0-5.5 4.47-10 10-10 5.52 0 10 4.5 10 10z" stroke="#333" fill="#fff"/><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,446.91459665169674) translate(5.024999999999999 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,446.91459665169674) translate(5.024999999999999 23.6)"/><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,446.91459665169674) translate(4.850000000000001 23.6)"/><path d="M80.75 494.9c0-4.4 3.58-8 8-8h354c4.42 0 8 3.6 8 8v24c0 4.43-3.58 8-8 8h-354c-4.42 0-8-3.57-8-8z" fill="none"/><path d="M100.75 502.9c0 5.54-4.48 10-10 10s-10-4.46-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10z" stroke="#333" fill="#fff"/><path d="M95.75 502.9c0 2.78-2.24 5-5 5s-5-2.22-5-5c0-2.75 2.24-5 5-5s5 2.25 5 5z" stroke="#333" fill="#333"/><path d="M224.08 502.9c0 5.54-4.47 10-10 10-5.52 0-10-4.46-10-10 0-5.5 4.48-10 10-10 5.53 0 10 4.5 10 10zM347.42 502.9c0 5.54-4.48 10-10 10-5.53 0-10-4.46-10-10 0-5.5 4.47-10 10-10 5.52 0 10 4.5 10 10z" stroke="#333" fill="#fff"/><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,486.91459665169685) translate(5.024999999999999 23.6)"/><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,486.91459665169685) translate(5.024999999999999 23.6)"/><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,486.91459665169685) translate(4.850000000000001 23.6)"/><path d="M80.75 534.9c0-4.4 3.58-8 8-8h354c4.42 0 8 3.6 8 8v24c0 4.43-3.58 8-8 8h-354c-4.42 0-8-3.57-8-8z" fill="none"/><path d="M100.75 542.9c0 5.54-4.48 10-10 10s-10-4.46-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10z" stroke="#333" fill="#fff"/><path d="M95.75 542.9c0 2.78-2.24 5-5 5s-5-2.22-5-5c0-2.75 2.24-5 5-5s5 2.25 5 5z" stroke="#333" fill="#333"/><path d="M224.08 542.9c0 5.54-4.47 10-10 10-5.52 0-10-4.46-10-10 0-5.5 4.48-10 10-10 5.53 0 10 4.5 10 10zM347.42 542.9c0 5.54-4.48 10-10 10-5.53 0-10-4.46-10-10 0-5.5 4.47-10 10-10 5.52 0 10 4.5 10 10z" stroke="#333" fill="#fff"/><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,526.9145966516968) translate(5.024999999999999 23.6)"/><g><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,526.9145966516968) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,526.9145966516968) translate(4.850000000000001 23.6)"/></g><path d="M80.75 571.06c0-4.42 3.58-8 8-8h354c4.42 0 8 3.58 8 8v24c0 4.4-3.58 8-8 8h-354c-4.42 0-8-3.6-8-8z" fill="none"/><path d="M100.75 579.06c0 5.52-4.48 10-10 10s-10-4.48-10-10c0-5.53 4.48-10 10-10s10 4.47 10 10zM224.08 579.06c0 5.52-4.47 10-10 10-5.52 0-10-4.48-10-10 0-5.53 4.48-10 10-10 5.53 0 10 4.47 10 10z" stroke="#333" fill="#fff"/><path d="M219.08 579.06c0 2.76-2.24 5-5 5s-5-2.24-5-5c0-2.77 2.24-5 5-5s5 2.23 5 5z" stroke="#333" fill="#333"/><path d="M347.42 579.06c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10 0-5.53 4.47-10 10-10 5.52 0 10 4.47 10 10z" stroke="#333" fill="#fff"/><g><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,563.0560603292774) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,563.0560603292774) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,563.0560603292774) translate(4.850000000000001 23.6)"/></g><path d="M80.75 604.5c0-4.42 3.58-8 8-8h354c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8h-354c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M100.75 612.5c0 5.52-4.48 10-10 10s-10-4.48-10-10 4.48-10 10-10 10 4.48 10 10zM224.08 612.5c0 5.52-4.47 10-10 10-5.52 0-10-4.48-10-10s4.48-10 10-10c5.53 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><path d="M219.08 612.5c0 2.76-2.24 5-5 5s-5-2.24-5-5 2.24-5 5-5 5 2.24 5 5z" stroke="#333" fill="#333"/><path d="M347.42 612.5c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><g><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,596.5) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,596.5) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,596.5) translate(4.850000000000001 23.6)"/></g><path d="M80.75 638.88c0-4.42 3.58-8 8-8h354c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8h-354c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M100.75 646.88c0 5.52-4.48 10-10 10s-10-4.48-10-10c0-5.53 4.48-10 10-10s10 4.47 10 10zM224.08 646.88c0 5.52-4.47 10-10 10-5.52 0-10-4.48-10-10 0-5.53 4.48-10 10-10 5.53 0 10 4.47 10 10zM347.42 646.88c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10 0-5.53 4.47-10 10-10 5.52 0 10 4.47 10 10z" stroke="#333" fill="#fff"/><path d="M342.42 646.88c0 2.76-2.24 5-5 5s-5-2.24-5-5 2.24-5 5-5 5 2.24 5 5z" stroke="#333" fill="#333"/><g><use xlink:href="#bG" transform="matrix(1,0,0,1,112.75000000000006,630.8769343003341) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bH" transform="matrix(1,0,0,1,236.08333333333337,630.8769343003341) translate(5.024999999999999 23.6)"/></g><g><use xlink:href="#bI" transform="matrix(1,0,0,1,359.41666666666674,630.8769343003341) translate(4.850000000000001 23.6)"/></g><path d="M70 686.2c0-4.4 3.58-8 8-8h1142.75c4.42 0 8 3.6 8 8v352.3c0 4.42-3.58 8-8 8H78c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M80.75 696.13c0-4.4 3.58-8 8-8h535.88c4.4 0 8 3.6 8 8v33.8c0 4.42-3.6 8-8 8H88.75c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#bJ" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(0 24.4)"/><use xlink:href="#bK" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(36.900000000000006 24.4)"/><use xlink:href="#bL" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(58.80000000000001 24.4)"/><use xlink:href="#be" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(122.70000000000002 24.4)"/><use xlink:href="#bM" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(175.55 24.4)"/><use xlink:href="#bN" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(232.5 24.4)"/><use xlink:href="#bO" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(269.4 24.4)"/><use xlink:href="#am" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(389 24.4)"/><use xlink:href="#bP" transform="matrix(1,0,0,1,85.75,693.1349056624563) translate(409 24.4)"/></g><path d="M348.75 329.37c0-4.42 3.58-8 8-8h244c4.42 0 8 3.58 8 8v33.8c0 4.42-3.58 8-8 8h-244c-4.42 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#bQ" transform="matrix(1,0,0,1,353.75,326.3680376060253) translate(0 24.4)"/><use xlink:href="#bR" transform="matrix(1,0,0,1,353.75,326.3680376060253) translate(15 24.4)"/><use xlink:href="#bS" transform="matrix(1,0,0,1,353.75,326.3680376060253) translate(95.95 24.4)"/><use xlink:href="#bT" transform="matrix(1,0,0,1,353.75,326.3680376060253) translate(166.9 24.4)"/><use xlink:href="#bU" transform="matrix(1,0,0,1,353.75,326.3680376060253) translate(221.65 24.4)"/></g><path d="M1178.75 805.7c0-4.4 3.58-8 8-8h17.75c4.42 0 8 3.6 8 8v184c0 4.42-3.58 8-8 8h-17.75c-4.42 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><path d="M1180.75 807.7c0-4.4 3.58-8 8-8h13.75c4.42 0 8 3.6 8 8v13.76c0 4.4-3.58 8-8 8h-13.75c-4.42 0-8-3.6-8-8z" stroke="#333" fill="none"/><path d="M1186.7 820.53h17.85l-8.92-11.9z" stroke="#333" fill="#333"/><path d="M1180.75 839.46c0-4.42 3.58-8 8-8h13.75c4.42 0 8 3.58 8 8v52.25c0 4.42-3.58 8-8 8h-13.75c-4.42 0-8-3.58-8-8zM1182.75 863.58h25.75m-25.75 2h25.75m-25.75 2h25.75M1180.75 987.7c0 4.42 3.58 8 8 8h13.75c4.42 0 8-3.58 8-8v-13.74c0-4.42-3.58-8-8-8h-13.75c-4.42 0-8 3.58-8 8z" stroke="#333" fill="none"/><path d="M1186.7 974.88h17.85l-8.92 11.9z" stroke="#333" fill="#333"/><path d="M94.38 805.7c0-4.4 3.58-8 8-8h1063.37c4.42 0 8 3.6 8 8v184c0 4.42-3.58 8-8 8H102.37c-4.4 0-8-3.58-8-8z" fill="#fff"/><path d="M94.38 797.7h1079.37v36H94.37z" fill="#ccc"/><path d="M94.38 869.7h1079.37v36H94.37zM94.38 941.7h1079.37v36H94.37z" fill="#f0f0f0"/><path d="M510.1 797.7v200M94.38 805.7c0-4.4 3.58-8 8-8h1063.37c4.42 0 8 3.6 8 8v184c0 4.42-3.58 8-8 8H102.37c-4.4 0-8-3.58-8-8z" stroke="#333" fill="none"/><g><use xlink:href="#bV" transform="matrix(1,0,0,1,100.375,797.7051221754638) translate(177.025 21.6)"/></g><g><use xlink:href="#bW" transform="matrix(1,0,0,1,516.0870376126826,797.7051221754638) translate(282.55 21.6)"/><use xlink:href="#bX" transform="matrix(1,0,0,1,516.0870376126826,797.7051221754638) translate(324.5 21.6)"/></g><g><use xlink:href="#bY" transform="matrix(1,0,0,1,100.375,833.7051221754638) translate(199.5 21.6)"/></g><g><use xlink:href="#bZ" transform="matrix(1,0,0,1,516.0870376126826,833.7051221754638) translate(304 21.6)"/></g><g><use xlink:href="#bY" transform="matrix(1,0,0,1,100.375,869.7051221754638) translate(199.5 21.6)"/></g><g><use xlink:href="#ca" transform="matrix(1,0,0,1,516.0870376126826,869.7051221754638) translate(304 21.6)"/></g><g><use xlink:href="#bY" transform="matrix(1,0,0,1,100.375,905.7051221754638) translate(199.5 21.6)"/></g><g><use xlink:href="#cb" transform="matrix(1,0,0,1,516.0870376126826,905.7051221754638) translate(304 21.6)"/></g><g><use xlink:href="#bY" transform="matrix(1,0,0,1,100.375,941.7051221754638) translate(199.5 21.6)"/></g><g><use xlink:href="#cc" transform="matrix(1,0,0,1,516.0870376126826,941.7051221754638) translate(318.5 21.6)"/></g><path d="M94.37 757c0-4.42 3.6-8 8-8h418.25c4.42 0 8 3.58 8 8v21.64c0 4.42-3.58 8-8 8H102.37c-4.4 0-8-3.58-8-8z" stroke="#333" fill="#fff"/><g><use xlink:href="#ab" transform="matrix(1,0,0,1,104.37499999999994,752.764002783792) translate(0 19.1)"/><use xlink:href="#ac" transform="matrix(1,0,0,1,104.37499999999994,752.764002783792) translate(39 19.1)"/><use xlink:href="#ad" transform="matrix(1,0,0,1,104.37499999999994,752.764002783792) translate(54.5 19.1)"/></g><path d="M549.46 749c-6.24 0-11.3 5.06-11.3 11.3v15.05c0 6.23 5.06 11.3 11.3 11.3h127.7c6.24 0 11.3-5.07 11.3-11.3V760.3c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><g><use xlink:href="#ap" transform="matrix(1,0,0,1,548.1652877697843,752.764002783792) translate(23.599999999999994 19.1)"/><use xlink:href="#aq" transform="matrix(1,0,0,1,548.1652877697843,752.764002783792) translate(73.55 19.1)"/></g><path d="M725.75 749c-6.24 0-11.3 5.06-11.3 11.3v15.05c0 6.23 5.06 11.3 11.3 11.3h127.7c6.24 0 11.3-5.07 11.3-11.3V760.3c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><g><use xlink:href="#at" transform="matrix(1,0,0,1,724.4539568345326,752.764002783792) translate(22.625 19.1)"/><use xlink:href="#aq" transform="matrix(1,0,0,1,724.4539568345326,752.764002783792) translate(74.525 19.1)"/></g><path d="M595.04 694.2c-6.23 0-11.3 5.07-11.3 11.3v15.06c0 6.24 5.07 11.3 11.3 11.3h127.7c6.24 0 11.3-5.06 11.3-11.3V705.5c0-6.23-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><g><use xlink:href="#cd" transform="matrix(1,0,0,1,593.75,697.9788945272883) translate(34.05 19.1)"/></g><path d="M902.03 749c-6.23 0-11.3 5.06-11.3 11.3v15.05c0 6.23 5.07 11.3 11.3 11.3h127.7c6.25 0 11.3-5.07 11.3-11.3V760.3c0-6.24-5.05-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><g><use xlink:href="#U" transform="matrix(1,0,0,1,900.7426258992809,752.764002783792) translate(28.075000000000003 19.1)"/><use xlink:href="#ce" transform="matrix(1,0,0,1,900.7426258992809,752.764002783792) translate(83.025 19.1)"/></g><path d="M1078.32 749c-6.23 0-11.3 5.06-11.3 11.3v15.05c0 6.23 5.07 11.3 11.3 11.3h127.7c6.24 0 11.3-5.07 11.3-11.3V760.3c0-6.24-5.06-11.3-11.3-11.3z" stroke="#333" fill="#fff"/><g><use xlink:href="#at" transform="matrix(1,0,0,1,1077.0312949640293,752.764002783792) translate(4.099999999999994 19.1)"/><use xlink:href="#cf" transform="matrix(1,0,0,1,1077.0312949640293,752.764002783792) translate(56 19.1)"/></g><path d="M292 842.7c0-4.4 3.58-8 8-8h107.33c4.42 0 8 3.6 8 8v24c0 4.42-3.58 8-8 8H300c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M292 840.7h20v20h-20z" stroke="#333" fill="#fff"/><path d="M296 848.7l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M292 882.7c0-4.4 3.58-8 8-8h107.33c4.42 0 8 3.6 8 8v24c0 4.42-3.58 8-8 8H300c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M292 880.7h20v20h-20z" stroke="#333" fill="#fff"/><path d="M296 888.7l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M292 917.7c0-4.4 3.58-8 8-8h107.33c4.42 0 8 3.6 8 8v24c0 4.42-3.58 8-8 8H300c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M292 915.7h20v20h-20z" stroke="#333" fill="#fff"/><path d="M296 923.7l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M292 950.24c0-4.42 3.58-8 8-8h107.33c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8H300c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M292 948.24h20v20h-20z" stroke="#333" fill="#fff"/><path d="M296 956.24l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M94.38 1004.7c0-4.42 3.58-8 8-8h535.87c4.42 0 8 3.58 8 8v33.8c0 4.42-3.58 8-8 8H102.37c-4.4 0-8-3.58-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#cg" transform="matrix(1,0,0,1,99.375,1001.7) translate(0 24.4)"/><use xlink:href="#ch" transform="matrix(1,0,0,1,99.375,1001.7) translate(35 24.4)"/><use xlink:href="#ci" transform="matrix(1,0,0,1,99.375,1001.7) translate(87.85 24.4)"/></g><path d="M124.88 1111.57c0-4.42 3.58-8 8-8h1099c4.4 0 8 3.58 8 8v104c0 4.42-3.6 8-8 8h-1099c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M144.88 1123.57c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><path d="M139.88 1123.57c0 2.76-2.24 5-5 5-2.77 0-5-2.24-5-5s2.23-5 5-5c2.76 0 5 2.24 5 5z" stroke="#333" fill="#333"/><path d="M144.88 1163.57c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10zM144.88 1203.57c0 5.52-4.48 10-10 10-5.53 0-10-4.48-10-10s4.47-10 10-10c5.52 0 10 4.48 10 10z" stroke="#333" fill="#fff"/><g><use xlink:href="#au" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(0 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(58.900000000000006 23.6)"/><use xlink:href="#cj" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(77.85000000000001 23.6)"/><use xlink:href="#ck" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(140.65000000000003 23.6)"/><use xlink:href="#cl" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(151.60000000000002 23.6)"/><use xlink:href="#cm" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(185.60000000000002 23.6)"/><use xlink:href="#cn" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(215.60000000000002 23.6)"/><use xlink:href="#co" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(249.60000000000002 23.6)"/><use xlink:href="#cp" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(298.25 23.6)"/><use xlink:href="#cq" transform="matrix(1,0,0,1,156.875,1103.5700139189355) translate(404.1 23.6)"/></g><g><use xlink:href="#cr" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(0 23.6)"/><use xlink:href="#cs" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(36.95 23.6)"/><use xlink:href="#bb" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(123.85000000000001 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(192.75 23.6)"/><use xlink:href="#ct" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(211.7 23.6)"/><use xlink:href="#cu" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(271.65 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(341.65 23.6)"/><use xlink:href="#cw" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(394.59999999999997 23.6)"/><use xlink:href="#cx" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(414.59999999999997 23.6)"/><use xlink:href="#cy" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(463.54999999999995 23.6)"/><use xlink:href="#ck" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(502.44999999999993 23.6)"/><use xlink:href="#cz" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(513.4 23.6)"/><use xlink:href="#cm" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(554.3 23.6)"/><use xlink:href="#cA" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(584.3 23.6)"/><use xlink:href="#cB" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(653.25 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(711.15 23.6)"/><use xlink:href="#cp" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(730.1 23.6)"/><use xlink:href="#cq" transform="matrix(1,0,0,1,156.875,1143.5700139189355) translate(835.95 23.6)"/></g><g><use xlink:href="#au" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(0 23.6)"/><use xlink:href="#cC" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(58.900000000000006 23.6)"/><use xlink:href="#aK" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(81.80000000000001 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(138.15 23.6)"/><use xlink:href="#cD" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(157.1 23.6)"/><use xlink:href="#cx" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(187.1 23.6)"/><use xlink:href="#cE" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(236.05 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(275.05 23.6)"/><use xlink:href="#cF" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(294 23.6)"/><use xlink:href="#cG" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(366.8 23.6)"/><use xlink:href="#cH" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(410.8 23.6)"/><use xlink:href="#cI" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(477.70000000000005 23.6)"/><use xlink:href="#cJ" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(550.6500000000001 23.6)"/><use xlink:href="#cK" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(587.5500000000001 23.6)"/><use xlink:href="#cL" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(633.5000000000001 23.6)"/><use xlink:href="#ck" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(681.5000000000001 23.6)"/><use xlink:href="#cl" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(692.4500000000002 23.6)"/><use xlink:href="#cm" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(726.4500000000002 23.6)"/><use xlink:href="#cn" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(756.4500000000002 23.6)"/><use xlink:href="#co" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(790.4500000000002 23.6)"/><use xlink:href="#cp" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(839.1000000000001 23.6)"/><use xlink:href="#cq" transform="matrix(1,0,0,1,156.875,1183.5700139189355) translate(944.9500000000002 23.6)"/></g><path d="M273.75 1369.5c0-4.42 3.58-8 8-8h107.33c4.42 0 8 3.58 8 8v24c0 4.42-3.58 8-8 8H281.75c-4.42 0-8-3.58-8-8z" fill="none"/><path d="M273.75 1367.5h20v20h-20z" stroke="#333" fill="#fff"/><path d="M277.75 1375.5l4 6 8-8" stroke="#333" stroke-width="3" fill="none"/><path d="M80.75 1230.57c0-4.42 3.58-8 8-8H659c4.42 0 8 3.58 8 8v21.64c0 4.43-3.58 8-8 8H88.75c-4.42 0-8-3.57-8-8z" stroke="#000" stroke-opacity="0" fill="#fff" fill-opacity="0"/><g><use xlink:href="#aP" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(0 17.9)"/><use xlink:href="#aQ" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(66.85000000000001 17.9)"/><use xlink:href="#cM" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(123.75000000000001 17.9)"/><use xlink:href="#cN" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(210.5 17.9)"/><use xlink:href="#cO" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(250.45 17.9)"/><use xlink:href="#cP" transform="matrix(1,0,0,1,85.75000000000006,1227.5700139189353) translate(320.29999999999995 17.9)"/></g><path d="M116.75 1268.2c0-4.4 3.58-8 8-8h1099c4.42 0 8 3.6 8 8v104c0 4.43-3.58 8-8 8h-1099c-4.42 0-8-3.57-8-8z" fill="none"/><path d="M136.75 1280.2c0 5.53-4.48 10-10 10s-10-4.47-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10zM136.75 1320.2c0 5.53-4.48 10-10 10s-10-4.47-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10zM136.75 1360.2c0 5.53-4.48 10-10 10s-10-4.47-10-10c0-5.5 4.48-10 10-10s10 4.5 10 10z" stroke="#333" fill="#fff"/><path d="M131.75 1360.2c0 2.77-2.24 5-5 5s-5-2.23-5-5c0-2.75 2.24-5 5-5s5 2.25 5 5z" stroke="#333" fill="#333"/><g><use xlink:href="#cQ" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(0 23.6)"/><use xlink:href="#cR" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(66 23.6)"/><use xlink:href="#cC" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(114.95 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(137.85 23.6)"/><use xlink:href="#cS" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(190.8 23.6)"/><use xlink:href="#cT" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(245.8 23.6)"/><use xlink:href="#cw" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(276.75 23.6)"/><use xlink:href="#cU" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(296.75 23.6)"/><use xlink:href="#cV" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(353.65 23.6)"/><use xlink:href="#cW" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(373.65 23.6)"/><use xlink:href="#cC" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(421.65 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(444.54999999999995 23.6)"/><use xlink:href="#cT" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(497.49999999999994 23.6)"/><use xlink:href="#cX" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(528.4499999999999 23.6)"/><use xlink:href="#cY" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(598.4 23.6)"/><use xlink:href="#cT" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(643.35 23.6)"/><use xlink:href="#cZ" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(674.3000000000001 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(767.1500000000001 23.6)"/><use xlink:href="#da" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(786.1000000000001 23.6)"/><use xlink:href="#W" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(822.0500000000002 23.6)"/><use xlink:href="#db" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(842.0500000000002 23.6)"/><use xlink:href="#cD" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(885.9500000000002 23.6)"/><use xlink:href="#dc" transform="matrix(1,0,0,1,148.7500000000001,1260.2100417568552) translate(915.9500000000002 23.6)"/></g><g><use xlink:href="#cQ" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(0 23.6)"/><use xlink:href="#dd" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(66 23.6)"/><use xlink:href="#de" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(90 23.6)"/><use xlink:href="#cC" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(134 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(156.9 23.6)"/><use xlink:href="#df" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(209.85000000000002 23.6)"/><use xlink:href="#dg" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(234.85000000000002 23.6)"/><use xlink:href="#dh" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(249.85000000000002 23.6)"/><use xlink:href="#cE" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(297.8 23.6)"/><use xlink:href="#cT" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(336.8 23.6)"/><use xlink:href="#cZ" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(367.75 23.6)"/><use xlink:href="#cw" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(460.6 23.6)"/><use xlink:href="#cD" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(480.6 23.6)"/><use xlink:href="#cx" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(510.6 23.6)"/><use xlink:href="#di" transform="matrix(1,0,0,1,148.7500000000001,1300.2100417568552) translate(559.5500000000001 23.6)"/></g><g><use xlink:href="#dj" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(0 23.6)"/><use xlink:href="#dk" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(98.85000000000001 23.6)"/><use xlink:href="#cD" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(145.75 23.6)"/><use xlink:href="#cv" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(175.75 23.6)"/><use xlink:href="#dl" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(228.7 23.6)"/><use xlink:href="#dm" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(272.65 23.6)"/><use xlink:href="#dn" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(336.54999999999995 23.6)"/><use xlink:href="#do" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(383.49999999999994 23.6)"/><use xlink:href="#dp" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(448.4 23.6)"/><use xlink:href="#az" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(502.34999999999997 23.6)"/><use xlink:href="#dq" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(521.3 23.6)"/><use xlink:href="#dr" transform="matrix(1,0,0,1,148.7500000000001,1340.2100417568552) translate(559.25 23.6)"/></g><defs><path fill="#a9afb8" d="M140-251c80 0 125 45 125 126S219 4 139 4C58 4 15-44 15-125s44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="ds"/><path fill="#a9afb8" d="M169-182c-1-43-94-46-97-3 18 66 151 10 154 114 3 95-165 93-204 36-6-8-10-19-12-30l50-8c3 46 112 56 116 5-17-69-150-10-154-114-4-87 153-88 188-35 5 8 8 18 10 28" id="dt"/><path fill="#a9afb8" d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="du"/><g id="a"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#ds"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,20.39506172839506,0)" xlink:href="#dt"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,37.876543209876544,0)" xlink:href="#du"/></g><path fill="#a9afb8" d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="dv"/><path fill="#a9afb8" d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="dw"/><path fill="#a9afb8" d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="dx"/><path fill="#a9afb8" d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="dy"/><path fill="#a9afb8" d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="dz"/><path fill="#a9afb8" d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="dA"/><path fill="#a9afb8" d="M110-194c64 0 96 36 96 99 0 64-35 99-97 99-61 0-95-36-95-99 0-62 34-99 96-99zm-1 164c35 0 45-28 45-65 0-40-10-65-43-65-34 0-45 26-45 65 0 36 10 65 43 65" id="dB"/><path fill="#a9afb8" d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="dC"/><g id="b"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#dv"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,14.5679012345679,0)" xlink:href="#dw"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,24.76543209876543,0)" xlink:href="#dx"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,39.333333333333336,0)" xlink:href="#dy"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,53.90123456790123,0)" xlink:href="#dz"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,62.56913580246913,0)" xlink:href="#dA"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,69.85308641975308,0)" xlink:href="#dB"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,85.80493827160494,0)" xlink:href="#dC"/></g><path fill="#a9afb8" d="M67-93c0 74 22 123 53 168H70C40 30 18-18 18-93s22-123 52-168h50c-32 44-53 94-53 168" id="dD"/><path fill="#a9afb8" d="M186 0v-106H76V0H24v-248h52v99h110v-99h50V0h-50" id="dE"/><path fill="#a9afb8" d="M137-138c1-29-70-34-71-4 15 46 118 7 119 86 1 83-164 76-172 9l43-7c4 19 20 25 44 25 33 8 57-30 24-41C81-84 22-81 20-136c-2-80 154-74 161-7" id="dF"/><g id="c"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#dD"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,8.667901234567902,0)" xlink:href="#dE"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,27.53333333333333,0)" xlink:href="#dB"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,43.48518518518519,0)" xlink:href="#dF"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,58.05308641975308,0)" xlink:href="#dz"/></g><path fill="#a9afb8" d="M220-157c-53 9-28 100-34 157h-49v-107c1-27-5-49-29-50C55-147 81-57 75 0H25l-1-190h47c2 12-1 28 3 38 10-53 101-56 108 0 13-22 24-43 59-42 82 1 51 116 57 194h-49v-107c-1-25-5-48-29-50" id="dG"/><path fill="#a9afb8" d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="dH"/><g id="d"><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#dG"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,23.308641975308642,0)" xlink:href="#dB"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,39.260493827160495,0)" xlink:href="#dH"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,55.212345679012344,0)" xlink:href="#dx"/></g><path fill="#a9afb8" d="M102-93c0 74-22 123-52 168H0C30 29 54-18 53-93c0-74-22-123-53-168h50c30 45 52 94 52 168" id="dI"/><use transform="matrix(0.0728395061728395,0,0,0.0728395061728395,0,0)" xlink:href="#dI" id="e"/><path fill="#a9afb8" d="M153-248C145-148 188 4 80 4 36 3 13-21 6-62l32-5c4 25 16 42 43 43 27 0 39-20 39-49v-147H72v-28h81" id="dJ"/><path fill="#a9afb8" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="dK"/><path fill="#a9afb8" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="dL"/><path fill="#a9afb8" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="dM"/><g id="f"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dJ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,7.648148148148148,0)" xlink:href="#dK"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.14609053497942,0)" xlink:href="#dL"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,24.644032921810698,0)" xlink:href="#dM"/></g><path fill="#a9afb8" d="M240 0l2-218c-23 76-54 145-80 218h-23L58-218 59 0H30v-248h44l77 211c21-75 51-140 76-211h43V0h-30" id="dN"/><path fill="#a9afb8" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="dO"/><path fill="#a9afb8" d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="dP"/><path fill="#a9afb8" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="dQ"/><path fill="#a9afb8" d="M24 0v-261h32V0H24" id="dR"/><g id="g"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dN"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,12.704423868312757,0)" xlink:href="#dO"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.06111111111111,0)" xlink:href="#dP"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,24.559053497942386,0)" xlink:href="#dK"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,33.05699588477366,0)" xlink:href="#dQ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,41.55493827160494,0)" xlink:href="#dR"/></g><path fill="#a9afb8" d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="dS"/><path fill="#a9afb8" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="dT"/><path fill="#a9afb8" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="dU"/><g id="h"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dS"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,11.897119341563785,0)" xlink:href="#dR"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,15.25380658436214,0)" xlink:href="#dT"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,27.958230452674897,0)" xlink:href="#dU"/></g><path fill="#a9afb8" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="dV"/><path fill="#a9afb8" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="dW"/><path fill="#a9afb8" d="M9 0v-24l116-142H16v-24h144v24L44-24h123V0H9" id="dX"/><g id="i"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dN"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,12.704423868312757,0)" xlink:href="#dL"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,21.20236625514403,0)" xlink:href="#dV"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,26.25864197530864,0)" xlink:href="#dW"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,30.50761316872428,0)" xlink:href="#dO"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,33.864300411522635,0)" xlink:href="#dM"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,42.36224279835391,0)" xlink:href="#dQ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,50.86018518518518,0)" xlink:href="#dX"/></g><path fill="#a9afb8" d="M32 76v-337h29V76H32" id="dY"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dY" id="j"/><path fill="#a9afb8" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="dZ"/><path fill="#a9afb8" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="ea"/><g id="k"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#dS"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,11.897119341563785,0)" xlink:href="#dZ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,19.545267489711932,0)" xlink:href="#dW"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,23.79423868312757,0)" xlink:href="#dU"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,32.29218106995884,0)" xlink:href="#ea"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,40.79012345679012,0)" xlink:href="#dQ"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,49.288065843621396,0)" xlink:href="#dV"/></g><path fill="#a9afb8" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="eb"/><path fill="#a9afb8" d="M155-56V0h-30v-56H8v-25l114-167h33v167h35v25h-35zm-30-156c-27 46-58 90-88 131h88v-131" id="ec"/><path fill="#a9afb8" d="M68-38c1 34 0 65-14 84H32c9-13 17-26 17-46H33v-38h35" id="ed"/><g id="l"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#eb"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,8.497942386831275,0)" xlink:href="#ec"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.99588477366255,0)" xlink:href="#ed"/></g><path fill="#a9afb8" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="ee"/><path fill="#a9afb8" d="M101-251c68 0 85 55 85 127S166 4 100 4C33 4 14-52 14-124c0-73 17-127 87-127zm-1 229c47 0 54-49 54-102s-4-102-53-102c-51 0-55 48-55 102 0 53 5 102 54 102" id="ef"/><g id="m"><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,0,0)" xlink:href="#ee"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,8.497942386831275,0)" xlink:href="#ef"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,16.99588477366255,0)" xlink:href="#ee"/><use transform="matrix(0.04248971193415638,0,0,0.04248971193415638,25.493827160493822,0)" xlink:href="#ef"/></g><path fill="#333" d="M185-189c-5-48-123-54-124 2 14 75 158 14 163 119 3 78-121 87-175 55-17-10-28-26-33-46l33-7c5 56 141 63 141-1 0-78-155-14-162-118-5-82 145-84 179-34 5 7 8 16 11 25" id="eg"/><path fill="#333" d="M30 0v-248h187v28H63v79h144v27H63v87h162V0H30" id="eh"/><path fill="#333" d="M30 0v-248h33v221h125V0H30" id="ei"/><path fill="#333" d="M212-179c-10-28-35-45-73-45-59 0-87 40-87 99 0 60 29 101 89 101 43 0 62-24 78-52l27 14C228-24 195 4 139 4 59 4 22-46 18-125c-6-104 99-153 187-111 19 9 31 26 39 46" id="ej"/><path fill="#333" d="M127-220V0H93v-220H8v-28h204v28h-85" id="ek"/><path fill="#333" d="M30-248c118-7 216 8 213 122C240-48 200 0 122 0H30v-248zM63-27c89 8 146-16 146-99s-60-101-146-95v194" id="el"/><g id="n"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#ei"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,46,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,58.95,0)" xlink:href="#ek"/><use transform="matrix(0.05,0,0,0.05,69.9,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,81.9,0)" xlink:href="#el"/></g><path fill="#333" d="M63-220v92h138v28H63V0H30v-248h175v28H63" id="em"/><path fill="#333" d="M140-251c81 0 123 46 123 126C263-46 219 4 140 4 59 4 17-45 17-125s42-126 123-126zm0 227c63 0 89-41 89-101s-29-99-89-99c-61 0-89 39-89 99S79-25 140-24" id="en"/><path fill="#333" d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="eo"/><g id="o"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#em"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,24.950000000000003,0)" xlink:href="#eo"/></g><path fill="#333" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="ep"/><path fill="#333" d="M205 0l-28-72H64L36 0H1l101-248h38L239 0h-34zm-38-99l-47-123c-12 45-31 82-46 123h93" id="eq"/><path fill="#333" d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="er"/><g id="p"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,10.65,0)" xlink:href="#eq"/><use transform="matrix(0.05,0,0,0.05,21.3,0)" xlink:href="#ek"/><use transform="matrix(0.05,0,0,0.05,32.25,0)" xlink:href="#er"/></g><path fill="#333" d="M137-103V0h-34v-103L8-248h37l75 118 75-118h37" id="es"/><g id="q"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ek"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#es"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#eh"/></g><path fill="#333" d="M240 0l2-218c-23 76-54 145-80 218h-23L58-218 59 0H30v-248h44l77 211c21-75 51-140 76-211h43V0h-30" id="et"/><g id="r"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#et"/><use transform="matrix(0.05,0,0,0.05,14.950000000000001,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,28.950000000000003,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#ei"/></g><path fill="#333" d="M33 0v-248h34V0H33" id="eu"/><g id="s"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#el"/></g><path fill="#333" d="M209 0H11v-25l151-195H24v-28h176v25L50-27h159V0" id="ev"/><g id="t"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#ev"/><use transform="matrix(0.05,0,0,0.05,27.950000000000003,0)" xlink:href="#eh"/></g><path fill="#333" d="M33 0v-38h34V0H33" id="ew"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ew" id="u"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ew" id="v"/><path fill="#333" d="M0 4l72-265h28L28 4H0" id="ex"/><path fill="#333" d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="ey"/><path fill="#333" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="ez"/><path fill="#333" d="M108 0H70L1-190h34L89-25l56-165h34" id="eA"/><path fill="#333" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="eB"/><path fill="#333" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="eC"/><g id="w"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#eC"/></g><path fill="#333" d="M106-169C34-169 62-67 57 0H25v-261h32l-1 103c12-21 28-36 61-36 89 0 53 116 60 194h-32v-121c2-32-8-49-39-48" id="eD"/><g id="x"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ey"/></g><path fill="#333" d="M160-131c35 5 61 23 61 61C221 17 115-2 30 0v-248c76 3 177-17 177 60 0 33-19 50-47 57zm-97-11c50-1 110 9 110-42 0-47-63-36-110-37v79zm0 115c55-2 124 14 124-45 0-56-70-42-124-44v89" id="eE"/><path fill="#333" d="M155-56V0h-30v-56H8v-25l114-167h33v167h35v25h-35zm-30-156c-27 46-58 90-88 131h88v-131" id="eF"/><path fill="#333" d="M54-142c48-35 137-8 131 61C196 18 31 33 14-55l32-4c7 23 22 37 52 37 35-1 51-22 54-58 4-55-73-65-99-34H22l8-134h141v27H59" id="eG"/><path fill="#333" d="M110-160c48 1 74 30 74 79 0 53-28 85-80 85-65 0-83-55-86-122-5-90 50-162 133-122 14 7 22 21 27 39l-31 6c-5-40-67-38-82-6-9 19-15 44-15 74 11-20 30-34 60-33zm-7 138c34 0 49-23 49-58s-16-56-50-56c-29 0-50 16-49 49 1 36 15 65 50 65" id="eH"/><g id="y"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eE"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#eH"/></g><path fill="#333" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="eI"/><path fill="#333" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="eJ"/><path fill="#333" d="M126-127c33 6 58 20 58 59 0 88-139 92-164 29-3-8-5-16-6-25l32-3c6 27 21 44 54 44 32 0 52-15 52-46 0-38-36-46-79-43v-28c39 1 72-4 72-42 0-27-17-43-46-43-28 0-47 15-49 41l-32-3c6-42 35-63 81-64 48-1 79 21 79 65 0 36-21 52-52 59" id="eK"/><g id="z"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eF"/></g><path fill="#333" d="M134-131c28 9 52 24 51 62-1 50-34 73-85 73S17-19 16-69c0-36 21-54 49-61-75-25-45-126 34-121 46 3 78 18 79 63 0 33-17 51-44 57zm-34-11c31 1 46-15 46-44 0-28-17-43-47-42-29 0-46 13-45 42 1 28 16 44 46 44zm1 122c35 0 51-18 51-52 0-30-18-46-53-46-33 0-51 17-51 47 0 34 19 51 53 51" id="eL"/><path fill="#333" d="M101-251c68 0 85 55 85 127S166 4 100 4C33 4 14-52 14-124c0-73 17-127 87-127zm-1 229c47 0 54-49 54-102s-4-102-53-102c-51 0-55 48-55 102 0 53 5 102 54 102" id="eM"/><g id="A"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eM"/></g><path fill="#333" d="M143 4C61 4 22-44 18-125c-5-107 100-154 193-111 17 8 29 25 37 43l-32 9c-13-25-37-40-76-40-61 0-88 39-88 99 0 61 29 100 91 101 35 0 62-11 79-27v-45h-74v-28h105v86C228-13 192 4 143 4" id="eN"/><g id="B"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eN"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eC"/></g><path fill="#333" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="eO"/><g id="C"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#eO"/></g><g id="D"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eG"/></g><g id="E"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#ey"/></g><g id="F"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eH"/></g><g id="G"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#ez"/></g><path fill="#333" d="M64 0c3-98 48-159 88-221H18v-27h164v26C143-157 98-101 97 0H64" id="eP"/><g id="H"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eP"/></g><path fill="#333" d="M101-234c-31-9-42 10-38 44h38v23H63V0H32v-167H5v-23h27c-7-52 17-82 69-68v24" id="eQ"/><g id="I"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#eQ"/></g><g id="J"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#ey"/></g><g id="K"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eM"/></g><g id="L"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eL"/></g><g id="M"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eM"/></g><path fill="#333" d="M177-190C167-65 218 103 67 71c-23-6-38-20-44-43l32-5c15 47 100 32 89-28v-30C133-14 115 1 83 1 29 1 15-40 15-95c0-56 16-97 71-98 29-1 48 16 59 35 1-10 0-23 2-32h30zM94-22c36 0 50-32 50-73 0-42-14-75-50-75-39 0-46 34-46 75s6 73 46 73" id="eR"/><g id="N"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#eR"/></g><path fill="#333" d="M99-251c64 0 84 50 84 122C183-37 130 33 47-8c-14-7-20-23-25-40l30-5c6 39 69 39 84 7 9-19 16-44 16-74-10 22-31 35-62 35-49 0-73-33-73-83 0-54 28-83 82-83zm-1 141c31-1 51-18 51-49 0-36-14-67-51-67-34 0-49 23-49 58 0 34 15 58 49 58" id="eS"/><g id="O"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eS"/></g><g id="P"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,58,0)" xlink:href="#eD"/></g><path fill="#333" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="eT"/><path fill="#333" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="eU"/><g id="Q"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#ez"/></g><path fill="#333" d="M190 0L58-211 59 0H30v-248h39L202-35l-2-213h31V0h-41" id="eV"/><g id="R"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eV"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eM"/></g><g id="S"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#eM"/></g><g id="T"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eH"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eM"/></g><path fill="#333" d="M24 0v-261h32V0H24" id="eW"/><path fill="#333" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="eX"/><g id="U"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,44.95,0)" xlink:href="#eX"/></g><path fill="#333" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="eY"/><g id="V"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,18.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,22.9,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,32.9,0)" xlink:href="#eB"/></g><path fill="#333" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="eZ"/><g id="W"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eZ"/></g><path fill="#333" d="M206 0h-36l-40-164L89 0H53L-1-190h32L70-26l43-164h34l41 164 42-164h31" id="fa"/><g id="X"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#fa"/></g><g id="Y"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#fa"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eP" id="Z"/><path fill="#333" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="fb"/><g id="aa"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#eB"/></g><g id="ab"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/></g><path fill="#333" d="M18-150v-26h174v26H18zm0 90v-26h174v26H18" id="fc"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fc" id="ac"/><path fill="#333" d="M109-170H84l-4-78h32zm-65 0H19l-4-78h33" id="fd"/><path fill="#333" d="M80-196l47-18 7 23-49 13 32 44-20 13-27-46-27 45-21-12 33-44-49-13 8-23 47 19-2-53h23" id="fe"/><g id="ad"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fd"/><use transform="matrix(0.05,0,0,0.05,6.3500000000000005,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,16.35,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,26.35,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,35.35,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,40.35,0)" xlink:href="#fe"/><use transform="matrix(0.05,0,0,0.05,47.35,0)" xlink:href="#fd"/></g><path fill="#333" d="M169-182c-1-43-94-46-97-3 18 66 151 10 154 114 3 95-165 93-204 36-6-8-10-19-12-30l50-8c3 46 112 56 116 5-17-69-150-10-154-114-4-87 153-88 188-35 5 8 8 18 10 28" id="ff"/><path fill="#333" d="M185-48c-13 30-37 53-82 52C43 2 14-33 14-96s30-98 90-98c62 0 83 45 84 108H66c0 31 8 55 39 56 18 0 30-7 34-22zm-45-69c5-46-57-63-70-21-2 6-4 13-4 21h74" id="fg"/><path fill="#333" d="M25 0v-261h50V0H25" id="fh"/><path fill="#333" d="M190-63c-7 42-38 67-86 67-59 0-84-38-90-98-12-110 154-137 174-36l-49 2c-2-19-15-32-35-32-30 0-35 28-38 64-6 74 65 87 74 30" id="fi"/><path fill="#333" d="M115-3C79 11 28 4 28-45v-112H4v-33h27l15-45h31v45h36v33H77v99c-1 23 16 31 38 25v30" id="fj"/><g id="ae"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,37,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,47,0)" xlink:href="#fj"/></g><path fill="#333" d="M114-157C55-157 80-60 75 0H25v-261h50l-1 109c12-26 28-41 61-42 86-1 58 113 63 194h-50c-7-57 23-157-34-157" id="fk"/><g id="af"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#fg"/></g><path fill="#333" d="M110-194c64 0 96 36 96 99 0 64-35 99-97 99-61 0-95-36-95-99 0-62 34-99 96-99zm-1 164c35 0 45-28 45-65 0-40-10-65-43-65-34 0-45 26-45 65 0 36 10 65 43 65" id="fl"/><path fill="#333" d="M137-138c1-29-70-34-71-4 15 46 118 7 119 86 1 83-164 76-172 9l43-7c4 19 20 25 44 25 33 8 57-30 24-41C81-84 22-81 20-136c-2-80 154-74 161-7" id="fm"/><g id="ag"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fj"/></g><g id="ah"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#fl"/></g><path fill="#333" d="M135-194c52 0 70 43 70 98 0 56-19 99-73 100-30 1-46-15-58-35L72 0H24l1-261h50v104c11-23 29-37 60-37zM114-30c31 0 40-27 40-66 0-37-7-63-39-63s-41 28-41 65c0 36 8 64 40 64" id="fn"/><g id="ai"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fn"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fg"/></g><path fill="#333" d="M85 4C-2 5 27-109 22-190h50c7 57-23 150 33 157 60-5 35-97 40-157h50l1 190h-47c-2-12 1-28-3-38-12 25-28 42-61 42" id="fo"/><path fill="#333" d="M88-194c31-1 46 15 58 34l-1-101h50l1 261h-48c-2-10 0-23-3-31C134-8 116 4 84 4 32 4 16-41 15-95c0-56 19-97 73-99zm17 164c33 0 40-30 41-66 1-37-9-64-41-64s-38 30-39 65c0 43 13 65 39 65" id="fp"/><g id="aj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#fp"/></g><path fill="#333" d="M133-34C117-15 103 5 69 4 32 3 11-16 11-54c-1-60 55-63 116-61 1-26-3-47-28-47-18 1-26 9-28 27l-52-2c7-38 36-58 82-57s74 22 75 68l1 82c-1 14 12 18 25 15v27c-30 8-71 5-69-32zm-48 3c29 0 43-24 42-57-32 0-66-3-65 30 0 17 8 27 23 27" id="fq"/><g id="ak"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fm"/></g><path fill="#333" d="M220-157c-53 9-28 100-34 157h-49v-107c1-27-5-49-29-50C55-147 81-57 75 0H25l-1-190h47c2 12-1 28 3 38 10-53 101-56 108 0 13-22 24-43 59-42 82 1 51 116 57 194h-49v-107c-1-25-5-48-29-50" id="fr"/><g id="al"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fr"/><use transform="matrix(0.05,0,0,0.05,16,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,26.950000000000003,0)" xlink:href="#fp"/><use transform="matrix(0.05,0,0,0.05,37.900000000000006,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,47.900000000000006,0)" xlink:href="#fh"/></g><path fill="#333" d="M25-224v-37h50v37H25zM25 0v-190h50V0H25" id="fs"/><g id="am"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#fm"/></g><path fill="#333" d="M140-251c80 0 125 45 125 126S219 4 139 4C58 4 15-44 15-125s44-126 125-126zm-1 214c52 0 73-35 73-88 0-50-21-86-72-86-52 0-73 35-73 86s22 88 72 88" id="ft"/><path fill="#333" d="M24-248c120-7 223 5 221 122C244-46 201 0 124 0H24v-248zM76-40c74 7 117-18 117-86 0-67-45-88-117-82v168" id="fu"/><g id="an"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ft"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#fu"/></g><path fill="#333" d="M135-150c-39-12-60 13-60 57V0H25l-1-190h47c2 13-1 29 3 40 6-28 27-53 61-41v41" id="fv"/><path fill="#333" d="M135-194c87-1 58 113 63 194h-50c-7-57 23-157-34-157-59 0-34 97-39 157H25l-1-190h47c2 12-1 28 3 38 12-26 28-41 61-42" id="fw"/><g id="ao"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,37,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,42.95,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,58.900000000000006,0)" xlink:href="#fw"/></g><path fill="#333" d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="fx"/><path fill="#333" d="M179-190L93 31C79 59 56 82 12 73V49c39 6 53-20 64-50L1-190h34L92-34l54-156h33" id="fy"/><g id="ap"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eq"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#fy"/></g><g id="aq"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,12.899999999999999,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,17.9,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,27.899999999999995,0)" xlink:href="#fb"/></g><path fill="#333" d="M136-208V0H84v-208H4v-40h212v40h-80" id="fz"/><g id="ar"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fz"/><use transform="matrix(0.05,0,0,0.05,9.600000000000001,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,19.6,0)" xlink:href="#fn"/><use transform="matrix(0.05,0,0,0.05,30.55,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,35.550000000000004,0)" xlink:href="#fg"/></g><path fill="#333" d="M121-226c-27-7-43 5-38 36h38v33H83V0H34v-157H6v-33h28c-9-59 32-81 87-68v32" id="fA"/><path fill="#333" d="M195-6C206 82 75 100 31 46c-4-6-6-13-8-21l49-6c3 16 16 24 34 25 40 0 42-37 40-79-11 22-30 35-61 35-53 0-70-43-70-97 0-56 18-96 73-97 30 0 46 14 59 34l2-30h47zm-90-29c32 0 41-27 41-63 0-35-9-62-40-62-32 0-39 29-40 63 0 36 9 62 39 62" id="fB"/><path fill="#333" d="M35-132v-50h50v50H35zM35 0v-49h50V0H35" id="fC"/><g id="as"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,31.900000000000006,0)" xlink:href="#fA"/><use transform="matrix(0.05,0,0,0.05,37.85000000000001,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,42.85000000000001,0)" xlink:href="#fB"/><use transform="matrix(0.05,0,0,0.05,53.80000000000001,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,64.75000000000001,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,71.75000000000001,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,81.75000000000001,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,87.70000000000002,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,92.70000000000002,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,103.65000000000003,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,114.60000000000002,0)" xlink:href="#fC"/></g><path fill="#333" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="fD"/><g id="at"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,36.900000000000006,0)" xlink:href="#eT"/></g><g id="au"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ej"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#ez"/></g><path fill="#333" d="M47-170H22l-4-78h33" id="fE"/><g id="av"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#fE"/><use transform="matrix(0.05,0,0,0.05,42.35,0)" xlink:href="#eB"/></g><path fill="#333" d="M16-82v-28h88v28H16" id="fF"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fF" id="aw"/><g id="ax"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,51.95,0)" xlink:href="#eB"/></g><g id="ay"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,56.95,0)" xlink:href="#ey"/></g><g id="az"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#eT"/></g><g id="aA"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eD"/></g><g id="aB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#ew"/></g><g id="aC"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eI"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eM"/></g><g id="aD"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eB"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fF" id="aE"/><g id="aF"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#fa"/></g><g id="aG"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,48,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,51.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,56.95,0)" xlink:href="#fy"/></g><g id="aH"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eF"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,45,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,55,0)" xlink:href="#eM"/></g><g id="aI"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eN"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eE"/></g><g id="aJ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eG"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eH"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eM"/></g><g id="aK"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#fE"/><use transform="matrix(0.05,0,0,0.05,42.35,0)" xlink:href="#eB"/></g><g id="aL"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,20.85,0)" xlink:href="#eW"/></g><g id="aM"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/></g><g id="aN"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,49.95,0)" xlink:href="#ey"/></g><path fill="#333" d="M123 10C108 53 80 86 19 72V37c35 8 53-11 59-39L3-190h52l48 148c12-52 28-100 44-148h51" id="fG"/><g id="aO"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#fr"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#fr"/><use transform="matrix(0.05,0,0,0.05,54.95,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,64.95,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,71.95,0)" xlink:href="#fG"/><use transform="matrix(0.05,0,0,0.05,81.95,0)" xlink:href="#fC"/></g><path fill="#333" d="M67-125c0 53 21 87 73 88 37 1 54-22 65-47l45 17C233-25 199 4 140 4 58 4 20-42 15-125 8-235 124-281 211-232c18 10 29 29 36 50l-46 12c-8-25-30-41-62-41-52 0-71 34-72 86" id="fH"/><g id="aP"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fH"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,17.950000000000003,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,28.900000000000006,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,44.85000000000001,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#fv"/></g><path fill="#333" d="M135-194c53 0 70 44 70 98 0 56-19 98-73 100-31 1-45-17-59-34 3 33 2 69 2 105H25l-1-265h48c2 10 0 23 3 31 11-24 29-35 60-35zM114-30c33 0 39-31 40-66 0-38-9-64-40-64-56 0-55 130 0 130" id="fI"/><g id="aQ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fI"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#fG"/></g><path fill="#333" d="M67-93c0 74 22 123 53 168H70C40 30 18-18 18-93s22-123 52-168h50c-32 44-53 94-53 168" id="fJ"/><g id="aR"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fJ"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ft"/><use transform="matrix(0.05,0,0,0.05,19.95,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#fu"/></g><path fill="#333" d="M102-93c0 74-22 123-52 168H0C30 29 54-18 53-93c0-74-22-123-53-168h50c30 45 52 94 52 168" id="fK"/><g id="aS"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,37,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,42.95,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,58.900000000000006,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,69.85000000000001,0)" xlink:href="#fK"/><use transform="matrix(0.05,0,0,0.05,75.80000000000001,0)" xlink:href="#fC"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eJ" id="aT"/><g id="aU"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ff"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,42.95,0)" xlink:href="#fp"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,64.85000000000001,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,69.85000000000001,0)" xlink:href="#fg"/></g><g id="aV"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fI"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,48.900000000000006,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,59.85,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,70.80000000000001,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,81.75000000000001,0)" xlink:href="#fC"/></g><path fill="#333" d="M33-154v-36h34v36H33zM33 0v-36h34V0H33" id="fL"/><g id="aW"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eK"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fL"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#eM"/></g><g id="aX"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eU"/></g><g id="aY"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eS"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,50,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,60,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,70,0)" xlink:href="#eM"/></g><g id="aZ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,17,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,27,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eX"/></g><g id="ba"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#el"/></g><g id="bb"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eT"/></g><g id="bc"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eX"/></g><g id="bd"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fp"/></g><g id="be"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,37.85,0)" xlink:href="#fm"/></g><g id="bf"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,41.9,0)" xlink:href="#fC"/></g><path fill="#333" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="fM"/><g id="bg"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#eO"/></g><path fill="#333" d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="fN"/><g id="bh"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,37.95,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#fN"/></g><g id="bi"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,52.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,56.900000000000006,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,66.9,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,76.9,0)" xlink:href="#eX"/></g><g id="bj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#eB"/></g><g id="bk"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#eB"/></g><g id="bl"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eT"/></g><path fill="#333" d="M252-156c43 0 55 33 55 80 0 46-13 78-56 78-42 0-55-33-55-78 0-48 12-80 56-80zM93 0H65l162-248h28zM13-171c-1-47 13-79 56-79s55 33 55 79-13 79-55 79c-43 0-56-33-56-79zM251-17c28 0 30-29 30-59 0-31-1-60-29-60-29 0-31 29-31 60 0 29 2 59 30 59zM69-112c27 0 28-30 29-59 0-31-1-60-29-60-29 0-30 30-30 60s2 59 30 59" id="fO"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fO" id="bm"/><g id="bn"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eP"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eM"/></g><g id="bo"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#er"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#eX"/></g><g id="bp"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#fM"/></g><g id="bq"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#ez"/></g><g id="br"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eL"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eM"/></g><g id="bs"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eT"/></g><g id="bt"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,28,0)" xlink:href="#ez"/></g><g id="bu"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eQ"/></g><g id="bv"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,43.95,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,53.95,0)" xlink:href="#eR"/></g><g id="bw"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,63.900000000000006,0)" xlink:href="#eB"/></g><g id="bx"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,43.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,53.95,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,63.95,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,73.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,77.9,0)" xlink:href="#ez"/></g><g id="by"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#fb"/></g><path fill="#333" d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="fP"/><g id="bz"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fP"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/></g><g id="bA"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fy"/></g><g id="bB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eh"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,21,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,29.950000000000003,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,44.9,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,59.900000000000006,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,69.9,0)" xlink:href="#ey"/></g><g id="bC"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,44.900000000000006,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#eT"/></g><g id="bD"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eJ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eG"/></g><g id="bE"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,14.950000000000001,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eB"/></g><g id="bF"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eK"/></g><g id="bG"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,27.950000000000003,0)" xlink:href="#fD"/></g><g id="bH"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eE"/></g><path fill="#333" d="M266 0h-40l-56-210L115 0H75L2-248h35L96-30l15-64 43-154h32l59 218 59-218h35" id="fQ"/><g id="bI"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fQ"/><use transform="matrix(0.05,0,0,0.05,16.3,0)" xlink:href="#eq"/><use transform="matrix(0.05,0,0,0.05,28.3,0)" xlink:href="#ei"/></g><path fill="#333" d="M24 0v-248h52v208h133V0H24" id="fR"/><g id="bJ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fR"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,15.950000000000001,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fj"/></g><g id="bK"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fA"/></g><g id="bL"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,51.900000000000006,0)" xlink:href="#fv"/></g><path fill="#333" d="M231 0h-52l-39-155L100 0H48L-1-190h46L77-45c9-52 24-97 36-145h53l37 145 32-145h46" id="fS"/><g id="bM"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fS"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,24.950000000000003,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#fg"/></g><g id="bN"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fm"/></g><g id="bO"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,31.900000000000006,0)" xlink:href="#fA"/><use transform="matrix(0.05,0,0,0.05,37.85000000000001,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,42.85000000000001,0)" xlink:href="#fB"/><use transform="matrix(0.05,0,0,0.05,53.80000000000001,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,64.75000000000001,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,71.75000000000001,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,81.75000000000001,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,87.70000000000002,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,92.70000000000002,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,103.65000000000003,0)" xlink:href="#fw"/></g><g id="bP"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fI"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,41.900000000000006,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,46.900000000000006,0)" xlink:href="#fn"/><use transform="matrix(0.05,0,0,0.05,57.85,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,62.85,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,72.85000000000001,0)" xlink:href="#fC"/></g><path fill="#333" d="M52 0c1-96 47-148 87-207H15v-41h169v40c-36 62-79 113-81 208H52" id="fT"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fT" id="bQ"/><path fill="#333" d="M128 0H69L1-190h53L99-40l48-150h52" id="fU"/><g id="bR"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fU"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,50,0)" xlink:href="#fn"/><use transform="matrix(0.05,0,0,0.05,60.95,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,65.95,0)" xlink:href="#fg"/></g><g id="bS"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fp"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fU"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,45.95,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,55.95,0)" xlink:href="#fm"/></g><g id="bT"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fA"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#fo"/><use transform="matrix(0.05,0,0,0.05,27.85,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,38.800000000000004,0)" xlink:href="#fp"/></g><g id="bU"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#fw"/></g><g id="bV"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,44.95,0)" xlink:href="#eX"/></g><g id="bW"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#er"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#eX"/></g><g id="bX"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#ez"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ew" id="bY"/><g id="bZ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eI"/></g><g id="ca"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eJ"/></g><g id="cb"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eK"/></g><g id="cc"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#ew"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ew"/></g><g id="cd"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eo"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,27.950000000000003,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,33.900000000000006,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,43.900000000000006,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,52.900000000000006,0)" xlink:href="#eD"/></g><g id="ce"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eq"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eW"/></g><g id="cf"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,41.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,50.900000000000006,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,60.900000000000006,0)" xlink:href="#eT"/></g><path fill="#333" d="M23 0v-37h61v-169l-59 37v-38l62-41h46v211h57V0H23" id="fV"/><path fill="#333" d="M165-50V0h-47v-50H5v-38l105-160h55v161h33v37h-33zm-47-37l2-116L46-87h72" id="fW"/><path fill="#333" d="M101-251c68 0 84 54 84 127C185-50 166 4 99 4S15-52 14-124c-1-75 17-127 87-127zm-1 216c37-5 36-46 36-89s4-89-36-89c-39 0-36 45-36 89 0 43-3 85 36 89" id="fX"/><g id="cg"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fV"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fW"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fX"/></g><g id="ch"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fk"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,31.900000000000002,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,37.85,0)" xlink:href="#fm"/></g><g id="ci"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,45,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,50.95,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,60.95,0)" xlink:href="#fp"/></g><g id="cj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,43.85000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,53.85,0)" xlink:href="#eW"/></g><path fill="#333" d="M87 75C49 33 22-17 22-94c0-76 28-126 65-167h31c-38 41-64 92-64 168S80 34 118 75H87" id="fY"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fY" id="ck"/><g id="cl"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#eX"/></g><g id="cm"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eX"/></g><g id="cn"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eT"/></g><g id="co"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,14.65,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,19.65,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,29.649999999999995,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,38.65,0)" xlink:href="#eX"/></g><g id="cp"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,46.900000000000006,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,61.85,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,71.85000000000001,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,81.85000000000001,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,90.85000000000001,0)" xlink:href="#ez"/></g><path fill="#333" d="M33-261c38 41 65 92 65 168S71 34 33 75H2C39 34 66-17 66-93S39-220 2-261h31" id="fZ"/><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fZ" id="cq"/><path fill="#333" d="M232-93c-1 65-40 97-104 97C67 4 28-28 28-90v-158h33c8 89-33 224 67 224 102 0 64-133 71-224h33v155" id="ga"/><g id="cr"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ga"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,21.950000000000003,0)" xlink:href="#ez"/></g><path fill="#333" d="M145-31C134-9 116 4 85 4 32 4 16-35 15-94c0-59 17-99 70-100 32-1 48 14 60 33 0-11-1-24 2-32h30l-1 268h-32zM93-21c41 0 51-33 51-76s-8-73-50-73c-40 0-46 35-46 75s5 74 45 74" id="gb"/><g id="cs"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#gb"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,39,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,49,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,59,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,64,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,67.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,77.95,0)" xlink:href="#eW"/></g><g id="ct"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,45.95,0)" xlink:href="#eB"/></g><g id="cu"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eQ"/></g><g id="cv"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eB"/></g><g id="cw"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eX"/></g><g id="cx"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,33.95,0)" xlink:href="#ez"/></g><g id="cy"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,23.9,0)" xlink:href="#ez"/></g><g id="cz"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,12.949999999999998,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#fa"/></g><g id="cA"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,44.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,54.95,0)" xlink:href="#eB"/></g><g id="cB"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,3.95,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,18.900000000000002,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,28.900000000000002,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,38.900000000000006,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,47.900000000000006,0)" xlink:href="#eX"/></g><g id="cC"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#eW"/></g><g id="cD"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#ez"/></g><g id="cE"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/></g><path fill="#333" d="M68-38c1 34 0 65-14 84H32c9-13 17-26 17-46H33v-38h35" id="gc"/><g id="cF"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,43.85000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,53.85,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,57.80000000000001,0)" xlink:href="#gc"/></g><g id="cG"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#ez"/></g><g id="cH"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,18.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,22.9,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,41.9,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,51.900000000000006,0)" xlink:href="#ey"/></g><g id="cI"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,19,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,37.95,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,47.95,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,57.95,0)" xlink:href="#ez"/></g><g id="cJ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,16.900000000000002,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,21.900000000000002,0)" xlink:href="#eD"/></g><g id="cK"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,35,0)" xlink:href="#fb"/></g><g id="cL"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#eB"/></g><path fill="#333" d="M24 0v-248h195v40H76v63h132v40H76v65h150V0H24" id="gd"/><g id="cM"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fJ"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#gd"/><use transform="matrix(0.05,0,0,0.05,17.95,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,38.9,0)" xlink:href="#fn"/><use transform="matrix(0.05,0,0,0.05,49.85,0)" xlink:href="#fh"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,59.85,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,70.8,0)" xlink:href="#fB"/></g><g id="cN"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fw"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#fg"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#fS"/></g><g id="cO"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fm"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#fl"/><use transform="matrix(0.05,0,0,0.05,26.899999999999995,0)" xlink:href="#fv"/><use transform="matrix(0.05,0,0,0.05,33.9,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,43.9,0)" xlink:href="#fB"/><use transform="matrix(0.05,0,0,0.05,54.85,0)" xlink:href="#fg"/></g><g id="cP"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fI"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#fq"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#fi"/><use transform="matrix(0.05,0,0,0.05,50.95,0)" xlink:href="#fs"/><use transform="matrix(0.05,0,0,0.05,55.95,0)" xlink:href="#fj"/><use transform="matrix(0.05,0,0,0.05,61.900000000000006,0)" xlink:href="#fG"/><use transform="matrix(0.05,0,0,0.05,71.9,0)" xlink:href="#fK"/><use transform="matrix(0.05,0,0,0.05,77.85000000000001,0)" xlink:href="#fC"/></g><g id="cQ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ep"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,22,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,32,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,41,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,51,0)" xlink:href="#ey"/></g><g id="cR"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eg"/><use transform="matrix(0.05,0,0,0.05,26,0)" xlink:href="#el"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#fL"/></g><g id="cS"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,30,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,40,0)" xlink:href="#ey"/></g><g id="cT"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ez"/></g><g id="cU"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,12.950000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,26.900000000000002,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,36.900000000000006,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,46.900000000000006,0)" xlink:href="#eX"/></g><g id="cV"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eM"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ew"/></g><g id="cW"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#en"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,24,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,33,0)" xlink:href="#ez"/></g><g id="cX"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,14.949999999999998,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,24.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,34.95,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,39.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,49.95,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,59.95,0)" xlink:href="#ew"/></g><g id="cY"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ek"/><use transform="matrix(0.05,0,0,0.05,10.950000000000001,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,20.950000000000003,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#fy"/></g><g id="cZ"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,38.9,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,42.85,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,52.85,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,62.85,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,67.85,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,77.85,0)" xlink:href="#ey"/></g><g id="da"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,15,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,20.95,0)" xlink:href="#eT"/></g><g id="db"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,19.9,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,28.899999999999995,0)" xlink:href="#ez"/></g><g id="dc"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,35.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,39.900000000000006,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,49.900000000000006,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,59.900000000000006,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,68.9,0)" xlink:href="#ez"/></g><g id="dd"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eC"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fy"/></g><g id="de"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,29,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,34,0)" xlink:href="#fL"/></g><g id="df"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eT"/></g><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD" id="dg"/><g id="dh"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,13.949999999999998,0)" xlink:href="#eA"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#eT"/></g><g id="di"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,8.95,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,23.9,0)" xlink:href="#ez"/></g><g id="dj"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eu"/><use transform="matrix(0.05,0,0,0.05,5,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,19.950000000000003,0)" xlink:href="#eU"/><use transform="matrix(0.05,0,0,0.05,34.900000000000006,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,44.900000000000006,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,58.85,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,68.85000000000001,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,73.85000000000001,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,83.85000000000001,0)" xlink:href="#fL"/></g><g id="dk"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eE"/><use transform="matrix(0.05,0,0,0.05,12,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,17.95,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,21.9,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,31.899999999999995,0)" xlink:href="#eR"/></g><g id="dl"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#ex"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,28.950000000000003,0)" xlink:href="#eT"/></g><g id="dm"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,14,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,19.95,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,29.950000000000003,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,33.9,0)" xlink:href="#eR"/><use transform="matrix(0.05,0,0,0.05,43.9,0)" xlink:href="#eD"/><use transform="matrix(0.05,0,0,0.05,53.900000000000006,0)" xlink:href="#eX"/></g><g id="dn"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fa"/><use transform="matrix(0.05,0,0,0.05,22.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,32.95,0)" xlink:href="#fy"/></g><g id="do"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#fY"/><use transform="matrix(0.05,0,0,0.05,5.95,0)" xlink:href="#ey"/><use transform="matrix(0.05,0,0,0.05,15.949999999999998,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,25.950000000000003,0)" xlink:href="#eQ"/><use transform="matrix(0.05,0,0,0.05,30.950000000000003,0)" xlink:href="#fD"/><use transform="matrix(0.05,0,0,0.05,40.95,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,50.95,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,54.900000000000006,0)" xlink:href="#eX"/></g><g id="dp"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#fx"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,25,0)" xlink:href="#eY"/><use transform="matrix(0.05,0,0,0.05,28.950000000000003,0)" xlink:href="#eZ"/><use transform="matrix(0.05,0,0,0.05,38.95,0)" xlink:href="#eT"/></g><g id="dq"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eT"/><use transform="matrix(0.05,0,0,0.05,10,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,20,0)" xlink:href="#fa"/></g><g id="dr"><use transform="matrix(0.05,0,0,0.05,0,0)" xlink:href="#eO"/><use transform="matrix(0.05,0,0,0.05,9,0)" xlink:href="#eW"/><use transform="matrix(0.05,0,0,0.05,12.949999999999998,0)" xlink:href="#fM"/><use transform="matrix(0.05,0,0,0.05,22.95,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,31.950000000000003,0)" xlink:href="#eX"/><use transform="matrix(0.05,0,0,0.05,36.95,0)" xlink:href="#ez"/><use transform="matrix(0.05,0,0,0.05,46.95,0)" xlink:href="#fb"/><use transform="matrix(0.05,0,0,0.05,52.900000000000006,0)" xlink:href="#eB"/><use transform="matrix(0.05,0,0,0.05,61.900000000000006,0)" xlink:href="#fZ"/></g></defs></g></svg>
\ No newline at end of file
diff --git a/doc/dev/cephadm/design/storage_devices_and_osds.rst b/doc/dev/cephadm/design/storage_devices_and_osds.rst
new file mode 100644
index 000000000..bfe6daabb
--- /dev/null
+++ b/doc/dev/cephadm/design/storage_devices_and_osds.rst
@@ -0,0 +1,317 @@
+==============================================
+Storage Devices and OSDs Management Workflows
+==============================================
+The cluster storage devices are the physical storage devices installed in each of the cluster’s hosts. We need to execute different operations over them and also to retrieve information about physical features and working behavior.
+The basic use cases we have in this area are:
+
+- `1. Retrieve device information. Inventory`_
+- `2. Add OSDs`_
+- `3. Remove OSDs`_
+- `4. Replace OSDs`_
+
+1. Retrieve device information. Inventory
+=========================================
+We must be able to review what is the current state and condition of the cluster storage devices. We need the identification and features detail (including ident/fault led on/off capable) and if the device is used or not as an OSD/DB/WAL device.
+
+The information required for each device will be at least:
+::
+
+    Hostname   Path      Type  Serial    Size   Health  Ident  Fault  Available
+
+.. Note: A more optional extended view with other information fields could be also useful.
+
+In order to know what is the current condition of the device, we need to know what is the amount of storage used, the percentage of free space, the average number of IOPS and the fault led state.
+This information should be provided probably by the Ceph orchestrator which is the component where we have access to this kind of information.
+
+Another important question around retrieving device information is “efficiency�. The information about devices can be critical in components like the Orchestrator or the Dashboard, because this information usually is used to take decisions.
+When we talk about efficiency we need to be sure that all the points are covered:
+
+#. Get the complete information for each device in the most fast way.
+#. All the information about all the devices in one host is accessible always immediately.
+#. The information is constantly updated in each host. A device failure or the addition of a new device must be detected in the smallest possible timeframe
+#. Scalability. To work with thousands of devices in hundreds of hosts shouldn't be a problem.
+
+A. Current workflow:
+--------------------
+**CLI**:
+    Operations:
+
+.. prompt:: bash #
+
+  ceph orch device ls
+  ceph orch device ls json ( to get all the fields for each device )
+
+    Problems in current implementation:
+        * Does not scale.
+
+**GUI**:
+    Operations:
+        * cluster.Inventory section:
+            The cluster.Inventory section presents a basic list of the devices in the cluster. It is a fixed list with only a few fields. Only the “ident light on� operation is possible although we do not know if it is possible or not until the operation is launched.
+
+    Problems in current implementation:
+        * Does not scale (depends of the orchestrator)
+        * Rigid user experience
+
+B. Proposed workflow:
+---------------------
+
+**CLI**:
+The current API is good enough, we only need to be sure that we have:
+
+    - all the attribute/health/operative state fields from each device
+    - fast response
+    - scalable
+
+**GUI**:
+The inventory should be able to be customized in order to show the desired fields of information for each device. Being customizable also the position of each field(column) and the sort order.
+
+The inventory should be filtered using any of the fields present in the list of devices.
+
+A customized inventory list together with the filter and sort order should be able to be stored for easy utilization. In this way we can provide a set of interesting predefined inventory lists. For example:
+
+    - Devices available
+    - Devices more used (more average iops) (should be an alert/trigger)
+    - Devices biggers than n Gb
+
+The inventory should also provide a way to do directly operations over physical devices:
+
+    * Identify:  Start/stop to blink the identification light
+    * Create OSD: Create an OSD using this disk device if it is available.
+    * Remove OSD: Delete the OSD using this disk device.
+
+2. Add OSDs
+===========
+
+A. Current workflow
+--------------------
+
+**CLI**:
+We can specify specific devices or use a “drive group� specification to create OSD’s in different hosts. By default, the definition of the OSDs to create is “declarative“ unless you use the unmanaged parameter.
+
+.. prompt:: bash #
+
+  ceph orch daemon add osd <host>:device1,device2 [--unmanaged=true]  (manual approach)
+  ceph orch apply osd -i <json_file/yaml_file> [--dry-run] [--unmanaged=true]*  (Service Spec based approach)
+
+**GUI**:
+Implemented in the dashboard section “cluster.OSDs�.
+There is a button to create the OSDs, that presents a page dialog box to select the physical devices that are going to be part of the OSDs as main, WAL or Db devices.
+It Is very difficult to make a selection ( or to understand how to make the selection). This is even worse if your cluster has the same kind of devices, resulting in the weird thing that is not possible to create an OSD using only one storage device (because you cannot select it)
+The problem here is the UI has been designed to work with “drive groups� and not to work for the user. The “drive group� is an abstract concept that must be used only in the background. Users must not be aware of this concept.
+
+B. Proposed workflow
+--------------------
+
+**CLI and GUI**
+
+The utilization of “declarative� drive groups makes it very difficult to understand how to configure OSD’s and the implications. Also make difficult the implementation because the multiple possibilities and the big amount of different conditions that we can find in a production system makes very complex the right evaluation and use of a declarative description of the storage devices used.
+This results in unexpected situations. For example:
+* A disk cleaned can be reused automatically and without any warning for creating a new OSD.
+* New installed disks are used automatically for OSDs (without any warning)
+* Errors trying to recreate OSD’s in disk removed from the system.
+
+So there is an important thing to consider in order to simplify everything for the user and for the implementation:
+**Avoid the “declarative� use of the drive groups**
+
+**GUI**:
+
+The user should be able to define the set of physical disk devices that are going to be used to support OSD’s.
+This means to make simple things like create one OSD in a certain device, and also to define in an easy way how to create multiple OSD’s across multiple devices in different hosts.
+
+We should take into account different premises:
+
+We use only bluestore OSD’s, this means that in order to create an OSD we can decide between different strategies: consume only a single device for the OSD, use an additional device for the WAL, and/or use another different device for the DB.
+To split the different bluestore OSD data components between different devices only makes sense if the WAL/DB devices are faster than the main storage device.
+And the split of devices are always inside the same host, although the configuration will be applicable to other hosts with the same storage devices schema.
+
+A massive creation of OSDs in a production system can result in real disaster because rebalancing can affect negatively to the normal system performance.
+The same massive OSD creation in a cluster that is being installed for the first time probably is the desired behaviour.
+So we should provide a mechanism to allow the user to select in which way the OSDs are going to be created. It seems that we have two possibilities:
+* Fast creation - fast but harmful for performance -
+Create the OSDs directly
+* Conservative creation - Slow but respectful with performance -
+Create all the OSDs with 0 weight. Once all OSd’ are installed, start to assign the right weight to each OSD one by one.
+
+With all these premises into account it is proposed the following interface with two different modes:
+
+
+**Device mode**:
+
+An inventory list with all the available devices and filter/listing capability is presented to the user, the user can “play� with this list obtaining a set of preferred cluster physical storage devices.
+
+The user can select from the “preferred devices list� ,one, several or all the devices. These selected devices will be the ones used to create OSD’s (1 per physical device.).
+
+OSD ids coming from previously deleted OSDs can be available. The user should indicate if these ids must be used or not in the new OSDs.
+
+The user interface proposed could be like:
+
+.. image:: ./mockups/OSD_Creation_device_mode.svg
+   :align: center
+
+
+**Host mode**:
+
+Is basically an OSD configuration using the storage devices in a host. This configuration will be used as a base pattern to apply the same schema in other hosts.
+
+The user must select a base host.
+Once the host is selected, we should provide three lists (or ways to select) of available devices in the host:
+* “slow devices� with the “hdd� devices
+* “fast devices (WAL)� with the “sdd/nvme� devices that can be used for Bluestore WAL data
+* “fast devices(DB)� with the “sdd/nvme� devices that can be used for Bluestore DB data
+
+The user, using filters over the list of “slow devices� should select one,several,or all the devices in the list for OSD creation.
+If the user wants to split Bluestore data components in several devices, the same operation will be needed to be performed in the other two “fast devices� lists.
+
+OSD ids coming from previously deleted OSDs can be available. The user should indicate if these ids must be used or not in the new OSDs
+
+Once the devices are selected we can provide a “preview “ of the OSD’s that are going to be created. (the fast devices potentially will store the WAL/DB for several OSDs).
+
+.. Example: The user selects 8 slow storage devices for OSD, and 2 nvme for WAL and 1 sdd for DB. In each nvme device we will have the WAL for 4 OSD’s and in the sdd device we will have the DB for 8 OSD’s.
+
+OSD creation should have the inventory, and analyse to determine whether the OSD creation can be hybrid, dedicated - present those as options to the user (they never see a device group!) - then they click create.
+
+When the user is happy with the OSD configuration in the host, we should provide a way to present a list of hosts where it is possible to apply the same OSD configuration. The user will select from this list the hosts where he wants to create the OSD’s.
+
+A preview/summary of the creation of OSD’s in all the hosts must be provided, and if the user wants this configuration, then it will be applied, resulting in a bulk OSD creation in multiple hosts.
+
+Information about the progress of OSD creation in all the hosts should be provided.
+
+.. image:: ./mockups/OSD_Creation_host_mode.svg
+   :align: center
+
+
+Key points to consider:
+-----------------------
+
+**1. Context is everything**:
+The current OSD creation flow doesn’t provide any indications of available devices or hosts. This leaves the user clicking on the add button and seeing nothing, if there are no devices available - at which point the user assumes there are no available devices. Both host-mode and device-mode UI flows illustrate a couple of usability features that should be implemented as a bare minimum.
+
+    a. If there are no devices available, the add button should be disabled
+
+    b. The UI for OSD creation should include a summary of discovered hosts with disks and the total number of available disks that could be used for OSD creation. This should also show total raw. E.g. 5 hosts, 50 HDDs (80TB), 10 NVME (5TB)
+
+        - The discovered configuration could also
+            - Use the hosts rack ID annotations to look at the capacity from a fault domain perspective to ensure it’s balanced - and warn if not.
+            - Confirm whether the host configurations are identical (homogeneous). Heterogeneous configurations could therefore be accompanied by a INFO/WARN message in the UI to highlight the potential balance issues of heterogeneous clusters.
+
+    c. Once the deployment decision is made, display a summary of the selection, that the user CONFIRMs
+        * Total devices by type that would be used
+        * Total number of OSDs that would be created
+        * Overall raw capacity of the creation request, together with the potential raw cluster capacity once the OSD addition is complete
+        * Use a rule-of-thumb to determine approximate deployment time - set an expectation.
+
+**2. Enabling new capacity**:
+Policy option for how new disks are added to the cluster (this is present in both host-mode and device-mode designs)
+- Phased by OSD: All OSDs added are at weight 0. The orchestrator then reweights each OSD in turn to drive the rebalance
+- Phased by host: all OSDs on a given host are reweighted at the same time
+- Immediate: don’t use reweight. Bring the OSDs up/in straight away (on an empty cluster, this should be the default)
+
+**3. UI redesign**:
+Discover the devices, suggest a layout based on these devices combined with best practice, inform if there are is not enough flash for the number of HDDs, inform if there are no free devices, and also provide the advanced use case which is what we see today (which echoes the drive group approach)
+
+**4. Imperative not Declarative**:
+The use of declarative “drive groups� is a problem in several aspects:
+
+For the final user:
+
+The “admin persona� who is going to install a cluster by first time knows what is the current hardware composition and will create the OSD’s possibly using all the storage devices in the hosts of the cluster planified to harbor OSDs.
+
+But we are not telling the “admin persona� that this initial decision will be inmutable in the future and applied automatically without any warning.
+
+This will result in several undesired situations:
+
+    1. Storage devices with OSD’s cannot be used  for other purposes. Because they are reinstalled as OSD’s as soon as they are cleaned. Seems difficult to explain that if you do not want that, you need to add the device to a black list,  or create the OSD using the “unmanaged� parameter. (not provided in the UI)
+    Another horrible situation can be: you buy a new device for one of your hosts in order to store the minecraft server. You have bad luck and this device is more or less the same as the ones you used for OSD’s … then you won’t install your minecraft server because the device is automatically used for OSDs.
+    Another stressful situation… your lab team installed 10 new disks in your cluster, and they decided to do that just where you have more traffic in the cluster network. Rebalance of data will cause a funny situation for the “admin persona�.
+    This is a good example about  how we can make the users life more difficult managing OSDs
+
+    2. Probably after a couple of years the requirements will grow. New different storage devices will be added. And the “admin persona� will need to specify that these devices will harbor OSDs. Then we have to store the initial “drive group� used to create the initial OSDs, and also the new “drive group� definition for the new devices. So now we have more than one “drive group�, so this implies two possibilities, add a “drive groups� management tool, or merge “the two definitions" in only one!
+
+    In any case this is a good example about how we can make the users and developers life more difficult.
+
+All these things can be avoided using imperative drive groups, we are going to provide the same functionality but without all the undesired collateral effects.
+From the development point of view , this will also simplify things, so it seems a very good idea to move from “declarative� drive groups to “imperative� drive groups.
+
+.. Note: The current dashboard implementation of the functionality to create OSDs is trying to deal with “drive groups� , This is the reason that it will be so uncomfortable for the final user. The “drive group�concept should be completely hidden to the dashboard user.
+
+3. Remove OSDs
+==============
+
+A. Current workflow
+--------------------
+
+**CLI**:
+    * We can launch the command to delete a OSD (one by one)
+
+.. prompt:: bash #
+
+  ceph orch osd rm <svc_id(s)> [--replace] [--force]
+
+    * We can verify what is the status of the delete operation
+
+.. prompt:: bash #
+
+  ceph orch osd rm status
+
+    * Finally we can “clean� completely the device used in the OSD
+
+.. prompt:: bash #
+
+  ceph orch device zap my_hostname /dev/sdx
+
+**GUI**:
+
+In the cluster OSD section we have a button to execute different primitive operations over the OSD’s selected. One of these primitives is delete.
+
+When the “delete� primitive is selected and the action button is pressed, a dialog box to confirm the operation and a check box to ask about preserving the “osd id� is shown. After accepting nothing seems to happen….
+
+No way to know what is the progress of the delete operation.
+
+We tend to show all the primitives for osd management in the UI - question is, does that make the environment more complex? Should the UI focus on the key workflows of osd management to cover 90% of the work quickly and easily, and leave the 10% to the CLI?
+
+
+B. Proposed workflow
+---------------------
+
+**CLI**:
+
+    - Need a way to know in advance how much time is going to be needed to delete an OSD (if we rebalance data)
+    - The current set of command can satisfy main requirements
+
+**GUI**:
+
+The user should select the OSD (or set of OSDs) to remove from a list with filtering capabilities.
+
+The OSD removal should provide an option to preserve the OSD id for use when creating new OSD’s. An assessment about the time that is going to take the operation is another important  element to decide how to do the operation and when is the best moment.
+
+When the user decides to execute the removal operation, the system should follow a safe procedure, with a certain degree of intelligence.
+
+Depending of the OSD state (in(out, up/down) and the situation ( we are in a low/high cpu/network utilization time interval), probably we will need to do different things.
+
+* Direct removal of the OSD:
+
+we are going to execute the OSD deletion operations without any wait.
+
+* Safe OSD Removal:
+
+We want to remove the OSD in the most safe way. This means wait until we know that the OSD is not storing information. The user must receive a notification when it will be safe to remove the OSD
+
+* Scheduled OSD removal:
+
+We want to execute the removal in the future. Besides that,  it is probable that we only will want to execute the removal if the system utilization is below certain limit
+
+4. Replace OSDs
+===============
+
+A. Current workflow
+--------------------
+
+Is the same workflow used for removing OSDs, but we just need to use the "replace" parameter in order to preserve the OSD id for future use when we are deleting.
+In the GUI the replace parameter appears as a checkbox.
+
+
+B. Proposed workflow
+---------------------
+
+Follow the directives we have in the proposed workflow for OSD removal
diff --git a/doc/dev/cephadm/developing-cephadm.rst b/doc/dev/cephadm/developing-cephadm.rst
new file mode 100644
index 000000000..49b771caa
--- /dev/null
+++ b/doc/dev/cephadm/developing-cephadm.rst
@@ -0,0 +1,403 @@
+=======================
+Developing with cephadm
+=======================
+
+There are several ways to develop with cephadm.  Which you use depends
+on what you're trying to accomplish.
+
+vstart --cephadm
+================
+
+- Start a cluster with vstart, with cephadm configured
+- Manage any additional daemons with cephadm
+- Requires compiled ceph binaries
+
+In this case, the mon and manager at a minimum are running in the usual
+vstart way, not managed by cephadm.  But cephadm is enabled and the local
+host is added, so you can deploy additional daemons or add additional hosts.
+
+This works well for developing cephadm itself, because any mgr/cephadm
+or cephadm/cephadm code changes can be applied by kicking ceph-mgr
+with ``ceph mgr fail x``.  (When the mgr (re)starts, it loads the
+cephadm/cephadm script into memory.)
+
+::
+
+   MON=1 MGR=1 OSD=0 MDS=0 ../src/vstart.sh -d -n -x --cephadm
+
+- ``~/.ssh/id_dsa[.pub]`` is used as the cluster key.  It is assumed that
+  this key is authorized to ssh with no passphrase to root@`hostname`.
+- cephadm does not try to manage any daemons started by vstart.sh (any
+  nonzero number in the environment variables).  No service spec is defined
+  for mon or mgr.
+- You'll see health warnings from cephadm about stray daemons--that's because
+  the vstart-launched daemons aren't controlled by cephadm.
+- The default image is ``quay.io/ceph-ci/ceph:main``, but you can change
+  this by passing ``-o container_image=...`` or ``ceph config set global container_image ...``.
+
+
+cstart and cpatch
+=================
+
+The ``cstart.sh`` script will launch a cluster using cephadm and put the
+conf and keyring in your build dir, so that the ``bin/ceph ...`` CLI works
+(just like with vstart).  The ``ckill.sh`` script will tear it down.
+
+- A unique but stable fsid is stored in ``fsid`` (in the build dir).
+- The mon port is random, just like with vstart.
+- The container image is ``quay.io/ceph-ci/ceph:$tag`` where $tag is
+  the first 8 chars of the fsid.
+- If the container image doesn't exist yet when you run cstart for the
+  first time, it is built with cpatch.
+
+There are a few advantages here:
+
+- The cluster is a "normal" cephadm cluster that looks and behaves
+  just like a user's cluster would.  In contrast, vstart and teuthology
+  clusters tend to be special in subtle (and not-so-subtle) ways (e.g.
+  having the ``lockdep`` turned on).
+
+To start a test cluster::
+
+  sudo ../src/cstart.sh
+
+The last line of the output will be a line you can cut+paste to update
+the container image.  For instance::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e
+
+By default, cpatch will patch everything it can think of from the local
+build dir into the container image.  If you are working on a specific
+part of the system, though, can you get away with smaller changes so that
+cpatch runs faster.  For instance::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e --py
+
+will update the mgr modules (minus the dashboard).  Or::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e --core
+
+will do most binaries and libraries.  Pass ``-h`` to cpatch for all options.
+
+Once the container is updated, you can refresh/restart daemons by bouncing
+them with::
+
+  sudo systemctl restart ceph-`cat fsid`.target
+
+When you're done, you can tear down the cluster with::
+
+  sudo ../src/ckill.sh   # or,
+  sudo ../src/cephadm/cephadm rm-cluster --force --fsid `cat fsid`
+
+cephadm bootstrap --shared_ceph_folder
+======================================
+
+Cephadm can also be used directly without compiled ceph binaries.
+
+Run cephadm like so::
+
+  sudo ./cephadm bootstrap --mon-ip 127.0.0.1 \
+    --ssh-private-key /home/<user>/.ssh/id_rsa \
+    --skip-mon-network \
+    --skip-monitoring-stack --single-host-defaults \
+    --skip-dashboard \
+    --shared_ceph_folder /home/<user>/path/to/ceph/
+
+- ``~/.ssh/id_rsa`` is used as the cluster key.  It is assumed that
+  this key is authorized to ssh with no passphrase to root@`hostname`.
+
+Source code changes made in the ``pybind/mgr/`` directory then
+require a daemon restart to take effect.
+
+Kcli: a virtualization management tool to make easy orchestrators development
+=============================================================================
+`Kcli <https://github.com/karmab/kcli>`_ is meant to interact with existing
+virtualization providers (libvirt, KubeVirt, oVirt, OpenStack, VMware vSphere,
+GCP and AWS) and to easily deploy and customize VMs from cloud images.
+
+It allows you to setup an environment with several vms with your preferred
+configuration (memory, cpus, disks) and OS flavor.
+
+main advantages:
+----------------
+  - Fast. Typically you can have a completely new Ceph cluster ready to debug
+    and develop orchestrator features in less than 5 minutes.
+  - "Close to production" lab. The resulting lab is close to "real" clusters
+    in QE labs or even production. It makes it easy to test "real things" in
+    an almost "real" environment.
+  - Safe and isolated. Does not depend of the things you have installed in
+    your machine. And the vms are isolated from your environment.
+  - Easy to work "dev" environment. For "not compiled" software pieces,
+    for example any mgr module. It is an environment that allow you to test your
+    changes interactively.
+
+Installation:
+-------------
+Complete documentation in `kcli installation <https://kcli.readthedocs.io/en/latest/#installation>`_
+but we suggest to use the container image approach.
+
+So things to do:
+  - 1. Review `requirements <https://kcli.readthedocs.io/en/latest/#libvirt-hypervisor-requisites>`_
+    and install/configure whatever is needed to meet them.
+  - 2. get the kcli image and create one alias for executing the kcli command
+    ::
+
+        # podman pull quay.io/karmab/kcli
+        # alias kcli='podman run --net host -it --rm --security-opt label=disable -v $HOME/.ssh:/root/.ssh -v $HOME/.kcli:/root/.kcli -v /var/lib/libvirt/images:/var/lib/libvirt/images -v /var/run/libvirt:/var/run/libvirt -v $PWD:/workdir -v /var/tmp:/ignitiondir quay.io/karmab/kcli'
+
+.. note:: This assumes that /var/lib/libvirt/images is your default libvirt pool.... Adjust if using a different path
+
+.. note:: Once you have used your kcli tool to create and use different labs, we
+   suggest you stick to a given container tag and update your kcli alias.
+   Why? kcli uses a rolling release model and sticking to a specific
+   container tag will improve overall stability.
+   what we want is overall stability.
+
+Test your kcli installation:
+----------------------------
+See the kcli `basic usage workflow <https://kcli.readthedocs.io/en/latest/#basic-workflow>`_
+
+Create a Ceph lab cluster
+-------------------------
+In order to make this task simple, we are going to use a "plan".
+
+A "plan" is a file where you can define a set of vms with different settings.
+You can define hardware parameters (cpu, memory, disks ..), operating system and
+it also allows you to automate the installation and configuration of any
+software you want to have.
+
+There is a `repository <https://github.com/karmab/kcli-plans>`_ with a collection of
+plans that can be used for different purposes. And we have predefined plans to
+install Ceph clusters using Ceph ansible or cephadm, so let's create our first Ceph
+cluster using cephadm::
+
+# kcli create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml
+
+This will create a set of three vms using the plan file pointed by the url.
+After a few minutes, let's check the cluster:
+
+* Take a look to the vms created::
+
+  # kcli list vms
+
+* Enter in the bootstrap node::
+
+  # kcli ssh ceph-node-00
+
+* Take a look to the ceph cluster installed::
+
+  [centos@ceph-node-00 ~]$ sudo -i
+  [root@ceph-node-00 ~]# cephadm version
+  [root@ceph-node-00 ~]# cephadm shell
+  [ceph: root@ceph-node-00 /]# ceph orch host ls
+
+Create a Ceph cluster to make easy developing in mgr modules (Orchestrators and Dashboard)
+------------------------------------------------------------------------------------------
+The cephadm kcli plan (and cephadm) are prepared to do that.
+
+The idea behind this method is to replace several python mgr folders in each of
+the ceph daemons with the source code folders in your host machine.
+This "trick" will allow you to make changes in any orchestrator or dashboard
+module and test them intermediately. (only needed to disable/enable the mgr module)
+
+So in order to create a ceph cluster for development purposes you must use the
+same cephadm plan but with a new parameter pointing to your Ceph source code folder::
+
+  # kcli create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml -P ceph_dev_folder=/home/mycodefolder/ceph
+
+Ceph Dashboard development
+--------------------------
+Ceph dashboard module is not going to be loaded if previously you have not
+generated the frontend bundle.
+
+For now, in order load properly the Ceph Dashboardmodule and to apply frontend
+changes you have to run "ng build" on your laptop::
+
+  # Start local frontend build with watcher (in background):
+  sudo dnf install -y nodejs
+  cd <path-to-your-ceph-repo>
+  cd src/pybind/mgr/dashboard/frontend
+  sudo chown -R <your-user>:root dist node_modules
+  NG_CLI_ANALYTICS=false npm ci
+  npm run build -- --deleteOutputPath=false --watch &
+
+After saving your changes, the frontend bundle will be built again.
+When completed, you'll see::
+
+  "Localized bundle generation complete."
+
+Then you can reload your Dashboard browser tab.
+
+Cephadm box container (Podman inside Podman) development environment
+====================================================================
+
+As kcli has a long startup time, we created an alternative which is faster using
+Podman inside Podman. This approach has its downsides too as we have to
+simulate the creation of osds and addition of devices with loopback devices.
+
+Cephadm's box environment is simple to set up. The setup requires you to
+get the required Podman images for Ceph and what we call boxes.
+A box is the first layer of Podman containers which can be either a seed or a
+host. A seed is the main box which holds Cephadm and where you bootstrap the
+cluster. On the other hand, you have hosts with a SSH server setup so you can
+add those hosts to the cluster. The second layer, managed by Cephadm, inside the
+seed box, requires the Ceph image.
+
+.. warning:: This development environment is still experimental and can have unexpected
+             behaviour. Please take a look at the road map and the known issues section
+             to see what the development progress.
+
+Requirements
+------------
+
+* `podman-compose <https://github.com/containers/podman-compose>`_
+* lvm
+
+Setup
+-----
+
+In order to setup Cephadm's box run::
+
+  cd src/cephadm/box
+  ./box.py -v cluster setup
+
+.. note:: It is recommended to run box with verbose (-v) as it will show the output of
+          shell commands being run.
+
+After getting all needed images we can create a simple cluster without OSDs and hosts with::
+
+  ./box.py -v cluster start
+
+If you want to deploy the cluster with more OSDs and hosts::
+  # 3 osds and 3 hosts by default
+  sudo box -v cluster start --extended
+  # explicitly change number of hosts and osds
+  sudo box -v cluster start --extended --osds 5 --hosts 5
+
+.. warning:: OSDs are still not supported in the box implementation with Podman. It is
+             work in progress.
+
+
+Without the extended option, explicitly adding either more hosts or OSDs won't change the state
+of the cluster.
+
+.. note:: Cluster start will try to setup even if cluster setup was not called.
+.. note:: OSDs are created with loopback devices and hence, sudo is needed to
+   create loopback devices capable of holding OSDs.
+.. note::  Each osd will require 5GiB of space.
+
+After bootstrapping the cluster you can go inside the seed box in which you'll be
+able to run Cephadm commands::
+
+  ./box.py -v cluster bash
+  [root@8d52a7860245] cephadm --help
+  [root@8d52a7860245] cephadm shell
+  ...
+
+
+If you want to navigate to the dashboard enter https://localhost:8443 on you browser.
+
+You can also find the hostname and ip of each box container with::
+
+  ./box.py cluster list
+
+and you'll see something like::
+
+  IP               Name            Hostname
+  172.30.0.2       box_hosts_1     6283b7b51d91
+  172.30.0.3       box_hosts_3     3dcf7f1b25a4
+  172.30.0.4       box_seed_1      8d52a7860245
+  172.30.0.5       box_hosts_2     c3c7b3273bf1
+
+To remove the cluster and clean up run::
+
+  ./box.py cluster down
+
+If you just want to clean up the last cluster created run::
+
+  ./box.py cluster cleanup
+
+To check all available commands run::
+
+  ./box.py --help
+
+If you want to run the box with Docker you can. You'll have to specify which
+engine you want to you like::
+
+  ./box.py -v --engine docker cluster list
+
+With Docker commands like bootstrap and osd creation should be called with sudo
+since it requires privileges to create osds on VGs and LVs::
+
+  sudo ./box.py -v --engine docker cluster start --expanded
+
+.. warning:: Using Docker as the box engine is dangerous as there were some instances
+             where the Xorg session was killed.
+
+Known issues
+------------
+
+* If you get permission issues with Cephadm because it cannot infer the keyring
+  and configuration, please run cephadm like this example::
+
+    cephadm shell --config /etc/ceph/ceph.conf --keyring /etc/ceph/ceph.kerying
+
+* Docker containers run with the --privileged flag enabled which has been seen
+  to make some computers log out.
+* If SELinux is not disabled you'll probably see unexpected behaviour. For example:
+  if not all permissions of Ceph repo files are set to your user it will probably
+  fail starting with podman-compose.
+* If running a command it fails to run a podman command because it couldn't find the
+  container, you can debug by running the same podman-compose .. up command displayed
+  with the flag -v.
+
+Road map
+------------
+
+* Create osds with ``ceph-volume raw``.
+* Enable ceph-volume to mark loopback devices as a valid block device in
+  the inventory.
+* Make the box ready to run dashboard CI tests (including cluster expansion).
+
+Note regarding network calls from CLI handlers
+==============================================
+
+Executing any cephadm CLI commands like ``ceph orch ls`` will block the
+mon command handler thread within the MGR, thus preventing any concurrent
+CLI calls. Note that pressing ``^C`` will not resolve this situation,
+as *only* the client will be aborted, but not execution of the command
+within the orchestrator manager module itself. This means, cephadm will
+be completely unresponsive until the execution of the CLI handler is
+fully completed. Note that even ``ceph orch ps`` will not respond while
+another handler is executing.
+
+This means we should do very few synchronous calls to remote hosts.
+As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers.
+Everything else should be done asynchronously in other threads, like ``serve()``.
+
+Note regarding different variables used in the code
+===================================================
+
+* a ``service_type`` is something like mon, mgr, alertmanager etc defined
+  in ``ServiceSpec``
+* a ``service_id`` is the name of the service. Some services don't have
+  names.
+* a ``service_name`` is ``<service_type>.<service_id>``
+* a ``daemon_type`` is the same as the service_type, except for ingress,
+  which has the haproxy and keepalived daemon types.
+* a ``daemon_id`` is typically ``<service_id>.<hostname>.<random-string>``.
+  (Not the case for e.g. OSDs. OSDs are always called OSD.N)
+* a ``daemon_name`` is ``<daemon_type>.<daemon_id>``
+
+.. _compiling-cephadm:
+
+Compiling cephadm
+=================
+
+Recent versions of cephadm are based on `Python Zip Application`_ support, and
+are "compiled" from Python source code files in the ceph tree. To create your
+own copy of the cephadm "binary" use the script located at
+``src/cephadm/build.py`` in the Ceph tree.  The command should take the form
+``./src/cephadm/build.py [output]``.
+
+.. _Python Zip Application: https://peps.python.org/pep-0441/
diff --git a/doc/dev/cephadm/host-maintenance.rst b/doc/dev/cephadm/host-maintenance.rst
new file mode 100644
index 000000000..c54e0495c
--- /dev/null
+++ b/doc/dev/cephadm/host-maintenance.rst
@@ -0,0 +1,104 @@
+================
+Host Maintenance
+================
+
+All hosts that support Ceph daemons need to support maintenance activity, whether the host
+is physical or virtual. This means that management workflows should provide
+a simple and consistent way to support this operational requirement. This document defines
+the maintenance strategy that could be implemented in cephadm and mgr/cephadm.
+
+
+High Level Design
+=================
+Placing a host into maintenance, adopts the following workflow;
+
+#. confirm that the removal of the host does not impact data availability (the following
+   steps will assume it is safe to proceed)
+
+   * ``orch host ok-to-stop <host>`` would be used here
+
+#. if the host has osd daemons, apply noout to the host subtree to prevent data migration
+   from triggering during the planned maintenance slot.
+#. Stop the ceph target (all daemons stop)
+#. Disable the ceph target on that host, to prevent a reboot from automatically starting
+   ceph services again)
+
+
+Exiting Maintenance, is basically the reverse of the above sequence
+
+Admin Interaction
+=================
+The ceph orch command will be extended to support maintenance. 
+
+.. code-block::
+
+    ceph orch host maintenance enter <host> [ --force ]
+    ceph orch host maintenance exit <host>
+
+.. note:: In addition, the host's status should be updated to reflect whether it
+   is in maintenance or not. 
+
+The 'check' Option 
+__________________
+The orch host ok-to-stop command focuses on ceph daemons (mon, osd, mds), which
+provides the first check. However, a ceph cluster also uses other types of daemons
+for monitoring, management and non-native protocol support which means the 
+logic will need to consider service impact too. The 'check' option provides 
+this additional layer to alert the user of service impact to *secondary*
+daemons.
+
+The list below shows some of these additional daemons.
+
+* mgr (not included in ok-to-stop checks) 
+* prometheus, grafana, alertmanager
+* rgw
+* haproxy
+* iscsi gateways
+* ganesha gateways
+
+By using the --check option first, the Admin can choose whether to proceed. This 
+workflow is obviously optional for the CLI user, but could be integrated into the
+UI workflow to help less experienced administrators manage the cluster.
+
+By adopting this two-phase approach, a UI based workflow would look something
+like this.
+
+#. User selects a host to place into maintenance
+ 
+   * orchestrator checks for data **and** service impact
+#. If potential impact is shown, the next steps depend on the impact type
+   
+   * **data availability** : maintenance is denied, informing the user of the issue
+   * **service availability** : user is provided a list of affected services and 
+     asked to confirm
+
+
+Components Impacted
+===================
+Implementing this capability will require changes to the following; 
+
+* cephadm
+
+  * Add maintenance subcommand with the following 'verbs'; enter, exit, check
+
+* mgr/cephadm
+
+  * add methods to CephadmOrchestrator for enter/exit and check
+  * data gathering would be skipped for hosts in a maintenance state
+
+* mgr/orchestrator
+
+  * add CLI commands to OrchestratorCli which expose the enter/exit and check interaction
+
+
+Ideas for Future Work
+=====================
+#. When a host is placed into maintenance, the time of the event could be persisted. This
+   would allow the orchestrator layer to establish a maintenance window for the task and 
+   alert if the maintenance window has been exceeded.
+#. The maintenance process could support plugins to allow other integration tasks to be
+   initiated as part of the transition to and from maintenance. This plugin capability could
+   support actions like; 
+   
+   * alert suppression to 3rd party monitoring framework(s)
+   * service level reporting, to record outage windows
diff --git a/doc/dev/cephadm/index.rst b/doc/dev/cephadm/index.rst
new file mode 100644
index 000000000..f89e7ed26
--- /dev/null
+++ b/doc/dev/cephadm/index.rst
@@ -0,0 +1,15 @@
+===================================
+CEPHADM Developer Documentation
+===================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1
+
+
+   developing-cephadm
+   host-maintenance
+   compliance-check
+   Storage devices and OSDs management <./design/storage_devices_and_osds>
+   scalability-notes
diff --git a/doc/dev/cephadm/scalability-notes.rst b/doc/dev/cephadm/scalability-notes.rst
new file mode 100644
index 000000000..9faaee041
--- /dev/null
+++ b/doc/dev/cephadm/scalability-notes.rst
@@ -0,0 +1,95 @@
+#############################################
+ Notes and Thoughts on Cephadm's scalability
+#############################################
+
+*********************
+ About this document
+*********************
+
+This document does NOT define a specific proposal or some future work.
+Instead it merely lists a few thoughts that MIGHT be relevant for future
+cephadm enhancements.
+
+*******
+ Intro
+*******
+
+Current situation:
+
+Cephadm manages all registered hosts. This means that it periodically
+scrapes data from each host to identify changes on the host like:
+
+-  disk added/removed
+-  daemon added/removed
+-  host network/firewall etc has changed
+
+Currently, cephadm scrapes each host (up to 10 in parallel) every 6
+minutes, unless a refresh is forced manually.
+
+Refreshes for disks (ceph-volume), daemons (podman/docker), etc, happen
+in sequence.
+
+With the cephadm exporter, we have now reduced the time to scan hosts
+considerably, but the question remains:
+
+Is the cephadm-exporter sufficient to solve all future scalability
+issues?
+
+***********************************************
+ Considerations of cephadm-exporter's REST API
+***********************************************
+
+The cephadm-exporter uses HTTP to serve an endpoint to the hosts
+metadata. We MIGHT encounter some issues with this approach, which need
+to be mitigated at some point.
+
+-  With the cephadm-exporter we use SSH and HTTP to connect to each
+   host. Having two distinct transport layers feels odd, and we might
+   want to consider reducing it to only a single protocol.
+
+-  The current approach of delivering ``bin/cephadm`` to the host doesn't
+   allow the use of external dependencies. This means that we're stuck
+   with the built-in HTTP server lib, which isn't great for providing a
+   good developer experience. ``bin/cephadm`` needs to be packaged and
+   distributed (one way or the other) for us to make use of a better
+   http server library.
+
+************************
+ MON's config-key store
+************************
+
+After the ``mgr/cephadm`` queried metadata from each host, cephadm stores
+the data within the mon's k-v store.
+
+If each host would be allowed to write their own metadata to the store,
+``mgr/cephadm`` would no longer be required to gather the data.
+
+Some questions arise:
+
+-  ``mgr/cephadm`` now needs to query data from the config-key store,
+   instead of relying on cached data.
+
+-  cephadm knows three different types of data: (1) Data that is
+   critical and needs to be stored in the config-key store. (2) Data
+   that can be kept in memory only. (3) Data that can be stored in
+   RADOS pool. How can we apply this idea to those different types of
+   data.
+
+*******************************
+ Increase the worker pool size
+*******************************
+
+``mgr/cephadm`` is currently able to scrape 10 nodes at the same time.
+
+The scrape of a individual host takes the same amount of time persists.
+We'd just reduce the overall execution time.
+
+At best we can reach O(hosts) + O(daemons).
+
+*************************
+ Backwards compatibility
+*************************
+
+Any changes need to be backwards compatible or completely isolated from
+any existing functionality. There are running cephadm clusters out there
+that require an upgrade path.
diff --git a/doc/dev/cephfs-mirroring.rst b/doc/dev/cephfs-mirroring.rst
new file mode 100644
index 000000000..a804a0075
--- /dev/null
+++ b/doc/dev/cephfs-mirroring.rst
@@ -0,0 +1,409 @@
+================
+CephFS Mirroring
+================
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file
+system via `cephfs-mirror` tool. Snapshots are synchronized by mirroring
+snapshot data followed by creating a snapshot with the same name (for a given
+directory on the remote file system) as the snapshot being synchronized.
+
+Requirements
+------------
+
+The primary (local) and secondary (remote) Ceph clusters version should be
+Pacific or later.
+
+Key Idea
+--------
+
+For a given snapshot pair in a directory, `cephfs-mirror` daemon will rely on
+readdir diff to identify changes in a directory tree. The diffs are applied to
+directory in the remote file system thereby only synchronizing files that have
+changed between two snapshots.
+
+This feature is tracked here: https://tracker.ceph.com/issues/47034.
+
+Currently, snapshot data is synchronized by bulk copying to the remote
+filesystem.
+
+.. note:: Synchronizing hardlinks is not supported -- hardlinked files get
+   synchronized as separate files.
+
+Creating Users
+--------------
+
+Start by creating a user (on the primary/local cluster) for the mirror daemon.
+This user requires write capability on the metadata pool to create RADOS
+objects (index objects) for watch/notify operation and read capability on the
+data pool(s).
+
+.. prompt:: bash $
+
+   ceph auth get-or-create client.mirror mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'
+
+Create a user for each file system peer (on the secondary/remote cluster). This user needs
+to have full capabilities on the MDS (to take snapshots) and the OSDs::
+
+  $ ceph fs authorize <fs_name> client.mirror_remote / rwps
+
+This user should be used (as part of peer specification) when adding a peer.
+
+Starting Mirror Daemon
+----------------------
+
+Mirror daemon should be spawned using `systemctl(1)` unit files::
+
+  $ systemctl enable cephfs-mirror@mirror
+  $ systemctl start cephfs-mirror@mirror
+
+`cephfs-mirror` daemon can be run in foreground using::
+
+  $ cephfs-mirror --id mirror --cluster site-a -f
+
+.. note:: User used here is `mirror` as created in the `Creating Users` section.
+
+Mirroring Design
+----------------
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file system
+via `cephfs-mirror` tool. For a given directory, snapshots are synchronized by transferring
+snapshot data to the remote file system and creating a snapshot with the same name as the
+snapshot being synchronized.
+
+Snapshot Synchronization Order
+------------------------------
+
+Although the order in which snapshots get chosen for synchronization does not matter,
+snapshots are picked based on creation order (using snap-id).
+
+Snapshot Incarnation
+--------------------
+
+A snapshot may be deleted and recreated (with the same name) with different contents.
+An "old" snapshot could have been synchronized (earlier) and the recreation of the
+snapshot could have been done when mirroring was disabled. Using snapshot names to
+infer the point-of-continuation would result in the "new" snapshot (incarnation)
+never getting picked up for synchronization.
+
+Snapshots on the secondary file system stores the snap-id of the snapshot it was
+synchronized from. This metadata is stored in `SnapInfo` structure on the MDS.
+
+Interfaces
+----------
+
+`Mirroring` module (manager plugin) provides interfaces for managing directory snapshot
+mirroring. Manager interfaces are (mostly) wrappers around monitor commands for managing
+file system mirroring and is the recommended control interface.
+
+Mirroring Module and Interface
+------------------------------
+
+Mirroring module provides interface for managing directory snapshot mirroring. The module
+is implemented as a Ceph Manager plugin. Mirroring module does not manage spawning (and
+terminating) the mirror daemons. Right now the preferred way would be to start/stop
+mirror daemons via `systemctl(1)`. Going forward, deploying mirror daemons would be
+managed by `cephadm` (Tracker: http://tracker.ceph.com/issues/47261).
+
+The manager module is responsible for assigning directories to mirror daemons for
+synchronization. Multiple mirror daemons can be spawned to achieve concurrency in
+directory snapshot synchronization. When mirror daemons are spawned (or terminated)
+, the mirroring module discovers the modified set of mirror daemons and rebalances
+the directory assignment amongst the new set thus providing high-availability.
+
+.. note:: Multiple mirror daemons is currently untested. Only a single mirror daemon
+          is recommended.
+
+Mirroring module is disabled by default. To enable mirroring use::
+
+  $ ceph mgr module enable mirroring
+
+Mirroring module provides a family of commands to control mirroring of directory
+snapshots. To add or remove directories, mirroring needs to be enabled for a given
+file system. To enable mirroring use::
+
+  $ ceph fs snapshot mirror enable <fs_name>
+
+.. note:: Mirroring module commands use `fs snapshot mirror` prefix as compared to
+          the monitor commands which `fs mirror` prefix. Make sure to use module
+          commands.
+
+To disable mirroring, use::
+
+  $ ceph fs snapshot mirror disable <fs_name>
+
+Once mirroring is enabled, add a peer to which directory snapshots are to be mirrored.
+Peers follow `<client>@<cluster>` specification and get assigned a unique-id (UUID)
+when added. See `Creating Users` section on how to create Ceph users for mirroring.
+
+To add a peer use::
+
+  $ ceph fs snapshot mirror peer_add <fs_name> <remote_cluster_spec> [<remote_fs_name>] [<remote_mon_host>] [<cephx_key>]
+
+`<remote_fs_name>` is optional, and default to `<fs_name>` (on the remote cluster).
+
+This requires the remote cluster ceph configuration and user keyring to be available in
+the primary cluster. See `Bootstrap Peers` section to avoid this. `peer_add` additionally
+supports passing the remote cluster monitor address and the user key. However, bootstrapping
+a peer is the recommended way to add a peer.
+
+.. note:: Only a single peer is supported right now.
+
+To remove a peer use::
+
+  $ ceph fs snapshot mirror peer_remove <fs_name> <peer_uuid>
+
+.. note:: See `Mirror Daemon Status` section on how to figure out Peer UUID.
+
+To list file system mirror peers use::
+
+  $ ceph fs snapshot mirror peer_list <fs_name>
+
+To configure a directory for mirroring, use::
+
+  $ ceph fs snapshot mirror add <fs_name> <path>
+
+To stop a mirroring directory snapshots use::
+
+  $ ceph fs snapshot mirror remove <fs_name> <path>
+
+Only absolute directory paths are allowed. Also, paths are normalized by the mirroring
+module, therefore, `/a/b/../b` is equivalent to `/a/b`.
+
+  $ mkdir -p /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2
+  {}
+  $ ceph fs snapshot mirror add cephfs /d0/d1/../d1/d2
+  Error EEXIST: directory /d0/d1/d2 is already tracked
+
+Once a directory is added for mirroring, its subdirectory or ancestor directories are
+disallowed to be added for mirroring::
+
+  $ ceph fs snapshot mirror add cephfs /d0/d1
+  Error EINVAL: /d0/d1 is a ancestor of tracked path /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2/d3
+  Error EINVAL: /d0/d1/d2/d3 is a subtree of tracked path /d0/d1/d2
+
+Commands to check directory mapping (to mirror daemons) and directory distribution are
+detailed in `Mirror Daemon Status` section.
+
+Bootstrap Peers
+---------------
+
+Adding a peer (via `peer_add`) requires the peer cluster configuration and user keyring
+to be available in the primary cluster (manager host and hosts running the mirror daemon).
+This can be avoided by bootstrapping and importing a peer token. Peer bootstrap involves
+creating a bootstrap token on the peer cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap create backup_fs client.mirror_remote site-remote
+  {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="}
+
+`site-name` refers to a user-defined string to identify the remote filesystem. In context
+of `peer_add` interface, `site-name` is the passed in `cluster` name from `remote_cluster_spec`.
+
+Import the bootstrap token in the primary cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ==
+
+Mirror Daemon Status
+--------------------
+
+Mirror daemons get asynchronously notified about changes in file system mirroring status
+and/or peer updates.
+
+CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon
+status::
+
+  $ ceph fs snapshot mirror daemon status
+
+E.g::
+
+  $ ceph fs snapshot mirror daemon status | jq
+  [
+    {
+      "daemon_id": 284167,
+      "filesystems": [
+        {
+          "filesystem_id": 1,
+          "name": "a",
+          "directory_count": 1,
+          "peers": [
+            {
+              "uuid": "02117353-8cd1-44db-976b-eb20609aa160",
+              "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "ceph",
+                "fs_name": "backup_fs"
+              },
+              "stats": {
+                "failure_count": 1,
+                "recovery_count": 0
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+
+An entry per mirror daemon instance is displayed along with information such as configured
+peers and basic stats. For more detailed stats, use the admin socket interface as detailed
+below.
+
+CephFS mirror daemons provide admin socket commands for querying mirror status. To check
+available commands for mirror status use::
+
+  $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help
+  {
+      ....
+      ....
+      "fs mirror status cephfs@360": "get filesystem mirror status",
+      ....
+      ....
+  }
+
+Commands with `fs mirror status` prefix provide mirror status for mirror enabled
+file systems. Note that `cephfs@360` is of format `filesystem-name@filesystem-id`.
+This format is required since mirror daemons get asynchronously notified regarding
+file system mirror status (A file system can be deleted and recreated with the same
+name).
+
+Right now, the command provides minimal information regarding mirror status::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror status cephfs@360
+  {
+    "rados_inst": "192.168.0.5:0/1476644347",
+    "peers": {
+        "a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
+            "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "site-a",
+                "fs_name": "backup_fs"
+            }
+        }
+    },
+    "snap_dirs": {
+        "dir_count": 1
+    }
+  }
+
+`Peers` section in the command output above shows the peer information such as unique
+peer-id (UUID) and specification. The peer-id is required to remove an existing peer
+as mentioned in the `Mirror Module and Interface` section.
+
+Command with `fs mirror peer status` prefix provide peer synchronization status. This
+command is of format `filesystem-name@filesystem-id peer-uuid`::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+Synchronization stats such as `snaps_synced`, `snaps_deleted` and `snaps_renamed` are reset
+on daemon restart and/or when a directory is reassigned to another mirror daemon (when
+multiple mirror daemons are deployed).
+
+A directory can be in one of the following states::
+
+  - `idle`: The directory is currently not being synchronized
+  - `syncing`: The directory is currently being synchronized
+  - `failed`: The directory has hit upper limit of consecutive failures
+
+When a directory hits a configured number of consecutive synchronization failures, the
+mirror daemon marks it as `failed`. Synchronization for these directories are retried.
+By default, the number of consecutive failures before a directory is marked as failed
+is controlled by `cephfs_mirror_max_consecutive_failures_per_directory` configuration
+option (default: 10) and the retry interval for failed directories is controlled via
+`cephfs_mirror_retry_failed_directories_interval` configuration option (default: 60s).
+
+E.g., adding a regular file for synchronization would result in failed status::
+
+  $ ceph fs snapshot mirror add cephfs /f0
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    },
+    "/f0": {
+        "state": "failed",
+        "snaps_synced": 0,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+This allows a user to add a non-existent directory for synchronization. The mirror daemon
+would mark the directory as failed and retry (less frequently). When the directory comes
+to existence, the mirror daemons would unmark the failed state upon successful snapshot
+synchronization.
+
+When mirroring is disabled, the respective `fs mirror status` command for the file system
+will not show up in command help.
+
+Mirroring module provides a couple of commands to display directory mapping and distribution
+information. To check which mirror daemon a directory has been mapped to use::
+
+  $ ceph fs snapshot mirror dirmap cephfs /d0/d1/d2
+  {
+    "instance_id": "404148",
+    "last_shuffled": 1601284516.10986,
+    "state": "mapped"
+  }
+
+.. note:: `instance_id` is the RADOS instance-id associated with a mirror daemon.
+
+Other information such as `state` and `last_shuffled` are interesting when running
+multiple mirror daemons.
+
+When no mirror daemons are running the above command shows::
+
+  $ ceph fs snapshot mirror dirmap cephfs /d0/d1/d2
+  {
+    "reason": "no mirror daemons running",
+    "state": "stalled"
+  }
+
+Signifying that no mirror daemons are running and mirroring is stalled.
+
+Re-adding Peers
+---------------
+
+When re-adding (reassigning) a peer to a file system in another cluster, ensure that
+all mirror daemons have stopped synchronization to the peer. This can be checked
+via `fs mirror status` admin socket command (the `Peer UUID` should not show up
+in the command output). Also, it is recommended to purge synchronized directories
+from the peer  before re-adding it to another file system (especially those directories
+which might exist in the new primary file system). This is not required if re-adding
+a peer to the same primary file system it was earlier synchronized from.
+
+Feature Status
+--------------
+
+`cephfs-mirror` daemon is built by default (follows `WITH_CEPHFS` CMake rule).
diff --git a/doc/dev/cephfs-reclaim.rst b/doc/dev/cephfs-reclaim.rst
new file mode 100644
index 000000000..ce56f5eaa
--- /dev/null
+++ b/doc/dev/cephfs-reclaim.rst
@@ -0,0 +1,104 @@
+CephFS Reclaim Interface
+========================
+
+Introduction
+------------
+NFS servers typically do not track ephemeral state on stable storage. If
+the NFS server is restarted, then it will be resurrected with no
+ephemeral state, and the NFS clients are expected to send requests to
+reclaim what state they held during a grace period.
+
+In order to support this use-case, libcephfs has grown several functions
+that allow a client that has been stopped and restarted to destroy or
+reclaim state held by a previous incarnation of itself. This allows the
+client to reacquire state held by its previous incarnation, and to avoid
+the long wait for the old session to time out before releasing the state
+previously held.
+
+As soon as an NFS server running over cephfs goes down, it's racing
+against its MDS session timeout. If the Ceph session times out before
+the NFS grace period is started, then conflicting state could be
+acquired by another client. This mechanism also allows us to increase
+the timeout for these clients, to ensure that the server has a long
+window of time to be restarted.
+
+Setting the UUID
+----------------
+In order to properly reset or reclaim against the old session, we need a
+way to identify the old session. This done by setting a unique opaque
+value on the session using **ceph_set_uuid()**. The uuid value can be
+any string and is treated as opaque by the client.
+
+Setting the uuid directly can only be done on a new session, prior to
+mounting. When reclaim is performed the current session will inherit the
+old session's uuid.
+
+Starting Reclaim
+----------------
+After calling ceph_create and ceph_init on the resulting struct
+ceph_mount_info, the client should then issue ceph_start_reclaim,
+passing in the uuid of the previous incarnation of the client with any
+flags.
+
+CEPH_RECLAIM_RESET
+   This flag indicates that we do not intend to do any sort of reclaim
+   against the old session indicated by the given uuid, and that it
+   should just be discarded. Any state held by the previous client
+   should be released immediately.
+
+Finishing Reclaim
+-----------------
+After the Ceph client has completed all of its reclaim operations, the
+client should issue ceph_finish_reclaim to indicate that the reclaim is
+now complete.
+
+Setting Session Timeout (Optional)
+----------------------------------
+When a client dies and is restarted, and we need to preserve its state,
+we are effectively racing against the session expiration clock. In this
+situation we generally want a longer timeout since we expect to
+eventually kill off the old session manually.
+
+Example 1: Reset Old Session
+----------------------------
+This example just kills off the MDS session held by a previous instance
+of itself. An NFS server can start a grace period and then ask the MDS
+to tear down the old session. This allows clients to start reclaim
+immediately.
+
+(Note: error handling omitted for clarity)
+
+.. code-block:: c
+
+	struct ceph_mount_info *cmount;
+	const char *uuid = "foobarbaz";
+
+	/* Set up a new cephfs session, but don't mount it yet. */
+	rc = ceph_create(&cmount);
+	rc = ceph_init(&cmount);
+
+	/*
+	 * Set the timeout to 5 minutes to lengthen the window of time for
+	 * the server to restart, should it crash.
+	 */
+	ceph_set_session_timeout(cmount, 300);
+
+	/*
+	 * Start reclaim vs. session with old uuid. Before calling this,
+	 * all NFS servers that could acquire conflicting state _must_ be
+	 * enforcing their grace period locally.
+	 */
+	rc = ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET);
+
+	/* Declare reclaim complete */
+	rc = ceph_finish_reclaim(cmount);
+
+	/* Set uuid held by new session */
+	ceph_set_uuid(cmount, nodeid);
+
+	/*
+	 * Now mount up the file system and do normal open/lock operations to
+	 * satisfy reclaim requests.
+	 */
+	ceph_mount(cmount, rootpath);
+	...
diff --git a/doc/dev/cephfs-snapshots.rst b/doc/dev/cephfs-snapshots.rst
new file mode 100644
index 000000000..31c337fb9
--- /dev/null
+++ b/doc/dev/cephfs-snapshots.rst
@@ -0,0 +1,149 @@
+CephFS Snapshots
+================
+
+CephFS supports snapshots, generally created by invoking mkdir within the
+``.snap`` directory. Note this is a hidden, special directory, not visible
+during a directory listing.
+
+Overview
+-----------
+
+Generally, snapshots do what they sound like: they create an immutable view
+of the file system at the point in time they're taken. There are some headline
+features that make CephFS snapshots different from what you might expect:
+
+* Arbitrary subtrees. Snapshots are created within any directory you choose,
+  and cover all data in the file system under that directory.
+* Asynchronous. If you create a snapshot, buffered data is flushed out lazily,
+  including from other clients. As a result, "creating" the snapshot is
+  very fast.
+
+Important Data Structures
+-------------------------
+* SnapRealm: A `SnapRealm` is created whenever you create a snapshot at a new
+  point in the hierarchy (or, when a snapshotted inode is move outside of its
+  parent snapshot). SnapRealms contain an `sr_t srnode`, and `inodes_with_caps`
+  that are part of the snapshot. Clients also have a SnapRealm concept that
+  maintains less data but is used to associate a `SnapContext` with each open
+  file for writing.
+* sr_t: An `sr_t` is the on-disk snapshot metadata. It is part of the containing
+  directory and contains sequence counters, timestamps, the list of associated
+  snapshot IDs, and `past_parent_snaps`.
+* SnapServer: SnapServer manages snapshot ID allocation, snapshot deletion and
+  tracks list of effective snapshots in the file system. A file system only has
+  one instance of snapserver.
+* SnapClient: SnapClient is used to communicate with snapserver, each MDS rank
+  has its own snapclient instance. SnapClient also caches effective snapshots
+  locally.
+
+Creating a snapshot
+-------------------
+CephFS snapshot feature is enabled by default on new file system. To enable it
+on existing file systems, use command below.
+
+.. code::
+
+       $ ceph fs set <fs_name> allow_new_snaps true
+
+When snapshots are enabled, all directories in CephFS will have a special
+``.snap`` directory. (You may configure a different name with the ``client
+snapdir`` setting if you wish.)
+
+To create a CephFS snapshot, create a subdirectory under
+``.snap`` with a name of your choice. For example, to create a snapshot on
+directory "/1/2/3/", invoke ``mkdir /1/2/3/.snap/my-snapshot-name`` .
+
+.. note::
+   Snapshot names can not start with an underscore ('_'), as these names are
+   reserved for internal usage.
+
+.. note::
+   Snapshot names can not exceed 240 characters.  This is because the MDS makes
+   use of long snapshot names internally, which follow the format:
+   `_<SNAPSHOT-NAME>_<INODE-NUMBER>`.  Since filenames in general can't have
+   more than 255 characters, and `<node-id>` takes 13 characters, the long
+   snapshot names can take as much as 255 - 1 - 1 - 13 = 240.
+
+This is transmitted to the MDS Server as a
+CEPH_MDS_OP_MKSNAP-tagged `MClientRequest`, and initially handled in
+Server::handle_client_mksnap(). It allocates a `snapid` from the `SnapServer`,
+projects a new inode with the new SnapRealm, and commits it to the MDLog as
+usual. When committed, it invokes
+`MDCache::do_realm_invalidate_and_update_notify()`, which notifies all clients
+with caps on files under "/1/2/3/", about the new SnapRealm. When clients get
+the notifications, they update client-side SnapRealm hierarchy, link files
+under "/1/2/3/" to the new SnapRealm and generate a `SnapContext` for the
+new SnapRealm.
+
+Note that this *is not* a synchronous part of the snapshot creation!
+
+Updating a snapshot
+-------------------
+If you delete a snapshot, a similar process is followed. If you remove an inode
+out of its parent SnapRealm, the rename code creates a new SnapRealm for the
+renamed inode (if SnapRealm does not already exist), saves IDs of snapshots that
+are effective on the original parent SnapRealm into `past_parent_snaps` of the
+new SnapRealm, then follows a process similar to creating snapshot.
+
+Generating a SnapContext
+------------------------
+A RADOS `SnapContext` consists of a snapshot sequence ID (`snapid`) and all
+the snapshot IDs that an object is already part of. To generate that list, we
+combine `snapids` associated with the SnapRealm and all valid `snapids` in
+`past_parent_snaps`. Stale `snapids` are filtered out by SnapClient's cached
+effective snapshots.
+
+Storing snapshot data
+---------------------
+File data is stored in RADOS "self-managed" snapshots. Clients are careful to
+use the correct `SnapContext` when writing file data to the OSDs.
+
+Storing snapshot metadata
+-------------------------
+Snapshotted dentries (and their inodes) are stored in-line as part of the
+directory they were in at the time of the snapshot. *All dentries* include a
+`first` and `last` snapid for which they are valid. (Non-snapshotted dentries
+will have their `last` set to CEPH_NOSNAP).
+
+Snapshot writeback
+------------------
+There is a great deal of code to handle writeback efficiently. When a Client
+receives an `MClientSnap` message, it updates the local `SnapRealm`
+representation and its links to specific `Inodes`, and generates a `CapSnap`
+for the `Inode`. The `CapSnap` is flushed out as part of capability writeback,
+and if there is dirty data the `CapSnap` is used to block fresh data writes
+until the snapshot is completely flushed to the OSDs.
+
+In the MDS, we generate snapshot-representing dentries as part of the regular
+process for flushing them. Dentries with outstanding `CapSnap` data is kept
+pinned and in the journal.
+
+Deleting snapshots
+------------------
+Snapshots are deleted by invoking "rmdir" on the ".snap" directory they are
+rooted in. (Attempts to delete a directory which roots snapshots *will fail*;
+you must delete the snapshots first.) Once deleted, they are entered into the
+`OSDMap` list of deleted snapshots and the file data is removed by the OSDs.
+Metadata is cleaned up as the directory objects are read in and written back
+out again.
+
+Hard links
+----------
+Inode with multiple hard links is moved to a dummy global SnapRealm. The
+dummy SnapRealm covers all snapshots in the file system. The inode's data
+will be preserved for any new snapshot. These preserved data will cover
+snapshots on any linkage of the inode.
+
+Multi-FS
+---------
+Snapshots and multiple file systems don't interact well. Specifically, each
+MDS cluster allocates `snapids` independently; if you have multiple file systems
+sharing a single pool (via namespaces), their snapshots *will* collide and
+deleting one will result in missing file data for others. (This may even be
+invisible, not throwing errors to the user.) If each FS gets its own
+pool things probably work, but this isn't tested and may not be true.
+
+.. Note:: To avoid snap id collision between mon-managed snapshots and file system
+   snapshots, pools with mon-managed snapshots are not allowed to be attached
+   to a file system. Also, mon-managed snapshots can't be created in pools
+   already attached to a file system either.
diff --git a/doc/dev/cephx.rst b/doc/dev/cephx.rst
new file mode 100644
index 000000000..e060f7ec8
--- /dev/null
+++ b/doc/dev/cephx.rst
@@ -0,0 +1,406 @@
+=====
+Cephx
+=====
+
+.. _cephx:
+
+Intro
+-----
+
+The protocol design looks a lot like kerberos.  The authorizer "KDC"
+role is served by the monitor, who has a database of shared secrets
+for each entity.  Clients and non-monitor daemons all start by
+authenticating with the monitor to obtain tickets, mostly referred to
+in the code as authorizers.  These tickets provide both
+*authentication* and *authorization* in that they include a
+description of the *capabilities* for the entity, a concise structured
+description of what actions are allowed, that can be interpreted and
+enforced by the service daemons.
+
+Other references
+----------------
+
+- A write-up from 2012 on cephx as it existed at that time by Peter
+  Reiher: :ref:`cephx_2012_peter`
+
+Terms
+-----
+
+- *monitor(s)*: central authorization authority
+- *service*: the set of all daemons of a particular type (e.g., all
+  OSDs, all MDSs)
+- *client*: an entity or principal that is accessing the service
+- *entity name*: the string identifier for a principal
+  (e.g. client.admin, osd.123)
+- *ticket*: a bit of data that cryptographically asserts identify and
+  authorization
+
+- *principal*: a client or daemon, identified by a unique entity_name,
+  that shares a secret with the monitor.
+- *principal_secret*: principal secret, a shared secret (16 bytes)
+  known by the principal and the monitor
+- *mon_secret*: monitor secret, a shared secret known by all monitors
+- *service_secret*: a rotating secret known by all members of a
+  service class (e.g., all OSDs)
+
+- *auth ticket*: a ticket proving identity to the monitors
+- *service ticket*: a ticket proving identify and authorization to a
+  service
+
+  
+Terminology
+-----------
+
+``{foo, bar}^secret`` denotes encryption by secret.
+
+
+Context
+-------
+
+The authentication messages described here are specific to the cephx
+auth implementation.  The messages are transferred by the Messenger
+protocol or by MAuth messages, depending on the version of the
+messenger protocol.  See also :ref:`msgr2-protocol`.
+
+An initial (messenger) handshake negotiates an authentication method
+to be used (cephx vs none or krb or whatever) and an assertion of what
+entity the client or daemon is attempting to authenticate as.
+
+Phase I: obtaining auth ticket
+------------------------------
+
+The cephx exchange begins with the monitor knowing who the client
+claims to be, and an initial cephx message from the monitor to the
+client/principal.::
+
+  a->p : 
+    CephxServerChallenge {
+      u64 server_challenge     # random (by server)
+    }
+
+The client responds by adding its own challenge, and calculating a
+value derived from both challenges and its shared key
+principal_secret.::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_AUTH_SESSION_KEY
+    }
+    CephXAuthenticate {
+      u8 2                     # 2 means nautilus+
+      u64 client_challenge     # random (by client)
+      u64 key = {client_challenge ^ server_challenge}^principal_secret   # (roughly)
+      blob old_ticket          # old ticket, if we are reconnecting or renewing
+      u32 other_keys           # bit mask of service keys we want
+    }
+
+Prior to nautilus,::
+
+    CephXAuthenticate {
+      u8 1                     # 2 means nautilus+
+      u64 client_challenge     # random (by client)
+      u64 key = {client_challenge + server_challenge}^principal_secret   # (roughly)
+      blob old_ticket          # old ticket, if we are reconnecting or renewing
+    }
+
+The monitor looks up principal_secret in database, and verifies the
+key is correct.  If old_ticket is present, verify it is valid, and we
+can reuse the same global_id.  (Otherwise, a new global_id is assigned
+by the monitor.)::
+
+  a->p :
+    CephxReplyHeader {
+      u16 CEPHX_GET_AUTH_SESSION_KEY
+      s32 result (0)
+    }
+    u8 encoding_version = 1
+    u32 num_tickets ( = 1)
+    ticket_info           # (N = 1)
+
+plus (for Nautilus and later)::
+
+    u32 connection_secret_len      # in bytes
+    connection_secret^session_key
+    u32 other_keys_len             # bytes of other keys (encoded)
+    other_keys {
+      u8 encoding_version = 1
+      u32 num_tickets
+      service_ticket_info * N      # for each service ticket
+    }
+
+where::
+
+    ticket_info {
+      u32 service_id       # CEPH_ENTITY_TYPE_AUTH
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      {CephxTicketBlob ticket_blob}^existing session_key   # if we are renewing a ticket,
+      CephxTicketBlob ticket_blob                          # otherwise
+    }
+
+    service_ticket_info {
+      u32 service_id       # CEPH_ENTITY_TYPE_{OSD,MDS,MGR}
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      CephxTicketBlob ticket_blob
+    }
+
+    CephxServiceTicket {
+      CryptoKey session_key      # freshly generated (even if old_ticket is present)
+      utime_t expiration         # now + auth_mon_ticket_ttl
+    }
+
+    CephxTicketBlob {
+      u64 secret_id             # which service ticket encrypted this; -1 == monsecret, otherwise service's rotating key id
+      {CephXServiceTicketInfo ticket}^mon_secret
+    }
+
+    CephxServiceTicketInfo {
+      CryptoKey session_key     # same session_key as above
+      AuthTicket ticket
+    }
+
+    AuthTicket {
+      EntityName name           # client's identity, as proven by its possession of principal_secret
+      u64 global_id             # newly assigned, or from old_ticket
+      utime_t created, renew_after, expires
+      AuthCapsInfo       # what client is allowed to do
+      u32 flags = 0      # unused
+    }
+
+So: for each ticket, principal gets a part that it decrypts with its
+secret to get the session_key (CephxServiceTicket).  And the
+CephxTicketBlob is opaque (secured by the mon secret) but can be used
+later to prove who we are and what we can do (see CephxAuthorizer
+below).
+
+For Nautilus+, we also include the service tickets.
+
+The client can infer that the monitor is authentic because it can
+decrypt the service_ticket with its secret (i.e., the server has its
+secret key).
+
+
+Phase II: Obtaining service tickets (pre-nautilus)
+--------------------------------------------------
+
+Now the client needs the keys used to talk to non-monitors (osd, mds,
+mgr).::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_PRINCIPAL_SESSION_KEY
+    }
+    CephxAuthorizer authorizer      
+    CephxServiceTicketRequest {
+      u32 keys    # bitmask of CEPH_ENTITY_TYPE_NAME (MGR, OSD, MDS, etc)
+    }
+
+where::
+
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce                         # random from client
+      bool have_challenge = false       # not used here
+      u64 server_challenge_plus_one = 0 # not used here
+    }
+
+The monitor validates the authorizer by decrypting the auth_ticket
+with ``mon_secret`` and confirming that it says this principal is who
+they say they are in the CephxAuthorizer fields.  Note that the nonce
+random bytes aren't used here (the field exists for Phase III below).
+
+Assuming all is well, the authorizer can generate service tickets
+based on the CEPH_ENTITY_TYPE_* bits in the ``keys`` bitmask.
+
+The response looks like::
+
+    CephxResponseHeader {
+      u16 CEPHX_GET_PRINCIPAL_SESSION_KEY
+      s32 result (= 0)
+    }
+    u8 encoding_version = 1
+    u32 num_tickets
+    ticket_info * N
+  
+Where, as above,::
+
+    ticket_info {
+      u32 service_id      # CEPH_ENTITY_TYPE_{OSD,MGR,MDS}
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      CephxTicketBlob ticket_blob
+    }
+
+    CephxServiceTicket {
+      CryptoKey session_key
+      utime_t expiration
+    }
+
+    CephxTicketBlob {
+      u64 secret_id       # which version of the (rotating) service ticket encrypted this
+      {CephXServiceTicketInfo ticket}^rotating_service_secret
+    }
+
+    CephxServiceTicketInfo {
+      CryptoKey session_key
+      AuthTicket ticket
+    }
+
+    AuthTicket {
+      EntityName name
+      u64 global_id
+      utime_t created, renew_after, expires
+      AuthCapsInfo       # what you are allowed to do
+      u32 flags = 0      # unused
+    }
+
+This concludes the authentication exchange with the monitor.  The
+client or daemon now has tickets to talk to the mon and all other
+daemons of interest.
+
+
+Phase III: Opening a connection to a service
+--------------------------------------------
+
+When a connection is opened, an "authorizer" payload is sent::
+
+  p->s :
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce               # random from client
+      bool have_challenge = false
+      u64 server_challenge_plus_one = 0
+    }
+
+Note that prior to the Luminous v12.2.6 or Mimic v13.2.2 releases, the
+CephxAuthorize msg did not contain a challenge, and consisted only
+of::
+
+    CephxAuthorize msg {
+      u8 1
+      u64 nonce               # random from client
+    }
+    
+The server will inspect the auth_ticket CephxTicketBlob (by decrypting
+it with its current rotating service key).  If it is a pre-v12.2.6 or
+pre-v13.2.2 client, the server immediately replies with::
+
+  s->p :
+    {CephxAuthorizeReply reply}^session_key
+
+where::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+    }
+
+Otherwise, the server will respond with a challenge (to prevent replay
+attacks)::
+
+  s->p :
+    {CephxAuthorizeChallenge challenge}^session_key
+
+where::
+
+    CephxAuthorizeChallenge {
+      u64 server_challenge        # random from server
+    }
+
+The client decrypts and updates its CephxAuthorize msg accordingly,
+resending most of the same information as before::
+
+  p->s :
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+where::
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce                        # (new) random from client
+      bool have_challenge = true
+      u64 server_challenge_plus_one    # server_challenge + 1
+    }
+
+The server validates the ticket as before, and then also verifies the
+msg nonce has it's challenge + 1, confirming this is a live
+authentication attempt (not a replay).
+
+Finally, the server responds with a reply that proves its authenticity
+to the client.  It also includes some entropy to use for encryption of
+the session, if it is needed for the mode.::
+
+  s->p :
+    {CephxAuthorizeReply reply}^session_key
+
+where::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+      u32 connection_secret_length
+      connection secret
+    }
+
+Prior to nautilus, there is no connection secret::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+    }
+
+The client decrypts and confirms that the server incremented nonce
+properly and that this is thus a live authentication request and not a
+replay.
+
+
+Rotating service secrets
+------------------------
+
+Daemons make use of a rotating secret for their tickets instead of a
+fixed secret in order to limit the severity of a compromised daemon.
+If a daemon's secret key is compromised by an attacker, that daemon
+and its key can be removed from the monitor's database, but the
+attacker may also have obtained a copy of the service secret shared by
+all daemons.  To mitigate this, service keys rotate periodically so
+that after a period of time (auth_service_ticket_ttl) the key the
+attacker obtained will no longer be valid.::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_ROTATING_KEY
+    }
+
+  a->p :
+    CephxReplyHeader {
+      u16 CEPHX_GET_ROTATING_KEY
+      s32 result = 0
+    }
+    {CryptoKey service_key}^principal_secret
+
+That is, the new rotating key is simply protected by the daemon's
+rotating secret.
+
+Note that, as an implementation detail, the services keep the current
+key and the prior key on hand so that they can continue to validate
+requests while the key is being rotated.
diff --git a/doc/dev/cephx_protocol.rst b/doc/dev/cephx_protocol.rst
new file mode 100644
index 000000000..4b4a3a584
--- /dev/null
+++ b/doc/dev/cephx_protocol.rst
@@ -0,0 +1,341 @@
+.. _cephx_2012_peter:
+
+============================================================
+A Detailed Description of the Cephx Authentication Protocol
+============================================================
+
+Peter Reiher
+7/13/12
+
+This document provides deeper detail on the Cephx authorization protocol whose high level flow 
+is described in the memo by Yehuda (12/19/09).  Because this memo discusses details of 
+routines called and variables used, it represents a snapshot.  The code might be changed 
+subsequent to the creation of this document, and the document is not likely to be updated in
+lockstep.  With luck, code comments will indicate major changes in the way the protocol is
+implemented.
+
+Introduction
+-------------
+
+The basic idea of the protocol is based on Kerberos.  A client wishes to obtain something from 
+a server.  The server will only offer the requested service to authorized clients.  Rather 
+than requiring each server to deal with authentication and authorization issues, the system 
+uses an authorization server.  Thus, the client must first communicate with the authorization 
+server to authenticate itself and to obtain credentials that will grant it access to the
+service it wants.
+
+Authorization is not the same as authentication.  Authentication provides evidence that some 
+party is who it claims to be.  Authorization provides evidence that a particular party is
+allowed to do something.  Generally, secure authorization implies secure authentication 
+(since without authentication, you may authorize something for an imposter), but the reverse 
+is not necessarily true.  One can authenticate without authorizing.  The purpose 
+of this protocol is to authorize.
+
+The basic approach is to use symmetric cryptography throughout.  Each client C has its own
+secret key, known only to itself and the authorization server A.  Each server S has its own
+secret key, known only to itself and the authorization server A.  Authorization information 
+will be passed in tickets, encrypted with the secret key of the entity that offers the service.
+There will be a ticket that A gives to C, which permits C to ask A for other tickets.  This 
+ticket will be encrypted with A's key, since A is the one who needs to check it.  There will 
+later be tickets that A issues that allow C to communicate with S to ask for service.  These 
+tickets will be encrypted with S's key, since S needs to check them.   Since we wish to provide 
+security of the communications, as well, session keys are set up along with the tickets.  
+Currently, those session keys are only used for authentication purposes during this protocol 
+and the handshake between the client C and the server S, when the client provides its service 
+ticket.  They could be used for authentication or secrecy throughout, with some changes to 
+the system.
+
+Several parties need to prove something to each other if this protocol is to achieve its 
+desired security effects.
+
+1.  The client C must prove to the authenticator A that it really is C.  Since everything
+is being done via messages, the client must also prove that the message proving authenticity
+is fresh, and is not being replayed by an attacker.
+
+2.  The authenticator A must prove to client C that it really is the authenticator.  Again,
+proof that replay is not occurring is also required.
+
+3.  A and C must securely share a session key to be used for distribution of later
+authorization material between them.  Again, no replay is allowable, and the key must be
+known only to A and C.
+
+4.  A must receive evidence from C that allows A to look up C's authorized operations with
+server S.  
+
+5.  C must receive a ticket from A that will prove to S that C can perform its authorized
+operations.   This ticket must be usable only by C.
+
+6.  C must receive from A a session key to protect the communications between C and S.  The
+session key must be fresh and not the result of a replay.
+
+Getting Started With Authorization
+-----------------------------------
+
+When the client first needs to get service, it contacts the monitor.  At the moment, it has 
+no tickets.  Therefore, it uses the "unknown" protocol to talk to the monitor.  This protocol 
+is specified as ``CEPH_AUTH_UNKNOWN``.  The monitor also takes on the authentication server 
+role, A.  The remainder of the communications will use the cephx protocol (most of whose code 
+will be found in files in ``auth/cephx``).  This protocol is responsible for creating and 
+communicating the tickets spoken of above.  
+
+Currently, this document does not follow the pre-cephx protocol flow.  It starts up at the 
+point where the client has contacted the server and is ready to start the cephx protocol itself.
+
+Once we are in the cephx protocol, we can get the tickets.  First, C needs a ticket that 
+allows secure communications with A.  This ticket can then be used to obtain other tickets. 
+This is phase I of the protocol, and consists of a send from C to A and a response from A to C.
+Then, C needs a ticket to allow it to talk to S to get services.  This is phase II of the 
+protocol, and consists of a send from C to A and a response from A to C.
+
+Phase I:
+--------
+
+The client is set up to know that it needs certain things, using a variable called ``need``, 
+which is part of the ``AuthClientHandler`` class, which the ``CephxClientHandler`` inherits 
+from.  At this point, one thing that's encoded in the ``need`` variable is 
+``CEPH_ENTITY_TYPE_AUTH``, indicating that we need to start the authentication protocol 
+from scratch.  Since we're always talking to the same authorization server, if we've gone 
+through this step of the protocol before (and the resulting ticket/session hasn't timed out), 
+we can skip this step and just ask for client tickets.  But it must be done initially, and 
+we'll assume that we are in that state.
+
+The message C sends to A in phase I is build in ``CephxClientHandler::build_request()`` (in 
+``auth/cephx/CephxClientHandler.cc``).  This routine is used for more than one purpose.  
+In this case, we first call ``validate_tickets()`` (from routine 
+``CephXTicketManager::validate_tickets()`` which lives in ``auth/cephx/CephxProtocol.h``).  
+This code runs through the list of possible tickets to determine what we need, setting values 
+in the ``need`` flag as necessary.  Then we call ``ticket.get_handler()``.  This routine 
+(in ``CephxProtocol.h``) finds a ticket of the specified type (a ticket to perform 
+authorization) in the ticket map, creates a ticket handler object for it,  and puts the 
+handler into the right place in the map.  Then we hit specialized code to deal with individual 
+cases.  The case here is when we still need to authenticate to A (the 
+``if (need & CEPH_ENTITY_TYPE_AUTH)`` branch).
+
+We now create a message of type ``CEPHX_GET_AUTH_SESSION_KEY``.  We need to authenticate
+this message with C's secret key, so we fetch that from the local key repository.  We create
+a random challenge, whose purpose is to prevent replays.  We encrypt that challenge using
+``cephx_calc_client_server_challenge()``.  We already
+have a server challenge (a similar set of random bytes, but created by the server and sent to
+the client) from our pre-cephx stage.  We take both challenges and our secret key and 
+produce a combined encrypted challenge value, which goes into ``req.key``.
+
+If we have an old ticket, we store it in ``req.old_ticket``.  We're about to get a new one.
+
+The entire ``req`` structure, including the old ticket and the cryptographic hash of the two 
+challenges, gets put into the message.  Then we return from this function, and the 
+message is sent.
+
+We now switch over to the authenticator side, A.  The server receives the message that was 
+sent, of type ``CEPH_GET_AUTH_SESSION_KEY``.  The message gets handled in ``prep_auth()``,
+in ``mon/AuthMonitor.cc``, which calls ``handle_request()`` is ``CephxServiceHandler.cc`` to 
+do most of the work.  This routine, also, handles multiple cases.  
+
+The control flow is determined by the ``request_type`` in the ``cephx_header`` associated 
+with the message.  Our case here is ``CEPH_GET_AUTH_SESSION_KEY``.  We need the
+secret key A shares with C, so we call ``get_secret()`` from out local key repository to get 
+it. (It's called a ``key_server`` in the code, but it's not really a separate machine or
+processing entity. It's more like the place where locally used keys are kept.)  We should
+have set up a server challenge already with this client, so we make sure
+we really do have one.  (This variable is specific to a ``CephxServiceHandler``, so there 
+is a different one for each such structure we create, presumably one per client A is 
+dealing with.)  If there is no challenge, we'll need to start over, since we need to 
+check the client's crypto hash, which depends on a server challenge, in part.
+
+We now call the same routine the client used to calculate the hash, based on the same values: 
+the client challenge (which is in the incoming message), the server challenge (which we saved), 
+and the client's key (which we just obtained).  We check to see if the client sent the same 
+thing we expected.  If so, we know we're talking to the right client.  We know the session is 
+fresh, because it used the challenge we sent it to calculate its crypto hash.  So we can
+give it an authentication ticket.
+
+We fetch C's ``eauth`` structure.  This contains an ID, a key, and a set of caps (capabilities).
+
+The client sent us its old ticket in the message, if it had one.  If
+so, we set a flag, ``should_enc_ticket``, to true and set the global
+ID to the global ID in that old ticket.  If the attempt to decode its
+old ticket fails (most probably because it didn't have one),
+``should_enc_ticket`` remains false.  Now we set up the new ticket,
+filling in timestamps, the name of C, and the global ID provided in the
+method call (unless there was an old ticket).  We need a new session
+key to help the client communicate securely with us, not using its
+permanent key.  We set the service ID to ``CEPH_ENTITY_TYPE_AUTH``,
+which will tell the client C what to do with the message we send it.
+We build a cephx response header and call
+``cephx_build_service_ticket_reply()``.
+
+``cephx_build_service_ticket_reply()`` is in ``auth/cephx/CephxProtocol.cc``.  This 
+routine will build up the response message.   Much of it copies data from its parameters to 
+a message structure.  Part of that information (the session key and the validity period) 
+gets encrypted with C's permanent key.  If the ``should_encrypt_ticket`` flag is set, 
+encrypt it using the old ticket's key.  Otherwise, there was no old ticket key, so the 
+new ticket is not encrypted.  (It is, of course, already encrypted with A's permanent key.)  
+Presumably the point of this second encryption is to expose less material encrypted with 
+permanent keys.
+
+Then we call the key server's ``get_service_caps()`` routine on the entity name, with a 
+flag ``CEPH_ENTITY_TYPE_MON``, and capabilities, which will be filled in by this routine.  
+The use of that constant flag means we're going to get the client's caps for A, not for some 
+other data server.  The ticket here is to access the authorizer A, not the service S.  The 
+result of this call is that the caps variable  (a parameter to the routine we're in) is 
+filled in with the monitor capabilities that will allow C to  access A's authorization services.
+
+``handle_request()`` itself does not send the response message.  It builds up the 
+``result_bl``, which basically holds that message's contents, and the capabilities structure, 
+but it doesn't send the message.  We go back to ``prep_auth()``, in ``mon/AuthMonitor.cc``, 
+for that.    This routine does some fiddling around with the caps structure that just got 
+filled in.  There's a global ID that comes up as a result of this fiddling that is put into 
+the reply message.  The reply message is built here (mostly from the ``response_bl`` buffer) 
+and sent off.
+
+This completes Phase I of the protocol.  At this point, C has authenticated itself to A, and A has generated a new session key and ticket allowing C to obtain server tickets from A.
+
+Phase II
+--------
+
+This phase starts when C receives the message from A containing a new ticket and session key.
+The goal of this phase is to provide C with a session key and ticket allowing it to
+communicate with S.
+
+The message A sent to C is dispatched to ``build_request()`` in ``CephxClientHandler.cc``, 
+the same routine that was used early in Phase I to build the first message in the protocol.  
+This time, when ``validate_tickets()`` is called, the ``need`` variable will not contain 
+``CEPH_ENTITY_TYPE_AUTH``, so a different branch through the bulk of the routine will be 
+used.  This is the branch indicated by ``if (need)``.  We have a ticket for the authorizer, 
+but we still need service tickets.
+
+We must send another message to A to obtain the tickets (and session key) for the server 
+S.  We set the ``request_type`` of the message to ``CEPHX_GET_PRINCIPAL_SESSION_KEY`` and 
+call ``ticket_handler.build_authorizer()`` to obtain an authorizer.  This routine is in 
+``CephxProtocol.cc``.  We set the key for this authorizer to be the session key we just got 
+from A,and create a new nonce.  We put the global ID, the service ID, and the ticket into a 
+message buffer that is part of the authorizer.  Then we create a new ``CephXAuthorize`` 
+structure.  The nonce we just created goes there.  We encrypt this ``CephXAuthorize`` 
+structure with the current session key and stuff it into the authorizer's buffer.  We 
+return the authorizer.
+
+Back in ``build_request()``, we take the part of the authorizer that was just built (its 
+buffer, not the session key or anything else) and shove it into the buffer we're creating 
+for the message that will go to A.  Then we delete the authorizer.  We put the requirements 
+for what we want in ``req.keys``, and we put ``req`` into the buffer.  Then we return, and 
+the message gets sent.
+
+The authorizer A receives this message which is of type ``CEPHX_GET_PRINCIPAL_SESSION_KEY``.
+The message gets handled in ``prep_auth()``, in ``mon/AuthMonitor.cc``, which again calls 
+``handle_request()`` in ``CephxServiceHandler.cc`` to do most of the work.  
+
+In this case, ``handle_request()`` will take the ``CEPHX_GET_PRINCIPAL_SESSION_KEY`` case. 
+It will call ``cephx_verify_authorizer()`` in ``CephxProtocol.cc``.  Here, we will grab 
+a bunch of data out of the input buffer, including the global and service IDs and the ticket 
+for A.   The ticket contains a ``secret_id``, indicating which key is being used for it.     
+If the secret ID pulled out of the ticket was -1, the ticket does not specify which secret 
+key A should use.  In this case, A should use the key for the specific entity that C wants
+to contact, rather than a rotating key shared by all server entities of the same type.
+To get that key, A must consult the key repository to find the right key.   Otherwise, 
+there's already a structure obtained from the key repository to hold the necessary secret.  
+Server secrets rotate on a time expiration basis (key rotation is not covered in this
+document), so run through that structure to find its current secret.  Either way, A now 
+knows the secret key used to create this ticket.  Now decrypt the encrypted part of the 
+ticket, using this key.  It should be a ticket for A.  
+
+The ticket also contains a session key that C should have used to encrypt other parts of 
+this message.  Use that session key to decrypt the rest of the message.  
+
+Create a ``CephXAuthorizeReply`` to hold our reply.  Extract the nonce (which was in the stuff 
+we just decrypted), add 1 to it, and put the result in the reply.  Encrypt the reply and 
+put it in the buffer provided in the call to ``cephx_verify_authorizer()`` and return 
+to ``handle_request()``.  This will be used to prove to C that A (rather than an attacker) 
+created this response.
+
+Having verified that the message is valid and from C, now we need to build it a ticket for S.
+We need to know what S it wants to communicate with and what services it wants.  Pull the
+ticket request that describes those things out of its message.  Now run through the ticket
+request to see what it wanted.  (He could potentially be asking for multiple different
+services in the same request, but we will assume it's just one, for this discussion.)  Once we 
+know which service ID it's after, call ``build_session_auth_info()``.
+
+``build_session_auth_info()`` is in ``CephxKeyServer.cc``.  It checks to see if the 
+secret for the ``service_ID`` of S is available and puts it into the subfield of one of 
+the parameters, and calls the similarly named ``_build_session_auth_info()``, located in 
+the same file.      This routine loads up the new ``auth_info`` structure with the 
+ID of S, a ticket, and some timestamps for that ticket.  It generates a new session key 
+and puts it in the structure.   It then calls ``get_caps()`` to fill in the 
+``info.ticket`` caps field.  ``get_caps()`` is also in ``CephxKeyServer.cc``.  It fills the 
+``caps_info`` structure it is provided with caps for S allowed to C.
+
+Once ``build_session_auth_info()`` returns, A has a list of the capabilities allowed to 
+C for S.  We put a validity period based on the current TTL for this context into the info 
+structure, and put it into the ``info_vec`` structure we are preparing in response to the 
+message.  
+
+Now call ``build_cephx_response_header()``, also in ``CephxServiceHandler.cc``.   Fill in 
+the ``request_type``, which is ``CEPHX_GET_PRINCIPAL_SESSION_KEY``, a status of 0, 
+and the result buffer.  
+
+Now call ``cephx_build_service_ticket_reply()``, which is in ``CephxProtocol.cc``.  The 
+same routine was used towards the end of A's handling of its response in phase I.  Here, 
+the session key (now a session key to talk to S, not A) and the validity period for that 
+key will be encrypted with the existing session key shared between C and A.  
+The ``should_encrypt_ticket`` parameter is false here, and no key is provided for that 
+encryption.  The ticket in question, destined for S once C sends it there, is already 
+encrypted with S's secret.  So, essentially, this routine will put ID information, 
+the encrypted session key, and the ticket allowing C to talk to S into the buffer to 
+be sent to C.
+
+After this routine returns, we exit from ``handle_request()``, going back to ``prep_auth()`` 
+and ultimately to the underlying message send code.  
+
+The client receives this message. The nonce is checked as the message passes through
+``Pipe::connect()``, which is in ``msg/SimpleMessager.cc``.  In a lengthy ``while(1)`` loop in
+the middle of this routine, it gets an authorizer.  If the get was successful, eventually
+it will call ``verify_reply()``, which checks the nonce.  ``connect()`` never explicitly
+checks to see if it got an authorizer, which would suggest that failure to provide an
+authorizer would allow an attacker to skip checking of the nonce.  However, in many places,
+if there is no authorizer, important connection fields will get set to zero, which will
+ultimately cause the connection to fail to provide data.  It would be worth testing, but
+it looks like failure to provide an authorizer, which contains the nonce, would not be helpful
+to an attacker.
+
+The message eventually makes its way through to ``handle_response()``, in 
+``CephxClientHandler.cc``.    In this routine, we call ``get_handler()`` to get a ticket 
+handler to hold the ticket we have just received.  This routine is embedded in the definition 
+for a ``CephXTicketManager`` structure.  It takes a type (``CEPH_ENTITY_TYPE_AUTH``, in 
+this case) and looks through the ``tickets_map`` to find that type.  There should be one, and 
+it should have the session key of the session between C and A in its entry.  This key will 
+be used to decrypt the information provided by A, particularly the new session key allowing 
+C to talk to S.
+
+We then call ``verify_service_ticket_reply()``, in ``CephxProtocol.cc``.  This routine 
+needs to determine if the ticket is OK and also obtain the session key associated with this 
+ticket.  It decrypts the encrypted portion of the message buffer, using the session key 
+shared with A.  This ticket was not encrypted (well, not twice - tickets are always encrypted, 
+but sometimes double encrypted, which this one isn't).  So it can be stored in a service 
+ticket buffer directly.  We now grab the ticket out of that buffer.  
+
+The stuff we decrypted with the session key shared between C and A included the new session 
+key.  That's our current session key for this ticket, so set it.  Check validity and 
+set the expiration times.  Now return true, if we got this far.  
+
+Back in ``handle_response()``, we now call ``validate_tickets()`` to adjust what we think 
+we need, since we now have a ticket we didn't have before.  If we've taken care of 
+everything we need, we'll return 0.
+
+This ends phase II of the protocol.  We have now successfully set up a ticket and session key 
+for client C to talk to server S.  S will know that C is who it claims to be, since A will
+verify it.  C will know it is S it's talking to, again because A verified it.  The only
+copies of the session key for C and S to communicate were sent encrypted under the permanent
+keys of C and S, respectively, so no other party (excepting A, who is trusted by all) knows
+that session key.  The ticket will securely indicate to S what C is allowed to do, attested 
+to by A.  The nonces passed back and forth between A and C ensure that they have not been 
+subject to a replay attack.  C has not yet actually talked to S, but it is ready to.
+
+Much of the security here falls apart if one of the permanent keys is compromised.  Compromise
+of C's key means that the attacker can pose as C and obtain all of C's privileges, and can
+eavesdrop on C's legitimate conversations.  He can also pretend to be A, but only in 
+conversations with C.  Since it does not (by hypothesis) have keys for any services, he
+cannot generate any new tickets for services, though it can replay old tickets and session
+keys until S's permanent key is changed or the old tickets time out. 
+
+Compromise of S's key means that the attacker can pose as S to anyone, and can eavesdrop on 
+any user's conversation with S.  Unless some client's key is also compromised, the attacker
+cannot generate new fake client tickets for S, since doing so requires it to authenticate
+himself as A, using the client key it doesn't know.
diff --git a/doc/dev/config-key.rst b/doc/dev/config-key.rst
new file mode 100644
index 000000000..d7b79db2f
--- /dev/null
+++ b/doc/dev/config-key.rst
@@ -0,0 +1,68 @@
+===================
+ config-key layout
+===================
+
+*config-key* is a general-purpose key/value storage service offered by
+the mons.  Generally speaking, you can put whatever you want there.
+Current in-tree users should be captured here with their key layout
+schema.
+
+OSD dm-crypt keys
+=================
+
+Key::
+
+  dm-crypt/osd/$OSD_UUID/luks = <json string>
+
+The JSON payload has the form::
+
+  { "dm-crypt": <secret> }
+
+where the secret is a base64 encoded LUKS key.
+
+Created by the 'osd new' command (see OSDMonitor.cc).
+
+Consumed by ceph-volume, and similar tools. Normally access to the
+dm-crypt/osd/$OSD_UUID prefix is allowed by a client.osd-lockbox.$OSD_UUID
+cephx key, such that only the appropriate host can retrieve the LUKS key (which
+in turn decrypts the actual raw key, also stored on the device itself).
+
+
+ceph-mgr modules
+================
+
+The convention for keys is::
+
+  mgr/$MODULE/$option = $value
+
+or::
+
+  mgr/$MODULE/$MGRID/$option = $value
+
+For example,::
+
+  mgr/dashboard/server_port = 80
+  mgr/dashboard/foo/server_addr = 1.2.3.4
+  mgr/dashboard/bar/server_addr = 1.2.3.5
+
+
+Configuration
+=============
+
+Configuration options for clients and daemons are also stored in config-key.
+
+Keys take the form::
+
+  config/$option = $value
+  config/$type/$option = $value
+  config/$type.$id/$option = $value
+  config/$type.$id/$mask[/$mask2...]/$option = $value
+
+Where
+
+* `type` is a daemon type (`osd`, `mon`, `mds`, `mgr`, `client`)
+* `id` is a daemon id (e.g., `0`, `foo`), such that `$type.$id` is something like `osd.123` or `mds.foo`)
+* `mask` restricts who the option applies to, and can take two forms:
+
+  #. `$crush_type:$crush_value`.  For example, `rack:foorack`
+  #. `class:$classname`, in reference to CRUSH device classes (e.g., `ssd`)
diff --git a/doc/dev/config.rst b/doc/dev/config.rst
new file mode 100644
index 000000000..9cb20aee7
--- /dev/null
+++ b/doc/dev/config.rst
@@ -0,0 +1,283 @@
+=================================
+ Configuration Management System
+=================================
+
+The configuration management system exists to provide every daemon with the
+proper configuration information. The configuration can be viewed as a set of
+key-value pairs.
+
+How can the configuration be set? Well, there are several sources:
+
+ - the ceph configuration file, usually named ceph.conf
+ - command line arguments::
+
+    --debug-ms=1
+    --debug-monc=10
+
+   etc.
+ - arguments injected at runtime using ``injectargs`` or ``config set``
+
+
+The Configuration File
+======================
+
+Most configuration settings originate in the Ceph configuration file.
+
+How do we find the configuration file? Well, in order, we check:
+
+ - the default locations
+ - the environment variable ``CEPH_CONF``
+ - the command line argument ``-c``
+
+Each stanza of the configuration file describes the key-value pairs that will be in
+effect for a particular subset of the daemons. The "global" stanza applies to
+everything. The "mon", "osd", and "mds" stanzas specify settings to take effect
+for all monitors, all OSDs, and all mds servers, respectively.  A stanza of the
+form ``mon.$name``, ``osd.$name``, or ``mds.$name`` gives settings for the monitor, OSD, or
+MDS of that name, respectively. Configuration values that appear later in the
+file win over earlier ones.
+
+A sample configuration file can be found in src/sample.ceph.conf.
+
+
+Metavariables
+=============
+
+The configuration system allows any configuration value to be
+substituted into another value using the ``$varname`` syntax, similar
+to how bash shell expansion works.
+
+A few additional special metavariables are also defined:
+
+ - $host: expands to the current hostname
+ - $type: expands to one of "mds", "osd", "mon", or "client"
+ - $id: expands to the daemon identifier. For ``osd.0``, this would be ``0``; for ``mds.a``, it would be ``a``; for ``client.admin``, it would be ``admin``.
+ - $num: same as $id
+ - $name: expands to $type.$id
+
+
+Reading configuration values
+====================================================
+
+There are two ways for Ceph code to get configuration values. One way is to
+read it directly from a variable named ``g_conf``, or equivalently,
+``g_ceph_ctx->_conf``. The other is to register an observer that will be called
+every time the relevant configuration values change. This observer will be
+called soon after the initial configuration is read, and every time after that
+when one of the relevant values changes. Each observer tracks a set of keys
+and is invoked only when one of the relevant keys changes.
+
+The interface to implement is found in ``common/config_obs.h``.
+
+The observer method should be preferred in new code because
+
+ - It is more flexible, allowing the code to do whatever reinitialization needs
+   to be done to implement the new configuration value.
+ - It is the only way to create a std::string configuration variable that can
+   be changed by injectargs.
+ - Even for int-valued configuration options, changing the values in one thread
+   while another thread is reading them can lead to subtle and
+   impossible-to-diagnose bugs.
+
+For these reasons, reading directly from ``g_conf`` should be considered deprecated
+and not done in new code.  Do not ever alter ``g_conf``.
+
+Changing configuration values
+====================================================
+
+Configuration values can be changed by calling ``g_conf()->set_val``. After changing
+the configuration, you should call ``g_conf()->apply_changes`` to re-run all the
+affected configuration observers. For convenience, you can call
+``g_conf()->set_val_or_die`` to make a configuration change which you think should
+never fail.
+
+``injectargs``, ``parse_argv``, and ``parse_env`` are three other functions which modify
+the configuration. Just like with set_val, you should call apply_changes after
+calling these functions to make sure your changes get applied.
+
+
+Defining config options
+=======================
+
+Config options are defined in ``common/options/*.yaml.in``. The options are categorized
+by their consumers. If an option is only used by ceph-osd, it should go to
+``osd.yaml.in``. All the ``.yaml.in`` files are translated into ``.cc`` and ``.h`` files
+at build time by ``y2c.py``.
+
+Each option is represented using a YAML mapping (dictionary). A typical option looks like
+
+.. code-block:: yaml
+
+   - name: public_addr
+     type: addr
+     level: basic
+     desc: public-facing address to bind to
+     long_desc: The IP address for the public (front-side) network.
+       Set for each daemon.
+     services:
+     - mon
+     - mds
+     - osd
+     - mgr
+     flags:
+     - startup
+     with_legacy: true
+
+In which, following keys are allowed:
+
+level
+-----
+
+The ``level`` property of an option is an indicator for the probability the
+option is adjusted by an operator or a developer:
+
+.. describe:: basic
+
+   for basic config options that a normal operator is likely to adjust.
+
+.. describe:: advanced
+
+   for options that an operator *can* adjust, but should not touch unless they
+   understand what they are doing. Adjusting advanced options poorly can lead to
+   problems (performance or even data loss) if done incorrectly.
+
+.. describe:: dev
+
+   for options in place for use by developers only, either for testing purposes,
+   or to describe constants that no user should adjust but we prefer not to compile
+   into the code.
+
+``desc``, ``long_desc`` and ``fmt_desc``
+----------------------------------------
+
+.. describe:: desc
+
+   Short description of the option. Sentence fragment. e.g.
+
+   .. code-block:: yaml
+
+      desc: Default checksum algorithm to use
+
+.. describe:: long_desc
+
+   The long description is complete sentences, perhaps even multiple
+   paragraphs, and may include other detailed information or notes. e.g.
+
+   .. code-block:: yaml
+
+      long_desc: crc32c, xxhash32, and xxhash64 are available.  The _16 and _8 variants use
+        only a subset of the bits for more compact (but less reliable) checksumming.
+
+.. describe:: fmt_desc
+
+   The description formatted using reStructuredText. This property is
+   only used by the ``confval`` directive to render an option in the
+   document. e.g.:
+
+   .. code-block:: yaml
+
+      fmt_desc: The interval for "deep" scrubbing (fully reading all data). The
+        ``osd_scrub_load_threshold`` does not affect this setting.
+
+Default values
+--------------
+
+There is a default value for every config option. In some cases, there may
+also be a *daemon default* that only applies to code that declares itself
+as a daemon (in this case, the regular default only applies to non-daemons). Like:
+
+.. code-block:: yaml
+
+   default: crc32c
+
+Some literal postfixes are allowed when options with type of ``float``, ``size``
+and ``secs``, like:
+
+.. code-block:: yaml
+
+   - name: mon_scrub_interval
+     type: secs
+     default: 1_day
+   - name: osd_journal_size
+     type: size
+     default: 5_K
+
+For better readability, it is encouraged to use these literal postfixes when
+adding or updating the default value for an option.
+
+Service
+-------
+
+Service is a component name, like "common", "osd", "rgw", "mds", etc. It may
+be a list of components, like:
+
+.. code-block:: yaml
+
+   services:
+   - mon
+   - mds
+   - osd
+   - mgr
+
+For example, the rocksdb options affect both the osd and mon. If an option is put
+into a service specific ``.yaml.in`` file, the corresponding service is added to
+its ``services`` property automatically. For instance, ``osd_scrub_begin_hour``
+option is located in ``osd.yaml.in``, even its ``services`` is not specified
+explicitly in this file, this property still contains ``osd``.
+
+Tags
+----
+
+Tags identify options across services that relate in some way. For example:
+
+network
+  options affecting network configuration
+mkfs
+  options that only matter at mkfs time
+
+Like:
+
+.. code-block:: yaml
+
+   tags:
+   - network
+
+Enums
+-----
+
+For options with a defined set of allowed values:
+
+.. code-block:: yaml
+
+   enum_values:
+   - none
+   - crc32c
+   - crc32c_16
+   - crc32c_8
+   - xxhash32
+   - xxhash64
+
+Flags
+-----
+
+.. describe:: runtime
+
+   the value can be updated at runtime
+
+.. describe:: no_mon_update
+
+   Daemons/clients do not pull this value from the monitor config database.  We
+   disallow setting this option via ``ceph config set ...``.  This option should
+   be configured via ``ceph.conf`` or via the command line.
+
+.. describe:: startup
+
+   option takes effect only during daemon startup
+
+.. describe:: cluster_create
+
+   option only affects cluster creation
+
+.. describe:: create
+
+   option only affects daemon creation
diff --git a/doc/dev/context.rst b/doc/dev/context.rst
new file mode 100644
index 000000000..1a2b2cbfb
--- /dev/null
+++ b/doc/dev/context.rst
@@ -0,0 +1,20 @@
+=============
+ CephContext
+=============
+
+A CephContext represents a single view of the Ceph cluster. It comes complete
+with a configuration, a set of performance counters (PerfCounters), and a
+heartbeat map. You can find more information about CephContext in
+src/common/ceph_context.h.
+
+Generally, you will have only one CephContext in your application, called
+g_ceph_context. However, in library code, it is possible that the library user
+will initialize multiple CephContexts. For example, this would happen if he
+called rados_create more than once.
+
+A ceph context is required to issue log messages. Why is this? Well, without
+the CephContext, we would not know which log messages were disabled and which
+were enabled.  The dout() macro implicitly references g_ceph_context, so it
+can't be used in library code.  It is fine to use dout and derr in daemons, but
+in library code, you must use ldout and lderr, and pass in your own CephContext
+object. The compiler will enforce this restriction.
diff --git a/doc/dev/continuous-integration.rst b/doc/dev/continuous-integration.rst
new file mode 100644
index 000000000..5c2f15823
--- /dev/null
+++ b/doc/dev/continuous-integration.rst
@@ -0,0 +1,285 @@
+Continuous Integration Architecture
+===================================
+
+In Ceph, we rely on multiple CI pipelines in our development. Most of these pipelines
+are centered around Jenkins. And their configurations are generated using `Jenkins Job Builder`_.
+
+.. _Jenkins Job Builder: https://docs.openstack.org/infra/jenkins-job-builder/
+
+Let's take the ``make check`` performed by Jenkins as an example.
+
+ceph-pull-requests
+------------------
+
+``ceph-pull-requests`` is a jenkins job which gets triggered by a GitHub pull
+request or a trigger phrase like::
+
+    jenkins test make check
+
+There are multiple parties involved in this jenkins job:
+
+.. graphviz::
+
+   digraph  {
+     rankdir="LR";
+     github [
+      label="<git> git_repo | <webhooks> webhooks | <api> api";
+      shape=record;
+      href="https://github.com/ceph/ceph";
+     ];
+     subgraph cluster_lab {
+       label="Sepia Lab";
+       href="https://wiki.sepia.ceph.com/doku.php";
+       shape=circle;
+       apt_mirror [
+         href="http://apt-mirror.front.sepia.ceph.com";
+       ];
+       shaman [
+         href="https://shaman.ceph.com";
+       ];
+       chacra [
+         peripheries=3;
+         href="https://chacra.ceph.com";
+       ];
+       subgraph cluster_jenkins {
+         label="jenkins";
+         href="https://jenkins.ceph.com";
+         jenkins_controller [ label = "controller" ];
+         jenkins_agents [ label = "agents", peripheries=3 ];
+       };
+     };
+     {
+       rank=same;
+       package_repos [ peripheries=3 ];
+       pypi;
+       npm;
+     }
+     github:webhooks -> jenkins_controller [ label = "notify", color = "crimson" ];
+     jenkins_controller -> jenkins_agents [ label = "schedule jobs" ];
+     jenkins_agents -> github:git [ label = "git pull" ];
+     jenkins_agents -> shaman [ label = "query for chacra repo URL" ];
+     jenkins_agents -> chacra [ label = "pull build dependencies" ];
+     jenkins_agents -> package_repos [ label = "pull build dependencies" ];
+     jenkins_agents -> pypi [ label = "pull Python packages" ];
+     jenkins_agents -> npm [ label = "pull JavaScript packages" ];
+     jenkins_agents -> apt_mirror [ label = "pull build dependencies" ];
+     jenkins_agents -> github:api [ label = "update", color = "crimson" ];
+   }
+
+Where
+
+Sepia Lab
+   `Sepia Lab`_ is a test lab used by the Ceph project. This lab offers
+   the storage and computing resources required by our CI infra.
+
+Jenkins agents
+   are a set of machines which perform the CI jobs. In this case, they
+
+   #. pull the git repo from GitHub and
+   #. rebase the pull request against the latest master
+   #. set necessary environment variables
+   #. run ``run-make-check.sh``
+
+Chacra
+   is a server offering RESTful API allowing the clients to store and
+   retrieve binary packages. It also creates the repo for uploaded
+   packages automatically. Once a certain repo is created on chacra, the
+   configured shaman server is updated as well, then we can query shaman
+   for the corresponding repo address. Chacra not only hosts Ceph packages,
+   it also hosts quite a few other packages like various build dependencies.
+
+Shaman
+   is a server offering RESTful API allowing the clients to query the
+   information of repos hosted by chacra nodes. Shaman is also known
+   for its `Web UI`_. But please note, shaman does not build the
+   packages, it just offers information on the builds.
+
+As the following shows, `chacra`_ manages multiple projects whose metadata
+are stored in a database. These metadata are exposed via Shaman as a web
+service. `chacractl`_ is a utility to interact with the `chacra`_ service.
+
+.. graphviz::
+
+   digraph  {
+     libboost [
+       shape=cylinder;
+     ];
+     libzbd [
+       shape=cylinder;
+     ];
+     other_repos [
+       label="...";
+       shape=cylinder;
+     ];
+     postgresql [
+       shape=cylinder;
+       style=filled;
+     ]
+     shaman -> postgresql;
+     chacra -> postgresql;
+     chacractl -> chacra;
+     chacra -> libboost;
+     chacra -> libzbd;
+     chacra -> other_repos;
+   }
+
+.. _Sepia Lab: https://wiki.sepia.ceph.com/doku.php
+.. _Web UI: https://shaman.ceph.com
+
+build dependencies
+------------------
+
+Just like lots of other software projects, Ceph has both build-time and
+run-time dependencies. Most of time, we are inclined to use the packages
+prebuilt by the distro. But there are cases where
+
+- the necessary dependencies are either missing in the distro, or
+- their versions are too old, or
+- they are packaged without some important feature enabled.
+- we want to ensure that the version of a certain runtime dependency is
+  identical to the one we tested in our lab.
+
+No matter what the reason is, we either need to build them from source, or
+to package them as binary packages instead of using the ones shipped by the
+distro. Quite a few build-time dependencies are included as git submodules,
+but in order to avoid rebuilding these dependencies repeatedly, we pre-built
+some of them and uploaded them to our own repos. So, when performing
+``make check``, the building hosts in our CI just pull them from our internal
+repos hosting these packages instead of building them.
+
+So far, following packages are prebuilt for ubuntu focal, and then uploaded to
+`chacra`_:
+
+libboost
+    packages `boost`_. The packages' names are changed from ``libboost-*`` to
+    ``ceph-libboost-*``, and they are instead installed into ``/opt/ceph``, so
+    they don't interfere with the official ``libboost`` packages shipped by
+    distro. Its build scripts are hosted at https://github.com/ceph/ceph-boost.
+    See https://github.com/ceph/ceph-boost/commit/2a8ae02932b2a1fd6a68072da8ca0df2b99b805c
+    for an example of how to bump the version number. The commands used to
+    build 1.79 on a vanilla Ubuntu Focal OS are below.
+
+    .. prompt:: bash $
+
+       sudo apt install debhelper dctrl-tools chrpath libbz2-dev libicu-dev bison \
+         flex docbook-to-man help2man xsltproc doxygen dh-python python3-all-dev graphviz
+       wget http://download.ceph.com/qa/boost_1_79_0.tar.bz2
+       git clone https://github.com/ceph/ceph-boost
+       tar xjf boost_1_79_0.tar.bz2
+       cp -ra ceph-boost/debian boost_1_79_0/
+       pushd boost_1_79_0
+       export DEB_BUILD_OPTIONS='parallel=6 nodoc'
+       dpkg-buildpackage -us -uc -b
+       popd
+       BOOST_SHA=$(git ls-remote https://github.com/ceph/ceph-boost main | awk '{ print $1 }')
+       ls *.deb | chacractl binary create \
+         libboost/master/$BOOST_SHA/ubuntu/focal/amd64/flavors/default
+
+libzbd
+    packages `libzbd`_ . The upstream libzbd includes debian packaging already.
+
+libpmem
+    packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
+    pmdk, for an updated debian packaging, please see
+    https://github.com/ceph/ceph-ndctl .
+
+.. note::
+
+   please ensure that the package version and the release number of the
+   packaging are properly updated when updating/upgrading the packaging,
+   otherwise it would be difficult to tell which version of the package
+   is installed. We check the package version before trying to upgrade
+   it in ``install-deps.sh``.
+
+.. _boost: https://www.boost.org
+.. _libzbd: https://github.com/westerndigitalcorporation/libzbd
+.. _pmdk: https://github.com/pmem/pmdk
+
+But in addition to these libraries, ``ceph-mgr-dashboard``'s frontend uses lots of
+JavaScript packages. Quite a few of them are not packaged by distros. Not to
+mention the trouble of testing different combination of versions of these
+packages. So we decided to include these JavaScript packages in our dist tarball
+using ``make-dist``.
+
+Also, because our downstream might not want to use the prepackaged binaries when
+redistributing the precompiled Ceph packages, we also need to include these
+libraries in our dist tarball. They are
+
+- boost
+- liburing
+- pmdk
+
+``make-dist`` is a script used by our CI pipeline to create dist tarball so the
+tarball can be used to build the Ceph packages in a clean room environment. When
+we need to upgrade these third party libraries, we should
+
+- update the CMake script
+- rebuild the prebuilt packages and
+- update this script to reflect the change.
+
+Uploading Dependencies
+----------------------
+
+To ensure that prebuilt packages are available by the jenkins agents, we need to
+upload them to either ``apt-mirror.front.sepia.ceph.com`` or `chacra`_. To upload
+packages to the former would require the help of our lab administrator, so if we
+want to maintain the package repositories on regular basis, a better choice would be
+to manage them using `chacractl`_. `chacra`_ represents packages repositories using
+a resource hierarchy, like::
+
+  <project>/<branch>/<ref>/<distro>/<distro-version>/<arch>
+
+In which:
+
+project
+    in general, it is used for denoting a set of related packages. For instance,
+    ``libboost``.
+
+branch
+    branch of project. This mirrors the concept of a Git repo.
+
+ref
+    a unique id of a given version of a set packages. This id is used to reference
+    the set packages under the ``<project>/<branch>``. It is a good practice to
+    version the packaging recipes, like the ``debian`` directory for building DEB
+    packages and the ``spec`` for building RPM packages, and use the SHA1 of the
+    packaging recipe for the ``ref``. But you could also use a random string for
+    ``ref``, like the tag name of the built source tree.
+
+distro
+    the distro name for which the packages are built. Currently, following distros are
+    supported:
+
+    - centos
+    - debian
+    - fedora
+    - rhel
+    - ubuntu
+
+distro-version
+    the version of the distro. For instance, if a package is built on ubuntu focal,
+    the ``distro-version`` should be ``20.04``.
+
+arch
+    the architecture of the packages. It could be:
+
+    - arm64
+    - amd64
+    - noarch
+
+So, for example, we can upload the prebuilt boost packages to chacra like
+
+.. prompt:: bash $
+
+   ls *.deb | chacractl binary create \
+     libboost/master/099c0fd56b4a54457e288a2eff8fffdc0d416f7a/ubuntu/focal/amd64/flavors/default
+
+.. _chacra: https://github.com/ceph/chacra
+.. _chacractl: https://github.com/ceph/chacractl
+
+Update ``install-deps.sh``
+--------------------------
+
+We also need to update ``install-deps.sh`` to point the built script to the new
+repo. Please refer to the `script <https://github.com/ceph/ceph/blob/master/install-deps.sh>`_,
+for more details.
diff --git a/doc/dev/corpus.rst b/doc/dev/corpus.rst
new file mode 100644
index 000000000..4005f70c0
--- /dev/null
+++ b/doc/dev/corpus.rst
@@ -0,0 +1,100 @@
+
+Corpus structure
+================
+
+ceph.git/ceph-object-corpus is a submodule.::
+
+ bin/   # misc scripts
+ archive/$version/objects/$type/$hash  # a sample of encoded objects from a specific version
+
+You can also mark known or deliberate incompatibilities between versions with::
+
+ archive/$version/forward_incompat/$type
+
+The presence of a file indicates that new versions of code cannot
+decode old objects across that ``$version`` (this is normally the case).
+
+
+How to generate an object corpus
+--------------------------------
+
+.. highlight:: shell
+
+We can generate an object corpus for a particular version of ceph using the
+script of ``script/gen-corpus.sh``, or by following the instructions below:
+
+#. Checkout a clean repo (best not to do this where you normally work)::
+
+	git clone ceph.git
+	cd ceph
+	git submodule update --init --recursive
+
+#. Build with flag to dump objects to ``/tmp/foo``::
+
+	rm -rf /tmp/foo ; mkdir /tmp/foo
+	do_cmake.sh -DCMAKE_CXX_FLAGS="-DENCODE_DUMP_PATH=/tmp/foo"
+	cd build
+	make
+
+#. Start via vstart::
+
+	cd build
+	MON=3 MGR=2 OSD=3 MDS=3 RGW=1 ../src/vstart.sh -n -x
+
+#. Use as much functionality of the cluster as you can, to exercise as many object encoder methods as possible::
+
+	bin/ceph osd pool create mypool
+	bin/rados -p mypool bench 10 write -b 123
+	bin/ceph osd out 0
+	bin/ceph osd in 0
+	bin/init-ceph restart osd.1
+	for f in ../qa/workunits/cls/*.sh ; do PATH="bin:$PATH" $f ; done
+	PATH="bin:$PATH" ../qa/workunits/rados/test.sh
+	bin/ceph_test_librbd
+	bin/ceph_test_libcephfs
+	bin/init-ceph restart mds.a
+	../qa/workunits/rgw/run-s3tests.sh
+
+#. Stop::
+
+	../src/stop.sh
+
+#. Import the corpus (this will take a few minutes)::
+
+	../src/test/encoding/import.sh /tmp/foo `bin/ceph-dencoder version` ../ceph-object-corpus/archive
+	../src/test/encoding/import-generated.sh ../ceph-object-corpus/archive
+
+#. Prune it!  There will be a bazillion copies of various objects, and we only want a representative sample.::
+
+	pushd ../ceph-object-corpus
+	bin/prune-archive.sh
+	popd
+
+#. Verify the tests pass::
+
+	ctest -R readable.sh
+
+#. Commit it to the corpus repo and push::
+
+	pushd ../ceph-object-corpus
+	git checkout -b wip-new
+	git add archive/`../build/bin/ceph-dencoder version`
+	git commit -m `../build/bin/ceph-dencoder version`
+	git remote add cc git@github.com:ceph/ceph-object-corpus.git
+	git push cc wip-new
+	popd
+
+#. Go test it out::
+
+	cd my/regular/tree
+	cd ceph-object-corpus
+	git fetch origin
+	git checkout wip-new
+	cd ../build
+	ctest -R readable.sh
+
+#. If everything looks good, update the submodule master branch, and commit the submodule in ceph.git.
+
+
+
+
diff --git a/doc/dev/cpu-profiler.rst b/doc/dev/cpu-profiler.rst
new file mode 100644
index 000000000..3885293cd
--- /dev/null
+++ b/doc/dev/cpu-profiler.rst
@@ -0,0 +1,46 @@
+=====================
+ Installing Oprofile
+=====================
+
+The easiest way to profile Ceph's CPU consumption is to use the `oprofile`_
+system-wide profiler.
+
+.. _oprofile: http://oprofile.sourceforge.net/about/
+
+Installation
+============
+
+If you are using a Debian/Ubuntu distribution, you can install ``oprofile`` by
+executing the following::
+
+	sudo apt-get install oprofile oprofile-gui
+	
+
+Compiling Ceph for Profiling
+============================
+
+To compile Ceph for profiling, first clean everything. ::
+
+    git clean -dfx
+	
+Finally, compile Ceph. ::
+
+	./do-cmake.sh -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -O2 -g"
+    cd build
+    cmake --build .
+
+In this command, ``CMAKE_CXX_FLAGS`` is specified. This provides callgraph output.
+
+Ceph Configuration 
+==================
+
+Ensure that you disable ``lockdep``. Consider setting logging to 
+levels appropriate for a production cluster. See `Ceph Logging and Debugging`_ 
+for details.
+
+.. _Ceph Logging and Debugging: ../../rados/troubleshooting/log-and-debug
+
+See the `CPU Profiling`_ section of the RADOS Troubleshooting documentation for details on using Oprofile.
+
+
+.. _CPU Profiling: ../../rados/troubleshooting/cpu-profiling
diff --git a/doc/dev/crimson/crimson.rst b/doc/dev/crimson/crimson.rst
new file mode 100644
index 000000000..cbc20b773
--- /dev/null
+++ b/doc/dev/crimson/crimson.rst
@@ -0,0 +1,480 @@
+=======
+crimson
+=======
+
+Crimson is the code name of ``crimson-osd``, which is the next
+generation ``ceph-osd``. It improves performance when using fast network
+and storage devices, employing state-of-the-art technologies including
+DPDK and SPDK. BlueStore continues to support HDDs and slower SSDs.
+Crimson aims to be backward compatible with the classic ``ceph-osd``.
+
+.. highlight:: console
+
+Building Crimson
+================
+
+Crimson is not enabled by default. Enable it at build time by running::
+
+  $ WITH_SEASTAR=true ./install-deps.sh
+  $ mkdir build && cd build
+  $ cmake -DWITH_SEASTAR=ON ..
+
+Please note, `ASan`_ is enabled by default if Crimson is built from a source
+cloned using ``git``.
+
+.. _ASan: https://github.com/google/sanitizers/wiki/AddressSanitizer
+
+Testing crimson with cephadm
+===============================
+
+The Ceph CI/CD pipeline builds containers with
+``crimson-osd`` subsitituted for ``ceph-osd``.
+
+Once a branch at commit <sha1> has been built and is available in
+``shaman``, you can deploy it using the cephadm instructions outlined
+in :ref:`cephadm` with the following adaptations.
+
+First, while performing the initial bootstrap, use the ``--image`` flag to
+use a Crimson build:
+
+.. prompt:: bash #
+
+   cephadm --image quay.ceph.io/ceph-ci/ceph:<sha1>-crimson --allow-mismatched-release bootstrap ...
+
+You'll likely need to supply the ``--allow-mismatched-release`` flag to
+use a non-release branch.
+
+Additionally, prior to deploying OSDs, you'll need enable Crimson to
+direct the default pools to be created as Crimson pools.  From the cephadm shell run:
+
+.. prompt:: bash #
+
+   ceph config set global 'enable_experimental_unrecoverable_data_corrupting_features' crimson
+   ceph osd set-allow-crimson --yes-i-really-mean-it
+   ceph config set mon osd_pool_default_crimson true
+
+The first command enables the ``crimson`` experimental feature.  Crimson
+is highly experimental, and malfunctions including crashes
+and data loss are to be expected.
+
+The second enables the ``allow_crimson`` OSDMap flag.  The monitor will
+not allow ``crimson-osd`` to boot without that flag.
+
+The last causes pools to be created by default with the ``crimson`` flag.
+Crimson pools are restricted to operations supported by Crimson.
+``Crimson-osd`` won't instantiate PGs from non-Crimson pools.
+
+Running Crimson
+===============
+
+As you might expect, Crimson does not yet have as extensive a feature set as does ``ceph-osd``.
+
+object store backend
+--------------------
+
+At the moment, ``crimson-osd`` offers both native and alienized object store
+backends. The native object store backends perform IO using the SeaStar reactor.
+They are:
+
+.. describe:: cyanstore
+
+   CyanStore is modeled after memstore in the classic OSD.
+
+.. describe:: seastore
+
+   Seastore is still under active development.
+
+The alienized object store backends are backed by a thread pool, which
+is a proxy of the alienstore adaptor running in Seastar. The proxy issues
+requests to object stores running in alien threads, i.e., worker threads not
+managed by the Seastar framework. They are:
+
+.. describe:: memstore
+
+   The memory backed object store
+
+.. describe:: bluestore
+
+   The object store used by the classic ``ceph-osd``
+
+daemonize
+---------
+
+Unlike ``ceph-osd``, ``crimson-osd`` does not daemonize itself even if the
+``daemonize`` option is enabled. In order to read this option, ``crimson-osd``
+needs to ready its config sharded service, but this sharded service lives
+in the Seastar reactor. If we fork a child process and exit the parent after
+starting the Seastar engine, that will leave us with a single thread which is
+a replica of the thread that called `fork()`_. Tackling this problem in Crimson
+would unnecessarily complicate the code.
+
+Since supported GNU/Linux distributions use ``systemd``, which is able to
+daemonize the application, there is no need to daemonize ourselves. 
+Those using sysvinit can use ``start-stop-daemon`` to daemonize ``crimson-osd``.
+If this is does not work out, a helper utility may be devised.
+
+.. _fork(): http://pubs.opengroup.org/onlinepubs/9699919799/functions/fork.html
+
+logging
+-------
+
+``Crimson-osd`` currently uses the logging utility offered by Seastar. See
+``src/common/dout.h`` for the mapping between Ceph logging levels to
+the severity levels in Seastar. For instance, messages sent to ``derr``
+will be issued using ``logger::error()``, and the messages with a debug level
+greater than ``20`` will be issued using ``logger::trace()``.
+
++---------+---------+
+| ceph    | seastar |
++---------+---------+
+| < 0     | error   |
++---------+---------+
+|   0     | warn    |
++---------+---------+
+| [1, 6)  | info    |
++---------+---------+
+| [6, 20] | debug   |
++---------+---------+
+| >  20   | trace   |
++---------+---------+
+
+Note that ``crimson-osd``
+does not send log messages directly to a specified ``log_file``. It writes
+the logging messages to stdout and/or syslog. This behavior can be
+changed using ``--log-to-stdout`` and ``--log-to-syslog`` command line
+options. By default, ``log-to-stdout`` is enabled, and ``--log-to-syslog`` is disabled.
+
+
+vstart.sh
+---------
+
+The following options aree handy when using ``vstart.sh``,
+
+``--crimson``
+    Start ``crimson-osd`` instead of ``ceph-osd``.
+
+``--nodaemon``
+    Do not daemonize the service.
+
+``--redirect-output``
+    Redirect the ``stdout`` and ``stderr`` to ``out/$type.$num.stdout``.
+
+``--osd-args``
+    Pass extra command line options to ``crimson-osd`` or ``ceph-osd``.
+    This is useful for passing Seastar options to ``crimson-osd``. For
+    example, one can supply ``--osd-args "--memory 2G"`` to set the amount of
+    memory to use. Please refer to the output of::
+
+      crimson-osd --help-seastar
+
+    for additional Seastar-specific command line options.
+
+``--cyanstore``
+    Use CyanStore as the object store backend.
+
+``--bluestore``
+    Use the alienized BlueStore as the object store backend. This is the default.
+
+``--memstore``
+    Use the alienized MemStore as the object store backend.
+
+``--seastore``
+    Use SeaStore as the back end object store.
+
+``--seastore-devs``
+    Specify the block device used by SeaStore.
+
+``--seastore-secondary-devs``
+    Optional.  SeaStore supports multiple devices.  Enable this feature by
+    passing the block device to this option.
+
+``--seastore-secondary-devs-type``
+    Optional.  Specify the type of secondary devices.  When the secondary
+    device is slower than main device passed to ``--seastore-devs``, the cold
+    data in faster device will be evicted to the slower devices over time.
+    Valid types include ``HDD``, ``SSD``(default), ``ZNS``, and ``RANDOM_BLOCK_SSD``
+    Note secondary devices should not be faster than the main device.
+
+``--seastore``
+    Use SeaStore as the object store backend.
+
+To start a cluster with a single Crimson node, run::
+
+  $  MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x \
+    --without-dashboard --cyanstore \
+    --crimson --redirect-output \
+    --osd-args "--memory 4G"
+
+Here we assign 4 GiB memory and a single thread running on core-0 to ``crimson-osd``.
+
+Another SeaStore example::
+
+  $  MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x \
+    --without-dashboard --seastore \
+    --crimson --redirect-output \
+    --seastore-devs /dev/sda \
+    --seastore-secondary-devs /dev/sdb \
+    --seastore-secondary-devs-type HDD
+
+Stop this ``vstart`` cluster by running::
+
+  $ ../src/stop.sh --crimson
+
+Metrics and Tracing
+===================
+
+Crimson offers three ways to report stats and metrics.
+
+pg stats reported to mgr
+------------------------
+
+Crimson collects the per-pg, per-pool, and per-osd stats in a `MPGStats`
+message which is sent to the Ceph Managers. Manager modules can query
+them using the `MgrModule.get()` method.
+
+asock command
+-------------
+
+An admin socket command is offered for dumping metrics::
+
+  $ ceph tell osd.0 dump_metrics
+  $ ceph tell osd.0 dump_metrics reactor_utilization
+
+Here `reactor_utilization` is an optional string allowing us to filter
+the dumped metrics by prefix.
+
+Prometheus text protocol
+------------------------
+
+The listening port and address can be configured using the command line options of
+`--prometheus_port`
+see `Prometheus`_ for more details.
+
+.. _Prometheus: https://github.com/scylladb/seastar/blob/master/doc/prometheus.md
+
+Profiling Crimson
+=================
+
+fio
+---
+
+``crimson-store-nbd`` exposes configurable ``FuturizedStore`` internals as an
+NBD server for use with ``fio``.
+
+In order to use ``fio`` to test ``crimson-store-nbd``, perform the below steps.
+
+#. You will need to install ``libnbd``, and compile it into ``fio``
+
+   .. prompt:: bash $
+
+      apt-get install libnbd-dev
+      git clone git://git.kernel.dk/fio.git
+      cd fio
+      ./configure --enable-libnbd
+      make
+
+#. Build ``crimson-store-nbd``
+
+   .. prompt:: bash $
+
+      cd build
+      ninja crimson-store-nbd
+
+#. Run the ``crimson-store-nbd`` server with a block device. Specify
+   the path to the raw device, for example ``/dev/nvme1n1``, in place of the created
+   file for testing with a block device.
+
+   .. prompt:: bash $
+
+      export disk_img=/tmp/disk.img
+      export unix_socket=/tmp/store_nbd_socket.sock
+      rm -f $disk_img $unix_socket
+      truncate -s 512M $disk_img
+      ./bin/crimson-store-nbd \
+        --device-path $disk_img \
+        --smp 1 \
+        --mkfs true \
+        --type transaction_manager \
+        --uds-path ${unix_socket} &
+
+   Below are descriptions of these command line arguments:
+
+   ``--smp``
+     The number of CPU cores to use (Symmetric MultiProcessor)
+
+   ``--mkfs``
+     Initialize the device first.
+
+   ``--type``
+     The back end to use. If ``transaction_manager`` is specified, SeaStore's
+     ``TransactionManager`` and ``BlockSegmentManager`` are used to emulate a
+     block device. Otherwise, this option is used to choose a backend of
+     ``FuturizedStore``, where the whole "device" is divided into multiple
+     fixed-size objects whose size is specified by ``--object-size``. So, if
+     you are only interested in testing the lower-level implementation of
+     SeaStore like logical address translation layer and garbage collection
+     without the object store semantics, ``transaction_manager`` would be a
+     better choice.
+
+#. Create a ``fio`` job file named ``nbd.fio``
+
+   .. code:: ini
+
+      [global]
+      ioengine=nbd
+      uri=nbd+unix:///?socket=${unix_socket}
+      rw=randrw
+      time_based
+      runtime=120
+      group_reporting
+      iodepth=1
+      size=512M
+
+      [job0]
+      offset=0
+
+#. Test the Crimson object store, using the custom ``fio`` built just now
+
+   .. prompt:: bash $
+
+      ./fio nbd.fio
+
+CBT
+---
+We can use `cbt`_ for performance tests::
+
+  $ git checkout main
+  $ make crimson-osd
+  $ ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/baseline ../src/test/crimson/cbt/radosbench_4K_read.yaml
+  $ git checkout yet-another-pr
+  $ make crimson-osd
+  $ ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/yap ../src/test/crimson/cbt/radosbench_4K_read.yaml
+  $ ~/dev/cbt/compare.py -b /tmp/baseline -a /tmp/yap -v
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: bandwidth: (or (greater) (near 0.05)):: 0.183165/0.186155  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: iops_avg: (or (greater) (near 0.05)):: 46.0/47.0  => accepted
+  19:48:23 - WARNING  - cbt      - prefill/gen8/0: iops_stddev: (or (less) (near 0.05)):: 10.4403/6.65833  => rejected
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: latency_avg: (or (less) (near 0.05)):: 0.340868/0.333712  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: bandwidth: (or (greater) (near 0.05)):: 0.190447/0.177619  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: iops_avg: (or (greater) (near 0.05)):: 48.0/45.0  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: iops_stddev: (or (less) (near 0.05)):: 6.1101/9.81495  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: latency_avg: (or (less) (near 0.05)):: 0.325163/0.350251  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: bandwidth: (or (greater) (near 0.05)):: 1.24654/1.22336  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: iops_avg: (or (greater) (near 0.05)):: 319.0/313.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: iops_stddev: (or (less) (near 0.05)):: 0.0/0.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: latency_avg: (or (less) (near 0.05)):: 0.0497733/0.0509029  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: bandwidth: (or (greater) (near 0.05)):: 1.22717/1.11372  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: iops_avg: (or (greater) (near 0.05)):: 314.0/285.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: iops_stddev: (or (less) (near 0.05)):: 0.0/0.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: latency_avg: (or (less) (near 0.05)):: 0.0508262/0.0557337  => accepted
+  19:48:23 - WARNING  - cbt      - 1 tests failed out of 16
+
+Here we compile and run the same test against two branches: ``main`` and ``yet-another-pr``.
+We then compare the results. Along with every test case, a set of rules is defined to check for
+performance regressions when comparing the sets of test results. If a possible regression is found, the rule and
+corresponding test results are highlighted.
+
+.. _cbt: https://github.com/ceph/cbt
+
+Hacking Crimson
+===============
+
+
+Seastar Documents
+-----------------
+
+See `Seastar Tutorial <https://github.com/scylladb/seastar/blob/master/doc/tutorial.md>`_ .
+Or build a browsable version and start an HTTP server::
+
+  $ cd seastar
+  $ ./configure.py --mode debug
+  $ ninja -C build/debug docs
+  $ python3 -m http.server -d build/debug/doc/html
+
+You might want to install ``pandoc`` and other dependencies beforehand.
+
+Debugging Crimson
+=================
+
+Debugging with GDB
+------------------
+
+The `tips`_ for debugging Scylla also apply to Crimson.
+
+.. _tips: https://github.com/scylladb/scylla/blob/master/docs/dev/debugging.md#tips-and-tricks
+
+Human-readable backtraces with addr2line
+----------------------------------------
+
+When a Seastar application crashes, it leaves us with a backtrace of addresses, like::
+
+  Segmentation fault.
+  Backtrace:
+    0x00000000108254aa
+    0x00000000107f74b9
+    0x00000000105366cc
+    0x000000001053682c
+    0x00000000105d2c2e
+    0x0000000010629b96
+    0x0000000010629c31
+    0x00002a02ebd8272f
+    0x00000000105d93ee
+    0x00000000103eff59
+    0x000000000d9c1d0a
+    /lib/x86_64-linux-gnu/libc.so.6+0x000000000002409a
+    0x000000000d833ac9
+  Segmentation fault
+
+The ``seastar-addr2line`` utility provided by Seastar can be used to map these
+addresses to functions. The script expects input on ``stdin``,
+so we need to copy and paste the above addresses, then send EOF by inputting
+``control-D`` in the terminal.  One might  use ``echo`` or ``cat`` instead`::
+
+  $ ../src/seastar/scripts/seastar-addr2line -e bin/crimson-osd
+
+    0x00000000108254aa
+    0x00000000107f74b9
+    0x00000000105366cc
+    0x000000001053682c
+    0x00000000105d2c2e
+    0x0000000010629b96
+    0x0000000010629c31
+    0x00002a02ebd8272f
+    0x00000000105d93ee
+    0x00000000103eff59
+    0x000000000d9c1d0a
+    0x00000000108254aa
+  [Backtrace #0]
+  seastar::backtrace_buffer::append_backtrace() at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1136
+  seastar::print_with_backtrace(seastar::backtrace_buffer&) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1157
+  seastar::print_with_backtrace(char const*) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1164
+  seastar::sigsegv_action() at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5119
+  seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::operator()(int, siginfo_t*, void*) const at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5105
+  seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::_FUN(int, siginfo_t*, void*) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5101
+  ?? ??:0
+  seastar::smp::configure(boost::program_options::variables_map, seastar::reactor_config) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5418
+  seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /home/kefu/dev/ceph/build/../src/seastar/src/core/app-template.cc:173 (discriminator 5)
+  main at /home/kefu/dev/ceph/build/../src/crimson/osd/main.cc:131 (discriminator 1)
+
+Note that ``seastar-addr2line`` is able to extract addresses from
+its input, so you can also paste the log messages as below::
+
+  2020-07-22T11:37:04.500 INFO:teuthology.orchestra.run.smithi061.stderr:Backtrace:
+  2020-07-22T11:37:04.500 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e78dbc
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e7f0
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e8b8
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e985
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  /lib64/libpthread.so.0+0x0000000000012dbf
+
+Unlike the classic ``ceph-osd``, Crimson does not print a human-readable backtrace when it
+handles fatal signals like `SIGSEGV` or `SIGABRT`. It is also more complicated
+with a stripped binary. So instead of planting a signal handler for
+those signals into Crimson, we can use `script/ceph-debug-docker.sh` to map
+addresses in the backtrace::
+
+  # assuming you are under the source tree of ceph
+  $ ./src/script/ceph-debug-docker.sh  --flavor crimson master:27e237c137c330ebb82627166927b7681b20d0aa centos:8
+  ....
+  [root@3deb50a8ad51 ~]# wget -q https://raw.githubusercontent.com/scylladb/seastar/master/scripts/seastar-addr2line
+  [root@3deb50a8ad51 ~]# dnf install -q -y file
+  [root@3deb50a8ad51 ~]# python3 seastar-addr2line -e /usr/bin/crimson-osd
+  # paste the backtrace here
diff --git a/doc/dev/crimson/error-handling.rst b/doc/dev/crimson/error-handling.rst
new file mode 100644
index 000000000..185868e70
--- /dev/null
+++ b/doc/dev/crimson/error-handling.rst
@@ -0,0 +1,158 @@
+==============
+error handling
+==============
+
+
+In Seastar, a ``future`` represents a value not yet available but that can become
+available later. ``future`` can have one of following states:
+
+* unavailable: value is not available yet,
+* value,
+* failed: an exception was thrown when computing the value. This exception has
+  been captured and stored in the ``future`` instance via ``std::exception_ptr``.
+
+In the last case, the exception can be processed using ``future::handle_exception()`` or
+``future::handle_exception_type()``. Seastar even provides ``future::or_terminate()`` to
+terminate the program if the future fails.
+
+But in Crimson, quite a few errors are not serious enough to fail the program entirely.
+For instance, if we try to look up an object by its object id, and that operation could
+fail because the object does not exist or it is corrupted, we need to recover that object
+for fulfilling the request instead of terminating the process.
+
+In other words, these errors are expected. Moreover, the performance of the unhappy path
+should also be on par with that of the happy path. Also, we want to have a way to ensure
+that all expected errors are handled. It should be something like the statical analysis
+performed by compiler to spit a warning if any enum value is not handled in a ``switch-case``
+statement.
+
+Unfortunately, ``seastar::future`` is not able to satisfy these two requirements.
+
+* Seastar imposes re-throwing an exception to dispatch between different types of
+  exceptions. This is not very performant nor even scalable as locking in the language's
+  runtime can occur.
+* Seastar does not encode the expected exception type in the type of the returned
+  ``seastar::future``. Only the type of the value is encoded. This imposes huge
+  mental load on programmers as ensuring that all intended errors are indeed handled
+  requires manual code audit.
+
+.. highlight:: c++
+
+So, "errorator" is created. It is a wrapper around the vanilla ``seastar::future``.
+It addresses the performance and scalability issues while embedding the information
+about all expected types-of-errors to the type-of-future.::
+
+  using ertr = crimson::errorator<crimson::ct_error::enoent,
+                                  crimson::ct_error::einval>;
+
+In above example we defined an errorator that allows for two error types:
+
+* ``crimson::ct_error::enoent`` and
+* ``crimson::ct_error::einval``.
+
+These (and other ones in the ``crimson::ct_error`` namespace) are basically
+unthrowable wrappers over ``std::error_code`` to exclude accidental throwing
+and ensure signaling errors in a way that enables compile-time checking.
+
+The most fundamental thing in an errorator is a descendant of ``seastar::future``
+which can be used as e.g. function's return type::
+
+  static ertr::future<int> foo(int bar) {
+    if (bar == 42) {
+      return crimson::ct_error::einval::make();
+    } else {
+      return ertr::make_ready_future(bar);
+    }
+  }
+
+It's worth to note that returning an error that is not a part the errorator's error set
+would result in a compile-time error::
+
+  static ertr::future<int> foo(int bar) {
+    // Oops, input_output_error is not allowed in `ertr`.  static_assert() will
+    // terminate the compilation. This behaviour is absolutely fundamental for
+    // callers -- to figure out about all possible errors they need to worry
+    // about is enough to just take a look on the function's signature; reading
+    // through its implementation is not necessary anymore!
+    return crimson::ct_error::input_output_error::make();
+  }
+
+The errorator concept goes further. It not only provides callers with the information
+about all potential errors embedded in the function's type; it also ensures at the caller
+site that all these errors are handled. As the reader probably know, the main method
+in ``seastar::future`` is ``then()``. On errorated future it is available but only if errorator's
+error set is empty (literally: ``errorator<>::future``); otherwise callers have
+to use ``safe_then()`` instead::
+
+  seastar::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "the optimistic path! got bar=" << bar << std::endl
+        return ertr::now();
+      },
+      ertr::all_same_way(const std::error_code& err) {
+        // handling errors removes them from errorator's error set
+        std::cout << "the error path! got err=" << err << std::endl;
+        return ertr::now();
+      }).then([] {
+        // as all errors have been handled, errorator's error set became
+        // empty and the future instance returned from `safe_then()` has
+        // `then()` available!
+        return seastar::now();
+      });
+  }
+
+In the above example ``ertr::all_same_way`` has been used to handle all errors in the same
+manner. This is not obligatory -- a caller can handle each of them separately. Moreover,
+it can provide a handler for only a subset of errors. The price for that is the availability
+of ``then()``::
+
+  using einval_ertr = crimson::errorator<crimson::ct_error::einval>;
+
+  // we can't return seastar::future<> (aka errorator<>::future<>) as handling
+  // as this level deals only with enoent leaving einval without a handler.
+  // handling it becomes a responsibility of a caller of `baz()`.
+  einval_ertr::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "the optimistic path! got bar=" << bar << std::endl
+        return ertr::now();
+      },
+      // provide a handler only for crimson::ct_error::enoent.
+      // crimson::ct_error::einval stays unhandled!
+      crimson::ct_error::enoent::handle([] {
+        std::cout << "the enoent error path!" << std::endl;
+        return ertr::now();
+      }));
+    // .safe_then() above returned `errorator<crimson::ct_error::einval>::future<>`
+    // which lacks `then()`.
+  }
+
+That is, handling errors removes them from errorated future's error set. This works
+in the opposite direction too -- returning new errors in ``safe_then()`` appends them
+the error set. Of course, this set must be compliant with error set in the ``baz()``'s
+signature::
+
+  using broader_ertr = crimson::errorator<crimson::ct_error::enoent,
+                                          crimson::ct_error::einval,
+                                          crimson::ct_error::input_output_error>;
+
+  broader_ertr::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "oops, the optimistic path generates a new error!";
+        return crimson::ct_error::input_output_error::make();
+      },
+      // we have a special handler to delegate the handling up. For convenience,
+      // the same behaviour is available as single argument-taking variant of
+      // `safe_then()`.
+      ertr::pass_further{});
+  }
+
+As it can be seen, handling and signaling errors in ``safe_then()`` is basically
+an operation on the error set checked at compile-time.
+
+More details can be found in `the slides from ceph::errorator<> throw/catch-free,
+compile time-checked exceptions for seastar::future<>
+<https://www.slideshare.net/ScyllaDB/cepherrorator-throwcatchfree-compile-timechecked-exceptions-for-seastarfuture>`_
+presented at the Seastar Summit 2019.
diff --git a/doc/dev/crimson/index.rst b/doc/dev/crimson/index.rst
new file mode 100644
index 000000000..55f071825
--- /dev/null
+++ b/doc/dev/crimson/index.rst
@@ -0,0 +1,11 @@
+===============================
+Crimson developer documentation
+===============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
+
diff --git a/doc/dev/crimson/osd.rst b/doc/dev/crimson/osd.rst
new file mode 100644
index 000000000..f7f132b3f
--- /dev/null
+++ b/doc/dev/crimson/osd.rst
@@ -0,0 +1,54 @@
+osd
+===
+
+.. graphviz::
+
+   digraph osd {
+     node [shape = doublecircle]; "start" "end";
+     node [shape = circle];
+     start -> preboot;
+     waiting_for_healthy [label = "waiting\nfor\nhealthy"];
+     waiting_for_healthy -> waiting_for_healthy [label = "tick"];
+     waiting_for_healthy -> preboot [label = "i am healthy!"];
+     preboot -> booting [label = "send(MOSDBoot)"];
+     booting -> active [label = "recv(osdmap<up>)"];
+     active -> prestop [label = "stop()"];
+     active -> preboot [label = "recv(osdmap<down>)"];
+     active -> end [label = "kill(SIGINT)"];
+     active -> waiting_for_healthy [label = "i am unhealthy!"]
+     prestop -> end [label = "recv(osdmap<down>)"];
+   }
+
+.. describe:: waiting_for_healthy
+
+   If an OSD daemon is able to connected to its heartbeat peers, and its own
+   internal heartbeat does not fail, it is considered healthy. Otherwise, it
+   puts itself in the state of `waiting_for_healthy`, and check its own
+   reachability and internal heartbeat periodically.
+
+.. describe:: preboot
+
+   OSD sends an `MOSDBoot` message to the connected monitor to inform the
+   cluster that it's ready to serve, so that the quorum can mark it `up`
+   in the osdmap.
+
+.. describe:: booting
+
+   Before being marked as `up`, an OSD has to stay in its `booting` state.
+
+.. describe:: active
+
+   Upon receiving an osdmap marking the OSD as `up`, it transits to `active`
+   state. After that, it is entitled to do its business. But the OSD service
+   can be fully stopped or suspended due to various reasons. For instance,
+   the osd services can be stopped by administrator manually, or marked `stop`
+   in the osdmap. Or any of its IP addresses does not match with the
+   corresponding one configured in osdmap, it transits to `preboot` if
+   it considers itself healthy.
+
+.. describe:: prestop
+
+   The OSD transits to `prestop` unconditionally upon request of `stop`.
+   But before bidding us farewell, it tries to get the acknowledge from
+   the monitor by sending an `MOSDMarkMeDown`, and waiting for an response
+   of updated osdmap or another `MOSDMarkMeDown` message.
diff --git a/doc/dev/crimson/pipeline.rst b/doc/dev/crimson/pipeline.rst
new file mode 100644
index 000000000..e9115c6d7
--- /dev/null
+++ b/doc/dev/crimson/pipeline.rst
@@ -0,0 +1,97 @@
+==============================
+The ``ClientRequest`` pipeline
+==============================
+
+In crimson, exactly like in the classical OSD, a client request has data and
+ordering dependencies which must be satisfied before processing (actually
+a particular phase of) can begin. As one of the goals behind crimson is to
+preserve the compatibility with the existing OSD incarnation, the same semantic
+must be assured. An obvious example of such data dependency is the fact that
+an OSD needs to have a version of OSDMap that matches the one used by the client
+(``Message::get_min_epoch()``).
+
+If a dependency is not satisfied, the processing stops. It is crucial to note
+the same must happen to all other requests that are sequenced-after (due to
+their ordering requirements).
+
+There are a few cases when the blocking of a client request can happen.
+
+
+  ``ClientRequest::ConnectionPipeline::await_map``
+    wait for particular OSDMap version is available at the OSD level
+  ``ClientRequest::ConnectionPipeline::get_pg``
+    wait a particular PG becomes available on OSD
+  ``ClientRequest::PGPipeline::await_map``
+    wait on a PG being advanced to particular epoch
+  ``ClientRequest::PGPipeline::wait_for_active``
+    wait for a PG to become *active* (i.e. have ``is_active()`` asserted)
+  ``ClientRequest::PGPipeline::recover_missing``
+    wait on an object to be recovered (i.e. leaving the ``missing`` set)
+  ``ClientRequest::PGPipeline::get_obc``
+    wait on an object to be available for locking. The ``obc`` will be locked
+    before this operation is allowed to continue
+  ``ClientRequest::PGPipeline::process``
+    wait if any other ``MOSDOp`` message is handled against this PG
+
+At any moment, a ``ClientRequest`` being served should be in one and only one
+of the phases described above. Similarly, an object denoting particular phase
+can host not more than a single ``ClientRequest`` the same time. At low-level
+this is achieved with a combination of a barrier and an exclusive lock.
+They implement the semantic of a semaphore with a single slot for these exclusive
+phases.
+
+As the execution advances, request enters next phase and leaves the current one
+freeing it for another ``ClientRequest`` instance. All these phases form a pipeline
+which assures the order is preserved.
+
+These pipeline phases are divided into two ordering domains: ``ConnectionPipeline``
+and ``PGPipeline``. The former ensures order across a client connection while
+the latter does that across a PG. That is, requests originating from the same
+connection are executed in the same order as they were sent by the client.
+The same applies to the PG domain: when requests from multiple connections reach
+a PG, they are executed in the same order as they entered a first blocking phase
+of the ``PGPipeline``.
+
+Comparison with the classical OSD
+----------------------------------
+As the audience of this document are Ceph Developers, it seems reasonable to
+match the phases of crimson's ``ClientRequest`` pipeline with the blocking
+stages in the classical OSD. The names in the right column are names of
+containers (lists and maps) used to implement these stages. They are also
+already documented in the ``PG.h`` header.
+
++----------------------------------------+--------------------------------------+
+| crimson                                | ceph-osd waiting list		|
++========================================+======================================+
+|``ConnectionPipeline::await_map``       | ``OSDShardPGSlot::waiting`` and	|
+|``ConnectionPipeline::get_pg``          | ``OSDShardPGSlot::waiting_peering``	|
++----------------------------------------+--------------------------------------+
+|``PGPipeline::await_map``               | ``PG::waiting_for_map``		|
++----------------------------------------+--------------------------------------+
+|``PGPipeline::wait_for_active``         | ``PG::waiting_for_peered``		|
+|                                        +--------------------------------------+
+|                                        | ``PG::waiting_for_flush``		|
+|                                        +--------------------------------------+
+|                                        | ``PG::waiting_for_active``		|
++----------------------------------------+--------------------------------------+
+|To be done (``PG_STATE_LAGGY``)         | ``PG::waiting_for_readable``		|
++----------------------------------------+--------------------------------------+
+|To be done                              | ``PG::waiting_for_scrub``		|
++----------------------------------------+--------------------------------------+
+|``PGPipeline::recover_missing``         | ``PG::waiting_for_unreadable_object``|
+|                                        +--------------------------------------+
+|                                        | ``PG::waiting_for_degraded_object``	|
++----------------------------------------+--------------------------------------+
+|To be done (proxying)                   | ``PG::waiting_for_blocked_object``	|
++----------------------------------------+--------------------------------------+
+|``PGPipeline::get_obc``                 | *obc rwlocks*			|
++----------------------------------------+--------------------------------------+
+|``PGPipeline::process``                 | ``PG::lock`` (roughly)		|
++----------------------------------------+--------------------------------------+
+
+
+As the last word it might be worth to emphasize that the ordering implementations
+in both classical OSD and in crimson are stricter than a theoretical minimum one
+required by the RADOS protocol. For instance, we could parallelize read operations
+targeting the same object at the price of extra complexity but we don't -- the
+simplicity has won.
diff --git a/doc/dev/crimson/poseidonstore.rst b/doc/dev/crimson/poseidonstore.rst
new file mode 100644
index 000000000..7c54c029a
--- /dev/null
+++ b/doc/dev/crimson/poseidonstore.rst
@@ -0,0 +1,586 @@
+===============
+ PoseidonStore
+===============
+
+Key concepts and goals
+======================
+
+* As one of the pluggable backend stores for Crimson, PoseidonStore targets only
+  high-end NVMe SSDs (not concerned with ZNS devices).
+* Designed entirely for low CPU consumption
+
+  - Hybrid update strategies for different data types (in-place, out-of-place) to
+    minimize CPU consumption by reducing host-side GC.
+  - Remove a black-box component like RocksDB and a file abstraction layer in BlueStore
+    to avoid unnecessary overheads (e.g., data copy and serialization/deserialization)
+  - Utilize NVMe feature (atomic large write command, Atomic Write Unit Normal).
+    Make use of io_uring, new kernel asynchronous I/O interface, to selectively use the interrupt
+    driven mode for CPU efficiency (or polled mode for low latency).
+* Sharded data/processing model
+
+Background
+----------
+
+Both in-place and out-of-place update strategies have their pros and cons.
+
+* Log-structured store
+
+  Log-structured based storage system is a typical example that adopts an update-out-of-place approach.
+  It never modifies the written data. Writes always go to the end of the log. It enables I/O sequentializing.
+
+  * Pros
+
+    - Without a doubt, one sequential write is enough to store the data
+    - It naturally supports transaction (this is no overwrite, so the store can rollback
+      previous stable state)
+    - Flash friendly (it mitigates GC burden on SSDs)
+  * Cons
+
+    - There is host-side GC that induces overheads
+
+      - I/O amplification (host-side)
+      - More host-CPU consumption
+
+    - Slow metadata lookup
+    - Space overhead (live and unused data co-exist)
+
+* In-place update store
+
+  The update-in-place strategy has been used widely for conventional file systems such as ext4 and xfs.
+  Once a block has been placed in a given disk location, it doesn't move.
+  Thus, writes go to the corresponding location in the disk.
+
+  * Pros
+
+    - Less host-CPU consumption (No host-side GC is required)
+    - Fast lookup
+    - No additional space for log-structured, but there is internal fragmentation
+  * Cons
+
+    - More writes occur to record the data (metadata and data section are separated)
+    - It cannot support transaction. Some form of WAL required to ensure update atomicity
+      in the general case
+    - Flash unfriendly (Give more burdens on SSDs due to device-level GC)
+
+Motivation and Key idea
+-----------------------
+
+In modern distributed storage systems, a server node can be equipped with multiple
+NVMe storage devices. In fact, ten or more NVMe SSDs could be attached on a server.
+As a result, it is hard to achieve NVMe SSD's full performance due to the limited CPU resources
+available in a server node. In such environments, CPU tends to become a performance bottleneck.
+Thus, now we should focus on minimizing host-CPU consumption, which is the same as the Crimson's objective.
+
+Towards an object store highly optimized for CPU consumption, three design choices have been made.
+
+* **PoseidonStore does not have a black-box component like RocksDB in BlueStore.**
+
+  Thus, it can avoid unnecessary data copy and serialization/deserialization overheads.
+  Moreover, we can remove an unnecessary file abstraction layer, which was required to run RocksDB.
+  Object data and metadata is now directly mapped to the disk blocks.
+  Eliminating all these overheads will reduce CPU consumption (e.g., pre-allocation, NVME atomic feature).
+
+* **PoseidonStore uses hybrid update strategies for different data size, similar to BlueStore.**
+
+  As we discussed, both in-place and out-of-place update strategies have their pros and cons.
+  Since CPU is only bottlenecked under small I/O workloads, we chose update-in-place for small I/Os to minimize CPU consumption
+  while choosing update-out-of-place for large I/O to avoid double write. Double write for small data may be better than host-GC overhead
+  in terms of CPU consumption in the long run. Although it leaves GC entirely up to SSDs,
+
+* **PoseidonStore makes use of io_uring, new kernel asynchronous I/O interface to exploit interrupt-driven I/O.**
+
+  User-space driven I/O solutions like SPDK provide high I/O performance by avoiding syscalls and enabling zero-copy
+  access from the application. However, it does not support interrupt-driven I/O, which is only possible with kernel-space driven I/O.
+  Polling is good for low-latency but bad for CPU efficiency. On the other hand, interrupt is good for CPU efficiency and bad for
+  low-latency (but not that bad as I/O size increases). Note that network acceleration solutions like DPDK also excessively consume
+  CPU resources for polling. Using polling both for network and storage processing aggravates CPU consumption.
+  Since network is typically much faster and has a higher priority than storage, polling should be applied only to network processing.
+
+high-end NVMe SSD has enough powers to handle more works. Also, SSD lifespan is not a practical concern these days
+(there is enough program-erase cycle limit [#f1]_). On the other hand, for large I/O workloads, the host can afford process host-GC.
+Also, the host can garbage collect invalid objects more effectively when their size is large
+
+Observation
+-----------
+
+Two data types in Ceph
+
+* Data (object data)
+
+  - The cost of double write is high
+  - The best method to store this data is in-place update
+
+    - At least two operations required to store the data: 1) data and 2) location of
+      data. Nevertheless, a constant number of operations would be better than out-of-place
+      even if it aggravates WAF in SSDs
+
+* Metadata or small data (e.g., object_info_t, snapset, pg_log, and collection)
+
+  - Multiple small-sized metadata entries for an object
+  - The best solution to store this data is WAL + Using cache
+
+    - The efficient way to store metadata is to merge all metadata related to data
+      and store it though a single write operation even though it requires background
+      flush to update the data partition
+
+
+Design
+======
+.. ditaa::
+
+  +-WAL partition-|----------------------Data partition-------------------------------+
+  | Sharded partition                                                                 |
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Sharded partition 2
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Sharded partition N
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Global information (in reverse order)
+  +-----------------------------------------------------------------------------------+
+  | Global WAL -> | | SB | Freelist |                                                 |
+  +-----------------------------------------------------------------------------------+
+
+
+* WAL
+
+  - Log, metadata and small data are stored in the WAL partition
+  - Space within the WAL partition is continually reused in a circular manner
+  - Flush data to trim WAL as necessary
+* Disk layout
+
+  - Data blocks are metadata blocks or data blocks
+  - Freelist manages the root of free space B+tree
+  - Super block contains management info for a data partition
+  - Onode radix tree info contains the root of onode radix tree
+
+
+I/O procedure
+-------------
+* Write
+
+  For incoming writes, data is handled differently depending on the request size;
+  data is either written twice (WAL) or written in a log-structured manner.
+
+  #. If Request Size ≤ Threshold (similar to minimum allocation size in BlueStore)
+
+     Write data and metadata to [WAL] —flush—> Write them to [Data section (in-place)] and
+     [Metadata section], respectively.
+
+     Since the CPU becomes the bottleneck for small I/O workloads, in-place update scheme is used.
+     Double write for small data may be better than host-GC overhead in terms of CPU consumption
+     in the long run
+  #. Else if Request Size > Threshold
+
+     Append data to [Data section (log-structure)] —> Write the corresponding metadata to [WAL]
+     —flush—> Write the metadata to [Metadata section]
+
+  For large I/O workloads, the host can afford process host-GC
+  Also, the host can garbage collect invalid objects more effectively when their size is large
+
+  Note that Threshold can be configured to a very large number so that only the scenario (1) occurs.
+  With this design, we can control the overall I/O procedure with the optimizations for crimson
+  as described above.
+
+  * Detailed flow
+
+    We make use of a NVMe write command which provides atomicity guarantees (Atomic Write Unit Power Fail)
+    For example, 512 Kbytes of data can be atomically written at once without fsync().
+
+    * stage 1
+
+      - if the data is small
+        WAL (written) --> | TxBegin A | Log Entry | TxEnd A |
+        Append a log entry that contains pg_log, snapset, object_infot_t and block allocation
+        using NVMe atomic write command on the WAL
+      - if the data is large
+        Data partition (written) --> | Data blocks |
+    * stage 2
+
+      - if the data is small
+        No need.
+      - if the data is large
+        Then, append the metadata to WAL.
+        WAL --> | TxBegin A | Log Entry | TxEnd A |
+
+* Read
+
+  - Use the cached object metadata to find out the data location
+  - If not cached, need to search WAL after checkpoint and Object meta partition to find the
+    latest meta data
+
+* Flush (WAL --> Data partition)
+
+  - Flush WAL entries that have been committed. There are two conditions
+    (1. the size of WAL is close to full, 2. a signal to flush).
+    We can mitigate the overhead of frequent flush via batching processing, but it leads to
+    delaying completion.
+
+
+Crash consistency
+------------------
+
+* Large case
+
+  #. Crash occurs right after writing Data blocks
+
+     - Data partition --> | Data blocks |
+     - We don't need to care this case. Data is not allocated yet. The blocks will be reused.
+  #. Crash occurs right after WAL
+
+     - Data partition --> | Data blocks |
+     - WAL --> | TxBegin A | Log Entry | TxEnd A |
+     - Write procedure is completed, so there is no data loss or inconsistent state
+
+* Small case
+
+  #. Crash occurs right after writing WAL
+
+     - WAL --> | TxBegin A | Log Entry| TxEnd A |
+     - All data has been written
+
+
+Comparison
+----------
+
+* Best case (pre-allocation)
+
+  - Only need writes on both WAL and Data partition without updating object metadata (for the location).
+* Worst case
+
+  - At least three writes are required additionally on WAL, object metadata, and data blocks.
+  - If the flush from WAL to the data partition occurs frequently, radix tree onode structure needs to be update
+    in many times. To minimize such overhead, we can make use of batch processing to minimize the update on the tree
+    (the data related to the object has a locality because it will have the same parent node, so updates can be minimized)
+
+* WAL needs to be flushed if the WAL is close to full or a signal to flush.
+
+  - The premise behind this design is OSD can manage the latest metadata as a single copy. So,
+    appended entries are not to be read
+* Either best of the worst case does not produce severe I/O amplification (it produce I/Os, but I/O rate is constant)
+  unlike LSM-tree DB (the proposed design is similar to LSM-tree which has only level-0)
+
+
+Detailed Design
+===============
+
+* Onode lookup
+
+  * Radix tree
+    Our design is entirely based on the prefix tree. Ceph already makes use of the characteristic of OID's prefix to split or search
+    the OID (e.g., pool id + hash + oid). So, the prefix tree fits well to store or search the object. Our scheme is designed
+    to lookup the prefix tree efficiently.
+
+  * Sharded partition
+    A few bits (leftmost bits of the hash) of the OID determine a sharded partition where the object is located.
+    For example, if the number of partitions is configured as four, The entire space of the hash in hobject_t
+    can be divided into four domains (0x0xxx ~ 0x3xxx, 0x4xxx ~ 0x7xxx, 0x8xxx ~ 0xBxxx and 0xCxxx ~ 0xFxxx).
+
+  * Ondisk onode
+
+    .. code-block:: c
+
+            struct onode {
+              extent_tree block_maps;
+              b+_tree omaps;
+              map xattrs;
+            }
+
+    onode contains the radix tree nodes for lookup, which means we can search for objects using tree node information in onode.
+    Also, if the data size is small, the onode can embed the data and xattrs.
+    The onode is fixed size (256 or 512 byte). On the other hands, omaps and block_maps are variable-length by using pointers in the onode.
+
+    .. ditaa::
+
+           +----------------+------------+--------+
+           | on\-disk onode | block_maps | omaps  |
+           +----------+-----+------------+--------+
+                      |           ^         ^
+                      |           |         |
+                      +-----------+---------+
+
+
+  * Lookup
+    The location of the root of onode tree is specified on Onode radix tree info, so we can find out where the object
+    is located by using the root of prefix tree. For example, shared partition is determined by OID as described above.
+    Using the rest of the OID's bits and radix tree, lookup procedure find outs the location of the onode.
+    The extent tree (block_maps) contains where data chunks locate, so we finally figure out the data location.
+
+
+* Allocation
+
+  * Sharded partitions
+
+    The entire disk space is divided into several  data chunks called sharded partition (SP).
+    Each SP has its own data structures to manage the partition.
+
+  * Data allocation
+
+    As we explained above, the management infos (e.g., super block, freelist info, onode radix tree info) are pre-allocated
+    in each shared partition. Given OID, we can map any data in Data block section to the extent tree in the onode.
+    Blocks can be allocated by searching the free space tracking data structure (we explain below).
+
+    ::
+
+            +-----------------------------------+
+            | onode radix tree root node block  |
+            |          (Per-SP Meta)            |
+            |                                   |
+            |           # of records            |
+            |    left_sibling / right_sibling   |
+            | +--------------------------------+|
+            | | keys[# of records]             ||
+            | | +-----------------------------+||
+            | | |    start onode ID           |||
+            | | |           ...               |||
+            | | +-----------------------------+||
+            | +--------------------------------||
+            | +--------------------------------+|
+            | | ptrs[# of records]             ||
+            | | +-----------------------------+||
+            | | |       SP block number       |||
+            | | |           ...               |||
+            | | +-----------------------------+||
+            | +--------------------------------+|
+            +-----------------------------------+
+
+  * Free space tracking
+    The freespace is tracked on a per-SP basis. We can use extent-based B+tree in XFS for free space tracking.
+    The freelist info contains the root of free space B+tree. Granularity is a data block in Data blocks partition.
+    The data block is the smallest and fixed size unit of data.
+
+    ::
+
+             +-----------------------------------+
+             | Free space B+tree root node block |
+             |          (Per-SP Meta)            |
+             |                                   |
+             |           # of records            |
+             |    left_sibling / right_sibling   |
+             | +--------------------------------+|
+             | | keys[# of records]             ||
+             | | +-----------------------------+||
+             | | |   startblock / blockcount   |||
+             | | |           ...               |||
+             | | +-----------------------------+||
+             | +--------------------------------||
+             | +--------------------------------+|
+             | | ptrs[# of records]             ||
+             | | +-----------------------------+||
+             | | |       SP block number       |||
+             | | |           ...               |||
+             | | +-----------------------------+||
+             | +--------------------------------+|
+             +-----------------------------------+
+
+* Omap and xattr
+  In this design, omap and xattr data is tracked by b+tree in onode. The onode only has the root node of b+tree.
+  The root node contains entries which indicate where the key onode exists.
+  So, if we know the onode, omap can be found via omap b+tree.
+
+* Fragmentation
+
+  - Internal fragmentation
+
+    We pack different types of data/metadata in a single block as many as possible to reduce internal fragmentation.
+    Extent-based B+tree may help reduce this further by allocating contiguous blocks that best fit for the object
+
+  - External fragmentation
+
+    Frequent object create/delete may lead to external fragmentation
+    In this case, we need cleaning work (GC-like) to address this.
+    For this, we are referring the NetApp’s Continuous Segment Cleaning, which seems similar to the SeaStore’s approach
+    Countering Fragmentation in an Enterprise Storage System (NetApp, ACM TOS, 2020)
+
+.. ditaa::
+
+
+       +---------------+-------------------+-------------+
+       | Freelist info | Onode radix tree  | Data blocks +-------+
+       +---------------+---------+---------+-+-----------+       |
+                                 |           |                   |
+            +--------------------+           |                   |
+            |                                |                   |
+            |        OID                     |                   |
+            |                                |                   |
+        +---+---+                            |                   |
+        | Root  |                            |                   |
+        +---+---+                            |                   |
+            |                                |                   |
+            v                                |                   |
+       /-----------------------------\       |                   |
+       |          Radix tree         |       |                   v
+       +---------+---------+---------+       |      /---------------\
+       | onode   | ...     | ...     |       |      | Num Chunk     |
+       +---------+---------+---------+       |      |               |
+    +--+ onode   | ...     | ...     |       |      | <Offset, len> |
+    |  +---------+---------+---------+       |      | <Offset, len> +-------+
+    |                                        |      | ...           |       |
+    |                                        |      +---------------+       |
+    |                                        |      ^                       |
+    |                                        |      |                       |
+    |                                        |      |                       |
+    |                                        |      |                       |
+    |  /---------------\  /-------------\    |      |                       v
+    +->| onode         |  | onode       |<---+      |       /------------+------------\
+       +---------------+  +-------------+           |       | Block0     | Block1     |
+       | OID           |  | OID         |           |       +------------+------------+
+       | Omaps         |  | Omaps       |           |       | Data       | Data       |
+       | Data Extent   |  | Data Extent +-----------+       +------------+------------+
+       +---------------+  +-------------+
+
+WAL
+---
+Each SP has a WAL.
+The data written to the WAL are metadata updates, free space update and small data.
+Note that only data smaller than the predefined threshold needs to be written to the WAL.
+The larger data is written to the unallocated free space and its onode's extent_tree is updated accordingly
+(also on-disk extent tree). We statically allocate WAL partition aside from data partition pre-configured.
+
+
+Partition and Reactor thread
+----------------------------
+In early stage development, PoseidonStore will employ static allocation of partition. The number of sharded partitions
+is fixed and the size of each partition also should be configured before running cluster.
+But, the number of partitions can grow as below. We leave this as a future work.
+Also, each reactor thread has a static set of SPs.
+
+.. ditaa::
+
+   +------+------+-------------+------------------+
+   | SP 1 | SP N | -->     <-- | global partition |
+   +------+------+-------------+------------------+
+
+
+
+Cache
+-----
+There are mainly two cache data structures; onode cache and block cache.
+It looks like below.
+
+#. Onode cache:
+   lru_map <OID, OnodeRef>;
+#. Block cache (data and omap):
+   Data cache --> lru_map <paddr, value>
+
+To fill the onode data structure, the target onode needs to be retrieved using the prefix tree.
+Block cache is used for caching a block contents. For a transaction, all the updates to blocks
+(including object meta block, data block) are first performed in the in-memory block cache.
+After writing a transaction to the WAL, the dirty blocks are flushed to their respective locations in the
+respective partitions.
+PoseidonStore can configure cache size for each type. Simple LRU cache eviction strategy can be used for both.
+
+
+Sharded partitions (with cross-SP transaction)
+----------------------------------------------
+The entire disk space is divided into a number of chunks called sharded partitions (SP).
+The prefixes of the parent collection ID (original collection ID before collection splitting. That is, hobject.hash)
+is used to map any collections to SPs.
+We can use BlueStore's approach for collection splitting, changing the number of significant bits for the collection prefixes.
+Because the prefixes of the parent collection ID do not change even after collection splitting, the mapping between
+the collection and SP are maintained.
+The number of SPs may be configured to match the number of CPUs allocated for each disk so that each SP can hold
+a number of objects large enough for cross-SP transaction not to occur.
+
+In case of need of cross-SP transaction, we could use the global WAL. The coordinator thread (mainly manages global partition) handles
+cross-SP transaction via acquire the source SP and target SP locks before processing the cross-SP transaction.
+Source and target probably are blocked.
+
+For the load unbalanced situation,
+Poseidonstore can create partitions to make full use of entire space efficiently and provide load balaning.
+
+
+CoW/Clone
+---------
+As for CoW/Clone, a clone has its own onode like other normal objects.
+
+Although each clone has its own onode, data blocks should be shared between the original object and clones
+if there are no changes on them to minimize the space overhead.
+To do so, the reference count for the data blocks is needed to manage those shared data blocks.
+
+To deal with the data blocks which has the reference count, poseidon store makes use of shared_blob
+which maintains the referenced data block.
+
+As shown the figure as below,
+the shared_blob tracks the data blocks shared between other onodes by using a reference count.
+The shared_blobs are managed by shared_blob_list in the superblock.
+
+
+.. ditaa::
+
+
+    /----------\         /----------\
+    | Object A |         | Object B |
+    +----------+         +----------+
+    | Extent   |         | Extent   |
+    +---+--+---+         +--+----+--+
+        |  |                |    |
+        |  |     +----------+    |
+        |  |     |               |
+        |  +---------------+     |
+        |        |         |     |
+        v        v         v     v
+    +---------------+---------------+
+    | Data block 1  | Data block 2  |
+    +-------+-------+------+--------+
+            |              |
+            v              v
+    /---------------+---------------\
+    | shared_blob 1 | shared_blob 2 |
+    +---------------+---------------+ shared_blob_list
+    | refcount      | refcount      |
+    +---------------+---------------+
+
+Plans
+=====
+
+All PRs should contain unit tests to verify its minimal functionality.
+
+* WAL and block cache implementation
+
+  As a first step, we are going to build the WAL including the I/O procedure to read/write the WAL.
+  With WAL development, the block cache needs to be developed together.
+  Besides, we are going to add an I/O library to read/write from/to the NVMe storage to
+  utilize NVMe feature and the asynchronous interface.
+
+* Radix tree and onode
+
+  First, submit a PR against this file with a more detailed on disk layout and lookup strategy for the onode radix tree.
+  Follow up with implementation based on the above design once design PR is merged.
+  The second PR will be the implementation regarding radix tree which is the key structure to look up
+  objects.
+
+* Extent tree
+
+  This PR is the extent tree to manage data blocks in the onode. We build the extent tree, and
+  demonstrate how it works when looking up the object.
+
+* B+tree for omap
+
+  We will put together a simple key/value interface for omap. This probably will be a separate PR.
+
+* CoW/Clone
+
+  To support CoW/Clone, shared_blob and shared_blob_list will be added.
+
+* Integration to Crimson as to I/O interfaces
+
+  At this stage, interfaces for interacting with Crimson such as queue_transaction(), read(), clone_range(), etc.
+  should work right.
+
+* Configuration
+
+  We will define Poseidon store configuration in detail.
+
+* Stress test environment and integration to teuthology
+
+  We will add stress tests and teuthology suites.
+
+.. rubric:: Footnotes
+
+.. [#f1] Stathis Maneas, Kaveh Mahdaviani, Tim Emami, Bianca Schroeder: A Study of SSD Reliability in Large Scale Enterprise Storage Deployments. FAST 2020: 137-149
diff --git a/doc/dev/cxx.rst b/doc/dev/cxx.rst
new file mode 100644
index 000000000..e8ab9d4de
--- /dev/null
+++ b/doc/dev/cxx.rst
@@ -0,0 +1,27 @@
+C++17 and libstdc++ ABI
+=======================
+
+Ceph has switched over to C++17 in mimic. To build Ceph on old distros without
+GCC-7, it is required to install GCC-7 from additionary repos. On RHEL/CentOS,
+we are using devtoolset-7_ from SCLs_ for building Ceph. But devltoolset-7 is
+always using the old ABI_ even if ``_GLIBCXX_USE_CXX11_ABI=1`` is defined. So,
+on RHEL/CentOS, the old implementations of ``std::string`` and ``std::list``
+are still used. In other words, ``std::string`` is still copy-on-write, and
+``std::list::size()`` is still O(n) on these distros. But on Ubuntu Xenial,
+Ceph is built using the new ABI. So, because we are still using libstdc++ and
+devtoolset for building packages on RHEL/CentOS, please do not rely on the
+behavior of the new ABI or the old one.
+
+For those who argue that "GCC supports dual ABI!", here comes the long story.
+The problem is in the system shared library and ``libstdc++_nonshared.a`` model.
+If some symbol is exported from the system shared library, we must use that, and
+cannot override it. Also, the dual ABI support requires several of the system
+shared library symbols to behave differently (e.g. for locale facets, need
+to register twice as many, one set for old ABI, another for new ABI). So, this
+leaves us with no options but to stick with the old ABI, if we want to enable
+the built binaries to run on old distros where only the libstdc++ with the old
+ABI is available.
+
+.. _ABI: https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
+.. _devtoolset-7: https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/
+.. _SCLs: https://www.softwarecollections.org/
diff --git a/doc/dev/dashboard/ui_goals.rst b/doc/dev/dashboard/ui_goals.rst
new file mode 100644
index 000000000..4e68ec1f5
--- /dev/null
+++ b/doc/dev/dashboard/ui_goals.rst
@@ -0,0 +1,78 @@
+===========================
+Ceph Dashboard Design Goals
+===========================
+
+.. note:: This document is intended to provide a focal point for discussing the overall design
+   principles for mgr/dashboard
+
+Introduction
+============
+
+Most distributed storage architectures are inherently complex and can present a management challenge
+to Operations teams who are typically stretched across multiple product and platform disciplines. In
+general terms, the complexity of any solution can have a direct bearing on the operational costs
+incurred to manage it. The answer is simple...make it simple :)
+
+This document is intended to highlight Ceph Dashboard design goals which may help to  
+
+* reduce complexity
+* increase productivity
+* improve time-to-value
+* increase observability
+
+ 
+Understanding the Persona of the Target User
+============================================
+
+Ceph has historically been administered from the CLI. The CLI has always and will always offer the
+richest, most flexible way to install and manage a Ceph cluster. Administrators who require and
+demand this level of control are unlikely to adopt a UI for anything more than a technical curiosity.
+
+The relevance of the UI is therefore more critical for a new SysAdmin, where it can help technology
+adoption and reduce the operational friction that is normally experienced when implementing a new
+solution.
+
+Understanding the target user persona is therefore a fundamental first step in design. Attempting to
+design a UI that meets the requirements of a 'seasoned' Ceph Administrator or Developer, and a
+relatively new SysAdmin is unlikely to satisfy either user group.
+
+Design Principles
+=================
+
+Key Principles
+______________
+
+
+#. **Clarity and consistency**. The UI should ensure the data shown is unambiguous and consistent across
+   different views
+#. **Data timeliness**. Data displayed in the UI must be timely. State information **must** be reasonably
+   recent for it to be relevant and acted upon with confidence. In addition, the age of the data should
+   be shown as age (e.g. 20s ago) rather than UTC timestamps to make it more immediately consumable by
+   the Administrator.
+#. **Automate through workflows**. If the admin has to follow a 'recipe' to perform a task, the goal of
+   the dashboard UI should be to implement the flow.
+#. **Provide a natural next step**. The UI **is** the *expert system*, so instead of expecting the user
+   to know where they go next, the UI should lead them. This means linking components together to
+   establish a flow and deeper integration between the alertmanager implementation and the dashboard
+   elements enabling an Admin to efficiently step from alert to affected component.
+#. **Platform visibility**. The platform (OS and hardware configuration) is a fundamental component of the
+   solution, so providing platform level insights can help deliver a more holistic view of the Ceph cluster.
+#. **Jargon Busting**. Jargon is an unavoidable component of most systems. However, a good system will
+   include inline help to support new and infrequent users of the UI.
+
+
+Common Pitfalls
+_______________
+
+* Don't re-implement CLI commands in the UI. The sysadmin will likely use the CLI primitives in scripts
+  to automate tasks, so by simply adding a CLI feature we miss the workflow and add complexity, which
+  potentially 'bloats' the UI.
+* Don't think like a developer...try and adopt the mindset of an Administrator, who only works with the
+  Ceph cluster part-time - this is the reality for today's Operations teams.
+
+
+Focus On User Experience
+========================
+Ultimately, the goal must be to move away from pushing complexity onto the GUI user through multi-step
+workflows like iSCSI configuration or setting specific cluster flags in defined sequences. Simplicity
+should be the goal for the UI...let's leave the complexity to the CLI.
diff --git a/doc/dev/deduplication.rst b/doc/dev/deduplication.rst
new file mode 100644
index 000000000..554031840
--- /dev/null
+++ b/doc/dev/deduplication.rst
@@ -0,0 +1,257 @@
+===============
+ Deduplication
+===============
+
+
+Introduction
+============
+
+Applying data deduplication on an existing software stack is not easy 
+due to additional metadata management and original data processing 
+procedure. 
+
+In a typical deduplication system, the input source as a data
+object is split into multiple chunks by a chunking algorithm.
+The deduplication system then compares each chunk with
+the existing data chunks, stored in the storage previously.
+To this end, a fingerprint index that stores the hash value
+of each chunk is employed by the deduplication system
+in order to easily find the existing chunks by comparing
+hash value rather than searching all contents that reside in
+the underlying storage.
+
+There are many challenges in order to implement deduplication on top
+of Ceph. Among them, two issues are essential for deduplication.
+First is managing scalability of fingerprint index; Second is
+it is complex to ensure compatibility between newly applied
+deduplication metadata and existing metadata.
+
+Key Idea
+========
+1. Content hashing (Double hashing): Each client can find an object data 
+for an object ID using CRUSH. With CRUSH, a client knows object's location
+in Base tier. 
+By hashing object's content at Base tier, a new OID (chunk ID) is generated.
+Chunk tier stores in the new OID that has a partial content of original object.
+
+ Client 1 -> OID=1 -> HASH(1's content)=K -> OID=K -> 
+ CRUSH(K) -> chunk's location
+
+
+2. Self-contained object: The external metadata design
+makes difficult for integration with storage feature support
+since existing storage features cannot recognize the
+additional external data structures. If we can design data
+deduplication system without any external component, the
+original storage features can be reused.
+
+More details in https://ieeexplore.ieee.org/document/8416369
+
+Design
+======
+
+.. ditaa::
+
+           +-------------+
+           | Ceph Client |
+           +------+------+
+                  ^
+     Tiering is   |  
+    Transparent   |               Metadata
+        to Ceph   |           +---------------+
+     Client Ops   |           |               |   
+                  |    +----->+   Base Pool   |
+                  |    |      |               |
+                  |    |      +-----+---+-----+
+                  |    |            |   ^ 
+                  v    v            |   |   Dedup metadata in Base Pool
+           +------+----+--+         |   |   (Dedup metadata contains chunk offsets
+           |   Objecter   |         |   |    and fingerprints)
+           +-----------+--+         |   |
+                       ^            |   |   Data in Chunk Pool
+                       |            v   |
+                       |      +-----+---+-----+
+                       |      |               |
+                       +----->|  Chunk Pool   |
+                              |               |
+                              +---------------+
+                                    Data
+
+
+Pool-based object management:
+We define two pools.
+The metadata pool stores metadata objects and the chunk pool stores
+chunk objects. Since these two pools are divided based on
+the purpose and usage, each pool can be managed more
+efficiently according to its different characteristics. Base
+pool and the chunk pool can separately select a redundancy
+scheme between replication and erasure coding depending on
+its usage and each pool can be placed in a different storage
+location depending on the required performance.
+
+Regarding how to use, please see ``osd_internals/manifest.rst``
+
+Usage Patterns
+==============
+
+Each Ceph interface layer presents unique opportunities and costs for
+deduplication and tiering in general.
+
+RadosGW
+-------
+
+S3 big data workloads seem like a good opportunity for deduplication.  These
+objects tend to be write once, read mostly objects which don't see partial
+overwrites.  As such, it makes sense to fingerprint and dedup up front.
+
+Unlike cephfs and rbd, radosgw has a system for storing
+explicit metadata in the head object of a logical s3 object for
+locating the remaining pieces.  As such, radosgw could use the
+refcounting machinery (``osd_internals/refcount.rst``) directly without
+needing direct support from rados for manifests.
+
+RBD/Cephfs
+----------
+
+RBD and CephFS both use deterministic naming schemes to partition
+block devices/file data over rados objects.  As such, the redirection
+metadata would need to be included as part of rados, presumably
+transparently.
+
+Moreover, unlike radosgw, rbd/cephfs rados objects can see overwrites.
+For those objects, we don't really want to perform dedup, and we don't
+want to pay a write latency penalty in the hot path to do so anyway.
+As such, performing tiering and dedup on cold objects in the background
+is likely to be preferred.
+   
+One important wrinkle, however, is that both rbd and cephfs workloads
+often feature usage of snapshots.  This means that the rados manifest
+support needs robust support for snapshots.
+
+RADOS Machinery
+===============
+
+For more information on rados redirect/chunk/dedup support, see ``osd_internals/manifest.rst``.
+For more information on rados refcount support, see ``osd_internals/refcount.rst``.
+
+Status and Future Work
+======================
+
+At the moment, there exists some preliminary support for manifest
+objects within the OSD as well as a dedup tool.
+
+RadosGW data warehouse workloads probably represent the largest
+opportunity for this feature, so the first priority is probably to add
+direct support for fingerprinting and redirects into the refcount pool
+to radosgw.
+
+Aside from radosgw, completing work on manifest object support in the
+OSD particularly as it relates to snapshots would be the next step for
+rbd and cephfs workloads.
+
+How to use deduplication
+========================
+
+ * This feature is highly experimental and is subject to change or removal.
+
+Ceph provides deduplication using RADOS machinery.
+Below we explain how to perform deduplication. 
+
+
+1. Estimate space saving ratio of a target pool using ``ceph-dedup-tool``.
+
+.. code:: bash
+
+    ceph-dedup-tool --op estimate --pool $POOL --chunk-size chunk_size  
+      --chunk-algorithm fixed|fastcdc --fingerprint-algorithm sha1|sha256|sha512
+      --max-thread THREAD_COUNT
+
+This CLI command will show how much storage space can be saved when deduplication
+is applied on the pool. If the amount of the saved space is higher than user's expectation,
+the pool probably is worth performing deduplication. 
+Users should specify $POOL where the object---the users want to perform
+deduplication---is stored. The users also need to run ceph-dedup-tool multiple time
+with varying ``chunk_size`` to find the optimal chunk size. Note that the
+optimal value probably differs in the content of each object in case of fastcdc
+chunk algorithm (not fixed). Example output:
+
+::
+
+    {
+      "chunk_algo": "fastcdc",
+      "chunk_sizes": [
+        {
+          "target_chunk_size": 8192,
+          "dedup_bytes_ratio": 0.4897049
+          "dedup_object_ratio": 34.567315
+          "chunk_size_average": 64439,
+          "chunk_size_stddev": 33620
+        }
+      ],
+      "summary": {
+        "examined_objects": 95,
+        "examined_bytes": 214968649
+      }
+    }
+
+The above is an example output when executing ``estimate``. ``target_chunk_size`` is the same as
+``chunk_size`` given by the user. ``dedup_bytes_ratio`` shows how many bytes are redundant from 
+examined bytes. For instance, 1 - ``dedup_bytes_ratio`` means the percentage of saved storage space.
+``dedup_object_ratio`` is the generated chunk objects / ``examined_objects``. ``chunk_size_average`` 
+means that the divided chunk size on average when performing CDC---this may differnet from ``target_chunk_size``
+because CDC genarates different chunk-boundary depending on the content. ``chunk_size_stddev``
+represents the standard deviation of the chunk size. 
+
+
+2. Create chunk pool. 
+
+.. code:: bash
+
+  ceph osd pool create CHUNK_POOL
+    
+
+3. Run dedup command (there are two ways).
+  
+.. code:: bash
+
+    ceph-dedup-tool --op sample-dedup --pool POOL --chunk-pool CHUNK_POOL --chunk-size 
+    CHUNK_SIZE --chunk-algorithm fastcdc --fingerprint-algorithm sha1|sha256|sha512 
+    --chunk-dedup-threshold THRESHOLD --max-thread THREAD_COUNT ----sampling-ratio SAMPLE_RATIO
+    --wakeup-period WAKEUP_PERIOD --loop --snap
+
+The ``sample-dedup`` comamnd spawns threads specified by ``THREAD_COUNT`` to deduplicate objects on
+the ``POOL``. According to sampling-ratio---do a full search if ``SAMPLE_RATIO`` is 100, the threads selectively
+perform deduplication if the chunk is redundant over ``THRESHOLD`` times during iteration.
+If --loop is set, the theads will wakeup after ``WAKEUP_PERIOD``. If not, the threads will exit after one iteration.
+
+.. code:: bash
+
+    ceph-dedup-tool --op object-dedup --pool POOL --object OID --chunk-pool CHUNK_POOL
+      --fingerprint-algorithm sha1|sha256|sha512 --dedup-cdc-chunk-size CHUNK_SIZE
+
+The ``object-dedup`` command triggers deduplication on the RADOS object specified by ``OID``.
+All parameters shown above must be specified. ``CHUNK_SIZE`` should be taken from
+the results of step 1 above.
+Note that when this command is executed, ``fastcdc`` will be set by default and other parameters
+such as ``FP`` and ``CHUNK_SIZE`` will be set as defaults for the pool.
+Deduplicated objects will appear in the chunk pool. If the object is mutated over time, user needs to re-run
+``object-dedup`` because chunk-boundary should be recalculated based on updated contents.
+The user needs to specify ``snap`` if the target object is snapshotted. After deduplication is done, the target
+object size in ``POOL`` is zero (evicted) and chunks objects are genereated---these appear in ``CHUNK_POOL``.
+
+
+4. Read/write I/Os
+
+After step 3, the users don't need to consider anything about I/Os. Deduplicated objects are
+completely compatible with existing RADOS operations.
+
+
+5. Run scrub to fix reference count 
+
+Reference mismatches can on rare occasions occur to false positives when handling reference counts for
+deduplicated RADOS objects. These mismatches will be fixed by periodically scrubbing the pool:
+
+.. code:: bash
+
+    ceph-dedup-tool --op chunk-scrub --op chunk-scrub --chunk-pool CHUNK_POOL --pool POOL --max-thread THREAD_COUNT
+  
diff --git a/doc/dev/delayed-delete.rst b/doc/dev/delayed-delete.rst
new file mode 100644
index 000000000..31f3e6b97
--- /dev/null
+++ b/doc/dev/delayed-delete.rst
@@ -0,0 +1,13 @@
+=========================
+ CephFS delayed deletion
+=========================
+
+The deletion of a file does not immediately remove its data. Each of the file's
+underlying objects must be removed independently. If these objects were removed 
+immediately, the client would have to send ``size_of_file / stripe_size *
+replication_count`` messages. This would consume significant bandwith and would
+slow the client unacceptably. If snapshots exist, their existence can prevent
+the deletion of objects associated with them.
+
+In these cases, such files are (1) marked as deleted on the MDS and (2) deleted
+lazily.
diff --git a/doc/dev/dev_cluster_deployment.rst b/doc/dev/dev_cluster_deployment.rst
new file mode 100644
index 000000000..7ee65a0c7
--- /dev/null
+++ b/doc/dev/dev_cluster_deployment.rst
@@ -0,0 +1,167 @@
+.. _dev_deploying_a_development_cluster:
+
+=================================
+ Deploying a development cluster
+=================================
+
+In order to develop on ceph, a Ceph utility,
+*vstart.sh*, allows you to deploy fake local cluster for development purpose.
+
+Usage
+=====
+
+It allows to deploy a fake local cluster on your machine for development purpose. It starts rgw, mon, osd and/or mds, or all of them if not specified.
+
+To start your development cluster, type the following::
+
+	vstart.sh [OPTIONS]...
+
+In order to stop the cluster, you can type::
+
+	./stop.sh
+
+Options
+=======
+
+.. option:: -b, --bluestore
+
+    Use bluestore as the objectstore backend for osds.
+
+.. option:: --cache <pool>
+
+    Set a cache-tier for the specified pool.
+
+.. option:: -d, --debug
+
+    Launch in debug mode.
+
+.. option:: -e
+
+    Create an erasure pool.
+
+.. option:: --hitset <pool> <hit_set_type>
+
+    Enable hitset tracking.
+
+.. option:: -i ip_address
+
+    Bind to the specified *ip_address* instead of guessing and resolve from hostname.
+
+.. option:: -k
+
+    Keep old configuration files instead of overwriting these.
+
+.. option:: -K, --kstore
+
+    Use kstore as the osd objectstore backend.
+
+.. option:: -l, --localhost
+
+    Use localhost instead of hostname.
+
+.. option:: -m ip[:port]
+
+    Specifies monitor *ip* address and *port*.
+
+.. option:: --memstore
+
+    Use memstore as the objectstore backend for osds
+
+.. option:: --multimds <count>
+
+    Allow multimds with maximum active count.
+
+.. option:: -n, --new
+
+    Create a new cluster.
+
+.. option:: -N, --not-new
+
+    Reuse existing cluster config (default).
+
+.. option:: --nodaemon
+
+    Use ceph-run as wrapper for mon/osd/mds.
+
+.. option:: --nolockdep
+
+    Disable lockdep
+
+.. option:: -o <config>
+
+    Add *config* to all sections in the ceph configuration.
+
+.. option:: --rgw_port <port>
+
+    Specify ceph rgw http listen port.
+
+.. option:: --rgw_frontend <frontend>
+
+    Specify the rgw frontend configuration (default is civetweb).
+
+.. option:: --rgw_compression <compression_type>
+
+    Specify the rgw compression plugin (default is disabled).
+
+.. option:: --smallmds
+
+    Configure mds with small limit cache size.
+
+.. option:: --short
+
+    Short object names only; necessary for ext4 dev
+
+.. option:: --valgrind[_{osd,mds,mon}] 'valgrind_toolname [args...]'
+
+    Launch the osd/mds/mon/all the ceph binaries using valgrind with the specified tool and arguments.
+
+.. option:: --without-dashboard
+
+    Do not run using mgr dashboard.
+
+.. option:: -x
+
+    Enable cephx (on by default).
+
+.. option:: -X
+
+    Disable cephx.
+
+
+Environment variables
+=====================
+
+{OSD,MDS,MON,RGW}
+
+These environment variables will contains the number of instances of the desired ceph process you want to start.
+
+Example: ::
+
+	OSD=3 MON=3 RGW=1 vstart.sh
+
+
+============================================================
+ Deploying multiple development clusters on the same machine
+============================================================
+
+In order to bring up multiple ceph clusters on the same machine, *mstart.sh* a
+small wrapper around the above *vstart* can help.
+
+Usage
+=====
+
+To start multiple clusters, you would run mstart for each cluster you would want
+to deploy, and it will start monitors, rgws for each cluster on different ports
+allowing you to run multiple mons, rgws etc. on the same cluster. Invoke it in
+the following way::
+
+  mstart.sh <cluster-name> <vstart options>
+
+For eg::
+
+  ./mstart.sh cluster1 -n
+
+
+For stopping the cluster, you do::
+
+  ./mstop.sh <cluster-name>
diff --git a/doc/dev/developer_guide/basic-workflow.rst b/doc/dev/developer_guide/basic-workflow.rst
new file mode 100644
index 000000000..27000fa2b
--- /dev/null
+++ b/doc/dev/developer_guide/basic-workflow.rst
@@ -0,0 +1,587 @@
+.. _basic workflow dev guide:
+
+Basic Workflow
+==============
+
+The following chart illustrates the basic Ceph development workflow:
+
+.. ditaa::
+
+            Upstream Code                       Your Local Environment
+
+           /----------\        git clone           /-------------\
+           |   Ceph   | -------------------------> | ceph/main   |
+           \----------/                            \-------------/
+                ^                                    |
+                |                                    | git branch fix_1
+                | git merge                          |
+                |                                    v
+           /----------------\  git commit --amend   /-------------\
+           | ninja check    |---------------------> | ceph/fix_1  |
+           | ceph--qa--suite|                       \-------------/
+           \----------------/                        |
+                ^                                    | fix changes
+                |                                    | test changes
+                | review                             | git commit
+                |                                    |
+                |                                    v
+           /--------------\                        /-------------\
+           |   github     |<---------------------- | ceph/fix_1  |
+           | pull request |         git push       \-------------/
+           \--------------/
+
+This page assumes that you are a new contributor with an idea for a bugfix or
+an enhancement, but you do not know how to proceed. Watch the `Getting Started
+with Ceph Development <https://www.youtube.com/watch?v=t5UIehZ1oLs>`_ video for
+a practical summary of this workflow.
+
+Updating the tracker
+--------------------
+
+Find the :ref:`issue-tracker` (Redmine) number of the bug you intend to fix. If
+no tracker issue exists, create one. There is only one case in which you do not
+have to create a Redmine tracker issue: the case of minor documentation changes.
+
+Simple documentation cleanup does not require a corresponding tracker issue.
+Major documentation changes do require a tracker issue. Major documentation
+changes include adding new documentation chapters or files, and making 
+substantial changes to the structure or content of the documentation.
+
+A (Redmine) tracker ticket explains the issue (bug) to other Ceph developers to
+keep them informed as the bug nears resolution. Provide a useful, clear title
+and include detailed information in the description. When composing the title
+of the ticket, ask yourself "If I need to search for this ticket two years from
+now, which keywords am I likely to search for?" Then include those keywords in
+the title.
+
+If your tracker permissions are elevated, assign the bug to yourself by setting
+the ``Assignee`` field. If your tracker permissions have not been elevated,
+just add a comment with a short message that says "I am working on this issue".
+
+Ceph Workflow Overview
+----------------------
+
+Three repositories are involved in the Ceph workflow. They are:
+
+1. The upstream repository (ceph/ceph)
+2. Your fork of the upstream repository (your_github_id/ceph)
+3. Your local working copy of the repository (on your workstation)
+
+The procedure for making changes to the Ceph repository is as follows:
+
+#. Configure your local environment
+
+   #. :ref:`Create a fork<forking>` of the "upstream Ceph"
+      repository.
+
+   #. :ref:`Clone the fork<cloning>` to your local filesystem.
+
+#. Fix the bug
+
+   #. :ref:`Synchronize local main with upstream main<synchronizing>`.
+         
+   #. :ref:`Create a bugfix branch<bugfix_branch>` in your local working copy.
+         
+   #. :ref:`Make alterations to the local working copy of the repository in your
+      local filesystem<fixing_bug_locally>`.
+   
+   #. :ref:`Push the changes in your local working copy to your fork<push_changes>`.
+
+#. Create a Pull Request to push the change upstream.
+
+   #. Create a Pull Request that asks for your changes to be added into the
+      "upstream Ceph" repository.
+
+Preparing Your Local Working Copy of the Ceph Repository 
+--------------------------------------------------------
+
+The procedures in this section, "Preparing Your Local Working Copy of the Ceph
+Repository", must be followed only when you are first setting up your local
+environment. If this is your first time working with the Ceph project, then
+these commands are necessary and are the first commands that you should run.
+
+.. _forking:
+
+Creating a Fork of the Ceph Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+See the `GitHub documentation
+<https://help.github.com/articles/fork-a-repo/#platform-linux>`_ for
+detailed instructions on forking. In short, if your GitHub username is
+"mygithubaccount", your fork of the upstream repo will appear at
+``https://github.com/mygithubaccount/ceph``. 
+
+.. _cloning:
+
+Cloning Your Fork  
+^^^^^^^^^^^^^^^^^
+
+After you have created your fork, clone it by running the following command:
+
+.. prompt:: bash $
+
+   git clone https://github.com/mygithubaccount/ceph
+
+You must fork the Ceph repository before you clone it.  If you fail to fork,
+you cannot open a `GitHub pull request
+<https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request>`_.
+
+For more information on using GitHub, refer to `GitHub Help
+<https://help.github.com/>`_.
+
+Configuring Your Local Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The commands in this section configure your local git environment so that it
+generates "Signed-off-by:" tags. These commands also set up your local
+environment so that it can stay synchronized with the upstream repository.
+
+These commands are necessary only during the initial setup of your local
+working copy. Another way to say that is "These commands are necessary
+only the first time that you are working with the Ceph repository. They are,
+however, unavoidable, and if you fail to run them then you will not be able
+to work on the Ceph repository.".
+
+1. Configure your local git environment with your name and email address.  
+
+   .. note::
+      These commands will work only from within the ``ceph/`` directory
+      that was created when you cloned your fork.
+
+   .. prompt:: bash $
+
+      git config user.name "FIRST_NAME LAST_NAME"
+      git config user.email "MY_NAME@example.com"
+
+2. Add the upstream repo as a "remote" and fetch it:
+
+   .. prompt:: bash $
+
+      git remote add ceph https://github.com/ceph/ceph.git
+      git fetch ceph
+
+   These commands fetch all the branches and commits from ``ceph/ceph.git`` to
+   the local git repo as ``remotes/ceph/$BRANCH_NAME`` and can be referenced as
+   ``ceph/$BRANCH_NAME`` in local git commands.
+
+Fixing the Bug
+--------------
+
+.. _synchronizing:
+
+Synchronizing Local Main with Upstream Main
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In your local working copy, there is a copy of the ``main`` branch in
+``remotes/origin/main``. This is called "local main". This copy of the
+main branch (https://github.com/your_github_id/ceph.git) is "frozen in time"
+at the moment that you cloned it, but the upstream repo
+(https://github.com/ceph/ceph.git, typically abbreviated to ``ceph/ceph.git``)
+that it was forked from is not frozen in time: the upstream repo is still being
+updated by other contributors. 
+
+Because upstream main is continually receiving updates from other
+contributors, your fork will drift farther and farther from the state of the
+upstream repo when you cloned it.
+
+Keep your fork's ``main`` branch synchronized with upstream main to reduce drift
+between your fork's main branch and the upstream main branch.
+
+Here are the commands for keeping your fork synchronized with the
+upstream repository:
+
+.. prompt:: bash $
+
+   git fetch ceph
+   git checkout main 
+   git reset --hard ceph/main
+   git push -u origin main
+
+Follow this procedure often to keep your local ``main`` in sync with upstream
+``main``.
+
+If the command ``git status`` returns a line that reads "Untracked files", see
+:ref:`the procedure on updating submodules <update-submodules>`.
+
+.. _bugfix_branch:
+
+Creating a Bugfix branch
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Create a branch for your bugfix:
+
+.. prompt:: bash $
+
+   git checkout main 
+   git checkout -b fix_1
+   git push -u origin fix_1
+
+The first command (git checkout main) makes sure that the bugfix branch
+"fix_1" is created from the most recent state of the main branch of the
+upstream repository. 
+
+The second command (git checkout -b fix_1) creates a "bugfix branch" called
+"fix_1" in your local working copy of the repository. The changes that you make
+in order to fix the bug will be committed to this branch.
+
+The third command (git push -u origin fix_1) pushes the bugfix branch from
+your local working repository to your fork of the upstream repository.
+
+.. _fixing_bug_locally:
+
+Fixing the bug in the local working copy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. **Updating the tracker**
+
+   In the `Ceph issue tracker <https://tracker.ceph.com>`_, change the status
+   of the tracker issue to "In progress".  This communicates to other Ceph
+   contributors that you have begun working on a fix, which helps to avoid
+   duplication of effort. If you don't have permission to change that field,
+   just comment that you are working on the issue. 
+
+#. **Fixing the bug itself**
+
+   This guide cannot tell you how to fix the bug that you have chosen to fix.
+   This guide assumes that you know what required improvement, and that you
+   know what to do to provide that improvement.
+
+   It might be that your fix is simple and requires only minimal testing. But
+   that's unlikely. It is more likely that the process of fixing your bug will
+   be iterative and will involve trial, error, skill, and patience. 
+
+   For a detailed discussion of the tools available for validating bugfixes,
+   see the chapters on testing.
+
+Pushing the Fix to Your Fork
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   
+You have finished work on the bugfix. You have tested the bugfix, and you
+believe that it works. 
+   
+#. Commit the changes to your local working copy.
+
+   Commit the changes to the `fix_1` branch of your local working copy by using
+   the ``--signoff`` option (here represented as the `s` portion of the `-as`
+   flag): 
+
+   .. prompt:: bash $
+
+      git commit -as
+
+   .. _push_changes:
+
+#. Push the changes to your fork:
+
+   Push the changes from the `fix_1` branch of your local working copy to the
+   `fix_1` branch of your fork of the upstream repository:
+
+   .. prompt:: bash $
+
+      git push origin fix_1
+   
+   .. note::
+
+      In the command ``git push origin fix_1``, ``origin`` is the name of your
+      fork of the upstream Ceph repository, and can be thought of as a nickname
+      for ``git@github.com:username/ceph.git``, where ``username`` is your
+      GitHub username.
+
+      It is possible that ``origin`` is not the name of your fork. Discover the
+      name of your fork by running ``git remote -v``, as shown here:
+
+      .. code-block:: bash
+
+         $ git remote -v
+         ceph	https://github.com/ceph/ceph.git (fetch)
+         ceph	https://github.com/ceph/ceph.git (push)
+         origin	git@github.com:username/ceph.git (fetch)
+         origin	git@github.com:username/ceph.git (push)
+
+      The line::
+     
+         origin git@github.com:username/ceph.git (fetch) 
+      
+      and the line:: 
+        
+         origin git@github.com:username/ceph.git (push) 
+         
+      provide the information that "origin" is the name of your fork of the
+      Ceph repository.
+
+
+Opening a GitHub pull request
+-----------------------------
+
+After you have pushed the bugfix to your fork, open a GitHub pull request
+(PR). This makes your bugfix visible to the community of Ceph contributors.
+They will review it. They may perform additional testing on your bugfix, and
+they might request changes to the bugfix.
+
+Be prepared to receive suggestions and constructive criticism in the form of
+comments within the PR. 
+
+If you don't know how to create and manage pull requests, read `this GitHub
+pull request tutorial`_.
+
+.. _`this GitHub pull request tutorial`:
+   https://help.github.com/articles/using-pull-requests/
+
+To learn what constitutes a "good" pull request, see
+the `Git Commit Good Practice`_ article at the `OpenStack Project Wiki`_.
+
+.. _`Git Commit Good Practice`: https://wiki.openstack.org/wiki/GitCommitMessages
+.. _`OpenStack Project Wiki`: https://wiki.openstack.org/wiki/Main_Page
+
+See also our own `Submitting Patches
+<https://github.com/ceph/ceph/blob/main/SubmittingPatches.rst>`_ document.
+
+After your pull request (PR) has been opened, update the :ref:`issue-tracker`
+by adding a comment directing other contributors to your PR. The comment can be
+as simple as this::
+
+    *PR*: https://github.com/ceph/ceph/pull/$NUMBER_OF_YOUR_PULL_REQUEST
+
+Understanding Automated PR validation
+-------------------------------------
+
+When you create or update your PR, the Ceph project's `Continuous Integration
+(CI) <https://en.wikipedia.org/wiki/Continuous_integration>`_ infrastructure
+automatically tests it. At the time of this writing (May 2022), the automated
+CI testing included many tests. These five are among them:
+
+#. a test to check that the commits are properly signed (see :ref:`submitting-patches`):
+#. a test to check that the documentation builds
+#. a test to check that the submodules are unmodified
+#. a test to check that the API is in order
+#. a :ref:`make check<make-check>` test
+
+Additional tests may be run depending on which files your PR modifies.
+
+The :ref:`make check<make-check>` test builds the PR and runs it through a
+battery of tests. These tests run on servers that are operated by the Ceph
+Continuous Integration (CI) team. When the tests have completed their run, the
+result is shown on GitHub in the pull request itself.
+
+Test your modifications before you open a PR.  Refer to the chapters
+on testing for details.
+
+Notes on PR make check test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The GitHub :ref:`make check<make-check>` test is driven by a Jenkins instance.
+
+Jenkins merges your PR branch into the latest version of the base branch before
+it starts any tests. This means that you don't have to rebase the PR in order
+to pick up any fixes.
+
+You can trigger PR tests at any time by adding a comment to the PR - the
+comment should contain the string "test this please". Since a human who is
+subscribed to the PR might interpret that as a request for him or her to test
+the PR, you must address Jenkins directly. For example, write "jenkins retest
+this please". If you need to run only one of the tests, you can request it with
+a command like "jenkins test signed". A list of these requests is automatically
+added to the end of each new PR's description, so check there to find the
+single test you need.
+
+If there is a build failure and you aren't sure what caused it, check the
+:ref:`make check<make-check>` log. To access the make check log, click the
+"details" (next to the :ref:`make check<make-check>` test in the PR) link to
+enter the Jenkins web GUI. Then click "Console Output" (on the left).
+
+Jenkins is configured to search logs for strings that are known to have been
+associated with :ref:`make check<make-check>` failures in the past. However,
+there is no guarantee that these known strings are associated with any given
+:ref:`make check<make-check>` failure. You'll have to read through the log to
+determine the cause of your specific failure.
+
+Integration tests AKA ceph-qa-suite
+-----------------------------------
+
+It may be necessary to test your fix on real Ceph clusters that run on physical
+or virtual hardware. Tests designed for this purpose live in the `ceph/qa
+sub-directory`_ and are run via the `teuthology framework`_.
+
+.. _`ceph/qa sub-directory`: https://github.com/ceph/ceph/tree/main/qa/
+.. _`teuthology repository`: https://github.com/ceph/teuthology
+.. _`teuthology framework`: https://github.com/ceph/teuthology
+
+The Ceph community has access to the `Sepia lab
+<https://wiki.sepia.ceph.com/doku.php>`_ where `integration tests`_ can be run
+on physical hardware.
+
+Other contributors might add tags like `needs-qa` to your PR. This allows PRs
+to be merged into a single branch and then efficiently tested together.
+Teuthology test suites can take hours (and even days in some cases) to
+complete, so batching tests reduces contention for resources and saves a lot of
+time.
+
+To request access to the Sepia lab, start `here
+<https://wiki.sepia.ceph.com/doku.php?id=vpnaccess>`_.
+
+Integration testing is discussed in more detail in the `integration
+tests`_ chapter.
+
+.. _integration tests: ../testing_integration_tests/tests-integration-testing-teuthology-intro
+
+Code review
+-----------
+
+Once your bugfix has been thoroughly tested, or even during this process,
+it will be subjected to code review by other developers. This typically
+takes the form of comments in the PR itself, but can be supplemented
+by discussions on :ref:`irc` and the :ref:`mailing-list`.
+
+Amending your PR
+----------------
+
+While your PR is going through testing and `Code Review`_, you can
+modify it at any time by editing files in your local branch.
+
+After updates are committed locally (to the ``fix_1`` branch in our
+example), they need to be pushed to GitHub so they appear in the PR.
+
+Modifying the PR is done by adding commits to the ``fix_1`` branch upon
+which it is based, often followed by rebasing to modify the branch's git
+history. See `this tutorial
+<https://www.atlassian.com/git/tutorials/rewriting-history>`_ for a good
+introduction to rebasing. When you are done with your modifications, you
+will need to force push your branch with:
+
+.. prompt:: bash $
+
+   git push --force origin fix_1
+
+Why do we take these extra steps instead of simply adding additional commits
+the PR?  It is best practice for a PR to consist of a single commit; this
+makes for clean history, eases peer review of your changes, and facilitates
+merges.  In rare circumstances it also makes it easier to cleanly revert
+changes.
+
+Merging
+-------
+
+The bugfix process completes when a project lead merges your PR.
+
+When this happens, it is a signal for you (or the lead who merged the PR)
+to change the :ref:`issue-tracker` status to "Resolved". Some issues may be
+flagged for backporting, in which case the status should be changed to
+"Pending Backport" (see the :ref:`backporting` chapter for details).
+
+See also :ref:`merging` for more information on merging.
+
+Proper Merge Commit Format
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is the most basic form of a merge commit::
+
+       doc/component: title of the commit 
+
+       Reviewed-by: Reviewer Name <rname@example.com>
+
+This consists of two parts:
+
+#. The title of the commit / PR to be merged.
+#. The name and email address of the reviewer. Enclose the reviewer's email 
+   address in angle brackets.
+
+Using a browser extension to auto-fill the merge  message
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you use a browser for merging GitHub PRs, the easiest way to fill in
+the merge message is with the `"Ceph GitHub Helper Extension"
+<https://github.com/tspmelo/ceph-github-helper>`_ (available for `Chrome
+<https://chrome.google.com/webstore/detail/ceph-github-helper/ikpfebikkeabmdnccbimlomheocpgkmn>`_
+and `Firefox <https://addons.mozilla.org/en-US/firefox/addon/ceph-github-helper/>`_).
+
+After enabling this extension, if you go to a GitHub PR page, a vertical helper
+will be displayed at the top-right corner. If you click on the user silhouette button
+the merge message input will be automatically populated.
+
+Using .githubmap to Find a Reviewer's Email Address
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you cannot find the email address of the reviewer on his or her GitHub
+page, you can look it up in the **.githubmap** file, which can be found in
+the repository at **/ceph/.githubmap**.
+
+Using "git log" to find a Reviewer's Email Address
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you cannot find a reviewer's email address by using the above methods, you
+can search the git log for their email address. Reviewers are likely to have
+committed something before.  If they have made previous contributions, the git
+log will probably contain their email address.
+
+Use the following command
+
+.. prompt:: bash [branch-under-review]$
+
+   git log
+
+Using ptl-tool to Generate Merge Commits
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Another method of generating merge commits involves using Patrick Donnelly's
+**ptl-tool** pull commits. This tool can be found at
+**/ceph/src/script/ptl-tool.py**.  Merge commits that have been generated by
+the **ptl-tool** have the following form::
+
+     Merge PR #36257 into main 
+     * refs/pull/36257/head:
+             client: move client_lock to _unmount()
+             client: add timer_lock support
+     Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+Miscellaneous
+-------------
+
+--set-upstream
+^^^^^^^^^^^^^^
+
+If you forget to include the ``--set-upstream origin x`` option in your ``git
+push`` command, you will see the following error message:
+
+::
+
+   fatal: The current branch {x} has no upstream branch.
+   To push the current branch and set the remote as upstream, use
+      git push --set-upstream origin {x}
+
+To set up git to automatically create the upstream branch that corresponds to
+the branch in your local working copy, run this command from within the
+``ceph/`` directory:
+
+.. prompt:: bash $
+
+   git config --global push.autoSetupRemote true
+
+Deleting a Branch Locally
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To delete the branch named ``localBranchName`` from the local working copy, run
+a command of this form:
+
+.. prompt:: bash $
+
+   git branch -d localBranchName
+
+Deleting a Branch Remotely
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To delete the branch named ``remoteBranchName`` from the remote upstream branch
+(which is also your fork of ``ceph/ceph``, as described in :ref:`forking`), run
+a command of this form:
+
+.. prompt:: bash $
+
+   git push origin --delete remoteBranchName
+
+Searching a File Longitudinally for a String
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To search for the commit that introduced a given string (in this example, that
+string is ``foo``) into a given file (in this example, that file is
+``file.rst``), run a command of this form:
+
+.. prompt:: bash $
+
+   git log -S 'foo' file.rst
diff --git a/doc/dev/developer_guide/dash-devel.rst b/doc/dev/developer_guide/dash-devel.rst
new file mode 100644
index 000000000..1277cecc5
--- /dev/null
+++ b/doc/dev/developer_guide/dash-devel.rst
@@ -0,0 +1,2748 @@
+.. _dashdevel:
+
+Ceph Dashboard Developer Documentation
+======================================
+
+.. contents:: Table of Contents
+
+Feature Design
+--------------
+
+To promote collaboration on new Ceph Dashboard features, the first step is
+the definition of a design document. These documents then form the basis of
+implementation scope and permit wider participation in the evolution of the
+Ceph Dashboard UI.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Design Documents:
+
+    UI Design Goals <../dashboard/ui_goals>
+
+
+Preliminary Steps
+-----------------
+
+The following documentation chapters expect a running Ceph cluster and at
+least a running ``dashboard`` manager module (with few exceptions). This
+chapter gives an introduction on how to set up such a system for development,
+without the need to set up a full-blown production environment. All options
+introduced in this chapter are based on a so called ``vstart`` environment.
+
+.. note::
+
+  Every ``vstart`` environment needs Ceph `to be compiled`_ from its GitHub
+  repository, though Docker environments simplify that step by providing a
+  shell script that contains those instructions.
+
+  One exception to this rule are the `build-free`_ capabilities of
+  `ceph-dev`_. See below for more information.
+
+.. _to be compiled: https://docs.ceph.com/docs/master/install/build-ceph/
+
+vstart
+~~~~~~
+
+"vstart" is actually a shell script in the ``src/`` directory of the Ceph
+repository (``src/vstart.sh``). It is used to start a single node Ceph
+cluster on the machine where it is executed. Several required and some
+optional Ceph internal services are started automatically when it is used to
+start a Ceph cluster. vstart is the basis for the three most commonly used
+development environments in Ceph Dashboard.
+
+You can read more about vstart in :ref:`Deploying a development cluster
+<dev_deploying_a_development_cluster>`. Additional information for developers
+can also be found in the `Developer Guide`_.
+
+.. _Developer Guide: https://docs.ceph.com/docs/master/dev/quick_guide/
+
+Host-based vs Docker-based Development Environments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This document introduces you to three different development environments, all
+based on vstart. Those are:
+
+- vstart running on your host system
+
+- vstart running in a Docker environment
+
+  * ceph-dev-docker_
+  * ceph-dev_
+
+  Besides their independent development branches and sometimes slightly
+  different approaches, they also differ with respect to their underlying
+  operating systems.
+
+  ========= ======================  ========
+  Release   ceph-dev-docker         ceph-dev
+  ========= ======================  ========
+  Mimic     openSUSE Leap 15        CentOS 7
+  Nautilus  openSUSE Leap 15        CentOS 7
+  Octopus   openSUSE Leap 15.2      CentOS 8
+  --------- ----------------------  --------
+  Master    openSUSE Tumbleweed     CentOS 8
+  ========= ======================  ========
+
+.. note::
+
+  Independently of which of these environments you will choose, you need to
+  compile Ceph in that environment. If you compiled Ceph on your host system,
+  you would have to recompile it on Docker to be able to switch to a Docker
+  based solution. The same is true vice versa. If you previously used a
+  Docker development environment and compiled Ceph there and you now want to
+  switch to your host system, you will also need to recompile Ceph (or
+  compile Ceph using another separate repository).
+
+  `ceph-dev`_ is an exception to this rule as one of the options it provides
+  is `build-free`_. This is accomplished through a Ceph installation using
+  RPM system packages. You will still be able to work with a local GitHub
+  repository like you are used to.
+
+
+Development environment on your host system
+...........................................
+
+- No need to learn or have experience with Docker, jump in right away.
+
+- Limited amount of scripts to support automation (like Ceph compilation).
+
+- No pre-configured easy-to-start services (Prometheus, Grafana, etc).
+
+- Limited amount of host operating systems supported, depending on which
+  Ceph version is supposed to be used.
+
+- Dependencies need to be installed on your host.
+
+- You might find yourself in the situation where you need to upgrade your
+  host operating system (for instance due to a change of the GCC version used
+  to compile Ceph).
+
+
+Development environments based on Docker
+........................................
+
+- Some overhead in learning Docker if you are not used to it yet.
+
+- Both Docker projects provide you with scripts that help you getting started
+  and automate recurring tasks.
+
+- Both Docker environments come with partly pre-configured external services
+  which can be used to attach to or complement Ceph Dashboard features, like
+
+  - Prometheus
+  - Grafana
+  - Node-Exporter
+  - Shibboleth
+  - HAProxy
+
+- Works independently of the operating system you use on your host.
+
+
+.. _build-free: https://github.com/rhcs-dashboard/ceph-dev#quick-install-rpm-based
+
+vstart on your host system
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The vstart script is usually called from your `build/` directory like so:
+
+.. code::
+
+  ../src/vstart.sh -n -d
+
+In this case ``-n`` ensures that a new vstart cluster is created and that a
+possibly previously created cluster isn't re-used. ``-d`` enables debug
+messages in log files. There are several more options to chose from. You can
+get a list using the ``--help`` argument.
+
+At the end of the output of vstart, there should be information about the
+dashboard and its URLs::
+
+  vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output.
+
+  dashboard urls: https://192.168.178.84:41259, https://192.168.178.84:43259, https://192.168.178.84:45259
+    w/ user/pass: admin / admin
+  restful urls: https://192.168.178.84:42259, https://192.168.178.84:44259, https://192.168.178.84:46259
+    w/ user/pass: admin / 598da51f-8cd1-4161-a970-b2944d5ad200
+
+During development (especially in backend development), you also want to
+check on occasions if the dashboard manager module is still running. To do so
+you can call `./bin/ceph mgr services` manually. It will list all the URLs of
+successfully enabled services. Only URLs of services which are available over
+HTTP(S) will be listed there. Ceph Dashboard is one of these services. It
+should look similar to the following output:
+
+.. code::
+
+  $ ./bin/ceph mgr services
+  {
+      "dashboard": "https://home:41931/",
+      "restful": "https://home:42931/"
+  }
+
+By default, this environment uses a randomly chosen port for Ceph Dashboard
+and you need to use this command to find out which one it has become.
+
+Docker
+~~~~~~
+
+Docker development environments usually ship with a lot of useful scripts.
+``ceph-dev-docker`` for instance contains a file called `start-ceph.sh`,
+which cleans up log files, always starts a Rados Gateway service, sets some
+Ceph Dashboard configuration options and automatically runs a frontend proxy,
+all before or after starting up your vstart cluster.
+
+Instructions on how to use those environments are contained in their
+respective repository README files.
+
+- ceph-dev-docker_
+- ceph-dev_
+
+.. _ceph-dev-docker: https://github.com/ricardoasmarques/ceph-dev-docker
+.. _ceph-dev: https://github.com/rhcs-dashboard/ceph-dev
+
+Frontend Development
+--------------------
+
+Before you can start the dashboard from within a development environment, you
+will need to generate the frontend code and either use a compiled and running
+Ceph cluster (e.g. started by ``vstart.sh``) or the standalone development web
+server.
+
+The build process is based on `Node.js <https://nodejs.org/>`_ and requires the
+`Node Package Manager <https://www.npmjs.com/>`_ ``npm`` to be installed.
+
+Prerequisites
+~~~~~~~~~~~~~
+
+ * Node 18.17.0 or higher
+ * NPM 9.6.7 or higher
+
+nodeenv:
+  During Ceph's build we create a virtualenv with ``node`` and ``npm``
+  installed, which can be used as an alternative to installing node/npm in your
+  system.
+
+  If you want to use the node installed in the virtualenv you just need to
+  activate the virtualenv before you run any npm commands. To activate it run
+  ``. build/src/pybind/mgr/dashboard/node-env/bin/activate``.
+
+  Once you finish, you can simply run ``deactivate`` and exit the virtualenv.
+
+Angular CLI:
+  If you do not have the `Angular CLI <https://github.com/angular/angular-cli>`_
+  installed globally, then you need to execute ``ng`` commands with an
+  additional ``npm run`` before it.
+
+Package installation
+~~~~~~~~~~~~~~~~~~~~
+
+Run ``npm ci`` in directory ``src/pybind/mgr/dashboard/frontend`` to
+install the required packages locally.
+
+Adding or updating packages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run the following commands to add/update a package::
+
+  npm install <PACKAGE_NAME>
+  npm ci
+
+Setting up a Development Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create the ``proxy.conf.json`` file based on ``proxy.conf.json.sample``.
+
+Run ``npm start`` for a dev server.
+Navigate to ``http://localhost:4200/``. The app will automatically
+reload if you change any of the source files.
+
+Code Scaffolding
+~~~~~~~~~~~~~~~~
+
+Run ``ng generate component component-name`` to generate a new
+component. You can also use
+``ng generate directive|pipe|service|class|guard|interface|enum|module``.
+
+Build the Project
+~~~~~~~~~~~~~~~~~
+
+Run ``npm run build`` to build the project. The build artifacts will be
+stored in the ``dist/`` directory. Use the ``--prod`` flag for a
+production build (``npm run build -- --prod``). Navigate to ``https://localhost:8443``.
+
+Build the Code Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run ``npm run doc-build`` to generate code docs in the ``documentation/``
+directory. To make them accessible locally for a web browser, run
+``npm run doc-serve`` and they will become available at ``http://localhost:8444``.
+With ``npm run compodoc -- <opts>`` you may
+`fully configure it <https://compodoc.app/guides/usage.html>`_.
+
+Code linting and formatting
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We use the following tools to lint and format the code in all our TS, SCSS and
+HTML files:
+
+- `codelyzer <http://codelyzer.com/>`_
+- `html-linter <https://github.com/chinchiheather/html-linter>`_
+- `htmllint-cli <https://github.com/htmllint/htmllint-cli>`_
+- `Prettier <https://prettier.io/>`_
+- `ESLint <https://eslint.org/>`_
+- `stylelint <https://stylelint.io/>`_
+
+We added 2 npm scripts to help run these tools:
+
+- ``npm run lint``, will check frontend files against all linters
+- ``npm run fix``, will try to fix all the detected linting errors
+
+Ceph Dashboard and Bootstrap
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently we are using Bootstrap on the Ceph Dashboard as a CSS framework. This means that most of our SCSS and HTML
+code can make use of all the utilities and other advantages Bootstrap is offering. In the past we often have used our
+own custom styles and this lead to more and more variables with a single use and double defined variables which
+sometimes are forgotten to be removed or it led to styling be inconsistent because people forgot to change a color or to
+adjust a custom SCSS class.
+
+To get the current version of Bootstrap used inside Ceph please refer to the ``package.json`` and search for:
+
+- ``bootstrap``: For the Bootstrap version used.
+- ``@ng-bootstrap``: For the version of the Angular bindings which we are using.
+
+So for the future please do the following when visiting a component:
+
+- Does this HTML/SCSS code use custom code? - If yes: Is it needed? --> Clean it up before changing the things you want
+  to fix or change.
+- If you are creating a new component: Please make use of Bootstrap as much as reasonably possible! Don't try to
+  reinvent the wheel.
+- If possible please look up if Bootstrap has guidelines on how to extend it properly to do achieve what you want to
+  achieve.
+
+The more bootstrap alike our code is the easier it is to theme, to maintain and the less bugs we will have. Also since
+Bootstrap is a framework which tries to have usability and user experience in mind we increase both points
+exponentially. The biggest benefit of all is that there is less code for us to maintain which makes it easier to read
+for beginners and even more easy for people how are already familiar with the code.
+
+Writing Unit Tests
+~~~~~~~~~~~~~~~~~~
+
+To write unit tests most efficient we have a small collection of tools,
+we use within test suites.
+
+Those tools can be found under
+``src/pybind/mgr/dashboard/frontend/src/testing/``, especially take
+a look at ``unit-test-helper.ts``.
+
+There you will be able to find:
+
+``configureTestBed`` that replaces the initial ``TestBed``
+methods. It takes the same arguments as ``TestBed.configureTestingModule``.
+Using it will run your tests a lot faster in development, as it doesn't
+recreate everything from scratch on every test. To use the default behaviour
+pass ``true`` as the second argument.
+
+``PermissionHelper`` to help determine if
+the correct actions are shown based on the current permissions and selection
+in a list.
+
+``FormHelper`` which makes testing a form a lot easier
+with a few simple methods. It allows you to set a control or multiple
+controls, expect if a control is valid or has an error or just do both with
+one method. Additional you can expect a template element or multiple elements
+to be visible in the rendered template.
+
+Running Unit Tests
+~~~~~~~~~~~~~~~~~~
+
+Run ``npm run test`` to execute the unit tests via `Jest
+<https://facebook.github.io/jest/>`_.
+
+If you get errors on all tests, it could be because `Jest
+<https://facebook.github.io/jest/>`__ or something else was updated.
+There are a few ways how you can try to resolve this:
+
+- Remove all modules with ``rm -rf dist node_modules`` and run ``npm install``
+  again in order to reinstall them
+- Clear the cache of jest by running ``npx jest --clearCache``
+
+Running End-to-End (E2E) Tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We use `Cypress <https://www.cypress.io/>`__ to run our frontend E2E tests.
+
+E2E Prerequisites
+.................
+
+You need to previously build the frontend.
+
+In some environments, depending on your user permissions and the CYPRESS_CACHE_FOLDER,
+you might need to run ``npm ci`` with the ``--unsafe-perm`` flag.
+
+You might need to install additional packages to be able to run Cypress.
+Please run ``npx cypress verify`` to verify it.
+
+run-frontend-e2e-tests.sh
+.........................
+
+Our ``run-frontend-e2e-tests.sh`` script is the go to solution when you wish to
+do a full scale e2e run.
+It will verify if everything needed is installed, start a new vstart cluster
+and run the full test suite.
+
+Start all frontend E2E tests with::
+
+  $ cd src/pybind/mgr/dashboard
+  $ ./run-frontend-e2e-tests.sh
+
+Report:
+  You can follow the e2e report on the terminal and you can find the screenshots
+  of failed test cases by opening the following directory::
+
+    src/pybind/mgr/dashboard/frontend/cypress/screenshots/
+
+Device:
+  You can force the script to use a specific device with the ``-d`` flag::
+
+    $ ./run-frontend-e2e-tests.sh -d <chrome|chromium|electron|docker>
+
+Remote:
+  By default this script will stop and start a new vstart cluster.
+  If you want to run the tests outside the ceph environment, you will need to
+  manually define the dashboard url using ``-r`` and, optionally, credentials
+  (``-u``, ``-p``)::
+
+    $ ./run-frontend-e2e-tests.sh -r <DASHBOARD_URL> -u <E2E_LOGIN_USER> -p <E2E_LOGIN_PWD>
+
+Note:
+  When using docker, as your device, you might need to run the script with sudo
+  permissions.
+
+run-cephadm-e2e-tests.sh
+.........................
+
+``run-cephadm-e2e-tests.sh`` runs a subset of E2E tests to verify that the Dashboard and cephadm as
+Orchestrator backend behave correctly.
+
+Prerequisites: you need to install `KCLI
+<https://kcli.readthedocs.io/en/latest/>`_ and Node.js in your local machine.
+
+Configure KCLI plan requirements::
+
+  $ sudo chown -R $(id -un) /var/lib/libvirt/images
+  $ mkdir -p /var/lib/libvirt/images/ceph-dashboard
+  $ kcli create pool -p /var/lib/libvirt/images/ceph-dashboard ceph-dashboard
+  $ kcli create network -c 192.168.100.0/24 ceph-dashboard
+
+Note:
+  This script is aimed to be run as jenkins job so the cleanup is triggered only in a jenkins
+  environment. In local, the user will shutdown the cluster when desired (i.e. after debugging).
+
+Start E2E tests by running::
+
+  $ cd <your/ceph/repo/dir>
+  $ sudo chown -R $(id -un) src/pybind/mgr/dashboard/frontend/{dist,node_modules,src/environments}
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh
+
+Note:
+  In fedora 35, there can occur a permission error when trying to mount the shared_folders. This can be
+  fixed by running::
+
+    $ sudo setfacl -R -m u:qemu:rwx <abs-path-to-your-user-home>
+
+  or also by setting the appropriate permission to your $HOME directory
+
+You can also start a cluster in development mode (so the frontend build starts in watch mode and you
+only have to reload the page for the changes to be reflected) by running::
+
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh --dev-mode
+
+Note:
+  Add ``--expanded`` if you need a cluster ready to deploy services (one with enough monitor
+  daemons spread across different hosts and enough OSDs).
+
+Test your changes by running:
+
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh
+
+Shutdown the cluster by running:
+
+  $ kcli delete plan -y ceph
+  $ # In development mode, also kill the npm build watch process (e.g., pkill -f "ng build")
+
+Other running options
+.....................
+
+During active development, it is not recommended to run the previous script,
+as it is not prepared for constant file changes.
+Instead you should use one of the following commands:
+
+- ``npm run e2e`` - This will run ``ng serve`` and open the Cypress Test Runner.
+- ``npm run e2e:ci`` - This will run ``ng serve`` and run the Cypress Test Runner once.
+- ``npx cypress run`` - This calls cypress directly and will run the Cypress Test Runner.
+  You need to have a running frontend server.
+- ``npx cypress open`` - This calls cypress directly and will open the Cypress Test Runner.
+  You need to have a running frontend server.
+
+Calling Cypress directly has the advantage that you can use any of the available
+`flags <https://docs.cypress.io/guides/guides/command-line.html#cypress-run>`__
+to customize your test run and you don't need to start a frontend server each time.
+
+Using one of the ``open`` commands, will open a cypress application where you
+can see all the test files you have and run each individually.
+This is going to be run in watch mode, so if you make any changes to test files,
+it will retrigger the test run.
+This cannot be used inside docker, as it requires X11 environment to be able to open.
+
+By default Cypress will look for the web page at ``https://localhost:4200/``.
+If you are serving it in a different URL you will need to configure it by
+exporting the environment variable CYPRESS_BASE_URL with the new value.
+E.g.: ``CYPRESS_BASE_URL=https://localhost:41076/ npx cypress open``
+
+CYPRESS_CACHE_FOLDER
+.....................
+
+When installing cypress via npm, a binary of the cypress app will also be
+downloaded and stored in a cache folder.
+This removes the need to download it every time you run ``npm ci`` or even when
+using cypress in a separate project.
+
+By default Cypress uses ~/.cache to store the binary.
+To prevent changes to the user home directory, we have changed this folder to
+``/ceph/build/src/pybind/mgr/dashboard/cypress``, so when you build ceph or run
+``run-frontend-e2e-tests.sh`` this is the directory Cypress will use.
+
+When using any other command to install or run cypress,
+it will go back to the default directory. It is recommended that you export the
+CYPRESS_CACHE_FOLDER environment variable with a fixed directory, so you always
+use the same directory no matter which command you use.
+
+
+Writing End-to-End Tests
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The PagerHelper class
+.....................
+
+The ``PageHelper`` class is supposed to be used for general purpose code that
+can be used on various pages or suites.
+
+Examples are
+
+- ``navigateTo()`` - Navigates to a specific page and waits for it to load
+- ``getFirstTableCell()`` - returns the first table cell. You can also pass a
+  string with the desired content and it will return the first cell that
+  contains it.
+- ``getTabsCount()`` - returns the amount of tabs
+
+Every method that could be useful on several pages belongs there. Also, methods
+which enhance the derived classes of the PageHelper belong there. A good
+example for such a case is the ``restrictTo()`` decorator. It ensures that a
+method implemented in a subclass of PageHelper is called on the correct page.
+It will also show a developer-friendly warning if this is not the case.
+
+Subclasses of PageHelper
+........................
+
+Helper Methods
+""""""""""""""
+
+In order to make code reusable which is specific for a particular suite, make
+sure to put it in a derived class of the ``PageHelper``. For instance, when
+talking about the pool suite, such methods would be ``create()``, ``exist()``
+and ``delete()``. These methods are specific to a pool but are useful for other
+suites.
+
+Methods that return HTML elements which can only be found on a specific page,
+should be either implemented in the helper methods of the subclass of PageHelper
+or as own methods of the subclass of PageHelper.
+
+Using PageHelpers
+"""""""""""""""""
+
+In any suite, an instance of the specific ``Helper`` class should be
+instantiated and called directly.
+
+.. code:: TypeScript
+
+  const pools = new PoolPageHelper();
+
+  it('should create a pool', () => {
+    pools.exist(poolName, false);
+    pools.navigateTo('create');
+    pools.create(poolName, 8);
+    pools.exist(poolName, true);
+  });
+
+Code Style
+..........
+
+Please refer to the official `Cypress Core Concepts
+<https://docs.cypress.io/guides/core-concepts/introduction-to-cypress.html#Cypress-Can-Be-Simple-Sometimes>`__
+for a better insight on how to write and structure tests.
+
+``describe()`` vs ``it()``
+""""""""""""""""""""""""""
+
+Both ``describe()`` and ``it()`` are function blocks, meaning that any
+executable code necessary for the test can be contained in either block.
+However, Typescript scoping rules still apply, therefore any variables declared
+in a ``describe`` are available to the ``it()`` blocks inside of it.
+
+``describe()`` typically are containers for tests, allowing you to break tests
+into multiple parts. Likewise, any setup that must be made before your tests are
+run can be initialized within the ``describe()`` block. Here is an example:
+
+.. code:: TypeScript
+
+  describe('create, edit & delete image test', () => {
+    const poolName = 'e2e_images_pool';
+
+    before(() => {
+      cy.login();
+      pools.navigateTo('create');
+      pools.create(poolName, 8, 'rbd');
+      pools.exist(poolName, true);
+    });
+
+    beforeEach(() => {
+      cy.login();
+      images.navigateTo();
+    });
+
+    //...
+
+  });
+
+As shown, we can initiate the variable ``poolName`` as well as run commands
+before our test suite begins (creating a pool). ``describe()`` block messages
+should include what the test suite is.
+
+``it()`` blocks typically are parts of an overarching test. They contain the
+functionality of the test suite, each performing individual roles.
+Here is an example:
+
+.. code:: TypeScript
+
+  describe('create, edit & delete image test', () => {
+    //...
+
+    it('should create image', () => {
+      images.createImage(imageName, poolName, '1');
+      images.getFirstTableCell(imageName).should('exist');
+    });
+
+    it('should edit image', () => {
+      images.editImage(imageName, poolName, newImageName, '2');
+      images.getFirstTableCell(newImageName).should('exist');
+    });
+
+    //...
+  });
+
+As shown from the previous example, our ``describe()`` test suite is to create,
+edit and delete an image. Therefore, each ``it()`` completes one of these steps,
+one for creating, one for editing, and so on. Likewise, every ``it()`` blocks
+message should be in lowercase and written so long as "it" can be the prefix of
+the message. For example, ``it('edits the test image' () => ...)`` vs.
+``it('image edit test' () => ...)``. As shown, the first example makes
+grammatical sense with ``it()`` as the prefix whereas the second message does
+not. ``it()`` should describe what the individual test is doing and what it
+expects to happen.
+
+
+Visual Regression Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For visual regression testing, we use `Applitools Eyes <https://applitools.com/products-eyes/>`_
+an AI powered automated  visual regression testing tool.
+Applitools integrates with our existing Cypress E2E tests.
+The tests currently are located at: ``ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/visualTests`` and
+follow the naming convention: ``<component-name>.vrt-spec.ts``.
+
+Running Visual Regression Tests Locally
+.......................................
+
+To run the tests locally, you'll need an Applitools API key, if you don't have one, you can sign up
+for a free account. After obtaining the API key, export it as an environment variable: ``APPLITOOLS_API_KEY``.
+
+Now you can run the tests like normal cypress E2E tests, using either ``npx cypress open`` or in headless mode by running ``npx cypress run``.
+
+Capturing Screenshots
+.....................
+
+Baseline screenshots are the screenshots against which checkpoint screenshots
+(or the screenshots from your feature branch) will be tested.
+
+To capture baseline screenshots, you can run the tests against the master branch,
+and then switch to your feature branch and run the tests again to capture checkpoint screenshots.
+
+Now to see your screenshots, login to applitools.com and on the landing page you'll be greeted with
+applitools eyes test runner, where you can see all your screenshots. And if there's any visual regression or difference (diff) between your baseline and checkpoint screenshots, they'll be highlighted with a mask over the diff.
+
+Writing More Visual Regression Tests
+....................................
+
+Please refer to `Applitools's official cypress sdk documentation <https://www.npmjs.com/package/@applitools/eyes-cypress#usage>`_ to write more tests.
+
+Visual Regression Tests In Jenkins
+..................................
+
+Currently, all visual regression tests are being run under `ceph dashboard tests <https://jenkins.ceph.com/job/ceph-dashboard-pull-requests>`_ GitHub check in the Jenkins job.
+
+Accepting or Rejecting Differences
+..................................
+
+Currently, only the ceph dashboard team has read and write access to the applitools test runner. If any differences are reported by the tests, and you want to accept them and update the baseline screenshots, or if the differences are due to a genuine regression you can fail them. To perform the above actions, please follow `this <https://applitools.com/docs/topics/test-manager/pages/page-test-results/tm-accepting-and-rejecting-steps.html>`_ guide.
+
+Debugging Regressions
+.....................
+
+If you're running the tests locally and regressions are reported, you can take advantage of `Applitools's Root Cause Analysis feature <https://applitools.com/docs/topics/test-manager/viewers/root-cause-analysis.html>`_ to find the cause of the regression.
+
+
+Differences between Frontend Unit Tests and End-to-End (E2E) Tests / FAQ
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+General introduction about testing and E2E/unit tests
+
+
+What are E2E/unit tests designed for?
+.....................................
+
+E2E test:
+
+It requires a fully functional system and tests the interaction of all components
+of the application (Ceph, back-end, front-end).
+E2E tests are designed to mimic the behavior of the user when interacting with the application
+- for example when it comes to workflows like creating/editing/deleting an item.
+Also the tests should verify that certain items are displayed as a user would see them
+when clicking through the UI (for example a menu entry or a pool that has been
+created during a test and the pool and its properties should be displayed in the table).
+
+Angular Unit Tests:
+
+Unit tests, as the name suggests, are tests for smaller units of the code.
+Those tests are designed for testing all kinds of Angular components (e.g. services, pipes etc.).
+They do not require a connection to the backend, hence those tests are independent of it.
+The expected data of the backend is mocked in the frontend and by using this data
+the functionality of the frontend can be tested without having to have real data from the backend.
+As previously mentioned, data is either mocked or, in a simple case, contains a static input,
+a function call and an expected static output.
+More complex examples include the state of a component (attributes of the component class),
+that define how the output changes according to the given input.
+
+Which E2E/unit tests are considered to be valid?
+................................................
+
+This is not easy to answer, but new tests that are written in the same way as already existing
+dashboard tests should generally be considered valid.
+Unit tests should focus on the component to be tested.
+This is either an Angular component, directive, service, pipe, etc.
+
+E2E tests should focus on testing the functionality of the whole application.
+Approximately a third of the overall E2E tests should verify the correctness
+of user visible elements.
+
+How should an E2E/unit test look like?
+......................................
+
+Unit tests should focus on the described purpose
+and shouldn't try to test other things in the same `it` block.
+
+E2E tests should contain a description that either verifies
+the correctness of a user visible element or a complete process
+like for example the creation/validation/deletion of a pool.
+
+What should an E2E/unit test cover?
+...................................
+
+E2E tests should mostly, but not exclusively, cover interaction with the backend.
+This way the interaction with the backend is utilized to write integration tests.
+
+A unit test should mostly cover critical or complex functionality
+of a component (Angular Components, Services, Pipes, Directives, etc).
+
+What should an E2E/unit test NOT cover?
+.......................................
+
+Avoid duplicate testing: do not write E2E tests for what's already
+been covered as frontend-unit tests and vice versa.
+It may not be possible to completely avoid an overlap.
+
+Unit tests should not be used to extensively click through components and E2E tests
+shouldn't be used to extensively test a single component of Angular.
+
+Best practices/guideline
+........................
+
+As a general guideline we try to follow the 70/20/10 approach - 70% unit tests,
+20% integration tests and 10% end-to-end tests.
+For further information please refer to `this document
+<https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html>`__
+and the included "Testing Pyramid".
+
+Further Help
+~~~~~~~~~~~~
+
+To get more help on the Angular CLI use ``ng help`` or go check out the
+`Angular CLI
+README <https://github.com/angular/angular-cli/blob/master/README.md>`__.
+
+Example of a Generator
+~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # Create module 'Core'
+    src/app> ng generate module core -m=app --routing
+
+    # Create module 'Auth' under module 'Core'
+    src/app/core> ng generate module auth -m=core --routing
+    or, alternatively:
+    src/app> ng generate module core/auth -m=core --routing
+
+    # Create component 'Login' under module 'Auth'
+    src/app/core/auth> ng generate component login -m=core/auth
+    or, alternatively:
+    src/app> ng generate component core/auth/login -m=core/auth
+
+Frontend Typescript Code Style Guide Recommendations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Group the imports based on its source and separate them with a blank
+line.
+
+The source groups can be either from Angular, external or internal.
+
+Example:
+
+.. code:: javascript
+
+    import { Component } from '@angular/core';
+    import { Router } from '@angular/router';
+
+    import { ToastrManager } from 'ngx-toastr';
+
+    import { Credentials } from '../../../shared/models/credentials.model';
+    import { HostService } from './services/host.service';
+
+Frontend components
+~~~~~~~~~~~~~~~~~~~
+
+There are several components that can be reused on different pages.
+This components are declared on the components module:
+`src/pybind/mgr/dashboard/frontend/src/app/shared/components`.
+
+Helper
+~~~~~~
+
+This component should be used to provide additional information to the user.
+
+Example:
+
+.. code:: html
+
+    <cd-helper>
+      Some <strong>helper</strong> html text
+    </cd-helper>
+
+Terminology and wording
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Instead of using the Ceph component names, the approach
+suggested is to use the logical/generic names (Block over RBD, Filesystem over
+CephFS, Object over RGW). Nevertheless, as Ceph-Dashboard cannot completely hide
+the Ceph internals, some Ceph-specific names might remain visible.
+
+Regarding the wording for action labels and other textual elements (form titles,
+buttons, etc.), the chosen approach is to follow `these guidelines
+<https://www.patternfly.org/styles/terminology-and-wording/#terminology-and-wording-for-action-labels>`_.
+As a rule of thumb, 'Create' and 'Delete' are the proper wording for most forms,
+instead of 'Add' and 'Remove', unless some already created item is either added
+or removed to/from a set of items (e.g.: 'Add permission' to a user vs. 'Create
+(new) permission').
+
+In order to enforce the use of this wording, a service ``ActionLabelsI18n`` has
+been created, which provides translated labels for use in UI elements.
+
+Frontend branding
+~~~~~~~~~~~~~~~~~
+
+Every vendor can customize the 'Ceph dashboard' to his needs. No matter if
+logo, HTML-Template or TypeScript, every file inside the frontend folder can be
+replaced.
+
+To replace files, open ``./frontend/angular.json`` and scroll to the section
+``fileReplacements`` inside the production configuration. Here you can add the
+files you wish to brand. We recommend to place the branded version of a file in
+the same directory as the original one and to add a ``.brand`` to the file
+name, right in front of the file extension. A ``fileReplacement`` could for
+example look like this:
+
+.. code:: javascript
+
+    {
+      "replace": "src/app/core/auth/login/login.component.html",
+      "with": "src/app/core/auth/login/login.component.brand.html"
+    }
+
+To serve or build the branded user interface run:
+
+    $ npm run start -- --prod
+
+or
+
+    $ npm run build -- --prod
+
+Unfortunately it's currently not possible to use multiple configurations when
+serving or building the UI at the same time. That means a configuration just
+for the branding ``fileReplacements`` is not an option, because you want to use
+the production configuration anyway
+(https://github.com/angular/angular-cli/issues/10612).
+Furthermore it's also not possible to use glob expressions for
+``fileReplacements``. As long as the feature hasn't been implemented, you have
+to add the file replacements manually to the angular.json file
+(https://github.com/angular/angular-cli/issues/12354).
+
+Nevertheless you should stick to the suggested naming scheme because it makes
+it easier for you to use glob expressions once it's supported in the future.
+
+To change the variable defaults or add your own ones you can overwrite them in
+``./frontend/src/styles/vendor/_variables.scss``.
+Just reassign the variable you want to change, for example ``$color-primary: teal;``
+To overwrite or extend the default CSS, you can add your own styles in
+``./frontend/src/styles/vendor/_style-overrides.scss``.
+
+UI Style Guide
+~~~~~~~~~~~~~~
+
+The style guide is created to document Ceph Dashboard standards and maintain
+consistency across the project. Its an effort to make it easier for
+contributors to process designing and deciding mockups and designs for
+Dashboard.
+
+The development environment for Ceph Dashboard has live reloading enabled so
+any changes made in UI are reflected in open browser windows. Ceph Dashboard
+uses Bootstrap as the main third-party CSS library.
+
+Avoid duplication of code. Be consistent with the existing UI by reusing
+existing SCSS declarations as much as possible.
+
+Always check for existing code similar to what you want to write.
+You should always try to keep the same look-and-feel as the existing code.
+
+Colors
+......
+
+All the colors used in Ceph Dashboard UI are listed in
+`frontend/src/styles/defaults/_bootstrap-defaults.scss`. If using new color
+always define color variables in the `_bootstrap-defaults.scss` and
+use the variable instead of hard coded color values so that changes to the
+color are reflected in similar UI elements.
+
+The main color for the Ceph Dashboard is `$primary`. The primary color is
+used in navigation components and as the `$border-color` for input components of
+form.
+
+The secondary color is `$secondary` and is the background color for Ceph
+Dashboard.
+
+Buttons
+.......
+
+Buttons are used for performing actions such as: “Submit�, “Edit, “Create" and
+“Update�.
+
+**Forms:** When using to submit forms anywhere in the Dashboard, the main action
+button should use the `cd-submit-button` component and the secondary button should
+use `cd-back-button` component. The text on the action button should be same as the
+form title and follow a title case. The text on the secondary button should be
+`Cancel`. `Perform action` button should always be on right while `Cancel`
+button should always be on left.
+
+**Modals**: The main action button should use the `cd-submit-button` component and
+the secondary button should use `cd-back-button` component. The text on the action
+button should follow a title case and correspond to the action to be performed.
+The text on the secondary button should be `Close`.
+
+**Disclosure Button:** Disclosure buttons should be used to allow users to
+display and hide additional content in the interface.
+
+**Action Button**: Use the action button to perform actions such as edit or update
+a component. All action button should have an icon corresponding to the actions they
+perform and button text should follow title case. The button color should be the
+same as the form's main button color.
+
+**Drop Down Buttons:** Use dropdown buttons to display predefined lists of
+actions. All drop down buttons have icons corresponding to the action they
+perform.
+
+Links
+.....
+
+Use text hyperlinks as navigation to guide users to a new page in the application
+or to anchor users to a section within a page. The color of the hyperlinks
+should be `$primary`.
+
+Forms
+.....
+
+Mark invalid form fields with red outline and show a meaningful error message.
+Use red as font color for message and be as specific as possible.
+`This field is required.` should be the exact error message for required fields.
+Mark valid forms with a green outline and a green tick at the end of the form.
+Sections should not have a bigger header than the parent.
+
+Modals
+......
+
+Blur any interface elements in the background to bring the modal content into
+focus. The heading of the modal should reflect the action it can perform and
+should be clearly mentioned at the top of the modal. Use `cd-back-button`
+component in the footer for closing the modal.
+
+Icons
+.....
+
+We use `Fork Awesome <https://forkaweso.me/Fork-Awesome/>`_ classes for icons.
+We have a list of used icons in `src/app/shared/enum/icons.enum.ts`, these
+should be referenced in the HTML, so its easier to change them later. When
+icons are next to text, they should be center-aligned horizontally. If icons
+are stacked, they should also be center-aligned vertically. Use small icons
+with buttons. For notifications use large icons.
+
+Navigation
+..........
+
+For local navigation use tabs. For overall navigation use expandable vertical
+navigation to collapse and expand items as needed.
+
+Alerts and notifications
+........................
+
+Default notification should have `text-info` color. Success notification should
+have `text-success` color. Failure notification should have `text-danger` color.
+
+Error Handling
+~~~~~~~~~~~~~~
+
+For handling front-end errors, there is a generic Error Component which can be
+found in ``./src/pybind/mgr/dashboard/frontend/src/app/core/error``. For
+reporting a new error, you can simply extend the ``DashboardError`` class
+in ``error.ts`` file and add specific header and message for the new error. Some
+generic error classes are already in place such as ``DashboardNotFoundError``
+and ``DashboardForbiddenError`` which can be called and reused in different
+scenarios.
+
+For example - ``throw new DashboardNotFoundError()``.
+
+Internationalization (i18n)
+---------------------------
+
+How to extract messages from source code?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To extract the I18N messages from the templates and the TypeScript files just
+run the following command in ``src/pybind/mgr/dashboard/frontend``::
+
+  $ npm run i18n:extract
+
+This will extract all marked messages from the HTML templates first and then
+add all marked strings from the TypeScript files to the translation template.
+Since the extraction from TypeScript files is still not supported by Angular
+itself, we are using the
+`ngx-translator <https://github.com/ngx-translate/i18n-polyfill>`_ extractor to
+parse the TypeScript files.
+
+When the command ran successfully, it should have created or updated the file
+``src/locale/messages.xlf``.
+
+The file isn't tracked by git, you can just use it to start with the
+translation offline or add/update the resource files on transifex.
+
+Supported languages
+~~~~~~~~~~~~~~~~~~~
+
+All our supported languages should be registered in both exports in
+``supported-languages.enum.ts`` and have a corresponding test in
+``language-selector.component.spec.ts``.
+
+The ``SupportedLanguages`` enum will provide the list for the default language selection.
+
+Translating process
+~~~~~~~~~~~~~~~~~~~
+
+To facilitate the translation process of the dashboard we are using a web tool
+called `transifex <https://www.transifex.com/>`_.
+
+If you wish to help translating to any language just go to our `transifex
+project page <https://www.transifex.com/ceph/ceph-dashboard/>`_, join the
+project and you can start translating immediately.
+
+All translations will then be reviewed and later pushed upstream.
+
+Updating translated messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any time there are new messages translated and reviewed in a specific language
+we should update the translation file upstream.
+
+To do that, check the settings in the i18n config file
+``src/pybind/mgr/dashboard/frontend/i18n.config.json``:: and make sure that the
+organization is *ceph*, the project is *ceph-dashboard* and the resource is
+the one you want to pull from and push to e.g. *Master:master*. To find a list
+of available resources visit `<https://www.transifex.com/ceph/ceph-dashboard/content/>`_.
+
+After you checked the config go to the directory ``src/pybind/mgr/dashboard/frontend`` and run::
+
+  $ npm run i18n
+
+This command will extract all marked messages from the HTML templates and
+TypeScript files. Once the source file has been created it will push it to
+transifex and pull the latest translations. It will also fill all the
+untranslated strings with the source string.
+The tool will ask you for an api token, unless you added it by running:
+
+  $ npm run i18n:token
+
+To create a transifex api token visit `<https://www.transifex.com/user/settings/api/>`_.
+
+After the command ran successfully, build the UI and check if everything is
+working as expected. You also might want to run the frontend tests.
+
+Add a new release resource to transifex
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to organize the translations, we create a
+`transifex resource <https://www.transifex.com/ceph/ceph-dashboard/content/>`_
+for every Ceph release. This means, once a new version has been released, the
+``src/pybind/mgr/dashboard/frontend/i18n.config.json`` needs to be updated on
+the release branch.
+
+Please replace::
+
+"resource": "Master:master"
+
+by::
+
+"resource": "<Release-name>:<release-name>"
+
+E.g. the resource definition for the pacific release::
+
+"resource": "Pacific:pacific"
+
+Note:
+  The first part of the resource definition (before the colon) needs to be
+  written with a capital letter.
+
+Suggestions
+~~~~~~~~~~~
+
+Strings need to start and end in the same line as the element:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <span i18n>
+    Foo
+  </span>
+
+  <!-- recommended -->
+  <span i18n>Foo</span>
+
+
+  <!-- avoid -->
+  <span i18n>
+    Foo bar baz.
+    Foo bar baz.
+  </span>
+
+  <!-- recommended -->
+  <span i18n>Foo bar baz.
+    Foo bar baz.</span>
+
+Isolated interpolations should not be translated:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <span i18n>{{ foo }}</span>
+
+  <!-- recommended -->
+  <span>{{ foo }}</span>
+
+Interpolations used in a sentence should be kept in the translation:
+
+.. code-block:: html
+
+  <!-- recommended -->
+  <span i18n>There are {{ x }} OSDs.</span>
+
+Remove elements that are outside the context of the translation:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <label i18n>
+    Profile
+    <span class="required"></span>
+  </label>
+
+  <!-- recommended -->
+  <label>
+    <ng-container i18n>Profile<ng-container>
+    <span class="required"></span>
+  </label>
+
+Keep elements that affect the sentence:
+
+.. code-block:: html
+
+  <!-- recommended -->
+  <span i18n>Profile <b>foo</b> will be removed.</span>
+
+
+.. _accessibility:
+
+Accessibility
+-------------
+
+Many parts of the Ceph Dashboard are modeled on `Web Content Accessibility Guidelines (WCAG) 2.1 <https://www.w3.org/TR/WCAG21/>`_  level A accessibility conformance guidelines. 
+By implementing accessibility best practices, you are improving the usability of the Ceph Dashboard for blind and visually impaired users.
+
+Summary
+~~~~~~~
+
+A few things you should check before introducing a new code change include:
+
+1) Add `ARIA labels and descriptions <https://www.w3.org/TR/wai-aria/>`_ to actionable HTML elements.
+2) Don't forget to tag ARIA labels/descriptions or any user-readable text for translation (i18n-title, i18n-aria-label...).
+3) Add `ARIA roles <https://www.w3.org/TR/wai-aria/#usage_intro>`_ to tag HTML elements that behave different from their intended behaviour (<a> tags behaving as <buttons>) or that provide extended behaviours (roles).
+4) Avoid poor `color contrast choices <https://www.w3.org/TR/WCAG21/#contrast-minimum>`_ (foreground-background) when styling a component. Here are some :ref:`tools <color-contrast-checkers>` you can use.
+5) When testing menus or dropdowns, be sure to scan them with an :ref:`accessibility checker <accessibility-checkers>` in both opened and closed states. Sometimes issues are hidden when menus are closed.
+
+.. _accessibility-checkers:
+
+Accessibility checkers
+~~~~~~~~~~~~~~~~~~~~~~
+
+During development, you can test the accessibility compliance of your features using one of the tools below:
+
+- `Accessibility insights plugin <https://accessibilityinsights.io/downloads/>`_
+- `Site Improve plugin <https://www.siteimprove.com/integrations/browser-extensions/>`_
+- `Axe devtools <https://www.deque.com/axe/devtools/>`_
+
+Testing with two or more of these tools can greatly improve the detection of accessibility violations.
+
+.. _color-contrast-checkers:
+
+Color contrast checkers
+~~~~~~~~~~~~~~~~~~~~~~~
+
+When adding new colors, making sure they are accessible is also important. Here are some tools which can help with color contrast testing:
+
+- `Accessible web color-contrast checker <https://accessibleweb.com/color-contrast-checker/>`_
+- `Colorsafe generator <https://colorsafe.co/>`_
+
+Accessibility linters
+~~~~~~~~~~~~~~~~~~~~~
+
+If you use VSCode, you may install the `axe accessibility linter <https://marketplace.visualstudio.com/items?itemName=deque-systems.vscode-axe-linter>`_,
+which can help you catch and fix potential issues during development.
+
+Accessibility testing
+~~~~~~~~~~~~~~~~~~~~~
+
+Our e2e testing suite, which is based on Cypress, supports the addition of accessibility tests using `axe-core <https://github.com/dequelabs/axe-core>`_ 
+and `cypress-axe <https://github.com/component-driven/cypress-axe>`_. A custom Cypress command, `cy.checkAccessibility`, can also be used directly. 
+This is a great way to prevent accessibility regressions on high impact components.
+
+Tests can be found under the `a11y folder <./src/pybind/mgr/dashboard/frontend/cypress/integration/a11y>`_ in the dashboard. Here is an example:
+
+.. code:: TypeScript
+
+  describe('Navigation accessibility', { retries: 0 }, () => {
+    const shared = new NavigationPageHelper();
+  
+    beforeEach(() => {
+      cy.login();
+      shared.navigateTo();
+    });
+  
+    it('top-nav should have no accessibility violations', () => {
+      cy.injectAxe();
+      cy.checkAccessibility('.cd-navbar-top');
+    });
+  
+    it('sidebar should have no accessibility violations', () => {
+      cy.injectAxe();
+      cy.checkAccessibility('nav[id=sidebar]');
+    });
+  
+  });
+
+Additional guidelines
+~~~~~~~~~~~~~~~~~~~~~
+
+If you're unsure about which UI pattern to follow in order to implement an accessibility fix, `patternfly <https://www.patternfly.org/v4/accessibility/accessibility-fundamentals>`_ guidelines can be used.
+
+Backend Development
+-------------------
+
+The Python backend code of this module requires a number of Python modules to be
+installed. They are listed in file ``requirements.txt``. Using `pip
+<https://pypi.python.org/pypi/pip>`_ you may install all required dependencies
+by issuing ``pip install -r requirements.txt`` in directory
+``src/pybind/mgr/dashboard``.
+
+If you're using the `ceph-dev-docker development environment
+<https://github.com/ricardoasmarques/ceph-dev-docker/>`_, simply run
+``./install_deps.sh`` from the toplevel directory to install them.
+
+Unit Testing
+~~~~~~~~~~~~
+
+In dashboard we have two different kinds of backend tests:
+
+1. Unit tests based on ``tox``
+2. API tests based on Teuthology.
+
+Unit tests based on tox
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+We included a ``tox`` configuration file that will run the unit tests under
+Python 3, as well as linting tools to guarantee the uniformity of code.
+
+You need to install ``tox`` and ``coverage`` before running it. To install the
+packages in your system, either install it via your operating system's package
+management tools, e.g. by running ``dnf install python-tox python-coverage`` on
+Fedora Linux.
+
+Alternatively, you can use Python's native package installation method::
+
+  $ pip install tox
+  $ pip install coverage
+
+To run the tests, run ``src/script/run_tox.sh`` in the dashboard directory (where
+``tox.ini`` is located)::
+
+  ## Run Python 3 tests+lint commands:
+  $ ../../../script/run_tox.sh --tox-env py3,lint,check
+
+  ## Run Python 3 arbitrary command (e.g. 1 single test):
+  $ ../../../script/run_tox.sh --tox-env py3 "" tests/test_rgw_client.py::RgwClientTest::test_ssl_verify
+
+You can also run tox instead of ``run_tox.sh``::
+
+  ## Run Python 3 tests command:
+  $ tox -e py3
+
+  ## Run Python 3 arbitrary command (e.g. 1 single test):
+  $ tox -e py3 tests/test_rgw_client.py::RgwClientTest::test_ssl_verify
+
+Python files can be automatically fixed and formatted according to PEP8
+standards by using ``run_tox.sh --tox-env fix`` or ``tox -e fix``.
+
+We also collect coverage information from the backend code when you run tests. You can check the
+coverage information provided by the tox output, or by running the following
+command after tox has finished successfully::
+
+  $ coverage html
+
+This command will create a directory ``htmlcov`` with an HTML representation of
+the code coverage of the backend.
+
+API tests based on Teuthology
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+How to run existing API tests:
+  To run the API tests against a real Ceph cluster, we leverage the Teuthology
+  framework. This has the advantage of catching bugs originated from changes in
+  the internal Ceph code.
+
+  Our ``run-backend-api-tests.sh`` script will start a ``vstart`` Ceph cluster
+  before running the Teuthology tests, and then it stops the cluster after the
+  tests are run. Of course this implies that you have built/compiled Ceph
+  previously.
+
+  Start all dashboard tests by running::
+
+    $ ./run-backend-api-tests.sh
+
+  Or, start one or multiple specific tests by specifying the test name::
+
+    $ ./run-backend-api-tests.sh tasks.mgr.dashboard.test_pool.PoolTest
+
+  Or, ``source`` the script and run the tests manually::
+
+    $ source run-backend-api-tests.sh
+    $ run_teuthology_tests [tests]...
+    $ cleanup_teuthology
+
+How to write your own tests:
+  There are two possible ways to write your own API tests:
+
+  The first is by extending one of the existing test classes in the
+  ``qa/tasks/mgr/dashboard`` directory.
+
+  The second way is by adding your own API test module if you're creating a new
+  controller for example. To do so you'll just need to add the file containing
+  your new test class to the ``qa/tasks/mgr/dashboard`` directory and implement
+  all your tests here.
+
+  .. note:: Don't forget to add the path of the newly created module to
+    ``modules`` section in ``qa/suites/rados/mgr/tasks/dashboard.yaml``.
+
+  Short example: Let's assume you created a new controller called
+  ``my_new_controller.py`` and the related test module
+  ``test_my_new_controller.py``. You'll need to add
+  ``tasks.mgr.dashboard.test_my_new_controller`` to the ``modules`` section in
+  the ``dashboard.yaml`` file.
+
+  Also, if you're removing test modules please keep in mind to remove the
+  related section. Otherwise the Teuthology test run will fail.
+
+  Please run your API tests on your dev environment (as explained above)
+  before submitting a pull request. Also make sure that a full QA run in
+  Teuthology/sepia lab (based on your changes) has completed successfully
+  before it gets merged. You don't need to schedule the QA run yourself, just
+  add the 'needs-qa' label to your pull request as soon as you think it's ready
+  for merging (e.g. make check was successful, the pull request is approved and
+  all comments have been addressed). One of the developers who has access to
+  Teuthology/the sepia lab will take care of it and report the result back to
+  you.
+
+
+How to add a new controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A controller is a Python class that extends from the ``BaseController`` class
+and is decorated with either the ``@Controller``, ``@ApiController`` or
+``@UiApiController`` decorators. The Python class must be stored inside a Python
+file located under the ``controllers`` directory. The Dashboard module will
+automatically load your new controller upon start.
+
+``@ApiController`` and ``@UiApiController`` are both specializations of the
+``@Controller`` decorator.
+
+The ``@ApiController`` should be used for controllers that provide an API-like
+REST interface and the ``@UiApiController`` should be used for endpoints consumed
+by the UI but that are not part of the 'public' API. For any other kinds of
+controllers the ``@Controller`` decorator should be used.
+
+A controller has a URL prefix path associated that is specified in the
+controller decorator, and all endpoints exposed by the controller will share
+the same URL prefix path.
+
+A controller's endpoint is exposed by implementing a method on the controller
+class decorated with the ``@Endpoint`` decorator.
+
+For example create a file ``ping.py`` under ``controllers`` directory with the
+following code:
+
+.. code-block:: python
+
+  from ..tools import Controller, ApiController, UiApiController, BaseController, Endpoint
+
+  @Controller('/ping')
+  class Ping(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+  @ApiController('/ping')
+  class ApiPing(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+  @UiApiController('/ping')
+  class UiApiPing(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+The ``hello`` endpoint of the ``Ping`` controller can be reached by the
+following URL: https://mgr_hostname:8443/ping/hello using HTTP GET requests.
+As you can see the controller URL path ``/ping`` is concatenated to the
+method name ``hello`` to generate the endpoint's URL.
+
+In the case of the ``ApiPing`` controller, the ``hello`` endpoint can be
+reached by the following URL: https://mgr_hostname:8443/api/ping/hello using a
+HTTP GET request.
+The API controller URL path ``/ping`` is prefixed by the ``/api`` path and then
+concatenated to the method name ``hello`` to generate the endpoint's URL.
+Internally, the ``@ApiController`` is actually calling the ``@Controller``
+decorator by passing an additional decorator parameter called ``base_url``::
+
+  @ApiController('/ping') <=> @Controller('/ping', base_url="/api")
+
+``UiApiPing`` works in a similar way than the ``ApiPing``, but the URL will be
+prefixed by ``/ui-api``: https://mgr_hostname:8443/ui-api/ping/hello. ``UiApiPing`` is
+also a ``@Controller`` extension::
+
+  @UiApiController('/ping') <=> @Controller('/ping', base_url="/ui-api")
+
+The ``@Endpoint`` decorator also supports many parameters to customize the
+endpoint:
+
+* ``method="GET"``: the HTTP method allowed to access this endpoint.
+* ``path="/<method_name>"``: the URL path of the endpoint, excluding the
+  controller URL path prefix.
+* ``path_params=[]``: list of method parameter names that correspond to URL
+  path parameters. Can only be used when ``method in ['POST', 'PUT']``.
+* ``query_params=[]``: list of method parameter names that correspond to URL
+  query parameters.
+* ``json_response=True``: indicates if the endpoint response should be
+  serialized in JSON format.
+* ``proxy=False``: indicates if the endpoint should be used as a proxy.
+
+An endpoint method may have parameters declared. Depending on the HTTP method
+defined for the endpoint the method parameters might be considered either
+path parameters, query parameters, or body parameters.
+
+For ``GET`` and ``DELETE`` methods, the method's non-optional parameters are
+considered path parameters by default. Optional parameters are considered
+query parameters. By specifying the ``query_parameters`` in the endpoint
+decorator it is possible to make a non-optional parameter to be a query
+parameter.
+
+For ``POST`` and ``PUT`` methods, all method parameters are considered
+body parameters by default. To override this default, one can use the
+``path_params`` and ``query_params`` to specify which method parameters are
+path and query parameters respectively.
+Body parameters are decoded from the request body, either from a form format, or
+from a dictionary in JSON format.
+
+Let's use an example to better understand the possible ways to customize an
+endpoint:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Endpoint
+
+  @Controller('/ping')
+  class Ping(BaseController):
+
+    # URL: /ping/{key}?opt1=...&opt2=...
+    @Endpoint(path="/", query_params=['opt1'])
+    def index(self, key, opt1, opt2=None):
+      """..."""
+
+    # URL: /ping/{key}?opt1=...&opt2=...
+    @Endpoint(query_params=['opt1'])
+    def __call__(self, key, opt1, opt2=None):
+      """..."""
+
+    # URL: /ping/post/{key1}/{key2}
+    @Endpoint('POST', path_params=['key1', 'key2'])
+    def post(self, key1, key2, data1, data2=None):
+      """..."""
+
+
+In the above example we see how the ``path`` option can be used to override the
+generated endpoint URL in order to not use the method's name in the URL. In the
+``index`` method we set the ``path`` to ``"/"`` to generate an endpoint that is
+accessible by the root URL of the controller.
+
+An alternative approach to generate an endpoint that is accessible through just
+the controller's path URL is by using the ``__call__`` method, as we show in
+the above example.
+
+From the third method you can see that the path parameters are collected from
+the URL by parsing the list of values separated by slashes ``/`` that come
+after the URL path ``/ping`` for ``index`` method case, and ``/ping/post`` for
+the ``post`` method case.
+
+Defining path parameters in endpoints's URLs using python methods's parameters
+is very easy but it is still a bit strict with respect to the position of these
+parameters in the URL structure.
+Sometimes we may want to explicitly define a URL scheme that
+contains path parameters mixed with static parts of the URL.
+Our controller infrastructure also supports the declaration of URL paths with
+explicit path parameters at both the controller level and method level.
+
+Consider the following example:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Endpoint
+
+  @Controller('/ping/{node}/stats')
+  class Ping(BaseController):
+
+    # URL: /ping/{node}/stats/{date}/latency?unit=...
+    @Endpoint(path="/{date}/latency")
+    def latency(self, node, date, unit="ms"):
+      """ ..."""
+
+In this example we explicitly declare a path parameter ``{node}`` in the
+controller URL path, and a path parameter ``{date}`` in the ``latency``
+method. The endpoint for the ``latency`` method is then accessible through
+the URL: https://mgr_hostname:8443/ping/{node}/stats/{date}/latency .
+
+For a full set of examples on how to use the ``@Endpoint``
+decorator please check the unit test file: ``tests/test_controllers.py``.
+There you will find many examples of how to customize endpoint methods.
+
+
+Implementing Proxy Controller
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you might need to relay some requests from the Dashboard frontend
+directly to an external service.
+For that purpose we provide a decorator called ``@Proxy``.
+(As a concrete example, check the ``controllers/rgw.py`` file where we
+implemented an RGW Admin Ops proxy.)
+
+
+The ``@Proxy`` decorator is a wrapper of the ``@Endpoint`` decorator that
+already customizes the endpoint for working as a proxy.
+A proxy endpoint works by capturing the URL path that follows the controller
+URL prefix path, and does not do any decoding of the request body.
+
+Example:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Proxy
+
+  @Controller('/foo/proxy')
+  class FooServiceProxy(BaseController):
+
+    @Proxy()
+    def proxy(self, path, **params):
+      """
+      if requested URL is "/foo/proxy/access/service?opt=1"
+      then path is "access/service" and params is {'opt': '1'}
+      """
+
+
+How does the RESTController work?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We also provide a simple mechanism to create REST based controllers using the
+``RESTController`` class. Any class which inherits from ``RESTController`` will,
+by default, return JSON.
+
+The ``RESTController`` is basically an additional abstraction layer which eases
+and unifies the work with collections. A collection is just an array of objects
+with a specific type. ``RESTController`` enables some default mappings of
+request types and given parameters to specific method names. This may sound
+complicated at first, but it's fairly easy. Lets have look at the following
+example:
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+  @ApiController('ping')
+  class Ping(RESTController):
+    def list(self):
+      return {"msg": "Hello"}
+
+    def get(self, id):
+      return self.objects[id]
+
+In this case, the ``list`` method is automatically used for all requests to
+``api/ping`` where no additional argument is given and where the request type
+is ``GET``. If the request is given an additional argument, the ID in our
+case, it won't map to ``list`` anymore but to ``get`` and return the element
+with the given ID (assuming that ``self.objects`` has been filled before). The
+same applies to other request types:
+
++--------------+------------+----------------+-------------+
+| Request type | Arguments  | Method         | Status Code |
++==============+============+================+=============+
+| GET          | No         | list           | 200         |
++--------------+------------+----------------+-------------+
+| PUT          | No         | bulk_set       | 200         |
++--------------+------------+----------------+-------------+
+| POST         | No         | create         | 201         |
++--------------+------------+----------------+-------------+
+| DELETE       | No         | bulk_delete    | 204         |
++--------------+------------+----------------+-------------+
+| GET          | Yes        | get            | 200         |
++--------------+------------+----------------+-------------+
+| PUT          | Yes        | set            | 200         |
++--------------+------------+----------------+-------------+
+| DELETE       | Yes        | delete         | 204         |
++--------------+------------+----------------+-------------+
+
+To use a custom endpoint for the above listed methods, you can
+use ``@RESTController.MethodMap``
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+    @RESTController.MethodMap(version='0.1')
+    def create(self):
+      return {"msg": "Hello"}
+
+This decorator supports three parameters to customize the
+endpoint:
+
+* ``resource"``: resource id.
+* ``status=200``: set the HTTP status response code
+* ``version``: version
+
+How to use a custom API endpoint in a RESTController?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you don't have any access restriction you can use ``@Endpoint``. If you
+have set a permission scope to restrict access to your endpoints,
+``@Endpoint`` will fail, as it doesn't know which permission property should be
+used. To use a custom endpoint inside a restricted ``RESTController`` use
+``@RESTController.Collection`` instead. You can also choose
+``@RESTController.Resource`` if you have set a ``RESOURCE_ID`` in your
+``RESTController`` class.
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+  @ApiController('ping', Scope.Ping)
+  class Ping(RESTController):
+    RESOURCE_ID = 'ping'
+
+    @RESTController.Resource('GET')
+    def some_get_endpoint(self):
+      return {"msg": "Hello"}
+
+    @RESTController.Collection('POST')
+    def some_post_endpoint(self, **data):
+      return {"msg": data}
+
+Both decorators also support five parameters to customize the
+endpoint:
+
+* ``method="GET"``: the HTTP method allowed to access this endpoint.
+* ``path="/<method_name>"``: the URL path of the endpoint, excluding the
+  controller URL path prefix.
+* ``status=200``: set the HTTP status response code
+* ``query_params=[]``: list of method parameter names that correspond to URL
+  query parameters.
+* ``version``: version
+
+How to restrict access to a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All controllers require authentication by default.
+If you require that the controller can be accessed without authentication,
+then you can add the parameter ``secure=False`` to the controller decorator.
+
+Example:
+
+.. code-block:: python
+
+  import cherrypy
+  from . import ApiController, RESTController
+
+
+  @ApiController('ping', secure=False)
+  class Ping(RESTController):
+    def list(self):
+      return {"msg": "Hello"}
+
+How to create a dedicated UI endpoint which uses the 'public' API?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes we want to combine multiple calls into one single call
+to save bandwidth or for other performance reasons.
+In order to achieve that, we first have to create an ``@UiApiController`` which
+is used for endpoints consumed by the UI but that are not part of the
+'public' API. Let the ui class inherit from the REST controller class.
+Now you can use all methods from the api controller.
+
+Example:
+
+.. code-block:: python
+
+  import cherrypy
+  from . import UiApiController, ApiController, RESTController
+
+
+  @ApiController('ping', secure=False)  # /api/ping
+  class Ping(RESTController):
+    def list(self):
+      return self._list()
+
+    def _list(self):  # To not get in conflict with the JSON wrapper
+      return [1,2,3]
+
+
+  @UiApiController('ping', secure=False)  # /ui-api/ping
+  class PingUi(Ping):
+    def list(self):
+      return self._list() + [4, 5, 6]
+
+How to access the manager module instance from a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We provide the manager module instance as a global variable that can be
+imported in any module.
+
+Example:
+
+.. code-block:: python
+
+  import logging
+  import cherrypy
+  from .. import mgr
+  from ..tools import ApiController, RESTController
+
+  logger = logging.getLogger(__name__)
+
+  @ApiController('servers')
+  class Servers(RESTController):
+    def list(self):
+      logger.debug('Listing available servers')
+      return {'servers': mgr.list_servers()}
+
+
+How to write a unit test for a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We provide a test helper class called ``ControllerTestCase`` to easily create
+unit tests for your controller.
+
+If we want to write a unit test for the above ``Ping`` controller, create a
+``test_ping.py`` file under the ``tests`` directory with the following code:
+
+.. code-block:: python
+
+  from .helper import ControllerTestCase
+  from .controllers.ping import Ping
+
+
+  class PingTest(ControllerTestCase):
+      @classmethod
+      def setup_test(cls):
+          cp_config = {'tools.authenticate.on': True}
+          cls.setup_controllers([Ping], cp_config=cp_config)
+
+      def test_ping(self):
+          self._get("/api/ping")
+          self.assertStatus(200)
+          self.assertJsonBody({'msg': 'Hello'})
+
+The ``ControllerTestCase`` class starts by initializing a CherryPy webserver.
+Then it will call the ``setup_test()`` class method where we can explicitly
+load the controllers that we want to test. In the above example we are only
+loading the ``Ping`` controller. We can also provide ``cp_config`` in order to
+update the controller's cherrypy config (e.g. enable authentication as shown in the example).
+
+How to update or create new dashboards in grafana?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We are using ``jsonnet`` and ``grafonnet-lib`` to write code for the grafana dashboards.
+All the dashboards are written inside ``grafana_dashboards.jsonnet`` file in the
+monitoring/grafana/dashboards/jsonnet directory.
+
+We generate the dashboard json files directly from this jsonnet file by running this
+command in the grafana/dashboards directory:
+``jsonnet -m . jsonnet/grafana_dashboards.jsonnet``.
+(For the above command to succeed we need ``jsonnet`` package installed and ``grafonnet-lib``
+directory cloned in our machine. Please refer -
+``https://grafana.github.io/grafonnet-lib/getting-started/`` in case you have some trouble.)
+
+To update an existing grafana dashboard or to create a new one, we need to update
+the ``grafana_dashboards.jsonnet`` file and generate the new/updated json files using the
+above mentioned command. For people who are not familiar with grafonnet or jsonnet implementation
+can follow this doc - ``https://grafana.github.io/grafonnet-lib/``.
+
+Example grafana dashboard in jsonnet format:
+
+To specify the grafana dashboard properties such as title, uid etc we can create a local function -
+
+::
+
+    local dashboardSchema(title, uid, time_from, refresh, schemaVersion, tags,timezone, timepicker)
+
+To add a graph panel we can specify the graph schema in a local function such as -
+
+::
+
+    local graphPanelSchema(title, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource)
+
+and then use these functions inside the dashboard definition like -
+
+::
+
+    {
+        radosgw-sync-overview.json: //json file name to be generated
+
+        dashboardSchema(
+          'RGW Sync Overview', 'rgw-sync-overview', 'now-1h', '15s', .., .., ..
+        )
+
+        .addPanels([
+          graphPanelSchema(
+            'Replication (throughput) from Source Zone', 'Bps', null, .., .., ..)
+        ])
+    }
+
+The valid grafonnet-lib attributes can be found here - ``https://grafana.github.io/grafonnet-lib/api-docs/``.
+
+
+How to listen for manager notifications in a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The manager notifies the modules of several types of cluster events, such
+as cluster logging event, etc...
+
+Each module has a "global" handler function called ``notify`` that the manager
+calls to notify the module. But this handler function must not block or spend
+too much time processing the event notification.
+For this reason we provide a notification queue that controllers can register
+themselves with to receive cluster notifications.
+
+The example below represents a controller that implements a very simple live
+log viewer page:
+
+.. code-block:: python
+
+  import collections
+
+  import cherrypy
+
+  from ..tools import ApiController, BaseController, NotificationQueue
+
+
+  @ApiController('livelog')
+  class LiveLog(BaseController):
+      log_buffer = collections.deque(maxlen=1000)
+
+      def __init__(self):
+          super(LiveLog, self).__init__()
+          NotificationQueue.register(self.log, 'clog')
+
+      def log(self, log_struct):
+          self.log_buffer.appendleft(log_struct)
+
+      @cherrypy.expose
+      def default(self):
+          ret = '<html><meta http-equiv="refresh" content="2" /><body>'
+          for l in self.log_buffer:
+              ret += "{}<br>".format(l)
+          ret += "</body></html>"
+          return ret
+
+As you can see above, the ``NotificationQueue`` class provides a register
+method that receives the function as its first argument, and receives the
+"notification type" as the second argument.
+You can omit the second argument of the ``register`` method, and in that case
+you are registering to listen all notifications of any type.
+
+Here is an list of notification types (these might change in the future) that
+can be used:
+
+* ``clog``: cluster log notifications
+* ``command``: notification when a command issued by ``MgrModule.send_command``
+  completes
+* ``perf_schema_update``: perf counters schema update
+* ``mon_map``: monitor map update
+* ``fs_map``: cephfs map update
+* ``osd_map``: OSD map update
+* ``service_map``: services (RGW, RBD-Mirror, etc.) map update
+* ``mon_status``: monitor status regular update
+* ``health``: health status regular update
+* ``pg_summary``: regular update of PG status information
+
+
+How to write a unit test when a controller accesses a Ceph module?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Consider the following example that implements a controller that retrieves the
+list of RBD images of the ``rbd`` pool:
+
+.. code-block:: python
+
+  import rbd
+  from .. import mgr
+  from ..tools import ApiController, RESTController
+
+
+  @ApiController('rbdimages')
+  class RbdImages(RESTController):
+      def __init__(self):
+          self.ioctx = mgr.rados.open_ioctx('rbd')
+          self.rbd = rbd.RBD()
+
+      def list(self):
+          return [{'name': n} for n in self.rbd.list(self.ioctx)]
+
+In the example above, we want to mock the return value of the ``rbd.list``
+function, so that we can test the JSON response of the controller.
+
+The unit test code will look like the following:
+
+.. code-block:: python
+
+  import mock
+  from .helper import ControllerTestCase
+
+
+  class RbdImagesTest(ControllerTestCase):
+      @mock.patch('rbd.RBD.list')
+      def test_list(self, rbd_list_mock):
+          rbd_list_mock.return_value = ['img1', 'img2']
+          self._get('/api/rbdimages')
+          self.assertJsonBody([{'name': 'img1'}, {'name': 'img2'}])
+
+
+
+How to add a new configuration setting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you need to store some configuration setting for a new feature, we already
+provide an easy mechanism for you to specify/use the new config setting.
+
+For instance, if you want to add a new configuration setting to hold the
+email address of the dashboard admin, just add a setting name as a class
+attribute to the ``Options`` class in the ``settings.py`` file::
+
+  # ...
+  class Options(object):
+    # ...
+
+    ADMIN_EMAIL_ADDRESS = ('admin@admin.com', str)
+
+The value of the class attribute is a pair composed by the default value for that
+setting, and the python type of the value.
+
+By declaring the ``ADMIN_EMAIL_ADDRESS`` class attribute, when you restart the
+dashboard module, you will automatically gain two additional CLI commands to
+get and set that setting::
+
+  $ ceph dashboard get-admin-email-address
+  $ ceph dashboard set-admin-email-address <value>
+
+To access, or modify the config setting value from your Python code, either
+inside a controller or anywhere else, you just need to import the ``Settings``
+class and access it like this:
+
+.. code-block:: python
+
+  from settings import Settings
+
+  # ...
+  tmp_var = Settings.ADMIN_EMAIL_ADDRESS
+
+  # ....
+  Settings.ADMIN_EMAIL_ADDRESS = 'myemail@admin.com'
+
+The settings management implementation will make sure that if you change a
+setting value from the Python code you will see that change when accessing
+that setting from the CLI and vice-versa.
+
+
+How to run a controller read-write operation asynchronously?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some controllers might need to execute operations that alter the state of the
+Ceph cluster. These operations might take some time to execute and to maintain
+a good user experience in the Web UI, we need to run those operations
+asynchronously and return immediately to frontend some information that the
+operations are running in the background.
+
+To help in the development of the above scenario we added the support for
+asynchronous tasks. To trigger the execution of an asynchronous task we must
+use the following class method of the ``TaskManager`` class::
+
+  from ..tools import TaskManager
+  # ...
+  TaskManager.run(name, metadata, func, args, kwargs)
+
+* ``name`` is a string that can be used to group tasks. For instance
+  for RBD image creation tasks we could specify ``"rbd/create"`` as the
+  name, or similarly ``"rbd/remove"`` for RBD image removal tasks.
+
+* ``metadata`` is a dictionary where we can store key-value pairs that
+  characterize the task. For instance, when creating a task for creating
+  RBD images we can specify the metadata argument as
+  ``{'pool_name': "rbd", image_name': "test-img"}``.
+
+* ``func`` is the python function that implements the operation code, which
+  will be executed asynchronously.
+
+* ``args`` and ``kwargs`` are the positional and named arguments that will be
+  passed to ``func`` when the task manager starts its execution.
+
+The ``TaskManager.run`` method triggers the asynchronous execution of function
+``func`` and returns a ``Task`` object.
+The ``Task`` provides the public method ``Task.wait(timeout)``, which can be
+used to wait for the task to complete up to a timeout defined in seconds and
+provided as an argument. If no argument is provided the ``wait`` method
+blocks until the task is finished.
+
+The ``Task.wait`` is very useful for tasks that usually are fast to execute but
+that sometimes may take a long time to run.
+The return value of the ``Task.wait`` method is a pair ``(state, value)``
+where ``state`` is a string with following possible values:
+
+* ``VALUE_DONE = "done"``
+* ``VALUE_EXECUTING = "executing"``
+
+The ``value`` will store the result of the execution of function ``func`` if
+``state == VALUE_DONE``. If ``state == VALUE_EXECUTING`` then
+``value == None``.
+
+The pair ``(name, metadata)`` should unequivocally identify the task being
+run, which means that if you try to trigger a new task that matches the same
+``(name, metadata)`` pair of the currently running task, then the new task
+is not created and you get the task object of the current running task.
+
+For instance, consider the following example:
+
+.. code-block:: python
+
+  task1 = TaskManager.run("dummy/task", {'attr': 2}, func)
+  task2 = TaskManager.run("dummy/task", {'attr': 2}, func)
+
+If the second call to ``TaskManager.run`` executes while the first task is
+still executing then it will return the same task object:
+``assert task1 == task2``.
+
+
+How to get the list of executing and finished asynchronous tasks?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The list of executing and finished tasks is included in the ``Summary``
+controller, which is already polled every 5 seconds by the dashboard frontend.
+But we also provide a dedicated controller to get the same list of executing
+and finished tasks.
+
+The ``Task`` controller exposes the ``/api/task`` endpoint that returns the
+list of executing and finished tasks. This endpoint accepts the ``name``
+parameter that accepts a glob expression as its value.
+For instance, an HTTP GET request of the URL ``/api/task?name=rbd/*``
+will return all executing and finished tasks which name starts with ``rbd/``.
+
+To prevent the finished tasks list from growing unbounded, we will always
+maintain the 10 most recent finished tasks, and the remaining older finished
+tasks will be removed when reaching a TTL of 1 minute. The TTL is calculated
+using the timestamp when the task finished its execution. After a minute, when
+the finished task information is retrieved, either by the summary controller or
+by the task controller, it is automatically deleted from the list and it will
+not be included in further task queries.
+
+Each executing task is represented by the following dictionary::
+
+  {
+    'name': "name",  # str
+    'metadata': { },  # dict
+    'begin_time': "2018-03-14T15:31:38.423605Z",  # str (ISO 8601 format)
+    'progress': 0  # int (percentage)
+  }
+
+Each finished task is represented by the following dictionary::
+
+  {
+    'name': "name",  # str
+    'metadata': { },  # dict
+    'begin_time': "2018-03-14T15:31:38.423605Z",  # str (ISO 8601 format)
+    'end_time': "2018-03-14T15:31:39.423605Z",  # str (ISO 8601 format)
+    'duration': 0.0,  # float
+    'progress': 0  # int (percentage)
+    'success': True,  # bool
+    'ret_value': None,  # object, populated only if 'success' == True
+    'exception': None,  # str, populated only if 'success' == False
+  }
+
+
+How to use asynchronous APIs with asynchronous tasks?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``TaskManager.run`` method as described in a previous section, is well
+suited for calling blocking functions, as it runs the function inside a newly
+created thread. But sometimes we want to call some function of an API that is
+already asynchronous by nature.
+
+For these cases we want to avoid creating a new thread for just running a
+non-blocking function, and want to leverage the asynchronous nature of the
+function. The ``TaskManager.run`` is already prepared to be used with
+non-blocking functions by passing an object of the type ``TaskExecutor`` as an
+additional parameter called ``executor``. The full method signature of
+``TaskManager.run``::
+
+  TaskManager.run(name, metadata, func, args=None, kwargs=None, executor=None)
+
+
+The ``TaskExecutor`` class is responsible for code that executes a given task
+function, and defines three methods that can be overridden by
+subclasses::
+
+  def init(self, task)
+  def start(self)
+  def finish(self, ret_value, exception)
+
+The ``init`` method is called before the running the task function, and
+receives the task object (of class ``Task``).
+
+The ``start`` method runs the task function. The default implementation is to
+run the task function in the current thread context.
+
+The ``finish`` method should be called when the task function finishes with
+either the ``ret_value`` populated with the result of the execution, or with
+an exception object in the case that execution raised an exception.
+
+To leverage the asynchronous nature of a non-blocking function, the developer
+should implement a custom executor by creating a subclass of the
+``TaskExecutor`` class, and provide an instance of the custom executor class
+as the ``executor`` parameter of the ``TaskManager.run``.
+
+To better understand the expressive power of executors, we write a full example
+of use a custom executor to execute the ``MgrModule.send_command`` asynchronous
+function:
+
+.. code-block:: python
+
+  import json
+  from mgr_module import CommandResult
+  from .. import mgr
+  from ..tools import ApiController, RESTController, NotificationQueue, \
+                      TaskManager, TaskExecutor
+
+
+  class SendCommandExecutor(TaskExecutor):
+      def __init__(self):
+          super(SendCommandExecutor, self).__init__()
+          self.tag = None
+          self.result = None
+
+      def init(self, task):
+          super(SendCommandExecutor, self).init(task)
+
+          # we need to listen for 'command' events to know when the command
+          # finishes
+          NotificationQueue.register(self._handler, 'command')
+
+          # store the CommandResult object to retrieve the results
+          self.result = self.task.fn_args[0]
+          if len(self.task.fn_args) > 4:
+              # the user specified a tag for the command, so let's use it
+              self.tag = self.task.fn_args[4]
+          else:
+              # let's generate a unique tag for the command
+              self.tag = 'send_command_{}'.format(id(self))
+              self.task.fn_args.append(self.tag)
+
+      def _handler(self, data):
+          if data == self.tag:
+              # the command has finished, notifying the task with the result
+              self.finish(self.result.wait(), None)
+              # deregister listener to avoid memory leaks
+              NotificationQueue.deregister(self._handler, 'command')
+
+
+  @ApiController('test')
+  class Test(RESTController):
+
+      def _run_task(self, osd_id):
+          task = TaskManager.run("test/task", {}, mgr.send_command,
+                                 [CommandResult(''), 'osd', osd_id,
+                                  json.dumps({'prefix': 'perf histogram dump'})],
+                                 executor=SendCommandExecutor())
+          return task.wait(1.0)
+
+      def get(self, osd_id):
+          status, value = self._run_task(osd_id)
+          return {'status': status, 'value': value}
+
+
+The above ``SendCommandExecutor`` executor class can be used for any call to
+``MgrModule.send_command``. This means that we should need just one custom
+executor class implementation for each non-blocking API that we use in our
+controllers.
+
+The default executor, used when no executor object is passed to
+``TaskManager.run``, is the ``ThreadedExecutor``. You can check its
+implementation in the ``tools.py`` file.
+
+
+How to update the execution progress of an asynchronous task?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The asynchronous tasks infrastructure provides support for updating the
+execution progress of an executing task.
+The progress can be updated from within the code the task is executing, which
+usually is the place where we have the progress information available.
+
+To update the progress from within the task code, the ``TaskManager`` class
+provides a method to retrieve the current task object::
+
+  TaskManager.current_task()
+
+The above method is only available when using the default executor
+``ThreadedExecutor`` for executing the task.
+The ``current_task()`` method returns the current ``Task`` object. The
+``Task`` object provides two public methods to update the execution progress
+value: the ``set_progress(percentage)``, and the ``inc_progress(delta)``
+methods.
+
+The ``set_progress`` method receives as argument an integer value representing
+the absolute percentage that we want to set to the task.
+
+The ``inc_progress`` method receives as argument an integer value representing
+the delta we want to increment to the current execution progress percentage.
+
+Take the following example of a controller that triggers a new task and
+updates its progress:
+
+.. code-block:: python
+
+  import random
+  import time
+  import cherrypy
+  from ..tools import TaskManager, ApiController, BaseController
+
+
+  @ApiController('dummy_task')
+  class DummyTask(BaseController):
+      def _dummy(self):
+          top = random.randrange(100)
+          for i in range(top):
+              TaskManager.current_task().set_progress(i*100/top)
+              # or TaskManager.current_task().inc_progress(100/top)
+              time.sleep(1)
+          return "finished"
+
+      @cherrypy.expose
+      @cherrypy.tools.json_out()
+      def default(self):
+          task = TaskManager.run("dummy/task", {}, self._dummy)
+          return task.wait(5)  # wait for five seconds
+
+
+How to deal with asynchronous tasks in the front-end?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All executing and most recently finished asynchronous tasks are displayed on
+"Background-Tasks" and if finished on "Recent-Notifications" in the menu bar.
+For each task a operation name for three states (running, success and failure),
+a function that tells who is involved and error descriptions, if any, have to
+be provided. This can be  achieved by appending
+``TaskManagerMessageService.messages``.  This has to be done to achieve
+consistency among all tasks and states.
+
+Operation Object
+  Ensures consistency among all tasks. It consists of three verbs for each
+  different state f.e.
+  ``{running: 'Creating', failure: 'create', success: 'Created'}``.
+
+#. Put running operations in present participle f.e. ``'Updating'``.
+#. Failed messages always start with ``'Failed to '`` and should be continued
+   with the operation in present tense f.e. ``'update'``.
+#. Put successful operations in past tense f.e. ``'Updated'``.
+
+Involves Function
+  Ensures consistency among all messages of a task, it resembles who's
+  involved by the operation. It's a function that returns a string which
+  takes the metadata from the task to return f.e.
+  ``"RBD 'somePool/someImage'"``.
+
+Both combined create the following messages:
+
+* Failure => ``"Failed to create RBD 'somePool/someImage'"``
+* Running => ``"Creating RBD 'somePool/someImage'"``
+* Success => ``"Created RBD 'somePool/someImage'"``
+
+For automatic task handling use ``TaskWrapperService.wrapTaskAroundCall``.
+
+If for some reason ``wrapTaskAroundCall`` is not working for you,
+you have to subscribe to your asynchronous task manually through
+``TaskManagerService.subscribe``, and provide it with a callback,
+in case of a success to notify the user. A notification can
+be triggered with ``NotificationService.notifyTask``. It will use
+``TaskManagerMessageService.messages`` to display a message based on the state
+of a task.
+
+Notifications of API errors are handled by ``ApiInterceptorService``.
+
+Usage example:
+
+.. code-block:: javascript
+
+  export class TaskManagerMessageService {
+    // ...
+    messages = {
+      // Messages for task 'rbd/create'
+      'rbd/create': new TaskManagerMessage(
+        // Message prefixes
+        ['create', 'Creating', 'Created'],
+        // Message suffix
+        (metadata) => `RBD '${metadata.pool_name}/${metadata.image_name}'`,
+        (metadata) => ({
+          // Error code and description
+          '17': `Name is already used by RBD '${metadata.pool_name}/${
+                 metadata.image_name}'.`
+        })
+      ),
+      // ...
+    };
+    // ...
+  }
+
+  export class RBDFormComponent {
+    // ...
+    createAction() {
+      const request = this.createRequest();
+      // Subscribes to 'call' with submitted 'task' and handles notifications
+      return this.taskWrapper.wrapTaskAroundCall({
+        task: new FinishedTask('rbd/create', {
+          pool_name: request.pool_name,
+          image_name: request.name
+        }),
+        call: this.rbdService.create(request)
+      });
+    }
+    // ...
+  }
+
+
+REST API documentation
+~~~~~~~~~~~~~~~~~~~~~~
+Ceph-Dashboard provides two types of documentation for the **Ceph RESTful API**:
+
+* **Static documentation**: available at :ref:`mgr ceph api`. This comes from a versioned specification located at ``src/pybind/mgr/dashboard/openapi.yaml``.
+* **Interactive documentation**: available from a running Ceph-Dashboard instance (top-right ``?`` icon > API Docs).
+
+If changes are made to the ``controllers/`` directory, it's very likely that
+they will result in changes to the generated OpenAPI specification. For that
+reason, a checker has been implemented to block unintended changes. This check
+is automatically triggered by the Pull Request CI (``make check``) and can be
+also manually invoked: ``tox -e openapi-check``.
+
+If that checker failed, it means that the current Pull Request is modifying the
+Ceph API and therefore:
+
+#. The versioned OpenAPI specification should be updated explicitly: ``tox -e openapi-fix``.
+#. The team @ceph/api will be requested for reviews (this is automated via GitHub CODEOWNERS), in order to asses the impact of changes.
+
+Additionally, Sphinx documentation can be generated from the OpenAPI
+specification with ``tox -e openapi-doc``.
+
+The Ceph RESTful OpenAPI specification is dynamically generated from the
+``Controllers`` in ``controllers/`` directory.  However, by default it is not
+very detailed, so there are two decorators that can and should be used to add
+more information:
+
+* ``@EndpointDoc()`` for documentation of endpoints. It has four optional arguments
+  (explained below): ``description``, ``group``, ``parameters`` and
+  ``responses``.
+* ``@ControllerDoc()`` for documentation of controller or group associated with
+  the endpoints. It only takes the two first arguments: ``description`` and
+  ``group``.
+
+
+``description``: A a string with a short (1-2 sentences) description of the object.
+
+
+``group``: By default, an endpoint is grouped together with other endpoints
+within the same controller class. ``group`` is a string that can be used to
+assign an endpoint or all endpoints in a class to another controller or a
+conceived group name.
+
+
+``parameters``: A dict used to describe path, query or request body parameters.
+By default, all parameters for an endpoint are listed on the Swagger UI page,
+including information of whether the parameter is optional/required and default
+values. However, there will be no description of the parameter and the parameter
+type will only be displayed in some cases.
+When adding information, each parameters should be described as in the example
+below. Note that the parameter type should be expressed as a built-in python
+type and not as a string. Allowed values are ``str``, ``int``, ``bool``, ``float``.
+
+.. code-block:: python
+
+ @EndpointDoc(parameters={'my_string': (str, 'Description of my_string')})
+ def method(my_string): pass
+
+For body parameters, more complex cases are possible. If the parameter is a
+dictionary, the type should be replaced with a ``dict`` containing its nested
+parameters. When describing nested parameters, the same format as other
+parameters is used. However, all nested parameters are set as required by default.
+If the nested parameter is optional this must be specified as for ``item2`` in
+the example below. If a nested parameters is set to optional, it is also
+possible to specify the default value (this will not be provided automatically
+for nested parameters).
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={
+    'my_dictionary': ({
+      'item1': (str, 'Description of item1'),
+      'item2': (str, 'Description of item2', True),  # item2 is optional
+      'item3': (str, 'Description of item3', True, 'foo'),  # item3 is optional with 'foo' as default value
+  }, 'Description of my_dictionary')})
+  def method(my_dictionary): pass
+
+If the parameter is a ``list`` of primitive types, the type should be
+surrounded with square brackets.
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={'my_list': ([int], 'Description of my_list')})
+  def method(my_list): pass
+
+If the parameter is a ``list`` with nested parameters, the nested parameters
+should be placed in a dictionary and surrounded with square brackets.
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={
+    'my_list': ([{
+      'list_item': (str, 'Description of list_item'),
+      'list_item2': (str, 'Description of list_item2')
+  }], 'Description of my_list')})
+  def method(my_list): pass
+
+
+``responses``: A dict used for describing responses. Rules for describing
+responses are the same as for request body parameters, with one difference:
+responses also needs to be assigned to the related response code as in the
+example below:
+
+.. code-block:: python
+
+  @EndpointDoc(responses={
+    '400':{'my_response': (str, 'Description of my_response')}})
+  def method(): pass
+
+
+Error Handling in Python
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Good error handling is a key requirement in creating a good user experience
+and providing a good API.
+
+Dashboard code should not duplicate C++ code. Thus, if error handling in C++
+is sufficient to provide good feedback, a new wrapper to catch these errors
+is not necessary. On the other hand, input validation is the best place to
+catch errors and generate the best error messages. If required, generate
+errors as soon as possible.
+
+The backend provides few standard ways of returning errors.
+
+First, there is a generic Internal Server Error::
+
+    Status Code: 500
+    {
+        "version": <cherrypy version, e.g. 13.1.0>,
+        "detail": "The server encountered an unexpected condition which prevented it from fulfilling the request.",
+    }
+
+
+For errors generated by the backend, we provide a standard error
+format::
+
+    Status Code: 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+    }
+
+
+In case, the API Endpoints uses @ViewCache to temporarily cache results,
+the error looks like so::
+
+    Status Code 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+        'status': 3,          # Indicating the @ViewCache error status
+    }
+
+In case, the API Endpoints uses a task the error looks like so::
+
+    Status Code 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+        "task": {             # Information about the task itself
+            "name": "taskname",
+            "metadata": {...}
+        }
+    }
+
+
+Our WebUI should show errors generated by the API to the user. Especially
+field-related errors in wizards and dialogs or show non-intrusive notifications.
+
+Handling exceptions in Python should be an exception. In general, we
+should have few exception handlers in our project. Per default, propagate
+errors to the API, as it will take care of all exceptions anyway. In general,
+log the exception by adding ``logger.exception()`` with a description to the
+handler.
+
+We need to distinguish between user errors from internal errors and
+programming errors. Using different exception types will ease the
+task for the API layer and for the user interface:
+
+Standard Python errors, like ``SystemError``, ``ValueError`` or ``KeyError``
+will end up as internal server errors in the API.
+
+In general, do not ``return`` error responses in the REST API. They will be
+returned by the  error handler. Instead, raise the appropriate exception.
+
+Plug-ins
+~~~~~~~~
+
+New functionality can be provided by means of a plug-in architecture. Among the
+benefits this approach brings in, loosely coupled development is one of the most
+notable. As the Ceph Dashboard grows in feature richness, its code-base becomes
+more and more complex. The hook-based nature of a plug-in architecture allows to
+extend functionality in a controlled manner, and isolate the scope of the
+changes.
+
+Ceph Dashboard relies on `Pluggy <https://pluggy.readthedocs.io>`_ to provide
+for plug-ing support. On top of pluggy, an interface-based approach has been
+implemented, with some safety checks (method override and abstract method
+checks).
+
+In order to create a new plugin, the following steps are required:
+
+#. Add a new file under ``src/pybind/mgr/dashboard/plugins``.
+#. Import the ``PLUGIN_MANAGER`` instance and the ``Interfaces``.
+#. Create a class extending the desired interfaces. The plug-in library will
+   check if all the methods of the interfaces have been properly overridden.
+#. Register the plugin in the ``PLUGIN_MANAGER`` instance.
+#. Import the plug-in from within the Ceph Dashboard ``module.py`` (currently no
+   dynamic loading is implemented).
+
+The available Mixins (helpers) are:
+
+- ``CanMgr``: provides the plug-in with access to the ``mgr`` instance under ``self.mgr``.
+
+The available Interfaces are:
+
+- ``Initializable``: requires overriding ``init()`` hook. This method is run at
+  the very beginning of the dashboard module, right after all imports have been
+  performed.
+- ``Setupable``: requires overriding ``setup()`` hook. This method is run in the
+  Ceph Dashboard ``serve()`` method, right after CherryPy has been configured,
+  but before it is started. It's a placeholder for the plug-in initialization
+  logic.
+- ``HasOptions``: requires overriding ``get_options()`` hook by returning a list
+  of ``Options()``. The options returned here are added to the
+  ``MODULE_OPTIONS``.
+- ``HasCommands``: requires overriding ``register_commands()`` hook by defining
+  the commands the plug-in can handle and decorating them with ``@CLICommand``.
+  The commands can be optionally returned, so that they can be invoked
+  externally (which makes unit testing easier).
+- ``HasControllers``: requires overriding ``get_controllers()`` hook by defining
+  and returning the controllers as usual.
+- ``FilterRequest.BeforeHandler``: requires overriding
+  ``filter_request_before_handler()`` hook. This method receives a
+  ``cherrypy.request`` object for processing. A usual implementation of this
+  method will allow some requests to pass or will raise a ``cherrypy.HTTPError``
+  based on the ``request`` metadata and other conditions.
+
+New interfaces and hooks should be added as soon as they are required to
+implement new functionality. The above list only comprises the hooks needed for
+the existing plugins.
+
+A sample plugin implementation would look like this:
+
+.. code-block:: python
+
+  # src/pybind/mgr/dashboard/plugins/mute.py
+
+  from . import PLUGIN_MANAGER as PM
+  from . import interfaces as I
+
+  from mgr_module import CLICommand, Option
+  import cherrypy
+
+  @PM.add_plugin
+  class Mute(I.CanMgr, I.Setupable, I.HasOptions, I.HasCommands,
+                       I.FilterRequest.BeforeHandler, I.HasControllers):
+    @PM.add_hook
+    def get_options(self):
+      return [Option('mute', default=False, type='bool')]
+
+    @PM.add_hook
+    def setup(self):
+      self.mute = self.mgr.get_module_option('mute')
+
+    @PM.add_hook
+    def register_commands(self):
+      @CLICommand("dashboard mute")
+      def _(mgr):
+        self.mute = True
+        self.mgr.set_module_option('mute', True)
+        return 0
+
+    @PM.add_hook
+    def filter_request_before_handler(self, request):
+      if self.mute:
+        raise cherrypy.HTTPError(500, "I'm muted :-x")
+
+    @PM.add_hook
+    def get_controllers(self):
+      from ..controllers import ApiController, RESTController
+
+      @ApiController('/mute')
+      class MuteController(RESTController):
+        def get(_):
+          return self.mute
+
+      return [MuteController]
+
+
+Additionally, a helper for creating plugins ``SimplePlugin`` is provided. It
+facilitates the basic tasks (Options, Commands, and common Mixins). The previous
+plugin could be rewritten like this:
+
+.. code-block:: python
+
+  from . import PLUGIN_MANAGER as PM
+  from . import interfaces as I
+  from .plugin import SimplePlugin as SP
+
+  import cherrypy
+
+  @PM.add_plugin
+  class Mute(SP, I.Setupable, I.FilterRequest.BeforeHandler, I.HasControllers):
+    OPTIONS = [
+        SP.Option('mute', default=False, type='bool')
+    ]
+
+    def shut_up(self):
+      self.set_option('mute', True)
+      self.mute = True
+      return 0
+
+    COMMANDS = [
+        SP.Command("dashboard mute", handler=shut_up)
+    ]
+
+    @PM.add_hook
+    def setup(self):
+      self.mute = self.get_option('mute')
+
+    @PM.add_hook
+    def filter_request_before_handler(self, request):
+      if self.mute:
+        raise cherrypy.HTTPError(500, "I'm muted :-x")
+
+    @PM.add_hook
+    def get_controllers(self):
+      from ..controllers import ApiController, RESTController
+
+      @ApiController('/mute')
+      class MuteController(RESTController):
+        def get(_):
+          return self.mute
+
+      return [MuteController]
diff --git a/doc/dev/developer_guide/debugging-gdb.rst b/doc/dev/developer_guide/debugging-gdb.rst
new file mode 100644
index 000000000..153144431
--- /dev/null
+++ b/doc/dev/developer_guide/debugging-gdb.rst
@@ -0,0 +1,43 @@
+GDB - The GNU Project Debugger
+==============================
+
+`The GNU Project Debugger (GDB) <https://www.sourceware.org/gdb>`_ is
+a powerful tool that allows you to analyze the execution flow
+of a process.
+GDB can help to find bugs, uncover crash errors or track the
+source code during execution of a development cluster.
+It can also be used to debug Teuthology test runs.
+
+GET STARTED WITH GDB
+--------------------
+
+Basic usage with examples can be found `here. <https://geeksforgeeks.org/gdb-command-in-linux-with-examples>`_
+GDB can be attached to a running process. For instance, after deploying a
+development cluster, the process number (PID) of a ``ceph-osd`` daemon can be found in::
+
+    $ cd build
+    $ cat out/osd.0.pid
+
+Attaching gdb to the process::
+
+    $ gdb ./bin/ceph-osd -p <pid>
+
+.. note::
+    It is recommended to compile without any optimizations (``-O0`` gcc flag)
+    in order to avoid elimination of intermediate values.
+
+Stopping for breakpoints while debugging may cause timeouts, so the following
+configuration options are suggested::
+
+        [osd]
+        osd_op_thread_timeout = 1500
+        osd_op_thread_suicide_timeout = 1500
+
+Debugging Teuthology Tests
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``src/script/ceph-debug-docker.sh`` can be used to analyze Teuthology failures::
+
+    $ ./ceph-debug-docker.sh <branch-name>
+
+Refer to the script header for more information.
diff --git a/doc/dev/developer_guide/essentials.rst b/doc/dev/developer_guide/essentials.rst
new file mode 100644
index 000000000..5a31e430b
--- /dev/null
+++ b/doc/dev/developer_guide/essentials.rst
@@ -0,0 +1,346 @@
+Essentials (tl;dr)
+==================
+
+This chapter presents essential information that every Ceph developer needs
+to know.
+
+Leads
+-----
+
+The Ceph project was created by Sage Weil and is led by the Ceph Leadership
+Team (CLT). In addition, each major project component has its own lead. The
+following table shows all the leads and their nicks on `GitHub`_:
+
+.. _github: https://github.com/
+
+========= ================ =============
+Scope     Lead             GitHub nick
+========= ================ =============
+Ceph      Sage Weil        liewegas
+RADOS     Neha Ojha        neha-ojha
+RGW       Yehuda Sadeh     yehudasa
+RGW       Matt Benjamin    mattbenjamin
+RBD       Ilya Dryomov     dis 
+CephFS    Venky Shankar    vshankar
+Dashboard Ernesto Puerta   epuertat
+MON       Joao Luis        jecluis
+Build/Ops Ken Dreyer       ktdreyer
+Docs      Zac Dover        zdover23
+========= ================ =============
+
+The Ceph-specific acronyms in the table are explained in
+:doc:`/architecture`.
+
+History
+-------
+
+See the `History chapter of the Wikipedia article`_.
+
+.. _`History chapter of the Wikipedia article`: https://en.wikipedia.org/wiki/Ceph_%28software%29#History
+
+Licensing
+---------
+
+Ceph is free software.
+
+Unless stated otherwise, the Ceph source code is distributed under the
+terms of the LGPL2.1 or LGPL3.0. For full details, see the file
+`COPYING`_ in the top-level directory of the source-code tree.
+
+.. _`COPYING`:
+  https://github.com/ceph/ceph/blob/master/COPYING
+
+Source code repositories
+------------------------
+
+The source code of Ceph lives on `GitHub`_ in a number of repositories below
+the `Ceph "organization"`_.
+
+.. _`Ceph "organization"`: https://github.com/ceph
+
+A working knowledge of git_ is essential to make a meaningful contribution to the project as a developer.
+
+.. _git: https://git-scm.com/doc
+
+Although the `Ceph "organization"`_ includes several software repositories,
+this document covers only one: https://github.com/ceph/ceph.
+
+Redmine issue tracker
+---------------------
+
+Although `GitHub`_ is used for code, Ceph-related issues (Bugs, Features,
+Backports, Documentation, etc.) are tracked at http://tracker.ceph.com,
+which is powered by `Redmine`_.
+
+.. _Redmine: http://www.redmine.org
+
+The tracker has a Ceph project with a number of subprojects loosely
+corresponding to the various architectural components (see
+:doc:`/architecture`).
+
+Mere `registration`_ in the tracker automatically grants permissions
+sufficient to open new issues and comment on existing ones.
+
+.. _registration: http://tracker.ceph.com/account/register
+
+To report a bug or propose a new feature, `jump to the Ceph project`_ and
+click on `New issue`_.
+
+.. _`jump to the Ceph project`: http://tracker.ceph.com/projects/ceph
+.. _`New issue`: http://tracker.ceph.com/projects/ceph/issues/new
+
+Slack
+-----
+
+Ceph's Slack is https://ceph-storage.slack.com/.
+
+.. _mailing-list:
+
+Mailing lists
+-------------
+
+Ceph Development Mailing List
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``dev@ceph.io`` list is for discussion about the development of Ceph,
+its interoperability with other technology, and the operations of the
+project itself.
+
+The email discussion list for Ceph development is open to all. Subscribe by
+sending a message to ``dev-request@ceph.io`` with the following line in the
+body of the message::
+
+    subscribe ceph-devel
+
+
+Ceph Client Patch Review Mailing List
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``ceph-devel@vger.kernel.org`` list is for discussion and patch review
+for the Linux kernel Ceph client component. Note that this list used to
+be an all-encompassing list for developers. When searching the archives, 
+remember that this list contains the generic devel-ceph archives before mid-2018.
+
+Subscribe to the list covering the Linux kernel Ceph client component by sending
+a message to ``majordomo@vger.kernel.org`` with the following line in the body
+of the message::
+
+    subscribe ceph-devel
+
+
+Other Ceph Mailing Lists
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are also `other Ceph-related mailing lists`_.
+
+.. _`other Ceph-related mailing lists`: https://ceph.com/irc/
+
+.. _irc:
+
+
+IRC
+---
+
+In addition to mailing lists, the Ceph community also communicates in real time
+using `Internet Relay Chat`_.
+
+.. _`Internet Relay Chat`: http://www.irchelp.org/
+
+The Ceph community gathers in the #ceph channel of the Open and Free Technology
+Community (OFTC) IRC network.
+
+Created in 1988, Internet Relay Chat (IRC) is a relay-based, real-time chat
+protocol. It is mainly designed for group (many-to-many) communication in
+discussion forums called channels, but also allows one-to-one communication via
+private message. On IRC you can talk to many other members using Ceph, on
+topics ranging from idle chit-chat to support questions. Though a channel might
+have many people in it at any one time, they might not always be at their
+keyboard; so if no-one responds, just wait around and someone will hopefully
+answer soon enough.
+
+Registration
+~~~~~~~~~~~~
+
+If you intend to use the IRC service on a continued basis, you are advised to
+register an account. Registering gives you a unique IRC identity and allows you
+to access channels where unregistered users have been locked out for technical
+reasons.
+
+See ``the official OFTC (Open and Free Technology Community) documentation's
+registration instructions
+<https://www.oftc.net/Services/#register-your-account>`` to learn how to
+register your IRC account.
+
+Channels
+~~~~~~~~
+
+To connect to the OFTC IRC network, download an IRC client and configure it to
+connect to ``irc.oftc.net``. Then join one or more of the channels. Discussions
+inside #ceph are logged and archives are available online.
+
+Here are the real-time discussion channels for the Ceph community:
+
+  -  #ceph
+  -  #ceph-devel
+  -  #cephfs
+  -  #ceph-dashboard
+  -  #ceph-orchestrators
+  -  #sepia
+
+
+.. _submitting-patches:
+
+Submitting patches
+------------------
+
+The canonical instructions for submitting patches are contained in the
+file `CONTRIBUTING.rst`_ in the top-level directory of the source-code
+tree. There may be some overlap between this guide and that file.
+
+.. _`CONTRIBUTING.rst`:
+  https://github.com/ceph/ceph/blob/main/CONTRIBUTING.rst
+
+All newcomers are encouraged to read that file carefully.
+
+Building from source
+--------------------
+
+See instructions at :doc:`/install/build-ceph`.
+
+Using ccache to speed up local builds
+-------------------------------------
+`ccache`_ can make the process of rebuilding the ceph source tree faster. 
+
+Before you use `ccache`_ to speed up your rebuilds of the ceph source tree,
+make sure that your source tree is clean and will produce no build failures.
+When you have a clean source tree, you can confidently use `ccache`_, secure in
+the knowledge that you're not using a dirty tree.
+
+Old build artifacts can cause build failures. You might introduce these
+artifacts unknowingly when switching from one branch to another. If you see
+build errors when you attempt a local build, follow the procedure below to
+clean your source tree.
+
+Cleaning the Source Tree
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. prompt:: bash $
+
+  ninja clean
+  
+.. note:: The following commands will remove everything in the source tree 
+          that isn't tracked by git. Make sure to back up your log files 
+          and configuration options before running these commands.
+
+.. prompt:: bash $
+
+   git clean -fdx; git submodule foreach git clean -fdx
+
+Building Ceph with ccache
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``ccache`` is available as a package in most distros. To build ceph with
+ccache, run the following command.
+
+.. prompt:: bash $
+
+  cmake -DWITH_CCACHE=ON ..
+
+Using ccache to Speed Up Build Times
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``ccache`` can be used for speeding up all builds of the system. For more
+details, refer to the `run modes`_ section of the ccache manual. The default
+settings of ``ccache`` can be displayed with the ``ccache -s`` command.
+
+.. note:: We recommend overriding the ``max_size``. The default is 10G.
+          Use a larger value, like 25G. Refer to the `configuration`_ section
+          of the ccache manual for more information.
+
+To further increase the cache hit rate and reduce compile times in a
+development environment, set the version information and build timestamps to
+fixed values. This makes it unnecessary to rebuild the binaries that contain
+this information.
+
+This can be achieved by adding the following settings to the ``ccache``
+configuration file ``ccache.conf``::
+
+  sloppiness = time_macros
+  run_second_cpp = true
+
+Now, set the environment variable ``SOURCE_DATE_EPOCH`` to a fixed value (a
+UNIX timestamp) and set ``ENABLE_GIT_VERSION`` to ``OFF`` when running
+``cmake``:
+
+.. prompt:: bash $
+
+  export SOURCE_DATE_EPOCH=946684800
+  cmake -DWITH_CCACHE=ON -DENABLE_GIT_VERSION=OFF ..
+
+.. note:: Binaries produced with these build options are not suitable for
+  production or debugging purposes, as they do not contain the correct build
+  time and git version information.
+
+.. _`ccache`: https://ccache.samba.org/
+.. _`run modes`: https://ccache.samba.org/manual.html#_run_modes
+.. _`configuration`: https://ccache.samba.org/manual.html#_configuration
+
+Development-mode cluster
+------------------------
+
+See :doc:`/dev/quick_guide`.
+
+Kubernetes/Rook development cluster
+-----------------------------------
+
+See :ref:`kubernetes-dev`
+
+.. _backporting:
+
+Backporting
+-----------
+
+All bugfixes should be merged to the ``main`` branch before being
+backported. To flag a bugfix for backporting, make sure it has a
+`tracker issue`_ associated with it and set the ``Backport`` field to a
+comma-separated list of previous releases (e.g. "hammer,jewel") that you think
+need the backport.
+The rest (including the actual backporting) will be taken care of by the
+`Stable Releases and Backports`_ team.
+
+.. _`tracker issue`: http://tracker.ceph.com/
+.. _`Stable Releases and Backports`: http://tracker.ceph.com/projects/ceph-releases/wiki
+
+Dependabot
+----------
+
+Dependabot is a GitHub bot that scans the dependencies in the repositories for
+security vulnerabilities (CVEs). If a fix is available for a discovered CVE,
+Dependabot creates a pull request to update the dependency.
+
+Dependabot also indicates the compatibility score of the upgrade. This score is
+based on the number of CI failures that occur in other GitHub repositories
+where the fix was applied. 
+
+With some configuration, Dependabot can perform non-security updates (for
+example, it can upgrade to the latest minor version or patch version).
+
+Dependabot supports `several languages and package managers
+<https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates#supported-repositories-and-ecosystems>`_.
+As of July 2022, the Ceph project receives alerts only from pip (based on the
+`requirements.txt` files) and npm (`package*.json`). It is possible to extend
+these alerts to git submodules, Golang, and Java. As of July 2022, there is no
+support for C++ package managers such as vcpkg, conan, C++20 modules.
+
+Many of the dependencies discovered by Dependabot will best be updated
+elsewhere than the Ceph Github repository (distribution packages, for example,
+will be a better place to update some of the dependencies). Nonetheless, the
+list of new and existing vulnerabilities generated by Dependabot will be
+useful.
+
+`Here is an example of a Dependabot pull request.
+<https://github.com/ceph/ceph/pull/46998>`_
+
+Guidance for use of cluster log
+-------------------------------
+
+If your patches emit messages to the Ceph cluster log, please consult
+this: :doc:`/dev/logging`.
diff --git a/doc/dev/developer_guide/index.rst b/doc/dev/developer_guide/index.rst
new file mode 100644
index 000000000..e9832bea6
--- /dev/null
+++ b/doc/dev/developer_guide/index.rst
@@ -0,0 +1,25 @@
+============================================
+Contributing to Ceph: A Guide for Developers
+============================================
+
+:Author: Loic Dachary
+:Author: Nathan Cutler
+:License: Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0)
+
+.. note:: You may also be interested in the :doc:`/dev/internals` documentation.
+
+.. toctree::
+   :maxdepth: 1
+
+   Introduction <intro>
+   Essentials <essentials>
+   What is Merged and When <merging>
+   Issue tracker <issue-tracker>
+   Basic workflow <basic-workflow>
+   Tests: Unit Tests <tests-unit-tests>
+   Tests: Integration Tests (Teuthology) <testing_integration_tests/index>
+   Tests: Running Tests (Locally) <running-tests-locally>
+   Ceph Dashboard Developer Documentation (formerly HACKING.rst) <dash-devel>
+   Tracing Developer Documentation <jaegertracing>
+   Cephadm Developer Documentation  <../cephadm/index>
+   Debugging with GDB <debugging-gdb>
diff --git a/doc/dev/developer_guide/intro.rst b/doc/dev/developer_guide/intro.rst
new file mode 100644
index 000000000..67b449c55
--- /dev/null
+++ b/doc/dev/developer_guide/intro.rst
@@ -0,0 +1,25 @@
+Introduction
+============
+
+This guide has two aims. First, it should lower the barrier to entry for
+software developers who wish to get involved in the Ceph project. Second,
+it should serve as a reference for Ceph developers.
+
+We assume that readers are already familiar with Ceph (the distributed
+object store and file system designed to provide excellent performance,
+reliability and scalability). If not, please refer to the `project website`_
+and especially the `publications list`_. Another way to learn about what's
+happening in Ceph is to check out our `youtube channel`_ , where we post Tech
+Talks, Code walk-throughs and Ceph Developer Monthly recordings.
+
+.. _`project website`: https://ceph.com
+.. _`publications list`: https://ceph.com/publications/
+.. _`youtube channel`: https://www.youtube.com/c/CephStorage
+
+Since this document is to be consumed by developers, who are assumed to
+have Internet access, topics covered elsewhere, either within the Ceph
+documentation or elsewhere on the web, are treated by linking. If you
+notice that a link is broken or if you know of a better link, please
+`report it as a bug`_.
+
+.. _`report it as a bug`: http://tracker.ceph.com/projects/ceph/issues/new
diff --git a/doc/dev/developer_guide/issue-tracker.rst b/doc/dev/developer_guide/issue-tracker.rst
new file mode 100644
index 000000000..eae68f3f0
--- /dev/null
+++ b/doc/dev/developer_guide/issue-tracker.rst
@@ -0,0 +1,39 @@
+.. _issue-tracker:
+
+Issue Tracker
+=============
+
+See `Redmine Issue Tracker`_ for a brief introduction to the Ceph Issue
+Tracker.
+
+Ceph developers use the issue tracker to
+
+1. keep track of issues - bugs, fix requests, feature requests, backport
+requests, etc.
+
+2. communicate with other developers and keep them informed as work
+on the issues progresses.
+
+Issue tracker conventions
+-------------------------
+
+When you start working on an existing issue, it's nice to let the other
+developers know this - to avoid duplication of labor. Typically, this is
+done by changing the :code:`Assignee` field (to yourself) and changing the
+:code:`Status` to *In progress*. Newcomers to the Ceph community typically do
+not have sufficient privileges to update these fields, however: they can
+simply update the issue with a brief note.
+
+.. table:: Meanings of some commonly used statuses
+
+   ================ ===========================================
+   Status           Meaning
+   ================ ===========================================
+   New              Initial status
+   In Progress      Somebody is working on it
+   Need Review      Pull request is open with a fix
+   Pending Backport Fix has been merged, backport(s) pending
+   Resolved         Fix and backports (if any) have been merged
+   ================ ===========================================
+
+.. _Redmine issue tracker: https://tracker.ceph.com
diff --git a/doc/dev/developer_guide/jaegertracing.rst b/doc/dev/developer_guide/jaegertracing.rst
new file mode 100644
index 000000000..73a48ad83
--- /dev/null
+++ b/doc/dev/developer_guide/jaegertracing.rst
@@ -0,0 +1,63 @@
+JAEGER- DISTRIBUTED TRACING
+===========================
+
+Jaeger + Opentracing provides ready to use tracing services for distributed
+systems and is becoming the widely used standard because of their simplicity and
+standardization.
+
+We use a modified `jaeger-cpp-client
+<https://github.com/ceph/jaeger-client-cpp>`_ the backend provided to the
+Opentracing API, which is responsible for the collection of spans, these spans
+are made with the use of smart pointers that carry the timestamp, TraceID and other
+meta info like a specific tag/log associated with the span to uniquely identify
+it across the distributed system.
+
+
+BASIC ARCHITECTURE AND TERMINOLOGY
+----------------------------------
+
+refer to the `Ceph Tracing documentation <../../../jaegertracing/#basic-architecture-and-terminology>`_
+
+
+HOW TO GET STARTED USING TRACING?
+---------------------------------
+
+Enabling jaegertracing with Ceph needs deployment Jaeger daemons + compiling
+Ceph with Jaeger, orchestrated to be used in vstart cluster for developers, this
+uses a jaeger `all-in-one docker
+<https://www.jaegertracing.io/docs/1.22/getting-started/#all-in-one>`_ which
+isn't recommended for production, but for testing purposes. Let's look at all the
+steps needed:
+
+  1. Update system with Jaeger dependencies, using install-deps::
+
+     $ WITH_JAEGER=true ./install-deps.sh
+
+  2. Compile Ceph with Jaeger enabled:
+
+    - for precompiled build::
+
+      $ cd build
+      $ cmake -DWITH_JAEGER=ON ..
+
+    - for fresh compilation using do_cmake.sh::
+
+      $ ./do_cmake.sh -DWITH_JAEGER=ON && ninja vstart
+
+  3. After successful compiling, start a vstart cluster with `--jaeger` which
+  will deploy `jaeger all-in-one <https://www.jaegertracing.io/docs/1.20/getting-started/#all-in-one>`_
+  using container deployment services(docker/podman)::
+
+   $ MON=1 MGR=0 OSD=1 ../src/vstart.sh --with-jaeger
+
+  if the deployment is unsuccessful, you can deploy `all-in-one
+  <https://www.jaegertracing.io/docs/1.20/getting- started/#all-in-one>`_
+  service manually and start vstart cluster without jaeger as well.
+
+
+  4. Test the traces using rados-bench write::
+
+     $ bin/rados -p test bench 5 write --no-cleanup
+
+.. seealso::
+ `using-jaeger-cpp-client-for-distributed-tracing-in-ceph <https://medium.com/@deepikaupadhyay/using-jaeger-cpp-client-for-distributed-tracing-in-ceph-8b1f4906ca2>`
\ No newline at end of file
diff --git a/doc/dev/developer_guide/merging.rst b/doc/dev/developer_guide/merging.rst
new file mode 100644
index 000000000..7e41bd483
--- /dev/null
+++ b/doc/dev/developer_guide/merging.rst
@@ -0,0 +1,138 @@
+.. _merging:
+
+Commit merging:  scope and cadence
+==================================
+
+Commits are merged into branches according to criteria specific to each phase
+of the Ceph release lifecycle. This chapter codifies these criteria.
+
+Development releases (i.e. x.0.z)
+---------------------------------
+
+What ?
+^^^^^^
+
+* Features
+* Bug fixes
+
+Where ?
+^^^^^^^
+
+Features are merged to the *main* branch. Bug fixes should be merged to the
+corresponding named branch (e.g. *nautilus* for 14.0.z, *pacific* for 16.0.z,
+etc.). However, this is not mandatory - bug fixes and documentation
+enhancements can be merged to the *main* branch as well, since the *main*
+branch is itself occasionally merged to the named branch during the development
+releases phase. In either case, if a bug fix is important it can also be
+flagged for backport to one or more previous stable releases.
+
+When ?
+^^^^^^
+
+After each stable release, candidate branches for previous releases enter
+phase 2 (see below).  For example: the *jewel* named branch was created when
+the *infernalis* release candidates entered phase 2. From this point on,
+*main* was no longer associated with *infernalis*. After he named branch of
+the next stable release is created, *main* will be occasionally merged into
+it.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The latest stable release branch is merged periodically into main.
+* The main branch is merged periodically into the branch of the stable release.
+* The main is merged into the stable release branch
+  immediately after each development (x.0.z) release.
+
+Stable release candidates (i.e. x.1.z) phase 1
+----------------------------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes only
+
+Where ?
+^^^^^^^
+
+The stable release branch (e.g. *jewel* for 10.0.z, *luminous*
+for 12.0.z, etc.) or *main*.  Bug fixes should be merged to the named
+branch corresponding to the stable release candidate (e.g. *jewel* for
+10.1.z) or to *main*. During this phase, all commits to *main* will be
+merged to the named branch, and vice versa. In other words, it makes
+no difference whether a commit is merged to the named branch or to
+*main* - it will make it into the next release candidate either way.
+
+When ?
+^^^^^^
+
+After the first stable release candidate is published, i.e. after the
+x.1.0 tag is set in the release branch.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The stable release branch is merged periodically into *main*.
+* The *main* branch is merged periodically into the stable release branch.
+* The *main* branch is merged into the stable release branch
+  immediately after each x.1.z release candidate.
+
+Stable release candidates (i.e. x.1.z) phase 2
+----------------------------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes only
+
+Where ?
+^^^^^^^
+
+The stable release branch (e.g. *mimic* for 13.0.z, *octopus* for 15.0.z
+,etc.). During this phase, all commits to the named branch will be merged into
+*main*. Cherry-picking to the named branch during release candidate phase 2
+is performed manually since the official backporting process begins only when
+the release is pronounced "stable".
+
+When ?
+^^^^^^
+
+After the CLT announces that it is time for phase 2 to happen.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The stable release branch is occasionally merged into main.
+
+Stable releases (i.e. x.2.z)
+----------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes
+* Features are sometime accepted
+* Commits should be cherry-picked from *main* when possible
+* Commits that are not cherry-picked from *main* must pertain to a bug unique to
+  the stable release
+* See also the `backport HOWTO`_ document
+
+.. _`backport HOWTO`:
+  http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO#HOWTO
+
+Where ?
+^^^^^^^
+
+The stable release branch (*hammer* for 0.94.x, *infernalis* for 9.2.x,
+etc.)
+
+When ?
+^^^^^^
+
+After the stable release is published, i.e. after the "vx.2.0" tag is set in
+the release branch.
+
+Branch merges
+^^^^^^^^^^^^^
+
+Never
diff --git a/doc/dev/developer_guide/running-tests-locally.rst b/doc/dev/developer_guide/running-tests-locally.rst
new file mode 100644
index 000000000..262683bfb
--- /dev/null
+++ b/doc/dev/developer_guide/running-tests-locally.rst
@@ -0,0 +1,171 @@
+Running Unit Tests
+==================
+
+How to run s3-tests locally
+---------------------------
+
+RGW code can be tested by building Ceph locally from source, starting a vstart
+cluster, and running the "s3-tests" suite against it.
+
+The following instructions should work on jewel and above.
+
+Step 1 - build Ceph
+^^^^^^^^^^^^^^^^^^^
+
+Refer to :doc:`/install/build-ceph`.
+
+You can do step 2 separately while it is building.
+
+Step 2 - vstart
+^^^^^^^^^^^^^^^
+
+When the build completes, and still in the top-level directory of the git
+clone where you built Ceph, do the following, for cmake builds::
+
+    cd build/
+    RGW=1 ../src/vstart.sh -n
+
+This will produce a lot of output as the vstart cluster is started up. At the
+end you should see a message like::
+
+    started.  stop.sh to stop.  see out/* (e.g. 'tail -f out/????') for debug output.
+
+This means the cluster is running.
+
+
+Step 3 - run s3-tests
+^^^^^^^^^^^^^^^^^^^^^
+
+.. highlight:: console
+
+To run the s3tests suite do the following::
+
+   $ ../qa/workunits/rgw/run-s3tests.sh
+
+
+Running test using vstart_runner.py
+-----------------------------------
+CephFS and Ceph Manager code is be tested using `vstart_runner.py`_.
+
+Running your first test
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+The Python tests in Ceph repository can be executed on your local machine
+using `vstart_runner.py`_. To do that, you'd need `teuthology`_ installed::
+
+    $ virtualenv --python=python3 venv
+    $ source venv/bin/activate
+    $ pip install 'setuptools >= 12'
+    $ pip install teuthology[test]@git+https://github.com/ceph/teuthology
+    $ deactivate
+
+The above steps installs teuthology in a virtual environment. Before running
+a test locally, build Ceph successfully from the source (refer
+:doc:`/install/build-ceph`) and do::
+
+    $ cd build
+    $ ../src/vstart.sh -n -d -l
+    $ source ~/path/to/teuthology/venv/bin/activate
+
+To run a specific test, say `test_reconnect_timeout`_ from
+`TestClientRecovery`_ in ``qa/tasks/cephfs/test_client_recovery``, you can
+do::
+
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery.test_reconnect_timeout
+
+The above command runs vstart_runner.py and passes the test to be executed as
+an argument to vstart_runner.py. In a similar way, you can also run the group
+of tests in the following manner::
+
+    $ # run all tests in class TestClientRecovery
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery
+    $ # run  all tests in test_client_recovery.py
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery
+
+Based on the argument passed, vstart_runner.py collects tests and executes as
+it would execute a single test.
+
+vstart_runner.py can take the following options -
+
+--clear-old-log             deletes old log file before running the test
+--create                    create Ceph cluster before running a test
+--create-cluster-only       creates the cluster and quits; tests can be issued
+                            later
+--interactive               drops a Python shell when a test fails
+--log-ps-output             logs ps output; might be useful while debugging
+--teardown                  tears Ceph cluster down after test(s) has finished
+                            running
+--kclient                   use the kernel cephfs client instead of FUSE
+--brxnet=<net/mask>         specify a new net/mask for the mount clients' network
+                            namespace container (Default: 192.168.0.0/16)
+
+.. note:: If using the FUSE client, ensure that the fuse package is installed
+          and enabled on the system and that ``user_allow_other`` is added
+          to ``/etc/fuse.conf``.
+
+.. note:: If using the kernel client, the user must have the ability to run
+          commands with passwordless sudo access.
+
+.. note:: A failure on the kernel client may crash the host, so it's
+          recommended to use this functionality within a virtual machine.
+
+Internal working of vstart_runner.py -
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+vstart_runner.py primarily does three things -
+
+* collects and runs the tests
+    vstart_runner.py setups/teardowns the cluster and collects and runs the
+    test. This is implemented using methods ``scan_tests()``, ``load_tests()``
+    and ``exec_test()``. This is where all the options that vstart_runner.py
+    takes are implemented along with other features like logging and copying
+    the traceback to the bottom of the log.
+
+* provides an interface for issuing and testing shell commands
+    The tests are written assuming that the cluster exists on remote machines.
+    vstart_runner.py provides an interface to run the same tests with the
+    cluster that exists within the local machine. This is done using the class
+    ``LocalRemote``. Class ``LocalRemoteProcess`` can manage the process that
+    executes the commands from ``LocalRemote``, class ``LocalDaemon`` provides
+    an interface to handle Ceph daemons and class ``LocalFuseMount`` can
+    create and handle FUSE mounts.
+
+* provides an interface to operate Ceph cluster
+    ``LocalCephManager`` provides methods to run Ceph cluster commands with
+    and without admin socket and ``LocalCephCluster`` provides methods to set
+    or clear ``ceph.conf``.
+
+.. note:: vstart_runner.py deletes "adjust-ulimits" and "ceph-coverage" from
+          the command arguments unconditionally since they are not applicable
+          when tests are run on a developer's machine.
+
+.. note:: "omit_sudo" is re-set to False unconditionally in cases of commands
+          "passwd" and "chown".
+
+.. note:: The presence of binary file named after the first argument is
+          checked in ``<ceph-repo-root>/build/bin/``. If present, the first
+          argument is replaced with the path to binary file.
+
+Running Workunits Using vstart_enviroment.sh
+--------------------------------------------
+
+Code can be tested by building Ceph locally from source, starting a vstart
+cluster, and running any suite against it.
+Similar to S3-Tests, other workunits can be run against by configuring your environment.
+
+Set up the environment
+^^^^^^^^^^^^^^^^^^^^^^
+
+Configure your environment::
+
+    $ . ./build/vstart_enviroment.sh
+
+Running a test
+^^^^^^^^^^^^^^
+
+To run a workunit (e.g ``mon/osd.sh``) do the following::
+
+    $ ./qa/workunits/mon/osd.sh
+
+.. _test_reconnect_timeout: https://github.com/ceph/ceph/blob/master/qa/tasks/cephfs/test_client_recovery.py#L133
+.. _TestClientRecovery: https://github.com/ceph/ceph/blob/master/qa/tasks/cephfs/test_client_recovery.py#L86
+.. _teuthology: https://github.com/ceph/teuthology
+.. _vstart_runner.py: https://github.com/ceph/ceph/blob/master/qa/tasks/vstart_runner.py
diff --git a/doc/dev/developer_guide/testing_integration_tests/index.rst b/doc/dev/developer_guide/testing_integration_tests/index.rst
new file mode 100644
index 000000000..363e2d212
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/index.rst
@@ -0,0 +1,16 @@
+=======================
+Teuthology User Guide
+=======================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+   :titlesonly:
+
+   Introduction <tests-integration-testing-teuthology-intro>
+   Workflow <tests-integration-testing-teuthology-workflow>
+   Debugging Tips <tests-integration-testing-teuthology-debugging-tips>
+   Kernel Development <tests-integration-testing-teuthology-kernel>
+   Sentry Notes <tests-sentry-developers-guide>
+
diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-debugging-tips.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-debugging-tips.rst
new file mode 100644
index 000000000..a959240ba
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-debugging-tips.rst
@@ -0,0 +1,158 @@
+.. _tests-integration-testing-teuthology-debugging-tips:
+
+Analyzing and Debugging A Teuthology Job
+========================================
+
+To learn more about how to schedule an integration test, refer to `Scheduling
+Test Run`_.
+
+Viewing Test Results
+--------------------
+
+When a teuthology run has been completed successfully, use `pulpito`_ dashboard
+to view the results::
+
+   http://pulpito.front.sepia.ceph.com/<job-name>/<job-id>/
+
+.. _pulpito: https://pulpito.ceph.com
+
+or ssh into the teuthology server to view the results of the integration test:
+
+  .. prompt:: bash $
+
+    ssh <username>@teuthology.front.sepia.ceph.com
+
+and access `teuthology archives`_, as in this example:
+
+  .. prompt:: bash $
+
+     nano /a/teuthology-2021-01-06_07:01:02-rados-master-distro-basic-smithi/
+
+.. note:: This requires you to have access to the Sepia lab. To learn how to
+          request access to the Sepia lab, see: 
+          https://ceph.github.io/sepia/adding_users/
+
+Identifying Failed Jobs
+-----------------------
+
+On pulpito, a job in red means either a failed job or a dead job. A job is
+combination of daemons and configurations defined in the yaml fragments in
+`qa/suites`_ . Teuthology uses these configurations and runs the tasks listed
+in `qa/tasks`_, which are commands that set up the test environment and test
+Ceph's components. These tasks cover a large subset of use cases and help to
+expose bugs not exposed by `make check`_ testing.
+
+.. _make check: ../tests-integration-testing-teuthology-intro/#make-check
+
+A job failure might be caused by one or more of the following reasons:
+
+* environment setup (`testing on varied
+  systems <https://github.com/ceph/ceph/tree/master/qa/distros/supported>`_):
+  testing compatibility with stable releases for supported versions.
+
+* permutation of config values: for instance, `qa/suites/rados/thrash
+  <https://github.com/ceph/ceph/tree/master/qa/suites/rados/thrash>`_ ensures
+  that we run thrashing tests against Ceph under stressful workloads so that we
+  can catch corner-case bugs. The final setup config yaml file used for testing
+  can be accessed at::
+
+  /a/<job-name>/<job-id>/orig.config.yaml
+
+More details about config.yaml can be found at `detailed test config`_
+
+Triaging the cause of failure
+------------------------------
+
+When a job fails, you will need to read its teuthology log in order to triage
+the cause of its failure. Use the job's name and id from pulpito to locate your
+failed job's teuthology log::
+
+   http://qa-proxy.ceph.com/<job-name>/<job-id>/teuthology.log
+
+Open the log file::
+
+   /a/<job-name>/<job-id>/teuthology.log
+
+For example:
+
+  .. prompt:: bash $ 
+
+     nano /a/teuthology-2021-01-06_07:01:02-rados-master-distro-basic-smithi/5759282/teuthology.log
+
+Every job failure is recorded in the teuthology log as a Traceback and is 
+added to the job summary.
+
+Find the ``Traceback`` keyword and search the call stack and the logs for
+issues that caused the failure. Usually the traceback will include the command
+that failed.
+
+.. note:: The teuthology logs are deleted from time to time. If you are unable
+          to access the link in this example, just use any other case from
+          http://pulpito.front.sepia.ceph.com/
+
+Reporting the Issue
+-------------------
+
+In short: first check to see if your job failure was caused by a known issue,
+and if it wasn't, raise a tracker ticket. 
+
+After you have triaged the cause of the failure and you have determined that it
+wasn't caused by the changes that you made to the code, this might indicate
+that you have encountered a known failure in the upstream branch (in the
+example we're considering in this section, the upstream branch is "octopus").
+If the failure was not caused by the changes you made to the code, go to
+https://tracker.ceph.com and look for tracker issues related to the failure by
+using keywords spotted in the failure under investigation.
+
+If you find a similar issue on https://tracker.ceph.com, leave a comment on
+that issue explaining the failure as you understand it and make sure to
+include a link to your recent test run. If you don't find a similar issue,
+create a new tracker ticket for this issue and explain the cause of your job's
+failure as thoroughly as you can. If you're not sure what caused the job's
+failure, ask one of the team members for help.
+
+Debugging an issue using interactive-on-error
+---------------------------------------------
+
+When you encounter a job failure during testing, you should attempt to
+reproduce it. This is where ``--interactive-on-error`` comes in. This
+section explains how to use ``interactive-on-error`` and what it does. 
+
+When you have verified that a job has failed, run the same job again in
+teuthology but add the `interactive-on-error`_ flag::
+
+    ideepika@teuthology:~/teuthology$ ./virtualenv/bin/teuthology -v --lock --block $<your-config-yaml> --interactive-on-error
+
+Use either `custom config.yaml`_ or the yaml file from the failed job. If
+you use the yaml file from the failed job, copy ``orig.config.yaml`` to
+your local directory::
+
+    ideepika@teuthology:~/teuthology$ cp /a/teuthology-2021-01-06_07:01:02-rados-master-distro-basic-smithi/5759282/orig.config.yaml test.yaml
+    ideepika@teuthology:~/teuthology$ ./virtualenv/bin/teuthology -v --lock --block test.yaml --interactive-on-error
+
+If a job fails when the ``interactive-on-error`` flag is used, teuthology
+will lock the machines required by ``config.yaml``. Teuthology will halt
+the testing machines and hold them in the state that they were in at the
+time of the job failure. You will be put into an interactive python
+session. From there, you can ssh into the system to investigate the cause
+of the job failure.  
+
+After you have investigated the failure, just terminate the session.
+Teuthology will then clean up the session and unlock the machines.
+
+Suggested Resources
+--------------------
+
+  * `Testing Ceph: Pains & Pleasures <https://www.youtube.com/watch?v=gj1OXrKdSrs>`_
+  * `Teuthology Training <https://www.youtube.com/playlist?list=PLrBUGiINAakNsOwHaIM27OBGKezQbUdM->`_
+  * `Intro to Teuthology <https://www.youtube.com/watch?v=WiEUzoS6Nc4>`_
+
+.. _Scheduling Test Run: ../tests-integration-testing-teuthology-workflow/#scheduling-test-run
+.. _detailed test config: https://docs.ceph.com/projects/teuthology/en/latest/detailed_test_config.html
+.. _teuthology archives: ../tests-integration-testing-teuthology-workflow/#teuthology-archives
+.. _qa/suites: https://github.com/ceph/ceph/tree/master/qa/suites
+.. _qa/tasks: https://github.com/ceph/ceph/tree/master/qa/tasks
+.. _interactive-on-error: https://docs.ceph.com/projects/teuthology/en/latest/detailed_test_config.html#troubleshooting
+.. _custom config.yaml: https://docs.ceph.com/projects/teuthology/en/latest/detailed_test_config.html#test-configuration
+.. _testing priority: ../tests-integration-testing-teuthology-intro/#testing-priority
+.. _thrash: https://github.com/ceph/ceph/tree/master/qa/suites/rados/thrash
diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst
new file mode 100644
index 000000000..3cbe51241
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst
@@ -0,0 +1,660 @@
+.. _tests-integration-testing-teuthology-intro:
+
+Testing - Integration Tests - Introduction
+==========================================
+
+Ceph has two types of tests: :ref:`make check <make-check>` tests and
+integration tests. When a test requires multiple machines, root access, or lasts
+for a long time (for example, to simulate a realistic Ceph workload), it is
+deemed to be an integration test. Integration tests are organized into "suites",
+which are defined in the `ceph/qa sub-directory`_ and run with the
+``teuthology-suite`` command.
+
+The ``teuthology-suite`` command is part of the `teuthology framework`_.
+In the sections that follow we attempt to provide a detailed introduction
+to that framework from the perspective of a beginning Ceph developer.
+
+Teuthology consumes packages
+----------------------------
+
+It may take some time to understand the significance of this fact, but it
+is `very` significant. It means that automated tests can be conducted on
+multiple platforms using the same packages (RPM, DEB) that can be
+installed on any machine running those platforms.
+
+Teuthology has a `list of platforms that it supports
+<https://github.com/ceph/ceph/tree/master/qa/distros/supported>`_ (as of
+September 2020 the list consisted of "RHEL/CentOS 8" and "Ubuntu 18.04"). It
+expects to be provided pre-built Ceph packages for these platforms.  Teuthology
+deploys these platforms on machines (bare-metal or cloud-provisioned), installs
+the packages on them, and deploys Ceph clusters on them - all as called for by
+the test.
+
+The Nightlies
+-------------
+
+A number of integration tests are run on a regular basis in the `Sepia
+lab`_ against the official Ceph repositories (on the ``master`` development
+branch and the stable branches). Traditionally, these tests are called "the
+nightlies" because the Ceph core developers used to live and work in
+the same time zone and from their perspective the tests were run overnight.
+
+The results of nightly test runs are published at http://pulpito.ceph.com/
+under the user ``teuthology``. The developer nick appears in URL of the the
+test results and in the first column of the Pulpito dashboard.  The results are
+also reported on the `ceph-qa mailing list <https://ceph.com/irc/>`_.
+
+Testing Priority
+----------------
+
+In brief: in the ``teuthology-suite`` command option ``-p <N>``, set the value of ``<N>`` to a number lower than 1000. An explanation of why follows.
+
+The ``teuthology-suite`` command includes an option ``-p <N>``. This option specifies the priority of the jobs submitted to the queue. The lower the value of ``N``, the higher the priority.
+
+The default value of ``N`` is ``1000``. This is the same priority value given to the nightly tests (the nightlies). Often, the volume of testing done during the nightly tests is so great that the full number of nightly tests do not get run during the time allotted for their run.
+
+Set the value of ``N`` lower than ``1000``, or your tests will not have priority over the nightly tests. This means that they might never run.
+
+Select your job's priority (the value of ``N``) in accordance with the following guidelines:
+
+.. list-table::
+   :widths: 30 30
+   :header-rows: 1
+
+   * - Priority
+     - Explanation
+   * - **N < 10**
+     - Use this if the sky is falling and some group of tests must be run ASAP.
+   * - **10 <= N < 50**
+     - Use this if your tests are urgent and blocking other important
+       development.
+   * - **50 <= N < 75**
+     - Use this if you are testing a particular feature/fix and running fewer
+       than about 25 jobs. This range is also used for urgent release testing.
+   * - **75 <= N < 100**
+     - Tech Leads regularly schedule integration tests with this priority to
+       verify pull requests against master.
+   * - **100 <= N < 150**
+     - This priority is used for QE validation of point releases.
+   * - **150 <= N < 200**
+     - Use this priority for 100 jobs or fewer that test a particular feature
+       or fix.  Results are available in about 24 hours.
+   * - **200 <= N < 1000**
+     - Use this priority for large test runs.  Results are available in about a
+       week.
+
+To see how many jobs the ``teuthology-suite`` command will trigger, use the
+``--dry-run`` flag. If you are happy with the number of jobs returned by the
+dry run, issue the ``teuthology-suite`` command again without ``--dry-run`` and
+with ``-p`` and an appropriate number as an argument. 
+
+To skip the priority check, use ``--force-priority``. Be considerate of the needs of other developers to run tests, and use ``--force-priority`` only in emergencies. 
+
+Suites Inventory
+----------------
+
+The ``suites`` directory of the `ceph/qa sub-directory`_ contains all the
+integration tests for all the Ceph components.
+
+.. list-table:: **Suites**
+
+  * - **Component**
+    - **Function**
+
+  * - `ceph-deploy <https://github.com/ceph/ceph/tree/master/qa/suites/ceph-deploy>`_
+    - install a Ceph cluster with ``ceph-deploy`` (`ceph-deploy man page`_)
+
+  * - `dummy <https://github.com/ceph/ceph/tree/master/qa/suites/dummy>`_
+    - get a machine, do nothing and return success (commonly used to verify
+      that the integration testing infrastructure works as expected)
+
+  * - `fs <https://github.com/ceph/ceph/tree/master/qa/suites/fs>`_
+    - test CephFS mounted using kernel and FUSE clients, also with multiple MDSs.
+
+  * - `krbd <https://github.com/ceph/ceph/tree/master/qa/suites/krbd>`_
+    - test the RBD kernel module
+
+  * - `powercycle <https://github.com/ceph/ceph/tree/master/qa/suites/powercycle>`_
+    - verify the Ceph cluster behaves when machines are powered off and on
+      again
+
+  * - `rados <https://github.com/ceph/ceph/tree/master/qa/suites/rados>`_
+    - run Ceph clusters including OSDs and MONs, under various conditions of
+      stress
+
+  * - `rbd <https://github.com/ceph/ceph/tree/master/qa/suites/rbd>`_
+    - run RBD tests using actual Ceph clusters, with and without qemu
+
+  * - `rgw <https://github.com/ceph/ceph/tree/master/qa/suites/rgw>`_
+    - run RGW tests using actual Ceph clusters
+
+  * - `smoke <https://github.com/ceph/ceph/tree/master/qa/suites/smoke>`_
+    - run tests that exercise the Ceph API with an actual Ceph cluster
+
+  * - `teuthology <https://github.com/ceph/ceph/tree/master/qa/suites/teuthology>`_ 
+    - verify that teuthology can run integration tests, with and without OpenStack
+
+  * - `upgrade <https://github.com/ceph/ceph/tree/master/qa/suites/upgrade>`_
+    - for various versions of Ceph, verify that upgrades can happen without disrupting an ongoing workload (`Upgrade Testing`_)
+
+teuthology-describe
+-------------------
+
+``teuthology-describe`` was added to the `teuthology framework`_ to facilitate
+documentation and better understanding of integration tests.
+
+Tests can be documented by embedding ``meta:`` annotations in the yaml files
+used to define the tests. The results can be seen in the `teuthology-describe
+usecases`_
+
+Since this is a new feature, many yaml files have yet to be annotated.
+Developers are encouraged to improve the coverage and the quality of the
+documentation. 
+
+How to run integration tests 
+----------------------------
+
+Typically, the `Sepia lab`_ is used to run integration tests. But as a new Ceph
+developer, you will probably not have access to the `Sepia lab`_.  You might
+however be able to run some integration tests in an environment separate from
+the `Sepia lab`_ . Ask members from the relevant team how to do this. 
+
+One way to run your own integration tests is to set up a teuthology cluster on
+bare metal. Setting up a teuthology cluster on bare metal is a complex task.
+Here are `some notes
+<https://docs.ceph.com/projects/teuthology/en/latest/LAB_SETUP.html>`_ to get
+you started if you decide that you are interested in undertaking the complex
+task of setting up a teuthology cluster on bare metal.
+
+Running integration tests on your code contributions and publishing the results
+allows reviewers to verify that changes to the code base do not cause
+regressions, and allows reviewers to analyze test failures when they occur.
+
+Every teuthology cluster, whether bare-metal or cloud-provisioned, has a
+so-called "teuthology machine" from which tests suites are triggered using the
+``teuthology-suite`` command.
+
+A detailed and up-to-date description of each `teuthology-suite`_ option is
+available by running the following command on the teuthology machine:
+
+.. prompt:: bash $
+
+   teuthology-suite --help
+
+.. _teuthology-suite: https://docs.ceph.com/projects/teuthology/en/latest/commands/teuthology-suite.html
+
+How integration tests are defined
+---------------------------------
+
+Integration tests are defined by yaml files found in the ``suites``
+subdirectory of the `ceph/qa sub-directory`_ and implemented by python
+code found in the ``tasks`` subdirectory. Some tests ("standalone tests")
+are defined in a single yaml file, while other tests are defined by a
+directory tree containing yaml files that are combined, at runtime, into a
+larger yaml file.
+
+
+.. _reading-standalone-test:
+
+Reading a standalone test
+-------------------------
+
+Let us first examine a standalone test, or "singleton".
+
+Here is a commented example using the integration test
+`rados/singleton/all/admin-socket.yaml
+<https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/admin-socket.yaml>`_
+
+.. code-block:: yaml
+
+      roles:
+      - - mon.a
+        - osd.0
+        - osd.1
+      tasks:
+      - install:
+      - ceph:
+      - admin_socket:
+          osd.0:
+            version:
+            git_version:
+            help:
+            config show:
+            config set filestore_dump_file /tmp/foo:
+            perf dump:
+            perf schema:
+
+The ``roles`` array determines the composition of the cluster (how
+many MONs, OSDs, etc.) on which this test is designed to run, as well
+as how these roles will be distributed over the machines in the
+testing cluster. In this case, there is only one element in the
+top-level array: therefore, only one machine is allocated to the
+test. The nested array declares that this machine shall run a MON with
+id ``a`` (that is the ``mon.a`` in the list of roles) and two OSDs
+(``osd.0`` and ``osd.1``).
+
+The body of the test is in the ``tasks`` array: each element is
+evaluated in order, causing the corresponding python file found in the
+``tasks`` subdirectory of the `teuthology repository`_ or
+`ceph/qa sub-directory`_ to be run. "Running" in this case means calling
+the ``task()`` function defined in that file.
+
+In this case, the `install
+<https://github.com/ceph/teuthology/blob/master/teuthology/task/install/__init__.py>`_
+task comes first. It installs the Ceph packages on each machine (as
+defined by the ``roles`` array). A full description of the ``install``
+task is `found in the python file
+<https://github.com/ceph/teuthology/blob/master/teuthology/task/install/__init__.py>`_
+(search for "def task").
+
+The ``ceph`` task, which is documented `here
+<https://github.com/ceph/ceph/blob/master/qa/tasks/ceph.py>`__ (again,
+search for "def task"), starts OSDs and MONs (and possibly MDSs as well)
+as required by the ``roles`` array. In this example, it will start one MON
+(``mon.a``) and two OSDs (``osd.0`` and ``osd.1``), all on the same
+machine. Control moves to the next task when the Ceph cluster reaches
+``HEALTH_OK`` state.
+
+The next task is ``admin_socket`` (`source code
+<https://github.com/ceph/ceph/blob/master/qa/tasks/admin_socket.py>`_).
+The parameter of the ``admin_socket`` task (and any other task) is a
+structure which is interpreted as documented in the task. In this example
+the parameter is a set of commands to be sent to the admin socket of
+``osd.0``. The task verifies that each of them returns on success (i.e.
+exit code zero).
+
+This test can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados/singleton/all/admin-socket.yaml fs/ext4.yaml
+
+Test descriptions
+-----------------
+
+Each test has a "test description", which is similar to a directory path,
+but not the same. In the case of a standalone test, like the one in
+`Reading a standalone test`_, the test description is identical to the
+relative path (starting from the ``suites/`` directory of the
+`ceph/qa sub-directory`_) of the yaml file defining the test.
+
+Much more commonly, tests are defined not by a single yaml file, but by a
+`directory tree of yaml files`. At runtime, the tree is walked and all yaml
+files (facets) are combined into larger yaml "programs" that define the
+tests. A full listing of the yaml defining the test is included at the
+beginning of every test log.
+
+In these cases, the description of each test consists of the
+subdirectory under `suites/
+<https://github.com/ceph/ceph/tree/master/qa/suites>`_ containing the
+yaml facets, followed by an expression in curly braces (``{}``) consisting of
+a list of yaml facets in order of concatenation. For instance the
+test description::
+
+  ceph-deploy/basic/{distros/centos_7.0.yaml tasks/ceph-deploy.yaml}
+
+signifies the concatenation of two files:
+
+* ceph-deploy/basic/distros/centos_7.0.yaml
+* ceph-deploy/basic/tasks/ceph-deploy.yaml
+
+How tests are built from directories
+------------------------------------
+
+As noted in the previous section, most tests are not defined in a single
+yaml file, but rather as a `combination` of files collected from a
+directory tree within the ``suites/`` subdirectory of the `ceph/qa sub-directory`_.
+
+The set of all tests defined by a given subdirectory of ``suites/`` is
+called an "integration test suite", or a "teuthology suite".
+
+Combination of yaml facets is controlled by special files (``%`` and
+``+``) that are placed within the directory tree and can be thought of as
+operators.  The ``%`` file is the "convolution" operator and ``+``
+signifies concatenation.
+
+Convolution operator
+^^^^^^^^^^^^^^^^^^^^
+
+The convolution operator, implemented as a (typically empty) file called ``%``,
+tells teuthology to construct a test matrix from yaml facets found in
+subdirectories below the directory containing the operator.
+
+For example, the `ceph-deploy suite
+<https://github.com/ceph/ceph/tree/master/qa/suites/ceph-deploy/>`_ is
+defined by the ``suites/ceph-deploy/`` tree, which consists of the files and
+subdirectories in the following structure
+
+.. code-block:: none
+
+  qa/suites/ceph-deploy
+  ├── %
+  ├── distros
+  │   ├── centos_latest.yaml
+  │   └── ubuntu_latest.yaml
+  └── tasks
+      ├── ceph-admin-commands.yaml
+      └── rbd_import_export.yaml
+
+This is interpreted as a 2x1 matrix consisting of two tests:
+
+1. ceph-deploy/basic/{distros/centos_7.0.yaml tasks/ceph-deploy.yaml}
+2. ceph-deploy/basic/{distros/ubuntu_16.04.yaml tasks/ceph-deploy.yaml}
+
+i.e. the concatenation of centos_7.0.yaml and ceph-deploy.yaml and
+the concatenation of ubuntu_16.04.yaml and ceph-deploy.yaml, respectively.
+In human terms, this means that the task found in ``ceph-deploy.yaml`` is
+intended to run on both CentOS 7.0 and Ubuntu 16.04.
+
+Without the file percent, the ``ceph-deploy`` tree would be interpreted as
+three standalone tests:
+
+* ceph-deploy/basic/distros/centos_7.0.yaml
+* ceph-deploy/basic/distros/ubuntu_16.04.yaml
+* ceph-deploy/basic/tasks/ceph-deploy.yaml
+
+(which would of course be wrong in this case).
+
+Referring to the `ceph/qa sub-directory`_, you will notice that the
+``centos_7.0.yaml`` and ``ubuntu_16.04.yaml`` files in the
+``suites/ceph-deploy/basic/distros/`` directory are implemented as symlinks.
+By using symlinks instead of copying, a single file can appear in multiple
+suites. This eases the maintenance of the test framework as a whole.
+
+All the tests generated from the ``suites/ceph-deploy/`` directory tree
+(also known as the "ceph-deploy suite") can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite ceph-deploy
+
+An individual test from the `ceph-deploy suite`_ can be run by adding the
+``--filter`` option
+
+.. prompt:: bash $
+
+   teuthology-suite \
+      --machine-type smithi \
+      --suite ceph-deploy/basic \
+      --filter 'ceph-deploy/basic/{distros/ubuntu_16.04.yaml tasks/ceph-deploy.yaml}'
+
+.. note:: To run a standalone test like the one in `Reading a standalone
+   test`_, ``--suite`` alone is sufficient. If you want to run a single
+   test from a suite that is defined as a directory tree, ``--suite`` must
+   be combined with ``--filter``. This is because the ``--suite`` option
+   understands POSIX relative paths only.
+
+Nested Subsets
+^^^^^^^^^^^^^^
+
+Suites can get quite large with the combinatorial explosion of yaml
+configurations. At the time of writing, the ``rados``` suite is more than
+100,000 jobs. For this reason, scheduling often uses the ``--subset`` option to
+only run a subset of the jobs (see also: :ref:`subset`). However, this applies
+only at the top-level of the suite being run (e.g. ``fs``). That may
+incidentally inflate the ratio of jobs for some larger sub-suites (like
+``fs:workload``) vs.  smaller but critical suites (like ``fs:volumes``).
+
+It is therefore attractive to automatically subset some sub-suites which are
+never run fully. This is done by providing an integer divisor for the ``%``
+convolution operator file instead of leaving it empty. That divisor
+automatically subsets the resulting matrix. For example, if the convolution
+file ``%`` contains ``2``, the matrix will be divided into two using the same
+logic as the ``--subset`` mechanism.
+
+Note the numerator is not specified as with the ``--subset`` option as there is
+no meaningful way to express this when there could be several layers of
+nesting.  Instead, a random subset is selected (1 of 2 in our example). The
+choice is based off the random seed (``--seed``) used for the scheduling.
+Remember that seed is saved in the results so that a ``--rerun`` of failed
+tests will still preserve the correct numerator (subset of subsets).
+
+You can disable nested subsets using the ``--no-nested-subset`` argument to
+``teuthology-suite``.
+
+Concatenation operator
+^^^^^^^^^^^^^^^^^^^^^^
+
+For even greater flexibility in sharing yaml files between suites, the
+special file plus (``+``) can be used to concatenate files within a
+directory. For instance, consider the `suites/rbd/thrash
+<https://github.com/ceph/ceph/tree/master/qa/suites/rbd/thrash>`_
+tree
+
+.. code-block:: none
+
+  qa/suites/rbd/thrash
+  ├── %
+  ├── clusters
+  │   ├── +
+  │   ├── fixed-2.yaml
+  │   └── openstack.yaml
+  └── workloads
+      ├── rbd_api_tests_copy_on_read.yaml
+      ├── rbd_api_tests.yaml
+      └── rbd_fsx_rate_limit.yaml
+
+This creates two tests:
+
+* rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests.yaml}
+
+Because the ``clusters/`` subdirectory contains the special file plus
+(``+``), all the other files in that subdirectory (``fixed-2.yaml`` and
+``openstack.yaml`` in this case) are concatenated together
+and treated as a single file. Without the special file plus, they would
+have been convolved with the files from the workloads directory to create
+a 2x2 matrix:
+
+* rbd/thrash/{clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/openstack.yaml workloads/rbd_api_tests.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml workloads/rbd_api_tests.yaml}
+
+The ``clusters/fixed-2.yaml`` file is shared among many suites to
+define the following ``roles``
+
+.. code-block:: yaml
+
+  roles:
+  - [mon.a, mon.c, osd.0, osd.1, osd.2, client.0]
+  - [mon.b, osd.3, osd.4, osd.5, client.1]
+
+The ``rbd/thrash`` suite as defined above, consisting of two tests,
+can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rbd/thrash
+
+A single test from the rbd/thrash suite can be run by adding the
+``--filter`` option
+
+.. prompt:: bash $
+
+   teuthology-suite \
+      --machine-type smithi \
+      --suite rbd/thrash \
+      --filter 'rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}'
+
+.. _upgrade-testing:
+
+Upgrade Testing
+^^^^^^^^^^^^^^^
+
+Using the upgrade suite we are able to verify that upgrades from earlier releases can complete
+successfully without disrupting any ongoing workload.
+Each Release branch upgrade directory includes 2-x upgrade testing.
+Meaning, we are able to test the upgrade from 2 preceding releases to the current one.
+The upgrade sequence is done in `parallel <https://github.com/ceph/teuthology/blob/main/teuthology/task/parallel.py>`_
+with other given workloads.
+
+For instance, the upgrade test directory from the Quincy release branch is as follows:
+
+.. code-block:: none
+
+  .
+  ├── octopus-x
+  └── pacific-x
+
+It is possible to test upgrades from Octopus (2-x) or from Pacific (1-x) to Quincy (x).
+A simple upgrade test consists the following order:
+
+.. code-block:: none
+
+  ├── 0-start.yaml
+  ├── 1-tasks.yaml
+  ├── upgrade-sequence.yaml
+  └── workload
+
+After starting the cluster with the older release we begin running the given ``workload``
+and the ``upgrade-sequnce`` in parallel.
+
+.. code-block:: yaml
+
+  - print: "**** done start parallel"
+  - parallel:
+      - workload
+      - upgrade-sequence
+  - print: "**** done end parallel"
+
+While the ``workload`` directory consists regular yaml files just as in any other suite,
+the ``upgrade-sequnce`` is resposible for running the upgrade and awaitng its completion:
+
+.. code-block:: yaml
+
+  - print: "**** done start upgrade, wait"
+  ...
+    mon.a:
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
+      - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done\
+  ...
+  - print: "**** done end upgrade, wait..."
+
+
+It is also possible to upgrade in stages while running workloads in between those:
+
+.. code-block:: none
+
+  ├── %
+  ├── 0-cluster
+  ├── 1-ceph-install
+  ├── 2-partial-upgrade
+  ├── 3-thrash
+  ├── 4-workload
+  ├── 5-finish-upgrade.yaml
+  ├── 6-quincy.yaml
+  └── 8-final-workload
+
+After starting a cluster we upgrade only 2/3 of the cluster (``2-partial-upgrade``).
+The next stage is running thrash tests and given workload tests. Later on, continuing to upgrade the
+rest of the cluster (``5-finish-upgrade.yaml``).
+The last stage is requiring the updated release (``ceph require-osd-release quincy``,
+``ceph osd set-require-min-compat-client quincy``) and running the ``final-workload``.
+
+Position Independent Linking
+----------------------------
+
+Under the ``qa/suites`` directory are ``.qa`` symbolic links in every
+directory. Each link is recursive by always linking to ``../.qa/``. The final
+terminating link is in the ``qa/`` directory itself as ``qa/.qa -> .``. This
+layout of symbolic links allows a suite to be easily copied or moved without
+breaking a number of symbolic links. For example::
+
+    qa/suites/fs/upgrade/nofs/centos_latest.yaml -> .qa/distros/supported/centos_latest.yaml
+
+If we copy the ``nofs`` suite somewhere else, add a parent directory above
+``nofs``, or move the ``centos_latest.yaml`` fragment into a sub-directory, the
+link will not break. Compare to::
+
+    qa/suites/fs/upgrade/nofs/centos_latest.yaml -> ../../../../distros/supported/centos_latest.yaml
+
+If the link is moved, it is very likely it will break because the number of
+parent directories to reach the ``distros`` directory may change.
+
+When adding new directories or suites, it is recommended to also remember
+adding ``.qa`` symbolic links. A trivial find command may do this for you:
+
+.. prompt:: bash $
+
+   find qa/suites/ -type d -execdir ln -sfT ../.qa/ {}/.qa \;
+
+
+Filtering tests by their description
+------------------------------------
+
+When a few jobs fail and need to be run again, the ``--filter`` option
+can be used to select tests with a matching description. For instance, if the
+``rados`` suite fails the `all/peer.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/peer.yaml>`_ test, the following will only
+run the tests that contain this file
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados --filter all/peer.yaml
+
+The ``--filter-out`` option does the opposite (it matches tests that do `not`
+contain a given string), and can be combined with the ``--filter`` option.
+
+Both ``--filter`` and ``--filter-out`` take a comma-separated list of strings
+(which means the comma character is implicitly forbidden in filenames found in
+the `ceph/qa sub-directory`_). For instance
+
+.. prompt:: bash $
+
+  teuthology-suite --machine-type smithi --suite rados --filter all/peer.yaml,all/rest-api.yaml
+
+will run tests that contain either
+`all/peer.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/peer.yaml>`_
+or
+`all/rest-api.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/rest-api.yaml>`_
+
+Each string is looked up anywhere in the test description and has to
+be an exact match: they are not regular expressions.
+
+
+.. _subset:
+
+Reducing the number of tests
+----------------------------
+
+The ``rados`` suite generates tens or even hundreds of thousands of tests out
+of a few hundred files. This happens because teuthology constructs test
+matrices from subdirectories wherever it encounters a file named ``%``. For
+instance, all tests in the `rados/basic suite
+<https://github.com/ceph/ceph/tree/master/qa/suites/rados/basic>`_ run with
+different messenger types: ``simple``, ``async`` and ``random``, because they
+are combined (via the special file ``%``) with the `msgr directory
+<https://github.com/ceph/ceph/tree/master/qa/suites/rados/basic/msgr>`_
+
+All integration tests are required to be run before a Ceph release is
+published. When merely verifying whether a contribution can be merged without
+risking a trivial regression, it is enough to run a subset. The ``--subset``
+option can be used to reduce the number of tests that are triggered. For
+instance
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados --subset 0/4000
+
+will run as few tests as possible. The tradeoff in this case is that
+not all combinations of test variations will together,
+but no matter how small a ratio is provided in the ``--subset``,
+teuthology will still ensure that all files in the suite are in at
+least one test. Understanding the actual logic that drives this
+requires reading the teuthology source code.
+
+Note: some suites are now using a **nested subset** feature that automatically
+applies a subset to a carefully chosen set of YAML configurations. You may
+disable this behavior (for some custom filtering, perhaps) using the
+``--no-nested-subset`` option.
+
+The ``--limit`` option only runs the first ``N`` tests in the suite:
+this is rarely useful, however, because there is no way to control which
+test will be first.
+
+.. _ceph/qa sub-directory: https://github.com/ceph/ceph/tree/master/qa
+.. _Sepia Lab: https://wiki.sepia.ceph.com/doku.php
+.. _teuthology repository: https://github.com/ceph/teuthology
+.. _teuthology framework: https://github.com/ceph/teuthology
+.. _teuthology-describe usecases: https://gist.github.com/jdurgin/09711d5923b583f60afc
+.. _ceph-deploy man page: ../../../../man/8/ceph-deploy
diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-kernel.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-kernel.rst
new file mode 100644
index 000000000..e7c20ee24
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-kernel.rst
@@ -0,0 +1,71 @@
+.. _tests-integration-testing-teuthology-kernel:
+
+Integration Tests for Kernel Development
+========================================
+
+
+.. _kernel-cephfs:
+
+CephFS
+------
+
+The ``fs`` suite runs various kernels as described by the `kernel YAML
+fragments`_. These are symbolically linked by other sub-suites under the ``fs``
+suite.
+
+The matrix of fragments allows for testing the following configurations:
+
+* The "stock" kernel on RHEL 8 (i.e. the kernel that ships with it).
+
+* The `testing branch`_ by the kernel development team which represents the
+  patches undergoing active testing. These patches may or may not be in the next
+  upstream kernel release and include a mix of CephFS or kRBD changes. For the
+  testing kernel, we test with whatever distributions are specified by the
+  sub-suite. For example, the ``fs:functional`` sub-suite uses a random selection
+  of the `supported random distros`_.
+
+
+
+
+Testing custom kernels
+----------------------
+
+If you have a kernel branch on `ceph-client.git`_ and have built it using
+shaman, then you can also test that easily by specifying an override for the
+kernel. This is done via a YAML fragment passed to the ``teuthology-suite``
+command:
+
+::
+
+    $ cat custom-kernel.yaml
+    overrides:
+      kernel:
+        branch: for-linus
+
+This specifies an override for the kernel branch specified in the suite's
+matrix. You can also specify an override as a tag or SHA1 for the ``kernel``
+task. When overriding the kernel, you should reduce the selection of jobs as
+the matrix will include a number of kernel configurations you won't care to
+test, as mentioned in the :ref:`kernel-cephfs` section; the override YAML will
+apply to all configurations of the kernel so it will result in duplicate tests.
+The command to run tests will look like:
+
+.. prompt:: bash $
+
+    teuthology-suite ... --suite fs --filter k-testing custom-kernel.yaml
+
+Where ``...`` indicates other typical options that are normally specified when
+running ``teuthology-suite``. The important filter ``--filter k-testing``
+will limit the selection of jobs to those using the ``testing`` branch of the
+kernel (see the `k-testing.yaml`_ file). So you'll only select jobs using the
+kernel client with the ``testing`` branch. Your custom YAML file,
+``custom-kernel.yaml``, will further override the ``testing`` branch to use
+whatever you specify.
+
+
+
+.. _kernel YAML fragments: https://github.com/ceph/ceph/tree/63f84c50e0851d456fc38b3330945c54162dd544/qa/cephfs/mount/kclient/overrides/distro
+.. _ceph-client.git: https://github.com/ceph/ceph-client/tree/testing
+.. _testing branch: https://github.com/ceph/ceph-client/tree/testing
+.. _supported random distros: https://github.com/ceph/ceph/blob/63f84c50e0851d456fc38b3330945c54162dd544/qa/suites/fs/functional/distro
+.. _k-testing.yaml: https://github.com/ceph/ceph/blob/63f84c50e0851d456fc38b3330945c54162dd544/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml
diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
new file mode 100644
index 000000000..64b006c57
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
@@ -0,0 +1,293 @@
+.. _tests-integration-testing-teuthology-workflow:
+
+Integration Tests using Teuthology Workflow
+===========================================
+
+Scheduling Test Run
+-------------------
+
+Getting binaries
+****************
+
+Ceph binaries must be built for your branch before you can use teuthology to run integration tests on them. Follow these steps to build the Ceph binaries:
+
+#. Push the branch to the `ceph-ci`_ repository. This triggers the process of
+   building the binaries on the Jenkins CI.
+
+#. To ensure that the build process has been initiated, confirm that the branch
+   name has appeared in the list of "Latest Builds Available" at `Shaman`_.
+   Soon after you start the build process, the testing infrastructure adds
+   other, similarly-named builds to the list of "Latest Builds Available".
+   The names of these new builds will contain the names of various Linux
+   distributions of Linux and will be used to test your build against those
+   Linux distributions. 
+
+#. Wait for the packages to be built and uploaded to `Chacra`_, and wait for
+   the repositories offering the packages to be created. The entries for the
+   branch names in the list of "Latest Builds Available" on `Shaman`_ will turn
+   green to indicate that the packages have been uploaded to `Chacra`_ and to
+   indicate that their repositories have been created.  Wait until each entry
+   is coloured green. This usually takes between two and three hours depending
+   on the availability of the machines.
+   
+   The Chacra URL for a particular build can be queried from `the Chacra site`_.
+
+.. note:: The branch to be pushed on ceph-ci can be any branch. The branch does
+   not have to be a PR branch.
+
+.. note:: If you intend to push master or any other standard branch, check
+   `Shaman`_ beforehand since it might already have completed builds for it.
+
+.. _the Chacra site: https://shaman.ceph.com/api/search/?status=ready&project=ceph
+
+
+Triggering Tests
+****************
+
+After you have built Ceph binaries for your branch, you can run tests using
+teuthology. This procedure explains how to run tests using teuthology.
+
+#. Log in to the teuthology machine:
+
+   .. prompt:: bash $
+
+       ssh <username>@teuthology.front.sepia.ceph.com
+
+   This requires Sepia lab access. To request access to the Sepia lab, see:
+   https://ceph.github.io/sepia/adding_users/
+
+#. Run the ``teuthology-suite`` command:
+
+   .. prompt:: bash $
+
+        teuthology-suite -v \
+        -m smithi \
+        -c wip-devname-feature-x \
+        -s fs \
+        -p 110 \
+        --filter "cephfs-shell" \
+        -e foo@gmail.com \
+
+   The options in the above command are defined here: 
+
+      =============  =========================================================
+         Option        Meaning
+      =============  =========================================================
+        -v            verbose
+        -m            machine name
+        -c            the name of the branch that was pushed on ceph-ci
+        -s            test-suite name
+        -p            the higher the number, the lower the priority of 
+                      the job
+        --filter      filter tests in a given suite. The argument
+                      passed to this filter specifies which test you 
+                      want to run
+        -e <email>    When tests finish or time out, send an email to the
+                      specified address. Can also be specified in 
+                      ~/.teuthology.yaml as 'results_email'
+      =============  =========================================================
+
+   .. note:: The priority number present in the command above is a placeholder. 
+      Do not use it in your own tests. See `Testing Priority`_ for information 
+      about recommended values.
+
+   .. note:: Do not issue a command without a priority number. The default 
+      value is 1000, a value so large that your job is unlikely ever to run.
+
+   Run ``teuthology-suite --help`` to read descriptions of these and other 
+   available options.
+
+#. Wait for the tests to run. ``teuthology-suite`` prints a link to
+   `Pulpito`_ where the test results can be viewed.
+
+
+
+Other frequently used/useful options are ``-d`` (or ``--distro``),
+``--distroversion``, ``--filter-out``, ``--timeout``, ``flavor``, ``-rerun``,
+``-l`` (for limiting number of jobs) , ``-N`` (for how many times the job will
+run), and ``--subset`` (used to reduce the number of tests that are triggered). Run
+``teuthology-suite --help`` to read descriptions of these and other options.
+
+.. _teuthology_testing_qa_changes:
+
+Testing QA changes (without re-building binaries)
+*************************************************
+
+If you are making changes only in the ``qa/`` directory, you do not have to
+rebuild the binaries before you re-run tests. If you make changes only in
+``qa/``, you can use the binaries built for the ceph-ci branch to re-run tests.
+You just have to make sure to tell the ``teuthology-suite`` command to use a
+separate branch for running the tests.
+
+If you made changes only in ``qa/``
+(https://github.com/ceph/ceph/tree/master/qa), you do not need to rebuild the
+binaries. You can use existing binaries that are built periodically for master and other stable branches and run your test changes against them.
+Your branch with the qa changes can be tested by passing two extra arguments to the ``teuthology-suite`` command: (1) ``--suite-repo``, specifying your ceph repo, and (2) ``--suite-branch``, specifying your branch name. 
+
+For example, if you want to make changes in ``qa/`` after testing ``branch-x``
+(for which the ceph-ci branch is ``wip-username-branch-x``), run the following
+command
+
+.. prompt:: bash $
+
+   teuthology-suite -v \
+    -m smithi \
+    -c wip-username-branch-x \
+    -s fs \
+    -p 50 \
+    --filter cephfs-shell
+
+Then make modifications locally, update the PR branch, and trigger tests from
+your PR branch as follows:
+
+.. prompt:: bash $
+
+   teuthology-suite -v \
+    -m smithi \
+    -c wip-username-branch-x \
+    -s fs -p 50 \
+    --filter cephfs-shell \
+    --suite-repo https://github.com/$username/ceph \
+    --suite-branch branch-x
+
+You can verify that the tests were run using this branch by looking at the
+values for the keys ``suite_branch``, ``suite_repo`` and ``suite_sha1`` in the
+job config printed at the beginning of the teuthology job.
+
+.. note:: If you are making changes that are not in the ``qa/`` directory, 
+          you must follow the standard process of triggering builds, waiting 
+          for the builds to finish, then triggering tests and waiting for 
+          the test results. 
+
+About Suites and Filters
+************************
+
+See `Suites Inventory`_ for a list of available suites of integration tests.
+Each directory under ``qa/suites`` in the Ceph repository is an integration
+test suite, and arguments appropriate to follow ``-s`` can be found there.
+
+Keywords for filtering tests can be found in
+``qa/suites/<suite-name>/<subsuite-name>/tasks`` and can be used as arguments
+for ``--filter``. Each YAML file in that directory can trigger tests; using the
+name of the file without its filename extension as an argument to the
+``--filter`` triggers those tests. 
+
+For example, in the command above in the :ref:`Testing QA Changes
+<teuthology_testing_qa_changes>` section, ``cephfs-shell`` is specified. 
+This works because there is a file named ``cephfs-shell.yaml`` in
+``qa/suites/fs/basic_functional/tasks/``.
+
+If the filename doesn't suggest what kind of tests it triggers, search the
+contents of the file for the ``modules`` attribute. For ``cephfs-shell.yaml``
+the ``modules`` attribute is ``tasks.cephfs.test_cephfs_shell``. This means
+that it triggers all tests in ``qa/tasks/cephfs/test_cephfs_shell.py``.
+
+Viewing Test Results
+---------------------
+
+Pulpito Dashboard
+*****************
+
+After the teuthology job is scheduled, the status and results of the test run 
+can be checked at https://pulpito.ceph.com/.
+
+Teuthology Archives
+*******************
+
+After the tests have finished running, the log for the job can be obtained by
+clicking on the job ID at the Pulpito page associated with your tests. It's
+more convenient to download the log and then view it rather than viewing it in
+an internet browser since these logs can easily be up to 1 GB in size. It is
+easier to ssh into the teuthology machine (``teuthology.front.sepia.ceph.com``)
+and access the following path::
+
+    /ceph/teuthology-archive/<test-id>/<job-id>/teuthology.log
+
+For example: for the above test ID, the path is::
+
+   /ceph/teuthology-archive/teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi/4588482/teuthology.log
+
+This method can be used to view the log more quickly than would be possible through a browser.
+
+.. note:: To access archives more conveniently, ``/a/`` has been symbolically
+   linked to ``/ceph/teuthology-archive/``. For instance, to access the previous
+   example, we can use something like::
+
+   /a/teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi/4588482/teuthology.log
+
+Killing Tests
+-------------
+``teuthology-kill`` can be used to kill jobs that have been running
+unexpectedly for several hours, or when developers want to terminate tests
+before they complete.
+
+Here is the command that terminates jobs:
+
+.. prompt:: bash $
+
+   teuthology-kill -r teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi
+
+Let's call the argument passed to ``-r`` as test ID. It can be found
+easily in the link to the Pulpito page for the tests you triggered. For
+example, for the above test ID, the link is - http://pulpito.front.sepia.ceph.com/teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi/
+
+Re-running Tests
+----------------
+
+The ``teuthology-suite`` command has a ``-r`` (or ``--rerun``) option, which
+allows you to re-run tests. This is handy when your tests have failed or end
+up dead. The ``--rerun`` option takes the name of a teuthology run as an
+argument. Option ``-R`` (or ``--rerun-statuses``) can be passed along with
+``-r`` to choose which kind of tests should be picked from the run. For
+example, you can re-run only those tests from previous run which had ended up
+as dead. Following is a practical example:
+
+.. prompt:: bash $ 
+
+   teuthology-suite -v \
+    -m smithi \
+    -c wip-rishabh-fs-test_cephfs_shell-fix \
+    -p 50 \
+    --r teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi \
+    -R fail,dead,queued \
+    -e $CEPH_QA_MAIL
+
+Following's the definition of new options introduced in this section:
+
+      =======================  ===============================================
+         Option                     Meaning
+      =======================  ===============================================
+        -r, --rerun             Attempt to reschedule a run, selecting only
+                                those jobs whose status are mentioned by
+                                --rerun-status.
+        -R, --rerun-statuses    A comma-separated list of statuses to be used
+                                with --rerun. Supported statuses: 'dead',
+                                'fail', 'pass', 'queued', 'running' and
+                                'waiting'. Default value: 'fail,dead'
+      =======================  ===============================================
+
+Naming the ceph-ci branch
+-------------------------
+Prepend your branch with your name before you push it to ceph-ci. For example,
+a branch named ``feature-x`` should be named ``wip-$yourname-feature-x``, where
+``$yourname`` is replaced with your name. Identifying your branch with your
+name makes your branch easily findable on Shaman and Pulpito.
+
+If you are using one of the stable branches (`quincy`, `pacific`, etc.), include
+the name of that stable branch in your ceph-ci branch name.
+For example, the ``feature-x`` PR branch should be named 
+``wip-feature-x-nautilus``. *This is not just a convention. This ensures that your branch is built in the correct environment.*
+
+Delete the branch from ceph-ci when you no longer need it. If you are
+logged in to GitHub, all your branches on ceph-ci can be found here:
+https://github.com/ceph/ceph-ci/branches.
+
+.. _ceph-ci: https://github.com/ceph/ceph-ci
+.. _Chacra: https://github.com/ceph/chacra/blob/master/README.rst
+.. _Pulpito: http://pulpito.front.sepia.ceph.com/
+.. _Running Your First Test: ../../running-tests-locally/#running-your-first-test
+.. _Shaman: https://shaman.ceph.com/builds/ceph/
+.. _Suites Inventory: ../tests-integration-testing-teuthology-intro/#suites-inventory
+.. _Testing Priority: ../tests-integration-testing-teuthology-intro/#testing-priority
+.. _Triggering Tests: ../tests-integration-testing-teuthology-workflow/#triggering-tests
+.. _tests-sentry-developers-guide: ../tests-sentry-developers-guide/
diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-sentry-developers-guide.rst b/doc/dev/developer_guide/testing_integration_tests/tests-sentry-developers-guide.rst
new file mode 100644
index 000000000..94dfae39a
--- /dev/null
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-sentry-developers-guide.rst
@@ -0,0 +1,6 @@
+.. _tests-sentry-developers-guide:
+
+Sentry Notes
+============
+
+To be updated. Feel free to contribute.
diff --git a/doc/dev/developer_guide/tests-unit-tests.rst b/doc/dev/developer_guide/tests-unit-tests.rst
new file mode 100644
index 000000000..72d724d98
--- /dev/null
+++ b/doc/dev/developer_guide/tests-unit-tests.rst
@@ -0,0 +1,177 @@
+Testing - unit tests
+====================
+
+The Ceph GitHub repository has two types of tests: unit tests (also called
+``make check`` tests) and integration tests. Strictly speaking, the
+``make check`` tests are not "unit tests", but rather tests that can be run
+easily on a single build machine after compiling Ceph from source, whereas
+integration tests require package installation and multi-machine clusters to
+run.
+
+.. _make-check:
+
+What does "make check" mean?
+----------------------------
+
+After compiling Ceph, the code can be run through a battery of tests. For
+historical reasons, this is often referred to as ``make check`` even though
+the actual command used to run the tests is now ``ctest``. To be included in
+this group of tests, a test must:
+
+* bind ports that do not conflict with other tests
+* not require root access
+* not require more than one machine to run
+* complete within a few minutes
+
+For the sake of simplicity, this class of tests is referred to as "make
+check tests" or "unit tests". This is meant to distinguish these tests from
+the more complex "integration tests" that are run via the `teuthology
+framework`_.
+
+While it is possible to run ``ctest`` directly, it can be tricky to correctly
+set up your environment for it. Fortunately, there is a script that makes it
+easy to run the unit tests on your code. This script can be run from the
+top-level directory of the Ceph source tree by invoking:
+
+  .. prompt:: bash $
+
+     ./run-make-check.sh
+
+You will need a minimum of 8GB of RAM and 32GB of free drive space for this
+command to complete successfully on x86_64 architectures; other architectures
+may have different requirements. Depending on your hardware, it can take from
+twenty minutes to three hours to complete.
+
+
+How unit tests are declared
+---------------------------
+
+Unit tests are declared in the ``CMakeLists.txt`` file, which is found in the
+``./src`` directory. The ``add_ceph_test`` and ``add_ceph_unittest`` CMake
+functions are used to declare unit tests.  ``add_ceph_test`` and
+``add_ceph_unittest`` are themselves defined in
+``./cmake/modules/AddCephTest.cmake``. 
+
+Some unit tests are scripts and other unit tests are binaries that are
+compiled during the build process.  
+
+* ``add_ceph_test`` function - used to declare unit test scripts 
+* ``add_ceph_unittest`` function - used for unit test binaries
+
+Unit testing of CLI tools
+-------------------------
+Some of the CLI tools are tested using special files ending with the extension
+``.t`` and stored under ``./src/test/cli``. These tests are run using a tool
+called `cram`_ via a shell script called ``./src/test/run-cli-tests``.
+`cram`_ tests that are not suitable for ``make check`` can also be run by
+teuthology using the `cram task`_.
+
+.. _`cram`: https://bitheap.org/cram/
+.. _`cram task`: https://github.com/ceph/ceph/blob/master/qa/tasks/cram.py
+
+Tox-based testing of Python modules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Some of the Python modules in Ceph use `tox <https://tox.readthedocs.io/en/latest/>`_ 
+to run their unit tests.
+
+Most of these Python modules can be found in the directory ``./src/pybind/``.
+
+Currently (December 2020) the following modules use **tox**:
+
+* Cephadm (``./src/cephadm/tox.ini``)
+* Ceph Manager Python API (``./src/pybind/mgr``)
+
+  * ``./src/pybind/mgr/tox.ini``
+    
+  * ``./src/pybind/mgr/dashboard/tox.ini``
+
+  * ``./src/pybind/tox.ini``
+
+* Dashboard (``./src/pybind/mgr/dashboard``)
+* Python common (``./src/python-common/tox.ini``)
+* CephFS (``./src/tools/cephfs/tox.ini``)
+* ceph-volume
+
+  * ``./src/ceph-volume/tox.ini``
+
+  * ``./src/ceph-volume/plugin/zfs/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini``
+
+Configuring Tox environments and tasks 
+""""""""""""""""""""""""""""""""""""""
+Most tox configurations support multiple environments and tasks. 
+
+The list of environments and tasks that are supported is in the ``tox.ini``
+file, under ``envlist``. For example, here are the first three lines of
+``./src/cephadm/tox.ini``::
+
+   [tox]
+   envlist = py3, mypy
+   skipsdist=true
+
+In this example, the ``Python 3`` and ``mypy`` environments are specified.
+
+The list of environments can be retrieved with the following command:
+
+  .. prompt:: bash $
+
+     tox --list
+
+Or:
+
+  .. prompt:: bash $
+
+     tox -l
+
+Running Tox
+"""""""""""
+To run **tox**, just execute ``tox`` in the directory containing
+``tox.ini``.  If you do not specify any environments (for example, ``-e
+$env1,$env2``), then ``tox`` will run all environments. Jenkins will run
+``tox`` by executing ``./src/script/run_tox.sh``.
+
+Here are some examples from Ceph Dashboard that show how to specify different
+environments and run options::
+
+  ## Run Python 2+3 tests+lint commands:
+  $ tox -e py27,py3,lint,check
+
+  ## Run Python 3 tests+lint commands:
+  $ tox -e py3,lint,check
+
+  ## To run it as Jenkins would:  
+  $ ../../../script/run_tox.sh --tox-env py3,lint,check
+
+Manager core unit tests
+"""""""""""""""""""""""
+
+Currently only doctests_ inside ``mgr_util.py`` are run.
+
+To add more files to be tested inside the core of the manager, open the
+``tox.ini`` file and add the files to be tested  at the end of the line that
+includes ``mgr_util.py``.
+
+.. _doctests: https://docs.python.org/3/library/doctest.html
+
+Unit test caveats
+-----------------
+
+#. Unlike the various Ceph daemons and ``ceph-fuse``, the unit tests are
+   linked against the default memory allocator (glibc) unless they are
+   explicitly linked against something else. This enables tools such as
+   **valgrind** to be used in the tests.
+
+#. Google Test unit testing library hides the client output from the shell.
+   In order to debug the client after setting the desired debug level
+   (e.g ``ceph config set client debug_rbd 20``), the debug log file can
+   be found at ``build/out/client.admin.<pid>.log``.
+   This can also be handy when examining teuthology failed unit test
+   jobs, the job's debug level can be set at the relevant yaml file.
+
+.. _make check:
+.. _teuthology framework: https://github.com/ceph/teuthology
diff --git a/doc/dev/development-workflow.rst b/doc/dev/development-workflow.rst
new file mode 100644
index 000000000..dfcab929d
--- /dev/null
+++ b/doc/dev/development-workflow.rst
@@ -0,0 +1,248 @@
+=====================
+Development workflows
+=====================
+
+This page explains the workflows a developer is expected to follow to
+implement the goals that are part of the Ceph release cycle. It does not
+go into technical details and is designed to provide a high level view
+instead. Each chapter is about a given goal such as ``Merging bug
+fixes or features`` or ``Publishing point releases and backporting``.
+
+A key aspect of all workflows is that none of them blocks another. For
+instance, a bug fix can be backported and merged to a stable branch
+while the next point release is being published. For that specific
+example to work, a branch should be created to avoid any
+interference. In practice it is not necessary for Ceph because:
+
+* there are few people involved
+* the frequency of backports is not too high
+* the reviewers, who know a release is being published, are unlikely
+  to merge anything that may cause issues
+
+This ad-hoc approach implies the workflows are changed on a regular
+basis to adapt. For instance, ``quality engineers`` were not involved
+in the workflow to publish ``dumpling`` point releases. The number of
+commits being backported to ``firefly`` made it impractical for developers
+tasked to write code or fix bugs to also run and verify the full suite
+of integration tests. Inserting ``quality engineers`` makes it
+possible for someone to participate in the workflow by analyzing test
+results.
+
+The workflows are not enforced when they impose an overhead that does
+not make sense. For instance, if the release notes for a point release
+were not written prior to checking all integration tests, they can be
+committed to the stable branch and the result sent for publication
+without going through another run of integration tests.
+
+Release Cycle
+=============
+
+::
+
+    Ceph              hammer                             infernalis
+    Developer          CDS                                  CDS 
+    Summit              |                                    |
+                        |                                    |
+    development         |                                    |
+    release             |  v0.88  v0.89  v0.90   ...         |  v9.0.0
+                   --v--^----^--v---^------^--v-     ---v----^----^---  2015       
+                     |          |             |         |
+    stable         giant        |             |      hammer
+    release        v0.87        |             |      v0.94
+                                |             |          
+    point                    firefly       dumpling
+    release                  v0.80.8       v0.67.12
+
+
+Four times a year, the development roadmap is discussed online during
+the `Ceph Developer Summit <http://tracker.ceph.com/projects/ceph/wiki/Planning#Ceph-Developer-Summit>`_. A
+new stable release (hammer, infernalis, jewel ...) is published at the same
+frequency.  Every other release (firefly, hammer, jewel...) is a `Long Term
+Stable (LTS) <../../releases>`_.  See `Understanding the release cycle
+<../../releases#understanding-the-release-cycle>`_ for more information.
+
+Merging bug fixes or features
+=============================
+
+The development branch is ``master`` and the workflow followed by all
+developers can be summarized as follows:
+
+* The developer prepares a series of commits
+* The developer submits the series of commits via a pull request
+* A reviewer is assigned the pull request
+* When the pull request looks good to the reviewer, it is merged into
+  an integration branch by the tester
+* After a successful run of integration tests, the pull request is
+  merged by the tester
+
+The ``developer`` is the author of a series of commits. The
+``reviewer`` is responsible for providing feedback to the developer on
+a regular basis and the developer is invited to ping the reviewer if
+nothing happened after a week. After the ``reviewer`` is satisfied
+with the pull request, (s)he passes it to the ``tester``. The
+``tester`` is responsible for running teuthology integration tests on
+the pull request. If nothing happens within a month the ``reviewer`` is
+invited to ping the ``tester``.
+
+Resolving bug reports and implementing features
+===============================================
+
+All bug reports and feature requests are in the `issue tracker
+<http://tracker.ceph.com>`_ and the workflow can be summarized as
+follows:
+
+* The reporter creates the issue with priority ``Normal``
+* A developer may pick the issue right away
+* During a bi-weekly bug scrub, the team goes over all new issue and
+  assign them a priority
+* The bugs with higher priority are worked on first
+
+Each ``team`` is responsible for a project, managed by :ref:`leads <governance>`.
+
+The ``developer`` assigned to an issue is responsible for it. The
+status of an open issue can be:
+
+* ``New``: it is unclear if the issue needs work.
+* ``Verified``: the bug can be reproduced or showed up multiple times
+* ``In Progress``: the developer is working on it this week
+* ``Pending Backport``: the fix needs to be backported to the stable
+  releases listed in the backport field
+
+For each ``Pending Backport`` issue, there exists at least one issue
+in the ``Backport`` tracker to record the work done to cherry pick the
+necessary commits from the master branch to the target stable branch.
+See `the backporter manual
+<http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO>`_ for more
+information.
+
+Running and interpreting teuthology integration tests
+=====================================================
+
+The :doc:`/dev/sepia` runs `teuthology
+<https://github.com/ceph/teuthology/>`_ integration tests `on a regular basis <http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_monitor_the_automated_tests_AKA_nightlies#Automated-tests-AKA-nightlies>`_ and the
+results are posted on `pulpito <http://pulpito.ceph.com/>`_ and the
+`ceph-qa mailing list <https://ceph.com/irc/>`_.
+
+* The job failures are `analyzed by quality engineers and developers
+  <http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_monitor_the_automated_tests_AKA_nightlies#List-of-suites-and-watchers>`_
+* If the cause is environmental (e.g. network connectivity), an issue
+  is created in the `sepia lab project
+  <http://tracker.ceph.com/projects/lab/issues/new>`_
+* If the bug is known, a pulpito URL to the failed job is added to the issue
+* If the bug is new, an issue is created
+
+The ``quality engineer`` is either a developer or a member of the QE
+team. There is at least one integration test suite per project:
+
+* `rgw <https://github.com/ceph/ceph/tree/master/qa/suites/rgw>`_ suite
+* `CephFS <https://github.com/ceph/ceph/tree/master/qa/suites/fs>`_ suite
+* `rados <https://github.com/ceph/ceph/tree/master/qa/suites/rados>`_ suite
+* `rbd <https://github.com/ceph/ceph/tree/master/qa/suites/rbd>`_ suite
+
+and many others such as
+
+* `upgrade <https://github.com/ceph/ceph/tree/master/qa/suites/upgrade>`_ suites
+* `power-cyle <https://github.com/ceph/ceph/tree/master/qa/suites/powercycle>`_ suite
+* ...
+
+Preparing a new release
+=======================
+
+A release is prepared in a dedicated branch, different from the
+``master`` branch.
+
+* For a stable releases it is the branch matching the release code
+  name (dumpling, firefly, etc.)
+* For a development release it is the ``next`` branch
+
+The workflow expected of all developers to stabilize the release
+candidate is the same as the normal development workflow with the
+following differences:
+
+* The pull requests must target the stable branch or next instead of
+  master
+* The reviewer rejects pull requests that are not bug fixes
+* The ``Backport`` issues matching a teuthology test failure and set
+  with priority ``Urgent`` must be fixed before the release
+
+Cutting a new stable release
+============================
+
+A new stable release can be cut when:
+
+* all ``Backport`` issues with priority ``Urgent`` are fixed
+* integration and upgrade tests run successfully
+
+Publishing a new stable release implies a risk of regression or
+discovering new bugs during the upgrade, no matter how carefully it is
+tested. The decision to cut a release must take this into account: it
+may not be wise to publish a stable release that only fixes a few
+minor bugs. For instance if only one commit has been backported to a
+stable release that is not a LTS, it is better to wait until there are
+more.
+
+When a stable release is to be retired, it may be safer to
+recommend an upgrade to the next LTS release instead of
+proposing a new point release to fix a problem. For instance, the
+``dumpling`` v0.67.11 release has bugs related to backfilling which have
+been fixed in ``firefly`` v0.80.x. A backport fixing these backfilling
+bugs has been tested in the draft point release ``dumpling`` v0.67.12 but
+they are large enough to introduce a risk of regression. As ``dumpling``
+is to be retired, users suffering from this bug can
+upgrade to ``firefly`` to fix it. Unless users manifest themselves and ask
+for ``dumpling`` v0.67.12, this draft release may never be published.
+
+* The ``Ceph lead`` decides a new stable release must be published
+* The ``release master`` gets approval from all leads
+* The ``release master`` writes and commits the release notes
+* The ``release master`` informs the ``quality engineer`` that the
+  branch is ready for testing
+* The ``quality engineer`` runs additional integration tests
+* If the ``quality engineer`` discovers new bugs that require an
+  ``Urgent Backport``, the release goes back to being prepared, it
+  was not ready after all
+* The ``quality engineer`` informs the ``publisher`` that the branch
+  is ready for release
+* The ``publisher`` `creates the packages and sets the release tag
+  <../release-process>`_
+
+The person responsible for each role is:
+
+* Sage Weil is the ``Ceph lead``
+* Sage Weil is the ``release master`` for major stable releases
+  (``firefly`` 0.80, ``hammer`` 0.94 etc.)
+* Loic Dachary is the ``release master`` for stable point releases
+  (``firefly`` 0.80.10, ``hammer`` 0.94.1 etc.)
+* Yuri Weinstein is the ``quality engineer``
+* Alfredo Deza is the ``publisher``
+
+Cutting a new development release
+=================================
+
+The publication workflow of a development release is the same as
+preparing a new release and cutting it, with the following
+differences:
+
+* The ``next`` branch is reset to the tip of ``master`` after
+  publication
+* The ``quality engineer`` is not required to run additional tests,
+  the ``release master`` directly informs the ``publisher`` that the
+  release is ready to be published.
+
+Publishing point releases and backporting
+=========================================
+
+The publication workflow of the point releases is the same as
+preparing a new release and cutting it, with the following
+differences:
+
+* The ``backport`` field of each issue contains the code name of the
+  stable release
+* There is exactly one issue in the ``Backport`` tracker for each
+  stable release to which the issue is backported
+* All commits are cherry-picked with ``git cherry-pick -x`` to
+  reference the original commit
+
+See `the backporter manual
+<http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO>`_ for more
+information.
diff --git a/doc/dev/documenting.rst b/doc/dev/documenting.rst
new file mode 100644
index 000000000..1b9468a50
--- /dev/null
+++ b/doc/dev/documenting.rst
@@ -0,0 +1,146 @@
+==================
+ Documenting Ceph
+==================
+
+User documentation
+==================
+
+The documentation on docs.ceph.com is generated from the reStructuredText
+sources in ``/doc/`` in the Ceph git repository.
+
+Please make sure that your changes are written in a way that is intended
+for end users of the software, unless you are making additions in
+``/doc/dev/``, which is the section for developers.
+
+All pull requests that modify user-facing functionality must
+include corresponding updates to documentation: see 
+`Submitting Patches`_ for more detail.
+
+Check your .rst syntax is working as expected by using the "View"
+button in the github user interface when looking at a diff on
+an .rst file, or build the docs locally using the ``admin/build-doc``
+script.
+
+For more information about the Ceph documentation, see 
+:doc:`/start/documenting-ceph`.
+
+Code Documentation
+==================
+
+C and C++ can be documented with Doxygen_, using the subset of Doxygen
+markup supported by Breathe_.
+
+.. _Doxygen: http://www.doxygen.nl/
+.. _Breathe: https://github.com/michaeljones/breathe
+
+The general format for function documentation is
+
+.. code-block:: c
+
+  /**
+   * Short description
+   *
+   * Detailed description when necessary
+   *
+   * preconditions, postconditions, warnings, bugs or other notes
+   *
+   * parameter reference
+   * return value (if non-void)
+   */
+
+This should be in the header where the function is declared, and
+functions should be grouped into logical categories. The `librados C
+API`_ provides a complete example. It is pulled into Sphinx by
+`librados.rst`_, which is rendered at :doc:`/rados/api/librados`.
+
+To generate the doxygen documentation in HTML format use:
+
+::
+
+   # cmake --build . --target doxygen
+
+HTML output will be under: ``build-doc/doxygen/html`` 
+
+.. _`librados C API`: https://github.com/ceph/ceph/blob/master/src/include/rados/librados.h
+.. _`librados.rst`: https://github.com/ceph/ceph/raw/master/doc/rados/api/librados.rst
+
+Drawing diagrams
+================
+
+Graphviz
+--------
+
+You can use Graphviz_, as explained in the `Graphviz extension documentation`_.
+
+.. _Graphviz: http://graphviz.org/
+.. _`Graphviz extension documentation`: https://www.sphinx-doc.org/en/master/usage/extensions/graphviz.html
+
+.. graphviz::
+
+   digraph "example" {
+     foo -> bar;
+     bar -> baz;
+     bar -> th
+   }
+
+Most of the time, you'll want to put the actual DOT source in a
+separate file, like this::
+
+  .. graphviz:: myfile.dot
+
+See the `Dot User's Manual <https://www.graphviz.org/pdf/dotguide.pdf>`_ by
+Emden R. Gansner, Eleftherios Koutsofios, and Stephen North for examples of
+digraphs. This is especially useful if this is your first time encountering
+GraphViz.
+
+Ditaa
+-----
+
+You can use Ditaa_:
+
+.. _Ditaa: http://ditaa.sourceforge.net/
+
+.. ditaa::
+
+   +--------------+   /=----\
+   | hello, world |-->| hi! |
+   +--------------+   \-----/
+
+
+Blockdiag
+---------
+
+If a use arises, we can integrate Blockdiag_. It is a Graphviz-style
+declarative language for drawing things, and includes:
+
+- `block diagrams`_: boxes and arrows (automatic layout, as opposed to
+  Ditaa_)
+- `sequence diagrams`_: timelines and messages between them
+- `activity diagrams`_: subsystems and activities in them
+- `network diagrams`_: hosts, LANs, IP addresses etc (with `Cisco
+  icons`_ if wanted)
+
+.. _Blockdiag: http://blockdiag.com/en/
+.. _`Cisco icons`: https://pypi.org/project/blockdiagcontrib-cisco/
+.. _`block diagrams`: http://blockdiag.com/en/blockdiag/
+.. _`sequence diagrams`: http://blockdiag.com/en/seqdiag/index.html
+.. _`activity diagrams`: http://blockdiag.com/en/actdiag/index.html
+.. _`network diagrams`: http://blockdiag.com/en/nwdiag/
+
+
+Inkscape
+--------
+
+You can use Inkscape to generate scalable vector graphics.
+https://inkscape.org/en/ for restructuredText documents.
+
+If you generate diagrams with Inkscape, you should
+commit both the Scalable Vector Graphics (SVG) file and export a
+Portable Network Graphic (PNG) file. Reference the PNG file.
+
+By committing the SVG file, others will be able to update the
+SVG diagrams using Inkscape.
+
+HTML5 will support SVG inline.
+
+.. _`Submitting Patches`: https://github.com/ceph/ceph/blob/master/SubmittingPatches.rst
diff --git a/doc/dev/dpdk.rst b/doc/dev/dpdk.rst
new file mode 100644
index 000000000..cdc31a7c6
--- /dev/null
+++ b/doc/dev/dpdk.rst
@@ -0,0 +1,172 @@
+=========================
+Ceph messenger DPDKStack
+=========================
+
+Compiling DPDKStack
+===================
+
+Ceph dpdkstack is not compiled by default. Therefore, you need to recompile and
+enable the DPDKstack component.
+Optionally install ``dpdk-devel`` or ``dpdk-dev`` on distros with precompiled DPDK packages, and compile
+
+.. prompt:: bash $
+
+   do_cmake.sh -DWITH_DPDK=ON
+
+
+Setting the DPDK Network Adapter
+================================
+
+Most mainstream NICs support SR-IOV and can be virtualized into multiple VF NICs.
+Each OSD uses some dedicated NICs through DPDK. The mon, mgr and client use the PF NICs
+through the POSIX protocol stack.
+
+Load the driver on which DPDK depends:
+
+.. prompt:: bash #
+
+   modprobe vfio
+   modprobe vfio_pci
+
+Configure Hugepage by editing ``/etc/sysctl.conf`` ::
+
+  vm.nr_hugepages = xxx
+
+Configure the number of VFs based on the number of OSDs:
+
+.. prompt:: bash #
+
+   echo $numvfs > /sys/class/net/$port/device/sriov_numvfs
+
+Binding NICs to DPDK Applications:
+
+.. prompt:: bash #
+
+   dpdk-devbind.py -b vfio-pci 0000:xx:yy.z
+
+
+Configuring OSD DPDKStack
+==========================
+
+By default, the DPDK RTE initialization process requires the root privileges
+for accessing various resources in system. To grant the root access to
+the ``ceph`` user:
+
+.. prompt:: bash #
+
+   usermod -G root ceph
+
+The OSD selects the NICs using ``ms_dpdk_devs_allowlist``:
+
+#. Configure a single NIC.
+
+   .. code-block:: ini
+
+      ms_dpdk_devs_allowlist=-a 0000:7d:010
+
+   or
+
+   .. code-block:: ini
+
+      ms_dpdk_devs_allowlist=--allow=0000:7d:010
+
+#. Configure the Bond Network Adapter
+
+   .. code-block:: ini
+
+      ms_dpdk_devs_allowlist=--allow=0000:7d:01.0 --allow=0000:7d:02.6 --vdev=net_bonding0,mode=2,slave=0000:7d:01.0,slave=0000:7d:02.6
+
+DPDK-related configuration items are as follows:
+
+.. code-block:: ini
+
+   [osd]
+   ms_type=async+dpdk
+   ms_async_op_threads=1
+
+   ms_dpdk_port_id=0
+   ms_dpdk_gateway_ipv4_addr=172.19.36.1
+   ms_dpdk_netmask_ipv4_addr=255.255.255.0
+   ms_dpdk_hugepages=/dev/hugepages
+   ms_dpdk_hw_flow_control=false
+   ms_dpdk_lro=false
+   ms_dpdk_enable_tso=false
+   ms_dpdk_hw_queue_weight=1
+   ms_dpdk_memory_channel=2
+   ms_dpdk_debug_allow_loopback = true
+
+   [osd.x]
+   ms_dpdk_coremask=0xf0
+   ms_dpdk_host_ipv4_addr=172.19.36.51
+   public_addr=172.19.36.51
+   cluster_addr=172.19.36.51
+   ms_dpdk_devs_allowlist=--allow=0000:7d:01.1
+
+Debug and Optimization
+======================
+
+Locate faults based on logs and adjust logs to a proper level:
+
+.. code-block:: ini
+
+   debug_dpdk=xx
+   debug_ms=xx
+
+if the log contains a large number of retransmit messages,reduce the value of ms_dpdk_tcp_wmem.
+
+Run the perf dump command to view DPDKStack statistics:
+
+.. prompt:: bash $
+
+   ceph daemon osd.$i perf dump | grep dpdk
+
+
+if the ``dpdk_device_receive_nombuf_errors`` keeps increasing, check whether the
+throttling exceeds the limit:
+
+.. prompt:: bash $
+
+   ceph daemon osd.$i perf dump | grep throttle-osd_client -A 7 | grep "get_or_fail_fail"
+   ceph daemon osd.$i perf dump | grep throttle-msgr_dispatch_throttler -A 7 | grep "get_or_fail_fail"
+
+if the throttling exceeds the threshold, increase the throttling threshold or
+disable the throttling.
+
+Check whether the network adapter is faulty or abnormal.Run the following 
+command to obtain the network adapter status and statistics:
+
+.. prompt:: bash $
+
+   ceph daemon osd.$i show_pmd_stats
+   ceph daemon osd.$i show_pmd_xstats
+
+Some DPDK versions (eg. dpdk-20.11-3.e18.aarch64) or NIC TSOs are abnormal,
+try disabling tso:
+
+.. code-block:: ini
+
+   ms_dpdk_enable_tso=false
+
+if VF NICs support multiple queues, more NIC queues can be allocated to a
+single core to improve performance:
+
+.. code-block:: ini
+
+   ms_dpdk_hw_queues_per_qp=4
+
+
+Status and Future Work
+======================
+
+Compared with POSIX Stack, in the multi-concurrency test, DPDKStack has the same
+4K random write performance, 8K random write performance is improved by 28%, and
+1 MB packets are unstable. In the single-latency test,the 4K and 8K random write
+latency is reduced by 15% (the lower the latency is, the better).
+
+At a high level, our future work plan is:
+
+  OSD multiple network support (public network and cluster network)
+  The public and cluster network adapters can be configured.When connecting or
+  listening,the public or cluster network adapters can be selected based on the
+  IP address.During msgr-work initialization,initialize both the public and cluster
+  network adapters and create two DPDKQueuePairs.
diff --git a/doc/dev/encoding.rst b/doc/dev/encoding.rst
new file mode 100644
index 000000000..8ec3bb22d
--- /dev/null
+++ b/doc/dev/encoding.rst
@@ -0,0 +1,226 @@
+
+Serialization (encode/decode)
+=============================
+
+When a structure is sent over the network or written to disk, it is
+encoded into a string of bytes. Usually (but not always -- multiple
+serialization facilities coexist in Ceph) serializable structures
+have ``encode`` and ``decode`` methods that write and read from
+``bufferlist`` objects representing byte strings.
+
+Terminology
+-----------
+It is best to think not in the domain of daemons and clients but
+encoders and decoders. An encoder serializes a structure into a bufferlist
+while a decoder does the opposite.
+
+Encoders and decoders can be referred collectively as dencoders.
+
+Dencoders (both encoders and docoders) live within daemons and clients.
+For instance, when an RBD client issues an IO operation, it prepares
+an instance of the ``MOSDOp`` structure and encodes it into a bufferlist
+that is put on the wire.
+An OSD reads these bytes and decodes them back into an ``MOSDOp`` instance.
+Here encoder was used by the client while decoder by the OSD. However,
+these roles can swing -- just imagine handling of the response: OSD encodes
+the ``MOSDOpReply`` while RBD clients decode.
+
+Encoder and decoder operate accordingly to a format which is defined
+by a programmer by implementing the ``encode`` and ``decode`` methods.
+
+Principles for format change
+----------------------------
+It is not unusual that the format of serialization changes. This
+process requires careful attention from during both development
+and review.
+
+The general rule is that a decoder must understand what had been
+encoded by an encoder. Most of the problems come from ensuring
+that compatibility continues between old decoders and new encoders
+as well as new decoders and old decoders. One should assume
+that -- if not otherwise derogated -- any mix (old/new) is
+possible in a cluster. There are 2 main reasons for that:
+
+1. Upgrades. Although there are recommendations related to the order
+   of entity types (mons/osds/clients), it is not mandatory and
+   no assumption should be made about it.
+2. Huge variability of client versions. It was always the case
+   that kernel (and thus kernel clients) upgrades are decoupled
+   from Ceph upgrades. Moreover, proliferation of containerization
+   bring the variability even to e.g. ``librbd`` -- now user space
+   libraries live on the container own.
+
+With this being said, there are few rules limiting the degree
+of interoperability between dencoders:
+
+* ``n-2`` for dencoding between daemons,
+* ``n-3`` hard requirement for client-involved scenarios,
+* ``n-3..``  soft requirements for clinet-involved scenarios. Ideally
+  every client should be able to talk any version of daemons.
+
+As the underlying reasons are the same, the rules dencoders
+follow are virtually the same as for deprecations of our features
+bits. See the ``Notes on deprecation`` in ``src/include/ceph_features.h``.
+
+Frameworks
+----------
+Currently multiple genres of dencoding helpers co-exist.
+
+* encoding.h (the most proliferated one),
+* denc.h (performance optimized, seen mostly in ``BlueStore``),
+* the `Message` hierarchy.
+
+Although details vary, the interoperability rules stay the same.
+
+Adding a field to a structure
+-----------------------------
+
+You can see examples of this all over the Ceph code, but here's an
+example:
+
+.. code-block:: cpp
+
+    class AcmeClass
+    {
+        int member1;
+        std::string member2;
+
+        void encode(bufferlist &bl)
+        {
+            ENCODE_START(1, 1, bl);
+            ::encode(member1, bl);
+            ::encode(member2, bl);
+            ENCODE_FINISH(bl);
+        }
+
+        void decode(bufferlist::iterator &bl)
+        {
+            DECODE_START(1, bl);
+            ::decode(member1, bl);
+            ::decode(member2, bl);
+            DECODE_FINISH(bl);
+        }
+    };
+
+The ``ENCODE_START`` macro writes a header that specifies a *version* and
+a *compat_version* (both initially 1).  The message version is incremented
+whenever a change is made to the encoding.  The compat_version is incremented
+only if the change will break existing decoders -- decoders are tolerant
+of trailing bytes, so changes that add fields at the end of the structure
+do not require incrementing compat_version.
+
+The ``DECODE_START`` macro takes an argument specifying the most recent
+message version that the code can handle.  This is compared with the
+compat_version encoded in the message, and if the message is too new then
+an exception will be thrown.  Because changes to compat_version are rare,
+this isn't usually something to worry about when adding fields.
+
+In practice, changes to encoding usually involve simply adding the desired fields
+at the end of the ``encode`` and ``decode`` functions, and incrementing
+the versions in ``ENCODE_START`` and ``DECODE_START``.  For example, here's how
+to add a third field to ``AcmeClass``:
+
+.. code-block:: cpp
+
+    class AcmeClass
+    {
+        int member1;
+        std::string member2;
+        std::vector<std::string> member3;
+
+        void encode(bufferlist &bl)
+        {
+            ENCODE_START(2, 1, bl);
+            ::encode(member1, bl);
+            ::encode(member2, bl);
+            ::encode(member3, bl);
+            ENCODE_FINISH(bl);
+        }
+
+        void decode(bufferlist::iterator &bl)
+        {
+            DECODE_START(2, bl);
+            ::decode(member1, bl);
+            ::decode(member2, bl);
+            if (struct_v >= 2) {
+                ::decode(member3, bl);
+            }
+            DECODE_FINISH(bl);
+        }
+    };
+
+Note that the compat_version did not change because the encoded message
+will still be decodable by versions of the code that only understand
+version 1 -- they will just ignore the trailing bytes where we encode ``member3``.
+
+In the ``decode`` function, decoding the new field is conditional: this is
+because we might still be passed older-versioned messages that do not
+have the field.  The ``struct_v`` variable is a local set by the ``DECODE_START``
+macro.
+
+# Into the weeeds
+
+The append-extendability of our dencoders is a result of the forward
+compatibility that the ``ENCODE_START`` and ``DECODE_FINISH`` macros bring.
+
+They are implementing extendibility facilities. An encoder, when filling
+the bufferlist, prepends three fields: version of the current format,
+minimal version of a decoder compatible with it and the total size of
+all encoded fields.
+
+.. code-block:: cpp
+
+        /**
+         * start encoding block
+         *
+         * @param v current (code) version of the encoding
+         * @param compat oldest code version that can decode it
+         * @param bl bufferlist to encode to
+         *
+         */
+        #define ENCODE_START(v, compat, bl)                             \
+          __u8 struct_v = v;                                            \
+          __u8 struct_compat = compat;                                  \
+          ceph_le32 struct_len;                                         \
+          auto filler = (bl).append_hole(sizeof(struct_v) +             \
+            sizeof(struct_compat) + sizeof(struct_len));                \
+          const auto starting_bl_len = (bl).length();                   \
+          using ::ceph::encode;                                         \
+          do {
+
+The ``struct_len`` field allows the decoder to eat all the bytes that were
+left undecoded in the user-provided ``decode`` implementation.
+Analogically, decoders tracks how much input has been decoded in the
+user-provided ``decode`` methods.
+
+.. code-block:: cpp
+
+        #define DECODE_START(bl)		                        \
+          unsigned struct_end = 0;					\
+          __u32 struct_len;						\
+          decode(struct_len, bl);					\
+          ...                                                           \
+          struct_end = bl.get_off() + struct_len;			\
+          }								\
+          do {
+
+
+Decoder uses this information to discard the extra bytes it does not
+understand. Advancing bufferlist is critical as dencoders tend to be nested;
+just leaving it intact would work only for the very last ``deocde`` call
+in a nested structure.
+
+.. code-block:: cpp
+
+        #define DECODE_FINISH(bl)					\
+          } while (false);						\
+          if (struct_end) {						\
+            ...                                                         \
+            if (bl.get_off() < struct_end)				\
+              bl += struct_end - bl.get_off();				\
+          }
+
+
+This entire, cooperative mechanism allows encoder (its further revisions)
+to generate more byte stream (due to e.g. adding a new field at the end)
+and not worry that the residue will crash older decoder revisions.
diff --git a/doc/dev/erasure-coded-pool.rst b/doc/dev/erasure-coded-pool.rst
new file mode 100644
index 000000000..8ad697702
--- /dev/null
+++ b/doc/dev/erasure-coded-pool.rst
@@ -0,0 +1,135 @@
+Erasure Coded pool
+==================
+
+Purpose
+-------
+
+Erasure-coded pools require less storage space compared to replicated
+pools.  The erasure-coding support has higher computational requirements and
+only supports a subset of the operations allowed on an object (for instance,
+partial write is not supported).
+
+Use cases
+---------
+
+Cold storage
+~~~~~~~~~~~~
+
+An erasure-coded pool is created to store a large number of 1GB
+objects (imaging, genomics, etc.) and 10% of them are read per
+month. New objects are added every day and the objects are not
+modified after being written. On average there is one write for 10,000
+reads.
+
+A replicated pool is created and set as a cache tier for the
+erasure coded pool. An agent demotes objects (i.e. moves them from the
+replicated pool to the erasure-coded pool) if they have not been
+accessed in a week.
+
+The erasure-coded pool CRUSH rule targets hardware designed for
+cold storage with high latency and slow access time. The replicated
+pool CRUSH rule targets faster hardware to provide better response
+times.
+
+Cheap multidatacenter storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ten datacenters are connected with dedicated network links. Each
+datacenter contains the same amount of storage with no power-supply
+backup and no air-cooling system.
+
+An erasure-coded pool is created with a CRUSH rule that will
+ensure no data loss if at most three datacenters fail
+simultaneously. The overhead is 50% with erasure code configured to
+split data in six (k=6) and create three coding chunks (m=3). With
+replication the overhead would be 400% (four replicas).
+
+Interface
+---------
+
+Set up an erasure-coded pool::
+
+ $ ceph osd pool create ecpool erasure
+
+Set up an erasure-coded pool and the associated CRUSH rule ``ecrule``::
+
+ $ ceph osd crush rule create-erasure ecrule
+ $ ceph osd pool create ecpool erasure default ecrule
+
+Set the CRUSH failure domain to osd (instead of host, which is the default)::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-failure-domain=osd
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-failure-domain=osd
+ $ ceph osd pool create ecpool erasure myprofile
+
+Control the parameters of the erasure code plugin::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     k=3 m=2
+ $ ceph osd erasure-code-profile get myprofile
+ k=3
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ $ ceph osd pool create ecpool erasure myprofile
+
+Choose an alternate erasure code plugin::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     plugin=example technique=xor
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=example
+ technique=xor
+ $ ceph osd pool create ecpool 12 12 erasure \
+     myprofile
+
+Display the default erasure code profile::
+
+  $ ceph osd erasure-code-profile ls
+  default
+  $ ceph osd erasure-code-profile get default
+  k=2
+  m=2
+  plugin=jerasure
+  technique=reed_sol_van
+
+Create a profile to set the data to be distributed on six OSDs (k+m=6) and sustain the loss of three OSDs (m=3) without losing data::
+
+  $ ceph osd erasure-code-profile set myprofile k=3 m=3
+  $ ceph osd erasure-code-profile get myprofile
+  k=3
+  m=3
+  plugin=jerasure
+  technique=reed_sol_van
+  $ ceph osd erasure-code-profile ls
+  default
+  myprofile
+
+Remove a profile that is no longer in use (otherwise it will fail with EBUSY)::
+
+  $ ceph osd erasure-code-profile ls
+  default
+  myprofile
+  $ ceph osd erasure-code-profile rm myprofile
+  $ ceph osd erasure-code-profile ls
+  default
+
+Set the rule to ssd (instead of default)::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-root=ssd
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-root=ssd
+
diff --git a/doc/dev/file-striping.rst b/doc/dev/file-striping.rst
new file mode 100644
index 000000000..405c9718d
--- /dev/null
+++ b/doc/dev/file-striping.rst
@@ -0,0 +1,161 @@
+File striping
+=============
+
+The text below describes how files from Ceph file system clients are
+stored across objects stored in RADOS.
+
+ceph_file_layout
+----------------
+
+Ceph distributes (stripes) the data for a given file across a number
+of underlying objects.  The way file data is mapped to those objects
+is defined by the ceph_file_layout structure.  The data distribution
+is a modified RAID 0, where data is striped across a set of objects up
+to a (per-file) fixed size, at which point another set of objects
+holds the file's data.  The second set also holds no more than the
+fixed amount of data, and then another set is used, and so on.
+
+Defining some terminology will go a long way toward explaining the
+way file data is laid out across Ceph objects.
+
+- file
+    A collection of contiguous data, named from the perspective of
+    the Ceph client (i.e., a file on a Linux system using Ceph
+    storage).  The data for a file is divided into fixed-size
+    "stripe units," which are stored in ceph "objects."
+- stripe unit
+    The size (in bytes) of a block of data used in the RAID 0
+    distribution of a file.  All stripe units for a file have equal
+    size.  The last stripe unit is typically incomplete--i.e. it
+    represents the data at the end of the file as well as unused
+    "space" beyond it up to the end of the fixed stripe unit size.
+- stripe count
+    The number of consecutive stripe units that constitute a RAID 0
+    "stripe" of file data.
+- stripe
+    A contiguous range of file data, RAID 0 striped across "stripe
+    count" objects in fixed-size "stripe unit" blocks.
+- object
+    A collection of data maintained by Ceph storage.  Objects are
+    used to hold portions of Ceph client files.
+- object set
+    A set of objects that together represent a contiguous portion of
+    a file.
+
+Three fields in the ceph_file_layout structure define this mapping::
+
+	u32 fl_stripe_unit;
+	u32 fl_stripe_count;
+	u32 fl_object_size;
+
+(They are actually maintained in their on-disk format, __le32.)
+
+The role of the first two fields should be clear from the
+definitions above.
+
+The third field is the maximum size (in bytes) of an object used to
+back file data.  The object size is a multiple of the stripe unit.
+
+A file's data is blocked into stripe units, and consecutive stripe
+units are stored on objects in an object set.  The number of objects
+in a set is the same as the stripe count.  No object storing file
+data will exceed the file's designated object size, so after some
+fixed number of complete stripes, a new object set is used to store
+subsequent file data.
+
+Note that by default, Ceph uses a simple striping strategy in which
+object_size equals stripe_unit and stripe_count is 1.  This simply
+puts one stripe_unit in each object.  
+
+Here's a more complex example::
+
+    file size = 1 trillion = 1000000000000 bytes
+
+    fl_stripe_unit = 64KB = 65536 bytes
+    fl_stripe_count = 5 stripe units per stripe
+    fl_object_size = 64GB = 68719476736 bytes
+
+This means::
+
+    file stripe size = 64KB * 5 = 320KB = 327680 bytes
+    each object holds 64GB / 64KB = 1048576 stripe units
+    file object set size = 64GB * 5 = 320GB = 343597383680 bytes
+	(also 1048576 stripe units * 327680 bytes per stripe unit)
+
+So the file's 1 trillion bytes can be divided into complete object
+sets, then complete stripes, then complete stripe units, and finally
+a single incomplete stripe unit::
+
+    - 1 trillion bytes / 320GB per object set = 2 complete object sets
+        (with 312805232640 bytes remaining)
+    - 312805232640 bytes / 320KB per stripe = 954605 complete stripes
+        (with 266240 bytes remaining)
+    - 266240 bytes / 64KB per stripe unit = 4 complete stripe units
+        (with 4096 bytes remaining)
+    - and the final incomplete stripe unit holds those 4096 bytes.
+
+The ASCII art below attempts to capture this::
+
+     _________   _________   _________   _________   _________
+    /object  0\ /object  1\ /object  2\ /object  3\ /object  4\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe |
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | stripe 0
+  b |     0   | |     1   | |     2   | |     3   | |     4   |
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe |
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | stripe 1
+  t |     5   | |     6   | |     7   | |     8   | |     9   |
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  0 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048575
+    | 5242875 | | 5242876 | | 5242877 | | 5242878 | | 5242879 |
+    \=========/ \=========/ \=========/ \=========/ \=========/
+
+     _________   _________   _________   _________   _________
+    /object  5\ /object  6\ /object  7\ /object  8\ /object  9\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048576
+  b | 5242880 | | 5242881 | | 5242882 | | 5242883 | | 5242884 |
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048577
+  t | 5242885 | | 5242886 | | 5242887 | | 5242888 | | 5242889 |
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  1 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097151
+    | 10485755| | 10485756| | 10485757| | 10485758| | 10485759|
+    \=========/ \=========/ \=========/ \=========/ \=========/
+
+     _________   _________   _________   _________   _________
+    /object 10\ /object 11\ /object 12\ /object 13\ /object 14\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097152
+  b | 10485760| | 10485761| | 10485762| | 10485763| | 10485764|
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097153
+  t | 10485765| | 10485766| | 10485767| | 10485768| | 10485769|
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  2 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 3051756
+    | 15258780| | 15258781| | 15258782| | 15258783| | 15258784|
+    |---------| |---------| |---------| |---------| |---------|
+    |  stripe | |  stripe | |  stripe | |  stripe | | (partial| (partial
+    |   unit  | |   unit  | |   unit  | |   unit  | |  stripe | stripe
+    | 15258785| | 15258786| | 15258787| | 15258788| |  unit)  | 3051757)
+    \=========/ \=========/ \=========/ \=========/ \=========/
diff --git a/doc/dev/freebsd.rst b/doc/dev/freebsd.rst
new file mode 100644
index 000000000..b1645b873
--- /dev/null
+++ b/doc/dev/freebsd.rst
@@ -0,0 +1,53 @@
+==============================
+FreeBSD Implementation details
+==============================
+
+
+Disk layout
+-----------
+
+Current implementation works on ZFS pools
+
+* created in /var/lib/ceph
+* One ZFS pool per OSD, like::
+
+    gpart create -s GPT ada1
+    gpart add -t freebsd-zfs -l osd1 ada1
+    zpool create -o mountpoint=/var/lib/ceph/osd/osd.1 osd
+
+* Maybe add some cache and log (ZIL)? Assuming that ada2 is an SSD::
+
+    gpart create -s GPT ada2
+    gpart add -t freebsd-zfs -l osd1-log -s 1G ada2
+    zpool add osd1 log gpt/osd1-log
+    gpart add -t freebsd-zfs -l osd1-cache -s 10G ada2
+    zpool add osd1 log gpt/osd1-cache
+
+* Note: *UFS2 does not allow large xattribs*
+
+
+Configuration
+-------------
+
+As per FreeBSD default parts of extra software go into ``/usr/local/``. Which
+means that for ``/etc/ceph.conf`` the default location is
+``/usr/local/etc/ceph/ceph.conf``. Smartest thing to do is to create a softlink
+from ``/etc/ceph`` to ``/usr/local/etc/ceph``::
+
+  ln -s /usr/local/etc/ceph /etc/ceph
+
+A sample file is provided in ``/usr/local/share/doc/ceph/sample.ceph.conf``
+
+
+MON creation
+------------
+
+Monitors are created by following the manual creation steps on::
+
+  https://docs.ceph.com/en/latest/install/manual-freebsd-deployment/
+
+
+OSD creation
+------------
+
+OSDs can be manually created only, see :ref:`freebsd_adding_osds`
diff --git a/doc/dev/generatedocs.rst b/doc/dev/generatedocs.rst
new file mode 100644
index 000000000..8632eb176
--- /dev/null
+++ b/doc/dev/generatedocs.rst
@@ -0,0 +1,83 @@
+Building Ceph Documentation
+===========================
+
+Ceph utilizes Python's Sphinx documentation tool. For details on
+the Sphinx documentation tool, refer to `The Sphinx Documentation Tool <https://www.sphinx-doc.org/en/master/>`_.
+
+To build the Ceph documentation set, you must:
+
+1. Clone the Ceph repository
+2. Install the required tools
+3. Build the documents
+4. Demo the documents (Optional)
+
+.. highlight:: bash
+
+Clone the Ceph Repository
+-------------------------
+
+To clone the Ceph repository, you must have ``git`` installed
+on your local host. To install ``git``, execute::
+
+	sudo apt-get install git
+
+To clone the Ceph repository, execute::
+
+	git clone git://github.com/ceph/ceph
+
+You should have a full copy of the Ceph repository.
+
+
+Install the Required Tools
+--------------------------
+
+To build the Ceph documentation, some dependencies are required.
+To know what packages are needed, you can launch this command::
+
+    cd ceph
+    admin/build-doc
+
+If dependencies are missing, the command above will fail
+with a message that suggests you a command to install all
+missing dependencies.
+
+
+Build the Documents
+-------------------
+
+Once you have installed all the dependencies, execute the build (the
+same command as above)::
+
+	cd ceph
+	admin/build-doc
+
+Once you build the documentation set, you may navigate to the source directory to view it::
+
+	cd build-doc/output
+
+There should be an ``html`` directory and a ``man`` directory containing documentation
+in HTML and manpage formats respectively.
+
+``admin/build-doc`` takes a long time to prepare the environment and build the document.
+But you can just rebuild the document on changes using::
+
+  admin/build-doc livehtml
+
+This feature uses ``sphinx-autobuild`` under the hood. You can also pass options to it. For
+instance, to open the browser after building the documentation::
+
+  admin/build-doc livehtml -- --open-browser
+
+Please see `sphinx-autobuild <https://pypi.org/project/sphinx-autobuild/>`_ for more details.
+
+Demo the Documents
+-------------------
+
+Once you build the documentation, as described above, you can demo the rendered documents
+by running ``serve-doc``::
+
+	cd ceph
+	admin/serve-doc
+
+This will serve the ``build-doc/output/html`` directory over port 8080 via 
+Python's ``SimpleHTTPServer`` module.
diff --git a/doc/dev/health-reports.rst b/doc/dev/health-reports.rst
new file mode 100644
index 000000000..7769c6d8c
--- /dev/null
+++ b/doc/dev/health-reports.rst
@@ -0,0 +1,69 @@
+==============
+Health Reports
+==============
+
+
+How to Get Reports
+==================
+
+In general, there are two channels to retrieve the health reports:
+
+ceph (CLI)
+   which sends ``health`` mon command for retrieving the health status of the cluster
+mgr module
+   which calls ``mgr.get('health')`` for the same report in the form of a JSON encoded string
+
+The following diagrams outline the involved parties and how the interact when the clients
+query for the reports:
+
+
+Where are the Reports Generated
+===============================
+
+Aggregator of Aggregators
+-------------------------
+
+Health reports are aggregated from multiple Paxos services:
+
+- AuthMonitor
+- HealthMonitor
+- MDSMonitor
+- MgrMonitor
+- MgrStatMonitor
+- MonmapMonitor
+- OSDMonitor
+
+When persisting the pending changes in their own domain, each of them identifies the
+health related issues and store them into the monstore with the prefix of ``health``
+using the same transaction. For instance, ``OSDMonitor`` checks a pending new osdmap
+for possible issues, like down OSDs and missing scrub flag in a pool, and then stores
+the encoded form of the health reports along with the new osdmap. These reports are
+later loaded and decoded, so they can be collected on demand. When it comes to
+``MDSMonitor``, it persists the health metrics in the beacon sent by the MDS daemons,
+and prepares health reports when storing the pending changes.
+
+
+So, if we want to add a new warning related to cephfs, probably the best place to
+start is ``MDSMonitor::encode_pending()``, where health reports are collected from
+the latest ``FSMap`` and the health metrics reported by MDS daemons.
+
+But it's noteworthy that ``MgrStatMonitor`` does *not* prepare the reports by itself,
+it just stores whatever the health reports received from mgr!
+
+ceph-mgr -- A Delegate Aggregator
+---------------------------------
+
+In Ceph, mgr is created to share the burden of monitor, which is used to establish
+the consensus of information which is critical to keep the cluster function.
+Apparently, osdmap, mdsmap and monmap fall into this category. But what about the
+aggregated statistics of the cluster? They are crucial for the administrator to
+understand the status of the cluster, but they might not be that important to keep
+the cluster running. To address this scalability issue,  we offloaded the work of
+collecting and aggregating the metrics to mgr.
+
+Now, mgr is responsible for receiving and processing the ``MPGStats`` messages from
+OSDs. And we also developed a protocol allowing a daemon to periodically report its
+metrics and status to mgr using ``MMgrReport``. On the mgr side, it periodically sends
+an aggregated report to the ``MgrStatMonitor`` service on mon. As explained earlier,
+this service just persists the health reports in the aggregated report to the monstore.
+
diff --git a/doc/dev/iana.rst b/doc/dev/iana.rst
new file mode 100644
index 000000000..d2daf3395
--- /dev/null
+++ b/doc/dev/iana.rst
@@ -0,0 +1,16 @@
+IANA Numbers
+============
+
+Private Enterprise Number (PEN) Assignment
+------------------------------------------
+
+50495
+
+Organization ``Ceph``.
+
+Port number (monitor)
+---------------------
+
+3300
+
+That's 0xce4, or ce4h, or (sort of) "ceph."
diff --git a/doc/dev/internals.rst b/doc/dev/internals.rst
new file mode 100644
index 000000000..a894394c9
--- /dev/null
+++ b/doc/dev/internals.rst
@@ -0,0 +1,52 @@
+================
+ Ceph Internals
+================
+
+.. note:: If you're looking for how to use Ceph as a library from your
+   own software, please see :doc:`/api/index`.
+
+You can start a development mode Ceph cluster, after compiling the source, with::
+
+	cd build
+	OSD=3 MON=3 MGR=3 ../src/vstart.sh -n -x
+	# check that it's there
+	bin/ceph health
+
+.. rubric:: Mailing list
+
+The ``dev@ceph.io`` list is for discussion about the development of Ceph,
+its interoperability with other technology, and the operations of the
+project itself.  Subscribe by sending a message to ``dev-request@ceph.io``
+with the line::
+
+ subscribe ceph-devel
+
+in the body of the message.
+
+The ceph-devel@vger.kernel.org list is for discussion
+and patch review for the Linux kernel Ceph client component.
+Subscribe by sending a message to ``majordomo@vger.kernel.org`` with the line::
+
+ subscribe ceph-devel
+
+in the body of the message.
+
+.. raw:: html
+
+   <!---
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
+   osd_internals/index*
+   mds_internals/index*
+   radosgw/index*
+   ceph-volume/index*
+   crimson/index*
+
+.. raw:: html
+
+   --->
diff --git a/doc/dev/kubernetes.rst b/doc/dev/kubernetes.rst
new file mode 100644
index 000000000..75b100b24
--- /dev/null
+++ b/doc/dev/kubernetes.rst
@@ -0,0 +1,228 @@
+
+.. _kubernetes-dev:
+
+=======================================
+Hacking on Ceph in Kubernetes with Rook
+=======================================
+
+.. warning::
+
+    This is *not* official user documentation for setting up production
+    Ceph clusters with Kubernetes.  It is aimed at developers who want
+    to hack on Ceph in Kubernetes.
+
+This guide is aimed at Ceph developers getting started with running
+in a Kubernetes environment. It assumes that you may be hacking on Rook,
+Ceph or both, so everything is built from source.
+
+TL;DR for hacking on MGR modules
+================================
+
+Make your changes to the Python code base and then from Ceph's
+``build`` directory, run::
+
+    ../src/script/kubejacker/kubejacker.sh '192.168.122.1:5000'
+
+where ``'192.168.122.1:5000'`` is a local docker registry and
+Rook's ``CephCluster`` CR uses ``image: 192.168.122.1:5000/ceph/ceph:latest``.
+
+1. Build a kubernetes cluster
+=============================
+
+Before installing Ceph/Rook, make sure you've got a working kubernetes
+cluster with some nodes added (i.e. ``kubectl get nodes`` shows you something).
+The rest of this guide assumes that your development workstation has network
+access to your kubernetes cluster, such that ``kubectl`` works from your
+workstation.
+
+`There are many ways <https://kubernetes.io/docs/setup/>`_
+to build a kubernetes cluster: here we include some tips/pointers on where
+to get started.
+
+`kubic-terraform-kvm <https://github.com/kubic-project/kubic-terraform-kvm>`_
+might also be an option.
+
+Or `Host your own <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/>`_ with
+``kubeadm``.
+
+Some Tips
+---------
+
+Here are some tips for a smoother ride with
+
+``kubeadm``:
+
+- If you have previously added any yum/deb repos for kubernetes packages,
+  disable them before trying to use the packages.cloud.google.com repository.
+  If you don't, you'll get quite confusing conflicts.
+- Even if your distro already has docker, make sure you're installing it
+  a version from docker.com that is within the range mentioned in the
+  kubeadm install instructions. Especially, note that the docker in CentOS 7, 8
+  will *not* work.
+
+``minikube``:
+
+- Start up minikube by passing local docker registry address::
+  ``minikube start --driver=docker --insecure-registry='192.168.122.1:5000'``
+
+Hosted elsewhere
+----------------
+
+If you do not have any servers to hand, you might try a pure
+container provider such as Google Compute Engine. Your mileage may
+vary when it comes to what kinds of storage devices are visible
+to your kubernetes cluster.
+
+Make sure you check how much it's costing you before you spin up a big cluster!
+
+
+2. Run a docker registry
+========================
+
+Run this somewhere accessible from both your workstation and your
+kubernetes cluster (i.e. so that ``docker push/pull`` just works everywhere).
+This is likely to be the same host you're using as your kubernetes master.
+
+1. Install the ``docker-distribution`` package.
+2. If you want to configure the port, edit ``/etc/docker-distribution/registry/config.yml``
+3. Enable the registry service:
+
+::
+
+    systemctl enable docker-distribution
+    systemctl start docker-distribution
+
+You may need to mark the registry as **insecure**.
+
+3. Build Rook
+=============
+
+.. note::
+
+    Building Rook is **not required** to make changes to Ceph.
+
+Install Go if you don't already have it.
+
+Download the Rook source code:
+
+::
+
+    go get github.com/rook/rook
+
+    # Ignore this warning, as Rook is not a conventional go package
+    can't load package: package github.com/rook/rook: no Go files in /home/jspray/go/src/github.com/rook/rook
+
+You will now have a Rook source tree in ~/go/src/github.com/rook/rook -- you may
+be tempted to clone it elsewhere, but your life will be easier if you
+leave it in your GOPATH.
+
+Run ``make`` in the root of your Rook tree to build its binaries and containers:
+
+::
+
+    make
+    ...
+    === saving image build-9204c79b/ceph-amd64
+    === docker build build-9204c79b/ceph-toolbox-base-amd64
+    sha256:653bb4f8d26d6178570f146fe637278957e9371014ea9fce79d8935d108f1eaa
+    === docker build build-9204c79b/ceph-toolbox-amd64
+    sha256:445d97b71e6f8de68ca1c40793058db0b7dd1ebb5d05789694307fd567e13863
+    === caching image build-9204c79b/ceph-toolbox-base-amd64
+
+You can use ``docker image ls`` to see the resulting built images.  The
+images you care about are the ones with tags ending "ceph-amd64" (used
+for the Rook operator and Ceph daemons) and "ceph-toolbox-amd64" (used
+for the "toolbox" container where the CLI is run).
+
+4. Build Ceph
+=============
+
+.. note::
+
+    Building Ceph is **not required** to make changes to MGR modules
+    written in Python.
+
+
+The Rook containers and the Ceph containers are independent now. Note that
+Rook's Ceph client libraries need to communicate with the Ceph cluster,
+therefore a compatible major version is required.
+
+You can run a Registry docker container with access to your Ceph source
+tree using a command like:
+
+::
+
+    docker run -i -v /my/ceph/src:/my/ceph/src -p 192.168.122.1:5000:5000 -t --name registry registry:2
+
+
+Once you have built Ceph, you can inject the resulting binaries into
+the Rook container image using the ``kubejacker.sh`` script (run from
+your build directory but from *outside* your build container).
+
+5. Run Kubejacker
+=================
+
+``kubejacker`` needs access to your docker registry. Execute the script
+to build a docker image containing your latest Ceph binaries:
+
+::
+
+    build$ ../src/script/kubejacker/kubejacker.sh "<host>:<port>"
+
+
+Now you've got your freshly built Rook and freshly built Ceph into
+a single container image, ready to run.  Next time you change something
+in Ceph, you can re-run this to update your image and restart your
+kubernetes containers. If you change something in Rook, then re-run the Rook
+build, and the Ceph build too.
+
+5. Run a Rook cluster
+=====================
+
+Please refer to `Rook's documentation <https://rook.io/docs/rook/master/ceph-quickstart.html>`_
+for setting up a Rook operator, a Ceph cluster and the toolbox.
+
+The Rook source tree includes example .yaml files in
+``cluster/examples/kubernetes/ceph/``. Copy these into
+a working directory, and edit as necessary to configure
+the setup you want:
+
+- Ensure that ``spec.cephVersion.image`` points to your docker registry::
+
+    spec:
+      cephVersion:
+        allowUnsupported: true
+        image: 192.168.122.1:5000/ceph/ceph:latest
+
+Then, load the configuration into the kubernetes API using ``kubectl``:
+
+::
+
+    kubectl apply -f ./cluster-test.yaml
+
+Use ``kubectl -n rook-ceph get pods`` to check the operator
+pod the Ceph daemons and toolbox are is coming up.
+
+Once everything is up and running,
+you should be able to open a shell in the toolbox container and
+run ``ceph status``.
+
+If your mon services start but the rest don't, it could be that they're
+unable to form a quorum due to a Kubernetes networking issue: check that
+containers in your Kubernetes cluster can ping containers on other nodes.
+
+Cheat sheet
+===========
+
+Open a shell in your toolbox container::
+
+    kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath="{.items[0].metadata.name}") -- bash
+
+Inspect the Rook operator container's logs::
+
+    kubectl -n rook-ceph logs -l app=rook-ceph-operator
+
+Inspect the ceph-mgr container's logs::
+
+    kubectl -n rook-ceph logs -l app=rook-ceph-mgr
+
diff --git a/doc/dev/libs.rst b/doc/dev/libs.rst
new file mode 100644
index 000000000..203dd38b0
--- /dev/null
+++ b/doc/dev/libs.rst
@@ -0,0 +1,18 @@
+======================
+ Library architecture
+======================
+
+Ceph is structured into libraries which are built and then combined together to
+make executables and other libraries.
+
+- libcommon: a collection of utilities which are available to nearly every ceph
+  library and executable. In general, libcommon should not contain global
+  variables, because it is intended to be linked into libraries such as
+  libcephfs.so.
+
+- libglobal: a collection of utilities focused on the needs of Ceph daemon
+  programs. In here you will find pidfile management functions, signal
+  handlers, and so forth.
+
+.. todo:: document other libraries
+
diff --git a/doc/dev/logging.rst b/doc/dev/logging.rst
new file mode 100644
index 000000000..67d3de141
--- /dev/null
+++ b/doc/dev/logging.rst
@@ -0,0 +1,106 @@
+
+Use of the cluster log
+======================
+
+(Note: none of this applies to the local "dout" logging.  This is about
+the cluster log that we send through the mon daemons)
+
+Severity
+--------
+
+Use ERR for situations where the cluster cannot do its job for some reason.
+For example: we tried to do a write, but it returned an error, or we tried
+to read something, but it's corrupt so we can't, or we scrubbed a PG but
+the data was inconsistent so we can't recover.
+
+Use WRN for incidents that the cluster can handle, but have some abnormal/negative
+aspect, such as a temporary degradation of service, or an unexpected internal
+value.  For example, a metadata error that can be auto-fixed, or a slow operation.
+
+Use INFO for ordinary cluster operations that do not indicate a fault in
+Ceph.  It is especially important that INFO level messages are clearly
+worded and do not cause confusion or alarm.
+
+Frequency
+---------
+
+It is important that messages of all severities are not excessively
+frequent.  Consumers may be using a rotating log buffer that contains
+messages of all severities, so even DEBUG messages could interfere
+with proper display of the latest INFO messages if the DEBUG messages
+are too frequent.
+
+Remember that if you have a bad state (as opposed to event), that is
+what health checks are for -- do not spam the cluster log to indicate
+a continuing unhealthy state.
+
+Do not emit cluster log messages for events that scale with
+the number of clients or level of activity on the system, or for
+events that occur regularly in normal operation.  For example, it
+would be inappropriate to emit a INFO message about every
+new client that connects (scales with #clients), or to emit and INFO
+message about every CephFS subtree migration (occurs regularly).
+
+Language and formatting
+-----------------------
+
+(Note: these guidelines matter much less for DEBUG-level messages than
+ for INFO and above.  Concentrate your efforts on making INFO/WRN/ERR
+ messages as readable as possible.)
+
+Use the passive voice.  For example, use "Object xyz could not be read", rather
+than "I could not read the object xyz".
+
+Print long/big identifiers, such as inode numbers, as hex, prefixed
+with an 0x so that the user can tell it is hex.  We do this because
+the 0x makes it unambiguous (no equivalent for decimal), and because
+the hex form is more likely to fit on the screen.
+
+Print size quantities as a human readable MB/GB/etc, including the unit
+at the end of the number.  Exception: if you are specifying an offset,
+where precision is essential to the meaning, then you can specify
+the value in bytes (but print it as hex).
+
+Make a good faith effort to fit your message on a single line.  It does
+not have to be guaranteed, but it should at least usually be
+the case.  That means, generally, no printing of lists unless there
+are only a few items in the list.
+
+Use nouns that are meaningful to the user, and defined in the
+documentation.  Common acronyms are OK -- don't waste screen space
+typing "Rados Object Gateway" instead of RGW.  Do not use internal
+class names like "MDCache" or "Objecter".  It is okay to mention
+internal structures if they are the direct subject of the message,
+for example in a corruption, but use plain English.
+Example: instead of "Objecter requests" say "OSD client requests"
+Example: it is okay to mention internal structure in the context
+of "Corrupt session table" (but don't say "Corrupt SessionTable")
+
+Where possible, describe the consequence for system availability, rather
+than only describing the underlying state.  For example, rather than
+saying "MDS myfs.0 is replaying", say that "myfs is degraded, waiting
+for myfs.0 to finish starting".
+
+While common acronyms are fine, don't randomly truncate words.  It's not
+"dir ino", it's "directory inode".
+
+If you're logging something that "should never happen", i.e. a situation
+where it would be an assertion, but we're helpfully not crashing, then
+make that clear in the language -- this is probably not a situation
+that the user can remediate themselves.
+
+Avoid UNIX/programmer jargon.  Instead of "errno", just say "error" (or
+preferably give something more descriptive than the number!)
+
+Do not mention cluster map epochs unless they are essential to
+the meaning of the message.  For example, "OSDMap epoch 123 is corrupt"
+would be okay (the epoch is the point of the message), but saying "OSD
+123 is down in OSDMap epoch 456" would not be (the osdmap and epoch
+concepts are an implementation detail, the down-ness of the OSD
+is the real message).  Feel free to send additional detail to
+the daemon's local log (via `dout`/`derr`).
+
+If you log a problem that may go away in the future, make sure you
+also log when it goes away.  Whatever priority you logged the original
+message at, log the "going away" message at INFO.
+
diff --git a/doc/dev/logs.rst b/doc/dev/logs.rst
new file mode 100644
index 000000000..7e703e541
--- /dev/null
+++ b/doc/dev/logs.rst
@@ -0,0 +1,55 @@
+============
+ Debug logs
+============
+
+The main debugging tool for Ceph is the dout and derr logging functions.
+Collectively, these are referred to as "dout logging."
+
+Dout has several log faculties, which can be set at various log
+levels using the configuration management system. So it is possible to enable
+debugging just for the messenger, by setting debug_ms to 10, for example.
+
+The dout macro avoids even generating log messages which are not going to be
+used, by enclosing them in an "if" statement. What this means is that if you
+have the debug level set at 0, and you run this code::
+
+	dout(20) << "myfoo() = " << myfoo() << dendl;
+
+
+myfoo() will not be called here.
+
+Unfortunately, the performance of debug logging is relatively low. This is
+because there is a single, process-wide mutex which every debug output
+statement takes, and every debug output statement leads to a write() system
+call or a call to syslog(). There is also a computational overhead to using C++
+streams to consider. So you will need to be parsimonious in your logging to get
+the best performance.
+
+Sometimes, enabling logging can hide race conditions and other bugs by changing
+the timing of events. Keep this in mind when debugging.
+
+Performance counters
+====================
+
+Ceph daemons use performance counters to track key statistics like number of
+inodes pinned. Performance counters are essentially sets of integers and floats
+which can be set, incremented, and read using the PerfCounters API.
+
+A PerfCounters object is usually associated with a single subsystem.  It
+contains multiple counters. This object is thread-safe because it is protected
+by an internal mutex. You can create multiple PerfCounters objects.
+
+Currently, three types of performance counters are supported: u64 counters,
+float counters, and long-run floating-point average counters. These are created
+by PerfCountersBuilder::add_u64, PerfCountersBuilder::add_fl, and
+PerfCountersBuilder::add_fl_avg, respectively. u64 and float counters simply
+provide a single value which can be updated, incremented, and read atomically.
+floating-pointer average counters provide two values: the current total, and
+the number of times the total has been changed. This is intended to provide a
+long-run average value.
+
+Performance counter information can be read in JSON format from the
+administrative socket (admin_sock). This is implemented as a UNIX domain
+socket. The Ceph performance counter plugin for collectd shows an example of how
+to access this information. Another example can be found in the unit tests for
+the administrative sockets.
diff --git a/doc/dev/macos.rst b/doc/dev/macos.rst
new file mode 100644
index 000000000..4f966be3d
--- /dev/null
+++ b/doc/dev/macos.rst
@@ -0,0 +1,50 @@
+build on MacOS
+==============
+
+Since we've switched to C++ 17, and the default clang shipped with Xcode 9 does not support all the C++ 17 language features, it's suggested to install clang using brew::
+
+  brew install llvm
+
+and install all the necessary bits::
+
+  brew install snappy ccache cmake pkg-config
+  pip install cython
+
+install FUSE if you want to build the FUSE support::
+
+  brew cask install osxfuse
+
+then, under the source directory of Ceph::
+
+  mkdir build
+  cd build
+  export PKG_CONFIG_PATH=/usr/local/Cellar/nss/3.48/lib/pkgconfig:/usr/local/Cellar/openssl/1.0.2t/lib/pkgconfig
+  cmake .. -DBOOST_J=4 \
+    -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang \
+    -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++ \
+    -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/opt/llvm/lib" \
+    -DENABLE_GIT_VERSION=OFF \
+    -DSNAPPY_ROOT_DIR=/usr/local/Cellar/snappy/1.1.7_1 \
+    -DWITH_BABELTRACE=OFF \
+    -DWITH_BLUESTORE=OFF \
+    -DWITH_CCACHE=OFF \
+    -DWITH_CEPHFS=OFF \
+    -DWITH_KRBD=OFF \
+    -DWITH_LIBCEPHFS=OFF \
+    -DWITH_LTTNG=OFF \
+    -DWITH_LZ4=OFF \
+    -DWITH_MANPAGE=ON \
+    -DWITH_MGR=OFF \
+    -DWITH_MGR_DASHBOARD_FRONTEND=OFF \
+    -DWITH_RADOSGW=OFF \
+    -DWITH_RDMA=OFF \
+    -DWITH_SPDK=OFF \
+    -DWITH_SYSTEMD=OFF \
+    -DWITH_TESTS=OFF \
+    -DWITH_XFS=OFF
+
+The paths to ``nss`` and ``snappy`` might vary if newer versions of the packages are installed.
+
+Also, please consider using boost v1.69 to address the bug of https://github.com/boostorg/atomic/issues/15.
+
+Currently, the most practical uses for Ceph on MacOS might be FUSE and some other librados based applications.
diff --git a/doc/dev/mds_internals/data-structures.rst b/doc/dev/mds_internals/data-structures.rst
new file mode 100644
index 000000000..c77175a16
--- /dev/null
+++ b/doc/dev/mds_internals/data-structures.rst
@@ -0,0 +1,44 @@
+MDS internal data structures
+==============================
+
+*CInode*
+  CInode contains the metadata of a file, there is one CInode for each file.
+  The CInode stores information like who owns the file, how big the file is.
+
+*CDentry*
+  CDentry is the glue that holds inodes and files together by relating inode to
+  file/directory names. A CDentry links to at most one CInode (it may not link
+  to any CInode). A CInode may be linked by multiple CDentries.
+
+*CDir*
+  CDir only exists for directory inode, it's used to link CDentries under the
+  directory. A CInode can have multiple CDir when the directory is fragmented.
+
+These data structures are linked together as::
+
+  CInode
+  CDir
+   |   \
+   |      \
+   |         \
+  CDentry   CDentry
+  CInode    CInode
+  CDir      CDir
+   |         |  \
+   |         |     \
+   |         |        \
+  CDentry   CDentry  CDentry
+  CInode    CInode   CInode
+
+As this doc is being written, size of CInode is about 1400 bytes, size of CDentry
+is about 400 bytes, size of CDir is about 700 bytes. These data structures are
+quite large. Please be careful if you want to add new fields to them.
+
+*OpenFileTable*
+  Open file table tracks open files and their ancestor directories. Recovering
+  MDS can easily get open files' paths, significantly reducing the time of
+  loading inodes for open files. Each entry in the table corresponds to an inode,
+  it records linkage information (parent inode and dentry name) of the inode. MDS
+  can constructs the inode's path by recursively lookup parent inode's linkage.
+  Open file table is stored in omap of RADOS objects, table entries correspond to
+  KV pairs in omap.
diff --git a/doc/dev/mds_internals/exports.rst b/doc/dev/mds_internals/exports.rst
new file mode 100644
index 000000000..c5b0e3915
--- /dev/null
+++ b/doc/dev/mds_internals/exports.rst
@@ -0,0 +1,76 @@
+
+===============
+Subtree exports
+===============
+
+Normal Migration
+----------------
+
+The exporter begins by doing some checks in export_dir() to verify
+that it is permissible to export the subtree at this time.  In
+particular, the cluster must not be degraded, the subtree root may not
+be freezing or frozen (\ie already exporting, or nested beneath
+something that is exporting), and the path must be pinned (\ie not
+conflicted with a rename).  If these conditions are met, the subtree
+freeze is initiated, and the exporter is committed to the subtree
+migration, barring an intervening failure of the importer or itself.
+
+The MExportDirDiscover serves simply to ensure that the base directory
+being exported is open on the destination node.  It is pinned by the
+importer to prevent it from being trimmed.  This occurs before the
+exporter completes the freeze of the subtree to ensure that the
+importer is able to replicate the necessary metadata.  When the
+exporter receives the MExportDirDiscoverAck, it allows the freeze to proceed.
+
+The MExportDirPrep message then follows to populate a spanning tree that
+includes all dirs, inodes, and dentries necessary to reach any nested
+exports within the exported region.  This replicates metadata as well,
+but it is pushed out by the exporter, avoiding deadlock with the
+regular discover and replication process.  The importer is responsible
+for opening the bounding directories from any third parties before
+acknowledging.  This ensures that the importer has correct dir_auth
+information about where authority is delegated for all points nested
+within the subtree being migrated.  While processing the MExportDirPrep,
+the importer freezes the entire subtree region to prevent any new
+replication or cache expiration.
+
+The warning stage occurs only if the base subtree directory is open by
+nodes other than the importer and exporter.  If so, then a
+MExportDirNotify message informs any bystanders that the authority for
+the region is temporarily ambiguous.  In particular, bystanders who
+are trimming items from their cache must send MCacheExpire messages to
+both the old and new authorities.  This is necessary to ensure that
+the surviving authority reliably receives all expirations even if the
+importer or exporter fails.  While the subtree is frozen (on both the
+importer and exporter), expirations will not be immediately processed;
+instead, they will be queued until the region is unfrozen and it can
+be determined that the node is or is not authoritative for the region.
+
+The MExportDir message sends the actual subtree metadata to the importer.
+Upon receipt, the importer inserts the data into its cache, logs a
+copy in the EImportStart, and replies with an MExportDirAck.  The exporter
+can now log an EExport, which ultimately specifies that
+the export was a success.  In the presence of failures, it is the
+existence  of the EExport that disambiguates authority during recovery.
+
+Once logged, the exporter will send an MExportDirNotify to any
+bystanders, informing them that the authority is no longer ambiguous
+and cache expirations should be sent only to the new authority (the
+importer).  Once these are acknowledged, implicitly flushing the
+bystander to exporter message streams of any stray expiration notices,
+the exporter unfreezes the subtree, cleans up its state, and sends a
+final MExportDirFinish to the importer.  Upon receipt, the importer logs
+an EImportFinish(true), unfreezes its subtree, and cleans up its
+state.
+
+
+PARTIAL FAILURE RECOVERY
+
+
+
+RECOVERY FROM JOURNAL
+
+
+
+
+
diff --git a/doc/dev/mds_internals/index.rst b/doc/dev/mds_internals/index.rst
new file mode 100644
index 000000000..c8c82ad10
--- /dev/null
+++ b/doc/dev/mds_internals/index.rst
@@ -0,0 +1,10 @@
+==============================
+MDS developer documentation
+==============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
diff --git a/doc/dev/mds_internals/locking.rst b/doc/dev/mds_internals/locking.rst
new file mode 100644
index 000000000..cfd934f3f
--- /dev/null
+++ b/doc/dev/mds_internals/locking.rst
@@ -0,0 +1,172 @@
+Ceph MDS Locker
+===============
+
+Why use locks?
+--------------
+
+Locking infrastructure in MDS is (obviously) to protect the state of various metadata. MDS has different locks covering different portions of inode and dentry. Moreover, MDS uses different kinds of locks since different metadata (in inode and dentry) have different behaviour in different situations. The MDS cache is distributed across multiple MDS ranks and across all clients. The locking infrastructure serves to ensure that all ranks and clients are consistent in their view of the file system.
+
+Data managed by the MDS can be very large to practically have the entire data set in the memory of a single metadata server. This also results in a single point of failure. The MDS therefore has a concept of Distributed Subtree Partition. A directory tree can be divided into smaller sub-trees. This is done by recording heat (access frequency) of each node in the directory tree. When a sub-tree heat reaches a configured threshold, the MDS divides the sub-tree by splitting the directory fragment. Each fragment is responsible for a part of the original directory, however, there will be a single authority node for these fragments. Each MDS can bear the read and write requests after fragmentation. If a file is very frequently accessed, the MDS will generate multiple copies distributed across active MDSs to satisfy concurrent I/Os. Typically, there are multiple clients reading and writing to files. The MDS defines locking rules for the associated metadata, e.g., metadata which is rarely modified concurrently such as UID/GID for an inode, a shared read and exclusive write access rule would suffice. However, statistics of a directory may need to be updated by multiple clients at the same time. This large directory may have been divided (fragmented) into multiple shards and different clients could write to different shards. These shards can share the read and also support simultaneous writes.
+
+Therefore, in addition to different lock types that cover different metadata pieces for an inode, the MDS has lock classes that define access rules for a particular lock type. Lock types and classes are explained further in this document.
+
+Lock Types
+----------
+
+MDS defines a handful of lock types associated with different metadata for an inode or dentry. Lock type protecting metadata for an inode and dentry are as follows::
+
+  CEPH_LOCK_DN       - dentry
+  CEPH_LOCK_DVERSION - dentry version
+  CEPH_LOCK_IVERSION - inode version
+  CEPH_LOCK_IAUTH    - mode, uid, gid
+  CEPH_LOCK_ILINK    - nlink
+  CEPH_LOCK_IDFT     - dirfragtree, frags
+  CEPH_LOCK_IFILE    - mtime, atime, size, truncate_seq, truncate_size, client_ranges, inline_data
+  CEPH_LOCK_INEST    - rstats
+  CEPH_LOCK_IXATTR   - xattrs
+  CEPH_LOCK_ISNAP    - snaps
+  CEPH_LOCK_IFLOCK   - file locks
+  CEPH_LOCK_IPOLICY  - layout, quota, export_pin, ephemeral_*
+
+.. note:: Locking rules when modifying `ctime` is a bit different - either under `versionlock` or under no specific lock at all (i.e., it can be modified with other locks held, e.g., when modifying (say) uid/gid under `CEPH_LOCK_IAUTH`).
+
+Lock Classes
+------------
+
+Lock classes define locking behaviour for the associated lock type necessary for handling distributed locks. The MDS defines 3 lock classes::
+
+  LocalLock   - Used for data that does not require distributed locking such as inode or dentry version information. Local locks are versioned locks.
+
+  SimpleLock  - Used for data that requires shared read and mutually exclusive write. This lock class is also the base class for other lock classes and specifies most of the locking behaviour for implementing distributed locks.
+
+  ScatterLock - Used for data that requires shared read and shared write. Typical use is where an MDS can delegate some authority to other MDS replicas, e.g., replica MDSs can satisfy read capabilities for clients.
+
+.. note::  In addition, MDS defines FileLock which is a special case of ScatterLock used for data that requires shared read and shared write, but also for protecting other pieces of metadata that require shared read and mutually exclusive write.
+
+Classification of lock types are as follows::
+
+   SimpleLock
+     CEPH_LOCK_DN
+     CEPH_LOCK_IAUTH
+     CEPH_LOCK_ILINK
+     CEPH_LOCK_IXATTR
+     CEPH_LOCK_ISNAP
+     CEPH_LOCK_IFLOCK
+     CEPH_LOCK_IPOLICY
+
+   ScatterLock
+     CEPH_LOCK_INEST
+     CEPH_LOCK_IDFT
+
+   FileLock
+     CEPH_LOCK_IFILE
+
+   LocalLock
+     CEPH_LOCK_DVERSION
+     CEPH_LOCK_IVERSION
+
+Read, Write and Exclusive Locks
+-------------------------------
+
+There are 3 modes in which a lock can be acquired::
+
+  rdlock - shared read lock
+  wrlock - shared write lock
+  xlock  - exclusive lock
+
+`rdlock` and `xlock` are self explanatory.
+
+`wrlock` is special since it allows concurrent writers and is valid for `ScatterLock` and `FileLock` class. From the earlier section it can be seen that `INEST` and `IDFT` are of `ScatterLock` class. `wrlock` allows multiple writers at the same time, .e.g., when a (large) directory is split into multiple shards (after fragmentation) and each shard is "assigned" to an active MDS. When new files are created under these directories, the recursive stats are independently updated on the active MDSs. Later, to fetch the updated stats, the "scattered" data is aggregated ("gathered") on the auth MDS (of the inode); which typically happens when a `rdlock` is requested on this lock type.
+
+.. note:: MDS also defines `remote_wrlock` which is primarily used during rename operations when the destination dentry is on another (active) MDS than the source MDS.
+
+Lock States and Lock State Machine
+----------------------------------
+
+MDS defines various lock states (defined in `src/mds/locks.h` source). Not all lock states are valid for a given lock class. Each lock class defines its own lock transition rules and are organized as Lock State Machines. The lock states (`LOCK_*`) are not locks themselves, but control if a lock is allowed to be taken. Each state follows `LOCK_<STATE>` or `LOCK_<FROM_STATE>_<TO_STATE>` naming terminology and can be summed up as::
+
+  LOCK_SYNC  - anybody (ANY) can read lock, no one can write lock and exclusive lock
+  LOCK_LOCK  - no one can read lock, only primary (AUTH) mds can write lock or exclusive lock
+  LOCK_MIX   - anybody (ANY) can write lock, no one can read lock or exclusive lock
+  LOCK_XLOCK - someone (client) is holding a exclusive lock
+
+The Lock Transition table (section) use the following notions::
+
+  ANY  - Auth or Replica MDS
+  AUTH - Auth MDS
+  XCL  - Auth MDS or Exclusive client
+
+Other lock states (such as `LOCK_XSYN`, `LOCK_TSYN`, etc..) are additional states that are defined as an optimization for certain client behaviour (`LOCK_XSYN` allows clients to keep the buffered writes and not flush it to the OSDs and temporarily pausing writes).
+
+Intermediate lock states (`LOCK_<FROM_STATE>_<TO_STATE>`) denote transition of a lock from one state (`<FROM_STATE>`) to another (`<TO_STATE>`).
+
+Each lock class defines its own Lock State Machine and can be found in `src/mds/locks.c` source. The state machines are explained when discussing Lock Transition in the section below.
+
+Lock Transition
+---------------
+
+Transition of lock from one state to another is mostly prompted by a (client) request or a change that the MDS is undergoing, such as tree migration. Let's consider a simple case of two clients: One client does a `stat()` (`getattr()` or `lookup()`) to fetch UID/GID of a inode, and the other client does a `setattr()` to change the UID/GID of the same inode. The first client (most likely) has `As` (iauth shared) caps issued to it by the MDS. Now, when the other client does a `setattr()` call to the MDS, the MDS adds a `xlock` to the inodes' `authlock` (`CEPH_LOCK_IAUTH`)::
+
+  Server::handle_client_setattr()
+      if (mask & (CEPH_SETATTR_MODE|CEPH_SETATTR_UID|CEPH_SETATTR_GID|CEPH_SETATTR_BTIME|CEPH_SETATTR_KILL_SGUID))
+        lov.add_xlock(&cur->authlock);
+
+Note that the MDS adds a bunch of other locks for this inode, but for now let's only work on IAUTH. Now, `CEPH_LOCK_IAUTH` is a `SimpleLock` class, and its lock transition state machine is::
+
+                      // stable     loner  rep state  r     rp   rd   wr   fwr  l    x    caps,other
+    [LOCK_SYNC]      = { 0,         false, LOCK_SYNC, ANY,  0,   ANY, 0,   0,   ANY, 0,   CEPH_CAP_GSHARED,0,0,CEPH_CAP_GSHARED },
+    [LOCK_LOCK_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, AUTH, XCL, XCL, 0,   0,   XCL, 0,   0,0,0,0 },
+    [LOCK_EXCL_SYNC] = { LOCK_SYNC, true,  LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   0,CEPH_CAP_GSHARED,0,0 },
+    [LOCK_SNAP_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, 0,    0,   0,   0,   AUTH,0,   0,   0,0,0,0 },
+
+    [LOCK_LOCK]      = { 0,         false, LOCK_LOCK, AUTH, 0,   REQ, 0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SYNC_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, ANY,  0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_EXCL_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   0,0,0,0 },
+
+    [LOCK_PREXLOCK]  = { LOCK_LOCK, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   ANY, 0,0,0,0 },
+    [LOCK_XLOCK]     = { LOCK_SYNC, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_XLOCKDONE] = { LOCK_SYNC, false, LOCK_LOCK, XCL,  XCL, XCL, 0,   0,   XCL, 0,   0,0,CEPH_CAP_GSHARED,0 },
+    [LOCK_LOCK_XLOCK]= { LOCK_PREXLOCK,false,LOCK_LOCK,0,   XCL, 0,   0,   0,   0,   XCL, 0,0,0,0 },
+
+    [LOCK_EXCL]      = { 0,         true,  LOCK_LOCK, 0,    0,   REQ, XCL, 0,   0,   0,   0,CEPH_CAP_GEXCL|CEPH_CAP_GSHARED,0,0 },
+    [LOCK_SYNC_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, ANY,  0,   0,   0,   0,   0,   0,   0,CEPH_CAP_GSHARED,0,0 },
+    [LOCK_LOCK_EXCL] = { LOCK_EXCL, false, LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   CEPH_CAP_GSHARED,0,0,0 },
+
+    [LOCK_REMOTEXLOCK]={ LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+
+The state transition entries are of type `sm_state_t` from `src/mds/locks.h` source. TODO: Describe these in detail.
+
+We reach a point where the MDS fills in `LockOpVec` and invokes
+`Locker::acquire_locks()`, which according to the lock type and the mode
+(`rdlock`, etc..) tries to acquire that particular lock. Starting state for
+the lock is `LOCK_SYNC` (this may not always be the case, but consider this
+for simplicity). To acquire `xlock` for `iauth`, the MDS refers to the state
+transition table. If the current state allows the lock to be acquired, the MDS
+grabs the lock (which is just incrementing a counter). The current state
+(`LOCK_SYNC`) does not allow `xlock` to be acquired (column `x` in `LOCK_SYNC`
+state), thereby requiring a lock state switch. At this point, the MDS switches
+to an intermediate state `LOCK_SYNC_LOCK` - signifying transitioning from
+`LOCK_SYNC` to `LOCK_LOCK` state. The intermediate state has a couple of
+purposes - a. The intermediate state defines what caps are allowed to be held
+by clients thereby revoking caps that are not allowed be held in this state,
+and b. preventing new locks to be acquired. At this point the MDS sends cap
+revoke messages to clients::
+
+  2021-11-22T07:18:20.040-0500 7fa66a3bd700  7 mds.0.locker: issue_caps allowed=pLsXsFscrl, xlocker allowed=pLsXsFscrl on [inode 0x10000000003 [2,head] /testfile auth v142 ap=1 DIRTYPARENT s=0 n(v0 rc2021-11-22T06:21:45.015746-0500 1=1+0) (iauth sync->lock) (iversion lock) caps={94134=pAsLsXsFscr/-@1,94138=pLsXsFscr/-@1} | request=1 lock=1 caps=1 dirtyparent=1 dirty=1 authpin=1 0x5633ffdac000]
+  2021-11-22T07:18:20.040-0500 7fa66a3bd700 20 mds.0.locker: client.94134 pending pAsLsXsFscr allowed pLsXsFscrl wanted -
+  2021-11-22T07:18:20.040-0500 7fa66a3bd700  7 mds.0.locker: sending MClientCaps to client.94134 seq 2 new pending pLsXsFscr was pAsLsXsFscr
+
+As seen above, `client.94134` has `As` caps, which are getting revoked by the
+MDS. After the caps have been revoked, the MDS can continue to transition to
+further states: `LOCK_SYNC_LOCK` to `LOCK_LOCK`. Since the goal is to acquire
+`xlock`, the state transition continues (as per the lock transition state
+machine)::
+
+  LOCK_LOCK -> LOCK_LOCK_XLOCK
+  LOCK_LOCK_XLOCK -> LOCK_PREXLOCK
+  LOCK_PREXLOCK -> LOCK_XLOCK
+
+finally, acquiring `xlock` on `iauth`.
+
+
+TODO: Explain locking order and path traversal locking.
diff --git a/doc/dev/mempool_accounting.rst b/doc/dev/mempool_accounting.rst
new file mode 100644
index 000000000..d86220ae8
--- /dev/null
+++ b/doc/dev/mempool_accounting.rst
@@ -0,0 +1,49 @@
+What is a mempool?
+------------------
+A memory pool (mempool) is a method for tracking memory consumption. Memory pools represent the memory consumption
+of C++ classes and containers, and they are used to assess memory leaks and other insights around memory usage with
+low overhead. Each memory pool tracks the number of bytes and items it contains. Mempools are statically declared,
+and they serve the purpose of identifying memory-related checks in BlueStore.
+
+Some common mempools that we can track
+--------------------------------------
+
+- ``bloom_filter``: tracks objects already in the cache pool in order to determine which objects in the pool are
+  being accessed
+- ``bluestore_alloc``: accounts for actual allocations done by an allocator
+- ``bluestore_inline_bl``: measures encoded length of an inline buffer
+- ``bluestore_fsck``: file system consistency check for BlueStore metadata; helps in tracking and debugging during OSD repairs
+- ``bluestore_txc``: accounts for committed transactions counter
+- ``bluestore_writing_deferred``: measures small writes which are deferred, i.e. first written into RocksDB WAL and later
+  flushed to the disk
+- ``bluestore_writing``: accounts for in-flight write buffers
+- ``bluefs``:  file-system-like interface; provides just enough functionality to allow RocksDB to store its “files� and share
+  the same raw device(s) with BlueStore
+- ``buffer_anon``: stores arbitrary buffer data
+- ``buffer_meta``: all the metadata associated with buffer anon buffers
+- ``bluestore_cache_data``: mempool for writing and writing deferred
+- ``bluestore_cache_onode``: object node (onode) metadata in the BlueStore cache
+- ``bluestore_cache_meta``: key under PREFIX_OBJ where we are stored
+- ``bluestore_cache_other``: right now accounts for:
+
+  - ``map_t``: used to track raw extents on disk for SharedBlob and for the in-memory Blob with the blob namespace
+  - ``coll_map``: collections_map
+  - ``csum_data``: checksum data
+- ``bluestore_cache_buffer``: accounts for buffer cache shards
+- ``bluestore_extent``: a logical (as well as physical) extent, pointing to some portion of a blob
+- ``bluestore_blob``: in-memory blob metadata associated cached buffers
+- ``bluestore_shared_blob``: in-memory shared blob state; stores a reference to the set of collections it belongs to
+  (includes cached buffers)
+- ``bluefs_file_reader``: accounts for bluefs file reader buffer
+- ``bluefs_file_writer``: accounts for bluefs file writer buffer
+
+Check mempools usage
+~~~~~~~~~~~~~~~~~~~~
+
+Command to see BlueStore memory allocation in these mempools::
+
+     $ ceph daemon osd.NNN dump_mempools
+
+
+.. note:: see more:
+    https://github.com/ceph/ceph/blob/main/src/include/mempool.h
diff --git a/doc/dev/messenger.rst b/doc/dev/messenger.rst
new file mode 100644
index 000000000..729538913
--- /dev/null
+++ b/doc/dev/messenger.rst
@@ -0,0 +1,33 @@
+============================
+ Messenger notes
+============================
+
+Messenger is the Ceph network layer implementation. Currently Ceph supports
+one messenger type: "async".
+
+ceph_perf_msgr
+==============
+
+ceph_perf_msgr is used to do benchmark for messenger module only and can help
+to find the bottleneck or time consuming within messenger moduleIt just like
+"iperf", we need to start server-side program firstly:
+
+# ./ceph_perf_msgr_server 172.16.30.181:10001 1 0
+
+The first argument is ip:port pair which is telling the destination address the
+client need to specified. The second argument configures the server threads. The
+third argument tells the "think time"(us) when dispatching messages. After Giant,
+CEPH_OSD_OP message which is the actual client read/write io request is fast
+dispatched without queueing to Dispatcher, in order to achieve better performance.
+So CEPH_OSD_OP message will be processed inline, "think time" is used by mock
+this "inline process" process.
+
+# ./ceph_perf_msgr_client 172.16.30.181:10001 1 32 10000 10 4096 
+
+The first argument is specified the server ip:port, and the second argument is
+used to specify client threads. The third argument specify the concurrency(the
+max inflight messages for each client thread), the fourth argument specify the
+io numbers will be issued to server per client thread. The fifth argument is
+used to indicate the "think time" for client thread when receiving messages,
+this is also used to mock the client fast dispatch process. The last argument
+specify the message data length to issue.
diff --git a/doc/dev/mon-bootstrap.rst b/doc/dev/mon-bootstrap.rst
new file mode 100644
index 000000000..8cd7e3986
--- /dev/null
+++ b/doc/dev/mon-bootstrap.rst
@@ -0,0 +1,212 @@
+===================
+ Monitor bootstrap
+===================
+
+Terminology:
+
+* ``cluster``: a set of monitors
+* ``quorum``: an active set of monitors consisting of a majority of the cluster
+
+In order to initialize a new monitor, it must always be fed:
+
+#. a logical name
+#. secret keys
+#. a cluster fsid (uuid)
+
+In addition, a monitor needs to know two things:
+
+#. what address to bind to
+#. who its peers are (if any)
+
+There are a range of ways to do both.
+
+Logical id
+==========
+
+The logical id should be unique across the cluster.  It will be
+appended to ``mon.`` to logically describe the monitor in the Ceph
+cluster.  For example, if the logical id is ``foo``, the monitor's
+name will be ``mon.foo``.
+
+For most users, there is no more than one monitor per host, which
+makes the short hostname logical choice.
+
+Secret keys
+===========
+
+The ``mon.`` secret key is stored a ``keyring`` file in the ``mon data`` directory.  It can be generated
+with a command like::
+
+        ceph-authtool --create-keyring /path/to/keyring --gen-key -n mon.
+
+When creating a new monitor cluster, the keyring should also contain a ``client.admin`` key that can be used
+to administer the system::
+
+        ceph-authtool /path/to/keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
+
+The resulting keyring is fed to ``ceph-mon --mkfs`` with the ``--keyring <keyring>`` command-line argument.
+
+Cluster fsid
+============
+
+The cluster fsid is a normal uuid, like that generated by the ``uuidgen`` command.  It
+can be provided to the monitor in two ways:
+
+#. via the ``--fsid <uuid>`` command-line argument (or config file option)
+#. via a monmap provided to the new monitor via the ``--monmap <path>`` command-line argument.
+
+Monitor address
+===============
+
+The monitor address can be provided in several ways.
+
+#. via the ``--public-addr <ip[:port]>`` command-line option (or config file option)
+#. via the ``--public-network <cidr>`` command-line option (or config file option)
+#. via the monmap provided via ``--monmap <path>``, if it includes a monitor with our name
+#. via the bootstrap monmap (provided via ``--inject-monmap <path>`` or generated from ``--mon-host <list>``) if it includes a monitor with no name (``noname-<something>``) and an address configured on the local host.
+
+Peers
+=====
+
+The monitor peers are provided in several ways:
+
+#. via the initial monmap, provided via ``--monmap <filename>``
+#. via the bootstrap monmap generated from ``--mon-host <list>``
+#. via the bootstrap monmap generated from ``[mon.*]`` sections with the deprecated ``mon addr`` options in the config file (note that this method is *not* recommended and does not support binding to both v1 and v2 protocol addresses)
+#. dynamically via the admin socket
+
+However, these methods are not completely interchangeable because of
+the complexity of creating a new monitor cluster without danger of
+races.
+
+Cluster creation
+================
+
+There are three basic approaches to creating a cluster:
+
+#. Create a new cluster by specifying the monitor names and addresses ahead of time.
+#. Create a new cluster by specifying the monitor names ahead of time, and dynamically setting the addresses as ``ceph-mon`` daemons configure themselves.
+#. Create a new cluster by specifying the monitor addresses ahead of time.
+
+
+Names and addresses
+-------------------
+
+Generate a monmap using ``monmaptool`` with the names and addresses of the initial
+monitors.  The generated monmap will also include a cluster fsid.  Feed that monmap
+to each monitor daemon::
+
+        ceph-mon --mkfs -i <name> --monmap <initial_monmap> --keyring <initial_keyring>
+
+When the daemons start, they will know exactly who they and their peers are.
+
+
+Addresses only
+--------------
+
+The initial monitor addresses can be specified with the ``mon host`` configuration value,
+either via a config file or the command-line argument.  This method has the advantage that
+a single global config file for the cluster can have a line like::
+
+     mon host = a.foo.com, b.foo.com, c.foo.com
+
+and will also serve to inform any ceph clients or daemons who the monitors are.
+
+The ``ceph-mon`` daemons will need to be fed the initial keyring and cluster fsid to 
+initialize themselves:
+
+     ceph-mon --mkfs -i <name> --fsid <uuid> --keyring <initial_keyring>
+
+When the daemons first start up, they will share their names with each other and form a
+new cluster.
+
+Names only
+----------
+
+In dynamic "cloud" environments, the cluster creator may not (yet)
+know what the addresses of the monitors are going to be.  Instead,
+they may want machines to configure and start themselves in parallel
+and, as they come up, form a new cluster on their own.  The problem is
+that the monitor cluster relies on strict majorities to keep itself
+consistent, and in order to "create" a new cluster, it needs to know
+what the *initial* set of monitors will be.
+
+This can be done with the ``mon initial members`` config option, which
+should list the ids of the initial monitors that are allowed to create
+the cluster::
+
+     mon initial members = foo, bar, baz
+
+The monitors can then be initialized by providing the other pieces of
+information (the keyring, cluster fsid, and a way of determining
+their own address).  For example::
+
+     ceph-mon --mkfs -i <name> --mon-initial-hosts 'foo,bar,baz' --keyring <initial_keyring> --public-addr <ip>
+
+When these daemons are started, they will know their own address, but
+not their peers.  They can learn those addresses via the admin socket::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hint <peer ip>
+
+Once they learn enough of their peers from the initial member set,
+they will be able to create the cluster.
+
+
+Cluster expansion
+=================
+
+Cluster expansion is slightly less demanding than creation, because
+the creation of the initial quorum is not an issue and there is no
+worry about creating separately independent clusters.
+
+New nodes can be forced to join an existing cluster in two ways:
+
+#. by providing no initial monitor peers addresses, and feeding them dynamically.
+#. by specifying the ``mon initial members`` config option to prevent the new nodes from forming a new, independent cluster, and feeding some existing monitors via any available method.
+
+Initially peerless expansion
+----------------------------
+
+Create a new monitor and give it no peer addresses other than its own.  For
+example::
+
+     ceph-mon --mkfs -i <myid> --fsid <fsid> --keyring <mon secret key> --public-addr <ip>
+
+Once the daemon starts, you can give it one or more peer addresses (preferably a bare IP address with no port; the mon will set the addr types and ports for you) to join with::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hint <peer ip>
+
+Alternatively, you can explicitly specify the addrvec_t with::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hintv <peer addrvec>
+
+For example,::
+
+     ceph daemon mon.new add_bootstrap_peer_hintv v2:1.2.3.4:3300,v1:1.2.3.4:6789
+
+This monitor will never participate in cluster creation; it can only
+join an existing cluster.
+
+Note that the address(es) specified should match exactly the addresses
+the new monitor is binding too.  If, for example, the new mon binds to
+only a v2 address but a v2 and v1 address are provided, there is some
+possibility of confusion in the mons.
+
+Expanding with initial members
+------------------------------
+
+You can feed the new monitor some peer addresses initially and avoid badness by also
+setting ``mon initial members``.  For example::
+
+     ceph-mon --mkfs -i <myid> --fsid <fsid> --keyring <mon secret key> --public-addr <ip> --mon-host foo,bar,baz
+
+When the daemon is started, ``mon initial members`` must be set via the command line or config file::
+
+     ceph-mon -i <myid> --mon-initial-members foo,bar,baz
+
+to prevent any risk of split-brain.
+  
+
+
+
+
diff --git a/doc/dev/mon-elections.rst b/doc/dev/mon-elections.rst
new file mode 100644
index 000000000..1f346aece
--- /dev/null
+++ b/doc/dev/mon-elections.rst
@@ -0,0 +1,132 @@
+.. _dev_mon_elections:
+
+=================
+Monitor Elections
+=================
+
+The Original Algorithm
+======================
+Historically, monitor leader elections have been very simple: the lowest-ranked
+monitor wins!
+
+This is accomplished using a low-state "Elector" module (though it has now
+been split into an Elector that handles message-passing, and an ElectionLogic
+that makes the voting choices). It tracks the election epoch and not much
+else. Odd epochs are elections; even epochs have a leader and let the monitor
+do its ongoing work. When a timeout occurs or the monitor asks for a
+new election, we bump the epoch and send out Propose messages to all known
+monitors.
+In general, if we receive an old message we either drop it or trigger a new
+election (if we think the sender is newly-booted and needs to join quorum). If
+we receive a message from a newer epoch, we bump up our epoch to match and
+either Defer to the Proposer or else bump the epoch again and Propose
+ourselves if we expect to win over them. When we receive a Propose within
+our current epoch, we either Defer to the sender or ignore them (we ignore them
+if they are of a higher rank than us, or higher than the rank we have already
+deferred to).
+(Note that if we have the highest rank it is possible for us to defer to every
+other monitor in sequence within the same election epoch!)
+
+This resolves under normal circumstances because all monitors agree on the
+priority voting order, and epochs are only bumped when a monitor isn't
+participating or sees a possible conflict with the known proposers.
+
+The Problems
+==============
+The original algorithm didn't work at all under a variety of netsplit
+conditions. This didn't manifest often in practice but has become
+important as the community and commercial vendors move Ceph into
+spaces requiring the use of "stretch clusters".
+
+The New Algorithms
+==================
+We still default to the original ("classic") election algorithm, but
+support letting users change to new ones via the CLI. These
+algorithms are implemented as different functions and switch statements
+within the ElectionLogic class.
+
+The first algorithm is very simple: "disallow" lets you add monitors
+to a list of disallowed leaders.
+The second, "connectivity", incorporates connection score ratings
+and elects the monitor with the best score.
+
+Algorithm: disallow
+===================
+If a monitor is in the disallowed list, it always defers to another
+monitor, no matter the rank. Otherwise, it is the same as the classic
+algorithm is.
+Since changing the disallowed list requires a paxos update, monitors
+in an election together should always have the same set. This means
+the election order is constant and static across the full monitor set
+and elections resolve trivially (assuming a connected network).
+
+This algorithm really just exists as a demo and stepping-stone to
+the more advanced connectivity mode, but it may have utility in asymmetric
+networks and clusters.
+
+Algorithm: connectivity
+=======================
+This algorithm takes as input scores for each connection
+(both ways, discussed in the next section) and attempts to elect the monitor
+with the highest total score. We keep the same basic message-passing flow as the
+classic algorithm, in which elections are driven by reacting to Propose messages.
+But this has several challenges since unlike ranks, scores are not static (and
+might change during an election!). To guarantee an election epoch does not
+produce multiple leaders, we must maintain two key invariants:
+* Monitors must maintain static scores during an election epoch
+* Any deferral must be transitive -- if A defers to B and then to C,
+B had better defer to C as well!
+
+We handle these very explicitly: by branching a copy stable_peer_tracker
+of our peer_tracker scoring object whenever starting an election (or
+bumping the epoch), and by refusing to defer to a monitor if it won't
+be deferred to by our current leader choice. (All Propose messages include
+a copy of the scores the leader is working from, so peers can evaluate them.)
+
+Of course, those modifications can easily block. To guarantee forward progress,
+we make several further adjustments:
+* If we want to defer to a new peer, but have already deferred to a peer
+whose scores don't allow that, we bump the election epoch and start()
+the election over again.
+* All election messages include the scores the sender is aware of.
+
+This guarantees we will resolve the election as long as the network is
+reasonably stable (even if disconnected): As long as all score "views"
+result in the same deferral order, an election will complete normally. And by
+broadly sharing scores across the full set of monitors, monitors rapidly
+converge on the global newest state.
+
+This algorithm has one further important feature compared to the classic and
+disallowed handlers: it can ignore out-of-quorum peers. Normally, whenever
+a monitor B receives a Propose from an out-of-quorum peer C, B will itself trigger
+a new election to give C an opportunity to join. But because the
+highest-scoring monitor A may be netsplit from C, this is not desirable. So in
+the connectivity election algorithm, B only "forwards" Propose messages when B's
+scores indicate the cluster would choose a leader other than A.
+
+Connection Scoring
+==================
+We implement scoring within the ConnectionTracker class, which is
+driven by the Elector and provided to ElectionLogic as a resource. Elector
+is responsible for sending out MMonPing messages, and for reporting the
+results in to the ConnectionTracker as calls to report_[live|dead]_connection
+with the relevant peer and the time units the call counts for. (These time units
+are seconds in the monitor, but the ConnectionTracker is agnostic and our unit
+tests count simple time steps.)
+
+We configure a "half life" and each report updates the peer's current status
+(alive or dead) and its total score. The new score is current_score * (1 - units_alive / (2 * half_life)) + (units_alive / (2 * half_life)). (For a dead report, we of course
+subtract the new delta, rather than adding it).
+
+We can further encode and decode the ConnectionTracker for wire transmission,
+and receive_peer_report()s of a full ConnectionTracker (containing all
+known scores) or a ConnectionReport (representing a single peer's scores)
+to slurp up the scores from peers. These scores are of course all versioned so
+we are in no danger of accidentally going backwards in time.
+We can query an individual connection score (if the connection is down, it's 0)
+or the total score of a specific monitor, which is the connection score from all
+other monitors going in to that one.
+
+By default, we consider pings failed after 2 seconds (mon_elector_ping_timeout)
+and ping live connections every second (mon_elector_ping_divisor). The halflife
+is 12 hours (mon_con_tracker_score_halflife).
diff --git a/doc/dev/mon-on-disk-formats.rst b/doc/dev/mon-on-disk-formats.rst
new file mode 100644
index 000000000..a64994fc0
--- /dev/null
+++ b/doc/dev/mon-on-disk-formats.rst
@@ -0,0 +1,91 @@
+##############
+ON-DISK FORMAT
+##############
+
+
+************
+UPGRADE PATH
+************
+
+On-disk formats, or even data structure formats, may be changed during an
+upgrade. Services wishing to do so, may so do it via the
+`PaxosService::upgrade_format()` call path. There is no formalized, unified
+format versioning; the `PaxosService` class keeps track of its
+`format_version` through a key in the store, assumed an `unsigned int`,  but
+it will be the service's responsibility to give meaning to those versions.
+
+AUTH MONITOR
+============
+
+versions
+--------
+
+versions are represented with a single `unsigned int`. By default, the value
+zero represents the absence of a formal upgraded format. The first format
+version was introduced in Dumpling; clusters upgrading to Dumpling saw their
+format version being increased from zero to one::
+
+  0 to 1 - introduced in v0.65, dev release for v0.67 dumpling
+  1 to 2 - introduced in v12.0.2, dev release for luminous
+  2 to 3 - introduced in mimic
+
+  0 - all clusters pre-dumpling
+  1 - all clusters dumpling+ and pre-luminous
+  2 - all clusters luminous+ and pre-mimic
+  3 - all clusters mimic+
+
+  version 1: introduces new-style monitor caps (i.e., profiles)
+  version 2: introduces mgr caps and bootstrap-mgr key
+  version 3: creates all bootstrap and admin keys if they don't yet exist
+
+callstack
+---------
+
+format_version set on `PaxosService::refresh()`::
+
+  - initially called from Monitor::refresh_from_paxos
+    - initially called from Monitor::init_paxos()
+      - initially called from Monitor::preinit()
+
+AuthMonitor::upgrade_format() called by `PaxosService::_active()`::
+
+  - called from C_Committed callback, from PaxosService::propose_pending()
+  - called from C_Active callback, from PaxosService::_active()
+  - called from PaxosService::election_finished()
+
+  - on a freshly deployed cluster, upgrade_format() will be first called
+    *after* create_initial().
+  - on an existing cluster, upgrade_format() will be called after the first
+    election.
+
+  - upgrade_format() is irrelevant on a freshly deployed cluster, as there is
+    no format to upgrade at this point.
+
+boil down
+---------
+
+* if `format_version >= current_version` then format is up-to-date, return.
+* if `features doesn't contain LUMINOUS` then `current_version = 1`
+* else if `features doesn't contain MIMIC` then `current_version = 2`
+* else `current_version = 3`
+
+if `format_version == 0`::
+
+  - upgrade to format version 1
+    - move to new-style monitor caps (i.e., profiles):
+      - set daemon profiles for existing entities
+      - set profile for existing bootstrap keys
+
+if `format_version == 1`::
+
+  - upgrade to format version 2
+    - for existing entities:
+      - add new cap for mgr
+    - for existing 'mgr' entities, fix 'mon' caps due to bug from kraken
+      setting 'allow \*', and set 'allow profile mgr' instead.
+    - add bootstrap-mgr key.
+
+if `format_version == 2`::
+
+  - upgrade to format version 3
+    - create all bootstrap keys if they don't currently exist
diff --git a/doc/dev/mon-osdmap-prune.rst b/doc/dev/mon-osdmap-prune.rst
new file mode 100644
index 000000000..6ff059b84
--- /dev/null
+++ b/doc/dev/mon-osdmap-prune.rst
@@ -0,0 +1,415 @@
+===========================
+FULL OSDMAP VERSION PRUNING
+===========================
+
+For each incremental osdmap epoch, the monitor will keep a full osdmap
+epoch in the store.
+
+While this is great when serving osdmap requests from clients, allowing
+us to fulfill their request without having to recompute the full osdmap
+from a myriad of incrementals, it can also become a burden once we start
+keeping an unbounded number of osdmaps.
+
+The monitors will attempt to keep a bounded number of osdmaps in the store.
+This number is defined (and configurable) via ``mon_min_osdmap_epochs``, and
+defaults to 500 epochs. Generally speaking, we will remove older osdmap
+epochs once we go over this limit.
+
+However, there are a few constraints to removing osdmaps. These are all
+defined in ``OSDMonitor::get_trim_to()``.
+
+In the event one of these conditions is not met, we may go over the bounds
+defined by ``mon_min_osdmap_epochs``. And if the cluster does not meet the
+trim criteria for some time (e.g., unclean pgs), the monitor may start
+keeping a lot of osdmaps. This can start putting pressure on the underlying
+key/value store, as well as on the available disk space.
+
+One way to mitigate this problem would be to stop keeping full osdmap
+epochs on disk. We would have to rebuild osdmaps on-demand, or grab them
+from cache if they had been recently served. We would still have to keep
+at least one osdmap, and apply all incrementals on top of either this
+oldest map epoch kept in the store or a more recent map grabbed from cache.
+While this would be feasible, it seems like a lot of cpu (and potentially
+IO) would be going into rebuilding osdmaps.
+
+Additionally, this would prevent the aforementioned problem going forward,
+but would do nothing for stores currently in a state that would truly
+benefit from not keeping osdmaps.
+
+This brings us to full osdmap pruning.
+
+Instead of not keeping full osdmap epochs, we are going to prune some of
+them when we have too many.
+
+Deciding whether we have too many will be dictated by a configurable option
+``mon_osdmap_full_prune_min`` (default: 10000). The pruning algorithm will be
+engaged once we go over this threshold.
+
+We will not remove all ``mon_osdmap_full_prune_min`` full osdmap epochs
+though. Instead, we are going to poke some holes in the sequence of full
+maps. By default, we will keep one full osdmap per 10 maps since the last
+map kept; i.e., if we keep epoch 1, we will also keep epoch 10 and remove
+full map epochs 2 to 9. The size of this interval is configurable with
+``mon_osdmap_full_prune_interval``.
+
+Essentially, we are proposing to keep ~10% of the full maps, but we will
+always honour the minimum number of osdmap epochs, as defined by
+``mon_min_osdmap_epochs``, and these won't be used for the count of the
+minimum versions to prune. For instance, if we have on-disk versions
+[1..50000], we would allow the pruning algorithm to operate only over
+osdmap epochs [1..49500); but, if have on-disk versions [1..10200], we
+won't be pruning because the algorithm would only operate on versions
+[1..9700), and this interval contains less versions than the minimum
+required by ``mon_osdmap_full_prune_min``.
+
+
+ALGORITHM
+=========
+
+Say we have 50,000 osdmap epochs in the store, and we're using the
+defaults for all configurable options.
+
+::
+
+    -----------------------------------------------------------
+    |1|2|..|10|11|..|100|..|1000|..|10000|10001|..|49999|50000|
+    -----------------------------------------------------------
+     ^ first                                            last ^
+
+We will prune when all the following constraints are met:
+
+1. number of versions is greater than ``mon_min_osdmap_epochs``;
+
+2. the number of versions between ``first`` and ``prune_to`` is greater (or
+   equal) than ``mon_osdmap_full_prune_min``, with ``prune_to`` being equal to
+   ``last`` minus ``mon_min_osdmap_epochs``.
+
+If any of these conditions fails, we will *not* prune any maps.
+
+Furthermore, if it is known that we have been pruning, but since then we
+are no longer satisfying at least one of the above constraints, we will
+not continue to prune. In essence, we only prune full osdmaps if the
+number of epochs in the store so warrants it.
+
+As pruning will create gaps in the sequence of full maps, we need to keep
+track of the intervals of missing maps. We do this by keeping a manifest of
+pinned maps -- i.e., a list of maps that, by being pinned, are not to be
+pruned.
+
+While pinned maps are not removed from the store, maps between two consecutive
+pinned maps will; and the number of maps to be removed will be dictated by the
+configurable option ``mon_osdmap_full_prune_interval``. The algorithm makes an
+effort to keep pinned maps apart by as many maps as defined by this option,
+but in the event of corner cases it may allow smaller intervals. Additionally,
+as this is a configurable option that is read any time a prune iteration
+occurs, there is the possibility this interval will change if the user changes
+this config option.
+
+Pinning maps is performed lazily: we will be pinning maps as we are removing
+maps. This grants us more flexibility to change the prune interval while
+pruning is happening, but also simplifies considerably the algorithm, as well
+as the information we need to keep in the manifest. Below we show a simplified
+version of the algorithm:::
+
+    manifest.pin(first)
+    last_to_prune = last - mon_min_osdmap_epochs
+
+    while manifest.get_last_pinned() + prune_interval < last_to_prune AND
+          last_to_prune - first > mon_min_osdmap_epochs AND
+          last_to_prune - first > mon_osdmap_full_prune_min AND
+          num_pruned < mon_osdmap_full_prune_txsize:
+      
+      last_pinned = manifest.get_last_pinned()
+      new_pinned = last_pinned + prune_interval
+      manifest.pin(new_pinned)
+      for e in (last_pinned .. new_pinned):
+        store.erase(e)
+        ++num_pruned
+
+In essence, the algorithm ensures that the first version in the store is
+*always* pinned. After all, we need a starting point when rebuilding maps, and
+we can't simply remove the earliest map we have; otherwise we would be unable
+to rebuild maps for the very first pruned interval.
+
+Once we have at least one pinned map, each iteration of the algorithm can
+simply base itself on the manifest's last pinned map (which we can obtain by
+reading the element at the tail of the manifest's pinned maps list).
+
+We'll next need to determine the interval of maps to be removed: all the maps
+from ``last_pinned`` up to ``new_pinned``, which in turn is nothing more than
+``last_pinned`` plus ``mon_osdmap_full_prune_interval``. We know that all maps
+between these two values, ``last_pinned`` and ``new_pinned`` can be removed,
+considering ``new_pinned`` has been pinned.
+
+The algorithm ceases to execute as soon as one of the two initial
+preconditions is not met, or if we do not meet two additional conditions that
+have no weight on the algorithm's correctness:
+
+1. We will stop if we are not able to create a new pruning interval properly
+   aligned with ``mon_osdmap_full_prune_interval`` that is lower than
+   ``last_pruned``. There is no particular technical reason why we enforce
+   this requirement, besides allowing us to keep the intervals with an
+   expected size, and preventing small, irregular intervals that would be
+   bound to happen eventually (e.g., pruning continues over the course of
+   several iterations, removing one or two or three maps each time).
+
+2. We will stop once we know that we have pruned more than a certain number of
+   maps. This value is defined by ``mon_osdmap_full_prune_txsize``, and
+   ensures we don't spend an unbounded number of cycles pruning maps. We don't
+   enforce this value religiously (deletes do not cost much), but we make an
+   effort to honor it.
+
+We could do the removal in one go, but we have no idea how long that would
+take. Therefore, we will perform several iterations, removing at most
+``mon_osdmap_full_prune_txsize`` osdmaps per iteration.
+
+In the end, our on-disk map sequence will look similar to::
+
+    ------------------------------------------
+    |1|10|20|30|..|49500|49501|..|49999|50000|
+    ------------------------------------------
+     ^ first                           last ^
+
+
+Because we are not pruning all versions in one go, we need to keep state
+about how far along on our pruning we are. With that in mind, we have
+created a data structure, ``osdmap_manifest_t``, that holds the set of pinned
+maps:::
+
+    struct osdmap_manifest_t:
+        set<version_t> pinned;
+
+Given we are only pinning maps while we are pruning, we don't need to keep
+track of additional state about the last pruned version. We know as a matter
+of fact that we have pruned all the intermediate maps between any two
+consecutive pinned maps.
+
+The question one could ask, though, is how can we be sure we pruned all the
+intermediate maps if, for instance, the monitor crashes. To ensure we are
+protected against such an event, we always write the osdmap manifest to disk
+on the same transaction that is deleting the maps. This way we have the
+guarantee that, if the monitor crashes, we will read the latest version of the
+manifest: either containing the newly pinned maps, meaning we also pruned the
+in-between maps; or we will find the previous version of the osdmap manifest,
+which will not contain the maps we were pinning at the time we crashed, given
+the transaction on which we would be writing the updated osdmap manifest was
+not applied (alongside with the maps removal).
+
+The osdmap manifest will be written to the store each time we prune, with an
+updated list of pinned maps. It is written in the transaction effectively
+pruning the maps, so we guarantee the manifest is always up to date. As a
+consequence of this criteria, the first time we will write the osdmap manifest
+is the first time we prune. If an osdmap manifest does not exist, we can be
+certain we do not hold pruned map intervals.
+
+We will rely on the manifest to ascertain whether we have pruned maps
+intervals. In theory, this will always be the on-disk osdmap manifest, but we
+make sure to read the on-disk osdmap manifest each time we update from paxos;
+this way we always ensure having an up to date in-memory osdmap manifest.
+
+Once we finish pruning maps, we will keep the manifest in the store, to
+allow us to easily find which maps have been pinned (instead of checking
+the store until we find a map). This has the added benefit of allowing us to
+quickly figure out which is the next interval we need to prune (i.e., last
+pinned plus the prune interval). This doesn't however mean we will forever
+keep the osdmap manifest: the osdmap manifest will no longer be required once
+the monitor trims osdmaps and the earliest available epoch in the store is
+greater than the last map we pruned.
+
+The same conditions from ``OSDMonitor::get_trim_to()`` that force the monitor
+to keep a lot of osdmaps, thus requiring us to prune, may eventually change
+and allow the monitor to remove some of its oldest maps.
+
+MAP TRIMMING
+------------
+
+If the monitor trims maps, we must then adjust the osdmap manifest to
+reflect our pruning status, or remove the manifest entirely if it no longer
+makes sense to keep it. For instance, take the map sequence from before, but
+let us assume we did not finish pruning all the maps.::
+
+    -------------------------------------------------------------
+    |1|10|20|30|..|490|500|501|502|..|49500|49501|..|49999|50000|
+    -------------------------------------------------------------
+     ^ first            ^ pinned.last()                   last ^
+
+    pinned = {1, 10, 20, ..., 490, 500}
+
+Now let us assume that the monitor will trim up to epoch 501. This means
+removing all maps prior to epoch 501, and updating the ``first_committed``
+pointer to ``501``. Given removing all those maps would invalidate our
+existing pruning efforts, we can consider our pruning has finished and drop
+our osdmap manifest. Doing so also simplifies starting a new prune, if all
+the starting conditions are met once we refreshed our state from the
+store.
+
+We would then have the following map sequence: ::
+
+    ---------------------------------------
+    |501|502|..|49500|49501|..|49999|50000|
+    ---------------------------------------
+     ^ first                        last ^
+
+However, imagine a slightly more convoluted scenario: the monitor will trim
+up to epoch 491. In this case, epoch 491 has been previously pruned from the
+store.
+
+Given we will always need to have the oldest known map in the store, before
+we trim we will have to check whether that map is in the prune interval
+(i.e., if said map epoch belongs to ``[ pinned.first()..pinned.last() )``).
+If so, we need to check if this is a pinned map, in which case we don't have
+much to be concerned aside from removing lower epochs from the manifest's
+pinned list. On the other hand, if the map being trimmed to is not a pinned
+map, we will need to rebuild said map and pin it, and only then will we remove
+the pinned maps prior to the map's epoch. 
+
+In this case, we would end up with the following sequence:::
+
+    -----------------------------------------------
+    |491|500|501|502|..|49500|49501|..|49999|50000|
+    -----------------------------------------------
+     ^   ^- pinned.last()                   last ^
+     `- first
+
+There is still an edge case that we should mention. Consider that we are
+going to trim up to epoch 499, which is the very last pruned epoch.
+
+Much like the scenario above, we would end up writing osdmap epoch 499 to
+the store; but what should we do about pinned maps and pruning?
+
+The simplest solution is to drop the osdmap manifest. After all, given we
+are trimming to the last pruned map, and we are rebuilding this map, we can
+guarantee that all maps greater than e 499 are sequential (because we have
+not pruned any of them). In essence, dropping the osdmap manifest in this
+case is essentially the same as if we were trimming over the last pruned
+epoch: we can prune again later if we meet the required conditions.
+
+And, with this, we have fully dwelled into full osdmap pruning. Later in this
+document one can find detailed `REQUIREMENTS, CONDITIONS & INVARIANTS` for the
+whole algorithm, from pruning to trimming. Additionally, the next section
+details several additional checks to guarantee the sanity of our configuration
+options. Enjoy.
+
+
+CONFIGURATION OPTIONS SANITY CHECKS
+-----------------------------------
+
+We perform additional checks before pruning to ensure all configuration
+options involved are sane:
+
+1. If ``mon_osdmap_full_prune_interval`` is zero we will not prune; we
+   require an actual positive number, greater than one, to be able to prune
+   maps. If the interval is one, we would not actually be pruning any maps, as
+   the interval between pinned maps would essentially be a single epoch. This
+   means we would have zero maps in-between pinned maps, hence no maps would
+   ever be pruned.
+
+2. If ``mon_osdmap_full_prune_min`` is zero we will not prune; we require a
+   positive, greater than zero, value so we know the threshold over which we
+   should prune. We don't want to guess.
+
+3. If ``mon_osdmap_full_prune_interval`` is greater than
+   ``mon_osdmap_full_prune_min`` we will not prune, as it is impossible to
+   ascertain a proper prune interval.
+
+4. If ``mon_osdmap_full_prune_txsize`` is lower than
+   ``mon_osdmap_full_prune_interval`` we will not prune; we require a
+   ``txsize`` with a value at least equal than ``interval``, and (depending on
+   the value of the latter) ideally higher.
+
+
+REQUIREMENTS, CONDITIONS & INVARIANTS
+-------------------------------------
+
+REQUIREMENTS
+~~~~~~~~~~~~
+
+* All monitors in the quorum need to support pruning.
+
+* Once pruning has been enabled, monitors not supporting pruning will not be
+  allowed in the quorum, nor will be allowed to synchronize.
+
+* Removing the osdmap manifest results in disabling the pruning feature quorum
+  requirement. This means that monitors not supporting pruning will be allowed
+  to synchronize and join the quorum, granted they support any other features
+  required.
+
+
+CONDITIONS & INVARIANTS
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* Pruning has never happened, or we have trimmed past its previous
+  intervals:::
+
+    invariant: first_committed > 1
+
+    condition: pinned.empty() AND !store.exists(manifest)
+
+
+* Pruning has happened at least once:::
+
+    invariant: first_committed > 0
+    invariant: !pinned.empty())
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+
+      precond: pinned.last() < prune_to AND
+               pinned.last() + prune_interval < prune_to
+
+     postcond: pinned.size() > old_pinned.size() AND
+               (for each v in [pinned.first()..pinned.last()]:
+                 if pinned.count(v) > 0: store.exists_full(v)
+                 else: !store.exists_full(v)
+               )
+
+
+* Pruning has finished:::
+
+    invariant: first_committed > 0
+    invariant: !pinned.empty()
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+
+    condition: pinned.last() == prune_to OR
+               pinned.last() + prune_interval < prune_to
+
+
+* Pruning intervals can be trimmed:::
+
+    precond:   OSDMonitor::get_trim_to() > 0
+
+    condition: !pinned.empty()
+
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+    invariant: pinned.first() <= OSDMonitor::get_trim_to()
+    invariant: pinned.last() >= OSDMonitor::get_trim_to()
+
+* Trim pruned intervals:::
+
+    invariant: !pinned.empty()
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+    invariant: pinned.first() <= OSDMonitor::get_trim_to()
+    invariant: pinned.last() >= OSDMonitor::get_trim_to()
+
+    postcond:  pinned.empty() OR
+               (pinned.first() == OSDMonitor::get_trim_to() AND
+                pinned.last() > pinned.first() AND
+                (for each v in [0..pinned.first()]:
+                  !store.exists(v) AND
+                  !store.exists_full(v)
+                ) AND
+                (for each m in [pinned.first()..pinned.last()]:
+                  if pinned.count(m) > 0: store.exists_full(m)
+                  else: !store.exists_full(m) AND store.exists(m)
+                )
+               )
+    postcond:  !pinned.empty() OR
+               (!store.exists(manifest) AND
+                (for each v in [pinned.first()..pinned.last()]:
+                  !store.exists(v) AND
+                  !store.exists_full(v)
+                )
+               )
+
diff --git a/doc/dev/msgr2.rst b/doc/dev/msgr2.rst
new file mode 100644
index 000000000..ecd6c8258
--- /dev/null
+++ b/doc/dev/msgr2.rst
@@ -0,0 +1,956 @@
+.. _msgr2-protocol:
+
+msgr2 protocol (msgr2.0 and msgr2.1)
+====================================
+
+This is a revision of the legacy Ceph on-wire protocol that was
+implemented by the SimpleMessenger.  It addresses performance and
+security issues.
+
+Goals
+-----
+
+This protocol revision has several goals relative to the original protocol:
+
+* *Flexible handshaking*.  The original protocol did not have a
+  sufficiently flexible protocol negotiation that allows for features
+  that were not required.
+* *Encryption*.  We will incorporate encryption over the wire.
+* *Performance*.  We would like to provide for protocol features
+  (e.g., padding) that keep computation and memory copies out of the
+  fast path where possible.
+* *Signing*.  We will allow for traffic to be signed (but not
+  necessarily encrypted).  This is not implemented.
+
+Definitions
+-----------
+
+* *client* (C): the party initiating a (TCP) connection
+* *server* (S): the party accepting a (TCP) connection
+* *connection*: an instance of a (TCP) connection between two processes.
+* *entity*: a ceph entity instantiation, e.g. 'osd.0'.  each entity
+  has one or more unique entity_addr_t's by virtue of the 'nonce'
+  field, which is typically a pid or random value.
+* *session*: a stateful session between two entities in which message
+  exchange is ordered and lossless.  A session might span multiple
+  connections if there is an interruption (TCP connection disconnect).
+* *frame*: a discrete message sent between the peers.  Each frame
+  consists of a tag (type code), payload, and (if signing
+  or encryption is enabled) some other fields.  See below for the
+  structure.
+* *tag*: a type code associated with a frame.  The tag
+  determines the structure of the payload.
+
+Phases
+------
+
+A connection has four distinct phases:
+
+#. banner
+#. authentication frame exchange
+#. message flow handshake frame exchange
+#. message frame exchange
+
+Banner
+------
+
+Both the client and server, upon connecting, send a banner::
+
+  "ceph v2\n"
+  __le16 banner payload length
+  banner payload
+
+A banner payload has the form::
+
+  __le64 peer_supported_features
+  __le64 peer_required_features
+
+This is a new, distinct feature bit namespace (CEPH_MSGR2_*).
+Currently, only CEPH_MSGR2_FEATURE_REVISION_1 is defined. It is
+supported but not required, so that msgr2.0 and msgr2.1 peers
+can talk to each other.
+
+If the remote party advertises required features we don't support, we
+can disconnect.
+
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | send banner     |
+                |----+       +----|
+                |    |       |    |
+                |    +-------+--->|
+                | send banner|    |
+                |<-----------+    |
+                |                 |
+
+Frame format
+------------
+
+After the banners are exchanged, all further communication happens
+in frames.  The exact format of the frame depends on the connection
+mode (msgr2.0-crc, msgr2.0-secure, msgr2.1-crc or msgr2.1-secure).
+All connections start in crc mode (either msgr2.0-crc or msgr2.1-crc,
+depending on peer_supported_features from the banner).
+
+Each frame has a 32-byte preamble::
+
+  __u8 tag
+  __u8 number of segments
+  {
+    __le32 segment length
+    __le16 segment alignment
+  } * 4
+  __u8 flags
+  reserved (1 byte)
+  __le32 preamble crc
+
+An empty frame has one empty segment.  A non-empty frame can have
+between one and four segments, all segments except the last may be
+empty.
+
+If there are less than four segments, unused (trailing) segment
+length and segment alignment fields are zeroed.
+
+### Currently supported flags
+
+  1. FRAME_EARLY_DATA_COMPRESSED (see :ref:`msgr-post-compression`)
+
+The reserved bytes are zeroed.
+
+The preamble checksum is CRC32-C.  It covers everything up to
+itself (28 bytes) and is calculated and verified irrespective of
+the connection mode (i.e. even if the frame is encrypted).
+
+### msgr2.0-crc mode
+
+A msgr2.0-crc frame has the form::
+
+  preamble (32 bytes)
+  {
+    segment payload
+  } * number of segments
+  epilogue (17 bytes)
+
+where epilogue is::
+
+  __u8 late_flags
+  {
+    __le32 segment crc
+  } * 4
+
+late_flags is used for frame abortion.  After transmitting the
+preamble and the first segment, the sender can fill the remaining
+segments with zeros and set a flag to indicate that the receiver must
+drop the frame.  This allows the sender to avoid extra buffering
+when a frame that is being put on the wire is revoked (i.e. yanked
+out of the messenger): payload buffers can be unpinned and handed
+back to the user immediately, without making a copy or blocking
+until the whole frame is transmitted.  Currently this is used only
+by the kernel client, see ceph_msg_revoke().
+
+The segment checksum is CRC32-C.  For "used" empty segments, it is
+set to (__le32)-1.  For unused (trailing) segments, it is zeroed.
+
+The crcs are calculated just to protect against bit errors.
+No authenticity guarantees are provided, unlike in msgr1 which
+attempted to provide some authenticity guarantee by optionally
+signing segment lengths and crcs with the session key.
+
+Issues:
+
+1. As part of introducing a structure for a generic frame with
+   variable number of segments suitable for both control and
+   message frames, msgr2.0 moved the crc of the first segment of
+   the message frame (ceph_msg_header2) into the epilogue.
+
+   As a result, ceph_msg_header2 can no longer be safely
+   interpreted before the whole frame is read off the wire.
+   This is a regression from msgr1, because in order to scatter
+   the payload directly into user-provided buffers and thus avoid
+   extra buffering and copying when receiving message frames,
+   ceph_msg_header2 must be available in advance -- it stores
+   the transaction id which the user buffers are keyed on.
+   The implementation has to choose between forgoing this
+   optimization or acting on an unverified segment.
+
+2. late_flags is not covered by any crc.  Since it stores the
+   abort flag, a single bit flip can result in a completed frame
+   being dropped (causing the sender to hang waiting for a reply)
+   or, worse, in an aborted frame with garbage segment payloads
+   being dispatched.
+
+   This was the case with msgr1 and got carried over to msgr2.0.
+
+### msgr2.1-crc mode
+
+Differences from msgr2.0-crc:
+
+1. The crc of the first segment is stored at the end of the
+   first segment, not in the epilogue.  The epilogue stores up to
+   three crcs, not up to four.
+
+   If the first segment is empty, (__le32)-1 crc is not generated.
+
+2. The epilogue is generated only if the frame has more than one
+   segment (i.e. at least one of second to fourth segments is not
+   empty).  Rationale: If the frame has only one segment, it cannot
+   be aborted and there are no crcs to store in the epilogue.
+
+3. Unchecksummed late_flags is replaced with late_status which
+   builds in bit error detection by using a 4-bit nibble per flag
+   and two code words that are Hamming Distance = 4 apart (and not
+   all zeros or ones).  This comes at the expense of having only
+   one reserved flag, of course.
+
+Some example frames:
+
+* A 0+0+0+0 frame (empty, no epilogue)::
+
+    preamble (32 bytes)
+
+* A 20+0+0+0 frame (no epilogue)::
+
+    preamble (32 bytes)
+    segment1 payload (20 bytes)
+    __le32 segment1 crc
+
+* A 0+70+0+0 frame::
+
+    preamble (32 bytes)
+    segment2 payload (70 bytes)
+    epilogue (13 bytes)
+
+* A 20+70+0+350 frame::
+
+    preamble (32 bytes)
+    segment1 payload (20 bytes)
+    __le32 segment1 crc
+    segment2 payload (70 bytes)
+    segment4 payload (350 bytes)
+    epilogue (13 bytes)
+
+where epilogue is::
+
+  __u8 late_status
+  {
+    __le32 segment crc
+  } * 3
+
+Hello
+-----
+
+* TAG_HELLO: client->server and server->client::
+
+    __u8 entity_type
+    entity_addr_t peer_socket_address
+
+  - We immediately share our entity type and the address of the peer (which can be useful
+    for detecting our effective IP address, especially in the presence of NAT).
+
+
+Authentication
+--------------
+
+* TAG_AUTH_REQUEST: client->server::
+
+    __le32 method;  // CEPH_AUTH_{NONE, CEPHX, ...}
+    __le32 num_preferred_modes;
+    list<__le32> mode  // CEPH_CON_MODE_*
+    method specific payload
+
+* TAG_AUTH_BAD_METHOD server -> client: reject client-selected auth method::
+
+    __le32 method
+    __le32 negative error result code
+    __le32 num_methods
+    list<__le32> allowed_methods // CEPH_AUTH_{NONE, CEPHX, ...}
+    __le32 num_modes
+    list<__le32> allowed_modes   // CEPH_CON_MODE_*
+
+  - Returns the attempted auth method, and error code (-EOPNOTSUPP if
+    the method is unsupported), and the list of allowed authentication
+    methods.
+
+* TAG_AUTH_REPLY_MORE: server->client::
+
+    __le32 len;
+    method specific payload
+
+* TAG_AUTH_REQUEST_MORE: client->server::
+
+    __le32 len;
+    method specific payload
+
+* TAG_AUTH_DONE: (server->client)::
+
+    __le64 global_id
+    __le32 connection mode // CEPH_CON_MODE_*
+    method specific payload
+
+  - The server is the one to decide authentication has completed and what
+    the final connection mode will be.
+
+
+Example of authentication phase interaction when the client uses an
+allowed authentication method:
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |        auth more|
+                |                 |
+                |auth more        |
+                |---------------->|
+                |<----------------|
+                |        auth done|
+
+
+Example of authentication phase interaction when the client uses a forbidden
+authentication method as the first attempt:
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |   bad method    |
+                |                 |
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |        auth more|
+                |                 |
+                | auth more       |
+                |---------------->|
+                |<----------------|
+                |        auth done|
+
+
+Post-auth frame format
+----------------------
+
+Depending on the negotiated connection mode from TAG_AUTH_DONE, the
+connection either stays in crc mode or switches to the corresponding
+secure mode (msgr2.0-secure or msgr2.1-secure).
+
+### msgr2.0-secure mode
+
+A msgr2.0-secure frame has the form::
+
+  {
+    preamble (32 bytes)
+    {
+      segment payload
+      zero padding (out to 16 bytes)
+    } * number of segments
+    epilogue (16 bytes)
+  } ^ AES-128-GCM cipher
+  auth tag (16 bytes)
+
+where epilogue is::
+
+    __u8 late_flags
+    zero padding (15 bytes)
+
+late_flags has the same meaning as in msgr2.0-crc mode.
+
+Each segment and the epilogue are zero padded out to 16 bytes.
+Technically, GCM doesn't require any padding because Counter mode
+(the C in GCM) essentially turns a block cipher into a stream cipher.
+But, if the overall input length is not a multiple of 16 bytes, some
+implicit zero padding would occur internally because GHASH function
+used by GCM for generating auth tags only works on 16-byte blocks.
+
+Issues:
+
+1. The sender encrypts the whole frame using a single nonce
+   and generating a single auth tag.  Because segment lengths are
+   stored in the preamble, the receiver has no choice but to decrypt
+   and interpret the preamble without verifying the auth tag -- it
+   can't even tell how much to read off the wire to get the auth tag
+   otherwise!  This creates a decryption oracle, which, in conjunction
+   with Counter mode malleability, could lead to recovery of sensitive
+   information.
+
+   This issue extends to the first segment of the message frame as
+   well.  As in msgr2.0-crc mode, ceph_msg_header2 cannot be safely
+   interpreted before the whole frame is read off the wire.
+
+2. Deterministic nonce construction with a 4-byte counter field
+   followed by an 8-byte fixed field is used.  The initial values are
+   taken from the connection secret -- a random byte string generated
+   during the authentication phase.  Because the counter field is
+   only four bytes long, it can wrap and then repeat in under a day,
+   leading to GCM nonce reuse and therefore a potential complete
+   loss of both authenticity and confidentiality for the connection.
+   This was addressed by disconnecting before the counter repeats
+   (CVE-2020-1759).
+
+### msgr2.1-secure mode
+
+Differences from msgr2.0-secure:
+
+1. The preamble, the first segment and the rest of the frame are
+   encrypted separately, using separate nonces and generating
+   separate auth tags.  This gets rid of unverified plaintext use
+   and keeps msgr2.1-secure mode close to msgr2.1-crc mode, allowing
+   the implementation to receive message frames in a similar fashion
+   (little to no buffering, same scatter/gather logic, etc).
+
+   In order to reduce the number of en/decryption operations per
+   frame, the preamble is grown by a fixed size inline buffer (48
+   bytes) that the first segment is inlined into, either fully or
+   partially.  The preamble auth tag covers both the preamble and the
+   inline buffer, so if the first segment is small enough to be fully
+   inlined, it becomes available after a single decryption operation.
+
+2. As in msgr2.1-crc mode, the epilogue is generated only if the
+   frame has more than one segment.  The rationale is even stronger,
+   as it would require an extra en/decryption operation.
+
+3. For consistency with msgr2.1-crc mode, late_flags is replaced
+   with late_status (the built-in bit error detection isn't really
+   needed in secure mode).
+
+4. In accordance with `NIST Recommendation for GCM`_, deterministic
+   nonce construction with a 4-byte fixed field followed by an 8-byte
+   counter field is used.  An 8-byte counter field should never repeat
+   but the nonce reuse protection put in place for msgr2.0-secure mode
+   is still there.
+
+   The initial values are the same as in msgr2.0-secure mode.
+
+   .. _`NIST Recommendation for GCM`: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf
+
+As in msgr2.0-secure mode, each segment is zero padded out to
+16 bytes.  If the first segment is fully inlined, its padding goes
+to the inline buffer.  Otherwise, the padding is on the remainder.
+The corollary to this is that the inline buffer is consumed in
+16-byte chunks.
+
+The unused portion of the inline buffer is zeroed.
+
+Some example frames:
+
+* A 0+0+0+0 frame (empty, nothing to inline, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      zero padding (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 20+0+0+0 frame (first segment fully inlined, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (20 bytes)
+      zero padding (28 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 0+70+0+0 frame (nothing to inline)::
+
+    {
+      preamble (32 bytes)
+      zero padding (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 20+70+0+350 frame (first segment fully inlined)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (20 bytes)
+      zero padding (28 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      segment4 payload (350 bytes)
+      zero padding (2 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 105+0+0+0 frame (first segment partially inlined, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment1 payload remainder (57 bytes)
+      zero padding (7 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 105+70+0+350 frame (first segment partially inlined)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment1 payload remainder (57 bytes)
+      zero padding (7 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      segment4 payload (350 bytes)
+      zero padding (2 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+where epilogue is::
+
+    __u8 late_status
+    zero padding (15 bytes)
+
+late_status has the same meaning as in msgr2.1-crc mode.
+
+Compression
+-----------
+Compression handshake is implemented using msgr2 feature-based handshaking.
+In this phase, the client will indicate the server if on-wire-compression can be used for message transmitting, 
+in addition to the list of supported compression methods. If on-wire-compression is enabled for both client and server, 
+the server will choose a compression method based on client's request and its' own preferences. 
+Once the handshake is completed, both peers have setup their compression handlers (if desired). 
+
+* TAG_COMPRESSION_REQUEST (client->server): declares compression capabilities and requirements::
+
+    bool  is_compress
+    std::vector<uint32_t> preferred_methods 
+
+  - if the client identifies that both peers support compression feature, it initiates the handshake.
+  - is_compress flag indicates whether the client's configuration is to use compression.
+  - preferred_methods is a list of compression algorithms that are supported by the client.
+
+* TAG_COMPRESSION_DONE (server->client) : determines on compression settings::
+
+    bool is_compress
+    uint32_t  method
+
+  - the server determines whether compression is possible according to the configuration.
+  - if it is possible, it will pick the most prioritized compression method that is also supported by the client.
+  - if none exists, it will determine that session between the peers will be handled without compression.
+
+.. ditaa::
+
+           +---------+              +--------+
+           | Client  |              | Server |
+           +---------+              +--------+
+                | compression request    |
+                |----------------------->|
+                |<-----------------------|
+                |   compression done     |
+
+# msgr2.x-secure mode
+
+Combining compression with encryption introduces security implications.
+Compression will not be possible when using secure mode, unless configured specifically by an admin. 
+
+.. _msgr-post-compression:
+
+Post-compression frame format 
+-----------------------------
+Depending on the negotiated connection mode from TAG_COMPRESSION_DONE, the connection is able to accept/send compressed frames or process all frames as decompressed.
+
+# msgr2.x-force mode
+
+All subsequent frames that will be sent via the connection will be compressed if compression requirements are met (e.g, the frames size).
+
+For compressed frames, the sending peer will enable the FRAME_EARLY_DATA_COMPRESSED flag, thus allowing the accepting peer to detect it and decompress the frame.
+
+# msgr2.x-none mode
+
+FRAME_EARLY_DATA_COMPRESSED flag will be disabled in preamble.
+
+
+Message flow handshake
+----------------------
+
+In this phase the peers identify each other and (if desired) reconnect to
+an established session.
+
+* TAG_CLIENT_IDENT (client->server): identify ourselves::
+
+    __le32 num_addrs
+    entity_addrvec_t*num_addrs entity addrs
+    entity_addr_t target entity addr
+    __le64 gid (numeric part of osd.0, client.123456, ...)
+    __le64 global_seq
+    __le64 features supported (CEPH_FEATURE_* bitmask)
+    __le64 features required (CEPH_FEATURE_* bitmask)
+    __le64 flags (CEPH_MSG_CONNECT_* bitmask)
+    __le64 cookie
+
+  - client will send first, server will reply with same.  if this is a
+    new session, the client and server can proceed to the message exchange.
+  - the target addr is who the client is trying to connect *to*, so
+    that the server side can close the connection if the client is
+    talking to the wrong daemon.
+  - type.gid (entity_name_t) is set here, by combining the type shared in the hello
+    frame with the gid here.  this means we don't need it
+    in the header of every message.  it also means that we can't send
+    messages "from" other entity_name_t's.  the current
+    implementations set this at the top of _send_message etc so this
+    shouldn't break any existing functionality.  implementation will
+    likely want to mask this against what the authenticated credential
+    allows.
+  - cookie is the client cookie used to identify a session, and can be used
+    to reconnect to an existing session.
+  - we've dropped the 'protocol_version' field from msgr1
+
+* TAG_IDENT_MISSING_FEATURES (server->client): complain about a TAG_IDENT
+  with too few features::
+
+    __le64 features we require that the peer didn't advertise
+
+* TAG_SERVER_IDENT (server->client): accept client ident and identify server::
+
+    __le32 num_addrs
+    entity_addrvec_t*num_addrs entity addrs
+    __le64 gid (numeric part of osd.0, client.123456, ...)
+    __le64 global_seq
+    __le64 features supported (CEPH_FEATURE_* bitmask)
+    __le64 features required (CEPH_FEATURE_* bitmask)
+    __le64 flags (CEPH_MSG_CONNECT_* bitmask)
+    __le64 cookie
+
+  - The server cookie can be used by the client if it is later disconnected
+    and wants to reconnect and resume the session.
+
+* TAG_RECONNECT (client->server): reconnect to an established session::
+
+    __le32 num_addrs
+    entity_addr_t * num_addrs
+    __le64 client_cookie
+    __le64 server_cookie
+    __le64 global_seq
+    __le64 connect_seq
+    __le64 msg_seq (the last msg seq received)
+
+* TAG_RECONNECT_OK (server->client): acknowledge a reconnect attempt::
+
+    __le64 msg_seq (last msg seq received)
+
+  - once the client receives this, the client can proceed to message exchange.
+  - once the server sends this, the server can proceed to message exchange.
+
+* TAG_RECONNECT_RETRY_SESSION (server only): fail reconnect due to stale connect_seq
+
+* TAG_RECONNECT_RETRY_GLOBAL (server only): fail reconnect due to stale global_seq
+
+* TAG_RECONNECT_WAIT (server only): fail reconnect due to connect race.
+
+  - Indicates that the server is already connecting to the client, and
+    that direction should win the race.  The client should wait for that
+    connection to complete.
+
+* TAG_RESET_SESSION (server only): ask client to reset session::
+
+      __u8 full
+
+  - full flag indicates whether peer should do a full reset, i.e., drop
+    message queue.
+
+
+Example of failure scenarios:
+
+* First client's client_ident message is lost, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------X       |
+                |                     |
+                | client_ident(a)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                | session established |
+                |                     |
+
+
+* Server's server_ident message is lost, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------------->|
+                |        X------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                |                     |
+                | client_ident(a)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(c) | s_cookie(c)
+                |                     |
+                | session established |
+                |                     |
+
+
+* Server's server_ident message is lost, and then server reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------------->|
+                |        X------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                |                     |
+                |     reconnect(a, b) |
+                |<--------------------|
+                |-------------------->|
+                | reset_session(F)    |
+                |                     |
+                |     client_ident(a) | c_cookie(a)
+                |<--------------------|
+                |-------------------->|
+    s_cookie(c) | server_ident(c)     |
+                |                     |
+
+
+* Connection failure after session is established, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+                |        X------------|
+                |                     |
+                | reconnect(a, b)     |
+                |-------------------->|
+                |<--------------------|
+                |        reconnect_ok |
+                |                     |
+
+
+* Connection failure after session is established because server reset,
+  and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+                |        X------------| reset
+                |                     |
+                | reconnect(a, b)     |
+                |-------------------->|
+                |<--------------------|
+                |  reset_session(RC*) |
+                |                     |
+    c_cookie(c) | client_ident(c)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(d) | s_cookie(d)
+                |                     |
+
+RC* means that the reset session full flag depends on the policy.resetcheck
+of the connection.
+
+
+* Connection failure after session is established because client reset,
+  and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+          reset |        X------------|
+                |                     |
+    c_cookie(c) | client_ident(c)     |
+                |-------------------->|
+                |<--------------------| reset if policy.resetcheck
+                |     server_ident(d) | s_cookie(d)
+                |                     |
+
+
+Message exchange
+----------------
+
+Once a session is established, we can exchange messages.
+
+* TAG_MSG: a message::
+
+    ceph_msg_header2
+    front
+    middle
+    data_pre_padding
+    data
+
+  - The ceph_msg_header2 is modified from ceph_msg_header:
+      * include an ack_seq.  This avoids the need for a TAG_ACK
+        message most of the time.
+      * remove the src field, which we now get from the message flow
+        handshake (TAG_IDENT).
+      * specifies the data_pre_padding length, which can be used to
+        adjust the alignment of the data payload.  (NOTE: is this is
+        useful?)
+
+* TAG_ACK: acknowledge receipt of message(s)::
+
+    __le64 seq
+
+  - This is only used for stateful sessions.
+
+* TAG_KEEPALIVE2: check for connection liveness::
+
+    ceph_timespec stamp
+
+  - Time stamp is local to sender.
+
+* TAG_KEEPALIVE2_ACK: reply to a keepalive2::
+
+    ceph_timestamp stamp
+
+  - Time stamp is from the TAG_KEEPALIVE2 we are responding to.
+
+* TAG_CLOSE: terminate a connection
+
+  Indicates that a connection should be terminated. This is equivalent
+  to a hangup or reset (i.e., should trigger ms_handle_reset).  It
+  isn't strictly necessary or useful as we could just disconnect the
+  TCP connection.
+
+
+Example of protocol interaction (WIP)
+_____________________________________
+
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | send banner       |
+                |----+       +------|
+                |    |       |      |
+                |    +-------+----->|
+                | send banner|      |
+                |<-----------+      |
+                |                   |
+                |  send new stream  |
+                |------------------>|
+                | auth request      |
+                |------------------>|
+                |<------------------|
+                |   bad method      |
+                |                   |
+                | auth request      |
+                |------------------>|
+                |<------------------|
+                |         auth more |
+                |                   |
+                | auth more         |
+                |------------------>|
+                |<------------------|
+                |         auth done |
+                |                   |
+
+
+.. graphviz::
+   :caption: client side state machine
+
+   digraph lossy_client {
+     node [shape = doublecircle]; "start_connect" "closed";
+     node [shape = oval];
+     start_connect -> banner_connecting [label = "<connected>"];
+     subgraph hello_banner {
+       banner_connecting -> hello_connecting [label = "banner exchange"];
+       hello_connecting -> banner_connecting [label = "hello exchange"];
+       label = "hello banner exchange";
+       color = blue;
+     }
+     banner_connecting -> auth_connecting [label = "<exchange done>"];
+     auth_connecting -> auth_connecting [label = "auth reply more"];
+     auth_connecting -> auth_connecting [label = "auth bad method"];
+     auth_connecting -> auth_connecting_sign [label = "auth done"];
+     auth_connecting_sign -> session_connecting [label = "auth signature"];
+     session_connecting -> wait [label = "wait"];
+     wait -> start_connect [label = "<backoff>"];
+     session_connecting -> closed [label = "ident missing features"];
+     session_connecting -> ready [label = "server ident", tooltip = "set peer_name, peer_addr and connection features"];
+     ready -> ready [label = "keep alive"];
+   }
+
+.. graphviz::
+   :caption: server side state machine
+
+   digraph lossy_server {
+     node [shape = doublecircle]; "start_accept" "closed";
+     node [shape = oval];
+     start_accept -> banner_accepting [label = "<accepted>"];
+     subgraph hello_banner {
+       banner_accepting -> hello_accepting [label = "banner exchange"];
+       hello_accepting -> banner_accepting [label = "hello exchange"];
+       label = "hello banner exchange";
+       color = blue;
+     };
+     banner_accepting -> auth_accepting [label = "<exchange done>"];
+     auth_accepting -> auth_accepting_more [label = "auth_request => 0"];
+     auth_accepting -> auth_accepting_sign [label = "auth_request => 1"];
+     auth_accepting_more -> auth_accepting_more [label = "auth_request => 0"];
+     auth_accepting_more -> auth_accepting_sign [label = "auth_request => 1"];
+     auth_accepting_more -> standby [label = "auth_request => EBUSY"];
+     auth_accepting_more -> auth_accepting_more [label = "auth_request => *"];
+     auth_accepting -> standby [label = "auth_request => EBUSY"];
+     auth_accepting -> auth_accepting [label = "send <auth bad method>"];
+     auth_accepting_sign -> session_accepting [label = "auth signature"];
+     session_accepting -> session_accepting [label = "reconnect"];
+     session_accepting -> closed [label = "ident missing features"];
+     session_accepting -> ready [label = "client ident", tooltip = "set connection features"];
+     ready -> ready [label = "keep alive"];
+   }
diff --git a/doc/dev/network-encoding.rst b/doc/dev/network-encoding.rst
new file mode 100644
index 000000000..d5d1a6d15
--- /dev/null
+++ b/doc/dev/network-encoding.rst
@@ -0,0 +1,215 @@
+==================
+ Network Encoding
+==================
+
+This describes the encoding used to serialize data.  It doesn't cover specific
+objects/messages but focuses on the base types.
+
+The types are not self documenting in any way.  They can not be decoded unless
+you know what they are.
+
+Conventions
+===========
+
+Integers
+--------
+
+The integer types used will be named ``{signed}{size}{endian}``.  For example
+``u16le`` is an unsigned 16 bit integer encoded in little endian byte order
+while ``s64be`` is a signed 64 bit integer in big endian.  Additionally ``u8``
+and ``s8`` will represent signed and unsigned bytes respectively.  Signed
+integers use two's complement encoding.
+
+Complex Types
+-------------
+
+This document will use a c-like syntax for describing structures.  The
+structure represents the data that will go over the wire.  There will be no
+padding between the elements and the elements will be sent in the order they
+appear.  For example::
+
+	struct foo {
+		u8    tag;
+		u32le data;
+	}
+
+When encoding the values ``0x05`` and ``0x12345678`` respectively will appear on
+the wire as ``05 78 56 34 12``.
+
+Variable Arrays
+---------------
+
+Unlike c, length arrays can be used anywhere in structures and will be inline in
+the protocol.  Furthermore the length may be described using an earlier item in
+the structure.
+
+::
+	
+	struct blob {
+		u32le size;
+		u8    data[size];
+		u32le checksum;
+	}
+
+This structure is encoded as a 32 bit size, followed by ``size`` data bytes,
+then a 32 bit checksum.
+
+Primitive Aliases
+-----------------
+
+These types are just aliases for primitive types.
+
+::
+	
+	// From /src/include/types.h
+	
+	typedef u32le epoch_t;
+	typedef u32le ceph_seq_t;
+	typedef u64le ceph_tid_t;
+	typedef u64le version_t;
+
+
+Structures
+==========
+
+These are the way structures are encoded.  Note that these structures don't
+actually exist in the source but are the way that different types are encoded.
+
+Optional
+--------
+
+Optionals are represented as a presence byte, followed by the item if it exists.
+
+::
+	
+	struct ceph_optional<T> {
+		u8 present;
+		T  element[present? 1 : 0]; // Only if present is non-zero.
+	}
+
+Optionals are used to encode ``boost::optional`` and, since introducing
+C++17 to Ceph, ``std::optional``.
+
+Pair
+----
+
+Pairs are simply the first item followed by the second.
+
+::
+	
+	struct ceph_pair<A,B> {
+		A a;
+		B b;
+	}
+
+Pairs are used to encode ``std::pair``.
+
+Triple
+------
+
+Triples are simply the tree elements one after another.
+
+::
+	
+	struct ceph_triple<A,B,C> {
+		A a;
+		B b;
+		C c;
+	}
+
+Triples are used to encode ``ceph::triple``.
+
+
+List
+----
+
+Lists are represented as an element count followed by that many elements.
+
+::
+	
+	struct ceph_list<T> {
+		u32le length;
+		T     elements[length];
+	}
+
+.. note::
+	The size of the elements in the list are not necessarily uniform.
+
+Lists are used to encode ``std::list``, ``std::vector``, ``std::deque``,
+``std::set`` and ``ceph::unordered_set``.
+
+Blob
+----
+
+A Blob is simply a list of bytes.
+
+::
+	
+	struct ceph_string {
+		ceph_list<u8>;
+	}
+	
+	// AKA
+	
+	struct ceph_string {
+		u32le size;
+		u8    data[size];
+	}
+
+Blobs are used to encode ``std::string``, ``const char *`` and ``bufferlist``.
+
+.. note::
+	The content of a Blob is arbitrary binary data.
+
+Map
+---
+
+Maps are a list of pairs.
+
+::
+	
+	struct ceph_map<K,V> {
+		ceph_list<ceph_pair<K,V>>;
+	}
+	
+	// AKA
+	
+	struct ceph_map<K,V> {
+		u32le length;
+		ceph_pair<K,V> entries[length];
+	}
+
+Maps are used to encode ``std::map``, ``std::multimap`` and
+``ceph::unordered_map``.
+
+Complex Types
+=============
+
+These aren't hard to find in the source but the common ones are listed here for
+convenience.
+
+utime_t
+-------
+
+::
+	
+	// From /src/include/utime.h
+	struct utime_t {
+		u32le tv_sec;  // Seconds since epoch.
+		u32le tv_nsec; // Nanoseconds since the last second.
+	}
+
+ceph_entity_name
+----------------
+
+::
+	
+	// From /src/include/msgr.h
+	struct ceph_entity_name {
+		u8    type; // CEPH_ENTITY_TYPE_*
+		u64le num;
+	}
+	
+	// CEPH_ENTITY_TYPE_* defined in /src/include/msgr.h
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/network-protocol.rst b/doc/dev/network-protocol.rst
new file mode 100644
index 000000000..d766a3211
--- /dev/null
+++ b/doc/dev/network-protocol.rst
@@ -0,0 +1,197 @@
+==================
+ Network Protocol
+==================
+
+This file describes the network protocol used by Ceph.  In order to understand
+the way the structures are defined it is recommended to read the introduction
+of :doc:`/dev/network-encoding` first.
+
+Hello
+=====
+
+The protocol starts with a handshake that confirms that both nodes are talking
+ceph and shares some basic information.
+
+Banner
+------
+
+The first action is the server sending banner to the client.  The banner is
+defined in ``CEPH_BANNER`` from ``src/include/msgr.h``.  This is followed by
+the server's then client's address each encoded as a ``entity_addr_t``.
+
+Once the client verifies that the servers banner matches its own it replies with
+its banner and its address.
+
+Connect
+-------
+
+Once the banners have been verified and the addresses exchanged the connection
+negotiation begins.  First the client sends a ``ceph_msg_connect`` structure
+with its information.
+
+::
+	
+	// From src/include/msgr.h
+	struct ceph_msg_connect {
+		u64le features;            // Supported features (CEPH_FEATURE_*)
+		u32le host_type;           // CEPH_ENTITY_TYPE_*
+		u32le global_seq;          // Number of connections initiated by this host.
+		u32le connect_seq;         // Number of connections initiated in this session.
+		u32le protocol_version;
+		u32le authorizer_protocol;
+		u32le authorizer_len;
+		u8    flags;               // CEPH_MSG_CONNECT_*
+		u8    authorizer[authorizer_len];
+	}
+
+Connect Reply
+-------------
+
+Once the connect has been sent the connection has effectively been opened,
+however the first message the server sends must be a connect reply message.
+
+::
+	
+	struct ceph_msg_connect_reply {
+		u8    tag; // Tag indicating response code.
+		u64le features;
+		u32le global_seq;
+		u32le connect_seq;
+		u32le protocol_version;
+		u32le authorizer_len;
+		u8    flags;
+		u8    authorizer[authorizer_len];
+	}
+
+MSGR Protocol
+=============
+
+This is a low level protocol over which messages are delivered.  The messages
+at this level consist of a tag byte, identifying the type of message, followed
+by the message data.
+
+::
+	
+	// Virtual structure.
+	struct {
+		u8 tag; // CEPH_MSGR_TAG_*
+		u8 data[]; // Length depends on tag and data.
+	}
+
+The length of ``data`` is determined by the tag byte and depending on the
+message type via information in the ``data`` array itself.
+
+.. note::
+	There is no way to determine the length of the message if you do not
+	understand the type of message.
+
+The message tags are defined in ``src/include/msgr.h`` and the current ones
+are listed below along with the data they include.  Note that the defined
+structures don't exist in the source and are merely for representing the
+protocol.
+
+CEPH_MSGR_TAG_CLOSE (0x06)
+--------------------------
+
+::
+	
+	struct ceph_msgr_close {
+		u8 tag = 0x06;
+		u8 data[0]; // No data.
+	}
+
+The close message indicates that the connection is being closed.
+
+CEPH_MSGR_TAG_MSG (0x07)
+------------------------
+
+::
+	
+	struct ceph_msgr_msg {
+		u8 tag = 0x07;
+		ceph_msg_header header;
+		u8 front [header.front_len ];
+		u8 middle[header.middle_len];
+		u8 data  [header.data_len  ];
+		ceph_msg_footer footer;
+	}
+	
+	// From src/include/msgr.h
+	struct ceph_msg_header {
+		u64le seq;       // Sequence number.
+		u64le tid;       // Transaction ID.
+		u16le type;      // Message type (CEPH_MSG_* or MSG_*).
+		u16le priority;  // Priority (higher is more important).
+		u16le version;   // Version of message encoding.
+		
+		u32le front_len;  // The size of the front section.
+		u32le middle_len; // The size of the middle section.
+		u32le data_len;   // The size of the data section.
+		u16le data_off;   // The way data should be aligned by the receiver.
+		
+		ceph_entity_name src; // Information about the sender.
+		
+		u16le compat_version; // Oldest compatible encoding version.
+		u16le reserved;       // Unused.
+		u32le crc;            // CRC of header.
+	}
+	
+	// From src/include/msgr.h
+	struct ceph_msg_footer {
+		u32le front_crc;  // Checksums of the various sections.
+		u32le middle_crc; //
+		u32le data_crc;   //
+		u64le sig; // Cryptographic signature.
+		u8    flags;
+	}
+
+Messages are the business logic of Ceph.  They are what is used to send data and
+requests between nodes.  The message header contains the length of the message
+so unknown messages can be handled gracefully.
+
+There are two names for the message type constants ``CEPH_MSG_*`` and ``MSG_*``.
+The only difference between the two is that the first are considered "public"
+while the second is for internal use only.  There is no protocol-level
+difference.
+
+CEPH_MSGR_TAG_ACK (0x08)
+------------------------
+
+::
+	
+	struct ceph_msgr_ack {
+		u8    tag = 0x08;
+		u64le seq; // The sequence number of the message being acknowledged.
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE (0x09)
+------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive {
+		u8 tag = 0x09;
+		u8 data[0]; // No data.
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE2 (0x0E)
+-------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive2 {
+		u8      tag = 0x0E;
+		utime_t timestamp;
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE2_ACK (0x0F)
+-----------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive2_ack {
+		u8      tag = 0x0F;
+		utime_t timestamp;
+	}
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/object-store.rst b/doc/dev/object-store.rst
new file mode 100644
index 000000000..73ea148bb
--- /dev/null
+++ b/doc/dev/object-store.rst
@@ -0,0 +1,62 @@
+====================================
+ Object Store Architecture Overview
+====================================
+
+.. graphviz::
+
+   digraph object_store {
+    size="7,7";
+    node [color=lightblue2, style=filled, fontname="Serif"];
+
+    "testrados" -> "librados"
+    "testradospp" -> "librados"
+
+    "rbd" -> "librados"
+
+    "radostool" -> "librados"
+
+    "radosgw-admin" -> "radosgw"
+
+    "radosgw" -> "librados"
+
+    "radosacl" -> "librados"
+
+    "librados" -> "objecter"
+
+    "ObjectCacher" -> "Filer"
+
+    "dumpjournal" -> "Journaler"
+
+    "Journaler" -> "Filer"
+
+    "SyntheticClient" -> "Filer"
+    "SyntheticClient" -> "objecter"
+
+    "Filer" -> "objecter"
+
+    "objecter" -> "OSDMap"
+
+    "ceph-osd" -> "PG"
+    "ceph-osd" -> "ObjectStore"
+
+    "crushtool" -> "CrushWrapper"
+
+    "OSDMap" -> "CrushWrapper"
+
+    "OSDMapTool" -> "OSDMap"
+
+    "PG" -> "PrimaryLogPG"
+    "PG" -> "ObjectStore"
+    "PG" -> "OSDMap"
+
+    "PrimaryLogPG" -> "ObjectStore"
+    "PrimaryLogPG" -> "OSDMap"
+
+    "ObjectStore" -> "BlueStore"
+
+    "BlueStore" -> "rocksdb"
+  }
+
+
+.. todo:: write more here
+
diff --git a/doc/dev/osd-class-path.rst b/doc/dev/osd-class-path.rst
new file mode 100644
index 000000000..66796eedd
--- /dev/null
+++ b/doc/dev/osd-class-path.rst
@@ -0,0 +1,28 @@
+=======================
+ OSD class path issues
+=======================
+
+::
+
+  $ rbd create rbd/test --size 100M
+  2021-03-16 01:26:59.012 7fe41426f080 -1 librbd::PoolMetadata: list: failed listing metadata: (95) Operation not supported
+  2021-03-16 01:26:59.012 7fe41426f080 -1 librbd::Config: apply_pool_overrides: failed to read pool config overrides: (95) Operation not supported
+  2021-03-16 01:26:59.012 7fe400ff9640 -1 librbd::image::CreateRequest: 0x55d62341bb30 handle_add_image_to_directory: error adding image to directory: (95) Operation not supported
+  rbd: create error: (95) Operation not supported
+
+After adding ``--debug-ms=1``, you can see which OSD is contacted.
+In the contacted OSD's log, you'll find the reason why this happens,
+and usually it is because the "rados classes" can't be loaded at runtime with ``dlopen``:
+
+::
+
+  2021-03-16 01:26:59.013 7f6c6dff3640 10 _load_class rbd from /usr/local/lib64/rados-classes/libcls_rbd.so
+  2021-03-16 01:26:59.013 7f6c6dff3640  0 _load_class could not stat class /usr/local/lib64/rados-classes/libcls_rbd.so: (2) No such file or directory
+  2021-03-16 01:26:59.013 7f6c6dff3640 -1 osd.3 112 class rbd open got (2) No such file or directory
+  2021-03-16 01:26:59.013 7f6c6dff3640  1 -- [...] --> [...] -- osd_op_reply(5 rbd_directory [call rbd.dir_add_image] v0'0 uv0 ondisk = -95 ((95) Operation not supported)) v8 -- 0x7f6c6800fed0 con 0x7f6cb80100c0
+
+This means the OSD could not find ``libcls_rbd.so``.
+You can customize the load path of these modules in ``ceph.conf`` with ``osd_class_dir``.
+By default, this is ``$libdir/rados-classes``, so when developing, you likely have to adjust the path.
+
+These class libraries are used for extending RADOS, see :ref:`rados-objclass-api-sdk`.
diff --git a/doc/dev/osd_internals/async_recovery.rst b/doc/dev/osd_internals/async_recovery.rst
new file mode 100644
index 000000000..aea5b70db
--- /dev/null
+++ b/doc/dev/osd_internals/async_recovery.rst
@@ -0,0 +1,53 @@
+=====================
+Asynchronous Recovery
+=====================
+
+Ceph Placement Groups (PGs) maintain a log of write transactions to
+facilitate speedy recovery of data. During recovery, each of these PG logs
+is used to determine which content in each OSD is missing or outdated.
+This obviates the need to scan all RADOS objects.
+See :ref:`Log Based PG <log-based-pg>` for more details on this process.
+
+Prior to the Nautilus release this recovery process was synchronous: it
+blocked writes to a RADOS object until it was recovered. In contrast,
+backfill could allow writes to proceed (assuming enough up-to-date replicas
+were available) by temporarily assigning a different acting set, and
+backfilling an OSD outside of the acting set. In some circumstances
+this ends up being significantly better for availability, e.g. if the
+PG log contains 3000 writes to disjoint objects.  When the PG log contains
+thousands of entries, it could actually be faster (though not as safe) to
+trade backfill for recovery by deleting and redeploying the containing
+OSD than to iterate through the PG log.  Recovering several megabytes
+of RADOS object data (or even worse, several megabytes of omap keys,
+notably RGW bucket indexes) can drastically increase latency for a small
+update, and combined with requests spread across many degraded objects
+it is a recipe for slow requests.
+
+To avoid this we can perform recovery in the background on an OSD
+out-of-band of the live acting set, similar to backfill, but still using
+the PG log to determine what needs to be done. This is known as *asynchronous
+recovery*.
+
+The threshold for performing asynchronous recovery instead of synchronous
+recovery is not a clear-cut. There are a few criteria which
+need to be met for asynchronous recovery:
+
+* Try to keep ``min_size`` replicas available
+* Use the approximate magnitude of the difference in length of
+  logs combined with historical missing objects to estimate the cost of
+  recovery
+* Use the parameter ``osd_async_recovery_min_cost`` to determine
+  when asynchronous recovery is appropriate
+
+With the existing peering process, when we choose the acting set we
+have not fetched the PG log from each peer; we have only the bounds of
+it and other metadata from their ``pg_info_t``. It would be more expensive
+to fetch and examine every log at this point, so we only consider an
+approximate check for log length for now. In Nautilus, we improved
+the accounting of missing objects, so post-Nautilus this information
+is also used to determine the cost of recovery.
+
+While async recovery is occurring, writes to members of the acting set
+may proceed, but we need to send their log entries to the async
+recovery targets (just like we do for backfill OSDs) so that they
+can completely catch up.
diff --git a/doc/dev/osd_internals/backfill_reservation.rst b/doc/dev/osd_internals/backfill_reservation.rst
new file mode 100644
index 000000000..3c380dcf6
--- /dev/null
+++ b/doc/dev/osd_internals/backfill_reservation.rst
@@ -0,0 +1,93 @@
+====================
+Backfill Reservation
+====================
+
+When a new OSD joins a cluster all PGs with it in their acting sets must
+eventually backfill.  If all of these backfills happen simultaneously
+they will present excessive load on the OSD:  the "thundering herd"
+effect.
+
+The ``osd_max_backfills`` tunable limits the number of outgoing or
+incoming backfills that are active on a given OSD. Note that this limit is
+applied separately to incoming and to outgoing backfill operations.
+Thus there can be as many as ``osd_max_backfills * 2`` backfill operations
+in flight on each OSD.  This subtlety is often missed, and Ceph
+operators can be puzzled as to why more ops are observed than expected.
+
+Each ``OSDService`` now has two AsyncReserver instances: one for backfills going
+from the OSD (``local_reserver``) and one for backfills going to the OSD
+(``remote_reserver``).  An ``AsyncReserver`` (``common/AsyncReserver.h``)
+manages a queue by priority of waiting items and a set of current reservation
+holders.  When a slot frees up, the ``AsyncReserver`` queues the ``Context*``
+associated with the next item on the highest priority queue in the finisher
+provided to the constructor.
+
+For a primary to initiate a backfill it must first obtain a reservation from
+its own ``local_reserver``.  Then it must obtain a reservation from the backfill
+target's ``remote_reserver`` via a ``MBackfillReserve`` message. This process is
+managed by sub-states of ``Active`` and ``ReplicaActive`` (see the sub-states
+of ``Active`` in PG.h).  The reservations are dropped either on the ``Backfilled``
+event, which is sent on the primary before calling ``recovery_complete``
+and on the replica on receipt of the ``BackfillComplete`` progress message),
+or upon leaving ``Active`` or ``ReplicaActive``.
+
+It's important to always grab the local reservation before the remote
+reservation in order to prevent a circular dependency.
+
+We minimize the risk of data loss by prioritizing the order in
+which PGs are recovered.  Admins can override the default order by using
+``force-recovery`` or ``force-backfill``. A ``force-recovery`` with op
+priority ``255`` will start before a ``force-backfill`` op at priority ``254``.
+
+If recovery is needed because a PG is below ``min_size`` a base priority of
+``220`` is used. This is incremented by the number of OSDs short of the pool's
+``min_size`` as well as a value relative to the pool's ``recovery_priority``.
+The resultant priority is capped at ``253`` so that it does not confound forced
+ops as described above. Under ordinary circumstances a recovery op is
+prioritized at ``180`` plus a value relative to the pool's ``recovery_priority``.
+The resultant priority is capped at ``219``.
+
+If backfill is needed because the number of acting OSDs is less than
+the pool's ``min_size``, a priority of ``220`` is used.  The number of OSDs
+short of the pool's ``min_size`` is added as well as a value relative to
+the pool's ``recovery_priority``.  The total priority is limited to ``253``.
+
+If backfill is needed because a PG is undersized,
+a priority of ``140`` is used.  The number of OSDs below the size of the pool is
+added as well as a value relative to the pool's ``recovery_priority``.  The
+resultant priority is capped at ``179``.  If a backfill op is
+needed because a PG is degraded, a priority of ``140`` is used.  A value
+relative to the pool's ``recovery_priority`` is added.  The resultant priority
+is capped at ``179`` .  Under ordinary circumstances a
+backfill op priority of ``100`` is used.  A value relative to the pool's
+``recovery_priority`` is added.  The total priority is capped at ``139``.
+
+.. list-table:: Backfill and Recovery op priorities
+   :widths: 20 20 20
+   :header-rows: 1
+
+   * - Description
+     - Base priority
+     - Maximum priority
+   * - Backfill
+     - 100
+     - 139
+   * - Degraded Backfill
+     - 140
+     - 179
+   * - Recovery
+     - 180
+     - 219
+   * - Inactive Recovery
+     - 220
+     - 253
+   * - Inactive Backfill
+     - 220
+     - 253
+   * - force-backfill
+     - 254
+     -
+   * - force-recovery
+     - 255
+     -
+
diff --git a/doc/dev/osd_internals/erasure_coding.rst b/doc/dev/osd_internals/erasure_coding.rst
new file mode 100644
index 000000000..40064961b
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding.rst
@@ -0,0 +1,87 @@
+==============================
+Erasure Coded Placement Groups
+==============================
+
+Glossary
+--------
+
+*chunk* 
+   When the encoding function is called, it returns chunks of the same
+   size as each other. There are two kinds of chunks: (1) *data
+   chunks*, which can be concatenated to reconstruct the original
+   object, and (2) *coding chunks*, which can be used to rebuild a
+   lost chunk.
+
+*chunk rank*
+   The index of a chunk, as determined by the encoding function. The
+   rank of the first chunk is 0, the rank of the second chunk is 1,
+   and so on.
+
+*K*
+   The number of data chunks into which an object is divided. For
+   example, if *K* = 2, then a 10KB object is divided into two objects
+   of 5KB each.
+
+*M* 
+   The number of coding chunks computed by the encoding function. *M*
+   is equal to the number of OSDs that can be missing from the cluster
+   without the cluster suffering data loss. For example, if there are
+   two coding chunks, then two OSDs can be missing without data loss.
+
+*N*
+   The number of data chunks plus the number of coding chunks: that
+   is, *K* + *M*.
+
+*rate*
+   The proportion of the total chunks containing useful information:
+   that is, *K* divided by *N*. For example, suppose that *K* = 9 and
+   *M* = 3. This would mean that *N* = 12 (because *K* + *M* = 9 + 3).
+   Therefore, the *rate* (*K* / *N*) would be 9 / 12 = 0.75. In other
+   words, 75% of the chunks would contain useful information.
+
+*shard* (also called *strip*)
+   An ordered sequence of chunks of the same rank from the same object. For a
+   given placement group, each OSD contains shards of the same rank. In the
+   special case in which an object is encoded with only one call to the
+   encoding function, the term *chunk* may be used instead of *shard* because
+   the shard is made of a single chunk. The chunks in a shard are ordered
+   according to the rank of the stripe (see *stripe* below) they belong to.
+
+
+*stripe* 
+   If an object is so large that encoding it requires more than one
+   call to the encoding function, each of these calls creates a set of
+   chunks called a *stripe*.
+
+The definitions are illustrated as follows (PG stands for placement group):
+::
+ 
+                 OSD 40                       OSD 33
+       +-------------------------+ +-------------------------+
+       |      shard 0 - PG 10    | |      shard 1 - PG 10    |
+       |+------ object O -------+| |+------ object O -------+|
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   0   |||    stripe 0         ||| |||    stripe 0         ||| 
+       ||+---------------------+|| ||+---------------------+||
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   1   |||    stripe 1         ||| |||    stripe 1         |||
+       ||+---------------------+|| ||+---------------------+||
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   2   |||    stripe 2         ||| |||    stripe 2         |||
+       ||+---------------------+|| ||+---------------------+||
+       |+-----------------------+| |+-----------------------+|
+       |         ...             | |         ...             |
+       +-------------------------+ +-------------------------+
+
+Table of contents
+-----------------
+
+.. toctree::
+   :maxdepth: 1
+
+   Developer notes <erasure_coding/developer_notes>
+   Jerasure plugin <erasure_coding/jerasure>
+   High level design document <erasure_coding/ecbackend>
diff --git a/doc/dev/osd_internals/erasure_coding/developer_notes.rst b/doc/dev/osd_internals/erasure_coding/developer_notes.rst
new file mode 100644
index 000000000..586b4b71b
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/developer_notes.rst
@@ -0,0 +1,223 @@
+============================
+Erasure Code developer notes
+============================
+
+Introduction
+------------
+
+Each chapter of this document explains an aspect of the implementation
+of the erasure code within Ceph. It is mostly based on examples being
+explained to demonstrate how things work. 
+
+Reading and writing encoded chunks from and to OSDs
+---------------------------------------------------
+
+An erasure coded pool stores each object as K+M chunks. It is divided
+into K data chunks and M coding chunks. The pool is configured to have
+a size of K+M so that each chunk is stored in an OSD in the acting
+set. The rank of the chunk is stored as an attribute of the object.
+
+Let's say an erasure coded pool is created to use five OSDs ( K+M =
+5 ) and sustain the loss of two of them ( M = 2 ).
+
+When the object *NYAN* containing *ABCDEFGHI* is written to it, the
+erasure encoding function splits the content in three data chunks,
+simply by dividing the content in three : the first contains *ABC*,
+the second *DEF* and the last *GHI*. The content will be padded if the
+content length is not a multiple of K. The function also creates two
+coding chunks : the fourth with *YXY* and the fifth with *GQC*. Each
+chunk is stored in an OSD in the acting set. The chunks are stored in
+objects that have the same name ( *NYAN* ) but reside on different
+OSDs. The order in which the chunks were created must be preserved and
+is stored as an attribute of the object ( shard_t ), in addition to its
+name. Chunk *1* contains *ABC* and is stored on *OSD5* while chunk *4*
+contains *YXY* and is stored on *OSD3*.
+
+::
+ 
+                             +-------------------+
+                        name |        NYAN       |
+                             +-------------------+
+                     content |      ABCDEFGHI    |
+                             +--------+----------+
+                                      |
+                                      |
+                                      v
+                               +------+------+
+               +---------------+ encode(3,2) +-----------+
+               |               +--+--+---+---+           |
+               |                  |  |   |               |
+               |          +-------+  |   +-----+         |
+               |          |          |         |         |
+            +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+      name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+            +------+   +------+   +------+  +------+  +------+
+     shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+            +------+   +------+   +------+  +------+  +------+
+   content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+            +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+               |          |          |         |         |
+               |          |          |         |         |
+               |          |       +--+---+     |         |
+               |          |       | OSD1 |     |         |
+               |          |       +------+     |         |
+               |          |       +------+     |         |
+               |          +------>| OSD2 |     |         |
+               |                  +------+     |         |
+               |                  +------+     |         |
+               |                  | OSD3 |<----+         |
+               |                  +------+               |
+               |                  +------+               |
+               |                  | OSD4 |<--------------+
+               |                  +------+
+               |                  +------+
+               +----------------->| OSD5 |
+                                  +------+
+
+
+
+
+When the object *NYAN* is read from the erasure coded pool, the
+decoding function reads three chunks : chunk *1* containing *ABC*,
+chunk *3* containing *GHI* and chunk *4* containing *YXY* and rebuild
+the original content of the object *ABCDEFGHI*. The decoding function
+is informed that the chunks *2* and *5* are missing ( they are called
+*erasures* ). The chunk *5* could not be read because the *OSD4* is
+*out*. 
+
+The decoding function could be called as soon as three chunks are
+read : *OSD2* was the slowest and its chunk does not need to be taken into
+account. This optimization is not implemented in Firefly.
+
+::
+ 
+                             +-------------------+
+                        name |        NYAN       |
+                             +-------------------+
+                     content |      ABCDEFGHI    |
+                             +--------+----------+
+                                      ^
+                                      |
+                                      |
+                               +------+------+
+                               | decode(3,2) |
+                               | erasures 2,5|
+               +-------------->|             |
+               |               +-------------+
+               |                     ^   ^
+               |                     |   +-----+
+               |                     |         |
+            +--+---+   +------+   +--+---+  +--+---+
+      name  | NYAN |   | NYAN |   | NYAN |  | NYAN |
+            +------+   +------+   +------+  +------+
+     shard  |  1   |   |  2   |   |  3   |  |  4   |
+            +------+   +------+   +------+  +------+
+   content  | ABC  |   | DEF  |   | GHI  |  | YXY  |
+            +--+---+   +--+---+   +--+---+  +--+---+
+               ^          .          ^         ^
+               |    TOO   .          |         |
+               |    SLOW  .       +--+---+     |
+               |          ^       | OSD1 |     |
+               |          |       +------+     |
+               |          |       +------+     |
+               |          +-------| OSD2 |     |
+               |                  +------+     |
+               |                  +------+     |
+               |                  | OSD3 |-----+
+               |                  +------+
+               |                  +------+
+               |                  | OSD4 | OUT
+               |                  +------+
+               |                  +------+
+               +------------------| OSD5 |
+                                  +------+
+
+
+Erasure code library
+--------------------
+
+Using `Reed-Solomon <https://en.wikipedia.org/wiki/Reed_Solomon>`_,
+with parameters K+M, object O is encoded by dividing it into chunks O1,
+O2, ...  OM and computing coding chunks P1, P2, ... PK. Any K chunks
+out of the available K+M chunks can be used to obtain the original
+object.  If data chunk O2 or coding chunk P2 are lost, they can be
+repaired using any K chunks out of the K+M chunks. If more than M
+chunks are lost, it is not possible to recover the object.
+
+Reading the original content of object O can be a simple
+concatenation of O1, O2, ... OM, because the plugins are using
+`systematic codes
+<https://en.wikipedia.org/wiki/Systematic_code>`_. Otherwise the chunks
+must be given to the erasure code library *decode* method to retrieve
+the content of the object.
+
+Performance depend on the parameters to the encoding functions and
+is also influenced by the packet sizes used when calling the encoding
+functions ( for Cauchy or Liberation for instance ): smaller packets
+means more calls and more overhead.
+
+Although Reed-Solomon is provided as a default, Ceph uses it via an
+`abstract API <https://github.com/ceph/ceph/blob/v0.78/src/erasure-code/ErasureCodeInterface.h>`_ designed to
+allow each pool to choose the plugin that implements it using
+key=value pairs stored in an `erasure code profile`_. 
+
+.. _erasure code profile: ../../../erasure-coded-pool
+
+::
+ 
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-failure-domain=osd
+ $ ceph osd erasure-code-profile get myprofile
+ directory=/usr/lib/ceph/erasure-code
+ k=2
+ m=1
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-failure-domain=osd
+ $ ceph osd pool create ecpool erasure myprofile
+
+The *plugin* is dynamically loaded from *directory*  and expected to
+implement the *int __erasure_code_init(char *plugin_name, char *directory)* function 
+which is responsible for registering an object derived from *ErasureCodePlugin* 
+in the registry. The `ErasureCodePluginExample <https://github.com/ceph/ceph/blob/v0.78/src/test/erasure-code/ErasureCodePluginExample.cc>`_ plugin reads:
+
+::
+ 
+  ErasureCodePluginRegistry &instance = 
+                             ErasureCodePluginRegistry::instance();
+  instance.add(plugin_name, new ErasureCodePluginExample());
+
+The *ErasureCodePlugin* derived object must provide a factory method
+from which the concrete implementation of the *ErasureCodeInterface*
+object can be generated. The `ErasureCodePluginExample plugin <https://github.com/ceph/ceph/blob/v0.78/src/test/erasure-code/ErasureCodePluginExample.cc>`_ reads:
+
+::
+ 
+  virtual int factory(const map<std::string,std::string> &parameters,
+                      ErasureCodeInterfaceRef *erasure_code) {
+    *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample(parameters));
+    return 0;
+  } 
+
+The *parameters* argument is the list of *key=value* pairs that were
+set in the erasure code profile, before the pool was created.
+
+::
+ 
+  ceph osd erasure-code-profile set myprofile \
+     directory=<dir>         \ # mandatory
+     plugin=jerasure         \ # mandatory
+     m=10                    \ # optional and plugin dependent
+     k=3                     \ # optional and plugin dependent
+     technique=reed_sol_van  \ # optional and plugin dependent
+
+Notes
+-----
+
+If the objects are large, it may be impractical to encode and decode
+them in memory. However, when using *RBD* a 1TB device is divided in
+many individual 4MB objects and *RGW* does the same.
+
+Encoding and decoding is implemented in the OSD. Although it could be
+implemented client side for read write, the OSD must be able to encode
+and decode on its own when scrubbing.
diff --git a/doc/dev/osd_internals/erasure_coding/ecbackend.rst b/doc/dev/osd_internals/erasure_coding/ecbackend.rst
new file mode 100644
index 000000000..877a08a38
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/ecbackend.rst
@@ -0,0 +1,206 @@
+=================================
+ECBackend Implementation Strategy
+=================================
+
+Miscellaneous initial design notes
+==================================
+
+The initial (and still true for EC pools without the hacky EC
+overwrites debug flag enabled) design for EC pools restricted
+EC pools to operations that can be easily rolled back:
+
+- CEPH_OSD_OP_APPEND: We can roll back an append locally by
+  including the previous object size as part of the PG log event.
+- CEPH_OSD_OP_DELETE: The possibility of rolling back a delete
+  requires that we retain the deleted object until all replicas have
+  persisted in the deletion event. Erasure Coded backend will therefore
+  need to store objects with the version at which they were created
+  included in the key provided to the filestore.  Old versions of an
+  object can be pruned when all replicas have committed up to the log
+  event deleting the object.
+- CEPH_OSD_OP_(SET|RM)ATTR: If we include the prior value of the attr
+  to be set or removed, we can roll back these operations locally.
+
+Log entries contain a structure explaining how to locally undo the
+operation represented by the operation
+(see osd_types.h:TransactionInfo::LocalRollBack).
+
+PGTemp and Crush
+----------------
+
+Primaries are able to request a temp acting set mapping in order to
+allow an up-to-date OSD to serve requests while a new primary is
+backfilled (and for other reasons).  An erasure coded PG needs to be
+able to designate a primary for these reasons without putting it in
+the first position of the acting set.  It also needs to be able to
+leave holes in the requested acting set.
+
+Core Changes:
+
+- OSDMap::pg_to_*_osds needs to separately return a primary.  For most
+  cases, this can continue to be acting[0].
+- MOSDPGTemp (and related OSD structures) needs to be able to specify
+  a primary as well as an acting set.
+- Much of the existing code base assumes that acting[0] is the primary
+  and that all elements of acting are valid.  This needs to be cleaned
+  up since the acting set may contain holes.
+
+Distinguished acting set positions
+----------------------------------
+
+With the replicated strategy, all replicas of a PG are
+interchangeable.  With erasure coding, different positions in the
+acting set have different pieces of the erasure coding scheme and are
+not interchangeable.  Worse, crush might cause chunk 2 to be written
+to an OSD which happens already to contain an (old) copy of chunk 4.
+This means that the OSD and PG messages need to work in terms of a
+type like pair<shard_t, pg_t> in order to distinguish different PG
+chunks on a single OSD.
+
+Because the mapping of an object name to object in the filestore must
+be 1-to-1, we must ensure that the objects in chunk 2 and the objects
+in chunk 4 have different names.  To that end, the object store must
+include the chunk id in the object key.
+
+Core changes:
+
+- The object store `ghobject_t needs to also include a chunk id
+  <https://github.com/ceph/ceph/blob/firefly/src/common/hobject.h#L241>`_ making it more like
+  tuple<hobject_t, gen_t, shard_t>.
+- coll_t needs to include a shard_t.
+- The OSD pg_map and similar PG mappings need to work in terms of a
+  spg_t (essentially
+  pair<pg_t, shard_t>).  Similarly, pg->pg messages need to include
+  a shard_t
+- For client->PG messages, the OSD will need a way to know which PG
+  chunk should get the message since the OSD may contain both a
+  primary and non-primary chunk for the same PG
+
+Object Classes
+--------------
+
+Reads from object classes will return ENOTSUP on EC pools by invoking
+a special SYNC read.
+
+Scrub
+-----
+
+The main catch, however, for EC pools is that sending a crc32 of the
+stored chunk on a replica isn't particularly helpful since the chunks
+on different replicas presumably store different data.  Because we
+don't support overwrites except via DELETE, however, we have the
+option of maintaining a crc32 on each chunk through each append.
+Thus, each replica instead simply computes a crc32 of its own stored
+chunk and compares it with the locally stored checksum.  The replica
+then reports to the primary whether the checksums match.
+
+With overwrites, all scrubs are disabled for now until we work out
+what to do (see doc/dev/osd_internals/erasure_coding/proposals.rst).
+
+Crush
+-----
+
+If crush is unable to generate a replacement for a down member of an
+acting set, the acting set should have a hole at that position rather
+than shifting the other elements of the acting set out of position.
+
+=========
+ECBackend
+=========
+
+MAIN OPERATION OVERVIEW
+=======================
+
+A RADOS put operation can span
+multiple stripes of a single object. There must be code that
+tessellates the application level write into a set of per-stripe write
+operations -- some whole-stripes and up to two partial
+stripes. Without loss of generality, for the remainder of this
+document, we will focus exclusively on writing a single stripe (whole
+or partial). We will use the symbol "W" to represent the number of
+blocks within a stripe that are being written, i.e., W <= K.
+
+There are three data flows for handling a write into an EC stripe. The
+choice of which of the three data flows to choose is based on the size
+of the write operation and the arithmetic properties of the selected
+parity-generation algorithm.
+
+(1) Whole stripe is written/overwritten
+(2) A read-modify-write operation is performed.
+
+WHOLE STRIPE WRITE
+------------------
+
+This is a simple case, and is already performed in the existing code
+(for appends, that is). The primary receives all of the data for the
+stripe in the RADOS request, computes the appropriate parity blocks
+and send the data and parity blocks to their destination shards which
+write them. This is essentially the current EC code.
+
+READ-MODIFY-WRITE
+-----------------
+
+The primary determines which of the K-W blocks are to be unmodified,
+and reads them from the shards. Once all of the data is received it is
+combined with the received new data and new parity blocks are
+computed. The modified blocks are sent to their respective shards and
+written. The RADOS operation is acknowledged.
+
+OSD Object Write and Consistency
+--------------------------------
+
+Regardless of the algorithm chosen above, writing of the data is a two-
+phase process: commit and rollforward. The primary sends the log
+entries with the operation described (see
+osd_types.h:TransactionInfo::(LocalRollForward|LocalRollBack).  
+In all cases, the "commit" is performed in place, possibly leaving some
+information required for a rollback in a write-aside object.  The
+rollforward phase occurs once all acting set replicas have committed
+the commit, it then removes the rollback information.
+
+In the case of overwrites of existing stripes, the rollback information
+has the form of a sparse object containing the old values of the
+overwritten extents populated using clone_range.  This is essentially
+a place-holder implementation, in real life, bluestore will have an
+efficient primitive for this.
+
+The rollforward part can be delayed since we report the operation as
+committed once all replicas have been committed.  Currently, whenever we
+send a write, we also indicate that all previously committed
+operations should be rolled forward (see
+ECBackend::try_reads_to_commit).  If there aren't any in the pipeline
+when we arrive at the waiting_rollforward queue, we start a dummy
+write to move things along (see the Pipeline section later on and
+ECBackend::try_finish_rmw).
+
+ExtentCache
+-----------
+
+It's pretty important to be able to pipeline writes on the same
+object.  For this reason, there is a cache of extents written by
+cacheable operations.  Each extent remains pinned until the operations
+referring to it are committed.  The pipeline prevents rmw operations
+from running until uncacheable transactions (clones, etc) are flushed
+from the pipeline.
+
+See ExtentCache.h for a detailed explanation of how the cache
+states correspond to the higher level invariants about the conditions
+under which concurrent operations can refer to the same object.
+
+Pipeline
+--------
+
+Reading src/osd/ExtentCache.h should have given a good idea of how
+operations might overlap.  There are several states involved in
+processing a write operation and an important invariant which
+isn't enforced by PrimaryLogPG at a higher level which needs to be
+managed by ECBackend.  The important invariant is that we can't
+have uncacheable and rmw operations running at the same time
+on the same object.  For simplicity, we simply enforce that any
+operation which contains an rmw operation must wait until
+all in-progress uncacheable operations complete.
+
+There are improvements to be made here in the future.
+
+For more details, see ECBackend::waiting_* and
+ECBackend::try_<from>_to_<to>.
diff --git a/doc/dev/osd_internals/erasure_coding/jerasure.rst b/doc/dev/osd_internals/erasure_coding/jerasure.rst
new file mode 100644
index 000000000..ac3636720
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/jerasure.rst
@@ -0,0 +1,35 @@
+===============
+jerasure plugin
+===============
+
+Introduction
+------------
+
+The parameters interpreted by the ``jerasure`` plugin are:
+
+::
+ 
+  ceph osd erasure-code-profile set myprofile \
+     directory=<dir>         \ # plugin directory absolute path
+     plugin=jerasure         \ # plugin name (only jerasure)
+     k=<k>                   \ # data chunks (default 2)
+     m=<m>                   \ # coding chunks (default 2)
+     technique=<technique>   \ # coding technique
+
+The coding techniques can be chosen among *reed_sol_van*,
+*reed_sol_r6_op*, *cauchy_orig*, *cauchy_good*, *liberation*,
+*blaum_roth* and *liber8tion*.
+
+The *src/erasure-code/jerasure* directory contains the
+implementation. It is a wrapper around the code found at
+`https://github.com/ceph/jerasure <https://github.com/ceph/jerasure>`_
+and `https://github.com/ceph/gf-complete
+<https://github.com/ceph/gf-complete>`_ , pinned to the latest stable
+version in *.gitmodules*. These repositories are copies of the
+upstream repositories `http://jerasure.org/jerasure/jerasure
+<http://jerasure.org/jerasure/jerasure>`_ and
+`http://jerasure.org/jerasure/gf-complete
+<http://jerasure.org/jerasure/gf-complete>`_ . The difference
+between the two, if any, should match pull requests against upstream.
+Note that as of 2023, the ``jerasure.org`` web site may no longer be
+legitimate and/or associated with the original project.
diff --git a/doc/dev/osd_internals/erasure_coding/proposals.rst b/doc/dev/osd_internals/erasure_coding/proposals.rst
new file mode 100644
index 000000000..8a30727b3
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/proposals.rst
@@ -0,0 +1,385 @@
+:orphan:
+
+=================================
+Proposed Next Steps for ECBackend
+=================================
+
+PARITY-DELTA-WRITE
+------------------
+
+RMW operations current require 4 network hops (2 round trips).  In
+principle, for some codes, we can reduce this to 3 by sending the
+update to the replicas holding the data blocks and having them
+compute a delta to forward onto the parity blocks.
+
+The primary reads the current values of the "W" blocks and then uses
+the new values of the "W" blocks to compute parity-deltas for each of
+the parity blocks.  The W blocks and the parity delta-blocks are sent
+to their respective shards.
+
+The choice of whether to use a read-modify-write or a
+parity-delta-write is complex policy issue that is TBD in the details
+and is likely to be heavily dependent on the computational costs
+associated with a parity-delta vs. a regular parity-generation
+operation. However, it is believed that the parity-delta scheme is
+likely to be the preferred choice, when available.
+
+The internal interface to the erasure coding library plug-ins needs to
+be extended to support the ability to query if parity-delta
+computation is possible for a selected algorithm as well as an
+interface to the actual parity-delta computation algorithm when
+available.
+
+Stripe Cache
+------------
+
+It may be a good idea to extend the current ExtentCache usage to
+cache some data past when the pinning operation releases it.
+One application pattern that is important to optimize is the small
+block sequential write operation (think of the journal of a journaling
+file system or a database transaction log). Regardless of the chosen
+redundancy algorithm, it is advantageous for the primary to
+retain/buffer recently read/written portions of a stripe in order to
+reduce network traffic. The dynamic contents of this cache may be used
+in the determination of whether a read-modify-write or a
+parity-delta-write is performed. The sizing of this cache is TBD, but
+we should plan on allowing at least a few full stripes per active
+client. Limiting the cache occupancy on a per-client basis will reduce
+the noisy neighbor problem.
+
+Recovery and Rollback Details
+=============================
+
+Implementing a Rollback-able Prepare Operation
+----------------------------------------------
+
+The prepare operation is implemented at each OSD through a simulation
+of a versioning or copy-on-write capability for modifying a portion of
+an object.
+
+When a prepare operation is performed, the new data is written into a
+temporary object. The PG log for the
+operation will contain a reference to the temporary object so that it
+can be located for recovery purposes as well as a record of all of the
+shards which are involved in the operation. 
+
+In order to avoid fragmentation (and hence, future read performance),
+creation of the temporary object needs special attention. The name of
+the temporary object affects its location within the KV store. Right
+now its unclear whether it's desirable for the name to locate near the
+base object or whether a separate subset of keyspace should be used
+for temporary objects. Sam believes that colocation with the base
+object is preferred (he suggests using the generation counter of the
+ghobject for temporaries).  Whereas Allen believes that using a
+separate subset of keyspace is desirable since these keys are
+ephemeral and we don't want to actually colocate them with the base
+object keys. Perhaps some modeling here can help resolve this
+issue. The data of the temporary object wants to be located as close
+to the data of the base object as possible. This may be best performed
+by adding a new ObjectStore creation primitive that takes the base
+object as an additional parameter that is a hint to the allocator.
+
+Sam: I think that the short lived thing may be a red herring.  We'll
+be updating the donor and primary objects atomically, so it seems like
+we'd want them adjacent in the key space, regardless of the donor's
+lifecycle.
+
+The apply operation moves the data from the temporary object into the
+correct position within the base object and deletes the associated
+temporary object. This operation is done using a specialized
+ObjectStore primitive. In the current ObjectStore interface, this can
+be done using the clonerange function followed by a delete, but can be
+done more efficiently with a specialized move primitive.
+Implementation of the specialized primitive on FileStore can be done
+by copying the data. Some file systems have extensions that might also
+be able to implement this operation (like a defrag API that swaps
+chunks between files). It is expected that NewStore will be able to
+support this efficiently and natively (It has been noted that this
+sequence requires that temporary object allocations, which tend to be
+small, be efficiently converted into blocks for main objects and that
+blocks that were formerly inside of main objects must be reusable with
+minimal overhead)
+
+The prepare and apply operations can be separated arbitrarily in
+time. If a read operation accesses an object that has been altered by
+a prepare operation (but without a corresponding apply operation) it
+must return the data after the prepare operation. This is done by
+creating an in-memory database of objects which have had a prepare
+operation without a corresponding apply operation. All read operations
+must consult this in-memory data structure in order to get the correct
+data. It should explicitly recognized that it is likely that there
+will be multiple prepare operations against a single base object and
+the code must handle this case correctly. This code is implemented as
+a layer between ObjectStore and all existing readers.  Annoyingly,
+we'll want to trash this state when the interval changes, so the first
+thing that needs to happen after activation is that the primary and
+replicas apply up to last_update so that the empty cache will be
+correct.
+
+During peering, it is now obvious that an unapplied prepare operation
+can easily be rolled back simply by deleting the associated temporary
+object and removing that entry from the in-memory data structure.
+
+Partial Application Peering/Recovery modifications
+--------------------------------------------------
+
+Some writes will be small enough to not require updating all of the
+shards holding data blocks.  For write amplification minimization
+reasons, it would be best to avoid writing to those shards at all,
+and delay even sending the log entries until the next write which
+actually hits that shard.
+
+The delaying (buffering) of the transmission of the prepare and apply
+operations for witnessing OSDs creates new situations that peering
+must handle. In particular the logic for determining the authoritative
+last_update value (and hence the selection of the OSD which has the
+authoritative log) must be modified to account for the valid but
+missing (i.e., delayed/buffered) pglog entries to which the
+authoritative OSD was only a witness to.
+
+Because a partial write might complete without persisting a log entry
+on every replica, we have to do a bit more work to determine an
+authoritative last_update.  The constraint (as with a replicated PG)
+is that last_update >= the most recent log entry for which a commit
+was sent to the client (call this actual_last_update).  Secondarily,
+we want last_update to be as small as possible since any log entry
+past actual_last_update (we do not apply a log entry until we have
+sent the commit to the client) must be able to be rolled back.  Thus,
+the smaller a last_update we choose, the less recovery will need to
+happen (we can always roll back, but rolling a replica forward may
+require an object rebuild).  Thus, we will set last_update to 1 before
+the oldest log entry we can prove cannot have been committed.  In
+current master, this is simply the last_update of the shortest log
+from that interval (because that log did not persist any entry past
+that point -- a precondition for sending a commit to the client).  For
+this design, we must consider the possibility that any log is missing
+at its head log entries in which it did not participate.  Thus, we
+must determine the most recent interval in which we went active
+(essentially, this is what find_best_info currently does).  We then
+pull the log from each live osd from that interval back to the minimum
+last_update among them.  Then, we extend all logs from the
+authoritative interval until each hits an entry in which it should
+have participated, but did not record.  The shortest of these extended
+logs must therefore contain any log entry for which we sent a commit
+to the client -- and the last entry gives us our last_update.
+
+Deep scrub support
+------------------
+
+The simple answer here is probably our best bet.  EC pools can't use
+the omap namespace at all right now.  The simplest solution would be
+to take a prefix of the omap space and pack N M byte L bit checksums
+into each key/value.  The prefixing seems like a sensible precaution
+against eventually wanting to store something else in the omap space.
+It seems like any write will need to read at least the blocks
+containing the modified range.  However, with a code able to compute
+parity deltas, we may not need to read a whole stripe.  Even without
+that, we don't want to have to write to blocks not participating in
+the write.  Thus, each shard should store checksums only for itself.
+It seems like you'd be able to store checksums for all shards on the
+parity blocks, but there may not be distinguished parity blocks which
+are modified on all writes (LRC or shec provide two examples).  L
+should probably have a fixed number of options (16, 32, 64?) and be
+configurable per-pool at pool creation.  N, M should be likewise be
+configurable at pool creation with sensible defaults.
+
+We need to handle online upgrade.  I think the right answer is that
+the first overwrite to an object with an append only checksum
+removes the append only checksum and writes in whatever stripe
+checksums actually got written.  The next deep scrub then writes
+out the full checksum omap entries.
+
+RADOS Client Acknowledgement Generation Optimization
+====================================================
+
+Now that the recovery scheme is understood, we can discuss the
+generation of the RADOS operation acknowledgement (ACK) by the
+primary ("sufficient" from above). It is NOT required that the primary
+wait for all shards to complete their respective prepare
+operations. Using our example where the RADOS operations writes only
+"W" chunks of the stripe, the primary will generate and send W+M
+prepare operations (possibly including a send-to-self). The primary
+need only wait for enough shards to be written to ensure recovery of
+the data, Thus after writing W + M chunks you can afford the lost of M
+chunks. Hence the primary can generate the RADOS ACK after W+M-M => W
+of those prepare operations are completed.
+
+Inconsistent object_info_t versions
+===================================
+
+A natural consequence of only writing the blocks which actually
+changed is that we don't want to update the object_info_t of the
+objects which didn't.  I actually think it would pose a problem to do
+so: pg ghobject namespaces are generally large, and unless the osd is
+seeing a bunch of overwrites on a small set of objects, I'd expect
+each write to be far enough apart in the backing ghobject_t->data
+mapping to each constitute a random metadata update.  Thus, we have to
+accept that not every shard will have the current version in its
+object_info_t.  We can't even bound how old the version on a
+particular shard will happen to be.  In particular, the primary does
+not necessarily have the current version.  One could argue that the
+parity shards would always have the current version, but not every
+code necessarily has designated parity shards which see every write
+(certainly LRC, iirc shec, and even with a more pedestrian code, it
+might be desirable to rotate the shards based on object hash).  Even
+if you chose to designate a shard as witnessing all writes, the pg
+might be degraded with that particular shard missing.  This is a bit
+tricky, currently reads and writes implicitly return the most recent
+version of the object written.  On reads, we'd have to read K shards
+to answer that question.  We can get around that by adding a "don't
+tell me the current version" flag.  Writes are more problematic: we
+need an object_info from the most recent write in order to form the
+new object_info and log_entry.
+
+A truly terrifying option would be to eliminate version and
+prior_version entirely from the object_info_t.  There are a few
+specific purposes it serves:
+
+#. On OSD startup, we prime the missing set by scanning backwards
+   from last_update to last_complete comparing the stored object's
+   object_info_t to the version of most recent log entry.
+#. During backfill, we compare versions between primary and target
+   to avoid some pushes. We use it elsewhere as well
+#. While pushing and pulling objects, we verify the version.
+#. We return it on reads and writes and allow the librados user to
+   assert it atomically on writesto allow the user to deal with write
+   races (used extensively by rbd).
+
+Case (3) isn't actually essential, just convenient.  Oh well.  (4)
+is more annoying. Writes are easy since we know the version.  Reads
+are tricky because we may not need to read from all of the replicas.
+Simplest solution is to add a flag to rados operations to just not
+return the user version on read.  We can also just not support the
+user version assert on ec for now (I think?  Only user is rgw bucket
+indices iirc, and those will always be on replicated because they use
+omap).
+
+We can avoid (1) by maintaining the missing set explicitly.  It's
+already possible for there to be a missing object without a
+corresponding log entry (Consider the case where the most recent write
+is to an object which has not been updated in weeks.  If that write
+becomes divergent, the written object needs to be marked missing based
+on the prior_version which is not in the log.)  THe PGLog already has
+a way of handling those edge cases (see divergent_priors).  We'd
+simply expand that to contain the entire missing set and maintain it
+atomically with the log and the objects.  This isn't really an
+unreasonable option, the additional keys would be fewer than the
+existing log keys + divergent_priors and aren't updated in the fast
+write path anyway.
+
+The second case is a bit trickier.  It's really an optimization for
+the case where a pg became not in the acting set long enough for the
+logs to no longer overlap but not long enough for the PG to have
+healed and removed the old copy.  Unfortunately, this describes the
+case where a node was taken down for maintenance with noout set. It's
+probably not acceptable to re-backfill the whole OSD in such a case,
+so we need to be able to quickly determine whether a particular shard
+is up to date given a valid acting set of other shards.
+
+Let ordinary writes which do not change the object size not touch the
+object_info at all.  That means that the object_info version won't
+match the pg log entry version.  Include in the pg_log_entry_t the
+current object_info version as well as which shards participated (as
+mentioned above).  In addition to the object_info_t attr, record on
+each shard s a vector recording for each other shard s' the most
+recent write which spanned both s and s'.  Operationally, we maintain
+an attr on each shard containing that vector.  A write touching S
+updates the version stamp entry for each shard in S on each shard in
+S's attribute (and leaves the rest alone).  If we have a valid acting
+set during backfill, we must have a witness of every write which
+completed -- so taking the max of each entry over all of the acting
+set shards must give us the current version for each shard.  During
+recovery, we set the attribute on the recovery target to that max
+vector (Question: with LRC, we may not need to touch much of the
+acting set to recover a particular shard -- can we just use the max of
+the shards we used to recovery, or do we need to grab the version
+vector from the rest of the acting set as well?  I'm not sure, not a
+big deal anyway, I think).
+
+The above lets us perform blind writes without knowing the current
+object version (log entry version, that is) while still allowing us to
+avoid backfilling up to date objects.  The only catch is that our
+backfill scans will can all replicas, not just the primary and the
+backfill targets.
+
+It would be worth adding into scrub the ability to check the
+consistency of the gathered version vectors -- probably by just
+taking 3 random valid subsets and verifying that they generate
+the same authoritative version vector.
+
+Implementation Strategy
+=======================
+
+It goes without saying that it would be unwise to attempt to do all of
+this in one massive PR.  It's also not a good idea to merge code which
+isn't being tested.  To that end, it's worth thinking a bit about
+which bits can be tested on their own (perhaps with a bit of temporary
+scaffolding).
+
+We can implement the overwrite friendly checksumming scheme easily
+enough with the current implementation.  We'll want to enable it on a
+per-pool basis (probably using a flag which we'll later repurpose for
+actual overwrite support).  We can enable it in some of the ec
+thrashing tests in the suite.  We can also add a simple test
+validating the behavior of turning it on for an existing ec pool
+(later, we'll want to be able to convert append-only ec pools to
+overwrite ec pools, so that test will simply be expanded as we go).
+The flag should be gated by the experimental feature flag since we
+won't want to support this as a valid configuration -- testing only.
+We need to upgrade append only ones in place during deep scrub.
+
+Similarly, we can implement the unstable extent cache with the current
+implementation, it even lets us cut out the readable ack the replicas
+send to the primary after the commit which lets it release the lock.
+Same deal, implement, gate with experimental flag, add to some of the
+automated tests.  I don't really see a reason not to use the same flag
+as above.
+
+We can certainly implement the move-range primitive with unit tests
+before there are any users.  Adding coverage to the existing
+objectstore tests would suffice here.
+
+Explicit missing set can be implemented now, same deal as above --
+might as well even use the same feature bit.
+
+The TPC protocol outlined above can actually be implemented an append
+only EC pool.  Same deal as above, can even use the same feature bit.
+
+The RADOS flag to suppress the read op user version return can be
+implemented immediately.  Mostly just needs unit tests.
+
+The version vector problem is an interesting one.  For append only EC
+pools, it would be pointless since all writes increase the size and
+therefore update the object_info.  We could do it for replicated pools
+though.  It's a bit silly since all "shards" see all writes, but it
+would still let us implement and partially test the augmented backfill
+code as well as the extra pg log entry fields -- this depends on the
+explicit pg log entry branch having already merged.  It's not entirely
+clear to me that this one is worth doing separately.  It's enough code
+that I'd really prefer to get it done independently, but it's also a
+fair amount of scaffolding that will be later discarded.
+
+PGLog entries need to be able to record the participants and log
+comparison needs to be modified to extend logs with entries they
+wouldn't have witnessed.  This logic should be abstracted behind
+PGLog so it can be unittested -- that would let us test it somewhat
+before the actual ec overwrites code merges.
+
+Whatever needs to happen to the ec plugin interface can probably be
+done independently of the rest of this (pending resolution of
+questions below).
+
+The actual nuts and bolts of performing the ec overwrite it seems to
+me can't be productively tested (and therefore implemented) until the
+above are complete, so best to get all of the supporting code in
+first.
+
+Open Questions
+==============
+
+Is there a code we should be using that would let us compute a parity
+delta without rereading and reencoding the full stripe?  If so, is it
+the kind of thing we need to design for now, or can it be reasonably
+put off?
+
+What needs to happen to the EC plugin interface?
diff --git a/doc/dev/osd_internals/index.rst b/doc/dev/osd_internals/index.rst
new file mode 100644
index 000000000..7e82914aa
--- /dev/null
+++ b/doc/dev/osd_internals/index.rst
@@ -0,0 +1,10 @@
+==============================
+OSD developer documentation
+==============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
diff --git a/doc/dev/osd_internals/last_epoch_started.rst b/doc/dev/osd_internals/last_epoch_started.rst
new file mode 100644
index 000000000..c31cc66b5
--- /dev/null
+++ b/doc/dev/osd_internals/last_epoch_started.rst
@@ -0,0 +1,60 @@
+======================
+last_epoch_started
+======================
+
+``info.last_epoch_started`` records an activation epoch ``e`` for interval ``i``
+such that all writes committed in ``i`` or earlier are reflected in the
+local info/log and no writes after ``i`` are reflected in the local
+info/log.  Since no committed write is ever divergent, even if we
+get an authoritative log/info with an older ``info.last_epoch_started``,
+we can leave our ``info.last_epoch_started`` alone since no writes could
+have committed in any intervening interval (See PG::proc_master_log).
+
+``info.history.last_epoch_started`` records a lower bound on the most
+recent interval in which the PG as a whole went active and accepted
+writes.  On a particular OSD it is also an upper bound on the
+activation epoch of intervals in which writes in the local PG log
+occurred:  we update it before accepting writes.  Because all
+committed writes are committed by all acting set OSDs, any
+non-divergent writes ensure that ``history.last_epoch_started`` was
+recorded by all acting set members in the interval.  Once peering has
+queried one OSD from each interval back to some seen
+``history.last_epoch_started``, it follows that no interval after the max
+``history.last_epoch_started`` can have reported writes as committed
+(since we record it before recording client writes in an interval).
+Thus, the minimum ``last_update`` across all infos with
+``info.last_epoch_started >= MAX(history.last_epoch_started)`` must be an
+upper bound on writes reported as committed to the client.
+
+We update ``info.last_epoch_started`` with the initial activation message,
+but we only update ``history.last_epoch_started`` after the new
+``info.last_epoch_started`` is persisted (possibly along with the first
+write).  This ensures that we do not require an OSD with the most
+recent ``info.last_epoch_started`` until all acting set OSDs have recorded
+it.
+
+In ``find_best_info``, we do include ``info.last_epoch_started`` values when
+calculating ``max_last_epoch_started_found`` because we want to avoid
+designating a log entry divergent which in a prior interval would have
+been non-divergent since it might have been used to serve a read.  In
+``activate()``, we use the peer's ``last_epoch_started`` value as a bound on
+how far back divergent log entries can be found.
+
+However, in a case like
+
+.. code::
+
+  calc_acting osd.0 1.4e( v 473'302 (292'200,473'302] local-les=473 n=4 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.1 1.4e( v 473'302 (293'202,473'302] lb 0//0//-1 local-les=477 n=0 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.4 1.4e( v 473'302 (120'121,473'302] local-les=473 n=4 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.5 1.4e( empty local-les=0 n=0 ec=5 les/c 473/473 556/556/556
+
+since osd.1 is the only one which recorded info.les=477, while osd.4,osd.0
+(which were the acting set in that interval) did not (osd.4 restarted and osd.0
+did not get the message in time), the PG is marked incomplete when
+either osd.4 or osd.0 would have been valid choices. To avoid this, we do not
+consider ``info.les`` for incomplete peers when calculating
+``min_last_epoch_started_found``.  It would not have been in the acting
+set, so we must have another OSD from that interval anyway (if
+``maybe_went_rw``).  If that OSD does not remember that ``info.les``, then we
+cannot have served reads.
diff --git a/doc/dev/osd_internals/log_based_pg.rst b/doc/dev/osd_internals/log_based_pg.rst
new file mode 100644
index 000000000..99cffd3d9
--- /dev/null
+++ b/doc/dev/osd_internals/log_based_pg.rst
@@ -0,0 +1,208 @@
+.. _log-based-pg:
+
+============
+Log Based PG
+============
+
+Background
+==========
+
+Why PrimaryLogPG?
+-----------------
+
+Currently, consistency for all ceph pool types is ensured by primary
+log-based replication. This goes for both erasure-coded (EC) and
+replicated pools.
+
+Primary log-based replication
+-----------------------------
+
+Reads must return data written by any write which completed (where the
+client could possibly have received a commit message).  There are lots
+of ways to handle this, but Ceph's architecture makes it easy for
+everyone at any map epoch to know who the primary is.  Thus, the easy
+answer is to route all writes for a particular PG through a single
+ordering primary and then out to the replicas.  Though we only
+actually need to serialize writes on a single RADOS object (and even then,
+the partial ordering only really needs to provide an ordering between
+writes on overlapping regions), we might as well serialize writes on
+the whole PG since it lets us represent the current state of the PG
+using two numbers: the epoch of the map on the primary in which the
+most recent write started (this is a bit stranger than it might seem
+since map distribution itself is asynchronous -- see Peering and the
+concept of interval changes) and an increasing per-PG version number
+-- this is referred to in the code with type ``eversion_t`` and stored as
+``pg_info_t::last_update``.  Furthermore, we maintain a log of "recent"
+operations extending back at least far enough to include any
+*unstable* writes (writes which have been started but not committed)
+and objects which aren't up-to-date locally (see recovery and
+backfill).  In practice, the log will extend much further
+(``osd_min_pg_log_entries`` when clean and ``osd_max_pg_log_entries`` when not
+clean) because it's handy for quickly performing recovery.
+
+Using this log, as long as we talk to a non-empty subset of the OSDs
+which must have accepted any completed writes from the most recent
+interval in which we accepted writes, we can determine a conservative
+log which must contain any write which has been reported to a client
+as committed.  There is some freedom here, we can choose any log entry
+between the oldest head remembered by an element of that set (any
+newer cannot have completed without that log containing it) and the
+newest head remembered (clearly, all writes in the log were started,
+so it's fine for us to remember them) as the new head.  This is the
+main point of divergence between replicated pools and EC pools in
+``PG/PrimaryLogPG``: replicated pools try to choose the newest valid
+option to avoid the client needing to replay those operations and
+instead recover the other copies.  EC pools instead try to choose
+the *oldest* option available to them.
+
+The reason for this gets to the heart of the rest of the differences
+in implementation: one copy will not generally be enough to
+reconstruct an EC object.  Indeed, there are encodings where some log
+combinations would leave unrecoverable objects (as with a ``k=4,m=2`` encoding
+where 3 of the replicas remember a write, but the other 3 do not -- we
+don't have 3 copies of either version).  For this reason, log entries
+representing *unstable* writes (writes not yet committed to the
+client) must be rollbackable using only local information on EC pools.
+Log entries in general may therefore be rollbackable (and in that case,
+via a delayed application or via a set of instructions for rolling
+back an inplace update) or not.  Replicated pool log entries are
+never able to be rolled back.
+
+For more details, see ``PGLog.h/cc``, ``osd_types.h:pg_log_t``,
+``osd_types.h:pg_log_entry_t``, and peering in general.
+
+ReplicatedBackend/ECBackend unification strategy
+================================================
+
+PGBackend
+---------
+
+The fundamental difference between replication and erasure coding
+is that replication can do destructive updates while erasure coding
+cannot.  It would be really annoying if we needed to have two entire
+implementations of ``PrimaryLogPG`` since there
+are really only a few fundamental differences:
+
+#. How reads work -- async only, requires remote reads for EC
+#. How writes work -- either restricted to append, or must write aside and do a
+   tpc
+#. Whether we choose the oldest or newest possible head entry during peering
+#. A bit of extra information in the log entry to enable rollback
+
+and so many similarities
+
+#. All of the stats and metadata for objects
+#. The high level locking rules for mixing client IO with recovery and scrub
+#. The high level locking rules for mixing reads and writes without exposing
+   uncommitted state (which might be rolled back or forgotten later)
+#. The process, metadata, and protocol needed to determine the set of osds
+   which participated in the most recent interval in which we accepted writes
+#. etc.
+
+Instead, we choose a few abstractions (and a few kludges) to paper over the differences:
+
+#. ``PGBackend``
+#. ``PGTransaction``
+#. ``PG::choose_acting`` chooses between ``calc_replicated_acting`` and ``calc_ec_acting``
+#. Various bits of the write pipeline disallow some operations based on pool
+   type -- like omap operations, class operation reads, and writes which are
+   not aligned appends (officially, so far) for EC
+#. Misc other kludges here and there
+
+``PGBackend`` and ``PGTransaction`` enable abstraction of differences 1 and 2 above
+and the addition of 4 as needed to the log entries.
+
+The replicated implementation is in ``ReplicatedBackend.h/cc`` and doesn't
+require much additional explanation.  More detail on the ``ECBackend`` can be
+found in ``doc/dev/osd_internals/erasure_coding/ecbackend.rst``.
+
+PGBackend Interface Explanation
+===============================
+
+Note: this is from a design document that predated the Firefly release
+and is probably out of date w.r.t. some of the method names.
+
+Readable vs Degraded
+--------------------
+
+For a replicated pool, an object is readable IFF it is present on
+the primary (at the right version).  For an EC pool, we need at least
+`m` shards present to perform a read, and we need it on the primary.  For
+this reason, ``PGBackend`` needs to include some interfaces for determining
+when recovery is required to serve a read vs a write.  This also
+changes the rules for when peering has enough logs to prove that it
+
+Core Changes:
+
+- | ``PGBackend`` needs to be able to return ``IsPG(Recoverable|Readable)Predicate``
+  | objects to allow the user to make these determinations.
+
+Client Reads
+------------
+
+Reads from a replicated pool can always be satisfied
+synchronously by the primary OSD.  Within an erasure coded pool,
+the primary will need to request data from some number of replicas in
+order to satisfy a read.  ``PGBackend`` will therefore need to provide
+separate ``objects_read_sync`` and ``objects_read_async`` interfaces where
+the former won't be implemented by the ``ECBackend``.
+
+``PGBackend`` interfaces:
+
+- ``objects_read_sync``
+- ``objects_read_async``
+
+Scrubs
+------
+
+We currently have two scrub modes with different default frequencies:
+
+#. [shallow] scrub: compares the set of objects and metadata, but not
+   the contents
+#. deep scrub: compares the set of objects, metadata, and a CRC32 of
+   the object contents (including omap)
+
+The primary requests a scrubmap from each replica for a particular
+range of objects.  The replica fills out this scrubmap for the range
+of objects including, if the scrub is deep, a CRC32 of the contents of
+each object.  The primary gathers these scrubmaps from each replica
+and performs a comparison identifying inconsistent objects.
+
+Most of this can work essentially unchanged with erasure coded PG with
+the caveat that the ``PGBackend`` implementation must be in charge of
+actually doing the scan.
+
+
+``PGBackend`` interfaces:
+
+- ``be_*``
+
+Recovery
+--------
+
+The logic for recovering an object depends on the backend.  With
+the current replicated strategy, we first pull the object replica
+to the primary and then concurrently push it out to the replicas.
+With the erasure coded strategy, we probably want to read the
+minimum number of replica chunks required to reconstruct the object
+and push out the replacement chunks concurrently.
+
+Another difference is that objects in erasure coded PG may be
+unrecoverable without being unfound.  The ``unfound`` state
+should probably be renamed to ``unrecoverable``.  Also, the
+``PGBackend`` implementation will have to be able to direct the search
+for PG replicas with unrecoverable object chunks and to be able
+to determine whether a particular object is recoverable.
+
+
+Core changes:
+
+- ``s/unfound/unrecoverable``
+
+PGBackend interfaces:
+
+- `on_local_recover_start <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L60>`_
+- `on_local_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L66>`_
+- `on_global_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L78>`_
+- `on_peer_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L83>`_
+- `begin_peer_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L90>`_
diff --git a/doc/dev/osd_internals/manifest.rst b/doc/dev/osd_internals/manifest.rst
new file mode 100644
index 000000000..7be4350ea
--- /dev/null
+++ b/doc/dev/osd_internals/manifest.rst
@@ -0,0 +1,589 @@
+========
+Manifest
+========
+
+
+Introduction
+============
+
+As described in ``../deduplication.rst``, adding transparent redirect
+machinery to RADOS would enable a more capable tiering solution
+than RADOS currently has with "cache/tiering".
+
+See ``../deduplication.rst``
+
+At a high level, each object has a piece of metadata embedded in
+the ``object_info_t`` which can map subsets of the object data payload
+to (refcounted) objects in other pools.
+
+This document exists to detail:
+
+1. Manifest data structures
+2. Rados operations for manipulating manifests.
+3. Status and Plans
+
+
+Intended Usage Model
+====================
+
+RBD
+---
+
+For RBD, the primary goal is for either an OSD-internal agent or a
+cluster-external agent to be able to transparently shift portions
+of the constituent 4MB extents between a dedup pool and a hot base
+pool.
+
+As such, RBD operations (including class operations and snapshots)
+must have the same observable results regardless of the current
+status of the object.
+
+Moreover, tiering/dedup operations must interleave with RBD operations
+without changing the result.
+
+Thus, here is a sketch of how I'd expect a tiering agent to perform
+basic operations:
+
+* Demote cold RBD chunk to slow pool:
+
+  1. Read object, noting current user_version.
+  2. In memory, run CDC implementation to fingerprint object.
+  3. Write out each resulting extent to an object in the cold pool
+     using the CAS class.
+  4. Submit operation to base pool:
+
+     * ``ASSERT_VER`` with the user version from the read to fail if the
+       object has been mutated since the read.
+     * ``SET_CHUNK`` for each of the extents to the corresponding object
+       in the base pool.
+     * ``EVICT_CHUNK`` for each extent to free up space in the base pool.
+       Results in each chunk being marked ``MISSING``.
+
+  RBD users should then either see the state prior to the demotion or
+  subsequent to it.
+
+  Note that between 3 and 4, we potentially leak references, so a
+  periodic scrub would be needed to validate refcounts.
+
+* Promote cold RBD chunk to fast pool.
+
+  1. Submit ``TIER_PROMOTE``
+
+For clones, all of the above would be identical except that the
+initial read would need a ``LIST_SNAPS`` to determine which clones exist
+and the ``PROMOTE`` or ``SET_CHUNK``/``EVICT`` operations would need to include
+the ``cloneid``.
+
+RadosGW
+-------
+
+For reads, RADOS Gateway (RGW) could operate as RBD does above relying on the
+manifest machinery in the OSD to hide the distinction between the object
+being dedup'd or present in the base pool
+
+For writes, RGW could operate as RBD does above, but could
+optionally have the freedom to fingerprint prior to doing the write.
+In that case, it could immediately write out the target objects to the
+CAS pool and then atomically write an object with the corresponding
+chunks set.
+
+Status and Future Work
+======================
+
+At the moment, initial versions of a manifest data structure along
+with IO path support and rados control operations exist.  This section
+is meant to outline next steps.
+
+At a high level, our future work plan is:
+
+- Cleanups: Address immediate inconsistencies and shortcomings outlined
+  in the next section.
+- Testing: Rados relies heavily on teuthology failure testing to validate
+  features like cache/tiering.  We'll need corresponding tests for
+  manifest operations.
+- Snapshots: We want to be able to deduplicate portions of clones
+  below the level of the rados snapshot system.  As such, the
+  rados operations below need to be extended to work correctly on
+  clones (e.g.: we should be able to call ``SET_CHUNK`` on a clone, clear the
+  corresponding extent in the base pool, and correctly maintain OSD metadata).
+- Cache/tiering: Ultimately, we'd like to be able to deprecate the existing
+  cache/tiering implementation, but to do that we need to ensure that we
+  can address the same use cases.
+
+
+Cleanups
+--------
+
+The existing implementation has some things that need to be cleaned up:
+
+* ``SET_REDIRECT``: Should create the object if it doesn't exist, otherwise
+  one couldn't create an object atomically as a redirect.
+* ``SET_CHUNK``:
+
+  * Appears to trigger a new clone as user_modify gets set in
+    ``do_osd_ops``.  This probably isn't desirable, see Snapshots section
+    below for some options on how generally to mix these operations
+    with snapshots.  At a minimum, ``SET_CHUNK`` probably shouldn't set
+    user_modify.
+  * Appears to assume that the corresponding section of the object
+    does not exist (sets ``FLAG_MISSING``) but does not check whether the
+    corresponding extent exists already in the object.  Should always
+    leave the extent clean.
+  * Appears to clear the manifest unconditionally if not chunked,
+    that's probably wrong.  We should return an error if it's a
+    ``REDIRECT`` ::
+
+	case CEPH_OSD_OP_SET_CHUNK:
+	  if (oi.manifest.is_redirect()) {
+	    result = -EINVAL;
+	    goto fail;
+	  }
+
+
+* ``TIER_PROMOTE``:
+
+  * ``SET_REDIRECT`` clears the contents of the object.  ``PROMOTE`` appears
+    to copy them back in, but does not unset the redirect or clear the
+    reference. This violates the invariant that a redirect object
+    should be empty in the base pool.  In particular, as long as the
+    redirect is set, it appears that all operations will be proxied
+    even after the promote defeating the purpose.  We do want ``PROMOTE``
+    to be able to atomically replace a redirect with the actual
+    object, so the solution is to clear the redirect at the end of the
+    promote.
+  * For a chunked manifest, we appear to flush prior to promoting.
+    Promotion will often be used to prepare an object for low latency
+    reads and writes, accordingly, the only effect should be to read
+    any ``MISSING`` extents into the base pool.  No flushing should be done.
+
+* High Level:
+
+  * It appears that ``FLAG_DIRTY`` should never be used for an extent pointing
+    at a dedup extent.  Writing the mutated extent back to the dedup pool
+    requires writing a new object since the previous one cannot be mutated,
+    just as it would if it hadn't been dedup'd yet.  Thus, we should always
+    drop the reference and remove the manifest pointer.
+
+  * There isn't currently a way to "evict" an object region.  With the above
+    change to ``SET_CHUNK`` to always retain the existing object region, we
+    need an ``EVICT_CHUNK`` operation to then remove the extent.
+
+
+Testing
+-------
+
+We rely really heavily on randomized failure testing.  As such, we need
+to extend that testing to include dedup/manifest support as well.  Here's
+a short list of the touchpoints:
+
+* Thrasher tests like ``qa/suites/rados/thrash/workloads/cache-snaps.yaml``
+
+  That test, of course, tests the existing cache/tiering machinery.  Add
+  additional files to that directory that instead setup a dedup pool.  Add
+  support to ``ceph_test_rados`` (``src/test/osd/TestRados*``).
+
+* RBD tests
+
+  Add a test that runs an RBD workload concurrently with blind
+  promote/evict operations.
+
+* RGW
+
+  Add a test that runs a rgw workload concurrently with blind
+  promote/evict operations.
+
+
+Snapshots
+---------
+
+Fundamentally we need to be able to manipulate the manifest
+status of clones because we want to be able to dynamically promote,
+flush (if the state was dirty when the clone was created), and evict
+extents from clones.
+
+As such, the plan is to allow the ``object_manifest_t`` for each clone
+to be independent.  Here's an incomplete list of the high level
+tasks:
+
+* Modify the op processing pipeline to permit ``SET_CHUNK``, ``EVICT_CHUNK``
+  to operation directly on clones.
+* Ensure that recovery checks the object_manifest prior to trying to
+  use the overlaps in clone_range.  ``ReplicatedBackend::calc_*_subsets``
+  are the two methods that would likely need to be modified.
+
+See ``snaps.rst`` for a rundown of the ``librados`` snapshot system and OSD
+support details.  I'd like to call out one particular data structure
+we may want to exploit.
+
+The dedup-tool needs to be updated to use ``LIST_SNAPS`` to discover
+clones as part of leak detection.
+
+An important question is how we deal with the fact that many clones
+will frequently have references to the same backing chunks at the same
+offset.  In particular, ``make_writeable`` will generally create a clone
+that shares the same ``object_manifest_t`` references with the exception
+of any extents modified in that transaction.  The metadata that
+commits as part of that transaction must therefore map onto the same
+refcount as before because otherwise we'd have to first increment
+refcounts on backing objects (or risk a reference to a dead object)
+Thus, we introduce a simple convention: consecutive clones which
+share a reference at the same offset share the same refcount.  This
+means that a write that invokes ``make_writeable`` may decrease refcounts,
+but not increase them.  This has some consequences for removing clones.
+Consider the following sequence ::
+
+  write foo [0, 1024)
+  flush foo ->
+    head: [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1
+  snapshot 10
+  write foo [0, 512) ->
+    head:               [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1
+  flush foo ->
+    head: [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1, refcount(ccc)=1
+  snapshot 20
+  write foo [0, 512) (same contents as the original write)
+    head:               [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1
+  flush foo
+    head: [0, 512) aaa, [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1, refcount(ccc)=1
+
+What should be the refcount for ``aaa`` be at the end?  By our
+above rule, it should be ``2`` since the two ```aaa``` refs are not
+contiguous.  However, consider removing clone ``20`` ::
+
+  initial:
+    head: [0, 512) aaa, [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=2, refcount(bbb)=1, refcount(ccc)=1
+  trim 20
+    head: [0, 512) aaa, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1, refcount(ccc)=0
+
+At this point, our rule dictates that ``refcount(aaa)`` is `1`.
+This means that removing ``20`` needs to check for refs held by
+the clones on either side which will then match.
+
+See ``osd_types.h:object_manifest_t::calc_refs_to_drop_on_removal``
+for the logic implementing this rule.
+
+This seems complicated, but it gets us two valuable properties:
+
+1) The refcount change from make_writeable will not block on
+   incrementing a ref
+2) We don't need to load the ``object_manifest_t`` for every clone
+   to determine how to handle removing one -- just the ones
+   immediately preceding and succeeding it.
+
+All clone operations will need to consider adjacent ``chunk_maps``
+when adding or removing references.
+
+Data Structures
+===============
+
+Each RADOS object contains an ``object_manifest_t`` embedded within the
+``object_info_t`` (see ``osd_types.h``):
+
+::
+  
+        struct object_manifest_t {
+                enum {
+                        TYPE_NONE = 0,
+                        TYPE_REDIRECT = 1,
+                        TYPE_CHUNKED = 2,
+                };
+                uint8_t type;  // redirect, chunked, ...
+                hobject_t redirect_target;
+                std::map<uint64_t, chunk_info_t> chunk_map;
+        }
+
+The ``type`` enum reflects three possible states an object can be in:
+
+1. ``TYPE_NONE``: normal RADOS object
+2. ``TYPE_REDIRECT``: object payload is backed by a single object
+   specified by ``redirect_target``
+3. ``TYPE_CHUNKED: object payload is distributed among objects with
+   size and offset specified by the ``chunk_map``. ``chunk_map`` maps
+   the offset of the chunk to a ``chunk_info_t`` as shown below, also
+   specifying the ``length``, target `OID`, and ``flags``.
+
+::
+
+        struct chunk_info_t {
+          typedef enum {
+            FLAG_DIRTY = 1, 
+            FLAG_MISSING = 2,
+            FLAG_HAS_REFERENCE = 4,
+            FLAG_HAS_FINGERPRINT = 8,
+          } cflag_t;
+          uint32_t offset;
+          uint32_t length;
+          hobject_t oid;
+          cflag_t flags;   // FLAG_*
+
+
+``FLAG_DIRTY`` at this time can happen if an extent with a fingerprint
+is written.  This should be changed to drop the fingerprint instead.
+
+
+Request Handling
+================
+
+Similarly to cache/tiering, the initial touchpoint is
+``maybe_handle_manifest_detail``.
+
+For manifest operations listed below, we return ``NOOP`` and continue onto
+dedicated handling within ``do_osd_ops``.
+
+For redirect objects which haven't been promoted (apparently ``oi.size >
+0`` indicates that it's present?) we proxy reads and writes.
+
+For reads on ``TYPE_CHUNKED``, if ``can_proxy_chunked_read`` (basically, all
+of the ops are reads of extents in the ``object_manifest_t chunk_map``),
+we proxy requests to those objects.
+
+
+RADOS Interface
+================
+
+To set up deduplication one must provision two pools. One will act as the 
+base pool and the other will act as the chunk pool. The base pool need to be
+configured with the ``fingerprint_algorithm`` option as follows.
+
+::
+
+  ceph osd pool set $BASE_POOL fingerprint_algorithm sha1|sha256|sha512 
+  --yes-i-really-mean-it
+
+Create objects ::
+
+  rados -p base_pool put foo ./foo
+  rados -p chunk_pool put foo-chunk ./foo-chunk
+
+Make a manifest object ::
+
+  rados -p base_pool set-chunk foo $START_OFFSET $END_OFFSET --target-pool chunk_pool foo-chunk $START_OFFSET --with-reference
+
+Operations:
+
+* ``set-redirect``
+
+  Set a redirection between a ``base_object`` in the ``base_pool`` and a ``target_object`` 
+  in the ``target_pool``.
+  A redirected object will forward all operations from the client to the 
+  ``target_object``. ::
+
+        void set_redirect(const std::string& tgt_obj, const IoCtx& tgt_ioctx,
+		      uint64_t tgt_version, int flag = 0);
+  
+        rados -p base_pool set-redirect <base_object> --target-pool <target_pool> 
+         <target_object>
+
+  Returns ``ENOENT`` if the object does not exist (TODO: why?)
+  Returns ``EINVAL`` if the object already is a redirect.
+
+  Takes a reference to target as part of operation, can possibly leak a ref
+  if the acting set resets and the client dies between taking the ref and
+  recording the redirect.
+
+  Truncates object, clears omap, and clears xattrs as a side effect.
+
+  At the top of ``do_osd_ops``, does not set user_modify.
+
+  This operation is not a user mutation and does not trigger a clone to be created.
+
+  There are two purposes of ``set_redirect``:
+
+  1. Redirect all operation to the target object (like proxy)
+  2. Cache when ``tier_promote`` is called (redirect will be cleared at this time).
+
+* ``set-chunk`` 
+
+  Set the ``chunk-offset`` in a ``source_object`` to make a link between it and a 
+  ``target_object``. ::
+
+        void set_chunk(uint64_t src_offset, uint64_t src_length, const IoCtx& tgt_ioctx,
+                   std::string tgt_oid, uint64_t tgt_offset, int flag = 0);
+  
+        rados -p base_pool set-chunk <source_object> <offset> <length> --target-pool 
+         <caspool> <target_object> <target-offset> 
+
+  Returns ``ENOENT`` if the object does not exist (TODO: why?)
+  Returns ``EINVAL`` if the object already is a redirect.
+  Returns ``EINVAL`` if on ill-formed parameter buffer.
+  Returns ``ENOTSUPP`` if existing mapped chunks overlap with new chunk mapping.
+
+  Takes references to targets as part of operation, can possibly leak refs
+  if the acting set resets and the client dies between taking the ref and
+  recording the redirect.
+
+  Truncates object, clears omap, and clears xattrs as a side effect.
+
+  This operation is not a user mutation and does not trigger a clone to be created.
+
+  TODO: ``SET_CHUNK`` appears to clear the manifest unconditionally if it's not chunked. ::
+
+       if (!oi.manifest.is_chunked()) {
+         oi.manifest.clear();
+       }
+
+* ``evict-chunk``
+
+  Clears an extent from an object leaving only the manifest link between
+  it and the ``target_object``. ::
+
+        void evict_chunk(
+	  uint64_t offset, uint64_t length, int flag = 0);
+  
+        rados -p base_pool evict-chunk <offset> <length> <object>
+
+  Returns ``EINVAL`` if the extent is not present in the manifest.
+
+  Note: this does not exist yet.
+
+
+* ``tier-promote`` 
+
+  Promotes the object ensuring that subsequent reads and writes will be local ::
+
+        void tier_promote();
+
+        rados -p base_pool tier-promote <obj-name> 
+
+  Returns ``ENOENT`` if the object does not exist
+
+  For a redirect manifest, copies data to head.
+
+  TODO: Promote on a redirect object needs to clear the redirect.
+
+  For a chunked manifest, reads all MISSING extents into the base pool,
+  subsequent reads and writes will be served from the base pool.
+
+  Implementation Note: For a chunked manifest, calls ``start_copy`` on itself.  The
+  resulting ``copy_get`` operation will issue reads which will then be redirected by
+  the normal manifest read machinery.
+
+  Does not set the ``user_modify`` flag.
+
+  Future work will involve adding support for specifying a ``clone_id``.
+
+* ``unset-manifest``
+
+  Unset the manifest info in the object that has manifest. ::
+
+        void unset_manifest();
+
+        rados -p base_pool unset-manifest <obj-name>
+
+  Clears manifest chunks or redirect.  Lazily releases references, may
+  leak.
+
+  ``do_osd_ops`` seems not to include it in the ``user_modify=false`` ``ignorelist``,
+  and so will trigger a snapshot.  Note, this will be true even for a
+  redirect though ``SET_REDIRECT`` does not flip ``user_modify``.  This should
+  be fixed -- ``unset-manifest`` should not be a ``user_modify``.
+
+* ``tier-flush``
+
+  Flush the object which has chunks to the chunk pool. ::
+
+        void tier_flush();
+
+        rados -p base_pool tier-flush <obj-name>
+
+  Included in the ``user_modify=false`` ``ignorelist``, does not trigger a clone.
+
+  Does not evict the extents.
+
+
+ceph-dedup-tool
+===============
+
+``ceph-dedup-tool`` has two features: finding an optimal chunk offset for dedup chunking 
+and fixing the reference count (see ``./refcount.rst``).
+
+* Find an optimal chunk offset
+
+  a. Fixed chunk  
+
+  To find out a fixed chunk length, you need to run the following command many 
+  times while changing the ``chunk_size``. ::
+
+            ceph-dedup-tool --op estimate --pool $POOL --chunk-size chunk_size  
+              --chunk-algorithm fixed --fingerprint-algorithm sha1|sha256|sha512
+
+  b. Rabin chunk(Rabin-Karp algorithm) 
+
+  Rabin-Karp is a string-searching algorithm based
+  on a rolling hash. But a rolling hash is not enough to do deduplication because 
+  we don't know the chunk boundary. So, we need content-based slicing using 
+  a rolling hash for content-defined chunking.
+  The current implementation uses the simplest approach: look for chunk boundaries 
+  by inspecting the rolling hash for pattern (like the
+  lower N bits are all zeroes). 
+      
+  Users who want to use deduplication need to find an ideal chunk offset.
+  To find out ideal chunk offset, users should discover
+  the optimal configuration for their data workload via ``ceph-dedup-tool``.
+  This information will then be used for object chunking through
+  the ``set-chunk`` API. ::
+
+              ceph-dedup-tool --op estimate --pool $POOL --min-chunk min_size  
+                --chunk-algorithm rabin --fingerprint-algorithm rabin
+
+  ``ceph-dedup-tool`` has many options to utilize ``rabin chunk``.
+  These are options for ``rabin chunk``. ::
+
+              --mod-prime <uint64_t>
+              --rabin-prime <uint64_t>
+              --pow <uint64_t>
+              --chunk-mask-bit <uint32_t>
+              --window-size <uint32_t>
+              --min-chunk <uint32_t>
+              --max-chunk <uint64_t>
+
+   Users need to refer following equation to use above options for ``rabin chunk``. ::
+
+              rabin_hash = 
+                (rabin_hash * rabin_prime + new_byte - old_byte * pow) % (mod_prime)
+
+  c. Fixed chunk vs content-defined chunk
+
+  Content-defined chunking may or not be optimal solution.
+  For example,
+
+  Data chunk ``A`` : ``abcdefgabcdefgabcdefg``
+
+  Let's think about Data chunk ``A``'s deduplication. The ideal chunk offset is
+  from ``1`` to ``7`` (``abcdefg``). So, if we use fixed chunk, ``7`` is optimal chunk length.
+  But, in the case of content-based slicing, the optimal chunk length
+  could not be found (dedup ratio will not be 100%).
+  Because we need to find optimal parameter such
+  as boundary bit, window size and prime value. This is as easy as fixed chunk.
+  But, content defined chunking is very effective in the following case.
+
+    Data chunk ``B`` : ``abcdefgabcdefgabcdefg``
+
+    Data chunk ``C`` : ``Tabcdefgabcdefgabcdefg``
+
+
+* Fix reference count
+  
+  The key idea behind of reference counting for dedup is false-positive, which means 
+  ``(manifest object (no ref),, chunk object(has ref))`` happen instead of 
+  ``(manifest object (has ref), chunk 1(no ref))``.
+  To fix such inconsistencies, ``ceph-dedup-tool`` supports ``chunk_scrub``. ::
+
+          ceph-dedup-tool --op chunk_scrub --chunk_pool $CHUNK_POOL
+
diff --git a/doc/dev/osd_internals/map_message_handling.rst b/doc/dev/osd_internals/map_message_handling.rst
new file mode 100644
index 000000000..f8104f3fd
--- /dev/null
+++ b/doc/dev/osd_internals/map_message_handling.rst
@@ -0,0 +1,131 @@
+===========================
+Map and PG Message handling
+===========================
+
+Overview
+--------
+The OSD handles routing incoming messages to PGs, creating the PG if necessary
+in some cases.
+
+PG messages generally come in two varieties:
+
+  1. Peering Messages
+  2. Ops/SubOps
+
+There are several ways in which a message might be dropped or delayed.  It is
+important that the message delaying does not result in a violation of certain
+message ordering requirements on the way to the relevant PG handling logic:
+
+  1. Ops referring to the same object must not be reordered.
+  2. Peering messages must not be reordered.
+  3. Subops must not be reordered.
+
+MOSDMap
+-------
+MOSDMap messages may come from either monitors or other OSDs.  Upon receipt, the
+OSD must perform several tasks:
+
+  1. Persist the new maps to the filestore.
+     Several PG operations rely on having access to maps dating back to the last
+     time the PG was clean.
+  2. Update and persist the superblock.
+  3. Update OSD state related to the current map.
+  4. Expose new maps to PG processes via *OSDService*.
+  5. Remove PGs due to pool removal.
+  6. Queue dummy events to trigger PG map catchup.
+
+Each PG asynchronously catches up to the currently published map during
+process_peering_events before processing the event.  As a result, different
+PGs may have different views as to the "current" map.
+
+One consequence of this design is that messages containing submessages from
+multiple PGs (MOSDPGInfo, MOSDPGQuery, MOSDPGNotify) must tag each submessage
+with the PG's epoch as well as tagging the message as a whole with the OSD's
+current published epoch.
+
+MOSDPGOp/MOSDPGSubOp
+--------------------
+See OSD::dispatch_op, OSD::handle_op, OSD::handle_sub_op
+
+MOSDPGOps are used by clients to initiate rados operations. MOSDSubOps are used
+between OSDs to coordinate most non peering activities including replicating
+MOSDPGOp operations.
+
+OSD::require_same_or_newer map checks that the current OSDMap is at least
+as new as the map epoch indicated on the message.  If not, the message is
+queued in OSD::waiting_for_osdmap via OSD::wait_for_new_map.  Note, this
+cannot violate the above conditions since any two messages will be queued
+in order of receipt and if a message is received with epoch e0, a later message
+from the same source must be at epoch at least e0.  Note that two PGs from
+the same OSD count for these purposes as different sources for single PG
+messages.  That is, messages from different PGs may be reordered.
+
+
+MOSDPGOps follow the following process:
+
+  1. OSD::handle_op: validates permissions and crush mapping.
+     discard the request if they are not connected and the client cannot get the reply ( See OSD::op_is_discardable )
+     See OSDService::handle_misdirected_op
+     See PG::op_has_sufficient_caps
+     See OSD::require_same_or_newer_map
+  2. OSD::enqueue_op
+
+MOSDSubOps follow the following process:
+
+  1. OSD::handle_sub_op checks that sender is an OSD
+  2. OSD::enqueue_op
+
+OSD::enqueue_op calls PG::queue_op which checks waiting_for_map before calling OpWQ::queue which adds the op to the queue of the PG responsible for handling it.
+
+OSD::dequeue_op is then eventually called, with a lock on the PG.  At
+this time, the op is passed to PG::do_request, which checks that:
+
+  1. the PG map is new enough (PG::must_delay_op)
+  2. the client requesting the op has enough permissions (PG::op_has_sufficient_caps)
+  3. the op is not to be discarded (PG::can_discard_{request,op,subop,scan,backfill})
+  4. the PG is active (PG::flushed boolean)
+  5. the op is a CEPH_MSG_OSD_OP and the PG is in PG_STATE_ACTIVE state and not in PG_STATE_REPLAY 
+
+If these conditions are not met, the op is either discarded or queued for later processing. If all conditions are met, the op is processed according to its type:
+
+  1. CEPH_MSG_OSD_OP is handled by PG::do_op
+  2. MSG_OSD_SUBOP is handled by PG::do_sub_op
+  3. MSG_OSD_SUBOPREPLY is handled by PG::do_sub_op_reply
+  4. MSG_OSD_PG_SCAN is handled by PG::do_scan
+  5. MSG_OSD_PG_BACKFILL is handled by PG::do_backfill
+
+CEPH_MSG_OSD_OP processing
+--------------------------
+
+PrimaryLogPG::do_op handles CEPH_MSG_OSD_OP op and will queue it
+
+  1. in wait_for_all_missing if it is a CEPH_OSD_OP_PGLS for a designated snapid and some object updates are still missing
+  2. in waiting_for_active if the op may write but the scrubber is working
+  3. in waiting_for_missing_object if the op requires an object or a snapdir or a specific snap that is still missing
+  4. in waiting_for_degraded_object if the op may write an object or a snapdir that is degraded, or if another object blocks it ("blocked_by")
+  5. in waiting_for_backfill_pos if the op requires an object that will be available after the backfill is complete
+  6. in waiting_for_ack if an ack from another OSD is expected
+  7. in waiting_for_ondisk if the op is waiting for a write to complete
+
+Peering Messages
+----------------
+See OSD::handle_pg_(notify|info|log|query)
+
+Peering messages are tagged with two epochs:
+
+  1. epoch_sent: map epoch at which the message was sent
+  2. query_epoch: map epoch at which the message triggering the message was sent
+
+These are the same in cases where there was no triggering message.  We discard
+a peering message if the message's query_epoch if the PG in question has entered
+a new epoch (See PG::old_peering_evt, PG::queue_peering_event).  Notifies,
+infos, notifies, and logs are all handled as PG::PeeringMachine events and
+are wrapped by PG::queue_* by PG::CephPeeringEvts, which include the created
+state machine event along with epoch_sent and query_epoch in order to
+generically check PG::old_peering_message upon insertion and removal from the
+queue.
+
+Note, notifies, logs, and infos can trigger the creation of a PG.  See
+OSD::get_or_create_pg.
+
+
diff --git a/doc/dev/osd_internals/mclock_wpq_cmp_study.rst b/doc/dev/osd_internals/mclock_wpq_cmp_study.rst
new file mode 100644
index 000000000..31ad18409
--- /dev/null
+++ b/doc/dev/osd_internals/mclock_wpq_cmp_study.rst
@@ -0,0 +1,476 @@
+=========================================
+ QoS Study with mClock and WPQ Schedulers
+=========================================
+
+Introduction
+============
+
+The mClock scheduler provides three controls for each service using it. In Ceph,
+the services using mClock are for example client I/O, background recovery,
+scrub, snap trim and PG deletes. The three controls such as *weight*,
+*reservation* and *limit* are used for predictable allocation of resources to
+each service in proportion to its weight subject to the constraint that the
+service receives at least its reservation and no more than its limit. In Ceph,
+these controls are used to allocate IOPS for each service type provided the IOPS
+capacity of each OSD is known. The mClock scheduler is based on
+`the dmClock algorithm`_. See :ref:`dmclock-qos` section for more details.
+
+Ceph's use of mClock was primarily experimental and approached with an
+exploratory mindset. This is still true with other organizations and individuals
+who continue to either use the codebase or modify it according to their needs.
+
+DmClock exists in its own repository_. Before the Ceph *Pacific* release,
+mClock could be enabled by setting the :confval:`osd_op_queue` Ceph option to
+"mclock_scheduler". Additional mClock parameters like *reservation*, *weight*
+and *limit* for each service type could be set using Ceph options.
+For example, ``osd_mclock_scheduler_client_[res,wgt,lim]`` is one such option.
+See :ref:`dmclock-qos` section for more details. Even with all the mClock
+options set, the full capability of mClock could not be realized due to:
+
+- Unknown OSD capacity in terms of throughput (IOPS).
+- No limit enforcement. In other words, services using mClock were allowed to
+  exceed their limits resulting in the desired QoS goals not being met.
+- Share of each service type not distributed across the number of operational
+  shards.
+
+To resolve the above, refinements were made to the mClock scheduler in the Ceph
+code base. See :doc:`/rados/configuration/mclock-config-ref`. With the
+refinements, the usage of mClock is a bit more user-friendly and intuitive. This
+is one step of many to refine and optimize the way mClock is used in Ceph.
+
+Overview
+========
+
+A comparison study was performed as part of efforts to refine the mClock
+scheduler. The study involved running tests with client ops and background
+recovery operations in parallel with the two schedulers. The results were
+collated and then compared. The following statistics were compared between the
+schedulers from the test results for each service type:
+
+- External client
+
+  - Average throughput(IOPS),
+  - Average and percentile(95th, 99th, 99.5th) latency,
+
+- Background recovery
+
+  - Average recovery throughput,
+  - Number of misplaced objects recovered per second
+
+Test Environment
+================
+
+1. **Software Configuration**: CentOS 8.1.1911 Linux Kernel 4.18.0-193.6.3.el8_2.x86_64
+2. **CPU**: 2 x Intel® Xeon® CPU E5-2650 v3 @ 2.30GHz
+3. **nproc**: 40
+4. **System Memory**: 64 GiB
+5. **Tuned-adm Profile**: network-latency
+6. **CephVer**: 17.0.0-2125-g94f550a87f (94f550a87fcbda799afe9f85e40386e6d90b232e) quincy (dev)
+7. **Storage**:
+
+  - Intel® NVMe SSD DC P3700 Series (SSDPE2MD800G4) [4 x 800GB]
+  - Seagate Constellation 7200 RPM 64MB Cache SATA 6.0Gb/s HDD (ST91000640NS) [4 x 1TB]
+
+Test Methodology
+================
+
+Ceph cbt_ was used to test the recovery scenarios. A new recovery test to
+generate background recoveries with client I/Os in parallel was created.
+See the next section for the detailed test steps. The test was executed 3 times
+with the default *Weighted Priority Queue (WPQ)* scheduler for comparison
+purposes. This was done to establish a credible mean value to compare
+the mClock scheduler results at a later point.
+
+After this, the same test was executed with mClock scheduler and with different
+mClock profiles i.e., *high_client_ops*, *balanced* and *high_recovery_ops* and
+the results collated for comparison. With each profile, the test was
+executed 3 times, and the average of those runs are reported in this study.
+
+.. note:: Tests with HDDs were performed with and without the bluestore WAL and
+          dB configured. The charts discussed further below help bring out the
+          comparison across the schedulers and their configurations.
+
+Establish Baseline Client Throughput (IOPS)
+===========================================
+
+Before the actual recovery tests, the baseline throughput was established for
+both the SSDs and the HDDs on the test machine by following the steps mentioned
+in the :doc:`/rados/configuration/mclock-config-ref` document under
+the "Benchmarking Test Steps Using CBT" section. For this study, the following
+baseline throughput for each device type was determined:
+
++--------------------------------------+-------------------------------------------+
+|  Device Type                         | Baseline Throughput(@4KiB Random Writes)  |
++======================================+===========================================+
+| **NVMe SSD**                         | 21500 IOPS (84 MiB/s)                     |
++--------------------------------------+-------------------------------------------+
+| **HDD (with bluestore WAL & dB)**    | 340 IOPS (1.33 MiB/s)                     |
++--------------------------------------+-------------------------------------------+
+| **HDD (without bluestore WAL & dB)** | 315 IOPS (1.23 MiB/s)                     |
++--------------------------------------+-------------------------------------------+
+
+.. note:: The :confval:`bluestore_throttle_bytes` and
+          :confval:`bluestore_throttle_deferred_bytes` for SSDs were determined to be
+          256 KiB. For HDDs, it was 40MiB. The above throughput was obtained
+          by running 4 KiB random writes at a queue depth of 64 for 300 secs.
+
+MClock Profile Allocations
+==========================
+
+The low-level mClock shares per profile are shown in the tables below. For
+parameters like *reservation* and *limit*, the shares are represented as a
+percentage of the total OSD capacity. For the *high_client_ops* profile, the
+*reservation* parameter is set to 50% of the total OSD capacity. Therefore, for
+the NVMe(baseline 21500 IOPS) device, a minimum of 10750 IOPS is reserved for
+client operations. These allocations are made under the hood once
+a profile is enabled.
+
+The *weight* parameter is unitless. See :ref:`dmclock-qos`.
+
+high_client_ops(default)
+````````````````````````
+
+This profile allocates more reservation and limit to external clients ops
+when compared to background recoveries and other internal clients within
+Ceph. This profile is enabled by default.
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 50%         | 2      | MAX   |
++------------------------+-------------+--------+-------+
+| background recovery    | 25%         | 1      | 100%  |
++------------------------+-------------+--------+-------+
+| background best effort | 25%         | 2      | MAX   |
++------------------------+-------------+--------+-------+
+
+balanced
+`````````
+
+This profile allocates equal reservations to client ops and background
+recovery ops. The internal best effort client get a lower reservation
+but a very high limit so that they can complete quickly if there are
+no competing services.
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 40%         | 1      | 100%  |
++------------------------+-------------+--------+-------+
+| background recovery    | 40%         | 1      | 150%  |
++------------------------+-------------+--------+-------+
+| background best effort | 20%         | 2      | MAX   |
++------------------------+-------------+--------+-------+
+
+high_recovery_ops
+`````````````````
+
+This profile allocates more reservation to background recoveries when
+compared to external clients and other internal clients within Ceph. For
+example, an admin may enable this profile temporarily to speed-up background
+recoveries during non-peak hours.
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 30%         | 1      | 80%   |
++------------------------+-------------+--------+-------+
+| background recovery    | 60%         | 2      | 200%  |
++------------------------+-------------+--------+-------+
+| background best effort | 1 (MIN)     | 2      | MAX   |
++------------------------+-------------+--------+-------+
+
+custom
+```````
+
+The custom profile allows the user to have complete control of the mClock
+and Ceph config parameters. To use this profile, the user must have a deep
+understanding of the workings of Ceph and the mClock scheduler. All the
+*reservation*, *weight* and *limit* parameters of the different service types
+must be set manually along with any Ceph option(s). This profile may be used
+for experimental and exploratory purposes or if the built-in profiles do not
+meet the requirements. In such cases, adequate testing must be performed prior
+to enabling this profile.
+
+
+Recovery Test Steps
+===================
+
+Before bringing up the Ceph cluster, the following mClock configuration
+parameters were set appropriately based on the obtained baseline throughput
+from the previous section:
+
+- :confval:`osd_mclock_max_capacity_iops_hdd`
+- :confval:`osd_mclock_max_capacity_iops_ssd`
+- :confval:`osd_mclock_profile`
+
+See :doc:`/rados/configuration/mclock-config-ref` for more details.
+
+Test Steps(Using cbt)
+`````````````````````
+
+1. Bring up the Ceph cluster with 4 osds.
+2. Configure the OSDs with replication factor 3.
+3. Create a recovery pool to populate recovery data.
+4. Create a client pool and prefill some objects in it.
+5. Create the recovery thread and mark an OSD down and out.
+6. After the cluster handles the OSD down event, recovery data is
+   prefilled into the recovery pool. For the tests involving SSDs, prefill 100K
+   4MiB objects into the recovery pool. For the tests involving HDDs, prefill
+   5K 4MiB objects into the recovery pool.
+7. After the prefill stage is completed, the downed OSD is brought up and in.
+   The backfill phase starts at this point.
+8. As soon as the backfill/recovery starts, the test proceeds to initiate client
+   I/O on the client pool on another thread using a single client.
+9. During step 8 above, statistics related to the client latency and
+   bandwidth are captured by cbt. The test also captures the total number of
+   misplaced objects and the number of misplaced objects recovered per second.
+
+To summarize, the steps above creates 2 pools during the test. Recovery is
+triggered on one pool and client I/O is triggered on the other simultaneously.
+Statistics captured during the tests are discussed below.
+
+
+Non-Default Ceph Recovery Options
+`````````````````````````````````
+
+Apart from the non-default bluestore throttle already mentioned above, the
+following set of Ceph recovery related options were modified for tests with both
+the WPQ and mClock schedulers.
+
+- :confval:`osd_max_backfills` = 1000
+- :confval:`osd_recovery_max_active` = 1000
+- :confval:`osd_async_recovery_min_cost` = 1
+
+The above options set a high limit on the number of concurrent local and
+remote backfill operations per OSD. Under these conditions the capability of the
+mClock scheduler was tested and the results are discussed below.
+
+Test Results
+============
+
+Test Results With NVMe SSDs
+```````````````````````````
+
+Client Throughput Comparison
+----------------------------
+
+The chart below shows the average client throughput comparison across the
+schedulers and their respective configurations.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Throughput_NVMe_SSD_WPQ_vs_mClock.png
+
+
+WPQ(def) in the chart shows the average client throughput obtained
+using the WPQ scheduler with all other Ceph configuration settings set to
+default values. The default setting for :confval:`osd_max_backfills` limits the number
+of concurrent local and remote backfills or recoveries per OSD to 1. As a
+result, the average client throughput obtained is impressive at just over 18000
+IOPS when compared to the baseline value which is 21500 IOPS.
+
+However, with WPQ scheduler along with non-default options mentioned in section
+`Non-Default Ceph Recovery Options`_, things are quite different as shown in the
+chart for WPQ(BST). In this case, the average client throughput obtained drops
+dramatically to only 2544 IOPS. The non-default recovery options clearly had a
+significant impact on the client throughput. In other words, recovery operations
+overwhelm the client operations. Sections further below discuss the recovery
+rates under these conditions.
+
+With the non-default options, the same test was executed with mClock and with
+the default profile (*high_client_ops*) enabled. As per the profile allocation,
+the reservation goal of 50% (10750 IOPS) is being met with an average throughput
+of 11209 IOPS during the course of recovery operations. This is more than 4x
+times the throughput obtained with WPQ(BST).
+
+Similar throughput with the *balanced* (11017 IOPS) and *high_recovery_ops*
+(11153 IOPS) profile was obtained as seen in the chart above. This clearly
+demonstrates that mClock is able to provide the desired QoS for the client
+with multiple concurrent backfill/recovery operations in progress.
+
+Client Latency Comparison
+-------------------------
+
+The chart below shows the average completion latency (*clat*) along with the
+average 95th, 99th and 99.5th percentiles across the schedulers and their
+respective configurations.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Latency_Percentiles_NVMe_SSD_WPQ_vs_mClock.png
+
+The average *clat* latency obtained with WPQ(Def) was 3.535 msec. But in this
+case the number of concurrent recoveries was very much limited at an average of
+around 97 objects/sec or ~388 MiB/s and a major contributing factor to the low
+latency seen by the client.
+
+With WPQ(BST) and with the non-default recovery options, things are very
+different with the average *clat* latency shooting up to an average of almost
+25 msec which is 7x times worse! This is due to the high number of concurrent
+recoveries which was measured to be ~350 objects/sec or ~1.4 GiB/s which is
+close to the maximum OSD bandwidth.
+
+With mClock enabled and with the default *high_client_ops* profile, the average
+*clat* latency was 5.688 msec which is impressive considering the high number
+of concurrent active background backfill/recoveries. The recovery rate was
+throttled down by mClock to an average of 80 objects/sec or ~320 MiB/s according
+to the minimum profile allocation of 25% of the maximum OSD bandwidth thus
+allowing the client operations to meet the QoS goal.
+
+With the other profiles like *balanced* and *high_recovery_ops*, the average
+client *clat* latency didn't change much and stayed between 5.7 - 5.8 msec with
+variations in the average percentile latency as observed from the chart above.
+
+.. image:: ../../images/mclock_wpq_study/Clat_Latency_Comparison_NVMe_SSD_WPQ_vs_mClock.png
+
+Perhaps a more interesting chart is the comparison chart shown above that
+tracks the average *clat* latency variations through the duration of the test.
+The chart shows the differences in the average latency between the
+WPQ and mClock profiles). During the initial phase of the test, for about 150
+secs, the differences in the average latency between the WPQ scheduler and
+across the profiles of mClock scheduler are quite evident and self explanatory.
+The *high_client_ops* profile shows the lowest latency followed by *balanced*
+and *high_recovery_ops* profiles. The WPQ(BST) had the highest average latency
+through the course of the test.
+
+Recovery Statistics Comparison
+------------------------------
+
+Another important aspect to consider is how the recovery bandwidth and recovery
+time are affected by mClock profile settings. The chart below outlines the
+recovery rates and times for each mClock profile and how they differ with the
+WPQ scheduler. The total number of objects to be recovered in all the cases was
+around 75000 objects as observed in the chart below.
+
+.. image:: ../../images/mclock_wpq_study/Recovery_Rate_Comparison_NVMe_SSD_WPQ_vs_mClock.png
+
+Intuitively, the *high_client_ops* should impact recovery operations the most
+and this is indeed the case as it took an average of 966 secs for the
+recovery to complete at 80 Objects/sec. The recovery bandwidth as expected was
+the lowest at an average of ~320 MiB/s.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Obj_Rec_Throughput_NVMe_SSD_WPQ_vs_mClock.png
+
+The *balanced* profile provides a good middle ground by allocating the same
+reservation and weight to client and recovery operations. The recovery rate
+curve falls between the *high_recovery_ops* and *high_client_ops* curves with
+an average bandwidth of ~480 MiB/s and taking an average of ~647 secs at ~120
+Objects/sec to complete the recovery.
+
+The *high_recovery_ops* profile provides the fastest way to complete recovery
+operations at the expense of other operations. The recovery bandwidth was
+nearly 2x the bandwidth at ~635 MiB/s when compared to the bandwidth observed
+using the *high_client_ops* profile. The average object recovery rate was ~159
+objects/sec and completed the fastest in approximately 488 secs.
+
+Test Results With HDDs (WAL and dB configured)
+``````````````````````````````````````````````
+
+The recovery tests were performed on HDDs with bluestore WAL and dB configured
+on faster NVMe SSDs. The baseline throughput measured was 340 IOPS.
+
+Client Throughput & latency Comparison
+--------------------------------------
+
+The average client throughput comparison for WPQ and mClock and its profiles
+are shown in the chart below.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Throughput_HDD_WALdB_WPQ_vs_mClock.png
+
+With WPQ(Def), the average client throughput obtained was ~308 IOPS since the
+the number of concurrent recoveries was very much limited. The average *clat*
+latency was ~208 msec.
+
+However for WPQ(BST), due to concurrent recoveries client throughput is affected
+significantly with 146 IOPS and an average *clat* latency of 433 msec.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_WALdB_WPQ_vs_mClock.png
+
+With the *high_client_ops* profile, mClock was able to meet the QoS requirement
+for client operations with an average throughput of 271 IOPS which is nearly
+80% of the baseline throughput at an average *clat* latency of 235 msecs.
+
+For *balanced* and *high_recovery_ops* profiles, the average client throughput
+came down marginally to ~248 IOPS and ~240 IOPS respectively. The average *clat*
+latency as expected increased to ~258 msec and ~265 msec respectively.
+
+.. image:: ../../images/mclock_wpq_study/Clat_Latency_Comparison_HDD_WALdB_WPQ_vs_mClock.png
+
+The *clat* latency comparison chart above provides a more comprehensive insight
+into the differences in latency through the course of the test. As observed
+with the NVMe SSD case, *high_client_ops* profile shows the lowest latency in
+the HDD case as well followed by the *balanced* and *high_recovery_ops* profile.
+It's fairly easy to discern this between the profiles during the first 200 secs
+of the test.
+
+Recovery Statistics Comparison
+------------------------------
+
+The charts below compares the recovery rates and times. The total number of
+objects to be recovered in all the cases using HDDs with WAL and dB was around
+4000 objects as observed in the chart below.
+
+.. image:: ../../images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_WALdB_WPQ_vs_mClock.png
+
+As expected, the *high_client_ops* impacts recovery operations the most as it
+took an average of  ~1409 secs for the recovery to complete at ~3 Objects/sec.
+The recovery bandwidth as expected was the lowest at ~11 MiB/s.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_WALdB_WPQ_vs_mClock.png
+
+The *balanced* profile as expected provides a decent compromise with an an
+average bandwidth of ~16.5 MiB/s and taking an average of ~966 secs at ~4
+Objects/sec to complete the recovery.
+
+The *high_recovery_ops* profile is the fastest with nearly 2x the bandwidth at
+~21 MiB/s when compared to the *high_client_ops* profile. The average object
+recovery rate was ~5 objects/sec and completed in approximately 747 secs. This
+is somewhat similar to the recovery time observed with WPQ(Def) at 647 secs with
+a bandwidth of 23 MiB/s and at a rate of 5.8 objects/sec.
+
+Test Results With HDDs (No WAL and dB configured)
+`````````````````````````````````````````````````
+
+The recovery tests were also performed on HDDs without bluestore WAL and dB
+configured. The baseline throughput measured was 315 IOPS.
+
+This type of configuration without WAL and dB configured is probably rare
+but testing was nevertheless performed to get a sense of how mClock performs
+under a very restrictive environment where the OSD capacity is at the lower end.
+The sections and charts below are very similar to the ones presented above and
+are provided here for reference.
+
+Client Throughput & latency Comparison
+--------------------------------------
+
+The average client throughput, latency and percentiles are compared as before
+in the set of charts shown below.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
+
+.. image:: ../../images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_NoWALdB_WPQ_vs_mClock.png
+
+.. image:: ../../images/mclock_wpq_study/Clat_Latency_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
+
+Recovery Statistics Comparison
+------------------------------
+
+The recovery rates and times are shown in the charts below.
+
+.. image:: ../../images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
+
+.. image:: ../../images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
+
+Key Takeaways and Conclusion
+============================
+
+- mClock is able to provide the desired QoS using profiles to allocate proper
+  *reservation*, *weight* and *limit* to the service types.
+- By using the cost per I/O and the cost per byte parameters, mClock can
+  schedule operations appropriately for the different device types(SSD/HDD).
+
+The study so far shows promising results with the refinements made to the mClock
+scheduler. Further refinements to mClock and profile tuning are planned. Further
+improvements will also be based on feedback from broader testing on larger
+clusters and with different workloads.
+
+.. _the dmClock algorithm: https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Gulati.pdf
+.. _repository: https://github.com/ceph/dmclock
+.. _cbt: https://github.com/ceph/cbt
diff --git a/doc/dev/osd_internals/osd_overview.rst b/doc/dev/osd_internals/osd_overview.rst
new file mode 100644
index 000000000..192ddf8ca
--- /dev/null
+++ b/doc/dev/osd_internals/osd_overview.rst
@@ -0,0 +1,106 @@
+===
+OSD
+===
+
+Concepts
+--------
+
+*Messenger*
+   See src/msg/Messenger.h
+
+   Handles sending and receipt of messages on behalf of the OSD.  The OSD uses
+   two messengers: 
+
+     1. cluster_messenger - handles traffic to other OSDs, monitors
+     2. client_messenger - handles client traffic
+
+	 This division allows the OSD to be configured with different interfaces for
+	 client and cluster traffic.
+
+*Dispatcher*
+   See src/msg/Dispatcher.h
+
+	 OSD implements the Dispatcher interface.  Of particular note is ms_dispatch,
+	 which serves as the entry point for messages received via either the client
+	 or cluster messenger.  Because there are two messengers, ms_dispatch may be
+	 called from at least two threads.  The osd_lock is always held during
+	 ms_dispatch.
+
+*WorkQueue*
+  See src/common/WorkQueue.h
+
+  The WorkQueue class abstracts the process of queueing independent tasks
+  for asynchronous execution.  Each OSD process contains workqueues for
+  distinct tasks:
+
+    1. OpWQ: handles ops (from clients) and subops (from other OSDs).
+       Runs in the op_tp threadpool.
+    2. PeeringWQ: handles peering tasks and pg map advancement
+       Runs in the op_tp threadpool.
+       See Peering
+    3. CommandWQ: handles commands (pg query, etc)
+       Runs in the command_tp threadpool.
+    4. RecoveryWQ: handles recovery tasks.
+       Runs in the recovery_tp threadpool.
+    5. SnapTrimWQ: handles snap trimming
+       Runs in the disk_tp threadpool.
+       See SnapTrimmer
+    6. ScrubWQ: handles primary scrub path
+       Runs in the disk_tp threadpool.
+       See Scrub
+    7. ScrubFinalizeWQ: handles primary scrub finalize
+       Runs in the disk_tp threadpool.
+       See Scrub
+    8. RepScrubWQ: handles replica scrub path
+       Runs in the disk_tp threadpool
+       See Scrub
+    9. RemoveWQ: Asynchronously removes old pg directories
+       Runs in the disk_tp threadpool
+       See PGRemoval
+
+*ThreadPool*
+  See src/common/WorkQueue.h
+  See also above.
+
+  There are 4 OSD threadpools:
+
+    1. op_tp: handles ops and subops
+    2. recovery_tp: handles recovery tasks
+    3. disk_tp: handles disk intensive tasks
+    4. command_tp: handles commands
+
+*OSDMap*
+  See src/osd/OSDMap.h
+
+  The crush algorithm takes two inputs: a picture of the cluster
+  with status information about which nodes are up/down and in/out, 
+  and the pgid to place.  The former is encapsulated by the OSDMap.
+  Maps are numbered by *epoch* (epoch_t).  These maps are passed around
+  within the OSD as std::tr1::shared_ptr<const OSDMap>.
+
+  See MapHandling
+
+*PG*
+  See src/osd/PG.* src/osd/PrimaryLogPG.*
+
+  Objects in rados are hashed into *PGs* and *PGs* are placed via crush onto
+  OSDs.  The PG structure is responsible for handling requests pertaining to
+  a particular *PG* as well as for maintaining relevant metadata and controlling
+  recovery.
+
+*OSDService*
+  See src/osd/OSD.cc OSDService
+
+  The OSDService acts as a broker between PG threads and OSD state which allows
+  PGs to perform actions using OSD services such as workqueues and messengers.
+  This is still a work in progress.  Future cleanups will focus on moving such
+  state entirely from the OSD into the OSDService.
+
+Overview
+--------
+  See src/ceph_osd.cc
+
+  The OSD process represents one leaf device in the crush hierarchy.  There
+  might be one OSD process per physical machine, or more than one if, for
+  example, the user configures one OSD instance per disk.
+  
diff --git a/doc/dev/osd_internals/osdmap_versions.txt b/doc/dev/osd_internals/osdmap_versions.txt
new file mode 100644
index 000000000..2bf247dcf
--- /dev/null
+++ b/doc/dev/osd_internals/osdmap_versions.txt
@@ -0,0 +1,259 @@
+releases:
+
+ <0.48  pre-argonaut, dev
+ 0.48   argonaut
+ 0.56   bobtail
+ 0.61   cuttlefish
+ 0.67   dumpling
+ 0.72   emperor
+ 0.80   firefly
+ 0.87   giant
+ 0.94   hammer
+ 9.1.0  infernalis rc
+ 9.2.0  infernalis
+ 10.2.0 jewel
+ 11.2.0 kraken
+ 12.2.0 luminous
+ 13.2.0 mimic
+ 14.2.0 nautilus (to-be)
+
+osdmap:
+
+type / v / cv / ev / commit / version / date
+
+map / 1 / - / - / 017788a6ecb570038632de31904dd2e1314dc7b7 / 0.11 / 2009
+inc / 1 / - / - /
+      * initial
+map / 2 / - / - / 020350e19a5dc03cd6cedd7494e434295580615f / 0.13 / 2009
+inc / 2 / - / - /
+      * pg_temp
+map / 3 / - / - / 1ebcebf6fff056a0c0bdf82dde69356e271be27e / 0.19 / 2009
+inc / 3 / - / - /
+      * heartbeat_addr
+map / 4 / - / - / 3ced5e7de243edeccfd20a90ec2034206c920795 / 0.19 / 2010
+inc / 4 / - / - /
+      * pools removed from map
+map / 5 / - / 5 / c4892bed6f49df396df3cbf9ed561c7315bd2442 / 0.20 / 2010
+inc / 5 / - / 5 /
+      * pool names moved to first part of encoding
+        * adds CEPH_OSDMAP_INC_VERSION_EXT (for extended part of map)
+        * adds CEPH_OSDMAP_VERSION_EXT (for extended part of map)
+        * adds 'ev' (extended version) during encode() and decode
+map / 5 / - / 5 / bc9cb9311f1b946898b5256eab500856fccf5c83 / 0.22 / 2010
+inc / 5 / - / 6 /
+      * separate up client/osd
+        * increments CEPH_OSDMAP_INC_VERSION_EXT to 6
+        * CEPH_OSDMAP_INC_VERSION stays at 5
+map / 5 / - / 6 / 7f70112052c7fc3ba46f9e475fa575d85e8b16b2 / 0.22 / 2010
+inc / 5 / - / 6 /
+      * add osd_cluster_addr to full map
+        * increments CEPH_OSDMAP_VERSION_EXT to 6
+        * CEPH_OSDMAP_VERSION stays at 5
+map / 5 / - / 7 / 2ced4e24aef64f2bc7d55b73abb888c124512eac / 0.28 / 2011
+inc / 5 / - / 7 /
+      * add cluster_snapshot field
+        * increments CEPH_OSDMAP_VERSION_EXT to 7
+        * increments CEPH_OSDMAP_INC_VERSION_EXT to 7
+        * CEPH_OSDMAP_INC_VERSION stays at 5
+        * CEPH_OSDMAP_VERSION stays at 5
+map / 6 / - / 7 / d1ce410842ca51fad3aa100a52815a39e5fe6af6 / 0.35 / 2011
+inc / 6 / - / 7 /
+      * encode/decode old + new versions
+        * adds encode_client_old() (basically transitioning from uint32 to
+          uint64)
+        * bumps osdmap version to 6, old clients stay at 5
+        * starts using in-function versions (i.e., _u16 v = 6)
+map / 6 / - / 7 / b297d1edecaf31a48cff6c37df2ee266e51cdec1 / 0.38 / 2011
+inc / 6 / - / 7 /
+      * make encoding conditional based on features
+        * essentially checks whether features & CEPH_FEATURE_PGID64 and opts
+          to either use encode_client_old() or encode()
+map / 6 / - / 7 / 0f0c59478894c9ca7fa04fc32e854648192a9fae / 0.38 / 2011
+inc / 6 / - / 7 /
+      * move stuff from osdmap.h to osdmap.cc
+map / 6 / 8 / ca4311e5e39cec8fad85fad3e67eea968707e9eb / 0.47 / 2012
+inc / 6 / 8 /
+      * store uuid per osd
+        * bumps osdmap::incremental extended version to 8; in function
+        * bumps osdmap's extended version to 8; in function
+map / 6 / - / 8 / 5125daa6d78e173a8dbc75723a8fdcd279a44bcd / 0.47 / 2012
+inc / 6 / - / 8 /
+      * drop defines
+        * drops defines for CEPH_OSDMAP_*_VERSION from rados.h
+map / 6 / 9 / e9f051ef3c49a080b24d7811a16aefb64beacbbd / 0.53 / 2012
+inc / 6 / 9 /
+      * add osd_xinfo_t
+        * osdmap::incremental ext version bumped to 9
+        * osdmap's ext version bumped to 9
+        * because we're adding osd_xinfo_t to the map
+map / 6 / - / 10 / 1fee4ccd5277b52292e255daf458330eef5f0255 / 0.64 / 2013
+inc / 6 / - / 10 /
+      * encode front hb addr
+        * osdmap::incremental ext version bumped to 10
+        * osdmap's ext version bumped to 10
+        * because we're adding osd_addrs->hb_front_addr to map
+
+// below we have the change to ENCODE_START() for osdmap and others
+// this means client-usable data and extended osd data get to have their
+// own ENCODE_START()'s, hence their versions start at 1 again.
+
+map / 7 / 1 / 1 / 3d7c69fb0986337dc72e466dc39d93e5ab406062 / 0.77 / 2014
+inc / 7 / 1 / 1 / b55c45e85dbd5d2513a4c56b3b74dcafd03f20b1 / 0.77 / 2014
+      * introduces ENCODE_START() approach to osdmap, and the 'features'
+        argument we currently see in ::encode() functions
+      * same, but for osdmap::incremental
+map / 7 / 1 / 1 / b9208b47745fdd53d36b682bebfc01e913347092 / 0.77 / 2014
+inc / 7 / 1 / 2 /
+      * include features argument in incremental.
+map / 7 / 2 / 1 / cee914290c5540eb1fb9d70faac70a581381c29b / 0.78 / 2014
+inc / 7 / 2 / 2 /
+      * add osd_primary_affinity
+map / 7 / 3 / 1 / c4f8f265955d54f33c79cde02c1ab2fe69ab1ab0 / 0.78 / 2014
+inc / 7 / 3 / 2 /
+      * add new/old erasure code profiles
+map / 8 / 3 / 1 / 3dcf5b9636bb9e0cd6484d18f151b457e1a0c328 / 0.91 / 2014
+inc / 8 / 3 / 2 /
+      * encode crc
+map / 8 / 3 / 1 / 04679c5451e353c966f6ed00b33fa97be8072a79 / 9.1.0 / 2015
+inc / 8 / 3 / 2 /
+      * simply ensures encode_features are filled to CEPH_FEATURE_PGID64 when
+        decoding an incremental if struct_v >= 6; else keeps it at zero.
+      * otherwise, if we get an incremental from hammer (which has
+        struct_v = 6) we would be decoding it as if it were a map from before
+        CEPH_FEATURES_PGID64 (which was introduced in 0.35, pre-argonaut)
+map / 8 / 3 / 2 / 5c6b9d9dcd0a225e3a2b154c20a623868c269346 / 12.0.1 / 2017
+inc / 8 / 3 / 3 /
+      * add (near)full_ratio
+        * used to live in pgmap, moving to osdmap for luminous
+      * conditional on SERVER_LUMINOUS feature being present
+        * osdmap::incremental::encode(): conditional on ev >= 3
+        * osdmap::incremental::decode(): conditional on ev >= 3, else -1
+        * osdmap::encode(): conditional on ev >= 2
+        * osdmap::decode(): conditional on ev >= 0, else 0
+map / 8 / 4 / 2 / 27d6f4373bafa24450f6dbb4e4252c2d9c2c1448 / 12.0.2 / 2017
+inc / 8 / 4 / 3 /
+      * add pg_remap and pg_remap_items
+        * first forces a pg to map to a particular value; second replaces
+          specific osds with specific other osds in crush mapping.
+      * inc conditional on SERVER_LUMINOUS feature being present
+        * osdmap::incremental::encode(): conditional on cv >= 4
+        * osdmap::incremental::decode(): conditional on cv >= 4
+      * map conditional on OSDMAP_REMAP feature being present
+        * osdmap::encode(): if not feature, cv = 3; encode on cv >= 4
+        * osdmap::decode(): conditional on cv >= 4
+map / 8 / 4 / 3 / 27d6f4373bafa24450f6dbb4e4252c2d9c2c1448 / 12.0.2 / 2017
+inc / 8 / 4 / 4 /
+      * handle backfillfull_ratio like nearfull and full
+      * inc:
+        * osdmap::incremental::encode(): conditional on ev >= 3
+        * osdmap::incremental::decode(): conditional on ev >= 4, else -1
+      * map:
+        * osdmap::encode(): conditional on ev >= 2
+        * osdmap::decode(): conditional on ev >= 3, else 0
+map / 8 / 4 / 3 / a1c66468232002c9f36033226f5db0a5751e8d18 / 12.0.3 / 2017
+inc / 8 / 4 / 4 /
+      * add require_min_compat_client field
+      * inc:
+        * osdmap::incremental::encode() conditional on ev >= 4
+        * osdmap::incremental::decode() conditional on ev >= 4
+      map:
+        * osdmap::encode() conditional on ev >= 3
+        * osdmap::decode() conditional on ev >= 3
+map / 8 / 4 / 4 / 4a09e9431de3084b1ca98af11b28f822fde4ffbe / 12.0.3 / 2017
+inc / 8 / 4 / 5 /
+      * bumps encoding version for require_min_compat_client
+          * otherwise osdmap::decode() would throw exception when decoding
+            old maps
+      * inc:
+        * osdmap::incremental::encode() no conditional on ev >= 3
+        * osdmap::incremental::decode() conditional on ev >= 5
+      * map:
+      * osdmap::encode() conditional on ev >= 2
+      * osdmap::decode() conditional on ev >= 4
+map / 8 / 4 / 5 / 3d4c4d9d9da07e1456331c43acc998d2008ca8ea / 12.1.0 / 2017
+inc / 8 / 4 / 6 /
+      * add require_osd_release numeric field
+      * new_require_min_compat_client:
+        * osdmap::incremental::encode() conditional on ev >= 5
+        * osdmap::encode() conditional on ev >= 4
+      * require_osd_release:
+        * osdmap::incremental::encode() conditional on ev >= 6
+        * osdmap::incremental::decode() conditional on ev >= 6 (else, -1)
+        * osdmap::encode() conditional on ev >= 5
+        * osdmap::decode() conditional on ev >= 5 (else, -1)
+map / 8 / 4 / 5 / f22997e24bda4e6476e15d5d4ad9737861f9741f / 12.1.0 / 2017
+inc / 8 / 4 / 6 /
+        * switch (require_)min_compat_client to integers instead of strings
+        * osdmap::incremental::encode() conditional on ev >= 6
+        * osdmap::incremental::decode():
+          * if ev == 5, decode string and translate to release number
+          * if ev >= 6, decode integer
+        * osdmap::encode() conditional on ev >= 4
+        * osdmap::decode():
+          * if ev == 4, decode string and translate to release number
+          * if ev >= 5, decode integer
+map / 8 / 4 / 6 / a8fb39b57884d96201fa502b17bc9395ec38c1b3 / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * make incremental's `new_state` 32 bits instead of 8 bits
+        * implies forcing 8 bits on
+          * osdmap::incremental::encode_client_old()
+          * osdmap::incremental::encode_classic()
+          * osdmap::incremental::decode_classic()
+        * osdmap::incremental::encode() conditional on cv >= 5, else force 8b.
+        * osdmap::incremental::decode() conditional on cv >= 5, else force 8b.
+map / 8 / 5 / 6 / 3c1e58215bbb98f71aae30904f9010a57a58da81 / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * same as above
+map / 8 / 6 / 6 / 48158ec579b708772fae82daaa6cb5dcaf5ac5dd / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * add crush_version
+        * osdmap::encode() conditional on cv >= 6
+        * osdmap::decode() conditional on cv >= 6
+map / 8 / 7 / 6 / 553048fbf97af999783deb7e992c8ecfa5e55500 / 13.0.2 / 2017
+inc / 8 / 6 / 6 /
+        * track newly removed and purged snaps in each epoch
+          * new_removed_snaps
+          * new_purged_snaps
+        * osdmap::encode() conditional on cv >= 7
+          * if SERVER_MIMIC not in features, cv = 6
+        * osdmap::decode() conditional cv >= 7
+map / 8 / 8 / 6 / f99c2a9fec65ad3ce275ef24bd167ee03275d3d7 / 14.0.1 / 2018
+inc / 8 / 7 / 6 /
+        * fix pre-addrvec compat
+        * osdmap::encode() conditional on cv >= 8, else encode client addrs
+          one by one in a loop.
+        * osdmap::decode() just bumps version (?)
+map / 8 / 8 / 7 / 9fb1e521c7c75c124b0dbf193e8b65ff1b5f461e / 14.0.1 / 2018
+inc / 8 / 7 / 7 /
+        * make cluster addrs into addrvecs too
+          * this will allow single-step upgrade from msgr1 to msgr2
+map / 8 / 9 / 7 / d414f0b43a69f3c2db8e454d795be881496237c6 / 14.0.1 / 2018
+inc / 8 / 8 / 7 /
+        * store last_up_change and last_in_change
+        * osdmap::encode() conditional on cv >= 9
+        * osdmap::decode() conditional on cv >= 9
+
+
+
+osd_info_t:
+v / commit / version / date / reason
+
+1 / e574c84a6a0c5a5070dc72d5f5d3d17914ef824a / 0.19 / 2010 / add struct_v
+
+osd_xinfo_t:
+v / commit / version / date
+
+1 / e9f051ef3c49a080b24d7811a16aefb64beacbbd / 0.53 / 2012
+      * add osd_xinfo_t
+2 / 31743d50a109a463d664ec9cf764d5405db507bd / 0.75 / 2013
+      * add features bit mask to osd_xinfo_t
+3 / 87722a42c286d4d12190b86b6d06d388e2953ba0 / 0.82 / 2014
+      * remember osd weight when auto-marking osds out
+
+rados.h:
+v / commit / version / date / reason
+
+- / 147c6f51e34a875ab65624df04baa8ef89296ddd / 0.19 / 2010 / move versions
+  3 / CEPH_OSDMAP_INC_VERSION
+  3 / CEPH_OSDMAP_VERSION
+  2 / CEPH_PG_POOL_VERSION
diff --git a/doc/dev/osd_internals/partial_object_recovery.rst b/doc/dev/osd_internals/partial_object_recovery.rst
new file mode 100644
index 000000000..a22f63348
--- /dev/null
+++ b/doc/dev/osd_internals/partial_object_recovery.rst
@@ -0,0 +1,148 @@
+=======================
+Partial Object Recovery
+=======================
+
+Partial Object Recovery improves the efficiency of log-based recovery (vs
+backfill). Original log-based recovery calculates missing_set based on pg_log
+differences.
+
+The whole object should be recovery from one OSD to another
+if the object is indicated modified by pg_log regardless of how much
+content in the object is really modified. That means a 4M object,
+which is just modified 4k inside, should recovery the whole 4M object
+rather than the modified 4k content. In addition, object map should be
+also recovered even if it is not modified at all.
+
+Partial Object Recovery is designed to solve the problem mentioned above.
+In order to achieve the goals, two things should be done: 
+
+1. logging where the object is modified is necessary
+2. logging whether the object_map of an object is modified is also necessary
+
+class ObjectCleanRegion is introduced to do what we want.
+clean_offsets is a variable of interval_set<uint64_t>
+and is used to indicate the unmodified content in an object.
+clean_omap is a variable of bool indicating whether object_map is modified.
+new_object means that osd does not exist for an object
+max_num_intervals is an upbound of the number of intervals in clean_offsets
+so that the memory cost of clean_offsets is always bounded.
+
+The shortest clean interval will be trimmed if the number of intervals
+in clean_offsets exceeds the boundary.
+
+    etc. max_num_intervals=2, clean_offsets:{[5~10], [20~5]}
+
+    then new interval [30~10] will evict out the shortest one [20~5]
+
+    finally, clean_offsets becomes {[5~10], [30~10]}
+
+Procedures for Partial Object Recovery
+======================================
+
+Firstly, OpContext and pg_log_entry_t should contain ObjectCleanRegion.
+In do_osd_ops(), finish_copyfrom(), finish_promote(), corresponding content
+in ObjectCleanRegion should mark dirty so that trace the modification of an object.
+Also update ObjectCleanRegion in OpContext to its pg_log_entry_t.
+
+Secondly, pg_missing_set can build and rebuild correctly.
+when calculating pg_missing_set during peering process,
+also merge ObjectCleanRegion in each pg_log_entry_t.
+
+    etc. object aa has pg_log:
+        26'101 {[0~4096, 8192~MAX], false}
+
+        26'104 {0~8192, 12288~MAX, false}
+
+        28'108 {[0~12288, 16384~MAX], true}
+
+    missing_set for object aa: merge pg_log above --> {[0~4096, 16384~MAX], true}.
+    which means 4096~16384 is modified and object_map is also modified on version 28'108
+
+Also, OSD may be crash after merge log.
+Therefore, we need to read_log and rebuild pg_missing_set. For example, pg_log is:
+
+    object aa: 26'101 {[0~4096, 8192~MAX], false}
+
+    object bb: 26'102 {[0~4096, 8192~MAX], false}
+
+    object cc: 26'103 {[0~4096, 8192~MAX], false}
+
+    object aa: 26'104 {0~8192, 12288~MAX, false}
+
+    object dd: 26'105 {[0~4096, 8192~MAX], false}
+
+    object aa: 28'108 {[0~12288, 16384~MAX], true}
+
+Originally, if bb,cc,dd is recovered, and aa is not.
+So we need to rebuild pg_missing_set for object aa,
+and find aa is modified on version 28'108.
+If version in object_info is 26'96 < 28'108,
+we don't need to consider 26'104 and 26'101 because the whole object will be recovered.
+However, Partial Object Recovery should also require us to rebuild ObjectCleanRegion.
+
+Knowing whether the object is modified is not enough.
+
+Therefore, we also need to traverse the pg_log before,
+that says 26'104 and 26'101 also > object_info(26'96)
+and rebuild pg_missing_set for object aa based on those three logs: 28'108, 26'104, 26'101.
+The way how to merge logs is the same as mentioned above
+
+Finally, finish the push and pull process based on pg_missing_set.
+Updating copy_subset in recovery_info based on ObjectCleanRegion in pg_missing_set.
+copy_subset indicates the intervals of content need to pull and push.
+
+The complicated part here is submit_push_data
+and serval cases should be considered separately.
+what we need to consider is how to deal with the object data,
+object data makes up of omap_header, xattrs, omap, data:
+    
+case 1: first && complete: since object recovering is finished in a single PushOp,
+we would like to preserve the original object and overwrite on the object directly.
+Object will not be removed and touch a new one.
+
+    issue 1: As object is not removed, old xattrs remain in the old object
+    but maybe updated in new object. Overwriting for the same key or adding new keys is correct,
+    but removing keys will be wrong. 
+    In order to solve this issue, We need to remove the all original xattrs in the object, and then update new xattrs.
+
+    issue 2: As object is not removed,
+    object_map may be recovered depending on the clean_omap.
+    Therefore, if recovering clean_omap, we need to remove old omap of the object for the same reason
+    since omap updating may also be a deletion.
+    Thus, in this case, we should do:
+
+        1) clear xattrs of the object
+        2) clear omap of the object if omap recovery is needed
+        3) truncate the object into recovery_info.size
+        4) recovery omap_header
+        5) recovery xattrs, and recover omap if needed
+        6) punch zeros for original object if fiemap tells nothing there
+        7) overwrite object content which is modified
+        8) finish recovery
+
+case 2:  first && !complete: object recovering should be done in multiple times.
+Here, target_oid will indicate a new temp_object in pgid_TEMP,
+so the issues are a bit difference.
+
+    issue 1: As object is newly created, there is no need to deal with xattrs
+
+    issue 2: As object is newly created,
+    and object_map may not be transmitted depending on clean_omap.
+    Therefore, if clean_omap is true, we need to clone object_map from original object.
+    issue 3: As object is newly created, and unmodified data will not be transmitted.
+    Therefore, we need to clone unmodified data from the original object.
+    Thus, in this case, we should do:
+
+        1) remove the temp object
+        2) create a new temp object
+        3) set alloc_hint for the new temp object
+        4) truncate new temp object to recovery_info.size
+        5) recovery omap_header
+        6) clone object_map from original object if omap is clean
+        7) clone unmodified object_data from original object
+        8) punch zeros for the new temp object
+        9) recovery xattrs, and recover omap if needed
+        10) overwrite object content which is modified
+        11) remove the original object
+        12) move and rename the new temp object to replace the original object
+        13) finish recovery
diff --git a/doc/dev/osd_internals/past_intervals.rst b/doc/dev/osd_internals/past_intervals.rst
new file mode 100644
index 000000000..5b594df1a
--- /dev/null
+++ b/doc/dev/osd_internals/past_intervals.rst
@@ -0,0 +1,93 @@
+=============
+PastIntervals
+=============
+
+Purpose
+-------
+
+There are two situations where we need to consider the set of all acting-set
+OSDs for a PG back to some epoch ``e``:
+
+ * During peering, we need to consider the acting set for every epoch back to
+   ``last_epoch_started``, the last epoch in which the PG completed peering and
+   became active.
+   (see :doc:`/dev/osd_internals/last_epoch_started` for a detailed explanation)
+ * During recovery, we need to consider the acting set for every epoch back to
+   ``last_epoch_clean``, the last epoch at which all of the OSDs in the acting
+   set were fully recovered, and the acting set was full.
+
+For either of these purposes, we could build such a set by iterating backwards
+from the current OSDMap to the relevant epoch.  Instead, we maintain a structure
+PastIntervals for each PG.
+
+An ``interval`` is a contiguous sequence of OSDMap epochs where the PG mapping
+didn't change.  This includes changes to the acting set, the up set, the
+primary, and several other parameters fully spelled out in
+PastIntervals::check_new_interval.
+
+Maintenance and Trimming
+------------------------
+
+The PastIntervals structure stores a record for each ``interval`` back to
+last_epoch_clean.  On each new ``interval`` (See AdvMap reactions,
+PeeringState::should_restart_peering, and PeeringState::start_peering_interval)
+each OSD with the PG will add the new ``interval`` to its local PastIntervals.
+Activation messages to OSDs which do not already have the PG contain the
+sender's PastIntervals so that the recipient needn't rebuild it.  (See
+PeeringState::activate needs_past_intervals).
+
+PastIntervals are trimmed in two places.  First, when the primary marks the
+PG clean, it clears its past_intervals instance
+(PeeringState::try_mark_clean()).  The replicas will do the same thing when
+they receive the info (See PeeringState::update_history).
+
+The second, more complex, case is in PeeringState::start_peering_interval.  In
+the event of a "map gap", we assume that the PG actually has gone clean, but we
+haven't received a pg_info_t with the updated ``last_epoch_clean`` value yet.
+To explain this behavior, we need to discuss OSDMap trimming.
+
+OSDMap Trimming
+---------------
+
+OSDMaps are created by the Monitor quorum and gossiped out to the OSDs.  The
+Monitor cluster also determines when OSDs (and the Monitors) are allowed to
+trim old OSDMap epochs.  For the reasons explained above in this document, the
+primary constraint is that we must retain all OSDMaps back to some epoch such
+that all PGs have been clean at that or a later epoch (min_last_epoch_clean).
+(See OSDMonitor::get_trim_to).
+
+The Monitor quorum determines min_last_epoch_clean through MOSDBeacon messages
+sent periodically by each OSDs.  Each message contains a set of PGs for which
+the OSD is primary at that moment as well as the min_last_epoch_clean across
+that set.  The Monitors track these values in OSDMonitor::last_epoch_clean.
+
+There is a subtlety in the min_last_epoch_clean value used by the OSD to
+populate the MOSDBeacon.  OSD::collect_pg_stats invokes PG::with_pg_stats to
+obtain the lec value, which actually uses
+pg_stat_t::get_effective_last_epoch_clean() rather than
+info.history.last_epoch_clean.  If the PG is currently clean,
+pg_stat_t::get_effective_last_epoch_clean() is the current epoch rather than
+last_epoch_clean -- this works because the PG is clean at that epoch and it
+allows OSDMaps to be trimmed during periods where OSDMaps are being created
+(due to snapshot activity, perhaps), but no PGs are undergoing ``interval``
+changes.
+
+Back to PastIntervals
+---------------------
+
+We can now understand our second trimming case above.  If OSDMaps have been
+trimmed up to epoch ``e``, we know that the PG must have been clean at some epoch
+>= ``e`` (indeed, **all** PGs must have been), so we can drop our PastIntevals.
+
+This dependency also pops up in PeeringState::check_past_interval_bounds().
+PeeringState::get_required_past_interval_bounds takes as a parameter
+oldest_epoch, which comes from OSDSuperblock::cluster_osdmap_trim_lower_bound.
+We use cluster_osdmap_trim_lower_bound rather than a specific osd's oldest_map
+because we don't necessarily trim all MOSDMap::cluster_osdmap_trim_lower_bound.
+In order to avoid doing too much work at once we limit the amount of osdmaps
+trimmed using ``osd_target_transaction_size`` in OSD::trim_maps().
+For this reason, a specific OSD's oldest_map can lag behind
+OSDSuperblock::cluster_osdmap_trim_lower_bound
+for a while.
+
+See https://tracker.ceph.com/issues/49689 for an example.
diff --git a/doc/dev/osd_internals/pg.rst b/doc/dev/osd_internals/pg.rst
new file mode 100644
index 000000000..397d4ab5d
--- /dev/null
+++ b/doc/dev/osd_internals/pg.rst
@@ -0,0 +1,31 @@
+====
+PG
+====
+
+Concepts
+--------
+
+*Peering Interval*
+  See PG::start_peering_interval.
+  See PG::acting_up_affected
+  See PG::PeeringState::Reset
+
+  A peering interval is a maximal set of contiguous map epochs in which the
+  up and acting sets did not change.  PG::PeeringMachine represents a
+  transition from one interval to another as passing through
+  PeeringState::Reset.  On PG::PeeringState::AdvMap PG::acting_up_affected can
+  cause the pg to transition to Reset.
+  
+
+Peering Details and Gotchas
+---------------------------
+For an overview of peering, see `Peering <../../peering>`_.
+
+  * PG::flushed defaults to false and is set to false in
+    PG::start_peering_interval.  Upon transitioning to PG::PeeringState::Started
+    we send a transaction through the pg op sequencer which, upon complete,
+    sends a FlushedEvt which sets flushed to true.  The primary cannot go
+    active until this happens (See PG::PeeringState::WaitFlushedPeering).
+    Replicas can go active but cannot serve ops (writes or reads).
+    This is necessary because we cannot read our ondisk state until unstable
+    transactions from the previous interval have cleared.
diff --git a/doc/dev/osd_internals/pg_removal.rst b/doc/dev/osd_internals/pg_removal.rst
new file mode 100644
index 000000000..c5fe0e1ab
--- /dev/null
+++ b/doc/dev/osd_internals/pg_removal.rst
@@ -0,0 +1,56 @@
+==========
+PG Removal
+==========
+
+See OSD::_remove_pg, OSD::RemoveWQ
+
+There are two ways for a pg to be removed from an OSD:
+
+  1. MOSDPGRemove from the primary
+  2. OSD::advance_map finds that the pool has been removed
+
+In either case, our general strategy for removing the pg is to
+atomically set the metadata objects (pg->log_oid, pg->biginfo_oid) to
+backfill and asynchronously remove the pg collections.  We do not do
+this inline because scanning the collections to remove the objects is
+an expensive operation.
+
+OSDService::deleting_pgs tracks all pgs in the process of being
+deleted.  Each DeletingState object in deleting_pgs lives while at
+least one reference to it remains.  Each item in RemoveWQ carries a
+reference to the DeletingState for the relevant pg such that
+deleting_pgs.lookup(pgid) will return a null ref only if there are no
+collections currently being deleted for that pg. 
+
+The DeletingState for a pg also carries information about the status
+of the current deletion and allows the deletion to be cancelled.
+The possible states are:
+
+  1. QUEUED: the PG is in the RemoveWQ
+  2. CLEARING_DIR: the PG's contents are being removed synchronously
+  3. DELETING_DIR: the PG's directories and metadata being queued for removal
+  4. DELETED_DIR: the final removal transaction has been queued
+  5. CANCELED: the deletion has been cancelled
+
+In 1 and 2, the deletion can be cancelled.  Each state transition
+method (and check_canceled) returns false if deletion has been
+cancelled and true if the state transition was successful.  Similarly,
+try_stop_deletion() returns true if it succeeds in cancelling the
+deletion.  Additionally, try_stop_deletion() in the event that it
+fails to stop the deletion will not return until the final removal
+transaction is queued.  This ensures that any operations queued after
+that point will be ordered after the pg deletion.
+
+OSD::_create_lock_pg must handle two cases:
+
+  1. Either there is no DeletingStateRef for the pg, or it failed to cancel
+  2. We succeeded in cancelling the deletion.
+
+In case 1., we proceed as if there were no deletion occurring, except that
+we avoid writing to the PG until the deletion finishes.  In case 2., we
+proceed as in case 1., except that we first mark the PG as backfilling.
+
+Similarly, OSD::osr_registry ensures that the OpSequencers for those
+pgs can be reused for a new pg if created before the old one is fully
+removed, ensuring that operations on the new pg are sequenced properly
+with respect to operations on the old one.
diff --git a/doc/dev/osd_internals/pgpool.rst b/doc/dev/osd_internals/pgpool.rst
new file mode 100644
index 000000000..45a252bd4
--- /dev/null
+++ b/doc/dev/osd_internals/pgpool.rst
@@ -0,0 +1,22 @@
+==================
+PGPool
+==================
+
+PGPool is a structure used to manage and update the status of removed
+snapshots.  It does this by maintaining two fields, cached_removed_snaps - the
+current removed snap set and newly_removed_snaps - newly removed snaps in the
+last epoch. In OSD::load_pgs the osd map is recovered from the pg's file store
+and passed down to OSD::_get_pool where a PGPool object is initialised with the
+map.
+
+With each new map we receive we call PGPool::update with the new map. In that
+function we build a list of newly removed snaps
+(pg_pool_t::build_removed_snaps) and merge that with our cached_removed_snaps.
+This function included checks to make sure we only do this update when things
+have changed or there has been a map gap.
+
+When we activate the pg we initialise the snap trim queue from
+cached_removed_snaps and subtract the purged_snaps we have already purged
+leaving us with the list of snaps that need to be trimmed. Trimming is later
+performed asynchronously by the snap_trim_wq.
+
diff --git a/doc/dev/osd_internals/recovery_reservation.rst b/doc/dev/osd_internals/recovery_reservation.rst
new file mode 100644
index 000000000..a24ac1b15
--- /dev/null
+++ b/doc/dev/osd_internals/recovery_reservation.rst
@@ -0,0 +1,83 @@
+====================
+Recovery Reservation
+====================
+
+Recovery reservation extends and subsumes backfill reservation. The
+reservation system from backfill recovery is used for local and remote
+reservations.
+
+When a PG goes active, first it determines what type of recovery is
+necessary, if any. It may need log-based recovery, backfill recovery,
+both, or neither.
+
+In log-based recovery, the primary first acquires a local reservation
+from the OSDService's local_reserver. Then a MRemoteReservationRequest
+message is sent to each replica in order of OSD number. These requests
+will always be granted (i.e., cannot be rejected), but they may take
+some time to be granted if the remotes have already granted all their
+remote reservation slots.
+
+After all reservations are acquired, log-based recovery proceeds as it
+would without the reservation system.
+
+After log-based recovery completes, the primary releases all remote
+reservations. The local reservation remains held. The primary then
+determines whether backfill is necessary. If it is not necessary, the
+primary releases its local reservation and waits in the Recovered state
+for all OSDs to indicate that they are clean.
+
+If backfill recovery occurs after log-based recovery, the local
+reservation does not need to be reacquired since it is still held from
+before. If it occurs immediately after activation (log-based recovery
+not possible/necessary), the local reservation is acquired according to
+the typical process.
+
+Once the primary has its local reservation, it requests a remote
+reservation from the backfill target. This reservation CAN be rejected,
+for instance if the OSD is too full (backfillfull_ratio osd setting).
+If the reservation is rejected, the primary drops its local
+reservation, waits (osd_backfill_retry_interval), and then retries. It
+will retry indefinitely.
+
+Once the primary has the local and remote reservations, backfill
+proceeds as usual. After backfill completes the remote reservation is
+dropped.
+
+Finally, after backfill (or log-based recovery if backfill was not
+necessary), the primary drops the local reservation and enters the
+Recovered state. Once all the PGs have reported they are clean, the
+primary enters the Clean state and marks itself active+clean.
+
+-----------------
+Dump Reservations
+-----------------
+
+An OSD daemon command dumps total local and remote reservations::
+
+  ceph daemon osd.<id> dump_recovery_reservations
+
+
+--------------
+Things to Note
+--------------
+
+We always grab the local reservation first, to prevent a circular
+dependency. We grab remote reservations in order of OSD number for the
+same reason.
+
+The recovery reservation state chart controls the PG state as reported
+to the monitor. The state chart can set:
+
+ - recovery_wait: waiting for local/remote reservations
+ - recovering: recovering
+ - recovery_toofull: recovery stopped, OSD(s) above full ratio
+ - backfill_wait: waiting for remote backfill reservations
+ - backfilling: backfilling
+ - backfill_toofull: backfill stopped, OSD(s) above backfillfull ratio
+
+
+--------
+See Also
+--------
+
+The Active substate of the automatically generated OSD state diagram.
diff --git a/doc/dev/osd_internals/refcount.rst b/doc/dev/osd_internals/refcount.rst
new file mode 100644
index 000000000..3324b63e5
--- /dev/null
+++ b/doc/dev/osd_internals/refcount.rst
@@ -0,0 +1,45 @@
+========
+Refcount
+========
+
+
+Introduction
+============
+
+Deduplication, as described in ../deduplication.rst, needs a way to
+maintain a target pool of deduplicated chunks with atomic ref
+refcounting.  To that end, there exists an osd object class 
+refcount responsible for using the object class machinery to
+maintain refcounts on deduped chunks and ultimately remove them
+as the refcount hits 0.
+
+Class Interface
+===============
+
+See cls/refcount/cls_refcount_client*
+
+* cls_refcount_get
+
+  Atomically increments the refcount with specified tag ::
+
+    void cls_refcount_get(librados::ObjectWriteOperation& op, const string& tag, bool implicit_ref = false);
+
+* cls_refcount_put
+
+  Atomically decrements the refcount specified by passed tag ::
+
+    void cls_refcount_put(librados::ObjectWriteOperation& op, const string& tag, bool implicit_ref = false);
+
+* cls_refcount_Set
+
+  Atomically sets the set of refcounts with passed list of tags ::
+
+    void cls_refcount_set(librados::ObjectWriteOperation& op, list<string>& refs);
+
+* cls_refcount_read
+
+  Dumps the current set of ref tags for the object ::
+
+    int cls_refcount_read(librados::IoCtx& io_ctx, string& oid, list<string> *refs, bool implicit_ref = false);
+
+  
diff --git a/doc/dev/osd_internals/scrub.rst b/doc/dev/osd_internals/scrub.rst
new file mode 100644
index 000000000..149509799
--- /dev/null
+++ b/doc/dev/osd_internals/scrub.rst
@@ -0,0 +1,41 @@
+
+Scrub internals and diagnostics
+===============================
+
+Scrubbing Behavior Table
+------------------------
+
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+|                                          Flags  | none     | noscrub   | nodeep_scrub  | noscrub/nodeep_scrub |
++=================================================+==========+===========+===============+======================+
+| Periodic tick                                   |   S      |    X      |     S         |         X            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Periodic tick after osd_deep_scrub_interval     |   D      |    D      |     S         |         X            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated scrub                                 |   S      |    S      |     S         |         S            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated scrub after osd_deep_scrub_interval   |   D      |    D      |     S         |         S            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated deep scrub                            |   D      |    D      |     D         |         D            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+
+- X = Do nothing
+- S = Do regular scrub
+- D = Do deep scrub
+
+State variables
+---------------
+
+- Periodic tick state is ``!must_scrub && !must_deep_scrub && !time_for_deep``
+- Periodic tick after ``osd_deep_scrub_interval state is !must_scrub && !must_deep_scrub && time_for_deep``
+- Initiated scrub state is ``must_scrub && !must_deep_scrub && !time_for_deep``
+- Initiated scrub after ``osd_deep_scrub_interval`` state is ``must_scrub && !must_deep_scrub && time_for_deep``
+- Initiated deep scrub state is ``must_scrub && must_deep_scrub``
+
+Scrub Reservations
+------------------
+
+An OSD daemon command dumps total local and remote reservations::
+
+  ceph daemon osd.<id> dump_scrub_reservations
+
diff --git a/doc/dev/osd_internals/snaps.rst b/doc/dev/osd_internals/snaps.rst
new file mode 100644
index 000000000..5ebd0884a
--- /dev/null
+++ b/doc/dev/osd_internals/snaps.rst
@@ -0,0 +1,128 @@
+======
+Snaps
+======
+
+Overview
+--------
+Rados supports two related snapshotting mechanisms:
+
+  1. *pool snaps*: snapshots are implicitly applied to all objects
+     in a pool
+  2. *self managed snaps*: the user must provide the current *SnapContext*
+     on each write.
+
+These two are mutually exclusive, only one or the other can be used on
+a particular pool.
+
+The *SnapContext* is the set of snapshots currently defined for an object
+as well as the most recent snapshot (the *seq*) requested from the mon for
+sequencing purposes (a *SnapContext* with a newer *seq* is considered to
+be more recent).
+
+The difference between *pool snaps* and *self managed snaps* from the
+OSD's point of view lies in whether the *SnapContext* comes to the OSD
+via the client's MOSDOp or via the most recent OSDMap.
+
+See OSD::make_writeable
+
+Ondisk Structures
+-----------------
+Each object has in the PG collection a *head* object (or *snapdir*, which we
+will come to shortly) and possibly a set of *clone* objects.
+Each hobject_t has a snap field.  For the *head* (the only writeable version
+of an object), the snap field is set to CEPH_NOSNAP.  For the *clones*, the
+snap field is set to the *seq* of the *SnapContext* at their creation.
+When the OSD services a write, it first checks whether the most recent
+*clone* is tagged with a snapid prior to the most recent snap represented
+in the *SnapContext*.  If so, at least one snapshot has occurred between
+the time of the write and the time of the last clone.  Therefore, prior
+to performing the mutation, the OSD creates a new clone for servicing
+reads on snaps between the snapid of the last clone and the most recent
+snapid.
+
+The *head* object contains a *SnapSet* encoded in an attribute, which tracks
+
+  1. The full set of snaps defined for the object
+  2. The full set of clones which currently exist
+  3. Overlapping intervals between clones for tracking space usage
+  4. Clone size
+
+If the *head* is deleted while there are still clones, a *snapdir* object
+is created instead to house the *SnapSet*.
+
+Additionally, the *object_info_t* on each clone includes a vector of snaps
+for which clone is defined.
+
+Snap Removal
+------------
+To remove a snapshot, a request is made to the *Monitor* cluster to
+add the snapshot id to the list of purged snaps (or to remove it from
+the set of pool snaps in the case of *pool snaps*).  In either case,
+the *PG* adds the snap to its *snap_trimq* for trimming.
+
+A clone can be removed when all of its snaps have been removed.  In
+order to determine which clones might need to be removed upon snap
+removal, we maintain a mapping from snap to *hobject_t* using the
+*SnapMapper*.
+
+See PrimaryLogPG::SnapTrimmer, SnapMapper
+
+This trimming is performed asynchronously by the snap_trim_wq while the
+PG is clean and not scrubbing.
+
+  #. The next snap in PG::snap_trimq is selected for trimming
+  #. We determine the next object for trimming out of PG::snap_mapper.
+     For each object, we create a log entry and repop updating the
+     object info and the snap set (including adjusting the overlaps).
+     If the object is a clone which no longer belongs to any live snapshots,
+     it is removed here. (See PrimaryLogPG::trim_object() when new_snaps
+     is empty.)
+  #. We also locally update our *SnapMapper* instance with the object's
+     new snaps.
+  #. The log entry containing the modification of the object also
+     contains the new set of snaps, which the replica uses to update
+     its own *SnapMapper* instance.
+  #. The primary shares the info with the replica, which persists
+     the new set of purged_snaps along with the rest of the info.
+
+
+
+Recovery
+--------
+Because the trim operations are implemented using repops and log entries,
+normal PG peering and recovery maintain the snap trimmer operations with
+the caveat that push and removal operations need to update the local
+*SnapMapper* instance.  If the purged_snaps update is lost, we merely
+retrim a now empty snap.
+
+SnapMapper
+----------
+*SnapMapper* is implemented on top of map_cacher<string, bufferlist>,
+which provides an interface over a backing store such as the file system
+with async transactions.  While transactions are incomplete, the map_cacher
+instance buffers unstable keys allowing consistent access without having
+to flush the filestore.  *SnapMapper* provides two mappings:
+
+  1. hobject_t -> set<snapid_t>: stores the set of snaps for each clone
+     object
+  2. snapid_t -> hobject_t: stores the set of hobjects with the snapshot
+     as one of its snaps
+
+Assumption: there are lots of hobjects and relatively few snaps.  The
+first encoding has a stringification of the object as the key and an
+encoding of the set of snaps as a value.  The second mapping, because there
+might be many hobjects for a single snap, is stored as a collection of keys
+of the form stringify(snap)_stringify(object) such that stringify(snap)
+is constant length.  These keys have a bufferlist encoding
+pair<snapid, hobject_t> as a value.  Thus, creating or trimming a single
+object does not involve reading all objects for any snap.  Additionally,
+upon construction, the *SnapMapper* is provided with a mask for filtering
+the objects in the single SnapMapper keyspace belonging to that PG.
+
+Split
+-----
+The snapid_t -> hobject_t key entries are arranged such that for any PG,
+up to 8 prefixes need to be checked to determine all hobjects in a particular
+snap for a particular PG.  Upon split, the prefixes to check on the parent
+are adjusted such that only the objects remaining in the PG will be visible.
+The children will immediately have the correct mapping.
diff --git a/doc/dev/osd_internals/stale_read.rst b/doc/dev/osd_internals/stale_read.rst
new file mode 100644
index 000000000..5493bb1f4
--- /dev/null
+++ b/doc/dev/osd_internals/stale_read.rst
@@ -0,0 +1,102 @@
+Preventing Stale Reads
+======================
+
+We write synchronously to all replicas before sending an ACK to the
+client, which limits the potential for inconsistency
+in the write path.  However, by default we serve reads from just
+one replica (the lead/primary OSD for each PG), and the
+client will use whatever OSDMap is has to select the OSD from which to read.
+In most cases, this is fine: either the client map is correct,
+or the OSD that we think is the primary for the object knows that it
+is not the primary anymore, and can feed the client an updated map
+that indicates a newer primary.
+
+They key is to ensure that this is *always* true.  In particular, we
+need to ensure that an OSD that is fenced off from its peers and has
+not learned about a map update does not continue to service read
+requests from similarly stale clients at any point after which a new
+primary may have been allowed to make a write.
+
+We accomplish this via a mechanism that works much like a read lease.
+Each pool may have a ``read_lease_interval`` property which defines
+how long this is, although by default we simply set it to
+``osd_pool_default_read_lease_ratio`` (default: .8) times the
+``osd_heartbeat_grace``.  (This way the lease will generally have
+expired by the time we mark a failed OSD down.)
+
+readable_until
+--------------
+
+Primary and replica both track a couple of values:
+
+* *readable_until* is how long we are allowed to service (read)
+  requests before *our* "lease" expires.
+* *readable_until_ub* is an upper bound on *readable_until* for any
+  OSD in the acting set.
+
+The primary manages these two values by sending *pg_lease_t* messages
+to replicas that increase the upper bound.  Once all acting OSDs have
+acknowledged they've seen the higher bound, the primary increases its
+own *readable_until* and shares that (in a subsequent *pg_lease_t*
+message).  The resulting invariant is that any acting OSDs'
+*readable_until* is always <= any acting OSDs' *readable_until_ub*.
+
+In order to avoid any problems with clock skew, we use monotonic
+clocks (which are only accurate locally and unaffected by time
+adjustments) throughout to manage these leases.  Peer OSDs calculate
+upper and lower bounds on the deltas between OSD-local clocks,
+allowing the primary to share timestamps based on its local clock
+while replicas translate that to an appropriate bound in for their own
+local clocks.
+
+Prior Intervals
+---------------
+
+Whenever there is an interval change, we need to have an upper bound
+on the *readable_until* values for any OSDs in the prior interval.
+All OSDs from that interval have this value (*readable_until_ub*), and
+share it as part of the pg_history_t during peering.
+
+Because peering may involve OSDs that were not already communicating
+before and may not have bounds on their clock deltas, the bound in
+*pg_history_t* is shared as a simple duration before the upper bound
+expires.  This means that the bound slips forward in time due to the
+transit time for the peering message, but that is generally quite
+short, and moving the bound later in time is safe since it is an
+*upper* bound.
+
+PG "laggy" state
+----------------
+
+While the PG is active, *pg_lease_t* and *pg_lease_ack_t* messages are
+regularly exchanged.  However, if a client request comes in and the
+lease has expired (*readable_until* has passed), the PG will go into a
+*LAGGY* state and request will be blocked.  Once the lease is renewed,
+the request(s) will be requeued.
+
+PG "wait" state
+---------------
+
+If peering completes but the prior interval's OSDs may still be
+readable, the PG will go into the *WAIT* state until sufficient time
+has passed.  Any OSD requests will block during that period.  Recovery
+may proceed while in this state, since the logical, user-visible
+content of objects does not change.
+
+Dead OSDs
+---------
+
+Generally speaking, we need to wait until prior intervals' OSDs *know*
+that they should no longer be readable.  If an OSD is known to have
+crashed (e.g., because the process is no longer running, which we may
+infer because we get a ECONNREFUSED error), then we can infer that it
+is not readable.
+
+Similarly, if an OSD is marked down, gets a map update telling it so,
+and then informs the monitor that it knows it was marked down, we can
+similarly infer that it is not still serving requests for a prior interval.
+
+When a PG is in the *WAIT* state, it will watch new maps for OSDs'
+*dead_epoch* value indicating they are aware of their dead-ness.  If
+all down OSDs from prior interval are so aware, we can exit the WAIT
+state early.
diff --git a/doc/dev/osd_internals/watch_notify.rst b/doc/dev/osd_internals/watch_notify.rst
new file mode 100644
index 000000000..8c2ce09ba
--- /dev/null
+++ b/doc/dev/osd_internals/watch_notify.rst
@@ -0,0 +1,81 @@
+============
+Watch Notify
+============
+
+See librados for the watch/notify interface.
+
+Overview
+--------
+The object_info (See osd/osd_types.h) tracks the set of watchers for
+a particular object persistently in the object_info_t::watchers map.
+In order to track notify progress, we also maintain some ephemeral
+structures associated with the ObjectContext.
+
+Each Watch has an associated Watch object (See osd/Watch.h).  The
+ObjectContext for a watched object will have a (strong) reference
+to one Watch object per watch, and each Watch object holds a
+reference to the corresponding ObjectContext.  This circular reference
+is deliberate and is broken when the Watch state is discarded on
+a new peering interval or removed upon timeout expiration or an
+unwatch operation.
+
+A watch tracks the associated connection via a strong
+ConnectionRef Watch::conn.  The associated connection has a
+WatchConState stashed in the OSD::Session for tracking associated
+Watches in order to be able to notify them upon ms_handle_reset()
+(via WatchConState::reset()).
+
+Each Watch object tracks the set of currently un-acked notifies.
+start_notify() on a Watch object adds a reference to a new in-progress
+Notify to the Watch and either:
+
+* if the Watch is *connected*, sends a Notify message to the client
+* if the Watch is *unconnected*, does nothing.
+
+When the Watch becomes connected (in PrimaryLogPG::do_osd_op_effects),
+Notifies are resent to all remaining tracked Notify objects.
+
+Each Notify object tracks the set of un-notified Watchers via
+calls to complete_watcher().  Once the remaining set is empty or the
+timeout expires (cb, registered in init()) a notify completion
+is sent to the client.
+
+Watch Lifecycle
+---------------
+A watch may be in one of 5 states:
+
+1. Non existent.
+2. On disk, but not registered with an object context.
+3. Connected
+4. Disconnected, callback registered with timer
+5. Disconnected, callback in queue for scrub or is_degraded
+
+Case 2 occurs between when an OSD goes active and the ObjectContext
+for an object with watchers is loaded into memory due to an access.
+During Case 2, no state is registered for the watch.  Case 2
+transitions to Case 4 in PrimaryLogPG::populate_obc_watchers() during
+PrimaryLogPG::find_object_context.  Case 1 becomes case 3 via
+OSD::do_osd_op_effects due to a watch operation.  Case 4,5 become case
+3 in the same way. Case 3 becomes case 4 when the connection resets
+on a watcher's session.
+
+Cases 4&5 can use some explanation.  Normally, when a Watch enters Case
+4, a callback is registered with the OSDService::watch_timer to be
+called at timeout expiration.  At the time that the callback is
+called, however, the pg might be in a state where it cannot write
+to the object in order to remove the watch (i.e., during a scrub
+or while the object is degraded).  In that case, we use
+Watch::get_delayed_cb() to generate another Context for use from
+the callbacks_for_degraded_object and Scrubber::callbacks lists.
+In either case, Watch::unregister_cb() does the right thing
+(SafeTimer::cancel_event() is harmless for contexts not registered
+with the timer).
+
+Notify Lifecycle
+----------------
+The notify timeout is simpler: a timeout callback is registered when
+the notify is init()'d.  If all watchers ack notifies before the
+timeout occurs, the timeout is canceled and the client is notified
+of the notify completion.  Otherwise, the timeout fires, the Notify
+object pings each Watch via cancel_notify to remove itself, and
+sends the notify completion to the client early.
diff --git a/doc/dev/osd_internals/wbthrottle.rst b/doc/dev/osd_internals/wbthrottle.rst
new file mode 100644
index 000000000..9b67efbb6
--- /dev/null
+++ b/doc/dev/osd_internals/wbthrottle.rst
@@ -0,0 +1,28 @@
+==================
+Writeback Throttle
+==================
+
+Previously, the filestore had a problem when handling large numbers of
+small ios.  We throttle dirty data implicitly via the journal, but
+a large number of inodes can be dirtied without filling the journal
+resulting in a very long sync time when the sync finally does happen.
+The flusher was not an adequate solution to this problem since it
+forced writeback of small writes too eagerly killing performance.
+
+WBThrottle tracks unflushed io per hobject_t and ::fsyncs in lru
+order once the start_flusher threshold is exceeded for any of
+dirty bytes, dirty ios, or dirty inodes.  While any of these exceed
+the hard_limit, we block on throttle() in _do_op.
+
+See src/os/WBThrottle.h, src/osd/WBThrottle.cc
+
+To track the open FDs through the writeback process, there is now an
+fdcache to cache open fds.  lfn_open now returns a cached FDRef which
+implicitly closes the fd once all references have expired.
+
+Filestore syncs have a sideeffect of flushing all outstanding objects
+in the wbthrottle.
+
+lfn_unlink clears the cached FDRef and wbthrottle entries for the
+unlinked object when the last link is removed and asserts that all
+outstanding FDRefs for that object are dead.
diff --git a/doc/dev/peering.rst b/doc/dev/peering.rst
new file mode 100644
index 000000000..3960e14ca
--- /dev/null
+++ b/doc/dev/peering.rst
@@ -0,0 +1,270 @@
+======================
+Peering
+======================
+
+Concepts
+--------
+
+*Peering*
+   the process of bringing all of the OSDs that store
+   a Placement Group (PG) into agreement about the state
+   of all of the objects (and their metadata) in that PG.
+   Note that agreeing on the state does not mean that
+   they all have the latest contents.
+
+*Acting set*
+   the ordered list of OSDs who are (or were as of some epoch)
+   responsible for a particular PG.
+
+*Up set*
+   the ordered list of OSDs responsible for a particular PG for
+   a particular epoch according to CRUSH.  Normally this
+   is the same as the *acting set*, except when the *acting set* has been
+   explicitly overridden via *PG temp* in the OSDMap.
+
+*PG temp* 
+   a temporary placement group acting set used while backfilling the
+   primary osd. Let say acting is [0,1,2] and we are
+   active+clean. Something happens and acting is now [3,1,2]. osd 3 is
+   empty and can't serve reads although it is the primary. osd.3 will
+   see that and request a *PG temp* of [1,2,3] to the monitors using a
+   MOSDPGTemp message so that osd.1 temporarily becomes the
+   primary. It will select osd.3 as a backfill peer and continue to
+   serve reads and writes while osd.3 is backfilled. When backfilling
+   is complete, *PG temp* is discarded and the acting set changes back
+   to [3,1,2] and osd.3 becomes the primary.
+
+*current interval* or *past interval*
+   a sequence of OSD map epochs during which the *acting set* and *up
+   set* for particular PG do not change
+
+*primary*
+   the (by convention first) member of the *acting set*,
+   who is responsible for coordination peering, and is
+   the only OSD that will accept client initiated
+   writes to objects in a placement group.
+
+*replica*
+   a non-primary OSD in the *acting set* for a placement group
+   (and who has been recognized as such and *activated* by the primary).
+
+*stray*
+   an OSD who is not a member of the current *acting set*, but
+   has not yet been told that it can delete its copies of a
+   particular placement group.
+
+*recovery*
+   ensuring that copies of all of the objects in a PG
+   are on all of the OSDs in the *acting set*.  Once
+   *peering* has been performed, the primary can start
+   accepting write operations, and *recovery* can proceed
+   in the background.
+
+*PG info*
+   basic metadata about the PG's creation epoch, the version
+   for the most recent write to the PG, *last epoch started*, *last
+   epoch clean*, and the beginning of the *current interval*.  Any
+   inter-OSD communication about PGs includes the *PG info*, such that
+   any OSD that knows a PG exists (or once existed) also has a lower
+   bound on *last epoch clean* or *last epoch started*.
+
+*PG log*
+   a list of recent updates made to objects in a PG.
+   Note that these logs can be truncated after all OSDs
+   in the *acting set* have acknowledged up to a certain
+   point.
+
+*missing set*
+   Each OSD notes update log entries and if they imply updates to
+   the contents of an object, adds that object to a list of needed
+   updates.  This list is called the *missing set* for that <OSD,PG>.
+
+*Authoritative History*
+   a complete, and fully ordered set of operations that, if
+   performed, would bring an OSD's copy of a Placement Group
+   up to date.
+
+*epoch*
+   a (monotonically increasing) OSD map version number
+
+*last epoch start*
+   the last epoch at which all nodes in the *acting set*
+   for a particular placement group agreed on an
+   *authoritative history*.  At this point, *peering* is
+   deemed to have been successful.
+
+*up_thru*
+   before a primary can successfully complete the *peering* process,
+   it must inform a monitor that is alive through the current
+   OSD map epoch by having the monitor set its *up_thru* in the osd
+   map.  This helps peering ignore previous *acting sets* for which
+   peering never completed after certain sequences of failures, such as
+   the second interval below:
+
+   - *acting set* = [A,B]
+   - *acting set* = [A]
+   - *acting set* = [] very shortly after (e.g., simultaneous failure, but staggered detection)
+   - *acting set* = [B] (B restarts, A does not)
+
+*last epoch clean*
+   the last epoch at which all nodes in the *acting set*
+   for a particular placement group were completely
+   up to date (both PG logs and object contents).
+   At this point, *recovery* is deemed to have been
+   completed.
+
+Description of the Peering Process
+----------------------------------
+
+The *Golden Rule* is that no write operation to any PG
+is acknowledged to a client until it has been persisted
+by all members of the *acting set* for that PG.  This means
+that if we can communicate with at least one member of
+each *acting set* since the last successful *peering*, someone
+will have a record of every (acknowledged) operation
+since the last successful *peering*.
+This means that it should be possible for the current
+primary to construct and disseminate a new *authoritative history*.
+
+It is also important to appreciate the role of the OSD map
+(list of all known OSDs and their states, as well as some
+information about the placement groups) in the *peering*
+process:
+
+   When OSDs go up or down (or get added or removed)
+   this has the potential to affect the *active sets*
+   of many placement groups.
+
+   Before a primary successfully completes the *peering*
+   process, the OSD map must reflect that the OSD was alive
+   and well as of the first epoch in the *current interval*.
+
+   Changes can only be made after successful *peering*.
+
+Thus, a new primary can use the latest OSD map along with a recent
+history of past maps to generate a set of *past intervals* to
+determine which OSDs must be consulted before we can successfully
+*peer*.  The set of past intervals is bounded by *last epoch started*,
+the most recent *past interval* for which we know *peering* completed.
+The process by which an OSD discovers a PG exists in the first place is
+by exchanging *PG info* messages, so the OSD always has some lower
+bound on *last epoch started*.
+
+The high level process is for the current PG primary to:
+
+  1. get a recent OSD map (to identify the members of the all
+     interesting *acting sets*, and confirm that we are still the
+     primary).
+
+  #. generate a list of *past intervals* since *last epoch started*.
+     Consider the subset of those for which *up_thru* was greater than
+     the first interval epoch by the last interval epoch's OSD map; that is,
+     the subset for which *peering* could have completed before the *acting
+     set* changed to another set of OSDs.
+
+     Successful *peering* will require that we be able to contact at
+     least one OSD from each of *past interval*'s *acting set*.
+
+  #. ask every node in that list for its *PG info*, which includes the most
+     recent write made to the PG, and a value for *last epoch started*.  If
+     we learn about a *last epoch started* that is newer than our own, we can
+     prune older *past intervals* and reduce the peer OSDs we need to contact.
+
+  #. if anyone else has (in its PG log) operations that I do not have,
+     instruct them to send me the missing log entries so that the primary's
+     *PG log* is up to date (includes the newest write)..
+
+  #. for each member of the current *acting set*:
+
+     a. ask it for copies of all PG log entries since *last epoch start*
+	so that I can verify that they agree with mine (or know what
+	objects I will be telling it to delete).
+
+	If the cluster failed before an operation was persisted by all
+	members of the *acting set*, and the subsequent *peering* did not
+	remember that operation, and a node that did remember that
+	operation later rejoined, its logs would record a different
+	(divergent) history than the *authoritative history* that was
+	reconstructed in the *peering* after the failure.
+
+	Since the *divergent* events were not recorded in other logs
+	from that *acting set*, they were not acknowledged to the client,
+	and there is no harm in discarding them (so that all OSDs agree
+	on the *authoritative history*).  But, we will have to instruct
+	any OSD that stores data from a divergent update to delete the
+	affected (and now deemed to be apocryphal) objects.
+
+     #. ask it for its *missing set* (object updates recorded
+	in its PG log, but for which it does not have the new data).
+	This is the list of objects that must be fully replicated
+	before we can accept writes.
+
+  #. at this point, the primary's PG log contains an *authoritative history* of
+     the placement group, and the OSD now has sufficient
+     information to bring any other OSD in the *acting set* up to date.
+
+  #. if the primary's *up_thru* value in the current OSD map is not greater than
+     or equal to the first epoch in the *current interval*, send a request to the
+     monitor to update it, and wait until receive an updated OSD map that reflects
+     the change.
+
+  #. for each member of the current *acting set*:
+
+     a. send them log updates to bring their PG logs into agreement with
+	my own (*authoritative history*) ... which may involve deciding
+	to delete divergent objects.
+
+     #. await acknowledgment that they have persisted the PG log entries.
+
+  #. at this point all OSDs in the *acting set* agree on all of the meta-data,
+     and would (in any future *peering*) return identical accounts of all
+     updates.
+
+     a. start accepting client write operations (because we have unanimous
+	agreement on the state of the objects into which those updates are
+	being accepted).  Note, however, that if a client tries to write to an
+        object it will be promoted to the front of the recovery queue, and the
+        write willy be applied after it is fully replicated to the current *acting set*.
+
+     #. update the *last epoch started* value in our local *PG info*, and instruct
+	other *active set* OSDs to do the same.
+
+     #. start pulling object data updates that other OSDs have, but I do not.  We may
+	need to query OSDs from additional *past intervals* prior to *last epoch started*
+	(the last time *peering* completed) and following *last epoch clean* (the last epoch that
+	recovery completed) in order to find copies of all objects.
+
+     #. start pushing object data updates to other OSDs that do not yet have them.
+
+	We push these updates from the primary (rather than having the replicas
+	pull them) because this allows the primary to ensure that a replica has
+	the current contents before sending it an update write.  It also makes
+	it possible for a single read (from the primary) to be used to write
+	the data to multiple replicas.  If each replica did its own pulls,
+	the data might have to be read multiple times.
+
+  #. once all replicas store the all copies of all objects (that
+     existed prior to the start of this epoch) we can update *last
+     epoch clean* in the *PG info*, and we can dismiss all of the
+     *stray* replicas, allowing them to delete their copies of objects
+     for which they are no longer in the *acting set*.
+
+     We could not dismiss the *strays* prior to this because it was possible
+     that one of those *strays* might hold the sole surviving copy of an
+     old object (all of whose copies disappeared before they could be
+     replicated on members of the current *acting set*).
+
+Generate a State Model
+----------------------
+
+Use the `gen_state_diagram.py <https://github.com/ceph/ceph/blob/master/doc/scripts/gen_state_diagram.py>`_ script to generate a copy of the latest peering state model::
+
+        $ git clone https://github.com/ceph/ceph.git
+        $ cd ceph
+        $ cat src/osd/PeeringState.h src/osd/PeeringState.cc | doc/scripts/gen_state_diagram.py > doc/dev/peering_graph.generated.dot
+        $ sed -i 's/7,7/1080,1080/' doc/dev/peering_graph.generated.dot
+        $ dot -Tsvg doc/dev/peering_graph.generated.dot > doc/dev/peering_graph.generated.svg
+
+Sample state model:
+
+.. graphviz:: peering_graph.generated.dot
diff --git a/doc/dev/perf.rst b/doc/dev/perf.rst
new file mode 100644
index 000000000..57742eec4
--- /dev/null
+++ b/doc/dev/perf.rst
@@ -0,0 +1,55 @@
+Using perf
+==========
+
+Top::
+
+  sudo perf top -p `pidof ceph-osd`
+
+To capture some data with call graphs::
+
+  sudo perf record -p `pidof ceph-osd` -F 99 --call-graph dwarf -- sleep 60
+
+To view by caller (where you can see what each top function calls)::
+
+  sudo perf report --call-graph caller
+
+To view by callee (where you can see who calls each top function)::  
+
+  sudo perf report --call-graph callee
+
+:note: If the caller/callee views look the same you may be
+       suffering from a kernel bug; upgrade to 4.8 or later.
+
+Common Issues
+-------------
+
+Ceph use `RelWithDebInfo` as its default `CMAKE_BUILD_TYPE`. Hence `-O2 -g` is
+used to compile the tree in this case. And the `-O2` optimization level
+enables `-fomit-frame-pointer` by default. But this prevents stack profilers
+from accessing the complete stack information. So one can disable this option
+when launching `cmake` ::
+
+  cmake -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer"
+
+or when building the tree::
+
+  make CMAKE_CXX_FLAGS="-fno-omit-frame-pointer"
+
+
+Flamegraphs
+-----------
+
+First, get things set up::
+
+  cd ~/src
+  git clone https://github.com/brendangregg/FlameGraph
+
+Run ceph, then record some perf data::
+
+  sudo perf record -p `pidof ceph-osd` -F 99 --call-graph dwarf -- sleep 60
+
+Then generate the flamegraph::
+
+  sudo perf script | ~/src/FlameGraph/stackcollapse-perf.pl > /tmp/folded
+  ~/src/FlameGraph/flamegraph.pl /tmp/folded > /tmp/perf.svg
+  firefox /tmp/perf.svg
diff --git a/doc/dev/perf_counters.rst b/doc/dev/perf_counters.rst
new file mode 100644
index 000000000..a64d14d33
--- /dev/null
+++ b/doc/dev/perf_counters.rst
@@ -0,0 +1,247 @@
+===============
+ Perf counters
+===============
+
+The perf counters provide generic internal infrastructure for gauges and counters.  The counted values can be both integer and float. There is also an "average" type (normally float) that combines a sum and num counter which can be divided to provide an average.
+
+The intention is that this data will be collected and aggregated by a tool like ``collectd`` or ``statsd`` and fed into a tool like ``graphite`` for graphing and analysis.  Also, note the :doc:`../mgr/prometheus` and the :doc:`../mgr/telemetry`.
+
+Users and developers can also access perf counter data locally to check a cluster's overall health, identify workload patterns, monitor cluster performance by daemon types, and troubleshoot issues with latency, throttling, memory management, etc. (see :ref:`Access`)
+
+.. _Access:
+
+Access
+------
+
+The perf counter data is accessed via the admin socket.  For example::
+
+   ceph daemon osd.0 perf schema
+   ceph daemon osd.0 perf dump
+
+
+Collections
+-----------
+
+The values are grouped into named collections, normally representing a subsystem or an instance of a subsystem.  For example, the internal ``throttle`` mechanism reports statistics on how it is throttling, and each instance is named something like::
+
+
+    throttle-msgr_dispatch_throttler-hbserver
+    throttle-msgr_dispatch_throttler-client
+    throttle-filestore_bytes
+    ...
+
+
+Schema
+------
+
+The ``perf schema`` command dumps a json description of which values are available, and what their type is.  Each named value as a ``type`` bitfield, with the following bits defined.
+
++------+-------------------------------------+
+| bit  | meaning                             |
++======+=====================================+
+| 1    | floating point value                |
++------+-------------------------------------+
+| 2    | unsigned 64-bit integer value       |
++------+-------------------------------------+
+| 4    | average (sum + count pair), where   |
++------+-------------------------------------+
+| 8    | counter (vs gauge)                  |
++------+-------------------------------------+
+
+Every value will have either bit 1 or 2 set to indicate the type
+(float or integer).
+
+If bit 8 is set (counter), the value is monotonically increasing and
+the reader may want to subtract off the previously read value to get
+the delta during the previous interval.
+
+If bit 4 is set (average), there will be two values to read, a sum and
+a count.  If it is a counter, the average for the previous interval
+would be sum delta (since the previous read) divided by the count
+delta.  Alternatively, dividing the values outright would provide the
+lifetime average value.  Normally these are used to measure latencies
+(number of requests and a sum of request latencies), and the average
+for the previous interval is what is interesting.
+
+Instead of interpreting the bit fields, the ``metric type`` has a
+value of either ``gauge`` or ``counter``, and the ``value type``
+property will be one of ``real``, ``integer``, ``real-integer-pair``
+(for a sum + real count pair), or ``integer-integer-pair`` (for a
+sum + integer count pair).
+
+Here is an example of the schema output::
+
+  {
+    "throttle-bluestore_throttle_bytes": {
+        "val": {
+            "type": 2,
+            "metric_type": "gauge",
+            "value_type": "integer",
+            "description": "Currently available throttle",
+            "nick": ""
+        },
+        "max": {
+            "type": 2,
+            "metric_type": "gauge",
+            "value_type": "integer",
+            "description": "Max value for throttle",
+            "nick": ""
+        },
+        "get_started": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Number of get calls, increased before wait",
+            "nick": ""
+        },
+        "get": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Gets",
+            "nick": ""
+        },
+        "get_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Got data",
+            "nick": ""
+        },
+        "get_or_fail_fail": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Get blocked during get_or_fail",
+            "nick": ""
+        },
+        "get_or_fail_success": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Successful get during get_or_fail",
+            "nick": ""
+        },
+        "take": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Takes",
+            "nick": ""
+        },
+        "take_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Taken data",
+            "nick": ""
+        },
+        "put": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Puts",
+            "nick": ""
+        },
+        "put_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Put data",
+            "nick": ""
+        },
+        "wait": {
+            "type": 5,
+            "metric_type": "gauge",
+            "value_type": "real-integer-pair",
+            "description": "Waiting latency",
+            "nick": ""
+        }
+  }
+
+
+Dump
+----
+
+The actual dump is similar to the schema, except that average values are grouped.  For example::
+
+ {
+   "throttle-msgr_dispatch_throttler-hbserver" : {
+      "get_or_fail_fail" : 0,
+      "get_sum" : 0,
+      "max" : 104857600,
+      "put" : 0,
+      "val" : 0,
+      "take" : 0,
+      "get_or_fail_success" : 0,
+      "wait" : {
+         "avgcount" : 0,
+         "sum" : 0
+      },
+      "get" : 0,
+      "take_sum" : 0,
+      "put_sum" : 0
+   },
+   "throttle-msgr_dispatch_throttler-client" : {
+      "get_or_fail_fail" : 0,
+      "get_sum" : 82760,
+      "max" : 104857600,
+      "put" : 2637,
+      "val" : 0,
+      "take" : 0,
+      "get_or_fail_success" : 0,
+      "wait" : {
+         "avgcount" : 0,
+         "sum" : 0
+      },
+      "get" : 2637,
+      "take_sum" : 0,
+      "put_sum" : 82760
+   }
+ }
+
+Labeled Perf Counters
+---------------------
+
+A Ceph daemon has the ability to emit a set of perf counter instances with varying labels. These counters are intended for visualizing specific metrics in 3rd party tools like Prometheus and Grafana.
+
+For example, the below counters show the number of put requests for different users on different buckets::
+
+  {
+      "rgw": [
+          {
+              "labels": {
+                  "Bucket: "bkt1",
+                  "User: "user1",
+              },
+              "counters": {
+                  "put": 1,
+              },
+          },
+          {
+              "labels": {},
+              "counters": {
+                  "put": 4,
+              },
+          },
+          {
+              "labels": {
+                  "Bucket: "bkt1",
+                  "User: "user2",
+              },
+              "counters": {
+                  "put": 3,
+              },
+          },
+      ]
+  }
+
+All labeled and unlabeled perf counters can be viewed with ``ceph daemon {daemon id} counter dump``. 
+
+All labeled and unlabeled perf counter's schema can be viewed with ``ceph daemon {daemon id} counter schema``.
+
+In the above example the second counter without labels is a counter that would also be shown in ``ceph daemon {daemon id} perf dump``.
+
+Since the ``counter dump`` and ``counter schema`` commands can be used to view both types of counters it is not recommended to use the ``perf dump`` and ``perf schema`` commands which are retained for backwards compatibility and continue to emit only non-labeled counters.
+
+Some perf counters that are emitted via ``perf dump`` and ``perf schema`` may become labeled in future releases and as such will no longer be emitted by ``perf dump`` and ``perf schema`` respectively.
diff --git a/doc/dev/perf_histograms.rst b/doc/dev/perf_histograms.rst
new file mode 100644
index 000000000..429c00400
--- /dev/null
+++ b/doc/dev/perf_histograms.rst
@@ -0,0 +1,677 @@
+=================
+ Perf histograms
+=================
+
+The perf histograms build on perf counters infrastructure. Histograms are built for a number of counters and simplify gathering data on which groups of counter values occur most often over time. 
+Perf histograms are currently unsigned 64-bit integer counters, so they're mostly useful for time and sizes. Data dumped by perf histogram can then be feed into other analysis tools/scripts.
+
+Access
+------
+
+The perf histogram data are accessed via the admin socket.  For example::
+
+   ceph daemon osd.0 perf histogram schema
+   ceph daemon osd.0 perf histogram dump
+
+
+Collections
+-----------
+
+The histograms are grouped into named collections, normally representing a subsystem or an instance of a subsystem.  For example, the internal ``throttle`` mechanism reports statistics on how it is throttling, and each instance is named something like::
+
+
+    op_r_latency_out_bytes_histogram
+    op_rw_latency_in_bytes_histogram
+    op_rw_latency_out_bytes_histogram
+    ...
+
+
+Schema
+------
+
+The ``perf histogram schema`` command dumps a json description of which values are available, and what their type is.  Each named value as a ``type`` bitfield, with the 5-th bit always set and following bits defined.
+
++------+-------------------------------------+
+| bit  | meaning                             |
++======+=====================================+
+| 1    | floating point value                |
++------+-------------------------------------+
+| 2    | unsigned 64-bit integer value       |
++------+-------------------------------------+
+| 4    | average (sum + count pair)          |
++------+-------------------------------------+
+| 8    | counter (vs gauge)                  |
++------+-------------------------------------+
+
+In other words, histogram of type "18" is a histogram of unsigned 64-bit integer values (16 + 2).
+
+Here is an example of the schema output::
+
+    {
+        "AsyncMessenger::Worker-0": {},
+        "AsyncMessenger::Worker-1": {},
+        "AsyncMessenger::Worker-2": {},
+        "mutex-WBThrottle::lock": {},
+        "objecter": {},
+        "osd": {
+            "op_r_latency_out_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of operation latency (including queue time) +   da  ta read",
+                "nick": ""
+            },
+            "op_w_latency_in_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of operation latency (including queue time) +   da  ta written",
+                "nick": ""
+            },
+            "op_rw_latency_in_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of rw operation latency (including queue time)   +   data written",
+                "nick": ""
+            },
+            "op_rw_latency_out_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of rw operation latency (including queue time)   +   data read",
+                "nick": ""
+            }
+        }
+    }
+
+
+Dump
+----
+
+The actual dump is similar to the schema, except that there are actual value groups.  For example::
+
+    "osd": {
+        "op_r_latency_out_bytes_histogram": {
+            "axes": [
+                {
+                    "name": "Latency (usec)",
+                    "min": 0,
+                    "quant_size": 100000,
+                    "buckets": 32,
+                    "scale_type": "log2",
+                    "ranges": [
+                        {
+                            "max": -1
+                        },
+                        {
+                            "min": 0,
+                            "max": 99999
+                        },
+                        {
+                            "min": 100000,
+                            "max": 199999
+                        },
+                        {
+                            "min": 200000,
+                            "max": 399999
+                        },
+                        {
+                            "min": 400000,
+                            "max": 799999
+                        },
+                        {
+                            "min": 800000,
+                            "max": 1599999
+                        },
+                        {
+                            "min": 1600000,
+                            "max": 3199999
+                        },
+                        {
+                            "min": 3200000,
+                            "max": 6399999
+                        },
+                        {
+                            "min": 6400000,
+                            "max": 12799999
+                        },
+                        {
+                            "min": 12800000,
+                            "max": 25599999
+                        },
+                        {
+                            "min": 25600000,
+                            "max": 51199999
+                        },
+                        {
+                            "min": 51200000,
+                            "max": 102399999
+                        },
+                        {
+                            "min": 102400000,
+                            "max": 204799999
+                        },
+                        {
+                            "min": 204800000,
+                            "max": 409599999
+                        },
+                        {
+                            "min": 409600000,
+                            "max": 819199999
+                        },
+                        {
+                            "min": 819200000,
+                            "max": 1638399999
+                        },
+                        {
+                            "min": 1638400000,
+                            "max": 3276799999
+                        },
+                        {
+                            "min": 3276800000,
+                            "max": 6553599999
+                        },
+                        {
+                            "min": 6553600000,
+                            "max": 13107199999
+                        },
+                        {
+                            "min": 13107200000,
+                            "max": 26214399999
+                        },
+                        {
+                            "min": 26214400000,
+                            "max": 52428799999
+                        },
+                        {
+                            "min": 52428800000,
+                            "max": 104857599999
+                        },
+                        {
+                            "min": 104857600000,
+                            "max": 209715199999
+                        },
+                        {
+                            "min": 209715200000,
+                            "max": 419430399999
+                        },
+                        {
+                            "min": 419430400000,
+                            "max": 838860799999
+                        },
+                        {
+                            "min": 838860800000,
+                            "max": 1677721599999
+                        },
+                        {
+                            "min": 1677721600000,
+                            "max": 3355443199999
+                        },
+                        {
+                            "min": 3355443200000,
+                            "max": 6710886399999
+                        },
+                        {
+                            "min": 6710886400000,
+                            "max": 13421772799999
+                        },
+                        {
+                            "min": 13421772800000,
+                            "max": 26843545599999
+                        },
+                        {
+                            "min": 26843545600000,
+                            "max": 53687091199999
+                        },
+                        },
+                        {
+                            "min": 53687091200000
+                        }
+                    ]
+                },
+                {
+                    "name": "Request size (bytes)",
+                    "min": 0,
+                    "quant_size": 512,
+                    "buckets": 32,
+                    "scale_type": "log2",
+                    "ranges": [
+                        {
+                            "max": -1
+                        },
+                        {
+                            "min": 0,
+                            "max": 511
+                        },
+                        {
+                            "min": 512,
+                            "max": 1023
+                        },
+                        {
+                            "min": 1024,
+                            "max": 2047
+                        },
+                        {
+                            "min": 2048,
+                            "max": 4095
+                        },
+                        {
+                            "min": 4096,
+                            "max": 8191
+                        },
+                        {
+                            "min": 8192,
+                            "max": 16383
+                        },
+                        {
+                            "min": 16384,
+                            "max": 32767
+                        },
+                        {
+                            "min": 32768,
+                            "max": 65535
+                        },
+                        {
+                            "min": 65536,
+                            "max": 131071
+                        },
+                        {
+                            "min": 131072,
+                            "max": 262143
+                        },
+                        {
+                            "min": 262144,
+                            "max": 524287
+                        },
+                        {
+                            "min": 524288,
+                            "max": 1048575
+                        },
+                        {
+                            "min": 1048576,
+                            "max": 2097151
+                        },
+                        {
+                            "min": 2097152,
+                            "max": 4194303
+                        },
+                        {
+                            "min": 4194304,
+                            "max": 8388607
+                        },
+                        {
+                            "min": 8388608,
+                            "max": 16777215
+                        },
+                        {
+                            "min": 16777216,
+                            "max": 33554431
+                        },
+                        {
+                            "min": 33554432,
+                            "max": 67108863
+                        },
+                        {
+                            "min": 67108864,
+                            "max": 134217727
+                        },
+                        {
+                            "min": 134217728,
+                            "max": 268435455
+                        },
+                        {
+                            "min": 268435456,
+                            "max": 536870911
+                        },
+                        {
+                            "min": 536870912,
+                            "max": 1073741823
+                        },
+                        {
+                            "min": 1073741824,
+                            "max": 2147483647
+                        },
+                        {
+                            "min": 2147483648,
+                            "max": 4294967295
+                        },
+                        {
+                            "min": 4294967296,
+                            "max": 8589934591
+                        },
+                        {
+                            "min": 8589934592,
+                            "max": 17179869183
+                        },
+                        {
+                            "min": 17179869184,
+                            "max": 34359738367
+                        },
+                        {
+                            "min": 34359738368,
+                            "max": 68719476735
+                        },
+                        {
+                            "min": 68719476736,
+                            "max": 137438953471
+                        },
+                        {
+                            "min": 137438953472,
+                            "max": 274877906943
+                        },
+                        {
+                            "min": 274877906944
+                        }
+                    ]
+                }
+            ],
+            "values": [
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ]
+            ]
+        }
+    },
+
+This represents the 2D histogram, consisting of 9 history entries and 32 value groups per each history entry.
+"Ranges" element denote value bounds for each of value groups. "Buckets" denote amount of value groups ("buckets"), 
+"Min" is a minimum accepted value, "quant_size" is quantization unit and "scale_type" is either "log2" (logarithmic
+scale) or "linear" (linear scale).
+You can use histogram_dump.py tool (see src/tools/histogram_dump.py) for quick visualisation of existing histogram
+data.
diff --git a/doc/dev/placement-group.rst b/doc/dev/placement-group.rst
new file mode 100644
index 000000000..e29be2fa6
--- /dev/null
+++ b/doc/dev/placement-group.rst
@@ -0,0 +1,210 @@
+============================
+ PG (Placement Group) notes
+============================
+
+Miscellaneous copy-pastes from emails, when this gets cleaned up it
+should move out of /dev.
+
+Overview
+========
+
+PG = "placement group". When placing data in the cluster, objects are
+mapped into PGs, and those PGs are mapped onto OSDs. We use the
+indirection so that we can group objects, which reduces the amount of
+per-object metadata we need to keep track of and processes we need to
+run (it would be prohibitively expensive to track eg the placement
+history on a per-object basis). Increasing the number of PGs can
+reduce the variance in per-OSD load across your cluster, but each PG
+requires a bit more CPU and memory on the OSDs that are storing it. We
+try and ballpark it at 100 PGs/OSD, although it can vary widely
+without ill effects depending on your cluster. You hit a bug in how we
+calculate the initial PG number from a cluster description.
+
+There are a couple of different categories of PGs; the 6 that exist
+(in the original emailer's ``ceph -s`` output) are "local" PGs which
+are tied to a specific OSD. However, those aren't actually used in a
+standard Ceph configuration.
+
+
+Mapping algorithm (simplified)
+==============================
+
+| > How does the Object->PG mapping look like, do you map more than one object on
+| > one PG, or do you sometimes map an object to more than one PG? How about the
+| > mapping of PGs to OSDs, does one PG belong to exactly one OSD?
+| >
+| > Does one PG represent a fixed amount of storage space?
+
+Many objects map to one PG.
+
+Each object maps to exactly one PG.
+
+One PG maps to a single list of OSDs, where the first one in the list
+is the primary and the rest are replicas.
+
+Many PGs can map to one OSD.
+
+A PG represents nothing but a grouping of objects; you configure the
+number of PGs you want, number of OSDs * 100 is a good starting point
+, and all of your stored objects are pseudo-randomly evenly distributed
+to the PGs. So a PG explicitly does NOT represent a fixed amount of
+storage; it represents 1/pg_num'th of the storage you happen to have
+on your OSDs.
+
+Ignoring the finer points of CRUSH and custom placement, it goes
+something like this in pseudocode::
+
+	locator = object_name
+	obj_hash = hash(locator)
+	pg = obj_hash % num_pg
+	OSDs_for_pg = crush(pg)  # returns a list of OSDs
+	primary = osds_for_pg[0]
+	replicas = osds_for_pg[1:]
+
+If you want to understand the crush() part in the above, imagine a
+perfectly spherical datacenter in a vacuum ;) that is, if all OSDs
+have weight 1.0, and there is no topology to the data center (all OSDs
+are on the top level), and you use defaults, etc, it simplifies to
+consistent hashing; you can think of it as::
+
+	def crush(pg):
+	   all_osds = ['osd.0', 'osd.1', 'osd.2', ...]
+	   result = []
+	   # size is the number of copies; primary+replicas
+	   while len(result) < size:
+	       r = hash(pg)
+	       chosen = all_osds[ r % len(all_osds) ]
+	       if chosen in result:
+	           # OSD can be picked only once
+	           continue
+	       result.append(chosen)
+	   return result
+
+User-visible PG States
+======================
+
+.. todo:: diagram of states and how they can overlap
+
+*creating*
+  the PG is still being created
+
+*active*
+  requests to the PG will be processed
+
+*clean*
+  all objects in the PG are replicated the correct number of times
+
+*down*
+  a replica with necessary data is down, so the pg is offline
+
+*recovery_unfound*
+  recovery could not finish because object(s) are unfound.
+
+*backfill_unfound*
+  backfill could not finish because object(s) are unfound.
+
+*premerge*
+  the PG is in a quiesced-IO state due to an impending PG merge.  That
+  happens when pg_num_pending < pg_num, and applies to the PGs with
+  pg_num_pending <= ps < pg_num as well as the corresponding peer PG
+  that it is merging with.
+
+*scrubbing*
+  the PG is being checked for inconsistencies
+
+*degraded*
+  some objects in the PG are not replicated enough times yet
+
+*inconsistent*
+  replicas of the PG are not consistent (e.g. objects are
+  the wrong size, objects are missing from one replica *after* recovery
+  finished, etc.)
+
+*peering*
+  the PG is undergoing the :doc:`/dev/peering` process
+
+*repair*
+  the PG is being checked and any inconsistencies found will be repaired (if possible)
+
+*recovering*
+  objects are being migrated/synchronized with replicas
+
+*backfill_wait*
+  the PG is waiting in line to start backfill
+
+*incomplete*
+  a pg is missing a necessary period of history from its
+  log.  If you see this state, report a bug, and try to start any
+  failed OSDs that may contain the needed information.
+
+*stale*
+  the PG is in an unknown state - the monitors have not received
+  an update for it since the PG mapping changed.
+
+*remapped*
+  the PG is temporarily mapped to a different set of OSDs from what
+  CRUSH specified
+
+*deep*
+  In conjunction with *scrubbing* the scrub is a deep scrub
+
+*backfilling*
+  a special case of recovery, in which the entire contents of
+  the PG are scanned and synchronized, instead of inferring what
+  needs to be transferred from the PG logs of recent operations
+
+*backfill_toofull*
+  backfill reservation rejected, OSD too full
+
+*recovery_wait*
+  the PG is waiting for the local/remote recovery reservations
+
+*undersized*
+  the PG can't select enough OSDs given its size
+
+*activating*
+  the PG is peered but not yet active
+
+*peered*
+  the PG peered but can't go active
+
+*snaptrim*
+  the PG is trimming snaps
+
+*snaptrim_wait*
+  the PG is queued to trim snaps
+
+*recovery_toofull*
+  recovery reservation rejected, OSD too full
+
+*snaptrim_error*
+  the PG could not complete snap trimming due to errors
+
+*forced_recovery*
+  the PG has been marked for highest priority recovery
+
+*forced_backfill*
+  the PG has been marked for highest priority backfill
+
+*failed_repair*
+  an attempt to repair the PG has failed.  Manual intervention is required.
+
+
+OMAP STATISTICS
+===============
+
+Omap statistics are gathered during deep scrub and displayed in the output of
+the following commands::
+
+  ceph pg dump
+  ceph pg dump all
+  ceph pg dump summary
+  ceph pg dump pgs
+  ceph pg dump pools
+  ceph pg ls
+
+As these statistics are not updated continuously they may be quite inaccurate in
+an environment where deep scrubs are run infrequently and/or there is a lot of
+omap activity. As such they should not be relied on for exact accuracy but
+rather used as a guide. Running a deep scrub and checking these statistics
+immediately afterwards should give a good indication of current omap usage.
diff --git a/doc/dev/quick_guide.rst b/doc/dev/quick_guide.rst
new file mode 100644
index 000000000..bccca0239
--- /dev/null
+++ b/doc/dev/quick_guide.rst
@@ -0,0 +1,158 @@
+=================================
+ Developer Guide (Quick)
+=================================
+
+This guide will describe how to build and test Ceph for development.
+
+Development
+-----------
+
+The ``run-make-check.sh`` script will install Ceph dependencies,
+compile everything in debug mode and run a number of tests to verify
+the result behaves as expected.
+
+.. prompt:: bash $
+
+   ./run-make-check.sh
+
+Optionally if you want to work on a specific component of Ceph,
+install the dependencies and build Ceph in debug mode with required cmake flags.
+
+Example:
+
+.. prompt:: bash $
+
+   ./install-deps.sh
+   ./do_cmake.sh -DWITH_MANPAGE=OFF -DWITH_BABELTRACE=OFF -DWITH_MGR_DASHBOARD_FRONTEND=OFF
+
+You can also turn off building of some core components that are not relevant to
+your development:
+
+.. prompt:: bash $
+
+   ./do_cmake.sh ... -DWITH_RBD=OFF -DWITH_KRBD=OFF -DWITH_RADOSGW=OFF
+
+Finally, build ceph:
+
+.. prompt:: bash $
+
+   cmake --build build [--target <target>...]
+
+Omit ``--target...`` if you want to do a full build.
+
+
+Running a development deployment
+--------------------------------
+
+Ceph contains a script called ``vstart.sh`` (see also
+:doc:`/dev/dev_cluster_deployment`) which allows developers to quickly test
+their code using a simple deployment on your development system. Once the build
+finishes successfully, start the ceph deployment using the following command:
+
+.. prompt:: bash $
+
+   cd build
+   ../src/vstart.sh -d -n
+
+You can also configure ``vstart.sh`` to use only one monitor and one metadata server by using the following:
+
+.. prompt:: bash $
+
+   env MON=1 MDS=1 ../src/vstart.sh -d -n -x
+
+Most logs from the cluster can be found in ``build/out``.
+
+The system creates two pools on startup: `cephfs_data_a` and `cephfs_metadata_a`.  Let's get some stats on
+the current pools:
+
+.. code-block:: console
+
+  $ bin/ceph osd pool stats
+  *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
+  pool cephfs_data_a id 1
+    nothing is going on
+	
+  pool cephfs_metadata_a id 2
+    nothing is going on
+	
+  $ bin/ceph osd pool stats cephfs_data_a
+  *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
+  pool cephfs_data_a id 1
+    nothing is going on
+
+  $ bin/rados df
+  POOL_NAME         USED OBJECTS CLONES COPIES MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR
+  cephfs_data_a        0       0      0      0                  0       0        0      0  0      0    0
+  cephfs_metadata_a 2246      21      0     63                  0       0        0      0  0     42 8192
+
+  total_objects    21
+  total_used       244G
+  total_space      1180G
+
+
+Make a pool and run some benchmarks against it:
+
+.. prompt:: bash $
+
+   bin/ceph osd pool create mypool
+   bin/rados -p mypool bench 10 write -b 123
+
+Place a file into the new pool:
+
+.. prompt:: bash $
+
+   bin/rados -p mypool put objectone <somefile>
+   bin/rados -p mypool put objecttwo <anotherfile>
+
+List the objects in the pool:
+
+.. prompt:: bash $
+
+   bin/rados -p mypool ls
+
+Once you are done, type the following to stop the development ceph deployment:
+
+.. prompt:: bash $
+
+   ../src/stop.sh
+
+Resetting your vstart environment
+---------------------------------
+
+The vstart script creates out/ and dev/ directories which contain
+the cluster's state.  If you want to quickly reset your environment,
+you might do something like this:
+
+.. prompt:: bash [build]$
+
+   ../src/stop.sh
+   rm -rf out dev
+   env MDS=1 MON=1 OSD=3 ../src/vstart.sh -n -d
+
+Running a RadosGW development environment
+-----------------------------------------
+
+Set the ``RGW`` environment variable when running vstart.sh to enable the RadosGW.
+
+.. prompt:: bash $
+
+   cd build
+   RGW=1 ../src/vstart.sh -d -n -x
+
+You can now use the swift python client to communicate with the RadosGW.
+
+.. prompt:: bash $
+
+   swift -A http://localhost:8000/auth -U test:tester -K testing list
+   swift -A http://localhost:8000/auth -U test:tester -K testing upload mycontainer ceph
+   swift -A http://localhost:8000/auth -U test:tester -K testing list
+
+
+Run unit tests
+--------------
+
+The tests are located in `src/tests`.  To run them type:
+
+.. prompt:: bash $
+
+   (cd build && ninja check)
diff --git a/doc/dev/rados-client-protocol.rst b/doc/dev/rados-client-protocol.rst
new file mode 100644
index 000000000..920c65f39
--- /dev/null
+++ b/doc/dev/rados-client-protocol.rst
@@ -0,0 +1,117 @@
+RADOS client protocol
+=====================
+
+This is very incomplete, but one must start somewhere.
+
+Basics
+------
+
+Requests are MOSDOp messages.  Replies are MOSDOpReply messages.
+
+An object request is targeted at an hobject_t, which includes a pool,
+hash value, object name, placement key (usually empty), and snapid.
+
+The hash value is a 32-bit hash value, normally generated by hashing
+the object name.  The hobject_t can be arbitrarily constructed,
+though, with any hash value and name.  Note that in the MOSDOp these
+components are spread across several fields and not logically
+assembled in an actual hobject_t member (mainly historical reasons).
+
+A request can also target a PG.  In this case, the *ps* value matches
+a specific PG, the object name is empty, and (hopefully) the ops in
+the request are PG ops.
+
+Either way, the request ultimately targets a PG, either by using the
+explicit pgid or by folding the hash value onto the current number of
+pgs in the pool.  The client sends the request to the primary for the
+associated PG.
+
+Each request is assigned a unique tid.
+
+Resends
+-------
+
+If there is a connection drop, the client will resend any outstanding
+requests.
+
+Any time there is a PG mapping change such that the primary changes,
+the client is responsible for resending the request.  Note that
+although there may be an interval change from the OSD's perspective
+(triggering PG peering), if the primary doesn't change then the client
+need not resend.
+
+There are a few exceptions to this rule:
+
+ * There is a last_force_op_resend field in the pg_pool_t in the
+   OSDMap.  If this changes, then the clients are forced to resend any
+   outstanding requests. (This happens when tiering is adjusted, for
+   example.)
+ * Some requests are such that they are resent on *any* PG interval
+   change, as defined by pg_interval_t's is_new_interval() (the same
+   criteria used by peering in the OSD).
+ * If the PAUSE OSDMap flag is set and unset.
+
+Each time a request is sent to the OSD the *attempt* field is incremented. The
+first time it is 0, the next 1, etc.
+
+Backoff
+-------
+
+Ordinarily the OSD will simply queue any requests it can't immediately
+process in memory until such time as it can.  This can become
+problematic because the OSD limits the total amount of RAM consumed by
+incoming messages: if either of the thresholds for the number of
+messages or the number of bytes is reached, new messages will not be
+read off the network socket, causing backpressure through the network.
+
+In some cases, though, the OSD knows or expects that a PG or object
+will be unavailable for some time and does not want to consume memory
+by queuing requests.  In these cases it can send a MOSDBackoff message
+to the client.
+
+A backoff request has four properties:
+
+#. the op code (block, unblock, or ack-block)
+#. *id*, a unique id assigned within this session
+#. hobject_t begin
+#. hobject_t end
+
+There are two types of backoff: a *PG* backoff will plug all requests
+targeting an entire PG at the client, as described by a range of the
+hash/hobject_t space [begin,end), while an *object* backoff will plug
+all requests targeting a single object (begin == end).
+
+When the client receives a *block* backoff message, it is now
+responsible for *not* sending any requests for hobject_ts described by
+the backoff.  The backoff remains in effect until the backoff is
+cleared (via an 'unblock' message) or the OSD session is closed.  A
+*ack_block* message is sent back to the OSD immediately to acknowledge
+receipt of the backoff.
+
+When an unblock is
+received, it will reference a specific id that the client previous had
+blocked.  However, the range described by the unblock may be smaller
+than the original range, as the PG may have split on the OSD.  The unblock
+should *only* unblock the range specified in the unblock message.  Any requests
+that fall within the unblock request range are reexamined and, if no other
+installed backoff applies, resent.
+
+On the OSD, Backoffs are also tracked across ranges of the hash space, and
+exist in three states:
+
+#. new
+#. acked
+#. deleting
+
+A newly installed backoff is set to *new* and a message is sent to the
+client.  When the *ack-block* message is received it is changed to the
+*acked* state.  The OSD may process other messages from the client that
+are covered by the backoff in the *new* state, but once the backoff is
+*acked* it should never see a blocked request unless there is a bug.
+
+If the OSD wants to a remove a backoff in the *acked* state it can
+simply remove it and notify the client.  If the backoff is in the
+*new* state it must move it to the *deleting* state and continue to
+use it to discard client requests until the *ack-block* message is
+received, at which point it can finally be removed.  This is necessary to
+preserve the order of operations processed by the OSD.
diff --git a/doc/dev/radosgw/admin/adminops_nonimplemented.rst b/doc/dev/radosgw/admin/adminops_nonimplemented.rst
new file mode 100644
index 000000000..e579bd5aa
--- /dev/null
+++ b/doc/dev/radosgw/admin/adminops_nonimplemented.rst
@@ -0,0 +1,495 @@
+==================
+ Admin Operations
+==================
+
+An admin API request will be done on a URI that starts with the configurable 'admin'
+resource entry point. Authorization for the admin API duplicates the S3 authorization
+mechanism. Some operations require that the user holds special administrative capabilities.
+The response entity type (XML or JSON) may be specified as the 'format' option in the
+request and defaults to JSON if not specified.
+
+Get Object
+==========
+
+Get an existing object. NOTE: Does not require owner to be non-suspended.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?object&format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be retrieved.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to be retrieved.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns the desired object.
+
+``object``
+
+:Description: The desired object.
+:Type: Object
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+Head Object
+===========
+
+Verify the existence of an object. If the object exists,
+metadata headers for the object will be returned.
+
+Syntax
+~~~~~~
+
+::
+
+	HEAD /{admin}/bucket?object HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be retrieved.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to be retrieved.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+Get Zone Info
+=============
+
+Get cluster information.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/zone&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns cluster pool configuration.
+
+``zone``
+
+:Description: Contains current cluster pool configuration.
+:Type: Container
+
+``domain_root``
+
+:Description: root of all buckets.
+:Type: String
+:Parent: ``cluster``
+
+``control_pool``
+
+:Description: 
+:Type: String
+:Parent: ``cluster``
+
+``gc_pool``
+
+:Description: Garbage collection pool.
+:Type: String
+:Parent: ``cluster``
+
+``log_pool``
+
+:Description: Log pool.
+:Type: String
+:Parent: ``cluster``
+
+``intent_log_pool``
+
+:Description: Intent log pool.
+:Type: String
+:Parent: ``cluster``
+
+``usage_log_pool``
+
+:Description: Usage log pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_keys_pool``
+
+:Description: User key pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_email_pool``
+
+:Description: User email pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_swift_pool``
+
+:Description: Pool of swift users.
+:Type: String
+:Parent: ``cluster``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Example Response
+~~~~~~~~~~~~~~~~
+
+::
+
+	HTTP/1.1 200
+	Content-Type: application/json
+
+	{
+	  "domain_root": ".rgw",
+	  "control_pool": ".rgw.control",
+	  "gc_pool": ".rgw.gc",
+	  "log_pool": ".log",
+	  "intent_log_pool": ".intent-log",
+	  "usage_log_pool": ".usage",
+	  "user_keys_pool": ".users",
+	  "user_email_pool": ".users.email",
+	  "user_swift_pool": ".users.swift",
+	  "user_uid_pool ": ".users.uid"
+	}
+
+
+
+Add Placement Pool
+==================
+
+Make a pool available for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``pool``
+
+:Description: The pool to be made available for data placement.
+:Type: String
+:Example: ``foo_pool``
+:Required: Yes
+
+``create``
+
+:Description: Creates the data pool if it does not exist.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Remove Placement Pool
+=====================
+
+Make a pool unavailable for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``pool``
+
+:Description: The existing pool to be made available for data placement.
+:Type: String
+:Example: ``foo_pool``
+:Required: Yes
+
+``destroy``
+
+:Description: Destroys the pool after removing it from the active set.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+List Available Data Placement Pools
+===================================
+
+List current pools available for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns a list of pools available for data placement.
+
+``pools``
+
+:Description: Contains currently available pools for data placement.
+:Type: Container
+
+
+
+List Expired Garbage Collection Items
+=====================================
+
+List objects scheduled for garbage collection.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/garbage?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If expired garbage collection items exist, a list of such objects
+will be returned.
+
+``garbage``
+
+:Description: Expired garbage collection items.
+:Type: Container
+
+``object``
+
+:Description: A container garbage collection object information.
+:Type: Container
+:Parent: ``garbage``
+
+``name``
+
+:Description: The name of the object.
+:Type: String
+:Parent: ``object``
+
+``expired``
+
+:Description: The date at which the object expired.
+:Type: String
+:Parent: ``object``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Manually Processes Garbage Collection Items
+===========================================
+
+List objects scheduled for garbage collection.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/garbage?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If expired garbage collection items exist, a list of removed objects
+will be returned.
+
+``garbage``
+
+:Description: Expired garbage collection items.
+:Type: Container
+
+``object``
+
+:Description: A container garbage collection object information.
+:Type: Container
+:Parent: ``garbage``
+
+``name``
+
+:Description: The name of the object.
+:Type: String
+:Parent: ``object``
+
+``expired``
+
+:Description: The date at which the object expired.
+:Type: String
+:Parent: ``object``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Show Log Objects
+================
+
+Show log objects
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/log?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``object``
+
+:Description: The log object to return.
+:Type: String:
+:Example: ``2012-10-11-09-4165.2-foo_bucket``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If no object is specified, returns the full list of log objects.
+
+``log-objects``
+
+:Description: A list of log objects.
+:Type: Container
+
+``object``
+
+:Description: The name of the log object.
+:Type: String
+
+``log``
+
+:Description: The contents of the log object.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Standard Error Responses
+========================
+
+``AccessDenied``
+
+:Description: Access denied.
+:Code: 403 Forbidden
+
+``InternalError``
+
+:Description: Internal server error.
+:Code: 500 Internal Server Error
+
+``NoSuchUser``
+
+:Description: User does not exist.
+:Code: 404 Not Found
+
+``NoSuchBucket``
+
+:Description: Bucket does not exist.
+:Code: 404 Not Found
+
+``NoSuchKey``
+
+:Description: No such access key.
+:Code: 404 Not Found
diff --git a/doc/dev/radosgw/bucket_index.rst b/doc/dev/radosgw/bucket_index.rst
new file mode 100644
index 000000000..6764641e0
--- /dev/null
+++ b/doc/dev/radosgw/bucket_index.rst
@@ -0,0 +1,75 @@
+==================
+Rados Bucket Index
+==================
+
+Buckets in RGW store their list of objects in a bucket index. Each index entry stores just enough metadata (size, etag, mtime, etc.) to serve API requests to list objects. These APIs are `ListObjectsV2`_ and `ListObjectVersions`_ in S3, and `GET Container`_ in Swift.
+
+.. note:: Buckets can be created as 'indexless'. Such buckets have no index, and cannot be listed.
+
+---------------------
+Consistency Guarantee
+---------------------
+
+RGW guarantees read-after-write consistency on object operations. This means that once a client receives a successful response to a write request, then the effects of that write must be visible to subsequent read requests.
+
+For example: if an S3 client sends a PutObject request to overwrite an existing object, followed by a GetObject request to read it back, RGW must not return the previous object's contents. It must either respond with the new object's contents, or with the result of a later object write or delete.
+
+This consistency guarantee applies to all object write requests (PutObject, DeleteObject, PutObjectAcl, etc) and all object read requests (HeadObject, GetObject, ListObjectsV2, etc).
+
+------------------
+Rados Object Model
+------------------
+
+S3/Swift objects, or 'API objects', are stored as rados objects in the rgw.buckets.data pool. Each API object is comprised of a head object and zero or more tail objects. Bucket index objects are stored in the rgw.buckets.index pool.
+
+When writing an object, its head object is written last. This acts as an atomic 'commit' to make it visible to read requests.
+
+-----------------------
+Sharding and Resharding
+-----------------------
+
+For a given bucket, the index may be split into several rados objects, called bucket index shards. In RADOS, multiple writes to the same object cannot run in parallel. By spreading the index over more rados objects, we increase its write parallelism. For a given object upload, the corresponding bucket index shard is selected based on a hash of the object's name.
+
+The default shard count for new buckets is 11, but can be overridden in the zonegroup's ``bucket_index_max_shards`` or ceph.conf's ``rgw_override_bucket_index_max_shards``. As the number of objects in a bucket grows, its index shard count will also increase as a result of dynamic resharding.
+
+Information about the bucket's index object layout is stored in ``RGWBucketInfo`` as ``struct rgw::BucketLayout`` from ``src/rgw/rgw_bucket_layout.h``. The resharding logic is in ``src/rgw/rgw_reshard.cc``.
+
+-----------------
+Index Transaction
+-----------------
+
+To keep the bucket index consistent, all object writes or deletes must also update the index accordingly. Because the head objects are stored in different rados objects than the bucket indices, we can't update both atomically with a single rados operation. In order to satisfy the `Consistency Guarantee`_ for listing operations, we have to coordinate these two object writes using a three-step bucket index transaction:
+
+#. Prepare a transaction on its bucket index object.
+#. Write or delete the head object.
+#. Commit the transaction on the bucket index object (or cancel the transaction if step 2 fails).
+
+Object writes and deletes may race with each other, so a given object may have more than one prepared transaction at a time. RGW considers an object entry to be 'pending' if there are any outstanding transactions, or 'completed' otherwise.
+
+This transaction is implemented in ``src/rgw/rgw_rados.cc`` as ``RGWRados::Object::Write::write_meta()`` for object writes, and ``RGWRados::Object::Delete::delete_obj()`` for object deletes. The bucket index operations are implemented in ``src/cls/rgw/cls_rgw.cc`` as ``rgw_bucket_prepare_op()`` and ``rgw_bucket_complete_op()``.
+
+-------
+Listing
+-------
+
+When listing objects, RGW will read all entries (pending and completed) from the bucket index. For any pending entries, it must check whether the head object exists before including that entry in the final listing.
+
+If an RGW crashes in the middle of an `Index Transaction`_, an index entry may get stuck in this 'pending' state. When bucket listing encounters these pending entries, it also sends information from the head object back to the bucket index so it can update the entry and resolve its stale transactions. This message is called 'dir suggest', because the bucket index treats it as a hint or suggestion.
+
+Bucket listing is implemented in ``src/rgw/rgw_rados.cc`` as ``RGWRados::Bucket::List::list_objects_ordered()`` and ``RGWRados::Bucket::List::list_objects_unordered()``. ``RGWRados::check_disk_state()`` is the part that reads the head object and encodes suggested changes. The corresponding bucket index operations are implemented in ``src/cls/rgw/cls_rgw.cc`` as ``rgw_bucket_list()`` and ``rgw_dir_suggest_changes()``.
+
+--------------------
+S3 Object Versioning
+--------------------
+
+For versioned buckets, the bucket index contains an entry for each object version and delete marker. In addition to sorting index entries by object name, it also has to sort object versions of the same name from newest to oldest.
+
+RGW stores a head object in the rgw.buckets.data pool for each object version. This rados object's oid is a combination of the object name and its version id.
+
+In S3, a GET/HEAD request for an object name will give you that object's "current" version. To support this, RGW stores an extra 'object logical head' (olh) object whose oid includes the object name only, that acts as an indirection to the head object of its current version. This indirection logic is implemented in ``src/rgw/rgw_rados.cc`` as ``RGWRados::follow_olh()``.
+
+To maintain the consistency between this olh object and the bucket index, the index keeps a separate 'olh' entry for each object name. This entry stores a log of all writes/deletes to its versions. In ``src/rgw/rgw_rados.cc``, ``RGWRados::apply_olh_log()`` replays this log to guarantee that this olh object converges on the same "current" version as the bucket index.
+
+.. _ListObjectsV2: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html
+.. _ListObjectVersions: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html
+.. _GET Container: https://docs.openstack.org/api-ref/object-store/?expanded=show-container-details-and-list-objects-detail#show-container-details-and-list-objects
diff --git a/doc/dev/radosgw/index.rst b/doc/dev/radosgw/index.rst
new file mode 100644
index 000000000..7cd4b64ab
--- /dev/null
+++ b/doc/dev/radosgw/index.rst
@@ -0,0 +1,14 @@
+=======================================
+ RADOS Gateway developer documentation
+=======================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1   
+   
+   
+   usage
+   Admin Ops Nonimplemented <admin/adminops_nonimplemented>
+   s3_compliance
+   bucket_index
diff --git a/doc/dev/radosgw/s3_compliance.rst b/doc/dev/radosgw/s3_compliance.rst
new file mode 100644
index 000000000..fc5526b4e
--- /dev/null
+++ b/doc/dev/radosgw/s3_compliance.rst
@@ -0,0 +1,310 @@
+===============================
+Rados Gateway S3 API Compliance
+===============================
+
+.. warning::
+	This document is a draft, it might not be accurate
+
+----------------------
+Naming code reference
+----------------------
+
+Here comes a BNF definition on how to name a feature in the code for referencing purpose : ::
+
+    name ::= request_type "_" ( header | operation ) ( "_" header_option )?
+    
+    request_type ::= "req" | "res"
+    
+    header ::= string
+    
+    operation ::= method resource
+    
+    method ::= "GET" | "PUT" | "POST" | "DELETE" | "OPTIONS" | "HEAD"
+    
+    resource ::= string
+    
+    header_option ::= string
+
+----------------------
+Common Request Headers
+----------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html
+
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Header               | Supported? | Code Links                                                                                              | Tests links |
++======================+============+=========================================================================================================+=============+
+| Authorization        | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1962 |             |
+|                      |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L2051 |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-Length       | Yes        |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-Type         | Yes        |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-MD5          | Yes        | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1249      |             |
+|                      |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1306      |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Date                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_auth_s3.cc#L164  |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Expect               | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest.cc#L1227    |             |
+|                      |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L802  |             |
+|                      |            | https://github.com/ceph/ceph/blob/76040d90f7eb9f9921a3b8dcd0f821ac2cd9c492/src/rgw/rgw_main.cc#L372     |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Host                 | ?          |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-date           | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_auth_s3.cc#L169  |             |
+|                      |            | should take precedence over DATE as mentioned here ->                                                   |             |
+|                      |            | http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html                            |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-security-token | No         |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+-----------------------
+Common Response Headers
+-----------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Header              | Supported? | Code Links                                                                                              | Tests links |
++=====================+============+=========================================================================================================+=============+
+| Content-Length      | Yes        |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Connection          | ?          |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Date                | ?          |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| ETag                | Yes        | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1312      |             |
+|                     |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1436      |             |
+|                     |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2222      |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L118  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L268  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L516  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1336 |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1486 |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1548 |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Server              | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-delete-marker | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-id-2          | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-request-id    | Yes        | https://github.com/ceph/ceph/commit/b711e3124f8f73c17ebd19b38807a1b77f201e44                            |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-version-id    | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+-------------------------
+Operations on the Service
+-------------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceOps.html
+
++------+-----------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Type | Operation | Supported? | Code links                                                                                              | Tests links |
++======+===========+============+=========================================================================================================+=============+
+| GET  | Service   | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L2094 |             |
+|      |           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1676 |             |
+|      |           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L185  |             |
++------+-----------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+---------------------
+Operations on Buckets
+---------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketOps.html
+
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| Type   | Operation              | Supported? | Code links                                                                                                 | Tests links |
++========+========================+============+============================================================================================================+=============+
+| DELETE | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1728    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L250 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L212 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L856      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L513      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L286      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L461     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1731    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1916         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket policy          | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket tagging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1676    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L185     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket acl             | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1697    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1728         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1344    |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1698    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1845         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/76040d90f7eb9f9921a3b8dcd0f821ac2cd9c492/src/rgw/rgw_main.cc#L345        |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket location        | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket policy          | ?          | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L232 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L58  |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket logging         | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1695    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L287     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket notification    | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket tagging         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket Object versions | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket requestPayment  | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket versioning      | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | List Multipart uploads | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1701    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest.cc#L877        |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2355         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2363         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| HEAD   | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1713    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1689    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L826          |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L834          |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1725    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L382     |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L437     |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L901          |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L945          |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket acl             | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1721    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1354    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1373    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1739         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1753         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1723    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1398    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1858         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1866         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket policy          | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket logging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket notification    | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket tagging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket requestPayment  | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket versioning      | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+
+---------------------
+Operations on Objects
+---------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
+
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Type    | Operation                 | Supported? | Code links                                                                                              | Tests links |
++=========+===========================+============+=========================================================================================================+=============+
+| DELETE  | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1796 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1516      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1524      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| DELETE  | Multiple objects          | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1739 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1616 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1626 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1641 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1667 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1516      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1524      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1767 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L71   |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L397       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L424       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L497       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L562       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L626       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L641       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L706       |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object acl                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object torrent            | No         |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| HEAD    | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1777 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L71   |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L397       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L424       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L497       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L562       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L626       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L641       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L706       |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| OPTIONS | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1814 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1418 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1951      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1968      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1993      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| POST    | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1742 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L631  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L694  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L700  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L707  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L759  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L771  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L781  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L795  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L929  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1037 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1059 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1134 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1344      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1360      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1365      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| POST    | Object restore            | ?          |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L481  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L493  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L507  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1786 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1119      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1217      |             |              
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1222      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object acl                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object copy               | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Initiate multipart upload | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Upload Part               | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Upload Part copy          | ?          |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Complete multipart upload | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Abort multipart upload    | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | List parts                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
diff --git a/doc/dev/radosgw/usage.rst b/doc/dev/radosgw/usage.rst
new file mode 100644
index 000000000..6c856fc7f
--- /dev/null
+++ b/doc/dev/radosgw/usage.rst
@@ -0,0 +1,84 @@
+============================
+Usage Design Overview
+============================
+
+
+
+
+Testing
+-------
+
+The current usage testing does the following:
+
+Following these operations:
+
+ - Create a few buckets
+ - Remove buckets
+ - Create a bucket
+ - Put object
+ - Remove object
+
+Test:
+
+1. Verify that 'usage show' with delete_obj category isn't empty after no more than 45 seconds (wait to flush)
+2. Check the following
+
+  - 'usage show'
+
+    - does not error out
+    - num of entries > 0
+    - num of summary entries > 0
+    - for every entry in categories check successful_ops > 0
+    - check that correct uid in the user summary
+
+
+  - 'usage show' with specified uid (--uid=<uid>')
+
+    - num of entries > 0
+    - num of summary entries > 0
+    - for every entry in categories check successful_ops > 0
+    - check that correct uid in the user summary
+
+  - 'usage show' with specified uid and specified categories (create_bucket,
+    put_obj, delete_obj, delete_bucket)
+
+    - for each category:
+      - does not error out
+      - num of entries > 0
+      - user in user summary is correct user
+      - length of categories entries under user summary is exactly 1
+      - name of category under user summary is correct name
+      - successful ops for the category > 0
+
+  - 'usage trim' with specified uid
+    - does not error
+    - check following 'usage show' shows complete usage info cleared for user
+
+
+Additional required testing:
+
+ - test multiple users
+
+   Do the same as in (2), with multiple users being set up.
+
+ - test with multiple buckets (> 1000 * factor, e.g., 2000)
+
+   Create multiple buckets, put objects in each. Account the number written data and verify
+   that usage reports show the expected number (up to a certain delta).
+
+ - verify usage show with a date/time range
+
+   Take timestamp of the beginning of the test, and the end of the test. Round timestamps to the
+   nearest hour (downward from start of test, upward from the end of test). List data starting
+   at end-time, make sure that no data is being shown. List data ending at start-time, make sure
+   that no data is shown. List data beginning at start-time, make sure that correct data is
+   displayed. List data ending end end-time, make sure that correct data is displayed. List
+   data beginning in begin-time, ending in end-time, make sure that correct data is displayed.
+
+ - verify usage trim with a date/time range
+
+   Take timestamp of the beginning of the test, and the end of the test. Round timestamps to the
+   nearest hour (downward from start of test, upward from the end of test). Trim data starting
+   at end-time, make sure that no data has been trimmed. Trim data ending at start-time, make sure
+   that no data has been trimmed.  Trim data beginning in begin-time, ending in end-time, make sure
+   that all data has been trimmed.
diff --git a/doc/dev/rbd-diff.rst b/doc/dev/rbd-diff.rst
new file mode 100644
index 000000000..083c13165
--- /dev/null
+++ b/doc/dev/rbd-diff.rst
@@ -0,0 +1,146 @@
+RBD Incremental Backup
+======================
+
+This is a simple streaming file format for representing a diff between
+two snapshots (or a snapshot and the head) of an RBD image.
+
+Header
+~~~~~~
+
+"rbd diff v1\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+From snap
+---------
+
+- u8: 'f'
+- le32: snap name length
+- snap name
+
+To snap
+-------
+
+- u8: 't'
+- le32: snap name length
+- snap name
+
+Size
+----
+
+- u8: 's'
+- le64: (ending) image size
+
+Data Records
+~~~~~~~~~~~~
+
+These records come in the second part of the sequence.
+
+Updated data
+------------
+
+- u8: 'w'
+- le64: offset
+- le64: length
+- length bytes of actual data
+
+Zero data
+---------
+
+- u8: 'z'
+- le64: offset
+- le64: length
+
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'e'
+
+
+Header
+~~~~~~
+
+"rbd diff v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+From snap
+---------
+
+- u8: 'f'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+To snap
+-------
+
+- u8: 't'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+Size
+----
+
+- u8: 's'
+- le64: length of appending data (8)
+- le64: (ending) image size
+
+Data Records
+~~~~~~~~~~~~
+
+These records come in the second part of the sequence.
+
+Updated data
+------------
+
+- u8: 'w'
+- le64: length of appending data (8 + 8 + length)
+- le64: offset
+- le64: length
+- length bytes of actual data
+
+Zero data
+---------
+
+- u8: 'z'
+- le64: length of appending data (8 + 8)
+- le64: offset
+- le64: length
+
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'e'
diff --git a/doc/dev/rbd-export.rst b/doc/dev/rbd-export.rst
new file mode 100644
index 000000000..2edb637f6
--- /dev/null
+++ b/doc/dev/rbd-export.rst
@@ -0,0 +1,104 @@
+RBD Export & Import
+===================
+
+This is a file format of an RBD image or snapshot. It's a sparse format
+for the full image. There are three recording sections in the file.
+
+(1) Header.
+(2) Metadata.
+(3) Diffs.
+
+Header
+~~~~~~
+
+"rbd image v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+Image order
+-----------
+
+- u8: 'O'
+- le64: length of appending data (8)
+- le64: image order
+
+Image format
+------------
+
+- u8: 'F'
+- le64: length of appending data (8)
+- le64: image format
+
+Image Features
+--------------
+
+- u8: 'T'
+- le64: length of appending data (8)
+- le64: image features
+
+Image Stripe unit
+-----------------
+
+- u8: 'U'
+- le64: length of appending data (8)
+- le64: image striping unit
+
+Image Stripe count
+------------------
+
+- u8: 'C'
+- le64: length of appending data (8)
+- le64: image striping count
+
+ImageMeta Key and Value
+-----------------------
+
+- u8: 'M'
+- le64: length of appending data (length of key + length of value + 4 * 2)
+- string: image-meta key
+- string: image-meta value
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'E'
+
+
+Diffs records
+~~~~~~~~~~~~~
+
+Record the all snapshots and the HEAD in this section. 
+
+Snap Protection status
+----------------------
+
+Record the snapshot's protection status if `--export-format=2`.
+- u8: 'p'
+- le64: length of appending data (8)
+- u8: snap protection status (0 for false, 1 for true)
+
+Others
+------
+
+- le64: number of diffs
+- Diffs ...
+
+Detail please refer to rbd-diff.rst
diff --git a/doc/dev/rbd-layering.rst b/doc/dev/rbd-layering.rst
new file mode 100644
index 000000000..e6e224ce4
--- /dev/null
+++ b/doc/dev/rbd-layering.rst
@@ -0,0 +1,281 @@
+============
+RBD Layering
+============
+
+RBD layering refers to the creation of copy-on-write clones of block
+devices. This allows for fast image creation, for example to clone a
+golden master image of a virtual machine into a new instance. To
+simplify the semantics, you can only create a clone of a snapshot -
+snapshots are always read-only, so the rest of the image is
+unaffected, and there's no possibility of writing to them
+accidentally.
+
+From a user's perspective, a clone is just like any other rbd image.
+You can take snapshots of them, read/write them, resize them, etc.
+There are no restrictions on clones from a user's viewpoint.
+
+Note: the terms `child` and `parent` below mean an rbd image created
+by cloning, and the rbd image snapshot a child was cloned from.
+
+Command line interface
+----------------------
+
+Before cloning a snapshot, you must mark it as protected, to prevent
+it from being deleted while child images refer to it:
+::
+
+    $ rbd snap protect pool/image@snap
+
+Then you can perform the clone:
+::
+
+    $ rbd clone [--parent] pool/parent@snap [--image] pool2/child1
+
+You can create a clone with different object sizes from the parent:
+::
+
+    $ rbd clone --order 25 pool/parent@snap pool2/child2
+
+To delete the parent, you must first mark it unprotected, which checks
+that there are no children left:
+::
+
+    $ rbd snap unprotect pool/image@snap
+    Cannot unprotect: Still in use by pool2/image2
+    $ rbd children pool/image@snap
+    pool2/child1
+    pool2/child2
+    $ rbd flatten pool2/child1
+    $ rbd rm pool2/child2
+    $ rbd snap rm pool/image@snap
+    Cannot remove a protected snapshot: pool/image@snap
+    $ rbd snap unprotect pool/image@snap
+
+Then the snapshot can be deleted like normal:
+::
+
+    $ rbd snap rm pool/image@snap
+
+Implementation
+--------------
+
+Data Flow
+^^^^^^^^^
+
+In the initial implementation, called 'trivial layering', there will
+be no tracking of which objects exist in a clone. A read that hits a
+non-existent object will attempt to read from the parent snapshot, and
+this will continue recursively until an object exists or an image with
+no parent is found. This is done through the normal read path from
+the parent, so differing object sizes between parents and children
+do not matter.
+
+Before a write to an object is performed, the object is checked for
+existence. If it doesn't exist, a copy-up operation is performed,
+which means reading the relevant range of data from the parent
+snapshot and writing it (plus the original write) to the child
+image. To prevent races with multiple writes trying to copy-up the
+same object, this copy-up operation will include an atomic create. If
+the atomic create fails, the original write is done instead. This
+copy-up operation is implemented as a class method so that extra
+metadata can be stored by it in the future. In trivial layering, the
+copy-up operation copies the entire range needed to the child object
+(that is, the full size of the child object). A future optimization
+could make this copy-up more fine-grained.
+
+Another future optimization could be storing a bitmap of which objects
+actually exist in a child. This would obviate the check for existence
+before each write, and let reads go directly to the parent if needed.
+
+These optimizations are discussed in:
+
+http://marc.info/?l=ceph-devel&m=129867273303846
+
+Parent/Child relationships
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Children store a reference to their parent in their header, as a tuple
+of (pool id, image id, snapshot id). This is enough information to
+open the parent and read from it.
+
+In addition to knowing which parent a given image has, we want to be
+able to tell if a protected snapshot still has children. This is
+accomplished with a new per-pool object, `rbd_children`, which maps
+(parent pool id, parent image id, parent snapshot id) to a list of
+child image ids. This is stored in the same pool as the child image
+because the client creating a clone already has read/write access to
+everything in this pool, but may not have write access to the parent's
+pool. This lets a client with read-only access to one pool clone a
+snapshot from that pool into a pool they have full access to. It
+increases the cost of unprotecting an image, since this needs to check
+for children in every pool, but this is a rare operation. It would
+likely only be done before removing old images, which is already much
+more expensive because it involves deleting every data object in the
+image.
+
+Protection
+^^^^^^^^^^
+
+Internally, protection_state is a field in the header object that
+can be in three states. "protected", "unprotected", and
+"unprotecting". The first two are set as the result of "rbd
+protect/unprotect". The "unprotecting" state is set while the "rbd
+unprotect" command checks for any child images. Only snapshots in the
+"protected" state may be cloned, so the "unprotected" state prevents
+a race like:
+
+1. A: walk through all pools, look for clones, find none
+2. B: create a clone
+3. A: unprotect parent
+4. A: rbd snap rm pool/parent@snap
+
+Resizing
+^^^^^^^^
+
+Resizing an rbd image is like truncating a sparse file. New space is
+treated as zeroes, and shrinking an rbd image deletes the contents
+beyond the old bounds. This means that if you have a 10G image full of
+data, and you resize it down to 5G and then up to 10G again, the last
+5G is treated as zeroes (and any objects that held that data were
+removed when the image was shrunk).
+
+Layering complicates this because the absence of an object no longer
+implies it should be treated as zeroes - if the object is part of a
+clone, it may mean that some data needs to be read from the parent.
+
+To preserve the resizing behavior for clones, we need to keep track of
+which objects could be stored in the parent. We can track this as the
+amount of overlap the child has with the parent, since resizing only
+changes the end of an image. When a child is created, its overlap
+is the size of the parent snapshot. On each subsequent resize, the
+overlap is `min(overlap, new_size)`. That is, shrinking the image
+may shrinks the overlap, but increasing the image's size does not
+change the overlap.
+
+Objects that do not exist past the overlap are treated as zeroes.
+Objects that do not exist before that point fall back to reading
+from the parent.
+
+Since this overlap changes over time, we store it as part of the
+metadata for a snapshot as well.
+
+Renaming
+^^^^^^^^
+
+Currently the rbd header object (that stores all the metadata about an
+image) is named after the name of the image. This makes renaming
+disrupt clients who have the image open (such as children reading from
+a parent). To avoid this, we can name the header object by the
+id of the image, which does not change. That is, the name of the
+header object could be `rbd_header.$id`, where $id is a unique id for
+the image in the pool.
+
+When a client opens an image, all it knows is the name. There is
+already a per-pool `rbd_directory` object that maps image names to
+ids, but if we relied on it to get the id, we could not open any
+images in that pool if that single object was unavailable. To avoid
+this dependency, we can store the id of an image in an object called
+`rbd_id.$image_name`, where $image_name is the name of the image. The
+per-pool `rbd_directory` object is still useful for listing all images
+in a pool, however.
+
+Header changes
+--------------
+
+The header needs a few new fields:
+
+* int64_t parent_pool_id
+* string parent_image_id
+* uint64_t parent_snap_id
+* uint64_t overlap (how much of the image may be referring to the parent)
+
+These are stored in a "parent" key, which is only present if the image
+has a parent.
+
+cls_rbd
+^^^^^^^
+
+Some new methods are needed:
+::
+
+    /***************** methods on the rbd header *********************/
+    /**
+     * Sets the parent and overlap keys.
+     * Fails if any of these keys exist, since the image already
+     * had a parent.
+     */
+    set_parent(uint64_t pool_id, string image_id, uint64_t snap_id)
+
+    /**
+     * returns the parent pool id, image id, snap id, and overlap, or -ENOENT
+     * if parent_pool_id does not exist or is -1
+     */
+    get_parent(uint64_t snapid)
+
+    /**
+     * Removes the parent key
+     */
+    remove_parent() // after all parent data is copied to the child
+
+    /*************** methods on the rbd_children object *****************/
+
+    add_child(uint64_t parent_pool_id, string parent_image_id,
+              uint64_t parent_snap_id, string image_id);
+    remove_child(uint64_t parent_pool_id, string parent_image_id,
+                 uint64_t parent_snap_id, string image_id);
+    /**
+     * List ids of a given parent
+     */
+    get_children(uint64_t parent_pool_id, string parent_image_id,
+                 uint64_t parent_snap_id, uint64_t max_return,
+                 string start);
+    /**
+     * list parent
+     */
+    get_parents(uint64_t max_return, uint64_t start_pool_id,
+                string start_image_id, string start_snap_id);
+
+
+    /************ methods on the rbd_id.$image_name object **************/
+
+    set_id(string id)
+    get_id()
+
+    /************** methods on the rbd_directory object *****************/
+
+    dir_get_id(string name);
+    dir_get_name(string id);
+    dir_list(string start_after, uint64_t max_return);
+    dir_add_image(string name, string id);
+    dir_remove_image(string name, string id);
+    dir_rename_image(string src, string dest, string id);
+
+Two existing methods will change if the image supports
+layering:
+::
+
+    snapshot_add - stores current overlap and has_parent with
+                   other snapshot metadata (images that don't have
+                   layering enabled aren't affected)
+
+    set_size     - will adjust the parent overlap down as needed.
+
+librbd
+^^^^^^
+
+Opening a child image opens its parent (and this will continue
+recursively as needed). This means that an ImageCtx will contain a
+pointer to the parent image context. Differing object sizes won't
+matter, since reading from the parent will go through the parent
+image context.
+
+Discard will need to change for layered images so that it only
+truncates objects, and does not remove them. If we removed objects, we
+could not tell if we needed to read them from the parent.
+
+A new clone method will be added, which takes the same arguments as
+create except size (size of the parent image is used).
+
+Instead of expanding the rbd_info struct, we will break the metadata
+retrieval into several API calls.  Right now, the only users of
+rbd_stat() other than 'rbd info' only use it to retrieve image size.
diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
new file mode 100644
index 000000000..6299c3783
--- /dev/null
+++ b/doc/dev/release-checklists.rst
@@ -0,0 +1,142 @@
+==================
+Release checklists
+==================
+
+Dev Kickoff
+===========
+
+These steps should be taken when starting a new major release, just after
+the previous release has been tagged (vX.2.0) and that tag has been merged
+back into master.
+
+X is the release we are just starting development on.  X-1 is the one
+that was just released (X-1).2.0.
+
+Versions and tags
+-----------------
+
+- [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
+- [x] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
+- [x] Update src/ceph_release with the new release name, number, and type ('dev')
+- [x] Initial tag vX.0.0 (so that we can distinguish from (and sort
+      after) the backported (X-1).2.Z versions.
+
+### Notes on tagging
+* Tags must be annonated as CMake determines `CEPH_GIT_NICE_VER` by
+calling `git describe --always`.
+* vX.0.0 are special ones in the sense they are pushed manually (unlike v.X.2.n
+which are handled by Jenkins).
+* vX.0.0 should point to a commit before the first one in a kickoff branch.
+
+Define release names and constants
+----------------------------------
+
+Make sure X (and, ideally, X+1) is defined:
+
+- [x] src/common/ceph_releases.h (`ceph_release_t`)
+- [x] src/common/ceph_strings.cc (`ceph_release_name()`)
+- [x] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
+- [x] src/include/rbd/librbd.h (`LIBRBD_VER_MINOR` to X)
+- [x] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
+- [x] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
+
+Scripts
+~~~~~~~
+
+- [x] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
+- [x] src/script/ceph-release-notes (X-1)
+- [ ] ceph-build.git scripts/build_utils.sh `release_from_version()`
+
+Misc
+~~~~
+- [x] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
+- [x] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
+- [x] update src/cephadm/cephadm (`DEFAULT_IMAGE_RELEASE` to X)
+
+Docs
+~~~~
+
+- [x] Remove ``doc/releases/*.rst``. This should leave behind ``doc/releases/releases.yml`` which is used for doc building purposes.
+- [x] Cherry-pick 8cf9ad62949516666ad0f2c0bb7726ef68e4d666 ("doc: add releases links to toc"). There will be trivial conflicts.
+- [x] Add redirect for new major release at `RTD <https://readthedocs.org/dashboard/ceph/redirects/>`_.
+
+Feature bits
+------------
+
+- [x] ensure that `SERVER_X` is defined
+- [x] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
+- [ ] look for features that (1) were present in X-2 and (2) have no
+  client dependency and mark them `DEPRECATED` as of X.
+
+
+Compatsets
+----------
+
+- [x] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
+- [x] mon/Monitor.cc (include in `get_supported_features()`)
+- [x] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
+- [x] mon/Monitor.cc (`calc_quorum_requirements()`)
+- [x] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
+
+Mon
+---
+
+- [x] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
+- [x] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
+- [x] mon/MgrMonitor.cc adjust `always_on_modules`
+- [x] common/options/global.yaml.in define `mon_debug_no_require_X`
+- [x] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
+- [x] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
+- [x] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
+- [x] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
+- [x] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
+- [x] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
+
+
+Code cleanup
+------------
+
+- [ ] search code for "after X-1" or "X" for conditional checks
+- [ ] search code for X-2 and X-3 (`CEPH_FEATURE_SERVER_*` and
+  `ceph_release_t::*`)
+- [ ] search code for `require_osd_release`
+- [ ] search code for `min_mon_release`
+
+QA suite
+--------
+
+- [x] create qa/suites/upgrade/(X-1)-x
+- [x] remove qa/suites/upgrade/(X-3)-x-*
+- [x] create qa/releases/X.yaml
+- [x] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
+
+
+ceph-build
+----------
+In the `ceph/ceph-build.git` repo:
+
+- [x] add the version -> X mapping (`release_from_version()` in `scripts/build_utils.sh`)
+- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/build_osc`)
+- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/setup_osc`)
+- [x] grep for previous release and add relevant build targets (e.g. for reef https://github.com/ceph/ceph-build/pull/2076 and https://github.com/ceph/ceph-build/pull/2119)
+
+
+After dev freeze
+================
+
+- [x] add release name to redmine (using https://tracker.ceph.com/custom_fields/16/edit)
+- [x] add release name to .github/milestone.yml for github actions to automatically add milestone to backports
+
+First release candidate
+=======================
+
+- [x] src/ceph_release: change type to `rc`
+- [x] opt-in to all telemetry channels, generate telemetry reports, and verify no sensitive details (like pools names) are collected
+
+
+First stable release
+====================
+
+- [x] src/ceph_release: change type `stable`
+- [ ] generate new object corpus for encoding/decoding tests - see :doc:`corpus`
+- [ ] src/cephadm/cephadm: update `LATEST_STABLE_RELEASE`
diff --git a/doc/dev/release-process.rst b/doc/dev/release-process.rst
new file mode 100644
index 000000000..3750759b8
--- /dev/null
+++ b/doc/dev/release-process.rst
@@ -0,0 +1,225 @@
+======================
+  Ceph Release Process
+======================
+
+Prerequisites
+=============
+
+Signing Machine
+---------------
+The signing machine is a virtual machine in the `Sepia lab
+<https://wiki.sepia.ceph.com/doku.php?id=start>`_. SSH access to the signing
+machine is limited to the usual Infrastructure Admins along with a few other
+component leads (e.g., nfs-ganesha, ceph-iscsi).
+
+The ``ubuntu`` user on the machine has some `build scripts <https://github.com/ceph/ceph-build/tree/main/scripts>`_ that help with pulling, pushing, and signing packages.
+
+The GPG signing key permanently lives on a `Nitrokey Pro <https://shop.nitrokey.com/shop/product/nkpr2-nitrokey-pro-2-3>`_ and is passed through to the VM via RHV. This helps to ensure that the key cannot be exported or leave the datacenter in any way.
+
+New Major Releases
+------------------
+For each new major (alphabetical) release, you must create one ``ceph-release`` RPM for each RPM repo (e.g., one for el8 and one for el9). `chacra <https://github.com/ceph/chacra>`_ is a python service we use to store DEB and RPM repos. The chacra repos are configured to include this ceph-release RPM, but it must be built separately. You must make sure that chacra is properly configured to include this RPM for each particular release.
+
+1. Update chacra so it is aware of the new Ceph release.  See `this PR <https://github.com/ceph/chacra/pull/219>`_ for an example.
+2. Redeploy chacra (e.g., ``ansible-playbook chacra.ceph.com.yml``)
+3. Run https://jenkins.ceph.com/view/all/job/ceph-release-rpm/
+
+Summarized build process
+========================
+
+1. QE finishes testing and finds a stopping point.  That commit is pushed to the ``$release-release`` branch in ceph.git (e.g., ``quincy-release``).  This allows work to continue in the working ``$release`` branch without having to freeze it during the release process.
+2. The Ceph Council approves and notifies the "Build Lead".
+3. The "Build Lead" starts the `Jenkins multijob <https://jenkins.ceph.com/view/all/job/ceph>`_, which triggers all builds.
+4. Packages are pushed to chacra.ceph.com.
+5. Packages are pulled from chacra.ceph.com to the Signer VM.
+6. Packages are signed.
+7. Packages are pushed to download.ceph.com.
+8. Release containers are built and pushed to quay.io.
+
+Hotfix Release Process Deviation
+--------------------------------
+
+A hotfix release has a couple differences.
+
+1. Check out the most recent tag. For example, if we're releasing a hotfix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
+2. ``git cherry-pick -x`` the necessary hotfix commits
+3. ``git push -f origin quincy-release``
+4. Notify the "Build Lead" to start the build.
+5. The "Build Lead" should set ``RELEASE_TYPE=HOTFIX`` instead of ``STABLE``.
+
+Security Release Process Deviation
+----------------------------------
+
+A security/CVE release is similar to a hotfix release with two differences:
+
+    1. The fix should be pushed to the `ceph-private <https://github.com/ceph/ceph-private>`_ repo instead of ceph.git (requires GitHub Admin Role).
+    2. The tags (e.g., v17.2.4) must be manually pushed to ceph.git by the "Build Lead."
+
+1. Check out the most recent tag. For example, if we're releasing a security fix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
+2. ``git cherry-pick -x`` the necessary security fix commits
+3. ``git remote add security git@github.com:ceph/ceph-private.git``
+4. ``git push -f security quincy-release``
+5. Notify the "Build Lead" to start the build.
+6. The "Build Lead" should set ``RELEASE_TYPE=SECURITY`` instead of ``STABLE``.
+7. Finally, the `ceph-tag <https://github.com/ceph/ceph-build/blob/main/ansible/roles/ceph-release/tasks/push.yml>`_ steps need to be manually run by the "Build Lead" as close to the Announcement time as possible::
+
+    # Example using quincy pretending 17.2.4 is the security release version
+    # Add the ceph-releases repo (also requires GitHub Admin Role). The `ceph-setup <https://jenkins.ceph.com/job/ceph-setup>`_ job will have already created and pushed the tag to ceph-releases.git.
+    git remote add releases git@github.com:ceph/ceph-releases.git
+    git fetch --all
+    # Check out the version commit
+    git checkout -f -B quincy-release releases/quincy-release
+    git push -f origin quincy-release
+    git push origin v17.2.4
+    # Now create a Pull Request of quincy-release targeting quincy to merge the version commit and security fixes back into the quincy branch
+
+1. Preparing the release branch
+===============================
+
+Once QE has determined a stopping point in the working (e.g., ``quincy``) branch, that commit should be pushed to the corresponding ``quincy-release`` branch.
+
+Notify the "Build Lead" that the release branch is ready.
+
+2. Starting the build
+=====================
+
+We'll use a stable/regular 15.2.17 release of Octopus as an example throughout this document.
+
+1. Browse to https://jenkins.ceph.com/view/all/job/ceph/build?delay=0sec
+2. Log in with GitHub OAuth
+3. Set the parameters as necessary::
+
+    BRANCH=octopus
+    TAG=checked
+    VERSION=15.2.17
+    RELEASE_TYPE=STABLE
+    ARCHS=x86_64 arm64
+
+4. Use https://docs.ceph.com/en/latest/start/os-recommendations/?highlight=debian#platforms to determine the ``DISTROS`` parameter.  For example,
+
+    +-------------------+-------------------------------------------+
+    | Release           | Distro Codemap                            |
+    +===================+===========================================+
+    | octopus (15.X.X)  | ``focal bionic centos7 centos8 buster``   |
+    +-------------------+-------------------------------------------+
+    | pacific (16.X.X)  | ``focal bionic centos8 buster bullseye``  |
+    +-------------------+-------------------------------------------+
+    | quincy (17.X.X)   | ``focal centos8 centos9 bullseye``        |
+    +-------------------+-------------------------------------------+
+
+5. Click ``Build``.
+
+3. Release Notes
+================
+
+Packages take hours to build. Use those hours to create the Release Notes and Announcements:
+
+1. ceph.git Release Notes (e.g., `v15.2.17's ceph.git (docs.ceph.com) PR <https://github.com/ceph/ceph/pull/47198>`_)
+2. ceph.io Release Notes (e.g., `v15.2.17's ceph.io.git (www.ceph.io) PR <https://github.com/ceph/ceph.io/pull/427>`_)
+3. E-mail announcement
+
+See `the Ceph Tracker wiki page that explains how to write the release notes <https://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_write_the_release_notes>`_. 
+
+4. Signing and Publishing the Build
+===================================
+
+#. Obtain the sha1 of the version commit from the `build job <https://jenkins.ceph.com/view/all/job/ceph>`_ or the ``sha1`` file created by the `ceph-setup <https://jenkins.ceph.com/job/ceph-setup/>`_ job.
+
+#. Download the packages from chacra.ceph.com to the signing virtual machine. These packages get downloaded to ``/opt/repos`` where the `Sepia Lab Long Running (Ceph) Cluster <https://wiki.sepia.ceph.com/doku.php?id=services:longrunningcluster>`_ is mounted.
+
+   .. prompt:: bash $
+
+      ssh ubuntu@signer.front.sepia.ceph.com
+      sync-pull ceph [pacific|quincy|etc] <sha1>
+ 
+   Example::
+    
+      $ sync-pull ceph octopus 8a82819d84cf884bd39c17e3236e0632ac146dc4
+      sync for: ceph octopus
+      ********************************************
+      Found the most packages (332) in ubuntu/bionic.
+      No JSON object could be decoded
+      No JSON object could be decoded
+      ubuntu@chacra.ceph.com:/opt/repos/ceph/octopus/8a82819d84cf884bd39c17e3236e0632ac146dc4/ubuntu/bionic/flavors/default/* /opt/repos/ceph/octopus-15.2.17/debian/jessie/
+      --------------------------------------------
+      receiving incremental file list
+      db/
+       db/checksums.db
+              180.22K 100%    2.23MB/s    0:00:00 (xfr#1, to-chk=463/467)
+      db/contents.cache.db
+              507.90K 100%    1.95MB/s    0:00:00 (xfr#2, to-chk=462/467)
+      db/packages.db
+      
+      etc...
+
+#. Sign the DEBs:
+
+   .. prompt:: bash
+
+      merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
+
+   Example::
+
+      $ merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
+      --> Starting path collection, looking for files to sign
+      --> 18 matching paths found
+      --> will sign with the following commands:
+      --> gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/bionic/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/focal/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      
+      etc...
+
+#. Sign the RPMs:
+
+   .. prompt:: bash
+
+      sign-rpms octopus
+
+   Example::
+   
+      $ sign-rpms octopus
+      Checking packages in: /opt/repos/ceph/octopus-15.2.17/centos/7
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/noarch/ceph-mgr-modules-core-15.2.17-0.el7.noarch.rpm
+   
+      etc...
+
+5. Publish the packages to download.ceph.com:
+
+   .. prompt:: bash $
+
+      sync-push octopus
+
+5. Build Containers
+===================
+
+Start the following two jobs:
+
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs/
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs-arm64/
+
+6. Announce the Release
+=======================
+
+Version Commit PR
+-----------------
+
+The `ceph-tag Jenkins job <https://jenkins.ceph.com/job/ceph-tag>`_ creates a Pull Request in ceph.git that targets the release branch.
+
+If this was a regular release (not a hotfix release or a security release), the only commit in that Pull Request should be the version commit.  For example, see `v15.2.17's version commit PR <https://github.com/ceph/ceph/pull/47520>`_. 
+
+Request a review and then merge the Pull Request.
+
+Announcing
+----------
+
+Publish the Release Notes on ceph.io before announcing the release by email, because the e-mail announcement references the ceph.io blog post.
diff --git a/doc/dev/seastore.rst b/doc/dev/seastore.rst
new file mode 100644
index 000000000..dd080092c
--- /dev/null
+++ b/doc/dev/seastore.rst
@@ -0,0 +1,323 @@
+==========
+ SeaStore
+==========
+
+Goals and Basics
+================
+
+* Target NVMe devices.  Not primarily concerned with pmem or HDD.
+* make use of SPDK for user-space driven IO
+* Use Seastar futures programming model to facilitate
+  run-to-completion and a sharded memory/processing model
+* Allow zero- (or minimal) data copying on read and write paths when
+  combined with a seastar-based messenger using DPDK
+
+Motivation and background
+-------------------------
+
+All flash devices are internally structured in terms of segments that
+can be written efficiently but must be erased in their entirety.  The
+NVMe device generally has limited knowledge about what data in a
+segment is still "live" (hasn't been logically discarded), making the
+inevitable garbage collection within the device inefficient.  We can
+design an on-disk layout that is friendly to GC at lower layers and
+drive garbage collection at higher layers.
+
+In principle a fine-grained discard could communicate our intent to
+the device, but in practice discard is poorly implemented in the
+device and intervening software layers.
+
+The basic idea is that all data will be stream out sequentially to
+large segments on the device.  In the SSD hardware, segments are
+likely to be on the order of 100's of MB to tens of GB.
+
+SeaStore's logical segments would ideally be perfectly aligned with
+the hardware segments.  In practice, it may be challenging to
+determine geometry and to sufficiently hint to the device that LBAs
+being written should be aligned to the underlying hardware.  In the
+worst case, we can structure our logical segments to correspond to
+e.g. 5x the physical segment size so that we have about ~20% of our
+data misaligned.
+
+When we reach some utilization threshold, we mix cleaning work in with
+the ongoing write workload in order to evacuate live data from
+previously written segments.  Once they are completely free we can
+discard the entire segment so that it can be erased and reclaimed by
+the device.
+
+The key is to mix a small bit of cleaning work with every write
+transaction to avoid spikes and variance in write latency.
+  
+Data layout basics
+------------------
+
+One or more cores/shards will be reading and writing to the device at
+once.  Each shard will have its own independent data it is operating
+on and stream to its own open segments.  Devices that support streams
+can be hinted accordingly so that data from different shards is not
+mixed on the underlying media.
+
+Persistent Memory
+-----------------
+
+As the initial sequential design above matures, we'll introduce
+persistent memory support for metadata and caching structures.
+
+Design
+======
+
+The design is based heavily on both f2fs and btrfs.  Each reactor
+manages its own root.  Prior to reusing a segment, we rewrite any live
+blocks to an open segment.
+
+Because we are only writing sequentially to open segments, we must
+“clean� one byte of an existing segment for every byte written at
+steady state.  Generally, we’ll need to reserve some portion of the
+usable capacity in order to ensure that write amplification remains
+acceptably low (20% for 2x? -- TODO: find prior work).  As a design
+choice, we want to avoid a background gc scheme as it tends to
+complicate estimating operation cost and tends to introduce
+non-deterministic latency behavior.  Thus, we want a set of structures
+that permits us to relocate blocks from existing segments inline with
+ongoing client IO.
+
+To that end, at a high level, we’ll maintain 2 basic metadata trees.
+First, we need a tree mapping ghobject_t->onode_t (onode_by_hobject).
+Second, we need a way to find live blocks within a segment and a way
+to decouple internal references from physical locations (lba_tree).
+
+Each onode contains xattrs directly as well as the top of the omap and
+extent trees (optimization: we ought to be able to fit small enough
+objects into the onode).
+
+Segment Layout
+--------------
+
+The backing storage is abstracted into a set of segments.  Each
+segment can be in one of 3 states: empty, open, closed.  The byte
+contents of a segment are a sequence of records.  A record is prefixed
+by a header (including length and checksums) and contains a sequence
+of deltas and/or blocks.  Each delta describes a logical mutation for
+some block.  Each included block is an aligned extent addressable by
+<segment_id_t, segment_offset_t>.  A transaction can be implemented by
+constructing a record combining deltas and updated blocks and writing
+it to an open segment.
+
+Note that segments will generally be large (something like >=256MB),
+so there will not typically be very many of them.
+
+record: [ header | delta | delta... | block | block ... ]
+segment: [ record ... ]
+
+See src/crimson/os/seastore/journal.h for Journal implementation
+See src/crimson/os/seastore/seastore_types.h for most seastore structures.
+
+Each shard will keep open N segments for writes
+
+- HDD: N is probably 1 on one shard
+- NVME/SSD: N is probably 2/shard, one for "journal" and one for
+  finished data records as their lifetimes are different.
+
+I think the exact number to keep open and how to partition writes
+among them will be a tuning question -- gc/layout should be flexible.
+Where practical, the goal is probably to partition blocks by expected
+lifetime so that a segment either has long lived or short lived
+blocks.
+
+The backing physical layer is exposed via a segment based interface.
+See src/crimson/os/seastore/segment_manager.h
+
+Journal and Atomicity
+---------------------
+
+One open segment is designated to be the journal.  A transaction is
+represented by an atomically written record.  A record will contain
+blocks written as part of the transaction as well as deltas which
+are logical mutations to existing physical extents.  Transaction deltas
+are always written to the journal.  If the transaction is associated
+with blocks written to other segments, final record with the deltas
+should be written only once the other blocks are persisted.  Crash
+recovery is done by finding the segment containing the beginning of
+the current journal, loading the root node, replaying the deltas, and
+loading blocks into the cache as needed.
+
+See src/crimson/os/seastore/journal.h
+
+Block Cache
+-----------
+
+Every block is in one of two states:
+
+- clean: may be in cache or not, reads may cause cache residence or
+  not
+- dirty: the current version of the record requires overlaying deltas
+  from the journal.  Must be fully present in the cache.
+
+Periodically, we need to trim the journal (else, we’d have to replay
+journal deltas from the beginning of time).  To do this, we need to
+create a checkpoint by rewriting the root blocks and all currently
+dirty blocks.  Note, we can do journal checkpoints relatively
+infrequently, and they needn’t block the write stream.
+
+Note, deltas may not be byte range modifications.  Consider a btree
+node structured with keys to the left and values to the right (common
+trick for improving point query/key scan performance).  Inserting a
+key/value into that node at the min would involve moving a bunch of
+bytes, which would be expensive (or verbose) to express purely as a
+sequence of byte operations.  As such, each delta indicates the type
+as well as the location of the corresponding extent.  Each block
+type can therefore implement CachedExtent::apply_delta as appropriate.
+
+See src/os/crimson/seastore/cached_extent.h.
+See src/os/crimson/seastore/cache.h.
+
+GC
+---
+
+Prior to reusing a segment, we must relocate all live blocks.  Because
+we only write sequentially to empty segments, for every byte we write
+to currently open segments, we need to clean a byte of an existing
+closed segment.  As a design choice, we’d like to avoid background
+work as it complicates estimating operation cost and has a tendency to
+create non-deterministic latency spikes.  Thus, under normal operation
+each seastore reactor will be inserting enough work to clean a segment
+at the same rate as incoming operations.
+
+In order to make this cheap for sparse segments, we need a way to
+positively identify dead blocks.  Thus, for every block written, an
+entry will be added to the lba tree with a pointer to the previous lba
+in the segment.  Any transaction that moves a block or modifies the
+reference set of an existing one will include deltas/blocks required
+to update the lba tree to update or remove the previous block
+allocation.  The gc state thus simply needs to maintain an iterator
+(of a sort) into the lba tree segment linked list for segment
+currently being cleaned and a pointer to the next record to be
+examined -- records not present in the allocation tree may still
+contain roots (like allocation tree blocks) and so the record metadata
+must be checked for a flag indicating root blocks.
+
+For each transaction, we evaluate a heuristic function of the
+currently available space and currently live space in order to
+determine whether we need to do cleaning work (could be simply a range
+of live/used space ratios).
+
+TODO: there is not yet a GC implementation
+
+Logical Layout
+==============
+
+Using the above block and delta semantics, we build two root level trees:
+- onode tree: maps hobject_t to onode_t
+- lba_tree: maps lba_t to lba_range_t
+
+Each of the above structures is comprised of blocks with mutations
+encoded in deltas.  Each node of the above trees maps onto a block.
+Each block is either physically addressed (root blocks and the
+lba_tree nodes) or is logically addressed (everything else).
+Physically addressed blocks are located by a paddr_t: <segment_id_t,
+segment_off_t> tuple and are marked as physically addressed in the
+record.  Logical blocks are addressed by laddr_t and require a lookup in
+the lba_tree to address.
+
+Because the cache/transaction machinery lives below the level of the
+lba tree, we can represent atomic mutations of the lba tree and other
+structures by simply including both in a transaction.
+
+LBAManager/BtreeLBAManager
+--------------------------
+
+Implementations of the LBAManager interface are responsible for managing
+the logical->physical mapping -- see crimson/os/seastore/lba_manager.h.
+
+The BtreeLBAManager implements this interface directly on top of
+Journal and SegmentManager using a wandering btree approach.
+
+Because SegmentManager does not let us predict the location of a
+committed record (a property of both SMR and Zone devices), references
+to blocks created within the same transaction will necessarily be
+*relative* addresses.  The BtreeLBAManager maintains an invariant by
+which the in-memory copy of any block will contain only absolute
+addresses when !is_pending() -- on_commit and complete_load fill in
+absolute addresses based on the actual block addr and on_delta_write
+does so based on the just committed record.  When is_pending(), if
+is_initial_pending references in memory are block_relative (because
+they will be written to the original block location) and
+record_relative otherwise (value will be written to delta).
+
+TransactionManager
+------------------
+
+The TransactionManager is responsible for presenting a unified
+interface on top of the Journal, SegmentManager, Cache, and
+LBAManager.  Users can allocate and mutate extents based on logical
+addresses with segment cleaning handled in the background.
+
+See crimson/os/seastore/transaction_manager.h
+
+Next Steps
+==========
+
+Journal
+-------
+
+- Support for scanning a segment to find physically addressed blocks
+- Add support for trimming the journal and releasing segments.
+
+Cache
+-----
+
+- Support for rewriting dirty blocks
+
+  - Need to add support to CachedExtent for finding/updating
+    dependent blocks
+  - Need to add support for adding dirty block writout to
+    try_construct_record
+
+LBAManager
+----------
+
+- Add support for pinning
+- Add segment -> laddr for use in GC
+- Support for locating remaining used blocks in segments
+
+GC
+---
+
+- Initial implementation
+- Support in BtreeLBAManager for tracking used blocks in segments
+- Heuristic for identifying segments to clean
+
+Other
+------
+
+- Add support for periodically generating a journal checkpoint.
+- Onode tree
+- Extent tree
+- Remaining ObjectStore integration
+
+ObjectStore considerations
+==========================
+
+Splits, merges, and sharding
+----------------------------
+
+One of the current ObjectStore requirements is to be able to split a
+collection (PG) in O(1) time.  Starting in mimic, we also need to be
+able to merge two collections into one (i.e., exactly the reverse of a
+split).
+
+However, the PGs that we split into would hash to different shards of
+the OSD in the current sharding scheme.  One can imagine replacing
+that sharding scheme with a temporary mapping directing the smaller
+child PG to the right shard since we generally then migrate that PG to
+another OSD anyway, but this wouldn't help us in the merge case where
+the constituent pieces may start out on different shards and
+ultimately need to be handled in the same collection (and be operated
+on via single transactions).
+
+This suggests that we likely need a way for data written via one shard
+to "switch ownership" and later be read and managed by a different
+shard.
+
+
+
diff --git a/doc/dev/sepia.rst b/doc/dev/sepia.rst
new file mode 100644
index 000000000..3064900c5
--- /dev/null
+++ b/doc/dev/sepia.rst
@@ -0,0 +1,8 @@
+Sepia community test lab
+========================
+
+The Ceph community maintains a test lab that is open to active contributors to
+the Ceph project. Please see the `Sepia wiki`_ for more information.
+
+.. _Sepia wiki: https://wiki.sepia.ceph.com/doku.php
+
diff --git a/doc/dev/session_authentication.rst b/doc/dev/session_authentication.rst
new file mode 100644
index 000000000..48fab623d
--- /dev/null
+++ b/doc/dev/session_authentication.rst
@@ -0,0 +1,160 @@
+==============================================
+Session Authentication for the Cephx Protocol
+==============================================
+Peter Reiher
+7/30/12
+
+The original Cephx protocol authenticated the client to the authenticator and set up a session 
+key used to authenticate the client to the server it needs to talk to.  It did not, however,
+authenticate the ongoing messages between the client and server.  Based on the fact that they
+share a secret key, these ongoing session messages can be easily authenticated by using the
+key to sign the messages.  
+
+This document describes changes to the code that allow such ongoing session authentication.
+The changes allow for future changes that permit other authentication protocols (and the 
+existing null NONE and UNKNOWN protocols) to handle signatures, but the only protocol that 
+actually does signatures, at the time of the writing, is the Cephx protocol.
+
+Introduction
+-------------
+
+This code comes into play after the Cephx protocol has completed.  At this point, the client and
+server share a secret key.  This key will be used for authentication.  For other protocols, there
+may or may not be such a key in place, and perhaps the actual procedures used to perform 
+signing will be different, so the code is written to be general.
+
+The "session" here is represented by an established pipe.  For such pipes, there should be a
+``session\_security`` structure attached to the pipe.  Whenever a message is to be sent on the
+pipe, code that handles the signature for this kind of session security will be called.  On the
+other end of the pipe, code that checks this kind of session security's message signatures will
+be called.  Messages that fail the signature check will not be processed further.  That implies
+that the sender had better be in agreement with the receiver on the session security being used,
+since otherwise messages will be uniformly dropped between them.
+
+The code is also prepared to handle encryption and decryption of session messages, which would
+add secrecy to the integrity provided by the signatures.  No protocol currently implemented
+encrypts the ongoing session messages, though.
+
+For this functionality to work, several steps are required.  First, the sender and receiver must have 
+a successful run of the cephx protocol to establish a shared key.  They must store that key somewhere
+that the pipe can get at later, to permit messages to be signed with it.  Sent messages must be
+signed, and received messages must have their signatures checked.
+
+The signature could be computed in a variety of ways, but currently its size is limited to 64 bits.
+A message's signature is placed in its footer, in a field called ``sig``.
+
+The signature code in Cephx can be turned on and off at runtime, using a Ceph boolean option called 
+``cephx\_sign\_messages``.  It is currently set to false, by default, so no messages will be signed.  It
+must be changed to true to cause signatures to be calculated and checked.
+
+Storing the Key
+---------------
+
+The key is needed to create signatures on the sending end and check signatures on the receiving end.
+In the future, if asymmetric crypto is an option, it's possible that two keys (a private one for
+this end of the pipe and a public one for the other end) would need to be stored. At this time,
+messages going in both directions will be signed with the same key, so only that key needs to be
+saved.
+
+The key is saved when the pipe is established.  On the client side, this happens in ``connect()``,
+which is located in ``msg/Pipe.cc``.  The key is obtained from a run of the Cephx protocol,
+which results in a successfully checked authorizer structure.  If there is such an authorizer
+available, the code calls ``get\_auth\_session\_handler()`` to create a new authentication session handler
+and stores it in the pipe data structure.  On the server side, a similar thing is done in 
+``accept()`` after the authorizer provided by the client has been verified.
+
+Once these things are done on either end of the connection, session authentication can start.
+
+These routines (``connect()`` and ``accept()``) are also used to handle situations where a new 
+session is being set up.  At this stage, no authorizer has been created yet, so there's no key.
+Special cases in the code that calls the signature code skip these calls when the 
+``CEPH\_AUTH\_UNKNOWN`` protocol is in use.  This protocol label is on the pre-authorizer
+messages in a session, indicating that negotiation on an authentication protocol is ongoing and
+thus signature is not possible.  There will be a reliable authentication operation later in this 
+session before anything sensitive should be passed, so this is not a security problem.
+
+Signing Messages
+----------------
+
+Messages are signed in the ``write\_message`` call located in ``msg/Pipe.cc``.  The actual
+signature process is to encrypt the CRCs for the message using the shared key.  Thus, we must
+defer signing until all CRCs have been computed.  The header CRC is computed last, so we
+call ``sign\_message()`` as soon as we've calculated that CRC.  
+
+``sign\_message()`` is a virtual function defined in ``auth/AuthSessionHandler.h``.  Thus,
+a specific version of it must be written for each authentication protocol supported.  Currently,
+only UNKNOWN, NONE and CEPHX are supported.  So there is a separate version of ``sign\_message()`` in
+``auth/unknown/AuthUnknownSessionHandler.h``, ``auth/none/AuthNoneSessionHandler.h`` and 
+``auth/cephx/CephxSessionHandler.cc``.  The UNKNOWN and NONE versions simply return 0, indicating 
+success.
+
+The CEPHX version is more extensive.  It is found in ``auth/cephx/CephxSessionHandler.cc``.  
+The first thing done is to determine if the run time option to handle signatures (see above) is on.  
+If not, the Cephx version of ``sign\_message()`` simply returns success without actually calculating 
+a signature or inserting it into the message.
+
+If the run time option is enabled, ``sign\_message()`` copies all of the message's CRCs (one from the
+header and three from the footer) into a buffer.  It calls ``encode\_encrypt()`` on the buffer,
+using the key obtained from the pipe's ``session\_security`` structure.  64 bits of the encrypted
+result are put into the message footer's signature field and a footer flag is set to indicate that
+the message was signed.  (This flag is a sanity check.  It is not regarded as definitive 
+evidence that the message was signed.  The presence of a ``session\_security`` structure at the
+receiving end requires a signature regardless of the value of this flag.)  If this all goes well,
+``sign\_message()`` returns 0.  If there is a problem anywhere along the line and no signature
+was computed, it returns ``SESSION\_SIGNATURE\_FAILURE``.
+
+Checking Signatures 
+-------------------
+
+The signature is checked by a routine called ``check\_message\_signature()``.  This is also a
+virtual function, defined in ``auth/AuthSessionHandler.h``.  So again there are specific versions
+for supported authentication protocols, such as UNKNOWN, NONE and CEPHX.  Again, the UNKNOWN and
+NONE versions are stored in ``auth/unknown/AuthUnknownSessionHandler.h`` and 
+``auth/none/AuthNoneSessionHandler.h``, respectively, and again they simply return 0, indicating
+success.
+
+The CEPHX version of ``check\_message\_signature()`` performs a real signature check.  This routine 
+(stored in ``auth/cephx/CephxSessionHandler.cc``) exits with success if the run time option has 
+disabled signatures.  Otherwise, it takes the CRCs from the header and footer, encrypts the result, 
+and compares it to the signature stored in the footer.  Since an earlier routine has checked that
+the CRCs actually match the contents of the message, it is unnecessary to recompute the CRCs
+on the raw data in the message.  The encryption is performed with the same ``encode\_encrypt()``
+routine used on the sending end, using the key stored in the local ``session\_security``
+data structure.  
+
+If everything checks out, the CEPHX routine returns 0, indicating success.  If there is a 
+problem, the routine returns ``SESSION\_SIGNATURE\_FAILURE``.
+
+Adding New Session Authentication Methods
+-----------------------------------------
+
+For the purpose of session authentication only (not the basic authentication of client and 
+server currently performed by the Cephx protocol), in addition to adding a new protocol, that
+protocol must have a ``sign\_message()`` routine and a ``check\_message\_signature`` routine.
+These routines will take a message pointer as a parameter and return 0 on success.  The procedure
+used to sign and check will be specific to the new method, but probably there will be a
+``session\_security`` structure attached to the pipe that contains a cryptographic key.  This
+structure will be either an ``AuthSessionHandler`` (found in ``auth/AuthSessionHandler.h``) 
+or a structure derived from that type.
+
+Adding Encryption to Sessions
+-----------------------------
+
+The existing code is partially, but not fully, set up to allow sessions to have their packets
+encrypted.  Part of adding encryption would be similar to adding a new authentication method.
+But one would also need to add calls to the encryption and decryption routines in ``write\_message()`` 
+and ``read\_message()``.  These calls would probably go near where the current calls for
+authentication are made.  You should consider whether you want to replace the existing calls
+with something more general that does whatever the chosen form of session security requires,
+rather than explicitly saying ``sign`` or ``encrypt``.
+
+Session Security Statistics
+---------------------------
+
+The existing Cephx authentication code keeps statistics on how many messages were signed, how
+many message signature were checked, and how many checks succeeded and failed.  It is prepared
+to keep similar statistics on encryption and decryption.  These statistics can be accessed through
+the call ``printAuthSessionHandlerStats`` in ``auth/AuthSessionHandler.cc``.  
+
+If new authentication or encryption methods are added, they should include code that keeps these
+statistics.
diff --git a/doc/dev/testing.rst b/doc/dev/testing.rst
new file mode 100644
index 000000000..37202e992
--- /dev/null
+++ b/doc/dev/testing.rst
@@ -0,0 +1,40 @@
+Testing notes
+=============
+
+
+build-integration-branch
+------------------------
+
+Setup
+^^^^^
+
+#. Create a github token at `<https://github.com/settings/tokens>`_
+   and put it in ``~/.github_token``.  Note that only the
+   ``public_repo`` under the ``repo`` section needs to be checked.
+
+#. Create a ceph repo label `wip-yourname-testing` if you don't
+   already have one at `<https://github.com/ceph/ceph/labels>`_.
+
+#. Create the ``ci`` remote::
+
+     git remote add ci git@github.com:ceph/ceph-ci
+
+Using
+^^^^^
+
+#. Tag some subset of `needs-qa` commits with your label (usually `wip-yourname-testing`).
+
+#. Create the integration branch::
+
+     git checkout master
+     git pull
+     ../src/script/build-integration-branch wip-yourname-testing
+
+#. Smoke test::
+
+     ./run-make-check.sh
+
+#. Push to ceph-ci::
+
+     git push ci $(git rev-parse --abbrev-ref HEAD)
+
diff --git a/doc/dev/versions.rst b/doc/dev/versions.rst
new file mode 100644
index 000000000..34ed74724
--- /dev/null
+++ b/doc/dev/versions.rst
@@ -0,0 +1,42 @@
+==================
+Public OSD Version
+==================
+
+We maintain two versions on disk: an eversion_t pg_log.head and a
+version_t info.user_version. Each object is tagged with both the pg
+version and user_version it was last modified with.  The PG version is
+modified by manipulating OpContext::at_version and then persisting it
+to the pg log as transactions, and is incremented in all the places it
+used to be. The user_version is modified by manipulating the new
+OpContext::user_at_version and is also persisted via the pg log
+transactions.
+user_at_version is modified only in PrimaryLogPG::prepare_transaction
+when the op was a "user modify" (a non-watch write), and the durable
+user_version is updated according to the following rules:
+1) set user_at_version to the maximum of ctx->new_obs.oi.user_version+1
+and info.last_user_version+1.
+2) set user_at_version to the maximum of itself and
+ctx->at_version.version.
+3) ctx->new_obs.oi.user_version = ctx->user_at_version (to change the
+object's user_version)
+
+This set of update semantics mean that for traditional pools the
+user_version will be equal to the past reassert_version, while for
+caching pools the object and PG user-version will be able to cross
+pools without making a total mess of things.
+In order to support old clients, we keep the old reassert_version but
+rename it to "bad_replay_version"; we fill it in as before: for writes
+it is set to the at_version (and is the proper replay version); for
+watches it is set to our user version; for ENOENT replies it is set to
+the replay version's epoch but the user_version's version. We also now
+fill in the version_t portion of the bad_replay_version on read ops as
+well as write ops, which should be fine for all old clients.
+
+For new clients, we prevent them from reading bad_replay_version and
+add two proper members: user_version and replay_version; user_version
+is filled in on every operation (reads included) while replay_version
+is filled in for writes.
+
+The objclass function get_current_version() now always returns the
+pg->info.last_user_version, which means it is guaranteed to contain
+the version of the last user update in the PG (including on reads!).
diff --git a/doc/dev/vstart-ganesha.rst b/doc/dev/vstart-ganesha.rst
new file mode 100644
index 000000000..4e77deb8b
--- /dev/null
+++ b/doc/dev/vstart-ganesha.rst
@@ -0,0 +1,45 @@
+==============================
+NFS CephFS-RGW Developer Guide
+==============================
+
+CephFS exports are supported since Octopus and RGW exports are supported since
+Quincy.
+
+Configuring NFS Ganesha to export CephFS with vstart
+====================================================
+
+1) Using ``cephadm``
+
+    .. code:: bash
+
+        $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d --cephadm
+
+    This will deploy a single NFS Ganesha daemon using ``vstart.sh``, where the
+    daemon will listen on the default NFS Ganesha port. Also cephfs export is
+    created.
+
+2) Using test orchestrator
+
+    .. code:: bash
+
+       $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d
+
+    Environment variable ``NFS`` is the number of NFS Ganesha daemons to be
+    deployed, each listening on a random port.
+
+    .. note:: NFS Ganesha packages must be pre-installed for this to work.
+
+Configuring NFS Ganesha to export RGW with vstart
+=================================================
+
+1) Using ``cephadm``
+
+    .. code:: bash
+
+        $ MON=1 OSD=3 RGW=1 NFS=1 ../src/vstart.sh -n -d --cephadm
+
+    This will deploy a single NFS Ganesha daemon using ``vstart.sh``, where the
+    daemon will listen on the default NFS Ganesha port. Also rgw export is
+    created.
+
+    .. note:: boto python module must be pre-installed for this to work.
diff --git a/doc/dev/wireshark.rst b/doc/dev/wireshark.rst
new file mode 100644
index 000000000..e03b3621c
--- /dev/null
+++ b/doc/dev/wireshark.rst
@@ -0,0 +1,41 @@
+=====================
+ Wireshark Dissector
+=====================
+
+Wireshark has support for the Ceph protocol and it will be shipped in the 1.12.1
+release.
+
+Using
+=====
+
+To use the Wireshark dissector you must build it from `git`__, the process is
+outlined in great detail in the `Building and Installing`__ section of the
+`Wireshark Users Guide`__.
+
+__ `Wireshark git`_
+__ WSUG_BI_
+__ WSUG_
+
+Developing
+==========
+
+The Ceph dissector lives in `Wireshark git`_ at
+``epan/dissectors/packet-ceph.c``.  At the top of that file there are some
+comments explaining how to insert new functionality or to update the encoding
+of existing types.
+
+Before you start hacking on Wireshark code you should look at the
+``doc/README.developer`` and ``doc/README.dissector`` documents as they explain
+the basics of writing dissectors.  After reading those two documents you should
+be prepared to work on the Ceph dissector.  `The Wireshark
+developers guide`__ also contains a lot of useful information but it is less
+directed and is more useful as a reference then an introduction.
+
+__ WSDG_
+
+.. _WSUG: https://www.wireshark.org/docs/wsug_html_chunked/
+.. _WSDG: https://www.wireshark.org/docs/wsdg_html_chunked/
+.. _WSUG_BI: https://www.wireshark.org/docs/wsug_html_chunked/ChapterBuildInstall.html
+.. _Wireshark git: https://www.wireshark.org/develop.html
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/zoned-storage.rst b/doc/dev/zoned-storage.rst
new file mode 100644
index 000000000..cea741d6b
--- /dev/null
+++ b/doc/dev/zoned-storage.rst
@@ -0,0 +1,134 @@
+=======================
+ Zoned Storage Support
+=======================
+
+http://zonedstorage.io
+
+Zoned Storage is a class of storage devices that enables host and storage
+devices to cooperate to achieve higher storage capacities, increased throughput,
+and lower latencies. The zoned storage interface is available through the SCSI
+Zoned Block Commands (ZBC) and Zoned Device ATA Command Set (ZAC) standards on
+Shingled Magnetic Recording (SMR) hard disks today and is also being adopted for
+NVMe Solid State Disks with the upcoming NVMe Zoned Namespaces (ZNS) standard.
+
+This project aims to enable Ceph to work on zoned storage drives and at the same
+time explore research problems related to adopting this new interface.  The
+first target is to enable non-overwrite workloads (e.g. RGW) on host-managed SMR
+(HM-SMR) drives and explore cleaning (garbage collection) policies.  HM-SMR
+drives are high capacity hard drives with the ZBC/ZAC interface.  The longer
+term goal is to support ZNS SSDs, as they become available, as well as overwrite
+workloads.
+
+The first patch in these series enabled writing data to HM-SMR drives.  This
+patch introduces ZonedFreelistManger, a FreelistManager implementation that
+passes enough information to ZonedAllocator to correctly initialize state of
+zones by tracking the write pointer and the number of dead bytes per zone.  We
+have to introduce a new FreelistManager implementation because with zoned
+devices a region of disk can be in three states (empty, used, and dead), whereas
+current BitmapFreelistManager tracks only two states (empty and used).  It is
+not possible to accurately initialize the state of zones in ZonedAllocator by
+tracking only two states.  The third planned patch will introduce a rudimentary
+cleaner to form a baseline for further research.
+
+Currently we can perform basic RADOS benchmarks on an OSD running on an HM-SMR
+drives, restart the OSD, and read the written data, and write new data, as can
+be seen below.
+
+Please contact Abutalib Aghayev <agayev@psu.edu> for questions.
+
+::
+   
+  $ sudo zbd report -i -n /dev/sdc
+  Device /dev/sdc:
+      Vendor ID: ATA HGST HSH721414AL T240
+      Zone model: host-managed
+      Capacity: 14000.520 GB (27344764928 512-bytes sectors)
+      Logical blocks: 3418095616 blocks of 4096 B
+      Physical blocks: 3418095616 blocks of 4096 B
+      Zones: 52156 zones of 256.0 MB
+      Maximum number of open zones: no limit
+      Maximum number of active zones: no limit
+  52156 / 52156 zones
+  $ MON=1 OSD=1 MDS=0 sudo ../src/vstart.sh --new --localhost --bluestore --bluestore-devs /dev/sdc --bluestore-zoned
+  <snipped verbose output>
+  $ sudo ./bin/ceph osd pool create bench 32 32
+  pool 'bench' created
+  $ sudo ./bin/rados bench -p bench 10 write --no-cleanup
+  hints = 1
+  Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
+  Object prefix: benchmark_data_h0.cc.journaling712.narwhal.p_29846
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        45        29   115.943       116    0.384175    0.407806
+      2      16        86        70   139.949       164    0.259845    0.391488
+      3      16       125       109   145.286       156     0.31727    0.404727
+      4      16       162       146   145.953       148    0.826671    0.409003
+      5      16       203       187   149.553       164     0.44815    0.404303
+      6      16       242       226   150.621       156    0.227488    0.409872
+      7      16       281       265   151.384       156    0.411896    0.408686
+      8      16       320       304   151.956       156    0.435135    0.411473
+      9      16       359       343   152.401       156    0.463699    0.408658
+     10      15       396       381   152.356       152    0.409554    0.410851
+  Total time run:         10.3305
+  Total writes made:      396
+  Write size:             4194304
+  Object size:            4194304
+  Bandwidth (MB/sec):     153.333
+  Stddev Bandwidth:       13.6561
+  Max bandwidth (MB/sec): 164
+  Min bandwidth (MB/sec): 116
+  Average IOPS:           38
+  Stddev IOPS:            3.41402
+  Max IOPS:               41
+  Min IOPS:               29
+  Average Latency(s):     0.411226
+  Stddev Latency(s):      0.180238
+  Max latency(s):         1.00844
+  Min latency(s):         0.108616
+  $ sudo ../src/stop.sh
+  $ # Notice the lack of "--new" parameter to vstart.sh
+  $ MON=1 OSD=1 MDS=0 sudo ../src/vstart.sh --localhost --bluestore --bluestore-devs /dev/sdc --bluestore-zoned  
+  <snipped verbose output>
+  $ sudo ./bin/rados bench -p bench 10 rand
+  hints = 1
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        61        45   179.903       180    0.117329    0.244067
+      2      16       116       100   199.918       220    0.144162    0.292305
+      3      16       174       158   210.589       232    0.170941    0.285481
+      4      16       251       235   234.918       308    0.241175    0.256543
+      5      16       316       300   239.914       260    0.206044    0.255882
+      6      15       392       377   251.206       308    0.137972    0.247426
+      7      15       458       443   252.984       264   0.0800146    0.245138
+      8      16       529       513   256.346       280    0.103529    0.239888
+      9      16       587       571   253.634       232    0.145535      0.2453
+     10      15       646       631   252.254       240    0.837727    0.246019
+  Total time run:       10.272
+  Total reads made:     646
+  Read size:            4194304
+  Object size:          4194304
+  Bandwidth (MB/sec):   251.558
+  Average IOPS:         62
+  Stddev IOPS:          10.005
+  Max IOPS:             77
+  Min IOPS:             45
+  Average Latency(s):   0.249385
+  Max latency(s):       0.888654
+  Min latency(s):       0.0103208
+  $ sudo ./bin/rados bench -p bench 10 write --no-cleanup
+  hints = 1
+  Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
+  Object prefix: benchmark_data_h0.aa.journaling712.narwhal.p_64416
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        46        30   119.949       120     0.52627    0.396166
+      2      16        82        66   131.955       144     0.48087    0.427311
+      3      16       123       107   142.627       164      0.3287    0.420614
+      4      16       158       142   141.964       140    0.405177    0.425993
+      5      16       192       176   140.766       136    0.514565    0.425175
+      6      16       224       208   138.635       128     0.69184    0.436672
+      7      16       261       245   139.967       148    0.459929    0.439502
+      8      16       301       285   142.468       160    0.250846    0.434799
+      9      16       336       320   142.189       140    0.621686    0.435457
+     10      16       374       358   143.166       152    0.460593    0.436384
+  
diff --git a/doc/favicon.ico b/doc/favicon.ico
new file mode 100644
index 000000000..90e538ba7
--- /dev/null
+++ b/doc/favicon.ico
diff --git a/doc/foundation.rst b/doc/foundation.rst
new file mode 100644
index 000000000..95a393410
--- /dev/null
+++ b/doc/foundation.rst
@@ -0,0 +1,118 @@
+
+.. _foundation:
+
+=================
+ Ceph Foundation
+=================
+
+The Ceph Foundation exists to enable industry members to collaborate
+and pool resources to support the Ceph project community. The
+Foundation provides an open, collaborative, and neutral home for
+project stakeholders to coordinate their development and community
+investments in the Ceph ecosystem.
+
+The Ceph Foundation is organized as a directed fund under the Linux
+Foundation. Premier and General Member organizations contribute a
+yearly fee to become members. Associate members are educational
+institutions or government organizations and are invited to join at no
+cost.
+
+For more information, see `https://ceph.com/foundation
+<https://ceph.com/foundation>`_.
+
+
+Members
+=======
+
+Premier
+-------
+
+* `Bloomberg <https://bloomberg.com>`_
+* `Clyso <https://www.clyso.com/en/>`_
+* `IBM <https://ibm.com>`_
+* `Intel <http://www.intel.com/>`_
+* `OVH <https://www.ovh.com/>`_
+* `Samsung Electronics <https://samsung.com/>`_
+* `Western Digital <https://www.wdc.com/>`_
+
+General
+-------
+
+* `42on <https://www.42on.com/>`_
+* `Akamai <https://www.akamai.com/>`_
+* `ARM <http://www.arm.com/>`_
+* `Canonical <https://www.canonical.com/>`_
+* `Cloudbase Solutions <https://cloudbase.it/>`_
+* `CloudFerro <https://cloudferro.com/>`_
+* `croit <http://www.croit.io/>`_
+* `EasyStack <https://www.easystack.io/>`_
+* `ISS <http://iss-integration.com/>`_
+* `Koor <http://koor.tech/>`_
+* `Linode <https://linode.com/>`_
+* `OSNexus <https://osnexus.com/>`_
+* `Seagate <https://seagate.com/>`_
+* `SinoRail <http://www.sinorail.com/>`_
+* `Vexxhost <https://vexxhost.com>`_
+
+Associate
+---------
+
+* `Boston University <http://www.bu.com/>`_
+* `Center for Research in Open Source Systems (CROSS) <http://cross.ucsc.edu/>`_
+* `CERN <https://home.cern/>`_
+* `FASRC <https://www.rc.fas.harvard.edu/>`_
+* `grnet <https://grnet.gr/>`_
+* `Monash University <http://www.monash.edu/>`_
+* `NRF SARAO <http://www.ska.ac.za/about/sarao/>`_
+* `Open Infrastructure Foundation <http://openinfra.dev>`_
+* `Science & Technology Facilities Councel (STFC) <https://stfc.ukri.org/>`_
+* `SWITCH <https://switch.ch/>`_
+* `University of Michigan <http://www.osris.org/>`_
+
+Governing Board
+===============
+
+The Governing Board consists of all Premier members, a representative
+for the General members, a representative for the Associate members,
+and a representative from the Ceph Leadership Team (the technical
+governance body). The board is responsible for:
+
+* Building and approving an annual budget for spending in support of
+  the Ceph project
+* Establishing ad-hoc committees to address current needs of the
+  project
+* Coordinating outreach or marketing
+* Meeting regularly to discuss Foundation activities, the status of
+  the Ceph project, and overall project strategy
+* Voting on any decisions or matters before the board
+
+The Ceph Foundation board is not responsible for and does not have any
+direct control over the technical governance of Ceph. Development and
+engineering activities are managed through traditional open source
+processes and are overseen by the :ref:`csc`. For more
+information see :ref:`governance`.
+
+Members
+-------
+
+* Anjaneya "Reddy" Chagam (Intel)
+* Carlos Maltzahn (UCSC) - Associate member representative
+* Dan van der Ster (CERN) - Ceph Council representative
+* Haomai Wang (XSKY)
+* James Page (Canonical)
+* Joachim Kraftmayer (Clyso)
+* Josh Durgin (IBM) - Ceph Council representative
+* Matias Bjorling (Western Digital)
+* Matthew Leonard (Bloomberg)
+* Mike Perez (IBM) - Ceph community manager
+* Myoungwon Oh (Samsung Electronics)
+* Neha Ojha (IBM) - Ceph Council Representative
+* Steven Umbehocker (OSNexus) - General member representative
+* Pawel Sadowski (OVH)
+* Vincent Hsu (IBM)
+
+Joining
+=======
+
+For information about joining the Ceph Foundation, please contact
+membership@linuxfoundation.org.
diff --git a/doc/glossary.rst b/doc/glossary.rst
new file mode 100644
index 000000000..4d7037684
--- /dev/null
+++ b/doc/glossary.rst
@@ -0,0 +1,460 @@
+===============
+ Ceph Glossary
+===============
+
+.. glossary::
+
+        Application
+                More properly called a :term:`client`, an application is any program
+                external to Ceph that uses a Ceph Cluster to store and
+                replicate data.
+
+	:ref:`BlueStore<rados_config_storage_devices_bluestore>`
+                OSD BlueStore is a storage back end used by OSD daemons, and
+                was designed specifically for use with Ceph. BlueStore was
+                introduced in the Ceph Kraken release. The Luminous release of
+                Ceph promoted BlueStore to the default OSD back end,
+                supplanting FileStore. As of the Reef release, FileStore is no
+                longer available as a storage back end.
+                
+                BlueStore stores objects directly on raw block devices or
+                partitions, and does not interact with mounted file systems.
+                BlueStore uses RocksDB's key/value database to map object names
+                to block locations on disk.
+
+        Bucket
+                In the context of :term:`RGW`, a bucket is a group of objects.
+                In a filesystem-based analogy in which objects are the
+                counterpart of files, buckets are the counterpart of
+                directories. :ref:`Multisite sync
+                policies<radosgw-multisite-sync-policy>` can be set on buckets,
+                to provide fine-grained control of data movement from one zone
+                to another zone. 
+                
+                The concept of the bucket has been taken from AWS S3. See also
+                `the AWS S3 page on creating buckets <https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-buckets-s3.html>`_
+                and `the AWS S3 'Buckets Overview' page <https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingBucket.html>`_.
+
+                OpenStack Swift uses the term "containers" for what RGW and AWS call "buckets". 
+                See `the OpenStack Storage API overview page <https://docs.openstack.org/swift/latest/api/object_api_v1_overview.html>`_.
+                
+	Ceph
+                Ceph is a distributed network storage and file system with
+                distributed metadata management and POSIX semantics.
+
+	Ceph Block Device
+                Also called "RADOS Block Device" and :term:`RBD`. A software
+                instrument that orchestrates the storage of block-based data in
+                Ceph. Ceph Block Device splits block-based application data
+                into "chunks". RADOS stores these chunks as objects. Ceph Block
+                Device orchestrates the storage of those objects across the
+                storage cluster. 
+
+	Ceph Block Storage
+                One of the three kinds of storage supported by Ceph (the other
+                two are object storage and file storage). Ceph Block Storage is
+                the block storage "product", which refers to block-storage
+                related services and capabilities when used in conjunction with
+                the collection of (1) ``librbd`` (a python module that provides
+                file-like access to :term:`RBD` images), (2) a hypervisor such
+                as QEMU or Xen, and (3) a hypervisor abstraction layer such as
+                ``libvirt``.
+
+	Ceph Client
+                Any of the Ceph components that can access a Ceph Storage
+                Cluster. This includes the Ceph Object Gateway, the Ceph Block
+                Device, the Ceph File System, and their corresponding
+                libraries. It also includes kernel modules, and FUSEs
+                (Filesystems in USERspace).
+
+	Ceph Client Libraries
+                The collection of libraries that can be used to interact with
+                components of the Ceph Cluster.
+
+	Ceph Cluster Map
+                See :term:`Cluster Map`
+
+	Ceph Dashboard
+                :ref:`The Ceph Dashboard<mgr-dashboard>` is a built-in
+                web-based Ceph management and monitoring application through
+                which you can inspect and administer various resources within
+                the cluster. It is implemented as a :ref:`ceph-manager-daemon`
+                module.
+
+	Ceph File System
+                See :term:`CephFS`
+
+	:ref:`CephFS<ceph-file-system>`
+                The **Ceph F**\ile **S**\ystem, or CephFS, is a
+                POSIX-compliant file system built on top of Ceph’s distributed
+                object store, RADOS.  See :ref:`CephFS Architecture
+                <arch-cephfs>` for more details.
+
+	Ceph Interim Release
+                See :term:`Releases`.
+
+	Ceph Kernel Modules
+                The collection of kernel modules that can be used to interact
+                with the Ceph Cluster (for example: ``ceph.ko``, ``rbd.ko``).
+
+	:ref:`Ceph Manager<ceph-manager-daemon>`
+                The Ceph manager daemon (ceph-mgr) is a daemon that runs
+                alongside monitor daemons to provide monitoring and interfacing
+                to external monitoring and management systems. Since the
+                Luminous release (12.x), no Ceph cluster functions properly
+                unless it contains a running ceph-mgr daemon. 
+
+	Ceph Manager Dashboard
+                See :term:`Ceph Dashboard`.
+
+	Ceph Metadata Server
+                See :term:`MDS`.
+
+	Ceph Monitor
+                A daemon that maintains a map of the state of the cluster. This
+                "cluster state" includes the monitor map, the manager map, the
+                OSD map, and the CRUSH map. A Ceph cluster must contain a
+                minimum of three running monitors in order to be both redundant
+                and highly-available. Ceph monitors and the nodes on which they
+                run are often referred to as "mon"s. See :ref:`Monitor Config
+                Reference <monitor-config-reference>`.
+
+	Ceph Node
+               A Ceph node is a unit of the Ceph Cluster that communicates with
+               other nodes in the Ceph Cluster in order to replicate and
+               redistribute data. All of the nodes together are called the
+               :term:`Ceph Storage Cluster`. Ceph nodes include :term:`OSD`\s,
+               :term:`Ceph Monitor`\s, :term:`Ceph Manager`\s, and
+               :term:`MDS`\es. The term "node" is usually equivalent to "host"
+               in the Ceph documentation. If you have a running Ceph Cluster,
+               you can list all of the nodes in it by running the command
+               ``ceph node ls all``.
+                
+	:ref:`Ceph Object Gateway<object-gateway>`
+                An object storage interface built on top of librados. Ceph
+                Object Gateway provides a RESTful gateway between applications
+                and Ceph storage clusters.
+
+	Ceph Object Storage
+                See :term:`Ceph Object Store`.
+
+	Ceph Object Store
+                A Ceph Object Store consists of a :term:`Ceph Storage Cluster`
+                and a :term:`Ceph Object Gateway` (RGW).
+
+	:ref:`Ceph OSD<rados_configuration_storage-devices_ceph_osd>`
+                Ceph **O**\bject **S**\torage **D**\aemon. The Ceph OSD
+                software, which interacts with logical disks (:term:`OSD`).
+                Around 2013, there was an attempt by "research and industry"
+                (Sage's own words) to insist on using the term "OSD" to mean
+                only "Object Storage Device", but the Ceph community has always
+                persisted in using the term to mean "Object Storage Daemon" and
+                no less an authority than Sage Weil himself confirms in
+                November of 2022 that "Daemon is more accurate for how Ceph is
+                built" (private correspondence between Zac Dover and Sage Weil,
+                07 Nov 2022). 
+
+	Ceph OSD Daemon
+                See :term:`Ceph OSD`.
+
+	Ceph OSD Daemons
+                See :term:`Ceph OSD`.
+
+	Ceph Platform
+                All Ceph software, which includes any piece of code hosted at
+                `https://github.com/ceph`_.
+
+	Ceph Point Release
+                See :term:`Releases`.
+
+	Ceph Project
+                The aggregate term for the people, software, mission and
+                infrastructure of Ceph.
+
+	Ceph Release
+                See :term:`Releases`.
+
+	Ceph Release Candidate
+                See :term:`Releases`.
+
+	Ceph Stable Release
+                See :term:`Releases`.
+
+	Ceph Stack
+		A collection of two or more components of Ceph.
+
+	:ref:`Ceph Storage Cluster<arch-ceph-storage-cluster>`
+                The collection of :term:`Ceph Monitor`\s, :term:`Ceph
+                Manager`\s, :term:`Ceph Metadata Server`\s, and :term:`OSD`\s
+                that work together to store and replicate data for use by
+                applications, Ceph Users, and :term:`Ceph Client`\s. Ceph
+                Storage Clusters receive data from :term:`Ceph Client`\s.
+
+	CephX
+                The Ceph authentication protocol. CephX authenticates users and
+                daemons. CephX operates like Kerberos, but it has no single
+                point of failure. See the :ref:`High-availability
+                Authentication section<arch_high_availability_authentication>`
+                of the Architecture document and the :ref:`CephX Configuration
+                Reference<rados-cephx-config-ref>`. 
+
+	Client
+                A client is any program external to Ceph that uses a Ceph
+                Cluster to store and replicate data. 
+
+	Cloud Platforms
+	Cloud Stacks
+                Third party cloud provisioning platforms such as OpenStack,
+                CloudStack, OpenNebula, and Proxmox VE.
+
+	Cluster Map
+                The set of maps consisting of the monitor map, OSD map, PG map,
+                MDS map, and CRUSH map, which together report the state of the
+                Ceph cluster. See :ref:`the "Cluster Map" section of the
+                Architecture document<architecture_cluster_map>` for details.
+
+	CRUSH
+                **C**\ontrolled **R**\eplication **U**\nder **S**\calable
+                **H**\ashing. The algorithm that Ceph uses to compute object
+                storage locations.
+
+	CRUSH rule
+                The CRUSH data placement rule that applies to a particular
+                pool or pools.
+
+        DAS
+                **D**\irect-\ **A**\ttached **S**\torage. Storage that is
+                attached directly to the computer accessing it, without passing
+                through a network.  Contrast with NAS and SAN.
+
+	:ref:`Dashboard<mgr-dashboard>`
+                A built-in web-based Ceph management and monitoring application
+                to administer various aspects and objects of the cluster. The
+                dashboard is implemented as a Ceph Manager module. See
+                :ref:`mgr-dashboard` for more details.
+
+	Dashboard Module
+                Another name for :term:`Dashboard`.
+
+	Dashboard Plugin
+        FQDN
+                **F**\ully **Q**\ualified **D**\omain **N**\ame. A domain name
+                that is applied to a node in a network and that specifies the
+                node's exact location in the tree hierarchy of the DNS.
+
+                In the context of Ceph cluster administration, FQDNs are often
+                applied to hosts. In this documentation, the term "FQDN" is
+                used mostly to distinguish between FQDNs and relatively simpler
+                hostnames, which do not specify the exact location of the host
+                in the tree hierarchy of the DNS but merely name the host.
+
+	Host
+                Any single machine or server in a Ceph Cluster. See :term:`Ceph
+                Node`.
+
+        Hybrid OSD  
+                Refers to an OSD that has both HDD and SSD drives.
+
+	LVM tags
+                **L**\ogical **V**\olume **M**\anager tags. Extensible metadata
+                for LVM volumes and groups. They are used to store
+                Ceph-specific information about devices and its relationship
+                with OSDs.
+
+	:ref:`MDS<cephfs_add_remote_mds>`
+                The Ceph **M**\eta\ **D**\ata **S**\erver daemon. Also referred
+                to as "ceph-mds". The Ceph metadata server daemon must be
+                running in any Ceph cluster that runs the CephFS file system.
+                The MDS stores all filesystem metadata. 
+
+	MGR
+                The Ceph manager software, which collects all the state from
+                the whole cluster in one place.
+
+	:ref:`MON<arch_monitor>`
+		The Ceph monitor software.
+
+	Node
+                See :term:`Ceph Node`.
+
+	Object Storage Device
+                See :term:`OSD`.
+
+	OSD
+                Probably :term:`Ceph OSD`, but not necessarily. Sometimes
+                (especially in older correspondence, and especially in
+                documentation that is not written specifically for Ceph), "OSD"
+                means "**O**\bject **S**\torage **D**\evice", which refers to a
+                physical or logical storage unit (for example: LUN). The Ceph
+                community has always used the term "OSD" to refer to
+                :term:`Ceph OSD Daemon` despite an industry push in the
+                mid-2010s to insist that "OSD" should refer to "Object Storage
+                Device", so it is important to know which meaning is intended. 
+
+	OSD fsid
+                This is a unique identifier used to identify an OSD. It is
+                found in the OSD path in a file called ``osd_fsid``. The
+                term ``fsid`` is used interchangeably with ``uuid``
+
+	OSD id
+                The integer that defines an OSD. It is generated by the
+                monitors during the creation of each OSD.
+
+	OSD uuid
+                This is the unique identifier of an OSD. This term is used
+                interchangeably with ``fsid``
+
+        Period
+                In the context of :term:`RGW`, a period is the configuration
+                state of the :term:`Realm`. The period stores the configuration
+                state of a multi-site configuration. When the period is updated,
+                the "epoch" is said thereby to have been changed.
+
+        Placement Groups (PGs)
+                Placement groups (PGs) are subsets of each logical Ceph pool.
+                Placement groups perform the function of placing objects (as a
+                group) into OSDs. Ceph manages data internally at
+                placement-group granularity: this scales better than would
+                managing individual (and therefore more numerous) RADOS
+                objects. A cluster that has a larger number of placement groups
+                (for example, 100 per OSD) is better balanced than an otherwise
+                identical cluster with a smaller number of placement groups. 
+                
+                Ceph's internal RADOS objects are each mapped to a specific
+                placement group, and each placement group belongs to exactly
+                one Ceph pool. 
+
+	:ref:`Pool<rados_pools>`
+		A pool is a logical partition used to store objects.
+
+	Pools
+                See :term:`pool`.
+
+	:ref:`Primary Affinity <rados_ops_primary_affinity>`
+                The characteristic of an OSD that governs the likelihood that
+                a given OSD will be selected as the primary OSD (or "lead
+                OSD") in an acting set. Primary affinity was introduced in
+                Firefly (v. 0.80). See :ref:`Primary Affinity
+                <rados_ops_primary_affinity>`.
+
+        Quorum	
+                Quorum is the state that exists when a majority of the
+                :ref:`Monitors<arch_monitor>` in the cluster are ``up``. A
+                minimum of three :ref:`Monitors<arch_monitor>` must exist in
+                the cluster in order for Quorum to be possible.
+
+	RADOS
+                **R**\eliable **A**\utonomic **D**\istributed **O**\bject
+                **S**\tore. RADOS is the object store that provides a scalable
+                service for variably-sized objects. The RADOS object store is
+                the core component of a Ceph cluster.  `This blog post from
+                2009
+                <https://ceph.io/en/news/blog/2009/the-rados-distributed-object-store/>`_
+                provides a beginner's introduction to RADOS. Readers interested
+                in a deeper understanding of RADOS are directed to `RADOS: A
+                Scalable, Reliable Storage Service for Petabyte-scale Storage
+                Clusters <https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf>`_.
+
+	RADOS Cluster
+                A proper subset of the Ceph Cluster consisting of
+                :term:`OSD`\s, :term:`Ceph Monitor`\s, and :term:`Ceph
+                Manager`\s.
+                
+	RADOS Gateway
+                See :term:`RGW`.
+
+	RBD
+                **R**\ADOS **B**\lock **D**\evice. See :term:`Ceph Block
+                Device`.
+
+        :ref:`Realm<rgw-realms>`
+                In the context of RADOS Gateway (RGW), a realm is a globally
+                unique namespace that consists of one or more zonegroups.
+
+        Releases
+
+	        Ceph Interim Release
+                        A version of Ceph that has not yet been put through
+                        quality assurance testing. May contain new features.
+
+                Ceph Point Release
+                        Any ad hoc release that includes only bug fixes and
+                        security fixes.
+
+                Ceph Release
+                        Any distinct numbered version of Ceph.
+
+                Ceph Release Candidate
+                        A major version of Ceph that has undergone initial
+                        quality assurance testing and is ready for beta
+                        testers.
+
+                Ceph Stable Release
+                        A major version of Ceph where all features from the
+                        preceding interim releases have been put through
+                        quality assurance testing successfully.
+
+	Reliable Autonomic Distributed Object Store
+                The core set of storage software which stores the user's data
+                (MON+OSD). See also :term:`RADOS`.
+
+	:ref:`RGW<object-gateway>`
+                **R**\ADOS **G**\ate\ **w**\ay.
+
+                Also called "Ceph Object Gateway". The component of Ceph that
+                provides a gateway to both the Amazon S3 RESTful API and the
+                OpenStack Swift API. 
+
+        scrubs
+
+                The processes by which Ceph ensures data integrity. During the
+                process of scrubbing, Ceph generates a catalog of all objects
+                in a placement group, then ensures that none of the objects are
+                missing or mismatched by comparing each primary object against
+                its replicas, which are stored across other OSDs. Any PG
+                is determined to have a copy of an object that is different
+                than the other copies or is missing entirely is marked
+                "inconsistent" (that is, the PG is marked "inconsistent"). 
+
+                There are two kinds of scrubbing: light scrubbing and deep
+                scrubbing (also called "normal scrubbing" and "deep scrubbing",
+                respectively). Light scrubbing is performed daily and does
+                nothing more than confirm that a given object exists and that
+                its metadata is correct. Deep scrubbing is performed weekly and
+                reads the data and uses checksums to ensure data integrity.
+
+                See :ref:`Scrubbing <rados_config_scrubbing>` in the RADOS OSD
+                Configuration Reference Guide and page 141 of *Mastering Ceph,
+                second edition* (Fisk, Nick. 2019).
+
+        secrets
+                Secrets are credentials used to perform digital authentication
+                whenever privileged users must access systems that require
+                authentication. Secrets can be passwords, API keys, tokens, SSH
+                keys, private certificates, or encryption keys.
+
+        SDS
+                **S**\oftware-**d**\efined **S**\torage.
+
+	systemd oneshot
+                A systemd ``type`` where a command is defined in ``ExecStart``
+                which will exit upon completion (it is not intended to
+                daemonize)
+
+	Teuthology
+		The collection of software that performs scripted tests on Ceph.
+
+        User
+                An individual or a system actor (for example, an application)
+                that uses Ceph clients to interact with the :term:`Ceph Storage
+                Cluster`. See :ref:`User<rados-ops-user>` and :ref:`User
+                Management<user-management>`.
+
+        Zone
+                In the context of :term:`RGW`, a zone is a logical group that
+                consists of one or more :term:`RGW` instances.  A zone's
+                configuration state is stored in the :term:`period`. See
+                :ref:`Zones<radosgw-zones>`.
+
+.. _https://github.com/ceph: https://github.com/ceph
+.. _Cluster Map: ../architecture#cluster-map   
diff --git a/doc/governance.rst b/doc/governance.rst
new file mode 100644
index 000000000..493a87666
--- /dev/null
+++ b/doc/governance.rst
@@ -0,0 +1,138 @@
+.. _governance:
+
+============
+ Governance
+============
+
+The Ceph open source community is guided by a few different groups.
+
+Key principles
+==============
+
+ * Decision-making is consensus-driven by those who participate.
+ * Leadership roles are defined primarily by responsibility, not prestige or seniority.
+ * It is normal and healthy for these roles to be passed on to others
+ * Everyone's role is ultimately to serve the users and participation
+   is voluntary.
+
+Bodies
+------
+
+Ceph Executive Council
+======================
+
+Responsibilities
+----------------
+
+ * Arbiter in cases where decisions cannot be reached by consensus
+ * Distribute key responsibilities amongst themselves or others
+ * Point of contact for the project
+ * Representatives for Ceph foundation board meetings
+ * Ensure things get done
+
+Membership
+----------
+
+ * 3 people
+ * Elected by the steering committee
+ * Candidates self-nominate or are nominated by other members
+ * Discussion of how roles/responsibilities may be delegated
+ * Ranked-choice vote by the steering committee
+ * 2 year terms, with one member being elected in even years, and the
+   other two in odd years
+ * Members may resign at any time, and the steering committee may vote
+   to appoint a replacement for the rest of their term
+ * members must involve >1 employer
+
+Current Members
+^^^^^^^^^^^^^^^
+
+* Dan van der Ster <daniel.vanderster@cern.ch>
+* Josh Durgin <jdurgin@redhat.com>
+* Neha Ojha <nojha@redhat.com>
+
+.. _csc:
+
+Ceph Steering Committee
+=======================
+
+Responsibilities
+----------------
+
+ * Elect executive council
+ * Amend governance model (by majority vote)
+ * Meet regularly to discuss and decide on tactical and strategic projects
+   and improvements
+
+Membership
+----------
+
+ * Developers, users, community members
+ * Voting members can be nominated and added/removed by existing
+   members (2/3 majority)
+ * Anyone may attend steering committee meetings as a non-voting participant
+ * Existing Ceph Leadership Team members are grandfathered in
+ * Membership reflected by an email list and on the Ceph website and
+   docs
+
+Current Members
+^^^^^^^^^^^^^^^
+
+ * Adam King <adking@redhat.com>
+ * Casey Bodley <cbodley@redhat.com>
+ * Dan van der Ster <dan.vanderster@clyso.com>
+ * David Orman <ormandj@1111systems.com>
+ * Ernesto Puerta <epuerta@redhat.com>
+ * Gregory Farnum <gfarnum@redhat.com>
+ * Haomai Wang <haomai@xsky.com>
+ * Ilya Dryomov <idryomov@redhat.com>
+ * Igor Fedotov <igor.fedotov@croit.io>
+ * Jeff Layton <jlayton@redhat.com>
+ * Josh Durgin <jdurgin@redhat.com>
+ * João Eduardo Luis <joao@suse.de>
+ * Ken Dreyer <kdreyer@redhat.com>
+ * Mark Nelson <mnelson@redhat.com>
+ * Matt Benjamin <mbenjami@redhat.com>
+ * Mike Perez <miperez@redhat.com>
+ * Myoungwon Oh <myoungwon.oh@samsung.com>
+ * Neha Ojha <nojha@redhat.com>
+ * Patrick Donnelly <pdonnell@redhat.com>
+ * Sam Just <sjust@redhat.com>
+ * Vikhyat Umrao <vikhyat@redhat.com>
+ * Xie Xingguo <xie.xingguo@zte.com.cn>
+ * Yehuda Sadeh <yehuda@redhat.com>
+ * Yingxin Cheng <yingxin.cheng@intel.com>
+ * Yuri Weinstein <yweinste@redhat.com>
+ * Zac Dover <zac.dover@gmail.com>
+
+.. _ctl:
+
+Component Team Leads
+====================
+
+Responsibilities
+----------------
+
+ * Manage a `component team`_ in Ceph
+ * Ensure PRs are reviewed and merged
+ * Ensure severe bug fixes are backported
+ * Run standups
+ * Bug triage, scrubs
+ * etc.
+
+Team leads are selected by the executive council, generally based on
+the recommendation by team members and outgoing lead.  Periodic
+rotation of lead responsibility among team members is encouraged.
+
+The Ceph Foundation
+-------------------
+
+The Ceph Foundation is organized as a directed fund under the Linux
+Foundation and is tasked with supporting the Ceph project community
+and ecosystem.  It has no direct control over the technical direction
+of the Ceph open source project beyond offering feedback and input
+into the collaborative development process.
+
+For more information, see :ref:`foundation`.
+
+.. _component team: https://ceph.io/en/community/team/
diff --git a/doc/images/CRUSH.jpg b/doc/images/CRUSH.jpg
new file mode 100644
index 000000000..ac7975935
--- /dev/null
+++ b/doc/images/CRUSH.jpg
diff --git a/doc/images/HAProxy_for_RGW.svg b/doc/images/HAProxy_for_RGW.svg
new file mode 100644
index 000000000..c2fc95f1f
--- /dev/null
+++ b/doc/images/HAProxy_for_RGW.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1182" height="877"><g transform="translate(-539 -40)" lucid:page-tab-id="0_0"><path d="M0 0h1870.4v1323.2H0z" fill="#fff"/><g filter="url(#a)"><path d="M1380 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40h-240c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,1385,285) translate(14.01 17.77777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1385,285) translate(65.80012345679013 17.77777777777778)"/></g><g filter="url(#a)"><path d="M920 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40H960c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,925,285) translate(14.01 17.77777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,925,285) translate(65.80012345679013 17.77777777777778)"/></g><g filter="url(#a)"><path d="M560 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40H600c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,565,285) translate(14.01 17.77777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,565,285) translate(65.80012345679013 17.77777777777778)"/></g><path d="M708 60c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,585,65) translate(18.95061728395062 39.65277777777778)"/><use xlink:href="#g" transform="matrix(1,0,0,1,585,65) translate(79.32098765432099 39.65277777777778)"/><path d="M1608 63c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,1485,68) translate(7.870370370370381 39.65277777777778)"/><use xlink:href="#h" transform="matrix(1,0,0,1,1485,68) translate(68.24074074074075 39.65277777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1485,68) translate(126.14197530864197 39.65277777777778)"/><path d="M700 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#i" transform="matrix(1,0,0,1,625,325) translate(17.098765432098766 39.65277777777778)"/><path d="M1058.5 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#i" transform="matrix(1,0,0,1,983.4990644204674,325) translate(17.098765432098766 39.65277777777778)"/><path d="M732.7 360H945.8" stroke="#5e5e5e" stroke-width="8" fill="none"/><path d="M714.45 360l14.26-4.64v9.28zM964.05 360l-14.26 4.64v-9.28z" stroke="#5e5e5e" stroke-width="8" fill="#5e5e5e"/><path d="M580 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H586c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#j" transform="matrix(1,0,0,1,592,512) translate(14.97530864197531 31.77777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,592,512) translate(108.67901234567901 31.77777777777778)"/><path d="M1518.5 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#i" transform="matrix(1,0,0,1,1443.4990644204672,325) translate(17.098765432098766 39.65277777777778)"/><path d="M1091.2 360h314.6" stroke="#5e5e5e" stroke-width="8" fill="none"/><path d="M1072.94 360l14.27-4.64v9.28zM1424.05 360l-14.26 4.64v-9.28z" stroke="#5e5e5e" stroke-width="8" fill="#5e5e5e"/><path d="M680 426.33c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6V474c0 3.3-2.7 6-6 6H686c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#k" transform="matrix(1,0,0,1,692,432.3333333333333) translate(3.864197530864189 17.77777777777778)"/><use xlink:href="#l" transform="matrix(1,0,0,1,692,432.3333333333333) translate(119.79012345679013 17.77777777777778)"/><use xlink:href="#m" transform="matrix(1,0,0,1,692,432.3333333333333) translate(32.19753086419753 44.44444444444444)"/><path d="M1188 60c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,1065,65) translate(18.95061728395062 39.65277777777778)"/><use xlink:href="#g" transform="matrix(1,0,0,1,1065,65) translate(79.32098765432099 39.65277777777778)"/><path d="M938.5 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#j" transform="matrix(1,0,0,1,950.4990644204674,512) translate(14.97530864197531 31.77777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,950.4990644204674,512) translate(108.67901234567901 31.77777777777778)"/><path d="M1398.5 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#j" transform="matrix(1,0,0,1,1410.4990644204672,512) translate(13.154320987654323 31.77777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1410.4990644204672,512) translate(106.85802469135803 31.77777777777778)"/><path d="M660 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M659.98 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M660 495.26L655.36 481h9.28z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1018.5 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1018.48 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M1018.5 495.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1478.5 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1478.48 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M1478.5 495.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M560 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H566c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,572,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,572,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#e" transform="matrix(1,0,0,1,572,829) translate(105.00617283950618 44.44444444444444)"/><path d="M740 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H746c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,752,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,752,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#d" transform="matrix(1,0,0,1,752,829) translate(105.00617283950618 44.44444444444444)"/><path d="M920 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H926c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,932,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,932,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#p" transform="matrix(1,0,0,1,932,829) translate(105.00617283950618 44.44444444444444)"/><path d="M1480 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,1492,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,1492,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#q" transform="matrix(1,0,0,1,1492,829) translate(105.00617283950618 44.44444444444444)"/><path d="M1058.5 426c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v47.67c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#k" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(3.864197530864189 17.77777777777778)"/><use xlink:href="#r" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(119.79012345679013 17.77777777777778)"/><use xlink:href="#s" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(34.11111111111111 44.44444444444444)"/><path d="M1518.5 426c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v47.67c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#k" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(2.0432098765432016 17.77777777777778)"/><use xlink:href="#t" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(117.96913580246914 17.77777777777778)"/><use xlink:href="#m" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(32.19753086419753 44.44444444444444)"/><path d="M1138.5 417.5v-5.74c0-3.3-2.7-6-6-6h-83.23" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1139.5 418.5h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1044.84 395.98l10.1 11.08-8.44 3.83z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1598.5 417.5v-6.98c0-3.3-2.7-6-6-6H1511" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1599.5 418.5h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1507.48 393.63l8.8 12.14-8.82 2.86z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M760 417.83V406c0-3.3-2.7-6-6-6h-56.86" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M761 418.83h-2v-1.02h2z" fill="#5e5e5e"/><path d="M691.67 391.43l11.58 9.54-7.82 4.98z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M659.9 582.5L641.6 797.07" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.9 582.6l-1.98-.16.08-.94h2z" fill="#b2b2b2"/><path d="M640.28 812.28l-3.4-14.6 9.23.78z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.56 582.33l149 217.9" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M661.4 581.78l-1.65 1.13-.96-1.4h2.4z" fill="#b2b2b2"/><path d="M818.17 812.83l-11.87-9.16 7.65-5.23z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.82 582.07L984.76 805" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.28 582.9l-2.04-1.4h3z" fill="#b2b2b2"/><path d="M997.33 813.67l-14.38-4.27 5.26-7.64z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.97 581.75l881.12 229.1" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.74 582.73l-3.9-1.02.06-.2h4.16z" fill="#b2b2b2"/><path d="M1556.87 814.7l-14.98.88 2.33-8.97z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1017.65 582.03l-361.9 223.74" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1018.15 582.9l-.86-1.4h3.1z" fill="#a3d977"/><path d="M642.75 813.8l9.7-11.45 4.87 7.9z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1017.85 582.26L831.97 801.4" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1018.6 582.93l-1.53-1.3.12-.13h2.6z" fill="#a3d977"/><path d="M822.1 813.03l5.7-13.88 7.06 6z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1018.4 582.5l-18.24 213.6" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1019.4 582.6l-1.98-.16.08-.94h2z" fill="#a3d977"/><path d="M998.86 811.3l-3.4-14.6 9.23.78z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1019.42 581.9l523.6 226.26" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1019.04 582.82l-3.06-1.32h3.64z" fill="#a3d977"/><path d="M1557.03 814.22l-14.93-1.4 3.67-8.52z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1477.54 581.77L657.82 810.53" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1481.66 581.66l-3.88 1.08-.35-1.24h4.18z" fill="#b2b2b2"/><path d="M643.12 814.63l12.5-8.3 2.48 8.93z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1477.56 581.83L837.43 809.3" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1477.87 582.8l-.46-1.3h4.1z" fill="#b2b2b2"/><path d="M823.05 814.42l11.9-9.15 3.1 8.74z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1477.6 581.94l-460.98 225.43" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1478.02 582.85l-.66-1.35h3.42z" fill="#b2b2b2"/><path d="M1002.9 814.08l10.8-10.43 4.06 8.33z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1478.83 582.44l75.08 215.6" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1479.78 582.14l-1.9.66-.44-1.3h2.12z" fill="#b2b2b2"/><path d="M1558.94 812.44l-9.07-11.94 8.75-3.05z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M860 450.14l178.5-.28" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M844.74 450.16L859 445.5v9.27zM1053.76 449.84l-14.26 4.66v-9.27z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1238.5 449.5h260" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1223.24 449.5l14.26-4.64v9.28zM1513.76 449.5l-14.26 4.64v-9.28z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1140 142.5v81.7c0 3.3-2.7 6-6 6h-103.9c-3.3 0-6 2.68-6 6v64.2" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1141 142.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1024.1 315.66l-4.63-14.27h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1560 145.5v80c0 3.3-2.7 6-6 6h-529.5c-3.3 0-6 2.7-6 6V300" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1561 145.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1018.5 315.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M660 142.5V224c0 3.3 2.7 6 6 6h346.5c3.3 0 6 2.7 6 6v64" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M661 142.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1018.5 315.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1250.5 400c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.47 0-4.5-2-4.5-4.5s2.03-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5z" fill="#5e5e5e"/><path d="M1246.12 404.5h-4.62v-9h4.62zM1378.5 404.5h-4.62v-9h4.62zM1090.5 858.5c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.65 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.65 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5z" fill="#5e5e5e"/><path d="M1086.12 863h-4.62v-9h4.62zM1478.5 863h-4.62v-9h4.62z" fill="#5e5e5e"/><path d="M742.5 540H754c3.3 0 6-2.7 6-6v-34" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M742.53 541h-1.03v-2h1.03z" fill="#5e5e5e"/><path d="M760 484.74l4.64 14.26h-9.28z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1101 540h31.5c3.3 0 6-2.7 6-6v-34.33" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1101.02 541H1100v-2h1.02z" fill="#5e5e5e"/><path d="M1138.5 484.4l4.63 14.27h-9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1561 540h31.5c3.3 0 6-2.7 6-6v-34.33" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1561.02 541H1560v-2h1.02z" fill="#5e5e5e"/><path d="M1598.5 484.4l4.63 14.27h-9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><defs><path fill="#333" d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="u"/><path fill="#333" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="v"/><path fill="#333" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="w"/><path fill="#333" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="x"/><g id="b"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#u"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,28.333333333333332,0)" xlink:href="#w"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,39.44444444444444,0)" xlink:href="#x"/></g><path fill="#333" d="M190 0L58-211 59 0H30v-248h39L202-35l-2-213h31V0h-41" id="y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#y" id="c"/><filter id="a" filterUnits="objectBoundingBox" x="-.03" y="-.01" width="1.06" height="1.06"><feOffset result="offOut" in="SourceAlpha" dy="6"/><feGaussianBlur result="blurOut" in="offOut" stdDeviation="5"/><feColorMatrix result="colorOut" in="blurOut" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.5019607843137255 0"/><feBlend in="SourceGraphic" in2="colorOut"/></filter><path fill="#333" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="z"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#z" id="d"/><path fill="#333" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="A"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#A" id="e"/><path fill="#333" d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="B"/><path fill="#333" d="M143 4C61 4 22-44 18-125c-5-107 100-154 193-111 17 8 29 25 37 43l-32 9c-13-25-37-40-76-40-61 0-88 39-88 99 0 61 29 100 91 101 35 0 62-11 79-27v-45h-74v-28h105v86C228-13 192 4 143 4" id="C"/><path fill="#333" d="M266 0h-40l-56-210L115 0H75L2-248h35L96-30l15-64 43-154h32l59 218 59-218h35" id="D"/><g id="f"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#B"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#C"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.2716049382716,0)" xlink:href="#D"/></g><path fill="#333" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="E"/><path fill="#333" d="M24 0v-261h32V0H24" id="F"/><path fill="#333" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="G"/><path fill="#333" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="H"/><path fill="#333" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="I"/><g id="g"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,11.11111111111111,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.864197530864196,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.20987654320987,0)" xlink:href="#I"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.55555555555555,0)" xlink:href="#x"/></g><g id="h"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,11.11111111111111,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.864197530864196,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.20987654320987,0)" xlink:href="#I"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.55555555555555,0)" xlink:href="#x"/></g><path fill="#333" d="M137 0h-34L2-248h35l83 218 83-218h36" id="J"/><path fill="#333" d="M33 0v-248h34V0H33" id="K"/><path fill="#333" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="L"/><g id="i"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#J"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#K"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.98765432098765,0)" xlink:href="#L"/></g><path fill="#333" d="M205 0l-28-72H64L36 0H1l101-248h38L239 0h-34zm-38-99l-47-123c-12 45-31 82-46 123h93" id="M"/><path fill="#333" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="N"/><path fill="#333" d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="O"/><path fill="#333" d="M179-190L93 31C79 59 56 82 12 73V49c39 6 53-20 64-50L1-190h34L92-34l54-156h33" id="P"/><g id="j"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#u"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#M"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,30.80246913580247,0)" xlink:href="#L"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.61728395061728,0)" xlink:href="#N"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,52.962962962962955,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,65.30864197530863,0)" xlink:href="#O"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,76.41975308641975,0)" xlink:href="#P"/></g><path fill="#333" d="M194 0L95-120 63-95V0H30v-248h33v124l119-124h40L117-140 236 0h-42" id="Q"/><path fill="#333" d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="R"/><path fill="#333" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="S"/><path fill="#333" d="M108 0H70L1-190h34L89-25l56-165h34" id="T"/><path fill="#333" d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="U"/><g id="k"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Q"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,27.160493827160494,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,39.50617283950617,0)" xlink:href="#R"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,51.85185185185185,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,64.19753086419753,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,69.07407407407408,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,73.95061728395062,0)" xlink:href="#T"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,85.06172839506173,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,97.40740740740742,0)" xlink:href="#U"/></g><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#A" id="l"/><path fill="#333" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="V"/><path fill="#333" d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="W"/><path fill="#333" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="X"/><g id="m"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#V"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,12.345679012345679,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,24.691358024691358,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,35.80246913580247,0)" xlink:href="#W"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,46.91358024691358,0)" xlink:href="#X"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,59.25925925925926,0)" xlink:href="#R"/></g><g id="n"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#B"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#C"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.2716049382716,0)" xlink:href="#D"/></g><path fill="#333" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="Y"/><g id="o"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#U"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,12.345679012345679,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,24.691358024691358,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,37.03703703703704,0)" xlink:href="#Y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,55.49382716049382,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,67.8395061728395,0)" xlink:href="#I"/></g><path fill="#333" d="M126-127c33 6 58 20 58 59 0 88-139 92-164 29-3-8-5-16-6-25l32-3c6 27 21 44 54 44 32 0 52-15 52-46 0-38-36-46-79-43v-28c39 1 72-4 72-42 0-27-17-43-46-43-28 0-47 15-49 41l-32-3c6-42 35-63 81-64 48-1 79 21 79 65 0 36-21 52-52 59" id="Z"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Z" id="p"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#I" id="q"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#z" id="r"/><g id="s"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,18.456790123456788,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,30.80246913580247,0)" xlink:href="#w"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,41.91358024691358,0)" xlink:href="#x"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,48.086419753086425,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,60.4320987654321,0)" xlink:href="#N"/></g><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#y" id="t"/></defs></g></svg>
\ No newline at end of file
diff --git a/doc/images/RADOS.jpg b/doc/images/RADOS.jpg
new file mode 100644
index 000000000..0793a8ce4
--- /dev/null
+++ b/doc/images/RADOS.jpg
diff --git a/doc/images/RBD.jpg b/doc/images/RBD.jpg
new file mode 100644
index 000000000..c2a2b8f16
--- /dev/null
+++ b/doc/images/RBD.jpg
diff --git a/doc/images/RDBSnapshots.jpg b/doc/images/RDBSnapshots.jpg
new file mode 100644
index 000000000..493d4686a
--- /dev/null
+++ b/doc/images/RDBSnapshots.jpg
diff --git a/doc/images/dashboard/invalid-credentials.png b/doc/images/dashboard/invalid-credentials.png
new file mode 100644
index 000000000..abd78b024
--- /dev/null
+++ b/doc/images/dashboard/invalid-credentials.png
diff --git a/doc/images/docreviewprocess.jpg b/doc/images/docreviewprocess.jpg
new file mode 100644
index 000000000..33f7163e6
--- /dev/null
+++ b/doc/images/docreviewprocess.jpg
diff --git a/doc/images/esx_chap.png b/doc/images/esx_chap.png
new file mode 100644
index 000000000..caa70668a
--- /dev/null
+++ b/doc/images/esx_chap.png
diff --git a/doc/images/esx_config_iscsi_main.png b/doc/images/esx_config_iscsi_main.png
new file mode 100644
index 000000000..7443379da
--- /dev/null
+++ b/doc/images/esx_config_iscsi_main.png
diff --git a/doc/images/esx_iscsi_recov_timeout.png b/doc/images/esx_iscsi_recov_timeout.png
new file mode 100644
index 000000000..99840ae9c
--- /dev/null
+++ b/doc/images/esx_iscsi_recov_timeout.png
diff --git a/doc/images/esx_web_client_storage_main.png b/doc/images/esx_web_client_storage_main.png
new file mode 100644
index 000000000..56442a8b7
--- /dev/null
+++ b/doc/images/esx_web_client_storage_main.png
diff --git a/doc/images/keycloak-adduser.png b/doc/images/keycloak-adduser.png
new file mode 100644
index 000000000..03d9048f3
--- /dev/null
+++ b/doc/images/keycloak-adduser.png
diff --git a/doc/images/keycloak-userclientmapper.png b/doc/images/keycloak-userclientmapper.png
new file mode 100644
index 000000000..814b5e07e
--- /dev/null
+++ b/doc/images/keycloak-userclientmapper.png
diff --git a/doc/images/keycloak-usercredentials.png b/doc/images/keycloak-usercredentials.png
new file mode 100644
index 000000000..a01ab7922
--- /dev/null
+++ b/doc/images/keycloak-usercredentials.png
diff --git a/doc/images/keycloak-userdetails.png b/doc/images/keycloak-userdetails.png
new file mode 100644
index 000000000..3251a3907
--- /dev/null
+++ b/doc/images/keycloak-userdetails.png
diff --git a/doc/images/keycloak-usertags.png b/doc/images/keycloak-usertags.png
new file mode 100644
index 000000000..ee3fecf48
--- /dev/null
+++ b/doc/images/keycloak-usertags.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_NoWALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_NoWALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..c368d06e3
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_NoWALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_WALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_WALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..fd5de6584
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_HDD_WALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_NVMe_SSD_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_NVMe_SSD_WPQ_vs_mClock.png
new file mode 100644
index 000000000..ac1a979f2
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Latency_Percentiles_NVMe_SSD_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..8a3d89e1e
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_WALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_WALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..a7c28bc70
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Throughput_HDD_WALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Client_Throughput_NVMe_SSD_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Client_Throughput_NVMe_SSD_WPQ_vs_mClock.png
new file mode 100644
index 000000000..b19bf1995
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Client_Throughput_NVMe_SSD_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..f3f6a5c97
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_NoWALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_WALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_WALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..89f138d85
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_HDD_WALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_NVMe_SSD_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_NVMe_SSD_WPQ_vs_mClock.png
new file mode 100644
index 000000000..8ad783a50
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Avg_Obj_Rec_Throughput_NVMe_SSD_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..bdce97e7b
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_WALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_WALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..e1c8efe70
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_HDD_WALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Clat_Latency_Comparison_NVMe_SSD_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_NVMe_SSD_WPQ_vs_mClock.png
new file mode 100644
index 000000000..c3016b217
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Clat_Latency_Comparison_NVMe_SSD_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..c75127073
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_NoWALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_WALdB_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_WALdB_WPQ_vs_mClock.png
new file mode 100644
index 000000000..281e788fe
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_HDD_WALdB_WPQ_vs_mClock.png
diff --git a/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_NVMe_SSD_WPQ_vs_mClock.png b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_NVMe_SSD_WPQ_vs_mClock.png
new file mode 100644
index 000000000..f5e823000
--- /dev/null
+++ b/doc/images/mclock_wpq_study/Recovery_Rate_Comparison_NVMe_SSD_WPQ_vs_mClock.png
diff --git a/doc/images/region-sync.png b/doc/images/region-sync.png
new file mode 100644
index 000000000..7f889a0e1
--- /dev/null
+++ b/doc/images/region-sync.png
diff --git a/doc/images/region-sync.svg b/doc/images/region-sync.svg
new file mode 100644
index 000000000..93b66f31d
--- /dev/null
+++ b/doc/images/region-sync.svg
@@ -0,0 +1,30426 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="744.09448819"
+   height="1052.3622047"
+   id="svg104536"
+   version="1.1"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="region-sync.svg"
+   inkscape:export-filename="/home/john/Pictures/region-sync.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs104538">
+    <linearGradient
+       id="linearGradient126200"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4884_">
+      <rect
+         height="11.504"
+         width="174.63"
+         y="360.13901"
+         x="-679.57202"
+         id="use84863" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4886_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84873" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4888_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84881" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4890_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-535.73901"
+         id="use84891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4892_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-535.73901"
+         id="use84899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99071">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99073" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4894_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-536.966"
+         id="use84911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4896_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-536.966"
+         id="use84919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99079">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99081" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4898_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4900_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99087">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99089" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4902_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4904_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84959" />
+    </clipPath>
+    <clipPath
+       id="clipPath99095">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4906_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-540.646"
+         id="use84971" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4908_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-540.646"
+         id="use84979" />
+    </clipPath>
+    <clipPath
+       id="clipPath99103">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99105" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4910_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-541.87299"
+         id="use84991" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4912_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-541.87299"
+         id="use84999" />
+    </clipPath>
+    <clipPath
+       id="clipPath99111">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4914_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85011" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4916_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85019" />
+    </clipPath>
+    <clipPath
+       id="clipPath99119">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99121" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4918_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85031" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4920_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85039" />
+    </clipPath>
+    <clipPath
+       id="clipPath99127">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99129" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4922_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85051" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4924_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85059" />
+    </clipPath>
+    <clipPath
+       id="clipPath99135">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99137" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4926_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-546.77899"
+         id="use85071" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4928_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-546.77899"
+         id="use85079" />
+    </clipPath>
+    <clipPath
+       id="clipPath99143">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99145" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4930_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85091" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4932_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85099" />
+    </clipPath>
+    <clipPath
+       id="clipPath99151">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99153" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4934_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85111" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4936_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85119" />
+    </clipPath>
+    <clipPath
+       id="clipPath99159">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99161" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4938_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85131" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4940_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85139" />
+    </clipPath>
+    <clipPath
+       id="clipPath99167">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99169" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4942_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85151" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4944_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85159" />
+    </clipPath>
+    <clipPath
+       id="clipPath99175">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99177" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4946_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85171" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4948_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85179" />
+    </clipPath>
+    <clipPath
+       id="clipPath99183">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99185" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4950_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85191" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4952_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85199" />
+    </clipPath>
+    <clipPath
+       id="clipPath99191">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99193" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4954_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-555.367"
+         id="use85211" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4956_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-555.367"
+         id="use85219" />
+    </clipPath>
+    <clipPath
+       id="clipPath99199">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99201" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4958_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85231" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4960_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85239" />
+    </clipPath>
+    <clipPath
+       id="clipPath99207">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99209" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4962_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85251" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4964_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85259" />
+    </clipPath>
+    <clipPath
+       id="clipPath99215">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99217" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4966_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85271" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4968_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85279" />
+    </clipPath>
+    <clipPath
+       id="clipPath99223">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99225" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4970_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85291" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4972_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85299" />
+    </clipPath>
+    <clipPath
+       id="clipPath99231">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99233" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4974_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85311" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4976_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85319" />
+    </clipPath>
+    <clipPath
+       id="clipPath99239">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99241" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4978_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85331" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4980_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85339" />
+    </clipPath>
+    <clipPath
+       id="clipPath99247">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99249" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4982_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85351" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4984_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85359" />
+    </clipPath>
+    <clipPath
+       id="clipPath99255">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99257" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4986_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-565.17999"
+         id="use85371" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4988_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-565.17999"
+         id="use85379" />
+    </clipPath>
+    <clipPath
+       id="clipPath99263">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99265" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4990_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85391" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4992_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85399" />
+    </clipPath>
+    <clipPath
+       id="clipPath99271">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99273" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4994_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85411" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4996_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85419" />
+    </clipPath>
+    <clipPath
+       id="clipPath99279">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99281" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4998_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85431" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5000_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85439" />
+    </clipPath>
+    <clipPath
+       id="clipPath99287">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99289" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5002_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85451" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5004_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85459" />
+    </clipPath>
+    <clipPath
+       id="clipPath99295">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99297" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5006_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-571.315"
+         id="use85471" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5008_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-571.31403"
+         id="use85479" />
+    </clipPath>
+    <clipPath
+       id="clipPath99303">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99305" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5010_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85491" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5012_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85499" />
+    </clipPath>
+    <clipPath
+       id="clipPath99311">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99313" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5014_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-573.76801"
+         id="use85511" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5016_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-573.76801"
+         id="use85519" />
+    </clipPath>
+    <clipPath
+       id="clipPath99319">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99321" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5018_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-574.99402"
+         id="use85531" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5020_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-574.99402"
+         id="use85539" />
+    </clipPath>
+    <clipPath
+       id="clipPath99327">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99329" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5022_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85551" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5024_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85559" />
+    </clipPath>
+    <clipPath
+       id="clipPath99335">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99337" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5026_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85571" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5028_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85579" />
+    </clipPath>
+    <clipPath
+       id="clipPath99343">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99345" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5030_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-578.67499"
+         id="use85591" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5032_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-578.67499"
+         id="use85599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99351">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99353" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5034_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85611" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5036_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99359">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99361" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5038_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-581.12799"
+         id="use85631" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5040_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-581.12799"
+         id="use85639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99367">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99369" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5042_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85651" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5044_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99375">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5046_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85671" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5048_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99383">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99385" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5050_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85691" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5052_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99391">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99393" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5054_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-586.03497"
+         id="use85711" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5056_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-586.03497"
+         id="use85719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99399">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99401" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5058_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85731" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5060_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85739" />
+    </clipPath>
+    <clipPath
+       id="clipPath99407">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5062_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-588.48901"
+         id="use85751" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5064_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-588.48901"
+         id="use85759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99415">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99417" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5066_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5068_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85779" />
+    </clipPath>
+    <clipPath
+       id="clipPath99423">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99425" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5070_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85791" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5072_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99431">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99433" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5074_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85811" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5076_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99439">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99441" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5078_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5080_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85839" />
+    </clipPath>
+    <clipPath
+       id="clipPath99447">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99449" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5082_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-594.62299"
+         id="use85851" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5084_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-594.62299"
+         id="use85859" />
+    </clipPath>
+    <clipPath
+       id="clipPath99455">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99457" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5086_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85871" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5088_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85879" />
+    </clipPath>
+    <clipPath
+       id="clipPath99463">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99465" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5090_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5092_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99471">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99473" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5094_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5096_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99479">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99481" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5098_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5100_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99487">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99489" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5102_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5104_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85959" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use84798" />
+    </clipPath>
+    <clipPath
+       id="clipPath99497">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99499" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4874_"
+         overflow="visible"
+         id="use84810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_"
+         overflow="visible"
+         id="use84818" />
+    </clipPath>
+    <clipPath
+       id="clipPath99511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99513" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4879_"
+         overflow="visible"
+         id="use84841" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_"
+         overflow="visible"
+         id="use84849" />
+    </clipPath>
+    <clipPath
+       id="clipPath99519">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99521" />
+    </clipPath>
+    <clipPath
+       id="clipPath99523">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99525" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_">
+      <polygon
+         points="-640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 -654.922,278.632 "
+         id="use84771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99536" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4864_"
+         overflow="visible"
+         id="use84732" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use84720" />
+    </clipPath>
+    <clipPath
+       id="clipPath99545">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use99547" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4861_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205299"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84696" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84698" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4860_">
+      <polygon
+         points="-521.64,278.632 -507.673,278.632 -507.673,273.567 -521.64,273.567 -523.432,276.108 "
+         id="use84693" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4858_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84681" />
+    </clipPath>
+    <clipPath
+       id="clipPath99556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99558" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_">
+      <polygon
+         points="-521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 "
+         id="use84656" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644" />
+    </clipPath>
+    <clipPath
+       id="clipPath99567">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99569" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99580" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500" />
+    </clipPath>
+    <clipPath
+       id="clipPath99589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99591" />
+    </clipPath>
+    <clipPath
+       id="clipPath99593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99595" />
+    </clipPath>
+    <clipPath
+       id="clipPath99597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99601">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99603" />
+    </clipPath>
+    <clipPath
+       id="clipPath99605">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99609">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99611" />
+    </clipPath>
+    <clipPath
+       id="clipPath99613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99615" />
+    </clipPath>
+    <clipPath
+       id="clipPath99617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99623" />
+    </clipPath>
+    <clipPath
+       id="clipPath99625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99627" />
+    </clipPath>
+    <clipPath
+       id="clipPath99629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99631" />
+    </clipPath>
+    <clipPath
+       id="clipPath99633">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99635" />
+    </clipPath>
+    <clipPath
+       id="clipPath99637">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99641">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99643" />
+    </clipPath>
+    <clipPath
+       id="clipPath99645">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99647" />
+    </clipPath>
+    <clipPath
+       id="clipPath99649">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99651" />
+    </clipPath>
+    <clipPath
+       id="clipPath99653">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99655" />
+    </clipPath>
+    <clipPath
+       id="clipPath99657">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99661">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99663" />
+    </clipPath>
+    <clipPath
+       id="clipPath99665">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99667" />
+    </clipPath>
+    <clipPath
+       id="clipPath99669">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99671" />
+    </clipPath>
+    <clipPath
+       id="clipPath99673">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99675" />
+    </clipPath>
+    <clipPath
+       id="clipPath99677">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99681">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99683" />
+    </clipPath>
+    <clipPath
+       id="clipPath99685">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99687" />
+    </clipPath>
+    <clipPath
+       id="clipPath99689">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99691" />
+    </clipPath>
+    <clipPath
+       id="clipPath99693">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99695" />
+    </clipPath>
+    <clipPath
+       id="clipPath99697">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99701">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99703" />
+    </clipPath>
+    <clipPath
+       id="clipPath99705">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99707" />
+    </clipPath>
+    <clipPath
+       id="clipPath99709">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99711" />
+    </clipPath>
+    <clipPath
+       id="clipPath99713">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99715" />
+    </clipPath>
+    <clipPath
+       id="clipPath99717">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99723" />
+    </clipPath>
+    <clipPath
+       id="clipPath99725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99727" />
+    </clipPath>
+    <clipPath
+       id="clipPath99729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99731" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445" />
+    </clipPath>
+    <clipPath
+       id="clipPath99743">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99745" />
+    </clipPath>
+    <clipPath
+       id="clipPath99747">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99749" />
+    </clipPath>
+    <clipPath
+       id="clipPath99751">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99753" />
+    </clipPath>
+    <clipPath
+       id="clipPath99755">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99757" />
+    </clipPath>
+    <clipPath
+       id="clipPath99759">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99761" />
+    </clipPath>
+    <clipPath
+       id="clipPath99763">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99765" />
+    </clipPath>
+    <clipPath
+       id="clipPath99767">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99769" />
+    </clipPath>
+    <clipPath
+       id="clipPath99771">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99773" />
+    </clipPath>
+    <clipPath
+       id="clipPath99775">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99777" />
+    </clipPath>
+    <clipPath
+       id="clipPath99779">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99781" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390" />
+    </clipPath>
+    <clipPath
+       id="clipPath99793">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99795" />
+    </clipPath>
+    <clipPath
+       id="clipPath99797">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99801">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99803" />
+    </clipPath>
+    <clipPath
+       id="clipPath99805">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99807" />
+    </clipPath>
+    <clipPath
+       id="clipPath99809">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99811" />
+    </clipPath>
+    <clipPath
+       id="clipPath99813">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99815" />
+    </clipPath>
+    <clipPath
+       id="clipPath99817">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99821">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99823" />
+    </clipPath>
+    <clipPath
+       id="clipPath99825">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99827" />
+    </clipPath>
+    <clipPath
+       id="clipPath99829">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99831" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_">
+      <polygon
+         points="-636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 -636.356,251.283 "
+         id="use84326" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4823_"
+         overflow="visible"
+         id="use84295" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4822_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.317"
+       y1="2.8515"
+       x2="146.317"
+       y2="2.8515"
+       gradientTransform="matrix(0.0299,24.8863,24.8863,-0.0299,-601.3687,-3384.4197)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84267" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84269" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84271" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84273" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4821_">
+      <polygon
+         points="-503.122,251.283 -503.124,231.934 -548.973,231.929 -548.973,251.221 -526.108,256.788 "
+         id="use84264" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4819_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84232" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4817_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84220" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4816_">
+      <path
+         d="m -516.985,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84217"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4814_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84197" />
+    </clipPath>
+    <clipPath
+       id="clipPath99874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99876" />
+    </clipPath>
+    <clipPath
+       id="clipPath99878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99880" />
+    </clipPath>
+    <clipPath
+       id="clipPath99882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99884" />
+    </clipPath>
+    <clipPath
+       id="clipPath99886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99888" />
+    </clipPath>
+    <clipPath
+       id="clipPath99890">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99892" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4812_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84185" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4811_">
+      <path
+         d="m -528.956,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84182"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4809_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84162" />
+    </clipPath>
+    <clipPath
+       id="clipPath99901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99903" />
+    </clipPath>
+    <clipPath
+       id="clipPath99905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99907" />
+    </clipPath>
+    <clipPath
+       id="clipPath99909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99911" />
+    </clipPath>
+    <clipPath
+       id="clipPath99913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99915" />
+    </clipPath>
+    <clipPath
+       id="clipPath99917">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99919" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4807_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84150" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84152" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4806_">
+      <path
+         d="m -540.927,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84147"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4804_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84127" />
+    </clipPath>
+    <clipPath
+       id="clipPath99928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99930" />
+    </clipPath>
+    <clipPath
+       id="clipPath99932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99934" />
+    </clipPath>
+    <clipPath
+       id="clipPath99936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99938" />
+    </clipPath>
+    <clipPath
+       id="clipPath99940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99942" />
+    </clipPath>
+    <clipPath
+       id="clipPath99944">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99946" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4802_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84115" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84117" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4801_">
+      <path
+         d="m -552.899,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84112"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4799_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84092" />
+    </clipPath>
+    <clipPath
+       id="clipPath99955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99957" />
+    </clipPath>
+    <clipPath
+       id="clipPath99959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99961" />
+    </clipPath>
+    <clipPath
+       id="clipPath99963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99965" />
+    </clipPath>
+    <clipPath
+       id="clipPath99967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99969" />
+    </clipPath>
+    <clipPath
+       id="clipPath99971">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99973" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4797_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,237.034,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84080" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84082" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4796_">
+      <path
+         d="m -564.87,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84077"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4794_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84057" />
+    </clipPath>
+    <clipPath
+       id="clipPath99982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99984" />
+    </clipPath>
+    <clipPath
+       id="clipPath99986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99988" />
+    </clipPath>
+    <clipPath
+       id="clipPath99990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99992" />
+    </clipPath>
+    <clipPath
+       id="clipPath99994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99996" />
+    </clipPath>
+    <clipPath
+       id="clipPath99998">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100000" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4792_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84045" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84047" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4791_">
+      <path
+         d="m -576.842,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84042"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4789_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84022" />
+    </clipPath>
+    <clipPath
+       id="clipPath100009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100011" />
+    </clipPath>
+    <clipPath
+       id="clipPath100013">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100015" />
+    </clipPath>
+    <clipPath
+       id="clipPath100017">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100019" />
+    </clipPath>
+    <clipPath
+       id="clipPath100021">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100023" />
+    </clipPath>
+    <clipPath
+       id="clipPath100025">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100027" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_"
+         overflow="visible"
+         id="use84007" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use83987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100036">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100038" />
+    </clipPath>
+    <clipPath
+       id="clipPath100040">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100044">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100046" />
+    </clipPath>
+    <clipPath
+       id="clipPath100048">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100050" />
+    </clipPath>
+    <clipPath
+       id="clipPath100052">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100054" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_"
+         overflow="visible"
+         id="use83972" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use83952" />
+    </clipPath>
+    <clipPath
+       id="clipPath100063">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100065" />
+    </clipPath>
+    <clipPath
+       id="clipPath100067">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100069" />
+    </clipPath>
+    <clipPath
+       id="clipPath100071">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100073" />
+    </clipPath>
+    <clipPath
+       id="clipPath100075">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100079">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100081" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_"
+         overflow="visible"
+         id="use83937" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use83917" />
+    </clipPath>
+    <clipPath
+       id="clipPath100090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100092" />
+    </clipPath>
+    <clipPath
+       id="clipPath100094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100096" />
+    </clipPath>
+    <clipPath
+       id="clipPath100098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100100" />
+    </clipPath>
+    <clipPath
+       id="clipPath100102">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100104" />
+    </clipPath>
+    <clipPath
+       id="clipPath100106">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100108" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_"
+         overflow="visible"
+         id="use83902" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use83882" />
+    </clipPath>
+    <clipPath
+       id="clipPath100117">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100119" />
+    </clipPath>
+    <clipPath
+       id="clipPath100121">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100123" />
+    </clipPath>
+    <clipPath
+       id="clipPath100125">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100127" />
+    </clipPath>
+    <clipPath
+       id="clipPath100129">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100131" />
+    </clipPath>
+    <clipPath
+       id="clipPath100133">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100135" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_"
+         overflow="visible"
+         id="use83867" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use83847" />
+    </clipPath>
+    <clipPath
+       id="clipPath100144">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100146" />
+    </clipPath>
+    <clipPath
+       id="clipPath100148">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100150" />
+    </clipPath>
+    <clipPath
+       id="clipPath100152">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100154" />
+    </clipPath>
+    <clipPath
+       id="clipPath100156">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100158" />
+    </clipPath>
+    <clipPath
+       id="clipPath100160">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100162" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_"
+         overflow="visible"
+         id="use83832" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use83812" />
+    </clipPath>
+    <clipPath
+       id="clipPath100171">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100173" />
+    </clipPath>
+    <clipPath
+       id="clipPath100175">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100177" />
+    </clipPath>
+    <clipPath
+       id="clipPath100179">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100181" />
+    </clipPath>
+    <clipPath
+       id="clipPath100183">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100185" />
+    </clipPath>
+    <clipPath
+       id="clipPath100187">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100189" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_"
+         overflow="visible"
+         id="use83797" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use83777" />
+    </clipPath>
+    <clipPath
+       id="clipPath100198">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100200" />
+    </clipPath>
+    <clipPath
+       id="clipPath100202">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100204" />
+    </clipPath>
+    <clipPath
+       id="clipPath100206">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100208" />
+    </clipPath>
+    <clipPath
+       id="clipPath100210">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100212" />
+    </clipPath>
+    <clipPath
+       id="clipPath100214">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100216" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_"
+         overflow="visible"
+         id="use83762" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use83742" />
+    </clipPath>
+    <clipPath
+       id="clipPath100225">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100227" />
+    </clipPath>
+    <clipPath
+       id="clipPath100229">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100231" />
+    </clipPath>
+    <clipPath
+       id="clipPath100233">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100235" />
+    </clipPath>
+    <clipPath
+       id="clipPath100237">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100239" />
+    </clipPath>
+    <clipPath
+       id="clipPath100241">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100243" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4747_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83730" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83732" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4746_">
+      <path
+         d="m -516.985,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83727"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4744_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83707" />
+    </clipPath>
+    <clipPath
+       id="clipPath100252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100254" />
+    </clipPath>
+    <clipPath
+       id="clipPath100256">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100258" />
+    </clipPath>
+    <clipPath
+       id="clipPath100260">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100262" />
+    </clipPath>
+    <clipPath
+       id="clipPath100264">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100266" />
+    </clipPath>
+    <clipPath
+       id="clipPath100268">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100270" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4742_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83695" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83697" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4741_">
+      <path
+         d="m -528.956,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83692"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4739_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83672" />
+    </clipPath>
+    <clipPath
+       id="clipPath100279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100281" />
+    </clipPath>
+    <clipPath
+       id="clipPath100283">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100285" />
+    </clipPath>
+    <clipPath
+       id="clipPath100287">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100289" />
+    </clipPath>
+    <clipPath
+       id="clipPath100291">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100293" />
+    </clipPath>
+    <clipPath
+       id="clipPath100295">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4737_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83660" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83662" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4736_">
+      <path
+         d="m -540.927,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83657"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4734_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83637" />
+    </clipPath>
+    <clipPath
+       id="clipPath100306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100308" />
+    </clipPath>
+    <clipPath
+       id="clipPath100310">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100312" />
+    </clipPath>
+    <clipPath
+       id="clipPath100314">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100316" />
+    </clipPath>
+    <clipPath
+       id="clipPath100318">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100320" />
+    </clipPath>
+    <clipPath
+       id="clipPath100322">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4732_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83625" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83627" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4731_">
+      <path
+         d="m -552.899,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83622"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4729_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83602" />
+    </clipPath>
+    <clipPath
+       id="clipPath100333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100335" />
+    </clipPath>
+    <clipPath
+       id="clipPath100337">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100339" />
+    </clipPath>
+    <clipPath
+       id="clipPath100341">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100343" />
+    </clipPath>
+    <clipPath
+       id="clipPath100345">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100347" />
+    </clipPath>
+    <clipPath
+       id="clipPath100349">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4727_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83590" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83592" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4726_">
+      <path
+         d="m -576.842,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83587"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4724_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83567" />
+    </clipPath>
+    <clipPath
+       id="clipPath100360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100362" />
+    </clipPath>
+    <clipPath
+       id="clipPath100364">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100366" />
+    </clipPath>
+    <clipPath
+       id="clipPath100368">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100370" />
+    </clipPath>
+    <clipPath
+       id="clipPath100372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100374" />
+    </clipPath>
+    <clipPath
+       id="clipPath100376">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_"
+         overflow="visible"
+         id="use83552" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use83532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100389" />
+    </clipPath>
+    <clipPath
+       id="clipPath100391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100393" />
+    </clipPath>
+    <clipPath
+       id="clipPath100395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100397" />
+    </clipPath>
+    <clipPath
+       id="clipPath100399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100401" />
+    </clipPath>
+    <clipPath
+       id="clipPath100403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_"
+         overflow="visible"
+         id="use83517" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use83497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100416" />
+    </clipPath>
+    <clipPath
+       id="clipPath100418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100420" />
+    </clipPath>
+    <clipPath
+       id="clipPath100422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100424" />
+    </clipPath>
+    <clipPath
+       id="clipPath100426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100428" />
+    </clipPath>
+    <clipPath
+       id="clipPath100430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_"
+         overflow="visible"
+         id="use83482" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use83462" />
+    </clipPath>
+    <clipPath
+       id="clipPath100441">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100443" />
+    </clipPath>
+    <clipPath
+       id="clipPath100445">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100447" />
+    </clipPath>
+    <clipPath
+       id="clipPath100449">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100451" />
+    </clipPath>
+    <clipPath
+       id="clipPath100453">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100455" />
+    </clipPath>
+    <clipPath
+       id="clipPath100457">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100459" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_"
+         overflow="visible"
+         id="use83447" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use83427" />
+    </clipPath>
+    <clipPath
+       id="clipPath100468">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100470" />
+    </clipPath>
+    <clipPath
+       id="clipPath100472">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100474" />
+    </clipPath>
+    <clipPath
+       id="clipPath100476">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100478" />
+    </clipPath>
+    <clipPath
+       id="clipPath100480">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100482" />
+    </clipPath>
+    <clipPath
+       id="clipPath100484">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100486" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_"
+         overflow="visible"
+         id="use83412" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use83392" />
+    </clipPath>
+    <clipPath
+       id="clipPath100495">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100499">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100501" />
+    </clipPath>
+    <clipPath
+       id="clipPath100503">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100505" />
+    </clipPath>
+    <clipPath
+       id="clipPath100507">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100509" />
+    </clipPath>
+    <clipPath
+       id="clipPath100511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100513" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_"
+         overflow="visible"
+         id="use83377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use83357" />
+    </clipPath>
+    <clipPath
+       id="clipPath100522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100524" />
+    </clipPath>
+    <clipPath
+       id="clipPath100526">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100528" />
+    </clipPath>
+    <clipPath
+       id="clipPath100530">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100536" />
+    </clipPath>
+    <clipPath
+       id="clipPath100538">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100540" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_"
+         overflow="visible"
+         id="use83342" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use83322" />
+    </clipPath>
+    <clipPath
+       id="clipPath100549">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100551" />
+    </clipPath>
+    <clipPath
+       id="clipPath100553">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100555" />
+    </clipPath>
+    <clipPath
+       id="clipPath100557">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100559" />
+    </clipPath>
+    <clipPath
+       id="clipPath100561">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100563" />
+    </clipPath>
+    <clipPath
+       id="clipPath100565">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100567" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4687_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83310" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83312" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4686_">
+      <path
+         d="m -516.985,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83307"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4684_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83287" />
+    </clipPath>
+    <clipPath
+       id="clipPath100576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100578" />
+    </clipPath>
+    <clipPath
+       id="clipPath100580">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100582" />
+    </clipPath>
+    <clipPath
+       id="clipPath100584">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100586" />
+    </clipPath>
+    <clipPath
+       id="clipPath100588">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100590" />
+    </clipPath>
+    <clipPath
+       id="clipPath100592">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100594" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4682_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83275" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83277" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4681_">
+      <path
+         d="m -528.956,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83272"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4679_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83252" />
+    </clipPath>
+    <clipPath
+       id="clipPath100603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100605" />
+    </clipPath>
+    <clipPath
+       id="clipPath100607">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100609" />
+    </clipPath>
+    <clipPath
+       id="clipPath100611">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100613" />
+    </clipPath>
+    <clipPath
+       id="clipPath100615">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100617" />
+    </clipPath>
+    <clipPath
+       id="clipPath100619">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100621" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4677_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83240" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83242" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4676_">
+      <path
+         d="m -540.927,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83237"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4674_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83217" />
+    </clipPath>
+    <clipPath
+       id="clipPath100630">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100632" />
+    </clipPath>
+    <clipPath
+       id="clipPath100634">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100636" />
+    </clipPath>
+    <clipPath
+       id="clipPath100638">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100640" />
+    </clipPath>
+    <clipPath
+       id="clipPath100642">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100644" />
+    </clipPath>
+    <clipPath
+       id="clipPath100646">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100648" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4672_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83205" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83207" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4671_">
+      <path
+         d="m -552.899,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83202"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4669_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83182" />
+    </clipPath>
+    <clipPath
+       id="clipPath100657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100659" />
+    </clipPath>
+    <clipPath
+       id="clipPath100661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100663" />
+    </clipPath>
+    <clipPath
+       id="clipPath100665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100667" />
+    </clipPath>
+    <clipPath
+       id="clipPath100669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100671" />
+    </clipPath>
+    <clipPath
+       id="clipPath100673">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100675" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4667_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83170" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83172" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4666_">
+      <path
+         d="m -564.87,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83167"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4664_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83147" />
+    </clipPath>
+    <clipPath
+       id="clipPath100684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100686" />
+    </clipPath>
+    <clipPath
+       id="clipPath100688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100690" />
+    </clipPath>
+    <clipPath
+       id="clipPath100692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100694" />
+    </clipPath>
+    <clipPath
+       id="clipPath100696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100698" />
+    </clipPath>
+    <clipPath
+       id="clipPath100700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100702" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4662_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83135" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83137" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4661_">
+      <path
+         d="m -576.842,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83132"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4659_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83112" />
+    </clipPath>
+    <clipPath
+       id="clipPath100711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100713" />
+    </clipPath>
+    <clipPath
+       id="clipPath100715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100717" />
+    </clipPath>
+    <clipPath
+       id="clipPath100719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100721" />
+    </clipPath>
+    <clipPath
+       id="clipPath100723">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100725" />
+    </clipPath>
+    <clipPath
+       id="clipPath100727">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100729" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_"
+         overflow="visible"
+         id="use83097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use83077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100738">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100740" />
+    </clipPath>
+    <clipPath
+       id="clipPath100742">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100744" />
+    </clipPath>
+    <clipPath
+       id="clipPath100746">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100748" />
+    </clipPath>
+    <clipPath
+       id="clipPath100750">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100752" />
+    </clipPath>
+    <clipPath
+       id="clipPath100754">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100756" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_"
+         overflow="visible"
+         id="use83062" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use83042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100765">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100767" />
+    </clipPath>
+    <clipPath
+       id="clipPath100769">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100771" />
+    </clipPath>
+    <clipPath
+       id="clipPath100773">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100775" />
+    </clipPath>
+    <clipPath
+       id="clipPath100777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100779" />
+    </clipPath>
+    <clipPath
+       id="clipPath100781">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100783" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_"
+         overflow="visible"
+         id="use83027" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use83007" />
+    </clipPath>
+    <clipPath
+       id="clipPath100792">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100794" />
+    </clipPath>
+    <clipPath
+       id="clipPath100796">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100798" />
+    </clipPath>
+    <clipPath
+       id="clipPath100800">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100802" />
+    </clipPath>
+    <clipPath
+       id="clipPath100804">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100806" />
+    </clipPath>
+    <clipPath
+       id="clipPath100808">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_"
+         overflow="visible"
+         id="use82992" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use82972" />
+    </clipPath>
+    <clipPath
+       id="clipPath100819">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100821" />
+    </clipPath>
+    <clipPath
+       id="clipPath100823">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100825" />
+    </clipPath>
+    <clipPath
+       id="clipPath100827">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100829" />
+    </clipPath>
+    <clipPath
+       id="clipPath100831">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100833" />
+    </clipPath>
+    <clipPath
+       id="clipPath100835">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100837" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_"
+         overflow="visible"
+         id="use82957" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use82937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100846">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100848" />
+    </clipPath>
+    <clipPath
+       id="clipPath100850">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100852" />
+    </clipPath>
+    <clipPath
+       id="clipPath100854">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100856" />
+    </clipPath>
+    <clipPath
+       id="clipPath100858">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100860" />
+    </clipPath>
+    <clipPath
+       id="clipPath100862">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100864" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_"
+         overflow="visible"
+         id="use82922" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use82902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100873">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100875" />
+    </clipPath>
+    <clipPath
+       id="clipPath100877">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100879" />
+    </clipPath>
+    <clipPath
+       id="clipPath100881">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100883" />
+    </clipPath>
+    <clipPath
+       id="clipPath100885">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100887" />
+    </clipPath>
+    <clipPath
+       id="clipPath100889">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100891" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4625_"
+         overflow="visible"
+         id="use82887" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use82867" />
+    </clipPath>
+    <clipPath
+       id="clipPath100900">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100904">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100906" />
+    </clipPath>
+    <clipPath
+       id="clipPath100908">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100910" />
+    </clipPath>
+    <clipPath
+       id="clipPath100912">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100914" />
+    </clipPath>
+    <clipPath
+       id="clipPath100916">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100918" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4622_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82855" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82857" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4621_">
+      <path
+         d="m -516.985,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82852"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4619_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82832" />
+    </clipPath>
+    <clipPath
+       id="clipPath100927">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100929" />
+    </clipPath>
+    <clipPath
+       id="clipPath100931">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100933" />
+    </clipPath>
+    <clipPath
+       id="clipPath100935">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100941" />
+    </clipPath>
+    <clipPath
+       id="clipPath100943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100945" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4617_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82820" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82822" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4616_">
+      <path
+         d="m -528.956,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82817"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4614_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82797" />
+    </clipPath>
+    <clipPath
+       id="clipPath100954">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100956" />
+    </clipPath>
+    <clipPath
+       id="clipPath100958">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100960" />
+    </clipPath>
+    <clipPath
+       id="clipPath100962">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100964" />
+    </clipPath>
+    <clipPath
+       id="clipPath100966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100968" />
+    </clipPath>
+    <clipPath
+       id="clipPath100970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100972" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4612_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82785" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82787" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4611_">
+      <path
+         d="m -540.927,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82782"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4609_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82762" />
+    </clipPath>
+    <clipPath
+       id="clipPath100981">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100983" />
+    </clipPath>
+    <clipPath
+       id="clipPath100985">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100989">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100991" />
+    </clipPath>
+    <clipPath
+       id="clipPath100993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100995" />
+    </clipPath>
+    <clipPath
+       id="clipPath100997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100999" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4607_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82750" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82752" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4606_">
+      <path
+         d="m -552.899,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82747"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4604_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82727" />
+    </clipPath>
+    <clipPath
+       id="clipPath101008">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101012">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101014" />
+    </clipPath>
+    <clipPath
+       id="clipPath101016">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101018" />
+    </clipPath>
+    <clipPath
+       id="clipPath101020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101022" />
+    </clipPath>
+    <clipPath
+       id="clipPath101024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101026" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4602_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82715" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82717" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4601_">
+      <path
+         d="m -564.87,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82712"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4599_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82692" />
+    </clipPath>
+    <clipPath
+       id="clipPath101035">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101037" />
+    </clipPath>
+    <clipPath
+       id="clipPath101039">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101041" />
+    </clipPath>
+    <clipPath
+       id="clipPath101043">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101045" />
+    </clipPath>
+    <clipPath
+       id="clipPath101047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101049" />
+    </clipPath>
+    <clipPath
+       id="clipPath101051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101053" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4597_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82680" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82682" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4596_">
+      <path
+         d="m -576.842,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82677"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4594_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82657" />
+    </clipPath>
+    <clipPath
+       id="clipPath101062">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101064" />
+    </clipPath>
+    <clipPath
+       id="clipPath101066">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101068" />
+    </clipPath>
+    <clipPath
+       id="clipPath101070">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101072" />
+    </clipPath>
+    <clipPath
+       id="clipPath101074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101076" />
+    </clipPath>
+    <clipPath
+       id="clipPath101078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101080" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_"
+         overflow="visible"
+         id="use82642" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use82622" />
+    </clipPath>
+    <clipPath
+       id="clipPath101089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101091" />
+    </clipPath>
+    <clipPath
+       id="clipPath101093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101095" />
+    </clipPath>
+    <clipPath
+       id="clipPath101097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101099" />
+    </clipPath>
+    <clipPath
+       id="clipPath101101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101103" />
+    </clipPath>
+    <clipPath
+       id="clipPath101105">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101107" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_"
+         overflow="visible"
+         id="use82607" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use82587" />
+    </clipPath>
+    <clipPath
+       id="clipPath101116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101118" />
+    </clipPath>
+    <clipPath
+       id="clipPath101120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101122" />
+    </clipPath>
+    <clipPath
+       id="clipPath101124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101126" />
+    </clipPath>
+    <clipPath
+       id="clipPath101128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101130" />
+    </clipPath>
+    <clipPath
+       id="clipPath101132">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101134" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_"
+         overflow="visible"
+         id="use82572" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use82552" />
+    </clipPath>
+    <clipPath
+       id="clipPath101143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101145" />
+    </clipPath>
+    <clipPath
+       id="clipPath101147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101149" />
+    </clipPath>
+    <clipPath
+       id="clipPath101151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101153" />
+    </clipPath>
+    <clipPath
+       id="clipPath101155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101157" />
+    </clipPath>
+    <clipPath
+       id="clipPath101159">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101161" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_"
+         overflow="visible"
+         id="use82537" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use82517" />
+    </clipPath>
+    <clipPath
+       id="clipPath101170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101172" />
+    </clipPath>
+    <clipPath
+       id="clipPath101174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101176" />
+    </clipPath>
+    <clipPath
+       id="clipPath101178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101180" />
+    </clipPath>
+    <clipPath
+       id="clipPath101182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101184" />
+    </clipPath>
+    <clipPath
+       id="clipPath101186">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101188" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_"
+         overflow="visible"
+         id="use82502" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482" />
+    </clipPath>
+    <clipPath
+       id="clipPath101197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101199" />
+    </clipPath>
+    <clipPath
+       id="clipPath101201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101203" />
+    </clipPath>
+    <clipPath
+       id="clipPath101205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101207" />
+    </clipPath>
+    <clipPath
+       id="clipPath101209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101211" />
+    </clipPath>
+    <clipPath
+       id="clipPath101213">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101215" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_"
+         overflow="visible"
+         id="use82467" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447" />
+    </clipPath>
+    <clipPath
+       id="clipPath101224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101226" />
+    </clipPath>
+    <clipPath
+       id="clipPath101228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101230" />
+    </clipPath>
+    <clipPath
+       id="clipPath101232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101234" />
+    </clipPath>
+    <clipPath
+       id="clipPath101236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101238" />
+    </clipPath>
+    <clipPath
+       id="clipPath101240">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101242" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101251">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101253" />
+    </clipPath>
+    <clipPath
+       id="clipPath101255">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101257" />
+    </clipPath>
+    <clipPath
+       id="clipPath101259">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101261" />
+    </clipPath>
+    <clipPath
+       id="clipPath101263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101265" />
+    </clipPath>
+    <clipPath
+       id="clipPath101267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101269" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_"
+         overflow="visible"
+         id="use82397" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4554_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82373" />
+    </clipPath>
+    <clipPath
+       id="clipPath101278">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101280" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use82359" />
+    </clipPath>
+    <clipPath
+       id="clipPath101284">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use101286" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use82345" />
+    </clipPath>
+    <clipPath
+       id="clipPath101290">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use101292" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_">
+      <polygon
+         points="-637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 -680.004,267.353 "
+         id="use82316" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4543_"
+         overflow="visible"
+         id="use82287" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4542_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1387.9897,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82261" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82263" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82265" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82267" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82269" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82271" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82273" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4541_">
+      <polygon
+         points="-547.185,267.353 -504.831,267.32 -504.824,253.133 -526.362,258.293 -547.178,253.224 "
+         id="use82258" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use82164" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use82174" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_"
+         overflow="visible"
+         id="use82182" />
+    </clipPath>
+    <clipPath
+       id="clipPath101336">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use101338" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_"
+         overflow="visible"
+         id="use82197" />
+    </clipPath>
+    <clipPath
+       id="clipPath101342">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use101344" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use82209" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_"
+         overflow="visible"
+         id="use82217" />
+    </clipPath>
+    <clipPath
+       id="clipPath101353">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use101355" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_"
+         overflow="visible"
+         id="use82232" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132" />
+    </clipPath>
+    <clipPath
+       id="clipPath101361">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101363" />
+    </clipPath>
+    <clipPath
+       id="clipPath101365">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101367" />
+    </clipPath>
+    <clipPath
+       id="clipPath101369">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101371" />
+    </clipPath>
+    <clipPath
+       id="clipPath101373">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101375" />
+    </clipPath>
+    <clipPath
+       id="clipPath101377">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101379" />
+    </clipPath>
+    <clipPath
+       id="clipPath101381">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101383" />
+    </clipPath>
+    <clipPath
+       id="clipPath101385">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101387" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_">
+      <polygon
+         points="-547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 "
+         id="use82113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083" />
+    </clipPath>
+    <clipPath
+       id="clipPath101398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101400" />
+    </clipPath>
+    <clipPath
+       id="clipPath101402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101404" />
+    </clipPath>
+    <clipPath
+       id="clipPath101406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101408" />
+    </clipPath>
+    <clipPath
+       id="clipPath101410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101416" />
+    </clipPath>
+    <clipPath
+       id="clipPath101418">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101420" />
+    </clipPath>
+    <clipPath
+       id="clipPath101422">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101424" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101435">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101437" />
+    </clipPath>
+    <clipPath
+       id="clipPath101439">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101441" />
+    </clipPath>
+    <clipPath
+       id="clipPath101443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101445" />
+    </clipPath>
+    <clipPath
+       id="clipPath101447">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101449" />
+    </clipPath>
+    <clipPath
+       id="clipPath101451">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101453" />
+    </clipPath>
+    <clipPath
+       id="clipPath101455">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101457" />
+    </clipPath>
+    <clipPath
+       id="clipPath101459">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101461" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4509_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534"
+       xlink:href="#SVGID_4509_"
+       inkscape:collect="always" />
+    <defs
+       id="defs84496">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4842_" />
+    </defs>
+    <clipPath
+       id="clipPath103391">
+      <use
+         id="use103393"
+         overflow="visible"
+         xlink:href="#SVGID_4842_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113428">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113430" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-9">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-2" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-4" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-9" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-0">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113442">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113444" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-0">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-9">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113450">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113452" />
+    </clipPath>
+    <clipPath
+       id="clipPath113454">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113456" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-6">
+      <polygon
+         points="-654.922,273.567 -656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 "
+         id="use84771-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113465">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113467" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-6">
+      <polygon
+         points="-588.288,273.567 -590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 "
+         id="use84732-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113476">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113478" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-1">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113489" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-3">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113498">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113500" />
+    </clipPath>
+    <clipPath
+       id="clipPath113502">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113504" />
+    </clipPath>
+    <clipPath
+       id="clipPath113506">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113508" />
+    </clipPath>
+    <clipPath
+       id="clipPath113510">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113512" />
+    </clipPath>
+    <clipPath
+       id="clipPath113514">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113516" />
+    </clipPath>
+    <clipPath
+       id="clipPath113518">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113520" />
+    </clipPath>
+    <clipPath
+       id="clipPath113522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113524" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113528">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113530" />
+    </clipPath>
+    <clipPath
+       id="clipPath113532">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113534" />
+    </clipPath>
+    <clipPath
+       id="clipPath113536">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113538" />
+    </clipPath>
+    <clipPath
+       id="clipPath113540">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113542" />
+    </clipPath>
+    <clipPath
+       id="clipPath113544">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113546" />
+    </clipPath>
+    <clipPath
+       id="clipPath113548">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113550" />
+    </clipPath>
+    <clipPath
+       id="clipPath113552">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113554" />
+    </clipPath>
+    <clipPath
+       id="clipPath113556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113558" />
+    </clipPath>
+    <clipPath
+       id="clipPath113560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113562" />
+    </clipPath>
+    <clipPath
+       id="clipPath113564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113566" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-8" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-7" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-5">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113580" />
+    </clipPath>
+    <clipPath
+       id="clipPath113582">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113584" />
+    </clipPath>
+    <clipPath
+       id="clipPath113586">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113588" />
+    </clipPath>
+    <clipPath
+       id="clipPath113590">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113592" />
+    </clipPath>
+    <clipPath
+       id="clipPath113594">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113596" />
+    </clipPath>
+    <clipPath
+       id="clipPath113598">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113600" />
+    </clipPath>
+    <clipPath
+       id="clipPath113602">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113604" />
+    </clipPath>
+    <clipPath
+       id="clipPath113606">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113608" />
+    </clipPath>
+    <clipPath
+       id="clipPath113610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113612" />
+    </clipPath>
+    <clipPath
+       id="clipPath113614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113616" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-9" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-0" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-4" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-0" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-0" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-1" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-3">
+      <polygon
+         points="-682.208,251.221 -659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 "
+         id="use84326-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-6" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-1" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-6" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-9">
+      <polygon
+         points="-615.653,251.221 -592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 "
+         id="use84295-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-8">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113649">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113651" />
+    </clipPath>
+    <clipPath
+       id="clipPath113653">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113655" />
+    </clipPath>
+    <clipPath
+       id="clipPath113657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113659" />
+    </clipPath>
+    <clipPath
+       id="clipPath113661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113663" />
+    </clipPath>
+    <clipPath
+       id="clipPath113665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113667" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-3">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113676">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113678" />
+    </clipPath>
+    <clipPath
+       id="clipPath113680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113682" />
+    </clipPath>
+    <clipPath
+       id="clipPath113684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113686" />
+    </clipPath>
+    <clipPath
+       id="clipPath113688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113690" />
+    </clipPath>
+    <clipPath
+       id="clipPath113692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113694" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-4">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113703">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113705" />
+    </clipPath>
+    <clipPath
+       id="clipPath113707">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113709" />
+    </clipPath>
+    <clipPath
+       id="clipPath113711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113713" />
+    </clipPath>
+    <clipPath
+       id="clipPath113715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113717" />
+    </clipPath>
+    <clipPath
+       id="clipPath113719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113721" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-5">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113732" />
+    </clipPath>
+    <clipPath
+       id="clipPath113734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113736" />
+    </clipPath>
+    <clipPath
+       id="clipPath113738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113740" />
+    </clipPath>
+    <clipPath
+       id="clipPath113742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113744" />
+    </clipPath>
+    <clipPath
+       id="clipPath113746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-5">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113757">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113759" />
+    </clipPath>
+    <clipPath
+       id="clipPath113761">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113763" />
+    </clipPath>
+    <clipPath
+       id="clipPath113765">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113767" />
+    </clipPath>
+    <clipPath
+       id="clipPath113769">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113771" />
+    </clipPath>
+    <clipPath
+       id="clipPath113773">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113775" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-9">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113786" />
+    </clipPath>
+    <clipPath
+       id="clipPath113788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113790" />
+    </clipPath>
+    <clipPath
+       id="clipPath113792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113794" />
+    </clipPath>
+    <clipPath
+       id="clipPath113796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113798" />
+    </clipPath>
+    <clipPath
+       id="clipPath113800">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113802" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-5">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113811">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113813" />
+    </clipPath>
+    <clipPath
+       id="clipPath113815">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113817" />
+    </clipPath>
+    <clipPath
+       id="clipPath113819">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113821" />
+    </clipPath>
+    <clipPath
+       id="clipPath113823">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113825" />
+    </clipPath>
+    <clipPath
+       id="clipPath113827">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113829" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-0">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-5">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113845" />
+    </clipPath>
+    <clipPath
+       id="clipPath113847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113849" />
+    </clipPath>
+    <clipPath
+       id="clipPath113851">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113853" />
+    </clipPath>
+    <clipPath
+       id="clipPath113855">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113857" />
+    </clipPath>
+    <clipPath
+       id="clipPath113859">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113861" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-1">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113872" />
+    </clipPath>
+    <clipPath
+       id="clipPath113874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113876" />
+    </clipPath>
+    <clipPath
+       id="clipPath113878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113880" />
+    </clipPath>
+    <clipPath
+       id="clipPath113882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113884" />
+    </clipPath>
+    <clipPath
+       id="clipPath113886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113888" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-6">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113899" />
+    </clipPath>
+    <clipPath
+       id="clipPath113901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113903" />
+    </clipPath>
+    <clipPath
+       id="clipPath113905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113907" />
+    </clipPath>
+    <clipPath
+       id="clipPath113909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113911" />
+    </clipPath>
+    <clipPath
+       id="clipPath113913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113915" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-5">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113926" />
+    </clipPath>
+    <clipPath
+       id="clipPath113928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113930" />
+    </clipPath>
+    <clipPath
+       id="clipPath113932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113934" />
+    </clipPath>
+    <clipPath
+       id="clipPath113936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113938" />
+    </clipPath>
+    <clipPath
+       id="clipPath113940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113942" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-3">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath113951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113953" />
+    </clipPath>
+    <clipPath
+       id="clipPath113955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113957" />
+    </clipPath>
+    <clipPath
+       id="clipPath113959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113961" />
+    </clipPath>
+    <clipPath
+       id="clipPath113963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113965" />
+    </clipPath>
+    <clipPath
+       id="clipPath113967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113969" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-2">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113980" />
+    </clipPath>
+    <clipPath
+       id="clipPath113982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113984" />
+    </clipPath>
+    <clipPath
+       id="clipPath113986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113988" />
+    </clipPath>
+    <clipPath
+       id="clipPath113990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113992" />
+    </clipPath>
+    <clipPath
+       id="clipPath113994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113996" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-2">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114010">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114012" />
+    </clipPath>
+    <clipPath
+       id="clipPath114014">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114016" />
+    </clipPath>
+    <clipPath
+       id="clipPath114018">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114020" />
+    </clipPath>
+    <clipPath
+       id="clipPath114022">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114024" />
+    </clipPath>
+    <clipPath
+       id="clipPath114026">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114028" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-4">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath114037">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114039" />
+    </clipPath>
+    <clipPath
+       id="clipPath114041">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114043" />
+    </clipPath>
+    <clipPath
+       id="clipPath114045">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114047" />
+    </clipPath>
+    <clipPath
+       id="clipPath114049">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114051" />
+    </clipPath>
+    <clipPath
+       id="clipPath114053">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114055" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-1">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath114064">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114066" />
+    </clipPath>
+    <clipPath
+       id="clipPath114068">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114070" />
+    </clipPath>
+    <clipPath
+       id="clipPath114072">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114074" />
+    </clipPath>
+    <clipPath
+       id="clipPath114076">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114078" />
+    </clipPath>
+    <clipPath
+       id="clipPath114080">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114082" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-8">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114091">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114093" />
+    </clipPath>
+    <clipPath
+       id="clipPath114095">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114097" />
+    </clipPath>
+    <clipPath
+       id="clipPath114099">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114101" />
+    </clipPath>
+    <clipPath
+       id="clipPath114103">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114105" />
+    </clipPath>
+    <clipPath
+       id="clipPath114107">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114109" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-6">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath114118">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114120" />
+    </clipPath>
+    <clipPath
+       id="clipPath114122">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114124" />
+    </clipPath>
+    <clipPath
+       id="clipPath114126">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114128" />
+    </clipPath>
+    <clipPath
+       id="clipPath114130">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114132" />
+    </clipPath>
+    <clipPath
+       id="clipPath114134">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114136" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-1">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114145">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114147" />
+    </clipPath>
+    <clipPath
+       id="clipPath114149">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114151" />
+    </clipPath>
+    <clipPath
+       id="clipPath114153">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114155" />
+    </clipPath>
+    <clipPath
+       id="clipPath114157">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114159" />
+    </clipPath>
+    <clipPath
+       id="clipPath114161">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114163" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-6">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-6">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114177">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114179" />
+    </clipPath>
+    <clipPath
+       id="clipPath114181">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114183" />
+    </clipPath>
+    <clipPath
+       id="clipPath114185">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114187" />
+    </clipPath>
+    <clipPath
+       id="clipPath114189">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114191" />
+    </clipPath>
+    <clipPath
+       id="clipPath114193">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114195" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-0">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114204">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114206" />
+    </clipPath>
+    <clipPath
+       id="clipPath114208">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114210" />
+    </clipPath>
+    <clipPath
+       id="clipPath114212">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114214" />
+    </clipPath>
+    <clipPath
+       id="clipPath114216">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114218" />
+    </clipPath>
+    <clipPath
+       id="clipPath114220">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114222" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-6">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114231">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114233" />
+    </clipPath>
+    <clipPath
+       id="clipPath114235">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114237" />
+    </clipPath>
+    <clipPath
+       id="clipPath114239">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114241" />
+    </clipPath>
+    <clipPath
+       id="clipPath114243">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114245" />
+    </clipPath>
+    <clipPath
+       id="clipPath114247">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114249" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-2">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114258">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114260" />
+    </clipPath>
+    <clipPath
+       id="clipPath114262">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114264" />
+    </clipPath>
+    <clipPath
+       id="clipPath114266">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114268" />
+    </clipPath>
+    <clipPath
+       id="clipPath114270">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114272" />
+    </clipPath>
+    <clipPath
+       id="clipPath114274">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114276" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-1">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114285">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114287" />
+    </clipPath>
+    <clipPath
+       id="clipPath114289">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114291" />
+    </clipPath>
+    <clipPath
+       id="clipPath114293">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114295" />
+    </clipPath>
+    <clipPath
+       id="clipPath114297">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114299" />
+    </clipPath>
+    <clipPath
+       id="clipPath114301">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114303" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-1">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath114312">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114314" />
+    </clipPath>
+    <clipPath
+       id="clipPath114316">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114318" />
+    </clipPath>
+    <clipPath
+       id="clipPath114320">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114322" />
+    </clipPath>
+    <clipPath
+       id="clipPath114324">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114326" />
+    </clipPath>
+    <clipPath
+       id="clipPath114328">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114330" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-1">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath114339">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114341" />
+    </clipPath>
+    <clipPath
+       id="clipPath114343">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114345" />
+    </clipPath>
+    <clipPath
+       id="clipPath114347">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114349" />
+    </clipPath>
+    <clipPath
+       id="clipPath114351">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114353" />
+    </clipPath>
+    <clipPath
+       id="clipPath114355">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114357" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-5">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114366">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114368" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114374" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-4" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-6" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-0" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-7" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-0" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-5" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-4">
+      <polygon
+         points="-659.18,258.293 -679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 "
+         id="use82316-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-3" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-2" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-5" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-7" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-2" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-2">
+      <polygon
+         points="-593.085,258.293 -613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 "
+         id="use82287-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-8">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-6">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath114407">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use114409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-7">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114413">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114415" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-4">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-7" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-8">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114424">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use114426" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-9">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114432">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114434" />
+    </clipPath>
+    <clipPath
+       id="clipPath114436">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114438" />
+    </clipPath>
+    <clipPath
+       id="clipPath114440">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114442" />
+    </clipPath>
+    <clipPath
+       id="clipPath114444">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114446" />
+    </clipPath>
+    <clipPath
+       id="clipPath114448">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114450" />
+    </clipPath>
+    <clipPath
+       id="clipPath114452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114454" />
+    </clipPath>
+    <clipPath
+       id="clipPath114456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114458" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-5" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-0" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-1">
+      <polygon
+         points="-613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 -571.658,188.287 "
+         id="use82064-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114469">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114471" />
+    </clipPath>
+    <clipPath
+       id="clipPath114473">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114475" />
+    </clipPath>
+    <clipPath
+       id="clipPath114477">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114479" />
+    </clipPath>
+    <clipPath
+       id="clipPath114481">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114483" />
+    </clipPath>
+    <clipPath
+       id="clipPath114485">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114487" />
+    </clipPath>
+    <clipPath
+       id="clipPath114489">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114491" />
+    </clipPath>
+    <clipPath
+       id="clipPath114493">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114495" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-0"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-4" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-3" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-0">
+      <polygon
+         points="-680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 -638.002,188.287 "
+         id="use81991-1" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient115446"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4657_-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-2-7"
+         overflow="visible"
+         id="use83097-8-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath118798">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use118800" />
+    </clipPath>
+    <defs
+       id="defs83093-1-3">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4655_-2-7"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102313-6-0">
+      <use
+         id="use102315-6-2"
+         overflow="visible"
+         xlink:href="#SVGID_4655_-2-7"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102317-9-1">
+      <stop
+         id="stop102319-7-7"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102321-3-3"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4873_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-79" />
+    </clipPath>
+    <clipPath
+       id="clipPath119610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119612" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-5">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-4" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-2" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-99" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-79" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-3">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119624">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119626" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-1">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-6">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119632">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119634" />
+    </clipPath>
+    <clipPath
+       id="clipPath119636">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119638" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-61" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-2">
+      <polygon
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 "
+         id="use84771-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-44" />
+    </clipPath>
+    <clipPath
+       id="clipPath119647">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119649" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-0">
+      <polygon
+         points="-588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 "
+         id="use84732-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119658">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119660" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-19">
+      <polygon
+         points="-588.288,208.524 -574.322,208.524 -574.322,203.458 -588.288,203.458 -590.081,206 "
+         id="use84619-38" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119671" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-4">
+      <polygon
+         points="-654.922,208.524 -640.956,208.524 -640.956,203.458 -654.922,203.458 -656.715,206 "
+         id="use84582-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath119680">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119682" />
+    </clipPath>
+    <clipPath
+       id="clipPath119684">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119686" />
+    </clipPath>
+    <clipPath
+       id="clipPath119688">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119690" />
+    </clipPath>
+    <clipPath
+       id="clipPath119692">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119694" />
+    </clipPath>
+    <clipPath
+       id="clipPath119696">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119698" />
+    </clipPath>
+    <clipPath
+       id="clipPath119700">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119702" />
+    </clipPath>
+    <clipPath
+       id="clipPath119704">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119706" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119710">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119712" />
+    </clipPath>
+    <clipPath
+       id="clipPath119714">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119716" />
+    </clipPath>
+    <clipPath
+       id="clipPath119718">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119720" />
+    </clipPath>
+    <clipPath
+       id="clipPath119722">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119724" />
+    </clipPath>
+    <clipPath
+       id="clipPath119726">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119728" />
+    </clipPath>
+    <clipPath
+       id="clipPath119730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119732" />
+    </clipPath>
+    <clipPath
+       id="clipPath119734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119736" />
+    </clipPath>
+    <clipPath
+       id="clipPath119738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119740" />
+    </clipPath>
+    <clipPath
+       id="clipPath119742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119744" />
+    </clipPath>
+    <clipPath
+       id="clipPath119746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-5" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-1" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-3">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119760">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119762" />
+    </clipPath>
+    <clipPath
+       id="clipPath119764">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119766" />
+    </clipPath>
+    <clipPath
+       id="clipPath119768">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119770" />
+    </clipPath>
+    <clipPath
+       id="clipPath119772">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119774" />
+    </clipPath>
+    <clipPath
+       id="clipPath119776">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119778" />
+    </clipPath>
+    <clipPath
+       id="clipPath119780">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119782" />
+    </clipPath>
+    <clipPath
+       id="clipPath119784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119786" />
+    </clipPath>
+    <clipPath
+       id="clipPath119788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119790" />
+    </clipPath>
+    <clipPath
+       id="clipPath119792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119794" />
+    </clipPath>
+    <clipPath
+       id="clipPath119796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119798" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-0" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-42" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-15" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-5">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-5" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-2" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-3" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-8">
+      <polygon
+         points="-636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 "
+         id="use84326-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-1" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-7" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-5" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-6">
+      <polygon
+         points="-569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 "
+         id="use84295-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-2">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119831">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119833" />
+    </clipPath>
+    <clipPath
+       id="clipPath119835">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119837" />
+    </clipPath>
+    <clipPath
+       id="clipPath119839">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119841" />
+    </clipPath>
+    <clipPath
+       id="clipPath119843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119845" />
+    </clipPath>
+    <clipPath
+       id="clipPath119847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119849" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-1">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-45"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath119858">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119860" />
+    </clipPath>
+    <clipPath
+       id="clipPath119862">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119864" />
+    </clipPath>
+    <clipPath
+       id="clipPath119866">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119868" />
+    </clipPath>
+    <clipPath
+       id="clipPath119870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119872" />
+    </clipPath>
+    <clipPath
+       id="clipPath119874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119876" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-7">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119885">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119887" />
+    </clipPath>
+    <clipPath
+       id="clipPath119889">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119891" />
+    </clipPath>
+    <clipPath
+       id="clipPath119893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119895" />
+    </clipPath>
+    <clipPath
+       id="clipPath119897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119899" />
+    </clipPath>
+    <clipPath
+       id="clipPath119901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119903" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-2">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-70"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119912">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119914" />
+    </clipPath>
+    <clipPath
+       id="clipPath119916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119918" />
+    </clipPath>
+    <clipPath
+       id="clipPath119920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119922" />
+    </clipPath>
+    <clipPath
+       id="clipPath119924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119926" />
+    </clipPath>
+    <clipPath
+       id="clipPath119928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119930" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-91" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-6">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119941" />
+    </clipPath>
+    <clipPath
+       id="clipPath119943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119945" />
+    </clipPath>
+    <clipPath
+       id="clipPath119947">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119949" />
+    </clipPath>
+    <clipPath
+       id="clipPath119951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119953" />
+    </clipPath>
+    <clipPath
+       id="clipPath119955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119957" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-3">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119968" />
+    </clipPath>
+    <clipPath
+       id="clipPath119970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119972" />
+    </clipPath>
+    <clipPath
+       id="clipPath119974">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119976" />
+    </clipPath>
+    <clipPath
+       id="clipPath119978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119980" />
+    </clipPath>
+    <clipPath
+       id="clipPath119982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119984" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-76" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-1">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath119993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119995" />
+    </clipPath>
+    <clipPath
+       id="clipPath119997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119999" />
+    </clipPath>
+    <clipPath
+       id="clipPath120001">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120003" />
+    </clipPath>
+    <clipPath
+       id="clipPath120005">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120007" />
+    </clipPath>
+    <clipPath
+       id="clipPath120009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-6">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83742-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120022" />
+    </clipPath>
+    <clipPath
+       id="clipPath120024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120026" />
+    </clipPath>
+    <clipPath
+       id="clipPath120028">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120030" />
+    </clipPath>
+    <clipPath
+       id="clipPath120032">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120034" />
+    </clipPath>
+    <clipPath
+       id="clipPath120036">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120038" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-85" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-8">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120049" />
+    </clipPath>
+    <clipPath
+       id="clipPath120051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120053" />
+    </clipPath>
+    <clipPath
+       id="clipPath120055">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120057" />
+    </clipPath>
+    <clipPath
+       id="clipPath120059">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120061" />
+    </clipPath>
+    <clipPath
+       id="clipPath120063">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120065" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-0">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120076" />
+    </clipPath>
+    <clipPath
+       id="clipPath120078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120080" />
+    </clipPath>
+    <clipPath
+       id="clipPath120082">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120084" />
+    </clipPath>
+    <clipPath
+       id="clipPath120086">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120088" />
+    </clipPath>
+    <clipPath
+       id="clipPath120090">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120092" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-68">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120101">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120103" />
+    </clipPath>
+    <clipPath
+       id="clipPath120105">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120107" />
+    </clipPath>
+    <clipPath
+       id="clipPath120109">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120111" />
+    </clipPath>
+    <clipPath
+       id="clipPath120113">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120115" />
+    </clipPath>
+    <clipPath
+       id="clipPath120117">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120119" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-77" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-2">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120128">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120130" />
+    </clipPath>
+    <clipPath
+       id="clipPath120132">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120134" />
+    </clipPath>
+    <clipPath
+       id="clipPath120136">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120138" />
+    </clipPath>
+    <clipPath
+       id="clipPath120140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120142" />
+    </clipPath>
+    <clipPath
+       id="clipPath120144">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120146" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-5">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120155">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120157" />
+    </clipPath>
+    <clipPath
+       id="clipPath120159">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120161" />
+    </clipPath>
+    <clipPath
+       id="clipPath120163">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120165" />
+    </clipPath>
+    <clipPath
+       id="clipPath120167">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120169" />
+    </clipPath>
+    <clipPath
+       id="clipPath120171">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120173" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-62"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-20">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120182">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120184" />
+    </clipPath>
+    <clipPath
+       id="clipPath120186">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120188" />
+    </clipPath>
+    <clipPath
+       id="clipPath120190">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120192" />
+    </clipPath>
+    <clipPath
+       id="clipPath120194">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120196" />
+    </clipPath>
+    <clipPath
+       id="clipPath120198">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120200" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-7">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83322-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120209">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120211" />
+    </clipPath>
+    <clipPath
+       id="clipPath120213">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120215" />
+    </clipPath>
+    <clipPath
+       id="clipPath120217">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120219" />
+    </clipPath>
+    <clipPath
+       id="clipPath120221">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120223" />
+    </clipPath>
+    <clipPath
+       id="clipPath120225">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120227" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-7">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120236">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120238" />
+    </clipPath>
+    <clipPath
+       id="clipPath120240">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120242" />
+    </clipPath>
+    <clipPath
+       id="clipPath120244">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120246" />
+    </clipPath>
+    <clipPath
+       id="clipPath120248">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120250" />
+    </clipPath>
+    <clipPath
+       id="clipPath120252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120254" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-42" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-1">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-49" />
+    </clipPath>
+    <clipPath
+       id="clipPath120263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120265" />
+    </clipPath>
+    <clipPath
+       id="clipPath120267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120269" />
+    </clipPath>
+    <clipPath
+       id="clipPath120271">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120273" />
+    </clipPath>
+    <clipPath
+       id="clipPath120275">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120277" />
+    </clipPath>
+    <clipPath
+       id="clipPath120279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120281" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-35" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-8">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120290">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120292" />
+    </clipPath>
+    <clipPath
+       id="clipPath120294">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120296" />
+    </clipPath>
+    <clipPath
+       id="clipPath120298">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120300" />
+    </clipPath>
+    <clipPath
+       id="clipPath120302">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120304" />
+    </clipPath>
+    <clipPath
+       id="clipPath120306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120308" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-9">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-97" />
+    </clipPath>
+    <clipPath
+       id="clipPath120317">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120319" />
+    </clipPath>
+    <clipPath
+       id="clipPath120321">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120323" />
+    </clipPath>
+    <clipPath
+       id="clipPath120325">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120327" />
+    </clipPath>
+    <clipPath
+       id="clipPath120329">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120331" />
+    </clipPath>
+    <clipPath
+       id="clipPath120333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120335" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-2">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120344">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120346" />
+    </clipPath>
+    <clipPath
+       id="clipPath120348">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120350" />
+    </clipPath>
+    <clipPath
+       id="clipPath120352">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120354" />
+    </clipPath>
+    <clipPath
+       id="clipPath120356">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120358" />
+    </clipPath>
+    <clipPath
+       id="clipPath120360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120362" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-64" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-9">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-36"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120371">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120373" />
+    </clipPath>
+    <clipPath
+       id="clipPath120375">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120377" />
+    </clipPath>
+    <clipPath
+       id="clipPath120379">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120381" />
+    </clipPath>
+    <clipPath
+       id="clipPath120383">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120385" />
+    </clipPath>
+    <clipPath
+       id="clipPath120387">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120389" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-2">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82867-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120400" />
+    </clipPath>
+    <clipPath
+       id="clipPath120402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120404" />
+    </clipPath>
+    <clipPath
+       id="clipPath120406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120408" />
+    </clipPath>
+    <clipPath
+       id="clipPath120410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120412" />
+    </clipPath>
+    <clipPath
+       id="clipPath120414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120416" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-1">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120425">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120427" />
+    </clipPath>
+    <clipPath
+       id="clipPath120429">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120431" />
+    </clipPath>
+    <clipPath
+       id="clipPath120433">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120435" />
+    </clipPath>
+    <clipPath
+       id="clipPath120437">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120439" />
+    </clipPath>
+    <clipPath
+       id="clipPath120441">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120443" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-7">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120454" />
+    </clipPath>
+    <clipPath
+       id="clipPath120456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120458" />
+    </clipPath>
+    <clipPath
+       id="clipPath120460">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120462" />
+    </clipPath>
+    <clipPath
+       id="clipPath120464">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120466" />
+    </clipPath>
+    <clipPath
+       id="clipPath120468">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120470" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-03" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-7">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-65">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120479">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120481" />
+    </clipPath>
+    <clipPath
+       id="clipPath120483">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120485" />
+    </clipPath>
+    <clipPath
+       id="clipPath120487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120489" />
+    </clipPath>
+    <clipPath
+       id="clipPath120491">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120493" />
+    </clipPath>
+    <clipPath
+       id="clipPath120495">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120497" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-5">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120506">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120508" />
+    </clipPath>
+    <clipPath
+       id="clipPath120510">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120512" />
+    </clipPath>
+    <clipPath
+       id="clipPath120514">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120516" />
+    </clipPath>
+    <clipPath
+       id="clipPath120518">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120520" />
+    </clipPath>
+    <clipPath
+       id="clipPath120522">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120524" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-3">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120533">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120535" />
+    </clipPath>
+    <clipPath
+       id="clipPath120537">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120539" />
+    </clipPath>
+    <clipPath
+       id="clipPath120541">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120543" />
+    </clipPath>
+    <clipPath
+       id="clipPath120545">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120547" />
+    </clipPath>
+    <clipPath
+       id="clipPath120549">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120551" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-8">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120562" />
+    </clipPath>
+    <clipPath
+       id="clipPath120564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120566" />
+    </clipPath>
+    <clipPath
+       id="clipPath120568">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120570" />
+    </clipPath>
+    <clipPath
+       id="clipPath120572">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120574" />
+    </clipPath>
+    <clipPath
+       id="clipPath120576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120578" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-29" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-11">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120587">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120589" />
+    </clipPath>
+    <clipPath
+       id="clipPath120591">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120593" />
+    </clipPath>
+    <clipPath
+       id="clipPath120595">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120597" />
+    </clipPath>
+    <clipPath
+       id="clipPath120599">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120601" />
+    </clipPath>
+    <clipPath
+       id="clipPath120603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120605" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-7">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120616" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120620">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120622" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-6" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-70" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-00" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-8" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-67" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-3">
+      <polygon
+         points="-680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 "
+         id="use82316-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-93"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-1" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-1" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-0" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-3" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-1">
+      <polygon
+         points="-613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 "
+         id="use82287-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-9">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-42" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-4">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120655">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use120657" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-5">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120663" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-89">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120672">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use120674" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-35">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120682" />
+    </clipPath>
+    <clipPath
+       id="clipPath120684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120686" />
+    </clipPath>
+    <clipPath
+       id="clipPath120688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120690" />
+    </clipPath>
+    <clipPath
+       id="clipPath120692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120694" />
+    </clipPath>
+    <clipPath
+       id="clipPath120696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120698" />
+    </clipPath>
+    <clipPath
+       id="clipPath120700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120702" />
+    </clipPath>
+    <clipPath
+       id="clipPath120704">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120706" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-2" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-11">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-38">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120717">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120719" />
+    </clipPath>
+    <clipPath
+       id="clipPath120721">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120723" />
+    </clipPath>
+    <clipPath
+       id="clipPath120725">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120727" />
+    </clipPath>
+    <clipPath
+       id="clipPath120729">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120731" />
+    </clipPath>
+    <clipPath
+       id="clipPath120733">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120735" />
+    </clipPath>
+    <clipPath
+       id="clipPath120737">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120739" />
+    </clipPath>
+    <clipPath
+       id="clipPath120741">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120743" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-4"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-08" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-7" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-3">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991-0" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient121727"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126204"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientUnits="userSpaceOnUse" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126210"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-7"
+       id="linearGradient126210-7"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-7"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126228"
+       xlink:href="#linearGradient126200-7"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126210-9"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126435"
+       xlink:href="#linearGradient126200-2"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126487"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientTransform="translate(190.5,52.1205)" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient126520-9"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-2"
+       id="linearGradient126520-4"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4843_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2-2" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_"
+         overflow="visible"
+         id="use84656-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use84644-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath126768">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use126770" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478-8" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480-2" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482-3" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use82132-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath126782">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126784" />
+    </clipPath>
+    <clipPath
+       id="clipPath126786">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126788" />
+    </clipPath>
+    <clipPath
+       id="clipPath126790">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126792" />
+    </clipPath>
+    <clipPath
+       id="clipPath126794">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126796" />
+    </clipPath>
+    <clipPath
+       id="clipPath126798">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126800" />
+    </clipPath>
+    <clipPath
+       id="clipPath126802">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126804" />
+    </clipPath>
+    <clipPath
+       id="clipPath126806">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126808" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_"
+         overflow="visible"
+         id="use82113-2" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904"
+       xlink:href="#SVGID_4521_-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4856_-4-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-2">
+      <polygon
+         points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 "
+         id="use84656-1-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath127140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127142" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath127146">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127148" />
+    </clipPath>
+    <clipPath
+       id="clipPath127150">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127152" />
+    </clipPath>
+    <clipPath
+       id="clipPath127154">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127156" />
+    </clipPath>
+    <clipPath
+       id="clipPath127158">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127160" />
+    </clipPath>
+    <clipPath
+       id="clipPath127162">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127164" />
+    </clipPath>
+    <clipPath
+       id="clipPath127166">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127168" />
+    </clipPath>
+    <clipPath
+       id="clipPath127170">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127172" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-6"
+       xlink:href="#SVGID_4521_-3-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-3" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-4" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-3">
+      <polygon
+         points="-524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 "
+         id="use82113-2-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24">
+      <polygon
+         id="use84656-1-9"
+         points="-521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 " />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0">
+      <rect
+         id="use84644-1-6"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9068">
+      <rect
+         id="use9070"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5">
+      <rect
+         id="use82132-5-25"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9074">
+      <rect
+         id="use9076"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9078">
+      <rect
+         id="use9080"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9082">
+      <rect
+         id="use9084"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9086">
+      <rect
+         id="use9088"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9090">
+      <rect
+         id="use9092"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9094">
+      <rect
+         id="use9096"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9098">
+      <rect
+         id="use9100"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61"
+       xlink:href="#SVGID_4521_-3-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6">
+      <polygon
+         id="use82113-2-5"
+         points="-528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 " />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9382"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(28.50002,64.927434)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <clipPath
+       id="SVGID_4550_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use82345-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath9448">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use9450" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-03" />
+    </clipPath>
+    <clipPath
+       id="clipPath9526">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9528" />
+    </clipPath>
+    <clipPath
+       id="clipPath9530">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9532" />
+    </clipPath>
+    <clipPath
+       id="clipPath9534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9536" />
+    </clipPath>
+    <clipPath
+       id="clipPath9538">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9540" />
+    </clipPath>
+    <clipPath
+       id="clipPath9542">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9544" />
+    </clipPath>
+    <defs
+       id="defs82478">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4568_" />
+    </defs>
+    <clipPath
+       id="clipPath101841">
+      <use
+         id="use101843"
+         overflow="visible"
+         xlink:href="#SVGID_4568_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4563_" />
+    </defs>
+    <clipPath
+       id="clipPath101814">
+      <use
+         id="use101816"
+         overflow="visible"
+         xlink:href="#SVGID_4563_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-43">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-01" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-39">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-27" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-6-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8-0"
+         overflow="visible"
+         id="use82345-0-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9882">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8-0"
+         overflow="visible"
+         id="use9884" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-65"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-8" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-4">
+      <polygon
+         points="-523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 "
+         id="use84656-1-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-67" />
+    </clipPath>
+    <clipPath
+       id="clipPath9893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9895" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-21" />
+    </clipPath>
+    <clipPath
+       id="clipPath9899">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9901" />
+    </clipPath>
+    <clipPath
+       id="clipPath9903">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9905" />
+    </clipPath>
+    <clipPath
+       id="clipPath9907">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9909" />
+    </clipPath>
+    <clipPath
+       id="clipPath9911">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9913" />
+    </clipPath>
+    <clipPath
+       id="clipPath9915">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9917" />
+    </clipPath>
+    <clipPath
+       id="clipPath9919">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9921" />
+    </clipPath>
+    <clipPath
+       id="clipPath9923">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9925" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-5"
+       xlink:href="#SVGID_4521_-3-09"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-09"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-1" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-2">
+      <polygon
+         points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 "
+         id="use82113-2-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-98">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use84798-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath9939">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9941" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4874_-4"
+         overflow="visible"
+         id="use84810-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-5" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-1" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-0" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_-1"
+         overflow="visible"
+         id="use84818-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9953">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9955" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4879_-3"
+         overflow="visible"
+         id="use84841-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_-4"
+         overflow="visible"
+         id="use84849-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath9961">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9963" />
+    </clipPath>
+    <clipPath
+       id="clipPath9965">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9967" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4869_-7"
+         overflow="visible"
+         id="use84771-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_-0"
+         overflow="visible"
+         id="use84759-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath9976">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_-0"
+         overflow="visible"
+         id="use9978" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-90" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-5">
+      <polygon
+         points="-574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 -588.288,278.632 "
+         id="use84732-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath9987">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9989" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath9993">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use9995" />
+    </clipPath>
+    <clipPath
+       id="clipPath9997">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use9999" />
+    </clipPath>
+    <clipPath
+       id="clipPath10001">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10003" />
+    </clipPath>
+    <clipPath
+       id="clipPath10005">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10007" />
+    </clipPath>
+    <clipPath
+       id="clipPath10009">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-0" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-5" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-31" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4826_-2"
+         overflow="visible"
+         id="use84326-14" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61-3"
+       xlink:href="#SVGID_4521_-3-0-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_-5-8"
+         overflow="visible"
+         id="use82113-2-5-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use82132-5-25-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10031">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10033" />
+    </clipPath>
+    <clipPath
+       id="clipPath10035">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10037" />
+    </clipPath>
+    <clipPath
+       id="clipPath10039">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10041" />
+    </clipPath>
+    <clipPath
+       id="clipPath10043">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10045" />
+    </clipPath>
+    <clipPath
+       id="clipPath10047">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10049" />
+    </clipPath>
+    <clipPath
+       id="clipPath10051">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10053" />
+    </clipPath>
+    <clipPath
+       id="clipPath10055">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10057" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4-9"
+         overflow="visible"
+         id="use84644-1-6-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10061">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4-9"
+         overflow="visible"
+         id="use10063" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9-3" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_-1-2"
+         overflow="visible"
+         id="use84656-1-9-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-4" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-9" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-0" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-4">
+      <polygon
+         points="-569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 -569.802,251.283 "
+         id="use84295-5" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_-4"
+         overflow="visible"
+         id="use84007-57" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use83987-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10085">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10087" />
+    </clipPath>
+    <clipPath
+       id="clipPath10089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10091" />
+    </clipPath>
+    <clipPath
+       id="clipPath10093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10095" />
+    </clipPath>
+    <clipPath
+       id="clipPath10097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10099" />
+    </clipPath>
+    <clipPath
+       id="clipPath10101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10103" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_-6"
+         overflow="visible"
+         id="use83972-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-83">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use83952-22" />
+    </clipPath>
+    <clipPath
+       id="clipPath10112">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10114" />
+    </clipPath>
+    <clipPath
+       id="clipPath10116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10118" />
+    </clipPath>
+    <clipPath
+       id="clipPath10120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10122" />
+    </clipPath>
+    <clipPath
+       id="clipPath10124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10126" />
+    </clipPath>
+    <clipPath
+       id="clipPath10128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10130" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-42"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_-6"
+         overflow="visible"
+         id="use83937-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-60">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use83917-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10139">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10141" />
+    </clipPath>
+    <clipPath
+       id="clipPath10143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10145" />
+    </clipPath>
+    <clipPath
+       id="clipPath10147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10149" />
+    </clipPath>
+    <clipPath
+       id="clipPath10151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10153" />
+    </clipPath>
+    <clipPath
+       id="clipPath10155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10157" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-30" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_-0"
+         overflow="visible"
+         id="use83902-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use83882-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10166">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10168" />
+    </clipPath>
+    <clipPath
+       id="clipPath10170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10172" />
+    </clipPath>
+    <clipPath
+       id="clipPath10174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10176" />
+    </clipPath>
+    <clipPath
+       id="clipPath10178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10180" />
+    </clipPath>
+    <clipPath
+       id="clipPath10182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10184" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-06" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_-7"
+         overflow="visible"
+         id="use83867-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use83847-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10193">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10195" />
+    </clipPath>
+    <clipPath
+       id="clipPath10197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10199" />
+    </clipPath>
+    <clipPath
+       id="clipPath10201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10203" />
+    </clipPath>
+    <clipPath
+       id="clipPath10205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10207" />
+    </clipPath>
+    <clipPath
+       id="clipPath10209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10211" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-32">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_-8"
+         overflow="visible"
+         id="use83832-48" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use83812-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10220">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10222" />
+    </clipPath>
+    <clipPath
+       id="clipPath10224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10226" />
+    </clipPath>
+    <clipPath
+       id="clipPath10228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10230" />
+    </clipPath>
+    <clipPath
+       id="clipPath10232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10234" />
+    </clipPath>
+    <clipPath
+       id="clipPath10236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10238" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-57">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_-2"
+         overflow="visible"
+         id="use83797-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use83777-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath10247">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10249" />
+    </clipPath>
+    <clipPath
+       id="clipPath10251">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10253" />
+    </clipPath>
+    <clipPath
+       id="clipPath10255">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10257" />
+    </clipPath>
+    <clipPath
+       id="clipPath10259">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10261" />
+    </clipPath>
+    <clipPath
+       id="clipPath10263">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10265" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-41" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_-6"
+         overflow="visible"
+         id="use83762-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-33"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-25" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-50">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_-7"
+         overflow="visible"
+         id="use83552-66" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use83532-81" />
+    </clipPath>
+    <clipPath
+       id="clipPath10279">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10281" />
+    </clipPath>
+    <clipPath
+       id="clipPath10283">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10285" />
+    </clipPath>
+    <clipPath
+       id="clipPath10287">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10289" />
+    </clipPath>
+    <clipPath
+       id="clipPath10291">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10293" />
+    </clipPath>
+    <clipPath
+       id="clipPath10295">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-52"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_-7"
+         overflow="visible"
+         id="use83517-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use83497-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath10306">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10308" />
+    </clipPath>
+    <clipPath
+       id="clipPath10310">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10312" />
+    </clipPath>
+    <clipPath
+       id="clipPath10314">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10316" />
+    </clipPath>
+    <clipPath
+       id="clipPath10318">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10320" />
+    </clipPath>
+    <clipPath
+       id="clipPath10322">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_-7"
+         overflow="visible"
+         id="use83482-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use83462-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10333">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10335" />
+    </clipPath>
+    <clipPath
+       id="clipPath10337">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10339" />
+    </clipPath>
+    <clipPath
+       id="clipPath10341">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10343" />
+    </clipPath>
+    <clipPath
+       id="clipPath10345">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10347" />
+    </clipPath>
+    <clipPath
+       id="clipPath10349">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-25">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_-7"
+         overflow="visible"
+         id="use83447-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use83427-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10360">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10362" />
+    </clipPath>
+    <clipPath
+       id="clipPath10364">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10366" />
+    </clipPath>
+    <clipPath
+       id="clipPath10368">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10370" />
+    </clipPath>
+    <clipPath
+       id="clipPath10372">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10374" />
+    </clipPath>
+    <clipPath
+       id="clipPath10376">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_-3"
+         overflow="visible"
+         id="use83412-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-07">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use83392-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath10387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10389" />
+    </clipPath>
+    <clipPath
+       id="clipPath10391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10393" />
+    </clipPath>
+    <clipPath
+       id="clipPath10395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10397" />
+    </clipPath>
+    <clipPath
+       id="clipPath10399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10401" />
+    </clipPath>
+    <clipPath
+       id="clipPath10403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_-3"
+         overflow="visible"
+         id="use83377-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use83357-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10416" />
+    </clipPath>
+    <clipPath
+       id="clipPath10418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10420" />
+    </clipPath>
+    <clipPath
+       id="clipPath10422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10424" />
+    </clipPath>
+    <clipPath
+       id="clipPath10426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10428" />
+    </clipPath>
+    <clipPath
+       id="clipPath10430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-37" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_-8"
+         overflow="visible"
+         id="use83342-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-6"
+         overflow="visible"
+         id="use83097-44" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use83077-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath10446">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10448" />
+    </clipPath>
+    <clipPath
+       id="clipPath10450">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10452" />
+    </clipPath>
+    <clipPath
+       id="clipPath10454">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10456" />
+    </clipPath>
+    <clipPath
+       id="clipPath10458">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10460" />
+    </clipPath>
+    <clipPath
+       id="clipPath10462">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10464" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_-3"
+         overflow="visible"
+         id="use83062-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use83042-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10473">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10475" />
+    </clipPath>
+    <clipPath
+       id="clipPath10477">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10479" />
+    </clipPath>
+    <clipPath
+       id="clipPath10481">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10483" />
+    </clipPath>
+    <clipPath
+       id="clipPath10485">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10487" />
+    </clipPath>
+    <clipPath
+       id="clipPath10489">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10491" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-87">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_-7"
+         overflow="visible"
+         id="use83027-49" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use83007-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath10500">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10502" />
+    </clipPath>
+    <clipPath
+       id="clipPath10504">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10506" />
+    </clipPath>
+    <clipPath
+       id="clipPath10508">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10510" />
+    </clipPath>
+    <clipPath
+       id="clipPath10512">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10514" />
+    </clipPath>
+    <clipPath
+       id="clipPath10516">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10518" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_-2"
+         overflow="visible"
+         id="use82992-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-45">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use82972-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10527">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10529" />
+    </clipPath>
+    <clipPath
+       id="clipPath10531">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10533" />
+    </clipPath>
+    <clipPath
+       id="clipPath10535">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10537" />
+    </clipPath>
+    <clipPath
+       id="clipPath10539">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10541" />
+    </clipPath>
+    <clipPath
+       id="clipPath10543">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10545" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-54" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_-5"
+         overflow="visible"
+         id="use82957-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use82937-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath10554">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10556" />
+    </clipPath>
+    <clipPath
+       id="clipPath10558">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10560" />
+    </clipPath>
+    <clipPath
+       id="clipPath10562">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10564" />
+    </clipPath>
+    <clipPath
+       id="clipPath10566">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10568" />
+    </clipPath>
+    <clipPath
+       id="clipPath10570">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10572" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_-9"
+         overflow="visible"
+         id="use82922-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use82902-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10581">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10583" />
+    </clipPath>
+    <clipPath
+       id="clipPath10585">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10587" />
+    </clipPath>
+    <clipPath
+       id="clipPath10589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10591" />
+    </clipPath>
+    <clipPath
+       id="clipPath10593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10595" />
+    </clipPath>
+    <clipPath
+       id="clipPath10597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10599" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-95" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-3">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-94"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-71" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_-7"
+         overflow="visible"
+         id="use82642-05" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use82622-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10615" />
+    </clipPath>
+    <clipPath
+       id="clipPath10617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10619" />
+    </clipPath>
+    <clipPath
+       id="clipPath10621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10623" />
+    </clipPath>
+    <clipPath
+       id="clipPath10625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10627" />
+    </clipPath>
+    <clipPath
+       id="clipPath10629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10631" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-03">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_-8"
+         overflow="visible"
+         id="use82607-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use82587-10" />
+    </clipPath>
+    <clipPath
+       id="clipPath10640">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10642" />
+    </clipPath>
+    <clipPath
+       id="clipPath10644">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10646" />
+    </clipPath>
+    <clipPath
+       id="clipPath10648">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10650" />
+    </clipPath>
+    <clipPath
+       id="clipPath10652">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10654" />
+    </clipPath>
+    <clipPath
+       id="clipPath10656">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10658" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_-5"
+         overflow="visible"
+         id="use82572-40" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use82552-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10667">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10669" />
+    </clipPath>
+    <clipPath
+       id="clipPath10671">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10673" />
+    </clipPath>
+    <clipPath
+       id="clipPath10675">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10677" />
+    </clipPath>
+    <clipPath
+       id="clipPath10679">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10681" />
+    </clipPath>
+    <clipPath
+       id="clipPath10683">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10685" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-27"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_-2"
+         overflow="visible"
+         id="use82537-22" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use82517-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10694">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10696" />
+    </clipPath>
+    <clipPath
+       id="clipPath10698">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10700" />
+    </clipPath>
+    <clipPath
+       id="clipPath10702">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10704" />
+    </clipPath>
+    <clipPath
+       id="clipPath10706">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10708" />
+    </clipPath>
+    <clipPath
+       id="clipPath10710">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10712" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-41"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_-9"
+         overflow="visible"
+         id="use82502-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10723" />
+    </clipPath>
+    <clipPath
+       id="clipPath10725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10727" />
+    </clipPath>
+    <clipPath
+       id="clipPath10729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10731" />
+    </clipPath>
+    <clipPath
+       id="clipPath10733">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10735" />
+    </clipPath>
+    <clipPath
+       id="clipPath10737">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10739" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_-3"
+         overflow="visible"
+         id="use82467-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-73">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath10748">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10750" />
+    </clipPath>
+    <clipPath
+       id="clipPath10752">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10754" />
+    </clipPath>
+    <clipPath
+       id="clipPath10756">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10758" />
+    </clipPath>
+    <clipPath
+       id="clipPath10760">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10762" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7"
+         overflow="visible"
+         id="use82397-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_-7"
+         overflow="visible"
+         id="use82359-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10771">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_-7"
+         overflow="visible"
+         id="use10773" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-50">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-7"
+         overflow="visible"
+         id="use82345-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-7"
+         overflow="visible"
+         id="use10779" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-8" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-7" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-9" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-0" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4546_-1"
+         overflow="visible"
+         id="use82316-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-7" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-4" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-50" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-8" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-4" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-53" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-16">
+      <polygon
+         points="-571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 -613.908,267.353 "
+         id="use82287-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_-5"
+         overflow="visible"
+         id="use82164-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_-7"
+         overflow="visible"
+         id="use82174-5" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-47" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_-0"
+         overflow="visible"
+         id="use82182-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10812">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_-7"
+         overflow="visible"
+         id="use10814" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_-4"
+         overflow="visible"
+         id="use82197-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10818">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_-5"
+         overflow="visible"
+         id="use10820" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-49">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_-1"
+         overflow="visible"
+         id="use82209-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_-5"
+         overflow="visible"
+         id="use82217-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10829">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_-1"
+         overflow="visible"
+         id="use10831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_-7"
+         overflow="visible"
+         id="use82232-1" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0-0"
+       id="linearGradient9382-1"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(197.19124,64.89703)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1-6" />
+    </linearGradient>
+    <linearGradient
+       id="SVGID_4757_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-3">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-44"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath14916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14918" />
+    </clipPath>
+    <clipPath
+       id="clipPath14920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14922" />
+    </clipPath>
+    <clipPath
+       id="clipPath14924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14926" />
+    </clipPath>
+    <clipPath
+       id="clipPath14928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14930" />
+    </clipPath>
+    <clipPath
+       id="clipPath14932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14934" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-5">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-0">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-29">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath14948">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14950" />
+    </clipPath>
+    <clipPath
+       id="clipPath14952">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14954" />
+    </clipPath>
+    <clipPath
+       id="clipPath14956">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14958" />
+    </clipPath>
+    <clipPath
+       id="clipPath14960">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14962" />
+    </clipPath>
+    <clipPath
+       id="clipPath14964">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14966" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-07" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-6">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-16">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-99">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath14980">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14982" />
+    </clipPath>
+    <clipPath
+       id="clipPath14984">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14986" />
+    </clipPath>
+    <clipPath
+       id="clipPath14988">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14990" />
+    </clipPath>
+    <clipPath
+       id="clipPath14992">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14994" />
+    </clipPath>
+    <clipPath
+       id="clipPath14996">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14998" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-90" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-69">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       y2="-148.46021"
+       x2="256.37631"
+       y1="-148.46021"
+       x1="255.37621"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient15092"
+       xlink:href="#SVGID_4627_-3"
+       inkscape:collect="always" />
+    <clipPath
+       id="SVGID_4564_-7-9-3-4-4-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0-9-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-43-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-01-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-8-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-9-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-8-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-1-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-3-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-1-6">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-57-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-33-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-7-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-25-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-50-0">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-66-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-5-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-0-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-0-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-5-8">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-44-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-8-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9"
+         overflow="visible"
+         id="use82397-7-5" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1"
+       id="linearGradient15887"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1-0"
+       id="linearGradient15887-0"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       id="SVGID_4557_-8-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9-1"
+         overflow="visible"
+         id="use82397-7-5-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9-1" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1-0-3"
+       id="linearGradient15887-0-3"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       id="SVGID_4557_-8-1-0-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0-3-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3-5-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2-5-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9-1-0"
+         overflow="visible"
+         id="use82397-7-5-6-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7-5-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9-1-0" />
+    </clipPath>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.93303299"
+     inkscape:cx="321.56814"
+     inkscape:cy="515.95091"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1535"
+     inkscape:window-height="876"
+     inkscape:window-x="65"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata104541">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <rect
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48500001;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:10;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect125410-1-2-7"
+       width="94.101288"
+       height="87.10128"
+       x="316.64056"
+       y="418.58807" />
+    <rect
+       style="fill:url(#linearGradient9382-1);fill-opacity:1;stroke:none"
+       id="rect126198-3-8-1"
+       width="108"
+       height="27"
+       x="308.19122"
+       y="494.25922" />
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499"
+       stroke-miterlimit="10"
+       x1="277.38089"
+       y1="315.69797"
+       x2="277.38089"
+       y2="533.44318"
+       id="line84244" />
+    <rect
+       style="fill:url(#linearGradient9382);fill-opacity:1;stroke:none"
+       id="rect126198-3-8"
+       width="108"
+       height="27"
+       x="139.50002"
+       y="494.28964" />
+    <g
+       transform="matrix(1.3542118,0,0,2.25,996.72494,-25.560183)"
+       id="g82158"
+       inkscape:transform-center-x="262.3321"
+       inkscape:transform-center-y="50.765304">
+      <defs
+         id="defs82160">
+        <rect
+           id="SVGID_4524_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101582">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4524_"
+           overflow="visible"
+           id="use101584" />
+      </clipPath>
+      <g
+         clip-path="url(#SVGID_4525_)"
+         id="g82166">
+        <g
+           id="g82168">
+          <defs
+             id="defs82170">
+            <rect
+               id="SVGID_4526_"
+               x="-679.67902"
+               y="293.259"
+               width="175.168"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101590">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4526_"
+               overflow="visible"
+               id="use101592" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4527_)"
+             id="g82176">
+            <defs
+               id="defs82178">
+              <rect
+                 id="SVGID_4528_"
+                 x="-679.57202"
+                 y="293.367"
+                 width="174.953"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101597">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4528_"
+                 overflow="visible"
+                 id="use101599" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101601"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101603" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101605" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4530_)"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4529_)"
+               width="174.953"
+               height="64.038002"
+               id="rect82189" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4527_)"
+             id="g82191">
+            <defs
+               id="defs82193">
+              <rect
+                 id="SVGID_4531_"
+                 x="-679.67902"
+                 y="293.259"
+                 width="175.168"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101611">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4531_"
+                 overflow="visible"
+                 id="use101613" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4532_)"
+               stroke-miterlimit="10"
+               width="174.953"
+               height="64.038002"
+               id="rect82199" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4525_)"
+         id="g82201">
+        <g
+           id="g82203">
+          <defs
+             id="defs82205">
+            <rect
+               id="SVGID_4533_"
+               x="-441.56699"
+               y="293.259"
+               width="45.873001"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101620">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4533_"
+               overflow="visible"
+               id="use101622" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4534_)"
+             id="g82211">
+            <defs
+               id="defs82213">
+              <rect
+                 id="SVGID_4535_"
+                 x="-441.45999"
+                 y="293.367"
+                 width="45.659"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101627">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4535_"
+                 overflow="visible"
+                 id="use101629" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101631"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101633" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101635" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4537_)"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4536_)"
+               width="45.659"
+               height="64.038002"
+               id="rect82224" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4534_)"
+             id="g82226">
+            <defs
+               id="defs82228">
+              <rect
+                 id="SVGID_4538_"
+                 x="-441.56699"
+                 y="293.259"
+                 width="45.873001"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101641">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4538_"
+                 overflow="visible"
+                 id="use101643" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4539_)"
+               stroke-miterlimit="10"
+               width="45.659"
+               height="64.038002"
+               id="rect82234" />
+          </g>
+        </g>
+      </g>
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="224.23932"
+       y="574.69031"
+       width="95.793755"
+       height="3.0712502"
+       id="rect82238"
+       inkscape:transform-center-x="437.28349"
+       inkscape:transform-center-y="-79.564828" />
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="272.20041"
+       y1="577.34521"
+       x2="272.20041"
+       y2="631.72772"
+       id="line82244"
+       inkscape:transform-center-x="437.21928"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="  -603.212,292.128 -603.212,283.832 -582.187,283.832 -582.187,292.128 "
+       id="polyline82246"
+       inkscape:transform-center-x="437.22048"
+       inkscape:transform-center-y="-33.395941" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82281"
+       inkscape:transform-center-x="437.2835"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82283">
+        <polygon
+           id="SVGID_4543_"
+           points="-613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101684">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4543_"
+           overflow="visible"
+           id="use101686" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101688"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101690" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101692" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101694" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101696" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101698" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101700" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101702" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101704" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4545_)"
+         x="-613.90802"
+         y="253.133"
+         clip-path="url(#SVGID_4544_)"
+         width="42.361"
+         height="14.22"
+         id="rect82306" />
+    </g>
+    <text
+       y="567.99066"
+       x="255.24185"
+       id="text84246"
+       inkscape:transform-center-x="438.70839"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="255.24185"
+         y="567.99066"
+         font-size="3.4203"
+         id="tspan84248">US-EAST</tspan>
+    </text>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 319.54934,544.29505 -0.0157,31.37625 -94.815,0.072 0.0157,-31.2435 46.48275,11.31975 0.1125,0.027 0.11475,-0.027 48.105,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0158,-32.46525 z"
+       id="path82308"
+       inkscape:transform-center-x="437.28456"
+       inkscape:transform-center-y="-95.835733" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82339"
+       inkscape:transform-center-x="449.81598"
+       inkscape:transform-center-y="45.666809">
+      <defs
+         id="defs82341">
+        <rect
+           id="SVGID_4549_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101738">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4549_"
+           overflow="visible"
+           id="use101740" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_)"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82347" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_)"
+         stroke-miterlimit="10"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82349" />
+    </g>
+    <text
+       y="704.88873"
+       x="275.14355"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82353"
+       inkscape:transform-center-x="503.22873"
+       inkscape:transform-center-y="71.647559">
+      <defs
+         id="defs82355">
+        <rect
+           id="SVGID_4551_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101748">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4551_"
+           overflow="visible"
+           id="use101750" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_)"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82361" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_)"
+         stroke-miterlimit="10"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82363" />
+    </g>
+    <text
+       y="730.87036"
+       x="201.73129"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82365"
+       inkscape:transform-center-x="503.47899"
+       inkscape:transform-center-y="71.524915">M</text>
+    <rect
+       style="fill:#ffffff"
+       x="87.659889"
+       y="664.59583"
+       width="21.581999"
+       height="21.584251"
+       id="rect82381"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10"
+       x="87.659889"
+       y="664.59583"
+       stroke-miterlimit="10"
+       width="21.581999"
+       height="21.584251"
+       id="rect82383"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:#81d0db"
+       x="89.169548"
+       y="666.1123"
+       width="18.557999"
+       height="18.553501"
+       id="rect82385"
+       inkscape:transform-center-x="610.97115"
+       inkscape:transform-center-y="19.598295" />
+    <rect
+       style="fill:#ffffff"
+       x="90.654625"
+       y="667.59052"
+       width="15.592501"
+       height="15.594751"
+       id="rect82387"
+       inkscape:transform-center-x="610.96883"
+       inkscape:transform-center-y="19.597131" />
+    <rect
+       style="fill:#ffffff"
+       x="93.307411"
+       y="670.24774"
+       width="10.282501"
+       height="10.278"
+       id="rect82389"
+       inkscape:transform-center-x="610.97103"
+       inkscape:transform-center-y="19.595982" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82391"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82393">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4555_"
+           d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101773">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4555_"
+           overflow="visible"
+           id="use101775" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101777"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101779" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101781" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4557_)"
+         x="-672.61401"
+         y="308.841"
+         clip-path="url(#SVGID_4556_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82404" />
+    </g>
+    <rect
+       id="rect82451"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4564_)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82453"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4564_)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82455"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4564_)"
+       y="307.41"
+       x="-650.10303"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82457"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4564_)"
+       y="308.06699"
+       x="-649.44299"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82459"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4564_)"
+       y="309.24799"
+       x="-648.26398"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82461"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82463">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4565_"
+           d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101827">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4565_"
+           overflow="visible"
+           id="use101829" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101831"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101833" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101835" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4567_)"
+         x="-648.67102"
+         y="308.841"
+         clip-path="url(#SVGID_4566_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82474" />
+    </g>
+    <path
+       id="path82484"
+       d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4569_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82486"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4569_)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82488"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4569_)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82490"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4569_)"
+       y="307.41"
+       x="-638.13202"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82492"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4569_)"
+       y="308.06699"
+       x="-637.47198"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82494"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4569_)"
+       y="309.24799"
+       x="-636.29199"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82496"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82498">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4570_"
+           d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101854">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4570_"
+           overflow="visible"
+           id="use101856" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101858"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46001"
+         x2="256.3761"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101860" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101862" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4572_)"
+         x="-636.69897"
+         y="308.841"
+         clip-path="url(#SVGID_4571_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82509" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82511"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82513">
+        <rect
+           id="SVGID_4573_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101868">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4573_"
+           overflow="visible"
+           id="use101870" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4574_)"
+         d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82519" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82521" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82523" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="307.41"
+         clip-path="url(#SVGID_4574_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82525" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="308.06699"
+         clip-path="url(#SVGID_4574_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82527" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="309.24799"
+         clip-path="url(#SVGID_4574_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82529" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82531"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82533">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4575_"
+           d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101881">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4575_"
+           overflow="visible"
+           id="use101883" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101885"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101887" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101889" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4577_)"
+         x="-624.72803"
+         y="308.841"
+         clip-path="url(#SVGID_4576_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82544" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82546"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82548">
+        <rect
+           id="SVGID_4578_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101895">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4578_"
+           overflow="visible"
+           id="use101897" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4579_)"
+         d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82554" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82556" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82558" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="307.41"
+         clip-path="url(#SVGID_4579_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82560" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="308.06699"
+         clip-path="url(#SVGID_4579_)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect82562" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="309.24799"
+         clip-path="url(#SVGID_4579_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82564" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82566"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82568">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4580_"
+           d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101908">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4580_"
+           overflow="visible"
+           id="use101910" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101912"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101914" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101916" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4582_)"
+         x="-612.75598"
+         y="308.841"
+         clip-path="url(#SVGID_4581_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82579" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82581"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82583">
+        <rect
+           id="SVGID_4583_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101922">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4583_"
+           overflow="visible"
+           id="use101924" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4584_)"
+         d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82589" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82591" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82593" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="307.41"
+         clip-path="url(#SVGID_4584_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82595" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="308.06699"
+         clip-path="url(#SVGID_4584_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82597" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="309.24799"
+         clip-path="url(#SVGID_4584_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82599" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82601"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82603">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4585_"
+           d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101935">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4585_"
+           overflow="visible"
+           id="use101937" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101939"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101941" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101943" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4587_)"
+         x="-600.78497"
+         y="308.841"
+         clip-path="url(#SVGID_4586_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82614" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82616"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82618">
+        <rect
+           id="SVGID_4588_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101949">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4588_"
+           overflow="visible"
+           id="use101951" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4589_)"
+         d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82624" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82626" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82628" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="307.41"
+         clip-path="url(#SVGID_4589_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82630" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="308.06699"
+         clip-path="url(#SVGID_4589_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82632" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="309.24799"
+         clip-path="url(#SVGID_4589_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82634" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82636"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82638">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4590_"
+           d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101962">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4590_"
+           overflow="visible"
+           id="use101964" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101966"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101968" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101970" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4592_)"
+         x="-588.81299"
+         y="308.841"
+         clip-path="url(#SVGID_4591_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82649" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82861"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="37.558931">
+      <defs
+         id="defs82863">
+        <rect
+           id="SVGID_4623_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102138">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4623_"
+           overflow="visible"
+           id="use102140" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4624_)"
+         d="m -514.925,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82869" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="318.323"
+         clip-path="url(#SVGID_4624_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82871" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="318.323"
+         clip-path="url(#SVGID_4624_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82873" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="318.99701"
+         clip-path="url(#SVGID_4624_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82875" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="319.65399"
+         clip-path="url(#SVGID_4624_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82877" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="320.83499"
+         clip-path="url(#SVGID_4624_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82879" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82881"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82883">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4625_"
+           d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102151">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4625_"
+           overflow="visible"
+           id="use102153" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102155"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46021"
+         x2="256.37631"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102157" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102159" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4627_)"
+         x="-672.61401"
+         y="320.42899"
+         clip-path="url(#SVGID_4626_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82894" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82896"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82898">
+        <rect
+           id="SVGID_4628_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102165">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4628_"
+           overflow="visible"
+           id="use102167" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4629_)"
+         d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82904" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="318.323"
+         clip-path="url(#SVGID_4629_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82906" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="318.323"
+         clip-path="url(#SVGID_4629_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82908" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="318.99701"
+         clip-path="url(#SVGID_4629_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82910" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="319.65399"
+         clip-path="url(#SVGID_4629_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82912" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="320.83499"
+         clip-path="url(#SVGID_4629_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82914" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82916"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82918">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4630_"
+           d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102178">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4630_"
+           overflow="visible"
+           id="use102180" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102182"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102184" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102186" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4632_)"
+         x="-660.64203"
+         y="320.42899"
+         clip-path="url(#SVGID_4631_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82929" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82931"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82933">
+        <rect
+           id="SVGID_4633_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102192">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4633_"
+           overflow="visible"
+           id="use102194" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4634_)"
+         d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82939" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82941" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82943" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="318.99701"
+         clip-path="url(#SVGID_4634_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82945" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="319.65399"
+         clip-path="url(#SVGID_4634_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82947" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="320.83499"
+         clip-path="url(#SVGID_4634_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82949" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82951"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82953">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4635_"
+           d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102205">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4635_"
+           overflow="visible"
+           id="use102207" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102209"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102211" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102213" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4637_)"
+         x="-648.67102"
+         y="320.42899"
+         clip-path="url(#SVGID_4636_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82964" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82966"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82968">
+        <rect
+           id="SVGID_4638_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102219">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4638_"
+           overflow="visible"
+           id="use102221" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4639_)"
+         d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82974" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82976" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82978" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="318.99701"
+         clip-path="url(#SVGID_4639_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82980" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="319.65399"
+         clip-path="url(#SVGID_4639_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82982" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="320.83499"
+         clip-path="url(#SVGID_4639_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82984" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82986"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82988">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4640_"
+           d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102232">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4640_"
+           overflow="visible"
+           id="use102234" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102236"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46001"
+         x2="256.37631"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102238" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102240" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4642_)"
+         x="-636.69897"
+         y="320.42899"
+         clip-path="url(#SVGID_4641_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82999" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83001"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83003">
+        <rect
+           id="SVGID_4643_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102246">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4643_"
+           overflow="visible"
+           id="use102248" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4644_)"
+         d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83009" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83011" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83013" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="318.99701"
+         clip-path="url(#SVGID_4644_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83015" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="319.65399"
+         clip-path="url(#SVGID_4644_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83017" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="320.83499"
+         clip-path="url(#SVGID_4644_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83019" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83021"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83023">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4645_"
+           d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102259">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4645_"
+           overflow="visible"
+           id="use102261" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102263"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102265" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102267" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4647_)"
+         x="-624.72803"
+         y="320.42899"
+         clip-path="url(#SVGID_4646_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83034" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83036"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83038">
+        <rect
+           id="SVGID_4648_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102273">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4648_"
+           overflow="visible"
+           id="use102275" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4649_)"
+         d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83044" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="318.323"
+         clip-path="url(#SVGID_4649_)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83046" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="318.323"
+         clip-path="url(#SVGID_4649_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83048" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="318.99701"
+         clip-path="url(#SVGID_4649_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83050" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="319.65399"
+         clip-path="url(#SVGID_4649_)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect83052" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="320.83499"
+         clip-path="url(#SVGID_4649_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83054" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83056"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83058">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4650_"
+           d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102286">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4650_"
+           overflow="visible"
+           id="use102288" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102290"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102292" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102294" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4652_)"
+         x="-612.75598"
+         y="320.42899"
+         clip-path="url(#SVGID_4651_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83069" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83071"
+       inkscape:transform-center-x="444.42945"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83073">
+        <rect
+           id="SVGID_4653_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102300">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4653_"
+           overflow="visible"
+           id="use102302" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4654_)"
+         d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83079" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83081" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83083" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="318.99701"
+         clip-path="url(#SVGID_4654_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83085" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="319.65399"
+         clip-path="url(#SVGID_4654_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83087" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="320.83499"
+         clip-path="url(#SVGID_4654_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83089" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83091"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83093">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4655_"
+           d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102313">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4655_"
+           overflow="visible"
+           id="use102315" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102317"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102319" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102321" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4657_)"
+         x="-588.81299"
+         y="320.42899"
+         clip-path="url(#SVGID_4656_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83104" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83316"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="63.583551">
+      <defs
+         id="defs83318">
+        <rect
+           id="SVGID_4688_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102489">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4688_"
+           overflow="visible"
+           id="use102491" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4689_)"
+         d="m -514.925,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83324" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="329.87"
+         clip-path="url(#SVGID_4689_)"
+         width="9.592"
+         height="9.592"
+         id="rect83326" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="329.87"
+         clip-path="url(#SVGID_4689_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83328" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="330.543"
+         clip-path="url(#SVGID_4689_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83330" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="331.20099"
+         clip-path="url(#SVGID_4689_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83332" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="332.38199"
+         clip-path="url(#SVGID_4689_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83334" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83336"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83338">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4690_"
+           d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102502">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4690_"
+           overflow="visible"
+           id="use102504" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102506"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102508" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102510" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4692_)"
+         x="-672.61401"
+         y="331.974"
+         clip-path="url(#SVGID_4691_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83349" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83351"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83353">
+        <rect
+           id="SVGID_4693_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102516">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4693_"
+           overflow="visible"
+           id="use102518" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4694_)"
+         d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83359" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="329.87"
+         clip-path="url(#SVGID_4694_)"
+         width="9.592"
+         height="9.592"
+         id="rect83361" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="329.87"
+         clip-path="url(#SVGID_4694_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83363" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="330.543"
+         clip-path="url(#SVGID_4694_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83365" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="331.20099"
+         clip-path="url(#SVGID_4694_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83367" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="332.38199"
+         clip-path="url(#SVGID_4694_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83369" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83371"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="71.648681">
+      <defs
+         id="defs83373">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4695_"
+           d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102529">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4695_"
+           overflow="visible"
+           id="use102531" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102533"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.35419"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102535" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102537" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4697_)"
+         x="-660.64203"
+         y="331.974"
+         clip-path="url(#SVGID_4696_)"
+         width="5.3839998"
+         height="5.3850002"
+         id="rect83384" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83386"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83388">
+        <rect
+           id="SVGID_4698_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102543">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4698_"
+           overflow="visible"
+           id="use102545" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4699_)"
+         d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83394" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_)"
+         width="9.592"
+         height="9.592"
+         id="rect83396" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83398" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="330.543"
+         clip-path="url(#SVGID_4699_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83400" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="331.20099"
+         clip-path="url(#SVGID_4699_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83402" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="332.38199"
+         clip-path="url(#SVGID_4699_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83404" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83406"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83408">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4700_"
+           d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102556">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4700_"
+           overflow="visible"
+           id="use102558" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102560"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102562" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102564" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4702_)"
+         x="-648.67102"
+         y="331.974"
+         clip-path="url(#SVGID_4701_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83419" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83421"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83423">
+        <rect
+           id="SVGID_4703_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102570">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4703_"
+           overflow="visible"
+           id="use102572" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4704_)"
+         d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83429" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_)"
+         width="9.592"
+         height="9.592"
+         id="rect83431" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83433" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="330.543"
+         clip-path="url(#SVGID_4704_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83435" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="331.20099"
+         clip-path="url(#SVGID_4704_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83437" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="332.38199"
+         clip-path="url(#SVGID_4704_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83439" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83441"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83443">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4705_"
+           d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102583">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4705_"
+           overflow="visible"
+           id="use102585" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102587"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3544"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102589" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102591" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4707_)"
+         x="-636.69897"
+         y="331.974"
+         clip-path="url(#SVGID_4706_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83454" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83456"
+       inkscape:transform-center-x="498.30125"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83458">
+        <rect
+           id="SVGID_4708_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102597">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4708_"
+           overflow="visible"
+           id="use102599" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4709_)"
+         d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83464" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="329.87"
+         clip-path="url(#SVGID_4709_)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83466" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="329.87"
+         clip-path="url(#SVGID_4709_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83468" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="330.543"
+         clip-path="url(#SVGID_4709_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83470" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="331.20099"
+         clip-path="url(#SVGID_4709_)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83472" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="332.38199"
+         clip-path="url(#SVGID_4709_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83474" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83476"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83478">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4710_"
+           d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102610">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4710_"
+           overflow="visible"
+           id="use102612" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102614"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102616" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102618" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4712_)"
+         x="-612.75598"
+         y="331.974"
+         clip-path="url(#SVGID_4711_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83489" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83491"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83493">
+        <rect
+           id="SVGID_4713_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102624">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4713_"
+           overflow="visible"
+           id="use102626" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4714_)"
+         d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83499" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_)"
+         width="9.592"
+         height="9.592"
+         id="rect83501" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83503" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="330.543"
+         clip-path="url(#SVGID_4714_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83505" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="331.20099"
+         clip-path="url(#SVGID_4714_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83507" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="332.38199"
+         clip-path="url(#SVGID_4714_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83509" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83511"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83513">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4715_"
+           d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102637">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4715_"
+           overflow="visible"
+           id="use102639" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102641"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102643" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102645" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4717_)"
+         x="-600.78497"
+         y="331.974"
+         clip-path="url(#SVGID_4716_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83524" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83526"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83528">
+        <rect
+           id="SVGID_4718_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102651">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4718_"
+           overflow="visible"
+           id="use102653" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4719_)"
+         d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83534" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_)"
+         width="9.592"
+         height="9.592"
+         id="rect83536" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83538" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="330.543"
+         clip-path="url(#SVGID_4719_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83540" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="331.20099"
+         clip-path="url(#SVGID_4719_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83542" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="332.38199"
+         clip-path="url(#SVGID_4719_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83544" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83546"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83548">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4720_"
+           d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102664">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4720_"
+           overflow="visible"
+           id="use102666" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102668"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102670" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102672" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4722_)"
+         x="-588.81299"
+         y="331.974"
+         clip-path="url(#SVGID_4721_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83559" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83736"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="89.828699">
+      <defs
+         id="defs83738">
+        <rect
+           id="SVGID_4748_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102813">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4748_"
+           overflow="visible"
+           id="use102815" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4749_)"
+         d="m -514.925,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83744" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="341.65302"
+         clip-path="url(#SVGID_4749_)"
+         width="9.592"
+         height="9.592"
+         id="rect83746" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="341.65302"
+         clip-path="url(#SVGID_4749_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83748" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="342.32599"
+         clip-path="url(#SVGID_4749_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83750" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="342.98401"
+         clip-path="url(#SVGID_4749_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83752" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="344.164"
+         clip-path="url(#SVGID_4749_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83754" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83756"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83758">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4750_"
+           d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102826">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4750_"
+           overflow="visible"
+           id="use102828" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102830"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102832" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102834" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4752_)"
+         x="-672.61401"
+         y="343.75699"
+         clip-path="url(#SVGID_4751_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83769" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83771"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83773">
+        <rect
+           id="SVGID_4753_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102840">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4753_"
+           overflow="visible"
+           id="use102842" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4754_)"
+         d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83779" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_)"
+         width="9.592"
+         height="9.592"
+         id="rect83781" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83783" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="342.32599"
+         clip-path="url(#SVGID_4754_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83785" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="342.98401"
+         clip-path="url(#SVGID_4754_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83787" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="344.164"
+         clip-path="url(#SVGID_4754_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83789" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83791"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83793">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4755_"
+           d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102853">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4755_"
+           overflow="visible"
+           id="use102855" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102857"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102859" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102861" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4757_)"
+         x="-660.64203"
+         y="343.75699"
+         clip-path="url(#SVGID_4756_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83804" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83806"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83808">
+        <rect
+           id="SVGID_4758_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102867">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4758_"
+           overflow="visible"
+           id="use102869" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4759_)"
+         d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83814" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_)"
+         width="9.592"
+         height="9.592"
+         id="rect83816" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83818" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="342.32599"
+         clip-path="url(#SVGID_4759_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83820" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="342.98401"
+         clip-path="url(#SVGID_4759_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83822" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="344.164"
+         clip-path="url(#SVGID_4759_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83824" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83826"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83828">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4760_"
+           d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102880">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4760_"
+           overflow="visible"
+           id="use102882" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102884"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102886" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102888" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4762_)"
+         x="-648.67102"
+         y="343.75699"
+         clip-path="url(#SVGID_4761_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83839" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83841"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83843">
+        <rect
+           id="SVGID_4763_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102894">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4763_"
+           overflow="visible"
+           id="use102896" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4764_)"
+         d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83849" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_)"
+         width="9.592"
+         height="9.592"
+         id="rect83851" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83853" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="342.32599"
+         clip-path="url(#SVGID_4764_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83855" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="342.98401"
+         clip-path="url(#SVGID_4764_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83857" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="344.164"
+         clip-path="url(#SVGID_4764_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83859" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83861"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83863">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4765_"
+           d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102907">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4765_"
+           overflow="visible"
+           id="use102909" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102911"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102913" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102915" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4767_)"
+         x="-636.69897"
+         y="343.75699"
+         clip-path="url(#SVGID_4766_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83874" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83876"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83878">
+        <rect
+           id="SVGID_4768_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102921">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4768_"
+           overflow="visible"
+           id="use102923" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4769_)"
+         d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83884" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_)"
+         width="9.592"
+         height="9.592"
+         id="rect83886" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83888" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="342.32599"
+         clip-path="url(#SVGID_4769_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83890" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="342.98401"
+         clip-path="url(#SVGID_4769_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83892" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="344.164"
+         clip-path="url(#SVGID_4769_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83894" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83896"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83898">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4770_"
+           d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102934">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4770_"
+           overflow="visible"
+           id="use102936" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102938"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102940" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102942" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4772_)"
+         x="-624.72803"
+         y="343.75699"
+         clip-path="url(#SVGID_4771_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83909" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83911"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83913">
+        <rect
+           id="SVGID_4773_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102948">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4773_"
+           overflow="visible"
+           id="use102950" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4774_)"
+         d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83919" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83921" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83923" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="342.32599"
+         clip-path="url(#SVGID_4774_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83925" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="342.98401"
+         clip-path="url(#SVGID_4774_)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83927" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="344.164"
+         clip-path="url(#SVGID_4774_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83929" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83931"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83933">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4775_"
+           d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102961">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4775_"
+           overflow="visible"
+           id="use102963" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102965"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102967" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102969" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4777_)"
+         x="-612.75598"
+         y="343.75699"
+         clip-path="url(#SVGID_4776_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83944" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83946"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83948">
+        <rect
+           id="SVGID_4778_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102975">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4778_"
+           overflow="visible"
+           id="use102977" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4779_)"
+         d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83954" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_)"
+         width="9.592"
+         height="9.592"
+         id="rect83956" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83958" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="342.32599"
+         clip-path="url(#SVGID_4779_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83960" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="342.98401"
+         clip-path="url(#SVGID_4779_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83962" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="344.164"
+         clip-path="url(#SVGID_4779_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83964" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83966"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83968">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4780_"
+           d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102988">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4780_"
+           overflow="visible"
+           id="use102990" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102992"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102994" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102996" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4782_)"
+         x="-600.78497"
+         y="343.75699"
+         clip-path="url(#SVGID_4781_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83979" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83981"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83983">
+        <rect
+           id="SVGID_4783_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103002">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4783_"
+           overflow="visible"
+           id="use103004" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4784_)"
+         d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83989" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_)"
+         width="9.592"
+         height="9.592"
+         id="rect83991" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83993" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="342.32599"
+         clip-path="url(#SVGID_4784_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83995" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="342.98401"
+         clip-path="url(#SVGID_4784_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83997" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="344.164"
+         clip-path="url(#SVGID_4784_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83999" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84001"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs84003">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4785_"
+           d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath103015">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4785_"
+           overflow="visible"
+           id="use103017" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103019"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop103021" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop103023" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4787_)"
+         x="-588.81299"
+         y="343.75699"
+         clip-path="url(#SVGID_4786_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect84014" />
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="220.3154"
+       y="492.99048"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84287"
+       inkscape:transform-center-x="437.28453"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84289"
+       inkscape:transform-center-x="437.28348"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84291">
+        <polygon
+           id="SVGID_4823_"
+           points="-592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103235">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4823_"
+           overflow="visible"
+           id="use103237" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103239"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103241" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103243" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103245" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103247" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103249" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4825_)"
+         clip-path="url(#SVGID_4824_)"
+         points="-615.683,231.929 -569.802,231.874 -569.772,256.788 -615.653,256.843 "
+         id="polygon84308" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 220.79459,496.5208 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.003,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+       id="path84310"
+       inkscape:transform-center-x="437.28484"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.2843"
+       x="243.95297"
+       id="text84312"
+       inkscape:transform-center-x="437.86922"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="243.95297"
+         y="513.2843"
+         font-size="5.1244"
+         id="tspan84314">RADOSGW</tspan>
+    </text>
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       points="-592.696,269.217 -590.628,267.424 -594.747,267.424 "
+       id="polygon84355"
+       inkscape:transform-center-x="437.19348"
+       inkscape:transform-center-y="-77.629816" />
+    <path
+       id="path84524"
+       d="m -365.856,302.434 c 0,-2.331 1.892,-4.222 4.225,-4.222 2.334,0 4.223,1.891 4.223,4.222 0,2.334 -1.889,4.226 -4.223,4.226 -2.333,0 -4.225,-1.892 -4.225,-4.226"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84526"
+       d="m -361.631,301.547 c 0.488,0 0.888,0.398 0.888,0.887 0,0.493 -0.4,0.892 -0.888,0.89 -0.491,0 -0.888,-0.397 -0.888,-0.89 0,-0.489 0.397,-0.887 0.888,-0.887"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84528"
+       d="m -347.575,299.54 c 0.487,0 0.886,0.395 0.886,0.886 0,0.494 -0.399,0.891 -0.886,0.891 -0.493,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.395,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84530"
+       d="m -350.839,302.809 c 0.49,0 0.886,0.397 0.886,0.886 0,0.494 -0.396,0.893 -0.886,0.893 -0.491,0 -0.889,-0.399 -0.889,-0.893 0,-0.489 0.398,-0.886 0.889,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84532"
+       d="m -350.838,299.54 c 0.489,0 0.885,0.395 0.885,0.886 0,0.494 -0.396,0.891 -0.885,0.891 -0.492,0 -0.89,-0.397 -0.89,-0.891 0,-0.491 0.398,-0.886 0.89,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84534"
+       d="m -347.576,302.809 c 0.488,0 0.888,0.397 0.888,0.886 0,0.494 -0.4,0.893 -0.888,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84536"
+       d="m -344.311,302.809 c 0.488,0 0.886,0.397 0.886,0.886 0,0.494 -0.398,0.893 -0.886,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84538"
+       d="m -344.31,299.54 c 0.487,0 0.885,0.395 0.885,0.886 0,0.494 -0.398,0.891 -0.885,0.891 -0.492,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.396,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84540"
+       y2="308.98499"
+       x2="-343.42499"
+       y1="308.98499"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84542"
+       y2="311.16699"
+       x2="-343.42499"
+       y1="311.16699"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84544"
+       y2="313.35001"
+       x2="-343.42499"
+       y1="313.35001"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84552"
+       points="-601.13,292.47 -605.249,292.47 -603.198,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84554"
+       points="-590.618,292.47 -594.737,292.47 -592.686,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84556"
+       points="-580.105,292.47 -584.224,292.47 -582.173,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84564"
+       points="-436.6,208.359 -434.532,206.566 -438.651,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84566"
+       points="-426.089,208.359 -424.021,206.566 -428.14,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84568"
+       points="-415.575,208.359 -413.507,206.566 -417.626,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84570"
+       points="-296.627,290.885 -294.559,289.091 -298.678,289.091 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84714"
+       inkscape:transform-center-x="434.5295"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84716">
+        <rect
+           id="SVGID_4862_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103539">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4862_"
+           overflow="visible"
+           id="use103541" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4863_)"
+         d="m -593.321,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84722" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4863_)"
+         stroke-miterlimit="10"
+         x1="-592.68903"
+         y1="276.099"
+         x2="-589.79303"
+         y2="276.099"
+         id="line84724" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84726"
+       inkscape:transform-center-x="413.59995"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84728">
+        <polygon
+           id="SVGID_4864_"
+           points="-590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103548">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4864_"
+           overflow="visible"
+           id="use103550" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103552"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.9404"
+         x2="-33.205299"
+         y2="249.9404"
+         gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103554" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103556" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4866_)"
+         x="-590.08099"
+         y="273.56699"
+         clip-path="url(#SVGID_4865_)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84739" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="-590.082,276.108 -588.289,278.632 -574.323,278.632 -574.323,273.567 -588.289,273.567 "
+       id="polygon84741"
+       inkscape:transform-center-x="413.60223"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.62451"
+       x="287.82977"
+       id="text84743"
+       inkscape:transform-center-x="412.33742"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="287.82977"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84745">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="291.70874"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84747">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="294.77774"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84749">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="302.49078"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84751">E</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,996.72494,-31.560183)"
+       id="g84792"
+       inkscape:transform-center-x="262.77489"
+       inkscape:transform-center-y="141.90493">
+      <defs
+         id="defs84794">
+        <rect
+           id="SVGID_4872_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103597">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4872_"
+           overflow="visible"
+           id="use103599" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4873_)"
+         d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+         id="path84800" />
+      <g
+         clip-path="url(#SVGID_4873_)"
+         id="g84802">
+        <g
+           id="g84804">
+          <defs
+             id="defs84806">
+            <rect
+               id="SVGID_4874_"
+               x="-679.90002"
+               y="360.13901"
+               width="174.95799"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103606">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4874_"
+               overflow="visible"
+               id="use103608" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4875_)"
+             id="g84812">
+            <defs
+               id="defs84814">
+              <rect
+                 id="SVGID_4876_"
+                 x="-679.90002"
+                 y="360.13901"
+                 width="174.95799"
+                 height="10.152" />
+            </defs>
+            <clipPath
+               id="clipPath103613">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4876_"
+                 overflow="visible"
+                 id="use103615" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient103617"
+               gradientUnits="userSpaceOnUse"
+               x1="40.563"
+               y1="147.0757"
+               x2="41.563"
+               y2="147.0757"
+               gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+              <stop
+                 offset="0"
+                 style="stop-color:#55AEB9"
+                 id="stop103619" />
+              <stop
+                 offset="0.486"
+                 style="stop-color:#B0D2D9"
+                 id="stop103621" />
+              <stop
+                 offset="0.8287"
+                 style="stop-color:#E6EFF1"
+                 id="stop103623" />
+              <stop
+                 offset="0.9939"
+                 style="stop-color:#FFFFFF"
+                 id="stop103625" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop103627" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4878_)"
+               x="-679.90002"
+               y="360.138"
+               clip-path="url(#SVGID_4877_)"
+               width="174.95799"
+               height="10.153"
+               id="rect84831" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4873_)"
+         id="g84833">
+        <g
+           id="g84835">
+          <defs
+             id="defs84837">
+            <rect
+               id="SVGID_4879_"
+               x="-679.88898"
+               y="360.13901"
+               width="174.94701"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103634">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4879_"
+               overflow="visible"
+               id="use103636" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4880_)"
+             id="g84843">
+            <defs
+               id="defs84845">
+              <rect
+                 id="SVGID_4881_"
+                 x="-679.88898"
+                 y="360.138"
+                 width="174.94701"
+                 height="10.153" />
+            </defs>
+            <clipPath
+               id="clipPath103641">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4881_"
+                 overflow="visible"
+                 id="use103643" />
+            </clipPath>
+            <polygon
+               style="fill:#f15d57"
+               clip-path="url(#SVGID_4882_)"
+               points="-679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 -504.943,360.138 -679.89,360.139 "
+               id="polygon84851" />
+          </g>
+        </g>
+      </g>
+      <rect
+         style="fill:#f15d57"
+         x="-679.90198"
+         y="360.267"
+         clip-path="url(#SVGID_4873_)"
+         width="174.96001"
+         height="2.641"
+         id="rect84853" />
+      <rect
+         style="fill:#55aeb9"
+         x="-679.88898"
+         y="370.29199"
+         clip-path="url(#SVGID_4873_)"
+         width="174.94701"
+         height="1.352"
+         id="rect84855" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-size:8px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="285.86542"
+       y="577.40869"
+       id="text113406"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan113408"
+         x="285.86542"
+         y="577.40869" /></text>
+    <text
+       y="832.10559"
+       x="242.41013"
+       id="text84343-2"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="197.43796"
+         y="832.10559"
+         font-size="5.1244"
+         id="tspan84345-2">MASTER REGION</tspan>
+    </text>
+    <text
+       y="926.36633"
+       x="288.45654"
+       id="text84343-2-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="249.24219"
+         y="926.36633"
+         font-size="5.1244"
+         id="tspan84345-2-5" />
+    </text>
+    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126352"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126354"><rect
+           id="rect126356"
+           width="0"
+           height="51"
+           x="247"
+           y="928.36218" /></flowRegion><flowPara
+         id="flowPara126358" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126403"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126405"><rect
+           id="rect126407"
+           width="185"
+           height="81"
+           x="286"
+           y="941.36218" /></flowRegion><flowPara
+         id="flowPara126409" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126411"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126413"><rect
+           id="rect126415"
+           width="89"
+           height="64"
+           x="285"
+           y="943.36218" /></flowRegion><flowPara
+         id="flowPara126417" /></flowRoot>    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="272"
+       y="529.64587"
+       id="text126456"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126458"
+         x="272"
+         y="529.64587">MASTER</tspan><tspan
+         sodipodi:role="line"
+         x="272"
+         y="540.89587"
+         id="tspan126460">ZONE</tspan></text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82107">
+      <defs
+         id="defs82109">
+        <polygon
+           id="SVGID_4519_"
+           points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 " />
+      </defs>
+      <clipPath
+         id="clipPath126820">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4519_"
+           overflow="visible"
+           id="use126822" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126824"
+         gradientUnits="userSpaceOnUse"
+         x1="40.840599"
+         y1="146.6937"
+         x2="41.840599"
+         y2="146.6937"
+         gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+        <stop
+           offset="0"
+           style="stop-color:#37424B"
+           id="stop126826" />
+        <stop
+           offset="0.12"
+           style="stop-color:#37424B"
+           id="stop126828" />
+        <stop
+           offset="0.88"
+           style="stop-color:#5E6A71"
+           id="stop126830" />
+        <stop
+           offset="1"
+           style="stop-color:#5E6A71"
+           id="stop126832" />
+      </linearGradient>
+      <rect
+         style="fill:url(#linearGradient126904)"
+         x="-547.302"
+         y="188.287"
+         clip-path="url(#SVGID_4520_-5)"
+         width="42.327"
+         height="10.191"
+         id="rect82124" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82126">
+      <defs
+         id="defs82128">
+        <rect
+           id="SVGID_4522_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126838">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4522_"
+           overflow="visible"
+           id="use126840" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82134" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82136" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-541.625"
+         d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82138" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-541.625"
+         d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82140" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-543.85303"
+         d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.85303"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82142" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-543.854"
+         d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.854"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82144" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82146" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82148" />
+    </g>
+    <text
+       y="319.3797"
+       x="267.68884"
+       id="text82150"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="267.68884"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82152">AP</tspan>
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="281.37094"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82154">P</tspan>
+    </text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84638">
+      <defs
+         id="defs84640">
+        <rect
+           id="SVGID_4852_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126879">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4852_"
+           overflow="visible"
+           id="use126881" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4853_-8)"
+         d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84646" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4853_-8)"
+         stroke-miterlimit="10"
+         x1="-526.03998"
+         y1="205.991"
+         x2="-523.14398"
+         y2="205.991"
+         id="line84648" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84650">
+      <defs
+         id="defs84652">
+        <polygon
+           id="SVGID_4854_"
+           points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 " />
+      </defs>
+      <clipPath
+         id="clipPath126888">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4854_"
+           overflow="visible"
+           id="use126890" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126892"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94031"
+         x2="-33.205101"
+         y2="249.94031"
+         gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop126894" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop126896" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4856_-4)"
+         x="-523.43201"
+         y="203.45799"
+         clip-path="url(#SVGID_4855_-2)"
+         width="15.759"
+         height="5.066"
+         id="rect84663" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       stroke-miterlimit="10"
+       points="-507.674,203.459 -521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 "
+       id="polygon84665" />
+    <text
+       y="363.54565"
+       x="306.66861"
+       id="text84667"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="306.66861"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84669">RE</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="315.84521"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84671">S</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="319.98297"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84673">T</tspan>
+    </text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="232.25806"
+       y="178.16862"
+       id="text127482"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127484"
+         x="232.25806"
+         y="178.16862" /></text>
+    <text
+       y="247.60512"
+       x="322.41013"
+       id="text84343-2-0"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="277.43796"
+         y="247.60512"
+         font-size="5.1244"
+         id="tspan84345-2-6">MASTER REGION</tspan>
+    </text>
+    <text
+       y="705.37921"
+       x="255.23141"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-9"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       id="g9504"
+       transform="translate(-230,34)">
+      <g
+         inkscape:transform-center-y="45.666809"
+         inkscape:transform-center-x="449.81598"
+         id="g82339-6"
+         transform="matrix(2.25,0,0,2.25,1701.0081,-85.193202)">
+        <defs
+           id="defs82341-9">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4549_-8" />
+        </defs>
+        <clipPath
+           id="clipPath101738-3">
+          <use
+             id="use101740-3"
+             overflow="visible"
+             xlink:href="#SVGID_4549_-8"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82347-9"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4550_-6)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82349-9"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4550_-6)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9-4"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="350.6293"
+         y="645.37921">M</text>
+    </g>
+    <path
+       id="path82449-2"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.5425,-25.466067)" />
+    <path
+       id="path82449-2-2"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1578.7139,-25.075587)" />
+    <path
+       id="path82449-2-0"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.5425,0.59793262)" />
+    <path
+       id="path82449-2-0-1"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.6425,26.597933)" />
+    <path
+       id="path82449-2-0-1-5"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-4)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.7422,53.197933)" />
+    <text
+       y="292.96655"
+       x="321.78174"
+       id="text84343-2-0-4-3"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="276.80957"
+         y="292.96655"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-3">WRITE / READ</tspan>
+    </text>
+    <text
+       y="262.38412"
+       x="322.19797"
+       id="text84343-2-0-4-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="277.2258"
+         y="262.38412"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-6">(United States)</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,1325.4162,-25.59059)"
+       id="g82158-9"
+       inkscape:transform-center-x="262.3321"
+       inkscape:transform-center-y="50.765304">
+      <defs
+         id="defs82160-5">
+        <rect
+           id="SVGID_4524_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101582-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4524_-5"
+           overflow="visible"
+           id="use101584-7" />
+      </clipPath>
+      <g
+         clip-path="url(#SVGID_4525_-0)"
+         id="g82166-4">
+        <g
+           id="g82168-7">
+          <defs
+             id="defs82170-2">
+            <rect
+               id="SVGID_4526_-7"
+               x="-679.67902"
+               y="293.259"
+               width="175.168"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101590-3">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4526_-7"
+               overflow="visible"
+               id="use101592-1" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4527_-7)"
+             id="g82176-6">
+            <defs
+               id="defs82178-7">
+              <rect
+                 id="SVGID_4528_-0"
+                 x="-679.57202"
+                 y="293.367"
+                 width="174.953"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101597-1">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4528_-0"
+                 overflow="visible"
+                 id="use101599-3" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101601-1"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101603-3" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101605-0" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4530_-2)"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4529_-9)"
+               width="174.953"
+               height="64.038002"
+               id="rect82189-5" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4527_-7)"
+             id="g82191-9">
+            <defs
+               id="defs82193-6">
+              <rect
+                 id="SVGID_4531_-4"
+                 x="-679.67902"
+                 y="293.259"
+                 width="175.168"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101611-1">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4531_-4"
+                 overflow="visible"
+                 id="use101613-8" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4532_-8)"
+               stroke-miterlimit="10"
+               width="174.953"
+               height="64.038002"
+               id="rect82199-7" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4525_-0)"
+         id="g82201-2">
+        <g
+           id="g82203-6">
+          <defs
+             id="defs82205-4">
+            <rect
+               id="SVGID_4533_-1"
+               x="-441.56699"
+               y="293.259"
+               width="45.873001"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101620-7">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4533_-1"
+               overflow="visible"
+               id="use101622-3" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4534_-49)"
+             id="g82211-7">
+            <defs
+               id="defs82213-4">
+              <rect
+                 id="SVGID_4535_-5"
+                 x="-441.45999"
+                 y="293.367"
+                 width="45.659"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101627-6">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4535_-5"
+                 overflow="visible"
+                 id="use101629-8" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101631-5"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101633-8" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101635-5" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4537_-7)"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4536_-4)"
+               width="45.659"
+               height="64.038002"
+               id="rect82224-8" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4534_-49)"
+             id="g82226-0">
+            <defs
+               id="defs82228-3">
+              <rect
+                 id="SVGID_4538_-7"
+                 x="-441.56699"
+                 y="293.259"
+                 width="45.873001"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101641-2">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4538_-7"
+                 overflow="visible"
+                 id="use101643-6" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4539_-0)"
+               stroke-miterlimit="10"
+               width="45.659"
+               height="64.038002"
+               id="rect82234-0" />
+          </g>
+        </g>
+      </g>
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="404.21451"
+       y="574.65991"
+       width="95.778"
+       height="3.0712502"
+       id="rect82236-5"
+       inkscape:transform-center-x="586.0074"
+       inkscape:transform-center-y="-79.564828" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82310-8"
+       inkscape:transform-center-x="585.9972"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82312-6">
+        <polygon
+           id="SVGID_4546_-1"
+           points="-679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101711-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4546_-1"
+           overflow="visible"
+           id="use101713-5" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101715-3"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101717-8" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101719-2" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101721-7" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101723-6" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101725-5" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101727-2" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101729-5" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101731-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4548_-6)"
+         x="-680.00299"
+         y="253.133"
+         clip-path="url(#SVGID_4547_-1)"
+         width="42.361"
+         height="14.22"
+         id="rect82335-2" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 499.52681,544.26464 -0.0157,31.37625 -94.815,0.072 0.0158,-31.2435 46.48274,11.31975 0.1125,0.027 0.11475,-0.027 48.10501,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0157,-32.46525 z"
+       id="path82337-5"
+       inkscape:transform-center-x="585.99834"
+       inkscape:transform-center-y="-95.835733" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82339-8"
+       inkscape:transform-center-x="449.81598"
+       inkscape:transform-center-y="45.666809">
+      <defs
+         id="defs82341-98">
+        <rect
+           id="SVGID_4549_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101738-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4549_-7"
+           overflow="visible"
+           id="use101740-4" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_-50)"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82347-4" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_-50)"
+         stroke-miterlimit="10"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82349-4" />
+    </g>
+    <text
+       y="704.8584"
+       x="603.83478"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-1"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82353-8"
+       inkscape:transform-center-x="503.22873"
+       inkscape:transform-center-y="71.647559">
+      <defs
+         id="defs82355-0">
+        <rect
+           id="SVGID_4551_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101748-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4551_-7"
+           overflow="visible"
+           id="use101750-2" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_-4)"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82361-9" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_-4)"
+         stroke-miterlimit="10"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82363-6" />
+    </g>
+    <text
+       y="730.84003"
+       x="530.42249"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82365-8"
+       inkscape:transform-center-x="503.47899"
+       inkscape:transform-center-y="71.524915">M</text>
+    <rect
+       style="fill:#ffffff"
+       x="416.3511"
+       y="664.56543"
+       width="21.581999"
+       height="21.584251"
+       id="rect82381-2"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10"
+       x="416.3511"
+       y="664.56543"
+       stroke-miterlimit="10"
+       width="21.581999"
+       height="21.584251"
+       id="rect82383-1"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:#81d0db"
+       x="417.8631"
+       y="666.08191"
+       width="18.557999"
+       height="18.553501"
+       id="rect82385-3"
+       inkscape:transform-center-x="610.97115"
+       inkscape:transform-center-y="19.598295" />
+    <rect
+       style="fill:#ffffff"
+       x="419.34586"
+       y="667.56012"
+       width="15.592501"
+       height="15.594751"
+       id="rect82387-5"
+       inkscape:transform-center-x="610.96883"
+       inkscape:transform-center-y="19.597131" />
+    <rect
+       style="fill:#ffffff"
+       x="422.00085"
+       y="670.21729"
+       width="10.282501"
+       height="10.278"
+       id="rect82389-9"
+       inkscape:transform-center-x="610.97103"
+       inkscape:transform-center-y="19.595982" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.591067)"
+       id="g82391-8"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82393-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4555_-7"
+           d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101773-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4555_-7"
+           overflow="visible"
+           id="use101775-6" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101777-2"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101779-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101781-1" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4557_-8)"
+         x="-672.61401"
+         y="308.841"
+         clip-path="url(#SVGID_4556_-9)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82404-9" />
+    </g>
+    <rect
+       id="rect82451-3"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4564_-73)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82453-2"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4564_-73)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82455-8"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4564_-73)"
+       y="307.41"
+       x="-650.10303"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82457-2"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4564_-73)"
+       y="308.06699"
+       x="-649.44299"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82459-8"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4564_-73)"
+       y="309.24799"
+       x="-648.26398"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82461-4"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82463-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4565_-3"
+           d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101827-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4565_-3"
+           overflow="visible"
+           id="use101829-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101831-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101833-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101835-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4567_-1)"
+         x="-648.67102"
+         y="308.841"
+         clip-path="url(#SVGID_4566_-3)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82474-5" />
+    </g>
+    <path
+       id="path82484-3"
+       d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4569_-7)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82486-0"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4569_-7)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82488-6"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4569_-7)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82490-6"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4569_-7)"
+       y="307.41"
+       x="-638.13202"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82492-4"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4569_-7)"
+       y="308.06699"
+       x="-637.47198"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82494-9"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4569_-7)"
+       y="309.24799"
+       x="-636.29199"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82496-3"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82498-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4570_-9"
+           d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101854-8">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4570_-9"
+           overflow="visible"
+           id="use101856-3" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101858-4"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46001"
+         x2="256.3761"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101860-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101862-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4572_-41)"
+         x="-636.69897"
+         y="308.841"
+         clip-path="url(#SVGID_4571_-0)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82509-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82511-7"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82513-7">
+        <rect
+           id="SVGID_4573_-4"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101868-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4573_-4"
+           overflow="visible"
+           id="use101870-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4574_-9)"
+         d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82519-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_-9)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82521-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_-9)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82523-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="307.41"
+         clip-path="url(#SVGID_4574_-9)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82525-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="308.06699"
+         clip-path="url(#SVGID_4574_-9)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82527-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="309.24799"
+         clip-path="url(#SVGID_4574_-9)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82529-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82531-3"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82533-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4575_-2"
+           d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101881-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4575_-2"
+           overflow="visible"
+           id="use101883-6" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101885-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101887-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101889-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4577_-27)"
+         x="-624.72803"
+         y="308.841"
+         clip-path="url(#SVGID_4576_-1)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82544-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82546-5"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82548-4">
+        <rect
+           id="SVGID_4578_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101895-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4578_-9"
+           overflow="visible"
+           id="use101897-5" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4579_-0)"
+         d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82554-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_-0)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82556-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_-0)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82558-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="307.41"
+         clip-path="url(#SVGID_4579_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82560-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="308.06699"
+         clip-path="url(#SVGID_4579_-0)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect82562-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="309.24799"
+         clip-path="url(#SVGID_4579_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82564-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82566-5"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82568-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4580_-5"
+           d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101908-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4580_-5"
+           overflow="visible"
+           id="use101910-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101912-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101914-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101916-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4582_-8)"
+         x="-612.75598"
+         y="308.841"
+         clip-path="url(#SVGID_4581_-2)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82579-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82581-3"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82583-0">
+        <rect
+           id="SVGID_4583_-4"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101922-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4583_-4"
+           overflow="visible"
+           id="use101924-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4584_-0)"
+         d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82589-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82591-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82593-2" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="307.41"
+         clip-path="url(#SVGID_4584_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82595-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="308.06699"
+         clip-path="url(#SVGID_4584_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82597-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="309.24799"
+         clip-path="url(#SVGID_4584_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82599-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82601-6"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82603-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4585_-8"
+           d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101935-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4585_-8"
+           overflow="visible"
+           id="use101937-8" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101939-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101941-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101943-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4587_-9)"
+         x="-600.78497"
+         y="308.841"
+         clip-path="url(#SVGID_4586_-03)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82614-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82616-8"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82618-9">
+        <rect
+           id="SVGID_4588_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101949-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4588_-2"
+           overflow="visible"
+           id="use101951-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4589_-7)"
+         d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82624-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_-7)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82626-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_-7)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82628-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="307.41"
+         clip-path="url(#SVGID_4589_-7)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82630-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="308.06699"
+         clip-path="url(#SVGID_4589_-7)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82632-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="309.24799"
+         clip-path="url(#SVGID_4589_-7)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82634-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82636-3"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82638-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4590_-7"
+           d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101962-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4590_-7"
+           overflow="visible"
+           id="use101964-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101966-2"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101968-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101970-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4592_-6)"
+         x="-588.81299"
+         y="308.841"
+         clip-path="url(#SVGID_4591_-2)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82649-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82896-3"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82898-8">
+        <rect
+           id="SVGID_4628_-3"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102165-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4628_-3"
+           overflow="visible"
+           id="use102167-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4629_-0)"
+         d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82904-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="318.323"
+         clip-path="url(#SVGID_4629_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82906-1" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="318.323"
+         clip-path="url(#SVGID_4629_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82908-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="318.99701"
+         clip-path="url(#SVGID_4629_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82910-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="319.65399"
+         clip-path="url(#SVGID_4629_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82912-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="320.83499"
+         clip-path="url(#SVGID_4629_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82914-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82916-7"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82918-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4630_-9"
+           d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102178-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4630_-9"
+           overflow="visible"
+           id="use102180-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102182-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102184-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102186-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4632_-7)"
+         x="-660.64203"
+         y="320.42899"
+         clip-path="url(#SVGID_4631_-7)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82929-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82931-1"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82933-7">
+        <rect
+           id="SVGID_4633_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102192-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4633_-6"
+           overflow="visible"
+           id="use102194-8" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4634_-0)"
+         d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82939-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82941-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82943-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="318.99701"
+         clip-path="url(#SVGID_4634_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82945-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="319.65399"
+         clip-path="url(#SVGID_4634_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82947-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="320.83499"
+         clip-path="url(#SVGID_4634_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82949-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82951-7"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82953-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4635_-5"
+           d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102205-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4635_-5"
+           overflow="visible"
+           id="use102207-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102209-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102211-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102213-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4637_-4)"
+         x="-648.67102"
+         y="320.42899"
+         clip-path="url(#SVGID_4636_-9)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82964-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82966-8"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82968-8">
+        <rect
+           id="SVGID_4638_-3"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102219-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4638_-3"
+           overflow="visible"
+           id="use102221-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4639_-45)"
+         d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82974-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_-45)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82976-3" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_-45)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82978-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="318.99701"
+         clip-path="url(#SVGID_4639_-45)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82980-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="319.65399"
+         clip-path="url(#SVGID_4639_-45)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82982-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="320.83499"
+         clip-path="url(#SVGID_4639_-45)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82984-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82986-8"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82988-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4640_-2"
+           d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102232-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4640_-2"
+           overflow="visible"
+           id="use102234-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102236-4"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46001"
+         x2="256.37631"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102238-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102240-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4642_-8)"
+         x="-636.69897"
+         y="320.42899"
+         clip-path="url(#SVGID_4641_-3)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82999-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83001-3"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83003-9">
+        <rect
+           id="SVGID_4643_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102246-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4643_-8"
+           overflow="visible"
+           id="use102248-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4644_-6)"
+         d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83009-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_-6)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83011-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_-6)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83013-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="318.99701"
+         clip-path="url(#SVGID_4644_-6)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83015-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="319.65399"
+         clip-path="url(#SVGID_4644_-6)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83017-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="320.83499"
+         clip-path="url(#SVGID_4644_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83019-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83021-5"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83023-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4645_-7"
+           d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102259-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4645_-7"
+           overflow="visible"
+           id="use102261-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102263-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102265-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102267-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4647_-0)"
+         x="-624.72803"
+         y="320.42899"
+         clip-path="url(#SVGID_4646_-87)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83034-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83036-0"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83038-8">
+        <rect
+           id="SVGID_4648_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102273-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4648_-2"
+           overflow="visible"
+           id="use102275-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4649_-4)"
+         d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83044-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="318.323"
+         clip-path="url(#SVGID_4649_-4)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83046-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="318.323"
+         clip-path="url(#SVGID_4649_-4)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83048-4" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="318.99701"
+         clip-path="url(#SVGID_4649_-4)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83050-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="319.65399"
+         clip-path="url(#SVGID_4649_-4)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect83052-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="320.83499"
+         clip-path="url(#SVGID_4649_-4)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83054-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83056-3"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83058-3">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4650_-3"
+           d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102286-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4650_-3"
+           overflow="visible"
+           id="use102288-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102290-5"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102292-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102294-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4652_-6)"
+         x="-612.75598"
+         y="320.42899"
+         clip-path="url(#SVGID_4651_-3)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83069-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83071-4"
+       inkscape:transform-center-x="444.42945"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83073-0">
+        <rect
+           id="SVGID_4653_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102300-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4653_-9"
+           overflow="visible"
+           id="use102302-8" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4654_-2)"
+         d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83079-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_-2)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83081-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_-2)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83083-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="318.99701"
+         clip-path="url(#SVGID_4654_-2)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83085-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="319.65399"
+         clip-path="url(#SVGID_4654_-2)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83087-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="320.83499"
+         clip-path="url(#SVGID_4654_-2)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83089-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83091-0"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83093-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4655_-6"
+           d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102313-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4655_-6"
+           overflow="visible"
+           id="use102315-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102317-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102319-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102321-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4657_-5)"
+         x="-588.81299"
+         y="320.42899"
+         clip-path="url(#SVGID_4656_-5)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83104-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.59059)"
+       id="g83336-4"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83338-2">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4690_-8"
+           d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102502-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4690_-8"
+           overflow="visible"
+           id="use102504-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102506-5"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102508-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102510-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4692_-5)"
+         x="-672.61401"
+         y="331.974"
+         clip-path="url(#SVGID_4691_-8)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83349-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83351-8"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83353-8">
+        <rect
+           id="SVGID_4693_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102516-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4693_-2"
+           overflow="visible"
+           id="use102518-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4694_-6)"
+         d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83359-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="329.87"
+         clip-path="url(#SVGID_4694_-6)"
+         width="9.592"
+         height="9.592"
+         id="rect83361-9" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="329.87"
+         clip-path="url(#SVGID_4694_-6)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83363-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="330.543"
+         clip-path="url(#SVGID_4694_-6)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83365-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="331.20099"
+         clip-path="url(#SVGID_4694_-6)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83367-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="332.38199"
+         clip-path="url(#SVGID_4694_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83369-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83371-6"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="71.648681">
+      <defs
+         id="defs83373-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4695_-3"
+           d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102529-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4695_-3"
+           overflow="visible"
+           id="use102531-5" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102533-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.35419"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102535-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102537-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4697_-7)"
+         x="-660.64203"
+         y="331.974"
+         clip-path="url(#SVGID_4696_-9)"
+         width="5.3839998"
+         height="5.3850002"
+         id="rect83384-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83386-5"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83388-6">
+        <rect
+           id="SVGID_4698_-1"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102543-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4698_-1"
+           overflow="visible"
+           id="use102545-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4699_-07)"
+         d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83394-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_-07)"
+         width="9.592"
+         height="9.592"
+         id="rect83396-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_-07)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83398-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="330.543"
+         clip-path="url(#SVGID_4699_-07)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83400-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="331.20099"
+         clip-path="url(#SVGID_4699_-07)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83402-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="332.38199"
+         clip-path="url(#SVGID_4699_-07)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83404-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83406-2"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83408-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4700_-3"
+           d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102556-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4700_-3"
+           overflow="visible"
+           id="use102558-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102560-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102562-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102564-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4702_-3)"
+         x="-648.67102"
+         y="331.974"
+         clip-path="url(#SVGID_4701_-9)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83419-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83421-8"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83423-5">
+        <rect
+           id="SVGID_4703_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102570-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4703_-5"
+           overflow="visible"
+           id="use102572-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4704_-1)"
+         d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83429-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_-1)"
+         width="9.592"
+         height="9.592"
+         id="rect83431-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_-1)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83433-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="330.543"
+         clip-path="url(#SVGID_4704_-1)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83435-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="331.20099"
+         clip-path="url(#SVGID_4704_-1)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83437-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="332.38199"
+         clip-path="url(#SVGID_4704_-1)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83439-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83441-7"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83443-2">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4705_-7"
+           d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102583-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4705_-7"
+           overflow="visible"
+           id="use102585-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102587-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3544"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102589-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102591-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4707_-8)"
+         x="-636.69897"
+         y="331.974"
+         clip-path="url(#SVGID_4706_-25)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83454-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83456-8"
+       inkscape:transform-center-x="498.30125"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83458-2">
+        <rect
+           id="SVGID_4708_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102597-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4708_-8"
+           overflow="visible"
+           id="use102599-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4709_-6)"
+         d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83464-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="329.87"
+         clip-path="url(#SVGID_4709_-6)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83466-8" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="329.87"
+         clip-path="url(#SVGID_4709_-6)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83468-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="330.543"
+         clip-path="url(#SVGID_4709_-6)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83470-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="331.20099"
+         clip-path="url(#SVGID_4709_-6)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83472-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="332.38199"
+         clip-path="url(#SVGID_4709_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83474-0" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83476-5"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83478-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4710_-7"
+           d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102610-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4710_-7"
+           overflow="visible"
+           id="use102612-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102614-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102616-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102618-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4712_-3)"
+         x="-612.75598"
+         y="331.974"
+         clip-path="url(#SVGID_4711_-5)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83489-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83491-9"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83493-3">
+        <rect
+           id="SVGID_4713_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102624-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4713_-2"
+           overflow="visible"
+           id="use102626-1" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4714_-7)"
+         d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83499-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_-7)"
+         width="9.592"
+         height="9.592"
+         id="rect83501-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_-7)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83503-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="330.543"
+         clip-path="url(#SVGID_4714_-7)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83505-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="331.20099"
+         clip-path="url(#SVGID_4714_-7)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83507-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="332.38199"
+         clip-path="url(#SVGID_4714_-7)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83509-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83511-7"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83513-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4715_-7"
+           d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102637-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4715_-7"
+           overflow="visible"
+           id="use102639-2" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102641-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102643-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102645-3" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4717_-52)"
+         x="-600.78497"
+         y="331.974"
+         clip-path="url(#SVGID_4716_-5)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83524-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83526-2"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83528-5">
+        <rect
+           id="SVGID_4718_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102651-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4718_-5"
+           overflow="visible"
+           id="use102653-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4719_-0)"
+         d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83534-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_-0)"
+         width="9.592"
+         height="9.592"
+         id="rect83536-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83538-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="330.543"
+         clip-path="url(#SVGID_4719_-0)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83540-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="331.20099"
+         clip-path="url(#SVGID_4719_-0)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83542-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="332.38199"
+         clip-path="url(#SVGID_4719_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83544-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83546-3"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83548-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4720_-7"
+           d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102664-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4720_-7"
+           overflow="visible"
+           id="use102666-8" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102668-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102670-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102672-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4722_-33)"
+         x="-588.81299"
+         y="331.974"
+         clip-path="url(#SVGID_4721_-50)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83559-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.59059)"
+       id="g83756-0"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83758-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4750_-6"
+           d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102826-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4750_-6"
+           overflow="visible"
+           id="use102828-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102830-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102832-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102834-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4752_-4)"
+         x="-672.61401"
+         y="343.75699"
+         clip-path="url(#SVGID_4751_-1)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83769-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83771-2"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83773-9">
+        <rect
+           id="SVGID_4753_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102840-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4753_-7"
+           overflow="visible"
+           id="use102842-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4754_-0)"
+         d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83779-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_-0)"
+         width="9.592"
+         height="9.592"
+         id="rect83781-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83783-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="342.32599"
+         clip-path="url(#SVGID_4754_-0)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83785-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="342.98401"
+         clip-path="url(#SVGID_4754_-0)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83787-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="344.164"
+         clip-path="url(#SVGID_4754_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83789-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83791-4"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83793-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4755_-2"
+           d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102853-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4755_-2"
+           overflow="visible"
+           id="use102855-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102857-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102859-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102861-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4757_-6)"
+         x="-660.64203"
+         y="343.75699"
+         clip-path="url(#SVGID_4756_-57)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83804-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83806-2"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83808-9">
+        <rect
+           id="SVGID_4758_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102867-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4758_-6"
+           overflow="visible"
+           id="use102869-5" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4759_-5)"
+         d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83814-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83816-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83818-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="342.32599"
+         clip-path="url(#SVGID_4759_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83820-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="342.98401"
+         clip-path="url(#SVGID_4759_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83822-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="344.164"
+         clip-path="url(#SVGID_4759_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83824-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83826-5"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83828-3">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4760_-8"
+           d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102880-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4760_-8"
+           overflow="visible"
+           id="use102882-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102884-8"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102886-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102888-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4762_-0)"
+         x="-648.67102"
+         y="343.75699"
+         clip-path="url(#SVGID_4761_-32)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83839-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83841-0"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83843-2">
+        <rect
+           id="SVGID_4763_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102894-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4763_-6"
+           overflow="visible"
+           id="use102896-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4764_-8)"
+         d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83849-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_-8)"
+         width="9.592"
+         height="9.592"
+         id="rect83851-9" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_-8)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83853-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="342.32599"
+         clip-path="url(#SVGID_4764_-8)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83855-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="342.98401"
+         clip-path="url(#SVGID_4764_-8)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83857-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="344.164"
+         clip-path="url(#SVGID_4764_-8)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83859-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83861-1"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83863-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4765_-7"
+           d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102907-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4765_-7"
+           overflow="visible"
+           id="use102909-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102911-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102913-9" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102915-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4767_-9)"
+         x="-636.69897"
+         y="343.75699"
+         clip-path="url(#SVGID_4766_-0)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83874-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83876-5"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83878-1">
+        <rect
+           id="SVGID_4768_-0"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102921-8">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4768_-0"
+           overflow="visible"
+           id="use102923-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4769_-5)"
+         d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83884-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83886-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83888-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="342.32599"
+         clip-path="url(#SVGID_4769_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83890-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="342.98401"
+         clip-path="url(#SVGID_4769_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83892-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="344.164"
+         clip-path="url(#SVGID_4769_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83894-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83896-1"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83898-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4770_-0"
+           d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102934-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4770_-0"
+           overflow="visible"
+           id="use102936-2" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102938-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102940-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102942-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4772_-5)"
+         x="-624.72803"
+         y="343.75699"
+         clip-path="url(#SVGID_4771_-8)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83909-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83911-4"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83913-1">
+        <rect
+           id="SVGID_4773_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102948-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4773_-2"
+           overflow="visible"
+           id="use102950-1" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4774_-60)"
+         d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83919-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_-60)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83921-3" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_-60)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83923-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="342.32599"
+         clip-path="url(#SVGID_4774_-60)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83925-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="342.98401"
+         clip-path="url(#SVGID_4774_-60)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83927-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="344.164"
+         clip-path="url(#SVGID_4774_-60)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83929-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83931-6"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83933-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4775_-6"
+           d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102961-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4775_-6"
+           overflow="visible"
+           id="use102963-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102965-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102967-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102969-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4777_-42)"
+         x="-612.75598"
+         y="343.75699"
+         clip-path="url(#SVGID_4776_-8)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83944-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83946-6"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83948-3">
+        <rect
+           id="SVGID_4778_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102975-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4778_-7"
+           overflow="visible"
+           id="use102977-3" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4779_-83)"
+         d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83954-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_-83)"
+         width="9.592"
+         height="9.592"
+         id="rect83956-5" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_-83)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83958-2" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="342.32599"
+         clip-path="url(#SVGID_4779_-83)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83960-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="342.98401"
+         clip-path="url(#SVGID_4779_-83)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83962-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="344.164"
+         clip-path="url(#SVGID_4779_-83)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83964-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83966-0"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83968-6">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4780_-6"
+           d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102988-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4780_-6"
+           overflow="visible"
+           id="use102990-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102992-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102994-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102996-1" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4782_-6)"
+         x="-600.78497"
+         y="343.75699"
+         clip-path="url(#SVGID_4781_-0)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83979-0" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83981-7"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83983-7">
+        <rect
+           id="SVGID_4783_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103002-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4783_-9"
+           overflow="visible"
+           id="use103004-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4784_-5)"
+         d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83989-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83991-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83993-9" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="342.32599"
+         clip-path="url(#SVGID_4784_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83995-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="342.98401"
+         clip-path="url(#SVGID_4784_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83997-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="344.164"
+         clip-path="url(#SVGID_4784_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83999-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84001-7"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs84003-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4785_-4"
+           d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath103015-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4785_-4"
+           overflow="visible"
+           id="use103017-4" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103019-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop103021-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop103023-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4787_-8)"
+         x="-588.81299"
+         y="343.75699"
+         clip-path="url(#SVGID_4786_-1)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect84014-2" />
+    </g>
+    <text
+       y="567.96027"
+       x="435.21774"
+       id="text84236-1"
+       inkscape:transform-center-x="587.42372"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="435.21774"
+         y="567.96027"
+         font-size="3.4203"
+         id="tspan84238-2">EU-EAST</tspan>
+    </text>
+    <rect
+       style="fill:#f15d57"
+       x="399.26007"
+       y="492.96008"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84318-4"
+       inkscape:transform-center-x="587.03108"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       transform="translate(328.69122,-0.03041)"
+       id="g9328-2">
+      <line
+         id="line84244-3-6"
+         y2="533.36865"
+         x2="121.81088"
+         y1="315.62341"
+         x1="121.81088"
+         stroke-miterlimit="10"
+         style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499" />
+      <g
+         id="g82107-4-5"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82109-6-6">
+          <polygon
+             points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 "
+             id="SVGID_4519_-5-8" />
+        </defs>
+        <clipPath
+           id="clipPath126820-6-9">
+          <use
+             id="use126822-8-2"
+             overflow="visible"
+             xlink:href="#SVGID_4519_-5-8"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+           y2="146.6937"
+           x2="41.840599"
+           y1="146.6937"
+           x1="40.840599"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126824-4-4">
+          <stop
+             id="stop126826-2-0"
+             style="stop-color:#37424B"
+             offset="0" />
+          <stop
+             id="stop126828-3-6"
+             style="stop-color:#37424B"
+             offset="0.12" />
+          <stop
+             id="stop126830-7-8"
+             style="stop-color:#5E6A71"
+             offset="0.88" />
+          <stop
+             id="stop126832-6-2"
+             style="stop-color:#5E6A71"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82124-4-8"
+           height="10.191"
+           width="42.327"
+           clip-path="url(#SVGID_4520_-5-6-7)"
+           y="188.287"
+           x="-547.302"
+           style="fill:url(#linearGradient126904-61-3)" />
+      </g>
+      <g
+         id="g82126-0-8"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82128-8-2">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4522_-8-1" />
+        </defs>
+        <clipPath
+           id="clipPath126838-9-7">
+          <use
+             id="use126840-87-3"
+             overflow="visible"
+             xlink:href="#SVGID_4522_-8-1"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82134-5-4"
+           d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82136-0-8"
+           d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <circle
+           id="circle82138-2-3"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82140-3-7"
+           r="0.63999999"
+           cy="193.623"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82142-3-6"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-543.85303"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-543.85303"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82144-4-4"
+           r="0.63999999"
+           cy="193.623"
+           cx="-543.854"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-543.854"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <path
+           id="path82146-1-5"
+           d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82148-5-2"
+           d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+      </g>
+      <text
+         style="font-size:40.5px"
+         id="text82150-7-1"
+         x="112.11884"
+         y="319.30515">
+        <tspan
+           id="tspan82152-4-3"
+           font-size="3.2027"
+           y="319.30515"
+           x="112.11884"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">AP</tspan>
+        <tspan
+           id="tspan82154-9-7"
+           font-size="3.2027"
+           y="319.30515"
+           x="125.80093"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">P</tspan>
+      </text>
+      <g
+         id="g84638-4-9"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84640-9-8">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4852_-4-9" />
+        </defs>
+        <clipPath
+           id="clipPath126879-0-5">
+          <use
+             id="use126881-7-3"
+             overflow="visible"
+             xlink:href="#SVGID_4852_-4-9"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path84646-0-7"
+           d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4853_-8-0-4)"
+           inkscape:connector-curvature="0"
+           style="fill:#5e6a71" />
+        <line
+           id="line84648-6-5"
+           y2="205.991"
+           x2="-523.14398"
+           y1="205.991"
+           x1="-526.03998"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4853_-8-0-4)"
+           style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10" />
+      </g>
+      <g
+         id="g84650-5-5"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84652-4-9">
+          <polygon
+             points="-521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 "
+             id="SVGID_4854_-1-2" />
+        </defs>
+        <clipPath
+           id="clipPath126888-1-5">
+          <use
+             id="use126890-4-7"
+             overflow="visible"
+             xlink:href="#SVGID_4854_-1-2"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)"
+           y2="249.94031"
+           x2="-33.205101"
+           y1="249.94031"
+           x1="-34.205299"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126892-7-0">
+          <stop
+             id="stop126894-5-7"
+             style="stop-color:#E6E8E7"
+             offset="0" />
+          <stop
+             id="stop126896-6-7"
+             style="stop-color:#FFFFFF"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect84663-7-0"
+           height="5.066"
+           width="15.759"
+           clip-path="url(#SVGID_4855_-2-24-5)"
+           y="203.45799"
+           x="-523.43201"
+           style="fill:url(#SVGID_4856_-4-1-0)" />
+      </g>
+      <polygon
+         id="polygon84665-6-1"
+         points="-521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 -507.674,203.459 "
+         stroke-miterlimit="10"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)"
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10" />
+      <text
+         style="font-size:40.5px"
+         id="text84667-5-9"
+         x="151.0986"
+         y="363.4711">
+        <tspan
+           id="tspan84669-5-9"
+           font-size="2.5652"
+           y="363.4711"
+           x="151.0986"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">RE</tspan>
+        <tspan
+           id="tspan84671-5-4"
+           font-size="2.5652"
+           y="363.4711"
+           x="160.27521"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">S</tspan>
+        <tspan
+           id="tspan84673-2-5"
+           font-size="2.5652"
+           y="363.4711"
+           x="164.41296"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">T</tspan>
+      </text>
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84320-7"
+       inkscape:transform-center-x="587.03111"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84322-0">
+        <polygon
+           id="SVGID_4826_-2"
+           points="-659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103260-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4826_-2"
+           overflow="visible"
+           id="use103262-4" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103264-9"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103266-7" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103268-8" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103270-3" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103272-6" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103274-7" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4828_-8)"
+         clip-path="url(#SVGID_4827_-1)"
+         points="-682.238,231.929 -636.356,231.874 -636.326,256.788 -682.208,256.843 "
+         id="polygon84339-1" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 399.73931,496.49039 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.002,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+       id="path84341-5"
+       inkscape:transform-center-x="587.03109"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.25391"
+       x="422.89722"
+       id="text84343-4"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="422.89722"
+         y="513.25391"
+         font-size="5.1244"
+         id="tspan84345-6">RADOSGW</tspan>
+    </text>
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="450.98538"
+       y1="577.31482"
+       x2="450.98538"
+       y2="631.69733"
+       id="line84349-2"
+       inkscape:transform-center-x="587.12555"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       stroke-miterlimit="10"
+       points="  -669.837,292.128 -669.837,283.832 -648.812,283.832 -648.812,292.128 "
+       id="polyline84351-0"
+       inkscape:transform-center-x="587.12673"
+       inkscape:transform-center-y="-33.395941" />
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       points="-657.251,267.424 -661.37,267.424 -659.319,269.217 "
+       id="polygon84353-1"
+       inkscape:transform-center-x="587.09523"
+       inkscape:transform-center-y="-77.629816" />
+    <polygon
+       id="polygon84546-2"
+       points="-671.896,292.47 -669.845,294.263 -667.777,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <polygon
+       id="polygon84548-8"
+       points="-661.384,292.47 -659.333,294.263 -657.265,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <polygon
+       id="polygon84550-0"
+       points="-650.871,292.47 -648.82,294.263 -646.752,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84753-5"
+       inkscape:transform-center-x="584.45598"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84755-4">
+        <rect
+           id="SVGID_4867_-0"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103568-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4867_-0"
+           overflow="visible"
+           id="use103570-2" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4868_-1)"
+         d="m -659.955,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84761-2" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4868_-1)"
+         stroke-miterlimit="10"
+         x1="-659.323"
+         y1="276.099"
+         x2="-656.427"
+         y2="276.099"
+         id="line84763-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84765-0"
+       inkscape:transform-center-x="563.5265"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84767-2">
+        <polygon
+           id="SVGID_4869_-7"
+           points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103577-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4869_-7"
+           overflow="visible"
+           id="use103579-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103581-2"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94051"
+         x2="-33.205299"
+         y2="249.94051"
+         gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103583-7" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103585-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4871_-4)"
+         x="-656.71503"
+         y="273.56699"
+         clip-path="url(#SVGID_4870_-0)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84778-5" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       stroke-miterlimit="10"
+       points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 "
+       id="polygon84780-4"
+       inkscape:transform-center-x="563.52648"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.59412"
+       x="466.59583"
+       id="text84782-6"
+       inkscape:transform-center-x="562.26258"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="466.59583"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84784-6">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="470.47482"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84786-0">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="473.54385"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84788-6">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="481.25684"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84790-4">E</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,1325.4162,-31.59059)"
+       id="g84792-1"
+       inkscape:transform-center-x="262.77489"
+       inkscape:transform-center-y="141.90493">
+      <defs
+         id="defs84794-1">
+        <rect
+           id="SVGID_4872_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103597-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4872_-8"
+           overflow="visible"
+           id="use103599-5" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4873_-98)"
+         d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+         id="path84800-2" />
+      <g
+         clip-path="url(#SVGID_4873_-98)"
+         id="g84802-7">
+        <g
+           id="g84804-9">
+          <defs
+             id="defs84806-7">
+            <rect
+               id="SVGID_4874_-4"
+               x="-679.90002"
+               y="360.13901"
+               width="174.95799"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103606-3">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4874_-4"
+               overflow="visible"
+               id="use103608-9" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4875_-1)"
+             id="g84812-3">
+            <defs
+               id="defs84814-8">
+              <rect
+                 id="SVGID_4876_-1"
+                 x="-679.90002"
+                 y="360.13901"
+                 width="174.95799"
+                 height="10.152" />
+            </defs>
+            <clipPath
+               id="clipPath103613-6">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4876_-1"
+                 overflow="visible"
+                 id="use103615-0" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient103617-6"
+               gradientUnits="userSpaceOnUse"
+               x1="40.563"
+               y1="147.0757"
+               x2="41.563"
+               y2="147.0757"
+               gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+              <stop
+                 offset="0"
+                 style="stop-color:#55AEB9"
+                 id="stop103619-5" />
+              <stop
+                 offset="0.486"
+                 style="stop-color:#B0D2D9"
+                 id="stop103621-2" />
+              <stop
+                 offset="0.8287"
+                 style="stop-color:#E6EFF1"
+                 id="stop103623-9" />
+              <stop
+                 offset="0.9939"
+                 style="stop-color:#FFFFFF"
+                 id="stop103625-9" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop103627-0" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4878_-2)"
+               x="-679.90002"
+               y="360.138"
+               clip-path="url(#SVGID_4877_-4)"
+               width="174.95799"
+               height="10.153"
+               id="rect84831-7" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4873_-98)"
+         id="g84833-5">
+        <g
+           id="g84835-6">
+          <defs
+             id="defs84837-5">
+            <rect
+               id="SVGID_4879_-3"
+               x="-679.88898"
+               y="360.13901"
+               width="174.94701"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103634-6">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4879_-3"
+               overflow="visible"
+               id="use103636-3" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4880_-2)"
+             id="g84843-9">
+            <defs
+               id="defs84845-0">
+              <rect
+                 id="SVGID_4881_-4"
+                 x="-679.88898"
+                 y="360.138"
+                 width="174.94701"
+                 height="10.153" />
+            </defs>
+            <clipPath
+               id="clipPath103641-9">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4881_-4"
+                 overflow="visible"
+                 id="use103643-2" />
+            </clipPath>
+            <polygon
+               style="fill:#f15d57"
+               clip-path="url(#SVGID_4882_-3)"
+               points="-504.943,360.138 -679.89,360.139 -679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 "
+               id="polygon84851-1" />
+          </g>
+        </g>
+      </g>
+      <rect
+         style="fill:#f15d57"
+         x="-679.90198"
+         y="360.267"
+         clip-path="url(#SVGID_4873_-98)"
+         width="174.96001"
+         height="2.641"
+         id="rect84853-2" />
+      <rect
+         style="fill:#55aeb9"
+         x="-679.88898"
+         y="370.29199"
+         clip-path="url(#SVGID_4873_-98)"
+         width="174.94701"
+         height="1.352"
+         id="rect84855-1" />
+    </g>
+    <text
+       y="832.0752"
+       x="571.10132"
+       id="text84343-2-09"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="526.12915"
+         y="832.0752"
+         font-size="5.1244"
+         id="tspan84345-2-7">SECONDARY REGION</tspan>
+    </text>
+    <polygon
+       id="polygon84546-9-3-6"
+       points="-667.777,292.47 -671.896,292.47 -669.845,294.263 "
+       clip-path="url(#SVGID_4843_-9-6-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1917.6233,-169.74298)" />
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="450.92264"
+       y="529.61548"
+       id="text126655-6-3"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126657-2-8"
+         x="450.92264"
+         y="529.61548">MASTER</tspan><tspan
+         sodipodi:role="line"
+         x="450.92264"
+         y="540.86548"
+         id="tspan126659-4-8">ZONE</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="363.76648"
+       y="400.13824"
+       id="text127418-7"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127420-8"
+         x="363.76648"
+         y="400.13824">METADATA</tspan><tspan
+         sodipodi:role="line"
+         x="363.76648"
+         y="411.38824"
+         id="tspan127422-7">SYNC</tspan></text>
+    <text
+       y="247.57471"
+       x="491.10132"
+       id="text84343-2-0-2"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="446.12915"
+         y="247.57471"
+         font-size="5.1244"
+         id="tspan84345-2-6-0">SECONDARY REGION</tspan>
+    </text>
+    <text
+       y="705.34888"
+       x="583.92261"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-9-7"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       id="g9504-6"
+       transform="translate(98.691216,33.96959)">
+      <g
+         inkscape:transform-center-y="45.666809"
+         inkscape:transform-center-x="449.81598"
+         id="g82339-6-5"
+         transform="matrix(2.25,0,0,2.25,1701.0081,-85.193202)">
+        <defs
+           id="defs82341-9-3">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4549_-8-0" />
+        </defs>
+        <clipPath
+           id="clipPath101738-3-9">
+          <use
+             id="use101740-3-4"
+             overflow="visible"
+             xlink:href="#SVGID_4549_-8-0"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82347-9-8"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4550_-6-4)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82349-9-3"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4550_-6-4)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9-4-7"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="350.6293"
+         y="645.37921">M</text>
+    </g>
+    <path
+       id="path82449-2-4"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-39)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.2338,-25.49647)" />
+    <path
+       id="path82449-2-2-4"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-3-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1907.5274,-25.10599)" />
+    <path
+       id="path82449-2-0-6"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-8)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.2338,0.56753)" />
+    <path
+       id="path82449-2-0-1-9"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-43)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.3338,26.56753)" />
+    <path
+       id="path82449-2-0-1-5-7"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-4-4)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.4335,53.16753)" />
+    <text
+       y="293.44104"
+       x="496.78619"
+       id="text84343-2-0-4-8"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="451.81403"
+         y="293.44104"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-9">WRITE / READ</tspan>
+    </text>
+    <text
+       y="262.3537"
+       x="490.88916"
+       id="text84343-2-0-4-6-1"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="445.91699"
+         y="262.3537"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-6-6">(Europe)</tspan>
+    </text>
+    <g
+       id="g15866"
+       transform="translate(66.384186,10.199929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5">
+          <use
+             id="use101775-6-5"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8">
+          <stop
+             id="stop101779-4-4"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4" />
+    </g>
+    <g
+       id="g15866-2"
+       transform="translate(66.384186,36.234929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3-5"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0-2"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3-6"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0-6"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0-8"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0-3">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9-1"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5-0">
+          <use
+             id="use101775-6-5-6"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9-1"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8-6">
+          <stop
+             id="stop101779-4-4-3"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3-8"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9-8"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2-5)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887-0)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7-5)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4-2" />
+    </g>
+    <g
+       id="g15866-2-3"
+       transform="translate(66.384186,62.334929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3-5-9"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0-2-8"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3-6-8"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0-6-5"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0-8-1"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0-3-9">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9-1-0"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5-0-4">
+          <use
+             id="use101775-6-5-6-5"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9-1-0"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8-6-8">
+          <stop
+             id="stop101779-4-4-3-2"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3-8-5"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9-8-1"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2-5-8)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887-0-3)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7-5-3)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4-2-3" />
+    </g>
+  </g>
+</svg>
diff --git a/doc/images/rgw-encryption-barbican.png b/doc/images/rgw-encryption-barbican.png
new file mode 100644
index 000000000..e3c10c4a7
--- /dev/null
+++ b/doc/images/rgw-encryption-barbican.png
diff --git a/doc/images/stack.png b/doc/images/stack.png
new file mode 100644
index 000000000..38eac41b0
--- /dev/null
+++ b/doc/images/stack.png
diff --git a/doc/images/win2016_iscsi_advanced_window.png b/doc/images/win2016_iscsi_advanced_window.png
new file mode 100755
index 000000000..cba70ea89
--- /dev/null
+++ b/doc/images/win2016_iscsi_advanced_window.png
diff --git a/doc/images/win2016_iscsi_connect_to_target.png b/doc/images/win2016_iscsi_connect_to_target.png
new file mode 100755
index 000000000..29d9e1267
--- /dev/null
+++ b/doc/images/win2016_iscsi_connect_to_target.png
diff --git a/doc/images/win2016_iscsi_devices_mpio.png b/doc/images/win2016_iscsi_devices_mpio.png
new file mode 100755
index 000000000..135d41bf5
--- /dev/null
+++ b/doc/images/win2016_iscsi_devices_mpio.png
diff --git a/doc/images/win2016_iscsi_discovery_tab.png b/doc/images/win2016_iscsi_discovery_tab.png
new file mode 100755
index 000000000..4ac5f71d9
--- /dev/null
+++ b/doc/images/win2016_iscsi_discovery_tab.png
diff --git a/doc/images/win2016_iscsi_target_tab.png b/doc/images/win2016_iscsi_target_tab.png
new file mode 100755
index 000000000..75437692b
--- /dev/null
+++ b/doc/images/win2016_iscsi_target_tab.png
diff --git a/doc/images/win2016_iscsi_target_tab2.png b/doc/images/win2016_iscsi_target_tab2.png
new file mode 100755
index 000000000..302fef918
--- /dev/null
+++ b/doc/images/win2016_iscsi_target_tab2.png
diff --git a/doc/images/win2016_mpclaim_output.png b/doc/images/win2016_mpclaim_output.png
new file mode 100644
index 000000000..73e1e5ed2
--- /dev/null
+++ b/doc/images/win2016_mpclaim_output.png
diff --git a/doc/images/win2016_mpio_set_failover_only.png b/doc/images/win2016_mpio_set_failover_only.png
new file mode 100755
index 000000000..a988e8a07
--- /dev/null
+++ b/doc/images/win2016_mpio_set_failover_only.png
diff --git a/doc/images/zone-sync.png b/doc/images/zone-sync.png
new file mode 100644
index 000000000..c7bf9efba
--- /dev/null
+++ b/doc/images/zone-sync.png
diff --git a/doc/images/zone-sync.svg b/doc/images/zone-sync.svg
new file mode 100644
index 000000000..b73302f5c
--- /dev/null
+++ b/doc/images/zone-sync.svg
@@ -0,0 +1,27090 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="379.63181"
+   height="638.90717"
+   id="svg104536"
+   version="1.1"
+   inkscape:version="1.2.2 (b0a8486541, 2022-12-01)"
+   sodipodi:docname="zone-sync.svg"
+   inkscape:export-filename="/home/john/Pictures/zone-sync.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb">
+  <defs
+     id="defs104538">
+    <linearGradient
+       id="linearGradient126200"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4884_">
+      <rect
+         height="11.504"
+         width="174.63"
+         y="360.13901"
+         x="-679.57202"
+         id="use84863" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4886_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84873" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4888_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84881" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4890_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-535.73901"
+         id="use84891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4892_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-535.73901"
+         id="use84899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99071">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99073" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4894_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-536.966"
+         id="use84911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4896_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-536.966"
+         id="use84919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99079">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99081" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4898_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4900_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99087">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99089" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4902_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4904_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84959" />
+    </clipPath>
+    <clipPath
+       id="clipPath99095">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4906_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-540.646"
+         id="use84971" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4908_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-540.646"
+         id="use84979" />
+    </clipPath>
+    <clipPath
+       id="clipPath99103">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99105" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4910_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-541.87299"
+         id="use84991" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4912_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-541.87299"
+         id="use84999" />
+    </clipPath>
+    <clipPath
+       id="clipPath99111">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4914_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85011" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4916_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85019" />
+    </clipPath>
+    <clipPath
+       id="clipPath99119">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99121" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4918_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85031" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4920_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85039" />
+    </clipPath>
+    <clipPath
+       id="clipPath99127">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99129" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4922_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85051" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4924_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85059" />
+    </clipPath>
+    <clipPath
+       id="clipPath99135">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99137" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4926_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-546.77899"
+         id="use85071" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4928_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-546.77899"
+         id="use85079" />
+    </clipPath>
+    <clipPath
+       id="clipPath99143">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99145" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4930_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85091" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4932_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85099" />
+    </clipPath>
+    <clipPath
+       id="clipPath99151">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99153" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4934_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85111" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4936_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85119" />
+    </clipPath>
+    <clipPath
+       id="clipPath99159">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99161" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4938_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85131" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4940_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85139" />
+    </clipPath>
+    <clipPath
+       id="clipPath99167">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99169" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4942_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85151" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4944_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85159" />
+    </clipPath>
+    <clipPath
+       id="clipPath99175">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99177" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4946_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85171" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4948_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85179" />
+    </clipPath>
+    <clipPath
+       id="clipPath99183">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99185" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4950_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85191" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4952_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85199" />
+    </clipPath>
+    <clipPath
+       id="clipPath99191">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99193" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4954_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-555.367"
+         id="use85211" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4956_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-555.367"
+         id="use85219" />
+    </clipPath>
+    <clipPath
+       id="clipPath99199">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99201" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4958_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85231" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4960_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85239" />
+    </clipPath>
+    <clipPath
+       id="clipPath99207">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99209" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4962_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85251" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4964_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85259" />
+    </clipPath>
+    <clipPath
+       id="clipPath99215">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99217" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4966_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85271" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4968_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85279" />
+    </clipPath>
+    <clipPath
+       id="clipPath99223">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99225" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4970_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85291" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4972_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85299" />
+    </clipPath>
+    <clipPath
+       id="clipPath99231">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99233" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4974_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85311" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4976_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85319" />
+    </clipPath>
+    <clipPath
+       id="clipPath99239">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99241" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4978_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85331" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4980_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85339" />
+    </clipPath>
+    <clipPath
+       id="clipPath99247">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99249" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4982_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85351" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4984_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85359" />
+    </clipPath>
+    <clipPath
+       id="clipPath99255">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99257" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4986_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-565.17999"
+         id="use85371" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4988_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-565.17999"
+         id="use85379" />
+    </clipPath>
+    <clipPath
+       id="clipPath99263">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99265" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4990_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85391" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4992_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85399" />
+    </clipPath>
+    <clipPath
+       id="clipPath99271">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99273" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4994_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85411" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4996_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85419" />
+    </clipPath>
+    <clipPath
+       id="clipPath99279">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99281" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4998_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85431" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5000_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85439" />
+    </clipPath>
+    <clipPath
+       id="clipPath99287">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99289" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5002_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85451" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5004_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85459" />
+    </clipPath>
+    <clipPath
+       id="clipPath99295">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99297" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5006_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-571.315"
+         id="use85471" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5008_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-571.31403"
+         id="use85479" />
+    </clipPath>
+    <clipPath
+       id="clipPath99303">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99305" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5010_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85491" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5012_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85499" />
+    </clipPath>
+    <clipPath
+       id="clipPath99311">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99313" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5014_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-573.76801"
+         id="use85511" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5016_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-573.76801"
+         id="use85519" />
+    </clipPath>
+    <clipPath
+       id="clipPath99319">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99321" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5018_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-574.99402"
+         id="use85531" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5020_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-574.99402"
+         id="use85539" />
+    </clipPath>
+    <clipPath
+       id="clipPath99327">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99329" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5022_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85551" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5024_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85559" />
+    </clipPath>
+    <clipPath
+       id="clipPath99335">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99337" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5026_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85571" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5028_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85579" />
+    </clipPath>
+    <clipPath
+       id="clipPath99343">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99345" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5030_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-578.67499"
+         id="use85591" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5032_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-578.67499"
+         id="use85599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99351">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99353" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5034_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85611" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5036_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99359">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99361" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5038_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-581.12799"
+         id="use85631" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5040_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-581.12799"
+         id="use85639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99367">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99369" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5042_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85651" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5044_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99375">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5046_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85671" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5048_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99383">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99385" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5050_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85691" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5052_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99391">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99393" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5054_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-586.03497"
+         id="use85711" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5056_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-586.03497"
+         id="use85719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99399">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99401" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5058_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85731" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5060_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85739" />
+    </clipPath>
+    <clipPath
+       id="clipPath99407">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5062_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-588.48901"
+         id="use85751" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5064_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-588.48901"
+         id="use85759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99415">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99417" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5066_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5068_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85779" />
+    </clipPath>
+    <clipPath
+       id="clipPath99423">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99425" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5070_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85791" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5072_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99431">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99433" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5074_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85811" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5076_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99439">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99441" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5078_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5080_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85839" />
+    </clipPath>
+    <clipPath
+       id="clipPath99447">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99449" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5082_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-594.62299"
+         id="use85851" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5084_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-594.62299"
+         id="use85859" />
+    </clipPath>
+    <clipPath
+       id="clipPath99455">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99457" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5086_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85871" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5088_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85879" />
+    </clipPath>
+    <clipPath
+       id="clipPath99463">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99465" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5090_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5092_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99471">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99473" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5094_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5096_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99479">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99481" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5098_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5100_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99487">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99489" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5102_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5104_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85959" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use84798" />
+    </clipPath>
+    <clipPath
+       id="clipPath99497">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99499" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_"
+         overflow="visible"
+         id="use84818" />
+    </clipPath>
+    <clipPath
+       id="clipPath99511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99513" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_"
+         overflow="visible"
+         id="use84849" />
+    </clipPath>
+    <clipPath
+       id="clipPath99519">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99521" />
+    </clipPath>
+    <clipPath
+       id="clipPath99523">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99525" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4869_"
+         overflow="visible"
+         id="use84771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_"
+         overflow="visible"
+         id="use84759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_"
+         overflow="visible"
+         id="use99536" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4864_"
+         overflow="visible"
+         id="use84732" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use84720" />
+    </clipPath>
+    <clipPath
+       id="clipPath99545">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use99547" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4861_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205299"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84696" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84698" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4860_">
+      <polygon
+         points="-507.673,273.567 -521.64,273.567 -523.432,276.108 -521.64,278.632 -507.673,278.632 "
+         id="use84693" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4858_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84681" />
+    </clipPath>
+    <clipPath
+       id="clipPath99556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99558" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_">
+      <polygon
+         points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 "
+         id="use84656" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644" />
+    </clipPath>
+    <clipPath
+       id="clipPath99567">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99569" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_">
+      <polygon
+         points="-588.288,208.524 -574.322,208.524 -574.322,203.458 -588.288,203.458 -590.081,206 "
+         id="use84619" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99580" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_">
+      <polygon
+         points="-654.922,208.524 -640.956,208.524 -640.956,203.458 -654.922,203.458 -656.715,206 "
+         id="use84582" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500" />
+    </clipPath>
+    <clipPath
+       id="clipPath99589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99591" />
+    </clipPath>
+    <clipPath
+       id="clipPath99593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99595" />
+    </clipPath>
+    <clipPath
+       id="clipPath99597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99601">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99603" />
+    </clipPath>
+    <clipPath
+       id="clipPath99605">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99609">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99611" />
+    </clipPath>
+    <clipPath
+       id="clipPath99613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99615" />
+    </clipPath>
+    <clipPath
+       id="clipPath99617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99623" />
+    </clipPath>
+    <clipPath
+       id="clipPath99625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99627" />
+    </clipPath>
+    <clipPath
+       id="clipPath99629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99631" />
+    </clipPath>
+    <clipPath
+       id="clipPath99633">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99635" />
+    </clipPath>
+    <clipPath
+       id="clipPath99637">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99641">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99643" />
+    </clipPath>
+    <clipPath
+       id="clipPath99645">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99647" />
+    </clipPath>
+    <clipPath
+       id="clipPath99649">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99651" />
+    </clipPath>
+    <clipPath
+       id="clipPath99653">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99655" />
+    </clipPath>
+    <clipPath
+       id="clipPath99657">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99661">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99663" />
+    </clipPath>
+    <clipPath
+       id="clipPath99665">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99667" />
+    </clipPath>
+    <clipPath
+       id="clipPath99669">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99671" />
+    </clipPath>
+    <clipPath
+       id="clipPath99673">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99675" />
+    </clipPath>
+    <clipPath
+       id="clipPath99677">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99681">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99683" />
+    </clipPath>
+    <clipPath
+       id="clipPath99685">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99687" />
+    </clipPath>
+    <clipPath
+       id="clipPath99689">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99691" />
+    </clipPath>
+    <clipPath
+       id="clipPath99693">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99695" />
+    </clipPath>
+    <clipPath
+       id="clipPath99697">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99701">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99703" />
+    </clipPath>
+    <clipPath
+       id="clipPath99705">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99707" />
+    </clipPath>
+    <clipPath
+       id="clipPath99709">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99711" />
+    </clipPath>
+    <clipPath
+       id="clipPath99713">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99715" />
+    </clipPath>
+    <clipPath
+       id="clipPath99717">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99723" />
+    </clipPath>
+    <clipPath
+       id="clipPath99725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99727" />
+    </clipPath>
+    <clipPath
+       id="clipPath99729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99731" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445" />
+    </clipPath>
+    <clipPath
+       id="clipPath99743">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99745" />
+    </clipPath>
+    <clipPath
+       id="clipPath99747">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99749" />
+    </clipPath>
+    <clipPath
+       id="clipPath99751">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99753" />
+    </clipPath>
+    <clipPath
+       id="clipPath99755">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99757" />
+    </clipPath>
+    <clipPath
+       id="clipPath99759">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99761" />
+    </clipPath>
+    <clipPath
+       id="clipPath99763">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99765" />
+    </clipPath>
+    <clipPath
+       id="clipPath99767">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99769" />
+    </clipPath>
+    <clipPath
+       id="clipPath99771">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99773" />
+    </clipPath>
+    <clipPath
+       id="clipPath99775">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99777" />
+    </clipPath>
+    <clipPath
+       id="clipPath99779">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99781" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390" />
+    </clipPath>
+    <clipPath
+       id="clipPath99793">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99795" />
+    </clipPath>
+    <clipPath
+       id="clipPath99797">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99801">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99803" />
+    </clipPath>
+    <clipPath
+       id="clipPath99805">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99807" />
+    </clipPath>
+    <clipPath
+       id="clipPath99809">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99811" />
+    </clipPath>
+    <clipPath
+       id="clipPath99813">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99815" />
+    </clipPath>
+    <clipPath
+       id="clipPath99817">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99821">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99823" />
+    </clipPath>
+    <clipPath
+       id="clipPath99825">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99827" />
+    </clipPath>
+    <clipPath
+       id="clipPath99829">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99831" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4826_"
+         overflow="visible"
+         id="use84326" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4823_"
+         overflow="visible"
+         id="use84295" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4822_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.317"
+       y1="2.8515"
+       x2="146.317"
+       y2="2.8515"
+       gradientTransform="matrix(0.0299,24.8863,24.8863,-0.0299,-601.3687,-3384.4197)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84267" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84269" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84271" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84273" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4821_">
+      <polygon
+         points="-548.973,231.929 -548.973,251.221 -526.108,256.788 -503.122,251.283 -503.124,231.934 "
+         id="use84264" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4819_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84232" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4817_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84220" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4816_">
+      <path
+         d="m -516.985,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84217"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4814_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84197" />
+    </clipPath>
+    <clipPath
+       id="clipPath99874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99876" />
+    </clipPath>
+    <clipPath
+       id="clipPath99878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99880" />
+    </clipPath>
+    <clipPath
+       id="clipPath99882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99884" />
+    </clipPath>
+    <clipPath
+       id="clipPath99886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99888" />
+    </clipPath>
+    <clipPath
+       id="clipPath99890">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99892" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4812_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84185" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4811_">
+      <path
+         d="m -528.956,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84182"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4809_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84162" />
+    </clipPath>
+    <clipPath
+       id="clipPath99901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99903" />
+    </clipPath>
+    <clipPath
+       id="clipPath99905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99907" />
+    </clipPath>
+    <clipPath
+       id="clipPath99909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99911" />
+    </clipPath>
+    <clipPath
+       id="clipPath99913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99915" />
+    </clipPath>
+    <clipPath
+       id="clipPath99917">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99919" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4807_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84150" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84152" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4806_">
+      <path
+         d="m -540.927,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84147"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4804_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84127" />
+    </clipPath>
+    <clipPath
+       id="clipPath99928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99930" />
+    </clipPath>
+    <clipPath
+       id="clipPath99932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99934" />
+    </clipPath>
+    <clipPath
+       id="clipPath99936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99938" />
+    </clipPath>
+    <clipPath
+       id="clipPath99940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99942" />
+    </clipPath>
+    <clipPath
+       id="clipPath99944">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99946" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4802_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84115" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84117" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4801_">
+      <path
+         d="m -552.899,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84112"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4799_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84092" />
+    </clipPath>
+    <clipPath
+       id="clipPath99955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99957" />
+    </clipPath>
+    <clipPath
+       id="clipPath99959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99961" />
+    </clipPath>
+    <clipPath
+       id="clipPath99963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99965" />
+    </clipPath>
+    <clipPath
+       id="clipPath99967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99969" />
+    </clipPath>
+    <clipPath
+       id="clipPath99971">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99973" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4797_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,237.034,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84080" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84082" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4796_">
+      <path
+         d="m -564.87,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84077"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4794_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84057" />
+    </clipPath>
+    <clipPath
+       id="clipPath99982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99984" />
+    </clipPath>
+    <clipPath
+       id="clipPath99986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99988" />
+    </clipPath>
+    <clipPath
+       id="clipPath99990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99992" />
+    </clipPath>
+    <clipPath
+       id="clipPath99994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99996" />
+    </clipPath>
+    <clipPath
+       id="clipPath99998">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100000" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4792_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84045" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84047" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4791_">
+      <path
+         d="m -576.842,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84042"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4789_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84022" />
+    </clipPath>
+    <clipPath
+       id="clipPath100009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100011" />
+    </clipPath>
+    <clipPath
+       id="clipPath100013">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100015" />
+    </clipPath>
+    <clipPath
+       id="clipPath100017">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100019" />
+    </clipPath>
+    <clipPath
+       id="clipPath100021">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100023" />
+    </clipPath>
+    <clipPath
+       id="clipPath100025">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100027" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_"
+         overflow="visible"
+         id="use84007" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use83987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100036">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100038" />
+    </clipPath>
+    <clipPath
+       id="clipPath100040">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100044">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100046" />
+    </clipPath>
+    <clipPath
+       id="clipPath100048">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100050" />
+    </clipPath>
+    <clipPath
+       id="clipPath100052">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100054" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_"
+         overflow="visible"
+         id="use83972" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use83952" />
+    </clipPath>
+    <clipPath
+       id="clipPath100063">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100065" />
+    </clipPath>
+    <clipPath
+       id="clipPath100067">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100069" />
+    </clipPath>
+    <clipPath
+       id="clipPath100071">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100073" />
+    </clipPath>
+    <clipPath
+       id="clipPath100075">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100079">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100081" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_"
+         overflow="visible"
+         id="use83937" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use83917" />
+    </clipPath>
+    <clipPath
+       id="clipPath100090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100092" />
+    </clipPath>
+    <clipPath
+       id="clipPath100094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100096" />
+    </clipPath>
+    <clipPath
+       id="clipPath100098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100100" />
+    </clipPath>
+    <clipPath
+       id="clipPath100102">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100104" />
+    </clipPath>
+    <clipPath
+       id="clipPath100106">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100108" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_"
+         overflow="visible"
+         id="use83902" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use83882" />
+    </clipPath>
+    <clipPath
+       id="clipPath100117">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100119" />
+    </clipPath>
+    <clipPath
+       id="clipPath100121">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100123" />
+    </clipPath>
+    <clipPath
+       id="clipPath100125">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100127" />
+    </clipPath>
+    <clipPath
+       id="clipPath100129">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100131" />
+    </clipPath>
+    <clipPath
+       id="clipPath100133">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100135" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_"
+         overflow="visible"
+         id="use83867" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use83847" />
+    </clipPath>
+    <clipPath
+       id="clipPath100144">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100146" />
+    </clipPath>
+    <clipPath
+       id="clipPath100148">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100150" />
+    </clipPath>
+    <clipPath
+       id="clipPath100152">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100154" />
+    </clipPath>
+    <clipPath
+       id="clipPath100156">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100158" />
+    </clipPath>
+    <clipPath
+       id="clipPath100160">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100162" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_"
+         overflow="visible"
+         id="use83832" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use83812" />
+    </clipPath>
+    <clipPath
+       id="clipPath100171">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100173" />
+    </clipPath>
+    <clipPath
+       id="clipPath100175">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100177" />
+    </clipPath>
+    <clipPath
+       id="clipPath100179">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100181" />
+    </clipPath>
+    <clipPath
+       id="clipPath100183">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100185" />
+    </clipPath>
+    <clipPath
+       id="clipPath100187">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100189" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_"
+         overflow="visible"
+         id="use83797" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use83777" />
+    </clipPath>
+    <clipPath
+       id="clipPath100198">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100200" />
+    </clipPath>
+    <clipPath
+       id="clipPath100202">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100204" />
+    </clipPath>
+    <clipPath
+       id="clipPath100206">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100208" />
+    </clipPath>
+    <clipPath
+       id="clipPath100210">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100212" />
+    </clipPath>
+    <clipPath
+       id="clipPath100214">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100216" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_"
+         overflow="visible"
+         id="use83762" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use83742" />
+    </clipPath>
+    <clipPath
+       id="clipPath100225">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100227" />
+    </clipPath>
+    <clipPath
+       id="clipPath100229">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100231" />
+    </clipPath>
+    <clipPath
+       id="clipPath100233">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100235" />
+    </clipPath>
+    <clipPath
+       id="clipPath100237">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100239" />
+    </clipPath>
+    <clipPath
+       id="clipPath100241">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100243" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4747_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83730" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83732" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4746_">
+      <path
+         d="m -516.985,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83727"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4744_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83707" />
+    </clipPath>
+    <clipPath
+       id="clipPath100252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100254" />
+    </clipPath>
+    <clipPath
+       id="clipPath100256">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100258" />
+    </clipPath>
+    <clipPath
+       id="clipPath100260">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100262" />
+    </clipPath>
+    <clipPath
+       id="clipPath100264">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100266" />
+    </clipPath>
+    <clipPath
+       id="clipPath100268">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100270" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4742_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83695" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83697" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4741_">
+      <path
+         d="m -528.956,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83692"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4739_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83672" />
+    </clipPath>
+    <clipPath
+       id="clipPath100279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100281" />
+    </clipPath>
+    <clipPath
+       id="clipPath100283">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100285" />
+    </clipPath>
+    <clipPath
+       id="clipPath100287">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100289" />
+    </clipPath>
+    <clipPath
+       id="clipPath100291">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100293" />
+    </clipPath>
+    <clipPath
+       id="clipPath100295">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4737_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83660" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83662" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4736_">
+      <path
+         d="m -540.927,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83657"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4734_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83637" />
+    </clipPath>
+    <clipPath
+       id="clipPath100306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100308" />
+    </clipPath>
+    <clipPath
+       id="clipPath100310">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100312" />
+    </clipPath>
+    <clipPath
+       id="clipPath100314">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100316" />
+    </clipPath>
+    <clipPath
+       id="clipPath100318">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100320" />
+    </clipPath>
+    <clipPath
+       id="clipPath100322">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4732_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83625" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83627" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4731_">
+      <path
+         d="m -552.899,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83622"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4729_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83602" />
+    </clipPath>
+    <clipPath
+       id="clipPath100333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100335" />
+    </clipPath>
+    <clipPath
+       id="clipPath100337">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100339" />
+    </clipPath>
+    <clipPath
+       id="clipPath100341">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100343" />
+    </clipPath>
+    <clipPath
+       id="clipPath100345">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100347" />
+    </clipPath>
+    <clipPath
+       id="clipPath100349">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4727_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83590" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83592" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4726_">
+      <path
+         d="m -576.842,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83587"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4724_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83567" />
+    </clipPath>
+    <clipPath
+       id="clipPath100360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100362" />
+    </clipPath>
+    <clipPath
+       id="clipPath100364">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100366" />
+    </clipPath>
+    <clipPath
+       id="clipPath100368">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100370" />
+    </clipPath>
+    <clipPath
+       id="clipPath100372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100374" />
+    </clipPath>
+    <clipPath
+       id="clipPath100376">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_"
+         overflow="visible"
+         id="use83552" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use83532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100389" />
+    </clipPath>
+    <clipPath
+       id="clipPath100391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100393" />
+    </clipPath>
+    <clipPath
+       id="clipPath100395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100397" />
+    </clipPath>
+    <clipPath
+       id="clipPath100399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100401" />
+    </clipPath>
+    <clipPath
+       id="clipPath100403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_"
+         overflow="visible"
+         id="use83517" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use83497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100416" />
+    </clipPath>
+    <clipPath
+       id="clipPath100418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100420" />
+    </clipPath>
+    <clipPath
+       id="clipPath100422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100424" />
+    </clipPath>
+    <clipPath
+       id="clipPath100426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100428" />
+    </clipPath>
+    <clipPath
+       id="clipPath100430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_"
+         overflow="visible"
+         id="use83482" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use83462" />
+    </clipPath>
+    <clipPath
+       id="clipPath100441">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100443" />
+    </clipPath>
+    <clipPath
+       id="clipPath100445">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100447" />
+    </clipPath>
+    <clipPath
+       id="clipPath100449">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100451" />
+    </clipPath>
+    <clipPath
+       id="clipPath100453">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100455" />
+    </clipPath>
+    <clipPath
+       id="clipPath100457">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100459" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_"
+         overflow="visible"
+         id="use83447" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use83427" />
+    </clipPath>
+    <clipPath
+       id="clipPath100468">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100470" />
+    </clipPath>
+    <clipPath
+       id="clipPath100472">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100474" />
+    </clipPath>
+    <clipPath
+       id="clipPath100476">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100478" />
+    </clipPath>
+    <clipPath
+       id="clipPath100480">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100482" />
+    </clipPath>
+    <clipPath
+       id="clipPath100484">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100486" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_"
+         overflow="visible"
+         id="use83412" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use83392" />
+    </clipPath>
+    <clipPath
+       id="clipPath100495">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100499">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100501" />
+    </clipPath>
+    <clipPath
+       id="clipPath100503">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100505" />
+    </clipPath>
+    <clipPath
+       id="clipPath100507">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100509" />
+    </clipPath>
+    <clipPath
+       id="clipPath100511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100513" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_"
+         overflow="visible"
+         id="use83377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use83357" />
+    </clipPath>
+    <clipPath
+       id="clipPath100522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100524" />
+    </clipPath>
+    <clipPath
+       id="clipPath100526">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100528" />
+    </clipPath>
+    <clipPath
+       id="clipPath100530">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100536" />
+    </clipPath>
+    <clipPath
+       id="clipPath100538">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100540" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_"
+         overflow="visible"
+         id="use83342" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use83322" />
+    </clipPath>
+    <clipPath
+       id="clipPath100549">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100551" />
+    </clipPath>
+    <clipPath
+       id="clipPath100553">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100555" />
+    </clipPath>
+    <clipPath
+       id="clipPath100557">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100559" />
+    </clipPath>
+    <clipPath
+       id="clipPath100561">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100563" />
+    </clipPath>
+    <clipPath
+       id="clipPath100565">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100567" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4687_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83310" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83312" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4686_">
+      <path
+         d="m -516.985,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83307"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4684_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83287" />
+    </clipPath>
+    <clipPath
+       id="clipPath100576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100578" />
+    </clipPath>
+    <clipPath
+       id="clipPath100580">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100582" />
+    </clipPath>
+    <clipPath
+       id="clipPath100584">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100586" />
+    </clipPath>
+    <clipPath
+       id="clipPath100588">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100590" />
+    </clipPath>
+    <clipPath
+       id="clipPath100592">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100594" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4682_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83275" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83277" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4681_">
+      <path
+         d="m -528.956,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83272"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4679_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83252" />
+    </clipPath>
+    <clipPath
+       id="clipPath100603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100605" />
+    </clipPath>
+    <clipPath
+       id="clipPath100607">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100609" />
+    </clipPath>
+    <clipPath
+       id="clipPath100611">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100613" />
+    </clipPath>
+    <clipPath
+       id="clipPath100615">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100617" />
+    </clipPath>
+    <clipPath
+       id="clipPath100619">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100621" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4677_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83240" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83242" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4676_">
+      <path
+         d="m -540.927,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83237"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4674_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83217" />
+    </clipPath>
+    <clipPath
+       id="clipPath100630">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100632" />
+    </clipPath>
+    <clipPath
+       id="clipPath100634">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100636" />
+    </clipPath>
+    <clipPath
+       id="clipPath100638">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100640" />
+    </clipPath>
+    <clipPath
+       id="clipPath100642">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100644" />
+    </clipPath>
+    <clipPath
+       id="clipPath100646">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100648" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4672_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83205" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83207" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4671_">
+      <path
+         d="m -552.899,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83202"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4669_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83182" />
+    </clipPath>
+    <clipPath
+       id="clipPath100657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100659" />
+    </clipPath>
+    <clipPath
+       id="clipPath100661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100663" />
+    </clipPath>
+    <clipPath
+       id="clipPath100665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100667" />
+    </clipPath>
+    <clipPath
+       id="clipPath100669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100671" />
+    </clipPath>
+    <clipPath
+       id="clipPath100673">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100675" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4667_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83170" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83172" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4666_">
+      <path
+         d="m -564.87,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83167"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4664_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83147" />
+    </clipPath>
+    <clipPath
+       id="clipPath100684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100686" />
+    </clipPath>
+    <clipPath
+       id="clipPath100688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100690" />
+    </clipPath>
+    <clipPath
+       id="clipPath100692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100694" />
+    </clipPath>
+    <clipPath
+       id="clipPath100696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100698" />
+    </clipPath>
+    <clipPath
+       id="clipPath100700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100702" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4662_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83135" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83137" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4661_">
+      <path
+         d="m -576.842,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83132"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4659_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83112" />
+    </clipPath>
+    <clipPath
+       id="clipPath100711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100713" />
+    </clipPath>
+    <clipPath
+       id="clipPath100715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100717" />
+    </clipPath>
+    <clipPath
+       id="clipPath100719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100721" />
+    </clipPath>
+    <clipPath
+       id="clipPath100723">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100725" />
+    </clipPath>
+    <clipPath
+       id="clipPath100727">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100729" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_"
+         overflow="visible"
+         id="use83097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use83077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100738">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100740" />
+    </clipPath>
+    <clipPath
+       id="clipPath100742">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100744" />
+    </clipPath>
+    <clipPath
+       id="clipPath100746">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100748" />
+    </clipPath>
+    <clipPath
+       id="clipPath100750">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100752" />
+    </clipPath>
+    <clipPath
+       id="clipPath100754">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100756" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_"
+         overflow="visible"
+         id="use83062" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use83042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100765">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100767" />
+    </clipPath>
+    <clipPath
+       id="clipPath100769">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100771" />
+    </clipPath>
+    <clipPath
+       id="clipPath100773">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100775" />
+    </clipPath>
+    <clipPath
+       id="clipPath100777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100779" />
+    </clipPath>
+    <clipPath
+       id="clipPath100781">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100783" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_"
+         overflow="visible"
+         id="use83027" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use83007" />
+    </clipPath>
+    <clipPath
+       id="clipPath100792">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100794" />
+    </clipPath>
+    <clipPath
+       id="clipPath100796">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100798" />
+    </clipPath>
+    <clipPath
+       id="clipPath100800">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100802" />
+    </clipPath>
+    <clipPath
+       id="clipPath100804">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100806" />
+    </clipPath>
+    <clipPath
+       id="clipPath100808">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_"
+         overflow="visible"
+         id="use82992" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use82972" />
+    </clipPath>
+    <clipPath
+       id="clipPath100819">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100821" />
+    </clipPath>
+    <clipPath
+       id="clipPath100823">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100825" />
+    </clipPath>
+    <clipPath
+       id="clipPath100827">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100829" />
+    </clipPath>
+    <clipPath
+       id="clipPath100831">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100833" />
+    </clipPath>
+    <clipPath
+       id="clipPath100835">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100837" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_"
+         overflow="visible"
+         id="use82957" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use82937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100846">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100848" />
+    </clipPath>
+    <clipPath
+       id="clipPath100850">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100852" />
+    </clipPath>
+    <clipPath
+       id="clipPath100854">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100856" />
+    </clipPath>
+    <clipPath
+       id="clipPath100858">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100860" />
+    </clipPath>
+    <clipPath
+       id="clipPath100862">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100864" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_"
+         overflow="visible"
+         id="use82922" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use82902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100873">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100875" />
+    </clipPath>
+    <clipPath
+       id="clipPath100877">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100879" />
+    </clipPath>
+    <clipPath
+       id="clipPath100881">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100883" />
+    </clipPath>
+    <clipPath
+       id="clipPath100885">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100887" />
+    </clipPath>
+    <clipPath
+       id="clipPath100889">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100891" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4625_"
+         overflow="visible"
+         id="use82887" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use82867" />
+    </clipPath>
+    <clipPath
+       id="clipPath100900">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100904">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100906" />
+    </clipPath>
+    <clipPath
+       id="clipPath100908">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100910" />
+    </clipPath>
+    <clipPath
+       id="clipPath100912">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100914" />
+    </clipPath>
+    <clipPath
+       id="clipPath100916">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100918" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4622_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82855" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82857" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4621_">
+      <path
+         d="m -516.985,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82852"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4619_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82832" />
+    </clipPath>
+    <clipPath
+       id="clipPath100927">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100929" />
+    </clipPath>
+    <clipPath
+       id="clipPath100931">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100933" />
+    </clipPath>
+    <clipPath
+       id="clipPath100935">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100941" />
+    </clipPath>
+    <clipPath
+       id="clipPath100943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100945" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4617_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82820" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82822" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4616_">
+      <path
+         d="m -528.956,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82817"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4614_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82797" />
+    </clipPath>
+    <clipPath
+       id="clipPath100954">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100956" />
+    </clipPath>
+    <clipPath
+       id="clipPath100958">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100960" />
+    </clipPath>
+    <clipPath
+       id="clipPath100962">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100964" />
+    </clipPath>
+    <clipPath
+       id="clipPath100966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100968" />
+    </clipPath>
+    <clipPath
+       id="clipPath100970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100972" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4612_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82785" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82787" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4611_">
+      <path
+         d="m -540.927,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82782"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4609_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82762" />
+    </clipPath>
+    <clipPath
+       id="clipPath100981">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100983" />
+    </clipPath>
+    <clipPath
+       id="clipPath100985">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100989">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100991" />
+    </clipPath>
+    <clipPath
+       id="clipPath100993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100995" />
+    </clipPath>
+    <clipPath
+       id="clipPath100997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100999" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4607_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82750" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82752" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4606_">
+      <path
+         d="m -552.899,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82747"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4604_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82727" />
+    </clipPath>
+    <clipPath
+       id="clipPath101008">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101012">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101014" />
+    </clipPath>
+    <clipPath
+       id="clipPath101016">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101018" />
+    </clipPath>
+    <clipPath
+       id="clipPath101020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101022" />
+    </clipPath>
+    <clipPath
+       id="clipPath101024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101026" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4602_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82715" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82717" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4601_">
+      <path
+         d="m -564.87,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82712"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4599_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82692" />
+    </clipPath>
+    <clipPath
+       id="clipPath101035">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101037" />
+    </clipPath>
+    <clipPath
+       id="clipPath101039">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101041" />
+    </clipPath>
+    <clipPath
+       id="clipPath101043">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101045" />
+    </clipPath>
+    <clipPath
+       id="clipPath101047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101049" />
+    </clipPath>
+    <clipPath
+       id="clipPath101051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101053" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4597_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82680" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82682" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4596_">
+      <path
+         d="m -576.842,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82677"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4594_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82657" />
+    </clipPath>
+    <clipPath
+       id="clipPath101062">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101064" />
+    </clipPath>
+    <clipPath
+       id="clipPath101066">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101068" />
+    </clipPath>
+    <clipPath
+       id="clipPath101070">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101072" />
+    </clipPath>
+    <clipPath
+       id="clipPath101074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101076" />
+    </clipPath>
+    <clipPath
+       id="clipPath101078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101080" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_"
+         overflow="visible"
+         id="use82642" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use82622" />
+    </clipPath>
+    <clipPath
+       id="clipPath101089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101091" />
+    </clipPath>
+    <clipPath
+       id="clipPath101093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101095" />
+    </clipPath>
+    <clipPath
+       id="clipPath101097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101099" />
+    </clipPath>
+    <clipPath
+       id="clipPath101101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101103" />
+    </clipPath>
+    <clipPath
+       id="clipPath101105">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101107" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_"
+         overflow="visible"
+         id="use82607" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use82587" />
+    </clipPath>
+    <clipPath
+       id="clipPath101116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101118" />
+    </clipPath>
+    <clipPath
+       id="clipPath101120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101122" />
+    </clipPath>
+    <clipPath
+       id="clipPath101124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101126" />
+    </clipPath>
+    <clipPath
+       id="clipPath101128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101130" />
+    </clipPath>
+    <clipPath
+       id="clipPath101132">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101134" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_"
+         overflow="visible"
+         id="use82572" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use82552" />
+    </clipPath>
+    <clipPath
+       id="clipPath101143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101145" />
+    </clipPath>
+    <clipPath
+       id="clipPath101147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101149" />
+    </clipPath>
+    <clipPath
+       id="clipPath101151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101153" />
+    </clipPath>
+    <clipPath
+       id="clipPath101155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101157" />
+    </clipPath>
+    <clipPath
+       id="clipPath101159">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101161" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_"
+         overflow="visible"
+         id="use82537" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use82517" />
+    </clipPath>
+    <clipPath
+       id="clipPath101170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101172" />
+    </clipPath>
+    <clipPath
+       id="clipPath101174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101176" />
+    </clipPath>
+    <clipPath
+       id="clipPath101178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101180" />
+    </clipPath>
+    <clipPath
+       id="clipPath101182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101184" />
+    </clipPath>
+    <clipPath
+       id="clipPath101186">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101188" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_"
+         overflow="visible"
+         id="use82502" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482" />
+    </clipPath>
+    <clipPath
+       id="clipPath101197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101199" />
+    </clipPath>
+    <clipPath
+       id="clipPath101201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101203" />
+    </clipPath>
+    <clipPath
+       id="clipPath101205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101207" />
+    </clipPath>
+    <clipPath
+       id="clipPath101209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101211" />
+    </clipPath>
+    <clipPath
+       id="clipPath101213">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101215" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_"
+         overflow="visible"
+         id="use82467" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447" />
+    </clipPath>
+    <clipPath
+       id="clipPath101224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101226" />
+    </clipPath>
+    <clipPath
+       id="clipPath101228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101230" />
+    </clipPath>
+    <clipPath
+       id="clipPath101232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101234" />
+    </clipPath>
+    <clipPath
+       id="clipPath101236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101238" />
+    </clipPath>
+    <clipPath
+       id="clipPath101240">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101242" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101251">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101253" />
+    </clipPath>
+    <clipPath
+       id="clipPath101255">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101257" />
+    </clipPath>
+    <clipPath
+       id="clipPath101259">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101261" />
+    </clipPath>
+    <clipPath
+       id="clipPath101263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101265" />
+    </clipPath>
+    <clipPath
+       id="clipPath101267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101269" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_"
+         overflow="visible"
+         id="use82397" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4554_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82373" />
+    </clipPath>
+    <clipPath
+       id="clipPath101278">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101280" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use82359" />
+    </clipPath>
+    <clipPath
+       id="clipPath101284">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use101286" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use82345" />
+    </clipPath>
+    <clipPath
+       id="clipPath101290">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use101292" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4546_"
+         overflow="visible"
+         id="use82316" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4543_"
+         overflow="visible"
+         id="use82287" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4542_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1387.9897,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82261" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82263" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82265" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82267" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82269" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82271" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82273" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4541_">
+      <polygon
+         points="-504.824,253.133 -526.362,258.293 -547.178,253.224 -547.185,267.353 -504.831,267.32 "
+         id="use82258" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_"
+         overflow="visible"
+         id="use82182" />
+    </clipPath>
+    <clipPath
+       id="clipPath101336">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use101338" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_"
+         overflow="visible"
+         id="use82197" />
+    </clipPath>
+    <clipPath
+       id="clipPath101342">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use101344" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_"
+         overflow="visible"
+         id="use82217" />
+    </clipPath>
+    <clipPath
+       id="clipPath101353">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use101355" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_"
+         overflow="visible"
+         id="use82232" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132" />
+    </clipPath>
+    <clipPath
+       id="clipPath101361">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101363" />
+    </clipPath>
+    <clipPath
+       id="clipPath101365">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101367" />
+    </clipPath>
+    <clipPath
+       id="clipPath101369">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101371" />
+    </clipPath>
+    <clipPath
+       id="clipPath101373">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101375" />
+    </clipPath>
+    <clipPath
+       id="clipPath101377">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101379" />
+    </clipPath>
+    <clipPath
+       id="clipPath101381">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101383" />
+    </clipPath>
+    <clipPath
+       id="clipPath101385">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101387" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_">
+      <polygon
+         points="-526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 "
+         id="use82113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083" />
+    </clipPath>
+    <clipPath
+       id="clipPath101398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101400" />
+    </clipPath>
+    <clipPath
+       id="clipPath101402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101404" />
+    </clipPath>
+    <clipPath
+       id="clipPath101406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101408" />
+    </clipPath>
+    <clipPath
+       id="clipPath101410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101416" />
+    </clipPath>
+    <clipPath
+       id="clipPath101418">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101420" />
+    </clipPath>
+    <clipPath
+       id="clipPath101422">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101424" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_">
+      <polygon
+         points="-571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 "
+         id="use82064" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101435">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101437" />
+    </clipPath>
+    <clipPath
+       id="clipPath101439">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101441" />
+    </clipPath>
+    <clipPath
+       id="clipPath101443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101445" />
+    </clipPath>
+    <clipPath
+       id="clipPath101447">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101449" />
+    </clipPath>
+    <clipPath
+       id="clipPath101451">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101453" />
+    </clipPath>
+    <clipPath
+       id="clipPath101455">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101457" />
+    </clipPath>
+    <clipPath
+       id="clipPath101459">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101461" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4509_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_">
+      <polygon
+         points="-638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 "
+         id="use81991" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534"
+       xlink:href="#SVGID_4509_"
+       inkscape:collect="always" />
+    <defs
+       id="defs84496">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4842_" />
+    </defs>
+    <clipPath
+       id="clipPath103391">
+      <use
+         id="use103393"
+         overflow="visible"
+         xlink:href="#SVGID_4842_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113428">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113430" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-9">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-2" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-4" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-9" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-0">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113442">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113444" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-0">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-9">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113450">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113452" />
+    </clipPath>
+    <clipPath
+       id="clipPath113454">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113456" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-6">
+      <polygon
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 "
+         id="use84771-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113465">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113467" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-6">
+      <polygon
+         points="-588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 "
+         id="use84732-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113476">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113478" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-1">
+      <polygon
+         points="-588.288,208.524 -574.322,208.524 -574.322,203.458 -588.288,203.458 -590.081,206 "
+         id="use84619-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113489" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-3">
+      <polygon
+         points="-654.922,208.524 -640.956,208.524 -640.956,203.458 -654.922,203.458 -656.715,206 "
+         id="use84582-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113498">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113500" />
+    </clipPath>
+    <clipPath
+       id="clipPath113502">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113504" />
+    </clipPath>
+    <clipPath
+       id="clipPath113506">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113508" />
+    </clipPath>
+    <clipPath
+       id="clipPath113510">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113512" />
+    </clipPath>
+    <clipPath
+       id="clipPath113514">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113516" />
+    </clipPath>
+    <clipPath
+       id="clipPath113518">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113520" />
+    </clipPath>
+    <clipPath
+       id="clipPath113522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113524" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113528">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113530" />
+    </clipPath>
+    <clipPath
+       id="clipPath113532">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113534" />
+    </clipPath>
+    <clipPath
+       id="clipPath113536">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113538" />
+    </clipPath>
+    <clipPath
+       id="clipPath113540">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113542" />
+    </clipPath>
+    <clipPath
+       id="clipPath113544">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113546" />
+    </clipPath>
+    <clipPath
+       id="clipPath113548">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113550" />
+    </clipPath>
+    <clipPath
+       id="clipPath113552">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113554" />
+    </clipPath>
+    <clipPath
+       id="clipPath113556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113558" />
+    </clipPath>
+    <clipPath
+       id="clipPath113560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113562" />
+    </clipPath>
+    <clipPath
+       id="clipPath113564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113566" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-8" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-7" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-5">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113580" />
+    </clipPath>
+    <clipPath
+       id="clipPath113582">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113584" />
+    </clipPath>
+    <clipPath
+       id="clipPath113586">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113588" />
+    </clipPath>
+    <clipPath
+       id="clipPath113590">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113592" />
+    </clipPath>
+    <clipPath
+       id="clipPath113594">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113596" />
+    </clipPath>
+    <clipPath
+       id="clipPath113598">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113600" />
+    </clipPath>
+    <clipPath
+       id="clipPath113602">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113604" />
+    </clipPath>
+    <clipPath
+       id="clipPath113606">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113608" />
+    </clipPath>
+    <clipPath
+       id="clipPath113610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113612" />
+    </clipPath>
+    <clipPath
+       id="clipPath113614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113616" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-9" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-0" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-4" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-0" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-0" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-1" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-3">
+      <polygon
+         points="-636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 "
+         id="use84326-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-6" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-1" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-6" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-9">
+      <polygon
+         points="-569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 "
+         id="use84295-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-8">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113649">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113651" />
+    </clipPath>
+    <clipPath
+       id="clipPath113653">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113655" />
+    </clipPath>
+    <clipPath
+       id="clipPath113657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113659" />
+    </clipPath>
+    <clipPath
+       id="clipPath113661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113663" />
+    </clipPath>
+    <clipPath
+       id="clipPath113665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113667" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-3">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113676">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113678" />
+    </clipPath>
+    <clipPath
+       id="clipPath113680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113682" />
+    </clipPath>
+    <clipPath
+       id="clipPath113684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113686" />
+    </clipPath>
+    <clipPath
+       id="clipPath113688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113690" />
+    </clipPath>
+    <clipPath
+       id="clipPath113692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113694" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-4">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113703">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113705" />
+    </clipPath>
+    <clipPath
+       id="clipPath113707">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113709" />
+    </clipPath>
+    <clipPath
+       id="clipPath113711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113713" />
+    </clipPath>
+    <clipPath
+       id="clipPath113715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113717" />
+    </clipPath>
+    <clipPath
+       id="clipPath113719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113721" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-5">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113732" />
+    </clipPath>
+    <clipPath
+       id="clipPath113734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113736" />
+    </clipPath>
+    <clipPath
+       id="clipPath113738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113740" />
+    </clipPath>
+    <clipPath
+       id="clipPath113742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113744" />
+    </clipPath>
+    <clipPath
+       id="clipPath113746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-5">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113757">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113759" />
+    </clipPath>
+    <clipPath
+       id="clipPath113761">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113763" />
+    </clipPath>
+    <clipPath
+       id="clipPath113765">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113767" />
+    </clipPath>
+    <clipPath
+       id="clipPath113769">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113771" />
+    </clipPath>
+    <clipPath
+       id="clipPath113773">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113775" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-9">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113786" />
+    </clipPath>
+    <clipPath
+       id="clipPath113788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113790" />
+    </clipPath>
+    <clipPath
+       id="clipPath113792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113794" />
+    </clipPath>
+    <clipPath
+       id="clipPath113796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113798" />
+    </clipPath>
+    <clipPath
+       id="clipPath113800">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113802" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-5">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113811">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113813" />
+    </clipPath>
+    <clipPath
+       id="clipPath113815">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113817" />
+    </clipPath>
+    <clipPath
+       id="clipPath113819">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113821" />
+    </clipPath>
+    <clipPath
+       id="clipPath113823">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113825" />
+    </clipPath>
+    <clipPath
+       id="clipPath113827">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113829" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-0">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-5">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113845" />
+    </clipPath>
+    <clipPath
+       id="clipPath113847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113849" />
+    </clipPath>
+    <clipPath
+       id="clipPath113851">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113853" />
+    </clipPath>
+    <clipPath
+       id="clipPath113855">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113857" />
+    </clipPath>
+    <clipPath
+       id="clipPath113859">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113861" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-1">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113872" />
+    </clipPath>
+    <clipPath
+       id="clipPath113874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113876" />
+    </clipPath>
+    <clipPath
+       id="clipPath113878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113880" />
+    </clipPath>
+    <clipPath
+       id="clipPath113882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113884" />
+    </clipPath>
+    <clipPath
+       id="clipPath113886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113888" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-6">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113899" />
+    </clipPath>
+    <clipPath
+       id="clipPath113901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113903" />
+    </clipPath>
+    <clipPath
+       id="clipPath113905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113907" />
+    </clipPath>
+    <clipPath
+       id="clipPath113909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113911" />
+    </clipPath>
+    <clipPath
+       id="clipPath113913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113915" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-5">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113926" />
+    </clipPath>
+    <clipPath
+       id="clipPath113928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113930" />
+    </clipPath>
+    <clipPath
+       id="clipPath113932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113934" />
+    </clipPath>
+    <clipPath
+       id="clipPath113936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113938" />
+    </clipPath>
+    <clipPath
+       id="clipPath113940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113942" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-3">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath113951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113953" />
+    </clipPath>
+    <clipPath
+       id="clipPath113955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113957" />
+    </clipPath>
+    <clipPath
+       id="clipPath113959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113961" />
+    </clipPath>
+    <clipPath
+       id="clipPath113963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113965" />
+    </clipPath>
+    <clipPath
+       id="clipPath113967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113969" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-2">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113980" />
+    </clipPath>
+    <clipPath
+       id="clipPath113982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113984" />
+    </clipPath>
+    <clipPath
+       id="clipPath113986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113988" />
+    </clipPath>
+    <clipPath
+       id="clipPath113990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113992" />
+    </clipPath>
+    <clipPath
+       id="clipPath113994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113996" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-2">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114010">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114012" />
+    </clipPath>
+    <clipPath
+       id="clipPath114014">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114016" />
+    </clipPath>
+    <clipPath
+       id="clipPath114018">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114020" />
+    </clipPath>
+    <clipPath
+       id="clipPath114022">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114024" />
+    </clipPath>
+    <clipPath
+       id="clipPath114026">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114028" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-4">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath114037">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114039" />
+    </clipPath>
+    <clipPath
+       id="clipPath114041">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114043" />
+    </clipPath>
+    <clipPath
+       id="clipPath114045">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114047" />
+    </clipPath>
+    <clipPath
+       id="clipPath114049">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114051" />
+    </clipPath>
+    <clipPath
+       id="clipPath114053">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114055" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-1">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath114064">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114066" />
+    </clipPath>
+    <clipPath
+       id="clipPath114068">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114070" />
+    </clipPath>
+    <clipPath
+       id="clipPath114072">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114074" />
+    </clipPath>
+    <clipPath
+       id="clipPath114076">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114078" />
+    </clipPath>
+    <clipPath
+       id="clipPath114080">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114082" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-8">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114091">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114093" />
+    </clipPath>
+    <clipPath
+       id="clipPath114095">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114097" />
+    </clipPath>
+    <clipPath
+       id="clipPath114099">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114101" />
+    </clipPath>
+    <clipPath
+       id="clipPath114103">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114105" />
+    </clipPath>
+    <clipPath
+       id="clipPath114107">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114109" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-6">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath114118">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114120" />
+    </clipPath>
+    <clipPath
+       id="clipPath114122">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114124" />
+    </clipPath>
+    <clipPath
+       id="clipPath114126">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114128" />
+    </clipPath>
+    <clipPath
+       id="clipPath114130">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114132" />
+    </clipPath>
+    <clipPath
+       id="clipPath114134">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114136" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-1">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114145">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114147" />
+    </clipPath>
+    <clipPath
+       id="clipPath114149">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114151" />
+    </clipPath>
+    <clipPath
+       id="clipPath114153">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114155" />
+    </clipPath>
+    <clipPath
+       id="clipPath114157">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114159" />
+    </clipPath>
+    <clipPath
+       id="clipPath114161">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114163" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-6">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-6">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114177">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114179" />
+    </clipPath>
+    <clipPath
+       id="clipPath114181">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114183" />
+    </clipPath>
+    <clipPath
+       id="clipPath114185">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114187" />
+    </clipPath>
+    <clipPath
+       id="clipPath114189">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114191" />
+    </clipPath>
+    <clipPath
+       id="clipPath114193">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114195" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-0">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114204">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114206" />
+    </clipPath>
+    <clipPath
+       id="clipPath114208">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114210" />
+    </clipPath>
+    <clipPath
+       id="clipPath114212">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114214" />
+    </clipPath>
+    <clipPath
+       id="clipPath114216">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114218" />
+    </clipPath>
+    <clipPath
+       id="clipPath114220">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114222" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-6">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114231">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114233" />
+    </clipPath>
+    <clipPath
+       id="clipPath114235">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114237" />
+    </clipPath>
+    <clipPath
+       id="clipPath114239">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114241" />
+    </clipPath>
+    <clipPath
+       id="clipPath114243">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114245" />
+    </clipPath>
+    <clipPath
+       id="clipPath114247">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114249" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-2">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114258">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114260" />
+    </clipPath>
+    <clipPath
+       id="clipPath114262">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114264" />
+    </clipPath>
+    <clipPath
+       id="clipPath114266">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114268" />
+    </clipPath>
+    <clipPath
+       id="clipPath114270">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114272" />
+    </clipPath>
+    <clipPath
+       id="clipPath114274">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114276" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-1">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114285">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114287" />
+    </clipPath>
+    <clipPath
+       id="clipPath114289">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114291" />
+    </clipPath>
+    <clipPath
+       id="clipPath114293">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114295" />
+    </clipPath>
+    <clipPath
+       id="clipPath114297">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114299" />
+    </clipPath>
+    <clipPath
+       id="clipPath114301">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114303" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-1">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath114312">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114314" />
+    </clipPath>
+    <clipPath
+       id="clipPath114316">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114318" />
+    </clipPath>
+    <clipPath
+       id="clipPath114320">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114322" />
+    </clipPath>
+    <clipPath
+       id="clipPath114324">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114326" />
+    </clipPath>
+    <clipPath
+       id="clipPath114328">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114330" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-1">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath114339">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114341" />
+    </clipPath>
+    <clipPath
+       id="clipPath114343">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114345" />
+    </clipPath>
+    <clipPath
+       id="clipPath114347">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114349" />
+    </clipPath>
+    <clipPath
+       id="clipPath114351">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114353" />
+    </clipPath>
+    <clipPath
+       id="clipPath114355">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114357" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-5">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114366">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114368" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114374" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-4" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-6" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-0" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-7" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-0" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-5" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-4">
+      <polygon
+         points="-680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 "
+         id="use82316-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-3" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-2" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-5" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-7" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-2" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-2">
+      <polygon
+         points="-613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 "
+         id="use82287-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-8">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-6">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath114407">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use114409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-7">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114413">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114415" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-4">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-7" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-8">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114424">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use114426" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-9">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114432">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114434" />
+    </clipPath>
+    <clipPath
+       id="clipPath114436">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114438" />
+    </clipPath>
+    <clipPath
+       id="clipPath114440">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114442" />
+    </clipPath>
+    <clipPath
+       id="clipPath114444">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114446" />
+    </clipPath>
+    <clipPath
+       id="clipPath114448">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114450" />
+    </clipPath>
+    <clipPath
+       id="clipPath114452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114454" />
+    </clipPath>
+    <clipPath
+       id="clipPath114456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114458" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-5" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-0" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-1">
+      <polygon
+         points="-594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 "
+         id="use82064-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114469">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114471" />
+    </clipPath>
+    <clipPath
+       id="clipPath114473">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114475" />
+    </clipPath>
+    <clipPath
+       id="clipPath114477">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114479" />
+    </clipPath>
+    <clipPath
+       id="clipPath114481">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114483" />
+    </clipPath>
+    <clipPath
+       id="clipPath114485">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114487" />
+    </clipPath>
+    <clipPath
+       id="clipPath114489">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114491" />
+    </clipPath>
+    <clipPath
+       id="clipPath114493">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114495" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-0"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-4" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-3" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-0">
+      <polygon
+         points="-661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 "
+         id="use81991-1" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient115446"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4657_-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-2-7"
+         overflow="visible"
+         id="use83097-8-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath118798">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use118800" />
+    </clipPath>
+    <defs
+       id="defs83093-1-3">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4655_-2-7"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102313-6-0">
+      <use
+         id="use102315-6-2"
+         overflow="visible"
+         xlink:href="#SVGID_4655_-2-7"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102317-9-1">
+      <stop
+         id="stop102319-7-7"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102321-3-3"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4873_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-79" />
+    </clipPath>
+    <clipPath
+       id="clipPath119610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119612" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-5">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-4" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-2" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-99" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-79" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-3">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119624">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119626" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-1">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-6">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119632">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119634" />
+    </clipPath>
+    <clipPath
+       id="clipPath119636">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119638" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-61" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-2">
+      <polygon
+         points="-640.956,273.567 -654.922,273.567 -656.715,276.108 -654.922,278.632 -640.956,278.632 "
+         id="use84771-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-44" />
+    </clipPath>
+    <clipPath
+       id="clipPath119647">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119649" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-0">
+      <polygon
+         points="-574.322,273.567 -588.288,273.567 -590.081,276.108 -588.288,278.632 -574.322,278.632 "
+         id="use84732-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119658">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119660" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-19">
+      <polygon
+         points="-574.322,203.458 -588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 "
+         id="use84619-38" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119671" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-4">
+      <polygon
+         points="-640.956,203.458 -654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 "
+         id="use84582-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath119680">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119682" />
+    </clipPath>
+    <clipPath
+       id="clipPath119684">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119686" />
+    </clipPath>
+    <clipPath
+       id="clipPath119688">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119690" />
+    </clipPath>
+    <clipPath
+       id="clipPath119692">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119694" />
+    </clipPath>
+    <clipPath
+       id="clipPath119696">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119698" />
+    </clipPath>
+    <clipPath
+       id="clipPath119700">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119702" />
+    </clipPath>
+    <clipPath
+       id="clipPath119704">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119706" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119710">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119712" />
+    </clipPath>
+    <clipPath
+       id="clipPath119714">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119716" />
+    </clipPath>
+    <clipPath
+       id="clipPath119718">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119720" />
+    </clipPath>
+    <clipPath
+       id="clipPath119722">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119724" />
+    </clipPath>
+    <clipPath
+       id="clipPath119726">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119728" />
+    </clipPath>
+    <clipPath
+       id="clipPath119730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119732" />
+    </clipPath>
+    <clipPath
+       id="clipPath119734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119736" />
+    </clipPath>
+    <clipPath
+       id="clipPath119738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119740" />
+    </clipPath>
+    <clipPath
+       id="clipPath119742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119744" />
+    </clipPath>
+    <clipPath
+       id="clipPath119746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-5" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-1" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-3">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119760">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119762" />
+    </clipPath>
+    <clipPath
+       id="clipPath119764">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119766" />
+    </clipPath>
+    <clipPath
+       id="clipPath119768">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119770" />
+    </clipPath>
+    <clipPath
+       id="clipPath119772">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119774" />
+    </clipPath>
+    <clipPath
+       id="clipPath119776">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119778" />
+    </clipPath>
+    <clipPath
+       id="clipPath119780">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119782" />
+    </clipPath>
+    <clipPath
+       id="clipPath119784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119786" />
+    </clipPath>
+    <clipPath
+       id="clipPath119788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119790" />
+    </clipPath>
+    <clipPath
+       id="clipPath119792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119794" />
+    </clipPath>
+    <clipPath
+       id="clipPath119796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119798" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-0" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-42" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-15" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-5">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-5" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-2" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-3" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-8">
+      <polygon
+         points="-682.207,231.929 -682.208,251.221 -659.342,256.788 -636.356,251.283 -636.358,231.934 "
+         id="use84326-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-1" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-7" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-5" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-6">
+      <polygon
+         points="-615.653,231.929 -615.653,251.221 -592.788,256.788 -569.802,251.283 -569.804,231.934 "
+         id="use84295-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-2">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119831">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119833" />
+    </clipPath>
+    <clipPath
+       id="clipPath119835">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119837" />
+    </clipPath>
+    <clipPath
+       id="clipPath119839">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119841" />
+    </clipPath>
+    <clipPath
+       id="clipPath119843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119845" />
+    </clipPath>
+    <clipPath
+       id="clipPath119847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119849" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-1">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-45"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath119858">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119860" />
+    </clipPath>
+    <clipPath
+       id="clipPath119862">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119864" />
+    </clipPath>
+    <clipPath
+       id="clipPath119866">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119868" />
+    </clipPath>
+    <clipPath
+       id="clipPath119870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119872" />
+    </clipPath>
+    <clipPath
+       id="clipPath119874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119876" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-7">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119885">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119887" />
+    </clipPath>
+    <clipPath
+       id="clipPath119889">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119891" />
+    </clipPath>
+    <clipPath
+       id="clipPath119893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119895" />
+    </clipPath>
+    <clipPath
+       id="clipPath119897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119899" />
+    </clipPath>
+    <clipPath
+       id="clipPath119901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119903" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-2">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-70"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119912">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119914" />
+    </clipPath>
+    <clipPath
+       id="clipPath119916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119918" />
+    </clipPath>
+    <clipPath
+       id="clipPath119920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119922" />
+    </clipPath>
+    <clipPath
+       id="clipPath119924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119926" />
+    </clipPath>
+    <clipPath
+       id="clipPath119928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119930" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-91" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-6">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119941" />
+    </clipPath>
+    <clipPath
+       id="clipPath119943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119945" />
+    </clipPath>
+    <clipPath
+       id="clipPath119947">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119949" />
+    </clipPath>
+    <clipPath
+       id="clipPath119951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119953" />
+    </clipPath>
+    <clipPath
+       id="clipPath119955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119957" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-3">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119968" />
+    </clipPath>
+    <clipPath
+       id="clipPath119970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119972" />
+    </clipPath>
+    <clipPath
+       id="clipPath119974">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119976" />
+    </clipPath>
+    <clipPath
+       id="clipPath119978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119980" />
+    </clipPath>
+    <clipPath
+       id="clipPath119982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119984" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-76" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-1">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath119993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119995" />
+    </clipPath>
+    <clipPath
+       id="clipPath119997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119999" />
+    </clipPath>
+    <clipPath
+       id="clipPath120001">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120003" />
+    </clipPath>
+    <clipPath
+       id="clipPath120005">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120007" />
+    </clipPath>
+    <clipPath
+       id="clipPath120009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-6">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83742-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120022" />
+    </clipPath>
+    <clipPath
+       id="clipPath120024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120026" />
+    </clipPath>
+    <clipPath
+       id="clipPath120028">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120030" />
+    </clipPath>
+    <clipPath
+       id="clipPath120032">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120034" />
+    </clipPath>
+    <clipPath
+       id="clipPath120036">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120038" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-85" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-8">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120049" />
+    </clipPath>
+    <clipPath
+       id="clipPath120051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120053" />
+    </clipPath>
+    <clipPath
+       id="clipPath120055">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120057" />
+    </clipPath>
+    <clipPath
+       id="clipPath120059">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120061" />
+    </clipPath>
+    <clipPath
+       id="clipPath120063">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120065" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-0">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120076" />
+    </clipPath>
+    <clipPath
+       id="clipPath120078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120080" />
+    </clipPath>
+    <clipPath
+       id="clipPath120082">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120084" />
+    </clipPath>
+    <clipPath
+       id="clipPath120086">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120088" />
+    </clipPath>
+    <clipPath
+       id="clipPath120090">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120092" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-68">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120101">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120103" />
+    </clipPath>
+    <clipPath
+       id="clipPath120105">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120107" />
+    </clipPath>
+    <clipPath
+       id="clipPath120109">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120111" />
+    </clipPath>
+    <clipPath
+       id="clipPath120113">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120115" />
+    </clipPath>
+    <clipPath
+       id="clipPath120117">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120119" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-77" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-2">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120128">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120130" />
+    </clipPath>
+    <clipPath
+       id="clipPath120132">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120134" />
+    </clipPath>
+    <clipPath
+       id="clipPath120136">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120138" />
+    </clipPath>
+    <clipPath
+       id="clipPath120140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120142" />
+    </clipPath>
+    <clipPath
+       id="clipPath120144">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120146" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-5">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120155">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120157" />
+    </clipPath>
+    <clipPath
+       id="clipPath120159">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120161" />
+    </clipPath>
+    <clipPath
+       id="clipPath120163">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120165" />
+    </clipPath>
+    <clipPath
+       id="clipPath120167">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120169" />
+    </clipPath>
+    <clipPath
+       id="clipPath120171">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120173" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-62"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-20">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120182">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120184" />
+    </clipPath>
+    <clipPath
+       id="clipPath120186">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120188" />
+    </clipPath>
+    <clipPath
+       id="clipPath120190">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120192" />
+    </clipPath>
+    <clipPath
+       id="clipPath120194">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120196" />
+    </clipPath>
+    <clipPath
+       id="clipPath120198">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120200" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-7">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83322-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120209">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120211" />
+    </clipPath>
+    <clipPath
+       id="clipPath120213">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120215" />
+    </clipPath>
+    <clipPath
+       id="clipPath120217">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120219" />
+    </clipPath>
+    <clipPath
+       id="clipPath120221">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120223" />
+    </clipPath>
+    <clipPath
+       id="clipPath120225">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120227" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-7">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120236">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120238" />
+    </clipPath>
+    <clipPath
+       id="clipPath120240">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120242" />
+    </clipPath>
+    <clipPath
+       id="clipPath120244">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120246" />
+    </clipPath>
+    <clipPath
+       id="clipPath120248">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120250" />
+    </clipPath>
+    <clipPath
+       id="clipPath120252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120254" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-42" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-1">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-49" />
+    </clipPath>
+    <clipPath
+       id="clipPath120263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120265" />
+    </clipPath>
+    <clipPath
+       id="clipPath120267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120269" />
+    </clipPath>
+    <clipPath
+       id="clipPath120271">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120273" />
+    </clipPath>
+    <clipPath
+       id="clipPath120275">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120277" />
+    </clipPath>
+    <clipPath
+       id="clipPath120279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120281" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-35" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-8">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120290">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120292" />
+    </clipPath>
+    <clipPath
+       id="clipPath120294">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120296" />
+    </clipPath>
+    <clipPath
+       id="clipPath120298">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120300" />
+    </clipPath>
+    <clipPath
+       id="clipPath120302">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120304" />
+    </clipPath>
+    <clipPath
+       id="clipPath120306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120308" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-9">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-97" />
+    </clipPath>
+    <clipPath
+       id="clipPath120317">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120319" />
+    </clipPath>
+    <clipPath
+       id="clipPath120321">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120323" />
+    </clipPath>
+    <clipPath
+       id="clipPath120325">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120327" />
+    </clipPath>
+    <clipPath
+       id="clipPath120329">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120331" />
+    </clipPath>
+    <clipPath
+       id="clipPath120333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120335" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-2">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120344">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120346" />
+    </clipPath>
+    <clipPath
+       id="clipPath120348">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120350" />
+    </clipPath>
+    <clipPath
+       id="clipPath120352">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120354" />
+    </clipPath>
+    <clipPath
+       id="clipPath120356">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120358" />
+    </clipPath>
+    <clipPath
+       id="clipPath120360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120362" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-64" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-9">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-36"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120371">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120373" />
+    </clipPath>
+    <clipPath
+       id="clipPath120375">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120377" />
+    </clipPath>
+    <clipPath
+       id="clipPath120379">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120381" />
+    </clipPath>
+    <clipPath
+       id="clipPath120383">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120385" />
+    </clipPath>
+    <clipPath
+       id="clipPath120387">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120389" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-2">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82867-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120400" />
+    </clipPath>
+    <clipPath
+       id="clipPath120402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120404" />
+    </clipPath>
+    <clipPath
+       id="clipPath120406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120408" />
+    </clipPath>
+    <clipPath
+       id="clipPath120410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120412" />
+    </clipPath>
+    <clipPath
+       id="clipPath120414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120416" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-1">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120425">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120427" />
+    </clipPath>
+    <clipPath
+       id="clipPath120429">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120431" />
+    </clipPath>
+    <clipPath
+       id="clipPath120433">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120435" />
+    </clipPath>
+    <clipPath
+       id="clipPath120437">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120439" />
+    </clipPath>
+    <clipPath
+       id="clipPath120441">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120443" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-7">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120454" />
+    </clipPath>
+    <clipPath
+       id="clipPath120456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120458" />
+    </clipPath>
+    <clipPath
+       id="clipPath120460">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120462" />
+    </clipPath>
+    <clipPath
+       id="clipPath120464">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120466" />
+    </clipPath>
+    <clipPath
+       id="clipPath120468">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120470" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-03" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-7">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-65">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120479">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120481" />
+    </clipPath>
+    <clipPath
+       id="clipPath120483">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120485" />
+    </clipPath>
+    <clipPath
+       id="clipPath120487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120489" />
+    </clipPath>
+    <clipPath
+       id="clipPath120491">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120493" />
+    </clipPath>
+    <clipPath
+       id="clipPath120495">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120497" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-5">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120506">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120508" />
+    </clipPath>
+    <clipPath
+       id="clipPath120510">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120512" />
+    </clipPath>
+    <clipPath
+       id="clipPath120514">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120516" />
+    </clipPath>
+    <clipPath
+       id="clipPath120518">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120520" />
+    </clipPath>
+    <clipPath
+       id="clipPath120522">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120524" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-3">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120533">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120535" />
+    </clipPath>
+    <clipPath
+       id="clipPath120537">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120539" />
+    </clipPath>
+    <clipPath
+       id="clipPath120541">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120543" />
+    </clipPath>
+    <clipPath
+       id="clipPath120545">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120547" />
+    </clipPath>
+    <clipPath
+       id="clipPath120549">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120551" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-8">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120562" />
+    </clipPath>
+    <clipPath
+       id="clipPath120564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120566" />
+    </clipPath>
+    <clipPath
+       id="clipPath120568">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120570" />
+    </clipPath>
+    <clipPath
+       id="clipPath120572">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120574" />
+    </clipPath>
+    <clipPath
+       id="clipPath120576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120578" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-29" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-11">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120587">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120589" />
+    </clipPath>
+    <clipPath
+       id="clipPath120591">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120593" />
+    </clipPath>
+    <clipPath
+       id="clipPath120595">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120597" />
+    </clipPath>
+    <clipPath
+       id="clipPath120599">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120601" />
+    </clipPath>
+    <clipPath
+       id="clipPath120603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120605" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-7">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120616" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120620">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120622" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-6" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-70" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-00" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-8" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-67" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-3">
+      <polygon
+         points="-637.642,253.133 -659.18,258.293 -679.997,253.224 -680.004,267.353 -637.65,267.32 "
+         id="use82316-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-93"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-1" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-1" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-0" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-3" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-1">
+      <polygon
+         points="-571.547,253.133 -593.085,258.293 -613.901,253.224 -613.908,267.353 -571.554,267.32 "
+         id="use82287-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-9">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-42" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-4">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120655">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use120657" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-5">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120663" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-89">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120672">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use120674" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-35">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120682" />
+    </clipPath>
+    <clipPath
+       id="clipPath120684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120686" />
+    </clipPath>
+    <clipPath
+       id="clipPath120688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120690" />
+    </clipPath>
+    <clipPath
+       id="clipPath120692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120694" />
+    </clipPath>
+    <clipPath
+       id="clipPath120696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120698" />
+    </clipPath>
+    <clipPath
+       id="clipPath120700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120702" />
+    </clipPath>
+    <clipPath
+       id="clipPath120704">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120706" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-2" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-11">
+      <polygon
+         points="-571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 "
+         id="use82064-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-38">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120717">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120719" />
+    </clipPath>
+    <clipPath
+       id="clipPath120721">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120723" />
+    </clipPath>
+    <clipPath
+       id="clipPath120725">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120727" />
+    </clipPath>
+    <clipPath
+       id="clipPath120729">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120731" />
+    </clipPath>
+    <clipPath
+       id="clipPath120733">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120735" />
+    </clipPath>
+    <clipPath
+       id="clipPath120737">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120739" />
+    </clipPath>
+    <clipPath
+       id="clipPath120741">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120743" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-4"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-08" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-7" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-3">
+      <polygon
+         points="-638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 "
+         id="use81991-0" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient121727"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126204"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientUnits="userSpaceOnUse" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126210"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-7"
+       id="linearGradient126210-7"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-7"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126228"
+       xlink:href="#linearGradient126200-7"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126210-9"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126435"
+       xlink:href="#linearGradient126200-2"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126487"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientTransform="translate(190.5,52.1205)" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126520"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient126520-9"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientTransform="translate(190.5,52.1205)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126539"
+       xlink:href="#linearGradient126200-2-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-2"
+       id="linearGradient126520-4"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientTransform="translate(190.5,52.1205)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126539-0"
+       xlink:href="#linearGradient126200-2-2"
+       inkscape:collect="always" />
+    <clipPath
+       id="SVGID_4843_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2-2" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_"
+         overflow="visible"
+         id="use84656-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use84644-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath126768">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use126770" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478-8" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480-2" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482-3" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use82132-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath126782">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126784" />
+    </clipPath>
+    <clipPath
+       id="clipPath126786">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126788" />
+    </clipPath>
+    <clipPath
+       id="clipPath126790">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126792" />
+    </clipPath>
+    <clipPath
+       id="clipPath126794">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126796" />
+    </clipPath>
+    <clipPath
+       id="clipPath126798">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126800" />
+    </clipPath>
+    <clipPath
+       id="clipPath126802">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126804" />
+    </clipPath>
+    <clipPath
+       id="clipPath126806">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126808" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_"
+         overflow="visible"
+         id="use82113-2" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904"
+       xlink:href="#SVGID_4521_-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4856_-4-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-2">
+      <polygon
+         points="-523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 "
+         id="use84656-1-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath127140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127142" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath127146">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127148" />
+    </clipPath>
+    <clipPath
+       id="clipPath127150">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127152" />
+    </clipPath>
+    <clipPath
+       id="clipPath127154">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127156" />
+    </clipPath>
+    <clipPath
+       id="clipPath127158">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127160" />
+    </clipPath>
+    <clipPath
+       id="clipPath127162">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127164" />
+    </clipPath>
+    <clipPath
+       id="clipPath127166">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127168" />
+    </clipPath>
+    <clipPath
+       id="clipPath127170">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127172" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-6"
+       xlink:href="#SVGID_4521_-3-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-3" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-4" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-3">
+      <polygon
+         points="-504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 "
+         id="use82113-2-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_-1"
+         overflow="visible"
+         id="use84656-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4"
+         overflow="visible"
+         id="use84644-1-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9068">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4"
+         overflow="visible"
+         id="use9070" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use82132-5-25" />
+    </clipPath>
+    <clipPath
+       id="clipPath9074">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9076" />
+    </clipPath>
+    <clipPath
+       id="clipPath9078">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9080" />
+    </clipPath>
+    <clipPath
+       id="clipPath9082">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9084" />
+    </clipPath>
+    <clipPath
+       id="clipPath9086">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9088" />
+    </clipPath>
+    <clipPath
+       id="clipPath9090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9092" />
+    </clipPath>
+    <clipPath
+       id="clipPath9094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9096" />
+    </clipPath>
+    <clipPath
+       id="clipPath9098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9100" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61"
+       xlink:href="#SVGID_4521_-3-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_-5"
+         overflow="visible"
+         id="use82113-2-5" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9379"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9382"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(164.46193,28.435994)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <clipPath
+       id="SVGID_4550_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use82345-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath9448">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use9450" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-03" />
+    </clipPath>
+    <clipPath
+       id="clipPath9526">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9528" />
+    </clipPath>
+    <clipPath
+       id="clipPath9530">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9532" />
+    </clipPath>
+    <clipPath
+       id="clipPath9534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9536" />
+    </clipPath>
+    <clipPath
+       id="clipPath9538">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9540" />
+    </clipPath>
+    <clipPath
+       id="clipPath9542">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9544" />
+    </clipPath>
+    <defs
+       id="defs82478">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4568_" />
+    </defs>
+    <clipPath
+       id="clipPath101841">
+      <use
+         id="use101843"
+         overflow="visible"
+         xlink:href="#SVGID_4568_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4563_" />
+    </defs>
+    <clipPath
+       id="clipPath101814">
+      <use
+         id="use101816"
+         overflow="visible"
+         xlink:href="#SVGID_4563_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath26747">
+      <g
+         id="g26751">
+        <rect
+           style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+           id="rect26749"
+           width="439.00146"
+           height="696.97461"
+           x="122.01721"
+           y="152.74527" />
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28686">
+      <g
+         id="g28694"
+         transform="matrix(0.2962963,0,0,0.2962963,-648.90222,109.95785)">
+        <g
+           id="g28692">
+          <g
+             id="g28690">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28688"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28716">
+      <g
+         id="g28724"
+         transform="matrix(0.44444444,0,0,0.44444444,-797.73062,91.646227)">
+        <g
+           id="g28722">
+          <g
+             id="g28720">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28718"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28726">
+      <g
+         id="g28734"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28732">
+          <g
+             id="g28730">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28728"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28746">
+      <g
+         id="g28754"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28752">
+          <g
+             id="g28750">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28748"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28766">
+      <g
+         id="g28774"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28772">
+          <g
+             id="g28770">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28768"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28776">
+      <g
+         id="g28784"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28782">
+          <g
+             id="g28780">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28778"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28786">
+      <g
+         id="g28794"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28792">
+          <g
+             id="g28790">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28788"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28796">
+      <g
+         id="g28804">
+        <g
+           id="g28802">
+          <g
+             id="g28800">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28798"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28806">
+      <g
+         id="g28814">
+        <g
+           id="g28812">
+          <g
+             id="g28810">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28808"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28836">
+      <g
+         id="g28844">
+        <g
+           id="g28842">
+          <g
+             id="g28840">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28838"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28846">
+      <g
+         id="g28854">
+        <g
+           id="g28852">
+          <g
+             id="g28850">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28848"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28866">
+      <g
+         id="g28874">
+        <g
+           id="g28872">
+          <g
+             id="g28870">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28868"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28876">
+      <g
+         id="g28884">
+        <g
+           id="g28882">
+          <g
+             id="g28880">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28878"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28896">
+      <g
+         id="g28904">
+        <g
+           id="g28902">
+          <g
+             id="g28900">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28898"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28916">
+      <g
+         id="g28924"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g28922">
+          <g
+             id="g28920">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28918"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28926">
+      <g
+         id="g28934">
+        <g
+           id="g28932">
+          <g
+             id="g28930">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28928"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28936">
+      <g
+         id="g28944">
+        <g
+           id="g28942">
+          <g
+             id="g28940">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28938"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28946">
+      <g
+         id="g28954">
+        <g
+           id="g28952">
+          <g
+             id="g28950">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28948"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28956">
+      <g
+         id="g28964">
+        <g
+           id="g28962">
+          <g
+             id="g28960">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28958"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28966">
+      <g
+         id="g28974">
+        <g
+           id="g28972">
+          <g
+             id="g28970">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28968"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath28976">
+      <g
+         id="g28984">
+        <g
+           id="g28982">
+          <g
+             id="g28980">
+            <rect
+               style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+               id="rect28978"
+               width="439.00146"
+               height="696.97461"
+               x="122.01721"
+               y="152.74527" />
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84716">
+      <rect
+         id="SVGID_4862_"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </defs>
+    <clipPath
+       id="clipPath103539">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use103541" />
+    </clipPath>
+    <defs
+       id="defs84767">
+      <polygon
+         id="SVGID_4869_"
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 " />
+    </defs>
+    <clipPath
+       id="clipPath103577">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4869_"
+         overflow="visible"
+         id="use103579" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient103581"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop103583" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop103585" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29010">
+      <g
+         id="g29020"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g29018"
+           transform="translate(1e-5)">
+          <g
+             id="g29016">
+            <g
+               id="g29014">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29012"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84728">
+      <polygon
+         id="SVGID_4864_"
+         points="-588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 " />
+    </defs>
+    <clipPath
+       id="clipPath103548">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4864_"
+         overflow="visible"
+         id="use103550" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient103552"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop103554" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop103556" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29040">
+      <g
+         id="g29050"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g29048"
+           transform="translate(1e-5)">
+          <g
+             id="g29046">
+            <g
+               id="g29044">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29042"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82283">
+      <polygon
+         id="SVGID_4543_"
+         points="-613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 " />
+    </defs>
+    <clipPath
+       id="clipPath101684">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4543_"
+         overflow="visible"
+         id="use101686" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient101688"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop101690" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop101692" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop101694" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop101696" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop101698" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop101700" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop101702" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop101704" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29076">
+      <g
+         id="g29086"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g29084"
+           transform="translate(1e-5)">
+          <g
+             id="g29082">
+            <g
+               id="g29080">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29078"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29143">
+      <g
+         id="g29153"
+         transform="matrix(0.2962963,0,0,0.2962963,-602.80741,109.97993)">
+        <g
+           id="g29151">
+          <g
+             id="g29149">
+            <g
+               id="g29147">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29145"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29179">
+      <g
+         id="g29189">
+        <g
+           id="g29187">
+          <g
+             id="g29185">
+            <g
+               id="g29183">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29181"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84291">
+      <polygon
+         id="SVGID_4823_"
+         points="-569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 " />
+    </defs>
+    <clipPath
+       id="clipPath103235">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4823_"
+         overflow="visible"
+         id="use103237" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient103239"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop103241" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop103243" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop103245" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop103247" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop103249" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29212">
+      <g
+         id="g29222"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g29220"
+           transform="translate(1e-5)">
+          <g
+             id="g29218">
+            <g
+               id="g29216">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29214"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84755">
+      <rect
+         id="SVGID_4867_"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </defs>
+    <clipPath
+       id="clipPath103568">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_"
+         overflow="visible"
+         id="use103570" />
+    </clipPath>
+    <defs
+       id="defs84652">
+      <polygon
+         id="SVGID_4854_"
+         points="-521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 " />
+    </defs>
+    <clipPath
+       id="clipPath126888">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_"
+         overflow="visible"
+         id="use126890" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient126892"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop126894" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop126896" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29248">
+      <g
+         id="g29258"
+         transform="matrix(0.2962963,0,0,0.2962963,-648.90222,109.95785)">
+        <g
+           id="g29256"
+           transform="translate(7.4999998e-6,1.375e-5)">
+          <g
+             id="g29254">
+            <g
+               id="g29252">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29250"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29955">
+      <g
+         id="g29965"
+         transform="matrix(0.44444444,0,0,0.44444444,-846.09071,-7.425108)">
+        <g
+           id="g29963">
+          <g
+             id="g29961">
+            <g
+               id="g29959">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29957"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29967">
+      <g
+         id="g29977"
+         transform="matrix(0.44444444,0,0,0.44444444,-846.0464,4.3971142)">
+        <g
+           id="g29975">
+          <g
+             id="g29973">
+            <g
+               id="g29971">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29969"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29979">
+      <g
+         id="g29989"
+         transform="matrix(0.44444444,0,0,0.44444444,-846.00196,15.95267)">
+        <g
+           id="g29987">
+          <g
+             id="g29985">
+            <g
+               id="g29983">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29981"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath29991">
+      <g
+         id="g30001"
+         transform="matrix(0.44444444,0,0,0.44444444,-762.07813,27.363123)">
+        <g
+           id="g29999">
+          <g
+             id="g29997">
+            <g
+               id="g29995">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect29993"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30003">
+      <g
+         id="g30013"
+         transform="matrix(0.44444444,0,0,0.44444444,-846.00196,27.53667)">
+        <g
+           id="g30011">
+          <g
+             id="g30009">
+            <g
+               id="g30007">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30005"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30015">
+      <g
+         id="g30025"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30023">
+          <g
+             id="g30021">
+            <g
+               id="g30019">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30017"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30027">
+      <g
+         id="g30037"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30035">
+          <g
+             id="g30033">
+            <g
+               id="g30031">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30029"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30039">
+      <g
+         id="g30049"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30047">
+          <g
+             id="g30045">
+            <g
+               id="g30043">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30041"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30051">
+      <g
+         id="g30061"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30059">
+          <g
+             id="g30057">
+            <g
+               id="g30055">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30053"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30063">
+      <g
+         id="g30073"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30071">
+          <g
+             id="g30069">
+            <g
+               id="g30067">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30065"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30075">
+      <g
+         id="g30085"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30083">
+          <g
+             id="g30081">
+            <g
+               id="g30079">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30077"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30411">
+      <g
+         id="g30421"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30419">
+          <g
+             id="g30417">
+            <g
+               id="g30415">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30413"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30423">
+      <g
+         id="g30433"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30431">
+          <g
+             id="g30429">
+            <g
+               id="g30427">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30425"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30435">
+      <g
+         id="g30445"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30443">
+          <g
+             id="g30441">
+            <g
+               id="g30439">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30437"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30447">
+      <g
+         id="g30457"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30455">
+          <g
+             id="g30453">
+            <g
+               id="g30451">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30449"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30459">
+      <g
+         id="g30469"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30467">
+          <g
+             id="g30465">
+            <g
+               id="g30463">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30461"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30471">
+      <g
+         id="g30481"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30479">
+          <g
+             id="g30477">
+            <g
+               id="g30475">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30473"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30495">
+      <g
+         id="g30505"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30503">
+          <g
+             id="g30501">
+            <g
+               id="g30499">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30497"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30507">
+      <g
+         id="g30517"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30515">
+          <g
+             id="g30513">
+            <g
+               id="g30511">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30509"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30519">
+      <g
+         id="g30529"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30527">
+          <g
+             id="g30525">
+            <g
+               id="g30523">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30521"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30531">
+      <g
+         id="g30541"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30539">
+          <g
+             id="g30537">
+            <g
+               id="g30535">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30533"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30543">
+      <g
+         id="g30553"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30551">
+          <g
+             id="g30549">
+            <g
+               id="g30547">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30545"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30567">
+      <g
+         id="g30577">
+        <g
+           id="g30575">
+          <g
+             id="g30573">
+            <g
+               id="g30571">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30569"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30579">
+      <g
+         id="g30589">
+        <g
+           id="g30587">
+          <g
+             id="g30585">
+            <g
+               id="g30583">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30581"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30591">
+      <g
+         id="g30601">
+        <g
+           id="g30599">
+          <g
+             id="g30597">
+            <g
+               id="g30595">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30593"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30603">
+      <g
+         id="g30613">
+        <g
+           id="g30611">
+          <g
+             id="g30609">
+            <g
+               id="g30607">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30605"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30615">
+      <g
+         id="g30625">
+        <g
+           id="g30623">
+          <g
+             id="g30621">
+            <g
+               id="g30619">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30617"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82128">
+      <rect
+         id="SVGID_4522_"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </defs>
+    <clipPath
+       id="clipPath126838">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126840" />
+    </clipPath>
+    <defs
+       id="defs84322">
+      <polygon
+         id="SVGID_4826_"
+         points="-636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 " />
+    </defs>
+    <clipPath
+       id="clipPath103260">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4826_"
+         overflow="visible"
+         id="use103262" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient103264"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop103266" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop103268" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop103270" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop103272" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop103274" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30660">
+      <g
+         id="g30670"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30668"
+           transform="translate(1e-5)">
+          <g
+             id="g30666">
+            <g
+               id="g30664">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30662"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82109">
+      <polygon
+         id="SVGID_4519_"
+         points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 " />
+    </defs>
+    <clipPath
+       id="clipPath126820">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_"
+         overflow="visible"
+         id="use126822" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient126824"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop126826" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop126828" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop126830" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop126832" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30692">
+      <g
+         id="g30702"
+         transform="matrix(0.2962963,0,0,0.2962963,-648.90222,109.95785)">
+        <g
+           id="g30700"
+           transform="translate(7.4999998e-6,1.375e-5)">
+          <g
+             id="g30698">
+            <g
+               id="g30696">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30694"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84640">
+      <rect
+         id="SVGID_4852_"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </defs>
+    <clipPath
+       id="clipPath126879">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use126881" />
+    </clipPath>
+    <defs
+       id="defs82312">
+      <polygon
+         id="SVGID_4546_"
+         points="-680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 " />
+    </defs>
+    <clipPath
+       id="clipPath101711">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4546_"
+         overflow="visible"
+         id="use101713" />
+    </clipPath>
+    <linearGradient
+       id="linearGradient101715"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop101717" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop101719" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop101721" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop101723" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop101725" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop101727" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop101729" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop101731" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30734">
+      <g
+         id="g30744"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30742"
+           transform="translate(1e-5)">
+          <g
+             id="g30740">
+            <g
+               id="g30738">
+              <rect
+                 style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                 id="rect30736"
+                 width="439.00146"
+                 height="696.97461"
+                 x="122.01721"
+                 y="152.74527" />
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82128-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4522_-8" />
+    </defs>
+    <clipPath
+       id="clipPath126838-9">
+      <use
+         id="use126840-87"
+         overflow="visible"
+         xlink:href="#SVGID_4522_-8"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83793">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4755_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102853">
+      <use
+         id="use102855"
+         overflow="visible"
+         xlink:href="#SVGID_4755_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)"
+       y2="-148.4299"
+       x2="256.3541"
+       y1="-148.4299"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102857">
+      <stop
+         id="stop102859"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102861"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30778">
+      <g
+         id="g30790"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30788"
+           transform="translate(1e-5)">
+          <g
+             id="g30786">
+            <g
+               id="g30784">
+              <g
+                 id="g30782">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30780"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83933">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4775_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102961">
+      <use
+         id="use102963"
+         overflow="visible"
+         xlink:href="#SVGID_4775_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)"
+       y2="-148.42999"
+       x2="256.3541"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102965">
+      <stop
+         id="stop102967"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102969"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30812">
+      <g
+         id="g30824"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30822"
+           transform="translate(1e-5)">
+          <g
+             id="g30820">
+            <g
+               id="g30818">
+              <g
+                 id="g30816">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30814"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83758">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4750_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102826">
+      <use
+         id="use102828"
+         overflow="visible"
+         xlink:href="#SVGID_4750_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)"
+       y2="-148.4301"
+       x2="256.3541"
+       y1="-148.4301"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102830">
+      <stop
+         id="stop102832"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102834"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30846">
+      <g
+         id="g30858"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30856"
+           transform="translate(1e-5)">
+          <g
+             id="g30854">
+            <g
+               id="g30852">
+              <g
+                 id="g30850">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30848"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83478">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4710_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102610">
+      <use
+         id="use102612"
+         overflow="visible"
+         xlink:href="#SVGID_4710_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)"
+       y2="-148.42999"
+       x2="256.3544"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102614">
+      <stop
+         id="stop102616"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102618"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30880">
+      <g
+         id="g30892"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30890"
+           transform="translate(1e-5)">
+          <g
+             id="g30888">
+            <g
+               id="g30886">
+              <g
+                 id="g30884">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30882"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83058">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4650_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102286">
+      <use
+         id="use102288"
+         overflow="visible"
+         xlink:href="#SVGID_4650_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102290">
+      <stop
+         id="stop102292"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102294"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30914">
+      <g
+         id="g30926"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30924"
+           transform="translate(1e-5)">
+          <g
+             id="g30922">
+            <g
+               id="g30920">
+              <g
+                 id="g30918">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30916"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82393">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4555_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101773">
+      <use
+         id="use101775"
+         overflow="visible"
+         xlink:href="#SVGID_4555_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       y2="-148.46021"
+       x2="256.3761"
+       y1="-148.46021"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101777">
+      <stop
+         id="stop101779"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101781"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30948">
+      <g
+         id="g30960"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30958"
+           transform="translate(1e-5)">
+          <g
+             id="g30956">
+            <g
+               id="g30954">
+              <g
+                 id="g30952">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30950"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82918">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4630_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102178">
+      <use
+         id="use102180"
+         overflow="visible"
+         xlink:href="#SVGID_4630_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102182">
+      <stop
+         id="stop102184"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102186"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath30982">
+      <g
+         id="g30994"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g30992"
+           transform="translate(1e-5)">
+          <g
+             id="g30990">
+            <g
+               id="g30988">
+              <g
+                 id="g30986">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect30984"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84640-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4852_-4" />
+    </defs>
+    <clipPath
+       id="clipPath126879-0">
+      <use
+         id="use126881-7"
+         overflow="visible"
+         xlink:href="#SVGID_4852_-4"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4748_" />
+    </defs>
+    <clipPath
+       id="clipPath102813">
+      <use
+         id="use102815"
+         overflow="visible"
+         xlink:href="#SVGID_4748_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83948">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4778_" />
+    </defs>
+    <clipPath
+       id="clipPath102975">
+      <use
+         id="use102977"
+         overflow="visible"
+         xlink:href="#SVGID_4778_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82533">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="SVGID_4575_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101881">
+      <use
+         id="use101883"
+         overflow="visible"
+         xlink:href="#SVGID_4575_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)"
+       y2="-148.4601"
+       x2="256.3761"
+       y1="-148.4601"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101885">
+      <stop
+         id="stop101887"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101889"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31042">
+      <g
+         id="g31054"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31052"
+           transform="translate(1e-5)">
+          <g
+             id="g31050">
+            <g
+               id="g31048">
+              <g
+                 id="g31046">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31044"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4768_" />
+    </defs>
+    <clipPath
+       id="clipPath102921">
+      <use
+         id="use102923"
+         overflow="visible"
+         xlink:href="#SVGID_4768_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82568">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4580_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101908">
+      <use
+         id="use101910"
+         overflow="visible"
+         xlink:href="#SVGID_4580_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)"
+       y2="-148.4601"
+       x2="256.3761"
+       y1="-148.4601"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101912">
+      <stop
+         id="stop101914"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101916"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31086">
+      <g
+         id="g31098"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31096"
+           transform="translate(1e-5)">
+          <g
+             id="g31094">
+            <g
+               id="g31092">
+              <g
+                 id="g31090">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31088"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82109-6">
+      <polygon
+         points="-528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 "
+         id="SVGID_4519_-5" />
+    </defs>
+    <clipPath
+       id="clipPath126820-6">
+      <use
+         id="use126822-8"
+         overflow="visible"
+         xlink:href="#SVGID_4519_-5"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126824-4">
+      <stop
+         id="stop126826-2"
+         style="stop-color:#37424B"
+         offset="0" />
+      <stop
+         id="stop126828-3"
+         style="stop-color:#37424B"
+         offset="0.12" />
+      <stop
+         id="stop126830-7"
+         style="stop-color:#5E6A71"
+         offset="0.88" />
+      <stop
+         id="stop126832-6"
+         style="stop-color:#5E6A71"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31122">
+      <g
+         id="g31134"
+         transform="matrix(0.2962963,0,0,0.2962963,-602.80741,109.97993)">
+        <g
+           id="g31132">
+          <g
+             id="g31130">
+            <g
+               id="g31128">
+              <g
+                 id="g31126">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31124"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82355">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4551_" />
+    </defs>
+    <clipPath
+       id="clipPath101748">
+      <use
+         id="use101750"
+         overflow="visible"
+         xlink:href="#SVGID_4551_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83458">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4708_" />
+    </defs>
+    <clipPath
+       id="clipPath102597">
+      <use
+         id="use102599"
+         overflow="visible"
+         xlink:href="#SVGID_4708_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83073">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4653_" />
+    </defs>
+    <clipPath
+       id="clipPath102300">
+      <use
+         id="use102302"
+         overflow="visible"
+         xlink:href="#SVGID_4653_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82953">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4635_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102205">
+      <use
+         id="use102207"
+         overflow="visible"
+         xlink:href="#SVGID_4635_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102209">
+      <stop
+         id="stop102211"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102213"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31182">
+      <g
+         id="g31194"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31192"
+           transform="translate(1e-5)">
+          <g
+             id="g31190">
+            <g
+               id="g31188">
+              <g
+                 id="g31186">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31184"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82603">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4585_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101935">
+      <use
+         id="use101937"
+         overflow="visible"
+         xlink:href="#SVGID_4585_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)"
+       y2="-148.46021"
+       x2="256.3761"
+       y1="-148.46021"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101939">
+      <stop
+         id="stop101941"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101943"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31216">
+      <g
+         id="g31228"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31226"
+           transform="translate(1e-5)">
+          <g
+             id="g31224">
+            <g
+               id="g31222">
+              <g
+                 id="g31220">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31218"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82341">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4549_" />
+    </defs>
+    <clipPath
+       id="clipPath101738">
+      <use
+         id="use101740"
+         overflow="visible"
+         xlink:href="#SVGID_4549_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82513">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4573_" />
+    </defs>
+    <clipPath
+       id="clipPath101868">
+      <use
+         id="use101870"
+         overflow="visible"
+         xlink:href="#SVGID_4573_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83423">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4703_" />
+    </defs>
+    <clipPath
+       id="clipPath102570">
+      <use
+         id="use102572"
+         overflow="visible"
+         xlink:href="#SVGID_4703_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4763_" />
+    </defs>
+    <clipPath
+       id="clipPath102894">
+      <use
+         id="use102896"
+         overflow="visible"
+         xlink:href="#SVGID_4763_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83388">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4698_" />
+    </defs>
+    <clipPath
+       id="clipPath102543">
+      <use
+         id="use102545"
+         overflow="visible"
+         xlink:href="#SVGID_4698_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82898">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4628_" />
+    </defs>
+    <clipPath
+       id="clipPath102165">
+      <use
+         id="use102167"
+         overflow="visible"
+         xlink:href="#SVGID_4628_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82583">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4583_" />
+    </defs>
+    <clipPath
+       id="clipPath101922">
+      <use
+         id="use101924"
+         overflow="visible"
+         xlink:href="#SVGID_4583_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83773">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4753_" />
+    </defs>
+    <clipPath
+       id="clipPath102840">
+      <use
+         id="use102842"
+         overflow="visible"
+         xlink:href="#SVGID_4753_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs84652-4">
+      <polygon
+         points="-507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 "
+         id="SVGID_4854_-1" />
+    </defs>
+    <clipPath
+       id="clipPath126888-1">
+      <use
+         id="use126890-4"
+         overflow="visible"
+         xlink:href="#SVGID_4854_-1"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)"
+       y2="249.94031"
+       x2="-33.205101"
+       y1="249.94031"
+       x1="-34.205299"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126892-7">
+      <stop
+         id="stop126894-5"
+         style="stop-color:#E6E8E7"
+         offset="0" />
+      <stop
+         id="stop126896-6"
+         style="stop-color:#FFFFFF"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31334">
+      <g
+         id="g31346"
+         transform="matrix(0.2962963,0,0,0.2962963,-602.80741,109.97993)">
+        <g
+           id="g31344">
+          <g
+             id="g31342">
+            <g
+               id="g31340">
+              <g
+                 id="g31338">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31336"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82498">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4570_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101854">
+      <use
+         id="use101856"
+         overflow="visible"
+         xlink:href="#SVGID_4570_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)"
+       y2="-148.46001"
+       x2="256.3761"
+       y1="-148.46001"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101858">
+      <stop
+         id="stop101860"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101862"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31368">
+      <g
+         id="g31380"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31378"
+           transform="translate(1e-5)">
+          <g
+             id="g31376">
+            <g
+               id="g31374">
+              <g
+                 id="g31372">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31370"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83808">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4758_" />
+    </defs>
+    <clipPath
+       id="clipPath102867">
+      <use
+         id="use102869"
+         overflow="visible"
+         xlink:href="#SVGID_4758_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83093">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4655_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102313">
+      <use
+         id="use102315"
+         overflow="visible"
+         xlink:href="#SVGID_4655_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102317">
+      <stop
+         id="stop102319"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102321"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31412">
+      <g
+         id="g31424"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31422"
+           transform="translate(1e-5)">
+          <g
+             id="g31420">
+            <g
+               id="g31418">
+              <g
+                 id="g31416">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31414"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83863">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4765_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102907">
+      <use
+         id="use102909"
+         overflow="visible"
+         xlink:href="#SVGID_4765_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)"
+       y2="-148.4299"
+       x2="256.3541"
+       y1="-148.4299"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102911">
+      <stop
+         id="stop102913"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102915"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31446">
+      <g
+         id="g31458"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31456"
+           transform="translate(1e-5)">
+          <g
+             id="g31454">
+            <g
+               id="g31452">
+              <g
+                 id="g31450">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31448"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82463">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4565_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101827">
+      <use
+         id="use101829"
+         overflow="visible"
+         xlink:href="#SVGID_4565_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)"
+       y2="-148.4601"
+       x2="256.3761"
+       y1="-148.4601"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101831">
+      <stop
+         id="stop101833"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101835"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31480">
+      <g
+         id="g31492"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31490"
+           transform="translate(1e-5)">
+          <g
+             id="g31488">
+            <g
+               id="g31486">
+              <g
+                 id="g31484">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31482"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83023">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="SVGID_4645_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102259">
+      <use
+         id="use102261"
+         overflow="visible"
+         xlink:href="#SVGID_4645_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102263">
+      <stop
+         id="stop102265"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102267"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31514">
+      <g
+         id="g31526"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31524"
+           transform="translate(1e-5)">
+          <g
+             id="g31522">
+            <g
+               id="g31520">
+              <g
+                 id="g31518">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31516"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82618">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4588_" />
+    </defs>
+    <clipPath
+       id="clipPath101949">
+      <use
+         id="use101951"
+         overflow="visible"
+         xlink:href="#SVGID_4588_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83443">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4705_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102583">
+      <use
+         id="use102585"
+         overflow="visible"
+         xlink:href="#SVGID_4705_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)"
+       y2="-148.4299"
+       x2="256.3544"
+       y1="-148.4299"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102587">
+      <stop
+         id="stop102589"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102591"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31558">
+      <g
+         id="g31570"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31568"
+           transform="translate(1e-5)">
+          <g
+             id="g31566">
+            <g
+               id="g31564">
+              <g
+                 id="g31562">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31560"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83318">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4688_" />
+    </defs>
+    <clipPath
+       id="clipPath102489">
+      <use
+         id="use102491"
+         overflow="visible"
+         xlink:href="#SVGID_4688_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82968">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4638_" />
+    </defs>
+    <clipPath
+       id="clipPath102219">
+      <use
+         id="use102221"
+         overflow="visible"
+         xlink:href="#SVGID_4638_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83983">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4783_" />
+    </defs>
+    <clipPath
+       id="clipPath103002">
+      <use
+         id="use103004"
+         overflow="visible"
+         xlink:href="#SVGID_4783_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82933">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4633_" />
+    </defs>
+    <clipPath
+       id="clipPath102192">
+      <use
+         id="use102194"
+         overflow="visible"
+         xlink:href="#SVGID_4633_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83373">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4695_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102529">
+      <use
+         id="use102531"
+         overflow="visible"
+         xlink:href="#SVGID_4695_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)"
+       y2="-148.4299"
+       x2="256.35419"
+       y1="-148.4299"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102533">
+      <stop
+         id="stop102535"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102537"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31632">
+      <g
+         id="g31644"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31642"
+           transform="translate(1e-5)">
+          <g
+             id="g31640">
+            <g
+               id="g31638">
+              <g
+                 id="g31636">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31634"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83353">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4693_" />
+    </defs>
+    <clipPath
+       id="clipPath102516">
+      <use
+         id="use102518"
+         overflow="visible"
+         xlink:href="#SVGID_4693_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82160">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4524_" />
+    </defs>
+    <clipPath
+       id="clipPath101582">
+      <use
+         id="use101584"
+         overflow="visible"
+         xlink:href="#SVGID_4524_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83828">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4760_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102880">
+      <use
+         id="use102882"
+         overflow="visible"
+         xlink:href="#SVGID_4760_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)"
+       y2="-148.42999"
+       x2="256.3541"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102884">
+      <stop
+         id="stop102886"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102888"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31722">
+      <g
+         id="g31734"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31732"
+           transform="translate(1e-5)">
+          <g
+             id="g31730">
+            <g
+               id="g31728">
+              <g
+                 id="g31726">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31724"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs84003">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4785_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath103015">
+      <use
+         id="use103017"
+         overflow="visible"
+         xlink:href="#SVGID_4785_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)"
+       y2="-148.42999"
+       x2="256.3541"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient103019">
+      <stop
+         id="stop103021"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop103023"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31756">
+      <g
+         id="g31768"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31766"
+           transform="translate(1e-5)">
+          <g
+             id="g31764">
+            <g
+               id="g31762">
+              <g
+                 id="g31760">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31758"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83548">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4720_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102664">
+      <use
+         id="use102666"
+         overflow="visible"
+         xlink:href="#SVGID_4720_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)"
+       y2="-148.42999"
+       x2="256.3544"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102668">
+      <stop
+         id="stop102670"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102672"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31790">
+      <g
+         id="g31802"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31800"
+           transform="translate(1e-5)">
+          <g
+             id="g31798">
+            <g
+               id="g31796">
+              <g
+                 id="g31794">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31792"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82638">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="SVGID_4590_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath101962">
+      <use
+         id="use101964"
+         overflow="visible"
+         xlink:href="#SVGID_4590_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)"
+       y2="-148.4601"
+       x2="256.3761"
+       y1="-148.4601"
+       x1="255.3761"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101966">
+      <stop
+         id="stop101968"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop101970"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31824">
+      <g
+         id="g31836"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31834"
+           transform="translate(1e-5)">
+          <g
+             id="g31832">
+            <g
+               id="g31830">
+              <g
+                 id="g31828">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31826"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82883">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4625_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102151">
+      <use
+         id="use102153"
+         overflow="visible"
+         xlink:href="#SVGID_4625_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)"
+       y2="-148.46021"
+       x2="256.37631"
+       y1="-148.46021"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102155">
+      <stop
+         id="stop102157"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102159"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31858">
+      <g
+         id="g31870"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31868"
+           transform="translate(1e-5)">
+          <g
+             id="g31866">
+            <g
+               id="g31864">
+              <g
+                 id="g31862">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31860"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83038">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4648_" />
+    </defs>
+    <clipPath
+       id="clipPath102273">
+      <use
+         id="use102275"
+         overflow="visible"
+         xlink:href="#SVGID_4648_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82988">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4640_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102232">
+      <use
+         id="use102234"
+         overflow="visible"
+         xlink:href="#SVGID_4640_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)"
+       y2="-148.46001"
+       x2="256.37631"
+       y1="-148.46001"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102236">
+      <stop
+         id="stop102238"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102240"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31902">
+      <g
+         id="g31914"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31912"
+           transform="translate(1e-5)">
+          <g
+             id="g31910">
+            <g
+               id="g31908">
+              <g
+                 id="g31906">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31904"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83408">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4700_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102556">
+      <use
+         id="use102558"
+         overflow="visible"
+         xlink:href="#SVGID_4700_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)"
+       y2="-148.42999"
+       x2="256.3544"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102560">
+      <stop
+         id="stop102562"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102564"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath31936">
+      <g
+         id="g31948"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g31946"
+           transform="translate(1e-5)">
+          <g
+             id="g31944">
+            <g
+               id="g31942">
+              <g
+                 id="g31940">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect31938"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82863">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4623_" />
+    </defs>
+    <clipPath
+       id="clipPath102138">
+      <use
+         id="use102140"
+         overflow="visible"
+         xlink:href="#SVGID_4623_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82548">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4578_" />
+    </defs>
+    <clipPath
+       id="clipPath101895">
+      <use
+         id="use101897"
+         overflow="visible"
+         xlink:href="#SVGID_4578_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs84794">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4872_" />
+    </defs>
+    <clipPath
+       id="clipPath103597">
+      <use
+         id="use103599"
+         overflow="visible"
+         xlink:href="#SVGID_4872_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83968">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4780_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102988">
+      <use
+         id="use102990"
+         overflow="visible"
+         xlink:href="#SVGID_4780_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)"
+       y2="-148.4301"
+       x2="256.3541"
+       y1="-148.4301"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102992">
+      <stop
+         id="stop102994"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102996"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath32027">
+      <g
+         id="g32039"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g32037"
+           transform="translate(1e-5)">
+          <g
+             id="g32035">
+            <g
+               id="g32033">
+              <g
+                 id="g32031">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect32029"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83003">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4643_" />
+    </defs>
+    <clipPath
+       id="clipPath102246">
+      <use
+         id="use102248"
+         overflow="visible"
+         xlink:href="#SVGID_4643_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4773_" />
+    </defs>
+    <clipPath
+       id="clipPath102948">
+      <use
+         id="use102950"
+         overflow="visible"
+         xlink:href="#SVGID_4773_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83528">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4718_" />
+    </defs>
+    <clipPath
+       id="clipPath102651">
+      <use
+         id="use102653"
+         overflow="visible"
+         xlink:href="#SVGID_4718_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83898">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="SVGID_4770_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102934">
+      <use
+         id="use102936"
+         overflow="visible"
+         xlink:href="#SVGID_4770_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)"
+       y2="-148.42999"
+       x2="256.3541"
+       y1="-148.42999"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102938">
+      <stop
+         id="stop102940"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102942"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath32091">
+      <g
+         id="g32103"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g32101"
+           transform="translate(1e-5)">
+          <g
+             id="g32099">
+            <g
+               id="g32097">
+              <g
+                 id="g32095">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect32093"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83493">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4713_" />
+    </defs>
+    <clipPath
+       id="clipPath102624">
+      <use
+         id="use102626"
+         overflow="visible"
+         xlink:href="#SVGID_4713_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs83513">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4715_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102637">
+      <use
+         id="use102639"
+         overflow="visible"
+         xlink:href="#SVGID_4715_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)"
+       y2="-148.4301"
+       x2="256.3544"
+       y1="-148.4301"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102641">
+      <stop
+         id="stop102643"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102645"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath32135">
+      <g
+         id="g32147"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g32145"
+           transform="translate(1e-5)">
+          <g
+             id="g32143">
+            <g
+               id="g32141">
+              <g
+                 id="g32139">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect32137"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs83338">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="SVGID_4690_"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102502">
+      <use
+         id="use102504"
+         overflow="visible"
+         xlink:href="#SVGID_4690_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)"
+       y2="-148.4301"
+       x2="256.3544"
+       y1="-148.4301"
+       x1="255.3541"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102506">
+      <stop
+         id="stop102508"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102510"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath32169">
+      <g
+         id="g32181"
+         transform="matrix(0.44444444,0,0,0.44444444,-774.10444,27.578499)">
+        <g
+           id="g32179"
+           transform="translate(1e-5)">
+          <g
+             id="g32177">
+            <g
+               id="g32175">
+              <g
+                 id="g32173">
+                <rect
+                   style="opacity:0.61;fill:#000000;stroke-width:1.00094"
+                   id="rect32171"
+                   width="439.00146"
+                   height="696.97461"
+                   x="122.01721"
+                   y="152.74527" />
+              </g>
+            </g>
+          </g>
+        </g>
+      </g>
+    </clipPath>
+    <defs
+       id="defs82341-9">
+      <rect
+         id="SVGID_4549_-8"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </defs>
+    <clipPath
+       id="clipPath101738-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use101740-3" />
+    </clipPath>
+    <defs
+       id="defs82205">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="SVGID_4533_" />
+    </defs>
+    <clipPath
+       id="clipPath101620">
+      <use
+         id="use101622"
+         overflow="visible"
+         xlink:href="#SVGID_4533_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82170">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="SVGID_4526_" />
+    </defs>
+    <clipPath
+       id="clipPath101590">
+      <use
+         id="use101592"
+         overflow="visible"
+         xlink:href="#SVGID_4526_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs84837">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="SVGID_4879_" />
+    </defs>
+    <clipPath
+       id="clipPath103634">
+      <use
+         id="use103636"
+         overflow="visible"
+         xlink:href="#SVGID_4879_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs84806">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="SVGID_4874_" />
+    </defs>
+    <clipPath
+       id="clipPath103606">
+      <use
+         id="use103608"
+         overflow="visible"
+         xlink:href="#SVGID_4874_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82213">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="SVGID_4535_" />
+    </defs>
+    <clipPath
+       id="clipPath101627">
+      <use
+         id="use101629"
+         overflow="visible"
+         xlink:href="#SVGID_4535_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)"
+       y2="60.902401"
+       x2="127.8011"
+       y1="60.902401"
+       x1="126.8011"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101631">
+      <stop
+         id="stop101633"
+         style="stop-color:#E6E8E7"
+         offset="0" />
+      <stop
+         id="stop101635"
+         style="stop-color:#FFFFFF"
+         offset="1" />
+    </linearGradient>
+    <defs
+       id="defs82193">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="SVGID_4531_" />
+    </defs>
+    <clipPath
+       id="clipPath101611">
+      <use
+         id="use101613"
+         overflow="visible"
+         xlink:href="#SVGID_4531_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82228">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="SVGID_4538_" />
+    </defs>
+    <clipPath
+       id="clipPath101641">
+      <use
+         id="use101643"
+         overflow="visible"
+         xlink:href="#SVGID_4538_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82178">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="SVGID_4528_" />
+    </defs>
+    <clipPath
+       id="clipPath101597">
+      <use
+         id="use101599"
+         overflow="visible"
+         xlink:href="#SVGID_4528_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)"
+       y2="60.902401"
+       x2="127.8011"
+       y1="60.902401"
+       x1="126.8011"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient101601">
+      <stop
+         id="stop101603"
+         style="stop-color:#E6E8E7"
+         offset="0" />
+      <stop
+         id="stop101605"
+         style="stop-color:#FFFFFF"
+         offset="1" />
+    </linearGradient>
+    <defs
+       id="defs84814">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="SVGID_4876_" />
+    </defs>
+    <clipPath
+       id="clipPath103613">
+      <use
+         id="use103615"
+         overflow="visible"
+         xlink:href="#SVGID_4876_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)"
+       y2="147.0757"
+       x2="41.563"
+       y1="147.0757"
+       x1="40.563"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient103617">
+      <stop
+         id="stop103619"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop103621"
+         style="stop-color:#B0D2D9"
+         offset="0.486" />
+      <stop
+         id="stop103623"
+         style="stop-color:#E6EFF1"
+         offset="0.8287" />
+      <stop
+         id="stop103625"
+         style="stop-color:#FFFFFF"
+         offset="0.9939" />
+      <stop
+         id="stop103627"
+         style="stop-color:#FFFFFF"
+         offset="1" />
+    </linearGradient>
+    <defs
+       id="defs84845">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="SVGID_4881_" />
+    </defs>
+    <clipPath
+       id="clipPath103641">
+      <use
+         id="use103643"
+         overflow="visible"
+         xlink:href="#SVGID_4881_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60681">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60683"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60685">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60687"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60689">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60691"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60693">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60695"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60697">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60699"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60701">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60703"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60705">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60707"
+         width="379.63181"
+         height="647.18188"
+         x="25.31588"
+         y="-9.4916677" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60709">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60711"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60713">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60715"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60717">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60719"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60721">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60723"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60725">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60727"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60729">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60731"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60733">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60735"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60737">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60739"
+         width="168.72525"
+         height="287.63638"
+         x="-780.32684"
+         y="72.128014" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60741">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60743"
+         width="168.72525"
+         height="287.63638"
+         x="-780.28253"
+         y="83.950233" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60745">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60747"
+         width="168.72525"
+         height="287.63638"
+         x="-780.2381"
+         y="95.505791" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60749">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60751"
+         width="168.72525"
+         height="287.63638"
+         x="-696.31427"
+         y="106.91624" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60753">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60755"
+         width="168.72525"
+         height="287.63638"
+         x="-780.2381"
+         y="107.08979" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60757">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60759"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60761">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60763"
+         width="168.72525"
+         height="287.63638"
+         x="-648.44501"
+         y="118.52407" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60765">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60767"
+         width="168.72525"
+         height="287.63638"
+         x="-648.44501"
+         y="118.52407" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60769">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60771"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60773">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60775"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="109.79829" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60777">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60779"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="109.79829" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60781">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60783"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="109.79829" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60785">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60787"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="109.79829" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60789">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60791"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60793">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60795"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60797">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60799"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60801">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60803"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60805">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60807"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60809">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60811"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60813">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60815"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60817">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60819"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60821">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60823"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60825">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60827"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60829">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60831"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60833">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60835"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60837">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60839"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60841">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60843"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60845">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60847"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60849">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60851"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60853">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60855"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60857">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60859"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60861">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60863"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60865">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60867"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60869">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60871"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60873">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60875"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60877">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60879"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60881">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60883"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60885">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60887"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60889">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60891"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60893">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60895"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60897">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60899"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60901">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60903"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60905">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60907"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60909">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60911"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60913">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60915"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60917">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60919"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60921">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60923"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60925">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60927"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60929">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60931"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60933">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60935"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60937">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60939"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60941">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60943"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60945">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60947"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60949">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60951"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60953">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60955"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60957">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60959"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60961">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60963"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60965">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60967"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60969">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60971"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60973">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60975"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60977">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60979"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60981">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60983"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60985">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60987"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60989">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60991"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60993">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60995"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath60997">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60999"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61001">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61003"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61005">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61007"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61009">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61011"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61013">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61015"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61017">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61019"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61021">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61023"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61025">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61027"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61029">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61031"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61033">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61035"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61037">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61039"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61041">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61043"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61045">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61047"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61049">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61051"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61053">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61055"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61057">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61059"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61061">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61063"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61065">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61067"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61069">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61071"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61073">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61075"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61077">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61079"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61081">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61083"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61085">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61087"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61089">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61091"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61093">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61095"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61097">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61099"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61101">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61103"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61105">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61107"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61109">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61111"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61113">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61115"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61117">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61119"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61121">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61123"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61125">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61127"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61129">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61131"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61133">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61135"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61137">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61139"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61141">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61143"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61145">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61147"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61149">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61151"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61153">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61155"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61157">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61159"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61161">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61163"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61165">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61167"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61169">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61171"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61173">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61175"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61177">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61179"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61181">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61183"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61185">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61187"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61189">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61191"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61193">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61195"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61197">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61199"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61201">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61203"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61205">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61207"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61209">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61211"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61213">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61215"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61217">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61219"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61221">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61223"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61225">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61227"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61229">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61231"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61233">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61235"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61237">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61239"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61241">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61243"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61245">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61247"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61249">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61251"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61253">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61255"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61257">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61259"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61261">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61263"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61265">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61267"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61269">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61271"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61273">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61275"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61277">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61279"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61281">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61283"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61285">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61287"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61289">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61291"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61293">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61295"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61297">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61299"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61301">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61303"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61305">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61307"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61309">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61311"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61313">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61315"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61317">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61319"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61321">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61323"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61325">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61327"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61329">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61331"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61333">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61335"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61337">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61339"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61341">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61343"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61345">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61347"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61349">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61351"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61353">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61355"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61357">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61359"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61361">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61363"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61365">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61367"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61369">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61371"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61373">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61375"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61377">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61379"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61381">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61383"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61385">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61387"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61389">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61391"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61393">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61395"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61397">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61399"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61401">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61403"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61405">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61407"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61409">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61411"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61413">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61415"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61417">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61419"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61421">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61423"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61425">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61427"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61429">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61431"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61433">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61435"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61437">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61439"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61441">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61443"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61445">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61447"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61449">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61451"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61453">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61455"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61457">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61459"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61461">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61463"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61465">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61467"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61469">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61471"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61473">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61475"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61477">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61479"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61481">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61483"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61485">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61487"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61489">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61491"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61493">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61495"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61497">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61499"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61501">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61503"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61505">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61507"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61509">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61511"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61513">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61515"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61517">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61519"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61521">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61523"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61525">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61527"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61529">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61531"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61533">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61535"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61537">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61539"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61541">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61543"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61545">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61547"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61549">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61551"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61553">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61555"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61557">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61559"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61561">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61563"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61565">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61567"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61569">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61571"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61573">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61575"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61577">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61579"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61581">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61583"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61585">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61587"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61589">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61591"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61593">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61595"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61597">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61599"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61601">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61603"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61605">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61607"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61609">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61611"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61613">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61615"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61617">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61619"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61621">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61623"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61625">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61627"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61629">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61631"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61633">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:4.01018;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61635"
+         width="280.33414"
+         height="287.63638"
+         x="-727.15228"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61637">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61639"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61641">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61643"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61645">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61647"
+         width="112.4835"
+         height="191.7576"
+         x="-605.36768"
+         y="163.00008" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61649">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61651"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61653">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61655"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61657">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61659"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61661">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61663"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61665">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61667"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61669">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61671"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61673">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61675"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61677">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61679"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61681">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61683"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61685">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61687"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61689">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61691"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61693">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61695"
+         width="112.4835"
+         height="191.7576"
+         x="-605.05963"
+         y="162.99326" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61697">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61699"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61701">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61703"
+         width="168.72525"
+         height="287.63638"
+         x="-731.96674"
+         y="171.19934" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61705">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61707"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61709">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61711"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61713">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61715"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61717">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61719"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61721">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61723"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61725">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61727"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61729">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61731"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61733">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61735"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61737">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61739"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61741">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61743"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61745">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61747"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61749">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61751"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61753">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61755"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61757">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61759"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61761">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61763"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61765">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61767"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61769">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61771"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61773">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61775"
+         width="112.4835"
+         height="191.7576"
+         x="-559.32135"
+         y="163.02754" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61777">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61779"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61781">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61783"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61785">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61787"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61789">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61791"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61793">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61795"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61797">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61799"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61801">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61803"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61805">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61807"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61809">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61811"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61813">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61815"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61817">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61819"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61821">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:2.07407;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61823"
+         width="112.4835"
+         height="191.7576"
+         x="-558.96484"
+         y="163.01535" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61825">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61827"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61829">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61831"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61833">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61835"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61837">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61839"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61841">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61843"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61845">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61847"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61849">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61851"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61853">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61855"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61857">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61859"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61861">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61863"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61865">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61867"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61869">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61871"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61873">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:3.11111;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61875"
+         width="168.72525"
+         height="287.63638"
+         x="-708.34058"
+         y="107.13162" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61877">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61879"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61881">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61883"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61885">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61887"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61889">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61891"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61893">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61895"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61897">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61899"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath61901">
+      <rect
+         style="opacity:1;fill:none;fill-opacity:1;stroke:#000000;stroke-width:7;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect61903"
+         width="379.63181"
+         height="647.18188"
+         x="147.96869"
+         y="178.99452" />
+    </clipPath>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.0729879"
+     inkscape:cx="355.54921"
+     inkscape:cy="235.79017"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1920"
+     inkscape:window-height="1011"
+     inkscape:window-x="0"
+     inkscape:window-y="32"
+     inkscape:window-maximized="1"
+     inkscape:showpageshadow="2"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1" />
+  <metadata
+     id="metadata104541">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-147.96869,-187.26927)">
+    <g
+       id="g93021">
+      <rect
+         style="fill:#ffffff"
+         id="rect6353"
+         width="445.59259"
+         height="660.79095"
+         x="122.65281"
+         y="188.48619"
+         clip-path="url(#clipPath61901)" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:1.02682;stroke-miterlimit:10;stroke-dasharray:3.07565, 3.07565"
+         stroke-miterlimit="10"
+         x1="413.3428"
+         y1="279.20651"
+         x2="413.3428"
+         y2="496.95172"
+         id="line84244"
+         clip-path="url(#clipPath61897)" />
+      <rect
+         style="fill:none;stroke:#5e6a71;stroke-width:0.485;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:10;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect125410-1-2"
+         width="94.101288"
+         height="87.10128"
+         x="287.91125"
+         y="382.12704"
+         clip-path="url(#clipPath61893)" />
+      <rect
+         style="fill:url(#linearGradient9382);fill-opacity:1;stroke:none"
+         id="rect126198-3-8"
+         width="108"
+         height="27"
+         x="275.46191"
+         y="457.79819"
+         clip-path="url(#clipPath61889)" />
+      <rect
+         style="fill:#f15d57"
+         x="211.4852"
+         y="538.19885"
+         width="95.778"
+         height="3.0712502"
+         id="rect82236"
+         inkscape:transform-center-x="586.0074"
+         inkscape:transform-center-y="-79.564828"
+         clip-path="url(#clipPath61885)" />
+      <rect
+         style="fill:#f15d57"
+         x="360.20123"
+         y="538.19885"
+         width="95.793755"
+         height="3.0712502"
+         id="rect82238"
+         inkscape:transform-center-x="437.28349"
+         inkscape:transform-center-y="-79.564828"
+         clip-path="url(#clipPath61881)" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.4815;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+         stroke-miterlimit="10"
+         x1="408.16232"
+         y1="540.85376"
+         x2="408.16232"
+         y2="595.23627"
+         id="line82244"
+         inkscape:transform-center-x="437.21928"
+         inkscape:transform-center-y="-51.254289"
+         clip-path="url(#clipPath61877)" />
+      <polyline
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         stroke-miterlimit="10"
+         points="  -603.212,292.128 -603.212,283.832 -582.187,283.832 -582.187,292.128 "
+         id="polyline82246"
+         inkscape:transform-center-x="437.22048"
+         inkscape:transform-center-y="-33.395941"
+         clip-path="url(#clipPath61873)" />
+      <rect
+         style="fill:url(#SVGID_4545_)"
+         x="-613.90802"
+         y="253.133"
+         clip-path="url(#clipPath61869)"
+         width="42.361"
+         height="14.22"
+         id="rect82306"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         style="fill:#e6e8e8"
+         inkscape:connector-curvature="0"
+         d="m 455.51125,507.80361 -0.0157,31.37625 -94.815,0.072 0.0157,-31.2435 46.48275,11.31975 0.1125,0.027 0.11475,-0.027 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0158,-32.46525 z"
+         id="path82308"
+         inkscape:transform-center-x="437.28456"
+         inkscape:transform-center-y="-95.835733"
+         clip-path="url(#clipPath61865)" />
+      <rect
+         style="fill:url(#SVGID_4548_)"
+         x="-680.00299"
+         y="253.133"
+         clip-path="url(#clipPath61861)"
+         width="42.361"
+         height="14.22"
+         id="rect82335"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         style="fill:#e6e8e8"
+         inkscape:connector-curvature="0"
+         d="m 306.7975,507.80361 -0.0157,31.37625 -94.815,0.072 0.0157,-31.2435 46.48274,11.31975 0.1125,0.027 0.11475,-0.027 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0157,-32.46525 z"
+         id="path82337"
+         inkscape:transform-center-x="585.99834"
+         inkscape:transform-center-y="-95.835733"
+         clip-path="url(#clipPath61857)" />
+      <text
+         y="531.49921"
+         x="240.48843"
+         id="text84236"
+         inkscape:transform-center-x="587.42372"
+         inkscape:transform-center-y="-90.601429"
+         style="font-size:27px"
+         clip-path="url(#clipPath61853)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:8px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#37424b"
+           x="240.48843"
+           y="531.49921"
+           font-size="3.4203"
+           id="tspan84238">US-WEST</tspan></text>
+      <text
+         y="531.49921"
+         x="391.20377"
+         id="text84246"
+         inkscape:transform-center-x="438.70839"
+         inkscape:transform-center-y="-90.601429"
+         style="font-size:27px"
+         inkscape:export-xdpi="90"
+         inkscape:export-ydpi="90"
+         inkscape:export-filename="/home/john/Pictures/zone-sync.png"
+         clip-path="url(#clipPath61849)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:8px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#37424b"
+           x="391.20377"
+           y="531.49921"
+           font-size="3.4203"
+           id="tspan84248">US-EAST</tspan></text>
+      <rect
+         style="fill:#f15d57"
+         x="356.27731"
+         y="456.49902"
+         width="103.6395"
+         height="3.8114998"
+         id="rect84287"
+         inkscape:transform-center-x="437.28453"
+         inkscape:transform-center-y="-160.89453"
+         clip-path="url(#clipPath61845)" />
+      <polygon
+         style="fill:url(#SVGID_4825_)"
+         clip-path="url(#clipPath61841)"
+         points="-615.683,231.929 -569.802,231.874 -569.772,256.788 -615.653,256.843 "
+         id="polygon84308"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         style="fill:#e6e8e8"
+         inkscape:connector-curvature="0"
+         d="m 356.7565,460.02936 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.003,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+         id="path84310"
+         inkscape:transform-center-x="437.28484"
+         inkscape:transform-center-y="-131.54096"
+         clip-path="url(#clipPath61837)" />
+      <text
+         y="476.79285"
+         x="379.91489"
+         id="text84312"
+         inkscape:transform-center-x="437.86922"
+         inkscape:transform-center-y="-134.7035"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:27px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'"
+         clip-path="url(#clipPath61833)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#37424b"
+           x="379.91489"
+           y="476.79285"
+           font-size="5.1244"
+           id="tspan84314">RADOSGW</tspan></text>
+      <rect
+         style="fill:#f15d57"
+         x="206.53078"
+         y="456.49902"
+         width="103.6395"
+         height="3.8114998"
+         id="rect84318"
+         inkscape:transform-center-x="587.03108"
+         inkscape:transform-center-y="-160.89453"
+         clip-path="url(#clipPath61829)" />
+      <line
+         id="line84244-3"
+         y2="496.8772"
+         x2="257.7728"
+         y1="279.13196"
+         x1="257.7728"
+         stroke-miterlimit="10"
+         style="fill:none;stroke:#5e6a71;stroke-width:1.02682;stroke-miterlimit:10;stroke-dasharray:3.07565, 3.07565"
+         clip-path="url(#clipPath61825)" />
+      <rect
+         id="rect82124-4"
+         height="10.191"
+         width="42.327"
+         clip-path="url(#clipPath61821)"
+         y="188.287"
+         x="-547.302"
+         style="fill:url(#linearGradient126904-61)"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <path
+         id="path82134-5"
+         d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+         clip-path="url(#clipPath61817)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <path
+         id="path82136-0"
+         d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+         clip-path="url(#clipPath61813)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <circle
+         id="circle82138-2"
+         r="0.63999999"
+         cy="191.26801"
+         cx="-541.625"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61809)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <circle
+         id="circle82140-3"
+         r="0.63999999"
+         cy="193.623"
+         cx="-541.625"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61805)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <circle
+         id="circle82142-3"
+         r="0.63999999"
+         cy="191.26801"
+         cx="-543.85303"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61801)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <circle
+         id="circle82144-4"
+         r="0.63999999"
+         cy="193.623"
+         cx="-543.854"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61797)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <path
+         id="path82146-1"
+         d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+         clip-path="url(#clipPath61793)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <path
+         id="path82148-5"
+         d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+         clip-path="url(#clipPath61789)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <text
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:40.5px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'"
+         id="text82150-7"
+         x="248.08075"
+         y="282.81369"
+         clip-path="url(#clipPath61785)"><tspan
+           id="tspan82152-4"
+           font-size="3.2027"
+           y="282.81369"
+           x="248.08075"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.8091px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#ffffff">AP</tspan><tspan
+           id="tspan82154-9"
+           font-size="3.2027"
+           y="282.81369"
+           x="261.76285"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.8091px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#ffffff">P</tspan></text>
+      <path
+         id="path84646-0"
+         d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61781)"
+         inkscape:connector-curvature="0"
+         style="fill:#5e6a71"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <line
+         id="line84648-6"
+         y2="205.991"
+         x2="-523.14398"
+         y1="205.991"
+         x1="-526.03998"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61777)"
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2034.475,-371.18227)" />
+      <polygon
+         id="polygon84665-6"
+         points="-523.433,206 -521.64,208.524 -507.674,208.524 -507.674,203.459 -521.64,203.459 "
+         stroke-miterlimit="10"
+         transform="matrix(3.375,0,0,3.375,2035.6783,-371.22344)"
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#clipPath61773)" />
+      <polygon
+         style="fill:url(#SVGID_4828_)"
+         clip-path="url(#clipPath61769)"
+         points="-682.238,231.929 -636.356,231.874 -636.326,256.788 -682.208,256.843 "
+         id="polygon84339"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         style="fill:#e6e8e8"
+         inkscape:connector-curvature="0"
+         d="m 207.01,460.02936 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.002,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+         id="path84341"
+         inkscape:transform-center-x="587.03109"
+         inkscape:transform-center-y="-131.54096"
+         clip-path="url(#clipPath61765)" />
+      <text
+         y="476.79285"
+         x="230.16792"
+         id="text84343"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:27px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'"
+         clip-path="url(#clipPath61761)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#37424b"
+           x="230.16792"
+           y="476.79285"
+           font-size="5.1244"
+           id="tspan84345">RADOSGW</tspan></text>
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.4815;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+         stroke-miterlimit="10"
+         x1="258.25607"
+         y1="540.85376"
+         x2="258.25607"
+         y2="595.23627"
+         id="line84349"
+         inkscape:transform-center-x="587.12555"
+         inkscape:transform-center-y="-51.254289"
+         clip-path="url(#clipPath61757)" />
+      <polyline
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         stroke-miterlimit="10"
+         points="  -669.837,292.128 -669.837,283.832 -648.812,283.832 -648.812,292.128 "
+         id="polyline84351"
+         inkscape:transform-center-x="587.12673"
+         inkscape:transform-center-y="-33.395941"
+         clip-path="url(#clipPath61753)" />
+      <polygon
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         points="-661.37,267.424 -659.319,269.217 -657.251,267.424 "
+         id="polygon84353"
+         inkscape:transform-center-x="587.09523"
+         inkscape:transform-center-y="-77.629816"
+         clip-path="url(#clipPath61749)" />
+      <polygon
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         points="-594.747,267.424 -592.696,269.217 -590.628,267.424 "
+         id="polygon84355"
+         inkscape:transform-center-x="437.19348"
+         inkscape:transform-center-y="-77.629816"
+         clip-path="url(#clipPath61745)" />
+      <path
+         id="path84524"
+         d="m -365.856,302.434 c 0,-2.331 1.892,-4.222 4.225,-4.222 2.334,0 4.223,1.891 4.223,4.222 0,2.334 -1.889,4.226 -4.223,4.226 -2.333,0 -4.225,-1.892 -4.225,-4.226"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#f15d57"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84526"
+         d="m -361.631,301.547 c 0.488,0 0.888,0.398 0.888,0.887 0,0.493 -0.4,0.892 -0.888,0.89 -0.491,0 -0.888,-0.397 -0.888,-0.89 0,-0.489 0.397,-0.887 0.888,-0.887"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84528"
+         d="m -347.575,299.54 c 0.487,0 0.886,0.395 0.886,0.886 0,0.494 -0.399,0.891 -0.886,0.891 -0.493,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.395,-0.886 0.888,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84530"
+         d="m -350.839,302.809 c 0.49,0 0.886,0.397 0.886,0.886 0,0.494 -0.396,0.893 -0.886,0.893 -0.491,0 -0.889,-0.399 -0.889,-0.893 0,-0.489 0.398,-0.886 0.889,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84532"
+         d="m -350.838,299.54 c 0.489,0 0.885,0.395 0.885,0.886 0,0.494 -0.396,0.891 -0.885,0.891 -0.492,0 -0.89,-0.397 -0.89,-0.891 0,-0.491 0.398,-0.886 0.89,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84534"
+         d="m -347.576,302.809 c 0.488,0 0.888,0.397 0.888,0.886 0,0.494 -0.4,0.893 -0.888,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84536"
+         d="m -344.311,302.809 c 0.488,0 0.886,0.397 0.886,0.886 0,0.494 -0.398,0.893 -0.886,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         id="path84538"
+         d="m -344.31,299.54 c 0.487,0 0.885,0.395 0.885,0.886 0,0.494 -0.398,0.891 -0.885,0.891 -0.492,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.396,-0.886 0.888,-0.886"
+         clip-path="url(#SVGID_4843_)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <line
+         id="line84540"
+         y2="308.98499"
+         x2="-343.42499"
+         y1="308.98499"
+         x1="-365.664"
+         stroke-miterlimit="10"
+         clip-path="url(#SVGID_4843_)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <line
+         id="line84542"
+         y2="311.16699"
+         x2="-343.42499"
+         y1="311.16699"
+         x1="-365.664"
+         stroke-miterlimit="10"
+         clip-path="url(#SVGID_4843_)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <line
+         id="line84544"
+         y2="313.35001"
+         x2="-343.42499"
+         y1="313.35001"
+         x1="-365.664"
+         stroke-miterlimit="10"
+         clip-path="url(#SVGID_4843_)"
+         style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <polygon
+         id="polygon84564"
+         points="-438.651,206.566 -436.6,208.359 -434.532,206.566 "
+         clip-path="url(#SVGID_4843_)"
+         style="fill:#f15d57"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <polygon
+         id="polygon84566"
+         points="-428.14,206.566 -426.089,208.359 -424.021,206.566 "
+         clip-path="url(#SVGID_4843_)"
+         style="fill:#f15d57"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <polygon
+         id="polygon84568"
+         points="-417.626,206.566 -415.575,208.359 -413.507,206.566 "
+         clip-path="url(#SVGID_4843_)"
+         style="fill:#f15d57"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <polygon
+         id="polygon84570"
+         points="-298.678,289.091 -296.627,290.885 -294.559,289.091 "
+         clip-path="url(#SVGID_4843_)"
+         style="fill:#f15d57"
+         transform="matrix(1.5,0,0,1.5,1199.2451,155.9225)" />
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61741)"
+         d="m -593.321,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84722"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#clipPath61737)"
+         stroke-miterlimit="10"
+         x1="-592.68903"
+         y1="276.099"
+         x2="-589.79303"
+         y2="276.099"
+         id="line84724"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         style="fill:url(#SVGID_4866_)"
+         x="-590.08099"
+         y="273.56699"
+         clip-path="url(#clipPath61733)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84739"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <polygon
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         stroke-miterlimit="10"
+         points="-588.289,278.632 -574.323,278.632 -574.323,273.567 -588.289,273.567 -590.082,276.108 "
+         id="polygon84741"
+         inkscape:transform-center-x="413.60223"
+         inkscape:transform-center-y="-60.127066"
+         clip-path="url(#clipPath61729)" />
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61725)"
+         d="m -659.955,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84761"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#clipPath61721)"
+         stroke-miterlimit="10"
+         x1="-659.323"
+         y1="276.099"
+         x2="-656.427"
+         y2="276.099"
+         id="line84763"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         style="fill:url(#SVGID_4871_)"
+         x="-656.71503"
+         y="273.56699"
+         clip-path="url(#clipPath61717)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84778"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <polygon
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         stroke-miterlimit="10"
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 "
+         id="polygon84780"
+         inkscape:transform-center-x="563.52648"
+         inkscape:transform-center-y="-60.127066"
+         clip-path="url(#clipPath61713)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:8px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none"
+         x="421.82733"
+         y="540.91724"
+         id="text113406"><tspan
+           sodipodi:role="line"
+           id="tspan113408"
+           x="421.82733"
+           y="540.91724" /></text>
+      <text
+         y="795.61414"
+         x="378.37204"
+         id="text84343-2"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-size:27px;text-align:center;text-anchor:middle"
+         clip-path="url(#clipPath61709)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#37424b"
+           x="333.39987"
+           y="795.61414"
+           font-size="5.1244"
+           id="tspan84345-2">MASTER ZONEGROUP</tspan></text>
+      <text
+         y="889.87488"
+         x="424.41846"
+         id="text84343-2-6"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-size:27px;text-align:center;text-anchor:middle"><tspan
+           style="font-size:11px;font-family:ApexSans-Book;text-align:center;text-anchor:middle;fill:#37424b"
+           x="385.2041"
+           y="889.87488"
+           font-size="5.1244"
+           id="tspan84345-2-5" /></text>
+      <flowRoot
+         xml:space="preserve"
+         id="flowRoot126352"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         transform="translate(135.96191,-36.49144)"><flowRegion
+           id="flowRegion126354"><rect
+             id="rect126356"
+             width="0"
+             height="51"
+             x="247"
+             y="928.36218" /></flowRegion><flowPara
+           id="flowPara126358" /></flowRoot>
+      <flowRoot
+         xml:space="preserve"
+         id="flowRoot126403"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         transform="translate(135.96191,-36.49144)"><flowRegion
+           id="flowRegion126405"><rect
+             id="rect126407"
+             width="185"
+             height="81"
+             x="286"
+             y="941.36218" /></flowRegion><flowPara
+           id="flowPara126409" /></flowRoot>
+      <flowRoot
+         xml:space="preserve"
+         id="flowRoot126411"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         transform="translate(135.96191,-36.49144)"><flowRegion
+           id="flowRegion126413"><rect
+             id="rect126415"
+             width="89"
+             height="64"
+             x="285"
+             y="943.36218" /></flowRegion><flowPara
+           id="flowPara126417" /></flowRoot>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none"
+         x="407.96191"
+         y="493.15442"
+         id="text126456"
+         clip-path="url(#clipPath61705)"><tspan
+           sodipodi:role="line"
+           id="tspan126458"
+           x="407.96191"
+           y="493.15442"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'">MASTER</tspan><tspan
+           sodipodi:role="line"
+           x="407.96191"
+           y="504.40442"
+           id="tspan126460"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'">ZONE</tspan></text>
+      <polygon
+         id="polygon84546-9-3"
+         points="-669.845,294.263 -667.777,292.47 -671.896,292.47 "
+         clip-path="url(#clipPath61701)"
+         style="fill:#f15d57"
+         transform="matrix(2.25,0,0,2.25,1794.8939,-206.20401)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:9px;line-height:125%;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none"
+         x="258.19333"
+         y="493.15442"
+         id="text126655-6"
+         clip-path="url(#clipPath61697)"><tspan
+           sodipodi:role="line"
+           id="tspan126657-2"
+           x="258.19333"
+           y="493.15442">SECONDARY</tspan><tspan
+           sodipodi:role="line"
+           x="258.19333"
+           y="504.40442"
+           id="tspan126659-4">ZONE</tspan></text>
+      <rect
+         style="fill:url(#linearGradient126904)"
+         x="-547.302"
+         y="188.287"
+         clip-path="url(#clipPath61693)"
+         width="42.327"
+         height="10.191"
+         id="rect82124"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61689)"
+         d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82134"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61685)"
+         d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82136"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         clip-path="url(#clipPath61681)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82138"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         clip-path="url(#clipPath61677)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82140"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         clip-path="url(#clipPath61673)"
+         stroke-miterlimit="10"
+         cx="-543.85303"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82142"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         clip-path="url(#clipPath61669)"
+         stroke-miterlimit="10"
+         cx="-543.854"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82144"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61665)"
+         d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82146"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61661)"
+         d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82148"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <text
+         y="282.88824"
+         x="403.65076"
+         id="text82150"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:40.5px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans'"
+         clip-path="url(#clipPath61657)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.8091px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#ffffff"
+           x="403.65076"
+           y="282.88824"
+           font-size="3.2027"
+           id="tspan82152">AP</tspan><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.8091px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';fill:#ffffff"
+           x="417.33286"
+           y="282.88824"
+           font-size="3.2027"
+           id="tspan82154">P</tspan></text>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61653)"
+         d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84646"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#clipPath61649)"
+         stroke-miterlimit="10"
+         x1="-526.03998"
+         y1="205.991"
+         x2="-523.14398"
+         y2="205.991"
+         id="line84648"
+         transform="matrix(3.375,0,0,3.375,2190.045,-371.10773)" />
+      <polygon
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+         transform="matrix(3.375,0,0,3.375,2191.0846,-371.13073)"
+         stroke-miterlimit="10"
+         points="-523.433,206 -521.64,208.524 -507.674,208.524 -507.674,203.459 -521.64,203.459 "
+         id="polygon84665"
+         clip-path="url(#clipPath61645)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none"
+         x="333.03717"
+         y="401.67719"
+         id="text127418"
+         clip-path="url(#clipPath61641)"><tspan
+           sodipodi:role="line"
+           id="tspan127420"
+           x="333.03717"
+           y="401.67719">DATA</tspan><tspan
+           sodipodi:role="line"
+           x="333.03717"
+           y="412.92719"
+           id="tspan127422">SYNC</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:9px;line-height:125%;font-family:Sans;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none"
+         x="368.21997"
+         y="141.67719"
+         id="text127482"><tspan
+           sodipodi:role="line"
+           id="tspan127484"
+           x="368.21997"
+           y="141.67719" /></text>
+      <text
+         y="211.11368"
+         x="378.37204"
+         id="text84343-2-0"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:27px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle"
+         clip-path="url(#clipPath61637)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#37424b"
+           x="333.39987"
+           y="211.11368"
+           font-size="5.1244"
+           id="tspan84345-2-6">MASTER ZONEGROUP</tspan></text>
+      <rect
+         id="rect82189"
+         height="64.038002"
+         width="174.953"
+         clip-path="url(#clipPath61633)"
+         y="293.366"
+         x="-679.57202"
+         style="fill:url(#SVGID_4530_)"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-62.051623)" />
+      <rect
+         id="rect82199"
+         height="64.038002"
+         width="174.953"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61629)"
+         y="293.366"
+         x="-679.57202"
+         style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-62.051623)" />
+      <path
+         id="path82347"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         clip-path="url(#clipPath61625)"
+         inkscape:connector-curvature="0"
+         style="fill:#f15d57"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82349"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61621)"
+         inkscape:connector-curvature="0"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351"
+         font-size="4.3343"
+         style="font-size:9.75218px;font-family:ApexSans-Medium;fill:#ffffff"
+         x="411.10547"
+         y="668.39728"
+         clip-path="url(#clipPath61617)">M</text>
+      <path
+         id="path82361"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         clip-path="url(#clipPath61613)"
+         inkscape:connector-curvature="0"
+         style="fill:#f15d57"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82363"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61609)"
+         inkscape:connector-curvature="0"
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <text
+         inkscape:transform-center-y="71.524915"
+         inkscape:transform-center-x="503.47899"
+         id="text82365"
+         font-size="4.3343"
+         style="font-size:9.75218px;font-family:ApexSans-Medium;fill:#ffffff"
+         x="337.69321"
+         y="694.37891"
+         clip-path="url(#clipPath61605)">M</text>
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381"
+         height="21.584251"
+         width="21.581999"
+         y="628.10437"
+         x="223.6218"
+         style="fill:#ffffff"
+         clip-path="url(#clipPath61601)" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="628.10437"
+         x="223.6218"
+         style="fill:none;stroke:#81d0db;stroke-width:0.963;stroke-miterlimit:10"
+         clip-path="url(#clipPath61597)" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385"
+         height="18.553501"
+         width="18.557999"
+         y="629.62085"
+         x="225.13145"
+         style="fill:#81d0db"
+         clip-path="url(#clipPath61593)" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387"
+         height="15.594751"
+         width="15.592501"
+         y="631.09906"
+         x="226.61653"
+         style="fill:#ffffff"
+         clip-path="url(#clipPath61589)" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389"
+         height="10.278"
+         width="10.282501"
+         y="633.75629"
+         x="229.26932"
+         style="fill:#ffffff"
+         clip-path="url(#clipPath61585)" />
+      <rect
+         id="rect82404"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61581)"
+         y="308.841"
+         x="-672.61401"
+         style="fill:url(#SVGID_4557_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="306.73599"
+         clip-path="url(#clipPath61577)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82451" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="306.73599"
+         clip-path="url(#clipPath61573)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82453" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="307.41"
+         clip-path="url(#clipPath61569)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82455" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="308.06699"
+         clip-path="url(#clipPath61565)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82457" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="309.24799"
+         clip-path="url(#clipPath61561)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82459" />
+      <rect
+         id="rect82474"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61557)"
+         y="308.841"
+         x="-648.67102"
+         style="fill:url(#SVGID_4567_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath61553)"
+         d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82484" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="306.73599"
+         clip-path="url(#clipPath61549)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82486" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="306.73599"
+         clip-path="url(#clipPath61545)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82488" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="307.41"
+         clip-path="url(#clipPath61541)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82490" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="308.06699"
+         clip-path="url(#clipPath61537)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82492" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="309.24799"
+         clip-path="url(#clipPath61533)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82494" />
+      <rect
+         id="rect82509"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61529)"
+         y="308.841"
+         x="-636.69897"
+         style="fill:url(#SVGID_4572_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82519"
+         d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61525)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82521"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61521)"
+         y="306.73599"
+         x="-626.83099"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82523"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61517)"
+         y="306.73599"
+         x="-626.83099"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82525"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61513)"
+         y="307.41"
+         x="-626.15997"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82527"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61509)"
+         y="308.06699"
+         x="-625.5"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82529"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61505)"
+         y="309.24799"
+         x="-624.32098"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82544"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61501)"
+         y="308.841"
+         x="-624.72803"
+         style="fill:url(#SVGID_4577_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82554"
+         d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61497)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82556"
+         height="9.5930004"
+         width="9.5930004"
+         clip-path="url(#clipPath61493)"
+         y="306.73599"
+         x="-614.85999"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82558"
+         height="9.5930004"
+         width="9.5930004"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61489)"
+         y="306.73599"
+         x="-614.86102"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82560"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61485)"
+         y="307.41"
+         x="-614.18799"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82562"
+         height="6.9310002"
+         width="6.9310002"
+         clip-path="url(#clipPath61481)"
+         y="308.06699"
+         x="-613.52899"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82564"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61477)"
+         y="309.24799"
+         x="-612.349"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82579"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath61473)"
+         y="308.841"
+         x="-612.75598"
+         style="fill:url(#SVGID_4582_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82589"
+         d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61469)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82591"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61465)"
+         y="306.73599"
+         x="-602.888"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82593"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61461)"
+         y="306.73599"
+         x="-602.888"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82595"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61457)"
+         y="307.41"
+         x="-602.21698"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82597"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61453)"
+         y="308.06699"
+         x="-601.55701"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82599"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61449)"
+         y="309.24799"
+         x="-600.37799"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82614"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath61445)"
+         y="308.841"
+         x="-600.78497"
+         style="fill:url(#SVGID_4587_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82624"
+         d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61441)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82626"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61437)"
+         y="306.73599"
+         x="-590.91699"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82628"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61433)"
+         y="306.73599"
+         x="-590.91699"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82630"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61429)"
+         y="307.41"
+         x="-590.24597"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82632"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61425)"
+         y="308.06699"
+         x="-589.586"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82634"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61421)"
+         y="309.24799"
+         x="-588.40698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82649"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath61417)"
+         y="308.841"
+         x="-588.81299"
+         style="fill:url(#SVGID_4592_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82871"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61413)"
+         y="318.323"
+         x="-674.71698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82873"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61409)"
+         y="318.323"
+         x="-674.71698"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82875"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61405)"
+         y="318.99701"
+         x="-674.04602"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82877"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61401)"
+         y="319.65399"
+         x="-673.38599"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82879"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61397)"
+         y="320.83499"
+         x="-672.20697"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82894"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61393)"
+         y="320.42899"
+         x="-672.61401"
+         style="fill:url(#SVGID_4627_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82904"
+         d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61389)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82906"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61385)"
+         y="318.323"
+         x="-662.745"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82908"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61381)"
+         y="318.323"
+         x="-662.74597"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82910"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61377)"
+         y="318.99701"
+         x="-662.07397"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82912"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61373)"
+         y="319.65399"
+         x="-661.414"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82914"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61369)"
+         y="320.83499"
+         x="-660.23499"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82929"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61365)"
+         y="320.42899"
+         x="-660.64203"
+         style="fill:url(#SVGID_4632_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82939"
+         d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61361)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82941"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61357)"
+         y="318.323"
+         x="-650.77399"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82943"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61353)"
+         y="318.323"
+         x="-650.77399"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82945"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61349)"
+         y="318.99701"
+         x="-650.10303"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82947"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61345)"
+         y="319.65399"
+         x="-649.44299"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82949"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61341)"
+         y="320.83499"
+         x="-648.26398"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82964"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61337)"
+         y="320.42899"
+         x="-648.67102"
+         style="fill:url(#SVGID_4637_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path82974"
+         d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61333)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82976"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61329)"
+         y="318.323"
+         x="-638.80298"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82978"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61325)"
+         y="318.323"
+         x="-638.80298"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82980"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61321)"
+         y="318.99701"
+         x="-638.13202"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82982"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61317)"
+         y="319.65399"
+         x="-637.47198"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82984"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61313)"
+         y="320.83499"
+         x="-636.29199"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect82999"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61309)"
+         y="320.42899"
+         x="-636.69897"
+         style="fill:url(#SVGID_4642_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83009"
+         d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61305)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83011"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61301)"
+         y="318.323"
+         x="-626.83099"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83013"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61297)"
+         y="318.323"
+         x="-626.83099"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83015"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61293)"
+         y="318.99701"
+         x="-626.15997"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83017"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61289)"
+         y="319.65399"
+         x="-625.5"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83019"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61285)"
+         y="320.83499"
+         x="-624.32098"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83034"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61281)"
+         y="320.42899"
+         x="-624.72803"
+         style="fill:url(#SVGID_4647_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83044"
+         d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61277)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83046"
+         height="9.5930004"
+         width="9.5930004"
+         clip-path="url(#clipPath61273)"
+         y="318.323"
+         x="-614.85999"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83048"
+         height="9.5930004"
+         width="9.5930004"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61269)"
+         y="318.323"
+         x="-614.86102"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83050"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61265)"
+         y="318.99701"
+         x="-614.18799"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83052"
+         height="6.9310002"
+         width="6.9310002"
+         clip-path="url(#clipPath61261)"
+         y="319.65399"
+         x="-613.52899"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83054"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61257)"
+         y="320.83499"
+         x="-612.349"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83069"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath61253)"
+         y="320.42899"
+         x="-612.75598"
+         style="fill:url(#SVGID_4652_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83079"
+         d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61249)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83081"
+         height="9.5930004"
+         width="9.592"
+         clip-path="url(#clipPath61245)"
+         y="318.323"
+         x="-590.91699"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83083"
+         height="9.5930004"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61241)"
+         y="318.323"
+         x="-590.91699"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83085"
+         height="8.2460003"
+         width="8.2480001"
+         clip-path="url(#clipPath61237)"
+         y="318.99701"
+         x="-590.24597"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83087"
+         height="6.9310002"
+         width="6.9299998"
+         clip-path="url(#clipPath61233)"
+         y="319.65399"
+         x="-589.586"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83089"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61229)"
+         y="320.83499"
+         x="-588.40698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83104"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath61225)"
+         y="320.42899"
+         x="-588.81299"
+         style="fill:url(#SVGID_4657_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83326"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61221)"
+         y="329.87"
+         x="-674.71698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83328"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61217)"
+         y="329.87"
+         x="-674.71698"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83330"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61213)"
+         y="330.543"
+         x="-674.04602"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83332"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61209)"
+         y="331.20099"
+         x="-673.38599"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83334"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61205)"
+         y="332.38199"
+         x="-672.20697"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83349"
+         height="5.3860002"
+         width="5.3839998"
+         clip-path="url(#clipPath61201)"
+         y="331.974"
+         x="-672.61401"
+         style="fill:url(#SVGID_4692_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83359"
+         d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61197)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83361"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61193)"
+         y="329.87"
+         x="-662.745"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83363"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61189)"
+         y="329.87"
+         x="-662.74597"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83365"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61185)"
+         y="330.543"
+         x="-662.07397"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83367"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61181)"
+         y="331.20099"
+         x="-661.414"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83369"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61177)"
+         y="332.38199"
+         x="-660.23499"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83384"
+         height="5.3850002"
+         width="5.3839998"
+         clip-path="url(#clipPath61173)"
+         y="331.974"
+         x="-660.64203"
+         style="fill:url(#SVGID_4697_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83394"
+         d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61169)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83396"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61165)"
+         y="329.87"
+         x="-650.77399"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83398"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61161)"
+         y="329.87"
+         x="-650.77399"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83400"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61157)"
+         y="330.543"
+         x="-650.10303"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83402"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61153)"
+         y="331.20099"
+         x="-649.44299"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83404"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61149)"
+         y="332.38199"
+         x="-648.26398"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83419"
+         height="5.3860002"
+         width="5.3839998"
+         clip-path="url(#clipPath61145)"
+         y="331.974"
+         x="-648.67102"
+         style="fill:url(#SVGID_4702_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83429"
+         d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61141)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83431"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61137)"
+         y="329.87"
+         x="-638.80298"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83433"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61133)"
+         y="329.87"
+         x="-638.80298"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83435"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61129)"
+         y="330.543"
+         x="-638.13202"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83437"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61125)"
+         y="331.20099"
+         x="-637.47198"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83439"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61121)"
+         y="332.38199"
+         x="-636.29199"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83454"
+         height="5.3860002"
+         width="5.3839998"
+         clip-path="url(#clipPath61117)"
+         y="331.974"
+         x="-636.69897"
+         style="fill:url(#SVGID_4707_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83464"
+         d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61113)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83466"
+         height="9.592"
+         width="9.5930004"
+         clip-path="url(#clipPath61109)"
+         y="329.87"
+         x="-614.85999"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83468"
+         height="9.592"
+         width="9.5930004"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61105)"
+         y="329.87"
+         x="-614.86102"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83470"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61101)"
+         y="330.543"
+         x="-614.18799"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83472"
+         height="6.9299998"
+         width="6.9310002"
+         clip-path="url(#clipPath61097)"
+         y="331.20099"
+         x="-613.52899"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83474"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61093)"
+         y="332.38199"
+         x="-612.349"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83489"
+         height="5.3860002"
+         width="5.3829999"
+         clip-path="url(#clipPath61089)"
+         y="331.974"
+         x="-612.75598"
+         style="fill:url(#SVGID_4712_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83499"
+         d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61085)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83501"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61081)"
+         y="329.87"
+         x="-602.888"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83503"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61077)"
+         y="329.87"
+         x="-602.888"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83505"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61073)"
+         y="330.543"
+         x="-602.21698"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83507"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61069)"
+         y="331.20099"
+         x="-601.55701"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83509"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61065)"
+         y="332.38199"
+         x="-600.37799"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83524"
+         height="5.3860002"
+         width="5.3829999"
+         clip-path="url(#clipPath61061)"
+         y="331.974"
+         x="-600.78497"
+         style="fill:url(#SVGID_4717_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83534"
+         d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61057)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83536"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61053)"
+         y="329.87"
+         x="-590.91699"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83538"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61049)"
+         y="329.87"
+         x="-590.91699"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83540"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61045)"
+         y="330.543"
+         x="-590.24597"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83542"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61041)"
+         y="331.20099"
+         x="-589.586"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83544"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61037)"
+         y="332.38199"
+         x="-588.40698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83559"
+         height="5.3860002"
+         width="5.3829999"
+         clip-path="url(#clipPath61033)"
+         y="331.974"
+         x="-588.81299"
+         style="fill:url(#SVGID_4722_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83746"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61029)"
+         y="341.65302"
+         x="-674.71698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83748"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath61025)"
+         y="341.65302"
+         x="-674.71698"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83750"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath61021)"
+         y="342.32599"
+         x="-674.04602"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83752"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath61017)"
+         y="342.98401"
+         x="-673.38599"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83754"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath61013)"
+         y="344.164"
+         x="-672.20697"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83769"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath61009)"
+         y="343.75699"
+         x="-672.61401"
+         style="fill:url(#SVGID_4752_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83779"
+         d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath61005)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83781"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath61001)"
+         y="341.65302"
+         x="-662.745"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83783"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60997)"
+         y="341.65302"
+         x="-662.74597"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83785"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60993)"
+         y="342.32599"
+         x="-662.07397"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83787"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60989)"
+         y="342.98401"
+         x="-661.414"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83789"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60985)"
+         y="344.164"
+         x="-660.23499"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83804"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath60981)"
+         y="343.75699"
+         x="-660.64203"
+         style="fill:url(#SVGID_4757_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83814"
+         d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60977)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83816"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath60973)"
+         y="341.65302"
+         x="-650.77399"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83818"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60969)"
+         y="341.65302"
+         x="-650.77399"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83820"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60965)"
+         y="342.32599"
+         x="-650.10303"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83822"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60961)"
+         y="342.98401"
+         x="-649.44299"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83824"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60957)"
+         y="344.164"
+         x="-648.26398"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83839"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath60953)"
+         y="343.75699"
+         x="-648.67102"
+         style="fill:url(#SVGID_4762_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83849"
+         d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60949)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83851"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath60945)"
+         y="341.65302"
+         x="-638.80298"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83853"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60941)"
+         y="341.65302"
+         x="-638.80298"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83855"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60937)"
+         y="342.32599"
+         x="-638.13202"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83857"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60933)"
+         y="342.98401"
+         x="-637.47198"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83859"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60929)"
+         y="344.164"
+         x="-636.29199"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83874"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath60925)"
+         y="343.75699"
+         x="-636.69897"
+         style="fill:url(#SVGID_4767_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83884"
+         d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60921)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83886"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath60917)"
+         y="341.65302"
+         x="-626.83099"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83888"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60913)"
+         y="341.65302"
+         x="-626.83099"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83890"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60909)"
+         y="342.32599"
+         x="-626.15997"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83892"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60905)"
+         y="342.98401"
+         x="-625.5"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83894"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60901)"
+         y="344.164"
+         x="-624.32098"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83909"
+         height="5.3839998"
+         width="5.3839998"
+         clip-path="url(#clipPath60897)"
+         y="343.75699"
+         x="-624.72803"
+         style="fill:url(#SVGID_4772_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83919"
+         d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60893)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83921"
+         height="9.592"
+         width="9.5930004"
+         clip-path="url(#clipPath60889)"
+         y="341.65302"
+         x="-614.85999"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83923"
+         height="9.592"
+         width="9.5930004"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60885)"
+         y="341.65302"
+         x="-614.86102"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83925"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60881)"
+         y="342.32599"
+         x="-614.18799"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83927"
+         height="6.9299998"
+         width="6.9310002"
+         clip-path="url(#clipPath60877)"
+         y="342.98401"
+         x="-613.52899"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83929"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60873)"
+         y="344.164"
+         x="-612.349"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83944"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath60869)"
+         y="343.75699"
+         x="-612.75598"
+         style="fill:url(#SVGID_4777_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83954"
+         d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60865)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83956"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath60861)"
+         y="341.65302"
+         x="-602.888"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83958"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60857)"
+         y="341.65302"
+         x="-602.888"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83960"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60853)"
+         y="342.32599"
+         x="-602.21698"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83962"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60849)"
+         y="342.98401"
+         x="-601.55701"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83964"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60845)"
+         y="344.164"
+         x="-600.37799"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83979"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath60841)"
+         y="343.75699"
+         x="-600.78497"
+         style="fill:url(#SVGID_4782_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <path
+         id="path83989"
+         d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         clip-path="url(#clipPath60837)"
+         inkscape:connector-curvature="0"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83991"
+         height="9.592"
+         width="9.592"
+         clip-path="url(#clipPath60833)"
+         y="341.65302"
+         x="-590.91699"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83993"
+         height="9.592"
+         width="9.592"
+         stroke-miterlimit="10"
+         clip-path="url(#clipPath60829)"
+         y="341.65302"
+         x="-590.91699"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83995"
+         height="8.2469997"
+         width="8.2480001"
+         clip-path="url(#clipPath60825)"
+         y="342.32599"
+         x="-590.24597"
+         style="fill:#81d0db"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83997"
+         height="6.9299998"
+         width="6.9299998"
+         clip-path="url(#clipPath60821)"
+         y="342.98401"
+         x="-589.586"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect83999"
+         height="4.5679998"
+         width="4.5700002"
+         clip-path="url(#clipPath60817)"
+         y="344.164"
+         x="-588.40698"
+         style="fill:#ffffff"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <rect
+         id="rect84014"
+         height="5.3839998"
+         width="5.3829999"
+         clip-path="url(#clipPath60813)"
+         y="343.75699"
+         x="-588.81299"
+         style="fill:url(#SVGID_4787_)"
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60809)"
+         points="-671.896,292.47 -669.845,294.263 -667.777,292.47 "
+         id="polygon84546" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60805)"
+         points="-661.384,292.47 -659.333,294.263 -657.265,292.47 "
+         id="polygon84548" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60801)"
+         points="-650.871,292.47 -648.82,294.263 -646.752,292.47 "
+         id="polygon84550" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60797)"
+         points="-605.249,292.47 -603.198,294.263 -601.13,292.47 "
+         id="polygon84552" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60793)"
+         points="-594.737,292.47 -592.686,294.263 -590.618,292.47 "
+         id="polygon84554" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1741.735,-62.051623)"
+         style="fill:#f15d57"
+         clip-path="url(#clipPath60789)"
+         points="-584.224,292.47 -582.173,294.263 -580.105,292.47 "
+         id="polygon84556" />
+      <path
+         id="path84800"
+         d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+         clip-path="url(#SVGID_4873_)"
+         inkscape:connector-curvature="0"
+         style="fill:#5e6a71"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-68.051623)" />
+      <rect
+         id="rect84831"
+         height="10.153"
+         width="174.95799"
+         clip-path="url(#clipPath60785)"
+         y="360.138"
+         x="-679.90002"
+         style="fill:url(#SVGID_4878_)"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-68.051623)" />
+      <polygon
+         id="polygon84851"
+         points="-679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 -504.943,360.138 -679.89,360.139 "
+         clip-path="url(#clipPath60781)"
+         style="fill:#f15d57"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-68.051623)" />
+      <rect
+         id="rect84853"
+         height="2.641"
+         width="174.96001"
+         clip-path="url(#clipPath60777)"
+         y="360.267"
+         x="-679.90198"
+         style="fill:#f15d57"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-68.051623)" />
+      <rect
+         id="rect84855"
+         height="1.352"
+         width="174.94701"
+         clip-path="url(#clipPath60773)"
+         y="370.29199"
+         x="-679.88898"
+         style="fill:#55aeb9"
+         transform="matrix(1.3542118,0,0,2.25,1132.6869,-68.051623)" />
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9"
+         font-size="4.3343"
+         style="font-size:9.75218px;font-family:ApexSans-Medium;fill:#ffffff"
+         x="391.19333"
+         y="668.88776"
+         clip-path="url(#clipPath60769)">M</text>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60765)"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         id="path82347-9"
+         transform="matrix(2.25,0,0,2.25,1606.97,-87.684642)" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.321;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60761)"
+         stroke-miterlimit="10"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 z"
+         id="path82349-9"
+         transform="matrix(2.25,0,0,2.25,1606.97,-87.684642)" />
+      <text
+         y="642.88776"
+         x="256.59122"
+         style="font-size:9.75218px;font-family:ApexSans-Medium;fill:#ffffff"
+         font-size="4.3343"
+         id="text82351-9-4"
+         inkscape:transform-center-x="450.06672"
+         inkscape:transform-center-y="45.543285"
+         clip-path="url(#clipPath60757)">M</text>
+      <path
+         transform="matrix(2.25,0,0,2.25,1903.5044,-61.957507)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60753)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1714.6758,-61.567027)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60749)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1903.5044,-35.893507)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60745)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1903.6044,-9.893507)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60741)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0-1" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1903.7041,16.706493)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#clipPath60737)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0-1-5" />
+      <text
+         y="256.97998"
+         x="304.05688"
+         id="text84343-2-0-4"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:27px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle"
+         clip-path="url(#clipPath60733)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#37424b"
+           x="259.08472"
+           y="256.97998"
+           font-size="5.1244"
+           id="tspan84345-2-6-5">READ ONLY</tspan></text>
+      <text
+         y="256.4751"
+         x="457.74365"
+         id="text84343-2-0-4-3"
+         inkscape:transform-center-x="587.62742"
+         inkscape:transform-center-y="-134.7035"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:27px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle"
+         clip-path="url(#clipPath60729)"><tspan
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#37424b"
+           x="412.77148"
+           y="256.4751"
+           font-size="5.1244"
+           id="tspan84345-2-6-5-3">WRITE / READ</tspan></text>
+      <g
+         id="g87377">
+        <text
+           y="225.89268"
+           x="378.15988"
+           id="text84343-2-0-4-6"
+           inkscape:transform-center-x="587.62742"
+           inkscape:transform-center-y="-134.7035"
+           style="font-size:27px;text-align:center;text-anchor:middle"
+           clip-path="url(#clipPath60725)"><tspan
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#37424b"
+             x="333.18771"
+             y="225.89268"
+             font-size="5.1244"
+             id="tspan84345-2-6-5-6">(United States)</tspan></text>
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:8.2259px;font-family:sans-serif;-inkscape-font-specification:sans-serif;text-align:center;text-anchor:middle;fill:#000000;stroke-width:0.685492"
+         x="282.56839"
+         y="561.89795"
+         id="text18066"
+         clip-path="url(#clipPath60721)"><tspan
+           sodipodi:role="line"
+           id="tspan18064"
+           x="282.56839"
+           y="561.89795"
+           style="stroke-width:0.685492">NATIVE</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:8.2259px;font-family:sans-serif;-inkscape-font-specification:sans-serif;text-align:center;text-anchor:middle;fill:#000000;stroke-width:0.685492"
+         x="432.63998"
+         y="562.07495"
+         id="text18066-2"
+         clip-path="url(#clipPath60717)"><tspan
+           sodipodi:role="line"
+           id="tspan18064-0"
+           x="432.63998"
+           y="562.07495"
+           style="stroke-width:0.685492">NATIVE</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:12px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#000000"
+         x="295.58789"
+         y="328.13101"
+         id="text18255"
+         clip-path="url(#clipPath60713)"><tspan
+           sodipodi:role="line"
+           id="tspan18253"
+           x="295.58789"
+           y="328.13101">REST</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:12px;font-family:'DejaVu Sans';-inkscape-font-specification:'DejaVu Sans';text-align:center;text-anchor:middle;fill:#000000"
+         x="454.07568"
+         y="328.0863"
+         id="text18255-2"
+         clip-path="url(#clipPath60709)"><tspan
+           sodipodi:role="line"
+           id="tspan18253-3"
+           x="454.07568"
+           y="328.0863">REST</tspan></text>
+      <path
+         style="fill:#f0f1f1;fill-opacity:1;stroke-width:0.116497"
+         d="m 305.33447,139.83555 c -1.5353,-2.15705 -2.83148,-3.9885 -2.88041,-4.06989 -0.0572,-0.0951 0.9412,-1.60821 2.7929,-4.2329 l 2.88186,-4.08491 h 23.27333 23.27333 v 8.1548 8.1548 H 331.4007 308.12591 Z m 32.24668,-0.42818 c 0.93699,-0.45843 1.30385,-1.08769 1.29904,-2.22817 -0.004,-0.9312 -0.18777,-1.37911 -0.76567,-1.86538 -0.48036,-0.4042 -0.68807,-0.4842 -2.30641,-0.88843 -1.48115,-0.36996 -1.74005,-0.54291 -1.83075,-1.22296 -0.0474,-0.35547 -0.003,-0.52653 0.2031,-0.78845 0.6608,-0.83835 2.11899,-0.88633 3.98156,-0.131 0.17963,0.0728 0.20387,0.006 0.20387,-0.56505 v -0.64772 l -0.43687,-0.12566 c -0.86722,-0.24944 -2.2724,-0.35278 -3.02226,-0.22226 -1.40688,0.24488 -2.15797,1.02245 -2.22618,2.30468 -0.0797,1.49771 0.67598,2.23462 2.71463,2.64726 0.96518,0.19536 1.65494,0.48012 1.91134,0.78906 0.24888,0.29988 0.28986,1.04242 0.0799,1.44837 -0.30027,0.58066 -0.78831,0.77633 -1.93302,0.775 -0.98365,-10e-4 -1.10221,-0.0331 -2.65031,-0.71487 -0.10741,-0.0473 -0.14562,0.1186 -0.14562,0.63218 v 0.69631 l 0.49511,0.13292 c 1.15073,0.30893 1.48493,0.35104 2.59352,0.32678 1.0115,-0.0221 1.25523,-0.069 1.83499,-0.35261 z m -19.13627,-1.60589 v -1.87856 l 0.99237,0.0521 c 0.93496,0.0491 1.0155,0.0745 1.39206,0.43948 0.21983,0.21307 0.73,1.03436 1.1337,1.82509 l 0.73401,1.4377 0.66986,0.001 c 0.36842,8.1e-4 0.66986,-0.0284 0.66986,-0.0648 0,-0.24918 -1.7292,-3.40276 -2.03171,-3.70527 l -0.37593,-0.37593 0.48783,-0.42436 c 0.59108,-0.51417 0.81828,-1.18732 0.7205,-2.13471 -0.0793,-0.76797 -0.36408,-1.29608 -0.89023,-1.65066 -0.6134,-0.41337 -1.11056,-0.49538 -3.00721,-0.49607 l -1.77658,-6.5e-4 v 4.42689 4.42689 h 0.64073 0.64074 z m 12.69818,1.29608 v -0.58248 h -2.21344 -2.21345 v -1.51447 -1.51446 h 2.09695 2.09695 v -0.52424 -0.52423 h -2.09695 -2.09695 v -1.28147 -1.28147 h 2.21345 2.21344 v -0.52424 -0.52423 h -2.85418 -2.85417 v 4.42689 4.42689 h 2.85417 2.85418 z m 12.69819,-3.32017 v -3.90265 h 1.57271 1.57271 v -0.52424 -0.52423 h -3.78615 -3.78616 v 0.52423 0.52424 h 1.57271 1.57271 v 3.90265 3.90266 h 0.64074 0.64073 z"
+         id="path49149"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60705)" />
+      <path
+         style="fill:#5e6a71;fill-opacity:1;stroke-width:0.041188"
+         d="m 149.80537,139.62238 c -1.5788,-2.22265 -2.87445,-4.05418 -2.87924,-4.07006 -0.005,-0.0159 1.28826,-1.86719 2.87345,-4.11402 l 2.88216,-4.08515 h 23.28249 23.28249 v 8.15521 8.15522 H 175.96131 152.6759 Z m 28.45609,0.0935 c 1.14799,-0.24773 1.79588,-0.80698 2.03981,-1.76075 0.15965,-0.62424 0.0825,-1.42191 -0.18945,-1.95857 -0.0613,-0.12096 -0.23419,-0.34268 -0.38422,-0.49271 -0.2173,-0.21729 -0.36062,-0.31534 -0.70471,-0.48211 -0.45685,-0.22141 -0.75797,-0.30949 -1.7271,-0.50521 -0.7467,-0.15081 -0.98724,-0.21988 -1.27692,-0.36672 -0.42219,-0.214 -0.57645,-0.48452 -0.57788,-1.01343 -8e-4,-0.29058 0.0153,-0.36562 0.12557,-0.58634 0.25339,-0.50708 0.75307,-0.76152 1.60341,-0.81648 0.66672,-0.0431 1.39515,0.0981 2.18297,0.42298 0.19255,0.0794 0.38253,0.15465 0.42217,0.1672 0.0664,0.021 0.0721,-0.0244 0.0721,-0.57454 v -0.59734 l -0.46336,-0.12664 c -0.87625,-0.23947 -1.1287,-0.27249 -2.0697,-0.27074 -0.75933,0.001 -0.90457,0.0126 -1.18943,0.0915 -0.56136,0.15551 -0.91399,0.35221 -1.26228,0.70413 -0.47857,0.48354 -0.6562,0.94146 -0.65737,1.6946 -0.001,0.91579 0.35417,1.53769 1.10151,1.9265 0.44431,0.23116 0.82089,0.34948 1.65748,0.52078 1.33967,0.27431 1.73789,0.46989 2.01108,0.98765 0.098,0.18576 0.11003,0.25358 0.11006,0.62044 3e-5,0.36705 -0.0124,0.43725 -0.11446,0.64502 -0.19885,0.40492 -0.62148,0.69315 -1.20352,0.82078 -0.3428,0.0752 -1.24166,0.0639 -1.62343,-0.0203 -0.45495,-0.10039 -1.00181,-0.29248 -1.47596,-0.51843 l -0.42218,-0.20119 v 0.6341 0.63409 l 0.32876,0.10385 c 0.4758,0.15031 1.08693,0.29834 1.44232,0.34937 0.1699,0.0244 0.35525,0.0525 0.41188,0.0624 0.25954,0.0453 1.47675,-0.017 1.83287,-0.0938 z m -18.39042,-1.90095 v -1.85346 l 0.75168,7.2e-4 c 1.09909,0.001 1.45393,0.0953 1.82867,0.48557 0.26165,0.27251 0.49727,0.68143 1.17342,2.03647 l 0.59088,1.18416 h 0.64905 c 0.35697,-10e-6 0.64904,-0.0103 0.64904,-0.023 0,-0.0615 -1.38249,-2.76002 -1.56667,-3.05805 -0.24349,-0.39401 -0.62713,-0.82154 -0.84011,-0.93624 -0.0808,-0.0435 -0.1465,-0.0888 -0.14605,-0.10062 4.5e-4,-0.0118 0.12637,-0.0805 0.27982,-0.15253 0.30847,-0.14484 0.63615,-0.43956 0.80044,-0.71993 0.20691,-0.35308 0.29634,-0.74691 0.29313,-1.29084 -0.006,-1.05405 -0.36649,-1.73924 -1.12368,-2.13713 -0.59787,-0.31417 -0.70622,-0.32785 -2.7321,-0.34496 l -1.80198,-0.0152 v 4.38923 4.38923 h 0.59723 0.59723 z m 12.80945,1.33861 v -0.51485 h -2.22415 -2.22415 v -1.56515 -1.56514 h 2.07999 2.07999 v -0.51485 -0.51485 h -2.07999 -2.07999 v -1.27682 -1.27683 h 2.16237 2.16236 v -0.51485 -0.51485 h -2.78018 -2.78019 v 4.38652 4.38651 h 2.84197 2.84197 z m 12.6447,-3.35682 v -3.87167 h 1.54455 1.54455 v -0.51485 -0.51485 h -3.70692 -3.70692 v 0.51485 0.51485 h 1.54455 1.54455 v 3.87167 3.87166 h 0.61782 0.61782 z"
+         id="path49151"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60701)" />
+      <path
+         style="fill:#5e6a71;fill-opacity:1;stroke-width:0.041188"
+         d="m 305.12035,140.83505 c -1.42717,-2.00868 -2.84073,-4.00065 -3.14127,-4.42659 l -0.54642,-0.77444 0.13817,-0.19669 c 0.076,-0.10818 1.48767,-2.11039 3.13707,-4.44935 l 2.9989,-4.25265 h 23.84172 23.84172 v 8.876 8.87601 l -23.83752,-9e-5 -23.83753,-8e-5 -2.59484,-3.65215 z m 49.5595,-5.23402 0.0104,-8.20669 h -23.30527 -23.30527 l -2.89037,4.09471 c -1.5897,2.25209 -2.89037,4.11616 -2.89037,4.14238 0,0.0511 5.71289,8.11468 5.78503,8.1654 0.0237,0.0166 10.5151,0.0259 23.31426,0.0206 l 23.27119,-0.01 z"
+         id="path54026"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60697)" />
+      <path
+         style="fill:#ffffff;fill-opacity:1;stroke-width:0.116497"
+         d="m 151.27151,141.5739 c -0.80092,-1.13346 -2.08979,-2.95668 -2.86417,-4.05158 l -1.40795,-1.99073 0.25302,-0.34309 c 0.13917,-0.1887 1.42247,-2.00754 2.85179,-4.04187 l 2.59876,-3.69878 h 23.22444 23.22443 v 8.09655 8.09655 l -23.21205,-0.003 -23.21206,-0.003 z m 27.50437,-1.956 c 1.1133,-0.3928 1.61991,-1.13291 1.61991,-2.36657 0,-1.56133 -0.76954,-2.2893 -2.91243,-2.75509 -1.6059,-0.34908 -1.98045,-0.60836 -1.98045,-1.37096 0,-0.34239 0.0808,-0.53382 0.33108,-0.78407 0.63872,-0.63871 1.73088,-0.7162 3.18587,-0.22605 0.36828,0.12406 0.72367,0.22557 0.78977,0.22557 0.0678,0 0.12017,-0.27526 0.12017,-0.6314 v -0.63139 l -0.80935,-0.18409 c -1.04889,-0.23856 -2.60092,-0.23847 -3.30137,1.9e-4 -0.81156,0.27652 -1.43659,0.96219 -1.60158,1.75693 -0.14511,0.69897 -0.0293,1.44993 0.30156,1.95482 0.2656,0.40536 1.19477,0.85185 2.33611,1.12256 1.8003,0.42702 2.14266,0.67846 2.14266,1.57372 0,0.57618 -0.3286,1.05099 -0.88955,1.28537 -0.78583,0.32834 -2.33171,0.1325 -3.44997,-0.43707 l -0.43687,-0.22252 v 0.70424 c 0,0.80635 -0.0206,0.7898 1.36503,1.09928 0.9109,0.20346 2.44591,0.14885 3.18941,-0.11347 z m -18.88358,-1.75515 v -1.8173 l 1.04097,0.0407 c 0.97132,0.038 1.06531,0.064 1.40476,0.38939 0.20008,0.19176 0.68838,0.991 1.0851,1.77608 l 0.72131,1.4274 0.73211,4.7e-4 0.7321,4.7e-4 -0.8132,-1.60266 c -0.44726,-0.88146 -0.99978,-1.82064 -1.22783,-2.08706 l -0.41464,-0.48441 0.39596,-0.24472 c 0.55668,-0.34405 0.81508,-0.94633 0.80906,-1.88576 -0.006,-0.92118 -0.26227,-1.51371 -0.86136,-1.99085 -0.5313,-0.42315 -1.33214,-0.55677 -3.34222,-0.55765 l -1.54359,-6e-4 v 4.42689 4.42689 h 0.64073 0.64074 z m 12.81468,1.29306 v -0.52424 h -2.21344 -2.21345 v -1.51446 -1.51446 h 2.09695 2.09695 v -0.58249 -0.58248 h -2.09695 -2.09695 v -1.22322 -1.22322 h 2.1552 2.15519 v -0.58456 -0.58456 l -2.82505,0.0312 -2.82506,0.0312 -0.0305,4.39777 -0.0305,4.39777 h 2.91383 2.91383 z m 12.69819,-3.32017 v -3.8444 h 1.51446 1.51446 v -0.58249 -0.58248 h -3.72791 -3.7279 v 0.58248 0.58249 h 1.51446 1.51446 v 3.8444 3.84441 h 0.69898 0.69899 z"
+         id="path54381"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60693)" />
+      <path
+         style="fill:#f0f1f1;fill-opacity:1;stroke-width:0.164752"
+         d="m 150.00652,139.62291 c -1.49512,-2.12602 -2.7184,-3.96474 -2.7184,-4.08604 0,-0.1213 1.22328,-1.97102 2.7184,-4.11049 l 2.71841,-3.88993 23.18881,-0.009 23.18882,-0.009 v 7.99046 7.99046 l -23.18882,-0.005 -23.18881,-0.005 z m 29.09805,-0.0639 c 1.11701,-0.50725 1.52891,-1.41935 1.30756,-2.89543 -0.14766,-0.98464 -0.81293,-1.63717 -2.03332,-1.99436 -2.22759,-0.65198 -2.51775,-0.76912 -2.68658,-1.08459 -0.24965,-0.46648 -0.0637,-1.03434 0.44873,-1.37008 0.55104,-0.36105 1.67651,-0.36896 2.85059,-0.02 0.49217,0.14627 0.91847,0.26594 0.94733,0.26594 0.0289,0 0.0525,-0.32748 0.0525,-0.72773 v -0.72773 l -0.83187,-0.1784 c -1.1475,-0.2461 -2.86016,-0.22347 -3.50585,0.0463 -1.55434,0.64944 -2.11573,2.94348 -0.97829,3.99762 0.23654,0.21923 1.06236,0.59169 1.84263,0.83108 2.14843,0.65913 2.32012,0.77103 2.32012,1.51216 0,1.47736 -1.76738,1.87645 -3.99692,0.90255 -0.34072,-0.14884 -0.66666,-0.22345 -0.7243,-0.16581 -0.0576,0.0576 -0.0381,0.42141 0.0435,0.80835 0.12737,0.60428 0.22957,0.73129 0.72431,0.90019 0.94021,0.32097 3.42236,0.26212 4.2199,-0.10006 z m -19.21294,-1.6779 v -1.82845 l 1.0297,0.11621 c 0.56633,0.0639 1.15649,0.22829 1.31146,0.36528 0.15497,0.13699 0.62182,0.9075 1.03745,1.71224 l 0.7557,1.46316 h 0.816 c 0.4488,0 0.816,-0.0495 0.816,-0.11002 0,-0.19458 -1.50546,-2.92002 -1.91197,-3.46138 l -0.3931,-0.5235 0.46605,-0.46605 c 0.81682,-0.81682 0.82586,-2.54094 0.0187,-3.56708 -0.46413,-0.59004 -1.24843,-0.76657 -3.41055,-0.76765 l -1.85346,-9.2e-4 v 4.4483 4.4483 h 0.65901 0.65901 z m 12.85064,1.25181 v -0.57663 h -2.22415 -2.22415 v -1.40039 -1.40039 h 2.14177 2.14178 v -0.65901 -0.65901 h -2.14178 -2.14177 v -1.15326 -1.15326 h 2.22415 2.22415 v -0.65901 -0.65901 h -2.96553 -2.96554 v 4.4483 4.4483 h 2.96554 2.96553 z m 12.68589,-3.21266 v -3.78929 h 1.56514 1.56514 v -0.65901 -0.65901 h -3.78929 -3.78929 v 0.65901 0.65901 h 1.48277 1.48276 v 3.78929 3.78929 h 0.74139 0.74138 z"
+         id="path54383"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60689)" />
+      <path
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:0.0582486"
+         d="m 149.29106,140.13447 c -1.66591,-2.34503 -3.0761,-4.33989 -3.13375,-4.43303 -0.12884,-0.20815 -0.52452,0.39541 3.18945,-4.86503 l 2.96812,-4.20404 h 23.81165 23.81165 v 8.88291 8.8829 l -23.8091,-1e-5 -23.8091,-1e-5 z m 50.00639,-4.55618 c 0,-4.4665 -0.0159,-8.16237 -0.0354,-8.21305 -0.0322,-0.0838 -2.13894,-0.0921 -23.29453,-0.0921 h -23.25917 l -0.17964,0.22682 c -0.0988,0.12475 -0.17964,0.25581 -0.17964,0.29124 0,0.0354 -0.0221,0.0644 -0.0491,0.0644 -0.027,0 -0.0985,0.0917 -0.15886,0.20387 -0.0604,0.11213 -0.12592,0.20387 -0.14563,0.20387 -0.0197,0 -0.0852,0.0917 -0.14562,0.20387 -0.0604,0.11213 -0.13004,0.20387 -0.15479,0.20387 -0.0248,0 -0.045,0.0393 -0.045,0.0874 0,0.0481 -0.0245,0.0874 -0.0543,0.0874 -0.0299,0 -0.069,0.0459 -0.087,0.10194 -0.018,0.0561 -0.0984,0.17808 -0.17864,0.27115 -0.0803,0.0931 -0.27046,0.3546 -0.42265,0.58119 -0.15219,0.22659 -0.32258,0.46337 -0.37865,0.52619 -0.0561,0.0628 -0.10193,0.13361 -0.10193,0.15732 0,0.0237 -0.0352,0.0783 -0.0783,0.12137 -0.043,0.043 -0.15181,0.19564 -0.2417,0.33909 -0.0899,0.14345 -0.17933,0.27393 -0.19875,0.28995 -0.0834,0.0688 -0.35501,0.46319 -0.35501,0.51549 0,0.0315 -0.0197,0.066 -0.0437,0.0767 -0.0679,0.0302 -0.36405,0.44161 -0.36405,0.50579 0,0.0315 -0.0222,0.066 -0.0494,0.0767 -0.0683,0.0269 -1.05736,1.4269 -1.05736,1.49674 0,0.0311 -0.0231,0.0653 -0.0514,0.076 -0.0641,0.0242 -1.11356,1.50651 -1.11356,1.57285 0,0.0266 0.0524,0.10082 0.1165,0.1649 0.0641,0.0641 0.1165,0.13249 0.1165,0.15204 0,0.0537 0.59474,0.86441 0.65048,0.88672 0.0267,0.0107 0.0485,0.0463 0.0485,0.0792 0,0.0706 0.57986,0.88271 0.65049,0.91099 0.0267,0.0107 0.0485,0.0503 0.0485,0.088 0,0.0377 0.0655,0.13923 0.14563,0.2256 0.0801,0.0864 0.14562,0.17305 0.14562,0.1926 0,0.0317 0.27462,0.42007 0.34916,0.49377 0.0162,0.016 0.15418,0.21541 0.30664,0.44309 0.15245,0.22769 0.32938,0.47416 0.39317,0.54771 0.0638,0.0736 0.116,0.15101 0.116,0.1721 0,0.0211 0.0472,0.0892 0.10485,0.15127 0.0577,0.0621 0.15506,0.19811 0.21643,0.30223 0.0614,0.10412 0.12618,0.18931 0.14402,0.18931 0.0178,0 0.0819,0.0917 0.14223,0.20387 0.0604,0.11212 0.13188,0.20387 0.15887,0.20387 0.027,0 0.0491,0.028 0.0491,0.0621 0,0.0516 0.24627,0.41862 0.34916,0.52035 0.0162,0.016 0.12984,0.17824 0.25252,0.36049 0.12269,0.18225 0.27341,0.3894 0.33493,0.46034 0.0615,0.0709 0.11186,0.14623 0.11186,0.16733 0,0.0211 0.0493,0.0912 0.10946,0.15584 0.0602,0.0646 0.18225,0.23339 0.27121,0.37504 0.089,0.14165 0.2263,0.34273 0.30519,0.44685 l 0.14345,0.18931 h 23.32163 23.32164 z"
+         id="path55282"
+         transform="translate(122.65281,188.48619)"
+         clip-path="url(#clipPath60685)" />
+      <rect
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:5.96092;stroke-linejoin:round;stroke-dasharray:none"
+         id="rect60677"
+         width="362.13153"
+         height="625.22229"
+         x="156.51996"
+         y="190.24973"
+         clip-path="url(#clipPath60681)" />
+    </g>
+  </g>
+</svg>
diff --git a/doc/images/zone-sync2.png b/doc/images/zone-sync2.png
new file mode 100644
index 000000000..67963dcd1
--- /dev/null
+++ b/doc/images/zone-sync2.png
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 000000000..8edc2cb09
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,125 @@
+=================
+ Welcome to Ceph
+=================
+
+Ceph delivers **object, block, and file storage in one unified system**.
+
+.. warning::
+
+   :ref:`If this is your first time using Ceph, read the "Basic Workflow"
+   page in the Ceph Developer Guide to learn how to contribute to the
+   Ceph project. (Click anywhere in this paragraph to read the "Basic
+   Workflow" page of the Ceph Developer Guide.) <basic workflow dev guide>`.
+
+.. note::
+
+   :ref:`If you want to make a commit to the documentation but you don't
+   know how to get started, read the "Documenting Ceph" page. (Click anywhere
+   in this paragraph to read the "Documenting Ceph" page.) <documenting_ceph>`.
+
+.. container:: columns-3
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Ceph Object Store</h3>
+
+      - RESTful Interface
+      - S3- and Swift-compliant APIs
+      - S3-style subdomains
+      - Unified S3/Swift namespace
+      - User management
+      - Usage tracking
+      - Striped objects
+      - Cloud solution integration
+      - Multi-site deployment
+      - Multi-site replication
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Ceph Block Device</h3>
+
+      - Thin-provisioned
+      - Images up to 16 exabytes
+      - Configurable striping
+      - In-memory caching
+      - Snapshots
+      - Copy-on-write cloning
+      - Kernel driver support
+      - KVM/libvirt support
+      - Back-end for cloud solutions
+      - Incremental backup
+      - Disaster recovery (multisite asynchronous replication)
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Ceph File System</h3>
+
+      - POSIX-compliant semantics
+      - Separates metadata from data
+      - Dynamic rebalancing
+      - Subdirectory snapshots
+      - Configurable striping
+      - Kernel driver support
+      - FUSE support
+      - NFS/CIFS deployable
+      - Use with Hadoop (replace HDFS)
+
+.. container:: columns-3
+
+   .. container:: column
+
+      See `Ceph Object Store`_ for additional details.
+
+   .. container:: column
+
+      See `Ceph Block Device`_ for additional details.
+
+   .. container:: column
+
+      See `Ceph File System`_ for additional details.
+
+Ceph is highly reliable, easy to manage, and free. The power of Ceph
+can transform your company's IT infrastructure and your ability to manage vast
+amounts of data. To try Ceph, see our `Getting Started`_ guides. To learn more
+about Ceph, see our `Architecture`_ section.
+
+
+
+.. _Ceph Object Store: radosgw
+.. _Ceph Block Device: rbd
+.. _Ceph File System: cephfs
+.. _Getting Started: install
+.. _Architecture: architecture
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+
+   start/intro
+   install/index
+   cephadm/index
+   rados/index
+   cephfs/index
+   rbd/index
+   radosgw/index
+   mgr/index
+   mgr/dashboard
+   monitoring/index
+   api/index
+   architecture
+   Developer Guide <dev/developer_guide/index>
+   dev/internals
+   governance
+   foundation
+   ceph-volume/index
+   releases/general
+   releases/index
+   security/index
+   Glossary <glossary>
+   Tracing <jaegertracing/index>
diff --git a/doc/install/build-ceph.rst b/doc/install/build-ceph.rst
new file mode 100644
index 000000000..28f1163ff
--- /dev/null
+++ b/doc/install/build-ceph.rst
@@ -0,0 +1,109 @@
+============
+ Build Ceph
+============
+
+You can get Ceph software by retrieving Ceph source code and building it yourself.
+To build Ceph, you need to set up a development environment, compile Ceph,
+and then either install in user space or build packages and install the packages.
+
+Build Prerequisites
+===================
+
+
+.. tip:: Check this section to see if there are specific prerequisites for your
+   Linux/Unix distribution.
+
+A debug build of Ceph may take around 40 gigabytes. If you want to build Ceph in
+a virtual machine (VM) please make sure total disk space on the VM is at least
+60 gigabytes.
+
+Please also be aware that some distributions of Linux, like CentOS, use Linux
+Volume Manager (LVM) for the default installation. LVM may reserve a large
+portion of disk space of a typical sized virtual disk for the operating system.
+
+Before you can build Ceph source code, you need to install several libraries
+and tools::
+
+	./install-deps.sh
+
+.. note:: Some distributions that support Google's memory profiler tool may use
+   a different package name (e.g., ``libgoogle-perftools4``).
+
+.. _build-ceph:
+
+Build Ceph
+==========
+
+Ceph is built using cmake. To build Ceph, navigate to your cloned Ceph
+repository and execute the following::
+
+    cd ceph
+    ./do_cmake.sh
+    cd build
+    ninja
+
+See `Installing a Build`_ to install a build in user space and `Ceph README.md`_
+doc for more details on build.
+
+Build Ceph Packages
+===================
+
+To build packages, you must clone the `Ceph`_ repository. You can create 
+installation packages from the latest code using ``dpkg-buildpackage`` for 
+Debian/Ubuntu or ``rpmbuild`` for the RPM Package Manager.
+
+.. tip:: When building on a multi-core CPU, use the ``-j`` and the number of 
+   cores * 2. For example, use ``-j4`` for a dual-core processor to accelerate 
+   the build.
+
+
+Advanced Package Tool (APT)
+---------------------------
+
+To create ``.deb`` packages for Debian/Ubuntu, ensure that you have cloned the 
+`Ceph`_ repository, installed the `Build Prerequisites`_ and installed 
+``debhelper``::
+
+	sudo apt-get install debhelper
+
+Once you have installed debhelper, you can build the packages::
+
+	sudo dpkg-buildpackage
+
+For multi-processor CPUs use the ``-j`` option to accelerate the build.
+
+
+RPM Package Manager
+-------------------
+
+To create ``.rpm`` packages, ensure that you have cloned the `Ceph`_ repository,
+installed the `Build Prerequisites`_ and installed ``rpm-build`` and 
+``rpmdevtools``::
+
+	yum install rpm-build rpmdevtools
+
+Once you have installed the tools, setup an RPM compilation environment::
+
+	rpmdev-setuptree
+
+Fetch the source tarball for the RPM compilation environment::
+
+	wget -P ~/rpmbuild/SOURCES/ https://download.ceph.com/tarballs/ceph-<version>.tar.bz2
+
+Or from the EU mirror::
+
+	wget -P ~/rpmbuild/SOURCES/ http://eu.ceph.com/tarballs/ceph-<version>.tar.bz2
+
+Extract the specfile::
+
+    tar --strip-components=1 -C ~/rpmbuild/SPECS/ --no-anchored -xvjf ~/rpmbuild/SOURCES/ceph-<version>.tar.bz2 "ceph.spec"
+
+Build the RPM packages::
+
+	rpmbuild -ba ~/rpmbuild/SPECS/ceph.spec
+
+For multi-processor CPUs use the ``-j`` option to accelerate the build.
+
+.. _Ceph: ../clone-source
+.. _Installing a Build: ../install-storage-cluster#installing-a-build
+.. _Ceph README.md: https://github.com/ceph/ceph#building-ceph
diff --git a/doc/install/clone-source.rst b/doc/install/clone-source.rst
new file mode 100644
index 000000000..2d09ef9eb
--- /dev/null
+++ b/doc/install/clone-source.rst
@@ -0,0 +1,195 @@
+=========================================
+ Cloning the Ceph Source Code Repository
+=========================================
+
+To clone a Ceph branch of the Ceph source code, go to `github Ceph
+Repository`_, select a branch (``main`` by default), and click the **Download
+ZIP** button.
+
+.. _github Ceph Repository: https://github.com/ceph/ceph
+
+To clone the entire git repository, :ref:`install <install-git>` and configure
+``git``.
+
+.. _install-git:
+
+Install Git
+===========
+
+To install ``git`` on Debian/Ubuntu, run the following command:
+
+.. prompt:: bash $
+
+   sudo apt-get install git
+
+
+To install ``git`` on CentOS/RHEL, run the following command:
+
+.. prompt:: bash $
+
+   sudo yum install git
+
+
+You must have a ``github`` account. If you do not have a ``github``
+account, go to `github.com`_ and register.  Follow the directions for setting
+up git at `Set Up Git`_.
+
+.. _github.com: https://github.com
+.. _Set Up Git: https://help.github.com/linux-set-up-git
+
+
+Add SSH Keys (Optional)
+=======================
+
+To commit code to Ceph or to clone the respository by using SSH
+(``git@github.com:ceph/ceph.git``), you must generate SSH keys for github. 
+
+.. tip:: If you want only to clone the repository, you can 
+   use ``git clone --recursive https://github.com/ceph/ceph.git`` 
+   without generating SSH keys.
+
+To generate SSH keys for ``github``, run the following command:
+
+.. prompt:: bash $
+
+   ssh-keygen
+
+To print the SSH key that you just generated and that you will add to your
+``github`` account, use the ``cat`` command. (The following example assumes you
+used the default file path.):
+
+.. prompt:: bash $
+
+   cat .ssh/id_rsa.pub
+
+Copy the public key.
+
+Go to your ``github`` account, click "Account Settings" (represented by the
+'tools' icon), and click "SSH Keys" on the left side navbar.
+
+Click "Add SSH key" in the "SSH Keys" list, enter a name for the key, paste the
+key you generated, and press the "Add key" button.
+
+
+Clone the Source
+================
+
+To clone the Ceph source code repository, run the following command:
+
+.. prompt:: bash $
+
+   git clone --recursive https://github.com/ceph/ceph.git
+
+After ``git clone`` has run, you should have a full copy of the Ceph 
+repository.
+
+.. tip:: Make sure you maintain the latest copies of the submodules included in
+   the repository. Running ``git status`` will tell you whether the submodules
+   are out of date. See :ref:`update-submodules` for more information.
+
+
+.. prompt:: bash $
+
+   cd ceph
+   git status
+
+.. _update-submodules:
+
+Updating Submodules
+-------------------
+
+#. Determine whether your submodules are out of date:
+
+   .. prompt:: bash $
+
+      git status
+
+   A. If your submodules are up to date 
+         If your submodules are up to date, the following console output will
+         appear: 
+
+         ::
+   
+           On branch main
+           Your branch is up to date with 'origin/main'.
+           
+           nothing to commit, working tree clean
+   
+         If you see this console output, then your submodules are up to date.
+         You do not need this procedure.
+
+
+   B. If your submodules are not up to date 
+         If your submodules are not up to date, you will see a message that
+         includes a list of "untracked files". The example here shows such a
+         list, which was generated from a real situation in which the
+         submodules were no longer current. Your list of files will not be the
+         same as this list of files, but this list is provided as an example.
+         If in your case any untracked files are listed, then you should
+         continue to the next step of this procedure.
+
+         ::
+
+            On branch main
+            Your branch is up to date with 'origin/main'.
+            
+            Untracked files:
+              (use "git add <file>..." to include in what will be committed)
+            src/pybind/cephfs/build/
+            src/pybind/cephfs/cephfs.c
+            src/pybind/cephfs/cephfs.egg-info/
+            src/pybind/rados/build/
+            src/pybind/rados/rados.c
+            src/pybind/rados/rados.egg-info/
+            src/pybind/rbd/build/
+            src/pybind/rbd/rbd.c
+            src/pybind/rbd/rbd.egg-info/
+            src/pybind/rgw/build/
+            src/pybind/rgw/rgw.c
+            src/pybind/rgw/rgw.egg-info/
+            
+            nothing added to commit but untracked files present (use "git add" to track)
+
+#. If your submodules are out of date, run the following commands:
+
+   .. prompt:: bash $
+
+      git submodule update --force --init --recursive
+      git clean -fdx
+      git submodule foreach git clean -fdx
+
+   If you still have problems with a submodule directory, use ``rm -rf
+   [directory name]`` to remove the directory. Then run ``git submodule update
+   --init --recursive`` again.
+
+#. Run ``git status`` again:
+
+   .. prompt:: bash $
+
+      git status
+   
+   Your submodules are up to date if you see the following message:
+
+   ::
+
+     On branch main
+     Your branch is up to date with 'origin/main'.
+     
+     nothing to commit, working tree clean
+
+Choose a Branch
+===============
+
+Once you clone the source code and submodules, your Ceph repository 
+will be on the ``main`` branch by default, which is the unstable 
+development branch. You may choose other branches too.
+
+- ``main``: The unstable development branch.
+- ``stable-release-name``: The name of the stable, `Active Releases`_. e.g. ``Pacific``
+- ``next``: The release candidate branch.
+
+::
+
+	git checkout main
+
+.. _Active Releases: https://docs.ceph.com/en/latest/releases/#active-releases
diff --git a/doc/install/containers.rst b/doc/install/containers.rst
new file mode 100644
index 000000000..41bf59c7d
--- /dev/null
+++ b/doc/install/containers.rst
@@ -0,0 +1,113 @@
+.. _containers:
+
+Ceph Container Images
+=====================
+
+.. important::
+
+   Using the ``:latest`` tag is discouraged. If you use the ``:latest`` 
+   tag, there is no guarantee that the same image will be on each of 
+   your hosts.  Under these conditions, upgrades might not work 
+   properly. Remember that ``:latest`` is a relative tag, and a moving
+   target.
+
+   Instead of the ``:latest`` tag, use explicit tags or image IDs. For
+   example:
+
+   ``podman pull ceph/ceph:v15.2.0``
+
+Official Releases
+-----------------
+
+Ceph Container images are available from Quay:
+
+  https://quay.io/repository/ceph/ceph
+  https://hub.docker.com/r/ceph
+
+ceph/ceph
+^^^^^^^^^
+
+- General purpose Ceph container with all necessary daemons and
+  dependencies installed.
+
++----------------------+--------------------------------------------------------------+
+| Tag                  | Meaning                                                      |
++----------------------+--------------------------------------------------------------+
+| vRELNUM              | Latest release in this series (e.g., *v14* = Nautilus)       |
++----------------------+--------------------------------------------------------------+
+| vRELNUM.2            | Latest *stable* release in this stable series (e.g., *v14.2*)|
++----------------------+--------------------------------------------------------------+
+| vRELNUM.Y.Z          | A specific release (e.g., *v14.2.4*)                         |
++----------------------+--------------------------------------------------------------+
+| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., *v14.2.4-20191203*)                  |
++----------------------+--------------------------------------------------------------+
+
+Legacy container images
+-----------------------
+
+Legacy container images are available from Docker Hub at::
+  
+  https://hub.docker.com/r/ceph
+
+ceph/daemon-base
+^^^^^^^^^^^^^^^^
+
+- General purpose Ceph container with all necessary daemons and
+  dependencies installed.
+- Basically the same as *ceph/ceph*, but with different tags.
+- Note that all of the *-devel* tags (and the *latest-master* tag) are based on
+  unreleased and generally untested packages from https://shaman.ceph.com.
+
+:note: This image will soon become an alias to *ceph/ceph*.
+
++------------------------+---------------------------------------------------------+
+| Tag                    | Meaning                                                 |
++------------------------+---------------------------------------------------------+
+| latest-master          | Build of master branch a last ceph-container.git update |
++------------------------+---------------------------------------------------------+
+| latest-master-devel    | Daily build of the master branch                        |
++------------------------+---------------------------------------------------------+
+| latest-RELEASE-devel   | Daily build of the *RELEASE* (e.g., nautilus) branch    |
++------------------------+---------------------------------------------------------+
+
+
+ceph/daemon
+^^^^^^^^^^^
+
+- *ceph/daemon-base* plus a collection of BASH scripts that are used
+  by ceph-nano and ceph-ansible to manage a Ceph cluster.
+
++------------------------+---------------------------------------------------------+
+| Tag                    | Meaning                                                 |
++------------------------+---------------------------------------------------------+
+| latest-master          | Build of master branch a last ceph-container.git update |
++------------------------+---------------------------------------------------------+
+| latest-master-devel    | Daily build of the master branch                        |
++------------------------+---------------------------------------------------------+
+| latest-RELEASE-devel   | Daily build of the *RELEASE* (e.g., nautilus) branch    |
++------------------------+---------------------------------------------------------+
+
+
+Development builds
+------------------
+
+We automatically build container images for development ``wip-*``
+branches in the ceph-ci.git repositories and push them to Quay at::
+
+  https://quay.ceph.io/organization/ceph-ci
+
+ceph-ci/ceph
+^^^^^^^^^^^^
+
+- This is analogous to the ceph/ceph image above
+- TODO: remove the ``wip-*`` limitation and also build ceph.git branches.
+
++------------------------------------+------------------------------------------------------+
+| Tag                                | Meaning                                              |
++------------------------------------+------------------------------------------------------+
+| BRANCH                             | Latest build of a given GIT branch (e.g., *wip-foo*) |
++------------------------------------+------------------------------------------------------+
+| BRANCH-SHORTSHA1-BASEOS-ARCH-devel | A specific build of a branch                         |
++------------------------------------+------------------------------------------------------+
+| SHA1                               | A specific build                                     |
++------------------------------------+------------------------------------------------------+
diff --git a/doc/install/get-packages.rst b/doc/install/get-packages.rst
new file mode 100644
index 000000000..d6edc177b
--- /dev/null
+++ b/doc/install/get-packages.rst
@@ -0,0 +1,382 @@
+.. _packages:
+
+==============
+ Get Packages
+==============
+
+To install Ceph and other enabling software, you need to retrieve packages from
+the Ceph repository. 
+
+There are three ways to get packages:
+
+- **Cephadm:** Cephadm can configure your Ceph repositories for you
+  based on a release name or a specific Ceph version.  Each
+  :term:`Ceph Node` in your cluster must have internet access.
+
+- **Configure Repositories Manually:** You can manually configure your
+  package management tool to retrieve Ceph packages and all enabling
+  software.  Each :term:`Ceph Node` in your cluster must have internet
+  access.
+
+- **Download Packages Manually:** Downloading packages manually is a convenient
+  way to install Ceph if your environment does not allow a :term:`Ceph Node` to
+  access the internet.
+
+Install packages with cephadm
+=============================
+
+#. Download cephadm
+
+.. prompt:: bash $
+   :substitutions:
+
+   curl --silent --remote-name --location https://download.ceph.com/rpm-|stable-release|/el9/noarch/cephadm
+   chmod +x cephadm
+
+#. Configure the Ceph repository based on the release name::
+
+     ./cephadm add-repo --release |stable-release|
+
+   For Octopus (15.2.0) and later releases, you can also specify a specific
+   version::
+
+     ./cephadm add-repo --version 15.2.1
+
+   For development packages, you can specify a specific branch name::
+
+     ./cephadm add-repo --dev my-branch
+
+#. Install the appropriate packages.  You can install them using your
+   package management tool (e.g., APT, Yum) directly, or you can
+   use the cephadm wrapper command.  For example::
+
+     ./cephadm install ceph-common
+   
+     
+Configure Repositories Manually
+===============================
+
+All Ceph deployments require Ceph packages (except for development). You should
+also add keys and recommended packages.
+
+- **Keys: (Recommended)** Whether you add repositories or download packages
+  manually, you should download keys to verify the packages. If you do not get
+  the keys, you may encounter security warnings.
+
+- **Ceph: (Required)** All Ceph deployments require Ceph release packages,
+  except for deployments that use development packages (development, QA, and
+  bleeding edge deployments only).
+
+- **Ceph Development: (Optional)** If you are developing for Ceph, testing Ceph
+  development builds, or if you want features from the bleeding edge of Ceph
+  development, you may get Ceph development packages.
+
+
+
+Add Keys
+--------
+
+Add a key to your system's list of trusted keys to avoid a security warning. For
+major releases (e.g., ``luminous``, ``mimic``, ``nautilus``) and development releases
+(``release-name-rc1``, ``release-name-rc2``), use the ``release.asc`` key.
+
+
+APT
+~~~
+
+To install the ``release.asc`` key, execute the following::
+
+	wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
+
+
+RPM
+~~~
+
+To install the ``release.asc`` key, execute the following::
+
+	sudo rpm --import 'https://download.ceph.com/keys/release.asc'
+
+Ceph Release Packages
+---------------------
+
+Release repositories use the ``release.asc`` key to verify packages.
+To install Ceph packages with the Advanced Package Tool (APT) or
+Yellowdog Updater, Modified (YUM), you must add Ceph repositories.
+
+You may find releases for Debian/Ubuntu (installed with APT) at::
+
+	https://download.ceph.com/debian-{release-name}
+
+You may find releases for CentOS/RHEL and others (installed with YUM) at::
+
+	https://download.ceph.com/rpm-{release-name}
+
+For Octopus and later releases, you can also configure a repository for a
+specific version ``x.y.z``.  For Debian/Ubuntu packages::
+
+  https://download.ceph.com/debian-{version}
+
+For RPMs::
+
+  https://download.ceph.com/rpm-{version}
+
+The major releases of Ceph are summarized at: `Releases`_
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+Debian Packages
+~~~~~~~~~~~~~~~
+
+Add a Ceph package repository to your system's list of APT sources. For newer
+versions of Debian/Ubuntu, call ``lsb_release -sc`` on the command line to
+get the short codename, and replace ``{codename}`` in the following command.
+
+.. prompt:: bash $
+   :substitutions:
+
+   sudo apt-add-repository 'deb https://download.ceph.com/debian-|stable-release|/ {codename} main'
+
+For early Linux distributions, you may execute the following command
+
+.. prompt:: bash $
+   :substitutions:
+
+   echo deb https://download.ceph.com/debian-|stable-release|/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+For earlier Ceph releases, replace ``{release-name}`` with the name  with the
+name of the Ceph release. You may call ``lsb_release -sc`` on the command  line
+to get the short codename, and replace ``{codename}`` in the following command.
+
+.. prompt:: bash $
+
+   sudo apt-add-repository 'deb https://download.ceph.com/debian-{release-name}/ {codename} main'
+
+For older Linux distributions, replace ``{release-name}`` with the name of the
+release
+
+.. prompt:: bash $
+
+	echo deb https://download.ceph.com/debian-{release-name}/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+For development release packages, add our package repository to your system's
+list of APT sources.  See `the testing Debian repository`_ for a complete list
+of Debian and Ubuntu releases supported.
+
+.. prompt:: bash $
+
+   echo deb https://download.ceph.com/debian-testing/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+
+RPM Packages
+~~~~~~~~~~~~
+
+RHEL
+^^^^
+
+For major releases, you may add a Ceph entry to the ``/etc/yum.repos.d``
+directory. Create a ``ceph.repo`` file. In the example below, replace
+``{ceph-release}`` with  a major release of Ceph (e.g., ``|stable-release|``)
+and ``{distro}`` with your Linux distribution (e.g., ``el8``, etc.).  You
+may view https://download.ceph.com/rpm-{ceph-release}/ directory to see which
+distributions Ceph supports. Some Ceph packages (e.g., EPEL) must take priority
+over standard packages, so you must ensure that you set
+``priority=2``.
+
+.. code-block:: ini
+
+	[ceph]
+	name=Ceph packages for $basearch
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-noarch]
+	name=Ceph noarch packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/noarch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-source]
+	name=Ceph source packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/SRPMS
+	enabled=0
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+
+For specific packages, you may retrieve them by downloading the release package
+by name. Our development process generates a new release of Ceph every 3-4
+weeks. These packages are faster-moving than the major releases.  Development
+packages have new features integrated quickly, while still undergoing several
+weeks of QA prior to release.
+
+The repository package installs the repository details on your local system for
+use with ``yum``. Replace ``{distro}`` with your Linux distribution, and
+``{release}`` with the specific release of Ceph
+
+.. prompt:: bash $
+
+    su -c 'rpm -Uvh https://download.ceph.com/rpms/{distro}/x86_64/ceph-{release}.el8.noarch.rpm'
+
+You can download the RPMs directly from
+
+.. code-block:: none
+
+   https://download.ceph.com/rpm-testing
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+openSUSE Leap 15.1
+^^^^^^^^^^^^^^^^^^
+
+You need to add the Ceph package repository to your list of zypper sources. This can be done with the following command
+
+.. code-block:: bash
+
+    zypper ar https://download.opensuse.org/repositories/filesystems:/ceph/openSUSE_Leap_15.1/filesystems:ceph.repo
+
+openSUSE Tumbleweed
+^^^^^^^^^^^^^^^^^^^
+
+The newest major release of Ceph is already available through the normal Tumbleweed repositories.
+There's no need to add another package repository manually.
+
+
+Ceph Development Packages
+-------------------------
+
+If you are developing Ceph and need to deploy and test specific Ceph branches,
+ensure that you remove repository entries for major releases first.
+
+
+DEB Packages
+~~~~~~~~~~~~
+
+We automatically build Ubuntu packages for current development branches in the
+Ceph source code repository.  These packages are intended for developers and QA
+only.
+
+Add the package repository to your system's list of APT sources, but
+replace ``{BRANCH}`` with the branch you'd like to use (e.g.,
+wip-hack, master).  See `the shaman page`_ for a complete
+list of distributions we build.
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/{BRANCH}/latest/ubuntu/$(lsb_release -sc)/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+.. note:: If the repository is not ready an HTTP 504 will be returned
+
+The use of ``latest`` in the url, means it will figure out which is the last
+commit that has been built. Alternatively, a specific sha1 can be specified.
+For Ubuntu Xenial and the master branch of Ceph, it would look like
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/master/53e772a45fdf2d211c0c383106a66e1feedec8fd/ubuntu/xenial/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+
+.. warning:: Development repositories are no longer available after two weeks.
+
+RPM Packages
+~~~~~~~~~~~~
+
+For current development branches, you may add a Ceph entry to the
+``/etc/yum.repos.d`` directory. The `the shaman page`_ can be used to retrieve the full details
+of a repo file. It can be retrieved via an HTTP request, for example
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/{BRANCH}/latest/centos/8/repo/ | sudo tee /etc/yum.repos.d/shaman.repo
+
+The use of ``latest`` in the url, means it will figure out which is the last
+commit that has been built. Alternatively, a specific sha1 can be specified.
+For CentOS 8 and the master branch of Ceph, it would look like
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/master/488e6be0edff7eb18343fd5c7e2d7ed56435888f/centos/8/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+
+.. warning:: Development repositories are no longer available after two weeks.
+
+.. note:: If the repository is not ready an HTTP 504 will be returned
+
+Download Packages Manually
+--------------------------
+
+If you are attempting to install behind a firewall in an environment without internet
+access, you must retrieve the packages (mirrored with all the necessary dependencies)
+before attempting an install.
+
+Debian Packages
+~~~~~~~~~~~~~~~
+
+The repository package installs the repository details on your local system for
+use with ``apt``. Replace ``{release}`` with the latest Ceph release. Replace
+``{version}`` with the latest Ceph version number. Replace ``{distro}`` with
+your Linux distribution codename. Replace ``{arch}`` with the CPU architecture.
+
+.. prompt:: bash $
+
+	wget -q https://download.ceph.com/debian-{release}/pool/main/c/ceph/ceph_{version}{distro}_{arch}.deb
+
+
+RPM Packages
+~~~~~~~~~~~~
+
+Ceph requires additional third party libraries.
+To add the EPEL repository, execute the following
+
+.. prompt:: bash $
+
+   sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
+Packages are currently built for the RHEL/CentOS8 (``el8``) platforms.  The
+repository package installs the repository details on your local system for use
+with ``yum``. Replace ``{distro}`` with your distribution.
+
+.. prompt:: bash $
+   :substitutions:
+
+   su -c 'rpm -Uvh https://download.ceph.com/rpm-|stable-release|/{distro}/noarch/ceph-{version}.{distro}.noarch.rpm'
+
+For example, for CentOS 8  (``el8``)
+
+.. prompt:: bash $
+   :substitutions:
+
+   su -c 'rpm -Uvh https://download.ceph.com/rpm-|stable-release|/el8/noarch/ceph-release-1-0.el8.noarch.rpm'
+
+You can download the RPMs directly from
+
+.. code-block:: none
+   :substitutions:
+
+   https://download.ceph.com/rpm-|stable-release|
+
+
+For earlier Ceph releases, replace ``{release-name}`` with the name
+with the name of the Ceph release. You may call ``lsb_release -sc`` on the command
+line to get the short codename.
+
+.. prompt:: bash $
+
+	su -c 'rpm -Uvh https://download.ceph.com/rpm-{release-name}/{distro}/noarch/ceph-{version}.{distro}.noarch.rpm'
+
+
+
+.. _Releases: https://docs.ceph.com/en/latest/releases/
+.. _the testing Debian repository: https://download.ceph.com/debian-testing/dists
+.. _the shaman page: https://shaman.ceph.com
+.. _Ceph Mirrors: ../mirrors
diff --git a/doc/install/get-tarballs.rst b/doc/install/get-tarballs.rst
new file mode 100644
index 000000000..175d0399b
--- /dev/null
+++ b/doc/install/get-tarballs.rst
@@ -0,0 +1,14 @@
+====================================
+ Downloading a Ceph Release Tarball
+====================================
+
+As Ceph development progresses, the Ceph team releases new versions of the
+source code. You may download source code tarballs for Ceph releases here:
+
+`Ceph Release Tarballs`_
+
+.. tip:: For international users: There might be a mirror close to you where download Ceph from. For more information see: `Ceph Mirrors`_.
+
+
+.. _Ceph Release Tarballs: https://download.ceph.com/tarballs/
+.. _Ceph Mirrors: ../mirrors
diff --git a/doc/install/index.rst b/doc/install/index.rst
new file mode 100644
index 000000000..58ef92a27
--- /dev/null
+++ b/doc/install/index.rst
@@ -0,0 +1,76 @@
+.. _install-overview:
+
+===============
+Installing Ceph
+===============
+
+There are multiple ways to install Ceph.  
+
+Recommended methods
+~~~~~~~~~~~~~~~~~~~
+
+:ref:`Cephadm <cephadm_deploying_new_cluster>` installs and manages a Ceph
+cluster that uses containers and systemd and is tightly integrated with the CLI
+and dashboard GUI.
+
+* cephadm supports only Octopus and newer releases.
+* cephadm is fully integrated with the orchestration API and fully supports the
+  CLI and dashboard features that are used to manage cluster deployment.
+* cephadm requires container support (in the form of Podman or Docker) and
+  Python 3.
+
+`Rook <https://rook.io/>`_ deploys and manages Ceph clusters running
+in Kubernetes, while also enabling management of storage resources and
+provisioning via Kubernetes APIs. We recommend Rook as the way to run Ceph in
+Kubernetes or to connect an existing Ceph storage cluster to Kubernetes.
+
+* Rook supports only Nautilus and newer releases of Ceph.
+* Rook is the preferred method for running Ceph on Kubernetes, or for
+  connecting a Kubernetes cluster to an existing (external) Ceph
+  cluster.
+* Rook supports the orchestrator API. Management features in the CLI and
+  dashboard are fully supported.
+
+Other methods
+~~~~~~~~~~~~~
+
+`ceph-ansible <https://docs.ceph.com/ceph-ansible/>`_ deploys and manages
+Ceph clusters using Ansible.
+
+* ceph-ansible is widely deployed.
+* ceph-ansible is not integrated with the orchestrator APIs that were
+  introduced in Nautilus and Octopus, which means that the management features
+  and dashboard integration introduced in Nautilus and Octopus are not
+  available in Ceph clusters deployed by means of ceph-ansible.
+
+
+`ceph-deploy <https://docs.ceph.com/projects/ceph-deploy/en/latest/>`_ is a
+tool that can be used to quickly deploy clusters. It is deprecated.
+
+  .. IMPORTANT::
+
+   ceph-deploy is not actively maintained. It is not tested on versions of Ceph
+   newer than Nautilus. It does not support RHEL8, CentOS 8, or newer operating
+   systems.
+
+`ceph-salt <https://github.com/ceph/ceph-salt>`_ installs Ceph using Salt and cephadm.
+
+`jaas.ai/ceph-mon <https://jaas.ai/ceph-mon>`_ installs Ceph using Juju.
+
+`github.com/openstack/puppet-ceph <https://github.com/openstack/puppet-ceph>`_  installs Ceph via Puppet.
+
+Ceph can also be :ref:`installed manually <install-manual>`.
+
+
+.. toctree::
+   :hidden:
+
+   index_manual
+
+Windows
+~~~~~~~
+
+For Windows installations, consult this document:
+`Windows installation guide`_.
+
+.. _Windows installation guide: ./windows-install
diff --git a/doc/install/index_manual.rst b/doc/install/index_manual.rst
new file mode 100644
index 000000000..e64b99cb9
--- /dev/null
+++ b/doc/install/index_manual.rst
@@ -0,0 +1,69 @@
+.. _install-manual:
+
+=======================
+ Installation (Manual)
+=======================
+
+
+Get Software
+============
+
+There are several methods for getting Ceph software. The easiest and most common
+method is to `get packages`_ by adding repositories for use with package
+management tools such as the Advanced Package Tool (APT) or Yellowdog Updater,
+Modified (YUM). You may also retrieve pre-compiled packages from the Ceph
+repository. Finally, you can retrieve tarballs or clone the Ceph source code
+repository and build Ceph yourself.
+
+
+.. toctree::
+   :maxdepth: 1
+
+	Get Packages <get-packages>
+	Get Tarballs <get-tarballs>
+	Clone Source <clone-source>
+	Build Ceph <build-ceph>
+    	Ceph Mirrors <mirrors>
+	Ceph Containers <containers>
+
+
+Install Software
+================
+
+Once you have the Ceph software (or added repositories), installing the software
+is easy. To install packages on each :term:`Ceph Node` in your cluster, use package
+management tools. You should install Yum Priorities for RHEL/CentOS and other
+distributions that use Yum if you intend to install the Ceph Object Gateway or
+QEMU.
+
+.. toctree::
+   :maxdepth: 1
+
+	Install Ceph Storage Cluster <install-storage-cluster>
+	Install Virtualization for Block <install-vm-cloud>
+
+
+Deploy a Cluster Manually
+=========================
+
+Once you have Ceph installed on your nodes, you can deploy a cluster manually.
+The manual procedure is primarily for exemplary purposes for those developing
+deployment scripts with Chef, Juju, Puppet, etc.
+
+.. toctree::
+
+	Manual Deployment <manual-deployment>
+	Manual Deployment on FreeBSD <manual-freebsd-deployment>
+
+Upgrade Software
+================
+
+As new versions of Ceph become available, you may upgrade your cluster to take
+advantage of new functionality. Read the upgrade documentation before you
+upgrade your cluster. Sometimes upgrading Ceph requires you to follow an upgrade
+sequence.
+
+.. toctree::
+   :maxdepth: 2
+
+.. _get packages: ../get-packages
diff --git a/doc/install/install-storage-cluster.rst b/doc/install/install-storage-cluster.rst
new file mode 100644
index 000000000..6bedecbac
--- /dev/null
+++ b/doc/install/install-storage-cluster.rst
@@ -0,0 +1,87 @@
+==============================
+ Install Ceph Storage Cluster
+==============================
+
+This guide describes installing Ceph packages manually. This procedure
+is only for users who are not installing with a deployment tool such as
+``cephadm``, ``chef``, ``juju``, etc. 
+
+
+Installing with APT
+===================
+
+Once you have added either release or development packages to APT, you should
+update APT's database and install Ceph::
+
+	sudo apt-get update && sudo apt-get install ceph ceph-mds
+
+
+Installing with RPM
+===================
+
+To install Ceph with RPMs, execute the following steps:
+
+
+#. Install ``yum-plugin-priorities``. ::
+
+	sudo yum install yum-plugin-priorities
+
+#. Ensure ``/etc/yum/pluginconf.d/priorities.conf`` exists.
+
+#. Ensure ``priorities.conf`` enables the plugin. :: 
+
+	[main]
+	enabled = 1
+
+#. Ensure your YUM ``ceph.repo`` entry includes ``priority=2``. See
+   `Get Packages`_ for details::
+
+	[ceph]
+	name=Ceph packages for $basearch
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-noarch]
+	name=Ceph noarch packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/noarch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-source]
+	name=Ceph source packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/SRPMS
+	enabled=0
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+
+#. Install pre-requisite packages::  
+
+	sudo yum install snappy leveldb gdisk python-argparse gperftools-libs
+
+
+Once you have added either release or development packages, or added a
+``ceph.repo`` file to ``/etc/yum.repos.d``, you can install Ceph packages. :: 
+
+	sudo yum install ceph
+
+
+Installing a Build
+==================
+
+If you build Ceph from source code, you may install Ceph in user space
+by executing the following:: 
+
+	sudo ninja install
+
+If you install Ceph locally, ``ninja`` will place the executables in
+``usr/local/bin``. You may add the Ceph configuration file to the
+``usr/local/bin`` directory to run Ceph from a single directory.
+
+.. _Get Packages: ../get-packages
diff --git a/doc/install/install-vm-cloud.rst b/doc/install/install-vm-cloud.rst
new file mode 100644
index 000000000..876422865
--- /dev/null
+++ b/doc/install/install-vm-cloud.rst
@@ -0,0 +1,132 @@
+=========================================
+ Install Virtualization for Block Device
+=========================================
+
+If you intend to use Ceph Block Devices and the Ceph Storage Cluster as a
+backend for Virtual Machines (VMs) or  :term:`Cloud Platforms` the QEMU/KVM and
+``libvirt`` packages are important for enabling VMs and cloud platforms.
+Examples of VMs include: QEMU/KVM, XEN, VMWare, LXC, VirtualBox, etc. Examples
+of Cloud Platforms include OpenStack, CloudStack, OpenNebula, etc.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+Install QEMU
+============
+
+QEMU KVM can interact with Ceph Block Devices via ``librbd``, which is an
+important feature for using Ceph with cloud platforms. Once you install QEMU,
+see `QEMU and Block Devices`_ for usage.
+
+
+Debian Packages
+---------------
+
+QEMU packages are incorporated into Ubuntu 12.04 Precise Pangolin and later
+versions. To  install QEMU, execute the following::
+
+	sudo apt-get install qemu
+
+
+RPM Packages
+------------
+
+To install QEMU, execute the following:
+
+
+#. Update your repositories. ::
+
+	sudo yum update
+
+#. Install QEMU for Ceph. ::
+
+	sudo yum install qemu-kvm qemu-kvm-tools qemu-img
+
+#. Install additional QEMU packages (optional)::
+
+	sudo yum install qemu-guest-agent qemu-guest-agent-win32
+
+
+Building QEMU
+-------------
+
+To build QEMU from source, use the following procedure::
+
+	cd {your-development-directory}
+	git clone git://git.qemu.org/qemu.git
+	cd qemu
+	./configure --enable-rbd
+	make; make install
+
+
+
+Install libvirt
+===============
+
+To use ``libvirt`` with Ceph, you must have a running Ceph Storage Cluster, and
+you must have installed and configured QEMU. See `Using libvirt with Ceph Block
+Device`_ for usage.
+
+
+Debian Packages
+---------------
+
+``libvirt`` packages are incorporated into Ubuntu 12.04 Precise Pangolin and
+later versions of Ubuntu. To install ``libvirt`` on these distributions,
+execute the following::
+
+	sudo apt-get update && sudo apt-get install libvirt-bin
+
+
+RPM Packages
+------------
+
+To use ``libvirt`` with a Ceph Storage Cluster, you must  have a running Ceph
+Storage Cluster and you must also install a version of QEMU with ``rbd`` format
+support.  See `Install QEMU`_ for details.
+
+
+``libvirt`` packages are incorporated into the recent CentOS/RHEL distributions.
+To install ``libvirt``, execute the following::
+
+	sudo yum install libvirt
+
+
+Building ``libvirt``
+--------------------
+
+To build ``libvirt`` from source, clone the ``libvirt`` repository and use
+`AutoGen`_ to generate the build. Then, execute ``make`` and ``make install`` to
+complete the installation. For example::
+
+	git clone git://libvirt.org/libvirt.git
+	cd libvirt
+	./autogen.sh
+	make
+	sudo make install
+
+See `libvirt Installation`_ for details.
+
+
+
+.. _libvirt Installation: http://www.libvirt.org/compiling.html
+.. _AutoGen: http://www.gnu.org/software/autogen/
+.. _QEMU and Block Devices: ../../rbd/qemu-rbd
+.. _Using libvirt with Ceph Block Device: ../../rbd/libvirt
diff --git a/doc/install/manual-deployment.rst b/doc/install/manual-deployment.rst
new file mode 100644
index 000000000..6716ecb5b
--- /dev/null
+++ b/doc/install/manual-deployment.rst
@@ -0,0 +1,494 @@
+===================
+ Manual Deployment
+===================
+
+All Ceph clusters require at least one monitor, and at least as many OSDs as
+copies of an object stored on the cluster.  Bootstrapping the initial monitor(s)
+is the first step in deploying a Ceph Storage Cluster. Monitor deployment also
+sets important criteria for the entire cluster, such as the number of replicas
+for pools, the number of placement groups per OSD, the heartbeat intervals,
+whether authentication is required, etc. Most of these values are set by
+default, so it's useful to know about them when setting up your cluster for
+production.
+
+We will set up a cluster with ``mon-node1`` as  the monitor node, and ``osd-node1`` and
+``osd-node2`` for OSD nodes.
+
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |    mon-node1   |
+           |                  +-------->+                |
+           |                  |         | cCCC           |
+           \---------+--------/         \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |    osd-node1   |
+                     +----------------->+                |
+                     |                  | cCCC           |
+                     |                  \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |    osd-node2   |
+                     +----------------->|                |
+                                        | cCCC           |
+                                        \----------------/
+
+
+Monitor Bootstrapping
+=====================
+
+Bootstrapping a monitor (a Ceph Storage Cluster, in theory) requires
+a number of things:
+
+- **Unique Identifier:** The ``fsid`` is a unique identifier for the cluster,
+  and stands for File System ID from the days when the Ceph Storage Cluster was
+  principally for the Ceph File System. Ceph now supports native interfaces,
+  block devices, and object storage gateway interfaces too, so ``fsid`` is a
+  bit of a misnomer.
+
+- **Cluster Name:** Ceph clusters have a cluster name, which is a simple string
+  without spaces. The default cluster name is ``ceph``, but you may specify
+  a different cluster name. Overriding the default cluster name is
+  especially useful when you are working with multiple clusters and you need to
+  clearly understand which cluster your are working with.
+
+  For example, when you run multiple clusters in a :ref:`multisite configuration <multisite>`,
+  the cluster name (e.g., ``us-west``, ``us-east``) identifies the cluster for
+  the current CLI session. **Note:** To identify the cluster name on the
+  command line interface, specify the Ceph configuration file with the
+  cluster name (e.g., ``ceph.conf``, ``us-west.conf``, ``us-east.conf``, etc.).
+  Also see CLI usage (``ceph --cluster {cluster-name}``).
+
+- **Monitor Name:** Each monitor instance within a cluster has a unique name.
+  In common practice, the Ceph Monitor name is the host name (we recommend one
+  Ceph Monitor per host, and no commingling of Ceph OSD Daemons with
+  Ceph Monitors). You may retrieve the short hostname with ``hostname -s``.
+
+- **Monitor Map:** Bootstrapping the initial monitor(s) requires you to
+  generate a monitor map. The monitor map requires the ``fsid``, the cluster
+  name (or uses the default), and at least one host name and its IP address.
+
+- **Monitor Keyring**: Monitors communicate with each other via a
+  secret key. You must generate a keyring with a monitor secret and provide
+  it when bootstrapping the initial monitor(s).
+
+- **Administrator Keyring**: To use the ``ceph`` CLI tools, you must have
+  a ``client.admin`` user. So you must generate the admin user and keyring,
+  and you must also add the ``client.admin`` user to the monitor keyring.
+
+The foregoing requirements do not imply the creation of a Ceph Configuration
+file. However, as a best practice, we recommend creating a Ceph configuration
+file and populating it with the ``fsid``, the ``mon initial members`` and the
+``mon host`` settings.
+
+You can get and set all of the monitor settings at runtime as well. However,
+a Ceph Configuration file may contain only those settings that override the
+default values. When you add settings to a Ceph configuration file, these
+settings override the default settings. Maintaining those settings in a
+Ceph configuration file makes it easier to maintain your cluster.
+
+The procedure is as follows:
+
+
+#. Log in to the initial monitor node(s)::
+
+	ssh {hostname}
+
+   For example::
+
+	ssh mon-node1
+
+
+#. Ensure you have a directory for the Ceph configuration file. By default,
+   Ceph uses ``/etc/ceph``. When you install ``ceph``, the installer will
+   create the ``/etc/ceph`` directory automatically. ::
+
+	ls /etc/ceph
+
+
+#. Create a Ceph configuration file. By default, Ceph uses
+   ``ceph.conf``, where ``ceph`` reflects the cluster name. Add a line
+   containing "[global]" to the configuration file. ::
+
+	sudo vim /etc/ceph/ceph.conf
+
+
+#. Generate a unique ID (i.e., ``fsid``) for your cluster. ::
+
+	uuidgen
+
+
+#. Add the unique ID to your Ceph configuration file. ::
+
+	fsid = {UUID}
+
+   For example::
+
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+#. Add the initial monitor(s) to your Ceph configuration file. ::
+
+	mon_initial_members = {hostname}[,{hostname}]
+
+   For example::
+
+	mon_initial_members = mon-node1
+
+
+#. Add the IP address(es) of the initial monitor(s) to your Ceph configuration
+   file and save the file. ::
+
+	mon_host = {ip-address}[,{ip-address}]
+
+   For example::
+
+	mon_host = 192.168.0.1
+
+   **Note:** You may use IPv6 addresses instead of IPv4 addresses, but
+   you must set ``ms_bind_ipv6`` to ``true``. See `Network Configuration
+   Reference`_ for details about network configuration.
+
+#. Create a keyring for your cluster and generate a monitor secret key. ::
+
+	sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+
+#. Generate an administrator keyring, generate a ``client.admin`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+
+#. Generate a bootstrap-osd keyring, generate a ``client.bootstrap-osd`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+
+#. Add the generated keys to the ``ceph.mon.keyring``. ::
+
+	sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+	sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+#. Change the owner for ``ceph.mon.keyring``. ::
+
+	sudo chown ceph:ceph /tmp/ceph.mon.keyring
+
+#. Generate a monitor map using the hostname(s), host IP address(es) and the FSID.
+   Save it as ``/tmp/monmap``::
+
+	monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
+
+   For example::
+
+	monmaptool --create --add mon-node1 192.168.0.1 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
+
+
+#. Create a default data directory (or directories) on the monitor host(s). ::
+
+	sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
+
+   For example::
+
+	sudo -u ceph mkdir /var/lib/ceph/mon/ceph-mon-node1
+
+   See `Monitor Config Reference - Data`_ for details.
+
+#. Populate the monitor daemon(s) with the monitor map and keyring. ::
+
+	sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+   For example::
+
+	sudo -u ceph ceph-mon --mkfs -i mon-node1 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+
+#. Consider settings for a Ceph configuration file. Common settings include
+   the following::
+
+	[global]
+	fsid = {cluster-id}
+	mon_initial_members = {hostname}[, {hostname}]
+	mon_host = {ip-address}[, {ip-address}]
+	public_network = {network}[, {network}]
+	cluster_network = {network}[, {network}]
+	auth_cluster required = cephx
+	auth_service required = cephx
+	auth_client required = cephx
+	osd_pool_default_size = {n}  # Write an object n times.
+	osd_pool_default_min_size = {n} # Allow writing n copies in a degraded state.
+	osd_pool_default_pg_num = {n}
+	osd_crush_chooseleaf_type = {n}
+
+   In the foregoing example, the ``[global]`` section of the configuration might
+   look like this::
+
+	[global]
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+	mon_initial_members = mon-node1
+	mon_host = 192.168.0.1
+	public_network = 192.168.0.0/24
+	auth_cluster_required = cephx
+	auth_service_required = cephx
+	auth_client_required = cephx
+	osd_pool_default_size = 3
+	osd_pool_default_min_size = 2
+	osd_pool_default_pg_num = 333
+	osd_crush_chooseleaf_type = 1
+
+
+#. Start the monitor(s).
+
+   Start the service with systemd::
+
+	sudo systemctl start ceph-mon@mon-node1
+
+#. Ensure to open firewall ports for ceph-mon.
+
+   Open the ports with firewalld::
+   
+        sudo firewall-cmd --zone=public --add-service=ceph-mon
+	sudo firewall-cmd --zone=public --add-service=ceph-mon --permanent
+
+#. Verify that the monitor is running. ::
+
+	sudo ceph -s
+
+   You should see output that the monitor you started is up and running, and
+   you should see a health error indicating that placement groups are stuck
+   inactive. It should look something like this::
+
+      cluster:
+        id:     a7f64266-0894-4f1e-a635-d0aeaca0e993
+        health: HEALTH_OK
+
+      services:
+        mon: 1 daemons, quorum mon-node1
+        mgr: mon-node1(active)
+        osd: 0 osds: 0 up, 0 in
+
+      data:
+        pools:   0 pools, 0 pgs
+        objects: 0 objects, 0 bytes
+        usage:   0 kB used, 0 kB / 0 kB avail
+        pgs:
+
+
+   **Note:** Once you add OSDs and start them, the placement group health errors
+   should disappear. See `Adding OSDs`_ for details.
+
+Manager daemon configuration
+============================
+
+On each node where you run a ceph-mon daemon, you should also set up a ceph-mgr daemon.
+
+See :ref:`mgr-administrator-guide`
+
+Adding OSDs
+===========
+
+Once you have your initial monitor(s) running, you should add OSDs. Your cluster
+cannot reach an ``active + clean`` state until you have enough OSDs to handle the
+number of copies of an object (e.g., ``osd_pool_default_size = 2`` requires at
+least two OSDs). After bootstrapping your monitor, your cluster has a default
+CRUSH map; however, the CRUSH map doesn't have any Ceph OSD Daemons mapped to
+a Ceph Node.
+
+
+Short Form
+----------
+
+Ceph provides the ``ceph-volume`` utility, which can prepare a logical volume, disk, or partition
+for use with Ceph. The ``ceph-volume`` utility creates the OSD ID by
+incrementing the index. Additionally, ``ceph-volume`` will add the new OSD to the
+CRUSH map under the host for you. Execute ``ceph-volume -h`` for CLI details.
+The ``ceph-volume`` utility automates the steps of the `Long Form`_ below. To
+create the first two OSDs with the short form procedure, execute the following for each OSD:
+
+#. Create the OSD. ::
+
+	copy /var/lib/ceph/bootstrap-osd/ceph.keyring from monitor node (mon-node1) to /var/lib/ceph/bootstrap-osd/ceph.keyring on osd node (osd-node1)
+	ssh {osd node}
+	sudo ceph-volume lvm create --data {data-path}
+
+   For example::
+
+    	scp -3 root@mon-node1:/var/lib/ceph/bootstrap-osd/ceph.keyring root@osd-node1:/var/lib/ceph/bootstrap-osd/ceph.keyring
+    
+	ssh osd-node1
+	sudo ceph-volume lvm create --data /dev/hdd1
+
+Alternatively, the creation process can be split in two phases (prepare, and
+activate):
+
+#. Prepare the OSD. ::
+
+	ssh {osd node}
+	sudo ceph-volume lvm prepare --data {data-path} {data-path}
+
+   For example::
+
+	ssh osd-node1
+	sudo ceph-volume lvm prepare --data /dev/hdd1
+
+   Once prepared, the ``ID`` and ``FSID`` of the prepared OSD are required for
+   activation. These can be obtained by listing OSDs in the current server::
+
+    sudo ceph-volume lvm list
+
+#. Activate the OSD::
+
+	sudo ceph-volume lvm activate {ID} {FSID}
+
+   For example::
+
+	sudo ceph-volume lvm activate 0 a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+Long Form
+---------
+
+Without the benefit of any helper utilities, create an OSD and add it to the
+cluster and CRUSH map with the following procedure. To create the first two
+OSDs with the long form procedure, execute the following steps for each OSD.
+
+.. note:: This procedure does not describe deployment on top of dm-crypt
+          making use of the dm-crypt 'lockbox'.
+
+#. Connect to the OSD host and become root. ::
+
+     ssh {node-name}
+     sudo bash
+
+#. Generate a UUID for the OSD. ::
+
+     UUID=$(uuidgen)
+
+#. Generate a cephx key for the OSD. ::
+
+     OSD_SECRET=$(ceph-authtool --gen-print-key)
+
+#. Create the OSD. Note that an OSD ID can be provided as an
+   additional argument to ``ceph osd new`` if you need to reuse a
+   previously-destroyed OSD id. We assume that the
+   ``client.bootstrap-osd`` key is present on the machine.  You may
+   alternatively execute this command as ``client.admin`` on a
+   different host where that key is present.::
+
+     ID=$(echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
+	ceph osd new $UUID -i - \
+	-n client.bootstrap-osd -k /var/lib/ceph/bootstrap-osd/ceph.keyring)
+
+   It is also possible to include a ``crush_device_class`` property in the JSON
+   to set an initial class other than the default (``ssd`` or ``hdd`` based on
+   the auto-detected device type).
+
+#. Create the default directory on your new OSD. ::
+
+     mkdir /var/lib/ceph/osd/ceph-$ID
+
+#. If the OSD is for a drive other than the OS drive, prepare it
+   for use with Ceph, and mount it to the directory you just created. ::
+
+     mkfs.xfs /dev/{DEV}
+     mount /dev/{DEV} /var/lib/ceph/osd/ceph-$ID
+
+#. Write the secret to the OSD keyring file. ::
+
+     ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-$ID/keyring \
+          --name osd.$ID --add-key $OSD_SECRET
+
+#. Initialize the OSD data directory. ::
+
+     ceph-osd -i $ID --mkfs --osd-uuid $UUID
+
+#. Fix ownership. ::
+
+     chown -R ceph:ceph /var/lib/ceph/osd/ceph-$ID
+
+#. After you add an OSD to Ceph, the OSD is in your configuration. However,
+   it is not yet running. You must start
+   your new OSD before it can begin receiving data.
+
+   For modern systemd distributions::
+
+     systemctl enable ceph-osd@$ID
+     systemctl start ceph-osd@$ID
+
+   For example::
+
+     systemctl enable ceph-osd@12
+     systemctl start ceph-osd@12
+
+
+Adding MDS
+==========
+
+In the below instructions, ``{id}`` is an arbitrary name, such as the hostname of the machine.
+
+#. Create the mds data directory.::
+
+	mkdir -p /var/lib/ceph/mds/{cluster-name}-{id}
+
+#. Create a keyring.::
+
+	ceph-authtool --create-keyring /var/lib/ceph/mds/{cluster-name}-{id}/keyring --gen-key -n mds.{id}
+
+#. Import the keyring and set caps.::
+
+	ceph auth add mds.{id} osd "allow rwx" mds "allow *" mon "allow profile mds" -i /var/lib/ceph/mds/{cluster}-{id}/keyring
+
+#. Add to ceph.conf.::
+
+	[mds.{id}]
+	host = {id}
+
+#. Start the daemon the manual way.::
+
+	ceph-mds --cluster {cluster-name} -i {id} -m {mon-hostname}:{mon-port} [-f]
+
+#. Start the daemon the right way (using ceph.conf entry).::
+
+	service ceph start
+
+#. If starting the daemon fails with this error::
+
+	mds.-1.0 ERROR: failed to authenticate: (22) Invalid argument
+
+   Then make sure you do not have a keyring set in ceph.conf in the global section; move it to the client section; or add a keyring setting specific to this mds daemon. And verify that you see the same key in the mds data directory and ``ceph auth get mds.{id}`` output.
+
+#. Now you are ready to `create a Ceph file system`_.
+
+
+Summary
+=======
+
+Once you have your monitor and two OSDs up and running, you can watch the
+placement groups peer by executing the following::
+
+	ceph -w
+
+To view the tree, execute the following::
+
+	ceph osd tree
+
+You should see output that looks something like this::
+
+	# id	weight	type name	up/down	reweight
+	-1	2	root default
+	-2	2		host osd-node1
+	0	1			osd.0	up	1
+	-3	1		host osd-node2
+	1	1			osd.1	up	1
+
+To add (or remove) additional monitors, see `Add/Remove Monitors`_.
+To add (or remove) additional Ceph OSD Daemons, see `Add/Remove OSDs`_.
+
+
+.. _Add/Remove Monitors: ../../rados/operations/add-or-rm-mons
+.. _Add/Remove OSDs: ../../rados/operations/add-or-rm-osds
+.. _Network Configuration Reference: ../../rados/configuration/network-config-ref
+.. _Monitor Config Reference - Data: ../../rados/configuration/mon-config-ref#data
+.. _create a Ceph file system: ../../cephfs/createfs
diff --git a/doc/install/manual-freebsd-deployment.rst b/doc/install/manual-freebsd-deployment.rst
new file mode 100644
index 000000000..0ed47e1ca
--- /dev/null
+++ b/doc/install/manual-freebsd-deployment.rst
@@ -0,0 +1,528 @@
+==============================
+ Manual Deployment on FreeBSD
+==============================
+
+This a largely a copy of the regular Manual Deployment with FreeBSD specifics.
+The difference lies in two parts: The underlying diskformat, and the way to use
+the tools.
+
+All Ceph clusters require at least one monitor, and at least as many OSDs as
+copies of an object stored on the cluster.  Bootstrapping the initial monitor(s)
+is the first step in deploying a Ceph Storage Cluster. Monitor deployment also
+sets important criteria for the entire cluster, such as the number of replicas
+for pools, the number of placement groups per OSD, the heartbeat intervals,
+whether authentication is required, etc. Most of these values are set by
+default, so it's useful to know about them when setting up your cluster for
+production.
+
+We will set up a cluster with ``node1`` as  the monitor node, and ``node2`` and
+``node3`` for OSD nodes.
+
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |     node1      |
+           |                  +-------->+                |
+           |                  |         | cCCC           |
+           \---------+--------/         \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node2      |
+                     +----------------->+                |
+                     |                  | cCCC           |
+                     |                  \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node3      |
+                     +----------------->|                |
+                                        | cCCC           |
+                                        \----------------/
+
+
+
+Disklayout on FreeBSD
+=====================
+
+Current implementation works on ZFS pools
+
+* All Ceph data is created in /var/lib/ceph
+* Log files go into /var/log/ceph
+* PID files go into /var/log/run
+* One ZFS pool is allocated per OSD, like::
+
+    gpart create -s GPT ada1
+    gpart add -t freebsd-zfs -l osd.1 ada1
+    zpool create -m /var/lib/ceph/osd/osd.1 osd.1 gpt/osd.1
+
+* Some cache and log (ZIL) can be attached.
+  Please note that this is different from the Ceph journals. Cache and log are
+  totally transparent for Ceph, and help the file system to keep the system
+  consistent and help performance.
+  Assuming that ada2 is an SSD::
+
+    gpart create -s GPT ada2
+    gpart add -t freebsd-zfs -l osd.1-log -s 1G ada2
+    zpool add osd.1 log gpt/osd.1-log
+    gpart add -t freebsd-zfs -l osd.1-cache -s 10G ada2
+    zpool add osd.1 log gpt/osd.1-cache
+
+* Note: *UFS2 does not allow large xattribs*
+
+
+Configuration
+-------------
+
+As per FreeBSD default parts of extra software go into ``/usr/local/``. Which
+means that for ``/etc/ceph.conf`` the default location is
+``/usr/local/etc/ceph/ceph.conf``. Smartest thing to do is to create a softlink
+from ``/etc/ceph`` to ``/usr/local/etc/ceph``::
+
+  ln -s /usr/local/etc/ceph /etc/ceph
+
+A sample file is provided in ``/usr/local/share/doc/ceph/sample.ceph.conf``
+Note that ``/usr/local/etc/ceph/ceph.conf`` will be found by most tools,
+linking it to ``/etc/ceph/ceph.conf`` will help with any scripts that are found
+in extra tools, scripts, and/or discussionlists.
+
+Monitor Bootstrapping
+=====================
+
+Bootstrapping a monitor (a Ceph Storage Cluster, in theory) requires
+a number of things:
+
+- **Unique Identifier:** The ``fsid`` is a unique identifier for the cluster,
+  and stands for File System ID from the days when the Ceph Storage Cluster was
+  principally for the Ceph File System. Ceph now supports native interfaces,
+  block devices, and object storage gateway interfaces too, so ``fsid`` is a
+  bit of a misnomer.
+
+- **Cluster Name:** Ceph clusters have a cluster name, which is a simple string
+  without spaces. The default cluster name is ``ceph``, but you may specify
+  a different cluster name. Overriding the default cluster name is
+  especially useful when you are working with multiple clusters and you need to
+  clearly understand which cluster your are working with.
+
+  For example, when you run multiple clusters in a :ref:`multisite configuration <multisite>`,
+  the cluster name (e.g., ``us-west``, ``us-east``) identifies the cluster for
+  the current CLI session. **Note:** To identify the cluster name on the
+  command line interface, specify the a Ceph configuration file with the
+  cluster name (e.g., ``ceph.conf``, ``us-west.conf``, ``us-east.conf``, etc.).
+  Also see CLI usage (``ceph --cluster {cluster-name}``).
+
+- **Monitor Name:** Each monitor instance within a cluster has a unique name.
+  In common practice, the Ceph Monitor name is the host name (we recommend one
+  Ceph Monitor per host, and no commingling of Ceph OSD Daemons with
+  Ceph Monitors). You may retrieve the short hostname with ``hostname -s``.
+
+- **Monitor Map:** Bootstrapping the initial monitor(s) requires you to
+  generate a monitor map. The monitor map requires the ``fsid``, the cluster
+  name (or uses the default), and at least one host name and its IP address.
+
+- **Monitor Keyring**: Monitors communicate with each other via a
+  secret key. You must generate a keyring with a monitor secret and provide
+  it when bootstrapping the initial monitor(s).
+
+- **Administrator Keyring**: To use the ``ceph`` CLI tools, you must have
+  a ``client.admin`` user. So you must generate the admin user and keyring,
+  and you must also add the ``client.admin`` user to the monitor keyring.
+
+The foregoing requirements do not imply the creation of a Ceph Configuration
+file. However, as a best practice, we recommend creating a Ceph configuration
+file and populating it with the ``fsid``, the ``mon initial members`` and the
+``mon host`` settings.
+
+You can get and set all of the monitor settings at runtime as well. However,
+a Ceph Configuration file may contain only those settings that override the
+default values. When you add settings to a Ceph configuration file, these
+settings override the default settings. Maintaining those settings in a
+Ceph configuration file makes it easier to maintain your cluster.
+
+The procedure is as follows:
+
+
+#. Log in to the initial monitor node(s)::
+
+	ssh {hostname}
+
+   For example::
+
+	ssh node1
+
+
+#. Ensure you have a directory for the Ceph configuration file. By default,
+   Ceph uses ``/etc/ceph``. When you install ``ceph``, the installer will
+   create the ``/etc/ceph`` directory automatically. ::
+
+	ls /etc/ceph
+
+#. Create a Ceph configuration file. By default, Ceph uses
+   ``ceph.conf``, where ``ceph`` reflects the cluster name. ::
+
+	sudo vim /etc/ceph/ceph.conf
+
+
+#. Generate a unique ID (i.e., ``fsid``) for your cluster. ::
+
+	uuidgen
+
+
+#. Add the unique ID to your Ceph configuration file. ::
+
+	fsid = {UUID}
+
+   For example::
+
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+#. Add the initial monitor(s) to your Ceph configuration file. ::
+
+	mon initial members = {hostname}[,{hostname}]
+
+   For example::
+
+	mon initial members = node1
+
+
+#. Add the IP address(es) of the initial monitor(s) to your Ceph configuration
+   file and save the file. ::
+
+	mon host = {ip-address}[,{ip-address}]
+
+   For example::
+
+	mon host = 192.168.0.1
+
+   **Note:** You may use IPv6 addresses instead of IPv4 addresses, but
+   you must set ``ms bind ipv6`` to ``true``. See `Network Configuration
+   Reference`_ for details about network configuration.
+
+#. Create a keyring for your cluster and generate a monitor secret key. ::
+
+	ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+
+#. Generate an administrator keyring, generate a ``client.admin`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+
+
+#. Add the ``client.admin`` key to the ``ceph.mon.keyring``. ::
+
+	ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+
+
+#. Generate a monitor map using the hostname(s), host IP address(es) and the FSID.
+   Save it as ``/tmp/monmap``::
+
+	monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
+
+   For example::
+
+	monmaptool --create --add node1 192.168.0.1 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
+
+
+#. Create a default data directory (or directories) on the monitor host(s). ::
+
+	sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
+
+   For example::
+
+	sudo mkdir /var/lib/ceph/mon/ceph-node1
+
+   See `Monitor Config Reference - Data`_ for details.
+
+#. Populate the monitor daemon(s) with the monitor map and keyring. ::
+
+	sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+   For example::
+
+	sudo -u ceph ceph-mon --mkfs -i node1 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+
+#. Consider settings for a Ceph configuration file. Common settings include
+   the following::
+
+	[global]
+	fsid = {cluster-id}
+	mon initial members = {hostname}[, {hostname}]
+	mon host = {ip-address}[, {ip-address}]
+	public network = {network}[, {network}]
+	cluster network = {network}[, {network}]
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = {n}
+	osd pool default size = {n}  # Write an object n times.
+	osd pool default min size = {n} # Allow writing n copy in a degraded state.
+	osd pool default pg num = {n}
+	osd pool default pgp num = {n}
+	osd crush chooseleaf type = {n}
+
+   In the foregoing example, the ``[global]`` section of the configuration might
+   look like this::
+
+	[global]
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+	mon initial members = node1
+	mon host = 192.168.0.1
+	public network = 192.168.0.0/24
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = 1024
+	osd pool default size = 3
+	osd pool default min size = 2
+	osd pool default pg num = 333
+	osd pool default pgp num = 333
+	osd crush chooseleaf type = 1
+
+#. Touch the ``done`` file.
+
+   Mark that the monitor is created and ready to be started::
+
+	sudo touch /var/lib/ceph/mon/ceph-node1/done
+
+#. And for FreeBSD an entry for every monitor needs to be added to the config
+   file. (The requirement will be removed in future releases).
+
+   The entry should look like::
+
+     [mon]
+         [mon.node1]
+             host = node1    # this name can be resolve
+
+
+#. Start the monitor(s).
+
+   For FreeBSD we use the rc.d init scripts (called bsdrc in Ceph)::
+
+	sudo service ceph start start mon.node1
+
+   For this to work /etc/rc.conf also needs the entry to enable ceph::
+     cat 'ceph_enable="YES"' >> /etc/rc.conf
+
+
+#. Verify that Ceph created the default pools. ::
+
+	ceph osd lspools
+
+   You should see output like this::
+
+	0 data
+	1 metadata
+	2 rbd
+
+#. Verify that the monitor is running. ::
+
+	ceph -s
+
+   You should see output that the monitor you started is up and running, and
+   you should see a health error indicating that placement groups are stuck
+   inactive. It should look something like this::
+
+	cluster a7f64266-0894-4f1e-a635-d0aeaca0e993
+	  health HEALTH_ERR 192 pgs stuck inactive; 192 pgs stuck unclean; no osds
+	  monmap e1: 1 mons at {node1=192.168.0.1:6789/0}, election epoch 1, quorum 0 node1
+	  osdmap e1: 0 osds: 0 up, 0 in
+	  pgmap v2: 192 pgs, 3 pools, 0 bytes data, 0 objects
+	     0 kB used, 0 kB / 0 kB avail
+	     192 creating
+
+   **Note:** Once you add OSDs and start them, the placement group health errors
+   should disappear. See the next section for details.
+
+.. _freebsd_adding_osds:
+
+Adding OSDs
+===========
+
+Once you have your initial monitor(s) running, you should add OSDs. Your cluster
+cannot reach an ``active + clean`` state until you have enough OSDs to handle the
+number of copies of an object (e.g., ``osd pool default size = 2`` requires at
+least two OSDs). After bootstrapping your monitor, your cluster has a default
+CRUSH map; however, the CRUSH map doesn't have any Ceph OSD Daemons mapped to
+a Ceph Node.
+
+
+Long Form
+---------
+
+Without the benefit of any helper utilities, create an OSD and add it to the
+cluster and CRUSH map with the following procedure. To create the first two
+OSDs with the long form procedure, execute the following on ``node2`` and
+``node3``:
+
+#. Connect to the OSD host. ::
+
+	ssh {node-name}
+
+#. Generate a UUID for the OSD. ::
+
+	uuidgen
+
+
+#. Create the OSD. If no UUID is given, it will be set automatically when the
+   OSD starts up. The following command will output the OSD number, which you
+   will need for subsequent steps. ::
+
+	ceph osd create [{uuid} [{id}]]
+
+
+#. Create the default directory on your new OSD. ::
+
+	ssh {new-osd-host}
+	sudo mkdir /var/lib/ceph/osd/{cluster-name}-{osd-number}
+
+   Above are the ZFS instructions to do this for FreeBSD.
+
+
+#. If the OSD is for a drive other than the OS drive, prepare it
+   for use with Ceph, and mount it to the directory you just created.
+
+
+#. Initialize the OSD data directory. ::
+
+	ssh {new-osd-host}
+	sudo ceph-osd -i {osd-num} --mkfs --mkkey --osd-uuid [{uuid}]
+
+   The directory must be empty before you can run ``ceph-osd`` with the
+   ``--mkkey`` option. In addition, the ceph-osd tool requires specification
+   of custom cluster names with the ``--cluster`` option.
+
+
+#. Register the OSD authentication key. The value of ``ceph`` for
+   ``ceph-{osd-num}`` in the path is the ``$cluster-$id``.  If your
+   cluster name differs from ``ceph``, use your cluster name instead.::
+
+	sudo ceph auth add osd.{osd-num} osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/{cluster-name}-{osd-num}/keyring
+
+
+#. Add your Ceph Node to the CRUSH map. ::
+
+	ceph [--cluster {cluster-name}] osd crush add-bucket {hostname} host
+
+   For example::
+
+	ceph osd crush add-bucket node1 host
+
+
+#. Place the Ceph Node under the root ``default``. ::
+
+	ceph osd crush move node1 root=default
+
+
+#. Add the OSD to the CRUSH map so that it can begin receiving data. You may
+   also decompile the CRUSH map, add the OSD to the device list, add the host as a
+   bucket (if it's not already in the CRUSH map), add the device as an item in the
+   host, assign it a weight, recompile it and set it. ::
+
+	ceph [--cluster {cluster-name}] osd crush add {id-or-name} {weight} [{bucket-type}={bucket-name} ...]
+
+   For example::
+
+	ceph osd crush add osd.0 1.0 host=node1
+
+
+#. After you add an OSD to Ceph, the OSD is in your configuration. However,
+   it is not yet running. The OSD is ``down`` and ``in``. You must start
+   your new OSD before it can begin receiving data.
+
+   For FreeBSD using rc.d init.
+
+   After adding the OSD to ``ceph.conf``::
+
+	sudo service ceph start osd.{osd-num}
+
+   For example::
+
+	sudo service ceph start osd.0
+	sudo service ceph start osd.1
+
+   In this case, to allow the start of the daemon at each reboot you
+   must create an empty file like this::
+
+	sudo touch /var/lib/ceph/osd/{cluster-name}-{osd-num}/bsdrc
+
+   For example::
+
+	sudo touch /var/lib/ceph/osd/ceph-0/bsdrc
+	sudo touch /var/lib/ceph/osd/ceph-1/bsdrc
+
+   Once you start your OSD, it is ``up`` and ``in``.
+
+
+
+Adding MDS
+==========
+
+In the below instructions, ``{id}`` is an arbitrary name, such as the hostname of the machine.
+
+#. Create the mds data directory.::
+
+	mkdir -p /var/lib/ceph/mds/{cluster-name}-{id}
+
+#. Create a keyring.::
+
+	ceph-authtool --create-keyring /var/lib/ceph/mds/{cluster-name}-{id}/keyring --gen-key -n mds.{id}
+
+#. Import the keyring and set caps.::
+
+	ceph auth add mds.{id} osd "allow rwx" mds "allow *" mon "allow profile mds" -i /var/lib/ceph/mds/{cluster}-{id}/keyring
+
+#. Add to ceph.conf.::
+
+	[mds.{id}]
+	host = {id}
+
+#. Start the daemon the manual way.::
+
+	ceph-mds --cluster {cluster-name} -i {id} -m {mon-hostname}:{mon-port} [-f]
+
+#. Start the daemon the right way (using ceph.conf entry).::
+
+	service ceph start
+
+#. If starting the daemon fails with this error::
+
+	mds.-1.0 ERROR: failed to authenticate: (22) Invalid argument
+
+   Then make sure you do not have a keyring set in ceph.conf in the global section; move it to the client section; or add a keyring setting specific to this mds daemon. And verify that you see the same key in the mds data directory and ``ceph auth get mds.{id}`` output.
+
+#. Now you are ready to `create a Ceph file system`_.
+
+
+Summary
+=======
+
+Once you have your monitor and two OSDs up and running, you can watch the
+placement groups peer by executing the following::
+
+	ceph -w
+
+To view the tree, execute the following::
+
+	ceph osd tree
+
+You should see output that looks something like this::
+
+	# id	weight	type name	up/down	reweight
+	-1	2	root default
+	-2	2		host node1
+	0	1			osd.0	up	1
+	-3	1		host node2
+	1	1			osd.1	up	1
+
+To add (or remove) additional monitors, see `Add/Remove Monitors`_.
+To add (or remove) additional Ceph OSD Daemons, see `Add/Remove OSDs`_.
+
+
+.. _Add/Remove Monitors: ../../rados/operations/add-or-rm-mons
+.. _Add/Remove OSDs: ../../rados/operations/add-or-rm-osds
+.. _Network Configuration Reference: ../../rados/configuration/network-config-ref
+.. _Monitor Config Reference - Data: ../../rados/configuration/mon-config-ref#data
+.. _create a Ceph file system: ../../cephfs/createfs
diff --git a/doc/install/mirrors.rst b/doc/install/mirrors.rst
new file mode 100644
index 000000000..11c60db8d
--- /dev/null
+++ b/doc/install/mirrors.rst
@@ -0,0 +1,65 @@
+=============
+ Ceph Mirrors
+=============
+
+For improved user experience multiple mirrors for Ceph are available around the
+world.
+
+These mirrors are kindly sponsored by various companies who want to support the
+Ceph project.
+
+
+Locations
+=========
+
+These mirrors are available on the following locations:
+
+- **EU: Netherlands**: http://eu.ceph.com/
+- **AU: Australia**: http://au.ceph.com/
+- **SE: Sweden**: http://se.ceph.com/
+- **DE: Germany**: http://de.ceph.com/
+- **FR: France**: http://fr.ceph.com/
+- **UK: UK**: http://uk.ceph.com
+- **US-Mid-West: Chicago**: http://mirrors.gigenet.com/ceph/
+- **US-West: US West Coast**: http://us-west.ceph.com/
+- **CN: China**: http://mirrors.ustc.edu.cn/ceph/
+
+You can replace all download.ceph.com URLs with any of the mirrors, for example:
+
+- http://download.ceph.com/tarballs/
+- http://download.ceph.com/debian-hammer/
+- http://download.ceph.com/rpm-hammer/
+
+Change this to:
+
+- http://eu.ceph.com/tarballs/
+- http://eu.ceph.com/debian-hammer/
+- http://eu.ceph.com/rpm-hammer/
+
+
+Mirroring
+=========
+
+You can easily mirror Ceph yourself using a Bash script and rsync. An easy-to-use
+script can be found at `GitHub`_.
+
+When mirroring Ceph, please keep the following guidelines in mind:
+
+- Choose a mirror close to you
+- Do not sync in a interval shorter than 3 hours
+- Avoid syncing at minute 0 of the hour, use something between 0 and 59
+
+
+Becoming a mirror
+=================
+
+If you want to provide a public mirror for other users of Ceph you can opt to
+become a official mirror.
+
+To make sure all mirrors meet the same standards some requirements have been
+set for all mirrors. These can be found on `GitHub`_.
+
+If you want to apply for an official mirror, please contact the ceph-users mailinglist.
+
+
+.. _GitHub: https://github.com/ceph/ceph/tree/master/mirroring
diff --git a/doc/install/windows-basic-config.rst b/doc/install/windows-basic-config.rst
new file mode 100644
index 000000000..0fe8a1b1b
--- /dev/null
+++ b/doc/install/windows-basic-config.rst
@@ -0,0 +1,48 @@
+:orphan:
+
+===========================
+Windows basic configuration
+===========================
+
+This page describes the minimum Ceph configuration required for using the
+client components on Windows.
+
+ceph.conf
+=========
+
+The default location for the ``ceph.conf`` file on Windows is
+``%ProgramData%\ceph\ceph.conf``, which usually expands to
+``C:\ProgramData\ceph\ceph.conf``.
+
+Below you may find a sample. Please fill in the monitor addresses
+accordingly.
+
+.. code:: ini
+
+    [global]
+        log to stderr = true
+        ; Uncomment the following in order to use the Windows Event Log
+        ; log to syslog = true
+
+        run dir = C:/ProgramData/ceph/out
+        crash dir = C:/ProgramData/ceph/out
+
+        ; Use the following to change the cephfs client log level
+        ; debug client = 2
+    [client]
+        keyring = C:/ProgramData/ceph/keyring
+        ; log file = C:/ProgramData/ceph/out/$name.$pid.log
+        admin socket = C:/ProgramData/ceph/out/$name.$pid.asok
+
+        ; client_permissions = true
+        ; client_mount_uid = 1000
+        ; client_mount_gid = 1000
+    [global]
+        mon host = <ceph_monitor_addresses>
+
+Don't forget to also copy your keyring file to the specified location and make
+sure that the configured directories exist (e.g. ``C:\ProgramData\ceph\out``).
+
+Please use slashes ``/`` instead of backslashes ``\`` as path separators
+within ``ceph.conf``.
+
diff --git a/doc/install/windows-install.rst b/doc/install/windows-install.rst
new file mode 100644
index 000000000..6da3e1723
--- /dev/null
+++ b/doc/install/windows-install.rst
@@ -0,0 +1,87 @@
+:orphan:
+
+==========================
+Installing Ceph on Windows
+==========================
+
+The Ceph client tools and libraries can be natively used on Windows. This avoids
+the need for additional layers such as iSCSI gateways or SMB shares,
+drastically improving the performance.
+
+Prerequisites
+=============
+
+Supported platforms
+-------------------
+
+Windows Server 2019 and Windows Server 2016 are supported. Previous Windows
+Server versions, including Windows client versions such as Windows 10, might
+work but haven't been tested.
+
+Windows Server 2016 does not provide Unix sockets, in which case some commands
+might be unavailable.
+
+Secure boot
+-----------
+
+The ``WNBD`` driver hasn't been signed by Microsoft, which means that Secure Boot
+must be disabled.
+
+Dokany
+------
+
+In order to mount Ceph filesystems, ``ceph-dokan`` requires Dokany to be
+installed. You may fetch the installer as well as the source code from the
+Dokany GitHub repository: https://github.com/dokan-dev/dokany/releases
+
+Make sure to install Dokany 2.0.5 or later.
+
+Unlike ``WNBD``, Dokany isn't included in the Ceph MSI installer.
+
+MSI installer
+=============
+
+Using the MSI installer is the recommended way of installing Ceph on Windows.
+It can be downloaded from here: https://cloudbase.it/ceph-for-windows/
+
+As mentioned earlier, the Ceph installer does not include Dokany, which has
+to be installed separately.
+
+A server reboot is required after uninstalling the driver, otherwise subsequent
+install attempts may fail.
+
+The following project allows building the MSI installer:
+https://github.com/cloudbase/ceph-windows-installer. It can either use prebuilt
+Ceph and WNBD binaries or compile them from scratch.
+
+Manual installation
+===================
+
+The following document describes the build process and manual installation:
+https://github.com/ceph/ceph/blob/master/README.windows.rst
+
+Configuration
+=============
+
+Please check the `Windows configuration sample`_ to get started.
+
+You'll also need a keyring file. The `General CephFS Prerequisites`_ page provides a
+simple example, showing how a new CephX user can be created and how its secret
+key can be retrieved.
+
+For more details on CephX user management, see the `Client Authentication`_
+and :ref:`User Management <user-management>`.
+
+Further reading
+===============
+
+* `RBD Windows documentation`_
+* `CephFS Windows documentation`_
+* `Windows troubleshooting`_
+
+.. _CephFS Windows documentation: ../../cephfs/ceph-dokan
+.. _Windows configuration sample: ../windows-basic-config
+.. _RBD Windows documentation: ../../rbd/rbd-windows/
+.. _Windows troubleshooting: ../windows-troubleshooting
+.. _General CephFS Prerequisites: ../../cephfs/mount-prerequisites
+.. _Client Authentication: ../../cephfs/client-auth
diff --git a/doc/install/windows-troubleshooting.rst b/doc/install/windows-troubleshooting.rst
new file mode 100644
index 000000000..c41d50b1d
--- /dev/null
+++ b/doc/install/windows-troubleshooting.rst
@@ -0,0 +1,98 @@
+:orphan:
+
+===============================
+Troubleshooting Ceph on Windows
+===============================
+
+MSI installer
+~~~~~~~~~~~~~
+
+The MSI source code can be consulted here:
+https://github.com/cloudbase/ceph-windows-installer
+
+The following command can be used to generate MSI logs
+
+.. code:: PowerShell
+
+    msiexec.exe /i $msi_full_path /l*v! $log_file
+
+WNBD driver installation failures will be logged here: ``C:\Windows\inf\setupapi.dev.log``.
+A server reboot is required after uninstalling the driver, otherwise subsequent
+install attempts may fail.
+
+Wnbd
+~~~~
+
+For ``WNBD`` troubleshooting, please check this page: https://github.com/cloudbase/wnbd#troubleshooting
+
+Privileges
+~~~~~~~~~~
+
+Most ``rbd-wnbd`` and ``rbd device`` commands require privileged rights. Make
+sure to use an elevated PowerShell or CMD command prompt.
+
+Crash dumps
+~~~~~~~~~~~
+
+Userspace crash dumps can be placed at a configurable location and enabled for all
+applications or just predefined ones, as outlined here:
+https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps.
+
+Whenever a Windows application crashes, an event will be submitted to the ``Application``
+Windows Event Log, having Event ID 1000. The entry will also include the process id,
+the faulting module name and path as well as the exception code.
+
+Please note that in order to analyze crash dumps, the debug symbols are required.
+We're currently building Ceph using ``MinGW``, so by default ``DWARF`` symbols will
+be embedded in the binaries. ``windbg`` does not support such symbols but ``gdb``
+can be used.
+
+``gdb`` can debug running Windows processes but it cannot open Windows minidumps.
+The following ``gdb`` fork may be used until this functionality is merged upstream:
+https://github.com/ssbssa/gdb/releases. As an alternative, ``DWARF`` symbols
+can be converted using ``cv2pdb`` but be aware that this tool has limited C++
+support.
+
+ceph tool
+~~~~~~~~~
+
+The ``ceph`` Python tool can't be used on Windows natively yet. With minor
+changes it may run, but the main issue is that Python doesn't currently allow
+using ``AF_UNIX`` on Windows: https://bugs.python.org/issue33408
+
+As an alternative, the ``ceph`` tool can be used through Windows Subsystem
+for Linux (WSL). For example, running Windows RBD daemons may be contacted by
+using:
+
+.. code:: bash
+
+    ceph daemon /mnt/c/ProgramData/ceph/out/ceph-client.admin.61436.1209215304.asok help
+
+IO counters
+~~~~~~~~~~~
+
+Along with the standard RBD perf counters, the ``libwnbd`` IO counters may be
+retrieved using:
+
+.. code:: PowerShell
+
+    rbd-wnbd stats $imageName
+
+At the same time, WNBD driver counters can be fetched using:
+
+.. code:: PowerShell
+
+    wnbd-client stats $mappingId
+
+Note that the ``wnbd-client`` mapping identifier will be the full RBD image spec
+(the ``device`` column of the ``rbd device list`` output).
+
+Missing libraries
+~~~~~~~~~~~~~~~~~
+
+The Ceph tools can silently exit with a -1073741515 return code if one of the
+required DLLs is missing or unsupported.
+
+The `Dependency Walker`_ tool can be used to determine the missing library.
+
+.. _Dependency Walker: https://www.dependencywalker.com/
diff --git a/doc/jaegertracing/index.rst b/doc/jaegertracing/index.rst
new file mode 100644
index 000000000..a3afd4787
--- /dev/null
+++ b/doc/jaegertracing/index.rst
@@ -0,0 +1,107 @@
+JAEGER- DISTRIBUTED TRACING
+===========================
+
+Jaeger provides ready-to-use tracing services for distributed systems. 
+
+BASIC ARCHITECTURE AND TERMINOLOGY
+----------------------------------
+
+* TRACE: A trace shows the data/execution path through a system.
+* SPAN: A single unit of a trace. A data structure that stores information such
+  as the operation name, timestamps, and the ordering within a trace.
+* JAEGER CLIENT: Language-specific implementations of the OpenTracing API.
+* JAEGER AGENT: A daemon that listens for spans sent over User Datagram
+  Protocol. The agent is meant to be placed on the same host as the
+  instrumented application. (The Jaeger agent acts like a sidecar listener.)
+* JAEGER COLLECTOR: A daemon that receives spans sent by the Jaeger agent. The
+  Jaeger collector then stitches the spans together to form a trace. (A databse
+  can be enabled to persist a database for these traces).
+* JAEGER QUERY AND CONSOLE FRONTEND: The UI-based frontend that presents
+  reports of the jaeger traces. Accessible at  http://<jaeger frontend host>:16686.
+
+Read more about jaeger tracing:.
+
+  https://www.jaegertracing.io/docs/
+
+JAEGER DEPLOYMENT
+-----------------
+
+Jaeger can be deployed using cephadm, and Jaeger can be deployed manually.
+
+Refer to one of the following:
+
+`Cephadm Jaeger Services Deployment <../cephadm/services/tracing/>`_
+
+`Jaeger Deployment <https://www.jaegertracing.io/docs/1.25/deployment/>`_
+
+`Jaeger Performance Tuning <https://www.jaegertracing.io/docs/1.25/performance-tuning/>`_
+
+
+Important Notes:
+^^^^^^^^^^^^^^^^
+
+- The Jaeger agent must be running on each host (and not running in all-in-one
+  mode). This is because spans are sent to the local Jaeger agent. Spans of
+  hosts that do not have active Jaeger agents will be lost.
+
+- Ceph tracers are configured to send tracers to agents that listen to port
+  6799. Use the option "--processor.jaeger-compact.server-host-port=6799" for
+  manual Jaeger deployments. 
+
+
+HOW TO ENABLE TRACING IN CEPH
+-----------------------------
+
+Tracing in Ceph is disabled by default.
+
+Tracing can be enabled globally, and tracing can also be enabled separately for
+each entity (for example, for rgw).
+
+Enable tracing globally:
+
+.. prompt:: bash $
+
+   ceph config set global jaeger_tracing_enable true
+
+
+Enable tracing for each entity:
+
+.. prompt:: bash $
+
+   ceph config set <entity> jaeger_tracing_enable true
+
+
+TRACES IN RGW
+-------------
+
+Traces run on RGW can be found under the Service `rgw` in the Jaeger Frontend.
+
+REQUESTS
+^^^^^^^^
+Every user request is traced. Each trace contains tags for `Operation name`,
+`User id`, `Object name` and `Bucket name`.
+
+There is also an `Upload id` tag for Multipart upload operations.
+
+The names of request traces have the following format: `<command> <transaction
+id>`.
+
+MULTIPART UPLOAD
+^^^^^^^^^^^^^^^^
+There is a kind of trace that consists of a span for each request made by a
+multipart upload, and it includes all `Put Object` requests.
+
+The names of multipart traces have the following format: `multipart_upload
+<upload id>`.
+
+
+rgw service in Jaeger Frontend:
+
+.. image:: ./rgw_jaeger.png
+  :width: 400
+
+
+osd service in Jaeger Frontend:
+
+.. image:: ./osd_jaeger.png
+  :width: 400
diff --git a/doc/jaegertracing/osd_jaeger.png b/doc/jaegertracing/osd_jaeger.png
new file mode 100644
index 000000000..b881e1623
--- /dev/null
+++ b/doc/jaegertracing/osd_jaeger.png
diff --git a/doc/jaegertracing/rgw_jaeger.png b/doc/jaegertracing/rgw_jaeger.png
new file mode 100644
index 000000000..9f063022c
--- /dev/null
+++ b/doc/jaegertracing/rgw_jaeger.png
diff --git a/doc/logo.png b/doc/logo.png
new file mode 100644
index 000000000..4cd8c7cc9
--- /dev/null
+++ b/doc/logo.png
diff --git a/doc/man/8/CMakeLists.txt b/doc/man/8/CMakeLists.txt
new file mode 100644
index 000000000..f1df5b4b4
--- /dev/null
+++ b/doc/man/8/CMakeLists.txt
@@ -0,0 +1,95 @@
+set(client_srcs
+  ceph-syn.rst
+  ceph-conf.rst
+  ceph.rst
+  ceph-authtool.rst
+  ceph-kvstore-tool.rst
+  rados.rst
+  ceph-post-file.rst
+  ceph-dencoder.rst)
+
+set(server_srcs
+  crushtool.rst
+  ceph-run.rst
+  mount.ceph.rst
+  mount.fuse.ceph.rst
+  ceph-create-keys.rst)
+if(WITH_TESTS)
+list(APPEND server_srcs
+  ceph-debugpack.rst)
+endif(WITH_TESTS)
+
+set(osd_srcs
+  ceph-clsinfo.rst
+  ceph-volume.rst
+  ceph-volume-systemd.rst
+  ceph-osd.rst
+  crushdiff.rst
+  osdmaptool.rst
+  ceph-bluestore-tool.rst)
+
+set(mon_srcs
+  ceph-mon.rst
+  monmaptool.rst)
+
+list(APPEND man_srcs
+  ${client_srcs}
+  ${server_srcs}
+  ${osd_srcs}
+  ${mon_srcs}
+  ceph-mds.rst
+  cephfs-top.rst
+  librados-config.rst
+  cephadm.rst
+  cephfs-mirror.rst)
+
+if(WITH_CEPHFS_SHELL)
+  list(APPEND man_srcs
+    cephfs-shell.rst)
+endif()
+
+if(HAVE_LIBFUSE)
+  list(APPEND man_srcs
+    ceph-fuse.rst
+    rbd-fuse.rst)
+endif()
+
+if(WITH_RADOSGW)
+  list(APPEND man_srcs
+	radosgw.rst
+	radosgw-admin.rst
+	rgw-orphan-list.rst
+	rgw-policy-check.rst
+	ceph-diff-sorted.rst)
+endif()
+
+if(WITH_RBD)
+  list(APPEND man_srcs
+	ceph-rbdnamer.rst
+	rbd-mirror.rst
+	rbd-replay-many.rst
+	rbd-replay-prep.rst
+	rbd-replay.rst
+	rbdmap.rst
+	rbd.rst)
+  if(LINUX)
+    list(APPEND man_srcs rbd-nbd.rst)
+  endif()
+  if(FREEBSD)
+    list(APPEND man_srcs rbd-ggate.rst)
+  endif()
+endif()
+
+list(APPEND man_srcs ceph-immutable-object-cache.rst)
+foreach(man ${man_srcs})
+  list(APPEND sphinx_input ${CMAKE_CURRENT_SOURCE_DIR}/${man})
+  # mount.ceph.rst => mount if we use
+  # get_filename_component(cmd ${man} NAME_WE)
+  string(REGEX REPLACE ".rst$" "" cmd ${man})
+  list(APPEND sphinx_output ${sphinx_output_dir}/${cmd}.8)
+  install(FILES ${sphinx_output_dir}/${cmd}.8
+    DESTINATION ${CEPH_MAN_DIR}/man8)
+endforeach()
+
+set(sphinx_input ${sphinx_input} PARENT_SCOPE)
+set(sphinx_output ${sphinx_output} PARENT_SCOPE)
diff --git a/doc/man/8/ceph-authtool.rst b/doc/man/8/ceph-authtool.rst
new file mode 100644
index 000000000..4e37f01a3
--- /dev/null
+++ b/doc/man/8/ceph-authtool.rst
@@ -0,0 +1,206 @@
+:orphan:
+
+=================================================
+ ceph-authtool -- ceph keyring manipulation tool
+=================================================
+
+.. program:: ceph-authtool
+
+Synopsis
+========
+
+| **ceph-authtool** *keyringfile*
+  [ -l | --list ]
+  [ -p | --print-key ]
+  [ -C | --create-keyring ]
+  [ -g | --gen-key ]
+  [ --gen-print-key ]
+  [ --import-keyring *otherkeyringfile* ]
+  [ -n | --name *entityname* ]
+  [ -a | --add-key *base64_key* ]
+  [ --cap *subsystem* *capability* ]
+  [ --caps *capfile* ]
+  [ --mode *mode* ]
+
+
+Description
+===========
+
+**ceph-authtool** is a utility to create, view, and modify a Ceph keyring
+file. A keyring file stores one or more Ceph authentication keys and
+possibly an associated capability specification. Each key is
+associated with an entity name, of the form
+``{client,mon,mds,osd}.name``.
+
+**WARNING** Ceph provides authentication and protection against
+man-in-the-middle attacks once secret keys are in place.  However,
+data over the wire is not encrypted, which may include the messages
+used to configure said keys. The system is primarily intended to be
+used in trusted environments.
+
+Options
+=======
+
+.. option:: -l, --list
+
+   will list all keys and capabilities present in the keyring
+
+.. option:: -p, --print-key
+
+   will print an encoded key for the specified entityname. This is
+   suitable for the ``mount -o secret=`` argument
+
+.. option:: -C, --create-keyring
+
+   will create a new keyring, overwriting any existing keyringfile
+
+.. option:: -g, --gen-key
+
+   will generate a new secret key for the specified entityname
+
+.. option:: --gen-print-key
+
+   will generate a new secret key for the specified entityname,
+   without altering the keyringfile, printing the secret to stdout
+
+.. option:: --import-keyring *secondkeyringfile*
+
+   will import the content of a given keyring to the keyringfile
+
+.. option:: -n, --name *name*
+
+   specify entityname to operate on
+
+.. option:: -a, --add-key *base64_key*
+
+   will add an encoded key to the keyring
+
+.. option:: --cap *subsystem* *capability*
+
+   will set the capability for given subsystem
+
+.. option:: --caps *capsfile*
+
+   will set all of capabilities associated with a given key, for all subsystems
+
+.. option:: --mode *mode*
+
+   will set the desired file mode to the keyring e.g: 0644, defaults to 0600
+
+
+Capabilities
+============
+
+The subsystem is the name of a Ceph subsystem: ``mon``, ``mds``, or
+``osd``.
+
+The capability is a string describing what the given user is allowed
+to do. This takes the form of a comma separated list of allow
+clauses with a permission specifier containing one or more of rwx for
+read, write, and execute permission. The ``allow *`` grants full
+superuser permissions for the given subsystem.
+
+For example::
+
+	# can read, write, and execute objects
+        osd = "allow rwx"
+
+	# can access mds server
+        mds = "allow"
+
+	# can modify cluster state (i.e., is a server daemon)
+        mon = "allow rwx"
+
+A librados user restricted to a single pool might look like::
+
+        mon = "allow r"
+
+        osd = "allow rw pool foo"
+
+A client using rbd with read access to one pool and read/write access to another::
+
+        mon = "allow r"
+
+        osd = "allow class-read object_prefix rbd_children, allow pool templates r class-read, allow pool vms rwx"
+
+A client mounting the file system with minimal permissions would need caps like::
+
+        mds = "allow"
+
+        osd = "allow rw pool data"
+
+        mon = "allow r"
+
+
+OSD Capabilities
+================
+
+In general, an osd capability follows the grammar::
+
+        osdcap  := grant[,grant...]
+        grant   := allow (match capspec | capspec match)
+        match   := [ pool[=]<poolname> | object_prefix <prefix>
+                    | namespace[=]<rados-namespace>
+                    | tag <application-name> <key>=<value> ]
+        capspec := * | [r][w][x] [class-read] [class-write]
+
+The capspec determines what kind of operations the entity can perform::
+
+    r           = read access to objects
+    w           = write access to objects
+    x           = can call any class method (same as class-read class-write)
+    class-read  = can call class methods that are reads
+    class-write = can call class methods that are writes
+    * or "all"  = equivalent to rwx, plus the ability to run osd admin commands,
+                  i.e. ceph osd tell ...
+
+The match criteria restrict a grant based on the pool being accessed.
+Grants are additive if the client fulfills the match condition. For
+example, if a client has the osd capabilities: "allow r object_prefix
+prefix, allow w pool foo, allow x pool bar", then it has rw access to
+pool foo, rx access to pool bar, and r access to objects whose
+names begin with 'prefix' in any pool.
+
+Caps file format
+================
+
+The caps file format consists of zero or more key/value pairs, one per
+line. The key and value are separated by an ``=``, and the value must
+be quoted (with ``'`` or ``"``) if it contains any whitespace. The key
+is the name of the Ceph subsystem (``osd``, ``mds``, ``mon``), and the
+value is the capability string (see above).
+
+
+Example
+=======
+
+To create a new keyring containing a key for client.foo with a 0644 file mode::
+
+        ceph-authtool -C -n client.foo --gen-key keyring --mode 0644
+
+To associate some capabilities with the key (namely, the ability to
+mount a Ceph file system)::
+
+        ceph-authtool -n client.foo --cap mds 'allow' --cap osd 'allow rw pool=data' --cap mon 'allow r' keyring
+
+To display the contents of the keyring::
+
+        ceph-authtool -l keyring
+
+When mounting a Ceph file system, you can grab the appropriately encoded secret key with::
+
+        mount -t ceph serverhost:/ mountpoint -o name=foo,secret=`ceph-authtool -p -n client.foo keyring`
+
+
+Availability
+============
+
+**ceph-authtool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-bluestore-tool.rst b/doc/man/8/ceph-bluestore-tool.rst
new file mode 100644
index 000000000..f6c88da09
--- /dev/null
+++ b/doc/man/8/ceph-bluestore-tool.rst
@@ -0,0 +1,240 @@
+:orphan:
+
+======================================================
+ ceph-bluestore-tool -- bluestore administrative tool
+======================================================
+
+.. program:: ceph-bluestore-tool
+
+Synopsis
+========
+
+| **ceph-bluestore-tool** *command*
+  [ --dev *device* ... ]
+  [ -i *osd_id* ]
+  [ --path *osd path* ]
+  [ --out-dir *dir* ]
+  [ --log-file | -l *filename* ]
+  [ --deep ]
+| **ceph-bluestore-tool** fsck|repair --path *osd path* [ --deep ]
+| **ceph-bluestore-tool** qfsck       --path *osd path*
+| **ceph-bluestore-tool** allocmap    --path *osd path*
+| **ceph-bluestore-tool** restore_cfb --path *osd path*
+| **ceph-bluestore-tool** show-label --dev *device* ...
+| **ceph-bluestore-tool** prime-osd-dir --dev *device* --path *osd path*
+| **ceph-bluestore-tool** bluefs-export --path *osd path* --out-dir *dir*
+| **ceph-bluestore-tool** bluefs-bdev-new-wal --path *osd path* --dev-target *new-device*
+| **ceph-bluestore-tool** bluefs-bdev-new-db --path *osd path* --dev-target *new-device*
+| **ceph-bluestore-tool** bluefs-bdev-migrate --path *osd path* --dev-target *new-device* --devs-source *device1* [--devs-source *device2*]
+| **ceph-bluestore-tool** free-dump|free-score --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+| **ceph-bluestore-tool** reshard --path *osd path* --sharding *new sharding* [ --sharding-ctrl *control string* ]
+| **ceph-bluestore-tool** show-sharding --path *osd path*
+
+
+Description
+===========
+
+**ceph-bluestore-tool** is a utility to perform low-level administrative
+operations on a BlueStore instance.
+
+Commands
+========
+
+:command:`help`
+
+   show help
+
+:command:`fsck` [ --deep ]
+
+   run consistency check on BlueStore metadata.  If *--deep* is specified, also read all object data and verify checksums.
+
+:command:`repair`
+
+   Run a consistency check *and* repair any errors we can.
+
+:command:`qfsck`
+
+   run consistency check on BlueStore metadata comparing allocator data (from RocksDB CFB when exists and if not uses allocation-file) with ONodes state.
+
+:command:`allocmap`
+
+   performs the same check done by qfsck and then stores a new allocation-file (command is disabled by default and requires a special build)
+
+:command:`restore_cfb`
+
+   Reverses changes done by the new NCB code (either through ceph restart or when running allocmap command) and restores RocksDB B Column-Family (allocator-map).
+
+
+:command:`bluefs-export`
+
+   Export the contents of BlueFS (i.e., RocksDB files) to an output directory.
+
+:command:`bluefs-bdev-sizes` --path *osd path*
+
+   Print the device sizes, as understood by BlueFS, to stdout.
+
+:command:`bluefs-bdev-expand` --path *osd path*
+
+   Instruct BlueFS to check the size of its block devices and, if they have
+   expanded, make use of the additional space. Please note that only the new
+   files created by BlueFS will be allocated on the preferred block device if
+   it has enough free space, and the existing files that have spilled over to
+   the slow device will be gradually removed when RocksDB performs compaction.
+   In other words, if there is any data spilled over to the slow device, it
+   will be moved to the fast device over time.
+
+:command:`bluefs-bdev-new-wal` --path *osd path* --dev-target *new-device*
+
+   Adds WAL device to BlueFS, fails if WAL device already exists.
+
+:command:`bluefs-bdev-new-db` --path *osd path* --dev-target *new-device*
+
+   Adds DB device to BlueFS, fails if DB device already exists.
+   
+:command:`bluefs-bdev-migrate` --dev-target *new-device* --devs-source *device1* [--devs-source *device2*]
+
+   Moves BlueFS data from source device(s) to the target one, source devices
+   (except the main one) are removed on success. Target device can be both
+   already attached or new device. In the latter case it's added to OSD
+   replacing one of the source devices. Following replacement rules apply
+   (in the order of precedence, stop on the first match):
+
+      - if source list has DB volume - target device replaces it.
+      - if source list has WAL volume - target device replace it.
+      - if source list has slow volume only - operation isn't permitted, requires explicit allocation via new-db/new-wal command.
+
+:command:`show-label` --dev *device* [...]
+
+   Show device label(s).	   
+
+:command:`free-dump` --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+
+   Dump all free regions in allocator.
+
+:command:`free-score` --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+
+   Give a [0-1] number that represents quality of fragmentation in allocator.
+   0 represents case when all free space is in one chunk. 1 represents worst possible fragmentation.
+
+:command:`reshard` --path *osd path* --sharding *new sharding* [ --resharding-ctrl *control string* ]
+
+   Changes sharding of BlueStore's RocksDB. Sharding is build on top of RocksDB column families.
+   This option allows to test performance of *new sharding* without need to redeploy OSD.
+   Resharding is usually a long process, which involves walking through entire RocksDB key space
+   and moving some of them to different column families.
+   Option --resharding-ctrl provides performance control over resharding process.
+   Interrupted resharding will prevent OSD from running.
+   Interrupted resharding does not corrupt data. It is always possible to continue previous resharding,
+   or select any other sharding scheme, including reverting to original one.
+
+:command:`show-sharding` --path *osd path*
+
+   Show sharding that is currently applied to BlueStore's RocksDB.
+
+Options
+=======
+
+.. option:: --dev *device*
+
+   Add *device* to the list of devices to consider
+
+.. option:: -i *osd_id*
+
+   Operate as OSD *osd_id*. Connect to monitor for OSD specific options.
+   If monitor is unavailable, add --no-mon-config to read from ceph.conf instead.
+
+.. option:: --devs-source *device*
+
+   Add *device* to the list of devices to consider as sources for migrate operation
+
+.. option:: --dev-target *device*
+
+   Specify target *device* migrate operation or device to add for adding new DB/WAL.
+
+.. option:: --path *osd path*
+
+   Specify an osd path.  In most cases, the device list is inferred from the symlinks present in *osd path*.  This is usually simpler than explicitly specifying the device(s) with --dev. Not necessary if -i *osd_id* is provided.
+
+.. option:: --out-dir *dir*
+
+   Output directory for bluefs-export
+
+.. option:: -l, --log-file *log file*
+
+   file to log to
+
+.. option:: --log-level *num*
+
+   debug log level.  Default is 30 (extremely verbose), 20 is very
+   verbose, 10 is verbose, and 1 is not very verbose.
+
+.. option:: --deep
+
+   deep scrub/repair (read and validate object data, not just metadata)
+
+.. option:: --allocator *name*
+
+   Useful for *free-dump* and *free-score* actions. Selects allocator(s).
+
+.. option:: --resharding-ctrl *control string*
+
+   Provides control over resharding process. Specifies how often refresh RocksDB iterator,
+   and how large should commit batch be before committing to RocksDB. Option format is:
+   <iterator_refresh_bytes>/<iterator_refresh_keys>/<batch_commit_bytes>/<batch_commit_keys>
+   Default: 10000000/10000/1000000/1000
+
+Additional ceph.conf options
+============================
+
+Any configuration option that is accepted by OSD can be also passed to **ceph-bluestore-tool**.
+Useful to provide necessary configuration options when access to monitor/ceph.conf is impossible and -i option cannot be used.
+
+Device labels
+=============
+
+Every BlueStore block device has a single block label at the beginning of the
+device.  You can dump the contents of the label with::
+
+  ceph-bluestore-tool show-label --dev *device*
+
+The main device will have a lot of metadata, including information
+that used to be stored in small files in the OSD data directory.  The
+auxiliary devices (db and wal) will only have the minimum required
+fields (OSD UUID, size, device type, birth time).
+
+OSD directory priming
+=====================
+
+You can generate the content for an OSD data directory that can start up a
+BlueStore OSD with the *prime-osd-dir* command::
+
+  ceph-bluestore-tool prime-osd-dir --dev *main device* --path /var/lib/ceph/osd/ceph-*id*
+
+BlueFS log rescue
+=====================
+
+Some versions of BlueStore were susceptible to BlueFS log growing extremely large -
+beyond the point of making booting OSD impossible. This state is indicated by
+booting that takes very long and fails in _replay function.
+
+This can be fixed by::
+  ceph-bluestore-tool fsck --path *osd path* --bluefs_replay_recovery=true
+
+It is advised to first check if rescue process would be successful::
+  ceph-bluestore-tool fsck --path *osd path* \
+  --bluefs_replay_recovery=true --bluefs_replay_recovery_disable_compact=true
+
+If above fsck is successful fix procedure can be applied.
+
+Availability
+============
+
+**ceph-bluestore-tool** is part of Ceph, a massively scalable,
+open-source, distributed storage system. Please refer to the Ceph
+documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-clsinfo.rst b/doc/man/8/ceph-clsinfo.rst
new file mode 100644
index 000000000..931631547
--- /dev/null
+++ b/doc/man/8/ceph-clsinfo.rst
@@ -0,0 +1,49 @@
+:orphan:
+
+===============================================
+ ceph-clsinfo -- show class object information
+===============================================
+
+.. program:: ceph-clsinfo
+
+Synopsis
+========
+
+| **ceph-clsinfo** [ *options* ] ... *filename*
+
+
+Description
+===========
+
+**ceph-clsinfo** can show name, version, and architecture information
+about a specific class object.
+
+
+Options
+=======
+
+.. option:: -n, --name
+
+   Shows the class name
+
+.. option:: -v, --version
+
+   Shows the class version
+
+.. option:: -a, --arch
+
+   Shows the class architecture
+
+
+Availability
+============
+
+**ceph-clsinfo** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-conf.rst b/doc/man/8/ceph-conf.rst
new file mode 100644
index 000000000..4fea4d43d
--- /dev/null
+++ b/doc/man/8/ceph-conf.rst
@@ -0,0 +1,149 @@
+:orphan:
+
+==================================
+ ceph-conf -- ceph conf file tool
+==================================
+
+.. program:: ceph-conf
+
+Synopsis
+========
+
+| **ceph-conf** -c *conffile* --list-all-sections
+| **ceph-conf** -c *conffile* -L
+| **ceph-conf** -c *conffile* -l *prefix*
+| **ceph-conf** *key* -s *section1* ...
+| **ceph-conf** [-s *section* ] [-r] --lookup *key*
+| **ceph-conf** [-s *section* ] *key*
+
+
+Description
+===========
+
+**ceph-conf** is a utility for getting information from a ceph
+configuration file. As with most Ceph programs, you can specify which
+Ceph configuration file to use with the ``-c`` flag.
+
+Note that unlike other ceph tools, **ceph-conf** will *only* read from
+config files (or return compiled-in default values)--it will *not*
+fetch config values from the monitor cluster.  For this reason it is
+recommended that **ceph-conf** only be used in legacy environments
+that are strictly config-file based.  New deployments and tools should
+instead rely on either querying the monitor explicitly for
+configuration (e.g., ``ceph config get <daemon> <option>``) or use
+daemons themselves to fetch effective config options (e.g.,
+``ceph-osd -i 123 --show-config-value osd_data``).  The latter option
+has the advantages of drawing from compiled-in defaults (which
+occasionally vary between daemons), config files, and the monitor's
+config database, providing the exact value that that daemon would be
+using if it were started.
+
+Actions
+=======
+
+**ceph-conf** performs one of the following actions:
+
+.. option:: -L, --list-all-sections
+
+   list all sections in the configuration file.
+
+.. option:: -l, --list-sections *prefix*
+
+   list the sections with the given *prefix*. For example, ``--list-sections mon``
+   would list all sections beginning with ``mon``.
+
+.. option:: --lookup *key*
+
+   search and print the specified configuration setting. Note:  ``--lookup`` is
+   the default action. If no other actions are given on the command line, we will
+   default to doing a lookup.
+
+.. option:: -h, --help
+
+   print a summary of usage.
+
+
+Options
+=======
+
+.. option:: -c *conffile*
+
+   the Ceph configuration file.
+
+.. option:: --filter-key *key*
+
+   filter section list to only include sections with given *key* defined.
+
+.. option:: --filter-key-value *key* ``=`` *value*
+
+   filter section list to only include sections with given *key*/*value* pair.
+
+.. option:: --name *type.id*
+
+   the Ceph name in which the sections are searched (default 'client.admin').
+   For example, if we specify ``--name osd.0``, the following sections will be
+   searched: [osd.0], [osd], [global]
+
+.. option:: --pid *pid*
+
+   override the ``$pid`` when expanding options. For example, if an option is
+   configured like ``/var/log/$name.$pid.log``, the ``$pid`` portion in its
+   value will be substituted using the PID of **ceph-conf** instead of the
+   PID of the process specified using the ``--name`` option.
+
+.. option:: -r, --resolve-search
+
+   search for the first file that exists and can be opened in the resulted
+   comma delimited search list.
+
+.. option:: -s, --section
+
+   additional sections to search.  These additional sections will be searched
+   before the sections that would normally be searched. As always, the first
+   matching entry we find will be returned.
+
+
+Examples
+========
+
+To find out what value osd 0 will use for the "osd data" option::
+
+        ceph-conf -c foo.conf  --name osd.0 --lookup "osd data"
+
+To find out what value will mds a use for the "log file" option::
+
+        ceph-conf -c foo.conf  --name mds.a "log file"
+
+To list all sections that begin with "osd"::
+
+        ceph-conf -c foo.conf -l osd
+
+To list all sections::
+
+        ceph-conf -c foo.conf -L
+
+To print the path of the "keyring" used by "client.0"::
+
+       ceph-conf --name client.0 -r -l keyring
+
+
+Files
+=====
+
+``/etc/ceph/$cluster.conf``, ``~/.ceph/$cluster.conf``, ``$cluster.conf``
+
+the Ceph configuration files to use if not specified.
+
+
+Availability
+============
+
+**ceph-conf** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please refer
+to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
diff --git a/doc/man/8/ceph-create-keys.rst b/doc/man/8/ceph-create-keys.rst
new file mode 100644
index 000000000..07f41adeb
--- /dev/null
+++ b/doc/man/8/ceph-create-keys.rst
@@ -0,0 +1,67 @@
+:orphan:
+
+===============================================
+ceph-create-keys -- ceph keyring generate tool
+===============================================
+
+.. program:: ceph-create-keys
+
+Synopsis
+========
+
+| **ceph-create-keys** [-h] [-v] [-t seconds] [--cluster *name*] --id *id*
+
+
+Description
+===========
+
+:program:`ceph-create-keys` is a utility to generate bootstrap keyrings using
+the given monitor when it is ready.
+
+It creates following auth entities (or users)
+
+``client.admin``
+
+    and its key for your client host.
+
+``client.bootstrap-{osd, rgw, mds}``
+
+    and their keys for bootstrapping corresponding services
+
+To list all users in the cluster::
+
+    ceph auth ls
+
+
+Options
+=======
+
+.. option:: --cluster
+
+   name of the cluster (default 'ceph').
+
+.. option:: -t
+
+   time out after **seconds** (default: 600) waiting for a response from the monitor
+
+.. option:: -i, --id
+
+   id of a ceph-mon that is coming up. **ceph-create-keys** will wait until it joins quorum.
+
+.. option:: -v, --verbose
+
+   be more verbose.
+
+
+Availability
+============
+
+**ceph-create-keys** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please refer
+to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-debugpack.rst b/doc/man/8/ceph-debugpack.rst
new file mode 100644
index 000000000..845738e2a
--- /dev/null
+++ b/doc/man/8/ceph-debugpack.rst
@@ -0,0 +1,50 @@
+:orphan:
+
+=============================================
+ ceph-debugpack -- ceph debug packer utility
+=============================================
+
+.. program:: ceph-debugpack
+
+Synopsis
+========
+
+| **ceph-debugpack** [ *options* ] *filename.tar.gz*
+
+
+Description
+===========
+
+**ceph-debugpack** will build a tarball containing various items that are
+useful for debugging crashes. The resulting tarball can be shared with
+Ceph developers when debugging a problem.
+
+The tarball will include the binaries for ceph-mds, ceph-osd, and ceph-mon, radosgw, any
+log files, the ceph.conf configuration file, any core files we can
+find, and (if the system is running) dumps of the current cluster state
+as reported by 'ceph report'.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+
+Availability
+============
+
+**ceph-debugpack** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`ceph-post-file <ceph-post-file>`\(8)
diff --git a/doc/man/8/ceph-dencoder.rst b/doc/man/8/ceph-dencoder.rst
new file mode 100644
index 000000000..ef7e972b8
--- /dev/null
+++ b/doc/man/8/ceph-dencoder.rst
@@ -0,0 +1,151 @@
+:orphan:
+
+==============================================
+ ceph-dencoder -- ceph encoder/decoder utility
+==============================================
+
+.. program:: ceph-dencoder
+
+Synopsis
+========
+
+| **ceph-dencoder** [commands...]
+
+
+Description
+===========
+
+**ceph-dencoder** is a utility to encode, decode, and dump ceph data
+structures.  It is used for debugging and for testing inter-version
+compatibility.
+
+**ceph-dencoder** takes a simple list of commands and performs them
+in order.
+
+Commands
+========
+
+.. option:: version
+
+   Print the version string for the **ceph-dencoder** binary.
+
+.. option:: import <file>
+
+   Read a binary blob of encoded data from the given file.  It will be
+   placed in an in-memory buffer.
+
+.. option:: export <file>
+
+   Write the contents of the current in-memory buffer to the given
+   file.
+
+.. option:: list_types
+
+   List the data types known to this build of **ceph-dencoder**.
+
+.. option:: type <name>
+
+   Select the given type for future ``encode`` or ``decode`` operations.
+
+.. option:: skip <bytes>
+
+   Seek <bytes> into the imported file before reading data structure, use
+   this with objects that have a preamble/header before the object of interest.
+
+.. option:: decode
+
+   Decode the contents of the in-memory buffer into an instance of the
+   previously selected type.  If there is an error, report it.
+
+.. option:: encode
+
+   Encode the contents of the in-memory instance of the previously
+   selected type to the in-memory buffer.
+
+.. option:: dump_json
+
+   Print a JSON-formatted description of the in-memory object.
+
+.. option:: count_tests
+
+   Print the number of built-in test instances of the previously
+   selected type that **ceph-dencoder** is able to generate.
+
+.. option:: select_test <n>
+
+   Select the given built-in test instance as the in-memory instance
+   of the type.
+
+.. option:: get_features
+
+   Print the decimal value of the feature set supported by this version
+   of **ceph-dencoder**.  Each bit represents a feature.  These correspond to
+   CEPH_FEATURE_* defines in src/include/ceph_features.h.
+
+.. option:: set_features <f>
+
+   Set the feature bits provided to ``encode`` to *f*.  This allows
+   you to encode objects such that they can be understood by old
+   versions of the software (for those types that support it).
+
+Example
+=======
+
+Say you want to examine an attribute on an object stored by ``ceph-osd``.  You can do this:
+
+::
+
+    $ cd /mnt/osd.12/current/2.b_head
+    $ attr -l foo_bar_head_EFE6384B
+    Attribute "ceph.snapset" has a 31 byte value for foo_bar_head_EFE6384B
+    Attribute "ceph._" has a 195 byte value for foo_bar_head_EFE6384B
+    $ attr foo_bar_head_EFE6384B -g ceph._ -q > /tmp/a
+    $ ceph-dencoder type object_info_t import /tmp/a decode dump_json
+    { "oid": { "oid": "foo",
+          "key": "bar",
+          "snapid": -2,
+          "hash": 4024842315,
+          "max": 0},
+      "locator": { "pool": 2,
+          "preferred": -1,
+          "key": "bar"},
+      "category": "",
+      "version": "9'1",
+      "prior_version": "0'0",
+      "last_reqid": "client.4116.0:1",
+      "size": 1681,
+      "mtime": "2012-02-21 08:58:23.666639",
+      "lost": 0,
+      "wrlock_by": "unknown.0.0:0",
+      "snaps": [],
+      "truncate_seq": 0,
+      "truncate_size": 0,
+      "watchers": {}}
+
+Alternatively, perhaps you wish to dump an internal CephFS metadata object, you might
+do that like this:
+
+::
+
+   $ rados -p metadata get mds_snaptable mds_snaptable.bin
+   $ ceph-dencoder type SnapServer skip 8 import mds_snaptable.bin decode dump_json
+   { "snapserver": { "last_snap": 1,
+      "pending_noop": [],
+      "snaps": [],
+      "need_to_purge": {},
+      "pending_create": [],
+      "pending_destroy": []}} 
+
+
+Availability
+============
+
+**ceph-dencoder** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-diff-sorted.rst b/doc/man/8/ceph-diff-sorted.rst
new file mode 100644
index 000000000..61e0772e1
--- /dev/null
+++ b/doc/man/8/ceph-diff-sorted.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+==========================================================
+ ceph-diff-sorted -- compare two sorted files line by line
+==========================================================
+
+.. program:: ceph-diff-sorted
+
+Synopsis
+========
+
+| **ceph-diff-sorted** *file1* *file2*
+
+Description
+===========
+
+:program:`ceph-diff-sorted` is a simplified *diff* utility optimized
+for comparing two files with lines that are lexically sorted.
+
+The output is simplified in comparison to that of the standard `diff`
+tool available in POSIX systems. Angle brackets ('<' and '>') are used
+to show lines that appear in one file but not the other. The output is
+not compatible with the `patch` tool.
+
+This tool was created in order to perform diffs of large files (e.g.,
+containing billions of lines) that the standard `diff` tool cannot
+handle efficiently. Knowing that the lines are sorted allows this to
+be done efficiently with minimal memory overhead.
+
+The sorting of each file needs to be done lexically. Most POSIX
+systems use the *LANG* environment variable to determine the `sort`
+tool's sorting order. To sort lexically we would need something such
+as:
+
+        $ LANG=C sort some-file.txt >some-file-sorted.txt
+
+Examples
+========
+
+Compare two files::
+
+        $ ceph-diff-sorted fileA.txt fileB.txt
+
+Exit Status
+===========
+
+When complete, the exit status will be set to one of the following:
+
+0
+  files same
+1
+  files different
+2
+  usage problem (e.g., wrong number of command-line arguments)
+3
+  problem opening input file
+4
+  bad file content (e.g., unsorted order or empty lines)
+
+
+Availability
+============
+
+:program:`ceph-diff-sorted` is part of Ceph, a massively scalable,
+open-source, distributed storage system.  Please refer to the Ceph
+documentation at https://docs.ceph.com for more information.
+
+See also
+========
+
+:doc:`rgw-orphan-list <rgw-orphan-list>`\(8)
diff --git a/doc/man/8/ceph-fuse.rst b/doc/man/8/ceph-fuse.rst
new file mode 100644
index 000000000..6c4cc8399
--- /dev/null
+++ b/doc/man/8/ceph-fuse.rst
@@ -0,0 +1,97 @@
+:orphan:
+
+=========================================
+ ceph-fuse -- FUSE-based client for ceph
+=========================================
+
+.. program:: ceph-fuse
+
+Synopsis
+========
+
+| **ceph-fuse** [-n *client.username*] [ -m *monaddr*:*port* ] *mountpoint* [ *fuse options* ]
+
+
+Description
+===========
+
+**ceph-fuse** is a FUSE ("Filesystem in USErspace") client for Ceph
+distributed file system. It will mount a ceph file system specified via the -m
+option or described by ceph.conf (see below) at the specific mount point. See
+`Mount CephFS using FUSE`_ for detailed information.
+
+The file system can be unmounted with::
+
+        fusermount -u mountpoint
+
+or by sending ``SIGINT`` to the ``ceph-fuse`` process.
+
+
+Options
+=======
+
+Any options not recognized by ceph-fuse will be passed on to libfuse.
+
+.. option:: -o opt,[opt...]
+
+   Mount options.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: -n client.{cephx-username}
+
+   Pass the name of CephX user whose secret key is be to used for mounting.
+
+.. option:: --id <client-id>
+
+   Pass the name of CephX user whose secret key is be to used for mounting.
+   ``--id`` takes just the ID of the client in contrast to ``-n``. For
+   example, ``--id 0`` for using ``client.0``.
+
+.. option:: -k <path-to-keyring>
+
+   Provide path to keyring; useful when it's absent in standard locations.
+
+.. option:: --client_mountpoint/-r root_directory
+
+   Use root_directory as the mounted root, rather than the full Ceph tree.
+
+.. option:: -f
+
+   Foreground: do not daemonize after startup (run in foreground). Do not generate a pid file.
+
+.. option:: -d
+
+   Run in foreground, send all log output to stderr and enable FUSE debugging
+   (-o debug).
+
+.. option:: -s
+
+   Disable multi-threaded operation.
+
+.. option:: --client_fs
+
+   Pass the name of Ceph FS to be mounted. Not passing this option mounts the
+   default Ceph FS on the Ceph cluster.
+
+Availability
+============
+
+**ceph-fuse** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+fusermount(8),
+:doc:`ceph <ceph>`\(8)
+
+.. _Mount CephFS using FUSE: ../../../cephfs/mount-using-fuse/
diff --git a/doc/man/8/ceph-immutable-object-cache.rst b/doc/man/8/ceph-immutable-object-cache.rst
new file mode 100644
index 000000000..5ae968d2f
--- /dev/null
+++ b/doc/man/8/ceph-immutable-object-cache.rst
@@ -0,0 +1,76 @@
+:orphan:
+
+======================================================================
+ ceph-immutable-object-cache -- Ceph daemon for immutable object cache
+======================================================================
+
+.. program:: ceph-immutable-object-cache
+
+Synopsis
+========
+
+| **ceph-immutable-object-cache**
+
+
+Description
+===========
+
+:program:`ceph-immutable-object-cache` is a daemon for object cache of RADOS
+objects among Ceph clusters. It will promote the objects to a local directory
+upon promote requests and future reads will be serviced from these cached
+objects.
+
+It connects to local clusters via the RADOS protocol, relying on
+default search paths to find ceph.conf files, monitor addresses and
+authentication information for them, i.e. ``/etc/ceph/$cluster.conf``,
+``/etc/ceph/$cluster.keyring``, and
+``/etc/ceph/$cluster.$name.keyring``, where ``$cluster`` is the
+human-friendly name of the cluster, and ``$name`` is the rados user to
+connect as, e.g. ``client.ceph-immutable-object-cache``.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for ceph-immutable-object-cache
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.ceph-immutable-object-cache)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`ceph-immutable-object-cache` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com for
+more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/ceph-kvstore-tool.rst b/doc/man/8/ceph-kvstore-tool.rst
new file mode 100644
index 000000000..3c0ff56e9
--- /dev/null
+++ b/doc/man/8/ceph-kvstore-tool.rst
@@ -0,0 +1,101 @@
+:orphan:
+
+=====================================================
+ ceph-kvstore-tool -- ceph kvstore manipulation tool
+=====================================================
+
+.. program:: ceph-kvstore-tool
+
+Synopsis
+========
+
+| **ceph-kvstore-tool** <leveldb|rocksdb|bluestore-kv> <store path> *command* [args...]
+
+
+Description
+===========
+
+:program:`ceph-kvstore-tool` is a kvstore manipulation tool. It allows users to manipulate
+leveldb/rocksdb's data (like OSD's omap) offline.
+
+Commands
+========
+
+:program:`ceph-kvstore-tool` utility uses many commands for debugging purpose
+which are as follows:
+
+:command:`list [prefix]`
+    Print key of all KV pairs stored with the URL encoded prefix.
+
+:command:`list-crc [prefix]`
+    Print CRC of all KV pairs stored with the URL encoded prefix.
+
+:command:`dump [prefix]`
+    Print key and value of all KV pairs stored with the URL encoded prefix.
+
+:command:`exists <prefix> [key]`
+    Check if there is any KV pair stored with the URL encoded prefix. If key
+    is also specified, check for the key with the prefix instead.
+
+:command:`get <prefix> <key> [out <file>]`
+    Get the value of the KV pair stored with the URL encoded prefix and key.
+    If file is also specified, write the value to the file.
+
+:command:`crc <prefix> <key>`
+    Get the CRC of the KV pair stored with the URL encoded prefix and key. 
+
+:command:`get-size [<prefix> <key>]`
+    Get estimated store size or size of value specified by prefix and key.
+
+:command:`set <prefix> <key> [ver <N>|in <file>]`
+    Set the value of the KV pair stored with the URL encoded prefix and key. 
+    The value could be *version_t* or text.
+
+:command:`rm <prefix> <key>`
+    Remove the KV pair stored with the URL encoded prefix and key.
+
+:command:`rm-prefix <prefix>`
+    Remove all KV pairs stored with the URL encoded prefix.
+
+:command:`store-copy <path> [num-keys-per-tx]`
+    Copy all KV pairs to another directory specified by ``path``. 
+    [num-keys-per-tx] is the number of KV pairs copied for a transaction.
+
+:command:`store-crc <path>`
+    Store CRC of all KV pairs to a file specified by ``path``.
+
+:command:`compact`
+    Subcommand ``compact`` is used to compact all data of kvstore. It will open
+    the database, and trigger a database's compaction. After compaction, some 
+    disk space may be released.
+
+:command:`compact-prefix <prefix>`
+    Compact all entries specified by the URL encoded prefix. 
+   
+:command:`compact-range <prefix> <start> <end>`
+    Compact some entries specified by the URL encoded prefix and range.
+
+:command:`destructive-repair`
+    Make a (potentially destructive) effort to recover a corrupted database.
+    Note that in the case of rocksdb this may corrupt an otherwise uncorrupted
+    database--use this only as a last resort!
+
+:command:`stats`
+    Prints statistics from underlying key-value database. This is only for informative purposes.
+    Format and information content may vary between releases. For RocksDB information includes
+    compactions stats, performance counters, memory usage and internal RocksDB stats. 
+
+:command:`histogram`
+    Presents key-value sizes distribution statistics from the underlying KV database.
+
+Availability
+============
+
+**ceph-kvstore-tool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-mds.rst b/doc/man/8/ceph-mds.rst
new file mode 100644
index 000000000..fd8fc800e
--- /dev/null
+++ b/doc/man/8/ceph-mds.rst
@@ -0,0 +1,87 @@
+:orphan:
+
+=========================================
+ ceph-mds -- ceph metadata server daemon
+=========================================
+
+.. program:: ceph-mds
+
+Synopsis
+========
+
+| **ceph-mds** -i <*ID*> [flags]
+
+
+Description
+===========
+
+**ceph-mds** is the metadata server daemon for the Ceph distributed file
+system. One or more instances of ceph-mds collectively manage the file
+system namespace, coordinating access to the shared OSD cluster.
+
+Each ceph-mds daemon instance should have a unique name. The name is used
+to identify daemon instances in the ceph.conf.
+
+Once the daemon has started, the monitor cluster will normally assign
+it a logical rank, or put it in a standby pool to take over for
+another daemon that crashes. Some of the specified options can cause
+other behaviors.
+
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run
+   <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --id/-i ID
+
+   Set ID portion of the MDS name. The ID should not start with a numeric digit.
+
+.. option:: --name/-n TYPE.ID
+
+   Set the MDS name of the format TYPE.ID. The TYPE is obviously 'mds'.
+   The ID should not start with a numeric digit.
+
+Availability
+============
+
+**ceph-mds** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to the Ceph documentation at
+https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-mon.rst b/doc/man/8/ceph-mon.rst
new file mode 100644
index 000000000..640c842d6
--- /dev/null
+++ b/doc/man/8/ceph-mon.rst
@@ -0,0 +1,99 @@
+:orphan:
+
+=================================
+ ceph-mon -- ceph monitor daemon
+=================================
+
+.. program:: ceph-mon
+
+Synopsis
+========
+
+| **ceph-mon** -i *monid* [ --mon-data *mondatapath* ]
+
+
+Description
+===========
+
+**ceph-mon** is the cluster monitor daemon for the Ceph distributed
+file system. One or more instances of **ceph-mon** form a Paxos
+part-time parliament cluster that provides extremely reliable and
+durable storage of cluster membership, configuration, and state.
+
+The *mondatapath* refers to a directory on a local file system storing
+monitor data. It is normally specified via the ``mon data`` option in
+the configuration file.
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: --mkfs
+
+   Initialize the ``mon data`` directory with seed information to form
+   and initial ceph file system or to join an existing monitor
+   cluster.  Three pieces of information must be provided:
+
+   - The cluster fsid.  This can come from a monmap (``--monmap <path>``) or
+     explicitly via ``--fsid <uuid>``.
+   - A list of monitors and their addresses.  This list of monitors
+     can come from a monmap (``--monmap <path>``), the ``mon host``
+     configuration value (in *ceph.conf* or via ``-m
+     host1,host2,...``), or (for backward compatibility) the deprecated ``mon addr`` lines in *ceph.conf*.  If this
+     monitor is to be part of the initial monitor quorum for a new
+     Ceph cluster, then it must be included in the initial list,
+     matching either the name or address of a monitor in the list.
+     When matching by address, either the ``public addr`` or ``public
+     subnet`` options may be used.
+   - The monitor secret key ``mon.``.  This must be included in the
+     keyring provided via ``--keyring <path>``.
+
+.. option:: --keyring
+
+   Specify a keyring for use with ``--mkfs``.
+
+.. option:: --no-config-file
+
+    Signal that we don't want to rely on a *ceph.conf*, either user provided
+    or the default, to run the daemon.  This will entail providing all
+    necessary options to the daemon as arguments.
+
+Availability
+============
+
+**ceph-mon** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer
+to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-monstore-tool.rst b/doc/man/8/ceph-monstore-tool.rst
new file mode 100644
index 000000000..9396df0b6
--- /dev/null
+++ b/doc/man/8/ceph-monstore-tool.rst
@@ -0,0 +1,90 @@
+:orphan:
+
+======================================================
+ ceph-monstore-tool -- ceph monstore manipulation tool
+======================================================
+
+.. program:: ceph-monstore-tool
+
+Synopsis
+========
+
+| **ceph-monstore-tool** <store path> <cmd> [args|options]
+
+
+Description
+===========
+
+:program:`ceph-monstore-tool` is used to manipulate MonitorDBStore's data
+(monmap, osdmap, etc.) offline. It is similar to `ceph-kvstore-tool`.
+
+The default RocksDB debug level is `0`. This can be changed using `--debug`.
+
+Note:
+    Ceph-specific options take the format `--option-name=VAL`
+    DO NOT FORGET THE EQUALS SIGN. ('=')
+    Command-specific options must be passed after a `--`
+    for example, `get monmap --debug -- --version 10 --out /tmp/foo`
+
+Commands
+========
+
+:program:`ceph-monstore-tool` uses many commands for debugging purposes:
+
+:command:`store-copy <path>`
+    Copy the store to PATH.
+
+:command:`get monmap [-- options]`
+    Get monmap (version VER if specified) (default: last committed).
+
+:command:`get osdmap [-- options]`
+    Get osdmap (version VER if specified) (default: last committed).
+
+:command:`get msdmap [-- options]`
+    Get msdmap (version VER if specified) (default: last committed).
+
+:command:`get mgr [-- options]`
+    Get mgrmap (version VER if specified) (default: last committed).
+
+:command:`get crushmap [-- options]`
+    Get crushmap (version VER if specified) (default: last committed).
+
+:command:`get osd_snap <key> [-- options]`
+    Get osd_snap key (`purged_snap` or `purged_epoch`).
+
+:command:`dump-keys`
+    Dump store keys to FILE (default: stdout).
+
+:command:`dump-paxos [-- options]`
+    Dump Paxos transactions  (-- -- help for more info).
+
+:command:`dump-trace FILE  [-- options]`
+    Dump contents of trace file FILE (-- --help for more info).
+
+:command:`replay-trace FILE  [-- options]`
+    Replay trace from FILE (-- --help for more info).
+
+:command:`random-gen [-- options]`
+    Add randomly genererated ops to the store (-- --help for more info).
+
+:command:`rewrite-crush [-- options]`
+    Add a rewrite commit to the store
+
+:command:`rebuild`
+    Rebuild store.
+
+:command:`rm <prefix> <key>`
+    Remove specified key from the store.
+
+Availability
+============
+
+**ceph-monstore-tool** is part of Ceph, a massively scalable, open-source,
+distributed storage system. See the Ceph documentation at
+https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-objectstore-tool.rst b/doc/man/8/ceph-objectstore-tool.rst
new file mode 100644
index 000000000..de1304733
--- /dev/null
+++ b/doc/man/8/ceph-objectstore-tool.rst
@@ -0,0 +1,478 @@
+:orphan:
+
+==============================================================
+ceph-objectstore-tool -- modify or examine the state of an OSD
+==============================================================
+
+Synopsis
+========
+
+
+| **ceph-objectstore-tool** --data-path *path to osd* [--op *list* ]
+
+
+
+Possible object operations:
+
+* (get|set)-bytes [file]
+* set-(attr|omap) [file]
+* (get|rm)-attr|omap)
+* get-omaphdr
+* set-omaphdr [file]
+* list-attrs
+* list-omap
+* remove|removeall
+* dump
+* set-size
+* clear-data-digest
+* remove-clone-metadata 
+
+
+Description
+===========
+
+**ceph-objectstore-tool** is a tool for modifying the state of an OSD. It facilitates manipulating an object's content, removing an object, listing the omap, manipulating the omap header, manipulating the omap key, listing object attributes, and manipulating object attribute keys.
+
+**ceph-objectstore-tool** provides two main modes: (1) a mode that specifies the "--op" argument (for example, **ceph-objectstore-tool** --data-path $PATH_TO_OSD --op $SELECT_OPERATION [--pgid $PGID] [--dry-run]), and (2) a mode for positional object operations. If the second mode is used, the object can be specified by ID or by the JSON output of the --op list. 
+
+| **ceph-objectstore-tool** --data-path *path to osd* [--pgid *$PG_ID* ][--op *command*]
+| **ceph-objectstore-tool** --data-path *path to osd* [ --op *list $OBJECT_ID*]
+
+Possible -op commands::
+
+* info
+* log
+* remove
+* mkfs
+* fsck
+* repair
+* fuse
+* dup
+* export
+* export-remove
+* import
+* list
+* list-slow-omap
+* fix-lost
+* list-pgs
+* dump-super
+* meta-list
+* get-osdmap
+* set-osdmap
+* get-inc-osdmap
+* set-inc-osdmap
+* mark-complete
+* reset-last-complete
+* update-mon-db
+* dump-export
+* trim-pg-log
+
+Installation
+============
+
+The `ceph-osd` package provides **ceph-objectstore-tool**.
+
+
+Examples
+========
+
+Modifying Objects
+-----------------
+These commands modify state of an OSD. The OSD must not be running when ceph-objectstore-tool is used.
+
+Listing Objects and Placement Groups
+------------------------------------
+
+Make sure that the target OSD is down::
+
+   systemctl status ceph-osd@$OSD_NUMBER
+
+Identify all objects within an OSD::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op list
+
+Identify all objects within a placement group::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID --op list
+
+Identify the placement group (PG) that an object belongs to::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op list $OBJECT_ID
+
+
+Fixing Lost Objects   
+-------------------
+
+Make sure the OSD is down::
+
+   systemctl status ceph-osd@OSD_NUMBER
+
+Fix all lost objects::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost
+
+Fix all the lost objects within a specified placement group::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID --op fix-lost
+
+Fix a lost object by its identifier::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost $OBJECT_ID
+
+Fix legacy lost objects::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost
+
+
+Manipulating an object's content
+--------------------------------
+
+1. Make sure that the target OSD is down::
+   
+    systemctl status ceph-osd@$OSD_NUMBER
+
+2. Find the object by listing the objects of the OSD or placement group.
+
+3. Before setting the bytes on the object, make a backup and a working copy of the object. Here is the syntactic form of that command::
+   
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-bytes > $OBJECT_FILE_NAME
+
+For example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.backup
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.working-copy
+
+The first command creates the back-up copy, and the second command creates the working copy.
+
+4. Edit the working copy object file.
+
+5. Set the bytes of the object::
+     
+     ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-bytes < $OBJECT_FILE_NAME
+
+For example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-bytes < zone_info.default.working-copy
+ 
+
+Removing an Object
+------------------
+
+Use **ceph-objectstore-tool** to remove objects. When an object is removed, its contents and references are removed from the placement group (PG).
+
+Remove an object (syntax)::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT remove
+
+Remove an object (example)::
+
+[root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' remove
+
+
+Listing the Object Map
+----------------------
+
+Use the ceph-objectstore-tool to list the contents of the object map (OMAP). The output is a list of keys.
+
+
+1. Verify the appropriate OSD is down:
+
+   Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+   Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+2. List the object map:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT list-omap
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-omap
+
+
+Manipulating the Object Map Header
+----------------------------------
+The **ceph-objectstore-tool** utility will output the object map (OMAP) header with the values associated with the object's keys.
+
+Prerequisites
+^^^^^^^^^^^^^
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+^^^^^^^^^
+
+  Verify that the target OSD is down:
+
+  Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+  Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+  Get the object map header:
+
+  Syntax::
+
+        ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omaphdr > $OBJECT_MAP_FILE_NAME
+
+  Example::
+
+        [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omaphdr > zone_info.default.omaphdr.txt
+
+  Set the object map header:
+
+  Syntax::
+
+        ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omaphdr < $OBJECT_MAP_FILE_NAME
+
+  Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  set-omaphdr < zone_info.default.omaphdr.txt
+
+
+Manipulating the Object Map Key
+-------------------------------
+
+Use the **ceph-objectstore-tool** utility to change the object map (OMAP) key. You need to provide the data path, the placement group identifier (PG ID), the object, and the key in the OMAP.
+Note
+
+Prerequisites
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+
+    Get the object map key:
+
+    Syntax::
+     
+       ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omap $KEY > $OBJECT_MAP_FILE_NAME
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omap "" > zone_info.default.omap.txt
+
+   Set the object map key:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-omap $KEY < $OBJECT_MAP_FILE_NAME
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omap "" < zone_info.default.omap.txt
+
+   Remove the object map key:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-omap $KEY
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-omap ""
+
+
+Listing an Object's Attributes
+-------------------------------
+
+Use the **ceph-objectstore-tool** utility to list an object's attributes. The output provides you with the object's keys and values.
+Note
+
+Prerequisites
+^^^^^^^^^^^^^
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+^^^^^^^^^
+
+   Verify that the target OSD is down:
+
+   Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+   Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+   List the object's attributes:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT list-attrs
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-attrs
+
+
+MANIPULATING THE OBJECT ATTRIBUTE KEY
+-------------------------------------
+
+Use the ceph-objectstore-tool utility to change an object's attributes. To manipulate the object's attributes you need the data and journal paths, the placement group identifier (PG ID), the object, and the key in the object's attribute.
+Note
+
+Prerequisites
+
+    * Having root access to the Ceph OSD node.  
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+
+    Verify that the target OSD is down.
+
+ Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+ Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+ Get the object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-attrs $KEY > $OBJECT_ATTRS_FILE_NAME
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0  --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-attrs "oid" > zone_info.default.attr.txt
+
+ Set an object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT  set-attrs $KEY < $OBJECT_ATTRS_FILE_NAME
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-attrs "oid" < zone_info.default.attr.txt
+
+ Remove an object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-attrs $KEY
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-attrs "oid"
+
+
+Options
+=======
+
+.. option:: --help          
+
+   produce help message
+
+.. option:: --type arg        
+
+   Arg is one of [bluestore (default), memstore]. This option is needed only if the tool can't tell the type from --data-path.
+ 
+.. option:: --data-path arg
+
+   path to object store, mandatory
+   
+.. option:: --journal-path arg
+
+   path to journal, use if tool can't find it
+   
+.. option:: --pgid arg
+
+   PG id, mandatory for info, log, remove, export, export-remove, mark-complete, trim-pg-log
+                             
+.. option:: --pool arg
+
+   Pool name
+
+.. option:: --op arg
+
+   Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, fix-lost, list-pgs, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, update-mon-db, dump-export, trim-pg-log]
+
+.. option:: --epoch arg
+
+   epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified
+
+.. option:: --file arg             
+   
+   path of file to export, export-remove, import, get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap
+
+.. option:: --mon-store-path arg
+
+   path of monstore to update-mon-db
+
+.. option:: --fsid arg
+
+   fsid for new store created by mkfs
+
+.. option:: --target-data-path arg
+
+   path of target object store (for --op dup)
+   
+.. option:: --mountpoint arg
+
+   fuse mountpoint
+
+.. option:: --format arg (=json-pretty) 
+
+   Output format which may be json, json-pretty, xml, xml-pretty
+
+.. option:: --debug
+
+   Enable diagnostic output to stderr
+
+.. option:: --force
+
+   Ignore some types of errors and proceed with operation - USE WITH CAUTION: CORRUPTION POSSIBLE NOW OR IN THE FUTURE
+
+.. option:: --skip-journal-replay
+
+   Disable journal replay
+
+.. option:: --skip-mount-omap
+
+   Disable mounting of omap
+
+.. option:: --head
+
+   Find head/snapdir when searching for objects by name
+
+.. option:: --dry-run
+
+   Don't modify the objectstore
+
+.. option:: --namespace arg
+
+   Specify namespace when searching for objects
+
+.. option:: --rmtype arg      
+
+   Specify corrupting object removal 'snapmap' or 'nosnapmap' - TESTING USE ONLY
+
+
+
+Error Codes
+===========
+"Mount failed with '(11) Resource temporarily unavailable" - This might mean that you have attempted to run **ceph-objectstore-tool** on a running OSD.
+
+Availability
+============
+
+**ceph-objectstore-tool** is part of Ceph, a massively scalable, open-source, distributed storage system. **ceph-objectstore-tool** is provided by the package `ceph-osd`. Refer to the Ceph documentation at http://ceph.com/docs for more information.
diff --git a/doc/man/8/ceph-osd.rst b/doc/man/8/ceph-osd.rst
new file mode 100644
index 000000000..5b631eeff
--- /dev/null
+++ b/doc/man/8/ceph-osd.rst
@@ -0,0 +1,140 @@
+:orphan:
+
+.. _ceph_osd-daemon:
+
+========================================
+ ceph-osd -- ceph object storage daemon
+========================================
+
+.. program:: ceph-osd
+
+Synopsis
+========
+
+| **ceph-osd** -i *osdnum* [ --osd-data *datapath* ] [ --osd-journal
+  *journal* ] [ --mkfs ] [ --mkjournal ] [--flush-journal] [--check-allows-journal] [--check-wants-journal] [--check-needs-journal] [ --mkkey ] [ --osdspec-affinity ]
+
+
+Description
+===========
+
+**ceph-osd** is the object storage daemon for the Ceph distributed file
+system. It is responsible for storing objects on a local file system
+and providing access to them over the network.
+
+The datapath argument should be a directory on a xfs file system
+where the object data resides. The journal is optional, and is only
+useful performance-wise when it resides on a different disk than
+datapath with low latency (ideally, an NVRAM device).
+
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: --osd-data osddata
+
+   Use object store at *osddata*.
+
+.. option:: --osd-journal journal
+
+   Journal updates to *journal*.
+
+.. option:: --check-wants-journal
+
+   Check whether a journal is desired.
+
+.. option:: --check-allows-journal
+
+   Check whether a journal is allowed.
+
+.. option:: --check-needs-journal
+
+   Check whether a journal is required.
+
+.. option:: --mkfs
+
+   Create an empty object repository. This also initializes the journal
+   (if one is defined).
+
+.. option:: --mkkey
+
+   Generate a new secret key. This is normally used in combination
+   with ``--mkfs`` as it is more convenient than generating a key by
+   hand with :doc:`ceph-authtool <ceph-authtool>`\(8).
+
+.. option:: --mkjournal
+
+   Create a new journal file to match an existing object repository.
+   This is useful if the journal device or file is wiped out due to a
+   disk or file system failure.
+
+.. option:: --flush-journal
+
+   Flush the journal to permanent store. This runs in the foreground
+   so you know when it's completed. This can be useful if you want to
+   resize the journal or need to otherwise destroy it: this guarantees
+   you won't lose data.
+
+.. option:: --get-cluster-fsid
+
+   Print the cluster fsid (uuid) and exit.
+
+.. option:: --get-osd-fsid
+
+   Print the OSD's fsid and exit.  The OSD's uuid is generated at
+   --mkfs time and is thus unique to a particular instantiation of
+   this OSD.
+
+.. option:: --get-journal-fsid
+
+   Print the journal's uuid.  The journal fsid is set to match the OSD
+   fsid at --mkfs time.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` for runtime configuration options.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --osdspec-affinity
+
+   Set an affinity to a certain OSDSpec.
+   This option can only be used in conjunction with --mkfs.
+
+Availability
+============
+
+**ceph-osd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-authtool <ceph-authtool>`\(8)
diff --git a/doc/man/8/ceph-post-file.rst b/doc/man/8/ceph-post-file.rst
new file mode 100644
index 000000000..313c67456
--- /dev/null
+++ b/doc/man/8/ceph-post-file.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+==================================================
+ ceph-post-file -- post files for ceph developers
+==================================================
+
+.. program:: ceph-post-file
+
+Synopsis
+========
+
+| **ceph-post-file** [-d *description] [-u *user*] *file or dir* ...
+
+
+Description
+===========
+
+**ceph-post-file** will upload files or directories to ceph.com for
+later analysis by Ceph developers.
+
+Each invocation uploads files or directories to a separate directory
+with a unique tag.  That tag can be passed to a developer or
+referenced in a bug report (http://tracker.ceph.com/).  Once the
+upload completes, the directory is marked non-readable and
+non-writeable to prevent access or modification by other users.
+
+Warning
+=======
+
+Basic measures are taken to make posted data be visible only to
+developers with access to ceph.com infrastructure. However, users
+should think twice and/or take appropriate precautions before
+posting potentially sensitive data (for example, logs or data
+directories that contain Ceph secrets).
+
+
+Options
+=======
+
+.. option:: -d *description*, --description *description*
+
+   Add a short description for the upload.  This is a good opportunity
+   to reference a bug number.  There is no default value.
+
+.. option:: -u *user*
+
+   Set the user metadata for the upload.  This defaults to `whoami`@`hostname -f`.
+
+Examples
+========
+
+To upload a single log::
+
+   ceph-post-file /var/log/ceph/ceph-mon.`hostname`.log
+
+To upload several directories::
+
+   ceph-post-file -d 'mon data directories' /var/log/ceph/mon/*
+
+
+Availability
+============
+
+**ceph-post-file** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-debugpack <ceph-debugpack>`\(8),
diff --git a/doc/man/8/ceph-rbdnamer.rst b/doc/man/8/ceph-rbdnamer.rst
new file mode 100644
index 000000000..fbf235e06
--- /dev/null
+++ b/doc/man/8/ceph-rbdnamer.rst
@@ -0,0 +1,36 @@
+:orphan:
+
+==================================================
+ ceph-rbdnamer -- udev helper to name RBD devices
+==================================================
+
+.. program:: ceph-rbdnamer
+
+
+Synopsis
+========
+
+| **ceph-rbdnamer** *num*
+
+
+Description
+===========
+
+**ceph-rbdnamer** prints the pool, namespace, image and snapshot names
+for a given RBD device to stdout. It is used by `udev` device manager
+to set up RBD device symlinks. The appropriate `udev` rules are
+provided in a file named `50-rbd.rules`.
+
+Availability
+============
+
+**ceph-rbdnamer** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-run.rst b/doc/man/8/ceph-run.rst
new file mode 100644
index 000000000..18ca2eea4
--- /dev/null
+++ b/doc/man/8/ceph-run.rst
@@ -0,0 +1,45 @@
+:orphan:
+
+=========================================
+ ceph-run -- restart daemon on core dump
+=========================================
+
+.. program:: ceph-run
+
+Synopsis
+========
+
+| **ceph-run** *command* ...
+
+
+Description
+===========
+
+**ceph-run** is a simple wrapper that will restart a daemon if it exits
+with a signal indicating it crashed and possibly core dumped (that is,
+signals 3, 4, 5, 6, 8, or 11).
+
+The command should run the daemon in the foreground. For Ceph daemons,
+that means the ``-f`` option.
+
+
+Options
+=======
+
+None
+
+
+Availability
+============
+
+**ceph-run** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-syn.rst b/doc/man/8/ceph-syn.rst
new file mode 100644
index 000000000..888cdb3e1
--- /dev/null
+++ b/doc/man/8/ceph-syn.rst
@@ -0,0 +1,99 @@
+:orphan:
+
+===============================================
+ ceph-syn -- ceph synthetic workload generator
+===============================================
+
+.. program:: ceph-syn
+
+Synopsis
+========
+
+| **ceph-syn** [ -m *monaddr*:*port* ] --syn *command* *...*
+
+
+Description
+===========
+
+**ceph-syn** is a simple synthetic workload generator for the Ceph
+distributed file system. It uses the userspace client library to
+generate simple workloads against a currently running file system. The
+file system need not be mounted via ceph-fuse(8) or the kernel client.
+
+One or more ``--syn`` command arguments specify the particular
+workload, as documented below.
+
+
+Options
+=======
+
+.. option:: -d
+
+   Detach from console and daemonize after startup.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --num_client num
+
+   Run num different clients, each in a separate thread.
+
+.. option:: --syn workloadspec
+
+   Run the given workload. May be specified as many times as
+   needed. Workloads will normally run sequentially.
+
+
+Workloads
+=========
+
+Each workload should be preceded by ``--syn`` on the command
+line. This is not a complete list.
+
+:command:`mknap` *path* *snapname*
+  Create a snapshot called *snapname* on *path*.
+
+:command:`rmsnap` *path* *snapname*
+  Delete snapshot called *snapname* on *path*.
+
+:command:`rmfile` *path*
+  Delete/unlink *path*.
+
+:command:`writefile` *sizeinmb* *blocksize*
+  Create a file, named after our client id, that is *sizeinmb* MB by
+  writing *blocksize* chunks.
+
+:command:`readfile` *sizeinmb* *blocksize*
+  Read file, named after our client id, that is *sizeinmb* MB by
+  writing *blocksize* chunks.
+
+:command:`rw` *sizeinmb* *blocksize*
+  Write file, then read it back, as above.
+
+:command:`makedirs` *numsubdirs* *numfiles* *depth*
+  Create a hierarchy of directories that is *depth* levels deep. Give
+  each directory *numsubdirs* subdirectories and *numfiles* files.
+
+:command:`walk`
+  Recursively walk the file system (like find).
+
+
+Availability
+============
+
+**ceph-syn** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-fuse <ceph-fuse>`\(8)
diff --git a/doc/man/8/ceph-volume-systemd.rst b/doc/man/8/ceph-volume-systemd.rst
new file mode 100644
index 000000000..b8578796c
--- /dev/null
+++ b/doc/man/8/ceph-volume-systemd.rst
@@ -0,0 +1,55 @@
+:orphan:
+
+=======================================================
+ ceph-volume-systemd -- systemd ceph-volume helper tool
+=======================================================
+
+.. program:: ceph-volume-systemd
+
+Synopsis
+========
+
+| **ceph-volume-systemd** *systemd instance name*
+
+
+Description
+===========
+:program:`ceph-volume-systemd` is a systemd helper tool that receives input
+from (dynamically created) systemd units so that activation of OSDs can
+proceed.
+
+It translates the input into a system call to ceph-volume for activation
+purposes only.
+
+
+Examples
+========
+Its input is the ``systemd instance name`` (represented by ``%i`` in a systemd
+unit), and it should be in the following format::
+
+    <ceph-volume subcommand>-<extra metadata>
+
+In the case of ``lvm`` a call could look like::
+
+    /usr/bin/ceph-volume-systemd lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Which in turn will call ``ceph-volume`` in the following way::
+
+    ceph-volume lvm trigger  0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Any other subcommand will need to have implemented a ``trigger`` command that
+can consume the extra metadata in this format.
+
+
+Availability
+============
+
+:program:`ceph-volume-systemd` is part of Ceph, a massively scalable,
+open-source, distributed storage system. Please refer to the documentation at
+http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8),
diff --git a/doc/man/8/ceph-volume.rst b/doc/man/8/ceph-volume.rst
new file mode 100644
index 000000000..8254cf400
--- /dev/null
+++ b/doc/man/8/ceph-volume.rst
@@ -0,0 +1,569 @@
+:orphan:
+
+=======================================================
+ ceph-volume -- Ceph OSD deployment and inspection tool
+=======================================================
+
+.. program:: ceph-volume
+
+Synopsis
+========
+
+**ceph-volume** [-h] [--cluster CLUSTER] [--log-level LOG_LEVEL]
+[--log-path LOG_PATH]
+
+**ceph-volume** **inventory**
+
+**ceph-volume** **lvm** [ *trigger* | *create* | *activate* | *prepare*
+| *zap* | *list* | *batch* | *new-wal* | *new-db* | *migrate* ]
+
+**ceph-volume** **simple** [ *trigger* | *scan* | *activate* ]
+
+
+Description
+===========
+
+:program:`ceph-volume` is a single purpose command line tool to deploy logical
+volumes as OSDs, trying to maintain a similar API to ``ceph-disk`` when
+preparing, activating, and creating OSDs.
+
+It deviates from ``ceph-disk`` by not interacting or relying on the udev rules
+that come installed for Ceph. These rules allow automatic detection of
+previously setup devices that are in turn fed into ``ceph-disk`` to activate
+them.
+
+
+Commands
+========
+
+inventory
+---------
+
+.. program:: ceph-volume inventory
+
+This subcommand provides information about a host's physical disc inventory and
+reports metadata about these discs. Among this metadata one can find disc
+specific data items (like model, size, rotational or solid state) as well as
+data items specific to ceph using a device, such as if it is available for
+use with ceph or if logical volumes are present.
+
+Examples::
+
+    ceph-volume inventory
+    ceph-volume inventory /dev/sda
+    ceph-volume inventory --format json-pretty
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --format
+
+   report format, valid values are ``plain`` (default),
+   ``json`` and ``json-pretty``
+
+lvm
+---
+
+.. program:: ceph-volume lvm
+
+By making use of LVM tags, the ``lvm`` sub-command is able to store and later
+re-discover and query devices associated with OSDs so that they can later
+activated.
+
+Subcommands:
+
+batch
+^^^^^
+
+.. program:: ceph-volume lvm batch
+
+Creates OSDs from a list of devices using a ``bluestore`` (default) setup. 
+It will create all necessary volume groups and logical volumes required to have a working OSD.
+
+Example usage with three devices::
+
+    ceph-volume lvm batch --bluestore /dev/sda /dev/sdb /dev/sdc
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --bluestore
+
+   Use the bluestore objectstore (default)
+
+.. option:: --yes
+
+   Skip the report and prompt to continue provisioning
+
+.. option:: --prepare
+
+   Only prepare OSDs, do not activate
+
+.. option:: --dmcrypt
+
+   Enable encryption for the underlying OSD devices
+
+.. option:: --crush-device-class
+
+   Define a CRUSH device class to assign the OSD to
+
+.. option:: --no-systemd
+
+   Do not enable or create any systemd units
+
+.. option:: --osds-per-device
+
+   Provision more than 1 (the default) OSD per device
+
+.. option:: --report
+
+   Report what the potential outcome would be for the current input (requires devices
+   to be passed in)
+
+.. option:: --format
+
+   Output format when reporting (used along with --report), can be one of 'pretty'
+   (default) or 'json'
+
+.. option:: --block-db-size
+
+   Set (or override) the "bluestore_block_db_size" value, in bytes
+
+.. option:: --journal-size
+
+   Override the "osd_journal_size" value, in megabytes
+
+Required positional arguments:
+
+.. describe:: <DEVICE>
+
+   Full path to a raw device, like ``/dev/sda``. Multiple
+   ``<DEVICE>`` paths can be passed in.
+
+activate
+^^^^^^^^
+
+.. program:: ceph-volume lvm activate
+
+Enables a systemd unit that persists the OSD ID and its UUID (also called
+``fsid`` in Ceph CLI tools), so that at boot time it can understand what OSD is
+enabled and needs to be mounted.
+
+Usage::
+
+    ceph-volume lvm activate --bluestore <osd id> <osd fsid>
+
+Optional Arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --auto-detect-objectstore
+
+   Automatically detect the objectstore by inspecting
+   the OSD
+
+.. option:: --bluestore
+
+   bluestore objectstore (default)
+
+.. option:: --all
+
+   Activate all OSDs found in the system
+
+.. option:: --no-systemd
+
+   Skip creating and enabling systemd units and starting of OSD
+   services
+
+Multiple OSDs can be activated at once by using the (idempotent) ``--all`` flag::
+
+    ceph-volume lvm activate --all
+
+
+prepare
+^^^^^^^
+
+.. program:: ceph-volume lvm prepare
+
+Prepares a logical volume to be used as an OSD and journal using a ``bluestore`` (default) setup. 
+It will not create or modify the logical volumes except for adding extra metadata.
+
+Usage::
+
+    ceph-volume lvm prepare --bluestore --data <data lv> --journal <journal device>
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --journal JOURNAL
+
+   logical group name, path to a logical volume, or path to a device
+
+.. option:: --bluestore
+
+   Use the bluestore objectstore (default)
+
+.. option:: --block.wal
+
+   Path to a bluestore block.wal logical volume or partition
+
+.. option:: --block.db
+
+   Path to a bluestore block.db logical volume or partition
+
+.. option:: --dmcrypt
+
+   Enable encryption for the underlying OSD devices
+
+.. option:: --osd-id OSD_ID
+
+   Reuse an existing OSD id
+
+.. option:: --osd-fsid OSD_FSID
+
+   Reuse an existing OSD fsid
+
+.. option:: --crush-device-class
+
+   Define a CRUSH device class to assign the OSD to
+
+Required arguments:
+
+.. option:: --data
+
+   A logical group name or a path to a logical volume
+
+For encrypting an OSD, the ``--dmcrypt`` flag must be added when preparing
+(also supported in the ``create`` sub-command).
+
+
+create
+^^^^^^
+
+Wraps the two-step process to provision a new osd (calling ``prepare`` first
+and then ``activate``) into a single one. The reason to prefer ``prepare`` and
+then ``activate`` is to gradually introduce new OSDs into a cluster, and
+avoiding large amounts of data being rebalanced.
+
+The single-call process unifies exactly what ``prepare`` and ``activate`` do,
+with the convenience of doing it all at once. Flags and general usage are
+equivalent to those of the ``prepare`` and ``activate`` subcommand.
+
+trigger
+^^^^^^^
+
+This subcommand is not meant to be used directly, and it is used by systemd so
+that it proxies input to ``ceph-volume lvm activate`` by parsing the
+input from systemd, detecting the UUID and ID associated with an OSD.
+
+Usage::
+
+    ceph-volume lvm trigger <SYSTEMD-DATA>
+
+The systemd "data" is expected to be in the format of::
+
+    <OSD ID>-<OSD UUID>
+
+The lvs associated with the OSD need to have been prepared previously,
+so that all needed tags and metadata exist.
+
+Positional arguments:
+
+.. describe:: <SYSTEMD_DATA>
+
+   Data from a systemd unit containing ID and UUID of the OSD.
+
+list
+^^^^
+
+List devices or logical volumes associated with Ceph. An association is
+determined if a device has information relating to an OSD. This is
+verified by querying LVM's metadata and correlating it with devices.
+
+The lvs associated with the OSD need to have been prepared previously by
+ceph-volume so that all needed tags and metadata exist.
+
+Usage::
+
+    ceph-volume lvm list
+
+List a particular device, reporting all metadata about it::
+
+    ceph-volume lvm list /dev/sda1
+
+List a logical volume, along with all its metadata (vg is a volume
+group, and lv the logical volume name)::
+
+    ceph-volume lvm list {vg/lv}
+
+Positional arguments:
+
+.. describe:: <DEVICE>
+
+   Either in the form of ``vg/lv`` for logical volumes,
+   ``/path/to/sda1`` or ``/path/to/sda`` for regular devices.
+
+
+zap
+^^^
+
+Zaps the given logical volume or partition. If given a path to a logical
+volume it must be in the format of vg/lv. Any file systems present
+on the given lv or partition will be removed and all data will be purged.
+
+However, the lv or partition will be kept intact.
+
+Usage, for logical volumes::
+
+      ceph-volume lvm zap {vg/lv}
+
+Usage, for logical partitions::
+
+      ceph-volume lvm zap /dev/sdc1
+
+For full removal of the device use the ``--destroy`` flag (allowed for all
+device types)::
+
+      ceph-volume lvm zap --destroy /dev/sdc1
+
+Multiple devices can be removed by specifying the OSD ID and/or the OSD FSID::
+
+      ceph-volume lvm zap --destroy --osd-id 1
+      ceph-volume lvm zap --destroy --osd-id 1 --osd-fsid C9605912-8395-4D76-AFC0-7DFDAC315D59
+
+
+Positional arguments:
+
+.. describe:: <DEVICE>
+
+   Either in the form of ``vg/lv`` for logical volumes,
+   ``/path/to/sda1`` or ``/path/to/sda`` for regular devices.
+
+
+new-wal
+^^^^^^^
+
+.. program:: ceph-volume lvm new-wal
+
+Attaches the given logical volume to OSD as a WAL. Logical volume
+name format is vg/lv. Fails if OSD has already got attached WAL.
+
+Usage::
+
+    ceph-volume lvm new-wal --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv>
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --target
+
+   logical volume name to attach as WAL
+
+new-db
+^^^^^^
+
+.. program:: ceph-volume lvm new-db
+
+Attaches the given logical volume to OSD as a DB. Logical volume
+name format is vg/lv. Fails if OSD has already got attached DB.
+
+Usage::
+
+    ceph-volume lvm new-db --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv>
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --target
+
+   logical volume name to attach as DB
+
+migrate
+^^^^^^^
+
+.. program:: ceph-volume lvm migrate
+
+Moves BlueFS data from source volume(s) to the target one, source volumes
+(except the main, i.e. data or block one) are removed on success. LVM volumes
+are permitted for Target only, both already attached or new one. In the latter
+case it is attached to the OSD replacing one of the source devices. Following
+replacement rules apply (in the order of precedence, stop on the first match):
+
+    - if source list has DB volume - target device replaces it.
+    - if source list has WAL volume - target device replace it.
+    - if source list has slow volume only - operation is not permitted,
+      requires explicit allocation via new-db/new-wal command.
+
+Usage::
+
+    ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv> --from {data|db|wal} [{data|db|wal} ...]
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --from
+
+   list of source device type names
+
+.. option:: --target
+
+   logical volume to move data to
+
+simple
+------
+
+Scan legacy OSD directories or data devices that may have been created by
+ceph-disk, or manually.
+
+Subcommands:
+
+activate
+^^^^^^^^
+
+.. program:: ceph-volume simple activate
+
+Enables a systemd unit that persists the OSD ID and its UUID (also called
+``fsid`` in Ceph CLI tools), so that at boot time it can understand what OSD is
+enabled and needs to be mounted, while reading information that was previously
+created and persisted at ``/etc/ceph/osd/`` in JSON format.
+
+Usage::
+
+    ceph-volume simple activate --bluestore <osd id> <osd fsid>
+
+Optional Arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --bluestore
+
+   bluestore objectstore (default)
+
+.. note::
+
+   It requires a matching JSON file with the following format::
+
+    /etc/ceph/osd/<osd id>-<osd fsid>.json
+
+
+scan
+^^^^
+
+.. program:: ceph-volume simple scan
+
+Scan a running OSD or data device for an OSD for metadata that can later be
+used to activate and manage the OSD with ceph-volume. The scan method will
+create a JSON file with the required information plus anything found in the OSD
+directory as well.
+
+Optionally, the JSON blob can be sent to stdout for further inspection.
+
+Usage on all running OSDs::
+
+    ceph-volume simple scan
+
+Usage on data devices::
+
+    ceph-volume simple scan <data device>
+
+Running OSD directories::
+
+    ceph-volume simple scan <path to osd dir>
+
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --stdout
+
+   Send the JSON blob to stdout
+
+.. option:: --force
+
+   If the JSON file exists at destination, overwrite it
+
+Optional Positional arguments:
+
+.. describe:: <DATA DEVICE or OSD DIR>
+
+   Actual data partition or a path to the running OSD
+
+trigger
+^^^^^^^
+
+This subcommand is not meant to be used directly, and it is used by systemd so
+that it proxies input to ``ceph-volume simple activate`` by parsing the
+input from systemd, detecting the UUID and ID associated with an OSD.
+
+Usage::
+
+    ceph-volume simple trigger <SYSTEMD-DATA>
+
+The systemd "data" is expected to be in the format of::
+
+    <OSD ID>-<OSD UUID>
+
+The JSON file associated with the OSD need to have been persisted previously by
+a scan (or manually), so that all needed metadata can be used.
+
+Positional arguments:
+
+.. describe:: <SYSTEMD_DATA>
+
+   Data from a systemd unit containing ID and UUID of the OSD.
+
+
+Availability
+============
+
+:program:`ceph-volume` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the documentation at http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8),
diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst
new file mode 100644
index 000000000..881c0d307
--- /dev/null
+++ b/doc/man/8/ceph.rst
@@ -0,0 +1,1666 @@
+:orphan:
+
+==================================
+ ceph -- ceph administration tool
+==================================
+
+.. program:: ceph
+
+Synopsis
+========
+
+| **ceph** **auth** [ *add* \| *caps* \| *del* \| *export* \| *get* \| *get-key* \| *get-or-create* \| *get-or-create-key* \| *import* \| *list* \| *print-key* \| *print_key* ] ...
+
+| **ceph** **compact**
+
+| **ceph** **config** [ *dump* | *ls* | *help* | *get* | *show* | *show-with-defaults* | *set* | *rm* | *log* | *reset* | *assimilate-conf* | *generate-minimal-conf* ] ...
+
+| **ceph** **config-key** [ *rm* | *exists* | *get* | *ls* | *dump* | *set* ] ...
+
+| **ceph** **daemon** *<name>* \| *<path>* *<command>* ...
+
+| **ceph** **daemonperf** *<name>* \| *<path>* [ *interval* [ *count* ] ]
+
+| **ceph** **df** *{detail}*
+
+| **ceph** **fs** [ *ls* \| *new* \| *reset* \| *rm* \| *authorize* ] ...
+
+| **ceph** **fsid**
+
+| **ceph** **health** *{detail}*
+
+| **ceph** **injectargs** *<injectedargs>* [ *<injectedargs>*... ]
+
+| **ceph** **log** *<logtext>* [ *<logtext>*... ]
+
+| **ceph** **mds** [ *compat* \| *fail* \| *rm* \| *rmfailed* \| *set_state* \| *stat* \| *repaired* ] ...
+
+| **ceph** **mon** [ *add* \| *dump* \| *enable_stretch_mode* \| *getmap* \| *remove* \| *stat* ] ...
+
+| **ceph** **osd** [ *blocklist* \| *blocked-by* \| *create* \| *new* \| *deep-scrub* \| *df* \| *down* \| *dump* \| *erasure-code-profile* \| *find* \| *getcrushmap* \| *getmap* \| *getmaxosd* \| *in* \| *ls* \| *lspools* \| *map* \| *metadata* \| *ok-to-stop* \| *out* \| *pause* \| *perf* \| *pg-temp* \| *force-create-pg* \| *primary-affinity* \| *primary-temp* \| *repair* \| *reweight* \| *reweight-by-pg* \| *rm* \| *destroy* \| *purge* \| *safe-to-destroy* \| *scrub* \| *set* \| *setcrushmap* \| *setmaxosd*  \| *stat* \| *tree* \| *unpause* \| *unset* ] ...
+
+| **ceph** **osd** **crush** [ *add* \| *add-bucket* \| *create-or-move* \| *dump* \| *get-tunable* \| *link* \| *move* \| *remove* \| *rename-bucket* \| *reweight* \| *reweight-all* \| *reweight-subtree* \| *rm* \| *rule* \| *set* \| *set-tunable* \| *show-tunables* \| *tunables* \| *unlink* ] ...
+
+| **ceph** **osd** **pool** [ *create* \| *delete* \| *get* \| *get-quota* \| *ls* \| *mksnap* \| *rename* \| *rmsnap* \| *set* \| *set-quota* \| *stats* ] ...
+
+| **ceph** **osd** **pool** **application** [ *disable* \| *enable* \| *get* \| *rm* \| *set* ] ...
+
+| **ceph** **osd** **tier** [ *add* \| *add-cache* \| *cache-mode* \| *remove* \| *remove-overlay* \| *set-overlay* ] ...
+
+| **ceph** **pg** [ *debug* \| *deep-scrub* \| *dump* \| *dump_json* \| *dump_pools_json* \| *dump_stuck* \| *getmap* \| *ls* \| *ls-by-osd* \| *ls-by-pool* \| *ls-by-primary* \| *map* \| *repair* \| *scrub* \| *stat* ] ...
+
+| **ceph** **quorum_status**
+
+| **ceph** **report** { *<tags>* [ *<tags>...* ] }
+
+| **ceph** **status**
+
+| **ceph** **sync** **force** {--yes-i-really-mean-it} {--i-know-what-i-am-doing}
+
+| **ceph** **tell** *<name (type.id)> <command> [options...]*
+
+| **ceph** **version**
+
+Description
+===========
+
+:program:`ceph` is a control utility which is used for manual deployment and maintenance
+of a Ceph cluster. It provides a diverse set of commands that allows deployment of
+monitors, OSDs, placement groups, MDS and overall maintenance, administration
+of the cluster.
+
+Commands
+========
+
+auth
+----
+
+Manage authentication keys. It is used for adding, removing, exporting
+or updating of authentication keys for a particular  entity such as a monitor or
+OSD. It uses some additional subcommands.
+
+Subcommand ``add`` adds authentication info for a particular entity from input
+file, or random key if no input is given and/or any caps specified in the command.
+
+Usage::
+
+	ceph auth add <entity> {<caps> [<caps>...]}
+
+Subcommand ``caps`` updates caps for **name** from caps specified in the command.
+
+Usage::
+
+	ceph auth caps <entity> <caps> [<caps>...]
+
+Subcommand ``del`` deletes all caps for ``name``.
+
+Usage::
+
+	ceph auth del <entity>
+
+Subcommand ``export`` writes keyring for requested entity, or master keyring if
+none given.
+
+Usage::
+
+	ceph auth export {<entity>}
+
+Subcommand ``get`` writes keyring file with requested key.
+
+Usage::
+
+	ceph auth get <entity>
+
+Subcommand ``get-key`` displays requested key.
+
+Usage::
+
+	ceph auth get-key <entity>
+
+Subcommand ``get-or-create`` adds authentication info for a particular entity
+from input file, or random key if no input given and/or any caps specified in the
+command.
+
+Usage::
+
+	ceph auth get-or-create <entity> {<caps> [<caps>...]}
+
+Subcommand ``get-or-create-key`` gets or adds key for ``name`` from system/caps
+pairs specified in the command.  If key already exists, any given caps must match
+the existing caps for that key.
+
+Usage::
+
+	ceph auth get-or-create-key <entity> {<caps> [<caps>...]}
+
+Subcommand ``import`` reads keyring from input file.
+
+Usage::
+
+	ceph auth import
+
+Subcommand ``ls`` lists authentication state.
+
+Usage::
+
+	ceph auth ls
+
+Subcommand ``print-key`` displays requested key.
+
+Usage::
+
+	ceph auth print-key <entity>
+
+Subcommand ``print_key`` displays requested key.
+
+Usage::
+
+	ceph auth print_key <entity>
+
+
+compact
+-------
+
+Causes compaction of monitor's leveldb storage.
+
+Usage::
+
+	ceph compact
+
+
+config
+------
+
+Configure the cluster. By default, Ceph daemons and clients retrieve their
+configuration options from monitor when they start, and are updated if any of
+the tracked options is changed at run time. It uses following additional
+subcommand.
+
+Subcommand ``dump`` to dump all options for the cluster
+
+Usage::
+
+	ceph config dump
+
+Subcommand ``ls`` to list all option names for the cluster
+
+Usage::
+
+	ceph config ls
+
+Subcommand ``help`` to describe the specified configuration option
+
+Usage::
+
+    ceph config help <option>
+
+Subcommand ``get`` to dump the option(s) for the specified entity.
+
+Usage::
+
+    ceph config get <who> {<option>}
+
+Subcommand ``show`` to display the running configuration of the specified
+entity. Please note, unlike ``get``, which only shows the options managed
+by monitor, ``show`` displays all the configurations being actively used.
+These options are pulled from several sources, for instance, the compiled-in
+default value, the monitor's configuration database, ``ceph.conf`` file on
+the host. The options can even be overridden at runtime. So, there is chance
+that the configuration options in the output of ``show`` could be different
+from those in the output of ``get``.
+
+Usage::
+
+	ceph config show {<who>}
+
+Subcommand ``show-with-defaults`` to display the running configuration along with the compiled-in defaults of the specified entity
+
+Usage::
+
+	ceph config show {<who>}
+
+Subcommand ``set`` to set an option for one or more specified entities
+
+Usage::
+
+    ceph config set <who> <option> <value> {--force}
+
+Subcommand ``rm`` to clear an option for one or more entities
+
+Usage::
+
+    ceph config rm <who> <option>
+
+Subcommand ``log`` to show recent history of config changes. If `count` option
+is omitted it defaults to 10.
+
+Usage::
+
+    ceph config log {<count>}
+
+Subcommand ``reset`` to revert configuration to the specified historical version
+
+Usage::
+
+    ceph config reset <version>
+
+
+Subcommand ``assimilate-conf`` to assimilate options from stdin, and return a
+new, minimal conf file
+
+Usage::
+
+    ceph config assimilate-conf -i <input-config-path> > <output-config-path>
+    ceph config assimilate-conf < <input-config-path>
+
+Subcommand ``generate-minimal-conf`` to generate a minimal ``ceph.conf`` file,
+which can be used for bootstrapping a daemon or a client.
+
+Usage::
+
+    ceph config generate-minimal-conf > <minimal-config-path>
+
+
+config-key
+----------
+
+Manage configuration key. Config-key is a general purpose key/value service
+offered by the monitors. This service is mainly used by Ceph tools and daemons
+for persisting various settings. Among which, ceph-mgr modules uses it for
+storing their options. It uses some additional subcommands.
+
+Subcommand ``rm`` deletes configuration key.
+
+Usage::
+
+	ceph config-key rm <key>
+
+Subcommand ``exists`` checks for configuration keys existence.
+
+Usage::
+
+	ceph config-key exists <key>
+
+Subcommand ``get`` gets the configuration key.
+
+Usage::
+
+	ceph config-key get <key>
+
+Subcommand ``ls`` lists configuration keys.
+
+Usage::
+
+	ceph config-key ls
+
+Subcommand ``dump`` dumps configuration keys and values.
+
+Usage::
+
+	ceph config-key dump
+
+Subcommand ``set`` puts configuration key and value.
+
+Usage::
+
+	ceph config-key set <key> {<val>}
+
+
+daemon
+------
+
+Submit admin-socket commands.
+
+Usage::
+
+	ceph daemon {daemon_name|socket_path} {command} ...
+
+Example::
+
+	ceph daemon osd.0 help
+
+
+daemonperf
+----------
+
+Watch performance counters from a Ceph daemon.
+
+Usage::
+
+	ceph daemonperf {daemon_name|socket_path} [{interval} [{count}]]
+
+
+df
+--
+
+Show cluster's free space status.
+
+Usage::
+
+	ceph df {detail}
+
+.. _ceph features:
+
+features
+--------
+
+Show the releases and features of all connected daemons and clients connected
+to the cluster, along with the numbers of them in each bucket grouped by the
+corresponding features/releases. Each release of Ceph supports a different set
+of features, expressed by the features bitmask. New cluster features require
+that clients support the feature, or else they are not allowed to connect to
+these new features. As new features or capabilities are enabled after an
+upgrade, older clients are prevented from connecting.
+
+Usage::
+
+    ceph features
+
+fs
+--
+
+Manage cephfs file systems. It uses some additional subcommands.
+
+Subcommand ``ls`` to list file systems
+
+Usage::
+
+	ceph fs ls
+
+Subcommand ``new`` to make a new file system using named pools <metadata> and <data>
+
+Usage::
+
+	ceph fs new <fs_name> <metadata> <data>
+
+Subcommand ``reset`` is used for disaster recovery only: reset to a single-MDS map
+
+Usage::
+
+	ceph fs reset <fs_name> {--yes-i-really-mean-it}
+
+Subcommand ``rm`` to disable the named file system
+
+Usage::
+
+	ceph fs rm <fs_name> {--yes-i-really-mean-it}
+
+Subcommand ``authorize`` creates a new client that will be authorized for the
+given path in ``<fs_name>``. Pass ``/`` to authorize for the entire FS.
+``<perms>`` below can be ``r``, ``rw`` or ``rwp``.
+
+Usage::
+
+    ceph fs authorize <fs_name> client.<client_id> <path> <perms> [<path> <perms>...]
+
+fsid
+----
+
+Show cluster's FSID/UUID.
+
+Usage::
+
+	ceph fsid
+
+
+health
+------
+
+Show cluster's health.
+
+Usage::
+
+	ceph health {detail}
+
+
+heap
+----
+
+Show heap usage info (available only if compiled with tcmalloc)
+
+Usage::
+
+	ceph tell <name (type.id)> heap dump|start_profiler|stop_profiler|stats
+
+Subcommand ``release`` to make TCMalloc to releases no-longer-used memory back to the kernel at once. 
+
+Usage::
+
+	ceph tell <name (type.id)> heap release
+
+Subcommand ``(get|set)_release_rate`` get or set the TCMalloc memory release rate. TCMalloc releases 
+no-longer-used memory back to the kernel gradually. the rate controls how quickly this happens. 
+Increase this setting to make TCMalloc to return unused memory more frequently. 0 means never return
+memory to system, 1 means wait for 1000 pages after releasing a page to system. It is ``1.0`` by default..
+
+Usage::
+
+	ceph tell <name (type.id)> heap get_release_rate|set_release_rate {<val>}
+
+injectargs
+----------
+
+Inject configuration arguments into monitor.
+
+Usage::
+
+	ceph injectargs <injected_args> [<injected_args>...]
+
+
+log
+---
+
+Log supplied text to the monitor log.
+
+Usage::
+
+	ceph log <logtext> [<logtext>...]
+
+
+mds
+---
+
+Manage metadata server configuration and administration. It uses some
+additional subcommands.
+
+Subcommand ``compat`` manages compatible features. It uses some additional
+subcommands.
+
+Subcommand ``rm_compat`` removes compatible feature.
+
+Usage::
+
+	ceph mds compat rm_compat <int[0-]>
+
+Subcommand ``rm_incompat`` removes incompatible feature.
+
+Usage::
+
+	ceph mds compat rm_incompat <int[0-]>
+
+Subcommand ``show`` shows mds compatibility settings.
+
+Usage::
+
+	ceph mds compat show
+
+Subcommand ``fail`` forces mds to status fail.
+
+Usage::
+
+	ceph mds fail <role|gid>
+
+Subcommand ``rm`` removes inactive mds.
+
+Usage::
+
+	ceph mds rm <int[0-]> <name> (type.id)>
+
+Subcommand ``rmfailed`` removes failed mds.
+
+Usage::
+
+	ceph mds rmfailed <int[0-]>
+
+Subcommand ``set_state`` sets mds state of <gid> to <numeric-state>.
+
+Usage::
+
+	ceph mds set_state <int[0-]> <int[0-20]>
+
+Subcommand ``stat`` shows MDS status.
+
+Usage::
+
+	ceph mds stat
+
+Subcommand ``repaired`` mark a damaged MDS rank as no longer damaged.
+
+Usage::
+
+	ceph mds repaired <role>
+
+mon
+---
+
+Manage monitor configuration and administration. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds new monitor named <name> at <addr>.
+
+Usage::
+
+	ceph mon add <name> <IPaddr[:port]>
+
+Subcommand ``dump`` dumps formatted monmap (optionally from epoch)
+
+Usage::
+
+	ceph mon dump {<int[0-]>}
+
+Subcommand ``getmap`` gets monmap.
+
+Usage::
+
+	ceph mon getmap {<int[0-]>}
+
+Subcommand ``enable_stretch_mode`` enables stretch mode, changing the peering
+rules and failure handling on all pools. For a given PG to successfully peer
+and be marked active, ``min_size`` replicas will now need to be active under all
+(currently two) CRUSH buckets of type <dividing_bucket>.
+
+<tiebreaker_mon> is the tiebreaker mon to use if a network split happens.
+
+<dividing_bucket> is the bucket type across which to stretch.
+This will typically be ``datacenter`` or other CRUSH hierarchy bucket type that
+denotes physically or logically distant subdivisions.
+
+<new_crush_rule> will be set as CRUSH rule for all pools.
+
+Usage::
+
+	ceph mon enable_stretch_mode <tiebreaker_mon> <new_crush_rule> <dividing_bucket>
+
+Subcommand ``remove`` removes monitor named <name>.
+
+Usage::
+
+	ceph mon remove <name>
+
+Subcommand ``stat`` summarizes monitor status.
+
+Usage::
+
+	ceph mon stat
+
+mgr
+---
+
+Ceph manager daemon configuration and management.
+
+Subcommand ``dump`` dumps the latest MgrMap, which describes the active
+and standby manager daemons.
+
+Usage::
+
+  ceph mgr dump
+
+Subcommand ``fail`` will mark a manager daemon as failed, removing it
+from the manager map.  If it is the active manager daemon a standby
+will take its place.
+
+Usage::
+
+  ceph mgr fail <name>
+
+Subcommand ``module ls`` will list currently enabled manager modules (plugins).
+
+Usage::
+
+  ceph mgr module ls
+
+Subcommand ``module enable`` will enable a manager module.  Available modules are included in MgrMap and visible via ``mgr dump``.
+
+Usage::
+
+  ceph mgr module enable <module>
+
+Subcommand ``module disable`` will disable an active manager module.
+
+Usage::
+
+  ceph mgr module disable <module>
+
+Subcommand ``metadata`` will report metadata about all manager daemons or, if the name is specified, a single manager daemon.
+
+Usage::
+
+  ceph mgr metadata [name]
+
+Subcommand ``versions`` will report a count of running daemon versions.
+
+Usage::
+
+  ceph mgr versions
+
+Subcommand ``count-metadata`` will report a count of any daemon metadata field.
+
+Usage::
+
+  ceph mgr count-metadata <field>
+
+.. _ceph-admin-osd:
+
+osd
+---
+
+Manage OSD configuration and administration. It uses some additional
+subcommands.
+
+Subcommand ``blocklist`` manage blocklisted clients. It uses some additional
+subcommands.
+
+Subcommand ``add`` add <addr> to blocklist (optionally until <expire> seconds
+from now)
+
+Usage::
+
+	ceph osd blocklist add <EntityAddr> {<float[0.0-]>}
+
+Subcommand ``ls`` show blocklisted clients
+
+Usage::
+
+	ceph osd blocklist ls
+
+Subcommand ``rm`` remove <addr> from blocklist
+
+Usage::
+
+	ceph osd blocklist rm <EntityAddr>
+
+Subcommand ``blocked-by`` prints a histogram of which OSDs are blocking their peers
+
+Usage::
+
+	ceph osd blocked-by
+
+Subcommand ``create`` creates new osd (with optional UUID and ID).
+
+This command is DEPRECATED as of the Luminous release, and will be removed in
+a future release.
+
+Subcommand ``new`` should instead be used.
+
+Usage::
+
+	ceph osd create {<uuid>} {<id>}
+
+Subcommand ``new`` can be used to create a new OSD or to recreate a previously
+destroyed OSD with a specific *id*. The new OSD will have the specified *uuid*,
+and the command expects a JSON file containing the base64 cephx key for auth
+entity *client.osd.<id>*, as well as optional base64 cepx key for dm-crypt
+lockbox access and a dm-crypt key. Specifying a dm-crypt requires specifying
+the accompanying lockbox cephx key.
+
+Usage::
+
+    ceph osd new {<uuid>} {<id>} -i {<params.json>}
+
+The parameters JSON file is optional but if provided, is expected to maintain
+a form of the following format::
+
+    {
+        "cephx_secret": "AQBWtwhZdBO5ExAAIDyjK2Bh16ZXylmzgYYEjg==",
+	"crush_device_class": "myclass"
+    }
+
+Or::
+
+    {
+        "cephx_secret": "AQBWtwhZdBO5ExAAIDyjK2Bh16ZXylmzgYYEjg==",
+        "cephx_lockbox_secret": "AQDNCglZuaeVCRAAYr76PzR1Anh7A0jswkODIQ==",
+        "dmcrypt_key": "<dm-crypt key>",
+	"crush_device_class": "myclass"
+    }
+
+Or::
+
+    {
+	"crush_device_class": "myclass"
+    }
+
+The "crush_device_class" property is optional. If specified, it will set the
+initial CRUSH device class for the new OSD.
+
+
+Subcommand ``crush`` is used for CRUSH management. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds or updates crushmap position and weight for <name> with
+<weight> and location <args>.
+
+Usage::
+
+	ceph osd crush add <osdname (id|osd.id)> <float[0.0-]> <args> [<args>...]
+
+Subcommand ``add-bucket`` adds no-parent (probably root) crush bucket <name> of
+type <type>.
+
+Usage::
+
+	ceph osd crush add-bucket <name> <type>
+
+Subcommand ``create-or-move`` creates entry or moves existing entry for <name>
+<weight> at/to location <args>.
+
+Usage::
+
+	ceph osd crush create-or-move <osdname (id|osd.id)> <float[0.0-]> <args>
+	[<args>...]
+
+Subcommand ``dump`` dumps crush map.
+
+Usage::
+
+	ceph osd crush dump
+
+Subcommand ``get-tunable`` get crush tunable straw_calc_version
+
+Usage::
+
+	ceph osd crush get-tunable straw_calc_version
+
+Subcommand ``link`` links existing entry for <name> under location <args>.
+
+Usage::
+
+	ceph osd crush link <name> <args> [<args>...]
+
+Subcommand ``move`` moves existing entry for <name> to location <args>.
+
+Usage::
+
+	ceph osd crush move <name> <args> [<args>...]
+
+Subcommand ``remove`` removes <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush remove <name> {<ancestor>}
+
+Subcommand ``rename-bucket`` renames bucket <srcname> to <dstname>
+
+Usage::
+
+	ceph osd crush rename-bucket <srcname> <dstname>
+
+Subcommand ``reweight`` change <name>'s weight to <weight> in crush map.
+
+Usage::
+
+	ceph osd crush reweight <name> <float[0.0-]>
+
+Subcommand ``reweight-all`` recalculate the weights for the tree to
+ensure they sum correctly
+
+Usage::
+
+	ceph osd crush reweight-all
+
+Subcommand ``reweight-subtree`` changes all leaf items beneath <name>
+to <weight> in crush map
+
+Usage::
+
+	ceph osd crush reweight-subtree <name> <weight>
+
+Subcommand ``rm`` removes <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush rm <name> {<ancestor>}
+
+Subcommand ``rule`` is used for creating crush rules. It uses some additional
+subcommands.
+
+Subcommand ``create-erasure`` creates crush rule <name> for erasure coded pool
+created with <profile> (default default).
+
+Usage::
+
+	ceph osd crush rule create-erasure <name> {<profile>}
+
+Subcommand ``create-simple`` creates crush rule <name> to start from <root>,
+replicate across buckets of type <type>, using a choose mode of <firstn|indep>
+(default firstn; indep best for erasure pools).
+
+Usage::
+
+	ceph osd crush rule create-simple <name> <root> <type> {firstn|indep}
+
+Subcommand ``dump`` dumps crush rule <name> (default all).
+
+Usage::
+
+	ceph osd crush rule dump {<name>}
+
+Subcommand ``ls`` lists crush rules.
+
+Usage::
+
+	ceph osd crush rule ls
+
+Subcommand ``rm`` removes crush rule <name>.
+
+Usage::
+
+	ceph osd crush rule rm <name>
+
+Subcommand ``set`` used alone, sets crush map from input file.
+
+Usage::
+
+	ceph osd crush set
+
+Subcommand ``set`` with osdname/osd.id update crushmap position and weight
+for <name> to <weight> with location <args>.
+
+Usage::
+
+	ceph osd crush set <osdname (id|osd.id)> <float[0.0-]> <args> [<args>...]
+
+Subcommand ``set-tunable`` set crush tunable <tunable> to <value>.  The only
+tunable that can be set is straw_calc_version.
+
+Usage::
+
+	ceph osd crush set-tunable straw_calc_version <value>
+
+Subcommand ``show-tunables`` shows current crush tunables.
+
+Usage::
+
+	ceph osd crush show-tunables
+
+Subcommand ``tree`` shows the crush buckets and items in a tree view.
+
+Usage::
+
+	ceph osd crush tree
+
+Subcommand ``tunables`` sets crush tunables values to <profile>.
+
+Usage::
+
+	ceph osd crush tunables legacy|argonaut|bobtail|firefly|hammer|optimal|default
+
+Subcommand ``unlink`` unlinks <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush unlink <name> {<ancestor>}
+
+Subcommand ``df`` shows OSD utilization
+
+Usage::
+
+	ceph osd df {plain|tree}
+
+Subcommand ``deep-scrub`` initiates deep scrub on specified osd.
+
+Usage::
+
+	ceph osd deep-scrub <who>
+
+Subcommand ``down`` sets osd(s) <id> [<id>...] down.
+
+Usage::
+
+	ceph osd down <ids> [<ids>...]
+
+Subcommand ``dump`` prints summary of OSD map.
+
+Usage::
+
+	ceph osd dump {<int[0-]>}
+
+Subcommand ``erasure-code-profile`` is used for managing the erasure code
+profiles. It uses some additional subcommands.
+
+Subcommand ``get`` gets erasure code profile <name>.
+
+Usage::
+
+	ceph osd erasure-code-profile get <name>
+
+Subcommand ``ls`` lists all erasure code profiles.
+
+Usage::
+
+	ceph osd erasure-code-profile ls
+
+Subcommand ``rm`` removes erasure code profile <name>.
+
+Usage::
+
+	ceph osd erasure-code-profile rm <name>
+
+Subcommand ``set`` creates erasure code profile <name> with [<key[=value]> ...]
+pairs. Add a --force at the end to override an existing profile (IT IS RISKY).
+
+Usage::
+
+	ceph osd erasure-code-profile set <name> {<profile> [<profile>...]}
+
+Subcommand ``find`` find osd <id> in the CRUSH map and shows its location.
+
+Usage::
+
+	ceph osd find <int[0-]>
+
+Subcommand ``getcrushmap`` gets CRUSH map.
+
+Usage::
+
+	ceph osd getcrushmap {<int[0-]>}
+
+Subcommand ``getmap`` gets OSD map.
+
+Usage::
+
+	ceph osd getmap {<int[0-]>}
+
+Subcommand ``getmaxosd`` shows largest OSD id.
+
+Usage::
+
+	ceph osd getmaxosd
+
+Subcommand ``in`` sets osd(s) <id> [<id>...] in.
+
+Usage::
+
+	ceph osd in <ids> [<ids>...]
+
+Subcommand ``lost`` marks osd as permanently lost. THIS DESTROYS DATA IF NO
+MORE REPLICAS EXIST, BE CAREFUL.
+
+Usage::
+
+	ceph osd lost <int[0-]> {--yes-i-really-mean-it}
+
+Subcommand ``ls`` shows all OSD ids.
+
+Usage::
+
+	ceph osd ls {<int[0-]>}
+
+Subcommand ``lspools`` lists pools.
+
+Usage::
+
+	ceph osd lspools {<int>}
+
+Subcommand ``map`` finds pg for <object> in <pool>.
+
+Usage::
+
+	ceph osd map <poolname> <objectname>
+
+Subcommand ``metadata`` fetches metadata for osd <id>.
+
+Usage::
+
+	ceph osd metadata {int[0-]} (default all)
+
+Subcommand ``out`` sets osd(s) <id> [<id>...] out.
+
+Usage::
+
+	ceph osd out <ids> [<ids>...]
+
+Subcommand ``ok-to-stop`` checks whether the list of OSD(s) can be
+stopped without immediately making data unavailable.  That is, all
+data should remain readable and writeable, although data redundancy
+may be reduced as some PGs may end up in a degraded (but active)
+state.  It will return a success code if it is okay to stop the
+OSD(s), or an error code and informative message if it is not or if no
+conclusion can be drawn at the current time.  When ``--max <num>`` is
+provided, up to <num> OSDs IDs will return (including the provided
+OSDs) that can all be stopped simultaneously.  This allows larger sets
+of stoppable OSDs to be generated easily by providing a single
+starting OSD and a max.  Additional OSDs are drawn from adjacent locations
+in the CRUSH hierarchy.
+
+Usage::
+
+  ceph osd ok-to-stop <id> [<ids>...] [--max <num>]
+
+Subcommand ``pause`` pauses osd.
+
+Usage::
+
+	ceph osd pause
+
+Subcommand ``perf`` prints dump of OSD perf summary stats.
+
+Usage::
+
+	ceph osd perf
+
+Subcommand ``pg-temp`` set pg_temp mapping pgid:[<id> [<id>...]] (developers
+only).
+
+Usage::
+
+	ceph osd pg-temp <pgid> {<id> [<id>...]}
+
+Subcommand ``force-create-pg`` forces creation of pg <pgid>.
+
+Usage::
+
+	ceph osd force-create-pg <pgid>
+
+
+Subcommand ``pool`` is used for managing data pools. It uses some additional
+subcommands.
+
+Subcommand ``create`` creates pool.
+
+Usage::
+
+	ceph osd pool create <poolname> {<int[0-]>} {<int[0-]>} {replicated|erasure}
+	{<erasure_code_profile>} {<rule>} {<int>} {--autoscale-mode=<on,off,warn>}
+
+Subcommand ``delete`` deletes pool.
+
+Usage::
+
+	ceph osd pool delete <poolname> {<poolname>} {--yes-i-really-really-mean-it}
+
+Subcommand ``get`` gets pool parameter <var>.
+
+Usage::
+
+	ceph osd pool get <poolname> size|min_size|pg_num|pgp_num|crush_rule|write_fadvise_dontneed
+
+Only for tiered pools::
+
+	ceph osd pool get <poolname> hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|
+	target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|
+	cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|
+	min_read_recency_for_promote|hit_set_grade_decay_rate|hit_set_search_last_n
+
+Only for erasure coded pools::
+
+	ceph osd pool get <poolname> erasure_code_profile
+
+Use ``all`` to get all pool parameters that apply to the pool's type::
+
+	ceph osd pool get <poolname> all
+
+Subcommand ``get-quota`` obtains object or byte limits for pool.
+
+Usage::
+
+	ceph osd pool get-quota <poolname>
+
+Subcommand ``ls`` list pools
+
+Usage::
+
+	ceph osd pool ls {detail}
+
+Subcommand ``mksnap`` makes snapshot <snap> in <pool>.
+
+Usage::
+
+	ceph osd pool mksnap <poolname> <snap>
+
+Subcommand ``rename`` renames <srcpool> to <destpool>.
+
+Usage::
+
+	ceph osd pool rename <poolname> <poolname>
+
+Subcommand ``rmsnap`` removes snapshot <snap> from <pool>.
+
+Usage::
+
+	ceph osd pool rmsnap <poolname> <snap>
+
+Subcommand ``set`` sets pool parameter <var> to <val>.
+
+Usage::
+
+	ceph osd pool set <poolname> size|min_size|pg_num|
+	pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|
+	hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool|
+	target_max_bytes|target_max_objects|cache_target_dirty_ratio|
+	cache_target_dirty_high_ratio|
+	cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|
+	min_read_recency_for_promote|write_fadvise_dontneed|hit_set_grade_decay_rate|
+	hit_set_search_last_n
+	<val> {--yes-i-really-mean-it}
+
+Subcommand ``set-quota`` sets object or byte limit on pool.
+
+Usage::
+
+	ceph osd pool set-quota <poolname> max_objects|max_bytes <val>
+
+Subcommand ``stats`` obtain stats from all pools, or from specified pool.
+
+Usage::
+
+	ceph osd pool stats {<name>}
+
+Subcommand ``application`` is used for adding an annotation to the given
+pool. By default, the possible applications are object, block, and file
+storage (corresponding app-names are "rgw", "rbd", and "cephfs"). However,
+there might be other applications as well. Based on the application, there
+may or may not be some processing conducted.
+
+Subcommand ``disable`` disables the given application on the given pool.
+
+Usage::
+
+        ceph osd pool application disable <pool-name> <app> {--yes-i-really-mean-it}
+
+Subcommand ``enable`` adds an annotation to the given pool for the mentioned
+application.
+
+Usage::
+
+        ceph osd pool application enable <pool-name> <app> {--yes-i-really-mean-it}
+
+Subcommand ``get`` displays the value for the given key that is associated
+with the given application of the given pool. Not passing the optional
+arguments would display all key-value pairs for all applications for all
+pools.
+
+Usage::
+
+        ceph osd pool application get {<pool-name>} {<app>} {<key>}
+
+Subcommand ``rm`` removes the key-value pair for the given key in the given
+application of the given pool.
+
+Usage::
+
+        ceph osd pool application rm <pool-name> <app> <key>
+
+Subcommand ``set`` associates or updates, if it already exists, a key-value
+pair with the given application for the given pool.
+
+Usage::
+
+        ceph osd pool application set <pool-name> <app> <key> <value>
+
+Subcommand ``primary-affinity`` adjust osd primary-affinity from 0.0 <=<weight>
+<= 1.0
+
+Usage::
+
+	ceph osd primary-affinity <osdname (id|osd.id)> <float[0.0-1.0]>
+
+Subcommand ``primary-temp`` sets primary_temp mapping pgid:<id>|-1 (developers
+only).
+
+Usage::
+
+	ceph osd primary-temp <pgid> <id>
+
+Subcommand ``repair`` initiates repair on a specified osd.
+
+Usage::
+
+	ceph osd repair <who>
+
+Subcommand ``reweight`` reweights osd to 0.0 < <weight> < 1.0.
+
+Usage::
+
+	osd reweight <int[0-]> <float[0.0-1.0]>
+
+Subcommand ``reweight-by-pg`` reweight OSDs by PG distribution
+[overload-percentage-for-consideration, default 120].
+
+Usage::
+
+	ceph osd reweight-by-pg {<int[100-]>} {<poolname> [<poolname...]}
+	{--no-increasing}
+
+Subcommand ``reweight-by-utilization`` reweights OSDs by utilization.  It only reweights
+outlier OSDs whose utilization exceeds the average, eg. the default 120%
+limits reweight to those OSDs that are more than 20% over the average.
+[overload-threshold, default 120 [max_weight_change, default 0.05 [max_osds_to_adjust, default 4]]] 
+
+Usage::
+
+	ceph osd reweight-by-utilization {<int[100-]> {<float[0.0-]> {<int[0-]>}}}
+	{--no-increasing}
+
+Subcommand ``rm`` removes osd(s) <id> [<id>...] from the OSD map.
+
+
+Usage::
+
+	ceph osd rm <ids> [<ids>...]
+
+Subcommand ``destroy`` marks OSD *id* as *destroyed*, removing its cephx
+entity's keys and all of its dm-crypt and daemon-private config key
+entries.
+
+This command will not remove the OSD from crush, nor will it remove the
+OSD from the OSD map. Instead, once the command successfully completes,
+the OSD will show marked as *destroyed*.
+
+In order to mark an OSD as destroyed, the OSD must first be marked as
+**lost**.
+
+Usage::
+
+    ceph osd destroy <id> {--yes-i-really-mean-it}
+
+
+Subcommand ``purge`` performs a combination of ``osd destroy``,
+``osd rm`` and ``osd crush remove``.
+
+Usage::
+
+    ceph osd purge <id> {--yes-i-really-mean-it}
+
+Subcommand ``safe-to-destroy`` checks whether it is safe to remove or
+destroy an OSD without reducing overall data redundancy or durability.
+It will return a success code if it is definitely safe, or an error
+code and informative message if it is not or if no conclusion can be
+drawn at the current time.
+
+Usage::
+
+  ceph osd safe-to-destroy <id> [<ids>...]
+
+Subcommand ``scrub`` initiates scrub on specified osd.
+
+Usage::
+
+	ceph osd scrub <who>
+
+Subcommand ``set`` sets cluster-wide <flag> by updating OSD map.
+The ``full`` flag is not honored anymore since the Mimic release, and
+``ceph osd set full`` is not supported in the Octopus release.
+
+Usage::
+
+	ceph osd set pause|noup|nodown|noout|noin|nobackfill|
+	norebalance|norecover|noscrub|nodeep-scrub|notieragent
+
+Subcommand ``setcrushmap`` sets crush map from input file.
+
+Usage::
+
+	ceph osd setcrushmap
+
+Subcommand ``setmaxosd`` sets new maximum osd value.
+
+Usage::
+
+	ceph osd setmaxosd <int[0-]>
+
+Subcommand ``set-require-min-compat-client`` enforces the cluster to be backward
+compatible with the specified client version. This subcommand prevents you from
+making any changes (e.g., crush tunables, or using new features) that
+would violate the current setting. Please note, This subcommand will fail if
+any connected daemon or client is not compatible with the features offered by
+the given <version>. To see the features and releases of all clients connected
+to cluster, please see `ceph features`_.
+
+Usage::
+
+    ceph osd set-require-min-compat-client <version>
+
+Subcommand ``stat`` prints summary of OSD map.
+
+Usage::
+
+	ceph osd stat
+
+Subcommand ``tier`` is used for managing tiers. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds the tier <tierpool> (the second one) to base pool <pool>
+(the first one).
+
+Usage::
+
+	ceph osd tier add <poolname> <poolname> {--force-nonempty}
+
+Subcommand ``add-cache`` adds a cache <tierpool> (the second one) of size <size>
+to existing pool <pool> (the first one).
+
+Usage::
+
+	ceph osd tier add-cache <poolname> <poolname> <int[0-]>
+
+Subcommand ``cache-mode`` specifies the caching mode for cache tier <pool>.
+
+Usage::
+
+	ceph osd tier cache-mode <poolname> writeback|proxy|readproxy|readonly|none
+
+Subcommand ``remove`` removes the tier <tierpool> (the second one) from base pool
+<pool> (the first one).
+
+Usage::
+
+	ceph osd tier remove <poolname> <poolname>
+
+Subcommand ``remove-overlay`` removes the overlay pool for base pool <pool>.
+
+Usage::
+
+	ceph osd tier remove-overlay <poolname>
+
+Subcommand ``set-overlay`` set the overlay pool for base pool <pool> to be
+<overlaypool>.
+
+Usage::
+
+	ceph osd tier set-overlay <poolname> <poolname>
+
+Subcommand ``tree`` prints OSD tree.
+
+Usage::
+
+	ceph osd tree {<int[0-]>}
+
+Subcommand ``unpause`` unpauses osd.
+
+Usage::
+
+	ceph osd unpause
+
+Subcommand ``unset`` unsets cluster-wide <flag> by updating OSD map.
+
+Usage::
+
+	ceph osd unset pause|noup|nodown|noout|noin|nobackfill|
+	norebalance|norecover|noscrub|nodeep-scrub|notieragent
+
+
+pg
+--
+
+It is used for managing the placement groups in OSDs. It uses some
+additional subcommands.
+
+Subcommand ``debug`` shows debug info about pgs.
+
+Usage::
+
+	ceph pg debug unfound_objects_exist|degraded_pgs_exist
+
+Subcommand ``deep-scrub`` starts deep-scrub on <pgid>.
+
+Usage::
+
+	ceph pg deep-scrub <pgid>
+
+Subcommand ``dump`` shows human-readable versions of pg map (only 'all' valid
+with plain).
+
+Usage::
+
+	ceph pg dump {all|summary|sum|delta|pools|osds|pgs|pgs_brief} [{all|summary|sum|delta|pools|osds|pgs|pgs_brief...]}
+
+Subcommand ``dump_json`` shows human-readable version of pg map in json only.
+
+Usage::
+
+	ceph pg dump_json {all|summary|sum|delta|pools|osds|pgs|pgs_brief} [{all|summary|sum|delta|pools|osds|pgs|pgs_brief...]}
+
+Subcommand ``dump_pools_json`` shows pg pools info in json only.
+
+Usage::
+
+	ceph pg dump_pools_json
+
+Subcommand ``dump_stuck`` shows information about stuck pgs.
+
+Usage::
+
+	ceph pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]}
+	{<int>}
+
+Subcommand ``getmap`` gets binary pg map to -o/stdout.
+
+Usage::
+
+	ceph pg getmap
+
+Subcommand ``ls`` lists pg with specific pool, osd, state
+
+Usage::
+
+	ceph pg ls {<int>} {<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-osd`` lists pg on osd [osd]
+
+Usage::
+
+	ceph pg ls-by-osd <osdname (id|osd.id)> {<int>}
+	{<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-pool`` lists pg with pool = [poolname]
+
+Usage::
+
+	ceph pg ls-by-pool <poolstr> {<int>} {<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-primary`` lists pg with primary = [osd]
+
+Usage::
+
+	ceph pg ls-by-primary <osdname (id|osd.id)> {<int>}
+	{<pg-state> [<pg-state>...]}
+
+Subcommand ``map`` shows mapping of pg to osds.
+
+Usage::
+
+	ceph pg map <pgid>
+
+Subcommand ``repair`` starts repair on <pgid>.
+
+Usage::
+
+	ceph pg repair <pgid>
+
+Subcommand ``scrub`` starts scrub on <pgid>.
+
+Usage::
+
+	ceph pg scrub <pgid>
+
+Subcommand ``stat`` shows placement group status.
+
+Usage::
+
+	ceph pg stat
+
+
+quorum
+------
+
+Cause a specific MON to enter or exit quorum.
+
+Usage::
+
+	ceph tell mon.<id> quorum enter|exit
+
+quorum_status
+-------------
+
+Reports status of monitor quorum.
+
+Usage::
+
+	ceph quorum_status
+
+
+report
+------
+
+Reports full status of cluster, optional title tag strings.
+
+Usage::
+
+	ceph report {<tags> [<tags>...]}
+
+
+status
+------
+
+Shows cluster status.
+
+Usage::
+
+	ceph status
+
+
+tell
+----
+
+Sends a command to a specific daemon.
+
+Usage::
+
+	ceph tell <name (type.id)> <command> [options...]
+
+
+List all available commands.
+
+Usage::
+
+ 	ceph tell <name (type.id)> help
+
+version
+-------
+
+Show mon daemon version
+
+Usage::
+
+	ceph version
+
+Options
+=======
+
+.. option:: -i infile
+
+   will specify an input file to be passed along as a payload with the
+   command to the monitor cluster. This is only used for specific
+   monitor commands.
+
+.. option:: -o outfile
+
+   will write any payload returned by the monitor cluster with its
+   reply to outfile.  Only specific monitor commands (e.g. osd getmap)
+   return a payload.
+
+.. option:: --setuser user
+
+   will apply the appropriate user ownership to the file specified by
+   the option '-o'.
+
+.. option:: --setgroup group
+
+   will apply the appropriate group ownership to the file specified by
+   the option '-o'.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ceph.conf configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: --id CLIENT_ID, --user CLIENT_ID
+
+   Client id for authentication.
+
+.. option:: --name CLIENT_NAME, -n CLIENT_NAME
+
+	Client name for authentication.
+
+.. option:: --cluster CLUSTER
+
+	Name of the Ceph cluster.
+
+.. option:: --admin-daemon ADMIN_SOCKET, daemon DAEMON_NAME
+
+	Submit admin-socket commands via admin sockets in /var/run/ceph.
+
+.. option:: --admin-socket ADMIN_SOCKET_NOPE
+
+	You probably mean --admin-daemon
+
+.. option:: -s, --status
+
+	Show cluster status.
+
+.. option:: -w, --watch
+
+	Watch live cluster changes on the default 'cluster' channel
+
+.. option:: -W, --watch-channel
+
+	Watch live cluster changes on any channel (cluster, audit, cephadm, or * for all)
+
+.. option:: --watch-debug
+
+	Watch debug events.
+
+.. option:: --watch-info
+
+	Watch info events.
+
+.. option:: --watch-sec
+
+	Watch security events.
+
+.. option:: --watch-warn
+
+	Watch warning events.
+
+.. option:: --watch-error
+
+	Watch error events.
+
+.. option:: --version, -v
+
+	Display version.
+
+.. option:: --verbose
+
+	Make verbose.
+
+.. option:: --concise
+
+	Make less verbose.
+
+.. option:: -f {json,json-pretty,xml,xml-pretty,plain,yaml}, --format
+
+	Format of output. Note: yaml is only valid for orch commands. 
+
+.. option:: --connect-timeout CLUSTER_TIMEOUT
+
+	Set a timeout for connecting to the cluster.
+
+.. option:: --no-increasing
+
+	 ``--no-increasing`` is off by default. So increasing the osd weight is allowed
+         using the ``reweight-by-utilization`` or ``test-reweight-by-utilization`` commands.
+         If this option is used with these commands, it will help not to increase osd weight
+         even the osd is under utilized.
+
+.. option:: --block
+
+	 block until completion (scrub and deep-scrub only)
+
+Availability
+============
+
+:program:`ceph` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8)
diff --git a/doc/man/8/cephadm.rst b/doc/man/8/cephadm.rst
new file mode 100644
index 000000000..0847066b6
--- /dev/null
+++ b/doc/man/8/cephadm.rst
@@ -0,0 +1,540 @@
+:orphan:
+
+=========================================
+ cephadm -- manage the local cephadm host
+=========================================
+
+.. program:: cephadm
+
+Synopsis
+========
+
+| **cephadm**** [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR]
+|               [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR]
+|               [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT]
+|               [--retry RETRY] [--no-container-init]
+|               {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install}
+|               ...
+
+
+| **cephadm** **pull**
+
+| **cephadm** --image IMAGE_NAME **inspect-image**
+
+| **cephadm** **ls** [-h] [--no-detail] [--legacy-dir LEGACY_DIR]
+
+| **cephadm** **list-networks**
+
+| **cephadm** **adopt** [-h] --name NAME --style STYLE [--cluster CLUSTER]
+|                       [--legacy-dir LEGACY_DIR] [--config-json CONFIG_JSON]
+|                       [--skip-firewalld] [--skip-pull]
+
+| **cephadm** **rm-daemon** [-h] --name NAME --fsid FSID [--force]
+|                           [--force-delete-data]
+
+| **cephadm** **rm-cluster** [-h] --fsid FSID [--force]
+
+| **cephadm** **run** [-h] --name NAME --fsid FSID
+
+| **cephadm** **shell** [-h] [--fsid FSID] [--name NAME] [--config CONFIG]
+                        [--keyring KEYRING] --mount [MOUNT [MOUNT ...]] [--env ENV]
+                        [--] [command [command ...]]
+
+| **cephadm** **enter** [-h] [--fsid FSID] --name NAME [command [command ...]]
+
+| **cephadm** **ceph-volume** [-h] [--fsid FSID] [--config-json CONFIG_JSON]
+                              [--config CONFIG] [--keyring KEYRING]
+                              command [command ...]
+
+| **cephadm** **unit**  [-h] [--fsid FSID] --name NAME command
+
+| **cephadm** **logs** [-h] [--fsid FSID] --name NAME [command [command ...]]
+
+| **cephadm** **bootstrap** [-h] [--config CONFIG] [--mon-id MON_ID]
+|                           [--mon-addrv MON_ADDRV] [--mon-ip MON_IP]
+|                           [--mgr-id MGR_ID] [--fsid FSID]
+|                           [--log-to-file] [--single-host-defaults]
+|                           [--output-dir OUTPUT_DIR]
+|                           [--output-keyring OUTPUT_KEYRING]
+|                           [--output-config OUTPUT_CONFIG]
+|                           [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY]
+|                           [--skip-ssh]
+|                           [--initial-dashboard-user INITIAL_DASHBOARD_USER]
+|                           [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD]
+|                           [--ssl-dashboard-port SSL_DASHBOARD_PORT]
+|                           [--dashboard-key DASHBOARD_KEY]
+|                           [--dashboard-crt DASHBOARD_CRT]
+|                           [--ssh-config SSH_CONFIG]
+|                           [--ssh-private-key SSH_PRIVATE_KEY]
+|                           [--ssh-public-key SSH_PUBLIC_KEY]
+|                           [--ssh-user SSH_USER] [--skip-mon-network]
+|                           [--skip-dashboard] [--dashboard-password-noupdate]
+|                           [--no-minimize-config] [--skip-ping-check]
+|                           [--skip-pull] [--skip-firewalld] [--allow-overwrite]
+|                           [--allow-fqdn-hostname] [--skip-prepare-host]
+|                           [--orphan-initial-daemons] [--skip-monitoring-stack]
+|                           [--apply-spec APPLY_SPEC]
+|                           [--registry-url REGISTRY_URL]
+|                           [--registry-username REGISTRY_USERNAME]
+|                           [--registry-password REGISTRY_PASSWORD]
+|                           [--registry-json REGISTRY_JSON]
+
+
+
+| **cephadm** **deploy** [-h] --name NAME --fsid FSID [--config CONFIG]
+|                        [--config-json CONFIG_JSON] [--keyring KEYRING]
+|                        [--key KEY] [--osd-fsid OSD_FSID] [--skip-firewalld]
+|                        [--tcp-ports TCP_PORTS] [--reconfig] [--allow-ptrace]
+
+| **cephadm** **check-host** [-h] [--expect-hostname EXPECT_HOSTNAME]
+
+| **cephadm** **prepare-host**
+
+| **cephadm** **add-repo** [-h] [--release RELEASE] [--version VERSION]
+|                          [--dev DEV] [--dev-commit DEV_COMMIT]
+|                          [--gpg-url GPG_URL] [--repo-url REPO_URL]
+
+
+| **cephadm** **rm-repo**
+
+| **cephadm** **install** [-h] [packages [packages ...]]
+
+| **cephadm** **registry-login** [-h] [--registry-url REGISTRY_URL]
+|                                [--registry-username REGISTRY_USERNAME]
+|                                [--registry-password REGISTRY_PASSWORD]
+|                                [--registry-json REGISTRY_JSON] [--fsid FSID]
+
+
+
+Description
+===========
+
+:program:`cephadm` is a command line tool to manage the local host for the cephadm orchestrator.
+
+It provides commands to investigate and modify the state of the current host.
+
+:program:`cephadm` is not required on all hosts, but useful when investigating a particular
+daemon.
+
+Options
+=======
+
+.. option:: --image IMAGE
+
+   container image. Can also be set via the
+   "CEPHADM_IMAGE" env var (default: None)
+
+.. option:: --docker
+
+   use docker instead of podman (default: False)
+
+.. option:: --data-dir DATA_DIR
+
+   base directory for daemon data (default: /var/lib/ceph)
+
+.. option:: --log-dir LOG_DIR
+
+   base directory for daemon logs (default: /var/log/ceph)
+
+.. option:: --logrotate-dir LOGROTATE_DIR
+
+   location of logrotate configuration files (default: /etc/logrotate.d)
+
+.. option:: --unit-dir UNIT_DIR
+
+   base directory for systemd units (default: /etc/systemd/system)
+
+.. option:: --verbose, -v
+
+   Show debug-level log messages (default: False)
+
+.. option:: --timeout TIMEOUT
+
+   timeout in seconds (default: None)
+
+.. option:: --retry RETRY
+
+   max number of retries (default: 10)
+
+.. option:: --no-container-init
+
+   do not run podman/docker with `--init` (default: False)
+
+
+Commands
+========
+
+add-repo
+--------
+
+configure local package repository to also include the ceph repository.
+
+Arguments:
+
+* [--release RELEASE]       use latest version of a named release (e.g., octopus)
+* [--version VERSION]       use specific upstream version (x.y.z)
+* [--dev DEV]               use specified bleeding edge build from git branch or tag
+* [--dev-commit DEV_COMMIT] use specified bleeding edge build from git commit
+* [--gpg-url GPG_URL]       specify alternative GPG key location
+* [--repo-url REPO_URL]     specify alternative repo location
+
+
+adopt
+-----
+
+Adopt a daemon deployed with a different deployment tool.
+
+Arguments:
+
+* [--name NAME, -n NAME]       daemon name (type.id)
+* [--style STYLE]              deployment style (legacy, ...)
+* [--cluster CLUSTER]          cluster name
+* [--legacy-dir LEGACY_DIR]    base directory for legacy daemon data
+* [--config-json CONFIG_JSON]  Additional configuration information in JSON format
+* [--skip-firewalld]           Do not configure firewalld
+* [--skip-pull]                do not pull the latest image before adopting
+
+Configuration:
+
+When starting the shell, cephadm looks for configuration in the following order.
+Only the first values found are used:
+
+1. An explicit, user provided path to a config file (``-c/--config`` option)
+2. Config file for daemon specified with ``--name`` parameter (``/var/lib/ceph/<fsid>/<daemon-name>/config``)
+3. ``/var/lib/ceph/<fsid>/config/ceph.conf`` if it exists
+4. The config file for a ``mon`` daemon (``/var/lib/ceph/<fsid>/mon.<mon-id>/config``) if it exists
+5. Finally: fallback to the default file ``/etc/ceph/ceph.conf``
+
+
+bootstrap
+---------
+
+Bootstrap a cluster on the local host. It deploys a MON and a MGR and then also automatically
+deploys the monitoring stack on this host (see --skip-monitoring-stack) and calls
+``ceph orch host add $(hostname)`` (see --skip-ssh).
+
+Arguments:
+
+* [--config CONFIG, -c CONFIG]    ceph conf file to incorporate
+* [--mon-id MON_ID]               mon id (default: local hostname)
+* [--mon-addrv MON_ADDRV]         mon IPs (e.g., [v2:localipaddr:3300,v1:localipaddr:6789])
+* [--mon-ip MON_IP]               mon IP
+* [--mgr-id MGR_ID]               mgr id (default: randomly generated)
+* [--fsid FSID]                   cluster FSID
+* [--log-to-file]                 configure cluster to log to traditional log files
+* [--single-host-defaults]        configure cluster to run on a single host
+* [--output-dir OUTPUT_DIR]       directory to write config, keyring, and pub key files
+* [--output-keyring OUTPUT_KEYRING] location to write keyring file with new cluster admin and mon keys
+* [--output-config OUTPUT_CONFIG] location to write conf file to connect to new cluster
+* [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY] location to write the cluster's public SSH key
+* [--skip-ssh                     skip setup of ssh key on local host
+* [--initial-dashboard-user INITIAL_DASHBOARD_USER] Initial user for the dashboard
+* [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] Initial password for the initial dashboard user
+* [--ssl-dashboard-port SSL_DASHBOARD_PORT] Port number used to connect with dashboard using SSL
+* [--dashboard-key DASHBOARD_KEY] Dashboard key
+* [--dashboard-crt DASHBOARD_CRT] Dashboard certificate
+* [--ssh-config SSH_CONFIG] SSH config
+* [--ssh-private-key SSH_PRIVATE_KEY] SSH private key
+* [--ssh-public-key SSH_PUBLIC_KEY] SSH public key
+* [--ssh-user SSH_USER]           set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users'
+* [--skip-mon-network]            set mon public_network based on bootstrap mon ip
+* [--skip-dashboard]              do not enable the Ceph Dashboard
+* [--dashboard-password-noupdate] stop forced dashboard password change
+* [--no-minimize-config]          do not assimilate and minimize the config file
+* [--skip-ping-check]             do not verify that mon IP is pingable
+* [--skip-pull]                   do not pull the latest image before bootstrapping
+* [--skip-firewalld]              Do not configure firewalld
+* [--allow-overwrite]             allow overwrite of existing --output-* config/keyring/ssh files
+* [--allow-fqdn-hostname]         allow hostname that is fully-qualified (contains ".")
+* [--skip-prepare-host]           Do not prepare host
+* [--orphan-initial-daemons]      Do not create initial mon, mgr, and crash service specs
+* [--skip-monitoring-stack]       Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter)
+* [--apply-spec APPLY_SPEC]       Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)
+* [--registry-url REGISTRY_URL]   url of custom registry to login to. e.g. docker.io, quay.io
+* [--registry-username REGISTRY_USERNAME] username of account to login to on custom registry
+* [--registry-password REGISTRY_PASSWORD] password of account to login to on custom registry
+* [--registry-json REGISTRY_JSON] JSON file containing registry login info (see registry-login command documentation)
+
+
+ceph-volume
+-----------
+
+Run ceph-volume inside a container::
+
+    cephadm ceph-volume inventory
+
+Positional arguments:
+* [command]               command
+
+Arguments:
+
+* [--fsid FSID]                    cluster FSID
+* [--config-json CONFIG_JSON]      JSON file with config and (client.bootstrap-osd) key
+* [--config CONFIG, -c CONFIG]     ceph conf file
+* [--keyring KEYRING, -k KEYRING]  ceph.keyring to pass through to the container
+
+
+check-host
+----------
+
+check host configuration to be suitable for a Ceph cluster.
+
+Arguments:
+
+* [--expect-hostname EXPECT_HOSTNAME] Check that hostname matches an expected value
+
+
+deploy
+------
+
+deploy a daemon on the local host. Used by the orchestrator CLI::
+
+    cephadm shell -- ceph orch apply <type> ...
+
+Arguments:
+
+* [--name NAME]               daemon name (type.id)
+* [--fsid FSID]               cluster FSID
+* [--config CONFIG, -c CONFIG] config file for new daemon
+* [--config-json CONFIG_JSON] Additional configuration information in JSON format
+* [--keyring KEYRING]         keyring for new daemon
+* [--key KEY]                 key for new daemon
+* [--osd-fsid OSD_FSID]       OSD uuid, if creating an OSD container
+* [--skip-firewalld]          Do not configure firewalld
+* [--tcp-ports                List of tcp ports to open in the host firewall
+* [--reconfig]                Reconfigure a previously deployed daemon
+* [--allow-ptrace]            Allow SYS_PTRACE on daemon container
+
+
+enter
+-----
+
+Run an interactive shell inside a running daemon container::
+
+    cephadm enter --name mgr.myhost.ysubfo
+
+Positional arguments:
+* [command]               command
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+install
+-------
+
+install ceph package(s)
+
+Positional arguments:
+
+* [packages]    packages
+
+
+inspect-image
+-------------
+
+Inspect local Ceph container image. From Reef onward, requires specifying
+the image to inspect with ``--image``::
+
+    cephadm --image IMAGE_NAME inspect-image
+
+list-networks
+-------------
+
+list IP networks
+
+
+ls
+--
+
+list daemon instances known to cephadm on **this** host::
+
+    $ cephadm ls
+    [
+        {
+            "style": "cephadm:v1",
+            "name": "mgr.storage-14b-1.ysubfo",
+            "fsid": "5110cb22-8332-11ea-9148-0894ef7e8bdc",
+            "enabled": true,
+            "state": "running",
+            "container_id": "8562de72370a3836473ecfff8a22c9ccdd99815386b4692a2b30924fb5493c44",
+            "container_image_name": "docker.io/ceph/ceph:v15",
+            "container_image_id": "bc83a388465f0568dab4501fb7684398dca8b50ca12a342a57f21815721723c2",
+            "version": "15.2.1",
+            "started": "2020-04-21T01:16:41.831456",
+            "created": "2020-04-21T01:16:41.775024",
+            "deployed": "2020-04-21T01:16:41.415021",
+            "configured": "2020-04-21T01:16:41.775024"
+        },
+    ...
+
+Arguments:
+
+* [--no-detail]             Do not include daemon status
+* [--legacy-dir LEGACY_DIR] Base directory for legacy daemon data
+
+logs
+----
+
+print journald logs for a daemon container::
+
+    cephadm logs --name mgr.myhost.ysubfo
+
+This is similar to::
+
+    journalctl -u mgr.myhost.ysubfo
+
+Can also specify additional journal arguments::
+
+    cephadm logs --name mgr.myhost.ysubfo -- -n 20 # last 20 lines
+    cephadm logs --name mgr.myhost.ysubfo -- -f # follow the log
+
+
+Positional arguments:
+
+* [command]               command (optional)
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+
+prepare-host
+------------
+
+prepare a host for cephadm use
+
+Arguments:
+
+* [--expect-hostname EXPECT_HOSTNAME] Set hostname
+
+
+pull
+----
+
+Pull the ceph image::
+
+    cephadm pull
+
+registry-login
+--------------
+
+Give cephadm login information for an authenticated registry (url, username and password).
+Cephadm will attempt to log the calling host into that registry::
+
+      cephadm registry-login --registry-url [REGISTRY_URL] --registry-username [USERNAME]
+                             --registry-password [PASSWORD]
+
+Can also use a JSON file containing the login info formatted as::
+
+      {
+       "url":"REGISTRY_URL",
+       "username":"REGISTRY_USERNAME",
+       "password":"REGISTRY_PASSWORD"
+      }
+
+and turn it in with command::
+
+      cephadm registry-login --registry-json [JSON FILE]
+
+Arguments:
+
+* [--registry-url REGISTRY_URL]   url of registry to login to. e.g. docker.io, quay.io
+* [--registry-username REGISTRY_USERNAME] username of account to login to on registry
+* [--registry-password REGISTRY_PASSWORD] password of account to login to on registry
+* [--registry-json REGISTRY_JSON] JSON file containing login info for custom registry
+* [--fsid FSID]                   cluster FSID
+
+rm-daemon
+---------
+
+Remove a specific daemon instance
+
+Arguments:
+
+* [--name NAME, -n NAME]  daemon name (type.id)
+* [--fsid FSID]           cluster FSID
+* [--force]               proceed, even though this may destroy valuable data
+* [--force-delete-data]   delete valuable daemon data instead of making a backup
+
+
+rm-cluster
+----------
+
+remove all daemons for a cluster
+
+Arguments:
+
+* [--fsid FSID]  cluster FSID
+* [--force]      proceed, even though this may destroy valuable data
+
+rm-repo
+-------
+
+remove package repository configuration
+
+run
+---
+
+run a ceph daemon, in a container, in the foreground
+
+Arguments:
+
+* [--name NAME, -n NAME]  daemon name (type.id)
+* [--fsid FSID]           cluster FSID
+
+
+shell
+-----
+
+Run an interactive shell::
+
+    cephadm shell
+
+Or one specific command inside a container::
+
+    cephadm shell -- ceph orch ls
+
+
+Positional arguments:
+
+* [command]               command (optional)
+
+Arguments:
+
+* [--fsid FSID]                   cluster FSID
+* [--name NAME, -n NAME]          daemon name (type.id)
+* [--config CONFIG, -c CONFIG]    ceph.conf to pass through to the container
+* [--keyring KEYRING, -k KEYRING] ceph.keyring to pass through to the container
+* [--mount MOUNT, -m MOUNT]       mount a file or directory under /mnt in the container
+* [--env ENV, -e ENV]             set environment variable
+
+
+unit
+----
+
+Operate on the daemon's systemd unit.
+
+Positional arguments:
+
+* [command]               systemd command (start, stop, restart, enable, disable, ...)
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+
+Availability
+============
+
+:program:`cephadm` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the documentation at http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-volume <ceph-volume>`\(8),
diff --git a/doc/man/8/cephfs-mirror.rst b/doc/man/8/cephfs-mirror.rst
new file mode 100644
index 000000000..12e2fb303
--- /dev/null
+++ b/doc/man/8/cephfs-mirror.rst
@@ -0,0 +1,83 @@
+:orphan:
+
+============================================================
+ cephfs-mirror -- Ceph daemon for mirroring CephFS snapshots
+============================================================
+
+.. program:: cephfs-mirror
+
+Synopsis
+========
+
+| **cephfs-mirror**
+
+
+Description
+===========
+
+:program:`cephfs-mirror` is a daemon for asynchronous mirroring of Ceph
+Filesystem snapshots among Ceph clusters.
+
+It connects to remote clusters via libcephfs, relying on default search
+paths to find ceph.conf files, i.e. ``/etc/ceph/$cluster.conf`` where
+``$cluster`` is the human-friendly name of the cluster.
+
+
+Options
+=======
+
+.. option:: --mon-host monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --keyring=<path-to-keyring>
+
+   Provide path to keyring; useful when it's absent in standard locations.
+
+.. option:: --log-file=<logfile>
+
+   file to log debug output
+
+.. option:: --debug-cephfs-mirror=<log-level>/<memory-level>
+
+   set cephfs-mirror debug level
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for cephfs-mirror
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name (eg. client.mirror)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`cephfs-mirror` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/cephfs-shell.rst b/doc/man/8/cephfs-shell.rst
new file mode 100644
index 000000000..a615dab98
--- /dev/null
+++ b/doc/man/8/cephfs-shell.rst
@@ -0,0 +1,650 @@
+:orphan:
+
+===================================================
+cephfs-shell -- Shell-like tool talking with CephFS
+===================================================
+
+.. program:: cephfs-shell
+
+Synopsis
+========
+
+| **cephfs-shell** [options] [command]
+| **cephfs-shell** [options] -- [command, command,...]
+
+Description
+===========
+
+CephFS Shell provides shell-like commands that directly interact with the
+Ceph File System.
+
+This tool can be used in interactive mode as well as in non-interactive mode.
+In former mode, cephfs-shell opens a shell session and after the given command
+is finished, it prints the prompt string and waits indefinitely. When the
+shell session is finished, cephfs-shell quits with the return value of last
+executed command. In non-interactive mode, cephfs-shell issues a command and
+exits right after the command's execution is complete with the command's
+return value.
+
+Behaviour of CephFS Shell can be tweaked using ``cephfs-shell.conf``. Refer to
+`CephFS Shell Configuration File`_ for details.
+
+Options
+=======
+
+.. option:: -b, --batch FILE
+
+   Path to batch file.
+
+.. option:: -c, --config FILE
+
+   Path to cephfs-shell.conf
+
+.. option:: -f, --fs FS
+
+   Name of filesystem to mount.
+
+.. option:: -t, --test FILE
+
+   Path to transcript(s) in FILE for testing
+
+.. note::
+
+    Latest version of the cmd2 module is required for running cephfs-shell.
+    If CephFS is installed through source, execute cephfs-shell in the build
+    directory. It can also be executed as following using virtualenv:
+
+.. code:: bash
+
+    [build]$ python3 -m venv venv && source venv/bin/activate && pip3 install cmd2
+    [build]$ source vstart_environment.sh && source venv/bin/activate && python3 ../src/tools/cephfs/shell/cephfs-shell
+
+Commands
+========
+
+.. note::
+
+    Apart from Ceph File System, CephFS Shell commands can also interact
+    directly with the local file system. To achieve this, ``!`` (an
+    exclamation point) must precede the CephFS Shell command.
+
+    Usage :
+
+        !<cephfs_shell_command>
+
+    For example,
+
+    .. code:: bash
+
+        CephFS:~/>>> !ls # Lists the local file system directory contents.
+        CephFS:~/>>> ls  # Lists the Ceph File System directory contents.
+
+mkdir
+-----
+
+Create the directory(ies), if they do not already exist.
+
+Usage : 
+        
+    mkdir [-option] <directory>... 
+
+* directory - name of the directory to be created.
+
+Options :
+  -m MODE    Sets the access mode for the new directory.
+  -p, --parent         Create parent directories as necessary. When this option is specified, no error is reported if a directory already exists.
+ 
+put
+---
+
+Copy a file/directory to Ceph File System from Local File System.
+
+Usage : 
+    
+        put [options] <source_path> <target_path>
+
+* source_path - local file/directory path to be copied to cephfs.
+    * if `.` copies all the file/directories in the local working directory.
+    * if `-`  Reads the input from stdin. 
+
+* target_path - remote directory path where the files/directories are to be copied to.
+    * if `.` files/directories are copied to the remote working directory.
+
+Options :
+   -f, --force        Overwrites the destination if it already exists.
+
+
+get
+---
+ 
+Copy a file from Ceph File System to Local File System.
+
+Usage : 
+
+    get [options] <source_path> <target_path>
+
+* source_path - remote file/directory path which is to be copied to local file system.
+    * if `.` copies all the file/directories in the remote working directory.
+                    
+* target_path - local directory path where the files/directories are to be copied to.
+    * if `.` files/directories are copied to the local working directory. 
+    * if `-` Writes output to stdout.
+
+Options:
+  -f, --force        Overwrites the destination if it already exists.
+
+ls
+--
+
+List all the files and directories in the current working directory.
+
+Usage : 
+    
+    ls [option] [directory]...
+
+* directory - name of directory whose files/directories are to be listed. 
+    * By default current working directory's files/directories are listed.
+
+Options:
+  -l, --long	    list with long format - show permissions
+  -r, --reverse     reverse sort     
+  -H                human readable
+  -a, -all          ignore entries starting with .
+  -S                Sort by file_size
+
+
+cat
+---
+
+Concatenate files and print on the standard output
+
+Usage : 
+
+    cat  <file>....
+
+* file - name of the file
+
+ln
+--
+
+Add a hard link to an existing file or create a symbolic link to an existing
+file or directory.
+
+Usage:
+
+    ln [options] <target> [link_name]
+
+* target - file/directory to which a link is to be created
+* link_name - link to target with the name link_name
+
+Options:
+  -s, --symbolic  Create symbolic link
+  -v, --verbose   Print name of each linked file
+  -f, --force     Force create link/symbolic link
+
+cd
+--
+
+Change current working directory.
+
+Usage : 
+    
+    cd [directory]
+        
+* directory - path/directory name. If no directory is mentioned it is changed to the root directory.
+    * If '.' moves to the parent directory of the current directory.
+
+cwd
+---
+
+Get current working directory.
+ 
+Usage : 
+    
+    cwd
+
+
+quit/Ctrl + D
+-------------
+
+Close the shell.
+
+chmod
+-----
+
+Change the permissions of file/directory.
+ 
+Usage : 
+    
+    chmod <mode> <file/directory>
+
+mv
+--
+
+Moves files/Directory from source to destination.
+
+Usage : 
+    
+    mv <source_path> <destination_path>
+
+rmdir
+-----
+
+Delete a directory(ies).
+
+Usage : 
+    
+    rmdir <directory_name>.....
+
+rm
+--
+
+Remove a file(es).
+
+Usage : 
+    
+    rm <file_name/pattern>...
+
+
+write
+-----
+
+Create and Write a file.
+
+Usage : 
+        
+        write <file_name>
+        <Enter Data>
+        Ctrl+D Exit.
+
+lls
+---
+
+Lists all files and directories in the specified directory.Current local directory files and directories are listed if no     path is mentioned
+
+Usage: 
+    
+    lls <path>.....
+
+lcd
+---
+
+Moves into the given local directory.
+
+Usage : 
+    
+    lcd <path>
+
+lpwd
+----
+
+Prints the absolute path of the current local directory.
+
+Usage : 
+    
+    lpwd
+
+
+umask
+-----
+
+Set and get the file mode creation mask 
+
+Usage : 
+    
+    umask [mode]
+
+alias
+-----
+
+Define or display aliases
+
+Usage: 
+
+    alias [name] | [<name> <value>]
+
+* name - name of the alias being looked up, added, or replaced
+* value - what the alias will be resolved to (if adding or replacing) this can contain spaces and does not need to be quoted
+
+run_pyscript
+------------
+
+Runs a python script file inside the console
+
+Usage: 
+    
+    run_pyscript <script_path> [script_arguments]
+
+* Console commands can be executed inside this script with cmd ("your command")
+  However, you cannot run nested "py" or "pyscript" commands from within this
+  script. Paths or arguments that contain spaces must be enclosed in quotes
+
+.. note:: This command is available as ``pyscript`` for cmd2 versions 0.9.13
+   or less.
+
+py
+--
+
+Invoke python command, shell, or script
+
+Usage : 
+
+        py <command>: Executes a Python command.
+        py: Enters interactive Python mode.
+
+shortcuts
+---------
+
+Lists shortcuts (aliases) available
+
+Usage :
+
+    shortcuts
+
+history
+-------
+
+View, run, edit, and save previously entered commands.
+
+Usage : 
+    
+    history [-h] [-r | -e | -s | -o FILE | -t TRANSCRIPT] [arg]
+
+Options:
+   -h             show this help message and exit
+   -r             run selected history items
+   -e             edit and then run selected history items
+   -s             script format; no separation lines
+   -o FILE        output commands to a script file
+   -t TRANSCRIPT  output commands and results to a transcript file
+
+unalias
+-------
+
+Unsets aliases
+ 
+Usage : 
+    
+    unalias [-a] name [name ...]
+
+* name - name of the alias being unset
+
+Options:
+   -a     remove all alias definitions
+
+set
+---
+
+Sets a settable parameter or shows current settings of parameters.
+
+Usage : 
+
+    set [-h] [-a] [-l] [settable [settable ...]]
+
+* Call without arguments for a list of settable parameters with their values.
+
+Options :
+  -h     show this help message and exit
+  -a     display read-only settings as well
+  -l     describe function of parameter
+
+edit
+----
+
+Edit a file in a text editor.
+
+Usage:  
+
+    edit [file_path]
+
+* file_path - path to a file to open in editor
+
+run_script
+----------
+
+Runs commands in script file that is encoded as either ASCII or UTF-8 text.
+Each command in the script should be separated by a newline.
+
+Usage:  
+    
+    run_script <file_path>
+
+
+* file_path - a file path pointing to a script
+
+.. note:: This command is available as ``load`` for cmd2 versions 0.9.13
+   or less.
+
+shell
+-----
+
+Execute a command as if at the OS prompt.
+
+Usage:  
+    
+    shell <command> [arguments]
+
+locate
+------
+
+Find an item in File System
+
+Usage:
+
+     locate [options] <name>
+
+Options :
+  -c       Count number of items found
+  -i       Ignore case 
+
+stat
+------
+
+Display file status.
+
+Usage :
+
+     stat [-h] <file_name> [file_name ...]
+
+Options :
+  -h     Shows the help message
+
+snap
+----
+
+Create or Delete Snapshot
+
+Usage:
+
+     snap {create|delete} <snap_name> <dir_name>
+
+* snap_name - Snapshot name to be created or deleted
+
+* dir_name - directory under which snapshot should be created or deleted
+
+setxattr
+--------
+
+Set extended attribute for a file
+
+Usage :
+
+     setxattr [-h] <path> <name> <value>
+
+*  path - Path to the file
+
+*  name - Extended attribute name to get or set
+
+*  value - Extended attribute value to be set
+
+Options:
+  -h, --help   Shows the help message
+
+getxattr
+--------
+
+Get extended attribute value for the name associated with the path
+
+Usage :
+
+     getxattr [-h] <path> <name>
+
+*  path - Path to the file
+
+*  name - Extended attribute name to get or set
+
+Options:
+  -h, --help   Shows the help message
+
+listxattr
+---------
+
+List extended attribute names associated with the path
+
+Usage :
+
+     listxattr [-h] <path>
+
+*  path - Path to the file
+
+Options:
+  -h, --help   Shows the help message
+
+df
+--
+
+Display amount of available disk space
+
+Usage :
+
+    df [-h] [file [file ...]]
+
+* file - name of the file
+
+Options:
+  -h, --help   Shows the help message
+
+du
+--
+
+Show disk usage of a directory
+
+Usage :
+
+    du [-h] [-r] [paths [paths ...]]
+
+* paths - name of the directory
+
+Options:
+  -h, --help   Shows the help message
+
+  -r     Recursive Disk usage of all directories
+
+
+quota
+-----
+
+Quota management for a Directory
+
+Usage :
+
+    quota [-h] [--max_bytes [MAX_BYTES]] [--max_files [MAX_FILES]] {get,set} path
+
+* {get,set} - quota operation type.
+
+* path - name of the directory.
+
+Options :
+  -h, --help   Shows the help message
+
+  --max_bytes MAX_BYTES    Set max cumulative size of the data under this directory
+
+  --max_files MAX_FILES    Set total number of files under this directory tree
+
+CephFS Shell Configuration File
+===============================
+By default, CephFS Shell looks for ``cephfs-shell.conf`` in the path provided
+by the environment variable ``CEPHFS_SHELL_CONF`` and then in user's home
+directory (``~/.cephfs-shell.conf``).
+
+Right now, CephFS Shell inherits all its options from its dependency ``cmd2``.
+Therefore, these options might vary with the version of ``cmd2`` installed on
+your system. Refer to ``cmd2`` docs for a description of these options.
+
+Following is a sample ``cephfs-shell.conf``
+
+.. code-block:: ini
+
+    [cephfs-shell]
+    prompt = CephFS:~/>>>
+    continuation_prompt = >
+
+    quiet = False
+    timing = False
+    colors = True
+    debug = False
+
+    abbrev = False
+    autorun_on_edit = False
+    echo = False
+    editor = vim
+    feedback_to_output = False
+    locals_in_py = True
+
+Exit Code
+=========
+
+Following exit codes are returned by cephfs shell
+
++-----------------------------------------------+-----------+
+| Error Type                                    | Exit Code |
++===============================================+===========+
+| Miscellaneous                                 |     1     |
++-----------------------------------------------+-----------+
+| Keyboard Interrupt                            |     2     |
++-----------------------------------------------+-----------+
+| Operation not permitted                       |     3     |
++-----------------------------------------------+-----------+
+| Permission denied                             |     4     |
++-----------------------------------------------+-----------+
+| No such file or directory                     |     5     |
++-----------------------------------------------+-----------+
+| I/O error                                     |     6     |
++-----------------------------------------------+-----------+
+| No space left on device                       |     7     |
++-----------------------------------------------+-----------+
+| File exists                                   |     8     |
++-----------------------------------------------+-----------+
+| No data available                             |     9     |
++-----------------------------------------------+-----------+
+| Invalid argument                              |     10    |
++-----------------------------------------------+-----------+
+| Operation not supported on transport endpoint |     11    |
++-----------------------------------------------+-----------+
+| Range error                                   |     12    |
++-----------------------------------------------+-----------+
+| Operation would block                         |     13    |
++-----------------------------------------------+-----------+
+| Directory not empty                           |     14    |
++-----------------------------------------------+-----------+
+| Not a directory                               |     15    |
++-----------------------------------------------+-----------+
+| Disk quota exceeded                           |     16    |
++-----------------------------------------------+-----------+
+| Broken pipe                                   |     17    |
++-----------------------------------------------+-----------+
+| Cannot send after transport endpoint shutdown |     18    |
++-----------------------------------------------+-----------+
+| Connection aborted                            |     19    |
++-----------------------------------------------+-----------+
+| Connection refused                            |     20    |
++-----------------------------------------------+-----------+
+| Connection reset                              |     21    |
++-----------------------------------------------+-----------+
+| Interrupted function call                     |     22    |
++-----------------------------------------------+-----------+
+
+Files
+=====
+
+``~/.cephfs-shell.conf``
diff --git a/doc/man/8/cephfs-top.rst b/doc/man/8/cephfs-top.rst
new file mode 100644
index 000000000..fd18ada77
--- /dev/null
+++ b/doc/man/8/cephfs-top.rst
@@ -0,0 +1,137 @@
+:orphan:
+
+==========================================
+ cephfs-top -- Ceph Filesystem Top Utility
+==========================================
+
+.. program:: cephfs-top
+
+Synopsis
+========
+
+| **cephfs-top** [flags]
+
+
+Description
+===========
+
+**cephfs-top** provides top(1) like functionality for Ceph Filesystem.
+Various client metrics are displayed and updated in realtime.
+
+Ceph Metadata Servers periodically send client metrics to Ceph Manager.
+``Stats`` plugin in Ceph Manager provides an interface to fetch these metrics.
+
+Options
+=======
+
+.. option:: --cluster
+
+   Cluster: Ceph cluster to connect. Defaults to ``ceph``.
+
+.. option:: --id
+
+   Id: Client used to connect to Ceph cluster. Defaults to ``fstop``.
+
+.. option:: --selftest
+
+   Perform a selftest. This mode performs a sanity check of ``stats`` module.
+
+.. option:: --conffile [CONFFILE]
+
+   Path to cluster configuration file
+
+.. option:: -d [DELAY], --delay [DELAY]
+
+   Refresh interval in seconds (default: 1)
+
+.. option:: --dump
+
+   Dump the metrics to stdout
+
+.. option:: --dumpfs <fs_name>
+
+   Dump the metrics of the given filesystem to stdout
+
+Descriptions of fields
+======================
+
+.. describe:: chit
+
+   cap hit rate
+
+.. describe:: dlease
+
+   dentry lease rate
+
+.. describe:: ofiles
+
+   number of opened files
+
+.. describe:: oicaps
+
+   number of pinned caps
+
+.. describe:: oinodes
+
+   number of opened inodes
+
+.. describe:: rtio
+
+   total size of read IOs
+
+.. describe:: wtio
+
+   total size of write IOs
+
+.. describe:: raio
+
+   average size of read IOs
+
+.. describe:: waio
+
+   average size of write IOs
+
+.. describe:: rsp
+
+   speed of read IOs compared with the last refresh
+
+.. describe:: wsp
+
+   speed of write IOs compared with the last refresh
+
+.. describe:: rlatavg
+
+   average read latency
+
+.. describe:: rlatsd
+
+   standard deviation (variance) for read latency
+
+.. describe:: wlatavg
+
+   average write latency
+
+.. describe:: wlatsd
+
+   standard deviation (variance) for write latency
+
+.. describe:: mlatavg
+
+   average metadata latency
+
+.. describe:: mlatsd
+
+   standard deviation (variance) for metadata latency
+
+Availability
+============
+
+**cephfs-top** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to the Ceph documentation at
+http://ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8)
diff --git a/doc/man/8/crushdiff.rst b/doc/man/8/crushdiff.rst
new file mode 100644
index 000000000..44ba4f561
--- /dev/null
+++ b/doc/man/8/crushdiff.rst
@@ -0,0 +1,118 @@
+:orphan:
+
+.. _crushdiff:
+
+=======================================
+ crushdiff -- ceph crush map test tool
+=======================================
+
+.. program:: crushdiff
+
+Synopsis
+========
+
+| **crushdiff** [ --osdmap *osdmap* ] [ --pg-dump *pg-dump* ]
+  [ --compiled ] [ --verbose ] *command* *crushmap*
+
+
+Description
+===========
+
+**crushdiff** is a utility that lets you test the effect of a crushmap
+change: number of pgs, objects, bytes moved. This is a wrapper around
+:doc:`osdmaptool <osdmaptool>`\(8), relying on its **--test-map-pgs-dump**
+option to get the list of changed pgs. Additionally it uses pg stats
+to calculate the numbers of objects and bytes moved.
+
+By default, **crushdiff** will use the cluster current osdmap and pg
+stats, which requires access to the cluster. Though one can use the
+**--osdmap** and **--pg-dump** options to test against previously
+obtained data.
+
+Options
+=======
+
+.. option:: --compiled
+
+   The input/output crushmap is compiled. If the options is not
+   specified the expected/returned crushmap is in txt (decompiled)
+   format.
+
+.. option:: --pg-dump <pg-dump>
+
+   JSON output of **ceph pg dump**. If not specified **crushdiff**
+   will try to get data running the command itself.
+
+.. option:: --osdmap <osdmap>
+
+   The cluster osdmap, obtained with **ceph osd getmap** command. If
+   not specified **crushdiff** will try to get data running the
+   command itself.
+
+.. option:: --verbose
+
+   Produce diagnostic output.
+
+Commands
+========
+
+:command:`compare` *crushmap*
+  Compare the crushmap from *crushmap* file with the crushmap from
+  the cluster osdmap. The output will show the expected number of pgs,
+  objects, bytes moved when the new crushmap is installed.
+
+:command:`export` *crushmap*
+  Export crushmap to *crushmap* file from the cluster osdmap.
+
+:command:`import` *crushmap*
+  Import crushmap from *crushmap* file to the cluster osdmap.
+
+Example
+=======
+
+Get the current crushmap::
+
+        crushdiff export cm.txt
+
+Edit the map::
+
+        $EDITOR cm.txt
+
+Check the result::
+
+        crushdiff compare cm.txt
+
+        79/416 (18.99%) pgs affected
+        281/1392 (20.19%) objects affected
+        80/1248 (6.41%) pg shards to move
+        281/4176 (6.73%) pg object shards to move
+        730.52Mi/10.55Gi (6.76%) bytes to move
+
+When running with **--verbose** option the output will also contain
+detailed information about the affected pgs, like below::
+
+        4.3	[0, 2, 1] -> [1, 4, 2]
+        4.b	[0, 1, 3] -> [2, 1, 3]
+        4.c	[4, 0, 1] -> [4, 1, 2]
+
+i.e. a pg number, and its old and the new osd active sets.
+
+If the result is satisfactory install the updated map::
+
+        crushdiff import cm.txt
+
+
+Availability
+============
+
+**crushdiff** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
+:doc:`osdmaptool <osdmaptool>`\(8),
diff --git a/doc/man/8/crushtool.rst b/doc/man/8/crushtool.rst
new file mode 100644
index 000000000..4c8486596
--- /dev/null
+++ b/doc/man/8/crushtool.rst
@@ -0,0 +1,302 @@
+:orphan:
+
+==========================================
+ crushtool -- CRUSH map manipulation tool
+==========================================
+
+.. program:: crushtool
+
+Synopsis
+========
+
+| **crushtool** ( -d *map* | -c *map.txt* | --build --num_osds *numosds*
+  *layer1* *...* | --test ) [ -o *outfile* ]
+
+
+Description
+===========
+
+**crushtool** is a utility that lets you create, compile, decompile
+and test CRUSH map files.
+
+CRUSH is a pseudo-random data distribution algorithm that efficiently
+maps input values (which, in the context of Ceph, correspond to Placement
+Groups) across a heterogeneous, hierarchically structured device map.
+The algorithm was originally described in detail in the following paper
+(although it has evolved some since then)::
+
+   http://www.ssrc.ucsc.edu/Papers/weil-sc06.pdf
+
+The tool has four modes of operation.
+
+.. option:: --compile|-c map.txt
+
+   will compile a plaintext map.txt into a binary map file.
+
+.. option:: --decompile|-d map
+
+   will take the compiled map and decompile it into a plaintext source
+   file, suitable for editing.
+
+.. option:: --build --num_osds {num-osds} layer1 ...
+
+   will create map with the given layer structure. See below for a
+   detailed explanation.
+
+.. option:: --test
+
+   will perform a dry run of a CRUSH mapping for a range of input
+   values ``[--min-x,--max-x]`` (default ``[0,1023]``) which can be
+   thought of as simulated Placement Groups. See below for a more
+   detailed explanation.
+
+Unlike other Ceph tools, **crushtool** does not accept generic options
+such as **--debug-crush** from the command line. They can, however, be
+provided via the CEPH_ARGS environment variable. For instance, to
+silence all output from the CRUSH subsystem::
+
+    CEPH_ARGS="--debug-crush 0" crushtool ...
+
+
+Running tests with --test
+=========================
+
+The test mode will use the input crush map ( as specified with **-i
+map** ) and perform a dry run of CRUSH mapping or random placement
+(if **--simulate** is set ). On completion, two kinds of reports can be
+created. 
+1) The **--show-...** option outputs human readable information
+on stderr. 
+2) The **--output-csv** option creates CSV files that are
+documented by the **--help-output** option.
+
+Note: Each Placement Group (PG) has an integer ID which can be obtained
+from ``ceph pg dump`` (for example PG 2.2f means pool id 2, PG id 32).
+The pool and PG IDs are combined by a function to get a value which is
+given to CRUSH to map it to OSDs. crushtool does not know about PGs or
+pools; it only runs simulations by mapping values in the range
+``[--min-x,--max-x]``.
+
+
+.. option:: --show-statistics
+
+   Displays a summary of the distribution. For instance::
+
+       rule 1 (metadata) num_rep 5 result size == 5:	1024/1024
+
+   shows that rule **1** which is named **metadata** successfully
+   mapped **1024** values to **result size == 5** devices when trying
+   to map them to **num_rep 5** replicas. When it fails to provide the
+   required mapping, presumably because the number of **tries** must
+   be increased, a breakdown of the failures is displayed. For instance::
+
+       rule 1 (metadata) num_rep 10 result size == 8:	4/1024
+       rule 1 (metadata) num_rep 10 result size == 9:	93/1024
+       rule 1 (metadata) num_rep 10 result size == 10:	927/1024
+
+   shows that although **num_rep 10** replicas were required, **4**
+   out of **1024** values ( **4/1024** ) were mapped to **result size
+   == 8** devices only.
+
+.. option:: --show-mappings
+
+   Displays the mapping of each value in the range ``[--min-x,--max-x]``.
+   For instance::
+
+       CRUSH rule 1 x 24 [11,6]
+
+   shows that value **24** is mapped to devices **[11,6]** by rule
+   **1**.
+
+   One of the following is required when using the ``--show-mappings`` option:
+   
+        (a) ``--num-rep`` 
+        (b) both ``--min-rep`` and ``--max-rep``
+
+   ``--num-rep`` stands for "number of replicas, indicates the number of
+   replicas in a pool, and is used to specify an exact number of replicas (for
+   example ``--num-rep 5``). ``--min-rep`` and ``--max-rep`` are used together
+   to specify a range of replicas (for example, ``--min-rep 1 --max-rep 10``).
+
+.. option:: --show-bad-mappings
+
+   Displays which value failed to be mapped to the required number of
+   devices. For instance::
+
+     bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+
+   shows that when rule **1** was required to map **7** devices, it
+   could map only six : **[8,10,2,11,6,9]**.
+
+.. option:: --show-utilization
+
+   Displays the expected and actual utilization for each device, for
+   each number of replicas. For instance::
+
+     device 0: stored : 951      expected : 853.333
+     device 1: stored : 963      expected : 853.333
+     ...
+
+   shows that device **0** stored **951** values and was expected to store **853**.
+   Implies **--show-statistics**.
+
+.. option:: --show-utilization-all
+
+   Displays the same as **--show-utilization** but does not suppress
+   output when the weight of a device is zero.
+   Implies **--show-statistics**.
+
+.. option:: --show-choose-tries
+
+   Displays how many attempts were needed to find a device mapping.
+   For instance::
+
+      0:     95224
+      1:      3745
+      2:      2225
+      ..
+
+   shows that **95224** mappings succeeded without retries, **3745**
+   mappings succeeded with one attempts, etc. There are as many rows
+   as the value of the **--set-choose-total-tries** option.
+
+.. option:: --output-csv
+
+   Creates CSV files (in the current directory) containing information
+   documented by **--help-output**. The files are named after the rule
+   used when collecting the statistics. For instance, if the rule
+   : 'metadata' is used, the CSV files will be::
+
+      metadata-absolute_weights.csv
+      metadata-device_utilization.csv
+      ...
+
+   The first line of the file shortly explains the column layout. For
+   instance::
+
+      metadata-absolute_weights.csv
+      Device ID, Absolute Weight
+      0,1
+      ...
+
+.. option:: --output-name NAME
+
+   Prepend **NAME** to the file names generated when **--output-csv**
+   is specified. For instance **--output-name FOO** will create
+   files::
+
+      FOO-metadata-absolute_weights.csv
+      FOO-metadata-device_utilization.csv
+      ...
+
+The **--set-...** options can be used to modify the tunables of the
+input crush map. The input crush map is modified in
+memory. For example::
+
+      $ crushtool -i mymap --test --show-bad-mappings
+      bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+
+could be fixed by increasing the **choose-total-tries** as follows:
+
+      $ crushtool -i mymap --test \
+          --show-bad-mappings \
+          --set-choose-total-tries 500
+
+Building a map with --build
+===========================
+
+The build mode will generate hierarchical maps. The first argument
+specifies the number of devices (leaves) in the CRUSH hierarchy. Each
+layer describes how the layer (or devices) preceding it should be
+grouped.
+
+Each layer consists of::
+
+       bucket ( uniform | list | tree | straw | straw2 ) size
+
+The **bucket** is the type of the buckets in the layer
+(e.g. "rack"). Each bucket name will be built by appending a unique
+number to the **bucket** string (e.g. "rack0", "rack1"...).
+
+The second component is the type of bucket: **straw** should be used
+most of the time.
+
+The third component is the maximum size of the bucket. A size of zero
+means a bucket of infinite capacity.
+
+
+Example
+=======
+
+Suppose we have two rows with two racks each and 20 nodes per rack. Suppose
+each node contains 4 storage devices for Ceph OSD Daemons. This configuration
+allows us to deploy 320 Ceph OSD Daemons. Lets assume a 42U rack with 2U nodes,
+leaving an extra 2U for a rack switch.
+
+To reflect our hierarchy of devices, nodes, racks and rows, we would execute
+the following::
+
+    $ crushtool -o crushmap --build --num_osds 320 \
+           node straw 4 \
+           rack straw 20 \
+           row straw 2 \
+           root straw 0
+    # id	weight	type name	reweight
+    -87	320	root root
+    -85	160		row row0
+    -81	80			rack rack0
+    -1	4				node node0
+    0	1					osd.0	1
+    1	1					osd.1	1
+    2	1					osd.2	1
+    3	1					osd.3	1
+    -2	4				node node1
+    4	1					osd.4	1
+    5	1					osd.5	1
+    ...
+
+CRUSH rules are created so the generated crushmap can be
+tested. They are the same rules as the ones created by default when
+creating a new Ceph cluster. They can be further edited with::
+
+       # decompile
+       crushtool -d crushmap -o map.txt
+
+       # edit
+       emacs map.txt
+
+       # recompile
+       crushtool -c map.txt -o crushmap
+
+Reclassify
+==========
+
+The *reclassify* function allows users to transition from older maps that
+maintain parallel hierarchies for OSDs of different types to a modern CRUSH
+map that makes use of the *device class* feature.  For more information,
+see https://docs.ceph.com/en/latest/rados/operations/crush-map-edits/#migrating-from-a-legacy-ssd-rule-to-device-classes.
+
+Example output from --test
+==========================
+
+See https://github.com/ceph/ceph/blob/master/src/test/cli/crushtool/set-choose.t
+for sample ``crushtool --test`` commands and output produced thereby.
+
+Availability
+============
+
+**crushtool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`osdmaptool <osdmaptool>`\(8),
+
+Authors
+=======
+
+John Wilkins, Sage Weil, Loic Dachary
diff --git a/doc/man/8/librados-config.rst b/doc/man/8/librados-config.rst
new file mode 100644
index 000000000..c89813138
--- /dev/null
+++ b/doc/man/8/librados-config.rst
@@ -0,0 +1,46 @@
+:orphan:
+
+=======================================================
+ librados-config -- display information about librados
+=======================================================
+
+.. program:: librados-config
+
+Synopsis
+========
+
+| **librados-config** [ --version ] [ --vernum ]
+
+
+Description
+===========
+
+**librados-config** is a utility that displays information about the
+  installed ``librados``.
+
+
+Options
+=======
+
+.. option:: --version
+
+   Display ``librados`` version
+
+.. option:: --vernum
+
+   Display the ``librados`` version code
+
+
+Availability
+============
+
+**librados-config** is part of Ceph, a massively scalable, open-source, distributed storage system.
+Please refer to the Ceph documentation at https://docs.ceph.com for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`rados <rados>`\(8)
diff --git a/doc/man/8/monmaptool.rst b/doc/man/8/monmaptool.rst
new file mode 100644
index 000000000..7787af015
--- /dev/null
+++ b/doc/man/8/monmaptool.rst
@@ -0,0 +1,140 @@
+:orphan:
+
+==========================================================
+ monmaptool -- ceph monitor cluster map manipulation tool
+==========================================================
+
+.. program:: monmaptool
+
+Synopsis
+========
+
+| **monmaptool** <action> [options] *mapfilename*
+
+
+Description
+===========
+
+**monmaptool** is a utility to create, view, and modify a monitor
+cluster map for the Ceph distributed storage system. The monitor map
+specifies the only fixed addresses in the Ceph distributed system.
+All other daemons bind to arbitrary addresses and register themselves
+with the monitors.
+
+When creating a map with --create, a new monitor map with a new,
+random UUID will be created. It should be followed by one or more
+monitor addresses.
+
+The default Ceph monitor port for messenger protocol v1 is 6789, and
+3300 for protocol v2.
+
+Multiple actions can be performed per invocation.
+
+
+Options
+=======
+
+.. option:: --print
+
+   print a plaintext dump of the map, after any modifications are
+   made.
+
+.. option:: --feature-list [plain|parseable]
+
+   list the enabled features as well as the available ones.
+
+   By default, a human readable output is produced.
+
+.. option:: --create
+
+   create a new monitor map with a new UUID (and with it, a new,
+   empty Ceph cluster).
+
+.. option:: --clobber
+
+   allow monmaptool to create a new mapfilename in place of an existing map.
+
+   Only useful when *--create* is used.
+
+.. option:: --generate
+
+   generate a new monmap based on the values on the command line or specified
+   in the ceph configuration.  This is, in order of preference,
+
+      #. ``--monmap filename`` to specify a monmap to load
+      #. ``--mon-host 'host1,ip2'`` to specify a list of hosts or ip addresses
+      #. ``[mon.foo]`` sections containing ``mon addr`` settings in the config. Note that this method is not recommended and support will be removed in a future release.
+
+.. option:: --filter-initial-members
+
+   filter the initial monmap by applying the ``mon initial members``
+   setting.  Monitors not present in that list will be removed, and
+   initial members not present in the map will be added with dummy
+   addresses.
+
+.. option:: --add name ip[:port]
+
+   add a monitor with the specified ip:port to the map.
+
+   If the *nautilus* feature is set, and the port is not, the monitor
+   will be added for both messenger protocols.
+
+.. option:: --addv name [protocol:ip:port[,...]]
+
+   add a monitor with the specified version:ip:port to the map.
+
+.. option:: --rm name
+
+   remove the monitor with the specified name from the map.
+
+.. option:: --fsid uuid
+
+   set the fsid to the given uuid.  If not specified with *--create*, a random fsid will be generated.
+
+.. option:: --feature-set value [--optional|--persistent]
+
+   enable a feature.
+
+.. option:: --feature-unset value [--optional|--persistent]
+
+   disable a feature.
+
+.. option:: --enable-all-features
+
+   enable all supported features.
+
+.. option:: --set-min-mon-release release
+
+   set the min_mon_release.
+
+Example
+=======
+
+To create a new map with three monitors (for a fresh Ceph cluster)::
+
+        monmaptool --create --add nodeA 192.168.0.10 --add nodeB 192.168.0.11 \
+          --add nodeC 192.168.0.12 --enable-all-features --clobber monmap
+
+To display the contents of the map::
+
+        monmaptool --print monmap
+
+To replace one monitor::
+
+        monmaptool --rm nodeA monmap
+        monmaptool --add nodeA 192.168.0.9 monmap
+
+
+Availability
+============
+
+**monmaptool** is part of Ceph, a massively scalable, open-source, distributed 
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com
+for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
diff --git a/doc/man/8/mount.ceph.rst b/doc/man/8/mount.ceph.rst
new file mode 100644
index 000000000..fbe8790dd
--- /dev/null
+++ b/doc/man/8/mount.ceph.rst
@@ -0,0 +1,259 @@
+:orphan:
+
+========================================
+ mount.ceph -- mount a Ceph file system
+========================================
+
+.. program:: mount.ceph
+
+Synopsis
+========
+
+| **mount.ceph** *name*@*fsid*.*fs_name*=/[*subdir*] *dir* [-o *options* ]
+
+
+Description
+===========
+
+**mount.ceph** is a helper for mounting the Ceph file system on a Linux host.
+It serves to resolve monitor hostname(s) into IP addresses and read
+authentication keys from disk; the Linux kernel client component does most of
+the real work. To mount a Ceph file system use::
+
+  mount.ceph name@07fe3187-00d9-42a3-814b-72a4d5e7d5be.fs_name=/ /mnt/mycephfs -o mon_addr=1.2.3.4
+
+where "name" is the RADOS client name (referred to hereafter as "RADOS user",
+and meaning any individual or system actor such as an application). 
+
+Mount helper can fill in the cluster FSID by reading the ceph configuration file.
+Its recommended to call the mount helper via mount(8) as per::
+
+  mount -t ceph name@.fs_name=/ /mnt/mycephfs -o mon_addr=1.2.3.4
+
+Note that the dot ``.`` still needs to be a part of the device string in this case.
+
+The first argument is the device part of the mount command. It includes the
+RADOS user for authentication, the file system name and a path within CephFS
+that will be mounted at the mount point.
+
+Monitor addresses can be passed using ``mon_addr`` mount option. Multiple monitor
+addresses can be passed by separating addresses with a slash (`/`). Only one
+monitor is needed to mount successfully; the client will learn about all monitors
+from any responsive monitor. However, it is a good idea to specify more than one
+in case the one happens to be down at the time of mount. Monitor addresses takes
+the form ip_address[:port]. If the port is not specified, the Ceph default of 6789
+is assumed.
+
+If monitor addresses are not specified, then **mount.ceph** will attempt to determine
+monitor addresses using local configuration files and/or DNS SRV records. In similar
+way, if authentication is enabled on Ceph cluster (which is done using CephX) and
+options ``secret`` and ``secretfile`` are not specified in the command, the mount
+helper will spawn a child process that will use the standard Ceph library routines
+to find a keyring and fetch the secret from it (including the monitor address and
+FSID if those not specified).
+
+A sub-directory of the file system can be mounted by specifying the (absolute)
+path to the sub-directory right after "=" in the device part of the mount command.
+
+Mount helper application conventions dictate that the first two options are
+device to be mounted and the mountpoint for that device. Options must be
+passed only after these fixed arguments.
+
+
+Options
+=======
+
+Basic
+-----
+
+:command:`conf`
+    Path to a ceph.conf file. This is used to initialize the Ceph context
+    for autodiscovery of monitor addresses and auth secrets. The default is
+    to use the standard search path for ceph.conf files.
+
+:command:`mount_timeout`
+    int (seconds), Default: 60
+
+:command:`ms_mode=<legacy|crc|secure|prefer-crc|prefer-secure>`
+    Set the connection mode that the client uses for transport. The available
+    modes are:
+
+    - ``legacy``: use messenger v1 protocol to talk to the cluster
+
+    - ``crc``: use messenger v2, without on-the-wire encryption
+
+    - ``secure``: use messenger v2, with on-the-wire encryption
+
+    - ``prefer-crc``: crc mode, if denied agree to secure mode
+
+    - ``prefer-secure``: secure mode, if denied agree to crc mode
+
+:command:`mon_addr`
+    Monitor address of the cluster in the form of ip_address[:port]
+
+:command:`fsid`
+    Cluster FSID. This can be found using `ceph fsid` command.
+
+:command:`secret`
+    secret key for use with CephX. This option is insecure because it exposes
+    the secret on the command line. To avoid this, use the secretfile option.
+
+:command:`secretfile`
+    path to file containing the secret key to use with CephX
+
+:command:`recover_session=<no|clean>`
+    Set auto reconnect mode in the case where the client is blocklisted. The
+    available modes are ``no`` and ``clean``. The default is ``no``.
+
+    - ``no``: never attempt to reconnect when client detects that it has been
+      blocklisted. Blocklisted clients will not attempt to reconnect and
+      their operations will fail too.
+
+    - ``clean``: client reconnects to the Ceph cluster automatically when it
+      detects that it has been blocklisted. During reconnect, client drops
+      dirty data/metadata, invalidates page caches and writable file handles.
+      After reconnect, file locks become stale because the MDS loses track of
+      them. If an inode contains any stale file locks, read/write on the inode
+      is not allowed until applications release all stale file locks.
+
+:command: `fs=<fs-name>`
+    Specify the non-default file system to be mounted, when using the old syntax.
+
+:command: `mds_namespace=<fs-name>`
+    A synonym of "fs=" (Deprecated).
+
+Advanced
+--------
+:command:`cap_release_safety`
+    int, Default: calculated
+
+:command:`caps_wanted_delay_max`
+    int, cap release delay, Default: 60
+
+:command:`caps_wanted_delay_min`
+    int, cap release delay, Default: 5
+
+:command:`dirstat`
+    funky `cat dirname` for stats, Default: off
+
+:command:`nodirstat`
+    no funky `cat dirname` for stats
+
+:command:`ip`
+    my ip
+
+:command:`noasyncreaddir`
+    no dcache readdir
+
+:command:`nocrc`
+    no data crc on writes
+
+:command:`noshare`
+    create a new client instance, instead of sharing an existing instance of
+    a client mounting the same cluster
+
+:command:`osdkeepalive`
+    int, Default: 5
+
+:command:`osd_idle_ttl`
+    int (seconds), Default: 60
+
+:command:`rasize`
+    int (bytes), max readahead. Default: 8388608 (8192*1024)
+
+:command:`rbytes`
+    Report the recursive size of the directory contents for st_size on
+    directories.  Default: off
+
+:command:`norbytes`
+    Do not report the recursive size of the directory contents for
+    st_size on directories.
+
+:command:`readdir_max_bytes`
+    int, Default: 524288 (512*1024)
+
+:command:`readdir_max_entries`
+    int, Default: 1024
+
+:command:`rsize`
+    int (bytes), max read size. Default: 16777216 (16*1024*1024)
+
+:command:`snapdirname`
+    string, set the name of the hidden snapdir. Default: .snap
+
+:command:`write_congestion_kb`
+    int (kb), max writeback in flight. scale with available
+    memory. Default: calculated from available memory
+
+:command:`wsize`
+    int (bytes), max write size. Default: 16777216 (16*1024*1024) (writeback
+    uses smaller of wsize and stripe unit)
+
+:command:`wsync`
+    Execute all namespace operations synchronously. This ensures that the
+    namespace operation will only complete after receiving a reply from
+    the MDS. This is the default.
+
+:command:`nowsync`
+    Allow the client to do namespace operations asynchronously. When this
+    option is enabled, a namespace operation may complete before the MDS
+    replies, if it has sufficient capabilities to do so.
+
+Examples
+========
+
+Mount the full file system::
+
+    mount -t ceph fs_user@.mycephfs2=/ /mnt/mycephfs
+
+Mount only part of the namespace/file system::
+
+    mount.ceph fs_user@.mycephfs2=/some/directory/in/cephfs /mnt/mycephfs
+
+Pass the monitor host's IP address, optionally::
+
+    mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs -o mon_addr=192.168.0.1
+
+Pass the port along with IP address if it's running on a non-standard port::
+
+    mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs -o mon_addr=192.168.0.1:7000
+
+If there are multiple monitors, pass each address separated by a `/`::
+
+   mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs -o mon_addr=192.168.0.1/192.168.0.2/192.168.0.3
+
+Pass secret key for CephX user optionally::
+
+    mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs -o secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==
+
+Pass file containing secret key to avoid leaving secret key in shell's command
+history::
+
+    mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs -o secretfile=/etc/ceph/fs_username.secret
+
+If authentication is disabled on Ceph cluster, omit the credential related option::
+
+    mount.ceph fs_user@.mycephfs2=/ /mnt/mycephfs
+
+To mount using the old syntax::
+
+    mount -t ceph 192.168.0.1:/ /mnt/mycephfs
+
+Availability
+============
+
+**mount.ceph** is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com
+for more information.
+
+Feature Availability
+====================
+
+The ``recover_session=`` option was added to mainline Linux kernels in v5.4.
+``wsync`` and ``nowsync`` were added in v5.7.
+
+See also
+========
+
+:doc:`ceph-fuse <ceph-fuse>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/mount.fuse.ceph.rst b/doc/man/8/mount.fuse.ceph.rst
new file mode 100644
index 000000000..6dacd4719
--- /dev/null
+++ b/doc/man/8/mount.fuse.ceph.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+====================================================
+ mount.fuse.ceph -- mount ceph-fuse from /etc/fstab.
+====================================================
+
+.. program:: mount.fuse.ceph
+
+Synopsis
+========
+
+| **mount.fuse.ceph** [-h] [-o OPTIONS [*OPTIONS* ...]]
+                      device [*device* ...]
+                      mountpoint [*mountpoint* ...]
+
+Description
+===========
+
+**mount.fuse.ceph** is a helper for mounting ceph-fuse from
+``/etc/fstab``.
+
+To use mount.fuse.ceph, add an entry in ``/etc/fstab`` like::
+
+  DEVICE    PATH        TYPE        OPTIONS
+  none      /mnt/ceph   fuse.ceph   ceph.id=admin,_netdev,defaults  0 0
+  none      /mnt/ceph   fuse.ceph   ceph.name=client.admin,_netdev,defaults  0 0
+  none      /mnt/ceph   fuse.ceph   ceph.id=myuser,ceph.conf=/etc/ceph/foo.conf,_netdev,defaults  0 0
+
+ceph-fuse options are specified in the ``OPTIONS`` column and must begin
+with '``ceph.``' prefix. This way ceph related fs options will be passed to
+ceph-fuse and others will be ignored by ceph-fuse.
+
+Options
+=======
+
+.. option:: ceph.id=<username>
+
+   Specify that the ceph-fuse will authenticate as the given user.
+
+.. option:: ceph.name=client.admin
+
+   Specify that the ceph-fuse will authenticate as client.admin
+
+.. option:: ceph.conf=/etc/ceph/foo.conf
+
+   Sets 'conf' option to /etc/ceph/foo.conf via ceph-fuse command line.
+
+
+Any valid ceph-fuse options can be passed this way.   
+
+Additional Info
+===============
+
+The old format /etc/fstab entries are also supported::
+
+  DEVICE                              PATH        TYPE        OPTIONS
+  id=admin                            /mnt/ceph   fuse.ceph   defaults   0 0
+  id=myuser,conf=/etc/ceph/foo.conf   /mnt/ceph   fuse.ceph   defaults   0 0
+
+Availability
+============
+
+**mount.fuse.ceph** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+See also
+========
+
+:doc:`ceph-fuse <ceph-fuse>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/osdmaptool.rst b/doc/man/8/osdmaptool.rst
new file mode 100644
index 000000000..4ef5458e0
--- /dev/null
+++ b/doc/man/8/osdmaptool.rst
@@ -0,0 +1,368 @@
+:orphan:
+
+.. _osdmaptool:
+
+======================================================
+ osdmaptool -- ceph osd cluster map manipulation tool
+======================================================
+
+.. program:: osdmaptool
+
+Synopsis
+========
+
+| **osdmaptool** *mapfilename* [--print] [--createsimple *numosd*
+  [--pgbits *bitsperosd* ] ] [--clobber]
+| **osdmaptool** *mapfilename* [--import-crush *crushmap*]
+| **osdmaptool** *mapfilename* [--export-crush *crushmap*]
+| **osdmaptool** *mapfilename* [--upmap *file*] [--upmap-max *max-optimizations*]
+  [--upmap-deviation *max-deviation*] [--upmap-pool *poolname*]
+  [--save] [--upmap-active]
+| **osdmaptool** *mapfilename* [--upmap-cleanup] [--upmap *file*]
+
+
+Description
+===========
+
+**osdmaptool** is a utility that lets you create, view, and manipulate
+OSD cluster maps from the Ceph distributed storage system. Notably, it
+lets you extract the embedded CRUSH map or import a new CRUSH map.
+It can also simulate the upmap balancer mode so you can get a sense of
+what is needed to balance your PGs.
+
+
+Options
+=======
+
+.. option:: --print
+
+   will simply make the tool print a plaintext dump of the map, after
+   any modifications are made.
+
+.. option:: --dump <format>
+
+   displays the map in plain text when <format> is 'plain', 'json' if specified
+   format is not supported. This is an alternative to the print option.
+
+.. option:: --clobber
+
+   will allow osdmaptool to overwrite mapfilename if changes are made.
+
+.. option:: --import-crush mapfile
+
+   will load the CRUSH map from mapfile and embed it in the OSD map.
+
+.. option:: --export-crush mapfile
+
+   will extract the CRUSH map from the OSD map and write it to
+   mapfile.
+
+.. option:: --createsimple numosd [--pg-bits bitsperosd] [--pgp-bits bits]
+
+   will create a relatively generic OSD map with the numosd devices.
+   If --pg-bits is specified, the initial placement group counts will
+   be set with bitsperosd bits per OSD. That is, the pg_num map
+   attribute will be set to numosd shifted by bitsperosd.
+   If --pgp-bits is specified, then the pgp_num map attribute will
+   be set to numosd shifted by bits. 
+
+.. option:: --create-from-conf
+
+   creates an osd map with default configurations.
+
+.. option:: --test-map-pgs [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the mappings from placement groups to OSDs.
+   If range is specified, then it iterates from first to last in the directory 
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-map-pgs-dump [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the summary of all placement groups and the mappings from them to the mapped OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs-dump --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-map-pgs-dump-all [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the summary of all placement groups and the mappings
+   from them to all the OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs-dump-all --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-random
+
+   does a random mapping of placement groups to the OSDs.
+
+.. option:: --test-map-pg <pgid>
+
+   map a particular placement group(specified by pgid) to the OSDs.
+
+.. option:: --test-map-object <objectname> [--pool <poolid>]
+
+   map a particular placement group(specified by objectname) to the OSDs.
+
+.. option:: --test-crush [--range-first <first> --range-last <last>]
+
+   map placement groups to acting OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-crush --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --mark-up-in
+
+   mark osds up and in (but do not persist).
+
+.. option:: --mark-out
+
+   mark an osd as out (but do not persist)
+
+.. option:: --mark-up <osdid>
+
+   mark an osd as up (but do not persist)
+
+.. option:: --mark-in <osdid>
+
+   mark an osd as in (but do not persist)
+
+.. option:: --tree
+
+   Displays a hierarchical tree of the map.
+
+.. option:: --clear-temp
+
+   clears pg_temp and primary_temp variables.
+
+.. option:: --clean-temps
+
+   clean pg_temps.
+
+.. option:: --health
+
+   dump health checks
+
+.. option:: --with-default-pool
+
+   include default pool when creating map
+
+.. option:: --upmap-cleanup <file>
+
+   clean up pg_upmap[_items] entries, writing commands to <file> [default: - for stdout]
+
+.. option:: --upmap <file>
+
+   calculate pg upmap entries to balance pg layout writing commands to <file> [default: - for stdout]
+
+.. option:: --upmap-max <max-optimizations>
+
+   set max upmap entries to calculate [default: 10]
+
+.. option:: --upmap-deviation <max-deviation>
+
+   max deviation from target [default: 5]
+
+.. option:: --upmap-pool <poolname>
+
+   restrict upmap balancing to 1 pool or the option can be repeated for multiple pools
+
+.. option:: --upmap-active
+
+   Act like an active balancer, keep applying changes until balanced
+
+.. option:: --adjust-crush-weight <osdid:weight>[,<osdid:weight>,<...>]
+
+   Change CRUSH weight of <osdid>
+
+.. option:: --save
+
+   write modified osdmap with upmap or crush-adjust changes
+
+.. option:: --read <file>
+
+   calculate pg upmap entries to balance pg primaries
+
+.. option:: --read-pool <poolname>
+
+   specify which pool the read balancer should adjust
+
+.. option:: --vstart
+
+   prefix upmap and read output with './bin/'
+
+Example
+=======
+
+To create a simple map with 16 devices::
+
+        osdmaptool --createsimple 16 osdmap --clobber
+
+To view the result::
+
+        osdmaptool --print osdmap
+
+To view the mappings of placement groups for pool 1::
+
+        osdmaptool osdmap --test-map-pgs-dump --pool 1
+
+        pool 1 pg_num 8
+        1.0     [0,2,1] 0
+        1.1     [2,0,1] 2
+        1.2     [0,1,2] 0
+        1.3     [2,0,1] 2
+        1.4     [0,2,1] 0
+        1.5     [0,2,1] 0
+        1.6     [0,1,2] 0
+        1.7     [1,0,2] 1
+        #osd    count   first   primary c wt    wt
+        osd.0   8       5       5       1       1
+        osd.1   8       1       1       1       1
+        osd.2   8       2       2       1       1
+         in 3
+         avg 8 stddev 0 (0x) (expected 2.3094 0.288675x))
+         min osd.0 8
+         max osd.0 8
+        size 0  0
+        size 1  0
+        size 2  0
+        size 3  8
+
+In which,
+ #. pool 1 has 8 placement groups. And two tables follow:
+ #. A table for placement groups. Each row presents a placement group. With columns of:
+
+    * placement group id,
+    * acting set, and
+    * primary OSD.
+ #. A table for all OSDs. Each row presents an OSD. With columns of:
+
+    * count of placement groups being mapped to this OSD,
+    * count of placement groups where this OSD is the first one in their acting sets,
+    * count of placement groups where this OSD is the primary of them,
+    * the CRUSH weight of this OSD, and
+    * the weight of this OSD.
+ #. Looking at the number of placement groups held by 3 OSDs. We have
+
+    * average, stddev, stddev/average, expected stddev, expected stddev / average
+    * min and max
+ #. The number of placement groups mapping to n OSDs. In this case, all 8 placement
+    groups are mapping to 3 different OSDs.
+
+In a less-balanced cluster, we could have following output for the statistics of
+placement group distribution, whose standard deviation is 1.41421::
+
+        #osd    count   first   primary c wt    wt
+        osd.0   8       5       5       1       1
+        osd.1   8       1       1       1       1
+        osd.2   8       2       2       1       1
+
+        #osd    count   first    primary c wt    wt
+        osd.0   33      9        9       0.0145874     1
+        osd.1   34      14       14      0.0145874     1
+        osd.2   31      7        7       0.0145874     1
+        osd.3   31      13       13      0.0145874     1
+        osd.4   30      14       14      0.0145874     1
+        osd.5   33      7        7       0.0145874     1
+         in 6
+         avg 32 stddev 1.41421 (0.0441942x) (expected 5.16398 0.161374x))
+         min osd.4 30
+         max osd.1 34
+        size 00
+        size 10
+        size 20
+        size 364
+
+To simulate the active balancer in upmap mode::
+
+        osdmaptool --upmap upmaps.out --upmap-active --upmap-deviation 6 --upmap-max 11 osdmap
+
+   osdmaptool: osdmap file 'osdmap'
+   writing upmap command output to: upmaps.out
+   checking for upmap cleanups
+   upmap, max-count 11, max deviation 6
+   pools movies photos metadata data
+   prepared 11/11 changes
+   Time elapsed 0.00310404 secs
+   pools movies photos metadata data
+   prepared 11/11 changes
+   Time elapsed 0.00283402 secs
+   pools data metadata movies photos
+   prepared 11/11 changes
+   Time elapsed 0.003122 secs
+   pools photos metadata data movies
+   prepared 11/11 changes
+   Time elapsed 0.00324372 secs
+   pools movies metadata data photos
+   prepared 1/11 changes
+   Time elapsed 0.00222609 secs
+   pools data movies photos metadata
+   prepared 0/11 changes
+   Time elapsed 0.00209916 secs
+   Unable to find further optimization, or distribution is already perfect
+   osd.0 pgs 41
+   osd.1 pgs 42
+   osd.2 pgs 42
+   osd.3 pgs 41
+   osd.4 pgs 46
+   osd.5 pgs 39
+   osd.6 pgs 39
+   osd.7 pgs 43
+   osd.8 pgs 41
+   osd.9 pgs 46
+   osd.10 pgs 46
+   osd.11 pgs 46
+   osd.12 pgs 46
+   osd.13 pgs 41
+   osd.14 pgs 40
+   osd.15 pgs 40
+   osd.16 pgs 39
+   osd.17 pgs 46
+   osd.18 pgs 46
+   osd.19 pgs 39
+   osd.20 pgs 42
+   Total time elapsed 0.0167765 secs, 5 rounds
+
+To simulate the active balancer in read mode, first make sure capacity is balanced
+by running the balancer in upmap mode. Then, balance the reads on a replicated pool with::
+
+        osdmaptool osdmap --read read.out --read-pool <pool name>
+
+   ./bin/osdmaptool: osdmap file 'om'
+   writing upmap command output to: read.out
+
+   ---------- BEFORE ------------ 
+   osd.0 | primary affinity: 1 | number of prims: 3
+   osd.1 | primary affinity: 1 | number of prims: 10
+   osd.2 | primary affinity: 1 | number of prims: 3
+ 
+   read_balance_score of 'cephfs.a.meta': 1.88
+
+
+   ---------- AFTER ------------ 
+   osd.0 | primary affinity: 1 | number of prims: 5
+   osd.1 | primary affinity: 1 | number of prims: 5
+   osd.2 | primary affinity: 1 | number of prims: 6
+ 
+   read_balance_score of 'cephfs.a.meta': 1.13
+
+  
+   num changes: 5
+
+Availability
+============
+
+**osdmaptool** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
diff --git a/doc/man/8/rados.rst b/doc/man/8/rados.rst
new file mode 100644
index 000000000..d606c3a5e
--- /dev/null
+++ b/doc/man/8/rados.rst
@@ -0,0 +1,404 @@
+:orphan:
+
+=======================================
+ rados -- rados object storage utility
+=======================================
+
+.. program:: rados
+
+Synopsis
+========
+
+| **rados** [ *options* ] [ *command* ]
+
+
+Description
+===========
+
+**rados** is a utility for interacting with a Ceph object storage
+cluster (RADOS), part of the Ceph distributed storage system.
+
+
+Global Options
+==============
+
+.. option:: --object-locator object_locator
+
+   Set object_locator for operation.
+
+.. option:: -p pool, --pool pool
+
+   Interact with the given pool. Required by most commands.
+
+.. option:: --target-pool pool
+
+   Select target pool by name.
+
+.. option:: --pgid
+
+   As an alternative to ``--pool``, ``--pgid`` also allow users to specify the
+   PG id to which the command will be directed. With this option, certain
+   commands like ``ls`` allow users to limit the scope of the command to the given PG.
+
+.. option:: -N namespace, --namespace namespace
+
+   Specify the rados namespace to use for the object.
+
+.. option:: --all
+
+   Use with ls to list objects in all namespaces.
+   Put in CEPH_ARGS environment variable to make this the default.
+
+.. option:: --default
+
+   Use with ls to list objects in default namespace.
+   Takes precedence over --all in case --all is in environment.
+
+.. option:: -s snap, --snap snap
+
+   Read from the given pool snapshot. Valid for all pool-specific read operations.
+
+.. option:: --create
+
+   Create the pool or directory that was specified.
+
+.. option:: -i infile
+
+   will specify an input file to be passed along as a payload with the
+   command to the monitor cluster. This is only used for specific
+   monitor commands.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: -b block_size
+
+  Set the block size for put/get/append ops and for write benchmarking.
+
+.. option:: --striper
+
+   Uses the striping API of rados rather than the default one.
+   Available for stat, stat2, get, put, append, truncate, rm, ls
+   and all xattr related operation.
+
+.. option:: -O object_size, --object-size object_size
+
+   Set the object size for put/get ops and for write benchmarking.
+
+.. option:: --max-objects
+
+   Set the max number of objects for write benchmarking.
+
+.. option:: --lock-cookie locker-cookie
+
+   Will set the lock cookie for acquiring advisory lock (lock get command).
+   If the cookie is not empty, this option must be passed to lock break command
+   to find the correct lock when releasing lock.
+
+.. option:: --target-locator
+
+   Use with cp to specify the locator of the new object.
+
+.. option:: --target-nspace
+
+   Use with cp to specify the namespace of the new object.
+
+
+Bench options
+=============
+
+.. option:: -t N, --concurrent-ios=N
+
+   Set number of concurrent I/O operations.
+
+.. option:: --show-time
+
+   Prefix output with date/time.
+
+.. option:: --no-verify
+
+   Do not verify contents of read objects.
+
+.. option:: --write-object
+
+   Write contents to the objects.
+
+.. option:: --write-omap
+
+   Write contents to the omap.
+
+.. option:: --write-xattr
+
+   Write contents to the extended attributes.
+
+
+Load gen options
+================
+
+.. option:: --num-objects
+
+   Total number of objects.
+
+.. option:: --min-object-size
+
+  Min object size.
+
+.. option:: --max-object-size
+
+   Max object size.
+
+.. option:: --min-op-len
+
+   Min io size of operations.
+
+.. option:: --max-op-len
+
+   Max io size of operations.
+
+.. option:: --max-ops
+
+   Max number of operations.
+
+.. option:: --max-backlog
+
+   Max backlog size.
+
+.. option:: --read-percent
+
+   Percent of operations that are read.
+
+.. option:: --target-throughput
+
+   Target throughput (in bytes).
+
+.. option:: --run-length
+
+   Total time (in seconds).
+
+.. option:: --offset-align
+
+   At what boundary to align random op offsets.
+
+
+Cache pools options
+===================
+
+.. option:: --with-clones
+
+   Include clones when doing flush or evict.
+
+
+OMAP options
+============
+
+.. option:: --omap-key-file file
+
+   Read the omap key from a file.
+
+
+Generic options
+===============
+
+.. option:: -c FILE, --conf FILE
+
+   Read configuration from the given configuration file.
+
+.. option:: --id ID
+
+   Set ID portion of my name.
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set cephx user name.
+
+.. option:: --cluster NAME
+
+   Set cluster name (default: ceph).
+
+.. option:: --setuser USER
+
+   Set uid to user or uid (and gid to user's gid).
+
+.. option:: --setgroup GROUP
+
+   Set gid to group or gid.
+
+.. option:: --version
+
+   Show version and quit.
+
+
+Global commands
+===============
+
+:command:`lspools`
+  List object pools
+
+:command:`df`
+  Show utilization statistics, including disk usage (bytes) and object
+  counts, over the entire system and broken down by pool.
+
+:command:`list-inconsistent-pg` *pool*
+  List inconsistent PGs in given pool.
+
+:command:`list-inconsistent-obj` *pgid*
+  List inconsistent objects in given PG.
+
+:command:`list-inconsistent-snapset` *pgid*
+  List inconsistent snapsets in given PG.
+
+
+Pool specific commands
+======================
+
+:command:`get` *name* *outfile*
+  Read object name from the cluster and write it to outfile.
+
+:command:`put` *name* *infile* [--offset offset]
+  Write object name with start offset (default:0) to the cluster with contents from infile.
+  **Warning:** The put command creates a single RADOS object, sized just as
+  large as your input file. Unless your objects are of reasonable and consistent sizes, that
+  is probably not what you want -- consider using RGW/S3, CephFS, or RBD instead.
+
+:command:`append` *name* *infile*
+  Append object name to the cluster with contents from infile.
+
+:command:`rm` [--force-full] *name* ...
+  Remove object(s) with name(s). With ``--force-full`` will remove when cluster is marked full.
+
+:command:`listwatchers` *name*
+  List the watchers of object name.
+
+:command:`ls` *outfile*
+  List objects in the given pool and write to outfile. Instead of ``--pool`` if ``--pgid`` will be specified, ``ls`` will only list the objects in the given PG.
+
+:command:`lssnap`
+  List snapshots for given pool.
+
+:command:`clonedata` *srcname* *dstname* --object-locator *key*
+  Clone object byte data from *srcname* to *dstname*.  Both objects must be stored with the locator key *key* (usually either *srcname* or *dstname*).  Object attributes and omap keys are not copied or cloned.
+
+:command:`mksnap` *foo*
+  Create pool snapshot named *foo*.
+
+:command:`rmsnap` *foo*
+  Remove pool snapshot named *foo*.
+
+:command:`bench` *seconds* *mode* [ -b *objsize* ] [ -t *threads* ]
+  Benchmark for *seconds*. The mode can be *write*, *seq*, or
+  *rand*. *seq* and *rand* are read benchmarks, either
+  sequential or random. Before running one of the reading benchmarks,
+  run a write benchmark with the *--no-cleanup* option. The default
+  object size is 4 MB, and the default number of simulated threads
+  (parallel writes) is 16. The *--run-name <label>* option is useful
+  for benchmarking a workload test from multiple clients. The *<label>*
+  is an arbitrary object name. It is "benchmark_last_metadata" by
+  default, and is used as the underlying object name for "read" and
+  "write" ops.
+  Note: -b *objsize* option is valid only in *write* mode.
+  Note: *write* and *seq* must be run on the same host otherwise the
+  objects created by *write* will have names that will fail *seq*.
+
+:command:`cleanup` [ --run-name *run_name* ] [ --prefix *prefix* ]
+  Clean up a previous benchmark operation.
+  Note: the default run-name is "benchmark_last_metadata"
+
+:command:`listxattr` *name*
+  List all extended attributes of an object.
+
+:command:`getxattr` *name* *attr*
+  Dump the extended attribute value of *attr* of an object.
+
+:command:`setxattr` *name* *attr* *value*
+  Set the value of *attr* in the extended attributes of an object.
+
+:command:`rmxattr` *name* *attr*
+  Remove *attr* from the extended attributes of an object.
+
+:command:`stat` *name*
+   Get stat (ie. mtime, size) of given object
+
+:command:`stat2` *name*
+   Get stat (similar to stat, but with high precision time) of given object
+
+:command:`listomapkeys` *name*
+  List all the keys stored in the object map of object name.
+
+:command:`listomapvals` *name*
+  List all key/value pairs stored in the object map of object name.
+  The values are dumped in hexadecimal.
+
+:command:`getomapval` [ --omap-key-file *file* ] *name* *key* [ *out-file* ]
+  Dump the hexadecimal value of key in the object map of object name.
+  If the optional *out-file* argument is not provided, the value will be
+  written to standard output.
+
+:command:`setomapval` [ --omap-key-file *file* ] *name* *key* [ *value* ]
+  Set the value of key in the object map of object name. If the optional
+  *value* argument is not provided, the value will be read from standard
+  input.
+
+:command:`rmomapkey` [ --omap-key-file *file* ] *name* *key*
+  Remove key from the object map of object name.
+
+:command:`getomapheader` *name*
+  Dump the hexadecimal value of the object map header of object name.
+
+:command:`setomapheader` *name* *value*
+  Set the value of the object map header of object name.
+
+:command:`export` *filename*
+  Serialize pool contents to a file or standard output.\n"
+
+:command:`import` [--dry-run] [--no-overwrite] < filename | - >
+  Load pool contents from a file or standard input
+
+
+Examples
+========
+
+To view cluster utilization::
+
+       rados df
+
+To get a list object in pool foo sent to stdout::
+
+       rados -p foo ls -
+
+To get a list of objects in PG 0.6::
+
+       rados --pgid 0.6 ls
+
+To write an object::
+
+       rados -p foo put myobject blah.txt
+
+To create a snapshot::
+
+       rados -p foo mksnap mysnap
+
+To delete the object::
+
+       rados -p foo rm myobject
+
+To read a previously snapshotted version of an object::
+
+       rados -p foo -s mysnap get myobject blah.txt.old
+
+To list inconsistent objects in PG 0.6::
+
+       rados list-inconsistent-obj 0.6 --format=json-pretty
+
+
+Availability
+============
+
+**rados** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst
new file mode 100644
index 000000000..1a6358330
--- /dev/null
+++ b/doc/man/8/radosgw-admin.rst
@@ -0,0 +1,1042 @@
+:orphan:
+
+=================================================================
+ radosgw-admin -- rados REST gateway user administration utility
+=================================================================
+
+.. program:: radosgw-admin
+
+Synopsis
+========
+
+| **radosgw-admin** *command* [ *options* *...* ]
+
+
+Description
+===========
+
+:program:`radosgw-admin` is a Ceph Object Gateway user administration utility. It
+is used to create and modify users.
+
+
+Commands
+========
+
+:program:`radosgw-admin` utility provides commands for administration purposes
+as follows:
+
+:command:`user create`
+  Create a new user.
+
+:command:`user modify`
+  Modify a user.
+
+:command:`user info`
+  Display information for a user including any subusers and keys.
+
+:command:`user rename`
+  Renames a user.
+
+:command:`user rm`
+  Remove a user.
+
+:command:`user suspend`
+  Suspend a user.
+
+:command:`user enable`
+  Re-enable user after suspension.
+
+:command:`user check`
+  Check user info.
+
+:command:`user stats`
+  Show user stats as accounted by the quota subsystem.
+
+:command:`user list`
+  List all users.
+
+:command:`caps add`
+  Add user capabilities.
+
+:command:`caps rm`
+  Remove user capabilities.
+
+:command:`subuser create`
+  Create a new subuser (primarily useful for clients using the Swift API).
+
+:command:`subuser modify`
+  Modify a subuser.
+
+:command:`subuser rm`
+  Remove a subuser.
+
+:command:`key create`
+  Create access key.
+
+:command:`key rm`
+  Remove access key.
+
+:command:`bucket list`
+  List buckets, or, if a bucket is specified with --bucket=<bucket>,
+  list its objects. Adding --allow-unordered
+  removes the ordering requirement, possibly generating results more
+  quickly for buckets with large number of objects.
+
+:command:`bucket limit check`
+  Show bucket sharding stats.
+
+:command:`bucket link`
+  Link bucket to specified user.
+
+:command:`bucket unlink`
+  Unlink bucket from specified user.
+
+:command:`bucket chown`
+  Change bucket ownership to the specified user and update object ACLs. 
+  Invoke with --marker to resume if the command is interrupted.
+
+:command:`bucket stats`
+  Returns bucket statistics.
+
+:command:`bucket rm`
+  Remove a bucket.
+
+:command:`bucket check`
+  Check bucket index.
+
+:command:`bucket rewrite`
+  Rewrite all objects in the specified bucket.
+
+:command:`bucket radoslist`
+  List the RADOS objects that contain the data for all objects in
+  the designated bucket, if --bucket=<bucket> is specified. 
+  Otherwise, list the RADOS objects that contain data for all 
+  buckets.
+
+:command:`bucket reshard`
+  Reshard a bucket's index.
+
+:command:`bucket sync disable`
+  Disable bucket sync.
+
+:command:`bucket sync enable`
+  Enable bucket sync.
+
+:command:`bi get`
+  Retrieve bucket index object entries.
+
+:command:`bi put`
+  Store bucket index object entries.
+
+:command:`bi list`
+  List raw bucket index entries.
+
+:command:`bi purge`
+  Purge bucket index entries.
+
+:command:`object rm`
+  Remove an object.
+
+:command:`object stat`
+  Stat an object for its metadata.
+
+:command:`object unlink`
+  Unlink object from bucket index.
+
+:command:`object rewrite`
+  Rewrite the specified object.
+
+:command:`objects expire`
+  Run expired objects cleanup.
+
+:command:`period rm`
+  Remove a period.
+
+:command:`period get`
+  Get the period info.
+
+:command:`period get-current`
+  Get the current period info.
+
+:command:`period pull`
+  Pull a period.
+
+:command:`period push`
+  Push a period.
+
+:command:`period list`
+  List all periods.
+
+:command:`period update`
+  Update the staging period.
+
+:command:`period commit`
+  Commit the staging period.
+
+:command:`quota set`
+  Set quota params.
+
+:command:`quota enable`
+  Enable quota.
+
+:command:`quota disable`
+  Disable quota.
+
+:command:`global quota get`
+  View global quota parameters.
+
+:command:`global quota set`
+  Set global quota parameters.
+
+:command:`global quota enable`
+  Enable a global quota.
+
+:command:`global quota disable`
+  Disable a global quota.
+
+:command:`realm create`
+  Create a new realm.
+
+:command:`realm rm`
+  Remove a realm.
+
+:command:`realm get`
+  Show the realm info.
+
+:command:`realm get-default`
+  Get the default realm name.
+
+:command:`realm list`
+  List all realms.
+
+:command:`realm list-periods`
+  List all realm periods.
+
+:command:`realm rename`
+  Rename a realm.
+
+:command:`realm set`
+  Set the realm info (requires infile).
+
+:command:`realm default`
+  Set the realm as default.
+
+:command:`realm pull`
+  Pull a realm and its current period.
+
+:command:`zonegroup add`
+  Add a zone to a zonegroup.
+
+:command:`zonegroup create`
+  Create a new zone group info.
+
+:command:`zonegroup default`
+  Set the default zone group.
+
+:command:`zonegroup rm`
+  Remove a zone group info.
+
+:command:`zonegroup get`
+  Show the zone group info.
+
+:command:`zonegroup modify`
+  Modify an existing zonegroup.
+
+:command:`zonegroup set`
+  Set the zone group info (requires infile).
+
+:command:`zonegroup remove`
+  Remove a zone from a zonegroup.
+
+:command:`zonegroup rename`
+  Rename a zone group.
+
+:command:`zonegroup list`
+  List all zone groups set on this cluster.
+
+:command:`zonegroup placement list`
+  List zonegroup's placement targets.
+
+:command:`zonegroup placement add`
+  Add a placement target id to a zonegroup.
+
+:command:`zonegroup placement modify`
+  Modify a placement target of a specific zonegroup.
+
+:command:`zonegroup placement rm`
+  Remove a placement target from a zonegroup.
+
+:command:`zonegroup placement default`
+  Set a zonegroup's default placement target.
+
+:command:`zone create`
+  Create a new zone.
+
+:command:`zone rm`
+  Remove a zone.
+
+:command:`zone get`
+  Show zone cluster params.
+
+:command:`zone set`
+  Set zone cluster params (requires infile).
+
+:command:`zone modify`
+  Modify an existing zone.
+
+:command:`zone list`
+  List all zones set on this cluster.
+
+:command:`metadata sync status`
+  Get metadata sync status.
+
+:command:`metadata sync init`
+  Init metadata sync.
+
+:command:`metadata sync run`
+  Run metadata sync.
+
+:command:`data sync status`
+  Get data sync status of the specified source zone.
+  
+:command:`data sync init`
+  Init data sync for the specified source zone.
+
+:command:`data sync run`
+  Run data sync for the specified source zone.
+
+:command:`sync error list`
+  List sync errors.
+
+:command:`sync error trim`
+  Trim sync errors.
+
+:command:`zone rename`
+  Rename a zone.
+
+:command:`zone placement list`
+  List a zone's placement targets.
+
+:command:`zone placement add`
+  Add a zone placement target.
+
+:command:`zone placement modify`
+  Modify a zone placement target.
+
+:command:`zone placement rm`
+  Remove a zone placement target.
+
+:command:`pool add`
+  Add an existing pool for data placement.
+
+:command:`pool rm`
+  Remove an existing pool from data placement set.
+
+:command:`pools list`
+  List placement active set.
+
+:command:`policy`
+  Display bucket/object policy.
+
+:command:`log list`
+  List log objects.
+
+:command:`log show`
+  Dump a log from specific object or (bucket + date + bucket-id).
+  (NOTE: required to specify formatting of date to "YYYY-MM-DD-hh")
+
+:command:`log rm`
+  Remove log object.
+
+:command:`usage show`
+  Show the usage information (with optional user and date range).
+
+:command:`usage trim`
+  Trim usage information (with optional user and date range).
+
+:command:`gc list`
+  Dump expired garbage collection objects (specify --include-all to list all
+  entries, including unexpired).
+
+:command:`gc process`
+  Manually process garbage.
+
+:command:`lc list`
+  List all bucket lifecycle progress.
+
+:command:`lc process`
+  Manually process lifecycle transitions.  If a bucket is specified (e.g., via
+  --bucket_id or via --bucket and optional --tenant), only that bucket
+  is processed.
+
+:command:`metadata get`
+  Get metadata info.
+
+:command:`metadata put`
+  Put metadata info.
+
+:command:`metadata rm`
+  Remove metadata info.
+
+:command:`metadata list`
+  List metadata info.
+
+:command:`mdlog list`
+  List metadata log which is needed for multi-site deployments.
+
+:command:`mdlog trim`
+  Trim metadata log manually instead of relying on the gateway's integrated log sync.
+  Before trimming, compare the listings and make sure the last sync was
+  complete, otherwise it can reinitiate a sync.
+
+:command:`mdlog status`
+  Read metadata log status.
+
+:command:`bilog list`
+  List bucket index log which is needed for multi-site deployments.
+
+:command:`bilog trim`
+  Trim bucket index log (use start-marker, end-marker) manually instead
+  of relying on the gateway's integrated log sync.
+  Before trimming, compare the listings and make sure the last sync was
+  complete, otherwise it can reinitiate a sync.
+
+:command:`datalog list`
+  List data log which is needed for multi-site deployments.
+
+:command:`datalog trim`
+  Trim data log manually instead of relying on the gateway's integrated log sync.
+  Before trimming, compare the listings and make sure the last sync was
+  complete, otherwise it can reinitiate a sync.
+
+:command:`datalog status`
+  Read data log status.
+
+:command:`orphans find`
+  Init and run search for leaked RADOS objects.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`orphans finish`
+  Clean up search for leaked RADOS objects.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`orphans list-jobs`
+  List the current orphans search job IDs.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`role create`
+  Create a new role for use with STS (Security Token Service).
+
+:command:`role rm`
+  Remove a role.
+
+:command:`role get`
+  Get a role.
+
+:command:`role list`
+  List the roles with specified path prefix.
+
+:command:`role modify`
+  Modify the assume role policy of an existing role.
+
+:command:`role-policy put`
+  Add/update permission policy to role.
+
+:command:`role-policy list`
+  List the policies attached to a role.
+
+:command:`role-policy get`
+  Get the specified inline policy document embedded with the given role.
+
+:command:`role-policy rm`
+  Remove the policy attached to a role
+
+:command:`reshard add`
+  Schedule a resharding of a bucket
+
+:command:`reshard list`
+  List all bucket resharding or scheduled to be resharded
+
+:command:`reshard process`
+  Process of scheduled reshard jobs
+
+:command:`reshard status`
+  Resharding status of a bucket
+
+:command:`reshard cancel`
+  Cancel resharding a bucket
+
+:command:`topic list`
+  List bucket notifications/pubsub topics                                                   
+
+:command:`topic get`
+  Get a bucket notifications/pubsub topic                                                   
+  
+:command:`topic rm`
+  Remove a bucket notifications/pubsub topic                                                
+
+:command:`subscription get`
+  Get a pubsub subscription definition
+
+:command:`subscription rm`
+  Remove a pubsub subscription
+
+:command:`subscription pull`
+  Show events in a pubsub subscription
+             
+:command:`subscription ack`
+  Acknowledge (remove) events in a pubsub subscription
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of selecting one
+   from ceph.conf).
+
+.. option:: --tenant=<tenant>
+
+   Name of the tenant.
+
+.. option:: --uid=uid
+
+   The user on which to operate.
+
+.. option:: --new-uid=uid
+
+   The new ID of the user. Used with 'user rename' command.
+
+.. option:: --subuser=<name>
+
+    Name of the subuser.
+
+.. option:: --access-key=<key>
+
+   S3 access key.
+
+.. option:: --email=email
+
+   The e-mail address of the user.
+
+.. option:: --secret/--secret-key=<key>
+
+   The secret key.
+
+.. option:: --gen-access-key
+
+    Generate random access key (for S3).
+
+
+.. option:: --gen-secret
+
+    Generate random secret key.
+
+.. option:: --key-type=<type>
+
+    Key type, options are: swift, s3.
+
+.. option:: --temp-url-key[-2]=<key>
+
+    Temporary URL key.
+
+.. option:: --max-buckets
+
+    Maximum number of buckets for a user (0 for no limit, negative value to disable bucket creation).
+    Default is 1000.
+
+.. option:: --access=<access>
+
+   Set the access permissions for the subuser.
+   Available access permissions are read, write, readwrite and full.
+
+.. option:: --display-name=<name>
+
+   The display name of the user.
+
+.. option:: --admin
+
+   Set the admin flag on the user.
+
+.. option:: --system
+
+   Set the system flag on the user.
+
+.. option:: --bucket=[tenant-id/]bucket
+
+   Specify the bucket name.  If tenant-id is not specified, the tenant-id
+   of the user (--uid) is used.
+
+.. option:: --pool=<pool>
+
+   Specify the pool name.
+   Also used with `orphans find` as data pool to scan for leaked rados objects.
+
+.. option:: --object=object
+
+   Specify the object name.
+
+.. option:: --date=yyyy-mm-dd
+
+   The date in the format yyyy-mm-dd.
+
+.. option:: --start-date=yyyy-mm-dd
+
+   The start date in the format yyyy-mm-dd.
+
+.. option:: --end-date=yyyy-mm-dd
+
+   The end date in the format yyyy-mm-dd.
+
+.. option:: --bucket-id=<bucket-id>
+
+   Specify the bucket id.
+
+.. option:: --bucket-new-name=[tenant-id/]<bucket>
+
+   Optional for `bucket link`; use to rename a bucket.
+   While the tenant-id can be specified, this is not
+   necessary in normal operation.
+
+.. option:: --shard-id=<shard-id>
+
+   Optional for mdlog list, bi list, data sync status. Required for ``mdlog trim``.
+
+.. option:: --max-entries=<entries>
+
+   Optional for listing operations to specify the max entries.
+
+.. option:: --purge-data
+
+   When specified, user removal will also purge the user's data.
+
+.. option:: --purge-keys
+
+   When specified, subuser removal will also purge the subuser' keys.
+   
+.. option:: --purge-objects
+
+   When specified, the bucket removal will also purge all objects in it.
+
+.. option:: --metadata-key=<key>
+
+   Key from which to retrieve metadata, used with ``metadata get``.
+
+.. option:: --remote=<remote>
+
+   Zone or zonegroup id of remote gateway.
+
+.. option:: --period=<id>
+
+   Period ID.
+
+.. option:: --url=<url>
+
+   URL for pushing/pulling period or realm.
+
+.. option:: --epoch=<number>
+
+   Period epoch.
+
+.. option:: --commit
+
+   Commit the period during 'period update'.
+
+.. option:: --staging
+
+   Get the staging period info.
+
+.. option:: --master
+
+   Set as master.
+
+.. option:: --master-zone=<id>
+
+   Master zone ID.
+
+.. option:: --rgw-realm=<name>
+
+   The realm name.
+
+.. option:: --realm-id=<id>
+
+   The realm ID.
+
+.. option:: --realm-new-name=<name>
+
+   New name for the realm.
+
+.. option:: --rgw-zonegroup=<name>
+
+   The zonegroup name.
+
+.. option:: --zonegroup-id=<id>
+
+   The zonegroup ID.
+
+.. option:: --zonegroup-new-name=<name>
+
+   The new name of the zonegroup.
+
+.. option:: --rgw-zone=<zone>
+
+   Zone in which the gateway is running.
+
+.. option:: --zone-id=<id>
+
+   The zone ID.
+
+.. option:: --zone-new-name=<name>
+
+   The new name of the zone.
+
+.. option:: --source-zone
+
+   The source zone for data sync.
+
+.. option:: --default
+
+   Set the entity (realm, zonegroup, zone) as default.
+
+.. option:: --read-only
+
+   Set the zone as read-only when adding to the zonegroup.
+
+.. option:: --placement-id
+
+   Placement ID for the zonegroup placement commands.
+
+.. option:: --tags=<list>
+
+   The list of tags for zonegroup placement add and modify commands.
+
+.. option:: --tags-add=<list>
+
+   The list of tags to add for zonegroup placement modify command.
+
+.. option:: --tags-rm=<list>
+
+   The list of tags to remove for zonegroup placement modify command.
+
+.. option:: --endpoints=<list>
+
+   The zone endpoints.
+
+.. option:: --index-pool=<pool>
+
+   The placement target index pool.
+
+.. option:: --data-pool=<pool>
+
+   The placement target data pool.
+
+.. option:: --data-extra-pool=<pool>
+
+   The placement target data extra (non-EC) pool.
+
+.. option:: --placement-index-type=<type>
+
+   The placement target index type (normal, indexless, or #id).
+
+.. option:: --placement-inline-data=<true>
+
+   Whether the placement target is configured to store a data chunk inline in head objects.
+
+.. option:: --tier-type=<type>
+
+   The zone tier type.
+
+.. option:: --tier-config=<k>=<v>[,...]
+
+   Set zone tier config keys, values.
+
+.. option:: --tier-config-rm=<k>[,...]
+
+   Unset zone tier config keys.
+
+.. option:: --sync-from-all[=false]
+
+   Set/reset whether zone syncs from all zonegroup peers.
+
+.. option:: --sync-from=[zone-name][,...]
+
+   Set the list of zones from which to sync.
+
+.. option:: --sync-from-rm=[zone-name][,...]
+
+   Remove zone(s) from list of zones from which to sync.
+
+.. option:: --bucket-index-max-shards
+
+   Override a zone's or zonegroup's default number of bucket index shards. This
+   option is accepted by the 'zone create', 'zone modify', 'zonegroup add',
+   and 'zonegroup modify' commands, and applies to buckets that are created
+   after the zone/zonegroup changes take effect.
+
+.. option:: --fix
+
+    Fix the bucket index in addition to checking it.
+
+.. option:: --check-objects
+
+    Bucket check: Rebuilds the bucket index according to actual object state.
+
+.. option:: --format=<format>
+
+    Specify output format for certain operations. Supported formats: xml, json.
+
+.. option:: --sync-stats
+
+    Option for the 'user stats' command. When specified, it will update user stats with
+    the current stats reported by the user's buckets indexes.
+
+.. option:: --show-config
+
+    Show configuration.
+
+.. option:: --show-log-entries=<flag>
+
+    Enable/disable dumping of log entries on log show.
+
+.. option:: --show-log-sum=<flag>
+
+    Enable/disable dump of log summation on log show.
+
+.. option:: --skip-zero-entries
+
+    Log show only dumps entries that don't have zero value in one of the numeric
+    field.
+
+.. option:: --infile
+
+    Specify a file to read when setting data.
+
+.. option:: --categories=<list>
+
+    Comma separated list of categories, used in usage show.
+
+.. option:: --caps=<caps>
+
+    List of capabilities (e.g., "usage=read, write; user=read").
+
+.. option:: --compression=<compression-algorithm>
+
+    Placement target compression algorithm (lz4|snappy|zlib|zstd).
+
+.. option:: --yes-i-really-mean-it
+
+    Required as a guardrail for certain destructive operations.
+
+.. option:: --min-rewrite-size
+
+    Specify the minimum object size for bucket rewrite (default 4M).
+
+.. option:: --max-rewrite-size
+
+    Specify the maximum object size for bucket rewrite (default ULLONG_MAX).
+
+.. option:: --min-rewrite-stripe-size
+
+    Specify the minimum stripe size for object rewrite (default 0). If the value
+    is set to 0, then the specified object will always be
+    rewritten when restriping.
+
+.. option:: --warnings-only
+
+   When specified with bucket limit check,
+   list only buckets nearing or over the current max objects per shard value.
+
+.. option:: --bypass-gc
+
+   When specified with bucket deletion,
+   triggers object deletion without involving GC.
+
+.. option:: --inconsistent-index
+
+   When specified with bucket deletion and bypass-gc set to true,
+   ignores bucket index consistency.
+
+.. option:: --max-concurrent-ios
+
+   Maximum concurrent bucket operations. Affects operations that
+   scan the bucket index, e.g., listing, deletion, and all scan/search
+   operations such as finding orphans or checking the bucket index.
+   The default is 32.
+
+Quota Options
+=============
+
+.. option:: --max-objects
+
+   Specify the maximum number of objects (negative value to disable).
+
+.. option:: --max-size
+
+    Specify the maximum object size (in B/K/M/G/T, negative value to disable).
+
+.. option:: --quota-scope
+
+    The scope of quota (bucket, user).
+
+
+Orphans Search Options
+======================
+
+.. option:: --num-shards
+
+    Number of shards to use for temporary scan info
+
+.. option:: --orphan-stale-secs
+
+   Number of seconds to wait before declaring an object to be an orphan.
+   The efault is 86400 (24 hours).
+
+.. option:: --job-id
+
+   Set the job id (for orphans find)
+
+
+Orphans list-jobs options
+=========================
+
+.. option:: --extra-info
+
+   Provide extra info in the job list.
+
+
+Role Options
+============
+
+.. option:: --role-name
+
+   The name of the role to create.
+
+.. option:: --path
+
+   The path to the role.
+
+.. option:: --assume-role-policy-doc
+
+   The trust relationship policy document that grants an entity permission to
+   assume the role.
+
+.. option:: --policy-name
+
+   The name of the policy document.
+
+.. option:: --policy-doc
+
+   The permission policy document.
+
+.. option:: --path-prefix
+
+   The path prefix for filtering the roles.
+
+
+Bucket Notifications/PubSub Options
+===================================
+.. option:: --topic                   
+
+   The bucket notifications/pubsub topic name.
+
+.. option:: --subscription
+
+   The pubsub subscription name.
+
+.. option:: --event-id
+
+   The event id in a pubsub subscription.
+
+
+Examples
+========
+
+Generate a new user::
+
+        $ radosgw-admin user create --display-name="johnny rotten" --uid=johnny
+        { "user_id": "johnny",
+          "rados_uid": 0,
+          "display_name": "johnny rotten",
+          "email": "",
+          "suspended": 0,
+          "subusers": [],
+          "keys": [
+                { "user": "johnny",
+                  "access_key": "TCICW53D9BQ2VGC46I44",
+                  "secret_key": "tfm9aHMI8X76L3UdgE+ZQaJag1vJQmE6HDb5Lbrz"}],
+          "swift_keys": []}
+
+Remove a user::
+
+        $ radosgw-admin user rm --uid=johnny
+
+Rename a user::
+
+        $ radosgw-admin user rename --uid=johnny --new-uid=joe
+        
+Remove a user and all associated buckets with their contents::
+
+        $ radosgw-admin user rm --uid=johnny --purge-data
+
+Remove a bucket::
+
+	$ radosgw-admin bucket rm --bucket=foo
+
+Link bucket to specified user::
+	
+	$ radosgw-admin bucket link --bucket=foo --bucket_id=<bucket id> --uid=johnny
+
+Unlink bucket from specified user::
+
+        $ radosgw-admin bucket unlink --bucket=foo --uid=johnny
+
+Rename a bucket::
+
+        $ radosgw-admin bucket link --bucket=foo --bucket-new-name=bar --uid=johnny
+
+Move a bucket from the old global tenant space to a specified tenant::
+
+        $ radosgw-admin bucket link --bucket=foo --uid='12345678$12345678'
+
+Link bucket to specified user and change object ACLs::
+
+        $ radosgw-admin bucket chown --bucket=foo --uid='12345678$12345678'
+
+Show the logs of a bucket from April 1st, 2012::
+
+        $ radosgw-admin log show --bucket=foo --date=2012-04-01-01 --bucket-id=default.14193.1
+
+Show usage information for user from March 1st to (but not including) April 1st, 2012::
+
+        $ radosgw-admin usage show --uid=johnny \
+                        --start-date=2012-03-01 --end-date=2012-04-01
+
+Show only summary of usage information for all users::
+
+        $ radosgw-admin usage show --show-log-entries=false
+
+Trim usage information for user until March 1st, 2012::
+
+        $ radosgw-admin usage trim --uid=johnny --end-date=2012-04-01
+
+
+Availability
+============
+
+:program:`radosgw-admin` is part of Ceph, a massively scalable, open-source,
+distributed storage system.  Please refer to the Ceph documentation at
+https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`radosgw <radosgw>`\(8)
diff --git a/doc/man/8/radosgw.rst b/doc/man/8/radosgw.rst
new file mode 100644
index 000000000..d31f3de8e
--- /dev/null
+++ b/doc/man/8/radosgw.rst
@@ -0,0 +1,229 @@
+:orphan:
+
+===============================
+ radosgw -- rados REST gateway
+===============================
+
+.. program:: radosgw
+
+Synopsis
+========
+
+| **radosgw**
+
+
+Description
+===========
+
+:program:`radosgw` is an HTTP REST gateway for the RADOS object store, a part
+of the Ceph distributed storage system. It is implemented as a FastCGI
+module using libfcgi, and can be used in conjunction with any FastCGI
+capable web server.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for radosgw
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.radosgw.gateway)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+.. option:: --rgw-region=region
+
+   The region where radosgw runs
+
+.. option:: --rgw-zone=zone
+
+   The zone where radosgw runs
+
+
+Configuration
+=============
+
+Earlier RADOS Gateway had to be configured with ``Apache`` and ``mod_fastcgi``.
+Now, ``mod_proxy_fcgi`` module is used instead of ``mod_fastcgi``.
+``mod_proxy_fcgi`` works differently than a traditional FastCGI module. This
+module requires the service of ``mod_proxy`` which provides support for the
+FastCGI protocol. So, to be able to handle FastCGI protocol, both ``mod_proxy``
+and ``mod_proxy_fcgi`` have to be present in the server. Unlike ``mod_fastcgi``,
+``mod_proxy_fcgi`` cannot start the application process. Some platforms have
+``fcgistarter`` for that purpose. However, external launching of application
+or process management may be available in the FastCGI application framework
+in use.
+
+``Apache`` must be configured in a way that enables ``mod_proxy_fcgi`` to be
+used with localhost tcp.
+
+The following steps show the configuration in Ceph's configuration file i.e,
+``/etc/ceph/ceph.conf`` and the gateway configuration file i.e,
+``/etc/httpd/conf.d/rgw.conf`` (RPM-based distros) or
+``/etc/apache2/conf-available/rgw.conf`` (Debian-based distros) with localhost
+tcp:
+
+#. For distros with Apache 2.2 and early versions of Apache 2.4 that use
+   localhost TCP, append the following contents to ``/etc/ceph/ceph.conf``::
+
+	[client.radosgw.gateway]
+	host = {hostname}
+	keyring = /etc/ceph/ceph.client.radosgw.keyring
+	log_file = /var/log/ceph/client.radosgw.gateway.log
+	rgw_frontends = fastcgi socket_port=9000 socket_host=0.0.0.0
+	rgw_print_continue = false
+
+#. Add the following content in the gateway configuration file:
+
+   For Debian/Ubuntu add in ``/etc/apache2/conf-available/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/apache2/rgw_error.log
+		CustomLog /var/log/apache2/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / fcgi://localhost:9000/
+
+		</VirtualHost>
+
+   For CentOS/RHEL add in ``/etc/httpd/conf.d/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/httpd/rgw_error.log
+		CustomLog /var/log/httpd/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / fcgi://localhost:9000/
+
+		</VirtualHost>
+
+#. Add the following content in the gateway configuration file:
+
+   For CentOS/RHEL add in ``/etc/httpd/conf.d/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/httpd/rgw_error.log
+		CustomLog /var/log/httpd/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / unix:///var/run/ceph/ceph.radosgw.gateway.fastcgi.sock|fcgi://localhost:9000/
+
+		</VirtualHost>
+
+#. Generate a key for radosgw to use for authentication with the cluster. ::
+
+	ceph-authtool -C -n client.radosgw.gateway --gen-key /etc/ceph/keyring.radosgw.gateway
+	ceph-authtool -n client.radosgw.gateway --cap mon 'allow rw' --cap osd 'allow rwx' /etc/ceph/keyring.radosgw.gateway
+
+#. Add the key to the auth entries. ::
+
+	ceph auth add client.radosgw.gateway --in-file=keyring.radosgw.gateway
+
+#. Start Apache and radosgw.
+
+   Debian/Ubuntu::
+
+		sudo /etc/init.d/apache2 start
+		sudo /etc/init.d/radosgw start
+
+   CentOS/RHEL::
+
+		sudo apachectl start
+		sudo /etc/init.d/ceph-radosgw start
+
+Usage Logging
+=============
+
+:program:`radosgw` maintains an asynchronous usage log. It accumulates
+statistics about user operations and flushes it periodically. The
+logs can be accessed and managed through :program:`radosgw-admin`.
+
+The information that is being logged contains total data transfer,
+total operations, and total successful operations. The data is being
+accounted in an hourly resolution under the bucket owner, unless the
+operation was done on the service (e.g., when listing a bucket) in
+which case it is accounted under the operating user.
+
+Following is an example configuration::
+
+        [client.radosgw.gateway]
+            rgw enable usage log = true
+            rgw usage log tick interval = 30
+            rgw usage log flush threshold = 1024
+            rgw usage max shards = 32
+            rgw usage max user shards = 1
+
+
+The total number of shards determines how many total objects hold the
+usage log information. The per-user number of shards specify how many
+objects hold usage information for a single user. The tick interval
+configures the number of seconds between log flushes, and the flush
+threshold specify how many entries can be kept before resorting to
+synchronous flush.
+
+
+Availability
+============
+
+:program:`radosgw` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`radosgw-admin <radosgw-admin>`\(8)
diff --git a/doc/man/8/rbd-fuse.rst b/doc/man/8/rbd-fuse.rst
new file mode 100644
index 000000000..7337b9214
--- /dev/null
+++ b/doc/man/8/rbd-fuse.rst
@@ -0,0 +1,61 @@
+:orphan:
+
+=======================================
+ rbd-fuse -- expose rbd images as files
+=======================================
+
+.. program:: rbd-fuse
+
+Synopsis
+========
+
+| **rbd-fuse** [ -p pool ] [-c conffile] *mountpoint* [ *fuse options* ]
+
+
+Note
+====
+
+**rbd-fuse** is not recommended for any production or high performance workloads.
+
+Description
+===========
+
+**rbd-fuse** is a FUSE ("Filesystem in USErspace") client for RADOS
+block device (rbd) images.  Given a pool containing rbd images,
+it will mount a userspace file system allowing access to those images
+as regular files at **mountpoint**.
+
+The file system can be unmounted with::
+
+        fusermount -u mountpoint
+
+or by sending ``SIGINT`` to the ``rbd-fuse`` process.
+
+
+Options
+=======
+
+Any options not recognized by rbd-fuse will be passed on to libfuse.
+
+.. option:: -c ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -p pool
+
+   Use *pool* as the pool to search for rbd images.  Default is ``rbd``.
+
+
+Availability
+============
+
+**rbd-fuse** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+fusermount(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-ggate.rst b/doc/man/8/rbd-ggate.rst
new file mode 100644
index 000000000..0c395f962
--- /dev/null
+++ b/doc/man/8/rbd-ggate.rst
@@ -0,0 +1,79 @@
+:orphan:
+
+==================================================
+ rbd-ggate -- map rbd images via FreeBSD GEOM Gate
+==================================================
+
+.. program:: rbd-ggate
+
+Synopsis
+========
+
+| **rbd-ggate** [--read-only] [--exclusive] [--device *ggate device*] map *image-spec* | *snap-spec*
+| **rbd-ggate** unmap *ggate device*
+| **rbd-ggate** list
+
+Description
+===========
+
+**rbd-ggate** is a client for RADOS block device (rbd) images. It will
+map a rbd image to a ggate (FreeBSD GEOM Gate class) device, allowing
+access it as regular local block device.
+
+Commands
+========
+
+map
+---
+
+Spawn a process responsible for the creation of ggate device and
+forwarding I/O requests between the GEOM Gate kernel subsystem and
+RADOS.
+
+unmap
+-----
+
+Destroy ggate device and terminate the process responsible for it.
+
+list
+----
+
+List mapped ggate devices.
+
+Options
+=======
+
+.. option:: --device *ggate device*
+
+   Specify ggate device path.
+
+.. option:: --read-only
+
+   Map read-only.
+
+.. option:: --exclusive
+
+   Forbid writes by other clients.
+
+Image and snap specs
+====================
+
+| *image-spec* is [*pool-name*]/*image-name*
+| *snap-spec*  is [*pool-name*]/*image-name*\ @\ *snap-name*
+
+The default for *pool-name* is "rbd".  If an image name contains a slash
+character ('/'), *pool-name* is required.
+
+Availability
+============
+
+**rbd-ggate** is part of Ceph, a massively scalable, open-source,
+distributed storage system. Please refer to the Ceph documentation at
+https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/rbd-mirror.rst b/doc/man/8/rbd-mirror.rst
new file mode 100644
index 000000000..652f71d2a
--- /dev/null
+++ b/doc/man/8/rbd-mirror.rst
@@ -0,0 +1,75 @@
+:orphan:
+
+===================================================
+ rbd-mirror -- Ceph daemon for mirroring RBD images
+===================================================
+
+.. program:: rbd-mirror
+
+Synopsis
+========
+
+| **rbd-mirror**
+
+
+Description
+===========
+
+:program:`rbd-mirror` is a daemon for asynchronous mirroring of RADOS
+block device (rbd) images among Ceph clusters. It replays changes to
+images in remote clusters in a local cluster, for disaster recovery.
+
+It connects to remote clusters via the RADOS protocol, relying on
+default search paths to find ceph.conf files, monitor addresses and
+authentication information for them, i.e. ``/etc/ceph/$cluster.conf``,
+``/etc/ceph/$cluster.keyring``, and
+``/etc/ceph/$cluster.$name.keyring``, where ``$cluster`` is the
+human-friendly name of the cluster, and ``$name`` is the rados user to
+connect as, e.g. ``client.rbd-mirror``.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for rbd-mirror
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.rbd-mirror)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`rbd-mirror` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at https://docs.ceph.com for
+more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-nbd.rst b/doc/man/8/rbd-nbd.rst
new file mode 100644
index 000000000..5ec7ce40f
--- /dev/null
+++ b/doc/man/8/rbd-nbd.rst
@@ -0,0 +1,97 @@
+:orphan:
+
+=========================================
+ rbd-nbd -- map rbd images to nbd device
+=========================================
+
+.. program:: rbd-nbd
+
+Synopsis
+========
+
+| **rbd-nbd** [-c conf] [--read-only] [--device *nbd device*] [--snap-id *snap-id*] [--nbds_max *limit*] [--max_part *limit*] [--exclusive] [--notrim] [--encryption-format *format*] [--encryption-passphrase-file *passphrase-file*] [--io-timeout *seconds*] [--reattach-timeout *seconds*] map *image-spec* | *snap-spec*
+| **rbd-nbd** unmap *nbd device* | *image-spec* | *snap-spec*
+| **rbd-nbd** list-mapped
+| **rbd-nbd** attach --device *nbd device* *image-spec* | *snap-spec*
+| **rbd-nbd** detach *nbd device* | *image-spec* | *snap-spec*
+
+Description
+===========
+
+**rbd-nbd** is a client for RADOS block device (rbd) images like rbd kernel module.
+It will map a rbd image to a nbd (Network Block Device) device, allowing access it
+as regular local block device.
+
+Options
+=======
+
+.. option:: -c ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: --read-only
+
+   Map read-only.
+
+.. option:: --nbds_max *limit*
+
+   Override the parameter nbds_max of NBD kernel module when modprobe, used to
+   limit the count of nbd device.
+
+.. option:: --max_part *limit*
+
+    Override for module param max_part.
+
+.. option:: --exclusive
+
+   Forbid writes by other clients.
+
+.. option:: --notrim
+
+   Turn off trim/discard.
+
+.. option:: --encryption-format
+
+   Image encryption format.
+   Possible values: *luks*, *luks1*, *luks2*
+
+.. option:: --encryption-passphrase-file
+
+   Path of file containing a passphrase for unlocking image encryption.
+
+.. option:: --io-timeout *seconds*
+
+   Override device timeout. Linux kernel will default to a 30 second request timeout.
+   Allow the user to optionally specify an alternate timeout.
+
+.. option:: --reattach-timeout *seconds*
+
+   Specify timeout for the kernel to wait for a new rbd-nbd process is
+   attached after the old process is detached. The default is 30
+   second.
+
+.. option:: --snap-id *snapid*
+
+   Specify a snapshot to map/unmap/attach/detach by ID instead of by name.
+
+Image and snap specs
+====================
+
+| *image-spec* is [*pool-name*]/*image-name*
+| *snap-spec*  is [*pool-name*]/*image-name*\ @\ *snap-name*
+
+The default for *pool-name* is "rbd".  If an image name contains a slash
+character ('/'), *pool-name* is required.
+
+Availability
+============
+
+**rbd-nbd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay-many.rst b/doc/man/8/rbd-replay-many.rst
new file mode 100644
index 000000000..8241f37cb
--- /dev/null
+++ b/doc/man/8/rbd-replay-many.rst
@@ -0,0 +1,73 @@
+:orphan:
+
+==================================================================================
+ rbd-replay-many -- replay a rados block device (RBD) workload on several clients
+==================================================================================
+
+.. program:: rbd-replay-many
+
+Synopsis
+========
+
+| **rbd-replay-many** [ *options* ] --original-image *name* *host1* [ *host2* [ ... ] ] -- *rbd_replay_args*
+
+
+Description
+===========
+
+**rbd-replay-many** is a utility for replaying a rados block device (RBD) workload on several clients.
+Although all clients use the same workload, they replay against separate images.
+This matches normal use of librbd, where each original client is a VM with its own image.
+
+Configuration and replay files are not automatically copied to clients.
+Replay images must already exist.
+
+
+Options
+=======
+
+.. option:: --original-image name
+
+   Specifies the name (and snap) of the originally traced image.
+   Necessary for correct name mapping.
+
+.. option:: --image-prefix prefix
+
+   Prefix of image names to replay against.
+   Specifying --image-prefix=foo results in clients replaying against foo-0, foo-1, etc.
+   Defaults to the original image name.
+
+.. option:: --exec program
+
+   Path to the rbd-replay executable.
+
+.. option:: --delay seconds
+
+   Delay between starting each client.  Defaults to 0.
+
+
+Examples
+========
+
+Typical usage::
+
+       rbd-replay-many host-0 host-1 --original-image=image -- -c ceph.conf replay.bin
+
+This results in the following commands being executed::
+
+       ssh host-0 'rbd-replay' --map-image 'image=image-0' -c ceph.conf replay.bin
+       ssh host-1 'rbd-replay' --map-image 'image=image-1' -c ceph.conf replay.bin
+
+
+Availability
+============
+
+**rbd-replay-many** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay <rbd-replay>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay-prep.rst b/doc/man/8/rbd-replay-prep.rst
new file mode 100644
index 000000000..7bcbaceb3
--- /dev/null
+++ b/doc/man/8/rbd-replay-prep.rst
@@ -0,0 +1,55 @@
+:orphan:
+
+====================================================================================
+ rbd-replay-prep -- prepare captured rados block device (RBD) workloads for replay
+====================================================================================
+
+.. program:: rbd-replay-prep
+
+Synopsis
+========
+
+| **rbd-replay-prep** [ --window *seconds* ] [ --anonymize ] *trace_dir* *replay_file*
+
+
+Description
+===========
+
+**rbd-replay-prep** processes raw rados block device (RBD) traces to prepare them for **rbd-replay**.
+
+
+Options
+=======
+
+.. option:: --window seconds
+
+   Requests further apart than 'seconds' seconds are assumed to be independent.
+
+.. option:: --anonymize
+
+   Anonymizes image and snap names.
+
+.. option:: --verbose
+
+   Print all processed events to console
+
+Examples
+========
+
+To prepare workload1-trace for replay::
+
+       rbd-replay-prep workload1-trace/ust/uid/1000/64-bit workload1
+
+
+Availability
+============
+
+**rbd-replay-prep** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay <rbd-replay>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay.rst b/doc/man/8/rbd-replay.rst
new file mode 100644
index 000000000..aeb29562c
--- /dev/null
+++ b/doc/man/8/rbd-replay.rst
@@ -0,0 +1,78 @@
+:orphan:
+
+=========================================================
+ rbd-replay -- replay rados block device (RBD) workloads
+=========================================================
+
+.. program:: rbd-replay
+
+Synopsis
+========
+
+| **rbd-replay** [ *options* ] *replay_file*
+
+
+Description
+===========
+
+**rbd-replay** is a utility for replaying rados block device (RBD) workloads.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf ceph.conf
+
+   Use ceph.conf configuration file instead of the default /etc/ceph/ceph.conf to
+   determine monitor addresses during startup.
+
+.. option:: -p pool, --pool pool
+
+   Interact with the given pool.  Defaults to 'rbd'.
+
+.. option:: --latency-multiplier
+
+   Multiplies inter-request latencies.  Default: 1.
+
+.. option:: --read-only
+
+   Only replay non-destructive requests.
+
+.. option:: --map-image rule
+
+   Add a rule to map image names in the trace to image names in the replay cluster.
+   A rule of image1@snap1=image2@snap2 would map snap1 of image1 to snap2 of image2.
+
+.. option:: --dump-perf-counters
+
+   **Experimental**
+   Dump performance counters to standard out before an image is closed.
+   Performance counters may be dumped multiple times if multiple images are closed,
+   or if the same image is opened and closed multiple times.
+   Performance counters and their meaning may change between versions.
+
+
+Examples
+========
+
+To replay workload1 as fast as possible::
+
+       rbd-replay --latency-multiplier=0 workload1
+
+To replay workload1 but use test_image instead of prod_image::
+
+       rbd-replay --map-image=prod_image=test_image workload1
+
+
+Availability
+============
+
+**rbd-replay** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay-prep <rbd-replay-prep>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
new file mode 100644
index 000000000..59e346209
--- /dev/null
+++ b/doc/man/8/rbd.rst
@@ -0,0 +1,1040 @@
+:orphan:
+
+===============================================
+ rbd -- manage rados block device (RBD) images
+===============================================
+
+.. program:: rbd
+
+Synopsis
+========
+
+| **rbd** [ -c *ceph.conf* ] [ -m *monaddr* ] [--cluster *cluster-name*]
+  [ -p | --pool *pool* ] [ *command* ... ]
+
+
+Description
+===========
+
+**rbd** is a utility for manipulating rados block device (RBD) images,
+used by the Linux rbd driver and the rbd storage driver for QEMU/KVM.
+RBD images are simple block devices that are striped over objects and
+stored in a RADOS object store. The size of the objects the image is
+striped over must be a power of two.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf ceph.conf
+
+   Use ceph.conf configuration file instead of the default /etc/ceph/ceph.conf to
+   determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: --cluster cluster-name
+
+   Use different cluster name as compared to default cluster name *ceph*.
+
+.. option:: -p pool-name, --pool pool-name
+
+   Interact with the given pool. Required by most commands.
+
+.. option:: --namespace namespace-name
+
+   Use a pre-defined image namespace within a pool
+
+.. option:: --no-progress
+
+   Do not output progress information (goes to standard error by
+   default for some commands).
+
+
+Parameters
+==========
+
+.. option:: --image-format format-id
+
+   Specifies which object layout to use. The default is 2.
+
+   * format 1 - (deprecated) Use the original format for a new rbd image. This
+     format is understood by all versions of librbd and the kernel rbd module,
+     but does not support newer features like cloning.
+
+   * format 2 - Use the second rbd format, which is supported by librbd since
+     the Bobtail release and the kernel rbd module since kernel 3.10 (except
+     for "fancy" striping, which is supported since kernel 4.17). This adds
+     support for cloning and is more easily extensible to allow more
+     features in the future.
+
+.. option:: -s size-in-M/G/T, --size size-in-M/G/T
+
+   Specifies the size of the new rbd image or the new size of the existing rbd
+   image in M/G/T.  If no suffix is given, unit M is assumed.
+
+.. option:: --object-size size-in-B/K/M
+
+   Specifies the object size in B/K/M.  Object size will be rounded up the
+   nearest power of two; if no suffix is given, unit B is assumed.  The default
+   object size is 4M, smallest is 4K and maximum is 32M.
+
+   The default value can be changed with the configuration option ``rbd_default_order``,
+   which takes a power of two (default object size is ``2 ^ rbd_default_order``).
+
+.. option:: --stripe-unit size-in-B/K/M
+
+   Specifies the stripe unit size in B/K/M.  If no suffix is given, unit B is
+   assumed.  See striping section (below) for more details.
+
+.. option:: --stripe-count num
+
+   Specifies the number of objects to stripe over before looping back
+   to the first object.  See striping section (below) for more details.
+
+.. option:: --snap snap
+
+   Specifies the snapshot name for the specific operation.
+
+.. option:: --id username
+
+   Specifies the username (without the ``client.`` prefix) to use with the map command.
+
+.. option:: --keyring filename
+
+   Specifies a keyring file containing a secret for the specified user
+   to use with the map command.  If not specified, the default keyring
+   locations will be searched.
+
+.. option:: --keyfile filename
+
+   Specifies a file containing the secret key of ``--id user`` to use with the map command.
+   This option is overridden by ``--keyring`` if the latter is also specified.
+
+.. option:: --shared lock-tag
+
+   Option for `lock add` that allows multiple clients to lock the
+   same image if they use the same tag. The tag is an arbitrary
+   string. This is useful for situations where an image must
+   be open from more than one client at once, like during
+   live migration of a virtual machine, or for use underneath
+   a clustered file system.
+
+.. option:: --format format
+
+   Specifies output formatting (default: plain, json, xml)
+
+.. option:: --pretty-format
+
+   Make json or xml formatted output more human-readable.
+
+.. option:: -o krbd-options, --options krbd-options
+
+   Specifies which options to use when mapping or unmapping an image via the
+   rbd kernel driver.  krbd-options is a comma-separated list of options
+   (similar to mount(8) mount options).  See kernel rbd (krbd) options section
+   below for more details.
+
+.. option:: --read-only
+
+   Map the image read-only.  Equivalent to -o ro.
+
+.. option:: --image-feature feature-name
+
+   Specifies which RBD format 2 feature should be enabled when creating
+   an image. Multiple features can be enabled by repeating this option
+   multiple times. The following features are supported:
+
+   * layering: layering support
+   * striping: striping v2 support
+   * exclusive-lock: exclusive locking support
+   * object-map: object map support (requires exclusive-lock)
+   * fast-diff: fast diff calculations (requires object-map)
+   * deep-flatten: snapshot flatten support
+   * journaling: journaled IO support (requires exclusive-lock)
+   * data-pool: erasure coded pool support
+
+.. option:: --image-shared
+
+   Specifies that the image will be used concurrently by multiple clients.
+   This will disable features that are dependent upon exclusive ownership
+   of the image.
+
+.. option:: --whole-object
+
+   Specifies that the diff should be limited to the extents of a full object
+   instead of showing intra-object deltas. When the object map feature is
+   enabled on an image, limiting the diff to the object extents will
+   dramatically improve performance since the differences can be computed
+   by examining the in-memory object map instead of querying RADOS for each
+   object within the image.
+
+.. option:: --limit
+
+   Specifies the limit for the number of snapshots permitted.
+
+Commands
+========
+
+.. TODO rst "option" directive seems to require --foo style options, parsing breaks on subcommands.. the args show up as bold too
+
+:command:`bench` --io-type <read | write | readwrite | rw> [--io-size *size-in-B/K/M/G/T*] [--io-threads *num-ios-in-flight*] [--io-total *size-in-B/K/M/G/T*] [--io-pattern seq | rand] [--rw-mix-read *read proportion in readwrite*] *image-spec*
+  Generate a series of IOs to the image and measure the IO throughput and
+  latency.  If no suffix is given, unit B is assumed for both --io-size and
+  --io-total.  Defaults are: --io-size 4096, --io-threads 16, --io-total 1G,
+  --io-pattern seq, --rw-mix-read 50.
+
+:command:`children` *snap-spec*
+  List the clones of the image at the given snapshot. This checks
+  every pool, and outputs the resulting poolname/imagename.
+
+  This requires image format 2.
+
+:command:`clone` [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--image-shared] *parent-snap-spec* *child-image-spec*
+  Will create a clone (copy-on-write child) of the parent snapshot.
+  Object size will be identical to that of the parent image unless
+  specified. Size will be the same as the parent snapshot. The --stripe-unit
+  and --stripe-count arguments are optional, but must be used together.
+
+  The parent snapshot must be protected (see `rbd snap protect`).
+  This requires image format 2.
+
+:command:`config global get` *config-entity* *key*
+  Get a global-level configuration override.
+
+:command:`config global list` [--format plain | json | xml] [--pretty-format] *config-entity*
+  List global-level configuration overrides.
+
+:command:`config global set` *config-entity* *key* *value*
+  Set a global-level configuration override.
+
+:command:`config global remove` *config-entity* *key*
+  Remove a global-level configuration override.
+
+:command:`config image get` *image-spec* *key*
+  Get an image-level configuration override.
+
+:command:`config image list` [--format plain | json | xml] [--pretty-format] *image-spec*
+  List image-level configuration overrides.
+
+:command:`config image set` *image-spec* *key* *value*
+  Set an image-level configuration override.
+
+:command:`config image remove` *image-spec* *key*
+  Remove an image-level configuration override.
+
+:command:`config pool get` *pool-name* *key*
+  Get a pool-level configuration override.
+
+:command:`config pool list` [--format plain | json | xml] [--pretty-format] *pool-name*
+  List pool-level configuration overrides.
+
+:command:`config pool set` *pool-name* *key* *value*
+  Set a pool-level configuration override.
+
+:command:`config pool remove` *pool-name* *key*
+  Remove a pool-level configuration override.
+
+:command:`cp` (*src-image-spec* | *src-snap-spec*) *dest-image-spec*
+  Copy the content of a src-image into the newly created dest-image.
+  dest-image will have the same size, object size, and image format as src-image.
+  Note: snapshots are not copied, use `deep cp` command to include
+  snapshots.
+
+:command:`create` (-s | --size *size-in-M/G/T*) [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--thick-provision] [--no-progress] [--image-feature *feature-name*]... [--image-shared] *image-spec*
+  Will create a new rbd image. You must also specify the size via --size.  The
+  --stripe-unit and --stripe-count arguments are optional, but must be used together.
+  If the --thick-provision is enabled, it will fully allocate storage for
+  the image at creation time. It will take a long time to do.
+  Note: thick provisioning requires zeroing the contents of the entire image.
+
+:command:`deep cp` (*src-image-spec* | *src-snap-spec*) *dest-image-spec*
+  Deep copy the content of a src-image into the newly created dest-image.
+  Dest-image will have the same size, object size, image format, and snapshots as src-image.
+
+:command:`device list` [-t | --device-type *device-type*] [--format plain | json | xml] --pretty-format
+  Show the rbd images that are mapped via the rbd kernel module
+  (default) or other supported device.
+
+:command:`device map` [-t | --device-type *device-type*] [--cookie *device-cookie*] [--show-cookie] [--snap-id *snap-id*] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec*
+  Map the specified image to a block device via the rbd kernel module
+  (default) or other supported device (*nbd* on Linux or *ggate* on
+  FreeBSD).
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device unmap` [-t | --device-type *device-type*] [-o | --options *device-options*] [--snap-id *snap-id*] *image-spec* | *snap-spec* | *device-path*
+  Unmap the block device that was mapped via the rbd kernel module
+  (default) or other supported device.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--cookie *device-cookie*] [--show-cookie] [--snap-id *snap-id*] [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec*
+  Attach the specified image to the specified block device (currently only
+  `nbd` on Linux). This operation is unsafe and should not be normally used.
+  In particular, specifying the wrong image or the wrong block device may
+  lead to data corruption as no validation is performed by `nbd` kernel driver.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device detach` [-t | --device-type *device-type*] [-o | --options *device-options*] [--snap-id *snap-id*] *image-spec* | *snap-spec* | *device-path*
+  Detach the block device that was mapped or attached (currently only `nbd`
+  on Linux). This operation is unsafe and should not be normally used.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`diff` [--from-snap *snap-name*] [--whole-object] *image-spec* | *snap-spec*
+  Dump a list of byte extents in the image that have changed since the specified start
+  snapshot, or since the image was created.  Each output line includes the starting offset
+  (in bytes), the length of the region (in bytes), and either 'zero' or 'data' to indicate
+  whether the region is known to be zeros or may contain other data.
+
+:command:`du` [-p | --pool *pool-name*] [*image-spec* | *snap-spec*] [--merge-snapshots]
+  Will calculate the provisioned and actual disk usage of all images and
+  associated snapshots within the specified pool.  It can also be used against
+  individual images and snapshots.
+
+  If the RBD fast-diff feature is not enabled on images, this operation will
+  require querying the OSDs for every potential object within the image.
+
+  The --merge-snapshots will merge snapshots used space into their parent images.
+
+:command:`encryption format` *image-spec* *format* *passphrase-file* [--cipher-alg *alg*]
+  Formats image to an encrypted format.
+  All data previously written to the image will become unreadable.
+  Supported formats: *luks1*, *luks2*.
+  Supported cipher algorithms: *aes-128*, *aes-256* (default).
+
+:command:`export` [--export-format *format (1 or 2)*] (*image-spec* | *snap-spec*) [*dest-path*]
+  Export image to dest path (use - for stdout).
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to export not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
+
+:command:`export-diff` [--from-snap *snap-name*] [--whole-object] (*image-spec* | *snap-spec*) *dest-path*
+  Export an incremental diff for an image to dest path (use - for stdout).  If
+  an initial snapshot is specified, only changes since that snapshot are included; otherwise,
+  any regions of the image that contain data are included.  The end snapshot is specified
+  using the standard --snap option or @snap syntax (see below).  The image diff format includes
+  metadata about image size changes, and the start and end snapshots.  It efficiently represents
+  discarded or 'zero' regions of the image.
+
+:command:`feature disable` *image-spec* *feature-name*...
+  Disable the specified feature on the specified image. Multiple features can
+  be specified.
+
+:command:`feature enable` *image-spec* *feature-name*...
+  Enable the specified feature on the specified image. Multiple features can
+  be specified.
+
+:command:`flatten` [--encryption-format *encryption-format* --encryption-passphrase-file *passphrase-file*]... *image-spec*
+  If image is a clone, copy all shared blocks from the parent snapshot and
+  make the child independent of the parent, severing the link between
+  parent snap and child.  The parent snapshot can be unprotected and
+  deleted if it has no further dependent clones.
+
+  This requires image format 2.
+
+:command:`group create` *group-spec*
+  Create a group.
+
+:command:`group image add` *group-spec* *image-spec*
+  Add an image to a group.
+
+:command:`group image list` *group-spec*
+  List images in a group.
+
+:command:`group image remove` *group-spec* *image-spec*
+  Remove an image from a group.
+
+:command:`group ls` [-p | --pool *pool-name*]
+  List rbd groups.
+
+:command:`group rename` *src-group-spec* *dest-group-spec*
+  Rename a group.  Note: rename across pools is not supported.
+
+:command:`group rm` *group-spec*
+  Delete a group.
+
+:command:`group snap create` *group-snap-spec*
+  Make a snapshot of a group.
+
+:command:`group snap list` *group-spec*
+  List snapshots of a group.
+
+:command:`group snap rm` *group-snap-spec*
+  Remove a snapshot from a group.
+
+:command:`group snap rename` *group-snap-spec* *snap-name*
+  Rename group's snapshot.
+
+:command:`group snap rollback` *group-snap-spec*
+  Rollback group to snapshot.
+
+:command:`image-meta get` *image-spec* *key*
+  Get metadata value with the key.
+
+:command:`image-meta list` *image-spec*
+  Show metadata held on the image. The first column is the key
+  and the second column is the value.
+
+:command:`image-meta remove` *image-spec* *key*
+  Remove metadata key with the value.
+
+:command:`image-meta set` *image-spec* *key* *value*
+  Set metadata key with the value. They will displayed in `image-meta list`.
+
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+  Create a new image and imports its data from path (use - for
+  stdin).  The import operation will try to create sparse rbd images 
+  if possible.  For import from stdin, the sparsification unit is
+  the data block size of the destination image (object size).
+
+  The --stripe-unit and --stripe-count arguments are optional, but must be
+  used together.
+
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to import not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
+
+:command:`import-diff` *src-path* *image-spec*
+  Import an incremental diff of an image and applies it to the current image.  If the diff
+  was generated relative to a start snapshot, we verify that snapshot already exists before
+  continuing.  If there was an end snapshot we verify it does not already exist before
+  applying the changes, and create the snapshot when we are done.
+  
+:command:`info` *image-spec* | *snap-spec*
+  Will dump information (such as size and object size) about a specific rbd image.
+  If image is a clone, information about its parent is also displayed.
+  If a snapshot is specified, whether it is protected is shown as well.
+
+:command:`journal client disconnect` *journal-spec*
+  Flag image journal client as disconnected.
+
+:command:`journal export` [--verbose] [--no-error] *src-journal-spec* *path-name*
+  Export image journal to path (use - for stdout). It can be make a backup
+  of the image journal especially before attempting dangerous operations.
+
+  Note that this command may not always work if the journal is badly corrupted.
+
+:command:`journal import` [--verbose] [--no-error] *path-name* *dest-journal-spec*
+  Import image journal from path (use - for stdin).
+
+:command:`journal info` *journal-spec*
+  Show information about image journal.
+
+:command:`journal inspect` [--verbose] *journal-spec*
+  Inspect and report image journal for structural errors.
+
+:command:`journal reset` *journal-spec*
+  Reset image journal.
+
+:command:`journal status` *journal-spec*
+  Show status of image journal.
+
+:command:`lock add` [--shared *lock-tag*] *image-spec* *lock-id*
+  Lock an image. The lock-id is an arbitrary name for the user's
+  convenience. By default, this is an exclusive lock, meaning it
+  will fail if the image is already locked. The --shared option
+  changes this behavior. Note that locking does not affect
+  any operation other than adding a lock. It does not
+  protect an image from being deleted.
+
+:command:`lock ls` *image-spec*
+  Show locks held on the image. The first column is the locker
+  to use with the `lock remove` command.
+
+:command:`lock rm` *image-spec* *lock-id* *locker*
+  Release a lock on an image. The lock id and locker are
+  as output by lock ls.
+
+:command:`ls` [-l | --long] [*pool-name*]
+  Will list all rbd images listed in the rbd_directory object.  With
+  -l, also show snapshots, and use longer-format output including
+  size, parent (if clone), format, etc.
+
+:command:`merge-diff` *first-diff-path* *second-diff-path* *merged-diff-path*
+  Merge two continuous incremental diffs of an image into one single diff. The
+  first diff's end snapshot must be equal with the second diff's start snapshot.
+  The first diff could be - for stdin, and merged diff could be - for stdout, which
+  enables multiple diff files to be merged using something like
+  'rbd merge-diff first second - | rbd merge-diff - third result'. Note this command
+  currently only support the source incremental diff with stripe_count == 1
+
+:command:`migration abort` *image-spec*
+  Cancel image migration. This step may be run after successful or
+  failed migration prepare or migration execute steps and returns the
+  image to its initial (before migration) state. All modifications to
+  the destination image are lost.
+
+:command:`migration commit` *image-spec*
+  Commit image migration. This step is run after a successful migration
+  prepare and migration execute steps and removes the source image data.
+
+:command:`migration execute` *image-spec*
+  Execute image migration. This step is run after a successful migration
+  prepare step and copies image data to the destination.
+
+:command:`migration prepare` [--order *order*] [--object-size *object-size*] [--image-feature *image-feature*] [--image-shared] [--stripe-unit *stripe-unit*] [--stripe-count *stripe-count*] [--data-pool *data-pool*] [--import-only] [--source-spec *json*] [--source-spec-path *path*] *src-image-spec* [*dest-image-spec*]
+  Prepare image migration. This is the first step when migrating an
+  image, i.e. changing the image location, format or other
+  parameters that can't be changed dynamically. The destination can
+  match the source, and in this case *dest-image-spec* can be omitted.
+  After this step the source image is set as a parent of the
+  destination image, and the image is accessible in copy-on-write mode
+  by its destination spec.
+
+  An image can also be migrated from a read-only import source by adding the
+  *--import-only* optional and providing a JSON-encoded *--source-spec* or a
+  path to a JSON-encoded source-spec file using the *--source-spec-path*
+  optionals.
+
+:command:`mirror image demote` *image-spec*
+  Demote a primary image to non-primary for RBD mirroring.
+
+:command:`mirror image disable` [--force] *image-spec*
+  Disable RBD mirroring for an image. If the mirroring is
+  configured in ``image`` mode for the image's pool, then it
+  can be explicitly disabled mirroring for each image within
+  the pool.
+
+:command:`mirror image enable` *image-spec* *mode*
+  Enable RBD mirroring for an image. If the mirroring is
+  configured in ``image`` mode for the image's pool, then it
+  can be explicitly enabled mirroring for each image within
+  the pool.
+
+  The mirror image mode can either be ``journal`` (default) or
+  ``snapshot``. The ``journal`` mode requires the RBD journaling
+  feature.
+
+:command:`mirror image promote` [--force] *image-spec*
+  Promote a non-primary image to primary for RBD mirroring.
+
+:command:`mirror image resync` *image-spec*
+  Force resync to primary image for RBD mirroring.
+
+:command:`mirror image status` *image-spec*
+  Show RBD mirroring status for an image.
+
+:command:`mirror pool demote` [*pool-name*]
+  Demote all primary images within a pool to non-primary.
+  Every mirroring enabled image will demoted in the pool.
+
+:command:`mirror pool disable` [*pool-name*]
+  Disable RBD mirroring by default within a pool. When mirroring
+  is disabled on a pool in this way, mirroring will also be
+  disabled on any images (within the pool) for which mirroring
+  was enabled explicitly.
+
+:command:`mirror pool enable` [*pool-name*] *mode*
+  Enable RBD mirroring by default within a pool.
+  The mirroring mode can either be ``pool`` or ``image``.
+  If configured in ``pool`` mode, all images in the pool
+  with the journaling feature enabled are mirrored.
+  If configured in ``image`` mode, mirroring needs to be
+  explicitly enabled (by ``mirror image enable`` command)
+  on each image.
+
+:command:`mirror pool info` [*pool-name*]
+  Show information about the pool mirroring configuration.
+  It includes mirroring mode, peer UUID, remote cluster name,
+  and remote client name.
+
+:command:`mirror pool peer add` [*pool-name*] *remote-cluster-spec*
+  Add a mirroring peer to a pool.
+  *remote-cluster-spec* is [*remote client name*\ @\ ]\ *remote cluster name*.
+
+  The default for *remote client name* is "client.admin".
+
+  This requires mirroring mode is enabled.
+
+:command:`mirror pool peer remove` [*pool-name*] *uuid*
+  Remove a mirroring peer from a pool. The peer uuid is available
+  from ``mirror pool info`` command.
+
+:command:`mirror pool peer set` [*pool-name*] *uuid* *key* *value*
+  Update mirroring peer settings.
+  The key can be either ``client`` or ``cluster``, and the value
+  is corresponding to remote client name or remote cluster name.
+
+:command:`mirror pool promote` [--force] [*pool-name*]
+  Promote all non-primary images within a pool to primary.
+  Every mirroring enabled image will promoted in the pool.
+
+:command:`mirror pool status` [--verbose] [*pool-name*]
+  Show status for all mirrored images in the pool.
+  With --verbose, also show additionally output status
+  details for every mirroring image in the pool.
+
+:command:`mirror snapshot schedule add` [-p | --pool *pool*] [--namespace *namespace*] [--image *image*] *interval* [*start-time*]
+  Add mirror snapshot schedule.
+
+:command:`mirror snapshot schedule list` [-R | --recursive] [--format *format*] [--pretty-format] [-p | --pool *pool*] [--namespace *namespace*] [--image *image*]
+  List mirror snapshot schedule.
+
+:command:`mirror snapshot schedule remove` [-p | --pool *pool*] [--namespace *namespace*] [--image *image*] *interval* [*start-time*]
+  Remove mirror snapshot schedule.
+
+:command:`mirror snapshot schedule status` [-p | --pool *pool*] [--format *format*] [--pretty-format] [--namespace *namespace*] [--image *image*]
+  Show mirror snapshot schedule status.
+
+:command:`mv` *src-image-spec* *dest-image-spec*
+  Rename an image.  Note: rename across pools is not supported.
+
+:command:`namespace create` *pool-name*/*namespace-name*
+  Create a new image namespace within the pool.
+
+:command:`namespace list` *pool-name*
+  List image namespaces defined within the pool.
+
+:command:`namespace remove` *pool-name*/*namespace-name*
+  Remove an empty image namespace from the pool.
+
+:command:`object-map check` *image-spec* | *snap-spec*
+  Verify the object map is correct.
+
+:command:`object-map rebuild` *image-spec* | *snap-spec*
+  Rebuild an invalid object map for the specified image. An image snapshot can be
+  specified to rebuild an invalid object map for a snapshot.
+
+:command:`pool init` [*pool-name*] [--force]
+  Initialize pool for use by RBD. Newly created pools must initialized
+  prior to use.
+
+:command:`resize` (-s | --size *size-in-M/G/T*) [--allow-shrink] [--encryption-format *encryption-format* --encryption-passphrase-file *passphrase-file*]... *image-spec*
+  Resize rbd image. The size parameter also needs to be specified.
+  The --allow-shrink option lets the size be reduced.
+  
+:command:`rm` *image-spec*
+  Delete an rbd image (including all data blocks). If the image has
+  snapshots, this fails and nothing is deleted.
+
+:command:`snap create` *snap-spec*
+  Create a new snapshot. Requires the snapshot name parameter specified.
+
+:command:`snap limit clear` *image-spec*
+  Remove any previously set limit on the number of snapshots allowed on
+  an image.
+
+:command:`snap limit set` [--limit] *limit* *image-spec*
+  Set a limit for the number of snapshots allowed on an image.
+
+:command:`snap ls` *image-spec*
+  Dump the list of snapshots inside a specific image.
+
+:command:`snap protect` *snap-spec*
+  Protect a snapshot from deletion, so that clones can be made of it
+  (see `rbd clone`).  Snapshots must be protected before clones are made;
+  protection implies that there exist dependent cloned children that
+  refer to this snapshot.  `rbd clone` will fail on a nonprotected
+  snapshot.
+
+  This requires image format 2.
+
+:command:`snap purge` *image-spec*
+  Remove all unprotected snapshots from an image.
+
+:command:`snap rename` *src-snap-spec* *dest-snap-spec*
+  Rename a snapshot. Note: rename across pools and images is not supported.
+
+:command:`snap rm` [--force] *snap-spec*
+  Remove the specified snapshot.
+
+:command:`snap rollback` *snap-spec*
+  Rollback image content to snapshot. This will iterate through the entire blocks
+  array and update the data head content to the snapshotted version.
+
+:command:`snap unprotect` *snap-spec*
+  Unprotect a snapshot from deletion (undo `snap protect`).  If cloned
+  children remain, `snap unprotect` fails.  (Note that clones may exist
+  in different pools than the parent snapshot.)
+
+  This requires image format 2.
+
+:command:`sparsify` [--sparse-size *sparse-size*] *image-spec*
+  Reclaim space for zeroed image extents. The default sparse size is
+  4096 bytes and can be changed via --sparse-size option with the
+  following restrictions: it should be power of two, not less than
+  4096, and not larger than image object size.
+
+:command:`status` *image-spec*
+  Show the status of the image, including which clients have it open.
+
+:command:`trash ls` [*pool-name*]
+  List all entries from trash.
+
+:command:`trash mv` *image-spec*
+  Move an image to the trash. Images, even ones actively in-use by 
+  clones, can be moved to the trash and deleted at a later time.
+
+:command:`trash purge` [*pool-name*]
+  Remove all expired images from trash.
+
+:command:`trash restore` *image-id*  
+  Restore an image from trash.
+
+:command:`trash rm` *image-id* 
+  Delete an image from trash. If image deferment time has not expired
+  you can not removed it unless use force. But an actively in-use by clones 
+  or has snapshots can not be removed.
+
+:command:`trash purge schedule add` [-p | --pool *pool*] [--namespace *namespace*] *interval* [*start-time*]
+  Add trash purge schedule.
+
+:command:`trash purge schedule list` [-R | --recursive] [--format *format*] [--pretty-format] [-p | --pool *pool*] [--namespace *namespace*]
+  List trash purge schedule.
+
+:command:`trash purge schedule remove` [-p | --pool *pool*] [--namespace *namespace*] *interval* [*start-time*]
+  Remove trash purge schedule.
+
+:command:`trash purge schedule status` [-p | --pool *pool*] [--format *format*] [--pretty-format] [--namespace *namespace*]
+  Show trash purge schedule status.
+
+:command:`watch` *image-spec*
+  Watch events on image.
+
+Image, snap, group and journal specs
+====================================
+
+| *image-spec*      is [*pool-name*/[*namespace-name*/]]\ *image-name*
+| *snap-spec*       is [*pool-name*/[*namespace-name*/]]\ *image-name*\ @\ *snap-name*
+| *group-spec*      is [*pool-name*/[*namespace-name*/]]\ *group-name*
+| *group-snap-spec* is [*pool-name*/[*namespace-name*/]]\ *group-name*\ @\ *snap-name*
+| *journal-spec*    is [*pool-name*/[*namespace-name*/]]\ *journal-name*
+
+The default for *pool-name* is "rbd" and *namespace-name* is "". If an image
+name contains a slash character ('/'), *pool-name* is required.
+
+The *journal-name* is *image-id*.
+
+You may specify each name individually, using --pool, --namespace, --image, and
+--snap options, but this is discouraged in favor of the above spec syntax.
+
+Striping
+========
+
+RBD images are striped over many objects, which are then stored by the
+Ceph distributed object store (RADOS).  As a result, read and write
+requests for the image are distributed across many nodes in the
+cluster, generally preventing any single node from becoming a
+bottleneck when individual images get large or busy.
+
+The striping is controlled by three parameters:
+
+.. option:: object-size
+
+  The size of objects we stripe over is a power of two. It will be rounded up the nearest power of two.
+  The default object size is 4 MB, smallest is 4K and maximum is 32M.
+
+.. option:: stripe_unit
+
+  Each [*stripe_unit*] contiguous bytes are stored adjacently in the same object, before we move on
+  to the next object.
+
+.. option:: stripe_count
+
+  After we write [*stripe_unit*] bytes to [*stripe_count*] objects, we loop back to the initial object
+  and write another stripe, until the object reaches its maximum size.  At that point,
+  we move on to the next [*stripe_count*] objects.
+
+By default, [*stripe_unit*] is the same as the object size and [*stripe_count*] is 1.  Specifying a different
+[*stripe_unit*] and/or [*stripe_count*] is often referred to as using "fancy" striping and requires format 2.
+
+
+Kernel rbd (krbd) options
+=========================
+
+Most of these options are useful mainly for debugging and benchmarking.  The
+default values are set in the kernel and may therefore depend on the version of
+the running kernel.
+
+Per client instance `rbd device map` options:
+
+* fsid=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee - FSID that should be assumed by
+  the client.
+
+* ip=a.b.c.d[:p] - IP and, optionally, port the client should use.
+
+* share - Enable sharing of client instances with other mappings (default).
+
+* noshare - Disable sharing of client instances with other mappings.
+
+* crc - Enable CRC32C checksumming for msgr1 on-the-wire protocol (default).
+  For msgr2.1 protocol this option is ignored: full checksumming is always on
+  in 'crc' mode and always off in 'secure' mode.
+
+* nocrc - Disable CRC32C checksumming for msgr1 on-the-wire protocol.  Note
+  that only payload checksumming is disabled, header checksumming is always on.
+  For msgr2.1 protocol this option is ignored.
+
+* cephx_require_signatures - Require msgr1 message signing feature (since 3.19,
+  default).  This option is deprecated and will be removed in the future as the
+  feature has been supported since the Bobtail release.
+
+* nocephx_require_signatures - Don't require msgr1 message signing feature
+  (since 3.19).  This option is deprecated and will be removed in the future.
+
+* tcp_nodelay - Disable Nagle's algorithm on client sockets (since 4.0,
+  default).
+
+* notcp_nodelay - Enable Nagle's algorithm on client sockets (since 4.0).
+
+* cephx_sign_messages - Enable message signing for msgr1 on-the-wire protocol
+  (since 4.4, default).  For msgr2.1 protocol this option is ignored: message
+  signing is built into 'secure' mode and not offered in 'crc' mode.
+
+* nocephx_sign_messages - Disable message signing for msgr1 on-the-wire protocol
+  (since 4.4).  For msgr2.1 protocol this option is ignored.
+
+* mount_timeout=x - A timeout on various steps in `rbd device map` and
+  `rbd device unmap` sequences (default is 60 seconds).  In particular,
+  since 4.2 this can be used to ensure that `rbd device unmap` eventually
+  times out when there is no network connection to a cluster.
+
+* osdkeepalive=x - OSD keepalive timeout (default is 5 seconds).
+
+* osd_idle_ttl=x - OSD idle TTL (default is 60 seconds).
+
+Per mapping (block device) `rbd device map` options:
+
+* rw - Map the image read-write (default).  Overridden by --read-only.
+
+* ro - Map the image read-only.  Equivalent to --read-only.
+
+* queue_depth=x - queue depth (since 4.2, default is 128 requests).
+
+* lock_on_read - Acquire exclusive lock on reads, in addition to writes and
+  discards (since 4.9).
+
+* exclusive - Disable automatic exclusive lock transitions (since 4.12).
+  Equivalent to --exclusive.
+
+* lock_timeout=x - A timeout on waiting for the acquisition of exclusive lock
+  (since 4.17, default is 0 seconds, meaning no timeout).
+
+* notrim - Turn off discard and write zeroes offload support to avoid
+  deprovisioning a fully provisioned image (since 4.17). When enabled, discard
+  requests will fail with -EOPNOTSUPP, write zeroes requests will fall back to
+  manually zeroing.
+
+* abort_on_full - Fail write requests with -ENOSPC when the cluster is full or
+  the data pool reaches its quota (since 5.0).  The default behaviour is to
+  block until the full condition is cleared.
+
+* alloc_size - Minimum allocation unit of the underlying OSD object store
+  backend (since 5.1, default is 64K bytes).  This is used to round off and
+  drop discards that are too small.  For bluestore, the recommended setting is
+  bluestore_min_alloc_size (currently set to 4K for all types of drives,
+  previously used to be set to 64K for hard disk drives and 16K for
+  solid-state drives).  For filestore with filestore_punch_hole = false, the
+  recommended setting is image object size (typically 4M).
+
+* crush_location=x - Specify the location of the client in terms of CRUSH
+  hierarchy (since 5.8).  This is a set of key-value pairs separated from
+  each other by '|', with keys separated from values by ':'.  Note that '|'
+  may need to be quoted or escaped to avoid it being interpreted as a pipe
+  by the shell.  The key is the bucket type name (e.g. rack, datacenter or
+  region with default bucket types) and the value is the bucket name.  For
+  example, to indicate that the client is local to rack "myrack", data center
+  "mydc" and region "myregion"::
+
+    crush_location=rack:myrack|datacenter:mydc|region:myregion
+
+  Each key-value pair stands on its own: "myrack" doesn't need to reside in
+  "mydc", which in turn doesn't need to reside in "myregion".  The location
+  is not a path to the root of the hierarchy but rather a set of nodes that
+  are matched independently, owning to the fact that bucket names are unique
+  within a CRUSH map.  "Multipath" locations are supported, so it is possible
+  to indicate locality for multiple parallel hierarchies::
+
+    crush_location=rack:myrack1|rack:myrack2|datacenter:mydc
+
+* read_from_replica=no - Disable replica reads, always pick the primary OSD
+  (since 5.8, default).
+
+* read_from_replica=balance - When issued a read on a replicated pool, pick
+  a random OSD for serving it (since 5.8).
+
+  This mode is safe for general use only since Octopus (i.e. after "ceph osd
+  require-osd-release octopus").  Otherwise it should be limited to read-only
+  workloads such as images mapped read-only everywhere or snapshots.
+
+* read_from_replica=localize - When issued a read on a replicated pool, pick
+  the most local OSD for serving it (since 5.8).  The locality metric is
+  calculated against the location of the client given with crush_location;
+  a match with the lowest-valued bucket type wins.  For example, with default
+  bucket types, an OSD in a matching rack is closer than an OSD in a matching
+  data center, which in turn is closer than an OSD in a matching region.
+
+  This mode is safe for general use only since Octopus (i.e. after "ceph osd
+  require-osd-release octopus").  Otherwise it should be limited to read-only
+  workloads such as images mapped read-only everywhere or snapshots.
+
+* compression_hint=none - Don't set compression hints (since 5.8, default).
+
+* compression_hint=compressible - Hint to the underlying OSD object store
+  backend that the data is compressible, enabling compression in passive mode
+  (since 5.8).
+
+* compression_hint=incompressible - Hint to the underlying OSD object store
+  backend that the data is incompressible, disabling compression in aggressive
+  mode (since 5.8).
+
+* ms_mode=legacy - Use msgr1 on-the-wire protocol (since 5.11, default).
+
+* ms_mode=crc - Use msgr2.1 on-the-wire protocol, select 'crc' mode, also
+  referred to as plain mode (since 5.11).  If the daemon denies 'crc' mode,
+  fail the connection.
+
+* ms_mode=secure - Use msgr2.1 on-the-wire protocol, select 'secure' mode
+  (since 5.11).  'secure' mode provides full in-transit encryption ensuring
+  both confidentiality and authenticity.  If the daemon denies 'secure' mode,
+  fail the connection.
+
+* ms_mode=prefer-crc - Use msgr2.1 on-the-wire protocol, select 'crc'
+  mode (since 5.11).  If the daemon denies 'crc' mode in favor of 'secure'
+  mode, agree to 'secure' mode.
+
+* ms_mode=prefer-secure - Use msgr2.1 on-the-wire protocol, select 'secure'
+  mode (since 5.11).  If the daemon denies 'secure' mode in favor of 'crc'
+  mode, agree to 'crc' mode.
+
+* rxbounce - Use a bounce buffer when receiving data (since 5.17).  The default
+  behaviour is to read directly into the destination buffer.  A bounce buffer
+  is needed if the destination buffer isn't guaranteed to be stable (i.e. remain
+  unchanged while it is being read to).  In particular this is the case for
+  Windows where a system-wide "dummy" (throwaway) page may be mapped into the
+  destination buffer in order to generate a single large I/O.  Otherwise,
+  "libceph: ... bad crc/signature" or "libceph: ... integrity error, bad crc"
+  errors and associated performance degradation are expected.
+
+* udev - Wait for udev device manager to finish executing all matching
+  "add" rules and release the device before exiting (default).  This option
+  is not passed to the kernel.
+
+* noudev - Don't wait for udev device manager.  When enabled, the device may
+  not be fully usable immediately on exit.
+
+`rbd device unmap` options:
+
+* force - Force the unmapping of a block device that is open (since 4.9).  The
+  driver will wait for running requests to complete and then unmap; requests
+  sent to the driver after initiating the unmap will be failed.
+
+* udev - Wait for udev device manager to finish executing all matching
+  "remove" rules and clean up after the device before exiting (default).
+  This option is not passed to the kernel.
+
+* noudev - Don't wait for udev device manager.
+
+
+Examples
+========
+
+To create a new rbd image that is 100 GB::
+
+       rbd create mypool/myimage --size 102400
+
+To use a non-default object size (8 MB)::
+
+       rbd create mypool/myimage --size 102400 --object-size 8M
+
+To delete an rbd image (be careful!)::
+
+       rbd rm mypool/myimage
+
+To create a new snapshot::
+
+       rbd snap create mypool/myimage@mysnap
+
+To create a copy-on-write clone of a protected snapshot::
+
+       rbd clone mypool/myimage@mysnap otherpool/cloneimage
+
+To see which clones of a snapshot exist::
+
+       rbd children mypool/myimage@mysnap
+
+To delete a snapshot::
+
+       rbd snap rm mypool/myimage@mysnap
+
+To map an image via the kernel with cephx enabled::
+
+       rbd device map mypool/myimage --id admin --keyfile secretfile
+
+To map an image via the kernel with different cluster name other than default *ceph*::
+
+       rbd device map mypool/myimage --cluster cluster-name
+
+To unmap an image::
+
+       rbd device unmap /dev/rbd0
+
+To create an image and a clone from it::
+
+       rbd import --image-format 2 image mypool/parent
+       rbd snap create mypool/parent@snap
+       rbd snap protect mypool/parent@snap
+       rbd clone mypool/parent@snap otherpool/child
+
+To create an image with a smaller stripe_unit (to better distribute small writes in some workloads)::
+
+       rbd create mypool/myimage --size 102400 --stripe-unit 65536B --stripe-count 16
+
+To change an image from one image format to another, export it and then
+import it as the desired image format::
+
+       rbd export mypool/myimage@snap /tmp/img
+       rbd import --image-format 2 /tmp/img mypool/myimage2
+
+To lock an image for exclusive use::
+
+       rbd lock add mypool/myimage mylockid
+
+To release a lock::
+
+       rbd lock remove mypool/myimage mylockid client.2485
+
+To list images from trash::
+
+       rbd trash ls mypool
+
+To defer delete an image (use *--expires-at* to set expiration time, default is now)::
+
+       rbd trash mv mypool/myimage --expires-at "tomorrow"
+
+To delete an image from trash (be careful!)::
+
+       rbd trash rm mypool/myimage-id
+
+To force delete an image from trash (be careful!)::
+
+       rbd trash rm mypool/myimage-id  --force
+
+To restore an image from trash::
+
+       rbd trash restore mypool/myimage-id
+
+To restore an image from trash and rename it::
+
+       rbd trash restore mypool/myimage-id --image mynewimage
+
+
+Availability
+============
+
+**rbd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`rados <rados>`\(8)
diff --git a/doc/man/8/rbdmap.rst b/doc/man/8/rbdmap.rst
new file mode 100644
index 000000000..1a9096a99
--- /dev/null
+++ b/doc/man/8/rbdmap.rst
@@ -0,0 +1,128 @@
+:orphan:
+
+=========================================
+ rbdmap -- map RBD devices at boot time
+=========================================
+
+.. program:: rbdmap
+
+Synopsis
+========
+
+| **rbdmap map**
+| **rbdmap unmap**
+
+
+Description
+===========
+
+**rbdmap** is a shell script that automates ``rbd map`` and ``rbd unmap``
+operations on one or more RBD (RADOS Block Device) images. While the script can be
+run manually by the system administrator at any time, the principal use case is
+automatic mapping/mounting of RBD images at boot time (and unmounting/unmapping
+at shutdown), as triggered by the init system (a systemd unit file,
+``rbdmap.service`` is included with the ceph-common package for this purpose).
+
+The script takes a single argument, which can be either "map" or "unmap".
+In either case, the script parses a configuration file (defaults to ``/etc/ceph/rbdmap``,
+but can be overridden via an environment variable ``RBDMAPFILE``). Each line
+of the configuration file corresponds to an RBD image which is to be mapped, or
+unmapped.
+
+The configuration file format is::
+
+    IMAGESPEC RBDOPTS
+
+where ``IMAGESPEC`` should be specified as ``POOLNAME/IMAGENAME`` (the pool
+name, a forward slash, and the image name), or merely ``IMAGENAME``, in which
+case the ``POOLNAME`` defaults to "rbd". ``RBDOPTS`` is an optional list of
+parameters to be passed to the underlying ``rbd map`` command. These parameters
+and their values should be specified as a comma-separated string::
+
+    PARAM1=VAL1,PARAM2=VAL2,...,PARAMN=VALN 
+
+This will cause the script to issue an ``rbd map`` command like the following::
+
+    rbd map POOLNAME/IMAGENAME --PARAM1 VAL1 --PARAM2 VAL2 
+
+(See the ``rbd`` manpage for a full list of possible options.)
+For parameters and values which contain commas or equality signs, a simple
+apostrophe can be used to prevent replacing them.
+
+When run as ``rbdmap map``, the script parses the configuration file, and for
+each RBD image specified attempts to first map the image (using the ``rbd map``
+command) and, second, to mount the image.
+
+When run as ``rbdmap unmap``, images listed in the configuration file will
+be unmounted and unmapped.
+
+``rbdmap unmap-all`` attempts to unmount and subsequently unmap all currently
+mapped RBD images, regardless of whether or not they are listed in the
+configuration file.
+
+If successful, the ``rbd map`` operation maps the image to a ``/dev/rbdX``
+device, at which point a udev rule is triggered to create a friendly device
+name symlink, ``/dev/rbd/POOLNAME/IMAGENAME``, pointing to the real mapped
+device.
+
+In order for mounting/unmounting to succeed, the friendly device name must
+have a corresponding entry in ``/etc/fstab``.
+
+When writing ``/etc/fstab`` entries for RBD images, it's a good idea to specify
+the "noauto" (or "nofail") mount option. This prevents the init system from
+trying to mount the device too early - before the device in question even
+exists. (Since ``rbdmap.service``
+executes a shell script, it is typically triggered quite late in the boot
+sequence.)
+
+
+Examples
+========
+
+Example ``/etc/ceph/rbdmap`` for three RBD images called "bar1", "bar2" and "bar3", 
+which are in pool "foopool"::
+
+    foopool/bar1    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
+    foopool/bar2    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
+    foopool/bar3    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring,options='lock_on_read,queue_depth=1024'
+
+Each line in the file contains two strings: the image spec and the options to
+be passed to ``rbd map``. These two lines get transformed into the following
+commands::
+
+    rbd map foopool/bar1 --id admin --keyring /etc/ceph/ceph.client.admin.keyring
+    rbd map foopool/bar2 --id admin --keyring /etc/ceph/ceph.client.admin.keyring
+    rbd map foopool/bar2 --id admin --keyring /etc/ceph/ceph.client.admin.keyring --options lock_on_read,queue_depth=1024
+
+If the images had XFS file systems on them, the corresponding ``/etc/fstab``
+entries might look like this::
+
+    /dev/rbd/foopool/bar1 /mnt/bar1 xfs noauto 0 0
+    /dev/rbd/foopool/bar2 /mnt/bar2 xfs noauto 0 0
+    /dev/rbd/foopool/bar3 /mnt/bar3 xfs noauto 0 0
+
+After creating the images and populating the ``/etc/ceph/rbdmap`` file, making
+the images get automatically mapped and mounted at boot is just a matter of
+enabling that unit::
+
+    systemctl enable rbdmap.service
+
+
+Options
+=======
+
+None
+
+
+Availability
+============
+
+**rbdmap** is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at
+https://docs.ceph.com for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8),
diff --git a/doc/man/8/rgw-orphan-list.rst b/doc/man/8/rgw-orphan-list.rst
new file mode 100644
index 000000000..7b36ec771
--- /dev/null
+++ b/doc/man/8/rgw-orphan-list.rst
@@ -0,0 +1,69 @@
+:orphan:
+
+==================================================================
+ rgw-orphan-list -- list rados objects that are not indexed by rgw
+==================================================================
+
+.. program:: rgw-orphan-list
+
+Synopsis
+========
+
+| **rgw-orphan-list**
+
+Description
+===========
+
+:program:`rgw-orphan-list` is an *EXPERIMENTAL* RADOS gateway user
+administration utility. It produces a listing of rados objects that
+are not directly or indirectly referenced through the bucket indexes
+on a pool. It places the results and intermediate files on the local
+filesystem rather than on the ceph cluster itself, and therefore will
+not itself consume additional cluster storage.
+
+In theory orphans should not exist. However because ceph evolves
+rapidly, bugs do crop up, and they may result in orphans that are left
+behind.
+
+In its current form this utility does not take any command-line
+arguments or options. It will list the available pools and prompt the
+user to enter the pool they would like to list orphans for.
+
+Behind the scenes it runs `rados ls` and `radosgw-admin bucket
+radoslist ...` and produces a list of those entries that appear in the
+former but not the latter. Those entries are presumed to be the
+orphans.
+
+Warnings
+========
+
+This utility is currently considered *EXPERIMENTAL*.
+
+This utility will produce false orphan entries for unindexed buckets
+since such buckets have no bucket indices that can provide the
+starting point for tracing.
+
+Options
+=======
+
+At present there are no options.
+
+Examples
+========
+
+Launch the tool::
+
+        $ rgw-orphan-list
+
+Availability
+============
+
+:program:`radosgw-admin` is part of Ceph, a massively scalable, open-source,
+distributed storage system.  Please refer to the Ceph documentation at
+https://docs.ceph.com for more information.
+
+See also
+========
+
+:doc:`radosgw-admin <radosgw-admin>`\(8)
+:doc:`ceph-diff-sorted <ceph-diff-sorted>`\(8)
diff --git a/doc/man/8/rgw-policy-check.rst b/doc/man/8/rgw-policy-check.rst
new file mode 100644
index 000000000..3f055711f
--- /dev/null
+++ b/doc/man/8/rgw-policy-check.rst
@@ -0,0 +1,55 @@
+:orphan:
+
+===================================================
+rgw-policy-check -- verify syntax of bucket policy
+===================================================
+
+.. program:: rgw-policy-check
+
+Synopsis
+========
+
+| **rgw-policy-check**
+   -t *tenant* [ *filename* ... ]
+
+
+Description
+===========
+
+This program reads one or more files containing bucket policy
+and determines if it is syntactically correct.
+It does not check to see if the policy makes sense;
+it only checks to see if the file would be accepted
+by the policy parsing logic inside
+:program:`radsogw`.
+
+More than one filename may be specified.  If no files are
+given, the program will read from stdin.
+
+On success, the program will say nothing.  On failure,
+the program will emit a error message indicating the
+problem.  The program will terminate with non-zero exit
+status if one or more policies could not be read or parsed.
+
+Options
+=======
+
+.. option: -t *tenant*
+
+   Specify *tenant* as the tenant.  This is required by the
+   policy parsing logic and is used to construct the internal
+   state representation of the policy.
+
+Availability
+============
+
+**rgw-policy-check** is part of Ceph, a massively scalable, open-source,
+distributed storage system.  Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`radosgw <radosgw>`\(8)
+
+.. _Bucket Policies: ../../radosgw/bucketpolicy.rst
diff --git a/doc/man/CMakeLists.txt b/doc/man/CMakeLists.txt
new file mode 100644
index 000000000..e81631bab
--- /dev/null
+++ b/doc/man/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(sphinx_input)
+set(sphinx_output)
+set(sphinx_output_dir ${CMAKE_BINARY_DIR}/doc/man)
+
+add_subdirectory(8)
+
+add_custom_command(
+  OUTPUT ${sphinx_output}
+  COMMAND ${SPHINX_BUILD} -b man -t man -d ${CMAKE_BINARY_DIR}/doc/doctrees -c ${CMAKE_SOURCE_DIR}/man ${CMAKE_CURRENT_SOURCE_DIR} ${sphinx_output_dir}
+  DEPENDS ${sphinx_input})
+
+add_custom_target(
+  manpages ALL
+  DEPENDS ${sphinx_output}
+  COMMENT "manpages building")
diff --git a/doc/man_index.rst b/doc/man_index.rst
new file mode 100644
index 000000000..989ed7e22
--- /dev/null
+++ b/doc/man_index.rst
@@ -0,0 +1,50 @@
+.. this is the man index/toctree reference. It is separate from the "regular"
+.. index so that it doesn't include things that are not tagged for man pages
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+
+   man/8/ceph-authtool
+   man/8/ceph-clsinfo
+   man/8/ceph-conf
+   man/8/ceph-create-keys
+   man/8/ceph-debugpack
+   man/8/ceph-dencoder
+   man/8/ceph-volume
+   man/8/ceph-volume-systemd
+   man/8/ceph-fuse
+   man/8/ceph-mds
+   man/8/ceph-mon
+   man/8/ceph-osd
+   man/8/ceph-post-file
+   man/8/ceph-rbdnamer
+   man/8/ceph-run
+   man/8/ceph-syn
+   man/8/ceph
+   man/8/cephadm
+   man/8/cephfs-top
+   man/8/cephfs-mirror
+   man/8/cephfs-shell
+   man/8/crushdiff
+   man/8/crushtool
+   man/8/librados-config
+   man/8/monmaptool
+   man/8/mount.ceph
+   man/8/osdmaptool
+   man/8/rados
+   man/8/radosgw-admin
+   man/8/radosgw
+   man/8/rbd-fuse
+   man/8/rbd-ggate
+   man/8/rbd-mirror
+   man/8/rbd-nbd
+   man/8/rbd-replay-many
+   man/8/rbd-replay-prep
+   man/8/rbd-replay
+   man/8/rbd
+   man/8/rbdmap
+   man/8/rgw-orphan-list
+   man/8/ceph-immutable-object-cache
+   man/8/ceph-diff-sorted
+   man/8/rgw-policy-check
diff --git a/doc/mgr/administrator.rst b/doc/mgr/administrator.rst
new file mode 100644
index 000000000..d59b013aa
--- /dev/null
+++ b/doc/mgr/administrator.rst
@@ -0,0 +1,178 @@
+.. _mgr-administrator-guide:
+
+ceph-mgr administrator's guide
+==============================
+
+Manual setup
+------------
+
+Usually, you would set up a ceph-mgr daemon using a tool such
+as ceph-ansible.  These instructions describe how to set up
+a ceph-mgr daemon manually.
+
+First, create an authentication key for your daemon::
+
+    ceph auth get-or-create mgr.$name mon 'allow profile mgr' osd 'allow *' mds 'allow *'
+
+Place that key as file named ``keyring`` into ``mgr data`` path, which for a cluster "ceph"
+and mgr $name "foo" would be ``/var/lib/ceph/mgr/ceph-foo`` respective ``/var/lib/ceph/mgr/ceph-foo/keyring``.
+
+Start the ceph-mgr daemon::
+
+    ceph-mgr -i $name
+
+Check that the mgr has come up by looking at the output
+of ``ceph status``, which should now include a mgr status line::
+
+    mgr active: $name
+
+Client authentication
+---------------------
+
+The manager is a new daemon which requires new CephX capabilities. If you upgrade
+a cluster from an old version of Ceph, or use the default install/deploy tools,
+your admin client should get this capability automatically. If you use tooling from
+elsewhere, you may get EACCES errors when invoking certain ceph cluster commands.
+To fix that, add a "mgr allow \*" stanza to your client's cephx capabilities by
+`Modifying User Capabilities`_.
+
+High availability
+-----------------
+
+In general, you should set up a ceph-mgr on each of the hosts
+running a ceph-mon daemon to achieve the same level of availability.
+
+By default, whichever ceph-mgr instance comes up first will be made
+active by the monitors, and the others will be standbys.  There is
+no requirement for quorum among the ceph-mgr daemons.
+
+If the active daemon fails to send a beacon to the monitors for
+more than :confval:`mon_mgr_beacon_grace`, then it will be replaced
+by a standby.
+
+If you want to preempt failover, you can explicitly mark a ceph-mgr
+daemon as failed using ``ceph mgr fail <mgr name>``.
+
+Performance and Scalability
+---------------------------
+
+All the mgr modules share a cache that can be enabled with
+``ceph config set mgr mgr_ttl_cache_expire_seconds <seconds>``, where seconds
+is the time to live of the cached python objects.
+
+It is recommended to enable the cache with a 10 seconds TTL when there are 500+
+osds or 10k+ pgs as internal structures might increase in size, and cause latency
+issues when requesting large structures. As an example, an OSDMap with 1000 osds
+has a approximate size of 4MiB. With heavy load, on a 3000 osd cluster there has
+been a 1.5x improvement enabling the cache.
+
+Furthermore, you can run ``ceph daemon mgr.${MGRNAME} perf dump`` to retrieve perf
+counters of a mgr module. In ``mgr.cache_hit`` and ``mgr.cache_miss`` you'll find the
+hit/miss ratio of the mgr cache.
+
+Using modules
+-------------
+
+Use the command ``ceph mgr module ls`` to see which modules are
+available, and which are currently enabled. Use ``ceph mgr module ls --format=json-pretty``
+to view detailed metadata about disabled modules. Enable or disable modules
+using the commands ``ceph mgr module enable <module>`` and
+``ceph mgr module disable <module>`` respectively.
+
+If a module is *enabled* then the active ceph-mgr daemon will load
+and execute it.  In the case of modules that provide a service,
+such as an HTTP server, the module may publish its address when it
+is loaded.  To see the addresses of such modules, use the command
+``ceph mgr services``.
+
+Some modules may also implement a special standby mode which runs on
+standby ceph-mgr daemons as well as the active daemon.  This enables
+modules that provide services to redirect their clients to the active
+daemon, if the client tries to connect to a standby.
+
+Consult the documentation pages for individual manager modules for more
+information about what functionality each module provides.
+
+Here is an example of enabling the :term:`Dashboard` module:
+
+.. code-block:: console
+
+	$ ceph mgr module ls
+	{
+		"enabled_modules": [
+			"restful",
+			"status"
+		],
+		"disabled_modules": [
+			"dashboard"
+		]
+	}
+
+	$ ceph mgr module enable dashboard
+	$ ceph mgr module ls
+	{
+		"enabled_modules": [
+			"restful",
+			"status",
+			"dashboard"
+		],
+		"disabled_modules": [
+		]
+	}
+
+	$ ceph mgr services
+	{
+		"dashboard": "http://myserver.com:7789/",
+		"restful": "https://myserver.com:8789/"
+	}
+
+
+The first time the cluster starts, it uses the :confval:`mgr_initial_modules`
+setting to override which modules to enable.  However, this setting
+is ignored through the rest of the lifetime of the cluster: only
+use it for bootstrapping.  For example, before starting your
+monitor daemons for the first time, you might add a section like
+this to your ``ceph.conf``:
+
+.. code-block:: ini
+
+    [mon]
+        mgr_initial_modules = dashboard balancer
+
+Module Pool
+-----------
+
+The manager creates a pool for use by its module to store state. The name of
+this pool is ``.mgr`` (with the leading ``.`` indicating a reserved pool
+name).
+
+.. note::
+
+   Prior to Quincy, the ``devicehealth`` module created a
+   ``device_health_metrics`` pool to store device SMART statistics. With
+   Quincy, this pool is automatically renamed to be the common manager module
+   pool.
+
+
+Calling module commands
+-----------------------
+
+Where a module implements command line hooks, the commands will
+be accessible as ordinary Ceph commands.  Ceph will automatically incorporate
+module commands into the standard CLI interface and route them appropriately to
+the module.::
+
+    ceph <command | help>
+
+Configuration
+-------------
+
+.. confval:: mgr_module_path
+.. confval:: mgr_initial_modules
+.. confval:: mgr_disabled_modules
+.. confval:: mgr_standby_modules
+.. confval:: mgr_data
+.. confval:: mgr_tick_period
+.. confval:: mon_mgr_beacon_grace
+
+.. _Modifying User Capabilities: ../../rados/operations/user-management/#modify-user-capabilities
diff --git a/doc/mgr/alerts.rst b/doc/mgr/alerts.rst
new file mode 100644
index 000000000..319d9d927
--- /dev/null
+++ b/doc/mgr/alerts.rst
@@ -0,0 +1,58 @@
+Alerts module
+=============
+
+The alerts module can send simple alert messages about cluster health
+via e-mail.  In the future, it will support other notification methods
+as well.
+
+:note: This module is *not* intended to be a robust monitoring
+       solution.  The fact that it is run as part of the Ceph cluster
+       itself is fundamentally limiting in that a failure of the
+       ceph-mgr daemon prevents alerts from being sent.  This module
+       can, however, be useful for standalone clusters that exist in
+       environments where existing monitoring infrastructure does not
+       exist.
+
+Enabling
+--------
+
+The *alerts* module is enabled with::
+
+  ceph mgr module enable alerts
+
+Configuration
+-------------
+
+To configure SMTP, all of the following config options must be set::
+
+  ceph config set mgr mgr/alerts/smtp_host *<smtp-server>*
+  ceph config set mgr mgr/alerts/smtp_destination *<email-address-to-send-to>*
+  ceph config set mgr mgr/alerts/smtp_sender *<from-email-address>*
+
+By default, the module will use SSL and port 465.  To change that,::
+
+  ceph config set mgr mgr/alerts/smtp_ssl false   # if not SSL
+  ceph config set mgr mgr/alerts/smtp_port *<port-number>*  # if not 465
+
+To authenticate to the SMTP server, you must set the user and password::
+
+  ceph config set mgr mgr/alerts/smtp_user *<username>*
+  ceph config set mgr mgr/alerts/smtp_password *<password>*
+
+By default, the name in the ``From:`` line is simply ``Ceph``.  To
+change that (e.g., to identify which cluster this is),::
+
+  ceph config set mgr mgr/alerts/smtp_from_name 'Ceph Cluster Foo'
+
+By default, the module will check the cluster health once per minute
+and, if there is a change, send a message.  To change that
+frequency,::
+
+  ceph config set mgr mgr/alerts/interval *<interval>*   # e.g., "5m" for 5 minutes
+
+Commands
+--------
+
+To force an alert to be send immediately,::
+
+  ceph alerts send
diff --git a/doc/mgr/capacity-card.png b/doc/mgr/capacity-card.png
new file mode 100644
index 000000000..59a70348c
--- /dev/null
+++ b/doc/mgr/capacity-card.png
diff --git a/doc/mgr/ceph_api/index.rst b/doc/mgr/ceph_api/index.rst
new file mode 100644
index 000000000..1cdc9a97b
--- /dev/null
+++ b/doc/mgr/ceph_api/index.rst
@@ -0,0 +1,92 @@
+.. _mgr ceph api:
+
+================
+Ceph RESTful API
+================
+
+Introduction
+============
+The **Ceph RESTful API** (henceforth **Ceph API**) is provided by the
+:ref:`mgr-dashboard` module. The Ceph API
+service is available at the same URL as the regular Ceph Dashboard, under the
+``/api`` base path (please refer to :ref:`dashboard-host-name-and-port`)::
+
+  http://<server_addr>:<server_port>/api
+
+or, if HTTPS is enabled (please refer to :ref:`dashboard-ssl-tls-support`)::
+
+  https://<server_addr>:<ssl_server_port>/api
+
+The Ceph API leverages the following standards:
+
+* `HTTP 1.1 <https://tools.ietf.org/html/rfc7231>`_ for API syntax and semantics,
+* `JSON <https://tools.ietf.org/html/rfc8259>`_ for content encoding,
+* `HTTP Content Negotiation <https://tools.ietf.org/html/rfc2295>`_ and `MIME <https://tools.ietf.org/html/rfc2045>`_ for versioning,
+* `OAuth 2.0 <https://tools.ietf.org/html/rfc6750>`_ and `JWT <https://tools.ietf.org/html/rfc7519>`_ for authentication and authorization.
+
+.. warning::
+  Some endpoints are still under active development, and should be carefully
+  used since new Ceph releases could bring backward incompatible changes.
+
+
+Authentication and Authorization
+================================
+
+Requests to the Ceph API pass through two access control checkpoints:
+
+* **Authentication**: ensures that the request is performed on behalf of an existing and valid user account.
+* **Authorization**: ensures that the previously authenticated user can in fact perform a specific action (create, read, update or delete) on the target endpoint.
+
+So, prior to start consuming the Ceph API, a valid JSON Web Token (JWT) has to
+be obtained, and it may then be reused for subsequent requests. The
+``/api/auth`` endpoint will provide the valid token:
+
+.. prompt:: bash $
+
+   curl -X POST "https://example.com:8443/api/auth" \
+   -H  "Accept: application/vnd.ceph.api.v1.0+json" \
+   -H  "Content-Type: application/json" \
+   -d '{"username": <username>, "password": <password>}'
+
+::
+
+    { "token": "<redacted_token>", ...}
+
+The token obtained must be passed together with every API request in the
+``Authorization`` HTTP header::
+
+  curl -H "Authorization: Bearer <token>" ...
+
+Authentication and authorization can be further configured from the
+Ceph CLI, the Ceph-Dashboard UI and the Ceph API itself (please refer to
+:ref:`dashboard-user-role-management`).
+
+Versioning
+==========
+
+One of the main goals of the Ceph API is to keep a stable interface. For this
+purpose, Ceph API is built upon the following principles:
+
+* **Mandatory**: in order to avoid implicit defaults, all endpoints require an explicit default version (starting with ``1.0``).
+* **Per-endpoint**: as this API wraps many different Ceph components, this allows for a finer-grained change control.
+   * **Content/MIME Type**: the version expected from a specific endpoint is stated by the ``Accept: application/vnd.ceph.api.v<major>.<minor>+json`` HTTP header. If the current Ceph API server is not able to address that specific major version, a `415 - Unsupported Media Type <https://tools.ietf.org/html/rfc7231#section-6.5.13>`_ response will be returned.
+* **Semantic Versioning**: with a ``major.minor`` version:
+   * Major changes are backward incompatible: they might result in non-additive changes to the request and/or response formats of a specific endpoint.
+   * Minor changes are backward/forward compatible: they basically consists of additive changes to the request or response formats of a specific endpoint.
+
+An example:
+
+.. prompt:: bash $
+
+   curl -X GET "https://example.com:8443/api/osd" \
+   -H  "Accept: application/vnd.ceph.api.v1.0+json" \
+   -H  "Authorization: Bearer <token>"
+
+
+Specification
+=============
+
+.. openapi:: ../../../src/pybind/mgr/dashboard/openapi.yaml
+    :group:
+    :examples:
+    :encoding: utf-8
diff --git a/doc/mgr/cli_api.rst b/doc/mgr/cli_api.rst
new file mode 100644
index 000000000..81a99ae44
--- /dev/null
+++ b/doc/mgr/cli_api.rst
@@ -0,0 +1,39 @@
+CLI API Commands Module
+=======================
+
+The CLI API module exposes most ceph-mgr python API via CLI. Furthermore, this API can be
+benchmarked for further testing.
+
+Enabling
+--------
+
+The *cli api commands* module is enabled with::
+
+  ceph mgr module enable cli_api
+
+To check that it is enabled, run::
+
+  ceph mgr module ls | grep cli_api
+
+Usage
+--------
+
+To run a mgr module command, run::
+
+  ceph mgr cli <command> <param>
+
+For example, use the following command to print the list of servers::
+
+  ceph mgr cli list_servers
+
+List all available mgr module commands with::
+
+  ceph mgr cli --help
+
+To benchmark a command, run::
+
+  ceph mgr cli_benchmark <number of calls> <number of threads> <command> <param>
+
+For example, use the following command to benchmark the command to get osd_map::
+
+  ceph mgr cli_benchmark 100 10 get osd_map
diff --git a/doc/mgr/cluster-utilization-card.png b/doc/mgr/cluster-utilization-card.png
new file mode 100644
index 000000000..fc0fd9ed1
--- /dev/null
+++ b/doc/mgr/cluster-utilization-card.png
diff --git a/doc/mgr/crash.rst b/doc/mgr/crash.rst
new file mode 100644
index 000000000..656c7a2d4
--- /dev/null
+++ b/doc/mgr/crash.rst
@@ -0,0 +1,103 @@
+Crash Module
+============
+The crash module collects information about daemon crashdumps and stores
+it in the Ceph cluster for later analysis.
+
+Enabling
+--------
+
+The *crash* module is enabled with::
+
+  ceph mgr module enable crash
+
+The *crash* upload key is generated with::
+
+  ceph auth get-or-create client.crash mon 'profile crash' mgr 'profile crash'
+
+On each node, you should store this key in
+``/etc/ceph/ceph.client.crash.keyring``.
+
+
+Automated collection
+--------------------
+
+Daemon crashdumps are dumped in ``/var/lib/ceph/crash`` by default; this can
+be configured with the option 'crash dir'.  Crash directories are named by
+time and date and a randomly-generated UUID, and contain a metadata file
+'meta' and a recent log file, with a "crash_id" that is the same.
+
+These crashes can be automatically submitted and persisted in the monitors'
+storage by using ``ceph-crash.service``.
+It watches the crashdump directory and uploads them with ``ceph crash post``.
+
+``ceph-crash`` tries some authentication names: ``client.crash.$hostname``,
+``client.crash`` and ``client.admin``.
+In order to successfully upload with ``ceph crash post``, these need
+the suitable permissions: ``mon profile crash`` and ``mgr profile crash``
+and a keyring needs to be in ``/etc/ceph``.
+
+
+Commands
+--------
+::
+
+  ceph crash post -i <metafile>
+
+Save a crash dump.  The metadata file is a JSON blob stored in the crash
+dir as ``meta``.  As usual, the ceph command can be invoked with ``-i -``,
+and will read from stdin.
+
+::
+
+  ceph crash rm <crashid>
+
+Remove a specific crash dump.
+
+::
+
+  ceph crash ls
+
+List the timestamp/uuid crashids for all new and archived crash info.
+
+::
+
+  ceph crash ls-new
+
+List the timestamp/uuid crashids for all newcrash info.
+
+::
+
+  ceph crash stat
+
+Show a summary of saved crash info grouped by age.
+
+::
+
+  ceph crash info <crashid>
+
+Show all details of a saved crash.
+
+::
+
+   ceph crash prune <keep>
+
+Remove saved crashes older than 'keep' days.  <keep> must be an integer.
+
+::
+
+   ceph crash archive <crashid>
+
+Archive a crash report so that it is no longer considered for the ``RECENT_CRASH`` health check and does not appear in the ``crash ls-new`` output (it will still appear in the ``crash ls`` output).
+
+::
+
+   ceph crash archive-all
+
+Archive all new crash reports.
+
+
+Options
+-------
+
+* ``mgr/crash/warn_recent_interval`` [default: 2 weeks] controls what constitutes "recent" for the purposes of raising the ``RECENT_CRASH`` health warning.
+* ``mgr/crash/retain_interval`` [default: 1 year] controls how long crash reports are retained by the cluster before they are automatically purged.
diff --git a/doc/mgr/dashboard-landing-page.png b/doc/mgr/dashboard-landing-page.png
new file mode 100644
index 000000000..77a1fe12b
--- /dev/null
+++ b/doc/mgr/dashboard-landing-page.png
diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst
new file mode 100644
index 000000000..696676aeb
--- /dev/null
+++ b/doc/mgr/dashboard.rst
@@ -0,0 +1,1655 @@
+.. _mgr-dashboard:
+
+Ceph Dashboard
+==============
+
+Overview
+--------
+
+The Ceph Dashboard is a built-in web-based Ceph management and monitoring
+application through which you can inspect and administer various aspects
+and resources within the cluster. It is implemented as a :ref:`ceph-manager-daemon` module.
+
+The original Ceph Dashboard that was shipped with Ceph Luminous started
+out as a simple read-only view into run-time information and performance
+data of Ceph clusters. It used a very simple architecture to achieve the
+original goal. However, there was growing demand for richer web-based
+management capabilities, to make it easier to administer Ceph for users that
+prefer a WebUI over the CLI.
+
+The new :term:`Ceph Dashboard` module adds web-based monitoring and
+administration to the Ceph Manager. The architecture and functionality of this new
+module are derived from
+and inspired by the `openATTIC Ceph management and monitoring tool
+<https://openattic.org/>`_. Development is actively driven by the
+openATTIC team at `SUSE <https://www.suse.com/>`_, with support from
+companies including `Red Hat <https://redhat.com/>`_ and members of the Ceph
+community.
+
+The dashboard module's backend code uses the CherryPy framework and implements
+a custom REST API. The WebUI implementation is based on
+Angular/TypeScript and includes both functionality from the original dashboard
+and new features originally developed for the standalone version
+of openATTIC. The Ceph Dashboard module is implemented as an
+application that provides a graphical representation of information and statistics
+through a web server hosted by ``ceph-mgr``.
+
+Feature Overview
+^^^^^^^^^^^^^^^^
+
+The dashboard provides the following features:
+
+* **Multi-User and Role Management**: The dashboard supports multiple user
+  accounts with different permissions (roles). User accounts and roles
+  can be managed via both the command line and the WebUI. The dashboard
+  supports various methods to enhance password security. Password
+  complexity rules may be configured, requiring users to change their password
+  after the first login or after a configurable time period. See
+  :ref:`dashboard-user-role-management` for details.
+* **Single Sign-On (SSO)**: The dashboard supports authentication
+  via an external identity provider using the SAML 2.0 protocol. See
+  :ref:`dashboard-sso-support` for details.
+* **SSL/TLS support**: All HTTP communication between the web browser and the
+  dashboard is secured via SSL. A self-signed certificate can be created with
+  a built-in command, but it's also possible to import custom certificates
+  signed and issued by a CA. See :ref:`dashboard-ssl-tls-support` for details.
+* **Auditing**: The dashboard backend can be configured to log all ``PUT``, ``POST``
+  and ``DELETE`` API requests in the Ceph audit log. See :ref:`dashboard-auditing`
+  for instructions on how to enable this feature.
+* **Internationalization (I18N)**: The language used for dashboard text can be
+  selected at run-time.
+
+The Ceph Dashboard offers the following monitoring and management capabilities:
+
+* **Overall cluster health**: Display performance and capacity metrics as well
+  as cluster status.
+* **Embedded Grafana Dashboards**: Ceph Dashboard
+  `Grafana`_ dashboards may be embedded in external applications and web pages
+  to surface information and performance metrics gathered by
+  the :ref:`mgr-prometheus` module. See
+  :ref:`dashboard-grafana` for details on how to configure this functionality.
+* **Cluster logs**: Display the latest updates to the cluster's event and
+  audit log files. Log entries can be filtered by priority, date or keyword.
+* **Hosts**: Display a list of all cluster hosts along with their
+  storage drives, which services are running, and which version of Ceph is
+  installed.
+* **Performance counters**: Display detailed service-specific statistics for
+  each running service.
+* **Monitors**: List all Mons, their quorum status, and open sessions.
+* **Monitoring**: Enable creation, re-creation, editing, and expiration of
+  Prometheus' silences, list the alerting configuration and all
+  configured and firing alerts. Show notifications for firing alerts.
+* **Configuration Editor**: Display all available configuration options,
+  their descriptions, types, default and currently set values.  These may be edited as well.
+* **Pools**: List Ceph pools and their details (e.g. applications,
+  pg-autoscaling, placement groups, replication size, EC profile, CRUSH
+  rules, quotas etc.)
+* **OSDs**: List OSDs, their status and usage statistics as well as
+  detailed information like attributes (OSD map), metadata, performance
+  counters and usage histograms for read/write operations. Mark OSDs
+  up/down/out, purge and reweight OSDs, perform scrub operations, modify
+  various scrub-related configuration options, select profiles to
+  adjust the level of backfilling activity. List all drives associated with an
+  OSD. Set and change the device class of an OSD, display and sort OSDs by
+  device class. Deploy OSDs on new drives and hosts.
+* **Device management**: List all hosts known by the orchestrator. List all
+  drives attached to a host and their properties. Display drive
+  health predictions and SMART data. Blink enclosure LEDs.
+* **iSCSI**: List all hosts that run the TCMU runner service, display all
+  images and their performance characteristics (read/write ops, traffic).
+  Create, modify, and delete iSCSI targets (via ``ceph-iscsi``). Display the
+  iSCSI gateway status and info about active initiators.
+  See :ref:`dashboard-iscsi-management` for instructions on how to configure
+  this feature.
+* **RBD**: List all RBD images and their properties (size, objects, features).
+  Create, copy, modify and delete RBD images (incl. snapshots) and manage RBD
+  namespaces. Define various I/O or bandwidth limitation settings on a global,
+  per-pool or per-image level. Create, delete and rollback snapshots of selected
+  images, protect/unprotect these snapshots against modification. Copy or clone
+  snapshots, flatten cloned images.
+* **RBD mirroring**: Enable and configure RBD mirroring to a remote Ceph server.
+  List active daemons and their status, pools and RBD images including
+  sync progress.
+* **CephFS**: List active file system clients and associated pools,
+  including usage statistics. Evict active CephFS clients. Manage CephFS
+  quotas and snapshots. Browse a CephFS directory structure.
+* **Object Gateway**: List all active object gateways and their performance
+  counters. Display and manage (add/edit/delete) object gateway users and their
+  details (e.g. quotas) as well as the users' buckets and their details (e.g.
+  placement targets, owner, quotas, versioning, multi-factor authentication).
+  See :ref:`dashboard-enabling-object-gateway` for configuration instructions.
+* **NFS**: Manage NFS exports of CephFS file systems and RGW S3 buckets via NFS
+  Ganesha. See :ref:`dashboard-nfs-ganesha-management` for details on how to
+  enable this functionality.
+* **Ceph Manager Modules**: Enable and disable Ceph Manager modules, manage
+  module-specific configuration settings.
+
+Overview of the Dashboard Landing Page
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The landing page of Ceph Dashboard serves as the home page and features metrics
+such as the overall cluster status, performance, and capacity. It provides real-time
+updates on any changes in the cluster and allows quick access to other sections of the dashboard.
+
+.. image:: dashboard-landing-page.png
+
+
+.. note::
+  You can change the landing page to the previous version from:
+  ``Cluster >> Manager Modules >> Dashboard >> Edit``.
+  Editing the ``FEATURE_TOGGLE_DASHBOARD`` option will change the landing page, from one view to another.
+
+  Note that the previous version of the landing page will be disabled in future releases.
+
+.. _dashboard-landing-page-details:
+
+Details
+"""""""
+Provides an overview of the cluster configuration, displaying various critical aspects of the cluster.
+
+.. image:: details-card.png
+
+.. _dashboard-landing-page-status:
+
+Status
+""""""
+Provides a visual indication of cluster health, and displays cluster alerts grouped by severity.
+
+.. image:: status-card-open.png
+
+.. _dashboard-landing-page-capacity:
+
+Capacity
+""""""""
+* **Used**: Displays the used capacity out of the total physical capacity provided by storage nodes (OSDs)
+* **Warning**: Displays the `nearfull` threshold of the OSDs
+* **Danger**: Displays the `full` threshold of the OSDs
+
+.. image:: capacity-card.png
+
+.. _dashboard-landing-page-inventory:
+
+Inventory
+"""""""""
+An inventory for all assets within the cluster.
+Provides direct access to subpages of the dashboard from each item of this card.
+
+.. image:: inventory-card.png
+
+.. _dashboard-landing-page-performance:
+
+Cluster Utilization
+"""""""""""""""""""
+* **Used Capacity**: Total capacity used of the cluster. The maximum value of the chart is the maximum capacity of the cluster.
+* **IOPS (Input/Output Operations Per Second)**: Number of read and write operations.
+* **Latency**: Amount of time that it takes to process a read or a write request.
+* **Client Throughput**: Amount of data that clients read or write to the cluster.
+* **Recovery Throughput**: Amount of recovery data that clients read or write to the cluster.
+
+
+.. image:: cluster-utilization-card.png
+
+Supported Browsers
+^^^^^^^^^^^^^^^^^^
+
+Ceph Dashboard is primarily tested and developed using the following web
+browsers:
+
++---------------------------------------------------------------+---------------------------------------+
+|                            Browser                            |               Versions                |
++===============================================================+=======================================+
+| `Chrome <https://www.google.com/chrome/>`_ and                | latest 2 major versions               |
+| `Chromium <https://www.chromium.org/>`_ based browsers        |                                       |
++---------------------------------------------------------------+---------------------------------------+
+| `Firefox <https://www.mozilla.org/firefox/>`_                 | latest 2 major versions               |
++---------------------------------------------------------------+---------------------------------------+
+| `Firefox ESR <https://www.mozilla.org/firefox/enterprise/>`_  | latest major version                  |
++---------------------------------------------------------------+---------------------------------------+
+
+While Ceph Dashboard might work in older browsers, we cannot guarantee compatibility and
+recommend keeping your browser up to date.
+
+Enabling
+--------
+
+If you have installed ``ceph-mgr-dashboard`` from distribution packages, the
+package management system should take care of installing all required
+dependencies.
+
+If you're building Ceph from source and want to start the dashboard from your
+development environment, please see the files ``README.rst`` and ``HACKING.rst``
+in the source directory ``src/pybind/mgr/dashboard``.
+
+Within a running Ceph cluster, the Ceph Dashboard is enabled with:
+
+.. prompt:: bash $
+
+   ceph mgr module enable dashboard
+
+Configuration
+-------------
+
+.. _dashboard-ssl-tls-support:
+
+SSL/TLS Support
+^^^^^^^^^^^^^^^
+
+All HTTP connections to the dashboard are secured with SSL/TLS by default.
+
+To get the dashboard up and running quickly, you can generate and install a
+self-signed certificate:
+
+.. prompt:: bash $
+
+   ceph dashboard create-self-signed-cert
+
+Note that most web browsers will complain about self-signed certificates
+and require explicit confirmation before establishing a secure connection to the
+dashboard.
+
+To properly secure a deployment and to remove the warning, a
+certificate that is issued by a certificate authority (CA) should be used.
+
+For example, a key pair can be generated with a command similar to:
+
+.. prompt:: bash $
+
+   openssl req -new -nodes -x509 \
+   -subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 \
+   -keyout dashboard.key -out dashboard.crt -extensions v3_ca
+
+The ``dashboard.crt`` file should then be signed by a CA. Once that is done, you
+can enable it for Ceph manager instances by running the following commands:
+
+.. prompt:: bash $
+
+   ceph dashboard set-ssl-certificate -i dashboard.crt
+   ceph dashboard set-ssl-certificate-key -i dashboard.key
+
+If unique certificates are desired for each manager instance,
+the name of the instance can be included as follows (where ``$name`` is the name
+of the ``ceph-mgr`` instance, usually the hostname):
+
+.. prompt:: bash $
+
+   ceph dashboard set-ssl-certificate $name -i dashboard.crt
+   ceph dashboard set-ssl-certificate-key $name -i dashboard.key
+
+SSL can also be disabled by setting this configuration value:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/ssl false
+
+This might be useful if the dashboard will be running behind a proxy which does
+not support SSL for its upstream servers or other situations where SSL is not
+wanted or required. See :ref:`dashboard-proxy-configuration` for more details.
+
+.. warning::
+
+  Use caution when disabling SSL as usernames and passwords will be sent to the
+  dashboard unencrypted.
+
+
+.. note::
+
+  You must restart Ceph manager processes after changing the SSL
+  certificate and key. This can be accomplished by either running ``ceph mgr
+  fail mgr`` or by disabling and re-enabling the dashboard module (which also
+  triggers the manager to respawn itself):
+  
+  .. prompt:: bash $
+
+     ceph mgr module disable dashboard
+     ceph mgr module enable dashboard
+
+.. _dashboard-host-name-and-port:
+
+Host Name and Port
+^^^^^^^^^^^^^^^^^^
+
+Like most web applications, the dashboard binds to a TCP/IP address and TCP port.
+
+By default, the ``ceph-mgr`` daemon hosting the dashboard (i.e., the currently
+active manager) will bind to TCP port 8443 or 8080 when SSL is disabled.
+
+If no specific address has been configured, the web app will bind to ``::``,
+which corresponds to all available IPv4 and IPv6 addresses.
+
+These defaults can be changed via the configuration key facility on a
+cluster-wide level (so they apply to all manager instances) as follows:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/server_addr $IP
+   ceph config set mgr mgr/dashboard/server_port $PORT
+   ceph config set mgr mgr/dashboard/ssl_server_port $PORT
+
+Since each ``ceph-mgr`` hosts its own instance of the dashboard, it may be
+necessary to configure them separately. The IP address and port for a specific
+manager instance can be changed with the following commands:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/$name/server_addr $IP
+   ceph config set mgr mgr/dashboard/$name/server_port $PORT
+   ceph config set mgr mgr/dashboard/$name/ssl_server_port $PORT
+
+Replace ``$name`` with the ID of the ceph-mgr instance hosting the dashboard.
+
+.. note::
+
+  The command ``ceph mgr services`` will show you all endpoints that are
+  currently configured. Look for the ``dashboard`` key to obtain the URL for
+  accessing the dashboard.
+
+Username and Password
+^^^^^^^^^^^^^^^^^^^^^
+
+In order to be able to log in, you need to create a user account and associate
+it with at least one role. We provide a set of predefined *system roles* that
+you can use. For more details please refer to the `User and Role Management`_
+section.
+
+To create a user with the administrator role you can use the following
+commands:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-user-create <username> -i <file-containing-password> administrator
+
+Account Lock-out
+^^^^^^^^^^^^^^^^
+
+It disables a user account if a user repeatedly enters the wrong credentials
+for multiple times. It is enabled by default to prevent brute-force or dictionary
+attacks. The user can get or set the default number of lock-out attempts using
+these commands respectively:
+
+.. prompt:: bash $
+
+   ceph dashboard get-account-lockout-attempts
+   ceph dashboard set-account-lockout-attempts <value:int>
+
+.. warning::
+
+  This feature can be disabled by setting the default number of lock-out attempts to 0.
+  However, by disabling this feature, the account is more vulnerable to brute-force or
+  dictionary based attacks. This can be disabled by:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-account-lockout-attempts 0
+
+Enable a Locked User
+^^^^^^^^^^^^^^^^^^^^
+
+If a user account is disabled as a result of multiple invalid login attempts, then
+it needs to be manually enabled by the administrator. This can be done by the following
+command:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-user-enable <username>
+
+Accessing the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^
+
+You can now access the dashboard using your (JavaScript-enabled) web browser, by
+pointing it to any of the host names or IP addresses and the selected TCP port
+where a manager instance is running: e.g., ``http(s)://<$IP>:<$PORT>/``.
+
+The dashboard page displays and requests a previously defined username and
+password.
+
+.. _dashboard-enabling-object-gateway:
+
+Enabling the Object Gateway Management Frontend
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When RGW is deployed with cephadm, the RGW credentials used by the
+dashboard will be automatically configured. You can also manually force the
+credentials to be set up with:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-credentials
+
+This will create an RGW user with uid ``dashboard`` for each realm in
+the system.
+
+If you've configured a custom 'admin' resource in your RGW admin API, you should set it here also:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-api-admin-resource <admin_resource>
+
+If you are using a self-signed certificate in your Object Gateway setup,
+you should disable certificate verification in the dashboard to avoid refused
+connections, e.g. caused by certificates signed by unknown CA or not matching
+the host name:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-api-ssl-verify False
+
+If the Object Gateway takes too long to process requests and the dashboard runs
+into timeouts, you can set the timeout value to your needs:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rest-requests-timeout <seconds>
+
+The default value is 45 seconds.
+
+.. _dashboard-iscsi-management:
+
+Enabling iSCSI Management
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Ceph Dashboard can manage iSCSI targets using the REST API provided by the
+``rbd-target-api`` service of the :ref:`ceph-iscsi`. Please make sure that it is
+installed and enabled on the iSCSI gateways.
+
+.. note::
+
+  The iSCSI management functionality of Ceph Dashboard depends on the latest
+  version 3 of the `ceph-iscsi <https://github.com/ceph/ceph-iscsi>`_ project.
+  Make sure that your operating system provides the correct version, otherwise
+  the dashboard will not enable the management features.
+
+If the ``ceph-iscsi`` REST API is configured in HTTPS mode and its using a self-signed
+certificate, you need to configure the dashboard to avoid SSL certificate
+verification when accessing ceph-iscsi API.
+
+To disable API SSL verification run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-iscsi-api-ssl-verification false
+
+The available iSCSI gateways must be defined using the following commands:
+
+.. prompt:: bash $
+
+   ceph dashboard iscsi-gateway-list
+   # Gateway URL format for a new gateway: <scheme>://<username>:<password>@<host>[:port]
+   ceph dashboard iscsi-gateway-add -i <file-containing-gateway-url> [<gateway_name>]
+   ceph dashboard iscsi-gateway-rm <gateway_name>
+
+
+.. _dashboard-grafana:
+
+Enabling the Embedding of Grafana Dashboards
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+`Grafana`_ pulls data from `Prometheus <https://prometheus.io/>`_. Although
+Grafana can use other data sources, the Grafana dashboards we provide contain
+queries that are specific to Prometheus. Our Grafana dashboards therefore
+require Prometheus as the data source. The Ceph :ref:`mgr-prometheus`
+module exports its data in the Prometheus exposition format. These Grafana
+dashboards rely on metric names from the Prometheus module and `Node exporter
+<https://prometheus.io/docs/guides/node-exporter/>`_. The Node exporter is a
+separate application that provides machine metrics.
+
+.. note::
+
+  Prometheus' security model presumes that untrusted users have access to the
+  Prometheus HTTP endpoint and logs. Untrusted users have access to all the
+  (meta)data Prometheus collects that is contained in the database, plus a
+  variety of operational and debugging information.
+
+  However, Prometheus' HTTP API is limited to read-only operations.
+  Configurations can *not* be changed using the API and secrets are not
+  exposed. Moreover, Prometheus has some built-in measures to mitigate the
+  impact of denial of service attacks.
+
+  Please see `Prometheus' Security model
+  <https://prometheus.io/docs/operating/security/>` for more detailed
+  information.
+
+Installation and Configuration using cephadm
+""""""""""""""""""""""""""""""""""""""""""""
+
+Grafana and Prometheus can be installed using :ref:`cephadm`. They will
+automatically be configured by ``cephadm``. Please see
+:ref:`mgr-cephadm-monitoring` documentation for more details on how to use
+``cephadm`` for installing and configuring Prometheus and Grafana.
+
+Manual Installation and Configuration
+"""""""""""""""""""""""""""""""""""""
+
+The following process describes how to configure Grafana and Prometheus
+manually. After you have installed Prometheus, Grafana, and the Node exporter
+on appropriate hosts, proceed with the following steps.
+
+#.  Enable the Ceph Exporter which comes as Ceph Manager module by running:
+
+    .. prompt:: bash $
+
+       ceph mgr module enable prometheus
+
+    More details can be found in the documentation of the :ref:`mgr-prometheus`.
+
+#.  Add the corresponding scrape configuration to Prometheus. This may look
+    like::
+
+      global:
+        scrape_interval: 5s
+
+      scrape_configs:
+        - job_name: 'prometheus'
+          static_configs:
+            - targets: ['localhost:9090']
+        - job_name: 'ceph'
+          static_configs:
+            - targets: ['localhost:9283']
+        - job_name: 'node-exporter'
+          static_configs:
+            - targets: ['localhost:9100']
+
+    .. note::
+
+      Please note that in the above example, Prometheus is configured
+      to scrape data from itself (port 9090), the Ceph manager module
+      `prometheus` (port 9283), which exports Ceph internal data, and the Node
+      Exporter (port 9100), which provides OS and hardware metrics for each host.
+
+      Depending on your configuration, you may need to change the hostname in
+      or add additional configuration entries for the Node
+      Exporter. It is unlikely that you will need to change the default TCP ports.
+
+      Moreover, you don't *need* to have more than one target for Ceph specific
+      data, provided by the `prometheus` mgr module. But it is recommended to
+      configure Prometheus to scrape Ceph specific data from all existing Ceph
+      managers. This enables a built-in high availability mechanism, so that
+      services run on a manager host will be restarted automatically on a different
+      manager host if one Ceph Manager goes down.
+
+#. Add Prometheus as data source to Grafana `using the Grafana Web UI <https://grafana.com/docs/grafana/latest/features/datasources/add-a-data-source/>`_.
+
+   .. IMPORTANT:: 
+      The data source must be named "Dashboard1".
+
+#.  Install the `vonage-status-panel and grafana-piechart-panel` plugins using:
+
+    .. prompt:: bash $
+
+      grafana-cli plugins install vonage-status-panel
+      grafana-cli plugins install grafana-piechart-panel
+
+#.  Add Dashboards to Grafana:
+
+    Dashboards can be added to Grafana by importing dashboard JSON files.
+    Use the following command to download the JSON files:
+
+    .. prompt:: bash $
+
+       wget https://raw.githubusercontent.com/ceph/ceph/main/monitoring/ceph-mixin/dashboards_out/<Dashboard-name>.json
+
+    You can find various dashboard JSON files `here <https://github.com/ceph/ceph/tree/
+    main/monitoring/ceph-mixin/dashboards_out>`_.
+
+    For Example, for ceph-cluster overview you can use:
+
+    .. prompt:: bash $
+
+       wget https://raw.githubusercontent.com/ceph/ceph/main/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
+
+    You may also author your own dashboards.
+
+#.  Configure anonymous mode in ``/etc/grafana/grafana.ini``::
+
+      [auth.anonymous]
+      enabled = true
+      org_name = Main Org.
+      org_role = Viewer
+
+    In newer versions of Grafana (starting with 6.2.0-beta1) a new setting named
+    ``allow_embedding`` has been introduced. This setting must be explicitly
+    set to ``true`` for the Grafana integration in Ceph Dashboard to work, as the
+    default is ``false``.
+
+    ::
+
+      [security]
+      allow_embedding = true
+
+Enabling RBD-Image monitoring
+"""""""""""""""""""""""""""""
+
+Monitoring of RBD images is disabled by default, as it can significantly impact
+performance. For more information please see :ref:`prometheus-rbd-io-statistics`.
+When disabled, the overview and details dashboards will be empty in Grafana and
+metrics will not be visible in Prometheus.
+
+Configuring Dashboard
+"""""""""""""""""""""
+
+After you have set up Grafana and Prometheus, you will need to configure the
+connection information that the Ceph Dashboard will use to access Grafana.
+
+You need to tell the dashboard on which URL the Grafana instance is
+running/deployed:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-api-url <grafana-server-url>  # default: ''
+
+The format of url is : `<protocol>:<IP-address>:<port>`
+
+.. note::
+
+  The Ceph Dashboard embeds Grafana dashboards via ``iframe`` HTML elements.
+  If Grafana is configured without SSL/TLS support, most browsers will block the
+  embedding of insecure content if SSL support is
+  enabled for the dashboard (which is the default). If you
+  can't see the embedded Grafana dashboards after enabling them as outlined
+  above, check your browser's documentation on how to unblock mixed content.
+  Alternatively, consider enabling SSL/TLS support in Grafana.
+
+If you are using a self-signed certificate for Grafana,
+disable certificate verification in the dashboard to avoid refused connections,
+which can be a result of certificates signed by an unknown CA or that do not
+match the host name:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-api-ssl-verify False
+
+You can also access Grafana directly to monitor your cluster.
+
+.. note::
+
+  Ceph Dashboard configuration information can also be unset. For example, to
+  clear the Grafana API URL we configured above:
+
+  .. prompt:: bash $
+
+     ceph dashboard reset-grafana-api-url
+
+Alternative URL for Browsers
+""""""""""""""""""""""""""""
+
+The Ceph Dashboard backend requires the Grafana URL to be able to verify the
+existence of Grafana Dashboards before the frontend even loads them. Due to the
+nature of how Grafana is implemented in Ceph Dashboard, this means that two
+working connections are required in order to be able to see Grafana graphs in
+Ceph Dashboard:
+
+- The backend (Ceph Mgr module) needs to verify the existence of the requested
+  graph. If this request succeeds, it lets the frontend know that it can safely
+  access Grafana.
+- The frontend then requests the Grafana graphs directly from the user's
+  browser using an iframe. The Grafana instance is accessed directly without any
+  detour through Ceph Dashboard.
+
+Now, it might be the case that your environment makes it difficult for the
+user's browser to directly access the URL configured in Ceph Dashboard. To solve
+this issue, a separate URL can be configured which will solely be used to tell
+the frontend (the user's browser) which URL it should use to access Grafana.
+This setting won't ever be changed automatically, unlike the GRAFANA_API_URL
+which is set by :ref:`cephadm` (only if cephadm is used to deploy monitoring
+services).
+
+To change the URL that is returned to the frontend issue the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-frontend-api-url <grafana-server-url>
+
+If no value is set for that option, it will simply fall back to the value of the
+GRAFANA_API_URL option. If set, it will instruct the browser to use this URL to
+access Grafana.
+
+.. _dashboard-sso-support:
+
+Enabling Single Sign-On (SSO)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Ceph Dashboard supports external authentication of users via the
+`SAML 2.0 <https://en.wikipedia.org/wiki/SAML_2.0>`_ protocol. You need to
+first create user accounts and associate them with desired roles, as
+authorization is performed by the Dashboard. However, the authentication
+process can be performed by an existing Identity Provider (IdP).
+
+.. note::
+
+  Ceph Dashboard SSO support relies on onelogin's
+  `python-saml <https://pypi.org/project/python-saml/>`_ library.
+  Please ensure that this library is installed on your system, either by using
+  your distribution's package management or via Python's `pip` installer.
+
+To configure SSO on Ceph Dashboard, you should use the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard sso setup saml2 <ceph_dashboard_base_url> <idp_metadata> {<idp_username_attribute>} {<idp_entity_id>} {<sp_x_509_cert>} {<sp_private_key>}
+
+Parameters:
+
+* **<ceph_dashboard_base_url>**: Base URL where Ceph Dashboard is accessible (e.g., `https://cephdashboard.local`)
+* **<idp_metadata>**: URL to remote (`http://`, `https://`) or local (`file://`) path or content of the IdP metadata XML (e.g., `https://myidp/metadata`, `file:///home/myuser/metadata.xml`).
+* **<idp_username_attribute>** *(optional)*: Attribute that should be used to get the username from the authentication response. Defaults to `uid`.
+* **<idp_entity_id>** *(optional)*: Use this when more than one entity id exists on the IdP metadata.
+* **<sp_x_509_cert> / <sp_private_key>** *(optional)*: File path of the certificate that should be used by Ceph Dashboard (Service Provider) for signing and encryption (these file paths should be accessible from the active ceph-mgr instance).
+
+.. note::
+
+  The issuer value of SAML requests will follow this pattern:  **<ceph_dashboard_base_url>**/auth/saml2/metadata
+
+To display the current SAML 2.0 configuration, use the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard sso show saml2
+
+.. note::
+
+  For more information about `onelogin_settings`, please check the `onelogin documentation <https://github.com/onelogin/python-saml>`_.
+
+To disable SSO:
+
+.. prompt:: bash $
+
+   ceph dashboard sso disable
+
+To check if SSO is enabled:
+
+.. prompt:: bash $
+
+   ceph dashboard sso status
+
+To enable SSO:
+
+.. prompt:: bash $
+
+   ceph dashboard sso enable saml2
+
+.. _dashboard-alerting:
+
+Enabling Prometheus Alerting
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To use Prometheus for alerting you must define `alerting rules
+<https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules>`_.
+These are managed by the `Alertmanager
+<https://prometheus.io/docs/alerting/alertmanager>`_.
+If you are not yet using the Alertmanager, `install it
+<https://github.com/prometheus/alertmanager#install>`_ as it receives
+and manages alerts from Prometheus.
+
+Alertmanager capabilities can be consumed by the dashboard in three different
+ways:
+
+#. Use the notification receiver of the dashboard.
+
+#. Use the Prometheus Alertmanager API.
+
+#. Use both sources simultaneously.
+
+All three methods notify you about alerts. You won't be notified
+twice if you use both sources, but you need to consume at least the Alertmanager API
+in order to manage silences.
+
+1. Use the notification receiver of the dashboard
+
+  This allows you to get notifications as `configured
+  <https://prometheus.io/docs/alerting/configuration/>`_ from the Alertmanager.
+  You will get notified inside the dashboard once a notification is send out,
+  but you are not able to manage alerts.
+
+  Add the dashboard receiver and the new route to your Alertmanager
+  configuration. This should look like::
+
+    route:
+      receiver: 'ceph-dashboard'
+    ...
+    receivers:
+      - name: 'ceph-dashboard'
+        webhook_configs:
+        - url: '<url-to-dashboard>/api/prometheus_receiver'
+
+
+  Ensure that the Alertmanager considers your SSL certificate in terms
+  of the dashboard as valid. For more information about the correct
+  configuration checkout the `<http_config> documentation
+  <https://prometheus.io/docs/alerting/configuration/#%3Chttp_config%3E>`_.
+
+2. Use the API of Prometheus and the Alertmanager
+
+  This allows you to manage alerts and silences and will enable the "Active
+  Alerts", "All Alerts" as well as the "Silences" tabs in the "Monitoring"
+  section of the "Cluster" menu entry.
+
+  Alerts can be sorted by name, job, severity, state and start time.
+  Unfortunately it's not possible to know when an alert was sent out through a
+  notification by the Alertmanager based on your configuration, that's why the
+  dashboard will notify the user on any visible change to an alert and will
+  notify the changed alert.
+
+  Silences can be sorted by id, creator, status, start, updated and end time.
+  Silences can be created in various ways, it's also possible to expire them.
+
+  #. Create from scratch
+
+  #. Based on a selected alert
+
+  #. Recreate from expired silence
+
+  #. Update a silence (which will recreate and expire it (default Alertmanager behaviour))
+
+  To use it, specify the host and port of the Alertmanager server:
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-alertmanager-api-host <alertmanager-host:port>  # default: ''
+
+  For example:
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-alertmanager-api-host 'http://localhost:9093'
+
+  To be able to see all configured alerts, you will need to configure the URL to
+  the Prometheus API. Using this API, the UI will also help you in verifying
+  that a new silence will match a corresponding alert.
+
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-prometheus-api-host <prometheus-host:port>  # default: ''
+
+  For example:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-prometheus-api-host 'http://localhost:9090'
+
+  After setting up the hosts, refresh your browser's dashboard window or tab.
+
+3. Use both methods
+
+  The behaviors of both methods are configured in a way that they
+  should not disturb each other, through annoying duplicated notifications
+  may pop up.
+
+If you are using a self-signed certificate in your Prometheus or your
+Alertmanager setup, you should disable certificate verification in the
+dashboard to avoid refused connections caused by certificates signed by
+an unknown CA or that do not match the host name.
+
+- For Prometheus:
+
+.. prompt:: bash $
+
+   ceph dashboard set-prometheus-api-ssl-verify False
+
+- For Alertmanager:
+
+.. prompt:: bash $
+
+   ceph dashboard set-alertmanager-api-ssl-verify False
+
+.. _dashboard-user-role-management:
+
+User and Role Management
+------------------------
+
+Password Policy
+^^^^^^^^^^^^^^^
+
+By default the password policy feature is enabled, which includes the
+following checks:
+
+- Is the password longer than N characters?
+- Are the old and new password the same?
+
+The password policy feature can be switched on or off completely:
+
+.. prompt:: bash $
+
+    ceph dashboard set-pwd-policy-enabled <true|false>
+
+The following individual checks can also be switched on or off:
+
+.. prompt:: bash $
+
+  ceph dashboard set-pwd-policy-check-length-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-oldpwd-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-username-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-exclusion-list-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-complexity-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-sequential-chars-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-repetitive-chars-enabled <true|false>
+
+Additionally the following options are available to configure password
+policy.
+
+- Minimum password length (defaults to 8):
+
+.. prompt:: bash $
+
+   ceph dashboard set-pwd-policy-min-length <N>
+
+- Minimum password complexity (defaults to 10):
+
+  .. prompt:: bash $
+
+     ceph dashboard set-pwd-policy-min-complexity <N>
+
+  Password complexity is calculated by classifying each character in
+  the password. The complexity count starts by 0. A character is rated by
+  the following rules in the given order.
+
+  - Increase by 1 if the character is a digit.
+  - Increase by 1 if the character is a lower case ASCII character.
+  - Increase by 2 if the character is an upper case ASCII character.
+  - Increase by 3 if the character is a special character like ``!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~``.
+  - Increase by 5 if the character has not been classified by one of the previous rules.
+
+- A list of comma separated words that are not allowed to be used in a
+  password:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-pwd-policy-exclusion-list <word>[,...]
+
+
+User Accounts
+^^^^^^^^^^^^^
+
+The Ceph Dashboard supports multiple user accounts. Each user account
+consists of a username, a password (stored in encrypted form using ``bcrypt``),
+an optional name, and an optional email address.
+
+If a new user is created via the Web UI, it is possible to set an option that the
+user must assign a new password when they log in for the first time.
+
+User accounts are stored in the monitors' configuration database, and are
+available to all ``ceph-mgr`` instances.
+
+We provide a set of CLI commands to manage user accounts:
+
+- *Show User(s)*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-show [<username>]
+
+- *Create User*:
+  
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-create [--enabled] [--force-password] [--pwd_update_required] <username> -i <file-containing-password> [<rolename>] [<name>] [<email>] [<pwd_expiration_date>]
+
+  To bypass password policy checks use the `force-password` option.
+  Add the option `pwd_update_required` so that a newly created user has
+  to change their password after the first login.
+
+- *Delete User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-delete <username>
+
+- *Change Password*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-password [--force-password] <username> -i <file-containing-password>
+
+- *Change Password Hash*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-password-hash <username> -i <file-containing-password-hash>
+
+  The hash must be a bcrypt hash and salt, e.g. ``$2b$12$Pt3Vq/rDt2y9glTPSV.VFegiLkQeIpddtkhoFetNApYmIJOY8gau2``.
+  This can be used to import users from an external database.
+
+- *Modify User (name, and email)*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-info <username> <name> <email>
+
+- *Disable User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-disable <username>
+
+- *Enable User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-enable <username>
+
+User Roles and Permissions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+User accounts are associated with a set of roles that define which
+dashboard functionality can be accessed.
+
+The Dashboard functionality/modules are grouped within a *security scope*.
+Security scopes are predefined and static. The current available security
+scopes are:
+
+- **hosts**: includes all features related to the ``Hosts`` menu
+  entry.
+- **config-opt**: includes all features related to management of Ceph
+  configuration options.
+- **pool**: includes all features related to pool management.
+- **osd**: includes all features related to OSD management.
+- **monitor**: includes all features related to monitor management.
+- **rbd-image**: includes all features related to RBD image
+  management.
+- **rbd-mirroring**: includes all features related to RBD mirroring
+  management.
+- **iscsi**: includes all features related to iSCSI management.
+- **rgw**: includes all features related to RADOS Gateway (RGW) management.
+- **cephfs**: includes all features related to CephFS management.
+- **nfs-ganesha**: includes all features related to NFS Ganesha management.
+- **manager**: include all features related to Ceph Manager
+  management.
+- **log**: include all features related to Ceph logs management.
+- **grafana**: include all features related to Grafana proxy.
+- **prometheus**: include all features related to Prometheus alert management.
+- **dashboard-settings**: allows to change dashboard settings.
+
+A *role* specifies a set of mappings between a *security scope* and a set of
+*permissions*. There are four types of permissions:
+
+- **read**
+- **create**
+- **update**
+- **delete**
+
+See below for an example of a role specification, in the form of a Python dictionary::
+
+  # example of a role
+  {
+    'role': 'my_new_role',
+    'description': 'My new role',
+    'scopes_permissions': {
+      'pool': ['read', 'create'],
+      'rbd-image': ['read', 'create', 'update', 'delete']
+    }
+  }
+
+The above role dictates that a user has *read* and *create* permissions for
+features related to pool management, and has full permissions for
+features related to RBD image management.
+
+The Dashboard provides a set of predefined roles that we call
+*system roles*, which can be used right away by a fresh Ceph Dashboard
+installation.
+
+The list of system roles are:
+
+- **administrator**: allows full permissions for all security scopes.
+- **read-only**: allows *read* permission for all security scopes except
+  dashboard settings.
+- **block-manager**: allows full permissions for *rbd-image*,
+  *rbd-mirroring*, and *iscsi* scopes.
+- **rgw-manager**: allows full permissions for the *rgw* scope
+- **cluster-manager**: allows full permissions for the *hosts*, *osd*,
+  *monitor*, *manager*, and *config-opt* scopes.
+- **pool-manager**: allows full permissions for the *pool* scope.
+- **cephfs-manager**: allows full permissions for the *cephfs* scope.
+
+The list of available roles can be retrieved with the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-role-show [<rolename>]
+
+You can also use the CLI to create new roles. The available commands are the
+following:
+
+- *Create Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-create <rolename> [<description>]
+
+- *Delete Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-delete <rolename>
+
+- *Add Scope Permissions to Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-add-scope-perms <rolename> <scopename> <permission> [<permission>...]
+
+- *Delete Scope Permission from Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-del-scope-perms <rolename> <scopename>
+
+To assign roles to users, the following commands are available:
+
+- *Set User Roles*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-roles <username> <rolename> [<rolename>...]
+
+- *Add Roles To User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-add-roles <username> <rolename> [<rolename>...]
+
+- *Delete Roles from User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-del-roles <username> <rolename> [<rolename>...]
+
+
+Example of User and Custom Role Creation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this section we show a complete example of the commands that
+create a user account that can manage RBD images, view and create Ceph pools,
+and has read-only access to other scopes.
+
+1. *Create the user*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-create bob -i <file-containing-password>
+
+2. *Create role and specify scope permissions*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-role-create rbd/pool-manager
+      ceph dashboard ac-role-add-scope-perms rbd/pool-manager rbd-image read create update delete
+      ceph dashboard ac-role-add-scope-perms rbd/pool-manager pool read create
+
+3. *Associate roles to user*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-set-roles bob rbd/pool-manager read-only
+
+.. _dashboard-proxy-configuration:
+
+Proxy Configuration
+-------------------
+
+In a Ceph cluster with multiple ``ceph-mgr`` instances, only the dashboard
+running on the currently active ``ceph-mgr`` daemon will serve incoming requests.
+Connections to the dashboard's TCP port on standby ``ceph-mgr`` instances
+will receive an HTTP redirect (303) to the active manager's dashboard URL.
+This enables you to point your browser to any ``ceph-mgr`` instance in
+order to access the dashboard.
+
+If you want to establish a fixed URL to reach the dashboard or if you don't want
+to allow direct connections to the manager nodes, you could set up a proxy that
+automatically forwards incoming requests to the active ``ceph-mgr``
+instance.
+
+Configuring a URL Prefix
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are accessing the dashboard via a reverse proxy,
+you may wish to service it under a URL prefix. To get the dashboard
+to use hyperlinks that include your prefix, you can set the
+``url_prefix`` setting:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/url_prefix $PREFIX
+
+so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.
+
+Disable the redirection
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If the dashboard is behind a load-balancing proxy like `HAProxy <https://www.haproxy.org/>`_
+you might want to disable redirection to prevent situations in which
+internal (unresolvable) URLs are published to the frontend client. Use the
+following command to get the dashboard to respond with an HTTP error (500 by default)
+instead of redirecting to the active dashboard:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_behaviour "error"
+
+To reset the setting to default redirection, use the following command:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_behaviour "redirect"
+
+Configure the error status code
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When redirection is disabled, you may want to customize the HTTP status
+code of standby dashboards. To do so you need to run the command:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_error_status_code 503
+
+Resolve IP address to hostname before redirect
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The redirect from a standby to the active dashboard is done via the IP
+address. This is done because resolving IP addresses to hostnames can be error
+prone in containerized environments. It is also the reason why the option is
+disabled by default.
+However, in some situations it might be helpful to redirect via the hostname.
+For example if the configured TLS certificate matches only the hostnames. To
+activate the redirection via the hostname run the following command::
+
+  $ ceph config set mgr mgr/dashboard/redirect_resolve_ip_addr True
+
+You can disable it again by::
+
+  $ ceph config set mgr mgr/dashboard/redirect_resolve_ip_addr False
+
+HAProxy example configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Below you will find an example configuration for SSL/TLS passthrough using
+`HAProxy <https://www.haproxy.org/>`_.
+
+Please note that this configuration works under the following conditions.
+If the dashboard fails over, the front-end client might receive a HTTP redirect
+(303) response and will be redirected to an unresolvable host. This happens when
+failover occurs between two HAProxy health checks. In this situation the
+previously active dashboard node will now respond with a 303 which points to
+the new active node. To prevent that situation you should consider disabling
+redirection on standby nodes.
+
+::
+
+  defaults
+    log global
+    option log-health-checks
+    timeout connect 5s
+    timeout client 50s
+    timeout server 450s
+
+  frontend dashboard_front
+    mode http
+    bind *:80
+    option httplog
+    redirect scheme https code 301 if !{ ssl_fc }
+
+  frontend dashboard_front_ssl
+    mode tcp
+    bind *:443
+    option tcplog
+    default_backend dashboard_back_ssl
+
+  backend dashboard_back_ssl
+    mode tcp
+    option httpchk GET /
+    http-check expect status 200
+    server x <HOST>:<PORT> ssl check verify none
+    server y <HOST>:<PORT> ssl check verify none
+    server z <HOST>:<PORT> ssl check verify none
+
+.. _dashboard-auditing:
+
+Auditing API Requests
+---------------------
+
+The REST API can log PUT, POST and DELETE requests to the Ceph
+audit log. This feature is disabled by default, but can be enabled with the
+following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-audit-api-enabled <true|false>
+
+If enabled, the following parameters are logged per each request:
+
+* from - The origin of the request, e.g. https://[::1]:44410
+* path - The REST API path, e.g. /api/auth
+* method - e.g. PUT, POST or DELETE
+* user - The name of the user, otherwise 'None'
+
+The logging of the request payload (the arguments and their values) is enabled
+by default. Execute the following command to disable this behaviour:
+
+.. prompt:: bash $
+
+   ceph dashboard set-audit-api-log-payload <true|false>
+
+A log entry may look like this::
+
+  2018-10-22 15:27:01.302514 mgr.x [INF] [DASHBOARD] from='https://[::ffff:127.0.0.1]:37022' path='/api/rgw/user/klaus' method='PUT' user='admin' params='{"max_buckets": "1000", "display_name": "Klaus Mustermann", "uid": "klaus", "suspended": "0", "email": "klaus.mustermann@ceph.com"}'
+
+.. _dashboard-nfs-ganesha-management:
+
+NFS-Ganesha Management
+----------------------
+
+The dashboard requires enabling the NFS module which will be used to manage
+NFS clusters and NFS exports. For more information check :ref:`mgr-nfs`.
+
+Plug-ins
+--------
+
+Plug-ins extend the functionality of the Ceph Dashboard in a modular
+and loosely coupled fashion.
+
+.. _Grafana: https://grafana.com/
+
+.. include:: dashboard_plugins/feature_toggles.inc.rst
+.. include:: dashboard_plugins/debug.inc.rst
+.. include:: dashboard_plugins/motd.inc.rst
+
+
+Troubleshooting the Dashboard
+-----------------------------
+
+Locating the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unsure of the location of the Ceph Dashboard, run the following command:
+
+.. prompt:: bash $
+
+   ceph mgr services | jq .dashboard
+
+::
+
+    "https://host:port"
+
+The command returns the URL where the Ceph Dashboard is located: ``https://<host>:<port>/``
+
+.. note::
+
+    Many Ceph tools return results in JSON format. We suggest that
+    you install the `jq <https://stedolan.github.io/jq>`_ command-line
+    utility to facilitate working with JSON data.
+
+
+Accessing the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unable to access the Ceph Dashboard, run the following
+commands:
+
+#. Verify the Ceph Dashboard module is enabled:
+
+   .. prompt:: bash $
+
+      ceph mgr module ls | jq .enabled_modules
+
+   Ensure the Ceph Dashboard module is listed in the return value of the
+   command. Example snipped output from the command above::
+
+    [
+      "dashboard",
+      "iostat",
+      "restful"
+    ]
+
+#. If it is not listed, activate the module with the following command:
+
+   .. prompt:: bash $
+
+      ceph mgr module enable dashboard
+
+#. Check the Ceph Dashboard and/or ``ceph-mgr`` log files for any errors.
+
+   * Check if ``ceph-mgr`` log messages are written to a file by:
+     
+     .. prompt:: bash $
+
+        ceph config get mgr log_to_file
+
+     ::
+
+        true
+
+   * Get the location of the log file (it's ``/var/log/ceph/<cluster-name>-<daemon-name>.log``
+     by default):
+
+     .. prompt:: bash $
+
+        ceph config get mgr log_file
+
+     ::
+
+        /var/log/ceph/$cluster-$name.log
+
+#. Ensure the SSL/TSL support is configured properly:
+
+   * Check if the SSL/TSL support is enabled:
+
+     .. prompt:: bash $
+
+        ceph config get mgr mgr/dashboard/ssl
+
+   * If the command returns ``true``, verify a certificate exists by:
+
+     .. prompt:: bash $
+
+        ceph config-key get mgr/dashboard/crt
+
+     and:
+
+     .. prompt:: bash $
+
+        ceph config-key get mgr/dashboard/key
+
+   * If it doesn't return ``true``, run the following command to generate a self-signed
+     certificate or follow the instructions outlined in
+     :ref:`dashboard-ssl-tls-support`:
+
+     .. prompt:: bash $
+
+        ceph dashboard create-self-signed-cert
+
+
+Trouble Logging into the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unable to log into the Ceph Dashboard and you receive the following
+error, run through the procedural checks below:
+
+.. image:: ../images/dashboard/invalid-credentials.png
+   :align: center
+
+#. Check that your user credentials are correct. If you are seeing the
+   notification message above when trying to log into the Ceph Dashboard, it
+   is likely you are using the wrong credentials. Double check your username
+   and password, and ensure that your keyboard's caps lock is not enabled by accident.
+
+#. If your user credentials are correct, but you are experiencing the same
+   error, check that the user account exists:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-show <username>
+
+   This command returns your user data. If the user does not exist, it will
+   print::
+
+      Error ENOENT: User <username> does not exist
+
+#. Check if the user is enabled:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-show <username> | jq .enabled
+
+   ::
+
+    true
+
+   Check if ``enabled`` is set to ``true`` for your user. If not the user is
+   not enabled, run:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-enable <username>
+
+Please see :ref:`dashboard-user-role-management` for more information.
+
+
+A Dashboard Feature is Not Working
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When an error occurs on the backend, you will usually receive an error
+notification on the frontend. Run through the following scenarios to debug.
+
+#. Check the Ceph Dashboard and ``ceph-mgr`` logfile(s) for any errors. These can
+   found by searching for keywords, such as *500 Internal Server Error*,
+   followed by ``traceback``. The end of a traceback contains more details about
+   what exact error occurred.
+#. Check your web browser's JavaScript Console for any errors.
+
+
+Ceph Dashboard Logs
+^^^^^^^^^^^^^^^^^^^
+
+Dashboard Debug Flag
+""""""""""""""""""""
+
+With this flag enabled, error traceback is included in backend responses.
+
+To enable this flag via the Ceph Dashboard, navigate from *Cluster* to *Manager
+modules*. Select *Dashboard module* and click the edit button. Click the
+*debug* checkbox and update.
+
+To enable it via the CLI, run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug enable
+
+
+Setting Logging Level of Dashboard Module
+"""""""""""""""""""""""""""""""""""""""""
+
+Setting the logging level to debug makes the log more verbose and helpful for
+debugging.
+
+#. Increase the logging level of manager daemons:
+
+   .. prompt:: bash $
+
+      ceph tell mgr config set debug_mgr 20
+
+#. Adjust the logging level of the Ceph Dashboard module via the Dashboard or
+   CLI:
+
+   * Navigate from *Cluster* to *Manager modules*. Select *Dashboard module*
+     and click the edit button. Modify the ``log_level`` configuration.
+   * To adjust it via the CLI, run the following command:
+
+     .. prompt:: bash $
+
+        bin/ceph config set mgr mgr/dashboard/log_level debug
+
+3. High log levels can result in considerable log volume, which can
+easily fill up your filesystem. Set a calendar reminder for an hour, a day,
+or a week in the future to revert this temporary logging increase.  This looks
+something like this:
+
+   .. prompt:: bash $
+
+      ceph config log
+
+   ::
+
+      ...
+      --- 11 --- 2020-11-07 11:11:11.960659 --- mgr.x/dashboard/log_level = debug ---
+      ...
+    
+   .. prompt:: bash $
+
+      ceph config reset 11
+
+.. _centralized-logging:
+
+Enable Centralized Logging in Dashboard
+"""""""""""""""""""""""""""""""""""""""
+
+To learn more about centralized logging, see :ref:`cephadm-monitoring-centralized-logs`
+
+1. Create the Loki service on any particular host using "Create Services" option.
+
+2. Similarly create the Promtail service which will be by default deployed 
+   on all the running hosts.
+
+3. To see debug-level messages as well as info-level events, run the following command via CLI:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/cephadm/log_to_cluster_level debug
+
+4. To enable logging to files, run the following commands via CLI:
+
+   .. prompt:: bash $
+
+      ceph config set global log_to_file true
+      ceph config set global mon_cluster_log_to_file true
+
+5. Click on the Daemon Logs tab under Cluster -> Logs.
+
+6. You can find some pre-defined labels there on clicking the Log browser button such as filename,
+   job etc that can help you query the logs at one go.
+
+7. You can query the logs with LogQL for advanced search and perform some
+   calculations as well - https://grafana.com/docs/loki/latest/logql/.
+
+
+Reporting issues from Dashboard
+"""""""""""""""""""""""""""""""
+
+Ceph-Dashboard provides two ways to create an issue in the Ceph Issue Tracker,
+either using the Ceph command line interface or by using the Ceph Dashboard
+user interface.
+
+To create an issue in the Ceph Issue Tracker, a user needs to have an account
+on the issue tracker. Under the ``my account`` tab in the Ceph Issue Tracker,
+the user can see their API access key. This key is used for authentication
+when creating a new issue. To store the Ceph API access key, in the CLI run:
+
+.. prompt:: bash $
+
+   ``ceph dashboard set-issue-tracker-api-key -i <file-containing-key>``
+
+Then on successful update, you can create an issue using:
+
+.. prompt:: bash $
+
+   ``ceph dashboard create issue <project> <tracker_type> <subject> <description>``
+
+The available projects to create an issue on are:
+#. dashboard
+#. block
+#. object
+#. file_system
+#. ceph_manager
+#. orchestrator
+#. ceph_volume
+#. core_ceph
+
+The available tracker types are:
+#. bug
+#. feature
+
+The subject and description are then set by the user.
+
+The user can also create an issue using the Dashboard user interface. The settings
+icon drop down menu on the top right of the navigation bar has the option to
+``Raise an issue``. On clicking it, a modal dialog opens that has the option to
+select the project and tracker from their respective drop down menus. The subject
+and multiline description are added by the user. The user can then submit the issue.
diff --git a/doc/mgr/dashboard_plugins/debug.inc.rst b/doc/mgr/dashboard_plugins/debug.inc.rst
new file mode 100644
index 000000000..883419cbf
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/debug.inc.rst
@@ -0,0 +1,43 @@
+.. _dashboard-debug:
+
+Debug
+^^^^^
+
+This plugin allows to customize the behaviour of the dashboard according to the
+debug mode. It can be enabled, disabled or checked with the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug status
+
+::
+
+  Debug: 'disabled'
+
+.. prompt:: bash $
+
+   ceph dashboard debug enable
+
+::
+
+  Debug: 'enabled'
+
+.. prompt:: bash $
+
+   ceph dashboard debug disable
+
+::
+
+  Debug: 'disabled'
+
+By default, it's disabled. This is the recommended setting for production
+deployments. If required, debug mode can be enabled without need of restarting.
+Currently, disabled debug mode equals to CherryPy ``production`` environment,
+while when enabled, it uses ``test_suite`` defaults (please refer to
+`CherryPy Environments
+<https://docs.cherrypy.org/en/latest/config.html#environments>`_ for more
+details).
+
+It also adds request uuid (``unique_id``) to Cherrypy on versions that don't
+support this. It additionally prints the ``unique_id`` to error responses and
+log messages.
diff --git a/doc/mgr/dashboard_plugins/feature_toggles.inc.rst b/doc/mgr/dashboard_plugins/feature_toggles.inc.rst
new file mode 100644
index 000000000..7c96b0faa
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/feature_toggles.inc.rst
@@ -0,0 +1,56 @@
+.. _dashboard-feature-toggles:
+
+Feature Toggles
+^^^^^^^^^^^^^^^
+
+This plug-in allows to enable or disable some features from the Ceph Dashboard
+on-demand. When a feature becomes disabled:
+
+- Its front-end elements (web pages, menu entries, charts, etc.) will become hidden.
+- Its associated REST API endpoints will reject any further requests (404, Not Found Error).
+
+The main purpose of this plug-in is to allow ad-hoc customizations of the workflows exposed
+by the dashboard. Additionally, it could allow for dynamically enabling experimental
+features with minimal configuration burden and no service impact.
+
+The list of features that can be enabled/disabled is:
+
+- **Block (RBD)**:
+   - Image Management: ``rbd``
+   - Mirroring: ``mirroring``
+   - iSCSI: ``iscsi``
+- **Filesystem (Cephfs)**: ``cephfs``
+- **Objects (RGW)**: ``rgw`` (including daemon, user and bucket management).
+- **NFS**: ``nfs-ganesha`` exports.
+
+By default all features come enabled.
+
+To retrieve a list of features and their current statuses:
+
+.. prompt:: bash $
+
+   ceph dashboard feature status
+
+::
+
+  Feature 'cephfs': 'enabled'
+  Feature 'iscsi': 'enabled'
+  Feature 'mirroring': 'enabled'
+  Feature 'rbd': 'enabled'
+  Feature 'rgw': 'enabled'
+  Feature 'nfs': 'enabled'
+
+To enable or disable the status of a single or multiple features:
+
+.. prompt:: bash $
+
+   ceph dashboard feature disable iscsi mirroring
+
+:: 
+
+  Feature 'iscsi': disabled
+  Feature 'mirroring': disabled
+
+After a feature status has changed, the API REST endpoints immediately respond to
+that change, while for the front-end UI elements, it may take up to 20 seconds to
+reflect it.
diff --git a/doc/mgr/dashboard_plugins/motd.inc.rst b/doc/mgr/dashboard_plugins/motd.inc.rst
new file mode 100644
index 000000000..0f9cc199a
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/motd.inc.rst
@@ -0,0 +1,36 @@
+.. _dashboard-motd:
+
+Message of the day (MOTD)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Displays a configured `message of the day` at the top of the Ceph Dashboard.
+
+The importance of a MOTD can be configured by its severity, which is
+`info`, `warning` or `danger`. The MOTD can expire after a given time,
+this means it will not be displayed in the UI anymore. Use the following
+syntax to specify the expiration time: `Ns|m|h|d|w` for seconds, minutes,
+hours, days and weeks. If the MOTD should expire after 2 hours, use `2h`
+or `5w` for 5 weeks. Use `0` to configure a MOTD that does not expire.
+
+To configure a MOTD, run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard motd set <severity:info|warning|danger> <expires> <message>
+
+To show the configured MOTD:
+
+.. prompt:: bash $
+
+   ceph dashboard motd get
+
+To clear the configured MOTD run:
+
+.. prompt:: bash $
+
+   ceph dashboard motd clear
+
+A MOTD with a `info` or `warning` severity can be closed by the user. The
+`info` MOTD is not displayed anymore until the local storage cookies are
+cleared or a new MOTD with a different severity is displayed. A MOTD with
+a 'warning' severity will be displayed again in a new session.
diff --git a/doc/mgr/details-card.png b/doc/mgr/details-card.png
new file mode 100644
index 000000000..0c219a890
--- /dev/null
+++ b/doc/mgr/details-card.png
diff --git a/doc/mgr/diskprediction.rst b/doc/mgr/diskprediction.rst
new file mode 100644
index 000000000..f4b697511
--- /dev/null
+++ b/doc/mgr/diskprediction.rst
@@ -0,0 +1,59 @@
+.. _diskprediction:
+
+=====================
+Diskprediction Module
+=====================
+
+The *diskprediction* module leverages Ceph device health check to collect disk health metrics and uses internal predictor module to produce the disk failure prediction and returns back to Ceph. It doesn't require any external server for data analysis and output results. Its internal predictor's accuracy is around 70%.
+
+Enabling
+========
+
+Run the following command to enable the *diskprediction_local* module in the Ceph
+environment::
+
+    ceph mgr module enable diskprediction_local
+
+
+To enable the local predictor::
+
+    ceph config set global device_failure_prediction_mode local
+
+To disable prediction::
+
+    ceph config set global device_failure_prediction_mode none
+
+
+*diskprediction_local* requires at least six datasets of device health metrics to
+make prediction of the devices' life expectancy. And these health metrics are
+collected only if health monitoring is :ref:`enabled <enabling-monitoring>`.
+
+Run the following command to retrieve the life expectancy of given device.
+
+::
+
+    ceph device predict-life-expectancy <device id>
+
+Configuration
+=============
+
+The module performs the prediction on a daily basis by default. You can adjust
+this interval with::
+
+  ceph config set mgr mgr/diskprediction_local/predict_interval <interval-in-seconds>
+
+Debugging
+=========
+
+If you want to debug the DiskPrediction module mapping to Ceph logging level,
+use the following command.
+
+::
+
+    [mgr]
+
+        debug mgr = 20
+
+With logging set to debug for the manager the module will print out logging
+message with prefix *mgr[diskprediction]* for easy filtering.
+
diff --git a/doc/mgr/hello.rst b/doc/mgr/hello.rst
new file mode 100644
index 000000000..725355fc9
--- /dev/null
+++ b/doc/mgr/hello.rst
@@ -0,0 +1,39 @@
+Hello World Module
+==================
+
+This is a simple module skeleton for documentation purposes.
+
+Enabling
+--------
+
+The *hello* module is enabled with::
+
+  ceph mgr module enable hello
+
+To check that it is enabled, run::
+
+  ceph mgr module ls
+
+After editing the module file (found in ``src/pybind/mgr/hello/module.py``), you can see changes by running::
+
+  ceph mgr module disable hello
+  ceph mgr module enable hello
+
+or::
+
+  init-ceph restart mgr
+
+To execute the module, run::
+
+  ceph hello
+
+The log is found at::
+
+  build/out/mgr.x.log
+
+
+Documenting
+-----------
+
+After adding a new mgr module, be sure to add its documentation to ``doc/mgr/module_name.rst``.
+Also, add a link to your new module into ``doc/mgr/index.rst``.
diff --git a/doc/mgr/index.rst b/doc/mgr/index.rst
new file mode 100644
index 000000000..4d20d5098
--- /dev/null
+++ b/doc/mgr/index.rst
@@ -0,0 +1,52 @@
+.. _ceph-manager-daemon:
+
+===================
+Ceph Manager Daemon
+===================
+
+The :term:`Ceph Manager` daemon (ceph-mgr) runs alongside monitor daemons,
+to provide additional monitoring and interfaces to external monitoring
+and management systems.
+
+Since the 12.x (*luminous*) Ceph release, the ceph-mgr daemon is required for
+normal operations.  The ceph-mgr daemon is an optional component in
+the 11.x (*kraken*) Ceph release.
+
+By default, the manager daemon requires no additional configuration, beyond
+ensuring it is running.  If there is no mgr daemon running, you will
+see a health warning to that effect, and some of the other information
+in the output of `ceph status` will be missing or stale until a mgr is started.
+
+Use your normal deployment tools, such as ceph-ansible or cephadm, to
+set up ceph-mgr daemons on each of your mon nodes.  It is not mandatory
+to place mgr daemons on the same nodes as mons, but it is almost always
+sensible.
+
+.. toctree::
+    :maxdepth: 1
+
+    Installation and Configuration <administrator>
+    Writing modules <modules>
+    Writing orchestrator plugins <orchestrator_modules>
+    Dashboard module <dashboard>
+    Ceph RESTful API <ceph_api/index>
+    Alerts module <alerts>
+    DiskPrediction module <diskprediction>
+    Local pool module <localpool>
+    RESTful module <restful>
+    Zabbix module <zabbix>
+    Prometheus module <prometheus>
+    Influx module <influx>
+    Hello module <hello>
+    Telegraf module <telegraf>
+    Telemetry module <telemetry>
+    Iostat module <iostat>
+    Crash module <crash>
+    Insights module <insights>
+    Orchestrator module <orchestrator>
+    Rook module <rook>
+    RGW module <rgw>
+    MDS Autoscaler module <mds_autoscaler>
+    NFS module <nfs>
+    Progress Module <progress>
+    CLI API Commands module <cli_api>
diff --git a/doc/mgr/influx.rst b/doc/mgr/influx.rst
new file mode 100644
index 000000000..2622d3919
--- /dev/null
+++ b/doc/mgr/influx.rst
@@ -0,0 +1,173 @@
+=============
+Influx Module 
+=============
+
+.. mgr_module:: influx
+
+The influx module continuously collects and sends time series data to an
+influxdb database.
+
+The influx module was introduced in the 13.x *Mimic* release.
+
+--------
+Enabling 
+--------
+
+To enable the module, use the following command:
+
+.. prompt:: bash $
+
+    ceph mgr module enable influx
+
+If you wish to subsequently disable the module, you can use the equivalent
+*disable* command:
+
+.. prompt:: bash $
+
+    ceph mgr module disable influx
+
+-------------
+Configuration 
+-------------
+
+For the influx module to send statistics to an InfluxDB server, it
+is necessary to configure the servers address and some authentication
+credentials.
+
+Set configuration values using the following command:
+
+.. prompt:: bash $
+
+    ceph config set mgr mgr/influx/<key> <value>
+
+
+The most important settings are :confval:`mgr/influx/hostname`,
+:confval:`mgr/influx/username` and :confval:`mgr/influx/password`.
+For example, a typical configuration might look like this:
+
+.. prompt:: bash $
+
+    ceph config set mgr mgr/influx/hostname influx.mydomain.com
+    ceph config set mgr mgr/influx/username admin123
+    ceph config set mgr mgr/influx/password p4ssw0rd
+    
+Following is the list of all configuration settings:
+
+.. confval:: hostname
+.. confval:: username
+.. confval:: password
+.. confval:: interval
+.. confval:: database
+.. confval:: port
+.. confval:: ssl
+.. confval:: verify_ssl
+.. confval:: threads
+.. confval:: batch_size
+
+---------
+Debugging 
+---------
+
+By default, a few debugging statements as well as error statements have been set to print in the log files. Users can add more if necessary.
+To make use of the debugging option in the module:
+
+- Add this to the ceph.conf file.
+
+  .. code-block:: ini
+
+     [mgr]
+        debug_mgr = 20  
+
+- Use this command ``ceph influx self-test``.
+- Check the log files. Users may find it easier to filter the log files using *mgr[influx]*.
+
+--------------------
+Interesting counters
+--------------------
+
+The following tables describe a subset of the values output by
+this module.
+
+^^^^^
+Pools
+^^^^^
+
++---------------+-----------------------------------------------------+
+|Counter        | Description                                         |
++===============+=====================================================+
+|stored         | Bytes stored in the pool not including copies       |
++---------------+-----------------------------------------------------+
+|max_avail      | Max available number of bytes in the pool           |
++---------------+-----------------------------------------------------+
+|objects        | Number of objects in the pool                       |
++---------------+-----------------------------------------------------+
+|wr_bytes       | Number of bytes written in the pool                 |
++---------------+-----------------------------------------------------+
+|dirty          | Number of bytes dirty in the pool                   |
++---------------+-----------------------------------------------------+
+|rd_bytes       | Number of bytes read in the pool                    |
++---------------+-----------------------------------------------------+
+|stored_raw     | Bytes used in pool including copies made            |
++---------------+-----------------------------------------------------+
+
+^^^^
+OSDs
+^^^^
+
++------------+------------------------------------+
+|Counter     | Description                        |
++============+====================================+
+|op_w        | Client write operations            |
++------------+------------------------------------+
+|op_in_bytes | Client operations total write size |
++------------+------------------------------------+
+|op_r        | Client read operations             |
++------------+------------------------------------+
+|op_out_bytes| Client operations total read size  |
++------------+------------------------------------+
+
+
++------------------------+--------------------------------------------------------------------------+
+|Counter                 | Description                                                              |
++========================+==========================================================================+
+|op_wip                  | Replication operations currently being processed (primary)               |
++------------------------+--------------------------------------------------------------------------+
+|op_latency              | Latency of client operations (including queue time)                      |
++------------------------+--------------------------------------------------------------------------+
+|op_process_latency      | Latency of client operations (excluding queue time)                      |           
++------------------------+--------------------------------------------------------------------------+
+|op_prepare_latency      | Latency of client operations (excluding queue time and wait for finished)|
++------------------------+--------------------------------------------------------------------------+
+|op_r_latency            | Latency of read operation (including queue time)                         |
++------------------------+--------------------------------------------------------------------------+
+|op_r_process_latency    | Latency of read operation (excluding queue time)                         |
++------------------------+--------------------------------------------------------------------------+
+|op_w_in_bytes           | Client data written                                                      |
++------------------------+--------------------------------------------------------------------------+
+|op_w_latency            | Latency of write operation (including queue time)                        |
++------------------------+--------------------------------------------------------------------------+
+|op_w_process_latency    | Latency of write operation (excluding queue time)                        |
++------------------------+--------------------------------------------------------------------------+
+|op_w_prepare_latency    | Latency of write operations (excluding queue time and wait for finished) |
++------------------------+--------------------------------------------------------------------------+
+|op_rw                   | Client read-modify-write operations                                      |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_in_bytes          | Client read-modify-write operations write in                             |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_out_bytes         | Client read-modify-write operations read out                             |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_latency           | Latency of read-modify-write operation (including queue time)            |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_process_latency   | Latency of read-modify-write operation (excluding queue time)            |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_prepare_latency   | Latency of read-modify-write operations (excluding queue time            |
+|                        | and wait for finished)                                                   |
++------------------------+--------------------------------------------------------------------------+
+|op_before_queue_op_lat  | Latency of IO before calling queue (before really queue into ShardedOpWq)|
+|                        | op_before_dequeue_op_lat                                                 |
++------------------------+--------------------------------------------------------------------------+
+|op_before_dequeue_op_lat| Latency of IO before calling dequeue_op(already dequeued and get PG lock)|
++------------------------+--------------------------------------------------------------------------+
+
+Latency counters are measured in microseconds unless otherwise specified in the description.
+
diff --git a/doc/mgr/insights.rst b/doc/mgr/insights.rst
new file mode 100644
index 000000000..37b8903f1
--- /dev/null
+++ b/doc/mgr/insights.rst
@@ -0,0 +1,52 @@
+Insights Module
+===============
+
+The insights module collects and exposes system information to the Insights Core
+data analysis framework. It is intended to replace explicit interrogation of
+Ceph CLIs and daemon admin sockets, reducing the API surface that Insights
+depends on. The insights reports contains the following:
+
+* **Health reports**. In addition to reporting the current health of the
+  cluster, the insights module reports a summary of the last 24 hours of health
+  checks. This feature is important for catching cluster health issues that are
+  transient and may not be present at the moment the report is generated. Health
+  checks are deduplicated to avoid unbounded data growth.
+
+* **Crash reports**. A summary of any daemon crashes in the past 24 hours is
+  included in the insights report. Crashes are reported as the number of crashes
+  per daemon type (e.g. `ceph-osd`) within the time window. Full details of a
+  crash may be obtained using the `crash module`_.
+
+* Software version, storage utilization, cluster maps, placement group summary,
+  monitor status, cluster configuration, and OSD metadata.
+
+Enabling
+--------
+
+The *insights* module is enabled with::
+
+  ceph mgr module enable insights
+
+Commands
+--------
+::
+
+  ceph insights
+
+Generate the full report.
+
+::
+
+  ceph insights prune-health <hours>
+
+Remove historical health data older than <hours>. Passing `0` for <hours> will
+clear all health data.
+
+This command is useful for cleaning the health history before automated nightly
+reports are generated, which may contain spurious health checks accumulated
+while performing system maintenance, or other health checks that have been
+resolved. There is no need to prune health data to reclaim storage space;
+garbage collection is performed regularly to remove old health data from
+persistent storage.
+
+.. _crash module: ../crash
diff --git a/doc/mgr/inventory-card.png b/doc/mgr/inventory-card.png
new file mode 100644
index 000000000..54317fc9f
--- /dev/null
+++ b/doc/mgr/inventory-card.png
diff --git a/doc/mgr/iostat.rst b/doc/mgr/iostat.rst
new file mode 100644
index 000000000..f9f849383
--- /dev/null
+++ b/doc/mgr/iostat.rst
@@ -0,0 +1,32 @@
+.. _mgr-iostat-overview:
+
+iostat
+======
+
+This module shows the current throughput and IOPS done on the Ceph cluster.
+
+Enabling
+--------
+
+To check if the *iostat* module is enabled, run::
+
+  ceph mgr module ls
+
+The module can be enabled with::
+
+  ceph mgr module enable iostat
+
+To execute the module, run::
+
+  ceph iostat
+
+To change the frequency at which the statistics are printed, use the ``-p``
+option::
+
+  ceph iostat -p <period in seconds>
+
+For example, use the following command to print the statistics every 5 seconds::
+
+  ceph iostat -p 5
+
+To stop the module, press Ctrl-C.
diff --git a/doc/mgr/localpool.rst b/doc/mgr/localpool.rst
new file mode 100644
index 000000000..2812925ca
--- /dev/null
+++ b/doc/mgr/localpool.rst
@@ -0,0 +1,39 @@
+Local Pool Module
+=================
+
+.. mgr_module:: localpool
+
+The *localpool* module can automatically create RADOS pools that are
+localized to a subset of the overall cluster.  For example, by default, it will
+create a pool for each distinct ``rack`` in the cluster.  This can be useful for
+deployments where it is desirable to distribute some data locally and other data
+globally across the cluster.  One use-case is measuring performance and testing
+behavior of specific drive, NIC, or chassis models in isolation.
+
+Enabling
+--------
+
+The *localpool* module is enabled with::
+
+  ceph mgr module enable localpool
+
+Configuring
+-----------
+
+The *localpool* module understands the following options:
+
+.. confval:: subtree
+.. confval:: failure_domain
+.. confval:: pg_num
+.. confval:: num_rep
+.. confval:: min_size
+.. confval:: prefix
+   :default: by-$subtreetype-
+
+These options are set via the config-key interface.  For example, to
+change the replication level to 2x with only 64 PGs, ::
+
+  ceph config set mgr mgr/localpool/num_rep 2
+  ceph config set mgr mgr/localpool/pg_num 64
+
+.. mgr_module:: None
diff --git a/doc/mgr/mds_autoscaler.rst b/doc/mgr/mds_autoscaler.rst
new file mode 100644
index 000000000..46fc44155
--- /dev/null
+++ b/doc/mgr/mds_autoscaler.rst
@@ -0,0 +1,23 @@
+MDS Autoscaler Module
+=====================
+
+The MDS Autoscaler Module monitors file systems to ensure sufficient MDS
+daemons are available. It works by adjusting the placement specification for
+the orchestrator backend of the MDS service. To enable, use:
+
+.. sh:
+
+   ceph mgr module enable mds_autoscaler
+
+The module will monitor the following file system settings to inform
+placement count adjustments:
+
+- ``max_mds`` file system setting
+- ``standby_count_wanted`` file system setting
+
+The Ceph monitor daemons are still responsible for promoting or stopping MDS
+according to these settings. The ``mds_autoscaler`` simply adjusts the
+number of MDS which are spawned by the orchestrator.
+
+.. note: There is no CLI or module configurations as of now. Enable or disable
+   the module to turn on or off.
diff --git a/doc/mgr/modules.rst b/doc/mgr/modules.rst
new file mode 100644
index 000000000..667664139
--- /dev/null
+++ b/doc/mgr/modules.rst
@@ -0,0 +1,735 @@
+
+
+.. _mgr-module-dev:
+
+ceph-mgr module developer's guide
+=================================
+
+.. warning::
+
+    This is developer documentation, describing Ceph internals that
+    are only relevant to people writing ceph-mgr modules.
+
+Creating a module
+-----------------
+
+In pybind/mgr/, create a python module.  Within your module, create a class
+that inherits from ``MgrModule``.  For ceph-mgr to detect your module, your
+directory must contain a file called `module.py`.
+
+The most important methods to override are:
+
+* a ``serve`` member function for server-type modules.  This
+  function should block forever.
+* a ``notify`` member function if your module needs to
+  take action when new cluster data is available.
+* a ``handle_command`` member function if your module
+  exposes CLI commands. But this approach for exposing commands
+  is deprecated. For more details, see :ref:`mgr-module-exposing-commands`.
+
+Some modules interface with external orchestrators to deploy
+Ceph services.  These also inherit from ``Orchestrator``, which adds
+additional methods to the base ``MgrModule`` class.  See
+:ref:`Orchestrator modules <orchestrator-modules>` for more on
+creating these modules.
+
+Installing a module
+-------------------
+
+Once your module is present in the location set by the
+``mgr module path`` configuration setting, you can enable it
+via the ``ceph mgr module enable`` command::
+
+  ceph mgr module enable mymodule
+
+Note that the MgrModule interface is not stable, so any modules maintained
+outside of the Ceph tree are liable to break when run against any newer
+or older versions of Ceph.
+
+.. _mgr module dev logging:
+
+Logging
+-------
+
+Logging in Ceph manager modules is done as in any other Python program. Just
+import the ``logging`` package and get a logger instance with the
+``logging.getLogger`` function.
+
+Each module has a ``log_level`` option that specifies the current Python
+logging level of the module.
+To change or query the logging level of the module use the following Ceph
+commands::
+
+  ceph config get mgr mgr/<module_name>/log_level
+  ceph config set mgr mgr/<module_name>/log_level <info|debug|critical|error|warning|>
+
+The logging level used upon the module's start is determined by the current
+logging level of the mgr daemon, unless if the ``log_level`` option was
+previously set with the ``config set ...`` command. The mgr daemon logging
+level is mapped to the module python logging level as follows:
+
+* <= 0 is CRITICAL
+* <= 1 is WARNING
+* <= 4 is INFO
+* <= +inf is DEBUG
+
+We can unset the module log level and fallback to the mgr daemon logging level
+by running the following command::
+
+  ceph config set mgr mgr/<module_name>/log_level ''
+
+By default, modules' logging messages are processed by the Ceph logging layer
+where they will be recorded in the mgr daemon's log file.
+But it's also possible to send a module's logging message to it's own file.
+
+The module's log file will be located in the same directory as the mgr daemon's
+log file with the following name pattern::
+
+   <mgr_daemon_log_file_name>.<module_name>.log
+
+To enable the file logging on a module use the following command::
+
+   ceph config set mgr mgr/<module_name>/log_to_file true
+
+When the module's file logging is enabled, module's logging messages stop
+being written to the mgr daemon's log file and are only written to the
+module's log file.
+
+It's also possible to check the status and disable the file logging with the
+following commands::
+
+   ceph config get mgr mgr/<module_name>/log_to_file
+   ceph config set mgr mgr/<module_name>/log_to_file false
+
+
+
+.. _mgr-module-exposing-commands:
+
+Exposing commands
+-----------------
+
+There are two approaches for exposing a command. The first method involves using
+the ``@CLICommand`` decorator to decorate the methods needed to handle a command.
+The second method uses a ``COMMANDS`` attribute defined for the module class.
+
+
+The CLICommand approach
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+   @CLICommand('antigravity send to blackhole',
+               perm='rw')
+   def send_to_blackhole(self, oid: str, blackhole: Optional[str] = None, inbuf: Optional[str] = None):
+       '''
+       Send the specified object to black hole
+       '''
+       obj = self.find_object(oid)
+       if obj is None:
+           return HandleCommandResult(-errno.ENOENT, stderr=f"object '{oid}' not found")
+       if blackhole is not None and inbuf is not None:
+           try:
+               location = self.decrypt(blackhole, passphrase=inbuf)
+           except ValueError:
+               return HandleCommandResult(-errno.EINVAL, stderr='unable to decrypt location')
+       else:
+           location = blackhole
+       self.send_object_to(obj, location)
+       return HandleCommandResult(stdout=f"the black hole swallowed '{oid}'")
+
+The first parameter passed to ``CLICommand`` is the "name" of the command.
+Since there are lots of commands in Ceph, we tend to group related commands
+with a common prefix. In this case, "antigravity" is used for this purpose.
+As the author is probably designing a module which is also able to launch
+rockets into the deep space.
+
+The `type annotations <https://www.python.org/dev/peps/pep-0484/>`_ for the
+method parameters are mandatory here, so the usage of the command can be
+properly reported to the ``ceph`` CLI, and the manager daemon can convert
+the serialized command parameters sent by the clients to the expected type
+before passing them to the handler method. With properly implemented types,
+one can also perform some sanity checks against the parameters!
+
+The names of the parameters are part of the command interface, so please
+try to take the backward compatibility into consideration when changing
+them. But you **cannot** change name of ``inbuf`` parameter, it is used
+to pass the content of the file specified by ``ceph --in-file`` option.
+
+The docstring of the method is used for the description of the command.
+
+The manager daemon cooks the usage of the command from these ingredients,
+like::
+
+  antigravity send to blackhole <oid> [<blackhole>]  Send the specified object to black hole
+
+as part of the output of ``ceph --help``.
+
+In addition to ``@CLICommand``, you could also use ``@CLIReadCommand`` or
+``@CLIWriteCommand`` if your command only requires read permissions or
+write permissions respectively.
+
+
+The COMMANDS Approach
+~~~~~~~~~~~~~~~~~~~~~
+
+This method uses the ``COMMANDS`` class attribute of your module to define
+a list of dicts like this::
+
+    COMMANDS = [
+        {
+            "cmd": "foobar name=myarg,type=CephString",
+            "desc": "Do something awesome",
+            "perm": "rw",
+            # optional:
+            "poll": "true"
+        }
+    ]
+
+The ``cmd`` part of each entry is parsed in the same way as internal
+Ceph mon and admin socket commands (see mon/MonCommands.h in
+the Ceph source for examples). Note that the "poll" field is optional,
+and is set to False by default; this indicates to the ``ceph`` CLI
+that it should call this command repeatedly and output results (see
+``ceph -h`` and its ``--period`` option).
+
+Each command is expected to return a tuple ``(retval, stdout, stderr)``.
+``retval`` is an integer representing a libc error code (e.g. EINVAL,
+EPERM, or 0 for no error), ``stdout`` is a string containing any
+non-error output, and ``stderr`` is a string containing any progress or
+error explanation output.  Either or both of the two strings may be empty.
+
+Implement the ``handle_command`` function to respond to the commands
+when they are sent:
+
+
+.. py:currentmodule:: mgr_module
+.. automethod:: MgrModule.handle_command
+
+
+Responses and Formatting
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Functions that handle manager commands are expected to return a three element
+tuple with the type signature ``Tuple[int, str, str]``. The first element is a
+return value/error code, where zero indicates no error and a negative `errno`_
+is typically used for error conditions.  The second element corresponds to the
+command's "output". The third element corresponds to the command's "error
+output" (akin to stderr) and is frequently used to report textual error details
+when the return code is non-zero. The ``mgr_module.HandleCommandResult`` type
+can also be used in lieu of a response tuple.
+
+.. _`errno`: https://man7.org/linux/man-pages/man3/errno.3.html
+
+When the implementation of a command raises an exception one of two possible
+approaches to handling the exception exist. First, the command function can do
+nothing and let the exception bubble up to the manager.  When this happens the
+manager will automatically set a return code to -EINVAL and record a trace-back
+in the error output. This trace-back can be very long in some cases. The second
+approach is to handle an exception within a try-except block and convert the
+exception to an error code that better fits the exception (converting a
+KeyError to -ENOENT, for example).  In this case the error output may also be
+set to something more specific and actionable by the one calling the command.
+
+In many cases, especially in more recent versions of Ceph, manager commands are
+designed to return structured output to the caller. Structured output includes
+machine-parsable data such as JSON, YAML, XML, etc. JSON is the most common
+structured output format returned by manager commands. As of Ceph Reef, there
+are a number of new decorators available from the ``object_format`` module that
+help manage formatting output and handling exceptions automatically.  The
+intent is that most of the implementation of a manager command can be written in
+an idiomatic (aka "Pythonic") style and the decorators will take care of most of
+the work needed to format the output and return manager response tuples.
+
+In most cases, net new code should use the ``Responder`` decorator. Example:
+
+.. code:: python
+
+   @CLICommand('antigravity list wormholes', perm='r')
+   @Responder()
+   def list_wormholes(self, oid: str, details: bool = False) -> List[Dict[str, Any]]:
+       '''List wormholes associated with the supplied oid.
+       '''
+       with self.open_wormhole_db() as db:
+           wormholes = db.query(oid=oid)
+       if not details:
+           return [{'name': wh.name} for wh in wormholes]
+       return [{'name': wh.name, 'age': wh.get_age(), 'destination': wh.dest}
+               for wh in wormholes]
+
+Formatting
+++++++++++
+
+The ``Responder`` decorator automatically takes care of converting Python
+objects into a response tuple with formatted output. By default, this decorator
+can automatically return JSON and YAML. When invoked from the command line the
+``--format`` flag can be used to select the response format. If left
+unspecified, JSON will be returned. The automatic formatting can be applied to
+any basic Python type: lists, dicts, str, int, etc. Other objects can be
+formatted automatically if they meet the ``SimpleDataProvider`` protocol - they
+provide a ``to_simplified`` method. The ``to_simplified`` function must return
+a simplified representation of the object made out of basic types.
+
+.. code:: python
+
+   class MyCleverObject:
+       def to_simplified(self) -> Dict[str, int]:
+           # returns a python object(s) made up from basic types
+           return {"gravitons": 999, "tachyons": 404}
+
+   @CLICommand('antigravity list wormholes', perm='r')
+   @Responder()
+   def list_wormholes(self, oid: str, details: bool = False) -> MyCleverObject:
+       '''List wormholes associated with the supplied oid.
+       '''
+       ...
+
+The behavior of the automatic output formatting can be customized and extednted
+to other types of formatting (XML, Plain Text, etc). As this is a complex
+topic, please refer to the module documentation for the ``object_format``
+module.
+
+
+
+Error Handling
+++++++++++++++
+
+Additionally, the ``Responder`` decorator can automatically handle converting
+some  exceptions into response tuples. Any raised exception inheriting from
+``ErrorResponseBase`` will be automatically converted into a response tuple.
+The common approach will be to use ``ErrorResponse``, an exception type that
+can be used directly and has arguments for the error output and return value or
+it can be constructed from an existing exception using the ``wrap``
+classmethod. The wrap classmethod will automatically use the exception text and
+if available the ``errno`` property of other exceptions.
+
+Converting our previous example to use this exception handling approach:
+
+.. code:: python
+
+   @CLICommand('antigravity list wormholes', perm='r')
+   @Responder()
+   def list_wormholes(self, oid: str, details: bool = False) -> List[Dict[str, Any]]:
+       '''List wormholes associated with the supplied oid.
+       '''
+       try:
+           with self.open_wormhole_db() as db:
+               wormholes = db.query(oid=oid)
+       except UnknownOIDError:
+            raise ErrorResponse(f"Unknown oid: {oid}", return_value=-errno.ENOENT)
+       except WormholeDBError as err:
+           raise ErrorResponse.wrap(err)
+       if not details:
+           return [{'name': wh.name} for wh in wormholes]
+       return [{'name': wh.name, 'age': wh.get_age(), 'destination': wh.dest}
+               for wh in wormholes]
+
+
+.. note:: Because the decorator can not determine the difference between a
+   programming mistake and an expected error condition it does not try to
+   catch all exceptions.
+
+
+
+Additional Decorators
++++++++++++++++++++++
+
+The ``object_format`` module provides additional decorators to complement
+``Responder`` but for cases where ``Responder`` is insufficient or too "heavy
+weight".
+
+The ``ErrorResponseHandler`` decorator exists for cases where you *must* still
+return a manager response tuple but want to handle errors as exceptions (as in
+typical Python code). In short, it works like ``Responder`` but only with
+regards to exceptions. Just like ``Responder`` it handles exceptions that
+inherit from ``ErrorResponseBase``. This can be useful in cases where you need
+to return raw data in the output. Example:
+
+.. code:: python
+
+   @CLICommand('antigravity dump config', perm='r')
+   @ErrorResponseHandler()
+   def dump_config(self, oid: str) -> Tuple[int, str, str]:
+       '''Dump configuration
+       '''
+       # we have no control over what data is inside the blob!
+       try:
+            blob = self.fetch_raw_config_blob(oid)
+            return 0, blob, ''
+       except KeyError:
+            raise ErrorResponse("Blob does not exist", return_value=-errno.ENOENT)
+
+
+The ``EmptyResponder`` decorator exists for cases where, on a success
+condition, no output should be generated at all. If you used ``Responder`` and
+default JSON formatting you may always see outputs like ``{}`` or ``[]`` if the
+command completes without error. Instead, ``EmptyResponder`` helps you create
+manager commands that obey the `Rule of Silence`_ when the command has no
+interesting output to emit on success. The functions that ``EmptyResponder``
+decorate should always return ``None``. Like both ``Responder`` and
+``ErrorResponseHandler`` exceptions that inhert from ``ErrorResponseBase`` will
+be automatically processed. Example:
+
+.. code:: python
+
+   @CLICommand('antigravity create wormhole', perm='rw')
+   @EmptyResponder()
+   def create_wormhole(self, oid: str, name: str) -> None:
+       '''Create a new wormhole.
+       '''
+       try:
+           with self.open_wormhole_db() as db:
+               wh = Wormhole(name)
+               db.insert(oid=oid, wormhole=wh)
+       except UnknownOIDError:
+           raise ErrorResponse(f"Unknown oid: {oid}", return_value=-errno.ENOENT)
+       except InvalidWormholeError as err:
+           raise ErrorResponse.wrap(err)
+       except WormholeDBError as err:
+           raise ErrorResponse.wrap(err)
+
+
+.. _`Rule of Silence`: http://www.linfo.org/rule_of_silence.html
+
+
+Configuration options
+---------------------
+
+Modules can load and store configuration options using the
+``set_module_option`` and ``get_module_option`` methods.
+
+.. note:: Use ``set_module_option`` and ``get_module_option`` to
+   manage user-visible configuration options that are not blobs (like
+   certificates). If you want to persist module-internal data or
+   binary configuration data consider using the `KV store`_.
+
+You must declare your available configuration options in the
+``MODULE_OPTIONS`` class attribute, like this:
+
+.. code-block:: python
+
+    MODULE_OPTIONS = [
+        Option(name="my_option")
+    ]
+
+If you try to use set_module_option or get_module_option on options not declared
+in ``MODULE_OPTIONS``, an exception will be raised.
+
+You may choose to provide setter commands in your module to perform
+high level validation.  Users can also modify configuration using
+the normal `ceph config set` command, where the configuration options
+for a mgr module are named like `mgr/<module name>/<option>`.
+
+If a configuration option is different depending on which node the mgr
+is running on, then use *localized* configuration (
+``get_localized_module_option``, ``set_localized_module_option``).
+This may be necessary for options such as what address to listen on.
+Localized options may also be set externally with ``ceph config set``,
+where they key name is like ``mgr/<module name>/<mgr id>/<option>``
+
+If you need to load and store data (e.g. something larger, binary, or multiline),
+use the KV store instead of configuration options (see next section).
+
+Hints for using config options:
+
+* Reads are fast: ceph-mgr keeps a local in-memory copy, so in many cases
+  you can just do a get_module_option every time you use a option, rather than
+  copying it out into a variable.
+* Writes block until the value is persisted (i.e. round trip to the monitor),
+  but reads from another thread will see the new value immediately.
+* If a user has used `config set` from the command line, then the new
+  value will become visible to `get_module_option` immediately, although the
+  mon->mgr update is asynchronous, so `config set` will return a fraction
+  of a second before the new value is visible on the mgr.
+* To delete a config value (i.e. revert to default), just pass ``None`` to
+  set_module_option.
+
+.. automethod:: MgrModule.get_module_option
+.. automethod:: MgrModule.set_module_option
+.. automethod:: MgrModule.get_localized_module_option
+.. automethod:: MgrModule.set_localized_module_option
+
+KV store
+--------
+
+Modules have access to a private (per-module) key value store, which
+is implemented using the monitor's "config-key" commands.  Use
+the ``set_store`` and ``get_store`` methods to access the KV store from
+your module.
+
+The KV store commands work in a similar way to the configuration
+commands.  Reads are fast, operating from a local cache.  Writes block
+on persistence and do a round trip to the monitor.
+
+This data can be access from outside of ceph-mgr using the
+``ceph config-key [get|set]`` commands.  Key names follow the same
+conventions as configuration options.  Note that any values updated
+from outside of ceph-mgr will not be seen by running modules until
+the next restart.  Users should be discouraged from accessing module KV
+data externally -- if it is necessary for users to populate data, modules
+should provide special commands to set the data via the module.
+
+Use the ``get_store_prefix`` function to enumerate keys within
+a particular prefix (i.e. all keys starting with a particular substring).
+
+
+.. automethod:: MgrModule.get_store
+.. automethod:: MgrModule.set_store
+.. automethod:: MgrModule.get_localized_store
+.. automethod:: MgrModule.set_localized_store
+.. automethod:: MgrModule.get_store_prefix
+
+
+Accessing cluster data
+----------------------
+
+Modules have access to the in-memory copies of the Ceph cluster's
+state that the mgr maintains.  Accessor functions as exposed
+as members of MgrModule.
+
+Calls that access the cluster or daemon state are generally going
+from Python into native C++ routines.  There is some overhead to this,
+but much less than for example calling into a REST API or calling into
+an SQL database.
+
+There are no consistency rules about access to cluster structures or
+daemon metadata.  For example, an OSD might exist in OSDMap but
+have no metadata, or vice versa.  On a healthy cluster these
+will be very rare transient states, but modules should be written
+to cope with the possibility.
+
+Note that these accessors must not be called in the modules ``__init__``
+function. This will result in a circular locking exception.
+
+.. automethod:: MgrModule.get
+.. automethod:: MgrModule.get_server
+.. automethod:: MgrModule.list_servers
+.. automethod:: MgrModule.get_metadata
+.. automethod:: MgrModule.get_daemon_status
+.. automethod:: MgrModule.get_perf_schema
+.. automethod:: MgrModule.get_counter
+.. automethod:: MgrModule.get_mgr_id
+.. automethod:: MgrModule.get_daemon_health_metrics
+
+Exposing health checks
+----------------------
+
+Modules can raise first class Ceph health checks, which will be reported
+in the output of ``ceph status`` and in other places that report on the
+cluster's health.
+
+If you use ``set_health_checks`` to report a problem, be sure to call
+it again with an empty dict to clear your health check when the problem
+goes away.
+
+.. automethod:: MgrModule.set_health_checks
+
+What if the mons are down?
+--------------------------
+
+The manager daemon gets much of its state (such as the cluster maps)
+from the monitor.  If the monitor cluster is inaccessible, whichever
+manager was active will continue to run, with the latest state it saw
+still in memory.
+
+However, if you are creating a module that shows the cluster state
+to the user then you may well not want to mislead them by showing
+them that out of date state.
+
+To check if the manager daemon currently has a connection to
+the monitor cluster, use this function:
+
+.. automethod:: MgrModule.have_mon_connection
+
+Reporting if your module cannot run
+-----------------------------------
+
+If your module cannot be run for any reason (such as a missing dependency),
+then you can report that by implementing the ``can_run`` function.
+
+.. automethod:: MgrModule.can_run
+
+Note that this will only work properly if your module can always be imported:
+if you are importing a dependency that may be absent, then do it in a
+try/except block so that your module can be loaded far enough to use
+``can_run`` even if the dependency is absent.
+
+Sending commands
+----------------
+
+A non-blocking facility is provided for sending monitor commands
+to the cluster.
+
+.. automethod:: MgrModule.send_command
+
+Receiving notifications
+-----------------------
+
+The manager daemon calls the ``notify`` function on all active modules
+when certain important pieces of cluster state are updated, such as the
+cluster maps.
+
+The actual data is not passed into this function, rather it is a cue for
+the module to go and read the relevant structure if it is interested.  Most
+modules ignore most types of notification: to ignore a notification
+simply return from this function without doing anything.
+
+.. automethod:: MgrModule.notify
+
+Accessing RADOS or CephFS
+-------------------------
+
+If you want to use the librados python API to access data stored in
+the Ceph cluster, you can access the ``rados`` attribute of your
+``MgrModule`` instance.  This is an instance of ``rados.Rados`` which
+has been constructed for you using the existing Ceph context (an internal
+detail of the C++ Ceph code) of the mgr daemon.
+
+Always use this specially constructed librados instance instead of
+constructing one by hand.
+
+Similarly, if you are using libcephfs to access the file system, then
+use the libcephfs ``create_with_rados`` to construct it from the
+``MgrModule.rados`` librados instance, and thereby inherit the correct context.
+
+Remember that your module may be running while other parts of the cluster
+are down: do not assume that librados or libcephfs calls will return
+promptly -- consider whether to use timeouts or to block if the rest of
+the cluster is not fully available.
+
+Implementing standby mode
+-------------------------
+
+For some modules, it is useful to run on standby manager daemons as well
+as on the active daemon.  For example, an HTTP server can usefully
+serve HTTP redirect responses from the standby managers so that
+the user can point his browser at any of the manager daemons without
+having to worry about which one is active.
+
+Standby manager daemons look for a subclass of ``StandbyModule``
+in each module.  If the class is not found then the module is not
+used at all on standby daemons.  If the class is found, then
+its ``serve`` method is called.  Implementations of ``StandbyModule``
+must inherit from ``mgr_module.MgrStandbyModule``.
+
+The interface of ``MgrStandbyModule`` is much restricted compared to
+``MgrModule`` -- none of the Ceph cluster state is available to
+the module.  ``serve`` and ``shutdown`` methods are used in the same
+way as a normal module class.  The ``get_active_uri`` method enables
+the standby module to discover the address of its active peer in
+order to make redirects.  See the ``MgrStandbyModule`` definition
+in the Ceph source code for the full list of methods.
+
+For an example of how to use this interface, look at the source code
+of the ``dashboard`` module.
+
+Communicating between modules
+-----------------------------
+
+Modules can invoke member functions of other modules.
+
+.. automethod:: MgrModule.remote
+
+Be sure to handle ``ImportError`` to deal with the case that the desired
+module is not enabled.
+
+If the remote method raises a python exception, this will be converted
+to a RuntimeError on the calling side, where the message string describes
+the exception that was originally thrown.  If your logic intends
+to handle certain errors cleanly, it is better to modify the remote method
+to return an error value instead of raising an exception.
+
+At time of writing, inter-module calls are implemented without
+copies or serialization, so when you return a python object, you're 
+returning a reference to that object to the calling module.  It
+is recommend *not* to rely on this reference passing, as in future the
+implementation may change to serialize arguments and return
+values.
+
+
+Shutting down cleanly
+---------------------
+
+If a module implements the ``serve()`` method, it should also implement
+the ``shutdown()`` method to shutdown cleanly: misbehaving modules
+may otherwise prevent clean shutdown of ceph-mgr.
+
+Limitations
+-----------
+
+It is not possible to call back into C++ code from a module's
+``__init__()`` method.  For example calling ``self.get_module_option()`` at
+this point will result in an assertion failure in ceph-mgr.  For modules
+that implement the ``serve()`` method, it usually makes sense to do most
+initialization inside that method instead.
+
+Debugging
+---------
+
+Apparently, we can always use the :ref:`mgr module dev logging` facility
+for debugging a ceph-mgr module. But some of us might miss `PDB`_ and the
+interactive Python interpreter. Yes, we can have them as well when developing
+ceph-mgr modules! ``ceph_mgr_repl.py`` can drop you into an interactive shell
+talking to ``selftest`` module. With this tool, one can peek and poke the
+ceph-mgr module, and use all the exposed facilities in quite the same way
+how we use the Python command line interpreter. For using ``ceph_mgr_repl.py``,
+we need to
+
+#. ready a Ceph cluster
+#. enable the ``selftest`` module
+#. setup the necessary environment variables
+#. launch the tool
+
+.. _PDB: https://docs.python.org/3/library/pdb.html
+
+Following is a sample session, in which the Ceph version is queried by
+inputting ``print(mgr.version)`` at the prompt. And later
+``timeit`` module is imported to measure the execution time of
+`mgr.get_mgr_id()`.
+
+.. code-block:: console
+
+   $ cd build
+   $ MDS=0 MGR=1 OSD=3 MON=1 ../src/vstart.sh -n -x
+   $ bin/ceph mgr module enable selftest
+   $ ../src/pybind/ceph_mgr_repl.py --show-env
+      $ export PYTHONPATH=/home/me/ceph/src/pybind:/home/me/ceph/build/lib/cython_modules/lib.3:/home/me/ceph/src/python-common:$PYTHONPATH
+      $ export LD_LIBRARY_PATH=/home/me/ceph/build/lib:$LD_LIBRARY_PATH
+   $ export PYTHONPATH=/home/me/ceph/src/pybind:/home/me/ceph/build/lib/cython_modules/lib.3:/home/me/ceph/src/python-common:$PYTHONPATH
+   $ export LD_LIBRARY_PATH=/home/me/ceph/build/lib:$LD_LIBRARY_PATH
+   $ ../src/pybind/ceph_mgr_repl.py
+   $ ../src/pybind/ceph_mgr_repl.py
+   Python 3.9.2 (default, Feb 28 2021, 17:03:44)
+   [GCC 10.2.1 20210110] on linux
+   Type "help", "copyright", "credits" or "license" for more information.
+   (MgrModuleInteractiveConsole)
+   [mgr self-test eval] >>> print(mgr.version)
+   ceph version Development (no_version) quincy (dev)
+   [mgr self-test eval] >>> from timeit import timeit
+   [mgr self-test eval] >>> timeit(mgr.get_mgr_id)
+   0.16303414600042743
+   [mgr self-test eval] >>>
+
+If you want to "talk" to a ceph-mgr module other than ``selftest`` using
+this tool, you can either add a command to the module you want to debug
+exactly like how ``mgr self-test eval`` command was added to ``selftest``. Or
+we can make this simpler by promoting the ``eval()`` method to a dedicated
+`Mixin`_ class and inherit your ``MgrModule`` subclass from it. And define
+a command with it. Assuming the prefix of the command is ``mgr my-module eval``,
+one can just put
+
+.. prompt:: bash $
+
+   ../src/pybind/ceph_mgr_repl.py --prefix "mgr my-module eval"
+
+
+.. _Mixin: _https://en.wikipedia.org/wiki/Mixin
+
+Is something missing?
+---------------------
+
+The ceph-mgr python interface is not set in stone.  If you have a need
+that is not satisfied by the current interface, please bring it up
+on the ceph-devel mailing list.  While it is desired to avoid bloating
+the interface, it is not generally very hard to expose existing data
+to the Python code when there is a good reason.
+
diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst
new file mode 100644
index 000000000..7e6637684
--- /dev/null
+++ b/doc/mgr/nfs.rst
@@ -0,0 +1,680 @@
+.. _mgr-nfs:
+
+=============================
+CephFS & RGW Exports over NFS
+=============================
+
+CephFS namespaces and RGW buckets can be exported over NFS protocol
+using the `NFS-Ganesha NFS server`_.
+
+The ``nfs`` manager module provides a general interface for managing
+NFS exports of either CephFS directories or RGW buckets.  Exports can
+be managed either via the CLI ``ceph nfs export ...`` commands
+or via the dashboard.
+
+The deployment of the nfs-ganesha daemons can also be managed
+automatically if either the :ref:`cephadm` or :ref:`mgr-rook`
+orchestrators are enabled.  If neither are in use (e.g., Ceph is
+deployed via an external orchestrator like Ansible or Puppet), the
+nfs-ganesha daemons must be manually deployed; for more information,
+see :ref:`nfs-ganesha-config`.
+
+.. note:: Starting with Ceph Pacific, the ``nfs`` mgr module must be enabled.
+
+NFS Cluster management
+======================
+
+.. _nfs-module-cluster-create:
+
+Create NFS Ganesha Cluster
+--------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster create <cluster_id> [<placement>] [--ingress] [--virtual_ip <value>] [--ingress-mode {default|keepalive-only|haproxy-standard|haproxy-protocol}] [--port <int>]
+
+This creates a common recovery pool for all NFS Ganesha daemons, new user based on
+``cluster_id``, and a common NFS Ganesha config RADOS object.
+
+.. note:: Since this command also brings up NFS Ganesha daemons using a ceph-mgr
+   orchestrator module (see :doc:`/mgr/orchestrator`) such as cephadm or rook, at
+   least one such module must be enabled for it to work.
+
+   Currently, NFS Ganesha daemon deployed by cephadm listens on the standard
+   port. So only one daemon will be deployed on a host.
+
+``<cluster_id>`` is an arbitrary string by which this NFS Ganesha cluster will be
+known (e.g., ``mynfs``).
+
+``<placement>`` is an optional string signifying which hosts should have NFS Ganesha
+daemon containers running on them and, optionally, the total number of NFS
+Ganesha daemons on the cluster (should you want to have more than one NFS Ganesha
+daemon running per node). For example, the following placement string means
+"deploy NFS Ganesha daemons on nodes host1 and host2 (one daemon per host)::
+
+    "host1,host2"
+
+and this placement specification says to deploy single NFS Ganesha daemon each
+on nodes host1 and host2 (for a total of two NFS Ganesha daemons in the
+cluster)::
+
+    "2 host1,host2"
+
+NFS can be deployed on a port other than 2049 (the default) with ``--port <port>``.
+
+To deploy NFS with a high-availability front-end (virtual IP and load balancer), add the
+``--ingress`` flag and specify a virtual IP address. This will deploy a combination
+of keepalived and haproxy to provide an high-availability NFS frontend for the NFS
+service.
+
+.. note:: The ingress implementation is not yet complete.  Enabling
+	  ingress will deploy multiple ganesha instances and balance
+	  load across them, but a host failure will not immediately
+	  cause cephadm to deploy a replacement daemon before the NFS
+	  grace period expires.  This high-availability functionality
+	  is expected to be completed by the Quincy release (March
+	  2022).
+
+For more details, refer :ref:`orchestrator-cli-placement-spec` but keep
+in mind that specifying the placement via a YAML file is not supported.
+
+Deployment of NFS daemons and the ingress service is asynchronous: the
+command may return before the services have completely started. You may
+wish to check that these services do successfully start and stay running.
+When using cephadm orchestration, these commands check service status:
+
+.. code:: bash
+
+    $ ceph orch ls --service_name=nfs.<cluster_id>
+    $ ceph orch ls --service_name=ingress.nfs.<cluster_id>
+
+
+Ingress
+-------
+
+The core *nfs* service will deploy one or more nfs-ganesha daemons,
+each of which will provide a working NFS endpoint.  The IP for each
+NFS endpoint will depend on which host the nfs-ganesha daemons are
+deployed.  By default, daemons are placed semi-randomly, but users can
+also explicitly control where daemons are placed; see
+:ref:`orchestrator-cli-placement-spec`.
+
+When a cluster is created with ``--ingress``, an *ingress* service is
+additionally deployed to provide load balancing and high-availability
+for the NFS servers.  A virtual IP is used to provide a known, stable
+NFS endpoint that all clients can use to mount.  Ceph will take care
+of the details of NFS redirecting traffic on the virtual IP to the
+appropriate backend NFS servers, and redeploying NFS servers when they
+fail.
+
+An optional ``--ingress-mode`` parameter can be provided to choose
+how the *ingress* service is configured:
+
+- Setting ``--ingress-mode keepalive-only`` deploys a simplified *ingress*
+  service that provides a virtual IP with the nfs server directly binding to
+  that virtual IP and leaves out any sort of load balancing or traffic
+  redirection. This setup will restrict users to deploying only 1 nfs daemon
+  as multiple cannot bind to the same port on the virtual IP.
+- Setting ``--ingress-mode haproxy-standard`` deploys a full *ingress* service
+  to provide load balancing and high-availability using HAProxy and keepalived.
+  Client IP addresses are not visible to the back-end NFS server and IP level
+  restrictions on NFS exports will not function.
+- Setting ``--ingress-mode haproxy-protocol`` deploys a full *ingress* service
+  to provide load balancing and high-availability using HAProxy and keepalived.
+  Client IP addresses are visible to the back-end NFS server and IP level
+  restrictions on NFS exports are usable. This mode requires NFS Ganesha version
+  5.0 or later.
+- Setting ``--ingress-mode default`` is equivalent to not providing any other
+  ingress mode by name. When no other ingress mode is specified by name
+  the default ingress mode used is ``haproxy-standard``.
+
+Ingress can be added to an existing NFS service (e.g., one initially created
+without the ``--ingress`` flag), and the basic NFS service can
+also be modified after the fact to include non-default options, by modifying
+the services directly.  For more information, see :ref:`cephadm-ha-nfs`.
+
+Show NFS Cluster IP(s)
+----------------------
+
+To examine an NFS cluster's IP endpoints, including the IPs for the individual NFS
+daemons, and the virtual IP (if any) for the ingress service,
+
+.. code:: bash
+
+    $ ceph nfs cluster info [<cluster_id>]
+
+.. note:: This will not work with the rook backend. Instead, expose the port with
+   the kubectl patch command and fetch the port details with kubectl get services
+   command::
+
+    $ kubectl patch service -n rook-ceph -p '{"spec":{"type": "NodePort"}}' rook-ceph-nfs-<cluster-name>-<node-id>
+    $ kubectl get services -n rook-ceph rook-ceph-nfs-<cluster-name>-<node-id>
+
+
+Delete NFS Ganesha Cluster
+--------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster rm <cluster_id>
+
+This deletes the deployed cluster.
+
+
+Removal of NFS daemons and the ingress service is asynchronous: the
+command may return before the services have been completely deleted. You may
+wish to check that these services are no longer reported. When using cephadm
+orchestration, these commands check service status:
+
+.. code:: bash
+
+    $ ceph orch ls --service_name=nfs.<cluster_id>
+    $ ceph orch ls --service_name=ingress.nfs.<cluster_id>
+
+
+Updating an NFS Cluster
+-----------------------
+
+In order to modify cluster parameters (like the port or placement), you need to
+use the orchestrator interface to update the NFS service spec.  The safest way to do
+that is to export the current spec, modify it, and then re-apply it.  For example,
+to modify the ``nfs.foo`` service,
+
+.. code:: bash
+
+    $ ceph orch ls --service-name nfs.foo --export > nfs.foo.yaml
+    $ vi nfs.foo.yaml
+    $ ceph orch apply -i nfs.foo.yaml
+
+For more information about the NFS service spec, see :ref:`deploy-cephadm-nfs-ganesha`.
+
+List NFS Ganesha Clusters
+-------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster ls
+
+This lists deployed clusters.
+
+.. _nfs-cluster-set:
+
+Set Customized NFS Ganesha Configuration
+----------------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config set <cluster_id> -i <config_file>
+
+With this the nfs cluster will use the specified config and it will have
+precedence over default config blocks.
+
+Example use cases include:
+
+#. Changing log level.  The logging level can be adjusted with the following config
+   fragment::
+
+     LOG {
+         COMPONENTS {
+             ALL = FULL_DEBUG;
+         }
+     }
+
+#. Adding custom export block.
+
+   The following sample block creates a single export. This export will not be
+   managed by `ceph nfs export` interface::
+
+    EXPORT {
+      Export_Id = 100;
+      Transports = TCP;
+      Path = /;
+      Pseudo = /ceph/;
+      Protocols = 4;
+      Access_Type = RW;
+      Attr_Expiration_Time = 0;
+      Squash = None;
+      FSAL {
+        Name = CEPH;
+        Filesystem = "filesystem name";
+        User_Id = "user id";
+        Secret_Access_Key = "secret key";
+      }
+    }
+
+.. note:: User specified in FSAL block should have proper caps for NFS-Ganesha
+   daemons to access ceph cluster. User can be created in following way using
+   `auth get-or-create`::
+
+         # ceph auth get-or-create client.<user_id> mon 'allow r' osd 'allow rw pool=.nfs namespace=<nfs_cluster_name>, allow rw tag cephfs data=<fs_name>' mds 'allow rw path=<export_path>'
+
+View Customized NFS Ganesha Configuration
+-----------------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config get <cluster_id>
+
+This will output the user defined configuration (if any).
+
+Reset NFS Ganesha Configuration
+-------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config reset <cluster_id>
+
+This removes the user defined configuration.
+
+.. note:: With a rook deployment, ganesha pods must be explicitly restarted
+   for the new config blocks to be effective.
+
+
+Export Management
+=================
+
+.. warning:: Currently, the nfs interface is not integrated with dashboard. Both
+   dashboard and nfs interface have different export requirements and
+   create exports differently. Management of dashboard created exports is not
+   supported.
+
+Create CephFS Export
+--------------------
+
+.. code:: bash
+
+    $ ceph nfs export create cephfs --cluster-id <cluster_id> --pseudo-path <pseudo_path> --fsname <fsname> [--readonly] [--path=/path/in/cephfs] [--client_addr <value>...] [--squash <value>] [--sectype <value>...]
+
+This creates export RADOS objects containing the export block, where
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique.
+
+``<fsname>`` is the name of the FS volume used by the NFS Ganesha cluster
+that will serve this export.
+
+``<path>`` is the path within cephfs. Valid path should be given and default
+path is '/'. It need not be unique. Subvolume path can be fetched using:
+
+.. code::
+
+   $ ceph fs subvolume getpath <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+``<client_addr>`` is the list of client address for which these export
+permissions will be applicable. By default all clients can access the export
+according to specified export permissions. See the `NFS-Ganesha Export Sample`_
+for permissible values.
+
+``<squash>`` defines the kind of user id squashing to be performed. The default
+value is `no_root_squash`. See the `NFS-Ganesha Export Sample`_ for
+permissible values.
+
+``<sectype>`` specifies which authentication methods will be used when
+connecting to the export. Valid values include "krb5p", "krb5i", "krb5", "sys",
+and "none". More than one value can be supplied. The flag may be specified
+multiple times (example: ``--sectype=krb5p --sectype=krb5i``) or multiple
+values may be separated by a comma (example: ``--sectype krb5p,krb5i``). The
+server will negotatiate a supported security type with the client preferring
+the supplied methods left-to-right.
+
+.. note:: Specifying values for sectype that require Kerberos will only function on servers
+          that are configured to support Kerberos. Setting up NFS-Ganesha to support Kerberos
+          is outside the scope of this document.
+
+.. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface.
+
+Create RGW Export
+-----------------
+
+There are two kinds of RGW exports:
+
+- a *user* export will export all buckets owned by an
+  RGW user, where the top-level directory of the export is a list of buckets.
+- a *bucket* export will export a single bucket, where the top-level directory contains
+  the objects in the bucket.
+
+RGW bucket export
+^^^^^^^^^^^^^^^^^
+  
+To export a *bucket*:
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id <cluster_id> --pseudo-path <pseudo_path> --bucket <bucket_name> [--user-id <user-id>] [--readonly] [--client_addr <value>...] [--squash <value>] [--sectype <value>...]
+
+For example, to export *mybucket* via NFS cluster *mynfs* at the pseudo-path */bucketdata* to any host in the ``192.168.10.0/24`` network
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --bucket mybucket --client_addr 192.168.10.0/24
+
+.. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface.
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique.
+
+``<bucket_name>`` is the name of the bucket that will be exported.
+
+``<user_id>`` is optional, and specifies which RGW user will be used for read and write
+operations to the bucket.  If it is not specified, the user who owns the bucket will be
+used.
+
+.. note:: Currently, if multi-site RGW is enabled, Ceph can only export RGW buckets in the default realm.
+
+``<client_addr>`` is the list of client address for which these export
+permissions will be applicable. By default all clients can access the export
+according to specified export permissions. See the `NFS-Ganesha Export Sample`_
+for permissible values.
+
+``<squash>`` defines the kind of user id squashing to be performed. The default
+value is `no_root_squash`. See the `NFS-Ganesha Export Sample`_ for
+permissible values.
+
+``<sectype>`` specifies which authentication methods will be used when
+connecting to the export. Valid values include "krb5p", "krb5i", "krb5", "sys",
+and "none". More than one value can be supplied. The flag may be specified
+multiple times (example: ``--sectype=krb5p --sectype=krb5i``) or multiple
+values may be separated by a comma (example: ``--sectype krb5p,krb5i``). The
+server will negotatiate a supported security type with the client preferring
+the supplied methods left-to-right.
+
+.. note:: Specifying values for sectype that require Kerberos will only function on servers
+          that are configured to support Kerberos. Setting up NFS-Ganesha to support Kerberos
+          is outside the scope of this document.
+
+RGW user export
+^^^^^^^^^^^^^^^
+
+To export an RGW *user*:
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id <cluster_id> --pseudo-path <pseudo_path> --user-id <user-id> [--readonly] [--client_addr <value>...] [--squash <value>]
+
+For example, to export *myuser* via NFS cluster *mynfs* at the pseudo-path */myuser* to any host in the ``192.168.10.0/24`` network
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --user-id myuser --client_addr 192.168.10.0/24
+
+
+Delete Export
+-------------
+
+.. code:: bash
+
+    $ ceph nfs export rm <cluster_id> <pseudo_path>
+
+This deletes an export in an NFS Ganesha cluster, where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the pseudo root path (must be an absolute path).
+
+List Exports
+------------
+
+.. code:: bash
+
+    $ ceph nfs export ls <cluster_id> [--detailed]
+
+It lists exports for a cluster, where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+With the ``--detailed`` option enabled it shows entire export block.
+
+Get Export
+----------
+
+.. code:: bash
+
+    $ ceph nfs export info <cluster_id> <pseudo_path>
+
+This displays export block for a cluster based on pseudo root name,
+where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the pseudo root path (must be an absolute path).
+
+
+Create or update export via JSON specification
+----------------------------------------------
+
+An existing export can be dumped in JSON format with:
+
+.. prompt:: bash #
+
+    ceph nfs export info *<cluster_id>* *<pseudo_path>*
+
+An export can be created or modified by importing a JSON description in the
+same format:
+
+.. prompt:: bash #
+
+    ceph nfs export apply *<cluster_id>* -i <json_file>
+
+For example,::
+
+   $ ceph nfs export info mynfs /cephfs > update_cephfs_export.json
+   $ cat update_cephfs_export.json
+   {
+     "export_id": 1,
+     "path": "/",
+     "cluster_id": "mynfs",
+     "pseudo": "/cephfs",
+     "access_type": "RW",
+     "squash": "no_root_squash",
+     "security_label": true,
+     "protocols": [
+       4
+     ],
+     "transports": [
+       "TCP"
+     ],
+     "fsal": {
+       "name": "CEPH",
+       "user_id": "nfs.mynfs.1",
+       "fs_name": "a",
+       "sec_label_xattr": ""
+     },
+     "clients": []
+   }
+
+The imported JSON can be a single dict describing a single export, or a JSON list
+containing multiple export dicts.
+
+The exported JSON can be modified and then reapplied.  Below, *pseudo*
+and *access_type* are modified.  When modifying an export, the
+provided JSON should fully describe the new state of the export (just
+as when creating a new export), with the exception of the
+authentication credentials, which will be carried over from the
+previous state of the export where possible.
+
+::
+
+   $ ceph nfs export apply mynfs -i update_cephfs_export.json
+   $ cat update_cephfs_export.json
+   {
+     "export_id": 1,
+     "path": "/",
+     "cluster_id": "mynfs",
+     "pseudo": "/cephfs_testing",
+     "access_type": "RO",
+     "squash": "no_root_squash",
+     "security_label": true,
+     "protocols": [
+       4
+     ],
+     "transports": [
+       "TCP"
+     ],
+     "fsal": {
+       "name": "CEPH",
+       "user_id": "nfs.mynfs.1",
+       "fs_name": "a",
+       "sec_label_xattr": ""
+     },
+     "clients": []
+   }
+
+An export can also be created or updated by injecting a Ganesha NFS EXPORT config
+fragment.  For example,::
+
+   $ ceph nfs export apply mynfs -i update_cephfs_export.conf
+   $ cat update_cephfs_export.conf
+   EXPORT {
+       FSAL {
+           name = "CEPH";
+           filesystem = "a";
+       }
+       export_id = 1;
+       path = "/";
+       pseudo = "/a";
+       access_type = "RW";
+       squash = "none";
+       attr_expiration_time = 0;
+       security_label = true;
+       protocols = 4;
+       transports = "TCP";
+   }
+
+
+Mounting
+========
+
+After the exports are successfully created and NFS Ganesha daemons are
+deployed, exports can be mounted with:
+
+.. code:: bash
+
+    $ mount -t nfs <ganesha-host-name>:<pseudo_path> <mount-point>
+
+For example, if the NFS cluster was created with ``--ingress --virtual-ip 192.168.10.10``
+and the export's pseudo-path was ``/foo``, the export can be mounted at ``/mnt`` with:
+
+.. code:: bash
+
+    $ mount -t nfs 192.168.10.10:/foo /mnt
+
+If the NFS service is running on a non-standard port number:
+
+.. code:: bash
+
+    $ mount -t nfs -o port=<ganesha-port> <ganesha-host-name>:<ganesha-pseudo_path> <mount-point>
+
+.. note:: Only NFS v4.0+ is supported.
+
+Troubleshooting
+===============
+
+Checking NFS-Ganesha logs with
+
+1) ``cephadm``: The NFS daemons can be listed with:
+
+   .. code:: bash
+
+	    $ ceph orch ps --daemon-type nfs
+
+   You can via the logs for a specific daemon (e.g., ``nfs.mynfs.0.0.myhost.xkfzal``) on
+   the relevant host with:
+
+   .. code:: bash
+
+      # cephadm logs --fsid <fsid> --name nfs.mynfs.0.0.myhost.xkfzal
+
+2) ``rook``:
+
+   .. code:: bash
+
+      $ kubectl logs -n rook-ceph rook-ceph-nfs-<cluster_id>-<node_id> nfs-ganesha
+
+The NFS log level can be adjusted using `nfs cluster config set` command (see :ref:`nfs-cluster-set`).
+
+
+.. _nfs-ganesha-config:
+
+
+Manual Ganesha deployment
+=========================
+
+It may be possible to deploy and manage the NFS ganesha daemons without
+orchestration frameworks such as cephadm or rook.
+
+.. note:: Manual configuration is not tested or fully documented; your
+          mileage may vary. If you make this work, please help us by
+          updating this documentation.
+
+Limitations
+------------
+
+If no orchestrator module is enabled for the Ceph Manager the NFS cluster
+management commands, such as those starting with ``ceph nfs cluster``, will not
+function. However, commands that manage NFS exports, like those prefixed with
+``ceph nfs export`` are expected to work as long as the necessary RADOS objects
+have already been created. The exact RADOS objects required are not documented
+at this time as support for this feature is incomplete. A curious reader can
+find some details about the object by reading the source code for the
+``mgr/nfs`` module (found in the ceph source tree under
+``src/pybind/mgr/nfs``).
+
+
+Requirements
+------------
+
+The following packages are required to enable CephFS and RGW exports with nfs-ganesha:
+
+-  ``nfs-ganesha``, ``nfs-ganesha-ceph``, ``nfs-ganesha-rados-grace`` and
+   ``nfs-ganesha-rados-urls`` packages (version 3.3 and above)
+
+Ganesha Configuration Hierarchy
+-------------------------------
+
+Cephadm and rook start each nfs-ganesha daemon with a minimal
+`bootstrap` configuration file that pulls from a shared `common`
+configuration stored in the ``.nfs`` RADOS pool and watches the common
+config for changes.  Each export is written to a separate RADOS object
+that is referenced by URL from the common config.
+
+.. ditaa::
+
+                             rados://$pool/$namespace/export-$i        rados://$pool/$namespace/userconf-nfs.$cluster_id
+                                      (export config)                          (user config)
+
+                        +----------+    +----------+    +----------+      +---------------------------+
+                        |          |    |          |    |          |      |                           |
+                        | export-1 |    | export-2 |    | export-3 |      | userconf-nfs.$cluster_id  |
+                        |          |    |          |    |          |      |                           |
+                        +----+-----+    +----+-----+    +-----+----+      +-------------+-------------+
+                             ^               ^                ^                         ^
+                             |               |                |                         |
+                             +--------------------------------+-------------------------+
+                                        %url |
+                                             |
+                                    +--------+--------+
+                                    |                 |  rados://$pool/$namespace/conf-nfs.$svc
+                                    |  conf+nfs.$svc  |  (common config)
+                                    |                 |
+                                    +--------+--------+
+                                             ^
+                                             |
+                                   watch_url |
+                     +----------------------------------------------+
+                     |                       |                      |
+                     |                       |                      |            RADOS
+             +----------------------------------------------------------------------------------+
+                     |                       |                      |            CONTAINER
+           watch_url |             watch_url |            watch_url |
+                     |                       |                      |
+            +--------+-------+      +--------+-------+      +-------+--------+
+            |                |      |                |      |                |  /etc/ganesha/ganesha.conf
+            |   nfs.$svc.a   |      |   nfs.$svc.b   |      |   nfs.$svc.c   |  (bootstrap config)
+            |                |      |                |      |                |
+            +----------------+      +----------------+      +----------------+
+
+
+.. _NFS-Ganesha NFS Server: https://github.com/nfs-ganesha/nfs-ganesha/wiki
+.. _NFS-Ganesha Export Sample: https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/export.txt
diff --git a/doc/mgr/orchestrator.rst b/doc/mgr/orchestrator.rst
new file mode 100644
index 000000000..7cb6ddeec
--- /dev/null
+++ b/doc/mgr/orchestrator.rst
@@ -0,0 +1,240 @@
+
+.. _orchestrator-cli-module:
+
+================
+Orchestrator CLI
+================
+
+This module provides a command line interface (CLI) for orchestrator modules.
+Orchestrator modules are ``ceph-mgr`` plugins that interface with external
+orchestration services.
+
+Definition of Terms
+===================
+
+The orchestrator CLI unifies multiple external orchestrators, so we need a
+common nomenclature for the orchestrator module: 
+
++--------------------------------------+---------------------------------------+
+| *host*                               | hostname (not the DNS name) of the    |
+|                                      | physical host. Not the podname,       |
+|                                      | container name, or hostname inside    |
+|                                      | the container.                        |
++--------------------------------------+---------------------------------------+
+| *service type*                       | The type of the service. e.g., nfs,   |
+|                                      | mds, osd, mon, rgw, mgr, iscsi        |
++--------------------------------------+---------------------------------------+
+| *service*                            | A logical service. Typically          |
+|                                      | comprised of multiple service         |
+|                                      | instances on multiple hosts for HA    |
+|                                      |                                       |
+|                                      | * ``fs_name`` for mds type            |
+|                                      | * ``rgw_zone`` for rgw type           |
+|                                      | * ``ganesha_cluster_id`` for nfs type |
++--------------------------------------+---------------------------------------+
+| *daemon*                             | A single instance of a service.       |
+|                                      | Usually a daemon, but maybe not       |
+|                                      | (e.g., might be a kernel service      |
+|                                      | like LIO or knfsd or whatever)        |
+|                                      |                                       |
+|                                      | This identifier should                |
+|                                      | uniquely identify the instance.       |
++--------------------------------------+---------------------------------------+
+
+Here is how the names relate: 
+
+* A *service* has a specific *service type*.
+* A *daemon* is a physical instance of a *service type*.
+
+.. note::
+
+    Orchestrator modules might implement only a subset of the commands listed
+    below. The implementation of the commands may differ between modules.
+
+Status
+======
+
+.. prompt:: bash $
+
+   ceph orch status [--detail]
+
+This command shows the current orchestrator mode and its high-level status
+(whether the orchestrator plugin is available and operational).
+
+
+..
+    Turn On Device Lights
+    ^^^^^^^^^^^^^^^^^^^^^
+    ::
+
+        ceph orch device ident-on <dev_id>
+        ceph orch device ident-on <dev_name> <host>
+        ceph orch device fault-on <dev_id>
+        ceph orch device fault-on <dev_name> <host>
+
+        ceph orch device ident-off <dev_id> [--force=true]
+        ceph orch device ident-off <dev_id> <host> [--force=true]
+        ceph orch device fault-off <dev_id> [--force=true]
+        ceph orch device fault-off <dev_id> <host> [--force=true]
+
+    where ``dev_id`` is the device id as listed in ``osd metadata``,
+    ``dev_name`` is the name of the device on the system and ``host`` is the host as
+    returned by ``orchestrator host ls``
+
+        ceph orch osd ident-on {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd ident-off {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd fault-on {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd fault-off {primary,journal,db,wal,all} <osd-id>
+
+    where ``journal`` is the filestore journal device, ``wal`` is the bluestore
+    write ahead log device, and ``all`` stands for all devices associated with the OSD
+
+
+.. _orchestrator-cli-stateless-services:
+
+Stateless services (MDS/RGW/NFS/rbd-mirror/iSCSI)
+=================================================
+
+.. note::
+
+   The orchestrator will not configure the services. See the relevant
+   documentation for details about how to configure particular services. 
+
+The ``name`` parameter identifies the kind of the group of instances. The
+following short list explains the meaning of the ``name`` parameter:
+
+* A CephFS file system identifies a group of MDS daemons.
+* A zone name identifies a group of RGWs.
+
+Creating/growing/shrinking/removing services:
+
+.. prompt:: bash $
+
+    ceph orch apply mds <fs_name> [--placement=<placement>] [--dry-run]
+    ceph orch apply rgw <name> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>] [--dry-run]
+    ceph orch apply nfs <name> <pool> [--namespace=<namespace>] [--placement=<placement>] [--dry-run]
+    ceph orch rm <service_name> [--force]
+
+where ``placement`` is a :ref:`orchestrator-cli-placement-spec`.
+
+e.g., ``ceph orch apply mds myfs --placement="3 host1 host2 host3"``
+
+Service Commands:
+
+.. prompt:: bash $
+
+    ceph orch <start|stop|restart|redeploy|reconfig> <service_name>
+
+.. note:: These commands apply only to cephadm containerized daemons.
+
+Options
+=======
+
+.. option:: start
+
+   Start the daemon on the corresponding host.
+
+.. option:: stop
+
+   Stop the daemon on the corresponding host.
+
+.. option:: restart
+
+   Restart the daemon on the corresponding host.
+
+.. option:: redeploy
+
+   Redeploy the Ceph daemon on the corresponding host. This will recreate the daemon directory
+   structure under ``/var/lib/ceph/<fsid>/<daemon-name>`` (if it doesn't exist), refresh its
+   configuration files, regenerate its unit-files and restarts the systemd daemon.
+
+.. option:: reconfig
+
+   Reconfigure the daemon on the corresponding host. This will refresh configuration files then restart the daemon.
+
+   .. note:: this command assumes the daemon directory ``/var/lib/ceph/<fsid>/<daemon-name>`` already exists.
+
+
+Configuring the Orchestrator CLI
+================================
+
+Enable the orchestrator by using the ``set backend`` command to select the orchestrator module that will be used:
+
+.. prompt:: bash $
+
+    ceph orch set backend <module>
+
+Example - Configuring the Orchestrator CLI
+------------------------------------------
+
+For example, to enable the Rook orchestrator module and use it with the CLI:
+
+.. prompt:: bash $
+
+    ceph mgr module enable rook
+    ceph orch set backend rook
+
+Confirm that the backend is properly configured:
+
+.. prompt:: bash $
+
+    ceph orch status
+
+Disable the Orchestrator
+------------------------
+
+To disable the orchestrator, use the empty string ``""``:
+
+.. prompt:: bash $
+
+    ceph orch set backend ""
+    ceph mgr module disable rook
+
+Current Implementation Status
+=============================
+
+This is an overview of the current implementation status of the orchestrators.
+
+=================================== ====== =========
+ Command                             Rook   Cephadm
+=================================== ====== =========
+ apply iscsi                         ⚪     ✔
+ apply mds                           ✔      ✔
+ apply mgr                           ⚪      ✔
+ apply mon                           ✔      ✔
+ apply nfs                           ✔      ✔
+ apply osd                           ✔      ✔
+ apply rbd-mirror                    ✔      ✔
+ apply cephfs-mirror                 ⚪      ✔
+ apply grafana                       ⚪      ✔
+ apply prometheus                    �      ✔
+ apply alertmanager                  �      ✔
+ apply node-exporter                 �      ✔
+ apply rgw                           ✔       ✔
+ apply container                     ⚪      ✔
+ apply snmp-gateway                  �      ✔
+ host add                            ⚪      ✔
+ host ls                             ✔      ✔
+ host rm                             ⚪      ✔
+ host maintenance enter              �      ✔
+ host maintenance exit               �      ✔
+ daemon status                       ⚪      ✔
+ daemon {stop,start,...}             ⚪      ✔
+ device {ident,fault}-(on,off}       ⚪      ✔
+ device ls                           ✔      ✔
+ iscsi add                           ⚪     ✔
+ mds add                             ⚪      ✔
+ nfs add                             ⚪      ✔
+ rbd-mirror add                      ⚪      ✔
+ rgw add                             ⚪     ✔
+ ls                                  ✔      ✔
+ ps                                  ✔      ✔
+ status                              ✔      ✔
+ upgrade                             �      ✔
+=================================== ====== =========
+
+where
+
+* ⚪ = not yet implemented
+* � = not applicable
+* ✔ = implemented
diff --git a/doc/mgr/orchestrator_modules.rst b/doc/mgr/orchestrator_modules.rst
new file mode 100644
index 000000000..a28b43059
--- /dev/null
+++ b/doc/mgr/orchestrator_modules.rst
@@ -0,0 +1,332 @@
+
+
+.. _orchestrator-modules:
+
+.. py:currentmodule:: orchestrator
+
+ceph-mgr orchestrator modules
+=============================
+
+.. warning::
+
+    This is developer documentation, describing Ceph internals that
+    are only relevant to people writing ceph-mgr orchestrator modules.
+
+In this context, *orchestrator* refers to some external service that
+provides the ability to discover devices and create Ceph services.  This
+includes external projects such as Rook.
+
+An *orchestrator module* is a ceph-mgr module (:ref:`mgr-module-dev`)
+which implements common management operations using a particular
+orchestrator.
+
+Orchestrator modules subclass the ``Orchestrator`` class: this class is
+an interface, it only provides method definitions to be implemented
+by subclasses.  The purpose of defining this common interface
+for different orchestrators is to enable common UI code, such as
+the dashboard, to work with various different backends.
+
+
+.. graphviz::
+
+    digraph G {
+        subgraph cluster_1 {
+            volumes [label="mgr/volumes"]
+            rook [label="mgr/rook"]
+            dashboard [label="mgr/dashboard"]
+            orchestrator_cli [label="mgr/orchestrator"]
+            orchestrator [label="Orchestrator Interface"]
+            cephadm [label="mgr/cephadm"]
+
+            label = "ceph-mgr";
+        }
+
+        volumes -> orchestrator
+        dashboard -> orchestrator
+        orchestrator_cli -> orchestrator
+        orchestrator -> rook -> rook_io
+        orchestrator -> cephadm
+
+
+        rook_io [label="Rook"]
+
+        rankdir="TB";
+    }
+
+Behind all the abstraction, the purpose of orchestrator modules is simple:
+enable Ceph to do things like discover available hardware, create and
+destroy OSDs, and run MDS and RGW services.
+
+A tutorial is not included here: for full and concrete examples, see
+the existing implemented orchestrator modules in the Ceph source tree.
+
+Glossary
+--------
+
+Stateful service
+  a daemon that uses local storage, such as OSD or mon.
+
+Stateless service
+  a daemon that doesn't use any local storage, such
+  as an MDS, RGW, nfs-ganesha, iSCSI gateway.
+
+Label
+  arbitrary string tags that may be applied by administrators
+  to hosts.  Typically administrators use labels to indicate
+  which hosts should run which kinds of service.  Labels are
+  advisory (from human input) and do not guarantee that hosts
+  have particular physical capabilities.
+
+Drive group
+  collection of block devices with common/shared OSD
+  formatting (typically one or more SSDs acting as
+  journals/dbs for a group of HDDs).
+
+Placement
+  choice of which host is used to run a service.
+
+Key Concepts
+------------
+
+The underlying orchestrator remains the source of truth for information
+about whether a service is running, what is running where, which
+hosts are available, etc.  Orchestrator modules should avoid taking
+any internal copies of this information, and read it directly from
+the orchestrator backend as much as possible.
+
+Bootstrapping hosts and adding them to the underlying orchestration
+system is outside the scope of Ceph's orchestrator interface.  Ceph
+can only work on hosts when the orchestrator is already aware of them.
+
+Where possible, placement of stateless services should be left up to the
+orchestrator.
+
+Completions and batching
+------------------------
+
+All methods that read or modify the state of the system can potentially
+be long running. Therefore the module needs to schedule those operations.
+
+Each orchestrator module implements its own underlying mechanisms
+for completions.  This might involve running the underlying operations
+in threads, or batching the operations up before later executing
+in one go in the background.  If implementing such a batching pattern, the
+module would do no work on any operation until it appeared in a list
+of completions passed into *process*.
+
+Error Handling
+--------------
+
+The main goal of error handling within orchestrator modules is to provide debug information to
+assist users when dealing with deployment errors.
+
+.. autoclass:: OrchestratorError
+.. autoclass:: NoOrchestrator
+.. autoclass:: OrchestratorValidationError
+
+
+In detail, orchestrators need to explicitly deal with different kinds of errors:
+
+1. No orchestrator configured
+
+   See :class:`NoOrchestrator`.
+
+2. An orchestrator doesn't implement a specific method.
+
+   For example, an Orchestrator doesn't support ``add_host``.
+
+   In this case, a ``NotImplementedError`` is raised.
+
+3. Missing features within implemented methods.
+
+   E.g. optional parameters to a command that are not supported by the
+   backend (e.g. the hosts field in :func:`Orchestrator.apply_mons` command with the rook backend).
+
+   See :class:`OrchestratorValidationError`.
+
+4. Input validation errors
+
+   The ``orchestrator`` module and other calling modules are supposed to
+   provide meaningful error messages.
+
+   See :class:`OrchestratorValidationError`.
+
+5. Errors when actually executing commands
+
+   The resulting Completion should contain an error string that assists in understanding the
+   problem. In addition, :func:`Completion.is_errored` is set to ``True``
+
+6. Invalid configuration in the orchestrator modules
+
+   This can be tackled similar to 5.
+
+
+All other errors are unexpected orchestrator issues and thus should raise an exception that are then
+logged into the mgr log file. If there is a completion object at that point,
+:func:`Completion.result` may contain an error message.
+
+
+Excluded functionality
+----------------------
+
+- Ceph's orchestrator interface is not a general purpose framework for
+  managing linux servers -- it is deliberately constrained to manage
+  the Ceph cluster's services only.
+- Multipathed storage is not handled (multipathing is unnecessary for
+  Ceph clusters).  Each drive is assumed to be visible only on
+  a single host.
+
+Host management
+---------------
+
+.. automethod:: Orchestrator.add_host
+.. automethod:: Orchestrator.remove_host
+.. automethod:: Orchestrator.get_hosts
+.. automethod:: Orchestrator.update_host_addr
+.. automethod:: Orchestrator.add_host_label
+.. automethod:: Orchestrator.remove_host_label
+
+.. autoclass:: HostSpec
+
+Devices
+-------
+
+.. automethod:: Orchestrator.get_inventory
+.. autoclass:: InventoryFilter
+
+.. py:currentmodule:: ceph.deployment.inventory
+
+.. autoclass:: Devices
+   :members:
+
+.. autoclass:: Device
+   :members:
+
+.. py:currentmodule:: orchestrator
+
+Placement
+---------
+
+A :ref:`orchestrator-cli-placement-spec` defines the placement of
+daemons of a specific service.
+
+In general, stateless services do not require any specific placement
+rules as they can run anywhere that sufficient system resources
+are available. However, some orchestrators may not include the
+functionality to choose a location in this way. Optionally, you can
+specify a location when creating a stateless service.
+
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: PlacementSpec
+   :members:
+
+.. py:currentmodule:: orchestrator
+
+
+Services
+--------
+
+.. autoclass:: ServiceDescription
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: ServiceSpec
+  :members:
+  :private-members:
+  :noindex:
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.describe_service
+
+.. automethod:: Orchestrator.service_action
+.. automethod:: Orchestrator.remove_service
+
+
+Daemons
+-------
+
+.. automethod:: Orchestrator.list_daemons
+.. automethod:: Orchestrator.remove_daemons
+.. automethod:: Orchestrator.daemon_action
+
+.. autoclass:: DaemonDescription
+.. autoclass:: DaemonDescriptionStatus
+
+OSD management
+--------------
+
+.. automethod:: Orchestrator.create_osds
+
+.. automethod:: Orchestrator.blink_device_light
+.. autoclass:: DeviceLightLoc
+
+.. _orchestrator-osd-replace:
+
+OSD Replacement
+^^^^^^^^^^^^^^^
+
+See :ref:`rados-replacing-an-osd` for the underlying process.
+
+Replacing OSDs is fundamentally a two-staged process, as users need to
+physically replace drives. The orchestrator therefore exposes this two-staged process.
+
+Phase one is a call to :meth:`Orchestrator.remove_daemons` with ``destroy=True`` in order to mark
+the OSD as destroyed.
+
+
+Phase two is a call to  :meth:`Orchestrator.create_osds` with a Drive Group with
+
+.. py:currentmodule:: ceph.deployment.drive_group
+
+:attr:`DriveGroupSpec.osd_id_claims` set to the destroyed OSD ids.
+
+.. py:currentmodule:: orchestrator
+
+Services
+--------
+
+.. automethod:: Orchestrator.add_daemon
+.. automethod:: Orchestrator.apply_mon
+.. automethod:: Orchestrator.apply_mgr
+.. automethod:: Orchestrator.apply_mds
+.. automethod:: Orchestrator.apply_rbd_mirror
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: RGWSpec
+  :noindex:
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.apply_rgw
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: NFSServiceSpec
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.apply_nfs
+
+Upgrades
+--------
+
+.. automethod:: Orchestrator.upgrade_available
+.. automethod:: Orchestrator.upgrade_start
+.. automethod:: Orchestrator.upgrade_status
+.. autoclass:: UpgradeStatusSpec
+
+Utility
+-------
+
+.. automethod:: Orchestrator.available
+.. automethod:: Orchestrator.get_feature_set
+
+Client Modules
+--------------
+
+.. autoclass:: OrchestratorClientMixin
+   :members:
diff --git a/doc/mgr/progress.rst b/doc/mgr/progress.rst
new file mode 100644
index 000000000..77a8a408a
--- /dev/null
+++ b/doc/mgr/progress.rst
@@ -0,0 +1,58 @@
+Progress Module
+===============
+
+The progress module is used to inform users about the recovery progress of PGs
+(Placement Groups) that are affected by events such as (1) OSDs being marked
+in or out and (2) ``pg_autoscaler`` trying to match the target PG number.
+
+The ``ceph -s`` command returns something called " Global Recovery Progress",
+which reports the overall recovery progress of PGs and is based on the number
+of PGs that are in the ``active+clean`` state.
+
+Enabling
+--------
+
+The *progress* module is enabled by default, but it can be enabled manually by
+running the following command:
+
+.. prompt:: bash #
+
+  ceph progress on
+
+The module can be disabled at anytime by running the following command:
+
+.. prompt:: bash #
+
+  ceph progress off
+
+Commands
+--------
+
+Show the summary of all the ongoing and completed events and their duration:
+
+.. prompt:: bash #
+
+  ceph progress
+
+Show the summary of ongoing and completed events in JSON format:
+
+.. prompt:: bash #
+
+  ceph progress json
+
+Clear all ongoing and completed events:
+
+.. prompt:: bash #
+
+  ceph progress clear
+
+PG Recovery Event
+-----------------
+
+An event for each PG affected by recovery event can be shown in
+`ceph progress` This is completely optional, and disabled by default
+due to CPU overheard:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/progress/allow_pg_recovery_event true
diff --git a/doc/mgr/prometheus.rst b/doc/mgr/prometheus.rst
new file mode 100644
index 000000000..25a7b0d08
--- /dev/null
+++ b/doc/mgr/prometheus.rst
@@ -0,0 +1,446 @@
+.. _mgr-prometheus:
+
+=================
+Prometheus Module
+=================
+
+Provides a Prometheus exporter to pass on Ceph performance counters
+from the collection point in ceph-mgr.  Ceph-mgr receives MMgrReport
+messages from all MgrClient processes (mons and OSDs, for instance)
+with performance counter schema data and actual counter data, and keeps
+a circular buffer of the last N samples.  This module creates an HTTP
+endpoint (like all Prometheus exporters) and retrieves the latest sample
+of every counter when polled (or "scraped" in Prometheus terminology).
+The HTTP path and query parameters are ignored; all extant counters
+for all reporting entities are returned in text exposition format.
+(See the Prometheus `documentation <https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details>`_.)
+
+Enabling prometheus output
+==========================
+
+The *prometheus* module is enabled with:
+
+.. prompt:: bash $
+
+   ceph mgr module enable prometheus
+
+Configuration
+-------------
+
+.. note::
+
+    The Prometheus manager module needs to be restarted for configuration changes to be applied.
+
+.. mgr_module:: prometheus
+.. confval:: server_addr
+.. confval:: server_port
+.. confval:: scrape_interval
+.. confval:: cache
+.. confval:: stale_cache_strategy
+.. confval:: rbd_stats_pools
+.. confval:: rbd_stats_pools_refresh_interval
+.. confval:: standby_behaviour
+.. confval:: standby_error_status_code
+.. confval:: exclude_perf_counters
+
+By default the module will accept HTTP requests on port ``9283`` on all IPv4
+and IPv6 addresses on the host.  The port and listen address are both
+configurable with ``ceph config set``, with keys
+``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``.  This port
+is registered with Prometheus's `registry
+<https://github.com/prometheus/prometheus/wiki/Default-port-allocations>`_.
+
+.. prompt:: bash $
+   
+   ceph config set mgr mgr/prometheus/server_addr 0.0.0.
+   ceph config set mgr mgr/prometheus/server_port 9283
+
+.. warning::
+
+    The :confval:`mgr/prometheus/scrape_interval` of this module should always be set to match
+    Prometheus' scrape interval to work properly and not cause any issues.
+
+The scrape interval in the module is used for caching purposes
+and to determine when a cache is stale.
+
+It is not recommended to use a scrape interval below 10 seconds.  It is
+recommended to use 15 seconds as scrape interval, though, in some cases it
+might be useful to increase the scrape interval.
+
+To set a different scrape interval in the Prometheus module, set
+``scrape_interval`` to the desired value:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/scrape_interval 20
+
+On large clusters (>1000 OSDs), the time to fetch the metrics may become
+significant.  Without the cache, the Prometheus manager module could, especially
+in conjunction with multiple Prometheus instances, overload the manager and lead
+to unresponsive or crashing Ceph manager instances.  Hence, the cache is enabled
+by default.  This means that there is a possibility that the cache becomes
+stale.  The cache is considered stale when the time to fetch the metrics from
+Ceph exceeds the configured :confval:`mgr/prometheus/scrape_interval`.
+
+If that is the case, **a warning will be logged** and the module will either
+
+* respond with a 503 HTTP status code (service unavailable) or,
+* it will return the content of the cache, even though it might be stale.
+
+This behavior can be configured. By default, it will return a 503 HTTP status
+code (service unavailable). You can set other options using the ``ceph config
+set`` commands.
+
+To tell the module to respond with possibly stale data, set it to ``return``:
+
+.. prompt:: bash $
+
+    ceph config set mgr mgr/prometheus/stale_cache_strategy return
+
+To tell the module to respond with "service unavailable", set it to ``fail``:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/stale_cache_strategy fail
+
+If you are confident that you don't require the cache, you can disable it:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/cache false
+
+If you are using the prometheus module behind some kind of reverse proxy or
+loadbalancer, you can simplify discovering the active instance by switching
+to ``error``-mode:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/standby_behaviour error
+
+If set, the prometheus module will respond with a HTTP error when requesting ``/``
+from the standby instance. The default error code is 500, but you can configure
+the HTTP response code with:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/standby_error_status_code 503
+
+Valid error codes are between 400-599.
+
+To switch back to the default behaviour, simply set the config key to ``default``:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/standby_behaviour default
+
+.. _prometheus-rbd-io-statistics:
+
+Ceph Health Checks
+------------------
+
+The mgr/prometheus module also tracks and maintains a history of Ceph health checks,
+exposing them to the Prometheus server as discrete metrics. This allows Prometheus
+alert rules to be configured for specific health check events.
+
+The metrics take the following form;
+
+::
+
+    # HELP ceph_health_detail healthcheck status by type (0=inactive, 1=active)
+    # TYPE ceph_health_detail gauge
+    ceph_health_detail{name="OSDMAP_FLAGS",severity="HEALTH_WARN"} 0.0
+    ceph_health_detail{name="OSD_DOWN",severity="HEALTH_WARN"} 1.0
+    ceph_health_detail{name="PG_DEGRADED",severity="HEALTH_WARN"} 1.0
+
+The health check history is made available through the following commands;
+
+::
+
+    healthcheck history ls [--format {plain|json|json-pretty}]
+    healthcheck history clear
+
+The ``ls`` command provides an overview of the health checks that the cluster has
+encountered, or since the last ``clear`` command was issued. The example below;
+
+::
+
+    [ceph: root@c8-node1 /]# ceph healthcheck history ls
+    Healthcheck Name          First Seen (UTC)      Last seen (UTC)       Count  Active
+    OSDMAP_FLAGS              2021/09/16 03:17:47   2021/09/16 22:07:40       2    No
+    OSD_DOWN                  2021/09/17 00:11:59   2021/09/17 00:11:59       1   Yes
+    PG_DEGRADED               2021/09/17 00:11:59   2021/09/17 00:11:59       1   Yes
+    3 health check(s) listed
+
+
+RBD IO statistics
+-----------------
+
+The module can optionally collect RBD per-image IO statistics by enabling
+dynamic OSD performance counters. The statistics are gathered for all images
+in the pools that are specified in the ``mgr/prometheus/rbd_stats_pools``
+configuration parameter. The parameter is a comma or space separated list
+of ``pool[/namespace]`` entries. If the namespace is not specified the
+statistics are collected for all namespaces in the pool.
+
+Example to activate the RBD-enabled pools ``pool1``, ``pool2`` and ``poolN``:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/rbd_stats_pools "pool1,pool2,poolN"
+
+The wildcard can be used to indicate all pools or namespaces:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/rbd_stats_pools "*"
+
+The module makes the list of all available images scanning the specified
+pools and namespaces and refreshes it periodically. The period is
+configurable via the ``mgr/prometheus/rbd_stats_pools_refresh_interval``
+parameter (in sec) and is 300 sec (5 minutes) by default. The module will
+force refresh earlier if it detects statistics from a previously unknown
+RBD image.
+
+Example to turn up the sync interval to 10 minutes:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/rbd_stats_pools_refresh_interval 600
+
+Ceph daemon performance counters metrics
+-----------------------------------------
+
+With the introduction of ``ceph-exporter`` daemon, the prometheus module will no longer export Ceph daemon
+perf counters as prometheus metrics by default. However, one may re-enable exporting these metrics by setting
+the module option ``exclude_perf_counters`` to ``false``:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/prometheus/exclude_perf_counters false
+
+Ceph daemon performance counters metrics
+-----------------------------------------
+
+With the introduction of ``ceph-exporter`` daemon, the prometheus module will no longer export Ceph daemon
+perf counters as prometheus metrics by default. However, one may re-enable exporting these metrics by setting
+the module option ``exclude_perf_counters`` to ``false``::
+
+    ceph config set mgr mgr/prometheus/exclude_perf_counters false
+
+Statistic names and labels
+==========================
+
+The names of the stats are exactly as Ceph names them, with
+illegal characters ``.``, ``-`` and ``::`` translated to ``_``,
+and ``ceph_`` prefixed to all names.
+
+
+All *daemon* statistics have a ``ceph_daemon`` label such as "osd.123"
+that identifies the type and ID of the daemon they come from.  Some
+statistics can come from different types of daemon, so when querying
+e.g. an OSD's RocksDB stats, you would probably want to filter
+on ceph_daemon starting with "osd" to avoid mixing in the monitor
+rocksdb stats.
+
+
+The *cluster* statistics (i.e. those global to the Ceph cluster)
+have labels appropriate to what they report on.  For example,
+metrics relating to pools have a ``pool_id`` label.
+
+
+The long running averages that represent the histograms from core Ceph
+are represented by a pair of ``<name>_sum`` and ``<name>_count`` metrics.
+This is similar to how histograms are represented in `Prometheus <https://prometheus.io/docs/concepts/metric_types/#histogram>`_
+and they can also be treated `similarly <https://prometheus.io/docs/practices/histograms/>`_.
+
+Pool and OSD metadata series
+----------------------------
+
+Special series are output to enable displaying and querying on
+certain metadata fields.
+
+Pools have a ``ceph_pool_metadata`` field like this:
+
+::
+
+    ceph_pool_metadata{pool_id="2",name="cephfs_metadata_a"} 1.0
+
+OSDs have a ``ceph_osd_metadata`` field like this:
+
+::
+
+    ceph_osd_metadata{cluster_addr="172.21.9.34:6802/19096",device_class="ssd",ceph_daemon="osd.0",public_addr="172.21.9.34:6801/19096",weight="1.0"} 1.0
+
+
+Correlating drive statistics with node_exporter
+-----------------------------------------------
+
+The prometheus output from Ceph is designed to be used in conjunction
+with the generic host monitoring from the Prometheus node_exporter.
+
+To enable correlation of Ceph OSD statistics with node_exporter's
+drive statistics, special series are output like this:
+
+::
+
+    ceph_disk_occupation_human{ceph_daemon="osd.0", device="sdd", exported_instance="myhost"}
+
+To use this to get disk statistics by OSD ID, use either the ``and`` operator or
+the ``*`` operator in your prometheus query. All metadata metrics (like ``
+ceph_disk_occupation_human`` have the value 1 so they act neutral with ``*``. Using ``*``
+allows to use ``group_left`` and ``group_right`` grouping modifiers, so that
+the resulting metric has additional labels from one side of the query.
+
+See the
+`prometheus documentation`__ for more information about constructing queries.
+
+__ https://prometheus.io/docs/prometheus/latest/querying/basics
+
+The goal is to run a query like
+
+::
+
+    rate(node_disk_written_bytes_total[30s]) and
+    on (device,instance) ceph_disk_occupation_human{ceph_daemon="osd.0"}
+
+Out of the box the above query will not return any metrics since the ``instance`` labels of
+both metrics don't match. The ``instance`` label of ``ceph_disk_occupation_human``
+will be the currently active MGR node.
+
+The following two section outline two approaches to remedy this.
+
+.. note::
+
+    If you need to group on the `ceph_daemon` label instead of `device` and
+    `instance` labels, using `ceph_disk_occupation_human` may not work reliably.
+    It is advised that you use `ceph_disk_occupation` instead.
+
+    The difference is that `ceph_disk_occupation_human` may group several OSDs
+    into the value of a single `ceph_daemon` label in cases where multiple OSDs
+    share a disk.
+
+Use label_replace
+=================
+
+The ``label_replace`` function (cp.
+`label_replace documentation <https://prometheus.io/docs/prometheus/latest/querying/functions/#label_replace>`_)
+can add a label to, or alter a label of, a metric within a query.
+
+To correlate an OSD and its disks write rate, the following query can be used:
+
+::
+
+    label_replace(
+        rate(node_disk_written_bytes_total[30s]),
+        "exported_instance",
+        "$1",
+        "instance",
+        "(.*):.*"
+    ) and on (device, exported_instance) ceph_disk_occupation_human{ceph_daemon="osd.0"}
+
+Configuring Prometheus server
+=============================
+
+honor_labels
+------------
+
+To enable Ceph to output properly-labeled data relating to any host,
+use the ``honor_labels`` setting when adding the ceph-mgr endpoints
+to your prometheus configuration.
+
+This allows Ceph to export the proper ``instance`` label without prometheus
+overwriting it. Without this setting, Prometheus applies an ``instance`` label
+that includes the hostname and port of the endpoint that the series came from.
+Because Ceph clusters have multiple manager daemons, this results in an
+``instance`` label that changes spuriously when the active manager daemon
+changes.
+
+If this is undesirable a custom ``instance`` label can be set in the
+Prometheus target configuration: you might wish to set it to the hostname
+of your first mgr daemon, or something completely arbitrary like "ceph_cluster".
+
+node_exporter hostname labels
+-----------------------------
+
+Set your ``instance`` labels to match what appears in Ceph's OSD metadata
+in the ``instance`` field.  This is generally the short hostname of the node.
+
+This is only necessary if you want to correlate Ceph stats with host stats,
+but you may find it useful to do it in all cases in case you want to do
+the correlation in the future.
+
+Example configuration
+---------------------
+
+This example shows a single node configuration running ceph-mgr and
+node_exporter on a server called ``senta04``. Note that this requires one
+to add an appropriate and unique ``instance`` label to each ``node_exporter`` target.
+
+This is just an example: there are other ways to configure prometheus
+scrape targets and label rewrite rules.
+
+prometheus.yml
+~~~~~~~~~~~~~~
+
+::
+
+    global:
+      scrape_interval:     15s
+      evaluation_interval: 15s
+
+    scrape_configs:
+      - job_name: 'node'
+        file_sd_configs:
+          - files:
+            - node_targets.yml
+      - job_name: 'ceph'
+        honor_labels: true
+        file_sd_configs:
+          - files:
+            - ceph_targets.yml
+
+
+ceph_targets.yml
+~~~~~~~~~~~~~~~~
+
+
+::
+
+    [
+        {
+            "targets": [ "senta04.mydomain.com:9283" ],
+            "labels": {}
+        }
+    ]
+
+
+node_targets.yml
+~~~~~~~~~~~~~~~~
+
+::
+
+    [
+        {
+            "targets": [ "senta04.mydomain.com:9100" ],
+            "labels": {
+                "instance": "senta04"
+            }
+        }
+    ]
+
+
+Notes
+=====
+
+Counters and gauges are exported; currently histograms and long-running
+averages are not.  It's possible that Ceph's 2-D histograms could be
+reduced to two separate 1-D histograms, and that long-running averages
+could be exported as Prometheus' Summary type.
+
+Timestamps, as with many Prometheus exporters, are established by
+the server's scrape time (Prometheus expects that it is polling the
+actual counter process synchronously).  It is possible to supply a
+timestamp along with the stat report, but the Prometheus team strongly
+advises against this.  This means that timestamps will be delayed by
+an unpredictable amount; it's not clear if this will be problematic,
+but it's worth knowing about.
diff --git a/doc/mgr/restful.rst b/doc/mgr/restful.rst
new file mode 100644
index 000000000..d684399fc
--- /dev/null
+++ b/doc/mgr/restful.rst
@@ -0,0 +1,189 @@
+Restful Module
+==============
+
+RESTful module offers the REST API access to the status of the cluster
+over an SSL-secured connection.
+
+Enabling
+--------
+
+The *restful* module is enabled with::
+
+  ceph mgr module enable restful
+
+You will also need to configure an SSL certificate below before the
+API endpoint is available.  By default the module will accept HTTPS
+requests on port ``8003`` on all IPv4 and IPv6 addresses on the host.
+
+Securing
+--------
+
+All connections to *restful* are secured with SSL.  You can generate a
+self-signed certificate with the command::
+
+  ceph restful create-self-signed-cert
+
+Note that with a self-signed certificate most clients will need a flag
+to allow a connection and/or suppress warning messages.  For example,
+if the ``ceph-mgr`` daemon is on the same host,::
+
+  curl -k https://localhost:8003/
+
+To properly secure a deployment, a certificate that is signed by the
+organization's certificate authority should be used.  For example, a key pair
+can be generated with a command similar to::
+
+  openssl req -new -nodes -x509 \
+    -subj "/O=IT/CN=ceph-mgr-restful" \
+    -days 3650 -keyout restful.key -out restful.crt -extensions v3_ca
+
+The ``restful.crt`` should then be signed by your organization's CA
+(certificate authority).  Once that is done, you can set it with::
+
+  ceph config-key set mgr/restful/$name/crt -i restful.crt
+  ceph config-key set mgr/restful/$name/key -i restful.key
+
+where ``$name`` is the name of the ``ceph-mgr`` instance (usually the
+hostname). If all manager instances are to share the same certificate,
+you can leave off the ``$name`` portion::
+
+  ceph config-key set mgr/restful/crt -i restful.crt
+  ceph config-key set mgr/restful/key -i restful.key
+
+
+Configuring IP and port
+-----------------------
+
+Like any other RESTful API endpoint, *restful* binds to an IP and
+port.  By default, the currently active ``ceph-mgr`` daemon will bind
+to port 8003 and any available IPv4 or IPv6 address on the host.
+
+Since each ``ceph-mgr`` hosts its own instance of *restful*, it may
+also be necessary to configure them separately. The IP and port
+can be changed via the configuration key facility::
+
+  ceph config set mgr mgr/restful/$name/server_addr $IP
+  ceph config set mgr mgr/restful/$name/server_port $PORT
+
+where ``$name`` is the ID of the ceph-mgr daemon (usually the hostname).
+
+These settings can also be configured cluster-wide and not manager
+specific.  For example,::
+
+  ceph config set mgr mgr/restful/server_addr $IP
+  ceph config set mgr mgr/restful/server_port $PORT
+
+If the port is not configured, *restful* will bind to port ``8003``.
+If the address it not configured, the *restful* will bind to ``::``,
+which corresponds to all available IPv4 and IPv6 addresses.
+
+.. _creating-an-api-user:
+
+Creating an API User
+-----------------------
+
+To create an API user, please run the following command::
+
+  ceph restful create-key <username>
+
+Replace ``<username>`` with the desired name of the user. For example, to create a user named
+``api``::
+
+  $ ceph restful create-key api
+  52dffd92-a103-4a10-bfce-5b60f48f764e
+
+The UUID generated from ``ceph restful create-key api`` acts as the key for the user.
+
+To list all of your API keys, please run the following command::
+
+  ceph restful list-keys
+
+The ``ceph restful list-keys`` command will output in JSON::
+
+  {
+  	"api": "52dffd92-a103-4a10-bfce-5b60f48f764e"
+  }
+
+You can use ``curl`` in order to test your user with the API. Here is an example::
+
+  curl -k https://api:52dffd92-a103-4a10-bfce-5b60f48f764e@<ceph-mgr>:<port>/server
+
+In the case above, we are using ``GET`` to fetch information from the ``server`` endpoint.
+
+Load balancer
+-------------
+
+Please note that *restful* will *only* start on the manager which
+is active at that moment. Query the Ceph cluster status to see which
+manager is active (e.g., ``ceph mgr dump``).  In order to make the
+API available via a consistent URL regardless of which manager
+daemon is currently active, you may want to set up a load balancer
+front-end to direct traffic to whichever manager endpoint is
+available.
+
+Available methods
+-----------------
+
+You can navigate to the ``/doc`` endpoint for full list of available
+endpoints and HTTP methods implemented for each endpoint.
+
+For example, if you want to use the PATCH method of the ``/osd/<id>``
+endpoint to set the state ``up`` of the OSD id ``1``, you can use the
+following curl command::
+
+  echo -En '{"up": true}' | curl --request PATCH --data @- --silent --insecure --user <user> 'https://<ceph-mgr>:<port>/osd/1'
+
+or you can use python to do so::
+
+  $ python
+  >> import requests
+  >> result = requests.patch(
+         'https://<ceph-mgr>:<port>/osd/1',
+         json={"up": True},
+         auth=("<user>", "<password>")
+     )
+  >> print result.json()
+
+Some of the other endpoints implemented in the *restful* module include
+
+* ``/config/cluster``: **GET**
+* ``/config/osd``: **GET**, **PATCH**
+* ``/crush/rule``: **GET**
+* ``/mon``: **GET**
+* ``/osd``: **GET**
+* ``/pool``: **GET**, **POST**
+* ``/pool/<arg>``: **DELETE**, **GET**, **PATCH**
+* ``/request``: **DELETE**, **GET**, **POST**
+* ``/request/<arg>``: **DELETE**, **GET**
+* ``/server``: **GET**
+
+The ``/request`` endpoint
+-------------------------
+
+You can use the ``/request`` endpoint to poll the state of a request
+you scheduled with any **DELETE**, **POST** or **PATCH** method. These
+methods are by default asynchronous since it may take longer for them
+to finish execution. You can modify this behaviour by appending
+``?wait=1`` to the request url. The returned request will then always
+be completed.
+
+The **POST** method of the ``/request`` method provides a passthrough
+for the ceph mon commands as defined in ``src/mon/MonCommands.h``.
+Let's consider the following command::
+
+  COMMAND("osd ls " \
+          "name=epoch,type=CephInt,range=0,req=false", \
+          "show all OSD ids", "osd", "r", "cli,rest")
+
+The **prefix** is **osd ls**. The optional argument's name is **epoch**
+and it is of type ``CephInt``, i.e. ``integer``. This means that you
+need to do the following **POST** request to schedule the command::
+
+  $ python
+  >> import requests
+  >> result = requests.post(
+         'https://<ceph-mgr>:<port>/request',
+         json={'prefix': 'osd ls', 'epoch': 0},
+         auth=("<user>", "<password>")
+     )
+  >> print result.json()
diff --git a/doc/mgr/rgw.rst b/doc/mgr/rgw.rst
new file mode 100644
index 000000000..a3f53280a
--- /dev/null
+++ b/doc/mgr/rgw.rst
@@ -0,0 +1,141 @@
+.. _mgr-rgw-module:
+
+RGW Module
+============
+The rgw module provides a simple interface to deploy RGW multisite.
+It helps with bootstrapping and configuring RGW realm, zonegroup and
+the different related entities.
+
+Enabling
+--------
+
+The *rgw* module is enabled with::
+
+  ceph mgr module enable rgw
+
+
+RGW Realm Operations
+-----------------------
+
+Bootstrapping RGW realm creates a new RGW realm entity, a new zonegroup,
+and a new zone. It configures a new system user that can be used for
+multisite sync operations. Under the hood this module instructs the
+orchestrator to create and deploy the corresponding RGW daemons. The module
+supports both passing the arguments through the cmd line or as a spec file:
+
+.. prompt:: bash #
+
+  ceph rgw realm bootstrap [--realm-name] [--zonegroup-name] [--zone-name] [--port] [--placement] [--start-radosgw]
+
+The command supports providing the configuration through a spec file (`-i option`):
+
+.. prompt:: bash #
+
+  ceph rgw realm bootstrap -i myrgw.yaml
+
+Following is an example of RGW multisite spec file:
+
+.. code-block:: yaml
+
+  rgw_realm: myrealm
+  rgw_zonegroup: myzonegroup
+  rgw_zone: myzone
+  placement:
+    hosts:
+     - ceph-node-1
+     - ceph-node-2
+  spec:
+    rgw_frontend_port: 5500
+
+.. note:: The spec file used by RGW has the same format as the one used by the orchestrator. Thus,
+          the user can provide any orchestration supported rgw parameters including advanced
+          configuration features such as SSL certificates etc.
+
+Users can also specify custom zone endpoints in the spec (or through the cmd line). In this case, no
+cephadm daemons will be launched. Following is an example RGW spec file with zone endpoints:
+
+.. code-block:: yaml
+
+  rgw_realm: myrealm
+  rgw_zonegroup: myzonegroup
+  rgw_zone: myzone
+  zone_endpoints: http://<rgw_host1>:<rgw_port1>, http://<rgw_host2>:<rgw_port2>
+
+
+Realm Credentials Token
+-----------------------
+
+Users can list the available tokens for the created (or already existing) realms.
+The token is a base64 string that encapsulates the realm information and its
+master zone endpoint authentication data. Following is an example of
+the `ceph rgw realm tokens` output:
+
+.. prompt:: bash #
+
+  ceph rgw realm tokens | jq
+
+.. code-block:: json
+
+  [
+    {
+      "realm": "myrealm1",
+      "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFs....NHlBTFhoIgp9"
+    },
+    {
+      "realm": "myrealm2",
+      "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFs....RUU12ZDB0Igp9"
+    }
+  ]
+
+User can use the token to pull a realm to create secondary zone on a
+different cluster that syncs with the master zone on the primary cluster
+by using `ceph rgw zone create` command and providing the corresponding token.
+
+Following is an example of zone spec file:
+
+.. code-block:: yaml
+
+  rgw_zone: my-secondary-zone
+  rgw_realm_token: <token>
+  placement:
+    hosts:
+     - ceph-node-1
+     - ceph-node-2
+  spec:
+    rgw_frontend_port: 5500
+
+
+.. prompt:: bash #
+
+  ceph rgw zone create -i zone-spec.yaml
+
+.. note:: The spec file used by RGW has the same format as the one used by the orchestrator. Thus,
+          the user can provide any orchestration supported rgw parameters including advanced
+          configuration features such as SSL certificates etc.
+
+Commands
+--------
+::
+
+  ceph rgw realm bootstrap -i spec.yaml
+
+Create a new realm + zonegroup + zone and deploy rgw daemons via the
+orchestrator using the information specified in the YAML file.
+
+::
+
+  ceph rgw realm tokens
+
+List the tokens of all the available realms
+
+::
+
+  ceph rgw zone create -i spec.yaml
+
+Join an existing realm by creating a new secondary zone (using the realm token)
+
+::
+
+  ceph rgw admin [*]
+
+RGW admin command
diff --git a/doc/mgr/rook.rst b/doc/mgr/rook.rst
new file mode 100644
index 000000000..1ae369623
--- /dev/null
+++ b/doc/mgr/rook.rst
@@ -0,0 +1,39 @@
+
+.. _mgr-rook:
+
+====
+Rook
+====
+
+Rook (https://rook.io/) is an orchestration tool that can run Ceph inside
+a Kubernetes cluster.
+
+The ``rook`` module provides integration between Ceph's orchestrator framework
+(used by modules such as ``dashboard`` to control cluster services) and
+Rook.
+
+Orchestrator modules only provide services to other modules, which in turn
+provide user interfaces.  To try out the rook module, you might like
+to use the :ref:`Orchestrator CLI <orchestrator-cli-module>` module.
+
+Requirements
+------------
+
+- Running ceph-mon and ceph-mgr services that were set up with Rook in
+  Kubernetes.
+- Rook 0.9 or newer.
+
+Configuration
+-------------
+
+Because a Rook cluster's ceph-mgr daemon is running as a Kubernetes pod, 
+the ``rook`` module can connect to the Kubernetes API without any explicit
+configuration.
+
+Development
+-----------
+
+If you are a developer, please see :ref:`kubernetes-dev` for instructions
+on setting up a development environment to work with this.
+
+
diff --git a/doc/mgr/status-card-open.png b/doc/mgr/status-card-open.png
new file mode 100644
index 000000000..4ea20921b
--- /dev/null
+++ b/doc/mgr/status-card-open.png
diff --git a/doc/mgr/telegraf.rst b/doc/mgr/telegraf.rst
new file mode 100644
index 000000000..781ff5592
--- /dev/null
+++ b/doc/mgr/telegraf.rst
@@ -0,0 +1,91 @@
+===============
+Telegraf Module 
+===============
+The Telegraf module collects and sends statistics series to a Telegraf agent.
+
+The Telegraf agent can buffer, aggregate, parse and process the data before
+sending it to an output which can be InfluxDB, ElasticSearch and many more.
+
+Currently the only way to send statistics to Telegraf from this module is to
+use the socket listener. The module can send statistics over UDP, TCP or
+a UNIX socket.
+
+The Telegraf module was introduced in the 13.x *Mimic* release.
+
+--------
+Enabling 
+--------
+
+To enable the module, use the following command:
+
+::
+
+    ceph mgr module enable telegraf
+
+If you wish to subsequently disable the module, you can use the corresponding
+*disable* command:
+
+::
+
+    ceph mgr module disable telegraf
+
+-------------
+Configuration 
+-------------
+
+For the telegraf module to send statistics to a Telegraf agent it is
+required to configure the address to send the statistics to.
+
+Set configuration values using the following command:
+
+::
+
+    ceph telegraf config-set <key> <value>
+
+
+The most important settings are ``address`` and ``interval``.
+
+For example, a typical configuration might look like this:
+
+::
+
+    ceph telegraf config-set address udp://:8094
+    ceph telegraf config-set interval 10
+    
+The default values for these configuration keys are:
+
+- address: unixgram:///tmp/telegraf.sock
+- interval: 15
+
+----------------
+Socket Listener
+----------------
+The module only supports sending data to Telegraf through the socket listener
+of the Telegraf module using the Influx data format.
+
+A typical Telegraf configuration might be
+
+::
+
+    [[inputs.socket_listener]]
+    # service_address = "tcp://:8094"
+    # service_address = "tcp://127.0.0.1:http"
+    # service_address = "tcp4://:8094"
+    # service_address = "tcp6://:8094"
+    # service_address = "tcp6://[2001:db8::1]:8094"
+    service_address = "udp://:8094"
+    # service_address = "udp4://:8094"
+    # service_address = "udp6://:8094"
+    # service_address = "unix:///tmp/telegraf.sock"
+    # service_address = "unixgram:///tmp/telegraf.sock"
+    data_format = "influx"
+
+In this case the `address` configuration option for the module would need to be set
+to:
+
+::
+
+  udp://:8094
+
+
+Refer to the Telegraf documentation for more configuration options.
diff --git a/doc/mgr/telemetry.rst b/doc/mgr/telemetry.rst
new file mode 100644
index 000000000..90d45766c
--- /dev/null
+++ b/doc/mgr/telemetry.rst
@@ -0,0 +1,292 @@
+.. _telemetry:
+
+Telemetry Module
+================
+
+The telemetry module sends anonymous data about the cluster back to the Ceph
+developers to help understand how Ceph is used and what problems users may
+be experiencing.
+
+This data is visualized on `public dashboards <https://telemetry-public.ceph.com/>`_
+that allow the community to quickly see summary statistics on how many clusters
+are reporting, their total capacity and OSD count, and version distribution
+trends.
+
+Channels
+--------
+
+The telemetry report is broken down into several "channels", each with
+a different type of information.  Assuming telemetry has been enabled,
+individual channels can be turned on and off.  (If telemetry is off,
+the per-channel setting has no effect.)
+
+* **basic** (default: on): Basic information about the cluster
+
+    - capacity of the cluster
+    - number of monitors, managers, OSDs, MDSs, object gateways, or other daemons
+    - software version currently being used
+    - number and types of RADOS pools and CephFS file systems
+    - names of configuration options that have been changed from their
+      default (but *not* their values)
+
+* **crash** (default: on): Information about daemon crashes, including
+
+    - type of daemon
+    - version of the daemon
+    - operating system (OS distribution, kernel version)
+    - stack trace identifying where in the Ceph code the crash occurred
+
+* **device** (default: on): Information about device metrics, including
+
+    - anonymized SMART metrics
+
+* **ident** (default: off): User-provided identifying information about
+  the cluster
+
+    - cluster description
+    - contact email address
+
+* **perf** (default: off): Various performance metrics of a cluster, which can be used to
+
+    - reveal overall cluster health
+    - identify workload patterns
+    - troubleshoot issues with latency, throttling, memory management, etc.
+    - monitor cluster performance by daemon
+
+The data being reported does *not* contain any sensitive
+data like pool names, object names, object contents, hostnames, or device
+serial numbers.
+
+It contains counters and statistics on how the cluster has been
+deployed, the version of Ceph, the distribution of the hosts and other
+parameters which help the project to gain a better understanding of
+the way Ceph is used.
+
+Data is sent secured to *https://telemetry.ceph.com*.
+
+Individual channels can be enabled or disabled with::
+
+  ceph telemetry enable channel basic
+  ceph telemetry enable channel crash
+  ceph telemetry enable channel device
+  ceph telemetry enable channel ident
+  ceph telemetry enable channel perf
+
+  ceph telemetry disable channel basic
+  ceph telemetry disable channel crash
+  ceph telemetry disable channel device
+  ceph telemetry disable channel ident
+  ceph telemetry disable channel perf
+
+Multiple channels can be enabled or disabled with::
+
+  ceph telemetry enable channel basic crash device ident perf
+  ceph telemetry disable channel basic crash device ident perf
+
+Channels can be enabled or disabled all at once with::
+
+  ceph telemetry enable channel all
+  ceph telemetry disable channel all
+
+Please note that telemetry should be on for these commands to take effect.
+
+List all channels with::
+
+  ceph telemetry channel ls
+
+  NAME      ENABLED    DEFAULT    DESC
+  basic     ON         ON         Share basic cluster information (size, version)
+  crash     ON         ON         Share metadata about Ceph daemon crashes (version, stack straces, etc)
+  device    ON         ON         Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)
+  ident     OFF        OFF        Share a user-provided description and/or contact email for the cluster
+  perf      ON         OFF        Share various performance metrics of a cluster
+
+
+Enabling Telemetry
+------------------
+
+To allow the *telemetry* module to start sharing data::
+
+  ceph telemetry on
+
+Please note: Telemetry data is licensed under the Community Data License
+Agreement - Sharing - Version 1.0 (https://cdla.io/sharing-1-0/). Hence,
+telemetry module can be enabled only after you add '--license sharing-1-0' to
+the 'ceph telemetry on' command.
+Once telemetry is on, please consider enabling channels which are off by
+default, such as the 'perf' channel. 'ceph telemetry on' output will list the
+exact command to enable these channels.
+
+Telemetry can be disabled at any time with::
+
+  ceph telemetry off
+
+Sample report
+-------------
+
+You can look at what data is reported at any time with the command::
+
+  ceph telemetry show
+
+If telemetry is off, you can preview a sample report with::
+
+  ceph telemetry preview
+
+Generating a sample report might take a few moments in big clusters (clusters
+with hundreds of OSDs or more).
+
+To protect your privacy, device reports are generated separately, and data such
+as hostname and device serial number is anonymized. The device telemetry is
+sent to a different endpoint and does not associate the device data with a
+particular cluster. To see a preview of the device report use the command::
+
+  ceph telemetry show-device
+
+If telemetry is off, you can preview a sample device report with::
+
+  ceph telemetry preview-device
+
+Please note: In order to generate the device report we use Smartmontools
+version 7.0 and up, which supports JSON output. 
+If you have any concerns about privacy with regard to the information included in
+this report, please contact the Ceph developers.
+
+In case you prefer to have a single output of both reports, and telemetry is on, use::
+
+  ceph telemetry show-all
+
+If you would like to view a single output of both reports, and telemetry is off, use::
+
+  ceph telemetry preview-all
+
+**Sample report by channel**
+
+When telemetry is on you can see what data is reported by channel with::
+
+  ceph telemetry show <channel_name>
+
+Please note: If telemetry is on, and <channel_name> is disabled, the command
+above will output a sample report by that channel, according to the collections
+the user is enrolled to. However this data is not reported, since the channel
+is disabled.
+
+If telemetry is off you can preview a sample report by channel with::
+
+  ceph telemetry preview <channel_name>
+
+Collections
+-----------
+
+Collections represent different aspects of data that we collect within a channel.
+
+List all collections with::
+
+  ceph telemetry collection ls
+
+  NAME                            STATUS                                               DESC
+  basic_base                      NOT REPORTING: NOT OPTED-IN                          Basic information about the cluster (capacity, number and type of daemons, version, etc.)
+  basic_mds_metadata              NOT REPORTING: NOT OPTED-IN                          MDS metadata
+  basic_pool_options_bluestore    NOT REPORTING: NOT OPTED-IN                          Per-pool bluestore config options
+  basic_pool_usage                NOT REPORTING: NOT OPTED-IN                          Default pool application and usage statistics
+  basic_rook_v01                  NOT REPORTING: NOT OPTED-IN                          Basic Rook deployment data
+  basic_usage_by_class            NOT REPORTING: NOT OPTED-IN                          Default device class usage statistics
+  crash_base                      NOT REPORTING: NOT OPTED-IN                          Information about daemon crashes (daemon type and version, backtrace, etc.)
+  device_base                     NOT REPORTING: NOT OPTED-IN                          Information about device health metrics
+  ident_base                      NOT REPORTING: NOT OPTED-IN, CHANNEL ident IS OFF    User-provided identifying information about the cluster
+  perf_memory_metrics             NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF     Heap stats and mempools for mon and mds
+  perf_perf                       NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF     Information about performance counters of the cluster
+
+Where:
+
+**NAME**: Collection name; prefix indicates the channel the collection belongs to.
+
+**STATUS**: Indicates whether the collection metrics are reported; this is
+determined by the status (enabled / disabled) of the channel the collection
+belongs to, along with the enrollment status of the collection (whether the user
+is opted-in to this collection).
+
+**DESC**: General description of the collection.
+
+See the diff between the collections you are enrolled to, and the new,
+available collections with::
+
+  ceph telemetry diff
+
+Enroll to the most recent collections with::
+
+  ceph telemetry on
+
+Then enable new channels that are off with::
+
+  ceph telemetry enable channel <channel_name>
+
+Interval
+--------
+
+The module compiles and sends a new report every 24 hours by default.
+You can adjust this interval with::
+
+  ceph config set mgr mgr/telemetry/interval 72    # report every three days
+
+Status
+--------
+
+The see the current configuration::
+
+  ceph telemetry status
+
+Manually sending telemetry
+--------------------------
+
+To ad hoc send telemetry data::
+
+  ceph telemetry send
+
+In case telemetry is not enabled (with 'ceph telemetry on'), you need to add
+'--license sharing-1-0' to 'ceph telemetry send' command.
+
+Sending telemetry through a proxy
+---------------------------------
+
+If the cluster cannot directly connect to the configured telemetry
+endpoint (default *telemetry.ceph.com*), you can configure a HTTP/HTTPS
+proxy server with::
+
+  ceph config set mgr mgr/telemetry/proxy https://10.0.0.1:8080
+
+You can also include a *user:pass* if needed::
+
+  ceph config set mgr mgr/telemetry/proxy https://ceph:telemetry@10.0.0.1:8080
+
+
+Contact and Description
+-----------------------
+
+A contact and description can be added to the report.  This is
+completely optional, and disabled by default.::
+
+  ceph config set mgr mgr/telemetry/contact 'John Doe <john.doe@example.com>'
+  ceph config set mgr mgr/telemetry/description 'My first Ceph cluster'
+  ceph config set mgr mgr/telemetry/channel_ident true
+
+Leaderboard
+-----------
+
+To participate in a leaderboard in the `public dashboards
+<https://telemetry-public.ceph.com/>`_, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/telemetry/leaderboard true
+
+The leaderboard displays basic information about the cluster. This includes the
+total storage capacity and the number of OSDs. To add a description of the
+cluster, run a command of the following form: 
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/telemetry/leaderboard_description 'Ceph cluster for Computational Biology at the University of XYZ'
+
+If the ``ident`` channel is enabled, its details will not be displayed in the
+leaderboard.
+
diff --git a/doc/mgr/zabbix.rst b/doc/mgr/zabbix.rst
new file mode 100644
index 000000000..f044b7a79
--- /dev/null
+++ b/doc/mgr/zabbix.rst
@@ -0,0 +1,153 @@
+Zabbix Module
+=============
+
+The Zabbix module actively sends information to a Zabbix server like:
+
+- Ceph status
+- I/O operations
+- I/O bandwidth
+- OSD status
+- Storage utilization
+
+Requirements
+------------
+
+The module requires that the *zabbix_sender* executable is present on *all*
+machines running ceph-mgr. It can be installed on most distributions using
+the package manager.
+
+Dependencies
+^^^^^^^^^^^^
+Installing zabbix_sender can be done under Ubuntu or CentOS using either apt
+or dnf.
+
+On Ubuntu Xenial:
+
+::
+
+    apt install zabbix-agent
+
+On Fedora:
+
+::
+
+    dnf install zabbix-sender
+
+
+Enabling
+--------
+You can enable the *zabbix* module with:
+
+::
+
+    ceph mgr module enable zabbix
+
+Configuration
+-------------
+
+Two configuration keys are vital for the module to work:
+
+- zabbix_host
+- identifier (optional)
+
+The parameter *zabbix_host* controls the hostname of the Zabbix server to which
+*zabbix_sender* will send the items. This can be a IP-Address if required by
+your installation.
+
+The *identifier* parameter controls the identifier/hostname to use as source
+when sending items to Zabbix. This should match the name of the *Host* in
+your Zabbix server.
+
+When the *identifier* parameter is not configured the ceph-<fsid> of the cluster
+will be used when sending data to Zabbix.
+
+This would for example be *ceph-c4d32a99-9e80-490f-bd3a-1d22d8a7d354*
+
+Additional configuration keys which can be configured and their default values:
+
+- zabbix_port: 10051
+- zabbix_sender: /usr/bin/zabbix_sender
+- interval: 60
+- discovery_interval: 100
+
+Configuration keys
+^^^^^^^^^^^^^^^^^^^
+
+Configuration keys can be set on any machine with the proper cephx credentials,
+these are usually Monitors where the *client.admin* key is present.
+
+::
+
+    ceph zabbix config-set <key> <value>
+
+For example:
+
+::
+
+    ceph zabbix config-set zabbix_host zabbix.localdomain
+    ceph zabbix config-set identifier ceph.eu-ams02.local
+
+The current configuration of the module can also be shown:
+
+::
+
+   ceph zabbix config-show
+
+
+Template
+^^^^^^^^
+A `template <https://raw.githubusercontent.com/ceph/ceph/master/src/pybind/mgr/zabbix/zabbix_template.xml>`_. 
+(XML) to be used on the Zabbix server can be found in the source directory of the module.
+
+This template contains all items and a few triggers. You can customize the triggers afterwards to fit your needs.
+
+
+Multiple Zabbix servers
+^^^^^^^^^^^^^^^^^^^^^^^
+It is possible to instruct zabbix module to send data to multiple Zabbix servers.
+
+Parameter *zabbix_host* can be set with multiple hostnames separated by commas.
+Hostnames (or IP addresses) can be followed by colon and port number. If a port
+number is not present module will use the port number defined in *zabbix_port*.
+
+For example:
+
+::
+
+    ceph zabbix config-set zabbix_host "zabbix1,zabbix2:2222,zabbix3:3333"
+
+
+Manually sending data
+---------------------
+If needed the module can be asked to send data immediately instead of waiting for
+the interval.
+
+This can be done with this command:
+
+::
+
+    ceph zabbix send
+
+The module will now send its latest data to the Zabbix server.
+
+Items discovery is accomplished also via zabbix_sender, and runs every `discovery_interval * interval` seconds. If you wish to launch discovery 
+manually, this can be done with this command:
+
+::
+
+    ceph zabbix discovery
+
+
+Debugging
+---------
+
+Should you want to debug the Zabbix module increase the logging level for
+ceph-mgr and check the logs.
+
+::
+
+    [mgr]
+        debug mgr = 20
+
+With logging set to debug for the manager the module will print various logging
+lines prefixed with *mgr[zabbix]* for easy filtering.
diff --git a/doc/mon/README.txt b/doc/mon/README.txt
new file mode 100644
index 000000000..fa1bf791a
--- /dev/null
+++ b/doc/mon/README.txt
@@ -0,0 +1,27 @@
+paxos-call-chain.dot describes to some detail the call chain involved in the
+Paxos algorithm, paying special consideration to the messages involved.
+
+This information is not easily obtainable by Doxygen, as it does not follow
+the call chain when messages are involved, since it becomes an async workflow.
+
+To obtain the graph one should run
+
+  dot -T<format> paxos-call-chain.dot -o paxos-call-chain.<format>
+
+e.g.,
+
+  dot -Tps paxos-call-chain.dot -o paxos-call-chain.ps
+
+or
+
+  dot -Tpng paxos-call-chain.dot -o paxos-call-chain.png
+
+It should do the trick.
+
+Also, for future reference, we consider that:
+  - boxed nodes refer to the Leader;
+  - elliptical nodes refer to the Peon;
+  - diamond shaped nodes refer to state changes;
+  - dotted lines illustrate a message being sent from the Leader to the Peon,
+  or vice-versa.
+
diff --git a/doc/mon/paxos-call-chain.dot b/doc/mon/paxos-call-chain.dot
new file mode 100644
index 000000000..cdca200f0
--- /dev/null
+++ b/doc/mon/paxos-call-chain.dot
@@ -0,0 +1,82 @@
+digraph Paxos {
+  concentrate=true
+//  subgraph cluster0 {
+    collect -> state_recovering;
+    collect -> send_mmp_collect -> handle_collect [style=dotted];
+    handle_collect -> send_mmp_last;
+    send_mmp_last -> handle_last [style=dotted];
+    handle_last -> collect [label="collect(last_pn)"];
+    handle_last -> send_mmp_commit;
+    handle_last -> state_active;
+    handle_last -> begin;
+    handle_last -> extend_lease;
+
+//    color=grey;
+//    style=filled;
+//    label="Post-Election call chain";
+//  }
+
+  election_finished -> leader_init;
+  leader_init -> collect [label="collect(0)"];
+  send_mmp_commit -> handle_commit [style=dotted];
+
+  begin -> state_updating;
+  begin -> send_mmp_begin;
+  begin -> commit;
+  
+  send_mmp_begin -> handle_begin [style=dotted];
+  handle_begin -> state_updating;
+  handle_begin -> send_mmp_accept;
+  send_mmp_accept -> handle_accept [style=dotted];
+
+  handle_accept -> extend_lease;
+  handle_accept -> state_active;
+  handle_accept -> commit;
+
+  extend_lease -> send_mmp_lease;
+  send_mmp_lease -> handle_lease [style=dotted];
+  handle_lease -> state_active;
+  handle_lease -> send_mmp_lease_ack;
+  send_mmp_lease_ack -> handle_lease_ack [style=dotted];
+
+  commit -> send_mmp_commit;
+  commit -> "last_committed++" [shape=box];
+  send_mmp_commit -> handle_commit [style=dotted];
+
+  handle_commit -> store_state;
+
+  propose_pending -> propose_new_value;
+  propose_new_value -> begin;
+
+  election_finished [label="Election Finished", shape=box, bgcolor=grey, style=filled];
+  collect	    [label="collect()", shape=box];
+  commit	    [label="commit()", shape=box];
+  begin		    [label="begin()", shape=box];
+  extend_lease	    [label="extend_lease()", shape=box];
+  store_state	    [label="store_state()"]; // peon
+
+  propose_pending   [label="PaxosService::propose_pending()"];
+  propose_new_value [label="propose_new_value"];
+
+  send_mmp_collect  [label="send(OP_COLLECT)", shape=box];
+  send_mmp_last	    [label="send(OP_LAST)"];
+  send_mmp_commit   [label="send(OP_COMMIT)", shape=box];
+  send_mmp_begin    [label="send(OP_BEGIN)", shape=box];
+  send_mmp_accept   [label="send(OP_ACCEPT)"];
+  send_mmp_lease    [label="send(OP_LEASE)", shape=box];
+  send_mmp_lease_ack [label="send(OP_LEASE_ACK)"];
+
+  handle_collect    [label="handle_collect()"];
+  handle_last	    [label="handle_last()", shape=box];
+  handle_begin	    [label="handle_begin()"];
+  handle_accept	    [label="handle_accept()", shape=box];
+  handle_lease	    [label="handle_lease()"];
+  handle_lease_ack  [label="handle_lease_ack()", shape=box];
+  handle_commit	    [label="handle_commit()"];
+
+  leader_init	    [label="Paxos::leader_init()", shape=box];
+
+  state_recovering  [label="RECOVERING", shape=diamond];
+  state_active	    [label="ACTIVE", shape=diamond];
+  state_updating    [label="UPDATING", shape=diamond];
+}
diff --git a/doc/monitoring/index.rst b/doc/monitoring/index.rst
new file mode 100644
index 000000000..2bf2aca90
--- /dev/null
+++ b/doc/monitoring/index.rst
@@ -0,0 +1,474 @@
+.. _monitoring:
+
+===================
+Monitoring overview
+===================
+
+The aim of this part of the documentation is to explain the Ceph monitoring
+stack and the meaning of the main Ceph metrics.
+
+With a good understand of the Ceph monitoring stack and metrics users can
+create customized monitoring tools, like Prometheus queries, Grafana
+dashboards, or scripts.
+
+
+Ceph Monitoring stack
+=====================
+
+Ceph provides a default monitoring stack wich is installed by cephadm and
+explained in the :ref:`Monitoring Services <mgr-cephadm-monitoring>` section of
+the cephadm documentation.
+
+
+Ceph metrics
+============
+
+The main source for Ceph metrics are the performance counters exposed by each
+Ceph daemon. The :doc:`../dev/perf_counters` are native Ceph monitoring data
+
+Performance counters are transformed into standard Prometheus metrics by the
+Ceph exporter daemon. This daemon runs on every Ceph cluster host and exposes a
+metrics end point where all the performance counters exposed by all the Ceph
+daemons running in the host are published in the form of Prometheus metrics.
+
+In addition to the Ceph exporter, there is another agent to expose Ceph
+metrics. It is the Prometheus manager module, wich exposes metrics related to
+the whole cluster, basically metrics that are not produced by individual Ceph
+daemons.
+
+The main source for obtaining Ceph metrics is the metrics endpoint exposed by
+the Cluster Prometheus server.  Ceph can provide you with the Prometheus
+endpoint where you can obtain the complete list of metrics (coming from Ceph
+exporter daemons and Prometheus manager module) and exeute queries.
+
+Use the following command to obtain the Prometheus server endpoint in your
+cluster:
+
+Example:
+
+.. code-block:: bash
+
+  # ceph orch ps --service_name prometheus
+  NAME                         HOST                          PORTS   STATUS          REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID
+  prometheus.cephtest-node-00  cephtest-node-00.cephlab.com  *:9095  running (103m)    50s ago   5w     142M        -  2.33.4   514e6a882f6e  efe3cbc2e521
+
+With this information you can connect to
+``http://cephtest-node-00.cephlab.com:9095`` to access the Prometheus server
+interface.
+
+And the complete list of metrics (with help) for your cluster will be available
+in:
+
+``http://cephtest-node-00.cephlab.com:9095/api/v1/targets/metadata``
+
+
+It is good to outline that the main tool allowing users to observe and monitor a Ceph cluster is the **Ceph dashboard**. It provides graphics where the most important cluster and service metrics are represented. Most of the examples in this document are extracted from the dashboard graphics or extrapolated from the metrics exposed by the Ceph dashboard.
+
+
+Performance metrics
+===================
+
+Main metrics used to measure Cluster Ceph performance:
+
+All metrics have the following labels:
+``ceph_daemon``: identifier of the OSD daemon generating the metric
+``instance``: the IP address of the ceph exporter instance exposing the metric.
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_osd_op_r{ceph_daemon="osd.0", instance="192.168.122.7:9283", job="ceph"} = 73981
+
+*Cluster I/O (throughput):*
+Use ``ceph_osd_op_r_out_bytes`` and ``ceph_osd_op_w_in_bytes`` to obtain the cluster throughput generated by clients
+
+Example:
+
+.. code-block:: bash
+
+  Writes (B/s):
+  sum(irate(ceph_osd_op_w_in_bytes[1m]))
+
+  Reads (B/s):
+  sum(irate(ceph_osd_op_r_out_bytes[1m]))
+
+
+*Cluster I/O (operations):*
+Use ``ceph_osd_op_r``, ``ceph_osd_op_w`` to obtain the number of operations generated by clients
+
+Example:
+
+.. code-block:: bash
+
+  Writes (ops/s):
+  sum(irate(ceph_osd_op_w[1m]))
+
+  Reads (ops/s):
+  sum(irate(ceph_osd_op_r[1m]))
+
+*Latency:*
+Use ``ceph_osd_op_latency_sum`` wich represents the delay before a OSD transfer of data begins following a client instruction for its transfer
+
+Example:
+
+.. code-block:: bash
+
+  sum(irate(ceph_osd_op_latency_sum[1m]))
+
+
+OSD performance
+===============
+
+The previous explained cluster performance metrics are based in OSD metrics, selecting the right label we can obtain for a single OSD the same performance information explained for the cluster:
+
+Example:
+
+.. code-block:: bash
+
+  OSD 0 read latency
+  irate(ceph_osd_op_r_latency_sum{ceph_daemon=~"osd.0"}[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m])
+
+  OSD 0 write IOPS
+  irate(ceph_osd_op_w{ceph_daemon=~"osd.0"}[1m])
+
+  OSD 0 write thughtput (bytes)
+  irate(ceph_osd_op_w_in_bytes{ceph_daemon=~"osd.0"}[1m])
+
+  OSD.0 total raw capacity available
+  ceph_osd_stat_bytes{ceph_daemon="osd.0", instance="cephtest-node-00.cephlab.com:9283", job="ceph"} = 536451481
+
+
+Physical disk performance:
+==========================
+
+Combining Prometheus ``node_exporter`` metrics with Ceph metrics we can have
+information about the performance provided by physical disks used by OSDs.
+
+Example:
+
+.. code-block:: bash
+
+  Read latency of device used by OSD 0:
+  label_replace(irate(node_disk_read_time_seconds_total[1m]) / irate(node_disk_reads_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Write latency of device used by OSD 0
+  label_replace(irate(node_disk_write_time_seconds_total[1m]) / irate(node_disk_writes_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  IOPS (device used by OSD.0)
+  reads:
+  label_replace(irate(node_disk_reads_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  writes:
+  label_replace(irate(node_disk_writes_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Throughput (device used by OSD.0)
+  reads:
+  label_replace(irate(node_disk_read_bytes_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  writes:
+  label_replace(irate(node_disk_written_bytes_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Physical Device Utilization (%) for OSD.0 in the last 5 minutes
+  label_replace(irate(node_disk_io_time_seconds_total[5m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+Pool metrics
+============
+
+These metrics have the following labels:
+``instance``: the ip address of the Ceph exporter daemon producing the metric.
+``pool_id``: identifier of the pool
+``job``: prometheus scrape job
+
+
+- ``ceph_pool_metadata``: Information about the pool It can be used together
+  with other metrics to provide more contextual information in queries and
+  graphs.  Apart of the three common labels this metric provide the following
+  extra labels:
+
+  - ``compression_mode``: compression used in the pool (lz4, snappy, zlib,
+    zstd, none). Example: compression_mode="none"
+
+  - ``description``: brief description of the pool type (replica:number of
+    replicas or Erasure code: ec profile). Example: description="replica:3"
+  - ``name``: name of the pool. Example: name=".mgr"
+  - ``type``: type of pool (replicated/erasure code). Example: type="replicated"
+
+- ``ceph_pool_bytes_used``: Total raw capacity consumed by user data and associated overheads by pool (metadata + redundancy):
+
+- ``ceph_pool_stored``: Total of CLIENT data stored in the pool
+
+- ``ceph_pool_compress_under_bytes``: Data eligible to be compressed in the pool
+
+- ``ceph_pool_compress_bytes_used``:  Data compressed in the pool
+
+- ``ceph_pool_rd``: CLIENT read operations per pool (reads per second)
+
+- ``ceph_pool_rd_bytes``: CLIENT read operations in bytes per pool
+
+- ``ceph_pool_wr``: CLIENT write operations per pool (writes per second)
+
+- ``ceph_pool_wr_bytes``: CLIENT write operation in bytes per pool
+
+
+**Useful queries**:
+
+.. code-block:: bash
+
+  Total raw capacity available in the cluster:
+  sum(ceph_osd_stat_bytes)
+
+  Total raw capacity consumed in the cluster (including metadata + redundancy):
+  sum(ceph_pool_bytes_used)
+
+  Total of CLIENT data stored in the cluster:
+  sum(ceph_pool_stored)
+
+  Compression savings:
+  sum(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used)
+
+  CLIENT IOPS for a pool (testrbdpool)
+  reads: irate(ceph_pool_rd[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+  writes: irate(ceph_pool_wr[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+
+  CLIENT Throughput for a pool
+  reads: irate(ceph_pool_rd_bytes[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+  writes: irate(ceph_pool_wr_bytes[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+
+Object metrics
+==============
+
+These metrics have the following labels:
+``instance``: the ip address of the ceph exporter daemon providing the metric
+``instance_id``: identifier of the rgw daemon
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_rgw_req{instance="192.168.122.7:9283", instance_id="154247", job="ceph"} = 12345
+
+
+Generic metrics
+---------------
+- ``ceph_rgw_metadata``: Provides generic information about the RGW daemon.  It
+  can be used together with other metrics to provide more contextual
+  information in queries and graphs. Apart from the three common labels, this
+  metric provides the following extra labels:
+
+  - ``ceph_daemon``: Name of the Ceph daemon. Example:
+    ceph_daemon="rgw.rgwtest.cephtest-node-00.sxizyq",
+  - ``ceph_version``: Version of Ceph daemon. Example: ceph_version="ceph
+    version 17.2.6 (d7ff0d10654d2280e08f1ab989c7cdf3064446a5) quincy (stable)",
+  - ``hostname``: Name of the host where the daemon runs. Example:
+    hostname:"cephtest-node-00.cephlab.com",
+
+- ``ceph_rgw_req``: Number total of requests for the daemon (GET+PUT+DELETE)
+    Useful to detect bottlenecks and optimize load distribution.
+
+- ``ceph_rgw_qlen``: RGW operations queue length for the daemon.
+    Useful to detect bottlenecks and optimize load distribution.
+
+- ``ceph_rgw_failed_req``: Aborted requests.
+    Useful to detect daemon errors
+
+
+GET operations: related metrics
+-------------------------------
+- ``ceph_rgw_get_initial_lat_count``: Number of get operations
+
+- ``ceph_rgw_get_initial_lat_sum``: Total latency time for the GET operations
+
+- ``ceph_rgw_get``: Number total of GET requests
+
+- ``ceph_rgw_get_b``: Total bytes transferred in GET operations
+
+
+Put operations: related metrics
+-------------------------------
+- ``ceph_rgw_put_initial_lat_count``: Number of get operations
+
+- ``ceph_rgw_put_initial_lat_sum``: Total latency time for the PUT operations
+
+- ``ceph_rgw_put``: Total number of PUT operations
+
+- ``ceph_rgw_get_b``: Total bytes transferred in PUT operations
+
+
+Useful queries
+--------------
+
+.. code-block:: bash
+
+  The average of get latencies:
+  rate(ceph_rgw_get_initial_lat_sum[30s]) / rate(ceph_rgw_get_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  The average of put latencies:
+  rate(ceph_rgw_put_initial_lat_sum[30s]) / rate(ceph_rgw_put_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Total requests per second:
+  rate(ceph_rgw_req[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Total number of "other" operations (LIST, DELETE)
+  rate(ceph_rgw_req[30s]) -  (rate(ceph_rgw_get[30s]) + rate(ceph_rgw_put[30s]))
+
+  GET latencies
+  rate(ceph_rgw_get_initial_lat_sum[30s]) /  rate(ceph_rgw_get_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  PUT latencies
+  rate(ceph_rgw_put_initial_lat_sum[30s]) /  rate(ceph_rgw_put_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Bandwidth consumed by GET operations
+  sum(rate(ceph_rgw_get_b[30s]))
+
+  Bandwidth consumed by PUT operations
+  sum(rate(ceph_rgw_put_b[30s]))
+
+  Bandwidth consumed by RGW instance (PUTs + GETs)
+  sum by (instance_id) (rate(ceph_rgw_get_b[30s]) + rate(ceph_rgw_put_b[30s])) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Http errors:
+  rate(ceph_rgw_failed_req[30s])
+
+
+Filesystem Metrics
+==================
+
+These metrics have the following labels:
+``ceph_daemon``: The name of the MDS daemon
+``instance``: the ip address (and port) of of the Ceph exporter daemon exposing the metric
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_mds_request{ceph_daemon="mds.test.cephtest-node-00.hmhsoh", instance="192.168.122.7:9283", job="ceph"} = 1452
+
+
+Main metrics
+------------
+
+- ``ceph_mds_metadata``: Provides general information about the MDS daemon.  It
+  can be used together with other metrics to provide more contextual
+  information in queries and graphs.  It provides the following extra labels:
+
+  - ``ceph_version``: MDS daemon Ceph version
+  - ``fs_id``: filesystem cluster id
+  - ``hostname``: Host name where the MDS daemon runs
+  - ``public_addr``: Public address where the MDS daemon runs
+  - ``rank``: Rank of the MDS daemon
+
+Example:
+
+.. code-block:: bash
+
+ ceph_mds_metadata{ceph_daemon="mds.test.cephtest-node-00.hmhsoh", ceph_version="ceph version 17.2.6 (d7ff0d10654d2280e08f1ab989c7cdf3064446a5) quincy (stable)", fs_id="-1", hostname="cephtest-node-00.cephlab.com", instance="cephtest-node-00.cephlab.com:9283", job="ceph", public_addr="192.168.122.145:6801/118896446", rank="-1"}
+
+
+- ``ceph_mds_request``: Total number of requests for the MDs daemon
+
+- ``ceph_mds_reply_latency_sum``: Reply latency total
+
+- ``ceph_mds_reply_latency_count``: Reply latency count
+
+- ``ceph_mds_server_handle_client_request``: Number of client requests
+
+- ``ceph_mds_sessions_session_count``: Session count
+
+- ``ceph_mds_sessions_total_load``: Total load
+
+- ``ceph_mds_sessions_sessions_open``: Sessions currently open
+
+- ``ceph_mds_sessions_sessions_stale``: Sessions currently stale
+
+- ``ceph_objecter_op_r``: Number of read operations
+
+- ``ceph_objecter_op_w``: Number of write operations
+
+- ``ceph_mds_root_rbytes``: Total number of bytes managed by the daemon
+
+- ``ceph_mds_root_rfiles``: Total number of files managed by the daemon
+
+
+Useful queries:
+---------------
+
+.. code-block:: bash
+
+  Total MDS daemons read workload:
+  sum(rate(ceph_objecter_op_r[1m]))
+
+  Total MDS daemons write workload:
+  sum(rate(ceph_objecter_op_w[1m]))
+
+  MDS daemon read workload: (daemon name is "mdstest")
+  sum(rate(ceph_objecter_op_r{ceph_daemon=~"mdstest"}[1m]))
+
+  MDS daemon write workload: (daemon name is "mdstest")
+  sum(rate(ceph_objecter_op_r{ceph_daemon=~"mdstest"}[1m]))
+
+  The average of reply latencies:
+  rate(ceph_mds_reply_latency_sum[30s]) / rate(ceph_mds_reply_latency_count[30s])
+
+  Total requests per second:
+  rate(ceph_mds_request[30s]) * on (instance) group_right (ceph_daemon) ceph_mds_metadata
+
+
+Block metrics
+=============
+
+By default RBD metrics for images are not available in order to provide the
+best performance in the prometheus manager module.
+
+To produce metrics for RBD images it is needed to configure properly the
+manager option ``mgr/prometheus/rbd_stats_pools``. For more information please
+see :ref:`prometheus-rbd-io-statistics`
+
+
+These metrics have the following labels:
+``image``: Name of the image which produces the metric value.
+``instance``: Node where the rbd metric is produced. (It points to the Ceph exporter daemon)
+``job``: Name of the Prometheus scrape job.
+``pool``: Image pool name.
+
+Example:
+
+.. code-block:: bash
+
+  ceph_rbd_read_bytes{image="test2", instance="cephtest-node-00.cephlab.com:9283", job="ceph", pool="testrbdpool"}
+
+
+Main metrics
+------------
+
+- ``ceph_rbd_read_bytes``: RBD image bytes read
+
+- ``ceph_rbd_read_latency_count``: RBD image reads latency count
+
+- ``ceph_rbd_read_latency_sum``: RBD image reads latency total
+
+- ``ceph_rbd_read_ops``: RBD image reads count
+
+- ``ceph_rbd_write_bytes``: RBD image bytes written
+
+- ``ceph_rbd_write_latency_count``: RBD image writes latency count
+
+- ``ceph_rbd_write_latency_sum``: RBD image writes latency total
+
+- ``ceph_rbd_write_ops``: RBD image writes count
+
+
+Useful queries
+--------------
+
+.. code-block:: bash
+
+  The average of read latencies:
+  rate(ceph_rbd_read_latency_sum[30s]) / rate(ceph_rbd_read_latency_count[30s]) * on (instance) group_left (ceph_daemon) ceph_rgw_metadata
+
+
+
+
diff --git a/doc/rados/api/index.rst b/doc/rados/api/index.rst
new file mode 100644
index 000000000..5422ce871
--- /dev/null
+++ b/doc/rados/api/index.rst
@@ -0,0 +1,25 @@
+.. _rados api:
+
+===========================
+ Ceph Storage Cluster APIs
+===========================
+
+The :term:`Ceph Storage Cluster` has a messaging layer protocol that enables
+clients to interact with a :term:`Ceph Monitor` and a :term:`Ceph OSD Daemon`.
+``librados`` provides this functionality to :term:`Ceph Client`\s in the form of
+a library.  All Ceph Clients either use ``librados`` or the same functionality
+encapsulated in ``librados`` to interact with the object store.  For example,
+``librbd`` and ``libcephfs`` leverage this functionality. You may use
+``librados`` to interact with Ceph directly (e.g., an application that talks to
+Ceph, your own interface to Ceph, etc.).
+
+
+.. toctree::
+   :maxdepth: 2 
+
+   Introduction to librados <librados-intro>
+   librados (C) <librados>
+   librados (C++) <libradospp>
+   librados (Python) <python>
+   libcephsqlite (SQLite) <libcephsqlite>
+   object class <objclass-sdk>
diff --git a/doc/rados/api/libcephsqlite.rst b/doc/rados/api/libcephsqlite.rst
new file mode 100644
index 000000000..beee4a466
--- /dev/null
+++ b/doc/rados/api/libcephsqlite.rst
@@ -0,0 +1,454 @@
+.. _libcephsqlite:
+
+================
+ Ceph SQLite VFS
+================
+
+This `SQLite VFS`_ may be used for storing and accessing a `SQLite`_ database
+backed by RADOS. This allows you to fully decentralize your database using
+Ceph's object store for improved availability, accessibility, and use of
+storage.
+
+Note what this is not: a distributed SQL engine. SQLite on RADOS can be thought
+of like RBD as compared to CephFS: RBD puts a disk image on RADOS for the
+purposes of exclusive access by a machine and generally does not allow parallel
+access by other machines; on the other hand, CephFS allows fully distributed
+access to a file system from many client mounts. SQLite on RADOS is meant to be
+accessed by a single SQLite client database connection at a given time.  The
+database may be manipulated safely by multiple clients only in a serial fashion
+controlled by RADOS locks managed by the Ceph SQLite VFS.
+
+
+Usage
+^^^^^
+
+Normal unmodified applications (including the sqlite command-line toolset
+binary) may load the *ceph* VFS using the `SQLite Extension Loading API`_.
+
+.. code:: sql
+
+    .LOAD libcephsqlite.so
+
+or during the invocation of ``sqlite3``
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so'
+
+A database file is formatted as a SQLite URI::
+
+    file:///<"*"poolid|poolname>:[namespace]/<dbname>?vfs=ceph
+
+The RADOS ``namespace`` is optional. Note the triple ``///`` in the path. The URI
+authority must be empty or localhost in SQLite. Only the path part of the URI
+is parsed. For this reason, the URI will not parse properly if you only use two
+``//``.
+
+A complete example of (optionally) creating a database and opening:
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///foo:bar/baz.db?vfs=ceph'
+
+Note you cannot specify the database file as the normal positional argument to
+``sqlite3``. This is because the ``.load libcephsqlite.so`` command is applied
+after opening the database, but opening the database depends on the extension
+being loaded first.
+
+An example passing the pool integer id and no RADOS namespace:
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///*2:/baz.db?vfs=ceph'
+
+Like other Ceph tools, the *ceph* VFS looks at some environment variables that
+help with configuring which Ceph cluster to communicate with and which
+credential to use. Here would be a typical configuration:
+
+.. code:: sh
+
+   export CEPH_CONF=/path/to/ceph.conf
+   export CEPH_KEYRING=/path/to/ceph.keyring
+   export CEPH_ARGS='--id myclientid'
+   ./runmyapp
+   # or
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///foo:bar/baz.db?vfs=ceph'
+
+The default operation would look at the standard Ceph configuration file path
+using the ``client.admin`` user.
+
+
+User
+^^^^
+
+The *ceph* VFS requires a user credential with read access to the monitors, the
+ability to blocklist dead clients of the database, and access to the OSDs
+hosting the database. This can be done with authorizations as simply as:
+
+.. code:: sh
+
+   ceph auth get-or-create client.X mon 'allow r, allow command "osd blocklist" with blocklistop=add' osd 'allow rwx'
+
+.. note:: The terminology change from ``blacklist`` to ``blocklist``; older clusters may require using the old terms.
+
+You may also simplify using the ``simple-rados-client-with-blocklist`` profile:
+
+.. code:: sh
+
+   ceph auth get-or-create client.X mon 'profile simple-rados-client-with-blocklist' osd 'allow rwx'
+
+To learn why blocklisting is necessary, see :ref:`libcephsqlite-corrupt`.
+
+
+Page Size
+^^^^^^^^^
+
+SQLite allows configuring the page size prior to creating a new database. It is
+advisable to increase this config to 65536 (64K) when using RADOS backed
+databases to reduce the number of OSD reads/writes and thereby improve
+throughput and latency.
+
+.. code:: sql
+
+   PRAGMA page_size = 65536
+
+You may also try other values according to your application needs but note that
+64K is the max imposed by SQLite.
+
+
+Cache
+^^^^^
+
+The ceph VFS does not do any caching of reads or buffering of writes. Instead,
+and more appropriately, the SQLite page cache is used. You may find it is too small
+for most workloads and should therefore increase it significantly:
+
+
+.. code:: sql
+
+   PRAGMA cache_size = 4096
+
+Which will cache 4096 pages or 256MB (with 64K ``page_cache``).
+
+
+Journal Persistence
+^^^^^^^^^^^^^^^^^^^
+
+By default, SQLite deletes the journal for every transaction. This can be
+expensive as the *ceph* VFS must delete every object backing the journal for each
+transaction. For this reason, it is much faster and simpler to ask SQLite to
+**persist** the journal. In this mode, SQLite will invalidate the journal via a
+write to its header. This is done as:
+
+.. code:: sql
+
+   PRAGMA journal_mode = PERSIST
+
+The cost of this may be increased unused space according to the high-water size
+of the rollback journal (based on transaction type and size).
+
+
+Exclusive Lock Mode
+^^^^^^^^^^^^^^^^^^^
+
+SQLite operates in a ``NORMAL`` locking mode where each transaction requires
+locking the backing database file. This can add unnecessary overhead to
+transactions when you know there's only ever one user of the database at a
+given time. You can have SQLite lock the database once for the duration of the
+connection using:
+
+.. code:: sql
+
+   PRAGMA locking_mode = EXCLUSIVE
+
+This can more than **halve** the time taken to perform a transaction. Keep in
+mind this prevents other clients from accessing the database.
+
+In this locking mode, each write transaction to the database requires 3
+synchronization events: once to write to the journal, another to write to the
+database file, and a final write to invalidate the journal header (in
+``PERSIST`` journaling mode).
+
+
+WAL Journal
+^^^^^^^^^^^
+
+The `WAL Journal Mode`_ is only available when SQLite is operating in exclusive
+lock mode. This is because it requires shared memory communication with other
+readers and writers when in the ``NORMAL`` locking mode.
+
+As with local disk databases, WAL mode may significantly reduce small
+transaction latency. Testing has shown it can provide more than 50% speedup
+over persisted rollback journals in exclusive locking mode. You can expect
+around 150-250 transactions per second depending on size.
+
+
+Performance Notes
+^^^^^^^^^^^^^^^^^
+
+The filing backend for the database on RADOS is asynchronous as much as
+possible.  Still, performance can be anywhere from 3x-10x slower than a local
+database on SSD. Latency can be a major factor. It is advisable to be familiar
+with SQL transactions and other strategies for efficient database updates.
+Depending on the performance of the underlying pool, you can expect small
+transactions to take up to 30 milliseconds to complete. If you use the
+``EXCLUSIVE`` locking mode, it can be reduced further to 15 milliseconds per
+transaction. A WAL journal in ``EXCLUSIVE`` locking mode can further reduce
+this as low as ~2-5 milliseconds (or the time to complete a RADOS write; you
+won't get better than that!).
+
+There is no limit to the size of a SQLite database on RADOS imposed by the Ceph
+VFS. There are standard `SQLite Limits`_ to be aware of, notably the maximum
+database size of 281 TB. Large databases may or may not be performant on Ceph.
+Experimentation for your own use-case is advised.
+
+Be aware that read-heavy queries could take significant amounts of time as
+reads are necessarily synchronous (due to the VFS API). No readahead is yet
+performed by the VFS.
+
+
+Recommended Use-Cases
+^^^^^^^^^^^^^^^^^^^^^
+
+The original purpose of this module was to support saving relational or large
+data in RADOS which needs to span multiple objects. Many current applications
+with trivial state try to use RADOS omap storage on a single object but this
+cannot scale without striping data across multiple objects. Unfortunately, it
+is non-trivial to design a store spanning multiple objects which is consistent
+and also simple to use. SQLite can be used to bridge that gap.
+
+
+Parallel Access
+^^^^^^^^^^^^^^^
+
+The VFS does not yet support concurrent readers. All database access is protected
+by a single exclusive lock.
+
+
+Export or Extract Database out of RADOS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The database is striped on RADOS and can be extracted using the RADOS cli toolset.
+
+.. code:: sh
+
+    rados --pool=foo --striper get bar.db local-bar.db
+    rados --pool=foo --striper get bar.db-journal local-bar.db-journal
+    sqlite3 local-bar.db ...
+
+Keep in mind the rollback journal is also striped and will need to be extracted
+as well if the database was in the middle of a transaction. If you're using
+WAL, that journal will need to be extracted as well.
+
+Keep in mind that extracting the database using the striper uses the same RADOS
+locks as those used by the *ceph* VFS. However, the journal file locks are not
+used by the *ceph* VFS (SQLite only locks the main database file) so there is a
+potential race with other SQLite clients when extracting both files. That could
+result in fetching a corrupt journal.
+
+Instead of manually extracting the files, it would be more advisable to use the
+`SQLite Backup`_ mechanism instead.
+
+
+Temporary Tables
+^^^^^^^^^^^^^^^^
+
+Temporary tables backed by the ceph VFS are not supported. The main reason for
+this is that the VFS lacks context about where it should put the database, i.e.
+which RADOS pool. The persistent database associated with the temporary
+database is not communicated via the SQLite VFS API.
+
+Instead, it's suggested to attach a secondary local or `In-Memory Database`_
+and put the temporary tables there. Alternatively, you may set a connection
+pragma:
+
+.. code:: sql
+
+   PRAGMA temp_store=memory
+
+
+.. _libcephsqlite-breaking-locks:
+
+Breaking Locks
+^^^^^^^^^^^^^^
+
+Access to the database file is protected by an exclusive lock on the first
+object stripe of the database. If the application fails without unlocking the
+database (e.g. a segmentation fault), the lock is not automatically unlocked,
+even if the client connection is blocklisted afterward. Eventually, the lock
+will timeout subject to the configurations::
+
+    cephsqlite_lock_renewal_timeout = 30000
+
+The timeout is in milliseconds. Once the timeout is reached, the OSD will
+expire the lock and allow clients to relock. When this occurs, the database
+will be recovered by SQLite and the in-progress transaction rolled back. The
+new client recovering the database will also blocklist the old client to
+prevent potential database corruption from rogue writes.
+
+The holder of the exclusive lock on the database will periodically renew the
+lock so it does not lose the lock. This is necessary for large transactions or
+database connections operating in ``EXCLUSIVE`` locking mode. The lock renewal
+interval is adjustable via::
+
+    cephsqlite_lock_renewal_interval = 2000
+
+This configuration is also in units of milliseconds.
+
+It is possible to break the lock early if you know the client is gone for good
+(e.g. blocklisted). This allows restoring database access to clients
+immediately. For example:
+
+.. code:: sh
+
+    $ rados --pool=foo --namespace bar lock info baz.db.0000000000000000 striper.lock
+    {"name":"striper.lock","type":"exclusive","tag":"","lockers":[{"name":"client.4463","cookie":"555c7208-db39-48e8-a4d7-3ba92433a41a","description":"SimpleRADOSStriper","expiration":"0.000000","addr":"127.0.0.1:0/1831418345"}]}
+
+    $ rados --pool=foo --namespace bar lock break baz.db.0000000000000000 striper.lock client.4463 --lock-cookie 555c7208-db39-48e8-a4d7-3ba92433a41a
+
+.. _libcephsqlite-corrupt:
+
+How to Corrupt Your Database
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There is the usual reading on `How to Corrupt Your SQLite Database`_ that you
+should review before using this tool. To add to that, the most likely way you
+may corrupt your database is by a rogue process transiently losing network
+connectivity and then resuming its work. The exclusive RADOS lock it held will
+be lost but it cannot know that immediately. Any work it might do after
+regaining network connectivity could corrupt the database.
+
+The *ceph* VFS library defaults do not allow for this scenario to occur. The Ceph
+VFS will blocklist the last owner of the exclusive lock on the database if it
+detects incomplete cleanup.
+
+By blocklisting the old client, it's no longer possible for the old client to
+resume its work on the database when it returns (subject to blocklist
+expiration, 3600 seconds by default). To turn off blocklisting the prior client, change::
+
+    cephsqlite_blocklist_dead_locker = false
+
+Do NOT do this unless you know database corruption cannot result due to other
+guarantees. If this config is true (the default), the *ceph* VFS will cowardly
+fail if it cannot blocklist the prior instance (due to lack of authorization,
+for example).
+
+One example where out-of-band mechanisms exist to blocklist the last dead
+holder of the exclusive lock on the database is in the ``ceph-mgr``. The
+monitors are made aware of the RADOS connection used for the *ceph* VFS and will
+blocklist the instance during ``ceph-mgr`` failover. This prevents a zombie
+``ceph-mgr`` from continuing work and potentially corrupting the database. For
+this reason, it is not necessary for the *ceph* VFS to do the blocklist command
+in the new instance of the ``ceph-mgr`` (but it still does so, harmlessly).
+
+To blocklist the *ceph* VFS manually, you may see the instance address of the
+*ceph* VFS using the ``ceph_status`` SQL function:
+
+.. code:: sql
+
+    SELECT ceph_status();
+
+.. code::
+
+    {"id":788461300,"addr":"172.21.10.4:0/1472139388"}
+
+You may easily manipulate that information using the `JSON1 extension`_:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_status(), '$.addr');
+
+.. code::
+
+   172.21.10.4:0/3563721180
+
+This is the address you would pass to the ceph blocklist command:
+
+.. code:: sh
+
+   ceph osd blocklist add 172.21.10.4:0/3082314560
+
+
+Performance Statistics
+^^^^^^^^^^^^^^^^^^^^^^
+
+The *ceph* VFS provides a SQLite function, ``ceph_perf``, for querying the
+performance statistics of the VFS. The data is from "performance counters" as
+in other Ceph services normally queried via an admin socket.
+
+.. code:: sql
+
+    SELECT ceph_perf();
+
+.. code::
+
+    {"libcephsqlite_vfs":{"op_open":{"avgcount":2,"sum":0.150001291,"avgtime":0.075000645},"op_delete":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"op_access":{"avgcount":1,"sum":0.003000026,"avgtime":0.003000026},"op_fullpathname":{"avgcount":1,"sum":0.064000551,"avgtime":0.064000551},"op_currenttime":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_close":{"avgcount":1,"sum":0.000000000,"avgtime":0.000000000},"opf_read":{"avgcount":3,"sum":0.036000310,"avgtime":0.012000103},"opf_write":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_truncate":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_sync":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_filesize":{"avgcount":2,"sum":0.000000000,"avgtime":0.000000000},"opf_lock":{"avgcount":1,"sum":0.158001360,"avgtime":0.158001360},"opf_unlock":{"avgcount":1,"sum":0.101000871,"avgtime":0.101000871},"opf_checkreservedlock":{"avgcount":1,"sum":0.002000017,"avgtime":0.002000017},"opf_filecontrol":{"avgcount":4,"sum":0.000000000,"avgtime":0.000000000},"opf_sectorsize":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_devicecharacteristics":{"avgcount":4,"sum":0.000000000,"avgtime":0.000000000}},"libcephsqlite_striper":{"update_metadata":0,"update_allocated":0,"update_size":0,"update_version":0,"shrink":0,"shrink_bytes":0,"lock":1,"unlock":1}}
+
+You may easily manipulate that information using the `JSON1 extension`_:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_sync.avgcount');
+
+.. code::
+
+    776
+
+That tells you the number of times SQLite has called the xSync method of the
+`SQLite IO Methods`_ of the VFS (for **all** open database connections in the
+process). You could analyze the performance stats before and after a number of
+queries to see the number of file system syncs required (this would just be
+proportional to the number of transactions). Alternatively, you may be more
+interested in the average latency to complete a write:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_write');
+
+.. code::
+
+    {"avgcount":7873,"sum":0.675005797,"avgtime":0.000085736}
+
+Which would tell you there have been 7873 writes with an average
+time-to-complete of 85 microseconds. That clearly shows the calls are executed
+asynchronously. Returning to sync:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_sync');
+
+.. code::
+
+    {"avgcount":776,"sum":4.802041199,"avgtime":0.006188197}
+
+6 milliseconds were spent on average executing a sync call. This gathers all of
+the asynchronous writes as well as an asynchronous update to the size of the
+striped file.
+
+
+Debugging
+^^^^^^^^^
+
+Debugging libcephsqlite can be turned on via::
+
+    debug_cephsqlite
+
+If running the ``sqlite3`` command-line tool, use:
+
+.. code:: sh
+
+   env CEPH_ARGS='--log_to_file true --log-file sqlite3.log --debug_cephsqlite 20 --debug_ms 1' sqlite3 ...
+
+This will save all the usual Ceph debugging to a file ``sqlite3.log`` for inspection.
+
+
+.. _SQLite: https://sqlite.org/index.html
+.. _SQLite VFS: https://www.sqlite.org/vfs.html
+.. _SQLite Backup: https://www.sqlite.org/backup.html
+.. _SQLite Limits: https://www.sqlite.org/limits.html
+.. _SQLite Extension Loading API: https://sqlite.org/c3ref/load_extension.html
+.. _In-Memory Database: https://www.sqlite.org/inmemorydb.html
+.. _WAL Journal Mode: https://sqlite.org/wal.html
+.. _How to Corrupt Your SQLite Database: https://www.sqlite.org/howtocorrupt.html
+.. _JSON1 Extension: https://www.sqlite.org/json1.html
+.. _SQLite IO Methods: https://www.sqlite.org/c3ref/io_methods.html
diff --git a/doc/rados/api/librados-intro.rst b/doc/rados/api/librados-intro.rst
new file mode 100644
index 000000000..5174188b4
--- /dev/null
+++ b/doc/rados/api/librados-intro.rst
@@ -0,0 +1,1051 @@
+==========================
+ Introduction to librados
+==========================
+
+The :term:`Ceph Storage Cluster` provides the basic storage service that allows
+:term:`Ceph` to uniquely deliver **object, block, and file storage** in one
+unified system. However, you are not limited to using the RESTful, block, or
+POSIX interfaces. Based upon :abbr:`RADOS (Reliable Autonomic Distributed Object
+Store)`, the ``librados`` API enables you to create your own interface to the
+Ceph Storage Cluster.
+
+The ``librados`` API enables you to interact with the two types of daemons in
+the Ceph Storage Cluster: 
+
+- The :term:`Ceph Monitor`, which maintains a master copy of the cluster map. 
+- The :term:`Ceph OSD Daemon` (OSD), which stores data as objects on a storage node.
+
+.. ditaa::
+            +---------------------------------+
+            |  Ceph Storage Cluster Protocol  |
+            |           (librados)            |
+            +---------------------------------+
+            +---------------+ +---------------+
+            |      OSDs     | |    Monitors   |
+            +---------------+ +---------------+
+
+This guide provides a high-level introduction to using ``librados``. 
+Refer to :doc:`../../architecture` for additional details of the Ceph
+Storage Cluster. To use the API, you need a running Ceph Storage Cluster. 
+See `Installation (Quick)`_ for details.
+
+
+Step 1: Getting librados
+========================
+
+Your client application must bind with ``librados`` to connect to the Ceph
+Storage Cluster. You must install ``librados`` and any required packages to
+write applications that use ``librados``. The ``librados`` API is written in
+C++, with additional bindings for C, Python, Java and PHP. 
+
+
+Getting librados for C/C++
+--------------------------
+
+To install ``librados`` development support files for C/C++ on Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo apt-get install librados-dev
+
+To install ``librados`` development support files for C/C++ on RHEL/CentOS
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install librados2-devel
+
+Once you install ``librados`` for developers, you can find the required 
+headers for C/C++ under ``/usr/include/rados``:
+
+.. prompt:: bash $
+
+   ls /usr/include/rados
+
+
+Getting librados for Python
+---------------------------
+
+The ``rados`` module provides ``librados`` support to Python
+applications. You may install ``python3-rados`` for Debian, Ubuntu, SLE or
+openSUSE or the ``python-rados`` package for CentOS/RHEL.
+
+To install ``librados`` development support files for Python on Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo apt-get install python3-rados
+
+To install ``librados`` development support files for Python on RHEL/CentOS
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install python-rados
+
+To install ``librados`` development support files for Python on SLE/openSUSE
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo zypper install python3-rados
+
+You can find the module under ``/usr/share/pyshared`` on Debian systems,
+or under ``/usr/lib/python*/site-packages`` on CentOS/RHEL systems.
+
+
+Getting librados for Java
+-------------------------
+
+To install ``librados`` for Java, you need to execute the following procedure:
+
+#. Install ``jna.jar``. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
+
+      sudo apt-get install libjna-java
+
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
+
+      sudo yum install jna
+
+   The JAR files are located in ``/usr/share/java``.
+
+#. Clone the ``rados-java`` repository:
+
+   .. prompt:: bash $
+
+      git clone --recursive https://github.com/ceph/rados-java.git
+
+#. Build the ``rados-java`` repository:
+
+   .. prompt:: bash $
+
+      cd rados-java
+      ant
+
+   The JAR file is located under ``rados-java/target``.
+
+#. Copy the JAR for RADOS to a common location (e.g., ``/usr/share/java``) and 
+   ensure that it and the JNA JAR are in your JVM's classpath. For example:
+
+   .. prompt:: bash $
+
+      sudo cp target/rados-0.1.3.jar /usr/share/java/rados-0.1.3.jar
+      sudo ln -s /usr/share/java/jna-3.2.7.jar /usr/lib/jvm/default-java/jre/lib/ext/jna-3.2.7.jar  
+      sudo ln -s /usr/share/java/rados-0.1.3.jar  /usr/lib/jvm/default-java/jre/lib/ext/rados-0.1.3.jar
+
+To build the documentation, execute the following:
+
+.. prompt:: bash $
+
+   ant docs
+
+
+Getting librados for PHP
+-------------------------
+
+To install the ``librados`` extension for PHP, you need to execute the following procedure:
+
+#. Install php-dev. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
+
+      sudo apt-get install php5-dev build-essential
+
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
+
+      sudo yum install php-devel
+
+#. Clone the ``phprados`` repository:
+
+   .. prompt:: bash $
+
+      git clone https://github.com/ceph/phprados.git
+
+#. Build ``phprados``:
+
+   .. prompt:: bash $
+
+      cd phprados
+      phpize
+      ./configure
+      make
+      sudo make install
+
+#. Enable ``phprados`` by adding the following line to ``php.ini``::
+   
+      extension=rados.so
+
+
+Step 2: Configuring a Cluster Handle
+====================================
+
+A :term:`Ceph Client`, via ``librados``, interacts directly with OSDs to store
+and retrieve data. To interact with OSDs, the client app must invoke
+``librados``  and connect to a Ceph Monitor. Once connected, ``librados``
+retrieves the  :term:`Cluster Map` from the Ceph Monitor. When the client app
+wants to read or write data, it creates an I/O context and binds to a
+:term:`Pool`. The pool has an associated :term:`CRUSH rule` that defines how it
+will place data in the storage cluster. Via the I/O context, the client 
+provides the object name to ``librados``, which takes the object name
+and the cluster map (i.e., the topology of the cluster) and `computes`_ the
+placement group and `OSD`_  for locating the data. Then the client application
+can read or write data. The client app doesn't need to learn about the topology
+of the cluster directly.
+
+.. ditaa::
+            +--------+  Retrieves  +---------------+
+            | Client |------------>|  Cluster Map  |
+            +--------+             +---------------+
+                 |
+                 v      Writes
+              /-----\
+              | obj |
+              \-----/
+                 |      To
+                 v
+            +--------+           +---------------+
+            |  Pool  |---------->| CRUSH Rule |
+            +--------+  Selects  +---------------+
+
+
+The Ceph Storage Cluster handle encapsulates the client configuration, including:
+
+- The `user ID`_ for ``rados_create()`` or user name for ``rados_create2()`` 
+  (preferred).
+- The :term:`cephx` authentication key
+- The monitor ID and IP address
+- Logging levels
+- Debugging levels
+
+Thus, the first steps in using the cluster from your app are to 1) create
+a cluster handle that your app will use to connect to the storage cluster,
+and then 2) use that handle to connect. To connect to the cluster, the
+app must supply a monitor address, a username and an authentication key
+(cephx is enabled by default).
+
+.. tip:: Talking to different Ceph Storage Clusters – or to the same cluster 
+   with different users – requires different cluster handles.
+
+RADOS provides a number of ways for you to set the required values. For
+the monitor and encryption key settings, an easy way to handle them is to ensure
+that your Ceph configuration file contains a ``keyring`` path to a keyring file
+and at least one monitor address (e.g., ``mon_host``). For example::
+
+	[global]
+	mon_host = 192.168.1.1
+	keyring = /etc/ceph/ceph.client.admin.keyring
+
+Once you create the handle, you can read a Ceph configuration file to configure
+the handle. You can also pass arguments to your app and parse them with the
+function for parsing command line arguments (e.g., ``rados_conf_parse_argv()``),
+or parse Ceph environment variables (e.g., ``rados_conf_parse_env()``). Some
+wrappers may not implement convenience methods, so you may need to implement
+these capabilities. The following diagram provides a high-level flow for the
+initial connection.
+
+
+.. ditaa::
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |               |
+                |-----+ create  |
+                |     | cluster |
+                |<----+ handle  |
+                |               |
+                |-----+ read    |
+                |     | config  |
+                |<----+ file    |
+                |               |
+                |    connect    |
+                |-------------->|
+                |               |
+                |<--------------|
+                |   connected   |
+                |               |
+
+
+Once connected, your app can invoke functions that affect the whole cluster
+with only the cluster handle. For example, once you have a cluster
+handle, you can:
+
+- Get cluster statistics
+- Use Pool Operation (exists, create, list, delete)
+- Get and set the configuration
+
+
+One of the powerful features of Ceph is the ability to bind to different pools.
+Each pool may have a different number of placement groups, object replicas and
+replication strategies. For example, a pool could be set up as a "hot" pool that
+uses SSDs for frequently used objects or a "cold" pool that uses erasure coding.
+
+The main difference in the various ``librados`` bindings is between C and
+the object-oriented bindings for C++, Java and Python. The object-oriented
+bindings use objects to represent cluster handles, IO Contexts, iterators,
+exceptions, etc.
+
+
+C Example
+---------
+
+For C, creating a simple cluster handle using the ``admin`` user, configuring
+it and connecting to the cluster might look something like this: 
+
+.. code-block:: c
+
+	#include <stdio.h>
+	#include <stdlib.h>
+	#include <string.h>
+	#include <rados/librados.h>
+
+	int main (int argc, const char **argv) 
+	{
+
+		/* Declare the cluster handle and required arguments. */
+		rados_t cluster;
+		char cluster_name[] = "ceph";
+		char user_name[] = "client.admin";
+		uint64_t flags = 0;
+	
+		/* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */  
+		int err;
+		err = rados_create2(&cluster, cluster_name, user_name, flags);
+
+		if (err < 0) {
+			fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nCreated a cluster handle.\n");
+		}
+
+
+		/* Read a Ceph configuration file to configure the cluster handle. */
+		err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nRead the config file.\n");
+		}
+
+		/* Read command line arguments */
+		err = rados_conf_parse_argv(cluster, argc, argv);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nRead the command line arguments.\n");
+		}
+
+		/* Connect to the cluster */
+		err = rados_connect(cluster);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nConnected to the cluster.\n");
+		}
+
+	}
+
+Compile your client and link to ``librados`` using ``-lrados``. For example:
+
+.. prompt:: bash $
+
+   gcc ceph-client.c -lrados -o ceph-client
+
+
+C++ Example
+-----------
+
+The Ceph project provides a C++ example in the ``ceph/examples/librados``
+directory. For C++, a simple cluster handle using the ``admin`` user requires
+you to initialize a ``librados::Rados`` cluster handle object:
+
+.. code-block:: c++
+
+	#include <iostream>
+	#include <string>
+	#include <rados/librados.hpp>
+
+	int main(int argc, const char **argv)
+	{
+
+		int ret = 0;
+
+		/* Declare the cluster handle and required variables. */	
+		librados::Rados cluster;
+		char cluster_name[] = "ceph";
+		char user_name[] = "client.admin";
+		uint64_t flags = 0; 
+	
+		/* Initialize the cluster handle with the "ceph" cluster name and "client.admin" user */ 
+		{
+			ret = cluster.init2(user_name, cluster_name, flags);
+			if (ret < 0) {
+				std::cerr << "Couldn't initialize the cluster handle! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Created a cluster handle." << std::endl;
+			}
+		}
+
+		/* Read a Ceph configuration file to configure the cluster handle. */	
+		{	
+			ret = cluster.conf_read_file("/etc/ceph/ceph.conf");	
+			if (ret < 0) {
+				std::cerr << "Couldn't read the Ceph configuration file! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Read the Ceph configuration file." << std::endl;
+			}
+		}
+		
+		/* Read command line arguments */
+		{
+			ret = cluster.conf_parse_argv(argc, argv);
+			if (ret < 0) {
+				std::cerr << "Couldn't parse command line options! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Parsed command line options." << std::endl;
+			}
+		}
+	
+		/* Connect to the cluster */
+		{
+			ret = cluster.connect();
+			if (ret < 0) {
+				std::cerr << "Couldn't connect to cluster! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Connected to the cluster." << std::endl;
+			}
+		}
+	
+		return 0;
+	}
+	
+
+Compile the source; then, link ``librados`` using ``-lrados``. 
+For example:
+
+.. prompt:: bash $
+
+   g++ -g -c ceph-client.cc -o ceph-client.o
+   g++ -g ceph-client.o -lrados -o ceph-client
+
+
+
+Python Example
+--------------
+
+Python uses the ``admin`` id and the ``ceph`` cluster name by default, and
+will read the standard ``ceph.conf`` file if the conffile parameter is
+set to the empty string. The Python binding converts C++ errors
+into exceptions.
+
+
+.. code-block:: python
+
+	import rados
+
+	try:
+		cluster = rados.Rados(conffile='')
+	except TypeError as e:
+		print('Argument validation error: {}'.format(e))
+		raise e
+		
+	print("Created cluster handle.")
+
+	try:
+		cluster.connect()
+	except Exception as e:
+		print("connection error: {}".format(e))
+		raise e
+	finally:
+		print("Connected to the cluster.")
+
+
+Execute the example to verify that it connects to your cluster:
+
+.. prompt:: bash $
+
+   python ceph-client.py
+
+
+Java Example
+------------
+
+Java requires you to specify the user ID (``admin``) or user name
+(``client.admin``), and uses the ``ceph`` cluster name by default . The Java
+binding converts C++-based errors into exceptions.
+
+.. code-block:: java
+
+	import com.ceph.rados.Rados;
+	import com.ceph.rados.RadosException;
+	
+	import java.io.File;
+	
+	public class CephClient {
+		public static void main (String args[]){
+	
+			try {
+				Rados cluster = new Rados("admin");
+				System.out.println("Created cluster handle.");
+	            
+				File f = new File("/etc/ceph/ceph.conf");
+				cluster.confReadFile(f);
+				System.out.println("Read the configuration file.");
+
+				cluster.connect();
+				System.out.println("Connected to the cluster.");            
+
+			} catch (RadosException e) {
+				System.out.println(e.getMessage() + ": " + e.getReturnValue());
+			}
+		}
+	}
+
+
+Compile the source; then, run it. If you have copied the JAR to
+``/usr/share/java`` and sym linked from your ``ext`` directory, you won't need
+to specify the classpath. For example:
+
+.. prompt:: bash $
+
+   javac CephClient.java
+   java CephClient
+
+
+PHP Example
+------------
+
+With the RADOS extension enabled in PHP you can start creating a new cluster handle very easily:
+
+.. code-block:: php
+
+	<?php
+
+	$r = rados_create();
+	rados_conf_read_file($r, '/etc/ceph/ceph.conf');
+	if (!rados_connect($r)) {
+		echo "Failed to connect to Ceph cluster";
+	} else {
+		echo "Successfully connected to Ceph cluster";
+	}
+
+
+Save this as rados.php and run the code:
+
+.. prompt:: bash $
+
+   php rados.php
+
+
+Step 3: Creating an I/O Context
+===============================
+
+Once your app has a cluster handle and a connection to a Ceph Storage Cluster,
+you may create an I/O Context and begin reading and writing data. An I/O Context
+binds the connection to a specific pool. The user must have appropriate
+`CAPS`_ permissions to access the specified pool. For example, a user with read
+access but not write access will only be able to read data. I/O Context 
+functionality includes:
+
+- Write/read data and extended attributes
+- List and iterate over objects and extended attributes
+- Snapshot pools, list snapshots, etc.
+
+
+.. ditaa::
+           +---------+     +---------+     +---------+
+           | Client  |     | Monitor |     |   OSD   |
+           +---------+     +---------+     +---------+
+                |               |               |
+                |-----+ create  |               |
+                |     | I/O     |               | 
+                |<----+ context |               |              
+                |               |               |
+                |  write data   |               |
+                |---------------+-------------->|
+                |               |               |
+                |  write ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |  write xattr  |               |
+                |---------------+-------------->|
+                |               |               |
+                |  xattr ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |   read data   |               |
+                |---------------+-------------->|
+                |               |               |
+                |   read ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |  remove data  |               |
+                |---------------+-------------->|
+                |               |               |
+                |  remove ack   |               |
+                |<--------------+---------------|
+
+
+
+RADOS enables you to interact both synchronously and asynchronously. Once your
+app has an I/O Context, read/write operations only require you to know the
+object/xattr name. The CRUSH algorithm encapsulated in ``librados`` uses the
+cluster map to identify the appropriate OSD. OSD daemons handle the replication,
+as described in `Smart Daemons Enable Hyperscale`_. The ``librados`` library also 
+maps objects to placement groups, as described in  `Calculating PG IDs`_.
+
+The following examples use the default ``data`` pool. However, you may also
+use the API to list pools, ensure they exist, or create and delete pools. For 
+the write operations, the examples illustrate how to use synchronous mode. For
+the read operations, the examples illustrate how to use asynchronous mode.
+
+.. important:: Use caution when deleting pools with this API. If you delete
+   a pool, the pool and ALL DATA in the pool will be lost.
+
+
+C Example
+---------
+
+
+.. code-block:: c
+
+	#include <stdio.h>
+	#include <stdlib.h>
+	#include <string.h>
+	#include <rados/librados.h>
+
+	int main (int argc, const char **argv) 
+	{
+		/* 
+		 * Continued from previous C example, where cluster handle and
+		 * connection are established. First declare an I/O Context. 
+		 */
+
+		rados_ioctx_t io;
+		char *poolname = "data";
+	
+		err = rados_ioctx_create(cluster, poolname, &io);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_shutdown(cluster);
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nCreated I/O context.\n");
+		}
+
+		/* Write data to the cluster synchronously. */	
+		err = rados_write(io, "hw", "Hello World!", 12, 0);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot write object \"hw\" to pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nWrote \"Hello World\" to object \"hw\".\n");
+		}
+	
+		char xattr[] = "en_US";
+		err = rados_setxattr(io, "hw", "lang", xattr, 5);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot write xattr to pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nWrote \"en_US\" to xattr \"lang\" for object \"hw\".\n");
+		}
+	
+		/*
+		 * Read data from the cluster asynchronously. 
+		 * First, set up asynchronous I/O completion.
+		 */
+		rados_completion_t comp;
+		err = rados_aio_create_completion(NULL, NULL, NULL, &comp);
+		if (err < 0) {
+			fprintf(stderr, "%s: Could not create aio completion: %s\n", argv[0], strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nCreated AIO completion.\n");
+		}
+
+		/* Next, read data using rados_aio_read. */
+		char read_res[100];
+		err = rados_aio_read(io, "hw", comp, read_res, 12, 0);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot read object. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRead object \"hw\". The contents are:\n %s \n", read_res);
+		}
+		
+		/* Wait for the operation to complete */
+		rados_aio_wait_for_complete(comp);
+		
+		/* Release the asynchronous I/O complete handle to avoid memory leaks. */
+		rados_aio_release(comp);		
+		
+	
+		char xattr_res[100];
+		err = rados_getxattr(io, "hw", "lang", xattr_res, 5);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot read xattr. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRead xattr \"lang\" for object \"hw\". The contents are:\n %s \n", xattr_res);
+		}
+
+		err = rados_rmxattr(io, "hw", "lang");
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot remove xattr. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRemoved xattr \"lang\" for object \"hw\".\n");
+		}
+
+		err = rados_remove(io, "hw");
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot remove object. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRemoved object \"hw\".\n");
+		}
+
+	}
+
+
+
+C++ Example
+-----------
+
+
+.. code-block:: c++
+
+	#include <iostream>
+	#include <string>
+	#include <rados/librados.hpp>
+
+	int main(int argc, const char **argv)
+	{
+
+		/* Continued from previous C++ example, where cluster handle and
+		 * connection are established. First declare an I/O Context. 
+		 */
+
+		librados::IoCtx io_ctx;
+		const char *pool_name = "data";
+		
+		{
+			ret = cluster.ioctx_create(pool_name, io_ctx);
+			if (ret < 0) {
+				std::cerr << "Couldn't set up ioctx! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Created an ioctx for the pool." << std::endl;
+			}
+		}
+		
+
+		/* Write an object synchronously. */
+		{
+			librados::bufferlist bl;
+			bl.append("Hello World!");
+			ret = io_ctx.write_full("hw", bl);
+			if (ret < 0) {
+				std::cerr << "Couldn't write object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Wrote new object 'hw' " << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Add an xattr to the object.
+		 */
+		{
+			librados::bufferlist lang_bl;
+			lang_bl.append("en_US");
+			ret = io_ctx.setxattr("hw", "lang", lang_bl);
+			if (ret < 0) {
+				std::cerr << "failed to set xattr version entry! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Set the xattr 'lang' on our object!" << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Read the object back asynchronously.
+		 */
+		{
+			librados::bufferlist read_buf;
+			int read_len = 4194304;
+
+			//Create I/O Completion.
+			librados::AioCompletion *read_completion = librados::Rados::aio_create_completion();
+			
+			//Send read request.
+			ret = io_ctx.aio_read("hw", read_completion, &read_buf, read_len, 0);
+			if (ret < 0) {
+				std::cerr << "Couldn't start read object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			}
+
+			// Wait for the request to complete, and check that it succeeded.
+			read_completion->wait_for_complete();
+			ret = read_completion->get_return_value();
+			if (ret < 0) {
+				std::cerr << "Couldn't read object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Read object hw asynchronously with contents.\n"
+				<< read_buf.c_str() << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Read the xattr.
+		 */
+		{
+			librados::bufferlist lang_res;
+			ret = io_ctx.getxattr("hw", "lang", lang_res);
+			if (ret < 0) {
+				std::cerr << "failed to get xattr version entry! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Got the xattr 'lang' from object hw!"
+				<< lang_res.c_str() << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Remove the xattr.
+		 */
+		{
+			ret = io_ctx.rmxattr("hw", "lang");
+			if (ret < 0) {
+				std::cerr << "Failed to remove xattr! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Removed the xattr 'lang' from our object!" << std::endl;
+			}
+		}
+		
+		/*
+		 * Remove the object.
+		 */
+		{
+			ret = io_ctx.remove("hw");
+			if (ret < 0) {
+				std::cerr << "Couldn't remove object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Removed object 'hw'." << std::endl;
+			}
+		}
+	}
+
+
+
+Python Example
+--------------
+
+.. code-block:: python
+
+	print("\n\nI/O Context and Object Operations")
+	print("=================================")
+	
+	print("\nCreating a context for the 'data' pool")
+	if not cluster.pool_exists('data'):
+		raise RuntimeError('No data pool exists')
+	ioctx = cluster.open_ioctx('data')
+	
+	print("\nWriting object 'hw' with contents 'Hello World!' to pool 'data'.")
+	ioctx.write("hw", b"Hello World!")
+	print("Writing XATTR 'lang' with value 'en_US' to object 'hw'")
+	ioctx.set_xattr("hw", "lang", b"en_US")
+	
+	
+	print("\nWriting object 'bm' with contents 'Bonjour tout le monde!' to pool
+	'data'.")
+	ioctx.write("bm", b"Bonjour tout le monde!")
+	print("Writing XATTR 'lang' with value 'fr_FR' to object 'bm'")
+	ioctx.set_xattr("bm", "lang", b"fr_FR")
+	
+	print("\nContents of object 'hw'\n------------------------")
+	print(ioctx.read("hw"))
+	
+	print("\n\nGetting XATTR 'lang' from object 'hw'")
+	print(ioctx.get_xattr("hw", "lang"))
+	
+	print("\nContents of object 'bm'\n------------------------")
+	print(ioctx.read("bm"))
+	
+	print("\n\nGetting XATTR 'lang' from object 'bm'")
+	print(ioctx.get_xattr("bm", "lang"))
+	
+	
+	print("\nRemoving object 'hw'")
+	ioctx.remove_object("hw")
+	
+	print("Removing object 'bm'")
+	ioctx.remove_object("bm")
+
+
+Java-Example
+------------
+
+.. code-block:: java
+
+	import com.ceph.rados.Rados;
+	import com.ceph.rados.RadosException;
+
+	import java.io.File;
+	import com.ceph.rados.IoCTX;
+
+	public class CephClient {
+        	public static void main (String args[]){
+
+                	try {
+				Rados cluster = new Rados("admin");
+				System.out.println("Created cluster handle.");
+
+                        	File f = new File("/etc/ceph/ceph.conf");
+                        	cluster.confReadFile(f);
+                        	System.out.println("Read the configuration file.");
+
+                        	cluster.connect();
+                        	System.out.println("Connected to the cluster.");
+
+				IoCTX io = cluster.ioCtxCreate("data");
+
+				String oidone = "hw";
+				String contentone = "Hello World!";
+				io.write(oidone, contentone); 
+
+				String oidtwo = "bm";
+				String contenttwo = "Bonjour tout le monde!";
+				io.write(oidtwo, contenttwo); 
+
+				String[] objects = io.listObjects();
+                       		for (String object: objects)
+					System.out.println(object);
+
+				io.remove(oidone);
+				io.remove(oidtwo);
+
+				cluster.ioCtxDestroy(io);
+
+                	} catch (RadosException e) {
+                        	System.out.println(e.getMessage() + ": " + e.getReturnValue());
+                	}
+        	}
+	}
+
+
+PHP Example
+-----------
+
+.. code-block:: php
+
+	<?php
+
+	$io = rados_ioctx_create($r, "mypool");
+	rados_write_full($io, "oidOne", "mycontents");
+	rados_remove("oidOne");
+	rados_ioctx_destroy($io);
+
+
+Step 4: Closing Sessions
+========================
+
+Once your app finishes with the I/O Context and cluster handle, the app should
+close the connection and shutdown the handle. For asynchronous I/O, the app
+should also ensure that pending asynchronous operations have completed.
+
+
+C Example
+---------
+
+.. code-block:: c
+
+	rados_ioctx_destroy(io);
+	rados_shutdown(cluster);	
+
+
+C++ Example
+-----------
+
+.. code-block:: c++
+
+	io_ctx.close();
+	cluster.shutdown();
+
+
+Java Example
+--------------
+
+.. code-block:: java
+
+	cluster.ioCtxDestroy(io);
+	cluster.shutDown();
+	
+	
+Python Example
+--------------
+
+.. code-block:: python
+
+	print("\nClosing the connection.")
+	ioctx.close()
+	
+	print("Shutting down the handle.")
+	cluster.shutdown()
+
+PHP Example
+-----------
+
+.. code-block:: php
+
+	rados_shutdown($r);
+
+
+
+.. _user ID: ../../operations/user-management#command-line-usage
+.. _CAPS: ../../operations/user-management#authorization-capabilities
+.. _Installation (Quick): ../../../start
+.. _Smart Daemons Enable Hyperscale: ../../../architecture#smart-daemons-enable-hyperscale
+.. _Calculating PG IDs: ../../../architecture#calculating-pg-ids
+.. _computes: ../../../architecture#calculating-pg-ids
+.. _OSD: ../../../architecture#mapping-pgs-to-osds
diff --git a/doc/rados/api/librados.rst b/doc/rados/api/librados.rst
new file mode 100644
index 000000000..3e202bd4b
--- /dev/null
+++ b/doc/rados/api/librados.rst
@@ -0,0 +1,187 @@
+==============
+ Librados (C)
+==============
+
+.. highlight:: c
+
+`librados` provides low-level access to the RADOS service. For an
+overview of RADOS, see :doc:`../../architecture`.
+
+
+Example: connecting and writing an object
+=========================================
+
+To use `Librados`, you instantiate a :c:type:`rados_t` variable (a cluster handle) and
+call :c:func:`rados_create()` with a pointer to it::
+
+	int err;
+	rados_t cluster;
+
+	err = rados_create(&cluster, NULL);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot create a cluster handle: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Then you configure your :c:type:`rados_t` to connect to your cluster,
+either by setting individual values (:c:func:`rados_conf_set()`),
+using a configuration file (:c:func:`rados_conf_read_file()`), using
+command line options (:c:func:`rados_conf_parse_argv`), or an
+environment variable (:c:func:`rados_conf_parse_env()`)::
+
+	err = rados_conf_read_file(cluster, "/path/to/myceph.conf");
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Once the cluster handle is configured, you can connect to the cluster with :c:func:`rados_connect()`::
+
+	err = rados_connect(cluster);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Then you open an "IO context", a :c:type:`rados_ioctx_t`, with :c:func:`rados_ioctx_create()`::
+
+	rados_ioctx_t io;
+	char *poolname = "mypool";
+
+	err = rados_ioctx_create(cluster, poolname, &io);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+Note that the pool you try to access must exist.
+
+Then you can use the RADOS data manipulation functions, for example
+write into an object called ``greeting`` with
+:c:func:`rados_write_full()`::
+
+	err = rados_write_full(io, "greeting", "hello", 5);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot write pool %s: %s\n", argv[0], poolname, strerror(-err));
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+In the end, you will want to close your IO context and connection to RADOS with :c:func:`rados_ioctx_destroy()` and :c:func:`rados_shutdown()`::
+
+	rados_ioctx_destroy(io);
+	rados_shutdown(cluster);
+
+
+Asynchronous IO
+===============
+
+When doing lots of IO, you often don't need to wait for one operation
+to complete before starting the next one. `Librados` provides
+asynchronous versions of several operations:
+
+* :c:func:`rados_aio_write`
+* :c:func:`rados_aio_append`
+* :c:func:`rados_aio_write_full`
+* :c:func:`rados_aio_read`
+
+For each operation, you must first create a
+:c:type:`rados_completion_t` that represents what to do when the
+operation is safe or complete by calling
+:c:func:`rados_aio_create_completion`. If you don't need anything
+special to happen, you can pass NULL::
+
+	rados_completion_t comp;
+	err = rados_aio_create_completion(NULL, NULL, NULL, &comp);
+	if (err < 0) {
+		fprintf(stderr, "%s: could not create aio completion: %s\n", argv[0], strerror(-err));
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+Now you can call any of the aio operations, and wait for it to
+be in memory or on disk on all replicas::
+
+	err = rados_aio_write(io, "foo", comp, "bar", 3, 0);
+	if (err < 0) {
+		fprintf(stderr, "%s: could not schedule aio write: %s\n", argv[0], strerror(-err));
+		rados_aio_release(comp);
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+	rados_aio_wait_for_complete(comp); // in memory
+	rados_aio_wait_for_safe(comp); // on disk
+
+Finally, we need to free the memory used by the completion with :c:func:`rados_aio_release`::
+
+	rados_aio_release(comp);
+
+You can use the callbacks to tell your application when writes are
+durable, or when read buffers are full. For example, if you wanted to
+measure the latency of each operation when appending to several
+objects, you could schedule several writes and store the ack and
+commit time in the corresponding callback, then wait for all of them
+to complete using :c:func:`rados_aio_flush` before analyzing the
+latencies::
+
+	typedef struct {
+		struct timeval start;
+		struct timeval ack_end;
+		struct timeval commit_end;
+	} req_duration;
+
+	void ack_callback(rados_completion_t comp, void *arg) {
+		req_duration *dur = (req_duration *) arg;
+		gettimeofday(&dur->ack_end, NULL);
+	}
+
+	void commit_callback(rados_completion_t comp, void *arg) {
+		req_duration *dur = (req_duration *) arg;
+		gettimeofday(&dur->commit_end, NULL);
+	}
+
+	int output_append_latency(rados_ioctx_t io, const char *data, size_t len, size_t num_writes) {
+		req_duration times[num_writes];
+		rados_completion_t comps[num_writes];
+		for (size_t i = 0; i < num_writes; ++i) {
+			gettimeofday(&times[i].start, NULL);
+			int err = rados_aio_create_completion((void*) &times[i], ack_callback, commit_callback, &comps[i]);
+			if (err < 0) {
+				fprintf(stderr, "Error creating rados completion: %s\n", strerror(-err));
+				return err;
+			}
+			char obj_name[100];
+			snprintf(obj_name, sizeof(obj_name), "foo%ld", (unsigned long)i);
+			err = rados_aio_append(io, obj_name, comps[i], data, len);
+			if (err < 0) {
+				fprintf(stderr, "Error from rados_aio_append: %s", strerror(-err));
+				return err;
+			}
+		}
+		// wait until all requests finish *and* the callbacks complete
+		rados_aio_flush(io);
+		// the latencies can now be analyzed
+		printf("Request # | Ack latency (s) | Commit latency (s)\n");
+		for (size_t i = 0; i < num_writes; ++i) {
+			// don't forget to free the completions
+			rados_aio_release(comps[i]);
+			struct timeval ack_lat, commit_lat;
+			timersub(&times[i].ack_end, &times[i].start, &ack_lat);
+			timersub(&times[i].commit_end, &times[i].start, &commit_lat);
+			printf("%9ld | %8ld.%06ld | %10ld.%06ld\n", (unsigned long) i, ack_lat.tv_sec, ack_lat.tv_usec, commit_lat.tv_sec, commit_lat.tv_usec);
+		}
+		return 0;
+	}
+
+Note that all the :c:type:`rados_completion_t` must be freed with :c:func:`rados_aio_release` to avoid leaking memory.
+
+
+API calls
+=========
+
+ .. autodoxygenfile:: rados_types.h
+ .. autodoxygenfile:: librados.h
diff --git a/doc/rados/api/libradospp.rst b/doc/rados/api/libradospp.rst
new file mode 100644
index 000000000..08483c8d4
--- /dev/null
+++ b/doc/rados/api/libradospp.rst
@@ -0,0 +1,9 @@
+==================
+ LibradosPP (C++)
+==================
+
+.. note:: The librados C++ API is not guaranteed to be API+ABI stable
+   between major releases. All applications using the librados C++ API must
+   be recompiled and relinked against a specific Ceph release.
+
+.. todo:: write me!
diff --git a/doc/rados/api/objclass-sdk.rst b/doc/rados/api/objclass-sdk.rst
new file mode 100644
index 000000000..90b8eb018
--- /dev/null
+++ b/doc/rados/api/objclass-sdk.rst
@@ -0,0 +1,39 @@
+.. _`rados-objclass-api-sdk`:
+
+===========================
+SDK for Ceph Object Classes
+===========================
+
+`Ceph` can be extended by creating shared object classes called `Ceph Object 
+Classes`. The existing framework to build these object classes has dependencies 
+on the internal functionality of `Ceph`, which restricts users to build object 
+classes within the tree. The aim of this project is to create an independent 
+object class interface, which can be used to build object classes outside the 
+`Ceph` tree. This allows us to have two types of object classes, 1) those that 
+have in-tree dependencies and reside in the tree and 2) those that can make use 
+of the `Ceph Object Class SDK framework` and can be built outside of the `Ceph` 
+tree because they do not depend on any internal implementation of `Ceph`. This 
+project decouples object class development from Ceph and encourages creation 
+and distribution of object classes as packages.
+
+In order to demonstrate the use of this framework, we have provided an example 
+called ``cls_sdk``, which is a very simple object class that makes use of the 
+SDK framework. This object class resides in the ``src/cls`` directory. 
+
+Installing objclass.h
+---------------------
+
+The object class interface that enables out-of-tree development of object 
+classes resides in ``src/include/rados/`` and gets installed with `Ceph` 
+installation. After running ``make install``, you should be able to see it 
+in ``<prefix>/include/rados``. ::
+
+        ls /usr/local/include/rados
+
+Using the SDK example
+---------------------
+
+The ``cls_sdk`` object class resides in ``src/cls/sdk/``. This gets built and 
+loaded into Ceph, with the Ceph build process. You can run the 
+``ceph_test_cls_sdk`` unittest, which resides in ``src/test/cls_sdk/``, 
+to test this class.
diff --git a/doc/rados/api/python.rst b/doc/rados/api/python.rst
new file mode 100644
index 000000000..346653a3d
--- /dev/null
+++ b/doc/rados/api/python.rst
@@ -0,0 +1,428 @@
+===================
+ Librados (Python)
+===================
+
+The ``rados`` module is a thin Python wrapper for ``librados``.
+
+Installation
+============
+
+To install Python libraries for Ceph, see `Getting librados for Python`_.
+
+
+Getting Started
+===============
+
+You can create your own Ceph client using Python. The following tutorial will
+show you how to import the Ceph Python module, connect to a Ceph cluster,  and
+perform object operations as a ``client.admin`` user.
+
+.. note:: To use the Ceph Python bindings, you must have access to a
+   running Ceph cluster. To set one up quickly, see `Getting Started`_.
+
+First, create a Python source file for your Ceph client.
+
+.. prompt:: bash
+
+	vim client.py
+
+
+Import the Module
+-----------------
+
+To use the ``rados`` module, import it into your source file.
+
+.. code-block:: python
+   :linenos:
+
+	import rados
+
+
+Configure a Cluster Handle
+--------------------------
+
+Before connecting to the Ceph Storage Cluster, create a cluster handle. By
+default, the cluster handle assumes a cluster named ``ceph`` (i.e., the default
+for deployment tools, and our Getting Started guides too),  and a
+``client.admin`` user name. You may change these defaults to suit your needs.
+
+To connect to the Ceph Storage Cluster, your application needs to know where to
+find the  Ceph Monitor. Provide this information to your application by
+specifying the path to your Ceph configuration file, which contains the location
+of the initial Ceph monitors.
+
+.. code-block:: python
+   :linenos:
+
+	import rados, sys
+
+	#Create Handle Examples.
+	cluster = rados.Rados(conffile='ceph.conf')
+	cluster = rados.Rados(conffile=sys.argv[1])
+	cluster = rados.Rados(conffile = 'ceph.conf', conf = dict (keyring = '/path/to/keyring'))
+
+Ensure that the ``conffile`` argument provides the path and file name of your
+Ceph configuration file. You may use the ``sys`` module to avoid hard-coding the
+Ceph configuration path and file name.
+
+Your Python client also requires a client keyring. For this example, we use the
+``client.admin`` key by default. If you would like to specify the keyring when
+creating the cluster handle, you may use the ``conf`` argument. Alternatively,
+you may specify the keyring path in your Ceph configuration file. For example,
+you may add something like the following line to your Ceph configuration file::
+
+	keyring = /path/to/ceph.client.admin.keyring
+
+For additional details on modifying your configuration via Python, see `Configuration`_.
+
+
+Connect to the Cluster
+----------------------
+
+Once you have a cluster handle configured, you may connect to the cluster.
+With a connection to the cluster, you may execute methods that return
+information about the cluster.
+
+.. code-block:: python
+   :linenos:
+   :emphasize-lines: 7
+
+	import rados, sys
+
+	cluster = rados.Rados(conffile='ceph.conf')
+	print("\nlibrados version: {}".format(str(cluster.version())))
+	print("Will attempt to connect to: {}".format(str(cluster.conf_get('mon host'))))
+
+	cluster.connect()
+	print("\nCluster ID: {}".format(cluster.get_fsid()))
+
+	print("\n\nCluster Statistics")
+	print("==================")
+	cluster_stats = cluster.get_cluster_stats()
+
+	for key, value in cluster_stats.items():
+		print(key, value)
+
+
+By default, Ceph authentication is ``on``. Your application will need to know
+the location of the keyring. The ``python-ceph`` module doesn't have the default
+location, so you need to specify the keyring path. The easiest way to specify
+the keyring is to add it to the Ceph configuration file. The following Ceph
+configuration file example uses the ``client.admin`` keyring.
+
+.. code-block:: ini
+   :linenos:
+
+	[global]
+	# ... elided configuration
+	keyring = /path/to/keyring/ceph.client.admin.keyring
+
+
+Manage Pools
+------------
+
+When connected to the cluster, the ``Rados`` API allows you to manage pools. You
+can list pools, check for the existence of a pool, create a pool and delete a
+pool.
+
+.. code-block:: python
+   :linenos:
+   :emphasize-lines: 6, 13, 18, 25
+
+	print("\n\nPool Operations")
+	print("===============")
+
+	print("\nAvailable Pools")
+	print("----------------")
+	pools = cluster.list_pools()
+
+	for pool in pools:
+		print(pool)
+
+	print("\nCreate 'test' Pool")
+	print("------------------")
+	cluster.create_pool('test')
+
+	print("\nPool named 'test' exists: {}".format(str(cluster.pool_exists('test'))))
+	print("\nVerify 'test' Pool Exists")
+	print("-------------------------")
+	pools = cluster.list_pools()
+
+	for pool in pools:
+		print(pool)
+
+	print("\nDelete 'test' Pool")
+	print("------------------")
+	cluster.delete_pool('test')
+	print("\nPool named 'test' exists: {}".format(str(cluster.pool_exists('test'))))
+
+
+Input/Output Context
+--------------------
+
+Reading from and writing to the Ceph Storage Cluster requires an input/output
+context (ioctx). You can create an ioctx with the ``open_ioctx()`` or
+``open_ioctx2()`` method of the ``Rados`` class. The ``ioctx_name`` parameter
+is the name of the  pool and ``pool_id`` is the ID of the pool you wish to use.
+
+.. code-block:: python
+   :linenos:
+
+	ioctx = cluster.open_ioctx('data')
+
+
+or
+
+.. code-block:: python
+   :linenos:
+
+        ioctx = cluster.open_ioctx2(pool_id)
+
+
+Once you have an I/O context, you can read/write objects, extended attributes,
+and perform a number of other operations. After you complete operations, ensure
+that you close the connection. For example:
+
+.. code-block:: python
+   :linenos:
+
+	print("\nClosing the connection.")
+	ioctx.close()
+
+
+Writing, Reading and Removing Objects
+-------------------------------------
+
+Once you create an I/O context, you can write objects to the cluster. If you
+write to an object that doesn't exist, Ceph creates it. If you write to an
+object that exists, Ceph overwrites it (except when you specify a range, and
+then it only overwrites the range). You may read objects (and object ranges)
+from the cluster. You may also remove objects from the cluster. For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 2, 5, 8
+
+	print("\nWriting object 'hw' with contents 'Hello World!' to pool 'data'.")
+	ioctx.write_full("hw", "Hello World!")
+
+	print("\n\nContents of object 'hw'\n------------------------\n")
+	print(ioctx.read("hw"))
+
+	print("\nRemoving object 'hw'")
+	ioctx.remove_object("hw")
+
+
+Writing and Reading XATTRS
+--------------------------
+
+Once you create an object, you can write extended attributes (XATTRs) to
+the object and read XATTRs from the object. For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 2, 5
+
+	print("\n\nWriting XATTR 'lang' with value 'en_US' to object 'hw'")
+	ioctx.set_xattr("hw", "lang", "en_US")
+
+	print("\n\nGetting XATTR 'lang' from object 'hw'\n")
+	print(ioctx.get_xattr("hw", "lang"))
+
+
+Listing Objects
+---------------
+
+If you want to examine the list of objects in a pool, you may
+retrieve the list of objects and iterate over them with the object iterator.
+For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 1, 6, 7, 13
+
+	object_iterator = ioctx.list_objects()
+
+	while True :
+
+		try :
+			rados_object = object_iterator.__next__()
+			print("Object contents = {}".format(rados_object.read()))
+
+		except StopIteration :
+			break
+
+	# Or alternatively
+	[print("Object contents = {}".format(obj.read())) for obj in ioctx.list_objects()]
+
+The ``Object`` class provides a file-like interface to an object, allowing
+you to read and write content and extended attributes. Object operations using
+the I/O context provide additional functionality and asynchronous capabilities.
+
+
+Cluster Handle API
+==================
+
+The ``Rados`` class provides an interface into the Ceph Storage Daemon.
+
+
+Configuration
+-------------
+
+The ``Rados`` class provides methods for getting and setting configuration
+values, reading the Ceph configuration file, and parsing arguments. You
+do not need to be connected to the Ceph Storage Cluster to invoke the following
+methods. See `Storage Cluster Configuration`_ for details on settings.
+
+.. currentmodule:: rados
+.. automethod:: Rados.conf_get(option)
+.. automethod:: Rados.conf_set(option, val)
+.. automethod:: Rados.conf_read_file(path=None)
+.. automethod:: Rados.conf_parse_argv(args)
+.. automethod:: Rados.version()
+
+
+Connection Management
+---------------------
+
+Once you configure your cluster handle, you may connect to the cluster, check
+the cluster ``fsid``, retrieve cluster statistics, and disconnect (shutdown)
+from the cluster. You may also assert that the cluster handle is in a particular
+state (e.g., "configuring", "connecting", etc.).
+
+.. automethod:: Rados.connect(timeout=0)
+.. automethod:: Rados.shutdown()
+.. automethod:: Rados.get_fsid()
+.. automethod:: Rados.get_cluster_stats()
+
+.. documented manually because it raises warnings because of *args usage in the
+.. signature
+
+.. py:class:: Rados
+
+   .. py:method:: require_state(*args)
+
+      Checks if the Rados object is in a special state
+
+      :param args: Any number of states to check as separate arguments
+      :raises: :class:`RadosStateError`
+
+
+Pool Operations
+---------------
+
+To use pool operation methods, you must connect to the Ceph Storage Cluster
+first.  You may list the available pools, create a pool, check to see if a pool
+exists,  and delete a pool.
+
+.. automethod:: Rados.list_pools()
+.. automethod:: Rados.create_pool(pool_name, crush_rule=None)
+.. automethod:: Rados.pool_exists()
+.. automethod:: Rados.delete_pool(pool_name)
+
+
+CLI Commands
+------------
+
+The Ceph CLI command is internally using the following librados Python binding methods.
+
+In order to send a command, choose the correct method and choose the correct target.
+
+.. automethod:: Rados.mon_command
+.. automethod:: Rados.osd_command
+.. automethod:: Rados.mgr_command
+.. automethod:: Rados.pg_command
+
+
+Input/Output Context API
+========================
+
+To write data to and read data from the Ceph Object Store, you must create
+an Input/Output context (ioctx). The `Rados` class provides `open_ioctx()`
+and `open_ioctx2()` methods. The remaining ``ioctx`` operations involve
+invoking methods of the `Ioctx` and other classes.
+
+.. automethod:: Rados.open_ioctx(ioctx_name)
+.. automethod:: Ioctx.require_ioctx_open()
+.. automethod:: Ioctx.get_stats()
+.. automethod:: Ioctx.get_last_version()
+.. automethod:: Ioctx.close()
+
+
+.. Pool Snapshots
+.. --------------
+
+.. The Ceph Storage Cluster allows you to make a snapshot of a pool's state.
+.. Whereas, basic pool operations only require a connection to the cluster,
+.. snapshots require an I/O context.
+
+.. Ioctx.create_snap(self, snap_name)
+.. Ioctx.list_snaps(self)
+.. SnapIterator.next(self)
+.. Snap.get_timestamp(self)
+.. Ioctx.lookup_snap(self, snap_name)
+.. Ioctx.remove_snap(self, snap_name)
+
+.. not published. This doesn't seem ready yet.
+
+Object Operations
+-----------------
+
+The Ceph Storage Cluster stores data as objects. You can read and write objects
+synchronously or asynchronously. You can read and write from offsets. An object
+has a name (or key) and data.
+
+
+.. automethod:: Ioctx.aio_write(object_name, to_write, offset=0, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.aio_write_full(object_name, to_write, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.aio_append(object_name, to_append, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.write(key, data, offset=0)
+.. automethod:: Ioctx.write_full(key, data)
+.. automethod:: Ioctx.aio_flush()
+.. automethod:: Ioctx.set_locator_key(loc_key)
+.. automethod:: Ioctx.aio_read(object_name, length, offset, oncomplete)
+.. automethod:: Ioctx.read(key, length=8192, offset=0)
+.. automethod:: Ioctx.stat(key)
+.. automethod:: Ioctx.trunc(key, size)
+.. automethod:: Ioctx.remove_object(key)
+
+
+Object Extended Attributes
+--------------------------
+
+You may set extended attributes (XATTRs) on an object. You can retrieve a list
+of objects or XATTRs and iterate over them.
+
+.. automethod:: Ioctx.set_xattr(key, xattr_name, xattr_value)
+.. automethod:: Ioctx.get_xattrs(oid)
+.. automethod:: XattrIterator.__next__()
+.. automethod:: Ioctx.get_xattr(key, xattr_name)
+.. automethod:: Ioctx.rm_xattr(key, xattr_name)
+
+
+
+Object Interface
+================
+
+From an I/O context, you can retrieve a list of objects from a pool and iterate
+over them. The object interface provide makes each object look like a file, and
+you may perform synchronous operations on the  objects. For asynchronous
+operations, you should use the I/O context methods.
+
+.. automethod:: Ioctx.list_objects()
+.. automethod:: ObjectIterator.__next__()
+.. automethod:: Object.read(length = 1024*1024)
+.. automethod:: Object.write(string_to_write)
+.. automethod:: Object.get_xattrs()
+.. automethod:: Object.get_xattr(xattr_name)
+.. automethod:: Object.set_xattr(xattr_name, xattr_value)
+.. automethod:: Object.rm_xattr(xattr_name)
+.. automethod:: Object.stat()
+.. automethod:: Object.remove()
+
+
+
+
+.. _Getting Started: ../../../start
+.. _Storage Cluster Configuration: ../../configuration
+.. _Getting librados for Python: ../librados-intro#getting-librados-for-python
diff --git a/doc/rados/command/list-inconsistent-obj.json b/doc/rados/command/list-inconsistent-obj.json
new file mode 100644
index 000000000..2bdc5f74c
--- /dev/null
+++ b/doc/rados/command/list-inconsistent-obj.json
@@ -0,0 +1,237 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "epoch": {
+      "description": "Scrub epoch",
+      "type": "integer"
+    },
+    "inconsistents": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "object": {
+            "description": "Identify a Ceph object",
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "nspace": {
+                "type": "string"
+              },
+              "locator": {
+                "type": "string"
+              },
+              "version": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "snap": {
+                "oneOf": [
+                  {
+                    "type": "string",
+                    "enum": [ "head", "snapdir" ]
+                  },
+                  {
+                    "type": "integer",
+                    "minimum": 0
+                  }
+                ]
+              }
+            },
+            "required": [
+              "name",
+              "nspace",
+              "locator",
+              "version",
+              "snap"
+            ]
+          },
+          "selected_object_info": {
+              "type": "object",
+              "description": "Selected object information",
+              "additionalProperties": true
+          },
+          "union_shard_errors": {
+            "description": "Union of all shard errors",
+            "type": "array",
+            "items": {
+              "enum": [
+                "missing",
+                "stat_error",
+                "read_error",
+                "data_digest_mismatch_info",
+                "omap_digest_mismatch_info",
+                "size_mismatch_info",
+                "ec_hash_error",
+                "ec_size_error",
+                "info_missing",
+                "info_corrupted",
+                "obj_size_info_mismatch",
+                "snapset_missing",
+                "snapset_corrupted",
+                "hinfo_missing",
+                "hinfo_corrupted"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "errors": {
+            "description": "Errors related to the analysis of this object",
+            "type": "array",
+            "items": {
+              "enum": [
+                "object_info_inconsistency",
+                "data_digest_mismatch",
+                "omap_digest_mismatch",
+                "size_mismatch",
+                "attr_value_mismatch",
+                "attr_name_mismatch",
+                "snapset_inconsistency",
+                "hinfo_inconsistency",
+		"size_too_large"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "shards": {
+            "description": "All found or expected shards",
+            "type": "array",
+            "items": {
+              "description": "Information about a particular shard of object",
+              "type": "object",
+              "properties": {
+                "object_info": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Object information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "snapset": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Snap set information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "hashinfo": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Erasure code hash information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "shard": {
+                  "type": "integer"
+                },
+                "osd": {
+                  "type": "integer"
+                },
+                "primary": {
+                  "type": "boolean"
+                },
+                "size": {
+                  "type": "integer"
+                },
+                "omap_digest": {
+                  "description": "Hex representation (e.g. 0x1abd1234)",
+                  "type": "string"
+                },
+                "data_digest": {
+                  "description": "Hex representation (e.g. 0x1abd1234)",
+                  "type": "string"
+                },
+                "errors": {
+                  "description": "Errors with this shard",
+                  "type": "array",
+                  "items": {
+                    "enum": [
+                      "missing",
+                      "stat_error",
+                      "read_error",
+                      "data_digest_mismatch_info",
+                      "omap_digest_mismatch_info",
+                      "size_mismatch_info",
+                      "ec_hash_error",
+                      "ec_size_error",
+                      "info_missing",
+                      "info_corrupted",
+                      "obj_size_info_mismatch",
+                      "snapset_missing",
+                      "snapset_corrupted",
+                      "hinfo_missing",
+                      "hinfo_corrupted"
+                    ]
+                  },
+                  "minItems": 0,
+                  "uniqueItems": true
+                },
+                "attrs": {
+                  "description": "If any shard's attr error is set then all attrs are here",
+                  "type": "array",
+                  "items": {
+                    "description": "Information about a particular shard of object",
+                    "type": "object",
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "value": {
+                        "type": "string"
+                      },
+                      "Base64": {
+                        "type": "boolean"
+                      }
+                    },
+                    "required": [
+                      "name",
+                      "value",
+                      "Base64"
+                    ],
+                    "additionalProperties": false
+                  }
+                }
+              },
+              "additionalProperties": false,
+              "required": [
+                "osd",
+                "primary",
+                "errors"
+              ]
+            }
+          }
+        },
+        "required": [
+          "object",
+          "union_shard_errors",
+          "errors",
+          "shards"
+        ]
+      }
+    }
+  },
+  "required": [
+    "epoch",
+    "inconsistents"
+  ]
+}
diff --git a/doc/rados/command/list-inconsistent-snap.json b/doc/rados/command/list-inconsistent-snap.json
new file mode 100644
index 000000000..55f1d53e9
--- /dev/null
+++ b/doc/rados/command/list-inconsistent-snap.json
@@ -0,0 +1,86 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "epoch": {
+      "description": "Scrub epoch",
+      "type": "integer"
+    },
+    "inconsistents": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "nspace": {
+            "type": "string"
+          },
+          "locator": {
+            "type": "string"
+          },
+          "snap": {
+            "oneOf": [
+              {
+                "type": "string",
+                "enum": [
+                  "head",
+                  "snapdir"
+                ]
+              },
+              {
+                "type": "integer",
+                "minimum": 0
+              }
+            ]
+          },
+          "errors": {
+            "description": "Errors for this object's snap",
+            "type": "array",
+            "items": {
+              "enum": [
+                "snapset_missing",
+                "snapset_corrupted",
+                "info_missing",
+                "info_corrupted",
+                "snapset_error",
+                "headless",
+                "size_mismatch",
+                "extra_clones",
+                "clone_missing"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "missing": {
+            "description": "List of missing clones if clone_missing error set",
+            "type": "array",
+            "items": {
+              "type": "integer"
+            }
+          },
+          "extra_clones": {
+            "description": "List of extra clones if extra_clones error set",
+            "type": "array",
+            "items": {
+              "type": "integer"
+            }
+          }
+        },
+        "required": [
+          "name",
+          "nspace",
+          "locator",
+          "snap",
+          "errors"
+        ]
+      }
+    }
+  },
+  "required": [
+    "epoch",
+    "inconsistents"
+  ]
+}
diff --git a/doc/rados/configuration/auth-config-ref.rst b/doc/rados/configuration/auth-config-ref.rst
new file mode 100644
index 000000000..fc14f4ee6
--- /dev/null
+++ b/doc/rados/configuration/auth-config-ref.rst
@@ -0,0 +1,379 @@
+.. _rados-cephx-config-ref:
+
+========================
+ CephX Config Reference
+========================
+
+The CephX protocol is enabled by default. The cryptographic authentication that
+CephX provides has some computational costs, though they should generally be
+quite low. If the network environment connecting your client and server hosts
+is very safe and you cannot afford authentication, you can disable it.
+**Disabling authentication is not generally recommended**.
+
+.. note:: If you disable authentication, you will be at risk of a
+   man-in-the-middle attack that alters your client/server messages, which
+   could have disastrous security effects.
+
+For information about creating users, see `User Management`_. For details on
+the architecture of CephX, see `Architecture - High Availability
+Authentication`_.
+
+
+Deployment Scenarios
+====================
+
+How you initially configure CephX depends on your scenario. There are two
+common strategies for deploying a Ceph cluster.  If you are a first-time Ceph
+user, you should probably take the easiest approach: using ``cephadm`` to
+deploy a cluster. But if your cluster uses other deployment tools (for example,
+Ansible, Chef, Juju, or Puppet), you will need either to use the manual
+deployment procedures or to configure your deployment tool so that it will
+bootstrap your monitor(s).
+
+Manual Deployment
+-----------------
+
+When you deploy a cluster manually, it is necessary to bootstrap the monitors
+manually and to create the ``client.admin`` user and keyring. To bootstrap
+monitors, follow the steps in `Monitor Bootstrapping`_. Follow these steps when
+using third-party deployment tools (for example, Chef, Puppet, and Juju).
+
+
+Enabling/Disabling CephX
+========================
+
+Enabling CephX is possible only if the keys for your monitors, OSDs, and
+metadata servers have already been deployed. If you are simply toggling CephX
+on or off, it is not necessary to repeat the bootstrapping procedures.
+
+Enabling CephX
+--------------
+
+When CephX is enabled, Ceph will look for the keyring in the default search
+path: this path includes ``/etc/ceph/$cluster.$name.keyring``. It is possible
+to override this search-path location by adding a ``keyring`` option in the
+``[global]`` section of your `Ceph configuration`_ file, but this is not
+recommended.
+
+To enable CephX on a cluster for which authentication has been disabled, carry
+out the following procedure.  If you (or your deployment utility) have already
+generated the keys, you may skip the steps related to generating keys.
+
+#. Create a ``client.admin`` key, and save a copy of the key for your client
+   host:
+
+   .. prompt:: bash $
+
+     ceph auth get-or-create client.admin mon 'allow *' mds 'allow *' mgr 'allow *' osd 'allow *' -o /etc/ceph/ceph.client.admin.keyring
+
+   **Warning:** This step will clobber any existing
+   ``/etc/ceph/client.admin.keyring`` file. Do not perform this step if a
+   deployment tool has already generated a keyring file for you. Be careful!
+
+#. Create a monitor keyring and generate a monitor secret key:
+
+   .. prompt:: bash $
+
+     ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+#. For each monitor, copy the monitor keyring into a ``ceph.mon.keyring`` file
+   in the monitor's ``mon data`` directory. For example, to copy the monitor
+   keyring to ``mon.a`` in a cluster called ``ceph``, run the following
+   command:
+
+   .. prompt:: bash $
+
+     cp /tmp/ceph.mon.keyring /var/lib/ceph/mon/ceph-a/keyring
+
+#. Generate a secret key for every MGR, where ``{$id}`` is the MGR letter:
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create mgr.{$id} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o /var/lib/ceph/mgr/ceph-{$id}/keyring
+
+#. Generate a secret key for every OSD, where ``{$id}`` is the OSD number:
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create osd.{$id} mon 'allow rwx' osd 'allow *' -o /var/lib/ceph/osd/ceph-{$id}/keyring
+
+#. Generate a secret key for every MDS, where ``{$id}`` is the MDS letter:
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create mds.{$id} mon 'allow rwx' osd 'allow *' mds 'allow *' mgr 'allow profile mds' -o /var/lib/ceph/mds/ceph-{$id}/keyring
+
+#. Enable CephX authentication by setting the following options in the
+   ``[global]`` section of your `Ceph configuration`_ file:
+
+   .. code-block:: ini
+
+      auth_cluster_required = cephx
+      auth_service_required = cephx
+      auth_client_required = cephx
+
+#. Start or restart the Ceph cluster. For details, see `Operating a Cluster`_.
+
+For details on bootstrapping a monitor manually, see `Manual Deployment`_.
+
+
+
+Disabling CephX
+---------------
+
+The following procedure describes how to disable CephX. If your cluster
+environment is safe, you might want to disable CephX in order to offset the
+computational expense of running authentication. **We do not recommend doing
+so.** However, setup and troubleshooting might be easier if authentication is
+temporarily disabled and subsequently re-enabled.
+
+#. Disable CephX authentication by setting the following options in the
+   ``[global]`` section of your `Ceph configuration`_ file:
+
+   .. code-block:: ini
+
+      auth_cluster_required = none
+      auth_service_required = none
+      auth_client_required = none
+
+#. Start or restart the Ceph cluster. For details, see `Operating a Cluster`_.
+
+
+Configuration Settings
+======================
+
+Enablement
+----------
+
+
+``auth_cluster_required``
+
+:Description: If this configuration setting is enabled, the Ceph Storage
+              Cluster daemons (that is, ``ceph-mon``, ``ceph-osd``,
+              ``ceph-mds``, and ``ceph-mgr``) are required to authenticate with
+              each other. Valid settings are ``cephx`` or ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+``auth_service_required``
+
+:Description: If this configuration setting is enabled, then Ceph clients can
+              access Ceph services only if those clients authenticate with the
+              Ceph Storage Cluster.  Valid settings are ``cephx`` or ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+``auth_client_required``
+
+:Description: If this configuration setting is enabled, then communication
+              between the Ceph client and Ceph Storage Cluster can be
+              established only if the Ceph Storage Cluster authenticates
+              against the Ceph client. Valid settings are ``cephx`` or
+              ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+.. index:: keys; keyring
+
+Keys
+----
+
+When Ceph is run with authentication enabled, ``ceph`` administrative commands
+and Ceph clients can access the Ceph Storage Cluster only if they use
+authentication keys.
+
+The most common way to make these keys available to ``ceph`` administrative
+commands and Ceph clients is to include a Ceph keyring under the ``/etc/ceph``
+directory. For Octopus and later releases that use ``cephadm``, the filename is
+usually ``ceph.client.admin.keyring``.  If the keyring is included in the
+``/etc/ceph`` directory, then it is unnecessary to specify a ``keyring`` entry
+in the Ceph configuration file.
+
+Because the Ceph Storage Cluster's keyring file contains the ``client.admin``
+key, we recommend copying the keyring file to nodes from which you run
+administrative commands.
+
+To perform this step manually, run the following command:
+
+.. prompt:: bash $
+
+   sudo scp {user}@{ceph-cluster-host}:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring
+
+.. tip:: Make sure that the ``ceph.keyring`` file has appropriate permissions
+   (for example, ``chmod 644``) set on your client machine.
+
+You can specify the key itself by using the ``key`` setting in the Ceph
+configuration file (this approach is not recommended), or instead specify a
+path to a keyfile by using the ``keyfile`` setting in the Ceph configuration
+file.
+
+``keyring``
+
+:Description: The path to the keyring file.
+:Type: String
+:Required: No
+:Default: ``/etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin``
+
+
+``keyfile``
+
+:Description: The path to a keyfile (that is, a file containing only the key).
+:Type: String
+:Required: No
+:Default: None
+
+
+``key``
+
+:Description: The key (that is, the text string of the key itself). We do not
+              recommend that you use this setting unless you know what you're
+              doing.
+:Type: String
+:Required: No
+:Default: None
+
+
+Daemon Keyrings
+---------------
+
+Administrative users or deployment tools (for example, ``cephadm``) generate
+daemon keyrings in the same way that they generate user keyrings. By default,
+Ceph stores the keyring of a daemon inside that daemon's data directory. The
+default keyring locations and the capabilities that are necessary for the
+daemon to function are shown below.
+
+``ceph-mon``
+
+:Location: ``$mon_data/keyring``
+:Capabilities: ``mon 'allow *'``
+
+``ceph-osd``
+
+:Location: ``$osd_data/keyring``
+:Capabilities: ``mgr 'allow profile osd' mon 'allow profile osd' osd 'allow *'``
+
+``ceph-mds``
+
+:Location: ``$mds_data/keyring``
+:Capabilities: ``mds 'allow' mgr 'allow profile mds' mon 'allow profile mds' osd 'allow rwx'``
+
+``ceph-mgr``
+
+:Location: ``$mgr_data/keyring``
+:Capabilities: ``mon 'allow profile mgr' mds 'allow *' osd 'allow *'``
+
+``radosgw``
+
+:Location: ``$rgw_data/keyring``
+:Capabilities: ``mon 'allow rwx' osd 'allow rwx'``
+
+
+.. note:: The monitor keyring (that is, ``mon.``) contains a key but no
+   capabilities, and this keyring is not part of the cluster ``auth`` database.
+
+The daemon's data-directory locations default to directories of the form::
+
+  /var/lib/ceph/$type/$cluster-$id
+
+For example, ``osd.12`` would have the following data directory::
+
+  /var/lib/ceph/osd/ceph-12
+
+It is possible to override these locations, but it is not recommended.
+
+
+.. index:: signatures
+
+Signatures
+----------
+
+Ceph performs a signature check that provides some limited protection against
+messages being tampered with in flight (for example, by a "man in the middle"
+attack).
+
+As with other parts of Ceph authentication, signatures admit of fine-grained
+control.  You can enable or disable signatures for service messages between
+clients and Ceph, and for messages between Ceph daemons.
+
+Note that even when signatures are enabled data is not encrypted in flight.
+
+``cephx_require_signatures``
+
+:Description: If this configuration setting is set to ``true``, Ceph requires
+              signatures on all message traffic between the Ceph client and the
+              Ceph Storage Cluster, and between daemons within the Ceph Storage
+              Cluster.
+
+.. note::
+          **ANTIQUATED NOTE:**
+
+          Neither Ceph Argonaut nor Linux kernel versions prior to 3.19
+          support signatures; if one of these clients is in use, ``cephx_require_signatures``
+          can be disabled in order to allow the client to connect.
+
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_cluster_require_signatures``
+
+:Description: If this configuration setting is set to ``true``, Ceph requires
+              signatures on all message traffic between Ceph daemons within the
+              Ceph Storage Cluster.
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_service_require_signatures``
+
+:Description: If this configuration setting is set to ``true``, Ceph requires
+              signatures on all message traffic between Ceph clients and the
+              Ceph Storage Cluster.
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_sign_messages``
+
+:Description: If this configuration setting is set to ``true``, and if the Ceph
+              version supports message signing, then Ceph will sign all
+              messages so that they are more difficult to spoof.
+
+:Type: Boolean
+:Default: ``true``
+
+
+Time to Live
+------------
+
+``auth_service_ticket_ttl``
+
+:Description: When the Ceph Storage Cluster sends a ticket for authentication
+              to a Ceph client, the Ceph Storage Cluster assigns that ticket a
+              Time To Live (TTL).
+
+:Type: Double
+:Default: ``60*60``
+
+
+.. _Monitor Bootstrapping: ../../../install/manual-deployment#monitor-bootstrapping
+.. _Operating a Cluster: ../../operations/operating
+.. _Manual Deployment: ../../../install/manual-deployment
+.. _Ceph configuration: ../ceph-conf
+.. _Architecture - High Availability Authentication: ../../../architecture#high-availability-authentication
+.. _User Management: ../../operations/user-management
diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst
new file mode 100644
index 000000000..3707be1aa
--- /dev/null
+++ b/doc/rados/configuration/bluestore-config-ref.rst
@@ -0,0 +1,552 @@
+==================================
+ BlueStore Configuration Reference 
+==================================
+
+Devices
+=======
+
+BlueStore manages either one, two, or in certain cases three storage devices.
+These *devices* are "devices" in the Linux/Unix sense. This means that they are
+assets listed under ``/dev`` or ``/devices``. Each of these devices may be an
+entire storage drive, or a partition of a storage drive, or a logical volume.
+BlueStore does not create or mount a conventional file system on devices that
+it uses; BlueStore reads and writes to the devices directly in a "raw" fashion.
+
+In the simplest case, BlueStore consumes all of a single storage device. This
+device is known as the *primary device*. The primary device is identified by
+the ``block`` symlink in the data directory.
+
+The data directory is a ``tmpfs`` mount. When this data directory is booted or
+activated by ``ceph-volume``, it is populated with metadata files and links
+that hold information about the OSD: for example, the OSD's identifier, the
+name of the cluster that the OSD belongs to, and the OSD's private keyring.
+
+In more complicated cases, BlueStore is deployed across one or two additional
+devices:
+
+* A *write-ahead log (WAL) device* (identified as ``block.wal`` in the data
+  directory) can be used to separate out BlueStore's internal journal or
+  write-ahead log. Using a WAL device is advantageous only if the WAL device
+  is faster than the primary device (for example, if the WAL device is an SSD
+  and the primary device is an HDD).
+* A *DB device* (identified as ``block.db`` in the data directory) can be used
+  to store BlueStore's internal metadata. BlueStore (or more precisely, the
+  embedded RocksDB) will put as much metadata as it can on the DB device in
+  order to improve performance. If the DB device becomes full, metadata will
+  spill back onto the primary device (where it would have been located in the
+  absence of the DB device). Again, it is advantageous to provision a DB device
+  only if it is faster than the primary device.
+
+If there is only a small amount of fast storage available (for example, less
+than a gigabyte), we recommend using the available space as a WAL device. But
+if more fast storage is available, it makes more sense to provision a DB
+device. Because the BlueStore journal is always placed on the fastest device
+available, using a DB device provides the same benefit that using a WAL device
+would, while *also* allowing additional metadata to be stored off the primary
+device (provided that it fits). DB devices make this possible because whenever
+a DB device is specified but an explicit WAL device is not, the WAL will be
+implicitly colocated with the DB on the faster device.
+
+To provision a single-device (colocated) BlueStore OSD, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph-volume lvm prepare --bluestore --data <device>
+
+To specify a WAL device or DB device, run the following command:
+   
+.. prompt:: bash $
+
+   ceph-volume lvm prepare --bluestore --data <device> --block.wal <wal-device> --block.db <db-device>
+
+.. note:: The option ``--data`` can take as its argument any of the the
+   following devices: logical volumes specified using *vg/lv* notation,
+   existing logical volumes, and GPT partitions.
+
+
+
+Provisioning strategies
+-----------------------
+
+BlueStore differs from Filestore in that there are several ways to deploy a
+BlueStore OSD. However, the overall deployment strategy for BlueStore can be
+clarified by examining just these two common arrangements:
+
+.. _bluestore-single-type-device-config:
+
+**block (data) only**
+^^^^^^^^^^^^^^^^^^^^^
+If all devices are of the same type (for example, they are all HDDs), and if
+there are no fast devices available for the storage of metadata, then it makes
+sense to specify the block device only and to leave ``block.db`` and
+``block.wal`` unseparated. The :ref:`ceph-volume-lvm` command for a single
+``/dev/sda`` device is as follows:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data /dev/sda
+
+If the devices to be used for a BlueStore OSD are pre-created logical volumes,
+then the :ref:`ceph-volume-lvm` call for an logical volume named
+``ceph-vg/block-lv`` is as follows:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data ceph-vg/block-lv
+
+.. _bluestore-mixed-device-config:
+
+**block and block.db**
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you have a mix of fast and slow devices (for example, SSD or HDD), then we
+recommend placing ``block.db`` on the faster device while ``block`` (that is,
+the data) is stored on the slower device (that is, the rotational drive).
+
+You must create these volume groups and these logical volumes manually. as The
+``ceph-volume`` tool is currently unable to do so [create them?] automatically.
+
+The following procedure illustrates the manual creation of volume groups and
+logical volumes.  For this example, we shall assume four rotational drives
+(``sda``, ``sdb``, ``sdc``, and ``sdd``) and one (fast) SSD (``sdx``). First,
+to create the volume groups, run the following commands:
+
+.. prompt:: bash $
+
+   vgcreate ceph-block-0 /dev/sda
+   vgcreate ceph-block-1 /dev/sdb
+   vgcreate ceph-block-2 /dev/sdc
+   vgcreate ceph-block-3 /dev/sdd
+
+Next, to create the logical volumes for ``block``, run the following commands:
+
+.. prompt:: bash $
+
+   lvcreate -l 100%FREE -n block-0 ceph-block-0
+   lvcreate -l 100%FREE -n block-1 ceph-block-1
+   lvcreate -l 100%FREE -n block-2 ceph-block-2
+   lvcreate -l 100%FREE -n block-3 ceph-block-3
+
+Because there are four HDDs, there will be four OSDs. Supposing that there is a
+200GB SSD in ``/dev/sdx``, we can create four 50GB logical volumes by running
+the following commands:
+
+.. prompt:: bash $
+
+   vgcreate ceph-db-0 /dev/sdx
+   lvcreate -L 50GB -n db-0 ceph-db-0
+   lvcreate -L 50GB -n db-1 ceph-db-0
+   lvcreate -L 50GB -n db-2 ceph-db-0
+   lvcreate -L 50GB -n db-3 ceph-db-0
+
+Finally, to create the four OSDs, run the following commands:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data ceph-block-0/block-0 --block.db ceph-db-0/db-0
+   ceph-volume lvm create --bluestore --data ceph-block-1/block-1 --block.db ceph-db-0/db-1
+   ceph-volume lvm create --bluestore --data ceph-block-2/block-2 --block.db ceph-db-0/db-2
+   ceph-volume lvm create --bluestore --data ceph-block-3/block-3 --block.db ceph-db-0/db-3
+
+After this procedure is finished, there should be four OSDs, ``block`` should
+be on the four HDDs, and each HDD should have a 50GB logical volume
+(specifically, a DB device) on the shared SSD.
+
+Sizing
+======
+When using a :ref:`mixed spinning-and-solid-drive setup
+<bluestore-mixed-device-config>`, it is important to make a large enough
+``block.db`` logical volume for BlueStore. The logical volumes associated with
+``block.db`` should have logical volumes that are *as large as possible*.
+
+It is generally recommended that the size of ``block.db`` be somewhere between
+1% and 4% of the size of ``block``. For RGW workloads, it is recommended that
+the ``block.db`` be at least 4% of the ``block`` size, because RGW makes heavy
+use of ``block.db`` to store metadata (in particular, omap keys). For example,
+if the ``block`` size is 1TB, then ``block.db`` should have a size of at least
+40GB. For RBD workloads, however, ``block.db`` usually needs no more than 1% to
+2% of the ``block`` size.
+
+In older releases, internal level sizes are such that the DB can fully utilize
+only those specific partition / logical volume sizes that correspond to sums of
+L0, L0+L1, L1+L2, and so on--that is, given default settings, sizes of roughly
+3GB, 30GB, 300GB, and so on. Most deployments do not substantially benefit from
+sizing that accommodates L3 and higher, though DB compaction can be facilitated
+by doubling these figures to 6GB, 60GB, and 600GB.
+
+Improvements in Nautilus 14.2.12, Octopus 15.2.6, and subsequent releases allow
+for better utilization of arbitrarily-sized DB devices. Moreover, the Pacific
+release brings experimental dynamic-level support. Because of these advances,
+users of older releases might want to plan ahead by provisioning larger DB
+devices today so that the benefits of scale can be realized when upgrades are
+made in the future.
+
+When *not* using a mix of fast and slow devices, there is no requirement to
+create separate logical volumes for ``block.db`` or ``block.wal``. BlueStore
+will automatically colocate these devices within the space of ``block``.
+
+Automatic Cache Sizing
+======================
+
+BlueStore can be configured to automatically resize its caches, provided that
+certain conditions are met: TCMalloc must be configured as the memory allocator
+and the ``bluestore_cache_autotune`` configuration option must be enabled (note
+that it is currently enabled by default). When automatic cache sizing is in
+effect, BlueStore attempts to keep OSD heap-memory usage under a certain target
+size (as determined by ``osd_memory_target``). This approach makes use of a
+best-effort algorithm and caches do not shrink smaller than the size defined by
+the value of ``osd_memory_cache_min``. Cache ratios are selected in accordance
+with a hierarchy of priorities.  But if priority information is not available,
+the values specified in the ``bluestore_cache_meta_ratio`` and
+``bluestore_cache_kv_ratio`` options are used as fallback cache ratios.
+
+.. confval:: bluestore_cache_autotune
+.. confval:: osd_memory_target
+.. confval:: bluestore_cache_autotune_interval
+.. confval:: osd_memory_base
+.. confval:: osd_memory_expected_fragmentation
+.. confval:: osd_memory_cache_min
+.. confval:: osd_memory_cache_resize_interval
+
+
+Manual Cache Sizing
+===================
+
+The amount of memory consumed by each OSD to be used for its BlueStore cache is
+determined by the ``bluestore_cache_size`` configuration option. If that option
+has not been specified (that is, if it remains at 0), then Ceph uses a
+different configuration option to determine the default memory budget:
+``bluestore_cache_size_hdd`` if the primary device is an HDD, or
+``bluestore_cache_size_ssd`` if the primary device is an SSD.
+
+BlueStore and the rest of the Ceph OSD daemon make every effort to work within
+this memory budget. Note that in addition to the configured cache size, there
+is also memory consumed by the OSD itself. There is additional utilization due
+to memory fragmentation and other allocator overhead. 
+
+The configured cache-memory budget can be used to store the following types of
+things:
+
+* Key/Value metadata (that is, RocksDB's internal cache)
+* BlueStore metadata
+* BlueStore data (that is, recently read or recently written object data)
+
+Cache memory usage is governed by the configuration options
+``bluestore_cache_meta_ratio`` and ``bluestore_cache_kv_ratio``.  The fraction
+of the cache that is reserved for data is governed by both the effective
+BlueStore cache size (which depends on the relevant
+``bluestore_cache_size[_ssd|_hdd]`` option and the device class of the primary
+device) and the "meta" and "kv" ratios.  This data fraction can be calculated
+with the following formula: ``<effective_cache_size> * (1 -
+bluestore_cache_meta_ratio - bluestore_cache_kv_ratio)``.
+
+.. confval:: bluestore_cache_size
+.. confval:: bluestore_cache_size_hdd
+.. confval:: bluestore_cache_size_ssd
+.. confval:: bluestore_cache_meta_ratio
+.. confval:: bluestore_cache_kv_ratio
+
+Checksums
+=========
+
+BlueStore checksums all metadata and all data written to disk. Metadata
+checksumming is handled by RocksDB and uses the `crc32c` algorithm. By
+contrast, data checksumming is handled by BlueStore and can use either
+`crc32c`, `xxhash32`, or `xxhash64`. Nonetheless, `crc32c` is the default
+checksum algorithm and it is suitable for most purposes.
+
+Full data checksumming increases the amount of metadata that BlueStore must
+store and manage. Whenever possible (for example, when clients hint that data
+is written and read sequentially), BlueStore will checksum larger blocks. In
+many cases, however, it must store a checksum value (usually 4 bytes) for every
+4 KB block of data.
+
+It is possible to obtain a smaller checksum value by truncating the checksum to
+one or two bytes and reducing the metadata overhead.  A drawback of this
+approach is that it increases the probability of a random error going
+undetected: about one in four billion given a 32-bit (4 byte) checksum, 1 in
+65,536 given a 16-bit (2 byte) checksum, and 1 in 256 given an 8-bit (1 byte)
+checksum. To use the smaller checksum values, select `crc32c_16` or `crc32c_8`
+as the checksum algorithm.
+
+The *checksum algorithm* can be specified either via a per-pool ``csum_type``
+configuration option or via the global configuration option. For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> csum_type <algorithm>
+
+.. confval:: bluestore_csum_type
+
+Inline Compression
+==================
+
+BlueStore supports inline compression using `snappy`, `zlib`, `lz4`, or `zstd`. 
+
+Whether data in BlueStore is compressed is determined by two factors: (1) the
+*compression mode* and (2) any client hints associated with a write operation.
+The compression modes are as follows:
+
+* **none**: Never compress data.
+* **passive**: Do not compress data unless the write operation has a
+  *compressible* hint set.
+* **aggressive**: Do compress data unless the write operation has an
+  *incompressible* hint set.
+* **force**: Try to compress data no matter what.
+
+For more information about the *compressible* and *incompressible* I/O hints,
+see :c:func:`rados_set_alloc_hint`.
+
+Note that data in Bluestore will be compressed only if the data chunk will be
+sufficiently reduced in size (as determined by the ``bluestore compression
+required ratio`` setting). No matter which compression modes have been used, if
+the data chunk is too big, then it will be discarded and the original
+(uncompressed) data will be stored instead. For example, if ``bluestore
+compression required ratio`` is set to ``.7``, then data compression will take
+place only if the size of the compressed data is no more than 70% of the size
+of the original data.
+
+The *compression mode*, *compression algorithm*, *compression required ratio*,
+*min blob size*, and *max blob size* settings can be specified either via a
+per-pool property or via a global config option. To specify pool properties,
+run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> compression_algorithm <algorithm>
+   ceph osd pool set <pool-name> compression_mode <mode>
+   ceph osd pool set <pool-name> compression_required_ratio <ratio>
+   ceph osd pool set <pool-name> compression_min_blob_size <size>
+   ceph osd pool set <pool-name> compression_max_blob_size <size>
+
+.. confval:: bluestore_compression_algorithm
+.. confval:: bluestore_compression_mode
+.. confval:: bluestore_compression_required_ratio
+.. confval:: bluestore_compression_min_blob_size
+.. confval:: bluestore_compression_min_blob_size_hdd
+.. confval:: bluestore_compression_min_blob_size_ssd
+.. confval:: bluestore_compression_max_blob_size
+.. confval:: bluestore_compression_max_blob_size_hdd
+.. confval:: bluestore_compression_max_blob_size_ssd
+
+.. _bluestore-rocksdb-sharding:
+
+RocksDB Sharding
+================
+
+BlueStore maintains several types of internal key-value data, all of which are
+stored in RocksDB. Each data type in BlueStore is assigned a unique prefix.
+Prior to the Pacific release, all key-value data was stored in a single RocksDB
+column family: 'default'. In Pacific and later releases, however, BlueStore can
+divide key-value data into several RocksDB column families. BlueStore achieves
+better caching and more precise compaction when keys are similar: specifically,
+when keys have similar access frequency, similar modification frequency, and a
+similar lifetime.  Under such conditions, performance is improved and less disk
+space is required during compaction (because each column family is smaller and
+is able to compact independently of the others).
+
+OSDs deployed in Pacific or later releases use RocksDB sharding by default.
+However, if Ceph has been upgraded to Pacific or a later version from a
+previous version, sharding is disabled on any OSDs that were created before
+Pacific.
+
+To enable sharding and apply the Pacific defaults to a specific OSD, stop the
+OSD and run the following command:
+
+    .. prompt:: bash #
+
+       ceph-bluestore-tool \
+        --path <data path> \
+        --sharding="m(3) p(3,0-12) o(3,0-13)=block_cache={type=binned_lru} l p" \
+        reshard
+
+.. confval:: bluestore_rocksdb_cf
+.. confval:: bluestore_rocksdb_cfs
+
+Throttling
+==========
+
+.. confval:: bluestore_throttle_bytes
+.. confval:: bluestore_throttle_deferred_bytes
+.. confval:: bluestore_throttle_cost_per_io
+.. confval:: bluestore_throttle_cost_per_io_hdd
+.. confval:: bluestore_throttle_cost_per_io_ssd
+
+SPDK Usage
+==========
+
+To use the SPDK driver for NVMe devices, you must first prepare your system.
+See `SPDK document`__.
+
+.. __: http://www.spdk.io/doc/getting_started.html#getting_started_examples
+
+SPDK offers a script that will configure the device automatically. Run this
+script with root permissions:
+
+.. prompt:: bash $
+
+   sudo src/spdk/scripts/setup.sh
+
+You will need to specify the subject NVMe device's device selector with the
+"spdk:" prefix for ``bluestore_block_path``.
+
+In the following example, you first find the device selector of an Intel NVMe
+SSD by running the following command:
+
+.. prompt:: bash $
+
+   lspci -mm -n -d -d 8086:0953
+
+The form of the device selector is either ``DDDD:BB:DD.FF`` or
+``DDDD.BB.DD.FF``.
+
+Next, supposing that ``0000:01:00.0`` is the device selector found in the
+output of the ``lspci`` command, you can specify the device selector by running
+the following command::
+
+  bluestore_block_path = "spdk:trtype:pcie traddr:0000:01:00.0"
+
+You may also specify a remote NVMeoF target over the TCP transport, as in the
+following example::
+
+  bluestore_block_path = "spdk:trtype:tcp traddr:10.67.110.197 trsvcid:4420 subnqn:nqn.2019-02.io.spdk:cnode1"
+
+To run multiple SPDK instances per node, you must make sure each instance uses
+its own DPDK memory by specifying for each instance the amount of DPDK memory
+(in MB) that the instance will use.
+
+In most cases, a single device can be used for data, DB, and WAL. We describe
+this strategy as *colocating* these components. Be sure to enter the below
+settings to ensure that all I/Os are issued through SPDK::
+
+  bluestore_block_db_path = ""
+  bluestore_block_db_size = 0
+  bluestore_block_wal_path = ""
+  bluestore_block_wal_size = 0
+
+If these settings are not entered, then the current implementation will
+populate the SPDK map files with kernel file system symbols and will use the
+kernel driver to issue DB/WAL I/Os.
+
+Minimum Allocation Size
+=======================
+
+There is a configured minimum amount of storage that BlueStore allocates on an
+underlying storage device. In practice, this is the least amount of capacity
+that even a tiny RADOS object can consume on each OSD's primary device. The
+configuration option in question--:confval:`bluestore_min_alloc_size`--derives
+its value from the value of either :confval:`bluestore_min_alloc_size_hdd` or
+:confval:`bluestore_min_alloc_size_ssd`, depending on the OSD's ``rotational``
+attribute. Thus if an OSD is created on an HDD, BlueStore is initialized with
+the current value of :confval:`bluestore_min_alloc_size_hdd`; but with SSD OSDs
+(including NVMe devices), Bluestore is initialized with the current value of
+:confval:`bluestore_min_alloc_size_ssd`.
+
+In Mimic and earlier releases, the default values were 64KB for rotational
+media (HDD) and 16KB for non-rotational media (SSD). The Octopus release
+changed the the default value for non-rotational media (SSD) to 4KB, and the
+Pacific release changed the default value for rotational media (HDD) to 4KB.
+
+These changes were driven by space amplification that was experienced by Ceph
+RADOS GateWay (RGW) deployments that hosted large numbers of small files
+(S3/Swift objects).
+
+For example, when an RGW client stores a 1 KB S3 object, that object is written
+to a single RADOS object. In accordance with the default
+:confval:`min_alloc_size` value, 4 KB of underlying drive space is allocated.
+This means that roughly 3 KB (that is, 4 KB minus 1 KB) is allocated but never
+used: this corresponds to 300% overhead or 25% efficiency. Similarly, a 5 KB
+user object will be stored as two RADOS objects, a 4 KB RADOS object and a 1 KB
+RADOS object, with the result that 4KB of device capacity is stranded. In this
+case, however, the overhead percentage is much smaller. Think of this in terms
+of the remainder from a modulus operation. The overhead *percentage* thus
+decreases rapidly as object size increases.
+
+There is an additional subtlety that is easily missed: the amplification
+phenomenon just described takes place for *each* replica. For example, when
+using the default of three copies of data (3R), a 1 KB S3 object actually
+strands roughly 9 KB of storage device capacity. If erasure coding (EC) is used
+instead of replication, the amplification might be even higher: for a ``k=4,
+m=2`` pool, our 1 KB S3 object allocates 24 KB (that is, 4 KB multiplied by 6)
+of device capacity.
+
+When an RGW bucket pool contains many relatively large user objects, the effect
+of this phenomenon is often negligible. However, with deployments that can
+expect a significant fraction of relatively small user objects, the effect
+should be taken into consideration.
+
+The 4KB default value aligns well with conventional HDD and SSD devices.
+However, certain novel coarse-IU (Indirection Unit) QLC SSDs perform and wear
+best when :confval:`bluestore_min_alloc_size_ssd` is specified at OSD creation
+to match the device's IU: this might be 8KB, 16KB, or even 64KB.  These novel
+storage drives can achieve read performance that is competitive with that of
+conventional TLC SSDs and write performance that is faster than that of HDDs,
+with higher density and lower cost than TLC SSDs.
+
+Note that when creating OSDs on these novel devices, one must be careful to
+apply the non-default value only to appropriate devices, and not to
+conventional HDD and SSD devices. Error can be avoided through careful ordering
+of OSD creation, with custom OSD device classes, and especially by the use of
+central configuration *masks*.
+
+In Quincy and later releases, you can use the
+:confval:`bluestore_use_optimal_io_size_for_min_alloc_size` option to allow
+automatic discovery of the correct value as each OSD is created. Note that the
+use of ``bcache``, ``OpenCAS``, ``dmcrypt``, ``ATA over Ethernet``, `iSCSI`, or
+other device-layering and abstraction technologies might confound the
+determination of correct values. Moreover, OSDs deployed on top of VMware
+storage have sometimes been found to report a ``rotational`` attribute that
+does not match the underlying hardware.
+
+We suggest inspecting such OSDs at startup via logs and admin sockets in order
+to ensure that their behavior is correct. Be aware that this kind of inspection
+might not work as expected with older kernels.  To check for this issue,
+examine the presence and value of ``/sys/block/<drive>/queue/optimal_io_size``.
+
+.. note:: When running Reef or a later Ceph release, the ``min_alloc_size``
+   baked into each OSD is conveniently reported by ``ceph osd metadata``.
+
+To inspect a specific OSD, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd metadata osd.1701 | egrep rotational\|alloc
+
+This space amplification might manifest as an unusually high ratio of raw to
+stored data as reported by ``ceph df``. There might also be ``%USE`` / ``VAR``
+values reported by ``ceph osd df`` that are unusually high in comparison to
+other, ostensibly identical, OSDs. Finally, there might be unexpected balancer
+behavior in pools that use OSDs that have mismatched ``min_alloc_size`` values.
+
+This BlueStore attribute takes effect *only* at OSD creation; if the attribute
+is changed later, a specific OSD's behavior will not change unless and until
+the OSD is destroyed and redeployed with the appropriate option value(s).
+Upgrading to a later Ceph release will *not* change the value used by OSDs that
+were deployed under older releases or with other settings.
+
+.. confval:: bluestore_min_alloc_size
+.. confval:: bluestore_min_alloc_size_hdd
+.. confval:: bluestore_min_alloc_size_ssd
+.. confval:: bluestore_use_optimal_io_size_for_min_alloc_size
+
+DSA (Data Streaming Accelerator) Usage
+======================================
+
+If you want to use the DML library to drive the DSA device for offloading
+read/write operations on persistent memory (PMEM) in BlueStore, you need to
+install `DML`_ and the `idxd-config`_ library. This will work only on machines
+that have a SPR (Sapphire Rapids) CPU.
+
+.. _dml: https://github.com/intel/dml
+.. _idxd-config: https://github.com/intel/idxd-config
+
+After installing the DML software, configure the shared work queues (WQs) with
+reference to the following WQ configuration example:
+
+.. prompt:: bash $
+
+   accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="myapp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
+   accel-config config-engine dsa0/engine0.1 --group-id=1
+   accel-config enable-device dsa0
+   accel-config enable-wq dsa0/wq0.1
diff --git a/doc/rados/configuration/ceph-conf.rst b/doc/rados/configuration/ceph-conf.rst
new file mode 100644
index 000000000..d8d5c9d03
--- /dev/null
+++ b/doc/rados/configuration/ceph-conf.rst
@@ -0,0 +1,715 @@
+.. _configuring-ceph:
+
+==================
+ Configuring Ceph
+==================
+
+When Ceph services start, the initialization process activates a set of
+daemons that run in the background. A :term:`Ceph Storage Cluster` runs at
+least three types of daemons:
+
+- :term:`Ceph Monitor` (``ceph-mon``)
+- :term:`Ceph Manager` (``ceph-mgr``)
+- :term:`Ceph OSD Daemon` (``ceph-osd``)
+
+Any Ceph Storage Cluster that supports the :term:`Ceph File System` also runs
+at least one :term:`Ceph Metadata Server` (``ceph-mds``). Any Cluster that
+supports :term:`Ceph Object Storage` runs Ceph RADOS Gateway daemons
+(``radosgw``).
+
+Each daemon has a number of configuration options, and each of those options
+has a default value. Adjust the behavior of the system by changing these
+configuration options. Make sure to understand the consequences before
+overriding the default values, as it is possible to significantly degrade the
+performance and stability of your cluster. Remember that default values
+sometimes change between releases. For this reason, it is best to review the
+version of this documentation that applies to your Ceph release.
+
+Option names
+============
+
+Each of the Ceph configuration options has a unique name that consists of words
+formed with lowercase characters and connected with underscore characters
+(``_``).
+
+When option names are specified on the command line, underscore (``_``) and
+dash (``-``) characters can be used interchangeably (for example,
+``--mon-host`` is equivalent to ``--mon_host``).
+
+When option names appear in configuration files, spaces can also be used in
+place of underscores or dashes. However, for the sake of clarity and
+convenience, we suggest that you consistently use underscores, as we do
+throughout this documentation.
+
+Config sources
+==============
+
+Each Ceph daemon, process, and library pulls its configuration from one or more
+of the several sources listed below. Sources that occur later in the list
+override those that occur earlier in the list (when both are present).
+
+- the compiled-in default value
+- the monitor cluster's centralized configuration database
+- a configuration file stored on the local host
+- environment variables
+- command-line arguments
+- runtime overrides that are set by an administrator
+
+One of the first things a Ceph process does on startup is parse the
+configuration options provided via the command line, via the environment, and
+via the local configuration file. Next, the process contacts the monitor
+cluster to retrieve centrally-stored configuration for the entire cluster.
+After a complete view of the configuration is available, the startup of the
+daemon or process will commence.
+
+.. _bootstrap-options:
+
+Bootstrap options
+-----------------
+
+Bootstrap options are configuration options that affect the process's ability
+to contact the monitors, to authenticate, and to retrieve the cluster-stored
+configuration.  For this reason, these options might need to be stored locally
+on the node, and set by means of a local configuration file. These options
+include the following:
+
+.. confval:: mon_host
+.. confval:: mon_host_override
+
+- :confval:`mon_dns_srv_name`
+- :confval:`mon_data`, :confval:`osd_data`, :confval:`mds_data`,
+  :confval:`mgr_data`, and similar options that define which local directory
+  the daemon stores its data in.
+- :confval:`keyring`, :confval:`keyfile`, and/or :confval:`key`, which can be
+  used to specify the authentication credential to use to authenticate with the
+  monitor. Note that in most cases the default keyring location is in the data
+  directory specified above.
+
+In most cases, there is no reason to modify the default values of these
+options. However, there is one exception to this: the :confval:`mon_host`
+option that identifies the addresses of the cluster's monitors. But when
+:ref:`DNS is used to identify monitors<mon-dns-lookup>`, a local Ceph
+configuration file can be avoided entirely.
+
+
+Skipping monitor config
+-----------------------
+
+The option ``--no-mon-config`` can be passed in any command in order to skip
+the step that retrieves configuration information from the cluster's monitors.
+Skipping this retrieval step can be useful in cases where configuration is
+managed entirely via configuration files, or when maintenance activity needs to
+be done but the monitor cluster is down.
+
+.. _ceph-conf-file:
+
+Configuration sections
+======================
+
+Each of the configuration options associated with a single process or daemon
+has a single value. However, the values for a configuration option can vary
+across daemon types, and can vary even across different daemons of the same
+type. Ceph options that are stored in the monitor configuration database or in
+local configuration files are grouped into sections |---| so-called "configuration
+sections" |---| to indicate which daemons or clients they apply to.
+
+
+These sections include the following:
+
+.. confsec:: global
+
+   Settings under ``global`` affect all daemons and clients
+   in a Ceph Storage Cluster.
+
+   :example: ``log_file = /var/log/ceph/$cluster-$type.$id.log``
+
+.. confsec:: mon
+
+   Settings under ``mon`` affect all ``ceph-mon`` daemons in
+   the Ceph Storage Cluster, and override the same setting in
+   ``global``.
+
+   :example: ``mon_cluster_log_to_syslog = true``
+
+.. confsec:: mgr
+
+   Settings in the ``mgr`` section affect all ``ceph-mgr`` daemons in
+   the Ceph Storage Cluster, and override the same setting in
+   ``global``.
+
+   :example: ``mgr_stats_period = 10``
+
+.. confsec:: osd
+
+   Settings under ``osd`` affect all ``ceph-osd`` daemons in
+   the Ceph Storage Cluster, and override the same setting in
+   ``global``.
+
+   :example: ``osd_op_queue = wpq``
+
+.. confsec:: mds
+
+   Settings in the ``mds`` section affect all ``ceph-mds`` daemons in
+   the Ceph Storage Cluster, and override the same setting in
+   ``global``.
+
+   :example: ``mds_cache_memory_limit = 10G``
+
+.. confsec:: client
+
+   Settings under ``client`` affect all Ceph clients
+   (for example, mounted Ceph File Systems, mounted Ceph Block Devices)
+   as well as RADOS Gateway (RGW) daemons.
+
+   :example: ``objecter_inflight_ops = 512``
+
+
+Configuration sections can also specify an individual daemon or client name. For example,
+``mon.foo``, ``osd.123``, and ``client.smith`` are all valid section names.
+
+
+Any given daemon will draw its settings from the global section, the daemon- or
+client-type section, and the section sharing its name. Settings in the
+most-specific section take precedence so precedence: for example, if the same
+option is specified in both :confsec:`global`, :confsec:`mon`, and ``mon.foo``
+on the same source (i.e. that is, in the same configuration file), the
+``mon.foo`` setting will be used.
+
+If multiple values of the same configuration option are specified in the same
+section, the last value specified takes precedence.
+
+Note that values from the local configuration file always take precedence over
+values from the monitor configuration database, regardless of the section in
+which they appear.
+
+.. _ceph-metavariables:
+
+Metavariables
+=============
+
+Metavariables dramatically simplify Ceph storage cluster configuration. When a
+metavariable is set in a configuration value, Ceph expands the metavariable at
+the time the configuration value is used. In this way, Ceph metavariables
+behave similarly to the way that variable expansion works in the Bash shell.
+
+Ceph supports the following metavariables:
+
+.. describe:: $cluster
+
+   Expands to the Ceph Storage Cluster name. Useful when running
+   multiple Ceph Storage Clusters on the same hardware.
+
+   :example: ``/etc/ceph/$cluster.keyring``
+   :default: ``ceph``
+
+.. describe:: $type
+
+   Expands to a daemon or process type (for example, ``mds``, ``osd``, or ``mon``)
+
+   :example: ``/var/lib/ceph/$type``
+
+.. describe:: $id
+
+   Expands to the daemon or client identifier. For
+   ``osd.0``, this would be ``0``; for ``mds.a``, it would
+   be ``a``.
+
+   :example: ``/var/lib/ceph/$type/$cluster-$id``
+
+.. describe:: $host
+
+   Expands to the host name where the process is running.
+
+.. describe:: $name
+
+   Expands to ``$type.$id``.
+
+   :example: ``/var/run/ceph/$cluster-$name.asok``
+
+.. describe:: $pid
+
+   Expands to daemon pid.
+
+   :example: ``/var/run/ceph/$cluster-$name-$pid.asok``
+
+
+Ceph configuration file
+=======================
+
+On startup, Ceph processes search for a configuration file in the
+following locations:
+
+#. ``$CEPH_CONF`` (that is, the path following the ``$CEPH_CONF``
+   environment variable)
+#. ``-c path/path``  (that is, the ``-c`` command line argument)
+#. ``/etc/ceph/$cluster.conf``
+#. ``~/.ceph/$cluster.conf``
+#. ``./$cluster.conf`` (that is, in the current working directory)
+#. On FreeBSD systems only, ``/usr/local/etc/ceph/$cluster.conf``
+
+Here ``$cluster`` is the cluster's name (default: ``ceph``).
+
+The Ceph configuration file uses an ``ini`` style syntax. You can add "comment
+text" after a pound sign (#) or a semi-colon semicolon (;). For example:
+
+.. code-block:: ini
+
+    # <--A number (#) sign number sign (#) precedes a comment.
+    ; A comment may be anything.
+    # Comments always follow a semi-colon semicolon (;) or a pound sign (#) on each line.
+    # The end of the line terminates a comment.
+    # We recommend that you provide comments in your configuration file(s).
+
+
+.. _ceph-conf-settings:
+
+Config file section names
+-------------------------
+
+The configuration file is divided into sections. Each section must begin with a
+valid configuration section name (see `Configuration sections`_, above) that is
+surrounded by square brackets. For example:
+
+.. code-block:: ini
+
+    [global]
+    debug_ms = 0
+
+    [osd]
+    debug_ms = 1
+
+    [osd.1]
+    debug_ms = 10
+
+    [osd.2]
+    debug_ms = 10
+
+Config file option values
+-------------------------
+
+The value of a configuration option is a string. If the string is too long to
+fit on a single line, you can put a backslash (``\``) at the end of the line
+and the backslash will act as a line continuation marker. In such a case, the
+value of the option will be the string after ``=`` in the current line,
+combined with the string in the next line. Here is an example::
+
+  [global]
+  foo = long long ago\
+  long ago
+
+In this example, the value of the "``foo``" option is "``long long ago long
+ago``".
+
+An option value typically ends with either a newline or a comment. For
+example:
+
+.. code-block:: ini
+
+    [global]
+    obscure_one = difficult to explain # I will try harder in next release
+    simpler_one = nothing to explain
+
+In this example, the value of the "``obscure one``" option is "``difficult to
+explain``" and the value of the "``simpler one`` options is "``nothing to
+explain``".
+
+When an option value contains spaces, it can be enclosed within single quotes
+or double quotes in order to make its scope clear and in order to make sure
+that the first space in the value is not interpreted as the end of the value.
+For example:
+
+.. code-block:: ini
+
+    [global]
+    line = "to be, or not to be"
+
+In option values, there are four characters that are treated as escape
+characters: ``=``, ``#``, ``;`` and ``[``. They are permitted to occur in an
+option value only if they are immediately preceded by the backslash character
+(``\``). For example:
+
+.. code-block:: ini
+
+    [global]
+    secret = "i love \# and \["
+
+Each configuration option falls under one of the following types:
+
+.. describe:: int
+
+   64-bit signed integer. Some SI suffixes are supported, such as "K", "M",
+   "G", "T", "P", and "E" (meaning, respectively, 10\ :sup:`3`, 10\ :sup:`6`,
+   10\ :sup:`9`, etc.). "B" is the only supported unit string. Thus "1K", "1M",
+   "128B" and "-1" are all valid option values. When a negative value is
+   assigned to a threshold option, this can indicate that the option is
+   "unlimited" -- that is, that there is no threshold or limit in effect.
+
+   :example: ``42``, ``-1``
+
+.. describe:: uint
+
+   This differs from ``integer`` only in that negative values are not
+   permitted.
+
+   :example: ``256``, ``0``
+
+.. describe:: str
+
+   A string encoded in UTF-8. Certain characters are not permitted. Reference
+   the above notes for the details.
+
+   :example: ``"hello world"``, ``"i love \#"``, ``yet-another-name``
+
+.. describe:: boolean
+
+   Typically either of the two values ``true`` or ``false``. However, any
+   integer is permitted: "0" implies ``false``, and any non-zero value implies
+   ``true``.
+
+   :example: ``true``, ``false``, ``1``, ``0``
+
+.. describe:: addr
+
+   A single address, optionally prefixed with ``v1``, ``v2`` or ``any`` for the
+   messenger protocol. If no prefix is specified, the ``v2`` protocol is used.
+   For more details, see :ref:`address_formats`.
+
+   :example: ``v1:1.2.3.4:567``, ``v2:1.2.3.4:567``, ``1.2.3.4:567``, ``2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::567``, ``[::1]:6789``
+
+.. describe:: addrvec
+
+   A set of addresses separated by ",". The addresses can be optionally quoted
+   with ``[`` and ``]``.
+
+   :example: ``[v1:1.2.3.4:567,v2:1.2.3.4:568]``, ``v1:1.2.3.4:567,v1:1.2.3.14:567``  ``[2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::567], [2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::568]``
+
+.. describe:: uuid
+
+   The string format of a uuid defined by `RFC4122
+   <https://www.ietf.org/rfc/rfc4122.txt>`_. Certain variants are also
+   supported: for more details, see `Boost document
+   <https://www.boost.org/doc/libs/1_74_0/libs/uuid/doc/uuid.html#String%20Generator>`_.
+
+   :example: ``f81d4fae-7dec-11d0-a765-00a0c91e6bf6``
+
+.. describe:: size
+
+   64-bit unsigned integer. Both SI prefixes and IEC prefixes are supported.
+   "B" is the only supported unit string. Negative values are not permitted.
+
+   :example: ``1Ki``, ``1K``, ``1KiB`` and ``1B``.
+
+.. describe:: secs
+
+   Denotes a duration of time. The default unit of time is the second.
+   The following units of time are supported:
+
+              * second: ``s``, ``sec``, ``second``, ``seconds``
+              * minute: ``m``, ``min``, ``minute``, ``minutes``
+              * hour: ``hs``, ``hr``, ``hour``, ``hours``
+              * day: ``d``, ``day``, ``days``
+              * week: ``w``, ``wk``, ``week``, ``weeks``
+              * month: ``mo``, ``month``, ``months``
+              * year: ``y``, ``yr``, ``year``, ``years``
+
+   :example: ``1 m``, ``1m`` and ``1 week``
+
+.. _ceph-conf-database:
+
+Monitor configuration database
+==============================
+
+The monitor cluster manages a database of configuration options that can be
+consumed by the entire cluster. This allows for streamlined central
+configuration management of the entire system. For ease of administration and
+transparency, the vast majority of configuration options can and should be
+stored in this database.
+
+Some settings might need to be stored in local configuration files because they
+affect the ability of the process to connect to the monitors, to authenticate,
+and to fetch configuration information. In most cases this applies only to the
+``mon_host`` option. This issue can be avoided by using :ref:`DNS SRV
+records<mon-dns-lookup>`.
+
+Sections and masks
+------------------
+
+Configuration options stored by the monitor can be stored in a global section,
+in a daemon-type section, or in a specific daemon section. In this, they are
+no different from the options in a configuration file.
+
+In addition, options may have a *mask* associated with them to further restrict
+which daemons or clients the option applies to. Masks take two forms:
+
+#. ``type:location`` where ``type`` is a CRUSH property like ``rack`` or
+   ``host``, and ``location`` is a value for that property. For example,
+   ``host:foo`` would limit the option only to daemons or clients
+   running on a particular host.
+#. ``class:device-class`` where ``device-class`` is the name of a CRUSH
+   device class (for example, ``hdd`` or ``ssd``). For example,
+   ``class:ssd`` would limit the option only to OSDs backed by SSDs.
+   (This mask has no effect on non-OSD daemons or clients.)
+
+In commands that specify a configuration option, the argument of the option (in
+the following examples, this is the "who" string) may be a section name, a
+mask, or a combination of both separated by a slash character (``/``). For
+example, ``osd/rack:foo`` would refer to all OSD daemons in the ``foo`` rack.
+
+When configuration options are shown, the section name and mask are presented
+in separate fields or columns to make them more readable.
+
+Commands
+--------
+
+The following CLI commands are used to configure the cluster:
+
+* ``ceph config dump`` dumps the entire monitor configuration
+  database for the cluster.
+
+* ``ceph config get <who>`` dumps the configuration options stored in
+  the monitor configuration database for a specific daemon or client
+  (for example, ``mds.a``).
+
+* ``ceph config get <who> <option>`` shows either a configuration value
+  stored in the monitor configuration database for a specific daemon or client
+  (for example, ``mds.a``), or, if that value is not present in the monitor
+  configuration database, the compiled-in default value.
+
+* ``ceph config set <who> <option> <value>`` specifies a configuration
+  option in the monitor configuration database.
+
+* ``ceph config show <who>`` shows the configuration for a running daemon.
+  These settings might differ from those stored by the monitors if there are
+  also local configuration files in use or if options have been overridden on
+  the command line or at run time. The source of the values of the options is
+  displayed in the output.
+
+* ``ceph config assimilate-conf -i <input file> -o <output file>`` ingests a
+  configuration file from *input file* and moves any valid options into the
+  monitor configuration database. Any settings that are unrecognized, are
+  invalid, or cannot be controlled by the monitor will be returned in an
+  abbreviated configuration file stored in *output file*. This command is
+  useful for transitioning from legacy configuration files to centralized
+  monitor-based configuration.
+
+Note that ``ceph config set <who> <option> <value>`` and ``ceph config get
+<who> <option>`` will not necessarily return the same values. The latter
+command will show compiled-in default values. In order to determine whether a
+configuration option is present in the monitor configuration database, run
+``ceph config dump``.
+
+Help
+====
+
+To get help for a particular option, run the following command:
+
+.. prompt:: bash $
+
+   ceph config help <option>
+
+For example:
+
+.. prompt:: bash $
+
+   ceph config help log_file
+
+::
+
+   log_file - path to log file
+    (std::string, basic)
+    Default (non-daemon):
+    Default (daemon): /var/log/ceph/$cluster-$name.log
+    Can update at runtime: false
+    See also: [log_to_stderr,err_to_stderr,log_to_syslog,err_to_syslog]
+
+or:
+
+.. prompt:: bash $
+
+   ceph config help log_file -f json-pretty
+
+::
+
+  {
+      "name": "log_file",
+      "type": "std::string",
+      "level": "basic",
+      "desc": "path to log file",
+      "long_desc": "",
+      "default": "",
+      "daemon_default": "/var/log/ceph/$cluster-$name.log",
+      "tags": [],
+      "services": [],
+      "see_also": [
+          "log_to_stderr",
+          "err_to_stderr",
+          "log_to_syslog",
+          "err_to_syslog"
+      ],
+      "enum_values": [],
+      "min": "",
+      "max": "",
+      "can_update_at_runtime": false
+  }
+
+The ``level`` property can be ``basic``, ``advanced``, or ``dev``.  The `dev`
+options are intended for use by developers, generally for testing purposes, and
+are not recommended for use by operators.
+
+.. note:: This command uses the configuration schema that is compiled into the
+   running monitors. If you have a mixed-version cluster (as might exist, for
+   example, during an upgrade), you might want to query the option schema from
+   a specific running daemon by running a command of the following form:
+
+.. prompt:: bash $
+
+   ceph daemon <name> config help [option]
+
+Runtime Changes
+===============
+
+In most cases, Ceph permits changes to the configuration of a daemon at
+run time. This can be used for increasing or decreasing the amount of logging
+output, for enabling or disabling debug settings, and for runtime optimization.
+
+Use the ``ceph config set`` command to update configuration options. For
+example, to enable the most verbose  debug log level on a specific OSD, run a
+command of the following form:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 debug_ms 20
+
+.. note:: If an option has been customized in a local configuration file, the
+   `central config
+   <https://ceph.io/en/news/blog/2018/new-mimic-centralized-configuration-management/>`_
+   setting will be ignored because it has a lower priority than the local
+   configuration file.
+
+.. note:: Log levels range from 0 to 20.
+
+Override values
+---------------
+
+Options can be set temporarily by using the Ceph CLI ``tell`` or ``daemon``
+interfaces on the Ceph CLI. These *override* values are ephemeral, which means
+that they affect only the current instance of the daemon and revert to
+persistently configured values when the daemon restarts.
+
+Override values can be set in two ways:
+
+#. From any host, send a message to a daemon with a command of the following
+   form:
+
+   .. prompt:: bash $
+
+      ceph tell <name> config set <option> <value>
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph tell osd.123 config set debug_osd 20
+
+   The ``tell`` command can also accept a wildcard as the daemon identifier.
+   For example, to adjust the debug level on all OSD daemons, run a command of
+   the following form:
+
+   .. prompt:: bash $
+
+      ceph tell osd.* config set debug_osd 20
+
+#. On the host where the daemon is running, connect to the daemon via a socket
+   in ``/var/run/ceph`` by running a command of the following form:
+
+   .. prompt:: bash $
+
+      ceph daemon <name> config set <option> <value>
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph daemon osd.4 config set debug_osd 20
+
+.. note:: In the output of the ``ceph config show`` command, these temporary
+   values are shown to have a source of ``override``.
+
+
+Viewing runtime settings
+========================
+
+You can see the current settings specified for a running daemon with the ``ceph
+config show`` command. For example, to see the (non-default) settings for the
+daemon ``osd.0``, run the following command:
+
+.. prompt:: bash $
+
+   ceph config show osd.0
+
+To see a specific setting, run the following command:
+
+.. prompt:: bash $
+
+   ceph config show osd.0 debug_osd
+
+To see all settings (including those with default values), run the following
+command:
+
+.. prompt:: bash $
+
+   ceph config show-with-defaults osd.0
+
+You can see all settings for a daemon that is currently running by connecting
+to it on the local host via the admin socket. For example, to dump all
+current settings, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config show
+
+To see non-default settings and to see where each value came from (for example,
+a config file, the monitor, or an override), run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config diff
+
+To see the value of a single setting, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config get debug_osd
+
+
+Changes introduced in Octopus
+=============================
+
+The Octopus release changed the way the configuration file is parsed.
+These changes are as follows:
+
+- Repeated configuration options are allowed, and no warnings will be
+  displayed. This means that the setting that comes last in the file is the one
+  that takes effect. Prior to this change, Ceph displayed warning messages when
+  lines containing duplicate options were encountered, such as::
+
+    warning line 42: 'foo' in section 'bar' redefined
+- Prior to Octopus, options containing invalid UTF-8 characters were ignored
+  with warning messages. But in Octopus, they are treated as fatal errors.
+- The backslash character ``\`` is used as the line-continuation marker that
+  combines the next line with the current one. Prior to Octopus, there was a
+  requirement that any end-of-line backslash be followed by a non-empty line.
+  But in Octopus, an empty line following a backslash is allowed.
+- In the configuration file, each line specifies an individual configuration
+  option. The option's name and its value are separated with ``=``, and the
+  value may be enclosed within single or double quotes. If an invalid
+  configuration is specified, we will treat it as an invalid configuration
+  file::
+
+    bad option ==== bad value
+- Prior to Octopus, if no section name was specified in the configuration file,
+  all options would be set as though they were within the :confsec:`global`
+  section. This approach is discouraged. Since Octopus, any configuration
+  file that has no section name must contain only a single option.
+
+.. |---|   unicode:: U+2014 .. EM DASH :trim:
diff --git a/doc/rados/configuration/common.rst b/doc/rados/configuration/common.rst
new file mode 100644
index 000000000..0b373f469
--- /dev/null
+++ b/doc/rados/configuration/common.rst
@@ -0,0 +1,207 @@
+.. _ceph-conf-common-settings:
+
+Common Settings
+===============
+
+The `Hardware Recommendations`_ section provides some hardware guidelines for
+configuring a Ceph Storage Cluster. It is possible for a single :term:`Ceph
+Node` to run multiple daemons. For example, a single node with multiple drives
+ususally runs one ``ceph-osd`` for each drive. Ideally, each node will be
+assigned to a particular type of process. For example, some nodes might run
+``ceph-osd`` daemons, other nodes might run ``ceph-mds`` daemons, and still
+other nodes might run ``ceph-mon`` daemons.
+
+Each node has a name. The name of a node can be found in its ``host`` setting.
+Monitors also specify a network address and port (that is, a domain name or IP
+address) that can be found in the ``addr`` setting. A basic configuration file
+typically specifies only minimal settings for each instance of monitor daemons.
+For example:
+
+
+
+
+.. code-block:: ini
+
+    [global]
+    mon_initial_members = ceph1
+    mon_host = 10.0.0.1
+
+.. important:: The ``host`` setting's value is the short name of the node. It
+   is not an FQDN. It is **NOT** an IP address. To retrieve the name of the
+   node, enter ``hostname -s`` on the command line. Unless you are deploying
+   Ceph manually, do not use ``host`` settings for anything other than initial
+   monitor setup.  **DO NOT** specify the ``host`` setting under individual
+   daemons when using deployment tools like ``chef`` or ``cephadm``. Such tools
+   are designed to enter the appropriate values for you in the cluster map.
+
+
+.. _ceph-network-config:
+
+Networks
+========
+
+For more about configuring a network for use with Ceph, see the `Network
+Configuration Reference`_ .
+
+
+Monitors
+========
+
+Ceph production clusters typically provision at least three :term:`Ceph
+Monitor` daemons to ensure availability in the event of a monitor instance
+crash. A minimum of three :term:`Ceph Monitor` daemons ensures that the Paxos
+algorithm is able to determine which version of the :term:`Ceph Cluster Map` is
+the most recent. It makes this determination by consulting a majority of Ceph
+Monitors in the quorum.
+
+.. note:: You may deploy Ceph with a single monitor, but if the instance fails,
+   the lack of other monitors might interrupt data-service availability.
+
+Ceph Monitors normally listen on port ``3300`` for the new v2 protocol, and on
+port ``6789`` for the old v1 protocol.
+
+By default, Ceph expects to store monitor data on the following path::
+
+    /var/lib/ceph/mon/$cluster-$id
+
+You or a deployment tool (for example, ``cephadm``) must create the
+corresponding directory. With metavariables fully expressed and a cluster named
+"ceph", the path specified in the above example evaluates to::
+
+    /var/lib/ceph/mon/ceph-a
+
+For additional details, see the `Monitor Config Reference`_.
+
+.. _Monitor Config Reference: ../mon-config-ref
+
+
+.. _ceph-osd-config:
+
+Authentication
+==============
+
+.. versionadded:: Bobtail 0.56
+
+Authentication is explicitly enabled or disabled in the ``[global]`` section of
+the Ceph configuration file, as shown here:
+
+.. code-block:: ini
+
+    auth_cluster_required = cephx
+    auth_service_required = cephx
+    auth_client_required = cephx
+
+In addition, you should enable message signing. For details, see `Cephx Config
+Reference`_.
+
+.. _Cephx Config Reference: ../auth-config-ref
+
+
+.. _ceph-monitor-config:
+
+
+OSDs
+====
+
+By default, Ceph expects to store a Ceph OSD Daemon's data on the following
+path::
+
+    /var/lib/ceph/osd/$cluster-$id
+
+You or a deployment tool (for example, ``cephadm``) must create the
+corresponding directory. With metavariables fully expressed and a cluster named
+"ceph", the path specified in the above example evaluates to::
+
+    /var/lib/ceph/osd/ceph-0
+
+You can override this path using the ``osd_data`` setting. We recommend that
+you do not change the default location. To create the default directory on your
+OSD host, run the following commands:
+
+.. prompt:: bash $
+
+    ssh {osd-host}
+    sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
+
+The ``osd_data`` path ought to lead to a mount point that has mounted on it a
+device that is distinct from the device that contains the operating system and
+the daemons. To use a device distinct from the device that contains the
+operating system and the daemons, prepare it for use with Ceph and mount it on
+the directory you just created by running the following commands:
+
+.. prompt:: bash $
+
+    ssh {new-osd-host}
+    sudo mkfs -t {fstype} /dev/{disk}
+    sudo mount -o user_xattr /dev/{disk} /var/lib/ceph/osd/ceph-{osd-number}
+
+We recommend using the ``xfs`` file system when running :command:`mkfs`. (The
+``btrfs`` and ``ext4`` file systems are not recommended and are no longer
+tested.)
+
+For additional configuration details, see `OSD Config Reference`_.
+
+
+Heartbeats
+==========
+
+During runtime operations, Ceph OSD Daemons check up on other Ceph OSD Daemons
+and report their findings to the Ceph Monitor. This process does not require
+you to provide any settings. However, if you have network latency issues, you
+might want to modify the default settings.
+
+For additional details, see `Configuring Monitor/OSD Interaction`_.
+
+
+.. _ceph-logging-and-debugging:
+
+Logs / Debugging
+================
+
+You might sometimes encounter issues with Ceph that require you to use Ceph's
+logging and debugging features. For details on log rotation, see `Debugging and
+Logging`_.
+
+.. _Debugging and Logging: ../../troubleshooting/log-and-debug
+
+
+Example ceph.conf
+=================
+
+.. literalinclude:: demo-ceph.conf
+   :language: ini
+
+.. _ceph-runtime-config:
+
+
+
+Naming Clusters (deprecated)
+============================
+
+Each Ceph cluster has an internal name. This internal name is used as part of
+configuration, and as part of "log file" names as well as part of directory
+names and as part of mountpoint names. This name defaults to "ceph". Previous
+releases of Ceph allowed one to specify a custom name instead, for example
+"ceph2". This option was intended to facilitate the running of multiple logical
+clusters on the same physical hardware, but in practice it was rarely
+exploited. Custom cluster names should no longer be attempted. Old
+documentation might lead readers to wrongly think that unique cluster names are
+required to use ``rbd-mirror``. They are not required.
+
+Custom cluster names are now considered deprecated and the ability to deploy
+them has already been removed from some tools, although existing custom-name
+deployments continue to operate. The ability to run and manage clusters with
+custom names might be progressively removed by future Ceph releases, so **it is
+strongly recommended to deploy all new clusters with the default name "ceph"**.
+
+Some Ceph CLI commands accept a ``--cluster`` (cluster name) option. This
+option is present only for the sake of backward compatibility. New tools and
+deployments cannot be relied upon to accommodate this option.
+
+If you need to allow multiple clusters to exist on the same host, use
+:ref:`cephadm`, which uses containers to fully isolate each cluster.
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+.. _Network Configuration Reference: ../network-config-ref
+.. _OSD Config Reference: ../osd-config-ref
+.. _Configuring Monitor/OSD Interaction: ../mon-osd-interaction
diff --git a/doc/rados/configuration/demo-ceph.conf b/doc/rados/configuration/demo-ceph.conf
new file mode 100644
index 000000000..8ba285a42
--- /dev/null
+++ b/doc/rados/configuration/demo-ceph.conf
@@ -0,0 +1,31 @@
+[global]
+fsid = {cluster-id}
+mon_initial_members = {hostname}[, {hostname}]
+mon_host = {ip-address}[, {ip-address}]
+
+#All clusters have a front-side public network.
+#If you have two network interfaces, you can configure a private / cluster 
+#network for RADOS object replication, heartbeats, backfill,
+#recovery, etc.
+public_network = {network}[, {network}]
+#cluster_network = {network}[, {network}] 
+
+#Clusters require authentication by default.
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+#Choose reasonable number of replicas and placement groups.
+osd_journal_size = {n}
+osd_pool_default_size = {n}  # Write an object n times.
+osd_pool_default_min_size = {n} # Allow writing n copies in a degraded state.
+osd_pool_default_pg_autoscale_mode = {mode} # on, off, or warn
+# Only used if autoscaling is off or warn:
+osd_pool_default_pg_num = {n}
+
+#Choose a reasonable crush leaf type.
+#0 for a 1-node cluster.
+#1 for a multi node cluster in a single rack
+#2 for a multi node, multi chassis cluster with multiple hosts in a chassis
+#3 for a multi node cluster with hosts across racks, etc.
+osd_crush_chooseleaf_type = {n}
diff --git a/doc/rados/configuration/filestore-config-ref.rst b/doc/rados/configuration/filestore-config-ref.rst
new file mode 100644
index 000000000..7aefe26b3
--- /dev/null
+++ b/doc/rados/configuration/filestore-config-ref.rst
@@ -0,0 +1,377 @@
+============================
+ Filestore Config Reference
+============================
+
+.. note:: Since the Luminous release of Ceph, Filestore has not been Ceph's
+   default storage back end. Since the Luminous release of Ceph, BlueStore has
+   been Ceph's default storage back end. However, Filestore OSDs are still
+   supported up to Quincy. Filestore OSDs are not supported in Reef. See
+   :ref:`OSD Back Ends <rados_config_storage_devices_osd_backends>`. See
+   :ref:`BlueStore Migration <rados_operations_bluestore_migration>` for
+   instructions explaining how to replace an existing Filestore back end with a
+   BlueStore back end.
+
+
+``filestore_debug_omap_check``
+
+:Description: Debugging check on synchronization. Expensive. For debugging only.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+.. index:: filestore; extended attributes
+
+Extended Attributes
+===================
+
+Extended Attributes (XATTRs) are important for Filestore OSDs. However, Certain
+disadvantages can occur when the underlying file system is used for the storage
+of XATTRs: some file systems have limits on the number of bytes that can be
+stored in XATTRs, and your file system might in some cases therefore run slower
+than would an alternative method of storing XATTRs. For this reason, a method
+of storing XATTRs extrinsic to the underlying file system might improve
+performance. To implement such an extrinsic method, refer to the following
+settings.
+
+If the underlying file system has no size limit, then Ceph XATTRs are stored as
+``inline xattr``, using the XATTRs provided by the file system. But if there is
+a size limit (for example, ext4 imposes a limit of 4 KB total), then some Ceph
+XATTRs will be stored in a key/value database when the limit is reached. More
+precisely, this begins to occur when either the
+``filestore_max_inline_xattr_size`` or ``filestore_max_inline_xattrs``
+threshold is reached.
+
+
+``filestore_max_inline_xattr_size``
+
+:Description: Defines the maximum size per object of an XATTR that can be
+              stored in the file system (for example, XFS, Btrfs, ext4). The
+              specified size should not be larger than the file system can
+              handle. Using the default value of 0 instructs Filestore to use
+              the value specific to the file system.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``filestore_max_inline_xattr_size_xfs``
+
+:Description: Defines the maximum size of an XATTR that can be stored in the
+              XFS file system.  This setting is used only if
+              ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``65536``
+
+
+``filestore_max_inline_xattr_size_btrfs``
+
+:Description: Defines the maximum size of an XATTR that can be stored in the
+              Btrfs file system.  This setting is used only if
+              ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``2048``
+
+
+``filestore_max_inline_xattr_size_other``
+
+:Description: Defines the maximum size of an XATTR that can be stored in other file systems.
+              This setting is used only if ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``512``
+
+
+``filestore_max_inline_xattrs``
+
+:Description: Defines the maximum number of XATTRs per object that can be stored in the file system.
+              Using the default value of 0 instructs Filestore to use the value specific to the file system.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``filestore_max_inline_xattrs_xfs``
+
+:Description: Defines the maximum number of XATTRs per object that can be stored in the XFS file system.
+              This setting is used only if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``10``
+
+
+``filestore_max_inline_xattrs_btrfs``
+
+:Description: Defines the maximum number of XATTRs per object that can be stored in the Btrfs file system.
+              This setting is used only if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``10``
+
+
+``filestore_max_inline_xattrs_other``
+
+:Description: Defines the maximum number of XATTRs per object that can be stored in other file systems.
+              This setting is used only if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``2``
+
+.. index:: filestore; synchronization
+
+Synchronization Intervals
+=========================
+
+Filestore must periodically quiesce writes and synchronize the file system.
+Each synchronization creates a consistent commit point. When the commit point
+is created, Filestore is able to free all journal entries up to that point.
+More-frequent synchronization tends to reduce both synchronization time and
+the amount of data that needs to remain in the journal. Less-frequent
+synchronization allows the backing file system to coalesce small writes and
+metadata updates, potentially increasing synchronization
+efficiency but also potentially increasing tail latency.
+
+
+``filestore_max_sync_interval``
+
+:Description: Defines the maximum interval (in seconds) for synchronizing Filestore.
+:Type: Double
+:Required: No
+:Default: ``5``
+
+
+``filestore_min_sync_interval``
+
+:Description: Defines the minimum interval (in seconds) for synchronizing Filestore.
+:Type: Double
+:Required: No
+:Default: ``.01``
+
+
+.. index:: filestore; flusher
+
+Flusher
+=======
+
+The Filestore flusher forces data from large writes to be written out using
+``sync_file_range`` prior to the synchronization.
+Ideally, this action reduces the cost of the eventual synchronization. In practice, however, disabling
+'filestore_flusher' seems in some cases to improve performance.
+
+
+``filestore_flusher``
+
+:Description: Enables the Filestore flusher.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+.. deprecated:: v.65
+
+``filestore_flusher_max_fds``
+
+:Description: Defines the maximum number of file descriptors for the flusher.
+:Type: Integer
+:Required: No
+:Default: ``512``
+
+.. deprecated:: v.65
+
+``filestore_sync_flush``
+
+:Description: Enables the synchronization flusher. 
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+.. deprecated:: v.65
+
+``filestore_fsync_flushes_journal_data``
+
+:Description: Flushes journal data during file-system synchronization.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+.. index:: filestore; queue
+
+Queue
+=====
+
+The following settings define limits on the size of the Filestore queue:
+
+``filestore_queue_max_ops``
+
+:Description: Defines the maximum number of in-progress operations that Filestore accepts before it blocks the queueing of any new operations. 
+:Type: Integer
+:Required: No. Minimal impact on performance.
+:Default: ``50``
+
+
+``filestore_queue_max_bytes``
+
+:Description: Defines the maximum number of bytes permitted per operation.
+:Type: Integer
+:Required: No
+:Default: ``100 << 20``
+
+
+.. index:: filestore; timeouts
+
+Timeouts
+========
+
+``filestore_op_threads``
+
+:Description: Defines the number of file-system operation threads that execute in parallel. 
+:Type: Integer
+:Required: No
+:Default: ``2``
+
+
+``filestore_op_thread_timeout``
+
+:Description: Defines the timeout (in seconds) for a file-system operation thread.
+:Type: Integer
+:Required: No
+:Default: ``60``
+
+
+``filestore_op_thread_suicide_timeout``
+
+:Description: Defines the timeout (in seconds) for a commit operation before the commit is cancelled.
+:Type: Integer
+:Required: No
+:Default: ``180``
+
+
+.. index:: filestore; btrfs
+
+B-Tree Filesystem
+=================
+
+
+``filestore_btrfs_snap``
+
+:Description: Enables snapshots for a ``btrfs`` Filestore.
+:Type: Boolean
+:Required: No. Used only for ``btrfs``.
+:Default: ``true``
+
+
+``filestore_btrfs_clone_range``
+
+:Description: Enables cloning ranges for a ``btrfs`` Filestore.
+:Type: Boolean
+:Required: No. Used only for ``btrfs``.
+:Default: ``true``
+
+
+.. index:: filestore; journal
+
+Journal
+=======
+
+
+``filestore_journal_parallel``
+
+:Description: Enables parallel journaling, default for ``btrfs``.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_journal_writeahead``
+
+:Description: Enables write-ahead journaling, default for XFS.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_journal_trailing``
+
+:Description: Deprecated. **Never use.**
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+Misc
+====
+
+
+``filestore_merge_threshold``
+
+:Description: Defines the minimum number of files permitted in a subdirectory before the subdirectory is merged into its parent directory.
+              NOTE: A negative value means that subdirectory merging is disabled.
+:Type: Integer
+:Required: No
+:Default: ``-10``
+
+
+``filestore_split_multiple``
+
+:Description:  ``(filestore_split_multiple * abs(filestore_merge_threshold) + (rand() % filestore_split_rand_factor)) * 16``
+               is the maximum number of files permitted in a subdirectory
+               before the subdirectory is split into child directories.
+
+:Type: Integer
+:Required: No
+:Default: ``2``
+
+
+``filestore_split_rand_factor``
+
+:Description:  A random factor added to the split threshold to avoid
+               too many (expensive) Filestore splits occurring at the same time.
+               For details, see ``filestore_split_multiple``.
+               To change this setting for an existing OSD, it is necessary to take the OSD
+               offline before running the ``ceph-objectstore-tool apply-layout-settings`` command.
+
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``20``
+
+
+``filestore_update_to``
+
+:Description: Limits automatic upgrades to a specified version of Filestore. Useful in cases in which you want to avoid upgrading to a specific version.
+:Type: Integer
+:Required: No
+:Default: ``1000``
+
+
+``filestore_blackhole``
+
+:Description: Drops any new transactions on the floor, similar to redirecting to NULL. 
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_dump_file``
+
+:Description: Defines the file that transaction dumps are stored on.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_kill_at``
+
+:Description: Injects a failure at the *n*\th opportunity.
+:Type: String
+:Required: No
+:Default: ``false``
+
+
+``filestore_fail_eio``
+
+:Description: Fail/Crash on EIO.
+:Type: Boolean
+:Required: No
+:Default: ``true``
diff --git a/doc/rados/configuration/general-config-ref.rst b/doc/rados/configuration/general-config-ref.rst
new file mode 100644
index 000000000..f4613456a
--- /dev/null
+++ b/doc/rados/configuration/general-config-ref.rst
@@ -0,0 +1,19 @@
+==========================
+ General Config Reference
+==========================
+
+.. confval:: admin_socket
+   :default: /var/run/ceph/$cluster-$name.asok
+.. confval:: pid_file
+.. confval:: chdir
+.. confval:: fatal_signal_handlers
+.. describe:: max_open_files
+
+   If set, when the :term:`Ceph Storage Cluster` starts, Ceph sets
+   the max open FDs at the OS level (i.e., the max # of file
+   descriptors). A suitably large value prevents Ceph Daemons from running out
+   of file descriptors.
+
+   :Type: 64-bit Integer
+   :Required: No
+   :Default: ``0``
diff --git a/doc/rados/configuration/index.rst b/doc/rados/configuration/index.rst
new file mode 100644
index 000000000..715b999d1
--- /dev/null
+++ b/doc/rados/configuration/index.rst
@@ -0,0 +1,53 @@
+===============
+ Configuration
+===============
+
+Each Ceph process, daemon, or utility draws its configuration from several
+sources on startup. Such sources can include (1) a local configuration, (2) the
+monitors, (3) the command line, and (4) environment variables.
+
+Configuration options can be set globally so that they apply (1) to all
+daemons, (2) to all daemons or services of a particular type, or (3) to only a
+specific daemon, process, or client.
+
+.. raw:: html
+
+	<table cellpadding="10"><colgroup><col width="50%"><col width="50%"></colgroup><tbody valign="top"><tr><td><h3>Configuring the Object Store</h3>
+
+For general object store configuration, refer to the following:
+
+.. toctree::
+   :maxdepth: 1
+
+   Storage devices <storage-devices>
+   ceph-conf
+
+
+.. raw:: html 
+
+	</td><td><h3>Reference</h3>
+
+To optimize the performance of your cluster, refer to the following:
+
+.. toctree::
+   :maxdepth: 1
+
+   Common Settings <common>
+   Network Settings <network-config-ref>
+   Messenger v2 protocol <msgr2>
+   Auth Settings <auth-config-ref>
+   Monitor Settings <mon-config-ref>
+   mon-lookup-dns
+   Heartbeat Settings <mon-osd-interaction>
+   OSD Settings <osd-config-ref>
+   DmClock Settings <mclock-config-ref>
+   BlueStore Settings <bluestore-config-ref>
+   FileStore Settings <filestore-config-ref>
+   Journal Settings <journal-ref>
+   Pool, PG & CRUSH Settings <pool-pg-config-ref.rst>
+   General Settings <general-config-ref>
+
+   
+.. raw:: html
+
+	</td></tr></tbody></table>
diff --git a/doc/rados/configuration/journal-ref.rst b/doc/rados/configuration/journal-ref.rst
new file mode 100644
index 000000000..5ce5a5e2d
--- /dev/null
+++ b/doc/rados/configuration/journal-ref.rst
@@ -0,0 +1,39 @@
+==========================
+ Journal Config Reference
+==========================
+.. warning:: Filestore has been deprecated in the Reef release and is no longer supported.
+.. index:: journal; journal configuration
+
+Filestore OSDs use a journal for two reasons: speed and consistency.  Note
+that since Luminous, the BlueStore OSD back end has been preferred and default.
+This information is provided for pre-existing OSDs and for rare situations where
+Filestore is preferred for new deployments.
+
+- **Speed:** The journal enables the Ceph OSD Daemon to commit small writes 
+  quickly. Ceph writes small, random i/o to the journal sequentially, which 
+  tends to speed up bursty workloads by allowing the backing file system more 
+  time to coalesce writes. The Ceph OSD Daemon's journal, however, can lead 
+  to spiky performance with short spurts of high-speed writes followed by 
+  periods without any write progress as the file system catches up to the 
+  journal.
+
+- **Consistency:** Ceph OSD Daemons require a file system interface that 
+  guarantees atomic compound operations. Ceph OSD Daemons write a description 
+  of the operation to the journal and apply the operation to the file system. 
+  This enables atomic updates to an object (for example, placement group 
+  metadata). Every few seconds--between ``filestore max sync interval`` and
+  ``filestore min sync interval``--the Ceph OSD Daemon stops writes and 
+  synchronizes the journal with the file system, allowing Ceph OSD Daemons to 
+  trim operations from the journal and reuse the space. On failure, Ceph 
+  OSD Daemons replay the journal starting after the last synchronization 
+  operation.
+
+Ceph OSD Daemons recognize the following journal settings: 
+
+.. confval:: journal_dio
+.. confval:: journal_aio
+.. confval:: journal_block_align
+.. confval:: journal_max_write_bytes
+.. confval:: journal_max_write_entries
+.. confval:: journal_align_min_size
+.. confval:: journal_zero_on_create
diff --git a/doc/rados/configuration/mclock-config-ref.rst b/doc/rados/configuration/mclock-config-ref.rst
new file mode 100644
index 000000000..a338aa6da
--- /dev/null
+++ b/doc/rados/configuration/mclock-config-ref.rst
@@ -0,0 +1,699 @@
+========================
+ mClock Config Reference
+========================
+
+.. index:: mclock; configuration
+
+QoS support in Ceph is implemented using a queuing scheduler based on `the
+dmClock algorithm`_. See :ref:`dmclock-qos` section for more details.
+
+To make the usage of mclock more user-friendly and intuitive, mclock config
+profiles are introduced. The mclock profiles mask the low level details from
+users, making it easier to configure and use mclock.
+
+The following input parameters are required for a mclock profile to configure
+the QoS related parameters:
+
+* total capacity (IOPS) of each OSD (determined automatically -
+  See `OSD Capacity Determination (Automated)`_)
+
+* the max sequential bandwidth capacity (MiB/s) of each OSD -
+  See *osd_mclock_max_sequential_bandwidth_[hdd|ssd]* option
+
+* an mclock profile type to enable
+
+Using the settings in the specified profile, an OSD determines and applies the
+lower-level mclock and Ceph parameters. The parameters applied by the mclock
+profile make it possible to tune the QoS between client I/O and background
+operations in the OSD.
+
+
+.. index:: mclock; mclock clients
+
+mClock Client Types
+===================
+
+The mclock scheduler handles requests from different types of Ceph services.
+Each service can be considered as a type of client from mclock's perspective.
+Depending on the type of requests handled, mclock clients are classified into
+the buckets as shown in the table below,
+
++------------------------+--------------------------------------------------------------+
+|  Client Type           | Request Types                                                |
++========================+==============================================================+
+| Client                 | I/O requests issued by external clients of Ceph              |
++------------------------+--------------------------------------------------------------+
+| Background recovery    | Internal recovery requests                                   |
++------------------------+--------------------------------------------------------------+
+| Background best-effort | Internal backfill, scrub, snap trim and PG deletion requests |
++------------------------+--------------------------------------------------------------+
+
+The mclock profiles allocate parameters like reservation, weight and limit
+(see :ref:`dmclock-qos`) differently for each client type. The next sections
+describe the mclock profiles in greater detail.
+
+
+.. index:: mclock; profile definition
+
+mClock Profiles - Definition and Purpose
+========================================
+
+A mclock profile is *“a configuration setting that when applied on a running
+Ceph cluster enables the throttling of the operations(IOPS) belonging to
+different client classes (background recovery, scrub, snaptrim, client op,
+osd subop)�*.
+
+The mclock profile uses the capacity limits and the mclock profile type selected
+by the user to determine the low-level mclock resource control configuration
+parameters and apply them transparently. Additionally, other Ceph configuration
+parameters are also applied. Please see sections below for more information.
+
+The low-level mclock resource control parameters are the *reservation*,
+*limit*, and *weight* that provide control of the resource shares, as
+described in the :ref:`dmclock-qos` section.
+
+
+.. index:: mclock; profile types
+
+mClock Profile Types
+====================
+
+mclock profiles can be broadly classified into *built-in* and *custom* profiles,
+
+Built-in Profiles
+-----------------
+Users can choose between the following built-in profile types:
+
+.. note:: The values mentioned in the tables below represent the proportion
+          of the total IOPS capacity of the OSD allocated for the service type.
+
+* balanced (default)
+* high_client_ops
+* high_recovery_ops
+
+balanced (*default*)
+^^^^^^^^^^^^^^^^^^^^
+The *balanced* profile is the default mClock profile. This profile allocates
+equal reservation/priority to client operations and background recovery
+operations. Background best-effort ops are given lower reservation and therefore
+take a longer time to complete when are are competing operations. This profile
+helps meet the normal/steady-state requirements of the cluster. This is the
+case when external client performance requirement is not critical and there are
+other background operations that still need attention within the OSD.
+
+But there might be instances that necessitate giving higher allocations to either
+client ops or recovery ops. In order to deal with such a situation, the alternate
+built-in profiles may be enabled by following the steps mentioned in next sections.
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 50%         | 1      | MAX   |
++------------------------+-------------+--------+-------+
+| background recovery    | 50%         | 1      | MAX   |
++------------------------+-------------+--------+-------+
+| background best-effort | MIN         | 1      | 90%   |
++------------------------+-------------+--------+-------+
+
+high_client_ops
+^^^^^^^^^^^^^^^
+This profile optimizes client performance over background activities by
+allocating more reservation and limit to client operations as compared to
+background operations in the OSD. This profile, for example, may be enabled
+to provide the needed performance for I/O intensive applications for a
+sustained period of time at the cost of slower recoveries. The table shows
+the resource control parameters set by the profile:
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 60%         | 2      | MAX   |
++------------------------+-------------+--------+-------+
+| background recovery    | 40%         | 1      | MAX   |
++------------------------+-------------+--------+-------+
+| background best-effort | MIN         | 1      | 70%   |
++------------------------+-------------+--------+-------+
+
+high_recovery_ops
+^^^^^^^^^^^^^^^^^
+This profile optimizes background recovery performance as compared to external
+clients and other background operations within the OSD. This profile, for
+example, may be enabled by an administrator temporarily to speed-up background
+recoveries during non-peak hours. The table shows the resource control
+parameters set by the profile:
+
++------------------------+-------------+--------+-------+
+|  Service Type          | Reservation | Weight | Limit |
++========================+=============+========+=======+
+| client                 | 30%         | 1      | MAX   |
++------------------------+-------------+--------+-------+
+| background recovery    | 70%         | 2      | MAX   |
++------------------------+-------------+--------+-------+
+| background best-effort | MIN         | 1      | MAX   |
++------------------------+-------------+--------+-------+
+
+.. note:: Across the built-in profiles, internal background best-effort clients
+          of mclock include "backfill", "scrub", "snap trim", and "pg deletion"
+          operations.
+
+
+Custom Profile
+--------------
+This profile gives users complete control over all the mclock configuration
+parameters. This profile should be used with caution and is meant for advanced
+users, who understand mclock and Ceph related configuration options.
+
+
+.. index:: mclock; built-in profiles
+
+mClock Built-in Profiles -  Locked Config Options
+=================================================
+The below sections describe the config options that are locked to certain values
+in order to ensure mClock scheduler is able to provide predictable QoS.
+
+mClock Config Options
+---------------------
+.. important:: These defaults cannot be changed using any of the config
+   subsytem commands like *config set* or via the *config daemon* or *config
+   tell* interfaces. Although the above command(s) report success, the mclock
+   QoS parameters are reverted to their respective built-in profile defaults.
+
+When a built-in profile is enabled, the mClock scheduler calculates the low
+level mclock parameters [*reservation*, *weight*, *limit*] based on the profile
+enabled for each client type. The mclock parameters are calculated based on
+the max OSD capacity provided beforehand. As a result, the following mclock
+config parameters cannot be modified when using any of the built-in profiles:
+
+- :confval:`osd_mclock_scheduler_client_res`
+- :confval:`osd_mclock_scheduler_client_wgt`
+- :confval:`osd_mclock_scheduler_client_lim`
+- :confval:`osd_mclock_scheduler_background_recovery_res`
+- :confval:`osd_mclock_scheduler_background_recovery_wgt`
+- :confval:`osd_mclock_scheduler_background_recovery_lim`
+- :confval:`osd_mclock_scheduler_background_best_effort_res`
+- :confval:`osd_mclock_scheduler_background_best_effort_wgt`
+- :confval:`osd_mclock_scheduler_background_best_effort_lim`
+
+Recovery/Backfill Options
+-------------------------
+.. warning:: The recommendation is to not change these options as the built-in
+   profiles are optimized based on them. Changing these defaults can result in
+   unexpected performance outcomes.
+
+The following recovery and backfill related Ceph options are overridden to
+mClock defaults:
+
+- :confval:`osd_max_backfills`
+- :confval:`osd_recovery_max_active`
+- :confval:`osd_recovery_max_active_hdd`
+- :confval:`osd_recovery_max_active_ssd`
+
+The following table shows the mClock defaults which is the same as the current
+defaults. This is done to maximize the performance of the foreground (client)
+operations:
+
++----------------------------------------+------------------+----------------+
+|  Config Option                         | Original Default | mClock Default |
++========================================+==================+================+
+| :confval:`osd_max_backfills`           | 1                | 1              |
++----------------------------------------+------------------+----------------+
+| :confval:`osd_recovery_max_active`     | 0                | 0              |
++----------------------------------------+------------------+----------------+
+| :confval:`osd_recovery_max_active_hdd` | 3                | 3              |
++----------------------------------------+------------------+----------------+
+| :confval:`osd_recovery_max_active_ssd` | 10               | 10             |
++----------------------------------------+------------------+----------------+
+
+The above mClock defaults, can be modified only if necessary by enabling
+:confval:`osd_mclock_override_recovery_settings` (default: false). The
+steps for this is discussed in the
+`Steps to Modify mClock Max Backfills/Recovery Limits`_ section.
+
+Sleep Options
+-------------
+If any mClock profile (including "custom") is active, the following Ceph config
+sleep options are disabled (set to 0),
+
+- :confval:`osd_recovery_sleep`
+- :confval:`osd_recovery_sleep_hdd`
+- :confval:`osd_recovery_sleep_ssd`
+- :confval:`osd_recovery_sleep_hybrid`
+- :confval:`osd_scrub_sleep`
+- :confval:`osd_delete_sleep`
+- :confval:`osd_delete_sleep_hdd`
+- :confval:`osd_delete_sleep_ssd`
+- :confval:`osd_delete_sleep_hybrid`
+- :confval:`osd_snap_trim_sleep`
+- :confval:`osd_snap_trim_sleep_hdd`
+- :confval:`osd_snap_trim_sleep_ssd`
+- :confval:`osd_snap_trim_sleep_hybrid`
+
+The above sleep options are disabled to ensure that mclock scheduler is able to
+determine when to pick the next op from its operation queue and transfer it to
+the operation sequencer. This results in the desired QoS being provided across
+all its clients.
+
+
+.. index:: mclock; enable built-in profile
+
+Steps to Enable mClock Profile
+==============================
+
+As already mentioned, the default mclock profile is set to *balanced*.
+The other values for the built-in profiles include *high_client_ops* and
+*high_recovery_ops*.
+
+If there is a requirement to change the default profile, then the option
+:confval:`osd_mclock_profile` may be set during runtime by using the following
+command:
+
+  .. prompt:: bash #
+
+    ceph config set osd.N osd_mclock_profile <value>
+
+For example, to change the profile to allow faster recoveries on "osd.0", the
+following command can be used to switch to the *high_recovery_ops* profile:
+
+  .. prompt:: bash #
+
+    ceph config set osd.0 osd_mclock_profile high_recovery_ops
+
+.. note:: The *custom* profile is not recommended unless you are an advanced
+          user.
+
+And that's it! You are ready to run workloads on the cluster and check if the
+QoS requirements are being met.
+
+
+Switching Between Built-in and Custom Profiles
+==============================================
+
+There may be situations requiring switching from a built-in profile to the
+*custom* profile and vice-versa. The following sections outline the steps to
+accomplish this.
+
+Steps to Switch From a Built-in to the Custom Profile
+-----------------------------------------------------
+
+The following command can be used to switch to the *custom* profile:
+
+  .. prompt:: bash #
+
+    ceph config set osd osd_mclock_profile custom
+
+For example, to change the profile to *custom* on all OSDs, the following
+command can be used:
+
+  .. prompt:: bash #
+
+    ceph config set osd osd_mclock_profile custom
+
+After switching to the *custom* profile, the desired mClock configuration
+option may be modified. For example, to change the client reservation IOPS
+ratio for a specific OSD (say osd.0) to 0.5 (or 50%), the following command
+can be used:
+
+  .. prompt:: bash #
+
+    ceph config set osd.0 osd_mclock_scheduler_client_res 0.5
+
+.. important:: Care must be taken to change the reservations of other services
+   like recovery and background best effort accordingly to ensure that the sum
+   of the reservations do not exceed the maximum proportion (1.0) of the IOPS
+   capacity of the OSD.
+
+.. tip::  The reservation and limit parameter allocations are per-shard based on
+   the type of backing device (HDD/SSD) under the OSD. See
+   :confval:`osd_op_num_shards_hdd` and :confval:`osd_op_num_shards_ssd` for
+   more details.
+
+Steps to Switch From the Custom Profile to a Built-in Profile
+-------------------------------------------------------------
+
+Switching from the *custom* profile to a built-in profile requires an
+intermediate step of removing the custom settings from the central config
+database for the changes to take effect.
+
+The following sequence of commands can be used to switch to a built-in profile:
+
+#. Set the desired built-in profile using:
+
+   .. prompt:: bash #
+
+     ceph config set osd <mClock Configuration Option>
+
+   For example, to set the built-in profile to ``high_client_ops`` on all
+   OSDs, run the following command:
+
+   .. prompt:: bash #
+
+     ceph config set osd osd_mclock_profile high_client_ops
+#. Determine the existing custom mClock configuration settings in the central
+   config database using the following command:
+
+   .. prompt:: bash #
+
+     ceph config dump
+#. Remove the custom mClock configuration settings determined in the previous
+   step from the central config database:
+
+   .. prompt:: bash #
+
+     ceph config rm osd <mClock Configuration Option>
+
+   For example, to remove the configuration option
+   :confval:`osd_mclock_scheduler_client_res` that was set on all OSDs, run the
+   following command:
+
+   .. prompt:: bash #
+
+     ceph config rm osd osd_mclock_scheduler_client_res
+#. After all existing custom mClock configuration settings have been removed
+   from the central config database, the configuration settings pertaining to
+   ``high_client_ops`` will come into effect. For e.g., to verify the settings
+   on osd.0 use:
+
+   .. prompt:: bash #
+
+     ceph config show osd.0
+
+Switch Temporarily Between mClock Profiles
+------------------------------------------
+
+To switch between mClock profiles on a temporary basis, the following commands
+may be used to override the settings:
+
+.. warning:: This section is for advanced users or for experimental testing. The
+   recommendation is to not use the below commands on a running cluster as it
+   could have unexpected outcomes.
+
+.. note:: The configuration changes on an OSD using the below commands are
+   ephemeral and are lost when it restarts. It is also important to note that
+   the config options overridden using the below commands cannot be modified
+   further using the *ceph config set osd.N ...* command. The changes will not
+   take effect until a given OSD is restarted. This is intentional, as per the
+   config subsystem design. However, any further modification can still be made
+   ephemerally using the commands mentioned below.
+
+#. Run the *injectargs* command as shown to override the mclock settings:
+
+   .. prompt:: bash #
+
+     ceph tell osd.N injectargs '--<mClock Configuration Option>=<value>'
+
+   For example, the following command overrides the
+   :confval:`osd_mclock_profile` option on osd.0:
+
+   .. prompt:: bash #
+
+     ceph tell osd.0 injectargs '--osd_mclock_profile=high_recovery_ops'
+
+
+#. An alternate command that can be used is:
+
+   .. prompt:: bash #
+
+     ceph daemon osd.N config set <mClock Configuration Option> <value>
+
+   For example, the following command overrides the
+   :confval:`osd_mclock_profile` option on osd.0:
+
+   .. prompt:: bash #
+
+     ceph daemon osd.0 config set osd_mclock_profile high_recovery_ops
+
+The individual QoS-related config options for the *custom* profile can also be
+modified ephemerally using the above commands.
+
+
+Steps to Modify mClock Max Backfills/Recovery Limits
+====================================================
+
+This section describes the steps to modify the default max backfills or recovery
+limits if the need arises.
+
+.. warning:: This section is for advanced users or for experimental testing. The
+   recommendation is to retain the defaults as is on a running cluster as
+   modifying them could have unexpected performance outcomes. The values may
+   be modified only if the cluster is unable to cope/showing poor performance
+   with the default settings or for performing experiments on a test cluster.
+
+.. important:: The max backfill/recovery options that can be modified are listed
+   in section `Recovery/Backfill Options`_. The modification of the mClock
+   default backfills/recovery limit is gated by the
+   :confval:`osd_mclock_override_recovery_settings` option, which is set to
+   *false* by default. Attempting to modify any default recovery/backfill
+   limits without setting the gating option will reset that option back to the
+   mClock defaults along with a warning message logged in the cluster log. Note
+   that it may take a few seconds for the default value to come back into
+   effect. Verify the limit using the *config show* command as shown below.
+
+#. Set the :confval:`osd_mclock_override_recovery_settings` config option on all
+   osds to *true* using:
+
+   .. prompt:: bash #
+
+     ceph config set osd osd_mclock_override_recovery_settings true
+
+#. Set the desired max backfill/recovery option using:
+
+   .. prompt:: bash #
+
+     ceph config set osd osd_max_backfills <value>
+
+   For example, the following command modifies the :confval:`osd_max_backfills`
+   option on all osds to 5.
+
+   .. prompt:: bash #
+
+     ceph config set osd osd_max_backfills 5
+
+#. Wait for a few seconds and verify the running configuration for a specific
+   OSD using:
+
+   .. prompt:: bash #
+
+     ceph config show osd.N | grep osd_max_backfills
+
+   For example, the following command shows the running configuration of
+   :confval:`osd_max_backfills` on osd.0.
+
+   .. prompt:: bash #
+
+     ceph config show osd.0 | grep osd_max_backfills
+
+#. Reset the :confval:`osd_mclock_override_recovery_settings` config option on
+   all osds to *false* using:
+
+   .. prompt:: bash #
+
+     ceph config set osd osd_mclock_override_recovery_settings false
+
+
+OSD Capacity Determination (Automated)
+======================================
+
+The OSD capacity in terms of total IOPS is determined automatically during OSD
+initialization. This is achieved by running the OSD bench tool and overriding
+the default value of ``osd_mclock_max_capacity_iops_[hdd, ssd]`` option
+depending on the device type. No other action/input is expected from the user
+to set the OSD capacity.
+
+.. note:: If you wish to manually benchmark OSD(s) or manually tune the
+          Bluestore throttle parameters, see section
+          `Steps to Manually Benchmark an OSD (Optional)`_.
+
+You may verify the capacity of an OSD after the cluster is brought up by using
+the following command:
+
+  .. prompt:: bash #
+
+    ceph config show osd.N osd_mclock_max_capacity_iops_[hdd, ssd]
+
+For example, the following command shows the max capacity for "osd.0" on a Ceph
+node whose underlying device type is SSD:
+
+  .. prompt:: bash #
+
+    ceph config show osd.0 osd_mclock_max_capacity_iops_ssd
+
+Mitigation of Unrealistic OSD Capacity From Automated Test
+----------------------------------------------------------
+In certain conditions, the OSD bench tool may show unrealistic/inflated result
+depending on the drive configuration and other environment related conditions.
+To mitigate the performance impact due to this unrealistic capacity, a couple
+of threshold config options depending on the osd's device type are defined and
+used:
+
+- :confval:`osd_mclock_iops_capacity_threshold_hdd` = 500
+- :confval:`osd_mclock_iops_capacity_threshold_ssd` = 80000
+
+The following automated step is performed:
+
+Fallback to using default OSD capacity (automated)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If OSD bench reports a measurement that exceeds the above threshold values
+depending on the underlying device type, the fallback mechanism reverts to the
+default value of :confval:`osd_mclock_max_capacity_iops_hdd` or
+:confval:`osd_mclock_max_capacity_iops_ssd`. The threshold config options
+can be reconfigured based on the type of drive used. Additionally, a cluster
+warning is logged in case the measurement exceeds the threshold. For example, ::
+
+    2022-10-27T15:30:23.270+0000 7f9b5dbe95c0  0 log_channel(cluster) log [WRN]
+    : OSD bench result of 39546.479392 IOPS exceeded the threshold limit of
+    25000.000000 IOPS for osd.1. IOPS capacity is unchanged at 21500.000000
+    IOPS. The recommendation is to establish the osd's IOPS capacity using other
+    benchmark tools (e.g. Fio) and then override
+    osd_mclock_max_capacity_iops_[hdd|ssd].
+
+If the default capacity doesn't accurately represent the OSD's capacity, the
+following additional step is recommended to address this:
+
+Run custom drive benchmark if defaults are not accurate (manual)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If the default OSD capacity is not accurate, the recommendation is to run a
+custom benchmark using your preferred tool (e.g. Fio) on the drive and then
+override the ``osd_mclock_max_capacity_iops_[hdd, ssd]`` option as described
+in the `Specifying  Max OSD Capacity`_ section.
+
+This step is highly recommended until an alternate mechansim is worked upon.
+
+Steps to Manually Benchmark an OSD (Optional)
+=============================================
+
+.. note:: These steps are only necessary if you want to override the OSD
+          capacity already determined automatically during OSD initialization.
+          Otherwise, you may skip this section entirely.
+
+.. tip:: If you have already determined the benchmark data and wish to manually
+         override the max osd capacity for an OSD, you may skip to section
+         `Specifying  Max OSD Capacity`_.
+
+
+Any existing benchmarking tool (e.g. Fio) can be used for this purpose. In this
+case, the steps use the *Ceph OSD Bench* command described in the next section.
+Regardless of the tool/command used, the steps outlined further below remain the
+same.
+
+As already described in the :ref:`dmclock-qos` section, the number of
+shards and the bluestore's throttle parameters have an impact on the mclock op
+queues. Therefore, it is critical to set these values carefully in order to
+maximize the impact of the mclock scheduler.
+
+:Number of Operational Shards:
+  We recommend using the default number of shards as defined by the
+  configuration options ``osd_op_num_shards``, ``osd_op_num_shards_hdd``, and
+  ``osd_op_num_shards_ssd``. In general, a lower number of shards will increase
+  the impact of the mclock queues.
+
+:Bluestore Throttle Parameters:
+  We recommend using the default values as defined by
+  :confval:`bluestore_throttle_bytes` and
+  :confval:`bluestore_throttle_deferred_bytes`. But these parameters may also be
+  determined during the benchmarking phase as described below.
+
+OSD Bench Command Syntax
+------------------------
+
+The :ref:`osd-subsystem` section describes the OSD bench command. The syntax
+used for benchmarking is shown below :
+
+.. prompt:: bash #
+
+  ceph tell osd.N bench [TOTAL_BYTES] [BYTES_PER_WRITE] [OBJ_SIZE] [NUM_OBJS]
+
+where,
+
+* ``TOTAL_BYTES``: Total number of bytes to write
+* ``BYTES_PER_WRITE``: Block size per write
+* ``OBJ_SIZE``: Bytes per object
+* ``NUM_OBJS``: Number of objects to write
+
+Benchmarking Test Steps Using OSD Bench
+---------------------------------------
+
+The steps below use the default shards and detail the steps used to determine
+the correct bluestore throttle values (optional).
+
+#. Bring up your Ceph cluster and login to the Ceph node hosting the OSDs that
+   you wish to benchmark.
+#. Run a simple 4KiB random write workload on an OSD using the following
+   commands:
+
+   .. note:: Note that before running the test, caches must be cleared to get an
+             accurate measurement.
+
+   For example, if you are running the benchmark test on osd.0, run the following
+   commands:
+
+   .. prompt:: bash #
+
+     ceph tell osd.0 cache drop
+
+   .. prompt:: bash #
+
+     ceph tell osd.0 bench 12288000 4096 4194304 100
+
+#. Note the overall throughput(IOPS) obtained from the output of the osd bench
+   command. This value is the baseline throughput(IOPS) when the default
+   bluestore throttle options are in effect.
+#. If the intent is to determine the bluestore throttle values for your
+   environment, then set the two options, :confval:`bluestore_throttle_bytes`
+   and :confval:`bluestore_throttle_deferred_bytes` to 32 KiB(32768 Bytes) each
+   to begin with. Otherwise, you may skip to the next section.
+#. Run the 4KiB random write test as before using OSD bench.
+#. Note the overall throughput from the output and compare the value
+   against the baseline throughput recorded in step 3.
+#. If the throughput doesn't match with the baseline, increment the bluestore
+   throttle options by 2x and repeat steps 5 through 7 until the obtained
+   throughput is very close to the baseline value.
+
+For example, during benchmarking on a machine with NVMe SSDs, a value of 256 KiB
+for both bluestore throttle and deferred bytes was determined to maximize the
+impact of mclock. For HDDs, the corresponding value was 40 MiB, where the
+overall throughput was roughly equal to the baseline throughput. Note that in
+general for HDDs, the bluestore throttle values are expected to be higher when
+compared to SSDs.
+
+
+Specifying  Max OSD Capacity
+----------------------------
+
+The steps in this section may be performed only if you want to override the
+max osd capacity automatically set during OSD initialization. The option
+``osd_mclock_max_capacity_iops_[hdd, ssd]`` for an OSD can be set by running the
+following command:
+
+  .. prompt:: bash #
+
+     ceph config set osd.N osd_mclock_max_capacity_iops_[hdd,ssd] <value>
+
+For example, the following command sets the max capacity for a specific OSD
+(say "osd.0") whose underlying device type is HDD to 350 IOPS:
+
+  .. prompt:: bash #
+
+    ceph config set osd.0 osd_mclock_max_capacity_iops_hdd 350
+
+Alternatively, you may specify the max capacity for OSDs within the Ceph
+configuration file under the respective [osd.N] section. See
+:ref:`ceph-conf-settings` for more details.
+
+
+.. index:: mclock; config settings
+
+mClock Config Options
+=====================
+
+.. confval:: osd_mclock_profile
+.. confval:: osd_mclock_max_capacity_iops_hdd
+.. confval:: osd_mclock_max_capacity_iops_ssd
+.. confval:: osd_mclock_max_sequential_bandwidth_hdd
+.. confval:: osd_mclock_max_sequential_bandwidth_ssd
+.. confval:: osd_mclock_force_run_benchmark_on_init
+.. confval:: osd_mclock_skip_benchmark
+.. confval:: osd_mclock_override_recovery_settings
+.. confval:: osd_mclock_iops_capacity_threshold_hdd
+.. confval:: osd_mclock_iops_capacity_threshold_ssd
+
+.. _the dmClock algorithm: https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Gulati.pdf
diff --git a/doc/rados/configuration/mon-config-ref.rst b/doc/rados/configuration/mon-config-ref.rst
new file mode 100644
index 000000000..e0a12d093
--- /dev/null
+++ b/doc/rados/configuration/mon-config-ref.rst
@@ -0,0 +1,642 @@
+.. _monitor-config-reference:
+
+==========================
+ Monitor Config Reference
+==========================
+
+Understanding how to configure a :term:`Ceph Monitor` is an important part of
+building a reliable :term:`Ceph Storage Cluster`. **All Ceph Storage Clusters
+have at least one monitor**. The monitor complement usually remains fairly
+consistent, but you can add, remove or replace a monitor in a cluster. See
+`Adding/Removing a Monitor`_ for details.
+
+
+.. index:: Ceph Monitor; Paxos
+
+Background
+==========
+
+Ceph Monitors maintain a "master copy" of the :term:`Cluster Map`. 
+
+The :term:`Cluster Map` makes it possible for :term:`Ceph client`\s to
+determine the location of all Ceph Monitors, Ceph OSD Daemons, and Ceph
+Metadata Servers. Clients do this by connecting to one Ceph Monitor and
+retrieving a current cluster map. Ceph clients must connect to a Ceph Monitor
+before they can read from or write to Ceph OSD Daemons or Ceph Metadata
+Servers. A Ceph client that has a current copy of the cluster map and the CRUSH
+algorithm can compute the location of any RADOS object within the cluster. This
+makes it possible for Ceph clients to talk directly to Ceph OSD Daemons. Direct
+communication between clients and Ceph OSD Daemons improves upon traditional
+storage architectures that required clients to communicate with a central
+component.  See `Scalability and High Availability`_ for more on this subject.
+
+The Ceph Monitor's primary function is to maintain a master copy of the cluster
+map. Monitors also provide authentication and logging services. All changes in
+the monitor services are written by the Ceph Monitor to a single Paxos
+instance, and Paxos writes the changes to a key/value store. This provides
+strong consistency. Ceph Monitors are able to query the most recent version of
+the cluster map during sync operations, and they use the key/value store's
+snapshots and iterators (using RocksDB) to perform store-wide synchronization.
+
+.. ditaa::
+ /-------------\               /-------------\
+ |   Monitor   | Write Changes |    Paxos    |
+ |   cCCC      +-------------->+   cCCC      |
+ |             |               |             |
+ +-------------+               \------+------/
+ |    Auth     |                      |
+ +-------------+                      | Write Changes
+ |    Log      |                      |
+ +-------------+                      v
+ | Monitor Map |               /------+------\
+ +-------------+               | Key / Value |
+ |   OSD Map   |               |    Store    |
+ +-------------+               |  cCCC       |
+ |   PG Map    |               \------+------/
+ +-------------+                      ^
+ |   MDS Map   |                      | Read Changes
+ +-------------+                      |
+ |    cCCC     |*---------------------+
+ \-------------/
+
+.. index:: Ceph Monitor; cluster map
+
+Cluster Maps
+------------
+
+The cluster map is a composite of maps, including the monitor map, the OSD map,
+the placement group map and the metadata server map. The cluster map tracks a
+number of important things: which processes are ``in`` the Ceph Storage Cluster;
+which processes that are ``in`` the Ceph Storage Cluster are ``up`` and running
+or ``down``; whether, the placement groups are ``active`` or ``inactive``, and
+``clean`` or in some other state; and, other details that reflect the current
+state of the cluster such as the total amount of storage space, and the amount
+of storage used.
+
+When there is a significant change in the state of the cluster--e.g., a Ceph OSD
+Daemon goes down, a placement group falls into a degraded state, etc.--the
+cluster map gets updated to reflect the current state of the cluster.
+Additionally, the Ceph Monitor also maintains a history of the prior states of
+the cluster. The monitor map, OSD map, placement group map and metadata server
+map each maintain a history of their map versions. We call each version an
+"epoch."
+
+When operating your Ceph Storage Cluster, keeping track of these states is an
+important part of your system administration duties. See `Monitoring a Cluster`_
+and `Monitoring OSDs and PGs`_ for additional details.
+
+.. index:: high availability; quorum
+
+Monitor Quorum
+--------------
+
+Our Configuring ceph section provides a trivial `Ceph configuration file`_ that
+provides for one monitor in the test cluster. A cluster will run fine with a
+single monitor; however, **a single monitor is a single-point-of-failure**. To
+ensure high availability in a production Ceph Storage Cluster, you should run
+Ceph with multiple monitors so that the failure of a single monitor **WILL NOT**
+bring down your entire cluster.
+
+When a Ceph Storage Cluster runs multiple Ceph Monitors for high availability,
+Ceph Monitors use `Paxos`_ to establish consensus about the master cluster map.
+A consensus requires a majority of monitors running to establish a quorum for
+consensus about the cluster map (e.g., 1; 2 out of 3; 3 out of 5; 4 out of 6;
+etc.).
+
+.. confval:: mon_force_quorum_join
+
+.. index:: Ceph Monitor; consistency
+
+Consistency
+-----------
+
+When you add monitor settings to your Ceph configuration file, you need to be
+aware of some of the architectural aspects of Ceph Monitors. **Ceph imposes
+strict consistency requirements** for a Ceph monitor when discovering another
+Ceph Monitor within the cluster. Whereas, Ceph Clients and other Ceph daemons
+use the Ceph configuration file to discover monitors, monitors discover each
+other using the monitor map (monmap), not the Ceph configuration file.
+
+A Ceph Monitor always refers to the local copy of the monmap when discovering
+other Ceph Monitors in the Ceph Storage Cluster. Using the monmap instead of the
+Ceph configuration file avoids errors that could break the cluster (e.g., typos
+in ``ceph.conf`` when specifying a monitor address or port). Since monitors use
+monmaps for discovery and they share monmaps with clients and other Ceph
+daemons, **the monmap provides monitors with a strict guarantee that their
+consensus is valid.**
+
+Strict consistency also applies to updates to the monmap. As with any other
+updates on the Ceph Monitor, changes to the monmap always run through a
+distributed consensus algorithm called `Paxos`_. The Ceph Monitors must agree on
+each update to the monmap, such as adding or removing a Ceph Monitor, to ensure
+that each monitor in the quorum has the same version of the monmap. Updates to
+the monmap are incremental so that Ceph Monitors have the latest agreed upon
+version, and a set of previous versions. Maintaining a history enables a Ceph
+Monitor that has an older version of the monmap to catch up with the current
+state of the Ceph Storage Cluster.
+
+If Ceph Monitors were to discover each other through the Ceph configuration file
+instead of through the monmap, additional risks would be introduced because
+Ceph configuration files are not updated and distributed automatically. Ceph
+Monitors might inadvertently use an older Ceph configuration file, fail to
+recognize a Ceph Monitor, fall out of a quorum, or develop a situation where
+`Paxos`_ is not able to determine the current state of the system accurately.
+
+
+.. index:: Ceph Monitor; bootstrapping monitors
+
+Bootstrapping Monitors
+----------------------
+
+In most configuration and deployment cases, tools that deploy Ceph help
+bootstrap the Ceph Monitors by generating a monitor map for you (e.g.,
+``cephadm``, etc). A Ceph Monitor requires a few explicit
+settings:
+
+- **Filesystem ID**: The ``fsid`` is the unique identifier for your
+  object store. Since you can run multiple clusters on the same
+  hardware, you must specify the unique ID of the object store when
+  bootstrapping a monitor.  Deployment tools usually do this for you
+  (e.g., ``cephadm`` can call a tool like ``uuidgen``), but you
+  may specify the ``fsid`` manually too.
+  
+- **Monitor ID**: A monitor ID is a unique ID assigned to each monitor within 
+  the cluster. It is an alphanumeric value, and by convention the identifier 
+  usually follows an alphabetical increment (e.g., ``a``, ``b``, etc.). This 
+  can be set in a Ceph configuration file (e.g., ``[mon.a]``, ``[mon.b]``, etc.), 
+  by a deployment tool, or using the ``ceph`` commandline.
+
+- **Keys**: The monitor must have secret keys. A deployment tool such as 
+  ``cephadm`` usually does this for you, but you may
+  perform this step manually too. See `Monitor Keyrings`_ for details.
+
+For additional details on bootstrapping, see `Bootstrapping a Monitor`_.
+
+.. index:: Ceph Monitor; configuring monitors
+
+Configuring Monitors
+====================
+
+To apply configuration settings to the entire cluster, enter the configuration
+settings under ``[global]``. To apply configuration settings to all monitors in
+your cluster, enter the configuration settings under ``[mon]``. To apply
+configuration settings to specific monitors, specify the monitor instance 
+(e.g., ``[mon.a]``). By convention, monitor instance names use alpha notation.
+
+.. code-block:: ini
+
+	[global]
+
+	[mon]		
+		
+	[mon.a]
+		
+	[mon.b]
+		
+	[mon.c]
+
+
+Minimum Configuration
+---------------------
+
+The bare minimum monitor settings for a Ceph monitor via the Ceph configuration
+file include a hostname and a network address for each monitor. You can configure
+these under ``[mon]`` or under the entry for a specific monitor.
+
+.. code-block:: ini
+
+	[global]
+		mon_host = 10.0.0.2,10.0.0.3,10.0.0.4
+
+.. code-block:: ini
+
+	[mon.a]
+		host = hostname1
+		mon_addr = 10.0.0.10:6789
+
+See the `Network Configuration Reference`_ for details.
+
+.. note:: This minimum configuration for monitors assumes that a deployment 
+   tool generates the ``fsid`` and the ``mon.`` key for you.
+
+Once you deploy a Ceph cluster, you **SHOULD NOT** change the IP addresses of
+monitors. However, if you decide to change the monitor's IP address, you
+must follow a specific procedure. See :ref:`Changing a Monitor's IP address` for
+details.
+
+Monitors can also be found by clients by using DNS SRV records. See `Monitor lookup through DNS`_ for details.
+
+Cluster ID
+----------
+
+Each Ceph Storage Cluster has a unique identifier (``fsid``). If specified, it
+usually appears under the ``[global]`` section of the configuration file.
+Deployment tools usually generate the ``fsid`` and store it in the monitor map,
+so the value may not appear in a configuration file. The ``fsid`` makes it
+possible to run daemons for multiple clusters on the same hardware.
+
+.. confval:: fsid
+
+.. index:: Ceph Monitor; initial members
+
+Initial Members
+---------------
+
+We recommend running a production Ceph Storage Cluster with at least three Ceph
+Monitors to ensure high availability. When you run multiple monitors, you may
+specify the initial monitors that must be members of the cluster in order to
+establish a quorum. This may reduce the time it takes for your cluster to come
+online.
+
+.. code-block:: ini
+
+	[mon]		
+		mon_initial_members = a,b,c
+
+
+.. confval:: mon_initial_members
+
+.. index:: Ceph Monitor; data path
+
+Data
+----
+
+Ceph provides a default path where Ceph Monitors store data. For optimal
+performance in a production Ceph Storage Cluster, we recommend running Ceph
+Monitors on separate hosts and drives from Ceph OSD Daemons. As leveldb uses
+``mmap()`` for writing the data, Ceph Monitors flush their data from memory to disk
+very often, which can interfere with Ceph OSD Daemon workloads if the data
+store is co-located with the OSD Daemons.
+
+In Ceph versions 0.58 and earlier, Ceph Monitors store their data in plain files. This 
+approach allows users to inspect monitor data with common tools like ``ls``
+and ``cat``. However, this approach didn't provide strong consistency.
+
+In Ceph versions 0.59 and later, Ceph Monitors store their data as key/value
+pairs. Ceph Monitors require `ACID`_ transactions. Using a data store prevents
+recovering Ceph Monitors from running corrupted versions through Paxos, and it
+enables multiple modification operations in one single atomic batch, among other
+advantages.
+
+Generally, we do not recommend changing the default data location. If you modify
+the default location, we recommend that you make it uniform across Ceph Monitors
+by setting it in the ``[mon]`` section of the configuration file.
+
+
+.. confval:: mon_data
+.. confval:: mon_data_size_warn
+.. confval:: mon_data_avail_warn
+.. confval:: mon_data_avail_crit
+.. confval:: mon_warn_on_crush_straw_calc_version_zero
+.. confval:: mon_warn_on_legacy_crush_tunables
+.. confval:: mon_crush_min_required_version
+.. confval:: mon_warn_on_osd_down_out_interval_zero
+.. confval:: mon_warn_on_slow_ping_ratio
+.. confval:: mon_warn_on_slow_ping_time
+.. confval:: mon_warn_on_pool_no_redundancy
+.. confval:: mon_cache_target_full_warn_ratio
+.. confval:: mon_health_to_clog
+.. confval:: mon_health_to_clog_tick_interval
+.. confval:: mon_health_to_clog_interval
+
+.. index:: Ceph Storage Cluster; capacity planning, Ceph Monitor; capacity planning
+
+.. _storage-capacity:
+
+Storage Capacity
+----------------
+
+When a Ceph Storage Cluster gets close to its maximum capacity
+(see``mon_osd_full ratio``), Ceph prevents you from writing to or reading from OSDs
+as a safety measure to prevent data loss. Therefore, letting a
+production Ceph Storage Cluster approach its full ratio is not a good practice,
+because it sacrifices high availability. The default full ratio is ``.95``, or
+95% of capacity. This a very aggressive setting for a test cluster with a small
+number of OSDs.
+
+.. tip:: When monitoring your cluster, be alert to warnings related to the 
+   ``nearfull`` ratio. This means that a failure of some OSDs could result
+   in a temporary service disruption if one or more OSDs fails. Consider adding
+   more OSDs to increase storage capacity.
+
+A common scenario for test clusters involves a system administrator removing an
+OSD from the Ceph Storage Cluster, watching the cluster rebalance, then removing
+another OSD, and another, until at least one OSD eventually reaches the full
+ratio and the cluster locks up. We recommend a bit of capacity
+planning even with a test cluster. Planning enables you to gauge how much spare
+capacity you will need in order to maintain high availability. Ideally, you want
+to plan for a series of Ceph OSD Daemon failures where the cluster can recover
+to an ``active+clean`` state without replacing those OSDs
+immediately. Cluster operation continues in the ``active+degraded`` state, but this
+is not ideal for normal operation and should be addressed promptly.
+
+The following diagram depicts a simplistic Ceph Storage Cluster containing 33
+Ceph Nodes with one OSD per host, each OSD reading from
+and writing to a 3TB drive. So this exemplary Ceph Storage Cluster has a maximum
+actual capacity of 99TB. With a ``mon osd full ratio`` of ``0.95``, if the Ceph
+Storage Cluster falls to 5TB of remaining capacity, the cluster will not allow
+Ceph Clients to read and write data. So the Ceph Storage Cluster's operating
+capacity is 95TB, not 99TB.
+
+.. ditaa::
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | Rack 1 |  | Rack 2 |  | Rack 3 |  | Rack 4 |  | Rack 5 |  | Rack 6 |
+ | cCCC   |  | cF00   |  | cCCC   |  | cCCC   |  | cCCC   |  | cCCC   |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 1  |  | OSD 7  |  | OSD 13 |  | OSD 19 |  | OSD 25 |  | OSD 31 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 2  |  | OSD 8  |  | OSD 14 |  | OSD 20 |  | OSD 26 |  | OSD 32 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 3  |  | OSD 9  |  | OSD 15 |  | OSD 21 |  | OSD 27 |  | OSD 33 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 4  |  | OSD 10 |  | OSD 16 |  | OSD 22 |  | OSD 28 |  | Spare  | 
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 5  |  | OSD 11 |  | OSD 17 |  | OSD 23 |  | OSD 29 |  | Spare  |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 6  |  | OSD 12 |  | OSD 18 |  | OSD 24 |  | OSD 30 |  | Spare  |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+
+It is normal in such a cluster for one or two OSDs to fail. A less frequent but
+reasonable scenario involves a rack's router or power supply failing, which
+brings down multiple OSDs simultaneously (e.g., OSDs 7-12). In such a scenario,
+you should still strive for a cluster that can remain operational and achieve an
+``active + clean`` state--even if that means adding a few hosts with additional
+OSDs in short order. If your capacity utilization is too high, you may not lose
+data, but you could still sacrifice data availability while resolving an outage
+within a failure domain if capacity utilization of the cluster exceeds the full
+ratio. For this reason, we recommend at least some rough capacity planning.
+
+Identify two numbers for your cluster:
+
+#. The number of OSDs. 
+#. The total capacity of the cluster 
+
+If you divide the total capacity of your cluster by the number of OSDs in your
+cluster, you will find the mean average capacity of an OSD within your cluster.
+Consider multiplying that number by the number of OSDs you expect will fail
+simultaneously during normal operations (a relatively small number). Finally
+multiply the capacity of the cluster by the full ratio to arrive at a maximum
+operating capacity; then, subtract the number of amount of data from the OSDs
+you expect to fail to arrive at a reasonable full ratio. Repeat the foregoing
+process with a higher number of OSD failures (e.g., a rack of OSDs) to arrive at
+a reasonable number for a near full ratio.
+
+The following settings only apply on cluster creation and are then stored in
+the OSDMap. To clarify, in normal operation the values that are used by OSDs
+are those found in the OSDMap, not those in the configuration file or central
+config store.
+
+.. code-block:: ini
+
+	[global]
+		mon_osd_full_ratio = .80
+		mon_osd_backfillfull_ratio = .75
+		mon_osd_nearfull_ratio = .70
+
+
+``mon_osd_full_ratio`` 
+
+:Description: The threshold percentage of device space utilized before an OSD is 
+              considered ``full``.
+
+:Type: Float
+:Default: ``0.95``
+
+
+``mon_osd_backfillfull_ratio``
+
+:Description: The threshold percentage of device space utilized before an OSD is
+              considered too ``full`` to backfill.
+
+:Type: Float
+:Default: ``0.90``
+
+
+``mon_osd_nearfull_ratio`` 
+
+:Description: The threshold percentage of device space used before an OSD is 
+              considered ``nearfull``.
+
+:Type: Float
+:Default: ``0.85``
+
+
+.. tip:: If some OSDs are nearfull, but others have plenty of capacity, you 
+         may have an inaccurate CRUSH weight set for the nearfull OSDs.
+
+.. tip:: These settings only apply during cluster creation. Afterwards they need
+         to be changed in the OSDMap using ``ceph osd set-nearfull-ratio`` and
+         ``ceph osd set-full-ratio``
+
+.. index:: heartbeat
+
+Heartbeat
+---------
+
+Ceph monitors know about the cluster by requiring reports from each OSD, and by
+receiving reports from OSDs about the status of their neighboring OSDs. Ceph
+provides reasonable default settings for monitor/OSD interaction; however,  you
+may modify them as needed. See `Monitor/OSD Interaction`_ for details.
+
+
+.. index:: Ceph Monitor; leader, Ceph Monitor; provider, Ceph Monitor; requester, Ceph Monitor; synchronization
+
+Monitor Store Synchronization
+-----------------------------
+
+When you run a production cluster with multiple monitors (recommended), each
+monitor checks to see if a neighboring monitor has a more recent version of the
+cluster map (e.g., a map in a neighboring monitor with one or more epoch numbers
+higher than the most current epoch in the map of the instant monitor).
+Periodically, one monitor in the cluster may fall behind the other monitors to
+the point where it must leave the quorum, synchronize to retrieve the most
+current information about the cluster, and then rejoin the quorum. For the
+purposes of synchronization, monitors may assume one of three roles: 
+
+#. **Leader**: The `Leader` is the first monitor to achieve the most recent
+   Paxos version of the cluster map.
+
+#. **Provider**: The `Provider` is a monitor that has the most recent version
+   of the cluster map, but wasn't the first to achieve the most recent version.
+
+#. **Requester:** A `Requester` is a monitor that has fallen behind the leader
+   and must synchronize in order to retrieve the most recent information about
+   the cluster before it can rejoin the quorum.
+
+These roles enable a leader to delegate synchronization duties to a provider,
+which prevents synchronization requests from overloading the leader--improving
+performance. In the following diagram, the requester has learned that it has
+fallen behind the other monitors. The requester asks the leader to synchronize,
+and the leader tells the requester to synchronize with a provider.
+
+
+.. ditaa::
+           +-----------+          +---------+          +----------+
+           | Requester |          | Leader  |          | Provider |
+           +-----------+          +---------+          +----------+
+                  |                    |                     |
+                  |                    |                     |
+                  | Ask to Synchronize |                     |
+                  |------------------->|                     |
+                  |                    |                     |
+                  |<-------------------|                     |
+                  | Tell Requester to  |                     |
+                  | Sync with Provider |                     |
+                  |                    |                     |
+                  |               Synchronize                |
+                  |--------------------+-------------------->|
+                  |                    |                     |
+                  |<-------------------+---------------------|
+                  |        Send Chunk to Requester           |
+                  |         (repeat as necessary)            |
+                  |    Requester Acks Chuck to Provider      |
+                  |--------------------+-------------------->|
+                  |                    |
+                  |   Sync Complete    |
+                  |    Notification    |
+                  |------------------->|
+                  |                    |
+                  |<-------------------|
+                  |        Ack         |
+                  |                    |
+
+
+Synchronization always occurs when a new monitor joins the cluster. During
+runtime operations, monitors may receive updates to the cluster map at different
+times. This means the leader and provider roles may migrate from one monitor to
+another. If this happens while synchronizing (e.g., a provider falls behind the
+leader), the provider can terminate synchronization with a requester.
+
+Once synchronization is complete, Ceph performs trimming across the cluster. 
+Trimming requires that the placement groups are ``active+clean``.
+
+
+.. confval:: mon_sync_timeout
+.. confval:: mon_sync_max_payload_size
+.. confval:: paxos_max_join_drift
+.. confval:: paxos_stash_full_interval
+.. confval:: paxos_propose_interval
+.. confval:: paxos_min
+.. confval:: paxos_min_wait
+.. confval:: paxos_trim_min
+.. confval:: paxos_trim_max
+.. confval:: paxos_service_trim_min
+.. confval:: paxos_service_trim_max
+.. confval:: paxos_service_trim_max_multiplier
+.. confval:: mon_mds_force_trim_to
+.. confval:: mon_osd_force_trim_to
+.. confval:: mon_osd_cache_size
+.. confval:: mon_election_timeout
+.. confval:: mon_lease
+.. confval:: mon_lease_renew_interval_factor
+.. confval:: mon_lease_ack_timeout_factor
+.. confval:: mon_accept_timeout_factor
+.. confval:: mon_min_osdmap_epochs
+.. confval:: mon_max_log_epochs
+
+
+.. index:: Ceph Monitor; clock
+
+.. _mon-config-ref-clock:
+
+Clock
+-----
+
+Ceph daemons pass critical messages to each other, which must be processed
+before daemons reach a timeout threshold. If the clocks in Ceph monitors
+are not synchronized, it can lead to a number of anomalies. For example:
+
+- Daemons ignoring received messages (e.g., timestamps outdated)
+- Timeouts triggered too soon/late when a message wasn't received in time.
+
+See `Monitor Store Synchronization`_ for details.
+
+
+.. tip:: You must configure NTP or PTP daemons on your Ceph monitor hosts to 
+         ensure that the monitor cluster operates with synchronized clocks.
+         It can be advantageous to have monitor hosts sync with each other
+         as well as with multiple quality upstream time sources.
+
+Clock drift may still be noticeable with NTP even though the discrepancy is not
+yet harmful. Ceph's clock drift / clock skew warnings may get triggered even 
+though NTP maintains a reasonable level of synchronization. Increasing your 
+clock drift may be tolerable under such circumstances; however, a number of 
+factors such as workload, network latency, configuring overrides to default 
+timeouts and the `Monitor Store Synchronization`_ settings may influence 
+the level of acceptable clock drift without compromising Paxos guarantees.
+
+Ceph provides the following tunable options to allow you to find 
+acceptable values.
+
+.. confval:: mon_tick_interval
+.. confval:: mon_clock_drift_allowed
+.. confval:: mon_clock_drift_warn_backoff
+.. confval:: mon_timecheck_interval
+.. confval:: mon_timecheck_skew_interval
+
+Client
+------
+
+.. confval:: mon_client_hunt_interval
+.. confval:: mon_client_ping_interval
+.. confval:: mon_client_max_log_entries_per_message
+.. confval:: mon_client_bytes
+
+.. _pool-settings:
+
+Pool settings
+=============
+
+Since version v0.94 there is support for pool flags which allow or disallow changes to be made to pools.
+Monitors can also disallow removal of pools if appropriately configured. The inconvenience of this guardrail
+is far outweighed by the number of accidental pool (and thus data) deletions it prevents.
+
+.. confval:: mon_allow_pool_delete
+.. confval:: osd_pool_default_ec_fast_read
+.. confval:: osd_pool_default_flag_hashpspool
+.. confval:: osd_pool_default_flag_nodelete
+.. confval:: osd_pool_default_flag_nopgchange
+.. confval:: osd_pool_default_flag_nosizechange
+
+For more information about the pool flags see :ref:`Pool values <setpoolvalues>`.
+
+Miscellaneous
+=============
+
+.. confval:: mon_max_osd
+.. confval:: mon_globalid_prealloc
+.. confval:: mon_subscribe_interval
+.. confval:: mon_stat_smooth_intervals
+.. confval:: mon_probe_timeout
+.. confval:: mon_daemon_bytes
+.. confval:: mon_max_log_entries_per_event
+.. confval:: mon_osd_prime_pg_temp
+.. confval:: mon_osd_prime_pg_temp_max_time
+.. confval:: mon_osd_prime_pg_temp_max_estimate
+.. confval:: mon_mds_skip_sanity
+.. confval:: mon_max_mdsmap_epochs
+.. confval:: mon_config_key_max_entry_size
+.. confval:: mon_scrub_interval
+.. confval:: mon_scrub_max_keys
+.. confval:: mon_compact_on_start
+.. confval:: mon_compact_on_bootstrap
+.. confval:: mon_compact_on_trim
+.. confval:: mon_cpu_threads
+.. confval:: mon_osd_mapping_pgs_per_chunk
+.. confval:: mon_session_timeout
+.. confval:: mon_osd_cache_size_min
+.. confval:: mon_memory_target
+.. confval:: mon_memory_autotune
+
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
+.. _Monitor Keyrings: ../../../dev/mon-bootstrap#secret-keys
+.. _Ceph configuration file: ../ceph-conf/#monitors
+.. _Network Configuration Reference: ../network-config-ref
+.. _Monitor lookup through DNS: ../mon-lookup-dns
+.. _ACID: https://en.wikipedia.org/wiki/ACID
+.. _Adding/Removing a Monitor: ../../operations/add-or-rm-mons
+.. _Monitoring a Cluster: ../../operations/monitoring
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg
+.. _Bootstrapping a Monitor: ../../../dev/mon-bootstrap
+.. _Monitor/OSD Interaction: ../mon-osd-interaction
+.. _Scalability and High Availability: ../../../architecture#scalability-and-high-availability
diff --git a/doc/rados/configuration/mon-lookup-dns.rst b/doc/rados/configuration/mon-lookup-dns.rst
new file mode 100644
index 000000000..129a083c4
--- /dev/null
+++ b/doc/rados/configuration/mon-lookup-dns.rst
@@ -0,0 +1,58 @@
+.. _mon-dns-lookup:
+
+===============================
+Looking up Monitors through DNS
+===============================
+
+Since Ceph version 11.0.0 (Kraken), RADOS has supported looking up monitors
+through DNS.
+
+The addition of the ability to look up monitors through DNS means that daemons
+and clients do not require a *mon host* configuration directive in their
+``ceph.conf`` configuration file.
+
+With a DNS update, clients and daemons can be made aware of changes
+in the monitor topology. To be more precise and technical, clients look up the
+monitors by using ``DNS SRV TCP`` records. 
+
+By default, clients and daemons look for the TCP service called *ceph-mon*,
+which is configured by the *mon_dns_srv_name* configuration directive.
+
+
+.. confval:: mon_dns_srv_name
+
+Example
+-------
+When the DNS search domain is set to *example.com* a DNS zone file might contain the following elements.
+
+First, create records for the Monitors, either IPv4 (A) or IPv6 (AAAA).
+
+::
+
+    mon1.example.com. AAAA 2001:db8::100
+    mon2.example.com. AAAA 2001:db8::200
+    mon3.example.com. AAAA 2001:db8::300
+
+::
+
+    mon1.example.com. A 192.168.0.1
+    mon2.example.com. A 192.168.0.2
+    mon3.example.com. A 192.168.0.3
+
+
+With those records now existing we can create the SRV TCP records with the name *ceph-mon* pointing to the three Monitors.
+
+::
+
+    _ceph-mon._tcp.example.com. 60 IN SRV 10 20 6789 mon1.example.com.
+    _ceph-mon._tcp.example.com. 60 IN SRV 10 30 6789 mon2.example.com.
+    _ceph-mon._tcp.example.com. 60 IN SRV 20 50 6789 mon3.example.com.
+
+Now all Monitors are running on port *6789*, with priorities 10, 10, 20 and weights 20, 30, 50 respectively.
+
+Monitor clients choose monitor by referencing the SRV records. If a cluster has multiple Monitor SRV records
+with the same priority value, clients and daemons will load balance the connections to Monitors in proportion
+to the values of the SRV weight fields.
+
+For the above example, this will result in approximate 40% of the clients and daemons connecting to mon1,
+60% of them connecting to mon2. However, if neither of them is reachable, then mon3 will be reconsidered as a fallback.
diff --git a/doc/rados/configuration/mon-osd-interaction.rst b/doc/rados/configuration/mon-osd-interaction.rst
new file mode 100644
index 000000000..8cf09707d
--- /dev/null
+++ b/doc/rados/configuration/mon-osd-interaction.rst
@@ -0,0 +1,245 @@
+=====================================
+ Configuring Monitor/OSD Interaction
+=====================================
+
+.. index:: heartbeat
+
+After you have completed your initial Ceph configuration, you may deploy and run
+Ceph.  When you execute a command such as ``ceph health`` or ``ceph -s``,  the
+:term:`Ceph Monitor` reports on the current state of the :term:`Ceph Storage
+Cluster`. The Ceph Monitor knows about the Ceph Storage Cluster by requiring
+reports from each :term:`Ceph OSD Daemon`, and by receiving reports from Ceph
+OSD Daemons about the status of their neighboring Ceph OSD Daemons. If the Ceph
+Monitor doesn't receive reports, or if it receives reports of changes in the
+Ceph Storage Cluster, the Ceph Monitor updates the status of the :term:`Ceph
+Cluster Map`.
+
+Ceph provides reasonable default settings for Ceph Monitor/Ceph OSD Daemon
+interaction. However, you may override the defaults. The following sections
+describe how Ceph Monitors and Ceph OSD Daemons interact for the purposes of
+monitoring the Ceph Storage Cluster.
+
+.. index:: heartbeat interval
+
+OSDs Check Heartbeats
+=====================
+
+Each Ceph OSD Daemon checks the heartbeat of other Ceph OSD Daemons at random
+intervals less than every 6 seconds.  If a neighboring Ceph OSD Daemon doesn't
+show a heartbeat within a 20 second grace period, the Ceph OSD Daemon may
+consider the neighboring Ceph OSD Daemon ``down`` and report it back to a Ceph
+Monitor, which will update the Ceph Cluster Map. You may change this grace
+period by adding an ``osd heartbeat grace`` setting under the ``[mon]``
+and ``[osd]`` or ``[global]`` section of your Ceph configuration file,
+or by setting the value at runtime.
+
+
+.. ditaa::
+           +---------+          +---------+
+           |  OSD 1  |          |  OSD 2  |
+           +---------+          +---------+
+                |                    |
+                |----+ Heartbeat     |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |       Check        |
+                |     Heartbeat      |
+                |------------------->|
+                |                    |
+                |<-------------------|
+                |   Heart Beating    |
+                |                    |
+                |----+ Heartbeat     |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |       Check        |
+                |     Heartbeat      |
+                |------------------->|
+                |                    |
+                |----+ Grace         |
+                |    | Period        |
+                |<---+ Exceeded      |
+                |                    |
+                |----+ Mark          |
+                |    | OSD 2         |
+                |<---+ Down          |
+
+
+.. index:: OSD down report
+
+OSDs Report Down OSDs
+=====================
+
+By default, two Ceph OSD Daemons from different hosts must report to the Ceph
+Monitors that another Ceph OSD Daemon is ``down`` before the Ceph Monitors
+acknowledge that the reported Ceph OSD Daemon is ``down``. But there is chance
+that all the OSDs reporting the failure are hosted in a rack with a bad switch
+which has trouble connecting to another OSD. To avoid this sort of false alarm,
+we consider the peers reporting a failure a proxy for a potential "subcluster"
+over the overall cluster that is similarly laggy. This is clearly not true in
+all cases, but will sometimes help us localize the grace correction to a subset
+of the system that is unhappy. ``mon osd reporter subtree level`` is used to
+group the peers into the "subcluster" by their common ancestor type in CRUSH
+map. By default, only two reports from different subtree are required to report
+another Ceph OSD Daemon ``down``. You can change the number of reporters from
+unique subtrees and the common ancestor type required to report a Ceph OSD
+Daemon ``down`` to a Ceph Monitor by adding an ``mon osd min down reporters``
+and ``mon osd reporter subtree level`` settings  under the ``[mon]`` section of
+your Ceph configuration file, or by setting the value at runtime.
+
+
+.. ditaa::
+
+           +---------+     +---------+      +---------+
+           |  OSD 1  |     |  OSD 2  |      | Monitor |
+           +---------+     +---------+      +---------+
+                |               |                |
+                | OSD 3 Is Down |                |
+                |---------------+--------------->|
+                |               |                |
+                |               |                |
+                |               | OSD 3 Is Down  |
+                |               |--------------->|
+                |               |                |
+                |               |                |
+                |               |                |---------+ Mark
+                |               |                |         | OSD 3
+                |               |                |<--------+ Down
+
+
+.. index:: peering failure
+
+OSDs Report Peering Failure
+===========================
+
+If a Ceph OSD Daemon cannot peer with any of the Ceph OSD Daemons defined in its
+Ceph configuration file (or the cluster map), it will ping a Ceph Monitor for
+the most recent copy of the cluster map every 30 seconds. You can change the
+Ceph Monitor heartbeat interval by adding an ``osd mon heartbeat interval``
+setting under the ``[osd]`` section of your Ceph configuration file, or by
+setting the value at runtime.
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+     +---------+
+           |  OSD 1  |     |  OSD 2  |     | OSD 3 |     | Monitor |
+           +---------+     +---------+     +-------+     +---------+
+                |               |              |              |
+                |  Request To   |              |              |
+                |     Peer      |              |              |
+                |-------------->|              |              |
+                |<--------------|              |              |
+                |    Peering                   |              |
+                |                              |              |
+                |  Request To                  |              |
+                |     Peer                     |              |
+                |----------------------------->|              |
+                |                                             |
+                |----+ OSD Monitor                            |
+                |    | Heartbeat                              |
+                |<---+ Interval Exceeded                      |
+                |                                             |
+                |         Failed to Peer with OSD 3           |
+                |-------------------------------------------->|
+                |<--------------------------------------------|
+                |          Receive New Cluster Map            |
+
+
+.. index:: OSD status
+
+OSDs Report Their Status
+========================
+
+If an Ceph OSD Daemon doesn't report to a Ceph Monitor, the Ceph Monitor will
+consider the Ceph OSD Daemon ``down`` after the  ``mon osd report timeout``
+elapses. A Ceph OSD Daemon sends a report to a Ceph Monitor when a reportable
+event such as a failure, a change in placement group stats, a change in
+``up_thru`` or when it boots within 5 seconds. You can change the Ceph OSD
+Daemon minimum report interval by adding an ``osd mon report interval``
+setting under the ``[osd]`` section of your Ceph configuration file, or by
+setting the value at runtime. A Ceph OSD Daemon sends a report to a Ceph
+Monitor every 120 seconds irrespective of whether any notable changes occur.
+You can change the Ceph Monitor report interval by adding an ``osd mon report
+interval max`` setting under the ``[osd]`` section of your Ceph configuration
+file, or by setting the value at runtime.
+
+
+.. ditaa::
+
+           +---------+          +---------+
+           |  OSD 1  |          | Monitor |
+           +---------+          +---------+
+                |                    |
+                |----+ Report Min    |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |----+ Reportable    |
+                |    | Event         |
+                |<---+ Occurs        |
+                |                    |
+                |     Report To      |
+                |      Monitor       |
+                |------------------->|
+                |                    |
+                |----+ Report Max    |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |     Report To      |
+                |      Monitor       |
+                |------------------->|
+                |                    |
+                |----+ Monitor       |
+                |    | Fails         |
+                |<---+               |
+                                     +----+ Monitor OSD
+                                     |    | Report Timeout
+                                     |<---+ Exceeded
+                                     |
+                                     +----+ Mark
+                                     |    | OSD 1
+                                     |<---+ Down
+
+
+
+
+Configuration Settings
+======================
+
+When modifying heartbeat settings, you should include them in the ``[global]``
+section of your configuration file.
+
+.. index:: monitor heartbeat
+
+Monitor Settings
+----------------
+
+.. confval:: mon_osd_min_up_ratio
+.. confval:: mon_osd_min_in_ratio
+.. confval:: mon_osd_laggy_halflife
+.. confval:: mon_osd_laggy_weight
+.. confval:: mon_osd_laggy_max_interval
+.. confval:: mon_osd_adjust_heartbeat_grace
+.. confval:: mon_osd_adjust_down_out_interval
+.. confval:: mon_osd_auto_mark_in
+.. confval:: mon_osd_auto_mark_auto_out_in
+.. confval:: mon_osd_auto_mark_new_in
+.. confval:: mon_osd_down_out_interval
+.. confval:: mon_osd_down_out_subtree_limit
+.. confval:: mon_osd_report_timeout
+.. confval:: mon_osd_min_down_reporters
+.. confval:: mon_osd_reporter_subtree_level
+
+.. index:: OSD heartbeat
+
+OSD Settings
+------------
+
+.. confval:: osd_heartbeat_interval
+.. confval:: osd_heartbeat_grace
+.. confval:: osd_mon_heartbeat_interval
+.. confval:: osd_mon_heartbeat_stat_stale
+.. confval:: osd_mon_report_interval
diff --git a/doc/rados/configuration/msgr2.rst b/doc/rados/configuration/msgr2.rst
new file mode 100644
index 000000000..33fe4e022
--- /dev/null
+++ b/doc/rados/configuration/msgr2.rst
@@ -0,0 +1,257 @@
+.. _msgr2:
+
+Messenger v2
+============
+
+What is it
+----------
+
+The messenger v2 protocol, or msgr2, is the second major revision on
+Ceph's on-wire protocol.  It brings with it several key features:
+
+* A *secure* mode that encrypts all data passing over the network
+* Improved encapsulation of authentication payloads, enabling future
+  integration of new authentication modes like Kerberos
+* Improved earlier feature advertisement and negotiation, enabling
+  future protocol revisions
+
+Ceph daemons can now bind to multiple ports, allowing both legacy Ceph
+clients and new v2-capable clients to connect to the same cluster.
+
+By default, monitors now bind to the new IANA-assigned port ``3300``
+(ce4h or 0xce4) for the new v2 protocol, while also binding to the
+old default port ``6789`` for the legacy v1 protocol.
+
+.. _address_formats:
+
+Address formats
+---------------
+
+Prior to Nautilus, all network addresses were rendered like
+``1.2.3.4:567/89012`` where there was an IP address, a port, and a
+nonce to uniquely identify a client or daemon on the network.
+Starting with Nautilus, we now have three different address types:
+
+* **v2**: ``v2:1.2.3.4:578/89012`` identifies a daemon binding to a
+  port speaking the new v2 protocol
+* **v1**: ``v1:1.2.3.4:578/89012`` identifies a daemon binding to a
+  port speaking the legacy v1 protocol.  Any address that was
+  previously shown with any prefix is now shown as a ``v1:`` address.
+* **TYPE_ANY** ``any:1.2.3.4:578/89012`` identifies a client that can
+  speak either version of the protocol. Prior to nautilus, clients would appear as
+  ``1.2.3.4:0/123456``, where the port of 0 indicates they are clients
+  and do not accept incoming connections.  Starting with Nautilus,
+  these clients are now internally represented by a **TYPE_ANY**
+  address, and still shown with no prefix, because they may
+  connect to daemons using the v2 or v1 protocol, depending on what
+  protocol(s) the daemons are using.
+
+Because daemons now bind to multiple ports, they are now described by
+a vector of addresses instead of a single address.  For example,
+dumping the monitor map on a Nautilus cluster now includes lines
+like::
+
+  epoch 1
+  fsid 50fcf227-be32-4bcb-8b41-34ca8370bd16
+  last_changed 2019-02-25 11:10:46.700821
+  created 2019-02-25 11:10:46.700821
+  min_mon_release 14 (nautilus)
+  0: [v2:10.0.0.10:3300/0,v1:10.0.0.10:6789/0] mon.foo
+  1: [v2:10.0.0.11:3300/0,v1:10.0.0.11:6789/0] mon.bar
+  2: [v2:10.0.0.12:3300/0,v1:10.0.0.12:6789/0] mon.baz
+
+The bracketed list or vector of addresses means that the same daemon can be
+reached on multiple ports (and protocols).  Any client or other daemon
+connecting to that daemon will use the v2 protocol (listed first) if
+possible; otherwise it will back to the legacy v1 protocol.  Legacy
+clients will only see the v1 addresses and will continue to connect as
+they did before, with the v1 protocol.
+
+Starting in Nautilus, the ``mon_host`` configuration option and ``-m
+<mon-host>`` command line options support the same bracketed address
+vector syntax.
+
+
+Bind configuration options
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Two new configuration options control whether the v1 and/or v2
+protocol is used:
+
+  * :confval:`ms_bind_msgr1` [default: true] controls whether a daemon binds
+    to a port speaking the v1 protocol
+  * :confval:`ms_bind_msgr2` [default: true] controls whether a daemon binds
+    to a port speaking the v2 protocol
+
+Similarly, two options control whether IPv4 and IPv6 addresses are used:
+
+  * :confval:`ms_bind_ipv4` [default: true] controls whether a daemon binds
+    to an IPv4 address
+  * :confval:`ms_bind_ipv6` [default: false] controls whether a daemon binds
+    to an IPv6 address
+
+.. note:: The ability to bind to multiple ports has paved the way for
+   dual-stack IPv4 and IPv6 support.  That said, dual-stack operation is
+   not yet supported as of Quincy v17.2.0.
+
+Connection modes
+----------------
+
+The v2 protocol supports two connection modes:
+
+* *crc* mode provides:
+
+  - a strong initial authentication when the connection is established
+    (with cephx, mutual authentication of both parties with protection
+    from a man-in-the-middle or eavesdropper), and
+  - a crc32c integrity check to protect against bit flips due to flaky
+    hardware or cosmic rays
+
+  *crc* mode does *not* provide:
+
+  - secrecy (an eavesdropper on the network can see all
+    post-authentication traffic as it goes by) or
+  - protection from a malicious man-in-the-middle (who can deliberate
+    modify traffic as it goes by, as long as they are careful to
+    adjust the crc32c values to match)
+
+* *secure* mode provides:
+
+  - a strong initial authentication when the connection is established
+    (with cephx, mutual authentication of both parties with protection
+    from a man-in-the-middle or eavesdropper), and
+  - full encryption of all post-authentication traffic, including a
+    cryptographic integrity check.
+
+  In Nautilus, secure mode uses the `AES-GCM
+  <https://en.wikipedia.org/wiki/Galois/Counter_Mode>`_ stream cipher,
+  which is generally very fast on modern processors (e.g., faster than
+  a SHA-256 cryptographic hash).
+
+Connection mode configuration options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For most connections, there are options that control which modes are used:
+
+.. confval:: ms_cluster_mode
+.. confval:: ms_service_mode
+.. confval:: ms_client_mode
+
+There are a parallel set of options that apply specifically to
+monitors, allowing administrators to set different (usually more
+secure) requirements on communication with the monitors.
+
+.. confval:: ms_mon_cluster_mode
+.. confval:: ms_mon_service_mode
+.. confval:: ms_mon_client_mode
+
+
+Compression modes
+-----------------
+
+The v2 protocol supports two compression modes:
+
+* *force* mode provides:
+
+  - In multi-availability zones deployment, compressing replication messages between OSDs saves latency.
+  - In the public cloud, inter-AZ communications are expensive. Thus, minimizing message 
+    size reduces network costs to cloud provider.
+  - When using instance storage on AWS (probably other public clouds as well) the instances with NVMe
+    provide low network bandwidth relative to the device bandwidth. 
+    In this case, NW compression can improve the overall performance since this is clearly 
+    the bottleneck.
+
+* *none* mode provides:
+
+  - messages are transmitted without compression. 
+
+
+Compression mode configuration options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For all connections, there is an option that controls compression usage in secure mode
+
+.. confval:: ms_compress_secure
+
+There is a parallel set of options that apply specifically to OSDs, 
+allowing administrators to set different requirements on communication between OSDs.
+
+.. confval:: ms_osd_compress_mode
+.. confval:: ms_osd_compress_min_size
+.. confval:: ms_osd_compression_algorithm
+
+Transitioning from v1-only to v2-plus-v1
+----------------------------------------
+
+By default, ``ms_bind_msgr2`` is true starting with Nautilus 14.2.z.
+However, until the monitors start using v2, only limited services will
+start advertising v2 addresses.
+
+For most users, the monitors are binding to the default legacy port ``6789``
+for the v1 protocol.  When this is the case, enabling v2 is as simple as:
+
+.. prompt:: bash $
+
+   ceph mon enable-msgr2
+
+If the monitors are bound to non-standard ports, you will need to
+specify an additional port for v2 explicitly.  For example, if your
+monitor ``mon.a`` binds to ``1.2.3.4:1111``, and you want to add v2 on
+port ``1112``:
+
+.. prompt:: bash $
+
+   ceph mon set-addrs a [v2:1.2.3.4:1112,v1:1.2.3.4:1111]
+
+Once the monitors bind to v2, each daemon will start advertising a v2
+address when it is next restarted.
+
+
+.. _msgr2_ceph_conf:
+
+Updating ceph.conf and mon_host
+-------------------------------
+
+Prior to Nautilus, a CLI user or daemon will normally discover the
+monitors via the ``mon_host`` option in ``/etc/ceph/ceph.conf``.  The
+syntax for this option has expanded starting with Nautilus to allow
+support the new bracketed list format.  For example, an old line
+like::
+
+  mon_host = 10.0.0.1:6789,10.0.0.2:6789,10.0.0.3:6789
+
+Can be changed to::
+
+  mon_host = [v2:10.0.0.1:3300/0,v1:10.0.0.1:6789/0],[v2:10.0.0.2:3300/0,v1:10.0.0.2:6789/0],[v2:10.0.0.3:3300/0,v1:10.0.0.3:6789/0]
+
+However, when default ports are used (``3300`` and ``6789``), they can
+be omitted::
+
+  mon_host = 10.0.0.1,10.0.0.2,10.0.0.3
+
+Once v2 has been enabled on the monitors, ``ceph.conf`` may need to be
+updated to either specify no ports (this is usually simplest), or
+explicitly specify both the v2 and v1 addresses.  Note, however, that
+the new bracketed syntax is only understood by Nautilus and later, so
+do not make that change on hosts that have not yet had their ceph
+packages upgraded.
+
+When you are updating ``ceph.conf``, note the new ``ceph config
+generate-minimal-conf`` command (which generates a barebones config
+file with just enough information to reach the monitors) and the
+``ceph config assimilate-conf`` (which moves config file options into
+the monitors' configuration database) may be helpful.  For example,::
+
+  # ceph config assimilate-conf < /etc/ceph/ceph.conf
+  # ceph config generate-minimal-config > /etc/ceph/ceph.conf.new
+  # cat /etc/ceph/ceph.conf.new
+  # minimal ceph.conf for 0e5a806b-0ce5-4bc6-b949-aa6f68f5c2a3
+  [global]
+          fsid = 0e5a806b-0ce5-4bc6-b949-aa6f68f5c2a3
+          mon_host = [v2:10.0.0.1:3300/0,v1:10.0.0.1:6789/0]
+  # mv /etc/ceph/ceph.conf.new /etc/ceph/ceph.conf
+
+Protocol
+--------
+
+For a detailed description of the v2 wire protocol, see :ref:`msgr2-protocol`.
diff --git a/doc/rados/configuration/network-config-ref.rst b/doc/rados/configuration/network-config-ref.rst
new file mode 100644
index 000000000..81e85c5d1
--- /dev/null
+++ b/doc/rados/configuration/network-config-ref.rst
@@ -0,0 +1,355 @@
+=================================
+ Network Configuration Reference
+=================================
+
+Network configuration is critical for building a high performance  :term:`Ceph
+Storage Cluster`. The Ceph Storage Cluster does not perform  request routing or
+dispatching on behalf of the :term:`Ceph Client`. Instead, Ceph Clients make
+requests directly to Ceph OSD Daemons. Ceph OSD Daemons perform data replication
+on behalf of Ceph Clients, which means replication and other factors impose
+additional loads on Ceph Storage Cluster networks.
+
+Our Quick Start configurations provide a trivial Ceph configuration file that
+sets monitor IP addresses and daemon host names only. Unless you specify a
+cluster network, Ceph assumes a single "public" network. Ceph functions just
+fine with a public network only, but you may see significant performance
+improvement with a second "cluster" network in a large cluster.
+
+It is possible to run a Ceph Storage Cluster with two networks: a public
+(client, front-side) network and a cluster (private, replication, back-side)
+network.  However, this approach
+complicates network configuration (both hardware and software) and does not usually
+have a significant impact on overall performance.  For this reason, we recommend
+that for resilience and capacity dual-NIC systems either active/active bond
+these interfaces or implement a layer 3 multipath strategy with eg. FRR.
+
+If, despite the complexity, one still wishes to use two networks, each
+:term:`Ceph Node` will need to have more than one network interface or VLAN. See `Hardware
+Recommendations - Networks`_ for additional details.
+
+.. ditaa::
+                               +-------------+
+                               | Ceph Client |
+                               +----*--*-----+
+                                    |  ^
+                            Request |  : Response
+                                    v  |
+ /----------------------------------*--*-------------------------------------\
+ |                              Public Network                               |
+ \---*--*------------*--*-------------*--*------------*--*------------*--*---/
+     ^  ^            ^  ^             ^  ^            ^  ^            ^  ^
+     |  |            |  |             |  |            |  |            |  |
+     |  :            |  :             |  :            |  :            |  :
+     v  v            v  v             v  v            v  v            v  v
+ +---*--*---+    +---*--*---+     +---*--*---+    +---*--*---+    +---*--*---+
+ | Ceph MON |    | Ceph MDS |     | Ceph OSD |    | Ceph OSD |    | Ceph OSD |
+ +----------+    +----------+     +---*--*---+    +---*--*---+    +---*--*---+
+                                      ^  ^            ^  ^            ^  ^
+     The cluster network relieves     |  |            |  |            |  |
+     OSD replication and heartbeat    |  :            |  :            |  :
+     traffic from the public network. v  v            v  v            v  v
+ /------------------------------------*--*------------*--*------------*--*---\
+ |   cCCC                      Cluster Network                               |
+ \---------------------------------------------------------------------------/
+
+
+IP Tables
+=========
+
+By default, daemons `bind`_ to ports within the ``6800:7300`` range. You may
+configure this range at your discretion. Before configuring your IP tables,
+check the default ``iptables`` configuration.
+
+.. prompt:: bash $
+
+   sudo iptables -L
+
+Some Linux distributions include rules that reject all inbound requests
+except SSH from all network interfaces. For example:: 
+
+	REJECT all -- anywhere anywhere reject-with icmp-host-prohibited
+
+You will need to delete these rules on both your public and cluster networks
+initially, and replace them with appropriate rules when you are ready to 
+harden the ports on your Ceph Nodes.
+
+
+Monitor IP Tables
+-----------------
+
+Ceph Monitors listen on ports ``3300`` and ``6789`` by
+default. Additionally, Ceph Monitors always operate on the public
+network. When you add the rule using the example below, make sure you
+replace ``{iface}`` with the public network interface (e.g., ``eth0``,
+``eth1``, etc.), ``{ip-address}`` with the IP address of the public
+network and ``{netmask}`` with the netmask for the public network. :
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface} -p tcp -s {ip-address}/{netmask} --dport 6789 -j ACCEPT
+
+
+MDS and Manager IP Tables
+-------------------------
+
+A :term:`Ceph Metadata Server` or :term:`Ceph Manager` listens on the first 
+available port on the public network beginning at port 6800. Note that this 
+behavior is not deterministic, so if you are running more than one OSD or MDS
+on the same host, or if you restart the daemons within a short window of time,
+the daemons will bind to higher ports. You should open the entire 6800-7300
+range by default.  When you add the rule using the example below, make sure
+you replace ``{iface}`` with the public network interface (e.g., ``eth0``,
+``eth1``, etc.), ``{ip-address}`` with the IP address of the public network
+and ``{netmask}`` with the netmask of the public network.
+
+For example:
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface} -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
+
+
+OSD IP Tables
+-------------
+
+By default, Ceph OSD Daemons `bind`_ to the first available ports on a Ceph Node
+beginning at port 6800.  Note that this behavior is not deterministic, so if you
+are running more than one OSD or MDS on the same host, or if you restart the
+daemons within a short window of time, the daemons will bind to higher ports.
+Each Ceph OSD Daemon on a Ceph Node may use up to four ports:
+
+#. One for talking to clients and monitors.
+#. One for sending data to other OSDs.
+#. Two for heartbeating on each interface.
+
+.. ditaa::
+              /---------------\
+              |      OSD      |
+              |           +---+----------------+-----------+
+              |           | Clients & Monitors | Heartbeat |
+              |           +---+----------------+-----------+
+              |               |
+              |           +---+----------------+-----------+
+              |           | Data Replication   | Heartbeat |
+              |           +---+----------------+-----------+
+              | cCCC          |
+              \---------------/
+
+When a daemon fails and restarts without letting go of the port, the restarted
+daemon will bind to a new port. You should open the entire 6800-7300 port range
+to handle this possibility.
+
+If you set up separate public and cluster networks, you must add rules for both
+the public network and the cluster network, because clients will connect using
+the public network and other Ceph OSD Daemons will connect using the cluster
+network. When you add the rule using the example below, make sure you replace
+``{iface}`` with the network interface (e.g., ``eth0``, ``eth1``, etc.),
+``{ip-address}`` with the IP address and ``{netmask}`` with the netmask of the
+public or cluster network. For example:
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface}  -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
+
+.. tip:: If you run Ceph Metadata Servers on the same Ceph Node as the 
+   Ceph OSD Daemons, you can consolidate the public network configuration step. 
+
+
+Ceph Networks
+=============
+
+To configure Ceph networks, you must add a network configuration to the
+``[global]`` section of the configuration file. Our 5-minute Quick Start
+provides a trivial Ceph configuration file that assumes one public network
+with client and server on the same network and subnet. Ceph functions just fine
+with a public network only. However, Ceph allows you to establish much more
+specific criteria, including multiple IP network and subnet masks for your
+public network. You can also establish a separate cluster network to handle OSD
+heartbeat, object replication and recovery traffic. Don't confuse the IP
+addresses you set in your configuration with the public-facing IP addresses
+network clients may use to access your service. Typical internal IP networks are
+often ``192.168.0.0`` or ``10.0.0.0``.
+
+.. tip:: If you specify more than one IP address and subnet mask for
+   either the public or the cluster network, the subnets within the network
+   must be capable of routing to each other. Additionally, make sure you
+   include each IP address/subnet in your IP tables and open ports for them
+   as necessary.
+
+.. note:: Ceph uses `CIDR`_ notation for subnets (e.g., ``10.0.0.0/24``).
+
+When you have configured your networks, you may restart your cluster or restart
+each daemon. Ceph daemons bind dynamically, so you do not have to restart the
+entire cluster at once if you change your network configuration.
+
+
+Public Network
+--------------
+
+To configure a public network, add the following option to the ``[global]``
+section of your Ceph configuration file. 
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		public_network = {public-network/netmask}
+
+.. _cluster-network:
+
+Cluster Network
+---------------
+
+If you declare a cluster network, OSDs will route heartbeat, object replication
+and recovery traffic over the cluster network. This may improve performance
+compared to using a single network. To configure a cluster network, add the
+following option to the ``[global]`` section of your Ceph configuration file. 
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		cluster_network = {cluster-network/netmask}
+
+We prefer that the cluster network is **NOT** reachable from the public network
+or the Internet for added security.
+
+IPv4/IPv6 Dual Stack Mode
+-------------------------
+
+If you want to run in an IPv4/IPv6 dual stack mode and want to define your public and/or
+cluster networks, then you need to specify both your IPv4 and IPv6 networks for each:
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		public_network = {IPv4 public-network/netmask}, {IPv6 public-network/netmask}
+
+This is so that Ceph can find a valid IP address for both address families.
+
+If you want just an IPv4 or an IPv6 stack environment, then make sure you set the `ms bind`
+options correctly.
+
+.. note::
+   Binding to IPv4 is enabled by default, so if you just add the option to bind to IPv6
+   you'll actually put yourself into dual stack mode. If you want just IPv6, then disable IPv4 and
+   enable IPv6. See `Bind`_ below.
+
+Ceph Daemons
+============
+
+Monitor daemons are each configured to bind to a specific IP address.  These
+addresses are normally configured by your deployment tool.  Other components
+in the Ceph cluster discover the monitors via the ``mon host`` configuration
+option, normally specified in the ``[global]`` section of the ``ceph.conf`` file.
+
+.. code-block:: ini
+
+     [global]
+         mon_host = 10.0.0.2, 10.0.0.3, 10.0.0.4
+
+The ``mon_host`` value can be a list of IP addresses or a name that is
+looked up via DNS.  In the case of a DNS name with multiple A or AAAA
+records, all records are probed in order to discover a monitor.  Once
+one monitor is reached, all other current monitors are discovered, so
+the ``mon host`` configuration option only needs to be sufficiently up
+to date such that a client can reach one monitor that is currently online.
+
+The MGR, OSD, and MDS daemons will bind to any available address and
+do not require any special configuration.  However, it is possible to
+specify a specific IP address for them to bind to with the ``public
+addr`` (and/or, in the case of OSD daemons, the ``cluster addr``)
+configuration option.  For example,
+
+.. code-block:: ini
+
+	[osd.0]
+		public_addr = {host-public-ip-address}
+		cluster_addr = {host-cluster-ip-address}
+
+.. topic:: One NIC OSD in a Two Network Cluster
+
+   Generally, we do not recommend deploying an OSD host with a single network interface in a 
+   cluster with two networks. However, you may accomplish this by forcing the 
+   OSD host to operate on the public network by adding a ``public_addr`` entry
+   to the ``[osd.n]`` section of the Ceph configuration file, where ``n`` 
+   refers to the ID of the OSD with one network interface. Additionally, the public
+   network and cluster network must be able to route traffic to each other, 
+   which we don't recommend for security reasons.
+
+
+Network Config Settings
+=======================
+
+Network configuration settings are not required. Ceph assumes a public network
+with all hosts operating on it unless you specifically configure a cluster 
+network.
+
+
+Public Network
+--------------
+
+The public network configuration allows you specifically define IP addresses
+and subnets for the public network. You may specifically assign static IP 
+addresses or override ``public_network`` settings using the ``public_addr``
+setting for a specific daemon.
+
+.. confval:: public_network
+.. confval:: public_addr
+
+Cluster Network
+---------------
+
+The cluster network configuration allows you to declare a cluster network, and
+specifically define IP addresses and subnets for the cluster network. You may
+specifically assign static IP  addresses or override ``cluster_network``
+settings using the ``cluster_addr`` setting for specific OSD daemons.
+
+
+.. confval:: cluster_network
+.. confval:: cluster_addr
+
+Bind
+----
+
+Bind settings set the default port ranges Ceph OSD and MDS daemons use. The
+default range is ``6800:7300``. Ensure that your `IP Tables`_ configuration
+allows you to use the configured port range.
+
+You may also enable Ceph daemons to bind to IPv6 addresses instead of IPv4
+addresses.
+
+.. confval:: ms_bind_port_min
+.. confval:: ms_bind_port_max
+.. confval:: ms_bind_ipv4
+.. confval:: ms_bind_ipv6
+.. confval:: public_bind_addr
+
+TCP
+---
+
+Ceph disables TCP buffering by default.
+
+.. confval:: ms_tcp_nodelay
+.. confval:: ms_tcp_rcvbuf
+
+General Settings
+----------------
+
+.. confval:: ms_type
+.. confval:: ms_async_op_threads
+.. confval:: ms_initial_backoff
+.. confval:: ms_max_backoff
+.. confval:: ms_die_on_bad_msg
+.. confval:: ms_dispatch_throttle_bytes
+.. confval:: ms_inject_socket_failures
+
+
+.. _Scalability and High Availability: ../../../architecture#scalability-and-high-availability
+.. _Hardware Recommendations - Networks: ../../../start/hardware-recommendations#networks
+.. _hardware recommendations: ../../../start/hardware-recommendations
+.. _Monitor / OSD Interaction: ../mon-osd-interaction
+.. _Message Signatures: ../auth-config-ref#signatures
+.. _CIDR: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing
+.. _Nagle's Algorithm: https://en.wikipedia.org/wiki/Nagle's_algorithm
diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
new file mode 100644
index 000000000..060121200
--- /dev/null
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -0,0 +1,445 @@
+======================
+ OSD Config Reference
+======================
+
+.. index:: OSD; configuration
+
+You can configure Ceph OSD Daemons in the Ceph configuration file (or in recent
+releases, the central config store), but Ceph OSD
+Daemons can use the default values and a very minimal configuration. A minimal
+Ceph OSD Daemon configuration sets ``host`` and
+uses default values for nearly everything else.
+
+Ceph OSD Daemons are numerically identified in incremental fashion, beginning
+with ``0`` using the following convention. ::
+
+	osd.0
+	osd.1
+	osd.2
+
+In a configuration file, you may specify settings for all Ceph OSD Daemons in
+the cluster by adding configuration settings to the ``[osd]`` section of your
+configuration file. To add settings directly to a specific Ceph OSD Daemon
+(e.g., ``host``), enter  it in an OSD-specific section of your configuration
+file. For example:
+
+.. code-block:: ini
+
+	[osd]
+		osd_journal_size = 5120
+
+	[osd.0]
+		host = osd-host-a
+
+	[osd.1]
+		host = osd-host-b
+
+
+.. index:: OSD; config settings
+
+General Settings
+================
+
+The following settings provide a Ceph OSD Daemon's ID, and determine paths to
+data and journals. Ceph deployment scripts typically generate the UUID
+automatically.
+
+.. warning:: **DO NOT** change the default paths for data or journals, as it
+             makes it more problematic to troubleshoot Ceph later.
+
+When using Filestore, the journal size should be at least twice the product of the expected drive
+speed multiplied by ``filestore_max_sync_interval``. However, the most common
+practice is to partition the journal drive (often an SSD), and mount it such
+that Ceph uses the entire partition for the journal.
+
+.. confval:: osd_uuid
+.. confval:: osd_data
+.. confval:: osd_max_write_size
+.. confval:: osd_max_object_size
+.. confval:: osd_client_message_size_cap
+.. confval:: osd_class_dir
+   :default: $libdir/rados-classes
+
+.. index:: OSD; file system
+
+File System Settings
+====================
+Ceph builds and mounts file systems which are used for Ceph OSDs.
+
+``osd_mkfs_options {fs-type}``
+
+:Description: Options used when creating a new Ceph Filestore OSD of type {fs-type}.
+
+:Type: String
+:Default for xfs: ``-f -i 2048``
+:Default for other file systems: {empty string}
+
+For example::
+  ``osd_mkfs_options_xfs = -f -d agcount=24``
+
+``osd_mount_options {fs-type}``
+
+:Description: Options used when mounting a Ceph Filestore OSD of type {fs-type}.
+
+:Type: String
+:Default for xfs: ``rw,noatime,inode64``
+:Default for other file systems: ``rw, noatime``
+
+For example::
+  ``osd_mount_options_xfs = rw, noatime, inode64, logbufs=8``
+
+
+.. index:: OSD; journal settings
+
+Journal Settings
+================
+
+This section applies only to the older Filestore OSD back end.  Since Luminous
+BlueStore has been default and preferred.
+
+By default, Ceph expects that you will provision a Ceph OSD Daemon's journal at
+the following path, which is usually a symlink to a device or partition::
+
+	/var/lib/ceph/osd/$cluster-$id/journal
+
+When using a single device type (for example, spinning drives), the journals
+should be *colocated*: the logical volume (or partition) should be in the same
+device as the ``data`` logical volume.
+
+When using a mix of fast (SSDs, NVMe) devices with slower ones (like spinning
+drives) it makes sense to place the journal on the faster device, while
+``data`` occupies the slower device fully.
+
+The default ``osd_journal_size`` value is 5120 (5 gigabytes), but it can be
+larger, in which case it will need to be set in the ``ceph.conf`` file.
+A value of 10 gigabytes is common in practice::
+
+	osd_journal_size = 10240
+
+
+.. confval:: osd_journal
+.. confval:: osd_journal_size
+
+See `Journal Config Reference`_ for additional details.
+
+
+Monitor OSD Interaction
+=======================
+
+Ceph OSD Daemons check each other's heartbeats and report to monitors
+periodically. Ceph can use default values in many cases. However, if your
+network has latency issues, you may need to adopt longer intervals. See
+`Configuring Monitor/OSD Interaction`_ for a detailed discussion of heartbeats.
+
+
+Data Placement
+==============
+
+See `Pool & PG Config Reference`_ for details.
+
+
+.. index:: OSD; scrubbing
+
+.. _rados_config_scrubbing:
+
+Scrubbing
+=========
+
+One way that Ceph ensures data integrity is by "scrubbing" placement groups.
+Ceph scrubbing is analogous to ``fsck`` on the object storage layer. Ceph
+generates a catalog of all objects in each placement group and compares each
+primary object to its replicas, ensuring that no objects are missing or
+mismatched. Light scrubbing checks the object size and attributes, and is
+usually done daily. Deep scrubbing reads the data and uses checksums to ensure
+data integrity, and is usually done weekly. The freqeuncies of both light
+scrubbing and deep scrubbing are determined by the cluster's configuration,
+which is fully under your control and subject to the settings explained below
+in this section.
+
+Although scrubbing is important for maintaining data integrity, it can reduce
+the performance of the Ceph cluster. You can adjust the following settings to
+increase or decrease the frequency and depth of scrubbing operations.
+
+
+.. confval:: osd_max_scrubs
+.. confval:: osd_scrub_begin_hour
+.. confval:: osd_scrub_end_hour
+.. confval:: osd_scrub_begin_week_day
+.. confval:: osd_scrub_end_week_day
+.. confval:: osd_scrub_during_recovery
+.. confval:: osd_scrub_load_threshold
+.. confval:: osd_scrub_min_interval
+.. confval:: osd_scrub_max_interval
+.. confval:: osd_scrub_chunk_min
+.. confval:: osd_scrub_chunk_max
+.. confval:: osd_scrub_sleep
+.. confval:: osd_deep_scrub_interval
+.. confval:: osd_scrub_interval_randomize_ratio
+.. confval:: osd_deep_scrub_stride
+.. confval:: osd_scrub_auto_repair
+.. confval:: osd_scrub_auto_repair_num_errors
+
+.. index:: OSD; operations settings
+
+Operations
+==========
+
+.. confval:: osd_op_num_shards
+.. confval:: osd_op_num_shards_hdd
+.. confval:: osd_op_num_shards_ssd
+.. confval:: osd_op_queue
+.. confval:: osd_op_queue_cut_off
+.. confval:: osd_client_op_priority
+.. confval:: osd_recovery_op_priority
+.. confval:: osd_scrub_priority
+.. confval:: osd_requested_scrub_priority
+.. confval:: osd_snap_trim_priority
+.. confval:: osd_snap_trim_sleep
+.. confval:: osd_snap_trim_sleep_hdd
+.. confval:: osd_snap_trim_sleep_ssd
+.. confval:: osd_snap_trim_sleep_hybrid
+.. confval:: osd_op_thread_timeout
+.. confval:: osd_op_complaint_time
+.. confval:: osd_op_history_size
+.. confval:: osd_op_history_duration
+.. confval:: osd_op_log_threshold
+.. confval:: osd_op_thread_suicide_timeout
+.. note:: See https://old.ceph.com/planet/dealing-with-some-osd-timeouts/ for
+   more on ``osd_op_thread_suicide_timeout``. Be aware that this is a link to a
+   reworking of a blog post from 2017, and that its conclusion will direct you
+   back to this page "for more information".
+
+.. _dmclock-qos:
+
+QoS Based on mClock
+-------------------
+
+Ceph's use of mClock is now more refined and can be used by following the
+steps as described in `mClock Config Reference`_.
+
+Core Concepts
+`````````````
+
+Ceph's QoS support is implemented using a queueing scheduler
+based on `the dmClock algorithm`_. This algorithm allocates the I/O
+resources of the Ceph cluster in proportion to weights, and enforces
+the constraints of minimum reservation and maximum limitation, so that
+the services can compete for the resources fairly. Currently the
+*mclock_scheduler* operation queue divides Ceph services involving I/O
+resources into following buckets:
+
+- client op: the iops issued by client
+- osd subop: the iops issued by primary OSD
+- snap trim: the snap trimming related requests
+- pg recovery: the recovery related requests
+- pg scrub: the scrub related requests
+
+And the resources are partitioned using following three sets of tags. In other
+words, the share of each type of service is controlled by three tags:
+
+#. reservation: the minimum IOPS allocated for the service.
+#. limitation: the maximum IOPS allocated for the service.
+#. weight: the proportional share of capacity if extra capacity or system
+   oversubscribed.
+
+In Ceph, operations are graded with "cost". And the resources allocated
+for serving various services are consumed by these "costs". So, for
+example, the more reservation a services has, the more resource it is
+guaranteed to possess, as long as it requires. Assuming there are 2
+services: recovery and client ops:
+
+- recovery: (r:1, l:5, w:1)
+- client ops: (r:2, l:0, w:9)
+
+The settings above ensure that the recovery won't get more than 5
+requests per second serviced, even if it requires so (see CURRENT
+IMPLEMENTATION NOTE below), and no other services are competing with
+it. But if the clients start to issue large amount of I/O requests,
+neither will they exhaust all the I/O resources. 1 request per second
+is always allocated for recovery jobs as long as there are any such
+requests. So the recovery jobs won't be starved even in a cluster with
+high load. And in the meantime, the client ops can enjoy a larger
+portion of the I/O resource, because its weight is "9", while its
+competitor "1". In the case of client ops, it is not clamped by the
+limit setting, so it can make use of all the resources if there is no
+recovery ongoing.
+
+CURRENT IMPLEMENTATION NOTE: the current implementation enforces the limit
+values. Therefore, if a service crosses the enforced limit, the op remains
+in the operation queue until the limit is restored.
+
+Subtleties of mClock
+````````````````````
+
+The reservation and limit values have a unit of requests per
+second. The weight, however, does not technically have a unit and the
+weights are relative to one another. So if one class of requests has a
+weight of 1 and another a weight of 9, then the latter class of
+requests should get 9 executed at a 9 to 1 ratio as the first class.
+However that will only happen once the reservations are met and those
+values include the operations executed under the reservation phase.
+
+Even though the weights do not have units, one must be careful in
+choosing their values due how the algorithm assigns weight tags to
+requests. If the weight is *W*, then for a given class of requests,
+the next one that comes in will have a weight tag of *1/W* plus the
+previous weight tag or the current time, whichever is larger. That
+means if *W* is sufficiently large and therefore *1/W* is sufficiently
+small, the calculated tag may never be assigned as it will get a value
+of the current time. The ultimate lesson is that values for weight
+should not be too large. They should be under the number of requests
+one expects to be serviced each second.
+
+Caveats
+```````
+
+There are some factors that can reduce the impact of the mClock op
+queues within Ceph. First, requests to an OSD are sharded by their
+placement group identifier. Each shard has its own mClock queue and
+these queues neither interact nor share information among them. The
+number of shards can be controlled with the configuration options
+:confval:`osd_op_num_shards`, :confval:`osd_op_num_shards_hdd`, and
+:confval:`osd_op_num_shards_ssd`. A lower number of shards will increase the
+impact of the mClock queues, but may have other deleterious effects.
+
+Second, requests are transferred from the operation queue to the
+operation sequencer, in which they go through the phases of
+execution. The operation queue is where mClock resides and mClock
+determines the next op to transfer to the operation sequencer. The
+number of operations allowed in the operation sequencer is a complex
+issue. In general we want to keep enough operations in the sequencer
+so it's always getting work done on some operations while it's waiting
+for disk and network access to complete on other operations. On the
+other hand, once an operation is transferred to the operation
+sequencer, mClock no longer has control over it. Therefore to maximize
+the impact of mClock, we want to keep as few operations in the
+operation sequencer as possible. So we have an inherent tension.
+
+The configuration options that influence the number of operations in
+the operation sequencer are :confval:`bluestore_throttle_bytes`,
+:confval:`bluestore_throttle_deferred_bytes`,
+:confval:`bluestore_throttle_cost_per_io`,
+:confval:`bluestore_throttle_cost_per_io_hdd`, and
+:confval:`bluestore_throttle_cost_per_io_ssd`.
+
+A third factor that affects the impact of the mClock algorithm is that
+we're using a distributed system, where requests are made to multiple
+OSDs and each OSD has (can have) multiple shards. Yet we're currently
+using the mClock algorithm, which is not distributed (note: dmClock is
+the distributed version of mClock).
+
+Various organizations and individuals are currently experimenting with
+mClock as it exists in this code base along with their modifications
+to the code base. We hope you'll share you're experiences with your
+mClock and dmClock experiments on the ``ceph-devel`` mailing list.
+
+.. confval:: osd_async_recovery_min_cost
+.. confval:: osd_push_per_object_cost
+.. confval:: osd_mclock_scheduler_client_res
+.. confval:: osd_mclock_scheduler_client_wgt
+.. confval:: osd_mclock_scheduler_client_lim
+.. confval:: osd_mclock_scheduler_background_recovery_res
+.. confval:: osd_mclock_scheduler_background_recovery_wgt
+.. confval:: osd_mclock_scheduler_background_recovery_lim
+.. confval:: osd_mclock_scheduler_background_best_effort_res
+.. confval:: osd_mclock_scheduler_background_best_effort_wgt
+.. confval:: osd_mclock_scheduler_background_best_effort_lim
+
+.. _the dmClock algorithm: https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Gulati.pdf
+
+.. index:: OSD; backfilling
+
+Backfilling
+===========
+
+When you add or remove Ceph OSD Daemons to a cluster, CRUSH will
+rebalance the cluster by moving placement groups to or from Ceph OSDs
+to restore balanced utilization. The process of migrating placement groups and
+the objects they contain can reduce the cluster's operational performance
+considerably. To maintain operational performance, Ceph performs this migration
+with 'backfilling', which allows Ceph to set backfill operations to a lower
+priority than requests to read or write data.
+
+
+.. confval:: osd_max_backfills
+.. confval:: osd_backfill_scan_min
+.. confval:: osd_backfill_scan_max
+.. confval:: osd_backfill_retry_interval
+
+.. index:: OSD; osdmap
+
+OSD Map
+=======
+
+OSD maps reflect the OSD daemons operating in the cluster. Over time, the
+number of map epochs increases. Ceph provides some settings to ensure that
+Ceph performs well as the OSD map grows larger.
+
+.. confval:: osd_map_dedup
+.. confval:: osd_map_cache_size
+.. confval:: osd_map_message_max
+
+.. index:: OSD; recovery
+
+Recovery
+========
+
+When the cluster starts or when a Ceph OSD Daemon crashes and restarts, the OSD
+begins peering with other Ceph OSD Daemons before writes can occur.  See
+`Monitoring OSDs and PGs`_ for details.
+
+If a Ceph OSD Daemon crashes and comes back online, usually it will be out of
+sync with other Ceph OSD Daemons containing more recent versions of objects in
+the placement groups. When this happens, the Ceph OSD Daemon goes into recovery
+mode and seeks to get the latest copy of the data and bring its map back up to
+date. Depending upon how long the Ceph OSD Daemon was down, the OSD's objects
+and placement groups may be significantly out of date. Also, if a failure domain
+went down (e.g., a rack), more than one Ceph OSD Daemon may come back online at
+the same time. This can make the recovery process time consuming and resource
+intensive.
+
+To maintain operational performance, Ceph performs recovery with limitations on
+the number recovery requests, threads and object chunk sizes which allows Ceph
+perform well in a degraded state.
+
+.. confval:: osd_recovery_delay_start
+.. confval:: osd_recovery_max_active
+.. confval:: osd_recovery_max_active_hdd
+.. confval:: osd_recovery_max_active_ssd
+.. confval:: osd_recovery_max_chunk
+.. confval:: osd_recovery_max_single_start
+.. confval:: osd_recover_clone_overlap
+.. confval:: osd_recovery_sleep
+.. confval:: osd_recovery_sleep_hdd
+.. confval:: osd_recovery_sleep_ssd
+.. confval:: osd_recovery_sleep_hybrid
+.. confval:: osd_recovery_priority
+
+Tiering
+=======
+
+.. confval:: osd_agent_max_ops
+.. confval:: osd_agent_max_low_ops
+
+See `cache target dirty high ratio`_ for when the tiering agent flushes dirty
+objects within the high speed mode.
+
+Miscellaneous
+=============
+
+.. confval:: osd_default_notify_timeout
+.. confval:: osd_check_for_log_corruption
+.. confval:: osd_delete_sleep
+.. confval:: osd_delete_sleep_hdd
+.. confval:: osd_delete_sleep_ssd
+.. confval:: osd_delete_sleep_hybrid
+.. confval:: osd_command_max_records
+.. confval:: osd_fast_fail_on_connection_refused
+
+.. _pool: ../../operations/pools
+.. _Configuring Monitor/OSD Interaction: ../mon-osd-interaction
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg#peering
+.. _Pool & PG Config Reference: ../pool-pg-config-ref
+.. _Journal Config Reference: ../journal-ref
+.. _cache target dirty high ratio: ../../operations/pools#cache-target-dirty-high-ratio
+.. _mClock Config Reference: ../mclock-config-ref
diff --git a/doc/rados/configuration/pool-pg-config-ref.rst b/doc/rados/configuration/pool-pg-config-ref.rst
new file mode 100644
index 000000000..902c80346
--- /dev/null
+++ b/doc/rados/configuration/pool-pg-config-ref.rst
@@ -0,0 +1,46 @@
+.. _rados_config_pool_pg_crush_ref:
+
+======================================
+ Pool, PG and CRUSH Config Reference
+======================================
+
+.. index:: pools; configuration
+
+Ceph uses default values to determine how many placement groups (PGs) will be
+assigned to each pool. We recommend overriding some of the defaults.
+Specifically, we recommend setting a pool's replica size and overriding the
+default number of placement groups. You can set these values when running
+`pool`_ commands. You can also override the defaults by adding new ones in the
+``[global]`` section of your Ceph configuration file.
+
+
+.. literalinclude:: pool-pg.conf
+   :language: ini
+
+.. confval:: mon_max_pool_pg_num
+.. confval:: mon_pg_stuck_threshold
+.. confval:: mon_pg_warn_min_per_osd
+.. confval:: mon_pg_warn_min_objects
+.. confval:: mon_pg_warn_min_pool_objects
+.. confval:: mon_pg_check_down_all_threshold
+.. confval:: mon_pg_warn_max_object_skew
+.. confval:: mon_delta_reset_interval
+.. confval:: osd_crush_chooseleaf_type
+.. confval:: osd_crush_initial_weight
+.. confval:: osd_pool_default_crush_rule
+.. confval:: osd_pool_erasure_code_stripe_unit
+.. confval:: osd_pool_default_size
+.. confval:: osd_pool_default_min_size
+.. confval:: osd_pool_default_pg_num
+.. confval:: osd_pool_default_pgp_num
+.. confval:: osd_pool_default_pg_autoscale_mode
+.. confval:: osd_pool_default_flags
+.. confval:: osd_max_pgls
+.. confval:: osd_min_pg_log_entries
+.. confval:: osd_max_pg_log_entries
+.. confval:: osd_default_data_pool_replay_window
+.. confval:: osd_max_pg_per_osd_hard_ratio
+
+.. _pool: ../../operations/pools
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg#peering
+.. _Weighting Bucket Items: ../../operations/crush-map#weightingbucketitems
diff --git a/doc/rados/configuration/pool-pg.conf b/doc/rados/configuration/pool-pg.conf
new file mode 100644
index 000000000..6765d37df
--- /dev/null
+++ b/doc/rados/configuration/pool-pg.conf
@@ -0,0 +1,21 @@
+[global]
+
+	# By default, Ceph makes three replicas of RADOS objects. If you want
+	# to maintain four copies of an object the default value--a primary
+	# copy and three replica copies--reset the default values as shown in
+	# 'osd_pool_default_size'. If you want to allow Ceph to accept an I/O
+	# operation to a degraded PG, set 'osd_pool_default_min_size' to a
+	# number less than the 'osd_pool_default_size' value.
+
+	osd_pool_default_size = 3  # Write an object three times.
+	osd_pool_default_min_size = 2 # Accept an I/O operation to a PG that has two copies of an object.
+
+	# Note: by default, PG autoscaling is enabled and this value is used only
+	# in specific circumstances. It is however still recommend to set it.
+	# Ensure you have a realistic number of placement groups. We recommend
+	# approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
+	# divided by the number of replicas (i.e., 'osd_pool_default_size'). So for
+	# 10 OSDs and 'osd_pool_default_size' = 4, we'd recommend approximately
+	# (100 * 10) / 4 = 250.
+	# Always use the nearest power of two.
+	osd_pool_default_pg_num = 256
diff --git a/doc/rados/configuration/storage-devices.rst b/doc/rados/configuration/storage-devices.rst
new file mode 100644
index 000000000..c83e87da7
--- /dev/null
+++ b/doc/rados/configuration/storage-devices.rst
@@ -0,0 +1,93 @@
+=================
+ Storage Devices
+=================
+
+There are several Ceph daemons in a storage cluster:
+
+.. _rados_configuration_storage-devices_ceph_osd:
+
+* **Ceph OSDs** (Object Storage Daemons) store most of the data
+  in Ceph. Usually each OSD is backed by a single storage device.
+  This can be a traditional hard disk (HDD) or a solid state disk
+  (SSD). OSDs can also be backed by a combination of devices: for
+  example, a HDD for most data and an SSD (or partition of an
+  SSD) for some metadata. The number of OSDs in a cluster is
+  usually a function of the amount of data to be stored, the size
+  of each storage device, and the level and type of redundancy
+  specified (replication or erasure coding).
+* **Ceph Monitor** daemons manage critical cluster state. This
+  includes cluster membership and authentication information.
+  Small clusters require only a few gigabytes of storage to hold
+  the monitor database. In large clusters, however, the monitor
+  database can reach sizes of tens of gigabytes to hundreds of
+  gigabytes.  
+* **Ceph Manager** daemons run alongside monitor daemons, providing
+  additional monitoring and providing interfaces to external
+  monitoring and management systems.
+
+.. _rados_config_storage_devices_osd_backends:
+
+OSD Back Ends
+=============
+
+There are two ways that OSDs manage the data they store.  As of the Luminous
+12.2.z release, the default (and recommended) back end is *BlueStore*.  Prior
+to the Luminous release, the default (and only) back end was *Filestore*.
+
+.. _rados_config_storage_devices_bluestore:
+
+BlueStore
+---------
+
+BlueStore is a special-purpose storage back end designed specifically for
+managing data on disk for Ceph OSD workloads.  BlueStore's design is based on
+a decade of experience of supporting and managing Filestore OSDs. 
+
+Key BlueStore features include:
+
+* Direct management of storage devices. BlueStore consumes raw block devices or
+  partitions. This avoids intervening layers of abstraction (such as local file
+  systems like XFS) that can limit performance or add complexity.
+* Metadata management with RocksDB. RocksDB's key/value database is embedded
+  in order to manage internal metadata, including the mapping of object
+  names to block locations on disk.
+* Full data and metadata checksumming. By default, all data and
+  metadata written to BlueStore is protected by one or more
+  checksums. No data or metadata is read from disk or returned
+  to the user without being verified.
+* Inline compression.  Data can be optionally compressed before being written
+  to disk.
+* Multi-device metadata tiering. BlueStore allows its internal
+  journal (write-ahead log) to be written to a separate, high-speed
+  device (like an SSD, NVMe, or NVDIMM) for increased performance.  If
+  a significant amount of faster storage is available, internal
+  metadata can be stored on the faster device.
+* Efficient copy-on-write. RBD and CephFS snapshots rely on a
+  copy-on-write *clone* mechanism that is implemented efficiently in
+  BlueStore. This results in efficient I/O both for regular snapshots
+  and for erasure-coded pools (which rely on cloning to implement
+  efficient two-phase commits).
+
+For more information, see :doc:`bluestore-config-ref` and :doc:`/rados/operations/bluestore-migration`.
+
+FileStore
+---------
+.. warning:: Filestore has been deprecated in the Reef release and is no longer supported.
+
+
+FileStore is the legacy approach to storing objects in Ceph. It
+relies on a standard file system (normally XFS) in combination with a
+key/value database (traditionally LevelDB, now RocksDB) for some
+metadata.
+
+FileStore is well-tested and widely used in production. However, it
+suffers from many performance deficiencies due to its overall design
+and its reliance on a traditional file system for object data storage.
+
+Although FileStore is capable of functioning on most POSIX-compatible
+file systems (including btrfs and ext4), we recommend that only the
+XFS file system be used with Ceph. Both btrfs and ext4 have known bugs and
+deficiencies and their use may lead to data loss. By default, all Ceph
+provisioning tools use XFS.
+
+For more information, see :doc:`filestore-config-ref`.
diff --git a/doc/rados/index.rst b/doc/rados/index.rst
new file mode 100644
index 000000000..b506b7a7e
--- /dev/null
+++ b/doc/rados/index.rst
@@ -0,0 +1,81 @@
+.. _rados-index:
+
+======================
+ Ceph Storage Cluster
+======================
+
+The :term:`Ceph Storage Cluster` is the foundation for all Ceph deployments.
+Based upon :abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, Ceph
+Storage Clusters consist of several types of daemons: 
+
+  1. a :term:`Ceph OSD Daemon` (OSD) stores data as objects on a storage node
+  2. a :term:`Ceph Monitor` (MON) maintains a master copy of the cluster map. 
+  3. a :term:`Ceph Manager`  manager daemon
+       
+A Ceph Storage Cluster might contain thousands of storage nodes. A
+minimal system has at least one Ceph Monitor and two Ceph OSD
+Daemons for data replication. 
+
+The Ceph File System, Ceph Object Storage and Ceph Block Devices read data from
+and write data to the Ceph Storage Cluster.
+
+.. container:: columns-3
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Config and Deploy</h3>
+
+      Ceph Storage Clusters have a few required settings, but most configuration
+      settings have default values. A typical deployment uses a deployment tool
+      to define a cluster and bootstrap a monitor. See :ref:`cephadm` for details.
+
+      .. toctree::
+         :maxdepth: 2
+
+         Configuration <configuration/index>
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Operations</h3>
+
+      Once you have deployed a Ceph Storage Cluster, you may begin operating
+      your cluster.
+
+      .. toctree::
+         :maxdepth: 2
+
+         Operations <operations/index>
+
+      .. toctree::
+         :maxdepth: 1
+
+         Man Pages <man/index>
+
+      .. toctree::
+         :hidden:
+
+         troubleshooting/index
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>APIs</h3>
+
+      Most Ceph deployments use `Ceph Block Devices`_, `Ceph Object Storage`_ and/or the
+      `Ceph File System`_. You  may also develop applications that talk directly to
+      the Ceph Storage Cluster.
+
+      .. toctree::
+         :maxdepth: 2
+
+         APIs <api/index>
+
+.. _Ceph Block Devices: ../rbd/
+.. _Ceph File System: ../cephfs/
+.. _Ceph Object Storage: ../radosgw/
+.. _Deployment: ../cephadm/
diff --git a/doc/rados/man/index.rst b/doc/rados/man/index.rst
new file mode 100644
index 000000000..bac56aa46
--- /dev/null
+++ b/doc/rados/man/index.rst
@@ -0,0 +1,32 @@
+=======================
+ Object Store Manpages
+=======================
+
+.. toctree:: 
+   :maxdepth: 1
+
+   ../../man/8/ceph-volume.rst
+   ../../man/8/ceph-volume-systemd.rst
+   ../../man/8/ceph.rst
+   ../../man/8/ceph-authtool.rst
+   ../../man/8/ceph-clsinfo.rst
+   ../../man/8/ceph-conf.rst
+   ../../man/8/ceph-debugpack.rst
+   ../../man/8/ceph-dencoder.rst
+   ../../man/8/ceph-mon.rst
+   ../../man/8/ceph-osd.rst
+   ../../man/8/ceph-kvstore-tool.rst
+   ../../man/8/ceph-run.rst
+   ../../man/8/ceph-syn.rst
+   ../../man/8/crushdiff.rst
+   ../../man/8/crushtool.rst
+   ../../man/8/librados-config.rst
+   ../../man/8/monmaptool.rst
+   ../../man/8/osdmaptool.rst
+   ../../man/8/rados.rst
+
+
+.. toctree::
+	:hidden:
+	
+	../../man/8/ceph-post-file.rst
diff --git a/doc/rados/operations/add-or-rm-mons.rst b/doc/rados/operations/add-or-rm-mons.rst
new file mode 100644
index 000000000..3688bb798
--- /dev/null
+++ b/doc/rados/operations/add-or-rm-mons.rst
@@ -0,0 +1,458 @@
+.. _adding-and-removing-monitors:
+
+==========================
+ Adding/Removing Monitors
+==========================
+
+It is possible to add monitors to a running cluster as long as redundancy is
+maintained. To bootstrap a monitor, see `Manual Deployment`_ or `Monitor
+Bootstrap`_.
+
+.. _adding-monitors:
+
+Adding Monitors
+===============
+
+Ceph monitors serve as the single source of truth for the cluster map. It is
+possible to run a cluster with only one monitor, but for a production cluster
+it is recommended to have at least three monitors provisioned and in quorum.
+Ceph monitors use a variation of the `Paxos`_ algorithm to maintain consensus
+about maps and about other critical information across the cluster. Due to the
+nature of Paxos, Ceph is able to maintain quorum (and thus establish
+consensus) only if a majority of the monitors are ``active``.
+
+It is best to run an odd number of monitors. This is because a cluster that is
+running an odd number of monitors is more resilient than a cluster running an
+even number. For example, in a two-monitor deployment, no failures can be
+tolerated if quorum is to be maintained; in a three-monitor deployment, one
+failure can be tolerated; in a four-monitor deployment, one failure can be
+tolerated; and in a five-monitor deployment, two failures can be tolerated. In
+general, a cluster running an odd number of monitors is best because it avoids
+what is called the *split brain* phenomenon. In short, Ceph is able to operate
+only if a majority of monitors are ``active`` and able to communicate with each
+other, (for example: there must be a single monitor, two out of two monitors,
+two out of three monitors, three out of five monitors, or the like).
+
+For small or non-critical deployments of multi-node Ceph clusters, it is
+recommended to deploy three monitors. For larger clusters or for clusters that
+are intended to survive a double failure, it is recommended to deploy five
+monitors. Only in rare circumstances is there any justification for deploying
+seven or more monitors.
+
+It is possible to run a monitor on the same host that is running an OSD.
+However, this approach has disadvantages: for example: `fsync` issues with the
+kernel might weaken performance, monitor and OSD daemons might be inactive at
+the same time and cause disruption if the node crashes, is rebooted, or is
+taken down for maintenance. Because of these risks, it is instead
+recommended to run monitors and managers on dedicated hosts.
+
+.. note:: A *majority* of monitors in your cluster must be able to 
+   reach each other in order for quorum to be established.
+
+Deploying your Hardware
+-----------------------
+
+Some operators choose to add a new monitor host at the same time that they add
+a new monitor. For details on the minimum recommendations for monitor hardware,
+see `Hardware Recommendations`_. Before adding a monitor host to the cluster,
+make sure that there is an up-to-date version of Linux installed.
+
+Add the newly installed monitor host to a rack in your cluster, connect the
+host to the network, and make sure that the host has network connectivity.
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+
+Installing the Required Software
+--------------------------------
+
+In manually deployed clusters, it is necessary to install Ceph packages
+manually. For details, see `Installing Packages`_. Configure SSH so that it can
+be used by a user that has passwordless authentication and root permissions.
+
+.. _Installing Packages: ../../../install/install-storage-cluster
+
+
+.. _Adding a Monitor (Manual):
+
+Adding a Monitor (Manual)
+-------------------------
+
+The procedure in this section creates a ``ceph-mon`` data directory, retrieves
+both the monitor map and the monitor keyring, and adds a ``ceph-mon`` daemon to
+the cluster. The procedure might result in a Ceph cluster that contains only
+two monitor daemons. To add more monitors until there are enough ``ceph-mon``
+daemons to establish quorum, repeat the procedure.
+
+This is a good point at which to define the new monitor's ``id``. Monitors have
+often been named with single letters (``a``, ``b``, ``c``, etc.), but you are
+free to define the ``id`` however you see fit. In this document, ``{mon-id}``
+refers to the ``id`` exclusive of the ``mon.`` prefix: for example, if
+``mon.a`` has been chosen as the ``id`` of a monitor, then ``{mon-id}`` is
+``a``.                                               ???
+
+#. Create a data directory on the machine that will host the new monitor:
+
+   .. prompt:: bash $
+
+    ssh {new-mon-host}
+    sudo mkdir /var/lib/ceph/mon/ceph-{mon-id}
+
+#. Create a temporary directory ``{tmp}`` that will contain the files needed
+   during this procedure. This directory should be different from the data
+   directory created in the previous step. Because this is a temporary
+   directory, it can be removed after the procedure is complete:
+
+   .. prompt:: bash $
+
+    mkdir {tmp}
+
+#. Retrieve the keyring for your monitors (``{tmp}`` is the path to the
+   retrieved keyring and ``{key-filename}`` is the name of the file that
+   contains the retrieved monitor key):
+
+   .. prompt:: bash $
+
+      ceph auth get mon. -o {tmp}/{key-filename}
+
+#. Retrieve the monitor map (``{tmp}`` is the path to the retrieved monitor map
+   and ``{map-filename}`` is the name of the file that contains the retrieved
+   monitor map):
+
+   .. prompt:: bash $
+
+      ceph mon getmap -o {tmp}/{map-filename}
+
+#. Prepare the monitor's data directory, which was created in the first step.
+   The following command must specify the path to the monitor map (so that
+   information about a quorum of monitors and their ``fsid``\s can be
+   retrieved) and specify the path to the monitor keyring:
+
+   .. prompt:: bash $
+
+      sudo ceph-mon -i {mon-id} --mkfs --monmap {tmp}/{map-filename} --keyring {tmp}/{key-filename}
+
+#. Start the new monitor. It will automatically join the cluster. To provide
+   information to the daemon about which address to bind to, use either the
+   ``--public-addr {ip}`` option or the ``--public-network {network}`` option.
+   For example:
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --public-addr {ip:port}
+
+.. _removing-monitors:
+
+Removing Monitors
+=================
+
+When monitors are removed from a cluster, it is important to remember
+that Ceph monitors use Paxos to maintain consensus about the cluster
+map. Such consensus is possible only if the number of monitors is sufficient
+to establish quorum.
+
+
+.. _Removing a Monitor (Manual):
+
+Removing a Monitor (Manual)
+---------------------------
+
+The procedure in this section removes a ``ceph-mon`` daemon from the cluster.
+The procedure might result in a Ceph cluster that contains a number of monitors
+insufficient to maintain quorum, so plan carefully. When replacing an old
+monitor with a new monitor, add the new monitor first, wait for quorum to be
+established, and then remove the old monitor. This ensures that quorum is not
+lost.
+
+
+#. Stop the monitor:
+
+   .. prompt:: bash $
+
+      service ceph -a stop mon.{mon-id}
+
+#. Remove the monitor from the cluster:
+
+   .. prompt:: bash $
+
+      ceph mon remove {mon-id}
+
+#. Remove the monitor entry from the ``ceph.conf`` file:
+
+.. _rados-mon-remove-from-unhealthy: 
+
+
+Removing Monitors from an Unhealthy Cluster
+-------------------------------------------
+
+The procedure in this section removes a ``ceph-mon`` daemon from an unhealthy
+cluster (for example, a cluster whose monitors are unable to form a quorum).
+
+#. Stop all ``ceph-mon`` daemons on all monitor hosts:
+
+   .. prompt:: bash $
+
+      ssh {mon-host}
+      systemctl stop ceph-mon.target
+
+   Repeat this step on every monitor host.
+
+#. Identify a surviving monitor and log in to the monitor's host:
+
+   .. prompt:: bash $
+
+      ssh {mon-host}
+
+#. Extract a copy of the ``monmap`` file by running a command of the following
+   form:
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --extract-monmap {map-path}
+
+   Here is a more concrete example. In this example, ``hostname`` is the
+   ``{mon-id}`` and ``/tmp/monpap`` is the ``{map-path}``:
+
+   .. prompt:: bash $
+
+      ceph-mon -i `hostname` --extract-monmap /tmp/monmap
+
+#. Remove the non-surviving or otherwise problematic monitors:
+
+   .. prompt:: bash $
+
+      monmaptool {map-path} --rm {mon-id}
+
+   For example, suppose that there are three monitors |---| ``mon.a``, ``mon.b``,
+   and ``mon.c`` |---| and that only ``mon.a`` will survive:
+
+   .. prompt:: bash $
+
+      monmaptool /tmp/monmap --rm b
+      monmaptool /tmp/monmap --rm c
+
+#. Inject the surviving map that includes the removed monitors into the
+   monmap of the surviving monitor(s):
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --inject-monmap {map-path}
+
+   Continuing with the above example, inject a map into monitor ``mon.a`` by
+   running the following command:
+
+   .. prompt:: bash $
+
+      ceph-mon -i a --inject-monmap /tmp/monmap
+
+
+#. Start only the surviving monitors.
+
+#. Verify that the monitors form a quorum by running the command ``ceph -s``.
+
+#. The data directory of the removed monitors is in ``/var/lib/ceph/mon``:
+   either archive this data directory in a safe location or delete this data
+   directory. However, do not delete it unless you are confident that the
+   remaining monitors are healthy and sufficiently redundant. Make sure that
+   there is enough room for the live DB to expand and compact, and make sure
+   that there is also room for an archived copy of the DB. The archived copy
+   can be compressed.
+
+.. _Changing a Monitor's IP address:
+
+Changing a Monitor's IP Address
+===============================
+
+.. important:: Existing monitors are not supposed to change their IP addresses.
+
+Monitors are critical components of a Ceph cluster. The entire system can work
+properly only if the monitors maintain quorum, and quorum can be established
+only if the monitors have discovered each other by means of their IP addresses.
+Ceph has strict requirements on the discovery of monitors.
+
+Although the ``ceph.conf`` file is used by Ceph clients and other Ceph daemons
+to discover monitors, the monitor map is used by monitors to discover each
+other. This is why it is necessary to obtain the current ``monmap`` at the time
+a new monitor is created: as can be seen above in `Adding a Monitor (Manual)`_,
+the ``monmap`` is one of the arguments required by the ``ceph-mon -i {mon-id}
+--mkfs`` command. The following sections explain the consistency requirements
+for Ceph monitors, and also explain a number of safe ways to change a monitor's
+IP address.
+
+
+Consistency Requirements
+------------------------
+
+When a monitor discovers other monitors in the cluster, it always refers to the
+local copy of the monitor map. Using the monitor map instead of using the
+``ceph.conf`` file avoids errors that could break the cluster (for example,
+typos or other slight errors in ``ceph.conf`` when a monitor address or port is
+specified). Because monitors use monitor maps for discovery and because they
+share monitor maps with Ceph clients and other Ceph daemons, the monitor map
+provides monitors with a strict guarantee that their consensus is valid.
+
+Strict consistency also applies to updates to the monmap. As with any other
+updates on the monitor, changes to the monmap always run through a distributed
+consensus algorithm called `Paxos`_. The monitors must agree on each update to
+the monmap, such as adding or removing a monitor, to ensure that each monitor
+in the quorum has the same version of the monmap. Updates to the monmap are
+incremental so that monitors have the latest agreed upon version, and a set of
+previous versions, allowing a monitor that has an older version of the monmap
+to catch up with the current state of the cluster.
+
+There are additional advantages to using the monitor map rather than
+``ceph.conf`` when monitors discover each other. Because ``ceph.conf`` is not
+automatically updated and distributed, its use would bring certain risks:
+monitors might use an outdated ``ceph.conf`` file, might fail to recognize a
+specific monitor, might fall out of quorum, and might develop a situation in
+which `Paxos`_ is unable to accurately ascertain the current state of the
+system. Because of these risks, any changes to an existing monitor's IP address
+must be made with great care.
+
+.. _operations_add_or_rm_mons_changing_mon_ip:
+
+Changing a Monitor's IP address (Preferred Method)
+--------------------------------------------------
+
+If a monitor's IP address is changed only in the ``ceph.conf`` file, there is
+no guarantee that the other monitors in the cluster will receive the update.
+For this reason, the preferred method to change a monitor's IP address is as
+follows: add a new monitor with the desired IP address (as described in `Adding
+a Monitor (Manual)`_), make sure that the new monitor successfully joins the
+quorum, remove the monitor that is using the old IP address, and update the
+``ceph.conf`` file to ensure that clients and other daemons are made aware of
+the new monitor's IP address.
+
+For example, suppose that there are three monitors in place:: 
+
+    [mon.a]
+        host = host01
+        addr = 10.0.0.1:6789
+    [mon.b]
+        host = host02
+        addr = 10.0.0.2:6789
+    [mon.c]
+        host = host03
+        addr = 10.0.0.3:6789
+
+To change ``mon.c`` so that its name is ``host04`` and its IP address is
+``10.0.0.4``: (1) follow the steps in `Adding a Monitor (Manual)`_ to add a new
+monitor ``mon.d``, (2) make sure that ``mon.d`` is  running before removing
+``mon.c`` or else quorum will be broken, and (3) follow the steps in `Removing
+a Monitor (Manual)`_ to remove ``mon.c``. To move all three monitors to new IP
+addresses, repeat this process.
+
+Changing a Monitor's IP address (Advanced Method)
+-------------------------------------------------
+
+There are cases in which the method outlined in :ref"`<Changing a Monitor's IP
+Address (Preferred Method)> operations_add_or_rm_mons_changing_mon_ip` cannot
+be used. For example, it might be necessary to move the cluster's monitors to a
+different network, to a different part of the datacenter, or to a different
+datacenter altogether. It is still possible to change the monitors' IP
+addresses, but a different method must be used.
+
+For such cases, a new monitor map with updated IP addresses for every monitor
+in the cluster must be generated and injected on each monitor. Although this
+method is not particularly easy, such a major migration is unlikely to be a
+routine task. As stated at the beginning of this section, existing monitors are
+not supposed to change their IP addresses.
+
+Continue with the monitor configuration in the example from :ref"`<Changing a
+Monitor's IP Address (Preferred Method)>
+operations_add_or_rm_mons_changing_mon_ip` . Suppose that all of the monitors
+are to be moved from the ``10.0.0.x`` range to the ``10.1.0.x`` range, and that
+these networks are unable to communicate. Carry out the following procedure:
+
+#. Retrieve the monitor map (``{tmp}`` is the path to the retrieved monitor
+   map, and ``{filename}`` is the name of the file that contains the retrieved
+   monitor map):
+
+   .. prompt:: bash $
+
+      ceph mon getmap -o {tmp}/{filename}
+
+#. Check the contents of the monitor map:
+
+   .. prompt:: bash $
+
+      monmaptool --print {tmp}/{filename}
+
+   ::    
+
+    monmaptool: monmap file {tmp}/{filename}
+    epoch 1
+    fsid 224e376d-c5fe-4504-96bb-ea6332a19e61
+    last_changed 2012-12-17 02:46:41.591248
+    created 2012-12-17 02:46:41.591248
+    0: 10.0.0.1:6789/0 mon.a
+    1: 10.0.0.2:6789/0 mon.b
+    2: 10.0.0.3:6789/0 mon.c
+
+#. Remove the existing monitors from the monitor map:
+
+   .. prompt:: bash $
+
+      monmaptool --rm a --rm b --rm c {tmp}/{filename}
+
+   ::
+
+    monmaptool: monmap file {tmp}/{filename}
+    monmaptool: removing a
+    monmaptool: removing b
+    monmaptool: removing c
+    monmaptool: writing epoch 1 to {tmp}/{filename} (0 monitors)
+
+#. Add the new monitor locations to the monitor map:
+
+   .. prompt:: bash $
+
+      monmaptool --add a 10.1.0.1:6789 --add b 10.1.0.2:6789 --add c 10.1.0.3:6789 {tmp}/{filename}
+
+   ::
+
+      monmaptool: monmap file {tmp}/{filename}
+      monmaptool: writing epoch 1 to {tmp}/{filename} (3 monitors)
+
+#. Check the new contents of the monitor map:
+
+   .. prompt:: bash $
+
+       monmaptool --print {tmp}/{filename}
+
+   ::
+
+    monmaptool: monmap file {tmp}/{filename}
+    epoch 1
+    fsid 224e376d-c5fe-4504-96bb-ea6332a19e61
+    last_changed 2012-12-17 02:46:41.591248
+    created 2012-12-17 02:46:41.591248
+    0: 10.1.0.1:6789/0 mon.a
+    1: 10.1.0.2:6789/0 mon.b
+    2: 10.1.0.3:6789/0 mon.c
+
+At this point, we assume that the monitors (and stores) have been installed at
+the new location. Next, propagate the modified monitor map to the new monitors,
+and inject the modified monitor map into each new monitor.
+
+#. Make sure all of your monitors have been stopped. Never inject into a
+   monitor while the monitor daemon is running.
+
+#. Inject the monitor map:
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --inject-monmap {tmp}/{filename}
+
+#. Restart all of the monitors.
+
+Migration to the new location is now complete. The monitors should operate
+successfully.
+
+
+
+.. _Manual Deployment: ../../../install/manual-deployment
+.. _Monitor Bootstrap: ../../../dev/mon-bootstrap
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
+
+.. |---|   unicode:: U+2014 .. EM DASH
+   :trim:
diff --git a/doc/rados/operations/add-or-rm-osds.rst b/doc/rados/operations/add-or-rm-osds.rst
new file mode 100644
index 000000000..1a6621148
--- /dev/null
+++ b/doc/rados/operations/add-or-rm-osds.rst
@@ -0,0 +1,419 @@
+======================
+ Adding/Removing OSDs
+======================
+
+When a cluster is up and running, it is possible to add or remove OSDs. 
+
+Adding OSDs
+===========
+
+OSDs can be added to a cluster in order to expand the cluster's capacity and
+resilience. Typically, an OSD is a Ceph ``ceph-osd`` daemon running on one
+storage drive within a host machine. But if your host machine has multiple
+storage drives, you may map one ``ceph-osd`` daemon for each drive on the
+machine.
+
+It's a good idea to check the capacity of your cluster so that you know when it
+approaches its capacity limits. If your cluster has reached its ``near full``
+ratio, then you should add OSDs to expand your cluster's capacity.
+
+.. warning:: Do not add an OSD after your cluster has reached its ``full
+   ratio``. OSD failures that occur after the cluster reaches its ``near full
+   ratio`` might cause the cluster to exceed its ``full ratio``.
+
+
+Deploying your Hardware
+-----------------------
+
+If you are also adding a new host when adding a new OSD, see `Hardware
+Recommendations`_ for details on minimum recommendations for OSD hardware. To
+add an OSD host to your cluster, begin by making sure that an appropriate 
+version of Linux has been installed on the host machine and that all initial
+preparations for your storage drives have been carried out. For details, see
+`Filesystem Recommendations`_.
+
+Next, add your OSD host to a rack in your cluster, connect the host to the
+network, and ensure that the host has network connectivity. For details, see
+`Network Configuration Reference`_.
+
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+.. _Filesystem Recommendations: ../../configuration/filesystem-recommendations
+.. _Network Configuration Reference: ../../configuration/network-config-ref
+
+Installing the Required Software
+--------------------------------
+
+If your cluster has been manually deployed, you will need to install Ceph
+software packages manually. For details, see `Installing Ceph (Manual)`_.
+Configure SSH for the appropriate user to have both passwordless authentication
+and root permissions.
+
+.. _Installing Ceph (Manual): ../../../install
+
+
+Adding an OSD (Manual)
+----------------------
+
+The following procedure sets up a ``ceph-osd`` daemon, configures this OSD to
+use one drive, and configures the cluster to distribute data to the OSD. If
+your host machine has multiple drives, you may add an OSD for each drive on the
+host by repeating this procedure.
+
+As the following procedure will demonstrate, adding an OSD involves creating a
+metadata directory for it, configuring a data storage drive, adding the OSD to
+the cluster, and then adding it to the CRUSH map.
+
+When you add the OSD to the CRUSH map, you will need to consider the weight you
+assign to the new OSD. Since storage drive capacities increase over time, newer
+OSD hosts are likely to have larger hard drives than the older hosts in the
+cluster have and therefore might have greater weight as well.
+
+.. tip:: Ceph works best with uniform hardware across pools. It is possible to
+   add drives of dissimilar size and then adjust their weights accordingly.
+   However, for best performance, consider a CRUSH hierarchy that has drives of
+   the same type and size. It is better to add larger drives uniformly to
+   existing hosts. This can be done incrementally, replacing smaller drives
+   each time the new drives are added.
+
+#. Create the new OSD by running a command of the following form. If you opt
+   not to specify a UUID in this command, the UUID will be set automatically
+   when the OSD starts up. The OSD number, which is needed for subsequent
+   steps, is found in the command's output:
+
+   .. prompt:: bash $
+
+      ceph osd create [{uuid} [{id}]]
+
+   If the optional parameter {id} is specified it will be used as the OSD ID.
+   However, if the ID number is already in use, the command will fail.
+
+   .. warning:: Explicitly specifying the ``{id}`` parameter is not
+      recommended. IDs are allocated as an array, and any skipping of entries
+      consumes extra memory. This memory consumption can become significant if
+      there are large gaps or if clusters are large. By leaving the ``{id}``
+      parameter unspecified, we ensure that Ceph uses the smallest ID number
+      available and that these problems are avoided.
+
+#. Create the default directory for your new OSD by running commands of the
+   following form:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
+
+#. If the OSD will be created on a drive other than the OS drive, prepare it
+   for use with Ceph. Run commands of the following form:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      sudo mkfs -t {fstype} /dev/{drive}
+      sudo mount -o user_xattr /dev/{hdd} /var/lib/ceph/osd/ceph-{osd-number}
+
+#. Initialize the OSD data directory by running commands of the following form:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      ceph-osd -i {osd-num} --mkfs --mkkey
+
+   Make sure that the directory is empty before running ``ceph-osd``.
+
+#. Register the OSD authentication key by running a command of the following
+   form:
+
+   .. prompt:: bash $
+
+      ceph auth add osd.{osd-num} osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-{osd-num}/keyring
+
+   This presentation of the command has ``ceph-{osd-num}`` in the listed path
+   because many clusters have the name ``ceph``. However, if your cluster name
+   is not ``ceph``, then the string ``ceph`` in ``ceph-{osd-num}`` needs to be
+   replaced with your cluster name. For example, if your cluster name is
+   ``cluster1``, then the path in the command should be
+   ``/var/lib/ceph/osd/cluster1-{osd-num}/keyring``.
+
+#. Add the OSD to the CRUSH map by running the following command. This allows
+   the OSD to begin receiving data. The ``ceph osd crush add`` command can add
+   OSDs to the CRUSH hierarchy wherever you want. If you specify one or more
+   buckets, the command places the OSD in the most specific of those buckets,
+   and it moves that bucket underneath any other buckets that you have
+   specified. **Important:** If you specify only the root bucket, the command
+   will attach the OSD directly to the root, but CRUSH rules expect OSDs to be
+   inside of hosts. If the OSDs are not inside hosts, the OSDS will likely not
+   receive any data.
+
+   .. prompt:: bash $
+
+      ceph osd crush add {id-or-name} {weight}  [{bucket-type}={bucket-name} ...]
+
+   Note that there is another way to add a new OSD to the CRUSH map: decompile
+   the CRUSH map, add the OSD to the device list, add the host as a bucket (if
+   it is not already in the CRUSH map), add the device as an item in the host,
+   assign the device a weight, recompile the CRUSH map, and set the CRUSH map.
+   For details, see `Add/Move an OSD`_. This is rarely necessary with recent
+   releases (this sentence was written the month that Reef was released).
+
+
+.. _rados-replacing-an-osd:
+
+Replacing an OSD
+----------------
+
+.. note:: If the procedure in this section does not work for you, try the
+   instructions in the ``cephadm`` documentation:
+   :ref:`cephadm-replacing-an-osd`.
+
+Sometimes OSDs need to be replaced: for example, when a disk fails, or when an
+administrator wants to reprovision OSDs with a new back end (perhaps when
+switching from Filestore to BlueStore). Replacing an OSD differs from `Removing
+the OSD`_ in that the replaced OSD's ID and CRUSH map entry must be kept intact
+after the OSD is destroyed for replacement.
+
+
+#. Make sure that it is safe to destroy the OSD:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy osd.{id} ; do sleep 10 ; done
+
+#. Destroy the OSD:
+
+   .. prompt:: bash $
+
+      ceph osd destroy {id} --yes-i-really-mean-it
+
+#. *Optional*: If the disk that you plan to use is not a new disk and has been
+   used before for other purposes, zap the disk:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap /dev/sdX
+
+#. Prepare the disk for replacement by using the ID of the OSD that was
+   destroyed in previous steps:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm prepare --osd-id {id} --data /dev/sdX
+
+#. Finally, activate the OSD:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm activate {id} {fsid}
+
+Alternatively, instead of carrying out the final two steps (preparing the disk
+and activating the OSD), you can re-create the OSD by running a single command
+of the following form:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --osd-id {id} --data /dev/sdX
+
+Starting the OSD
+----------------
+
+After an OSD is added to Ceph, the OSD is in the cluster. However, until it is
+started, the OSD is considered ``down`` and ``in``. The OSD is not running and
+will be unable to receive data. To start an OSD, either run ``service ceph``
+from your admin host or run a command of the following form to start the OSD
+from its host machine:
+
+   .. prompt:: bash $
+
+      sudo systemctl start ceph-osd@{osd-num}
+
+After the OSD is started, it is considered ``up`` and ``in``.
+
+Observing the Data Migration
+----------------------------
+
+After the new OSD has been added to the CRUSH map, Ceph begins rebalancing the
+cluster by migrating placement groups (PGs) to the new OSD. To observe this
+process by using the `ceph`_ tool, run the following command:
+
+   .. prompt:: bash $
+
+      ceph -w
+
+Or:
+
+   .. prompt:: bash $
+
+      watch ceph status
+
+The PG states will first change from ``active+clean`` to ``active, some
+degraded objects`` and then return to ``active+clean`` when migration
+completes. When you are finished observing, press Ctrl-C to exit.
+
+.. _Add/Move an OSD: ../crush-map#addosd
+.. _ceph: ../monitoring
+
+
+Removing OSDs (Manual)
+======================
+
+It is possible to remove an OSD manually while the cluster is running: you
+might want to do this in order to reduce the size of the cluster or when
+replacing hardware. Typically, an OSD is a Ceph ``ceph-osd`` daemon running on
+one storage drive within a host machine. Alternatively, if your host machine
+has multiple storage drives, you might need to remove multiple ``ceph-osd``
+daemons: one daemon for each drive on the machine. 
+
+.. warning:: Before you begin the process of removing an OSD, make sure that
+   your cluster is not near its ``full ratio``. Otherwise the act of removing
+   OSDs might cause the cluster to reach or exceed its ``full ratio``.
+
+
+Taking the OSD ``out`` of the Cluster
+-------------------------------------
+
+OSDs are typically ``up`` and ``in`` before they are removed from the cluster.
+Before the OSD can be removed from the cluster, the OSD must be taken ``out``
+of the cluster so that Ceph can begin rebalancing and copying its data to other
+OSDs. To take an OSD ``out`` of the cluster, run a command of the following
+form:
+
+   .. prompt:: bash $
+
+      ceph osd out {osd-num}
+
+
+Observing the Data Migration
+----------------------------
+
+After the OSD has been taken ``out`` of the cluster, Ceph begins rebalancing
+the cluster by migrating placement groups out of the OSD that was removed. To
+observe this process by using the `ceph`_ tool, run the following command:
+
+   .. prompt:: bash $
+
+      ceph -w
+
+The PG states will change from ``active+clean`` to ``active, some degraded
+objects`` and will then return to ``active+clean`` when migration completes.
+When you are finished observing, press Ctrl-C to exit.
+
+.. note:: Under certain conditions, the action of taking ``out`` an OSD
+   might lead CRUSH to encounter a corner case in which some PGs remain stuck
+   in the ``active+remapped`` state. This problem sometimes occurs in small
+   clusters with few hosts (for example, in a small testing cluster). To
+   address this problem, mark the OSD ``in`` by running a command of the
+   following form:
+
+   .. prompt:: bash $
+
+      ceph osd in {osd-num}
+
+   After the OSD has come back to its initial state, do not mark the OSD
+   ``out`` again. Instead, set the OSD's weight to ``0`` by running a command
+   of the following form:
+
+   .. prompt:: bash $
+
+      ceph osd crush reweight osd.{osd-num} 0
+
+   After the OSD has been reweighted, observe the data migration and confirm
+   that it has completed successfully. The difference between marking an OSD
+   ``out`` and reweighting the OSD to ``0`` has to do with the bucket that
+   contains the OSD. When an OSD is marked ``out``, the weight of the bucket is
+   not changed. But when an OSD is reweighted to ``0``, the weight of the
+   bucket is updated (namely, the weight of the OSD is subtracted from the
+   overall weight of the bucket). When operating small clusters, it can
+   sometimes be preferable to use the above reweight command.
+
+
+Stopping the OSD
+----------------
+
+After you take an OSD ``out`` of the cluster, the OSD might still be running.
+In such a case, the OSD is ``up`` and ``out``. Before it is removed from the
+cluster, the OSD must be stopped by running commands of the following form:
+
+   .. prompt:: bash $
+
+      ssh {osd-host}
+      sudo systemctl stop ceph-osd@{osd-num}
+
+After the OSD has been stopped, it is ``down``.
+
+
+Removing the OSD
+----------------
+
+The following procedure removes an OSD from the cluster map, removes the OSD's
+authentication key, removes the OSD from the OSD map, and removes the OSD from
+the ``ceph.conf`` file. If your host has multiple drives, it might be necessary
+to remove an OSD from each drive by repeating this procedure.
+
+#. Begin by having the cluster forget the OSD. This step removes the OSD from
+   the CRUSH map, removes the OSD's authentication key, and removes the OSD
+   from the OSD map. (The :ref:`purge subcommand <ceph-admin-osd>` was
+   introduced in Luminous. For older releases, see :ref:`the procedure linked
+   here <ceph_osd_purge_procedure_pre_luminous>`.):
+
+   .. prompt:: bash $
+
+      ceph osd purge {id} --yes-i-really-mean-it
+
+
+#. Navigate to the host where the master copy of the cluster's
+   ``ceph.conf`` file is kept:
+
+   .. prompt:: bash $
+
+      ssh {admin-host}
+      cd /etc/ceph
+      vim ceph.conf
+
+#. Remove the OSD entry from your ``ceph.conf`` file (if such an entry
+   exists)::
+
+    [osd.1]
+        host = {hostname}
+
+#. Copy the updated ``ceph.conf`` file from the location on the host where the
+   master copy of the cluster's ``ceph.conf`` is kept to the ``/etc/ceph``
+   directory of the other hosts in your cluster.
+
+.. _ceph_osd_purge_procedure_pre_luminous:
+
+If your Ceph cluster is older than Luminous, you will be unable to use the
+``ceph osd purge`` command. Instead, carry out the following procedure:
+
+#. Remove the OSD from the CRUSH map so that it no longer receives data (for
+   more details, see `Remove an OSD`_):
+
+   .. prompt:: bash $
+
+      ceph osd crush remove {name}
+
+   Instead of removing the OSD from the CRUSH map, you might opt for one of two
+   alternatives: (1) decompile the CRUSH map, remove the OSD from the device
+   list, and remove the device from the host bucket; (2) remove the host bucket
+   from the CRUSH map (provided that it is in the CRUSH map and that you intend
+   to remove the host), recompile the map, and set it:
+
+
+#. Remove the OSD authentication key:
+
+   .. prompt:: bash $
+
+      ceph auth del osd.{osd-num}
+
+#. Remove the OSD:
+
+   .. prompt:: bash $
+
+      ceph osd rm {osd-num}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd rm 1
+
+.. _Remove an OSD: ../crush-map#removeosd
diff --git a/doc/rados/operations/balancer.rst b/doc/rados/operations/balancer.rst
new file mode 100644
index 000000000..aa4eab93c
--- /dev/null
+++ b/doc/rados/operations/balancer.rst
@@ -0,0 +1,221 @@
+.. _balancer:
+
+Balancer Module
+=======================
+
+The *balancer* can optimize the allocation of placement groups (PGs) across
+OSDs in order to achieve a balanced distribution. The balancer can operate
+either automatically or in a supervised fashion.
+
+
+Status
+------
+
+To check the current status of the balancer, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer status
+
+
+Automatic balancing
+-------------------
+
+When the balancer is in ``upmap`` mode, the automatic balancing feature is
+enabled by default. For more details, see :ref:`upmap`.  To disable the
+balancer, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer off
+
+The balancer mode can be changed from ``upmap`` mode to ``crush-compat`` mode.
+``crush-compat`` mode is backward compatible with older clients.  In
+``crush-compat`` mode, the balancer automatically makes small changes to the
+data distribution in order to ensure that OSDs are utilized equally.
+
+
+Throttling
+----------
+
+If the cluster is degraded (that is, if an OSD has failed and the system hasn't
+healed itself yet), then the balancer will not make any adjustments to the PG
+distribution.
+
+When the cluster is healthy, the balancer will incrementally move a small
+fraction of unbalanced PGs in order to improve distribution.  This fraction
+will not exceed a certain threshold that defaults to 5%. To adjust this
+``target_max_misplaced_ratio`` threshold setting, run the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr target_max_misplaced_ratio .07   # 7%
+
+The balancer sleeps between runs. To set the number of seconds for this
+interval of sleep, run the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/sleep_interval 60
+
+To set the time of day (in HHMM format) at which automatic balancing begins,
+run the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/begin_time 0000
+
+To set the time of day (in HHMM format) at which automatic balancing ends, run
+the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_time 2359
+
+Automatic balancing can be restricted to certain days of the week.  To restrict
+it to a specific day of the week or later (as with crontab, ``0`` is Sunday,
+``1`` is Monday, and so on), run the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/begin_weekday 0
+
+To restrict automatic balancing to a specific day of the week or earlier
+(again, ``0`` is Sunday, ``1`` is Monday, and so on), run the following
+command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_weekday 6
+
+Automatic balancing can be restricted to certain pools. By default, the value
+of this setting is an empty string, so that all pools are automatically
+balanced.  To restrict automatic balancing to specific pools, retrieve their
+numeric pool IDs (by running the :command:`ceph osd pool ls detail` command),
+and then run the following command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/pool_ids 1,2,3
+
+
+Modes
+-----
+
+There are two supported balancer modes:
+
+#. **crush-compat**. This mode uses the compat weight-set feature (introduced
+   in Luminous) to manage an alternative set of weights for devices in the
+   CRUSH hierarchy. When the balancer is operating in this mode, the normal
+   weights should remain set to the size of the device in order to reflect the
+   target amount of data intended to be stored on the device. The balancer will
+   then optimize the weight-set values, adjusting them up or down in small
+   increments, in order to achieve a distribution that matches the target
+   distribution as closely as possible. (Because PG placement is a pseudorandom
+   process, it is subject to a natural amount of variation; optimizing the
+   weights serves to counteract that natural variation.)
+
+   Note that this mode is *fully backward compatible* with older clients: when
+   an OSD Map and CRUSH map are shared with older clients, Ceph presents the
+   optimized weights as the "real" weights.
+
+   The primary limitation of this mode is that the balancer cannot handle
+   multiple CRUSH hierarchies with different placement rules if the subtrees of
+   the hierarchy share any OSDs. (Such sharing of OSDs is not typical and,
+   because of the difficulty of managing the space utilization on the shared
+   OSDs, is generally not recommended.)
+
+#. **upmap**. In Luminous and later releases, the OSDMap can store explicit
+   mappings for individual OSDs as exceptions to the normal CRUSH placement
+   calculation. These ``upmap`` entries provide fine-grained control over the
+   PG mapping. This balancer mode optimizes the placement of individual PGs in
+   order to achieve a balanced distribution.  In most cases, the resulting
+   distribution is nearly perfect: that is, there is an equal number of PGs on
+   each OSD (±1 PG, since the total number might not divide evenly).
+
+   To use ``upmap``, all clients must be Luminous or newer.
+
+The default mode is ``upmap``. The mode can be changed to ``crush-compat`` by
+running the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer mode crush-compat
+
+Supervised optimization
+-----------------------
+
+Supervised use of the balancer can be understood in terms of three distinct
+phases:
+
+#. building a plan
+#. evaluating the quality of the data distribution, either for the current PG
+   distribution or for the PG distribution that would result after executing a
+   plan
+#. executing the plan
+
+To evaluate the current distribution, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer eval
+
+To evaluate the distribution for a single pool, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer eval <pool-name>
+
+To see the evaluation in greater detail, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer eval-verbose ...
+
+To instruct the balancer to generate a plan (using the currently configured
+mode), make up a name (any useful identifying string) for the plan, and run the
+following command:
+
+   .. prompt:: bash $
+
+      ceph balancer optimize <plan-name>
+
+To see the contents of a plan, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer show <plan-name>
+
+To display all plans, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer ls
+
+To discard an old plan, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer rm <plan-name>
+
+To see currently recorded plans, examine the output of the following status
+command:
+
+   .. prompt:: bash $
+
+      ceph balancer status
+
+To evaluate the distribution that would result from executing a specific plan,
+run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer eval <plan-name>
+
+If a plan is expected to improve the distribution (that is, the plan's score is
+lower than the current cluster state's score), you can execute that plan by
+running the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer execute <plan-name>
diff --git a/doc/rados/operations/bluestore-migration.rst b/doc/rados/operations/bluestore-migration.rst
new file mode 100644
index 000000000..d24782c46
--- /dev/null
+++ b/doc/rados/operations/bluestore-migration.rst
@@ -0,0 +1,357 @@
+.. _rados_operations_bluestore_migration:
+
+=====================
+ BlueStore Migration
+=====================
+.. warning:: Filestore has been deprecated in the Reef release and is no longer supported.
+	     Please migrate to BlueStore.
+
+Each OSD must be formatted as either Filestore or BlueStore. However, a Ceph
+cluster can operate with a mixture of both Filestore OSDs and BlueStore OSDs.
+Because BlueStore is superior to Filestore in performance and robustness, and
+because Filestore is not supported by Ceph releases beginning with Reef, users
+deploying Filestore OSDs should transition to BlueStore. There are several
+strategies for making the transition to BlueStore.
+
+BlueStore is so different from Filestore that an individual OSD cannot be
+converted in place. Instead, the conversion process must use either (1) the
+cluster's normal replication and healing support, or (2) tools and strategies
+that copy OSD content from an old (Filestore) device to a new (BlueStore) one.
+
+Deploying new OSDs with BlueStore
+=================================
+
+Use BlueStore when deploying new OSDs (for example, when the cluster is
+expanded). Because this is the default behavior, no specific change is
+needed.
+
+Similarly, use BlueStore for any OSDs that have been reprovisioned after
+a failed drive was replaced.
+
+Converting existing OSDs
+========================
+
+"Mark-``out``" replacement
+--------------------------
+
+The simplest approach is to verify that the cluster is healthy and
+then follow these steps for each Filestore OSD in succession: mark the OSD
+``out``, wait for the data to replicate across the cluster, reprovision the OSD, 
+mark the OSD back ``in``, and wait for recovery to complete before proceeding
+to the next OSD. This approach is easy to automate, but it entails unnecessary
+data migration that carries costs in time and SSD wear.
+
+#. Identify a Filestore OSD to replace::
+
+     ID=<osd-id-number>
+     DEVICE=<disk-device>
+
+   #. Determine whether a given OSD is Filestore or BlueStore:
+
+      .. prompt:: bash $
+
+         ceph osd metadata $ID | grep osd_objectstore
+
+   #. Get a current count of Filestore and BlueStore OSDs:
+
+      .. prompt:: bash $
+
+         ceph osd count-metadata osd_objectstore
+
+#. Mark a Filestore OSD ``out``:
+
+   .. prompt:: bash $
+
+      ceph osd out $ID
+
+#. Wait for the data to migrate off this OSD:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy $ID ; do sleep 60 ; done
+
+#. Stop the OSD:
+
+   .. prompt:: bash $
+
+      systemctl kill ceph-osd@$ID
+
+   .. _osd_id_retrieval: 
+
+#. Note which device the OSD is using:
+
+   .. prompt:: bash $
+
+      mount | grep /var/lib/ceph/osd/ceph-$ID
+
+#. Unmount the OSD:
+
+   .. prompt:: bash $
+
+      umount /var/lib/ceph/osd/ceph-$ID
+
+#. Destroy the OSD's data. Be *EXTREMELY CAREFUL*! These commands will destroy
+   the contents of the device; you must be certain that the data on the device is
+   not needed (in other words, that the cluster is healthy) before proceeding:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap $DEVICE
+
+#. Tell the cluster that the OSD has been destroyed (and that a new OSD can be
+   reprovisioned with the same OSD ID):
+
+   .. prompt:: bash $
+
+      ceph osd destroy $ID --yes-i-really-mean-it
+
+#. Provision a BlueStore OSD in place by using the same OSD ID. This requires
+   you to identify which device to wipe, and to make certain that you target
+   the correct and intended device, using the information that was retrieved in
+   the :ref:`"Note which device the OSD is using" <osd_id_retrieval>` step.  BE
+   CAREFUL!  Note that you may need to modify these commands when dealing with
+   hybrid OSDs:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --bluestore --data $DEVICE --osd-id $ID
+
+#. Repeat.
+
+You may opt to (1) have the balancing of the replacement BlueStore OSD take
+place concurrently with the draining of the next Filestore OSD, or instead
+(2) follow the same procedure for multiple OSDs in parallel. In either case,
+however, you must ensure that the cluster is fully clean (in other words, that
+all data has all replicas) before destroying any OSDs. If you opt to reprovision
+multiple OSDs in parallel, be **very** careful to destroy OSDs only within a
+single CRUSH failure domain (for example, ``host`` or ``rack``). Failure to
+satisfy this requirement will reduce the redundancy and availability of your
+data and increase the risk of data loss (or even guarantee data loss).
+
+Advantages:
+
+* Simple.
+* Can be done on a device-by-device basis.
+* No spare devices or hosts are required.
+
+Disadvantages:
+
+* Data is copied over the network twice: once to another OSD in the cluster (to
+  maintain the specified number of replicas), and again back to the
+  reprovisioned BlueStore OSD.
+
+"Whole host" replacement
+------------------------
+
+If you have a spare host in the cluster, or sufficient free space to evacuate
+an entire host for use as a spare, then the conversion can be done on a
+host-by-host basis so that each stored copy of the data is migrated only once.
+
+To use this approach, you need an empty host that has no OSDs provisioned.
+There are two ways to do this: either by using a new, empty host that is not
+yet part of the cluster, or by offloading data from an existing host that is
+already part of the cluster.
+
+Using a new, empty host
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Ideally the host will have roughly the same capacity as each of the other hosts
+you will be converting.  Add the host to the CRUSH hierarchy, but do not attach
+it to the root:
+
+
+.. prompt:: bash $
+
+   NEWHOST=<empty-host-name>
+   ceph osd crush add-bucket $NEWHOST host
+
+Make sure that Ceph packages are installed on the new host.
+
+Using an existing host
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to use an existing host that is already part of the cluster,
+and if there is sufficient free space on that host so that all of its data can
+be migrated off to other cluster hosts, you can do the following (instead of
+using a new, empty host):
+
+.. prompt:: bash $
+
+   OLDHOST=<existing-cluster-host-to-offload>
+   ceph osd crush unlink $OLDHOST default
+
+where "default" is the immediate ancestor in the CRUSH map. (For
+smaller clusters with unmodified configurations this will normally
+be "default", but it might instead be a rack name.) You should now
+see the host at the top of the OSD tree output with no parent:
+
+.. prompt:: bash $
+
+   bin/ceph osd tree
+
+::
+
+  ID CLASS WEIGHT  TYPE NAME     STATUS REWEIGHT PRI-AFF
+  -5             0 host oldhost
+  10   ssd 1.00000     osd.10        up  1.00000 1.00000
+  11   ssd 1.00000     osd.11        up  1.00000 1.00000
+  12   ssd 1.00000     osd.12        up  1.00000 1.00000
+  -1       3.00000 root default
+  -2       3.00000     host foo
+   0   ssd 1.00000         osd.0     up  1.00000 1.00000
+   1   ssd 1.00000         osd.1     up  1.00000 1.00000
+   2   ssd 1.00000         osd.2     up  1.00000 1.00000
+  ...
+
+If everything looks good, jump directly to the :ref:`"Wait for the data
+migration to complete" <bluestore_data_migration_step>` step below and proceed
+from there to clean up the old OSDs.
+
+Migration process
+^^^^^^^^^^^^^^^^^
+
+If you're using a new host, start at :ref:`the first step
+<bluestore_migration_process_first_step>`. If you're using an existing host,
+jump to :ref:`this step <bluestore_data_migration_step>`.
+
+.. _bluestore_migration_process_first_step:
+
+#. Provision new BlueStore OSDs for all devices:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --bluestore --data /dev/$DEVICE
+
+#. Verify that the new OSDs have joined the cluster:
+
+   .. prompt:: bash $
+
+      ceph osd tree
+
+   You should see the new host ``$NEWHOST`` with all of the OSDs beneath
+   it, but the host should *not* be nested beneath any other node in the
+   hierarchy (like ``root default``).  For example, if ``newhost`` is
+   the empty host, you might see something like::
+
+     $ bin/ceph osd tree
+     ID CLASS WEIGHT  TYPE NAME     STATUS REWEIGHT PRI-AFF
+     -5             0 host newhost
+     10   ssd 1.00000     osd.10        up  1.00000 1.00000
+     11   ssd 1.00000     osd.11        up  1.00000 1.00000
+     12   ssd 1.00000     osd.12        up  1.00000 1.00000
+     -1       3.00000 root default
+     -2       3.00000     host oldhost1
+      0   ssd 1.00000         osd.0     up  1.00000 1.00000
+      1   ssd 1.00000         osd.1     up  1.00000 1.00000
+      2   ssd 1.00000         osd.2     up  1.00000 1.00000
+     ...
+
+#. Identify the first target host to convert :
+
+   .. prompt:: bash $
+
+      OLDHOST=<existing-cluster-host-to-convert>
+
+#. Swap the new host into the old host's position in the cluster:
+
+   .. prompt:: bash $
+
+      ceph osd crush swap-bucket $NEWHOST $OLDHOST
+
+   At this point all data on ``$OLDHOST`` will begin migrating to the OSDs on
+   ``$NEWHOST``.  If there is a difference between the total capacity of the
+   old hosts and the total capacity of the new hosts, you may also see some
+   data migrate to or from other nodes in the cluster. Provided that the hosts
+   are similarly sized, however, this will be a relatively small amount of
+   data.
+
+   .. _bluestore_data_migration_step:
+
+#. Wait for the data migration to complete:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy $(ceph osd ls-tree $OLDHOST); do sleep 60 ; done
+
+#. Stop all old OSDs on the now-empty ``$OLDHOST``:
+
+   .. prompt:: bash $
+
+      ssh $OLDHOST
+      systemctl kill ceph-osd.target
+      umount /var/lib/ceph/osd/ceph-*
+
+#. Destroy and purge the old OSDs:
+
+   .. prompt:: bash $
+
+      for osd in `ceph osd ls-tree $OLDHOST`; do
+         ceph osd purge $osd --yes-i-really-mean-it
+      done
+
+#. Wipe the old OSDs. This requires you to identify which devices are to be
+   wiped manually. BE CAREFUL! For each device:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap $DEVICE
+
+#. Use the now-empty host as the new host, and repeat:
+
+   .. prompt:: bash $
+
+      NEWHOST=$OLDHOST
+
+Advantages:
+
+* Data is copied over the network only once.
+* An entire host's OSDs are converted at once.
+* Can be parallelized, to make possible the conversion of multiple hosts at the same time.
+* No host involved in this process needs to have a spare device.
+
+Disadvantages:
+
+* A spare host is required.
+* An entire host's worth of OSDs will be migrating data at a time. This
+  is likely to impact overall cluster performance.
+* All migrated data still makes one full hop over the network.
+
+Per-OSD device copy
+-------------------
+A single logical OSD can be converted by using the ``copy`` function
+included in ``ceph-objectstore-tool``. This requires that the host have one or more free
+devices to provision a new, empty BlueStore OSD. For
+example, if each host in your cluster has twelve OSDs, then you need a
+thirteenth unused OSD so that each OSD can be converted before the
+previous OSD is reclaimed to convert the next OSD.
+
+Caveats:
+
+* This approach requires that we prepare an empty BlueStore OSD but that we do not allocate
+  a new OSD ID to it. The ``ceph-volume`` tool does not support such an operation. **IMPORTANT:**
+  because the setup of *dmcrypt* is closely tied to the identity of the OSD, this approach does not
+  work with encrypted OSDs.
+
+* The device must be manually partitioned.
+
+* An unsupported user-contributed script that demonstrates this process may be found here:
+  https://github.com/ceph/ceph/blob/master/src/script/contrib/ceph-migrate-bluestore.bash
+
+Advantages:
+
+* Provided that the 'noout' or the 'norecover'/'norebalance' flags are set on the OSD or the
+  cluster while the conversion process is underway, little or no data migrates over the
+  network during the conversion.
+
+Disadvantages:
+
+* Tooling is not fully implemented, supported, or documented.
+  
+* Each host must have an appropriate spare or empty device for staging.
+  
+* The OSD is offline during the conversion, which means new writes to PGs
+  with the OSD in their acting set may not be ideally redundant until the
+  subject OSD comes up and recovers. This increases the risk of data
+  loss due to an overlapping failure. However, if another OSD fails before
+  conversion and startup have completed, the original Filestore OSD can be
+  started to provide access to its original data.
diff --git a/doc/rados/operations/cache-tiering.rst b/doc/rados/operations/cache-tiering.rst
new file mode 100644
index 000000000..127b0141f
--- /dev/null
+++ b/doc/rados/operations/cache-tiering.rst
@@ -0,0 +1,557 @@
+===============
+ Cache Tiering
+===============
+
+.. warning:: Cache tiering has been deprecated in the Reef release as it
+             has lacked a maintainer for a very long time. This does not mean
+             it will be certainly removed, but we may choose to remove it
+             without much further notice.
+
+A cache tier provides Ceph Clients with better I/O performance for a subset of
+the data stored in a backing storage tier. Cache tiering involves creating a
+pool of relatively fast/expensive storage devices (e.g., solid state drives)
+configured to act as a cache tier, and a backing pool of either erasure-coded
+or relatively slower/cheaper devices configured to act as an economical storage
+tier. The Ceph objecter handles where to place the objects and the tiering
+agent determines when to flush objects from the cache to the backing storage
+tier. So the cache tier and the backing storage tier are completely transparent 
+to Ceph clients.
+
+
+.. ditaa::
+           +-------------+
+           | Ceph Client |
+           +------+------+
+                  ^
+     Tiering is   |  
+    Transparent   |              Faster I/O
+        to Ceph   |           +---------------+
+     Client Ops   |           |               |   
+                  |    +----->+   Cache Tier  |
+                  |    |      |               |
+                  |    |      +-----+---+-----+
+                  |    |            |   ^ 
+                  v    v            |   |   Active Data in Cache Tier
+           +------+----+--+         |   |
+           |   Objecter   |         |   |
+           +-----------+--+         |   |
+                       ^            |   |   Inactive Data in Storage Tier
+                       |            v   |
+                       |      +-----+---+-----+
+                       |      |               |
+                       +----->|  Storage Tier |
+                              |               |
+                              +---------------+
+                                 Slower I/O
+
+
+The cache tiering agent handles the migration of data between the cache tier 
+and the backing storage tier automatically. However, admins have the ability to
+configure how this migration takes place by setting the ``cache-mode``. There are
+two main scenarios:
+
+- **writeback** mode: If the base tier and the cache tier are configured in
+  ``writeback`` mode, Ceph clients receive an ACK from the base tier every time
+  they write data to it. Then the cache tiering agent determines whether
+  ``osd_tier_default_cache_min_write_recency_for_promote`` has been set. If it
+  has been set and the data has been written more than a specified number of
+  times per interval, the data is promoted to the cache tier.
+
+  When Ceph clients need access to data stored in the base tier, the cache
+  tiering agent reads the data from the base tier and returns it to the client.
+  While data is being read from the base tier, the cache tiering agent consults
+  the value of ``osd_tier_default_cache_min_read_recency_for_promote`` and
+  decides whether to promote that data from the base tier to the cache tier.
+  When data has been promoted from the base tier to the cache tier, the Ceph
+  client is able to perform I/O operations on it using the cache tier. This is
+  well-suited for mutable data (for example, photo/video editing, transactional
+  data).
+
+- **readproxy** mode: This mode will use any objects that already
+  exist in the cache tier, but if an object is not present in the
+  cache the request will be proxied to the base tier.  This is useful
+  for transitioning from ``writeback`` mode to a disabled cache as it
+  allows the workload to function properly while the cache is drained,
+  without adding any new objects to the cache.
+
+Other cache modes are:
+
+- **readonly** promotes objects to the cache on read operations only; write
+  operations are forwarded to the base tier. This mode is intended for
+  read-only workloads that do not require consistency to be enforced by the
+  storage system. (**Warning**: when objects are updated in the base tier,
+  Ceph makes **no** attempt to sync these updates to the corresponding objects
+  in the cache. Since this mode is considered experimental, a
+  ``--yes-i-really-mean-it`` option must be passed in order to enable it.)
+
+- **none** is used to completely disable caching.
+
+
+A word of caution
+=================
+
+Cache tiering will *degrade* performance for most workloads.  Users should use
+extreme caution before using this feature.
+
+* *Workload dependent*: Whether a cache will improve performance is
+  highly dependent on the workload.  Because there is a cost
+  associated with moving objects into or out of the cache, it can only
+  be effective when there is a *large skew* in the access pattern in
+  the data set, such that most of the requests touch a small number of
+  objects.  The cache pool should be large enough to capture the
+  working set for your workload to avoid thrashing.
+
+* *Difficult to benchmark*: Most benchmarks that users run to measure
+  performance will show terrible performance with cache tiering, in
+  part because very few of them skew requests toward a small set of
+  objects, it can take a long time for the cache to "warm up," and
+  because the warm-up cost can be high.
+
+* *Usually slower*: For workloads that are not cache tiering-friendly,
+  performance is often slower than a normal RADOS pool without cache
+  tiering enabled.
+
+* *librados object enumeration*: The librados-level object enumeration
+  API is not meant to be coherent in the presence of the case.  If
+  your application is using librados directly and relies on object
+  enumeration, cache tiering will probably not work as expected.
+  (This is not a problem for RGW, RBD, or CephFS.)
+
+* *Complexity*: Enabling cache tiering means that a lot of additional
+  machinery and complexity within the RADOS cluster is being used.
+  This increases the probability that you will encounter a bug in the system
+  that other users have not yet encountered and will put your deployment at a
+  higher level of risk.
+
+Known Good Workloads
+--------------------
+
+* *RGW time-skewed*: If the RGW workload is such that almost all read
+  operations are directed at recently written objects, a simple cache
+  tiering configuration that destages recently written objects from
+  the cache to the base tier after a configurable period can work
+  well.
+
+Known Bad Workloads
+-------------------
+
+The following configurations are *known to work poorly* with cache
+tiering.
+
+* *RBD with replicated cache and erasure-coded base*: This is a common
+  request, but usually does not perform well.  Even reasonably skewed
+  workloads still send some small writes to cold objects, and because
+  small writes are not yet supported by the erasure-coded pool, entire
+  (usually 4 MB) objects must be migrated into the cache in order to
+  satisfy a small (often 4 KB) write.  Only a handful of users have
+  successfully deployed this configuration, and it only works for them
+  because their data is extremely cold (backups) and they are not in
+  any way sensitive to performance.
+
+* *RBD with replicated cache and base*: RBD with a replicated base
+  tier does better than when the base is erasure coded, but it is
+  still highly dependent on the amount of skew in the workload, and
+  very difficult to validate.  The user will need to have a good
+  understanding of their workload and will need to tune the cache
+  tiering parameters carefully.
+
+
+Setting Up Pools
+================
+
+To set up cache tiering, you must have two pools. One will act as the 
+backing storage and the other will act as the cache.
+
+
+Setting Up a Backing Storage Pool
+---------------------------------
+
+Setting up a backing storage pool typically involves one of two scenarios: 
+
+- **Standard Storage**: In this scenario, the pool stores multiple copies
+  of an object in the Ceph Storage Cluster.
+
+- **Erasure Coding:** In this scenario, the pool uses erasure coding to 
+  store data much more efficiently with a small performance tradeoff.
+
+In the standard storage scenario, you can setup a CRUSH rule to establish 
+the failure domain (e.g., osd, host, chassis, rack, row, etc.). Ceph OSD 
+Daemons perform optimally when all storage drives in the rule are of the 
+same size, speed (both RPMs and throughput) and type. See `CRUSH Maps`_ 
+for details on creating a rule. Once you have created a rule, create 
+a backing storage pool. 
+
+In the erasure coding scenario, the pool creation arguments will generate the
+appropriate rule automatically. See `Create a Pool`_ for details.
+
+In subsequent examples, we will refer to the backing storage pool 
+as ``cold-storage``.
+
+
+Setting Up a Cache Pool
+-----------------------
+
+Setting up a cache pool follows the same procedure as the standard storage
+scenario, but with this difference: the drives for the cache tier are typically
+high performance drives that reside in their own servers and have their own
+CRUSH rule.  When setting up such a rule, it should take account of the hosts
+that have the high performance drives while omitting the hosts that don't. See
+:ref:`CRUSH Device Class<crush-map-device-class>` for details.
+
+
+In subsequent examples, we will refer to the cache pool as ``hot-storage`` and
+the backing pool as ``cold-storage``.
+
+For cache tier configuration and default values, see 
+`Pools - Set Pool Values`_.
+
+
+Creating a Cache Tier
+=====================
+
+Setting up a cache tier involves associating a backing storage pool with
+a cache pool:
+
+.. prompt:: bash $
+
+   ceph osd tier add {storagepool} {cachepool}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier add cold-storage hot-storage
+
+To set the cache mode, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd tier cache-mode {cachepool} {cache-mode}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier cache-mode hot-storage writeback
+
+The cache tiers overlay the backing storage tier, so they require one
+additional step: you must direct all client traffic from the storage pool to 
+the cache pool. To direct client traffic directly to the cache pool, execute 
+the following:
+
+.. prompt:: bash $
+
+   ceph osd tier set-overlay {storagepool} {cachepool}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier set-overlay cold-storage hot-storage
+
+
+Configuring a Cache Tier
+========================
+
+Cache tiers have several configuration options. You may set
+cache tier configuration options with the following usage:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} {key} {value}
+   
+See `Pools - Set Pool Values`_ for details.
+
+
+Target Size and Type
+--------------------
+
+Ceph's production cache tiers use a `Bloom Filter`_ for the ``hit_set_type``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} hit_set_type bloom
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage hit_set_type bloom
+
+The ``hit_set_count`` and ``hit_set_period`` define how many such HitSets to
+store, and how much time each HitSet should cover:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} hit_set_count 12
+   ceph osd pool set {cachepool} hit_set_period 14400
+   ceph osd pool set {cachepool} target_max_bytes 1000000000000
+
+.. note:: A larger ``hit_set_count`` results in more RAM consumed by
+          the ``ceph-osd`` process.
+
+Binning accesses over time allows Ceph to determine whether a Ceph client
+accessed an object at least once, or more than once over a time period 
+("age" vs "temperature").
+
+The ``min_read_recency_for_promote`` defines how many HitSets to check for the
+existence of an object when handling a read operation. The checking result is
+used to decide whether to promote the object asynchronously. Its value should be
+between 0 and ``hit_set_count``. If it's set to 0, the object is always promoted.
+If it's set to 1, the current HitSet is checked. And if this object is in the
+current HitSet, it's promoted. Otherwise not. For the other values, the exact
+number of archive HitSets are checked. The object is promoted if the object is
+found in any of the most recent ``min_read_recency_for_promote`` HitSets.
+
+A similar parameter can be set for the write operation, which is
+``min_write_recency_for_promote``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} min_read_recency_for_promote 2
+   ceph osd pool set {cachepool} min_write_recency_for_promote 2
+
+.. note:: The longer the period and the higher the
+   ``min_read_recency_for_promote`` and
+   ``min_write_recency_for_promote``values, the more RAM the ``ceph-osd``
+   daemon consumes. In particular, when the agent is active to flush
+   or evict cache objects, all ``hit_set_count`` HitSets are loaded
+   into RAM.
+
+
+Cache Sizing
+------------
+
+The cache tiering agent performs two main functions: 
+
+- **Flushing:** The agent identifies modified (or dirty) objects and forwards
+  them to the storage pool for long-term storage.
+  
+- **Evicting:** The agent identifies objects that haven't been modified 
+  (or clean) and evicts the least recently used among them from the cache.
+
+
+Absolute Sizing
+~~~~~~~~~~~~~~~
+
+The cache tiering agent can flush or evict objects based upon the total number
+of bytes or the total number of objects. To specify a maximum number of bytes,
+execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} target_max_bytes {#bytes}
+
+For example, to flush or evict at 1 TB, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage target_max_bytes 1099511627776
+
+To specify the maximum number of objects, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} target_max_objects {#objects}
+
+For example, to flush or evict at 1M objects, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage target_max_objects 1000000
+
+.. note:: Ceph is not able to determine the size of a cache pool automatically, so
+   the configuration on the absolute size is required here, otherwise the
+   flush/evict will not work. If you specify both limits, the cache tiering
+   agent will begin flushing or evicting when either threshold is triggered.
+
+.. note:: All client requests will be blocked only when  ``target_max_bytes`` or
+   ``target_max_objects`` reached
+
+Relative Sizing
+~~~~~~~~~~~~~~~
+
+The cache tiering agent can flush or evict objects relative to the size of the
+cache pool(specified by ``target_max_bytes`` / ``target_max_objects`` in
+`Absolute sizing`_).  When the cache pool consists of a certain percentage of
+modified (or dirty) objects, the cache tiering agent will flush them to the
+storage pool. To set the ``cache_target_dirty_ratio``, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_dirty_ratio {0.0..1.0}
+
+For example, setting the value to ``0.4`` will begin flushing modified
+(dirty) objects when they reach 40% of the cache pool's capacity:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_dirty_ratio 0.4
+
+When the dirty objects reaches a certain percentage of its capacity, flush dirty
+objects with a higher speed. To set the ``cache_target_dirty_high_ratio``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_dirty_high_ratio {0.0..1.0}
+
+For example, setting the value to ``0.6`` will begin aggressively flush dirty
+objects when they reach 60% of the cache pool's capacity. obviously, we'd
+better set the value between dirty_ratio and full_ratio:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_dirty_high_ratio 0.6
+
+When the cache pool reaches a certain percentage of its capacity, the cache
+tiering agent will evict objects to maintain free capacity. To set the 
+``cache_target_full_ratio``, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_full_ratio {0.0..1.0}
+
+For example, setting the value to ``0.8`` will begin flushing unmodified
+(clean) objects when they reach 80% of the cache pool's capacity:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_full_ratio 0.8
+
+
+Cache Age
+---------
+
+You can specify the minimum age of an object before the cache tiering agent 
+flushes a recently modified (or dirty) object to the backing storage pool:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_min_flush_age {#seconds}
+
+For example, to flush modified (or dirty) objects after 10 minutes, execute the
+following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_min_flush_age 600
+
+You can specify the minimum age of an object before it will be evicted from the
+cache tier:
+
+.. prompt:: bash $
+
+   ceph osd pool {cache-tier} cache_min_evict_age {#seconds}
+
+For example, to evict objects after 30 minutes, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_min_evict_age 1800
+
+
+Removing a Cache Tier
+=====================
+
+Removing a cache tier differs depending on whether it is a writeback 
+cache or a read-only cache.
+
+
+Removing a Read-Only Cache
+--------------------------
+
+Since a read-only cache does not have modified data, you can disable
+and remove it without losing any recent changes to objects in the cache. 
+
+#. Change the cache-mode to ``none`` to disable it.:
+
+   .. prompt:: bash 
+
+      ceph osd tier cache-mode {cachepool} none
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode hot-storage none
+
+#. Remove the cache pool from the backing pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove {storagepool} {cachepool}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove cold-storage hot-storage
+
+
+Removing a Writeback Cache
+--------------------------
+
+Since a writeback cache may have modified data, you must take steps to ensure 
+that you do not lose any recent changes to objects in the cache before you 
+disable and remove it.
+
+
+#. Change the cache mode to ``proxy`` so that new and modified objects will 
+   flush to the backing storage pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode {cachepool} proxy
+
+   For example: 
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode hot-storage proxy
+
+
+#. Ensure that the cache pool has been flushed. This may take a few minutes:
+
+   .. prompt:: bash $
+
+      rados -p {cachepool} ls
+
+   If the cache pool still has objects, you can flush them manually. 
+   For example:
+
+   .. prompt:: bash $
+
+      rados -p {cachepool} cache-flush-evict-all
+
+
+#. Remove the overlay so that clients will not direct traffic to the cache.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove-overlay {storagetier}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove-overlay cold-storage
+
+
+#. Finally, remove the cache tier pool from the backing storage pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove {storagepool} {cachepool} 
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove cold-storage hot-storage
+
+
+.. _Create a Pool: ../pools#create-a-pool
+.. _Pools - Set Pool Values: ../pools#set-pool-values
+.. _Bloom Filter: https://en.wikipedia.org/wiki/Bloom_filter
+.. _CRUSH Maps: ../crush-map
+.. _Absolute Sizing: #absolute-sizing
diff --git a/doc/rados/operations/change-mon-elections.rst b/doc/rados/operations/change-mon-elections.rst
new file mode 100644
index 000000000..7418ea363
--- /dev/null
+++ b/doc/rados/operations/change-mon-elections.rst
@@ -0,0 +1,100 @@
+.. _changing_monitor_elections:
+
+=======================================
+Configuring Monitor Election Strategies
+=======================================
+
+By default, the monitors are in ``classic`` mode. We recommend staying in this
+mode unless you have a very specific reason.
+
+If you want to switch modes BEFORE constructing the cluster, change the ``mon
+election default strategy`` option. This option takes an integer value:
+
+* ``1`` for ``classic``
+* ``2`` for ``disallow``
+* ``3`` for ``connectivity``
+
+After your cluster has started running, you can change strategies by running a
+command of the following form:
+
+  $ ceph mon set election_strategy {classic|disallow|connectivity}
+
+Choosing a mode
+===============
+
+The modes other than ``classic`` provide specific features. We recommend staying
+in ``classic`` mode if you don't need these extra features because it is the
+simplest mode.
+
+.. _rados_operations_disallow_mode:
+
+Disallow Mode
+=============
+
+The ``disallow`` mode allows you to mark monitors as disallowed. Disallowed
+monitors participate in the quorum and serve clients, but cannot be elected
+leader. You might want to use this mode for monitors that are far away from
+clients.
+
+To disallow a monitor from being elected leader, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph mon add disallowed_leader {name}
+
+To remove a monitor from the disallowed list and allow it to be elected leader,
+run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph mon rm disallowed_leader {name}
+
+To see the list of disallowed leaders, examine the output of the following
+command:
+
+.. prompt:: bash $
+
+   ceph mon dump
+
+Connectivity Mode
+=================
+
+The ``connectivity`` mode evaluates connection scores that are provided by each
+monitor for its peers and elects the monitor with the highest score. This mode
+is designed to handle network partitioning (also called *net-splits*): network
+partitioning might occur if your cluster is stretched across multiple data
+centers or otherwise has a non-uniform or unbalanced network topology.
+
+The ``connectivity`` mode also supports disallowing monitors from being elected
+leader by using the same commands that were presented in :ref:`Disallow Mode <rados_operations_disallow_mode>`.
+
+Examining connectivity scores
+=============================
+
+The monitors maintain connection scores even if they aren't in ``connectivity``
+mode. To examine a specific monitor's connection scores, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph daemon mon.{name} connection scores dump
+
+Scores for an individual connection range from ``0`` to ``1`` inclusive and
+include whether the connection is considered alive or dead (as determined by
+whether it returned its latest ping before timeout).
+
+Connectivity scores are expected to remain valid. However, if during
+troubleshooting you determine that these scores have for some reason become
+invalid, drop the history and reset the scores by running a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph daemon mon.{name} connection scores reset
+
+Resetting connectivity scores carries little risk: monitors will still quickly
+determine whether a connection is alive or dead and trend back to the previous
+scores if those scores were accurate. Nevertheless, resetting scores ought to
+be unnecessary and it is not recommended unless advised by your support team
+or by a developer.
diff --git a/doc/rados/operations/control.rst b/doc/rados/operations/control.rst
new file mode 100644
index 000000000..033f831cd
--- /dev/null
+++ b/doc/rados/operations/control.rst
@@ -0,0 +1,665 @@
+.. index:: control, commands
+
+==================
+ Control Commands
+==================
+
+
+Monitor Commands
+================
+
+To issue monitor commands, use the ``ceph`` utility:
+
+.. prompt:: bash $
+
+   ceph [-m monhost] {command}
+
+In most cases, monitor commands have the following form:
+
+.. prompt:: bash $
+
+   ceph {subsystem} {command}
+
+
+System Commands
+===============
+
+To display the current cluster status, run the following commands:
+
+.. prompt:: bash $
+
+   ceph -s
+   ceph status
+
+To display a running summary of cluster status and major events, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph -w
+
+To display the monitor quorum, including which monitors are participating and
+which one is the leader, run the following commands:
+
+.. prompt:: bash $
+
+   ceph mon stat
+   ceph quorum_status
+
+To query the status of a single monitor, including whether it is in the quorum,
+run the following command:
+
+.. prompt:: bash $
+
+   ceph tell mon.[id] mon_status
+
+Here the value of ``[id]`` can be found by consulting the output of ``ceph
+-s``.
+
+
+Authentication Subsystem
+========================
+
+To add an OSD keyring for a specific OSD, run the following command:
+
+.. prompt:: bash $
+
+   ceph auth add {osd} {--in-file|-i} {path-to-osd-keyring}
+
+To list the cluster's keys and their capabilities, run the following command:
+
+.. prompt:: bash $
+
+   ceph auth ls
+
+
+Placement Group Subsystem
+=========================
+
+To display the statistics for all placement groups (PGs), run the following
+command:
+
+.. prompt:: bash $
+
+   ceph pg dump [--format {format}]
+
+Here the valid formats are ``plain`` (default), ``json`` ``json-pretty``,
+``xml``, and ``xml-pretty``.  When implementing monitoring tools and other
+tools, it is best to use the ``json`` format.  JSON parsing is more
+deterministic than the ``plain`` format (which is more human readable), and the
+layout is much more consistent from release to release. The ``jq`` utility is
+very useful for extracting data from JSON output.
+
+To display the statistics for all PGs stuck in a specified state, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph pg dump_stuck inactive|unclean|stale|undersized|degraded [--format {format}] [-t|--threshold {seconds}]
+
+Here ``--format`` may be ``plain`` (default), ``json``, ``json-pretty``,
+``xml``, or ``xml-pretty``.
+
+The ``--threshold`` argument determines the time interval (in seconds) for a PG
+to be considered ``stuck`` (default: 300).
+
+PGs might be stuck in any of the following states:
+
+**Inactive** 
+
+    PGs are unable to process reads or writes because they are waiting for an
+    OSD that has the most up-to-date data to return to an ``up`` state.
+
+
+**Unclean** 
+
+    PGs contain objects that have not been replicated the desired number of
+    times. These PGs have not yet completed the process of recovering.
+
+
+**Stale** 
+
+    PGs are in an unknown state, because the OSDs that host them have not
+    reported to the monitor cluster for a certain period of time (specified by
+    the ``mon_osd_report_timeout`` configuration setting).
+
+
+To delete a ``lost`` object or revert an object to its prior state, either by
+reverting it to its previous version or by deleting it because it was just
+created and has no previous version, run the following command:
+
+.. prompt:: bash $
+
+   ceph pg {pgid} mark_unfound_lost revert|delete
+
+
+.. _osd-subsystem:
+
+OSD Subsystem
+=============
+
+To query OSD subsystem status, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd stat
+
+To write a copy of the most recent OSD map to a file (see :ref:`osdmaptool
+<osdmaptool>`), run the following command:
+
+.. prompt:: bash $
+
+   ceph osd getmap -o file
+
+To write a copy of the CRUSH map from the most recent OSD map to a file, run
+the following command:
+
+.. prompt:: bash $
+
+   ceph osd getcrushmap -o file
+
+Note that this command is functionally equivalent to the following two
+commands:
+
+.. prompt:: bash $
+
+   ceph osd getmap -o /tmp/osdmap
+   osdmaptool /tmp/osdmap --export-crush file
+
+To dump the OSD map, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd dump [--format {format}]
+
+The ``--format`` option accepts the following arguments: ``plain`` (default),
+``json``, ``json-pretty``, ``xml``, and ``xml-pretty``. As noted above, JSON is
+the recommended format for tools, scripting, and other forms of automation. 
+
+To dump the OSD map as a tree that lists one OSD per line and displays
+information about the weights and states of the OSDs, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph osd tree [--format {format}]
+
+To find out where a specific RADOS object is stored in the system, run a
+command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd map <pool-name> <object-name>
+
+To add or move a new OSD (specified by its ID, name, or weight) to a specific
+CRUSH location, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush set {id} {weight} [{loc1} [{loc2} ...]]
+
+To remove an existing OSD from the CRUSH map, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {name}
+
+To remove an existing bucket from the CRUSH map, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {bucket-name}
+
+To move an existing bucket from one position in the CRUSH hierarchy to another,
+run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush move {id} {loc1} [{loc2} ...]
+
+To set the CRUSH weight of a specific OSD (specified by ``{name}``) to
+``{weight}``, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush reweight {name} {weight}
+
+To mark an OSD as ``lost``, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd lost {id} [--yes-i-really-mean-it]
+
+.. warning::
+   This could result in permanent data loss. Use with caution!
+
+To create a new OSD, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd create [{uuid}]
+
+If no UUID is given as part of this command, the UUID will be set automatically
+when the OSD starts up.
+
+To remove one or more specific OSDs, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd rm [{id}...]
+
+To display the current ``max_osd`` parameter in the OSD map, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph osd getmaxosd
+
+To import a specific CRUSH map, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd setcrushmap -i file
+
+To set the ``max_osd`` parameter in the OSD map, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd setmaxosd
+
+The parameter has a default value of 10000. Most operators will never need to
+adjust it.
+
+To mark a specific OSD ``down``, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd down {osd-num}
+
+To mark a specific OSD ``out`` (so that no data will be allocated to it), run
+the following command:
+
+.. prompt:: bash $
+
+   ceph osd out {osd-num}
+
+To mark a specific OSD ``in`` (so that data will be allocated to it), run the
+following command:
+
+.. prompt:: bash $
+
+   ceph osd in {osd-num}
+
+By using the "pause flags" in the OSD map, you can pause or unpause I/O
+requests.  If the flags are set, then no I/O requests will be sent to any OSD.
+When the flags are cleared, then pending I/O requests will be resent. To set or
+clear pause flags, run one of the following commands:
+
+.. prompt:: bash $
+
+   ceph osd pause
+   ceph osd unpause
+
+You can assign an override or ``reweight`` weight value to a specific OSD if
+the normal CRUSH distribution seems to be suboptimal. The weight of an OSD
+helps determine the extent of its I/O requests and data storage: two OSDs with
+the same weight will receive approximately the same number of I/O requests and
+store approximately the same amount of data. The ``ceph osd reweight`` command
+assigns an override weight to an OSD. The weight value is in the range 0 to 1,
+and the command forces CRUSH to relocate a certain amount (1 - ``weight``) of
+the data that would otherwise be on this OSD. The command does not change the
+weights of the buckets above the OSD in the CRUSH map. Using the command is
+merely a corrective measure: for example, if one of your OSDs is at 90% and the
+others are at 50%, you could reduce the outlier weight to correct this
+imbalance. To assign an override weight to a specific OSD, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph osd reweight {osd-num} {weight}
+
+.. note:: Any assigned override reweight value will conflict with the balancer.
+   This means that if the balancer is in use, all override reweight values
+   should be ``1.0000`` in order to avoid suboptimal cluster behavior.
+
+A cluster's OSDs can be reweighted in order to maintain balance if some OSDs
+are being disproportionately utilized. Note that override or ``reweight``
+weights have values relative to one another that default to 1.00000; their
+values are not absolute, and these weights must be distinguished from CRUSH
+weights (which reflect the absolute capacity of a bucket, as measured in TiB).
+To reweight OSDs by utilization, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd reweight-by-utilization [threshold [max_change [max_osds]]] [--no-increasing]
+
+By default, this command adjusts the override weight of OSDs that have ±20% of
+the average utilization, but you can specify a different percentage in the
+``threshold`` argument. 
+
+To limit the increment by which any OSD's reweight is to be changed, use the
+``max_change`` argument (default: 0.05). To limit the number of OSDs that are
+to be adjusted, use the ``max_osds`` argument (default: 4). Increasing these
+variables can accelerate the reweighting process, but perhaps at the cost of
+slower client operations (as a result of the increase in data movement).
+
+You can test the ``osd reweight-by-utilization`` command before running it. To
+find out which and how many PGs and OSDs will be affected by a specific use of
+the ``osd reweight-by-utilization`` command, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd test-reweight-by-utilization [threshold [max_change max_osds]] [--no-increasing]
+
+The ``--no-increasing`` option can be added to the ``reweight-by-utilization``
+and ``test-reweight-by-utilization`` commands in order to prevent any override
+weights that are currently less than 1.00000 from being increased. This option
+can be useful in certain circumstances: for example, when you are hastily
+balancing in order to remedy ``full`` or ``nearfull`` OSDs, or when there are
+OSDs being evacuated or slowly brought into service.
+
+Operators of deployments that utilize Nautilus or newer (or later revisions of
+Luminous and Mimic) and that have no pre-Luminous clients might likely instead
+want to enable the `balancer`` module for ``ceph-mgr``.
+
+The blocklist can be modified by adding or removing an IP address or a CIDR
+range. If an address is blocklisted, it will be unable to connect to any OSD.
+If an OSD is contained within an IP address or CIDR range that has been
+blocklisted, the OSD will be unable to perform operations on its peers when it
+acts as a client: such blocked operations include tiering and copy-from
+functionality. To add or remove an IP address or CIDR range to the blocklist,
+run one of the following commands:
+
+.. prompt:: bash $
+
+   ceph osd blocklist ["range"] add ADDRESS[:source_port][/netmask_bits] [TIME]
+   ceph osd blocklist ["range"] rm ADDRESS[:source_port][/netmask_bits]
+
+If you add something to the blocklist with the above ``add`` command, you can
+use the ``TIME`` keyword to specify the length of time (in seconds) that it
+will remain on the blocklist (default: one hour). To add or remove a CIDR
+range, use the ``range`` keyword in the above commands.
+
+Note that these commands are useful primarily in failure testing. Under normal
+conditions, blocklists are maintained automatically and do not need any manual
+intervention.
+
+To create or delete a snapshot of a specific storage pool, run one of the
+following commands:
+
+.. prompt:: bash $
+
+   ceph osd pool mksnap {pool-name} {snap-name}
+   ceph osd pool rmsnap {pool-name} {snap-name}
+
+To create, delete, or rename a specific storage pool, run one of the following
+commands:
+
+.. prompt:: bash $
+
+   ceph osd pool create {pool-name} [pg_num [pgp_num]]
+   ceph osd pool delete {pool-name} [{pool-name} --yes-i-really-really-mean-it]
+   ceph osd pool rename {old-name} {new-name}
+
+To change a pool setting, run the following command: 
+
+.. prompt:: bash $
+
+   ceph osd pool set {pool-name} {field} {value}
+
+The following are valid fields:
+
+    * ``size``: The number of copies of data in the pool.
+    * ``pg_num``: The PG number.
+    * ``pgp_num``: The effective number of PGs when calculating placement.
+    * ``crush_rule``: The rule number for mapping placement.
+
+To retrieve the value of a pool setting, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} {field}
+
+Valid fields are:
+
+    * ``pg_num``: The PG number.
+    * ``pgp_num``: The effective number of PGs when calculating placement.
+
+To send a scrub command to a specific OSD, or to all OSDs (by using ``*``), run
+the following command:
+
+.. prompt:: bash $
+
+   ceph osd scrub {osd-num}
+
+To send a repair command to a specific OSD, or to all OSDs (by using ``*``),
+run the following command:
+
+.. prompt:: bash $
+
+   ceph osd repair N
+
+You can run a simple throughput benchmark test against a specific OSD. This
+test writes a total size of ``TOTAL_DATA_BYTES`` (default: 1 GB) incrementally,
+in multiple write requests that each have a size of ``BYTES_PER_WRITE``
+(default: 4 MB). The test is not destructive and it will not overwrite existing
+live OSD data, but it might temporarily affect the performance of clients that
+are concurrently accessing the OSD. To launch this benchmark test, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph tell osd.N bench [TOTAL_DATA_BYTES] [BYTES_PER_WRITE]
+
+To clear the caches of a specific OSD during the interval between one benchmark
+run and another, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell osd.N cache drop
+
+To retrieve the cache statistics of a specific OSD, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell osd.N cache status
+
+MDS Subsystem
+=============
+
+To change the configuration parameters of a running metadata server, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph tell mds.{mds-id} config set {setting} {value}
+
+Example:
+
+.. prompt:: bash $
+
+   ceph tell mds.0 config set debug_ms 1
+
+To enable debug messages, run the following command:
+
+.. prompt:: bash $
+
+   ceph mds stat
+
+To display the status of all metadata servers, run the following command:
+
+.. prompt:: bash $
+
+   ceph mds fail 0
+
+To mark the active metadata server as failed (and to trigger failover to a
+standby if a standby is present), run the following command:
+
+.. todo:: ``ceph mds`` subcommands missing docs: set, dump, getmap, stop, setmap
+
+
+Mon Subsystem
+=============
+
+To display monitor statistics, run the following command:
+
+.. prompt:: bash $
+
+   ceph mon stat
+
+This command returns output similar to the following:
+
+::
+
+    e2: 3 mons at {a=127.0.0.1:40000/0,b=127.0.0.1:40001/0,c=127.0.0.1:40002/0}, election epoch 6, quorum 0,1,2 a,b,c
+
+There is a ``quorum`` list at the end of the output. It lists those monitor
+nodes that are part of the current quorum.
+
+To retrieve this information in a more direct way, run the following command:
+
+.. prompt:: bash $
+
+   ceph quorum_status -f json-pretty
+
+This command returns output similar to the following:
+
+.. code-block:: javascript    
+
+    {
+        "election_epoch": 6,
+        "quorum": [
+        0,
+        1,
+        2
+        ],
+        "quorum_names": [
+        "a",
+        "b",
+        "c"
+        ],
+        "quorum_leader_name": "a",
+        "monmap": {
+        "epoch": 2,
+        "fsid": "ba807e74-b64f-4b72-b43f-597dfe60ddbc",
+        "modified": "2016-12-26 14:42:09.288066",
+        "created": "2016-12-26 14:42:03.573585",
+        "features": {
+            "persistent": [
+            "kraken"
+            ],
+            "optional": []
+        },
+        "mons": [
+            {
+            "rank": 0,
+            "name": "a",
+            "addr": "127.0.0.1:40000\/0",
+            "public_addr": "127.0.0.1:40000\/0"
+            },
+            {
+            "rank": 1,
+            "name": "b",
+            "addr": "127.0.0.1:40001\/0",
+            "public_addr": "127.0.0.1:40001\/0"
+            },
+            {
+            "rank": 2,
+            "name": "c",
+            "addr": "127.0.0.1:40002\/0",
+            "public_addr": "127.0.0.1:40002\/0"
+            }
+        ]
+        }
+    }
+      
+
+The above will block until a quorum is reached.
+
+To see the status of a specific monitor, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell mon.[name] mon_status
+
+Here the value of ``[name]`` can be found by consulting the output of the
+``ceph quorum_status`` command. This command returns output similar to the
+following:
+
+::
+
+    {
+        "name": "b",
+        "rank": 1,
+        "state": "peon",
+        "election_epoch": 6,
+        "quorum": [
+        0,
+        1,
+        2
+        ],
+        "features": {
+        "required_con": "9025616074522624",
+        "required_mon": [
+            "kraken"
+        ],
+        "quorum_con": "1152921504336314367",
+        "quorum_mon": [
+            "kraken"
+        ]
+        },
+        "outside_quorum": [],
+        "extra_probe_peers": [],
+        "sync_provider": [],
+        "monmap": {
+        "epoch": 2,
+        "fsid": "ba807e74-b64f-4b72-b43f-597dfe60ddbc",
+        "modified": "2016-12-26 14:42:09.288066",
+        "created": "2016-12-26 14:42:03.573585",
+        "features": {
+            "persistent": [
+            "kraken"
+            ],
+            "optional": []
+        },
+        "mons": [
+            {
+            "rank": 0,
+            "name": "a",
+            "addr": "127.0.0.1:40000\/0",
+            "public_addr": "127.0.0.1:40000\/0"
+            },
+            {
+            "rank": 1,
+            "name": "b",
+            "addr": "127.0.0.1:40001\/0",
+            "public_addr": "127.0.0.1:40001\/0"
+            },
+            {
+            "rank": 2,
+            "name": "c",
+            "addr": "127.0.0.1:40002\/0",
+            "public_addr": "127.0.0.1:40002\/0"
+            }
+        ]
+        }
+    }
+
+To see a dump of the monitor state, run the following command:
+
+.. prompt:: bash $
+
+   ceph mon dump
+
+This command returns output similar to the following:
+
+::
+
+    dumped monmap epoch 2
+    epoch 2
+    fsid ba807e74-b64f-4b72-b43f-597dfe60ddbc
+    last_changed 2016-12-26 14:42:09.288066
+    created 2016-12-26 14:42:03.573585
+    0: 127.0.0.1:40000/0 mon.a
+    1: 127.0.0.1:40001/0 mon.b
+    2: 127.0.0.1:40002/0 mon.c
diff --git a/doc/rados/operations/crush-map-edits.rst b/doc/rados/operations/crush-map-edits.rst
new file mode 100644
index 000000000..46a4a4f74
--- /dev/null
+++ b/doc/rados/operations/crush-map-edits.rst
@@ -0,0 +1,746 @@
+Manually editing the CRUSH Map
+==============================
+
+.. note:: Manually editing the CRUSH map is an advanced administrator
+   operation. For the majority of installations, CRUSH changes can be
+   implemented via the Ceph CLI and do not require manual CRUSH map edits. If
+   you have identified a use case where manual edits *are* necessary with a
+   recent Ceph release, consider contacting the Ceph developers at dev@ceph.io
+   so that future versions of Ceph do not have this problem.
+
+To edit an existing CRUSH map, carry out the following procedure:
+
+#. `Get the CRUSH map`_.
+#. `Decompile`_ the CRUSH map.
+#.  Edit at least one of the following sections: `Devices`_, `Buckets`_, and
+    `Rules`_. Use a text editor for this task.
+#. `Recompile`_ the CRUSH map.
+#. `Set the CRUSH map`_.
+
+For details on setting the CRUSH map rule for a specific pool, see `Set Pool
+Values`_.
+
+.. _Get the CRUSH map: #getcrushmap
+.. _Decompile: #decompilecrushmap
+.. _Devices: #crushmapdevices
+.. _Buckets: #crushmapbuckets
+.. _Rules: #crushmaprules
+.. _Recompile: #compilecrushmap
+.. _Set the CRUSH map: #setcrushmap
+.. _Set Pool Values: ../pools#setpoolvalues
+
+.. _getcrushmap:
+
+Get the CRUSH Map
+-----------------
+
+To get the CRUSH map for your cluster, run a command of the following form:
+
+.. prompt:: bash $
+
+    ceph osd getcrushmap -o {compiled-crushmap-filename}
+
+Ceph outputs (``-o``) a compiled CRUSH map to the filename that you have
+specified. Because the CRUSH map is in a compiled form, you must first
+decompile it before you can edit it.
+
+.. _decompilecrushmap:
+
+Decompile the CRUSH Map
+-----------------------
+
+To decompile the CRUSH map, run a command of the following form:
+
+.. prompt:: bash $
+
+    crushtool -d {compiled-crushmap-filename} -o {decompiled-crushmap-filename}
+
+.. _compilecrushmap:
+
+Recompile the CRUSH Map
+-----------------------
+
+To compile the CRUSH map, run a command of the following form:
+
+.. prompt:: bash $
+
+    crushtool -c {decompiled-crushmap-filename} -o {compiled-crushmap-filename}
+
+.. _setcrushmap:
+
+Set the CRUSH Map
+-----------------
+
+To set the CRUSH map for your cluster, run a command of the following form:
+
+.. prompt:: bash $
+
+    ceph osd setcrushmap -i {compiled-crushmap-filename}
+
+Ceph loads (``-i``) a compiled CRUSH map from the filename that you have
+specified.
+
+Sections
+--------
+
+A CRUSH map has six main sections:
+
+#. **tunables:** The preamble at the top of the map describes any *tunables*
+   that are not a part of legacy CRUSH behavior. These tunables correct for old
+   bugs, optimizations, or other changes that have been made over the years to
+   improve CRUSH's behavior.
+
+#. **devices:** Devices are individual OSDs that store data.
+
+#. **types**: Bucket ``types`` define the types of buckets that are used in
+   your CRUSH hierarchy. 
+
+#. **buckets:** Buckets consist of a hierarchical aggregation of storage
+   locations (for example, rows, racks, chassis, hosts) and their assigned
+   weights. After the bucket ``types`` have been defined, the CRUSH map defines
+   each node in the hierarchy, its type, and which devices or other nodes it
+   contains.
+
+#. **rules:** Rules define policy about how data is distributed across
+   devices in the hierarchy.
+
+#. **choose_args:** ``choose_args`` are alternative weights associated with
+   the hierarchy that have been adjusted in order to optimize data placement. A
+   single ``choose_args`` map can be used for the entire cluster, or a number
+   of ``choose_args`` maps can be created such that each map is crafted for a
+   particular pool.
+
+
+.. _crushmapdevices:
+
+CRUSH-Map Devices
+-----------------
+
+Devices are individual OSDs that store data. In this section, there is usually
+one device defined for each OSD daemon in your cluster. Devices are identified
+by an ``id`` (a non-negative integer) and a ``name`` (usually ``osd.N``, where
+``N`` is the device's ``id``).
+
+
+.. _crush-map-device-class:
+
+A device can also have a *device class* associated with it: for example,
+``hdd`` or ``ssd``. Device classes make it possible for devices to be targeted
+by CRUSH rules. This means that device classes allow CRUSH rules to select only
+OSDs that match certain characteristics. For example, you might want an RBD
+pool associated only with SSDs and a different RBD pool associated only with
+HDDs.
+
+To see a list of devices, run the following command:
+
+.. prompt:: bash #
+
+   ceph device ls 
+
+The output of this command takes the following form:
+
+::
+
+    device {num} {osd.name} [class {class}]
+
+For example:
+
+.. prompt:: bash #
+
+    ceph device ls
+
+::
+
+    device 0 osd.0 class ssd
+    device 1 osd.1 class hdd
+    device 2 osd.2
+    device 3 osd.3
+
+In most cases, each device maps to a corresponding ``ceph-osd`` daemon. This
+daemon might map to a single storage device, a pair of devices (for example,
+one for data and one for a journal or metadata), or in some cases a small RAID
+device or a partition of a larger storage device.
+
+
+CRUSH-Map Bucket Types
+----------------------
+
+The second list in the CRUSH map defines 'bucket' types. Buckets facilitate a
+hierarchy of nodes and leaves. Node buckets (also known as non-leaf buckets)
+typically represent physical locations in a hierarchy. Nodes aggregate other
+nodes or leaves. Leaf buckets represent ``ceph-osd`` daemons and their
+corresponding storage media.
+
+.. tip:: In the context of CRUSH, the term "bucket" is used to refer to
+   a node in the hierarchy (that is, to a location or a piece of physical
+   hardware). In the context of RADOS Gateway APIs, however, the term
+   "bucket" has a different meaning.
+
+To add a bucket type to the CRUSH map, create a new line under the list of
+bucket types. Enter ``type`` followed by a unique numeric ID and a bucket name.
+By convention, there is exactly one leaf bucket type and it is ``type 0``;
+however, you may give the leaf bucket any name you like (for example: ``osd``,
+``disk``, ``drive``, ``storage``)::
+
+    # types
+    type {num} {bucket-name}
+
+For example::
+
+    # types
+    type 0 osd
+    type 1 host
+    type 2 chassis
+    type 3 rack
+    type 4 row
+    type 5 pdu
+    type 6 pod
+    type 7 room
+    type 8 datacenter
+    type 9 zone
+    type 10 region
+    type 11 root
+
+.. _crushmapbuckets:
+
+CRUSH-Map Bucket Hierarchy
+--------------------------
+
+The CRUSH algorithm distributes data objects among storage devices according to
+a per-device weight value, approximating a uniform probability distribution.
+CRUSH distributes objects and their replicas according to the hierarchical
+cluster map you define. The CRUSH map represents the available storage devices
+and the logical elements that contain them.
+
+To map placement groups (PGs) to OSDs across failure domains, a CRUSH map
+defines a hierarchical list of bucket types under ``#types`` in the generated
+CRUSH map. The purpose of creating a bucket hierarchy is to segregate the leaf
+nodes according to their failure domains (for example: hosts, chassis, racks,
+power distribution units, pods, rows, rooms, and data centers). With the
+exception of the leaf nodes that represent OSDs, the hierarchy is arbitrary and
+you may define it according to your own needs.
+
+We recommend adapting your CRUSH map to your preferred hardware-naming
+conventions and using bucket names that clearly reflect the physical
+hardware. Clear naming practice can make it easier to administer the cluster
+and easier to troubleshoot problems when OSDs malfunction (or other hardware
+malfunctions) and the administrator needs access to physical hardware.
+
+
+In the following example, the bucket hierarchy has a leaf bucket named ``osd``
+and two node buckets named ``host`` and ``rack``:
+
+.. ditaa::
+                           +-----------+
+                           | {o}rack   |
+                           |   Bucket  |
+                           +-----+-----+
+                                 |
+                 +---------------+---------------+
+                 |                               |
+           +-----+-----+                   +-----+-----+
+           | {o}host   |                   | {o}host   |
+           |   Bucket  |                   |   Bucket  |
+           +-----+-----+                   +-----+-----+
+                 |                               |
+         +-------+-------+               +-------+-------+
+         |               |               |               |
+   +-----+-----+   +-----+-----+   +-----+-----+   +-----+-----+
+   |    osd    |   |    osd    |   |    osd    |   |    osd    |
+   |   Bucket  |   |   Bucket  |   |   Bucket  |   |   Bucket  |
+   +-----------+   +-----------+   +-----------+   +-----------+
+
+.. note:: The higher-numbered ``rack`` bucket type aggregates the
+   lower-numbered ``host`` bucket type.
+
+Because leaf nodes reflect storage devices that have already been declared
+under the ``#devices`` list at the beginning of the CRUSH map, there is no need
+to declare them as bucket instances. The second-lowest bucket type in your
+hierarchy is typically used to aggregate the devices (that is, the
+second-lowest bucket type is usually the computer that contains the storage
+media and, such as ``node``, ``computer``, ``server``, ``host``, or
+``machine``). In high-density environments, it is common to have multiple hosts
+or nodes in a single chassis (for example, in the cases of blades or twins). It
+is important to anticipate the potential consequences of chassis failure -- for
+example, during the replacement of a chassis in case of a node failure, the
+chassis's hosts or nodes (and their associated OSDs) will be in a ``down``
+state.
+
+To declare a bucket instance, do the following: specify its type, give it a
+unique name (an alphanumeric string), assign it a unique ID expressed as a
+negative integer (this is optional), assign it a weight relative to the total
+capacity and capability of the item(s) in the bucket, assign it a bucket
+algorithm (usually ``straw2``), and specify the bucket algorithm's hash
+(usually ``0``, a setting that reflects the hash algorithm ``rjenkins1``). A
+bucket may have one or more items. The items may consist of node buckets or
+leaves. Items may have a weight that reflects the relative weight of the item.
+
+To declare a node bucket, use the following syntax::
+
+    [bucket-type] [bucket-name] {
+        id [a unique negative numeric ID]
+        weight [the relative capacity/capability of the item(s)]
+        alg [the bucket type: uniform | list | tree | straw | straw2 ]
+        hash [the hash type: 0 by default]
+        item [item-name] weight [weight]
+    }
+
+For example, in the above diagram, two host buckets (referred to in the
+declaration below as ``node1`` and ``node2``) and one rack bucket (referred to
+in the declaration below as ``rack1``) are defined. The OSDs are declared as
+items within the host buckets::
+
+    host node1 {
+        id -1
+        alg straw2
+        hash 0
+        item osd.0 weight 1.00
+        item osd.1 weight 1.00
+    }
+
+    host node2 {
+        id -2
+        alg straw2
+        hash 0
+        item osd.2 weight 1.00
+        item osd.3 weight 1.00
+    }
+
+    rack rack1 {
+        id -3
+        alg straw2
+        hash 0
+        item node1 weight 2.00
+        item node2 weight 2.00
+    }
+
+.. note:: In this example, the rack bucket does not contain any OSDs. Instead,
+   it contains lower-level host buckets and includes the sum of their weight in
+   the item entry.
+
+
+.. topic:: Bucket Types
+
+   Ceph supports five bucket types. Each bucket type provides a balance between
+   performance and reorganization efficiency, and each is different from the
+   others. If you are unsure of which bucket type to use, use the ``straw2``
+   bucket. For a more technical discussion of bucket types than is offered
+   here, see **Section 3.4** of `CRUSH - Controlled, Scalable, Decentralized
+   Placement of Replicated Data`_. 
+   
+   The bucket types are as follows:
+
+    #. **uniform**: Uniform buckets aggregate devices that have **exactly** 
+       the same weight. For example, when hardware is commissioned or
+       decommissioned, it is often done in sets of machines that have exactly
+       the same physical configuration (this can be the case, for example,
+       after bulk purchases). When storage devices have exactly the same
+       weight, you may use the ``uniform`` bucket type, which allows CRUSH to
+       map replicas into uniform buckets in constant time. If your devices have
+       non-uniform weights, you should not use the uniform bucket algorithm.
+
+    #. **list**: List buckets aggregate their content as linked lists. The 
+       behavior of list buckets is governed by the :abbr:`RUSH (Replication
+       Under Scalable Hashing)`:sub:`P` algorithm. In the behavior of this
+       bucket type, an object is either relocated to the newest device in
+       accordance with an appropriate probability, or it remains on the older
+       devices as before. This results in optimal data migration when items are
+       added to the bucket. The removal of items from the middle or the tail of
+       the list, however, can result in a significant amount of unnecessary
+       data movement. This means that list buckets are most suitable for
+       circumstances in which they **never shrink or very rarely shrink**.
+
+    #. **tree**: Tree buckets use a binary search tree. They are more efficient
+       at dealing with buckets that contain many items than are list buckets.
+       The behavior of tree buckets is governed by the :abbr:`RUSH (Replication
+       Under Scalable Hashing)`:sub:`R` algorithm. Tree buckets reduce the
+       placement time to 0(log\ :sub:`n`). This means that tree buckets are
+       suitable for managing large sets of devices or nested buckets.
+
+    #. **straw**: Straw buckets allow all items in the bucket to "compete" 
+       against each other for replica placement through a process analogous to
+       drawing straws. This is different from the behavior of list buckets and
+       tree buckets, which use a divide-and-conquer strategy that either gives
+       certain items precedence (for example, those at the beginning of a list)
+       or obviates the need to consider entire subtrees of items. Such an
+       approach improves the performance of the replica placement process, but
+       can also introduce suboptimal reorganization behavior when the contents
+       of a bucket change due an addition, a removal, or the re-weighting of an
+       item.
+
+        * **straw2**: Straw2 buckets improve on Straw by correctly avoiding 
+          any data movement between items when neighbor weights change. For
+          example, if the weight of a given item changes (including during the
+          operations of adding it to the cluster or removing it from the
+          cluster), there will be data movement to or from only that item.
+          Neighbor weights are not taken into account.
+
+
+.. topic:: Hash
+
+   Each bucket uses a hash algorithm. As of Reef, Ceph supports the
+   ``rjenkins1`` algorithm. To select ``rjenkins1`` as the hash algorithm,
+   enter ``0`` as your hash setting.
+
+.. _weightingbucketitems:
+
+.. topic:: Weighting Bucket Items
+
+   Ceph expresses bucket weights as doubles, which allows for fine-grained
+   weighting. A weight is the relative difference between device capacities. We
+   recommend using ``1.00`` as the relative weight for a 1 TB storage device.
+   In such a scenario, a weight of ``0.50`` would represent approximately 500
+   GB, and a weight of ``3.00`` would represent approximately 3 TB. Buckets
+   higher in the CRUSH hierarchy have a weight that is the sum of the weight of
+   the leaf items aggregated by the bucket.
+
+
+.. _crushmaprules:
+
+CRUSH Map Rules
+---------------
+
+CRUSH maps have rules that include data placement for a pool: these are
+called "CRUSH rules". The default CRUSH map has one rule for each pool. If you
+are running a large cluster, you might create many pools and each of those
+pools might have its own non-default CRUSH rule.
+
+
+.. note:: In most cases, there is no need to modify the default rule. When a
+   new pool is created, by default the rule will be set to the value ``0``
+   (which indicates the default CRUSH rule, which has the numeric ID ``0``).
+
+CRUSH rules define policy that governs how data is distributed across the devices in
+the hierarchy. The rules define placement as well as replication strategies or
+distribution policies that allow you to specify exactly how CRUSH places data
+replicas. For example, you might create one rule selecting a pair of targets for
+two-way mirroring, another rule for selecting three targets in two different data
+centers for three-way replication, and yet another rule for erasure coding across
+six storage devices. For a detailed discussion of CRUSH rules, see **Section 3.2**
+of `CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_.
+
+A rule takes the following form::
+
+    rule <rulename> {
+
+        id [a unique integer ID]
+        type [replicated|erasure]
+        step take <bucket-name> [class <device-class>]
+        step [choose|chooseleaf] [firstn|indep] <N> type <bucket-type>
+        step emit
+    }
+
+
+``id``
+   :Description: A unique integer that identifies the rule.
+   :Purpose: A component of the rule mask.
+   :Type: Integer
+   :Required: Yes
+   :Default: 0
+
+
+``type``
+   :Description: Denotes the type of replication strategy to be enforced by the
+                 rule.
+   :Purpose: A component of the rule mask.
+   :Type: String
+   :Required: Yes
+   :Default: ``replicated``
+   :Valid Values: ``replicated`` or ``erasure``
+
+
+``step take <bucket-name> [class <device-class>]``
+   :Description: Takes a bucket name and iterates down the tree. If
+                 the ``device-class`` argument is specified, the argument must
+                 match a class assigned to OSDs within the cluster. Only
+                 devices belonging to the class are included.
+   :Purpose: A component of the rule.
+   :Required: Yes
+   :Example: ``step take data``
+
+
+
+``step choose firstn {num} type {bucket-type}``
+   :Description: Selects ``num`` buckets of the given type from within the
+                 current bucket. ``{num}`` is usually the number of replicas in
+                 the pool (in other words, the pool size).
+
+                 - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (as many buckets as are available).
+                 - If ``pool-num-replicas > {num} > 0``, choose that many buckets.
+                 - If ``{num} < 0``, choose ``pool-num-replicas - {num}`` buckets.
+
+   :Purpose: A component of the rule.
+   :Prerequisite: Follows ``step take`` or ``step choose``.
+   :Example: ``step choose firstn 1 type row``
+
+
+``step chooseleaf firstn {num} type {bucket-type}``
+   :Description: Selects a set of buckets of the given type and chooses a leaf
+                 node (that is, an OSD) from the subtree of each bucket in that set of buckets. The
+                 number of buckets in the set is usually the number of replicas in
+                 the pool (in other words, the pool size).
+
+                 - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (as many buckets as are available).
+                 - If ``pool-num-replicas > {num} > 0``, choose that many buckets.
+                 - If ``{num} < 0``, choose ``pool-num-replicas - {num}`` buckets.
+   :Purpose: A component of the rule. Using ``chooseleaf`` obviates the need to select a device in a separate step.
+   :Prerequisite: Follows ``step take`` or ``step choose``.
+   :Example: ``step chooseleaf firstn 0 type row``
+
+
+``step emit``
+   :Description: Outputs the current value on the top of the stack and empties 
+                 the stack. Typically used
+                 at the end of a rule, but may also be used to choose from different
+                 trees in the same rule.
+
+   :Purpose: A component of the rule.
+   :Prerequisite: Follows ``step choose``.
+   :Example: ``step emit``
+
+.. important:: A single CRUSH rule can be assigned to multiple pools, but
+   a single pool cannot have multiple CRUSH rules.
+
+``firstn`` or ``indep``
+
+   :Description: Determines which replacement strategy CRUSH uses when items (OSDs)
+                 are marked ``down`` in the CRUSH map. When this rule is used
+                 with replicated pools, ``firstn`` is used. When this rule is
+                 used with erasure-coded pools, ``indep`` is used.
+
+                 Suppose that a PG is stored on OSDs 1, 2, 3, 4, and 5 and then
+                 OSD 3 goes down.
+
+                 When in ``firstn`` mode, CRUSH simply adjusts its calculation
+                 to select OSDs 1 and 2, then selects 3 and discovers that 3 is
+                 down, retries and selects 4 and 5, and finally goes on to
+                 select a new OSD: OSD 6. The final CRUSH mapping
+                 transformation is therefore 1, 2, 3, 4, 5 → 1, 2, 4, 5, 6.
+
+                 However, if you were storing an erasure-coded pool, the above
+                 sequence would have changed the data that is mapped to OSDs 4,
+                 5, and 6. The ``indep`` mode attempts to avoid this unwanted
+                 consequence. When in ``indep`` mode, CRUSH can be expected to
+                 select 3, discover that 3 is down, retry, and select 6. The
+                 final CRUSH mapping transformation is therefore 1, 2, 3, 4, 5
+                 → 1, 2, 6, 4, 5.
+
+.. _crush-reclassify:
+
+Migrating from a legacy SSD rule to device classes
+--------------------------------------------------
+
+Prior to the Luminous release's introduction of the *device class* feature, in
+order to write rules that applied to a specialized device type (for example,
+SSD), it was necessary to manually edit the CRUSH map and maintain a parallel
+hierarchy for each device type. The device class feature provides a more
+transparent way to achieve this end.
+
+However, if your cluster is migrated from an existing manually-customized
+per-device map to new device class-based rules, all data in the system will be
+reshuffled.
+
+The ``crushtool`` utility has several commands that can transform a legacy rule
+and hierarchy and allow you to start using the new device class rules. There
+are three possible types of transformation:
+
+#. ``--reclassify-root <root-name> <device-class>``
+
+   This command examines everything under ``root-name`` in the hierarchy and
+   rewrites any rules that reference the specified root and that have the
+   form ``take <root-name>`` so that they instead have the
+   form ``take <root-name> class <device-class>``. The command also renumbers
+   the buckets in such a way that the old IDs are used for the specified
+   class's "shadow tree" and as a result no data movement takes place.
+
+   For example, suppose you have the following as an existing rule::
+
+     rule replicated_rule {
+        id 0
+        type replicated
+        step take default
+        step chooseleaf firstn 0 type rack
+        step emit
+     }
+
+   If the root ``default`` is reclassified as class ``hdd``, the new rule will
+   be as follows::
+
+     rule replicated_rule {
+        id 0
+        type replicated
+        step take default class hdd
+        step chooseleaf firstn 0 type rack
+        step emit
+     }
+
+#. ``--set-subtree-class <bucket-name> <device-class>``
+
+   This command marks every device in the subtree that is rooted at *bucket-name*
+   with the specified device class.
+
+   This command is typically used in conjunction with the ``--reclassify-root`` option
+   in order to ensure that all devices in that root are labeled with the
+   correct class. In certain circumstances, however, some of those devices
+   are correctly labeled with a different class and must not be relabeled. To
+   manage this difficulty, one can exclude the ``--set-subtree-class``
+   option. The remapping process will not be perfect, because the previous rule
+   had an effect on devices of multiple classes but the adjusted rules will map
+   only to devices of the specified device class. However, when there are not many
+   outlier devices, the resulting level of data movement is often within tolerable
+   limits.
+
+
+#. ``--reclassify-bucket <match-pattern> <device-class> <default-parent>``
+
+   This command allows you to merge a parallel type-specific hierarchy with the
+   normal hierarchy. For example, many users have maps that resemble the
+   following::
+
+     host node1 {
+        id -2           # do not change unnecessarily
+        # weight 109.152
+        alg straw2
+        hash 0  # rjenkins1
+        item osd.0 weight 9.096
+        item osd.1 weight 9.096
+        item osd.2 weight 9.096
+        item osd.3 weight 9.096
+        item osd.4 weight 9.096
+        item osd.5 weight 9.096
+        ...
+     }
+
+     host node1-ssd {
+        id -10          # do not change unnecessarily
+        # weight 2.000
+        alg straw2
+        hash 0  # rjenkins1
+        item osd.80 weight 2.000
+    ...
+     }
+
+     root default {
+        id -1           # do not change unnecessarily
+        alg straw2
+        hash 0  # rjenkins1
+        item node1 weight 110.967
+        ...
+     }
+
+     root ssd {
+        id -18          # do not change unnecessarily
+        # weight 16.000
+        alg straw2
+        hash 0  # rjenkins1
+        item node1-ssd weight 2.000
+    ...
+     }
+
+   This command reclassifies each bucket that matches a certain
+   pattern. The pattern can be of the form ``%suffix`` or ``prefix%``. For
+   example, in the above example, we would use the pattern
+   ``%-ssd``. For each matched bucket, the remaining portion of the
+   name (corresponding to the ``%`` wildcard) specifies the *base bucket*. All
+   devices in the matched bucket are labeled with the specified
+   device class and then moved to the base bucket. If the base bucket
+   does not exist (for example, ``node12-ssd`` exists but ``node12`` does
+   not), then it is created and linked under the specified
+   *default parent* bucket. In each case, care is taken to preserve
+   the old bucket IDs for the new shadow buckets in order to prevent data
+   movement. Any rules with ``take`` steps that reference the old
+   buckets are adjusted accordingly.
+
+
+#. ``--reclassify-bucket <bucket-name> <device-class> <base-bucket>``
+
+   The same command can also be used without a wildcard in order to map a
+   single bucket. For example, in the previous example, we want the
+   ``ssd`` bucket to be mapped to the ``default`` bucket.
+
+#. The final command to convert the map that consists of the above fragments
+   resembles the following:
+
+   .. prompt:: bash $
+
+      ceph osd getcrushmap -o original
+      crushtool -i original --reclassify \
+        --set-subtree-class default hdd \
+        --reclassify-root default hdd \
+        --reclassify-bucket %-ssd ssd default \
+        --reclassify-bucket ssd ssd default \
+        -o adjusted
+
+``--compare`` flag
+------------------
+
+A ``--compare`` flag is available to make sure that the conversion performed in
+:ref:`Migrating from a legacy SSD rule to device classes <crush-reclassify>` is
+correct. This flag tests a large sample of inputs against the CRUSH map and
+checks that the expected result is output. The options that control these
+inputs are the same as the options that apply to the ``--test`` command. For an
+illustration of how this ``--compare`` command applies to the above example,
+see the following:
+
+.. prompt:: bash $
+
+   crushtool -i original --compare adjusted
+
+::
+
+  rule 0 had 0/10240 mismatched mappings (0)
+  rule 1 had 0/10240 mismatched mappings (0)
+  maps appear equivalent
+
+If the command finds any differences, the ratio of remapped inputs is reported
+in the parentheses.
+
+When you are satisfied with the adjusted map, apply it to the cluster by
+running the following command:
+
+.. prompt:: bash $
+
+   ceph osd setcrushmap -i adjusted
+
+Manually Tuning CRUSH
+---------------------
+
+If you have verified that all clients are running recent code, you can adjust
+the CRUSH tunables by extracting the CRUSH map, modifying the values, and
+reinjecting the map into the cluster. The procedure is carried out as follows:
+
+#. Extract the latest CRUSH map:
+
+   .. prompt:: bash $
+
+      ceph osd getcrushmap -o /tmp/crush
+
+#. Adjust tunables. In our tests, the following values appear to result in the 
+   best behavior for both large and small clusters. The procedure requires that
+   you specify the ``--enable-unsafe-tunables`` flag in the ``crushtool``
+   command. Use this option with **extreme care**:
+
+   .. prompt:: bash $
+
+      crushtool -i /tmp/crush --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 -o /tmp/crush.new
+
+#. Reinject the modified map:
+
+   .. prompt:: bash $
+
+     ceph osd setcrushmap -i /tmp/crush.new
+
+Legacy values
+-------------
+
+To set the legacy values of the CRUSH tunables, run the following command:
+
+.. prompt:: bash $
+
+   crushtool -i /tmp/crush --set-choose-local-tries 2 --set-choose-local-fallback-tries 5 --set-choose-total-tries 19 --set-chooseleaf-descend-once 0 --set-chooseleaf-vary-r 0 -o /tmp/crush.legacy
+
+The special ``--enable-unsafe-tunables`` flag is required. Be careful when
+running old versions of the ``ceph-osd`` daemon after reverting to legacy
+values, because the feature bit is not perfectly enforced.
+
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst
new file mode 100644
index 000000000..39151e6d4
--- /dev/null
+++ b/doc/rados/operations/crush-map.rst
@@ -0,0 +1,1147 @@
+============
+ CRUSH Maps
+============
+
+The :abbr:`CRUSH (Controlled Replication Under Scalable Hashing)` algorithm
+computes storage locations in order to determine how to store and retrieve
+data.  CRUSH allows Ceph clients to communicate with OSDs directly rather than
+through a centralized server or broker. By using an algorithmically-determined
+method of storing and retrieving data, Ceph avoids a single point of failure, a
+performance bottleneck, and a physical limit to its scalability.
+
+CRUSH uses a map of the cluster (the CRUSH map) to map data to OSDs,
+distributing the data across the cluster in accordance with configured
+replication policy and failure domains. For a detailed discussion of CRUSH, see
+`CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_
+
+CRUSH maps contain a list of :abbr:`OSDs (Object Storage Devices)` and a
+hierarchy of "buckets" (``host``\s, ``rack``\s) and rules that govern how CRUSH
+replicates data within the cluster's pools. By reflecting the underlying
+physical organization of the installation, CRUSH can model (and thereby
+address) the potential for correlated device failures.  Some factors relevant
+to the CRUSH hierarchy include chassis, racks, physical proximity, a shared
+power source, shared networking, and failure domains. By encoding this
+information into the CRUSH map, CRUSH placement policies distribute object
+replicas across failure domains while maintaining the desired distribution. For
+example, to address the possibility of concurrent failures, it might be
+desirable to ensure that data replicas are on devices that reside in or rely
+upon different shelves, racks, power supplies, controllers, or physical
+locations.
+
+When OSDs are deployed, they are automatically added to the CRUSH map under a
+``host`` bucket that is named for the node on which the OSDs run. This
+behavior, combined with the configured CRUSH failure domain, ensures that
+replicas or erasure-code shards are distributed across hosts and that the
+failure of a single host or other kinds of failures will not affect
+availability. For larger clusters, administrators must carefully consider their
+choice of failure domain. For example, distributing replicas across racks is
+typical for mid- to large-sized clusters.
+
+
+CRUSH Location
+==============
+
+The location of an OSD within the CRUSH map's hierarchy is referred to as its
+``CRUSH location``. The specification of a CRUSH location takes the form of a
+list of key-value pairs. For example, if an OSD is in a particular row, rack,
+chassis, and host, and is also part of the 'default' CRUSH root (which is the
+case for most clusters), its CRUSH location can be specified as follows::
+
+  root=default row=a rack=a2 chassis=a2a host=a2a1
+
+.. note::
+
+   #. The order of the keys does not matter.
+   #. The key name (left of ``=``) must be a valid CRUSH ``type``. By default,
+      valid CRUSH types include ``root``, ``datacenter``, ``room``, ``row``,
+      ``pod``, ``pdu``, ``rack``, ``chassis``, and ``host``. These defined
+      types suffice for nearly all clusters, but can be customized by
+      modifying the CRUSH map.
+   #. Not all keys need to be specified. For example, by default, Ceph
+      automatically sets an ``OSD``'s location as ``root=default
+      host=HOSTNAME`` (as determined by the output of ``hostname -s``).
+
+The CRUSH location for an OSD can be modified by adding the ``crush location``
+option in ``ceph.conf``. When this option has been added, every time the OSD
+starts it verifies that it is in the correct location in the CRUSH map and
+moves itself if it is not. To disable this automatic CRUSH map management, add
+the following to the ``ceph.conf`` configuration file in the ``[osd]``
+section::
+
+   osd crush update on start = false
+
+Note that this action is unnecessary in most cases.
+
+
+Custom location hooks
+---------------------
+
+A custom location hook can be used to generate a more complete CRUSH location
+on startup. The CRUSH location is determined by, in order of preference:
+
+#. A ``crush location`` option in ``ceph.conf``
+#. A default of ``root=default host=HOSTNAME`` where the hostname is determined
+   by the output of the ``hostname -s`` command
+
+A script can be written to provide additional location fields (for example,
+``rack`` or ``datacenter``) and the hook can be enabled via the following
+config option::
+
+   crush location hook = /path/to/customized-ceph-crush-location
+
+This hook is passed several arguments (see below). The hook outputs a single
+line to ``stdout`` that contains the CRUSH location description. The output
+resembles the following:::
+
+  --cluster CLUSTER --id ID --type TYPE
+
+Here the cluster name is typically ``ceph``, the ``id`` is the daemon
+identifier or (in the case of OSDs) the OSD number, and the daemon type is
+``osd``, ``mds, ``mgr``, or ``mon``.
+
+For example, a simple hook that specifies a rack location via a value in the
+file ``/etc/rack`` might be as follows::
+
+  #!/bin/sh
+  echo "host=$(hostname -s) rack=$(cat /etc/rack) root=default"
+
+
+CRUSH structure
+===============
+
+The CRUSH map consists of (1) a hierarchy that describes the physical topology
+of the cluster and (2) a set of rules that defines data placement policy. The
+hierarchy has devices (OSDs) at the leaves and internal nodes corresponding to
+other physical features or groupings: hosts, racks, rows, data centers, and so
+on. The rules determine how replicas are placed in terms of that hierarchy (for
+example, 'three replicas in different racks').
+
+Devices
+-------
+
+Devices are individual OSDs that store data (usually one device for each
+storage drive).  Devices are identified by an ``id`` (a non-negative integer)
+and a ``name`` (usually ``osd.N``, where ``N`` is the device's ``id``).
+
+In Luminous and later releases, OSDs can have a *device class* assigned (for
+example, ``hdd`` or ``ssd`` or ``nvme``), allowing them to be targeted by CRUSH
+rules. Device classes are especially useful when mixing device types within
+hosts.
+
+.. _crush_map_default_types:
+
+Types and Buckets
+-----------------
+
+"Bucket", in the context of CRUSH, is a term for any of the internal nodes in
+the hierarchy: hosts, racks, rows, and so on. The CRUSH map defines a series of
+*types* that are used to identify these nodes. Default types include:
+
+- ``osd`` (or ``device``)
+- ``host``
+- ``chassis``
+- ``rack``
+- ``row``
+- ``pdu``
+- ``pod``
+- ``room``
+- ``datacenter``
+- ``zone``
+- ``region``
+- ``root``
+
+Most clusters use only a handful of these types, and other types can be defined
+as needed.
+
+The hierarchy is built with devices (normally of type ``osd``) at the leaves
+and non-device types as the internal nodes. The root node is of type ``root``.
+For example:
+
+
+.. ditaa::
+
+                        +-----------------+ 
+                        |{o}root default  |
+                        +--------+--------+     
+                                 |
+                 +---------------+---------------+
+                 |                               |
+          +------+------+                 +------+------+
+          |{o}host foo  |                 |{o}host bar  | 
+          +------+------+                 +------+------+
+                 |                               |
+         +-------+-------+               +-------+-------+
+         |               |               |               |
+   +-----+-----+   +-----+-----+   +-----+-----+   +-----+-----+
+   |   osd.0   |   |   osd.1   |   |   osd.2   |   |   osd.3   | 
+   +-----------+   +-----------+   +-----------+   +-----------+
+
+
+Each node (device or bucket) in the hierarchy has a *weight* that indicates the
+relative proportion of the total data that should be stored by that device or
+hierarchy subtree. Weights are set at the leaves, indicating the size of the
+device. These weights automatically sum in an 'up the tree' direction: that is,
+the weight of the ``root`` node will be the sum of the weights of all devices
+contained under it. Weights are typically measured in tebibytes (TiB).
+
+To get a simple view of the cluster's CRUSH hierarchy, including weights, run
+the following command:
+
+.. prompt:: bash $
+
+   ceph osd tree
+
+Rules
+-----
+
+CRUSH rules define policy governing how data is distributed across the devices
+in the hierarchy. The rules define placement as well as replication strategies
+or distribution policies that allow you to specify exactly how CRUSH places
+data replicas. For example, you might create one rule selecting a pair of
+targets for two-way mirroring, another rule for selecting three targets in two
+different data centers for three-way replication, and yet another rule for
+erasure coding across six storage devices. For a detailed discussion of CRUSH
+rules, see **Section 3.2** of `CRUSH - Controlled, Scalable, Decentralized
+Placement of Replicated Data`_.
+
+CRUSH rules can be created via the command-line by specifying the *pool type*
+that they will govern (replicated or erasure coded), the *failure domain*, and
+optionally a *device class*.  In rare cases, CRUSH rules must be created by
+manually editing the CRUSH map.
+
+To see the rules that are defined for the cluster, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush rule ls
+
+To view the contents of the rules, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush rule dump
+
+.. _device_classes:
+
+Device classes
+--------------
+
+Each device can optionally have a *class* assigned. By default, OSDs
+automatically set their class at startup to `hdd`, `ssd`, or `nvme` in
+accordance with the type of device they are backed by.
+
+To explicitly set the device class of one or more OSDs, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd crush set-device-class <class> <osd-name> [...]
+
+Once a device class has been set, it cannot be changed to another class until
+the old class is unset. To remove the old class of one or more OSDs, run a
+command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush rm-device-class <osd-name> [...]
+
+This restriction allows administrators to set device classes that won't be
+changed on OSD restart or by a script.
+
+To create a placement rule that targets a specific device class, run a command
+of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-replicated <rule-name> <root> <failure-domain> <class>
+
+To apply the new placement rule to a specific pool, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> crush_rule <rule-name>
+
+Device classes are implemented by creating one or more "shadow" CRUSH
+hierarchies.  For each device class in use, there will be a shadow hierarchy
+that contains only devices of that class. CRUSH rules can then distribute data
+across the relevant shadow hierarchy.  This approach is fully backward
+compatible with older Ceph clients. To view the CRUSH hierarchy with shadow
+items displayed, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd crush tree --show-shadow
+
+Some older clusters that were created before the Luminous release rely on
+manually crafted CRUSH maps to maintain per-device-type hierarchies. For these
+clusters, there is a *reclassify* tool available that can help them transition
+to device classes without triggering unwanted data movement (see
+:ref:`crush-reclassify`).
+
+Weight sets
+-----------
+
+A *weight set* is an alternative set of weights to use when calculating data
+placement. The normal weights associated with each device in the CRUSH map are
+set in accordance with the device size and indicate how much data should be
+stored where. However, because CRUSH is a probabilistic pseudorandom placement
+process, there is always some variation from this ideal distribution (in the
+same way that rolling a die sixty times will likely not result in exactly ten
+ones and ten sixes). Weight sets allow the cluster to perform numerical
+optimization based on the specifics of your cluster (for example: hierarchy,
+pools) to achieve a balanced distribution.
+
+Ceph supports two types of weight sets:
+
+#. A **compat** weight set is a single alternative set of weights for each
+   device and each node in the cluster. Compat weight sets cannot be expected
+   to correct all anomalies (for example, PGs for different pools might be of
+   different sizes and have different load levels, but are mostly treated alike
+   by the balancer).  However, they have the major advantage of being *backward
+   compatible* with previous versions of Ceph. This means that even though
+   weight sets were first introduced in Luminous v12.2.z, older clients (for
+   example, Firefly) can still connect to the cluster when a compat weight set
+   is being used to balance data.
+
+#. A **per-pool** weight set is more flexible in that it allows placement to
+   be optimized for each data pool. Additionally, weights can be adjusted
+   for each position of placement, allowing the optimizer to correct for a
+   subtle skew of data toward devices with small weights relative to their
+   peers (an effect that is usually apparent only in very large clusters
+   but that can cause balancing problems).
+
+When weight sets are in use, the weights associated with each node in the
+hierarchy are visible in a separate column (labeled either as ``(compat)`` or
+as the pool name) in the output of the following command:
+
+.. prompt:: bash #
+
+   ceph osd tree
+
+If both *compat* and *per-pool* weight sets are in use, data placement for a
+particular pool will use its own per-pool weight set if present. If only
+*compat* weight sets are in use, data placement will use the compat weight set.
+If neither are in use, data placement will use the normal CRUSH weights.
+
+Although weight sets can be set up and adjusted manually, we recommend enabling
+the ``ceph-mgr`` *balancer* module to perform these tasks automatically if the
+cluster is running Luminous or a later release.
+
+Modifying the CRUSH map
+=======================
+
+.. _addosd:
+
+Adding/Moving an OSD
+--------------------
+
+.. note:: Under normal conditions, OSDs automatically add themselves to the
+   CRUSH map when they are created. The command in this section is rarely
+   needed.
+
+
+To add or move an OSD in the CRUSH map of a running cluster, run a command of
+the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush set {name} {weight} root={root} [{bucket-type}={bucket-name} ...]
+
+For details on this command's parameters, see the following:
+
+``name``
+   :Description: The full name of the OSD.
+   :Type: String
+   :Required: Yes
+   :Example: ``osd.0``
+
+
+``weight``
+   :Description: The CRUSH weight of the OSD. Normally, this is its size, as measured in terabytes (TB).
+   :Type: Double
+   :Required: Yes
+   :Example: ``2.0``
+
+
+``root``
+   :Description: The root node of the CRUSH hierarchy in which the OSD resides (normally ``default``).
+   :Type: Key-value pair.
+   :Required: Yes
+   :Example: ``root=default``
+
+
+``bucket-type``
+   :Description: The OSD's location in the CRUSH hierarchy.
+   :Type: Key-value pairs.
+   :Required: No
+   :Example: ``datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1``
+
+In the following example, the command adds ``osd.0`` to the hierarchy, or moves
+``osd.0`` from a previous location:
+
+.. prompt:: bash $
+
+   ceph osd crush set osd.0 1.0 root=default datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1
+
+
+Adjusting OSD weight
+--------------------
+
+.. note:: Under normal conditions, OSDs automatically add themselves to the
+   CRUSH map with the correct weight when they are created. The command in this
+   section is rarely needed.
+
+To adjust an OSD's CRUSH weight in a running cluster, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd crush reweight {name} {weight}
+
+For details on this command's parameters, see the following:
+
+``name``
+   :Description: The full name of the OSD.
+   :Type: String
+   :Required: Yes
+   :Example: ``osd.0``
+
+
+``weight``
+   :Description: The CRUSH weight of the OSD.
+   :Type: Double
+   :Required: Yes
+   :Example: ``2.0``
+
+
+.. _removeosd:
+
+Removing an OSD
+---------------
+
+.. note:: OSDs are normally removed from the CRUSH map as a result of the
+   `ceph osd purge`` command. This command is rarely needed.
+
+To remove an OSD from the CRUSH map of a running cluster, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {name}
+
+For details on the ``name`` parameter, see the following:
+
+``name``
+   :Description: The full name of the OSD.
+   :Type: String
+   :Required: Yes
+   :Example: ``osd.0``
+
+
+Adding a CRUSH Bucket
+---------------------
+
+.. note:: Buckets are implicitly created when an OSD is added and the command
+   that creates it specifies a ``{bucket-type}={bucket-name}`` as part of the
+   OSD's location (provided that a bucket with that name does not already
+   exist). The command in this section is typically used when manually
+   adjusting the structure of the hierarchy after OSDs have already been
+   created. One use of this command is to move a series of hosts to a new
+   rack-level bucket.  Another use of this command is to add new ``host``
+   buckets (OSD nodes) to a dummy ``root`` so that the buckets don't receive
+   any data until they are ready to receive data. When they are ready, move the
+   buckets to the ``default`` root or to any other root as described below.
+
+To add a bucket in the CRUSH map of a running cluster, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd crush add-bucket {bucket-name} {bucket-type}
+
+For details on this command's parameters, see the following:
+
+``bucket-name``
+   :Description: The full name of the bucket.
+   :Type: String
+   :Required: Yes
+   :Example: ``rack12``
+
+
+``bucket-type``
+   :Description: The type of the bucket. This type must already exist in the CRUSH hierarchy.
+   :Type: String
+   :Required: Yes
+   :Example: ``rack``
+
+In the following example, the command adds the ``rack12`` bucket to the hierarchy:
+
+.. prompt:: bash $
+
+   ceph osd crush add-bucket rack12 rack
+
+Moving a Bucket
+---------------
+
+To move a bucket to a different location or position in the CRUSH map
+hierarchy, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush move {bucket-name} {bucket-type}={bucket-name}, [...]
+
+For details on this command's parameters, see the following:
+
+``bucket-name``
+   :Description: The name of the bucket that you are moving.
+   :Type: String
+   :Required: Yes
+   :Example: ``foo-bar-1``
+
+``bucket-type``
+   :Description: The bucket's new location in the CRUSH hierarchy.
+   :Type: Key-value pairs.
+   :Required: No
+   :Example: ``datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1``
+
+Removing a Bucket
+-----------------
+
+To remove a bucket from the CRUSH hierarchy, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {bucket-name}
+
+.. note:: A bucket must already be empty before it is removed from the CRUSH
+   hierarchy. In other words, there must not be OSDs or any other CRUSH buckets
+   within it.
+
+For details on the ``bucket-name`` parameter, see the following:
+
+``bucket-name``
+   :Description: The name of the bucket that is being removed.
+   :Type: String
+   :Required: Yes
+   :Example: ``rack12``
+
+In the following example, the command removes the ``rack12`` bucket from the
+hierarchy:
+
+.. prompt:: bash $
+
+   ceph osd crush remove rack12
+
+Creating a compat weight set
+----------------------------
+
+.. note:: Normally this action is done automatically if needed by the
+   ``balancer`` module (provided that the module is enabled).
+
+To create a *compat* weight set, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set create-compat
+
+To adjust the weights of the compat weight set, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set reweight-compat {name} {weight}
+
+To destroy the compat weight set, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set rm-compat
+
+Creating per-pool weight sets
+-----------------------------
+
+To create a weight set for a specific pool, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set create {pool-name} {mode}
+
+.. note:: Per-pool weight sets can be used only if all servers and daemons are
+   running Luminous v12.2.z or a later release.
+
+For details on this command's parameters, see the following:
+
+``pool-name``
+   :Description: The name of a RADOS pool.
+   :Type: String
+   :Required: Yes
+   :Example: ``rbd``
+
+``mode``
+   :Description: Either ``flat`` or ``positional``. A *flat* weight set 
+                 assigns a single weight to all devices or buckets. A 
+                 *positional* weight set has a potentially different 
+                 weight for each position in the resulting placement 
+                 mapping. For example: if a pool has a replica count of 
+                 ``3``, then a positional weight set will have three
+                 weights for each device and bucket.
+   :Type: String
+   :Required: Yes
+   :Example: ``flat``
+
+To adjust the weight of an item in a weight set, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set reweight {pool-name} {item-name} {weight [...]}
+
+To list existing weight sets, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set ls
+
+To remove a weight set, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set rm {pool-name}
+
+
+Creating a rule for a replicated pool
+-------------------------------------
+
+When you create a CRUSH rule for a replicated pool, there is an important
+decision to make: selecting a failure domain. For example, if you select a
+failure domain of ``host``, then CRUSH will ensure that each replica of the
+data is stored on a unique host.  Alternatively, if you select a failure domain
+of ``rack``, then each replica of the data will be stored in a different rack.
+Your selection of failure domain should be guided by the size and its CRUSH
+topology. 
+
+The entire cluster hierarchy is typically nested beneath a root node that is
+named ``default``. If you have customized your hierarchy, you might want to
+create a rule nested beneath some other node in the hierarchy.  In creating
+this rule for the customized hierarchy, the node type doesn't matter, and in
+particular the rule does not have to be nested beneath a ``root`` node.
+
+It is possible to create a rule that restricts data placement to a specific
+*class* of device. By default, Ceph OSDs automatically classify themselves as
+either ``hdd`` or ``ssd`` in accordance with the underlying type of device
+being used. These device classes can be customized. One might set the ``device
+class`` of OSDs to ``nvme`` to distinguish the from SATA SSDs, or one might set
+them to something arbitrary like ``ssd-testing`` or ``ssd-ethel`` so that rules
+and pools may be flexibly constrained to use (or avoid using) specific subsets
+of OSDs based on specific requirements. 
+
+To create a rule for a replicated pool, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-replicated {name} {root} {failure-domain-type} [{class}]
+
+For details on this command's parameters, see the following:
+
+``name``
+   :Description: The name of the rule.
+   :Type: String
+   :Required: Yes
+   :Example: ``rbd-rule``
+
+``root``
+   :Description: The name of the CRUSH hierarchy node under which data is to be placed.
+   :Type: String
+   :Required: Yes
+   :Example: ``default``
+
+``failure-domain-type``
+   :Description: The type of CRUSH nodes used for the replicas of the failure domain.
+   :Type: String
+   :Required: Yes
+   :Example: ``rack``
+
+``class``
+   :Description: The device class on which data is to be placed.
+   :Type: String
+   :Required: No
+   :Example: ``ssd``
+
+Creating a rule for an erasure-coded pool
+-----------------------------------------
+
+For an erasure-coded pool, similar decisions need to be made: what the failure
+domain is, which node in the hierarchy data will be placed under (usually
+``default``), and whether placement is restricted to a specific device class.
+However, erasure-code pools are created in a different way: there is a need to
+construct them carefully with reference to the erasure code plugin in use. For
+this reason, these decisions must be incorporated into the **erasure-code
+profile**.  A CRUSH rule will then be created from the erasure-code profile,
+either explicitly or automatically when the profile is used to create a pool.
+
+To list the erasure-code profiles, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile ls
+
+To view a specific existing profile, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile get {profile-name}
+
+Under normal conditions, profiles should never be modified; instead, a new
+profile should be created and used when creating either a new pool or a new
+rule for an existing pool.
+
+An erasure-code profile consists of a set of key-value pairs. Most of these
+key-value pairs govern the behavior of the erasure code that encodes data in
+the pool. However, key-value pairs that begin with ``crush-`` govern the CRUSH
+rule that is created.
+
+The relevant erasure-code profile properties are as follows:
+
+ * **crush-root**: the name of the CRUSH node under which to place data
+   [default: ``default``].
+ * **crush-failure-domain**: the CRUSH bucket type used in the distribution of
+   erasure-coded shards [default: ``host``].
+ * **crush-device-class**: the device class on which to place data [default:
+   none, which means that all devices are used].
+ * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the
+   number of erasure-code shards, affecting the resulting CRUSH rule.
+
+ After a profile is defined, you can create a CRUSH rule by running a command
+ of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-erasure {name} {profile-name}
+
+.. note: When creating a new pool, it is not necessary to create the rule
+   explicitly. If only the erasure-code profile is specified and the rule
+   argument is omitted, then Ceph will create the CRUSH rule automatically.
+
+
+Deleting rules
+--------------
+
+To delete rules that are not in use by pools, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd crush rule rm {rule-name}
+
+.. _crush-map-tunables:
+
+Tunables
+========
+
+The CRUSH algorithm that is used to calculate the placement of data has been
+improved over time. In order to support changes in behavior, we have provided
+users with sets of tunables that determine which legacy or optimal version of
+CRUSH is to be used. 
+
+In order to use newer tunables, all Ceph clients and daemons must support the
+new major release of CRUSH. Because of this requirement, we have created
+``profiles`` that are named after the Ceph version in which they were
+introduced. For example, the ``firefly`` tunables were first supported by the
+Firefly release and do not work with older clients (for example, clients
+running Dumpling).  After a cluster's tunables profile is changed from a legacy
+set to a newer or ``optimal`` set, the ``ceph-mon`` and ``ceph-osd`` options
+will prevent older clients that do not support the new CRUSH features from
+connecting to the cluster.
+
+argonaut (legacy)
+-----------------
+
+The legacy CRUSH behavior used by Argonaut and older releases works fine for
+most clusters, provided that not many OSDs have been marked ``out``.
+
+bobtail (CRUSH_TUNABLES2)
+-------------------------
+
+The ``bobtail`` tunable profile provides the following improvements:
+
+ * For hierarchies with a small number of devices in leaf buckets, some PGs
+   might map to fewer than the desired number of replicas, resulting in
+   ``undersized`` PGs.  This is known to happen in the case of hierarchies with
+   ``host`` nodes that have a small number of OSDs (1 to 3) nested beneath each
+   host.
+
+ * For large clusters, a small percentage of PGs might map to fewer than the
+   desired number of OSDs. This is known to happen when there are multiple
+   hierarchy layers in use (for example,, ``row``, ``rack``, ``host``,
+   ``osd``).
+
+ * When one or more OSDs are marked ``out``, data tends to be redistributed
+   to nearby OSDs instead of across the entire hierarchy.
+
+The tunables introduced in the Bobtail release are as follows:
+
+ * ``choose_local_tries``: Number of local retries. The legacy value is ``2``,
+   and the optimal value is ``0``.
+
+ * ``choose_local_fallback_tries``: The legacy value is ``5``, and the optimal
+   value is 0.
+
+ * ``choose_total_tries``: Total number of attempts to choose an item.  The
+   legacy value is ``19``, but subsequent testing indicates that a value of
+   ``50`` is more appropriate for typical clusters. For extremely large
+   clusters, an even larger value might be necessary.
+
+ * ``chooseleaf_descend_once``: Whether a recursive ``chooseleaf`` attempt will
+   retry, or try only once and allow the original placement to retry. The
+   legacy default is ``0``, and the optimal value is ``1``.
+
+Migration impact:
+
+ * Moving from the ``argonaut`` tunables to the ``bobtail`` tunables triggers a
+   moderate amount of data movement. Use caution on a cluster that is already
+   populated with data.
+
+firefly (CRUSH_TUNABLES3)
+-------------------------
+
+chooseleaf_vary_r
+~~~~~~~~~~~~~~~~~
+
+This ``firefly`` tunable profile fixes a problem with ``chooseleaf`` CRUSH step
+behavior. This problem arose when a large fraction of OSDs were marked ``out``, which resulted in PG mappings with too few OSDs.
+
+This profile was introduced in the Firefly release, and adds a new tunable as follows:
+
+ * ``chooseleaf_vary_r``: Whether a recursive chooseleaf attempt will start
+   with a non-zero value of ``r``, as determined by the number of attempts the
+   parent has already made. The legacy default value is ``0``, but with this
+   value CRUSH is sometimes unable to find a mapping. The optimal value (in
+   terms of computational cost and correctness) is ``1``.
+
+Migration impact:
+
+ * For existing clusters that store a great deal of data, changing this tunable
+   from ``0`` to ``1`` will trigger a large amount of data migration; a value
+   of ``4`` or ``5`` will allow CRUSH to still find a valid mapping and will
+   cause less data to move.
+
+straw_calc_version tunable
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There were problems with the internal weights calculated and stored in the
+CRUSH map for ``straw`` algorithm buckets. When there were buckets with a CRUSH
+weight of ``0`` or with a mix of different and unique weights, CRUSH would
+distribute data incorrectly (that is, not in proportion to the weights).
+
+This tunable, introduced in the Firefly release, is as follows:
+
+ * ``straw_calc_version``: A value of ``0`` preserves the old, broken
+   internal-weight calculation; a value of ``1`` fixes the problem.
+
+Migration impact:
+
+ * Changing this tunable to a value of ``1`` and then adjusting a straw bucket
+   (either by adding, removing, or reweighting an item or by using the
+   reweight-all command) can trigger a small to moderate amount of data
+   movement provided that the cluster has hit one of the problematic
+   conditions.
+
+This tunable option is notable in that it has absolutely no impact on the
+required kernel version in the client side.
+
+hammer (CRUSH_V4)
+-----------------
+
+The ``hammer`` tunable profile does not affect the mapping of existing CRUSH
+maps simply by changing the profile. However:
+
+ * There is a new bucket algorithm supported: ``straw2``. This new algorithm
+   fixes several limitations in the original ``straw``. More specifically, the
+   old ``straw`` buckets would change some mappings that should not have
+   changed when a weight was adjusted, while ``straw2`` achieves the original
+   goal of changing mappings only to or from the bucket item whose weight has
+   changed.
+
+ * The ``straw2`` type is the default type for any newly created buckets.
+
+Migration impact:
+
+ * Changing a bucket type from ``straw`` to ``straw2`` will trigger a small
+   amount of data movement, depending on how much the bucket items' weights
+   vary from each other. When the weights are all the same no data will move,
+   and the more variance there is in the weights the more movement there will
+   be.
+
+jewel (CRUSH_TUNABLES5)
+-----------------------
+
+The ``jewel`` tunable profile improves the overall behavior of CRUSH. As a
+result, significantly fewer mappings change when an OSD is marked ``out`` of
+the cluster. This improvement results in significantly less data movement.
+
+The new tunable introduced in the Jewel release is as follows:
+
+ * ``chooseleaf_stable``: Determines whether a recursive chooseleaf attempt
+   will use a better value for an inner loop that greatly reduces the number of
+   mapping changes when an OSD is marked ``out``. The legacy value is ``0``,
+   and the new value of ``1`` uses the new approach.
+
+Migration impact:
+
+ * Changing this value on an existing cluster will result in a very large
+   amount of data movement because nearly every PG mapping is likely to change.
+
+Client versions that support CRUSH_TUNABLES2
+--------------------------------------------
+
+ * v0.55 and later, including Bobtail (v0.56.x)
+ * Linux kernel version v3.9 and later (for the CephFS and RBD kernel clients)
+
+Client versions that support CRUSH_TUNABLES3
+--------------------------------------------
+
+ * v0.78 (Firefly) and later
+ * Linux kernel version v3.15 and later (for the CephFS and RBD kernel clients)
+
+Client versions that support CRUSH_V4
+-------------------------------------
+
+ * v0.94 (Hammer) and later
+ * Linux kernel version v4.1 and later (for the CephFS and RBD kernel clients)
+
+Client versions that support CRUSH_TUNABLES5
+--------------------------------------------
+
+ * v10.0.2 (Jewel) and later
+ * Linux kernel version v4.5 and later (for the CephFS and RBD kernel clients)
+
+"Non-optimal tunables" warning
+------------------------------
+
+In v0.74 and later versions, Ceph will raise a health check ("HEALTH_WARN crush
+map has non-optimal tunables") if any of the current CRUSH tunables have
+non-optimal values: that is, if any fail to have the optimal values from the
+:ref:` ``default`` profile
+<rados_operations_crush_map_default_profile_definition>`.  There are two
+different ways to silence the alert:
+
+1. Adjust the CRUSH tunables on the existing cluster so as to render them
+   optimal. Making this adjustment will trigger some data movement
+   (possibly as much as 10%). This approach is generally preferred to the
+   other approach, but special care must be taken in situations where
+   data movement might affect performance: for example, in production clusters.
+   To enable optimal tunables, run the following command:
+
+   .. prompt:: bash $
+
+      ceph osd crush tunables optimal
+
+   There are several potential problems that might make it preferable to revert
+   to the previous values of the tunables. The new values might generate too
+   much load for the cluster to handle, the new values might unacceptably slow
+   the operation of the cluster, or there might be a client-compatibility
+   problem. Such client-compatibility problems can arise when using old-kernel
+   CephFS or RBD clients, or pre-Bobtail ``librados`` clients.  To revert to
+   the previous values of the tunables, run the following command:
+
+   .. prompt:: bash $
+
+      ceph osd crush tunables legacy
+
+2. To silence the alert without making any changes to CRUSH,
+   add the following option to the ``[mon]`` section of your ceph.conf file::
+
+      mon_warn_on_legacy_crush_tunables = false
+
+   In order for this change to take effect, you will need to either restart
+   the monitors or run the following command to apply the option to the
+   monitors while they are still running:
+
+   .. prompt:: bash $
+
+      ceph tell mon.\* config set mon_warn_on_legacy_crush_tunables false
+
+
+Tuning CRUSH
+------------
+
+When making adjustments to CRUSH tunables, keep the following considerations in
+mind:
+
+ * Adjusting the values of CRUSH tunables will result in the shift of one or
+   more PGs from one storage node to another. If the Ceph cluster is already
+   storing a great deal of data, be prepared for significant data movement.
+ * When the ``ceph-osd`` and ``ceph-mon`` daemons get the updated map, they
+   immediately begin rejecting new connections from clients that do not support
+   the new feature. However, already-connected clients are effectively
+   grandfathered in, and any of these clients that do not support the new
+   feature will malfunction.
+ * If the CRUSH tunables are set to newer (non-legacy) values and subsequently
+   reverted to the legacy values, ``ceph-osd`` daemons will not be required to
+   support any of the newer CRUSH features associated with the newer
+   (non-legacy) values. However, the OSD peering process requires the
+   examination and understanding of old maps. For this reason, **if the cluster
+   has previously used non-legacy CRUSH values, do not run old versions of
+   the** ``ceph-osd`` **daemon** -- even if the latest version of the map has
+   been reverted so as to use the legacy defaults.
+
+The simplest way to adjust CRUSH tunables is to apply them in matched sets
+known as *profiles*. As of the Octopus release, Ceph supports the following
+profiles:
+
+ * ``legacy``: The legacy behavior from argonaut and earlier.
+ * ``argonaut``: The legacy values supported by the argonaut release.
+ * ``bobtail``: The values supported by the bobtail release.
+ * ``firefly``: The values supported by the firefly release.
+ * ``hammer``: The values supported by the hammer release.
+ * ``jewel``: The values supported by the jewel release.
+ * ``optimal``: The best values for the current version of Ceph.
+   .. _rados_operations_crush_map_default_profile_definition:
+ * ``default``: The default values of a new cluster that has been installed
+   from scratch. These values, which depend on the current version of Ceph, are
+   hardcoded and are typically a mix of optimal and legacy values.  These
+   values often correspond to the ``optimal`` profile of either the previous
+   LTS (long-term service) release or the most recent release for which most
+   users are expected to have up-to-date clients.
+
+To apply a profile to a running cluster, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd crush tunables {PROFILE}
+
+This action might trigger a great deal of data movement. Consult release notes
+and documentation before changing the profile on a running cluster. Consider
+throttling recovery and backfill parameters in order to limit the backfill
+resulting from a specific change.
+
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
+
+
+Tuning Primary OSD Selection
+============================
+
+When a Ceph client reads or writes data, it first contacts the primary OSD in
+each affected PG's acting set. By default, the first OSD in the acting set is
+the primary OSD (also known as the "lead OSD"). For example, in the acting set
+``[2, 3, 4]``, ``osd.2`` is listed first and is therefore the primary OSD.
+However, sometimes it is clear that an OSD is not well suited to act as the
+lead as compared with other OSDs (for example, if the OSD has a slow drive or a
+slow controller). To prevent performance bottlenecks (especially on read
+operations) and at the same time maximize the utilization of your hardware, you
+can influence the selection of the primary OSD either by adjusting "primary
+affinity" values, or by crafting a CRUSH rule that selects OSDs that are better
+suited to act as the lead rather than other OSDs.
+
+To determine whether tuning Ceph's selection of primary OSDs will improve
+cluster performance, pool redundancy strategy must be taken into account. For
+replicated pools, this tuning can be especially useful, because by default read
+operations are served from the primary OSD of each PG. For erasure-coded pools,
+however, the speed of read operations can be increased by enabling **fast
+read** (see :ref:`pool-settings`).
+
+.. _rados_ops_primary_affinity:
+
+Primary Affinity
+----------------
+
+**Primary affinity** is a characteristic of an OSD that governs the likelihood
+that a given OSD will be selected as the primary OSD (or "lead OSD") in a given
+acting set. A primary affinity value can be any real number in the range ``0``
+to ``1``, inclusive.
+
+As an example of a common scenario in which it can be useful to adjust primary
+affinity values, let us suppose that a cluster contains a mix of drive sizes:
+for example, suppose it contains some older racks with 1.9 TB SATA SSDs and
+some newer racks with 3.84 TB SATA SSDs. The latter will on average be assigned
+twice the number of PGs and will thus serve twice the number of write and read
+operations -- they will be busier than the former. In such a scenario, you
+might make a rough assignment of primary affinity as inversely proportional to
+OSD size. Such an assignment will not be 100% optimal, but it can readily
+achieve a 15% improvement in overall read throughput by means of a more even
+utilization of SATA interface bandwidth and CPU cycles. This example is not
+merely a thought experiment meant to illustrate the theoretical benefits of
+adjusting primary affinity values; this fifteen percent improvement was
+achieved on an actual Ceph cluster.
+
+By default, every Ceph OSD has a primary affinity value of ``1``. In a cluster
+in which every OSD has this default value, all OSDs are equally likely to act
+as a primary OSD.
+
+By reducing the value of a Ceph OSD's primary affinity, you make CRUSH less
+likely to select the OSD as primary in a PG's acting set. To change the weight
+value associated with a specific OSD's primary affinity, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd primary-affinity <osd-id> <weight>
+
+The primary affinity of an OSD can be set to any real number in the range
+``[0-1]`` inclusive, where ``0`` indicates that the OSD may not be used as
+primary and ``1`` indicates that the OSD is maximally likely to be used as a
+primary. When the weight is between these extremes, its value indicates roughly
+how likely it is that CRUSH will select the OSD associated with it as a
+primary.
+
+The process by which CRUSH selects the lead OSD is not a mere function of a
+simple probability determined by relative affinity values. Nevertheless,
+measurable results can be achieved even with first-order approximations of
+desirable primary affinity values.
+
+
+Custom CRUSH Rules
+------------------
+
+Some clusters balance cost and performance by mixing SSDs and HDDs in the same
+replicated pool. By setting the primary affinity of HDD OSDs to ``0``,
+operations will be directed to an SSD OSD in each acting set. Alternatively,
+you can define a CRUSH rule that always selects an SSD OSD as the primary OSD
+and then selects HDDs for the remaining OSDs. Given this rule, each PG's acting
+set will contain an SSD OSD as the primary and have the remaining OSDs on HDDs.
+
+For example, see the following CRUSH rule::
+
+    rule mixed_replicated_rule {
+            id 11
+            type replicated
+            step take default class ssd
+            step chooseleaf firstn 1 type host
+            step emit
+            step take default class hdd
+            step chooseleaf firstn 0 type host
+            step emit
+    }
+
+This rule chooses an SSD as the first OSD. For an ``N``-times replicated pool,
+this rule selects ``N+1`` OSDs in order to guarantee that ``N`` copies are on
+different hosts, because the first SSD OSD might be colocated with any of the
+``N`` HDD OSDs.
+
+To avoid this extra storage requirement, you might place SSDs and HDDs in
+different hosts. However, taking this approach means that all client requests
+will be received by hosts with SSDs. For this reason, it might be advisable to
+have faster CPUs for SSD OSDs and more modest CPUs for HDD OSDs, since the
+latter will under normal circumstances perform only recovery operations. Here
+the CRUSH roots ``ssd_hosts`` and ``hdd_hosts`` are under a strict requirement
+not to contain any of the same servers, as seen in the following CRUSH rule::
+
+        rule mixed_replicated_rule_two {
+               id 1
+               type replicated
+               step take ssd_hosts class ssd
+               step chooseleaf firstn 1 type host
+               step emit
+               step take hdd_hosts class hdd
+               step chooseleaf firstn -1 type host
+               step emit
+        }
+
+.. note:: If a primary SSD OSD fails, then requests to the associated PG will
+   be temporarily served from a slower HDD OSD until the PG's data has been
+   replicated onto the replacement primary SSD OSD.
+
+
diff --git a/doc/rados/operations/data-placement.rst b/doc/rados/operations/data-placement.rst
new file mode 100644
index 000000000..3d3be65ec
--- /dev/null
+++ b/doc/rados/operations/data-placement.rst
@@ -0,0 +1,47 @@
+=========================
+ Data Placement Overview
+=========================
+
+Ceph stores, replicates, and rebalances data objects across a RADOS cluster
+dynamically. Because different users store objects in different pools for
+different purposes on many OSDs, Ceph operations require a certain amount of
+data- placement planning. The main data-placement planning concepts in Ceph
+include:
+
+- **Pools:** Ceph stores data within pools, which are logical groups used for
+  storing objects. Pools manage the number of placement groups, the number of
+  replicas, and the CRUSH rule for the pool. To store data in a pool, it is
+  necessary to be an authenticated user with permissions for the pool. Ceph is
+  able to make snapshots of pools. For additional details, see `Pools`_.
+
+- **Placement Groups:** Ceph maps objects to placement groups. Placement
+  groups (PGs) are shards or fragments of a logical object pool that place
+  objects as a group into OSDs. Placement groups reduce the amount of
+  per-object metadata that is necessary for Ceph to store the data in OSDs. A
+  greater number of placement groups (for example, 100 PGs per OSD as compared
+  with 50 PGs per OSD) leads to better balancing. For additional details, see
+  :ref:`placement groups`.
+
+- **CRUSH Maps:**  CRUSH plays a major role in allowing Ceph to scale while
+  avoiding certain pitfalls, such as performance bottlenecks, limitations to
+  scalability, and single points of failure. CRUSH maps provide the physical
+  topology of the cluster to the CRUSH algorithm, so that it can determine both
+  (1) where the data for an object and its replicas should be stored and (2)
+  how to store that data across failure domains so as to improve data safety.
+  For additional details, see `CRUSH Maps`_.
+
+- **Balancer:** The balancer is a feature that automatically optimizes the
+  distribution of placement groups across devices in order to achieve a
+  balanced data distribution, in order to maximize the amount of data that can
+  be stored in the cluster, and in order to evenly distribute the workload
+  across OSDs.
+
+It is possible to use the default values for each of the above components.
+Default values are recommended for a test cluster's initial setup. However,
+when planning a large Ceph cluster, values should be customized for
+data-placement operations with reference to the different roles played by
+pools, placement groups, and CRUSH.
+
+.. _Pools: ../pools
+.. _CRUSH Maps: ../crush-map
+.. _Balancer: ../balancer
diff --git a/doc/rados/operations/devices.rst b/doc/rados/operations/devices.rst
new file mode 100644
index 000000000..f92f622d5
--- /dev/null
+++ b/doc/rados/operations/devices.rst
@@ -0,0 +1,227 @@
+.. _devices:
+
+Device Management
+=================
+
+Device management allows Ceph to address hardware failure. Ceph tracks hardware
+storage devices (HDDs, SSDs) to see which devices are managed by which daemons.
+Ceph also collects health metrics about these devices. By doing so, Ceph can
+provide tools that predict hardware failure and can automatically respond to
+hardware failure.
+
+Device tracking
+---------------
+
+To see a list of the storage devices that are in use, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+Alternatively, to list devices by daemon or by host, run a command of one of
+the following forms:
+
+.. prompt:: bash $
+
+   ceph device ls-by-daemon <daemon>
+   ceph device ls-by-host <host>
+
+To see information about the location of an specific device and about how the
+device is being consumed, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device info <devid>
+
+Identifying physical devices
+----------------------------
+
+To make the replacement of failed disks easier and less error-prone, you can
+(in some cases) "blink" the drive's LEDs on hardware enclosures by running a
+command of the following form::
+
+  device light on|off <devid> [ident|fault] [--force]
+
+.. note:: Using this command to blink the lights might not work. Whether it
+   works will depend upon such factors as your kernel revision, your SES
+   firmware, or the setup of your HBA.
+
+The ``<devid>`` parameter is the device identification. To retrieve this
+information, run the following command:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+The ``[ident|fault]`` parameter determines which kind of light will blink.  By
+default, the `identification` light is used.
+
+.. note:: This command works only if the Cephadm or the Rook `orchestrator
+   <https://docs.ceph.com/docs/master/mgr/orchestrator/#orchestrator-cli-module>`_
+   module is enabled.  To see which orchestrator module is enabled, run the
+   following command:
+
+   .. prompt:: bash $
+
+      ceph orch status
+
+The command that makes the drive's LEDs blink is `lsmcli`. To customize this
+command, configure it via a Jinja2 template by running commands of the
+following forms::
+
+   ceph config-key set mgr/cephadm/blink_device_light_cmd "<template>"
+   ceph config-key set mgr/cephadm/<host>/blink_device_light_cmd "lsmcli local-disk-{{ ident_fault }}-led-{{'on' if on else 'off'}} --path '{{ path or dev }}'"
+
+The following arguments can be used to customize the Jinja2 template:
+
+* ``on``
+    A boolean value.
+* ``ident_fault``
+    A string that contains `ident` or `fault`.
+* ``dev``
+    A string that contains the device ID: for example, `SanDisk_X400_M.2_2280_512GB_162924424784`.
+* ``path``
+    A string that contains the device path: for example, `/dev/sda`.
+
+.. _enabling-monitoring:
+
+Enabling monitoring
+-------------------
+
+Ceph can also monitor the health metrics associated with your device. For
+example, SATA drives implement a standard called SMART that provides a wide
+range of internal metrics about the device's usage and health (for example: the
+number of hours powered on, the number of power cycles, the number of
+unrecoverable read errors). Other device types such as SAS and NVMe present a
+similar set of metrics (via slightly different standards).  All of these
+metrics can be collected by Ceph via the ``smartctl`` tool.
+
+You can enable or disable health monitoring by running one of the following
+commands:
+
+.. prompt:: bash $
+
+   ceph device monitoring on
+   ceph device monitoring off
+
+Scraping
+--------
+
+If monitoring is enabled, device metrics will be scraped automatically at
+regular intervals. To configure that interval, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/devicehealth/scrape_frequency <seconds>
+
+By default, device metrics are scraped once every 24 hours.
+
+To manually scrape all devices, run the following command:
+   
+.. prompt:: bash $
+
+   ceph device scrape-health-metrics
+
+To scrape a single device, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device scrape-health-metrics <device-id>
+
+To scrape a single daemon's devices, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device scrape-daemon-health-metrics <who>
+
+To retrieve the stored health metrics for a device (optionally for a specific
+timestamp),  run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device get-health-metrics <devid> [sample-timestamp]
+
+Failure prediction
+------------------
+
+Ceph can predict drive life expectancy and device failures by analyzing the
+health metrics that it collects. The prediction modes are as follows:
+
+* *none*: disable device failure prediction.
+* *local*: use a pre-trained prediction model from the ``ceph-mgr`` daemon.
+
+To configure the prediction mode, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph config set global device_failure_prediction_mode <mode>
+
+Under normal conditions, failure prediction runs periodically in the
+background.  For this reason, life expectancy values might be populated only
+after a significant amount of time has passed.  The life expectancy of all
+devices is displayed in the output of the following command:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+To see the metadata of a specific device, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device info <devid>
+
+To explicitly force prediction of a specific device's life expectancy, run a
+command of the following form:
+
+.. prompt:: bash $
+
+   ceph device predict-life-expectancy <devid>
+
+In addition to Ceph's internal device failure prediction, you might have an
+external source of information about device failures. To inform Ceph of a
+specific device's life expectancy, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph device set-life-expectancy <devid> <from> [<to>]
+
+Life expectancies are expressed as a time interval. This means that the
+uncertainty of the life expectancy can be expressed in the form of a range of
+time, and perhaps a wide range of time. The interval's end can be left
+unspecified.
+
+Health alerts
+-------------
+
+The ``mgr/devicehealth/warn_threshold`` configuration option controls the
+health check for an expected device failure. If the device is expected to fail
+within the specified time interval, an alert is raised.
+
+To check the stored life expectancy of all devices and generate any appropriate
+health alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph device check-health
+
+Automatic Migration
+-------------------
+
+The ``mgr/devicehealth/self_heal`` option (enabled by default) automatically
+migrates data away from devices that are expected to fail soon. If this option
+is enabled, the module marks such devices ``out`` so that automatic migration
+will occur.
+
+.. note:: The ``mon_osd_min_up_ratio`` configuration option can help prevent
+   this process from cascading to total failure. If the "self heal" module
+   marks ``out`` so many OSDs that the ratio value of ``mon_osd_min_up_ratio``
+   is exceeded, then the cluster raises the ``DEVICE_HEALTH_TOOMANY`` health
+   check. For instructions on what to do in this situation, see
+   :ref:`DEVICE_HEALTH_TOOMANY<rados_health_checks_device_health_toomany>`.
+
+The ``mgr/devicehealth/mark_out_threshold`` configuration option specifies the
+time interval for automatic migration. If a device is expected to fail within
+the specified time interval, it will be automatically marked ``out``.
diff --git a/doc/rados/operations/erasure-code-clay.rst b/doc/rados/operations/erasure-code-clay.rst
new file mode 100644
index 000000000..1cffa32f5
--- /dev/null
+++ b/doc/rados/operations/erasure-code-clay.rst
@@ -0,0 +1,240 @@
+================
+CLAY code plugin
+================
+
+CLAY (short for coupled-layer) codes are erasure codes designed to bring about significant savings 
+in terms of network bandwidth and disk IO when a failed node/OSD/rack is being repaired. Let:
+
+	d = number of OSDs contacted during repair
+
+If *jerasure* is configured with *k=8* and *m=4*, losing one OSD requires 
+reading from the *d=8* others to repair. And recovery of say a 1GiB needs
+a download of 8 X 1GiB = 8GiB of information.
+
+However, in the case of the *clay* plugin *d* is configurable within the limits:
+
+	k+1 <= d <= k+m-1 
+
+By default, the clay code plugin picks *d=k+m-1* as it provides the greatest savings in terms 
+of network bandwidth and disk IO. In the case of the *clay* plugin configured with 
+*k=8*, *m=4* and *d=11* when a single OSD fails, d=11 osds are contacted and 
+250MiB is downloaded from each of them, resulting in a total download of 11 X 250MiB = 2.75GiB 
+amount of information. More general parameters are provided below. The benefits are substantial 
+when the repair is carried out for a rack that stores information on the order of 
+Terabytes.
+
+	+-------------+---------------------------------------------------------+
+	| plugin      | total amount of disk IO                                 |
+	+=============+=========================================================+
+	|jerasure,isa | :math:`k S`                                             |
+	+-------------+---------------------------------------------------------+
+	| clay        | :math:`\frac{d S}{d - k + 1} = \frac{(k + m - 1) S}{m}` |
+	+-------------+---------------------------------------------------------+
+
+where *S* is the amount of data stored on a single OSD undergoing repair. In the table above, we have 
+used the largest possible value of *d* as this will result in the smallest amount of data download needed
+to achieve recovery from an OSD failure.
+
+Erasure-code profile examples
+=============================
+
+An example configuration that can be used to observe reduced bandwidth usage:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set CLAYprofile \
+      plugin=clay \
+      k=4 m=2 d=5 \
+      crush-failure-domain=host
+   ceph osd pool create claypool erasure CLAYprofile
+
+
+Creating a clay profile
+=======================
+
+To create a new clay code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+        plugin=clay \
+        k={data-chunks} \
+        m={coding-chunks} \
+        [d={helper-chunks}] \
+        [scalar_mds={plugin-name}] \
+        [technique={technique-name}] \
+        [crush-failure-domain={bucket-type}] \
+        [crush-device-class={device-class}] \
+        [directory={directory}] \
+        [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split into **data-chunks** parts,
+              each of which is stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``d={helper-chunks}``
+
+:Description: Number of OSDs requested to send data during recovery of
+              a single chunk. *d* needs to be chosen such that
+              k+1 <= d <= k+m-1. The larger the *d*, the better the savings.
+
+:Type: Integer
+:Required: No.
+:Default: k+m-1
+
+``scalar_mds={jerasure|isa|shec}``
+
+:Description: **scalar_mds** specifies the plugin that is used as a 
+             building block in the layered construction. It can be 
+             one of *jerasure*, *isa*, *shec*
+
+:Type: String
+:Required: No.
+:Default: jerasure
+
+``technique={technique}``
+
+:Description: **technique** specifies the technique that will be picked
+             within the 'scalar_mds' plugin specified. Supported techniques
+             are 'reed_sol_van', 'reed_sol_r6_op', 'cauchy_orig', 
+             'cauchy_good', 'liber8tion' for jerasure, 'reed_sol_van',
+             'cauchy' for isa and 'single', 'multiple' for shec.
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van (for jerasure, isa), single (for shec)
+
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+
+Notion of sub-chunks
+====================
+
+The Clay code is able to save in terms of disk IO, network bandwidth as it
+is a vector code and it is able to view and manipulate data within a chunk 
+at a finer granularity termed as a sub-chunk. The number of sub-chunks within 
+a chunk for a Clay code is given by:
+
+	sub-chunk count = :math:`q^{\frac{k+m}{q}}`, where :math:`q = d - k + 1`
+
+
+During repair of an OSD, the helper information requested
+from an available OSD is only a fraction of a chunk. In fact, the number
+of sub-chunks within a chunk that are accessed during repair is given by:
+
+	repair sub-chunk count = :math:`\frac{sub---chunk \: count}{q}`
+
+Examples
+--------
+
+#. For a configuration with *k=4*, *m=2*, *d=5*, the sub-chunk count is
+   8 and  the repair sub-chunk count is 4. Therefore, only half of a chunk is read 
+   during repair.
+#. When *k=8*, *m=4*, *d=11* the sub-chunk count is 64 and repair sub-chunk count
+   is 16. A quarter of a chunk is read from an available OSD for repair of a failed 
+   chunk.
+
+
+
+How to choose a configuration given a workload
+==============================================
+
+Only a few sub-chunks are read of all the sub-chunks within a chunk. These sub-chunks
+are not necessarily stored consecutively within a chunk. For best disk IO 
+performance, it is helpful to read contiguous data. For this reason, it is suggested that
+you choose stripe-size such that the sub-chunk size is sufficiently large.
+
+For a given stripe-size (that's fixed based on a workload), choose ``k``, ``m``, ``d`` such that:
+
+	sub-chunk size = :math:`\frac{stripe-size}{k sub-chunk count}` = 4KB, 8KB, 12KB ...
+
+#. For large size workloads for which the stripe size is large, it is easy to choose k, m, d.
+   For example consider a stripe-size of size 64MB, choosing *k=16*, *m=4* and *d=19* will
+   result in a sub-chunk count of 1024 and a sub-chunk size of 4KB.
+#. For small size workloads, *k=4*, *m=2* is a good configuration that provides both network
+   and disk IO benefits.
+
+Comparisons with LRC
+====================
+
+Locally Recoverable Codes (LRC) are also designed in order to save in terms of network
+bandwidth, disk IO during single OSD recovery. However, the focus in LRCs is to keep the
+number of OSDs contacted during repair (d) to be minimal, but this comes at the cost of storage overhead.
+The *clay* code has a storage overhead m/k. In the case of an *lrc*, it stores (k+m)/d parities in
+addition to the ``m`` parities resulting in a storage overhead (m+(k+m)/d)/k. Both *clay* and *lrc*
+can recover from the failure of any ``m`` OSDs.
+
+	+-----------------+----------------------------------+----------------------------------+
+	| Parameters      | disk IO, storage overhead (LRC)  | disk IO, storage overhead (CLAY) |
+	+=================+================+=================+==================================+
+	| (k=10, m=4)     | 7 * S, 0.6 (d=7)                 | 3.25 * S, 0.4 (d=13)             |
+	+-----------------+----------------------------------+----------------------------------+
+	| (k=16, m=4)     | 4 * S, 0.5625 (d=4)              | 4.75 * S, 0.25 (d=19)            |
+	+-----------------+----------------------------------+----------------------------------+
+
+
+where ``S`` is the amount of data stored of single OSD being recovered.
diff --git a/doc/rados/operations/erasure-code-isa.rst b/doc/rados/operations/erasure-code-isa.rst
new file mode 100644
index 000000000..9a43f89a2
--- /dev/null
+++ b/doc/rados/operations/erasure-code-isa.rst
@@ -0,0 +1,107 @@
+=======================
+ISA erasure code plugin
+=======================
+
+The *isa* plugin encapsulates the `ISA
+<https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version/>`_
+library.
+
+Create an isa profile
+=====================
+
+To create a new *isa* erasure code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+     plugin=isa \
+     technique={reed_sol_van|cauchy} \
+     [k={data-chunks}] \
+     [m={coding-chunks}] \
+     [crush-root={root}] \
+     [crush-failure-domain={bucket-type}] \
+     [crush-device-class={device-class}] \
+     [directory={directory}] \
+     [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: No.
+:Default: 7
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 3
+
+``technique={reed_sol_van|cauchy}``
+
+:Description: The ISA plugin comes in two `Reed Solomon
+              <https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction>`_
+              forms. If *reed_sol_van* is set, it is `Vandermonde
+              <https://en.wikipedia.org/wiki/Vandermonde_matrix>`_, if
+              *cauchy* is set, it is `Cauchy
+              <https://en.wikipedia.org/wiki/Cauchy_matrix>`_.
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
diff --git a/doc/rados/operations/erasure-code-jerasure.rst b/doc/rados/operations/erasure-code-jerasure.rst
new file mode 100644
index 000000000..8a0207748
--- /dev/null
+++ b/doc/rados/operations/erasure-code-jerasure.rst
@@ -0,0 +1,123 @@
+============================
+Jerasure erasure code plugin
+============================
+
+The *jerasure* plugin is the most generic and flexible plugin, it is
+also the default for Ceph erasure coded pools. 
+
+The *jerasure* plugin encapsulates the `Jerasure
+<https://github.com/ceph/jerasure>`_ library. It is
+recommended to read the ``jerasure`` documentation to
+understand the parameters. Note that the ``jerasure.org``
+web site as of 2023 may no longer be connected to the original
+project or legitimate.
+
+Create a jerasure profile
+=========================
+
+To create a new *jerasure* erasure code profile:
+ 
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+     plugin=jerasure \
+     k={data-chunks} \
+     m={coding-chunks} \
+     technique={reed_sol_van|reed_sol_r6_op|cauchy_orig|cauchy_good|liberation|blaum_roth|liber8tion} \
+     [crush-root={root}] \
+     [crush-failure-domain={bucket-type}] \
+     [crush-device-class={device-class}] \
+     [directory={directory}] \
+     [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``technique={reed_sol_van|reed_sol_r6_op|cauchy_orig|cauchy_good|liberation|blaum_roth|liber8tion}``
+
+:Description: The more flexible technique is *reed_sol_van* : it is
+              enough to set *k* and *m*. The *cauchy_good* technique
+              can be faster but you need to chose the *packetsize*
+              carefully. All of *reed_sol_r6_op*, *liberation*,
+              *blaum_roth*, *liber8tion* are *RAID6* equivalents in
+              the sense that they can only be configured with *m=2*. 
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van
+
+``packetsize={bytes}``
+
+:Description: The encoding will be done on packets of *bytes* size at
+              a time. Choosing the right packet size is difficult. The
+              *jerasure* documentation contains extensive information
+              on this topic.
+
+:Type: Integer
+:Required: No.
+:Default: 2048
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
diff --git a/doc/rados/operations/erasure-code-lrc.rst b/doc/rados/operations/erasure-code-lrc.rst
new file mode 100644
index 000000000..5329603b9
--- /dev/null
+++ b/doc/rados/operations/erasure-code-lrc.rst
@@ -0,0 +1,388 @@
+======================================
+Locally repairable erasure code plugin
+======================================
+
+With the *jerasure* plugin, when an erasure coded object is stored on
+multiple OSDs, recovering from the loss of one OSD requires reading
+from *k* others. For instance if *jerasure* is configured with
+*k=8* and *m=4*, recovering from the loss of one OSD requires reading
+from eight others.
+
+The *lrc* erasure code plugin creates local parity chunks to enable
+recovery using fewer surviving OSDs. For instance if *lrc* is configured with
+*k=8*, *m=4* and *l=4*, it will create an additional parity chunk for
+every four OSDs. When a single OSD is lost, it can be recovered with
+only four OSDs instead of eight.
+
+Erasure code profile examples
+=============================
+
+Reduce recovery bandwidth between hosts
+---------------------------------------
+
+Although it is probably not an interesting use case when all hosts are
+connected to the same switch, reduced bandwidth usage can actually be
+observed.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      k=4 m=2 l=3 \
+      crush-failure-domain=host
+   ceph osd pool create lrcpool erasure LRCprofile
+
+
+Reduce recovery bandwidth between racks
+---------------------------------------
+
+In Firefly the bandwidth reduction will only be observed if the primary
+OSD is in the same rack as the lost chunk.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      k=4 m=2 l=3 \
+      crush-locality=rack \
+      crush-failure-domain=host
+   ceph osd pool create lrcpool erasure LRCprofile
+
+
+Create an lrc profile
+=====================
+
+To create a new lrc erasure code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+       plugin=lrc \
+       k={data-chunks} \
+       m={coding-chunks} \
+       l={locality} \
+       [crush-root={root}] \
+       [crush-locality={bucket-type}] \
+       [crush-failure-domain={bucket-type}] \
+       [crush-device-class={device-class}] \
+       [directory={directory}] \
+       [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``l={locality}``
+
+:Description: Group the coding and data chunks into sets of size
+              **locality**. For instance, for **k=4** and **m=2**,
+              when **locality=3** two groups of three are created.
+              Each set can be recovered without reading chunks
+              from another set.
+
+:Type: Integer
+:Required: Yes.
+:Example: 3
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-locality={bucket-type}``
+
+:Description: The type of the CRUSH bucket in which each set of chunks
+              defined by **l** will be stored. For instance, if it is
+              set to **rack**, each group of **l** chunks will be
+              placed in a different rack. It is used to create a
+              CRUSH rule step such as **step choose rack**. If it is not
+              set, no such grouping is done.
+
+:Type: String
+:Required: No.
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+Low level plugin configuration
+==============================
+
+The sum of **k** and **m** must be a multiple of the **l** parameter.
+The low level configuration parameters however do not enforce this
+restriction and it may be advantageous to use them for specific
+purposes. It is for instance possible to define two groups, one with 4
+chunks and another with 3 chunks. It is also possible to recursively
+define locality sets, for instance datacenters and racks into
+datacenters. The **k/m/l** are implemented by generating a low level
+configuration.
+
+The *lrc* erasure code plugin recursively applies erasure code
+techniques so that recovering from the loss of some chunks only
+requires a subset of the available chunks, most of the time.
+
+For instance, when three coding steps are described as::
+
+   chunk nr    01234567
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+where *c* are coding chunks calculated from the data chunks *D*, the
+loss of chunk *7* can be recovered with the last four chunks. And the
+loss of chunk *2* chunk can be recovered with the first four
+chunks.
+
+Erasure code profile examples using low level configuration
+===========================================================
+
+Minimal testing
+---------------
+
+It is strictly equivalent to using a *K=2* *M=1* erasure code profile. The *DD*
+implies *K=2*, the *c* implies *M=1* and the *jerasure* plugin is used
+by default.:
+
+.. prompt:: bash $
+   
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      mapping=DD_ \
+      layers='[ [ "DDc", "" ] ]'
+   ceph osd pool create lrcpool erasure LRCprofile
+
+Reduce recovery bandwidth between hosts
+---------------------------------------
+
+Although it is probably not an interesting use case when all hosts are
+connected to the same switch, reduced bandwidth usage can actually be
+observed. It is equivalent to **k=4**, **m=2** and **l=3** although
+the layout of the chunks is different. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+        plugin=lrc \
+        mapping=__DD__DD \
+        layers='[
+                  [ "_cDD_cDD", "" ],
+                  [ "cDDD____", "" ],
+                  [ "____cDDD", "" ],
+                ]'
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+
+Reduce recovery bandwidth between racks
+---------------------------------------
+
+In Firefly the reduced bandwidth will only be observed if the primary OSD is in
+the same rack as the lost chunk. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+       plugin=lrc \
+       mapping=__DD__DD \
+       layers='[
+                 [ "_cDD_cDD", "" ],
+                 [ "cDDD____", "" ],
+                 [ "____cDDD", "" ],
+               ]' \
+       crush-steps='[
+                       [ "choose", "rack", 2 ],
+                       [ "chooseleaf", "host", 4 ],
+                      ]'
+  
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+Testing with different Erasure Code backends
+--------------------------------------------
+
+LRC now uses jerasure as the default EC backend. It is possible to
+specify the EC backend/algorithm on a per layer basis using the low
+level configuration. The second argument in layers='[ [ "DDc", "" ] ]'
+is actually an erasure code profile to be used for this level. The
+example below specifies the ISA backend with the cauchy technique to
+be used in the lrcpool.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      mapping=DD_ \
+      layers='[ [ "DDc", "plugin=isa technique=cauchy" ] ]'
+   ceph osd pool create lrcpool erasure LRCprofile
+
+You could also use a different erasure code profile for each
+layer. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+        plugin=lrc \
+        mapping=__DD__DD \
+        layers='[
+                  [ "_cDD_cDD", "plugin=isa technique=cauchy" ],
+                  [ "cDDD____", "plugin=isa" ],
+                  [ "____cDDD", "plugin=jerasure" ],
+                ]'
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+
+
+Erasure coding and decoding algorithm
+=====================================
+
+The steps found in the layers description::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+are applied in order. For instance, if a 4K object is encoded, it will
+first go through *step 1* and be divided in four 1K chunks (the four
+uppercase D). They are stored in the chunks 2, 3, 6 and 7, in
+order. From these, two coding chunks are calculated (the two lowercase
+c). The coding chunks are stored in the chunks 1 and 5, respectively.
+
+The *step 2* re-uses the content created by *step 1* in a similar
+fashion and stores a single coding chunk *c* at position 0. The last four
+chunks, marked with an underscore (*_*) for readability, are ignored.
+
+The *step 3* stores a single coding chunk *c* at position 4. The three
+chunks created by *step 1* are used to compute this coding chunk,
+i.e. the coding chunk from *step 1* becomes a data chunk in *step 3*.
+
+If chunk *2* is lost::
+
+   chunk nr    01234567
+
+   step 1      _c D_cDD
+   step 2      cD D____
+   step 3      __ _cDDD
+
+decoding will attempt to recover it by walking the steps in reverse
+order: *step 3* then *step 2* and finally *step 1*.
+
+The *step 3* knows nothing about chunk *2* (i.e. it is an underscore)
+and is skipped.
+
+The coding chunk from *step 2*, stored in chunk *0*, allows it to
+recover the content of chunk *2*. There are no more chunks to recover
+and the process stops, without considering *step 1*.
+
+Recovering chunk *2* requires reading chunks *0, 1, 3* and writing
+back chunk *2*.
+
+If chunk *2, 3, 6* are lost::
+
+   chunk nr    01234567
+
+   step 1      _c  _c D
+   step 2      cD  __ _
+   step 3      __  cD D
+
+The *step 3* can recover the content of chunk *6*::
+
+   chunk nr    01234567
+
+   step 1      _c  _cDD
+   step 2      cD  ____
+   step 3      __  cDDD
+
+The *step 2* fails to recover and is skipped because there are two
+chunks missing (*2, 3*) and it can only recover from one missing
+chunk.
+
+The coding chunk from *step 1*, stored in chunk *1, 5*, allows it to
+recover the content of chunk *2, 3*::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+Controlling CRUSH placement
+===========================
+
+The default CRUSH rule provides OSDs that are on different hosts. For instance::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+needs exactly *8* OSDs, one for each chunk. If the hosts are in two
+adjacent racks, the first four chunks can be placed in the first rack
+and the last four in the second rack. So that recovering from the loss
+of a single OSD does not require using bandwidth between the two
+racks.
+
+For instance::
+
+   crush-steps='[ [ "choose", "rack", 2 ], [ "chooseleaf", "host", 4 ] ]'
+
+will create a rule that will select two crush buckets of type
+*rack* and for each of them choose four OSDs, each of them located in
+different buckets of type *host*.
+
+The CRUSH rule can also be manually crafted for finer control.
diff --git a/doc/rados/operations/erasure-code-profile.rst b/doc/rados/operations/erasure-code-profile.rst
new file mode 100644
index 000000000..947b34c1f
--- /dev/null
+++ b/doc/rados/operations/erasure-code-profile.rst
@@ -0,0 +1,128 @@
+.. _erasure-code-profiles:
+
+=====================
+Erasure code profiles
+=====================
+
+Erasure code is defined by a **profile** and is used when creating an
+erasure coded pool and the associated CRUSH rule.
+
+The **default** erasure code profile (which is created when the Ceph
+cluster is initialized) will split the data into 2 equal-sized chunks,
+and have 2 parity chunks of the same size. It will take as much space
+in the cluster as a 2-replica pool but can sustain the data loss of 2
+chunks out of 4. It is described as a profile with **k=2** and **m=2**,
+meaning the information is spread over four OSD (k+m == 4) and two of
+them can be lost.
+
+To improve redundancy without increasing raw storage requirements, a
+new profile can be created. For instance, a profile with **k=10** and
+**m=4** can sustain the loss of four (**m=4**) OSDs by distributing an
+object on fourteen (k+m=14) OSDs. The object is first divided in
+**10** chunks (if the object is 10MB, each chunk is 1MB) and **4**
+coding chunks are computed, for recovery (each coding chunk has the
+same size as the data chunk, i.e. 1MB). The raw space overhead is only
+40% and the object will not be lost even if four OSDs break at the
+same time.
+
+.. _list of available plugins:
+
+.. toctree::
+	:maxdepth: 1
+
+	erasure-code-jerasure
+	erasure-code-isa
+	erasure-code-lrc
+	erasure-code-shec
+	erasure-code-clay
+
+osd erasure-code-profile set
+============================
+
+To create a new erasure code profile::
+
+	ceph osd erasure-code-profile set {name} \
+             [{directory=directory}] \
+             [{plugin=plugin}] \
+             [{stripe_unit=stripe_unit}] \
+             [{key=value} ...] \
+             [--force]
+
+Where:
+
+``{directory=directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``{plugin=plugin}``
+
+:Description: Use the erasure code **plugin** to compute coding chunks
+              and recover missing chunks. See the `list of available
+              plugins`_ for more information.
+
+:Type: String
+:Required: No.
+:Default: jerasure
+
+``{stripe_unit=stripe_unit}``
+
+:Description: The amount of data in a data chunk, per stripe. For
+              example, a profile with 2 data chunks and stripe_unit=4K
+              would put the range 0-4K in chunk 0, 4K-8K in chunk 1,
+              then 8K-12K in chunk 0 again. This should be a multiple
+              of 4K for best performance. The default value is taken
+              from the monitor config option
+              ``osd_pool_erasure_code_stripe_unit`` when a pool is
+              created.  The stripe_width of a pool using this profile
+              will be the number of data chunks multiplied by this
+              stripe_unit.
+
+:Type: String
+:Required: No.
+
+``{key=value}``
+
+:Description: The semantic of the remaining key/value pairs is defined
+              by the erasure code plugin.
+
+:Type: String
+:Required: No.
+
+``--force``
+
+:Description: Override an existing profile by the same name, and allow
+              setting a non-4K-aligned stripe_unit.
+
+:Type: String
+:Required: No.
+
+osd erasure-code-profile rm
+============================
+
+To remove an erasure code profile::
+
+	ceph osd erasure-code-profile rm {name}
+
+If the profile is referenced by a pool, the deletion will fail.
+
+.. warning:: Removing an erasure code profile using ``osd erasure-code-profile rm`` does not automatically delete the associated CRUSH rule associated with the erasure code profile. It is recommended to manually remove the associated CRUSH rule using ``ceph osd crush rule remove {rule-name}`` to avoid unexpected behavior.
+
+osd erasure-code-profile get
+============================
+
+To display an erasure code profile::
+
+	ceph osd erasure-code-profile get {name}
+
+osd erasure-code-profile ls
+===========================
+
+To list the names of all erasure code profiles::
+
+	ceph osd erasure-code-profile ls
+
diff --git a/doc/rados/operations/erasure-code-shec.rst b/doc/rados/operations/erasure-code-shec.rst
new file mode 100644
index 000000000..4e8f59b0b
--- /dev/null
+++ b/doc/rados/operations/erasure-code-shec.rst
@@ -0,0 +1,145 @@
+========================
+SHEC erasure code plugin
+========================
+
+The *shec* plugin encapsulates the `multiple SHEC
+<http://tracker.ceph.com/projects/ceph/wiki/Shingled_Erasure_Code_(SHEC)>`_
+library. It allows ceph to recover data more efficiently than Reed Solomon codes.
+
+Create an SHEC profile
+======================
+
+To create a new *shec* erasure code profile:
+
+.. prompt:: bash $
+
+        ceph osd erasure-code-profile set {name} \
+             plugin=shec \
+             [k={data-chunks}] \
+             [m={coding-chunks}] \
+             [c={durability-estimator}] \
+             [crush-root={root}] \
+             [crush-failure-domain={bucket-type}] \
+             [crush-device-class={device-class}] \
+             [directory={directory}] \
+             [--force]
+
+Where:
+
+``k={data-chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: No.
+:Default: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding-chunks** for each object and store them on
+              different OSDs. The number of **coding-chunks** does not necessarily
+              equal the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 3
+
+``c={durability-estimator}``
+
+:Description: The number of parity chunks each of which includes each data chunk in its
+              calculation range. The number is used as a **durability estimator**.
+              For instance, if c=2, 2 OSDs can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 2
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+Brief description of SHEC's layouts
+===================================
+
+Space Efficiency
+----------------
+
+Space efficiency is a ratio of data chunks to all ones in a object and
+represented as k/(k+m).
+In order to improve space efficiency, you should increase k or decrease m:
+
+        space efficiency of SHEC(4,3,2) = :math:`\frac{4}{4+3}` = 0.57
+        SHEC(5,3,2) or SHEC(4,2,2) improves SHEC(4,3,2)'s space efficiency
+
+Durability
+----------
+
+The third parameter of SHEC (=c) is a durability estimator, which approximates
+the number of OSDs that can be down without losing data.
+
+``durability estimator of SHEC(4,3,2) = 2``
+
+Recovery Efficiency
+-------------------
+
+Describing calculation of recovery efficiency is beyond the scope of this document,
+but at least increasing m without increasing c achieves improvement of recovery efficiency.
+(However, we must pay attention to the sacrifice of space efficiency in this case.)
+
+``SHEC(4,2,2) -> SHEC(4,3,2) : achieves improvement of recovery efficiency``
+
+Erasure code profile examples
+=============================
+
+
+.. prompt:: bash $
+
+          ceph osd erasure-code-profile set SHECprofile \
+             plugin=shec \
+             k=8 m=4 c=3 \
+             crush-failure-domain=host
+          ceph osd pool create shecpool erasure SHECprofile
diff --git a/doc/rados/operations/erasure-code.rst b/doc/rados/operations/erasure-code.rst
new file mode 100644
index 000000000..e2bd3c296
--- /dev/null
+++ b/doc/rados/operations/erasure-code.rst
@@ -0,0 +1,272 @@
+.. _ecpool:
+
+==============
+ Erasure code
+==============
+
+By default, Ceph `pools <../pools>`_ are created with the type "replicated". In
+replicated-type pools, every object is copied to multiple disks. This
+multiple copying is the method of data protection known as "replication".
+
+By contrast, `erasure-coded <https://en.wikipedia.org/wiki/Erasure_code>`_
+pools use a method of data protection that is different from replication. In
+erasure coding, data is broken into fragments of two kinds: data blocks and
+parity blocks. If a drive fails or becomes corrupted, the parity blocks are
+used to rebuild the data. At scale, erasure coding saves space relative to
+replication.
+
+In this documentation, data blocks are referred to as "data chunks"
+and parity blocks are referred to as "coding chunks".
+
+Erasure codes are also called "forward error correction codes". The
+first forward error correction code was developed in 1950 by Richard
+Hamming at Bell Laboratories.
+
+
+Creating a sample erasure-coded pool
+------------------------------------
+
+The simplest erasure-coded pool is similar to `RAID5
+<https://en.wikipedia.org/wiki/Standard_RAID_levels#RAID_5>`_ and
+requires at least three hosts:
+
+.. prompt:: bash $
+
+   ceph osd pool create ecpool erasure
+
+::
+
+   pool 'ecpool' created
+
+.. prompt:: bash $
+
+   echo ABCDEFGHI | rados --pool ecpool put NYAN -
+   rados --pool ecpool get NYAN -
+   
+::
+
+   ABCDEFGHI
+
+Erasure-code profiles
+---------------------
+
+The default erasure-code profile can sustain the overlapping loss of two OSDs
+without losing data. This erasure-code profile is equivalent to a replicated
+pool of size three, but with different storage requirements: instead of
+requiring 3TB to store 1TB, it requires only 2TB to store 1TB. The default
+profile can be displayed with this command:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile get default
+   
+::
+
+   k=2
+   m=2
+   plugin=jerasure
+   crush-failure-domain=host
+   technique=reed_sol_van
+
+.. note::
+  The profile just displayed is for the *default* erasure-coded pool, not the
+  *simplest* erasure-coded pool. These two pools are not the same:
+
+   The default erasure-coded pool has two data chunks (K) and two coding chunks
+   (M). The profile of the default erasure-coded pool is "k=2 m=2".
+
+   The simplest erasure-coded pool has two data chunks (K) and one coding chunk
+   (M). The profile of the simplest erasure-coded pool is "k=2 m=1".
+
+Choosing the right profile is important because the profile cannot be modified
+after the pool is created. If you find that you need an erasure-coded pool with
+a profile different than the one you have created, you must create a new pool
+with a different (and presumably more carefully considered) profile. When the
+new pool is created, all objects from the wrongly configured pool must be moved
+to the newly created pool. There is no way to alter the profile of a pool after
+the pool has been created.
+
+The most important parameters of the profile are *K*, *M*, and
+*crush-failure-domain* because they define the storage overhead and
+the data durability. For example, if the desired architecture must
+sustain the loss of two racks with a storage overhead of 67%,
+the following profile can be defined:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set myprofile \
+       k=3 \
+       m=2 \
+       crush-failure-domain=rack
+   ceph osd pool create ecpool erasure myprofile
+   echo ABCDEFGHI | rados --pool ecpool put NYAN -
+   rados --pool ecpool get NYAN -
+
+::
+
+    ABCDEFGHI
+
+The *NYAN* object will be divided in three (*K=3*) and two additional
+*chunks* will be created (*M=2*). The value of *M* defines how many
+OSDs can be lost simultaneously without losing any data. The
+*crush-failure-domain=rack* will create a CRUSH rule that ensures
+no two *chunks* are stored in the same rack.
+
+.. ditaa::
+                            +-------------------+
+                       name |       NYAN        |
+                            +-------------------+
+                    content |     ABCDEFGHI     |
+                            +--------+----------+
+                                     |
+                                     |
+                                     v
+                              +------+------+
+              +---------------+ encode(3,2) +-----------+
+              |               +--+--+---+---+           |
+              |                  |  |   |               |
+              |          +-------+  |   +-----+         |
+              |          |          |         |         |
+           +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+     name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+           +------+   +------+   +------+  +------+  +------+
+    shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+           +------+   +------+   +------+  +------+  +------+
+  content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+           +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+              |          |          |         |         |
+              |          |          v         |         |
+              |          |       +--+---+     |         |
+              |          |       | OSD1 |     |         |
+              |          |       +------+     |         |
+              |          |                    |         |
+              |          |       +------+     |         |
+              |          +------>| OSD2 |     |         |
+              |                  +------+     |         |
+              |                               |         |
+              |                  +------+     |         |
+              |                  | OSD3 |<----+         |
+              |                  +------+               |
+              |                                         |
+              |                  +------+               |
+              |                  | OSD4 |<--------------+
+              |                  +------+
+              |
+              |                  +------+
+              +----------------->| OSD5 |
+                                 +------+
+
+ 
+More information can be found in the `erasure-code profiles
+<../erasure-code-profile>`_ documentation.
+
+
+Erasure Coding with Overwrites
+------------------------------
+
+By default, erasure-coded pools work only with operations that
+perform full object writes and appends (for example, RGW).
+
+Since Luminous, partial writes for an erasure-coded pool may be
+enabled with a per-pool setting. This lets RBD and CephFS store their
+data in an erasure-coded pool:
+
+.. prompt:: bash $
+
+    ceph osd pool set ec_pool allow_ec_overwrites true
+
+This can be enabled only on a pool residing on BlueStore OSDs, since
+BlueStore's checksumming is used during deep scrubs to detect bitrot
+or other corruption. Using Filestore with EC overwrites is not only
+unsafe, but it also results in lower performance compared to BlueStore.
+
+Erasure-coded pools do not support omap, so to use them with RBD and
+CephFS you must instruct them to store their data in an EC pool and
+their metadata in a replicated pool. For RBD, this means using the
+erasure-coded pool as the ``--data-pool`` during image creation:
+
+.. prompt:: bash $
+
+    rbd create --size 1G --data-pool ec_pool replicated_pool/image_name
+
+For CephFS, an erasure-coded pool can be set as the default data pool during
+file system creation or via `file layouts <../../../cephfs/file-layouts>`_.
+
+
+Erasure-coded pools and cache tiering
+-------------------------------------
+
+.. note:: Cache tiering is deprecated in Reef.
+
+Erasure-coded pools require more resources than replicated pools and
+lack some of the functionality supported by replicated pools (for example, omap).
+To overcome these limitations, one can set up a `cache tier <../cache-tiering>`_
+before setting up the erasure-coded pool.
+
+For example, if the pool *hot-storage* is made of fast storage, the following commands
+will place the *hot-storage* pool as a tier of *ecpool* in *writeback*
+mode:
+
+.. prompt:: bash $
+
+   ceph osd tier add ecpool hot-storage
+   ceph osd tier cache-mode hot-storage writeback
+   ceph osd tier set-overlay ecpool hot-storage
+
+The result is that every write and read to the *ecpool* actually uses
+the *hot-storage* pool and benefits from its flexibility and speed.
+
+More information can be found in the `cache tiering
+<../cache-tiering>`_ documentation. Note, however, that cache tiering
+is deprecated and may be removed completely in a future release.
+
+Erasure-coded pool recovery
+---------------------------
+If an erasure-coded pool loses any data shards, it must recover them from others.
+This recovery involves reading from the remaining shards, reconstructing the data, and
+writing new shards.
+
+In Octopus and later releases, erasure-coded pools can recover as long as there are at least *K* shards
+available. (With fewer than *K* shards, you have actually lost data!)
+
+Prior to Octopus, erasure-coded pools required that at least ``min_size`` shards be
+available, even if ``min_size`` was greater than ``K``. This was a conservative
+decision made out of an abundance of caution when designing the new pool
+mode. As a result, however, pools with lost OSDs but without complete data loss were
+unable to recover and go active without manual intervention to temporarily change
+the ``min_size`` setting.
+
+We recommend that ``min_size`` be ``K+1`` or greater to prevent loss of writes and
+loss of data.
+
+
+
+Glossary
+--------
+
+*chunk*
+   When the encoding function is called, it returns chunks of the same size as each other. There are two
+   kinds of chunks: (1) *data chunks*, which can be concatenated to reconstruct the original object, and
+   (2) *coding chunks*, which can be used to rebuild a lost chunk.
+
+*K*
+   The number of data chunks into which an object is divided. For example, if *K* = 2, then a 10KB object
+   is divided into two objects of 5KB each.
+
+*M*
+   The number of coding chunks computed by the encoding function. *M* is equal to the number of OSDs that can
+   be missing from the cluster without the cluster suffering data loss. For example, if there are two coding
+   chunks, then two OSDs can be missing without data loss.
+
+Table of contents
+-----------------
+
+.. toctree::
+    :maxdepth: 1
+
+    erasure-code-profile
+    erasure-code-jerasure
+    erasure-code-isa
+    erasure-code-lrc
+    erasure-code-shec
+    erasure-code-clay
diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst
new file mode 100644
index 000000000..d52465602
--- /dev/null
+++ b/doc/rados/operations/health-checks.rst
@@ -0,0 +1,1619 @@
+.. _health-checks:
+
+===============
+ Health checks 
+===============
+
+Overview
+========
+
+There is a finite set of health messages that a Ceph cluster can raise. These
+messages are known as *health checks*. Each health check has a unique
+identifier.
+
+The identifier is a terse human-readable string -- that is, the identifier is
+readable in much the same way as a typical variable name. It is intended to
+enable tools (for example, UIs) to make sense of health checks and present them
+in a way that reflects their meaning.
+
+This page lists the health checks that are raised by the monitor and manager
+daemons. In addition to these, you might see health checks that originate
+from MDS daemons (see :ref:`cephfs-health-messages`), and health checks
+that are defined by ``ceph-mgr`` python modules.
+
+Definitions
+===========
+
+Monitor
+-------
+
+DAEMON_OLD_VERSION
+__________________
+
+Warn if one or more old versions of Ceph are running on any daemons.  A health
+check is raised if multiple versions are detected.  This condition must exist
+for a period of time greater than ``mon_warn_older_version_delay`` (set to one
+week by default) in order for the health check to be raised. This allows most
+upgrades to proceed without the occurrence of a false warning. If the upgrade
+is paused for an extended time period, ``health mute`` can be used by running
+``ceph health mute DAEMON_OLD_VERSION --sticky``. Be sure, however, to run
+``ceph health unmute DAEMON_OLD_VERSION`` after the upgrade has finished.
+
+MON_DOWN
+________
+
+One or more monitor daemons are currently down. The cluster requires a majority
+(more than one-half) of the monitors to be available. When one or more monitors
+are down, clients might have a harder time forming their initial connection to
+the cluster, as they might need to try more addresses before they reach an
+operating monitor.
+
+The down monitor daemon should be restarted as soon as possible to reduce the
+risk of a subsequent monitor failure leading to a service outage.
+
+MON_CLOCK_SKEW
+______________
+
+The clocks on the hosts running the ceph-mon monitor daemons are not
+well-synchronized. This health check is raised if the cluster detects a clock
+skew greater than ``mon_clock_drift_allowed``.
+
+This issue is best resolved by synchronizing the clocks by using a tool like
+``ntpd`` or ``chrony``.
+
+If it is impractical to keep the clocks closely synchronized, the
+``mon_clock_drift_allowed`` threshold can also be increased. However, this
+value must stay significantly below the ``mon_lease`` interval in order for the
+monitor cluster to function properly.
+
+MON_MSGR2_NOT_ENABLED
+_____________________
+
+The :confval:`ms_bind_msgr2` option is enabled but one or more monitors are
+not configured to bind to a v2 port in the cluster's monmap. This
+means that features specific to the msgr2 protocol (for example, encryption)
+are unavailable on some or all connections.
+
+In most cases this can be corrected by running the following command:
+
+.. prompt:: bash $
+
+   ceph mon enable-msgr2
+
+After this command is run, any monitor configured to listen on the old default
+port (6789) will continue to listen for v1 connections on 6789 and begin to
+listen for v2 connections on the new default port 3300.
+
+If a monitor is configured to listen for v1 connections on a non-standard port
+(that is, a port other than 6789), then the monmap will need to be modified
+manually.
+
+
+MON_DISK_LOW
+____________
+
+One or more monitors are low on disk space. This health check is raised if the
+percentage of available space on the file system used by the monitor database
+(normally ``/var/lib/ceph/mon``) drops below the percentage value
+``mon_data_avail_warn`` (default: 30%).
+
+This alert might indicate that some other process or user on the system is
+filling up the file system used by the monitor. It might also
+indicate that the monitor database is too large (see ``MON_DISK_BIG``
+below).
+
+If space cannot be freed, the monitor's data directory might need to be
+moved to another storage device or file system (this relocation process must be carried out while the monitor
+daemon is not running).
+
+
+MON_DISK_CRIT
+_____________
+
+One or more monitors are critically low on disk space. This health check is raised if the
+percentage of available space on the file system used by the monitor database
+(normally ``/var/lib/ceph/mon``) drops below the percentage value
+``mon_data_avail_crit`` (default: 5%). See ``MON_DISK_LOW``, above.
+
+MON_DISK_BIG
+____________
+
+The database size for one or more monitors is very large. This health check is
+raised if the size of the monitor database is larger than
+``mon_data_size_warn`` (default: 15 GiB).
+
+A large database is unusual, but does not necessarily indicate a problem.
+Monitor databases might grow in size when there are placement groups that have
+not reached an ``active+clean`` state in a long time.
+
+This alert might also indicate that the monitor's database is not properly
+compacting, an issue that has been observed with some older versions of leveldb
+and rocksdb. Forcing a compaction with ``ceph daemon mon.<id> compact`` might
+shrink the database's on-disk size.
+
+This alert might also indicate that the monitor has a bug that prevents it from
+pruning the cluster metadata that it stores. If the problem persists, please
+report a bug.
+
+To adjust the warning threshold, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set global mon_data_size_warn <size>
+
+
+AUTH_INSECURE_GLOBAL_ID_RECLAIM
+_______________________________
+
+One or more clients or daemons that are connected to the cluster are not
+securely reclaiming their ``global_id`` (a unique number that identifies each
+entity in the cluster) when reconnecting to a monitor. The client is being
+permitted to connect anyway because the
+``auth_allow_insecure_global_id_reclaim`` option is set to ``true`` (which may
+be necessary until all Ceph clients have been upgraded) and because the
+``auth_expose_insecure_global_id_reclaim`` option is set to ``true`` (which
+allows monitors to detect clients with "insecure reclaim" sooner by forcing
+those clients to reconnect immediately after their initial authentication).
+
+To identify which client(s) are using unpatched Ceph client code, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+If you collect a dump of the clients that are connected to an individual
+monitor and examine the ``global_id_status`` field in the output of the dump,
+you can see the ``global_id`` reclaim behavior of those clients. Here
+``reclaim_insecure`` means that a client is unpatched and is contributing to
+this health check.  To effect a client dump, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell mon.\* sessions
+
+We strongly recommend that all clients in the system be upgraded to a newer
+version of Ceph that correctly reclaims ``global_id`` values. After all clients
+have been updated, run the following command to stop allowing insecure
+reconnections:
+
+.. prompt:: bash $
+
+   ceph config set mon auth_allow_insecure_global_id_reclaim false
+
+If it is impractical to upgrade all clients immediately, you can temporarily
+silence this alert by running the following command:
+
+.. prompt:: bash $
+
+   ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1w   # 1 week
+
+Although we do NOT recommend doing so, you can also disable this alert
+indefinitely by running the following command:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_warn_on_insecure_global_id_reclaim false
+
+AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED
+_______________________________________
+
+Ceph is currently configured to allow clients that reconnect to monitors using
+an insecure process to reclaim their previous ``global_id``. Such reclaiming is
+allowed because, by default, ``auth_allow_insecure_global_id_reclaim`` is set
+to ``true``. It might be necessary to leave this setting enabled while existing
+Ceph clients are upgraded to newer versions of Ceph that correctly and securely
+reclaim their ``global_id``.
+
+If the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` health check has not also been
+raised and if the ``auth_expose_insecure_global_id_reclaim`` setting has not
+been disabled (it is enabled by default), then there are currently no clients
+connected that need to be upgraded. In that case, it is safe to disable
+``insecure global_id reclaim`` by running the following command:
+
+.. prompt:: bash $
+
+   ceph config set mon auth_allow_insecure_global_id_reclaim false
+
+On the other hand, if there are still clients that need to be upgraded, then
+this alert can be temporarily silenced by running the following command:
+
+.. prompt:: bash $
+
+   ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w   # 1 week
+
+Although we do NOT recommend doing so, you can also disable this alert indefinitely
+by running the following command:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
+
+
+Manager
+-------
+
+MGR_DOWN
+________
+
+All manager daemons are currently down. The cluster should normally have at
+least one running manager (``ceph-mgr``) daemon. If no manager daemon is
+running, the cluster's ability to monitor itself will be compromised, and parts
+of the management API will become unavailable (for example, the dashboard will
+not work, and most CLI commands that report metrics or runtime state will
+block). However, the cluster will still be able to perform all I/O operations
+and to recover from failures.
+
+The "down" manager daemon should be restarted as soon as possible to ensure
+that the cluster can be monitored (for example, so that the ``ceph -s``
+information is up to date, or so that metrics can be scraped by Prometheus).
+
+
+MGR_MODULE_DEPENDENCY
+_____________________
+
+An enabled manager module is failing its dependency check. This health check
+typically comes with an explanatory message from the module about the problem.
+
+For example, a module might report that a required package is not installed: in
+this case, you should install the required package and restart your manager
+daemons.
+
+This health check is applied only to enabled modules. If a module is not
+enabled, you can see whether it is reporting dependency issues in the output of
+`ceph module ls`.
+
+
+MGR_MODULE_ERROR
+________________
+
+A manager module has experienced an unexpected error. Typically, this means
+that an unhandled exception was raised from the module's `serve` function. The
+human-readable description of the error might be obscurely worded if the
+exception did not provide a useful description of itself.
+
+This health check might indicate a bug: please open a Ceph bug report if you
+think you have encountered a bug.
+
+However, if you believe the error is transient, you may restart your manager
+daemon(s) or use ``ceph mgr fail`` on the active daemon in order to force
+failover to another daemon.
+
+OSDs
+----
+
+OSD_DOWN
+________
+
+One or more OSDs are marked "down". The ceph-osd daemon might have been
+stopped, or peer OSDs might be unable to reach the OSD over the network.
+Common causes include a stopped or crashed daemon, a "down" host, or a network
+outage.
+
+Verify that the host is healthy, the daemon is started, and the network is
+functioning. If the daemon has crashed, the daemon log file
+(``/var/log/ceph/ceph-osd.*``) might contain debugging information.
+
+OSD_<crush type>_DOWN
+_____________________
+
+(for example, OSD_HOST_DOWN, OSD_ROOT_DOWN)
+
+All of the OSDs within a particular CRUSH subtree are marked "down" (for
+example, all OSDs on a host).
+
+OSD_ORPHAN
+__________
+
+An OSD is referenced in the CRUSH map hierarchy, but does not exist.
+
+To remove the OSD from the CRUSH map hierarchy, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd crush rm osd.<id>
+
+OSD_OUT_OF_ORDER_FULL
+_____________________
+
+The utilization thresholds for `nearfull`, `backfillfull`, `full`, and/or
+`failsafe_full` are not ascending. In particular, the following pattern is
+expected: `nearfull < backfillfull`, `backfillfull < full`, and `full <
+failsafe_full`.
+
+To adjust these utilization thresholds, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd set-nearfull-ratio <ratio>
+   ceph osd set-backfillfull-ratio <ratio>
+   ceph osd set-full-ratio <ratio>
+
+
+OSD_FULL
+________
+
+One or more OSDs have exceeded the `full` threshold and are preventing the
+cluster from servicing writes.
+
+To check utilization by pool, run the following command:
+
+.. prompt:: bash $
+
+   ceph df
+
+To see the currently defined `full` ratio, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd dump | grep full_ratio
+
+A short-term workaround to restore write availability is to raise the full
+threshold by a small amount. To do so, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd set-full-ratio <ratio>
+
+Additional OSDs should be deployed in order to add new storage to the cluster,
+or existing data should be deleted in order to free up space in the cluster.
+
+OSD_BACKFILLFULL
+________________
+
+One or more OSDs have exceeded the `backfillfull` threshold or *would* exceed
+it if the currently-mapped backfills were to finish, which will prevent data
+from rebalancing to this OSD. This alert is an early warning that
+rebalancing might be unable to complete and that the cluster is approaching
+full.
+
+To check utilization by pool, run the following command:
+
+.. prompt:: bash $
+
+   ceph df
+
+OSD_NEARFULL
+____________
+
+One or more OSDs have exceeded the `nearfull` threshold. This alert is an early
+warning that the cluster is approaching full.
+
+To check utilization by pool, run the following command:
+
+.. prompt:: bash $
+
+   ceph df
+
+OSDMAP_FLAGS
+____________
+
+One or more cluster flags of interest have been set. These flags include:
+
+* *full* - the cluster is flagged as full and cannot serve writes
+* *pauserd*, *pausewr* - there are paused reads or writes
+* *noup* - OSDs are not allowed to start
+* *nodown* - OSD failure reports are being ignored, and that means that the
+  monitors will not mark OSDs "down"
+* *noin* - OSDs that were previously marked ``out`` are not being marked
+  back ``in`` when they start
+* *noout* - "down" OSDs are not automatically being marked ``out`` after the
+  configured interval
+* *nobackfill*, *norecover*, *norebalance* - recovery or data
+  rebalancing is suspended
+* *noscrub*, *nodeep_scrub* - scrubbing is disabled
+* *notieragent* - cache-tiering activity is suspended
+
+With the exception of *full*, these flags can be set or cleared by running the
+following commands:
+
+.. prompt:: bash $
+
+   ceph osd set <flag>
+   ceph osd unset <flag>
+
+OSD_FLAGS
+_________
+
+One or more OSDs or CRUSH {nodes,device classes} have a flag of interest set.
+These flags include:
+
+* *noup*: these OSDs are not allowed to start
+* *nodown*: failure reports for these OSDs will be ignored
+* *noin*: if these OSDs were previously marked ``out`` automatically
+  after a failure, they will not be marked ``in`` when they start
+* *noout*: if these OSDs are "down" they will not automatically be marked
+  ``out`` after the configured interval
+
+To set and clear these flags in batch, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd set-group <flags> <who>
+   ceph osd unset-group <flags> <who>
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd set-group noup,noout osd.0 osd.1
+   ceph osd unset-group noup,noout osd.0 osd.1
+   ceph osd set-group noup,noout host-foo
+   ceph osd unset-group noup,noout host-foo
+   ceph osd set-group noup,noout class-hdd
+   ceph osd unset-group noup,noout class-hdd
+
+OLD_CRUSH_TUNABLES
+__________________
+
+The CRUSH map is using very old settings and should be updated. The oldest set
+of tunables that can be used (that is, the oldest client version that can
+connect to the cluster) without raising this health check is determined by the
+``mon_crush_min_required_version`` config option.  For more information, see
+:ref:`crush-map-tunables`.
+
+OLD_CRUSH_STRAW_CALC_VERSION
+____________________________
+
+The CRUSH map is using an older, non-optimal method of calculating intermediate
+weight values for ``straw`` buckets.
+
+The CRUSH map should be updated to use the newer method (that is:
+``straw_calc_version=1``). For more information, see :ref:`crush-map-tunables`.
+
+CACHE_POOL_NO_HIT_SET
+_____________________
+
+One or more cache pools are not configured with a *hit set* to track
+utilization. This issue prevents the tiering agent from identifying cold
+objects that are to be flushed and evicted from the cache.
+
+To configure hit sets on the cache pool, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set <poolname> hit_set_type <type>
+   ceph osd pool set <poolname> hit_set_period <period-in-seconds>
+   ceph osd pool set <poolname> hit_set_count <number-of-hitsets>
+   ceph osd pool set <poolname> hit_set_fpp <target-false-positive-rate>
+
+OSD_NO_SORTBITWISE
+__________________
+
+No pre-Luminous v12.y.z OSDs are running, but the ``sortbitwise`` flag has not
+been set.
+
+The ``sortbitwise`` flag must be set in order for OSDs running Luminous v12.y.z
+or newer to start. To safely set the flag, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd set sortbitwise
+
+OSD_FILESTORE
+__________________
+
+Warn if OSDs are running Filestore. The Filestore OSD back end has been
+deprecated; the BlueStore back end has been the default object store since the
+Ceph Luminous release.
+
+The 'mclock_scheduler' is not supported for Filestore OSDs. For this reason,
+the default 'osd_op_queue' is set to 'wpq' for Filestore OSDs and is enforced
+even if the user attempts to change it.
+
+
+
+.. prompt:: bash $
+
+   ceph report | jq -c '."osd_metadata" | .[] | select(.osd_objectstore | contains("filestore")) | {id, osd_objectstore}'
+
+**In order to upgrade to Reef or a later release, you must first migrate any
+Filestore OSDs to BlueStore.**
+
+If you are upgrading a pre-Reef release to Reef or later, but it is not
+feasible to migrate Filestore OSDs to BlueStore immediately, you can
+temporarily silence this alert by running the following command:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_FILESTORE
+
+Since this migration can take a considerable amount of time to complete, we
+recommend that you begin the process well in advance of any update to Reef or
+to later releases.
+
+POOL_FULL
+_________
+
+One or more pools have reached their quota and are no longer allowing writes.
+
+To see pool quotas and utilization, run the following command:
+
+.. prompt:: bash $
+
+   ceph df detail
+
+If you opt to raise the pool quota, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <poolname> max_objects <num-objects>
+   ceph osd pool set-quota <poolname> max_bytes <num-bytes>
+
+If not, delete some existing data to reduce utilization.
+
+BLUEFS_SPILLOVER
+________________
+
+One or more OSDs that use the BlueStore back end have been allocated `db`
+partitions (that is, storage space for metadata, normally on a faster device),
+but because that space has been filled, metadata has "spilled over" onto the
+slow device. This is not necessarily an error condition or even unexpected
+behavior, but may result in degraded performance. If the administrator had
+expected that all metadata would fit on the faster device, this alert indicates
+that not enough space was provided.
+
+To disable this alert on all OSDs, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set osd bluestore_warn_on_bluefs_spillover false
+
+Alternatively, to disable the alert on a specific OSD, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 bluestore_warn_on_bluefs_spillover false
+
+To secure more metadata space, you can destroy and reprovision the OSD in
+question. This process involves data migration and recovery.
+
+It might also be possible to expand the LVM logical volume that backs the `db`
+storage. If the underlying LV has been expanded, you must stop the OSD daemon
+and inform BlueFS of the device-size change by running the following command:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-$ID
+
+BLUEFS_AVAILABLE_SPACE
+______________________
+
+To see how much space is free for BlueFS, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore bluefs available
+
+This will output up to three values: ``BDEV_DB free``, ``BDEV_SLOW free``, and
+``available_from_bluestore``. ``BDEV_DB`` and ``BDEV_SLOW`` report the amount
+of space that has been acquired by BlueFS and is now considered free. The value
+``available_from_bluestore`` indicates the ability of BlueStore to relinquish
+more space to BlueFS.  It is normal for this value to differ from the amount of
+BlueStore free space, because the BlueFS allocation unit is typically larger
+than the BlueStore allocation unit.  This means that only part of the BlueStore
+free space will be available for BlueFS.
+
+BLUEFS_LOW_SPACE
+_________________
+
+If BlueFS is running low on available free space and there is not much free
+space available from BlueStore (in other words, `available_from_bluestore` has
+a low value), consider reducing the BlueFS allocation unit size. To simulate
+available space when the allocation unit is different, run the following
+command: 
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore bluefs available <alloc-unit-size>
+
+BLUESTORE_FRAGMENTATION
+_______________________
+
+As BlueStore operates, the free space on the underlying storage will become
+fragmented.  This is normal and unavoidable, but excessive fragmentation causes
+slowdown.  To inspect BlueStore fragmentation, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore allocator score block
+
+The fragmentation score is given in a [0-1] range.
+[0.0 .. 0.4] tiny fragmentation
+[0.4 .. 0.7] small, acceptable fragmentation
+[0.7 .. 0.9] considerable, but safe fragmentation
+[0.9 .. 1.0] severe fragmentation, might impact BlueFS's ability to get space from BlueStore
+
+To see a detailed report of free fragments, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore allocator dump block
+
+For OSD processes that are not currently running, fragmentation can be
+inspected with `ceph-bluestore-tool`. To see the fragmentation score, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-score
+
+To dump detailed free chunks, run the following command:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-dump
+
+BLUESTORE_LEGACY_STATFS
+_______________________
+
+One or more OSDs have BlueStore volumes that were created prior to the
+Nautilus release. (In Nautilus, BlueStore tracks its internal usage
+statistics on a granular, per-pool basis.)
+
+If *all* OSDs
+are older than Nautilus, this means that the per-pool metrics are
+simply unavailable. But if there is a mixture of pre-Nautilus and
+post-Nautilus OSDs, the cluster usage statistics reported by ``ceph
+df`` will be inaccurate.
+
+The old OSDs can be updated to use the new usage-tracking scheme by stopping
+each OSD, running a repair operation, and then restarting the OSD. For example,
+to update ``osd.123``, run the following commands:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+To disable this alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_legacy_statfs false
+
+BLUESTORE_NO_PER_POOL_OMAP
+__________________________
+
+One or more OSDs have volumes that were created prior to the Octopus release.
+(In Octopus and later releases, BlueStore tracks omap space utilization by
+pool.)
+
+If there are any BlueStore OSDs that do not have the new tracking enabled, the
+cluster will report an approximate value for per-pool omap usage based on the
+most recent deep scrub.
+
+The OSDs can be updated to track by pool by stopping each OSD, running a repair
+operation, and then restarting the OSD. For example, to update ``osd.123``, run
+the following commands:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+To disable this alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_no_per_pool_omap false
+
+BLUESTORE_NO_PER_PG_OMAP
+__________________________
+
+One or more OSDs have volumes that were created prior to Pacific.  (In Pacific
+and later releases Bluestore tracks omap space utilitzation by Placement Group
+(PG).)
+
+Per-PG omap allows faster PG removal when PGs migrate.
+
+The older OSDs can be updated to track by PG by stopping each OSD, running a
+repair operation, and then restarting the OSD. For example, to update
+``osd.123``, run the following commands:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+To disable this alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_no_per_pg_omap false
+
+
+BLUESTORE_DISK_SIZE_MISMATCH
+____________________________
+
+One or more BlueStore OSDs have an internal inconsistency between the size of
+the physical device and the metadata that tracks its size. This inconsistency
+can lead to the OSD(s) crashing in the future.
+
+The OSDs that have this inconsistency should be destroyed and reprovisioned. Be
+very careful to execute this procedure on only one OSD at a time, so as to
+minimize the risk of losing any data. To execute this procedure, where ``$N``
+is the OSD that has the inconsistency, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd out osd.$N
+   while ! ceph osd safe-to-destroy osd.$N ; do sleep 1m ; done
+   ceph osd destroy osd.$N
+   ceph-volume lvm zap /path/to/device
+   ceph-volume lvm create --osd-id $N --data /path/to/device
+
+.. note::
+
+   Wait for this recovery procedure to completely on one OSD before running it
+   on the next.
+
+BLUESTORE_NO_COMPRESSION
+________________________
+
+One or more OSDs is unable to load a BlueStore compression plugin.  This issue
+might be caused by a broken installation, in which the ``ceph-osd`` binary does
+not match the compression plugins. Or it might be caused by a recent upgrade in
+which the ``ceph-osd`` daemon was not restarted.
+
+To resolve this issue, verify that all of the packages on the host that is
+running the affected OSD(s) are correctly installed and that the OSD daemon(s)
+have been restarted. If the problem persists, check the OSD log for information
+about the source of the problem.
+
+BLUESTORE_SPURIOUS_READ_ERRORS
+______________________________
+
+One or more BlueStore OSDs detect spurious read errors on the main device.
+BlueStore has recovered from these errors by retrying disk reads.  This alert
+might indicate issues with underlying hardware, issues with the I/O subsystem,
+or something similar.  In theory, such issues can cause permanent data
+corruption.  Some observations on the root cause of spurious read errors can be
+found here: https://tracker.ceph.com/issues/22464
+
+This alert does not require an immediate response, but the affected host might
+need additional attention: for example, upgrading the host to the latest
+OS/kernel versions and implementing hardware-resource-utilization monitoring.
+
+To disable this alert on all OSDs, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set osd bluestore_warn_on_spurious_read_errors false
+
+Or, to disable this alert on a specific OSD, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 bluestore_warn_on_spurious_read_errors false
+
+Device health
+-------------
+
+DEVICE_HEALTH
+_____________
+
+One or more OSD devices are expected to fail soon, where the warning threshold
+is determined by the ``mgr/devicehealth/warn_threshold`` config option.
+
+Because this alert applies only to OSDs that are currently marked ``in``, the
+appropriate response to this expected failure is (1) to mark the OSD ``out`` so
+that data is migrated off of the OSD, and then (2) to remove the hardware from
+the system. Note that this marking ``out`` is normally done automatically if
+``mgr/devicehealth/self_heal`` is enabled (as determined by
+``mgr/devicehealth/mark_out_threshold``).
+
+To check device health, run the following command:
+
+.. prompt:: bash $
+
+   ceph device info <device-id>
+
+Device life expectancy is set either by a prediction model that the mgr runs or
+by an external tool that is activated by running the following command:
+
+.. prompt:: bash $
+
+   ceph device set-life-expectancy <device-id> <from> <to>
+
+You can change the stored life expectancy manually, but such a change usually
+doesn't accomplish anything. The reason for this is that whichever tool
+originally set the stored life expectancy will probably undo your change by
+setting it again, and a change to the stored value does not affect the actual
+health of the hardware device.
+
+DEVICE_HEALTH_IN_USE
+____________________
+
+One or more devices (that is, OSDs) are expected to fail soon and have been
+marked ``out`` of the cluster (as controlled by
+``mgr/devicehealth/mark_out_threshold``), but they are still participating in
+one or more Placement Groups. This might be because the OSD(s) were marked
+``out`` only recently and data is still migrating, or because data cannot be
+migrated off of the OSD(s) for some reason (for example, the cluster is nearly
+full, or the CRUSH hierarchy is structured so that there isn't another suitable
+OSD to migrate the data to).
+
+This message can be silenced by disabling self-heal behavior (that is, setting
+``mgr/devicehealth/self_heal`` to ``false``), by adjusting
+``mgr/devicehealth/mark_out_threshold``, or by addressing whichever condition
+is preventing data from being migrated off of the ailing OSD(s).
+
+.. _rados_health_checks_device_health_toomany:
+
+DEVICE_HEALTH_TOOMANY
+_____________________
+
+Too many devices (that is, OSDs) are expected to fail soon, and because
+``mgr/devicehealth/self_heal`` behavior is enabled, marking ``out`` all of the
+ailing OSDs would exceed the cluster's ``mon_osd_min_in_ratio`` ratio.  This
+ratio prevents a cascade of too many OSDs from being automatically marked
+``out``.
+
+You should promptly add new OSDs to the cluster to prevent data loss, or
+incrementally replace the failing OSDs.
+
+Alternatively, you can silence this health check by adjusting options including
+``mon_osd_min_in_ratio`` or ``mgr/devicehealth/mark_out_threshold``.  Be
+warned, however, that this will increase the likelihood of unrecoverable data
+loss.
+
+
+Data health (pools & placement groups)
+--------------------------------------
+
+PG_AVAILABILITY
+_______________
+
+Data availability is reduced. In other words, the cluster is unable to service
+potential read or write requests for at least some data in the cluster.  More
+precisely, one or more Placement Groups (PGs) are in a state that does not
+allow I/O requests to be serviced. Any of the following PG states are
+problematic if they do not clear quickly: *peering*, *stale*, *incomplete*, and
+the lack of *active*.
+
+For detailed information about which PGs are affected, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+In most cases, the root cause of this issue is that one or more OSDs are
+currently ``down``: see ``OSD_DOWN`` above.
+
+To see the state of a specific problematic PG, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+PG_DEGRADED
+___________
+
+Data redundancy is reduced for some data: in other words, the cluster does not
+have the desired number of replicas for all data (in the case of replicated
+pools) or erasure code fragments (in the case of erasure-coded pools).  More
+precisely, one or more Placement Groups (PGs):
+
+* have the *degraded* or *undersized* flag set, which means that there are not
+  enough instances of that PG in the cluster; or
+* have not had the *clean* state set for a long time.
+
+For detailed information about which PGs are affected, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+In most cases, the root cause of this issue is that one or more OSDs are
+currently "down": see ``OSD_DOWN`` above.
+
+To see the state of a specific problematic PG, run the following command:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+
+PG_RECOVERY_FULL
+________________
+
+Data redundancy might be reduced or even put at risk for some data due to a
+lack of free space in the cluster. More precisely, one or more Placement Groups
+have the *recovery_toofull* flag set, which means that the cluster is unable to
+migrate or recover data because one or more OSDs are above the ``full``
+threshold.
+
+For steps to resolve this condition, see *OSD_FULL* above.
+
+PG_BACKFILL_FULL
+________________
+
+Data redundancy might be reduced or even put at risk for some data due to a
+lack of free space in the cluster. More precisely, one or more Placement Groups
+have the *backfill_toofull* flag set, which means that the cluster is unable to
+migrate or recover data because one or more OSDs are above the ``backfillfull``
+threshold.
+
+For steps to resolve this condition, see *OSD_BACKFILLFULL* above.
+
+PG_DAMAGED
+__________
+
+Data scrubbing has discovered problems with data consistency in the cluster.
+More precisely, one or more Placement Groups either (1) have the *inconsistent*
+or ``snaptrim_error`` flag set, which indicates that an earlier data scrub
+operation found a problem, or (2) have the *repair* flag set, which means that
+a repair for such an inconsistency is currently in progress.
+
+For more information, see :doc:`pg-repair`.
+
+OSD_SCRUB_ERRORS
+________________
+
+Recent OSD scrubs have discovered inconsistencies. This alert is generally
+paired with *PG_DAMAGED* (see above).
+
+For more information, see :doc:`pg-repair`.
+
+OSD_TOO_MANY_REPAIRS
+____________________
+
+The count of read repairs has exceeded the config value threshold
+``mon_osd_warn_num_repaired`` (default: ``10``).  Because scrub handles errors
+only for data at rest, and because any read error that occurs when another
+replica is available will be repaired immediately so that the client can get
+the object data, there might exist failing disks that are not registering any
+scrub errors. This repair count is maintained as a way of identifying any such
+failing disks.
+
+
+LARGE_OMAP_OBJECTS
+__________________
+
+One or more pools contain large omap objects, as determined by
+``osd_deep_scrub_large_omap_object_key_threshold`` (threshold for the number of
+keys to determine what is considered a large omap object) or
+``osd_deep_scrub_large_omap_object_value_sum_threshold`` (the threshold for the
+summed size in bytes of all key values to determine what is considered a large
+omap object) or both.  To find more information on object name, key count, and
+size in bytes, search the cluster log for 'Large omap object found'. This issue
+can be caused by RGW-bucket index objects that do not have automatic resharding
+enabled. For more information on resharding, see :ref:`RGW Dynamic Bucket Index
+Resharding <rgw_dynamic_bucket_index_resharding>`.
+
+To adjust the thresholds mentioned above, run the following commands:
+
+.. prompt:: bash $
+
+   ceph config set osd osd_deep_scrub_large_omap_object_key_threshold <keys>
+   ceph config set osd osd_deep_scrub_large_omap_object_value_sum_threshold <bytes>
+
+CACHE_POOL_NEAR_FULL
+____________________
+
+A cache-tier pool is nearly full, as determined by the ``target_max_bytes`` and
+``target_max_objects`` properties of the cache pool. Once the pool reaches the
+target threshold, write requests to the pool might block while data is flushed
+and evicted from the cache. This state normally leads to very high latencies
+and poor performance.
+
+To adjust the cache pool's target size, run the following commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set <cache-pool-name> target_max_bytes <bytes>
+   ceph osd pool set <cache-pool-name> target_max_objects <objects>
+
+There might be other reasons that normal cache flush and evict activity are
+throttled: for example, reduced availability of the base tier, reduced
+performance of the base tier, or overall cluster load.
+
+TOO_FEW_PGS
+___________
+
+The number of Placement Groups (PGs) that are in use in the cluster is below
+the configurable threshold of ``mon_pg_warn_min_per_osd`` PGs per OSD. This can
+lead to suboptimal distribution and suboptimal balance of data across the OSDs
+in the cluster, and a reduction of overall performance.
+
+If data pools have not yet been created, this condition is expected.
+
+To address this issue, you can increase the PG count for existing pools or
+create new pools.  For more information, see
+:ref:`choosing-number-of-placement-groups`.
+
+POOL_PG_NUM_NOT_POWER_OF_TWO
+____________________________
+
+One or more pools have a ``pg_num`` value that is not a power of two.  Although
+this is not strictly incorrect, it does lead to a less balanced distribution of
+data because some Placement Groups will have roughly twice as much data as
+others have.
+
+This is easily corrected by setting the ``pg_num`` value for the affected
+pool(s) to a nearby power of two. To do so, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <value>
+
+To disable this health check, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false
+
+POOL_TOO_FEW_PGS
+________________
+
+One or more pools should probably have more Placement Groups (PGs), given the
+amount of data that is currently stored in the pool. This issue can lead to
+suboptimal distribution and suboptimal balance of data across the OSDs in the
+cluster, and a reduction of overall performance. This alert is raised only if
+the ``pg_autoscale_mode`` property on the pool is set to ``warn``.
+
+To disable the alert, entirely disable auto-scaling of PGs for the pool by
+running the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode off
+
+To allow the cluster to automatically adjust the number of PGs for the pool,
+run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode on
+
+Alternatively, to manually set the number of PGs for the pool to the
+recommended amount, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+For more information, see :ref:`choosing-number-of-placement-groups` and
+:ref:`pg-autoscaler`.
+
+TOO_MANY_PGS
+____________
+
+The number of Placement Groups (PGs) in use in the cluster is above the
+configurable threshold of ``mon_max_pg_per_osd`` PGs per OSD. If this threshold
+is exceeded, the cluster will not allow new pools to be created, pool `pg_num`
+to be increased, or pool replication to be increased (any of which, if allowed,
+would lead to more PGs in the cluster). A large number of PGs can lead to
+higher memory utilization for OSD daemons, slower peering after cluster state
+changes (for example, OSD restarts, additions, or removals), and higher load on
+the Manager and Monitor daemons.
+
+The simplest way to mitigate the problem is to increase the number of OSDs in
+the cluster by adding more hardware. Note that, because the OSD count that is
+used for the purposes of this health check is the number of ``in`` OSDs,
+marking ``out`` OSDs ``in`` (if there are any ``out`` OSDs available) can also
+help. To do so, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd in <osd id(s)>
+
+For more information, see :ref:`choosing-number-of-placement-groups`.
+
+POOL_TOO_MANY_PGS
+_________________
+
+One or more pools should probably have fewer Placement Groups (PGs), given the
+amount of data that is currently stored in the pool. This issue can lead to
+higher memory utilization for OSD daemons, slower peering after cluster state
+changes (for example, OSD restarts, additions, or removals), and higher load on
+the Manager and Monitor daemons. This alert is raised only if the
+``pg_autoscale_mode`` property on the pool is set to ``warn``.
+
+To disable the alert, entirely disable auto-scaling of PGs for the pool by
+running the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode off
+
+To allow the cluster to automatically adjust the number of PGs for the pool,
+run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode on
+
+Alternatively, to manually set the number of PGs for the pool to the
+recommended amount, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+For more information, see :ref:`choosing-number-of-placement-groups` and
+:ref:`pg-autoscaler`.
+
+
+POOL_TARGET_SIZE_BYTES_OVERCOMMITTED
+____________________________________
+
+One or more pools have a ``target_size_bytes`` property that is set in order to
+estimate the expected size of the pool, but the value(s) of this property are
+greater than the total available storage (either by themselves or in
+combination with other pools).
+
+This alert is usually an indication that the ``target_size_bytes`` value for
+the pool is too large and should be reduced or set to zero. To reduce the
+``target_size_bytes`` value or set it to zero, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> target_size_bytes 0
+
+The above command sets the value of ``target_size_bytes`` to zero. To set the
+value of ``target_size_bytes`` to a non-zero value, replace the ``0`` with that
+non-zero value.
+
+For more information, see :ref:`specifying_pool_target_size`.
+
+POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO
+____________________________________
+
+One or more pools have both ``target_size_bytes`` and ``target_size_ratio`` set
+in order to estimate the expected size of the pool.  Only one of these
+properties should be non-zero. If both are set to a non-zero value, then
+``target_size_ratio`` takes precedence and ``target_size_bytes`` is ignored.
+
+To reset ``target_size_bytes`` to zero, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> target_size_bytes 0
+
+For more information, see :ref:`specifying_pool_target_size`.
+
+TOO_FEW_OSDS
+____________
+
+The number of OSDs in the cluster is below the configurable threshold of
+``osd_pool_default_size``. This means that some or all data may not be able to
+satisfy the data protection policy specified in CRUSH rules and pool settings.
+
+SMALLER_PGP_NUM
+_______________
+
+One or more pools have a ``pgp_num`` value less than ``pg_num``. This alert is
+normally an indication that the Placement Group (PG) count was increased
+without any increase in the placement behavior.
+
+This disparity is sometimes brought about deliberately, in order to separate
+out the `split` step when the PG count is adjusted from the data migration that
+is needed when ``pgp_num`` is changed.
+
+This issue is normally resolved by setting ``pgp_num`` to match ``pg_num``, so
+as to trigger the data migration, by running the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool> pgp_num <pg-num-value>
+
+MANY_OBJECTS_PER_PG
+___________________
+
+One or more pools have an average number of objects per Placement Group (PG)
+that is significantly higher than the overall cluster average. The specific
+threshold is determined by the ``mon_pg_warn_max_object_skew`` configuration
+value.
+
+This alert is usually an indication that the pool(s) that contain most of the
+data in the cluster have too few PGs, or that other pools that contain less
+data have too many PGs. See *TOO_MANY_PGS* above.
+
+To silence the health check, raise the threshold by adjusting the
+``mon_pg_warn_max_object_skew`` config option on the managers.
+
+The health check will be silenced for a specific pool only if
+``pg_autoscale_mode`` is set to ``on``.
+
+POOL_APP_NOT_ENABLED
+____________________
+
+A pool exists but the pool has not been tagged for use by a particular
+application.
+
+To resolve this issue, tag the pool for use by an application. For
+example, if the pool is used by RBD, run the following command:
+
+.. prompt:: bash $
+
+   rbd pool init <poolname>
+
+Alternatively, if the pool is being used by a custom application (here 'foo'),
+you can label the pool by running the following low-level command:
+
+.. prompt:: bash $
+
+   ceph osd pool application enable foo
+
+For more information, see :ref:`associate-pool-to-application`.
+
+POOL_FULL
+_________
+
+One or more pools have reached (or are very close to reaching) their quota. The
+threshold to raise this health check is determined by the
+``mon_pool_quota_crit_threshold`` configuration option.
+
+Pool quotas can be adjusted up or down (or removed) by running the following
+commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <pool> max_bytes <bytes>
+   ceph osd pool set-quota <pool> max_objects <objects>
+
+To disable a quota, set the quota value to 0.
+
+POOL_NEAR_FULL
+______________
+
+One or more pools are approaching a configured fullness threshold.
+
+One of the several thresholds that can raise this health check is determined by
+the ``mon_pool_quota_warn_threshold`` configuration option.
+
+Pool quotas can be adjusted up or down (or removed) by running the following
+commands:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <pool> max_bytes <bytes>
+   ceph osd pool set-quota <pool> max_objects <objects>
+
+To disable a quota, set the quota value to 0.
+
+Other thresholds that can raise the two health checks above are
+``mon_osd_nearfull_ratio`` and ``mon_osd_full_ratio``. For details and
+resolution, see :ref:`storage-capacity` and :ref:`no-free-drive-space`.
+
+OBJECT_MISPLACED
+________________
+
+One or more objects in the cluster are not stored on the node that CRUSH would
+prefer that they be stored on. This alert is an indication that data migration
+due to a recent cluster change has not yet completed.
+
+Misplaced data is not a dangerous condition in and of itself; data consistency
+is never at risk, and old copies of objects will not be removed until the
+desired number of new copies (in the desired locations) has been created.
+
+OBJECT_UNFOUND
+______________
+
+One or more objects in the cluster cannot be found. More precisely, the OSDs
+know that a new or updated copy of an object should exist, but no such copy has
+been found on OSDs that are currently online.
+
+Read or write requests to unfound objects will block.
+
+Ideally, a "down" OSD that has a more recent copy of the unfound object can be
+brought back online. To identify candidate OSDs, check the peering state of the
+PG(s) responsible for the unfound object. To see the peering state, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+On the other hand, if the latest copy of the object is not available, the
+cluster can be told to roll back to a previous version of the object. For more
+information, see :ref:`failures-osd-unfound`.
+
+SLOW_OPS
+________
+
+One or more OSD requests or monitor requests are taking a long time to process.
+This alert might be an indication of extreme load, a slow storage device, or a
+software bug.
+
+To query the request queue for the daemon that is causing the slowdown, run the
+following command from the daemon's host:
+
+.. prompt:: bash $
+
+   ceph daemon osd.<id> ops
+
+To see a summary of the slowest recent requests, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon osd.<id> dump_historic_ops
+
+To see the location of a specific OSD, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd find osd.<id>
+
+PG_NOT_SCRUBBED
+_______________
+
+One or more Placement Groups (PGs) have not been scrubbed recently. PGs are
+normally scrubbed within an interval determined by
+:confval:`osd_scrub_max_interval` globally. This interval can be overridden on
+per-pool basis by changing the value of the variable
+:confval:`scrub_max_interval`. This health check is raised if a certain
+percentage (determined by ``mon_warn_pg_not_scrubbed_ratio``) of the interval
+has elapsed after the time the scrub was scheduled and no scrub has been
+performed.
+
+PGs will be scrubbed only if they are flagged as ``clean`` (which means that
+they are to be cleaned, and not that they have been examined and found to be
+clean). Misplaced or degraded PGs will not be flagged as ``clean`` (see
+*PG_AVAILABILITY* and *PG_DEGRADED* above).
+
+To manually initiate a scrub of a clean PG, run the following command:
+
+.. prompt: bash $
+
+   ceph pg scrub <pgid>
+
+PG_NOT_DEEP_SCRUBBED
+____________________
+
+One or more Placement Groups (PGs) have not been deep scrubbed recently. PGs
+are normally scrubbed every :confval:`osd_deep_scrub_interval` seconds at most.
+This health check is raised if a certain percentage (determined by
+``mon_warn_pg_not_deep_scrubbed_ratio``) of the interval has elapsed after the
+time the scrub was scheduled and no scrub has been performed.
+
+PGs will receive a deep scrub only if they are flagged as *clean* (which means
+that they are to be cleaned, and not that they have been examined and found to
+be clean). Misplaced or degraded PGs might not be flagged as ``clean`` (see
+*PG_AVAILABILITY* and *PG_DEGRADED* above).
+
+To manually initiate a deep scrub of a clean PG, run the following command:
+
+.. prompt:: bash $
+
+   ceph pg deep-scrub <pgid>
+
+
+PG_SLOW_SNAP_TRIMMING
+_____________________
+
+The snapshot trim queue for one or more PGs has exceeded the configured warning
+threshold. This alert indicates either that an extremely large number of
+snapshots was recently deleted, or that OSDs are unable to trim snapshots
+quickly enough to keep up with the rate of new snapshot deletions.
+
+The warning threshold is determined by the ``mon_osd_snap_trim_queue_warn_on``
+option (default: 32768).
+
+This alert might be raised if OSDs are under excessive load and unable to keep
+up with their background work, or if the OSDs' internal metadata database is
+heavily fragmented and unable to perform. The alert might also indicate some
+other performance issue with the OSDs.
+
+The exact size of the snapshot trim queue is reported by the ``snaptrimq_len``
+field of ``ceph pg ls -f json-detail``.
+
+Stretch Mode
+------------
+
+INCORRECT_NUM_BUCKETS_STRETCH_MODE
+__________________________________
+
+Stretch mode currently only support 2 dividing buckets with OSDs, this warning suggests
+that the number of dividing buckets is not equal to 2 after stretch mode is enabled.
+You can expect unpredictable failures and MON assertions until the condition is fixed.
+
+We encourage you to fix this by removing additional dividing buckets or bump the
+number of dividing buckets to 2.
+
+UNEVEN_WEIGHTS_STRETCH_MODE
+___________________________
+
+The 2 dividing buckets must have equal weights when stretch mode is enabled.
+This warning suggests that the 2 dividing buckets have uneven weights after
+stretch mode is enabled. This is not immediately fatal, however, you can expect
+Ceph to be confused when trying to process transitions between dividing buckets.
+
+We encourage you to fix this by making the weights even on both dividing buckets.
+This can be done by making sure the combined weight of the OSDs on each dividing
+bucket are the same.
+
+Miscellaneous
+-------------
+
+RECENT_CRASH
+____________
+
+One or more Ceph daemons have crashed recently, and the crash(es) have not yet
+been acknowledged and archived by the administrator. This alert might indicate
+a software bug, a hardware problem (for example, a failing disk), or some other
+problem.
+
+To list recent crashes, run the following command:
+
+.. prompt:: bash $
+
+   ceph crash ls-new
+
+To examine information about a specific crash, run the following command:
+
+.. prompt:: bash $
+
+   ceph crash info <crash-id>
+
+To silence this alert, you can archive the crash (perhaps after the crash
+has been examined by an administrator) by running the following command:
+
+.. prompt:: bash $
+
+   ceph crash archive <crash-id>
+
+Similarly, to archive all recent crashes, run the following command:
+
+.. prompt:: bash $
+
+   ceph crash archive-all
+
+Archived crashes will still be visible by running the command ``ceph crash
+ls``, but not by running the command ``ceph crash ls-new``.
+
+The time period that is considered recent is determined by the option
+``mgr/crash/warn_recent_interval`` (default: two weeks).
+
+To entirely disable this alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set mgr/crash/warn_recent_interval 0
+
+RECENT_MGR_MODULE_CRASH
+_______________________
+
+One or more ``ceph-mgr`` modules have crashed recently, and the crash(es) have
+not yet been acknowledged and archived by the administrator.  This alert
+usually indicates a software bug in one of the software modules that are
+running inside the ``ceph-mgr`` daemon. The module that experienced the problem
+might be disabled as a result, but other modules are unaffected and continue to
+function as expected.
+
+As with the *RECENT_CRASH* health check, a specific crash can be inspected by
+running the following command:
+
+.. prompt:: bash $
+
+   ceph crash info <crash-id>
+
+To silence this alert, you can archive the crash (perhaps after the crash has
+been examined by an administrator) by running the following command:
+
+.. prompt:: bash $
+
+   ceph crash archive <crash-id>
+
+Similarly, to archive all recent crashes, run the following command:
+
+.. prompt:: bash $
+
+   ceph crash archive-all
+
+Archived crashes will still be visible by running the command ``ceph crash ls``
+but not by running the command ``ceph crash ls-new``.
+
+The time period that is considered recent is determined by the option
+``mgr/crash/warn_recent_interval`` (default: two weeks).
+
+To entirely disable this alert, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set mgr/crash/warn_recent_interval 0
+
+TELEMETRY_CHANGED
+_________________
+
+Telemetry has been enabled, but because the contents of the telemetry report
+have changed in the meantime, telemetry reports will not be sent.
+
+Ceph developers occasionally revise the telemetry feature to include new and
+useful information, or to remove information found to be useless or sensitive.
+If any new information is included in the report, Ceph requires the
+administrator to re-enable telemetry. This requirement ensures that the
+administrator has an opportunity to (re)review the information that will be
+shared.
+
+To review the contents of the telemetry report, run the following command:
+
+.. prompt:: bash $
+
+   ceph telemetry show
+
+Note that the telemetry report consists of several channels that may be
+independently enabled or disabled. For more information, see :ref:`telemetry`.
+
+To re-enable telemetry (and silence the alert), run the following command:
+
+.. prompt:: bash $
+
+   ceph telemetry on
+
+To disable telemetry (and silence the alert), run the following command:
+
+.. prompt:: bash $
+
+   ceph telemetry off
+
+AUTH_BAD_CAPS
+_____________
+
+One or more auth users have capabilities that cannot be parsed by the monitors.
+As a general rule, this alert indicates that there are one or more daemon types
+that the user is not authorized to use to perform any action.
+
+This alert is most likely to be raised after an upgrade if (1) the capabilities
+were set with an older version of Ceph that did not properly validate the
+syntax of those capabilities, or if (2) the syntax of the capabilities has
+changed.
+
+To remove the user(s) in question, run the following command:
+
+.. prompt:: bash $
+
+   ceph auth rm <entity-name>
+
+(This resolves the health check, but it prevents clients from being able to
+authenticate as the removed user.)
+
+Alternatively, to update the capabilities for the user(s), run the following
+command:
+
+.. prompt:: bash $
+
+   ceph auth <entity-name> <daemon-type> <caps> [<daemon-type> <caps> ...]
+
+For more information about auth capabilities, see :ref:`user-management`.
+
+OSD_NO_DOWN_OUT_INTERVAL
+________________________
+
+The ``mon_osd_down_out_interval`` option is set to zero, which means that the
+system does not automatically perform any repair or healing operations when an
+OSD fails. Instead, an administrator an external orchestrator must manually
+mark "down" OSDs as ``out`` (by running ``ceph osd out <osd-id>``) in order to
+trigger recovery.
+
+This option is normally set to five or ten minutes, which should be enough time
+for a host to power-cycle or reboot.
+
+To silence this alert, set ``mon_warn_on_osd_down_out_interval_zero`` to
+``false`` by running the following command:
+
+.. prompt:: bash $
+
+   ceph config global mon mon_warn_on_osd_down_out_interval_zero false
+
+DASHBOARD_DEBUG
+_______________
+
+The Dashboard debug mode is enabled. This means that if there is an error while
+processing a REST API request, the HTTP error response will contain a Python
+traceback. This mode should be disabled in production environments because such
+a traceback might contain and expose sensitive information.
+
+To disable the debug mode, run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug disable
diff --git a/doc/rados/operations/index.rst b/doc/rados/operations/index.rst
new file mode 100644
index 000000000..15525c1d3
--- /dev/null
+++ b/doc/rados/operations/index.rst
@@ -0,0 +1,99 @@
+.. _rados-operations:
+
+====================
+ Cluster Operations
+====================
+
+.. raw:: html
+
+	<table><colgroup><col width="50%"><col width="50%"></colgroup><tbody valign="top"><tr><td><h3>High-level Operations</h3>
+
+High-level cluster operations consist primarily of starting, stopping, and
+restarting a cluster with the ``ceph`` service;  checking the cluster's health;
+and, monitoring an operating cluster.
+
+.. toctree::
+	:maxdepth: 1
+
+	operating
+	health-checks
+	monitoring
+	monitoring-osd-pg
+	user-management
+	pg-repair
+
+.. raw:: html
+
+	</td><td><h3>Data Placement</h3>
+
+Once you have your cluster up and running, you may begin working with data
+placement. Ceph supports petabyte-scale data storage clusters, with storage
+pools and placement groups that distribute data across the cluster using Ceph's
+CRUSH algorithm.
+
+.. toctree::
+	:maxdepth: 1
+
+	data-placement
+	pools
+	erasure-code
+	cache-tiering
+	placement-groups
+	upmap
+        read-balancer
+        balancer
+	crush-map
+	crush-map-edits
+	stretch-mode
+	change-mon-elections
+
+
+
+.. raw:: html
+
+	</td></tr><tr><td><h3>Low-level Operations</h3>
+
+Low-level cluster operations consist of starting, stopping, and restarting a
+particular daemon within a cluster; changing the settings of a particular
+daemon or subsystem; and, adding a daemon to the cluster or removing a	daemon
+from the cluster. The most common use cases for low-level operations include
+growing or shrinking the Ceph cluster and replacing legacy or failed hardware
+with new hardware.
+
+.. toctree::
+	:maxdepth: 1
+
+	add-or-rm-osds
+	add-or-rm-mons
+	devices
+	bluestore-migration
+	Command Reference <control>
+
+
+
+.. raw:: html
+
+	</td><td><h3>Troubleshooting</h3>
+
+Ceph is still on the leading edge, so you may encounter situations that require
+you to evaluate your Ceph configuration and modify your logging and debugging
+settings to identify and remedy issues you are encountering with your cluster.
+
+.. toctree::
+	:maxdepth: 1
+
+	../troubleshooting/community
+	../troubleshooting/troubleshooting-mon
+	../troubleshooting/troubleshooting-osd
+	../troubleshooting/troubleshooting-pg
+	../troubleshooting/log-and-debug
+	../troubleshooting/cpu-profiling
+	../troubleshooting/memory-profiling
+
+
+
+
+.. raw:: html
+
+	</td></tr></tbody></table>
+
diff --git a/doc/rados/operations/monitoring-osd-pg.rst b/doc/rados/operations/monitoring-osd-pg.rst
new file mode 100644
index 000000000..b0a6767a1
--- /dev/null
+++ b/doc/rados/operations/monitoring-osd-pg.rst
@@ -0,0 +1,556 @@
+=========================
+ Monitoring OSDs and PGs
+=========================
+
+High availability and high reliability require a fault-tolerant approach to
+managing hardware and software issues. Ceph has no single point of failure and
+it can service requests for data even when in a "degraded" mode. Ceph's `data
+placement`_ introduces a layer of indirection to ensure that data doesn't bind
+directly to specific OSDs. For this reason, tracking system faults
+requires finding the `placement group`_ (PG) and the underlying OSDs at the
+root of the problem.
+
+.. tip:: A fault in one part of the cluster might prevent you from accessing a
+   particular object, but that doesn't mean that you are prevented from
+   accessing other objects.  When you run into a fault, don't panic. Just
+   follow the steps for monitoring your OSDs and placement groups, and then
+   begin troubleshooting.
+
+Ceph is self-repairing. However, when problems persist, monitoring OSDs and
+placement groups will help you identify the problem.
+
+
+Monitoring OSDs
+===============
+
+An OSD is either *in* service (``in``) or *out* of service (``out``). An OSD is
+either running and reachable (``up``), or it is not running and not reachable
+(``down``). 
+
+If an OSD is ``up``, it may be either ``in`` service (clients can read and
+write data) or it is ``out`` of service. If the OSD was ``in`` but then due to
+a failure or a manual action was set to the ``out`` state, Ceph will migrate
+placement groups to the other OSDs to maintin the configured redundancy.  
+
+If an OSD is ``out`` of service, CRUSH will not assign placement groups to it.
+If an OSD is ``down``, it will also be ``out``.
+
+.. note:: If an OSD is ``down`` and ``in``, there is a problem and this
+   indicates that the cluster is not in a healthy state.
+
+.. ditaa::
+
+           +----------------+        +----------------+
+           |                |        |                |
+           |   OSD #n In    |        |   OSD #n Up    |
+           |                |        |                |
+           +----------------+        +----------------+
+                   ^                         ^
+                   |                         |
+                   |                         |
+                   v                         v
+           +----------------+        +----------------+
+           |                |        |                |
+           |   OSD #n Out   |        |   OSD #n Down  |
+           |                |        |                |
+           +----------------+        +----------------+
+
+If you run the commands ``ceph health``, ``ceph -s``, or ``ceph -w``,
+you might notice that the cluster does not always show ``HEALTH OK``. Don't
+panic. There are certain circumstances in which it is expected and normal that
+the cluster will **NOT** show ``HEALTH OK``:
+
+#. You haven't started the cluster yet.
+#. You have just started or restarted the cluster and it's not ready to show
+   health statuses yet, because the PGs are in the process of being created and
+   the OSDs are in the process of peering.
+#. You have just added or removed an OSD.
+#. You have just have modified your cluster map.
+
+Checking to see if OSDs are ``up`` and running is an important aspect of monitoring them:
+whenever the cluster is up and running, every OSD that is ``in`` the cluster should also
+be ``up`` and running. To see if all of the cluster's OSDs are running, run the following
+command:
+
+.. prompt:: bash $
+
+    ceph osd stat
+
+The output provides the following information: the total number of OSDs (x),
+how many OSDs are ``up`` (y), how many OSDs are ``in`` (z), and the map epoch (eNNNN). ::
+
+    x osds: y up, z in; epoch: eNNNN
+
+If the number of OSDs that are ``in`` the cluster is greater than the number of
+OSDs that are ``up``, run the following command to identify the ``ceph-osd``
+daemons that are not running:
+
+.. prompt:: bash $
+
+    ceph osd tree
+
+:: 
+
+    #ID CLASS WEIGHT  TYPE NAME             STATUS REWEIGHT PRI-AFF
+     -1       2.00000 pool openstack
+     -3       2.00000 rack dell-2950-rack-A
+     -2       2.00000 host dell-2950-A1
+      0   ssd 1.00000      osd.0                up  1.00000 1.00000
+      1   ssd 1.00000      osd.1              down  1.00000 1.00000
+
+.. tip:: Searching through a well-designed CRUSH hierarchy to identify the physical
+   locations of particular OSDs might help you troubleshoot your cluster.
+
+If an OSD is ``down``, start it by running the following command:
+
+.. prompt:: bash $
+
+    sudo systemctl start ceph-osd@1
+
+For problems associated with OSDs that have stopped or won't restart, see `OSD Not Running`_.
+
+
+PG Sets
+=======
+
+When CRUSH assigns a PG to OSDs, it takes note of how many replicas of the PG
+are required by the pool and then assigns each replica to a different OSD.
+For example, if the pool requires three replicas of a PG, CRUSH might assign
+them individually to ``osd.1``, ``osd.2`` and ``osd.3``. CRUSH seeks a
+pseudo-random placement that takes into account the failure domains that you
+have set in your `CRUSH map`_; for this reason, PGs are rarely assigned to
+immediately adjacent OSDs in a large cluster.
+
+Ceph processes client requests with the **Acting Set** of OSDs: this is the set
+of OSDs that currently have a full and working version of a PG shard and that
+are therefore responsible for handling requests. By contrast, the **Up Set** is
+the set of OSDs that contain a shard of a specific PG. Data is moved or copied
+to the **Up Set**, or planned to be moved or copied, to the **Up Set**. See
+:ref:`Placement Group Concepts <rados_operations_pg_concepts>`.
+
+Sometimes an OSD in the Acting Set is ``down`` or otherwise unable to
+service requests for objects in the PG. When this kind of situation
+arises, don't panic. Common examples of such a situation include:
+
+- You added or removed an OSD, CRUSH reassigned the PG to 
+  other OSDs, and this reassignment changed the composition of the Acting Set and triggered
+  the migration of data by means of a "backfill" process.
+- An OSD was ``down``, was restarted, and is now ``recovering``.
+- An OSD in the Acting Set is ``down`` or unable to service requests,
+  and another OSD has temporarily assumed its duties.
+
+Typically, the Up Set and the Acting Set are identical. When they are not, it
+might indicate that Ceph is migrating the PG (in other words, that the PG has
+been remapped), that an OSD is recovering, or that there is a problem with the
+cluster (in such scenarios, Ceph usually shows a "HEALTH WARN" state with a
+"stuck stale" message).
+
+To retrieve a list of PGs, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg dump
+
+To see which OSDs are within the Acting Set and the Up Set for a specific PG, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg map {pg-num}
+
+The output provides the following information: the osdmap epoch (eNNN), the PG number
+({pg-num}), the OSDs in the Up Set (up[]), and the OSDs in the Acting Set
+(acting[])::
+
+    osdmap eNNN pg {raw-pg-num} ({pg-num}) -> up [0,1,2] acting [0,1,2]
+
+.. note:: If the Up Set and the Acting Set do not match, this might indicate
+   that the cluster is rebalancing itself or that there is a problem with 
+   the cluster.
+
+
+Peering
+=======
+
+Before you can write data to a PG, it must be in an ``active`` state and it
+will preferably be in a ``clean`` state. For Ceph to determine the current
+state of a PG, peering must take place.  That is, the primary OSD of the PG
+(that is, the first OSD in the Acting Set) must peer with the secondary and
+OSDs so that consensus on the current state of the PG can be established. In
+the following diagram, we assume a pool with three replicas of the PG:
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+
+           |  OSD 1  |     |  OSD 2  |     | OSD 3 |
+           +---------+     +---------+     +-------+
+                |               |              |
+                |  Request To   |              |
+                |     Peer      |              |
+                |-------------->|              |
+                |<--------------|              |
+                |    Peering                   |
+                |                              |
+                |         Request To           |
+                |            Peer              |
+                |----------------------------->|
+                |<-----------------------------|
+                |          Peering             |
+
+The OSDs also report their status to the monitor. For details, see `Configuring Monitor/OSD
+Interaction`_. To troubleshoot peering issues, see `Peering
+Failure`_.
+
+
+Monitoring PG States
+====================
+
+If you run the commands ``ceph health``, ``ceph -s``, or ``ceph -w``,
+you might notice that the cluster does not always show ``HEALTH OK``. After
+first checking to see if the OSDs are running, you should also check PG
+states. There are certain PG-peering-related circumstances in which it is expected
+and normal that the cluster will **NOT** show ``HEALTH OK``:
+
+#. You have just created a pool and the PGs haven't peered yet.
+#. The PGs are recovering.
+#. You have just added an OSD to or removed an OSD from the cluster.
+#. You have just modified your CRUSH map and your PGs are migrating.
+#. There is inconsistent data in different replicas of a PG.
+#. Ceph is scrubbing a PG's replicas.
+#. Ceph doesn't have enough storage capacity to complete backfilling operations.
+
+If one of these circumstances causes Ceph to show ``HEALTH WARN``, don't
+panic. In many cases, the cluster will recover on its own. In some cases, however, you
+might need to take action. An important aspect of monitoring PGs is to check their
+status as ``active`` and ``clean``: that is, it is important to ensure that, when the
+cluster is up and running, all PGs are ``active`` and (preferably) ``clean``.
+To see the status of every PG, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg stat
+
+The output provides the following information: the total number of PGs (x), how many
+PGs are in a particular state such as ``active+clean`` (y), and the
+amount of data stored (z). ::
+
+    x pgs: y active+clean; z bytes data, aa MB used, bb GB / cc GB avail
+
+.. note:: It is common for Ceph to report multiple states for PGs (for example,
+   ``active+clean``, ``active+clean+remapped``, ``active+clean+scrubbing``.
+
+Here Ceph shows not only the PG states, but also storage capacity used (aa),
+the amount of storage capacity remaining (bb), and the total storage capacity
+of the PG. These values can be important in a few cases:
+
+- The cluster is reaching its ``near full ratio`` or ``full ratio``.
+- Data is not being distributed across the cluster due to an error in the
+  CRUSH configuration.
+
+
+.. topic:: Placement Group IDs
+
+   PG IDs consist of the pool number (not the pool name) followed by a period
+   (.) and a hexadecimal number. You can view pool numbers and their names from
+   in the output of ``ceph osd lspools``. For example, the first pool that was
+   created corresponds to pool number ``1``. A fully qualified PG ID has the
+   following form::
+
+       {pool-num}.{pg-id}
+
+   It typically resembles the following:: 
+
+    1.1701b
+
+
+To retrieve a list of PGs, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg dump
+
+To format the output in JSON format and save it to a file, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg dump -o {filename} --format=json
+
+To query a specific PG, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg {poolnum}.{pg-id} query
+
+Ceph will output the query in JSON format.
+
+The following subsections describe the most common PG states in detail.
+
+
+Creating
+--------
+
+PGs are created when you create a pool: the command that creates a pool
+specifies the total number of PGs for that pool, and when the pool is created
+all of those PGs are created as well. Ceph will echo ``creating`` while it is
+creating PGs. After the PG(s) are created, the OSDs that are part of a PG's
+Acting Set will peer. Once peering is complete, the PG status should be
+``active+clean``. This status means that Ceph clients begin writing to the
+PG.
+
+.. ditaa::
+
+       /-----------\       /-----------\       /-----------\
+       | Creating  |------>|  Peering  |------>|  Active   |
+       \-----------/       \-----------/       \-----------/
+
+Peering
+-------
+
+When a PG peers, the OSDs that store the replicas of its data converge on an
+agreed state of the data and metadata within that PG. When peering is complete,
+those OSDs agree about the state of that PG. However, completion of the peering
+process does **NOT** mean that each replica has the latest contents.
+
+.. topic:: Authoritative History
+
+   Ceph will **NOT** acknowledge a write operation to a client until that write
+   operation is persisted by every OSD in the Acting Set. This practice ensures
+   that at least one member of the Acting Set will have a record of every
+   acknowledged write operation since the last successful peering operation.
+   
+   Given an accurate record of each acknowledged write operation, Ceph can
+   construct a new authoritative history of the PG--that is, a complete and
+   fully ordered set of operations that, if performed, would bring an OSD’s
+   copy of the PG up to date.
+
+
+Active
+------
+
+After Ceph has completed the peering process, a PG should become ``active``.
+The ``active`` state means that the data in the PG is generally available for
+read and write operations in the primary and replica OSDs.
+
+
+Clean 
+-----
+
+When a PG is in the ``clean`` state, all OSDs holding its data and metadata
+have successfully peered and there are no stray replicas. Ceph has replicated
+all objects in the PG the correct number of times.
+
+
+Degraded
+--------
+
+When a client writes an object to the primary OSD, the primary OSD is
+responsible for writing the replicas to the replica OSDs. After the primary OSD
+writes the object to storage, the PG will remain in a ``degraded``
+state until the primary OSD has received an acknowledgement from the replica
+OSDs that Ceph created the replica objects successfully. 
+
+The reason that a PG can be ``active+degraded`` is that an OSD can be
+``active`` even if it doesn't yet hold all of the PG's objects. If an OSD goes
+``down``, Ceph marks each PG assigned to the OSD as ``degraded``. The PGs must
+peer again when the OSD comes back online. However, a client can still write a
+new object to a ``degraded`` PG if it is ``active``.
+
+If an OSD is ``down`` and the ``degraded`` condition persists, Ceph might mark the
+``down`` OSD as ``out`` of the cluster and remap the data from the ``down`` OSD
+to another OSD. The time between being marked ``down`` and being marked ``out``
+is determined by ``mon_osd_down_out_interval``, which is set to ``600`` seconds
+by default.
+
+A PG can also be in the ``degraded`` state because there are one or more
+objects that Ceph expects to find in the PG but that Ceph cannot find. Although
+you cannot read or write to unfound objects, you can still access all of the other
+objects in the ``degraded`` PG.
+
+
+Recovering
+----------
+
+Ceph was designed for fault-tolerance, because hardware and other server
+problems are expected or even routine. When an OSD goes ``down``, its contents
+might fall behind the current state of other replicas in the PGs. When the OSD
+has returned to the ``up`` state, the contents of the PGs must be updated to
+reflect that current state. During that time period, the OSD might be in a
+``recovering`` state.
+
+Recovery is not always trivial, because a hardware failure might cause a
+cascading failure of multiple OSDs. For example, a network switch for a rack or
+cabinet might fail, which can cause the OSDs of a number of host machines to
+fall behind the current state of the cluster. In such a scenario, general
+recovery is possible only if each of the OSDs recovers after the fault has been
+resolved.]
+
+Ceph provides a number of settings that determine how the cluster balances the
+resource contention between the need to process new service requests and the
+need to recover data objects and restore the PGs to the current state. The
+``osd_recovery_delay_start`` setting allows an OSD to restart, re-peer, and
+even process some replay requests before starting the recovery process. The
+``osd_recovery_thread_timeout`` setting determines the duration of a thread
+timeout, because multiple OSDs might fail, restart, and re-peer at staggered
+rates.  The ``osd_recovery_max_active`` setting limits the number of recovery
+requests an OSD can entertain simultaneously, in order to prevent the OSD from
+failing to serve.  The ``osd_recovery_max_chunk`` setting limits the size of
+the recovered data chunks, in order to prevent network congestion.
+
+
+Back Filling
+------------
+
+When a new OSD joins the cluster, CRUSH will reassign PGs from OSDs that are
+already in the cluster to the newly added OSD. It can put excessive load on the
+new OSD to force it to immediately accept the reassigned PGs. Back filling the
+OSD with the PGs allows this process to begin in the background. After the
+backfill operations have completed, the new OSD will begin serving requests as
+soon as it is ready.
+
+During the backfill operations, you might see one of several states:
+``backfill_wait`` indicates that a backfill operation is pending, but is not
+yet underway; ``backfilling`` indicates that a backfill operation is currently
+underway; and ``backfill_toofull`` indicates that a backfill operation was
+requested but couldn't be completed due to insufficient storage capacity. When
+a PG cannot be backfilled, it might be considered ``incomplete``.
+
+The ``backfill_toofull`` state might be transient. It might happen that, as PGs
+are moved around, space becomes available. The ``backfill_toofull`` state is
+similar to ``backfill_wait`` in that backfill operations can proceed as soon as
+conditions change.
+
+Ceph provides a number of settings to manage the load spike associated with the
+reassignment of PGs to an OSD (especially a new OSD). The ``osd_max_backfills``
+setting specifies the maximum number of concurrent backfills to and from an OSD
+(default: 1). The ``backfill_full_ratio`` setting allows an OSD to refuse a
+backfill request if the OSD is approaching its full ratio (default: 90%). This
+setting can be changed with the ``ceph osd set-backfillfull-ratio`` command. If
+an OSD refuses a backfill request, the ``osd_backfill_retry_interval`` setting
+allows an OSD to retry the request after a certain interval (default: 30
+seconds). OSDs can also set ``osd_backfill_scan_min`` and
+``osd_backfill_scan_max`` in order to manage scan intervals (default: 64 and
+512, respectively).
+
+
+Remapped
+--------
+
+When the Acting Set that services a PG changes, the data migrates from the old
+Acting Set to the new Acting Set. Because it might take time for the new
+primary OSD to begin servicing requests, the old primary OSD might be required
+to continue servicing requests until the PG data migration is complete. After
+data migration has completed, the mapping uses the primary OSD of the new
+Acting Set.
+
+
+Stale
+-----
+
+Although Ceph uses heartbeats in order to ensure that hosts and daemons are
+running, the ``ceph-osd`` daemons might enter a ``stuck`` state where they are
+not reporting statistics in a timely manner (for example, there might be a
+temporary network fault). By default, OSD daemons report their PG, up through,
+boot, and failure statistics every half second (that is, in accordance with a
+value of ``0.5``), which is more frequent than the reports defined by the
+heartbeat thresholds. If the primary OSD of a PG's Acting Set fails to report
+to the monitor or if other OSDs have reported the primary OSD ``down``, the
+monitors will mark the PG ``stale``.
+
+When you start your cluster, it is common to see the ``stale`` state until the
+peering process completes. After your cluster has been running for a while,
+however, seeing PGs in the ``stale`` state indicates that the primary OSD for
+those PGs is ``down`` or not reporting PG statistics to the monitor.
+
+
+Identifying Troubled PGs
+========================
+
+As previously noted, a PG is not necessarily having problems just because its
+state is not ``active+clean``. When PGs are stuck, this might indicate that
+Ceph cannot perform self-repairs. The stuck states include:
+
+- **Unclean**: PGs contain objects that have not been replicated the desired
+  number of times. Under normal conditions, it can be assumed that these PGs
+  are recovering.
+- **Inactive**: PGs cannot process reads or writes because they are waiting for
+  an OSD that has the most up-to-date data to come back ``up``.
+- **Stale**: PG are in an unknown state, because the OSDs that host them have
+  not reported to the monitor cluster for a certain period of time (determined
+  by ``mon_osd_report_timeout``).
+
+To identify stuck PGs, run the following command:
+
+.. prompt:: bash $
+
+    ceph pg dump_stuck [unclean|inactive|stale|undersized|degraded]
+
+For more detail, see `Placement Group Subsystem`_. To troubleshoot stuck PGs,
+see `Troubleshooting PG Errors`_.
+
+
+Finding an Object Location
+==========================
+
+To store object data in the Ceph Object Store, a Ceph client must: 
+
+#. Set an object name
+#. Specify a `pool`_
+
+The Ceph client retrieves the latest cluster map, the CRUSH algorithm
+calculates how to map the object to a PG, and then the algorithm calculates how
+to dynamically assign the PG to an OSD. To find the object location given only
+the object name and the pool name, run a command of the following form:
+
+.. prompt:: bash $
+
+    ceph osd map {poolname} {object-name} [namespace]
+
+.. topic:: Exercise: Locate an Object
+
+        As an exercise, let's create an object. We can specify an object name, a path
+        to a test file that contains some object data, and a pool name by using the
+        ``rados put`` command on the command line. For example:
+
+        .. prompt:: bash $
+   
+           rados put {object-name} {file-path} --pool=data
+           rados put test-object-1 testfile.txt --pool=data
+   
+        To verify that the Ceph Object Store stored the object, run the
+        following command:
+   
+        .. prompt:: bash $
+
+           rados -p data ls
+   
+        To identify the object location, run the following commands:
+
+        .. prompt:: bash $
+
+           ceph osd map {pool-name} {object-name}
+           ceph osd map data test-object-1
+
+        Ceph should output the object's location. For example:: 
+
+           osdmap e537 pool 'data' (1) object 'test-object-1' -> pg 1.d1743484 (1.4) -> up ([0,1], p0) acting ([0,1], p0)
+
+        To remove the test object, simply delete it by running the ``rados rm``
+        command. For example:
+
+        .. prompt:: bash $
+
+           rados rm test-object-1 --pool=data
+
+As the cluster evolves, the object location may change dynamically. One benefit
+of Ceph's dynamic rebalancing is that Ceph spares you the burden of manually
+performing the migration. For details, see the `Architecture`_ section.
+
+.. _data placement: ../data-placement
+.. _pool: ../pools
+.. _placement group: ../placement-groups
+.. _Architecture: ../../../architecture
+.. _OSD Not Running: ../../troubleshooting/troubleshooting-osd#osd-not-running
+.. _Troubleshooting PG Errors: ../../troubleshooting/troubleshooting-pg#troubleshooting-pg-errors
+.. _Peering Failure: ../../troubleshooting/troubleshooting-pg#failures-osd-peering
+.. _CRUSH map: ../crush-map
+.. _Configuring Monitor/OSD Interaction: ../../configuration/mon-osd-interaction/
+.. _Placement Group Subsystem: ../control#placement-group-subsystem
diff --git a/doc/rados/operations/monitoring.rst b/doc/rados/operations/monitoring.rst
new file mode 100644
index 000000000..a9171f2d8
--- /dev/null
+++ b/doc/rados/operations/monitoring.rst
@@ -0,0 +1,644 @@
+======================
+ Monitoring a Cluster
+======================
+
+After you have a running cluster, you can use the ``ceph`` tool to monitor your
+cluster. Monitoring a cluster typically involves checking OSD status, monitor
+status, placement group status, and metadata server status.
+
+Using the command line
+======================
+
+Interactive mode
+----------------
+
+To run the ``ceph`` tool in interactive mode, type ``ceph`` at the command line
+with no arguments. For example:
+
+.. prompt:: bash $
+
+    ceph
+
+.. prompt:: ceph>
+    :prompts: ceph>
+
+    health
+    status
+    quorum_status
+    mon stat
+
+Non-default paths
+-----------------
+
+If you specified non-default locations for your configuration or keyring when
+you install the cluster, you may specify their locations to the ``ceph`` tool
+by running the following command:
+
+.. prompt:: bash $
+
+   ceph -c /path/to/conf -k /path/to/keyring health
+
+Checking a Cluster's Status
+===========================
+
+After you start your cluster, and before you start reading and/or writing data,
+you should check your cluster's status.
+
+To check a cluster's status, run the following command:
+
+.. prompt:: bash $
+
+   ceph status
+
+Alternatively, you can run the following command:
+
+.. prompt:: bash $
+
+   ceph -s
+
+In interactive mode, this operation is performed by typing ``status`` and
+pressing **Enter**:
+
+.. prompt:: ceph>
+    :prompts: ceph>
+
+    status
+
+Ceph will print the cluster status. For example, a tiny Ceph "demonstration
+cluster" that is running one instance of each service (monitor, manager, and
+OSD) might print the following:
+
+::
+
+  cluster:
+    id:     477e46f1-ae41-4e43-9c8f-72c918ab0a20
+    health: HEALTH_OK
+   
+  services:
+    mon: 3 daemons, quorum a,b,c
+    mgr: x(active)
+    mds: cephfs_a-1/1/1 up  {0=a=up:active}, 2 up:standby
+    osd: 3 osds: 3 up, 3 in
+  
+  data:
+    pools:   2 pools, 16 pgs
+    objects: 21 objects, 2.19K
+    usage:   546 GB used, 384 GB / 931 GB avail
+    pgs:     16 active+clean
+
+
+How Ceph Calculates Data Usage
+------------------------------
+
+The ``usage`` value reflects the *actual* amount of raw storage used. The ``xxx
+GB / xxx GB`` value means the amount available (the lesser number) of the
+overall storage capacity of the cluster. The notional number reflects the size
+of the stored data before it is replicated, cloned or snapshotted.  Therefore,
+the amount of data actually stored typically exceeds the notional amount
+stored, because Ceph creates replicas of the data and may also use storage
+capacity for cloning and snapshotting.
+
+
+Watching a Cluster
+==================
+
+Each daemon in the Ceph cluster maintains a log of events, and the Ceph cluster
+itself maintains a *cluster log* that records high-level events about the
+entire Ceph cluster.  These events are logged to disk on monitor servers (in
+the default location ``/var/log/ceph/ceph.log``), and they can be monitored via
+the command line.
+
+To follow the cluster log, run the following command:
+
+.. prompt:: bash $
+
+   ceph -w
+
+Ceph will print the status of the system, followed by each log message as it is
+added. For example:
+
+:: 
+
+  cluster:
+    id:     477e46f1-ae41-4e43-9c8f-72c918ab0a20
+    health: HEALTH_OK
+  
+  services:
+    mon: 3 daemons, quorum a,b,c
+    mgr: x(active)
+    mds: cephfs_a-1/1/1 up  {0=a=up:active}, 2 up:standby
+    osd: 3 osds: 3 up, 3 in
+  
+  data:
+    pools:   2 pools, 16 pgs
+    objects: 21 objects, 2.19K
+    usage:   546 GB used, 384 GB / 931 GB avail
+    pgs:     16 active+clean
+  
+  
+  2017-07-24 08:15:11.329298 mon.a mon.0 172.21.9.34:6789/0 23 : cluster [INF] osd.0 172.21.9.34:6806/20527 boot
+  2017-07-24 08:15:14.258143 mon.a mon.0 172.21.9.34:6789/0 39 : cluster [INF] Activating manager daemon x
+  2017-07-24 08:15:15.446025 mon.a mon.0 172.21.9.34:6789/0 47 : cluster [INF] Manager daemon x is now available
+
+Instead of printing log lines as they are added, you might want to print only
+the most recent lines. Run ``ceph log last [n]`` to see the most recent ``n``
+lines from the cluster log.
+
+Monitoring Health Checks
+========================
+
+Ceph continuously runs various *health checks*. When
+a health check fails, this failure is reflected in the output of ``ceph status`` and
+``ceph health``. The cluster log receives messages that
+indicate when a check has failed and when the cluster has recovered.
+
+For example, when an OSD goes down, the ``health`` section of the status
+output is updated as follows:
+
+::
+
+    health: HEALTH_WARN
+            1 osds down
+            Degraded data redundancy: 21/63 objects degraded (33.333%), 16 pgs unclean, 16 pgs degraded
+
+At the same time, cluster log messages are emitted to record the failure of the 
+health checks:
+
+::
+
+    2017-07-25 10:08:58.265945 mon.a mon.0 172.21.9.34:6789/0 91 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)
+    2017-07-25 10:09:01.302624 mon.a mon.0 172.21.9.34:6789/0 94 : cluster [WRN] Health check failed: Degraded data redundancy: 21/63 objects degraded (33.333%), 16 pgs unclean, 16 pgs degraded (PG_DEGRADED)
+
+When the OSD comes back online, the cluster log records the cluster's return
+to a healthy state:
+
+::
+
+    2017-07-25 10:11:11.526841 mon.a mon.0 172.21.9.34:6789/0 109 : cluster [WRN] Health check update: Degraded data redundancy: 2 pgs unclean, 2 pgs degraded, 2 pgs undersized (PG_DEGRADED)
+    2017-07-25 10:11:13.535493 mon.a mon.0 172.21.9.34:6789/0 110 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs unclean, 2 pgs degraded, 2 pgs undersized)
+    2017-07-25 10:11:13.535577 mon.a mon.0 172.21.9.34:6789/0 111 : cluster [INF] Cluster is now healthy
+
+Network Performance Checks
+--------------------------
+
+Ceph OSDs send heartbeat ping messages to each other in order to monitor daemon
+availability and network performance. If a single delayed response is detected,
+this might indicate nothing more than a busy OSD. But if multiple delays
+between distinct pairs of OSDs are detected, this might indicate a failed
+network switch, a NIC failure, or a layer 1 failure.
+
+By default, a heartbeat time that exceeds 1 second (1000 milliseconds) raises a
+health check (a ``HEALTH_WARN``. For example:
+
+::
+
+    HEALTH_WARN Slow OSD heartbeats on back (longest 1118.001ms)
+
+In the output of the ``ceph health detail`` command, you can see which OSDs are
+experiencing delays and how long the delays are. The output of ``ceph health
+detail`` is limited to ten lines. Here is an example of the output you can
+expect from the ``ceph health detail`` command::
+
+    [WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1118.001ms)
+        Slow OSD heartbeats on back from osd.0 [dc1,rack1] to osd.1 [dc1,rack1] 1118.001 msec possibly improving
+        Slow OSD heartbeats on back from osd.0 [dc1,rack1] to osd.2 [dc1,rack2] 1030.123 msec
+        Slow OSD heartbeats on back from osd.2 [dc1,rack2] to osd.1 [dc1,rack1] 1015.321 msec
+        Slow OSD heartbeats on back from osd.1 [dc1,rack1] to osd.0 [dc1,rack1] 1010.456 msec
+
+To see more detail and to collect a complete dump of network performance
+information, use the ``dump_osd_network`` command. This command is usually sent
+to a Ceph Manager Daemon, but it can be used to collect information about a
+specific OSD's interactions by sending it to that OSD. The default threshold
+for a slow heartbeat is 1 second (1000 milliseconds), but this can be
+overridden by providing a number of milliseconds as an argument.
+
+To show all network performance data with a specified threshold of 0, send the
+following command to the mgr:
+
+.. prompt:: bash $
+
+   ceph daemon /var/run/ceph/ceph-mgr.x.asok dump_osd_network 0
+
+::
+
+    {
+        "threshold": 0,
+        "entries": [
+            {
+                "last update": "Wed Sep  4 17:04:49 2019",
+                "stale": false,
+                "from osd": 2,
+                "to osd": 0,
+                "interface": "front",
+                "average": {
+                    "1min": 1.023,
+                    "5min": 0.860,
+                    "15min": 0.883
+                },
+                "min": {
+                    "1min": 0.818,
+                    "5min": 0.607,
+                    "15min": 0.607
+                },
+                "max": {
+                    "1min": 1.164,
+                    "5min": 1.173,
+                    "15min": 1.544
+                },
+                "last": 0.924
+            },
+            {
+                "last update": "Wed Sep  4 17:04:49 2019",
+                "stale": false,
+                "from osd": 2,
+                "to osd": 0,
+                "interface": "back",
+                "average": {
+                    "1min": 0.968,
+                    "5min": 0.897,
+                    "15min": 0.830
+                },
+                "min": {
+                    "1min": 0.860,
+                    "5min": 0.563,
+                    "15min": 0.502
+                },
+                "max": {
+                    "1min": 1.171,
+                    "5min": 1.216,
+                    "15min": 1.456
+                },
+                "last": 0.845
+            },
+            {
+                "last update": "Wed Sep  4 17:04:48 2019",
+                "stale": false,
+                "from osd": 0,
+                "to osd": 1,
+                "interface": "front",
+                "average": {
+                    "1min": 0.965,
+                    "5min": 0.811,
+                    "15min": 0.850
+                },
+                "min": {
+                    "1min": 0.650,
+                    "5min": 0.488,
+                    "15min": 0.466
+                },
+                "max": {
+                    "1min": 1.252,
+                    "5min": 1.252,
+                    "15min": 1.362
+                },
+            "last": 0.791
+        },
+        ...
+
+
+
+Muting Health Checks
+--------------------
+
+Health checks can be muted so that they have no effect on the overall
+reported status of the cluster. For example, if the cluster has raised a
+single health check and then you mute that health check, then the cluster will report a status of ``HEALTH_OK``.
+To mute a specific health check, use the health check code that corresponds to that health check (see :ref:`health-checks`), and 
+run the following command:
+
+.. prompt:: bash $
+
+   ceph health mute <code>
+
+For example, to mute an ``OSD_DOWN`` health check, run the following command:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN
+
+Mutes are reported as part of the short and long form of the ``ceph health`` command's output.
+For example, in the above scenario, the cluster would report:
+
+.. prompt:: bash $
+
+   ceph health
+
+::
+
+   HEALTH_OK (muted: OSD_DOWN)
+
+.. prompt:: bash $
+
+   ceph health detail
+
+::
+
+   HEALTH_OK (muted: OSD_DOWN)
+   (MUTED) OSD_DOWN 1 osds down
+       osd.1 is down
+
+A mute can be removed by running the following command:
+
+.. prompt:: bash $
+
+   ceph health unmute <code>
+
+For example:
+
+.. prompt:: bash $
+
+   ceph health unmute OSD_DOWN
+
+A "health mute" can have a TTL (**T**\ime **T**\o **L**\ive)
+associated with it: this means that the mute will automatically expire
+after a specified period of time. The TTL is specified as an optional
+duration argument, as seen in the following examples:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN 4h    # mute for 4 hours
+   ceph health mute MON_DOWN 15m   # mute for 15 minutes
+
+Normally, if a muted health check is resolved (for example, if the OSD that raised the ``OSD_DOWN`` health check 
+in the example above has come back up), the mute goes away. If the health check comes
+back later, it will be reported in the usual way.
+
+It is possible to make a health mute "sticky": this means that the mute will remain even if the
+health check clears. For example, to make a health mute "sticky", you might run the following command:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN 1h --sticky   # ignore any/all down OSDs for next hour
+
+Most health mutes disappear if the unhealthy condition that triggered the health check gets worse.
+For example, suppose that there is one OSD down and the health check is muted. In that case, if
+one or more additional OSDs go down, then the health mute disappears. This behavior occurs in any health check with a threshold value.
+
+
+Checking a Cluster's Usage Stats
+================================
+
+To check a cluster's data usage and data distribution among pools, use the
+``df`` command. This option is similar to Linux's ``df`` command. Run the
+following command:
+
+.. prompt:: bash $
+
+   ceph df
+
+The output of ``ceph df`` resembles the following::
+
+   CLASS     SIZE    AVAIL     USED  RAW USED  %RAW USED
+   ssd    202 GiB  200 GiB  2.0 GiB   2.0 GiB       1.00
+   TOTAL  202 GiB  200 GiB  2.0 GiB   2.0 GiB       1.00
+
+   --- POOLS ---
+   POOL                   ID  PGS   STORED   (DATA)   (OMAP)   OBJECTS     USED  (DATA)   (OMAP)   %USED  MAX AVAIL  QUOTA OBJECTS  QUOTA BYTES  DIRTY  USED COMPR  UNDER COMPR
+   device_health_metrics   1    1  242 KiB   15 KiB  227 KiB         4  251 KiB  24 KiB  227 KiB       0    297 GiB            N/A          N/A      4         0 B          0 B
+   cephfs.a.meta           2   32  6.8 KiB  6.8 KiB      0 B        22   96 KiB  96 KiB      0 B       0    297 GiB            N/A          N/A     22         0 B          0 B
+   cephfs.a.data           3   32      0 B      0 B      0 B         0      0 B     0 B      0 B       0     99 GiB            N/A          N/A      0         0 B          0 B
+   test                    4   32   22 MiB   22 MiB   50 KiB       248   19 MiB  19 MiB   50 KiB       0    297 GiB            N/A          N/A    248         0 B          0 B
+   
+- **CLASS:** For example, "ssd" or "hdd".
+- **SIZE:** The amount of storage capacity managed by the cluster.
+- **AVAIL:** The amount of free space available in the cluster.
+- **USED:** The amount of raw storage consumed by user data (excluding
+  BlueStore's database).
+- **RAW USED:** The amount of raw storage consumed by user data, internal
+  overhead, and reserved capacity.
+- **%RAW USED:** The percentage of raw storage used. Watch this number in
+  conjunction with ``full ratio`` and ``near full ratio`` to be forewarned when
+  your cluster approaches the fullness thresholds. See `Storage Capacity`_.
+
+
+**POOLS:**
+
+The POOLS section of the output provides a list of pools and the *notional*
+usage of each pool. This section of the output **DOES NOT** reflect replicas,
+clones, or snapshots. For example, if you store an object with 1MB of data,
+then the notional usage will be 1MB, but the actual usage might be 2MB or more
+depending on the number of replicas, clones, and snapshots.
+
+- **ID:** The number of the specific node within the pool.
+- **STORED:** The actual amount of data that the user has stored in a pool.
+  This is similar to the USED column in earlier versions of Ceph, but the
+  calculations (for BlueStore!) are more precise (in that gaps are properly
+  handled).
+
+  - **(DATA):** Usage for RBD (RADOS Block Device), CephFS file data, and RGW
+    (RADOS Gateway) object data.
+  - **(OMAP):** Key-value pairs. Used primarily by CephFS and RGW (RADOS
+    Gateway) for metadata storage.
+
+- **OBJECTS:** The notional number of objects stored per pool (that is, the
+  number of objects other than replicas, clones, or snapshots). 
+- **USED:** The space allocated for a pool over all OSDs. This includes space
+  for replication, space for allocation granularity, and space for the overhead
+  associated with erasure-coding. Compression savings and object-content gaps
+  are also taken into account. However, BlueStore's database is not included in
+  the amount reported under USED.
+
+  - **(DATA):** Object usage for RBD (RADOS Block Device), CephFS file data,
+    and RGW (RADOS Gateway) object data.
+  - **(OMAP):** Object key-value pairs. Used primarily by CephFS and RGW (RADOS
+    Gateway) for metadata storage.
+
+- **%USED:** The notional percentage of storage used per pool.
+- **MAX AVAIL:** An estimate of the notional amount of data that can be written
+  to this pool.
+- **QUOTA OBJECTS:** The number of quota objects.
+- **QUOTA BYTES:** The number of bytes in the quota objects.
+- **DIRTY:** The number of objects in the cache pool that have been written to
+  the cache pool but have not yet been flushed to the base pool. This field is
+  available only when cache tiering is in use.
+- **USED COMPR:** The amount of space allocated for compressed data. This
+  includes compressed data in addition to all of the space required for
+  replication, allocation granularity, and erasure- coding overhead.
+- **UNDER COMPR:** The amount of data that has passed through compression
+  (summed over all replicas) and that is worth storing in a compressed form.
+
+
+.. note:: The numbers in the POOLS section are notional. They do not include
+   the number of replicas, clones, or snapshots. As a result, the sum of the
+   USED and %USED amounts in the POOLS section of the output will not be equal
+   to the sum of the USED and %USED amounts in the RAW section of the output.
+
+.. note:: The MAX AVAIL value is a complicated function of the replication or
+   the kind of erasure coding used, the CRUSH rule that maps storage to
+   devices, the utilization of those devices, and the configured
+   ``mon_osd_full_ratio`` setting.
+
+
+Checking OSD Status
+===================
+
+To check if OSDs are ``up`` and ``in``, run the
+following command:
+
+.. prompt:: bash #
+
+  ceph osd stat
+
+Alternatively, you can run the following command:
+
+.. prompt:: bash #
+
+  ceph osd dump
+
+To view OSDs according to their position in the CRUSH map, run the following
+command:
+
+.. prompt:: bash #
+
+   ceph osd tree
+
+To print out a CRUSH tree that displays a host, its OSDs, whether the OSDs are
+``up``, and the weight of the OSDs, run the following command:
+
+.. code-block:: bash
+
+   #ID CLASS WEIGHT  TYPE NAME             STATUS REWEIGHT PRI-AFF
+    -1       3.00000 pool default
+    -3       3.00000 rack mainrack
+    -2       3.00000 host osd-host
+     0   ssd 1.00000         osd.0             up  1.00000 1.00000
+     1   ssd 1.00000         osd.1             up  1.00000 1.00000
+     2   ssd 1.00000         osd.2             up  1.00000 1.00000
+
+See `Monitoring OSDs and Placement Groups`_.
+
+Checking Monitor Status
+=======================
+
+If your cluster has multiple monitors, then you need to perform certain
+"monitor status" checks.  After starting the cluster and before reading or
+writing data, you should check quorum status. A quorum must be present when
+multiple monitors are running to ensure proper functioning of your Ceph
+cluster. Check monitor status regularly in order to ensure that all of the
+monitors are running.
+
+To display the monitor map, run the following command:
+
+.. prompt:: bash $
+
+   ceph mon stat
+
+Alternatively, you can run the following command:
+
+.. prompt:: bash $
+
+   ceph mon dump
+
+To check the quorum status for the monitor cluster, run the following command:
+
+.. prompt:: bash $
+
+   ceph quorum_status
+
+Ceph returns the quorum status. For example, a Ceph cluster that consists of
+three monitors might return the following:
+
+.. code-block:: javascript
+
+    { "election_epoch": 10,
+      "quorum": [
+            0,
+            1,
+            2],
+      "quorum_names": [
+        "a",
+        "b",
+        "c"],
+      "quorum_leader_name": "a",
+      "monmap": { "epoch": 1,
+          "fsid": "444b489c-4f16-4b75-83f0-cb8097468898",
+          "modified": "2011-12-12 13:28:27.505520",
+          "created": "2011-12-12 13:28:27.505520",
+          "features": {"persistent": [
+                "kraken",
+                "luminous",
+                "mimic"],
+        "optional": []
+          },
+          "mons": [
+                { "rank": 0,
+                  "name": "a",
+                  "addr": "127.0.0.1:6789/0",
+              "public_addr": "127.0.0.1:6789/0"},
+                { "rank": 1,
+                  "name": "b",
+                  "addr": "127.0.0.1:6790/0",
+              "public_addr": "127.0.0.1:6790/0"},
+                { "rank": 2,
+                  "name": "c",
+                  "addr": "127.0.0.1:6791/0",
+              "public_addr": "127.0.0.1:6791/0"}
+               ]
+      }
+    }
+
+Checking MDS Status
+===================
+
+Metadata servers provide metadata services for CephFS. Metadata servers have
+two sets of states: ``up | down`` and ``active | inactive``. To check if your
+metadata servers are ``up`` and ``active``, run the following command:
+
+.. prompt:: bash $
+
+   ceph mds stat
+
+To display details of the metadata servers, run the following command:
+
+.. prompt:: bash $
+
+   ceph fs dump
+
+
+Checking Placement Group States
+===============================
+
+Placement groups (PGs) map objects to OSDs. PGs are monitored in order to
+ensure that they are ``active`` and ``clean``.  See `Monitoring OSDs and
+Placement Groups`_.
+
+.. _Monitoring OSDs and Placement Groups: ../monitoring-osd-pg
+
+.. _rados-monitoring-using-admin-socket:
+
+Using the Admin Socket
+======================
+
+The Ceph admin socket allows you to query a daemon via a socket interface.  By
+default, Ceph sockets reside under ``/var/run/ceph``. To access a daemon via
+the admin socket, log in to the host that is running the daemon and run one of
+the two following commands:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name}
+   ceph daemon {path-to-socket-file}
+
+For example, the following commands are equivalent to each other:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 foo
+   ceph daemon /var/run/ceph/ceph-osd.0.asok foo
+
+To view the available admin-socket commands, run the following command:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name} help
+
+Admin-socket commands enable you to view and set your configuration at runtime.
+For more on viewing your configuration, see `Viewing a Configuration at
+Runtime`_. There are two methods of setting configuration value at runtime: (1)
+using the admin socket, which bypasses the monitor and requires a direct login
+to the host in question, and (2) using the ``ceph tell {daemon-type}.{id}
+config set`` command, which relies on the monitor and does not require a direct
+login.
+
+.. _Viewing a Configuration at Runtime: ../../configuration/ceph-conf#viewing-a-configuration-at-runtime
+.. _Storage Capacity: ../../configuration/mon-config-ref#storage-capacity
diff --git a/doc/rados/operations/operating.rst b/doc/rados/operations/operating.rst
new file mode 100644
index 000000000..f4a2fd988
--- /dev/null
+++ b/doc/rados/operations/operating.rst
@@ -0,0 +1,174 @@
+=====================
+ Operating a Cluster
+=====================
+
+.. index:: systemd; operating a cluster
+
+
+Running Ceph with systemd
+=========================
+
+In all distributions that support systemd (CentOS 7, Fedora, Debian
+Jessie 8 and later, and SUSE), systemd files (and NOT legacy SysVinit scripts) 
+are used to manage Ceph daemons. Ceph daemons therefore behave like any other daemons 
+that can be controlled by the ``systemctl`` command, as in the following examples:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph.target       # start all daemons
+   sudo systemctl status ceph-osd@12      # check status of osd.12
+
+To list all of the Ceph systemd units on a node, run the following command:
+
+.. prompt:: bash $
+
+   sudo systemctl status ceph\*.service ceph\*.target
+
+
+Starting all daemons
+--------------------
+
+To start all of the daemons on a Ceph node (regardless of their type), run the
+following command:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph.target
+
+
+Stopping all daemons
+--------------------
+
+To stop all of the daemons on a Ceph node (regardless of their type), run the
+following command:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph\*.service ceph\*.target
+
+
+Starting all daemons by type
+----------------------------
+
+To start all of the daemons of a particular type on a Ceph node, run one of the
+following commands:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd.target
+   sudo systemctl start ceph-mon.target
+   sudo systemctl start ceph-mds.target
+
+
+Stopping all daemons by type
+----------------------------
+
+To stop all of the daemons of a particular type on a Ceph node, run one of the
+following commands:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-osd\*.service ceph-osd.target
+   sudo systemctl stop ceph-mon\*.service ceph-mon.target
+   sudo systemctl stop ceph-mds\*.service ceph-mds.target
+
+
+Starting a daemon
+-----------------
+
+To start a specific daemon instance on a Ceph node, run one of the
+following commands:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd@{id}
+   sudo systemctl start ceph-mon@{hostname}
+   sudo systemctl start ceph-mds@{hostname}
+
+For example:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd@1
+   sudo systemctl start ceph-mon@ceph-server
+   sudo systemctl start ceph-mds@ceph-server
+
+
+Stopping a daemon
+-----------------
+
+To stop a specific daemon instance on a Ceph node, run one of the
+following commands:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-osd@{id}
+   sudo systemctl stop ceph-mon@{hostname}
+   sudo systemctl stop ceph-mds@{hostname}
+
+For example:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-osd@1
+   sudo systemctl stop ceph-mon@ceph-server
+   sudo systemctl stop ceph-mds@ceph-server
+
+
+.. index:: sysvinit; operating a cluster
+
+Running Ceph with SysVinit
+==========================
+
+Each time you start, restart, or stop Ceph daemons, you must specify at least one option and one command.
+Likewise, each time you start, restart, or stop your entire cluster, you must specify at least one option and one command.
+In both cases, you can also specify a daemon type or a daemon instance. ::
+
+    {commandline} [options] [commands] [daemons]
+
+The ``ceph`` options include:
+
++-----------------+----------+-------------------------------------------------+
+| Option          | Shortcut | Description                                     |
++=================+==========+=================================================+
+| ``--verbose``   |  ``-v``  | Use verbose logging.                            |
++-----------------+----------+-------------------------------------------------+
+| ``--valgrind``  | ``N/A``  | (Dev and QA only) Use `Valgrind`_ debugging.    |
++-----------------+----------+-------------------------------------------------+
+| ``--allhosts``  |  ``-a``  | Execute on all nodes listed in ``ceph.conf``.   |
+|                 |          | Otherwise, it only executes on ``localhost``.   |
++-----------------+----------+-------------------------------------------------+
+| ``--restart``   | ``N/A``  | Automatically restart daemon if it core dumps.  |
++-----------------+----------+-------------------------------------------------+
+| ``--norestart`` | ``N/A``  | Do not restart a daemon if it core dumps.       |
++-----------------+----------+-------------------------------------------------+
+| ``--conf``      |  ``-c``  | Use an alternate configuration file.            |
++-----------------+----------+-------------------------------------------------+
+
+The ``ceph`` commands include:
+
++------------------+------------------------------------------------------------+
+| Command          | Description                                                |
++==================+============================================================+
+|    ``start``     | Start the daemon(s).                                       |
++------------------+------------------------------------------------------------+
+|    ``stop``      | Stop the daemon(s).                                        |
++------------------+------------------------------------------------------------+
+|  ``forcestop``   | Force the daemon(s) to stop. Same as ``kill -9``.          |
++------------------+------------------------------------------------------------+
+|   ``killall``    | Kill all daemons of a particular type.                     |
++------------------+------------------------------------------------------------+
+|  ``cleanlogs``   | Cleans out the log directory.                              |
++------------------+------------------------------------------------------------+
+| ``cleanalllogs`` | Cleans out **everything** in the log directory.            |
++------------------+------------------------------------------------------------+
+
+The ``[daemons]`` option allows the ``ceph`` service to target specific daemon types
+in order to perform subsystem operations. Daemon types include:
+
+- ``mon``
+- ``osd``
+- ``mds``
+
+.. _Valgrind: http://www.valgrind.org/
+.. _initctl: http://manpages.ubuntu.com/manpages/raring/en/man8/initctl.8.html
diff --git a/doc/rados/operations/pg-concepts.rst b/doc/rados/operations/pg-concepts.rst
new file mode 100644
index 000000000..83062b53a
--- /dev/null
+++ b/doc/rados/operations/pg-concepts.rst
@@ -0,0 +1,104 @@
+.. _rados_operations_pg_concepts:
+
+==========================
+ Placement Group Concepts
+==========================
+
+When you execute commands like ``ceph -w``, ``ceph osd dump``, and other 
+commands related to placement groups, Ceph may return values using some
+of the following terms: 
+
+*Peering*
+   The process of bringing all of the OSDs that store
+   a Placement Group (PG) into agreement about the state
+   of all of the objects (and their metadata) in that PG.
+   Note that agreeing on the state does not mean that
+   they all have the latest contents.
+
+*Acting Set*
+   The ordered list of OSDs who are (or were as of some epoch)
+   responsible for a particular placement group.
+
+*Up Set*
+   The ordered list of OSDs responsible for a particular placement
+   group for a particular epoch according to CRUSH. Normally this
+   is the same as the *Acting Set*, except when the *Acting Set* has 
+   been explicitly overridden via ``pg_temp`` in the OSD Map.
+
+*Current Interval* or *Past Interval*
+   A sequence of OSD map epochs during which the *Acting Set* and *Up
+   Set* for particular placement group do not change.
+
+*Primary*
+   The member (and by convention first) of the *Acting Set*,
+   that is responsible for coordination peering, and is
+   the only OSD that will accept client-initiated
+   writes to objects in a placement group.
+
+*Replica*
+   A non-primary OSD in the *Acting Set* for a placement group
+   (and who has been recognized as such and *activated* by the primary).
+
+*Stray*
+   An OSD that is not a member of the current *Acting Set*, but
+   has not yet been told that it can delete its copies of a
+   particular placement group.
+
+*Recovery*
+   Ensuring that copies of all of the objects in a placement group
+   are on all of the OSDs in the *Acting Set*.  Once *Peering* has 
+   been performed, the *Primary* can start accepting write operations, 
+   and *Recovery* can proceed in the background.
+
+*PG Info* 
+   Basic metadata about the placement group's creation epoch, the version
+   for the most recent write to the placement group, *last epoch started*, 
+   *last epoch clean*, and the beginning of the *current interval*.  Any
+   inter-OSD communication about placement groups includes the *PG Info*, 
+   such that any OSD that knows a placement group exists (or once existed) 
+   also has a lower bound on *last epoch clean* or *last epoch started*.
+
+*PG Log*
+   A list of recent updates made to objects in a placement group.
+   Note that these logs can be truncated after all OSDs
+   in the *Acting Set* have acknowledged up to a certain
+   point.
+
+*Missing Set*
+   Each OSD notes update log entries and if they imply updates to
+   the contents of an object, adds that object to a list of needed
+   updates.  This list is called the *Missing Set* for that ``<OSD,PG>``.
+
+*Authoritative History*
+   A complete, and fully ordered set of operations that, if
+   performed, would bring an OSD's copy of a placement group
+   up to date.
+
+*Epoch*
+   A (monotonically increasing) OSD map version number
+
+*Last Epoch Start*
+   The last epoch at which all nodes in the *Acting Set*
+   for a particular placement group agreed on an
+   *Authoritative History*.  At this point, *Peering* is
+   deemed to have been successful.
+
+*up_thru*
+   Before a *Primary* can successfully complete the *Peering* process,
+   it must inform a monitor that is alive through the current
+   OSD map *Epoch* by having the monitor set its *up_thru* in the osd
+   map.  This helps *Peering* ignore previous *Acting Sets* for which
+   *Peering* never completed after certain sequences of failures, such as
+   the second interval below:
+
+   - *acting set* = [A,B]
+   - *acting set* = [A]
+   - *acting set* = [] very shortly after (e.g., simultaneous failure, but staggered detection)
+   - *acting set* = [B] (B restarts, A does not)
+
+*Last Epoch Clean*
+   The last *Epoch* at which all nodes in the *Acting set*
+   for a particular placement group were completely
+   up to date (both placement group logs and object contents).
+   At this point, *recovery* is deemed to have been
+   completed.
diff --git a/doc/rados/operations/pg-repair.rst b/doc/rados/operations/pg-repair.rst
new file mode 100644
index 000000000..609318fca
--- /dev/null
+++ b/doc/rados/operations/pg-repair.rst
@@ -0,0 +1,118 @@
+============================
+Repairing PG Inconsistencies
+============================
+Sometimes a Placement Group (PG) might become ``inconsistent``. To return the PG
+to an ``active+clean`` state, you must first determine which of the PGs has become
+inconsistent and then run the ``pg repair`` command on it. This page contains
+commands for diagnosing PGs and the command for repairing PGs that have become
+inconsistent.
+
+.. highlight:: console
+
+Commands for Diagnosing PG Problems
+===================================
+The commands in this section provide various ways of diagnosing broken PGs.
+
+To see a high-level (low-detail) overview of Ceph cluster health, run the
+following command:
+
+.. prompt:: bash #
+
+   ceph health detail
+
+To see more detail on the status of the PGs, run the following command:
+
+.. prompt:: bash #
+
+   ceph pg dump --format=json-pretty
+
+To see a list of inconsistent PGs, run the following command:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-pg {pool}
+
+To see a list of inconsistent RADOS objects, run the following command:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-obj {pgid}
+
+To see a list of inconsistent snapsets in a specific PG, run the following
+command:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-snapset {pgid}
+
+
+Commands for Repairing PGs
+==========================
+The form of the command to repair a broken PG is as follows:
+
+.. prompt:: bash #
+
+   ceph pg repair {pgid}
+
+Here ``{pgid}`` represents the id of the affected PG.
+
+For example:
+
+.. prompt:: bash #
+
+   ceph pg repair 1.4
+
+.. note:: PG IDs have the form ``N.xxxxx``, where ``N`` is the number of the
+   pool that contains the PG. The command ``ceph osd listpools`` and the
+   command ``ceph osd dump | grep pool`` return a list of pool numbers.
+
+More Information on PG Repair
+=============================
+Ceph stores and updates the checksums of objects stored in the cluster. When a
+scrub is performed on a PG, the OSD attempts to choose an authoritative copy
+from among its replicas. Only one of the possible cases is consistent. After
+performing a deep scrub, Ceph calculates the checksum of an object that is read
+from disk and compares it to the checksum that was previously recorded. If the
+current checksum and the previously recorded checksum do not match, that
+mismatch is considered to be an inconsistency. In the case of replicated pools,
+any mismatch between the checksum of any replica of an object and the checksum
+of the authoritative copy means that there is an inconsistency. The discovery
+of these inconsistencies cause a PG's state to be set to ``inconsistent``.
+
+The ``pg repair`` command attempts to fix inconsistencies of various kinds. If
+``pg repair`` finds an inconsistent PG, it attempts to overwrite the digest of
+the inconsistent copy with the digest of the authoritative copy. If ``pg
+repair`` finds an inconsistent replicated pool, it marks the inconsistent copy
+as missing. In the case of replicated pools, recovery is beyond the scope of
+``pg repair``.
+
+In the case of erasure-coded and BlueStore pools, Ceph will automatically
+perform repairs if ``osd_scrub_auto_repair`` (default ``false``) is set to
+``true`` and if no more than ``osd_scrub_auto_repair_num_errors`` (default
+``5``) errors are found.
+
+The ``pg repair`` command will not solve every problem. Ceph does not
+automatically repair PGs when they are found to contain inconsistencies.
+
+The checksum of a RADOS object or an omap is not always available. Checksums
+are calculated incrementally. If a replicated object is updated
+non-sequentially, the write operation involved in the update changes the object
+and invalidates its checksum. The whole object is not read while the checksum
+is recalculated. The ``pg repair`` command is able to make repairs even when
+checksums are not available to it, as in the case of Filestore. Users working
+with replicated Filestore pools might prefer manual repair to ``ceph pg
+repair``.
+
+This material is relevant for Filestore, but not for BlueStore, which has its
+own internal checksums. The matched-record checksum and the calculated checksum
+cannot prove that any specific copy is in fact authoritative. If there is no
+checksum available, ``pg repair`` favors the data on the primary, but this
+might not be the uncorrupted replica. Because of this uncertainty, human
+intervention is necessary when an inconsistency is discovered. This
+intervention sometimes involves use of ``ceph-objectstore-tool``.
+
+External Links
+==============
+https://ceph.io/geen-categorie/ceph-manually-repair-object/ - This page
+contains a walkthrough of the repair of a PG. It is recommended reading if you
+want to repair a PG but have never done so.
diff --git a/doc/rados/operations/pg-states.rst b/doc/rados/operations/pg-states.rst
new file mode 100644
index 000000000..495229d92
--- /dev/null
+++ b/doc/rados/operations/pg-states.rst
@@ -0,0 +1,118 @@
+========================
+ Placement Group States
+========================
+
+When checking a cluster's status (e.g., running ``ceph -w`` or ``ceph -s``), 
+Ceph will report on the status of the placement groups. A placement group has 
+one or more states. The optimum state for placement groups in the placement group
+map is ``active + clean``. 
+
+*creating*
+  Ceph is still creating the placement group.
+
+*activating*
+  The placement group is peered but not yet active.
+
+*active*
+  Ceph will process requests to the placement group.
+
+*clean*
+  Ceph replicated all objects in the placement group the correct number of times.
+
+*down*
+  A replica with necessary data is down, so the placement group is offline.
+
+*laggy*
+  A replica is not acknowledging new leases from the primary in a timely fashion; IO is temporarily paused.
+
+*wait*
+  The set of OSDs for this PG has just changed and IO is temporarily paused until the previous interval's leases expire.
+
+*scrubbing*
+  Ceph is checking the placement group metadata for inconsistencies.
+
+*deep*
+  Ceph is checking the placement group data against stored checksums.
+
+*degraded*
+  Ceph has not replicated some objects in the placement group the correct number of times yet.
+
+*inconsistent*
+  Ceph detects inconsistencies in the one or more replicas of an object in the placement group
+  (e.g. objects are the wrong size, objects are missing from one replica *after* recovery finished, etc.).
+
+*peering*
+  The placement group is undergoing the peering process
+
+*repair*
+  Ceph is checking the placement group and repairing any inconsistencies it finds (if possible).
+
+*recovering*
+  Ceph is migrating/synchronizing objects and their replicas.
+
+*forced_recovery*
+  High recovery priority of that PG is enforced by user.
+
+*recovery_wait*
+  The placement group is waiting in line to start recover.
+
+*recovery_toofull*
+  A recovery operation is waiting because the destination OSD is over its
+  full ratio.
+
+*recovery_unfound*
+  Recovery stopped due to unfound objects.
+
+*backfilling*
+  Ceph is scanning and synchronizing the entire contents of a placement group
+  instead of inferring what contents need to be synchronized from the logs of
+  recent operations. Backfill is a special case of recovery.
+
+*forced_backfill*
+  High backfill priority of that PG is enforced by user.
+
+*backfill_wait*
+  The placement group is waiting in line to start backfill.
+
+*backfill_toofull*
+  A backfill operation is waiting because the destination OSD is over
+  the backfillfull ratio.
+
+*backfill_unfound*
+  Backfill stopped due to unfound objects.
+
+*incomplete*
+  Ceph detects that a placement group is missing information about
+  writes that may have occurred, or does not have any healthy
+  copies. If you see this state, try to start any failed OSDs that may
+  contain the needed information. In the case of an erasure coded pool
+  temporarily reducing min_size may allow recovery.
+
+*stale*
+  The placement group is in an unknown state - the monitors have not received
+  an update for it since the placement group mapping changed.
+
+*remapped*
+  The placement group is temporarily mapped to a different set of OSDs from what
+  CRUSH specified.
+
+*undersized*
+  The placement group has fewer copies than the configured pool replication level.
+
+*peered*
+  The placement group has peered, but cannot serve client IO due to not having
+  enough copies to reach the pool's configured min_size parameter.  Recovery
+  may occur in this state, so the pg may heal up to min_size eventually.
+
+*snaptrim*
+  Trimming snaps.
+
+*snaptrim_wait*
+  Queued to trim snaps.
+
+*snaptrim_error*
+  Error stopped trimming snaps.
+
+*unknown*
+  The ceph-mgr hasn't yet received any information about the PG's state from an
+  OSD since mgr started up.
diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
new file mode 100644
index 000000000..dda4a0177
--- /dev/null
+++ b/doc/rados/operations/placement-groups.rst
@@ -0,0 +1,897 @@
+.. _placement groups:
+
+==================
+ Placement Groups
+==================
+
+.. _pg-autoscaler:
+
+Autoscaling placement groups
+============================
+
+Placement groups (PGs) are an internal implementation detail of how Ceph
+distributes data. Autoscaling provides a way to manage PGs, and especially to
+manage the number of PGs present in different pools.  When *pg-autoscaling* is
+enabled, the cluster is allowed to make recommendations or automatic
+adjustments with respect to the number of PGs for each pool (``pgp_num``) in
+accordance with expected cluster utilization and expected pool utilization.
+
+Each pool has a ``pg_autoscale_mode`` property that can be set to ``off``,
+``on``, or ``warn``:
+
+* ``off``: Disable autoscaling for this pool. It is up to the administrator to
+  choose an appropriate ``pgp_num`` for each pool. For more information, see
+  :ref:`choosing-number-of-placement-groups`.
+* ``on``: Enable automated adjustments of the PG count for the given pool.
+* ``warn``: Raise health checks when the PG count is in need of adjustment.
+
+To set the autoscaling mode for an existing pool, run a command of the
+following form:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> pg_autoscale_mode <mode>
+
+For example, to enable autoscaling on pool ``foo``, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool set foo pg_autoscale_mode on
+
+There is also a ``pg_autoscale_mode`` setting for any pools that are created
+after the initial setup of the cluster. To change this setting, run a command
+of the following form:
+
+.. prompt:: bash #
+
+   ceph config set global osd_pool_default_pg_autoscale_mode <mode>
+
+You can disable or enable the autoscaler for all pools with the ``noautoscale``
+flag. By default, this flag is set to ``off``, but you can set it to ``on`` by
+running the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool set noautoscale
+
+To set the ``noautoscale`` flag to ``off``, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool unset noautoscale
+
+To get the value of the flag, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool get noautoscale
+
+Viewing PG scaling recommendations
+----------------------------------
+
+To view each pool, its relative utilization, and any recommended changes to the
+PG count, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool autoscale-status
+
+The output will resemble the following::
+
+   POOL    SIZE  TARGET SIZE  RATE  RAW CAPACITY   RATIO  TARGET RATIO  EFFECTIVE RATIO BIAS PG_NUM  NEW PG_NUM  AUTOSCALE BULK
+   a     12900M                3.0        82431M  0.4695                                          8         128  warn      True
+   c         0                 3.0        82431M  0.0000        0.2000           0.9884  1.0      1          64  warn      True
+   b         0        953.6M   3.0        82431M  0.0347                                          8              warn      False
+
+- **POOL** is the name of the pool. 
+
+- **SIZE** is the amount of data stored in the pool. 
+  
+- **TARGET SIZE** (if present) is the amount of data that is expected to be
+  stored in the pool, as specified by the administrator. The system uses the
+  greater of the two values for its calculation.
+
+- **RATE** is the multiplier for the pool that determines how much raw storage
+  capacity is consumed. For example, a three-replica pool will have a ratio of
+  3.0, and a ``k=4 m=2`` erasure-coded pool will have a ratio of 1.5.
+
+- **RAW CAPACITY** is the total amount of raw storage capacity on the specific
+  OSDs that are responsible for storing the data of the pool (and perhaps the
+  data of other pools). 
+
+- **RATIO** is the ratio of (1) the storage consumed by the pool to (2) the
+  total raw storage capacity. In order words, RATIO is defined as 
+  (SIZE * RATE) / RAW CAPACITY.
+
+- **TARGET RATIO** (if present) is the ratio of the expected storage of this
+  pool (that is, the amount of storage that this pool is expected to consume,
+  as specified by the administrator) to the expected storage of all other pools
+  that have target ratios set.  If both ``target_size_bytes`` and
+  ``target_size_ratio`` are specified, then ``target_size_ratio`` takes
+  precedence.
+
+- **EFFECTIVE RATIO** is the result of making two adjustments to the target
+  ratio:
+
+  #. Subtracting any capacity expected to be used by pools that have target
+     size set.
+
+  #. Normalizing the target ratios among pools that have target ratio set so
+     that collectively they target cluster capacity. For example, four pools
+     with target_ratio 1.0 would have an effective ratio of 0.25.
+
+  The system's calculations use whichever of these two ratios (that is, the 
+  target ratio and the effective ratio) is greater.
+
+- **BIAS** is used as a multiplier to manually adjust a pool's PG in accordance
+  with prior information about how many PGs a specific pool is expected to
+  have.
+
+- **PG_NUM** is either the current number of PGs associated with the pool or,
+  if a ``pg_num`` change is in progress, the current number of PGs that the
+  pool is working towards. 
+
+- **NEW PG_NUM** (if present) is the value that the system is recommending the
+  ``pg_num`` of the pool to be changed to. It is always a power of 2, and it is
+  present only if the recommended value varies from the current value by more
+  than the default factor of ``3``. To adjust this factor (in the following
+  example, it is changed to ``2``), run the following command:
+
+  .. prompt:: bash #
+
+     ceph osd pool set threshold 2.0
+
+- **AUTOSCALE** is the pool's ``pg_autoscale_mode`` and is set to ``on``,
+  ``off``, or ``warn``.
+
+- **BULK** determines whether the pool is ``bulk``. It has a value of ``True``
+  or ``False``. A ``bulk`` pool is expected to be large and should initially
+  have a large number of PGs so that performance does not suffer]. On the other
+  hand, a pool that is not ``bulk`` is expected to be small (for example, a
+  ``.mgr`` pool or a meta pool).
+
+.. note::
+
+   If the ``ceph osd pool autoscale-status`` command returns no output at all,
+   there is probably at least one pool that spans multiple CRUSH roots.  This
+   'spanning pool' issue can happen in scenarios like the following:
+   when a new deployment auto-creates the ``.mgr`` pool on the ``default``
+   CRUSH root, subsequent pools are created with rules that constrain them to a
+   specific shadow CRUSH tree. For example, if you create an RBD metadata pool
+   that is constrained to ``deviceclass = ssd`` and an RBD data pool that is
+   constrained to ``deviceclass = hdd``, you will encounter this issue. To
+   remedy this issue, constrain the spanning pool to only one device class. In
+   the above scenario, there is likely to be a ``replicated-ssd`` CRUSH rule in
+   effect, and the ``.mgr`` pool can be constrained to ``ssd`` devices by
+   running the following commands:
+
+   .. prompt:: bash #
+
+      ceph osd pool set .mgr crush_rule replicated-ssd
+      ceph osd pool set pool 1 crush_rule to replicated-ssd
+
+   This intervention will result in a small amount of backfill, but
+   typically this traffic completes quickly.
+
+
+Automated scaling
+-----------------
+
+In the simplest approach to automated scaling, the cluster is allowed to
+automatically scale ``pgp_num`` in accordance with usage. Ceph considers the
+total available storage and the target number of PGs for the whole system,
+considers how much data is stored in each pool, and apportions PGs accordingly.
+The system is conservative with its approach, making changes to a pool only
+when the current number of PGs (``pg_num``) varies by more than a factor of 3
+from the recommended number.
+
+The target number of PGs per OSD is determined by the ``mon_target_pg_per_osd``
+parameter (default: 100), which can be adjusted by running the following
+command:
+
+.. prompt:: bash #
+
+   ceph config set global mon_target_pg_per_osd 100
+
+The autoscaler analyzes pools and adjusts on a per-subtree basis.  Because each
+pool might map to a different CRUSH rule, and each rule might distribute data
+across different devices, Ceph will consider the utilization of each subtree of
+the hierarchy independently. For example, a pool that maps to OSDs of class
+``ssd`` and a pool that maps to OSDs of class ``hdd`` will each have optimal PG
+counts that are determined by how many of these two different device types
+there are.
+
+If a pool uses OSDs under two or more CRUSH roots (for example, shadow trees
+with both ``ssd`` and ``hdd`` devices), the autoscaler issues a warning to the
+user in the manager log. The warning states the name of the pool and the set of
+roots that overlap each other. The autoscaler does not scale any pools with
+overlapping roots because this condition can cause problems with the scaling
+process. We recommend constraining each pool so that it belongs to only one
+root (that is, one OSD class) to silence the warning and ensure a successful
+scaling process.
+
+.. _managing_bulk_flagged_pools:
+
+Managing pools that are flagged with ``bulk``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a pool is flagged ``bulk``, then the autoscaler starts the pool with a full
+complement of PGs and then scales down the number of PGs only if the usage
+ratio across the pool is uneven.  However, if a pool is not flagged ``bulk``,
+then the autoscaler starts the pool with minimal PGs and creates additional PGs
+only if there is more usage in the pool.
+
+To create a pool that will be flagged ``bulk``, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool create <pool-name> --bulk
+
+To set or unset the ``bulk`` flag of an existing pool, run the following
+command:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> bulk <true/false/1/0>
+
+To get the ``bulk`` flag of an existing pool, run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool get <pool-name> bulk
+
+.. _specifying_pool_target_size:
+
+Specifying expected pool size
+-----------------------------
+
+When a cluster or pool is first created, it consumes only a small fraction of
+the total cluster capacity and appears to the system as if it should need only
+a small number of PGs. However, in some cases, cluster administrators know
+which pools are likely to consume most of the system capacity in the long run.
+When Ceph is provided with this information, a more appropriate number of PGs
+can be used from the beginning, obviating subsequent changes in ``pg_num`` and
+the associated overhead cost of relocating data.
+
+The *target size* of a pool can be specified in two ways: either in relation to
+the absolute size (in bytes) of the pool, or as a weight relative to all other
+pools that have ``target_size_ratio`` set.
+
+For example, to tell the system that ``mypool`` is expected to consume 100 TB,
+run the following command:
+
+.. prompt:: bash #
+
+   ceph osd pool set mypool target_size_bytes 100T
+
+Alternatively, to tell the system that ``mypool`` is expected to consume a
+ratio of 1.0 relative to other pools that have ``target_size_ratio`` set,
+adjust the ``target_size_ratio`` setting of ``my pool`` by running the
+following command:
+
+.. prompt:: bash # 
+
+   ceph osd pool set mypool target_size_ratio 1.0
+
+If `mypool` is the only pool in the cluster, then it is expected to use 100% of
+the total cluster capacity. However, if the cluster contains a second pool that
+has ``target_size_ratio`` set to 1.0, then both pools are expected to use 50%
+of the total cluster capacity.
+
+The ``ceph osd pool create`` command has two command-line options that can be
+used to set the target size of a pool at creation time: ``--target-size-bytes
+<bytes>`` and ``--target-size-ratio <ratio>``.
+
+Note that if the target-size values that have been specified are impossible
+(for example, a capacity larger than the total cluster), then a health check
+(``POOL_TARGET_SIZE_BYTES_OVERCOMMITTED``) will be raised.
+
+If both ``target_size_ratio`` and ``target_size_bytes`` are specified for a
+pool, then the latter will be ignored, the former will be used in system
+calculations, and a health check (``POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO``)
+will be raised.
+
+Specifying bounds on a pool's PGs
+---------------------------------
+
+It is possible to specify both the minimum number and the maximum number of PGs
+for a pool. 
+
+Setting a Minimum Number of PGs and a Maximum Number of PGs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a minimum is set, then Ceph will not itself reduce (nor recommend that you
+reduce) the number of PGs to a value below the configured value. Setting a
+minimum serves to establish a lower bound on the amount of parallelism enjoyed
+by a client during I/O, even if a pool is mostly empty. 
+
+If a maximum is set, then Ceph will not itself increase (or recommend that you
+increase) the number of PGs to a value above the configured value.
+
+To set the minimum number of PGs for a pool, run a command of the following
+form:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> pg_num_min <num>
+
+To set the maximum number of PGs for a pool, run a command of the following
+form:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> pg_num_max <num>
+
+In addition, the ``ceph osd pool create`` command has two command-line options
+that can be used to specify the minimum or maximum PG count of a pool at
+creation time: ``--pg-num-min <num>`` and ``--pg-num-max <num>``.
+
+.. _preselection:
+
+Preselecting pg_num
+===================
+
+When creating a pool with the following command, you have the option to
+preselect the value of the ``pg_num`` parameter:
+
+.. prompt:: bash #
+
+   ceph osd pool create {pool-name} [pg_num]
+
+If you opt not to specify ``pg_num`` in this command, the cluster uses the PG
+autoscaler to automatically configure the parameter in accordance with the
+amount of data that is stored in the pool (see :ref:`pg-autoscaler` above).
+
+However, your decision of whether or not to specify ``pg_num`` at creation time
+has no effect on whether the parameter will be automatically tuned by the
+cluster afterwards. As seen above, autoscaling of PGs is enabled or disabled by
+running a command of the following form:
+    
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} pg_autoscale_mode (on|off|warn)
+
+Without the balancer, the suggested target is approximately 100 PG replicas on
+each OSD. With the balancer, an initial target of 50 PG replicas on each OSD is
+reasonable.
+
+The autoscaler attempts to satisfy the following conditions:
+
+- the number of PGs per OSD should be proportional to the amount of data in the
+  pool
+- there should be 50-100 PGs per pool, taking into account the replication
+  overhead or erasure-coding fan-out of each PG's replicas across OSDs
+
+Use of Placement Groups
+=======================
+
+A placement group aggregates objects within a pool. The tracking of RADOS
+object placement and object metadata on a per-object basis is computationally
+expensive. It would be infeasible for a system with millions of RADOS
+objects to efficiently track placement on a per-object basis.
+
+.. ditaa::
+           /-----\  /-----\  /-----\  /-----\  /-----\
+           | obj |  | obj |  | obj |  | obj |  | obj |
+           \-----/  \-----/  \-----/  \-----/  \-----/
+              |        |        |        |        |
+              +--------+--------+        +---+----+
+              |                              |
+              v                              v
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+               |                              |
+               +------------------------------+
+                             |
+                             v
+                  +-----------------------+
+                  |        Pool           |
+                  |                       |
+                  +-----------------------+
+
+The Ceph client calculates which PG a RADOS object should be in. As part of
+this calculation, the client hashes the object ID and performs an operation
+involving both the number of PGs in the specified pool and the pool ID. For
+details, see `Mapping PGs to OSDs`_.
+
+The contents of a RADOS object belonging to a PG are stored in a set of OSDs.
+For example, in a replicated pool of size two, each PG will store objects on
+two OSDs, as shown below:
+
+.. ditaa::
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+        |             |               |             |
+        v             v               v             v
+   /----------\  /----------\    /----------\  /----------\
+   |          |  |          |    |          |  |          |
+   |  OSD #1  |  |  OSD #2  |    |  OSD #2  |  |  OSD #3  |
+   |          |  |          |    |          |  |          |
+   \----------/  \----------/    \----------/  \----------/
+
+
+If OSD #2 fails, another OSD will be assigned to Placement Group #1 and then
+filled with copies of all objects in OSD #1. If the pool size is changed from
+two to three, an additional OSD will be assigned to the PG and will receive
+copies of all objects in the PG.
+
+An OSD assigned to a PG is not owned exclusively by that PG; rather, the OSD is
+shared with other PGs either from the same pool or from other pools. In our
+example, OSD #2 is shared by Placement Group #1 and Placement Group #2. If OSD
+#2 fails, then Placement Group #2 must restore copies of objects (by making use
+of OSD #3).
+
+When the number of PGs increases, several consequences ensue. The new PGs are
+assigned OSDs. The result of the CRUSH function changes, which means that some
+objects from the already-existing PGs are copied to the new PGs and removed
+from the old ones.
+
+Factors Relevant To Specifying pg_num
+=====================================
+
+On the one hand, the criteria of data durability and even distribution across
+OSDs weigh in favor of a high number of PGs. On the other hand, the criteria of
+saving CPU resources and minimizing memory usage weigh in favor of a low number
+of PGs.
+
+.. _data durability:
+
+Data durability
+---------------
+
+When an OSD fails, the risk of data loss is increased until replication of the
+data it hosted is restored to the configured level. To illustrate this point,
+let's imagine a scenario that results in permanent data loss in a single PG:
+
+#. The OSD fails and all copies of the object that it contains are lost.  For
+   each object within the PG, the number of its replicas suddenly drops from
+   three to two.
+
+#. Ceph starts recovery for this PG by choosing a new OSD on which to re-create
+   the third copy of each object.
+
+#. Another OSD within the same PG fails before the new OSD is fully populated
+   with the third copy. Some objects will then only have one surviving copy.
+
+#. Ceph selects yet another OSD and continues copying objects in order to
+   restore the desired number of copies.
+
+#. A third OSD within the same PG fails before recovery is complete. If this
+   OSD happened to contain the only remaining copy of an object, the object is
+   permanently lost.
+
+In a cluster containing 10 OSDs with 512 PGs in a three-replica pool, CRUSH
+will give each PG three OSDs.  Ultimately, each OSD hosts :math:`\frac{(512 *
+3)}{10} = ~150` PGs. So when the first OSD fails in the above scenario,
+recovery will begin for all 150 PGs at the same time.
+
+The 150 PGs that are being recovered are likely to be homogeneously distributed
+across the 9 remaining OSDs. Each remaining OSD is therefore likely to send
+copies of objects to all other OSDs and also likely to receive some new objects
+to be stored because it has become part of a new PG.
+
+The amount of time it takes for this recovery to complete depends on the
+architecture of the Ceph cluster. Compare two setups: (1) Each OSD is hosted by
+a 1 TB SSD on a single machine, all of the OSDs are connected to a 10 Gb/s
+switch, and the recovery of a single OSD completes within a certain number of
+minutes. (2) There are two OSDs per machine using HDDs with no SSD WAL+DB and
+a 1 Gb/s switch. In the second setup, recovery will be at least one order of
+magnitude slower.
+
+In such a cluster, the number of PGs has almost no effect on data durability.
+Whether there are 128 PGs per OSD or 8192 PGs per OSD, the recovery will be no
+slower or faster.
+
+However, an increase in the number of OSDs can increase the speed of recovery.
+Suppose our Ceph cluster is expanded from 10 OSDs to 20 OSDs.  Each OSD now
+participates in only ~75 PGs rather than ~150 PGs. All 19 remaining OSDs will
+still be required to replicate the same number of objects in order to recover.
+But instead of there being only 10 OSDs that have to copy ~100 GB each, there
+are now 20 OSDs that have to copy only 50 GB each. If the network had
+previously been a bottleneck, recovery now happens twice as fast.
+
+Similarly, suppose that our cluster grows to 40 OSDs. Each OSD will host only
+~38 PGs. And if an OSD dies, recovery will take place faster than before unless
+it is blocked by another bottleneck. Now, however, suppose that our cluster
+grows to 200 OSDs. Each OSD will host only ~7 PGs. And if an OSD dies, recovery
+will happen across at most :math:`\approx 21 = (7 \times 3)` OSDs
+associated with these PGs. This means that recovery will take longer than when
+there were only 40 OSDs. For this reason, the number of PGs should be
+increased.
+
+No matter how brief the recovery time is, there is always a chance that an
+additional OSD will fail while recovery is in progress.  Consider the cluster
+with 10 OSDs described above: if any of the OSDs fail, then :math:`\approx 17`
+(approximately 150 divided by 9) PGs will have only one remaining copy. And if
+any of the 8 remaining OSDs fail, then 2 (approximately 17 divided by 8) PGs
+are likely to lose their remaining objects. This is one reason why setting
+``size=2`` is risky.
+
+When the number of OSDs in the cluster increases to 20, the number of PGs that
+would be damaged by the loss of three OSDs significantly decreases. The loss of
+a second OSD degrades only approximately :math:`4` or (:math:`\frac{75}{19}`)
+PGs rather than :math:`\approx 17` PGs, and the loss of a third OSD results in
+data loss only if it is one of the 4 OSDs that contains the remaining copy.
+This means -- assuming that the probability of losing one OSD during recovery
+is 0.0001% -- that the probability of data loss when three OSDs are lost is
+:math:`\approx 17 \times 10 \times 0.0001%` in the cluster with 10 OSDs, and
+only :math:`\approx 4 \times 20 \times 0.0001%` in the cluster with 20 OSDs.
+
+In summary, the greater the number of OSDs, the faster the recovery and the
+lower the risk of permanently losing a PG due to cascading failures. As far as
+data durability is concerned, in a cluster with fewer than 50 OSDs, it doesn't
+much matter whether there are 512 or 4096 PGs.
+
+.. note::  It can take a long time for an OSD that has been recently added to
+   the cluster to be populated with the PGs assigned to it.  However, no object
+   degradation or impact on data durability will result from the slowness of
+   this process since Ceph populates data into the new PGs before removing it
+   from the old PGs.
+
+.. _object distribution:
+
+Object distribution within a pool
+---------------------------------
+
+Under ideal conditions, objects are evenly distributed across PGs. Because
+CRUSH computes the PG for each object but does not know how much data is stored
+in each OSD associated with the PG, the ratio between the number of PGs and the
+number of OSDs can have a significant influence on data distribution.
+
+For example, suppose that there is only a single PG for ten OSDs in a
+three-replica pool. In that case, only three OSDs would be used because CRUSH
+would have no other option. However, if more PGs are available, RADOS objects are
+more likely to be evenly distributed across OSDs.  CRUSH makes every effort to
+distribute OSDs evenly across all existing PGs.
+
+As long as there are one or two orders of magnitude more PGs than OSDs, the
+distribution is likely to be even. For example: 256 PGs for 3 OSDs, 512 PGs for
+10 OSDs, or 1024 PGs for 10 OSDs.
+
+However, uneven data distribution can emerge due to factors other than the
+ratio of PGs to OSDs. For example, since CRUSH does not take into account the
+size of the RADOS objects, the presence of a few very large RADOS objects can
+create an imbalance. Suppose that one million 4 KB RADOS objects totaling 4 GB
+are evenly distributed among 1024 PGs on 10 OSDs. These RADOS objects will
+consume 4 GB / 10 = 400 MB on each OSD. If a single 400 MB RADOS object is then
+added to the pool, the three OSDs supporting the PG in which the RADOS object
+has been placed will each be filled with 400 MB + 400 MB = 800 MB but the seven
+other OSDs will still contain only 400 MB.
+
+.. _resource usage:
+
+Memory, CPU and network usage
+-----------------------------
+
+Every PG in the cluster imposes memory, network, and CPU demands upon OSDs and
+MONs. These needs must be met at all times and are increased during recovery.
+Indeed, one of the main reasons PGs were developed was to share this overhead
+by clustering objects together.
+
+For this reason, minimizing the number of PGs saves significant resources.
+
+.. _choosing-number-of-placement-groups:
+
+Choosing the Number of PGs
+==========================
+
+.. note: It is rarely necessary to do the math in this section by hand.
+   Instead, use the ``ceph osd pool autoscale-status`` command in combination
+   with the ``target_size_bytes`` or ``target_size_ratio`` pool properties. For
+   more information, see :ref:`pg-autoscaler`.
+
+If you have more than 50 OSDs, we recommend approximately 50-100 PGs per OSD in
+order to balance resource usage, data durability, and data distribution. If you
+have fewer than 50 OSDs, follow the guidance in the `preselection`_ section.
+For a single pool, use the following formula to get a baseline value:
+
+  Total PGs = :math:`\frac{OSDs \times 100}{pool \: size}`
+
+Here **pool size** is either the number of replicas for replicated pools or the
+K+M sum for erasure-coded pools. To retrieve this sum, run the command ``ceph
+osd erasure-code-profile get``.
+
+Next, check whether the resulting baseline value is consistent with the way you
+designed your Ceph cluster to maximize `data durability`_ and `object
+distribution`_ and to minimize `resource usage`_.
+
+This value should be **rounded up to the nearest power of two**.
+
+Each pool's ``pg_num`` should be a power of two. Other values are likely to
+result in uneven distribution of data across OSDs. It is best to increase
+``pg_num`` for a pool only when it is feasible and desirable to set the next
+highest power of two. Note that this power of two rule is per-pool; it is
+neither necessary nor easy to align the sum of all pools' ``pg_num`` to a power
+of two.
+
+For example, if you have a cluster with 200 OSDs and a single pool with a size
+of 3 replicas, estimate the number of PGs as follows:
+
+  :math:`\frac{200 \times 100}{3} = 6667`. Rounded up to the nearest power of 2: 8192.
+
+When using multiple data pools to store objects, make sure that you balance the
+number of PGs per pool against the number of PGs per OSD so that you arrive at
+a reasonable total number of PGs. It is important to find a number that
+provides reasonably low variance per OSD without taxing system resources or
+making the peering process too slow.
+
+For example, suppose you have a cluster of 10 pools, each with 512 PGs on 10
+OSDs. That amounts to 5,120 PGs distributed across 10 OSDs, or 512 PGs per OSD.
+This cluster will not use too many resources. However, in a cluster of 1,000
+pools, each with 512 PGs on 10 OSDs, the OSDs will have to handle ~50,000 PGs
+each. This cluster will require significantly more resources and significantly
+more time for peering.
+
+For determining the optimal number of PGs per OSD, we recommend the `PGCalc`_
+tool.
+
+
+.. _setting the number of placement groups:
+
+Setting the Number of PGs
+=========================
+
+Setting the initial number of PGs in a pool must be done at the time you create
+the pool. See `Create a Pool`_ for details. 
+
+However, even after a pool is created, if the ``pg_autoscaler`` is not being
+used to manage ``pg_num`` values, you can change the number of PGs by running a
+command of the following form:
+
+.. prompt:: bash # 
+
+   ceph osd pool set {pool-name} pg_num {pg_num}
+
+If you increase the number of PGs, your cluster will not rebalance until you
+increase the number of PGs for placement (``pgp_num``). The ``pgp_num``
+parameter specifies the number of PGs that are to be considered for placement
+by the CRUSH algorithm. Increasing ``pg_num`` splits the PGs in your cluster,
+but data will not be migrated to the newer PGs until ``pgp_num`` is increased.
+The ``pgp_num`` parameter should be equal to the ``pg_num`` parameter. To
+increase the number of PGs for placement, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} pgp_num {pgp_num}
+
+If you decrease the number of PGs, then ``pgp_num`` is adjusted automatically.
+In releases of Ceph that are Nautilus and later (inclusive), when the
+``pg_autoscaler`` is not used, ``pgp_num`` is automatically stepped to match
+``pg_num``. This process manifests as periods of remapping of PGs and of
+backfill, and is expected behavior and normal.
+
+.. _rados_ops_pgs_get_pg_num:
+
+Get the Number of PGs
+=====================
+
+To get the number of PGs in a pool, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph osd pool get {pool-name} pg_num
+
+
+Get a Cluster's PG Statistics
+=============================
+
+To see the details of the PGs in your cluster, run a command of the following
+form:
+
+.. prompt:: bash #
+
+   ceph pg dump [--format {format}]
+
+Valid formats are ``plain`` (default) and ``json``.
+
+
+Get Statistics for Stuck PGs
+============================
+
+To see the statistics for all PGs that are stuck in a specified state, run a
+command of the following form:
+
+.. prompt:: bash #
+
+   ceph pg dump_stuck inactive|unclean|stale|undersized|degraded [--format <format>] [-t|--threshold <seconds>]
+
+- **Inactive** PGs cannot process reads or writes because they are waiting for
+  enough OSDs with the most up-to-date data to come ``up`` and ``in``.
+
+- **Undersized** PGs contain objects that have not been replicated the desired
+  number of times. Under normal conditions, it can be assumed that these PGs
+  are recovering.
+
+- **Stale** PGs are in an unknown state -- the OSDs that host them have not
+  reported to the monitor cluster for a certain period of time (determined by
+  ``mon_osd_report_timeout``).
+
+Valid formats are ``plain`` (default) and ``json``. The threshold defines the
+minimum number of seconds the PG is stuck before it is included in the returned
+statistics (default: 300).
+
+
+Get a PG Map
+============
+
+To get the PG map for a particular PG, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph pg map {pg-id}
+
+For example: 
+
+.. prompt:: bash #
+
+   ceph pg map 1.6c
+
+Ceph will return the PG map, the PG, and the OSD status. The output resembles
+the following:
+
+.. prompt:: bash #
+
+   osdmap e13 pg 1.6c (1.6c) -> up [1,0] acting [1,0]
+
+
+Get a PG's Statistics
+=====================
+
+To see statistics for a particular PG, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph pg {pg-id} query
+
+
+Scrub a PG
+==========
+
+To scrub a PG, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph pg scrub {pg-id}
+
+Ceph checks the primary and replica OSDs, generates a catalog of all objects in
+the PG, and compares the objects against each other in order to ensure that no
+objects are missing or mismatched and that their contents are consistent. If
+the replicas all match, then a final semantic sweep takes place to ensure that
+all snapshot-related object metadata is consistent.  Errors are reported in
+logs.
+
+To scrub all PGs from a specific pool, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph osd pool scrub {pool-name}
+
+
+Prioritize backfill/recovery of PG(s)
+=====================================
+
+You might encounter a situation in which multiple PGs require recovery or
+backfill, but the data in some PGs is more important than the data in others
+(for example, some PGs hold data for images that are used by running machines
+and other PGs are used by inactive machines and hold data that is less
+relevant). In that case, you might want to prioritize recovery or backfill of
+the PGs with especially important data so that the performance of the cluster
+and the availability of their data are restored sooner. To designate specific
+PG(s) as prioritized during recovery, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph pg force-recovery {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+
+To mark specific PG(s) as prioritized during backfill, run a command of the
+following form:
+
+.. prompt:: bash #
+
+   ceph pg force-backfill {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+
+These commands instruct Ceph to perform recovery or backfill on the specified
+PGs before processing the other PGs. Prioritization does not interrupt current
+backfills or recovery, but places the specified PGs at the top of the queue so
+that they will be acted upon next.  If you change your mind or realize that you
+have prioritized the wrong PGs, run one or both of the following commands:
+
+.. prompt:: bash #
+
+   ceph pg cancel-force-recovery {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+   ceph pg cancel-force-backfill {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+
+These commands remove the ``force`` flag from the specified PGs, so that the
+PGs will be processed in their usual order. As in the case of adding the
+``force`` flag, this affects only those PGs that are still queued but does not
+affect PGs currently undergoing recovery.
+
+The ``force`` flag is cleared automatically after recovery or backfill of the
+PGs is complete.
+
+Similarly, to instruct Ceph to prioritize all PGs from a specified pool (that
+is, to perform recovery or backfill on those PGs first), run one or both of the
+following commands:
+
+.. prompt:: bash #
+
+   ceph osd pool force-recovery {pool-name}
+   ceph osd pool force-backfill {pool-name}
+
+These commands can also be cancelled. To revert to the default order, run one
+or both of the following commands:
+
+.. prompt:: bash #
+
+   ceph osd pool cancel-force-recovery {pool-name}
+   ceph osd pool cancel-force-backfill {pool-name}
+
+.. warning:: These commands can break the order of Ceph's internal priority
+   computations, so use them with caution! If you have multiple pools that are
+   currently sharing the same underlying OSDs, and if the data held by certain
+   pools is more important than the data held by other pools, then we recommend
+   that you run a command of the following form to arrange a custom
+   recovery/backfill priority for all pools:
+
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} recovery_priority {value}
+
+For example, if you have twenty pools, you could make the most important pool  
+priority ``20``, and the next most important pool priority ``19``, and so on.  
+
+Another option is to set the recovery/backfill priority for only a proper
+subset of pools. In such a scenario, three important pools might (all) be
+assigned priority ``1`` and all other pools would be left without an assigned
+recovery/backfill priority.  Another possibility is to select three important
+pools and set their recovery/backfill priorities to ``3``, ``2``, and ``1``
+respectively.
+
+.. important:: Numbers of greater value have higher priority than numbers of
+   lesser value when using ``ceph osd pool set {pool-name} recovery_priority
+   {value}`` to set their recovery/backfill priority. For example, a pool with
+   the recovery/backfill priority ``30`` has a higher priority than a pool with
+   the recovery/backfill priority ``15``.
+
+Reverting Lost RADOS Objects
+============================
+
+If the cluster has lost one or more RADOS objects and you have decided to
+abandon the search for the lost data, you must mark the unfound objects
+``lost``.
+
+If every possible location has been queried and all OSDs are ``up`` and ``in``,
+but certain RADOS objects are still lost, you might have to give up on those
+objects. This situation can arise when rare and unusual combinations of
+failures allow the cluster to learn about writes that were performed before the
+writes themselves were recovered.
+
+The command to mark a RADOS object ``lost`` has only one supported option:
+``revert``. The ``revert`` option will either roll back to a previous version
+of the RADOS object (if it is old enough to have a previous version) or forget
+about it entirely (if it is too new to have a previous version). To mark the
+"unfound" objects ``lost``, run a command of the following form:
+
+
+.. prompt:: bash #
+
+   ceph pg {pg-id} mark_unfound_lost revert|delete
+
+.. important:: Use this feature with caution. It might confuse applications
+   that expect the object(s) to exist.
+
+
+.. toctree::
+        :hidden:
+
+        pg-states
+        pg-concepts
+
+
+.. _Create a Pool: ../pools#createpool
+.. _Mapping PGs to OSDs: ../../../architecture#mapping-pgs-to-osds
+.. _pgcalc: https://old.ceph.com/pgcalc/
diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
new file mode 100644
index 000000000..dda9e844e
--- /dev/null
+++ b/doc/rados/operations/pools.rst
@@ -0,0 +1,751 @@
+.. _rados_pools:
+
+=======
+ Pools
+=======
+Pools are logical partitions that are used to store objects.
+
+Pools provide:
+
+- **Resilience**: It is possible to set the number of OSDs that are allowed to
+  fail without any data being lost. If your cluster uses replicated pools, the
+  number of OSDs that can fail without data loss is equal to the number of
+  replicas.
+  
+  For example: a typical configuration stores an object and two replicas
+  (copies) of each RADOS object (that is: ``size = 3``), but you can configure
+  the number of replicas on a per-pool basis. For `erasure-coded pools
+  <../erasure-code>`_, resilience is defined as the number of coding chunks
+  (for example, ``m = 2`` in the default **erasure code profile**).
+
+- **Placement Groups**: You can set the number of placement groups (PGs) for
+  the pool. In a typical configuration, the target number of PGs is
+  approximately one hundred PGs per OSD. This provides reasonable balancing
+  without consuming excessive computing resources.  When setting up multiple
+  pools, be careful to set an appropriate number of PGs for each pool and for
+  the cluster as a whole. Each PG belongs to a specific pool: when multiple
+  pools use the same OSDs, make sure that the **sum** of PG replicas per OSD is
+  in the desired PG-per-OSD target range. To calculate an appropriate number of
+  PGs for your pools, use the `pgcalc`_ tool.
+
+- **CRUSH Rules**: When data is stored in a pool, the placement of the object
+  and its replicas (or chunks, in the case of erasure-coded pools) in your
+  cluster is governed by CRUSH rules. Custom CRUSH rules can be created for a
+  pool if the default rule does not fit your use case.
+
+- **Snapshots**: The command ``ceph osd pool mksnap`` creates a snapshot of a
+  pool.
+
+Pool Names
+==========
+
+Pool names beginning with ``.`` are reserved for use by Ceph's internal
+operations. Do not create or manipulate pools with these names.
+
+
+List Pools
+==========
+
+There are multiple ways to get the list of pools in your cluster.
+
+To list just your cluster's pool names (good for scripting), execute:
+
+.. prompt:: bash $
+
+   ceph osd pool ls
+
+::
+
+   .rgw.root
+   default.rgw.log
+   default.rgw.control
+   default.rgw.meta
+
+To list your cluster's pools with the pool number, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd lspools
+
+::
+
+   1 .rgw.root
+   2 default.rgw.log
+   3 default.rgw.control
+   4 default.rgw.meta
+
+To list your cluster's pools with additional information, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool ls detail
+
+::
+
+   pool 1 '.rgw.root' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 19 flags hashpspool stripe_width 0 application rgw read_balance_score 4.00
+   pool 2 'default.rgw.log' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 21 flags hashpspool stripe_width 0 application rgw read_balance_score 4.00
+   pool 3 'default.rgw.control' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 23 flags hashpspool stripe_width 0 application rgw read_balance_score 4.00
+   pool 4 'default.rgw.meta' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 25 flags hashpspool stripe_width 0 pg_autoscale_bias 4 application rgw read_balance_score 4.00
+
+To get even more information, you can execute this command with the ``--format`` (or ``-f``) option and the ``json``, ``json-pretty``, ``xml`` or ``xml-pretty`` value.
+
+.. _createpool:
+
+Creating a Pool
+===============
+
+Before creating a pool, consult `Pool, PG and CRUSH Config Reference`_.  Your
+Ceph configuration file contains a setting (namely, ``pg_num``) that determines
+the number of PGs.  However, this setting's default value is NOT appropriate
+for most systems.  In most cases, you should override this default value when
+creating your pool.  For details on PG numbers, see `setting the number of
+placement groups`_
+
+For example:
+
+.. prompt:: bash $
+
+    osd_pool_default_pg_num = 128
+    osd_pool_default_pgp_num = 128
+
+.. note:: In Luminous and later releases, each pool must be associated with the
+   application that will be using the pool. For more information, see
+   `Associating a Pool with an Application`_ below.
+
+To create a pool, run one of the following commands:
+
+.. prompt:: bash $
+
+    ceph osd pool create {pool-name} [{pg-num} [{pgp-num}]] [replicated] \
+             [crush-rule-name] [expected-num-objects]
+
+or:
+
+.. prompt:: bash $
+
+    ceph osd pool create {pool-name} [{pg-num} [{pgp-num}]]   erasure \
+             [erasure-code-profile] [crush-rule-name] [expected_num_objects] [--autoscale-mode=<on,off,warn>]
+
+For a brief description of the elements of the above commands, consult the
+following:
+
+.. describe:: {pool-name}
+
+   The name of the pool. It must be unique.
+
+   :Type: String
+   :Required: Yes.
+
+.. describe:: {pg-num}
+
+   The total number of PGs in the pool. For details on calculating an
+   appropriate number, see :ref:`placement groups`. The default value ``8`` is
+   NOT suitable for most systems.
+
+  :Type: Integer
+  :Required: Yes.
+  :Default: 8
+
+.. describe:: {pgp-num}
+
+   The total number of PGs for placement purposes. This **should be equal to
+   the total number of PGs**, except briefly while ``pg_num`` is being
+   increased or decreased. 
+
+  :Type: Integer
+  :Required: Yes. If no value has been specified in the command, then the default value is used (unless a different value has been set in Ceph configuration).
+  :Default: 8
+
+.. describe:: {replicated|erasure}
+
+   The pool type. This can be either **replicated** (to recover from lost OSDs
+   by keeping multiple copies of the objects) or **erasure** (to achieve a kind
+   of `generalized parity RAID <../erasure-code>`_ capability).  The
+   **replicated** pools require more raw storage but can implement all Ceph
+   operations. The **erasure** pools require less raw storage but can perform
+   only some Ceph tasks and may provide decreased performance.
+
+  :Type: String
+  :Required: No.
+  :Default: replicated
+
+.. describe:: [crush-rule-name]
+
+   The name of the CRUSH rule to use for this pool. The specified rule must
+   exist; otherwise the command will fail.
+
+   :Type: String
+   :Required: No.
+   :Default: For **replicated** pools, it is the rule specified by the :confval:`osd_pool_default_crush_rule` configuration variable. This rule must exist.  For **erasure** pools, it is the ``erasure-code`` rule if the ``default`` `erasure code profile`_ is used or the ``{pool-name}`` rule  if not. This rule will be created implicitly if it doesn't already exist.
+
+.. describe:: [erasure-code-profile=profile]
+
+   For **erasure** pools only. Instructs Ceph to use the specified `erasure
+   code profile`_. This profile must be an existing profile as defined by **osd
+   erasure-code-profile set**.
+
+  :Type: String
+  :Required: No.
+
+.. _erasure code profile: ../erasure-code-profile
+
+.. describe:: --autoscale-mode=<on,off,warn>
+
+   - ``on``: the Ceph cluster will autotune or recommend changes to the number of PGs in your pool based on actual usage.
+   - ``warn``: the Ceph cluster will autotune or recommend changes to the number of PGs in your pool based on actual usage.
+   - ``off``: refer to :ref:`placement groups` for more information.
+
+  :Type: String
+  :Required: No.
+  :Default: The default behavior is determined by the :confval:`osd_pool_default_pg_autoscale_mode` option.
+
+.. describe:: [expected-num-objects]
+
+   The expected number of RADOS objects for this pool. By setting this value and
+   assigning a negative value to **filestore merge threshold**, you arrange
+   for the PG folder splitting to occur at the time of pool creation and
+   avoid the latency impact that accompanies runtime folder splitting.
+
+   :Type: Integer
+   :Required: No.
+   :Default: 0, no splitting at the time of pool creation.
+
+.. _associate-pool-to-application:
+
+Associating a Pool with an Application
+======================================
+
+Pools need to be associated with an application before they can be used. Pools
+that are intended for use with CephFS and pools that are created automatically
+by RGW are associated automatically. Pools that are intended for use with RBD
+should be initialized with the ``rbd`` tool (see `Block Device Commands`_ for
+more information).
+
+For other cases, you can manually associate a free-form application name to a
+pool by running the following command.:
+
+.. prompt:: bash $
+
+   ceph osd pool application enable {pool-name} {application-name}
+
+.. note:: CephFS uses the application name ``cephfs``, RBD uses the
+   application name ``rbd``, and RGW uses the application name ``rgw``.
+
+Setting Pool Quotas
+===================
+
+To set pool quotas for the maximum number of bytes and/or the maximum number of
+RADOS objects per pool, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota {pool-name} [max_objects {obj-count}] [max_bytes {bytes}]
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota data max_objects 10000
+
+To remove a quota, set its value to ``0``.
+
+
+Deleting a Pool
+===============
+
+To delete a pool, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool delete {pool-name} [{pool-name} --yes-i-really-really-mean-it]
+
+To remove a pool, you must set the ``mon_allow_pool_delete`` flag to ``true``
+in the monitor's configuration. Otherwise, monitors will refuse to remove
+pools.
+
+For more information, see `Monitor Configuration`_.
+
+.. _Monitor Configuration: ../../configuration/mon-config-ref
+
+If there are custom rules for a pool that is no longer needed, consider
+deleting those rules.
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} crush_rule
+
+For example, if the custom rule is "123", check all pools to see whether they
+use the rule by running the following command:
+
+.. prompt:: bash $
+
+    ceph osd dump | grep "^pool" | grep "crush_rule 123"
+
+If no pools use this custom rule, then it is safe to delete the rule from the
+cluster.
+
+Similarly, if there are users with permissions restricted to a pool that no
+longer exists, consider deleting those users by running commands of the
+following forms:
+
+.. prompt:: bash $
+
+    ceph auth ls | grep -C 5 {pool-name}
+    ceph auth del {user}
+
+
+Renaming a Pool
+===============
+
+To rename a pool, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool rename {current-pool-name} {new-pool-name}
+
+If you rename a pool for which an authenticated user has per-pool capabilities,
+you must update the user's capabilities ("caps") to refer to the new pool name.
+
+
+Showing Pool Statistics
+=======================
+
+To show a pool's utilization statistics, run the following command:
+
+.. prompt:: bash $
+
+   rados df
+
+To obtain I/O information for a specific pool or for all pools, run a command
+of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool stats [{pool-name}]
+
+
+Making a Snapshot of a Pool
+===========================
+
+To make a snapshot of a pool, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool mksnap {pool-name} {snap-name}
+
+Removing a Snapshot of a Pool
+=============================
+
+To remove a snapshot of a pool, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool rmsnap {pool-name} {snap-name}
+
+.. _setpoolvalues:
+
+Setting Pool Values
+===================
+
+To assign values to a pool's configuration keys, run a command of the following
+form:
+
+.. prompt:: bash $
+
+   ceph osd pool set {pool-name} {key} {value}
+
+You may set values for the following keys:
+
+.. _compression_algorithm:
+
+.. describe:: compression_algorithm
+   
+   :Description: Sets the inline compression algorithm used in storing data on the underlying BlueStore back end. This key's setting overrides the global setting :confval:`bluestore_compression_algorithm`.
+   :Type: String
+   :Valid Settings: ``lz4``, ``snappy``, ``zlib``, ``zstd``
+
+.. describe:: compression_mode
+   
+   :Description: Sets the policy for the inline compression algorithm used in storing data on the underlying BlueStore back end. This key's setting overrides the global setting :confval:`bluestore_compression_mode`.
+   :Type: String
+   :Valid Settings: ``none``, ``passive``, ``aggressive``, ``force``
+
+.. describe:: compression_min_blob_size
+
+   
+   :Description: Sets the minimum size for the compression of chunks: that is, chunks smaller than this are not compressed.  This key's setting overrides the following global settings:
+   
+   * :confval:`bluestore_compression_min_blob_size` 
+   * :confval:`bluestore_compression_min_blob_size_hdd`
+   * :confval:`bluestore_compression_min_blob_size_ssd`
+
+   :Type: Unsigned Integer
+
+
+.. describe:: compression_max_blob_size
+   
+   :Description: Sets the maximum size for chunks: that is, chunks larger than this are broken into smaller blobs of this size before compression is performed.
+   :Type: Unsigned Integer
+
+.. _size:
+
+.. describe:: size
+   
+   :Description: Sets the number of replicas for objects in the pool. For further details, see `Setting the Number of RADOS Object Replicas`_. Replicated pools only.
+   :Type: Integer
+
+.. _min_size:
+
+.. describe:: min_size
+   
+   :Description: Sets the minimum number of replicas required for I/O.  For further details, see `Setting the Number of RADOS Object Replicas`_.  For erasure-coded pools, this should be set to a value greater than 'k'. If I/O is allowed at the value 'k', then there is no redundancy and data will be lost in the event of a permanent OSD failure. For more information, see `Erasure Code <../erasure-code>`_
+   :Type: Integer
+   :Version: ``0.54`` and above
+
+.. _pg_num:
+
+.. describe:: pg_num
+   
+   :Description: Sets the effective number of PGs to use when calculating data placement.
+   :Type: Integer
+   :Valid Range: ``0`` to ``mon_max_pool_pg_num``. If set to ``0``, the value of ``osd_pool_default_pg_num`` will be used. 
+
+.. _pgp_num:
+
+.. describe:: pgp_num
+   
+   :Description: Sets the effective number of PGs to use when calculating data placement.
+   :Type: Integer
+   :Valid Range: Between ``1`` and the current value of ``pg_num``.
+
+.. _crush_rule:
+
+.. describe:: crush_rule
+   
+   :Description: Sets the CRUSH rule that Ceph uses to map object placement within the pool.
+   :Type: String
+
+.. _allow_ec_overwrites:
+
+.. describe:: allow_ec_overwrites
+   
+   :Description: Determines whether writes to an erasure-coded pool are allowed to update only part of a RADOS object. This allows CephFS and RBD to use an EC (erasure-coded) pool for user data (but not for metadata). For more details, see `Erasure Coding with Overwrites`_.
+   :Type: Boolean
+
+   .. versionadded:: 12.2.0
+   
+.. describe:: hashpspool
+
+   :Description: Sets and unsets the HASHPSPOOL flag on a given pool.
+   :Type: Integer
+   :Valid Range: 1 sets flag, 0 unsets flag
+
+.. _nodelete:
+
+.. describe:: nodelete
+
+   :Description: Sets and unsets the NODELETE flag on a given pool.
+   :Type: Integer
+   :Valid Range: 1 sets flag, 0 unsets flag
+   :Version: Version ``FIXME``
+
+.. _nopgchange:
+
+.. describe:: nopgchange
+
+   :Description: Sets and unsets the NOPGCHANGE flag on a given pool.
+   :Type: Integer
+   :Valid Range: 1 sets flag, 0 unsets flag
+   :Version: Version ``FIXME``
+
+.. _nosizechange:
+
+.. describe:: nosizechange
+
+   :Description: Sets and unsets the NOSIZECHANGE flag on a given pool.
+   :Type: Integer
+   :Valid Range: 1 sets flag, 0 unsets flag
+   :Version: Version ``FIXME``
+
+.. _bulk:
+
+.. describe:: bulk
+
+   :Description: Sets and unsets the bulk flag on a given pool.
+   :Type: Boolean
+   :Valid Range: ``true``/``1`` sets flag, ``false``/``0`` unsets flag
+
+.. _write_fadvise_dontneed:
+
+.. describe:: write_fadvise_dontneed
+
+   :Description: Sets and unsets the WRITE_FADVISE_DONTNEED flag on a given pool.
+   :Type: Integer
+   :Valid Range: ``1`` sets flag, ``0`` unsets flag
+
+.. _noscrub:
+
+.. describe:: noscrub
+
+   :Description: Sets and unsets the NOSCRUB flag on a given pool.
+   :Type: Integer
+   :Valid Range: ``1`` sets flag, ``0`` unsets flag
+
+.. _nodeep-scrub:
+
+.. describe:: nodeep-scrub
+
+   :Description: Sets and unsets the NODEEP_SCRUB flag on a given pool.
+   :Type: Integer
+   :Valid Range: ``1`` sets flag, ``0`` unsets flag
+
+.. _target_max_bytes:
+
+.. describe:: target_max_bytes
+   
+   :Description: Ceph will begin flushing or evicting objects when the
+                 ``max_bytes`` threshold is triggered.
+   :Type: Integer
+   :Example: ``1000000000000``  #1-TB
+
+.. _target_max_objects:
+
+.. describe:: target_max_objects
+   
+   :Description: Ceph will begin flushing or evicting objects when the
+                 ``max_objects`` threshold is triggered.
+   :Type: Integer
+   :Example: ``1000000`` #1M objects
+
+.. _fast_read:
+
+.. describe:: fast_read
+   
+   :Description: For erasure-coded pools, if this flag is turned ``on``, the
+                 read request issues "sub reads" to all shards, and then waits
+                 until it receives enough shards to decode before it serves 
+                 the client. If *jerasure* or *isa* erasure plugins are in 
+                 use, then after the first *K* replies have returned, the 
+                 client's request is served immediately using the data decoded 
+                 from these replies. This approach sacrifices resources in 
+                 exchange for better performance. This flag is supported only 
+                 for erasure-coded pools.
+   :Type: Boolean 
+   :Defaults: ``0``
+
+.. _scrub_min_interval:
+
+.. describe:: scrub_min_interval
+   
+   :Description: Sets the minimum interval (in seconds) for successive scrubs of the pool's PGs when the load is low. If the default value of ``0`` is in effect, then the value of ``osd_scrub_min_interval`` from central config is used.
+
+   :Type: Double
+   :Default: ``0``
+
+.. _scrub_max_interval:
+
+.. describe:: scrub_max_interval
+   
+   :Description: Sets the maximum interval (in seconds) for scrubs of the pool's PGs regardless of cluster load. If the value of ``scrub_max_interval`` is ``0``, then the value ``osd_scrub_max_interval`` from central config is used.
+
+   :Type: Double
+   :Default: ``0``
+
+.. _deep_scrub_interval:
+
+.. describe:: deep_scrub_interval
+   
+   :Description: Sets the interval (in seconds) for pool “deep� scrubs of the pool's PGs. If the value of ``deep_scrub_interval`` is ``0``, the value ``osd_deep_scrub_interval`` from central config is used.
+
+   :Type: Double
+   :Default: ``0``
+
+.. _recovery_priority:
+
+.. describe:: recovery_priority
+   
+   :Description: Setting this value adjusts a pool's computed reservation priority. This value must be in the range ``-10`` to ``10``. Any pool assigned a negative value will be given a lower priority than any new pools, so users are directed to assign negative values to low-priority pools.
+
+   :Type: Integer
+   :Default: ``0``
+
+
+.. _recovery_op_priority:
+
+.. describe:: recovery_op_priority
+   
+   :Description: Sets the recovery operation priority for a specific pool's PGs. This overrides the general priority determined by :confval:`osd_recovery_op_priority`.
+
+   :Type: Integer
+   :Default: ``0``
+
+
+Getting Pool Values
+===================
+
+To get a value from a pool's key, run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} {key}
+
+
+You may get values from the following keys:
+
+
+``size``
+
+:Description: See size_.
+
+:Type: Integer
+
+
+``min_size``
+
+:Description: See min_size_.
+
+:Type: Integer
+:Version: ``0.54`` and above
+
+
+``pg_num``
+
+:Description: See pg_num_.
+
+:Type: Integer
+
+
+``pgp_num``
+
+:Description: See pgp_num_.
+
+:Type: Integer
+:Valid Range: Equal to or less than ``pg_num``.
+
+
+``crush_rule``
+
+:Description: See crush_rule_.
+
+
+``target_max_bytes``
+
+:Description: See target_max_bytes_.
+
+:Type: Integer
+
+
+``target_max_objects``
+
+:Description: See target_max_objects_.
+
+:Type: Integer
+
+
+``fast_read``
+
+:Description: See fast_read_.
+
+:Type: Boolean
+
+
+``scrub_min_interval``
+
+:Description: See scrub_min_interval_.
+
+:Type: Double
+
+
+``scrub_max_interval``
+
+:Description: See scrub_max_interval_.
+
+:Type: Double
+
+
+``deep_scrub_interval``
+
+:Description: See deep_scrub_interval_.
+
+:Type: Double
+
+
+``allow_ec_overwrites``
+
+:Description: See allow_ec_overwrites_.
+
+:Type: Boolean
+
+
+``recovery_priority``
+
+:Description: See recovery_priority_.
+
+:Type: Integer
+
+
+``recovery_op_priority``
+
+:Description: See recovery_op_priority_.
+
+:Type: Integer
+
+
+Setting the Number of RADOS Object Replicas
+===========================================
+
+To set the number of data replicas on a replicated pool, run a command of the
+following form:
+
+.. prompt:: bash $
+
+   ceph osd pool set {poolname} size {num-replicas}
+
+.. important:: The ``{num-replicas}`` argument includes the primary object
+   itself.  For example, if you want there to be two replicas of the object in
+   addition to the original object (for a total of three instances of the
+   object) specify ``3`` by running the following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set data size 3
+
+You may run the above command for each pool. 
+
+.. Note:: An object might accept I/Os in degraded mode with fewer than ``pool
+   size`` replicas. To set a minimum number of replicas required for I/O, you
+   should use the ``min_size`` setting.  For example, you might run the
+   following command:
+
+.. prompt:: bash $
+
+   ceph osd pool set data min_size 2
+
+This command ensures that no object in the data pool will receive I/O if it has
+fewer than ``min_size`` (in this case, two) replicas.
+
+
+Getting the Number of Object Replicas
+=====================================
+
+To get the number of object replicas, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd dump | grep 'replicated size'
+
+Ceph will list pools and highlight the ``replicated size`` attribute.  By
+default, Ceph creates two replicas of an object (a total of three copies, for a
+size of ``3``).
+
+Managing pools that are flagged with ``--bulk``
+===============================================
+See :ref:`managing_bulk_flagged_pools`.
+
+
+.. _pgcalc: https://old.ceph.com/pgcalc/
+.. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
+.. _Bloom Filter: https://en.wikipedia.org/wiki/Bloom_filter
+.. _setting the number of placement groups: ../placement-groups#set-the-number-of-placement-groups
+.. _Erasure Coding with Overwrites: ../erasure-code#erasure-coding-with-overwrites
+.. _Block Device Commands: ../../../rbd/rados-rbd-cmds/#create-a-block-device-pool
diff --git a/doc/rados/operations/read-balancer.rst b/doc/rados/operations/read-balancer.rst
new file mode 100644
index 000000000..0833e4326
--- /dev/null
+++ b/doc/rados/operations/read-balancer.rst
@@ -0,0 +1,64 @@
+.. _read_balancer:
+
+=======================================
+Operating the Read (Primary) Balancer
+=======================================
+
+You might be wondering: How can I improve performance in my Ceph cluster?
+One important data point you can check is the ``read_balance_score`` on each
+of your replicated pools.
+
+This metric, available via ``ceph osd pool ls detail`` (see :ref:`rados_pools`
+for more details) indicates read performance, or how balanced the primaries are
+for each replicated pool. In most cases, if a ``read_balance_score`` is above 1
+(for instance, 1.5), this means that your pool has unbalanced primaries and that
+you may want to try improving your read performance with the read balancer.
+
+Online Optimization
+===================
+
+At present, there is no online option for the read balancer. However, we plan to add
+the read balancer as an option to the :ref:`balancer` in the next Ceph version
+so it can be enabled to run automatically in the background like the upmap balancer.
+
+Offline Optimization
+====================
+
+Primaries are updated with an offline optimizer that is built into the
+:ref:`osdmaptool`.
+
+#. Grab the latest copy of your osdmap:
+
+   .. prompt:: bash $
+
+      ceph osd getmap -o om
+
+#. Run the optimizer:
+
+   .. prompt:: bash $
+
+      osdmaptool om --read out.txt --read-pool <pool name> [--vstart] 
+
+   It is highly recommended that you run the capacity balancer before running the
+   balancer to ensure optimal results. See :ref:`upmap` for details on how to balance
+   capacity in a cluster.
+
+#. Apply the changes:
+
+   .. prompt:: bash $
+
+      source out.txt
+
+   In the above example, the proposed changes are written to the output file
+   ``out.txt``. The commands in this procedure are normal Ceph CLI commands
+   that can be run in order to apply the changes to the cluster.
+
+   If you are working in a vstart cluster, you may pass the ``--vstart`` parameter
+   as shown above so the CLI commands are formatted with the `./bin/` prefix.
+
+   Note that any time the number of pgs changes (for instance, if the pg autoscaler [:ref:`pg-autoscaler`]
+   kicks in), you should consider rechecking the scores and rerunning the balancer if needed.
+
+To see some details about what the tool is doing, you can pass
+``--debug-osd 10`` to ``osdmaptool``. To see even more details, pass
+``--debug-osd 20`` to ``osdmaptool``.
diff --git a/doc/rados/operations/stretch-mode.rst b/doc/rados/operations/stretch-mode.rst
new file mode 100644
index 000000000..f797b5b91
--- /dev/null
+++ b/doc/rados/operations/stretch-mode.rst
@@ -0,0 +1,262 @@
+.. _stretch_mode:
+
+================
+Stretch Clusters
+================
+
+
+Stretch Clusters
+================
+
+A stretch cluster is a cluster that has servers in geographically separated
+data centers, distributed over a WAN. Stretch clusters have LAN-like high-speed
+and low-latency connections, but limited links. Stretch clusters have a higher
+likelihood of (possibly asymmetric) network splits, and a higher likelihood of
+temporary or complete loss of an entire data center (which can represent
+one-third to one-half of the total cluster).
+
+Ceph is designed with the expectation that all parts of its network and cluster
+will be reliable and that failures will be distributed randomly across the
+CRUSH map. Even if a switch goes down and causes the loss of many OSDs, Ceph is
+designed so that the remaining OSDs and monitors will route around such a loss. 
+
+Sometimes this cannot be relied upon. If you have a "stretched-cluster"
+deployment in which much of your cluster is behind a single network component,
+you might need to use **stretch mode** to ensure data integrity.
+
+We will here consider two standard configurations: a configuration with two
+data centers (or, in clouds, two availability zones), and a configuration with
+three data centers (or, in clouds, three availability zones).
+
+In the two-site configuration, Ceph expects each of the sites to hold a copy of
+the data, and Ceph also expects there to be a third site that has a tiebreaker
+monitor. This tiebreaker monitor picks a winner if the network connection fails
+and both data centers remain alive.
+
+The tiebreaker monitor can be a VM. It can also have high latency relative to
+the two main sites.
+
+The standard Ceph configuration is able to survive MANY network failures or
+data-center failures without ever compromising data availability. If enough
+Ceph servers are brought back following a failure, the cluster *will* recover.
+If you lose a data center but are still able to form a quorum of monitors and
+still have all the data available, Ceph will maintain availability. (This
+assumes that the cluster has enough copies to satisfy the pools' ``min_size``
+configuration option, or (failing that) that the cluster has CRUSH rules in
+place that will cause the cluster to re-replicate the data until the
+``min_size`` configuration option has been met.)
+
+Stretch Cluster Issues
+======================
+
+Ceph does not permit the compromise of data integrity and data consistency
+under any circumstances. When service is restored after a network failure or a
+loss of Ceph nodes, Ceph will restore itself to a state of normal functioning
+without operator intervention.  
+
+Ceph does not permit the compromise of data integrity or data consistency, but
+there are situations in which *data availability* is compromised. These
+situations can occur even though there are enough clusters available to satisfy
+Ceph's consistency and sizing constraints. In some situations, you might
+discover that your cluster does not satisfy those constraints.
+
+The first category of these failures that we will discuss involves inconsistent
+networks -- if there is a netsplit (a disconnection between two servers that
+splits the network into two pieces), Ceph might be unable to mark OSDs ``down``
+and remove them from the acting PG sets. This failure to mark ODSs ``down``
+will occur, despite the fact that the primary PG is unable to replicate data (a
+situation that, under normal non-netsplit circumstances, would result in the
+marking of affected OSDs as ``down`` and their removal from the PG). If this
+happens, Ceph will be unable to satisfy its durability guarantees and
+consequently IO will not be permitted.
+
+The second category of failures that we will discuss involves the situation in
+which the constraints are not sufficient to guarantee the replication of data
+across data centers, though it might seem that the data is correctly replicated
+across data centers. For example, in a scenario in which there are two data
+centers named Data Center A and Data Center B, and the CRUSH rule targets three
+replicas and places a replica in each data center with a ``min_size`` of ``2``,
+the PG might go active with two replicas in Data Center A and zero replicas in
+Data Center B. In a situation of this kind, the loss of Data Center A means
+that the data is lost and Ceph will not be able to operate on it. This
+situation is surprisingly difficult to avoid using only standard CRUSH rules.
+
+
+Stretch Mode
+============
+Stretch mode is designed to handle deployments in which you cannot guarantee the
+replication of data across two data centers. This kind of situation can arise
+when the cluster's CRUSH rule specifies that three copies are to be made, but 
+then a copy is placed in each data center with a ``min_size`` of 2. Under such
+conditions, a placement group can become active with two copies in the first
+data center and no copies in the second data center. 
+
+
+Entering Stretch Mode
+---------------------
+
+To enable stretch mode, you must set the location of each monitor, matching
+your CRUSH map. This procedure shows how to do this.
+
+
+#. Place ``mon.a`` in your first data center:
+
+   .. prompt:: bash $
+
+      ceph mon set_location a datacenter=site1
+
+#. Generate a CRUSH rule that places two copies in each data center.
+   This requires editing the CRUSH map directly:
+
+   .. prompt:: bash $
+
+      ceph osd getcrushmap > crush.map.bin
+      crushtool -d crush.map.bin -o crush.map.txt
+
+#. Edit the ``crush.map.txt`` file to add a new rule. Here there is only one
+   other rule (``id 1``), but you might need to use a different rule ID. We
+   have two data-center buckets named ``site1`` and ``site2``:
+
+   ::
+
+      rule stretch_rule {
+             id 1
+             min_size 1
+             max_size 10
+             type replicated
+             step take site1
+             step chooseleaf firstn 2 type host
+             step emit
+             step take site2
+             step chooseleaf firstn 2 type host
+             step emit
+     }
+
+#. Inject the CRUSH map to make the rule available to the cluster:
+
+   .. prompt:: bash $
+
+      crushtool -c crush.map.txt -o crush2.map.bin
+      ceph osd setcrushmap -i crush2.map.bin
+
+#. Run the monitors in connectivity mode. See `Changing Monitor Elections`_.
+
+#. Command the cluster to enter stretch mode. In this example, ``mon.e`` is the
+   tiebreaker monitor and we are splitting across data centers. The tiebreaker
+   monitor must be assigned a data center that is neither ``site1`` nor
+   ``site2``. For this purpose you can create another data-center bucket named
+   ``site3`` in your CRUSH and place ``mon.e`` there:
+
+   .. prompt:: bash $
+
+      ceph mon set_location e datacenter=site3
+      ceph mon enable_stretch_mode e stretch_rule datacenter
+
+When stretch mode is enabled, PGs will become active only when they peer
+across data centers (or across whichever CRUSH bucket type was specified),
+assuming both are alive. Pools will increase in size from the default ``3`` to
+``4``, and two copies will be expected in each site. OSDs will be allowed to
+connect to monitors only if they are in the same data center as the monitors.
+New monitors will not be allowed to join the cluster if they do not specify a
+location.
+
+If all OSDs and monitors in one of the data centers become inaccessible at once,
+the surviving data center enters a "degraded stretch mode". A warning will be
+issued, the ``min_size`` will be reduced to ``1``, and the cluster will be
+allowed to go active with the data in the single remaining site. The pool size
+does not change, so warnings will be generated that report that the pools are
+too small -- but a special stretch mode flag will prevent the OSDs from
+creating extra copies in the remaining data center. This means that the data
+center will keep only two copies, just as before.
+
+When the missing data center comes back, the cluster will enter a "recovery
+stretch mode". This changes the warning and allows peering, but requires OSDs
+only from the data center that was ``up`` throughout the duration of the
+downtime. When all PGs are in a known state, and are neither degraded nor
+incomplete, the cluster transitions back to regular stretch mode, ends the
+warning, restores ``min_size`` to its original value (``2``), requires both
+sites to peer, and no longer requires the site that was up throughout the
+duration of the downtime when peering (which makes failover to the other site
+possible, if needed).
+
+.. _Changing Monitor elections: ../change-mon-elections
+
+Limitations of Stretch Mode 
+===========================
+When using stretch mode, OSDs must be located at exactly two sites. 
+
+Two monitors should be run in each data center, plus a tiebreaker in a third
+(or in the cloud) for a total of five monitors. While in stretch mode, OSDs
+will connect only to monitors within the data center in which they are located.
+OSDs *DO NOT* connect to the tiebreaker monitor.
+
+Erasure-coded pools cannot be used with stretch mode. Attempts to use erasure
+coded pools with stretch mode will fail. Erasure coded pools cannot be created
+while in stretch mode. 
+
+To use stretch mode, you will need to create a CRUSH rule that provides two
+replicas in each data center. Ensure that there are four total replicas: two in
+each data center. If pools exist in the cluster that do not have the default
+``size`` or ``min_size``, Ceph will not enter stretch mode. An example of such
+a CRUSH rule is given above.
+
+Because stretch mode runs with ``min_size`` set to ``1`` (or, more directly,
+``min_size 1``), we recommend enabling stretch mode only when using OSDs on
+SSDs (including NVMe OSDs). Hybrid HDD+SDD or HDD-only OSDs are not recommended
+due to the long time it takes for them to recover after connectivity between
+data centers has been restored. This reduces the potential for data loss.
+
+In the future, stretch mode might support erasure-coded pools and might support
+deployments that have more than two data centers.
+
+Other commands
+==============
+
+Replacing a failed tiebreaker monitor
+-------------------------------------
+
+Turn on a new monitor and run the following command:
+
+.. prompt:: bash $
+
+   ceph mon set_new_tiebreaker mon.<new_mon_name>
+
+This command protests if the new monitor is in the same location as the
+existing non-tiebreaker monitors. **This command WILL NOT remove the previous
+tiebreaker monitor.** Remove the previous tiebreaker monitor yourself.
+
+Using "--set-crush-location" and not "ceph mon set_location"
+------------------------------------------------------------
+
+If you write your own tooling for deploying Ceph, use the
+``--set-crush-location`` option when booting monitors instead of running ``ceph
+mon set_location``. This option accepts only a single ``bucket=loc`` pair (for
+example, ``ceph-mon --set-crush-location 'datacenter=a'``), and that pair must
+match the bucket type that was specified when running ``enable_stretch_mode``.
+
+Forcing recovery stretch mode
+-----------------------------
+
+When in stretch degraded mode, the cluster will go into "recovery" mode
+automatically when the disconnected data center comes back. If that does not
+happen or you want to enable recovery mode early, run the following command:
+
+.. prompt:: bash $
+
+   ceph osd force_recovery_stretch_mode --yes-i-really-mean-it
+
+Forcing normal stretch mode
+---------------------------
+
+When in recovery mode, the cluster should go back into normal stretch mode when
+the PGs are healthy. If this fails to happen or if you want to force the
+cross-data-center peering early and are willing to risk data downtime (or have
+verified separately that all the PGs can peer, even if they aren't fully
+recovered), run the following command:
+
+.. prompt:: bash $
+
+   ceph osd force_healthy_stretch_mode --yes-i-really-mean-it
+
+This command can be used to to remove the ``HEALTH_WARN`` state, which recovery
+mode generates.
diff --git a/doc/rados/operations/upmap.rst b/doc/rados/operations/upmap.rst
new file mode 100644
index 000000000..8541680d8
--- /dev/null
+++ b/doc/rados/operations/upmap.rst
@@ -0,0 +1,113 @@
+.. _upmap:
+
+=======================================
+Using pg-upmap
+=======================================
+
+In Luminous v12.2.z and later releases, there is a *pg-upmap* exception table
+in the OSDMap that allows the cluster to explicitly map specific PGs to
+specific OSDs. This allows the cluster to fine-tune the data distribution to,
+in most cases, uniformly distribute PGs across OSDs.
+
+However, there is an important caveat when it comes to this new feature: it
+requires all clients to understand the new *pg-upmap* structure in the OSDMap.
+
+Online Optimization
+===================
+
+Enabling
+--------
+
+In order to use ``pg-upmap``, the cluster cannot have any pre-Luminous clients.
+By default, new clusters enable the *balancer module*, which makes use of
+``pg-upmap``. If you want to use a different balancer or you want to make your
+own custom ``pg-upmap`` entries, you might want to turn off the balancer in
+order to avoid conflict: 
+
+.. prompt:: bash $
+
+   ceph balancer off
+
+To allow use of the new feature on an existing cluster, you must restrict the
+cluster to supporting only Luminous (and newer) clients.  To do so, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph osd set-require-min-compat-client luminous
+
+This command will fail if any pre-Luminous clients or daemons are connected to
+the monitors. To see which client versions are in use, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph features
+
+Balancer Module
+---------------
+
+The `balancer` module for ``ceph-mgr`` will automatically balance the number of
+PGs per OSD. See :ref:`balancer`
+
+Offline Optimization
+====================
+
+Upmap entries are updated with an offline optimizer that is built into the
+:ref:`osdmaptool`.
+
+#. Grab the latest copy of your osdmap:
+
+   .. prompt:: bash $
+
+      ceph osd getmap -o om
+
+#. Run the optimizer:
+
+   .. prompt:: bash $
+
+      osdmaptool om --upmap out.txt [--upmap-pool <pool>] \ 
+      [--upmap-max <max-optimizations>] \ 
+      [--upmap-deviation <max-deviation>] \ 
+      [--upmap-active]
+
+   It is highly recommended that optimization be done for each pool
+   individually, or for sets of similarly utilized pools. You can specify the
+   ``--upmap-pool`` option multiple times. "Similarly utilized pools" means
+   pools that are mapped to the same devices and that store the same kind of
+   data (for example, RBD image pools are considered to be similarly utilized;
+   an RGW index pool and an RGW data pool are not considered to be similarly
+   utilized).
+
+   The ``max-optimizations`` value determines the maximum number of upmap
+   entries to identify. The default is `10` (as is the case with the
+   ``ceph-mgr`` balancer module), but you should use a larger number if you are
+   doing offline optimization.  If it cannot find any additional changes to
+   make (that is, if the pool distribution is perfect), it will stop early.
+
+   The ``max-deviation`` value defaults to `5`. If an OSD's PG count varies
+   from the computed target number by no more than this amount it will be
+   considered perfect.
+
+   The ``--upmap-active`` option simulates the behavior of the active balancer
+   in upmap mode. It keeps cycling until the OSDs are balanced and reports how
+   many rounds have occurred and how long each round takes. The elapsed time
+   for rounds indicates the CPU load that ``ceph-mgr`` consumes when it computes
+   the next optimization plan.
+
+#. Apply the changes:
+
+   .. prompt:: bash $
+
+      source out.txt
+
+   In the above example, the proposed changes are written to the output file
+   ``out.txt``. The commands in this procedure are normal Ceph CLI commands
+   that can be run in order to apply the changes to the cluster.
+
+The above steps can be repeated as many times as necessary to achieve a perfect
+distribution of PGs for each set of pools.
+
+To see some (gory) details about what the tool is doing, you can pass
+``--debug-osd 10`` to ``osdmaptool``. To see even more details, pass
+``--debug-crush 10`` to ``osdmaptool``.
diff --git a/doc/rados/operations/user-management.rst b/doc/rados/operations/user-management.rst
new file mode 100644
index 000000000..130c02002
--- /dev/null
+++ b/doc/rados/operations/user-management.rst
@@ -0,0 +1,840 @@
+.. _user-management:
+
+=================
+ User Management
+=================
+
+This document describes :term:`Ceph Client` users, and describes the process by
+which they perform authentication and authorization so that they can access the
+:term:`Ceph Storage Cluster`. Users are either individuals or system actors
+(for example, applications) that use Ceph clients to interact with the Ceph
+Storage Cluster daemons.
+
+.. ditaa::
+            +-----+
+            | {o} |
+            |     |
+            +--+--+       /---------\               /---------\
+               |          |  Ceph   |               |  Ceph   |
+            ---+---*----->|         |<------------->|         |
+               |     uses | Clients |               | Servers |
+               |          \---------/               \---------/
+            /--+--\
+            |     |
+            |     |
+             actor
+
+
+When Ceph runs with authentication and authorization enabled (both are enabled
+by default), you must specify a user name and a keyring that contains the
+secret key of the specified user (usually these are specified via the command
+line). If you do not specify a user name, Ceph will use ``client.admin`` as the
+default user name. If you do not specify a keyring, Ceph will look for a
+keyring via the ``keyring`` setting in the Ceph configuration. For example, if
+you execute the ``ceph health`` command without specifying a user or a keyring,
+Ceph will assume that the keyring is in ``/etc/ceph/ceph.client.admin.keyring``
+and will attempt to use that keyring. The following illustrates this behavior:
+
+.. prompt:: bash $
+
+   ceph health
+
+Ceph will interpret the command like this:
+
+.. prompt:: bash $
+
+   ceph -n client.admin --keyring=/etc/ceph/ceph.client.admin.keyring health
+
+Alternatively, you may use the ``CEPH_ARGS`` environment variable to avoid
+re-entry of the user name and secret.
+
+For details on configuring the Ceph Storage Cluster to use authentication, see
+`Cephx Config Reference`_. For details on the architecture of Cephx, see
+`Architecture - High Availability Authentication`_.
+
+Background
+==========
+
+No matter what type of Ceph client is used (for example: Block Device, Object
+Storage, Filesystem, native API), Ceph stores all data as RADOS objects within
+`pools`_.  Ceph users must have access to a given pool in order to read and
+write data, and Ceph users must have execute permissions in order to use Ceph's
+administrative commands. The following concepts will help you understand
+Ceph['s] user management.
+
+.. _rados-ops-user:
+
+User
+----
+
+A user is either an individual or a system actor (for example, an application).
+Creating users allows you to control who (or what) can access your Ceph Storage
+Cluster, its pools, and the data within those pools.
+
+Ceph has the concept of a ``type`` of user. For purposes of user management,
+the type will always be ``client``. Ceph identifies users in a "period-
+delimited form" that consists of the user type and the user ID: for example,
+``TYPE.ID``, ``client.admin``, or ``client.user1``. The reason for user typing
+is that the Cephx protocol is used not only by clients but also non-clients,
+such as Ceph Monitors, OSDs, and Metadata Servers. Distinguishing the user type
+helps to distinguish between client users and other users. This distinction
+streamlines access control, user monitoring, and traceability.
+
+Sometimes Ceph's user type might seem confusing, because the Ceph command line
+allows you to specify a user with or without the type, depending upon your
+command line usage. If you specify ``--user`` or ``--id``, you can omit the
+type. For example, ``client.user1`` can be entered simply as ``user1``. On the
+other hand, if you specify ``--name`` or ``-n``, you must supply the type and
+name: for example, ``client.user1``. We recommend using the type and name as a
+best practice wherever possible.
+
+.. note:: A Ceph Storage Cluster user is not the same as a Ceph Object Storage
+   user or a Ceph File System user. The Ceph Object Gateway uses a Ceph Storage
+   Cluster user to communicate between the gateway daemon and the storage
+   cluster, but the Ceph Object Gateway has its own user-management
+   functionality for end users. The Ceph File System uses POSIX semantics, and
+   the user space associated with the Ceph File System is not the same as the
+   user space associated with a Ceph Storage Cluster user.
+
+Authorization (Capabilities)
+----------------------------
+
+Ceph uses the term "capabilities" (caps) to describe the permissions granted to
+an authenticated user to exercise the functionality of the monitors, OSDs, and
+metadata servers. Capabilities can also restrict access to data within a pool,
+a namespace within a pool, or a set of pools based on their application tags.
+A Ceph administrative user specifies the capabilities of a user when creating
+or updating that user.
+
+Capability syntax follows this form::
+
+    {daemon-type} '{cap-spec}[, {cap-spec} ...]'
+
+- **Monitor Caps:** Monitor capabilities include ``r``, ``w``, ``x`` access
+  settings, and can be applied in aggregate from pre-defined profiles with
+  ``profile {name}``. For example::
+
+    mon 'allow {access-spec} [network {network/prefix}]'
+
+    mon 'profile {name}'
+
+  The ``{access-spec}`` syntax is as follows: ::
+
+        * | all | [r][w][x]
+
+  The optional ``{network/prefix}`` is a standard network name and prefix
+  length in CIDR notation (for example, ``10.3.0.0/16``).  If
+  ``{network/prefix}`` is present, the monitor capability can be used only by
+  clients that connect from the specified network.
+
+- **OSD Caps:** OSD capabilities include ``r``, ``w``, ``x``, and
+  ``class-read`` and ``class-write`` access settings. OSD capabilities can be
+  applied in aggregate from pre-defined profiles with ``profile {name}``. In
+  addition, OSD capabilities allow for pool and namespace settings. ::
+
+    osd 'allow {access-spec} [{match-spec}] [network {network/prefix}]'
+
+    osd 'profile {name} [pool={pool-name} [namespace={namespace-name}]] [network {network/prefix}]'
+
+  There are two alternative forms of the ``{access-spec}`` syntax: ::
+
+        * | all | [r][w][x] [class-read] [class-write]
+
+        class {class name} [{method name}]
+
+  There are two alternative forms of the optional ``{match-spec}`` syntax::
+
+        pool={pool-name} [namespace={namespace-name}] [object_prefix {prefix}]
+
+        [namespace={namespace-name}] tag {application} {key}={value}
+
+  The optional ``{network/prefix}`` is a standard network name and prefix
+  length in CIDR notation (for example, ``10.3.0.0/16``). If
+  ``{network/prefix}`` is present, the OSD capability can be used only by
+  clients that connect from the specified network.
+
+- **Manager Caps:** Manager (``ceph-mgr``) capabilities include ``r``, ``w``,
+  ``x`` access settings, and can be applied in aggregate from pre-defined
+  profiles with ``profile {name}``. For example::
+
+    mgr 'allow {access-spec} [network {network/prefix}]'
+
+    mgr 'profile {name} [{key1} {match-type} {value1} ...] [network {network/prefix}]'
+
+  Manager capabilities can also be specified for specific commands, for all
+  commands exported by a built-in manager service, or for all commands exported
+  by a specific add-on module. For example::
+
+        mgr 'allow command "{command-prefix}" [with {key1} {match-type} {value1} ...] [network {network/prefix}]'
+
+        mgr 'allow service {service-name} {access-spec} [network {network/prefix}]'
+
+        mgr 'allow module {module-name} [with {key1} {match-type} {value1} ...] {access-spec} [network {network/prefix}]'
+
+  The ``{access-spec}`` syntax is as follows: ::
+
+        * | all | [r][w][x]
+
+  The ``{service-name}`` is one of the following: ::
+
+        mgr | osd | pg | py
+
+  The ``{match-type}`` is one of the following: ::
+
+        = | prefix | regex
+
+- **Metadata Server Caps:** For administrators, use ``allow *``. For all other
+  users (for example, CephFS clients), consult :doc:`/cephfs/client-auth`
+
+.. note:: The Ceph Object Gateway daemon (``radosgw``) is a client of the
+          Ceph Storage Cluster. For this reason, it is not represented as 
+          a Ceph Storage Cluster daemon type.
+
+The following entries describe access capabilities.
+
+``allow``
+
+:Description: Precedes access settings for a daemon. Implies ``rw``
+              for MDS only.
+
+
+``r``
+
+:Description: Gives the user read access. Required with monitors to retrieve
+              the CRUSH map.
+
+
+``w``
+
+:Description: Gives the user write access to objects.
+
+
+``x``
+
+:Description: Gives the user the capability to call class methods
+              (that is, both read and write) and to conduct ``auth``
+              operations on monitors.
+
+
+``class-read``
+
+:Descriptions: Gives the user the capability to call class read methods.
+               Subset of ``x``.
+
+
+``class-write``
+
+:Description: Gives the user the capability to call class write methods.
+              Subset of ``x``.
+
+
+``*``, ``all``
+
+:Description: Gives the user read, write, and execute permissions for a
+              particular daemon/pool, as well as the ability to execute
+              admin commands.
+
+
+The following entries describe valid capability profiles:
+
+``profile osd`` (Monitor only)
+
+:Description: Gives a user permissions to connect as an OSD to other OSDs or
+              monitors. Conferred on OSDs in order to enable OSDs to handle replication
+              heartbeat traffic and status reporting.
+
+
+``profile mds`` (Monitor only)
+
+:Description: Gives a user permissions to connect as an MDS to other MDSs or
+              monitors.
+
+
+``profile bootstrap-osd`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an OSD. Conferred on
+              deployment tools such as ``ceph-volume`` and ``cephadm``
+              so that they have permissions to add keys when
+              bootstrapping an OSD.
+
+
+``profile bootstrap-mds`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap a metadata server.
+              Conferred on deployment tools such as ``cephadm``
+              so that they have permissions to add keys when bootstrapping
+              a metadata server.
+
+``profile bootstrap-rbd`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an RBD user.
+              Conferred on deployment tools such as ``cephadm``
+              so that they have permissions to add keys when bootstrapping
+              an RBD user.
+
+``profile bootstrap-rbd-mirror`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an ``rbd-mirror`` daemon
+              user. Conferred on deployment tools such as ``cephadm`` so that
+              they have permissions to add keys when bootstrapping an
+              ``rbd-mirror`` daemon.
+
+``profile rbd`` (Manager, Monitor, and OSD)
+
+:Description: Gives a user permissions to manipulate RBD images. When used as a
+              Monitor cap, it provides the user with the minimal privileges
+              required by an RBD client application; such privileges include
+              the ability to blocklist other client users. When used as an OSD
+              cap, it provides an RBD client application with read-write access
+              to the specified pool. The Manager cap supports optional ``pool``
+              and ``namespace`` keyword arguments.
+
+``profile rbd-mirror`` (Monitor only)
+
+:Description: Gives a user permissions to manipulate RBD images and retrieve
+              RBD mirroring config-key secrets. It provides the minimal
+              privileges required for the user to manipulate the ``rbd-mirror``
+              daemon.
+
+``profile rbd-read-only`` (Manager and OSD)
+
+:Description: Gives a user read-only permissions to RBD images. The Manager cap
+              supports optional ``pool`` and ``namespace`` keyword arguments.
+
+``profile simple-rados-client`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, and PG data.
+              Intended for use by direct librados client applications.
+
+``profile simple-rados-client-with-blocklist`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, and PG data.
+              Intended for use by direct librados client applications. Also
+              includes permissions to add blocklist entries to build
+              high-availability (HA) applications.
+
+``profile fs-client`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, PG, and MDS
+              data. Intended for CephFS clients.
+
+``profile role-definer`` (Monitor and Auth)
+
+:Description: Gives a user **all** permissions for the auth subsystem, read-only
+              access to monitors, and nothing else. Useful for automation
+              tools. Do not assign this unless you really, **really** know what
+              you're doing, as the security ramifications are substantial and
+              pervasive.
+
+``profile crash`` (Monitor and MGR)
+
+:Description: Gives a user read-only access to monitors. Used in conjunction
+              with the manager ``crash`` module to upload daemon crash
+              dumps into monitor storage for later analysis.
+
+Pool
+----
+
+A pool is a logical partition where users store data.
+In Ceph deployments, it is common to create a pool as a logical partition for
+similar types of data. For example, when deploying Ceph as a back end for
+OpenStack, a typical deployment would have pools for volumes, images, backups
+and virtual machines, and such users as ``client.glance`` and ``client.cinder``.
+
+Application Tags
+----------------
+
+Access may be restricted to specific pools as defined by their application
+metadata. The ``*`` wildcard may be used for the ``key`` argument, the
+``value`` argument, or both. The ``all`` tag is a synonym for ``*``.
+
+Namespace
+---------
+
+Objects within a pool can be associated to a namespace: that is, to a logical group of
+objects within the pool. A user's access to a pool can be associated with a
+namespace so that reads and writes by the user can take place only within the
+namespace. Objects written to a namespace within the pool can be accessed only
+by users who have access to the namespace.
+
+.. note:: Namespaces are primarily useful for applications written on top of
+   ``librados``. In such situations, the logical grouping provided by
+   namespaces  can obviate the need to create different pools. In Luminous and
+   later releases, Ceph Object Gateway uses namespaces for various metadata
+   objects.
+
+The rationale for namespaces is this: namespaces are relatively less
+computationally expensive than pools, which (pools) can be a computationally
+expensive method of segregating data sets between different authorized users.
+
+For example, a pool ought to host approximately 100 placement-group replicas
+per OSD. This means that a cluster with 1000 OSDs and three 3R replicated pools
+would have (in a single pool) 100,000 placement-group replicas, and that means
+that it has 33,333 Placement Groups.
+
+By contrast, writing an object to a namespace simply associates the namespace
+to the object name without incurring the computational overhead of a separate
+pool. Instead of creating a separate pool for a user or set of users, you can
+use a namespace. 
+
+.. note::
+
+   Namespaces are available only when using ``librados``.
+
+
+Access may be restricted to specific RADOS namespaces by use of the ``namespace``
+capability. Limited globbing of namespaces (that is, use of wildcards (``*``)) is supported: if the last character
+of the specified namespace is ``*``, then access is granted to any namespace
+starting with the provided argument.
+
+Managing Users
+==============
+
+User management functionality provides Ceph Storage Cluster administrators with
+the ability to create, update, and delete users directly in the Ceph Storage
+Cluster.
+
+When you create or delete users in the Ceph Storage Cluster, you might need to
+distribute keys to clients so that they can be added to keyrings. For details, see `Keyring
+Management`_.
+
+Listing Users
+-------------
+
+To list the users in your cluster, run the following command:
+
+.. prompt:: bash $
+
+    ceph auth ls
+
+Ceph will list all users in your cluster. For example, in a two-node
+cluster, ``ceph auth ls`` will provide an output that resembles the following::
+
+    installed auth entries:
+
+    osd.0
+        key: AQCvCbtToC6MDhAATtuT70Sl+DymPCfDSsyV4w==
+        caps: [mon] allow profile osd
+        caps: [osd] allow *
+    osd.1
+        key: AQC4CbtTCFJBChAAVq5spj0ff4eHZICxIOVZeA==
+        caps: [mon] allow profile osd
+        caps: [osd] allow *
+    client.admin
+        key: AQBHCbtT6APDHhAA5W00cBchwkQjh3dkKsyPjw==
+        caps: [mds] allow
+        caps: [mon] allow *
+        caps: [osd] allow *
+    client.bootstrap-mds
+        key: AQBICbtTOK9uGBAAdbe5zcIGHZL3T/u2g6EBww==
+        caps: [mon] allow profile bootstrap-mds
+    client.bootstrap-osd
+        key: AQBHCbtT4GxqORAADE5u7RkpCN/oo4e5W0uBtw==
+        caps: [mon] allow profile bootstrap-osd
+
+Note that, according to the ``TYPE.ID`` notation for users, ``osd.0`` is a
+user of type ``osd`` and an ID of ``0``, and ``client.admin`` is a user of type
+``client`` and an ID of ``admin`` (that is, the default ``client.admin`` user).
+Note too that each entry has a ``key: <value>`` entry, and also has one or more
+``caps:`` entries.
+
+To save the output of ``ceph auth ls`` to a file, use the ``-o {filename}`` option.
+
+
+Getting a User
+--------------
+
+To retrieve a specific user, key, and capabilities, run the following command:
+
+.. prompt:: bash $
+
+   ceph auth get {TYPE.ID}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph auth get client.admin
+
+To save the output of ``ceph auth get`` to a file, use the ``-o {filename}`` option. Developers may also run the following command:
+
+.. prompt:: bash $
+
+   ceph auth export {TYPE.ID}
+
+The ``auth export`` command is identical to ``auth get``.
+
+.. _rados_ops_adding_a_user:
+
+Adding a User
+-------------
+
+Adding a user creates a user name (that is, ``TYPE.ID``), a secret key, and
+any capabilities specified in the command that creates the user.
+
+A user's key allows the user to authenticate with the Ceph Storage Cluster.
+The user's capabilities authorize the user to read, write, or execute on Ceph
+monitors (``mon``), Ceph OSDs (``osd``) or Ceph Metadata Servers (``mds``).
+
+There are a few ways to add a user:
+
+- ``ceph auth add``: This command is the canonical way to add a user. It
+  will create the user, generate a key, and add any specified capabilities.
+
+- ``ceph auth get-or-create``: This command is often the most convenient way
+  to create a user, because it returns a keyfile format with the user name
+  (in brackets) and the key. If the user already exists, this command
+  simply returns the user name and key in the keyfile format. To save the output to
+  a file, use the ``-o {filename}`` option.
+
+- ``ceph auth get-or-create-key``: This command is a convenient way to create
+  a user and return the user's key and nothing else. This is useful for clients that
+  need only the key (for example, libvirt). If the user already exists, this command
+  simply returns the key. To save the output to
+  a file, use the ``-o {filename}`` option.
+
+It is possible, when creating client users, to create a user with no capabilities. A user
+with no capabilities is useless beyond mere authentication, because the client
+cannot retrieve the cluster map from the monitor. However, you might want to create a user
+with no capabilities and wait until later to add capabilities to the user by using the ``ceph auth caps`` comand.
+
+A typical user has at least read capabilities on the Ceph monitor and
+read and write capabilities on Ceph OSDs. A user's OSD permissions
+are often restricted so that the user can access only one particular pool.
+In the following example, the commands (1) add a client named ``john`` that has read capabilities on the Ceph monitor
+and read and write capabilities on the pool named ``liverpool``, (2) authorize a client named ``paul`` to have read capabilities on the Ceph monitor and
+read and write capabilities on the pool named ``liverpool``, (3) authorize a client named ``george`` to have read capabilities on the Ceph monitor and
+read and write capabilities on the pool named ``liverpool`` and use the keyring named ``george.keyring`` to make this authorization, and (4) authorize
+a client named ``ringo`` to have read capabilities on the Ceph monitor and read and write capabilities on the pool named ``liverpool`` and use the key
+named ``ringo.key`` to make this authorization:
+
+.. prompt:: bash $
+
+   ceph auth add client.john mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth get-or-create client.paul mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth get-or-create client.george mon 'allow r' osd 'allow rw pool=liverpool' -o george.keyring
+   ceph auth get-or-create-key client.ringo mon 'allow r' osd 'allow rw pool=liverpool' -o ringo.key
+
+.. important:: Any user that has capabilities on OSDs will have access to ALL pools in the cluster
+   unless that user's access has been restricted to a proper subset of the pools in the cluster.
+
+
+.. _modify-user-capabilities:
+
+Modifying User Capabilities
+---------------------------
+
+The ``ceph auth caps`` command allows you to specify a user and change that
+user's capabilities. Setting new capabilities will overwrite current capabilities.
+To view current capabilities, run ``ceph auth get USERTYPE.USERID``. 
+To add capabilities, run a command of the following form (and be sure to specify the existing capabilities):
+
+.. prompt:: bash $
+
+   ceph auth caps USERTYPE.USERID {daemon} 'allow [r|w|x|*|...] [pool={pool-name}] [namespace={namespace-name}]' [{daemon} 'allow [r|w|x|*|...] [pool={pool-name}] [namespace={namespace-name}]']
+
+For example:
+
+.. prompt:: bash $
+
+   ceph auth get client.john
+   ceph auth caps client.john mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth caps client.paul mon 'allow rw' osd 'allow rwx pool=liverpool'
+   ceph auth caps client.brian-manager mon 'allow *' osd 'allow *'
+
+For additional details on capabilities, see `Authorization (Capabilities)`_.
+
+Deleting a User
+---------------
+
+To delete a user, use ``ceph auth del``:
+
+.. prompt:: bash $
+
+   ceph auth del {TYPE}.{ID}
+
+Here ``{TYPE}`` is either ``client``, ``osd``, ``mon``, or ``mds``,
+and ``{ID}`` is the user name or the ID of the daemon.
+
+
+Printing a User's Key
+---------------------
+
+To print a user's authentication key to standard output, run the following command:
+
+.. prompt:: bash $
+
+   ceph auth print-key {TYPE}.{ID}
+
+Here ``{TYPE}`` is either ``client``, ``osd``, ``mon``, or ``mds``,
+and ``{ID}`` is the user name or the ID of the daemon.
+
+When it is necessary to populate client software with a user's key (as in the case of libvirt),
+you can print the user's key by running the following command:
+
+.. prompt:: bash $
+
+   mount -t ceph serverhost:/ mountpoint -o name=client.user,secret=`ceph auth print-key client.user`
+
+Importing a User
+----------------
+
+To import one or more users, use ``ceph auth import`` and
+specify a keyring as follows:
+
+.. prompt:: bash $
+
+   ceph auth import -i /path/to/keyring
+
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph auth import -i /etc/ceph/ceph.keyring
+
+.. note:: The Ceph storage cluster will add new users, their keys, and their
+   capabilities and will update existing users, their keys, and their
+   capabilities.
+
+Keyring Management
+==================
+
+When you access Ceph via a Ceph client, the Ceph client will look for a local
+keyring. Ceph presets the ``keyring`` setting with four keyring
+names by default. For this reason, you do not have to set the keyring names in your Ceph configuration file
+unless you want to override these defaults (which is not recommended). The four default keyring names are as follows:
+
+- ``/etc/ceph/$cluster.$name.keyring``
+- ``/etc/ceph/$cluster.keyring``
+- ``/etc/ceph/keyring``
+- ``/etc/ceph/keyring.bin``
+
+The ``$cluster`` metavariable found in the first two default keyring names above
+is your Ceph cluster name as defined by the name of the Ceph configuration
+file: for example, if the Ceph configuration file is named ``ceph.conf``,
+then your Ceph cluster name is ``ceph`` and the second name above would be
+``ceph.keyring``. The ``$name`` metavariable is the user type and user ID:
+for example, given the user ``client.admin``, the first name above would be
+``ceph.client.admin.keyring``.
+
+.. note:: When running commands that read or write to ``/etc/ceph``, you might
+   need to use ``sudo`` to run the command as ``root``.
+
+After you create a user (for example, ``client.ringo``), you must get the key and add
+it to a keyring on a Ceph client so that the user can access the Ceph Storage
+Cluster.
+
+The `User Management`_ section details how to list, get, add, modify, and delete
+users directly in the Ceph Storage Cluster. In addition, Ceph provides the
+``ceph-authtool`` utility to allow you to manage keyrings from a Ceph client.
+
+Creating a Keyring
+------------------
+
+When you use the procedures in the `Managing Users`_ section to create users,
+you must provide user keys to the Ceph client(s). This is required so that the Ceph client(s)
+can retrieve the key for the specified user and authenticate that user against the Ceph
+Storage Cluster. Ceph clients access keyrings in order to look up a user name and
+retrieve the user's key.
+
+The ``ceph-authtool`` utility allows you to create a keyring. To create an
+empty keyring, use ``--create-keyring`` or ``-C``. For example:
+
+.. prompt:: bash $
+
+   ceph-authtool --create-keyring /path/to/keyring
+
+When creating a keyring with multiple users, we recommend using the cluster name
+(of the form ``$cluster.keyring``) for the keyring filename and saving the keyring in the
+``/etc/ceph`` directory. By doing this, you ensure that the ``keyring`` configuration default setting
+will pick up the filename without requiring you to specify the filename in the local copy
+of your Ceph configuration file. For example, you can create ``ceph.keyring`` by
+running the following command:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -C /etc/ceph/ceph.keyring
+
+When creating a keyring with a single user, we recommend using the cluster name,
+the user type, and the user name, and saving the keyring in the ``/etc/ceph`` directory.
+For example, we recommend that the ``client.admin`` user use ``ceph.client.admin.keyring``.
+
+To create a keyring in ``/etc/ceph``, you must do so as ``root``. This means
+that the file will have ``rw`` permissions for the ``root`` user only, which is
+appropriate when the keyring contains administrator keys. However, if you
+intend to use the keyring for a particular user or group of users, be sure to use ``chown`` or ``chmod`` to establish appropriate keyring
+ownership and access.
+
+Adding a User to a Keyring
+--------------------------
+
+When you :ref:`Add a user<rados_ops_adding_a_user>` to the Ceph Storage
+Cluster, you can use the `Getting a User`_ procedure to retrieve a user, key,
+and capabilities and then save the user to a keyring.
+
+If you want to use only one user per keyring, the `Getting a User`_ procedure with
+the ``-o`` option will save the output in the keyring file format. For example,
+to create a keyring for the ``client.admin`` user, run the following command:
+
+.. prompt:: bash $
+
+   sudo ceph auth get client.admin -o /etc/ceph/ceph.client.admin.keyring
+
+Notice that the file format in this command is the file format conventionally used when manipulating the keyrings of individual users.
+
+If you want to import users to a keyring, you can use ``ceph-authtool``
+to specify the destination keyring and the source keyring.
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool /etc/ceph/ceph.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+
+Creating a User
+---------------
+
+Ceph provides the `Adding a User`_ function to create a user directly in the Ceph
+Storage Cluster. However, you can also create a user, keys, and capabilities
+directly on a Ceph client keyring, and then import the user to the Ceph
+Storage Cluster. For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.keyring
+
+For additional details on capabilities, see `Authorization (Capabilities)`_.
+
+You can also create a keyring and add a new user to the keyring simultaneously.
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -C /etc/ceph/ceph.keyring -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx' --gen-key
+
+In the above examples, the new user ``client.ringo`` has been added only to the
+keyring. The new user has not been added to the Ceph Storage Cluster.
+
+To add the new user ``client.ringo`` to the Ceph Storage Cluster, run the following command:
+
+.. prompt:: bash $
+
+   sudo ceph auth add client.ringo -i /etc/ceph/ceph.keyring
+
+Modifying a User
+----------------
+
+To modify the capabilities of a user record in a keyring, specify the keyring
+and the user, followed by the capabilities. For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool /etc/ceph/ceph.keyring -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx'
+
+To update the user in the Ceph Storage Cluster, you must update the user
+in the keyring to the user entry in the Ceph Storage Cluster. To do so, run the following command:
+
+.. prompt:: bash $
+
+   sudo ceph auth import -i /etc/ceph/ceph.keyring
+
+For details on updating a Ceph Storage Cluster user from a
+keyring, see `Importing a User`_
+
+You may also :ref:`Modify user capabilities<modify-user-capabilities>` directly in the cluster, store the
+results to a keyring file, and then import the keyring into your main
+``ceph.keyring`` file.
+
+Command Line Usage
+==================
+
+Ceph supports the following usage for user name and secret:
+
+``--id`` | ``--user``
+
+:Description: Ceph identifies users with a type and an ID: the form of this user identification is ``TYPE.ID``, and examples of the type and ID are
+              ``client.admin`` and ``client.user1``. The ``id``, ``name`` and
+              ``-n`` options allow you to specify the ID portion of the user
+              name (for example, ``admin``, ``user1``, ``foo``). You can specify
+              the user with the ``--id`` and omit the type. For example,
+              to specify user ``client.foo``, run the following commands:
+
+              .. prompt:: bash $
+
+                 ceph --id foo --keyring /path/to/keyring health
+                 ceph --user foo --keyring /path/to/keyring health
+
+
+``--name`` | ``-n``
+
+:Description: Ceph identifies users with a type and an ID: the form of this user identification is ``TYPE.ID``, and examples of the type and ID are
+              ``client.admin`` and ``client.user1``. The ``--name`` and ``-n``
+              options allow you to specify the fully qualified user name.
+              You are required to specify the user type (typically ``client``) with the
+              user ID. For example:
+
+              .. prompt:: bash $
+
+                 ceph --name client.foo --keyring /path/to/keyring health
+                 ceph -n client.foo --keyring /path/to/keyring health
+
+
+``--keyring``
+
+:Description: The path to the keyring that contains one or more user names and
+              secrets. The ``--secret`` option provides the same functionality,
+              but it does not work with Ceph RADOS Gateway, which uses
+              ``--secret`` for another purpose. You may retrieve a keyring with
+              ``ceph auth get-or-create`` and store it locally. This is a
+              preferred approach, because you can switch user names without
+              switching the keyring path. For example:
+
+              .. prompt:: bash $
+
+                 sudo rbd map --id foo --keyring /path/to/keyring mypool/myimage
+
+
+.. _pools: ../pools
+
+Limitations
+===========
+
+The ``cephx`` protocol authenticates Ceph clients and servers to each other. It
+is not intended to handle authentication of human users or application programs
+that are run on their behalf. If your access control
+needs require that kind of authentication, you will need to have some other mechanism, which is likely to be specific to the
+front end that is used to access the Ceph object store. This other mechanism would ensure that only acceptable users and programs are able to run on the
+machine that Ceph permits to access its object store.
+
+The keys used to authenticate Ceph clients and servers are typically stored in
+a plain text file on a trusted host. Appropriate permissions must be set on the plain text file.
+
+.. important:: Storing keys in plaintext files has security shortcomings, but
+   they are difficult to avoid, given the basic authentication methods Ceph
+   uses in the background. Anyone setting up Ceph systems should be aware of
+   these shortcomings.
+
+In particular, user machines, especially portable machines, should not
+be configured to interact directly with Ceph, since that mode of use would
+require the storage of a plaintext authentication key on an insecure machine.
+Anyone who stole that machine or obtained access to it could
+obtain a key that allows them to authenticate their own machines to Ceph.
+
+Instead of permitting potentially insecure machines to access a Ceph object
+store directly, you should require users to sign in to a trusted machine in
+your environment, using a method that provides sufficient security for your
+purposes. That trusted machine will store the plaintext Ceph keys for the
+human users. A future version of Ceph might address these particular
+authentication issues more fully.
+
+At present, none of the Ceph authentication protocols provide secrecy for
+messages in transit. As a result, an eavesdropper on the wire can hear and understand
+all data sent between clients and servers in Ceph, even if the eavesdropper cannot create or
+alter the data. Similarly, Ceph does not include options to encrypt user data in the
+object store. Users can, of course, hand-encrypt and store their own data in the Ceph
+object store, but Ceph itself provides no features to perform object
+encryption. Anyone storing sensitive data in Ceph should consider
+encrypting their data before providing it to the Ceph system.
+
+
+.. _Architecture - High Availability Authentication: ../../../architecture#high-availability-authentication
+.. _Cephx Config Reference: ../../configuration/auth-config-ref
diff --git a/doc/rados/troubleshooting/community.rst b/doc/rados/troubleshooting/community.rst
new file mode 100644
index 000000000..c0d7be10c
--- /dev/null
+++ b/doc/rados/troubleshooting/community.rst
@@ -0,0 +1,37 @@
+====================
+ The Ceph Community
+====================
+
+Ceph-users email list
+=====================
+
+The Ceph community is an excellent source of information and help. For
+operational issues with Ceph we recommend that you `subscribe to the ceph-users
+email list`_. When you no longer want to receive emails, you can `unsubscribe
+from the ceph-users email list`_.
+
+Ceph-devel email list
+=====================
+
+You can also `subscribe to the ceph-devel email list`_. You should do so if
+your issue is:
+
+- Likely related to a bug
+- Related to a development release package
+- Related to a development testing package
+- Related to your own builds
+
+If you no longer want to receive emails from the ``ceph-devel`` email list, you
+can `unsubscribe from the ceph-devel email list`_.
+
+Ceph report
+===========
+
+.. tip:: Community members can help you if you provide them with detailed
+   information about your problem. Attach the output of the ``ceph report``
+   command to help people understand your issues.
+
+.. _subscribe to the ceph-devel email list: mailto:dev-join@ceph.io
+.. _unsubscribe from the ceph-devel email list: mailto:dev-leave@ceph.io
+.. _subscribe to the ceph-users email list: mailto:ceph-users-join@ceph.io
+.. _unsubscribe from the ceph-users email list: mailto:ceph-users-leave@ceph.io
diff --git a/doc/rados/troubleshooting/cpu-profiling.rst b/doc/rados/troubleshooting/cpu-profiling.rst
new file mode 100644
index 000000000..b7fdd1d41
--- /dev/null
+++ b/doc/rados/troubleshooting/cpu-profiling.rst
@@ -0,0 +1,80 @@
+===============
+ CPU Profiling
+===============
+
+If you built Ceph from source and compiled Ceph for use with `oprofile`_
+you can profile Ceph's CPU usage. See `Installing Oprofile`_ for details.
+
+
+Initializing oprofile
+=====================
+
+``oprofile`` must be initalized the first time it is used. Locate the
+``vmlinux`` image that corresponds to the kernel you are running:
+
+.. prompt:: bash $
+
+   ls /boot
+   sudo opcontrol --init
+   sudo opcontrol --setup --vmlinux={path-to-image} --separate=library --callgraph=6
+
+
+Starting oprofile
+=================
+
+Run the following command to start ``oprofile``: 
+
+.. prompt:: bash $
+
+   opcontrol --start
+
+
+Stopping oprofile
+=================
+
+Run the following command to stop ``oprofile``: 
+
+.. prompt:: bash $
+
+   opcontrol --stop
+    
+    
+Retrieving oprofile Results
+===========================
+
+Run the following command to retrieve the top ``cmon`` results: 
+
+.. prompt:: bash $
+
+   opreport -gal ./cmon | less    
+    
+
+Run the following command to retrieve the top ``cmon`` results, with call
+graphs attached: 
+
+.. prompt:: bash $
+
+   opreport -cal ./cmon | less    
+    
+.. important:: After you have reviewed the results, reset ``oprofile`` before
+   running it again. The act of resetting ``oprofile`` removes data from the
+   session directory.
+
+
+Resetting oprofile
+==================
+
+Run the following command to reset ``oprofile``:  
+
+.. prompt:: bash $
+
+   sudo opcontrol --reset   
+   
+.. important:: Reset ``oprofile`` after analyzing data. This ensures that 
+   results from prior tests do not get mixed in with the results of the current
+   test. 
+
+.. _oprofile: http://oprofile.sourceforge.net/about/
+.. _Installing Oprofile: ../../../dev/cpu-profiler
+
+
diff --git a/doc/rados/troubleshooting/index.rst b/doc/rados/troubleshooting/index.rst
new file mode 100644
index 000000000..b481ee1dc
--- /dev/null
+++ b/doc/rados/troubleshooting/index.rst
@@ -0,0 +1,19 @@
+=================
+ Troubleshooting
+=================
+
+You may encounter situations that require you to examine your configuration,
+consult the documentation, modify your logging output, troubleshoot monitors
+and OSDs, profile memory and CPU usage, and, in the last resort, reach out to
+the Ceph community for help.
+
+.. toctree::
+   :maxdepth: 1
+   
+   community
+   log-and-debug
+   troubleshooting-mon
+   troubleshooting-osd
+   troubleshooting-pg
+   memory-profiling
+   cpu-profiling
diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
new file mode 100644
index 000000000..929c3f53f
--- /dev/null
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -0,0 +1,430 @@
+=======================
+ Logging and Debugging
+=======================
+
+Ceph component debug log levels can be adjusted at runtime, while services are
+running. In some circumstances you might want to adjust debug log levels in
+``ceph.conf`` or in the central config store. Increased debug logging can be
+useful if you are encountering issues when operating your cluster.  By default,
+Ceph log files are in ``/var/log/ceph``.
+
+.. tip:: Remember that debug output can slow down your system, and that this
+   latency sometimes hides race conditions.
+
+Debug logging is resource intensive. If you encounter a problem in a specific
+component of your cluster, begin troubleshooting by enabling logging for only
+that component of the cluster. For example, if your OSDs are running without
+errors, but your metadata servers are not, enable logging for any specific
+metadata server instances that are having problems. Continue by enabling
+logging for each subsystem only as needed.
+
+.. important:: Verbose logging sometimes generates over 1 GB of data per hour.
+   If the disk that your operating system runs on (your "OS disk") reaches its
+   capacity, the node associated with that disk will stop working.
+
+Whenever you enable or increase the rate of debug logging, make sure that you
+have ample capacity for log files, as this may dramatically increase their
+size.  For details on rotating log files, see `Accelerating Log Rotation`_.
+When your system is running well again, remove unnecessary debugging settings
+in order to ensure that your cluster runs optimally. Logging debug-output
+messages is a slow process and a potential waste of your cluster's resources.
+
+For details on available settings, see `Subsystem, Log and Debug Settings`_.
+
+Runtime
+=======
+
+To see the configuration settings at runtime, log in to a host that has a
+running daemon and run a command of the following form:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name} config show | less
+
+For example:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config show | less
+
+To activate Ceph's debugging output (that is, the ``dout()`` logging function)
+at runtime, inject arguments into the runtime configuration by running a ``ceph
+tell`` command of the following form:
+
+..  prompt:: bash $
+
+    ceph tell {daemon-type}.{daemon id or *} config set {name} {value}
+
+Here ``{daemon-type}`` is ``osd``, ``mon``, or ``mds``. Apply the runtime
+setting either to a specific daemon (by specifying its ID) or to all daemons of
+a particular type (by using the ``*`` operator).  For example, to increase
+debug logging for a specific ``ceph-osd`` daemon named ``osd.0``, run the
+following command:
+
+..  prompt:: bash $
+
+    ceph tell osd.0 config set debug_osd 0/5
+
+The ``ceph tell`` command goes through the monitors. However, if you are unable
+to bind to the monitor, there is another method that can be used to activate
+Ceph's debugging output: use the ``ceph daemon`` command to log in to the host
+of a specific daemon and change the daemon's configuration. For example:
+
+.. prompt:: bash $
+
+   sudo ceph daemon osd.0 config set debug_osd 0/5
+
+For details on available settings, see `Subsystem, Log and Debug Settings`_.
+
+
+Boot Time
+=========
+
+To activate Ceph's debugging output (that is, the ``dout()`` logging function)
+at boot time, you must add settings to your Ceph configuration file.
+Subsystems that are common to all daemons are set under ``[global]`` in the
+configuration file. Subsystems for a specific daemon are set under the relevant
+daemon section in the configuration file (for example, ``[mon]``, ``[osd]``,
+``[mds]``). Here is an example that shows possible debugging settings in a Ceph
+configuration file:
+
+.. code-block:: ini
+
+    [global]
+        debug_ms = 1/5
+        
+    [mon]
+        debug_mon = 20
+        debug_paxos = 1/5
+        debug_auth = 2
+         
+     [osd]
+         debug_osd = 1/5
+         debug_filestore = 1/5
+         debug_journal = 1
+         debug_monc = 5/20
+         
+    [mds]
+        debug_mds = 1
+        debug_mds_balancer = 1
+
+
+For details, see `Subsystem, Log and Debug Settings`_.
+
+
+Accelerating Log Rotation
+=========================
+
+If your log filesystem is nearly full, you can accelerate log rotation by
+modifying the Ceph log rotation file at ``/etc/logrotate.d/ceph``. To increase
+the frequency of log rotation (which will guard against a filesystem reaching
+capacity), add a ``size`` directive after the ``weekly`` frequency directive.
+To smooth out volume spikes, consider changing ``weekly`` to ``daily`` and
+consider changing ``rotate`` to ``30``. The procedure for adding the size
+setting is shown immediately below. 
+
+#. Note the default settings of the ``/etc/logrotate.d/ceph`` file::
+
+      rotate 7
+      weekly
+      compress
+      sharedscripts
+
+#. Modify them by adding a ``size`` setting::
+
+      rotate 7
+      weekly
+      size 500M
+      compress
+      sharedscripts
+
+#. Start the crontab editor for your user space:
+
+   .. prompt:: bash $
+
+      crontab -e
+
+#. Add an entry to crontab that instructs cron to check the
+   ``etc/logrotate.d/ceph`` file::
+
+      30 * * * * /usr/sbin/logrotate /etc/logrotate.d/ceph >/dev/null 2>&1
+
+In this example, the ``etc/logrotate.d/ceph`` file will be checked every 30
+minutes.
+
+Valgrind
+========
+
+When you are debugging your cluster's performance, you might find it necessary
+to track down memory and threading issues. The Valgrind tool suite can be used
+to detect problems in a specific daemon, in a particular type of daemon, or in
+the entire cluster. Because Valgrind is computationally expensive, it should be
+used only when developing or debugging Ceph, and it will slow down your system
+if used at other times. Valgrind messages are logged to ``stderr``. 
+
+
+Subsystem, Log and Debug Settings
+=================================
+
+Debug logging output is typically enabled via subsystems. 
+
+Ceph Subsystems
+---------------
+
+For each subsystem, there is a logging level for its output logs (a so-called
+"log level") and a logging level for its in-memory logs (a so-called "memory
+level"). Different values may be set for these two logging levels in each
+subsystem. Ceph's logging levels operate on a scale of ``1`` to ``20``, where
+``1`` is terse and ``20`` is verbose [#f1]_.  As a general rule, the in-memory
+logs are not sent to the output log unless one or more of the following
+conditions obtain:
+
+- a fatal signal is raised or
+- an ``assert`` in source code is triggered or
+- upon requested. Please consult `document on admin socket
+  <http://docs.ceph.com/en/latest/man/8/ceph/#daemon>`_ for more details.
+
+.. warning ::
+   .. [#f1] In certain rare cases, there are logging levels that can take a value greater than 20. The resulting logs are extremely verbose.
+
+Log levels and memory levels can be set either together or separately. If a
+subsystem is assigned a single value, then that value determines both the log
+level and the memory level. For example, ``debug ms = 5`` will give the ``ms``
+subsystem a log level of ``5`` and a memory level of ``5``.  On the other hand,
+if a subsystem is assigned two values that are separated by a forward slash
+(/), then the first value determines the log level and the second value
+determines the memory level. For example, ``debug ms = 1/5`` will give the
+``ms`` subsystem a log level of ``1`` and a memory level of ``5``. See the
+following:
+
+.. code-block:: ini 
+
+    debug {subsystem} = {log-level}/{memory-level}
+    #for example
+    debug mds balancer = 1/20
+
+The following table provides a list of Ceph subsystems and their default log and
+memory levels. Once you complete your logging efforts, restore the subsystems
+to their default level or to a level suitable for normal operations.
+
++--------------------------+-----------+--------------+
+| Subsystem                | Log Level | Memory Level |
++==========================+===========+==============+
+| ``default``              |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``lockdep``              |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``context``              |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``crush``                |     1     |      1       |
++--------------------------+-----------+--------------+
+| ``mds``                  |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``mds balancer``         |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``mds log``              |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``mds log expire``       |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``mds migrator``         |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``buffer``               |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``timer``                |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``filer``                |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``striper``              |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``objecter``             |     0     |      1       |
++--------------------------+-----------+--------------+
+| ``rados``                |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``rbd``                  |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``rbd mirror``           |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``rbd replay``           |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``rbd pwl``              |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``journaler``            |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``objectcacher``         |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``immutable obj cache``  |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``client``               |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``osd``                  |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``optracker``            |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``objclass``             |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``filestore``            |     1     |      3       |
++--------------------------+-----------+--------------+
+| ``journal``              |     1     |      3       |
++--------------------------+-----------+--------------+
+| ``ms``                   |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``mon``                  |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``monc``                 |     0     |      10      |
++--------------------------+-----------+--------------+
+| ``paxos``                |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``tp``                   |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``auth``                 |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``crypto``               |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``finisher``             |     1     |      1       |
++--------------------------+-----------+--------------+
+| ``reserver``             |     1     |      1       |
++--------------------------+-----------+--------------+
+| ``heartbeatmap``         |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``perfcounter``          |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rgw``                  |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rgw sync``             |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rgw datacache``        |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rgw access``           |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rgw dbstore``          |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``javaclient``           |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``asok``                 |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``throttle``             |     1     |      1       |
++--------------------------+-----------+--------------+
+| ``refs``                 |     0     |      0       |
++--------------------------+-----------+--------------+
+| ``compressor``           |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``bluestore``            |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``bluefs``               |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``bdev``                 |     1     |      3       |
++--------------------------+-----------+--------------+
+| ``kstore``               |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``rocksdb``              |     4     |      5       |
++--------------------------+-----------+--------------+
+| ``leveldb``              |     4     |      5       |
++--------------------------+-----------+--------------+
+| ``fuse``                 |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``mgr``                  |     2     |      5       |
++--------------------------+-----------+--------------+
+| ``mgrc``                 |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``dpdk``                 |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``eventtrace``           |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``prioritycache``        |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``test``                 |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``cephfs mirror``        |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``cepgsqlite``           |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore``             |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore onode``       |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore odata``       |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore ompap``       |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore tm``          |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore t``           |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore cleaner``     |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore epm``         |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore lba``         |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore fixedkv tree``|     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore cache``       |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore journal``     |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore device``      |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``seastore backref``     |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``alienstore``           |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``mclock``               |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``cyanstore``            |     0     |      5       |
++--------------------------+-----------+--------------+
+| ``ceph exporter``        |     1     |      5       |
++--------------------------+-----------+--------------+
+| ``memstore``             |     1     |      5       |
++--------------------------+-----------+--------------+
+
+
+Logging Settings
+----------------
+
+It is not necessary to specify logging and debugging settings in the Ceph
+configuration file, but you may override default settings when needed. Ceph
+supports the following settings:
+
+.. confval:: log_file
+.. confval:: log_max_new
+.. confval:: log_max_recent
+.. confval:: log_to_file
+.. confval:: log_to_stderr
+.. confval:: err_to_stderr
+.. confval:: log_to_syslog
+.. confval:: err_to_syslog
+.. confval:: log_flush_on_exit
+.. confval:: clog_to_monitors
+.. confval:: clog_to_syslog
+.. confval:: mon_cluster_log_to_syslog
+.. confval:: mon_cluster_log_file
+
+OSD
+---
+
+.. confval:: osd_debug_drop_ping_probability
+.. confval:: osd_debug_drop_ping_duration
+
+Filestore
+---------
+
+.. confval:: filestore_debug_omap_check
+
+MDS
+---
+
+- :confval:`mds_debug_scatterstat`
+- :confval:`mds_debug_frag`
+- :confval:`mds_debug_auth_pins`
+- :confval:`mds_debug_subtrees`
+
+RADOS Gateway
+-------------
+
+- :confval:`rgw_log_nonexistent_bucket`
+- :confval:`rgw_log_object_name`
+- :confval:`rgw_log_object_name_utc`
+- :confval:`rgw_enable_ops_log`
+- :confval:`rgw_enable_usage_log`
+- :confval:`rgw_usage_log_flush_threshold`
+- :confval:`rgw_usage_log_tick_interval`
diff --git a/doc/rados/troubleshooting/memory-profiling.rst b/doc/rados/troubleshooting/memory-profiling.rst
new file mode 100644
index 000000000..8e58f2d76
--- /dev/null
+++ b/doc/rados/troubleshooting/memory-profiling.rst
@@ -0,0 +1,203 @@
+==================
+ Memory Profiling
+==================
+
+Ceph Monitor, OSD, and MDS can report ``TCMalloc`` heap profiles. Install
+``google-perftools`` if you want to generate these. Your OS distribution might
+package this under a different name (for example, ``gperftools``), and your OS
+distribution might use a different package manager. Run a command similar to
+this one to install ``google-perftools``: 
+
+.. prompt:: bash 
+
+    sudo apt-get install google-perftools
+
+The profiler dumps output to your ``log file`` directory (``/var/log/ceph``).
+See `Logging and Debugging`_ for details.
+
+To view the profiler logs with Google's performance tools, run the following
+command:
+
+.. prompt:: bash
+
+    google-pprof --text {path-to-daemon}  {log-path/filename}
+
+For example::
+
+    $ ceph tell osd.0 heap start_profiler
+    $ ceph tell osd.0 heap dump
+    osd.0 tcmalloc heap stats:------------------------------------------------
+    MALLOC:        2632288 (    2.5 MiB) Bytes in use by application
+    MALLOC: +       499712 (    0.5 MiB) Bytes in page heap freelist
+    MALLOC: +       543800 (    0.5 MiB) Bytes in central cache freelist
+    MALLOC: +       327680 (    0.3 MiB) Bytes in transfer cache freelist
+    MALLOC: +      1239400 (    1.2 MiB) Bytes in thread cache freelists
+    MALLOC: +      1142936 (    1.1 MiB) Bytes in malloc metadata
+    MALLOC:   ------------
+    MALLOC: =      6385816 (    6.1 MiB) Actual memory used (physical + swap)
+    MALLOC: +            0 (    0.0 MiB) Bytes released to OS (aka unmapped)
+    MALLOC:   ------------
+    MALLOC: =      6385816 (    6.1 MiB) Virtual address space used
+    MALLOC:
+    MALLOC:            231              Spans in use
+    MALLOC:             56              Thread heaps in use
+    MALLOC:           8192              Tcmalloc page size
+    ------------------------------------------------
+    Call ReleaseFreeMemory() to release freelist memory to the OS (via madvise()).
+    Bytes released to the OS take up virtual address space but no physical memory.
+    $ google-pprof --text \
+                   /usr/bin/ceph-osd  \
+                   /var/log/ceph/ceph-osd.0.profile.0001.heap
+     Total: 3.7 MB
+     1.9  51.1%  51.1%      1.9  51.1% ceph::log::Log::create_entry
+     1.8  47.3%  98.4%      1.8  47.3% std::string::_Rep::_S_create
+     0.0   0.4%  98.9%      0.0   0.6% SimpleMessenger::add_accept_pipe
+     0.0   0.4%  99.2%      0.0   0.6% decode_message
+     ...
+
+Performing another heap dump on the same daemon creates another file. It is
+convenient to compare the new file to a file created by a previous heap dump to
+show what has grown in the interval. For example::
+
+    $ google-pprof --text --base out/osd.0.profile.0001.heap \
+          ceph-osd out/osd.0.profile.0003.heap
+     Total: 0.2 MB
+     0.1  50.3%  50.3%      0.1  50.3% ceph::log::Log::create_entry
+     0.1  46.6%  96.8%      0.1  46.6% std::string::_Rep::_S_create
+     0.0   0.9%  97.7%      0.0  26.1% ReplicatedPG::do_op
+     0.0   0.8%  98.5%      0.0   0.8% __gnu_cxx::new_allocator::allocate
+
+See `Google Heap Profiler`_ for additional details.
+
+After you have installed the heap profiler, start your cluster and begin using
+the heap profiler. You can enable or disable the heap profiler at runtime, or
+ensure that it runs continuously. When running commands based on the examples
+that follow, do the following:
+
+#. replace ``{daemon-type}`` with ``mon``, ``osd`` or ``mds`` 
+#. replace ``{daemon-id}`` with the OSD number or the MON ID or the MDS ID 
+
+
+Starting the Profiler
+---------------------
+
+To start the heap profiler, run a command of the following form: 
+
+.. prompt:: bash
+
+   ceph tell {daemon-type}.{daemon-id} heap start_profiler
+
+For example:
+
+.. prompt:: bash
+
+   ceph tell osd.1 heap start_profiler
+
+Alternatively, if the ``CEPH_HEAP_PROFILER_INIT=true`` variable is found in the
+environment, the profile will be started when the daemon starts running.
+
+Printing Stats
+--------------
+
+To print out statistics, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph  tell {daemon-type}.{daemon-id} heap stats
+
+For example:
+
+.. prompt:: bash
+
+   ceph tell osd.0 heap stats
+
+.. note:: The reporting of stats with this command does not require the
+   profiler to be running and does not dump the heap allocation information to
+   a file.
+
+
+Dumping Heap Information
+------------------------
+
+To dump heap information, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph tell {daemon-type}.{daemon-id} heap dump
+
+For example:
+
+.. prompt:: bash
+
+   ceph tell mds.a heap dump
+
+.. note:: Dumping heap information works only when the profiler is running.
+
+
+Releasing Memory
+----------------
+
+To release memory that ``tcmalloc`` has allocated but which is not being used
+by the Ceph daemon itself, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph tell {daemon-type}{daemon-id} heap release
+
+For example:
+
+.. prompt:: bash
+
+    ceph tell osd.2 heap release
+
+
+Stopping the Profiler
+---------------------
+
+To stop the heap profiler, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph tell {daemon-type}.{daemon-id} heap stop_profiler
+
+For example:
+
+.. prompt:: bash
+
+   ceph tell osd.0 heap stop_profiler
+
+.. _Logging and Debugging: ../log-and-debug
+.. _Google Heap Profiler: http://goog-perftools.sourceforge.net/doc/heap_profiler.html
+
+Alternative Methods of  Memory Profiling
+----------------------------------------
+
+Running Massif heap profiler with Valgrind
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Massif heap profiler tool can be used with Valgrind to measure how much
+heap memory is used. This method is well-suited to troubleshooting RadosGW.
+
+See the `Massif documentation
+<https://valgrind.org/docs/manual/ms-manual.html>`_ for more information.
+
+Install Valgrind from the package manager for your distribution then start the
+Ceph daemon you want to troubleshoot:
+
+.. prompt:: bash
+
+   sudo -u ceph valgrind --max-threads=1024 --tool=massif /usr/bin/radosgw -f --cluster ceph --name NAME --setuser ceph --setgroup ceph
+
+When this command has completed its run, a file with a name of the form
+``massif.out.<pid>`` will be saved in your current working directory. To run
+the command above, the user who runs it must have write permissions in the
+current directory.
+
+Run the ``ms_print`` command to get a graph and statistics from the collected
+data in the ``massif.out.<pid>`` file:
+
+.. prompt:: bash
+
+   ms_print massif.out.12345
+
+The output of this command is helpful when submitting a bug report.
diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
new file mode 100644
index 000000000..1170da7c3
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -0,0 +1,713 @@
+.. _rados-troubleshooting-mon:
+
+==========================
+ Troubleshooting Monitors
+==========================
+
+.. index:: monitor, high availability
+
+Even if a cluster experiences monitor-related problems, the cluster is not
+necessarily in danger of going down. If a cluster has lost multiple monitors,
+it can still remain up and running as long as there are enough surviving
+monitors to form a quorum.
+   
+If your cluster is having monitor-related problems, we recommend that you
+consult the following troubleshooting information.
+
+Initial Troubleshooting
+=======================
+
+The first steps in the process of troubleshooting Ceph Monitors involve making
+sure that the Monitors are running and that they are able to communicate with
+the network and on the network. Follow the steps in this section to rule out
+the simplest causes of Monitor malfunction.
+
+#. **Make sure that the Monitors are running.**
+
+    Make sure that the Monitor (*mon*) daemon processes (``ceph-mon``) are
+    running. It might be the case that the mons have not be restarted after an
+    upgrade. Checking for this simple oversight can save hours of painstaking
+    troubleshooting. 
+    
+    It is also important to make sure that the manager daemons (``ceph-mgr``)
+    are running. Remember that typical cluster configurations provide one
+    Manager (``ceph-mgr``) for each Monitor (``ceph-mon``).
+
+    .. note:: In releases prior to v1.12.5, Rook will not run more than two
+       managers.
+
+#. **Make sure that you can reach the Monitor nodes.**
+
+    In certain rare cases, ``iptables`` rules might be blocking access to
+    Monitor nodes or TCP ports. These rules might be left over from earlier
+    stress testing or rule development. To check for the presence of such
+    rules, SSH into each Monitor node and use ``telnet`` or ``nc`` or a similar
+    tool to attempt to connect to each of the other Monitor nodes on ports
+    ``tcp/3300`` and ``tcp/6789``. 
+
+#. **Make sure that the "ceph status" command runs and receives a reply from the cluster.**
+
+    If the ``ceph status`` command receives a reply from the cluster, then the
+    cluster is up and running. Monitors answer to a ``status`` request only if
+    there is a formed quorum. Confirm that one or more ``mgr`` daemons are
+    reported as running. In a cluster with no deficiencies, ``ceph status``
+    will report that all ``mgr`` daemons are running.
+
+    If the ``ceph status`` command does not receive a reply from the cluster,
+    then there are probably not enough Monitors ``up`` to form a quorum. If the
+    ``ceph -s`` command is run with no further options specified, it connects
+    to an arbitrarily selected Monitor. In certain cases, however, it might be
+    helpful to connect to a specific Monitor (or to several specific Monitors
+    in sequence) by adding the ``-m`` flag to the command: for example, ``ceph
+    status -m mymon1``.
+
+#. **None of this worked. What now?**
+
+    If the above solutions have not resolved your problems, you might find it
+    helpful to examine each individual Monitor in turn. Even if no quorum has
+    been formed, it is possible to contact each Monitor individually and
+    request its status by using the ``ceph tell mon.ID mon_status`` command
+    (here ``ID`` is the Monitor's identifier).
+
+    Run the ``ceph tell mon.ID mon_status`` command for each Monitor in the
+    cluster. For more on this command's output, see :ref:`Understanding
+    mon_status
+    <rados_troubleshoting_troubleshooting_mon_understanding_mon_status>`.
+
+    There is also an alternative method for contacting each individual Monitor:
+    SSH into each Monitor node and query the daemon's admin socket. See
+    :ref:`Using the Monitor's Admin
+    Socket<rados_troubleshoting_troubleshooting_mon_using_admin_socket>`.
+
+.. _rados_troubleshoting_troubleshooting_mon_using_admin_socket:
+
+Using the monitor's admin socket
+================================
+
+A monitor's admin socket allows you to interact directly with a specific daemon
+by using a Unix socket file. This file is found in the monitor's ``run``
+directory. The admin socket's default directory is
+``/var/run/ceph/ceph-mon.ID.asok``, but this can be overridden and the admin
+socket might be elsewhere, especially if your cluster's daemons are deployed in
+containers. If you cannot find it, either check your ``ceph.conf`` for an
+alternative path or run the following command:
+    
+.. prompt:: bash $
+
+   ceph-conf --name mon.ID --show-config-value admin_socket
+
+The admin socket is available for use only when the monitor daemon is running.
+Whenever the monitor has been properly shut down, the admin socket is removed.
+However, if the monitor is not running and the admin socket persists, it is
+likely that the monitor has been improperly shut down.  In any case, if the
+monitor is not running, it will be impossible to use the admin socket, and the
+``ceph`` command is likely to return ``Error 111: Connection Refused``.
+
+To access the admin socket, run a ``ceph tell`` command of the following form
+(specifying the daemon that you are interested in):
+
+.. prompt:: bash $
+
+   ceph tell mon.<id> mon_status
+
+This command passes a ``help`` command to the specific running monitor daemon
+``<id>`` via its admin socket. If you know the full path to the admin socket
+file, this can be done more directly by running the following command:
+
+.. prompt:: bash $
+
+   ceph --admin-daemon <full_path_to_asok_file> <command>
+
+Running ``ceph help`` shows all supported commands that are available through
+the admin socket. See especially ``config get``, ``config show``, ``mon stat``,
+and ``quorum_status``.
+
+.. _rados_troubleshoting_troubleshooting_mon_understanding_mon_status:
+
+Understanding mon_status
+========================
+
+The status of the monitor (as reported by the ``ceph tell mon.X mon_status``
+command) can always be obtained via the admin socket. This command outputs a
+great deal of information about the monitor (including the information found in
+the output of the ``quorum_status`` command).
+
+To understand this command's output, let us consider the following example, in
+which we see the output of ``ceph tell mon.c mon_status``::
+
+  { "name": "c",
+    "rank": 2,
+    "state": "peon",
+    "election_epoch": 38,
+    "quorum": [
+          1,
+          2],
+    "outside_quorum": [],
+    "extra_probe_peers": [],
+    "sync_provider": [],
+    "monmap": { "epoch": 3,
+        "fsid": "5c4e9d53-e2e1-478a-8061-f543f8be4cf8",
+        "modified": "2013-10-30 04:12:01.945629",
+        "created": "2013-10-29 14:14:41.914786",
+        "mons": [
+              { "rank": 0,
+                "name": "a",
+                "addr": "127.0.0.1:6789\/0"},
+              { "rank": 1,
+                "name": "b",
+                "addr": "127.0.0.1:6790\/0"},
+              { "rank": 2,
+                "name": "c",
+                "addr": "127.0.0.1:6795\/0"}]}}
+
+It is clear that there are three monitors in the monmap (*a*, *b*, and *c*),
+the quorum is formed by only two monitors, and *c* is in the quorum as a
+*peon*.
+
+**Which monitor is out of the quorum?**
+
+  The answer is **a** (that is, ``mon.a``).
+
+**Why?**
+
+  When the ``quorum`` set is examined, there are clearly two monitors in the
+  set: *1* and *2*. But these are not monitor names. They are monitor ranks, as
+  established in the current ``monmap``. The ``quorum`` set does not include
+  the monitor that has rank 0, and according to the ``monmap`` that monitor is
+  ``mon.a``.
+
+**How are monitor ranks determined?**
+
+  Monitor ranks are calculated (or recalculated) whenever monitors are added or
+  removed. The calculation of ranks follows a simple rule: the **greater** the
+  ``IP:PORT`` combination, the **lower** the rank. In this case, because
+  ``127.0.0.1:6789`` is lower than the other two ``IP:PORT`` combinations,
+  ``mon.a`` has the highest rank: namely, rank 0.
+  
+
+Most Common Monitor Issues
+===========================
+
+The Cluster Has Quorum but at Least One Monitor is Down
+-------------------------------------------------------
+
+When the cluster has quorum but at least one monitor is down, ``ceph health
+detail`` returns a message similar to the following::
+
+      $ ceph health detail
+      [snip]
+      mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum)
+
+**How do I troubleshoot a Ceph cluster that has quorum but also has at least one monitor down?**
+
+  #. Make sure that ``mon.a`` is running.
+
+  #. Make sure that you can connect to ``mon.a``'s node from the
+     other Monitor nodes. Check the TCP ports as well. Check ``iptables`` and
+     ``nf_conntrack`` on all nodes and make sure that you are not
+     dropping/rejecting connections.
+
+  If this initial troubleshooting doesn't solve your problem, then further
+  investigation is necessary.
+
+  First, check the problematic monitor's ``mon_status`` via the admin
+  socket as explained in `Using the monitor's admin socket`_ and
+  `Understanding mon_status`_.
+
+  If the Monitor is out of the quorum, then its state will be one of the
+  following: ``probing``, ``electing`` or ``synchronizing``. If the state of
+  the Monitor is ``leader`` or ``peon``, then the Monitor believes itself to be
+  in quorum but the rest of the cluster believes that it is not in quorum. It
+  is possible that a Monitor that is in one of the ``probing``, ``electing``,
+  or ``synchronizing`` states has entered the quorum during the process of
+  troubleshooting. Check ``ceph status`` again to determine whether the Monitor
+  has entered quorum during your troubleshooting. If the Monitor remains out of
+  the quorum, then proceed with the investigations described in this section of
+  the documentation.
+  
+
+**What does it mean when a Monitor's state is ``probing``?**
+
+  If ``ceph health detail`` shows that a Monitor's state is
+  ``probing``, then the Monitor is still looking for the other Monitors. Every
+  Monitor remains in this state for some time when it is started. When a
+  Monitor has connected to the other Monitors specified in the ``monmap``, it
+  ceases to be in the ``probing`` state. The amount of time that a Monitor is
+  in the ``probing`` state depends upon the parameters of the cluster of which
+  it is a part. For example, when a Monitor is a part of a single-monitor
+  cluster (never do this in production), the monitor passes through the probing
+  state almost instantaneously. In a multi-monitor cluster, the Monitors stay
+  in the ``probing`` state until they find enough monitors to form a quorum
+  |---| this means that if two out of three Monitors in the cluster are
+  ``down``, the one remaining Monitor stays in the ``probing``  state
+  indefinitely until you bring one of the other monitors up.
+
+  If quorum has been established, then the Monitor daemon should be able to
+  find the other Monitors quickly, as long as they can be reached. If a Monitor
+  is stuck in the ``probing`` state and you have exhausted the procedures above
+  that describe the troubleshooting of communications between the Monitors,
+  then it is possible that the problem Monitor is trying to reach the other
+  Monitors at a wrong address. ``mon_status`` outputs the ``monmap`` that is
+  known to the monitor: determine whether the other Monitors' locations as
+  specified in the ``monmap`` match the locations of the Monitors in the
+  network. If they do not, see `Recovering a Monitor's Broken monmap`_.
+  If the locations of the Monitors as specified in the ``monmap`` match the
+  locations of the Monitors in the network, then the persistent
+  ``probing`` state could  be related to severe clock skews amongst the monitor
+  nodes.  See `Clock Skews`_.  If the information in `Clock Skews`_ does not
+  bring the Monitor out of the ``probing`` state, then prepare your system logs
+  and ask the Ceph community for help. See `Preparing your logs`_ for
+  information about the proper preparation of logs.
+
+
+**What does it mean when a Monitor's state is ``electing``?**
+
+  If ``ceph health detail`` shows that a Monitor's state is ``electing``, the
+  monitor is in the middle of an election. Elections typically complete
+  quickly, but sometimes the monitors can get stuck in what is known as an
+  *election storm*. See :ref:`Monitor Elections <dev_mon_elections>` for more
+  on monitor elections.
+  
+  The presence of election storm might indicate clock skew among the monitor
+  nodes. See `Clock Skews`_ for more information. 
+  
+  If your clocks are properly synchronized, search the mailing lists and bug
+  tracker for issues similar to your issue. The ``electing`` state is not
+  likely to persist. In versions of Ceph after the release of Cuttlefish, there
+  is no obvious reason other than clock skew that explains why an ``electing``
+  state would persist.  
+  
+  It is possible to investigate the cause of a persistent ``electing`` state if
+  you put the problematic Monitor into a ``down`` state while you investigate.
+  This is possible only if there are enough surviving Monitors to form quorum. 
+
+**What does it mean when a Monitor's state is ``synchronizing``?**
+
+  If ``ceph health detail`` shows that the Monitor is ``synchronizing``, the
+  monitor is catching up with the rest of the cluster so that it can join the
+  quorum. The amount of time that it takes for the Monitor to synchronize with
+  the rest of the quorum is a function of the size of the cluster's monitor
+  store, the cluster's size, and the state of the cluster. Larger and degraded
+  clusters generally keep Monitors in the ``synchronizing`` state longer than
+  do smaller, new clusters.
+
+  A Monitor that changes its state from ``synchronizing`` to ``electing`` and
+  then back to ``synchronizing`` indicates a problem: the cluster state may be
+  advancing (that is, generating new maps) too fast for the synchronization
+  process to keep up with the pace of the creation of the new maps. This issue
+  presented more frequently prior to the Cuttlefish release than it does in
+  more recent releases, because the synchronization process has since been
+  refactored and enhanced to avoid this dynamic. If you experience this in
+  later versions, report the issue in the `Ceph bug tracker
+  <https://tracker.ceph.com>`_. Prepare and provide logs to substantiate any
+  bug you raise. See `Preparing your logs`_ for information about the proper
+  preparation of logs.
+
+**What does it mean when a Monitor's state is ``leader`` or ``peon``?**
+
+  If ``ceph health detail`` shows that the Monitor is in the ``leader`` state
+  or in the ``peon`` state, it is likely that clock skew is present. Follow the
+  instructions in `Clock Skews`_. If you have followed those instructions and
+  ``ceph health detail`` still shows that the Monitor is in the ``leader``
+  state or the ``peon`` state, report the issue in the `Ceph bug tracker
+  <https://tracker.ceph.com>`_. If you raise an issue, provide logs to
+  substantiate it. See `Preparing your logs`_ for information about the
+  proper preparation of logs.
+
+
+Recovering a Monitor's Broken ``monmap``
+----------------------------------------
+
+This is how a ``monmap`` usually looks, depending on the number of
+monitors::
+
+
+      epoch 3
+      fsid 5c4e9d53-e2e1-478a-8061-f543f8be4cf8
+      last_changed 2013-10-30 04:12:01.945629
+      created 2013-10-29 14:14:41.914786
+      0: 127.0.0.1:6789/0 mon.a
+      1: 127.0.0.1:6790/0 mon.b
+      2: 127.0.0.1:6795/0 mon.c
+      
+This may not be what you have however. For instance, in some versions of
+early Cuttlefish there was a bug that could cause your ``monmap``
+to be nullified.  Completely filled with zeros. This means that not even
+``monmaptool`` would be able to make sense of cold, hard, inscrutable zeros.
+It's also possible to end up with a monitor with a severely outdated monmap,
+notably if the node has been down for months while you fight with your vendor's
+TAC.  The subject ``ceph-mon`` daemon might be unable to find the surviving
+monitors (e.g., say ``mon.c`` is down; you add a new monitor ``mon.d``,
+then remove ``mon.a``, then add a new monitor ``mon.e`` and remove
+``mon.b``; you will end up with a totally different monmap from the one
+``mon.c`` knows).
+
+In this situation you have two possible solutions:
+
+Scrap the monitor and redeploy
+
+  You should only take this route if you are positive that you won't
+  lose the information kept by that monitor; that you have other monitors
+  and that they are running just fine so that your new monitor is able
+  to synchronize from the remaining monitors. Keep in mind that destroying
+  a monitor, if there are no other copies of its contents, may lead to
+  loss of data.
+
+Inject a monmap into the monitor
+
+  These are the basic steps:
+
+  Retrieve the ``monmap`` from the surviving monitors and inject it into the
+  monitor whose ``monmap`` is corrupted or lost.
+
+  Implement this solution by carrying out the following procedure:
+
+  1. Is there a quorum of monitors? If so, retrieve the ``monmap`` from the
+     quorum::
+
+      $ ceph mon getmap -o /tmp/monmap
+
+  2. If there is no quorum, then retrieve the ``monmap`` directly from another
+     monitor that has been stopped (in this example, the other monitor has
+     the ID ``ID-FOO``)::
+
+      $ ceph-mon -i ID-FOO --extract-monmap /tmp/monmap
+
+  3. Stop the monitor you are going to inject the monmap into.
+
+  4. Inject the monmap::
+
+      $ ceph-mon -i ID --inject-monmap /tmp/monmap
+
+  5. Start the monitor
+
+  .. warning:: Injecting ``monmaps`` can cause serious problems because doing
+     so will overwrite the latest existing ``monmap`` stored on the monitor. Be
+     careful!
+
+Clock Skews
+-----------
+
+The Paxos consensus algorithm requires close time synchroniziation, which means
+that clock skew among the monitors in the quorum can have a serious effect on
+monitor operation. The resulting behavior can be puzzling. To avoid this issue,
+run a clock synchronization tool on your monitor nodes: for example, use
+``Chrony`` or the legacy ``ntpd`` utility. Configure each monitor nodes so that
+the `iburst` option is in effect and so that each monitor has multiple peers,
+including the following: 
+
+* Each other
+* Internal ``NTP`` servers
+* Multiple external, public pool servers
+
+.. note:: The ``iburst`` option sends a burst of eight packets instead of the
+   usual single packet, and is used during the process of getting two peers
+   into initial synchronization.
+
+Furthermore, it is advisable to synchronize *all* nodes in your cluster against
+internal and external servers, and perhaps even against your monitors. Run
+``NTP`` servers on bare metal: VM-virtualized clocks are not suitable for
+steady timekeeping. See `https://www.ntp.org <https://www.ntp.org>`_ for more
+information about the Network Time Protocol (NTP). Your organization might
+already have quality internal ``NTP`` servers available.  Sources for ``NTP``
+server appliances include the following:
+
+* Microsemi (formerly Symmetricom) `https://microsemi.com <https://www.microsemi.com/product-directory/3425-timing-synchronization>`_
+* EndRun `https://endruntechnologies.com <https://endruntechnologies.com/products/ntp-time-servers>`_
+* Netburner `https://www.netburner.com <https://www.netburner.com/products/network-time-server/pk70-ex-ntp-network-time-server>`_
+
+Clock Skew Questions and Answers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**What's the maximum tolerated clock skew?**
+
+  By default, monitors allow clocks to drift up to a maximum of 0.05 seconds
+  (50 milliseconds).
+
+**Can I increase the maximum tolerated clock skew?**
+
+  Yes, but we strongly recommend against doing so. The maximum tolerated clock
+  skew is configurable via the ``mon-clock-drift-allowed`` option, but it is
+  almost certainly a bad idea to make changes to this option. The clock skew
+  maximum is in place because clock-skewed monitors cannot be relied upon. The
+  current default value has proven its worth at alerting the user before the
+  monitors encounter serious problems. Changing this value might cause
+  unforeseen effects on the stability of the monitors and overall cluster
+  health.
+
+**How do I know whether there is a clock skew?**
+
+  The monitors will warn you via the cluster status ``HEALTH_WARN``. When clock
+  skew is present, the ``ceph health detail`` and ``ceph status`` commands
+  return an output resembling the following::
+
+      mon.c addr 10.10.0.1:6789/0 clock skew 0.08235s > max 0.05s (latency 0.0045s)
+
+  In this example, the monitor ``mon.c`` has been flagged as suffering from 
+  clock skew.
+
+  In Luminous and later releases, it is possible to check for a clock skew by
+  running the ``ceph time-sync-status`` command. Note that the lead monitor
+  typically has the numerically lowest IP address. It will always show ``0``:
+  the reported offsets of other monitors are relative to the lead monitor, not
+  to any external reference source.
+
+**What should I do if there is a clock skew?**
+
+  Synchronize your clocks. Using an NTP client might help. However, if you
+  are already using an NTP client and you still encounter clock skew problems,
+  determine whether the NTP server that you are using is remote to your network
+  or instead hosted on your network. Hosting your own NTP servers tends to
+  mitigate clock skew problems.
+
+
+Client Can't Connect or Mount
+-----------------------------
+
+Check your IP tables. Some operating-system install utilities add a ``REJECT``
+rule to ``iptables``. ``iptables`` rules will reject all clients other than
+``ssh`` that try to connect to the host. If your monitor host's IP tables have
+a ``REJECT`` rule in place, clients that are connecting from a separate node
+will fail and will raise a timeout error. Any ``iptables`` rules that reject
+clients trying to connect to Ceph daemons must be addressed. For example::
+
+    REJECT all -- anywhere anywhere reject-with icmp-host-prohibited
+
+It might also be necessary to add rules to iptables on your Ceph hosts to
+ensure that clients are able to access the TCP ports associated with your Ceph
+monitors (default: port 6789) and Ceph OSDs (default: 6800 through 7300). For
+example::
+
+    iptables -A INPUT -m multiport -p tcp -s {ip-address}/{netmask} --dports 6789,6800:7300 -j ACCEPT
+
+
+Monitor Store Failures
+======================
+
+Symptoms of store corruption
+----------------------------
+
+Ceph monitors store the :term:`Cluster Map` in a key-value store.  If key-value
+store corruption causes a monitor to fail, then the monitor log might contain
+one of the following error messages::
+
+  Corruption: error in middle of record
+
+or::
+
+  Corruption: 1 missing files; e.g.: /var/lib/ceph/mon/mon.foo/store.db/1234567.ldb
+
+Recovery using healthy monitor(s)
+---------------------------------
+
+If there are surviving monitors, we can always :ref:`replace
+<adding-and-removing-monitors>` the corrupted monitor with a new one. After the
+new monitor boots, it will synchronize with a healthy peer. After the new
+monitor is fully synchronized, it will be able to serve clients.
+
+.. _mon-store-recovery-using-osds:
+
+Recovery using OSDs
+-------------------
+
+Even if all monitors fail at the same time, it is possible to recover the
+monitor store by using information stored in OSDs. You are encouraged to deploy
+at least three (and preferably five) monitors in a Ceph cluster. In such a
+deployment, complete monitor failure is unlikely. However, unplanned power loss
+in a data center whose disk settings or filesystem settings are improperly
+configured could cause the underlying filesystem to fail and this could kill
+all of the monitors. In such a case, data in the OSDs can be used to recover
+the monitors.  The following is such a script and can be used to recover the
+monitors:
+
+
+.. code-block:: bash
+
+  ms=/root/mon-store
+  mkdir $ms
+  
+  # collect the cluster map from stopped OSDs
+  for host in $hosts; do
+    rsync -avz $ms/. user@$host:$ms.remote
+    rm -rf $ms
+    ssh user@$host <<EOF
+      for osd in /var/lib/ceph/osd/ceph-*; do
+        ceph-objectstore-tool --data-path \$osd --no-mon-config --op update-mon-db --mon-store-path $ms.remote
+      done
+  EOF
+    rsync -avz user@$host:$ms.remote/. $ms
+  done
+  
+  # rebuild the monitor store from the collected map, if the cluster does not
+  # use cephx authentication, we can skip the following steps to update the
+  # keyring with the caps, and there is no need to pass the "--keyring" option.
+  # i.e. just use "ceph-monstore-tool $ms rebuild" instead
+  ceph-authtool /path/to/admin.keyring -n mon. \
+    --cap mon 'allow *'
+  ceph-authtool /path/to/admin.keyring -n client.admin \
+    --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'
+  # add one or more ceph-mgr's key to the keyring. in this case, an encoded key
+  # for mgr.x is added, you can find the encoded key in
+  # /etc/ceph/${cluster}.${mgr_name}.keyring on the machine where ceph-mgr is
+  # deployed
+  ceph-authtool /path/to/admin.keyring --add-key 'AQDN8kBe9PLWARAAZwxXMr+n85SBYbSlLcZnMA==' -n mgr.x \
+    --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'
+  # If your monitors' ids are not sorted by ip address, please specify them in order.
+  # For example. if mon 'a' is 10.0.0.3, mon 'b' is 10.0.0.2, and mon 'c' is  10.0.0.4,
+  # please passing "--mon-ids b a c".
+  # In addition, if your monitors' ids are not single characters like 'a', 'b', 'c', please
+  # specify them in the command line by passing them as arguments of the "--mon-ids"
+  # option. if you are not sure, please check your ceph.conf to see if there is any
+  # sections named like '[mon.foo]'. don't pass the "--mon-ids" option, if you are
+  # using DNS SRV for looking up monitors.
+  ceph-monstore-tool $ms rebuild -- --keyring /path/to/admin.keyring --mon-ids alpha beta gamma
+  
+  # make a backup of the corrupted store.db just in case!  repeat for
+  # all monitors.
+  mv /var/lib/ceph/mon/mon.foo/store.db /var/lib/ceph/mon/mon.foo/store.db.corrupted
+
+  # move rebuild store.db into place.  repeat for all monitors.
+  mv $ms/store.db /var/lib/ceph/mon/mon.foo/store.db
+  chown -R ceph:ceph /var/lib/ceph/mon/mon.foo/store.db
+
+This script performs the following steps:
+
+#. Collects the map from each OSD host.
+#. Rebuilds the store.
+#. Fills the entities in the keyring file with appropriate capabilities.
+#. Replaces the corrupted store on ``mon.foo`` with the recovered copy.
+
+
+Known limitations
+~~~~~~~~~~~~~~~~~
+
+The above recovery tool is unable to recover the following information:
+
+- **Certain added keyrings**: All of the OSD keyrings added using the ``ceph
+  auth add`` command are recovered from the OSD's copy, and the
+  ``client.admin`` keyring is imported using ``ceph-monstore-tool``. However,
+  the MDS keyrings and all other keyrings will be missing in the recovered
+  monitor store. You might need to manually re-add them.
+
+- **Creating pools**: If any RADOS pools were in the process of being created,
+  that state is lost. The recovery tool operates on the assumption that all
+  pools have already been created. If there are PGs that are stuck in the
+  'unknown' state after the recovery for a partially created pool, you can
+  force creation of the *empty* PG by running the ``ceph osd force-create-pg``
+  command. Note that this will create an *empty* PG, so take this action only
+  if you know the pool is empty.
+
+- **MDS Maps**: The MDS maps are lost.
+
+
+Everything Failed! Now What?
+============================
+
+Reaching out for help
+---------------------
+
+You can find help on IRC in #ceph and #ceph-devel on OFTC (server
+irc.oftc.net), or at ``dev@ceph.io`` and ``ceph-users@lists.ceph.com``. Make
+sure that you have prepared your logs and that you have them ready upon
+request.
+
+See https://ceph.io/en/community/connect/ for current (as of October 2023)
+information on getting in contact with the upstream Ceph community.
+
+
+Preparing your logs
+-------------------
+
+The default location for monitor logs is ``/var/log/ceph/ceph-mon.FOO.log*``.
+However, if they are not there, you can find their current location by running
+the following command:
+
+.. prompt:: bash
+
+   ceph-conf --name mon.FOO --show-config-value log_file
+
+The amount of information in the logs is determined by the debug levels in the
+cluster's configuration files. If Ceph is using the default debug levels, then
+your logs might be missing important information that would help the upstream
+Ceph community address your issue.
+
+To make sure your monitor logs contain relevant information, you can raise
+debug levels. Here we are interested in information from the monitors.  As with
+other components, the monitors have different parts that output their debug
+information on different subsystems.
+
+If you are an experienced Ceph troubleshooter, we recommend raising the debug
+levels of the most relevant subsystems. Of course, this approach might not be
+easy for beginners. In most cases, however, enough information to address the
+issue will be secured if the following debug levels are entered::
+
+      debug_mon = 10
+      debug_ms = 1
+
+Sometimes these debug levels do not yield enough information. In such cases,
+members of the upstream Ceph community might ask you to make additional changes
+to these or to other debug levels. In any case, it is better for us to receive
+at least some useful information than to receive an empty log.
+
+
+Do I need to restart a monitor to adjust debug levels?
+------------------------------------------------------
+
+No, restarting a monitor is not necessary. Debug levels may be adjusted by
+using two different methods, depending on whether or not there is a quorum:
+
+There is a quorum
+
+  Either inject the debug option into the specific monitor that needs to 
+  be debugged::
+
+        ceph tell mon.FOO config set debug_mon 10/10
+
+  Or inject it into all monitors at once::
+
+        ceph tell mon.* config set debug_mon 10/10
+
+
+There is no quorum
+
+  Use the admin socket of the specific monitor that needs to be debugged
+  and directly adjust the monitor's configuration options::
+
+      ceph daemon mon.FOO config set debug_mon 10/10
+
+
+To return the debug levels to their default values, run the above commands
+using the debug level ``1/10`` rather than ``10/10``. To check a monitor's
+current values, use the admin socket and run either of the following commands:
+
+  .. prompt:: bash
+
+     ceph daemon mon.FOO config show
+
+or:
+
+  .. prompt:: bash
+
+     ceph daemon mon.FOO config get 'OPTION_NAME'
+
+
+
+I Reproduced the problem with appropriate debug levels. Now what?
+-----------------------------------------------------------------
+
+We prefer that you send us only the portions of your logs that are relevant to
+your monitor problems. Of course, it might not be easy for you to determine
+which portions are relevant so we are willing to accept complete and
+unabridged logs. However, we request that you avoid sending logs containing
+hundreds of thousands of lines with no additional clarifying information. One
+common-sense way of making our task easier is to write down the current time
+and date when you are reproducing the problem and then extract portions of your
+logs based on that information.
+
+Finally, reach out to us on the mailing lists or IRC or Slack, or by filing a
+new issue on the `tracker`_.
+
+.. _tracker: http://tracker.ceph.com/projects/ceph/issues/new
+
+.. |---|   unicode:: U+2014 .. EM DASH
+   :trim:
diff --git a/doc/rados/troubleshooting/troubleshooting-osd.rst b/doc/rados/troubleshooting/troubleshooting-osd.rst
new file mode 100644
index 000000000..035947d7e
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-osd.rst
@@ -0,0 +1,787 @@
+======================
+ Troubleshooting OSDs
+======================
+
+Before troubleshooting the cluster's OSDs, check the monitors
+and the network. 
+
+First, determine whether the monitors have a quorum. Run the ``ceph health``
+command or the ``ceph -s`` command and if Ceph shows ``HEALTH_OK`` then there
+is a monitor quorum. 
+
+If the monitors don't have a quorum or if there are errors with the monitor
+status, address the monitor issues before proceeding by consulting the material
+in `Troubleshooting Monitors <../troubleshooting-mon>`_.
+
+Next, check your networks to make sure that they are running properly. Networks
+can have a significant impact on OSD operation and performance. Look for
+dropped packets on the host side and CRC errors on the switch side.
+
+
+Obtaining Data About OSDs
+=========================
+
+When troubleshooting OSDs, it is useful to collect different kinds of
+information about the OSDs. Some information comes from the practice of
+`monitoring OSDs`_ (for example, by running the ``ceph osd tree`` command).
+Additional information concerns the topology of your cluster, and is discussed
+in the following sections.
+
+
+Ceph Logs
+---------
+
+Ceph log files are stored under ``/var/log/ceph``. Unless the path has been
+changed (or you are in a containerized environment that stores logs in a
+different location), the log files can be listed by running the following
+command:
+
+.. prompt:: bash
+
+   ls /var/log/ceph
+
+If there is not enough log detail, change the logging level. To ensure that
+Ceph performs adequately under high logging volume, see `Logging and
+Debugging`_.
+
+
+
+Admin Socket
+------------
+
+Use the admin socket tool to retrieve runtime information. First, list the
+sockets of Ceph's daemons by running the following command:
+
+.. prompt:: bash
+
+   ls /var/run/ceph
+
+Next, run a command of the following form (replacing ``{daemon-name}`` with the
+name of a specific daemon: for example, ``osd.0``):
+
+.. prompt:: bash
+
+   ceph daemon {daemon-name} help
+
+Alternatively, run the command with a ``{socket-file}`` specified (a "socket
+file" is a specific file in ``/var/run/ceph``):
+
+.. prompt:: bash
+
+   ceph daemon {socket-file} help
+
+The admin socket makes many tasks possible, including:
+
+- Listing Ceph configuration at runtime
+- Dumping historic operations
+- Dumping the operation priority queue state
+- Dumping operations in flight
+- Dumping perfcounters
+
+Display Free Space
+------------------
+
+Filesystem issues may arise. To display your filesystems' free space, run the
+following command:
+
+.. prompt:: bash
+
+   df -h
+
+To see this command's supported syntax and options, run ``df --help``.
+
+I/O Statistics
+--------------
+
+The `iostat`_ tool can be used to identify I/O-related issues. Run the
+following command:
+
+.. prompt:: bash
+
+   iostat -x
+
+
+Diagnostic Messages
+-------------------
+
+To retrieve diagnostic messages from the kernel, run the ``dmesg`` command and
+specify the output with ``less``, ``more``, ``grep``, or ``tail``. For
+example: 
+
+.. prompt:: bash
+
+    dmesg | grep scsi
+
+Stopping without Rebalancing
+============================
+
+It might be occasionally necessary to perform maintenance on a subset of your
+cluster or to resolve a problem that affects a failure domain (for example, a
+rack).  However, when you stop OSDs for maintenance, you might want to prevent
+CRUSH from automatically rebalancing the cluster. To avert this rebalancing
+behavior, set the cluster to ``noout`` by running the following command:
+
+.. prompt:: bash
+
+   ceph osd set noout
+
+.. warning:: This is more a thought exercise offered for the purpose of giving
+   the reader a sense of failure domains and CRUSH behavior than a suggestion
+   that anyone in the post-Luminous world run ``ceph osd set noout``. When the
+   OSDs return to an ``up`` state, rebalancing will resume and the change
+   introduced by the ``ceph osd set noout`` command will be reverted.
+
+In Luminous and later releases, however, it is a safer approach to flag only
+affected OSDs.  To add or remove a ``noout`` flag to a specific OSD, run a
+command like the following:
+
+.. prompt:: bash
+
+   ceph osd add-noout osd.0
+   ceph osd rm-noout  osd.0
+
+It is also possible to flag an entire CRUSH bucket. For example, if you plan to
+take down ``prod-ceph-data1701`` in order to add RAM, you might run the
+following command:
+
+.. prompt:: bash
+
+   ceph osd set-group noout prod-ceph-data1701
+
+After the flag is set, stop the OSDs and any other colocated
+Ceph services within the failure domain that requires maintenance work::
+
+   systemctl stop ceph\*.service ceph\*.target
+
+.. note:: When an OSD is stopped, any placement groups within the OSD are
+   marked as ``degraded``.
+
+After the maintenance is complete, it will be necessary to restart the OSDs
+and any other daemons that have stopped. However, if the host was rebooted as
+part of the maintenance, they do not need to be restarted and will come back up
+automatically. To restart OSDs or other daemons, use a command of the following
+form:
+
+.. prompt:: bash
+
+   sudo systemctl start ceph.target
+
+Finally, unset the ``noout`` flag as needed by running commands like the
+following:
+
+.. prompt:: bash
+
+   ceph osd unset noout
+   ceph osd unset-group noout prod-ceph-data1701
+
+Many contemporary Linux distributions employ ``systemd`` for service
+management.  However, for certain operating systems (especially older ones) it
+might be necessary to issue equivalent ``service`` or ``start``/``stop``
+commands.
+
+
+.. _osd-not-running:
+
+OSD Not Running
+===============
+
+Under normal conditions, restarting a ``ceph-osd`` daemon will allow it to
+rejoin the cluster and recover.
+
+
+An OSD Won't Start
+------------------
+
+If the cluster has started but an OSD isn't starting, check the following:
+
+- **Configuration File:** If you were not able to get OSDs running from a new
+  installation, check your configuration file to ensure it conforms to the
+  standard (for example, make sure that it says ``host`` and not ``hostname``,
+  etc.).
+
+- **Check Paths:** Ensure that the paths specified in the configuration
+  correspond to the paths for data and metadata that actually exist (for
+  example, the paths to the journals, the WAL, and the DB). Separate the OSD
+  data from the metadata in order to see whether there are errors in the
+  configuration file and in the actual mounts. If so, these errors might
+  explain why OSDs are not starting. To store the metadata on a separate block
+  device, partition or LVM the drive and assign one partition per OSD.
+
+- **Check Max Threadcount:** If the cluster has a node with an especially high
+  number of OSDs, it might be hitting the default maximum number of threads
+  (usually 32,000).  This is especially likely to happen during recovery.
+  Increasing the maximum number of threads to the maximum possible number of
+  threads allowed (4194303) might help with the problem. To increase the number
+  of threads to the maximum, run the following command:
+
+  .. prompt:: bash
+
+     sysctl -w kernel.pid_max=4194303
+
+  If this increase resolves the issue, you must make the increase permanent by
+  including a ``kernel.pid_max`` setting either in a file under
+  ``/etc/sysctl.d`` or within the master ``/etc/sysctl.conf`` file. For
+  example::
+
+     kernel.pid_max = 4194303
+
+- **Check ``nf_conntrack``:** This connection-tracking and connection-limiting
+  system causes problems for many production Ceph clusters. The problems often
+  emerge slowly and subtly. As cluster topology and client workload grow,
+  mysterious and intermittent connection failures and performance glitches
+  occur more and more, especially at certain times of the day. To begin taking
+  the measure of your problem, check the ``syslog`` history for "table full"
+  events. One way to address this kind of problem is as follows: First, use the
+  ``sysctl`` utility to assign ``nf_conntrack_max`` a much higher value. Next,
+  raise the value of ``nf_conntrack_buckets`` so that ``nf_conntrack_buckets``
+  × 8 = ``nf_conntrack_max``; this action might require running commands
+  outside of ``sysctl`` (for example, ``"echo 131072 >
+  /sys/module/nf_conntrack/parameters/hashsize``). Another way to address the
+  problem is to blacklist the associated kernel modules in order to disable
+  processing altogether. This approach is powerful, but fragile. The modules
+  and the order in which the modules must be listed can vary among kernel
+  versions. Even when blacklisted, ``iptables`` and ``docker`` might sometimes
+  activate connection tracking anyway, so we advise a "set and forget" strategy
+  for the tunables. On modern systems, this approach will not consume
+  appreciable resources.
+
+- **Kernel Version:** Identify the kernel version and distribution that are in
+  use. By default, Ceph uses third-party tools that might be buggy or come into
+  conflict with certain distributions or kernel versions (for example, Google's
+  ``gperftools`` and ``TCMalloc``). Check the `OS recommendations`_ and the
+  release notes for each Ceph version in order to make sure that you have
+  addressed any issues related to your kernel.
+
+- **Segment Fault:** If there is a segment fault, increase log levels and
+  restart the problematic daemon(s). If segment faults recur, search the Ceph
+  bug tracker `https://tracker.ceph/com/projects/ceph
+  <https://tracker.ceph.com/projects/ceph/>`_ and the ``dev`` and
+  ``ceph-users`` mailing list archives `https://ceph.io/resources
+  <https://ceph.io/resources>`_ to see if others have experienced and reported
+  these issues. If this truly is a new and unique failure, post to the ``dev``
+  email list and provide the following information: the specific Ceph release
+  being run, ``ceph.conf`` (with secrets XXX'd out), your monitor status
+  output, and excerpts from your log file(s).
+
+
+An OSD Failed
+-------------
+
+When an OSD fails, this means that a ``ceph-osd`` process is unresponsive or
+has died and that the corresponding OSD has been marked ``down``. Surviving
+``ceph-osd`` daemons will report to the monitors that the OSD appears to be
+down, and a new status will be visible in the output of the ``ceph health``
+command, as in the following example:
+
+.. prompt:: bash
+
+   ceph health
+
+::
+
+   HEALTH_WARN 1/3 in osds are down
+
+This health alert is raised whenever there are one or more OSDs marked ``in``
+and ``down``. To see which OSDs are ``down``, add ``detail`` to the command as in
+the following example:
+
+.. prompt:: bash
+
+   ceph health detail
+
+::
+
+   HEALTH_WARN 1/3 in osds are down
+   osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
+
+Alternatively, run the following command:
+
+.. prompt:: bash
+
+    ceph osd tree down
+
+If there is a drive failure or another fault that is preventing a given
+``ceph-osd`` daemon from functioning or restarting, then there should be an
+error message present in its log file under ``/var/log/ceph``.
+
+If the ``ceph-osd`` daemon stopped because of a heartbeat failure or a
+``suicide timeout`` error, then the underlying drive or filesystem might be
+unresponsive. Check ``dmesg`` output and `syslog`  output for drive errors or
+kernel errors. It might be necessary to specify certain flags (for example,
+``dmesg -T`` to see human-readable timestamps) in order to avoid mistaking old
+errors for new errors.
+
+If an entire host's OSDs are ``down``, check to see if there is a network
+error or a hardware issue with the host.
+
+If the OSD problem is the result of a software error (for example, a failed
+assertion or another unexpected error), search for reports of the issue in the
+`bug tracker <https://tracker.ceph/com/projects/ceph>`_ , the `dev mailing list
+archives <https://lists.ceph.io/hyperkitty/list/dev@ceph.io/>`_, and the
+`ceph-users mailing list archives
+<https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/>`_.  If there is no
+clear fix or existing bug, then :ref:`report the problem to the ceph-devel
+email list <Get Involved>`.
+
+
+.. _no-free-drive-space:
+
+No Free Drive Space
+-------------------
+
+If an OSD is full, Ceph prevents data loss by ensuring that no new data is
+written to the OSD. In an properly running cluster, health checks are raised
+when the cluster's OSDs and pools approach certain "fullness" ratios. The
+``mon_osd_full_ratio`` threshold defaults to ``0.95`` (or 95% of capacity):
+this is the point above which clients are prevented from writing data. The
+``mon_osd_backfillfull_ratio`` threshold defaults to ``0.90`` (or 90% of
+capacity): this is the point above which backfills will not start. The
+``mon_osd_nearfull_ratio`` threshold defaults to ``0.85`` (or 85% of capacity):
+this is the point at which it raises the ``OSD_NEARFULL`` health check.
+
+OSDs within a cluster will vary in how much data is allocated to them by Ceph.
+To check "fullness" by displaying data utilization for every OSD, run the
+following command:
+
+.. prompt:: bash
+
+   ceph osd df
+
+To check "fullness" by displaying a cluster’s overall data usage and data
+distribution among pools, run the following command:
+
+.. prompt:: bash
+
+   ceph df 
+
+When examining the output of the ``ceph df`` command, pay special attention to
+the **most full** OSDs, as opposed to the percentage of raw space used. If a
+single outlier OSD becomes full, all writes to this OSD's pool might fail as a
+result. When ``ceph df`` reports the space available to a pool, it considers
+the ratio settings relative to the *most full* OSD that is part of the pool. To
+flatten the distribution, two approaches are available: (1) Using the
+``reweight-by-utilization`` command to progressively move data from excessively
+full OSDs or move data to insufficiently full OSDs, and (2) in later revisions
+of Luminous and subsequent releases, exploiting the ``ceph-mgr`` ``balancer``
+module to perform the same task automatically.
+
+To adjust the "fullness" ratios, run a command or commands of the following
+form:
+
+.. prompt:: bash
+
+   ceph osd set-nearfull-ratio <float[0.0-1.0]>
+   ceph osd set-full-ratio <float[0.0-1.0]>
+   ceph osd set-backfillfull-ratio <float[0.0-1.0]>
+
+Sometimes full cluster issues arise because an OSD has failed. This can happen
+either because of a test or because the cluster is small, very full, or
+unbalanced. When an OSD or node holds an excessive percentage of the cluster's
+data, component failures or natural growth can result in the ``nearfull`` and
+``full`` ratios being exceeded.  When testing Ceph's resilience to OSD failures
+on a small cluster, it is advised to leave ample free disk space and to
+consider temporarily lowering the OSD ``full ratio``, OSD ``backfillfull
+ratio``, and OSD ``nearfull ratio``.
+
+The "fullness" status of OSDs is visible in the output of the ``ceph health``
+command, as in the following example:
+
+.. prompt:: bash
+
+   ceph health
+
+::
+
+  HEALTH_WARN 1 nearfull osd(s)
+
+For details, add the ``detail`` command as in the following example:
+
+.. prompt:: bash
+
+    ceph health detail
+
+::
+
+    HEALTH_ERR 1 full osd(s); 1 backfillfull osd(s); 1 nearfull osd(s)
+    osd.3 is full at 97%
+    osd.4 is backfill full at 91%
+    osd.2 is near full at 87%
+
+To address full cluster issues, it is recommended to add capacity by adding
+OSDs. Adding new OSDs allows the cluster to redistribute data to newly
+available storage. Search for ``rados bench`` orphans that are wasting space.
+
+If a legacy Filestore OSD cannot be started because it is full, it is possible
+to reclaim space by deleting a small number of placement group directories in
+the full OSD.
+
+.. important:: If you choose to delete a placement group directory on a full
+   OSD, **DO NOT** delete the same placement group directory on another full
+   OSD. **OTHERWISE YOU WILL LOSE DATA**. You **MUST** maintain at least one
+   copy of your data on at least one OSD. Deleting placement group directories
+   is a rare and extreme intervention. It is not to be undertaken lightly.
+
+See `Monitor Config Reference`_ for more information.
+
+
+OSDs are Slow/Unresponsive
+==========================
+
+OSDs are sometimes slow or unresponsive. When troubleshooting this common
+problem, it is advised to eliminate other possibilities before investigating
+OSD performance issues. For example, be sure to confirm that your network(s)
+are working properly, to verify that your OSDs are running, and to check
+whether OSDs are throttling recovery traffic.
+
+.. tip:: In pre-Luminous releases of Ceph, ``up`` and ``in`` OSDs were
+   sometimes not available or were otherwise slow because recovering OSDs were
+   consuming system resources. Newer releases provide better recovery handling
+   by preventing this phenomenon.
+
+
+Networking Issues
+-----------------
+
+As a distributed storage system, Ceph relies upon networks for OSD peering and
+replication, recovery from faults, and periodic heartbeats. Networking issues
+can cause OSD latency and flapping OSDs. For more information, see `Flapping
+OSDs`_.
+
+To make sure that Ceph processes and Ceph-dependent processes are connected and
+listening, run the following commands:
+
+.. prompt:: bash
+
+   netstat -a | grep ceph
+   netstat -l | grep ceph
+   sudo netstat -p | grep ceph
+
+To check network statistics, run the following command:
+
+.. prompt:: bash
+
+   netstat -s
+
+Drive Configuration
+-------------------
+
+An SAS or SATA storage drive should house only one OSD, but a NVMe drive can
+easily house two or more. However, it is possible for read and write throughput
+to bottleneck if other processes share the drive. Such processes include:
+journals / metadata, operating systems, Ceph monitors, ``syslog`` logs, other
+OSDs, and non-Ceph processes.
+
+Because Ceph acknowledges writes *after* journaling, fast SSDs are an
+attractive option for accelerating response time -- particularly when using the
+``XFS`` or ``ext4`` filesystems for legacy FileStore OSDs.  By contrast, the
+``Btrfs`` file system can write and journal simultaneously. (However, use of
+``Btrfs`` is not recommended for production deployments.)
+
+.. note:: Partitioning a drive does not change its total throughput or
+   sequential read/write limits. Throughput might be improved somewhat by
+   running a journal in a separate partition, but it is better still to run
+   such a journal in a separate physical drive.
+   
+.. warning:: Reef does not support FileStore. Releases after Reef do not
+   support FileStore. Any information that mentions FileStore is pertinent only
+   to the Quincy release of Ceph and to releases prior to Quincy.
+
+
+Bad Sectors / Fragmented Disk
+-----------------------------
+
+Check your drives for bad blocks, fragmentation, and other errors that can
+cause significantly degraded performance. Tools that are useful in checking for
+drive errors include ``dmesg``, ``syslog`` logs, and ``smartctl`` (found in the
+``smartmontools`` package).
+
+.. note:: ``smartmontools`` 7.0 and late provides NVMe stat passthrough and
+   JSON output.
+
+
+Co-resident Monitors/OSDs
+-------------------------
+
+Although monitors are relatively lightweight processes, performance issues can
+result when monitors are run on the same host machine as an OSD. Monitors issue
+many ``fsync()`` calls and this can interfere with other workloads. The danger
+of performance issues is especially acute when the monitors are co-resident on
+the same storage drive as an OSD. In addition, if the monitors are running an
+older kernel (pre-3.0) or a kernel with no ``syncfs(2)`` syscall, then multiple
+OSDs running on the same host might make so many commits as to undermine each
+other's performance.  This problem sometimes results in what is called "the
+bursty writes".
+
+
+Co-resident Processes
+---------------------
+
+Significant OSD latency can result from processes that write data to Ceph (for
+example, cloud-based solutions and virtual machines) while operating on the
+same hardware as OSDs. For this reason, making such processes co-resident with
+OSDs is not generally recommended. Instead, the recommended practice is to
+optimize certain hosts for use with Ceph and use other hosts for other
+processes. This practice of separating Ceph operations from other applications
+might help improve performance and might also streamline troubleshooting and
+maintenance.
+
+Running co-resident processes on the same hardware is sometimes called
+"convergence". When using Ceph, engage in convergence only with expertise and
+after consideration.
+
+
+Logging Levels
+--------------
+
+Performance issues can result from high logging levels. Operators sometimes
+raise logging levels in order to track an issue and then forget to lower them
+afterwards. In such a situation, OSDs might consume valuable system resources to
+write needlessly verbose logs onto the disk. Anyone who does want to use high logging
+levels is advised to consider mounting a drive to the default path for logging
+(for example, ``/var/log/ceph/$cluster-$name.log``).
+
+Recovery Throttling
+-------------------
+
+Depending upon your configuration, Ceph may reduce recovery rates to maintain
+client or OSD performance, or it may increase recovery rates to the point that
+recovery impacts client or OSD performance. Check to see if the client or OSD
+is recovering.
+
+
+Kernel Version
+--------------
+
+Check the kernel version that you are running. Older kernels may lack updates
+that improve Ceph performance. 
+
+
+Kernel Issues with SyncFS
+-------------------------
+
+If you have kernel issues with SyncFS, try running one OSD per host to see if
+performance improves. Old kernels might not have a recent enough version of
+``glibc`` to support ``syncfs(2)``.
+
+
+Filesystem Issues
+-----------------
+
+In post-Luminous releases, we recommend deploying clusters with the BlueStore
+back end.  When running a pre-Luminous release, or if you have a specific
+reason to deploy OSDs with the previous Filestore backend, we recommend
+``XFS``.
+
+We recommend against using ``Btrfs`` or ``ext4``.  The ``Btrfs`` filesystem has
+many attractive features, but bugs may lead to performance issues and spurious
+ENOSPC errors.  We do not recommend ``ext4`` for Filestore OSDs because
+``xattr`` limitations break support for long object names, which are needed for
+RGW.
+
+For more information, see `Filesystem Recommendations`_.
+
+.. _Filesystem Recommendations: ../configuration/filesystem-recommendations
+
+Insufficient RAM
+----------------
+
+We recommend a *minimum* of 4GB of RAM per OSD daemon and we suggest rounding
+up from 6GB to 8GB. During normal operations, you may notice that ``ceph-osd``
+processes use only a fraction of that amount.  You might be tempted to use the
+excess RAM for co-resident applications or to skimp on each node's memory
+capacity. However, when OSDs experience recovery their memory utilization
+spikes. If there is insufficient RAM available during recovery, OSD performance
+will slow considerably and the daemons may even crash or be killed by the Linux
+``OOM Killer``.
+
+
+Blocked Requests or Slow Requests
+---------------------------------
+
+When a ``ceph-osd`` daemon is slow to respond to a request, the cluster log
+receives messages reporting ops that are taking too long. The warning threshold
+defaults to 30 seconds and is configurable via the ``osd_op_complaint_time``
+setting.
+
+Legacy versions of Ceph complain about ``old requests``::
+
+    osd.0 192.168.106.220:6800/18813 312 : [WRN] old request osd_op(client.5099.0:790 fatty_26485_object789 [write 0~4096] 2.5e54f643) v4 received at 2012-03-06 15:42:56.054801 currently waiting for sub ops
+
+Newer versions of Ceph complain about ``slow requests``::
+
+    {date} {osd.num} [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.005692 secs
+    {date} {osd.num}  [WRN] slow request 30.005692 seconds old, received at {date-time}: osd_op(client.4240.0:8 benchmark_data_ceph-1_39426_object7 [write 0~4194304] 0.69848840) v4 currently waiting for subops from [610]
+
+Possible causes include:
+
+- A failing drive (check ``dmesg`` output)
+- A bug in the kernel file system (check ``dmesg`` output)
+- An overloaded cluster (check system load, iostat, etc.)
+- A bug in the ``ceph-osd`` daemon.
+
+Possible solutions:
+
+- Remove VMs from Ceph hosts
+- Upgrade kernel
+- Upgrade Ceph
+- Restart OSDs
+- Replace failed or failing components
+
+Debugging Slow Requests
+-----------------------
+
+If you run ``ceph daemon osd.<id> dump_historic_ops`` or ``ceph daemon osd.<id>
+dump_ops_in_flight``, you will see a set of operations and a list of events
+each operation went through. These are briefly described below.
+
+Events from the Messenger layer:
+
+- ``header_read``: The time that the messenger first started reading the message off the wire.
+- ``throttled``: The time that the messenger tried to acquire memory throttle space to read
+  the message into memory.
+- ``all_read``: The time that the messenger finished reading the message off the wire.
+- ``dispatched``: The time that the messenger gave the message to the OSD.
+- ``initiated``: This is identical to ``header_read``. The existence of both is a
+  historical oddity.
+
+Events from the OSD as it processes ops:
+
+- ``queued_for_pg``: The op has been put into the queue for processing by its PG.
+- ``reached_pg``: The PG has started performing the op.
+- ``waiting for \*``: The op is waiting for some other work to complete before
+  it can proceed (for example, a new OSDMap; the scrubbing of its object
+  target; the completion of a PG's peering; all as specified in the message).
+- ``started``: The op has been accepted as something the OSD should do and 
+  is now being performed.
+- ``waiting for subops from``: The op has been sent to replica OSDs.
+
+Events from ```Filestore```:
+
+- ``commit_queued_for_journal_write``: The op has been given to the FileStore.
+- ``write_thread_in_journal_buffer``: The op is in the journal's buffer and is waiting
+  to be persisted (as the next disk write).
+- ``journaled_completion_queued``: The op was journaled to disk and its callback
+  has been queued for invocation.
+
+Events from the OSD after data has been given to underlying storage:
+
+- ``op_commit``: The op has been committed (that is, written to journal) by the
+  primary OSD.
+- ``op_applied``: The op has been `write()'en
+  <https://www.freebsd.org/cgi/man.cgi?write(2)>`_ to the backing FS (that is,
+  applied in memory but not flushed out to disk) on the primary.
+- ``sub_op_applied``: ``op_applied``, but for a replica's "subop".
+- ``sub_op_committed``: ``op_commit``, but for a replica's subop (only for EC pools).
+- ``sub_op_commit_rec/sub_op_apply_rec from <X>``: The primary marks this when it
+  hears about the above, but for a particular replica (i.e. ``<X>``).
+- ``commit_sent``: We sent a reply back to the client (or primary OSD, for sub ops).
+
+Some of these events may appear redundant, but they cross important boundaries
+in the internal code (such as passing data across locks into new threads).
+
+
+Flapping OSDs
+=============
+
+"Flapping" is the term for the phenomenon of an OSD being repeatedly marked
+``up`` and then ``down`` in rapid succession.  This section explains how to
+recognize flapping, and how to mitigate it.
+
+When OSDs peer and check heartbeats, they use the cluster (back-end) network
+when it is available. See `Monitor/OSD Interaction`_ for details.
+
+The upstream Ceph community has traditionally recommended separate *public*
+(front-end) and *private* (cluster / back-end / replication) networks. This
+provides the following benefits:
+
+#. Segregation of (1) heartbeat traffic and replication/recovery traffic
+   (private) from (2) traffic from clients and between OSDs and monitors
+   (public). This helps keep one stream of traffic from DoS-ing the other,
+   which could in turn result in a cascading failure.
+
+#. Additional throughput for both public and private traffic.
+
+In the past, when common networking technologies were measured in a range
+encompassing 100Mb/s and 1Gb/s, this separation was often critical. But with
+today's 10Gb/s, 40Gb/s, and 25/50/100Gb/s networks, the above capacity concerns
+are often diminished or even obviated.  For example, if your OSD nodes have two
+network ports, dedicating one to the public and the other to the private
+network means that you have no path redundancy.  This degrades your ability to
+endure network maintenance and network failures without significant cluster or
+client impact. In situations like this, consider instead using both links for
+only a public network: with bonding (LACP) or equal-cost routing (for example,
+FRR) you reap the benefits of increased throughput headroom, fault tolerance,
+and reduced OSD flapping.
+
+When a private network (or even a single host link) fails or degrades while the
+public network continues operating normally, OSDs may not handle this situation
+well. In such situations, OSDs use the public network to report each other
+``down`` to the monitors, while marking themselves ``up``. The monitors then
+send out-- again on the public network--an updated cluster map with the
+affected OSDs marked `down`. These OSDs reply to the monitors "I'm not dead
+yet!", and the cycle repeats. We call this scenario 'flapping`, and it can be
+difficult to isolate and remediate. Without a private network, this irksome
+dynamic is avoided: OSDs are generally either ``up`` or ``down`` without
+flapping.
+
+If something does cause OSDs to 'flap' (repeatedly being marked ``down`` and
+then ``up`` again), you can force the monitors to halt the flapping by
+temporarily freezing their states:
+
+.. prompt:: bash
+
+   ceph osd set noup      # prevent OSDs from getting marked up
+   ceph osd set nodown    # prevent OSDs from getting marked down
+
+These flags are recorded in the osdmap:
+
+.. prompt:: bash
+
+   ceph osd dump | grep flags
+
+::
+
+   flags no-up,no-down
+
+You can clear these flags with:
+
+.. prompt:: bash
+
+   ceph osd unset noup
+   ceph osd unset nodown
+
+Two other flags are available, ``noin`` and ``noout``, which prevent booting
+OSDs from being marked ``in`` (allocated data) or protect OSDs from eventually
+being marked ``out`` (regardless of the current value of
+``mon_osd_down_out_interval``).
+
+.. note:: ``noup``, ``noout``, and ``nodown`` are temporary in the sense that
+   after the flags are cleared, the action that they were blocking should be
+   possible shortly thereafter. But the ``noin`` flag prevents OSDs from being
+   marked ``in`` on boot, and any daemons that started while the flag was set
+   will remain that way.
+
+.. note:: The causes and effects of flapping can be mitigated somewhat by
+   making careful adjustments to ``mon_osd_down_out_subtree_limit``,
+   ``mon_osd_reporter_subtree_level``, and ``mon_osd_min_down_reporters``.
+   Derivation of optimal settings depends on cluster size, topology, and the
+   Ceph release in use. The interaction of all of these factors is subtle and
+   is beyond the scope of this document.
+
+
+.. _iostat: https://en.wikipedia.org/wiki/Iostat
+.. _Ceph Logging and Debugging: ../../configuration/ceph-conf#ceph-logging-and-debugging
+.. _Logging and Debugging: ../log-and-debug
+.. _Debugging and Logging: ../debug
+.. _Monitor/OSD Interaction: ../../configuration/mon-osd-interaction
+.. _Monitor Config Reference: ../../configuration/mon-config-ref
+.. _monitoring your OSDs: ../../operations/monitoring-osd-pg
+
+.. _monitoring OSDs: ../../operations/monitoring-osd-pg/#monitoring-osds
+
+.. _subscribe to the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=subscribe+ceph-devel
+.. _unsubscribe from the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=unsubscribe+ceph-devel
+.. _subscribe to the ceph-users email list: mailto:ceph-users-join@lists.ceph.com
+.. _unsubscribe from the ceph-users email list: mailto:ceph-users-leave@lists.ceph.com
+.. _OS recommendations: ../../../start/os-recommendations
+.. _ceph-devel: ceph-devel@vger.kernel.org
diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
new file mode 100644
index 000000000..74d04bd9f
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -0,0 +1,782 @@
+====================
+ Troubleshooting PGs
+====================
+
+Placement Groups Never Get Clean
+================================
+
+If, after you have created your cluster, any Placement Groups (PGs) remain in
+the ``active`` status, the ``active+remapped`` status or the
+``active+degraded`` status and never achieves an ``active+clean`` status, you
+likely have a problem with your configuration.
+
+In such a situation, it may be necessary to review the settings in the `Pool,
+PG and CRUSH Config Reference`_ and make appropriate adjustments.
+
+As a general rule, run your cluster with more than one OSD and a pool size
+greater than two object replicas.
+
+.. _one-node-cluster:
+
+One Node Cluster
+----------------
+
+Ceph no longer provides documentation for operating on a single node.  Systems
+designed for distributed computing by definition do not run on a single node.
+The mounting of client kernel modules on a single node that contains a Ceph
+daemon may cause a deadlock due to issues with the Linux kernel itself (unless
+VMs are used as clients). You can experiment with Ceph in a one-node
+configuration, in spite of the limitations as described herein.
+
+To create a cluster on a single node, you must change the
+``osd_crush_chooseleaf_type`` setting from the default of ``1`` (meaning
+``host`` or ``node``) to ``0`` (meaning ``osd``) in your Ceph configuration
+file before you create your monitors and OSDs. This tells Ceph that an OSD is
+permitted to place another OSD on the same host. If you are trying to set up a
+single-node cluster and ``osd_crush_chooseleaf_type`` is greater than ``0``,
+Ceph will attempt to place the PGs of one OSD with the PGs of another OSD on
+another node, chassis, rack, row, or datacenter depending on the setting.
+
+.. tip:: DO NOT mount kernel clients directly on the same node as your Ceph
+   Storage Cluster. Kernel conflicts can arise. However, you can mount kernel
+   clients within virtual machines (VMs) on a single node.
+
+If you are creating OSDs using a single disk, you must manually create
+directories for the data first.
+
+
+Fewer OSDs than Replicas
+------------------------
+
+If two OSDs are in an ``up`` and ``in`` state, but the placement gropus are not
+in an ``active + clean`` state, you may have an ``osd_pool_default_size`` set
+to greater than ``2``.
+
+There are a few ways to address this situation. If you want to operate your
+cluster in an ``active + degraded`` state with two replicas, you can set the
+``osd_pool_default_min_size`` to ``2`` so that you can write objects in an
+``active + degraded`` state. You may also set the ``osd_pool_default_size``
+setting to ``2`` so that you have only two stored replicas (the original and
+one replica). In such a case, the cluster should achieve an ``active + clean``
+state.
+
+.. note:: You can make the changes while the cluster is running. If you make
+   the changes in your Ceph configuration file, you might need to restart your
+   cluster.
+
+
+Pool Size = 1
+-------------
+
+If you have ``osd_pool_default_size`` set to ``1``, you will have only one copy
+of the object. OSDs rely on other OSDs to tell them which objects they should
+have. If one OSD has a copy of an object and there is no second copy, then
+there is no second OSD to tell the first OSD that it should have that copy. For
+each placement group mapped to the first OSD (see ``ceph pg dump``), you can
+force the first OSD to notice the placement groups it needs by running a
+command of the following form:
+
+.. prompt:: bash
+
+   ceph osd force-create-pg <pgid>
+
+
+CRUSH Map Errors
+----------------
+
+If any placement groups in your cluster are unclean, then there might be errors
+in your CRUSH map.
+
+
+Stuck Placement Groups
+======================
+
+It is normal for placement groups to enter "degraded" or "peering" states after
+a component failure. Normally, these states reflect the expected progression
+through the failure recovery process. However, a placement group that stays in
+one of these states for a long time might be an indication of a larger problem.
+For this reason, the Ceph Monitors will warn when placement groups get "stuck"
+in a non-optimal state. Specifically, we check for:
+
+* ``inactive`` - The placement group has not been ``active`` for too long (that
+  is, it hasn't been able to service read/write requests).
+
+* ``unclean`` - The placement group has not been ``clean`` for too long (that
+  is, it hasn't been able to completely recover from a previous failure).
+
+* ``stale`` - The placement group status has not been updated by a
+  ``ceph-osd``.  This indicates that all nodes storing this placement group may
+  be ``down``.
+
+List stuck placement groups by running one of the following commands:
+
+.. prompt:: bash
+
+   ceph pg dump_stuck stale
+   ceph pg dump_stuck inactive
+   ceph pg dump_stuck unclean
+
+- Stuck ``stale`` placement groups usually indicate that key ``ceph-osd``
+  daemons are not running.
+- Stuck ``inactive`` placement groups usually indicate a peering problem (see
+  :ref:`failures-osd-peering`).
+- Stuck ``unclean`` placement groups usually indicate that something is
+  preventing recovery from completing, possibly unfound objects (see
+  :ref:`failures-osd-unfound`);
+
+
+
+.. _failures-osd-peering:
+
+Placement Group Down - Peering Failure
+======================================
+
+In certain cases, the ``ceph-osd`` `peering` process can run into problems,
+which can prevent a PG from becoming active and usable. In such a case, running
+the command ``ceph health detail`` will report something similar to the following:
+
+.. prompt:: bash
+
+   ceph health detail
+
+::
+
+    HEALTH_ERR 7 pgs degraded; 12 pgs down; 12 pgs peering; 1 pgs recovering; 6 pgs stuck unclean; 114/3300 degraded (3.455%); 1/3 in osds are down
+    ...
+    pg 0.5 is down+peering
+    pg 1.4 is down+peering
+    ...
+    osd.1 is down since epoch 69, last address 192.168.106.220:6801/8651
+
+Query the cluster to determine exactly why the PG is marked ``down`` by running a command of the following form:
+
+.. prompt:: bash
+
+   ceph pg 0.5 query
+
+.. code-block:: javascript
+
+ { "state": "down+peering",
+   ...
+   "recovery_state": [
+        { "name": "Started\/Primary\/Peering\/GetInfo",
+          "enter_time": "2012-03-06 14:40:16.169679",
+          "requested_info_from": []},
+        { "name": "Started\/Primary\/Peering",
+          "enter_time": "2012-03-06 14:40:16.169659",
+          "probing_osds": [
+                0,
+                1],
+          "blocked": "peering is blocked due to down osds",
+          "down_osds_we_would_probe": [
+                1],
+          "peering_blocked_by": [
+                { "osd": 1,
+                  "current_lost_at": 0,
+                  "comment": "starting or marking this osd lost may let us proceed"}]},
+        { "name": "Started",
+          "enter_time": "2012-03-06 14:40:16.169513"}
+    ]
+ }
+
+The ``recovery_state`` section tells us that peering is blocked due to down
+``ceph-osd`` daemons, specifically ``osd.1``. In this case, we can start that
+particular ``ceph-osd`` and recovery will proceed.
+
+Alternatively, if there is a catastrophic failure of ``osd.1`` (for example, if
+there has been a disk failure), the cluster can be informed that the OSD is
+``lost`` and the cluster can be instructed that it must cope as best it can.
+
+.. important:: Informing the cluster that an OSD has been lost is dangerous
+   because the cluster cannot guarantee that the other copies of the data are
+   consistent and up to date.
+
+To report an OSD ``lost`` and to instruct Ceph to continue to attempt recovery
+anyway, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph osd lost 1
+
+Recovery will proceed.
+
+
+.. _failures-osd-unfound:
+
+Unfound Objects
+===============
+
+Under certain combinations of failures, Ceph may complain about ``unfound``
+objects, as in this example:
+
+.. prompt:: bash
+
+   ceph health detail
+
+::
+
+   HEALTH_WARN 1 pgs degraded; 78/3778 unfound (2.065%)
+   pg 2.4 is active+degraded, 78 unfound
+
+This means that the storage cluster knows that some objects (or newer copies of
+existing objects) exist, but it hasn't found copies of them.  Here is an
+example of how this might come about for a PG whose data is on two OSDS, which
+we will call "1" and "2":
+
+* 1 goes down
+* 2 handles some writes, alone
+* 1 comes up
+* 1 and 2 re-peer, and the objects missing on 1 are queued for recovery.
+* Before the new objects are copied, 2 goes down.
+
+At this point, 1 knows that these objects exist, but there is no live
+``ceph-osd`` that has a copy of the objects. In this case, IO to those objects
+will block, and the cluster will hope that the failed node comes back soon.
+This is assumed to be preferable to returning an IO error to the user.
+
+.. note:: The situation described immediately above is one reason that setting
+   ``size=2`` on a replicated pool and ``m=1`` on an erasure coded pool risks
+   data loss.
+
+Identify which objects are unfound by running a command of the following form:
+
+.. prompt:: bash
+
+   ceph pg 2.4 list_unfound [starting offset, in json]
+
+.. code-block:: javascript
+
+  {
+    "num_missing": 1,
+    "num_unfound": 1,
+    "objects": [
+        {
+            "oid": {
+                "oid": "object",
+                "key": "",
+                "snapid": -2,
+                "hash": 2249616407,
+                "max": 0,
+                "pool": 2,
+                "namespace": ""
+            },
+            "need": "43'251",
+            "have": "0'0",
+            "flags": "none",
+            "clean_regions": "clean_offsets: [], clean_omap: 0, new_object: 1",
+            "locations": [
+                "0(3)",
+                "4(2)"
+            ]
+        }
+    ],
+    "state": "NotRecovering",
+    "available_might_have_unfound": true,
+    "might_have_unfound": [
+        {
+            "osd": "2(4)",
+            "status": "osd is down"
+        }
+    ],
+    "more": false
+  }
+
+If there are too many objects to list in a single result, the ``more`` field
+will be true and you can query for more.  (Eventually the command line tool
+will hide this from you, but not yet.)
+
+Now you can identify which OSDs have been probed or might contain data.
+
+At the end of the listing (before ``more: false``), ``might_have_unfound`` is
+provided when ``available_might_have_unfound`` is true.  This is equivalent to
+the output of ``ceph pg #.# query``.  This eliminates the need to use ``query``
+directly.  The ``might_have_unfound`` information given behaves the same way as
+that ``query`` does, which is described below.  The only difference is that
+OSDs that have the status of ``already probed`` are ignored.
+
+Use of ``query``:
+
+.. prompt:: bash
+
+   ceph pg 2.4 query
+
+.. code-block:: javascript
+
+   "recovery_state": [
+        { "name": "Started\/Primary\/Active",
+          "enter_time": "2012-03-06 15:15:46.713212",
+          "might_have_unfound": [
+                { "osd": 1,
+                  "status": "osd is down"}]},
+
+In this case, the cluster knows that ``osd.1`` might have data, but it is
+``down``. Here is the full range of possible states:
+
+* already probed
+* querying
+* OSD is down
+* not queried (yet)
+
+Sometimes it simply takes some time for the cluster to query possible
+locations.
+
+It is possible that there are other locations where the object might exist that
+are not listed. For example: if an OSD is stopped and taken out of the cluster
+and then the cluster fully recovers, and then through a subsequent set of
+failures the cluster ends up with an unfound object, the cluster will ignore
+the removed OSD. (This scenario, however, is unlikely.)
+
+If all possible locations have been queried and objects are still lost, you may
+have to give up on the lost objects. This, again, is possible only when unusual
+combinations of failures have occurred that allow the cluster to learn about
+writes that were performed before the writes themselves have been recovered. To
+mark the "unfound" objects as "lost", run a command of the following form:
+
+.. prompt:: bash
+
+   ceph pg 2.5 mark_unfound_lost revert|delete
+
+Here the final argument (``revert|delete``) specifies how the cluster should
+deal with lost objects.
+
+The ``delete`` option will cause the cluster to forget about them entirely.
+
+The ``revert`` option (which is not available for erasure coded pools) will
+either roll back to a previous version of the object or (if it was a new
+object) forget about the object entirely. Use ``revert`` with caution, as it
+may confuse applications that expect the object to exist.
+
+Homeless Placement Groups
+=========================
+
+It is possible that every OSD that has copies of a given placement group fails.
+If this happens, then the subset of the object store that contains those
+placement groups becomes unavailable and the monitor will receive no status
+updates for those placement groups. The monitor marks as ``stale`` any
+placement group whose primary OSD has failed. For example:
+
+.. prompt:: bash
+
+   ceph health
+
+::
+
+    HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+
+Identify which placement groups are ``stale`` and which were the last OSDs to
+store the ``stale`` placement groups by running the following command:
+
+.. prompt:: bash
+
+   ceph health detail
+
+::
+
+   HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+   ...
+   pg 2.5 is stuck stale+active+remapped, last acting [2,0]
+   ...
+   osd.10 is down since epoch 23, last address 192.168.106.220:6800/11080
+   osd.11 is down since epoch 13, last address 192.168.106.220:6803/11539
+   osd.12 is down since epoch 24, last address 192.168.106.220:6806/11861
+
+This output indicates that placement group 2.5 (``pg 2.5``) was last managed by
+``osd.0`` and ``osd.2``. Restart those OSDs to allow the cluster to recover
+that placement group.
+
+
+Only a Few OSDs Receive Data
+============================
+
+If only a few of the nodes in the cluster are receiving data, check the number
+of placement groups in the pool as instructed in the :ref:`Placement Groups
+<rados_ops_pgs_get_pg_num>` documentation. Since placement groups get mapped to
+OSDs in an operation involving dividing the number of placement groups in the
+cluster by the number of OSDs in the cluster, a small number of placement
+groups (the remainder, in this operation) are sometimes not distributed across
+the cluster. In situations like this, create a pool with a placement group
+count that is a multiple of the number of OSDs. See `Placement Groups`_ for
+details. See the :ref:`Pool, PG, and CRUSH Config Reference
+<rados_config_pool_pg_crush_ref>` for instructions on changing the default
+values used to determine how many placement groups are assigned to each pool.
+
+
+Can't Write Data
+================
+
+If the cluster is up, but some OSDs are down and you cannot write data, make
+sure that you have the minimum number of OSDs running in the pool. If you don't
+have the minimum number of OSDs running in the pool, Ceph will not allow you to
+write data to it because there is no guarantee that Ceph can replicate your
+data. See ``osd_pool_default_min_size`` in the :ref:`Pool, PG, and CRUSH
+Config Reference <rados_config_pool_pg_crush_ref>` for details.
+
+
+PGs Inconsistent
+================
+
+If the command ``ceph health detail`` returns an ``active + clean +
+inconsistent`` state, this might indicate an error during scrubbing. Identify
+the inconsistent placement group or placement groups by running the following
+command:
+
+.. prompt:: bash
+
+    $ ceph health detail
+
+::
+
+    HEALTH_ERR 1 pgs inconsistent; 2 scrub errors
+    pg 0.6 is active+clean+inconsistent, acting [0,1,2]
+    2 scrub errors
+
+Alternatively, run this command if you prefer to inspect the output in a
+programmatic way:
+
+.. prompt:: bash
+
+   $ rados list-inconsistent-pg rbd
+
+::
+
+    ["0.6"]
+
+There is only one consistent state, but in the worst case, we could have
+different inconsistencies in multiple perspectives found in more than one
+object. If an object named ``foo`` in PG ``0.6`` is truncated, the output of
+``rados list-inconsistent-pg rbd`` will look something like this:
+
+.. prompt:: bash
+
+   rados list-inconsistent-obj 0.6 --format=json-pretty
+
+.. code-block:: javascript
+
+    {
+        "epoch": 14,
+        "inconsistents": [
+            {
+                "object": {
+                    "name": "foo",
+                    "nspace": "",
+                    "locator": "",
+                    "snap": "head",
+                    "version": 1
+                },
+                "errors": [
+                    "data_digest_mismatch",
+                    "size_mismatch"
+                ],
+                "union_shard_errors": [
+                    "data_digest_mismatch_info",
+                    "size_mismatch_info"
+                ],
+                "selected_object_info": "0:602f83fe:::foo:head(16'1 client.4110.0:1 dirty|data_digest|omap_digest s 968 uv 1 dd e978e67f od ffffffff alloc_hint [0 0 0])",
+                "shards": [
+                    {
+                        "osd": 0,
+                        "errors": [],
+                        "size": 968,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xe978e67f"
+                    },
+                    {
+                        "osd": 1,
+                        "errors": [],
+                        "size": 968,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xe978e67f"
+                    },
+                    {
+                        "osd": 2,
+                        "errors": [
+                            "data_digest_mismatch_info",
+                            "size_mismatch_info"
+                        ],
+                        "size": 0,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xffffffff"
+                    }
+                ]
+            }
+        ]
+    }
+
+In this case, the output indicates the following:
+
+* The only inconsistent object is named ``foo``, and its head has
+  inconsistencies.
+* The inconsistencies fall into two categories:
+
+  #. ``errors``: these errors indicate inconsistencies between shards, without
+     an indication of which shard(s) are bad. Check for the ``errors`` in the
+     ``shards`` array, if available, to pinpoint the problem.
+
+     * ``data_digest_mismatch``: the digest of the replica read from ``OSD.2``
+       is different from the digests of the replica reads of ``OSD.0`` and
+       ``OSD.1``
+     * ``size_mismatch``: the size of the replica read from ``OSD.2`` is ``0``,
+       but the size reported by ``OSD.0`` and ``OSD.1`` is ``968``.
+
+  #. ``union_shard_errors``: the union of all shard-specific ``errors`` in the
+     ``shards`` array. The ``errors`` are set for the shard with the problem.
+     These errors include ``read_error`` and other similar errors. The
+     ``errors`` ending in ``oi`` indicate a comparison with
+     ``selected_object_info``. Examine the ``shards`` array to determine
+     which shard has which error or errors.
+
+     * ``data_digest_mismatch_info``: the digest stored in the ``object-info``
+       is not ``0xffffffff``, which is calculated from the shard read from
+       ``OSD.2``
+     * ``size_mismatch_info``: the size stored in the ``object-info`` is
+       different from the size read from ``OSD.2``. The latter is ``0``.
+
+.. warning:: If ``read_error`` is listed in a shard's ``errors`` attribute, the
+   inconsistency is likely due to physical storage errors. In cases like this,
+   check the storage used by that OSD. 
+   
+   Examine the output of ``dmesg`` and ``smartctl`` before attempting a drive
+   repair.
+
+To repair the inconsistent placement group, run a command of the following
+form:
+
+.. prompt:: bash
+
+   ceph pg repair {placement-group-ID}
+    
+.. warning: This command overwrites the "bad" copies with "authoritative"
+   copies. In most cases, Ceph is able to choose authoritative copies from all
+   the available replicas by using some predefined criteria. This, however,
+   does not work in every case. For example, it might be the case that the
+   stored data digest is missing, which means that the calculated digest is
+   ignored when Ceph chooses the authoritative copies. Be aware of this, and
+   use the above command with caution.
+
+
+If you receive ``active + clean + inconsistent`` states periodically due to
+clock skew, consider configuring the `NTP
+<https://en.wikipedia.org/wiki/Network_Time_Protocol>`_ daemons on your monitor
+hosts to act as peers. See `The Network Time Protocol <http://www.ntp.org>`_
+and Ceph :ref:`Clock Settings <mon-config-ref-clock>` for more information.
+
+
+Erasure Coded PGs are not active+clean
+======================================
+
+If CRUSH fails to find enough OSDs to map to a PG, it will show as a
+``2147483647`` which is ``ITEM_NONE`` or ``no OSD found``. For example::
+
+     [2,1,6,0,5,8,2147483647,7,4]
+
+Not enough OSDs
+---------------
+
+If the Ceph cluster has only eight OSDs and an erasure coded pool needs nine
+OSDs, the cluster will show "Not enough OSDs". In this case, you either create
+another erasure coded pool that requires fewer OSDs, by running commands of the
+following form:
+
+.. prompt:: bash
+
+     ceph osd erasure-code-profile set myprofile k=5 m=3
+     ceph osd pool create erasurepool erasure myprofile
+
+or add new OSDs, and the PG will automatically use them.
+
+CRUSH constraints cannot be satisfied
+-------------------------------------
+
+If the cluster has enough OSDs, it is possible that the CRUSH rule is imposing
+constraints that cannot be satisfied. If there are ten OSDs on two hosts and
+the CRUSH rule requires that no two OSDs from the same host are used in the
+same PG, the mapping may fail because only two OSDs will be found. Check the
+constraint by displaying ("dumping") the rule, as shown here:
+
+.. prompt:: bash
+
+   ceph osd crush rule ls
+
+::
+
+    [
+        "replicated_rule",
+        "erasurepool"]
+    $ ceph osd crush rule dump erasurepool
+    { "rule_id": 1,
+      "rule_name": "erasurepool",
+      "type": 3,
+      "steps": [
+            { "op": "take",
+              "item": -1,
+              "item_name": "default"},
+            { "op": "chooseleaf_indep",
+              "num": 0,
+              "type": "host"},
+            { "op": "emit"}]}
+
+
+Resolve this problem by creating a new pool in which PGs are allowed to have
+OSDs residing on the same host by running the following commands:
+
+.. prompt:: bash
+
+   ceph osd erasure-code-profile set myprofile crush-failure-domain=osd
+   ceph osd pool create erasurepool erasure myprofile
+
+CRUSH gives up too soon
+-----------------------
+
+If the Ceph cluster has just enough OSDs to map the PG (for instance a cluster
+with a total of nine OSDs and an erasure coded pool that requires nine OSDs per
+PG), it is possible that CRUSH gives up before finding a mapping. This problem
+can be resolved by:
+
+* lowering the erasure coded pool requirements to use fewer OSDs per PG (this
+  requires the creation of another pool, because erasure code profiles cannot
+  be modified dynamically).
+
+* adding more OSDs to the cluster (this does not require the erasure coded pool
+  to be modified, because it will become clean automatically)
+
+* using a handmade CRUSH rule that tries more times to find a good mapping.
+  This can be modified for an existing CRUSH rule by setting
+  ``set_choose_tries`` to a value greater than the default.
+
+First, verify the problem by using  ``crushtool`` after extracting the crushmap
+from the cluster. This ensures that your experiments do not modify the Ceph
+cluster and that they operate only on local files:
+
+.. prompt:: bash
+
+   ceph osd crush rule dump erasurepool
+
+::
+
+    { "rule_id": 1,
+      "rule_name": "erasurepool",
+      "type": 3,
+      "steps": [
+            { "op": "take",
+              "item": -1,
+              "item_name": "default"},
+            { "op": "chooseleaf_indep",
+              "num": 0,
+              "type": "host"},
+            { "op": "emit"}]}
+    $ ceph osd getcrushmap > crush.map
+    got crush map from osdmap epoch 13
+    $ crushtool -i crush.map --test --show-bad-mappings \
+       --rule 1 \
+       --num-rep 9 \
+       --min-x 1 --max-x $((1024 * 1024))
+    bad mapping rule 8 x 43 num_rep 9 result [3,2,7,1,2147483647,8,5,6,0]
+    bad mapping rule 8 x 79 num_rep 9 result [6,0,2,1,4,7,2147483647,5,8]
+    bad mapping rule 8 x 173 num_rep 9 result [0,4,6,8,2,1,3,7,2147483647]
+
+Here, ``--num-rep`` is the number of OSDs that the erasure code CRUSH rule
+needs, ``--rule`` is the value of the ``rule_id`` field that was displayed by
+``ceph osd crush rule dump``. This test will attempt to map one million values
+(in this example, the range defined by ``[--min-x,--max-x]``) and must display
+at least one bad mapping. If this test outputs nothing, all mappings have been
+successful and you can be assured that the problem with your cluster is not
+caused by bad mappings.
+
+Changing the value of set_choose_tries
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#. Decompile the CRUSH map to edit the CRUSH rule by running the following
+   command:
+
+   .. prompt:: bash
+
+      crushtool --decompile crush.map > crush.txt
+
+#. Add the following line to the rule::
+
+      step set_choose_tries 100
+
+   The relevant part of the ``crush.txt`` file will resemble this::
+
+      rule erasurepool {
+              id 1
+              type erasure
+              step set_chooseleaf_tries 5
+              step set_choose_tries 100
+              step take default
+              step chooseleaf indep 0 type host
+              step emit
+      }
+
+#. Recompile and retest the CRUSH rule:
+
+   .. prompt:: bash
+
+      crushtool --compile crush.txt -o better-crush.map
+
+#. When all mappings succeed, display a histogram of the number of tries that
+   were necessary to find all of the mapping by using the
+   ``--show-choose-tries`` option of the ``crushtool`` command, as in the
+   following example:
+
+   .. prompt:: bash
+
+      crushtool -i better-crush.map --test --show-bad-mappings \
+       --show-choose-tries \
+       --rule 1 \
+       --num-rep 9 \
+       --min-x 1 --max-x $((1024 * 1024))
+    ...
+    11:        42
+    12:        44
+    13:        54
+    14:        45
+    15:        35
+    16:        34
+    17:        30
+    18:        25
+    19:        19
+    20:        22
+    21:        20
+    22:        17
+    23:        13
+    24:        16
+    25:        13
+    26:        11
+    27:        11
+    28:        13
+    29:        11
+    30:        10
+    31:         6
+    32:         5
+    33:        10
+    34:         3
+    35:         7
+    36:         5
+    37:         2
+    38:         5
+    39:         5
+    40:         2
+    41:         5
+    42:         4
+    43:         1
+    44:         2
+    45:         2
+    46:         3
+    47:         1
+    48:         0
+    ...
+    102:         0
+    103:         1
+    104:         0
+    ...
+
+   This output indicates that it took eleven tries to map forty-two PGs, twelve
+   tries to map forty-four PGs etc. The highest number of tries is the minimum
+   value of ``set_choose_tries`` that prevents bad mappings (for example,
+   ``103`` in the above output, because it did not take more than 103 tries for
+   any PG to be mapped).
+
+.. _check: ../../operations/placement-groups#get-the-number-of-placement-groups
+.. _Placement Groups: ../../operations/placement-groups
+.. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
diff --git a/doc/radosgw/STS.rst b/doc/radosgw/STS.rst
new file mode 100644
index 000000000..bc89b89da
--- /dev/null
+++ b/doc/radosgw/STS.rst
@@ -0,0 +1,297 @@
+===========
+STS in Ceph
+===========
+
+Secure Token Service is a web service in AWS that returns a set of temporary security credentials for authenticating federated users.
+The link to official AWS documentation can be found here: https://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html.
+
+Ceph Object Gateway implements a subset of STS APIs that provide temporary credentials for identity and access management.
+These temporary credentials can be used to make subsequent S3 calls which will be authenticated by the STS engine in Ceph Object Gateway.
+Permissions of the temporary credentials can be further restricted via an IAM policy passed as a parameter to the STS APIs.
+
+STS REST APIs
+=============
+
+The following STS REST APIs have been implemented in Ceph Object Gateway:
+
+1. AssumeRole: Returns a set of temporary credentials that can be used for 
+cross-account access. The temporary credentials will have permissions that are
+allowed by both - permission policies attached with the Role and policy attached
+with the AssumeRole API.
+
+Parameters:
+    **RoleArn** (String/ Required): ARN of the Role to Assume.
+
+    **RoleSessionName** (String/ Required): An Identifier for the assumed role
+    session.
+
+    **Policy** (String/ Optional): An IAM Policy in JSON format.
+
+    **DurationSeconds** (Integer/ Optional): The duration in seconds of the session.
+    Its default value is 3600.
+
+    **ExternalId** (String/ Optional): A unique Id that might be used when a role is
+    assumed in another account.
+
+    **SerialNumber** (String/ Optional): The Id number of the MFA device associated
+    with the user making the AssumeRole call.
+
+    **TokenCode** (String/ Optional): The value provided by the MFA device, if the
+    trust policy of the role being assumed requires MFA.
+
+2. AssumeRoleWithWebIdentity: Returns a set of temporary credentials for users that
+have been authenticated by a web/mobile app by an OpenID Connect /OAuth2.0 Identity Provider.
+Currently Keycloak has been tested and integrated with RGW.
+
+Parameters:
+    **RoleArn** (String/ Required): ARN of the Role to Assume.
+
+    **RoleSessionName** (String/ Required): An Identifier for the assumed role
+    session.
+
+    **Policy** (String/ Optional): An IAM Policy in JSON format.
+
+    **DurationSeconds** (Integer/ Optional): The duration in seconds of the session.
+    Its default value is 3600.
+
+    **ProviderId** (String/ Optional): Fully qualified host component of the domain name
+    of the IDP. Valid only for OAuth2.0 tokens (not for OpenID Connect tokens).
+
+    **WebIdentityToken** (String/ Required): The OpenID Connect/ OAuth2.0 token, which the
+    application gets in return after authenticating its user with an IDP.
+
+Before invoking AssumeRoleWithWebIdentity, an OpenID Connect Provider entity (which the web application
+authenticates with), needs to be created in RGW.
+
+The trust between the IDP and the role is created by adding a condition to the role's trust policy, which
+allows access only to applications which satisfy the given condition.
+All claims of the JWT are supported in the condition of the role's trust policy.
+An example of a policy that uses the 'aud' claim in the condition is of the form::
+
+    '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":["arn:aws:iam:::oidc-provider/<URL of IDP>"]},"Action":["sts:AssumeRoleWithWebIdentity"],"Condition":{"StringEquals":{"<URL of IDP> :app_id":"<aud>"}}}]}'''
+
+The app_id in the condition above must match the 'aud' claim of the incoming token.
+
+An example of a policy that uses the 'sub' claim in the condition is of the form::
+
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :sub\":\"<sub>\"\}\}\}\]\}"
+
+Similarly, an example of a policy that uses 'azp' claim in the condition is of the form::
+
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :azp\":\"<azp>\"\}\}\}\]\}"
+
+A shadow user is created corresponding to every federated user. The user id is derived from the 'sub' field of the incoming web token.
+The user is created in a separate namespace - 'oidc' such that the user id doesn't clash with any other user ids in rgw. The format of the user id
+is - <tenant>$<user-namespace>$<sub> where user-namespace is 'oidc' for users that authenticate with oidc providers.
+
+RGW now supports Session tags that can be passed in the web token to AssumeRoleWithWebIdentity call. More information related to Session Tags can be found here
+:doc:`session-tags`.
+
+STS Configuration
+=================
+
+The following configurable options have to be added for STS integration::
+
+  [client.{your-rgw-name}]
+  rgw_sts_key = {sts key for encrypting the session token}
+  rgw_s3_auth_use_sts = true
+
+Notes: 
+
+* By default, STS and S3 APIs co-exist in the same namespace, and both S3
+  and STS APIs can be accessed via the same endpoint in Ceph Object Gateway.
+* The ``rgw_sts_key`` needs to be a hex-string consisting of exactly 16 characters.
+
+Examples
+========
+1. In order to get the example to work, make sure that the user TESTER has the ``roles`` capability assigned:
+
+.. code-block:: console
+
+   radosgw-admin caps add --uid="TESTER" --caps="roles=*"
+
+2. The following is an example of the AssumeRole API call, which shows steps to create a role, assign a policy to it
+   (that allows access to S3 resources), assuming a role to get temporary credentials and accessing S3 resources using
+   those credentials. In this example, TESTER1 assumes a role created by TESTER, to access S3 resources owned by TESTER,
+   according to the permission policy attached to the role.
+
+.. code-block:: python
+
+    import boto3
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=<access_key of TESTER>,
+    aws_secret_access_key=<secret_key of TESTER>,
+    endpoint_url=<IAM URL>,
+    region_name=''
+    )
+
+    policy_document = '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER1"]},"Action":["sts:AssumeRole"]}]}'''
+
+    role_response = iam_client.create_role(
+    AssumeRolePolicyDocument=policy_document,
+    Path='/',
+    RoleName='S3Access',
+    )
+
+    role_policy = '''{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}}'''
+
+    response = iam_client.put_role_policy(
+    RoleName='S3Access',
+    PolicyName='Policy1',
+    PolicyDocument=role_policy
+    )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id=<access_key of TESTER1>,
+    aws_secret_access_key=<secret_key of TESTER1>,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = sts_client.assume_role(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=3600
+    )
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url=<S3 URL>,
+    region_name='',)
+
+    bucket_name = 'my-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+    resp = s3client.list_buckets()
+
+2. The following is an example of AssumeRoleWithWebIdentity API call, where an external app that has users authenticated with
+an OpenID Connect/ OAuth2 IDP (Keycloak in this example), assumes a role to get back temporary credentials and access S3 resources
+according to permission policy of the role.
+
+.. code-block:: python
+
+    import boto3
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=<access_key of TESTER>,
+    aws_secret_access_key=<secret_key of TESTER>,
+    endpoint_url=<IAM URL>,
+    region_name=''
+    )
+
+    oidc_response = iam_client.create_open_id_connect_provider(
+        Url=<URL of the OpenID Connect Provider,
+        ClientIDList=[
+            <Client id registered with the IDP>
+        ],
+        ThumbprintList=[
+            <Thumbprint of the IDP>
+     ]
+    )
+
+    policy_document = '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/demo"]},"Action":["sts:AssumeRoleWithWebIdentity"],"Condition":{"StringEquals":{"localhost:8080/auth/realms/demo:app_id":"customer-portal"}}}]}'''
+    role_response = iam_client.create_role(
+    AssumeRolePolicyDocument=policy_document,
+    Path='/',
+    RoleName='S3Access',
+    )
+
+    role_policy = '''{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}}'''
+
+    response = iam_client.put_role_policy(
+        RoleName='S3Access',
+        PolicyName='Policy1',
+        PolicyDocument=role_policy
+    )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id=<access_key of TESTER1>,
+    aws_secret_access_key=<secret_key of TESTER1>,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = client.assume_role_with_web_identity(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=3600,
+    WebIdentityToken=<Web Token>
+    )
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url=<S3 URL>,
+    region_name='',)
+
+    bucket_name = 'my-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+    resp = s3client.list_buckets()
+
+How to obtain thumbprint of an OpenID Connect Provider IDP
+==========================================================
+1. Take the OpenID connect provider's URL and add /.well-known/openid-configuration
+to it to get the URL to get the IDP's configuration document. For example, if the URL
+of the IDP is http://localhost:8000/auth/realms/quickstart, then the URL to get the
+document from is http://localhost:8000/auth/realms/quickstart/.well-known/openid-configuration
+
+2. Use the following curl command to get the configuration document from the URL described
+in step 1::
+
+    curl -k -v \
+      -X GET \
+      -H "Content-Type: application/x-www-form-urlencoded" \
+      "http://localhost:8000/auth/realms/quickstart/.well-known/openid-configuration" \
+    | jq .
+
+ 3. From the response of step 2, use the value of "jwks_uri" to get the certificate of the IDP,
+ using the following code::
+     curl -k -v \
+      -X GET \
+      -H "Content-Type: application/x-www-form-urlencoded" \
+      "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/certs" \
+      | jq .
+
+3. Copy the result of "x5c" in the response above, in a file certificate.crt, and add
+'-----BEGIN CERTIFICATE-----' at the beginning and "-----END CERTIFICATE-----"
+at the end.
+
+4. Use the following OpenSSL command to get the certificate thumbprint::
+
+    openssl x509 -in certificate.crt -fingerprint -noout
+
+5. The result of the above command in step 4, will be a SHA1 fingerprint, like the following::
+
+    SHA1 Fingerprint=F7:D7:B3:51:5D:D0:D3:19:DD:21:9A:43:A9:EA:72:7A:D6:06:52:87
+
+6.  Remove the colons from the result above to get the final thumbprint which can be as input
+while creating the OpenID Connect Provider entity in IAM::
+
+    F7D7B3515DD0D319DD219A43A9EA727AD6065287
+
+Roles in RGW
+============
+
+More information for role manipulation can be found here
+:doc:`role`.
+
+OpenID Connect Provider in RGW
+==============================
+
+More information for OpenID Connect Provider entity manipulation
+can be found here
+:doc:`oidc`.
+
+Keycloak integration with Radosgw
+=================================
+
+Steps for integrating Radosgw with Keycloak can be found here
+:doc:`keycloak`.
+
+STSLite
+=======
+STSLite has been built on STS, and documentation for the same can be found here
+:doc:`STSLite`.
\ No newline at end of file
diff --git a/doc/radosgw/STSLite.rst b/doc/radosgw/STSLite.rst
new file mode 100644
index 000000000..7880e373f
--- /dev/null
+++ b/doc/radosgw/STSLite.rst
@@ -0,0 +1,196 @@
+=========
+STS Lite
+=========
+
+Ceph Object Gateway provides support for a subset of Amazon Secure Token Service
+(STS) APIs. STS Lite is an extension of STS and builds upon one of its APIs to
+decrease the load on external IDPs like Keystone and LDAP.
+
+A set of temporary security credentials is returned after authenticating
+a set of AWS credentials with the external IDP. These temporary credentials can be used
+to make subsequent S3 calls which will be authenticated by the STS engine in Ceph,
+resulting in less load on the Keystone/ LDAP server.
+
+Temporary and limited privileged credentials can be obtained for a local user
+also using the STS Lite API.
+
+STS Lite REST APIs
+==================
+
+The following STS Lite REST API is part of STS Lite in Ceph Object Gateway:
+
+1. GetSessionToken: Returns a set of temporary credentials for a set of AWS
+credentials. After initial authentication with Keystone/ LDAP, the temporary
+credentials returned can be used to make subsequent S3 calls. The temporary
+credentials will have the same permission as that of the AWS credentials.
+
+Parameters:
+    **DurationSeconds** (Integer/ Optional): The duration in seconds for which the
+    credentials should remain valid. Its default value is 3600. Its default max
+    value is 43200 which is can be configured using rgw sts max session duration.
+
+    **SerialNumber** (String/ Optional): The Id number of the MFA device associated 
+    with the user making the GetSessionToken call.
+
+    **TokenCode** (String/ Optional): The value provided by the MFA device, if MFA is required.
+
+An administrative user needs to attach a policy to allow invocation of GetSessionToken API using its permanent
+credentials and to allow subsequent S3 operations invocation using only the temporary credentials returned
+by GetSessionToken.
+
+The user attaching the policy needs to have admin caps. For example::
+
+    radosgw-admin caps add --uid="TESTER" --caps="user-policy=*"
+
+The following is the policy that needs to be attached to a user 'TESTER1'::
+
+    user_policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Action\":\"s3:*\",\"Resource\":[\"*\"],\"Condition\":{\"BoolIfExists\":{\"sts:authentication\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":\"sts:GetSessionToken\",\"Resource\":\"*\",\"Condition\":{\"BoolIfExists\":{\"sts:authentication\":\"false\"}}}]}"
+
+
+STS Lite Configuration
+======================
+
+The following configurable options are available for STS Lite integration::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting the session token}
+  rgw s3 auth use sts = true
+
+The above STS configurables can be used with the Keystone configurables if one
+needs to use STS Lite in conjunction with Keystone. The complete set of
+configurable options will be::
+
+  [client.{your-rgw-name}]
+  rgw_sts_key = {sts key for encrypting/ decrypting the session token, exactly 16 hex characters}
+  rgw_s3_auth_use_sts = true
+
+  rgw keystone url = {keystone server url:keystone server admin port}
+  rgw keystone admin project = {keystone admin project name}
+  rgw keystone admin tenant = {keystone service tenant name}
+  rgw keystone admin domain = {keystone admin domain name}
+  rgw keystone api version = {keystone api version}
+  rgw keystone implicit tenants = {true for private tenant for each new user}
+  rgw keystone admin password = {keystone service tenant user name}
+  rgw keystone admin user = keystone service tenant user password}
+  rgw keystone accepted roles = {accepted user roles}
+  rgw keystone token cache size = {number of tokens to cache}
+  rgw s3 auth use keystone = true
+
+The details of the integrating ldap with Ceph Object Gateway can be found here:
+:doc:`keystone`
+
+The complete set of configurables to use STS Lite with LDAP are::
+
+  [client.{your-rgw-name}]
+  rgw_sts_key = {sts key for encrypting/ decrypting the session token, exactly 16 hex characters}
+  rgw_s3_auth_use_sts = true
+
+  rgw_s3_auth_use_ldap = true
+  rgw_ldap_uri = {LDAP server to use}
+  rgw_ldap_binddn = {Distinguished Name (DN) of the service account}
+  rgw_ldap_secret = {password for the service account}
+  rgw_ldap_searchdn = {base in the directory information tree for searching users}
+  rgw_ldap_dnattr = {attribute being used in the constructed search filter to match a username}
+  rgw_ldap_searchfilter = {search filter}
+
+The details of the integrating ldap with Ceph Object Gateway can be found here:
+:doc:`ldap-auth`
+
+Note: By default, STS and S3 APIs co-exist in the same namespace, and both S3
+and STS APIs can be accessed via the same endpoint in Ceph Object Gateway.
+
+Example showing how to Use STS Lite with Keystone
+=================================================
+
+The following are the steps needed to use STS Lite with Keystone. Boto 3.x has
+been used to write an example code to show the integration of STS Lite with
+Keystone.
+
+1. Generate EC2 credentials :
+
+.. code-block:: javascript
+
+  openstack ec2 credentials create
+  +------------+--------------------------------------------------------+
+  | Field      | Value                                                  |
+  +------------+--------------------------------------------------------+
+  | access     | b924dfc87d454d15896691182fdeb0ef                       |
+  | links      | {u'self': u'http://192.168.0.15/identity/v3/users/     |
+  |            | 40a7140e424f493d8165abc652dc731c/credentials/          |
+  |            | OS-EC2/b924dfc87d454d15896691182fdeb0ef'}              |
+  | project_id | c703801dccaf4a0aaa39bec8c481e25a                       |
+  | secret     | 6a2142613c504c42a94ba2b82147dc28                       |
+  | trust_id   | None                                                   |
+  | user_id    | 40a7140e424f493d8165abc652dc731c                       |
+  +------------+--------------------------------------------------------+
+
+2. Use the credentials created in the step 1. to get back a set of temporary
+   credentials using GetSessionToken API.
+
+.. code-block:: python
+
+    import boto3
+ 
+    access_key = <ec2 access key>
+    secret_key = <ec2 secret key>
+
+    client = boto3.client('sts',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = client.get_session_token(
+        DurationSeconds=43200
+    )
+
+3. The temporary credentials obtained in step 2. can be used for making S3 calls:
+
+.. code-block:: python
+
+    s3client = boto3.client('s3',
+      aws_access_key_id = response['Credentials']['AccessKeyId'],
+      aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+      aws_session_token = response['Credentials']['SessionToken'],
+      endpoint_url=<S3 URL>,
+      region_name='')
+
+    bucket = s3client.create_bucket(Bucket='my-new-shiny-bucket')
+    response = s3client.list_buckets()
+    for bucket in response["Buckets"]:
+        print("{name}\t{created}".format(
+                    name = bucket['Name'],
+                    created = bucket['CreationDate'],
+        ))
+
+Similar steps can be performed for using GetSessionToken with LDAP.
+
+Limitations and Workarounds
+===========================
+
+1. Keystone currently supports only S3 requests, hence in order to successfully 
+authenticate an STS request, the following workaround needs to be added to boto
+to the following file - botocore/auth.py
+
+Lines 13-16 have been added as a workaround in the code block below:
+
+.. code-block:: python
+
+  class SigV4Auth(BaseSigner):
+    """
+    Sign a request with Signature V4.
+    """
+    REQUIRES_REGION = True
+
+    def __init__(self, credentials, service_name, region_name):
+        self.credentials = credentials
+        # We initialize these value here so the unit tests can have
+        # valid values.  But these will get overridden in ``add_auth``
+        # later for real requests.
+        self._region_name = region_name
+        if service_name == 'sts':
+            self._service_name = 's3'
+        else:
+            self._service_name = service_name
+
diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
new file mode 100644
index 000000000..8d70252fe
--- /dev/null
+++ b/doc/radosgw/admin.rst
@@ -0,0 +1,715 @@
+=============
+ Admin Guide
+=============
+
+Once you have your Ceph Object Storage service up and running, you may
+administer the service with user management, access controls, quotas 
+and usage tracking among other features.
+
+
+User Management
+===============
+
+Ceph Object Storage user management refers to users of the Ceph Object Storage
+service (i.e., not the Ceph Object Gateway as a user of the Ceph Storage
+Cluster). You must create a user, access key and secret to enable end users to
+interact with Ceph Object Gateway services.
+
+There are two user types: 
+
+- **User:** The term 'user' reflects a user of the S3 interface.
+
+- **Subuser:** The term 'subuser' reflects a user of the Swift interface. A subuser
+  is associated to a user .
+  
+.. ditaa::
+           +---------+
+           |   User  |
+           +----+----+  
+                |     
+                |     +-----------+
+                +-----+  Subuser  |
+                      +-----------+
+
+You can create, modify, view, suspend and remove users and subusers. In addition
+to user and subuser IDs, you may add a display name and an email address for a
+user.  You can specify a key and secret, or generate a key and secret
+automatically. When generating or specifying keys, note that user IDs correspond
+to an S3 key type and subuser IDs correspond to a swift key type. Swift keys
+also have access levels of ``read``, ``write``, ``readwrite`` and ``full``.
+
+
+Create a User
+-------------
+
+To create a user (S3 interface), execute the following::
+
+	radosgw-admin user create --uid={username} --display-name="{display-name}" [--email={email}]
+
+For example:: 	
+	
+  radosgw-admin user create --uid=johndoe --display-name="John Doe" --email=john@example.com
+  
+.. code-block:: javascript
+  
+  { "user_id": "johndoe",
+    "display_name": "John Doe",
+    "email": "john@example.com",
+    "suspended": 0,
+    "max_buckets": 1000,
+    "subusers": [],
+    "keys": [
+          { "user": "johndoe",
+            "access_key": "11BS02LGFB6AL6H1ADMW",
+            "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}],
+    "swift_keys": [],
+    "caps": [],
+    "op_mask": "read, write, delete",
+    "default_placement": "",
+    "placement_tags": [],
+    "bucket_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "user_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "temp_url_keys": []}
+
+Creating a user also creates an ``access_key`` and ``secret_key`` entry for use
+with any S3 API-compatible client.  
+
+.. important:: Check the key output. Sometimes ``radosgw-admin``
+   generates a JSON escape (``\``) character, and some clients
+   do not know how to handle JSON escape characters. Remedies include 
+   removing the JSON escape character (``\``), encapsulating the string
+   in quotes, regenerating the key and ensuring that it 
+   does not have a JSON escape character or specify the key and secret 
+   manually.
+
+
+Create a Subuser
+----------------
+
+To create a subuser (Swift interface) for the user, you must specify the user ID
+(``--uid={username}``), a subuser ID and the access level for the subuser. ::
+
+  radosgw-admin subuser create --uid={uid} --subuser={uid} --access=[ read | write | readwrite | full ]
+
+For example::
+
+  radosgw-admin subuser create --uid=johndoe --subuser=johndoe:swift --access=full
+
+
+.. note:: ``full`` is not ``readwrite``, as it also includes the access control policy.
+
+.. code-block:: javascript
+
+  { "user_id": "johndoe",
+    "display_name": "John Doe",
+    "email": "john@example.com",
+    "suspended": 0,
+    "max_buckets": 1000,
+    "subusers": [
+          { "id": "johndoe:swift",
+            "permissions": "full-control"}],
+    "keys": [
+          { "user": "johndoe",
+            "access_key": "11BS02LGFB6AL6H1ADMW",
+            "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}],
+    "swift_keys": [],
+    "caps": [],
+    "op_mask": "read, write, delete",
+    "default_placement": "",
+    "placement_tags": [],
+    "bucket_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "user_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "temp_url_keys": []}
+
+
+Get User Info
+-------------
+
+To get information about a user, you must specify ``user info`` and the user ID
+(``--uid={username}``) . :: 
+
+	radosgw-admin user info --uid=johndoe
+
+
+
+Modify User Info
+----------------
+
+To modify information about a user, you must specify the user ID (``--uid={username}``)
+and the attributes you want to modify. Typical modifications are to keys and secrets,
+email addresses, display names and access levels. For example:: 
+
+	radosgw-admin user modify --uid=johndoe --display-name="John E. Doe"
+
+To modify subuser values, specify ``subuser modify``, user ID and the subuser ID. For example::
+
+	radosgw-admin subuser modify --uid=johndoe --subuser=johndoe:swift --access=full
+
+
+User Enable/Suspend
+-------------------
+
+When you create a user, the user is enabled by default. However, you may suspend
+user  privileges and re-enable them at a later time. To suspend a user, specify
+``user suspend`` and the user ID. ::
+
+	radosgw-admin user suspend --uid=johndoe
+
+To re-enable a suspended user, specify ``user enable`` and the user ID. :: 
+
+	radosgw-admin user enable --uid=johndoe
+	
+.. note:: Disabling the user disables the subuser.
+
+
+Remove a User
+-------------
+
+When you remove a user, the user and subuser are removed from the system.
+However, you may remove just the subuser if you wish. To remove a user (and
+subuser), specify ``user rm`` and the user ID. ::
+
+	radosgw-admin user rm --uid=johndoe
+
+To remove the subuser only, specify ``subuser rm`` and the subuser ID. ::
+
+	radosgw-admin subuser rm --subuser=johndoe:swift
+
+
+Options include:
+
+- **Purge Data:** The ``--purge-data`` option purges all data associated 
+  to the UID.
+  
+- **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
+  to the UID.
+
+
+Remove a Subuser
+----------------
+
+When you remove a sub user, you are removing access to the Swift interface. 
+The user will remain in the system. To remove the subuser, specify 
+``subuser rm`` and the subuser ID. ::
+
+	radosgw-admin subuser rm --subuser=johndoe:swift
+
+
+
+Options include:
+  
+- **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
+  to the UID.
+
+
+Add / Remove a Key
+------------------------
+
+Both users and subusers require the key to access the S3 or Swift interface. To
+use S3, the user needs a key pair which is composed of an access key and a 
+secret key. On the other hand, to use Swift, the user typically needs a secret 
+key (password), and use it together with the associated user ID. You may create
+a key and either specify or generate the access key and/or secret key. You may 
+also remove a key. Options include:
+
+- ``--key-type=<type>`` specifies the key type. The options are: s3, swift
+- ``--access-key=<key>`` manually specifies an S3 access key.
+- ``--secret-key=<key>`` manually specifies a S3 secret key or a Swift secret key.
+- ``--gen-access-key`` automatically generates a random S3 access key.
+- ``--gen-secret`` automatically generates a random S3 secret key or a random Swift secret key.
+
+An example how to add a specified S3 key pair for a user. ::
+
+	radosgw-admin key create --uid=foo --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey
+
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "keys": [
+      { "user": "foo",
+        "access_key": "fooAccessKey",
+        "secret_key": "fooSecretKey"}],
+  }
+
+Note that you may create multiple S3 key pairs for a user.
+
+To attach a specified swift secret key for a subuser. ::
+
+	radosgw-admin key create --subuser=foo:bar --key-type=swift --secret-key barSecret
+
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "subusers": [
+       { "id": "foo:bar",
+         "permissions": "full-control"}],
+    "swift_keys": [
+      { "user": "foo:bar",
+        "secret_key": "asfghjghghmgm"}]}
+
+Note that a subuser can have only one swift secret key.
+
+Subusers can also be used with S3 APIs if the subuser is associated with a S3 key pair. ::	
+
+	radosgw-admin key create --subuser=foo:bar --key-type=s3 --access-key barAccessKey --secret-key barSecretKey
+	
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "subusers": [
+       { "id": "foo:bar",
+         "permissions": "full-control"}],
+    "keys": [
+      { "user": "foo:bar",
+        "access_key": "barAccessKey",
+        "secret_key": "barSecretKey"}],
+  }
+
+
+To remove a S3 key pair, specify the access key. :: 
+
+	radosgw-admin key rm --uid=foo --key-type=s3 --access-key=fooAccessKey 
+
+To remove the swift secret key. ::
+
+	radosgw-admin key rm --subuser=foo:bar --key-type=swift
+
+
+Add / Remove Admin Capabilities
+-------------------------------
+
+The Ceph Storage Cluster provides an administrative API that enables  users to
+execute administrative functions via the REST API. By default, users do NOT have
+access to this API. To enable a user to exercise  administrative functionality,
+provide the user with administrative capabilities.
+
+To add administrative capabilities to a user, execute the following:: 
+
+	radosgw-admin caps add --uid={uid} --caps={caps}
+
+
+You can add read, write or all capabilities to users, buckets, metadata and 
+usage (utilization). For example::
+
+	--caps="[users|buckets|metadata|usage|zone|amz-cache|info|bilog|mdlog|datalog|user-policy|oidc-provider|roles|ratelimit]=[*|read|write|read, write]"
+
+For example::
+
+	radosgw-admin caps add --uid=johndoe --caps="users=*;buckets=*"
+
+
+To remove administrative capabilities from a user, execute the following:: 
+
+	radosgw-admin caps rm --uid=johndoe --caps={caps}
+
+
+Quota Management
+================
+
+The Ceph Object Gateway enables you to set quotas on users and buckets owned by
+users. Quotas include the maximum number of objects in a bucket and the maximum
+storage size a bucket can hold.
+
+- **Bucket:** The ``--bucket`` option allows you to specify a quota for
+  buckets the user owns.
+
+- **Maximum Objects:** The ``--max-objects`` setting allows you to specify
+  the maximum number of objects. A negative value disables this setting.
+  
+- **Maximum Size:** The ``--max-size`` option allows you to specify a quota
+  size in B/K/M/G/T, where B is the default. A negative value disables this setting.
+  
+- **Quota Scope:** The ``--quota-scope`` option sets the scope for the quota.
+  The options are ``bucket`` and ``user``. Bucket quotas apply to buckets a 
+  user owns. User quotas apply to a user.
+
+
+Set User Quota
+--------------
+
+Before you enable a quota, you must first set the quota parameters.
+For example:: 
+
+	radosgw-admin quota set --quota-scope=user --uid=<uid> [--max-objects=<num objects>] [--max-size=<max size>]
+
+For example:: 
+
+	radosgw-admin quota set --quota-scope=user --uid=johndoe --max-objects=1024 --max-size=1024B
+
+
+A negative value for num objects and / or max size means that the
+specific quota attribute check is disabled.
+
+
+Enable/Disable User Quota
+-------------------------
+
+Once you set a user quota, you may enable it. For example:: 
+
+	radosgw-admin quota enable --quota-scope=user --uid=<uid>
+
+You may disable an enabled user quota. For example:: 
+
+	radosgw-admin quota disable --quota-scope=user --uid=<uid>
+
+
+Set Bucket Quota
+----------------
+
+Bucket quotas apply to the buckets owned by the specified ``uid``. They are
+independent of the user. ::
+
+	radosgw-admin quota set --uid=<uid> --quota-scope=bucket [--max-objects=<num objects>] [--max-size=<max size]
+
+A negative value for num objects and / or max size means that the
+specific quota attribute check is disabled.
+
+
+Enable/Disable Bucket Quota
+---------------------------
+
+Once you set a bucket quota, you may enable it. For example:: 
+
+	radosgw-admin quota enable --quota-scope=bucket --uid=<uid>
+
+You may disable an enabled bucket quota. For example:: 
+
+	radosgw-admin quota disable --quota-scope=bucket --uid=<uid>
+
+
+Get Quota Settings
+------------------
+
+You may access each user's quota settings via the user information
+API. To read user quota setting information with the CLI interface, 
+execute the following::
+
+	radosgw-admin user info --uid=<uid>
+
+
+Update Quota Stats
+------------------
+
+Quota stats get updated asynchronously. You can update quota
+statistics for all users and all buckets manually to retrieve
+the latest quota stats. ::
+
+	radosgw-admin user stats --uid=<uid> --sync-stats
+
+.. _rgw_user_usage_stats:
+
+Get User Usage Stats
+--------------------
+
+To see how much of the quota a user has consumed, execute the following::
+
+	radosgw-admin user stats --uid=<uid>
+
+.. note:: You should execute ``radosgw-admin user stats`` with the 
+   ``--sync-stats`` option to receive the latest data.
+
+Default Quotas
+--------------
+
+You can set default quotas in the config.  These defaults are used when
+creating a new user and have no effect on existing users. If the
+relevant default quota is set in config, then that quota is set on the
+new user, and that quota is enabled.  See ``rgw bucket default quota max objects``,
+``rgw bucket default quota max size``, ``rgw user default quota max objects``, and
+``rgw user default quota max size`` in `Ceph Object Gateway Config Reference`_
+
+Quota Cache
+-----------
+
+Quota statistics are cached on each RGW instance.  If there are multiple
+instances, then the cache can keep quotas from being perfectly enforced, as
+each instance will have a different view of quotas.  The options that control
+this are ``rgw bucket quota ttl``, ``rgw user quota bucket sync interval`` and
+``rgw user quota sync interval``.  The higher these values are, the more
+efficient quota operations are, but the more out-of-sync multiple instances
+will be.  The lower these values are, the closer to perfect enforcement
+multiple instances will achieve.  If all three are 0, then quota caching is
+effectively disabled, and multiple instances will have perfect quota
+enforcement.  See `Ceph Object Gateway Config Reference`_
+
+Reading / Writing Global Quotas
+-------------------------------
+
+You can read and write global quota settings in the period configuration. To
+view the global quota settings::
+
+	radosgw-admin global quota get
+
+The global quota settings can be manipulated with the ``global quota``
+counterparts of the ``quota set``, ``quota enable``, and ``quota disable``
+commands. ::
+
+	radosgw-admin global quota set --quota-scope bucket --max-objects 1024
+	radosgw-admin global quota enable --quota-scope bucket
+
+.. note:: In a multisite configuration, where there is a realm and period
+   present, changes to the global quotas must be committed using ``period
+   update --commit``. If there is no period present, the rados gateway(s) must
+   be restarted for the changes to take effect.
+
+
+Rate Limit Management
+=====================
+
+The Ceph Object Gateway makes it possible to set rate limits on users and
+buckets.  "Rate limit" includes the maximum number of read operations (read
+ops) and write operations (write ops) per minute and the number of bytes per
+minute that can be written or read per user or per bucket.
+
+Operations that use the ``GET`` method or the ``HEAD`` method in their REST
+requests are "read requests". All other requests are "write requests".  
+
+Each object gateway tracks per-user metrics separately from bucket metrics.
+These metrics are not shared with other gateways. The configured limits should
+be divided by the number of active object gateways. For example, if "user A" is
+to be be limited to 10 ops per minute and there are two object gateways in the
+cluster, then the limit on "user A" should be ``5`` (10 ops per minute / 2
+RGWs).  If the requests are **not** balanced between RGWs, the rate limit might
+be underutilized. For example: if the ops limit is ``5`` and there are two
+RGWs, **but** the Load Balancer sends load to only one of those RGWs, the
+effective limit is 5 ops, because this limit is enforced per RGW. If the rate
+limit that has been set for the bucket has been reached but the rate limit that
+has been set for the user has not been reached, then the request is cancelled.
+The contrary holds as well: if the rate limit that has been set for the user
+has been reached but the rate limit that has been set for the bucket has not
+been reached, then the request is cancelled.
+
+The accounting of bandwidth happens only after a request has been accepted.
+This means that requests will proceed even if the bucket rate limit or user
+rate limit is reached during the execution of the request. The RGW keeps track
+of a "debt" consisting of bytes used in excess of the configured value; users
+or buckets that incur this kind of debt are prevented  from sending more
+requests until the "debt" has been repaid. The maximum size of the "debt" is
+twice the max-read/write-bytes per minute. If "user A" is subject to a 1-byte
+read limit per minute and they attempt to GET an object that is 1 GB in size,
+then the ``GET`` action will fail. After "user A" has completed this 1 GB
+operation, RGW blocks the user's requests for up to two minutes. After this
+time has elapsed, "user A" will be able to send ``GET`` requests again.
+
+
+- **Bucket:** The ``--bucket`` option allows you to specify a rate limit for a
+  bucket.
+
+- **User:** The ``--uid`` option allows you to specify a rate limit for a
+  user.
+
+- **Maximum Read Ops:** The ``--max-read-ops`` setting allows you to specify
+  the maximum number of read ops per minute per RGW. A 0 value disables this setting (which means unlimited access).
+  
+- **Maximum Read Bytes:** The ``--max-read-bytes`` setting allows you to specify
+  the maximum number of read bytes per minute per RGW. A 0 value disables this setting (which means unlimited access).
+
+- **Maximum Write Ops:** The ``--max-write-ops`` setting allows you to specify
+  the maximum number of write ops per minute per RGW. A 0 value disables this setting (which means unlimited access).
+  
+- **Maximum Write Bytes:** The ``--max-write-bytes`` setting allows you to specify
+  the maximum number of write bytes per minute per RGW. A 0 value disables this setting (which means unlimited access).
+ 
+- **Rate Limit Scope:** The ``--ratelimit-scope`` option sets the scope for the rate limit.
+  The options are ``bucket`` , ``user`` and ``anonymous``. Bucket rate limit apply to buckets. 
+  The user rate limit applies to a user. Anonymous applies to an unauthenticated user.
+  Anonymous scope is only available for global rate limit.
+
+
+Set User Rate Limit
+-------------------
+
+Before you enable a rate limit, you must first set the rate limit parameters.
+For example:: 
+
+	radosgw-admin ratelimit set --ratelimit-scope=user --uid=<uid> <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
+  [--max-write-ops=<num ops>] [--max-write-bytes=<num bytes>]>
+
+For example:: 
+
+	radosgw-admin ratelimit set --ratelimit-scope=user --uid=johndoe --max-read-ops=1024 --max-write-bytes=10240
+
+
+A 0 value for num ops and / or num bytes means that the
+specific rate limit attribute check is disabled.
+
+Get User Rate Limit
+-------------------
+
+Get the current configured rate limit parameters
+For example:: 
+
+	radosgw-admin ratelimit get --ratelimit-scope=user --uid=<uid>
+
+For example:: 
+
+	radosgw-admin ratelimit get --ratelimit-scope=user --uid=johndoe
+
+
+A 0 value for num ops and / or num bytes means that the
+specific rate limit attribute check is disabled.
+
+
+Enable/Disable User Rate Limit
+------------------------------
+
+Once you set a user rate limit, you may enable it. For example:: 
+
+	radosgw-admin ratelimit enable --ratelimit-scope=user --uid=<uid>
+
+You may disable an enabled user rate limit. For example:: 
+
+	radosgw-admin ratelimit disable --ratelimit-scope=user --uid=johndoe
+
+
+Set Bucket Rate Limit
+---------------------
+
+Before you enable a rate limit, you must first set the rate limit parameters.
+For example:: 
+
+	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=<bucket> <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
+  [--max-write-ops=<num ops>] [--max-write-bytes=<num bytes>]>
+
+For example:: 
+
+	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=mybucket --max-read-ops=1024 --max-write-bytes=10240
+
+
+A 0 value for num ops and / or num bytes means that the
+specific rate limit attribute check is disabled.
+
+Get Bucket Rate Limit
+---------------------
+
+Get the current configured rate limit parameters
+For example:: 
+
+	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=<bucket>
+
+For example:: 
+
+	radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket=mybucket
+
+
+A 0 value for num ops and / or num bytes means that the
+specific rate limit attribute check is disabled.
+
+
+Enable/Disable Bucket Rate Limit
+--------------------------------
+
+Once you set a bucket rate limit, you may enable it. For example:: 
+
+	radosgw-admin ratelimit enable --ratelimit-scope=bucket --bucket=<bucket>
+
+You may disable an enabled bucket rate limit. For example:: 
+
+	radosgw-admin ratelimit disable --ratelimit-scope=bucket --uid=mybucket
+
+
+Reading / Writing Global Rate Limit Configuration
+-------------------------------------------------
+
+You can read and write global rate limit settings in the period configuration. To
+view the global rate limit settings::
+
+	radosgw-admin global ratelimit get
+
+The global rate limit settings can be manipulated with the ``global ratelimit``
+counterparts of the ``ratelimit set``, ``ratelimit enable``, and ``ratelimit disable``
+commands. Per user and per bucket ratelimit configuration is overriding the global configuration::
+
+	radosgw-admin global ratelimit set --ratelimit-scope bucket --max-read-ops=1024
+	radosgw-admin global ratelimit enable --ratelimit-scope bucket
+
+The global rate limit can configure rate limit scope for all authenticated users::
+
+  radosgw-admin global ratelimit set --ratelimit-scope user --max-read-ops=1024
+  radosgw-admin global ratelimit enable --ratelimit-scope user
+
+The global rate limit can configure rate limit scope for all unauthenticated users::
+  
+  radosgw-admin global ratelimit set --ratelimit-scope=anonymous --max-read-ops=1024
+  radosgw-admin global ratelimit enable --ratelimit-scope=anonymous
+
+.. note:: In a multisite configuration, where there is a realm and period
+   present, changes to the global rate limit must be committed using ``period
+   update --commit``. If there is no period present, the rados gateway(s) must
+   be restarted for the changes to take effect.
+
+Usage
+=====
+
+The Ceph Object Gateway logs usage for each user. You can track
+user usage within date ranges too.
+
+- Add ``rgw enable usage log = true`` in [client.rgw] section of ceph.conf and restart the radosgw service. 
+
+Options include: 
+
+- **Start Date:** The ``--start-date`` option allows you to filter usage
+  stats from a particular start date and an optional start time
+  (**format:** ``yyyy-mm-dd [HH:MM:SS]``).
+
+- **End Date:** The ``--end-date`` option allows you to filter usage up
+  to a particular date and an optional end time
+  (**format:** ``yyyy-mm-dd [HH:MM:SS]``). 
+  
+- **Log Entries:** The ``--show-log-entries`` option allows you to specify
+  whether or not to include log entries with the usage stats 
+  (options: ``true`` | ``false``).
+
+.. note:: You may specify time with minutes and seconds, but it is stored 
+   with 1 hour resolution.
+
+
+Show Usage
+----------
+
+To show usage statistics, specify the ``usage show``. To show usage for a
+particular user, you must specify a user ID. You may also specify a start date,
+end date, and whether or not to show log entries.::
+
+	radosgw-admin usage show --uid=johndoe --start-date=2012-03-01 --end-date=2012-04-01
+
+You may also show a summary of usage information for all users by omitting a user ID. ::
+
+	radosgw-admin usage show --show-log-entries=false
+
+
+Trim Usage
+----------
+
+With heavy use, usage logs can begin to take up storage space. You can trim
+usage logs for all users and for specific users. You may also specify date
+ranges for trim operations. ::
+
+	radosgw-admin usage trim --start-date=2010-01-01 --end-date=2010-12-31
+	radosgw-admin usage trim --uid=johndoe	
+	radosgw-admin usage trim --uid=johndoe --end-date=2013-12-31
+
+
+.. _radosgw-admin: ../../man/8/radosgw-admin/
+.. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/
+.. _Ceph Object Gateway Config Reference: ../config-ref/
diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst
new file mode 100644
index 000000000..0974b95c5
--- /dev/null
+++ b/doc/radosgw/adminops.rst
@@ -0,0 +1,2166 @@
+.. _radosgw admin ops:
+
+==================
+ Admin Operations
+==================
+
+An admin API request will be done on a URI that starts with the configurable 'admin'
+resource entry point. Authorization for the admin API duplicates the S3 authorization
+mechanism. Some operations require that the user holds special administrative capabilities.
+The response entity type (XML or JSON) may be specified as the 'format' option in the
+request and defaults to JSON if not specified.
+
+Info
+====
+
+Get RGW cluster/endpoint information.
+
+:caps: info=read
+
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/info?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None.
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains an ``info`` section.
+
+``info``
+
+:Description: A container for all returned information.
+:Type: Container
+
+``cluster_id``
+
+:Description: The (typically unique) identifier for the controlling
+	      backing store for the RGW cluster.  In the typical case,
+	      this is value returned from librados::rados::cluster_fsid().
+:Type: String
+:Parent: ``info``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+
+Get Usage
+=========
+
+Request bandwidth usage information.
+
+Note: this feature is disabled by default, can be enabled by setting ``rgw
+enable usage log = true`` in the appropriate section of ceph.conf. For changes
+in ceph.conf to take effect, radosgw process restart is needed.
+
+:caps: usage=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/usage?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested. If not specified will apply to all users.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``start``
+
+:Description: Date and (optional) time that specifies the start time of the requested data.
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+``end``
+
+:Description: Date and (optional) time that specifies the end time of the requested data (non-inclusive).
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+
+``show-entries``
+
+:Description: Specifies whether data entries should be returned.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+
+``show-summary``
+
+:Description: Specifies whether data summary should be returned.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the requested information.
+
+``usage``
+
+:Description: A container for the usage information.
+:Type: Container
+
+``entries``
+
+:Description: A container for the usage entries information.
+:Type: Container
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``owner``
+
+:Description: The name of the user that owns the buckets.
+:Type: String
+
+``bucket``
+
+:Description: The bucket name.
+:Type: String
+
+``time``
+
+:Description: Time lower bound for which data is being specified (rounded to the beginning of the first relevant hour).
+:Type: String
+
+``epoch``
+
+:Description: The time specified in seconds since 1/1/1970.
+:Type: String
+
+``categories``
+
+:Description: A container for stats categories.
+:Type: Container
+
+``entry``
+
+:Description: A container for stats entry.
+:Type: Container
+
+``category``
+
+:Description: Name of request category for which the stats are provided.
+:Type: String
+
+``bytes_sent``
+
+:Description: Number of bytes sent by the RADOS Gateway.
+:Type: Integer
+
+``bytes_received``
+
+:Description: Number of bytes received by the RADOS Gateway.
+:Type: Integer
+
+``ops``
+
+:Description: Number of operations.
+:Type: Integer
+
+``successful_ops``
+
+:Description: Number of successful operations.
+:Type: Integer
+
+``summary``
+
+:Description: A container for stats summary.
+:Type: Container
+
+``total``
+
+:Description: A container for stats summary aggregated total.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Trim Usage
+==========
+
+Remove usage information. With no dates specified, removes all usage
+information.
+
+Note: this feature is disabled by default, can be enabled by setting ``rgw
+enable usage log = true`` in the appropriate section of ceph.conf. For changes
+in ceph.conf to take effect, radosgw process restart is needed.
+
+:caps: usage=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/usage?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested. If not specified will apply to all users.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``start``
+
+:Description: Date and (optional) time that specifies the start time of the requested data.
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+``end``
+
+:Description: Date and (optional) time that specifies the end time of the requested data (none inclusive).
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+
+``remove-all``
+
+:Description: Required when uid is not specified, in order to acknowledge multi user data removal.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Get User Info
+=============
+
+Get user information.
+
+:caps: users=read
+
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Create User
+===========
+
+Create a new user. By default, a S3 key pair will be created automatically
+and returned in the response. If only one of ``access-key`` or ``secret-key``
+is provided, the omitted key will be automatically generated. By default, a
+generated key is added to the keyring without replacing an existing key pair.
+If ``access-key`` is specified and refers to an existing key owned by the user
+then it will be modified.
+
+.. versionadded:: Luminous
+
+A ``tenant`` may either be specified as a part of uid or as an additional
+request param.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be created.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+A tenant name may also specified as a part of ``uid``, by following the syntax
+``tenant$user``, refer to :ref:`Multitenancy <rgw-multitenancy>` for more details.
+
+``display-name``
+
+:Description: The display name of the user to be created.
+:Type: String
+:Example: ``foo user``
+:Required: Yes
+
+
+``email``
+
+:Description: The email address associated with the user.
+:Type: String
+:Example: ``foo@bar.com``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3`` [``s3``]
+:Required: No
+
+``access-key``
+
+:Description: Specify access key.
+:Type: String
+:Example: ``ABCD0EF12GHIJ2K34LMN``
+:Required: No
+
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``user-caps``
+
+:Description: User capabilities.
+:Type: String
+:Example: ``usage=read, write; users=read``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+``max-buckets``
+
+:Description: Specify the maximum number of buckets the user can own.
+:Type: Integer
+:Example: 500 [1000]
+:Required: No
+
+``suspended``
+
+:Description: Specify whether the user should be suspended.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+.. versionadded:: Jewel
+
+``tenant``
+
+:Description: the Tenant under which a user is a part of.
+:Type: string
+:Example: tenant1
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``tenant``
+
+:Description: The tenant which user is a part of.
+:Type: String
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``UserExists``
+
+:Description: Attempt to create existing user.
+:Code: 409 Conflict
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+``EmailExists``
+
+:Description: Provided email address exists.
+:Code: 409 Conflict
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+
+Modify User
+===========
+
+Modify a user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be modified.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``display-name``
+
+:Description: The display name of the user to be modified.
+:Type: String
+:Example: ``foo user``
+:Required: No
+
+``email``
+
+:Description: The email address to be associated with the user.
+:Type: String
+:Example: ``foo@bar.com``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``access-key``
+
+:Description: Specify access key.
+:Type: String
+:Example: ``ABCD0EF12GHIJ2K34LMN``
+:Required: No
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3``
+:Required: No
+
+``max-buckets``
+
+:Description: Specify the maximum number of buckets the user can own.
+:Type: Integer
+:Example: 500 [1000]
+:Required: No
+
+``suspended``
+
+:Description: Specify whether the user should be suspended.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+``op-mask``
+
+:Description: The op-mask of the user to be modified.
+:Type: String
+:Example: ``read, write, delete, *``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+``EmailExists``
+
+:Description: Provided email address exists.
+:Code: 409 Conflict
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+Remove User
+===========
+
+Remove an existing user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be removed.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes.
+
+``purge-data``
+
+:Description: When specified the buckets and objects belonging
+              to the user will also be removed.
+:Type: Boolean
+:Example: True
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Create Subuser
+==============
+
+Create a new subuser (primarily useful for clients using the Swift API).
+Note that in general for a subuser to be useful, it must be granted
+permissions by specifying ``access``. As with user creation if
+``subuser`` is specified without ``secret``, then a secret key will
+be automatically generated.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?subuser&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which a subuser is to  be created.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+``subuser``
+
+:Description: Specify the subuser ID to be created.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift (default), s3.
+:Type: String
+:Example: ``swift`` [``swift``]
+:Required: No
+
+``access``
+
+:Description: Set access permissions for sub-user, should be one
+              of ``read, write, readwrite, full``.
+:Type: String
+:Example: ``read``
+:Required: No
+
+``generate-secret``
+
+:Description: Generate the secret key.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the subuser information.
+
+
+``subusers``
+
+:Description: Subusers associated with the user account.
+:Type: Container
+
+``id``
+
+:Description: Subuser id.
+:Type: String
+:Parent: ``subusers``
+
+``permissions``
+
+:Description: Subuser access to user account.
+:Type: String
+:Parent: ``subusers``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``SubuserExists``
+
+:Description: Specified subuser exists.
+:Code: 409 Conflict
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidAccess``
+
+:Description: Invalid subuser access specified.
+:Code: 400 Bad Request
+
+Modify Subuser
+==============
+
+Modify an existing subuser
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/user?subuser&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which the subuser is to be modified.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``subuser``
+
+:Description: The subuser ID to be modified.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``generate-secret``
+
+:Description: Generate a new secret key for the subuser,
+              replacing the existing key.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``secret``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift (default), s3 .
+:Type: String
+:Example: ``swift`` [``swift``]
+:Required: No
+
+``access``
+
+:Description: Set access permissions for sub-user, should be one
+              of ``read, write, readwrite, full``.
+:Type: String
+:Example: ``read``
+:Required: No
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the subuser information.
+
+
+``subusers``
+
+:Description: Subusers associated with the user account.
+:Type: Container
+
+``id``
+
+:Description: Subuser id.
+:Type: String
+:Parent: ``subusers``
+
+``permissions``
+
+:Description: Subuser access to user account.
+:Type: String
+:Parent: ``subusers``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidAccess``
+
+:Description: Invalid subuser access specified.
+:Code: 400 Bad Request
+
+Remove Subuser
+==============
+
+Remove an existing subuser
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?subuser&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which the subuser is to be removed.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+``subuser``
+
+:Description: The subuser ID to be removed.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``purge-keys``
+
+:Description: Remove keys belonging to the subuser.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+None.
+
+Create Key
+==========
+
+Create a new key. If a ``subuser`` is specified then by default created keys
+will be swift type. If only one of ``access-key`` or ``secret-key`` is provided the
+committed key will be automatically generated, that is if only ``secret-key`` is
+specified then ``access-key`` will be automatically generated. By default, a
+generated key is added to the keyring without replacing an existing key pair.
+If ``access-key`` is specified and refers to an existing key owned by the user
+then it will be modified. The response is a container listing all keys of the same
+type as the key created. Note that when creating a swift key, specifying the option
+``access-key`` will have no effect. Additionally, only one swift key may be held by
+each user or subuser.
+
+:caps: users=write
+
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?key&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to receive the new key.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``subuser``
+
+:Description: The subuser ID to receive the new key.
+:Type: String
+:Example: ``sub_foo``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3`` [``s3``]
+:Required: No
+
+``access-key``
+
+:Description: Specify the access key.
+:Type: String
+:Example: ``AB01C2D3EF45G6H7IJ8K``
+:Required: No
+
+``secret-key``
+
+:Description: Specify the secret key.
+:Type: String
+:Example: ``0ab/CdeFGhij1klmnopqRSTUv1WxyZabcDEFgHij``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``keys``
+
+:Description: Keys of type created associated with this user account.
+:Type: Container
+
+``user``
+
+:Description: The user account associated with the key.
+:Type: String
+:Parent: ``keys``
+
+``access-key``
+
+:Description: The access key.
+:Type: String
+:Parent: ``keys``
+
+``secret-key``
+
+:Description: The secret key
+:Type: String
+:Parent: ``keys``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+Remove Key
+==========
+
+Remove an existing key.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?key&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``access-key``
+
+:Description: The S3 access key belonging to the S3 key pair to remove.
+:Type: String
+:Example: ``AB01C2D3EF45G6H7IJ8K``
+:Required: Yes
+
+``uid``
+
+:Description: The user to remove the key from.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``subuser``
+
+:Description: The subuser to remove the key from.
+:Type: String
+:Example: ``sub_foo``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be removed, options are: swift, s3.
+              NOTE: Required to remove swift key.
+:Type: String
+:Example: ``swift``
+:Required: No
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Get Bucket Info
+===============
+
+Get information about a subset of the existing buckets. If ``uid`` is specified
+without ``bucket`` then all buckets belonging to the user will be returned. If
+``bucket`` alone is specified, information for that particular bucket will be
+retrieved.
+
+:caps: buckets=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to return info on.
+:Type: String
+:Example: ``foo_bucket``
+:Required: No
+
+``uid``
+
+:Description: The user to retrieve bucket information for.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``stats``
+
+:Description: Return bucket statistics.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful the request returns a buckets container containing
+the desired bucket information.
+
+``stats``
+
+:Description: Per bucket information.
+:Type: Container
+
+``buckets``
+
+:Description: Contains a list of one or more bucket containers.
+:Type: Container
+
+``bucket``
+
+:Description: Container for single bucket information.
+:Type: Container
+:Parent: ``buckets``
+
+``name``
+
+:Description: The name of the bucket.
+:Type: String
+:Parent: ``bucket``
+
+``pool``
+
+:Description: The pool the bucket is stored in.
+:Type: String
+:Parent: ``bucket``
+
+``id``
+
+:Description: The unique bucket id.
+:Type: String
+:Parent: ``bucket``
+
+``marker``
+
+:Description: Internal bucket tag.
+:Type: String
+:Parent: ``bucket``
+
+``owner``
+
+:Description: The user id of the bucket owner.
+:Type: String
+:Parent: ``bucket``
+
+``usage``
+
+:Description: Storage usage information.
+:Type: Container
+:Parent: ``bucket``
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+:Parent: ``bucket``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IndexRepairFailed``
+
+:Description: Bucket index repair failed.
+:Code: 409 Conflict
+
+Check Bucket Index
+==================
+
+Check the index of an existing bucket. NOTE: to check multipart object
+accounting with ``check-objects``, ``fix`` must be set to True.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?index&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to return info on.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``check-objects``
+
+:Description: Check multipart object accounting.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``fix``
+
+:Description: Also fix the bucket index when checking.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IndexRepairFailed``
+
+:Description: Bucket index repair failed.
+:Code: 409 Conflict
+
+Remove Bucket
+=============
+
+Delete an existing bucket.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/bucket?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to remove.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``purge-objects``
+
+:Description: Remove a buckets objects before deletion.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketNotEmpty``
+
+:Description: Attempted to delete non-empty bucket.
+:Code: 409 Conflict
+
+``ObjectRemovalFailed``
+
+:Description: Unable to remove objects.
+:Code: 409 Conflict
+
+Unlink Bucket
+=============
+
+Unlink a bucket from a specified user. Primarily useful for changing
+bucket ownership.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/bucket?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to unlink.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``uid``
+
+:Description: The user ID to unlink the bucket from.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketUnlinkFailed``
+
+:Description: Unable to unlink bucket from specified user.
+:Code: 409 Conflict
+
+Link Bucket
+===========
+
+Link a bucket to a specified user, unlinking the bucket from
+any previous user.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/bucket?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket name to unlink.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``bucket-id``
+
+:Description: The bucket id to unlink.
+:Type: String
+:Example: ``dev.6607669.420``
+:Required: No
+
+``uid``
+
+:Description: The user ID to link the bucket to.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: Container for single bucket information.
+:Type: Container
+
+``name``
+
+:Description: The name of the bucket.
+:Type: String
+:Parent: ``bucket``
+
+``pool``
+
+:Description: The pool the bucket is stored in.
+:Type: String
+:Parent: ``bucket``
+
+``id``
+
+:Description: The unique bucket id.
+:Type: String
+:Parent: ``bucket``
+
+``marker``
+
+:Description: Internal bucket tag.
+:Type: String
+:Parent: ``bucket``
+
+``owner``
+
+:Description: The user id of the bucket owner.
+:Type: String
+:Parent: ``bucket``
+
+``usage``
+
+:Description: Storage usage information.
+:Type: Container
+:Parent: ``bucket``
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+:Parent: ``bucket``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketUnlinkFailed``
+
+:Description: Unable to unlink bucket from specified user.
+:Code: 409 Conflict
+
+``BucketLinkFailed``
+
+:Description: Unable to link bucket to specified user.
+:Code: 409 Conflict
+
+Remove Object
+=============
+
+Remove an existing object. NOTE: Does not require owner to be non-suspended.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/bucket?object&format=json HTTP/1.1
+	Host: {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be removed.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to remove.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+``ObjectRemovalFailed``
+
+:Description: Unable to remove objects.
+:Code: 409 Conflict
+
+
+
+Get Bucket or Object Policy
+===========================
+
+Read the policy of an object or bucket.
+
+:caps: buckets=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?policy&format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to read the policy from.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to read the policy from.
+:Type: String
+:Example: ``foo.txt``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns the object or bucket policy
+
+``policy``
+
+:Description: Access control policy.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IncompleteBody``
+
+:Description: Either bucket was not specified for a bucket policy request or bucket
+              and object were not specified for an object policy request.
+:Code: 400 Bad Request
+
+Add A User Capability
+=====================
+
+Add an administrative capability to a specified user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?caps&format=json HTTP/1.1
+	Host: {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to add an administrative capability to.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``user-caps``
+
+:Description: The administrative capability to add to the user.
+:Type: String
+:Example: ``usage=read,write;user=write``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user's capabilities.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+Example Request
+~~~~~~~~~~~~~~~
+
+::
+
+	PUT /{admin}/user?caps&user-caps=usage=read,write;user=write&format=json HTTP/1.1
+	Host: {fqdn}
+	Content-Type: text/plain
+	Authorization: {your-authorization-token}
+
+
+
+Remove A User Capability
+========================
+
+Remove an administrative capability from a specified user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?caps&format=json HTTP/1.1
+	Host: {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to remove an administrative capability from.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``user-caps``
+
+:Description: The administrative capabilities to remove from the user.
+:Type: String
+:Example: ``usage=read, write``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user's capabilities.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidCapability``
+
+:Description: Attempt to remove an invalid admin capability.
+:Code: 400 Bad Request
+
+``NoSuchCap``
+
+:Description: User does not possess specified capability.
+:Code: 404 Not Found
+
+
+Quotas
+======
+
+The Admin Operations API enables you to set quotas on users and on bucket owned
+by users. See `Quota Management`_ for additional details. Quotas include the
+maximum number of objects in a bucket and the maximum storage size in megabytes.
+
+To view quotas, the user must have a ``users=read`` capability. To set,
+modify or disable a quota, the user must have ``users=write`` capability.
+See the `Admin Guide`_ for details.
+
+Valid parameters for quotas include:
+
+- **Bucket:** The ``bucket`` option allows you to specify a quota for
+  buckets owned by a user.
+
+- **Maximum Objects:** The ``max-objects`` setting allows you to specify
+  the maximum number of objects. A negative value disables this setting.
+
+- **Maximum Size:** The ``max-size`` option allows you to specify a quota
+  for the maximum number of bytes. The ``max-size-kb`` option allows you
+  to specify it in KiB. A negative value disables this setting.
+
+- **Quota Type:** The ``quota-type`` option sets the scope for the quota.
+  The options are ``bucket`` and ``user``.
+
+- **Enable/Disable Quota:** The ``enabled`` option specifies whether the
+  quota should be enabled. The value should be either 'True' or 'False'.
+
+Get User Quota
+~~~~~~~~~~~~~~
+
+To get a quota, the user must have ``users`` capability set with ``read``
+permission. ::
+
+	GET /admin/user?quota&uid=<uid>&quota-type=user
+
+
+Set User Quota
+~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``users`` capability set with ``write``
+permission. ::
+
+	PUT /admin/user?quota&uid=<uid>&quota-type=user
+
+
+The content must include a JSON representation of the quota settings
+as encoded in the corresponding read operation.
+
+
+Get Bucket Quota
+~~~~~~~~~~~~~~~~
+
+To get a quota, the user must have ``users`` capability set with ``read``
+permission. ::
+
+	GET /admin/user?quota&uid=<uid>&quota-type=bucket
+
+
+Set Bucket Quota
+~~~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``users`` capability set with ``write``
+permission. ::
+
+	PUT /admin/user?quota&uid=<uid>&quota-type=bucket
+
+The content must include a JSON representation of the quota settings
+as encoded in the corresponding read operation.
+
+
+Set Quota for an Individual Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``buckets`` capability set with ``write``
+permission. ::
+
+	PUT /admin/bucket?quota&uid=<uid>&bucket=<bucket-name>
+
+The content must include a JSON representation of the quota settings
+as mentioned in Set Bucket Quota section above.
+
+
+
+Rate Limit
+==========
+
+The Admin Operations API enables you to set and get ratelimit configurations on users and on bucket and global rate limit configurations. See `Rate Limit Management`_ for additional details. 
+Rate Limit includes the maximum number of operations and/or bytes per minute, separated by read and/or write, to a bucket and/or by a user and the maximum storage size in megabytes.
+
+To view rate limit, the user must have a ``ratelimit=read`` capability. To set,
+modify or disable a ratelimit, the user must have ``ratelimit=write`` capability.
+See the `Admin Guide`_ for details.
+
+Valid parameters for quotas include:
+
+- **Bucket:** The ``bucket`` option allows you to specify a rate limit for
+  a bucket.
+
+- **User:** The ``uid`` option allows you to specify a rate limit for a user.
+
+- **Maximum Read Bytes:** The ``max-read-bytes`` setting allows you to specify
+  the maximum number of read bytes per minute. A 0 value disables this setting.
+
+- **Maximum Write Bytes:** The ``max-write-bytes`` setting allows you to specify
+  the maximum number of write bytes per minute. A 0 value disables this setting.
+
+- **Maximum Read Ops:** The ``max-read-ops`` setting allows you to specify
+  the maximum number of read ops per minute. A 0 value disables this setting.
+
+- **Maximum Write Ops:** The ``max-write-ops`` setting allows you to specify
+  the maximum number of write ops per minute. A 0 value disables this setting.
+
+- **Global:** The ``global`` option allows you to specify a global rate limit.
+  The value should be either 'True' or 'False'.
+
+- **Rate Limit Scope:** The ``ratelimit-scope`` option sets the scope for the rate limit.
+  The options are ``bucket`` , ``user`` and ``anonymous``.
+  ``anonymous`` is only valid for setting global configuration
+
+- **Enable/Disable Rate Limit:** The ``enabled`` option specifies whether the
+  rate limit should be enabled.  The value should be either 'True' or 'False'.
+
+Get User Rate Limit
+~~~~~~~~~~~~~~~~~~~
+
+To get a rate limit, the user must have ``ratelimit`` capability set with ``read``
+permission. ::
+
+	GET /{admin}/ratelimit?ratelimit-scope=user&uid=<uid>
+
+
+Set User Rate Limit
+~~~~~~~~~~~~~~~~~~~
+
+To set a rate limit, the user must have ``ratelimit`` capability set with ``write``
+permission. ::
+
+	POST /{admin}/ratelimit?ratelimit-scope=user&uid=<uid><[&max-read-bytes=<bytes>][&max-write-bytes=<bytes>][&max-read-ops=<ops>][&max-write-ops=<ops>][enabled=<True|False>]>
+
+
+
+Get Bucket Rate Limit
+~~~~~~~~~~~~~~~~~~~~~
+
+To get a rate limit, the user must have ``users`` capability set with ``read``
+permission. ::
+
+	GET /{admin}/ratelimit?bucket=<bucket>&ratelimit-scope=bucket
+
+
+
+Set Rate Limit for an Individual Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a rate limit, the user must have ``ratelimit`` capability set with ``write``
+permission. ::
+
+	POST /{admin}/ratelimit?bucket=<bucket-name>&ratelimit-scope=bucket<[&max-read-bytes=<bytes>][&max-write-bytes=<bytes>][&max-read-ops=<ops>][&max-write-ops=<ops>]>
+
+
+
+Get Global Rate Limit
+~~~~~~~~~~~~~~~~~~~~~
+
+To get a global rate limit, the user must have ``ratelimit`` capability set with ``read``
+permission. ::
+
+	GET /{admin}/ratelimit?global=<True|False>
+
+
+
+Set Global User Rate Limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a rate limit, the user must have ``ratelimit`` capability set with ``write``
+permission. ::
+
+	POST /{admin}/ratelimit?ratelimit-scope=user&global=<True|False><[&max-read-bytes=<bytes>][&max-write-bytes=<bytes>][&max-read-ops=<ops>][&max-write-ops=<ops>][enabled=<True|False>]>
+
+
+
+Set Global Rate Limit Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a rate limit, the user must have ``ratelimit`` capability set with ``write``
+permission. ::
+
+	POST /{admin}/ratelimit?ratelimit-scope=bucket&global=<True|False><[&max-read-bytes=<bytes>][&max-write-bytes=<bytes>][&max-read-ops=<ops>][&max-write-ops=<ops>]>
+
+
+
+Set Global Anonymous User Rate Limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a rate limit, the user must have ``ratelimit`` capability set with ``write``
+permission. ::
+
+	POST /{admin}/ratelimit?ratelimit-scope=anon&global=<True|False><[&max-read-bytes=<bytes>][&max-write-bytes=<bytes>][&max-read-ops=<ops>][&max-write-ops=<ops>][enabled=<True|False>]>
+
+
+
+Standard Error Responses
+========================
+
+``AccessDenied``
+
+:Description: Access denied.
+:Code: 403 Forbidden
+
+``InternalError``
+
+:Description: Internal server error.
+:Code: 500 Internal Server Error
+
+``NoSuchUser``
+
+:Description: User does not exist.
+:Code: 404 Not Found
+
+``NoSuchBucket``
+
+:Description: Bucket does not exist.
+:Code: 404 Not Found
+
+``NoSuchKey``
+
+:Description: No such access key.
+:Code: 404 Not Found
+
+
+
+
+Binding libraries
+========================
+
+``Golang``
+
+ - `ceph/go-ceph`_
+ - `IrekFasikhov/go-rgwadmin`_
+ - `QuentinPerez/go-radosgw`_
+ 
+``Java``
+
+ - `twonote/radosgw-admin4j`_
+
+``PHP``
+
+ - `lbausch/php-ceph-radosgw-admin`_
+ - `myENA/php-rgw-api`_
+
+``Python``
+
+ - `UMIACS/rgwadmin`_
+ - `valerytschopp/python-radosgw-admin`_
+
+
+
+.. _Admin Guide: ../admin
+.. _Quota Management: ../admin#quota-management
+.. _IrekFasikhov/go-rgwadmin: https://github.com/IrekFasikhov/go-rgwadmin
+.. _QuentinPerez/go-radosgw: https://github.com/QuentinPerez/go-radosgw
+.. _ceph/go-ceph: https://github.com/ceph/go-ceph/
+.. _Rate Limit Management: ../admin#rate-limit-management
+.. _IrekFasikhov/go-rgwadmin: https://github.com/IrekFasikhov/go-rgwadmin
+.. _QuentinPerez/go-radosgw: https://github.com/QuentinPerez/go-radosgw
+.. _twonote/radosgw-admin4j: https://github.com/twonote/radosgw-admin4j
+.. _lbausch/php-ceph-radosgw-admin: https://github.com/lbausch/php-ceph-radosgw-admin
+.. _myENA/php-rgw-api: https://github.com/myENA/php-rgw-api
+.. _UMIACS/rgwadmin: https://github.com/UMIACS/rgwadmin
+.. _valerytschopp/python-radosgw-admin: https://github.com/valerytschopp/python-radosgw-admin
diff --git a/doc/radosgw/api.rst b/doc/radosgw/api.rst
new file mode 100644
index 000000000..cb31284e0
--- /dev/null
+++ b/doc/radosgw/api.rst
@@ -0,0 +1,16 @@
+.. _radosgw api:
+
+===============
+librgw (Python)
+===============
+
+.. highlight:: python
+
+The `rgw` python module provides file-like access to rgw.
+
+API Reference
+=============
+
+.. automodule:: rgw
+    :members: LibRGWFS, FileHandle
+
diff --git a/doc/radosgw/archive-sync-module.rst b/doc/radosgw/archive-sync-module.rst
new file mode 100644
index 000000000..b121ee6b1
--- /dev/null
+++ b/doc/radosgw/archive-sync-module.rst
@@ -0,0 +1,44 @@
+===================
+Archive Sync Module
+===================
+
+.. versionadded:: Nautilus
+
+This sync module leverages the versioning feature of the S3 objects in RGW to
+have an archive zone that captures the different versions of the S3 objects
+as they occur over time in the other zones.
+
+An archive zone allows to have a history of versions of S3 objects that can
+only be eliminated through the gateways associated with the archive zone.
+
+This functionality is useful to have a configuration where several
+non-versioned zones replicate their data and metadata through their zone
+gateways (mirror configuration) providing high availability to the end users,
+while the archive zone captures all the data updates and metadata for
+consolidate them as versions of S3 objects.
+
+Including an archive zone in a multizone configuration allows you to have the
+flexibility of an S3 object history in one only zone while saving the space
+that the replicas of the versioned S3 objects would consume in the rest of the
+zones.
+
+
+
+Archive Sync Tier Type Configuration
+------------------------------------
+
+How to Configure
+~~~~~~~~~~~~~~~~
+
+See `Multisite Configuration`_ for how to multisite config instructions. The
+archive sync module requires a creation of a new zone. The zone tier type needs
+to be defined as ``archive``:
+
+::
+
+    # radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --endpoints={http://fqdn}[,{http://fqdn}]
+                                --tier-type=archive
+
+.. _Multisite Configuration: ../multisite
diff --git a/doc/radosgw/barbican.rst b/doc/radosgw/barbican.rst
new file mode 100644
index 000000000..a90d063fb
--- /dev/null
+++ b/doc/radosgw/barbican.rst
@@ -0,0 +1,123 @@
+==============================
+OpenStack Barbican Integration
+==============================
+
+OpenStack `Barbican`_ can be used as a secure key management service for
+`Server-Side Encryption`_.
+
+.. image:: ../images/rgw-encryption-barbican.png
+
+#. `Configure Keystone`_
+#. `Create a Keystone user`_
+#. `Configure the Ceph Object Gateway`_
+#. `Create a key in Barbican`_
+
+Configure Keystone
+==================
+
+Barbican depends on Keystone for authorization and access control of its keys.
+
+See `OpenStack Keystone Integration`_.
+
+Create a Keystone user
+======================
+
+Create a new user that will be used by the Ceph Object Gateway to retrieve
+keys.
+
+For example::
+
+   user = rgwcrypt-user
+   pass = rgwcrypt-password
+   tenant = rgwcrypt
+
+See OpenStack documentation for `Manage projects, users, and roles`_.
+
+Create a key in Barbican
+========================
+
+See Barbican documentation for `How to Create a Secret`_. Requests to
+Barbican must include a valid Keystone token in the ``X-Auth-Token`` header.
+
+.. note:: Server-side encryption keys must be 256-bit long and base64 encoded.
+
+Example request::
+
+   POST /v1/secrets HTTP/1.1
+   Host: barbican.example.com:9311
+   Accept: */*
+   Content-Type: application/json
+   X-Auth-Token: 7f7d588dd29b44df983bc961a6b73a10
+   Content-Length: 299
+   
+   {
+       "name": "my-key",
+       "expiration": "2016-12-28T19:14:44.180394",
+       "algorithm": "aes",
+       "bit_length": 256,
+       "mode": "cbc",
+       "payload": "6b+WOZ1T3cqZMxgThRcXAQBrS5mXKdDUphvpxptl9/4=",
+       "payload_content_type": "application/octet-stream",
+       "payload_content_encoding": "base64"
+   }
+
+Response::
+
+   {"secret_ref": "http://barbican.example.com:9311/v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723"}
+
+In the response, ``d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723`` is the key id that
+can be used in any `SSE-KMS`_ request.
+
+This newly created key is not accessible by user ``rgwcrypt-user``. This
+privilege must be added with an ACL. See `How to Set/Replace ACL`_ for more
+details.
+
+Example request (assuming that the Keystone id of ``rgwcrypt-user`` is
+``906aa90bd8a946c89cdff80d0869460f``)::
+
+   PUT /v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723/acl HTTP/1.1
+   Host: barbican.example.com:9311
+   Accept: */*
+   Content-Type: application/json
+   X-Auth-Token: 7f7d588dd29b44df983bc961a6b73a10
+   Content-Length: 101
+
+   {
+       "read":{
+       "users":[ "906aa90bd8a946c89cdff80d0869460f" ],
+       "project-access": true
+       }
+   }
+
+Response::
+
+   {"acl_ref": "http://barbican.example.com:9311/v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723/acl"}
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Barbican as a KMS and add information
+about the Barbican server and Keystone user::
+
+   rgw crypt s3 kms backend = barbican
+   rgw barbican url = http://barbican.example.com:9311
+   rgw keystone barbican user = rgwcrypt-user
+   rgw keystone barbican password = rgwcrypt-password
+
+When using Keystone API version 2::
+
+   rgw keystone barbican tenant = rgwcrypt
+
+When using API version 3::
+
+   rgw keystone barbican project
+   rgw keystone barbican domain
+
+
+.. _Barbican: https://wiki.openstack.org/wiki/Barbican
+.. _Server-Side Encryption: ../encryption
+.. _OpenStack Keystone Integration: ../keystone
+.. _Manage projects, users, and roles: https://docs.openstack.org/admin-guide/cli-manage-projects-users-and-roles.html#create-a-user
+.. _How to Create a Secret: https://developer.openstack.org/api-guide/key-manager/secrets.html#how-to-create-a-secret
+.. _SSE-KMS: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+.. _How to Set/Replace ACL: https://developer.openstack.org/api-guide/key-manager/acls.html#how-to-set-replace-acl
diff --git a/doc/radosgw/bucketpolicy.rst b/doc/radosgw/bucketpolicy.rst
new file mode 100644
index 000000000..99f40c5d7
--- /dev/null
+++ b/doc/radosgw/bucketpolicy.rst
@@ -0,0 +1,216 @@
+===============
+Bucket Policies
+===============
+
+.. versionadded:: Luminous
+
+The Ceph Object Gateway supports a subset of the Amazon S3 policy
+language applied to buckets.
+
+
+Creation and Removal
+====================
+
+Bucket policies are managed through standard S3 operations rather than
+radosgw-admin.
+
+For example, one may use s3cmd to set or delete a policy thus::
+
+  $ cat > examplepol
+  {
+    "Version": "2012-10-17",
+    "Statement": [{
+      "Effect": "Allow",
+      "Principal": {"AWS": ["arn:aws:iam::usfolks:user/fred:subuser"]},
+      "Action": "s3:PutObjectAcl",
+      "Resource": [
+        "arn:aws:s3:::happybucket/*"
+      ]
+    }]
+  }
+
+  $ s3cmd setpolicy examplepol s3://happybucket
+  $ s3cmd delpolicy s3://happybucket
+
+
+Limitations
+===========
+
+Currently, we support only the following actions:
+
+- s3:AbortMultipartUpload
+- s3:CreateBucket
+- s3:DeleteBucketPolicy
+- s3:DeleteBucket
+- s3:DeleteBucketWebsite
+- s3:DeleteObject
+- s3:DeleteObjectVersion
+- s3:DeleteReplicationConfiguration
+- s3:GetAccelerateConfiguration
+- s3:GetBucketAcl
+- s3:GetBucketCORS
+- s3:GetBucketLocation
+- s3:GetBucketLogging
+- s3:GetBucketNotification
+- s3:GetBucketPolicy
+- s3:GetBucketRequestPayment
+- s3:GetBucketTagging
+- s3:GetBucketVersioning
+- s3:GetBucketWebsite
+- s3:GetLifecycleConfiguration
+- s3:GetObjectAcl
+- s3:GetObject
+- s3:GetObjectTorrent
+- s3:GetObjectVersionAcl
+- s3:GetObjectVersion
+- s3:GetObjectVersionTorrent
+- s3:GetReplicationConfiguration
+- s3:IPAddress
+- s3:NotIpAddress
+- s3:ListAllMyBuckets
+- s3:ListBucketMultipartUploads
+- s3:ListBucket
+- s3:ListBucketVersions
+- s3:ListMultipartUploadParts
+- s3:PutAccelerateConfiguration
+- s3:PutBucketAcl
+- s3:PutBucketCORS
+- s3:PutBucketLogging
+- s3:PutBucketNotification
+- s3:PutBucketPolicy
+- s3:PutBucketRequestPayment
+- s3:PutBucketTagging
+- s3:PutBucketVersioning
+- s3:PutBucketWebsite
+- s3:PutLifecycleConfiguration
+- s3:PutObjectAcl
+- s3:PutObject
+- s3:PutObjectVersionAcl
+- s3:PutReplicationConfiguration
+- s3:RestoreObject
+
+We do not yet support setting policies on users, groups, or roles.
+
+We use the RGW ‘tenant’ identifier in place of the Amazon twelve-digit
+account ID. In the future we may allow you to assign an account ID to
+a tenant, but for now if you want to use policies between AWS S3 and
+RGW S3 you will have to use the Amazon account ID as the tenant ID when
+creating users.
+
+Under AWS, all tenants share a single namespace. RGW gives every
+tenant its own namespace of buckets. There may be an option to enable
+an AWS-like 'flat' bucket namespace in future versions. At present, to
+access a bucket belonging to another tenant, address it as
+"tenant:bucket" in the S3 request.
+
+In AWS, a bucket policy can grant access to another account, and that
+account owner can then grant access to individual users with user
+permissions. Since we do not yet support user, role, and group
+permissions, account owners will currently need to grant access
+directly to individual users, and granting an entire account access to
+a bucket grants access to all users in that account.
+
+Bucket policies do not yet support string interpolation.
+
+For all requests, condition keys we support are:
+- aws:CurrentTime
+- aws:EpochTime
+- aws:PrincipalType
+- aws:Referer
+- aws:SecureTransport
+- aws:SourceIp
+- aws:UserAgent
+- aws:username
+
+We support certain s3 condition keys for bucket and object requests.
+
+.. versionadded:: Mimic
+
+Bucket Related Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++-----------------------+----------------------+----------------+
+| Permission            | Condition Keys       | Comments       |
++-----------------------+----------------------+----------------+
+|                       | s3:x-amz-acl         |                |
+|                       | s3:x-amz-grant-<perm>|                |
+|s3:createBucket        | where perm is one of |                |
+|                       | read/write/read-acp  |                |
+|                       | write-acp/           |                |
+|                       | full-control         |                |
++-----------------------+----------------------+----------------+
+|                       | s3:prefix            |                |
+|                       +----------------------+----------------+
+| s3:ListBucket &       | s3:delimiter         |                |
+|                       +----------------------+----------------+
+| s3:ListBucketVersions | s3:max-keys          |                |
++-----------------------+----------------------+----------------+
+| s3:PutBucketAcl       | s3:x-amz-acl         |                |
+|                       | s3:x-amz-grant-<perm>|                |
++-----------------------+----------------------+----------------+
+
+.. _tag_policy:
+
+Object Related Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++-----------------------------+-----------------------------------------------+-------------------+
+|Permission                   |Condition Keys                                 | Comments          |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|                             |s3:x-amz-acl & s3:x-amz-grant-<perm>           |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-copy-source                           |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-server-side-encryption                |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|s3:PutObject                 |s3:x-amz-server-side-encryption-aws-kms-key-id |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-metadata-directive                    |PUT & COPY to      |
+|                             |                                               |overwrite/preserve |
+|                             |                                               |metadata in COPY   |
+|                             |                                               |requests           |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:RequestObjectTag/<tag-key>                  |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:PutObjectAcl              |s3:x-amz-acl & s3-amz-grant-<perm>             |                   |
+|s3:PutObjectVersionAcl       |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:ExistingObjectTag/<tag-key>                 |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|                             |s3:RequestObjectTag/<tag-key>                  |                   |
+|s3:PutObjectTagging &        +-----------------------------------------------+-------------------+
+|s3:PutObjectVersionTagging   |s3:ExistingObjectTag/<tag-key>                 |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObject &               |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersion          |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObjectAcl &            |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersionAcl       |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObjectTagging &        |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersionTagging   |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:DeleteObjectTagging &     |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:DeleteObjectVersionTagging|                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+
+
+More may be supported soon as we integrate with the recently rewritten
+Authentication/Authorization subsystem.
+
+Swift
+=====
+
+There is no way to set bucket policies under Swift, but bucket
+policies that have been set govern Swift as well as S3 operations.
+
+Swift credentials are matched against Principals specified in a policy
+in a way specific to whatever backend is being used.
diff --git a/doc/radosgw/cloud-sync-module.rst b/doc/radosgw/cloud-sync-module.rst
new file mode 100644
index 000000000..a601bd503
--- /dev/null
+++ b/doc/radosgw/cloud-sync-module.rst
@@ -0,0 +1,244 @@
+=========================
+Cloud Sync Module
+=========================
+
+.. versionadded:: Mimic
+
+This module syncs zone data to a remote cloud service. The sync is unidirectional; data is not synced back from the
+remote zone. The goal of this module is to enable syncing data to multiple cloud providers. The currently supported
+cloud providers are those that are compatible with AWS (S3).
+
+User credentials for the remote cloud object store service need to be configured. Since many cloud services impose limits
+on the number of buckets that each user can create, the mapping of source objects and buckets is configurable.
+It is possible to configure different targets to different buckets and bucket prefixes. Note that source ACLs will not
+be preserved. It is possible to map permissions of specific source users to specific destination users.
+
+Due to API limitations there is no way to preserve original object modification time and ETag. The cloud sync module 
+stores these as metadata attributes on the destination objects.
+
+
+
+Cloud Sync Tier Type Configuration
+-------------------------------------
+
+Trivial Configuration:
+~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    {
+      "connection": {
+        "access_key": <access>,
+        "secret": <secret>,
+        "endpoint": <endpoint>,
+        "host_style": <path | virtual>,
+      },
+      "acls": [ { "type": <id | email | uri>,
+                  "source_id": <source_id>,
+                  "dest_id": <dest_id> } ... ],
+      "target_path": <target_path>,
+    }
+
+
+Non Trivial Configuration:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    {
+      "default": {
+        "connection": {
+            "access_key": <access>,
+            "secret": <secret>,
+            "endpoint": <endpoint>,
+            "host_style" <path | virtual>,
+        },
+        "acls": [
+        {
+          "type" : <id | email | uri>,   #  optional, default is id
+          "source_id": <id>,
+          "dest_id": <id>
+        } ... ]
+        "target_path": <path> # optional
+      },
+      "connections": [
+          {
+            "connection_id": <id>,
+            "access_key": <access>,
+            "secret": <secret>,
+            "endpoint": <endpoint>,
+            "host_style" <path | virtual>,  # optional
+          } ... ],
+      "acl_profiles": [
+          {
+            "acls_id": <id>, # acl mappings
+            "acls": [ {
+                "type": <id | email | uri>,
+                "source_id": <id>,
+                "dest_id": <id>
+              } ... ]
+          }
+      ],
+      "profiles": [
+          {
+           "source_bucket": <source>,
+           "connection_id": <connection_id>,
+           "acls_id": <mappings_id>,
+           "target_path": <dest>,          # optional
+          } ... ],
+    }
+
+
+.. Note:: Trivial configuration can coincide with the non-trivial one.
+
+
+* ``connection`` (container)
+
+Represents a connection to the remote cloud service. Contains ``connection_id``, ``access_key``,
+``secret``, ``endpoint``, and ``host_style``.
+
+* ``access_key`` (string)
+
+The remote cloud access key that will be used for a specific connection.
+
+* ``secret`` (string)
+
+The secret key for the remote cloud service.
+
+* ``endpoint`` (string)
+
+URL of remote cloud service endpoint.
+
+* ``host_style`` (path | virtual)
+
+Type of host style to be used when accessing remote cloud endpoint (default: ``path``).
+
+* ``acls`` (array)
+
+Contains a list of ``acl_mappings``.
+
+* ``acl_mapping`` (container)
+
+Each ``acl_mapping`` structure contains ``type``, ``source_id``, and ``dest_id``. These
+will define the ACL mutation that will be done on each object. An ACL mutation allows converting source
+user id to a destination id.
+
+* ``type`` (id | email | uri)
+
+ACL type: ``id`` defines user id, ``email`` defines user by email, and ``uri`` defines user by ``uri`` (group).
+
+* ``source_id`` (string)
+
+ID of user in the source zone.
+
+* ``dest_id`` (string)
+
+ID of user in the destination.
+
+* ``target_path`` (string)
+
+A string that defines how the target path is created. The target path specifies a prefix to which
+the source object name is appended. The target path configurable can include any of the following
+variables:
+- ``sid``: unique string that represents the sync instance ID
+- ``zonegroup``: the zonegroup name
+- ``zonegroup_id``: the zonegroup ID
+- ``zone``: the zone name
+- ``zone_id``: the zone id
+- ``bucket``: source bucket name
+- ``owner``: source bucket owner ID
+
+For example: ``target_path = rgwx-${zone}-${sid}/${owner}/${bucket}``
+
+
+* ``acl_profiles`` (array)
+
+An array of ``acl_profile``.
+
+* ``acl_profile`` (container)
+ 
+Each profile contains ``acls_id`` (string) that represents the profile, and ``acls`` array that
+holds a list of ``acl_mappings``.
+
+* ``profiles`` (array)
+
+A list of profiles. Each profile contains the following:
+- ``source_bucket``: either a bucket name, or a bucket prefix (if ends with ``*``) that defines the source bucket(s) for this profile
+- ``target_path``: as defined above
+- ``connection_id``: ID of the connection that will be used for this profile
+- ``acls_id``: ID of ACLs profile that will be used for this profile
+
+
+S3 Specific Configurables:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently cloud sync will only work with backends that are compatible with AWS S3. There are
+a few configurables that can be used to tweak its behavior when accessing these cloud services:
+
+::
+
+    {
+      "multipart_sync_threshold": {object_size},
+      "multipart_min_part_size": {part_size}
+    }
+
+
+* ``multipart_sync_threshold`` (integer)
+
+Objects this size or larger will be synced to the cloud using multipart upload.
+
+* ``multipart_min_part_size`` (integer)
+
+Minimum parts size to use when syncing objects using multipart upload.
+
+
+How to Configure
+~~~~~~~~~~~~~~~~
+
+See :ref:`multisite` for how to multisite config instructions. The cloud sync module requires a creation of a new zone. The zone
+tier type needs to be defined as ``cloud``:
+
+::
+
+    # radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --endpoints={http://fqdn}[,{http://fqdn}]
+                                --tier-type=cloud
+
+
+The tier configuration can be then done using the following command
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config={key}={val}[,{key}={val}]
+
+The ``key`` in the configuration specifies the config variable that needs to be updated, and
+the ``val`` specifies its new value. Nested values can be accessed using period. For example:
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=connection.access_key={key},connection.secret={secret}
+
+
+Configuration array entries can be accessed by specifying the specific entry to be referenced enclosed
+in square brackets, and adding new array entry can be done by using `[]`. Index value of `-1` references
+the last entry in the array. At the moment it is not possible to create a new entry and reference it
+again at the same command.
+For example, creating a new profile for buckets starting with {prefix}:
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=profiles[].source_bucket={prefix}'*'
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=profiles[-1].connection_id={conn_id},profiles[-1].acls_id={acls_id}
+
+
+An entry can be removed by using ``--tier-config-rm={key}``.
diff --git a/doc/radosgw/cloud-transition.rst b/doc/radosgw/cloud-transition.rst
new file mode 100644
index 000000000..c00ad790b
--- /dev/null
+++ b/doc/radosgw/cloud-transition.rst
@@ -0,0 +1,368 @@
+================
+Cloud Transition
+================
+
+This feature enables data transition to a remote cloud service as part of `Lifecycle Configuration <https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html>`__ via :ref:`storage_classes`. The transition is unidirectional; data cannot be transitioned back from the remote zone. The goal of this feature is to enable data transition to multiple cloud providers. The currently supported cloud providers are those that are compatible with AWS (S3).
+
+Special storage class of tier type ``cloud-s3`` is used to configure the remote cloud S3 object store service to which the data needs to be transitioned. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool.
+
+User credentials for the remote cloud object store service need to be configured. Note that source ACLs will not
+be preserved. It is possible to map permissions of specific source users to specific destination users.
+
+
+Cloud Storage Class Configuration
+---------------------------------
+
+::
+
+    {
+      "access_key": <access>,
+      "secret": <secret>,
+      "endpoint": <endpoint>,
+      "region": <region>,
+      "host_style": <path | virtual>,
+      "acls": [ { "type": <id | email | uri>,
+                  "source_id": <source_id>,
+                  "dest_id": <dest_id> } ... ],
+      "target_path": <target_path>,
+      "target_storage_class": <target-storage-class>,
+      "multipart_sync_threshold": {object_size},
+      "multipart_min_part_size": {part_size},
+      "retain_head_object": <true | false>
+    }
+
+
+Cloud Transition Specific Configurables:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``access_key`` (string)
+
+The remote cloud S3 access key that will be used for a specific connection.
+
+* ``secret`` (string)
+
+The secret key for the remote cloud S3 service.
+
+* ``endpoint`` (string)
+
+URL of remote cloud S3 service endpoint.
+
+* ``region`` (string)
+
+The remote cloud S3 service region name.
+
+* ``host_style`` (path | virtual)
+
+Type of host style to be used when accessing remote cloud S3 endpoint (default: ``path``).
+
+* ``acls`` (array)
+
+Contains a list of ``acl_mappings``.
+
+* ``acl_mapping`` (container)
+
+Each ``acl_mapping`` structure contains ``type``, ``source_id``, and ``dest_id``. These
+will define the ACL mutation that will be done on each object. An ACL mutation allows converting source
+user id to a destination id.
+
+* ``type`` (id | email | uri)
+
+ACL type: ``id`` defines user id, ``email`` defines user by email, and ``uri`` defines user by ``uri`` (group).
+
+* ``source_id`` (string)
+
+ID of user in the source zone.
+
+* ``dest_id`` (string)
+
+ID of user in the destination.
+
+* ``target_path`` (string)
+
+A string that defines how the target path is created. The target path specifies a prefix to which
+the source 'bucket-name/object-name' is appended. If not specified the target_path created is "rgwx-${zonegroup}-${storage-class}-cloud-bucket".
+
+For example: ``target_path = rgwx-archive-${zonegroup}/``
+
+* ``target_storage_class`` (string)
+
+A string that defines the target storage class to which the object transitions to. If not specified, object is transitioned to STANDARD storage class.
+
+* ``retain_head_object`` (true | false)
+
+If true, retains the metadata of the object transitioned to cloud. If false (default), the object is deleted post transition.
+This option is ignored for current versioned objects. For more details, refer to section "Versioned Objects" below.
+
+
+S3 Specific Configurables:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently cloud transition will only work with backends that are compatible with AWS S3. There are
+a few configurables that can be used to tweak its behavior when accessing these cloud services:
+
+::
+
+    {
+      "multipart_sync_threshold": {object_size},
+      "multipart_min_part_size": {part_size}
+    }
+
+
+* ``multipart_sync_threshold`` (integer)
+
+Objects this size or larger will be transitioned to the cloud using multipart upload.
+
+* ``multipart_min_part_size`` (integer)
+
+Minimum parts size to use when transitioning objects using multipart upload.
+
+
+How to Configure
+~~~~~~~~~~~~~~~~
+
+See :ref:`adding_a_storage_class` for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud-s3``
+
+.. note:: If you have not done any previous `Multisite Configuration`_,
+          a ``default`` zone and zonegroup are created for you, and changes
+          to the zone/zonegroup will not take effect until the Ceph Object
+          Gateways are restarted. If you have created a realm for multisite,
+          the zone/zonegroup changes will take effect once the changes are
+          committed with ``radosgw-admin period update --commit``.
+
+::
+
+    # radosgw-admin zonegroup placement add --rgw-zonegroup={zone-group-name} \
+                                            --placement-id={placement-id} \
+                                            --storage-class={storage-class-name} \
+                                            --tier-type=cloud-s3 
+
+For example:
+
+::
+
+    # radosgw-admin zonegroup placement add --rgw-zonegroup=default \
+                                            --placement-id=default-placement \
+                                            --storage-class=CLOUDTIER --tier-type=cloud-s3
+    [
+        {
+            "key": "default-placement",
+            "val": {
+                "name": "default-placement",
+                "tags": [],
+                "storage_classes": [
+                    "CLOUDTIER",
+                    "STANDARD"
+                ],
+                "tier_targets": [
+                    {
+                        "key": "CLOUDTIER",
+                        "val": {
+                            "tier_type": "cloud-s3",
+                            "storage_class": "CLOUDTIER",
+                            "retain_head_object": "false",
+                            "s3": {
+                                "endpoint": "",
+                                "access_key": "",
+                                "secret": "",
+                                "host_style": "path",
+                                "target_storage_class": "",
+                                "target_path": "",
+                                "acl_mappings": [],
+                                "multipart_sync_threshold": 33554432,
+                                "multipart_min_part_size": 33554432
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    ]
+
+
+.. note:: Once a storage class is created of ``--tier-type=cloud-s3``, it cannot be later modified to any other storage class type.
+
+The tier configuration can be then done using the following command
+
+::
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \
+                                               --placement-id={placement-id} \
+                                               --storage-class={storage-class-name} \
+                                               --tier-config={key}={val}[,{key}={val}]
+
+The ``key`` in the configuration specifies the config variable that needs to be updated, and
+the ``val`` specifies its new value.
+
+
+For example:
+
+::
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup default \
+                                               --placement-id default-placement \
+                                               --storage-class CLOUDTIER \
+                                               --tier-config=endpoint=http://XX.XX.XX.XX:YY,\
+                                               access_key=<access_key>,secret=<secret>, \
+                                               multipart_sync_threshold=44432, \
+                                               multipart_min_part_size=44432, \
+                                               retain_head_object=true
+
+Nested values can be accessed using period. For example:
+
+::
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \
+                                               --placement-id={placement-id} \
+                                               --storage-class={storage-class-name} \
+                                               --tier-config=acls.source_id=${source-id}, \
+                                               acls.dest_id=${dest-id}
+
+
+
+Configuration array entries can be accessed by specifying the specific entry to be referenced enclosed
+in square brackets, and adding new array entry can be done by using `[]`.
+For example, creating a new acl array entry:
+
+::
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \
+                                               --placement-id={placement-id} \
+                                               --storage-class={storage-class-name} \
+                                               --tier-config=acls[].source_id=${source-id}, \
+                                               acls[${source-id}].dest_id=${dest-id}, \
+                                               acls[${source-id}].type=email
+
+An entry can be removed by using ``--tier-config-rm={key}``.
+
+For example,
+
+::
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup default \
+                                               --placement-id default-placement \
+                                               --storage-class CLOUDTIER \
+                                               --tier-config-rm=acls.source_id=testid
+
+    # radosgw-admin zonegroup placement modify --rgw-zonegroup default \
+                                               --placement-id default-placement \
+                                               --storage-class CLOUDTIER \
+                                               --tier-config-rm=target_path
+
+The storage class can be removed using the following command
+
+::
+
+    # radosgw-admin zonegroup placement rm --rgw-zonegroup={zone-group-name} \
+                                           --placement-id={placement-id} \
+                                           --storage-class={storage-class-name}
+
+For example,
+
+::
+
+    # radosgw-admin zonegroup placement rm --rgw-zonegroup default \
+                                           --placement-id default-placement \
+                                           --storage-class CLOUDTIER
+    [
+        {
+            "key": "default-placement",
+            "val": {
+                "name": "default-placement",
+                "tags": [],
+                "storage_classes": [
+                    "STANDARD"
+                ]
+            }
+        }
+    ]
+
+Object modification & Limitations
+----------------------------------
+
+The cloud storage class once configured can then be used like any other storage class in the bucket lifecycle rules. For example,
+
+::
+
+    <Transition>
+      <StorageClass>CLOUDTIER</StorageClass>
+      ....
+      ....
+    </Transition>
+
+
+Since the transition is unidirectional, while configuring S3 lifecycle rules, the cloud storage class should be specified last among all the storage classes the object transitions to. Subsequent rules (if any) do not apply post transition to the cloud.
+
+Due to API limitations there is no way to preserve original object modification time and ETag but they get stored as metadata attributes on the destination objects, as shown below:
+
+::
+
+   x-amz-meta-rgwx-source: rgw
+   x-amz-meta-rgwx-source-etag: ed076287532e86365e841e92bfc50d8c
+   x-amz-meta-rgwx-source-key: lc.txt
+   x-amz-meta-rgwx-source-mtime: 1608546349.757100363
+   x-amz-meta-rgwx-versioned-epoch: 0
+
+In order to allow some cloud services detect the source and map the user-defined 'x-amz-meta-' attributes, below two additional new attributes are added to the objects being transitioned 
+
+::
+    
+   x-rgw-cloud : true/false
+   (set to "true", by default, if the object is being transitioned from RGW)
+
+   x-rgw-cloud-keep-attrs : true/false
+   (if set to default value "true", the cloud service should map and store all the x-amz-meta-* attributes. If it cannot, then the operation should fail.
+    if set to "false", the cloud service can ignore such attributes and just store the object data being sent.)
+
+
+By default, post transition, the source object gets deleted. But it is possible to retain its metadata but with updated values (like storage-class and object-size) by setting config option 'retain_head_object' to true. However GET on those objects shall still fail with 'InvalidObjectState' error.
+
+For example,
+::
+
+    # s3cmd info s3://bucket/lc.txt
+    s3://bucket/lc.txt (object):
+       File size: 12
+       Last mod:  Mon, 21 Dec 2020 10:25:56 GMT
+       MIME type: text/plain
+       Storage:   CLOUDTIER
+       MD5 sum:   ed076287532e86365e841e92bfc50d8c
+       SSE:       none
+       Policy:    none
+       CORS:      none
+       ACL:       M. Tester: FULL_CONTROL
+       x-amz-meta-s3cmd-attrs: atime:1608466266/ctime:1597606156/gid:0/gname:root/md5:ed076287532e86365e841e92bfc50d8c/mode:33188/mtime:1597605793/uid:0/uname:root
+
+    # s3cmd get s3://bucket/lc.txt lc_restore.txt
+    download: 's3://bucket/lc.txt' -> 'lc_restore.txt'  [1 of 1]
+    ERROR: S3 error: 403 (InvalidObjectState)
+
+To avoid object names collision across various buckets, source bucket name is prepended to the target object name. If the object is versioned, object versionid is appended to the end.
+
+Below is the sample object name format:
+::
+
+    s3://<target_path>/<source_bucket_name>/<source_object_name>(-<source_object_version_id>)
+
+
+Versioned Objects
+~~~~~~~~~~~~~~~~~
+
+For versioned and locked objects, similar semantics as that of LifecycleExpiration are applied as stated below.
+
+* If the object is current, post transitioning to cloud, it is made noncurrent with delete marker created.
+
+* If the object is noncurrent and is locked, its transition is skipped.
+
+
+Future Work
+-----------
+
+* Send presigned redirect or read-through the objects transitioned to cloud
+
+* Support s3:RestoreObject operation on cloud transitioned objects.
+
+* Federation between RGW and Cloud services.
+
+* Support transition to other cloud providers (like Azure).
+
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/compression.rst b/doc/radosgw/compression.rst
new file mode 100644
index 000000000..fba0681da
--- /dev/null
+++ b/doc/radosgw/compression.rst
@@ -0,0 +1,91 @@
+===========
+Compression
+===========
+
+.. versionadded:: Kraken
+
+The Ceph Object Gateway supports server-side compression of uploaded objects,
+using any of Ceph's existing compression plugins.
+
+.. note:: The Reef release added a :ref:`feature_compress_encrypted` zonegroup
+   feature to enable compression with `Server-Side Encryption`_.
+
+
+Configuration
+=============
+
+Compression can be enabled on a storage class in the Zone's placement target
+by providing the ``--compression=<type>`` option to the command
+``radosgw-admin zone placement modify``.
+
+The compression ``type`` refers to the name of the compression plugin to use
+when writing new object data. Each compressed object remembers which plugin
+was used, so changing this setting does not hinder the ability to decompress
+existing objects, nor does it force existing objects to be recompressed.
+
+This compression setting applies to all new objects uploaded to buckets using
+this placement target. Compression can be disabled by setting the ``type`` to
+an empty string or ``none``.
+
+For example::
+
+  $ radosgw-admin zone placement modify \
+        --rgw-zone default \
+        --placement-id default-placement \
+        --storage-class STANDARD \
+        --compression zlib
+  {
+  ...
+      "placement_pools": [
+          {
+              "key": "default-placement",
+              "val": {
+                  "index_pool": "default.rgw.buckets.index",
+                  "storage_classes": {
+                      "STANDARD": {
+                          "data_pool": "default.rgw.buckets.data",
+                          "compression_type": "zlib"
+                      }
+                  },
+                  "data_extra_pool": "default.rgw.buckets.non-ec",
+                  "index_type": 0,
+              }
+          }
+      ],
+  ...
+  }
+
+.. note:: A ``default`` zone is created for you if you have not done any
+   previous `Multisite Configuration`_.
+
+
+Statistics
+==========
+
+While all existing commands and APIs continue to report object and bucket
+sizes based their uncompressed data, compression statistics for a given bucket
+are included in its ``bucket stats``::
+
+  $ radosgw-admin bucket stats --bucket=<name>
+  {
+  ...
+      "usage": {
+          "rgw.main": {
+              "size": 1075028,
+              "size_actual": 1331200,
+              "size_utilized": 592035,
+              "size_kb": 1050,
+              "size_kb_actual": 1300,
+              "size_kb_utilized": 579,
+              "num_objects": 104
+          }
+      },
+  ...
+  }
+
+The ``size_utilized`` and ``size_kb_utilized`` fields represent the total
+size of compressed data, in bytes and kilobytes respectively.
+
+
+.. _`Server-Side Encryption`: ../encryption
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst
new file mode 100644
index 000000000..916ff4ff5
--- /dev/null
+++ b/doc/radosgw/config-ref.rst
@@ -0,0 +1,301 @@
+======================================
+ Ceph Object Gateway Config Reference
+======================================
+
+The following settings may added to the Ceph configuration file (i.e., usually
+``ceph.conf``) under the ``[client.radosgw.{instance-name}]`` section. The
+settings may contain default values. If you do not specify each setting in the
+Ceph configuration file, the default value will be set automatically.
+
+Configuration variables set under the ``[client.radosgw.{instance-name}]``
+section will not apply to rgw or radosgw-admin commands without an instance-name
+specified in the command. Thus variables meant to be applied to all RGW
+instances or all radosgw-admin options can be put into the ``[global]`` or the
+``[client]`` section to avoid specifying ``instance-name``.
+
+.. confval:: rgw_frontends
+.. confval:: rgw_data
+.. confval:: rgw_enable_apis
+.. confval:: rgw_cache_enabled
+.. confval:: rgw_cache_lru_size
+.. confval:: rgw_dns_name
+.. confval:: rgw_script_uri
+.. confval:: rgw_request_uri
+.. confval:: rgw_print_continue
+.. confval:: rgw_remote_addr_param
+.. confval:: rgw_op_thread_timeout
+.. confval:: rgw_op_thread_suicide_timeout
+.. confval:: rgw_thread_pool_size
+.. confval:: rgw_num_control_oids
+.. confval:: rgw_init_timeout
+.. confval:: rgw_mime_types_file
+.. confval:: rgw_s3_success_create_obj_status
+.. confval:: rgw_resolve_cname
+.. confval:: rgw_obj_stripe_size
+.. confval:: rgw_extended_http_attrs
+.. confval:: rgw_exit_timeout_secs
+.. confval:: rgw_get_obj_window_size
+.. confval:: rgw_get_obj_max_req_size
+.. confval:: rgw_multipart_min_part_size
+.. confval:: rgw_relaxed_s3_bucket_names
+.. confval:: rgw_list_buckets_max_chunk
+.. confval:: rgw_override_bucket_index_max_shards
+.. confval:: rgw_curl_wait_timeout_ms
+.. confval:: rgw_copy_obj_progress
+.. confval:: rgw_copy_obj_progress_every_bytes
+.. confval:: rgw_max_copy_obj_concurrent_io
+.. confval:: rgw_admin_entry
+.. confval:: rgw_content_length_compat
+.. confval:: rgw_bucket_quota_ttl
+.. confval:: rgw_user_quota_bucket_sync_interval
+.. confval:: rgw_user_quota_sync_interval
+.. confval:: rgw_bucket_default_quota_max_objects
+.. confval:: rgw_bucket_default_quota_max_size
+.. confval:: rgw_user_default_quota_max_objects
+.. confval:: rgw_user_default_quota_max_size
+.. confval:: rgw_verify_ssl
+.. confval:: rgw_max_chunk_size
+
+Lifecycle Settings
+==================
+
+Bucket Lifecycle configuration can be used to manage your objects so they are stored
+effectively throughout their lifetime. In past releases Lifecycle processing was rate-limited
+by single threaded processing. With the Nautilus release this has been addressed and the
+Ceph Object Gateway now allows for parallel thread processing of bucket lifecycles across
+additional Ceph Object Gateway instances and replaces the in-order
+index shard enumeration with a random ordered sequence.
+
+There are two options in particular to look at when looking to increase the
+aggressiveness of lifecycle processing:
+
+.. confval:: rgw_lc_max_worker
+.. confval:: rgw_lc_max_wp_worker
+
+These values can be tuned based upon your specific workload to further increase the
+aggressiveness of lifecycle processing. For a workload with a larger number of buckets (thousands)
+you would look at increasing the :confval:`rgw_lc_max_worker` value from the default value of 3 whereas for a
+workload with a smaller number of buckets but higher number of objects (hundreds of thousands)
+per bucket you would consider decreasing :confval:`rgw_lc_max_wp_worker` from the default value of 3.
+
+.. note:: When looking to tune either of these specific values please validate the
+   current Cluster performance and Ceph Object Gateway utilization before increasing.
+
+Garbage Collection Settings
+===========================
+
+The Ceph Object Gateway allocates storage for new objects immediately.
+
+The Ceph Object Gateway purges the storage space used for deleted and overwritten 
+objects in the Ceph Storage cluster some time after the gateway deletes the 
+objects from the bucket index. The process of purging the deleted object data 
+from the Ceph Storage cluster is known as Garbage Collection or GC.
+
+To view the queue of objects awaiting garbage collection, execute the following
+
+.. prompt:: bash $
+
+   radosgw-admin gc list
+
+.. note:: Specify ``--include-all`` to list all entries, including unexpired
+   Garbage Collection objects.
+
+Garbage collection is a background activity that may
+execute continuously or during times of low loads, depending upon how the
+administrator configures the Ceph Object Gateway. By default, the Ceph Object
+Gateway conducts GC operations continuously. Since GC operations are a normal
+part of Ceph Object Gateway operations, especially with object delete
+operations, objects eligible for garbage collection exist most of the time.
+
+Some workloads may temporarily or permanently outpace the rate of garbage
+collection activity. This is especially true of delete-heavy workloads, where
+many objects get stored for a short period of time and then deleted. For these
+types of workloads, administrators can increase the priority of garbage
+collection operations relative to other operations with the following
+configuration parameters.
+
+.. confval:: rgw_gc_max_objs
+.. confval:: rgw_gc_obj_min_wait
+.. confval:: rgw_gc_processor_max_time
+.. confval:: rgw_gc_processor_period
+.. confval:: rgw_gc_max_concurrent_io
+
+:Tuning Garbage Collection for Delete Heavy Workloads:
+
+As an initial step towards tuning Ceph Garbage Collection to be more
+aggressive the following options are suggested to be increased from their
+default configuration values::
+
+  rgw_gc_max_concurrent_io = 20
+  rgw_gc_max_trim_chunk = 64
+
+.. note:: Modifying these values requires a restart of the RGW service.
+
+Once these values have been increased from default please monitor for performance of the cluster during Garbage Collection to verify no adverse performance issues due to the increased values.
+
+Multisite Settings
+==================
+
+.. versionadded:: Jewel
+
+You may include the following settings in your Ceph configuration
+file under each ``[client.radosgw.{instance-name}]`` instance.
+
+.. confval:: rgw_zone
+.. confval:: rgw_zonegroup
+.. confval:: rgw_realm
+.. confval:: rgw_run_sync_thread
+.. confval:: rgw_data_log_window
+.. confval:: rgw_data_log_changes_size
+.. confval:: rgw_data_log_obj_prefix
+.. confval:: rgw_data_log_num_shards
+.. confval:: rgw_md_log_max_shards
+.. confval:: rgw_data_sync_poll_interval
+.. confval:: rgw_meta_sync_poll_interval
+.. confval:: rgw_bucket_sync_spawn_window
+.. confval:: rgw_data_sync_spawn_window
+.. confval:: rgw_meta_sync_spawn_window
+
+.. important:: The values of :confval:`rgw_data_log_num_shards` and
+   :confval:`rgw_md_log_max_shards` should not be changed after sync has
+   started.
+
+S3 Settings
+===========
+
+.. confval:: rgw_s3_auth_use_ldap
+
+Swift Settings
+==============
+
+.. confval:: rgw_enforce_swift_acls
+.. confval:: rgw_swift_tenant_name
+.. confval:: rgw_swift_token_expiration
+.. confval:: rgw_swift_url
+.. confval:: rgw_swift_url_prefix
+.. confval:: rgw_swift_auth_url
+.. confval:: rgw_swift_auth_entry
+.. confval:: rgw_swift_account_in_url
+.. confval:: rgw_swift_versioning_enabled
+.. confval:: rgw_trust_forwarded_https
+
+Logging Settings
+================
+
+.. confval:: rgw_log_nonexistent_bucket
+.. confval:: rgw_log_object_name
+.. confval:: rgw_log_object_name_utc
+.. confval:: rgw_usage_max_shards
+.. confval:: rgw_usage_max_user_shards
+.. confval:: rgw_enable_ops_log
+.. confval:: rgw_enable_usage_log
+.. confval:: rgw_ops_log_rados
+.. confval:: rgw_ops_log_socket_path
+.. confval:: rgw_ops_log_data_backlog
+.. confval:: rgw_usage_log_flush_threshold
+.. confval:: rgw_usage_log_tick_interval
+.. confval:: rgw_log_http_headers
+
+Keystone Settings
+=================
+
+.. confval:: rgw_keystone_url
+.. confval:: rgw_keystone_api_version
+.. confval:: rgw_keystone_admin_domain
+.. confval:: rgw_keystone_admin_project
+.. confval:: rgw_keystone_admin_token
+.. confval:: rgw_keystone_admin_token_path
+.. confval:: rgw_keystone_admin_tenant
+.. confval:: rgw_keystone_admin_user
+.. confval:: rgw_keystone_admin_password
+.. confval:: rgw_keystone_admin_password_path
+.. confval:: rgw_keystone_accepted_roles
+.. confval:: rgw_keystone_token_cache_size
+.. confval:: rgw_keystone_verify_ssl
+.. confval:: rgw_keystone_service_token_enabled
+.. confval:: rgw_keystone_service_token_accepted_roles
+.. confval:: rgw_keystone_expired_token_cache_expiration
+
+Server-side encryption Settings
+===============================
+
+.. confval:: rgw_crypt_s3_kms_backend
+
+Barbican Settings
+=================
+
+.. confval:: rgw_barbican_url
+.. confval:: rgw_keystone_barbican_user
+.. confval:: rgw_keystone_barbican_password
+.. confval:: rgw_keystone_barbican_tenant
+.. confval:: rgw_keystone_barbican_project
+.. confval:: rgw_keystone_barbican_domain
+
+HashiCorp Vault Settings
+========================
+
+.. confval:: rgw_crypt_vault_auth
+.. confval:: rgw_crypt_vault_token_file
+.. confval:: rgw_crypt_vault_addr
+.. confval:: rgw_crypt_vault_prefix
+.. confval:: rgw_crypt_vault_secret_engine
+.. confval:: rgw_crypt_vault_namespace
+
+SSE-S3 Settings
+===============
+
+.. confval:: rgw_crypt_sse_s3_backend
+.. confval:: rgw_crypt_sse_s3_vault_secret_engine
+.. confval:: rgw_crypt_sse_s3_key_template
+.. confval:: rgw_crypt_sse_s3_vault_auth
+.. confval:: rgw_crypt_sse_s3_vault_token_file
+.. confval:: rgw_crypt_sse_s3_vault_addr
+.. confval:: rgw_crypt_sse_s3_vault_prefix
+.. confval:: rgw_crypt_sse_s3_vault_namespace
+.. confval:: rgw_crypt_sse_s3_vault_verify_ssl
+.. confval:: rgw_crypt_sse_s3_vault_ssl_cacert
+.. confval:: rgw_crypt_sse_s3_vault_ssl_clientcert
+.. confval:: rgw_crypt_sse_s3_vault_ssl_clientkey
+
+
+QoS settings
+------------
+
+.. versionadded:: Nautilus
+
+The ``civetweb`` frontend has a threading model that uses a thread per
+connection and hence is automatically throttled by :confval:`rgw_thread_pool_size`
+configurable when it comes to accepting connections. The newer ``beast`` frontend is
+not restricted by the thread pool size when it comes to accepting new
+connections, so a scheduler abstraction is introduced in the Nautilus release
+to support future methods of scheduling requests.
+
+Currently the scheduler defaults to a throttler which throttles the active
+connections to a configured limit. QoS based on mClock is currently in an
+*experimental* phase and not recommended for production yet. Current
+implementation of *dmclock_client* op queue divides RGW ops on admin, auth
+(swift auth, sts) metadata & data requests.
+
+
+.. confval:: rgw_max_concurrent_requests
+.. confval:: rgw_scheduler_type
+.. confval:: rgw_dmclock_auth_res
+.. confval:: rgw_dmclock_auth_wgt
+.. confval:: rgw_dmclock_auth_lim
+.. confval:: rgw_dmclock_admin_res
+.. confval:: rgw_dmclock_admin_wgt
+.. confval:: rgw_dmclock_admin_lim
+.. confval:: rgw_dmclock_data_res
+.. confval:: rgw_dmclock_data_wgt
+.. confval:: rgw_dmclock_data_lim
+.. confval:: rgw_dmclock_metadata_res
+.. confval:: rgw_dmclock_metadata_wgt
+.. confval:: rgw_dmclock_metadata_lim
+
+.. _Architecture: ../../architecture#data-striping
+.. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/
+.. _Cluster Pools: ../../rados/operations/pools
+.. _Rados cluster handles: ../../rados/api/librados-intro/#step-2-configuring-a-cluster-handle
+.. _Barbican: ../barbican
+.. _Encryption: ../encryption
+.. _HTTP Frontends: ../frontends
diff --git a/doc/radosgw/d3n_datacache.rst b/doc/radosgw/d3n_datacache.rst
new file mode 100644
index 000000000..12d2850a5
--- /dev/null
+++ b/doc/radosgw/d3n_datacache.rst
@@ -0,0 +1,116 @@
+==================
+D3N RGW Data Cache
+==================
+
+.. contents::
+
+Datacenter-Data-Delivery Network (D3N) uses high-speed storage such as NVMe flash or DRAM to cache
+datasets on the access side.
+Such caching allows big data jobs to use the compute and fast storage resources available on each
+Rados Gateway node at the edge.
+
+Many datacenters include low-cost, centralized storage repositories, called data lakes,
+to store and share terabyte and petabyte-scale datasets.
+By necessity most distributed big-data analytic clusters such as Hadoop and Spark must
+depend on accessing a centrally located data lake that is relatively far away.
+Even with a well-designed datacenter network, cluster-to-data lake bandwidth is typically much less
+than the bandwidth of a solid-state storage located at an edge node.
+
+| D3N improves the performance of big-data jobs running in analysis clusters by speeding up recurring reads from the data lake.
+| The Rados Gateways act as cache servers for the back-end object store (OSDs), storing data locally for reuse.
+
+Architecture
+============
+
+D3N improves the performance of big-data jobs by speeding up repeatedly accessed dataset reads from the data lake.
+Cache servers are located in the datacenter on the access side of potential network and storage bottlenecks.
+D3Ns two-layer logical cache forms a traditional caching hierarchy :sup:`*`
+where caches nearer the client have the lowest access latency and overhead,
+while caches in higher levels in the hierarchy are slower (requiring multiple hops to access),
+The layer 1 cache server nearest to the client handles object requests by breaking them into blocks,
+returning any blocks which are cached locally, and forwarding missed requests to the block home location
+(as determined by consistent hashing) in the next layer.
+Cache misses are forwarded to successive logical caching layers until a miss at the top layer is resolved
+by a request to the data lake (Rados)
+
+:sup:`*` currently only layer 1 cache has been upstreamed.
+
+See `MOC D3N (Datacenter-scale Data Delivery Network)`_ and `Red Hat Research D3N Cache for Data Centers`_.
+
+Implementation
+==============
+
+- The D3N cache supports both the `S3` and `Swift` object storage interfaces.
+- D3N currently caches only tail objects, because they are immutable (by default it is parts of objects that are larger than 4MB).
+  (the NGINX `RGW Data cache and CDN`_ supports caching of all object sizes)
+
+
+Requirements
+------------
+
+- An SSD (/dev/nvme,/dev/pmem,/dev/shm) or similar block storage device, formatted
+  (filesystems other than XFS were not tested) and mounted.
+  It will be used as the cache backing store.
+  (depending on device performance, multiple RGWs may share a single device but each requires
+  a discrete directory on the device filesystem)
+
+Limitations
+-----------
+
+- D3N will not cache objects compressed by `Rados Gateway Compression`_ (OSD level compression is supported).
+- D3N will not cache objects encrypted by `Rados Gateway Encryption`_.
+- D3N will be disabled if the ``rgw_max_chunk_size`` config variable value differs from the ``rgw_obj_stripe_size`` config variable value.
+
+
+D3N Environment Setup
+=====================
+
+Running
+-------
+
+To enable D3N on an existing RGWs the following configuration entries are required
+in each Rados Gateways ceph.conf client section, for example for ``[client.rgw.8000]``::
+
+    [client.rgw.8000]
+      rgw_d3n_l1_local_datacache_enabled = true
+      rgw_d3n_l1_datacache_persistent_path = "/mnt/nvme0/rgw_datacache/client.rgw.8000/"
+      rgw_d3n_l1_datacache_size = 10737418240
+
+The above example assumes that the cache backing-store solid state device
+is mounted at `/mnt/nvme0` and has `10 GB` of free space available for the cache.
+
+The persistent path directory has to be created before starting the Gateway.
+(``mkdir -p /mnt/nvme0/rgw_datacache/client.rgw.8000/``)
+
+If another Gateway is co-located on the same machine, configure it's persistent path to a discrete directory,
+for example in the case of `[client.rgw.8001]` configure
+``rgw_d3n_l1_datacache_persistent_path = "/mnt/nvme0/rgw_datacache/client.rgw.8001/"``
+in the ``[client.rgw.8001]`` ceph.conf client section.
+
+In a multiple co-located Gateways configuration consider assigning clients with different workloads
+to each Gateway without a balancer in order to avoid cached data duplication.
+
+    NOTE: each time the Rados Gateway is restarted the content of the cache directory is purged.
+
+Logs
+----
+- D3N related log lines in `radosgw.*.log` contain the string ``d3n`` (case insensitive).
+- low level D3N logs can be enabled by the ``debug_rgw_datacache`` subsystem (up to ``debug_rgw_datacache=30``)
+
+
+CONFIG REFERENCE
+================
+The following D3N related settings can be added to the Ceph configuration file
+(i.e., usually `ceph.conf`) under the ``[client.rgw.{instance-name}]`` section.
+
+.. confval:: rgw_d3n_l1_local_datacache_enabled
+.. confval:: rgw_d3n_l1_datacache_persistent_path
+.. confval:: rgw_d3n_l1_datacache_size
+.. confval:: rgw_d3n_l1_eviction_policy
+
+
+.. _MOC D3N (Datacenter-scale Data Delivery Network): https://massopen.cloud/research-and-development/cloud-research/d3n/
+.. _Red Hat Research D3N Cache for Data Centers: https://research.redhat.com/blog/research_project/d3n-multilayer-cache/
+.. _Rados Gateway Compression: ../compression/
+.. _Rados Gateway Encryption: ../encryption/
+.. _RGW Data cache and CDN: ../rgw-cache/
diff --git a/doc/radosgw/dynamicresharding.rst b/doc/radosgw/dynamicresharding.rst
new file mode 100644
index 000000000..b8bd68d9e
--- /dev/null
+++ b/doc/radosgw/dynamicresharding.rst
@@ -0,0 +1,238 @@
+.. _rgw_dynamic_bucket_index_resharding:
+
+===================================
+RGW Dynamic Bucket Index Resharding
+===================================
+
+.. versionadded:: Luminous
+
+A large bucket index can lead to performance problems, which can
+be addressed by sharding bucket indexes.
+Until Luminous, changing the number of bucket shards (resharding)
+needed to be done offline, with RGW services disabled.
+Since the Luminous release Ceph has supported online bucket resharding.
+
+Each bucket index shard can handle its entries efficiently up until
+reaching a certain threshold. If this threshold is
+exceeded the system can suffer from performance issues. The dynamic
+resharding feature detects this situation and automatically increases
+the number of shards used by a bucket's index, resulting in a
+reduction of the number of entries in each shard. This
+process is transparent to the user. Writes to the target bucket
+are blocked (but reads are not) briefly during resharding process.
+
+By default dynamic bucket index resharding can only increase the
+number of bucket index shards to 1999, although this upper-bound is a
+configuration parameter (see Configuration below). When
+possible, the process chooses a prime number of shards in order to
+spread the number of entries across the bucket index
+shards more evenly.
+
+Detection of resharding opportunities runs as a background process
+that periodically
+scans all buckets. A bucket that requires resharding is added to
+a queue. A thread runs in the background and processes the queueued
+resharding tasks, one at a time and in order.
+
+Multisite
+=========
+
+With Ceph releases Prior to Reef, the Ceph Object Gateway (RGW) does not support
+dynamic resharding in a
+multisite environment. For information on dynamic resharding, see
+:ref:`Resharding <feature_resharding>` in the RGW multisite documentation.
+
+Configuration
+=============
+
+Enable/Disable dynamic bucket index resharding:
+
+- ``rgw_dynamic_resharding``:  true/false, default: true
+
+Configuration options that control the resharding process:
+
+- ``rgw_max_objs_per_shard``: maximum number of objects per bucket index shard before resharding is triggered, default: 100000
+
+- ``rgw_max_dynamic_shards``: maximum number of bucket index shards that dynamic resharding can increase to, default: 1999
+
+- ``rgw_reshard_bucket_lock_duration``: duration, in seconds, that writes to the bucket are locked during resharding, default: 360 (i.e., 6 minutes)
+
+- ``rgw_reshard_thread_interval``: maximum time, in seconds, between rounds of resharding queue processing, default: 600 seconds (i.e., 10 minutes)
+
+- ``rgw_reshard_num_logs``: number of shards for the resharding queue, default: 16
+
+Admin commands
+==============
+
+Add a bucket to the resharding queue
+------------------------------------
+
+::
+
+   # radosgw-admin reshard add --bucket <bucket_name> --num-shards <new number of shards>
+
+List resharding queue
+---------------------
+
+::
+
+   # radosgw-admin reshard list
+
+Process tasks on the resharding queue
+-------------------------------------
+
+::
+
+   # radosgw-admin reshard process
+
+Bucket resharding status
+------------------------
+
+::
+
+   # radosgw-admin reshard status --bucket <bucket_name>
+
+The output is a JSON array of 3 objects (reshard_status, new_bucket_instance_id, num_shards) per shard.
+
+For example, the output at each dynamic resharding stage is shown below:
+
+``1. Before resharding occurred:``
+::
+
+  [
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    }
+  ]
+
+``2. During resharding:``
+::
+
+  [
+    {
+        "reshard_status": "in-progress",
+        "new_bucket_instance_id": "1179f470-2ebf-4630-8ec3-c9922da887fd.8652.1",
+        "num_shards": 2
+    },
+    {
+        "reshard_status": "in-progress",
+        "new_bucket_instance_id": "1179f470-2ebf-4630-8ec3-c9922da887fd.8652.1",
+        "num_shards": 2
+    }
+  ]
+
+``3. After resharding completed:``
+::
+
+  [
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    },
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    }
+  ]
+
+
+Cancel pending bucket resharding
+--------------------------------
+
+Note: Bucket resharding operations cannot be cancelled while executing. ::
+
+   # radosgw-admin reshard cancel --bucket <bucket_name>
+
+Manual immediate bucket resharding
+----------------------------------
+
+::
+
+   # radosgw-admin bucket reshard --bucket <bucket_name> --num-shards <new number of shards>
+
+When choosing a number of shards, the administrator must anticipate each
+bucket's peak number of objects. Ideally one should aim for no
+more than 100000 entries per shard at any given time.
+
+Additionally, bucket index shards that are prime numbers are more effective
+in evenly distributing bucket index entries.
+For example, 7001 bucket index shards is better than 7000
+since the former is prime. A variety of web sites have lists of prime
+numbers; search for "list of prime numbers" with your favorite
+search engine to locate some web sites.
+
+Troubleshooting
+===============
+
+Clusters prior to Luminous 12.2.11 and Mimic 13.2.5 left behind stale bucket
+instance entries, which were not automatically cleaned up. This issue also affected
+LifeCycle policies, which were no longer applied to resharded buckets. Both of
+these issues could be worked around by running ``radosgw-admin`` commands.
+
+Stale instance management
+-------------------------
+
+List the stale instances in a cluster that are ready to be cleaned up.
+
+::
+
+   # radosgw-admin reshard stale-instances list
+
+Clean up the stale instances in a cluster. Note: cleanup of these
+instances should only be done on a single-site cluster.
+
+::
+
+   # radosgw-admin reshard stale-instances rm
+
+
+Lifecycle fixes
+---------------
+
+For clusters with resharded instances, it is highly likely that the old
+lifecycle processes would have flagged and deleted lifecycle processing as the
+bucket instance changed during a reshard. While this is fixed for buckets
+deployed on newer Ceph releases (from Mimic 13.2.6 and Luminous 12.2.12),
+older buckets that had lifecycle policies and that have undergone
+resharding must be fixed manually.
+
+The command to do so is:
+
+::
+
+   # radosgw-admin lc reshard fix --bucket {bucketname}
+
+
+If the ``--bucket`` argument is not provided, this
+command will try to fix lifecycle policies for all the buckets in the cluster.
+
+Object Expirer fixes
+--------------------
+
+Objects subject to Swift object expiration on older clusters may have
+been dropped from the log pool and never deleted after the bucket was
+resharded. This would happen if their expiration time was before the
+cluster was upgraded, but if their expiration was after the upgrade
+the objects would be correctly handled. To manage these expire-stale
+objects, ``radosgw-admin`` provides two subcommands.
+
+Listing:
+
+::
+
+   # radosgw-admin objects expire-stale list --bucket {bucketname}
+
+Displays a list of object names and expiration times in JSON format.
+
+Deleting:
+
+::
+
+   # radosgw-admin objects expire-stale rm --bucket {bucketname}
+
+
+Initiates deletion of such objects, displaying a list of object names, expiration times, and deletion status in JSON format.
diff --git a/doc/radosgw/elastic-sync-module.rst b/doc/radosgw/elastic-sync-module.rst
new file mode 100644
index 000000000..60c806e87
--- /dev/null
+++ b/doc/radosgw/elastic-sync-module.rst
@@ -0,0 +1,181 @@
+=========================
+ElasticSearch Sync Module
+=========================
+
+.. versionadded:: Kraken
+
+.. note::
+     As of 31 May 2020, only Elasticsearch 6 and lower are supported. ElasticSearch 7 is not supported.
+
+This sync module writes the metadata from other zones to `ElasticSearch`_. As of
+luminous this is a json of data fields we currently store in ElasticSearch.
+
+::
+
+   {
+        "_index" : "rgw-gold-ee5863d6",
+        "_type" : "object",
+        "_id" : "34137443-8592-48d9-8ca7-160255d52ade.34137.1:object1:null",
+        "_score" : 1.0,
+        "_source" : {
+          "bucket" : "testbucket123",
+          "name" : "object1",
+          "instance" : "null",
+          "versioned_epoch" : 0,
+          "owner" : {
+            "id" : "user1",
+            "display_name" : "user1"
+          },
+          "permissions" : [
+            "user1"
+          ],
+          "meta" : {
+            "size" : 712354,
+            "mtime" : "2017-05-04T12:54:16.462Z",
+            "etag" : "7ac66c0f148de9519b8bd264312c4d64"
+          }
+        }
+      }
+
+
+
+ElasticSearch tier type configurables
+-------------------------------------
+
+* ``endpoint``
+
+Specifies the Elasticsearch server endpoint to access
+
+* ``num_shards`` (integer)
+
+The number of shards that Elasticsearch will be configured with on
+data sync initialization. Note that this cannot be changed after init.
+Any change here requires rebuild of the Elasticsearch index and reinit
+of the data sync process.
+
+* ``num_replicas`` (integer)
+
+The number of the replicas that Elasticsearch will be configured with
+on data sync initialization.
+
+* ``explicit_custom_meta`` (true | false)
+
+Specifies whether all user custom metadata will be indexed, or whether
+user will need to configure (at the bucket level) what custom
+metadata entries should be indexed. This is false by default
+
+* ``index_buckets_list`` (comma separated list of strings)
+
+If empty, all buckets will be indexed. Otherwise, only buckets
+specified here will be indexed. It is possible to provide bucket
+prefixes (e.g., foo\*), or bucket suffixes (e.g., \*bar).
+
+* ``approved_owners_list`` (comma separated list of strings)
+
+If empty, buckets of all owners will be indexed (subject to other
+restrictions), otherwise, only buckets owned by specified owners will
+be indexed. Suffixes and prefixes can also be provided.
+
+* ``override_index_path`` (string)
+
+if not empty, this string will be used as the elasticsearch index
+path. Otherwise the index path will be determined and generated on
+sync initialization.
+
+
+End user metadata queries
+-------------------------
+
+.. versionadded:: Luminous
+
+Since the ElasticSearch cluster now stores object metadata, it is important that
+the ElasticSearch endpoint is not exposed to the public and only accessible to
+the cluster administrators. For exposing metadata queries to the end user itself
+this poses a problem since we'd want the user to only query their metadata and
+not of any other users, this would require the ElasticSearch cluster to
+authenticate users in a way similar to RGW does which poses a problem.
+
+As of Luminous RGW in the metadata master zone can now service end user
+requests. This allows for not exposing the elasticsearch endpoint in public and
+also solves the authentication and authorization problem since RGW itself can
+authenticate the end user requests. For this purpose RGW introduces a new query
+in the bucket APIs that can service elasticsearch requests. All these requests
+must be sent to the metadata master zone.
+
+Syntax
+~~~~~~
+
+Get an elasticsearch query
+``````````````````````````
+
+::
+
+   GET /{bucket}?query={query-expr}
+
+request params:
+ - max-keys: max number of entries to return
+ - marker: pagination marker
+
+``expression := [(]<arg> <op> <value> [)][<and|or> ...]``
+
+op is one of the following:
+<, <=, ==, >=, >
+
+For example ::
+
+  GET /?query=name==foo
+
+Will return all the indexed keys that user has read permission to, and
+are named 'foo'.
+
+The output will be a list of keys in XML that is similar to the S3
+list buckets response.
+
+Configure custom metadata fields
+````````````````````````````````
+
+Define which custom metadata entries should be indexed (under the
+specified bucket), and what are the types of these keys. If explicit
+custom metadata indexing is configured, this is needed so that rgw
+will index the specified custom metadata values. Otherwise it is
+needed in cases where the indexed metadata keys are of a type other
+than string.
+
+::
+
+   POST /{bucket}?mdsearch
+   x-amz-meta-search: <key [; type]> [, ...]
+
+Multiple metadata fields must be comma separated, a type can be forced for a
+field with a `;`. The currently allowed types are string(default), integer and
+date
+
+eg. if you want to index a custom object metadata x-amz-meta-year as int,
+x-amz-meta-date as type date and x-amz-meta-title as string, you'd do
+
+::
+
+   POST /mybooks?mdsearch
+   x-amz-meta-search: x-amz-meta-year;int, x-amz-meta-release-date;date, x-amz-meta-title;string
+
+
+Delete custom metadata configuration
+````````````````````````````````````
+
+Delete custom metadata bucket configuration.
+
+::
+
+   DELETE /<bucket>?mdsearch
+
+Get custom metadata configuration
+`````````````````````````````````
+
+Retrieve custom metadata bucket configuration.
+
+::
+
+   GET /<bucket>?mdsearch
+
+
+.. _`Elasticsearch`: https://github.com/elastic/elasticsearch
diff --git a/doc/radosgw/encryption.rst b/doc/radosgw/encryption.rst
new file mode 100644
index 000000000..e30fe1468
--- /dev/null
+++ b/doc/radosgw/encryption.rst
@@ -0,0 +1,96 @@
+==========
+Encryption
+==========
+
+.. versionadded:: Luminous
+
+The Ceph Object Gateway supports server-side encryption of uploaded objects,
+with 3 options for the management of encryption keys. Server-side encryption
+means that the data is sent over HTTP in its unencrypted form, and the Ceph
+Object Gateway stores that data in the Ceph Storage Cluster in encrypted form.
+
+.. note:: Requests for server-side encryption must be sent over a secure HTTPS
+          connection to avoid sending secrets in plaintext. If a proxy is used
+          for SSL termination, ``rgw trust forwarded https`` must be enabled
+          before forwarded requests will be trusted as secure.
+
+.. note:: Server-side encryption keys must be 256-bit long and base64 encoded.
+
+Customer-Provided Keys
+======================
+
+In this mode, the client passes an encryption key along with each request to
+read or write encrypted data. It is the client's responsibility to manage those
+keys and remember which key was used to encrypt each object.
+
+This is implemented in S3 according to the `Amazon SSE-C`_ specification.
+
+As all key management is handled by the client, no special Ceph configuration
+is needed to support this encryption mode.
+
+Key Management Service
+======================
+
+In this mode, an administrator stores keys in a secure key management service.
+These keys are then
+retrieved on demand by the Ceph Object Gateway to serve requests to encrypt
+or decrypt data.
+
+This is implemented in S3 according to the `Amazon SSE-KMS`_ specification.
+
+In principle, any key management service could be used here.  Currently
+integration with `Barbican`_, `Vault`_, and `KMIP`_ are implemented.
+
+See `OpenStack Barbican Integration`_, `HashiCorp Vault Integration`_,
+and `KMIP Integration`_.
+
+SSE-S3
+======
+
+This makes key management invisible to the user.  They are still stored
+in Vault, but they are automatically created and deleted by Ceph and
+retrieved as required to serve requests to encrypt
+or decrypt data.
+
+This is implemented in S3 according to the `Amazon SSE-S3`_ specification.
+
+In principle, any key management service could be used here.  Currently
+only integration with `Vault`_, is implemented.
+
+See `HashiCorp Vault Integration`_.
+
+Bucket Encryption APIs
+======================
+
+Bucket Encryption APIs to support server-side encryption with Amazon
+S3-managed keys (SSE-S3) or AWS KMS customer master keys (SSE-KMS). 
+
+See `PutBucketEncryption`_, `GetBucketEncryption`_, `DeleteBucketEncryption`_
+
+Automatic Encryption (for testing only)
+=======================================
+
+A ``rgw crypt default encryption key`` can be set in ceph.conf to force the
+encryption of all objects that do not otherwise specify an encryption mode.
+
+The configuration expects a base64-encoded 256 bit key. For example::
+
+  rgw crypt default encryption key = 4YSmvJtBv0aZ7geVgAsdpRnLBEwWSWlMIGnRS8a9TSA=
+
+.. important:: This mode is for diagnostic purposes only! The ceph configuration
+   file is not a secure method for storing encryption keys. Keys that are
+   accidentally exposed in this way should be considered compromised.
+
+
+.. _Amazon SSE-C: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+.. _Amazon SSE-KMS: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+.. _Amazon SSE-S3: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html
+.. _Barbican: https://wiki.openstack.org/wiki/Barbican
+.. _Vault: https://www.vaultproject.io/docs/
+.. _KMIP: http://www.oasis-open.org/committees/kmip/
+.. _PutBucketEncryption: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html
+.. _GetBucketEncryption: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html
+.. _DeleteBucketEncryption: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketEncryption.html
+.. _OpenStack Barbican Integration: ../barbican
+.. _HashiCorp Vault Integration: ../vault
+.. _KMIP Integration: ../kmip
diff --git a/doc/radosgw/frontends.rst b/doc/radosgw/frontends.rst
new file mode 100644
index 000000000..45b29cb6f
--- /dev/null
+++ b/doc/radosgw/frontends.rst
@@ -0,0 +1,163 @@
+.. _rgw_frontends:
+
+==============
+HTTP Frontends
+==============
+
+.. contents::
+
+The Ceph Object Gateway supports two embedded HTTP frontend libraries
+that can be configured with ``rgw_frontends``. See `Config Reference`_
+for details about the syntax.
+
+Beast
+=====
+
+.. versionadded:: Mimic
+
+The ``beast`` frontend uses the Boost.Beast library for HTTP parsing
+and the Boost.Asio library for asynchronous network i/o.
+
+Options
+-------
+
+``port`` and ``ssl_port``
+
+:Description: Sets the ipv4 & ipv6 listening port number. Can be specified multiple
+              times as in ``port=80 port=8000``.
+:Type: Integer
+:Default: ``80``
+
+
+``endpoint`` and ``ssl_endpoint``
+
+:Description: Sets the listening address in the form ``address[:port]``, where
+              the address is an IPv4 address string in dotted decimal form, or
+              an IPv6 address in hexadecimal notation surrounded by square
+              brackets. Specifying a IPv6 endpoint would listen to v6 only. The
+              optional port defaults to 80 for ``endpoint`` and 443 for
+              ``ssl_endpoint``. Can be specified multiple times as in
+              ``endpoint=[::1] endpoint=192.168.0.100:8000``.
+
+:Type: Integer
+:Default: None
+
+
+``ssl_certificate``
+
+:Description: Path to the SSL certificate file used for SSL-enabled endpoints.
+              If path is prefixed with ``config://``, the certificate will be
+              pulled from the ceph monitor ``config-key`` database.
+
+:Type: String
+:Default: None
+
+
+``ssl_private_key``
+
+:Description: Optional path to the private key file used for SSL-enabled
+              endpoints. If one is not given, the ``ssl_certificate`` file
+              is used as the private key.
+              If path is prefixed with ``config://``, the certificate will be
+              pulled from the ceph monitor ``config-key`` database.
+
+:Type: String
+:Default: None
+
+``ssl_options``
+
+:Description: Optional colon separated list of ssl context options:
+
+              ``default_workarounds`` Implement various bug workarounds.
+
+              ``no_compression`` Disable compression.
+
+              ``no_sslv2`` Disable SSL v2.
+
+              ``no_sslv3`` Disable SSL v3.
+
+              ``no_tlsv1`` Disable TLS v1.
+
+              ``no_tlsv1_1`` Disable TLS v1.1.
+
+              ``no_tlsv1_2`` Disable TLS v1.2.
+
+              ``single_dh_use`` Always create a new key when using tmp_dh parameters.
+
+:Type: String
+:Default: ``no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1``
+
+``ssl_ciphers``
+
+:Description: Optional list of one or more cipher strings separated by colons.
+              The format of the string is described in openssl's ciphers(1)
+              manual.
+
+:Type: String
+:Default: None
+
+``tcp_nodelay``
+
+:Description: If set the socket option will disable Nagle's algorithm on 
+              the connection which means that packets will be sent as soon 
+              as possible instead of waiting for a full buffer or timeout to occur.
+
+              ``1`` Disable Nagel's algorithm for all sockets.
+
+              ``0`` Keep the default: Nagel's algorithm enabled.
+
+:Type: Integer (0 or 1)
+:Default: 0
+
+``max_connection_backlog``
+
+:Description: Optional value to define the maximum size for the queue of
+              connections waiting to be accepted. If not configured, the value
+              from ``boost::asio::socket_base::max_connections`` will be used.
+
+:Type: Integer
+:Default: None
+
+``request_timeout_ms``
+
+:Description: The amount of time in milliseconds that Beast will wait
+              for more incoming data or outgoing data before giving up.
+              Setting this value to 0 will disable timeout.
+
+:Type: Integer
+:Default: ``65000``
+
+``rgw_thread_pool_size``
+
+:Description: Sets the number of threads spawned by Beast to handle
+              incoming HTTP connections. This effectively limits the number
+              of concurrent connections that the frontend can service.
+
+:Type: Integer
+:Default: ``512``
+
+``max_header_size``
+
+:Description: The maximum number of header bytes available for a single request.
+
+:Type: Integer
+:Default: ``16384``
+:Maximum: ``65536``
+
+
+Generic Options
+===============
+
+Some frontend options are generic and supported by all frontends:
+
+``prefix``
+
+:Description: A prefix string that is inserted into the URI of all
+              requests. For example, a swift-only frontend could supply
+              a uri prefix of ``/swift``.
+
+:Type: String
+:Default: None
+
+
+.. _Config Reference: ../config-ref
diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst
new file mode 100644
index 000000000..704436202
--- /dev/null
+++ b/doc/radosgw/index.rst
@@ -0,0 +1,87 @@
+.. _object-gateway:
+
+=====================
+ Ceph Object Gateway
+=====================
+
+:term:`Ceph Object Gateway` is an object storage interface built on top of
+``librados``. It provides a RESTful gateway between applications and Ceph
+Storage Clusters. :term:`Ceph Object Storage` supports two interfaces:
+
+#. **S3-compatible:** Provides object storage functionality with an interface
+   that is compatible with a large subset of the Amazon S3 RESTful API.
+
+#. **Swift-compatible:** Provides object storage functionality with an interface
+   that is compatible with a large subset of the OpenStack Swift API.
+
+Ceph Object Storage uses the Ceph Object Gateway daemon (``radosgw``), an HTTP
+server designed for interacting with a Ceph Storage Cluster. The Ceph Object
+Gateway provides interfaces that are compatible with both Amazon S3 and
+OpenStack Swift, and it has its own user management. Ceph Object Gateway can
+store data in the same Ceph Storage Cluster in which data from Ceph File System
+clients and Ceph Block Device clients is stored. The S3 API and the Swift API
+share a common namespace, which makes it possible to write data to a Ceph
+Storage Cluster with one API and then retrieve that data with the other API.
+
+.. ditaa::
+
+            +------------------------+ +------------------------+
+            |   S3 compatible API    | |  Swift compatible API  |
+            +------------------------+-+------------------------+
+            |                      radosgw                      |
+            +---------------------------------------------------+
+            |                      librados                     |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. note:: Ceph Object Storage does **NOT** use the Ceph Metadata Server.
+
+
+.. toctree::
+   :maxdepth: 1
+
+   HTTP Frontends <frontends>
+   Multisite Configuration <multisite>
+   Pool Placement and Storage Classes <placement>
+   Multisite Sync Policy Configuration <multisite-sync-policy>
+   Configuring Pools <pools>
+   Config Reference <config-ref>
+   Admin Guide <admin>
+   S3 API <s3>
+   Data caching and CDN <rgw-cache.rst>
+   Swift API <swift>
+   Admin Ops API <adminops>
+   Python binding <api>
+   Export over NFS <nfs>
+   OpenStack Keystone Integration <keystone>
+   OpenStack Barbican Integration <barbican>
+   HashiCorp Vault Integration <vault>
+   KMIP Integration <kmip>
+   Open Policy Agent Integration <opa>
+   Multi-tenancy <multitenancy>
+   Compression <compression>
+   LDAP Authentication <ldap-auth>
+   Server-Side Encryption <encryption>
+   Bucket Policy <bucketpolicy>
+   Dynamic bucket index resharding <dynamicresharding>
+   Multi factor authentication <mfa>
+   Sync Modules <sync-modules>
+   Bucket Notifications <notifications>
+   Data Layout in RADOS <layout>
+   STS <STS>
+   STS Lite <STSLite>
+   Keycloak <keycloak>
+   Session Tags <session-tags>
+   Role <role>
+   Orphan List and Associated Tooling <orphans>
+   OpenID Connect Provider <oidc>
+   troubleshooting
+   Manpage radosgw <../../man/8/radosgw>
+   Manpage radosgw-admin <../../man/8/radosgw-admin>
+   QAT Acceleration for Encryption and Compression <qat-accel>
+   S3-select <s3select>
+   Lua Scripting <lua-scripting>
+   D3N Data Cache <d3n_datacache>
+   Cloud Transition <cloud-transition>
+
diff --git a/doc/radosgw/keycloak.rst b/doc/radosgw/keycloak.rst
new file mode 100644
index 000000000..ec285a62f
--- /dev/null
+++ b/doc/radosgw/keycloak.rst
@@ -0,0 +1,138 @@
+.. _radosgw_keycloak:
+
+=================================
+Integrating Keycloak with RadosGW
+=================================
+
+If Keycloak is set up as an OpenID Connect Identity Provider, it can be used by
+mobile apps and web apps to authenticate their users. By using the web token
+returned by the authentication process, a mobile app or web app can call
+AssumeRoleWithWebIdentity, receive a set of temporary S3 credentials, and use
+those credentials to make S3 calls.
+
+Setting up Keycloak
+===================
+
+Documentation for installing and operating Keycloak can be found here:
+https://www.keycloak.org/guides.
+
+Configuring Keycloak to talk to RGW
+===================================
+
+To configure Keycloak to talk to RGW, add the following configurables::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting/ decrypting the session token}
+  rgw s3 auth use sts = true
+
+Fetching a web token with Keycloak
+==================================
+
+Several examples of apps authenticating with Keycloak can be found here:
+https://github.com/keycloak/keycloak-quickstarts/blob/latest/docs/getting-started.md.
+
+Here you might consider the example of the app-profile-jee-jsp app (in the link
+above). To fetch the access token (web token) for such an application using the
+grant type 'client_credentials', one can use client id and client secret as
+follows::
+
+    KC_REALM=demo
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+    # Request Tokens for credentials
+    KC_RESPONSE=$( \
+    curl -k -v -X POST \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "scope=openid" \
+    -d "grant_type=client_credentials" \
+    -d "client_id=$KC_CLIENT" \
+    -d "client_secret=$KC_CLIENT_SECRET" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token" \
+    | jq .
+    )
+
+    KC_ACCESS_TOKEN=$(echo $KC_RESPONSE| jq -r .access_token)
+
+It is also possible to fetch an access token for a particular user with the
+grant type 'password'. To fetch such an access token, use client id, client
+secret, username, and password as follows::
+
+    KC_REALM=demo
+    KC_USERNAME=<username>
+    KC_PASSWORD=<userpassword>
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+   # Request Tokens for credentials
+    KC_RESPONSE=$( \
+    curl -k -v -X POST \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "scope=openid" \
+    -d "grant_type=password" \
+    -d "client_id=$KC_CLIENT" \
+    -d "client_secret=$KC_CLIENT_SECRET" \
+    -d "username=$KC_USERNAME" \
+    -d "password=$KC_PASSWORD" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token" \
+    | jq .
+    )
+
+    KC_ACCESS_TOKEN=$(echo $KC_RESPONSE| jq -r .access_token)
+
+``KC_ACCESS_TOKEN`` can be used to invoke ``AssumeRoleWithWebIdentity``: see
+:doc:`STS`.
+
+Adding tags to a user in Keycloak
+=================================
+
+To create a user in Keycloak and add tags to it as its attributes, follow these
+steps:
+
+#. Add a user:
+
+   .. image:: ../images/keycloak-adduser.png
+      :align: center
+
+#. Add user details:
+
+   .. image:: ../images/keycloak-userdetails.png
+      :align: center
+
+#. Add user credentials:
+
+   .. image:: ../images/keycloak-usercredentials.png
+      :align: center
+
+#. Add tags to the 'attributes' tab of the user:
+
+   .. image:: ../images/keycloak-usertags.png
+      :align: center
+
+#. Add a protocol mapper that maps the user attribute to a client:
+
+   .. image:: ../images/keycloak-userclientmapper.png
+      :align: center
+
+After these steps have been completed, the tag 'Department' will appear in the
+JWT (web token), under the 'https://aws.amazon.com/tags' namespace.
+
+Tags can be verified by performing token introspection on a JWT. To introspect
+a token, use ``client id`` and ``client secret`` as follows::
+
+    KC_REALM=demo
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+    curl -k -v \
+    -X POST \
+    -u "$KC_CLIENT:$KC_CLIENT_SECRET" \
+    -d "token=$KC_ACCESS_TOKEN" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token/introspect" \
+    | jq .
diff --git a/doc/radosgw/keystone.rst b/doc/radosgw/keystone.rst
new file mode 100644
index 000000000..20edc3d24
--- /dev/null
+++ b/doc/radosgw/keystone.rst
@@ -0,0 +1,179 @@
+=====================================
+ Integrating with OpenStack Keystone
+=====================================
+
+It is possible to integrate the Ceph Object Gateway with Keystone, the OpenStack
+identity service. This sets up the gateway to accept Keystone as the users
+authority. A user that Keystone authorizes to access the gateway will also be
+automatically created on the Ceph Object Gateway (if didn't exist beforehand). A
+token that Keystone validates will be considered as valid by the gateway.
+
+The following configuration options are available for Keystone integration::
+
+	[client.radosgw.gateway]
+	rgw keystone api version = {keystone api version}
+	rgw keystone url = {keystone server url:keystone server admin port}
+	rgw keystone admin token = {keystone admin token}
+	rgw keystone admin token path = {path to keystone admin token} #preferred
+	rgw keystone accepted roles = {accepted user roles}
+	rgw keystone token cache size = {number of tokens to cache}
+	rgw keystone implicit tenants = {true for private tenant for each new user}
+
+It is also possible to configure a Keystone service tenant, user & password for
+Keystone (for v2.0 version of the OpenStack Identity API), similar to the way
+OpenStack services tend to be configured, this avoids the need for setting the
+shared secret ``rgw keystone admin token`` in the configuration file, which is
+recommended to be disabled in production environments. The service tenant
+credentials should have admin privileges, for more details refer the `OpenStack
+Keystone documentation`_, which explains the process in detail. The requisite
+configuration options for are::
+
+   rgw keystone admin user = {keystone service tenant user name}
+   rgw keystone admin password = {keystone service tenant user password}
+   rgw keystone admin password = {keystone service tenant user password path} # preferred
+   rgw keystone admin tenant = {keystone service tenant name}
+
+
+A Ceph Object Gateway user is mapped into a Keystone ``tenant``. A Keystone user
+has different roles assigned to it on possibly more than a single tenant. When
+the Ceph Object Gateway gets the ticket, it looks at the tenant, and the user
+roles that are assigned to that ticket, and accepts/rejects the request
+according to the ``rgw keystone accepted roles`` configurable.
+
+For a v3 version of the OpenStack Identity API you should replace
+``rgw keystone admin tenant`` with::
+
+   rgw keystone admin domain = {keystone admin domain name}
+   rgw keystone admin project = {keystone admin project name}
+
+For compatibility with previous versions of ceph, it is also
+possible to set ``rgw keystone implicit tenants`` to either
+``s3`` or ``swift``.  This has the effect of splitting
+the identity space such that the indicated protocol will
+only use implicit tenants, and the other protocol will
+never use implicit tenants.  Some older versions of ceph
+only supported implicit tenants with swift.
+
+Ocata (and later)
+-----------------
+
+Keystone itself needs to be configured to point to the Ceph Object Gateway as an
+object-storage endpoint::
+
+  openstack service create --name=swift \
+                           --description="Swift Service" \
+                           object-store
+  +-------------+----------------------------------+
+  | Field       | Value                            |
+  +-------------+----------------------------------+
+  | description | Swift Service                    |
+  | enabled     | True                             |
+  | id          | 37c4c0e79571404cb4644201a4a6e5ee |
+  | name        | swift                            |
+  | type        | object-store                     |
+  +-------------+----------------------------------+
+
+  openstack endpoint create --region RegionOne \
+       --publicurl   "http://radosgw.example.com:8080/swift/v1" \
+       --adminurl    "http://radosgw.example.com:8080/swift/v1" \
+       --internalurl "http://radosgw.example.com:8080/swift/v1" \
+       swift
+  +--------------+------------------------------------------+
+  | Field        | Value                                    |
+  +--------------+------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1 |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3         |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1 |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1 |
+  | region       | RegionOne                                |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee         |
+  | service_name | swift                                    |
+  | service_type | object-store                             |
+  +--------------+------------------------------------------+
+
+  $ openstack endpoint show object-store
+  +--------------+------------------------------------------+
+  | Field        | Value                                    |
+  +--------------+------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1 |
+  | enabled      | True                                     |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3         |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1 |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1 |
+  | region       | RegionOne                                |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee         |
+  | service_name | swift                                    |
+  | service_type | object-store                             |
+  +--------------+------------------------------------------+
+
+.. note:: If your radosgw ``ceph.conf`` sets the configuration option
+	  ``rgw swift account in url = true``, your ``object-store``
+	  endpoint URLs must be set to include the suffix
+	  ``/v1/AUTH_%(tenant_id)s`` (instead of just ``/v1``).
+
+The Keystone URL is the Keystone admin RESTful API URL. The admin token is the
+token that is configured internally in Keystone for admin requests.
+
+OpenStack Keystone may be terminated with a self signed ssl certificate, in
+order for radosgw to interact with Keystone in such a case, you could either
+install Keystone's ssl certificate in the node running radosgw. Alternatively
+radosgw could be made to not verify the ssl certificate at all (similar to
+OpenStack clients with a ``--insecure`` switch) by setting the value of the
+configurable ``rgw keystone verify ssl`` to false.
+
+
+.. _OpenStack Keystone documentation: http://docs.openstack.org/developer/keystone/configuringservices.html#setting-up-projects-users-and-roles
+
+Cross Project(Tenant) Access
+----------------------------
+
+In order to let a project (earlier called a 'tenant') access buckets belonging to a different project, the following config option needs to be enabled::
+
+   rgw swift account in url = true
+
+The Keystone object-store endpoint must accordingly be configured to include the AUTH_%(project_id)s suffix::
+
+   openstack endpoint create --region RegionOne \
+       --publicurl   "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       --adminurl    "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       --internalurl "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       swift
+  +--------------+--------------------------------------------------------------+
+  | Field        | Value                                                        |
+  +--------------+--------------------------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3                             |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | region       | RegionOne                                                    |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee                             |
+  | service_name | swift                                                        |
+  | service_type | object-store                                                 |
+  +--------------+--------------------------------------------------------------+
+
+Keystone integration with the S3 API
+------------------------------------
+
+It is possible to use Keystone for authentication even when using the
+S3 API (with AWS-like access and secret keys), if the ``rgw s3 auth
+use keystone`` option is set. For details, see
+:doc:`s3/authentication`.
+
+Service token support
+---------------------
+
+Service tokens can be enabled to support RadosGW Keystone integration
+to allow expired tokens when coupled with a valid service token in the request.
+
+Enable the support with ``rgw keystone service token enabled`` and use the
+``rgw keystone service token accepted roles`` option to specify which roles are considered
+service roles.
+
+The ``rgw keystone expired token cache expiration`` option can be used to tune the cache
+expiration for an expired token allowed with a service token, please note that this must
+be lower than the ``[token]/allow_expired_window`` option in the Keystone configuration.
+
+Enabling this will cause an expired token given in the X-Auth-Token header to be allowed
+if coupled with a X-Service-Token header that contains a valid token with the accepted
+roles. This can allow long running processes using a user token in X-Auth-Token to function
+beyond the expiration of the token.
diff --git a/doc/radosgw/kmip.rst b/doc/radosgw/kmip.rst
new file mode 100644
index 000000000..988897121
--- /dev/null
+++ b/doc/radosgw/kmip.rst
@@ -0,0 +1,219 @@
+================
+KMIP Integration
+================
+
+`KMIP`_ can be used as a secure key management service for
+`Server-Side Encryption`_ (SSE-KMS).
+
+.. ditaa::
+
+           +---------+       +---------+        +------+      +-------+
+           |  Client |       | RadosGW |        | KMIP |      |  OSD  |
+           +---------+       +---------+        +------+      +-------+
+                | create secret   |                 |             |
+                | key for key ID  |                 |             |
+                |-----------------+---------------->|             |
+                |                 |                 |             |
+                | upload object   |                 |             |
+                | with key ID     |                 |             |
+                |---------------->| request secret  |             |
+                |                 | key for key ID  |             |
+                |                 |---------------->|             |
+                |                 |<----------------|             |
+                |                 | return secret   |             |
+                |                 | key             |             |
+                |                 |                 |             |
+                |                 | encrypt object  |             |
+                |                 | with secret key |             |
+                |                 |--------------+  |             |
+                |                 |              |  |             |
+                |                 |<-------------+  |             |
+                |                 |                 |             |
+                |                 | store encrypted |             |
+                |                 | object          |             |
+                |                 |------------------------------>|
+
+#. `Setting KMIP Access for Ceph`_
+#. `Creating Keys in KMIP`_
+#. `Configure the Ceph Object Gateway`_
+#. `Upload object`_
+
+Before you can use KMIP with ceph, you will need to do three things.
+You will need to associate ceph with client information in KMIP,
+and configure ceph to use that client information.
+You will also need to create 1 or more keys in KMIP.
+
+Setting KMIP Access for Ceph
+============================
+
+Setting up Ceph in KMIP is very dependent on the mechanism(s) supported
+by your implementation of KMIP.  Two implementations are described
+here,
+
+1. `IBM Security Guardium Key Lifecycle Manager (SKLM)`__.  This is a well
+   supported commercial product.
+
+__ SKLM_
+
+2. PyKMIP_.  This is a small python project, suitable for experimental
+   and testing use only.
+
+Using IBM SKLM
+--------------
+
+IBM SKLM__ supports client authentication using certificates.
+Certificates may either be self-signed certificates created,
+for instance, using openssl, or certificates may be created
+using SKLM.  Ceph should then be configured (see below) to
+use KMIP and an attempt made to use it.  This will fail,
+but it will leave an "untrusted client device certificate" in SKLM.
+This can be then upgraded to a registered client using the web
+interface to complete the registration process.
+
+__ SKLM_
+
+Find untrusted clients under ``Advanced Configuration``,
+``Client Device Communication Certificates``.  Select
+``Modify SSL/KMIP Certificates for Clients``, then toggle the flag
+``allow the server to trust this certificate and communicate...``.
+
+Using PyKMIP 
+------------
+
+PyKMIP_ has no special registration process, it simply
+trusts the certificate.  However, the certificate has to
+be issued by a certificate authority that is trusted by
+pykmip.  PyKMIP also prefers that the certificate contain
+an extension for "extended key usage".  However, that
+can be defeated by specifying ``enable_tls_client_auth=False``
+in the server configuration.
+
+Creating Keys in KMIP
+=====================
+
+Some KMIP implementations come with a web interface or other
+administrative tools to create and manage keys.  Refer to your
+documentation on that if you wish to use it.  The KMIP protocol can also
+be used to create and manage keys.  PyKMIP comes with a python client
+library that can be used this way.
+
+In preparation to using the pykmip client, you'll need to have a valid
+kmip client key & certificate, such as the one you created for ceph.
+
+Next, you'll then need to download and install it::
+
+  virtualenv $HOME/my-kmip-env
+  source $HOME/my-kmip-env/bin/activate
+  pip install pykmip
+
+Then you'll need to prepare a configuration file
+for the client, something like this::
+
+   cat <<EOF >$HOME/my-kmip-configuration
+   [client]
+   host={hostname}
+   port=5696
+   certfile={clientcert}
+   keyfile={clientkey}
+   ca_certs={clientca}
+   ssl_version=PROTOCOL_TLSv1_2
+   EOF
+
+You will need to replace {hostname} with the name of your kmip host,
+also replace {clientcert} {clientkey} and {clientca} with pathnames to
+a suitable pem encoded certificate, such as the one you created for
+ceph to use.
+
+Now, you can run this python script directly from
+the shell::
+
+  python
+  from kmip.pie import client
+  from kmip import enums
+  import ssl
+  import os
+  import sys
+  import json
+  c = client.ProxyKmipClient(config_file=os.environ['HOME']+"/my-kmip-configuration")
+
+  while True:
+    l=sys.stdin.readline()
+    keyname=l.strip()
+    if keyname == "": break
+    with c:
+      key_id = c.create(
+	  enums.CryptographicAlgorithm.AES,
+	  256,
+	  operation_policy_name='default',
+	  name=keyname,
+	  cryptographic_usage_mask=[
+	      enums.CryptographicUsageMask.ENCRYPT,
+	      enums.CryptographicUsageMask.DECRYPT
+	  ]
+      )
+      c.activate(key_id)
+      attrs = c.get_attributes(uid=key_id)
+      r = {}
+      for a in attrs[1]:
+       r[str(a.attribute_name)] = str(a.attribute_value)
+      print (json.dumps(r))
+
+If this is all entered at the shell prompt, python will
+prompt with ">>>" then "..." until the script is read in,
+after which it will read and process names with no prompt
+until a blank line or end of file (^D) is given it, or
+an error occurs.  Of course you can turn this into a regular
+python script if you prefer.
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Vault as a KMS backend for
+server-side encryption::
+
+  rgw crypt s3 kms backend = kmip
+  rgw crypt kmip ca path: /etc/ceph/kmiproot.crt
+  rgw crypt kmip client cert: /etc/ceph/kmip-client.crt
+  rgw crypt kmip client key: /etc/ceph/private/kmip-client.key
+  rgw crypt kmip kms key template: pykmip-$keyid
+
+You may need to change the paths above to match where
+you actually want to store kmip certificate data.
+
+The kmip key template describes how ceph will modify
+the name given to it before it looks it up
+in kmip.  The default is just "$keyid".
+If you don't want ceph to see all your kmip
+keys, you can use this to limit ceph to just the
+designated subset of your kmip key namespace.
+
+Upload object
+=============
+
+When uploading an object to the Gateway, provide the SSE key ID in the request.
+As an example, for the kv engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt \
+  s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+  
+As an example, for the transit engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt \
+  s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+
+The Object Gateway will fetch the key from Vault, encrypt the object and store
+it in the bucket. Any request to download the object will make the Gateway
+automatically retrieve the correspondent key from Vault and decrypt the object.
+
+Note that the secret will be fetched from kmip using a name constructed
+from the key template, replacing ``$keyid`` with the key provided.
+
+With the ceph configuration given above,
+radosgw would fetch the secret from::
+
+  pykmip-mybucketkey
+
+.. _Server-Side Encryption: ../encryption
+.. _KMIP: http://www.oasis-open.org/committees/kmip/
+.. _SKLM: https://www.ibm.com/products/ibm-security-key-lifecycle-manager
+.. _PyKMIP: https://pykmip.readthedocs.io/en/latest/
diff --git a/doc/radosgw/layout.rst b/doc/radosgw/layout.rst
new file mode 100644
index 000000000..723adf827
--- /dev/null
+++ b/doc/radosgw/layout.rst
@@ -0,0 +1,208 @@
+===========================
+ Rados Gateway Data Layout
+===========================
+
+Although the source code is the ultimate guide, this document helps
+new developers to get up to speed with the implementation details.
+
+Introduction
+------------
+
+Swift offers something called a *container*, which we use interchangeably with
+the term *bucket*, so we say that RGW's buckets implement Swift containers.
+
+This document does not consider how RGW operates on these structures,
+e.g. the use of encode() and decode() methods for serialization and so on.
+
+Conceptual View
+---------------
+
+Although RADOS only knows about pools and objects with their xattrs and
+omap[1], conceptually RGW organizes its data into three different kinds:
+metadata, bucket index, and data.
+
+Metadata
+^^^^^^^^
+
+We have 3 'sections' of metadata: 'user', 'bucket', and 'bucket.instance'.
+You can use the following commands to introspect metadata entries: ::
+
+    $ radosgw-admin metadata list
+    $ radosgw-admin metadata list bucket
+    $ radosgw-admin metadata list bucket.instance
+    $ radosgw-admin metadata list user
+
+    $ radosgw-admin metadata get bucket:<bucket>
+    $ radosgw-admin metadata get bucket.instance:<bucket>:<bucket_id>
+    $ radosgw-admin metadata get user:<user>   # get or set
+    
+Some variables have been used in above commands, they are:
+
+- user: Holds user information
+- bucket: Holds a mapping between bucket name and bucket instance id
+- bucket.instance: Holds bucket instance information[2]
+
+Every metadata entry is kept on a single RADOS object. See below for implementation details.
+
+Note that the metadata is not indexed. When listing a metadata section we do a
+RADOS ``pgls`` operation on the containing pool.
+
+Bucket Index
+^^^^^^^^^^^^
+
+It's a different kind of metadata, and kept separately. The bucket index holds
+a key-value map in RADOS objects. By default it is a single RADOS object per
+bucket, but it is possible since Hammer to shard that map over multiple RADOS
+objects. The map itself is kept in omap, associated with each RADOS object.
+The key of each omap is the name of the objects, and the value holds some basic
+metadata of that object -- metadata that shows up when listing the bucket.
+Also, each omap holds a header, and we keep some bucket accounting metadata
+in that header (number of objects, total size, etc.).
+
+Note that we also hold other information in the bucket index, and it's kept in
+other key namespaces. We can hold the bucket index log there, and for versioned
+objects there is more information that we keep on other keys.
+
+Data
+^^^^
+
+Objects data is kept in one or more RADOS objects for each rgw object.
+
+Object Lookup Path
+------------------
+
+When accessing objects, REST APIs come to RGW with three parameters:
+account information (access key in S3 or account name in Swift),
+bucket or container name, and object name (or key). At present, RGW only
+uses account information to find out the user ID and for access control.
+Only the bucket name and object key are used to address the object in a pool.
+
+The user ID in RGW is a string, typically the actual user name from the user
+credentials and not a hashed or mapped identifier.
+
+When accessing a user's data, the user record is loaded from an object
+"<user_id>" in pool "default.rgw.meta" with namespace "users.uid".
+
+Bucket names are represented in the pool "default.rgw.meta" with namespace
+"root". Bucket record is
+loaded in order to obtain so-called marker, which serves as a bucket ID.
+
+The object is located in pool "default.rgw.buckets.data".
+Object name is "<marker>_<key>",
+for example "default.7593.4_image.png", where the marker is "default.7593.4"
+and the key is "image.png". Since these concatenated names are not parsed,
+only passed down to RADOS, the choice of the separator is not important and
+causes no ambiguity. For the same reason, slashes are permitted in object
+names (keys).
+
+It is also possible to create multiple data pools and make it so that
+different users\` buckets will be created in different RADOS pools by default,
+thus providing the necessary scaling. The layout and naming of these pools
+is controlled by a 'policy' setting.[3]
+
+An RGW object may consist of several RADOS objects, the first of which
+is the head that contains the metadata, such as manifest, ACLs, content type,
+ETag, and user-defined metadata. The metadata is stored in xattrs.
+The head may also contain up to :confval:`rgw_max_chunk_size` of object data, for efficiency
+and atomicity. The manifest describes how each object is laid out in RADOS
+objects.
+
+Bucket and Object Listing
+-------------------------
+
+Buckets that belong to a given user are listed in an omap of an object named
+"<user_id>.buckets" (for example, "foo.buckets") in pool "default.rgw.meta"
+with namespace "users.uid".
+These objects are accessed when listing buckets, when updating bucket
+contents, and updating and retrieving bucket statistics (e.g. for quota).
+
+See the user-visible, encoded class 'cls_user_bucket_entry' and its
+nested class 'cls_user_bucket' for the values of these omap entries.
+
+These listings are kept consistent with buckets in pool ".rgw".
+
+Objects that belong to a given bucket are listed in a bucket index,
+as discussed in sub-section 'Bucket Index' above. The default naming
+for index objects is ".dir.<marker>" in pool "default.rgw.buckets.index".
+
+Footnotes
+---------
+
+[1] Omap is a key-value store, associated with an object, in a way similar
+to how Extended Attributes associate with a POSIX file. An object's omap
+is not physically located in the object's storage, but its precise
+implementation is invisible and immaterial to RADOS Gateway.
+In Hammer, LevelDB is used to store omap data within each OSD; later releases
+default to RocksDB but can be configured to use LevelDB.
+
+[2] Before the Dumpling release, the 'bucket.instance' metadata did not
+exist and the 'bucket' metadata contained its information. It is possible
+to encounter such buckets in old installations.
+
+[3] Pool names changed with the Infernalis release.
+If you are looking at an older setup, some details may be different. In
+particular there was a different pool for each of the namespaces that are
+now being used inside the ``default.root.meta`` pool.
+
+Appendix: Compendium
+--------------------
+
+Known pools:
+
+.rgw.root
+  Unspecified region, zone, and global information records, one per object.
+
+<zone>.rgw.control
+  notify.<N>
+
+<zone>.rgw.meta
+  Multiple namespaces with different kinds of metadata:
+
+  namespace: root
+    <bucket>
+    .bucket.meta.<bucket>:<marker>   # see put_bucket_instance_info()
+
+    The tenant is used to disambiguate buckets, but not bucket instances.
+    Example::
+
+      .bucket.meta.prodtx:test%25star:default.84099.6
+      .bucket.meta.testcont:default.4126.1
+      .bucket.meta.prodtx:testcont:default.84099.4
+      prodtx/testcont
+      prodtx/test%25star
+      testcont
+
+  namespace: users.uid
+    Contains _both_ per-user information (RGWUserInfo) in "<user>" objects
+    and per-user lists of buckets in omaps of "<user>.buckets" objects.
+    The "<user>" may contain the tenant if non-empty, for example::
+
+      prodtx$prodt
+      test2.buckets
+      prodtx$prodt.buckets
+      test2
+
+  namespace: users.email
+    Unimportant
+
+  namespace: users.keys
+    47UA98JSTJZ9YAN3OS3O
+
+    This allows ``radosgw`` to look up users by their access keys during authentication.
+
+  namespace: users.swift
+    test:tester
+
+<zone>.rgw.buckets.index
+  Objects are named ".dir.<marker>", each contains a bucket index.
+  If the index is sharded, each shard appends the shard index after
+  the marker.
+
+<zone>.rgw.buckets.data
+  default.7593.4__shadow_.488urDFerTYXavx4yAd-Op8mxehnvTI_1
+  <marker>_<key>
+
+An example of a marker would be "default.16004.1" or "default.7593.4".
+The current format is "<zone>.<instance_id>.<bucket_id>". But once
+generated, a marker is not parsed again, so its format may change
+freely in the future.
diff --git a/doc/radosgw/ldap-auth.rst b/doc/radosgw/ldap-auth.rst
new file mode 100644
index 000000000..486d0c623
--- /dev/null
+++ b/doc/radosgw/ldap-auth.rst
@@ -0,0 +1,167 @@
+===================
+LDAP Authentication
+===================
+
+.. versionadded:: Jewel
+
+You can delegate the Ceph Object Gateway authentication to an LDAP server.
+
+How it works
+============
+
+The Ceph Object Gateway extracts the users LDAP credentials from a token. A
+search filter is constructed with the user name. The Ceph Object Gateway uses
+the configured service account to search the directory for a matching entry. If
+an entry is found, the Ceph Object Gateway attempts to bind to the found
+distinguished name with the password from the token. If the credentials are
+valid, the bind will succeed, and the Ceph Object Gateway will grant access and
+radosgw-user will be created with the provided username.
+
+You can limit the allowed users by setting the base for the search to a
+specific organizational unit or by specifying a custom search filter, for
+example requiring specific group membership, custom object classes, or
+attributes.
+
+The LDAP credentials must be available on the server to perform the LDAP
+authentication. Make sure to set the ``rgw`` log level low enough to hide the
+base-64-encoded credentials / access tokens.
+
+Requirements
+============
+
+- **LDAP or Active Directory:** A running LDAP instance accessible by the Ceph
+  Object Gateway
+- **Service account:** LDAP credentials to be used by the Ceph Object Gateway
+  with search permissions
+- **User account:** At least one user account in the LDAP directory
+- **Do not overlap LDAP and local users:** You should not use the same user
+  names for local users and for users being authenticated by using LDAP. The
+  Ceph Object Gateway cannot distinguish them and it treats them as the same
+  user.
+
+Sanity checks
+=============
+
+Use the ``ldapsearch`` utility to verify the service account or the LDAP connection:
+
+::
+
+  # ldapsearch -x -D "uid=ceph,ou=system,dc=example,dc=com" -W \
+  -H ldaps://example.com -b "ou=users,dc=example,dc=com" 'uid=*' dn
+
+.. note:: Make sure to use the same LDAP parameters like in the Ceph configuration file to
+          eliminate possible problems.
+
+Configuring the Ceph Object Gateway to use LDAP authentication
+==============================================================
+
+The following parameters in the Ceph configuration file are related to the LDAP
+authentication:
+
+- ``rgw_s3_auth_use_ldap``: Set this to ``true`` to enable S3 authentication with LDAP
+- ``rgw_ldap_uri``:  Specifies the LDAP server to use. Make sure to use the
+  ``ldaps://<fqdn>:<port>`` parameter to not transmit clear text credentials
+  over the wire.
+- ``rgw_ldap_binddn``: The Distinguished Name (DN) of the service account used
+  by the Ceph Object Gateway
+- ``rgw_ldap_secret``: Path to file containing credentials for ``rgw_ldap_binddn``
+- ``rgw_ldap_searchdn``: Specifies the base in the directory information tree
+  for searching users. This might be your users organizational unit or some
+  more specific Organizational Unit (OU).
+- ``rgw_ldap_dnattr``: The attribute being used in the constructed search
+  filter to match a username. Depending on your Directory Information Tree
+  (DIT) this would probably be ``uid`` or ``cn``. The generated filter string
+  will be, e.g., ``cn=some_username``.
+- ``rgw_ldap_searchfilter``: If not specified, the Ceph Object Gateway
+  automatically constructs the search filter with the ``rgw_ldap_dnattr``
+  setting. Use this parameter to narrow the list of allowed users in very
+  flexible ways. Consult the *Using a custom search filter to limit user access
+  section* for details
+
+Using a custom search filter to limit user access
+=================================================
+
+There are two ways to use the ``rgw_search_filter`` parameter:
+
+Specifying a partial filter to further limit the constructed search filter
+--------------------------------------------------------------------------
+
+An example for a partial filter:
+
+::
+
+  "objectclass=inetorgperson"
+
+The Ceph Object Gateway will generate the search filter as usual with the
+user name from the token and the value of ``rgw_ldap_dnattr``. The constructed
+filter is then combined with the partial filter from the ``rgw_search_filter``
+attribute. Depending on the user name and the settings the final search filter
+might become:
+
+::
+
+  "(&(uid=hari)(objectclass=inetorgperson))"
+
+So user ``hari`` will only be granted access if he is found in the LDAP
+directory, has an object class of ``inetorgperson``, and did specify a valid
+password.
+
+Specifying a complete filter
+----------------------------
+
+A complete filter must contain a ``@USERNAME@`` token which will be substituted
+with the user name during the authentication attempt. The ``rgw_ldap_dnattr``
+parameter is not used anymore in this case. For example, to limit valid users
+to a specific group, use the following filter:
+
+::
+
+  "(&(uid=@USERNAME@)(memberOf=cn=ceph-users,ou=groups,dc=mycompany,dc=com))"
+
+.. note:: Using the ``memberOf`` attribute in LDAP searches requires server side
+          support from you specific LDAP server implementation.
+
+Generating an access token for LDAP authentication
+==================================================
+
+The ``radosgw-token`` utility generates the access token based on the LDAP
+user name and password. It will output a base-64 encoded string which is the
+access token.
+
+::
+
+  # export RGW_ACCESS_KEY_ID="<username>"
+  # export RGW_SECRET_ACCESS_KEY="<password>"
+  # radosgw-token --encode
+
+.. important:: The access token is a base-64 encoded JSON struct and contains
+               the LDAP credentials as a clear text.
+
+Alternatively, users can also generate the token manually by base-64-encoding
+this JSON snippet, if they do not have the ``radosgw-token`` tool installed.
+
+::
+
+  {
+    "RGW_TOKEN": {
+      "version": 1,
+      "type": "ldap",
+      "id": "your_username",
+      "key": "your_clear_text_password_here"
+    }
+  }
+
+Using the access token
+======================
+
+Use your favorite S3 client and specify the token as the access key in your
+client or environment variables.
+
+::
+
+  # export AWS_ACCESS_KEY_ID=<base64-encoded token generated by radosgw-token>
+  # export AWS_SECRET_ACCESS_KEY="" # define this with an empty string, otherwise tools might complain about missing env variables.
+
+.. important:: The access token is a base-64 encoded JSON struct and contains
+               the LDAP credentials as a clear text. DO NOT share it unless
+               you want to share your clear text password!
diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst
new file mode 100644
index 000000000..c85f72a6e
--- /dev/null
+++ b/doc/radosgw/lua-scripting.rst
@@ -0,0 +1,570 @@
+=============
+Lua Scripting
+=============
+
+.. versionadded:: Pacific
+
+.. contents::
+
+This feature allows users to assign execution context to Lua scripts. The supported contexts are:
+
+ - ``prerequest`` which will execute a script before each operation is performed
+ - ``postrequest`` which will execute after each operation is performed
+ - ``background`` which will execute within a specified time interval
+ - ``getdata`` which will execute on objects' data when objects are downloaded
+ - ``putdata`` which will execute on objects' data when objects are uploaded
+
+A request (pre or post) or data (get or put) context script may be constrained to operations belonging to a specific tenant's users.
+The request context script can also access fields in the request and modify certain fields, as well as the `Global RGW Table`_.
+The data context script can access the content of the object as well as the request fields and the `Global RGW Table`_. 
+All Lua language features can be used in all contexts.
+
+By default, all Lua standard libraries are available in the script, however, in order to allow for other Lua modules to be used in the script, we support adding packages to an allowlist:
+
+  - All packages in the allowlist are being re-installed using the luarocks package manager on radosgw restart. Therefore a restart is needed for adding or removing of packages to take effect 
+  - To add a package that contains C source code that needs to be compiled, use the ``--allow-compilation`` flag. In this case a C compiler needs to be available on the host
+  - Lua packages are installed in, and used from, a directory local to the radosgw. Meaning that Lua packages in the allowlist are separated from any Lua packages available on the host.
+    By default, this directory would be ``/tmp/luarocks/<entity name>``. Its prefix part (``/tmp/luarocks/``) could be set to a different location via the ``rgw_luarocks_location`` configuration parameter. 
+    Note that this parameter should not be set to one of the default locations where luarocks install packages (e.g. ``$HOME/.luarocks``, ``/usr/lib64/lua``, ``/usr/share/lua``).
+	
+
+.. toctree::
+   :maxdepth: 1
+
+
+Script Management via CLI
+-------------------------
+
+To upload a script:
+   
+
+::
+   
+   # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+
+
+* When uploading a script with the ``background`` context, a tenant name should not be specified.
+* When uploading a script into a cluster deployed with cephadm, use the following command:
+
+::
+
+  # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+
+
+To print the content of the script to standard output:
+
+::
+   
+   # radosgw-admin script get --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+
+
+To remove the script:
+
+::
+   
+   # radosgw-admin script rm --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+
+
+Package Management via CLI
+--------------------------
+
+To add a package to the allowlist:
+
+::
+
+  # radosgw-admin script-package add --package={package name} [--allow-compilation]
+
+
+To add a specific version of a package to the allowlist:
+
+::
+
+  # radosgw-admin script-package add --package='{package name} {package version}' [--allow-compilation]
+
+
+* When adding a different version of a package which already exists in the list, the newly
+  added version will override the existing one.
+
+* When adding a package without a version specified, the latest version of the package
+  will be added.
+
+
+To remove a package from the allowlist:
+
+::
+
+  # radosgw-admin script-package rm --package={package name}
+
+
+To remove a specific version of a package from the allowlist:
+
+::
+
+  # radosgw-admin script-package rm --package='{package name} {package version}'
+
+
+* When removing a package without a version specified, any existing versions of the
+  package will be removed.
+
+
+To print the list of packages in the allowlist:
+
+::
+
+  # radosgw-admin script-package list
+
+
+Context Free Functions
+----------------------
+Debug Log
+~~~~~~~~~
+The ``RGWDebugLog()`` function accepts a string and prints it to the debug log with priority 20.
+Each log message is prefixed ``Lua INFO:``. This function has no return value.
+
+Request Fields
+-----------------
+
+.. warning:: This feature is experimental. Fields may be removed or renamed in the future.
+
+.. note::
+
+    - Although Lua is a case-sensitive language, field names provided by the radosgw are case-insensitive. Function names remain case-sensitive.
+    - Fields marked "optional" can have a nil value.
+    - Fields marked as "iterable" can be used by the pairs() function and with the # length operator.
+    - All table fields can be used with the bracket operator ``[]``.
+    - ``time`` fields are strings with the following format: ``%Y-%m-%d %H:%M:%S``.
+
+
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| Field                                              | Type     | Description                                                  | Iterable | Writeable | Optional |
++====================================================+==========+==============================================================+==========+===========+==========+
+| ``Request.RGWOp``                                  | string   | radosgw operation                                            | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.DecodedURI``                             | string   | decoded URI                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ContentLength``                          | integer  | size of the request                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.GenericAttributes``                      | table    | string to string generic attributes map                      | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response``                               | table    | response to the request                                      | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.HTTPStatusCode``                | integer  | HTTP status code                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.HTTPStatus``                    | string   | HTTP status text                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.RGWCode``                       | integer  | radosgw error code                                           | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.Message``                       | string   | response message                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.SwiftAccountName``                       | string   | swift account name                                           | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket``                                 | table    | info on the bucket                                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Tenant``                          | string   | tenant of the bucket                                         | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Name``                            | string   | bucket name (writeable only in ``prerequest`` context)       | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Marker``                          | string   | bucket marker (initial id)                                   | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Id``                              | string   | bucket id                                                    | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Count``                           | integer  | number of objects in the bucket                              | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Size``                            | integer  | total size of objects in the bucket                          | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.ZoneGroupId``                     | string   | zone group of the bucket                                     | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.CreationTime``                    | time     | creation time of the bucket                                  | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.MTime``                           | time     | modification time of the bucket                              | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota``                           | table    | bucket quota                                                 | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.MaxSize``                   | integer  | bucket quota max size                                        | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.MaxObjects``                | integer  | bucket quota max number of objects                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Reques.Bucket.Quota.Enabled``                    | boolean  | bucket quota is enabled                                      | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.Rounded``                   | boolean  | bucket quota is rounded to 4K                                | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule``                   | table    | bucket placement rule                                        | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule.Name``              | string   | bucket placement rule name                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule.StorageClass``      | string   | bucket placement rule storage class                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User``                            | table    | bucket owner                                                 | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User.Tenant``                     | string   | bucket owner tenant                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User.Id``                         | string   | bucket owner id                                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object``                                 | table    | info on the object                                           | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Name``                            | string   | object name                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Instance``                        | string   | object version                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Id``                              | string   | object id                                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Size``                            | integer  | object size                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.MTime``                           | time     | object mtime                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom``                               | table    | information on copy operation                                | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Tenant``                        | string   | tenant of the object copied from                             | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Bucket``                        | string   | bucket of the object copied from                             | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Object``                        | table    | object copied from. See: ``Request.Object``                  | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner``                            | table    | object owner                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner.DisplayName``                | string   | object owner display name                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner.User``                       | table    | object user. See: ``Request.Bucket.User``                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ZoneGroup.Name``                         | string   | name of zone group                                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ZoneGroup.Endpoint``                     | string   | endpoint of zone group                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl``                                | table    | user ACL                                                     | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Owner``                          | table    | user ACL owner. See: ``Request.ObjectOwner``                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants``                         | table    | user ACL map of string to grant                              | yes      | no        | no       |
+|                                                    |          | note: grants without an Id are not presented when iterated   |          |           |          |
+|                                                    |          | and only one of them can be accessed via brackets            |          |           |          |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"]``               | table    | user ACL grant                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].Type``          | integer  | user ACL grant type                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User``          | table    | user ACL grant user                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User.Tenant``   | table    | user ACL grant user tenant                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User.Id``       | table    | user ACL grant user id                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].GroupType``     | integer  | user ACL grant group type                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].Referer``       | string   | user ACL grant referer                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.BucketAcl``                              | table    | bucket ACL. See: ``Request.UserAcl``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectAcl``                              | table    | object ACL. See: ``Request.UserAcl``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Environment``                            | table    | string to string environment map                             | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy``                                 | table    | policy                                                       | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Text``                            | string   | policy text                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Id``                              | string   | policy Id                                                    | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Statements``                      | table    | list of string statements                                    | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserPolicies``                           | table    | list of user policies                                        | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserPolicies[<index>]``                  | table    | user policy. See: ``Request.Policy``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.RGWId``                                  | string   | radosgw host id: ``<host>-<zone>-<zonegroup>``               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP``                                   | table    | HTTP header                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Parameters``                        | table    | string to string parameter map                               | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Resources``                         | table    | string to string resource map                                | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Metadata``                          | table    | string to string metadata map                                | yes      | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.StorageClass``                      | string   | storage class                                                | no       | yes       | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Host``                              | string   | host name                                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Method``                            | string   | HTTP method                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.URI``                               | string   | URI                                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.QueryString``                       | string   | HTTP query string                                            | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Domain``                            | string   | domain name                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Time``                                   | time     | request time                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Dialect``                                | string   | "S3" or "Swift"                                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Id``                                     | string   | request Id                                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.TransactionId``                          | string   | transaction Id                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Tags``                                   | table    | object tags map                                              | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.User``                                   | table    | user that triggered the request                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.User.Tenant``                            | string   | triggering user tenant                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.User.Id``                                | string   | triggering user id                                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Trace``                                  | table    | info on trace                                                | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Trace.Enable``                           | boolean  | tracing is enabled                                           | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+
+Request Functions
+--------------------
+Operations Log
+~~~~~~~~~~~~~~
+The ``Request.Log()`` function prints the requests into the operations log. This function has no parameters. It returns 0 for success and an error code if it fails.
+
+Tracing
+~~~~~~~
+Tracing functions can be used only in the ``postrequest`` context.
+
+- ``Request.Trace.SetAttribute(<key>, <value>)`` - sets the attribute for the request's trace.
+  The function takes two arguments: the first is the ``key``, which should be a string, and the second is the ``value``, which can either be a string or a number (integer or double).
+  You may then locate specific traces by using this attribute.
+
+- ``Request.Trace.AddEvent(<name>, <attributes>)`` - adds an event to the first span of the request's trace
+  An event is defined by event name, event time, and zero or more event attributes.
+  The function accepts one or two arguments: A string containing the event ``name`` should be the first argument, followed by the event ``attributes``, which is optional for events without attributes.
+  An event's attributes must be a table of strings.
+
+Background Context
+--------------------
+The ``background`` context may be used for purposes that include analytics, monitoring, caching data for other context executions.
+- Background script execution default interval is 5 seconds.
+
+Data Context
+--------------------
+Both ``getdata`` and ``putdata`` contexts have the following fields:
+- ``Data`` which is read-only and iterable (byte by byte). In case that an object is uploaded or retrieved in multiple chunks, the ``Data`` field will hold data of one chunk at a time.
+- ``Offset`` which is holding the offset of the chunk within the entire object.
+- The ``Request`` fields and the background ``RGW`` table are also available in these contexts.
+
+Global RGW Table
+--------------------
+The ``RGW`` Lua table is accessible from all contexts and saves data written to it
+during execution so that it may be read and used later during other executions, from the same context of a different one.
+- Each RGW instance has its own private and ephemeral ``RGW`` Lua table that is lost when the daemon restarts. Note that ``background`` context scripts will run on every instance.
+- The maximum number of entries in the table is 100,000. Each entry has a string key a value with a combined length of no more than 1KB.
+A Lua script will abort with an error if the number of entries or entry size exceeds these limits.
+- The ``RGW`` Lua table uses string indices and can store values of type: string, integer, double and boolean
+
+Increment/Decrement Functions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since entries in the ``RGW`` table could be accessed from multiple places at the same time we need a way
+to atomically increment and decrement numeric values in it. For that the following functions should be used:
+- ``RGW.increment(<key>, [value])`` would increment the value of ``key`` by ``value`` if value is provided or by 1 if not
+- ``RGW.decrement(<key>, [value])`` would decrement the value of ``key`` by ``value`` if value is provided or by 1 if not
+- if the value of ``key`` is not numeric, the execution of the script would fail
+- if we try to increment or decrement by non-numeric values, the execution of the script would fail
+
+
+Lua Code Samples
+----------------
+- Print information on source and destination objects in case of copy:
+
+.. code-block:: lua
+ 
+  function print_object(object)
+    RGWDebugLog("  Name: " .. object.Name)
+    RGWDebugLog("  Instance: " .. object.Instance)
+    RGWDebugLog("  Id: " .. object.Id)
+    RGWDebugLog("  Size: " .. object.Size)
+    RGWDebugLog("  MTime: " .. object.MTime)
+  end
+
+  if Request.CopyFrom and Request.Object and Request.CopyFrom.Object then
+    RGWDebugLog("copy from object:")
+    print_object(Request.CopyFrom.Object)
+    RGWDebugLog("to object:")
+    print_object(Request.Object)
+  end
+
+- Print ACLs via a "generic function":
+
+.. code-block:: lua
+
+  function print_owner(owner)
+    RGWDebugLog("Owner:")
+    RGWDebugLog("  Display Name: " .. owner.DisplayName)
+    RGWDebugLog("  Id: " .. owner.User.Id)
+    RGWDebugLog("  Tenant: " .. owner.User.Tenant)
+  end
+
+  function print_acl(acl_type)
+    index = acl_type .. "ACL"
+    acl = Request[index]
+    if acl then
+      RGWDebugLog(acl_type .. "ACL Owner")
+      print_owner(acl.Owner)
+      RGWDebugLog("  there are " .. #acl.Grants .. " grant for owner")
+      for k,v in pairs(acl.Grants) do
+        RGWDebugLog("    Grant Key: " .. k)
+        RGWDebugLog("    Grant Type: " .. v.Type)
+        RGWDebugLog("    Grant Group Type: " .. v.GroupType)
+        RGWDebugLog("    Grant Referer: " .. v.Referer)
+        RGWDebugLog("    Grant User Tenant: " .. v.User.Tenant)
+        RGWDebugLog("    Grant User Id: " .. v.User.Id)
+      end
+    else
+      RGWDebugLog("no " .. acl_type .. " ACL in request: " .. Request.Id)
+    end 
+  end
+
+  print_acl("User")
+  print_acl("Bucket")
+  print_acl("Object")
+
+- Use of operations log only in case of errors:
+
+.. code-block:: lua
+  
+  if Request.Response.HTTPStatusCode ~= 200 then
+    RGWDebugLog("request is bad, use ops log")
+    rc = Request.Log()
+    RGWDebugLog("ops log return code: " .. rc)
+  end
+
+- Set values into the error message:
+
+.. code-block:: lua
+
+  if Request.Response.HTTPStatusCode == 500 then
+    Request.Response.Message = "<Message> something bad happened :-( </Message>"
+  end
+
+- Add metadata to objects that was not originally sent by the client:
+
+In the ``prerequest`` context we should add:
+
+.. code-block:: lua
+
+  if Request.RGWOp == 'put_obj' then
+    Request.HTTP.Metadata["x-amz-meta-mydata"] = "my value"
+  end
+
+In the ``postrequest`` context we look at the metadata:
+
+.. code-block:: lua
+
+  RGWDebugLog("number of metadata entries is: " .. #Request.HTTP.Metadata)
+  for k, v in pairs(Request.HTTP.Metadata) do
+    RGWDebugLog("key=" .. k .. ", " .. "value=" .. v)
+  end
+ 
+- Use modules to create Unix socket based, JSON encoded, "access log":
+
+First we should add the following packages to the allowlist:
+
+::
+
+  # radosgw-admin script-package add --package=luajson
+  # radosgw-admin script-package add --package=luasocket --allow-compilation
+
+
+Then, do a restart for the radosgw and upload the following script to the ``postrequest`` context:
+
+.. code-block:: lua
+
+  if Request.RGWOp == "get_obj" then
+    local json = require("json")
+    local socket = require("socket")
+    local unix = require("socket.unix")
+    local s = assert(unix())
+    E = {}
+
+    msg = {bucket = (Request.Bucket or (Request.CopyFrom or E).Bucket).Name,
+      time = Request.Time,
+      operation = Request.RGWOp,
+      http_status = Request.Response.HTTPStatusCode,
+      error_code = Request.Response.HTTPStatus,
+      object_size = Request.Object.Size,
+      trans_id = Request.TransactionId}
+
+    assert(s:connect("/tmp/socket"))
+    assert(s:send(json.encode(msg).."\n"))
+    assert(s:close())
+  end
+
+
+- Trace only requests of specific bucket
+
+Tracing is disabled by default, so we should enable tracing for this specific bucket
+
+.. code-block:: lua
+
+  if Request.Bucket.Name == "my-bucket" then
+      Request.Trace.Enable = true
+  end
+
+
+If `tracing is enabled <https://docs.ceph.com/en/latest/jaegertracing/#how-to-enable-tracing-in-ceph/>`_ on the RGW, the value of Request.Trace.Enable is true, so we should disable tracing for all other requests that do not match the bucket name.
+In the ``prerequest`` context:
+
+.. code-block:: lua
+
+  if Request.Bucket.Name ~= "my-bucket" then
+      Request.Trace.Enable = false
+  end
+
+Note that changing ``Request.Trace.Enable`` does not change the tracer's state, but disables or enables the tracing for the request only.
+
+
+- Add Information for requests traces
+
+in ``postrequest`` context, we can add attributes and events to the request's trace.
+
+.. code-block:: lua
+
+  Request.Trace.AddEvent("lua script execution started")
+
+  Request.Trace.SetAttribute("HTTPStatusCode", Request.Response.HTTPStatusCode)
+
+  event_attrs = {}
+  for k,v in pairs(Request.GenericAttributes) do
+    event_attrs[k] = v
+  end
+
+  Request.Trace.AddEvent("second event", event_attrs)
+
+- The entropy value of an object could be used to detect whether the object is encrypted. 
+  The following script calculates the entropy and size of uploaded objects and print to debug log
+
+in the ``putdata`` context, add the following script
+
+.. code-block:: lua
+
+	function object_entropy()
+		local byte_hist = {}
+		local byte_hist_size = 256 
+		for i = 1,byte_hist_size do
+			byte_hist[i] = 0 
+		end 
+		local total = 0 
+
+		for i, c in pairs(Data)  do
+			local byte = c:byte() + 1 
+			byte_hist[byte] = byte_hist[byte] + 1 
+			total = total + 1 
+		end 
+
+		entropy = 0 
+
+		for _, count in ipairs(byte_hist) do
+			if count ~= 0 then
+				local p = 1.0 * count / total
+				entropy = entropy - (p * math.log(p)/math.log(byte_hist_size))
+			end 
+		end 
+
+		return entropy
+	end
+
+	local full_name = Request.Bucket.Name.."\\"..Request.Object.Name
+	RGWDebugLog("entropy of chunk of: " .. full_name .. " at offset:" .. tostring(Offset)  ..  " is: " .. tostring(object_entropy()))
+	RGWDebugLog("payload size of chunk of: " .. full_name .. " is: " .. #Data)
+
diff --git a/doc/radosgw/mfa.rst b/doc/radosgw/mfa.rst
new file mode 100644
index 000000000..416f23af1
--- /dev/null
+++ b/doc/radosgw/mfa.rst
@@ -0,0 +1,102 @@
+.. _rgw_mfa:
+
+==========================================
+RGW Support for Multifactor Authentication
+==========================================
+
+.. versionadded:: Mimic
+
+The S3 multifactor authentication (MFA) feature allows
+users to require the use of one-time password when removing
+objects on certain buckets. The buckets need to be configured
+with versioning and MFA enabled which can be done through
+the S3 api.
+
+Time-based one time password tokens can be assigned to a user
+through radosgw-admin. Each token has a secret seed, and a serial
+id that is assigned to it. Tokens are added to the user, can
+be listed, removed, and can also be re-synchronized.
+
+Multisite
+=========
+
+While the MFA IDs are set on the user's metadata, the
+actual MFA one time password configuration resides in the local zone's
+osds. Therefore, in a multi-site environment it is advisable to use
+different tokens for different zones.
+
+
+Terminology
+=============
+
+-``TOTP``: Time-based One Time Password
+
+-``token serial``: a string that represents the ID of a TOTP token
+
+-``token seed``: the secret seed that is used to calculate the TOTP
+
+-``totp seconds``: the time resolution that is being used for TOTP generation
+
+-``totp window``: the number of TOTP tokens that are checked before and after the current token when validating token
+
+-``totp pin``: the valid value of a TOTP token at a certain time
+
+
+Admin commands
+==============
+
+Create a new MFA TOTP token
+------------------------------------
+
+::
+
+   # radosgw-admin mfa create --uid=<user-id> \
+                              --totp-serial=<serial> \
+                              --totp-seed=<seed> \
+                              [ --totp-seed-type=<hex|base32> ] \
+                              [ --totp-seconds=<num-seconds> ] \
+                              [ --totp-window=<twindow> ]
+
+List MFA TOTP tokens
+---------------------
+
+::
+
+   # radosgw-admin mfa list --uid=<user-id>
+
+
+Show MFA TOTP token
+------------------------------------
+
+::
+
+   # radosgw-admin mfa get --uid=<user-id> --totp-serial=<serial>
+
+
+Delete MFA TOTP token
+------------------------
+
+::
+
+   # radosgw-admin mfa remove --uid=<user-id> --totp-serial=<serial>
+
+
+Check MFA TOTP token
+--------------------------------
+
+Test a TOTP token pin, needed for validating that TOTP functions correctly. ::
+
+   # radosgw-admin mfa check --uid=<user-id> --totp-serial=<serial> \
+                             --totp-pin=<pin>
+
+
+Re-sync MFA TOTP token
+--------------------------------
+
+In order to re-sync the TOTP token (in case of time skew). This requires
+feeding two consecutive pins: the previous pin, and the current pin. ::
+
+   # radosgw-admin mfa resync --uid=<user-id> --totp-serial=<serial> \
+                              --totp-pin=<prev-pin> --totp=pin=<current-pin>
+
+
diff --git a/doc/radosgw/multisite-sync-policy.rst b/doc/radosgw/multisite-sync-policy.rst
new file mode 100644
index 000000000..7a6e7105c
--- /dev/null
+++ b/doc/radosgw/multisite-sync-policy.rst
@@ -0,0 +1,716 @@
+.. _radosgw-multisite-sync-policy:
+
+=====================
+Multisite Sync Policy
+=====================
+
+.. versionadded:: Octopus
+
+Multisite bucket-granularity sync policy provides fine grained control of data movement between buckets in different zones. It extends the zone sync mechanism. Previously buckets were being treated symmetrically, that is -- each (data) zone holds a mirror of that bucket that should be the same as all the other zones. Whereas leveraging the bucket-granularity sync policy is possible for buckets to diverge, and a bucket can pull data from other buckets (ones that don't share its name or its ID) in different zone.  The sync process was assuming therefore that the bucket sync source and the bucket sync destination were always referring to the same bucket, now that is not the case anymore.
+
+The sync policy supersedes the old zonegroup coarse configuration (sync_from*). The sync policy can be configured at the zonegroup level (and if it is configured it replaces the old style config), but it can also be configured at the bucket level.
+
+In the sync policy multiple groups that can contain lists of data-flow configurations can be defined, as well as lists of pipe configurations. The data-flow defines the flow of data between the different zones. It can define symmetrical data flow, in which multiple zones sync data from each other, and it can define directional data flow, in which the data moves in one way from one zone to another.
+
+A pipe defines the actual buckets that can use these data flows, and the properties that are associated with it (for example: source object prefix).
+
+A sync policy group can be in 3 states:
+
++----------------------------+----------------------------------------+
+|  Value                     | Description                            |
++============================+========================================+
+| ``enabled``                | sync is allowed and enabled            |
++----------------------------+----------------------------------------+
+| ``allowed``                | sync is allowed                        |
++----------------------------+----------------------------------------+
+| ``forbidden``              | sync (as defined by this group) is not |
+|                            | allowed and can override other groups  |
++----------------------------+----------------------------------------+
+
+A policy can be defined at the bucket level. A bucket level sync policy inherits the data flow of the zonegroup policy, and can only define a subset of what the zonegroup allows.
+
+A wildcard zone, and a wildcard bucket parameter in the policy defines all relevant zones, or all relevant buckets. In the context of a bucket policy it means the current bucket instance.  A disaster recovery configuration where entire zones are mirrored doesn't require configuring anything on the buckets. However, for a fine grained bucket sync it would be better to configure the pipes to be synced by allowing (status=allowed) them at the zonegroup level (e.g., using wildcards), but only enable the specific sync at the bucket level (status=enabled). If needed, the policy at the bucket level can limit the data movement to specific relevant zones.
+
+.. important:: Any changes to the zonegroup policy needs to be applied on the
+               zonegroup master zone, and require period update and commit. Changes
+               to the bucket policy needs to be applied on the zonegroup master
+               zone. The changes are dynamically handled by rgw.
+
+
+S3 Replication API
+~~~~~~~~~~~~~~~~~~
+
+The S3 bucket replication api has also been implemented, and allows users to create replication rules between different buckets. Note though that while the AWS replication feature allows bucket replication within the same zone, rgw does not allow it at the moment.  However, the rgw api also added a new 'Zone' array that allows users to select to what zones the specific bucket will be synced.
+
+
+Sync Policy Control Reference
+=============================
+
+
+Get Sync Policy
+~~~~~~~~~~~~~~~
+
+To retrieve the current zonegroup sync policy, or a specific bucket policy:
+
+::
+
+     # radosgw-admin sync policy get [--bucket=<bucket>]
+
+
+Create Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To create a sync policy group:
+
+::
+
+      # radosgw-admin sync group create [--bucket=<bucket>]                      \
+                                        --group-id=<group-id>                    \
+                                        --status=<enabled | allowed | forbidden> \
+                                       
+      
+Modify Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To modify a sync policy group:
+
+::
+
+      # radosgw-admin sync group modify [--bucket=<bucket>]                      \
+                                        --group-id=<group-id>                    \
+                                        --status=<enabled | allowed | forbidden> \
+
+
+Show Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To show a sync policy group:
+
+::
+
+      # radosgw-admin sync group get [--bucket=<bucket>]       \
+                                        --group-id=<group-id>
+                                       
+
+Remove Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To remove a sync policy group:
+
+::
+
+      # radosgw-admin sync group remove [--bucket=<bucket>]    \
+                                        --group-id=<group-id>
+
+
+
+Create Sync Flow
+~~~~~~~~~~~~~~~~
+
+- To create or update directional sync flow:
+
+::
+
+      # radosgw-admin sync group flow create [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=directional      \
+                                             --source-zone=<source_zone>  \
+                                             --dest-zone=<dest_zone>
+
+
+- To create or update symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow create [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical      \
+                                             --zones=<zones>
+
+
+Where zones are a comma separated list of all the zones that need to add to the flow.
+
+
+Remove Sync Flow Zones
+~~~~~~~~~~~~~~~~~~~~~~
+
+- To remove directional sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=directional      \
+                                             --source-zone=<source_zone>  \
+                                             --dest-zone=<dest_zone>
+
+
+- To remove specific zones from symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical      \
+                                             --zones=<zones>
+
+
+Where zones are a comma separated list of all zones to remove from the flow.
+
+                                             
+- To remove symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical
+
+
+Create Sync Pipe
+~~~~~~~~~~~~~~~~
+
+To create sync group pipe, or update its parameters:
+
+
+::
+
+      # radosgw-admin sync group pipe create [--bucket=<bucket>]                      \
+                                             --group-id=<group-id>                    \
+                                             --pipe-id=<pipe-id>                      \
+                                             --source-zones=<source_zones>            \
+                                             [--source-bucket=<source_buckets>]       \
+                                             [--source-bucket-id=<source_bucket_id>]  \
+                                             --dest-zones=<dest_zones>                \
+                                             [--dest-bucket=<dest_buckets>]           \
+                                             [--dest-bucket-id=<dest_bucket_id>]      \
+                                             [--prefix=<source_prefix>]               \
+                                             [--prefix-rm]                            \
+                                             [--tags-add=<tags>]                      \
+                                             [--tags-rm=<tags>]                       \
+                                             [--dest-owner=<owner>]                   \
+                                             [--storage-class=<storage_class>]        \
+                                             [--mode=<system | user>]                 \
+                                             [--uid=<user_id>]
+
+
+Zones are either a list of zones, or '*' (wildcard). Wildcard zones mean any zone that matches the sync flow rules.
+Buckets are either a bucket name, or '*' (wildcard). Wildcard bucket means the current bucket
+Prefix can be defined to filter source objects.
+Tags are passed by a comma separated list of 'key=value'.
+Destination owner can be set to force a destination owner of the objects. If user mode is selected, only the destination bucket owner can be set.
+Destination storage class can also be configured.
+User id can be set for user mode, and will be the user under which the sync operation will be executed (for permissions validation).
+
+
+Remove Sync Pipe
+~~~~~~~~~~~~~~~~
+
+To remove specific sync group pipe params, or the entire pipe:
+
+
+::
+
+      # radosgw-admin sync group pipe remove [--bucket=<bucket>]                     \
+                                             --group-id=<group-id>                   \
+                                             --pipe-id=<pipe-id>                     \
+                                             [--source-zones=<source_zones>]         \
+                                             [--source-bucket=<source_buckets>]      \
+                                             [--source-bucket-id=<source_bucket_id>] \
+                                             [--dest-zones=<dest_zones>]             \
+                                             [--dest-bucket=<dest_buckets>]          \
+                                             [--dest-bucket-id=<dest_bucket_id>]
+
+
+Sync Info
+~~~~~~~~~
+
+To get information about the expected sync sources and targets (as defined by the sync policy):
+
+::
+
+      # radosgw-admin sync info [--bucket=<bucket>]             \
+                                [--effective-zone-name=<zone>]
+
+
+Since a bucket can define a policy that defines data movement from it towards a different bucket at a different zone, when the policy is created we also generate a list of bucket dependencies that are used as hints when a sync of any particular bucket happens. The fact that a bucket references another bucket does not mean it actually syncs to/from it, as the data flow might not permit it.  
+
+
+Examples
+========
+
+The system in these examples includes 3 zones: ``us-east`` (the master zone), ``us-west``, ``us-west-2``.
+
+Example 1: Two Zones, Complete Mirror
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is similar to older (pre ``Octopus``) sync capabilities, but being done via the new sync policy engine. Note that changes to the zonegroup sync policy require a period update and commit.
+
+
+::
+
+      [us-east] $ radosgw-admin sync group create --group-id=group1 --status=allowed
+      [us-east] $ radosgw-admin sync group flow create --group-id=group1 \
+                                --flow-id=flow-mirror --flow-type=symmetrical \
+                                --zones=us-east,us-west
+      [us-east] $ radosgw-admin sync group pipe create --group-id=group1 \
+                                --pipe-id=pipe1 --source-zones='*' \
+                                --source-bucket='*' --dest-zones='*' \
+                                --dest-bucket='*'
+      [us-east] $ radosgw-admin sync group modify --group-id=group1 --status=enabled
+      [us-east] $ radosgw-admin period update --commit
+
+      $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "params": {
+      ...
+                  }
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+          }
+      }
+
+
+Note that the "id" field in the output above reflects the pipe rule
+that generated that entry, a single rule can generate multiple sync
+entries as can be seen in the example.
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  ...
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+      }
+
+
+
+Example 2: Directional, Entire Zone Backup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Also similar to older sync capabilities. In here we add a third zone, ``us-west-2`` that will be a replica of ``us-west``, but data will not be replicated back from it.
+
+::
+
+      [us-east] $ radosgw-admin sync group flow create --group-id=group1 \
+                                --flow-id=us-west-backup --flow-type=directional \
+                                --source-zone=us-west --dest-zone=us-west-2
+      [us-east] $ radosgw-admin period update --commit
+
+
+Note that us-west has two dests:
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              },
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west-2",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+      }
+
+
+Whereas us-west-2 has only source and no destinations:
+
+::
+
+      [us-west-2] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west-2",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          "dests": [],
+          ...
+      }
+
+      
+      
+Example 3: Mirror a Specific Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Using the same group configuration, but this time switching it to ``allowed`` state, which means that sync is allowed but not enabled.
+
+::
+
+      [us-east] $ radosgw-admin sync group modify --group-id=group1 --status=allowed
+      [us-east] $ radosgw-admin period update --commit
+
+
+And we will create a bucket level policy rule for existing bucket ``buck2``. Note that the bucket needs to exist before being able to set this policy, and admin commands that modify bucket policies need to run on the master zone, however, they do not require period update.  There is no need to change the data flow, as it is inherited from the zonegroup policy. A bucket policy flow will only be a subset of the flow defined in the zonegroup policy. Same goes for pipes, although a bucket policy can enable pipes that are not enabled (albeit not forbidden) at the zonegroup policy.
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck2 \
+                                --group-id=buck2-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck2 \
+                                --group-id=buck2-default --pipe-id=pipe1 \
+                                --source-zones='*' --dest-zones='*'
+
+
+
+Example 4: Limit Bucket Sync To Specific Zones
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This will only sync ``buck3`` to ``us-east`` (from any zone that flow allows to sync into ``us-east``).
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck3 \
+                                --group-id=buck3-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck3 \
+                                --group-id=buck3-default --pipe-id=pipe1 \
+                                --source-zones='*' --dest-zones=us-east
+
+
+
+Example 5: Sync From a Different Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note that bucket sync only works (currently) across zones and not within the same zone.
+
+Set ``buck4`` to pull data from ``buck5``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck4 '
+                                --group-id=buck4-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck4 \
+                                --group-id=buck4-default --pipe-id=pipe1 \
+                                --source-zones='*' --source-bucket=buck5 \
+                                --dest-zones='*'
+
+
+can also limit it to specific zones, for example the following will
+only sync data originated in us-west:
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe modify --bucket=buck4 \
+                                --group-id=buck4-default --pipe-id=pipe1 \
+                                --source-zones=us-west --source-bucket=buck5 \
+                                --dest-zones='*'
+
+
+Checking the sync info for ``buck5`` on ``us-west`` is interesting:
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck5
+      {
+          "sources": [],
+          "dests": [],
+          "hints": {
+              "sources": [],
+              "dests": [
+                  "buck4:115b12b3-....14433.2"
+              ]
+          },
+          "resolved-hints-1": {
+              "sources": [],
+              "dests": [
+                  {
+                      "id": "pipe1",
+                      "source": {
+                          "zone": "us-west",
+                          "bucket": "buck5"
+                      },
+                      "dest": {
+                          "zone": "us-east",
+                          "bucket": "buck4:115b12b3-....14433.2"
+                      },
+                      ...
+                  },
+                  {
+                      "id": "pipe1",
+                      "source": {
+                          "zone": "us-west",
+                          "bucket": "buck5"
+                      },
+                      "dest": {
+                          "zone": "us-west-2",
+                          "bucket": "buck4:115b12b3-....14433.2"
+                      },
+                      ...
+                  }
+              ]
+          },
+          "resolved-hints": {
+              "sources": [],
+              "dests": []
+          }
+      }
+
+
+Note that there are resolved hints, which means that the bucket ``buck5`` found about ``buck4`` syncing from it indirectly, and not from its own policy (the policy for ``buck5`` itself is empty).
+
+
+Example 6: Sync To Different Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The same mechanism can work for configuring data to be synced to (vs.  synced from as in the previous example). Note that internally data is still pulled from the source at the destination zone:
+
+Set ``buck6`` to "push" data to ``buck5``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck6 \
+                                --group-id=buck6-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck6 \
+                                --group-id=buck6-default --pipe-id=pipe1 \
+                                --source-zones='*' --source-bucket='*' \
+                                --dest-zones='*' --dest-bucket=buck5
+
+
+A wildcard bucket name means the current bucket in the context of bucket sync policy.
+
+Combined with the configuration in Example 5, we can now write data to ``buck6`` on ``us-east``, data will sync to ``buck5`` on ``us-west``, and from there it will be distributed to ``buck4`` on ``us-east``, and on ``us-west-2``.
+
+Example 7: Source Filters
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sync from ``buck8`` to ``buck9``, but only objects that start with ``foo/``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck8 \
+                                --group-id=buck8-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck8 \
+                                --group-id=buck8-default --pipe-id=pipe-prefix \
+                                --prefix=foo/ --source-zones='*' --dest-zones='*' \
+                                --dest-bucket=buck9
+
+
+Also sync from ``buck8`` to ``buck9`` any object that has the tags ``color=blue`` or ``color=red``:
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck8 \
+                                --group-id=buck8-default --pipe-id=pipe-tags \
+                                --tags-add=color=blue,color=red --source-zones='*' \
+                                --dest-zones='*' --dest-bucket=buck9
+
+
+And we can check the expected sync in ``us-east`` (for example):
+
+::
+
+      [us-east] $ radosgw-admin sync info --bucket=buck8
+      {
+          "sources": [],
+          "dests": [
+              {
+                  "id": "pipe-prefix",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck8:115b12b3-....14433.5"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck9"
+                  },
+                  "params": {
+                      "source": {
+                          "filter": {
+                              "prefix": "foo/",
+                              "tags": []
+                          }
+                      },
+                      ...
+                  }
+              },
+              {
+                  "id": "pipe-tags",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck8:115b12b3-....14433.5"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck9"
+                  },
+                  "params": {
+                      "source": {
+                          "filter": {
+                              "tags": [
+                                  {
+                                      "key": "color",
+                                      "value": "blue"
+                                  },
+                                  {
+                                      "key": "color",
+                                      "value": "red"
+                                  }
+                              ]
+                          }
+                      },
+                      ...
+                  }
+              }
+          ],
+          ...
+      }
+
+
+Note that there aren't any sources, only two different destinations (one for each configuration). When the sync process happens it will select the relevant rule for each object it syncs.
+
+Prefixes and tags can be combined, in which object will need to have both in order to be synced. The priority param can also be passed, and it can be used to determine when there are multiple different rules that are matched (and have the same source and destination), to determine which of the rules to be used.
+
+
+Example 8: Destination Params: Storage Class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Storage class of the destination objects can be configured:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck10 \
+                                --group-id=buck10-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck10 \
+                                --group-id=buck10-default \
+                                --pipe-id=pipe-storage-class \
+                                --source-zones='*' --dest-zones=us-west-2 \
+                                --storage-class=CHEAP_AND_SLOW
+
+
+Example 9: Destination Params: Destination Owner Translation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Set the destination objects owner as the destination bucket owner.
+This requires specifying the uid of the destination bucket:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck11 \
+                                --group-id=buck11-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck11 \
+                                --group-id=buck11-default --pipe-id=pipe-dest-owner \
+                                --source-zones='*' --dest-zones='*' \
+                                --dest-bucket=buck12 --dest-owner=joe
+
+Example 10: Destination Params: User Mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+User mode makes sure that the user has permissions to both read the objects, and write to the destination bucket. This requires that the uid of the user (which in its context the operation executes) is specified.
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe modify --bucket=buck11 \
+                                --group-id=buck11-default --pipe-id=pipe-dest-owner \
+                                --mode=user --uid=jenny
+
+
+
diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
new file mode 100644
index 000000000..c7627371d
--- /dev/null
+++ b/doc/radosgw/multisite.rst
@@ -0,0 +1,1690 @@
+.. _multisite:
+
+==========
+Multi-Site
+==========
+
+Single-zone Configurations and Multi-site Configurations
+========================================================
+
+Single-zone Configurations
+--------------------------
+
+A single-zone configuration typically consists of two things:
+
+#. One "zonegroup", which contains one zone. 
+#. One or more `ceph-radosgw` instances that have `ceph-radosgw` client
+   requests load-balanced between them. 
+
+In a typical single-zone configuration, there are multiple `ceph-radosgw`
+instances that make use of a single Ceph storage cluster.  
+
+Varieties of Multi-site Configuration
+-------------------------------------
+
+.. versionadded:: Jewel
+
+Beginning with the Kraken release, Ceph supports several multi-site
+configurations for the Ceph Object Gateway:
+
+- **Multi-zone:** A more advanced topology, the "multi-zone" configuration, is
+  possible. A multi-zone configuration consists of one zonegroup and
+  multiple zones, with each zone consisting of one or more `ceph-radosgw`
+  instances. **Each zone is backed by its own Ceph Storage Cluster.**
+  
+  The presence of multiple zones in a given zonegroup provides disaster
+  recovery for that zonegroup in the event that one of the zones experiences a
+  significant failure. Beginning with the Kraken release, each zone is active
+  and can receive write operations. A multi-zone configuration that contains
+  multiple active zones enhances disaster recovery and can also be used as a
+  foundation for content delivery networks. 
+
+- **Multi-zonegroups:** Ceph Object Gateway supports multiple zonegroups (which
+  were formerly called "regions"). Each zonegroup contains one or more zones.
+  If two zones are in the same zonegroup, and if that zonegroup is in the same
+  realm as a second zonegroup, then the objects stored in the two zones share
+  a global object namespace. This global object namespace ensures unique
+  object IDs across zonegroups and zones.
+
+  Each bucket is owned by the zonegroup where it was created (except where
+  overridden by the :ref:`LocationConstraint<s3_bucket_placement>` on
+  bucket creation), and its object data will only replicate to other zones in
+  that zonegroup. Any request for data in that bucket that are sent to other
+  zonegroups will redirect to the zonegroup where the bucket resides.
+
+  It can be useful to create multiple zonegroups when you want to share a
+  namespace of users and buckets across many zones, but isolate the object data
+  to a subset of those zones. It might be that you have several connected sites
+  that share storage, but only require a single backup for purposes of disaster
+  recovery. In such a case, it could make sense to create several zonegroups
+  with only two zones each to avoid replicating all objects to all zones.
+
+  In other cases, it might make more sense to isolate things in separate
+  realms, with each realm having a single zonegroup. Zonegroups provide
+  flexibility by making it possible to control the isolation of data and
+  metadata separately.
+
+- **Multiple Realms:** Beginning with the Kraken release, the Ceph Object
+  Gateway supports "realms", which are containers for zonegroups. Realms make
+  it possible to set policies that apply to multiple zonegroups. Realms have a
+  globally unique namespace and can contain either a single zonegroup or
+  multiple zonegroups. If you choose to make use of multiple realms, you can
+  define multiple namespaces and multiple configurations (this means that each
+  realm can have a configuration that is distinct from the configuration of
+  other realms).
+
+
+Diagram - Replication of Object Data Between Zones
+--------------------------------------------------
+
+The replication of object data between zones within a zonegroup looks
+something like this:
+
+.. image:: ../images/zone-sync.svg
+   :align: center
+
+At the top of this diagram, we see two applications (also known as "clients").
+The application on the right is both writing and reading data from the Ceph
+Cluster, by means of the RADOS Gateway (RGW). The application on the left is
+only *reading* data from the Ceph Cluster, by means of an instance of RADOS
+Gateway (RGW). In both cases (read-and-write and read-only), the transmssion of
+data is handled RESTfully.
+
+In the middle of this diagram, we see two zones, each of which contains an
+instance of RADOS Gateway (RGW). These instances of RGW are handling the
+movement of data from the applications to the zonegroup. The arrow from the
+master zone (US-EAST) to the secondary zone (US-WEST) represents an act of data
+synchronization.
+
+At the bottom of this diagram, we see the data distributed into the Ceph
+Storage Cluster.
+
+For additional details on setting up a cluster, see `Ceph Object Gateway for
+Production <https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/3/html/ceph_object_gateway_for_production/index/>`__.
+
+Functional Changes from Infernalis
+==================================
+
+Beginning with Kraken, each Ceph Object Gateway can be configured to work in an
+active-active zone mode. This makes it possible to write to non-master zones.
+
+The multi-site configuration is stored within a container called a "realm". The
+realm stores zonegroups, zones, and a time "period" with multiple epochs (which
+(the epochs) are used for tracking changes to the configuration). 
+
+Beginning with Kraken, the ``ceph-radosgw`` daemons handle the synchronization
+of data across zones, which eliminates the need for a separate synchronization
+agent. This new approach to synchronization allows the Ceph Object Gateway to
+operate with an "active-active" configuration instead of with an
+"active-passive" configuration.
+
+Requirements and Assumptions
+============================
+
+A multi-site configuration requires at least two Ceph storage clusters. The
+multi-site configuration must have at least two Ceph object gateway instances
+(one for each Ceph storage cluster).
+
+This guide assumes that at least two Ceph storage clusters are in
+geographically separate locations; however, the configuration can work on the
+same site. This guide also assumes two Ceph object gateway servers named
+``rgw1`` and ``rgw2``.
+
+.. important:: Running a single geographically-distributed Ceph storage cluster
+   is NOT recommended unless you have low latency WAN connections.
+
+A multi-site configuration requires a master zonegroup and a master zone. Each
+zonegroup requires a master zone. Zonegroups may have one or more secondary
+or non-master zones.
+
+In this guide, the ``rgw1`` host will serve as the master zone of the master
+zonegroup; and, the ``rgw2`` host will serve as the secondary zone of the
+master zonegroup.
+
+See `Pools`_ for instructions on creating and tuning pools for Ceph Object
+Storage.
+
+See `Sync Policy Config`_ for instructions on defining fine-grained bucket sync
+policy rules.
+
+.. _master-zone-label:
+
+Configuring a Master Zone
+=========================
+
+All gateways in a multi-site configuration retrieve their configurations from a
+``ceph-radosgw`` daemon that is on a host within both the master zonegroup and
+the master zone. To configure your gateways in a multi-site configuration,
+choose a ``ceph-radosgw`` instance to configure the master zonegroup and
+master zone.
+
+Create a Realm
+--------------
+
+A realm contains the multi-site configuration of zonegroups and zones. The
+realm enforces a globally unique namespace within itself.
+
+#. Create a new realm for the multi-site configuration by opening a command
+   line interface on a host that will serve in the master zonegroup and zone.
+   Then run the following command:
+
+   .. prompt:: bash #
+
+      radosgw-admin realm create --rgw-realm={realm-name} [--default]
+
+   For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin realm create --rgw-realm=movies --default
+
+   .. note:: If you intend the cluster to have a single realm, specify the ``--default`` flag.  
+
+      If ``--default`` is specified, ``radosgw-admin`` uses this realm by default. 
+      
+      If ``--default`` is not specified, you must specify either the ``--rgw-realm`` flag or the ``--realm-id`` flag to identify the realm when adding zonegroups and zones.
+
+#. After the realm has been created, ``radosgw-admin`` echoes back the realm
+   configuration. For example:
+
+   ::
+
+       {
+           "id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
+           "name": "movies",
+           "current_period": "1950b710-3e63-4c41-a19e-46a715000980",
+           "epoch": 1
+       }
+
+   .. note:: Ceph generates a unique ID for the realm, which can be used to rename the realm if the need arises.
+
+Create a Master Zonegroup
+--------------------------
+
+A realm must have at least one zonegroup which serves as the master zonegroup
+for the realm.
+
+#. To create a new master zonegroup for the multi-site configuration, open a
+   command-line interface on a host in the master zonegroup and zone. Then
+   run the following command:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup create --rgw-zonegroup={name} --endpoints={url} [--rgw-realm={realm-name}|--realm-id={realm-id}] --master --default
+
+   For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup create --rgw-zonegroup=us --endpoints=http://rgw1:80 --rgw-realm=movies --master --default
+
+   .. note:: If the realm will have only a single zonegroup, specify the ``--default`` flag. 
+
+      If ``--default`` is specified, ``radosgw-admin`` uses this zonegroup by default when adding new zones. 
+      
+      If ``--default`` is not specified, you must use either the ``--rgw-zonegroup`` flag or the ``--zonegroup-id`` flag to identify the zonegroup when adding or modifying zones.
+
+#. After creating the master zonegroup, ``radosgw-admin`` echoes back the
+   zonegroup configuration. For example:
+
+   ::
+   
+       {
+           "id": "f1a233f5-c354-4107-b36c-df66126475a6",
+           "name": "us",
+           "api_name": "us",
+           "is_master": "true",
+           "endpoints": [
+               "http:\/\/rgw1:80"
+           ],
+           "hostnames": [],
+           "hostnames_s3website": [],
+           "master_zone": "",
+           "zones": [],
+           "placement_targets": [],
+           "default_placement": "",
+           "realm_id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62"
+       }
+
+Create a Master Zone
+--------------------
+
+.. important:: Zones must be created on a Ceph Object Gateway node that will be
+   within the zone.
+
+Create a new master zone for the multi-site configuration by opening a command
+line interface on a host that serves in the master zonegroup and zone. Then
+run the following command:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                               --rgw-zone={zone-name} \
+                               --master --default \
+                               --endpoints={http://fqdn}[,{http://fqdn}]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east \
+                               --master --default \
+                               --endpoints={http://fqdn}[,{http://fqdn}]
+
+
+.. note:: The ``--access-key`` and ``--secret`` aren’t specified. These
+          settings will be added to the zone once the user is created in the
+          next section.
+
+.. important:: The following steps assume a multi-site configuration that uses
+   newly installed systems that aren’t storing data yet. DO NOT DELETE the
+   ``default`` zone and its pools if you are already using the zone to store
+   data, or the data will be deleted and unrecoverable.
+
+Delete Default Zonegroup and Zone
+----------------------------------
+
+#. Delete the ``default`` zone if it exists. Remove it from the default
+   zonegroup first.
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup delete --rgw-zonegroup=default --rgw-zone=default
+      radosgw-admin period update --commit
+      radosgw-admin zone delete --rgw-zone=default
+      radosgw-admin period update --commit
+      radosgw-admin zonegroup delete --rgw-zonegroup=default
+      radosgw-admin period update --commit
+
+#. Delete the ``default`` pools in your Ceph storage cluster if they exist.
+
+   .. important:: The following step assumes a multi-site configuration that uses newly installed systems that aren’t currently storing data. DO NOT DELETE the ``default`` zonegroup if you are already using it to store data.
+
+   .. prompt:: bash #
+   
+      ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it
+      ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
+      ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
+      ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it
+      ceph osd pool rm default.rgw.users.uid default.rgw.users.uid --yes-i-really-really-mean-it
+
+Create a System User
+--------------------
+
+#. The ``ceph-radosgw`` daemons must authenticate before pulling realm and
+   period information. In the master zone, create a "system user" to facilitate
+   authentication between daemons.
+
+   .. prompt:: bash #
+
+      radosgw-admin user create --uid="{user-name}" --display-name="{Display Name}" --system
+
+   For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin user create --uid="synchronization-user" --display-name="Synchronization User" --system
+
+#. Make a note of the ``access_key`` and ``secret_key``. The secondary zones
+   require them to authenticate against the master zone.
+
+#. Add the system user to the master zone:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --access-key={access-key} --secret={secret}
+      radosgw-admin period update --commit
+
+Update the Period
+-----------------
+
+After updating the master zone configuration, update the period.
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+.. note:: Updating the period changes the epoch, and ensures that other zones
+          will receive the updated configuration.
+
+Update the Ceph Configuration File
+----------------------------------
+
+Update the Ceph configuration file on master zone hosts by adding the
+``rgw_zone`` configuration option and the name of the master zone to the
+instance entry.
+
+::
+
+    [client.rgw.{instance-name}]
+    ...
+    rgw_zone={zone-name}
+
+For example:
+
+::
+
+    [client.rgw.rgw1]
+    host = rgw1
+    rgw frontends = "civetweb port=80"
+    rgw_zone=us-east
+
+Start the Gateway
+-----------------
+
+On the object gateway host, start and enable the Ceph Object Gateway
+service:
+
+.. prompt:: bash #
+
+   systemctl start ceph-radosgw@rgw.`hostname -s`
+   systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+.. _secondary-zone-label:
+
+Configuring Secondary Zones
+===========================
+
+Zones that are within a zonegroup replicate all data in order to ensure that
+every zone has the same data. When creating a secondary zone, run the following
+operations on a host identified to serve the secondary zone.
+
+.. note:: To add a second secondary zone (that is, a second non-master zone
+   within a zonegroup that already contains a secondary zone), follow :ref:`the
+   same procedures that are used for adding a secondary
+   zone<radosgw-multisite-secondary-zone-creating>`. Be sure to specify a
+   different zone name than the name of the first secondary zone.
+
+.. important:: Metadata operations (for example, user creation) must be
+   run on a host within the master zone. Bucket operations can be received
+   by the master zone or the secondary zone, but the secondary zone will
+   redirect bucket operations to the master zone. If the master zone is down,
+   bucket operations will fail.
+
+Pulling the Realm Configuration
+-------------------------------
+
+The URL path, access key, and secret of the master zone in the master zone
+group are used to pull the realm configuration to the host. When pulling the
+configuration of a non-default realm, specify the realm using the
+``--rgw-realm`` or ``--realm-id`` configuration options.
+
+.. prompt:: bash #
+
+   radosgw-admin realm pull --url={url-to-master-zone-gateway}
+   --access-key={access-key} --secret={secret}
+
+.. note:: Pulling the realm configuration also retrieves the remote's current
+   period configuration, and makes it the current period on this host as well.
+
+If this realm is the only realm, run the following command to make it the
+default realm:
+
+.. prompt:: bash #
+
+   radosgw-admin realm default --rgw-realm={realm-name}
+
+.. _radosgw-multisite-secondary-zone-creating:
+
+Creating a Secondary Zone
+-------------------------
+
+.. important:: When a zone is created, it must be on a Ceph Object Gateway node
+   within the zone.
+
+In order to create a secondary zone for the multi-site configuration, open a
+command line interface on a host identified to serve the secondary zone.
+Specify the zonegroup ID, the new zone name, and an endpoint for the zone.
+**DO NOT** use the ``--master`` or ``--default`` flags. Beginning in Kraken,
+all zones run in an active-active configuration by default, which means that a
+gateway client may write data to any zone and the zone will replicate the data
+to all other zones within the zonegroup.  If you want to prevent the secondary
+zone from accepting write operations, include the ``--read-only`` flag in the
+command in order to create an active-passive configuration between the master
+zone and the secondary zone. In any case, don't forget to provide the
+``access_key`` and ``secret_key`` of the generated system user that is stored
+in the master zone of the master zonegroup. Run the following command:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --access-key={system-key} --secret={secret} \
+                                --endpoints=http://{fqdn}:80 \
+                                [--read-only]
+
+For example:
+    
+    
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-west \
+                                --access-key={system-key} --secret={secret} \
+                                --endpoints=http://rgw2:80
+
+.. important:: The following steps assume a multi-site configuration that uses
+   newly installed systems that have not yet begun storing data. **DO NOT
+   DELETE the ``default`` zone or its pools** if you are already using it to
+   store data, or the data will be irretrievably lost.
+
+Delete the default zone if needed:
+
+.. prompt:: bash #
+
+   radosgw-admin zone delete --rgw-zone=default
+
+Finally, delete the default pools in your Ceph storage cluster if needed:
+
+.. prompt:: bash #
+
+   ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.users.uid default.rgw.users.uid --yes-i-really-really-mean-it
+   
+Updating the Ceph Configuration File
+------------------------------------
+
+To update the Ceph configuration file on the secondary zone hosts, add the
+``rgw_zone`` configuration option and the name of the secondary zone to the
+instance entry.
+
+::
+
+    [client.rgw.{instance-name}]
+    ...
+    rgw_zone={zone-name}
+
+For example:
+
+::
+
+    [client.rgw.rgw2]
+    host = rgw2
+    rgw frontends = "civetweb port=80"
+    rgw_zone=us-west
+
+Updating the Period
+-------------------
+
+After updating the master zone configuration, update the period:
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+.. note:: Updating the period changes the epoch, and ensures that other zones
+   will receive the updated configuration.
+          
+Starting the Gateway
+--------------------
+
+To start the gateway, start and enable the Ceph Object Gateway service by
+running the following commands on the object gateway host:
+
+.. prompt:: bash #
+
+   systemctl start ceph-radosgw@rgw.`hostname -s`
+   systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+Checking Synchronization Status
+-------------------------------
+
+After the secondary zone is up and running, you can check the synchronization
+status. The process of synchronization will copy users and buckets that were
+created in the master zone from the master zone to the secondary zone.
+
+.. prompt:: bash #
+
+   radosgw-admin sync status
+
+The output reports the status of synchronization operations. For example:
+
+::
+
+    realm f3239bc5-e1a8-4206-a81d-e1576480804d (earth)
+        zonegroup c50dbb7e-d9ce-47cc-a8bb-97d9b399d388 (us)
+             zone 4c453b70-4a16-4ce8-8185-1893b05d346e (us-west)
+    metadata sync syncing
+                  full sync: 0/64 shards
+                  metadata is caught up with master
+                  incremental sync: 64/64 shards
+        data sync source: 1ee9da3e-114d-4ae3-a8a4-056e8a17f532 (us-east)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+
+.. note:: Secondary zones accept bucket operations; however, secondary zones
+   redirect bucket operations to the master zone and then synchronize with the
+   master zone to receive the result of the bucket operations. If the master
+   zone is down, bucket operations executed on the secondary zone will fail,
+   but object operations should succeed.
+          
+          
+Verifying an Object
+-------------------
+
+By default, after the successful synchronization of an object there is no
+subsequent verification of the object. However, you can enable verification by
+setting :confval:`rgw_sync_obj_etag_verify` to ``true``. After this value is
+set to true, an MD5 checksum is used to verify the integrity of the data that
+was transferred from the source to the destination. This ensures the integrity
+of any object that has been fetched from a remote server over HTTP (including
+multisite sync). This option may decrease the performance of your RGW because
+it requires more computation.
+
+
+Maintenance
+===========
+
+Checking the Sync Status
+------------------------
+
+Information about the replication status of a zone can be queried with:
+
+.. prompt:: bash $
+
+   radosgw-admin sync status
+
+::
+
+            realm b3bc1c37-9c44-4b89-a03b-04c269bea5da (earth)
+        zonegroup f54f9b22-b4b6-4a0e-9211-fa6ac1693f49 (us)
+             zone adce11c9-b8ed-4a90-8bc5-3fc029ff0816 (us-2)
+            metadata sync syncing
+                  full sync: 0/64 shards
+                  incremental sync: 64/64 shards
+                  metadata is behind on 1 shards
+                  oldest incremental change not applied: 2017-03-22 10:20:00.0.881361s
+        data sync source: 341c2d81-4574-4d08-ab0f-5a2a7b168028 (us-1)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+                  source: 3b5d1a3f-3f27-4e4a-8f34-6072d4bb1275 (us-3)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+
+The output might be different, depending on the sync status. During sync, the
+shards are of two types:
+
+- **Behind shards** are shards that require a data sync (either a full data
+  sync or an incremental data sync) in order to be brought up to date.
+
+- **Recovery shards** are shards that encountered an error during sync and have
+  been marked for retry. The error occurs mostly on minor issues, such as
+  acquiring a lock on a bucket. Errors of this kind typically resolve on their
+  own.
+
+Check the logs
+--------------
+
+For multi-site deployments only, you can examine the metadata log (``mdlog``),
+the bucket index log (``bilog``), and the data log (``datalog``).  You can list
+them and also trim them. Trimming is not needed in most cases because
+:confval:`rgw_sync_log_trim_interval` is set to 20 minutes by default. It
+should not be necessary to trim the logs unless
+:confval:`rgw_sync_log_trim_interval` has been manually set to 0.
+
+Changing the Metadata Master Zone
+---------------------------------
+
+.. important:: Care must be taken when changing the metadata master zone by
+   promoting a zone to master. A zone that isn't finished syncing metadata from
+   the current master zone will be unable to serve any remaining entries if it
+   is promoted to master, and those metadata changes will be lost. For this
+   reason, we recommend waiting for a zone's ``radosgw-admin sync status`` to
+   complete the process of synchronizing the metadata before promoting the zone
+   to master.
+
+Similarly, if the current master zone is processing changes to metadata at the
+same time that another zone is being promoted to master, these changes are
+likely to be lost. To avoid losing these changes, we recommend shutting down
+any ``radosgw`` instances on the previous master zone. After the new master
+zone has been promoted, the previous master zone's new period can be fetched
+with ``radosgw-admin period pull`` and the gateway(s) can be restarted.
+
+To promote a zone to metadata master, run the following commands on that zone
+(in this example, the zone is zone ``us-2`` in zonegroup ``us``):
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone=us-2 --master
+   radosgw-admin zonegroup modify --rgw-zonegroup=us --master
+   radosgw-admin period update --commit
+
+This generates a new period, and the radosgw instance(s) in zone ``us-2`` sends
+this period to other zones.
+
+Failover and Disaster Recovery
+==============================
+
+Setting Up Failover to the Secondary Zone
+-----------------------------------------
+
+If the master zone fails, you can fail over to the secondary zone for
+disaster recovery by following these steps:
+
+#. Make the secondary zone the master and default zone. For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default
+
+   By default, Ceph Object Gateway runs in an active-active
+   configuration. However, if the cluster is configured to run in an
+   active-passive configuration, the secondary zone is a read-only zone.
+   To allow the secondary zone to receive write
+   operations, remove its ``--read-only`` status. For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default \
+                                   --read-only=false
+
+#. Update the period to make the changes take effect.
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+#. Finally, restart the Ceph Object Gateway.
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+Reverting from Failover
+-----------------------
+
+If the former master zone recovers, you can revert the failover operation by following these steps:
+
+#. From within the recovered zone, pull the latest realm configuration
+   from the current master zone:
+
+   .. prompt:: bash #
+
+      radosgw-admin realm pull --url={url-to-master-zone-gateway} \
+                                  --access-key={access-key} --secret={secret}
+
+#. Make the recovered zone the master and default zone:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default
+
+#. Update the period so that the changes take effect:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+#. Restart the Ceph Object Gateway in the recovered zone:
+
+   .. prompt:: bash #
+
+       systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+#. If the secondary zone needs to be a read-only configuration, update
+   the secondary zone:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --read-only
+
+#. Update the period so that the changes take effect:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+#. Restart the Ceph Object Gateway in the secondary zone:
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+.. _rgw-multisite-migrate-from-single-site:
+
+Migrating a Single-Site Deployment to Multi-Site
+=================================================
+
+To migrate from a single-site deployment with a ``default`` zonegroup and zone
+to a multi-site system, follow these steps:
+
+1. Create a realm. Replace ``<name>`` with the realm name:
+
+   .. prompt:: bash #
+
+      radosgw-admin realm create --rgw-realm=<name> --default
+
+2. Rename the default zonegroup and zone. Replace ``<name>`` with the zone name
+   or zonegroup name:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup rename --rgw-zonegroup default --zonegroup-new-name=<name>
+      radosgw-admin zone rename --rgw-zone default --zone-new-name us-east-1 --rgw-zonegroup=<name>
+
+3. Rename the default zonegroup's ``api_name``. Replace ``<name>`` with the zonegroup name:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup modify --api-name=<name> --rgw-zonegroup=<name>
+
+4. Configure the master zonegroup. Replace ``<name>`` with the realm name or
+   zonegroup name. Replace ``<fqdn>`` with the fully qualified domain name(s)
+   in the zonegroup:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup modify --rgw-realm=<name> --rgw-zonegroup=<name> --endpoints http://<fqdn>:80 --master --default
+
+5. Configure the master zone. Replace ``<name>`` with the realm name, zone
+   name, or zonegroup name. Replace ``<fqdn>`` with the fully qualified domain
+   name(s) in the zonegroup:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-realm=<name> --rgw-zonegroup=<name> \
+                                --rgw-zone=<name> --endpoints http://<fqdn>:80 \
+                                --access-key=<access-key> --secret=<secret-key> \
+                                --master --default
+
+6. Create a system user. Replace ``<user-id>`` with the username.  Replace
+   ``<display-name>`` with a display name. The display name is allowed to
+   contain spaces:
+
+   .. prompt:: bash #
+
+      radosgw-admin user create --uid=<user-id> \
+      --display-name="<display-name>" \ 
+      --access-key=<access-key> \ 
+      --secret=<secret-key> --system
+
+7. Commit the updated configuration:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+8. Restart the Ceph Object Gateway:
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+After completing this procedure, proceed to `Configure a Secondary
+Zone <#configure-secondary-zones>`_ and create a secondary zone
+in the master zonegroup.
+
+Multi-Site Configuration Reference
+==================================
+
+The following sections provide additional details and command-line
+usage for realms, periods, zonegroups and zones.
+
+For more details on every available configuration option, see
+``src/common/options/rgw.yaml.in``.  
+
+Alternatively, go to the :ref:`mgr-dashboard` configuration page (found under
+`Cluster`), where you can view and set all of the options. While on the page,
+set the level to ``advanced`` and search for RGW to see all basic and advanced
+configuration options. 
+
+.. _rgw-realms:
+
+Realms
+------
+
+A realm is a globally unique namespace that consists of one or more zonegroups.
+Zonegroups contain one or more zones. Zones contain buckets. Buckets contain
+objects. 
+
+Realms make it possible for the Ceph Object Gateway to support multiple
+namespaces and their configurations on the same hardware.
+
+Each realm is associated with a "period". A period represents the state
+of the zonegroup and zone configuration in time. Each time you make a
+change to a zonegroup or zone, you should update and commit the period.
+
+To ensure backward compatibility with Infernalis and earlier releases, the Ceph
+Object Gateway does not by default create a realm. However, as a best practice,
+we recommend that you create realms when creating new clusters.
+
+Create a Realm
+~~~~~~~~~~~~~~
+
+To create a realm, run ``realm create`` and specify the realm name.
+If the realm is the default, specify ``--default``.
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm={realm-name} [--default]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm=movies --default
+
+By specifying ``--default``, the realm will be called implicitly with
+each ``radosgw-admin`` call unless ``--rgw-realm`` and the realm name
+are explicitly provided.
+
+Make a Realm the Default
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+One realm in the list of realms should be the default realm. There may be only
+one default realm. If there is only one realm and it wasn’t specified as the
+default realm when it was created, make it the default realm. Alternatively, to
+change which realm is the default, run the following command:
+
+.. prompt:: bash #
+
+   radosgw-admin realm default --rgw-realm=movies
+
+.. note:: When the realm is default, the command line assumes
+   ``--rgw-realm=<realm-name>`` as an argument.
+
+Delete a Realm
+~~~~~~~~~~~~~~
+
+To delete a realm, run ``realm rm`` and specify the realm name:
+
+.. prompt:: bash #
+
+   radosgw-admin realm rm --rgw-realm={realm-name}
+
+For example:
+
+.. prompt:: bash #
+   
+   radosgw-admin realm rm --rgw-realm=movies
+
+Get a Realm
+~~~~~~~~~~~
+
+To get a realm, run ``realm get`` and specify the realm name:
+
+.. prompt:: bash #
+
+   radosgw-admin realm get --rgw-realm=<name>
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm get --rgw-realm=movies [> filename.json]
+
+::
+
+    {
+        "id": "0a68d52e-a19c-4e8e-b012-a8f831cb3ebc",
+        "name": "movies",
+        "current_period": "b0c5bbef-4337-4edd-8184-5aeab2ec413b",
+        "epoch": 1
+    }
+
+Set a Realm
+~~~~~~~~~~~
+
+To set a realm, run ``realm set``, specify the realm name, and use the
+``--infile=`` option (make sure that  the ``--infile`` option has an input file
+name as an argument):
+
+.. prompt:: bash #
+
+   radosgw-admin realm set --rgw-realm=<name> --infile=<infilename>
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm set --rgw-realm=movies --infile=filename.json
+
+List Realms
+~~~~~~~~~~~
+
+To list realms, run ``realm list``:
+
+.. prompt:: bash #
+
+   radosgw-admin realm list
+
+List Realm Periods
+~~~~~~~~~~~~~~~~~~
+
+To list realm periods, run ``realm list-periods``:
+
+.. prompt:: bash #
+
+   radosgw-admin realm list-periods
+
+Pull a Realm
+~~~~~~~~~~~~
+
+To pull a realm from the node that contains both the master zonegroup and
+master zone to a node that contains a secondary zonegroup or zone, run ``realm
+pull`` on the node that will receive the realm configuration:
+
+.. prompt:: bash #
+
+   radosgw-admin realm pull --url={url-to-master-zone-gateway} --access-key={access-key} --secret={secret}
+
+Rename a Realm
+~~~~~~~~~~~~~~
+
+A realm is not part of the period. Consequently, any renaming of the realm is
+applied only locally, and will therefore not get pulled when you run ``realm
+pull``. If you are renaming a realm that contains multiple zones, run the
+``rename`` command on each zone. 
+
+To rename a realm, run the following:
+
+.. prompt:: bash #
+
+   radosgw-admin realm rename --rgw-realm=<current-name> --realm-new-name=<new-realm-name>
+
+.. note:: DO NOT use ``realm set`` to change the ``name`` parameter. Doing so
+   changes the internal name only. If you use ``realm set`` to change the
+   ``name`` parameter, then ``--rgw-realm`` still expects the realm's old name.
+
+Zonegroups
+-----------
+
+Zonegroups make it possible for the Ceph Object Gateway to support multi-site
+deployments and a global namespace. Zonegroups were formerly called "regions"
+(in releases prior to and including Infernalis). 
+
+A zonegroup defines the geographic location of one or more Ceph Object Gateway
+instances within one or more zones.
+
+The configuration of zonegroups differs from typical configuration procedures,
+because not all of the zonegroup configuration settings are stored to a
+configuration file. 
+
+You can list zonegroups, get a zonegroup configuration, and set a zonegroup
+configuration.
+
+Creating a Zonegroup
+~~~~~~~~~~~~~~~~~~~~
+
+Creating a zonegroup consists of specifying the zonegroup name. Newly created
+zones reside in the default realm unless a different realm is specified by
+using the option ``--rgw-realm=<realm-name>``. 
+
+If the zonegroup is the default zonegroup, specify the ``--default`` flag. If
+the zonegroup is the master zonegroup, specify the ``--master`` flag. For
+example:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup create --rgw-zonegroup=<name> [--rgw-realm=<name>][--master] [--default]
+
+
+.. note:: Use ``zonegroup modify --rgw-zonegroup=<zonegroup-name>`` to modify
+          an existing zonegroup’s settings.
+
+Making a Zonegroup the Default
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One zonegroup in the list of zonegroups must be the default zonegroup.  There
+can be only one default zonegroup. In the case that there is only one zonegroup
+which was not designated the default zonegroup when it was created, use the
+following command to make it the default zonegroup. Commands of this form can
+be used to change which zonegroup is the default. 
+
+#. Designate a zonegroup as the default zonegroup:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup default --rgw-zonegroup=comedy
+
+   .. note:: When the zonegroup is default, the command line assumes that the name of the zonegroup will be the argument of the ``--rgw-zonegroup=<zonegroup-name>`` option. (In this example, ``<zonegroup-name>`` has been retained for the sake of consistency and legibility.)
+
+#. Update the period:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+Adding a Zone to a Zonegroup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This procedure explains how to add a zone to a zonegroup.
+
+#. Run the following command to add a zone to a zonegroup: 
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup add --rgw-zonegroup=<name> --rgw-zone=<name>
+
+#. Update the period:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+Removing a Zone from a Zonegroup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#. Run this command to remove a zone from a zonegroup:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup remove --rgw-zonegroup=<name> --rgw-zone=<name>
+
+#. Update the period:
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+Renaming a Zonegroup
+~~~~~~~~~~~~~~~~~~~~
+
+#. Run this command to rename the zonegroup:
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup rename --rgw-zonegroup=<name> --zonegroup-new-name=<name>
+
+#. Update the period:
+
+   .. prompt:: bash #
+   
+      radosgw-admin period update --commit
+
+Deleting a Zonegroup
+~~~~~~~~~~~~~~~~~~~~
+
+#. To delete a zonegroup, run the following command:
+
+   .. prompt:: bash #
+   
+      radosgw-admin zonegroup delete --rgw-zonegroup=<name>
+
+#. Update the period:
+
+   .. prompt:: bash #
+   
+      radosgw-admin period update --commit
+
+Listing Zonegroups
+~~~~~~~~~~~~~~~~~~
+
+A Ceph cluster contains a list of zonegroup. To list the zonegroups, run
+this command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup list
+
+The ``radosgw-admin`` returns a JSON formatted list of zonegroups.
+
+::
+
+    {
+        "default_info": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "zonegroups": [
+            "us"
+        ]
+    }
+
+Getting a Zonegroup Map
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To list the details of each zonegroup, run this command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup-map get
+
+.. note:: If you receive a ``failed to read zonegroup map`` error, run
+   ``radosgw-admin zonegroup-map update`` as ``root`` first.
+
+Getting a Zonegroup
+~~~~~~~~~~~~~~~~~~~~
+
+To view the configuration of a zonegroup, run this command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup get [--rgw-zonegroup=<zonegroup>]
+
+The zonegroup configuration looks like this:
+
+::
+
+    {
+        "id": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "name": "us",
+        "api_name": "us",
+        "is_master": "true",
+        "endpoints": [
+            "http:\/\/rgw1:80"
+        ],
+        "hostnames": [],
+        "hostnames_s3website": [],
+        "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e",
+        "zones": [
+            {
+                "id": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                "name": "us-east",
+                "endpoints": [
+                    "http:\/\/rgw1"
+                ],
+                "log_meta": "true",
+                "log_data": "true",
+                "bucket_index_max_shards": 0,
+                "read_only": "false"
+            },
+            {
+                "id": "d1024e59-7d28-49d1-8222-af101965a939",
+                "name": "us-west",
+                "endpoints": [
+                    "http:\/\/rgw2:80"
+                ],
+                "log_meta": "false",
+                "log_data": "true",
+                "bucket_index_max_shards": 0,
+                "read_only": "false"
+            }
+        ],
+        "placement_targets": [
+            {
+                "name": "default-placement",
+                "tags": []
+            }
+        ],
+        "default_placement": "default-placement",
+        "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe"
+    }
+
+Setting a Zonegroup
+~~~~~~~~~~~~~~~~~~~~
+
+The process of defining a zonegroup consists of creating a JSON object and
+specifying the required settings. Here is a list of the required settings:
+
+1. ``name``: The name of the zonegroup. Required.
+
+2. ``api_name``: The API name for the zonegroup. Optional.
+
+3. ``is_master``: Determines whether the zonegroup is the master zonegroup.
+   Required. **note:** You can only have one master zonegroup.
+
+4. ``endpoints``: A list of all the endpoints in the zonegroup. For example,
+   you may use multiple domain names to refer to the same zonegroup. Remember
+   to escape the forward slashes (``\/``). You may also specify a port
+   (``fqdn:port``) for each endpoint. Optional.
+
+5. ``hostnames``: A list of all the hostnames in the zonegroup. For example,
+   you may use multiple domain names to refer to the same zonegroup. Optional.
+   The ``rgw dns name`` setting will be included in this list automatically.
+   Restart the gateway daemon(s) after changing this setting.
+
+6. ``master_zone``: The master zone for the zonegroup. Optional. Uses
+   the default zone if not specified. **note:** You can only have one
+   master zone per zonegroup.
+
+7. ``zones``: A list of all zones within the zonegroup. Each zone has a name
+   (required), a list of endpoints (optional), and a setting that determines
+   whether the gateway will log metadata and data operations (false by
+   default).
+
+8. ``placement_targets``: A list of placement targets (optional). Each
+   placement target contains a name (required) for the placement target
+   and a list of tags (optional) so that only users with the tag can use
+   the placement target (that is, the user’s ``placement_tags`` field in
+   the user info).
+
+9. ``default_placement``: The default placement target for the object index and
+   object data. Set to ``default-placement`` by default. It is  also possible
+   to set a per-user default placement in the user info for each user.
+
+Setting a Zonegroup - Procedure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#. To set a zonegroup, create a JSON object that contains the required fields,
+   save the object to a file (for example, ``zonegroup.json``), and run the
+   following command:
+
+   .. prompt:: bash #
+   
+      radosgw-admin zonegroup set --infile zonegroup.json
+
+   Where ``zonegroup.json`` is the JSON file you created.
+
+   .. important:: The ``default`` zonegroup ``is_master`` setting is ``true`` by default. If you create an additional zonegroup and want to make it the master zonegroup, you must either set the ``default`` zonegroup ``is_master`` setting to ``false`` or delete the ``default`` zonegroup.
+
+#. Update the period:
+
+   .. prompt:: bash #
+   
+      radosgw-admin period update --commit
+
+Setting a Zonegroup Map
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The process of setting a zonegroup map comprises (1) creating a JSON object
+that consists of one or more zonegroups, and (2) setting the
+``master_zonegroup`` for the cluster. Each zonegroup in the zonegroup map
+consists of a key/value pair where the ``key`` setting is equivalent to the
+``name`` setting for an individual zonegroup configuration and the ``val`` is
+a JSON object consisting of an individual zonegroup configuration.
+
+You may only have one zonegroup with ``is_master`` equal to ``true``, and it
+must be specified as the ``master_zonegroup`` at the end of the zonegroup map.
+The following JSON object is an example of a default zonegroup map:
+
+::
+
+    {
+        "zonegroups": [
+            {
+                "key": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+                "val": {
+                    "id": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+                    "name": "us",
+                    "api_name": "us",
+                    "is_master": "true",
+                    "endpoints": [
+                        "http:\/\/rgw1:80"
+                    ],
+                    "hostnames": [],
+                    "hostnames_s3website": [],
+                    "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                    "zones": [
+                        {
+                            "id": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                            "name": "us-east",
+                            "endpoints": [
+                                "http:\/\/rgw1"
+                            ],
+                            "log_meta": "true",
+                            "log_data": "true",
+                            "bucket_index_max_shards": 0,
+                            "read_only": "false"
+                        },
+                        {
+                            "id": "d1024e59-7d28-49d1-8222-af101965a939",
+                            "name": "us-west",
+                            "endpoints": [
+                                "http:\/\/rgw2:80"
+                            ],
+                            "log_meta": "false",
+                            "log_data": "true",
+                            "bucket_index_max_shards": 0,
+                            "read_only": "false"
+                        }
+                    ],
+                    "placement_targets": [
+                        {
+                            "name": "default-placement",
+                            "tags": []
+                        }
+                    ],
+                    "default_placement": "default-placement",
+                    "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe"
+                }
+            }
+        ],
+        "master_zonegroup": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "bucket_quota": {
+            "enabled": false,
+            "max_size_kb": -1,
+            "max_objects": -1
+        },
+        "user_quota": {
+            "enabled": false,
+            "max_size_kb": -1,
+            "max_objects": -1
+        }
+    }
+
+#. To set a zonegroup map, run the following command:
+
+   .. prompt:: bash #
+   
+      radosgw-admin zonegroup-map set --infile zonegroupmap.json
+
+   In this command, ``zonegroupmap.json`` is the JSON file you created. Ensure
+   that you have zones created for the ones specified in the zonegroup map.
+
+#. Update the period:
+
+   .. prompt:: bash #
+   
+      radosgw-admin period update --commit
+
+.. _radosgw-zones:
+
+Zones
+-----
+
+A zone defines a logical group that consists of one or more Ceph Object Gateway
+instances. All RGWs in a given zone serve S3 objects that are backed by RADOS objects that are stored in the same set of pools in the same cluster. Ceph Object Gateway supports zones.
+
+The procedure for configuring zones differs from typical configuration
+procedures, because not all of the settings end up in a Ceph configuration
+file. 
+
+Zones can be listed. You can "get" a zone configuration and "set" a zone
+configuration.
+
+Creating a Zone
+~~~~~~~~~~~~~~~
+
+To create a zone, specify a zone name. If you are creating a master zone,
+specify the ``--master`` flag. Only one zone in a zonegroup may be a master
+zone. To add the zone to a zonegroup, specify the ``--rgw-zonegroup`` option
+with the zonegroup name.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone create --rgw-zone=<name> \
+                    [--zonegroup=<zonegroup-name]\
+                    [--endpoints=<endpoint>[,<endpoint>] \
+                    [--master] [--default] \
+                    --access-key $SYSTEM_ACCESS_KEY --secret $SYSTEM_SECRET_KEY
+
+After you have created the zone, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Deleting a Zone
+~~~~~~~~~~~~~~~
+
+To delete a zone, first remove it from the zonegroup:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup remove --zonegroup=<name>\
+                                     --zone=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Next, delete the zone:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone delete --rgw-zone<name>
+
+Finally, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+.. important:: Do not delete a zone without removing it from a zonegroup first.
+               Otherwise, updating the period will fail.
+
+If the pools for the deleted zone will not be used anywhere else,
+consider deleting the pools. Replace ``<del-zone>`` in the example below
+with the deleted zone’s name.
+
+.. important:: Only delete the pools with prepended zone names. Deleting the
+   root pool (for example, ``.rgw.root``) will remove all of the system’s
+   configuration.
+
+.. important:: When the pools are deleted, all of the data within them are
+   deleted in an unrecoverable manner. Delete the pools only if the pool's
+   contents are no longer needed.
+
+.. prompt:: bash #
+   
+   ceph osd pool rm <del-zone>.rgw.control <del-zone>.rgw.control --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.meta <del-zone>.rgw.meta --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.log <del-zone>.rgw.log --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.otp <del-zone>.rgw.otp --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.index <del-zone>.rgw.buckets.index --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.non-ec <del-zone>.rgw.buckets.non-ec --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.data <del-zone>.rgw.buckets.data --yes-i-really-really-mean-it
+
+Modifying a Zone
+~~~~~~~~~~~~~~~~
+
+To modify a zone, specify the zone name and the parameters you wish to
+modify.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone modify [options]
+
+Where ``[options]``:
+
+- ``--access-key=<key>``
+- ``--secret/--secret-key=<key>``
+- ``--master``
+- ``--default``
+- ``--endpoints=<list>``
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Listing Zones
+~~~~~~~~~~~~~
+
+As ``root``, to list the zones in a cluster, run the following command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone list
+
+Getting a Zone
+~~~~~~~~~~~~~~
+
+As ``root``, to get the configuration of a zone, run the following command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone get [--rgw-zone=<zone>]
+
+The ``default`` zone looks like this:
+
+::
+
+    { "domain_root": ".rgw",
+      "control_pool": ".rgw.control",
+      "gc_pool": ".rgw.gc",
+      "log_pool": ".log",
+      "intent_log_pool": ".intent-log",
+      "usage_log_pool": ".usage",
+      "user_keys_pool": ".users",
+      "user_email_pool": ".users.email",
+      "user_swift_pool": ".users.swift",
+      "user_uid_pool": ".users.uid",
+      "system_key": { "access_key": "", "secret_key": ""},
+      "placement_pools": [
+          {  "key": "default-placement",
+             "val": { "index_pool": ".rgw.buckets.index",
+                      "data_pool": ".rgw.buckets"}
+          }
+        ]
+      }
+
+Setting a Zone
+~~~~~~~~~~~~~~
+
+Configuring a zone involves specifying a series of Ceph Object Gateway
+pools. For consistency, we recommend using a pool prefix that is the
+same as the zone name. See
+`Pools <http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__
+for details of configuring pools.
+
+To set a zone, create a JSON object consisting of the pools, save the
+object to a file (e.g., ``zone.json``); then, run the following
+command, replacing ``{zone-name}`` with the name of the zone:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone set --rgw-zone={zone-name} --infile zone.json
+
+Where ``zone.json`` is the JSON file you created.
+
+Then, as ``root``, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Renaming a Zone
+~~~~~~~~~~~~~~~
+
+To rename a zone, specify the zone name and the new zone name.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone rename --rgw-zone=<name> --zone-new-name=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Zonegroup and Zone Settings
+----------------------------
+
+When configuring a default zonegroup and zone, the pool name includes
+the zone name. For example:
+
+-  ``default.rgw.control``
+
+To change the defaults, include the following settings in your Ceph
+configuration file under each ``[client.radosgw.{instance-name}]``
+instance.
+
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| Name                                | Description                       | Type    | Default               |
++=====================================+===================================+=========+=======================+
+| ``rgw_zone``                        | The name of the zone for the      | String  | None                  |
+|                                     | gateway instance.                 |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zonegroup``                   | The name of the zonegroup for     | String  | None                  |
+|                                     | the gateway instance.             |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zonegroup_root_pool``         | The root pool for the zonegroup.  | String  | ``.rgw.root``         |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zone_root_pool``              | The root pool for the zone.       | String  | ``.rgw.root``         |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_default_zone_group_info_oid`` | The OID for storing the default   | String  | ``default.zonegroup`` |
+|                                     | zonegroup. We do not recommend    |         |                       |
+|                                     | changing this setting.            |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+
+
+Zone Features
+=============
+
+Some multisite features require support from all zones before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
+
+On creation of new zones and zonegroups, all known features are supported and some features (see table below) are enabled by default. After upgrading an existing multisite configuration, however, new features must be enabled manually.
+
+Supported Features
+------------------
+
++-----------------------------------+---------+----------+
+| Feature                           | Release | Default  |
++===================================+=========+==========+
+| :ref:`feature_resharding`         | Reef    | Enabled  |
++-----------------------------------+---------+----------+
+| :ref:`feature_compress_encrypted` | Reef    | Disabled |
++-----------------------------------+---------+----------+
+
+.. _feature_resharding:
+
+resharding
+~~~~~~~~~~
+
+This feature allows buckets to be resharded in a multisite configuration
+without interrupting the replication of their objects. When
+``rgw_dynamic_resharding`` is enabled, it runs on each zone independently, and
+zones may choose different shard counts for the same bucket. When buckets are
+resharded manually with ``radosgw-admin bucket reshard``, only that zone's
+bucket is modified. A zone feature should only be marked as supported after all
+of its RGWs and OSDs have upgraded.
+
+.. note:: Dynamic resharding is not supported in multisite deployments prior to
+   the Reef release.
+
+
+.. _feature_compress_encrypted:
+
+compress-encrypted
+~~~~~~~~~~~~~~~~~~
+
+This feature enables support for combining `Server-Side Encryption`_ and
+`Compression`_ on the same object. Object data gets compressed before encryption.
+Prior to Reef, multisite would not replicate such objects correctly, so all zones
+must upgrade to Reef or later before enabling.
+
+.. warning:: The compression ratio may leak information about the encrypted data,
+   and allow attackers to distinguish whether two same-sized objects might contain
+   the same data. Due to these security considerations, this feature is disabled
+   by default.
+
+Commands
+--------
+
+Add support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+Remove support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Enable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Disable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+.. _`Pools`: ../pools
+.. _`Sync Policy Config`: ../multisite-sync-policy
+.. _`Server-Side Encryption`: ../encryption
+.. _`Compression`: ../compression
diff --git a/doc/radosgw/multitenancy.rst b/doc/radosgw/multitenancy.rst
new file mode 100644
index 000000000..09f5071c1
--- /dev/null
+++ b/doc/radosgw/multitenancy.rst
@@ -0,0 +1,169 @@
+.. _rgw-multitenancy:
+
+=================
+RGW Multi-tenancy
+=================
+
+.. versionadded:: Jewel
+
+The multi-tenancy feature allows to use buckets and users of the same
+name simultaneously by segregating them under so-called ``tenants``.
+This may be useful, for instance, to permit users of Swift API to
+create buckets with easily conflicting names such as "test" or "trove".
+
+From the Jewel release onward, each user and bucket lies under a tenant.
+For compatibility, a "legacy" tenant with an empty name is provided.
+Whenever a bucket is referred without an explicit tenant, an implicit
+tenant is used, taken from the user performing the operation. Since
+the pre-existing users are under the legacy tenant, they continue
+to create and access buckets as before. The layout of objects in RADOS
+is extended in a compatible way, ensuring a smooth upgrade to Jewel.
+
+Administering Users With Explicit Tenants
+=========================================
+
+Tenants as such do not have any operations on them. They appear and
+disappear as needed, when users are administered. In order to create,
+modify, and remove users with explicit tenants, either an additional
+option --tenant is supplied, or a syntax '<tenant>$<user>' is used
+in the parameters of the radosgw-admin command.
+
+Examples
+--------
+
+Create a user testx$tester to be accessed with S3::
+
+  # radosgw-admin --tenant testx --uid tester --display-name "Test User" --access_key TESTER --secret test123 user create
+
+Create a user testx$tester to be accessed with Swift::
+
+  # radosgw-admin --tenant testx --uid tester --display-name "Test User" --subuser tester:test --key-type swift --access full user create
+  # radosgw-admin --subuser 'testx$tester:test' --key-type swift --secret test123
+
+.. note:: The subuser with explicit tenant has to be quoted in the shell.
+
+   Tenant names may contain only alphanumeric characters and underscores.
+
+Accessing Buckets with Explicit Tenants
+=======================================
+
+When a client application accesses buckets, it always operates with
+credentials of a particular user. As mentioned above, every user belongs
+to a tenant. Therefore, every operation has an implicit tenant in its
+context, to be used if no tenant is specified explicitly. Thus a complete
+compatibility is maintained with previous releases, as long as the
+referred buckets and referring user belong to the same tenant.
+In other words, anything unusual occurs when accessing another tenant's
+buckets *only*.
+
+Extensions employed to specify an explicit tenant differ according
+to the protocol and authentication system used.
+
+S3
+--
+
+In case of S3, a colon character is used to separate tenant and bucket.
+Thus a sample URL would be::
+
+  https://ep.host.dom/tenant:bucket
+
+Here's a simple Python sample:
+
+.. code-block:: python
+   :linenos:
+
+	from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+	c = S3Connection(
+		aws_access_key_id="TESTER",
+		aws_secret_access_key="test123",
+		host="ep.host.dom",
+		calling_format = OrdinaryCallingFormat())
+	bucket = c.get_bucket("test5b:testbucket")
+
+Note that it's not possible to supply an explicit tenant using
+a hostname. Hostnames cannot contain colons, or any other separators
+that are not already valid in bucket names. Using a period creates an
+ambiguous syntax. Therefore, the bucket-in-URL-path format has to be
+used.
+
+Due to the fact that the native S3 API does not deal with
+multi-tenancy and radosgw's implementation does, things get a bit
+involved when dealing with signed URLs and public read ACLs.
+
+* A **signed URL** does contain the ``AWSAccessKeyId`` query
+  parameters, from which radosgw is able to discern the correct user
+  and tenant owning the bucket. In other words, an application
+  generating signed URLs should be able to take just the un-prefixed
+  bucket name, and produce a signed URL that itself contains the
+  bucket name without the tenant prefix. However, it is *possible* to
+  include the prefix if you so choose.
+
+  Thus, accessing a signed URL of an object ``bar`` in a container
+  ``foo`` belonging to the tenant ``7188e165c0ae4424ac68ae2e89a05c50``
+  would be possible either via
+  ``http://<host>:<port>/foo/bar?AWSAccessKeyId=b200fb6634c547199e436a0f93c0c46e&Expires=1542890806&Signature=eok6CYQC%2FDwmQQmqvY5jTg6ehXU%3D``,
+  or via
+  ``http://<host>:<port>/7188e165c0ae4424ac68ae2e89a05c50:foo/bar?AWSAccessKeyId=b200fb6634c547199e436a0f93c0c46e&Expires=1542890806&Signature=eok6CYQC%2FDwmQQmqvY5jTg6ehXU%3D``,
+  depending on whether or not the tenant prefix was passed in on
+  signature generation.
+
+* A bucket with a **public read ACL** is meant to be read by an HTTP
+  client *without* including any query parameters that would allow
+  radosgw to discern tenants. Thus, publicly readable objects must
+  always be accessed using the bucket name with the tenant prefix.
+
+  Thus, if you set a public read ACL on an object ``bar`` in a
+  container ``foo`` belonging to the tenant
+  ``7188e165c0ae4424ac68ae2e89a05c50``, you would need to access that
+  object via the public URL
+  ``http://<host>:<port>/7188e165c0ae4424ac68ae2e89a05c50:foo/bar``.
+
+Swift with built-in authenticator
+---------------------------------
+
+TBD -- not in test_multen.py yet
+
+Swift with Keystone
+-------------------
+
+In the default configuration, although native Swift has inherent
+multi-tenancy, radosgw does not enable multi-tenancy for the Swift
+API. This is to ensure that a setup with legacy buckets --- that is,
+buckets that were created before radosgw supported multitenancy ---,
+those buckets retain their dual-API capability to be queried and
+modified using either S3 or Swift.
+
+If you want to enable multitenancy for Swift, particularly if your
+users only ever authenticate against OpenStack Keystone, you should
+enable Keystone-based multitenancy with the following ``ceph.conf``
+configuration option::
+
+  rgw keystone implicit tenants = true
+
+Once you enable this option, any newly connecting user (whether they
+are using the Swift API, or Keystone-authenticated S3) will prompt
+radosgw to create a user named ``<tenant_id>$<tenant_id``, where
+``<tenant_id>`` is a Keystone tenant (project) UUID --- for example,
+``7188e165c0ae4424ac68ae2e89a05c50$7188e165c0ae4424ac68ae2e89a05c50``.
+
+Whenever that user then creates an Swift container, radosgw internally
+translates the given container name into
+``<tenant_id>/<container_name>``, such as
+``7188e165c0ae4424ac68ae2e89a05c50/foo``. This ensures that if there
+are two or more different tenants all creating a container named
+``foo``, radosgw is able to transparently discern them by their tenant
+prefix.
+
+It is also possible to limit the effects of implicit tenants
+to only apply to swift or s3, by setting ``rgw keystone implicit tenants``
+to either ``s3`` or ``swift``.  This will likely primarily
+be of use to users who had previously used implicit tenants
+with older versions of ceph, where implicit tenants
+only applied to the swift protocol.
+
+Notes and known issues
+----------------------
+
+Just to be clear, it is not possible to create buckets in other
+tenants at present. The owner of newly created bucket is extracted
+from authentication information.
diff --git a/doc/radosgw/nfs.rst b/doc/radosgw/nfs.rst
new file mode 100644
index 000000000..373765e10
--- /dev/null
+++ b/doc/radosgw/nfs.rst
@@ -0,0 +1,375 @@
+===
+NFS
+===
+
+.. versionadded:: Jewel
+
+.. note:: Only the NFSv4 protocol is supported when using a cephadm or rook based deployment.
+
+Ceph Object Gateway namespaces can be exported over the file-based
+NFSv4 protocols, alongside traditional HTTP access
+protocols (S3 and Swift).
+
+In particular, the Ceph Object Gateway can now be configured to
+provide file-based access when embedded in the NFS-Ganesha NFS server.
+
+The simplest and preferred way of managing nfs-ganesha clusters and rgw exports
+is using ``ceph nfs ...`` commands. See :doc:`/mgr/nfs` for more details.
+
+librgw
+======
+
+The librgw.so shared library (Unix) provides a loadable interface to
+Ceph Object Gateway services, and instantiates a full Ceph Object Gateway
+instance on initialization.
+
+In turn, librgw.so exports rgw_file, a stateful API for file-oriented
+access to RGW buckets and objects.  The API is general, but its design
+is strongly influenced by the File System Abstraction Layer (FSAL) API
+of NFS-Ganesha, for which it has been primarily designed.
+
+A set of Python bindings is also provided.
+
+Namespace Conventions
+=====================
+
+The implementation conforms to Amazon Web Services (AWS) hierarchical
+namespace conventions which map UNIX-style path names onto S3 buckets
+and objects.
+
+The top level of the attached namespace consists of S3 buckets,
+represented as NFS directories. Files and directories subordinate to
+buckets are each represented as objects, following S3 prefix and
+delimiter conventions, with '/' being the only supported path
+delimiter [#]_.
+
+For example, if an NFS client has mounted an RGW namespace at "/nfs",
+then a file "/nfs/mybucket/www/index.html" in the NFS namespace
+corresponds to an RGW object "www/index.html" in a bucket/container
+"mybucket."
+
+Although it is generally invisible to clients, the NFS namespace is
+assembled through concatenation of the corresponding paths implied by
+the objects in the namespace.  Leaf objects, whether files or
+directories, will always be materialized in an RGW object of the
+corresponding key name, "<name>" if a file, "<name>/" if a directory.
+Non-leaf directories (e.g., "www" above) might only be implied by
+their appearance in the names of one or more leaf objects. Directories
+created within NFS or directly operated on by an NFS client (e.g., via
+an attribute-setting operation such as chown or chmod) always have a
+leaf object representation used to store materialized attributes such
+as Unix ownership and permissions.
+
+Supported Operations
+====================
+
+The RGW NFS interface supports most operations on files and
+directories, with the following restrictions:
+
+- Links, including symlinks, are not supported.
+- NFS ACLs are not supported.
+
+  + Unix user and group ownership and permissions *are* supported.
+
+- Directories may not be moved/renamed.
+
+  + Files may be moved between directories.
+
+- Only full, sequential *write* I/O is supported
+
+  + i.e., write operations are constrained to be **uploads**.
+  + Many typical I/O operations such as editing files in place will necessarily fail as they perform non-sequential stores.
+  + Some file utilities *apparently* writing sequentially (e.g., some versions of GNU tar) may fail due to infrequent non-sequential stores.
+  + When mounting via NFS, sequential application I/O can generally be constrained to be written sequentially to the NFS server via a synchronous mount option (e.g. -osync in Linux).
+  + NFS clients which cannot mount synchronously (e.g., MS Windows) will not be able to upload files.
+
+Security
+========
+
+The RGW NFS interface provides a hybrid security model with the
+following characteristics:
+
+- NFS protocol security is provided by the NFS-Ganesha server, as negotiated by the NFS server and clients
+
+  + e.g., clients can by trusted (AUTH_SYS), or required to present Kerberos user credentials (RPCSEC_GSS)
+  + RPCSEC_GSS wire security can be integrity only (krb5i) or integrity and privacy (encryption, krb5p)
+  + various NFS-specific security and permission rules are available
+
+    * e.g., root-squashing
+
+- a set of RGW/S3 security credentials (unknown to NFS) is associated with each RGW NFS mount (i.e., NFS-Ganesha EXPORT)
+
+  + all RGW object operations performed via the NFS server will be performed by the RGW user associated with the credentials stored in the export being accessed (currently only RGW and RGW LDAP credentials are supported)
+
+    * additional RGW authentication types such as Keystone are not currently supported
+
+Manually configuring an NFS-Ganesha Instance
+============================================
+
+Each NFS RGW instance is an NFS-Ganesha server instance *embedding*
+a full Ceph RGW instance.
+
+Therefore, the RGW NFS configuration includes Ceph and Ceph Object
+Gateway-specific configuration in a local ceph.conf, as well as
+NFS-Ganesha-specific configuration in the NFS-Ganesha config file,
+ganesha.conf.
+
+ceph.conf
+---------
+
+Required ceph.conf configuration for RGW NFS includes:
+
+* valid [client.rgw.{instance-name}] section
+* valid values for minimal instance configuration, in particular, an installed and correct ``keyring``
+
+Other config variables (e.g., ``rgw data`` and ``rgw backend store``) are
+optional.
+
+A small number of config variables (e.g., ``rgw_nfs_namespace_expire_secs``)
+are unique to RGW NFS.
+
+In particular, front-end selection is handled specially by the librgw.so runtime.  By default, only the
+``rgw-nfs`` frontend is started. Additional frontends (e.g., ``beast``) are enabled via the
+``rgw nfs frontends`` config option.  Its syntax is identical to the ordinary ``rgw frontends`` option.
+Default options for non-default frontends are specified via ``rgw frontend defaults`` as normal.
+
+ganesha.conf
+------------
+
+A strictly minimal ganesha.conf for use with RGW NFS includes one
+EXPORT block with embedded FSAL block of type RGW::
+
+   EXPORT
+   {
+        Export_ID={numeric-id};
+        Path = "/";
+        Pseudo = "/";
+        Access_Type = RW;
+        SecType = "sys";
+        NFS_Protocols = 4;
+        Transport_Protocols = TCP;
+
+        # optional, permit unsquashed access by client "root" user
+        #Squash = No_Root_Squash;
+
+        FSAL {
+                Name = RGW;
+                User_Id = {s3-user-id};
+                Access_Key_Id ="{s3-access-key}";
+                Secret_Access_Key = "{s3-secret}";
+        }
+   }
+
+``Export_ID`` must have an integer value, e.g., "77"
+
+``Path`` (for RGW) should be "/"
+
+``Pseudo`` defines an NFSv4 pseudo root name (NFSv4 only)
+
+``SecType = sys;`` allows clients to attach without Kerberos
+authentication
+
+``Squash = No_Root_Squash;`` enables the client root user to override
+permissions (Unix convention).  When root-squashing is enabled,
+operations attempted by the root user are performed as if by the local
+"nobody" (and "nogroup") user on the NFS-Ganesha server
+
+The RGW FSAL additionally supports RGW-specific configuration
+variables in the RGW config section::
+
+ RGW {
+     cluster = "{cluster name, default 'ceph'}";
+     name = "client.rgw.{instance-name}";
+     ceph_conf = "/opt/ceph-rgw/etc/ceph/ceph.conf";
+     init_args = "-d --debug-rgw=16";
+ }
+
+``cluster`` sets a Ceph cluster name (must match the cluster being exported)
+
+``name`` sets an RGW instance name (must match the cluster being exported)
+
+``ceph_conf`` gives a path to a non-default ceph.conf file to use
+
+
+Other useful NFS-Ganesha configuration:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any EXPORT block which should support NFSv3 should include version 3
+in the NFS_Protocols setting. Additionally, NFSv3 is the last major
+version to support the UDP transport. To enable UDP, include it in the
+Transport_Protocols setting. For example::
+
+ EXPORT {
+  ...
+    NFS_Protocols = 3,4;
+    Transport_Protocols = UDP,TCP;
+  ...
+ }
+
+One important family of options pertains to interaction with the Linux
+idmapping service, which is used to normalize user and group names
+across systems.  Details of idmapper integration are not provided here.
+
+With Linux NFS clients, NFS-Ganesha can be configured
+to accept client-supplied numeric user and group identifiers with
+NFSv4, which by default stringifies these--this may be useful in small
+setups and for experimentation::
+
+ NFSV4 {
+     Allow_Numeric_Owners = true;
+     Only_Numeric_Owners = true;
+ }
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+NFS-Ganesha configuration problems are usually debugged by running the
+server with debugging options, controlled by the LOG config section.
+
+NFS-Ganesha log messages are grouped into various components, logging
+can be enabled separately for each component. Valid values for
+component logging include::
+
+  *FATAL* critical errors only
+  *WARN* unusual condition
+  *DEBUG* mildly verbose trace output
+  *FULL_DEBUG* verbose trace output
+
+Example::
+  
+  LOG {
+
+	Components {
+		MEMLEAKS = FATAL;
+		FSAL = FATAL;
+		NFSPROTO = FATAL;
+		NFS_V4 = FATAL;
+		EXPORT = FATAL;
+		FILEHANDLE = FATAL;
+		DISPATCH = FATAL;
+		CACHE_INODE = FATAL;
+		CACHE_INODE_LRU = FATAL;
+		HASHTABLE = FATAL;
+		HASHTABLE_CACHE = FATAL;
+		DUPREQ = FATAL;
+		INIT = DEBUG;
+		MAIN = DEBUG;
+		IDMAPPER = FATAL;
+		NFS_READDIR = FATAL;
+		NFS_V4_LOCK = FATAL;
+		CONFIG = FATAL;
+		CLIENTID = FATAL;
+		SESSIONS = FATAL;
+		PNFS = FATAL;
+		RW_LOCK = FATAL;
+		NLM = FATAL;
+		RPC = FATAL;
+		NFS_CB = FATAL;
+		THREAD = FATAL;
+		NFS_V4_ACL = FATAL;
+		STATE = FATAL;
+		FSAL_UP = FATAL;
+		DBUS = FATAL;
+	}
+	# optional: redirect log output
+ #	Facility {
+ #		name = FILE;
+ #		destination = "/tmp/ganesha-rgw.log";
+ #		enable = active;
+	}
+ }
+
+Running Multiple NFS Gateways
+=============================
+
+Each NFS-Ganesha instance acts as a full gateway endpoint, with the
+limitation that currently an NFS-Ganesha instance cannot be configured
+to export HTTP services. As with ordinary gateway instances, any
+number of NFS-Ganesha instances can be started, exporting the same or
+different resources from the cluster. This enables the clustering of
+NFS-Ganesha instances. However, this does not imply high availability.
+
+When regular gateway instances and NFS-Ganesha instances overlap the
+same data resources, they will be accessible from both the standard S3
+API and through the NFS-Ganesha instance as exported. You can
+co-locate the NFS-Ganesha instance with a Ceph Object Gateway instance
+on the same host. 
+
+RGW vs RGW NFS
+==============
+
+Exporting an NFS namespace and other RGW namespaces (e.g., S3 or Swift
+via the Civetweb HTTP front-end) from the same program instance is
+currently not supported.
+
+When adding objects and buckets outside of NFS, those objects will
+appear in the NFS namespace in the time set by
+``rgw_nfs_namespace_expire_secs``, which defaults to 300 seconds (5 minutes).
+Override the default value for ``rgw_nfs_namespace_expire_secs`` in the
+Ceph configuration file to change the refresh rate.
+
+If exporting Swift containers that do not conform to valid S3 bucket
+naming requirements, set ``rgw_relaxed_s3_bucket_names`` to true in the
+[client.rgw] section of the Ceph configuration file. For example,
+if a Swift container name contains underscores, it is not a valid S3
+bucket name and will be rejected unless ``rgw_relaxed_s3_bucket_names``
+is set to true.
+
+Configuring NFSv4 clients
+=========================
+
+To access the namespace, mount the configured NFS-Ganesha export(s)
+into desired locations in the local POSIX namespace. As noted, this
+implementation has a few unique restrictions:
+
+- NFS 4.1 and higher protocol flavors are preferred
+
+  + NFSv4 OPEN and CLOSE operations are used to track upload transactions
+
+- To upload data successfully, clients must preserve write ordering
+
+  + on Linux and many Unix NFS clients, use the -osync mount option
+
+Conventions for mounting NFS resources are platform-specific. The
+following conventions work on Linux and some Unix platforms:
+
+From the command line::
+
+  mount -t nfs -o nfsvers=4.1,noauto,soft,sync,proto=tcp <ganesha-host-name>:/ <mount-point>
+
+In /etc/fstab::
+
+<ganesha-host-name>:/ <mount-point> nfs noauto,soft,nfsvers=4.1,sync,proto=tcp 0 0
+
+Specify the NFS-Ganesha host name and the path to the mount point on
+the client.
+
+Configuring NFSv3 Clients
+=========================
+
+Linux clients can be configured to mount with NFSv3 by supplying
+``nfsvers=3`` and ``noacl`` as mount options. To use UDP as the
+transport, add ``proto=udp`` to the mount options. However, TCP is the
+preferred transport::
+
+  <ganesha-host-name>:/ <mount-point> nfs noauto,noacl,soft,nfsvers=3,sync,proto=tcp 0 0
+
+Configure the NFS Ganesha EXPORT block Protocols setting with version
+3 and the Transports setting with UDP if the mount will use version 3 with UDP.
+
+NFSv3 Semantics
+---------------
+
+Since NFSv3 does not communicate client OPEN and CLOSE operations to
+file servers, RGW NFS cannot use these operations to mark the
+beginning and ending of file upload transactions. Instead, RGW NFS
+starts a new upload when the first write is sent to a file at offset
+0, and finalizes the upload when no new writes to the file have been
+seen for a period of time, by default, 10 seconds. To change this
+timeout, set an alternate value for ``rgw_nfs_write_completion_interval_s``
+in the RGW section(s) of the Ceph configuration file. 
+
+References
+==========
+
+.. [#] http://docs.aws.amazon.com/AmazonS3/latest/dev/ListingKeysHierarchy.html
diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
new file mode 100644
index 000000000..1d18772b2
--- /dev/null
+++ b/doc/radosgw/notifications.rst
@@ -0,0 +1,547 @@
+====================
+Bucket Notifications
+====================
+
+.. versionadded:: Nautilus
+
+.. contents::
+
+Bucket notifications provide a mechanism for sending information out of radosgw
+when certain events happen on the bucket. Notifications can be sent to HTTP
+endpoints, AMQP0.9.1 endpoints, and Kafka endpoints.
+
+A user can create topics. A topic entity is defined by its name and is "per
+tenant". A user can associate its topics (via notification configuration) only
+with buckets it owns.
+
+A notification entity must be created in order to send event notifications for
+a specific bucket. A notification entity can be created either for a subset
+of event types or for all event types (which is the default). The
+notification may also filter out events based on matches of the prefixes and
+suffixes of (1) the keys, (2) the metadata attributes attached to the object,
+or (3) the object tags. Regular-expression matching can also be used on these
+to create filters. There can be multiple notifications for any specific topic,
+and the same topic can used for multiple notifications.
+
+REST API has been defined so as to provide configuration and control interfaces
+for the bucket notification mechanism.
+
+.. toctree::
+   :maxdepth: 1
+
+   S3 Bucket Notification Compatibility <s3-notification-compatibility>
+
+.. note:: To enable bucket notifications API, the `rgw_enable_apis` configuration parameter should contain: "notifications".
+
+Notification Reliability
+------------------------
+
+Notifications can be sent synchronously or asynchronously. This section
+describes the latency and reliability that you should expect for synchronous
+and asynchronous notifications.
+
+Synchronous Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Notifications can be sent synchronously, as part of the operation that
+triggered them. In this mode, the operation is acknowledged (acked) only after
+the notification is sent to the topic's configured endpoint. This means that
+the round trip time of the notification (the time it takes to send the
+notification to the topic's endpoint plus the time it takes to receive the
+acknowledgement) is added to the latency of the operation itself.
+
+.. note:: The original triggering operation is considered successful even if
+   the notification fails with an error, cannot be delivered, or times out.
+
+Asynchronous Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Notifications can be sent asynchronously. They are committed into persistent
+storage and then asynchronously sent to the topic's configured endpoint. In
+this case, the only latency added to the original operation is the latency
+added when the notification is committed to persistent storage.
+
+.. note:: If the notification fails with an error, cannot be delivered, or
+   times out, it is retried until it is successfully acknowledged.
+
+.. tip:: To minimize the latency added by asynchronous notification, we 
+   recommended placing the "log" pool on fast media.
+
+
+Topic Management via CLI
+------------------------
+
+Fetch the configuration of all topics associated with tenants by running the
+following command:
+
+.. prompt:: bash #
+
+   radosgw-admin topic list [--tenant={tenant}]
+
+
+Fetch the configuration of a specific topic by running the following command:
+
+.. prompt:: bash #
+
+   radosgw-admin topic get --topic={topic-name} [--tenant={tenant}]
+
+
+Remove a topic by running the following command: 
+
+.. prompt:: bash #
+
+   radosgw-admin topic rm --topic={topic-name} [--tenant={tenant}]
+
+
+Notification Performance Statistics
+-----------------------------------
+
+- ``pubsub_event_triggered``: a running counter of events that have at least one topic associated with them
+- ``pubsub_event_lost``: a running counter of events that had topics associated with them, but that were not pushed to any of the endpoints
+- ``pubsub_push_ok``: a running counter, for all notifications, of events successfully pushed to their endpoints
+- ``pubsub_push_fail``: a running counter, for all notifications, of events that failed to be pushed to their endpoints
+- ``pubsub_push_pending``: the gauge value of events pushed to an endpoint but not acked or nacked yet
+
+.. note::
+
+    ``pubsub_event_triggered`` and ``pubsub_event_lost`` are incremented per
+    event on each notification, but ``pubsub_push_ok`` and ``pubsub_push_fail``
+    are incremented per push action on each notification.
+
+Bucket Notification REST API
+----------------------------
+
+Topics
+~~~~~~
+
+.. note::
+
+    In all topic actions, the parameters are URL-encoded and sent in the
+    message body using this content type:
+    ``application/x-www-form-urlencoded``.
+   
+
+.. _Create a Topic:
+
+Create a Topic
+``````````````
+
+This creates a new topic. Provide the topic with push endpoint parameters,
+which will be used later when a notification is created. A response is
+generated. A successful response includes the topic's `ARN
+<https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_
+(the "Amazon Resource Name", a unique identifier used to reference the topic).
+To update a topic, use the same command that you used to create it (but when
+updating, use the name of an existing topic and different endpoint values).
+
+.. tip:: Any notification already associated with the topic must be re-created
+   in order for the topic to update.
+
+::
+
+   POST
+
+   Action=CreateTopic
+   &Name=<topic-name>
+   [&Attributes.entry.1.key=amqp-exchange&Attributes.entry.1.value=<exchange>]
+   [&Attributes.entry.2.key=amqp-ack-level&Attributes.entry.2.value=none|broker|routable]
+   [&Attributes.entry.3.key=verify-ssl&Attributes.entry.3.value=true|false]
+   [&Attributes.entry.4.key=kafka-ack-level&Attributes.entry.4.value=none|broker]
+   [&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=true|false]
+   [&Attributes.entry.6.key=ca-location&Attributes.entry.6.value=<file path>]
+   [&Attributes.entry.7.key=OpaqueData&Attributes.entry.7.value=<opaque data>]
+   [&Attributes.entry.8.key=push-endpoint&Attributes.entry.8.value=<endpoint>]
+   [&Attributes.entry.9.key=persistent&Attributes.entry.9.value=true|false]
+   [&Attributes.entry.10.key=cloudevents&Attributes.entry.10.value=true|false]
+   [&Attributes.entry.11.key=mechanism&Attributes.entry.11.value=<mechanism>]
+
+Request parameters:
+
+- push-endpoint: This is the URI of an endpoint to send push notifications to.
+- OpaqueData: Opaque data is set in the topic configuration and added to all
+  notifications that are triggered by the topic.
+- persistent: This indicates whether notifications to this endpoint are
+  persistent (=asynchronous) or not persistent. (This is "false" by default.)
+
+- HTTP endpoint
+
+ - URI: ``http[s]://<fqdn>[:<port]``
+ - port: This defaults to 80 for HTTP and 443 for HTTPS.
+ - verify-ssl: This indicates whether the server certificate is validated by
+   the client. (This is "true" by default.)
+ - cloudevents: This indicates whether the HTTP header should contain
+   attributes according to the `S3 CloudEvents Spec`_. (This is "false" by
+   default.)
+
+- AMQP0.9.1 endpoint
+
+ - URI: ``amqp[s]://[<user>:<password>@]<fqdn>[:<port>][/<vhost>]``
+ - user/password: This defaults to "guest/guest".
+ - user/password: This must be provided only over HTTPS. Topic creation
+   requests will otherwise be rejected.
+ - port: This defaults to 5672 for unencrypted connections and 5671 for
+   SSL-encrypted connections.
+ - vhost: This defaults to "/".
+ - verify-ssl: This indicates whether the server certificate is validated by
+   the client. (This is "true" by default.)
+ - If ``ca-location`` is provided and a secure connection is used, the
+   specified CA will be used to authenticate the broker. The default CA will
+   not be used.  
+ - amqp-exchange: The exchanges must exist and must be able to route messages
+   based on topics. This parameter is mandatory.
+ - amqp-ack-level: No end2end acking is required. Messages may persist in the
+   broker before being delivered to their final destinations. Three ack methods
+   exist:
+
+  - "none": The message is considered "delivered" if it is sent to the broker.
+  - "broker": The message is considered "delivered" if it is acked by the broker (default).
+  - "routable": The message is considered "delivered" if the broker can route to a consumer.
+
+.. tip:: The topic-name (see :ref:`Create a Topic`) is used for the
+   AMQP topic ("routing key" for a topic exchange).
+
+- Kafka endpoint
+
+ - URI: ``kafka://[<user>:<password>@]<fqdn>[:<port]``
+ - ``use-ssl``: If this is set to "true", a secure connection is used to
+   connect to the broker. (This is "false" by default.)
+ - ``ca-location``: If this is provided and a secure connection is used, the
+   specified CA will be used instead of the default CA to authenticate the
+   broker. 
+ - user/password may be provided over HTTPS. If not, the config parameter
+   `rgw_allow_notification_secrets_in_cleartext` must be `true` in order to create topic
+ - user/password may be provided along with ``use-ssl``.
+   The broker credentials will otherwise be sent over insecure transport
+ - ``mechanism`` may be provided together with user/password (default: ``PLAIN``).
+   The supported SASL mechanisms are:
+
+  - PLAIN
+  - SCRAM-SHA-256
+  - SCRAM-SHA-512
+  - GSSAPI
+  - OAUTHBEARER
+
+ - port: This defaults to 9092.
+ - kafka-ack-level: No end2end acking is required. Messages may persist in the
+   broker before being delivered to their final destinations. Two ack methods
+   exist:
+
+  - "none": Messages are considered "delivered" if sent to the broker.
+  - "broker": Messages are considered "delivered" if acked by the broker. (This
+    is the default.)
+
+.. note::
+
+    - The key-value pair of a specific parameter need not reside in the same
+      line as the parameter, and need not appear in any specific order, but it
+      must use the same index.
+    - Attribute indexing need not be sequential and need not start from any
+      specific value.
+    - `AWS Create Topic`_ provides a detailed explanation of the endpoint
+      attributes format. In our case, however, different keys and values are
+      used.
+
+The response has the following format:
+
+::
+
+    <CreateTopicResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <CreateTopicResult>
+            <TopicArn></TopicArn>
+        </CreateTopicResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </CreateTopicResponse>
+
+The topic `ARN
+<https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_
+in the response has the following format:
+
+::
+
+   arn:aws:sns:<zone-group>:<tenant>:<topic>
+
+Get Topic Attributes
+````````````````````
+
+This returns information about a specific topic. This includes push-endpoint
+information, if provided.
+
+::
+
+   POST
+
+   Action=GetTopicAttributes
+   &TopicArn=<topic-arn>
+
+The response has the following format:
+
+::
+
+    <GetTopicAttributesResponse>
+        <GetTopicAttributesResult>
+            <Attributes>
+                <entry>
+                    <key>User</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>Name</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>EndPoint</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>TopicArn</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>OpaqueData</key>
+                    <value></value>
+                </entry> 
+            </Attributes>
+        </GetTopicAttributesResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </GetTopicAttributesResponse>
+
+- User: the name of the user that created the topic.
+- Name: the name of the topic.
+- EndPoint: The JSON-formatted endpoint parameters, including:
+   - EndpointAddress: The push-endpoint URL.
+   - EndpointArgs: The push-endpoint args.
+   - EndpointTopic: The topic name to be sent to the endpoint (can be different
+     than the above topic name).
+   - HasStoredSecret: This is "true" if the endpoint URL contains user/password 
+     information. In this case, the request must be made over HTTPS. The "topic
+     get" request will otherwise be rejected.
+   - Persistent: This is "true" if the topic is persistent.
+- TopicArn: topic `ARN
+  <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
+- OpaqueData: The opaque data set on the topic.
+
+Get Topic Information
+`````````````````````
+
+This returns information about a specific topic. This includes push-endpoint
+information, if provided.  Note that this API is now deprecated in favor of the
+AWS compliant `GetTopicAttributes` API.
+
+::
+
+   POST
+
+   Action=GetTopic
+   &TopicArn=<topic-arn>
+
+The response has the following format:
+
+::
+
+    <GetTopicResponse>
+        <GetTopicResult>
+            <Topic>
+                <User></User>
+                <Name></Name>
+                <EndPoint>
+                    <EndpointAddress></EndpointAddress>
+                    <EndpointArgs></EndpointArgs>
+                    <EndpointTopic></EndpointTopic>
+                    <HasStoredSecret></HasStoredSecret>
+                    <Persistent></Persistent>
+                </EndPoint>
+                <TopicArn></TopicArn>
+                <OpaqueData></OpaqueData>
+            </Topic>
+        </GetTopicResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </GetTopicResponse>
+
+- User: The name of the user that created the topic.
+- Name: The name of the topic.
+- EndpointAddress: The push-endpoint URL.
+- EndpointArgs: The push-endpoint args.
+- EndpointTopic: The topic name to be sent to the endpoint (which can be
+  different than the above topic name).
+- HasStoredSecret: This is "true" if the endpoint URL contains user/password
+  information. In this case, the request must be made over HTTPS. The "topic
+  get" request will otherwise be rejected.
+- Persistent: "true" if topic is persistent.
+- TopicArn: topic `ARN
+  <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
+- OpaqueData: the opaque data set on the topic.
+
+Delete Topic
+````````````
+
+::
+
+   POST
+
+   Action=DeleteTopic
+   &TopicArn=<topic-arn>
+
+This deletes the specified topic.
+
+.. note::
+
+  - Deleting an unknown notification (for example, double delete) is not
+    considered an error.
+  - Deleting a topic does not automatically delete all notifications associated
+    with it.
+
+The response has the following format:
+
+::
+
+    <DeleteTopicResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </DeleteTopicResponse>
+
+List Topics
+```````````
+
+List all topics associated with a tenant.
+
+::
+
+   POST
+
+   Action=ListTopics
+
+The response has the following format:
+
+::
+
+    <ListTopicsResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <ListTopicsResult>
+            <Topics>
+                <member>
+                    <User></User>
+                    <Name></Name>
+                    <EndPoint>
+                        <EndpointAddress></EndpointAddress>
+                        <EndpointArgs></EndpointArgs>
+                        <EndpointTopic></EndpointTopic>
+                    </EndPoint>
+                    <TopicArn></TopicArn>
+                    <OpaqueData></OpaqueData>
+                </member>
+            </Topics>
+        </ListTopicsResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </ListTopicsResponse>
+
+- If the endpoint URL contains user/password information in any part of the
+  topic, the request must be made over HTTPS. The "topic list" request will
+  otherwise be rejected.
+
+Notifications
+~~~~~~~~~~~~~
+
+Detailed under: `Bucket Operations`_.
+
+.. note::
+
+    - "Abort Multipart Upload" request does not emit a notification
+    - Both "Initiate Multipart Upload" and "POST Object" requests will emit an ``s3:ObjectCreated:Post`` notification
+
+Events
+~~~~~~
+
+Events are in JSON format (regardless of the actual endpoint), and are S3-compatible.
+For example:
+
+::
+
+   {"Records":[
+       {
+           "eventVersion":"2.1",
+           "eventSource":"ceph:s3",
+           "awsRegion":"zonegroup1",
+           "eventTime":"2019-11-22T13:47:35.124724Z",
+           "eventName":"ObjectCreated:Put",
+           "userIdentity":{
+               "principalId":"tester"
+           },
+           "requestParameters":{
+               "sourceIPAddress":""
+           },
+           "responseElements":{
+               "x-amz-request-id":"503a4c37-85eb-47cd-8681-2817e80b4281.5330.903595",
+               "x-amz-id-2":"14d2-zone1-zonegroup1"
+           },
+           "s3":{
+               "s3SchemaVersion":"1.0",
+               "configurationId":"mynotif1",
+               "bucket":{
+                   "name":"mybucket1",
+                   "ownerIdentity":{
+                       "principalId":"tester"
+                   },
+                   "arn":"arn:aws:s3:zonegroup1::mybucket1",
+                   "id":"503a4c37-85eb-47cd-8681-2817e80b4281.5332.38"
+               },
+               "object":{
+                   "key":"myimage1.jpg",
+                   "size":"1024",
+                   "eTag":"37b51d194a7513e45b56f6524f2d51f2",
+                   "versionId":"",
+                   "sequencer": "F7E6D75DC742D108",
+                   "metadata":[],
+                   "tags":[]
+               }
+           },
+           "eventId":"",
+           "opaqueData":"me@example.com"
+       }
+   ]}
+
+- awsRegion: The zonegroup.
+- eventTime: The timestamp, indicating when the event was triggered.
+- eventName: For the list of supported events see: `S3 Notification
+  Compatibility`_. Note that eventName values do not start with the `s3:`
+  prefix.
+- userIdentity.principalId: The user that triggered the change.
+- requestParameters.sourceIPAddress: not supported
+- responseElements.x-amz-request-id: The request ID of the original change.
+- responseElements.x_amz_id_2: The RGW on which the change was made.
+- s3.configurationId: The notification ID that created the event.
+- s3.bucket.name: The name of the bucket.
+- s3.bucket.ownerIdentity.principalId: The owner of the bucket.
+- s3.bucket.arn: The ARN of the bucket.
+- s3.bucket.id: The ID of the bucket. (This is an extension to the S3
+  notification API.)
+- s3.object.key: The object key.
+- s3.object.size: The object size.
+- s3.object.eTag: The object etag.
+- s3.object.versionId: The object version, if the bucket is versioned. When a
+  copy is made, it includes the version of the target object. When a delete
+  marker is created, it includes the version of the delete marker.
+- s3.object.sequencer: The monotonically-increasing identifier of the "change
+  per object" (hexadecimal format).
+- s3.object.metadata: Any metadata set on the object that is sent as
+  ``x-amz-meta-`` (that is, any metadata set on the object that is sent as an
+  extension to the S3 notification API).
+- s3.object.tags: Any tags set on the object. (This is an extension to the S3
+  notification API.)
+- s3.eventId: The unique ID of the event, which could be used for acking. (This
+  is an extension to the S3 notification API.)
+- s3.opaqueData: This means that "opaque data" is set in the topic configuration
+  and is added to all notifications triggered by the topic. (This is an
+  extension to the S3 notification API.)
+
+.. _S3 Notification Compatibility: ../s3-notification-compatibility
+.. _AWS Create Topic: https://docs.aws.amazon.com/sns/latest/api/API_CreateTopic.html
+.. _Bucket Operations: ../s3/bucketops
+.. _S3 CloudEvents Spec: https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md
diff --git a/doc/radosgw/oidc.rst b/doc/radosgw/oidc.rst
new file mode 100644
index 000000000..46593f1d8
--- /dev/null
+++ b/doc/radosgw/oidc.rst
@@ -0,0 +1,97 @@
+===============================
+ OpenID Connect Provider in RGW
+===============================
+
+An entity describing the OpenID Connect Provider needs to be created in RGW, in order to establish trust between the two.
+
+REST APIs for Manipulating an OpenID Connect Provider
+=====================================================
+
+The following REST APIs can be used for creating and managing an OpenID Connect Provider entity in RGW.
+
+In order to invoke the REST admin APIs, a user with admin caps needs to be created.
+
+.. code-block:: javascript
+
+  radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create
+  radosgw-admin caps add --uid="TESTER" --caps="oidc-provider=*"
+
+
+CreateOpenIDConnectProvider
+---------------------------------
+
+Create an OpenID Connect Provider entity in RGW
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``ClientIDList.member.N``
+
+:Description: List of Client Ids that needs access to S3 resources.
+:Type: Array of Strings
+
+``ThumbprintList.member.N``
+
+:Description: List of OpenID Connect IDP's server certificates' thumbprints. A maximum of 5 thumbprints are allowed.
+:Type: Array of Strings
+
+``Url``
+
+:Description: URL of the IDP.
+:Type: String
+
+
+Example::
+  POST "<hostname>?Action=Action=CreateOpenIDConnectProvider
+    &ThumbprintList.list.1=F7D7B3515DD0D319DD219A43A9EA727AD6065287
+    &ClientIDList.list.1=app-profile-jsp
+    &Url=http://localhost:8080/auth/realms/quickstart
+
+
+DeleteOpenIDConnectProvider
+---------------------------
+
+Deletes an OpenID Connect Provider entity in RGW
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``OpenIDConnectProviderArn``
+
+:Description: ARN of the IDP which is returned by the Create API.
+:Type: String
+
+Example::
+  POST "<hostname>?Action=Action=DeleteOpenIDConnectProvider
+    &OpenIDConnectProviderArn=arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart
+
+
+GetOpenIDConnectProvider
+---------------------------
+
+Gets information about an IDP.
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``OpenIDConnectProviderArn``
+
+:Description: ARN of the IDP which is returned by the Create API.
+:Type: String
+
+Example::
+  POST "<hostname>?Action=Action=GetOpenIDConnectProvider
+    &OpenIDConnectProviderArn=arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart
+
+ListOpenIDConnectProviders
+--------------------------
+
+Lists information about all IDPs
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None
+
+Example::
+  POST "<hostname>?Action=Action=ListOpenIDConnectProviders
diff --git a/doc/radosgw/opa.rst b/doc/radosgw/opa.rst
new file mode 100644
index 000000000..f1b76b5ef
--- /dev/null
+++ b/doc/radosgw/opa.rst
@@ -0,0 +1,72 @@
+==============================
+Open Policy Agent Integration
+==============================
+
+Open Policy Agent (OPA) is a lightweight general-purpose policy engine
+that can be co-located with a service. OPA can be integrated as a
+sidecar, host-level daemon, or library.
+
+Services can offload policy decisions to OPA by executing queries. Hence,
+policy enforcement can be decoupled from policy decisions.
+
+Configure OPA
+=============
+
+To configure OPA, load custom policies into OPA that control the resources users
+are allowed to access. Relevant data or context can also be loaded into OPA to make decisions.
+
+Policies and data can be loaded into OPA in the following ways::
+  * OPA's RESTful APIs
+  * OPA's *bundle* feature that downloads policies and data from remote HTTP servers
+  * Filesystem
+
+Configure the Ceph Object Gateway
+=================================
+
+The following configuration options are available for OPA integration::
+
+     rgw use opa authz = {use opa server to authorize client requests}
+     rgw opa url = {opa server url:opa server port}
+     rgw opa token = {opa bearer token}
+     rgw opa verify ssl = {verify opa server ssl certificate}
+
+How does the RGW-OPA integration work
+=====================================
+
+After a user is authenticated, OPA can be used to check if the user is authorized
+to perform the given action on the resource. OPA responds with an allow or deny
+decision which is sent back to the RGW which enforces the decision.
+
+Example request::
+
+   POST /v1/data/ceph/authz HTTP/1.1
+   Host: opa.example.com:8181
+   Content-Type: application/json
+   
+   {
+       "input": {
+           "method": "GET",
+           "subuser": "subuser",
+           "user_info": {
+               "user_id": "john",
+               "display_name": "John"  
+           },
+           "bucket_info": {
+               "bucket": {
+                   "name": "Testbucket",
+                   "bucket_id": "testbucket" 
+               },
+               "owner": "john" 
+           }             
+       }
+   }
+
+Response::
+
+   {"result": true}
+
+The above is a sample request sent to OPA which contains information about the
+user, resource and the action to be performed on the resource. Based on the polices
+and data loaded into OPA, it will verify whether the request should be allowed or denied.
+In the sample request, RGW makes a POST request to the endpoint */v1/data/ceph/authz*,
+where *ceph* is the package name and *authz* is the rule name.
diff --git a/doc/radosgw/orphans.rst b/doc/radosgw/orphans.rst
new file mode 100644
index 000000000..bf6b10edf
--- /dev/null
+++ b/doc/radosgw/orphans.rst
@@ -0,0 +1,117 @@
+==================================
+Orphan List and Associated Tooling
+==================================
+
+.. version added:: Luminous
+
+.. contents::
+
+Orphans are RADOS objects that are left behind after their associated
+RGW objects are removed. Normally these RADOS objects are removed
+automatically, either immediately or through a process known as
+"garbage collection". Over the history of RGW, however, there may have
+been bugs that prevented these RADOS objects from being deleted, and
+these RADOS objects may be consuming space on the Ceph cluster without
+being of any use. From the perspective of RGW, we call such RADOS
+objects "orphans".
+
+Orphans Find -- DEPRECATED
+--------------------------
+
+The `radosgw-admin` tool has/had three subcommands to help manage
+orphans, however these subcommands are (or will soon be)
+deprecated. These subcommands are:
+
+.. prompt:: bash #
+
+   radosgw-admin orphans find ...
+   radosgw-admin orphans finish ...
+   radosgw-admin orphans list-jobs ...
+
+There are two key problems with these subcommands, however. First,
+these subcommands have not been actively maintained and therefore have
+not tracked RGW as it has evolved in terms of features and updates. As
+a result the confidence that these subcommands can accurately identify
+true orphans is presently low.
+
+Second, these subcommands store intermediate results on the cluster
+itself. This can be problematic when cluster administrators are
+confronting insufficient storage space and want to remove orphans as a
+means of addressing the issue. The intermediate results could strain
+the existing cluster storage capacity even further.
+
+For these reasons "orphans find" has been deprecated.
+
+Orphan List
+-----------
+
+Because "orphans find" has been deprecated, RGW now includes an
+additional tool -- 'rgw-orphan-list'. When run it will list the
+available pools and prompt the user to enter the name of the data
+pool. At that point the tool will, perhaps after an extended period of
+time, produce a local file containing the RADOS objects from the
+designated pool that appear to be orphans. The administrator is free
+to examine this file and the decide on a course of action, perhaps
+removing those RADOS objects from the designated pool.
+
+All intermediate results are stored on the local file system rather
+than the Ceph cluster. So running the 'rgw-orphan-list' tool should
+have no appreciable impact on the amount of cluster storage consumed.
+
+WARNING: Experimental Status
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The 'rgw-orphan-list' tool is new and therefore currently considered
+experimental. The list of orphans produced should be "sanity checked"
+before being used for a large delete operation.
+
+WARNING: Specifying a Data Pool
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a pool other than an RGW data pool is specified, the results of the
+tool will be erroneous. All RADOS objects found on such a pool will
+falsely be designated as orphans.
+
+WARNING: Unindexed Buckets
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+RGW allows for unindexed buckets, that is buckets that do not maintain
+an index of their contents. This is not a typical configuration, but
+it is supported. Because the 'rgw-orphan-list' tool uses the bucket
+indices to determine what RADOS objects should exist, objects in the
+unindexed buckets will falsely be listed as orphans.
+
+
+RADOS List
+----------
+
+One of the sub-steps in computing a list of orphans is to map each RGW
+object into its corresponding set of RADOS objects. This is done using
+a subcommand of 'radosgw-admin'.
+
+.. prompt:: bash #
+
+   radosgw-admin bucket radoslist [--bucket={bucket-name}]
+
+The subcommand will produce a list of RADOS objects that support all
+of the RGW objects. If a bucket is specified then the subcommand will
+only produce a list of RADOS objects that correspond back the RGW
+objects in the specified bucket.
+
+Note: Shared Bucket Markers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some administrators will be aware of the coding schemes used to name
+the RADOS objects that correspond to RGW objects, which include a
+"marker" unique to a given bucket.
+
+RADOS objects that correspond with the contents of one RGW bucket,
+however, may contain a marker that specifies a different bucket. This
+behavior is a consequence of the "shallow copy" optimization used by
+RGW. When larger objects are copied from bucket to bucket, only the
+"head" objects are actually copied, and the tail objects are
+shared. Those shared objects will contain the marker of the original
+bucket.
+
+.. _Data Layout in RADOS : ../layout
+.. _Pool Placement and Storage Classes : ../placement
diff --git a/doc/radosgw/placement.rst b/doc/radosgw/placement.rst
new file mode 100644
index 000000000..28c71783d
--- /dev/null
+++ b/doc/radosgw/placement.rst
@@ -0,0 +1,263 @@
+==================================
+Pool Placement and Storage Classes
+==================================
+
+.. contents::
+
+Placement Targets
+=================
+
+.. versionadded:: Jewel
+
+Placement targets control which `Pools`_ are associated with a particular
+bucket. A bucket's placement target is selected on creation, and cannot be
+modified. The ``radosgw-admin bucket stats`` command will display its
+``placement_rule``.
+
+The zonegroup configuration contains a list of placement targets with an
+initial target named ``default-placement``. The zone configuration then maps
+each zonegroup placement target name onto its local storage. This zone
+placement information includes the ``index_pool`` name for the bucket index,
+the ``data_extra_pool`` name for metadata about incomplete multipart uploads,
+and a ``data_pool`` name for each storage class.
+
+.. _storage_classes:
+
+Storage Classes
+===============
+
+.. versionadded:: Nautilus
+
+Storage classes are used to customize the placement of object data. S3 Bucket
+Lifecycle rules can automate the transition of objects between storage classes.
+
+Storage classes are defined in terms of placement targets. Each zonegroup
+placement target lists its available storage classes with an initial class
+named ``STANDARD``. The zone configuration is responsible for providing a
+``data_pool`` pool name for each of the zonegroup's storage classes.
+
+Zonegroup/Zone Configuration
+============================
+
+Placement configuration is performed with ``radosgw-admin`` commands on
+the zonegroups and zones.
+
+The zonegroup placement configuration can be queried with:
+
+::
+
+  $ radosgw-admin zonegroup get
+  {
+      "id": "ab01123f-e0df-4f29-9d71-b44888d67cd5",
+      "name": "default",
+      "api_name": "default",
+      ...
+      "placement_targets": [
+          {
+              "name": "default-placement",
+              "tags": [],
+              "storage_classes": [
+                  "STANDARD"
+              ]
+          }
+      ],
+      "default_placement": "default-placement",
+      ...
+  }
+
+The zone placement configuration can be queried with:
+
+::
+
+  $ radosgw-admin zone get
+  {
+      "id": "557cdcee-3aae-4e9e-85c7-2f86f5eddb1f",
+      "name": "default",
+      "domain_root": "default.rgw.meta:root",
+      ...
+      "placement_pools": [
+          {
+              "key": "default-placement",
+              "val": {
+                  "index_pool": "default.rgw.buckets.index",
+                  "storage_classes": {
+                      "STANDARD": {
+                          "data_pool": "default.rgw.buckets.data"
+                      }
+                  },
+                  "data_extra_pool": "default.rgw.buckets.non-ec",
+                  "index_type": 0,
+                  "inline_data": true
+              }
+          }
+      ],
+      ...
+  }
+
+.. note:: If you have not done any previous `Multisite Configuration`_,
+          a ``default`` zone and zonegroup are created for you, and changes
+          to the zone/zonegroup will not take effect until the Ceph Object
+          Gateways are restarted. If you have created a realm for multisite,
+          the zone/zonegroup changes will take effect once the changes are
+          committed with ``radosgw-admin period update --commit``.
+
+Adding a Placement Target
+-------------------------
+
+To create a new placement target named ``temporary``, start by adding it to
+the zonegroup:
+
+::
+
+  $ radosgw-admin zonegroup placement add \
+        --rgw-zonegroup default \
+        --placement-id temporary
+
+Then provide the zone placement info for that target:
+
+::
+
+  $ radosgw-admin zone placement add \
+        --rgw-zone default \
+        --placement-id temporary \
+        --data-pool default.rgw.temporary.data \
+        --index-pool default.rgw.temporary.index \
+        --data-extra-pool default.rgw.temporary.non-ec
+
+.. note:: With default placement target settings, RGW stores an object's first data chunk in the RADOS "head" object along
+          with xattr metadata. The `--placement-inline-data=false` flag may be passed with the `zone placement add` or
+          `zone placement modify` commands to change this behavior for new objects stored on the target.
+          When data is stored inline (default), it may provide an advantage for read/write workloads since the first chunk of
+          an object's data can be retrieved/stored in a single librados call along with object metadata. On the other hand, a
+          target that does not store data inline can provide a performance benefit for RGW client delete requests when
+          the BlueStore DB is located on faster storage than bucket data since it eliminates the need to access
+          slower devices synchronously while processing the client request. In that case, data associated with the deleted
+          objects is removed asynchronously in the background by garbage collection.                                          
+
+.. _adding_a_storage_class:
+
+Adding a Storage Class
+----------------------
+
+To add a new storage class named ``GLACIER`` to the ``default-placement`` target,
+start by adding it to the zonegroup:
+
+::
+
+  $ radosgw-admin zonegroup placement add \
+        --rgw-zonegroup default \
+        --placement-id default-placement \
+        --storage-class GLACIER
+
+Then provide the zone placement info for that storage class:
+
+::
+
+  $ radosgw-admin zone placement add \
+        --rgw-zone default \
+        --placement-id default-placement \
+        --storage-class GLACIER \
+        --data-pool default.rgw.glacier.data \
+        --compression lz4
+
+Customizing Placement
+=====================
+
+Default Placement
+-----------------
+
+By default, new buckets will use the zonegroup's ``default_placement`` target.
+This zonegroup setting can be changed with:
+
+::
+
+  $ radosgw-admin zonegroup placement default \
+        --rgw-zonegroup default \
+        --placement-id new-placement
+
+User Placement
+--------------
+
+A Ceph Object Gateway user can override the zonegroup's default placement
+target by setting a non-empty ``default_placement`` field in the user info.
+Similarly, the ``default_storage_class`` can override the ``STANDARD``
+storage class applied to objects by default.
+
+::
+
+  $ radosgw-admin user info --uid testid
+  {
+      ...
+      "default_placement": "",
+      "default_storage_class": "",
+      "placement_tags": [],
+      ...
+  }
+
+If a zonegroup's placement target contains any ``tags``, users will be unable
+to create buckets with that placement target unless their user info contains
+at least one matching tag in its ``placement_tags`` field. This can be useful
+to restrict access to certain types of storage.
+
+The ``radosgw-admin`` command can modify these fields directly with:
+
+::
+
+  $ radosgw-admin user modify \
+        --uid <user-id> \
+        --placement-id <default-placement-id> \
+        --storage-class <default-storage-class> \
+        --tags <tag1,tag2>
+
+.. _s3_bucket_placement:
+
+S3 Bucket Placement
+-------------------
+
+When creating a bucket with the S3 protocol, a placement target can be
+provided as part of the LocationConstraint to override the default placement
+targets from the user and zonegroup.
+
+Normally, the LocationConstraint must match the zonegroup's ``api_name``:
+
+::
+
+  <LocationConstraint>default</LocationConstraint>
+
+A custom placement target can be added to the ``api_name`` following a colon:
+
+::
+
+  <LocationConstraint>default:new-placement</LocationConstraint>
+
+Swift Bucket Placement
+----------------------
+
+When creating a bucket with the Swift protocol, a placement target can be
+provided in the HTTP header ``X-Storage-Policy``:
+
+::
+
+  X-Storage-Policy: new-placement
+
+Using Storage Classes
+=====================
+
+All placement targets have a ``STANDARD`` storage class which is applied to
+new objects by default. The user can override this default with its
+``default_storage_class``.
+
+To create an object in a non-default storage class, provide that storage class
+name in an HTTP header with the request. The S3 protocol uses the
+``X-Amz-Storage-Class`` header, while the Swift protocol uses the
+``X-Object-Storage-Class`` header.
+
+When using AWS S3 SDKs such as ``boto3``, it is important that non-default
+storage class names match those provided by AWS S3, or else the SDK
+will drop the request and raise an exception.
+
+S3 Object Lifecycle Management can then be used to move object data between
+storage classes using ``Transition`` actions.
+
+.. _`Pools`: ../pools
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/pools.rst b/doc/radosgw/pools.rst
new file mode 100644
index 000000000..bb1246c1f
--- /dev/null
+++ b/doc/radosgw/pools.rst
@@ -0,0 +1,57 @@
+=====
+Pools
+=====
+
+The Ceph Object Gateway uses several pools for its various storage needs,
+which are listed in the Zone object (see ``radosgw-admin zone get``). A
+single zone named ``default`` is created automatically with pool names
+starting with ``default.rgw.``, but a `Multisite Configuration`_ will have
+multiple zones.
+
+Tuning
+======
+
+When ``radosgw`` first tries to operate on a zone pool that does not
+exist, it will create that pool with the default values from
+``osd pool default pg num`` and ``osd pool default pgp num``. These defaults
+are sufficient for some pools, but others (especially those listed in
+``placement_pools`` for the bucket index and data) will require additional
+tuning. We recommend using the `Ceph Placement Group’s per Pool
+Calculator <https://old.ceph.com/pgcalc/>`__ to calculate a suitable number of
+placement groups for these pools. See
+`Pools <http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__
+for details on pool creation.
+
+.. _radosgw-pool-namespaces:
+
+Pool Namespaces
+===============
+
+.. versionadded:: Luminous
+
+Pool names particular to a zone follow the naming convention
+``{zone-name}.pool-name``. For example, a zone named ``us-east`` will
+have the following pools:
+
+-  ``.rgw.root``
+
+-  ``us-east.rgw.control``
+
+-  ``us-east.rgw.meta``
+
+-  ``us-east.rgw.log``
+
+-  ``us-east.rgw.buckets.index``
+
+-  ``us-east.rgw.buckets.data``
+
+The zone definitions list several more pools than that, but many of those
+are consolidated through the use of rados namespaces. For example, all of
+the following pool entries use namespaces of the ``us-east.rgw.meta`` pool::
+
+    "user_keys_pool": "us-east.rgw.meta:users.keys",
+    "user_email_pool": "us-east.rgw.meta:users.email",
+    "user_swift_pool": "us-east.rgw.meta:users.swift",
+    "user_uid_pool": "us-east.rgw.meta:users.uid",
+
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/qat-accel.rst b/doc/radosgw/qat-accel.rst
new file mode 100644
index 000000000..b275e8a19
--- /dev/null
+++ b/doc/radosgw/qat-accel.rst
@@ -0,0 +1,155 @@
+===============================================
+QAT Acceleration for Encryption and Compression
+===============================================
+
+Intel QAT (QuickAssist Technology) can provide extended accelerated encryption
+and compression services by offloading the actual encryption and compression
+request(s) to the hardware QuickAssist accelerators, which are more efficient
+in terms of cost and power than general purpose CPUs for those specific
+compute-intensive workloads.
+
+See `QAT Support for Compression`_ and `QAT based Encryption for RGW`_.
+
+
+QAT in the Software Stack 
+=========================
+
+Application developers can access QuickAssist features through the QAT API.
+The QAT API is the top-level API for QuickAssist technology, and enables easy
+interfacing between the customer application and the QuickAssist acceleration
+driver.
+
+The QAT API accesses the QuickAssist driver, which in turn drives the
+QuickAssist Accelerator hardware. The QuickAssist driver is responsible for
+exposing the acceleration services to the application software.
+
+A user can write directly to the QAT API, or the use of QAT can be done via
+frameworks that have been enabled by others including Intel (for example, zlib*,
+OpenSSL* libcrypto*, and the Linux* Kernel Crypto Framework).
+
+QAT Environment Setup
+=====================
+1. QuickAssist Accelerator hardware is necessary to make use of accelerated
+   encryption and compression services. And QAT driver in kernel space have to
+   be loaded to drive the hardware.
+
+The driver package can be downloaded from `Intel Quickassist Technology`_.
+
+2. The implementation for QAT based encryption is directly base on QAT API which
+   is included the driver package. But QAT support for compression depends on
+   QATzip project, which is a user space library which builds on top of the QAT
+   API. Currently, QATzip speeds up gzip compression and decompression at the
+   time of writing.
+
+See `QATzip`_.
+
+Implementation
+==============
+1. QAT based Encryption for RGW 
+
+`OpenSSL support for RGW encryption`_ has been merged into Ceph, and Intel also
+provides one `QAT Engine`_ for OpenSSL. So, theoretically speaking, QAT based
+encryption in Ceph can be directly supported through OpenSSl+QAT Engine.
+
+But the QAT Engine for OpenSSL currently supports chained operations only, and
+so Ceph will not be able to utilize QAT hardware feature for crypto operations
+based on OpenSSL crypto plugin. As a result, one QAT plugin based on native
+QAT API is added into crypto framework.
+
+2. QAT Support for Compression
+
+As mentioned above, QAT support for compression is based on QATzip library in
+user space, which is designed to take full advantage of the performance provided
+by QuickAssist Technology. Unlike QAT based encryption, QAT based compression
+is supported through a tool class for QAT acceleration rather than a compressor
+plugin. The common tool class can transparently accelerate the existing compression
+types, but only zlib compressor can be supported at the time of writing. So
+user is allowed to use it to speed up zlib compressor as long as the QAT
+hardware is available and QAT is capable to handle it.
+
+Configuration
+=============
+#. Prerequisites
+
+   Make sure the QAT driver with version v1.7.L.4.14.0 or higher has been installed.
+   Remember to set an environment variable "ICP_ROOT" for your QAT driver package
+   root directory. 
+
+   To enable the QAT based encryption and compression, user needs to modify the QAT
+   configuration files. For example, for Intel QuickAssist Adapter 8970 product, revise 
+   c6xx_dev0/1/2.conf in the directory ``/etc/`` and keep them the same, e.g.:
+
+   .. code-block:: ini
+        
+      #...
+      # User Process Instance Section
+      ##############################################
+      [CEPH]
+      NumberCyInstances = 1
+      NumberDcInstances = 1
+      NumProcesses = 8
+      LimitDevAccess = 1
+      # Crypto - User instance #0
+      Cy0Name = "SSL0"
+      Cy0IsPolled = 1
+      # List of core affinities
+      Cy0CoreAffinity = 0
+       
+      # Data Compression - User instance #0
+      Dc0Name = "Dc0"
+      Dc0IsPolled = 1
+      # List of core affinities
+      Dc0CoreAffinity = 0
+
+#. QAT based Encryption for RGW 
+
+   The CMake option ``WITH_QAT=ON`` must be configured. If you build Ceph from
+   source code (see: :ref:`build-ceph`), navigate to your cloned Ceph repository 
+   and execute the following:
+
+   .. prompt:: bash $ 
+
+      cd ceph
+      ./do_cmake.sh -DWITH_QAT=ON
+      cd build
+      ininja
+
+   .. note::
+     The section name of the QAT configuration files must be ``CEPH`` since 
+     the section name is set as "CEPH" in Ceph crypto source code.
+  
+   Then, edit the Ceph configuration file to make use of QAT based crypto plugin::
+
+      plugin crypto accelerator = crypto_qat
+
+#. QAT Support for Compression
+
+   Before starting, make sure both QAT driver and `QATzip`_  have been installed. Besides 
+   "ICP_ROOT", remember to set the environment variable "QZ_ROOT" for the root directory
+   of your QATzip source tree.
+
+   The following CMake options have to be configured to trigger QAT based compression
+   when building Ceph:
+  
+   .. prompt:: bash $
+
+      ./do_cmake.sh -DWITH_QAT=ON -DWITH_QATZIP=ON
+
+   Then, set an environment variable to clarify the section name of User Process Instance
+   Section in QAT configuration files, e.g.:
+  
+   .. prompt:: bash $
+
+      export QAT_SECTION_NAME=CEPH
+
+   Next, edit the Ceph configuration file to enable QAT support for compression::
+
+      qat compressor enabled=true
+
+
+.. _QAT Support for Compression: https://github.com/ceph/ceph/pull/19714
+.. _QAT based Encryption for RGW: https://github.com/ceph/ceph/pull/19386
+.. _Intel Quickassist Technology: https://01.org/intel-quickassist-technology
+.. _QATzip: https://github.com/intel/QATzip
+.. _OpenSSL support for RGW encryption: https://github.com/ceph/ceph/pull/15168
+.. _QAT Engine: https://github.com/intel/QAT_Engine
diff --git a/doc/radosgw/rgw-cache.rst b/doc/radosgw/rgw-cache.rst
new file mode 100644
index 000000000..116db8ed4
--- /dev/null
+++ b/doc/radosgw/rgw-cache.rst
@@ -0,0 +1,155 @@
+==========================
+RGW Data caching and CDN
+==========================
+
+.. versionadded:: Octopus
+
+.. contents::
+
+This feature adds to RGW the ability to securely cache objects and offload the workload from the cluster, using Nginx.
+After an object is accessed the first time it will be stored in the Nginx cache directory.
+When data is already cached, it need not be fetched from RGW. A permission check will be made against RGW to ensure the requesting user has access.
+This feature is based on some Nginx modules, ngx_http_auth_request_module, https://github.com/kaltura/nginx-aws-auth-module, Openresty for Lua capabilities.
+
+Currently, this feature will cache only AWSv4 requests (only s3 requests), caching-in the output of the 1st GET request
+and caching-out on subsequent GET requests, passing thru transparently PUT,POST,HEAD,DELETE and COPY requests.
+
+
+The feature introduces 2 new APIs: Auth and Cache.
+
+    NOTE: The `D3N RGW Data Cache`_ is an alternative data caching mechanism implemented natively in the Rados Gateway.
+
+New APIs
+-------------------------
+
+There are 2 new APIs for this feature:
+
+Auth API - The cache uses this to validate that a user can access the cached data
+
+Cache API - Adds the ability to override securely Range header, that way Nginx can use it is own smart cache on top of S3:
+https://www.nginx.com/blog/smart-efficient-byte-range-caching-nginx/
+Using this API gives the ability to read ahead objects when clients asking a specific range from the object.
+On subsequent accesses to the cached object, Nginx will satisfy requests for already-cached ranges from the cache. Uncached ranges will be read from RGW (and cached).
+
+Auth API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API Validates a specific authenticated access being made to the cache, using RGW's knowledge of the client credentials and stored access policy.
+Returns success if the encapsulated request would be granted.
+
+Cache API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API is meant to allow changing signed Range headers using a privileged user, cache user.
+
+Creating cache user
+
+::
+
+$ radosgw-admin user create --uid=<uid for cache user> --display-name="cache user" --caps="amz-cache=read"
+
+This user can send to the RGW the Cache API header ``X-Amz-Cache``, this header contains the headers from the original request(before changing the Range header).
+It means that ``X-Amz-Cache`` built from several headers.
+The headers that are building the ``X-Amz-Cache`` header are separated by char with ASCII code 177 and the header name and value are separated by char ASCII code 178.
+The RGW will check that the cache user is an authorized user and if it is a cache user,
+if yes it will use the ``X-Amz-Cache`` to revalidate that the user has permissions, using the headers from the X-Amz-Cache.
+During this flow, the RGW will override the Range header.
+
+
+Using Nginx with RGW
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Download the source of Openresty:
+
+::
+
+$ wget https://openresty.org/download/openresty-1.15.8.3.tar.gz
+
+git clone the AWS auth Nginx module:
+
+::
+
+$ git clone https://github.com/kaltura/nginx-aws-auth-module
+
+untar the openresty package:
+
+::
+
+$ tar xvzf openresty-1.15.8.3.tar.gz
+$ cd openresty-1.15.8.3
+
+Compile openresty, Make sure that you have pcre lib and openssl lib:
+
+::
+
+$ sudo yum install pcre-devel openssl-devel gcc curl zlib-devel nginx
+$ ./configure --add-module=<the nginx-aws-auth-module dir> --with-http_auth_request_module --with-http_slice_module --conf-path=/etc/nginx/nginx.conf
+$ gmake -j $(nproc)
+$ sudo gmake install
+$ sudo ln -sf /usr/local/openresty/bin/openresty /usr/bin/nginx
+
+Put in-place your Nginx configuration files and edit them according to your environment:
+
+All Nginx conf files are under: https://github.com/ceph/ceph/tree/main/examples/rgw/rgw-cache
+
+`nginx.conf` should go to `/etc/nginx/nginx.conf`
+
+`nginx-lua-file.lua` should go to `/etc/nginx/nginx-lua-file.lua`
+
+`nginx-default.conf` should go to `/etc/nginx/conf.d/nginx-default.conf`
+
+The parameters that are most likely to require adjustment according to the environment are located in the file `nginx-default.conf`
+
+Modify the example values of *proxy_cache_path* and *max_size* at:
+
+::
+
+ proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
+
+
+And modify the example *server* values to point to the RGWs URIs:
+
+::
+
+ server rgw1:8000 max_fails=2 fail_timeout=5s;
+ server rgw2:8000 max_fails=2 fail_timeout=5s;
+ server rgw3:8000 max_fails=2 fail_timeout=5s;
+
+| It is important to substitute the *access key* and *secret key* located in the `nginx.conf` with those belong to the user with the `amz-cache` caps
+| for example, create the `cache` user as following:
+
+::
+
+ radosgw-admin user create --uid=cacheuser --display-name="cache user" --caps="amz-cache=read" --access-key <access> --secret <secret>
+
+It is possible to use Nginx slicing which is a better method for streaming purposes.
+
+For using slice you should use `nginx-slicing.conf` and not `nginx-default.conf`
+
+Further information about Nginx slicing:
+
+https://docs.nginx.com/nginx/admin-guide/content-cache/content-caching/#byte-range-caching
+
+
+If you do not want to use the prefetch caching, It is possible to replace `nginx-default.conf` with `nginx-noprefetch.conf`
+Using `noprefetch` means that if the client is sending range request of 0-4095 and then 0-4096 Nginx will cache those requests separately, So it will need to fetch those requests twice.
+
+
+Run Nginx(openresty):
+
+::
+
+$ sudo systemctl restart nginx
+
+Appendix
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**A note about performance:** In certain instances like development environment, disabling the authentication by commenting the following line in `nginx-default.conf`:
+
+::
+
+ #auth_request /authentication;
+
+may (depending on the hardware) increases the performance significantly as it forgoes the auth API calls to radosgw.
+
+
+.. _D3N RGW Data Cache: ../d3n_datacache/
diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
new file mode 100644
index 000000000..e97449872
--- /dev/null
+++ b/doc/radosgw/role.rst
@@ -0,0 +1,570 @@
+======
+ Role
+======
+
+A role is similar to a user and has permission policies attached to it, that determine what a role can or can not do. A role can be assumed by any identity that needs it. If a user assumes a role, a set of dynamically created temporary credentials are returned to the user. A role can be used to delegate access to users, applications, services that do not have permissions to access some s3 resources.
+
+The following radosgw-admin commands can be used to create/ delete/ update a role and permissions associated with a role.
+
+Create a Role
+-------------
+
+To create a role, execute the following::
+
+	radosgw-admin role create --role-name={role-name} [--path=="{path to the role}"] [--assume-role-policy-doc={trust-policy-document}]
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``path``
+
+:Description: Path to the role. The default value is a slash(/).
+:Type: String
+
+``assume-role-policy-doc``
+
+:Description: The trust relationship policy document that grants an entity permission to assume the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role create --role-name=S3Access1 --path=/application_abc/component_xyz/ --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}
+  
+.. code-block:: javascript
+  
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+Delete a Role
+-------------
+
+To delete a role, execute the following::
+
+	radosgw-admin role delete --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role delete --role-name=S3Access1
+
+Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+
+Get a Role
+----------
+
+To get information about a role, execute the following::
+
+	radosgw-admin role get --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role get --role-name=S3Access1
+  
+.. code-block:: javascript
+  
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+List Roles
+----------
+
+To list roles with a specified path prefix, execute the following::
+
+	radosgw-admin role list [--path-prefix ={path prefix}]
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``path-prefix``
+
+:Description: Path prefix for filtering roles. If this is not specified, all roles are listed.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role list --path-prefix="/application"
+  
+.. code-block:: javascript
+  
+  [
+    {
+        "id": "3e1c0ff7-8f2b-456c-8fdf-20f428ba6a7f",
+        "name": "S3Access1",
+        "path": "/application_abc/component_xyz/",
+        "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+        "create_date": "2018-10-17T10:32:01.881Z",
+        "max_session_duration": 3600,
+        "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+    }
+  ]
+
+
+Update Assume Role Policy Document of a role
+--------------------------------------------
+
+To modify a role's assume role policy document, execute the following::
+
+	radosgw-admin role-trust-policy modify --role-name={role-name} --assume-role-policy-doc={trust-policy-document}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``assume-role-policy-doc``
+
+:Description: The trust relationship policy document that grants an entity permission to assume the role.
+:Type: String
+
+For example::
+
+  radosgw-admin role-trust-policy modify --role-name=S3Access1 --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER2\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}
+
+.. code-block:: javascript
+
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER2\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+In the above example, we are modifying the Principal from TESTER to TESTER2 in its assume role policy document.
+
+Add/ Update a Policy attached to a Role
+---------------------------------------
+
+To add or update the inline policy attached to a role, execute the following::
+
+	radosgw-admin role policy put --role-name={role-name} --policy-name={policy-name} --policy-doc={permission-policy-doc}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+``policy-doc``
+
+:Description: The Permission policy document.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:*\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\}
+
+For passing ``policy-doc`` as a file::
+
+  radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --infile policy-document.json
+
+In the above example, we are attaching a policy 'Policy1' to role 'S3Access1', which allows all s3 actions on 'example_bucket'.
+
+List Permission Policy Names attached to a Role
+-----------------------------------------------
+
+To list the names of permission policies attached to a role, execute the following::
+
+	radosgw-admin role policy get --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy list --role-name=S3Access1
+
+.. code-block:: javascript
+
+  [
+    "Policy1"
+  ]
+
+
+Get Permission Policy attached to a Role
+----------------------------------------
+
+To get a specific permission policy attached to a role, execute the following::
+
+	radosgw-admin role policy get --role-name={role-name} --policy-name={policy-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy get --role-name=S3Access1 --policy-name=Policy1
+
+.. code-block:: javascript
+
+  {
+    "Permission policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:*\"],\"Resource\":\"arn:aws:s3:::example_bucket\"}]}"
+  }
+
+
+Delete Policy attached to a Role
+--------------------------------
+
+To delete permission policy attached to a role, execute the following::
+
+	radosgw-admin role policy delete --role-name={role-name} --policy-name={policy-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy delete --role-name=S3Access1 --policy-name=Policy1
+
+
+Update a role
+-------------
+
+To update a role's max-session-duration, execute the following::
+
+	radosgw-admin role update --role-name={role-name} --max-session-duration={max-session-duration}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``max-session-duration``
+
+:Description: Maximum session duration for a role.
+:Type: String
+
+For example::
+
+  radosgw-admin role update --role-name=S3Access1 --max-session-duration=43200
+
+Note: This command currently can only be used to update max-session-duration.
+
+REST APIs for Manipulating a Role
+=================================
+
+In addition to the above radosgw-admin commands, the following REST APIs can be used for manipulating a role. For the request parameters and their explanations, refer to the sections above.
+
+In order to invoke the REST admin APIs, a user with admin caps needs to be created.
+
+.. code-block:: javascript
+
+  radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create
+  radosgw-admin caps add --uid="TESTER" --caps="roles=*"
+
+
+Create a Role
+-------------
+
+Example::
+  POST "<hostname>?Action=CreateRole&RoleName=S3Access&Path=/application_abc/component_xyz/&AssumeRolePolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+Delete a Role
+-------------
+
+Example::
+  POST "<hostname>?Action=DeleteRole&RoleName=S3Access"
+
+Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+
+Get a Role
+----------
+
+Example::
+  POST "<hostname>?Action=GetRole&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+List Roles
+----------
+
+Example::
+  POST "<hostname>?Action=ListRoles&RoleName=S3Access&PathPrefix=/application"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+Update Assume Role Policy Document
+----------------------------------
+
+Example::
+  POST "<hostname>?Action=UpdateAssumeRolePolicy&RoleName=S3Access&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER2\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
+
+Add/ Update a Policy attached to a Role
+---------------------------------------
+
+Example::
+  POST "<hostname>?Action=PutRolePolicy&RoleName=S3Access&PolicyName=Policy1&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:CreateBucket\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\}"
+
+List Permission Policy Names attached to a Role
+-----------------------------------------------
+
+Example::
+  POST "<hostname>?Action=ListRolePolicies&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <PolicyNames>
+    <member>Policy1</member>
+  </PolicyNames>
+
+
+Get Permission Policy attached to a Role
+----------------------------------------
+
+Example::
+  POST "<hostname>?Action=GetRolePolicy&RoleName=S3Access&PolicyName=Policy1"
+
+.. code-block:: XML
+
+  <GetRolePolicyResult>
+    <PolicyName>Policy1</PolicyName>
+    <RoleName>S3Access</RoleName>
+    <Permission_policy>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:CreateBucket"],"Resource":"arn:aws:s3:::example_bucket"}]}</Permission_policy>
+  </GetRolePolicyResult>
+
+
+Delete Policy attached to a Role
+--------------------------------
+
+Example::
+  POST "<hostname>?Action=DeleteRolePolicy&RoleName=S3Access&PolicyName=Policy1"
+
+Tag a role
+----------
+A role can have multivalued tags attached to it. These tags can be passed in as part of CreateRole REST API also.
+AWS does not support multi-valued role tags.
+
+Example::
+  POST "<hostname>?Action=TagRole&RoleName=S3Access&Tags.member.1.Key=Department&Tags.member.1.Value=Engineering"
+
+.. code-block:: XML
+
+  <TagRoleResponse>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000004-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </TagRoleResponse>
+
+
+List role tags
+--------------
+Lists the tags attached to a role.
+
+Example::
+  POST "<hostname>?Action=ListRoleTags&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <ListRoleTagsResponse>
+    <ListRoleTagsResult>
+      <Tags>
+        <member>
+          <Key>Department</Key>
+          <Value>Engineering</Value>
+        </member>
+      </Tags>
+    </ListRoleTagsResult>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000005-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </ListRoleTagsResponse>
+
+Delete role tags
+----------------
+Delete a tag/ tags attached to a role.
+
+Example::
+  POST "<hostname>?Action=UntagRoles&RoleName=S3Access&TagKeys.member.1=Department"
+
+.. code-block:: XML
+
+  <UntagRoleResponse>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000007-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </UntagRoleResponse>
+
+Update Role
+-----------
+
+Example::
+  POST "<hostname>?Action=UpdateRole&RoleName=S3Access&MaxSessionDuration=43200"
+
+.. code-block:: XML
+
+  <UpdateRoleResponse>
+    <UpdateRoleResult>
+      <ResponseMetadata>
+        <RequestId>tx000000000000000000007-00611f337e-1027-default</RequestId>
+      </ResponseMetadata>
+      </UpdateRoleResult>
+  </UpdateRoleResponse>
+
+Note: This API currently can only be used to update max-session-duration.
+
+Sample code for tagging, listing tags and untagging a role
+----------------------------------------------------------
+
+The following is sample code for adding tags to role, listing tags and untagging a role using boto3.
+
+.. code-block:: python
+
+    import boto3
+
+    access_key = 'TESTER'
+    secret_key = 'test123'
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url='http://s3.us-east.localhost:8000',
+    region_name=''
+    )
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"localhost:8080/auth/realms/quickstart:sub\":\"user1\"}}}]}"
+
+    print ("\n Creating Role with tags\n")
+    tags_list = [
+        {'Key':'Department','Value':'Engineering'}
+    ]
+    role_response = iam_client.create_role(
+        AssumeRolePolicyDocument=policy_document,
+        Path='/',
+        RoleName='S3Access',
+        Tags=tags_list,
+    )
+
+    print ("Adding tags to role\n")
+    response = iam_client.tag_role(
+                RoleName='S3Access',
+                Tags= [
+                        {'Key':'CostCenter','Value':'123456'}
+                    ]
+                )
+    print ("Listing role tags\n")
+    response = iam_client.list_role_tags(
+                RoleName='S3Access'
+                )
+    print (response)
+    print ("Untagging role\n")
+    response = iam_client.untag_role(
+        RoleName='S3Access',
+        TagKeys=[
+            'Department',
+        ]
+    )
+
+
+
diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
new file mode 100644
index 000000000..1627ed0c4
--- /dev/null
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -0,0 +1,149 @@
+=====================================
+S3 Bucket Notifications Compatibility
+=====================================
+
+Ceph's `Bucket Notifications`_ API follows `AWS S3 Bucket Notifications API`_. However, some differences exist, as listed below.
+
+
+.. note:: 
+
+    Compatibility is different depending on which of the above mechanism is used
+
+Supported Destination
+---------------------
+
+AWS supports: **SNS**, **SQS** and **Lambda** as possible destinations (AWS internal destinations). 
+Currently, we support: **HTTP/S**, **Kafka** and **AMQP**.
+
+We are using the **SNS** ARNs to represent the **HTTP/S**, **Kafka** and **AMQP** destinations.
+
+Notification Configuration XML
+------------------------------
+
+Following tags (and the tags inside them) are not supported:
+
++-----------------------------------+----------------------------------------------+
+| Tag                               | Remaks                                       |
++===================================+==============================================+
+| ``<QueueConfiguration>``          | not needed, we treat all destinations as SNS |
++-----------------------------------+----------------------------------------------+
+| ``<CloudFunctionConfiguration>``  | not needed, we treat all destinations as SNS |
++-----------------------------------+----------------------------------------------+
+
+REST API Extension 
+------------------
+
+Ceph's bucket notification API has the following extensions:
+
+- Deletion of a specific notification, or all notifications on a bucket, using the ``DELETE`` verb
+
+ - In S3, all notifications are deleted when the bucket is deleted, or when an empty notification is set on the bucket
+
+- Getting the information on a specific notification (when more than one exists on a bucket)
+
+  - In S3, it is only possible to fetch all notifications on a bucket
+
+- In addition to filtering based on prefix/suffix of object keys we support:
+
+  - Filtering based on regular expression matching
+
+  - Filtering based on metadata attributes attached to the object
+
+  - Filtering based on object tags
+
+- Each one of the additional filters extends the S3 API and using it will require extension of the client SDK (unless you are using plain HTTP). 
+
+- Filtering overlapping is allowed, so that same event could be sent as different notification
+
+
+Unsupported Fields in the Event Record
+--------------------------------------
+
+The records sent for bucket notification follows the format described in: `Event Message Structure`_.
+However, the ``requestParameters.sourceIPAddress`` field will be sent empty.
+
+
+Event Types
+-----------
+
++--------------------------------------------------------+-----------------------------------------+
+| Event                                                  | Note                                    |
++========================================================+=========================================+
+| ``s3:ObjectCreated:*``                                 | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectCreated:Put``                               | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectCreated:Post``                              | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectCreated:Copy``                              | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectCreated:CompleteMultipartUpload``           | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectRemoved:*``                                 | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectRemoved:Delete``                            | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectRemoved:DeleteMarkerCreated``               | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:Current``              | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:NonCurrent``           | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:DeleteMarker``         | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:AbortMultipartUpload`` | Defined, Ceph extension (not generated) |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Transition:Current``              | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectLifecycle:Transition:NonCurrent``           | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectSynced:*``                                  | Ceph extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectSynced:Create``                             | Ceph Extension                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectSynced:Delete``                             | Defined, Ceph extension (not generated) |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectSynced:DeletionMarkerCreated``              | Defined, Ceph extension (not generated) |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectRestore:Post``                              | Not applicable                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ObjectRestore:Complete``                          | Not applicable                          |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:ReducedRedundancyLostObject``                     | Not applicable                          |
++--------------------------------------------------------+-----------------------------------------+
+
+.. note:: 
+
+   The ``s3:ObjectRemoved:DeleteMarkerCreated`` event presents information on the latest version of the object
+
+.. note::
+
+   In case of multipart upload, an ``ObjectCreated:CompleteMultipartUpload`` notification will be sent at the end of the process.
+   
+.. note::
+
+   The ``s3:ObjectSynced:Create`` event is sent when an object successfully syncs to a zone. It must be explicitly set for each zone. 
+
+Topic Configuration
+-------------------
+In the case of bucket notifications, the topics management API will be derived from `AWS Simple Notification Service API`_. 
+Note that most of the API is not applicable to Ceph, and only the following actions are implemented:
+
+ - ``CreateTopic``
+ - ``DeleteTopic``
+ - ``ListTopics``
+
+We also have the following extensions to topic configuration: 
+
+ - In ``GetTopic`` we allow fetching a specific topic, instead of all user topics
+ - In ``CreateTopic``
+
+  - we allow setting endpoint attributes
+  - we allow setting opaque data that will be sent to the endpoint in the notification
+
+
+.. _AWS Simple Notification Service API: https://docs.aws.amazon.com/sns/latest/api/API_Operations.html
+.. _AWS S3 Bucket Notifications API: https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
+.. _Event Message Structure: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+.. _`Bucket Notifications`: ../notifications
+.. _`boto3 SDK filter extensions`: https://github.com/ceph/ceph/tree/main/examples/rgw/boto3
diff --git a/doc/radosgw/s3.rst b/doc/radosgw/s3.rst
new file mode 100644
index 000000000..cb5eb3adb
--- /dev/null
+++ b/doc/radosgw/s3.rst
@@ -0,0 +1,98 @@
+.. _radosgw s3:
+
+============================
+ Ceph Object Gateway S3 API
+============================
+
+Ceph supports a RESTful API that is compatible with the basic data access model of the `Amazon S3 API`_.
+
+API
+---
+
+.. toctree::
+   :maxdepth: 1
+
+   Common <s3/commons>
+   Authentication <s3/authentication>
+   Service Ops <s3/serviceops>
+   Bucket Ops <s3/bucketops>
+   Object Ops <s3/objectops>
+   C++ <s3/cpp>
+   C# <s3/csharp>
+   Java <s3/java>
+   Perl <s3/perl>
+   PHP <s3/php>
+   Python <s3/python>
+   Ruby <s3/ruby>
+
+
+Features Support
+----------------
+
+The following table describes the support status for current Amazon S3 functional features:
+
++---------------------------------+-----------------+----------------------------------------+
+| Feature                         | Status          | Remarks                                |
++=================================+=================+========================================+
+| **List Buckets**                | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Bucket**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Bucket**               | Supported       | Different set of canned ACLs           |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Lifecycle**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Replication**          | Partial         | Permitted only across zones            |
++---------------------------------+-----------------+----------------------------------------+
+| **Policy (Buckets, Objects)**   | Supported       | ACLs & bucket policies are supported   |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Website**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket ACLs (Get, Put)**      | Supported       | Different set of canned ACLs           |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Location**             | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Notification**         | Supported       | See `S3 Notification Compatibility`_   |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Object Versions**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Bucket Info (HEAD)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Request Payment**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Put Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Object ACLs (Get, Put)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object Info (HEAD)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **POST Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Copy Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Multipart Uploads**           | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Object Tagging**              | Supported       | See :ref:`tag_policy` for Policy verbs |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Tagging**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Storage Class**               | Supported       | See :ref:`storage_classes`             |
++---------------------------------+-----------------+----------------------------------------+
+
+Unsupported Header Fields
+-------------------------
+
+The following common request header fields are not supported:
+
++----------------------------+------------+
+| Name                       | Type       |
++============================+============+
+| **x-amz-id-2**             | Response   |
++----------------------------+------------+
+
+.. _Amazon S3 API: http://docs.aws.amazon.com/AmazonS3/latest/API/APIRest.html
+.. _S3 Notification Compatibility: ../s3-notification-compatibility
diff --git a/doc/radosgw/s3/authentication.rst b/doc/radosgw/s3/authentication.rst
new file mode 100644
index 000000000..64747cde2
--- /dev/null
+++ b/doc/radosgw/s3/authentication.rst
@@ -0,0 +1,235 @@
+=========================
+ Authentication and ACLs
+=========================
+
+Requests to the RADOS Gateway (RGW) can be either authenticated or 
+unauthenticated. RGW assumes unauthenticated requests are sent by an anonymous 
+user. RGW supports canned ACLs.
+
+Authentication
+--------------
+Authenticating a request requires including an access key and a Hash-based 
+Message Authentication Code (HMAC) in the request before it is sent to the 
+RGW server. RGW uses an S3-compatible authentication approach. 
+
+::
+
+	HTTP/1.1
+	PUT /buckets/bucket/object.mpeg
+	Host: cname.domain.com
+	Date: Mon, 2 Jan 2012 00:01:01 +0000
+	Content-Encoding: mpeg	
+	Content-Length: 9999999
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+In the foregoing example, replace ``{access-key}`` with the value for your access 
+key ID followed by a colon (``:``). Replace ``{hash-of-header-and-secret}`` with 
+a hash of the header string and the secret corresponding to the access key ID.
+
+To generate the hash of the header string and secret, you must:
+
+#. Get the value of the header string.
+#. Normalize the request header string into canonical form. 
+#. Generate an HMAC using a SHA-1 hashing algorithm.
+   See `RFC 2104`_ and `HMAC`_ for details.
+#. Encode the ``hmac`` result as base-64.
+
+To normalize the header into canonical form: 
+
+#. Get all fields beginning with ``x-amz-``.
+#. Ensure that the fields are all lowercase.
+#. Sort the fields lexicographically. 
+#. Combine multiple instances of the same field name into a 
+   single field and separate the field values with a comma.
+#. Replace white space and line breaks in field values with a single space.
+#. Remove white space before and after colons.
+#. Append a new line after each field.
+#. Merge the fields back into the header.
+
+Replace the ``{hash-of-header-and-secret}`` with the base-64 encoded HMAC string.
+
+Authentication against OpenStack Keystone
+-----------------------------------------
+
+In a radosgw instance that is configured with authentication against
+OpenStack Keystone, it is possible to use Keystone as an authoritative
+source for S3 API authentication. To do so, you must set:
+
+* the ``rgw keystone`` configuration options explained in :doc:`../keystone`,
+* ``rgw s3 auth use keystone = true``.
+
+In addition, a user wishing to use the S3 API must obtain an AWS-style
+access key and secret key. They can do so with the ``openstack ec2
+credentials create`` command::
+
+  $ openstack --os-interface public ec2 credentials create
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+  | Field      | Value                                                                                                                                       |
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+  | access     | c921676aaabbccdeadbeef7e8b0eeb2c                                                                                                            |
+  | links      | {u'self': u'https://auth.example.com:5000/v3/users/7ecbebaffeabbddeadbeefa23267ccbb24/credentials/OS-EC2/c921676aaabbccdeadbeef7e8b0eeb2c'} |
+  | project_id | 5ed51981aab4679851adeadbeef6ebf7                                                                                                            |
+  | secret     | ********************************                                                                                                            |
+  | trust_id   | None                                                                                                                                        |
+  | user_id    | 7ecbebaffeabbddeadbeefa23267cc24                                                                                                            |
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+
+The thus-generated access and secret key can then be used for S3 API
+access to radosgw.
+
+.. note:: Consider that most production radosgw deployments
+          authenticating against OpenStack Keystone are also set up
+          for :doc:`../multitenancy`, for which special
+          considerations apply with respect to S3 signed URLs and
+          public read ACLs.
+
+Access Control Lists (ACLs)
+---------------------------
+
+RGW supports S3-compatible ACL functionality. An ACL is a list of access grants
+that specify which operations a user can perform on a bucket or on an object.
+Each grant has a different meaning when applied to a bucket versus applied to 
+an object:
+
++------------------+--------------------------------------------------------+----------------------------------------------+
+| Permission       | Bucket                                                 | Object                                       |
++==================+========================================================+==============================================+
+| ``READ``         | Grantee can list the objects in the bucket.            | Grantee can read the object.                 |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``WRITE``        | Grantee can write or delete objects in the bucket.     | N/A                                          |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``READ_ACP``     | Grantee can read bucket ACL.                           | Grantee can read the object ACL.             |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``WRITE_ACP``    | Grantee can write bucket ACL.                          | Grantee can write to the object ACL.         |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``FULL_CONTROL`` | Grantee has full permissions for object in the bucket. | Grantee can read or write to the object ACL. |
++------------------+--------------------------------------------------------+----------------------------------------------+
+
+Internally, S3 operations are mapped to ACL permissions thus:
+
++---------------------------------------+---------------+
+| Operation                             | Permission    |
++=======================================+===============+
+| ``s3:GetObject``                      | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectTorrent``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersion``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionTorrent``        | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectTagging``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionTagging``        | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListAllMyBuckets``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucket``                     | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucketMultipartUploads``     | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucketVersions``             | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListMultipartUploadParts``       | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:AbortMultipartUpload``           | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:CreateBucket``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteBucket``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObject``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:s3DeleteObjectVersion``          | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObject``                      | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObjectTagging``               | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObjectVersionTagging``        | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObjectTagging``            | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObjectVersionTagging``     | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:RestoreObject``                  | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:GetAccelerateConfiguration``     | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketAcl``                   | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketCORS``                  | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketLocation``              | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketLogging``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketNotification``          | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketPolicy``                | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketRequestPayment``        | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketTagging``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketVersioning``            | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketWebsite``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetLifecycleConfiguration``      | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetObjectAcl``                   | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionAcl``            | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetReplicationConfiguration``    | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketEncryption``            | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:DeleteBucketPolicy``             | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:DeleteBucketWebsite``            | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:DeleteReplicationConfiguration`` | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutAccelerateConfiguration``     | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketAcl``                   | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketCORS``                  | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketLogging``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketNotification``          | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketPolicy``                | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketRequestPayment``        | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketTagging``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutPutBucketVersioning``         | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketWebsite``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutLifecycleConfiguration``      | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutObjectAcl``                   | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutObjectVersionAcl``            | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutReplicationConfiguration``    | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketEncryption``            | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+
+Some mappings, (e.g. ``s3:CreateBucket`` to ``WRITE``) are not
+applicable to S3 operation, but are required to allow Swift and S3 to
+access the same resources when things like Swift user ACLs are in
+play. This is one of the many reasons that you should use S3 bucket
+policies rather than S3 ACLs when possible.
+
+
+.. _RFC 2104: http://www.ietf.org/rfc/rfc2104.txt
+.. _HMAC: https://en.wikipedia.org/wiki/HMAC
diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst
new file mode 100644
index 000000000..17da3a935
--- /dev/null
+++ b/doc/radosgw/s3/bucketops.rst
@@ -0,0 +1,706 @@
+===================
+ Bucket Operations
+===================
+
+PUT Bucket
+----------
+Creates a new bucket. To create a bucket, you must have a user ID and a valid AWS Access Key ID to authenticate requests. You may not
+create buckets as an anonymous user.
+
+Constraints
+~~~~~~~~~~~
+In general, bucket names should follow domain name constraints.
+
+- Bucket names must be unique.
+- Bucket names cannot be formatted as IP address.
+- Bucket names can be between 3 and 63 characters long.
+- Bucket names must not contain uppercase characters or underscores.
+- Bucket names must start with a lowercase letter or number.
+- Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.). Bucket names can contain lowercase letters, numbers, and hyphens. Each label must start and end with a lowercase letter or a number.
+
+.. note:: The above constraints are relaxed if the option 'rgw_relaxed_s3_bucket_names' is set to true except that the bucket names must still be unique, cannot be formatted as IP address and can contain letters, numbers, periods, dashes and underscores for up to 255 characters long.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket} HTTP/1.1
+    Host: cname.domain.com
+    x-amz-acl: public-read-write
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Parameters
+~~~~~~~~~~
+
+
++---------------+----------------------+-----------------------------------------------------------------------------+------------+
+| Name          | Description          | Valid Values                                                                | Required   |
++===============+======================+=============================================================================+============+
+| ``x-amz-acl`` | Canned ACLs.         | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read`` | No         |
++---------------+----------------------+-----------------------------------------------------------------------------+------------+
+| ``x-amz-bucket-object-lock-enabled`` | Enable object lock on bucket. | ``true``, ``false``                         | No         |
++--------------------------------------+-------------------------------+---------------------------------------------+------------+
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++-------------------------------+-----------+----------------------------------------------------------------+
+| Name                          | Type      | Description                                                    |
++===============================+===========+================================================================+
+| ``CreateBucketConfiguration`` | Container | A container for the bucket configuration.                      |
++-------------------------------+-----------+----------------------------------------------------------------+
+| ``LocationConstraint``        | String    | A zonegroup api name, with optional :ref:`s3_bucket_placement` |
++-------------------------------+-----------+----------------------------------------------------------------+
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If the bucket name is unique, within constraints and unused, the operation will succeed.
+If a bucket with the same name already exists and the user is the bucket owner, the operation will succeed.
+If the bucket name is already in use, the operation will fail.
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``409``       | BucketAlreadyExists   | Bucket already exists under different user's ownership.  |
++---------------+-----------------------+----------------------------------------------------------+
+
+DELETE Bucket
+-------------
+
+Deletes a bucket. You can reuse bucket names following a successful bucket removal.
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket} HTTP/1.1
+    Host: cname.domain.com
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+---------------+------------------+
+| HTTP Status   | Status Code   | Description      |
++===============+===============+==================+
+| ``204``       | No Content    | Bucket removed.  |
++---------------+---------------+------------------+
+
+GET Bucket
+----------
+Returns a list of bucket objects.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?max-keys=25 HTTP/1.1
+    Host: cname.domain.com
+
+Parameters
+~~~~~~~~~~
+
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| Name                | Type      | Description                                                                                     |
++=====================+===========+=================================================================================================+
+| ``prefix``          | String    | Only returns objects that contain the specified prefix.                                         |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``delimiter``       | String    | The delimiter between the prefix and the rest of the object name.                               |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``marker``          | String    | A beginning index for the list of objects returned.                                             |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``max-keys``        | Integer   | The maximum number of keys to return. Default is 1000.                                          |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``allow-unordered`` | Boolean   | Non-standard extension. Allows results to be returned unordered. Cannot be used with delimiter. |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+---------------+--------------------+
+| HTTP Status   | Status Code   | Description        |
++===============+===============+====================+
+| ``200``       | OK            | Buckets retrieved  |
++---------------+---------------+--------------------+
+
+Bucket Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+``GET /{bucket}`` returns a container for buckets with the following fields.
+
++------------------------+-----------+----------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                      |
++========================+===========+==================================================================================+
+| ``ListBucketResult``   | Entity    | The container for the list of objects.                                           |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Name``               | String    | The name of the bucket whose contents will be returned.                          |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Prefix``             | String    | A prefix for the object keys.                                                    |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Marker``             | String    | A beginning index for the list of objects returned.                              |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``MaxKeys``            | Integer   | The maximum number of keys returned.                                             |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Delimiter``          | String    | If set, objects with the same prefix will appear in the ``CommonPrefixes`` list. |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``IsTruncated``        | Boolean   | If ``true``, only a subset of the bucket's contents were returned.               |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``CommonPrefixes``     | Container | If multiple objects contain the same prefix, they will appear in this list.      |
++------------------------+-----------+----------------------------------------------------------------------------------+
+
+Object Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+The ``ListBucketResult`` contains objects, where each object is within a ``Contents`` container.
+
++------------------------+-----------+------------------------------------------+
+| Name                   | Type      | Description                              |
++========================+===========+==========================================+
+| ``Contents``           | Object    | A container for the object.              |
++------------------------+-----------+------------------------------------------+
+| ``Key``                | String    | The object's key.                        |
++------------------------+-----------+------------------------------------------+
+| ``LastModified``       | Date      | The object's last-modified date/time.    |
++------------------------+-----------+------------------------------------------+
+| ``ETag``               | String    | An MD-5 hash of the object. (entity tag) |
++------------------------+-----------+------------------------------------------+
+| ``Size``               | Integer   | The object's size.                       |
++------------------------+-----------+------------------------------------------+
+| ``StorageClass``       | String    | Should always return ``STANDARD``.       |
++------------------------+-----------+------------------------------------------+
+| ``Type``               | String    | ``Appendable`` or ``Normal``.            |
++------------------------+-----------+------------------------------------------+
+
+Get Bucket Location
+-------------------
+Retrieves the bucket's region. The user needs to be the bucket owner
+to call this. A bucket can be constrained to a region by providing
+``LocationConstraint`` during a PUT request.
+
+Syntax
+~~~~~~
+Add the ``location`` subresource to bucket resource as shown below
+
+::
+
+   GET /{bucket}?location HTTP/1.1
+   Host: cname.domain.com
+
+   Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+
++------------------------+-----------+------------------------------------------+
+| Name                   | Type      | Description                              |
++========================+===========+==========================================+
+| ``LocationConstraint`` | String    | The region where bucket resides, empty   |
+|                        |           | string for default region                |
++------------------------+-----------+------------------------------------------+
+
+
+
+Get Bucket ACL
+--------------
+Retrieves the bucket access control list. The user needs to be the bucket
+owner or to have been granted ``READ_ACP`` permission on the bucket.
+
+Syntax
+~~~~~~
+Add the ``acl`` subresource to the bucket request as shown below.
+
+::
+
+    GET /{bucket}?acl HTTP/1.1
+    Host: cname.domain.com
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The bucket owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The bucket owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` bucket.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+PUT Bucket ACL
+--------------
+Sets an access control to an existing bucket. The user needs to be the bucket
+owner or to have been granted ``WRITE_ACP`` permission on the bucket.
+
+Syntax
+~~~~~~
+Add the ``acl`` subresource to the bucket request as shown below.
+
+::
+
+    PUT /{bucket}?acl HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the request.                                                                 |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The bucket owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The bucket owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` bucket.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+List Bucket Multipart Uploads
+-----------------------------
+
+``GET /?uploads`` returns a list of the current in-progress multipart uploads--i.e., the application initiates a multipart upload, but
+the service hasn't completed all the uploads yet.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?uploads HTTP/1.1
+
+Parameters
+~~~~~~~~~~
+
+You may specify parameters for ``GET /{bucket}?uploads``, but none of them are required.
+
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                          |
++========================+===========+======================================================================================+
+| ``prefix``             | String    | Returns in-progress uploads whose keys contains the specified prefix.                |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``delimiter``          | String    | The delimiter between the prefix and the rest of the object name.                    |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``key-marker``         | String    | The beginning marker for the list of uploads.                                        |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``max-keys``           | Integer   | The maximum number of in-progress uploads. The default is 1000.                      |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``max-uploads``        | Integer   | The maximum number of multipart uploads. The range from 1-1000. The default is 1000. |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``upload-id-marker``   | String    | Ignored if ``key-marker`` is not specified. Specifies the ``ID`` of first            |
+|                        |           | upload to list in lexicographical order at or following the ``ID``.                  |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``ListMultipartUploadsResult``          | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ListMultipartUploadsResult.Prefix``   | String      | The prefix specified by the ``prefix`` request parameter (if any).                                       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the bucket contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``KeyMarker``                           | String      | The key marker specified by the ``key-marker`` request parameter (if any).                               |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadIdMarker``                      | String      | The marker specified by the ``upload-id-marker`` request parameter (if any).                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextKeyMarker``                       | String      | The key marker to use in a subsequent request if ``IsTruncated`` is ``true``.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextUploadIdMarker``                  | String      | The upload ID marker to use in a subsequent request if ``IsTruncated`` is ``true``.                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``MaxUploads``                          | Integer     | The max uploads specified by the ``max-uploads`` request parameter.                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Delimiter``                           | String      | If set, objects with the same prefix will appear in the ``CommonPrefixes`` list.                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``IsTruncated``                         | Boolean     | If ``true``, only a subset of the bucket's upload contents were returned.                                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Upload``                              | Container   | A container for ``Key``, ``UploadId``, ``InitiatorOwner``, ``StorageClass``, and ``Initiated`` elements. |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key of the object once the multipart upload is complete.                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ``ID`` that identifies the multipart upload.                                                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiator``                           | Container   | Contains the ``ID`` and ``DisplayName`` of the user who initiated the upload.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``DisplayName``                         | String      | The initiator's display name.                                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ID``                                  | String      | The initiator's ID.                                                                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Owner``                               | Container   | A container for the ``ID`` and ``DisplayName`` of the user who owns the uploaded object.                 |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``StorageClass``                        | String      | The method used to store the resulting object. ``STANDARD`` or ``REDUCED_REDUNDANCY``                    |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiated``                           | Date        | The date and time the user initiated the upload.                                                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``CommonPrefixes``                      | Container   | If multiple objects contain the same prefix, they will appear in this list.                              |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``CommonPrefixes.Prefix``               | String      | The substring of the key after the prefix as defined by the ``prefix`` request parameter.                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+ENABLE/SUSPEND BUCKET VERSIONING
+--------------------------------
+
+``PUT /?versioning`` This subresource set the versioning state of an existing bucket. To set the versioning state, you must be the bucket owner.
+
+You can set the versioning state with one of the following values:
+
+- Enabled : Enables versioning for the objects in the bucket, All objects added to the bucket receive a unique version ID.
+- Suspended : Disables versioning for the objects in the bucket, All objects added to the bucket receive the version ID null.
+
+If the versioning state has never been set on a bucket, it has no versioning state; a GET versioning request does not return a versioning state value.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT  /{bucket}?versioning  HTTP/1.1
+
+REQUEST ENTITIES
+~~~~~~~~~~~~~~~~
+
++-----------------------------+-----------+---------------------------------------------------------------------------+
+| Name                        | Type      | Description                                                               |
++=============================+===========+===========================================================================+
+| ``VersioningConfiguration`` | Container | A container for the request.                                              |
++-----------------------------+-----------+---------------------------------------------------------------------------+
+| ``Status``                  | String    | Sets the versioning state of the bucket.  Valid Values: Suspended/Enabled |
++-----------------------------+-----------+---------------------------------------------------------------------------+
+
+PUT BUCKET OBJECT LOCK
+--------------------------------
+
+Places an Object Lock configuration on the specified bucket. The rule specified in the Object Lock configuration will be
+applied by default to every new object placed in the specified bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}?object-lock HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| Name                        | Type        | Description                                                                            | Required |
++=============================+=============+========================================================================================+==========+
+| ``ObjectLockConfiguration`` | Container   | A container for the request.                                                           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``ObjectLockEnabled``       | String      | Indicates whether this bucket has an Object Lock configuration enabled.                |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Rule``                    | Container   | The Object Lock rule in place for the specified bucket.                                |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``DefaultRetention``        | Container   | The default retention period applied to new objects placed in the specified bucket.    |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Mode``                    | String      | The default Object Lock retention mode. Valid Values:  GOVERNANCE/COMPLIANCE           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Days``                    | Integer     | The number of days specified for the default retention period.                         |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Years``                   | Integer     | The number of years specified for the default retention period.                        |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If the bucket object lock is not enabled when creating the bucket, the operation will fail.
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``400``       | MalformedXML          | The XML is not well-formed                               |
++---------------+-----------------------+----------------------------------------------------------+
+| ``409``       | InvalidBucketState    | The bucket object lock is not enabled                    |
++---------------+-----------------------+----------------------------------------------------------+
+
+GET BUCKET OBJECT LOCK
+--------------------------------
+
+Gets the Object Lock configuration for a bucket. The rule specified in the Object Lock configuration will be applied by
+default to every new object placed in the specified bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?object-lock HTTP/1.1
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| Name                        | Type        | Description                                                                            | Required |
++=============================+=============+========================================================================================+==========+
+| ``ObjectLockConfiguration`` | Container   | A container for the request.                                                           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``ObjectLockEnabled``       | String      | Indicates whether this bucket has an Object Lock configuration enabled.                |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Rule``                    | Container   | The Object Lock rule in place for the specified bucket.                                |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``DefaultRetention``        | Container   | The default retention period applied to new objects placed in the specified bucket.    |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Mode``                    | String      | The default Object Lock retention mode. Valid Values:  GOVERNANCE/COMPLIANCE           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Days``                    | Integer     | The number of days specified for the default retention period.                         |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Years``                   | Integer     | The number of years specified for the default retention period.                        |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+
+Create Notification
+-------------------
+
+Create a publisher for a specific bucket into a topic.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}?notification HTTP/1.1
+
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
+Parameters are XML encoded in the body of the request, in the following format:
+
+::
+
+   <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+       <TopicConfiguration>
+           <Id></Id>
+           <Topic></Topic>
+           <Event></Event>
+           <Filter>
+               <S3Key>
+                   <FilterRule>
+                       <Name></Name>
+                       <Value></Value>
+                   </FilterRule>
+        	    </S3Key>
+                <S3Metadata>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Metadata>
+                <S3Tags>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Tags>
+            </Filter>
+       </TopicConfiguration>
+   </NotificationConfiguration>
+
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| Name                          | Type      | Description                                                                          | Required |
++===============================+===========+======================================================================================+==========+
+| ``NotificationConfiguration`` | Container | Holding list of ``TopicConfiguration`` entities                                      | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``TopicConfiguration``        | Container | Holding ``Id``, ``Topic`` and list of ``Event`` entities                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Id``                        | String    | Name of the notification                                                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Topic``                     | String    | Topic ARN. Topic must be created beforehand                                          | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Event``                     | String    | List of supported events see: `S3 Notification Compatibility`_.  Multiple ``Event``  | No       |
+|                               |           | entities can be used. If omitted, all events are handled                             |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Filter``                    | Container | Holding ``S3Key``, ``S3Metadata`` and ``S3Tags`` entities                            | No       |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Key``                     | Container | Holding a list of ``FilterRule`` entities, for filtering based on object key.        | No       |
+|                               |           | At most, 3 entities may be in the list, with ``Name`` be ``prefix``, ``suffix`` or   |          |
+|                               |           | ``regex``. All filter rules in the list must match for the filter to match.          |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Metadata``                | Container | Holding a list of ``FilterRule`` entities, for filtering based on object metadata.   | No       |
+|                               |           | All filter rules in the list must match the metadata defined on the object. However, |          |
+|                               |           | the object still match if it has other metadata entries not listed in the filter.    |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Tags``                    | Container | Holding a list of ``FilterRule`` entities, for filtering based on object tags.       | No       |
+|                               |           | All filter rules in the list must match the tags defined on the object. However,     |          |
+|                               |           | the object still match it it has other tags not listed in the filter.                |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Key.FilterRule``          | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would  be: ``prefix``, ``suffix``  | Yes      |
+|                               |           | or ``regex``. The ``Value`` would hold the key prefix, key suffix or a regular       |          |
+|                               |           | expression for matching the key, accordingly.                                        |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Metadata.FilterRule``     | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would be the name of the metadata  | Yes      |
+|                               |           | attribute (e.g. ``x-amz-meta-xxx``). The ``Value`` would be the expected value for   |          | 
+|                               |           | this attribute.                                                                      |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Tags.FilterRule``         | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would be the tag key,              |  Yes     |
+|                               |           | and ``Value`` would be the tag value.                                                |          | 
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``400``       | MalformedXML          | The XML is not well-formed                               |
++---------------+-----------------------+----------------------------------------------------------+
+| ``400``       | InvalidArgument       | Missing Id; Missing/Invalid Topic ARN; Invalid Event     |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchKey             | The topic does not exist                                 |
++---------------+-----------------------+----------------------------------------------------------+
+
+
+Delete Notification
+-------------------
+
+Delete a specific, or all, notifications from a bucket.
+
+.. note:: 
+
+    - Notification deletion is an extension to the S3 notification API
+    - When the bucket is deleted, any notification defined on it is also deleted 
+    - Deleting an unknown notification (e.g. double delete) is not considered an error
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket}?notification[=<notification-id>] HTTP/1.1
+
+
+Parameters
+~~~~~~~~~~
+
++------------------------+-----------+----------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                            |
++========================+===========+========================================================================================+
+| ``notification-id``    | String    | Name of the notification. If not provided, all notifications on the bucket are deleted |
++------------------------+-----------+----------------------------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+
+Get/List Notification
+---------------------
+
+Get a specific notification, or list all notifications configured on a bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?notification[=<notification-id>] HTTP/1.1 
+
+
+Parameters
+~~~~~~~~~~
+
++------------------------+-----------+----------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                            |
++========================+===========+========================================================================================+
+| ``notification-id``    | String    | Name of the notification. If not provided, all notifications on the bucket are listed  |
++------------------------+-----------+----------------------------------------------------------------------------------------+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+Response is XML encoded in the body of the request, in the following format:
+
+::
+
+   <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+       <TopicConfiguration>
+           <Id></Id>
+           <Topic></Topic>
+           <Event></Event>
+           <Filter>
+               <S3Key>
+                   <FilterRule>
+                       <Name></Name>
+                       <Value></Value>
+                   </FilterRule>
+        	    </S3Key>
+                <S3Metadata>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Metadata>
+                <S3Tags>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Tags>
+            </Filter>
+       </TopicConfiguration>
+   </NotificationConfiguration>
+
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| Name                          | Type      | Description                                                                          | Required |
++===============================+===========+======================================================================================+==========+
+| ``NotificationConfiguration`` | Container | Holding list of ``TopicConfiguration`` entities                                      | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``TopicConfiguration``        | Container | Holding ``Id``, ``Topic`` and list of ``Event`` entities                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Id``                        | String    | Name of the notification                                                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Topic``                     | String    | Topic ARN                                                                            | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Event``                     | String    | Handled event. Multiple ``Event`` entities may exist                                 | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Filter``                    | Container | Holding the filters configured for this notification                                 | No       |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchKey             | The notification does not exist (if provided)            |
++---------------+-----------------------+----------------------------------------------------------+
+
+.. _S3 Notification Compatibility: ../../s3-notification-compatibility
diff --git a/doc/radosgw/s3/commons.rst b/doc/radosgw/s3/commons.rst
new file mode 100644
index 000000000..4b9b4a040
--- /dev/null
+++ b/doc/radosgw/s3/commons.rst
@@ -0,0 +1,113 @@
+=================
+ Common Entities
+=================
+
+.. toctree::
+   :maxdepth: -1
+
+Bucket and Host Name
+--------------------
+There are two different modes of accessing the buckets. The first (preferred) method
+identifies the bucket as the top-level directory in the URI. ::
+
+	GET /mybucket HTTP/1.1
+	Host: cname.domain.com
+
+The second method identifies the bucket via a virtual bucket host name. For example::
+
+	GET / HTTP/1.1
+	Host: mybucket.cname.domain.com
+
+To configure virtual hosted buckets, you can either set ``rgw_dns_name = cname.domain.com`` in ceph.conf, or add ``cname.domain.com`` to the list of ``hostnames`` in your zonegroup configuration. See `Ceph Object Gateway - Multisite Configuration`_ for more on zonegroups.
+
+.. tip:: We prefer the first method, because the second method requires expensive domain certification and DNS wild cards.
+
+.. tip:: You can define multiple hostname directly with the :confval:`rgw_dns_name` parameter.
+
+Common Request Headers
+----------------------
+
++--------------------+------------------------------------------+
+| Request Header     | Description                              |
++====================+==========================================+
+| ``CONTENT_LENGTH`` | Length of the request body.              |
++--------------------+------------------------------------------+
+| ``DATE``           | Request time and date (in UTC).          |
++--------------------+------------------------------------------+
+| ``HOST``           | The name of the host server.             |
++--------------------+------------------------------------------+
+| ``AUTHORIZATION``  | Authorization token.                     |
++--------------------+------------------------------------------+
+
+Common Response Status
+----------------------
+
++---------------+-----------------------------------+
+| HTTP Status   | Response Code                     |
++===============+===================================+
+| ``100``       | Continue                          |
++---------------+-----------------------------------+
+| ``200``       | Success                           |
++---------------+-----------------------------------+
+| ``201``       | Created                           |
++---------------+-----------------------------------+
+| ``202``       | Accepted                          |
++---------------+-----------------------------------+
+| ``204``       | NoContent                         |
++---------------+-----------------------------------+
+| ``206``       | Partial content                   |
++---------------+-----------------------------------+
+| ``304``       | NotModified                       |
++---------------+-----------------------------------+
+| ``400``       | InvalidArgument                   |
++---------------+-----------------------------------+
+| ``400``       | InvalidDigest                     |
++---------------+-----------------------------------+
+| ``400``       | BadDigest                         |
++---------------+-----------------------------------+
+| ``400``       | InvalidBucketName                 |
++---------------+-----------------------------------+
+| ``400``       | InvalidObjectName                 |
++---------------+-----------------------------------+
+| ``400``       | UnresolvableGrantByEmailAddress   |
++---------------+-----------------------------------+
+| ``400``       | InvalidPart                       |
++---------------+-----------------------------------+
+| ``400``       | InvalidPartOrder                  |
++---------------+-----------------------------------+
+| ``400``       | RequestTimeout                    |
++---------------+-----------------------------------+
+| ``400``       | EntityTooLarge                    |
++---------------+-----------------------------------+
+| ``403``       | AccessDenied                      |
++---------------+-----------------------------------+
+| ``403``       | UserSuspended                     |
++---------------+-----------------------------------+
+| ``403``       | RequestTimeTooSkewed              |
++---------------+-----------------------------------+
+| ``404``       | NoSuchKey                         |
++---------------+-----------------------------------+
+| ``404``       | NoSuchBucket                      |
++---------------+-----------------------------------+
+| ``404``       | NoSuchUpload                      |
++---------------+-----------------------------------+
+| ``405``       | MethodNotAllowed                  |
++---------------+-----------------------------------+
+| ``408``       | RequestTimeout                    |
++---------------+-----------------------------------+
+| ``409``       | BucketAlreadyExists               |
++---------------+-----------------------------------+
+| ``409``       | BucketNotEmpty                    |
++---------------+-----------------------------------+
+| ``411``       | MissingContentLength              |
++---------------+-----------------------------------+
+| ``412``       | PreconditionFailed                |
++---------------+-----------------------------------+
+| ``416``       | InvalidRange                      |
++---------------+-----------------------------------+
+| ``422``       | UnprocessableEntity               |
++---------------+-----------------------------------+
+| ``500``       | InternalError                     |
++---------------+-----------------------------------+
+
+.. _`Ceph Object Gateway - Multisite Configuration`: ../../multisite
diff --git a/doc/radosgw/s3/cpp.rst b/doc/radosgw/s3/cpp.rst
new file mode 100644
index 000000000..089c9c53a
--- /dev/null
+++ b/doc/radosgw/s3/cpp.rst
@@ -0,0 +1,337 @@
+.. _cpp:
+
+C++ S3 Examples
+===============
+
+Setup
+-----
+
+The following contains includes and globals that will be used in later examples:
+
+.. code-block:: cpp
+
+	#include "libs3.h"
+	#include <stdlib.h>
+	#include <iostream>
+	#include <fstream>
+
+	const char access_key[] = "ACCESS_KEY";
+	const char secret_key[] = "SECRET_KEY";
+	const char host[] = "HOST";
+	const char sample_bucket[] = "sample_bucket";
+	const char sample_key[] = "hello.txt";
+	const char sample_file[] = "resource/hello.txt";
+	const char *security_token = NULL;
+	const char *auth_region = NULL;
+
+	S3BucketContext bucketContext =
+	{
+		host,
+		sample_bucket,
+		S3ProtocolHTTP,
+		S3UriStylePath,
+		access_key,
+		secret_key,
+		security_token,
+		auth_region
+	};
+
+	S3Status responsePropertiesCallback(
+			const S3ResponseProperties *properties,
+			void *callbackData)
+	{
+		return S3StatusOK;
+	}
+
+	static void responseCompleteCallback(
+			S3Status status,
+			const S3ErrorDetails *error,
+			void *callbackData)
+	{
+		return;
+	}
+
+	S3ResponseHandler responseHandler =
+	{
+		&responsePropertiesCallback,
+		&responseCompleteCallback
+	};
+
+
+Creating (and Closing) a Connection
+-----------------------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: cpp
+
+	S3_initialize("s3", S3_INIT_ALL, host);
+	// Do stuff...
+	S3_deinitialize();
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name, owner ID, and display name
+for each bucket.
+
+.. code-block:: cpp
+
+	static S3Status listServiceCallback(
+			const char *ownerId,
+			const char *ownerDisplayName,
+			const char *bucketName,
+			int64_t creationDate, void *callbackData)
+	{
+		bool *header_printed = (bool*) callbackData;
+		if (!*header_printed) {
+			*header_printed = true;
+			printf("%-22s", "       Bucket");
+			printf("  %-20s  %-12s", "     Owner ID", "Display Name");
+			printf("\n");
+			printf("----------------------");
+			printf("  --------------------" "  ------------");
+			printf("\n");
+		}
+
+		printf("%-22s", bucketName);
+		printf("  %-20s  %-12s", ownerId ? ownerId : "", ownerDisplayName ? ownerDisplayName : "");
+		printf("\n");
+
+		return S3StatusOK;
+	}
+
+	S3ListServiceHandler listServiceHandler =
+	{
+		responseHandler,
+		&listServiceCallback
+	};
+	bool header_printed = false;
+	S3_list_service(S3ProtocolHTTP, access_key, secret_key, security_token, host, 
+                    auth_region, NULL, 0, &listServiceHandler, &header_printed);
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket.
+
+.. code-block:: cpp
+
+	S3_create_bucket(S3ProtocolHTTP, access_key, secret_key, NULL, host, sample_bucket, S3CannedAclPrivate, NULL, NULL, &responseHandler, NULL);
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and
+last modified date.
+
+.. code-block:: cpp
+
+	static S3Status listBucketCallback(
+			int isTruncated,
+			const char *nextMarker,
+			int contentsCount,
+			const S3ListBucketContent *contents,
+			int commonPrefixesCount,
+			const char **commonPrefixes,
+			void *callbackData)
+	{
+		printf("%-22s",	"      Object Name");
+		printf("  %-5s  %-20s", "Size", "   Last Modified");
+		printf("\n");
+		printf("----------------------");
+		printf("  -----" "  --------------------");
+		printf("\n");
+
+	    for (int i = 0; i < contentsCount; i++) {
+		char timebuf[256];
+			char sizebuf[16];
+		const S3ListBucketContent *content = &(contents[i]);
+			time_t t = (time_t) content->lastModified;
+
+			strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
+			sprintf(sizebuf, "%5llu", (unsigned long long) content->size);
+			printf("%-22s  %s  %s\n", content->key, sizebuf, timebuf);
+	    }
+
+	    return S3StatusOK;
+	}
+
+	S3ListBucketHandler listBucketHandler =
+	{
+		responseHandler,
+		&listBucketCallback
+	};
+	S3_list_bucket(&bucketContext, NULL, NULL, NULL, 0, NULL, 0, &listBucketHandler, NULL);
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: cpp
+
+	S3_delete_bucket(S3ProtocolHTTP, S3UriStylePath, access_key, secret_key, 0, host, sample_bucket, NULL, NULL, 0, &responseHandler, NULL);
+
+
+Creating an Object (from a file)
+--------------------------------
+
+This creates a file ``hello.txt``.
+
+.. code-block:: cpp
+
+	#include <sys/stat.h>
+	typedef struct put_object_callback_data
+	{
+	    FILE *infile;
+	    uint64_t contentLength;
+	} put_object_callback_data;
+
+
+	static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData)
+	{
+	    put_object_callback_data *data = (put_object_callback_data *) callbackData;
+
+	    int ret = 0;
+
+	    if (data->contentLength) {
+		int toRead = ((data->contentLength > (unsigned) bufferSize) ? (unsigned) bufferSize : data->contentLength);
+			ret = fread(buffer, 1, toRead, data->infile);
+	    }
+	    data->contentLength -= ret;
+	    return ret;
+	}
+
+	put_object_callback_data data;
+	struct stat statbuf;
+	if (stat(sample_file, &statbuf) == -1) {
+		fprintf(stderr, "\nERROR: Failed to stat file %s: ", sample_file);
+		perror(0);
+		exit(-1);
+	}
+
+	int contentLength = statbuf.st_size;
+	data.contentLength = contentLength;
+
+	if (!(data.infile = fopen(sample_file, "r"))) {
+		fprintf(stderr, "\nERROR: Failed to open input file %s: ", sample_file);
+		perror(0);
+		exit(-1);
+	}
+
+	S3PutObjectHandler putObjectHandler =
+	{
+		responseHandler,
+		&putObjectDataCallback
+	};
+
+	S3_put_object(&bucketContext, sample_key, contentLength, NULL, NULL, 0, &putObjectHandler, &data);
+	fclose(data.infile);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads a file and prints the contents.
+
+.. code-block:: cpp
+
+	static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData)
+	{
+		FILE *outfile = (FILE *) callbackData;
+		size_t wrote = fwrite(buffer, 1, bufferSize, outfile);
+		return ((wrote < (size_t) bufferSize) ? S3StatusAbortedByCallback : S3StatusOK);
+	}
+
+	S3GetObjectHandler getObjectHandler =
+	{
+		responseHandler,
+		&getObjectDataCallback
+	};
+	FILE *outfile = stdout;
+	S3_get_object(&bucketContext, sample_key, NULL, 0, 0, NULL, 0, &getObjectHandler, outfile);
+
+
+Delete an Object
+----------------
+
+This deletes an object.
+
+.. code-block:: cpp
+
+	S3ResponseHandler deleteResponseHandler =
+	{
+		0,
+		&responseCompleteCallback
+	};
+	S3_delete_object(&bucketContext, sample_key, 0, 0, &deleteResponseHandler, 0);
+
+
+Change an Object's ACL
+----------------------
+
+This changes an object's ACL to grant full control to another user.
+
+
+.. code-block:: cpp
+
+	#include <string.h>
+	char ownerId[] = "owner";
+	char ownerDisplayName[] = "owner";
+	char granteeId[] = "grantee";
+	char granteeDisplayName[] = "grantee";
+
+	S3AclGrant grants[] = {
+		{
+			S3GranteeTypeCanonicalUser,
+			{{}},
+			S3PermissionFullControl
+		},
+		{
+			S3GranteeTypeCanonicalUser,
+			{{}},
+			S3PermissionReadACP
+		},
+		{
+			S3GranteeTypeAllUsers,
+			{{}},
+			S3PermissionRead
+		}
+	};
+
+	strncpy(grants[0].grantee.canonicalUser.id, ownerId, S3_MAX_GRANTEE_USER_ID_SIZE);
+	strncpy(grants[0].grantee.canonicalUser.displayName, ownerDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+
+	strncpy(grants[1].grantee.canonicalUser.id, granteeId, S3_MAX_GRANTEE_USER_ID_SIZE);
+	strncpy(grants[1].grantee.canonicalUser.displayName, granteeDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+
+	S3_set_acl(&bucketContext, sample_key, ownerId, ownerDisplayName, 3, grants, 0, &responseHandler, 0);
+
+
+Generate Object Download URL (signed)
+-------------------------------------
+
+This generates a signed download URL that will be valid for 5 minutes.
+
+.. code-block:: cpp
+
+	#include <time.h>
+	char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE];
+	int64_t expires = time(NULL) + 60 * 5; // Current time + 5 minutes
+
+	S3_generate_authenticated_query_string(buffer, &bucketContext, sample_key, expires, NULL, "GET");
+
diff --git a/doc/radosgw/s3/csharp.rst b/doc/radosgw/s3/csharp.rst
new file mode 100644
index 000000000..af1c6e4b5
--- /dev/null
+++ b/doc/radosgw/s3/csharp.rst
@@ -0,0 +1,199 @@
+.. _csharp:
+
+C# S3 Examples
+==============
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: csharp
+
+	using System;
+	using Amazon;
+	using Amazon.S3;
+	using Amazon.S3.Model;
+
+	string accessKey = "put your access key here!";
+	string secretKey = "put your secret key here!";
+
+	AmazonS3Config config = new AmazonS3Config();
+	config.ServiceURL = "objects.dreamhost.com";
+
+	AmazonS3Client s3Client = new AmazonS3Client(
+		accessKey,
+		secretKey,
+		config
+		);
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: csharp
+
+	ListBucketsResponse response = client.ListBuckets();
+	foreach (S3Bucket b in response.Buckets)
+	{
+		Console.WriteLine("{0}\t{1}", b.BucketName, b.CreationDate);
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: csharp
+
+	PutBucketRequest request = new PutBucketRequest();
+	request.BucketName = "my-new-bucket";
+	client.PutBucket(request);
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: csharp
+
+	ListObjectsRequest request = new ListObjectsRequest();
+	request.BucketName = "my-new-bucket";
+	ListObjectsResponse response = client.ListObjects(request);
+	foreach (S3Object o in response.S3Objects)
+	{
+		Console.WriteLine("{0}\t{1}\t{2}", o.Key, o.Size, o.LastModified);
+	}
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: csharp
+
+	DeleteBucketRequest request = new DeleteBucketRequest();
+	request.BucketName = "my-new-bucket";
+	client.DeleteBucket(request);
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: csharp
+
+	PutObjectRequest request = new PutObjectRequest();
+	request.BucketName  = "my-new-bucket";
+	request.Key         = "hello.txt";
+	request.ContentType = "text/plain";
+	request.ContentBody = "Hello World!";
+	client.PutObject(request);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: csharp
+
+	PutACLRequest request = new PutACLRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "hello.txt";
+	request.CannedACL  = S3CannedACL.PublicRead;
+	client.PutACL(request);
+
+	PutACLRequest request2 = new PutACLRequest();
+	request2.BucketName = "my-new-bucket";
+	request2.Key        = "secret_plans.txt";
+	request2.CannedACL  = S3CannedACL.Private;
+	client.PutACL(request2);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``C:\Users\larry\Documents``
+
+.. code-block:: csharp
+
+	GetObjectRequest request = new GetObjectRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "perl_poetry.pdf";
+	GetObjectResponse response = client.GetObject(request);
+	response.WriteResponseStreamToFile("C:\\Users\\larry\\Documents\\perl_poetry.pdf");
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: csharp
+
+	DeleteObjectRequest request = new DeleteObjectRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "goodbye.txt";
+	client.DeleteObject(request);
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+
+   The C# S3 Library does not have a method for generating unsigned
+   URLs, so the following example only shows generating signed URLs.
+
+.. code-block:: csharp
+
+	GetPreSignedUrlRequest request = new GetPreSignedUrlRequest();
+	request.BucketName = "my-bucket-name";
+	request.Key        = "secret_plans.txt";
+	request.Expires    = DateTime.Now.AddHours(1);
+	request.Protocol   = Protocol.HTTP;
+	string url = client.GetPreSignedURL(request);
+	Console.WriteLine(url);
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
diff --git a/doc/radosgw/s3/java.rst b/doc/radosgw/s3/java.rst
new file mode 100644
index 000000000..057c09c2c
--- /dev/null
+++ b/doc/radosgw/s3/java.rst
@@ -0,0 +1,212 @@
+.. _java:
+
+Java S3 Examples
+================
+
+Setup
+-----
+
+The following examples may require some or all of the following java
+classes to be imported:
+
+.. code-block:: java
+
+	import java.io.ByteArrayInputStream;
+	import java.io.File;
+	import java.util.List;
+	import com.amazonaws.auth.AWSCredentials;
+	import com.amazonaws.auth.BasicAWSCredentials;
+	import com.amazonaws.util.StringUtils;
+	import com.amazonaws.services.s3.AmazonS3;
+	import com.amazonaws.services.s3.AmazonS3Client;
+	import com.amazonaws.services.s3.model.Bucket;
+	import com.amazonaws.services.s3.model.CannedAccessControlList;
+	import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+	import com.amazonaws.services.s3.model.GetObjectRequest;
+	import com.amazonaws.services.s3.model.ObjectListing;
+	import com.amazonaws.services.s3.model.ObjectMetadata;
+	import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+
+If you are just testing the Ceph Object Storage services, consider
+using HTTP protocol instead of HTTPS protocol. 
+
+First, import the ``ClientConfiguration`` and ``Protocol`` classes. 
+
+.. code-block:: java
+
+	import com.amazonaws.ClientConfiguration;
+	import com.amazonaws.Protocol;
+
+
+Then, define the client configuration, and add the client configuration
+as an argument for the S3 client.
+
+.. code-block:: java
+
+	AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+			 	
+	ClientConfiguration clientConfig = new ClientConfiguration();
+	clientConfig.setProtocol(Protocol.HTTP);
+			
+	AmazonS3 conn = new AmazonS3Client(credentials, clientConfig);
+	conn.setEndpoint("endpoint.com");
+
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: java
+
+	String accessKey = "insert your access key here!";
+	String secretKey = "insert your secret key here!";
+
+	AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+	AmazonS3 conn = new AmazonS3Client(credentials);
+	conn.setEndpoint("objects.dreamhost.com");
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: java
+
+	List<Bucket> buckets = conn.listBuckets();
+	for (Bucket bucket : buckets) {
+		System.out.println(bucket.getName() + "\t" +
+			StringUtils.fromDate(bucket.getCreationDate()));
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: java
+
+	Bucket bucket = conn.createBucket("my-new-bucket");
+
+
+Listing a Bucket's Content
+--------------------------
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: java
+
+	ObjectListing objects = conn.listObjects(bucket.getName());
+	do {
+		for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
+			System.out.println(objectSummary.getKey() + "\t" +
+				objectSummary.getSize() + "\t" +
+				StringUtils.fromDate(objectSummary.getLastModified()));
+		}
+		objects = conn.listNextBatchOfObjects(objects);
+	} while (objects.isTruncated());
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: java
+
+	conn.deleteBucket(bucket.getName());
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+.. attention::
+   not available
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: java
+
+	ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes());
+	conn.putObject(bucket.getName(), "hello.txt", input, new ObjectMetadata());
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: java
+
+	conn.setObjectAcl(bucket.getName(), "hello.txt", CannedAccessControlList.PublicRead);
+	conn.setObjectAcl(bucket.getName(), "secret_plans.txt", CannedAccessControlList.Private);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents``
+
+.. code-block:: java
+
+	conn.getObject(
+		new GetObjectRequest(bucket.getName(), "perl_poetry.pdf"),
+		new File("/home/larry/documents/perl_poetry.pdf")
+	);
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: java
+
+	conn.deleteObject(bucket.getName(), "goodbye.txt");
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+   The java library does not have a method for generating unsigned
+   URLs, so the example below just generates a signed URL.
+
+.. code-block:: java
+
+	GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket.getName(), "secret_plans.txt");
+	System.out.println(conn.generatePresignedUrl(request));
+
+The output will look something like this::
+
+   https://my-bucket-name.objects.dreamhost.com/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
diff --git a/doc/radosgw/s3/objectops.rst b/doc/radosgw/s3/objectops.rst
new file mode 100644
index 000000000..2ac52607f
--- /dev/null
+++ b/doc/radosgw/s3/objectops.rst
@@ -0,0 +1,558 @@
+Object Operations
+=================
+
+Put Object
+----------
+Adds an object to a bucket. You must have write permissions on the bucket to perform this operation.
+
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+
+Copy Object
+-----------
+To copy an object, use ``PUT`` and specify a destination bucket and the object name.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{dest-bucket}/{dest-object} HTTP/1.1
+    x-amz-copy-source: {source-bucket}/{source-object}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| Name                                 | Description                                     | Valid Values           | Required   |
++======================================+=================================================+========================+============+
+| **x-amz-copy-source**                | The source bucket name + object name.           | {bucket}/{obj}         | Yes        |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-acl**                        | A canned ACL.                                   | ``private``,           | No         |
+|                                      |                                                 | ``public-read``,       |            |
+|                                      |                                                 | ``public-read-write``, |            |
+|                                      |                                                 | ``authenticated-read`` |            |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-modified-since**     |  Copies only if modified since the timestamp.   |   Timestamp            | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-unmodified-since**   |  Copies only if unmodified since the timestamp. |   Timestamp            | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-match**              |  Copies only if object ETag matches ETag.       |   Entity Tag           | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-none-match**         |  Copies only if object ETag doesn't match.      |   Entity Tag           | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++------------------------+-------------+-----------------------------------------------+
+| Name                   | Type        | Description                                   |
++========================+=============+===============================================+
+| **CopyObjectResult**   | Container   |  A container for the response elements.       |
++------------------------+-------------+-----------------------------------------------+
+| **LastModified**       | Date        |  The last modified date of the source object. |
++------------------------+-------------+-----------------------------------------------+
+| **Etag**               | String      |  The ETag of the new object.                  |
++------------------------+-------------+-----------------------------------------------+
+
+Remove Object
+-------------
+
+Removes an object. Requires WRITE permission set on the containing bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket}/{object} HTTP/1.1
+
+
+
+Get Object
+----------
+Retrieves an object from a bucket within RADOS. 
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| Name                      | Description                                    | Valid Values                   | Required   |
++===========================+================================================+================================+============+
+| **range**                 | The range of the object to retrieve.           | Range: bytes=beginbyte-endbyte | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-modified-since**     | Gets only if modified since the timestamp.     | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-unmodified-since**   | Gets only if not modified since the timestamp. | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-match**              | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-none-match**         | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
++-------------------+--------------------------------------------------------------------------------------------+
+| Name              | Description                                                                                |
++===================+============================================================================================+
+| **Content-Range** | Data range, will only be returned if the range header field was specified in the request   |
++-------------------+--------------------------------------------------------------------------------------------+
+
+Get Object Info
+---------------
+
+Returns information about object. This request will return the same
+header information as with the Get Object request, but will include
+the metadata only, not the object data payload.
+
+Syntax
+~~~~~~
+
+::
+
+    HEAD /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| Name                      | Description                                    | Valid Values                   | Required   |
++===========================+================================================+================================+============+
+| **range**                 | The range of the object to retrieve.           | Range: bytes=beginbyte-endbyte | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-modified-since**     | Gets only if modified since the timestamp.     | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-unmodified-since**   | Gets only if not modified since the timestamp. | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-match**              | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-none-match**         | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+
+Get Object ACL
+--------------
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?acl HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the object owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The object owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The object owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` object.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+
+
+Set Object ACL
+--------------
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?acl
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the object owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The object owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The object owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` object.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+
+
+Initiate Multi-part Upload
+--------------------------
+
+Initiate a multi-part upload process.
+
+Syntax
+~~~~~~
+
+::
+
+    POST /{bucket}/{object}?uploads
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``InitiatedMultipartUploadsResult``     | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the object contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key specified by the ``key`` request parameter (if any).                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ID specified by the ``upload-id`` request parameter identifying the multipart upload (if any).       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+
+Multipart Upload Part
+---------------------
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?partNumber=&uploadId= HTTP/1.1
+
+HTTP Response
+~~~~~~~~~~~~~
+
+The following HTTP response may be returned:
+
++---------------+----------------+--------------------------------------------------------------------------+
+| HTTP Status   | Status Code    | Description                                                              |
++===============+================+==========================================================================+
+| **404**       | NoSuchUpload   | Specified upload-id does not match any initiated upload on this object   |
++---------------+----------------+--------------------------------------------------------------------------+
+
+List Multipart Upload Parts
+---------------------------
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?uploadId=123 HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``ListPartsResult``                     | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the object contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key specified by the ``key`` request parameter (if any).                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ID specified by the ``upload-id`` request parameter identifying the multipart upload (if any).       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiator``                           | Container   | Contains the ``ID`` and ``DisplayName`` of the user who initiated the upload.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ID``                                  | String      | The initiator's ID.                                                                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``DisplayName``                         | String      | The initiator's display name.                                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Owner``                               | Container   | A container for the ``ID`` and ``DisplayName`` of the user who owns the uploaded object.                 |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``StorageClass``                        | String      | The method used to store the resulting object. ``STANDARD`` or ``REDUCED_REDUNDANCY``                    |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``PartNumberMarker``                    | String      | The part marker to use in a subsequent request if ``IsTruncated`` is ``true``. Precedes the list.        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextPartNumberMarker``                | String      | The next part marker to use in a subsequent request if ``IsTruncated`` is ``true``. The end of the list. |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``MaxParts``                            | Integer     | The max parts allowed in the response as specified by the ``max-parts`` request parameter.               |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``IsTruncated``                         | Boolean     | If ``true``, only a subset of the object's upload contents were returned.                                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Part``                                | Container   | A container for ``LastModified``, ``PartNumber``, ``ETag`` and ``Size`` elements.                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``LastModified``                        | Date        | Date and time at which the part was uploaded.                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``PartNumber``                          | Integer     | The identification number of the part.                                                                   |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ETag``                                | String      | The part's entity tag.                                                                                   |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Size``                                | Integer     | The size of the uploaded part.                                                                           |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+
+
+Complete Multipart Upload
+-------------------------
+Assembles uploaded parts and creates a new object, thereby completing a multipart upload.
+
+Syntax
+~~~~~~
+
+::
+
+    POST /{bucket}/{object}?uploadId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| Name                             | Type        | Description                                         | Required |
++==================================+=============+=====================================================+==========+
+| ``CompleteMultipartUpload``      | Container   | A container consisting of one or more parts.        | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``Part``                         | Container   | A container for the ``PartNumber`` and ``ETag``.    | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``PartNumber``                   | Integer     | The identifier of the part.                         | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``ETag``                         | String      | The part's entity tag.                              | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-------------------------------------+-------------+-------------------------------------------------------+
+| Name                                | Type        | Description                                           |
++=====================================+=============+=======================================================+
+| **CompleteMultipartUploadResult**   | Container   | A container for the response.                         |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Location**                        | URI         | The resource identifier (path) of the new object.     |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Bucket**                          | String      | The name of the bucket that contains the new object.  |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Key**                             | String      | The object's key.                                     |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **ETag**                            | String      | The entity tag of the new object.                     |
++-------------------------------------+-------------+-------------------------------------------------------+
+
+Abort Multipart Upload
+----------------------
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket}/{object}?uploadId= HTTP/1.1
+
+
+
+Append Object
+-------------
+Append data to an object. You must have write permissions on the bucket to perform this operation.
+It is used to upload files in appending mode. The type of the objects created by the Append Object
+operation is Appendable Object, and the type of the objects uploaded with the Put Object operation is Normal Object.
+**Append Object can't be used if bucket versioning is enabled or suspended.**
+**Synced object will become normal in multisite, but you can still append to the original object.**
+**Compression and encryption features are disabled for Appendable objects.**
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?append&position= HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
++--------------------------------+------------------------------------------------------------------+
+| Name                           | Description                                                      |
++================================+==================================================================+
+| **x-rgw-next-append-position** | Next position to append object                                   |
++--------------------------------+------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+The following HTTP response may be returned:
+
++---------------+----------------------------+---------------------------------------------------+
+| HTTP Status   | Status Code                | Description                                       |
++===============+============================+===================================================+
+| **409**       | PositionNotEqualToLength   | Specified position does not match object length   |
++---------------+----------------------------+---------------------------------------------------+
+| **409**       | ObjectNotAppendable        | Specified object can not be appended              |
++---------------+----------------------------+---------------------------------------------------+
+| **409**       | InvalidBucketstate         | Bucket versioning is enabled or suspended         |
++---------------+----------------------------+---------------------------------------------------+
+
+
+Put Object Retention
+--------------------
+Places an Object Retention configuration on an object.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?retention&versionId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| Name                | Type        | Description                                                                   |  Required  |
++=====================+=============+===============================================================================+============+
+| ``Retention``       | Container   | A container for the request.                                                  |    Yes     |
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| ``Mode``            | String      | Retention mode for the specified object. Valid Values:  GOVERNANCE/COMPLIANCE |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+| ``RetainUntilDate`` | Timestamp   | Retention date. Format: 2020-01-05T00:00:00.000Z                              |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+
+
+Get Object Retention
+--------------------
+Gets an Object Retention configuration on an object.
+
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?retention&versionId= HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| Name                | Type        | Description                                                                   |  Required  |
++=====================+=============+===============================================================================+============+
+| ``Retention``       | Container   | A container for the request.                                                  |    Yes     |
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| ``Mode``            | String      | Retention mode for the specified object. Valid Values:  GOVERNANCE/COMPLIANCE |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+| ``RetainUntilDate`` | Timestamp   | Retention date. Format: 2020-01-05T00:00:00.000Z                              |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+
+
+Put Object Legal Hold
+---------------------
+Applies a Legal Hold configuration to the specified object.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?legal-hold&versionId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| Name           | Type        | Description                                                                            |  Required  |
++================+=============+========================================================================================+============+
+| ``LegalHold``  | Container   | A container for the request.                                                           |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| ``Status``     | String      | Indicates whether the specified object has a Legal Hold in place. Valid Values: ON/OFF |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+
+
+Get Object Legal Hold
+---------------------
+Gets an object's current Legal Hold status.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?legal-hold&versionId= HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| Name           | Type        | Description                                                                            |  Required  |
++================+=============+========================================================================================+============+
+| ``LegalHold``  | Container   | A container for the request.                                                           |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| ``Status``     | String      | Indicates whether the specified object has a Legal Hold in place. Valid Values: ON/OFF |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+
diff --git a/doc/radosgw/s3/perl.rst b/doc/radosgw/s3/perl.rst
new file mode 100644
index 000000000..f12e5c698
--- /dev/null
+++ b/doc/radosgw/s3/perl.rst
@@ -0,0 +1,192 @@
+.. _perl:
+
+Perl S3 Examples
+================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: perl
+
+	use Amazon::S3;
+	my $access_key = 'put your access key here!';
+	my $secret_key = 'put your secret key here!';
+
+	my $conn = Amazon::S3->new({
+		aws_access_key_id     => $access_key,
+		aws_secret_access_key => $secret_key,
+		host                  => 'objects.dreamhost.com',
+		secure                => 1,
+		retry                 => 1,
+	});
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of `Amazon::S3::Bucket`_ objects that you own.
+We'll also print out the bucket name and creation date of each bucket.
+
+.. code-block:: perl
+
+	my @buckets = @{$conn->buckets->{buckets} || []};
+	foreach my $bucket (@buckets) {
+		print $bucket->bucket . "\t" . $bucket->creation_date . "\n";
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: perl
+
+	my $bucket = $conn->add_bucket({ bucket => 'my-new-bucket' });
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with info about each object in the bucket.
+We'll also print out each object's name, the file size, and last
+modified date.
+
+.. code-block:: perl
+
+	my @keys = @{$bucket->list_all->{keys} || []};
+	foreach my $key (@keys) {
+		print "$key->{key}\t$key->{size}\t$key->{last_modified}\n";
+	}
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: perl
+
+	$conn->delete_bucket($bucket);
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available in the `Amazon::S3`_ perl module
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: perl
+
+	$bucket->add_key(
+		'hello.txt', 'Hello World!',
+		{ content_type => 'text/plain' },
+	);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable and
+``secret_plans.txt`` to be private.
+
+.. code-block:: perl
+
+	$bucket->set_acl({
+		key       => 'hello.txt',
+		acl_short => 'public-read',
+	});
+	$bucket->set_acl({
+		key       => 'secret_plans.txt',
+		acl_short => 'private',
+	});
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: perl
+
+	$bucket->get_key_filename('perl_poetry.pdf', undef,
+		'/home/larry/documents/perl_poetry.pdf');
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: perl
+
+	$bucket->delete_key('goodbye.txt');
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+Then this generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+   The `Amazon::S3`_ module does not have a way to generate download
+   URLs, so we are going to be using another module instead. Unfortunately,
+   most modules for generating these URLs assume that you are using Amazon,
+   so we have had to go with using a more obscure module, `Muck::FS::S3`_. This
+   should be the same as Amazon's sample S3 perl module, but this sample
+   module is not in CPAN. So, you can either use CPAN to install
+   `Muck::FS::S3`_, or install Amazon's sample S3 module manually. If you go
+   the manual route, you can remove ``Muck::FS::`` from the example below.
+
+.. code-block:: perl
+
+	use Muck::FS::S3::QueryStringAuthGenerator;
+	my $generator = Muck::FS::S3::QueryStringAuthGenerator->new(
+		$access_key,
+		$secret_key,
+		0, # 0 means use 'http'. set this to 1 for 'https'
+		'objects.dreamhost.com',
+	);
+
+	my $hello_url = $generator->make_bare_url($bucket->bucket, 'hello.txt');
+	print $hello_url . "\n";
+
+	$generator->expires_in(3600); # 1 hour = 3600 seconds
+	my $plans_url = $generator->get($bucket->bucket, 'secret_plans.txt');
+	print $plans_url . "\n";
+
+The output will look something like this::
+
+   http://objects.dreamhost.com:80/my-bucket-name/hello.txt
+   http://objects.dreamhost.com:80/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+
+.. _`Amazon::S3`: http://search.cpan.org/~tima/Amazon-S3-0.441/lib/Amazon/S3.pm
+.. _`Amazon::S3::Bucket`: http://search.cpan.org/~tima/Amazon-S3-0.441/lib/Amazon/S3/Bucket.pm
+.. _`Muck::FS::S3`: http://search.cpan.org/~mike/Muck-0.02/
+
diff --git a/doc/radosgw/s3/php.rst b/doc/radosgw/s3/php.rst
new file mode 100644
index 000000000..4878a3489
--- /dev/null
+++ b/doc/radosgw/s3/php.rst
@@ -0,0 +1,214 @@
+.. _php:
+
+PHP S3 Examples
+===============
+
+Installing AWS PHP SDK
+----------------------
+
+This installs AWS PHP SDK using composer (see here_ how to install composer).
+
+.. _here: https://getcomposer.org/download/
+  
+.. code-block:: bash
+
+	$ composer install aws/aws-sdk-php
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. note::
+
+   The client initialization requires a region so we use ``''``.
+
+.. code-block:: php
+
+	<?php
+	
+	use Aws\S3\S3Client;
+	
+	define('AWS_KEY', 'place access key here');
+	define('AWS_SECRET_KEY', 'place secret key here');
+	$ENDPOINT = 'http://objects.dreamhost.com';
+
+	// require the amazon sdk from your composer vendor dir
+	require __DIR__.'/vendor/autoload.php';
+
+	// Instantiate the S3 class and point it at the desired host
+	$client = new S3Client([
+	    'region' => '',
+	    'version' => '2006-03-01',
+	    'endpoint' => $ENDPOINT,
+	    'credentials' => [
+                'key' => AWS_KEY,
+		'secret' => AWS_SECRET_KEY
+	    ],
+	    // Set the S3 class to use objects.dreamhost.com/bucket
+	    // instead of bucket.objects.dreamhost.com
+	    'use_path_style_endpoint' => true
+	]);
+
+Listing Owned Buckets
+---------------------
+This gets a ``AWS\Result`` instance that is more convenient to visit using array access way.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: php
+
+	<?php
+	$listResponse = $client->listBuckets();
+	$buckets = $listResponse['Buckets'];
+	foreach ($buckets as $bucket) {
+    	    echo $bucket['Name'] . "\t" . $bucket['CreationDate'] . "\n";
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket`` and returns a
+``AWS\Result`` object.
+
+.. code-block:: php
+
+	<?php
+	$client->createBucket(['Bucket' => 'my-new-bucket']);
+
+
+List a Bucket's Content
+-----------------------
+
+This gets a ``AWS\Result`` instance that is more convenient to visit using array access way.
+This then prints out each object's name, the file size, and last modified date.
+
+.. code-block:: php
+
+	<?php
+	$objectsListResponse = $client->listObjects(['Bucket' => $bucketname]);
+	$objects = $objectsListResponse['Contents'] ?? [];
+	foreach ($objects as $object) {
+    	    echo $object['Key'] . "\t" . $object['Size'] . "\t" . $object['LastModified'] . "\n";
+	}
+
+.. note::
+
+   If there are more than 1000 objects in this bucket,
+   you need to check $objectsListResponse['isTruncated']
+   and run again with the name of the last key listed.
+   Keep doing this until isTruncated is not true.
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+This deletes the bucket called ``my-old-bucket`` and returns a
+``AWS\Result`` object
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: php
+
+	<?php
+	$client->deleteBucket(['Bucket' => 'my-old-bucket']);
+
+
+Creating an Object
+------------------
+
+This creates an object ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: php
+
+	<?php
+	$client->putObject([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'hello.txt',
+    	    'Body' => "Hello World!"
+	]);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable and
+``secret_plans.txt`` to be private.
+
+.. code-block:: php
+
+	<?php
+	$client->putObjectAcl([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'hello.txt',
+    	    'ACL' => 'public-read'
+	]);
+	$client->putObjectAcl([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'secret_plans.txt',
+    	    'ACL' => 'private'
+	]);
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: php
+
+	<?php
+	$client->deleteObject(['Bucket' => 'my-bucket-name', 'Key' => 'goodbye.txt']);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: php
+
+	<?php
+	$object = $client->getObject(['Bucket' => 'my-bucket-name', 'Key' => 'poetry.pdf']);
+	file_put_contents('/home/larry/documents/poetry.pdf', $object['Body']->getContents());
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``.
+This works because we made ``hello.txt`` public by setting
+the ACL above. This then generates a signed download URL
+for ``secret_plans.txt`` that will work for 1 hour.
+Signed download URLs will work for the time period even
+if the object is private (when the time period is up,
+the URL will stop working).
+
+.. code-block:: php
+
+	<?php
+	$hello_url = $client->getObjectUrl('my-bucket-name', 'hello.txt');
+	echo $hello_url."\n";
+	
+	$secret_plans_cmd = $client->getCommand('GetObject', ['Bucket' => 'my-bucket-name', 'Key' => 'secret_plans.txt']);
+	$request = $client->createPresignedRequest($secret_plans_cmd, '+1 hour');
+	echo $request->getUri()."\n";
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=sandboxAccessKey%2F20190116%2F%2Fs3%2Faws4_request&X-Amz-Date=20190116T125520Z&X-Amz-SignedHeaders=host&X-Amz-Expires=3600&X-Amz-Signature=61921f07c73d7695e47a2192cf55ae030f34c44c512b2160bb5a936b2b48d923
+
diff --git a/doc/radosgw/s3/python.rst b/doc/radosgw/s3/python.rst
new file mode 100644
index 000000000..35f682893
--- /dev/null
+++ b/doc/radosgw/s3/python.rst
@@ -0,0 +1,197 @@
+.. _python:
+
+Python S3 Examples
+==================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: python
+
+	import boto
+	import boto.s3.connection
+	access_key = 'put your access key here!'
+	secret_key = 'put your secret key here!'
+
+	conn = boto.connect_s3(
+		aws_access_key_id = access_key,
+		aws_secret_access_key = secret_key,
+		host = 'objects.dreamhost.com',
+                #is_secure=False,               # uncomment if you are not using ssl
+		calling_format = boto.s3.connection.OrdinaryCallingFormat(),
+		)
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: python
+
+	for bucket in conn.get_all_buckets():
+		print("{name}\t{created}".format(
+			name = bucket.name,
+			created = bucket.creation_date,
+		))
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: python
+
+	bucket = conn.create_bucket('my-new-bucket')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: python
+
+	for key in bucket.list():
+		print("{name}\t{size}\t{modified}".format(
+			name = key.name,
+			size = key.size,
+			modified = key.last_modified,
+		))
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: python
+
+	conn.delete_bucket(bucket.name)
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available in python
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: python
+
+	key = bucket.new_key('hello.txt')
+	key.set_contents_from_string('Hello World!')
+	
+
+Uploading an Object or a File
+-----------------------------
+
+This creates a file ``logo.png`` with the contents from the file ``"logo.png"``
+
+.. code-block:: python
+
+	key = bucket.new_key('logo.png')
+	key.set_contents_from_filename('logo.png')
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: python
+
+	hello_key = bucket.get_key('hello.txt')
+	hello_key.set_canned_acl('public-read')
+	plans_key = bucket.get_key('secret_plans.txt')
+	plans_key.set_canned_acl('private')
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: python
+
+	key = bucket.get_key('perl_poetry.pdf')
+	key.get_contents_to_filename('/home/larry/documents/perl_poetry.pdf')
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: python
+
+	bucket.delete_key('goodbye.txt')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: python
+
+	hello_key = bucket.get_key('hello.txt')
+	hello_url = hello_key.generate_url(0, query_auth=False, force_http=True)
+	print(hello_url)
+
+	plans_key = bucket.get_key('secret_plans.txt')
+	plans_url = plans_key.generate_url(3600, query_auth=True, force_http=True)
+	print(plans_url)
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+Using S3 API Extensions 
+-----------------------
+
+To use the boto3 client to tests the RadosGW extensions to the S3 API, the `extensions file`_ should be placed under: ``~/.aws/models/s3/2006-03-01/`` directory.
+For example, unordered list of objects could be fetched using:
+
+.. code-block:: python
+
+    print(conn.list_objects(Bucket='my-new-bucket', AllowUnordered=True))
+
+
+Without the extensions file, in the above example, boto3 would complain that the ``AllowUnordered`` argument is invalid.
+
+
+.. _extensions file: https://github.com/ceph/ceph/blob/main/examples/rgw/boto3/service-2.sdk-extras.json
diff --git a/doc/radosgw/s3/ruby.rst b/doc/radosgw/s3/ruby.rst
new file mode 100644
index 000000000..435b3c630
--- /dev/null
+++ b/doc/radosgw/s3/ruby.rst
@@ -0,0 +1,364 @@
+.. _ruby:
+
+Ruby `AWS::SDK`_ Examples (aws-sdk gem ~>2)
+===========================================
+
+Settings
+---------------------
+
+You can setup the connection on global way:
+
+.. code-block:: ruby
+
+	Aws.config.update(
+		endpoint: 'https://objects.dreamhost.com.',
+		access_key_id: 'my-access-key',
+		secret_access_key: 'my-secret-key',
+		force_path_style: true, 
+		region: 'us-east-1'
+	)
+
+
+and instantiate a client object:
+
+.. code-block:: ruby
+
+    	s3_client = Aws::S3::Client.new
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: ruby
+
+	s3_client.list_buckets.buckets.each do |bucket|
+		puts "#{bucket.name}\t#{bucket.creation_date}"
+	end
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: ruby
+
+	s3_client.create_bucket(bucket: 'my-new-bucket')
+
+If you want a private bucket: 
+
+`acl` option accepts: # private, public-read, public-read-write, authenticated-read
+
+.. code-block:: ruby
+
+	s3_client.create_bucket(bucket: 'my-new-bucket', acl: 'private')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with the contents of each object
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: ruby
+
+	s3_client.get_objects(bucket: 'my-new-bucket').contents.each do |object|
+		puts "#{object.key}\t#{object.size}\t#{object.last-modified}"
+	end
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: ruby
+
+	s3_client.delete_bucket(bucket: 'my-new-bucket')
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+First, you need to clear the bucket:
+
+.. code-block:: ruby
+
+	Aws::S3::Bucket.new('my-new-bucket', client: s3_client).clear!
+	
+after, you can destroy the bucket
+
+.. code-block:: ruby
+
+	s3_client.delete_bucket(bucket: 'my-new-bucket')
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: ruby
+
+	s3_client.put_object(
+		key: 'hello.txt',
+		body: 'Hello World!',
+		bucket: 'my-new-bucket',
+		content_type: 'text/plain'
+	)
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and ``secret_plans.txt``
+to be private.
+
+.. code-block:: ruby
+
+	s3_client.put_object_acl(bucket: 'my-new-bucket', key: 'hello.txt', acl: 'public-read')
+
+	s3_client.put_object_acl(bucket: 'my-new-bucket', key: 'private.txt', acl: 'private')
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: ruby
+
+	s3_client.get_object(bucket: 'my-new-bucket', key: 'poetry.pdf', response_target: '/home/larry/documents/poetry.pdf')
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: ruby
+
+	s3_client.delete_object(key: 'goodbye.txt', bucket: 'my-new-bucket')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: ruby
+
+	puts Aws::S3::Object.new(
+		key: 'hello.txt',
+		bucket_name: 'my-new-bucket',
+		client: s3_client
+	).public_url
+
+	puts Aws::S3::Object.new(
+		key: 'secret_plans.txt',
+		bucket_name: 'hermes_ceph_gem',
+		client: s3_client
+	).presigned_url(:get, expires_in: 60 * 60)
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+.. _`AWS::SDK`: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
+
+
+
+Ruby `AWS::S3`_ Examples (aws-s3 gem)
+=====================================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: ruby
+
+	AWS::S3::Base.establish_connection!(
+		:server            => 'objects.dreamhost.com',
+		:use_ssl           => true,
+		:access_key_id     => 'my-access-key',
+		:secret_access_key => 'my-secret-key'
+	)
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of `AWS::S3::Bucket`_ objects that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: ruby
+
+	AWS::S3::Service.buckets.each do |bucket|
+		puts "#{bucket.name}\t#{bucket.creation_date}"
+	end
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.create('my-new-bucket')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with the contents of each object
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: ruby
+
+	new_bucket = AWS::S3::Bucket.find('my-new-bucket')
+	new_bucket.each do |object|
+		puts "#{object.key}\t#{object.about['content-length']}\t#{object.about['last-modified']}"
+	end
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.delete('my-new-bucket')
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.delete('my-new-bucket', :force => true)
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: ruby
+
+	AWS::S3::S3Object.store(
+		'hello.txt',
+		'Hello World!',
+		'my-new-bucket',
+		:content_type => 'text/plain'
+	)
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and ``secret_plans.txt``
+to be private.
+
+.. code-block:: ruby
+
+	policy = AWS::S3::S3Object.acl('hello.txt', 'my-new-bucket')
+	policy.grants = [ AWS::S3::ACL::Grant.grant(:public_read) ]
+	AWS::S3::S3Object.acl('hello.txt', 'my-new-bucket', policy)
+
+	policy = AWS::S3::S3Object.acl('secret_plans.txt', 'my-new-bucket')
+	policy.grants = []
+	AWS::S3::S3Object.acl('secret_plans.txt', 'my-new-bucket', policy)
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: ruby
+
+	open('/home/larry/documents/poetry.pdf', 'w') do |file|
+		AWS::S3::S3Object.stream('poetry.pdf', 'my-new-bucket') do |chunk|
+			file.write(chunk)
+		end
+	end
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: ruby
+
+	AWS::S3::S3Object.delete('goodbye.txt', 'my-new-bucket')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: ruby
+
+	puts AWS::S3::S3Object.url_for(
+		'hello.txt',
+		'my-new-bucket',
+		:authenticated => false
+	)
+
+	puts AWS::S3::S3Object.url_for(
+		'secret_plans.txt',
+		'my-new-bucket',
+		:expires_in => 60 * 60
+	)
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+.. _`AWS::S3`: http://amazon.rubyforge.org/
+.. _`AWS::S3::Bucket`: http://amazon.rubyforge.org/doc/
+
diff --git a/doc/radosgw/s3/serviceops.rst b/doc/radosgw/s3/serviceops.rst
new file mode 100644
index 000000000..54b6ca375
--- /dev/null
+++ b/doc/radosgw/s3/serviceops.rst
@@ -0,0 +1,69 @@
+Service Operations
+==================
+
+List Buckets
+------------
+``GET /`` returns a list of buckets created by the user making the request. ``GET /`` only
+returns buckets created by an authenticated user. You cannot make an anonymous request.
+
+Syntax
+~~~~~~
+::
+
+	GET / HTTP/1.1
+	Host: cname.domain.com
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------------------+-------------+-----------------------------------------------------------------+
+| Name                       | Type        | Description                                                     |
++============================+=============+=================================================================+
+| ``Buckets``                | Container   | Container for list of buckets.                                  |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Bucket``                 | Container   | Container for bucket information.                               |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Name``                   | String      | Bucket name.                                                    |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``CreationDate``           | Date        | UTC time when the bucket was created.                           |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``ListAllMyBucketsResult`` | Container   | A container for the result.                                     |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Owner``                  | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.  |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``ID``                     | String      | The bucket owner's ID.                                          |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``DisplayName``            | String      | The bucket owner's display name.                                |
++----------------------------+-------------+-----------------------------------------------------------------+
+
+
+Get Usage Stats
+---------------
+
+Gets usage stats per user, similar to the admin command :ref:`rgw_user_usage_stats`.
+
+Syntax
+~~~~~~
+::
+
+	GET /?usage HTTP/1.1
+	Host: cname.domain.com
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------------------+-------------+-----------------------------------------------------------------+
+| Name                       | Type        | Description                                                     |
++============================+=============+=================================================================+
+| ``Summary``                | Container   | Summary of total stats by user.                                 |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalBytes``             | Integer     | Bytes used by user                                              |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalBytesRounded``      | Integer     | Bytes rounded to the nearest 4k boundary                        |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalEntries``           | Integer     | Total object entries                                            |
++----------------------------+-------------+-----------------------------------------------------------------+
diff --git a/doc/radosgw/s3select.rst b/doc/radosgw/s3select.rst
new file mode 100644
index 000000000..d46d4e96f
--- /dev/null
+++ b/doc/radosgw/s3select.rst
@@ -0,0 +1,796 @@
+===============
+ Ceph s3 select 
+===============
+
+.. contents::
+
+Overview
+--------
+
+The **S3 Select** engine creates an efficient pipe between clients and Ceph
+back end nodes. The S3 Select engine works best when implemented as closely as
+possible to back end storage.
+
+The S3 Select engine makes it possible to use an SQL-like syntax to select a
+restricted subset of data stored in an S3 object. The S3 Select engine
+facilitates the use of higher level, analytic applications (for example:
+SPARK-SQL). The ability of the S3 Select engine to target a proper subset of
+structed data within an S3 object decreases latency and increases throughput.
+
+For example: assume that a user needs to extract a single column that is
+filtered by another column, and that these colums are stored in a CSV file in
+an S3 object that is several GB in size. The following query performs this
+extraction: ``select customer-id from s3Object where age>30 and age<65;``
+
+Without the use of S3 Select, the whole S3 object must be retrieved from an OSD
+via RGW before the data is filtered and extracted. Significant network and CPU
+overhead are saved by "pushing down" the query into radosgw.
+
+**The bigger the object and the more accurate the query,
+the better the performance of s3select**.
+ 
+Basic Workflow
+--------------
+    
+S3 Select queries are sent to RGW via `AWS-CLI
+<https://docs.aws.amazon.com/cli/latest/reference/s3api/select-object-content.html>`_
+
+S3 Select passes the authentication and permission parameters as an incoming
+message (POST). ``RGWSelectObj_ObjStore_S3::send_response_data`` is the entry
+point and handles each fetched chunk according to the object key that was
+input.  ``send_response_data`` is the first to handle the input query: it
+extracts the query and other CLI parameters.
+   
+RGW executes an S3 Select query on each new fetched chunk (up to 4 MB). The
+current implementation supports CSV objects. CSV rows are sometimes "cut" in
+the middle by the limits of the chunks, and those broken-lines (the first or
+last per chunk) are skipped while processing the query. Such broken lines are
+stored and later merged with the next broken line (which belongs to the next
+chunk), and only then processed.
+
+For each processed chunk, an output message is formatted according to `aws
+specification
+<https://docs.aws.amazon.com/amazons3/latest/api/archive-restobjectselectcontent.html#archive-restobjectselectcontent-responses>`_
+and sent back to the client. RGW supports the following response:
+``{:event-type,records} {:content-type,application/octet-stream}
+{:message-type,event}``. For aggregation queries, the last chunk should be
+identified as the end of input. 
+
+        
+Basic Functionalities
+~~~~~~~~~~~~~~~~~~~~~
+
+**S3select** has a definite set of functionalities compliant with AWS.
+    
+The implemented software architecture supports basic arithmetic expressions,
+logical and compare expressions, including nested function calls and casting
+operators, which enables the user great flexibility. 
+
+review the below s3-select-feature-table_.
+
+
+Error Handling
+~~~~~~~~~~~~~~
+
+Upon an error being detected, RGW returns 400-Bad-Request and a specific error message sends back to the client.
+Currently, there are 2 main types of error.
+
+**Syntax error**: the s3select parser rejects user requests that are not aligned with parser syntax definitions, as     
+described in this documentation.
+Upon Syntax Error, the engine creates an error message that points to the location of the error.
+RGW sends back the error message in a specific error response. 
+
+**Processing Time error**: the runtime engine may detect errors that occur only on processing time, for that type of     
+error, a different error message would describe that.
+RGW sends back the error message in a specific error response.
+
+.. _s3-select-feature-table:
+
+Features Support
+----------------
+
+Currently only part of `AWS select command
+<https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html>`_
+is implemented, table below describes what is currently supported.
+
+The following table describes the current implementation for s3-select
+functionalities:
+
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Feature                         | Detailed        | Example  / Description                                                |
++=================================+=================+=======================================================================+
+| Arithmetic operators            | ^ * % / + - ( ) | select (int(_1)+int(_2))*int(_9) from s3object;                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 | ``%`` modulo    | select count(*) from s3object where cast(_1 as int)%2 = 0;            |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 | ``^`` power-of  | select cast(2^10 as int) from s3object;                               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Compare operators               | > < >= <= = !=  | select _1,_2 from s3object where (int(_1)+int(_3))>int(_5);           |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | AND OR NOT      | select count(*) from s3object where not (int(_1)>123 and int(_5)<200);|
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | is null         | return true/false for null indication in expression                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | is not null     | return true/false for null indication in expression                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator and NULL       | unknown state   | review null-handle_ observe how logical operator result with null.    |
+|                                 |                 | the following query return **0**.                                     |
+|                                 |                 |                                                                       |
+|                                 |                 | select count(*) from s3object where null and (3>2);                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Arithmetic operator with NULL   | unknown state   | review null-handle_ observe the results of binary operations with NULL|
+|                                 |                 | the following query return **0**.                                     |
+|                                 |                 |                                                                       |
+|                                 |                 | select count(*) from s3object where (null+1) and (3>2);               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| compare with NULL               | unknown state   | review null-handle_ observe results of compare operations with NULL   | 
+|                                 |                 | the following query return **0**.                                     |
+|                                 |                 |                                                                       |
+|                                 |                 | select count(*) from s3object where (null*1.5) != 3;                  |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| missing column                  | unknown state   | select count(*) from s3object where _1 is null;                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| query is filtering rows where predicate           | select count(*) from s3object where (_1 > 12 and _2 = 0) is not null; |
+| is returning non null results.                    |                                                                       |
+| this predicate will return null                   |                                                                       |
+| upon _1 or _2 is null                             |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| projection column               | similar to      | select case                                                           | 
+|                                 | switch/case     | cast(_1 as int) + 1                                                   |
+|                                 | default         | when 2 then "a"                                                       |
+|                                 |                 | when 3  then "b"                                                      |
+|                                 |                 | else "c" end from s3object;                                           |
+|                                 |                 |                                                                       | 
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| projection column               | similar to      | select case                                                           | 
+|                                 | if/then/else    | when (1+1=(2+1)*3) then 'case_1'                                      |
+|                                 |                 | when ((4*3)=(12)) then 'case_2'                                       |
+|                                 |                 | else 'case_else' end,                                                 |
+|                                 |                 | age*2 from s3object;                                                  | 
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | ``coalesce {expression,expression ...} :: return first non-null argument``              |
+|                                 |                                                                                         |
+|                                 | select coalesce(nullif(5,5),nullif(1,1.0),age+12) from s3object;                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | ``nullif {expr1,expr2} ::return null in case both arguments are equal,``                |
+|                                 | ``or else the first one``                                                               |
+|                                 |                                                                                         |
+|                                 | select nullif(cast(_1 as int),cast(_2 as int)) from s3object;                           |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | ``{expression} in ( .. {expression} ..)``                                               |
+|                                 |                                                                                         |
+|                                 | select count(*) from s3object                                                           | 
+|                                 | where 'ben' in (trim(_5),substring(_1,char_length(_1)-3,3),last_name);                  |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | ``{expression} between {expression} and {expression}``                                  | 
+|                                 |                                                                                         |
+|                                 | select count(*) from s3object                                                           | 
+|                                 | where substring(_3,char_length(_3),1) between "x" and trim(_1)                          |
+|                                 | and substring(_3,char_length(_3)-1,1) = ":";                                            |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | ``{expression} like {match-pattern}``                                                   |
+|                                 |                                                                                         |
+|                                 | select count(*) from s3object where first_name like '%de_';                             |
+|                                 |                                                                                         |
+|                                 | select count(*) from s3object where _1 like \"%a[r-s]\;                                 |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 | ``{expression} like {match-pattern} escape {char}``                                     |
+|                                 |                                                                                         |
+| logical operator                | select count(*) from s3object where  "jok_ai" like "%#_ai" escape "#";                  |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| true / false                    | select (cast(_1 as int)>123 = true) from s3object                                       |
+| predicate as a projection       | where address like '%new-york%';                                                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| an alias to                     | select (_1 like "_3_") as *likealias*,_1 from s3object                                  |
+| predicate as a projection       | where *likealias* = true and cast(_1 as int) between 800 and 900;                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | select cast(123 as int)%2 from s3object;                                                |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | select cast(123.456 as float)%2 from s3object;                                          |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | select cast('ABC0-9' as string),cast(substr('ab12cd',3,2) as int)*4  from s3object;     |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | select cast(5 as bool) from s3object;                                                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | select cast(substring('publish on 2007-01-01',12,10) as timestamp) from s3object;       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| non AWS casting operator        | select int(_1),int( 1.2 + 3.4) from s3object;                                           |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| non AWS casting operator        | select float(1.2) from s3object;                                                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| not AWS casting operator        | select to_timestamp('1999-10-10T12:23:44Z') from s3object;                              |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | sum             | select sum(int(_1)) from s3object;                                    |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | avg             | select avg(cast(_1 a float) + cast(_2 as int)) from s3object;         |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | min             | select min( int(_1) * int(_5) ) from s3object;                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | max             | select max(float(_1)),min(int(_5)) from s3object;                     |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | count           | select count(*) from s3object where (int(_1)+int(_3))>int(_5);        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | extract         | select count(*) from s3object where                                   |
+|                                 |                 | extract(year from to_timestamp(_2)) > 1950                            |
+|                                 |                 | and extract(year from to_timestamp(_1)) < 1960;                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | date_add        | select count(0) from s3object where                                   |
+|                                 |                 | date_diff(year,to_timestamp(_1),date_add(day,366,                     |
+|                                 |                 | to_timestamp(_1))) = 1;                                               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | date_diff       | select count(0) from s3object where                                   |
+|                                 |                 | date_diff(month,to_timestamp(_1),to_timestamp(_2))) = 2;              |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | utcnow          | select count(0) from s3object where                                   |
+|                                 |                 | date_diff(hours,utcnow(),date_add(day,1,utcnow())) = 24;              |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | to_string       | select to_string(                                                     |
+|                                 |                 | to_timestamp("2009-09-17T17:56:06.234567Z"),                          |
+|                                 |                 | "yyyyMMdd-H:m:s") from s3object;                                      |
+|                                 |                 |                                                                       |
+|                                 |                 | ``result: "20090917-17:56:6"``                                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | substring       | select count(0) from s3object where                                   |
+|                                 |                 | int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;          |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| substring with ``from`` negative number is valid  | select substring("123456789" from -4) from s3object;                  |
+| considered as first                               |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| substring with ``from`` zero ``for`` out-of-bound |  select substring("123456789" from 0 for 100) from s3object;          |
+| number is valid just as (first,last)              |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | trim            | select trim('   foobar   ') from s3object;                            |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | trim            | select trim(trailing from '   foobar   ') from s3object;              |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | trim            | select trim(leading from '   foobar   ') from s3object;               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | trim            | select trim(both '12' from  '1112211foobar22211122') from s3objects;  |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | lower/upper     | select lower('ABcD12#$e') from s3object;                              |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | char_length     | select count(*) from s3object where char_length(_3)=3;                |
+|                                 | character_length|                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Complex queries                 | select sum(cast(_1 as int)),                                                            |
+|                                 | max(cast(_3 as int)),                                                                   |
+|                                 | substring('abcdefghijklm',(2-1)*3+sum(cast(_1 as int))/sum(cast(_1 as int))+1,          |
+|                                 | (count() + count(0))/count(0)) from s3object;                                           |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| alias support                   |                 |  select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3                  | 
+|                                 |                 |  from s3object where a3>100 and a3<300;                               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+
+.. _null-handle:
+
+NULL
+~~~~
+NULL is a legit value in ceph-s3select systems similar to other DB systems, i.e. systems needs to handle the case where a value is NULL.
+
+The definition of NULL in our context, is missing/unknown, in that sense **NULL can not produce a value on ANY arithmetic operations** ( a + NULL will produce NULL value).
+
+The Same is with arithmetic comparison, **any comparison to NULL is NULL**, i.e. unknown.
+Below is a truth table contains the NULL use-case.
+
++---------------------------------+-----------------------------+
+| A is NULL                       | Result (NULL=UNKNOWN)       |
++=================================+=============================+
+| NOT A                           |  NULL                       |
++---------------------------------+-----------------------------+
+| A OR False                      |  NULL                       |
++---------------------------------+-----------------------------+
+| A OR True                       |  True                       |
++---------------------------------+-----------------------------+
+| A OR A                          |  NULL                       |
++---------------------------------+-----------------------------+
+| A AND False                     |  False                      |
++---------------------------------+-----------------------------+
+| A AND True                      |  NULL                       | 
++---------------------------------+-----------------------------+
+| A and A                         |  NULL                       |
++---------------------------------+-----------------------------+
+
+S3-select Function Interfaces
+-----------------------------
+
+Timestamp Functions
+~~~~~~~~~~~~~~~~~~~
+The timestamp functionalities as described in `AWS-specs
+<https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html>`_
+is fully implemented.
+
+ ``to_timestamp( string )`` : The casting operator converts string to timestamp
+ basic type.  to_timestamp operator is able to convert the following
+ ``YYYY-MM-DDTHH:mm:ss.SSSSSS+/-HH:mm`` , ``YYYY-MM-DDTHH:mm:ss.SSSSSSZ`` ,
+ ``YYYY-MM-DDTHH:mm:ss+/-HH:mm`` , ``YYYY-MM-DDTHH:mm:ssZ`` ,
+ ``YYYY-MM-DDTHH:mm+/-HH:mm`` , ``YYYY-MM-DDTHH:mmZ`` , ``YYYY-MM-DDT`` or
+ ``YYYYT`` string formats into timestamp.  Where time (or part of it) is
+ missing in the string format, zero's are replacing the missing parts. And for
+ missing month and day, 1 is default value for them.  Timezone part is in
+ format ``+/-HH:mm`` or ``Z`` , where the letter "Z" indicates Coordinated
+ Universal Time (UTC). Value of timezone can range between -12:00 and +14:00.
+
+ ``extract(date-part from timestamp)`` : The function extracts date-part from
+ input timestamp and returns it as integer.  Supported date-part : year, month,
+ week, day, hour, minute, second, timezone_hour, timezone_minute.
+
+ ``date_add(date-part, quantity, timestamp)`` : The function adds quantity
+ (integer) to date-part of timestamp and returns result as timestamp. It also
+ includes timezone in calculation.  Supported data-part : year, month, day,
+ hour, minute, second.
+
+ ``date_diff(date-part, timestamp, timestamp)`` : The function returns an
+ integer, a calculated result for difference between 2 timestamps according to
+ date-part. It includes timezone in calculation.  supported date-part : year,
+ month, day, hour, minute, second.
+
+ ``utcnow()`` : return timestamp of current time.
+
+ ``to_string(timestamp, format_pattern)`` : returns a string representation of
+ the input timestamp in the given input string format.
+
+to_string parameters
+~~~~~~~~~~~~~~~~~~~~
+
++--------------+-----------------+-----------------------------------------------------------------------------------+
+| Format       | Example         | Description                                                                       |
++==============+=================+===================================================================================+
+|    yy        | 69              |  2-digit year                                                                     |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    y         | 1969            |  4-digit year                                                                     |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    yyyy      | 1969            |  Zero-padded 4-digit year                                                         |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    M         | 1               |  Month of year                                                                    |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    MM        | 01              |  Zero-padded month of year                                                        |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    MMM       | Jan             |  Abbreviated month year name                                                      |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    MMMM      | January         |  Full month of year name                                                          |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    MMMMM     | J               |  Month of year first letter (NOTE: not valid for use with to_timestamp function)  |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    d         | 2               |  Day of month (1-31)                                                              |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    dd        | 02              |  Zero-padded day of month (01-31)                                                 |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    a         | AM              |  AM or PM of day                                                                  |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    h         | 3               |  Hour of day (1-12)                                                               |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    hh        | 03              |  Zero-padded hour of day (01-12)                                                  |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    H         | 3               |  Hour of day (0-23)                                                               |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    HH        | 03              |  Zero-padded hour of day (00-23)                                                  |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    m         | 4               |  Minute of hour (0-59)                                                            |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    mm        | 04              |  Zero-padded minute of hour (00-59)                                               |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    s         | 5               |  Second of minute (0-59)                                                          |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    ss        | 05              |  Zero-padded second of minute (00-59)                                             |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    S         | 0               |  Fraction of second (precision: 0.1, range: 0.0-0.9)                              |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    SS        | 6               |  Fraction of second (precision: 0.01, range: 0.0-0.99)                            |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    SSS       | 60              |  Fraction of second (precision: 0.001, range: 0.0-0.999)                          |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    SSSSSS    | 60000000        |  Fraction of second (maximum precision: 1 nanosecond, range: 0.0-0999999999)      |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    n         | 60000000        |  Nano of second                                                                   |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    X         | +07 or Z        |  Offset in hours or "Z" if the offset is 0                                        |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+|    XX or XXXX| +0700 or Z      |  Offset in hours and minutes or "Z" if the offset is 0                            |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+| XXX or XXXXX | +07:00 or Z     |  Offset in hours and minutes or "Z" if the offset is 0                            |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+| X            | 7               |  Offset in hours                                                                  |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+| xx or xxxx   | 700             |  Offset in hours and minutes                                                      |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+| xxx or xxxxx | +07:00          |  Offset in hours and minutes                                                      |
++--------------+-----------------+-----------------------------------------------------------------------------------+
+
+
+Aggregation Functions
+~~~~~~~~~~~~~~~~~~~~~
+
+``count()`` : return integer according to number of rows matching condition(if such exist).
+
+``sum(expression)`` : return a summary of expression per all rows matching condition(if such exist).
+
+``avg(expression)`` : return a average  of expression per all rows matching condition(if such exist).
+
+``max(expression)`` : return the maximal result for all expressions matching condition(if such exist).
+
+``min(expression)`` : return the minimal result for all expressions matching condition(if such exist).
+
+String Functions
+~~~~~~~~~~~~~~~~
+
+``substring(string,from,to)`` : substring( string ``from`` start [ ``for`` length ] )
+return a string extract from input string according to from,to inputs.
+``substring(string from )`` 
+``substring(string from for)`` 
+
+``char_length`` : return a number of characters in string (``character_length`` does the same).
+
+``trim`` : trim ( [[``leading`` | ``trailing`` | ``both`` remove_chars] ``from``] string )
+trims leading/trailing(or both) characters from target string, the default is blank character.
+
+``upper\lower`` : converts characters into lowercase/uppercase.
+
+SQL Limit Operator
+~~~~~~~~~~~~~~~~~~
+
+The SQL LIMIT operator is used to limit the number of rows processed by the query.
+Upon reaching the limit set by the user, the RGW stops fetching additional chunks.
+TODO : add examples, for aggregation and non-aggregation queries.
+
+Alias
+~~~~~
+**Alias** programming-construct is an essential part of s3-select language, it enables much better programming especially with objects containing many columns or in the case of complex queries.
+    
+Upon parsing the statement containing alias construct, it replaces alias with reference to correct projection column, on query execution time the reference is evaluated as any other expression.
+
+There is a risk that self(or cyclic) reference may occur causing stack-overflow(endless-loop), for that concern upon evaluating an alias, it is validated for cyclic reference.
+    
+Alias also maintains a result cache, meaning that successive uses of a given alias do not evaluate the expression again.  The result is instead returned from the cache.
+
+With each new row the cache is invalidated as the results may then differ.
+
+Testing
+~~~~~~~
+    
+``s3select`` contains several testing frameworks which provide a large coverage for its functionalities.
+
+(1) Tests comparison against a trusted engine, meaning,  C/C++ compiler is a trusted expression evaluator, 
+since the syntax for arithmetical and logical expressions are identical (s3select compare to C) 
+the framework runs equal expressions and validates their results.
+A dedicated expression generator produces different sets of expressions per each new test session. 
+
+(2) Compares results of queries whose syntax is different but which are semantically equivalent.
+This kind of test validates that different runtime flows produce an identical result 
+on each run with a different, random dataset.
+
+For example, on a dataset which contains a random numbers(1-1000)
+the following queries will produce identical results.
+``select count(*) from s3object where char_length(_3)=3;``
+``select count(*) from s3object where cast(_3 as int)>99 and cast(_3 as int)<1000;``
+
+(3) Constant dataset, the conventional way of testing. A query is processing a constant dataset, its result is validated against constant results.   
+
+Additional Syntax Support
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+S3select syntax supports table-alias ``select s._1 from s3object s where s._2 = ‘4’;``
+
+S3select syntax supports case insensitive ``Select SUM(Cast(_1 as int)) FROM S3Object;``
+
+S3select syntax supports statements without closing semicolon  ``select count(*) from s3object``
+
+
+Sending Query to RGW
+--------------------
+
+Any HTTP client can send an ``s3-select`` request to RGW, which must be compliant with `AWS Request syntax <https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#API_SelectObjectContent_RequestSyntax>`_.
+
+
+
+When sending an ``s3-select`` request to RGW using AWS CLI, clients must follow `AWS command reference <https://docs.aws.amazon.com/cli/latest/reference/s3api/select-object-content.html>`_.
+Below is an example:
+
+::
+
+ aws --endpoint-url http://localhost:8000 s3api select-object-content 
+  --bucket {BUCKET-NAME}  
+  --expression-type 'SQL'
+  --scan-range '{"Start" : 1000, "End" : 1000000}' 
+  --input-serialization 
+  '{"CSV": {"FieldDelimiter": "," , "QuoteCharacter": "\"" , "RecordDelimiter" : "\n" , "QuoteEscapeCharacter" : "\\" , "FileHeaderInfo": "USE" }, "CompressionType": "NONE"}' 
+  --output-serialization '{"CSV": {"FieldDelimiter": ":", "RecordDelimiter":"\t", "QuoteFields": "ALWAYS"}}' 
+  --key {OBJECT-NAME}
+  --request-progress '{"Enabled": True}'
+  --expression "select count(0) from s3object where int(_1)<10;" output.csv
+
+Input Serialization
+~~~~~~~~~~~~~~~~~~~
+
+**FileHeaderInfo** -> (string)
+Describes the first line of input. Valid values are:
+ 
+**NONE** : The first line is not a header.
+**IGNORE** : The first line is a header, but you can't use the header values to indicate the column in an expression.      
+it's possible to use column position (such as _1, _2, …) to indicate the column (``SELECT s._1 FROM S3OBJECT s``).
+**USE** : First line is a header, and you can use the header value to identify a column in an expression (``SELECT column_name FROM S3OBJECT``).
+
+**QuoteEscapeCharacter** -> (string) 
+A single character used for escaping the quotation mark character inside an already escaped value.
+
+**RecordDelimiter** -> (string) 
+A single character is used to separate individual records in the input. Instead of the default value, you can specify an arbitrary delimiter.
+
+**FieldDelimiter** -> (string) 
+A single character is used to separate individual fields in a record. You can specify an arbitrary delimiter.
+
+Output Serialization
+~~~~~~~~~~~~~~~~~~~~
+
+**AWS CLI example**
+
+    aws s3api select-object-content \
+    --bucket "mybucket" \
+    --key keyfile1 \
+    --expression "SELECT * FROM s3object s" \
+    --expression-type 'SQL' \
+    --request-progress '{"Enabled": false}' \
+    --input-serialization '{"CSV": {"FieldDelimiter": ","}, "CompressionType": "NONE"}' \
+    --output-serialization '{"CSV": {"FieldDelimiter": ":", "RecordDelimiter":"\\t", "QuoteFields": "ALWAYS"}}' /dev/stdout
+    
+    **QuoteFields** -> (string)
+    Indicates whether to use quotation marks around output fields.
+    **ALWAYS**: Always use quotation marks for output fields.
+    **ASNEEDED** (not implemented): Use quotation marks for output fields when needed.
+   
+    **RecordDelimiter** -> (string)
+    A single character is used to separate individual records in the output. Instead of the default value, you can specify an        
+    arbitrary delimiter.
+    
+    **FieldDelimiter** -> (string)
+    The value used to separate individual fields in a record. You can specify an arbitrary delimiter.
+
+Scan Range Option
+~~~~~~~~~~~~~~~~~
+
+    The scan range option to AWS-CLI enables the client to scan and process only a selected part of the object. 
+    This option reduces input/output operations and bandwidth by skipping parts of the object that are not of interest.
+    TODO : different data-sources (CSV, JSON, Parquet)
+
+CSV Parsing Behavior
+--------------------
+
+     The ``s3-select`` engine contains a CSV parser, which parses s3-objects as follows.   
+     - Each row ends with ``row-delimiter``.
+     - ``field-separator`` separates adjacent columns, successive instances of ``field separator`` define a NULL column.
+     - ``quote-character`` overrides ``field separator``, meaning that ``field separator`` is treated like any character between quotes.
+     - ``escape character`` disables interpretation of special characters, except for ``row delimiter``.
+    
+     Below are examples of CSV parsing rules.
+
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Feature                         | Description     | input ==> tokens                                                      |
++=================================+=================+=======================================================================+
+|     NULL                        | successive      | ,,1,,2,    ==> {null}{null}{1}{null}{2}{null}                         |
+|                                 | field delimiter |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     QUOTE                       | quote character | 11,22,"a,b,c,d",last ==> {11}{22}{"a,b,c,d"}{last}                    |
+|                                 | overrides       |                                                                       |
+|                                 | field delimiter |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     Escape                      | escape char     | 11,22,str=\\"abcd\\"\\,str2=\\"123\\",last                            |
+|                                 | overrides       | ==> {11}{22}{str="abcd",str2="123"}{last}                             |
+|                                 | meta-character. |                                                                       |
+|                                 | escape removed  |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     row delimiter               | no close quote, | 11,22,a="str,44,55,66                                                 |
+|                                 | row delimiter is| ==> {11}{22}{a="str,44,55,66}                                         |
+|                                 | closing line    |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     csv header info             | FileHeaderInfo  | "**USE**" value means each token on first line is column-name,        |
+|                                 | tag             | "**IGNORE**" value means to skip the first line                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+       
+
+JSON
+--------------------
+
+A JSON reader has been integrated with the ``s3select-engine``, which allows the client to use SQL statements to scan and extract information from JSON documents. 
+It should be noted that the data readers and parsers for CSV, Parquet, and JSON documents are separated from the SQL engine itself, so all of these readers use the same SQL engine.
+
+It's important to note that values in a JSON document can be nested in various ways, such as within objects or arrays.
+These objects and arrays can be nested within each other without any limitations.
+When using SQL to query a specific value in a JSON document, the client must specify the location of the value
+via a path in the SELECT statement.
+
+The SQL engine processes the SELECT statement in a row-based fashion.
+It uses the columns specified in the statement to perform its projection calculation, and each row contains values for these columns.
+In other words, the SQL engine processes each row one at a time (and aggregates results), using the values in the columns to perform SQL calculations.
+However, the generic structure of a JSON document does not have a row-and-column structure like CSV or Parquet.
+Instead, it is the SQL statement itself that defines the rows and columns when querying a JSON document.
+
+When querying JSON documents using SQL, the FROM clause in the SELECT statement defines the row boundaries.
+A row in a JSON document should be similar to how the row delimiter is used to define rows when querying CSV objects, and how row groups are used to define rows when querying Parquet objects.
+The statement "SELECT ... FROM s3object[*].aaa.bb.cc" instructs the reader to search for the path "aaa.bb.cc" and defines the row boundaries based on the occurrence of this path.
+A row begins when the reader encounters the path, and it ends when the reader exits the innermost part of the path, which in this case is the object "cc".
+
+NOTE : The semantics of querying JSON document may change and may not be the same as the current methodology described.
+
+TODO : relevant example for object and array values.
+
+A JSON Query Example
+--------------------
+
+::
+
+ {
+  "firstName": "Joe",
+  "lastName": "Jackson",
+  "gender": "male",
+  "age": "twenty",
+  "address": {
+  "streetAddress": "101",
+  "city": "San Diego",
+  "state": "CA"
+  },
+
+  "firstName": "Joe_2",
+  "lastName": "Jackson_2",
+  "gender": "male",
+  "age": 21,
+  "address": {
+  "streetAddress": "101",
+  "city": "San Diego",
+  "state": "CA"
+  },
+
+  "phoneNumbers": [
+    { "type": "home1", "number": "734928_1","addr": 11 },
+    { "type": "home2", "number": "734928_2","addr": 22 },
+    { "type": "home3", "number": "734928_3","addr": 33 },
+    { "type": "home4", "number": "734928_4","addr": 44 },
+    { "type": "home5", "number": "734928_5","addr": 55 },
+    { "type": "home6", "number": "734928_6","addr": 66 },
+    { "type": "home7", "number": "734928_7","addr": 77 },
+    { "type": "home8", "number": "734928_8","addr": 88 },
+    { "type": "home9", "number": "734928_9","addr": 99 },
+    { "type": "home10", "number": "734928_10","addr": 100 }
+  ],
+
+  "key_after_array": "XXX",
+
+  "description" : {
+    "main_desc" : "value_1",
+    "second_desc" : "value_2"
+  }
+ }
+
+  # the from-clause define a single row.
+  # _1 points to root object level.
+  # _1.age appears twice in Documnet-row, the last value is used for the operation.  
+  query = "select _1.firstname,_1.key_after_array,_1.age+4,_1.description.main_desc,_1.description.second_desc from s3object[*];";
+  expected_result = Joe_2,XXX,25,value_1,value_2
+
+
+  # the from-clause points the phonenumbers array (it defines the _1)
+  # each element in phoneNumbers array define a row. 
+  # in this case each element is an object contains 3 keys/values.
+  # the query "can not access" values outside phonenumbers array, the query can access only values appears on _1.phonenumbers path.
+  query = "select cast(substring(_1.number,1,6) as int) *10 from s3object[*].phonenumbers where _1.type='home2';";
+  expected_result = 7349280  
+
+
+BOTO3
+-----
+
+using BOTO3 is "natural" and easy due to AWS-cli support. 
+
+::
+
+ import pprint
+
+ def run_s3select(bucket,key,query,column_delim=",",row_delim="\n",quot_char='"',esc_char='\\',csv_header_info="NONE"):
+
+    s3 = boto3.client('s3',
+        endpoint_url=endpoint,
+        aws_access_key_id=access_key,
+        region_name=region_name,
+        aws_secret_access_key=secret_key)
+
+    result = ""
+    try:
+        r = s3.select_object_content(
+        Bucket=bucket,
+        Key=key,
+        ExpressionType='SQL',
+        InputSerialization = {"CSV": {"RecordDelimiter" : row_delim, "FieldDelimiter" : column_delim,"QuoteEscapeCharacter": esc_char, "QuoteCharacter": quot_char, "FileHeaderInfo": csv_header_info}, "CompressionType": "NONE"},
+        OutputSerialization = {"CSV": {}},
+        Expression=query,
+        RequestProgress = {"Enabled": progress})
+
+    except ClientError as c:
+        result += str(c)
+        return result
+
+    for event in r['Payload']:
+            if 'Records' in event:
+                result = ""
+                records = event['Records']['Payload'].decode('utf-8')
+                result += records
+            if 'Progress' in event:
+                print("progress")
+                pprint.pprint(event['Progress'],width=1)
+            if 'Stats' in event:
+                print("Stats")
+                pprint.pprint(event['Stats'],width=1)
+            if 'End' in event:
+                print("End")
+                pprint.pprint(event['End'],width=1)
+
+    return result
+
+
+
+
+  run_s3select(
+  "my_bucket",
+  "my_csv_object",
+  "select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from s3object where a3>100 and a3<300;")
+
+
+S3 SELECT Responses
+-------------------
+
+Error Response
+~~~~~~~~~~~~~~
+
+::
+
+   <?xml version="1.0" encoding="UTF-8"?>
+   <Error>
+     <Code>NoSuchKey</Code>
+     <Message>The resource you requested does not exist</Message>
+     <Resource>/mybucket/myfoto.jpg</Resource> 
+     <RequestId>4442587FB7D0A2F9</RequestId>
+   </Error>
+
+Report Response
+~~~~~~~~~~~~~~~
+::
+
+   HTTP/1.1 200
+   <?xml version="1.0" encoding="UTF-8"?>
+   <Payload>
+      <Records>
+         <Payload>blob</Payload>
+      </Records>
+      <Stats>
+         <Details>
+            <BytesProcessed>long</BytesProcessed>
+            <BytesReturned>long</BytesReturned>
+            <BytesScanned>long</BytesScanned>
+         </Details>
+      </Stats>
+      <Progress>
+         <Details>
+            <BytesProcessed>long</BytesProcessed>
+            <BytesReturned>long</BytesReturned>
+            <BytesScanned>long</BytesScanned>
+         </Details>
+      </Progress>
+      <Cont>
+      </Cont>
+      <End>
+      </End>
+   </Payload>
+
+Response Description
+~~~~~~~~~~~~~~~~~~~~
+
+For CEPH S3 Select, responses can be messages of the following types:
+
+**Records message**: Can contain a single record, partial records, or multiple records. Depending on the size of the result, a response can contain one or more of these messages.
+
+**Error message**: Upon an error being detected, RGW returns 400 Bad Request, and a specific error message sends back to the client, according to its type.
+
+**Continuation message**: Ceph S3 periodically sends this message to keep the TCP connection open.
+These messages appear in responses at random. The client must detect the message type and process it accordingly.
+
+**Progress message**: Ceph S3 periodically sends this message if requested. It contains information about the progress of a query that has started but has not yet been completed.  
+
+**Stats message**: Ceph S3 sends this message at the end of the request. It contains statistics about the query.
+
+**End message**: Indicates that the request is complete, and no more messages will be sent. You should not assume that request is complete until the client receives an End message.
diff --git a/doc/radosgw/session-tags.rst b/doc/radosgw/session-tags.rst
new file mode 100644
index 000000000..46722c382
--- /dev/null
+++ b/doc/radosgw/session-tags.rst
@@ -0,0 +1,427 @@
+=======================================================
+Session tags for Attribute Based Access Control in STS
+=======================================================
+
+Session tags are key-value pairs that can be passed while federating a user (currently it
+is only supported as part of the web token passed to AssumeRoleWithWebIdentity). The session
+tags are passed along as aws:PrincipalTag in the session credentials (temporary credentials)
+that is returned back by STS. These Principal Tags consists of the session tags that come in
+as part of the web token and the tags that are attached to the role being assumed. Please note
+that the tags have to be always specified in the following namespace: https://aws.amazon.com/tags.
+
+An example of the session tags that are passed in by the IDP in the web token is as follows:
+
+.. code-block:: python
+
+    {
+        "jti": "947960a3-7e91-4027-99f6-da719b0d4059",
+        "exp": 1627438044,
+        "nbf": 0,
+        "iat": 1627402044,
+        "iss": "http://localhost:8080/auth/realms/quickstart",
+        "aud": "app-profile-jsp",
+        "sub": "test",
+        "typ": "ID",
+        "azp": "app-profile-jsp",
+        "auth_time": 0,
+        "session_state": "3a46e3e7-d198-4a64-8b51-69682bcfc670",
+        "preferred_username": "test",
+        "email_verified": false,
+        "acr": "1",
+        "https://aws.amazon.com/tags": [
+            {
+                "principal_tags": {
+                    "Department": [
+                        "Engineering",
+                        "Marketing"
+                    ]
+                }
+            }
+        ],
+        "client_id": "app-profile-jsp",
+        "username": "test",
+        "active": true
+    }
+
+Steps to configure Keycloak to pass tags in the web token are described here:
+:ref:`radosgw_keycloak`.
+
+The trust policy must have 'sts:TagSession' permission if the web token passed
+in by the federated user contains session tags, otherwise the
+AssumeRoleWithWebIdentity action will fail. An example of the trust policy with
+sts:TagSession is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"localhost:8080/auth/realms/quickstart:sub":"test"}}
+	    }]
+	}
+
+Tag Keys
+========
+
+The following are the tag keys that can be used in the role's trust policy or the role's permission policy:
+
+1. aws:RequestTag: This key is used to compare the key-value pair passed in the request with the key-value pair
+in the role's trust policy. In case of AssumeRoleWithWebIdentity, the session tags that are passed by the idp
+in the web token can be used as aws:RequestTag in the role's trust policy based on which a federated user can be
+allowed to assume a role.
+
+An example of a role trust policy that uses aws:RequestTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"aws:RequestTag/Department":"Engineering"}}
+	    }]
+	}
+
+2. aws:PrincipalTag: This key is used to compare the key-value pair attached to the principal with the key-value pair
+in the policy. In case of AssumeRoleWithWebIdentity, the session tags that are passed by the idp in the web token appear
+as Principal tags in the temporary credentials once a user has been authenticated, and these tags can be used as
+aws:PrincipalTag in the role's permission policy.
+
+An example of a role permission policy that uses aws:PrincipalTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["s3:*"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket","arn:aws:s3::t1tenant:my-test-bucket/*],"+
+	        "Condition":{"StringEquals":{"aws:PrincipalTag/Department":"Engineering"}}
+	    }]
+	}
+
+3. iam:ResourceTag: This key is used to compare the key-value pair attached to the resource with the key-value pair
+in the policy. In case of AssumeRoleWithWebIdentity, tags attached to the role can be used to compare with that in
+the trust policy to allow a user to assume a role.
+RGW now supports REST APIs for tagging, listing tags and untagging actions on a role. More information related to
+role tagging can be found here :doc:`role`.
+
+An example of a role's trust policy that uses aws:ResourceTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"iam:ResourceTag/Department":"Engineering"}}
+	    }]
+	}
+
+For the above to work, you need to attach 'Department=Engineering' tag to the role.
+
+4. aws:TagKeys: This key is used to compare tags in the request with the tags in the policy. In case of
+AssumeRoleWithWebIdentity this can be used to check the tag keys in a role's trust policy before a user
+is allowed to assume a role.
+This can also be used in the role's permission policy.
+
+An example of a role's trust policy that uses aws:TagKeys is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"ForAllValues:StringEquals":{"aws:TagKeys":["Department"]}}
+	    }]
+	}
+
+'ForAllValues:StringEquals' tests whether every tag key in the request is a subset of the tag keys in the policy. So the above
+condition restricts the tag keys passed in the request.
+
+5. s3:ResourceTag: This key is used to compare tags present on the s3 resource (bucket or object) with the tags in
+the role's permission policy.
+
+An example of a role's permission policy that uses s3:ResourceTag is as follows:
+
+.. code-block:: python
+
+    {
+        "Version":"2012-10-17",
+        "Statement":[
+        {
+            "Effect":"Allow",
+            "Action":["s3:PutBucketTagging"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket\","arn:aws:s3::t1tenant:my-test-bucket/*"]
+        },
+        {
+            "Effect":"Allow",
+            "Action":["s3:*"],
+            "Resource":["*"],
+            "Condition":{"StringEquals":{"s3:ResourceTag/Department":\"Engineering"}}
+        }
+    }
+
+For the above to work, you need to attach 'Department=Engineering' tag to the bucket (and on the object too) on which you want this policy
+to be applied.
+
+More examples of policies using tags
+====================================
+
+1. To assume a role by matching the tags in the incoming request with the tag attached to the role.
+aws:RequestTag is the incoming tag in the JWT (access token) and iam:ResourceTag is the tag attached to the role being assumed:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"aws:RequestTag/Department":"${iam:ResourceTag/Department}"}}
+	    }]
+	}
+
+2. To evaluate a role's permission policy by matching principal tags with s3 resource tags.
+aws:PrincipalTag is the tag passed in along with the temporary credentials and s3:ResourceTag is the tag attached to
+the s3 resource (object/ bucket):
+
+.. code-block:: python
+
+
+    {
+        "Version":"2012-10-17",
+        "Statement":[
+        {
+            "Effect":"Allow",
+            "Action":["s3:PutBucketTagging"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket\","arn:aws:s3::t1tenant:my-test-bucket/*"]
+        },
+        {
+            "Effect":"Allow",
+            "Action":["s3:*"],
+            "Resource":["*"],
+            "Condition":{"StringEquals":{"s3:ResourceTag/Department":"${aws:PrincipalTag/Department}"}}
+        }
+    }
+
+Properties of Session Tags
+==========================
+
+1. Session Tags can be multi-valued. (Multi-valued session tags are not supported in AWS)
+2. A maximum of 50 session tags are allowed to be passed in by the IDP.
+3. The maximum size of a key allowed is 128 characters.
+4. The maximum size of a value allowed is 256 characters.
+5. The tag or the value can not start with "aws:".
+
+s3 Resource Tags
+================
+
+As stated above 's3:ResourceTag' key can be used for authorizing an s3 operation in RGW (this is not allowed in AWS).
+
+s3:ResourceTag is a key used to refer to tags that have been attached to an object or a bucket. Tags can be attached to an object or
+a bucket using REST APIs available for the same.
+
+The following table shows which s3 resource tag type (bucket/object) are supported for authorizing a particular operation.
+
++-----------------------------------+-------------------+
+| Operation                         | Tag type          |
++===================================+===================+
+| **GetObject**                     | Object tags       |
+| **GetObjectTags**                 |                   |
+| **DeleteObjectTags**              |                   |
+| **DeleteObject**                  |                   |
+| **PutACLs**                       |                   |
+| **InitMultipart**                 |                   |
+| **AbortMultipart**                |                   |
+| **ListMultipart**                 |                   |
+| **GetAttrs**                      |                   |
+| **PutObjectRetention**            |                   |
+| **GetObjectRetention**            |                   |
+| **PutObjectLegalHold**            |                   |
+| **GetObjectLegalHold**            |                   |
++-----------------------------------+-------------------+
+| **PutObjectTags**                 | Bucket tags       |
+| **GetBucketTags**                 |                   |
+| **PutBucketTags**                 |                   |
+| **DeleteBucketTags**              |                   |
+| **GetBucketReplication**          |                   |
+| **DeleteBucketReplication**       |                   |
+| **GetBucketVersioning**           |                   |
+| **SetBucketVersioning**           |                   |
+| **GetBucketWebsite**              |                   |
+| **SetBucketWebsite**              |                   |
+| **DeleteBucketWebsite**           |                   |
+| **StatBucket**                    |                   |
+| **ListBucket**                    |                   |
+| **GetBucketLogging**              |                   |
+| **GetBucketLocation**             |                   |
+| **DeleteBucket**                  |                   |
+| **GetLC**                         |                   |
+| **PutLC**                         |                   |
+| **DeleteLC**                      |                   |
+| **GetCORS**                       |                   |
+| **PutCORS**                       |                   |
+| **GetRequestPayment**             |                   |
+| **SetRequestPayment**             |                   |
+| **PutBucketPolicy**               |                   |
+| **GetBucketPolicy**               |                   |
+| **DeleteBucketPolicy**            |                   |
+| **PutBucketObjectLock**           |                   |
+| **GetBucketObjectLock**           |                   |
+| **GetBucketPolicyStatus**         |                   |
+| **PutBucketPublicAccessBlock**    |                   |
+| **GetBucketPublicAccessBlock**    |                   |
+| **DeleteBucketPublicAccessBlock** |                   |
++-----------------------------------+-------------------+
+| **GetACLs**                       | Bucket tags for   |
+| **PutACLs**                       | bucket ACLs       |
+|                                   | Object tags for   |
+|                                   | object ACLs       |
++-----------------------------------+-------------------+
+| **PutObject**                     | Object tags of    |
+| **CopyObject**                    | source object     |
+|                                   | Bucket tags of    |
+|                                   | destination bucket|
++-----------------------------------+-------------------+
+
+
+Sample code demonstrating usage of session tags
+===============================================
+
+The following is a sample code for tagging a role, a bucket, an object in it and using tag keys in a role's
+trust policy and its permission policy, assuming that a tag 'Department=Engineering' is passed in the
+JWT (access token) by the IDP
+
+.. code-block:: python
+
+    # -*- coding: utf-8 -*-
+
+    import boto3
+    import json
+    from nose.tools import eq_ as eq
+
+    access_key = 'TESTER'
+    secret_key = 'test123'
+    endpoint = 'http://s3.us-east.localhost:8000'
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = access_key,
+    aws_secret_access_key = secret_key,
+    endpoint_url = endpoint,
+    region_name='',)
+
+    s3res = boto3.resource('s3',
+            aws_access_key_id = access_key,
+            aws_secret_access_key = secret_key,
+            endpoint_url = endpoint,
+            region_name='',)
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url=endpoint,
+    region_name=''
+    )
+
+    bucket_name = 'test-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+
+    bucket_tagging = s3res.BucketTagging(bucket_name)
+    Set_Tag = bucket_tagging.put(Tagging={'TagSet':[{'Key':'Department', 'Value': 'Engineering'}]})
+    try:
+        response = iam_client.create_open_id_connect_provider(
+            Url='http://localhost:8080/auth/realms/quickstart',
+            ClientIDList=[
+                'app-profile-jsp',
+                'app-jee-jsp'
+            ],
+            ThumbprintList=[
+                'F7D7B3515DD0D319DD219A43A9EA727AD6065287'
+        ]
+        )
+    except ClientError as e:
+        print ("Provider already exists")
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\",\"sts:TagSession\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/Department\":\"${iam:ResourceTag/Department}\"}}}]}"
+    role_response = ""
+
+    print ("\n Getting Role \n")
+
+    try:
+        role_response = iam_client.get_role(
+            RoleName='S3Access'
+        )
+        print (role_response)
+    except ClientError as e:
+        if e.response['Code'] == 'NoSuchEntity':
+            print ("\n Creating Role \n")
+            tags_list = [
+                {'Key':'Department','Value':'Engineering'},
+            ]
+            role_response = iam_client.create_role(
+                AssumeRolePolicyDocument=policy_document,
+                Path='/',
+                RoleName='S3Access',
+                Tags=tags_list,
+            )
+            print (role_response)
+        else:
+            print("Unexpected error: %s" % e)
+
+    role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\",\"Condition\":{\"StringEquals\":{\"s3:ResourceTag/Department\":[\"${aws:PrincipalTag/Department}\"]}}}}"
+
+    response = iam_client.put_role_policy(
+                RoleName='S3Access',
+                PolicyName='Policy1',
+                PolicyDocument=role_policy
+            )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id='abc',
+    aws_secret_access_key='def',
+    endpoint_url = endpoint,
+    region_name = '',
+    )
+
+
+    print ("\n Assuming Role with Web Identity\n")
+    response = sts_client.assume_role_with_web_identity(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=900,
+    WebIdentityToken='<web-token>')
+
+    s3client2 = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url='http://s3.us-east.localhost:8000',
+    region_name='',)
+
+    bucket_body = 'this is a test file'
+    tags = 'Department=Engineering'
+    key = "test-1.txt"
+    s3_put_obj = s3client2.put_object(Body=bucket_body, Bucket=bucket_name, Key=key, Tagging=tags)
+    eq(s3_put_obj['ResponseMetadata']['HTTPStatusCode'],200)
+
+    s3_get_obj = s3client2.get_object(Bucket=bucket_name, Key=key)
+    eq(s3_get_obj['ResponseMetadata']['HTTPStatusCode'],200)
diff --git a/doc/radosgw/swift.rst b/doc/radosgw/swift.rst
new file mode 100644
index 000000000..24abd7728
--- /dev/null
+++ b/doc/radosgw/swift.rst
@@ -0,0 +1,79 @@
+.. _radosgw swift:
+
+===============================
+ Ceph Object Gateway Swift API
+===============================
+
+Ceph supports a RESTful API that is compatible with the basic data access model of the `Swift API`_.
+
+API
+---
+
+.. toctree::
+   :maxdepth: 1
+
+   Authentication <swift/auth>
+   Service Ops <swift/serviceops>
+   Container Ops <swift/containerops>
+   Object Ops <swift/objectops>
+   Temp URL Ops <swift/tempurl>
+   Tutorial <swift/tutorial>
+   Java <swift/java>
+   Python <swift/python>
+   Ruby <swift/ruby>
+
+
+Features Support
+----------------
+
+The following table describes the support status for current Swift functional features:
+
++---------------------------------+-----------------+----------------------------------------+
+| Feature                         | Status          | Remarks                                |
++=================================+=================+========================================+
+| **Authentication**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Account Metadata**        | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Swift ACLs**                  | Supported       | Supports a subset of Swift ACLs        |
++---------------------------------+-----------------+----------------------------------------+
+| **List Containers**             | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Container**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Container**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Container Metadata**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Update Container Metadata**   | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Container Metadata**   | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **List Objects**                | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Static Website**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Large Object**         | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Copy Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object Metadata**         | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Update Object Metadata**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Expiring Objects**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Temporary URLs**              | Partial Support | No support for container-level keys    |
++---------------------------------+-----------------+----------------------------------------+
+| **Object Versioning**           | Partial Support | No support for ``X-History-Location``  |
++---------------------------------+-----------------+----------------------------------------+
+| **CORS**                        | Not Supported   |                                        |
++---------------------------------+-----------------+----------------------------------------+
+
+.. _Swift API: https://developer.openstack.org/api-ref/object-store/index.html
diff --git a/doc/radosgw/swift/auth.rst b/doc/radosgw/swift/auth.rst
new file mode 100644
index 000000000..12d6b23ff
--- /dev/null
+++ b/doc/radosgw/swift/auth.rst
@@ -0,0 +1,82 @@
+================
+ Authentication
+================
+
+Swift API requests that require authentication must contain an
+``X-Storage-Token`` authentication token in the request header. 
+The token may be retrieved from RADOS Gateway, or from another authenticator.
+To obtain a token from RADOS Gateway, you must create a user. For example:: 
+
+    sudo radosgw-admin user create --subuser="{username}:{subusername}" --uid="{username}" 
+    --display-name="{Display Name}" --key-type=swift --secret="{password}" --access=full
+
+For details on RADOS Gateway administration, see `radosgw-admin`_. 
+
+.. _radosgw-admin: ../../../man/8/radosgw-admin/ 
+
+.. note::
+  For those used to the Swift API this is implementing the Swift auth v1.0 API, as such
+  `{username}` above is generally equivalent to a Swift `account` and `{subusername}`
+  is a user under that account.
+
+Auth Get
+--------
+
+To authenticate a user, make a request containing an ``X-Auth-User`` and a
+``X-Auth-Key`` in the header.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /auth HTTP/1.1
+    Host: swift.radosgwhost.com
+    X-Auth-User: johndoe
+    X-Auth-Key: R7UUOLFDI2ZI9PRCQ53K
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Auth-User`` 
+
+:Description: The key RADOS GW username to authenticate.
+:Type: String
+:Required: Yes
+
+``X-Auth-Key`` 
+
+:Description: The key associated to a RADOS GW username.
+:Type: String
+:Required: Yes
+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
+The response from the server should include an ``X-Auth-Token`` value. The 
+response may also contain a ``X-Storage-Url`` that provides the 
+``{api version}/{account}`` prefix that is specified in other requests
+throughout the API documentation.
+
+
+``X-Storage-Token`` 
+
+:Description: The authorization token for the ``X-Auth-User`` specified in the request.
+:Type: String
+
+
+``X-Storage-Url`` 
+
+:Description: The URL and ``{api version}/{account}`` path for the user.
+:Type: String
+
+A typical response looks like this:: 
+
+	HTTP/1.1 204 No Content
+	Date: Mon, 16 Jul 2012 11:05:33 GMT
+  	Server: swift
+  	X-Storage-Url: https://swift.radosgwhost.com/v1/ACCT-12345
+	X-Auth-Token: UOlCCC8TahFKlWuv9DB09TWHF0nDjpPElha0kAa
+	Content-Length: 0
+    Content-Type: text/plain; charset=UTF-8
diff --git a/doc/radosgw/swift/containerops.rst b/doc/radosgw/swift/containerops.rst
new file mode 100644
index 000000000..434b90ef5
--- /dev/null
+++ b/doc/radosgw/swift/containerops.rst
@@ -0,0 +1,341 @@
+======================
+ Container Operations
+======================
+
+A container is a mechanism for storing data objects. An account may
+have many containers, but container names must be unique. This API enables a 
+client to create a container, set access controls and metadata, 
+retrieve a container's contents, and delete a container. Since this API 
+makes requests related to information in a particular user's account, all 
+requests in this API must be authenticated unless a container's access control
+is deliberately made publicly accessible (i.e., allows anonymous requests).
+
+.. note:: The Amazon S3 API uses the term 'bucket' to describe a data container.
+   When you hear someone refer to a 'bucket' within the Swift API, the term 
+   'bucket' may be construed as the equivalent of the term 'container.'
+   
+One facet of object storage is that it does not support hierarchical paths
+or directories. Instead, it supports one level consisting of one or more
+containers, where each container may have objects. The RADOS Gateway's
+Swift-compatible API supports the notion of 'pseudo-hierarchical containers,'
+which is a means of using object naming to emulate a container (or directory)
+hierarchy without actually implementing one in the storage system. You may 
+name objects with pseudo-hierarchical names 
+(e.g., photos/buildings/empire-state.jpg), but container names cannot
+contain a forward slash (``/``) character.
+
+
+Create a Container
+==================
+
+To create a new container, make a ``PUT`` request with the API version, account,
+and the name of the new container. The container name must be unique, must not
+contain a forward-slash (/) character, and should be less than 256 bytes. You 
+may include access control headers and metadata headers in the request. The 
+operation is idempotent; that is, if you make a request to create a container
+that already exists, it will return with a HTTP 202 return code, but will not
+create another container.
+
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{api version}/{account}/{container} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Read: {comma-separated-uids}
+	X-Container-Write: {comma-separated-uids}
+	X-Container-Meta-{key}: {value}
+
+
+Headers
+~~~~~~~
+
+``X-Container-Read``
+
+:Description: The user IDs with read permissions for the container. 
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Write``
+
+:Description: The user IDs with write permissions for the container.
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If a container with the same name already exists, and the user is the
+container owner then the operation will succeed. Otherwise the operation
+will fail.
+
+``409``
+
+:Description: The container already exists under a different user's ownership.
+:Status Code: ``BucketAlreadyExists``
+
+
+
+
+List a Container's Objects
+==========================
+
+To list the objects within a container, make a ``GET`` request with the 
+API version, account, and the name of the container.  You can specify query 
+parameters to filter the full list, or leave out the parameters to return a list 
+of the first 10,000 object names stored in the container.
+
+
+Syntax
+~~~~~~
+
+::
+
+   GET /{api version}/{container} HTTP/1.1
+  	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Parameters
+~~~~~~~~~~
+
+``format``
+
+:Description: Defines the format of the result. 
+:Type: String
+:Valid Values: ``json`` | ``xml``
+:Required: No
+
+``prefix``
+
+:Description: Limits the result set to objects beginning with the specified prefix.
+:Type: String
+:Required: No
+
+``marker``
+
+:Description: Returns a list of results greater than the marker value.
+:Type: String
+:Required: No
+
+``limit``
+
+:Description: Limits the number of results to the specified value.
+:Type: Integer
+:Valid Range: 0 - 10,000
+:Required: No
+
+``delimiter``
+
+:Description: The delimiter between the prefix and the rest of the object name.
+:Type: String
+:Required: No
+
+``path``
+
+:Description: The pseudo-hierarchical path of the objects.
+:Type: String
+:Required: No
+
+``allow_unordered``
+
+:Description: Allows the results to be returned unordered to reduce computation overhead. Cannot be used with ``delimiter``.
+:Type: Boolean
+:Required: No
+:Non-Standard Extension: Yes
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``container``
+
+:Description: The container. 
+:Type: Container
+
+``object``
+
+:Description: An object within the container.
+:Type: Container
+
+``name``
+
+:Description: The name of an object within the container.
+:Type: String
+
+``hash``
+
+:Description: A hash code of the object's contents.
+:Type: String
+
+``last_modified``
+
+:Description: The last time the object's contents were modified.
+:Type: Date
+
+``content_type``
+
+:Description: The type of content within the object.
+:Type: String
+
+
+
+Update a Container's ACLs
+=========================
+
+When a user creates a container, the user has read and write access to the
+container by default. To allow other users to read a container's contents or
+write to a container, you must specifically enable the user. 
+You may also specify ``*`` in the ``X-Container-Read`` or ``X-Container-Write``
+settings, which effectively enables all users to either read from or write
+to the container. Setting ``*`` makes the container public. That is it 
+enables anonymous users to either read from or write to the container.
+
+.. note:: If you are planning to expose public read ACL functionality
+	  for the Swift API, it is strongly recommended to include the
+	  Swift account name in the endpoint definition, so as to most
+	  closely emulate the behavior of native OpenStack Swift. To
+	  do so, set the ``ceph.conf`` configuration option ``rgw
+	  swift account in url = true``, and update your Keystone
+	  endpoint to the URL suffix ``/v1/AUTH_%(tenant_id)s``
+	  (instead of just ``/v1``).
+
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Read: *
+	X-Container-Write: {uid1}, {uid2}, {uid3}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Container-Read``
+
+:Description: The user IDs with read permissions for the container. 
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Write``
+
+:Description: The user IDs with write permissions for the container.
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+
+Add/Update Container Metadata
+=============================
+
+To add metadata to a container, make a ``POST`` request with the API version, 
+account, and container name. You must have write permissions on the 
+container to add or update metadata.
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Meta-Color: red
+	X-Container-Meta-Taste: salty
+	
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Container-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+Enable Object Versioning for a Container
+========================================
+
+To enable object versioning a container, make a ``POST`` request with
+the API version, account, and container name. You must have write
+permissions on the container to add or update metadata.
+
+.. note:: Object versioning support is not enabled in radosgw by
+	  default; you must set ``rgw swift versioning enabled =
+	  true`` in ``ceph.conf`` to enable this feature.
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Versions-Location: {archive-container}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Versions-Location``
+
+:Description: The name of a container (the "archive container") that
+	      will be used to store versions of the objects in the
+	      container that the ``POST`` request is made on (the
+	      "current container"). The archive container need not
+	      exist at the time it is being referenced, but once
+	      ``X-Versions-Location`` is set on the current container,
+	      and object versioning is thus enabled, the archive
+	      container must exist before any further objects are
+	      updated or deleted in the current container.
+
+	      .. note:: ``X-Versions-Location`` is the only
+                        versioning-related header that radosgw
+                        interprets. ``X-History-Location``, supported
+                        by native OpenStack Swift, is currently not
+                        supported by radosgw.
+:Type: String
+:Required: No (if this header is passed with an empty value, object
+	   versioning on the current container is disabled, but the
+	   archive container continues to exist.)
+
+
+Delete a Container
+==================
+
+To delete a container, make a ``DELETE`` request with the API version, account,
+and the name of the container. The container must be empty. If you'd like to check 
+if the container is empty, execute a ``HEAD`` request against the container. Once 
+you have successfully removed the container, you will be able to reuse the container name.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{api version}/{account}/{container} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}    
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+``204``
+
+:Description: The container was removed.
+:Status Code: ``NoContent``
+
diff --git a/doc/radosgw/swift/java.rst b/doc/radosgw/swift/java.rst
new file mode 100644
index 000000000..8977a3b16
--- /dev/null
+++ b/doc/radosgw/swift/java.rst
@@ -0,0 +1,175 @@
+.. _java_swift:
+
+=====================
+ Java Swift Examples
+=====================
+
+Setup
+=====
+
+The following examples may require some or all of the following Java
+classes to be imported:
+
+.. code-block:: java
+
+       import org.javaswift.joss.client.factory.AccountConfig;
+       import org.javaswift.joss.client.factory.AccountFactory;
+       import org.javaswift.joss.client.factory.AuthenticationMethod;
+       import org.javaswift.joss.model.Account;
+       import org.javaswift.joss.model.Container;
+       import org.javaswift.joss.model.StoredObject;
+       import java.io.File;
+       import java.io.IOException;
+       import java.util.*;
+
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: java
+
+       String username = "USERNAME";
+       String password = "PASSWORD";
+       String authUrl  = "https://radosgw.endpoint/auth/1.0";
+
+       AccountConfig config = new AccountConfig();
+       config.setUsername(username);
+       config.setPassword(password);
+       config.setAuthUrl(authUrl);
+       config.setAuthenticationMethod(AuthenticationMethod.BASIC);
+       Account account = new AccountFactory(config).createAccount();
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       container.create();
+
+
+Create an Object
+================
+
+This creates an object ``foo.txt`` from the file named ``foo.txt`` in 
+the container ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.uploadObject(new File("foo.txt"));
+
+
+Add/Update Object Metadata
+==========================
+
+This adds the metadata key-value pair ``key``:``value`` to the object named
+``foo.txt`` in the container ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       Map<String, Object> metadata = new TreeMap<String, Object>();
+       metadata.put("key", "value");
+       object.setMetadata(metadata);
+
+
+List Owned Containers
+=====================
+
+This gets a list of Containers that you own.
+This also prints out the container name.
+
+.. code-block:: java
+
+       Collection<Container> containers = account.list();
+       for (Container currentContainer : containers) {
+           System.out.println(currentContainer.getName());
+       }
+
+The output will look something like this::
+
+	mahbuckat1
+	mahbuckat2
+	mahbuckat3
+
+
+List a Container's Content
+==========================
+
+This gets a list of objects in the container ``my-new-container``; and, it also 
+prints out each object's name, the file size, and last modified date:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       Collection<StoredObject> objects = container.list();
+       for (StoredObject currentObject : objects) {
+           System.out.println(currentObject.getName());
+       }
+
+The output will look something like this::
+
+   myphoto1.jpg
+   myphoto2.jpg
+
+
+Retrieve an Object's Metadata
+=============================
+
+This retrieves metadata and gets the MIME type for an object named ``foo.txt``
+in a container named ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       Map<String, Object> returnedMetadata = object.getMetadata();
+       for (String name : returnedMetadata.keySet()) {
+           System.out.println("META / "+name+": "+returnedMetadata.get(name));
+       }
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``foo.txt`` in the container ``my-new-container`` 
+and saves it in ``./outfile.txt``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.downloadObject(new File("outfile.txt"));
+
+
+Delete an Object
+================
+
+This deletes the object ``goodbye.txt`` in the container "my-new-container":
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.delete();
+
+
+Delete a Container
+==================
+
+This deletes a container named "my-new-container": 
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       container.delete();
+	
+.. note:: The container must be empty! Otherwise it won't work!
diff --git a/doc/radosgw/swift/objectops.rst b/doc/radosgw/swift/objectops.rst
new file mode 100644
index 000000000..fc8d21967
--- /dev/null
+++ b/doc/radosgw/swift/objectops.rst
@@ -0,0 +1,271 @@
+===================
+ Object Operations
+===================
+
+An object is a container for storing data and metadata. A container may
+have many objects, but the object names must be unique. This API enables a 
+client to create an object, set access controls and metadata, retrieve an 
+object's data and metadata, and delete an object. Since this API makes requests 
+related to information in a particular user's account, all requests in this API 
+must be authenticated unless the container or object's access control is 
+deliberately made publicly accessible (i.e., allows anonymous requests).
+
+
+Create/Update an Object
+=======================
+
+To create a new object, make a ``PUT`` request with the API version, account,
+container name and the name of the new object. You must have write permission
+on the container to create or update an object. The object name must be 
+unique within the container. The ``PUT`` request is not idempotent, so if you
+do not use a unique name, the request will update the object. However, you may
+use pseudo-hierarchical syntax in your object name to distinguish it from 
+another object of the same name if it is under a different pseudo-hierarchical 
+directory. You may include access control headers and metadata headers in the 
+request.
+
+
+Syntax
+~~~~~~
+
+::
+
+   PUT /{api version}/{account}/{container}/{object} HTTP/1.1 
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``ETag``
+
+:Description: An MD5 hash of the object's contents. Recommended. 
+:Type: String
+:Required: No
+
+
+``Content-Type``
+
+:Description: The type of content the object contains.
+:Type: String
+:Required: No
+
+
+``Transfer-Encoding``
+
+:Description: Indicates whether the object is part of a larger aggregate object.
+:Type: String
+:Valid Values: ``chunked``
+:Required: No
+
+
+Copy an Object
+==============
+
+Copying an object allows you to make a server-side copy of an object, so that
+you don't have to download it and upload it under another container/name.
+To copy the contents of one object to another object, you may make either a
+``PUT`` request or a ``COPY`` request with the API version, account, and the 
+container name. For a ``PUT`` request, use the destination container and object
+name in the request, and the source container and object in the request header.
+For a ``Copy`` request, use the source container and object in the request, and
+the destination container and object in the request header. You must have write 
+permission on the container to copy an object. The destination object name must be 
+unique within the container. The request is not idempotent, so if you do not use 
+a unique name, the request will update the destination object. However, you may 
+use pseudo-hierarchical syntax in your object name to distinguish the destination 
+object from the source object of the same name if it is under a different 
+pseudo-hierarchical directory. You may include access control headers and metadata 
+headers in the request.
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{api version}/{account}/{dest-container}/{dest-object} HTTP/1.1
+	X-Copy-From: {source-container}/{source-object}
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+or alternatively:
+
+::
+
+	COPY /{api version}/{account}/{source-container}/{source-object} HTTP/1.1
+	Destination: {dest-container}/{dest-object}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Copy-From``
+
+:Description: Used with a ``PUT`` request to define the source container/object path.
+:Type: String
+:Required: Yes, if using ``PUT``
+
+
+``Destination``
+
+:Description: Used with a ``COPY`` request to define the destination container/object path.
+:Type: String
+:Required: Yes, if using ``COPY``
+
+
+``If-Modified-Since``
+
+:Description: Only copies if modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+
+``If-Unmodified-Since``
+
+:Description: Only copies if not modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+``Copy-If-Match``
+
+:Description: Copies only if the ETag in the request matches the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+``Copy-If-None-Match``
+
+:Description: Copies only if the ETag in the request does not match the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+Delete an Object
+================
+
+To delete an object, make a ``DELETE`` request with the API version, account,
+container and object name. You must have write permissions on the container to delete
+an object within it. Once you have successfully deleted the object, you will be able to 
+reuse the object name.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Get an Object
+=============
+
+To retrieve an object, make a ``GET`` request with the API version, account,
+container and object name. You must have read permissions on the container to
+retrieve an object within it.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``range``
+
+:Description: To retrieve a subset of an object's contents, you may specify a byte range.
+:Type: Date
+:Required: No
+
+
+``If-Modified-Since``
+
+:Description: Only copies if modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+
+``If-Unmodified-Since``
+
+:Description: Only copies if not modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+``Copy-If-Match``
+
+:Description: Copies only if the ETag in the request matches the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+``Copy-If-None-Match``
+
+:Description: Copies only if the ETag in the request does not match the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
+``Content-Range``
+
+:Description: The range of the subset of object contents. Returned only if the range header field was specified in the request  
+
+
+Get Object Metadata
+===================
+
+To retrieve an object's metadata, make a ``HEAD`` request with the API version, 
+account, container and object name. You must have read permissions on the 
+container to retrieve metadata from an object within the container. This request
+returns the same header information as the request for the object itself, but
+it does not return the object's data.
+
+Syntax
+~~~~~~
+
+::
+
+	HEAD /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Add/Update Object Metadata
+==========================
+
+To add metadata to an object, make a ``POST`` request with the API version, 
+account, container and object name. You must have write permissions on the 
+parent container to add or update metadata.
+
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Object-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
diff --git a/doc/radosgw/swift/python.rst b/doc/radosgw/swift/python.rst
new file mode 100644
index 000000000..0b1f8d0da
--- /dev/null
+++ b/doc/radosgw/swift/python.rst
@@ -0,0 +1,114 @@
+.. _python_swift:
+
+=====================
+Python Swift Examples
+=====================
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: python
+
+	import swiftclient
+	user = 'account_name:username'
+	key = 'your_api_key'
+
+	conn = swiftclient.Connection(
+		user=user,
+		key=key,
+		authurl='https://objects.dreamhost.com/auth',
+	)
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``:
+
+.. code-block:: python
+
+	container_name = 'my-new-container'
+	conn.put_container(container_name)
+
+
+Create an Object
+================
+
+This creates a file ``hello.txt`` from the file named ``my_hello.txt``:
+
+.. code-block:: python
+
+	with open('hello.txt', 'r') as hello_file:
+		conn.put_object(container_name, 'hello.txt',
+						contents= hello_file.read(),
+						content_type='text/plain')
+
+
+List Owned Containers
+=====================
+
+This gets a list of containers that you own, and prints out the container name:
+
+.. code-block:: python
+
+	for container in conn.get_account()[1]:
+		print(container['name'])
+
+The output will look something like this::
+
+   mahbuckat1
+   mahbuckat2
+   mahbuckat3
+
+List a Container's Content
+==========================
+
+This gets a list of objects in the container, and prints out each
+object's name, the file size, and last modified date:
+
+.. code-block:: python
+
+	for data in conn.get_container(container_name)[1]:
+		print('{0}\t{1}\t{2}'.format(data['name'], data['bytes'], data['last_modified']))
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``hello.txt`` and saves it in
+``./my_hello.txt``:
+
+.. code-block:: python
+
+	obj_tuple = conn.get_object(container_name, 'hello.txt')
+	with open('my_hello.txt', 'w') as my_hello:
+		my_hello.write(obj_tuple[1])
+
+
+Delete an Object
+================
+
+This deletes the object ``hello.txt``:
+
+.. code-block:: python
+
+	conn.delete_object(container_name, 'hello.txt')
+
+Delete a Container
+==================
+
+.. note::
+
+   The container must be empty! Otherwise the request won't work!
+
+.. code-block:: python
+
+	conn.delete_container(container_name)
+
diff --git a/doc/radosgw/swift/ruby.rst b/doc/radosgw/swift/ruby.rst
new file mode 100644
index 000000000..a20b66d88
--- /dev/null
+++ b/doc/radosgw/swift/ruby.rst
@@ -0,0 +1,119 @@
+.. _ruby_swift:
+
+=====================
+ Ruby Swift Examples
+=====================
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: ruby
+
+	require 'cloudfiles'
+	username = 'account_name:user_name'
+	api_key  = 'your_secret_key'
+
+	conn = CloudFiles::Connection.new(
+		:username => username,
+		:api_key  => api_key,
+		:auth_url => 'http://objects.dreamhost.com/auth'
+	)
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``
+
+.. code-block:: ruby
+
+	container = conn.create_container('my-new-container')
+
+
+Create an Object
+================
+
+This creates a file ``hello.txt`` from the file named ``my_hello.txt``
+
+.. code-block:: ruby
+
+	obj = container.create_object('hello.txt')
+	obj.load_from_filename('./my_hello.txt')
+	obj.content_type = 'text/plain'
+
+
+
+List Owned Containers
+=====================
+
+This gets a list of Containers that you own, and also prints out 
+the container name:
+
+.. code-block:: ruby
+
+	conn.containers.each do |container|
+		puts container
+	end
+
+The output will look something like this::
+
+   mahbuckat1
+   mahbuckat2
+   mahbuckat3
+
+
+List a Container's Contents
+===========================
+
+This gets a list of objects in the container, and prints out each 
+object's name, the file size, and last modified date:
+
+.. code-block:: ruby
+
+	require 'date'  # not necessary in the next version
+
+	container.objects_detail.each do |name, data|
+		puts "#{name}\t#{data[:bytes]}\t#{data[:last_modified]}"
+	end
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``hello.txt`` and saves it in
+``./my_hello.txt``:
+
+.. code-block:: ruby
+
+	obj = container.object('hello.txt')
+	obj.save_to_filename('./my_hello.txt')
+
+
+Delete an Object
+================
+
+This deletes the object ``goodbye.txt``:
+
+.. code-block:: ruby
+
+	container.delete_object('goodbye.txt')
+	
+
+Delete a Container
+==================
+
+.. note::
+
+   The container must be empty! Otherwise the request won't work!
+
+.. code-block:: ruby
+
+	container.delete_container('my-new-container')
diff --git a/doc/radosgw/swift/serviceops.rst b/doc/radosgw/swift/serviceops.rst
new file mode 100644
index 000000000..a00f3d807
--- /dev/null
+++ b/doc/radosgw/swift/serviceops.rst
@@ -0,0 +1,76 @@
+====================
+ Service Operations
+====================
+
+To retrieve data about our Swift-compatible service, you may execute ``GET`` 
+requests using the ``X-Storage-Url`` value retrieved during authentication. 
+
+List Containers
+===============
+
+A ``GET`` request that specifies the API version and the account will return
+a list of containers for a particular user account. Since the request returns
+a particular user's containers, the request requires an authentication token.
+The request cannot be made anonymously.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{api version}/{account} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``limit``
+
+:Description: Limits the number of results to the specified value.
+:Type: Integer
+:Required: No
+
+``format``
+
+:Description: Defines the format of the result. 
+:Type: String
+:Valid Values: ``json`` | ``xml``
+:Required: No
+
+
+``marker``
+
+:Description: Returns a list of results greater than the marker value.
+:Type: String
+:Required: No
+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+The response contains a list of containers, or returns with an HTTP
+204 response code
+
+``account``
+
+:Description: A list for account information.
+:Type: Container
+
+``container``
+
+:Description: The list of containers.
+:Type: Container
+
+``name``
+
+:Description: The name of a container.
+:Type: String
+
+``bytes``
+
+:Description: The size of the container.
+:Type: Integer
\ No newline at end of file
diff --git a/doc/radosgw/swift/tempurl.rst b/doc/radosgw/swift/tempurl.rst
new file mode 100644
index 000000000..41dbb0ccb
--- /dev/null
+++ b/doc/radosgw/swift/tempurl.rst
@@ -0,0 +1,102 @@
+====================
+ Temp URL Operations
+====================
+
+To allow temporary access (for eg for `GET` requests) to objects
+without the need to share credentials, temp url functionality is
+supported by swift endpoint of radosgw. For this functionality,
+initially the value of `X-Account-Meta-Temp-URL-Key` and optionally
+`X-Account-Meta-Temp-URL-Key-2` should be set. The Temp URL
+functionality relies on a HMAC-SHA1 signature against these secret
+keys.
+
+.. note:: If you are planning to expose Temp URL functionality for the
+	  Swift API, it is strongly recommended to include the Swift
+	  account name in the endpoint definition, so as to most
+	  closely emulate the behavior of native OpenStack Swift. To
+	  do so, set the ``ceph.conf`` configuration option ``rgw
+	  swift account in url = true``, and update your Keystone
+	  endpoint to the URL suffix ``/v1/AUTH_%(tenant_id)s``
+	  (instead of just ``/v1``).
+
+
+POST Temp-URL Keys
+==================
+
+A ``POST`` request to the Swift account with the required key will set
+the secret temp URL key for the account, against which temporary URL
+access can be provided to accounts. Up to two keys are supported, and
+signatures are checked against both the keys, if present, so that keys
+can be rotated without invalidating the temporary URLs.
+
+.. note:: Native OpenStack Swift also supports the option to set
+          temporary URL keys at the container level, issuing a
+          ``POST`` or ``PUT`` request against a container that sets
+          ``X-Container-Meta-Temp-URL-Key`` or
+          ``X-Container-Meta-Temp-URL-Key-2``. This functionality is
+          not supported in radosgw; temporary URL keys can only be set
+          and used at the account level.
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{api version}/{account} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Account-Meta-Temp-URL-Key``
+
+:Description: A user-defined key that takes an arbitrary string value.
+:Type: String
+:Required: Yes
+
+``X-Account-Meta-Temp-URL-Key-2``
+
+:Description: A user-defined key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+GET Temp-URL Objects
+====================
+
+Temporary URL uses a cryptographic HMAC-SHA1 signature, which includes
+the following elements:
+
+#. The value of the Request method, "GET" for instance
+#. The expiry time, in format of seconds since the epoch, ie Unix time
+#. The request path starting from "v1" onwards
+
+The above items are normalized with newlines appended between them,
+and a HMAC is generated using the SHA-1 hashing algorithm against one
+of the Temp URL Keys posted earlier.
+
+A sample python script to demonstrate the above is given below:
+
+
+.. code-block:: python
+
+   import hmac
+   from hashlib import sha1
+   from time import time
+
+   method = 'GET'
+   host = 'https://objectstore.example.com/swift'
+   duration_in_seconds = 300  # Duration for which the url is valid
+   expires = int(time() + duration_in_seconds)
+   path = '/v1/your-bucket/your-object'
+   key = 'secret'
+   hmac_body = '%s\n%s\n%s' % (method, expires, path)
+   sig = hmac.new(key, hmac_body, sha1).hexdigest()
+   rest_uri = "{host}{path}?temp_url_sig={sig}&temp_url_expires={expires}".format(
+		host=host, path=path, sig=sig, expires=expires)
+   print(rest_uri)
+
+   # Example Output
+   # https://objectstore.example.com/swift/v1/your-bucket/your-object?temp_url_sig=ff4657876227fc6025f04fcf1e82818266d022c6&temp_url_expires=1423200992
+
diff --git a/doc/radosgw/swift/tutorial.rst b/doc/radosgw/swift/tutorial.rst
new file mode 100644
index 000000000..5d2889b19
--- /dev/null
+++ b/doc/radosgw/swift/tutorial.rst
@@ -0,0 +1,62 @@
+==========
+ Tutorial
+==========
+
+The Swift-compatible API tutorials follow a simple container-based object 
+lifecycle. The first step requires you to setup a connection between your 
+client and the RADOS Gateway server. Then, you may follow a natural 
+container and object lifecycle, including adding and retrieving object 
+metadata. See example code for the following languages:
+
+- `Java`_
+- `Python`_
+- `Ruby`_
+
+
+.. ditaa::
+
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |    Create a Connection     |------->|      Create a Container     |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |     Create an Object       |------->| Add/Update Object Metadata  |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |   List Owned Containers    |------->| List a Container's Contents |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           | Get an Object's Metadata   |------->|     Retrieve an Object      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |      Delete an Object      |------->|      Delete a Container     |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+
+.. _Java: ../java
+.. _Python: ../python
+.. _Ruby: ../ruby
diff --git a/doc/radosgw/sync-modules.rst b/doc/radosgw/sync-modules.rst
new file mode 100644
index 000000000..61797edc8
--- /dev/null
+++ b/doc/radosgw/sync-modules.rst
@@ -0,0 +1,97 @@
+============
+Sync Modules
+============
+
+.. versionadded:: Kraken
+
+The :ref:`multisite` functionality of RGW introduced in Jewel allowed the ability to
+create multiple zones and mirror data and metadata between them. ``Sync Modules``
+are built atop of the multisite framework that allows for forwarding data and
+metadata to a different external tier. A sync module allows for a set of actions
+to be performed whenever a change in data occurs (metadata ops like bucket or
+user creation etc. are also regarded as changes in data). As the rgw multisite
+changes are eventually consistent at remote sites, changes are propagated
+asynchronously. This would allow for unlocking use cases such as backing up the
+object storage to an external cloud cluster or a custom backup solution using
+tape drives, indexing metadata in ElasticSearch etc.
+
+A sync module configuration is local to a zone. The sync module determines
+whether the zone exports data or can only consume data that was modified in
+another zone. As of luminous the supported sync plugins are `elasticsearch`_,
+``rgw``, which is the default sync plugin that synchronizes data between the
+zones and ``log`` which is a trivial sync plugin that logs the metadata
+operation that happens in the remote zones. The following docs are written with
+the example of a zone using `elasticsearch sync module`_, the process would be similar
+for configuring any sync plugin
+
+.. toctree::
+   :maxdepth: 1
+
+   ElasticSearch Sync Module <elastic-sync-module>
+   Cloud Sync Module <cloud-sync-module>
+   Archive Sync Module <archive-sync-module>
+
+.. note ``rgw`` is the default sync plugin and there is no need to explicitly
+   configure this
+
+Requirements and Assumptions
+----------------------------
+
+Let us assume a simple multisite configuration as described in the :ref:`multisite`
+docs, of 2 zones ``us-east`` and ``us-west``, let's add a third zone
+``us-east-es`` which is a zone that only processes metadata from the other
+sites. This zone can be in the same or a different ceph cluster as ``us-east``.
+This zone would only consume metadata from other zones and RGWs in this zone
+will not serve any end user requests directly.
+
+
+Configuring Sync Modules
+------------------------
+
+Create the third zone similar to the :ref:`multisite` docs, for example
+
+::
+
+   # radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east-es \
+   --access-key={system-key} --secret={secret} --endpoints=http://rgw-es:80
+
+
+
+A sync module can be configured for this zone via the following
+
+::
+
+   # radosgw-admin zone modify --rgw-zone={zone-name} --tier-type={tier-type} --tier-config={set of key=value pairs}
+
+
+For example in the ``elasticsearch`` sync module
+
+::
+
+   # radosgw-admin zone modify --rgw-zone={zone-name} --tier-type=elasticsearch \
+                               --tier-config=endpoint=http://localhost:9200,num_shards=10,num_replicas=1
+
+
+For the various supported tier-config options refer to the `elasticsearch sync module`_ docs
+
+Finally update the period
+
+
+::
+
+    # radosgw-admin period update --commit
+
+
+Now start the radosgw in the zone
+
+::
+
+    # systemctl start ceph-radosgw@rgw.`hostname -s`
+    # systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+
+
+.. _`elasticsearch sync module`: ../elastic-sync-module
+.. _`elasticsearch`: ../elastic-sync-module
+.. _`cloud sync module`: ../cloud-sync-module
+.. _`archive sync module`: ../archive-sync-module
diff --git a/doc/radosgw/troubleshooting.rst b/doc/radosgw/troubleshooting.rst
new file mode 100644
index 000000000..4a084e82a
--- /dev/null
+++ b/doc/radosgw/troubleshooting.rst
@@ -0,0 +1,208 @@
+=================
+ Troubleshooting
+=================
+
+
+The Gateway Won't Start
+=======================
+
+If you cannot start the gateway (i.e., there is no existing ``pid``), 
+check to see if there is an existing ``.asok`` file from another 
+user. If an ``.asok`` file from another user exists and there is no
+running ``pid``, remove the ``.asok`` file and try to start the
+process again. This may occur when you start the process as a ``root`` user and 
+the startup script is trying to start the process as a 
+``www-data`` or ``apache`` user and an existing ``.asok`` is 
+preventing the script from starting the daemon.
+
+The radosgw init script (/etc/init.d/radosgw) also has a verbose argument that
+can provide some insight as to what could be the issue::
+
+  /etc/init.d/radosgw start -v
+
+or ::
+
+  /etc/init.d radosgw start --verbose
+
+HTTP Request Errors
+===================
+
+Examining the access and error logs for the web server itself is
+probably the first step in identifying what is going on.  If there is
+a 500 error, that usually indicates a problem communicating with the
+``radosgw`` daemon.  Ensure the daemon is running, its socket path is
+configured, and that the web server is looking for it in the proper
+location.
+
+
+Crashed ``radosgw`` process
+===========================
+
+If the ``radosgw`` process dies, you will normally see a 500 error
+from the web server (apache, nginx, etc.).  In that situation, simply
+restarting radosgw will restore service.
+
+To diagnose the cause of the crash, check the log in ``/var/log/ceph``
+and/or the core file (if one was generated).
+
+
+Blocked ``radosgw`` Requests
+============================
+
+If some (or all) radosgw requests appear to be blocked, you can get
+some insight into the internal state of the ``radosgw`` daemon via
+its admin socket.  By default, there will be a socket configured to
+reside in ``/var/run/ceph``, and the daemon can be queried with::
+
+ ceph daemon /var/run/ceph/client.rgw help
+ 
+ help                list available commands
+ objecter_requests   show in-progress osd requests
+ perfcounters_dump   dump perfcounters value
+ perfcounters_schema dump perfcounters schema
+ version             get protocol version
+
+Of particular interest::
+
+ ceph daemon /var/run/ceph/client.rgw objecter_requests
+ ...
+
+will dump information about current in-progress requests with the
+RADOS cluster.  This allows one to identify if any requests are blocked
+by a non-responsive OSD.  For example, one might see::
+
+  { "ops": [
+        { "tid": 1858,
+          "pg": "2.d2041a48",
+          "osd": 1,
+          "last_sent": "2012-03-08 14:56:37.949872",
+          "attempts": 1,
+          "object_id": "fatty_25647_object1857",
+          "object_locator": "@2",
+          "snapid": "head",
+          "snap_context": "0=[]",
+          "mtime": "2012-03-08 14:56:37.949813",
+          "osd_ops": [
+                "write 0~4096"]},
+        { "tid": 1873,
+          "pg": "2.695e9f8e",
+          "osd": 1,
+          "last_sent": "2012-03-08 14:56:37.970615",
+          "attempts": 1,
+          "object_id": "fatty_25647_object1872",
+          "object_locator": "@2",
+          "snapid": "head",
+          "snap_context": "0=[]",
+          "mtime": "2012-03-08 14:56:37.970555",
+          "osd_ops": [
+                "write 0~4096"]}],
+  "linger_ops": [],
+  "pool_ops": [],
+  "pool_stat_ops": [],
+  "statfs_ops": []}
+
+In this dump, two requests are in progress.  The ``last_sent`` field is
+the time the RADOS request was sent.  If this is a while ago, it suggests
+that the OSD is not responding.  For example, for request 1858, you could
+check the OSD status with::
+
+ ceph pg map 2.d2041a48
+ 
+ osdmap e9 pg 2.d2041a48 (2.0) -> up [1,0] acting [1,0]
+
+This tells us to look at ``osd.1``, the primary copy for this PG::
+
+ ceph daemon osd.1 ops
+ { "num_ops": 651,
+  "ops": [
+        { "description": "osd_op(client.4124.0:1858 fatty_25647_object1857 [write 0~4096] 2.d2041a48)",
+          "received_at": "1331247573.344650",
+          "age": "25.606449",
+          "flag_point": "waiting for sub ops",
+          "client_info": { "client": "client.4124",
+              "tid": 1858}},
+ ...
+
+The ``flag_point`` field indicates that the OSD is currently waiting
+for replicas to respond, in this case ``osd.0``.
+
+
+Java S3 API Troubleshooting
+===========================
+
+
+Peer Not Authenticated
+----------------------
+
+You may receive an error that looks like this:: 
+
+     [java] INFO: Unable to execute HTTP request: peer not authenticated
+
+The Java SDK for S3 requires a valid certificate from a recognized certificate
+authority, because it uses HTTPS by default. If you are just testing the Ceph
+Object Storage services, you can resolve this problem in a few ways:  
+
+#. Prepend the IP address or hostname with ``http://``. For example, change this::
+
+	conn.setEndpoint("myserver");
+
+   To:: 
+
+	conn.setEndpoint("http://myserver")
+
+#. After setting your credentials, add a client configuration and set the 
+   protocol to ``Protocol.HTTP``. :: 
+
+			AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+			
+			ClientConfiguration clientConfig = new ClientConfiguration();
+			clientConfig.setProtocol(Protocol.HTTP);
+			
+			AmazonS3 conn = new AmazonS3Client(credentials, clientConfig);
+
+
+
+405 MethodNotAllowed
+--------------------
+
+If you receive an 405 error, check to see if you have the S3 subdomain set up correctly. 
+You will need to have a wild card setting in your DNS record for subdomain functionality
+to work properly.
+
+Also, check to ensure that the default site is disabled. ::
+
+     [java] Exception in thread "main" Status Code: 405, AWS Service: Amazon S3, AWS Request ID: null, AWS Error Code: MethodNotAllowed, AWS Error Message: null, S3 Extended Request ID: null
+  
+  
+  
+Numerous objects in default.rgw.meta pool
+=========================================
+
+Clusters created prior to *jewel* have a metadata archival feature enabled by default, using the ``default.rgw.meta`` pool.
+This archive keeps all old versions of user and bucket metadata, resulting in large numbers of objects in the ``default.rgw.meta`` pool.
+
+Disabling the Metadata Heap
+---------------------------
+
+Users who want to disable this feature going forward should set the ``metadata_heap`` field to an empty string ``""``::
+
+  $ radosgw-admin zone get --rgw-zone=default > zone.json
+  [edit zone.json, setting "metadata_heap": ""]
+  $ radosgw-admin zone set --rgw-zone=default --infile=zone.json
+  $ radosgw-admin period update --commit
+
+This will stop new metadata from being written to the ``default.rgw.meta`` pool, but does not remove any existing objects or pool.
+
+Cleaning the Metadata Heap Pool
+-------------------------------
+
+Clusters created prior to *jewel* normally use ``default.rgw.meta`` only for the metadata archival feature.
+
+However, from *luminous* onwards, radosgw uses :ref:`Pool Namespaces <radosgw-pool-namespaces>` within ``default.rgw.meta`` for an entirely different purpose, that is, to store ``user_keys`` and other critical metadata.
+
+Users should check zone configuration before proceeding any cleanup procedures::
+
+  $ radosgw-admin zone get --rgw-zone=default | grep default.rgw.meta
+  [should not match any strings]
+
+Having confirmed that the pool is not used for any purpose, users may safely delete all objects in the ``default.rgw.meta`` pool, or optionally, delete the entire pool itself.
diff --git a/doc/radosgw/vault.rst b/doc/radosgw/vault.rst
new file mode 100644
index 000000000..da34a3919
--- /dev/null
+++ b/doc/radosgw/vault.rst
@@ -0,0 +1,442 @@
+===========================
+HashiCorp Vault Integration
+===========================
+
+HashiCorp `Vault`_ can be used as a secure key management service for
+`Server-Side Encryption`_ (SSE-KMS).
+
+.. ditaa::
+
+           +---------+       +---------+        +-------+     +-------+
+           |  Client |       | RadosGW |        | Vault |     |  OSD  |
+           +---------+       +---------+        +-------+     +-------+
+                | create secret   |                 |             |
+                | key for key ID  |                 |             |
+                |-----------------+---------------->|             |
+                |                 |                 |             |
+                | upload object   |                 |             |
+                | with key ID     |                 |             |
+                |---------------->| request secret  |             |
+                |                 | key for key ID  |             |
+                |                 |---------------->|             |
+                |                 |<----------------|             |
+                |                 | return secret   |             |
+                |                 | key             |             |
+                |                 |                 |             |
+                |                 | encrypt object  |             |
+                |                 | with secret key |             |
+                |                 |--------------+  |             |
+                |                 |              |  |             |
+                |                 |<-------------+  |             |
+                |                 |                 |             |
+                |                 | store encrypted |             |
+                |                 | object          |             |
+                |                 |------------------------------>|
+
+#. `Vault secrets engines`_
+#. `Vault authentication`_
+#. `Vault namespaces`_
+#. `Create a key in Vault`_
+#. `Configure the Ceph Object Gateway`_
+#. `Upload object`_
+
+Some examples below use the Vault command line utility to interact with
+Vault. You may need to set the following environment variable with the correct
+address of your Vault server to use this utility::
+
+  export VAULT_ADDR='https://vault-server-fqdn:8200'
+
+Vault secrets engines
+=====================
+
+Vault provides several secrets engines, which can store, generate, and encrypt
+data. Currently, the Object Gateway supports:
+
+- `KV secrets engine`_ version 2
+- `Transit engine`_
+
+KV secrets engine
+-----------------
+
+The KV secrets engine is used to store arbitrary key/value secrets in Vault. To
+enable the KV engine version 2 in Vault, use the following command::
+
+  vault secrets enable -path secret kv-v2
+
+The Object Gateway can be configured to use the KV engine version 2 with the
+following setting::
+
+  rgw crypt vault secret engine = kv
+
+Transit secrets engine
+----------------------
+
+The transit engine handles cryptographic functions on data in-transit. To enable
+it in Vault, use the following command::
+
+  vault secrets enable transit
+
+The Object Gateway can be configured to use the transit engine with the
+following setting::
+
+  rgw crypt vault secret engine = transit
+
+Vault authentication
+====================
+
+Vault supports several authentication mechanisms. Currently, the Object
+Gateway can be configured to authenticate to Vault using the
+`Token authentication method`_ or a `Vault agent`_.
+
+Most tokens in Vault have limited lifetimes and powers.  The only
+sort of Vault token that does not have a lifetime are root tokens.
+For all other tokens, it is necessary to periodically refresh them,
+either by performing initial authentication, or by renewing the token.
+Ceph does not have any logic to perform either operation.
+The simplest best way to use Vault tokens with ceph is to
+also run the Vault agent and have it refresh the token file.
+When the Vault agent is used in this mode, file system permissions
+can be used to restrict who has the use of tokens.
+
+Instead of having Vault agent refresh a token file, it can be told
+to act as a proxy server.  In this mode, Vault will add a token when
+necessary and add it to requests passed to it before forwarding them on
+to the real server.  Vault agent will still handle token renewal just
+as it would when storing a token in the filesystem.  In this mode, it
+is necessary to properly secure the network path rgw uses to reach the
+Vault agent, such as having the Vault agent listen only to localhost.
+
+Token policies for the object gateway
+-------------------------------------
+
+All Vault tokens have powers as specified by the polices attached
+to that token.  Multiple policies may be associated with one
+token.  You should only use the policies necessary for your
+configuration.
+
+When using the kv secret engine with the object gateway::
+
+  vault policy write rgw-kv-policy -<<EOF
+    path "secret/data/*" {
+      capabilities = ["read"]
+    }
+  EOF
+
+When using the transit secret engine with the object gateway::
+
+  vault policy write rgw-transit-policy -<<EOF
+    path "transit/keys/*" {
+      capabilities = [ "create", "update" ]
+      denied_parameters = {"exportable" = [], "allow_plaintext_backup" = [] }
+    }
+
+    path "transit/keys/*" {
+      capabilities = ["read", "delete"]
+    }
+
+    path "transit/keys/" {
+      capabilities = ["list"]
+    }
+
+    path "transit/keys/+/rotate" {
+      capabilities = [ "update" ]
+    }
+
+    path "transit/*" {
+      capabilities = [ "update" ]
+    }
+  EOF
+
+If you had previously used an older version of ceph with the
+transit secret engine, you might need the following policy::
+
+  vault policy write old-rgw-transit-policy -<<EOF
+    path "transit/export/encryption-key/*" {
+      capabilities = ["read"]
+    }
+  EOF
+
+If you are using both sse-kms and sse-s3, then you should point
+each to separate containers.  You could either use separate
+vault instances, or you could use either separately mounted
+transit instances, or different branches under a common transit
+point.  If you are not using separate vault instances, you can
+use these to point kms and sse-s3 to separate containers:
+``rgw_crypt_vault_prefix``
+and/or
+``rgw_crypt_sse_s3_vault_prefix``.
+When granting vault permissions to sse-kms bucket owners, you should
+not give them permission to muck around with sse-s3 keys;
+only ceph itself should be doing that.
+
+Token authentication
+--------------------
+
+.. note: Never use root tokens with ceph in production environments.
+
+The token authentication method expects a Vault token to be present in a
+plaintext file. The Object Gateway can be configured to use token authentication
+with the following settings::
+
+  rgw crypt vault auth = token
+  rgw crypt vault token file = /run/.rgw-vault-token
+  rgw crypt vault addr = https://vault-server-fqdn:8200
+
+Adjust these settings to match your configuration.
+For security reasons, the token file must be readable by the Object Gateway
+only.
+
+Vault agent
+-----------
+
+The Vault agent is a client daemon that provides authentication to Vault and
+manages token renewal and caching. It typically runs on the same host as the
+Object Gateway. With a Vault agent, it is possible to use other Vault
+authentication mechanism such as AppRole, AWS, Certs, JWT, and Azure.
+
+The Object Gateway can be configured to use a Vault agent with the following
+settings::
+
+  rgw crypt vault auth = agent
+  rgw crypt vault addr = http://127.0.0.1:8100
+
+You might set up vault agent as follows::
+
+  vault write auth/approle/role/rgw-ap \
+    token_policies=rgw-transit-policy,default \
+    token_max_ttl=60m
+
+Change the policy here to match your configuration.
+
+Get the role-id:
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-role-id
+
+Get the secret-id:
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-secret-id
+
+Create configuration for the Vault agent, such as::
+
+  pid_file = "/run/rgw-vault-agent-pid"
+  auto_auth {
+    method "AppRole" {
+      mount_path = "auth/approle"
+      config = {
+        role_id_file_path ="/usr/local/etc/vault/.rgw-ap-role-id"
+        secret_id_file_path ="/usr/local/etc/vault/.rgw-ap-secret-id"
+        remove_secret_id_file_after_reading ="false"
+      }
+    }
+  }
+  cache {
+    use_auto_auth_token = true
+  }
+  listener "tcp" {
+    address = "127.0.0.1:8100"
+    tls_disable = true
+  }
+  vault {
+    address = "https://vault-server-fqdn:8200"
+  }
+
+Then use systemctl or another method of your choice to run
+a persistent daemon with the following arguments::
+
+    /usr/local/bin/vault agent -config=/usr/local/etc/vault/rgw-agent.hcl
+
+Once the vault agent is running, you should find it listening
+to port 8100 on localhost, and you should be able to interact
+with it using the vault command.
+
+Vault namespaces
+================
+
+In the Enterprise version, Vault supports the concept of `namespaces`_, which
+allows centralized management for teams within an organization while ensuring
+that those teams operate within isolated environments known as tenants.
+
+The Object Gateway can be configured to access Vault within a particular
+namespace using the following configuration setting::
+
+  rgw crypt vault namespace = tenant1
+
+Create a key in Vault
+=====================
+
+.. note:: Keys for server-side encryption must be 256-bit long and base-64
+   encoded.
+
+Using the KV engine
+-------------------
+
+A key for server-side encryption can be created in the KV version 2 engine using
+the command line utility, as in the following example::
+
+  vault kv put secret/myproject/mybucketkey key=$(openssl rand -base64 32)
+
+Sample output::
+
+  ====== Metadata ======
+  Key              Value
+  ---              -----
+  created_time     2019-08-29T17:01:09.095824999Z
+  deletion_time    n/a
+  destroyed        false
+  version          1
+
+Note that in the KV secrets engine, secrets are stored as key-value pairs, and
+the Gateway expects the key name to be ``key``, i.e. the secret must be in the
+form ``key=<secret key>``.
+
+Using the Transit engine
+------------------------
+
+Keys created for use with the Transit engine should no longer be marked
+exportable.  They can be created with::
+
+  vault write -f transit/keys/mybucketkey
+
+The command above creates a keyring, which contains a key of type
+``aes256-gcm96`` by default. To verify that the key was correctly created, use
+the following command::
+
+  vault read transit/keys/mybucketkey
+
+Sample output::
+
+  Key     Value
+  ---     -----
+  derived                   false
+  exportable                false
+  name                      mybucketkey
+  type                      aes256-gcm96
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Vault as a KMS backend for
+server-side encryption::
+
+  rgw crypt s3 kms backend = vault
+
+Choose the Vault authentication method, e.g.::
+
+  rgw crypt vault auth = token
+  rgw crypt vault token file = /run/.rgw-vault-token
+  rgw crypt vault addr = https://vault-server-fqdn:8200
+
+Or::
+
+  rgw crypt vault auth = agent
+  rgw crypt vault addr = http://localhost:8100
+
+Choose the secrets engine::
+
+  rgw crypt vault secret engine = kv
+
+Or::
+
+  rgw crypt vault secret engine = transit
+
+Optionally, set the Vault namespace where encryption keys will be fetched from::
+
+  rgw crypt vault namespace = tenant1
+
+Finally, the URLs where the Gateway will retrieve encryption keys from Vault can
+be restricted by setting a path prefix. For instance, the Gateway can be
+restricted to fetch KV keys as follows::
+
+  rgw crypt vault prefix = /v1/secret/data
+
+Or, when using the transit secret engine::
+
+  rgw crypt vault prefix = /v1/transit
+
+In the example above, the Gateway would only fetch transit encryption keys under
+``https://vault-server:8200/v1/transit``.
+
+You can use custom ssl certs to authenticate with vault with help of
+following options::
+
+  rgw crypt vault verify ssl = true
+  rgw crypt vault ssl cacert = /etc/ceph/vault.ca
+  rgw crypt vault ssl clientcert = /etc/ceph/vault.crt
+  rgw crypt vault ssl clientkey = /etc/ceph/vault.key
+
+where vault.ca is CA certificate and vault.key/vault.crt are private key and ssl
+certificate generated for RGW to access the vault server. It highly recommended to
+set this option true, setting false is very dangerous and need to avoid since this
+runs in very secured environments.
+
+Transit engine compatibility support
+------------------------------------
+The transit engine has compatibility support for previous
+versions of ceph, which used the transit engine as a simple key store.
+
+There is a "compat" option which can be given to the transit
+engine to configure the compatibility support,
+
+To entirely disable backwards support, use::
+
+  rgw crypt vault secret engine = transit compat=0
+
+This will be the default in future versions. and is safe to use
+for new installs using the current version.
+
+This is the normal default with the current version::
+
+  rgw crypt vault secret engine = transit compat=1
+
+This enables the new engine for newly created objects,
+but still allows the old engine to be used for old objects.
+In order to access old and new objects, the vault token given
+to ceph must have both the old and new transit policies.
+
+To force use of only the old engine, use::
+
+  rgw crypt vault secret engine = transit compat=2
+
+This mode is automatically selected if the vault prefix
+ends in export/encryption-key, which was the previously
+documented setting.
+
+Upload object
+=============
+
+When uploading an object to the Gateway, provide the SSE key ID in the request.
+As an example, for the kv engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id myproject/mybucketkey
+  
+As an example, for the transit engine (new flavor), using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+
+The Object Gateway will fetch the key from Vault, encrypt the object and store
+it in the bucket. Any request to download the object will make the Gateway
+automatically retrieve the correspondent key from Vault and decrypt the object.
+
+Note that the secret will be fetched from Vault using a URL constructed by
+concatenating the base address (``rgw crypt vault addr``), the (optional)
+URL prefix (``rgw crypt vault prefix``), and finally the key ID. 
+
+In the kv engine example above, the Gateway would fetch the secret from::
+
+  http://vaultserver:8200/v1/secret/data/myproject/mybucketkey
+
+In the transit engine example above, the Gateway would encrypt the secret using this key::
+
+  http://vaultserver:8200/v1/transit/mybucketkey
+
+.. _Server-Side Encryption: ../encryption
+.. _Vault: https://www.vaultproject.io/docs/
+.. _Token authentication method: https://www.vaultproject.io/docs/auth/token.html
+.. _Vault agent: https://www.vaultproject.io/docs/agent/index.html
+.. _KV Secrets engine: https://www.vaultproject.io/docs/secrets/kv/
+.. _Transit engine: https://www.vaultproject.io/docs/secrets/transit
+.. _namespaces: https://www.vaultproject.io/docs/enterprise/namespaces/index.html
diff --git a/doc/rbd/api/index.rst b/doc/rbd/api/index.rst
new file mode 100644
index 000000000..27bb4485d
--- /dev/null
+++ b/doc/rbd/api/index.rst
@@ -0,0 +1,8 @@
+========================
+ Ceph Block Device APIs
+========================
+
+.. toctree::
+   :maxdepth: 2
+
+   librbd (Python) <librbdpy>
diff --git a/doc/rbd/api/librbdpy.rst b/doc/rbd/api/librbdpy.rst
new file mode 100644
index 000000000..7a74b0498
--- /dev/null
+++ b/doc/rbd/api/librbdpy.rst
@@ -0,0 +1,85 @@
+.. _rbd api py:
+
+================
+ Librbd (Python)
+================
+
+.. highlight:: python
+
+The `rbd` python module provides file-like access to RBD images.
+
+
+Example: Creating and writing to an image
+=========================================
+
+To use `rbd`, you must first connect to RADOS and open an IO
+context::
+
+    cluster = rados.Rados(conffile='my_ceph.conf')
+    cluster.connect()
+    ioctx = cluster.open_ioctx('mypool')
+
+Then you instantiate an :class:rbd.RBD object, which you use to create the
+image::
+
+    rbd_inst = rbd.RBD()
+    size = 4 * 1024**3  # 4 GiB
+    rbd_inst.create(ioctx, 'myimage', size)
+
+To perform I/O on the image, you instantiate an :class:rbd.Image object::
+
+    image = rbd.Image(ioctx, 'myimage')
+    data = b'foo' * 200
+    image.write(data, 0)
+
+This writes 'foo' to the first 600 bytes of the image. Note that data
+cannot be :type:unicode - `Librbd` does not know how to deal with
+characters wider than a :c:type:char.
+
+In the end, you will want to close the image, the IO context and the connection to RADOS::
+
+    image.close()
+    ioctx.close()
+    cluster.shutdown()
+
+To be safe, each of these calls would need to be in a separate :finally
+block::
+
+    cluster = rados.Rados(conffile='my_ceph_conf')
+    try:
+        cluster.connect()
+        ioctx = cluster.open_ioctx('my_pool')
+        try:
+            rbd_inst = rbd.RBD()
+            size = 4 * 1024**3  # 4 GiB
+            rbd_inst.create(ioctx, 'myimage', size)
+            image = rbd.Image(ioctx, 'myimage')
+            try:
+                data = b'foo' * 200
+                image.write(data, 0)
+            finally:
+                image.close()
+        finally:
+            ioctx.close()
+    finally:
+        cluster.shutdown()
+
+This can be cumbersome, so the :class:`Rados`, :class:`Ioctx`, and
+:class:`Image` classes can be used as context managers that close/shutdown
+automatically (see :pep:`343`). Using them as context managers, the
+above example becomes::
+
+    with rados.Rados(conffile='my_ceph.conf') as cluster:
+        with cluster.open_ioctx('mypool') as ioctx:
+            rbd_inst = rbd.RBD()
+            size = 4 * 1024**3  # 4 GiB
+            rbd_inst.create(ioctx, 'myimage', size)
+            with rbd.Image(ioctx, 'myimage') as image:
+                data = b'foo' * 200
+                image.write(data, 0)
+
+API Reference
+=============
+
+.. automodule:: rbd
+    :members: RBD, Image, SnapIterator
diff --git a/doc/rbd/disk.conf b/doc/rbd/disk.conf
new file mode 100644
index 000000000..3db9b8a11
--- /dev/null
+++ b/doc/rbd/disk.conf
@@ -0,0 +1,8 @@
+<disk type='network' device='disk'>
+	<source protocol='rbd' name='poolname/imagename'>
+		<host name='{fqdn}' port='6789'/>
+		<host name='{fqdn}' port='6790'/>
+		<host name='{fqdn}' port='6791'/>
+	</source>
+	<target dev='vda' bus='virtio'/>
+</disk>
diff --git a/doc/rbd/index.rst b/doc/rbd/index.rst
new file mode 100644
index 000000000..4a8029bba
--- /dev/null
+++ b/doc/rbd/index.rst
@@ -0,0 +1,72 @@
+===================
+ Ceph Block Device
+===================
+
+.. index:: Ceph Block Device; introduction
+
+A block is a sequence of bytes (often 512).
+Block-based storage interfaces are a mature and common way to store data on
+media including HDDs, SSDs, CDs, floppy disks, and even tape.
+The ubiquity of block device interfaces is a perfect fit for interacting
+with mass data storage including Ceph.
+
+Ceph block devices are thin-provisioned, resizable, and store data striped over
+multiple OSDs.  Ceph block devices leverage
+:abbr:`RADOS (Reliable Autonomic Distributed Object Store)` capabilities
+including snapshotting, replication and strong consistency. Ceph block
+storage clients communicate with Ceph clusters through kernel modules or
+the ``librbd`` library.
+
+.. ditaa::
+
+            +------------------------+ +------------------------+
+            |     Kernel Module      | |        librbd          |
+            +------------------------+-+------------------------+
+            |                   RADOS Protocol                  |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. note:: Kernel modules can use Linux page caching. For ``librbd``-based
+   applications, Ceph supports `RBD Caching`_.
+
+Ceph's block devices deliver high performance with vast scalability to
+`kernel modules`_, or to :abbr:`KVMs (kernel virtual machines)` such as `QEMU`_, and
+cloud-based computing systems like `OpenStack`_ and `CloudStack`_ that rely on
+libvirt and QEMU to integrate with Ceph block devices. You can use the same cluster
+to operate the :ref:`Ceph RADOS Gateway <object-gateway>`, the
+:ref:`Ceph File System <ceph-file-system>`, and Ceph block devices simultaneously.
+
+.. important:: To use Ceph Block Devices, you must have access to a running
+   Ceph cluster.
+
+.. toctree::
+        :maxdepth: 1
+
+	Basic Commands <rados-rbd-cmds>
+
+.. toctree::
+        :maxdepth: 2
+
+        Operations <rbd-operations>
+
+.. toctree::
+	:maxdepth: 2
+
+        Integrations <rbd-integrations>
+
+.. toctree::
+	:maxdepth: 2
+
+	Manpages <man/index>
+
+.. toctree::
+	:maxdepth: 2
+
+	APIs <api/index>
+
+.. _RBD Caching: ./rbd-config-ref/
+.. _kernel modules: ./rbd-ko/
+.. _QEMU: ./qemu-rbd/
+.. _OpenStack: ./rbd-openstack
+.. _CloudStack: ./rbd-cloudstack
diff --git a/doc/rbd/iscsi-initiator-esx.rst b/doc/rbd/iscsi-initiator-esx.rst
new file mode 100644
index 000000000..8bed6f2a2
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-esx.rst
@@ -0,0 +1,105 @@
+------------------------------
+iSCSI Initiator for VMware ESX
+------------------------------
+
+**Prerequisite:**
+
+-  VMware ESX 6.5 or later using Virtual Machine compatibility 6.5 with VMFS 6.
+
+**iSCSI Discovery and Multipath Device Setup:**
+
+The following instructions will use the default vSphere web client and esxcli.
+
+#. Enable Software iSCSI
+
+   .. image:: ../images/esx_web_client_storage_main.png
+      :align: center
+
+   Click on "Storage" from "Navigator", and select the "Adapters" tab.
+   From there right click "Configure iSCSI".
+
+#. Set Initiator Name
+
+   .. image:: ../images/esx_config_iscsi_main.png
+      :align: center
+
+   If the initiator name in the "Name & alias" section is not the same name
+   used when creating the client during gwcli setup or the initiator name used
+   in the ansible client_connections client variable, then ssh to the ESX
+   host and run the following esxcli commands to change the name.
+
+   Get the adapter name for Software iSCSI:
+
+   ::
+
+       > esxcli iscsi adapter list
+       > Adapter  Driver     State   UID            Description
+       > -------  ---------  ------  -------------  ----------------------
+       > vmhba64  iscsi_vmk  online  iscsi.vmhba64  iSCSI Software Adapter
+
+   In this example the software iSCSI adapter is vmhba64 and the initiator
+   name is iqn.1994-05.com.redhat:rh7-client:
+
+    ::
+
+        > esxcli iscsi adapter set -A vmhba64 -n iqn.1994-05.com.redhat:rh7-client
+
+#. Setup CHAP
+
+   .. image:: ../images/esx_chap.png
+      :align: center
+
+   Expand the CHAP authentication section, select "Do not use CHAP unless
+   required by target" and enter the CHAP credentials used in the gwcli
+   auth command or ansible client_connections credentials variable.
+
+   The Mutual CHAP authentication section should have "Do not use CHAP"
+   selected.
+
+   Warning: There is a bug in the web client where the requested CHAP
+   settings are not always used initially. On the iSCSI gateway kernel
+   logs you will see the error:
+
+    ::
+
+       > kernel: CHAP user or password not set for Initiator ACL
+       > kernel: Security negotiation failed.
+       > kernel: iSCSI Login negotiation failed.
+
+   To workaround this set the CHAP settings with the esxcli command. Here
+   authname is the username and secret is the password used in previous
+   examples:
+
+   ::
+
+       > esxcli iscsi adapter auth chap set --direction=uni --authname=myiscsiusername --secret=myiscsipassword --level=discouraged -A vmhba64
+
+#. Configure iSCSI Settings
+
+   .. image:: ../images/esx_iscsi_recov_timeout.png
+      :align: center
+
+   Expand Advanced settings and set the "RecoveryTimeout" to 25.
+
+#. Set the discovery address
+
+   .. image:: ../images/esx_config_iscsi_main.png
+      :align: center
+
+   In the Dynamic targets section, click "Add dynamic target" and under
+   Addresses add one of the gateway IP addresses added during the iSCSI
+   gateway setup stage in the gwcli section or an IP set in the ansible
+   gateway_ip_list variable. Only one address needs to be added as the gateways
+   have been setup so all the iSCSI portals are returned during discovery.
+
+   Finally, click the "Save configuration" button. In the Devices tab, you
+   should see the RBD image.
+
+   The LUN should be automatically configured and using the ALUA SATP and
+   MRU PSP. Other SATPs and PSPs must not be used. This can be verified with
+   the esxcli command:
+
+   ::
+
+       > esxcli storage nmp path list -d eui.your_devices_id
+
diff --git a/doc/rbd/iscsi-initiator-linux.rst b/doc/rbd/iscsi-initiator-linux.rst
new file mode 100644
index 000000000..bf8c930f3
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-linux.rst
@@ -0,0 +1,119 @@
+-------------------------
+iSCSI Initiator for Linux
+-------------------------
+
+**Prerequisite:**
+
+-  Package ``iscsi-initiator-utils``
+
+-  Package ``device-mapper-multipath``
+
+**Installing:**
+
+Install the iSCSI initiator and multipath tools:
+
+.. prompt:: bash #
+
+   yum install iscsi-initiator-utils
+   yum install device-mapper-multipath
+
+**Configuring:**
+
+#. Create the default ``/etc/multipath.conf`` file and enable the
+   ``multipathd`` service:
+
+   .. prompt:: bash #
+
+      mpathconf --enable --with_multipathd y
+
+#. Add the following to the ``/etc/multipath.conf`` file:
+
+   ::
+
+       devices {
+               device {
+                       vendor                 "LIO-ORG"
+                       product                "TCMU device"
+                       hardware_handler       "1 alua"
+                       path_grouping_policy   "failover"
+                       path_selector          "queue-length 0"
+                       failback               60
+                       path_checker           tur
+                       prio                   alua
+                       prio_args              exclusive_pref_bit
+                       fast_io_fail_tmo       25
+                       no_path_retry          queue
+               }
+       }
+
+#. Restart the ``multipathd`` service:
+
+   .. prompt:: bash #
+
+      systemctl reload multipathd
+
+**iSCSI Discovery and Setup:**
+
+#. Enable CHAP authentication and provide the initiator CHAP username
+   and password by uncommenting and setting the following options in
+   the ``/etc/iscsi/iscsid.conf`` file:
+
+   ::
+
+       node.session.auth.authmethod = CHAP
+       node.session.auth.username = myusername
+       node.session.auth.password = mypassword
+
+   If you intend to use mutual (bidirectional) authentication, provide the
+   target CHAP username and password:
+
+   ::
+
+       node.session.auth.username_in = mytgtusername
+       node.session.auth.password_in = mytgtpassword
+
+#. Discover the target portals:
+
+   .. prompt:: bash #
+
+      iscsiadm -m discovery -t st -p 192.168.56.101
+
+   ::
+
+       192.168.56.101:3260,1 iqn.2003-01.org.linux-iscsi.rheln1
+       192.168.56.102:3260,2 iqn.2003-01.org.linux-iscsi.rheln1
+
+#. Log in to the target:
+
+   .. prompt:: bash #
+
+      iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.rheln1 -l
+
+**Multipath IO Setup:**
+
+#. The multipath daemon (``multipathd``) uses the ``multipath.conf`` settings
+   to set up devices automatically. Running the ``multipath`` command shows
+   that the devices have been set up in a failover configuration. Notice that
+   each path has been placed into its own priority group: 
+
+   .. prompt:: bash #
+
+      multipath -ll
+
+   ::
+
+       mpathbt (360014059ca317516a69465c883a29603) dm-1 LIO-ORG ,IBLOCK
+       size=1.0G features='0' hwhandler='1 alua' wp=rw
+       |-+- policy='queue-length 0' prio=50 status=active
+       | `- 28:0:0:1 sde  8:64  active ready running
+       `-+- policy='queue-length 0' prio=10 status=enabled
+         `- 29:0:0:1 sdc  8:32  active ready running
+
+   You should now be able to use the RBD image in the same way that you would
+   use a normal multipath iSCSI disk.
+
+#. Log out of the target:
+
+   .. prompt:: bash #
+
+      iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.rheln1 -u
diff --git a/doc/rbd/iscsi-initiator-win.rst b/doc/rbd/iscsi-initiator-win.rst
new file mode 100644
index 000000000..7816059bc
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-win.rst
@@ -0,0 +1,102 @@
+-------------------------------------
+iSCSI Initiator for Microsoft Windows
+-------------------------------------
+
+**Prerequisite:**
+
+-  Microsoft Windows Server 2016 or later
+
+**iSCSI Initiator, Discovery and Setup:**
+
+#. Install the iSCSI initiator driver and MPIO tools.
+
+#. Launch the MPIO program, click on the "Discover Multi-Paths" tab, check the
+   "Add support for iSCSI devices� box, and click "Add". This will require a
+   reboot.
+
+#. On the iSCSI Initiator Properties window, on the "Discovery" tab, add a target
+   portal. Enter the IP address or DNS name and Port of the Ceph iSCSI gateway.
+
+#. On the “Targets� tab, select the target and click on “Connect�.
+
+#. On the “Connect To Target� window, select the “Enable multi-path� option, and
+   click the “Advanced� button.
+
+#. Under the "Connect using" section, select a “Target portal IP� . Select the
+   “Enable CHAP login on� and enter the "Name" and "Target secret" values from the
+   Ceph iSCSI Ansible client credentials section, and click OK.
+
+#. Repeat steps 5 and 6 for each target portal defined when setting up
+   the iSCSI gateway.
+
+**Multipath IO Setup:**
+
+Configuring the MPIO load balancing policy, setting the timeout and
+retry options are using PowerShell with the ``mpclaim`` command. The
+rest is done in the iSCSI Initiator tool.
+
+.. note::
+  It is recommended to increase the ``PDORemovePeriod`` option to 120
+  seconds from PowerShell. This value might need to be adjusted based
+  on the application. When all paths are down, and 120 seconds
+  expires, the operating system will start failing IO requests.
+
+::
+
+    Set-MPIOSetting -NewPDORemovePeriod 120
+
+::
+
+    mpclaim.exe -l -m 1
+
+::
+
+    mpclaim -s -m
+    MSDSM-wide Load Balance Policy: Fail Over Only
+
+#. Using the iSCSI Initiator tool, from the “Targets� tab, click on
+   the “Devices...� button.
+
+#. From the Devices window, select a disk and click the
+   “MPIO...� button.
+
+#. On the "Device Details" window the paths to each target portal is
+   displayed. If using the ``ceph-ansible`` setup method, the
+   iSCSI gateway will use ALUA to tell the iSCSI initiator which path
+   and iSCSI gateway should be used as the primary path. The Load
+   Balancing Policy “Fail Over Only� must be selected
+
+::
+
+    mpclaim -s -d $MPIO_DISK_ID
+
+.. note::
+  For the ``ceph-ansible`` setup method, there will be one
+  Active/Optimized path which is the path to the iSCSI gateway node
+  that owns the LUN, and there will be an Active/Unoptimized path for
+  each other iSCSI gateway node.
+
+**Tuning:**
+
+Consider using the following registry settings:
+
+-  Windows Disk Timeout
+
+   ::
+
+       HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Disk
+
+   ::
+
+       TimeOutValue = 65
+
+-  Microsoft iSCSI Initiator Driver
+
+   ::
+
+       HKEY_LOCAL_MACHINE\\SYSTEM\CurrentControlSet\Control\Class\{4D36E97B-E325-11CE-BFC1-08002BE10318}\<Instance_Number>\Parameters
+
+   ::
+   
+       LinkDownTime = 25
+       SRBTimeoutDelta = 15
diff --git a/doc/rbd/iscsi-initiators.rst b/doc/rbd/iscsi-initiators.rst
new file mode 100644
index 000000000..b24338952
--- /dev/null
+++ b/doc/rbd/iscsi-initiators.rst
@@ -0,0 +1,25 @@
+.. _configuring-the-iscsi-initiators:
+
+--------------------------------
+Configuring the iSCSI Initiators
+--------------------------------
+
+- `iSCSI Initiator for Linux <../iscsi-initiator-linux>`_
+
+- `iSCSI Initiator for Microsoft Windows <../iscsi-initiator-win>`_
+
+- `iSCSI Initiator for VMware ESX <../iscsi-initiator-esx>`_
+
+    .. warning::
+
+        Applications that use SCSI persistent group reservations (PGR) and
+        SCSI 2 based reservations are not supported when exporting a RBD image
+        through more than one iSCSI gateway.
+
+.. toctree::
+  :maxdepth: 1
+  :hidden:
+
+  Linux <iscsi-initiator-linux>
+  Microsoft Windows <iscsi-initiator-win>
+  VMware ESX <iscsi-initiator-esx>
diff --git a/doc/rbd/iscsi-monitoring.rst b/doc/rbd/iscsi-monitoring.rst
new file mode 100644
index 000000000..a36cc3cdb
--- /dev/null
+++ b/doc/rbd/iscsi-monitoring.rst
@@ -0,0 +1,85 @@
+------------------------------
+Monitoring Ceph iSCSI gateways
+------------------------------
+
+Ceph provides a tool for iSCSI gateway environments
+to monitor performance of exported RADOS Block Device (RBD) images.
+
+The ``gwtop`` tool is a ``top``-like tool that displays aggregated
+performance metrics of RBD images that are exported to clients over
+iSCSI. The metrics are sourced from a Performance Metrics Domain Agent
+(PMDA). Information from the Linux-IO target (LIO) PMDA is used to list
+each exported RBD image, the connected client, and its associated I/O
+metrics.
+
+**Requirements:**
+
+-  A running Ceph iSCSI gateway
+
+**Installing:**
+
+#. As ``root``, install the ``ceph-iscsi-tools`` package on each iSCSI
+   gateway node:
+
+   .. prompt:: bash #
+
+      yum install ceph-iscsi-tools
+
+#. As ``root``, install the performance co-pilot package on each iSCSI
+   gateway node:
+
+   .. prompt:: bash #
+
+      yum install pcp
+
+#. As ``root``, install the LIO PMDA package on each iSCSI gateway node:
+
+   .. prompt:: bash #
+
+      yum install pcp-pmda-lio
+
+#. As ``root``, enable and start the performance co-pilot service on
+   each iSCSI gateway node:
+
+   .. prompt:: bash #
+
+      systemctl enable pmcd
+      systemctl start pmcd
+
+#. As ``root``, register the ``pcp-pmda-lio`` agent:
+
+   .. prompt:: bash #
+
+      cd /var/lib/pcp/pmdas/lio
+      ./Install
+
+By default, ``gwtop`` assumes the iSCSI gateway configuration object is
+stored in a RADOS object called ``gateway.conf`` in the ``rbd`` pool.
+This configuration defines the iSCSI gateways to contact for gathering
+the performance statistics. This can be overridden by using either the
+``-g`` or ``-c`` flags. See ``gwtop --help`` for more details.
+
+The LIO configuration determines which type of performance statistics to
+extract from performance co-pilot. When ``gwtop`` starts it looks at the
+LIO configuration, and if it find user-space disks, then ``gwtop``
+selects the LIO collector automatically.
+
+**Example ``gwtop`` Outputs**
+
+::
+
+    gwtop  2/2 Gateways   CPU% MIN:  4 MAX:  5    Network Total In:    2M  Out:    3M   10:20:00
+    Capacity:   8G    Disks:   8   IOPS:  503   Clients:  1   Ceph: HEALTH_OK          OSDs:   3
+    Pool.Image       Src    Size     iops     rMB/s     wMB/s   Client
+    iscsi.t1703             500M        0      0.00      0.00
+    iscsi.testme1           500M        0      0.00      0.00
+    iscsi.testme2           500M        0      0.00      0.00
+    iscsi.testme3           500M        0      0.00      0.00
+    iscsi.testme5           500M        0      0.00      0.00
+    rbd.myhost_1      T       4G      504      1.95      0.00   rh460p(CON)
+    rbd.test_2                1G        0      0.00      0.00
+    rbd.testme              500M        0      0.00      0.00
+
+In the *Client* column, ``(CON)`` means the iSCSI initiator (client) is
+currently logged into the iSCSI gateway. If ``-multi-`` is displayed,
+then multiple clients are mapped to the single RBD image.
diff --git a/doc/rbd/iscsi-overview.rst b/doc/rbd/iscsi-overview.rst
new file mode 100644
index 000000000..879083c3f
--- /dev/null
+++ b/doc/rbd/iscsi-overview.rst
@@ -0,0 +1,57 @@
+.. _ceph-iscsi:
+
+==================
+Ceph iSCSI Gateway
+==================
+
+The iSCSI Gateway presents a Highly Available (HA) iSCSI target that exports
+RADOS Block Device (RBD) images as SCSI disks. The iSCSI protocol allows
+clients (initiators) to send SCSI commands to storage devices (targets) over a
+TCP/IP network, enabling clients without native Ceph client support to access
+Ceph block storage.  
+
+Each iSCSI gateway exploits the Linux IO target kernel subsystem (LIO) to
+provide iSCSI protocol support. LIO utilizes userspace passthrough (TCMU) to
+interact with Ceph's librbd library and expose RBD images to iSCSI clients.
+With Ceph’s iSCSI gateway you can provision a fully integrated block-storage
+infrastructure with all the features and benefits of a conventional Storage
+Area Network (SAN).
+
+.. ditaa::
+                  Cluster Network (optional)
+                 +-------------------------------------------+
+                 |             |               |             |
+             +-------+     +-------+       +-------+     +-------+
+             |       |     |       |       |       |     |       |
+             | OSD 1 |     | OSD 2 |       | OSD 3 |     | OSD N |
+             |    {s}|     |    {s}|       |    {s}|     |    {s}|
+             +-------+     +-------+       +-------+     +-------+
+                 |             |               |             |
+      +--------->|             |  +---------+  |             |<---------+
+      :          |             |  |   RBD   |  |             |          :
+      |          +----------------|  Image  |----------------+          |
+      |           Public Network  |    {d}  |                           |
+      |                           +---------+                           |
+      |                                                                 |
+      |                      +-------------------+                      |
+      |   +--------------+   | iSCSI Initiators  |   +--------------+   |
+      |   | iSCSI GW     |   |   +-----------+   |   | iSCSI GW     |   |
+      +-->|  RBD Module  |<--+   | Various   |   +-->|  RBD Module  |<--+
+          |              |   |   | Operating |   |   |              |
+          +--------------+   |   | Systems   |   |   +--------------+
+                             |   +-----------+   |
+                             +-------------------+
+
+.. warning::
+
+   The iSCSI gateway is in maintenance as of November 2022. This means that
+   it is no longer in active development and will not be updated to add
+   new features. 
+
+.. toctree::
+  :maxdepth: 1
+
+  Requirements <iscsi-requirements>
+  Configuring the iSCSI Target <iscsi-targets>
+  Configuring the iSCSI Initiators <iscsi-initiators>
+  Monitoring the iSCSI Gateways <iscsi-monitoring>
diff --git a/doc/rbd/iscsi-requirements.rst b/doc/rbd/iscsi-requirements.rst
new file mode 100644
index 000000000..50dfc2a27
--- /dev/null
+++ b/doc/rbd/iscsi-requirements.rst
@@ -0,0 +1,51 @@
+==========================
+iSCSI Gateway Requirements
+==========================
+
+It is recommended to provision two to four iSCSI gateway nodes to
+realize a highly available Ceph iSCSI gateway solution.
+
+For hardware recommendations, see :ref:`hardware-recommendations` .
+
+.. note::
+    On iSCSI gateway nodes the memory footprint is a function of
+    of the RBD images mapped and can grow to be large. Plan memory
+    requirements accordingly based on the number RBD images to be mapped.
+
+There are no specific iSCSI gateway options for the Ceph Monitors or
+OSDs, but it is important to lower the default heartbeat interval for
+detecting down OSDs to reduce the possibility of initiator timeouts.
+The following configuration options are suggested::
+
+        [osd]
+        osd heartbeat grace = 20
+        osd heartbeat interval = 5
+
+-   Updating Running State From a Ceph Monitor Node
+
+   ::
+
+       ceph tell <daemon_type>.<id> config set <parameter_name> <new_value>
+
+   ::
+
+       ceph tell osd.* config set osd_heartbeat_grace 20
+       ceph tell osd.* config set osd_heartbeat_interval 5
+
+-  Updating Running State On Each OSD Node
+
+   ::
+
+       ceph daemon <daemon_type>.<id> config set osd_client_watch_timeout 15
+
+   ::
+
+       ceph daemon osd.0 config set osd_heartbeat_grace 20
+       ceph daemon osd.0 config set osd_heartbeat_interval 5
+
+For more details on setting Ceph's configuration options, see
+:ref:`configuring-ceph`.  Be sure to persist these settings in
+``/etc/ceph.conf`` or, on Mimic and later releases, in the
+centralized config store.
+
+
diff --git a/doc/rbd/iscsi-target-ansible.rst b/doc/rbd/iscsi-target-ansible.rst
new file mode 100644
index 000000000..f89c4a0d2
--- /dev/null
+++ b/doc/rbd/iscsi-target-ansible.rst
@@ -0,0 +1,236 @@
+==========================================
+Configuring the iSCSI Target using Ansible
+==========================================
+
+The Ceph iSCSI gateway is the iSCSI target node and also a Ceph client
+node. The Ceph iSCSI gateway can be provisioned on dedicated node
+or be colocated on a Ceph Object Store Disk (OSD) node. The following steps will
+install and configure the Ceph iSCSI gateway for basic operation.
+
+**Requirements:**
+
+-  A running Ceph Luminous (12.2.x) cluster or newer
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  The ``ceph-iscsi`` package installed on all the iSCSI gateway nodes
+
+**Installation:**
+
+#. On the Ansible installer node, which could be either the administration node
+   or a dedicated deployment node, perform the following steps:
+
+   #. As ``root``, install the ``ceph-ansible`` package:
+
+      ::
+
+          # yum install ceph-ansible
+
+   #. Add an entry in ``/etc/ansible/hosts`` file for the gateway group:
+
+      ::
+
+          [iscsigws]
+          ceph-igw-1
+          ceph-igw-2
+
+.. note::
+  If co-locating the iSCSI gateway with an OSD node, then add the OSD node to the
+  ``[iscsigws]`` section.
+
+**Configuration:**
+
+The ``ceph-ansible`` package places a file in the ``/usr/share/ceph-ansible/group_vars/``
+directory called ``iscsigws.yml.sample``. Create a copy of this sample file named
+``iscsigws.yml``. Review the following Ansible variables and descriptions,
+and update accordingly. See the ``iscsigws.yml.sample`` for a full list of
+advanced variables.
+
++--------------------------------------+--------------------------------------+
+| Variable                             | Meaning/Purpose                      |
++======================================+======================================+
+| ``seed_monitor``                     | Each gateway needs access to the     |
+|                                      | ceph cluster for rados and rbd       |
+|                                      | calls. This means the iSCSI gateway  |
+|                                      | must have an appropriate             |
+|                                      | ``/etc/ceph/`` directory defined.    |
+|                                      | The ``seed_monitor`` host is used to |
+|                                      | populate the iSCSI gateway’s         |
+|                                      | ``/etc/ceph/`` directory.            |
++--------------------------------------+--------------------------------------+
+| ``cluster_name``                     | Define a custom storage cluster      |
+|                                      | name.                                |
++--------------------------------------+--------------------------------------+
+| ``gateway_keyring``                  | Define a custom keyring name.        |
++--------------------------------------+--------------------------------------+
+| ``deploy_settings``                  | If set to ``true``, then deploy the  |
+|                                      | settings when the playbook is ran.   |
++--------------------------------------+--------------------------------------+
+| ``perform_system_checks``            | This is a boolean value that checks  |
+|                                      | for multipath and lvm configuration  |
+|                                      | settings on each gateway. It must be |
+|                                      | set to true for at least the first   |
+|                                      | run to ensure multipathd and lvm are |
+|                                      | configured properly.                 |
++--------------------------------------+--------------------------------------+
+| ``api_user``                         | The user name for the API. The       |
+|                                      | default is `admin`.                  |
++--------------------------------------+--------------------------------------+
+| ``api_password``                     | The password for using the API. The  |
+|                                      | default is `admin`.                  |
++--------------------------------------+--------------------------------------+
+| ``api_port``                         | The TCP port number for using the    |
+|                                      | API. The default is `5000`.          |
++--------------------------------------+--------------------------------------+
+| ``api_secure``                       | True if TLS must be used. The        |
+|                                      | default is `false`. If true the user |
+|                                      | must create the necessary            |
+|                                      | certificate and key files. See the   |
+|                                      | gwcli man file for details.          |
++--------------------------------------+--------------------------------------+
+| ``trusted_ip_list``                  | A list of IPv4 or IPv6 addresses     |
+|                                      | who have access to the API. By       |
+|                                      | default, only the iSCSI gateway      |
+|                                      | nodes have access.                   |
++--------------------------------------+--------------------------------------+
+
+**Deployment:**
+
+Perform the following steps on the Ansible installer node.
+
+#. As ``root``, execute the Ansible playbook:
+
+   .. prompt:: bash #
+
+      cd /usr/share/ceph-ansible
+      ansible-playbook site.yml --limit iscsigws
+
+   .. note::
+    The Ansible playbook will handle RPM dependencies, setting up daemons,
+    and installing gwcli so it can be used to create iSCSI targets and export
+    RBD images as LUNs. In past versions, ``iscsigws.yml`` could define the
+    iSCSI target and other objects like clients, images and LUNs, but this is
+    no longer supported.
+
+#. Verify the configuration from an iSCSI gateway node:
+
+   .. prompt:: bash #
+
+      gwcli ls
+
+   .. note::
+    See the `Configuring the iSCSI Target using the Command Line Interface`_
+    section to create gateways, LUNs, and clients using the `gwcli` tool.
+
+   .. important::
+    Attempting to use the ``targetcli`` tool to change the configuration will
+    cause problems including ALUA misconfiguration and path failover
+    issues. There is the potential to corrupt data, to have mismatched
+    configuration across iSCSI gateways, and to have mismatched WWN information,
+    leading to client multipath problems.
+
+**Service Management:**
+
+The ``ceph-iscsi`` package installs the configuration management
+logic and a Systemd service called ``rbd-target-api``. When the Systemd
+service is enabled, the ``rbd-target-api`` will start at boot time and
+will restore the Linux IO state. The Ansible playbook disables the
+target service during the deployment. Below are the outcomes of when
+interacting with the ``rbd-target-api`` Systemd service.
+
+.. prompt:: bash #
+
+   systemctl <start|stop|restart|reload> rbd-target-api
+
+-  ``reload``
+
+   A reload request will force ``rbd-target-api`` to reread the
+   configuration and apply it to the current running environment. This
+   is normally not required, since changes are deployed in parallel from
+   Ansible to all iSCSI gateway nodes
+
+-  ``stop``
+
+   A stop request will close the gateway’s portal interfaces, dropping
+   connections to clients and wipe the current LIO configuration from
+   the kernel. This returns the iSCSI gateway to a clean state. When
+   clients are disconnected, active I/O is rescheduled to the other
+   iSCSI gateways by the client side multipathing layer.
+
+**Removing the Configuration:**
+
+The ``ceph-ansible`` package provides an Ansible playbook to
+remove the iSCSI gateway configuration and related RBD images. The
+Ansible playbook is ``/usr/share/ceph-ansible/purge_gateways.yml``. When
+this Ansible playbook is ran a prompted for the type of purge to
+perform:
+
+*lio* :
+
+In this mode the LIO configuration is purged on all iSCSI gateways that
+are defined. Disks that were created are left untouched within the Ceph
+storage cluster.
+
+*all* :
+
+When ``all`` is chosen, the LIO configuration is removed together with
+**all** RBD images that were defined within the iSCSI gateway
+environment, other unrelated RBD images will not be removed. Ensure the
+correct mode is chosen, this operation will delete data.
+
+.. warning::
+  A purge operation is destructive action against your iSCSI gateway
+  environment.
+
+.. warning::
+  A purge operation will fail, if RBD images have snapshots or clones
+  and are exported through the Ceph iSCSI gateway.
+
+.. highlight:: console
+
+::
+
+    [root@rh7-iscsi-client ceph-ansible]# ansible-playbook purge_gateways.yml
+    Which configuration elements should be purged? (all, lio or abort) [abort]: all
+
+
+    PLAY [Confirm removal of the iSCSI gateway configuration] *********************
+
+
+    GATHERING FACTS ***************************************************************
+    ok: [localhost]
+
+
+    TASK: [Exit playbook if user aborted the purge] *******************************
+    skipping: [localhost]
+
+
+    TASK: [set_fact ] *************************************************************
+    ok: [localhost]
+
+
+    PLAY [Removing the gateway configuration] *************************************
+
+
+    GATHERING FACTS ***************************************************************
+    ok: [ceph-igw-1]
+    ok: [ceph-igw-2]
+
+
+    TASK: [igw_purge | purging the gateway configuration] *************************
+    changed: [ceph-igw-1]
+    changed: [ceph-igw-2]
+
+
+    TASK: [igw_purge | deleting configured rbd devices] ***************************
+    changed: [ceph-igw-1]
+    changed: [ceph-igw-2]
+
+
+    PLAY RECAP ********************************************************************
+    ceph-igw-1                 : ok=3    changed=2    unreachable=0    failed=0
+    ceph-igw-2                 : ok=3    changed=2    unreachable=0    failed=0
+    localhost                  : ok=2    changed=0    unreachable=0    failed=0
+
+
+.. _Configuring the iSCSI Target using the Command Line Interface: ../iscsi-target-cli
diff --git a/doc/rbd/iscsi-target-cli-manual-install.rst b/doc/rbd/iscsi-target-cli-manual-install.rst
new file mode 100644
index 000000000..005f8aa94
--- /dev/null
+++ b/doc/rbd/iscsi-target-cli-manual-install.rst
@@ -0,0 +1,190 @@
+==============================
+Manual ceph-iscsi Installation
+==============================
+
+**Requirements**
+
+To complete the installation of ceph-iscsi, there are 4 steps:
+
+1. Install common packages from your Linux distribution's software repository
+2. Install Git to fetch the remaining packages directly from their Git repositories
+3. Ensure a compatible kernel is used
+4. Install all the components of ceph-iscsi and start associated daemons:
+
+   -  tcmu-runner
+   -  rtslib-fb
+   -  configshell-fb
+   -  targetcli-fb
+   -  ceph-iscsi
+
+
+1. Install Common Packages
+==========================
+
+The following packages will be used by ceph-iscsi and target tools.
+They must be installed from your Linux distribution's software repository
+on each machine that will be a iSCSI gateway:
+
+-  libnl3
+-  libkmod
+-  librbd1
+-  pyparsing
+-  python kmod
+-  python pyudev
+-  python gobject
+-  python urwid
+-  python pyparsing
+-  python rados
+-  python rbd
+-  python netifaces
+-  python crypto
+-  python requests
+-  python flask
+-  pyOpenSSL
+
+
+2. Install Git
+==============
+
+In order to install all the packages needed to run iSCSI with Ceph, you need to download them directly from their repository by using Git.
+On CentOS/RHEL execute:
+
+.. prompt:: bash >
+
+   sudo yum install git
+
+On Debian/Ubuntu execute:
+
+.. prompt:: bash >
+
+   sudo apt install git
+   
+To know more about Git and how it works, please, visit https://git-scm.com
+
+
+3. Ensure a compatible kernel is used
+=====================================
+
+Ensure you use a supported kernel that contains the required Ceph iSCSI patches:
+
+-  all Linux distribution with a kernel v4.16 or newer, or
+-  Red Hat Enterprise Linux or CentOS 7.5 or later (in these distributions ceph-iscsi support is backported)
+
+If you are already using a compatible kernel, you can go to next step.
+However, if you are NOT using a compatible kernel then check your distro's
+documentation for specific instructions on how to build this kernel. The only
+Ceph iSCSI specific requirements are that the following build options must be
+enabled:
+
+    .. code-block:: ini
+    
+       CONFIG_TARGET_CORE=m
+       CONFIG_TCM_USER2=m
+       CONFIG_ISCSI_TARGET=m
+
+
+4. Install ceph-iscsi
+========================================================
+
+Finally, the remaining tools can be fetched directly from their Git repositories and their associated services started
+
+
+tcmu-runner
+-----------
+
+   Installation:
+
+   .. prompt:: bash >
+
+      git clone https://github.com/open-iscsi/tcmu-runner
+      cd tcmu-runner
+
+   Run the following command to install all the needed dependencies:
+
+   .. prompt:: bash >
+
+      ./extra/install_dep.sh
+   
+   Now you can build the tcmu-runner.
+   To do so, use the following build command:
+
+   .. prompt:: bash >
+
+      cmake -Dwith-glfs=false -Dwith-qcow=false -DSUPPORT_SYSTEMD=ON -DCMAKE_INSTALL_PREFIX=/usr
+      make install
+
+   Enable and start the daemon:
+
+   .. prompt:: bash >
+
+      systemctl daemon-reload
+      systemctl enable tcmu-runner
+      systemctl start tcmu-runner
+
+
+rtslib-fb
+---------
+
+   Installation:
+
+   .. prompt:: bash >
+
+      git clone https://github.com/open-iscsi/rtslib-fb.git
+      cd rtslib-fb
+      python setup.py install
+
+configshell-fb
+--------------
+
+   Installation:
+
+   .. prompt:: bash >
+
+      git clone https://github.com/open-iscsi/configshell-fb.git
+      cd configshell-fb
+      python setup.py install
+
+targetcli-fb
+------------
+
+   Installation:
+
+   .. prompt:: bash >
+
+      git clone https://github.com/open-iscsi/targetcli-fb.git
+      cd targetcli-fb
+      python setup.py install
+      mkdir /etc/target
+      mkdir /var/target
+
+   .. warning:: The ceph-iscsi tools assume they are managing all targets
+      on the system. If targets have been setup and are being managed by
+      targetcli the target service must be disabled.
+
+ceph-iscsi
+-----------------
+
+   Installation:
+
+   .. prompt:: bash >
+
+      git clone https://github.com/ceph/ceph-iscsi.git
+      cd ceph-iscsi
+      python setup.py install --install-scripts=/usr/bin
+      cp usr/lib/systemd/system/rbd-target-gw.service /lib/systemd/system
+      cp usr/lib/systemd/system/rbd-target-api.service /lib/systemd/system
+
+   Enable and start the daemon:
+
+   .. prompt:: bash >
+
+      systemctl daemon-reload
+      systemctl enable rbd-target-gw
+      systemctl start rbd-target-gw
+      systemctl enable rbd-target-api
+      systemctl start rbd-target-api
+
+Installation is complete. Proceed to the setup section in the
+`main ceph-iscsi CLI page`_.
+
+.. _`main ceph-iscsi CLI page`: ../iscsi-target-cli
diff --git a/doc/rbd/iscsi-target-cli.rst b/doc/rbd/iscsi-target-cli.rst
new file mode 100644
index 000000000..44da56000
--- /dev/null
+++ b/doc/rbd/iscsi-target-cli.rst
@@ -0,0 +1,266 @@
+=============================================================
+Configuring the iSCSI Target using the Command Line Interface
+=============================================================
+
+The Ceph iSCSI gateway is both an iSCSI target  and a Ceph client;
+think of it as a "translator" between Ceph's RBD interface
+and the iSCSI standard. The Ceph iSCSI gateway can run on a
+standalone node or be colocated with other daemons eg. on
+a Ceph Object Store Disk (OSD) node.  When co-locating, ensure
+that sufficient CPU and memory are available to share.
+The following steps install and configure the Ceph iSCSI gateway for basic operation.
+
+**Requirements:**
+
+-  A running Ceph Luminous or later storage cluster
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  The following packages must be installed from your Linux distribution's software repository:
+
+   -  ``targetcli-2.1.fb47`` or newer package
+
+   -  ``python-rtslib-2.1.fb68`` or newer package
+
+   -  ``tcmu-runner-1.4.0`` or newer package
+
+   -  ``ceph-iscsi-3.2`` or newer package
+
+     .. important::
+        If previous versions of these packages exist, then they must
+        be removed first before installing the newer versions.
+
+Do the following steps on the Ceph iSCSI gateway node before proceeding
+to the *Installing* section:
+
+#. If the Ceph iSCSI gateway is not colocated on an OSD node, then copy
+   the Ceph configuration files, located in ``/etc/ceph/``, from a
+   running Ceph node in the storage cluster to the iSCSI Gateway node.
+   The Ceph configuration files must exist on the iSCSI gateway node
+   under ``/etc/ceph/``.
+
+#. Install and configure the `Ceph Command-line Interface`_
+
+#. If needed, open TCP ports 3260 and 5000 on the firewall.
+
+   .. note::
+      Access to port 5000 should be restricted to a trusted internal network or
+      only the individual hosts where ``gwcli`` is used or ``ceph-mgr`` daemons
+      are running.
+
+#. Create a new or use an existing RADOS Block Device (RBD).
+
+**Installing:**
+
+If you are using the upstream ceph-iscsi package follow the
+`manual install instructions`_.
+
+.. _`manual install instructions`: ../iscsi-target-cli-manual-install
+
+.. toctree::
+   :hidden:
+
+   iscsi-target-cli-manual-install
+
+For rpm based instructions execute the following commands:
+
+#. As ``root``, on all iSCSI gateway nodes, install the
+   ``ceph-iscsi`` package:
+
+   .. prompt:: bash #
+
+      yum install ceph-iscsi
+
+#. As ``root``, on all iSCSI gateway nodes, install the ``tcmu-runner``
+   package:
+
+   .. prompt:: bash #
+
+      yum install tcmu-runner
+
+**Setup:**
+
+#. gwcli requires a pool with the name ``rbd``, so it can store metadata
+   like the iSCSI configuration. To check if this pool has been created
+   run:
+
+   .. prompt:: bash #
+
+      ceph osd lspools
+
+   If it does not exist instructions for creating pools can be found on the
+   `RADOS pool operations page
+   <http://docs.ceph.com/en/latest/rados/operations/pools/>`_.
+
+#. As ``root``, on a iSCSI gateway node, create a file named
+   ``iscsi-gateway.cfg`` in the ``/etc/ceph/`` directory:
+
+   .. prompt:: bash #
+
+      touch /etc/ceph/iscsi-gateway.cfg
+
+   #. Edit the ``iscsi-gateway.cfg`` file and add the following lines:
+
+      .. code-block:: ini
+
+          [config]
+          # Name of the Ceph storage cluster. A suitable Ceph configuration file allowing
+          # access to the Ceph storage cluster from the gateway node is required, if not
+          # colocated on an OSD node.
+          cluster_name = ceph
+
+          # Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph
+          # directory and reference the filename here
+          gateway_keyring = ceph.client.admin.keyring
+
+
+          # API settings.
+          # The API supports a number of options that allow you to tailor it to your
+          # local environment. If you want to run the API under https, you will need to
+          # create cert/key files that are compatible for each iSCSI gateway node, that is
+          # not locked to a specific node. SSL cert and key files *must* be called
+          # 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory
+          # on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'
+          # to switch to https mode.
+
+          # To support the API, the bare minimum settings are:
+          api_secure = false
+
+          # Additional API configuration options are as follows, defaults shown.
+          # api_user = admin
+          # api_password = admin
+          # api_port = 5001
+          # trusted_ip_list = 192.168.0.10,192.168.0.11
+
+      .. note::
+        trusted_ip_list is a list of IP addresses on each iSCSI gateway that
+        will be used for management operations like target creation, LUN
+        exporting, etc. The IP can be the same that will be used for iSCSI
+        data, like READ/WRITE commands to/from the RBD image, but using
+        separate IPs is recommended.
+
+      .. important::
+        The ``iscsi-gateway.cfg`` file must be identical on all iSCSI gateway nodes.
+
+   #. As ``root``, copy the ``iscsi-gateway.cfg`` file to all iSCSI
+      gateway nodes.
+
+#. As ``root``, on all iSCSI gateway nodes, enable and start the API
+   service:
+
+   .. prompt:: bash #
+
+      systemctl daemon-reload
+       
+      systemctl enable rbd-target-gw
+      systemctl start rbd-target-gw
+
+      systemctl enable rbd-target-api
+      systemctl start rbd-target-api
+
+
+**Configuring:**
+
+gwcli will create and configure the iSCSI target and RBD images and copy the
+configuration across the gateways setup in the last section. Lower level
+tools including targetcli and rbd can be used to query the local configuration,
+but should not be used to modify it. This next section will demonstrate how
+to create a iSCSI target and export a RBD image as LUN 0.
+
+#. As ``root``, on a iSCSI gateway node, start the iSCSI gateway
+   command-line interface:
+
+   .. prompt:: bash #
+
+      gwcli
+
+#. Go to iscsi-targets and create a target with the name
+   iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw:
+
+   .. code-block:: console
+
+       > /> cd /iscsi-targets
+       > /iscsi-targets>  create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw
+
+#. Create the iSCSI gateways. The IPs used below are the ones that will be
+   used for iSCSI data like READ and WRITE commands. They can be the
+   same IPs used for management operations listed in trusted_ip_list,
+   but it is recommended that different IPs are used.
+
+   .. code-block:: console
+
+       > /iscsi-targets> cd iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/gateways
+       > /iscsi-target...-igw/gateways>  create ceph-gw-1 10.172.19.21
+       > /iscsi-target...-igw/gateways>  create ceph-gw-2 10.172.19.22
+
+   If not using RHEL/CentOS or using an upstream or ceph-iscsi-test kernel,
+   the skipchecks=true argument must be used. This will avoid the Red Hat kernel
+   and rpm checks:
+
+   .. code-block:: console
+
+       > /iscsi-targets> cd iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/gateways
+       > /iscsi-target...-igw/gateways>  create ceph-gw-1 10.172.19.21 skipchecks=true
+       > /iscsi-target...-igw/gateways>  create ceph-gw-2 10.172.19.22 skipchecks=true
+
+#. Add a RBD image with the name disk_1 in the pool rbd:
+
+   .. code-block:: console
+
+       > /iscsi-target...-igw/gateways> cd /disks
+       > /disks> create pool=rbd image=disk_1 size=90G
+
+#. Create a client with the initiator name iqn.1994-05.com.redhat:rh7-client:
+
+   .. code-block:: console
+
+       > /disks> cd /iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts
+       > /iscsi-target...eph-igw/hosts>  create iqn.1994-05.com.redhat:rh7-client
+
+#. Set the initiator CHAP username and password which the target would
+   use when authenticating the initiator:
+
+   .. code-block:: console
+
+       > /iscsi-target...at:rh7-client>  auth username=myusername password=mypassword
+
+   .. warning::
+      CHAP must always be configured. Without CHAP, the target will
+      reject any login requests.
+
+   To use mutual (bidirectional) authentication, also set the target CHAP
+   username and password which the initiator would use when authenticating
+   the target:
+
+   .. code-block:: console
+
+       > /iscsi-target...at:rh7-client>  auth username=myusername password=mypassword mutual_username=mytgtusername mutual_password=mytgtpassword
+
+   .. note::
+      CHAP usernames must be between 8 and 64 characters long.  Valid
+      characters: ``0`` to ``9``, ``a`` to ``z``, ``A`` to ``Z``, ``@``,
+      ``_``, ``-``, ``.``, ``:``.
+
+   .. note::
+      CHAP passwords must be between 12 and 16 characters long.  Valid
+      characters: ``0`` to ``9``, ``a`` to ``z``, ``A`` to ``Z``, ``@``,
+      ``_``, ``-``, ``/``.
+
+   .. note::
+      For mutual CHAP, initiator and target usernames and passwords
+      must not be the same.
+
+#. Add the disk to the client:
+
+   .. code-block:: console
+
+       > /iscsi-target...at:rh7-client> disk add rbd/disk_1
+
+The next step is to configure the iSCSI initiators.
+
+.. _`Ceph Command-line Interface`: ../../start/quick-rbd/#install-ceph
+
+.. toctree::
+   :hidden:
+
+   ../../start/quick-rbd
diff --git a/doc/rbd/iscsi-targets.rst b/doc/rbd/iscsi-targets.rst
new file mode 100644
index 000000000..d2a035283
--- /dev/null
+++ b/doc/rbd/iscsi-targets.rst
@@ -0,0 +1,27 @@
+=============
+iSCSI Targets
+=============
+
+Traditionally, block-level access to a Ceph storage cluster has been
+limited to QEMU and ``librbd``, which is a key enabler for adoption
+within OpenStack environments. Starting with the Ceph Luminous release,
+block-level access is expanding to offer standard iSCSI support allowing
+wider platform usage, and potentially opening new use cases.
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  A working Ceph Storage cluster, deployed with ``ceph-ansible`` or using the command-line interface
+
+-  iSCSI gateways nodes, which can either be colocated with OSD nodes or on dedicated nodes
+
+-  Separate network subnets for iSCSI front-end traffic and Ceph back-end traffic
+
+A choice of using Ansible or the command-line interface are the
+available deployment methods for installing and configuring the Ceph
+iSCSI gateway:
+
+.. toctree::
+  :maxdepth: 1
+
+  Using Ansible <iscsi-target-ansible>
+  Using the Command Line Interface <iscsi-target-cli>
diff --git a/doc/rbd/libvirt.rst b/doc/rbd/libvirt.rst
new file mode 100644
index 000000000..e3523f8a8
--- /dev/null
+++ b/doc/rbd/libvirt.rst
@@ -0,0 +1,323 @@
+=================================
+ Using libvirt with Ceph RBD
+=================================
+
+.. index:: Ceph Block Device; livirt
+
+The ``libvirt`` library creates a virtual machine abstraction layer between 
+hypervisor interfaces and the software applications that use them. With 
+``libvirt``, developers and system administrators can focus on a common 
+management framework, common API, and common shell interface (i.e., ``virsh``)
+to many different hypervisors, including: 
+
+- QEMU/KVM
+- XEN
+- LXC
+- VirtualBox
+- etc.
+
+Ceph block devices support QEMU/KVM. You can use Ceph block devices with
+software that interfaces with ``libvirt``. The following stack diagram
+illustrates how ``libvirt`` and QEMU use Ceph block devices via ``librbd``. 
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+The most common ``libvirt`` use case involves providing Ceph block devices to
+cloud solutions like OpenStack or CloudStack. The cloud solution uses
+``libvirt`` to  interact with QEMU/KVM, and QEMU/KVM interacts with Ceph block
+devices via  ``librbd``. See `Block Devices and OpenStack`_ and `Block Devices
+and CloudStack`_ for details. See `Installation`_ for installation details.
+
+You can also use Ceph block devices with ``libvirt``, ``virsh`` and the
+``libvirt`` API. See `libvirt Virtualization API`_ for details.
+
+
+To create VMs that use Ceph block devices, use the procedures in the following
+sections. In the exemplary embodiment, we have used ``libvirt-pool`` for the pool
+name, ``client.libvirt`` for the user name, and ``new-libvirt-image`` for  the
+image name. You may use any value you like, but ensure you replace those values
+when executing commands in the subsequent procedures.
+
+
+Configuring Ceph
+================
+
+To configure Ceph for use with ``libvirt``, perform the following steps:
+
+#. `Create a pool`_. The following example uses the 
+   pool name ``libvirt-pool``.::
+
+	ceph osd pool create libvirt-pool
+
+   Verify the pool exists. :: 
+
+	ceph osd lspools
+
+#. Use the ``rbd`` tool to initialize the pool for use by RBD::
+
+        rbd pool init <pool-name>
+
+#. `Create a Ceph User`_ (or use ``client.admin`` for version 0.9.7 and
+   earlier). The following example uses the Ceph user name ``client.libvirt``
+   and references ``libvirt-pool``. ::
+
+	ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=libvirt-pool'
+	
+   Verify the name exists. :: 
+   
+	ceph auth ls
+
+   **NOTE**: ``libvirt`` will access Ceph using the ID ``libvirt``, 
+   not the Ceph name ``client.libvirt``. See `User Management - User`_ and 
+   `User Management - CLI`_ for a detailed explanation of the difference 
+   between ID and name.	
+
+#. Use QEMU to `create an image`_ in your RBD pool. 
+   The following example uses the image name ``new-libvirt-image``
+   and references ``libvirt-pool``. ::
+
+	qemu-img create -f rbd rbd:libvirt-pool/new-libvirt-image 2G
+
+   Verify the image exists. :: 
+
+	rbd -p libvirt-pool ls
+
+   **NOTE:** You can also use `rbd create`_ to create an image, but we
+   recommend ensuring that QEMU is working properly.
+
+.. tip:: Optionally, if you wish to enable debug logs and the admin socket for
+   this client, you can add the following section to ``/etc/ceph/ceph.conf``::
+
+	[client.libvirt]
+	log file = /var/log/ceph/qemu-guest-$pid.log
+	admin socket = /var/run/ceph/$cluster-$type.$id.$pid.$cctid.asok
+
+   The ``client.libvirt`` section name should match the cephx user you created
+   above.  
+   If SELinux or AppArmor is enabled, note that this could prevent the client
+   process (qemu via libvirt) from doing some operations, such as writing logs
+   or operate the images or admin socket to the destination locations (``/var/
+   log/ceph`` or ``/var/run/ceph``). Additionally, make sure that the libvirt
+   and qemu users have appropriate access to the specified directory.
+
+
+Preparing the VM Manager
+========================
+
+You may use ``libvirt`` without a VM manager, but you may find it simpler to
+create your first domain with ``virt-manager``. 
+
+#. Install a virtual machine manager. See `KVM/VirtManager`_ for details. ::
+
+	sudo apt-get install virt-manager
+
+#. Download an OS image (if necessary).
+
+#. Launch the virtual machine manager. :: 
+
+	sudo virt-manager
+
+
+
+Creating a VM
+=============
+
+To create a VM with ``virt-manager``, perform the following steps:
+
+#. Press the **Create New Virtual Machine** button. 
+
+#. Name the new virtual machine domain. In the exemplary embodiment, we
+   use the name ``libvirt-virtual-machine``. You may use any name you wish,
+   but ensure you replace ``libvirt-virtual-machine`` with the name you 
+   choose in subsequent commandline and configuration examples. :: 
+
+	libvirt-virtual-machine
+
+#. Import the image. ::
+
+	/path/to/image/recent-linux.img
+
+   **NOTE:** Import a recent image. Some older images may not rescan for 
+   virtual devices properly.
+   
+#. Configure and start the VM.
+
+#. You may use ``virsh list`` to verify the VM domain exists. ::
+
+	sudo virsh list
+
+#. Login to the VM (root/root)
+
+#. Stop the VM before configuring it for use with Ceph.
+
+
+Configuring the VM
+==================
+
+When configuring the VM for use with Ceph, it is important  to use ``virsh``
+where appropriate. Additionally, ``virsh`` commands often require root
+privileges  (i.e., ``sudo``) and will not return appropriate results or notify
+you that root privileges are required. For a reference of ``virsh``
+commands, refer to `Virsh Command Reference`_.
+
+
+#. Open the configuration file with ``virsh edit``. :: 
+
+	sudo virsh edit {vm-domain-name}
+
+   Under ``<devices>`` there should be a ``<disk>`` entry. :: 
+
+	<devices>
+		<emulator>/usr/bin/kvm</emulator>
+		<disk type='file' device='disk'>
+			<driver name='qemu' type='raw'/>
+			<source file='/path/to/image/recent-linux.img'/>
+			<target dev='vda' bus='virtio'/>
+			<address type='drive' controller='0' bus='0' unit='0'/>
+		</disk>
+
+
+   Replace ``/path/to/image/recent-linux.img`` with the path to the OS image.
+   The minimum kernel for using the faster ``virtio`` bus is 2.6.25. See 
+   `Virtio`_ for details.
+
+   **IMPORTANT:** Use ``sudo virsh edit`` instead of a text editor. If you edit 
+   the configuration file under ``/etc/libvirt/qemu`` with a text editor, 
+   ``libvirt`` may not recognize the change. If there is a discrepancy between 
+   the contents of the XML file under ``/etc/libvirt/qemu`` and the result of 
+   ``sudo virsh dumpxml {vm-domain-name}``, then your VM may not work 
+   properly.
+   
+
+#. Add the Ceph RBD image you created as a ``<disk>`` entry. :: 
+
+	<disk type='network' device='disk'>
+		<source protocol='rbd' name='libvirt-pool/new-libvirt-image'>
+			<host name='{monitor-host}' port='6789'/>
+		</source>
+		<target dev='vdb' bus='virtio'/>
+	</disk>
+
+   Replace ``{monitor-host}`` with the name of your host, and replace the 
+   pool and/or image name as necessary. You may add multiple ``<host>`` 
+   entries for your Ceph monitors. The ``dev`` attribute is the logical
+   device name that will appear under the ``/dev`` directory of your 
+   VM. The optional ``bus`` attribute indicates the type of disk device to 
+   emulate. The valid settings are driver specific (e.g., "ide", "scsi", 
+   "virtio", "xen", "usb" or "sata").
+   
+   See `Disks`_ for details of the ``<disk>`` element, and its child elements
+   and attributes.
+	
+#. Save the file.
+
+#. If your Ceph Storage Cluster has `Ceph Authentication`_ enabled (it does by 
+   default), you must generate a secret. :: 
+
+	cat > secret.xml <<EOF
+	<secret ephemeral='no' private='no'>
+		<usage type='ceph'>
+			<name>client.libvirt secret</name>
+		</usage>
+	</secret>
+	EOF
+
+#. Define the secret. ::
+
+	sudo virsh secret-define --file secret.xml
+	{uuid of secret}
+
+#. Get the ``client.libvirt`` key and save the key string to a file. ::
+
+	ceph auth get-key client.libvirt | sudo tee client.libvirt.key
+
+#. Set the UUID of the secret. :: 
+
+	sudo virsh secret-set-value --secret {uuid of secret} --base64 $(cat client.libvirt.key) && rm client.libvirt.key secret.xml
+
+   You must also set the secret manually by adding the following ``<auth>`` 
+   entry to the ``<disk>`` element you entered earlier (replacing the
+   ``uuid`` value with the result from the command line example above). ::
+
+	sudo virsh edit {vm-domain-name}
+
+   Then, add ``<auth></auth>`` element to the domain configuration file::
+
+	...
+	</source>
+	<auth username='libvirt'>
+		<secret type='ceph' uuid='{uuid of secret}'/>
+	</auth>
+	<target ... 
+
+
+   **NOTE:** The exemplary ID is ``libvirt``, not the Ceph name 
+   ``client.libvirt`` as generated at step 2 of `Configuring Ceph`_. Ensure 
+   you use the ID component of the Ceph name you generated. If for some reason 
+   you need to regenerate the secret, you will have to execute 
+   ``sudo virsh secret-undefine {uuid}`` before executing 
+   ``sudo virsh secret-set-value`` again.
+
+
+Summary
+=======
+
+Once you have configured the VM for use with Ceph, you can start the VM.
+To verify that the VM and Ceph are communicating, you may perform the
+following procedures.
+
+
+#. Check to see if Ceph is running:: 
+
+	ceph health
+
+#. Check to see if the VM is running. :: 
+
+	sudo virsh list
+
+#. Check to see if the VM is communicating with Ceph. Replace 
+   ``{vm-domain-name}`` with the name of your VM domain:: 
+
+	sudo virsh qemu-monitor-command --hmp {vm-domain-name} 'info block'
+
+#. Check to see if the device from ``<target dev='vdb' bus='virtio'/>`` exists::
+   
+       virsh domblklist {vm-domain-name} --details
+
+If everything looks okay, you may begin using the Ceph block device 
+within your VM.
+
+
+.. _Installation: ../../install
+.. _libvirt Virtualization API: http://www.libvirt.org
+.. _Block Devices and OpenStack: ../rbd-openstack
+.. _Block Devices and CloudStack: ../rbd-cloudstack
+.. _Create a pool: ../../rados/operations/pools#create-a-pool
+.. _Create a Ceph User: ../../rados/operations/user-management#add-a-user
+.. _create an image: ../qemu-rbd#creating-images-with-qemu
+.. _Virsh Command Reference: http://www.libvirt.org/virshcmdref.html
+.. _KVM/VirtManager: https://help.ubuntu.com/community/KVM/VirtManager
+.. _Ceph Authentication: ../../rados/configuration/auth-config-ref
+.. _Disks: http://www.libvirt.org/formatdomain.html#elementsDisks
+.. _rbd create: ../rados-rbd-cmds#creating-a-block-device-image
+.. _User Management - User: ../../rados/operations/user-management#user
+.. _User Management - CLI: ../../rados/operations/user-management#command-line-usage
+.. _Virtio: http://www.linux-kvm.org/page/Virtio
diff --git a/doc/rbd/man/index.rst b/doc/rbd/man/index.rst
new file mode 100644
index 000000000..110273acc
--- /dev/null
+++ b/doc/rbd/man/index.rst
@@ -0,0 +1,16 @@
+============================
+ Ceph Block Device Manpages
+============================
+
+.. toctree::
+   :maxdepth: 1
+
+   rbd <../../man/8/rbd>
+   rbd-fuse <../../man/8/rbd-fuse>
+   rbd-nbd <../../man/8/rbd-nbd>
+   rbd-ggate <../../man/8/rbd-ggate>
+   rbd-map <../../man/8/rbdmap>
+   ceph-rbdnamer <../../man/8/ceph-rbdnamer>
+   rbd-replay-prep <../../man/8/rbd-replay-prep>
+   rbd-replay <../../man/8/rbd-replay>
+   rbd-replay-many <../../man/8/rbd-replay-many>
diff --git a/doc/rbd/qemu-rbd.rst b/doc/rbd/qemu-rbd.rst
new file mode 100644
index 000000000..281335ebe
--- /dev/null
+++ b/doc/rbd/qemu-rbd.rst
@@ -0,0 +1,219 @@
+========================
+ QEMU and Block Devices
+========================
+
+.. index:: Ceph Block Device; QEMU KVM
+
+The most frequent Ceph Block Device use case involves providing block device
+images to virtual machines. For example, a user may create  a "golden" image
+with an OS and any relevant software in an ideal configuration. Then the user
+takes a snapshot of the image. Finally the user clones the snapshot (potentially
+many times). See `Snapshots`_ for details. The ability to make copy-on-write
+clones of a snapshot means that Ceph can provision block device images to
+virtual machines quickly, because the client doesn't have to download the entire
+image each time it spins up a new virtual machine.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+Ceph Block Devices attach to QEMU virtual machines. For details on
+QEMU, see  `QEMU Open Source Processor Emulator`_. For QEMU documentation, see
+`QEMU Manual`_. For installation details, see `Installation`_.
+
+.. important:: To use Ceph Block Devices with QEMU, you must have access to a 
+   running Ceph cluster.
+
+
+Usage
+=====
+
+The QEMU command line expects you to specify the Ceph pool and image name. You
+may also specify a snapshot. 
+
+QEMU will assume that Ceph configuration resides in the default
+location (e.g., ``/etc/ceph/$cluster.conf``) and that you are executing
+commands as the default ``client.admin`` user unless you expressly specify
+another Ceph configuration file path or another user. When specifying a user,
+QEMU uses the ``ID`` rather than the full ``TYPE:ID``. See `User Management -
+User`_ for details. Do not prepend the client type (i.e., ``client.``) to the
+beginning of the user  ``ID``, or you will receive an authentication error. You
+should have the key for the ``admin`` user or the key of another user you
+specify with the ``:id={user}`` option in a keyring file stored in default path
+(i.e., ``/etc/ceph`` or the local directory with appropriate file ownership and
+permissions. Usage takes the following form::
+
+	qemu-img {command} [options] rbd:{pool-name}/{image-name}[@snapshot-name][:option1=value1][:option2=value2...]
+
+For example, specifying the ``id`` and ``conf`` options might look like the following:: 
+
+	qemu-img {command} [options] rbd:glance-pool/maipo:id=glance:conf=/etc/ceph/ceph.conf
+
+.. tip:: Configuration values containing ``:``, ``@``, or ``=`` can be escaped with a
+         leading ``\`` character.
+
+
+Creating Images with QEMU
+=========================
+
+You can create a block device image from QEMU. You must specify ``rbd``,  the
+pool name, and the name of the image you wish to create. You must also specify
+the size of the image. ::
+
+	qemu-img create -f raw rbd:{pool-name}/{image-name} {size}
+
+For example::
+
+	qemu-img create -f raw rbd:data/foo 10G
+
+.. important:: The ``raw`` data format is really the only sensible
+   ``format`` option to use with RBD. Technically, you could use other
+   QEMU-supported formats (such as ``qcow2`` or ``vmdk``), but doing
+   so would add additional overhead, and would also render the volume
+   unsafe for virtual machine live migration when caching (see below)
+   is enabled.
+
+
+Resizing Images with QEMU
+=========================
+
+You can resize a block device image from QEMU. You must specify ``rbd``, 
+the pool name, and the name of the image you wish to resize. You must also
+specify the size of the image. ::
+
+	qemu-img resize rbd:{pool-name}/{image-name} {size}
+
+For example::
+
+	qemu-img resize rbd:data/foo 10G
+
+
+Retrieving Image Info with QEMU
+===============================
+
+You can retrieve block device image information from QEMU. You must 
+specify ``rbd``, the pool name, and the name of the image. ::
+
+	qemu-img info rbd:{pool-name}/{image-name}
+
+For example::
+
+	qemu-img info rbd:data/foo
+
+
+Running QEMU with RBD
+=====================
+
+QEMU can pass a block device from the host on to a guest, but since
+QEMU 0.15, there's no need to map an image as a block device on
+the host. Instead, QEMU attaches an image as a virtual block
+device directly via ``librbd``. This strategy increases performance
+by avoiding context switches and taking advantage of `RBD caching`_.
+
+You can use ``qemu-img`` to convert existing virtual machine images to Ceph
+block device images. For example, if you have a qcow2 image, you could run::
+
+    qemu-img convert -f qcow2 -O raw debian_squeeze.qcow2 rbd:data/squeeze
+
+To run a virtual machine booting from that image, you could run::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze
+
+`RBD caching`_ can significantly improve performance.
+Since QEMU 1.2, QEMU's cache options control ``librbd`` caching::
+
+    qemu -m 1024 -drive format=rbd,file=rbd:data/squeeze,cache=writeback
+
+If you have an older version of QEMU, you can set the ``librbd`` cache
+configuration (like any Ceph configuration option) as part of the
+'file' parameter::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze:rbd_cache=true,cache=writeback
+
+.. important:: If you set rbd_cache=true, you must set cache=writeback
+   or risk data loss. Without cache=writeback, QEMU will not send
+   flush requests to librbd. If QEMU exits uncleanly in this
+   configuration, file systems on top of rbd can be corrupted.
+
+.. _RBD caching: ../rbd-config-ref/#rbd-cache-config-settings
+
+
+.. index:: Ceph Block Device; discard trim and libvirt
+
+Enabling Discard/TRIM
+=====================
+
+Since Ceph version 0.46 and QEMU version 1.1, Ceph Block Devices support the
+discard operation. This means that a guest can send TRIM requests to let a Ceph
+block device reclaim unused space. This can be enabled in the guest by mounting
+``ext4`` or ``XFS`` with the ``discard`` option.
+
+For this to be available to the guest, it must be explicitly enabled
+for the block device. To do this, you must specify a
+``discard_granularity`` associated with the drive::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze,id=drive1,if=none \
+         -device driver=ide-hd,drive=drive1,discard_granularity=512
+
+Note that this uses the IDE driver. The virtio driver supports discard since Linux kernel version 5.0.
+
+If using libvirt, edit your libvirt domain's configuration file using ``virsh
+edit`` to include the ``xmlns:qemu`` value. Then, add a ``qemu:commandline``
+block as a child of that domain. The following example shows how to set two
+devices with ``qemu id=`` to different ``discard_granularity`` values.
+
+.. code-block:: xml
+
+	<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
+		<qemu:commandline>
+			<qemu:arg value='-set'/>
+			<qemu:arg value='block.scsi0-0-0.discard_granularity=4096'/>
+			<qemu:arg value='-set'/>
+			<qemu:arg value='block.scsi0-0-1.discard_granularity=65536'/>
+		</qemu:commandline>
+	</domain>
+
+
+.. index:: Ceph Block Device; cache options
+
+QEMU Cache Options
+==================
+
+QEMU's cache options correspond to the following Ceph `RBD Cache`_ settings.
+
+Writeback::
+
+   rbd_cache = true
+
+Writethrough::
+
+   rbd_cache = true
+   rbd_cache_max_dirty = 0
+
+None:: 
+
+   rbd_cache = false
+
+QEMU's cache settings override Ceph's cache settings (including settings that
+are explicitly set in the Ceph configuration file).
+
+.. note:: Prior to QEMU v2.4.0, if you explicitly set `RBD Cache`_ settings
+   in the Ceph configuration file, your Ceph settings override the QEMU cache
+   settings.
+
+.. _QEMU Open Source Processor Emulator: http://wiki.qemu.org/Main_Page
+.. _QEMU Manual: http://wiki.qemu.org/Manual
+.. _RBD Cache: ../rbd-config-ref/
+.. _Snapshots: ../rbd-snapshot/
+.. _Installation: ../../install
+.. _User Management - User: ../../rados/operations/user-management#user
diff --git a/doc/rbd/rados-rbd-cmds.rst b/doc/rbd/rados-rbd-cmds.rst
new file mode 100644
index 000000000..0bbcb2611
--- /dev/null
+++ b/doc/rbd/rados-rbd-cmds.rst
@@ -0,0 +1,326 @@
+=============================
+ Basic Block Device Commands
+=============================
+
+.. index:: Ceph Block Device; image management
+
+The ``rbd`` command enables you to create, list, introspect and remove block
+device images. You can also use it to clone images, create snapshots,
+rollback an image to a snapshot, view a snapshot, etc. For details on using
+the ``rbd`` command, see `RBD – Manage RADOS Block Device (RBD) Images`_ for
+details. 
+
+.. important:: To use Ceph Block Device commands, you must have access to 
+   a running Ceph cluster.
+
+Create a Block Device Pool
+==========================
+
+#. Use the ``ceph`` tool to `create a pool`_.
+
+#. Use the ``rbd`` tool to initialize the pool for use by RBD:
+
+   .. prompt:: bash $
+
+      rbd pool init <pool-name>
+
+   .. note:: The ``rbd`` tool assumes a default pool name of 'rbd' if no pool
+      name is specified in the command.
+
+
+Create a Block Device User
+==========================
+
+Unless otherwise specified, the ``rbd`` command uses the Ceph user ID ``admin``
+to access the Ceph cluster. The ``admin`` Ceph user ID allows full
+administrative access to the cluster. We recommend that you acess the Ceph
+cluster with a Ceph user ID that has fewer permissions than the ``admin`` Ceph
+user ID does. We call this non-``admin`` Ceph user ID a "block device user" or
+"Ceph user".
+
+To `create a Ceph user`_, use the ``ceph auth get-or-create`` command to
+specify the Ceph user ID name, monitor caps (capabilities), and OSD caps
+(capabilities):
+
+.. prompt:: bash $
+
+   ceph auth get-or-create client.{ID} mon 'profile rbd' osd 'profile {profile name} [pool={pool-name}][, profile ...]' mgr 'profile rbd [pool={pool-name}]'
+
+For example: to create a Ceph user ID named ``qemu`` that has read-write access
+to the pool ``vms`` and read-only access to the pool ``images``, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph auth get-or-create client.qemu mon 'profile rbd' osd 'profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=images'
+
+The output from the ``ceph auth get-or-create`` command is the keyring for the
+specified Ceph user ID, which can be written to
+``/etc/ceph/ceph.client.{ID}.keyring``.
+
+.. note:: Specify the Ceph user ID by providing the ``--id {id} argument when
+   using the ``rbd`` command. This argument is optional. 
+
+Creating a Block Device Image
+=============================
+
+Before you can add a block device to a node, you must create an image for it in
+the :term:`Ceph Storage Cluster`. To create a block device image, run a command of this form: 
+
+.. prompt:: bash $
+
+   rbd create --size {megabytes} {pool-name}/{image-name}
+
+For example, to create a 1GB image named ``bar`` that stores information in a
+pool named ``swimmingpool``, run this command:
+
+.. prompt:: bash $
+
+   rbd create --size 1024 swimmingpool/bar
+
+If you don't specify a pool when you create an image, then the image will be
+stored in the default pool ``rbd``. For example, if you ran this command, you
+would create a 1GB image named ``foo`` that is stored in the default pool
+``rbd``:
+
+.. prompt:: bash $
+
+   rbd create --size 1024 foo
+
+.. note:: You must create a pool before you can specify it as a source. See
+   `Storage Pools`_ for details.
+
+Listing Block Device Images
+===========================
+
+To list block devices in the ``rbd`` pool, run the following command:
+
+.. prompt:: bash $
+
+   rbd ls
+
+.. note:: ``rbd`` is the default pool name, and ``rbd ls`` lists the commands
+   in the default pool.
+
+To list block devices in a particular pool, run the following command, but
+replace ``{poolname}`` with the name of the pool: 
+
+.. prompt:: bash $
+
+   rbd ls {poolname}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd ls swimmingpool
+
+To list "deferred delete" block devices in the ``rbd`` pool, run the
+following command:
+
+.. prompt:: bash $
+
+   rbd trash ls
+
+To list "deferred delete" block devices in a particular pool, run the 
+following command, but replace ``{poolname}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd trash ls {poolname}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash ls swimmingpool
+
+Retrieving Image Information
+============================
+
+To retrieve information from a particular image, run the following command, but
+replace ``{image-name}`` with the name for the image:
+
+.. prompt:: bash $
+
+   rbd info {image-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd info foo
+	
+To retrieve information from an image within a pool, run the following command,
+but replace ``{image-name}`` with the name of the image and replace
+``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd info {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd info swimmingpool/bar
+
+.. note:: Other naming conventions are possible, and might conflict with the
+   naming convention described here. For example, ``userid/<uuid>`` is a
+   possible name for an RBD image, and such a name might (at the least) be
+   confusing. 
+
+Resizing a Block Device Image
+=============================
+
+:term:`Ceph Block Device` images are thin provisioned. They don't actually use
+any physical storage until you begin saving data to them. However, they do have
+a maximum capacity that you set with the ``--size`` option. If you want to
+increase (or decrease) the maximum size of a Ceph Block Device image, run one
+of the following commands:
+
+Increasing the Size of a Block Device Image
+-------------------------------------------
+
+.. prompt:: bash $
+
+   rbd resize --size 2048 foo
+
+Decreasing the Size of a Block Device Image
+-------------------------------------------
+
+.. prompt:: bash $
+
+   rbd resize --size 2048 foo --allow-shrink
+
+
+Removing a Block Device Image
+=============================
+
+To remove a block device, run the following command, but replace
+``{image-name}`` with the name of the image you want to remove:
+
+.. prompt:: bash $
+
+   rbd rm {image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd rm foo
+
+Removing a Block Device from a Pool
+-----------------------------------
+
+To remove a block device from a pool, run the following command but replace
+``{image-name}`` with the name of the image to be removed, and replace
+``{pool-name}`` with the name of the pool from which the image is to be
+removed:
+
+.. prompt:: bash $
+
+   rbd rm {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd rm swimmingpool/bar
+
+"Defer Deleting" a Block Device from a Pool
+-------------------------------------------
+
+To defer delete a block device from a pool (which entails moving it to the
+"trash" and deleting it later), run the following command but replace
+``{image-name}`` with the name of the image to be moved to the trash and
+replace ``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd trash mv {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash mv swimmingpool/bar
+
+Removing a Deferred Block Device from a Pool
+--------------------------------------------
+
+To remove a deferred block device from a pool, run the following command but
+replace ``{image-}`` with the ID of the image to be removed, and replace
+``{pool-name}`` with the name of the pool from which the image is to be
+removed:
+
+.. prompt:: bash $
+
+   rbd trash rm {pool-name}/{image-}
+
+For example:
+
+.. prompt:: bash $
+   
+   rbd trash rm swimmingpool/2bf4474b0dc51
+
+.. note::
+
+  * You can move an image to the trash even if it has snapshot(s) or is
+    actively in use by clones. However, you cannot remove it from the trash
+    under those conditions.
+
+  * You can use ``--expires-at`` to set the deferment time (default is
+    ``now``). If the deferment time has not yet arrived, you cannot remove the
+    image unless you use ``--force``.
+
+Restoring a Block Device Image
+==============================
+
+To restore a deferred delete block device in the rbd pool, run the 
+following command but replace ``{image-id}`` with the ID of the image:
+
+.. prompt:: bash $
+
+   rbd trash restore {image-id}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash restore 2bf4474b0dc51
+
+Restoring a Block Device Image in a Specific Pool
+-------------------------------------------------
+
+To restore a deferred delete block device in a particular pool, run the
+following command but replace ``{image-id}`` with the ID of the image and
+replace ``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+  rbd trash restore {pool-name}/{image-id}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash restore swimmingpool/2bf4474b0dc51
+
+
+Renaming an Image While Restoring It
+------------------------------------
+
+You can also use ``--image`` to rename the image while restoring it. 
+
+For example:
+
+.. prompt:: bash $
+   
+   rbd trash restore swimmingpool/2bf4474b0dc51 --image new-name
+
+
+.. _create a pool: ../../rados/operations/pools/#create-a-pool
+.. _Storage Pools: ../../rados/operations/pools
+.. _RBD – Manage RADOS Block Device (RBD) Images: ../../man/8/rbd/
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
diff --git a/doc/rbd/rbd-cloudstack.rst b/doc/rbd/rbd-cloudstack.rst
new file mode 100644
index 000000000..1b961234b
--- /dev/null
+++ b/doc/rbd/rbd-cloudstack.rst
@@ -0,0 +1,157 @@
+=============================
+ Block Devices and CloudStack
+=============================
+
+You may use Ceph Block Device images with CloudStack 4.0 and higher through
+``libvirt``, which configures the QEMU interface to ``librbd``. Ceph stripes
+block device images as objects across the cluster, which means that large Ceph
+Block Device images have better performance than a standalone server!
+
+To use Ceph Block Devices with CloudStack 4.0 and higher, you must install QEMU,
+``libvirt``, and CloudStack first. We recommend using a separate physical host
+for your CloudStack installation. CloudStack recommends a minimum of 4GB of RAM
+and a dual-core processor, but more CPU and RAM will perform better. The
+following diagram depicts the CloudStack/Ceph technology stack.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                   CloudStack                      |
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. important:: To use Ceph Block Devices with CloudStack, you must have  
+   access to a running Ceph Storage Cluster.
+
+CloudStack integrates with Ceph's block devices to provide CloudStack with a
+back end for CloudStack's Primary Storage. The instructions below detail the
+setup for CloudStack Primary Storage.
+
+.. note:: We recommend installing with Ubuntu 14.04 or later so that 
+   you can use package installation instead of having to compile 
+   libvirt from source.
+
+Installing and configuring QEMU for use with CloudStack doesn't require any
+special handling. Ensure that you have a running Ceph Storage Cluster. Install
+QEMU and configure it for use with Ceph; then, install ``libvirt`` version
+0.9.13 or higher (you may need to compile from source) and ensure it is running
+with Ceph.
+
+
+.. note:: Ubuntu 14.04 and CentOS 7.2 will have ``libvirt`` with RBD storage
+   pool support enabled by default.
+
+.. index:: pools; CloudStack
+
+Create a Pool
+=============
+
+By default, Ceph block devices use the ``rbd`` pool. Create a pool for
+CloudStack NFS Primary Storage. Ensure your Ceph cluster is running, then create
+the pool. ::
+
+   ceph osd pool create cloudstack
+   
+See `Create a Pool`_ for details on specifying the number of placement groups
+for your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+A newly created pool must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pool::
+
+        rbd pool init cloudstack
+
+Create a Ceph User
+==================
+
+To access the Ceph cluster we require a Ceph user which has the correct
+credentials to access the ``cloudstack`` pool we just created. Although we could
+use ``client.admin`` for this, it's recommended to create a user with only
+access to the ``cloudstack`` pool. ::
+
+  ceph auth get-or-create client.cloudstack mon 'profile rbd' osd 'profile rbd pool=cloudstack'
+
+Use the information returned by the command in the next step when adding the 
+Primary Storage.
+
+See `User Management`_ for additional details.
+
+Add Primary Storage
+===================
+
+To add a Ceph block device as Primary Storage, the steps include: 
+
+#. Log in to the CloudStack UI.
+#. Click **Infrastructure** on the left side navigation bar. 
+#. Select **View All** under **Primary Storage**.
+#. Click the **Add Primary Storage** button on the top right hand side.
+#. Fill in the following information, according to your infrastructure setup:
+
+   - Scope (i.e. Cluster or Zone-Wide).
+   
+   - Zone.
+   
+   - Pod.
+   
+   - Cluster.
+   
+   - Name of Primary Storage.
+   
+   - For **Protocol**, select ``RBD``.
+   
+   - For **Provider**, select the appropriate provider type (i.e. DefaultPrimary, SolidFire, SolidFireShared, or CloudByte).  Depending on the provider chosen, fill out the information pertinent to your setup.
+   
+#. Add cluster information (``cephx`` is supported).
+
+   - For **RADOS Monitor**, provide the IP address of a Ceph monitor node.
+   
+   - For **RADOS Pool**, provide the name of an RBD pool.
+   
+   - For **RADOS User**, provide a user that has sufficient rights to the RBD pool. Note: Do not include the ``client.`` part of the user.
+   
+   - For **RADOS Secret**, provide the secret the user's secret.
+   
+   - **Storage Tags** are optional. Use tags at your own discretion. For more information about storage tags in CloudStack, refer to `Storage Tags`_.
+   
+#. Click **OK**.
+
+Create a Disk Offering
+======================
+
+To create a new disk offering, refer to `Create a New Disk Offering`_.
+Create a disk offering so that it matches the ``rbd`` tag.
+The ``StoragePoolAllocator`` will choose the  ``rbd``
+pool when searching for a suitable storage pool. If the disk offering doesn't
+match the ``rbd`` tag, the ``StoragePoolAllocator`` may select the pool you
+created (e.g., ``cloudstack``).
+
+
+Limitations
+===========
+
+- CloudStack will only bind to one monitor (You can however create a Round Robin DNS record over multiple monitors)
+
+
+
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+.. _Install and Configure QEMU: ../qemu-rbd
+.. _Install and Configure libvirt: ../libvirt
+.. _KVM Hypervisor Host Installation: http://docs.cloudstack.apache.org/en/latest/installguide/hypervisor/kvm.html
+.. _Storage Tags: http://docs.cloudstack.apache.org/en/latest/adminguide/storage.html#storage-tags
+.. _Create a New Disk Offering: http://docs.cloudstack.apache.org/en/latest/adminguide/service_offerings.html#creating-a-new-disk-offering
+.. _User Management: ../../rados/operations/user-management
diff --git a/doc/rbd/rbd-config-ref.rst b/doc/rbd/rbd-config-ref.rst
new file mode 100644
index 000000000..c21731adc
--- /dev/null
+++ b/doc/rbd/rbd-config-ref.rst
@@ -0,0 +1,265 @@
+=======================
+ Config Settings
+=======================
+
+See `Block Device`_ for additional details.
+
+Generic IO Settings
+===================
+
+.. confval:: rbd_compression_hint
+.. confval:: rbd_read_from_replica_policy
+.. confval:: rbd_default_order
+
+Cache Settings
+=======================
+
+.. sidebar:: Kernel Caching
+
+	The kernel driver for Ceph block devices can use the Linux page cache to
+	improve performance.
+
+The user space implementation of the Ceph block device (i.e., ``librbd``) cannot
+take advantage of the Linux page cache, so it includes its own in-memory
+caching, called "RBD caching." RBD caching behaves just like well-behaved hard
+disk caching.  When the OS sends a barrier or a flush request, all dirty data is
+written to the OSDs. This means that using write-back caching is just as safe as
+using a well-behaved physical hard disk with a VM that properly sends flushes
+(i.e. Linux kernel >= 2.6.32). The cache uses a Least Recently Used (LRU)
+algorithm, and in write-back mode it can coalesce contiguous requests for
+better throughput.
+
+The librbd cache is enabled by default and supports three different cache
+policies: write-around, write-back, and write-through. Writes return
+immediately under both the write-around and write-back policies, unless there
+are more than ``rbd_cache_max_dirty`` unwritten bytes to the storage cluster.
+The write-around policy differs from the write-back policy in that it does
+not attempt to service read requests from the cache, unlike the write-back
+policy, and is therefore faster for high performance write workloads. Under the
+write-through policy, writes return only when the data is on disk on all
+replicas, but reads may come from the cache.
+
+Prior to receiving a flush request, the cache behaves like a write-through cache
+to ensure safe operation for older operating systems that do not send flushes to
+ensure crash consistent behavior.
+
+If the librbd cache is disabled, writes and
+reads go directly to the storage cluster, and writes return only when the data
+is on disk on all replicas.
+
+.. note::
+   The cache is in memory on the client, and each RBD image has
+   its own.  Since the cache is local to the client, there's no coherency
+   if there are others accessing the image. Running GFS or OCFS on top of
+   RBD will not work with caching enabled.
+
+
+Option settings for RBD should be set in the ``[client]``
+section of your configuration file or the central config store. These settings
+include:
+
+.. confval:: rbd_cache
+.. confval:: rbd_cache_policy
+.. confval:: rbd_cache_writethrough_until_flush
+.. confval:: rbd_cache_size
+.. confval:: rbd_cache_max_dirty
+.. confval:: rbd_cache_target_dirty
+.. confval:: rbd_cache_max_dirty_age
+
+.. _Block Device: ../../rbd
+
+
+Read-ahead Settings
+=======================
+
+librbd supports read-ahead/prefetching to optimize small, sequential reads.
+This should normally be handled by the guest OS in the case of a VM,
+but boot loaders may not issue efficient reads. Read-ahead is automatically
+disabled if caching is disabled or if the policy is write-around.
+
+
+.. confval:: rbd_readahead_trigger_requests
+.. confval:: rbd_readahead_max_bytes
+.. confval:: rbd_readahead_disable_after_bytes
+
+Image Features
+==============
+
+RBD supports advanced features which can be specified via the command line when
+creating images or the default features can be configured via
+``rbd_default_features = <sum of feature numeric values>`` or
+``rbd_default_features = <comma-delimited list of CLI values>``.
+
+``Layering``
+
+:Description: Layering enables cloning.
+:Internal value: 1
+:CLI value: layering
+:Added in: v0.52 (Bobtail)
+:KRBD support: since v3.10
+:Default: yes
+
+``Striping v2``
+
+:Description: Striping spreads data across multiple objects. Striping helps with
+              parallelism for sequential read/write workloads.
+:Internal value: 2
+:CLI value: striping
+:Added in: v0.55 (Bobtail)
+:KRBD support: since v3.10 (default striping only, "fancy" striping added in v4.17)
+:Default: yes
+
+``Exclusive locking``
+
+:Description: When enabled, it requires a client to acquire a lock on an object
+              before making a write. Exclusive lock should only be enabled when
+              a single client is accessing an image at any given time.
+:Internal value: 4
+:CLI value: exclusive-lock
+:Added in: v0.92 (Hammer)
+:KRBD support: since v4.9
+:Default: yes
+
+``Object map``
+
+:Description: Object map support depends on exclusive lock support. Block
+              devices are thin provisioned, which means that they only store
+              data that actually has been written, ie. they are *sparse*. Object
+              map support helps track which objects actually exist (have data
+              stored on a device). Enabling object map support speeds up I/O
+              operations for cloning, importing and exporting a sparsely
+              populated image, and deleting.
+:Internal value: 8
+:CLI value: object-map
+:Added in: v0.93 (Hammer)
+:KRBD support: since v5.3
+:Default: yes
+
+
+``Fast-diff``
+
+:Description: Fast-diff support depends on object map support and exclusive lock
+              support. It adds another property to the object map, which makes
+              it much faster to generate diffs between snapshots of an image.
+              It is also much faster to calculate the actual data usage of a
+              snapshot or volume (``rbd du``).
+:Internal value: 16
+:CLI value: fast-diff
+:Added in: v9.0.1 (Infernalis)
+:KRBD support: since v5.3
+:Default: yes
+
+
+``Deep-flatten``
+
+:Description: Deep-flatten enables ``rbd flatten`` to work on all  snapshots of
+              an image, in addition to the image itself. Without it, snapshots
+              of an image will still rely on the parent, so the parent cannot be
+              deleted until the snapshots are first deleted. Deep-flatten makes
+              a parent independent of its clones, even if they have snapshots,
+              at the expense of using additional OSD device space.
+:Internal value: 32
+:CLI value: deep-flatten
+:Added in: v9.0.2 (Infernalis)
+:KRBD support: since v5.1
+:Default: yes
+
+
+``Journaling``
+
+:Description: Journaling support depends on exclusive lock support. Journaling
+              records all modifications to an image in the order they occur. RBD
+              mirroring can utilize the journal to replicate a crash-consistent
+              image to a remote cluster.  It is best to let ``rbd-mirror``
+              manage this feature only as needed, as enabling it long term may
+              result in substantial additional OSD space consumption.
+:Internal value: 64
+:CLI value: journaling
+:Added in: v10.0.1 (Jewel)
+:KRBD support: no
+:Default: no
+
+
+``Data pool``
+
+:Description: On erasure-coded pools, the image data block objects need to be stored on a separate pool from the image metadata.
+:Internal value: 128
+:Added in: v11.1.0 (Kraken)
+:KRBD support: since v4.11
+:Default: no
+
+
+``Operations``
+
+:Description: Used to restrict older clients from performing certain maintenance operations against an image (e.g. clone, snap create).
+:Internal value: 256
+:Added in: v13.0.2 (Mimic)
+:KRBD support: since v4.16
+
+
+``Migrating``
+
+:Description: Used to restrict older clients from opening an image when it is in migration state.
+:Internal value: 512
+:Added in: v14.0.1 (Nautilus)
+:KRBD support: no
+
+``Non-primary``
+
+:Description: Used to restrict changes to non-primary images using snapshot-based mirroring.
+:Internal value: 1024
+:Added in: v15.2.0 (Octopus)
+:KRBD support: no
+
+
+QoS Settings
+============
+
+librbd supports limiting per-image IO in several ways. These all apply
+to a given image within a given process - the same image used in
+multiple places, e.g. two separate VMs, would have independent limits.
+
+* **IOPS:** number of I/Os per second (any type of I/O)
+* **read IOPS:** number of read I/Os per second
+* **write IOPS:** number of write I/Os per second
+* **bps:** bytes per second (any type of I/O)
+* **read bps:** bytes per second read
+* **write bps:** bytes per second written
+
+Each of these limits operates independently of each other. They are
+all off by default. Every type of limit throttles I/O using a token
+bucket algorithm, with the ability to configure the limit (average
+speed over time) and potential for a higher rate (a burst) for a short
+period of time (burst_seconds). When any of these limits is reached,
+and there is no burst capacity left, librbd reduces the rate of that
+type of I/O to the limit.
+
+For example, if a read bps limit of 100MB was configured, but writes
+were not limited, writes could proceed as quickly as possible, while
+reads would be throttled to 100MB/s on average. If a read bps burst of
+150MB was set, and read burst seconds was set to five seconds, reads
+could proceed at 150MB/s for up to five seconds before dropping back
+to the 100MB/s limit.
+
+The following options configure these throttles:
+
+.. confval:: rbd_qos_iops_limit
+.. confval:: rbd_qos_iops_burst
+.. confval:: rbd_qos_iops_burst_seconds
+.. confval:: rbd_qos_read_iops_limit
+.. confval:: rbd_qos_read_iops_burst
+.. confval:: rbd_qos_read_iops_burst_seconds
+.. confval:: rbd_qos_write_iops_limit
+.. confval:: rbd_qos_write_iops_burst
+.. confval:: rbd_qos_write_iops_burst_seconds
+.. confval:: rbd_qos_bps_limit
+.. confval:: rbd_qos_bps_burst
+.. confval:: rbd_qos_bps_burst_seconds
+.. confval:: rbd_qos_read_bps_limit
+.. confval:: rbd_qos_read_bps_burst
+.. confval:: rbd_qos_read_bps_burst_seconds
+.. confval:: rbd_qos_write_bps_limit
+.. confval:: rbd_qos_write_bps_burst
+.. confval:: rbd_qos_write_bps_burst_seconds
+.. confval:: rbd_qos_schedule_tick_min
+.. confval:: rbd_qos_exclude_ops
diff --git a/doc/rbd/rbd-encryption.rst b/doc/rbd/rbd-encryption.rst
new file mode 100644
index 000000000..3f37a8b1c
--- /dev/null
+++ b/doc/rbd/rbd-encryption.rst
@@ -0,0 +1,246 @@
+======================
+ Image Encryption
+======================
+
+.. index:: Ceph Block Device; encryption
+
+Starting with the Pacific release, image-level encryption can be handled
+internally by RBD clients. This means you can set a secret key that will be
+used to encrypt a specific RBD image. This page describes the scope of the
+RBD encryption feature.
+
+.. note::
+   The ``krbd`` kernel module does not support encryption at this time.
+
+.. note::
+   External tools (e.g. dm-crypt, QEMU) can be used as well to encrypt
+   an RBD image, and the feature set and limitation set for that use may be
+   different than described here.
+
+Encryption Format
+=================
+
+By default, RBD images are not encrypted. To encrypt an RBD image, it needs to
+be formatted to one of the supported encryption formats. The format operation
+persists encryption metadata to the image. The encryption metadata usually
+includes information such as the encryption format and version, cipher
+algorithm and mode specification, as well as information used to secure the
+encryption key. The encryption key itself is protected by a user-kept secret
+(usually a passphrase), which is never persisted. The basic encryption format
+operation will require specifying the encryption format and a secret.
+
+Some of the encryption metadata may be stored as part of the image data,
+typically an encryption header will be written to the beginning of the raw
+image data. This means that the effective image size of the encrypted image may
+be lower than the raw image size. See the `Supported Formats`_ section for more
+details.
+
+.. note::
+   Unless explicitly (re-)formatted, clones of an encrypted image are
+   inherently encrypted using the same format and secret.
+
+.. note::
+   Clones of an encrypted image are always encrypted.
+   Re-formatting to plaintext is not supported.
+
+.. note::
+   Any data written to the image prior to its format may become unreadable,
+   though it may still occupy storage resources.
+
+.. note::
+   Images with the `journal feature`_ enabled cannot be formatted and encrypted
+   by RBD clients.
+
+Encryption Load
+=================
+
+Formatting an image is a necessary pre-requisite for enabling encryption.
+However, formatted images will still be treated as raw unencrypted images by
+all of the RBD APIs. In particular, an encrypted RBD image can be opened
+by the same APIs as any other image, and raw unencrypted data can be
+read / written. Such raw IOs may risk the integrity of the encryption format,
+for example by overriding encryption metadata located at the beginning of the
+image.
+
+In order to safely perform encrypted IO on the formatted image, an additional
+*encryption load* operation should be applied after opening the image. The
+encryption load operation requires supplying the encryption format and a secret
+for unlocking the encryption key for the image itself and each of its explicitly
+formatted ancestor images. Following a successful encryption load operation,
+all IOs for the opened image will be encrypted / decrypted. For a cloned
+image, this includes IOs for ancestor images as well. The encryption keys will
+be stored in-memory by the RBD client until the image is closed.
+
+.. note::
+   Once encryption has been loaded, no other encryption load / format
+   operations can be applied to the context of the opened image.
+
+.. note::
+   Once encryption has been loaded, API calls for retrieving the image size
+   and the parent overlap using the opened image context will return the
+   effective image size and the effective parent overlap respectively.
+
+.. note::
+   Once encryption has been loaded, API calls for resizing the image will
+   interpret the specified target size as effective image size.
+
+.. note::
+   If a clone of an encrypted image is explicitly formatted, the operation of
+   flattening the cloned image ceases to be transparent since the parent data
+   must be re-encrypted according to the cloned image format as it is copied
+   from the parent snapshot. If encryption is not loaded before the flatten
+   operation is issued, any parent data that was previously accessible in the
+   cloned image may become unreadable.
+
+.. note::
+   If a clone of an encrypted image is explicitly formatted, the operation of
+   shrinking the cloned image ceases to be transparent since in some cases
+   (e.g. if the cloned image has snapshots or if the cloned image is being
+   shrunk to a size that is not aligned with the object size) it involves
+   copying some data from the parent snapshot, similar to flattening. If
+   encryption is not loaded before the shrink operation is issued, any parent
+   data that was previously accessible in the cloned image may become
+   unreadable.
+
+.. note::
+   Encryption load can be automatically applied when mounting RBD images as
+   block devices via `rbd-nbd`_.
+
+Supported Formats
+=================
+
+LUKS
+~~~~~~~
+
+Both LUKS1 and LUKS2 are supported. The data layout is fully compliant with the
+LUKS specification. Thus, images formatted by RBD can be loaded using external
+LUKS-supporting tools such as dm-crypt or QEMU. Furthermore, existing LUKS
+data, created outside of RBD, can be imported (by copying the raw LUKS data
+into the image) and loaded by RBD encryption.
+
+.. note::
+   The LUKS formats are supported on Linux-based systems only.
+
+.. note::
+   Currently, only AES-128 and AES-256 encryption algorithms are supported.
+   Additionally, xts-plain64 is currently the only supported encryption mode.
+
+To use the LUKS format, start by formatting the image:
+
+.. prompt:: bash $
+
+    rbd encryption format [--cipher-alg {aes-128|aes-256}] {image-spec} {luks1|luks2} {passphrase-file}
+
+The encryption format operation generates a LUKS header and writes it to the
+beginning of the image. The header is appended with a single keyslot holding a
+randomly-generated encryption key, and is protected by the passphrase read from
+`passphrase-file`.
+
+.. note::
+   In older versions, if the content of `passphrase-file` ended with a newline
+   character, it was stripped off.
+
+By default, AES-256 in xts-plain64 mode (which is the current recommended mode,
+and the usual default for other tools) will be used. The format operation
+allows selecting AES-128 as well. Adding / removing passphrases is currently
+not supported by RBD, but can be applied to the raw RBD data using compatible
+tools such as cryptsetup.
+
+The LUKS header size can vary (up to 136MiB in LUKS2), but is usually up to
+16MiB, depending on the version of `libcryptsetup` installed. For optimal
+performance, the encryption format will set the data offset to be aligned with
+the image stripe period size. For example, expect a minimum overhead of 8MiB if
+using an image configured with an 8MiB object size and a minimum overhead of
+12MiB if using an image configured with a 4MiB object size and `stripe count`_
+of 3.
+
+In LUKS1, sectors, which are the minimal encryption units, are fixed at 512
+bytes. LUKS2 supports larger sectors, and for better performance we set
+the default sector size to the maximum of 4KiB. Writes which are either smaller
+than a sector, or are not aligned to a sector start, will trigger a guarded
+read-modify-write chain on the client, with a considerable latency penalty.
+A batch of such unaligned writes can lead to IO races which will further
+deteriorate performance. Thus it is advisable to avoid using RBD encryption
+in cases where incoming writes cannot be guaranteed to be sector-aligned.
+
+To map a LUKS-formatted image run:
+
+.. prompt:: bash #
+
+    rbd device map -t nbd -o encryption-passphrase-file={passphrase-file} {image-spec}
+
+Note that for security reasons, both the encryption format and encryption load
+operations are CPU-intensive, and may take a few seconds to complete. For the
+encryption operations of actual image IO, assuming AES-NI is enabled,
+a relative small microseconds latency should be added, as well as a small
+increase in CPU utilization.
+
+Examples
+========
+
+Create a LUKS2-formatted image with the effective size of 50GiB:
+
+.. prompt:: bash $
+
+    rbd create --size 50G mypool/myimage
+    rbd encryption format mypool/myimage luks2 passphrase.bin
+    rbd resize --size 50G --encryption-passphrase-file passphrase.bin mypool/myimage
+
+``rbd resize`` command at the end grows the image to compensate for the
+overhead associated with the LUKS2 header.
+
+Given a LUKS2-formatted image, create a LUKS2-formatted clone with the
+same effective size:
+
+.. prompt:: bash $
+
+    rbd snap create mypool/myimage@snap
+    rbd snap protect mypool/myimage@snap
+    rbd clone mypool/myimage@snap mypool/myclone
+    rbd encryption format mypool/myclone luks2 clone-passphrase.bin
+
+Given a LUKS2-formatted image with the effective size of 50GiB, create
+a LUKS1-formatted clone with the same effective size:
+
+.. prompt:: bash $
+
+    rbd snap create mypool/myimage@snap
+    rbd snap protect mypool/myimage@snap
+    rbd clone mypool/myimage@snap mypool/myclone
+    rbd encryption format mypool/myclone luks1 clone-passphrase.bin
+    rbd resize --size 50G --allow-shrink --encryption-passphrase-file clone-passphrase.bin --encryption-passphrase-file passphrase.bin mypool/myclone
+
+Since LUKS1 header is usually smaller than LUKS2 header, ``rbd resize``
+command at the end shrinks the cloned image to get rid of unneeded
+space allowance.
+
+Given a LUKS1-formatted image with the effective size of 50GiB, create
+a LUKS2-formatted clone with the same effective size:
+
+.. prompt:: bash $
+
+    rbd resize --size 51G mypool/myimage
+    rbd snap create mypool/myimage@snap
+    rbd snap protect mypool/myimage@snap
+    rbd clone mypool/myimage@snap mypool/myclone
+    rbd encryption format mypool/myclone luks2 clone-passphrase.bin
+    rbd resize --size 50G --allow-shrink --encryption-passphrase-file passphrase.bin mypool/myimage
+    rbd resize --size 50G --allow-shrink --encryption-passphrase-file clone-passphrase.bin --encryption-passphrase-file passphrase.bin mypool/myclone
+
+Since LUKS2 header is usually bigger than LUKS1 header, ``rbd resize``
+command at the beginning temporarily grows the parent image to reserve
+some extra space in the parent snapshot and consequently the cloned
+image. This is necessary to make all parent data accessible in the
+cloned image. ``rbd resize`` commands at the end shrink the parent
+image back to its original size (this does not impact the parent
+snapshot) and also the cloned image to get rid of unused reserved
+space.
+
+The same applies to creating a formatted clone of an unformatted
+(plaintext) image since an unformatted image does not have a header at
+all.
+
+.. _journal feature: ../rbd-mirroring/#enable-image-journaling-feature
+.. _Supported Formats: #supported-formats
+.. _rbd-nbd: ../../man/8/rbd-nbd
+.. _stripe count: ../../man/8/rbd/#striping
diff --git a/doc/rbd/rbd-exclusive-locks.rst b/doc/rbd/rbd-exclusive-locks.rst
new file mode 100644
index 000000000..f9b99dfb4
--- /dev/null
+++ b/doc/rbd/rbd-exclusive-locks.rst
@@ -0,0 +1,104 @@
+.. _rbd-exclusive-locks:
+
+====================
+ RBD Exclusive Locks
+====================
+
+.. index:: Ceph Block Device; RBD exclusive locks; exclusive-lock
+
+Exclusive locks are mechanisms designed to prevent multiple processes from
+accessing the same Rados Block Device (RBD) in an uncoordinated fashion.
+Exclusive locks are used heavily in virtualization (where they prevent VMs from
+clobbering each other's writes) and in `RBD mirroring`_ (where they are a
+prerequisite for journaling in journal-based mirroring and fast generation of
+incremental diffs in snapshot-based mirroring).
+
+The ``exclusive-lock`` feature is enabled on newly created images. This default
+can be overridden via the ``rbd_default_features`` configuration option or the
+``--image-feature`` and ``--image-shared`` options for ``rbd create`` command.
+
+.. note::
+   Many image features, including ``object-map`` and ``fast-diff``, depend upon
+   exclusive locking. Disabling the ``exclusive-lock`` feature will negatively
+   affect the performance of some operations.
+
+To maintain multi-client access, the ``exclusive-lock`` feature implements
+automatic cooperative lock transitions between clients. It ensures that only
+a single client can write to an RBD image at any given time and thus protects
+internal image structures such as the object map, the journal or the `PWL
+cache`_ from concurrent modification.
+
+Exclusive locking is mostly transparent to the user:
+
+* Whenever a client (a ``librbd`` process or, in case of a ``krbd`` client,
+  a client node's kernel) needs to handle a write to an RBD image on which
+  exclusive locking has been enabled, it first acquires an exclusive lock on
+  the image. If the lock is already held by some other client, that client is
+  requested to release it.
+
+* Whenever a client that holds an exclusive lock on an RBD image gets
+  a request to release the lock, it stops handling writes, flushes its caches
+  and releases the lock.
+
+* Whenever a client that holds an exclusive lock on an RBD image terminates
+  gracefully, the lock is also released gracefully.
+
+* A graceful release of an exclusive lock on an RBD image (whether by request
+  or due to client termination) enables another, subsequent, client to acquire
+  the lock and start handling writes.
+
+.. warning::
+   By default, the ``exclusive-lock`` feature does not prevent two or more
+   concurrently running clients from opening the same RBD image and writing to
+   it in turns (whether on the same node or not). In effect, their writes just
+   get linearized as the lock is automatically transitioned back and forth in
+   a cooperative fashion.
+
+.. note::
+   To disable automatic lock transitions between clients, the
+   ``RBD_LOCK_MODE_EXCLUSIVE`` flag may be specified when acquiring the
+   exclusive lock. This is exposed by the ``--exclusive`` option for ``rbd
+   device map`` command.
+
+
+Blocklisting
+============
+
+Sometimes a client that previously held an exclusive lock on an RBD image does
+not terminate gracefully, but dies abruptly. This may be because the client
+process received a ``KILL`` or ``ABRT`` signal, or because the client node
+underwent a hard reboot or suffered a power failure. In cases like this, the
+lock is never gracefully released. This means that any new client that comes up
+and attempts to write to the image must break the previously held exclusive
+lock.
+
+However, a process (or kernel thread) may hang or merely lose network
+connectivity to the Ceph cluster for some amount of time. In that case,
+breaking the lock would be potentially catastrophic: the hung process or
+connectivity issue could resolve itself and the original process might then
+compete with one that started in the interim, thus accessing RBD data in an
+uncoordinated and destructive manner.
+
+In the event that a lock cannot be acquired in the standard graceful manner,
+the overtaking process not only breaks the lock but also blocklists the
+previous lock holder. This is negotiated between the new client process and the
+Ceph Monitor. 
+
+* Upon receiving the blocklist request, the monitor instructs the relevant OSDs
+  to no longer serve requests from the old client process;
+* after the associated OSD map update is complete, the new client can break the
+  previously held lock;
+* after the new client has acquired the lock, it can commence writing
+  to the image.
+
+Blocklisting is thus a form of storage-level resource `fencing`_.
+
+.. note::
+   In order for blocklisting to work, the client must have the ``osd
+   blocklist`` capability. This capability is included in the ``profile
+   rbd`` capability profile, which should be set generally on all Ceph
+   :ref:`client identities <user-management>` using RBD.
+
+.. _RBD mirroring: ../rbd-mirroring
+.. _PWL cache: ../rbd-persistent-write-log-cache
+.. _fencing: https://en.wikipedia.org/wiki/Fencing_(computing)
diff --git a/doc/rbd/rbd-integrations.rst b/doc/rbd/rbd-integrations.rst
new file mode 100644
index 000000000..f55604a6f
--- /dev/null
+++ b/doc/rbd/rbd-integrations.rst
@@ -0,0 +1,16 @@
+=========================================
+ Ceph Block Device 3rd Party Integration
+=========================================
+
+.. toctree::
+   :maxdepth: 1
+
+   Kernel Modules <rbd-ko>
+   QEMU <qemu-rbd>
+   libvirt <libvirt>
+   Kubernetes <rbd-kubernetes>
+   Nomad <rbd-nomad>
+   OpenStack <rbd-openstack>
+   CloudStack <rbd-cloudstack>
+   LIO iSCSI Gateway <iscsi-overview>
+   Windows <rbd-windows>
diff --git a/doc/rbd/rbd-ko.rst b/doc/rbd/rbd-ko.rst
new file mode 100644
index 000000000..70c407839
--- /dev/null
+++ b/doc/rbd/rbd-ko.rst
@@ -0,0 +1,59 @@
+==========================
+ Kernel Module Operations
+==========================
+
+.. index:: Ceph Block Device; kernel module
+
+.. important:: To use kernel module operations, you must have a running Ceph cluster.
+
+Get a List of Images
+====================
+
+To mount a block device image, first return a list of the images. ::
+
+	rbd list
+
+Map a Block Device
+==================
+
+Use ``rbd`` to map an image name to a kernel module. You must specify the 
+image name, the pool name, and the user name. ``rbd`` will load RBD kernel
+module on your behalf if it's not already loaded. ::
+
+  sudo rbd device map {pool-name}/{image-name} --id {user-name}
+
+For example:: 
+
+  sudo rbd device map rbd/myimage --id admin
+ 
+If you use `cephx`_ authentication, you must also specify a secret.  It may come
+from a keyring or a file containing the secret. ::
+
+  sudo rbd device map rbd/myimage --id admin --keyring /path/to/keyring
+  sudo rbd device map rbd/myimage --id admin --keyfile /path/to/file
+
+
+Show Mapped Block Devices
+=========================
+
+To show block device images mapped to kernel modules with the ``rbd``,
+specify ``device list`` arguments. ::
+
+	rbd device list
+
+
+Unmapping a Block Device
+========================
+
+To unmap a block device image with the ``rbd`` command, specify the
+``device unmap`` arguments and the device name (i.e., by convention the
+same as the block device image name). :: 
+
+	sudo rbd device unmap /dev/rbd/{poolname}/{imagename}
+
+For example::
+
+	sudo rbd device unmap /dev/rbd/rbd/foo
+
+
+.. _cephx: ../../rados/operations/user-management/
diff --git a/doc/rbd/rbd-kubernetes.rst b/doc/rbd/rbd-kubernetes.rst
new file mode 100644
index 000000000..ccec4813a
--- /dev/null
+++ b/doc/rbd/rbd-kubernetes.rst
@@ -0,0 +1,364 @@
+==============================
+ Block Devices and Kubernetes
+==============================
+
+You may use Ceph Block Device images with Kubernetes v1.13 and later through
+`ceph-csi`_, which dynamically provisions RBD images to back Kubernetes
+`volumes`_ and maps these RBD images as block devices (optionally mounting
+a file system contained within the image) on worker nodes running
+`pods`_ that reference an RBD-backed volume. Ceph stripes block device images as
+objects across the cluster, which means that large Ceph Block Device images have
+better performance than a standalone server!
+
+To use Ceph Block Devices with Kubernetes v1.13 and higher, you must install
+and configure ``ceph-csi`` within your Kubernetes environment. The following
+diagram depicts the Kubernetes/Ceph technology stack.
+
+.. ditaa::
+            +---------------------------------------------------+
+            |                   Kubernetes                      |
+            +---------------------------------------------------+
+            |                    ceph--csi                      |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +------------------------+ +------------------------+
+            |                        | |        rbd--nbd        |
+            |     Kernel Modules     | +------------------------+
+            |                        | |         librbd         |
+            +------------------------+-+------------------------+
+            |                   RADOS Protocol                  |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+.. important::
+   ``ceph-csi`` uses the RBD kernel modules by default which may not support all
+   Ceph `CRUSH tunables`_ or `RBD image features`_.
+
+Create a Pool
+=============
+
+By default, Ceph block devices use the ``rbd`` pool. Create a pool for
+Kubernetes volume storage. Ensure your Ceph cluster is running, then create
+the pool. ::
+
+        $ ceph osd pool create kubernetes
+
+See `Create a Pool`_ for details on specifying the number of placement groups
+for your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+A newly created pool must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pool::
+
+        $ rbd pool init kubernetes
+
+Configure ceph-csi
+==================
+
+Setup Ceph Client Authentication
+--------------------------------
+
+Create a new user for Kubernetes and `ceph-csi`. Execute the following and
+record the generated key::
+
+    $ ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=kubernetes' mgr 'profile rbd pool=kubernetes'
+    [client.kubernetes]
+        key = AQD9o0Fd6hQRChAAt7fMaSZXduT3NWEqylNpmg==
+
+Generate `ceph-csi` `ConfigMap`
+-------------------------------
+
+The `ceph-csi` requires a `ConfigMap` object stored in Kubernetes to define the
+the Ceph monitor addresses for the Ceph cluster. Collect both the Ceph cluster
+unique `fsid` and the monitor addresses::
+
+        $ ceph mon dump
+        <...>
+        fsid b9127830-b0cc-4e34-aa47-9d1a2e9949a8
+        <...>
+        0: [v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0] mon.a
+        1: [v2:192.168.1.2:3300/0,v1:192.168.1.2:6789/0] mon.b
+        2: [v2:192.168.1.3:3300/0,v1:192.168.1.3:6789/0] mon.c
+
+.. note::
+   ``ceph-csi`` currently only supports the `legacy V1 protocol`_.
+
+Generate a `csi-config-map.yaml` file similar to the example below, substituting
+the `fsid` for "clusterID", and the monitor addresses for "monitors"::
+
+        $ cat <<EOF > csi-config-map.yaml
+        ---
+        apiVersion: v1
+        kind: ConfigMap
+        data:
+          config.json: |-
+            [
+              {
+                "clusterID": "b9127830-b0cc-4e34-aa47-9d1a2e9949a8",
+                "monitors": [
+                  "192.168.1.1:6789",
+                  "192.168.1.2:6789",
+                  "192.168.1.3:6789"
+                ]
+              }
+            ]
+        metadata:
+          name: ceph-csi-config
+        EOF
+
+Once generated, store the new `ConfigMap` object in Kubernetes::
+
+        $ kubectl apply -f csi-config-map.yaml
+
+Recent versions of `ceph-csi` also require an additional `ConfigMap` object to
+define Key Management Service (KMS) provider details.  If KMS isn't set up, put
+an empty configuration in a `csi-kms-config-map.yaml` file or refer to examples
+at https://github.com/ceph/ceph-csi/tree/master/examples/kms::
+
+        $ cat <<EOF > csi-kms-config-map.yaml
+        ---
+        apiVersion: v1
+        kind: ConfigMap
+        data:
+          config.json: |-
+            {}
+        metadata:
+          name: ceph-csi-encryption-kms-config
+        EOF
+
+Once generated, store the new `ConfigMap` object in Kubernetes::
+
+        $ kubectl apply -f csi-kms-config-map.yaml
+
+Recent versions of `ceph-csi` also require yet another `ConfigMap` object
+to define Ceph configuration to add to ceph.conf file inside CSI containers::
+
+        $ cat <<EOF > ceph-config-map.yaml
+        ---
+        apiVersion: v1
+        kind: ConfigMap
+        data:
+          ceph.conf: |
+            [global]
+            auth_cluster_required = cephx
+            auth_service_required = cephx
+            auth_client_required = cephx
+          # keyring is a required key and its value should be empty
+          keyring: |
+        metadata:
+          name: ceph-config
+        EOF
+
+Once generated, store the new `ConfigMap` object in Kubernetes::
+
+        $ kubectl apply -f ceph-config-map.yaml
+
+Generate `ceph-csi` cephx `Secret`
+----------------------------------
+
+`ceph-csi` requires the cephx credentials for communicating with the Ceph
+cluster. Generate a `csi-rbd-secret.yaml` file similar to the example below,
+using the newly created Kubernetes user id and cephx key::
+
+        $ cat <<EOF > csi-rbd-secret.yaml
+        ---
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: csi-rbd-secret
+          namespace: default
+        stringData:
+          userID: kubernetes
+          userKey: AQD9o0Fd6hQRChAAt7fMaSZXduT3NWEqylNpmg==
+        EOF
+
+Once generated, store the new `Secret` object in Kubernetes::
+
+        $ kubectl apply -f csi-rbd-secret.yaml
+
+Configure `ceph-csi` Plugins
+----------------------------
+
+Create the required `ServiceAccount` and RBAC `ClusterRole`/`ClusterRoleBinding`
+Kubernetes objects. These objects do not necessarily need to be customized for
+your Kubernetes environment and therefore can be used as-is from the `ceph-csi`
+deployment YAMLs::
+
+        $ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
+        $ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
+
+Finally, create the `ceph-csi` provisioner and node plugins. With the
+possible exception of the `ceph-csi` container release version, these objects do
+not necessarily need to be customized for your Kubernetes environment and
+therefore can be used as-is from the `ceph-csi` deployment YAMLs::
+
+        $ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
+        $ kubectl apply -f csi-rbdplugin-provisioner.yaml
+        $ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
+        $ kubectl apply -f csi-rbdplugin.yaml
+
+.. important::
+   The provisioner and node plugin YAMLs will, by default, pull the development
+   release of the `ceph-csi` container (quay.io/cephcsi/cephcsi:canary).
+   The YAMLs should be updated to use a release version container for
+   production workloads.
+
+Using Ceph Block Devices
+========================
+
+Create a `StorageClass`
+-----------------------
+
+The Kubernetes `StorageClass` defines a class of storage. Multiple `StorageClass`
+objects can be created to map to different quality-of-service levels (i.e. NVMe
+vs HDD-based pools) and features.
+
+For example, to create a `ceph-csi` `StorageClass` that maps to the `kubernetes`
+pool created above, the following YAML file can be used after ensuring that the
+"clusterID" property matches your Ceph cluster's `fsid`::
+
+        $ cat <<EOF > csi-rbd-sc.yaml
+        ---
+        apiVersion: storage.k8s.io/v1
+        kind: StorageClass
+        metadata:
+           name: csi-rbd-sc
+        provisioner: rbd.csi.ceph.com
+        parameters:
+           clusterID: b9127830-b0cc-4e34-aa47-9d1a2e9949a8
+           pool: kubernetes
+           imageFeatures: layering
+           csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
+           csi.storage.k8s.io/provisioner-secret-namespace: default
+           csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
+           csi.storage.k8s.io/controller-expand-secret-namespace: default
+           csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
+           csi.storage.k8s.io/node-stage-secret-namespace: default
+        reclaimPolicy: Delete
+        allowVolumeExpansion: true
+        mountOptions:
+           - discard
+        EOF
+        $ kubectl apply -f csi-rbd-sc.yaml
+
+Note that in Kubernetes v1.14 and v1.15 volume expansion feature was in alpha
+status and required enabling `ExpandCSIVolumes` feature gate.
+
+Create a `PersistentVolumeClaim`
+--------------------------------
+
+A `PersistentVolumeClaim` is a request for abstract storage resources by a user.
+The `PersistentVolumeClaim` would then be associated to a `Pod` resource to
+provision a `PersistentVolume`, which would be backed by a Ceph block image.
+An optional `volumeMode` can be included to select between a mounted file system
+(default) or raw block device-based volume.
+
+Using `ceph-csi`, specifying `Filesystem` for `volumeMode` can support both
+`ReadWriteOnce` and `ReadOnlyMany` `accessMode` claims, and specifying `Block`
+for `volumeMode` can support `ReadWriteOnce`, `ReadWriteMany`, and
+`ReadOnlyMany` `accessMode` claims.
+
+For example, to create a block-based `PersistentVolumeClaim` that utilizes
+the `ceph-csi`-based `StorageClass` created above, the following YAML can be
+used to request raw block storage from the `csi-rbd-sc` `StorageClass`::
+
+        $ cat <<EOF > raw-block-pvc.yaml
+        ---
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: raw-block-pvc
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          volumeMode: Block
+          resources:
+            requests:
+              storage: 1Gi
+          storageClassName: csi-rbd-sc
+        EOF
+        $ kubectl apply -f raw-block-pvc.yaml
+
+The following demonstrates and example of binding the above
+`PersistentVolumeClaim` to a `Pod` resource as a raw block device::
+
+        $ cat <<EOF > raw-block-pod.yaml
+        ---
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: pod-with-raw-block-volume
+        spec:
+          containers:
+            - name: fc-container
+              image: fedora:26
+              command: ["/bin/sh", "-c"]
+              args: ["tail -f /dev/null"]
+              volumeDevices:
+                - name: data
+                  devicePath: /dev/xvda
+          volumes:
+            - name: data
+              persistentVolumeClaim:
+                claimName: raw-block-pvc
+        EOF
+        $ kubectl apply -f raw-block-pod.yaml
+
+To create a file-system-based `PersistentVolumeClaim` that utilizes the
+`ceph-csi`-based `StorageClass` created above, the following YAML can be used to
+request a mounted file system (backed by an RBD image) from the `csi-rbd-sc`
+`StorageClass`::
+
+        $ cat <<EOF > pvc.yaml
+        ---
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: rbd-pvc
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          volumeMode: Filesystem
+          resources:
+            requests:
+              storage: 1Gi
+          storageClassName: csi-rbd-sc
+        EOF
+        $ kubectl apply -f pvc.yaml
+
+The following demonstrates and example of binding the above
+`PersistentVolumeClaim` to a `Pod` resource as a mounted file system::
+
+        $ cat <<EOF > pod.yaml
+        ---
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: csi-rbd-demo-pod
+        spec:
+          containers:
+            - name: web-server
+              image: nginx
+              volumeMounts:
+                - name: mypvc
+                  mountPath: /var/lib/www/html
+          volumes:
+            - name: mypvc
+              persistentVolumeClaim:
+                claimName: rbd-pvc
+                readOnly: false
+        EOF
+        $ kubectl apply -f pod.yaml
+
+.. _ceph-csi: https://github.com/ceph/ceph-csi/
+.. _volumes: https://kubernetes.io/docs/concepts/storage/volumes/
+.. _pods: https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+.. _CRUSH tunables: ../../rados/operations/crush-map/#tunables
+.. _RBD image features: ../rbd-config-ref/#image-features
+.. _legacy V1 protocol: ../../rados/configuration/msgr2/#address-formats
diff --git a/doc/rbd/rbd-live-migration.rst b/doc/rbd/rbd-live-migration.rst
new file mode 100644
index 000000000..c3e09193d
--- /dev/null
+++ b/doc/rbd/rbd-live-migration.rst
@@ -0,0 +1,367 @@
+======================
+ Image Live-Migration
+======================
+
+.. index:: Ceph Block Device; live-migration
+
+RBD images can be live-migrated between different pools within the same cluster;
+between different image formats and layouts; or from external data sources.
+When started, the source will be deep-copied to the destination image, pulling
+all snapshot history while preserving the sparse allocation of data where
+possible.
+
+By default, when live-migrating RBD images within the same Ceph cluster, the
+source image will be marked read-only and all clients will instead redirect
+IOs to the new target image. In addition, this mode can optionally preserve the
+link to the source image's parent to preserve sparseness, or it can flatten the
+image during the migration to remove the dependency on the source image's
+parent.
+
+The live-migration process can also be used in an import-only mode where the
+source image remains unmodified and the target image can be linked to an
+external data source such as a backing file, HTTP(s) file, or S3 object.
+
+The live-migration copy process can safely run in the background while the new
+target image is in use. There is currently a requirement to temporarily stop
+using the source image before preparing a migration when not using the
+import-only mode of operation. This helps to ensure that the client using the
+image is updated to point to the new target image.
+
+.. note::
+   Image live-migration requires the Ceph Nautilus release or later. Support for
+   external data sources requires the Ceph Pacific release of later. The
+   ``krbd`` kernel module does not support live-migration at this time.
+
+
+.. ditaa::
+
+           +-------------+               +-------------+
+           | {s} c999    |               | {s}         |
+           |  Live       | Target refers |  Live       |
+           |  migration  |<-------------*|  migration  |
+           |  source     |  to Source    |  target     |
+           |             |               |             |
+           | (read only) |               | (writable)  |
+           +-------------+               +-------------+
+
+               Source                        Target
+
+The live-migration process is comprised of three steps:
+
+#. **Prepare Migration:** The initial step creates the new target image and
+   links the target image to the source. When not configured in the import-only
+   mode, the source image will also be linked to the target image and marked
+   read-only.
+
+   Similar to `layered images`_, attempts to read uninitialized data extents
+   within the target image will internally redirect the read to the source
+   image, and writes to uninitialized extents within the target will internally
+   deep-copy the overlapping source image block to the target image.
+
+
+#. **Execute Migration:** This is a background operation that deep-copies all
+   initialized blocks from the source image to the target. This step can be
+   run while clients are actively using the new target image.
+
+
+#. **Finish Migration:** Once the background migration process has completed,
+   the migration can be committed or aborted. Committing the migration will
+   remove the cross-links between the source and target images, and will
+   remove the source image if not configured in the import-only mode. Aborting
+   the migration will remove the cross-links, and will remove the target image.
+
+Prepare Migration
+=================
+
+The default live-migration process for images within the same Ceph cluster is
+initiated by running the `rbd migration prepare` command, providing the source
+and target images::
+
+    $ rbd migration prepare migration_source [migration_target]
+
+The `rbd migration prepare` command accepts all the same layout optionals as the
+`rbd create` command, which allows changes to the immutable image on-disk
+layout. The `migration_target` can be skipped if the goal is only to change the
+on-disk layout, keeping the original image name.
+
+All clients using the source image must be stopped prior to preparing a
+live-migration. The prepare step will fail if it finds any running clients with
+the image open in read/write mode. Once the prepare step is complete, the
+clients can be restarted using the new target image name. Attempting to restart
+the clients using the source image name will result in failure.
+
+The `rbd status` command will show the current state of the live-migration::
+
+    $ rbd status migration_target
+    Watchers: none
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: prepared
+
+Note that the source image will be moved to the RBD trash to avoid mistaken
+usage during the migration process::
+
+    $ rbd info migration_source
+    rbd: error opening image migration_source: (2) No such file or directory
+    $ rbd trash ls --all
+    5e2cba2f62e migration_source
+
+
+Prepare Import-Only Migration
+=============================
+
+The import-only live-migration process is initiated by running the same
+`rbd migration prepare` command, but adding the `--import-only` optional
+and providing a JSON-encoded ``source-spec`` to describe how to access
+the source image data. This ``source-spec`` can either be passed
+directly via the `--source-spec` optional, or via a file or STDIN via the
+`--source-spec-path` optional::
+
+        $ rbd migration prepare --import-only --source-spec "<JSON>" migration_target
+
+The `rbd migration prepare` command accepts all the same layout optionals as the
+`rbd create` command.
+
+The `rbd status` command will show the current state of the live-migration::
+
+        $ rbd status migration_target
+        Watchers: none
+        Migration:
+	        source: {"stream":{"file_path":"/mnt/image.raw","type":"file"},"type":"raw"}
+        	destination: rbd/migration_target (ac69113dc1d7)
+	        state: prepared
+
+The general format for the ``source-spec`` JSON is as follows::
+
+        {
+            "type": "<format-type>",
+            <format unique parameters>
+            "stream": {
+                "type": "<stream-type>",
+                <stream unique parameters>
+            }
+        }
+
+The following formats are currently supported: ``native``, ``qcow``, and
+``raw``. The following streams are currently supported: ``file``, ``http``, and
+``s3``.
+
+Formats
+~~~~~~~
+
+The ``native`` format can be used to describe a native RBD image within a
+Ceph cluster as the source image. Its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "native",
+            "pool_name": "<pool-name>",
+            ["pool_id": <pool-id>,] (optional alternative to "pool_name")
+            ["pool_namespace": "<pool-namespace",] (optional)
+            "image_name": "<image-name>",
+            ["image_id": "<image-id>",] (optional if image in trash)
+            "snap_name": "<snap-name>",
+            ["snap_id": "<snap-id>",] (optional alternative to "snap_name")
+        }
+
+Note that the ``native`` format does not include the ``stream`` object since
+it utilizes native Ceph operations. For example, to import from the image
+``rbd/ns1/image1@snap1``, the ``source-spec`` could be encoded as::
+
+        {
+            "type": "native",
+            "pool_name": "rbd",
+            "pool_namespace": "ns1",
+            "image_name": "image1",
+            "snap_name": "snap1"
+        }
+
+The ``qcow`` format can be used to describe a QCOW (QEMU copy-on-write) block
+device. Both the QCOW (v1) and QCOW2 formats are currently supported with the
+exception of advanced features such as compression, encryption, backing
+files, and external data files. Support for these missing features may be added
+in a future release. The ``qcow`` format data can be linked to any supported
+stream source described below. For example, its base ``source-spec`` JSON is
+encoded as follows::
+
+        {
+            "type": "qcow",
+            "stream": {
+                <stream unique parameters>
+            }
+        }
+
+The ``raw`` format can be used to describe a thick-provisioned, raw block device
+export (i.e. `rbd export --export-format 1 <snap-spec>`). The ``raw`` format
+data can be linked to any supported stream source described below. For example,
+its base ``source-spec`` JSON is encoded as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                <stream unique parameters for HEAD, non-snapshot revision>
+            },
+            "snapshots": [
+                {
+                    "type": "raw",
+                    "name": "<snapshot-name>",
+                    "stream": {
+                        <stream unique parameters for snapshot>
+                    }
+                },
+            ] (optional oldest to newest ordering of snapshots)
+        }
+
+The inclusion of the ``snapshots`` array is optional and currently only supports
+thick-provisioned ``raw`` snapshot exports.
+
+Additional formats such as RBD export-format v2 and RBD export-diff
+snapshots will be added in a future release.
+
+Streams
+~~~~~~~
+
+The ``file`` stream can be used to import from a locally accessible POSIX file
+source. Its ``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "file",
+                "file_path": "<file-path>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+"/mnt/image.raw", its ``source-spec`` JSON is encoded as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "file",
+                "file_path": "/mnt/image.raw"
+            }
+        }
+
+The ``http`` stream can be used to import from a remote HTTP or HTTPS web
+server. Its ``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "http",
+                "url": "<url-path>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+``http://download.ceph.com/image.raw``, its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "http",
+                "url": "http://download.ceph.com/image.raw"
+            }
+        }
+
+The ``s3`` stream can be used to import from a remote S3 bucket. Its
+``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "s3",
+                "url": "<url-path>",
+                "access_key": "<access-key>",
+                "secret_key": "<secret-key>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+`http://s3.ceph.com/bucket/image.raw`, its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "s3",
+                "url": "http://s3.ceph.com/bucket/image.raw",
+                "access_key": "NX5QOQKC6BH2IDN8HC7A",
+                "secret_key": "LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn"
+            }
+        }
+
+.. note::
+  The ``access_key`` and ``secret_key`` parameters support storing the keys in
+  the MON config-key store by prefixing the key values with ``config://``
+  followed by the path in the MON config-key store to the value. Values can be
+  stored in the config-key store via ``ceph config-key set <key-path> <value>``
+  (e.g. ``ceph config-key set rbd/s3/access_key NX5QOQKC6BH2IDN8HC7A``).
+
+Execute Migration
+=================
+
+After preparing the live-migration, the image blocks from the source image
+must be copied to the target image. This is accomplished by running the
+`rbd migration execute` command::
+
+    $ rbd migration execute migration_target
+    Image migration: 100% complete...done.
+
+The `rbd status` command will also provide feedback on the progress of the
+migration block deep-copy process::
+
+    $ rbd status migration_target
+    Watchers:
+    	watcher=1.2.3.4:0/3695551461 client.123 cookie=123
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: executing (32% complete)
+
+
+Commit Migration
+================
+
+Once the live-migration has completed deep-copying all data blocks from the
+source image to the target, the migration can be committed::
+
+    $ rbd status migration_target
+    Watchers: none
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: executed
+    $ rbd migration commit migration_target
+    Commit image migration: 100% complete...done.
+
+If the `migration_source` image is a parent of one or more clones, the `--force`
+option will need to be specified after ensuring all descendent clone images are
+not in use.
+
+Committing the live-migration will remove the cross-links between the source
+and target images, and will remove the source image::
+
+    $ rbd trash list --all
+
+
+Abort Migration
+===============
+
+If you wish to revert the prepare or execute step, run the `rbd migration abort`
+command to revert the migration process::
+
+        $ rbd migration abort migration_target
+        Abort image migration: 100% complete...done.
+
+Aborting the migration will result in the target image being deleted and access
+to the original source image being restored::
+
+        $ rbd ls
+        migration_source
+
+
+.. _layered images: ../rbd-snapshot/#layering
diff --git a/doc/rbd/rbd-mirroring.rst b/doc/rbd/rbd-mirroring.rst
new file mode 100644
index 000000000..74a2a364e
--- /dev/null
+++ b/doc/rbd/rbd-mirroring.rst
@@ -0,0 +1,538 @@
+===============
+ RBD Mirroring
+===============
+
+.. index:: Ceph Block Device; mirroring
+
+RBD images can be asynchronously mirrored between two Ceph clusters. This
+capability is available in two modes:
+
+* **Journal-based**: This mode uses the RBD journaling image feature to ensure
+  point-in-time, crash-consistent replication between clusters. Every write to
+  the RBD image is first recorded to the associated journal before modifying the
+  actual image. The remote cluster will read from this associated journal and
+  replay the updates to its local copy of the image. Since each write to the
+  RBD image will result in two writes to the Ceph cluster, expect write
+  latencies to nearly double while using the RBD journaling image feature.
+
+* **Snapshot-based**: This mode uses periodically scheduled or manually
+  created RBD image mirror-snapshots to replicate crash-consistent RBD images
+  between clusters. The remote cluster will determine any data or metadata
+  updates between two mirror-snapshots and copy the deltas to its local copy of
+  the image. With the help of the RBD ``fast-diff`` image feature, updated data
+  blocks can be quickly determined without the need to scan the full RBD image.
+  Since this mode is not as fine-grained as journaling, the complete delta 
+  between two snapshots will need to be synced prior to use during a failover
+  scenario. Any partially applied set of deltas will be rolled back at moment
+  of failover.
+
+.. note:: journal-based mirroring requires the Ceph Jewel release or later;
+   snapshot-based mirroring requires the Ceph Octopus release or later.
+
+Mirroring is configured on a per-pool basis within peer clusters and can be
+configured on a specific subset of images within the pool.  You can also mirror
+all images within a given pool when using journal-based
+mirroring. Mirroring is configured using the ``rbd`` command. The
+``rbd-mirror`` daemon is responsible for pulling image updates from the remote
+peer cluster and applying them to the image within the local cluster.
+
+Depending on the desired needs for replication, RBD mirroring can be configured
+for either one- or two-way replication:
+
+* **One-way Replication**: When data is only mirrored from a primary cluster to
+  a secondary cluster, the ``rbd-mirror`` daemon runs only on the secondary
+  cluster.
+
+* **Two-way Replication**: When data is mirrored from primary images on one
+  cluster to non-primary images on another cluster (and vice-versa), the
+  ``rbd-mirror`` daemon runs on both clusters.
+
+.. important:: Each instance of the ``rbd-mirror`` daemon must be able to
+   connect to both the local and remote Ceph clusters simultaneously (i.e.
+   all monitor and OSD hosts). Additionally, the network must have sufficient
+   bandwidth between the two data centers to handle mirroring workload.
+
+Pool Configuration
+==================
+
+The following procedures demonstrate how to perform the basic administrative
+tasks to configure mirroring using the ``rbd`` command. Mirroring is
+configured on a per-pool basis.
+
+These pool configuration steps should be performed on both peer clusters. These
+procedures assume that both clusters, named "site-a" and "site-b", are accessible
+from a single host for clarity.
+
+See the `rbd`_ manpage for additional details of how to connect to different
+Ceph clusters.
+
+.. note:: The cluster name in the following examples corresponds to a Ceph
+   configuration file of the same name (e.g. /etc/ceph/site-b.conf).  See the
+   `ceph-conf`_ documentation for how to configure multiple clusters.  Note
+   that ``rbd-mirror`` does **not** require the source and destination clusters
+   to have unique internal names; both can and should call themselves ``ceph``.
+   The config `files` that ``rbd-mirror`` needs for local and remote clusters
+   can be named arbitrarily, and containerizing the daemon is one strategy
+   for maintaining them outside of ``/etc/ceph`` to avoid confusion.
+
+Enable Mirroring
+----------------
+
+To enable mirroring on a pool with ``rbd``, issue the ``mirror pool enable``
+subcommand with the pool name, the mirroring mode, and an optional friendly
+site name to describe the local cluster::
+
+        rbd mirror pool enable [--site-name {local-site-name}] {pool-name} {mode}
+
+The mirroring mode can either be ``image`` or ``pool``:
+
+* **image**: When configured in ``image`` mode, mirroring must
+  `explicitly enabled`_ on each image.
+* **pool** (default):  When configured in ``pool`` mode, all images in the pool
+  with the journaling feature enabled are mirrored.
+
+For example::
+
+        $ rbd --cluster site-a mirror pool enable --site-name site-a image-pool image
+        $ rbd --cluster site-b mirror pool enable --site-name site-b image-pool image
+
+The site name can also be specified when creating or importing a new
+`bootstrap token`_.
+
+The site name can be changed later using the same ``mirror pool enable``
+subcommand but note that the local site name and the corresponding site name
+used by the remote cluster generally must match.
+
+Disable Mirroring
+-----------------
+
+To disable mirroring on a pool with ``rbd``, specify the ``mirror pool disable``
+command and the pool name::
+
+        rbd mirror pool disable {pool-name}
+
+When mirroring is disabled on a pool in this way, mirroring will also be
+disabled on any images (within the pool) for which mirroring was enabled
+explicitly.
+
+For example::
+
+        $ rbd --cluster site-a mirror pool disable image-pool
+        $ rbd --cluster site-b mirror pool disable image-pool
+
+Bootstrap Peers
+---------------
+
+In order for the ``rbd-mirror`` daemon to discover its peer cluster, the peer
+must be registered and a user account must be created.
+This process can be automated with ``rbd`` and the
+``mirror pool peer bootstrap create`` and ``mirror pool peer bootstrap import``
+commands.
+
+To manually create a new bootstrap token with ``rbd``, issue the
+``mirror pool peer bootstrap create`` subcommand, a pool name, and an
+optional friendly site name to describe the local cluster::
+
+        rbd mirror pool peer bootstrap create [--site-name {local-site-name}] {pool-name}
+
+The output of ``mirror pool peer bootstrap create`` will be a token that should
+be provided to the ``mirror pool peer bootstrap import`` command. For example,
+on site-a::
+
+        $ rbd --cluster site-a mirror pool peer bootstrap create --site-name site-a image-pool
+        eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
+
+To manually import the bootstrap token created by another cluster with ``rbd``,
+specify the ``mirror pool peer bootstrap import`` command, the pool name, a file
+path to the created token (or '-' to read from standard input), along with an
+optional friendly site name to describe the local cluster and a mirroring
+direction (defaults to rx-tx for bidirectional mirroring, but can also be set
+to rx-only for unidirectional mirroring)::
+
+        rbd mirror pool peer bootstrap import [--site-name {local-site-name}] [--direction {rx-only or rx-tx}] {pool-name} {token-path}
+
+For example, on site-b::
+
+        $ cat <<EOF > token
+        eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
+        EOF
+        $ rbd --cluster site-b mirror pool peer bootstrap import --site-name site-b image-pool token
+
+Add Cluster Peer Manually
+-------------------------
+
+Cluster peers can be specified manually if desired or if the above bootstrap
+commands are not available with the currently installed Ceph release.
+
+The remote ``rbd-mirror`` daemon will need access to the local cluster to
+perform mirroring. A new local Ceph user should be created for the remote
+daemon to use. To `create a Ceph user`_, with ``ceph`` specify the
+``auth get-or-create`` command, user name, monitor caps, and OSD caps::
+
+        $ ceph auth get-or-create client.rbd-mirror-peer mon 'profile rbd-mirror-peer' osd 'profile rbd'
+
+The resulting keyring should be copied to the other cluster's ``rbd-mirror``
+daemon hosts if not using the Ceph monitor ``config-key`` store described below.
+
+To manually add a mirroring peer Ceph cluster with ``rbd``, specify the
+``mirror pool peer add`` command, the pool name, and a cluster specification::
+
+        rbd mirror pool peer add {pool-name} {client-name}@{cluster-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b
+        $ rbd --cluster site-b mirror pool peer add image-pool client.rbd-mirror-peer@site-a
+
+By default, the ``rbd-mirror`` daemon needs to have access to a Ceph
+configuration file located at ``/etc/ceph/{cluster-name}.conf`` that provides
+the addresses of the peer cluster's monitors, in addition to a keyring for
+``{client-name}`` located in the default or configured keyring search paths
+(e.g. ``/etc/ceph/{cluster-name}.{client-name}.keyring``).
+
+Alternatively, the peer cluster's monitor and/or client key can be securely
+stored within the local Ceph monitor ``config-key`` store. To specify the
+peer cluster connection attributes when adding a mirroring peer, use the
+``--remote-mon-host`` and ``--remote-key-file`` optionals. For example::
+
+        $ cat <<EOF > remote-key-file
+        AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==
+        EOF
+        $ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b --remote-mon-host 192.168.1.1,192.168.1.2 --remote-key-file remote-key-file
+        $ rbd --cluster site-a mirror pool info image-pool --all
+        Mode: pool
+        Peers: 
+          UUID                                 NAME   CLIENT                 MON_HOST                KEY                                      
+          587b08db-3d33-4f32-8af8-421e77abb081 site-b client.rbd-mirror-peer 192.168.1.1,192.168.1.2 AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw== 
+
+Remove Cluster Peer
+-------------------
+
+To remove a mirroring peer Ceph cluster with ``rbd``, specify the
+``mirror pool peer remove`` command, the pool name, and the peer UUID
+(available from the ``rbd mirror pool info`` command)::
+
+        rbd mirror pool peer remove {pool-name} {peer-uuid}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool peer remove image-pool 55672766-c02b-4729-8567-f13a66893445
+        $ rbd --cluster site-b mirror pool peer remove image-pool 60c0e299-b38f-4234-91f6-eed0a367be08
+
+Data Pools
+----------
+
+When creating images in the destination cluster, ``rbd-mirror`` selects a data
+pool as follows:
+
+#. If the destination cluster has a default data pool configured (with the
+   ``rbd_default_data_pool`` configuration option), it will be used.
+#. Otherwise, if the source image uses a separate data pool, and a pool with the
+   same name exists on the destination cluster, that pool will be used.
+#. If neither of the above is true, no data pool will be set.
+
+Image Configuration
+===================
+
+Unlike pool configuration, image configuration only needs to be performed
+against a single mirroring peer Ceph cluster.
+
+Mirrored RBD images are designated as either primary or non-primary. This is a
+property of the image and not the pool. Images that are designated as
+non-primary cannot be modified.
+
+Images are automatically promoted to primary when mirroring is first enabled on
+an image (either implicitly if the pool mirror mode was ``pool`` and the image
+has the journaling image feature enabled, or `explicitly enabled`_ by the
+``rbd`` command if the pool mirror mode was ``image``).
+
+Enable Image Mirroring
+----------------------
+
+If mirroring is configured in ``image`` mode for the image's pool, then it
+is necessary to explicitly enable mirroring for each image within the pool.
+To enable mirroring for a specific image with ``rbd``, specify the
+``mirror image enable`` command along with the pool, image name, and mode::
+
+        rbd mirror image enable {pool-name}/{image-name} {mode}
+
+The mirror image mode can either be ``journal`` or ``snapshot``:
+
+* **journal** (default): When configured in ``journal`` mode, mirroring will
+  utilize the RBD journaling image feature to replicate the image contents. If
+  the RBD journaling image feature is not yet enabled on the image, it will be
+  automatically enabled.
+
+* **snapshot**:  When configured in ``snapshot`` mode, mirroring will utilize
+  RBD image mirror-snapshots to replicate the image contents. Once enabled, an
+  initial mirror-snapshot will automatically be created. Additional RBD image
+  `mirror-snapshots`_ can be created by the ``rbd`` command.
+
+For example::
+
+        $ rbd --cluster site-a mirror image enable image-pool/image-1 snapshot
+        $ rbd --cluster site-a mirror image enable image-pool/image-2 journal
+
+Enable Image Journaling Feature
+-------------------------------
+
+RBD journal-based mirroring uses the RBD image journaling feature to ensure that
+the replicated image always remains crash-consistent. When using the ``image``
+mirroring mode, the journaling feature will be automatically enabled when
+mirroring is enabled on the image. When using the ``pool`` mirroring mode,
+before an image can be mirrored to a peer cluster, the RBD image journaling
+feature must be enabled. The feature can be enabled at image creation time by
+providing the ``--image-feature exclusive-lock,journaling`` option to the
+``rbd`` command.
+
+Alternatively, the journaling feature can be dynamically enabled on
+pre-existing RBD images. To enable journaling with ``rbd``, specify
+the ``feature enable`` command, the pool and image name, and the feature name::
+
+        rbd feature enable {pool-name}/{image-name} {feature-name}
+
+For example::
+
+        $ rbd --cluster site-a feature enable image-pool/image-1 journaling
+
+.. note:: The journaling feature is dependent on the exclusive-lock feature. If
+   the exclusive-lock feature is not already enabled, it should be enabled prior
+   to enabling the journaling feature.
+
+.. tip:: You can enable journaling on all new images by default by adding
+   ``rbd default features = 125`` to your Ceph configuration file.
+
+.. tip:: ``rbd-mirror`` tunables are set by default to values suitable for
+   mirroring an entire pool.  When using ``rbd-mirror`` to migrate single
+   volumes been clusters you may achieve substantial performance gains
+   by setting ``rbd_mirror_journal_max_fetch_bytes=33554432`` and
+   ``rbd_journal_max_payload_bytes=8388608`` within the ``[client]`` config
+   section of the local or centralized configuration.  Note that these
+   settings may allow ``rbd-mirror`` to present a substantial write workload
+   to the destination cluster:  monitor cluster performance closely during
+   migrations and test carefully before running multiple migrations in parallel.
+
+Create Image Mirror-Snapshots
+-----------------------------
+
+When using snapshot-based mirroring, mirror-snapshots will need to be created
+whenever it is desired to mirror the changed contents of the RBD image. To
+create a mirror-snapshot manually with ``rbd``, specify the
+``mirror image snapshot`` command along with the pool and image name::
+
+        rbd mirror image snapshot {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image snapshot image-pool/image-1
+
+By default up to ``5`` mirror-snapshots will be created per-image. The most
+recent mirror-snapshot is automatically pruned if the limit is reached.
+The limit can be overridden via the ``rbd_mirroring_max_mirroring_snapshots``
+configuration option if required. Additionally, mirror-snapshots are
+automatically deleted when the image is removed or when mirroring is disabled.
+
+Mirror-snapshots can also be automatically created on a periodic basis if
+mirror-snapshot schedules are defined. The mirror-snapshot can be scheduled
+globally, per-pool, or per-image levels. Multiple mirror-snapshot schedules can
+be defined at any level, but only the most-specific snapshot schedules that
+match an individual mirrored image will run.
+
+To create a mirror-snapshot schedule with ``rbd``, specify the
+``mirror snapshot schedule add`` command along with an optional pool or
+image name; interval; and optional start time::
+
+        rbd mirror snapshot schedule add [--pool {pool-name}] [--image {image-name}] {interval} [{start-time}]
+
+The ``interval`` can be specified in days, hours, or minutes using ``d``, ``h``,
+``m`` suffix respectively. The optional ``start-time`` can be specified using
+the ISO 8601 time format. For example::
+
+        $ rbd --cluster site-a mirror snapshot schedule add --pool image-pool 24h 14:00:00-05:00
+        $ rbd --cluster site-a mirror snapshot schedule add --pool image-pool --image image1 6h
+
+To remove a mirror-snapshot schedules with ``rbd``, specify the
+``mirror snapshot schedule remove`` command with options that match the
+corresponding ``add`` schedule command.
+
+To list all snapshot schedules for a specific level (global, pool, or image)
+with ``rbd``, specify the ``mirror snapshot schedule ls`` command along with
+an optional pool or image name. Additionally, the ``--recursive`` option can
+be specified to list all schedules at the specified level and below. For
+example::
+
+        $ rbd --cluster site-a mirror snapshot schedule ls --pool image-pool --recursive
+        POOL        NAMESPACE IMAGE  SCHEDULE                            
+        image-pool  -         -      every 1d starting at 14:00:00-05:00 
+        image-pool            image1 every 6h                            
+
+To view the status for when the next snapshots will be created for
+snapshot-based mirroring RBD images with ``rbd``, specify the
+``mirror snapshot schedule status`` command along with an optional pool or
+image name::
+
+        rbd mirror snapshot schedule status [--pool {pool-name}] [--image {image-name}]
+
+For example::
+
+        $ rbd --cluster site-a mirror snapshot schedule status
+        SCHEDULE TIME       IMAGE             
+        2020-02-26 18:00:00 image-pool/image1 
+
+Disable Image Mirroring
+-----------------------
+
+To disable mirroring for a specific image with ``rbd``, specify the
+``mirror image disable`` command along with the pool and image name::
+
+        rbd mirror image disable {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image disable image-pool/image-1
+
+Image Promotion and Demotion
+----------------------------
+
+In a failover scenario where the primary designation needs to be moved to the
+image in the peer Ceph cluster, access to the primary image should be stopped
+(e.g. power down the VM or remove the associated drive from a VM), demote the
+current primary image, promote the new primary image, and resume access to the
+image on the alternate cluster.
+
+.. note:: RBD only provides the necessary tools to facilitate an orderly
+   failover of an image. An external mechanism is required to coordinate the
+   full failover process (e.g. closing the image before demotion).
+
+To demote a specific image to non-primary with ``rbd``, specify the
+``mirror image demote`` command along with the pool and image name::
+
+        rbd mirror image demote {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image demote image-pool/image-1
+
+To demote all primary images within a pool to non-primary with ``rbd``, specify
+the ``mirror pool demote`` command along with the pool name::
+
+        rbd mirror pool demote {pool-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool demote image-pool
+
+To promote a specific image to primary with ``rbd``, specify the
+``mirror image promote`` command along with the pool and image name::
+
+        rbd mirror image promote [--force] {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-b mirror image promote image-pool/image-1
+
+To promote all non-primary images within a pool to primary with ``rbd``, specify
+the ``mirror pool promote`` command along with the pool name::
+
+        rbd mirror pool promote [--force] {pool-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool promote image-pool
+
+.. tip:: Since the primary / non-primary status is per-image, it is possible to
+   have two clusters split the IO load and stage failover / failback.
+
+.. note:: Promotion can be forced using the ``--force`` option. Forced
+   promotion is needed when the demotion cannot be propagated to the peer
+   Ceph cluster (e.g. Ceph cluster failure, communication outage). This will
+   result in a split-brain scenario between the two peers and the image will no
+   longer be in-sync until a `force resync command`_ is issued.
+
+Force Image Resync
+------------------
+
+If a split-brain event is detected by the ``rbd-mirror`` daemon, it will not
+attempt to mirror the affected image until corrected. To resume mirroring for an
+image, first `demote the image`_ determined to be out-of-date and then request a
+resync to the primary image. To request an image resync with ``rbd``, specify
+the ``mirror image resync`` command along with the pool and image name::
+
+        rbd mirror image resync {pool-name}/{image-name}
+
+For example::
+
+        $ rbd mirror image resync image-pool/image-1
+
+.. note:: The ``rbd`` command only flags the image as requiring a resync. The
+   local cluster's ``rbd-mirror`` daemon process is responsible for performing
+   the resync asynchronously.
+
+Mirror Status
+=============
+
+The peer cluster replication status is stored for every primary mirrored image.
+This status can be retrieved using the ``mirror image status`` and
+``mirror pool status`` commands.
+
+To request the mirror image status with ``rbd``, specify the
+``mirror image status`` command along with the pool and image name::
+
+        rbd mirror image status {pool-name}/{image-name}
+
+For example::
+
+        $ rbd mirror image status image-pool/image-1
+
+To request the mirror pool summary status with ``rbd``, specify the
+``mirror pool status`` command along with the pool name::
+
+        rbd mirror pool status {pool-name}
+
+For example::
+
+        $ rbd mirror pool status image-pool
+
+.. note:: Adding ``--verbose`` option to the ``mirror pool status`` command will
+   additionally output status details for every mirroring image in the pool.
+
+rbd-mirror Daemon
+=================
+
+The two ``rbd-mirror`` daemons are responsible for watching image journals on
+the remote, peer cluster and replaying the journal events against the local
+cluster. The RBD image journaling feature records all modifications to the
+image in the order they occur. This ensures that a crash-consistent mirror of
+the remote image is available locally.
+
+The ``rbd-mirror`` daemon is available within the optional ``rbd-mirror``
+distribution package.
+
+.. important:: Each ``rbd-mirror`` daemon requires the ability to connect
+   to both clusters simultaneously.
+.. warning:: Pre-Luminous releases: only run a single ``rbd-mirror`` daemon per
+   Ceph cluster.
+
+Each ``rbd-mirror`` daemon should use a unique Ceph user ID. To
+`create a Ceph user`_, with ``ceph`` specify the ``auth get-or-create``
+command, user name, monitor caps, and OSD caps::
+
+  ceph auth get-or-create client.rbd-mirror.{unique id} mon 'profile rbd-mirror' osd 'profile rbd'
+
+The ``rbd-mirror`` daemon can be managed by ``systemd`` by specifying the user
+ID as the daemon instance::
+
+  systemctl enable ceph-rbd-mirror@rbd-mirror.{unique id}
+
+The ``rbd-mirror`` can also be run in foreground by ``rbd-mirror`` command::
+
+  rbd-mirror -f --log-file={log_path}
+
+.. _rbd: ../../man/8/rbd
+.. _ceph-conf: ../../rados/configuration/ceph-conf/#running-multiple-clusters
+.. _explicitly enabled: #enable-image-mirroring
+.. _bootstrap token: #bootstrap-peers
+.. _force resync command: #force-image-resync
+.. _demote the image: #image-promotion-and-demotion
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
+.. _mirror-snapshots: #create-image-mirror-snapshots
diff --git a/doc/rbd/rbd-nomad.rst b/doc/rbd/rbd-nomad.rst
new file mode 100644
index 000000000..66d87d6ce
--- /dev/null
+++ b/doc/rbd/rbd-nomad.rst
@@ -0,0 +1,475 @@
+=========================
+ Block Devices and Nomad
+=========================
+
+Like Kubernetes, Nomad can use Ceph Block Device. This is made possible by
+`ceph-csi`_, which allows you to dynamically provision RBD images or import
+existing RBD images.
+
+Every version of Nomad is compatible with `ceph-csi`_, but the reference
+version of Nomad that was used to generate the procedures and guidance in this
+document is Nomad v1.1.2, the latest version available at the time of the
+writing of the document.
+
+To use Ceph Block Devices with Nomad, you must install
+and configure ``ceph-csi`` within your Nomad environment. The following
+diagram shows the Nomad/Ceph technology stack.
+
+.. ditaa::
+            +-------------------------+-------------------------+
+            |      Container          |          ceph--csi      |
+            |                         |            node         |
+            |          ^              |                 ^       |
+            |          |              |                 |       |
+            +----------+--------------+-------------------------+
+            |          |                                |       |
+            |          v                                |       |
+            |                       Nomad               |       |
+            |                                           |       |
+            +---------------------------------------------------+
+            |                       ceph--csi                   |
+            |                       controller                  |
+            +--------+------------------------------------------+
+                     |                                  |
+                     | configures       maps            |
+                     +---------------+ +----------------+
+                                     | | 
+                                     v v
+            +------------------------+ +------------------------+
+            |                        | |        rbd--nbd        |
+            |     Kernel Modules     | +------------------------+
+            |                        | |         librbd         |
+            +------------------------+-+------------------------+
+            |                   RADOS Protocol                  |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. note::
+    Nomad has many possible task drivers, but this example uses only a Docker container.
+
+.. important::
+   ``ceph-csi`` uses the RBD kernel modules by default, which may not support
+   all Ceph `CRUSH tunables`_ or `RBD image features`_.
+
+Create a Pool
+=============
+
+By default, Ceph block devices use the ``rbd`` pool. Ensure that your Ceph
+cluster is running, then create a pool for Nomad persistent storage:
+
+.. prompt:: bash $
+
+  ceph osd pool create nomad
+
+See `Create a Pool`_ for details on specifying the number of placement groups
+for your pools. See `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+A newly created pool must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pool:
+
+.. prompt:: bash $
+
+  rbd pool init nomad
+
+Configure ceph-csi
+==================
+
+Ceph Client Authentication Setup
+--------------------------------
+
+Create a new user for Nomad and `ceph-csi`. Execute the following command and
+record the generated key:
+
+.. code-block:: console
+
+  $ ceph auth get-or-create client.nomad mon 'profile rbd' osd 'profile rbd pool=nomad' mgr 'profile rbd pool=nomad'
+  [client.nomad]
+          key = AQAlh9Rgg2vrDxAARy25T7KHabs6iskSHpAEAQ==
+
+
+Configure Nomad
+---------------
+
+Configuring Nomad to Allow Containers to Use Privileged Mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, Nomad doesn't allow containers to use privileged mode. We must
+configure Nomad so that it allows containers to use privileged mode. Edit the
+Nomad configuration file by adding the following configuration block to
+`/etc/nomad.d/nomad.hcl`::
+
+    plugin "docker" {
+      config {
+        allow_privileged = true
+      }
+    }
+
+Loading the rbd module
+~~~~~~~~~~~~~~~~~~~~~~
+
+Nomad must have the `rbd` module loaded. Run the following command to confirm that the `rbd` module is loaded:
+
+.. code-block:: console
+
+  $ lsmod | grep rbd
+  rbd                    94208  2
+  libceph               364544  1 rbd
+
+If the `rbd` module is not loaded, load it:
+
+.. prompt:: bash $
+
+  sudo modprobe rbd
+
+Restarting Nomad
+~~~~~~~~~~~~~~~~
+
+Restart Nomad:
+
+.. prompt:: bash $
+
+  sudo systemctl restart nomad
+
+
+Create ceph-csi controller and plugin nodes
+===========================================
+
+The `ceph-csi`_ plugin requires two components:
+
+- **Controller plugin**: communicates with the provider's API.
+- **Node plugin**: executes tasks on the client.
+
+.. note::
+    We'll set the ceph-csi's version in those files. See `ceph-csi release`_
+    for information about ceph-csi's compatibility with other versions.
+
+Configure controller plugin
+---------------------------
+
+The controller plugin requires the Ceph monitor addresses of the Ceph
+cluster. Collect both (1) the Ceph cluster unique `fsid` and (2) the monitor
+addresses:
+
+.. code-block:: console
+
+  $ ceph mon dump
+  <...>
+  fsid b9127830-b0cc-4e34-aa47-9d1a2e9949a8
+  <...>
+  0: [v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0] mon.a
+  1: [v2:192.168.1.2:3300/0,v1:192.168.1.2:6789/0] mon.b
+  2: [v2:192.168.1.3:3300/0,v1:192.168.1.3:6789/0] mon.c
+
+Generate a ``ceph-csi-plugin-controller.nomad`` file similar to the example
+below. Substitute the `fsid` for "clusterID", and the monitor addresses for
+"monitors"::
+
+    job "ceph-csi-plugin-controller" {
+      datacenters = ["dc1"]
+      group "controller" {
+        network {
+          port "metrics" {}
+        }
+        task "ceph-controller" {
+          template {
+            data        = <<EOF
+    [{
+        "clusterID": "b9127830-b0cc-4e34-aa47-9d1a2e9949a8",
+        "monitors": [
+            "192.168.1.1",
+            "192.168.1.2",
+            "192.168.1.3"
+        ]
+    }]
+    EOF
+            destination = "local/config.json"
+            change_mode = "restart"
+          }
+          driver = "docker"
+          config {
+            image = "quay.io/cephcsi/cephcsi:v3.3.1"
+            volumes = [
+              "./local/config.json:/etc/ceph-csi-config/config.json"
+            ]
+            mounts = [
+              {
+                type     = "tmpfs"
+                target   = "/tmp/csi/keys"
+                readonly = false
+                tmpfs_options = {
+                  size = 1000000 # size in bytes
+                }
+              }
+            ]
+            args = [
+              "--type=rbd",
+              "--controllerserver=true",
+              "--drivername=rbd.csi.ceph.com",
+              "--endpoint=unix://csi/csi.sock",
+              "--nodeid=${node.unique.name}",
+              "--instanceid=${node.unique.name}-controller",
+              "--pidlimit=-1",
+              "--logtostderr=true",
+              "--v=5",
+              "--metricsport=$${NOMAD_PORT_metrics}"
+            ]
+          }
+          resources {
+            cpu    = 500
+            memory = 256
+          }
+          service {
+            name = "ceph-csi-controller"
+            port = "metrics"
+            tags = [ "prometheus" ]
+          }
+          csi_plugin {
+            id        = "ceph-csi"
+            type      = "controller"
+            mount_dir = "/csi"
+          }
+        }
+      }
+    }
+
+Configure plugin node
+---------------------
+
+Generate a ``ceph-csi-plugin-nodes.nomad`` file similar to the example below.
+Substitute the `fsid` for "clusterID" and the monitor addresses for
+"monitors"::
+
+    job "ceph-csi-plugin-nodes" {
+      datacenters = ["dc1"]
+      type        = "system"
+      group "nodes" {
+        network {
+          port "metrics" {}
+        }
+        task "ceph-node" {
+          driver = "docker"
+          template {
+            data        = <<EOF
+    [{
+        "clusterID": "b9127830-b0cc-4e34-aa47-9d1a2e9949a8",
+        "monitors": [
+            "192.168.1.1",
+            "192.168.1.2",
+            "192.168.1.3"
+        ]
+    }]
+    EOF
+            destination = "local/config.json"
+            change_mode = "restart"
+          }
+          config {
+            image = "quay.io/cephcsi/cephcsi:v3.3.1"
+            volumes = [
+              "./local/config.json:/etc/ceph-csi-config/config.json"
+            ]
+            mounts = [
+              {
+                type     = "tmpfs"
+                target   = "/tmp/csi/keys"
+                readonly = false
+                tmpfs_options = {
+                  size = 1000000 # size in bytes
+                }
+              }
+            ]
+            args = [
+              "--type=rbd",
+              "--drivername=rbd.csi.ceph.com",
+              "--nodeserver=true",
+              "--endpoint=unix://csi/csi.sock",
+              "--nodeid=${node.unique.name}",
+              "--instanceid=${node.unique.name}-nodes",
+              "--pidlimit=-1",
+              "--logtostderr=true",
+              "--v=5",
+              "--metricsport=$${NOMAD_PORT_metrics}"
+            ]
+            privileged = true
+          }
+          resources {
+            cpu    = 500
+            memory = 256
+          }
+          service {
+            name = "ceph-csi-nodes"
+            port = "metrics"
+            tags = [ "prometheus" ]
+          }
+          csi_plugin {
+            id        = "ceph-csi"
+            type      = "node"
+            mount_dir = "/csi"
+          }
+        }
+      }
+    }
+
+Start plugin controller and node
+--------------------------------
+
+To start the plugin controller and the Nomad node, run the following commands:
+
+.. prompt:: bash $
+
+  nomad job run ceph-csi-plugin-controller.nomad
+  nomad job run ceph-csi-plugin-nodes.nomad
+
+The `ceph-csi`_ image will be downloaded.
+
+Check the plugin status after a few minutes:
+
+.. code-block:: console
+
+  $ nomad plugin status ceph-csi
+  ID                   = ceph-csi
+  Provider             = rbd.csi.ceph.com
+  Version              = 3.3.1
+  Controllers Healthy  = 1
+  Controllers Expected = 1
+  Nodes Healthy        = 1
+  Nodes Expected       = 1
+
+  Allocations
+  ID        Node ID   Task Group  Version  Desired  Status   Created    Modified
+  23b4db0c  a61ef171  nodes       4        run      running  3h26m ago  3h25m ago
+  fee74115  a61ef171  controller  6        run      running  3h26m ago  3h25m ago
+
+Using Ceph Block Devices
+========================
+
+Create rbd image
+----------------
+
+``ceph-csi`` requires the cephx credentials for communicating with the Ceph
+cluster. Generate a ``ceph-volume.hcl`` file similar to the example below,
+using the newly created nomad user id and cephx key::
+
+    id = "ceph-mysql"
+    name = "ceph-mysql"
+    type = "csi"
+    plugin_id = "ceph-csi"
+    capacity_max = "200G"
+    capacity_min = "100G"
+
+    capability {
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+    }
+
+    secrets {
+      userID  = "admin"
+      userKey = "AQAlh9Rgg2vrDxAARy25T7KHabs6iskSHpAEAQ=="
+    }
+
+    parameters {
+      clusterID = "b9127830-b0cc-4e34-aa47-9d1a2e9949a8"
+      pool = "nomad"
+      imageFeatures = "layering"
+    }
+
+After the ``ceph-volume.hcl`` file has been generated, create the volume:
+
+.. prompt:: bash $
+
+  nomad volume create ceph-volume.hcl
+
+Use rbd image with a container
+------------------------------
+
+As an exercise in using an rbd image with a container, modify the Hashicorp
+`nomad stateful`_ example.
+
+Generate a ``mysql.nomad`` file similar to the example below::
+
+    job "mysql-server" {
+      datacenters = ["dc1"]
+      type        = "service"
+      group "mysql-server" {
+        count = 1
+        volume "ceph-mysql" {
+          type      = "csi"
+          attachment_mode = "file-system"
+          access_mode     = "single-node-writer"
+          read_only = false
+          source    = "ceph-mysql"
+        }
+        network {
+          port "db" {
+            static = 3306
+          }
+        }
+        restart {
+          attempts = 10
+          interval = "5m"
+          delay    = "25s"
+          mode     = "delay"
+        }
+        task "mysql-server" {
+          driver = "docker"
+          volume_mount {
+            volume      = "ceph-mysql"
+            destination = "/srv"
+            read_only   = false
+          }
+          env {
+            MYSQL_ROOT_PASSWORD = "password"
+          }
+          config {
+            image = "hashicorp/mysql-portworx-demo:latest"
+            args  = ["--datadir", "/srv/mysql"]
+            ports = ["db"]
+          }
+          resources {
+            cpu    = 500
+            memory = 1024
+          }
+          service {
+            name = "mysql-server"
+            port = "db"
+            check {
+              type     = "tcp"
+              interval = "10s"
+              timeout  = "2s"
+            }
+          }
+        }
+      }
+    }
+
+Start the job:
+
+.. prompt:: bash $
+
+  nomad job run mysql.nomad
+
+Check the status of the job:
+
+.. code-block:: console
+
+  $ nomad job status mysql-server
+  ...
+  Status        = running
+  ...
+  Allocations
+  ID        Node ID   Task Group    Version  Desired  Status   Created  Modified
+  38070da7  9ad01c63  mysql-server  0        run      running  6s ago   3s ago
+
+To check that data are persistent, modify the database, purge the job, then
+create it using the same file. The same RBD image will be used (re-used,
+really).
+
+.. _ceph-csi: https://github.com/ceph/ceph-csi/
+.. _csi: https://www.nomadproject.io/docs/internals/plugins/csi
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+.. _CRUSH tunables: ../../rados/operations/crush-map/#tunables
+.. _RBD image features: ../rbd-config-ref/#image-features
+.. _nomad stateful: https://learn.hashicorp.com/tutorials/nomad/stateful-workloads-csi-volumes?in=nomad/stateful-workloads#create-the-job-file
+.. _ceph-csi release: https://github.com/ceph/ceph-csi#ceph-csi-container-images-and-release-compatibility
diff --git a/doc/rbd/rbd-openstack.rst b/doc/rbd/rbd-openstack.rst
new file mode 100644
index 000000000..7d64b3548
--- /dev/null
+++ b/doc/rbd/rbd-openstack.rst
@@ -0,0 +1,395 @@
+=============================
+ Block Devices and OpenStack
+=============================
+
+.. index:: Ceph Block Device; OpenStack
+
+You can attach Ceph Block Device images to OpenStack instances through ``libvirt``,
+which configures the QEMU interface to ``librbd``. Ceph stripes block volumes
+across multiple OSDs within the cluster, which means that large volumes can
+realize better performance than local drives on a standalone server!
+
+To use Ceph Block Devices with OpenStack, you must install QEMU, ``libvirt``,
+and OpenStack first. We recommend using a separate physical node for your
+OpenStack installation. OpenStack recommends a minimum of 8GB of RAM and a
+quad-core processor. The following diagram depicts the OpenStack/Ceph
+technology stack.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                    OpenStack                      |
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. important:: To use Ceph Block Devices with OpenStack, you must have
+   access to a running Ceph Storage Cluster.
+
+Three parts of OpenStack integrate with Ceph's block devices:
+
+- **Images**: OpenStack Glance manages images for VMs. Images are immutable.
+  OpenStack treats images as binary blobs and downloads them accordingly.
+
+- **Volumes**: Volumes are block devices. OpenStack uses volumes to boot VMs,
+  or to attach volumes to running VMs. OpenStack manages volumes using
+  Cinder services.
+
+- **Guest Disks**: Guest disks are guest operating system disks. By default,
+  when you boot a virtual machine, its disk appears as a file on the file system
+  of the hypervisor (usually under ``/var/lib/nova/instances/<uuid>/``). Prior
+  to OpenStack Havana, the only way to boot a VM in Ceph was to use the
+  boot-from-volume functionality of Cinder. However, now it is possible to boot
+  every virtual machine inside Ceph directly without using Cinder, which is
+  advantageous because it allows you to perform maintenance operations easily
+  with the live-migration process. Additionally, if your hypervisor dies it is
+  also convenient to trigger ``nova evacuate`` and reinstate the virtual machine
+  elsewhere almost seamlessly. In doing so,
+  :ref:`exclusive locks <rbd-exclusive-locks>` prevent multiple
+  compute nodes from concurrently accessing the guest disk.
+
+
+You can use OpenStack Glance to store images as Ceph Block Devices, and you
+can use Cinder to boot a VM using a copy-on-write clone of an image.
+
+The instructions below detail the setup for Glance, Cinder and Nova, although
+they do not have to be used together. You may store images in Ceph block devices
+while running VMs using a local disk, or vice versa.
+
+.. important:: Using QCOW2 for hosting a virtual machine disk is NOT recommended.
+   If you want to boot virtual machines in Ceph (ephemeral backend or boot
+   from volume), please use the ``raw`` image format within Glance.
+
+.. index:: pools; OpenStack
+
+Create a Pool
+=============
+
+By default, Ceph block devices live within the ``rbd`` pool. You may use any
+suitable pool by specifying it explicitly. We recommend creating a pool for
+Cinder and a pool for Glance. Ensure your Ceph cluster is running, then create the pools. ::
+
+    ceph osd pool create volumes
+    ceph osd pool create images
+    ceph osd pool create backups
+    ceph osd pool create vms
+
+See `Create a Pool`_ for detail on specifying the number of placement groups for
+your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+Newly created pools must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pools::
+
+        rbd pool init volumes
+        rbd pool init images
+        rbd pool init backups
+        rbd pool init vms
+
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+
+
+Configure OpenStack Ceph Clients
+================================
+
+The nodes running ``glance-api``, ``cinder-volume``, ``nova-compute`` and
+``cinder-backup`` act as Ceph clients. Each requires the ``ceph.conf`` file::
+
+  ssh {your-openstack-server} sudo tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf
+
+
+Install Ceph client packages
+----------------------------
+
+On the ``glance-api`` node, you will need the Python bindings for ``librbd``::
+
+  sudo apt-get install python-rbd
+  sudo yum install python-rbd
+
+On the ``nova-compute``, ``cinder-backup`` and on the ``cinder-volume`` node,
+use both the Python bindings and the client command line tools::
+
+  sudo apt-get install ceph-common
+  sudo yum install ceph-common
+
+
+Setup Ceph Client Authentication
+--------------------------------
+
+If you have `cephx authentication`_ enabled, create a new user for Nova/Cinder
+and Glance. Execute the following::
+
+    ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images'
+    ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms'
+    ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups'
+
+Add the keyrings for ``client.cinder``, ``client.glance``, and
+``client.cinder-backup`` to the appropriate nodes and change their ownership::
+
+  ceph auth get-or-create client.glance | ssh {your-glance-api-server} sudo tee /etc/ceph/ceph.client.glance.keyring
+  ssh {your-glance-api-server} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
+  ceph auth get-or-create client.cinder | ssh {your-volume-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
+  ssh {your-cinder-volume-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
+  ceph auth get-or-create client.cinder-backup | ssh {your-cinder-backup-server} sudo tee /etc/ceph/ceph.client.cinder-backup.keyring
+  ssh {your-cinder-backup-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring
+
+Nodes running ``nova-compute`` need the keyring file for the ``nova-compute``
+process::
+
+  ceph auth get-or-create client.cinder | ssh {your-nova-compute-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
+
+They also need to store the secret key of the ``client.cinder`` user in
+``libvirt``. The libvirt process needs it to access the cluster while attaching
+a block device from Cinder.
+
+Create a temporary copy of the secret key on the nodes running
+``nova-compute``::
+
+  ceph auth get-key client.cinder | ssh {your-compute-node} tee client.cinder.key
+
+Then, on the compute nodes, add the secret key to ``libvirt`` and remove the
+temporary copy of the key::
+
+  uuidgen
+  457eb676-33da-42ec-9a8c-9293d545c337
+
+  cat > secret.xml <<EOF
+  <secret ephemeral='no' private='no'>
+    <uuid>457eb676-33da-42ec-9a8c-9293d545c337</uuid>
+    <usage type='ceph'>
+      <name>client.cinder secret</name>
+    </usage>
+  </secret>
+  EOF
+  sudo virsh secret-define --file secret.xml
+  Secret 457eb676-33da-42ec-9a8c-9293d545c337 created
+  sudo virsh secret-set-value --secret 457eb676-33da-42ec-9a8c-9293d545c337 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml
+
+Save the uuid of the secret for configuring ``nova-compute`` later.
+
+.. important:: You don't necessarily need the UUID on all the compute nodes.
+   However from a platform consistency perspective, it's better to keep the
+   same UUID.
+
+.. _cephx authentication: ../../rados/configuration/auth-config-ref/#enabling-disabling-cephx
+
+
+Configure OpenStack to use Ceph
+===============================
+
+Configuring Glance
+------------------
+
+Glance can use multiple back ends to store images. To use Ceph block devices by
+default, configure Glance like the following.
+
+
+Kilo and after
+~~~~~~~~~~~~~~
+
+Edit ``/etc/glance/glance-api.conf`` and add under the ``[glance_store]`` section::
+
+    [glance_store]
+    stores = rbd
+    default_store = rbd
+    rbd_store_pool = images
+    rbd_store_user = glance
+    rbd_store_ceph_conf = /etc/ceph/ceph.conf
+    rbd_store_chunk_size = 8
+
+For more information about the configuration options available in Glance please refer to the OpenStack Configuration Reference: http://docs.openstack.org/.
+
+Enable copy-on-write cloning of images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note that this exposes the back end location via Glance's API, so the endpoint
+with this option enabled should not be publicly accessible.
+
+Any OpenStack version except Mitaka
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want to enable copy-on-write cloning of images, also add under the ``[DEFAULT]`` section::
+
+    show_image_direct_url = True
+
+Disable cache management (any OpenStack version)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Disable the Glance cache management to avoid images getting cached under ``/var/lib/glance/image-cache/``,
+assuming your configuration file has ``flavor = keystone+cachemanagement``::
+
+    [paste_deploy]
+    flavor = keystone
+
+Image properties
+~~~~~~~~~~~~~~~~
+
+We recommend to use the following properties for your images:
+
+- ``hw_scsi_model=virtio-scsi``: add the virtio-scsi controller and get better performance and support for discard operation
+- ``hw_disk_bus=scsi``: connect every cinder block devices to that controller
+- ``hw_qemu_guest_agent=yes``: enable the QEMU guest agent
+- ``os_require_quiesce=yes``: send fs-freeze/thaw calls through the QEMU guest agent
+
+
+Configuring Cinder
+------------------
+
+OpenStack requires a driver to interact with Ceph block devices. You must also
+specify the pool name for the block device. On your OpenStack node, edit
+``/etc/cinder/cinder.conf`` by adding::
+
+    [DEFAULT]
+    ...
+    enabled_backends = ceph
+    glance_api_version = 2
+    ...
+    [ceph]
+    volume_driver = cinder.volume.drivers.rbd.RBDDriver
+    volume_backend_name = ceph
+    rbd_pool = volumes
+    rbd_ceph_conf = /etc/ceph/ceph.conf
+    rbd_flatten_volume_from_snapshot = false
+    rbd_max_clone_depth = 5
+    rbd_store_chunk_size = 4
+    rados_connect_timeout = -1
+
+If you are using `cephx authentication`_, also configure the user and uuid of
+the secret you added to ``libvirt`` as documented earlier::
+
+    [ceph]
+    ...
+    rbd_user = cinder
+    rbd_secret_uuid = 457eb676-33da-42ec-9a8c-9293d545c337
+
+Note that if you are configuring multiple cinder back ends,
+``glance_api_version = 2`` must be in the ``[DEFAULT]`` section.
+
+
+Configuring Cinder Backup
+-------------------------
+
+OpenStack Cinder Backup requires a specific daemon so don't forget to install it.
+On your Cinder Backup node, edit ``/etc/cinder/cinder.conf`` and add::
+
+    backup_driver = cinder.backup.drivers.ceph
+    backup_ceph_conf = /etc/ceph/ceph.conf
+    backup_ceph_user = cinder-backup
+    backup_ceph_chunk_size = 134217728
+    backup_ceph_pool = backups
+    backup_ceph_stripe_unit = 0
+    backup_ceph_stripe_count = 0
+    restore_discard_excess_bytes = true
+
+
+Configuring Nova to attach Ceph RBD block device
+------------------------------------------------
+
+In order to attach Cinder devices (either normal block or by issuing a boot
+from volume), you must tell Nova (and libvirt) which user and UUID to refer to
+when attaching the device. libvirt will refer to this user when connecting and
+authenticating with the Ceph cluster. ::
+
+    [libvirt]
+    ...
+    rbd_user = cinder
+    rbd_secret_uuid = 457eb676-33da-42ec-9a8c-9293d545c337
+
+These two flags are also used by the Nova ephemeral back end.
+
+
+Configuring Nova
+----------------
+
+In order to boot virtual machines directly from Ceph volumes, you must
+configure the ephemeral backend for Nova.
+
+It is recommended to enable the RBD cache in your Ceph configuration file; this
+has been enabled by default since the Giant release. Moreover, enabling the
+client admin socket allows the collection of metrics and can be invaluable
+for troubleshooting.
+
+This socket can be accessed on the hypervisor (Nova compute) node::
+
+    ceph daemon /var/run/ceph/ceph-client.cinder.19195.32310016.asok help
+
+To enable RBD cache and admin sockets, ensure that on each hypervisor's
+``ceph.conf`` contains::
+
+    [client]
+        rbd cache = true
+        rbd cache writethrough until flush = true
+        admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
+        log file = /var/log/qemu/qemu-guest-$pid.log
+        rbd concurrent management ops = 20
+
+Configure permissions for these directories::
+
+    mkdir -p /var/run/ceph/guests/ /var/log/qemu/
+    chown qemu:libvirtd /var/run/ceph/guests /var/log/qemu/
+
+Note that user ``qemu`` and group ``libvirtd`` can vary depending on your system.
+The provided example works for RedHat based systems.
+
+.. tip:: If your virtual machine is already running you can simply restart it to enable the admin socket
+
+
+Restart OpenStack
+=================
+
+To activate the Ceph block device driver and load the block device pool name
+into the configuration, you must restart the related OpenStack services.
+For Debian based systems execute these commands on the appropriate nodes::
+
+    sudo glance-control api restart
+    sudo service nova-compute restart
+    sudo service cinder-volume restart
+    sudo service cinder-backup restart
+
+For Red Hat based systems execute::
+
+    sudo service openstack-glance-api restart
+    sudo service openstack-nova-compute restart
+    sudo service openstack-cinder-volume restart
+    sudo service openstack-cinder-backup restart
+
+Once OpenStack is up and running, you should be able to create a volume
+and boot from it.
+
+
+Booting from a Block Device
+===========================
+
+You can create a volume from an image using the Cinder command line tool::
+
+    cinder create --image-id {id of image} --display-name {name of volume} {size of volume}
+
+You can use `qemu-img`_ to convert from one format to another. For example::
+
+    qemu-img convert -f {source-format} -O {output-format} {source-filename} {output-filename}
+    qemu-img convert -f qcow2 -O raw precise-cloudimg.img precise-cloudimg.raw
+
+When Glance and Cinder are both using Ceph block devices, the image is a
+copy-on-write clone, so new volumes are created quickly. In the OpenStack
+dashboard, you can boot from that volume by performing the following steps:
+
+#. Launch a new instance.
+#. Choose the image associated to the copy-on-write clone.
+#. Select 'boot from volume'.
+#. Select the volume you created.
+
+.. _qemu-img: ../qemu-rbd/#running-qemu-with-rbd
diff --git a/doc/rbd/rbd-operations.rst b/doc/rbd/rbd-operations.rst
new file mode 100644
index 000000000..df702114b
--- /dev/null
+++ b/doc/rbd/rbd-operations.rst
@@ -0,0 +1,16 @@
+==============================
+ Ceph Block Device Operations
+==============================
+
+.. toctree::
+   :maxdepth: 1
+
+   Snapshots<rbd-snapshot>
+   Exclusive Locking <rbd-exclusive-locks>
+   Mirroring <rbd-mirroring>
+   Live-Migration <rbd-live-migration>
+   Persistent Read-only Cache <rbd-persistent-read-only-cache>
+   Persistent Write Log Cache <rbd-persistent-write-log-cache>
+   Encryption <rbd-encryption>
+   Config Settings (librbd) <rbd-config-ref/>
+   RBD Replay <rbd-replay>
diff --git a/doc/rbd/rbd-persistent-read-only-cache.rst b/doc/rbd/rbd-persistent-read-only-cache.rst
new file mode 100644
index 000000000..5bef7f592
--- /dev/null
+++ b/doc/rbd/rbd-persistent-read-only-cache.rst
@@ -0,0 +1,201 @@
+===============================
+ RBD Persistent Read-only Cache
+===============================
+
+.. index:: Ceph Block Device; Persistent Read-only Cache
+
+Shared, Read-only Parent Image Cache
+====================================
+
+`Cloned RBD images`_ usually modify only a small fraction of the parent
+image. For example, in a VDI use-case, VMs are cloned from the same
+base image and initially differ only by hostname and IP address. During
+booting, all of these VMs read portions of the same parent
+image data. If we have a local cache of the parent
+image, this speeds up reads on the caching host.  We also achieve
+reduction of client-to-cluster network traffic.
+RBD cache must be explicitly enabled in
+``ceph.conf``. The ``ceph-immutable-object-cache`` daemon is responsible for
+caching the parent content on the local disk, and future reads on that data
+will be serviced from the local cache.
+
+.. note:: RBD shared read-only parent image cache requires the Ceph Nautilus release or later.
+
+.. ditaa::
+
+            +--------------------------------------------------------+
+            |                         QEMU                           |
+            +--------------------------------------------------------+
+            |                librbd (cloned images)                  |
+            +-------------------+-+----------------------------------+
+            |      librados     | |  ceph--immutable--object--cache  |
+            +-------------------+ +----------------------------------+
+            |      OSDs/Mons    | |     local cached parent image    |
+            +-------------------+ +----------------------------------+
+
+
+Enable RBD Shared Read-only Parent Image Cache
+----------------------------------------------
+
+To enable RBD shared read-only parent image cache, the following Ceph settings
+need to added in the ``[client]`` `section`_ of your ``ceph.conf`` file::
+
+        rbd parent cache enabled = true
+        rbd plugins = parent_cache
+
+Immutable Object Cache Daemon
+=============================
+
+Introduction and Generic Settings
+---------------------------------
+
+The ``ceph-immutable-object-cache`` daemon is responsible for caching parent
+image content within its local caching directory. Using SSDs as the underlying
+storage is recommended because doing so provides better performance. 
+
+The key components of the daemon are:
+
+#. **Domain socket based IPC:** The daemon listens on a local domain socket at 
+   startup and waits for connections from librbd clients.
+
+#. **LRU based promotion/demotion policy:** The daemon maintains in-memory
+   statistics of cache hits for each cache file. It demotes the cold cache
+   if capacity reaches the configured threshold.
+
+#. **File-based caching store:** The daemon maintains a simple file-based cache
+   store. On promotion, the RADOS objects are fetched from RADOS cluster and
+   stored in the local caching directory.
+
+When each cloned RBD image is opened, ``librbd`` tries to connect to the cache
+daemon through its Unix domain socket. After ``librbd`` is successfully
+connected, it coordinates with the daemon upon every subsequent read. In the
+case of an uncached read, the daemon promotes the RADOS object to the local
+caching directory and the next read of the object is serviced from the cache.
+The daemon maintains simple LRU statistics, which are used to evict cold cache
+files when required (for example, when the cache is at capacity and under
+pressure). 
+
+Here are some important cache configuration settings:
+
+``immutable_object_cache_sock``
+
+:Description: The path to the domain socket used for communication between
+              librbd clients and the ceph-immutable-object-cache daemon.
+:Type: String
+:Required: No
+:Default: ``/var/run/ceph/immutable_object_cache_sock``
+
+
+``immutable_object_cache_path``
+
+:Description: The immutable object cache data directory.
+:Type: String
+:Required: No
+:Default: ``/tmp/ceph_immutable_object_cache``
+
+
+``immutable_object_cache_max_size``
+
+:Description: The max size for immutable cache.
+:Type: Size
+:Required: No
+:Default: ``1G``
+
+
+``immutable_object_cache_watermark``
+
+:Description: The high-water mark for the cache. The value is between (0, 1).
+              If the cache size reaches this threshold the daemon will start
+              to delete cold cache based on LRU statistics.
+:Type: Float
+:Required: No
+:Default: ``0.9``
+
+The ``ceph-immutable-object-cache`` daemon is available within the optional
+``ceph-immutable-object-cache`` distribution package.
+
+.. important:: ``ceph-immutable-object-cache`` daemon requires the ability to
+   connect RADOS clusters.
+
+Running the Immutable Object Cache Daemon
+-----------------------------------------
+
+``ceph-immutable-object-cache`` daemon should use a unique Ceph user ID.
+To `create a Ceph user`_, with ``ceph`` specify the ``auth get-or-create``
+command, user name, monitor caps, and OSD caps::
+
+  ceph auth get-or-create client.ceph-immutable-object-cache.{unique id} mon 'allow r' osd 'profile rbd-read-only'
+
+The ``ceph-immutable-object-cache`` daemon can be managed by ``systemd`` by specifying the user
+ID as the daemon instance::
+
+  systemctl enable ceph-immutable-object-cache@ceph-immutable-object-cache.{unique id}
+
+The ``ceph-immutable-object-cache`` can also be run in foreground by ``ceph-immutable-object-cache`` command::
+
+  ceph-immutable-object-cache -f --log-file={log_path}
+
+QOS Settings
+------------
+
+The immutable object cache supports throttling, controlled by the following settings:
+
+``immutable_object_cache_qos_schedule_tick_min``
+
+:Description: Minimum schedule tick for immutable object cache.
+:Type: Milliseconds
+:Required: No
+:Default: ``50``
+
+
+``immutable_object_cache_qos_iops_limit``
+
+:Description: The desired immutable object cache IO operations limit per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_iops_burst``
+
+:Description: The desired burst limit of immutable object cache IO operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_iops_burst_seconds``
+
+:Description: The desired burst duration in seconds of immutable object cache IO operations.
+:Type: Seconds
+:Required: No
+:Default: ``1``
+
+
+``immutable_object_cache_qos_bps_limit``
+
+:Description: The desired immutable object cache IO bytes limit per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_bps_burst``
+
+:Description: The desired burst limit of immutable object cache IO bytes.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_bps_burst_seconds``
+
+:Description: The desired burst duration in seconds of immutable object cache IO bytes.
+:Type: Seconds
+:Required: No
+:Default: ``1``
+
+.. _Cloned RBD Images: ../rbd-snapshot/#layering
+.. _section: ../../rados/configuration/ceph-conf/#configuration-sections
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
+
diff --git a/doc/rbd/rbd-persistent-write-log-cache.rst b/doc/rbd/rbd-persistent-write-log-cache.rst
new file mode 100644
index 000000000..af323962d
--- /dev/null
+++ b/doc/rbd/rbd-persistent-write-log-cache.rst
@@ -0,0 +1,139 @@
+================================
+ RBD Persistent Write Log Cache
+================================
+
+.. index:: Ceph Block Device; Persistent Write Log Cache
+
+Persistent Write Log Cache
+===========================
+
+The Persistent Write Log Cache (PWL) provides a persistent, fault-tolerant
+write-back cache for librbd-based RBD clients.
+
+This cache uses a log-ordered write-back design which maintains checkpoints
+internally so that writes that get flushed back to the cluster are always
+crash consistent. Even if the client cache is lost entirely, the disk image is
+still consistent but the data will appear to be stale.
+
+This cache can be used with PMEM or SSD as a cache device. For PMEM, the cache
+mode is called ``replica write log (rwl)``. At present, only local cache is
+supported, and the replica function is under development. For SSD, the cache
+mode is called ``ssd``.
+
+Usage
+=====
+
+The PWL cache manages the cache data in a persistent device. It looks for and
+creates cache files in a configured directory, and then caches data in the
+file.
+
+The PWL cache depends on the exclusive-lock feature. The cache can be loaded
+only after the exclusive lock is acquired.
+
+The cache provides two different persistence modes. In persistent-on-write mode,
+the writes are completed only when they are persisted to the cache device and
+will be readable after a crash. In persistent-on-flush mode, the writes are
+completed as soon as it no longer needs the caller's data buffer to complete
+the writes, but does not guarantee that writes will be readable after a crash.
+The data is persisted to the cache device when a flush request is received.
+
+Initially it defaults to the persistent-on-write mode and it switches to
+persistent-on-flush mode after the first flush request is received.
+
+Enable Cache
+========================================
+
+To enable the PWL cache, set the following configuration settings::
+
+        rbd_persistent_cache_mode = {cache-mode}
+        rbd_plugins = pwl_cache
+
+Value of {cache-mode} can be ``rwl``, ``ssd`` or ``disabled``. By default the
+cache is disabled.
+
+The ``rwl`` cache mode depends on libpmem library (part of PMDK). It should
+be universally available on x86_64 architecture and may also be available on
+ppc64le and aarch64 architectures on some distributions. It is not available
+on s390x architecture.
+
+Here are some cache configuration settings:
+
+- ``rbd_persistent_cache_path`` A file folder to cache data. This folder must
+  have DAX enabled (see `DAX`_) when using ``rwl`` mode to avoid performance
+  degradation.
+
+- ``rbd_persistent_cache_size`` The cache size per image. The minimum cache
+  size is 1 GB.
+
+The above configurations can be set per-host, per-pool, per-image etc. Eg, to
+set per-host, add the overrides to the appropriate `section`_ in the host's
+``ceph.conf`` file. To set per-pool, per-image, etc, please refer to the
+``rbd config`` `commands`_.
+
+Cache Status
+------------
+
+The PWL cache is enabled when the exclusive lock is acquired,
+and it is closed when the exclusive lock is released. To check the cache status,
+users may use the command ``rbd status``.  ::
+
+        rbd status {pool-name}/{image-name}
+
+The status of the cache is shown, including present, clean, cache size and the
+location as well as some basic metrics.
+
+For example::
+
+        $ rbd status rbd/foo
+        Watchers:
+                watcher=10.10.0.102:0/1061883624 client.25496 cookie=140338056493088
+        Persistent cache state:
+                host: sceph9
+                path: /mnt/nvme0/rbd-pwl.rbd.101e5824ad9a.pool
+                size: 1 GiB
+                mode: ssd
+                stats_timestamp: Sun Apr 10 13:26:32 2022
+                present: true   empty: false    clean: false
+                allocated: 509 MiB
+                cached: 501 MiB
+                dirty: 338 MiB
+                free: 515 MiB
+                hits_full: 1450 / 61%
+                hits_partial: 0 / 0%
+                misses: 924
+                hit_bytes: 192 MiB / 66%
+                miss_bytes: 97 MiB
+
+Flush Cache
+-----------
+
+To flush a cache file with ``rbd``, specify the ``persistent-cache flush``
+command, the pool name and the image name.  ::
+
+        rbd persistent-cache flush {pool-name}/{image-name}
+
+If the application dies unexpectedly, this command can also be used to flush
+the cache back to OSDs.
+
+For example::
+
+        $ rbd persistent-cache flush rbd/foo
+
+Invalidate Cache
+----------------
+
+To invalidate (discard) a cache file with ``rbd``, specify the
+``persistent-cache invalidate`` command, the pool name and the image name.  ::
+
+        rbd persistent-cache invalidate {pool-name}/{image-name}
+
+The command removes the cache metadata of the corresponding image, disables
+the cache feature and deletes the local cache file if it exists.
+
+For example::
+
+        $ rbd persistent-cache invalidate rbd/foo
+
+.. _section: ../../rados/configuration/ceph-conf/#configuration-sections
+.. _commands: ../../man/8/rbd#commands
+.. _DAX: https://www.kernel.org/doc/Documentation/filesystems/dax.txt
diff --git a/doc/rbd/rbd-replay.rst b/doc/rbd/rbd-replay.rst
new file mode 100644
index 000000000..b1fc4973f
--- /dev/null
+++ b/doc/rbd/rbd-replay.rst
@@ -0,0 +1,42 @@
+===================
+ RBD Replay
+===================
+
+.. index:: Ceph Block Device; RBD Replay
+
+RBD Replay is a set of tools for capturing and replaying RADOS Block Device
+(RBD) workloads. To capture an RBD workload, ``lttng-tools`` must be installed
+on the client, and ``librbd`` on the client must be the v0.87 (Giant) release 
+or later. To replay an RBD workload, ``librbd`` on the client must be the Giant
+release or later.
+
+Capture and replay takes three steps:
+
+#. Capture the trace.  Make sure to capture ``pthread_id`` context::
+
+    mkdir -p traces
+    lttng create -o traces librbd
+    lttng enable-event -u 'librbd:*'
+    lttng add-context -u -t pthread_id
+    lttng start
+    # run RBD workload here
+    lttng stop
+
+#. Process the trace with `rbd-replay-prep`_::
+
+    rbd-replay-prep traces/ust/uid/*/* replay.bin
+
+#. Replay the trace with `rbd-replay`_. Use read-only until you know 
+   it's doing what you want::
+
+    rbd-replay --read-only replay.bin
+
+.. important:: ``rbd-replay`` will destroy data by default.  Do not use against 
+   an image you wish to keep, unless you use the ``--read-only`` option.
+
+The replayed workload does not have to be against the same RBD image or even the
+same cluster as the captured workload. To account for differences, you may need
+to use the ``--pool`` and ``--map-image`` options of ``rbd-replay``.
+
+.. _rbd-replay: ../../man/8/rbd-replay
+.. _rbd-replay-prep: ../../man/8/rbd-replay-prep
diff --git a/doc/rbd/rbd-snapshot.rst b/doc/rbd/rbd-snapshot.rst
new file mode 100644
index 000000000..120dd8ec1
--- /dev/null
+++ b/doc/rbd/rbd-snapshot.rst
@@ -0,0 +1,368 @@
+===========
+ Snapshots
+===========
+
+.. index:: Ceph Block Device; snapshots
+
+A snapshot is a read-only logical copy of an image at a particular point in
+time: a checkpoint. One of the advanced features of Ceph block devices is that
+you can create snapshots of images to retain point-in-time state history.  Ceph
+also supports snapshot layering, which allows you to clone images (for example,
+VM images) quickly and easily. Ceph block device snapshots are managed using
+the ``rbd`` command and several higher-level interfaces, including `QEMU`_,
+`libvirt`_, `OpenStack`_, and `CloudStack`_.
+
+.. important:: To use RBD snapshots, you must have a running Ceph cluster.
+
+
+.. note:: Because RBD is unaware of any file system within an image (volume),
+   snapshots are merely `crash-consistent` unless they are coordinated within
+   the mounting (attaching) operating system. We therefore recommend that you
+   pause or stop I/O before taking a snapshot.  
+   
+   If the volume contains a file system, the file system should be in an
+   internally consistent state before a snapshot is taken. Snapshots taken
+   without write quiescing could need an `fsck` pass before they are mounted
+   again. To quiesce I/O you can use `fsfreeze` command. See the `fsfreeze(8)`
+   man page for more details. 
+   
+   For virtual machines, `qemu-guest-agent` can be used to automatically freeze
+   file systems when creating a snapshot.
+
+.. ditaa::
+
+           +------------+         +-------------+
+           | {s}        |         | {s} c999    |
+           |   Active   |<-------*|   Snapshot  |
+           |   Image    |         |   of Image  |
+           | (stop i/o) |         | (read only) |
+           +------------+         +-------------+
+
+
+Cephx Notes
+===========
+
+When `cephx`_ authentication is enabled (it is by default), you must specify a
+user name or ID and a path to the keyring containing the corresponding key. See
+:ref:`User Management <user-management>` for details. 
+
+.. prompt:: bash $
+
+	rbd --id {user-ID} --keyring /path/to/secret [commands]
+	rbd --name {username} --keyring /path/to/secret [commands]
+
+For example:
+
+.. prompt:: bash $
+
+	rbd --id admin --keyring /etc/ceph/ceph.keyring [commands]
+	rbd --name client.admin --keyring /etc/ceph/ceph.keyring [commands]
+
+.. tip:: Add the user and secret to the ``CEPH_ARGS`` environment variable to
+   avoid re-entry of these parameters.
+
+
+Snapshot Basics
+===============
+
+The following procedures demonstrate how to create, list, and remove
+snapshots using the ``rbd`` command.
+
+Create Snapshot
+---------------
+
+To create a snapshot, use the ``rbd snap create`` command and specify the pool
+name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap create {pool-name}/{image-name}@{snap-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap create rbd/foo@snapname
+	
+
+List Snapshots
+--------------
+
+To list the snapshots of an image, use the ``rbd snap ls`` command and specify
+the pool name and the image name:
+
+.. prompt:: bash $
+
+   rbd snap ls {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap ls rbd/foo
+
+
+Roll back Snapshot
+------------------
+
+To roll back to a snapshot,  use the ``rbd snap rollback`` command and specify
+the pool name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap rollback {pool-name}/{image-name}@{snap-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap rollback rbd/foo@snapname
+
+
+.. note:: Rolling back an image to a snapshot means overwriting the current
+   version of the image with data from a snapshot. The time it takes to execute
+   a rollback increases with the size of the image. It is **faster to clone**
+   from a snapshot **than to roll back** an image to a snapshot. Cloning from a
+   snapshot is the preferred method of returning to a pre-existing state.
+
+
+Delete a Snapshot
+-----------------
+
+To delete a snapshot, use the ``rbd snap rm`` command and specify the pool
+name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap rm {pool-name}/{image-name}@{snap-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd snap rm rbd/foo@snapname
+
+
+.. note:: Ceph OSDs delete data asynchronously, so deleting a snapshot  does
+   not immediately free up the capacity of the underlying OSDs. This process is
+   known as "snaptrim", and is referred to as such in ``ceph status`` output.
+
+Purge Snapshots
+---------------
+
+To delete all snapshots, use the ``rbd snap purge`` command and specify the
+pool name and the image name:
+
+.. prompt:: bash $
+
+   rbd snap purge {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap purge rbd/foo
+
+
+.. index:: Ceph Block Device; snapshot layering
+
+Layering
+========
+
+Ceph supports the ability to create many copy-on-write (COW) clones of a block
+device snapshot. Snapshot layering enables Ceph block device clients to create
+images very quickly. For example, you might create a block device image with a
+Linux VM written to it, snapshot the image, protect the snapshot, and create as
+many copy-on-write clones as you like. A snapshot is read-only, so cloning a
+snapshot simplifies semantics, making it possible to create clones rapidly.
+
+
+.. ditaa::
+
+           +-------------+              +-------------+
+           | {s} c999    |              | {s}         |
+           |  Snapshot   | Child refers |  COW Clone  |
+           |  of Image   |<------------*| of Snapshot |
+           |             |  to Parent   |             |
+           | (read only) |              | (writable)  |
+           +-------------+              +-------------+
+           
+               Parent                        Child
+
+.. note:: The terms "parent" and "child" refer to a Ceph block device snapshot
+   (parent) and the corresponding image cloned from the snapshot (child).
+   These terms are important for the command line usage below.
+   
+Each cloned image (child) stores a reference to its parent image, which enables
+the cloned image to open the parent snapshot and read it.
+
+A copy-on-write clone of a snapshot behaves exactly like any other Ceph
+block device image. You can read to, write from, clone, and resize cloned
+images. There are no special restrictions with cloned images. However, the
+copy-on-write clone of a snapshot depends on the snapshot, so you must 
+protect the snapshot before you clone it. The diagram below depicts this
+process.
+
+.. note:: Ceph supports the cloning of only "RBD format 2" images (that is,
+   images created without specifying ``--image-format 1``). The Linux kernel
+   client supports cloned images beginning with the 3.10 release.
+
+Getting Started with Layering
+-----------------------------
+
+Ceph block device layering is a simple process. You must have an image. You
+must create a snapshot of the image. You must protect the snapshot. After you
+have performed these steps, you can begin cloning the snapshot.
+
+.. ditaa::
+
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           | Create Block Device Image  |------->|      Create a Snapshot      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |   Protect the Snapshot     |------->|     Clone the Snapshot      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+
+
+The cloned image has a reference to the parent snapshot, and includes the pool
+ID, the image ID, and the snapshot ID. The inclusion of the pool ID means that
+you may clone snapshots from one pool to images in another pool.
+
+#. **Image Template:** A common use case for block device layering is to create
+   a base image and a snapshot that serves as a template for clones. For
+   example: a user may create an image for a Linux distribution (for example,
+   Ubuntu 22.04) and create a snapshot of it. The user may occasionally update
+   the image and create a new snapshot (by using such commands as ``sudo
+   apt-get update``, ``sudo apt-get upgrade``, or ``sudo apt-get dist-upgrade``
+   followed by ``rbd snap create``). As the image matures, the user can clone
+   any one of the snapshots.
+
+#. **Extended Template:** A more advanced use case includes extending a
+   template image to provide more information than a base image. For
+   example, a user may clone an image (for example, a VM template) and install
+   other software (for example, a database, a content management system, an
+   analytics system) and then snapshot the extended image, which may itself be
+   updated just like the base image.
+
+#. **Template Pool:** One way to use block device layering is to create a pool
+   that contains (1) base images that act as templates and (2) snapshots of
+   those templates. You may then extend read-only privileges to users so that
+   they may clone the snapshots even though they do not have permissions that
+   allow them to write or execute within the pool.
+
+#. **Image Migration/Recovery:** One way to use block device layering is to
+   migrate or recover data from one pool into another pool.
+
+Protecting a Snapshot
+---------------------
+
+Clones access the parent snapshots. All clones would break if a user
+inadvertently deleted the parent snapshot. To prevent data loss, you must 
+protect the snapshot before you can clone it:
+
+.. prompt:: bash $
+
+   rbd snap protect {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap protect rbd/foo@snapname
+
+.. note:: You cannot delete a protected snapshot.
+
+Cloning a Snapshot
+------------------
+
+To clone a snapshot, specify the parent pool, the parent image, and the parent
+snapshot; and also the child pool together with the image name. You must
+protect the snapshot before you can clone it:
+
+.. prompt:: bash $
+
+   rbd clone {pool-name}/{parent-image-name}@{snap-name} {pool-name}/{child-image-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd clone rbd/foo@snapname rbd/bar
+	
+
+.. note:: You may clone a snapshot from one pool to an image in another pool.
+   For example, you may maintain read-only images and snapshots as templates in
+   one pool, and writeable clones in another pool.
+
+Unprotecting a Snapshot
+-----------------------
+
+Before you can delete a snapshot, you must first unprotect it. Additionally,
+you may *NOT* delete snapshots that have references from clones. You must
+flatten or delete each clone of a snapshot before you can unprotect the
+snapshot:
+
+.. prompt:: bash $
+
+   rbd snap unprotect {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap unprotect rbd/foo@snapname
+
+
+Listing Children of a Snapshot
+------------------------------
+
+To list the children of a snapshot, use the ``rbd children`` command and
+specify the pool name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd children {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd children rbd/foo@snapname
+
+
+Flattening a Cloned Image
+-------------------------
+
+Cloned images retain a reference to the parent snapshot. When you remove the
+reference to the parent snapshot from the clone, you effectively "flatten" the
+clone by copying the data stored in the snapshot to the clone.  The time it
+takes to flatten a clone increases with the size of the snapshot.  To delete a
+snapshot, you must first flatten the child images (or delete them):
+
+.. prompt:: bash $
+
+   rbd flatten {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd flatten rbd/bar
+
+.. note:: Since a flattened image contains all the data stored in the snapshot,
+   a flattened image takes up more storage space than a layered clone does.
+
+
+.. _cephx: ../../rados/configuration/auth-config-ref/
+.. _QEMU: ../qemu-rbd/
+.. _OpenStack: ../rbd-openstack/
+.. _CloudStack: ../rbd-cloudstack/
+.. _libvirt: ../libvirt/
diff --git a/doc/rbd/rbd-windows.rst b/doc/rbd/rbd-windows.rst
new file mode 100644
index 000000000..df4bd172e
--- /dev/null
+++ b/doc/rbd/rbd-windows.rst
@@ -0,0 +1,235 @@
+==============
+RBD on Windows
+==============
+
+The ``rbd`` command can be used to create, remove, import, export, map or
+unmap images exactly like it would on Linux. Make sure to check the
+`RBD basic commands`_ guide.
+
+``librbd.dll`` is also available for applications that can natively use Ceph.
+
+Please check the `installation guide`_ to get started.
+
+Windows service
+===============
+On Windows, ``rbd-wnbd`` daemons are managed by a centralized service. This allows
+decoupling the daemons from the Windows session from which they originate. At
+the same time, the service is responsible of recreating persistent mappings,
+usually when the host boots.
+
+Note that only one such service may run per host.
+
+By default, all image mappings are persistent. Non-persistent mappings can be
+requested using the ``-onon-persistent`` ``rbd`` flag.
+
+Persistent mappings are recreated when the service starts, unless explicitly
+unmapped. The service disconnects the mappings when being stopped. This also
+allows adjusting the Windows service start order so that RBD images can be
+mapped before starting services that may depend on it, such as VMMS.
+
+In order to be able to reconnect the images, ``rbd-wnbd`` stores mapping
+information in the Windows registry at the following location:
+``SYSTEM\CurrentControlSet\Services\rbd-wnbd``.
+
+The following command can be used to configure the service. Please update
+the ``rbd-wnbd.exe`` path accordingly::
+
+    New-Service -Name "ceph-rbd" `
+                -Description "Ceph RBD Mapping Service" `
+                -BinaryPathName "c:\ceph\rbd-wnbd.exe service" `
+                -StartupType Automatic
+
+Note that the Ceph MSI installer takes care of creating the ``ceph-rbd``
+Windows service.
+
+Usage
+=====
+
+Integration
+-----------
+
+RBD images can be exposed to the OS and host Windows partitions or they can be
+attached to Hyper-V VMs in the same way as iSCSI disks.
+
+Starting with Openstack Wallaby, the Nova Hyper-V driver can attach RBD Cinder
+volumes to Hyper-V VMs.
+
+Mapping images
+--------------
+
+The workflow and CLI is similar to the Linux counterpart, with a few
+notable differences:
+
+* device paths cannot be requested. The disk number and path will be picked by
+  Windows. If a device path is provided by the used when mapping an image, it
+  will be used as an identifier, which can also be used when unmapping the
+  image.
+* the ``show`` command was added, which describes a specific mapping.
+  This can be used for retrieving the disk path.
+* the ``service`` command was added, allowing ``rbd-wnbd`` to run as a Windows service.
+  All mappings are by default persistent, being recreated when the service
+  stops, unless explicitly unmapped. The service disconnects the mappings
+  when being stopped.
+* the ``list`` command also includes a ``status`` column.
+
+The purpose of the ``service`` mode is to ensure that mappings survive reboots
+and that the Windows service start order can be adjusted so that RBD images can
+be mapped before starting services that may depend on it, such as VMMS.
+
+The mapped images can either be consumed by the host directly or exposed to
+Hyper-V VMs.
+
+Hyper-V VM disks
+----------------
+
+The following sample imports an RBD image and boots a Hyper-V VM using it::
+
+    # Feel free to use any other image. This one is convenient to use for
+    # testing purposes because it's very small (~15MB) and the login prompt
+    # prints the pre-configured password.
+    wget http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img `
+         -OutFile cirros-0.5.1-x86_64-disk.img
+
+    # We'll need to make sure that the imported images are raw (so no qcow2 or vhdx).
+    # You may get qemu-img from https://cloudbase.it/qemu-img-windows/
+    # You can add the extracted location to $env:Path or update the path accordingly.
+    qemu-img convert -O raw cirros-0.5.1-x86_64-disk.img cirros-0.5.1-x86_64-disk.raw
+
+    rbd import cirros-0.5.1-x86_64-disk.raw
+    # Let's give it a hefty 100MB size.
+    rbd resize cirros-0.5.1-x86_64-disk.raw --size=100MB
+
+    rbd device map cirros-0.5.1-x86_64-disk.raw
+
+    # Let's have a look at the mappings.
+    rbd device list
+    Get-Disk
+
+    $mappingJson = rbd-wnbd show cirros-0.5.1-x86_64-disk.raw --format=json
+    $mappingJson = $mappingJson | ConvertFrom-Json
+
+    $diskNumber = $mappingJson.disk_number
+
+    New-VM -VMName BootFromRBD -MemoryStartupBytes 512MB
+    # The disk must be turned offline before it can be passed to Hyper-V VMs
+    Set-Disk -Number $diskNumber -IsOffline $true
+    Add-VMHardDiskDrive -VMName BootFromRBD -DiskNumber $diskNumber
+    Start-VM -VMName BootFromRBD
+
+Windows partitions
+------------------
+
+The following sample creates an empty RBD image, attaches it to the host and
+initializes a partition::
+
+    rbd create blank_image --size=1G
+    rbd device map blank_image -onon-persistent
+
+    $mappingJson = rbd-wnbd show blank_image --format=json
+    $mappingJson = $mappingJson | ConvertFrom-Json
+
+    $diskNumber = $mappingJson.disk_number
+
+    # The disk must be online before creating or accessing partitions.
+    Set-Disk -Number $diskNumber -IsOffline $false
+
+    # Initialize the disk, partition it and create a filesystem.
+    Get-Disk -Number $diskNumber | `
+        Initialize-Disk -PassThru | `
+        New-Partition -AssignDriveLetter -UseMaximumSize | `
+        Format-Volume -Force -Confirm:$false
+
+    # Show the partition letter (for example, "D:" or "F:"):
+    (Get-Partition -DiskNumber $diskNumber).DriveLetter
+
+SAN policy
+----------
+
+The Windows SAN policy determines which disks will be automatically mounted.
+The default policy (``offlineShared``) specifies that:
+
+  All newly discovered disks that do not reside on a shared bus (such as SCSI
+  and iSCSI) are brought online and made read-write. Disks that are left
+  offline will be read-only by default."
+
+Note that recent WNBD driver versions report rbd-wnbd disks as SAS, which is
+also considered a shared bus. As a result, the disks will be offline and
+read-only by default.
+
+In order to turn a disk online (mounting the disk partitions) and clear the
+read-only flag, use the following commands::
+
+  Set-Disk -Number $diskNumber -IsOffline $false
+  Set-Disk -Number $diskNumber -IsReadOnly $false
+
+Please check the `Limitations`_ section to learn about the Windows limitations
+that affect automatically mounted disks.
+
+Windows documentation:
+
+* `SAN policy reference`_
+* `san command`_
+* `StorageSetting command`_
+
+Limitations
+-----------
+
+CSV support
+~~~~~~~~~~~
+
+At the moment, the Microsoft Failover Cluster can't use WNBD disks as
+Cluster Shared Volumes (CSVs) underlying storage. The main reason is that
+``WNBD`` and ``rbd-wnbd`` don't support the *SCSI Persistent Reservations*
+feature yet.
+
+Hyper-V disk addressing
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. warning::
+  Hyper-V identifies passthrough VM disks by number instead of SCSI ID, although
+  the disk number can change across host reboots. This means that the VMs can end
+  up using incorrect disks after rebooting the host, which is an important
+  security concern. This issue also affects iSCSI and Fibre Channel disks.
+
+There are a few possible ways of avoiding this Hyper-V limitation:
+
+* use an NTFS/ReFS partition to store VHDX image files instead of directly
+  attaching the RBD image. This may slightly impact the IO performance.
+* use the Hyper-V ``AutomaticStartAction`` setting to prevent the VMs from
+  booting with the incorrect disks and have a script that updates VM disks
+  attachments before powering them back on. The ``ElementName`` field of the
+  `Msvm_StorageAllocationSettingData`_ `WMI`_ class may be used to label VM
+  disk attachments.
+* use the Openstack Hyper-V driver, which automatically refreshes the VM disk
+  attachments before powering them back on.
+
+Automatically mounted disks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Disks that are marked as "online" or "writable" will remain so after being
+reconnected (e.g. due to host reboots, Ceph service restarts, etc).
+
+Unfortunately, Windows restores the disk status based on the disk number,
+ignoring the disk unique identifier. However, the disk numbers can change
+after being reconnected. This issue also affects iSCSI and Fibre Channel disks.
+
+Let's assume that the `SAN policy`_ is set to ``offlineShared``, three
+RBD images are attached and disk 1 is turned online. After a reboot, disk 1
+will become online but it may now correspond to a different RBD image. This can
+be an issue if the disk that was mounted on the host was actually meant for a
+VM.
+
+Troubleshooting
+===============
+
+Please consult the `Windows troubleshooting`_ page.
+
+.. _Windows troubleshooting: ../../install/windows-troubleshooting
+.. _installation guide: ../../install/windows-install
+.. _RBD basic commands: ../rados-rbd-cmds
+.. _WNBD driver: https://github.com/cloudbase/wnbd
+.. _Msvm_StorageAllocationSettingData: https://docs.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-storageallocationsettingdata
+.. _WMI: https://docs.microsoft.com/en-us/windows/win32/wmisdk/wmi-start-page
+.. _san command: https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/san
+.. _StorageSetting command: https://learn.microsoft.com/en-us/powershell/module/storage/set-storagesetting?view=windowsserver2022-ps
+.. _SAN policy reference: https://learn.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-partitionmanager-sanpolicy
diff --git a/doc/releases/argonaut.rst b/doc/releases/argonaut.rst
new file mode 100644
index 000000000..409d1af91
--- /dev/null
+++ b/doc/releases/argonaut.rst
@@ -0,0 +1,185 @@
+========
+Argonaut
+========
+
+Argonaut is the first stable release of Ceph.  It is named after a
+genus of octopuses, sometimes also referred to as paper nautiluses.
+
+
+v0.48.3 "argonaut"
+==================
+
+This release contains a critical fix that can prevent data loss or
+corruption after a power loss or kernel panic event.  Please upgrade
+immediately.
+
+Upgrading
+---------
+
+* If you are using the undocumented ``ceph-disk-prepare`` and
+  ``ceph-disk-activate`` tools, they have several new features and
+  some additional functionality.  Please review the changes in
+  behavior carefully before upgrading.
+* The .deb packages now require xfsprogs.
+
+Notable changes
+---------------
+
+* filestore: fix op_seq write order (fixes journal replay after power loss)
+* osd: fix occasional indefinitely hung "slow" request
+* osd: fix encoding for pool_snap_info_t when talking to pre-v0.48 clients
+* osd: fix heartbeat check
+* osd: reduce log noise about rbd watch
+* log: fixes for deadlocks in the internal logging code
+* log: make log buffer size adjustable
+* init script: fix for 'ceph status' across machines
+* radosgw: fix swift error handling
+* radosgw: fix swift authentication concurrency bug
+* radosgw: don't cache large objects
+* radosgw: fix some memory leaks
+* radosgw: fix timezone conversion on read
+* radosgw: relax date format restrictions
+* radosgw: fix multipart overwrite
+* radosgw: stop processing requests on client disconnect
+* radosgw: avoid adding port to url that already has a port
+* radosgw: fix copy to not override ETAG
+* common: make parsing of ip address lists more forgiving
+* common: fix admin socket compatibility with old protocol (for collectd plugin)
+* mon: drop dup commands on paxos reset
+* mds: fix loner selection for multiclient workloads
+* mds: fix compat bit checks
+* ceph-fuse: fix segfault on startup when keyring is missing
+* ceph-authtool: fix usage
+* ceph-disk-activate: misc backports
+* ceph-disk-prepare: misc backports
+* debian: depend on xfsprogs (we use xfs by default)
+* rpm: build rpms, some related Makefile changes
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.48.3argonaut.txt>`.
+
+v0.48.2 "argonaut"
+==================
+
+Upgrading
+---------
+
+* The default search path for keyring files now includes /etc/ceph/ceph.$name.keyring.  If such files are present on your cluster, be aware that by default they may now be used.
+
+* There are several changes to the upstart init files.  These have not been previously documented or recommended.  Any existing users should review the changes before upgrading.
+
+* The ceph-disk-prepare and ceph-disk-active scripts have been updated significantly.  These have not been previously documented or recommended.  Any existing users should review the changes before upgrading.
+
+Notable changes
+---------------
+
+* mkcephfs: fix keyring generation for mds, osd when default paths are used
+* radosgw: fix bug causing occasional corruption of per-bucket stats
+* radosgw: workaround to avoid previously corrupted stats from going negative
+* radosgw: fix bug in usage stats reporting on busy buckets
+* radosgw: fix Content-Range: header for objects bigger than 2 GB.
+* rbd: avoid leaving watch acting when command line tool errors out (avoids 30s delay on subsequent operations)
+* rbd: friendlier use of --pool/--image options for import (old calling convention still works)
+* librbd: fix rare snapshot creation race (could "lose" a snap when creation is concurrent)
+* librbd: fix discard handling when spanning holes
+* librbd: fix memory leak on discard when caching is enabled
+* objecter: misc fixes for op reordering
+* objecter: fix for rare startup-time deadlock waiting for osdmap
+* ceph: fix usage
+* mon: reduce log noise about "check_sub"
+* ceph-disk-activate: misc fixes, improvements
+* ceph-disk-prepare: partition and format osd disks automatically
+* upstart: start everyone on a reboot
+* upstart: always update the osd crush location on start if specified in the config
+* config: add /etc/ceph/ceph.$name.keyring to default keyring search path
+* ceph.spec: don't package crush headers
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.48.2argonaut.txt>`.
+
+v0.48.1 "argonaut"
+==================
+
+Upgrading
+---------
+
+* The radosgw usage trim function was effectively broken in v0.48.  Earlier it would remove more usage data than what was requested.  This is fixed in v0.48.1, but the fix is incompatible.  The v0.48 radosgw-admin tool cannot be used to initiate the trimming; please use the v0.48.1 version.
+
+* v0.48.1 now explicitly indicates support for the CRUSH_TUNABLES feature.  No other version of Ceph requires this, yet, but future versions will when the tunables are adjusted from their historical defaults.
+
+* There are no other compatibility changes between v0.48.1 and v0.48.
+
+Notable changes
+---------------
+
+* mkcephfs: use default 'keyring', 'osd data', 'osd journal' paths when not specified in conf
+* msgr: various fixes to socket error handling
+* osd: reduce scrub overhead
+* osd: misc peering fixes (past_interval sharing, pgs stuck in 'peering' states)
+* osd: fail on EIO in read path (do not silently ignore read errors from failing disks)
+* osd: avoid internal heartbeat errors by breaking some large transactions into pieces
+* osd: fix osdmap catch-up during startup (catch up and then add daemon to osdmap)
+* osd: fix spurious 'misdirected op' messages
+* osd: report scrub status via 'pg ... query'
+* rbd: fix race when watch registrations are resent
+* rbd: fix rbd image id assignment scheme (new image data objects have slightly different names)
+* rbd: fix perf stats for cache hit rate
+* rbd tool: fix off-by-one in key name (crash when empty key specified)
+* rbd: more robust udev rules
+* rados tool: copy object, pool commands
+* radosgw: fix in usage stats trimming
+* radosgw: misc API compatibility fixes (date strings, ETag quoting, swift headers, etc.)
+* ceph-fuse: fix locking in read/write paths
+* mon: fix rare race corrupting on-disk data
+* config: fix admin socket 'config set' command
+* log: fix in-memory log event gathering
+* debian: remove crush headers, include librados-config
+* rpm: add ceph-disk-{activate, prepare}
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.48.1argonaut.txt>`.
+
+v0.48 "argonaut"
+================
+
+Upgrading
+---------
+
+* This release includes a disk format upgrade.  Each ceph-osd daemon, upon startup, will migrate its locally stored data to the new format.  This process can take a while (for large object counts, even hours), especially on non-btrfs file systems.
+
+* To keep the cluster available while the upgrade is in progress, we recommend you upgrade a storage node or rack at a time, and wait for the cluster to recover each time.  To prevent the cluster from moving data around in response to the OSD daemons being down for minutes or hours, you may want to::
+
+    ceph osd set noout
+
+  This will prevent the cluster from marking down OSDs as "out" and re-replicating the data elsewhere. If you do this, be sure to clear the flag when the upgrade is complete::
+
+    ceph osd unset noout
+
+* There is a encoding format change internal to the monitor cluster. The monitor daemons are careful to switch to the new format only when all members of the quorum support it.  However, that means that a partial quorum with new code may move to the new format, and a recovering monitor running old code will be unable to join (it will crash).  If this occurs, simply upgrading the remaining monitor will resolve the problem.
+
+* The ceph tool's -s and -w commands from previous versions are incompatible with this version. Upgrade your client tools at the same time you upgrade the monitors if you rely on those commands.
+
+* It is not possible to downgrade from v0.48 to a previous version.
+
+Notable changes
+---------------
+
+* osd: stability improvements
+* osd: capability model simplification
+* osd: simpler/safer --mkfs (no longer removes all files; safe to re-run on active osd)
+* osd: potentially buggy FIEMAP behavior disabled by default
+* rbd: caching improvements
+* rbd: improved instrumentation
+* rbd: bug fixes
+* radosgw: new, scalable usage logging infrastructure
+* radosgw: per-user bucket limits
+* mon: streamlined process for setting up authentication keys
+* mon: stability improvements
+* mon: log message throttling
+* doc: improved documentation (ceph, rbd, radosgw, chef, etc.)
+* config: new default locations for daemon keyrings
+* config: arbitrary variable substitutions
+* improved 'admin socket' daemon admin interface (ceph --admin-daemon ...)
+* chef: support for multiple monitor clusters
+* upstart: basic support for monitors, mds, radosgw; osd support still a work in progress.
+
+The new default keyring locations mean that when enabling authentication (``auth supported = cephx``), keyring locations do not need to be specified if the keyring file is located inside the daemon's data directory (``/var/lib/ceph/$type/ceph-$id`` by default).
+
+There is also a lot of librbd code in this release that is laying the groundwork for the upcoming layering functionality, but is not actually used. Likewise, the upstart support is still incomplete and not recommended; we will backport that functionality later if it turns out to be non-disruptive.
diff --git a/doc/releases/bobtail.rst b/doc/releases/bobtail.rst
new file mode 100644
index 000000000..9cca4f19e
--- /dev/null
+++ b/doc/releases/bobtail.rst
@@ -0,0 +1,546 @@
+=======
+Bobtail
+=======
+
+Bobtail is the second stable release of Ceph.  It is named after the
+bobtail squid (order Sepiolida), a group of cephalopods closely related to cuttlefish.
+
+v0.56.7 "bobtail"
+=================
+
+This bobtail update fixes a range of radosgw bugs (including an easily
+triggered crash from multi-delete), a possible data corruption issue
+with power failure on XFS, and several OSD problems, including a
+memory "leak" that will affect aged clusters.
+
+Notable changes
+---------------
+
+* ceph-fuse: create finisher flags after fork()
+* debian: fix prerm/postinst hooks; do not restart daemons on upgrade
+* librados: fix async aio completion wakeup (manifests as rbd hang)
+* librados: fix hang when osd becomes full and then not full
+* librados: fix locking for aio completion refcounting
+* librbd python bindings: fix stripe_unit, stripe_count
+* librbd: make image creation default configurable
+* mon: fix validation of mds ids in mon commands
+* osd: avoid excessive disk updates during peering
+* osd: avoid excessive memory usage on scrub
+* osd: avoid heartbeat failure/suicide when scrubbing
+* osd: misc minor bug fixes
+* osd: use fdatasync instead of sync_file_range (may avoid xfs power-loss corruption)
+* rgw: escape prefix correctly when listing objects
+* rgw: fix copy attrs
+* rgw: fix crash on multi delete
+* rgw: fix locking/crash when using ops log socket
+* rgw: fix usage logging
+* rgw: handle deep uri resources
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.7.txt>`.
+
+
+v0.56.6 "bobtail"
+=================
+
+Notable changes
+---------------
+
+* rgw: fix garbage collection
+* rpm: fix package dependencies
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.6.txt>`.
+
+
+v0.56.5 "bobtail"
+=================
+
+Upgrading
+---------
+
+* ceph-disk[-prepare,-activate] behavior has changed in various ways.
+  There should not be any compatibility issues, but chef users should
+  be aware.
+
+Notable changes
+---------------
+
+* mon: fix recording of quorum feature set (important for argonaut -> bobtail -> cuttlefish mon upgrades)
+* osd: minor peering bug fixes
+* osd: fix a few bugs when pools are renamed
+* osd: fix occasionally corrupted pg stats
+* osd: fix behavior when broken v0.56[.0] clients connect
+* rbd: avoid FIEMAP ioctl on import (it is broken on some kernels)
+* librbd: fixes for several request/reply ordering bugs
+* librbd: only set STRIPINGV2 feature on new images when needed
+* librbd: new async flush method to resolve qemu hangs (requires QEMU update as well)
+* librbd: a few fixes to flatten
+* ceph-disk: support for dm-crypt
+* ceph-disk: many backports to allow bobtail deployments with ceph-deploy, chef
+* sysvinit: do not stop starting daemons on first failure
+* udev: fixed rules for redhat-based distros
+* build fixes for raring
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.5.txt>`.
+
+v0.56.4 "bobtail"
+=================
+
+Upgrading
+---------
+
+* There is a fix in the syntax for the output of 'ceph osd tree --format=json'.
+
+* The MDS disk format has changed from prior releases *and* from v0.57.  In particular,
+  upgrades to v0.56.4 are safe, but you cannot move from v0.56.4 to v0.57 if you are using
+  the MDS for CephFS; you must upgrade directly to v0.58 (or later) instead.
+
+Notable changes
+---------------
+
+* mon: fix bug in bringup with IPv6
+* reduce default memory utilization by internal logging (all daemons)
+* rgw: fix for bucket removal
+* rgw: reopen logs after log rotation
+* rgw: fix multipat upload listing
+* rgw: don't copy object when copied onto self
+* osd: fix caps parsing for pools with - or _
+* osd: allow pg log trimming when degraded, scrubbing, recoverying (reducing memory consumption)
+* osd: fix potential deadlock when 'journal aio = true'
+* osd: various fixes for collection creation/removal, rename, temp collections
+* osd: various fixes for PG split
+* osd: deep-scrub omap key/value data
+* osd: fix rare bug in journal replay
+* osd: misc fixes for snapshot tracking
+* osd: fix leak in recovery reservations on pool deletion
+* osd: fix bug in connection management
+* osd: fix for op ordering when rebalancing
+* ceph-fuse: report file system size with correct units
+* mds: get and set directory layout policies via virtual xattrs
+* mds: on-disk format revision (see upgrading note above)
+* mkcephfs, init-ceph: close potential security issues with predictable filenames
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.4.txt>`.
+
+v0.56.3 "bobtail"
+=================
+
+This release has several bug fixes surrounding OSD stability.  Most
+significantly, an issue with OSDs being unresponsive shortly after
+startup (and occasionally crashing due to an internal heartbeat check)
+is resolved.  Please upgrade.
+
+Upgrading
+---------
+
+* A bug was fixed in which the OSDMap epoch for PGs without any IO
+  requests was not recorded.  If there are pools in the cluster that
+  are completely idle (for example, the ``data`` and ``metadata``
+  pools normally used by CephFS), and a large number of OSDMap epochs
+  have elapsed since the ``ceph-osd`` daemon was last restarted, those
+  maps will get reprocessed when the daemon restarts.  This process
+  can take a while if there are a lot of maps.  A workaround is to
+  'touch' any idle pools with IO prior to restarting the daemons after
+  packages are upgraded::
+
+   rados bench 10 write -t 1 -b 4096 -p {POOLNAME}
+
+  This will typically generate enough IO to touch every PG in the pool
+  without generating significant cluster load, and also cleans up any
+  temporary objects it creates.
+
+Notable changes
+---------------
+
+* osd: flush peering work queue prior to start
+* osd: persist osdmap epoch for idle PGs
+* osd: fix and simplify connection handling for heartbeats
+* osd: avoid crash on invalid admin command
+* mon: fix rare races with monitor elections and commands
+* mon: enforce that OSD reweights be between 0 and 1 (NOTE: not CRUSH weights)
+* mon: approximate client, recovery bandwidth logging
+* radosgw: fixed some XML formatting to conform to Swift API inconsistency
+* radosgw: fix usage accounting bug; add repair tool
+* radosgw: make fallback URI configurable (necessary on some web servers)
+* librbd: fix handling for interrupted 'unprotect' operations
+* mds, ceph-fuse: allow file and directory layouts to be modified via virtual xattrs
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.3.txt>`.
+
+
+v0.56.2 "bobtail"
+=================
+
+This release has a wide range of bug fixes, stability improvements, and some performance improvements.  Please upgrade.
+
+Upgrading
+---------
+
+* The meaning of the 'osd scrub min interval' and 'osd scrub max
+  interval' has changed slightly.  The min interval used to be
+  meaningless, while the max interval would only trigger a scrub if
+  the load was sufficiently low.  Now, the min interval option works
+  the way the old max interval did (it will trigger a scrub after this
+  amount of time if the load is low), while the max interval will
+  force a scrub regardless of load.  The default options have been
+  adjusted accordingly.  If you have customized these in ceph.conf,
+  please review their values when upgrading.
+
+* CRUSH maps that are generated by default when calling ``ceph-mon
+  --mkfs`` directly now distribute replicas across hosts instead of
+  across OSDs.  Any provisioning tools that are being used by Ceph may
+  be affected, although probably for the better, as distributing across
+  hosts is a much more commonly sought behavior.  If you use
+  ``mkcephfs`` to create the cluster, the default CRUSH rule is still
+  inferred by the number of hosts and/or racks in the initial ceph.conf.
+
+Notable changes
+---------------
+
+* osd: snapshot trimming fixes
+* osd: scrub snapshot metadata
+* osd: fix osdmap trimming
+* osd: misc peering fixes
+* osd: stop heartbeating with peers if internal threads are stuck/hung
+* osd: PG removal is friendlier to other workloads
+* osd: fix recovery start delay (was causing very slow recovery)
+* osd: fix scheduling of explicitly requested scrubs
+* osd: fix scrub interval config options
+* osd: improve recovery vs client io tuning
+* osd: improve 'slow request' warning detail for better diagnosis
+* osd: default CRUSH map now distributes across hosts, not OSDs
+* osd: fix crash on 32-bit hosts triggered by librbd clients
+* librbd: fix error handling when talking to older OSDs
+* mon: fix a few rare crashes
+* ceph command: ability to easily adjust CRUSH tunables
+* radosgw: object copy does not copy source ACLs
+* rados command: fix omap command usage
+* sysvinit script: set ulimit -n properly on remote hosts
+* msgr: fix narrow race with message queuing
+* fixed compilation on some old distros (e.g., RHEL 5.x)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.2.txt>`.
+
+
+v0.56.1 "bobtail"
+=================
+
+This release has two critical fixes.  Please upgrade.
+
+Upgrading
+---------
+
+* There is a protocol compatibility problem between v0.56 and any
+  other version that is now fixed.  If your radosgw or RBD clients are
+  running v0.56, they will need to be upgraded too.  If they are
+  running a version prior to v0.56, they can be left as is.
+
+Notable changes
+---------------
+* osd: fix commit sequence for XFS, ext4 (or any other non-btrfs) to prevent data loss on power cycle or kernel panic
+* osd: fix compatibility for CALL operation
+* osd: process old osdmaps prior to joining cluster (fixes slow startup)
+* osd: fix a couple of recovery-related crashes
+* osd: fix large io requests when journal is in (non-default) aio mode
+* log: fix possible deadlock in logging code
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.56.1.txt>`.
+
+v0.56 "bobtail"
+===============
+
+Bobtail is the second stable release of Ceph, named in honor of the
+`Bobtail Squid`: https://en.wikipedia.org/wiki/Bobtail_squid.
+
+Key features since v0.48 "argonaut"
+-----------------------------------
+
+* Object Storage Daemon (OSD): improved threading, small-io performance, and performance during recovery
+* Object Storage Daemon (OSD): regular "deep" scrubbing of all stored data to detect latent disk errors
+* RADOS Block Device (RBD): support for copy-on-write clones of images.
+* RADOS Block Device (RBD): better client-side caching.
+* RADOS Block Device (RBD): advisory image locking
+* Rados Gateway (RGW): support for efficient usage logging/scraping (for billing purposes)
+* Rados Gateway (RGW): expanded S3 and Swift API coverage (e.g., POST, multi-object delete)
+* Rados Gateway (RGW): improved striping for large objects
+* Rados Gateway (RGW): OpenStack Keystone integration
+* RPM packages for Fedora, RHEL/CentOS, OpenSUSE, and SLES
+* mkcephfs: support for automatically formatting and mounting XFS and ext4 (in addition to btrfs)
+
+Upgrading
+---------
+
+Please refer to the document `Upgrading from Argonaut to Bobtail`_ for details.
+
+.. _Upgrading from Argonaut to Bobtail: ../install/upgrading-ceph/#upgrading-from-argonaut-to-bobtail
+
+* Cephx authentication is now enabled by default (since v0.55).
+  Upgrading a cluster without adjusting the Ceph configuration will
+  likely prevent the system from starting up on its own.  We recommend
+  first modifying the configuration to indicate that authentication is
+  disabled, and only then upgrading to the latest version::
+
+     auth client required = none
+     auth service required = none
+     auth cluster required = none
+
+* Ceph daemons can be upgraded one-by-one while the cluster is online
+  and in service.
+
+* The ``ceph-osd`` daemons must be upgraded and restarted *before* any
+  ``radosgw`` daemons are restarted, as they depend on some new
+  ceph-osd functionality.  (The ``ceph-mon``, ``ceph-osd``, and
+  ``ceph-mds`` daemons can be upgraded and restarted in any order.)
+
+* Once each individual daemon has been upgraded and restarted, it
+  cannot be downgraded.
+
+* The cluster of ``ceph-mon`` daemons will migrate to a new internal
+  on-wire protocol once all daemons in the quorum have been upgraded.
+  Upgrading only a majority of the nodes (e.g., two out of three) may
+  expose the cluster to a situation where a single additional failure
+  may compromise availability (because the non-upgraded daemon cannot
+  participate in the new protocol).  We recommend not waiting for an
+  extended period of time between ``ceph-mon`` upgrades.
+
+* The ops log and usage log for radosgw are now off by default.  If
+  you need these logs (e.g., for billing purposes), you must enable
+  them explicitly.  For logging of all operations to objects in the
+  ``.log`` pool (see ``radosgw-admin log ...``)::
+
+    rgw enable ops log = true
+
+  For usage logging of aggregated bandwidth usage (see ``radosgw-admin
+  usage ...``)::
+
+    rgw enable usage log = true
+
+* You should not create or use "format 2" RBD images until after all
+  ``ceph-osd`` daemons have been upgraded.  Note that "format 1" is
+  still the default.  You can use the new ``ceph osd ls`` and
+  ``ceph tell osd.N version`` commands to doublecheck your cluster.
+  ``ceph osd ls`` will give a list of all OSD IDs that are part of the
+  cluster, and you can use that to write a simple shell loop to display
+  all the OSD version strings: ::
+
+      for i in $(ceph osd ls); do
+          ceph tell osd.${i} version
+      done
+
+
+Compatibility changes
+---------------------
+
+* The 'ceph osd create [<uuid>]' command now rejects an argument that
+  is not a UUID.  (Previously it would take an optional integer
+  OSD id.)  This correct syntax has been 'ceph osd create [<uuid>]'
+  since v0.47, but the older calling convention was being silently
+  ignored.
+
+* The CRUSH map root nodes now have type ``root`` instead of type
+  ``pool``.  This avoids confusion with RADOS pools, which are not
+  directly related.  Any scripts or tools that use the ``ceph osd
+  crush ...`` commands may need to be adjusted accordingly.
+
+* The ``ceph osd pool create <poolname> <pgnum>`` command now requires
+  the ``pgnum`` argument. Previously this was optional, and would
+  default to 8, which was almost never a good number.
+
+* Degraded mode (when there fewer than the desired number of replicas)
+  is now more configurable on a per-pool basis, with the min_size
+  parameter. By default, with min_size 0, this allows I/O to objects
+  with N - floor(N/2) replicas, where N is the total number of
+  expected copies. Argonaut behavior was equivalent to having min_size
+  = 1, so I/O would always be possible if any completely up to date
+  copy remained. min_size = 1 could result in lower overall
+  availability in certain cases, such as flapping network partitions.
+
+* The sysvinit start/stop script now defaults to adjusting the max
+  open files ulimit to 16384.  On most systems the default is 1024, so
+  this is an increase and won't break anything.  If some system has a
+  higher initial value, however, this change will lower the limit.
+  The value can be adjusted explicitly by adding an entry to the
+  ``ceph.conf`` file in the appropriate section.  For example::
+
+     [global]
+             max open files = 32768
+
+* 'rbd lock list' and 'rbd showmapped' no longer use tabs as
+  separators in their output.
+
+* There is configurable limit on the number of PGs when creating a new
+  pool, to prevent a user from accidentally specifying a ridiculous
+  number for pg_num.  It can be adjusted via the 'mon max pool pg num'
+  option on the monitor, and defaults to 65536 (the current max
+  supported by the Linux kernel client).
+
+* The osd capabilities associated with a rados user have changed
+  syntax since 0.48 argonaut. The new format is mostly backwards
+  compatible, but there are two backwards-incompatible changes:
+
+  * specifying a list of pools in one grant, i.e.
+    'allow r pool=foo,bar' is now done in separate grants, i.e.
+    'allow r pool=foo, allow r pool=bar'.
+
+  * restricting pool access by pool owner ('allow r uid=foo') is
+    removed. This feature was not very useful and unused in practice.
+
+  The new format is documented in the ceph-authtool man page.
+
+* 'rbd cp' and 'rbd rename' use rbd as the default destination pool,
+  regardless of what pool the source image is in. Previously they
+  would default to the same pool as the source image.
+
+* 'rbd export' no longer prints a message for each object written. It
+  just reports percent complete like other long-lasting operations.
+
+* 'ceph osd tree' now uses 4 decimal places for weight so output is
+  nicer for humans
+
+* Several monitor operations are now idempotent:
+
+  * ceph osd pool create
+  * ceph osd pool delete
+  * ceph osd pool mksnap
+  * ceph osd rm
+  * ceph pg <pgid> revert
+
+Notable changes
+---------------
+
+* auth: enable cephx by default
+* auth: expanded authentication settings for greater flexibility
+* auth: sign messages when using cephx
+* build fixes for Fedora 18, CentOS/RHEL 6
+* ceph: new 'osd ls' and 'osd tell <osd.N> version' commands
+* ceph-debugpack: misc improvements
+* ceph-disk-prepare: creates and labels GPT partitions
+* ceph-disk-prepare: support for external journals, default mount/mkfs options, etc.
+* ceph-fuse/libcephfs: many misc fixes, admin socket debugging
+* ceph-fuse: fix handling for .. in root directory
+* ceph-fuse: many fixes (including memory leaks, hangs)
+* ceph-fuse: mount helper (mount.fuse.ceph) for use with /etc/fstab
+* ceph.spec: misc packaging fixes
+* common: thread pool sizes can now be adjusted at runtime
+* config: $pid is now available as a metavariable
+* crush: default root of tree type is now 'root' instead of 'pool' (to avoid confusiong wrt rados pools)
+* crush: fixed retry behavior with chooseleaf via tunable
+* crush: tunables documented; feature bit now present and enforced
+* libcephfs: java wrapper
+* librados: several bug fixes (rare races, locking errors)
+* librados: some locking fixes
+* librados: watch/notify fixes, misc memory leaks
+* librbd: a few fixes to 'discard' support
+* librbd: fine-grained striping feature
+* librbd: fixed memory leaks
+* librbd: fully functional and documented image cloning
+* librbd: image (advisory) locking
+* librbd: improved caching (of object non-existence)
+* librbd: 'flatten' command to sever clone parent relationship
+* librbd: 'protect'/'unprotect' commands to prevent clone parent from being deleted
+* librbd: clip requests past end-of-image.
+* librbd: fixes an issue with some windows guests running in qemu (remove floating point usage)
+* log: fix in-memory buffering behavior (to only write log messages on crash)
+* mds: fix ino release on abort session close, relative getattr path, mds shutdown, other misc items
+* mds: misc fixes
+* mkcephfs: fix for default keyring, osd data/journal locations
+* mkcephfs: support for formatting xfs, ext4 (as well as btrfs)
+* init: support for automatically mounting xfs and ext4 osd data directories
+* mon, radosgw, ceph-fuse: fixed memory leaks
+* mon: improved ENOSPC, fs error checking
+* mon: less-destructive ceph-mon --mkfs behavior
+* mon: misc fixes
+* mon: more informative info about stuck PGs in 'health detail'
+* mon: information about recovery and backfill in 'pg <pgid> query'
+* mon: new 'osd crush create-or-move ...' command
+* mon: new 'osd crush move ...' command lets you rearrange your CRUSH hierarchy
+* mon: optionally dump 'osd tree' in json
+* mon: configurable cap on maximum osd number (mon max osd)
+* mon: many bug fixes (various races causing ceph-mon crashes)
+* mon: new on-disk metadata to facilitate future mon changes (post-bobtail)
+* mon: election bug fixes
+* mon: throttle client messages (limit memory consumption)
+* mon: throttle osd flapping based on osd history (limits osdmap ΄thrashing' on overloaded or unhappy clusters)
+* mon: 'report' command for dumping detailed cluster status (e.g., for use when reporting bugs)
+* mon: osdmap flags like noup, noin now cause a health warning
+* msgr: improved failure handling code
+* msgr: many bug fixes
+* osd, mon: honor new 'nobackfill' and 'norecover' osdmap flags
+* osd, mon: use feature bits to lock out clients lacking CRUSH tunables when they are in use
+* osd: backfill reservation framework (to avoid flooding new osds with backfill data)
+* osd: backfill target reservations (improve performance during recovery)
+* osd: better tracking of recent slow operations
+* osd: capability grammar improvements, bug fixes
+* osd: client vs recovery io prioritization
+* osd: crush performance improvements
+* osd: default journal size to 5 GB
+* osd: experimental support for PG "splitting" (pg_num adjustment for existing pools)
+* osd: fix memory leak on certain error paths
+* osd: fixed detection of EIO errors from fs on read
+* osd: major refactor of PG peering and threading
+* osd: many bug fixes
+* osd: more/better dump info about in-progress operations
+* osd: new caps structure (see compatibility notes)
+* osd: new 'deep scrub' will compare object content across replicas (once per week by default)
+* osd: new 'lock' rados class for generic object locking
+* osd: optional 'min' pg size
+* osd: recovery reservations
+* osd: scrub efficiency improvement
+* osd: several out of order reply bug fixes
+* osd: several rare peering cases fixed
+* osd: some performance improvements related to request queuing
+* osd: use entire device if journal is a block device
+* osd: use syncfs(2) when kernel supports it, even if glibc does not
+* osd: various fixes for out-of-order op replies
+* rados: ability to copy, rename pools
+* rados: bench command now cleans up after itself
+* rados: 'cppool' command to copy rados pools
+* rados: 'rm' now accepts a list of objects to be removed
+* radosgw: POST support
+* radosgw: REST API for managing usage stats
+* radosgw: fix bug in bucket stat updates
+* radosgw: fix copy-object vs attributes
+* radosgw: fix range header for large objects, ETag quoting, GMT dates, other compatibility fixes
+* radosgw: improved garbage collection framework
+* radosgw: many small fixes, cleanups
+* radosgw: openstack keystone integration
+* radosgw: stripe large (non-multipart) objects
+* radosgw: support for multi-object deletes
+* radosgw: support for swift manifest objects
+* radosgw: vanity bucket dns names
+* radosgw: various API compatibility fixes
+* rbd: import from stdin, export to stdout
+* rbd: new 'ls -l' option to view images with metadata
+* rbd: use generic id and keyring options for 'rbd map'
+* rbd: don't issue usage on errors
+* udev: fix symlink creation for rbd images containing partitions
+* upstart: job files for all daemon types (not enabled by default)
+* wireshark: ceph protocol dissector patch updated
+
+
+v0.54
+=====
+
+Upgrading
+---------
+
+* The osd capabilities associated with a rados user have changed
+  syntax since 0.48 argonaut. The new format is mostly backwards
+  compatible, but there are two backwards-incompatible changes:
+
+  * specifying a list of pools in one grant, i.e.
+    'allow r pool=foo,bar' is now done in separate grants, i.e.
+    'allow r pool=foo, allow r pool=bar'.
+
+  * restricting pool access by pool owner ('allow r uid=foo') is
+    removed. This feature was not very useful and unused in practice.
+
+  The new format is documented in the ceph-authtool man page.
+
+* Bug fixes to the new osd capability format parsing properly validate
+  the allowed operations. If an existing rados user gets permissions
+  errors after upgrading, its capabilities were probably
+  misconfigured. See the ceph-authtool man page for details on osd
+  capabilities.
+
+* 'rbd lock list' and 'rbd showmapped' no longer use tabs as
+  separators in their output.
diff --git a/doc/releases/cuttlefish.rst b/doc/releases/cuttlefish.rst
new file mode 100644
index 000000000..a7ebaa3d8
--- /dev/null
+++ b/doc/releases/cuttlefish.rst
@@ -0,0 +1,720 @@
+==========
+Cuttlefish
+==========
+
+Cuttlefish is the 3rd stable release of Ceph. It is named after a type
+of cephalopod (order Sepiida) characterized by a unique internal shell, the
+cuttlebone, which is used for control of buoyancy.
+
+v0.61.9 "Cuttlefish"
+====================
+
+This point release resolves several low to medium-impact bugs across
+the code base, and fixes a performance problem (CPU utilization) with
+radosgw.  We recommend that all production cuttlefish users upgrade.
+
+Notable Changes
+---------------
+
+* ceph, ceph-authtool: fix help (Danny Al-Gaaf)
+* ceph-disk: partprobe after creating journal partition
+* ceph-disk: specific fs type when mounting (Alfredo Deza)
+* ceph-fuse: fix bug when compiled against old versions
+* ceph-fuse: fix use-after-free in caching code (Yan, Zheng)
+* ceph-fuse: misc caching bugs
+* ceph.spec: remove incorrect mod_fcgi dependency (Gary Lowell)
+* crush: fix name caching
+* librbd: fix bug when unpausing cluster (Josh Durgin)
+* mds: fix LAZYIO lock hang
+* mds: fix bug in file size recovery (after client crash)
+* mon: fix paxos recovery corner case
+* osd: fix exponential backoff for slow request warnings (Loic Dachary)
+* osd: fix readdir_r usage
+* osd: fix startup for long-stopped OSDs
+* rgw: avoid std::list::size() to avoid wasting CPU cycles (Yehuda Sadeh)
+* rgw: drain pending requests during write (fixes data safety issue) (Yehuda Sadeh)
+* rgw: fix authenticated users group ACL check (Yehuda Sadeh)
+* rgw: fix bug in POST (Yehuda Sadeh)
+* rgw: fix sysvinit script 'status' command, return value (Danny Al-Gaaf)
+* rgw: reduce default log level (Yehuda Sadeh)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.9.txt>`.
+
+v0.61.8 "Cuttlefish"
+====================
+
+This release includes a number of important issues, including rare
+race conditions in the OSD, a few monitor bugs, and fixes for RBD
+flush behavior.  We recommend that production users upgrade at their
+convenience.
+
+Notable Changes
+---------------
+
+* librados: fix async aio completion wakeup
+* librados: fix aio completion locking
+* librados: fix rare deadlock during shutdown
+* osd: fix race when queueing recovery operations
+* osd: fix possible race during recovery
+* osd: optionally preload rados classes on startup (disabled by default)
+* osd: fix journal replay corner condition
+* osd: limit size of peering work queue batch (to speed up peering)
+* mon: fix paxos recovery corner case
+* mon: fix rare hang when monmap updates during an election
+* mon: make 'osd pool mksnap ...' avoid exposing uncommitted state
+* mon: make 'osd pool rmsnap ...' not racy, avoid exposing uncommitted state
+* mon: fix bug during mon cluster expansion
+* rgw: fix crash during multi delete operation
+* msgr: fix race conditions during osd network reinitialization
+* ceph-disk: apply mount options when remounting
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.8.txt>`.
+
+
+v0.61.7 "Cuttlefish"
+====================
+
+This release fixes another regression preventing monitors to start after
+undergoing certain upgrade sequences, as well as some corner cases with
+Paxos and support for unusual device names in ceph-disk/ceph-deploy.
+
+Notable Changes
+---------------
+
+* mon: fix regression in latest full osdmap retrieval
+* mon: fix a long-standing bug in a paxos corner case
+* ceph-disk: improved support for unusual device names (e.g., /dev/cciss/c0d0)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.7.txt>`.
+
+
+v0.61.6 "Cuttlefish"
+====================
+
+This release fixes a regression in v0.61.5 that could prevent monitors
+from restarting.  This affects any cluster that was upgraded from a
+previous version of Ceph (and not freshly created with v0.61.5).
+
+All users are strongly recommended to upgrade.
+
+Notable Changes
+---------------
+
+* mon: record latest full osdmap
+* mon: work around previous bug in which latest full osdmap is not recorded
+* mon: avoid scrub while updating
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.6.txt>`.
+
+
+v0.61.5 "Cuttlefish"
+====================
+
+This release most improves stability of the monitor and fixes a few
+bugs with the ceph-disk utility (used by ceph-deploy).  We recommend
+that all v0.61.x users upgrade.
+
+Upgrading
+---------
+
+* This release fixes a 32-bit vs 64-bit arithmetic bug with the
+  feature bits.  An unfortunate consequence of the fix is that 0.61.4
+  (or earlier) ceph-mon daemons can't form a quorum with 0.61.5 (or
+  later) monitors.  To avoid the possibility of service disruption, we
+  recommend you upgrade all monitors at once.
+
+Notable Changes
+---------------
+
+* mon: misc sync improvements (faster, more reliable, better tuning)
+* mon: enable leveldb cache by default (big performance improvement)
+* mon: new scrub feature (primarily for diagnostic, testing purposes)
+* mon: fix occasional leveldb assertion on startup
+* mon: prevent reads until initial state is committed
+* mon: improved logic for trimming old osdmaps
+* mon: fix pick_addresses bug when expanding mon cluster
+* mon: several small paxos fixes, improvements
+* mon: fix bug osdmap trim behavior
+* osd: fix several bugs with PG stat reporting
+* osd: limit number of maps shared with peers (which could cause domino failures)
+* rgw: fix radosgw-admin buckets list (for all buckets)
+* mds: fix occasional client failure to reconnect
+* mds: fix bad list traversal after unlink
+* mds: fix underwater dentry cleanup (occasional crash after mds restart)
+* libcephfs, ceph-fuse: fix occasional hangs on umount
+* libcephfs, ceph-fuse: fix old bug with O_LAZY vs O_NOATIME confusion
+* ceph-disk: more robust journal device detection on RHEL/CentOS
+* ceph-disk: better, simpler locking
+* ceph-disk: do not inadvertantely mount over existing osd mounts
+* ceph-disk: better handling for unusual device names
+* sysvinit, upstart: handle symlinks in /var/lib/ceph/*
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.5.txt>`.
+
+
+v0.61.4 "Cuttlefish"
+====================
+
+This release resolves a possible data corruption on power-cycle when
+using XFS, a few outstanding problems with monitor sync, several
+problems with ceph-disk and ceph-deploy operation, and a problem with
+OSD memory usage during scrub.
+
+Upgrading
+---------
+
+* No issues.
+
+Notable Changes
+---------------
+
+* mon: fix daemon exit behavior when error is encountered on startup
+* mon: more robust sync behavior
+* osd: do not use sync_file_range(2), posix_fadvise(...DONTNEED) (can cause data corruption on power loss on XFS)
+* osd: avoid unnecessary log rewrite (improves peering speed)
+* osd: fix scrub efficiency bug (problematic on old clusters)
+* rgw: fix listing objects that start with underscore
+* rgw: fix deep URI resource, CORS bugs
+* librados python binding: fix truncate on 32-bit architectures
+* ceph-disk: fix udev rules
+* rpm: install sysvinit script on package install
+* ceph-disk: fix OSD start on machine reboot on Debian wheezy
+* ceph-disk: activate OSD when journal device appears second
+* ceph-disk: fix various bugs on RHEL/CentOS 6.3
+* ceph-disk: add 'zap' command
+* ceph-disk: add '[un]suppress-activate' command for preparing spare disks
+* upstart: start on runlevel [2345] (instead of after the first network interface starts)
+* ceph-fuse, libcephfs: handle mds session reset during session open
+* ceph-fuse, libcephfs: fix two capability revocation bugs
+* ceph-fuse: fix thread creation on startup
+* all daemons: create /var/run/ceph directory on startup if missing
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.4.txt>`.
+
+
+v0.61.3 "Cuttlefish"
+====================
+
+This release resolves a number of problems with the monitors and leveldb that users have
+been seeing.  Please upgrade.
+
+Upgrading
+---------
+
+* There is one known problem with mon upgrades from bobtail.  If the
+  ceph-mon conversion on startup is aborted or fails for some reason, we
+  do not correctly error out, but instead continue with (in certain cases)
+  odd results.  Please be careful if you have to restart the mons during
+  the upgrade.  A 0.61.4 release with a fix will be out shortly.
+
+* In the meantime, for current cuttlefish users, v0.61.3 is safe to use.
+
+
+Notable Changes
+---------------
+
+* mon: paxos state trimming fix (resolves runaway disk usage)
+* mon: finer-grained compaction on trim
+* mon: discard messages from disconnected clients (lowers load)
+* mon: leveldb compaction and other stats available via admin socket
+* mon: async compaction (lower overhead)
+* mon: fix bug incorrectly marking osds down with insufficient failure reports
+* osd: fixed small bug in pg request map
+* osd: avoid rewriting pg info on every osdmap
+* osd: avoid internal heartbeta timeouts when scrubbing very large objects
+* osd: fix narrow race with journal replay
+* mon: fixed narrow pg split race
+* rgw: fix leaked space when copying object
+* rgw: fix iteration over large/untrimmed usage logs
+* rgw: fix locking issue with ops log socket
+* rgw: require matching version of librados
+* librbd: make image creation defaults configurable (e.g., create format 2 images via qemu-img)
+* fix units in 'ceph df' output
+* debian: fix prerm/postinst hooks to start/stop daemons appropriately
+* upstart: allow uppercase daemons names (and thus hostnames)
+* sysvinit: fix enumeration of local daemons by type
+* sysvinit: fix osd weight calcuation when using -a
+* fix build on unsigned char platforms (e.g., arm)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.3.txt>`.
+
+
+v0.61.2 "Cuttlefish"
+====================
+
+This release disables a monitor debug log that consumes disk space and
+fixes a bug when upgrade some monitors from bobtail to cuttlefish.
+
+Notable Changes
+---------------
+
+* mon: fix conversion of stores with duplicated GV values
+* mon: disable 'mon debug dump transactions' by default
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.2.txt>`.
+
+
+v0.61.1 "Cuttlefish"
+====================
+
+This release fixes a problem when upgrading a bobtail cluster that had
+snapshots to cuttlefish.
+
+Notable Changes
+---------------
+
+* osd: handle upgrade when legacy snap collections are present; repair from previous failed restart
+* ceph-create-keys: fix race with ceph-mon startup (which broke 'ceph-deploy gatherkeys ...')
+* ceph-create-keys: gracefully handle bad response from ceph-osd
+* sysvinit: do not assume default osd_data when automatically weighting OSD
+* osd: avoid crash from ill-behaved classes using getomapvals
+* debian: fix squeeze dependency
+* mon: debug options to log or dump leveldb transactions
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.61.1.txt>`.
+
+v0.61 "Cuttlefish"
+==================
+
+Upgrading from v0.60
+--------------------
+
+* The ceph-deploy tool is now the preferred method of provisioning
+  new clusters.  For existing clusters created via mkcephfs that
+  would like to transition to the new tool, there is a migration
+  path, documented at `Transitioning to ceph-deploy`_.
+
+
+* The sysvinit script (/etc/init.d/ceph) will now verify (and, if
+  necessary, update) the OSD's position in the CRUSH map on startup.
+  (The upstart script has always worked this way.) By default, this
+  ensures that the OSD is under a 'host' with a name that matches the
+  hostname (``hostname -s``).  Legacy clusters create with mkcephfs do
+  this by default, so this should not cause any problems, but legacy
+  clusters with customized CRUSH maps with an alternate structure
+  should set ``osd crush update on start = false``.
+
+* radosgw-admin now uses the term zone instead of cluster to describe
+  each instance of the radosgw data store (and corresponding
+  collection of radosgw daemons).  The usage for the radosgw-admin
+  command and the 'rgw zone root pool' config options have changed
+  accordingly.
+
+* rbd progress indicators now go to standard error instead of standard
+  out.  (You can disable progress with --no-progress.)
+
+* The 'rbd resize ...' command now requires the --allow-shrink option
+  when resizing to a smaller size.  Expanding images to a larger size
+  is unchanged.
+
+* Please review the changes going back to 0.56.4 if you are upgrading
+  all the way from bobtail.
+
+* The old 'ceph stop_cluster' command has been removed.
+
+* The sysvinit script now uses the ceph.conf file on the remote host
+  when starting remote daemons via the '-a' option.  Note that if '-a'
+  is used in conjunction with '-c path', the path must also be present
+  on the remote host (it is not copied to a temporary file, as it was
+  previously).
+
+
+Upgrading from v0.56.4 "Bobtail"
+--------------------------------
+
+Please see `Upgrading from Bobtail to Cuttlefish`_ for details.
+
+.. _Upgrading from Bobtail to Cuttlefish: ../install/upgrading-ceph/#upgrading-from-bobtail-to-cuttlefish
+
+* The ceph-deploy tool is now the preferred method of provisioning
+  new clusters.  For existing clusters created via mkcephfs that
+  would like to transition to the new tool, there is a migration
+  path, documented at `Transitioning to ceph-deploy`_.
+
+.. _Transitioning to ceph-deploy: ../rados/deployment/ceph-deploy-transition
+
+* The sysvinit script (/etc/init.d/ceph) will now verify (and, if
+  necessary, update) the OSD's position in the CRUSH map on startup.
+  (The upstart script has always worked this way.) By default, this
+  ensures that the OSD is under a 'host' with a name that matches the
+  hostname (``hostname -s``).  Legacy clusters create with mkcephfs do
+  this by default, so this should not cause any problems, but legacy
+  clusters with customized CRUSH maps with an alternate structure
+  should set ``osd crush update on start = false``.
+
+* radosgw-admin now uses the term zone instead of cluster to describe
+  each instance of the radosgw data store (and corresponding
+  collection of radosgw daemons).  The usage for the radosgw-admin
+  command and the 'rgw zone root pool' config options have changed
+  accordingly.
+
+* rbd progress indicators now go to standard error instead of standard
+  out.  (You can disable progress with --no-progress.)
+
+* The 'rbd resize ...' command now requires the --allow-shrink option
+  when resizing to a smaller size.  Expanding images to a larger size
+  is unchanged.
+
+* Please review the changes going back to 0.56.4 if you are upgrading
+  all the way from bobtail.
+
+* The old 'ceph stop_cluster' command has been removed.
+
+* The sysvinit script now uses the ceph.conf file on the remote host
+  when starting remote daemons via the '-a' option.  Note that if '-a'
+  is used in conjunction with '-c path', the path must also be present
+  on the remote host (it is not copied to a temporary file, as it was
+  previously).
+
+* The monitor is using a completely new storage strategy and
+  intra-cluster protocol.  This means that cuttlefish and bobtail
+  monitors do not talk to each other.  When you upgrade each one, it
+  will convert its local data store to the new format.  Once you
+  upgrade a majority, the quorum will be formed using the new protocol
+  and the old monitors will be blocked out until they too get
+  upgraded.  For this reason, we recommend not running a mixed-version
+  cluster for very long.
+
+* ceph-mon now requires the creation of its data directory prior to
+  --mkfs, similarly to what happens on ceph-osd.  This directory is no
+  longer automatically created, and custom scripts should be adjusted to
+  reflect just that.
+
+* The monitor now enforces that MDS names be unique.  If you have
+  multiple daemons start with the same id (e.g., ``mds.a``) the
+  second one will implicitly mark the first as failed.  This makes
+  things less confusing and makes a daemon restart faster (we no
+  longer wait for the stopped daemon to time out) but existing
+  multi-mds configurations may need to be adjusted accordingly to give
+  daemons unique names.
+
+* The 'ceph osd pool delete <poolname>' and 'rados rmpool <poolname>'
+  now have safety interlocks with loud warnings that make you confirm
+  pool removal.  Any scripts currently rely on these functions zapping
+  data without confirmation need to be adjusted accordingly.
+
+
+Notable Changes from v0.60
+--------------------------
+
+* rbd: incremental backups
+* rbd: only set STRIPINGV2 feature if striping parameters are incompatible with old versions
+* rbd: require --allow-shrink for resizing images down
+* librbd: many bug fixes
+* rgw: management REST API
+* rgw: fix object corruption on COPY to self
+* rgw: new sysvinit script for rpm-based systems
+* rgw: allow buckets with '_'
+* rgw: CORS support
+* mon: many fixes
+* mon: improved trimming behavior
+* mon: fix data conversion/upgrade problem (from bobtail)
+* mon: ability to tune leveldb
+* mon: config-keys service to store arbitrary data on monitor
+* mon: 'osd crush add|link|unlink|add-bucket ...' commands
+* mon: trigger leveldb compaction on trim
+* osd: per-rados pool quotas (objects, bytes)
+* osd: tool to export, import, and delete PGs from an individual OSD data store
+* osd: notify mon on clean shutdown to avoid IO stall
+* osd: improved detection of corrupted journals
+* osd: ability to tune leveldb
+* osd: improve client request throttling
+* osd, librados: fixes to the LIST_SNAPS operation
+* osd: improvements to scrub error repair
+* osd: better prevention of wedging OSDs with ENOSPC
+* osd: many small fixes
+* mds: fix xattr handling on root inode
+* mds: fixed bugs in journal replay
+* mds: many fixes
+* librados: clean up snapshot constant definitions
+* libcephfs: calls to query CRUSH topology (used by Hadoop)
+* ceph-fuse, libcephfs: misc fixes to mds session management
+* ceph-fuse: disabled cache invalidation (again) due to potential deadlock with kernel
+* sysvinit: try to start all daemons despite early failures
+* ceph-disk: new 'list' command
+* ceph-disk: hotplug fixes for RHEL/CentOS
+* ceph-disk: fix creation of OSD data partitions on >2TB disks
+* osd: fix udev rules for RHEL/CentOS systems
+* fix daemon logging during initial startup
+
+Notable changes from v0.56 "Bobtail"
+------------------------------------
+* always use installed system leveldb (Gary Lowell)
+* auth: ability to require new cephx signatures on messages (still off by default)
+* buffer unit testing (Loic Dachary)
+* ceph tool: some CLI interface cleanups
+* ceph-disk: improve multicluster support, error handling (Sage Weil)
+* ceph-disk: support for dm-crypt (Alexandre Marangone)
+* ceph-disk: support for sysvinit, directories or partitions (not full disks)
+* ceph-disk: fix mkfs args on old distros (Alexandre Marangone)
+* ceph-disk: fix creation of OSD data partitions on >2TB disks
+* ceph-disk: hotplug fixes for RHEL/CentOS
+* ceph-disk: new 'list' command
+* ceph-fuse, libcephfs: misc fixes to mds session management
+* ceph-fuse: disabled cache invalidation (again) due to potential deadlock with kernel
+* ceph-fuse: enable kernel cache invalidation (Sam Lang)
+* ceph-fuse: fix statfs(2) reporting
+* ceph-fuse: session handling cleanup, bug fixes (Sage Weil)
+* crush: ability to create, remove rules via CLI
+* crush: update weights for all instances of an item, not just the first (Sage Weil)
+* fix daemon logging during initial startup
+* fixed log rotation (Gary Lowell)
+* init-ceph, mkcephfs: close a few security holes with -a  (Sage Weil)
+* libcephfs: calls to query CRUSH topology (used by Hadoop)
+* libcephfs: many fixes, cleanups with the Java bindings
+* libcephfs: new topo API requests for Hadoop (Noah Watkins)
+* librados: clean up snapshot constant definitions
+* librados: fix linger bugs (Josh Durgin)
+* librbd: fixed flatten deadlock (Josh Durgin)
+* librbd: fixed some locking issues with flatten (Josh Durgin)
+* librbd: many bug fixes
+* librbd: optionally wait for flush before enabling writeback (Josh Durgin)
+* many many cleanups (Danny Al-Gaaf)
+* mds, ceph-fuse: fix bugs with replayed requests after MDS restart (Sage Weil)
+* mds, ceph-fuse: manage layouts via xattrs
+* mds: allow xattrs on root
+* mds: fast failover between MDSs (enforce unique mds names)
+* mds: fix xattr handling on root inode
+* mds: fixed bugs in journal replay
+* mds: improve session cleanup (Sage Weil)
+* mds: many fixes (Yan Zheng)
+* mds: misc bug fixes with clustered MDSs and failure recovery
+* mds: misc bug fixes with readdir
+* mds: new encoding for all data types (to allow forward/backward compatibility) (Greg Farnum)
+* mds: store and update backpointers/traces on directory, file objects (Sam Lang)
+* mon: 'osd crush add|link|unlink|add-bucket ...' commands
+* mon: ability to tune leveldb
+* mon: approximate recovery, IO workload stats
+* mon: avoid marking entire CRUSH subtrees out (e.g., if an entire rack goes offline)
+* mon: config-keys service to store arbitrary data on monitor
+* mon: easy adjustment of crush tunables via 'ceph osd crush tunables ...'
+* mon: easy creation of crush rules vai 'ceph osd rule ...'
+* mon: fix data conversion/upgrade problem (from bobtail)
+* mon: improved trimming behavior
+* mon: many fixes
+* mon: new 'ceph df [detail]' command
+* mon: new checks for identifying and reporting clock drift
+* mon: rearchitected to utilize single instance of paxos and a key/value store (Joao Luis)
+* mon: safety check for pool deletion
+* mon: shut down safely if disk approaches full (Joao Luis)
+* mon: trigger leveldb compaction on trim
+* msgr: fix comparison of IPv6 addresses (fixes monitor bringup via ceph-deploy, chef)
+* msgr: fixed race in connection reset
+* msgr: optionally tune TCP buffer size to avoid throughput collapse (Jim Schutt)
+* much code cleanup and optimization (Danny Al-Gaaf)
+* osd, librados: ability to list watchers (David Zafman)
+* osd, librados: fixes to the LIST_SNAPS operation
+* osd, librados: new listsnaps command (David Zafman)
+* osd: a few journaling bug fixes
+* osd: ability to tune leveldb
+* osd: add 'noscrub', 'nodeepscrub' osdmap flags (David Zafman)
+* osd: better prevention of wedging OSDs with ENOSPC
+* osd: ceph-filestore-dump tool for debugging
+* osd: connection handling bug fixes
+* osd: deep-scrub omap keys/values
+* osd: default to libaio for the journal (some performance boost)
+* osd: fix hang in 'journal aio = true' mode (Sage Weil)
+* osd: fix pg log trimming (avoids memory bloat on degraded clusters)
+* osd: fix udev rules for RHEL/CentOS systems
+* osd: fixed bug in journal checksums (Sam Just)
+* osd: improved client request throttling
+* osd: improved handling when disk fills up (David Zafman)
+* osd: improved journal corruption detection (Sam Just)
+* osd: improved detection of corrupted journals
+* osd: improvements to scrub error repair
+* osd: make tracking of object snapshot metadata more efficient (Sam Just)
+* osd: many small fixes
+* osd: misc fixes to PG split (Sam Just)
+* osd: move pg info, log into leveldb (== better performance) (David Zafman)
+* osd: notify mon on clean shutdown to avoid IO stall
+* osd: per-rados pool quotas (objects, bytes)
+* osd: refactored watch/notify infrastructure (fixes protocol, removes many bugs) (Sam Just)
+* osd: support for improved hashing of PGs across OSDs via HASHPSPOOL pool flag and feature
+* osd: tool to export, import, and delete PGs from an individual OSD data store
+* osd: trim log more aggressively, avoid appearance of leak memory
+* osd: validate snap collections on startup
+* osd: verify snap collections on startup (Sam Just)
+* radosgw: ACL grants in headers (Caleb Miles)
+* radosgw: ability to listen to fastcgi via a port (Guilhem Lettron)
+* radosgw: fix object copy onto self (Yehuda Sadeh)
+* radosgw: misc fixes
+* rbd-fuse: new tool, package
+* rbd: avoid FIEMAP when importing from file (it can be buggy)
+* rbd: incremental backups
+* rbd: only set STRIPINGV2 feature if striping parameters are incompatible with old versions
+* rbd: require --allow-shrink for resizing images down
+* rbd: udevadm settle on map/unmap to avoid various races (Dan Mick)
+* rbd: wait for udev to settle in strategic places (avoid spurious errors, failures)
+* rgw: CORS support
+* rgw: allow buckets with '_'
+* rgw: fix Content-Length on 32-bit machines (Jan Harkes)
+* rgw: fix log rotation
+* rgw: fix object corruption on COPY to self
+* rgw: fixed >4MB range requests (Jan Harkes)
+* rgw: new sysvinit script for rpm-based systems
+* rpm/deb: do not remove /var/lib/ceph on purge (v0.59 was the only release to do so)
+* sysvinit: try to start all daemons despite early failures
+* upstart: automatically set osd weight based on df (Guilhem Lettron)
+* use less memory for logging by default
+
+
+v0.60
+=====
+
+Upgrading
+---------
+
+* Please note that the recently added librados 'list_snaps' function
+  call is in a state of flux and is changing slightly in v0.61.  You
+  are advised not to make use of it in v0.59 or v0.60.
+
+Notable Changes
+---------------
+
+* osd: make tracking of object snapshot metadata more efficient (Sam Just)
+* osd: misc fixes to PG split (Sam Just)
+* osd: improve journal corruption detection (Sam Just)
+* osd: improve handling when disk fills up (David Zafman)
+* osd: add 'noscrub', 'nodeepscrub' osdmap flags (David Zafman)
+* osd: fix hang in 'journal aio = true' mode (Sage Weil)
+* ceph-disk-prepare: fix mkfs args on old distros (Alexandre Marangone)
+* ceph-disk-activate: improve multicluster support, error handling (Sage Weil)
+* librbd: optionally wait for flush before enabling writeback (Josh Durgin)
+* crush: update weights for all instances of an item, not just the first (Sage Weil)
+* mon: shut down safely if disk approaches full (Joao Luis)
+* rgw: fix Content-Length on 32-bit machines (Jan Harkes)
+* mds: store and update backpointers/traces on directory, file objects (Sam Lang)
+* mds: improve session cleanup (Sage Weil)
+* mds, ceph-fuse: fix bugs with replayed requests after MDS restart (Sage Weil)
+* ceph-fuse: enable kernel cache invalidation (Sam Lang)
+* libcephfs: new topo API requests for Hadoop (Noah Watkins)
+* ceph-fuse: session handling cleanup, bug fixes (Sage Weil)
+* much code cleanup and optimization (Danny Al-Gaaf)
+* use less memory for logging by default
+* upstart: automatically set osd weight based on df (Guilhem Lettron)
+* init-ceph, mkcephfs: close a few security holes with -a  (Sage Weil)
+* rpm/deb: do not remove /var/lib/ceph on purge (v0.59 was the only release to do so)
+
+
+v0.59
+=====
+
+Upgrading
+---------
+
+* The monitor is using a completely new storage strategy and
+  intra-cluster protocol.  This means that v0.59 and pre-v0.59
+  monitors do not talk to each other.  When you upgrade each one, it
+  will convert its local data store to the new format.  Once you
+  upgrade a majority, the quorum will be formed using the new protocol
+  and the old monitors will be blocked out until they too get
+  upgraded.  For this reason, we recommend not running a mixed-version
+  cluster for very long.
+
+* ceph-mon now requires the creation of its data directory prior to
+  --mkfs, similarly to what happens on ceph-osd.  This directory is no
+  longer automatically created, and custom scripts should be adjusted to
+  reflect just that.
+
+
+Notable Changes
+---------------
+
+ * mon: rearchitected to utilize single instance of paxos and a key/value store (Joao Luis)
+ * mon: new 'ceph df [detail]' command
+ * osd: support for improved hashing of PGs across OSDs via HASHPSPOOL pool flag and feature
+ * osd: refactored watch/notify infrastructure (fixes protocol, removes many bugs) (Sam Just)
+ * osd, librados: ability to list watchers (David Zafman)
+ * osd, librados: new listsnaps command (David Zafman)
+ * osd: trim log more aggressively, avoid appearance of leak memory
+ * osd: misc split fixes
+ * osd: a few journaling bug fixes
+ * osd: connection handling bug fixes
+ * rbd: avoid FIEMAP when importing from file (it can be buggy)
+ * librados: fix linger bugs (Josh Durgin)
+ * librbd: fixed flatten deadlock (Josh Durgin)
+ * rgw: fixed >4MB range requests (Jan Harkes)
+ * rgw: fix log rotation
+ * mds: allow xattrs on root
+ * ceph-fuse: fix statfs(2) reporting
+ * msgr: optionally tune TCP buffer size to avoid throughput collapse (Jim Schutt)
+ * consume less memory for logging by default
+ * always use system leveldb (Gary Lowell)
+
+
+
+v0.58
+=====
+
+Upgrading
+---------
+
+* The monitor now enforces that MDS names be unique.  If you have
+  multiple daemons start with the same id (e.g., ``mds.a``) the
+  second one will implicitly mark the first as failed.  This makes
+  things less confusing and makes a daemon restart faster (we no
+  longer wait for the stopped daemon to time out) but existing
+  multi-mds configurations may need to be adjusted accordingly to give
+  daemons unique names.
+
+Notable Changes
+---------------
+
+ * librbd: fixed some locking issues with flatten (Josh Durgin)
+ * rbd: udevadm settle on map/unmap to avoid various races (Dan Mick)
+ * osd: move pg info, log into leveldb (== better performance) (David Zafman)
+ * osd: fix pg log trimming (avoids memory bloat on degraded clusters)
+ * osd: fixed bug in journal checksums (Sam Just)
+ * osd: verify snap collections on startup (Sam Just)
+ * ceph-disk-prepare/activate: support for dm-crypt (Alexandre Marangone)
+ * ceph-disk-prepare/activate: support for sysvinit, directories or partitions (not full disks)
+ * msgr: fixed race in connection reset
+ * msgr: fix comparison of IPv6 addresses (fixes monitor bringup via ceph-deploy, chef)
+ * radosgw: fix object copy onto self (Yehuda Sadeh)
+ * radosgw: ACL grants in headers (Caleb Miles)
+ * radosgw: ability to listen to fastcgi via a port (Guilhem Lettron)
+ * mds: new encoding for all data types (to allow forward/backward compatibility) (Greg Farnum)
+ * mds: fast failover between MDSs (enforce unique mds names)
+ * crush: ability to create, remove rules via CLI
+ * many many cleanups (Danny Al-Gaaf)
+ * buffer unit testing (Loic Dachary)
+ * fixed log rotation (Gary Lowell)
+
+v0.57
+=====
+
+This development release has a lot of additional functionality
+accumulated over the last couple months.  Most of the bug fixes (with
+the notable exception of the MDS related work) has already been
+backported to v0.56.x, and is not mentioned here.
+
+Upgrading
+---------
+
+* The 'ceph osd pool delete <poolname>' and 'rados rmpool <poolname>'
+  now have safety interlocks with loud warnings that make you confirm
+  pool removal.  Any scripts currently rely on these functions zapping
+  data without confirmation need to be adjusted accordingly.
+
+Notable Changes
+---------------
+
+* osd: default to libaio for the journal (some performance boost)
+* osd: validate snap collections on startup
+* osd: ceph-filestore-dump tool for debugging
+* osd: deep-scrub omap keys/values
+* ceph tool: some CLI interface cleanups
+* mon: easy adjustment of crush tunables via 'ceph osd crush tunables ...'
+* mon: easy creation of crush rules vai 'ceph osd rule ...'
+* mon: approximate recovery, IO workload stats
+* mon: avoid marking entire CRUSH subtrees out (e.g., if an entire rack goes offline)
+* mon: safety check for pool deletion
+* mon: new checks for identifying and reporting clock drift
+* radosgw: misc fixes
+* rbd: wait for udev to settle in strategic places (avoid spurious errors, failures)
+* rbd-fuse: new tool, package
+* mds, ceph-fuse: manage layouts via xattrs
+* mds: misc bug fixes with clustered MDSs and failure recovery
+* mds: misc bug fixes with readdir
+* libcephfs: many fixes, cleanups with the Java bindings
+* auth: ability to require new cephx signatures on messages (still off by default)
diff --git a/doc/releases/dumpling.rst b/doc/releases/dumpling.rst
new file mode 100644
index 000000000..b9946546e
--- /dev/null
+++ b/doc/releases/dumpling.rst
@@ -0,0 +1,947 @@
+========
+Dumpling
+========
+
+Dumpling is the 4th stable release of Ceph.  It is named after the
+dumpling squid (Euprymna tasmanica).
+
+v0.67.12 "Dumpling" (draft)
+===========================
+
+This stable update for Dumpling fixes a few longstanding issues with
+backfill in the OSD that can lead to stalled IOs.  There is also a fix
+for memory utilization for reads in librbd when caching is enabled,
+and then several other small fixes across the rest of the system.
+
+Dumpling users who have encountered IO stalls during backfill and who
+do not expect to upgrade to Firefly soon should upgrade.  Everyone
+else should upgrade to Firefly already.  This is likely to be the last stable
+release for the 0.67.x Dumpling series.
+
+
+Notable Changes
+---------------
+
+* buffer: fix buffer rebuild alignment corner case (#6614 #6003 Loic Dachary, Samuel Just)
+* ceph-disk: reprobe partitions after zap (#9665 #9721 Loic Dachary)
+* ceph-disk: use partx instead of partprobe when appropriate (Loic Dachary)
+* common: add $cctid meta variable (#6228 Adam Crume)
+* crush: fix get_full_location_ordered (Sage Weil)
+* crush: pick ruleset id that matches rule_id (#9675 Xiaoxi Chen)
+* libcephfs: fix tid wrap bug (#9869 Greg Farnum)
+* libcephfs: get osd location on -1 should return EINVAL (Sage Weil)
+* librados: fix race condition with C API and op timeouts (#9582 Sage Weil)
+* librbd: constrain max number of in-flight read requests (#9854 Jason Dillaman)
+* librbd: enforce cache size on read requests (Jason Dillaman)
+* librbd: fix invalid close in image open failure path (#10030 Jason Dillaman)
+* librbd: fix read hang on sparse files (Jason Dillaman)
+* librbd: gracefully handle deleted/renamed pools (#10270 #10122 Jason Dillaman)
+* librbd: protect list_children from invalid child pool ioctxs (#10123 Jason Dillaman)
+* mds: fix ctime updates from clients without dirty caps (#9514 Greg Farnum)
+* mds: fix rare NULL dereference in cap update path (Greg Farnum)
+* mds: fix assertion caused by system clock backwards (#11053 Yan, Zheng)
+* mds: store backtrace on straydir (Yan, Zheng)
+* osd: fix journal committed_thru update after replay (#6756 Samuel Just)
+* osd: fix memory leak, busy loop on snap trim (#9113 Samuel Just)
+* osd: fix misc peering, recovery bugs (#10168 Samuel Just)
+* osd: fix purged_snap field on backfill start (#9487 Sage Weil, Samuel Just)
+* osd: handle no-op write with snapshot corner case (#10262 Sage Weil, Loic Dachary)
+* osd: respect RWORDERED rados flag (Sage Weil)
+* osd: several backfill fixes and refactors (Samuel Just, David Zafman)
+* rgw: send http status reason explicitly in fastcgi (Yehuda Sadeh)
+
+v0.67.11 "Dumpling"
+===================
+
+This stable update for Dumpling fixes several important bugs that
+affect a small set of users.
+
+We recommend that all Dumpling users upgrade at their convenience.  If
+none of these issues are affecting your deployment there is no
+urgency.
+
+
+Notable Changes
+---------------
+
+* common: fix sending dup cluster log items (#9080 Sage Weil)
+* doc: several doc updates (Alfredo Deza)
+* libcephfs-java: fix build against older JNI headesr (Greg Farnum)
+* librados: fix crash in op timeout path (#9362 Matthias Kiefer, Sage Weil)
+* librbd: fix crash using clone of flattened image (#8845 Josh Durgin)
+* librbd: fix error path cleanup when failing to open image (#8912 Josh Durgin)
+* mon: fix crash when adjusting pg_num before any OSDs are added (#9052 Sage Weil)
+* mon: reduce log noise from paxos (Aanchal Agrawal, Sage Weil)
+* osd: allow scrub and snap trim thread pool IO priority to be adjusted (Sage Weil)
+* osd: fix mount/remount sync race (#9144 Sage Weil)
+
+
+v0.67.10 "Dumpling"
+===================
+
+This stable update release for Dumpling includes primarily fixes for
+RGW, including several issues with bucket listings and a potential
+data corruption problem when multiple multi-part uploads race.  There is also
+some throttling capability added in the OSD for scrub that can mitigate the
+performance impact on production clusters.
+
+We recommend that all Dumpling users upgrade at their convenience.
+
+Notable Changes
+---------------
+
+* ceph-disk: partprobe befoere settle, fixing dm-crypt (#6966, Eric Eastman)
+* librbd: add invalidate cache interface (Josh Durgin)
+* librbd: close image if remove_child fails (Ilya Dryomov)
+* librbd: fix potential null pointer dereference (Danny Al-Gaaf)
+* librbd: improve writeback checks, performance (Haomai Wang)
+* librbd: skip zeroes when copying image (#6257, Josh Durgin)
+* mon: fix rule(set) check on 'ceph pool set ... crush_ruleset ...' (#8599, John Spray)
+* mon: shut down if mon is removed from cluster (#6789, Joao Eduardo Luis)
+* osd: fix filestore perf reports to mon (Sage Weil)
+* osd: force any new or updated xattr into leveldb if E2BIG from XFS (#7779, Sage Weil)
+* osd: lock snapdir object during write to fix race with backfill (Samuel Just)
+* osd: option sleep during scrub (Sage Weil)
+* osd: set io priority on scrub and snap trim threads (Sage Weil)
+* osd: 'status' admin socket command (Sage Weil)
+* rbd: tolerate missing NULL terminator on block_name_prefix (#7577, Dan Mick)
+* rgw: calculate user manifest (#8169, Yehuda Sadeh)
+* rgw: fix abort on chunk read error, avoid using extra memory (#8289, Yehuda Sadeh)
+* rgw: fix buffer overflow on bucket instance id (#8608, Yehuda Sadeh)
+* rgw: fix crash in swift CORS preflight request (#8586, Yehuda Sadeh)
+* rgw: fix implicit removal of old objects on object creation (#8972, Patrycja Szablowska, Yehuda Sadeh)
+* rgw: fix MaxKeys in bucket listing (Yehuda Sadeh)
+* rgw: fix race with multiple updates to a single multipart object (#8269, Yehuda Sadeh)
+* rgw: improve bucket listing with delimiter (Yehuda Sadeh)
+* rgw: include NextMarker in bucket listing (#8858, Yehuda Sadeh)
+* rgw: return error early on non-existent bucket (#7064, Yehuda Sadeh)
+* rgw: set truncation flag correctly in bucket listing (Yehuda Sadeh)
+* sysvinit: continue starting daemons after pre-mount error (#8554, Sage Weil)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.10.txt>`.
+
+
+v0.67.9 "Dumpling"
+==================
+
+This Dumpling point release fixes several minor bugs. The most
+prevalent in the field is one that occasionally prevents OSDs from
+starting on recently created clusters.
+
+We recommend that all Dumpling users upgrade at their convenience.
+
+Notable Changes
+---------------
+
+* ceph-fuse, libcephfs: client admin socket command to kick and inspect MDS sessions (#8021, Zheng Yan)
+* monclient: fix failure detection during mon handshake (#8278, Sage Weil)
+* mon: set tid on no-op PGStatsAck messages (#8280, Sage Weil)
+* msgr: fix a rare bug with connection negotiation between OSDs (Guang Yang)
+* osd: allow snap trim throttling with simple delay (#6278, Sage Weil)
+* osd: check for splitting when processing recover/backfill reservations (#6565, Samuel Just)
+* osd: fix backfill position tracking (#8162, Samuel Just)
+* osd: fix bug in backfill stats (Samuel Just)
+* osd: fix bug preventing OSD startup for infant clusters (#8162, Greg Farnum)
+* osd: fix rare PG resurrection race causing an incomplete PG (#7740, Samuel Just)
+* osd: only complete replicas count toward min_size (#7805, Samuel Just)
+* rgw: allow setting ACLs with empty owner (#6892, Yehuda Sadeh)
+* rgw: send user manifest header field (#8170, Yehuda Sadeh)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.9.txt>`.
+
+
+v0.67.8 "Dumpling"
+==================
+
+This Dumpling point release fixes several non-critical issues since
+v0.67.7.  The most notable bug fixes are an auth fix in librbd
+(observed as an occasional crash from KVM), an improvement in the
+network failure detection with the monitor, and several hard to hit
+OSD crashes or hangs.
+
+We recommend that all users upgrade at their convenience.
+
+Upgrading
+---------
+
+* The 'rbd ls' function now returns success and returns an empty when a pool
+  does not store any rbd images.  Previously it would return an ENOENT error.
+
+* Ceph will now issue a health warning if the 'mon osd down out
+  interval' config option is set to zero.  This warning can be
+  disabled by adding 'mon warn on osd down out interval zero = false'
+  to ceph.conf.
+
+Notable Changes
+---------------
+
+* all: improve keepalive detection of failed monitor connections (#7888, Sage Weil)
+* ceph-fuse, libcephfs: pin inodes during readahead, fixing rare crash (#7867, Sage Weil)
+* librbd: make cache writeback a bit less aggressive (Sage Weil)
+* librbd: make symlink for qemu to detect librbd in RPM (#7293, Josh Durgin)
+* mon: allow 'hashpspool' pool flag to be set and unset (Loic Dachary)
+* mon: commit paxos state only after entire quorum acks, fixing rare race where prior round state is readable (#7736, Sage Weil)
+* mon: make elections and timeouts a bit more robust (#7212, Sage Weil)
+* mon: prevent extreme pool split operations (Greg Farnum)
+* mon: wait for quorum for get_version requests to close rare pool creation race (#7997, Sage Weil)
+* mon: warn on 'mon osd down out interval = 0' (#7784, Joao Luis)
+* msgr: fix byte-order for auth challenge, fixing auth errors on big-endian clients (#7977, Dan Mick)
+* msgr: fix occasional crash in authentication code (usually triggered by librbd) (#6840, Josh Durgin)
+* msgr: fix rebind() race (#6992, Xihui He)
+* osd: avoid timeouts during slow PG deletion (#6528, Samuel Just)
+* osd: fix bug in pool listing during recovery (#6633, Samuel Just)
+* osd: fix queue limits, fixing recovery stalls (#7706, Samuel Just)
+* osd: fix rare peering crashes (#6722, #6910, Samuel Just)
+* osd: fix rare recovery hang (#6681, Samuel Just)
+* osd: improve error handling on journal errors (#7738, Sage Weil)
+* osd: reduce load on the monitor from OSDMap subscriptions (Greg Farnum)
+* osd: rery GetLog on peer osd startup, fixing some rare peering stalls (#6909, Samuel Just)
+* osd: reset journal state on remount to fix occasional crash on OSD startup (#8019, Sage Weil)
+* osd: share maps with peers more aggressively (Greg Farnum)
+* rbd: make it harder to delete an rbd image that is currently in use (#7076, Ilya Drymov)
+* rgw: deny writes to secondary zone by non-system users (#6678, Yehuda Sadeh)
+* rgw: do'nt log system requests in usage log (#6889, Yehuda Sadeh)
+* rgw: fix bucket recreation (#6951, Yehuda Sadeh)
+* rgw: fix Swift range response (#7099, Julien Calvet, Yehuda Sadeh)
+* rgw: fix URL escaping (#8202, Yehuda Sadeh)
+* rgw: fix whitespace trimming in http headers (#7543, Yehuda Sadeh)
+* rgw: make multi-object deletion idempotent (#7346, Yehuda Sadeh)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.8.txt>`.
+
+v0.67.7 "Dumpling"
+==================
+
+This Dumpling point release fixes a few critical issues in v0.67.6.
+
+All v0.67.6 users are urgently encouraged to upgrade.  We also
+recommend that all v0.67.5 (or older) users upgrade.
+
+Upgrading
+---------
+
+* Once you have upgraded a radosgw instance or OSD to v0.67.7, you should not
+  downgrade to a previous version.
+
+Notable Changes
+---------------
+
+* ceph-disk: additional unit tests
+* librbd: revert caching behavior change in v0.67.6
+* osd: fix problem reading xattrs due to incomplete backport in v0.67.6
+* radosgw-admin: fix reading object policy
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.7.txt>`.
+
+
+v0.67.6 "Dumpling"
+==================
+
+.. note: This release contains a librbd bug that is fixed in v0.67.7.  Please upgrade to v0.67.7 and do not use v0.67.6.
+
+This Dumpling point release contains a number of important fixed for
+the OSD, monitor, and radosgw.  Most significantly, a change that
+forces large object attributes to spill over into leveldb has been
+backported that can prevent objects and the cluster from being damaged
+by large attributes (which can be induced via the radosgw).  There is
+also a set of fixes that improves data safety and RADOS semantics when
+the cluster becomes full and then non-full.
+
+We recommend that all 0.67.x Dumpling users skip this release and upgrade to v0.67.7.
+
+Upgrading
+---------
+
+* The OSD has long contained a feature that allows large xattrs to
+  spill over into the leveldb backing store in situations where not
+  all local file systems are able to store them reliably.  This option
+  is now enabled unconditionally in order to avoid rare cases where
+  storing large xattrs renders the object unreadable. This is known to
+  be triggered by very large multipart objects, but could be caused by
+  other workloads as well.  Although there is some small risk that
+  performance for certain workloads will degrade, it is more important
+  that data be retrievable.  Note that newer versions of Ceph (e.g.,
+  firefly) do some additional work to avoid the potential performance
+  regression in this case, but that is current considered too complex
+  for backport to the Dumpling stable series.
+
+* It is very dangerous to downgrade from v0.67.6 to a prior version of
+  Dumpling.  If the old version does not have 'filestore xattr use
+  omap = true' it may not be able to read all xattrs for an object and
+  can cause undefined behavior.
+
+Notable changes
+---------------
+
+* ceph-disk: misc bug fixes, particularly on RHEL (Loic Dachary, Alfredo Deza, various)
+* ceph-fuse, libcephfs: fix crash from read over certain sparseness patterns (Sage Weil)
+* ceph-fuse, libcephfs: fix integer overflow for sync reads racing with appends (Sage Weil)
+* ceph.spec: fix udev rule when building RPM under RHEL (Derek Yarnell)
+* common: fix crash from bad format from admin socket (Loic Dachary)
+* librados: add optional timeouts (Josh Durgin)
+* librados: do not leak budget when resending localized or redirected ops (Josh Durgin)
+* librados, osd: fix and improve full cluster handling (Josh Durgin)
+* librbd: fix use-after-free when updating perfcounters during image close (Josh Durgin)
+* librbd: remove limit on objects in cache (Josh Durgin)
+* mon: avoid on-disk full OSDMap corruption from pg_temp removal (Sage Weil)
+* mon: avoid stray pg_temp entries from pool deletion race (Joao Eduardo Luis)
+* mon: do not generate spurious MDSMaps from laggy daemons (Joao Eduardo Luis)
+* mon: fix error code from 'osd rm|down|out|in ...' commands (Loic Dachary)
+* mon: include all health items in summary output (John Spray)
+* osd: fix occasional race/crash during startup (Sage Weil)
+* osd: ignore stray OSDMap messages during init (Sage Weil)
+* osd: unconditionally let xattrs overflow into leveldb (David Zafman)
+* rados: fix a few error checks for the CLI (Josh Durgin)
+* rgw: convert legacy bucket info objects on demand (Yehuda Sadeh)
+* rgw: fix bug causing system users to lose privileges (Yehuda Sadeh)
+* rgw: fix CORS bugs related to headers and case sensitivity (Robin H. Johnson)
+* rgw: fix multipart object listing (Yehuda Sadeh)
+* rgw: fix racing object creations (Yehuda Sadeh)
+* rgw: fix racing object put and delete (Yehuda Sadeh)
+* rgw: fix S3 auth when using response-* query string params (Sylvain Munaut)
+* rgw: use correct secret key for POST authentication (Robin H. Johnson)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.6.txt>`.
+
+
+v0.67.5 "Dumpling"
+==================
+
+This release includes a few critical bug fixes for the radosgw,
+including a fix for hanging operations on large objects.  There are also
+several bug fixes for radosgw multi-site replications, and a few
+backported features.  Also, notably, the 'osd perf' command (which dumps
+recent performance information about active OSDs) has been backported.
+
+We recommend that all 0.67.x Dumpling users upgrade.
+
+Notable changes
+---------------
+
+* ceph-fuse: fix crash in caching code
+* mds: fix looping in populate_mydir()
+* mds: fix standby-replay race
+* mon: accept 'osd pool set ...' as string
+* mon: backport: 'osd perf' command to dump recent OSD performance stats
+* osd: add feature compat check for upcoming object sharding
+* osd: fix osd bench block size argument
+* rbd.py: increase parent name size limit
+* rgw: backport: allow wildcard in supported keystone roles
+* rgw: backport: improve swift COPY behavior
+* rgw: backport: log and open admin socket by default
+* rgw: backport: validate S3 tokens against keystone
+* rgw: fix bucket removal
+* rgw: fix client error code for chunked PUT failure
+* rgw: fix hang on large object GET
+* rgw: fix rare use-after-free
+* rgw: various DR bug fixes
+* sysvinit, upstart: prevent starting daemons using both init systems
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.5.txt>`.
+
+
+v0.67.4 "Dumpling"
+==================
+
+This point release fixes an important performance issue with radosgw,
+keystone authentication token caching, and CORS.  All users
+(especially those of rgw) are encouraged to upgrade.
+
+Notable changes
+---------------
+
+* crush: fix invalidation of cached names
+* crushtool: do not crash on non-unique bucket ids
+* mds: be more careful when decoding LogEvents
+* mds: fix heap check debugging commands
+* mon: avoid rebuilding old full osdmaps
+* mon: fix 'ceph crush move ...'
+* mon: fix 'ceph osd crush reweight ...'
+* mon: fix writeout of full osdmaps during trim
+* mon: limit size of transactions
+* mon: prevent both unmanaged and pool snaps
+* osd: disable xattr size limit (prevents upload of large rgw objects)
+* osd: fix recovery op throttling
+* osd: fix throttling of log messages for very slow requests
+* rgw: drain pending requests before completing write
+* rgw: fix CORS
+* rgw: fix inefficient list::size() usage
+* rgw: fix keystone token expiration
+* rgw: fix minor memory leaks
+* rgw: fix null termination of buffer
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.4.txt>`.
+
+
+v0.67.3 "Dumpling"
+==================
+
+This point release fixes a few important performance regressions with
+the OSD (both with CPU and disk utilization), as well as several other
+important but less common problems.  We recommend that all production users
+upgrade.
+
+Notable Changes
+---------------
+
+* ceph-disk: partprobe after creation journal partition
+* ceph-disk: specify fs type when mounting
+* ceph-post-file: new utility to help share logs and other files with ceph developers
+* libcephfs: fix truncate vs readahead race (crash)
+* mds: fix flock/fcntl lock deadlock
+* mds: fix rejoin loop when encountering pre-dumpling backpointers
+* mon: allow name and addr discovery during election stage
+* mon: always refresh after Paxos store_state (fixes recovery corner case)
+* mon: fix off-by-4x bug with osd byte counts
+* osd: add and disable 'pg log keys debug' by default
+* osd: add option to disable throttling
+* osd: avoid leveldb iterators for pg log append and trim
+* osd: fix readdir_r invocations
+* osd: use fdatasync instead of sync
+* radosgw: fix sysvinit script return status
+* rbd: relicense as LGPL2
+* rgw: flush pending data on multipart upload
+* rgw: recheck object name during S3 POST
+* rgw: reorder init/startup
+* rpm: fix debuginfo package build
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.3.txt>`.
+
+
+v0.67.2 "Dumpling"
+==================
+
+This is an important point release for Dumpling.  Most notably, it
+fixes a problem when upgrading directly from v0.56.x Bobtail to
+v0.67.x Dumpling (without stopping at v0.61.x Cuttlefish along the
+way).  It also fixes a problem with the CLI parsing of the CEPH_ARGS
+environment variable, high CPU utilization by the ceph-osd daemons,
+and cleans up the radosgw shutdown sequence.
+
+Notable Changes
+---------------
+
+* objecter: resend linger requests when cluster goes from full to non-full
+* ceph: parse CEPH_ARGS environment variable
+* librados: fix small memory leak
+* osd: remove old log objects on upgrade (fixes bobtail -> dumpling jump)
+* osd: disable PGLog::check() via config option (fixes CPU burn)
+* rgw: drain requests on shutdown
+* rgw: misc memory leaks on shutdown
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.2.txt>`.
+
+
+v0.67.1 "Dumpling"
+==================
+
+This is a minor point release for Dumpling that fixes problems with
+OpenStack and librbd hangs when caching is disabled.
+
+Notable changes
+---------------
+
+* librados, librbd: fix constructor for python bindings with certain
+  usages (in particular, that used by OpenStack)
+* librados, librbd: fix aio_flush wakeup when cache is disabled
+* librados: fix locking for aio completion refcounting
+* fixes 'ceph --admin-daemon ...' command error code on error
+* fixes 'ceph daemon ... config set ...' command for boolean config
+  options.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.67.1.txt>`.
+
+v0.67 "Dumpling"
+================
+
+This is the fourth major release of Ceph, code-named "Dumpling."  The
+headline features for this release include:
+
+* Multi-site support for radosgw.  This includes the ability to set up
+  separate "regions" in the same or different Ceph clusters that share
+  a single S3/Swift bucket/container namespace.
+
+* RESTful API endpoint for Ceph cluster administration.
+  ceph-rest-api, a wrapper around ceph_rest_api.py, can be used to
+  start up a test single-threaded HTTP server that provides access to
+  cluster information and administration in very similar ways to the
+  ceph commandline tool.  ceph_rest_api.py can be used as a WSGI
+  application for deployment in a more-capable web server.  See
+  ceph-rest-api.8 for more.
+
+* Object namespaces in librados.
+
+
+.. _dumpling-upgrade:
+
+Upgrade Sequencing
+------------------
+
+It is possible to do a rolling upgrade from Cuttlefish to Dumpling.
+
+#. Upgrade ceph-common on all nodes that will use the command line
+   'ceph' utility.
+#. Upgrade all monitors (upgrade ceph package, restart ceph-mon
+   daemons).  This can happen one daemon or host at a time.  Note that
+   because cuttlefish and dumpling monitors can't talk to each other,
+   all monitors should be upgraded in relatively short succession to
+   minimize the risk that an a untimely failure will reduce
+   availability.
+#. Upgrade all osds (upgrade ceph package, restart ceph-osd daemons).
+   This can happen one daemon or host at a time.
+#. Upgrade radosgw (upgrade radosgw package, restart radosgw daemons).
+
+
+Upgrading from v0.66
+--------------------
+
+* There is monitor internal protocol change, which means that v0.67
+  ceph-mon daemons cannot talk to v0.66 or older daemons.  We
+  recommend upgrading all monitors at once (or in relatively quick
+  succession) to minimize the possibility of downtime.
+
+* The output of 'ceph status --format=json' or 'ceph -s --format=json'
+  has changed to return status information in a more structured and
+  usable format.
+
+* The 'ceph pg dump_stuck [threshold]' command used to require a
+  --threshold or -t prefix to the threshold argument, but now does
+  not.
+
+* Many more ceph commands now output formatted information; select
+  with '--format=<format>', where <format> can be 'json', 'json-pretty',
+  'xml', or 'xml-pretty'.
+
+* The 'ceph pg <pgid> ...' commands (like 'ceph pg <pgid> query') are
+  deprecated in favor of 'ceph tell <pgid> ...'.  This makes the
+  distinction between 'ceph pg <command> <pgid>' and 'ceph pg <pgid>
+  <command>' less awkward by making it clearer that the 'tell'
+  commands are talking to the OSD serving the placement group, not the
+  monitor.
+
+* The 'ceph --admin-daemon <path> <command ...>' used to accept the
+  command and arguments as either a single string or as separate
+  arguments.  It will now only accept the command spread across
+  multiple arguments.  This means that any script which does something
+  like::
+
+    ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok 'config set debug_ms 1'
+
+  needs to remove the quotes.  Also, note that the above can now be
+  shortened to::
+
+    ceph daemon osd.0 config set debug_ms 1
+
+* The radosgw caps were inconsistently documented to be either 'mon =
+  allow r' or 'mon = allow rw'.  The 'mon = allow rw' is required for
+  radosgw to create its own pools.  All documentation has been updated
+  accordingly.
+
+* The radosgw copy object operation may return extra progress info
+  during the operation. At this point it will only happen when doing
+  cross zone copy operations. The S3 response will now return extra
+  <Progress> field under the <CopyResult> container. The Swift
+  response will now send the progress as a json array.
+
+* In v0.66 and v0.65 the HASHPSPOOL pool flag was enabled by default
+  for new pools, but has been disabled again until Linux kernel client
+  support reaches more distributions and users.
+
+* ceph-osd now requires a max file descriptor limit (e.g., ``ulimit -n
+  ...``) of at least
+  filestore_wbthrottle_(xfs|btrfs)_inodes_hard_limit (5000 by default)
+  in order to accommodate the new write back throttle system.  On
+  Ubuntu, upstart now sets the fd limit to 32k.  On other platforms,
+  the sysvinit script will set it to 32k by default (still
+  overridable via max_open_files).  If this field has been customized
+  in ceph.conf it should likely be adjusted upwards.
+
+Upgrading from v0.61 "Cuttlefish"
+---------------------------------
+
+In addition to the above notes about upgrading from v0.66:
+
+* There has been a huge revamp of the 'ceph' command-line interface
+  implementation.  The ``ceph-common`` client library needs to be
+  upgrade before ``ceph-mon`` is restarted in order to avoid problems
+  using the CLI (the old ``ceph`` client utility cannot talk to the
+  new ``ceph-mon``).
+
+* The CLI is now very careful about sending the 'status' one-liner
+  output to stderr and command output to stdout.  Scripts relying on
+  output should take care.
+
+* The 'ceph osd tell ...' and 'ceph mon tell ...' commands are no
+  longer supported.  Any callers should use::
+
+   ceph tell osd.<id or *> ...
+   ceph tell mon.<id or name or *> ...
+
+  The 'ceph mds tell ...' command is still there, but will soon also
+  transition to 'ceph tell mds.<id or name or \*> ...'
+
+* The 'ceph osd crush add ...' command used to take one of two forms::
+
+    ceph osd crush add 123 osd.123 <weight> <location ...>
+    ceph osd crush add osd.123 <weight> <location ...>
+
+  This is because the id and crush name are redundant.  Now only the
+  simple form is supported, where the osd name/id can either be a bare
+  id (integer) or name (osd.<id>)::
+
+    ceph osd crush add osd.123 <weight> <location ...>
+    ceph osd crush add 123 <weight> <location ...>
+
+* There is now a maximum RADOS object size, configurable via 'osd max
+  object size', defaulting to 100 GB.  Note that this has no effect on
+  RBD, CephFS, or radosgw, which all stripe over objects. If you are
+  using librados and storing objects larger than that, you will need
+  to adjust 'osd max object size', and should consider using smaller
+  objects instead.
+
+* The 'osd min down {reporters|reports}' config options have been
+  renamed to 'mon osd min down {reporters|reports}', and the
+  documentation has been updated to reflect that these options apply
+  to the monitors (who process failure reports) and not OSDs.  If you
+  have adjusted these settings, please update your ``ceph.conf``
+  accordingly.
+
+
+Notable changes since v0.66
+---------------------------
+
+* mon: sync improvements (performance and robustness)
+* mon: many bug fixes (paxos and services)
+* mon: fixed bugs in recovery and io rate reporting (negative/large values)
+* mon: collect metadata on osd performance
+* mon: generate health warnings from slow or stuck requests
+* mon: expanded --format=<json|xml|...> support for monitor commands
+* mon: scrub function for verifying data integrity
+* mon, osd: fix old osdmap trimming logic
+* mon: enable leveldb caching by default
+* mon: more efficient storage of PG metadata
+* ceph-rest-api: RESTful endpoint for administer cluster (mirrors CLI)
+* rgw: multi-region support
+* rgw: infrastructure to support georeplication of bucket and user metadata
+* rgw: infrastructure to support georeplication of bucket data
+* rgw: COPY object support between regions
+* rbd: /etc/ceph/rbdmap file for mapping rbd images on startup
+* osd: many bug fixes
+* osd: limit number of incremental osdmaps sent to peers (could cause osds to be wrongly marked down)
+* osd: more efficient small object recovery
+* osd, librados: support for object namespaces
+* osd: automatically enable xattrs on leveldb as necessary
+* mds: fix bug in LOOKUPINO (used by nfs reexport)
+* mds: fix O_TRUNC locking
+* msgr: fixed race condition in inter-osd network communication
+* msgr: fixed various memory leaks related to network sessions
+* ceph-disk: fixes for unusual device names, partition detection
+* hypertable: fixes for hypertable CephBroker bindings
+* use SSE4.2 crc32c instruction if present
+
+
+Notable changes since v0.61 "Cuttlefish"
+----------------------------------------
+
+* add 'config get' admin socket command
+* ceph-conf: --show-config-value now reflects daemon defaults
+* ceph-disk: add '[un]suppress-active DEV' command
+* ceph-disk: avoid mounting over an existing osd in /var/lib/ceph/osd/*
+* ceph-disk: fixes for unusual device names, partition detection
+* ceph-disk: improved handling of odd device names
+* ceph-disk: many fixes for RHEL/CentOS, Fedora, wheezy
+* ceph-disk: simpler, more robust locking
+* ceph-fuse, libcephfs: fix a few caps revocation bugs
+* ceph-fuse, libcephfs: fix read zeroing at EOF
+* ceph-fuse, libcephfs: fix request refcounting bug (hang on shutdown)
+* ceph-fuse, libcephfs: fix truncatation bug on >4MB files (Yan, Zheng)
+* ceph-fuse, libcephfs: fix for cap release/hang
+* ceph-fuse: add ioctl support
+* ceph-fuse: fixed long-standing O_NOATIME vs O_LAZY bug
+* ceph-rest-api: RESTful endpoint for administer cluster (mirrors CLI)
+* ceph, librados: fix resending of commands on mon reconnect
+* daemons: create /var/run/ceph as needed
+* debian wheezy: fix udev rules
+* debian, specfile: packaging cleanups
+* debian: fix upstart behavior with upgrades
+* debian: rgw: stop daemon on uninstall
+* debian: stop daemons on uninstall; fix dependencies
+* hypertable: fixes for hypertable CephBroker bindings
+* librados python binding cleanups
+* librados python: fix xattrs > 4KB (Josh Durgin)
+* librados: configurable max object size (default 100 GB)
+* librados: new calls to administer the cluster
+* librbd: ability to read from local replicas
+* librbd: locking tests (Josh Durgin)
+* librbd: make default options/features for newly created images (e.g., via qemu-img) configurable
+* librbd: parallelize delete, rollback, flatten, copy, resize
+* many many fixes from static code analysis (Danny Al-Gaaf)
+* mds: fix O_TRUNC locking
+* mds: fix bug in LOOKUPINO (used by nfs reexport)
+* mds: fix rare hang after client restart
+* mds: fix several bugs (Yan, Zheng)
+* mds: many backpointer improvements (Yan, Zheng)
+* mds: many fixes for mds clustering
+* mds: misc stability fixes (Yan, Zheng, Greg Farnum)
+* mds: new robust open-by-ino support (Yan, Zheng)
+* mds: support robust lookup by ino number (good for NFS) (Yan, Zheng)
+* mon, ceph: huge revamp of CLI and internal admin API. (Dan Mick)
+* mon, osd: fix old osdmap trimming logic
+* mon, osd: many memory leaks fixed
+* mon: better trim/compaction behavior
+* mon: collect metadata on osd performance
+* mon: enable leveldb caching by default
+* mon: expanded --format=<json|xml|...> support for monitor commands
+* mon: fix election timeout
+* mon: fix leveldb compression, trimming
+* mon: fix start fork behavior
+* mon: fix units in 'ceph df' output
+* mon: fix validation of mds ids from CLI commands
+* mon: fixed bugs in recovery and io rate reporting (negative/large values)
+* mon: generate health warnings from slow or stuck requests
+* mon: many bug fixes (paxos and services, sync)
+* mon: many stability fixes (Joao Luis)
+* mon: more efficient storage of PG metadata
+* mon: new --extract-monmap to aid disaster recovery
+* mon: new capability syntax
+* mon: scrub function for verifying data integrity
+* mon: simplify PaxosService vs Paxos interaction, fix readable/writeable checks
+* mon: sync improvements (performance and robustness)
+* mon: tuning, performance improvements
+* msgr: fix various memory leaks
+* msgr: fixed race condition in inter-osd network communication
+* msgr: fixed various memory leaks related to network sessions
+* osd, librados: support for object namespaces
+* osd, mon: optionally dump leveldb transactions to a log
+* osd: automatically enable xattrs on leveldb as necessary
+* osd: avoid osd flapping from asymmetric network failure
+* osd: break blacklisted client watches (David Zafman)
+* osd: close narrow journal race
+* osd: do not use fadvise(DONTNEED) on XFS (data corruption on power cycle)
+* osd: fix for an op ordering bug
+* osd: fix handling for split after upgrade from bobtail
+* osd: fix incorrect mark-down of osds
+* osd: fix internal heartbeart timeouts when scrubbing very large objects
+* osd: fix memory/network inefficiency during deep scrub
+* osd: fixed problem with front-side heartbeats and mixed clusters (David Zafman)
+* osd: limit number of incremental osdmaps sent to peers (could cause osds to be wrongly marked down)
+* osd: many bug fixes
+* osd: monitor both front and back interfaces
+* osd: more efficient small object recovery
+* osd: new writeback throttling (for less bursty write performance) (Sam Just)
+* osd: pg log (re)writes are now vastly more efficient (faster peering) (Sam Just)
+* osd: ping/heartbeat on public and private interfaces
+* osd: prioritize recovery for degraded PGs
+* osd: re-use partially deleted PG contents when present (Sam Just)
+* osd: recovery and peering performance improvements
+* osd: resurrect partially deleted PGs
+* osd: verify both front and back network are working before rejoining cluster
+* rados: clonedata command for cli
+* radosgw-admin: create keys for new users by default
+* rbd: /etc/ceph/rbdmap file for mapping rbd images on startup
+* rgw: COPY object support between regions
+* rgw: fix CORS bugs
+* rgw: fix locking issue, user operation mask,
+* rgw: fix radosgw-admin buckets list (Yehuda Sadeh)
+* rgw: fix usage log scanning for large, untrimmed logs
+* rgw: handle deep uri resources
+* rgw: infrastructure to support georeplication of bucket and user metadata
+* rgw: infrastructure to support georeplication of bucket data
+* rgw: multi-region support
+* sysvinit: fix enumeration of local daemons
+* sysvinit: fix osd crush weight calculation when using -a
+* sysvinit: handle symlinks in /var/lib/ceph/osd/*
+* use SSE4.2 crc32c instruction if present
+
+
+v0.66
+=====
+
+Upgrading
+---------
+
+* There is now a configurable maximum rados object size, defaulting to 100 GB.  If you
+  are using librados and storing objects larger than that, you will need to adjust
+  'osd max object size', and should consider using smaller objects instead.
+
+Notable changes
+---------------
+
+* osd: pg log (re)writes are now vastly more efficient (faster peering) (Sam Just)
+* osd: fixed problem with front-side heartbeats and mixed clusters (David Zafman)
+* mon: tuning, performance improvements
+* mon: simplify PaxosService vs Paxos interaction, fix readable/writeable checks
+* rgw: fix radosgw-admin buckets list (Yehuda Sadeh)
+* mds: support robust lookup by ino number (good for NFS) (Yan, Zheng)
+* mds: fix several bugs (Yan, Zheng)
+* ceph-fuse, libcephfs: fix truncatation bug on >4MB files (Yan, Zheng)
+* ceph/librados: fix resending of commands on mon reconnect
+* librados python: fix xattrs > 4KB (Josh Durgin)
+* librados: configurable max object size (default 100 GB)
+* msgr: fix various memory leaks
+* ceph-fuse: fixed long-standing O_NOATIME vs O_LAZY bug
+* ceph-fuse, libcephfs: fix request refcounting bug (hang on shutdown)
+* ceph-fuse, libcephfs: fix read zeroing at EOF
+* ceph-conf: --show-config-value now reflects daemon defaults
+* ceph-disk: simpler, more robust locking
+* ceph-disk: avoid mounting over an existing osd in /var/lib/ceph/osd/*
+* sysvinit: handle symlinks in /var/lib/ceph/osd/*
+
+
+v0.65
+=====
+
+Upgrading
+---------
+
+* Huge revamp of the 'ceph' command-line interface implementation.
+  The ``ceph-common`` client library needs to be upgrade before
+  ``ceph-mon`` is restarted in order to avoid problems using the CLI
+  (the old ``ceph`` client utility cannot talk to the new
+  ``ceph-mon``).
+
+* The CLI is now very careful about sending the 'status' one-liner
+  output to stderr and command output to stdout.  Scripts relying on
+  output should take care.
+
+* The 'ceph osd tell ...' and 'ceph mon tell ...' commands are no
+  longer supported.  Any callers should use::
+
+    ceph tell osd.<id or *> ...
+    ceph tell mon.<id or name or *> ...
+
+  The 'ceph mds tell ...' command is still there, but will soon also
+  transition to 'ceph tell mds.<id or name or \*> ...'
+
+* The 'ceph osd crush add ...' command used to take one of two forms::
+
+    ceph osd crush add 123 osd.123 <weight> <location ...>
+    ceph osd crush add osd.123 <weight> <location ...>
+
+  This is because the id and crush name are redundant.  Now only the
+  simple form is supported, where the osd name/id can either be a bare
+  id (integer) or name (osd.<id>)::
+
+    ceph osd crush add osd.123 <weight> <location ...>
+    ceph osd crush add 123 <weight> <location ...>
+
+* There is now a maximum RADOS object size, configurable via 'osd max
+  object size', defaulting to 100 GB.  Note that this has no effect on
+  RBD, CephFS, or radosgw, which all stripe over objects.
+
+
+Notable changes
+---------------
+
+* mon, ceph: huge revamp of CLI and internal admin API. (Dan Mick)
+* mon: new capability syntax
+* osd: do not use fadvise(DONTNEED) on XFS (data corruption on power cycle)
+* osd: recovery and peering performance improvements
+* osd: new writeback throttling (for less bursty write performance) (Sam Just)
+* osd: ping/heartbeat on public and private interfaces
+* osd: avoid osd flapping from asymmetric network failure
+* osd: re-use partially deleted PG contents when present (Sam Just)
+* osd: break blacklisted client watches (David Zafman)
+* mon: many stability fixes (Joao Luis)
+* mon, osd: many memory leaks fixed
+* mds: misc stability fixes (Yan, Zheng, Greg Farnum)
+* mds: many backpointer improvements (Yan, Zheng)
+* mds: new robust open-by-ino support (Yan, Zheng)
+* ceph-fuse, libcephfs: fix a few caps revocation bugs
+* librados: new calls to administer the cluster
+* librbd: locking tests (Josh Durgin)
+* ceph-disk: improved handling of odd device names
+* ceph-disk: many fixes for RHEL/CentOS, Fedora, wheezy
+* many many fixes from static code analysis (Danny Al-Gaaf)
+* daemons: create /var/run/ceph as needed
+
+
+v0.64
+=====
+
+Upgrading
+---------
+
+* New pools now have the HASHPSPOOL flag set by default to provide
+  better distribution over OSDs.  Support for this feature was
+  introduced in v0.59 and Linux kernel version v3.9.  If you wish to
+  access the cluster from an older kernel, set the 'osd pool default
+  flag hashpspool = false' option in your ceph.conf prior to creating
+  the cluster or creating new pools.  Note that the presence of any
+  pool in the cluster with the flag enabled will make the OSD require
+  support from all clients.
+
+Notable changes
+---------------
+
+* osd: monitor both front and back interfaces
+* osd: verify both front and back network are working before rejoining cluster
+* osd: fix memory/network inefficiency during deep scrub
+* osd: fix incorrect mark-down of osds
+* mon: fix start fork behavior
+* mon: fix election timeout
+* mon: better trim/compaction behavior
+* mon: fix units in 'ceph df' output
+* mon, osd: misc memory leaks
+* librbd: make default options/features for newly created images (e.g., via qemu-img) configurable
+* mds: many fixes for mds clustering
+* mds: fix rare hang after client restart
+* ceph-fuse: add ioctl support
+* ceph-fuse/libcephfs: fix for cap release/hang
+* rgw: handle deep uri resources
+* rgw: fix CORS bugs
+* ceph-disk: add '[un]suppress-active DEV' command
+* debian: rgw: stop daemon on uninstall
+* debian: fix upstart behavior with upgrades
+
+
+v0.63
+=====
+
+Upgrading
+---------
+
+* The 'osd min down {reporters|reports}' config options have been
+  renamed to 'mon osd min down {reporters|reports}', and the
+  documentation has been updated to reflect that these options apply
+  to the monitors (who process failure reports) and not OSDs.  If you
+  have adjusted these settings, please update your ``ceph.conf``
+  accordingly.
+
+Notable Changes
+---------------
+
+* librbd: parallelize delete, rollback, flatten, copy, resize
+* librbd: ability to read from local replicas
+* osd: resurrect partially deleted PGs
+* osd: prioritize recovery for degraded PGs
+* osd: fix internal heartbeart timeouts when scrubbing very large objects
+* osd: close narrow journal race
+* rgw: fix usage log scanning for large, untrimmed logs
+* rgw: fix locking issue, user operation mask,
+* initscript: fix osd crush weight calculation when using -a
+* initscript: fix enumeration of local daemons
+* mon: several fixes to paxos, sync
+* mon: new --extract-monmap to aid disaster recovery
+* mon: fix leveldb compression, trimming
+* add 'config get' admin socket command
+* rados: clonedata command for cli
+* debian: stop daemons on uninstall; fix dependencies
+* debian wheezy: fix udev rules
+* many many small fixes from coverity scan
+
+
+v0.62
+=====
+
+Notable Changes
+---------------
+
+* mon: fix validation of mds ids from CLI commands
+* osd: fix for an op ordering bug
+* osd, mon: optionally dump leveldb transactions to a log
+* osd: fix handling for split after upgrade from bobtail
+* debian, specfile: packaging cleanups
+* radosgw-admin: create keys for new users by default
+* librados python binding cleanups
+* misc code cleanups
diff --git a/doc/releases/emperor.rst b/doc/releases/emperor.rst
new file mode 100644
index 000000000..3c0bcb782
--- /dev/null
+++ b/doc/releases/emperor.rst
@@ -0,0 +1,654 @@
+=======
+Emperor
+=======
+
+Emperor is the 5th stable release of Ceph.  It is named after the emperor squid.
+
+
+v0.72.3 Emperor (pending release)
+=================================
+
+Upgrading
+---------
+
+* Monitor 'auth' read-only commands now expect the user to have 'rx' caps.
+  This is the same behavior that was present in dumpling, but in emperor
+  and more recent development releases the 'r' cap was sufficient.  Note that
+  this backported security fix will break mon keys that are using the following
+  commands but do not have the 'x' bit in the mon capability::
+
+    ceph auth export
+    ceph auth get
+    ceph auth get-key
+    ceph auth print-key
+    ceph auth list
+
+
+v0.72.2 Emperor
+===============
+
+This is the second bugfix release for the v0.72.x Emperor series.  We
+have fixed a hang in radosgw, and fixed (again) a problem with monitor
+CLI compatibility with mixed version monitors.  (In the future this
+will no longer be a problem.)
+
+Upgrading
+---------
+
+* The JSON schema for the 'osd pool set ...' command changed slightly.  Please
+  avoid issuing this particular command via the CLI while there is a mix of
+  v0.72.1 and v0.72.2 monitor daemons running.
+
+* As part of fix for #6796, 'ceph osd pool set <pool> <var> <arg>' now
+  receives <arg> as an integer instead of a string.  This affects how
+  'hashpspool' flag is set/unset: instead of 'true' or 'false', it now
+  must be '0' or '1'.
+
+
+Changes
+-------
+
+* mon: 'osd pool set ...' syntax change
+* osd: added test for missing on-disk HEAD object
+* osd: fix osd bench block size argument
+* rgw: fix hang on large object GET
+* rgw: fix rare use-after-free
+* rgw: various DR bug fixes
+* rgw: do not return error on empty owner when setting ACL
+* sysvinit, upstart: prevent starting daemons using both init systems
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.72.2.txt>`.
+
+v0.72.1 Emperor
+===============
+
+Important Note
+--------------
+
+When you are upgrading from Dumpling to Emperor, do not run any of the
+"ceph osd pool set" commands while your monitors are running separate versions.
+Doing so could result in inadvertently changing cluster configuration settings
+that exhaust compute resources in your OSDs.
+
+Changes
+-------
+
+* osd: fix upgrade bug #6761
+* ceph_filestore_tool: introduced tool to repair errors caused by #6761
+
+This release addresses issue #6761.  Upgrading to Emperor can cause
+reads to begin returning ENFILE (too many open files).  v0.72.1 fixes
+that upgrade issue and adds a tool ceph_filestore_tool to repair osd
+stores affected by this bug.
+
+To repair a cluster affected by this bug:
+
+#. Upgrade all osd machines to v0.72.1
+#. Install the ceph-test package on each osd machine to get ceph_filestore_tool
+#. Stop all osd processes
+#. To see all lost objects, run the following on each osd with the osd stopped and
+   the osd data directory mounted::
+
+     ceph_filestore_tool --list-lost-objects=true --filestore-path=<path-to-osd-filestore> --journal-path=<path-to-osd-journal>
+
+#. To fix all lost objects, run the following on each osd with the
+   osd stopped and the osd data directory mounted::
+
+     ceph_filestore_tool --fix-lost-objects=true --list-lost-objects=true --filestore-path=<path-to-osd-filestore> --journal-path=<path-to-osd-journal>
+
+#. Once lost objects have been repaired on each osd, you can restart
+   the cluster.
+
+Note, the ceph_filestore_tool performs a scan of all objects on the
+osd and may take some time.
+
+
+v0.72 Emperor
+=============
+
+This is the fifth major release of Ceph, the fourth since adopting a
+3-month development cycle.  This release brings several new features,
+including multi-datacenter replication for the radosgw, improved
+usability, and lands a lot of incremental performance and internal
+refactoring work to support upcoming features in Firefly.
+
+Important Note
+--------------
+
+When you are upgrading from Dumpling to Emperor, do not run any of the
+"ceph osd pool set" commands while your monitors are running separate versions.
+Doing so could result in inadvertently changing cluster configuration settings
+that exhaust compute resources in your OSDs.
+
+Highlights
+----------
+
+* common: improved crc32c performance
+* librados: new example client and class code
+* mds: many bug fixes and stability improvements
+* mon: health warnings when pool pg_num values are not reasonable
+* mon: per-pool performance stats
+* osd, librados: new object copy primitives
+* osd: improved interaction with backend file system to reduce latency
+* osd: much internal refactoring to support ongoing erasure coding and tiering support
+* rgw: bucket quotas
+* rgw: improved CORS support
+* rgw: performance improvements
+* rgw: validate S3 tokens against Keystone
+
+Coincident with core Ceph, the Emperor release also brings:
+
+* radosgw-agent: support for multi-datacenter replication for disaster recovery
+* tgt: improved support for iSCSI via upstream tgt
+
+Packages for both are available on ceph.com.
+
+Upgrade sequencing
+------------------
+
+There are no specific upgrade restrictions on the order or sequence of
+upgrading from 0.67.x Dumpling. However, you cannot run any of the
+"ceph osd pool set" commands while your monitors are running separate versions.
+Doing so could result in inadvertently changing cluster configuration settings
+and exhausting compute resources in your OSDs.
+
+It is also possible to do a rolling upgrade from 0.61.x Cuttlefish,
+but there are ordering restrictions.  (This is the same set of
+restrictions for Cuttlefish to Dumpling.)
+
+#. Upgrade ceph-common on all nodes that will use the command line 'ceph' utility.
+#. Upgrade all monitors (upgrade ceph package, restart ceph-mon
+   daemons).  This can happen one daemon or host at a time.  Note that
+   because cuttlefish and dumpling monitors can't talk to each other,
+   all monitors should be upgraded in relatively short succession to
+   minimize the risk that an a untimely failure will reduce
+   availability.
+#. Upgrade all osds (upgrade ceph package, restart ceph-osd daemons).
+   This can happen one daemon or host at a time.
+#. Upgrade radosgw (upgrade radosgw package, restart radosgw daemons).
+
+
+Upgrading from v0.71
+--------------------
+
+* ceph-fuse and radosgw now use the same default values for the admin
+  socket and log file paths that the other daemons (ceph-osd,
+  ceph-mon, etc.) do.  If you run these daemons as non-root, you may
+  need to adjust your ceph.conf to disable these options or to adjust
+  the permissions on /var/run/ceph and /var/log/ceph.
+
+Upgrading from v0.67 Dumpling
+-----------------------------
+
+* ceph-fuse and radosgw now use the same default values for the admin
+  socket and log file paths that the other daemons (ceph-osd,
+  ceph-mon, etc.) do.  If you run these daemons as non-root, you may
+  need to adjust your ceph.conf to disable these options or to adjust
+  the permissions on /var/run/ceph and /var/log/ceph.
+
+* The MDS now disallows snapshots by default as they are not
+  considered stable.  The command 'ceph mds set allow_snaps' will
+  enable them.
+
+* For clusters that were created before v0.44 (pre-argonaut, Spring
+  2012) and store radosgw data, the auto-upgrade from TMAP to OMAP
+  objects has been disabled.  Before upgrading, make sure that any
+  buckets created on pre-argonaut releases have been modified (e.g.,
+  by PUTing and then DELETEing an object from each bucket).  Any
+  cluster created with argonaut (v0.48) or a later release or not
+  using radosgw never relied on the automatic conversion and is not
+  affected by this change.
+
+* Any direct users of the 'tmap' portion of the librados API should be
+  aware that the automatic tmap -> omap conversion functionality has
+  been removed.
+
+* Most output that used K or KB (e.g., for kilobyte) now uses a
+  lower-case k to match the official SI convention.  Any scripts that
+  parse output and check for an upper-case K will need to be modified.
+
+* librados::Rados::pool_create_async() and librados::Rados::pool_delete_async()
+  don't drop a reference to the completion object on error, caller needs to take
+  care of that. This has never really worked correctly and we were leaking an
+  object
+
+* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the
+  specified location, as that's a job for 'ceph osd crush add'.  It will
+  however continue to work just the same as long as the osd already exists
+  in the crush map.
+
+* The OSD now enforces that class write methods cannot both mutate an
+  object and return data.  The rbd.assign_bid method, the lone
+  offender, has been removed.  This breaks compatibility with
+  pre-bobtail librbd clients by preventing them from creating new
+  images.
+
+* librados now returns on commit instead of ack for synchronous calls.
+  This is a bit safer in the case where both OSDs and the client crash, and
+  is probably how it should have been acting from the beginning. Users are
+  unlikely to notice but it could result in lower performance in some
+  circumstances. Those who care should switch to using the async interfaces,
+  which let you specify safety semantics precisely.
+
+* The C++ librados AioComplete::get_version() method was incorrectly
+  returning an int (usually 32-bits).  To avoid breaking library
+  compatibility, a get_version64() method is added that returns the
+  full-width value.  The old method is deprecated and will be removed
+  in a future release.  Users of the C++ librados API that make use of
+  the get_version() method should modify their code to avoid getting a
+  value that is truncated from 64 to to 32 bits.
+
+
+Notable Changes since v0.71
+---------------------------
+
+* build: fix [/usr]/sbin locations (Alan Somers)
+* ceph-fuse, radosgw: enable admin socket and logging by default
+* ceph: make -h behave when monitors are down
+* common: cache crc32c values where possible
+* common: fix looping on BSD (Alan Somers)
+* librados, mon: ability to query/ping out-of-quorum monitor status (Joao Luis)
+* librbd python bindings: fix parent image name limit (Josh Durgin)
+* mds: avoid leaking objects when deleting truncated files (Yan, Zheng)
+* mds: fix F_GETLK (Yan, Zheng)
+* mds: fix many bugs with stray (unlinked) inodes (Yan, Zheng)
+* mds: fix many directory fragmentation bugs (Yan, Zheng)
+* mon: allow (un)setting HASHPSPOOL flag on existing pools (Joao Luis)
+* mon: make 'osd pool rename' idempotent (Joao Luis)
+* osd: COPY_GET on-wire encoding improvements (Greg Farnum)
+* osd: bloom_filter encodability, fixes, cleanups (Loic Dachary, Sage Weil)
+* osd: fix handling of racing read vs write (Samuel Just)
+* osd: reduce blocking on backing fs (Samuel Just)
+* radosgw-agent: multi-region replication/DR
+* rgw: fix/improve swift COPY support (Yehuda Sadeh)
+* rgw: misc fixes to support DR (Josh Durgin, Yehuda Sadeh)
+* rgw: per-bucket quota (Yehuda Sadeh)
+* rpm: fix junit dependencies (Alan Grosskurth)
+
+Notable Changes since v0.67 Dumpling
+------------------------------------
+
+* build cleanly under clang (Christophe Courtaut)
+* build: Makefile refactor (Roald J. van Loon)
+* build: fix [/usr]/sbin locations (Alan Somers)
+* ceph-disk: fix journal preallocation
+* ceph-fuse, radosgw: enable admin socket and logging by default
+* ceph-fuse: fix problem with readahead vs truncate race (Yan, Zheng)
+* ceph-fuse: trim deleted inodes from cache (Yan, Zheng)
+* ceph-fuse: use newer fuse api (Jianpeng Ma)
+* ceph-kvstore-tool: new tool for working with leveldb (copy, crc) (Joao Luis)
+* ceph-post-file: new command to easily share logs or other files with ceph devs
+* ceph: improve parsing of CEPH_ARGS (Benoit Knecht)
+* ceph: make -h behave when monitors are down
+* ceph: parse CEPH_ARGS env variable
+* common: bloom_filter improvements, cleanups
+* common: cache crc32c values where possible
+* common: correct SI is kB not KB (Dan Mick)
+* common: fix looping on BSD (Alan Somers)
+* common: migrate SharedPtrRegistry to use boost::shared_ptr<> (Loic Dachary)
+* common: misc portability fixes (Noah Watkins)
+* crc32c: fix optimized crc32c code (it now detects arch support properly)
+* crc32c: improved intel-optimized crc32c support (~8x faster on my laptop!)
+* crush: fix name caching
+* doc: erasure coding design notes (Loic Dachary)
+* hadoop: removed old version of shim to avoid confusing users (Noah Watkins)
+* librados, mon: ability to query/ping out-of-quorum monitor status (Joao Luis)
+* librados: fix async aio completion wakeup
+* librados: fix installed header #includes (Dan Mick)
+* librados: get_version64() method for C++ API
+* librados: hello_world example (Greg Farnum)
+* librados: sync calls now return on commit (instead of ack) (Greg Farnum)
+* librbd python bindings: fix parent image name limit (Josh Durgin)
+* librbd, ceph-fuse: avoid some sources of ceph-fuse, rbd cache stalls
+* mds: avoid leaking objects when deleting truncated files (Yan, Zheng)
+* mds: fix F_GETLK (Yan, Zheng)
+* mds: fix LOOKUPSNAP bug
+* mds: fix heap profiler commands (Joao Luis)
+* mds: fix locking deadlock (David Disseldorp)
+* mds: fix many bugs with stray (unlinked) inodes (Yan, Zheng)
+* mds: fix many directory fragmentation bugs (Yan, Zheng)
+* mds: fix mds rejoin with legacy parent backpointer xattrs (Alexandre Oliva)
+* mds: fix rare restart/failure race during fs creation
+* mds: fix standby-replay when we fall behind (Yan, Zheng)
+* mds: fix stray directory purging (Yan, Zheng)
+* mds: notify clients about deleted files (so they can release from their cache) (Yan, Zheng)
+* mds: several bug fixes with clustered mds (Yan, Zheng)
+* mon, osd: improve osdmap trimming logic (Samuel Just)
+* mon, osd: initial CLI for configuring tiering
+* mon: a few 'ceph mon add' races fixed (command is now idempotent) (Joao Luis)
+* mon: allow (un)setting HASHPSPOOL flag on existing pools (Joao Luis)
+* mon: allow cap strings with . to be unquoted
+* mon: allow logging level of cluster log (/var/log/ceph/ceph.log) to be adjusted
+* mon: avoid rewriting full osdmaps on restart (Joao Luis)
+* mon: continue to discover peer addr info during election phase
+* mon: disallow CephFS snapshots until 'ceph mds set allow_new_snaps' (Greg Farnum)
+* mon: do not expose uncommitted state from 'osd crush {add,set} ...' (Joao Luis)
+* mon: fix 'ceph osd crush reweight ...' (Joao Luis)
+* mon: fix 'osd crush move ...' command for buckets (Joao Luis)
+* mon: fix byte counts (off by factor of 4) (Dan Mick, Joao Luis)
+* mon: fix paxos corner case
+* mon: kv properties for pools to support EC (Loic Dachary)
+* mon: make 'osd pool rename' idempotent (Joao Luis)
+* mon: modify 'auth add' semantics to make a bit more sense (Joao Luis)
+* mon: new 'osd perf' command to dump recent performance information (Samuel Just)
+* mon: new and improved 'ceph -s' or 'ceph status' command (more info, easier to read)
+* mon: some auth check cleanups (Joao Luis)
+* mon: track per-pool stats (Joao Luis)
+* mon: warn about pools with bad pg_num
+* mon: warn when mon data stores grow very large (Joao Luis)
+* monc: fix small memory leak
+* new wireshark patches pulled into the tree (Kevin Jones)
+* objecter, librados: redirect requests based on cache tier config
+* objecter: fix possible hang when cluster is unpaused (Josh Durgin)
+* osd, librados: add new COPY_FROM rados operation
+* osd, librados: add new COPY_GET rados operations (used by COPY_FROM)
+* osd: 'osd recover clone overlap limit' option to limit cloning during recovery (Samuel Just)
+* osd: COPY_GET on-wire encoding improvements (Greg Farnum)
+* osd: add 'osd heartbeat min healthy ratio' configurable (was hard-coded at 33%)
+* osd: add option to disable pg log debug code (which burns CPU)
+* osd: allow cap strings with . to be unquoted
+* osd: automatically detect proper xattr limits (David Zafman)
+* osd: avoid extra copy in erasure coding reference implementation (Loic Dachary)
+* osd: basic cache pool redirects (Greg Farnum)
+* osd: basic whiteout, dirty flag support (not yet used)
+* osd: bloom_filter encodability, fixes, cleanups (Loic Dachary, Sage Weil)
+* osd: clean up and generalize copy-from code (Greg Farnum)
+* osd: cls_hello OSD class example
+* osd: erasure coding doc updates (Loic Dachary)
+* osd: erasure coding plugin infrastructure, tests (Loic Dachary)
+* osd: experiemental support for ZFS (zfsonlinux.org) (Yan, Zheng)
+* osd: fix RWORDER flags
+* osd: fix exponential backoff of slow request warnings (Loic Dachary)
+* osd: fix handling of racing read vs write (Samuel Just)
+* osd: fix version value returned by various operations (Greg Farnum)
+* osd: generalized temp object infrastructure
+* osd: ghobject_t infrastructure for EC (David Zafman)
+* osd: improvements for compatset support and storage (David Zafman)
+* osd: infrastructure to copy objects from other OSDs
+* osd: instrument peering states (David Zafman)
+* osd: misc copy-from improvements
+* osd: opportunistic crc checking on stored data (off by default)
+* osd: properly enforce RD/WR flags for rados classes
+* osd: reduce blocking on backing fs (Samuel Just)
+* osd: refactor recovery using PGBackend (Samuel Just)
+* osd: remove old magical tmap->omap conversion
+* osd: remove old pg log on upgrade (Samuel Just)
+* osd: revert xattr size limit (fixes large rgw uploads)
+* osd: use fdatasync(2) instead of fsync(2) to improve performance (Sam Just)
+* pybind: fix blacklisting nonce (Loic Dachary)
+* radosgw-agent: multi-region replication/DR
+* rgw: complete in-progress requests before shutting down
+* rgw: default log level is now more reasonable (Yehuda Sadeh)
+* rgw: fix S3 auth with response-* query string params (Sylvain Munaut, Yehuda Sadeh)
+* rgw: fix a few minor memory leaks (Yehuda Sadeh)
+* rgw: fix acl group check (Yehuda Sadeh)
+* rgw: fix inefficient use of std::list::size() (Yehuda Sadeh)
+* rgw: fix major CPU utilization bug with internal caching (Yehuda Sadeh, Mark Nelson)
+* rgw: fix ordering of write operations (preventing data loss on crash) (Yehuda Sadeh)
+* rgw: fix ordering of writes for mulitpart upload (Yehuda Sadeh)
+* rgw: fix various CORS bugs (Yehuda Sadeh)
+* rgw: fix/improve swift COPY support (Yehuda Sadeh)
+* rgw: improve help output (Christophe Courtaut)
+* rgw: misc fixes to support DR (Josh Durgin, Yehuda Sadeh)
+* rgw: per-bucket quota (Yehuda Sadeh)
+* rgw: validate S3 tokens against keystone (Roald J. van Loon)
+* rgw: wildcard support for keystone roles (Christophe Courtaut)
+* rpm: fix junit dependencies (Alan Grosskurth)
+* sysvinit radosgw: fix status return code (Danny Al-Gaaf)
+* sysvinit rbdmap: fix error 'service rbdmap stop' (Laurent Barbe)
+* sysvinit: add condrestart command (Dan van der Ster)
+* sysvinit: fix shutdown order (mons last) (Alfredo Deza)
+
+
+
+v0.71
+=====
+
+This development release includes a significant amount of new code and
+refactoring, as well as a lot of preliminary functionality that will be needed
+for erasure coding and tiering support.  There are also several significant
+patch sets improving this with the MDS.
+
+Upgrading
+---------
+
+* The MDS now disallows snapshots by default as they are not
+  considered stable.  The command 'ceph mds set allow_snaps' will
+  enable them.
+
+* For clusters that were created before v0.44 (pre-argonaut, Spring
+  2012) and store radosgw data, the auto-upgrade from TMAP to OMAP
+  objects has been disabled.  Before upgrading, make sure that any
+  buckets created on pre-argonaut releases have been modified (e.g.,
+  by PUTing and then DELETEing an object from each bucket).  Any
+  cluster created with argonaut (v0.48) or a later release or not
+  using radosgw never relied on the automatic conversion and is not
+  affected by this change.
+
+* Any direct users of the 'tmap' portion of the librados API should be
+  aware that the automatic tmap -> omap conversion functionality has
+  been removed.
+
+* Most output that used K or KB (e.g., for kilobyte) now uses a
+  lower-case k to match the official SI convention.  Any scripts that
+  parse output and check for an upper-case K will need to be modified.
+
+Notable Changes
+---------------
+
+* build: Makefile refactor (Roald J. van Loon)
+* ceph-disk: fix journal preallocation
+* ceph-fuse: trim deleted inodes from cache (Yan, Zheng)
+* ceph-fuse: use newer fuse api (Jianpeng Ma)
+* ceph-kvstore-tool: new tool for working with leveldb (copy, crc) (Joao Luis)
+* common: bloom_filter improvements, cleanups
+* common: correct SI is kB not KB (Dan Mick)
+* common: misc portability fixes (Noah Watkins)
+* hadoop: removed old version of shim to avoid confusing users (Noah Watkins)
+* librados: fix installed header #includes (Dan Mick)
+* librbd, ceph-fuse: avoid some sources of ceph-fuse, rbd cache stalls
+* mds: fix LOOKUPSNAP bug
+* mds: fix standby-replay when we fall behind (Yan, Zheng)
+* mds: fix stray directory purging (Yan, Zheng)
+* mon: disallow CephFS snapshots until 'ceph mds set allow_new_snaps' (Greg Farnum)
+* mon, osd: improve osdmap trimming logic (Samuel Just)
+* mon: kv properties for pools to support EC (Loic Dachary)
+* mon: some auth check cleanups (Joao Luis)
+* mon: track per-pool stats (Joao Luis)
+* mon: warn about pools with bad pg_num
+* osd: automatically detect proper xattr limits (David Zafman)
+* osd: avoid extra copy in erasure coding reference implementation (Loic Dachary)
+* osd: basic cache pool redirects (Greg Farnum)
+* osd: basic whiteout, dirty flag support (not yet used)
+* osd: clean up and generalize copy-from code (Greg Farnum)
+* osd: erasure coding doc updates (Loic Dachary)
+* osd: erasure coding plugin infrastructure, tests (Loic Dachary)
+* osd: fix RWORDER flags
+* osd: fix exponential backoff of slow request warnings (Loic Dachary)
+* osd: generalized temp object infrastructure
+* osd: ghobject_t infrastructure for EC (David Zafman)
+* osd: improvements for compatset support and storage (David Zafman)
+* osd: misc copy-from improvements
+* osd: opportunistic crc checking on stored data (off by default)
+* osd: refactor recovery using PGBackend (Samuel Just)
+* osd: remove old magical tmap->omap conversion
+* pybind: fix blacklisting nonce (Loic Dachary)
+* rgw: default log level is now more reasonable (Yehuda Sadeh)
+* rgw: fix acl group check (Yehuda Sadeh)
+* sysvinit: fix shutdown order (mons last) (Alfredo Deza)
+
+v0.70
+=====
+
+Upgrading
+---------
+
+* librados::Rados::pool_create_async() and librados::Rados::pool_delete_async()
+  don't drop a reference to the completion object on error, caller needs to take
+  care of that. This has never really worked correctly and we were leaking an
+  object
+
+* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the
+  specified location, as that's a job for 'ceph osd crush add'.  It will
+  however continue to work just the same as long as the osd already exists
+  in the crush map.
+
+Notable Changes
+---------------
+
+* mon: a few 'ceph mon add' races fixed (command is now idempotent) (Joao Luis)
+* crush: fix name caching
+* rgw: fix a few minor memory leaks (Yehuda Sadeh)
+* ceph: improve parsing of CEPH_ARGS (Benoit Knecht)
+* mon: avoid rewriting full osdmaps on restart (Joao Luis)
+* crc32c: fix optimized crc32c code (it now detects arch support properly)
+* mon: fix 'ceph osd crush reweight ...' (Joao Luis)
+* osd: revert xattr size limit (fixes large rgw uploads)
+* mds: fix heap profiler commands (Joao Luis)
+* rgw: fix inefficient use of std::list::size() (Yehuda Sadeh)
+
+
+v0.69
+=====
+
+Upgrading
+---------
+
+* The sysvinit /etc/init.d/ceph script will, by default, update the
+  CRUSH location of an OSD when it starts.  Previously, if the
+  monitors were not available, this command would hang indefinitely.
+  Now, that step will time out after 10 seconds and the ceph-osd daemon
+  will not be started.
+
+* Users of the librados C++ API should replace users of get_version()
+  with get_version64() as the old method only returns a 32-bit value
+  for a 64-bit field.  The existing 32-bit get_version() method is now
+  deprecated.
+
+* The OSDs are now more picky that request payload match their
+  declared size.  A write operation across N bytes that includes M
+  bytes of data will now be rejected.  No known clients do this, but
+  the because the server-side behavior has changed it is possible that
+  an application misusing the interface may now get errors.
+
+* The OSD now enforces that class write methods cannot both mutate an
+  object and return data.  The rbd.assign_bid method, the lone
+  offender, has been removed.  This breaks compatibility with
+  pre-bobtail librbd clients by preventing them from creating new
+  images.
+
+* librados now returns on commit instead of ack for synchronous calls.
+  This is a bit safer in the case where both OSDs and the client crash, and
+  is probably how it should have been acting from the beginning. Users are
+  unlikely to notice but it could result in lower performance in some
+  circumstances. Those who care should switch to using the async interfaces,
+  which let you specify safety semantics precisely.
+
+* The C++ librados AioComplete::get_version() method was incorrectly
+  returning an int (usually 32-bits).  To avoid breaking library
+  compatibility, a get_version64() method is added that returns the
+  full-width value.  The old method is deprecated and will be removed
+  in a future release.  Users of the C++ librados API that make use of
+  the get_version() method should modify their code to avoid getting a
+  value that is truncated from 64 to to 32 bits.
+
+
+Notable Changes
+---------------
+
+* build cleanly under clang (Christophe Courtaut)
+* common: migrate SharedPtrRegistry to use boost::shared_ptr<> (Loic Dachary)
+* doc: erasure coding design notes (Loic Dachary)
+* improved intel-optimized crc32c support (~8x faster on my laptop!)
+* librados: get_version64() method for C++ API
+* mds: fix locking deadlock (David Disseldorp)
+* mon, osd: initial CLI for configuring tiering
+* mon: allow cap strings with . to be unquoted
+* mon: continue to discover peer addr info during election phase
+* mon: fix 'osd crush move ...' command for buckets (Joao Luis)
+* mon: warn when mon data stores grow very large (Joao Luis)
+* objecter, librados: redirect requests based on cache tier config
+* osd, librados: add new COPY_FROM rados operation
+* osd, librados: add new COPY_GET rados operations (used by COPY_FROM)
+* osd: add 'osd heartbeat min healthy ratio' configurable (was hard-coded at 33%)
+* osd: add option to disable pg log debug code (which burns CPU)
+* osd: allow cap strings with . to be unquoted
+* osd: fix version value returned by various operations (Greg Farnum)
+* osd: infrastructure to copy objects from other OSDs
+* osd: use fdatasync(2) instead of fsync(2) to improve performance (Sam Just)
+* rgw: fix major CPU utilization bug with internal caching (Yehuda Sadeh, Mark Nelson)
+* rgw: fix ordering of write operations (preventing data loss on crash) (Yehuda Sadeh)
+* rgw: fix ordering of writes for mulitpart upload (Yehuda Sadeh)
+* rgw: fix various CORS bugs (Yehuda Sadeh)
+* rgw: improve help output (Christophe Courtaut)
+* rgw: validate S3 tokens against keystone (Roald J. van Loon)
+* rgw: wildcard support for keystone roles (Christophe Courtaut)
+* sysvinit radosgw: fix status return code (Danny Al-Gaaf)
+* sysvinit rbdmap: fix error 'service rbdmap stop' (Laurent Barbe)
+
+v0.68
+=====
+
+Upgrading
+---------
+
+* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the
+  specified location, as that's a job for 'ceph osd crush add'.  It will
+  however continue to work just the same as long as the osd already exists
+  in the crush map.
+
+* The OSD now enforces that class write methods cannot both mutate an
+  object and return data.  The rbd.assign_bid method, the lone
+  offender, has been removed.  This breaks compatibility with
+  pre-bobtail librbd clients by preventing them from creating new
+  images.
+
+* librados now returns on commit instead of ack for synchronous calls.
+  This is a bit safer in the case where both OSDs and the client crash, and
+  is probably how it should have been acting from the beginning. Users are
+  unlikely to notice but it could result in lower performance in some
+  circumstances. Those who care should switch to using the async interfaces,
+  which let you specify safety semantics precisely.
+
+* The C++ librados AioComplete::get_version() method was incorrectly
+  returning an int (usually 32-bits).  To avoid breaking library
+  compatibility, a get_version64() method is added that returns the
+  full-width value.  The old method is deprecated and will be removed
+  in a future release.  Users of the C++ librados API that make use of
+  the get_version() method should modify their code to avoid getting a
+  value that is truncated from 64 to to 32 bits.
+
+
+
+Notable Changes
+---------------
+
+* ceph-fuse: fix problem with readahead vs truncate race (Yan, Zheng)
+* ceph-post-file: new command to easily share logs or other files with ceph devs
+* ceph: parse CEPH_ARGS env variable
+* librados: fix async aio completion wakeup
+* librados: hello_world example (Greg Farnum)
+* librados: sync calls now return on commit (instead of ack) (Greg Farnum)
+* mds: fix mds rejoin with legacy parent backpointer xattrs (Alexandre Oliva)
+* mds: fix rare restart/failure race during fs creation
+* mds: notify clients about deleted files (so they can release from their cache) (Yan, Zheng)
+* mds: several bug fixes with clustered mds (Yan, Zheng)
+* mon: allow logging level of cluster log (/var/log/ceph/ceph.log) to be adjusted
+* mon: do not expose uncommitted state from 'osd crush {add,set} ...' (Joao Luis)
+* mon: fix byte counts (off by factor of 4) (Dan Mick, Joao Luis)
+* mon: fix paxos corner case
+* mon: modify 'auth add' semantics to make a bit more sense (Joao Luis)
+* mon: new 'osd perf' command to dump recent performance information (Samuel Just)
+* mon: new and improved 'ceph -s' or 'ceph status' command (more info, easier to read)
+* monc: fix small memory leak
+* new wireshark patches pulled into the tree (Kevin Jones)
+* objecter: fix possible hang when cluster is unpaused (Josh Durgin)
+* osd: 'osd recover clone overlap limit' option to limit cloning during recovery (Samuel Just)
+* osd: cls_hello OSD class example
+* osd: experiemental support for ZFS (zfsonlinux.org) (Yan, Zheng)
+* osd: instrument peering states (David Zafman)
+* osd: properly enforce RD/WR flags for rados classes
+* osd: remove old pg log on upgrade (Samuel Just)
+* rgw: complete in-progress requests before shutting down
+* rgw: fix S3 auth with response-* query string params (Sylvain Munaut, Yehuda Sadeh)
+* sysvinit: add condrestart command (Dan van der Ster)
diff --git a/doc/releases/firefly.rst b/doc/releases/firefly.rst
new file mode 100644
index 000000000..0b2d943b8
--- /dev/null
+++ b/doc/releases/firefly.rst
@@ -0,0 +1,1787 @@
+=======
+Firefly
+=======
+
+Firefly is the 6th stable release of Ceph. It is named after the
+firefly squid (Watasenia scintillans).
+
+v0.80.11 Firefly
+================
+
+This is a bugfix release for Firefly.  This Firefly 0.80.x is nearing
+its planned end of life in January 2016 it may also be the last.
+
+We recommend that all Firefly users upgrade.
+
+For more detailed information, see :download:`the complete changelog
+<../changelog/v0.80.11.txt>`.
+
+Notable Changes
+---------------
+
+* build/ops: /etc/init.d/radosgw restart does not work correctly (`issue#11140 <http://tracker.ceph.com/issues/11140>`_, `pr#5831 <http://github.com/ceph/ceph/pull/5831>`_, Dmitry Yatsushkevich)
+* build/ops: Fix -Wno-format and -Werror=format-security options clash  (`issue#13417 <http://tracker.ceph.com/issues/13417>`_, `pr#6207 <http://github.com/ceph/ceph/pull/6207>`_, Boris Ranto)
+* build/ops: ceph-common needs python-argparse on older distros, but doesn't require it (`issue#12034 <http://tracker.ceph.com/issues/12034>`_, `pr#5217 <http://github.com/ceph/ceph/pull/5217>`_, Nathan Cutler)
+* build/ops: ceph.spec.in running fdupes unnecessarily (`issue#12301 <http://tracker.ceph.com/issues/12301>`_, `pr#5224 <http://github.com/ceph/ceph/pull/5224>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: 50-rbd.rules conditional is wrong (`issue#12166 <http://tracker.ceph.com/issues/12166>`_, `pr#5225 <http://github.com/ceph/ceph/pull/5225>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: useless %py_requires breaks SLE11-SP3 build (`issue#12351 <http://tracker.ceph.com/issues/12351>`_, `pr#5394 <http://github.com/ceph/ceph/pull/5394>`_, Nathan Cutler)
+* build/ops: fedora21 has junit, not junit4  (`issue#10728 <http://tracker.ceph.com/issues/10728>`_, `pr#6203 <http://github.com/ceph/ceph/pull/6203>`_, Ken Dreyer, Loic Dachary)
+* build/ops: upstart: configuration is too generous on restarts (`issue#11798 <http://tracker.ceph.com/issues/11798>`_, `pr#5992 <http://github.com/ceph/ceph/pull/5992>`_, Sage Weil)
+* common: Client admin socket leaks file descriptors (`issue#11535 <http://tracker.ceph.com/issues/11535>`_, `pr#4633 <http://github.com/ceph/ceph/pull/4633>`_, Jon Bernard)
+* common: FileStore calls syncfs(2) even it is not supported (`issue#12512 <http://tracker.ceph.com/issues/12512>`_, `pr#5529 <http://github.com/ceph/ceph/pull/5529>`_, Danny Al-Gaaf, Kefu Chai, Jianpeng Ma)
+* common: HeartBeat: include types (`issue#13088 <http://tracker.ceph.com/issues/13088>`_, `pr#6038 <http://github.com/ceph/ceph/pull/6038>`_, Sage Weil)
+* common: Malformed JSON command output when non-ASCII strings are present  (`issue#7387 <http://tracker.ceph.com/issues/7387>`_, `pr#4635 <http://github.com/ceph/ceph/pull/4635>`_, Kefu Chai, Tim Serong)
+* common: Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy (`issue#11762 <http://tracker.ceph.com/issues/11762>`_, `pr#5403 <http://github.com/ceph/ceph/pull/5403>`_, Ketor Meng)
+* common: Thread:pthread_attr_destroy(thread_attr) when done with it (`issue#12570 <http://tracker.ceph.com/issues/12570>`_, `pr#6325 <http://github.com/ceph/ceph/pull/6325>`_, Piotr Dałek, Zheng Qiankun)
+* common: ThreadPool add/remove work queue methods not thread safe (`issue#12662 <http://tracker.ceph.com/issues/12662>`_, `pr#5991 <http://github.com/ceph/ceph/pull/5991>`_, Jason Dillaman)
+* common: buffer: critical bufferlist::zero bug (`issue#12252 <http://tracker.ceph.com/issues/12252>`_, `pr#5388 <http://github.com/ceph/ceph/pull/5388>`_, Haomai Wang)
+* common: log: take mutex while opening fd (`issue#12465 <http://tracker.ceph.com/issues/12465>`_, `pr#5406 <http://github.com/ceph/ceph/pull/5406>`_, Samuel Just)
+* common: recursive lock of md_config_t (0) (`issue#12614 <http://tracker.ceph.com/issues/12614>`_, `pr#5814 <http://github.com/ceph/ceph/pull/5814>`_, Josh Durgin)
+* crush: take crashes due to invalid arg (`issue#11602 <http://tracker.ceph.com/issues/11602>`_, `pr#4769 <http://github.com/ceph/ceph/pull/4769>`_, Sage Weil)
+* doc: backport v0.80.10 release notes to firefly (`issue#11090 <http://tracker.ceph.com/issues/11090>`_, `pr#5307 <http://github.com/ceph/ceph/pull/5307>`_, Loic Dachary, Sage Weil)
+* doc: update docs to point to download.ceph.com (`issue#13162 <http://tracker.ceph.com/issues/13162>`_, `pr#5993 <http://github.com/ceph/ceph/pull/5993>`_, Alfredo Deza)
+* fs: MDSMonitor: handle MDSBeacon messages properly (`issue#11590 <http://tracker.ceph.com/issues/11590>`_, `pr#5199 <http://github.com/ceph/ceph/pull/5199>`_, Kefu Chai)
+* fs: client nonce collision due to unshared pid namespaces (`issue#13032 <http://tracker.ceph.com/issues/13032>`_, `pr#6087 <http://github.com/ceph/ceph/pull/6087>`_, Josh Durgin, Sage Weil)
+* librbd: Objectcacher setting max object counts too low (`issue#7385 <http://tracker.ceph.com/issues/7385>`_, `pr#4639 <http://github.com/ceph/ceph/pull/4639>`_, Jason Dillaman)
+* librbd: aio calls may block (`issue#11056 <http://tracker.ceph.com/issues/11056>`_, `pr#4854 <http://github.com/ceph/ceph/pull/4854>`_, Haomai Wang, Sage Weil, Jason Dillaman)
+* librbd: internal.cc: 1967: FAILED assert(watchers.size() == 1) (`issue#12176 <http://tracker.ceph.com/issues/12176>`_, `pr#5171 <http://github.com/ceph/ceph/pull/5171>`_, Jason Dillaman)
+* mon: Clock skew causes missing summary and confuses Calamari (`issue#11877 <http://tracker.ceph.com/issues/11877>`_, `pr#4867 <http://github.com/ceph/ceph/pull/4867>`_, Thorsten Behrens)
+* mon: EC pools are not allowed as cache pools, disallow in the mon (`issue#11650 <http://tracker.ceph.com/issues/11650>`_, `pr#5389 <http://github.com/ceph/ceph/pull/5389>`_, Samuel Just)
+* mon: Make it more difficult to delete pools in firefly (`issue#11800 <http://tracker.ceph.com/issues/11800>`_, `pr#4788 <http://github.com/ceph/ceph/pull/4788>`_, Sage Weil)
+* mon: MonitorDBStore: get_next_key() only if prefix matches (`issue#11786 <http://tracker.ceph.com/issues/11786>`_, `pr#5360 <http://github.com/ceph/ceph/pull/5360>`_, Joao Eduardo Luis)
+* mon: PaxosService: call post_refresh() instead of post_paxos_update() (`issue#11470 <http://tracker.ceph.com/issues/11470>`_, `pr#5358 <http://github.com/ceph/ceph/pull/5358>`_, Joao Eduardo Luis)
+* mon: add a cache layer over MonitorDBStore (`issue#12638 <http://tracker.ceph.com/issues/12638>`_, `pr#5698 <http://github.com/ceph/ceph/pull/5698>`_, Kefu Chai)
+* mon: adding exsting pool as tier with --force-nonempty clobbers removed_snaps (`issue#11493 <http://tracker.ceph.com/issues/11493>`_, `pr#5236 <http://github.com/ceph/ceph/pull/5236>`_, Sage Weil, Samuel Just)
+* mon: ceph fails to compile with boost 1.58 (`issue#11576 <http://tracker.ceph.com/issues/11576>`_, `pr#5129 <http://github.com/ceph/ceph/pull/5129>`_, Kefu Chai)
+* mon: does not check for IO errors on every transaction (`issue#13089 <http://tracker.ceph.com/issues/13089>`_, `pr#6091 <http://github.com/ceph/ceph/pull/6091>`_, Sage Weil)
+* mon: get pools health'info have error (`issue#12402 <http://tracker.ceph.com/issues/12402>`_, `pr#5410 <http://github.com/ceph/ceph/pull/5410>`_, renhwztetecs)
+* mon: increase globalid default for firefly (`issue#13255 <http://tracker.ceph.com/issues/13255>`_, `pr#6010 <http://github.com/ceph/ceph/pull/6010>`_, Sage Weil)
+* mon: pgmonitor: wrong at/near target max“ reporting (`issue#12401 <http://tracker.ceph.com/issues/12401>`_, `pr#5409 <http://github.com/ceph/ceph/pull/5409>`_, huangjun)
+* mon: register_new_pgs() should check ruleno instead of its index (`issue#12210 <http://tracker.ceph.com/issues/12210>`_, `pr#5404 <http://github.com/ceph/ceph/pull/5404>`_, Xinze Chi)
+* mon: scrub error (osdmap encoding mismatch?) upgrading from 0.80 to ~0.80.2 (`issue#8815 <http://tracker.ceph.com/issues/8815>`_, `issue#8674 <http://tracker.ceph.com/issues/8674>`_, `issue#9064 <http://tracker.ceph.com/issues/9064>`_, `pr#5200 <http://github.com/ceph/ceph/pull/5200>`_, Sage Weil, Zhiqiang Wang, Samuel Just)
+* mon: the output is wrong when runing ceph osd reweight (`issue#12251 <http://tracker.ceph.com/issues/12251>`_, `pr#5408 <http://github.com/ceph/ceph/pull/5408>`_, Joao Eduardo Luis)
+* objecter: can get stuck in redirect loop if osdmap epoch == last_force_op_resend (`issue#11026 <http://tracker.ceph.com/issues/11026>`_, `pr#4597 <http://github.com/ceph/ceph/pull/4597>`_, Jianpeng Ma, Sage Weil)
+* objecter: pg listing can deadlock when throttling is in use (`issue#9008 <http://tracker.ceph.com/issues/9008>`_, `pr#5043 <http://github.com/ceph/ceph/pull/5043>`_, Guang Yang)
+* objecter: resend linger ops on split (`issue#9806 <http://tracker.ceph.com/issues/9806>`_, `pr#5062 <http://github.com/ceph/ceph/pull/5062>`_, Josh Durgin, Samuel Just)
+* osd: Cleanup boost optionals for boost 1.56 (`issue#9983 <http://tracker.ceph.com/issues/9983>`_, `pr#5039 <http://github.com/ceph/ceph/pull/5039>`_, William A. Kennington III)
+* osd: LibRadosTwoPools[EC]PP.PromoteSnap failure (`issue#10052 <http://tracker.ceph.com/issues/10052>`_, `pr#5050 <http://github.com/ceph/ceph/pull/5050>`_, Sage Weil)
+* osd: Mutex Assert from PipeConnection::try_get_pipe (`issue#12437 <http://tracker.ceph.com/issues/12437>`_, `pr#5815 <http://github.com/ceph/ceph/pull/5815>`_, David Zafman)
+* osd: PG stuck with remapped (`issue#9614 <http://tracker.ceph.com/issues/9614>`_, `pr#5044 <http://github.com/ceph/ceph/pull/5044>`_, Guang Yang)
+* osd: PG::handle_advance_map: on_pool_change after handling the map change (`issue#12809 <http://tracker.ceph.com/issues/12809>`_, `pr#5988 <http://github.com/ceph/ceph/pull/5988>`_, Samuel Just)
+* osd: PGLog: split divergent priors as well (`issue#11069 <http://tracker.ceph.com/issues/11069>`_, `pr#4631 <http://github.com/ceph/ceph/pull/4631>`_, Samuel Just)
+* osd: PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out (`issue#11358 <http://tracker.ceph.com/issues/11358>`_, `pr#5287 <http://github.com/ceph/ceph/pull/5287>`_, Samuel Just)
+* osd: WBThrottle::clear_object: signal on cond when we reduce throttle values (`issue#12223 <http://tracker.ceph.com/issues/12223>`_, `pr#5822 <http://github.com/ceph/ceph/pull/5822>`_, Samuel Just)
+* osd: cache full mode still skips young objects (`issue#10006 <http://tracker.ceph.com/issues/10006>`_, `pr#5051 <http://github.com/ceph/ceph/pull/5051>`_, Xinze Chi, Zhiqiang Wang)
+* osd: crash creating/deleting pools (`issue#12429 <http://tracker.ceph.com/issues/12429>`_, `pr#5526 <http://github.com/ceph/ceph/pull/5526>`_, John Spray)
+* osd: explicitly specify OSD features in MOSDBoot (`issue#10911 <http://tracker.ceph.com/issues/10911>`_, `pr#4960 <http://github.com/ceph/ceph/pull/4960>`_, Sage Weil)
+* osd: is_new_interval() fixes (`issue#11771 <http://tracker.ceph.com/issues/11771>`_, `issue#10399 <http://tracker.ceph.com/issues/10399>`_, `pr#5726 <http://github.com/ceph/ceph/pull/5726>`_, Samuel Just, Jason Dillaman)
+* osd: make the all osd/filestore thread pool suicide timeouts separately configurable (`issue#11439 <http://tracker.ceph.com/issues/11439>`_, `pr#5823 <http://github.com/ceph/ceph/pull/5823>`_, Samuel Just)
+* osd: object creation by write cannot use an offset on an erasure coded pool (`issue#11507 <http://tracker.ceph.com/issues/11507>`_, `pr#4632 <http://github.com/ceph/ceph/pull/4632>`_, Jianpeng Ma, Loic Dachary)
+* osd: os/FileJournal: Fix journal write fail, align for direct io (`issue#12943 <http://tracker.ceph.com/issues/12943>`_, `pr#5619 <http://github.com/ceph/ceph/pull/5619>`_, Xie Rui)
+* osd: osd/PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size()) (`issue#12652 <http://tracker.ceph.com/issues/12652>`_, `pr#5820 <http://github.com/ceph/ceph/pull/5820>`_, Sage Weil)
+* osd: read on chunk-aligned xattr not handled (`issue#12309 <http://tracker.ceph.com/issues/12309>`_, `pr#5235 <http://github.com/ceph/ceph/pull/5235>`_, Sage Weil)
+* rgw: Change variable length array of std::strings (not legal in C++) to std::vector<std::string> (`issue#12467 <http://tracker.ceph.com/issues/12467>`_, `pr#4583 <http://github.com/ceph/ceph/pull/4583>`_, Daniel J. Hofmann)
+* rgw: Civetweb RGW appears to report full size of object as downloaded when only partially downloaded (`issue#11851 <http://tracker.ceph.com/issues/11851>`_, `pr#5234 <http://github.com/ceph/ceph/pull/5234>`_, Yehuda Sadeh)
+* rgw: Keystone PKI token expiration is not enforced (`issue#11367 <http://tracker.ceph.com/issues/11367>`_, `pr#4765 <http://github.com/ceph/ceph/pull/4765>`_, Anton Aksola)
+* rgw: Object copy bug (`issue#11639 <http://tracker.ceph.com/issues/11639>`_, `pr#4762 <http://github.com/ceph/ceph/pull/4762>`_, Javier M. Mellid)
+* rgw: RGW returns requested bucket name raw in "Bucket" response header (`issue#11860 <http://tracker.ceph.com/issues/11860>`_, `issue#12537 <http://tracker.ceph.com/issues/12537>`_, `pr#5730 <http://github.com/ceph/ceph/pull/5730>`_, Yehuda Sadeh, Wido den Hollander)
+* rgw: Swift API: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used (`issue#11036 <http://tracker.ceph.com/issues/11036>`_, `pr#5170 <http://github.com/ceph/ceph/pull/5170>`_, Radoslaw Zarzynski)
+* rgw: content length parsing calls strtol() instead of strtoll() (`issue#10701 <http://tracker.ceph.com/issues/10701>`_, `pr#5997 <http://github.com/ceph/ceph/pull/5997>`_, Yehuda Sadeh)
+* rgw: delete bucket does not remove .bucket.meta file (`issue#11149 <http://tracker.ceph.com/issues/11149>`_, `pr#4641 <http://github.com/ceph/ceph/pull/4641>`_, Orit Wasserman)
+* rgw: doesn't return 'x-timestamp' in header which is used by 'View Details' of OpenStack (`issue#8911 <http://tracker.ceph.com/issues/8911>`_, `pr#4584 <http://github.com/ceph/ceph/pull/4584>`_, Yehuda Sadeh)
+* rgw: init some manifest fields when handling explicit objs (`issue#11455 <http://tracker.ceph.com/issues/11455>`_, `pr#5729 <http://github.com/ceph/ceph/pull/5729>`_, Yehuda Sadeh)
+* rgw: logfile does not get chowned properly (`issue#12073 <http://tracker.ceph.com/issues/12073>`_, `pr#5233 <http://github.com/ceph/ceph/pull/5233>`_, Thorsten Behrens)
+* rgw: logrotate.conf calls service with wrong init script name (`issue#12043 <http://tracker.ceph.com/issues/12043>`_, `pr#5390 <http://github.com/ceph/ceph/pull/5390>`_, wuxingyi)
+* rgw: quota not respected in POST object (`issue#11323 <http://tracker.ceph.com/issues/11323>`_, `pr#4642 <http://github.com/ceph/ceph/pull/4642>`_, Sergey Arkhipov)
+* rgw: swift smoke test fails on TestAccountUTF8 (`issue#11091 <http://tracker.ceph.com/issues/11091>`_, `issue#11438 <http://tracker.ceph.com/issues/11438>`_, `issue#12939 <http://tracker.ceph.com/issues/12939>`_, `issue#12157 <http://tracker.ceph.com/issues/12157>`_, `issue#12158 <http://tracker.ceph.com/issues/12158>`_, `issue#12363 <http://tracker.ceph.com/issues/12363>`_, `pr#5532 <http://github.com/ceph/ceph/pull/5532>`_, Radoslaw Zarzynski, Orit Wasserman, Robin H. Johnson)
+* rgw: use correct objv_tracker for bucket instance (`issue#11416 <http://tracker.ceph.com/issues/11416>`_, `pr#4535 <http://github.com/ceph/ceph/pull/4535>`_, Yehuda Sadeh)
+* tests: ceph-fuse crash in test_client_recovery (`issue#12673 <http://tracker.ceph.com/issues/12673>`_, `pr#5813 <http://github.com/ceph/ceph/pull/5813>`_, Loic Dachary)
+* tests: kernel_untar_build fails on EL7 (`issue#11758 <http://tracker.ceph.com/issues/11758>`_, `pr#6000 <http://github.com/ceph/ceph/pull/6000>`_, Greg Farnum)
+* tests: qemu workunit refers to apt-mirror.front.sepia.ceph.com (`issue#13420 <http://tracker.ceph.com/issues/13420>`_, `pr#6328 <http://github.com/ceph/ceph/pull/6328>`_, Yuan Zhou, Sage Weil)
+* tools:  src/ceph-disk : disk zap sgdisk invocation (`issue#11143 <http://tracker.ceph.com/issues/11143>`_, `pr#4636 <http://github.com/ceph/ceph/pull/4636>`_, Thorsten Behrens, Owen Synge)
+* tools: ceph-disk: sometimes the journal symlink is not created (`issue#10146 <http://tracker.ceph.com/issues/10146>`_, `pr#5541 <http://github.com/ceph/ceph/pull/5541>`_, Dan van der Ster)
+* tools: ceph-disk: support NVMe device partitions (`issue#11612 <http://tracker.ceph.com/issues/11612>`_, `pr#4771 <http://github.com/ceph/ceph/pull/4771>`_, Ilja Slepnev)
+* tools: ceph-post-file fails on rhel7 (`issue#11836 <http://tracker.ceph.com/issues/11836>`_, `pr#5037 <http://github.com/ceph/ceph/pull/5037>`_, Joseph McDonald, Sage Weil)
+* tools: ceph_argparse_flag has no regular 3rd parameter (`issue#11543 <http://tracker.ceph.com/issues/11543>`_, `pr#4582 <http://github.com/ceph/ceph/pull/4582>`_, Thorsten Behrens)
+* tools: use a new disk as journal disk,ceph-disk prepare fail (`issue#10983 <http://tracker.ceph.com/issues/10983>`_, `pr#4630 <http://github.com/ceph/ceph/pull/4630>`_, Loic Dachary)
+
+
+v0.80.10 Firefly
+================
+
+This is a bugfix release for Firefly.
+
+We recommend that all Firefly users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.10.txt>`.
+
+Notable Changes
+---------------
+
+* build/ops: ceph.spec.in: package mkcephfs on EL6 (`issue#11955 <http://tracker.ceph.com/issues/11955>`_, `pr#4924 <http://github.com/ceph/ceph/pull/4924>`_, Ken Dreyer)
+* build/ops: debian: ceph-test and rest-bench debug packages should require their respective binary packages (`issue#11673 <http://tracker.ceph.com/issues/11673>`_, `pr#4766 <http://github.com/ceph/ceph/pull/4766>`_, Ken Dreyer)
+* build/ops: run RGW as root (`issue#11453 <http://tracker.ceph.com/issues/11453>`_, `pr#4638 <http://github.com/ceph/ceph/pull/4638>`_, Ken Dreyer)
+* common: messages/MWatchNotify: include an error code in the message (`issue#9193 <http://tracker.ceph.com/issues/9193>`_, `pr#3944 <http://github.com/ceph/ceph/pull/3944>`_, Sage Weil)
+* common: Rados.shutdown() dies with Illegal instruction (core dumped) (`issue#10153 <http://tracker.ceph.com/issues/10153>`_, `pr#3963 <http://github.com/ceph/ceph/pull/3963>`_, Federico Simoncelli)
+* common: SimpleMessenger: allow RESETSESSION whenever we forget an endpoint (`issue#10080 <http://tracker.ceph.com/issues/10080>`_, `pr#3915 <http://github.com/ceph/ceph/pull/3915>`_, Greg Farnum)
+* common: WorkQueue: make wait timeout on empty queue configurable (`issue#10817 <http://tracker.ceph.com/issues/10817>`_, `pr#3941 <http://github.com/ceph/ceph/pull/3941>`_, Samuel Just)
+* crush: set_choose_tries = 100 for erasure code rulesets (`issue#10353 <http://tracker.ceph.com/issues/10353>`_, `pr#3824 <http://github.com/ceph/ceph/pull/3824>`_, Loic Dachary)
+* doc: backport ceph-disk man page to Firefly (`issue#10724 <http://tracker.ceph.com/issues/10724>`_, `pr#3936 <http://github.com/ceph/ceph/pull/3936>`_, Nilamdyuti Goswami)
+* doc: Fix ceph command manpage to match ceph -h (`issue#10676 <http://tracker.ceph.com/issues/10676>`_, `pr#3996 <http://github.com/ceph/ceph/pull/3996>`_, David Zafman)
+* fs: mount.ceph: avoid spurious error message (`issue#10351 <http://tracker.ceph.com/issues/10351>`_, `pr#3927 <http://github.com/ceph/ceph/pull/3927>`_, Yan, Zheng)
+* librados: Fix memory leak in python rados bindings (`issue#10723 <http://tracker.ceph.com/issues/10723>`_, `pr#3935 <http://github.com/ceph/ceph/pull/3935>`_, Josh Durgin)
+* librados: fix resources leakage in RadosClient::connect() (`issue#10425 <http://tracker.ceph.com/issues/10425>`_, `pr#3828 <http://github.com/ceph/ceph/pull/3828>`_, Radoslaw Zarzynski)
+* librados: Translate operation flags from C APIs (`issue#10497 <http://tracker.ceph.com/issues/10497>`_, `pr#3930 <http://github.com/ceph/ceph/pull/3930>`_, Matt Richards)
+* librbd: acquire cache_lock before refreshing parent (`issue#5488 <http://tracker.ceph.com/issues/5488>`_, `pr#4206 <http://github.com/ceph/ceph/pull/4206>`_, Jason Dillaman)
+* librbd: snap_remove should ignore -ENOENT errors (`issue#11113 <http://tracker.ceph.com/issues/11113>`_, `pr#4245 <http://github.com/ceph/ceph/pull/4245>`_, Jason Dillaman)
+* mds: fix assertion caused by system clock backwards (`issue#11053 <http://tracker.ceph.com/issues/11053>`_, `pr#3970 <http://github.com/ceph/ceph/pull/3970>`_, Yan, Zheng)
+* mon: ignore osd failures from before up_from (`issue#10762 <http://tracker.ceph.com/issues/10762>`_, `pr#3937 <http://github.com/ceph/ceph/pull/3937>`_, Sage Weil)
+* mon: MonCap: take EntityName instead when expanding profiles (`issue#10844 <http://tracker.ceph.com/issues/10844>`_, `pr#3942 <http://github.com/ceph/ceph/pull/3942>`_, Joao Eduardo Luis)
+* mon: Monitor: fix timecheck rounds period (`issue#10546 <http://tracker.ceph.com/issues/10546>`_, `pr#3932 <http://github.com/ceph/ceph/pull/3932>`_, Joao Eduardo Luis)
+* mon: OSDMonitor: do not trust small values in osd epoch cache (`issue#10787 <http://tracker.ceph.com/issues/10787>`_, `pr#3823 <http://github.com/ceph/ceph/pull/3823>`_, Sage Weil)
+* mon: OSDMonitor: fallback to json-pretty in case of invalid formatter (`issue#9538 <http://tracker.ceph.com/issues/9538>`_, `pr#4475 <http://github.com/ceph/ceph/pull/4475>`_, Loic Dachary)
+* mon: PGMonitor: several stats output error fixes (`issue#10257 <http://tracker.ceph.com/issues/10257>`_, `pr#3826 <http://github.com/ceph/ceph/pull/3826>`_, Joao Eduardo Luis)
+* objecter: fix map skipping (`issue#9986 <http://tracker.ceph.com/issues/9986>`_, `pr#3952 <http://github.com/ceph/ceph/pull/3952>`_, Ding Dinghua)
+* osd: cache tiering: fix the atime logic of the eviction (`issue#9915 <http://tracker.ceph.com/issues/9915>`_, `pr#3949 <http://github.com/ceph/ceph/pull/3949>`_, Zhiqiang Wang)
+* osd: cancel_pull: requeue waiters (`issue#11244 <http://tracker.ceph.com/issues/11244>`_, `pr#4415 <http://github.com/ceph/ceph/pull/4415>`_, Samuel Just)
+* osd: check that source OSD is valid for MOSDRepScrub (`issue#9555 <http://tracker.ceph.com/issues/9555>`_, `pr#3947 <http://github.com/ceph/ceph/pull/3947>`_, Sage Weil)
+* osd: DBObjectMap: lock header_lock on sync() (`issue#9891 <http://tracker.ceph.com/issues/9891>`_, `pr#3948 <http://github.com/ceph/ceph/pull/3948>`_, Samuel Just)
+* osd: do not ignore deleted pgs on startup (`issue#10617 <http://tracker.ceph.com/issues/10617>`_, `pr#3933 <http://github.com/ceph/ceph/pull/3933>`_, Sage Weil)
+* osd: ENOENT on clone (`issue#11199 <http://tracker.ceph.com/issues/11199>`_, `pr#4385 <http://github.com/ceph/ceph/pull/4385>`_, Samuel Just)
+* osd: erasure-code-profile set races with erasure-code-profile rm (`issue#11144 <http://tracker.ceph.com/issues/11144>`_, `pr#4383 <http://github.com/ceph/ceph/pull/4383>`_, Loic Dachary)
+* osd: FAILED assert(soid < scrubber.start || soid >= scrubber.end) (`issue#11156 <http://tracker.ceph.com/issues/11156>`_, `pr#4185 <http://github.com/ceph/ceph/pull/4185>`_, Samuel Just)
+* osd: FileJournal: fix journalq population in do_read_entry() (`issue#6003 <http://tracker.ceph.com/issues/6003>`_, `pr#3960 <http://github.com/ceph/ceph/pull/3960>`_, Samuel Just)
+* osd: fix negative degraded objects during backfilling (`issue#7737 <http://tracker.ceph.com/issues/7737>`_, `pr#4021 <http://github.com/ceph/ceph/pull/4021>`_, Guang Yang)
+* osd: get the currently atime of the object in cache pool for eviction (`issue#9985 <http://tracker.ceph.com/issues/9985>`_, `pr#3950 <http://github.com/ceph/ceph/pull/3950>`_, Sage Weil)
+* osd: load_pgs: we need to handle the case where an upgrade from earlier versions which ignored non-existent pgs resurrects a pg with a prehistoric osdmap (`issue#11429 <http://tracker.ceph.com/issues/11429>`_, `pr#4556 <http://github.com/ceph/ceph/pull/4556>`_, Samuel Just)
+* osd: ObjectStore: Don't use largest_data_off to calc data_align. (`issue#10014 <http://tracker.ceph.com/issues/10014>`_, `pr#3954 <http://github.com/ceph/ceph/pull/3954>`_, Jianpeng Ma)
+* osd: osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::o... (`issue#10259 <http://tracker.ceph.com/issues/10259>`_, `pr#3827 <http://github.com/ceph/ceph/pull/3827>`_, Samuel Just)
+* osd: PG::actingset should be used when checking the number of acting OSDs for... (`issue#11454 <http://tracker.ceph.com/issues/11454>`_, `pr#4453 <http://github.com/ceph/ceph/pull/4453>`_, Guang Yang)
+* osd: PG::all_unfound_are_queried_or_lost for non-existent osds (`issue#10976 <http://tracker.ceph.com/issues/10976>`_, `pr#4416 <http://github.com/ceph/ceph/pull/4416>`_, Mykola Golub)
+* osd: PG: always clear_primary_state (`issue#10059 <http://tracker.ceph.com/issues/10059>`_, `pr#3955 <http://github.com/ceph/ceph/pull/3955>`_, Samuel Just)
+* osd: PGLog.h: 279: FAILED assert(log.log.size() == log_keys_debug.size()) (`issue#10718 <http://tracker.ceph.com/issues/10718>`_, `pr#4382 <http://github.com/ceph/ceph/pull/4382>`_, Samuel Just)
+* osd: PGLog: include rollback_info_trimmed_to in (read|write)_log (`issue#10157 <http://tracker.ceph.com/issues/10157>`_, `pr#3964 <http://github.com/ceph/ceph/pull/3964>`_, Samuel Just)
+* osd: pg stuck stale after create with activation delay (`issue#11197 <http://tracker.ceph.com/issues/11197>`_, `pr#4384 <http://github.com/ceph/ceph/pull/4384>`_, Samuel Just)
+* osd: ReplicatedPG: fail a non-blocking flush if the object is being scrubbed (`issue#8011 <http://tracker.ceph.com/issues/8011>`_, `pr#3943 <http://github.com/ceph/ceph/pull/3943>`_, Samuel Just)
+* osd: ReplicatedPG::on_change: clean up callbacks_for_degraded_object (`issue#8753 <http://tracker.ceph.com/issues/8753>`_, `pr#3940 <http://github.com/ceph/ceph/pull/3940>`_, Samuel Just)
+* osd: ReplicatedPG::scan_range: an object can disappear between the list and t... (`issue#10150 <http://tracker.ceph.com/issues/10150>`_, `pr#3962 <http://github.com/ceph/ceph/pull/3962>`_, Samuel Just)
+* osd: requeue blocked op before flush it was blocked on (`issue#10512 <http://tracker.ceph.com/issues/10512>`_, `pr#3931 <http://github.com/ceph/ceph/pull/3931>`_, Sage Weil)
+* rgw: check for timestamp for s3 keystone auth (`issue#10062 <http://tracker.ceph.com/issues/10062>`_, `pr#3958 <http://github.com/ceph/ceph/pull/3958>`_, Abhishek Lekshmanan)
+* rgw: civetweb should use unique request id (`issue#11720 <http://tracker.ceph.com/issues/11720>`_, `pr#4780 <http://github.com/ceph/ceph/pull/4780>`_, Orit Wasserman)
+* rgw: don't allow negative / invalid content length (`issue#11890 <http://tracker.ceph.com/issues/11890>`_, `pr#4829 <http://github.com/ceph/ceph/pull/4829>`_, Yehuda Sadeh)
+* rgw: fail s3 POST auth if keystone not configured (`issue#10698 <http://tracker.ceph.com/issues/10698>`_, `pr#3966 <http://github.com/ceph/ceph/pull/3966>`_, Yehuda Sadeh)
+* rgw: flush xml header on get acl request (`issue#10106 <http://tracker.ceph.com/issues/10106>`_, `pr#3961 <http://github.com/ceph/ceph/pull/3961>`_, Yehuda Sadeh)
+* rgw: generate new tag for object when setting object attrs (`issue#11256 <http://tracker.ceph.com/issues/11256>`_, `pr#4571 <http://github.com/ceph/ceph/pull/4571>`_, Yehuda Sadeh)
+* rgw: generate the "Date" HTTP header for civetweb. (`issue#11871,11891 <http://tracker.ceph.com/issues/11871,11891>`_, `pr#4851 <http://github.com/ceph/ceph/pull/4851>`_, Radoslaw Zarzynski)
+* rgw: keystone token cache does not work correctly (`issue#11125 <http://tracker.ceph.com/issues/11125>`_, `pr#4414 <http://github.com/ceph/ceph/pull/4414>`_, Yehuda Sadeh)
+* rgw: merge manifests correctly when there's prefix override (`issue#11622 <http://tracker.ceph.com/issues/11622>`_, `pr#4697 <http://github.com/ceph/ceph/pull/4697>`_, Yehuda Sadeh)
+* rgw: send appropriate op to cancel bucket index pending operation (`issue#10770 <http://tracker.ceph.com/issues/10770>`_, `pr#3938 <http://github.com/ceph/ceph/pull/3938>`_, Yehuda Sadeh)
+* rgw: shouldn't need to disable rgw_socket_path if frontend is configured (`issue#11160 <http://tracker.ceph.com/issues/11160>`_, `pr#4275 <http://github.com/ceph/ceph/pull/4275>`_, Yehuda Sadeh)
+* rgw: Swift API. Dump container's custom metadata. (`issue#10665 <http://tracker.ceph.com/issues/10665>`_, `pr#3934 <http://github.com/ceph/ceph/pull/3934>`_, Dmytro Iurchenko)
+* rgw: Swift API. Support for X-Remove-Container-Meta-{key} header. (`issue#10475 <http://tracker.ceph.com/issues/10475>`_, `pr#3929 <http://github.com/ceph/ceph/pull/3929>`_, Dmytro Iurchenko)
+* rgw: use correct objv_tracker for bucket instance (`issue#11416 <http://tracker.ceph.com/issues/11416>`_, `pr#4379 <http://github.com/ceph/ceph/pull/4379>`_, Yehuda Sadeh)
+* tests: force checkout of submodules (`issue#11157 <http://tracker.ceph.com/issues/11157>`_, `pr#4079 <http://github.com/ceph/ceph/pull/4079>`_, Loic Dachary)
+* tools: Backport ceph-objectstore-tool changes to firefly (`issue#12327 <http://tracker.ceph.com/issues/12327>`_, `pr#3866 <http://github.com/ceph/ceph/pull/3866>`_, David Zafman)
+* tools: ceph-objectstore-tool: Output only unsupported features when incomatible (`issue#11176 <http://tracker.ceph.com/issues/11176>`_, `pr#4126 <http://github.com/ceph/ceph/pull/4126>`_, David Zafman)
+* tools: ceph-objectstore-tool: Use exit status 11 for incompatible import attemp... (`issue#11139 <http://tracker.ceph.com/issues/11139>`_, `pr#4129 <http://github.com/ceph/ceph/pull/4129>`_, David Zafman)
+* tools: Fix do_autogen.sh so that -L is allowed (`issue#11303 <http://tracker.ceph.com/issues/11303>`_, `pr#4247 <http://github.com/ceph/ceph/pull/4247>`_, Alfredo Deza)
+
+v0.80.9 Firefly
+===============
+
+This is a bugfix release for firefly.  It fixes a performance
+regression in librbd, an important CRUSH misbehavior (see below), and
+several RGW bugs.  We have also backported support for flock/fcntl
+locks to ceph-fuse and libcephfs.
+
+We recommend that all Firefly users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.9.txt>`.
+
+Adjusting CRUSH maps
+--------------------
+
+* This point release fixes several issues with CRUSH that trigger
+  excessive data migration when adjusting OSD weights.  These are most
+  obvious when a very small weight change (e.g., a change from 0 to
+  .01) triggers a large amount of movement, but the same set of bugs
+  can also lead to excessive (though less noticeable) movement in
+  other cases.
+
+  However, because the bug may already have affected your cluster,
+  fixing it may trigger movement *back* to the more correct location.
+  For this reason, you must manually opt-in to the fixed behavior.
+
+  In order to set the new tunable to correct the behavior::
+
+     ceph osd crush set-tunable straw_calc_version 1
+
+  Note that this change will have no immediate effect.  However, from
+  this point forward, any 'straw' bucket in your CRUSH map that is
+  adjusted will get non-buggy internal weights, and that transition
+  may trigger some rebalancing.
+
+  You can estimate how much rebalancing will eventually be necessary
+  on your cluster with::
+
+     ceph osd getcrushmap -o /tmp/cm
+     crushtool -i /tmp/cm --num-rep 3 --test --show-mappings > /tmp/a 2>&1
+     crushtool -i /tmp/cm --set-straw-calc-version 1 -o /tmp/cm2
+     crushtool -i /tmp/cm2 --reweight -o /tmp/cm2
+     crushtool -i /tmp/cm2 --num-rep 3 --test --show-mappings > /tmp/b 2>&1
+     wc -l /tmp/a                          # num total mappings
+     diff -u /tmp/a /tmp/b | grep -c ^+    # num changed mappings
+
+   Divide the number of changed lines by the total number of lines in
+   /tmp/a.  We've found that most clusters are under 10%.
+
+   You can force all of this rebalancing to happen at once with::
+
+     ceph osd crush reweight-all
+
+   Otherwise, it will happen at some unknown point in the future when
+   CRUSH weights are next adjusted.
+
+Notable Changes
+---------------
+
+* ceph-fuse: flock, fcntl lock support (Yan, Zheng, Greg Farnum)
+* crush: fix straw bucket weight calculation, add straw_calc_version tunable (#10095 Sage Weil)
+* crush: fix tree bucket (Rongzu Zhu)
+* crush: fix underflow of tree weights (Loic Dachary, Sage Weil)
+* crushtool: add --reweight (Sage Weil)
+* librbd: complete pending operations before losing image (#10299 Jason Dillaman)
+* librbd: fix read caching performance regression (#9854 Jason Dillaman)
+* librbd: gracefully handle deleted/renamed pools (#10270 Jason Dillaman)
+* mon: fix dump of chooseleaf_vary_r tunable (Sage Weil)
+* osd: fix PG ref leak in snaptrimmer on peering (#10421 Kefu Chai)
+* osd: handle no-op write with snapshot (#10262 Sage Weil)
+* radosgw-admin: create subuser when creating user (#10103 Yehuda Sadeh)
+* rgw: change multipart uplaod id magic (#10271 Georgio Dimitrakakis, Yehuda Sadeh)
+* rgw: don't overwrite bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: enable IPv6 for embedded civetweb (#10965 Yehuda Sadeh)
+* rgw: fix partial swift GET (#10553 Yehuda Sadeh)
+* rgw: fix quota disable (#9907 Dong Lei)
+* rgw: index swift keys appropriately (#10471 Hemant Burman, Yehuda Sadeh)
+* rgw: make setattrs update bucket index (#5595 Yehuda Sadeh)
+* rgw: pass civetweb configurables (#10907 Yehuda Sadeh)
+* rgw: remove swift user manifest (DLO) hash calculation (#9973 Yehuda Sadeh)
+* rgw: return correct len for 0-len objects (#9877 Yehuda Sadeh)
+* rgw: S3 object copy content-type fix (#9478 Yehuda Sadeh)
+* rgw: send ETag on S3 object copy (#9479 Yehuda Sadeh)
+* rgw: send HTTP status reason explicitly in fastcgi (Yehuda Sadeh)
+* rgw: set ulimit -n from sysvinit (el6) init script (#9587 Sage Weil)
+* rgw: update swift subuser permission masks when authenticating (#9918 Yehuda Sadeh)
+* rgw: URL decode query params correctly (#10271 Georgio Dimitrakakis, Yehuda Sadeh)
+* rgw: use attrs when reading object attrs (#10307 Yehuda Sadeh)
+* rgw: use \r\n for http headers (#9254 Benedikt Fraunhofer, Yehuda Sadeh)
+
+
+v0.80.8 Firefly
+===============
+
+This is a long-awaited bugfix release for firefly.  It has several
+important (but relatively rare) OSD peering fixes, performance issues
+when snapshots are trimmed, several RGW fixes, a paxos corner case
+fix, and some packaging updates.
+
+We recommend that all users for v0.80.x firefly upgrade when it is
+convenient to do so.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.8.txt>`.
+
+Notable Changes
+---------------
+
+* build: remove stack-execute bit from assembled code sections (#10114 Dan Mick)
+* ceph-disk: fix dmcrypt key permissions (#9785 Loic Dachary)
+* ceph-disk: fix keyring location (#9653 Loic Dachary)
+* ceph-disk: make partition checks more robust (#9721 #9665 Loic Dachary)
+* ceph: cleanly shut down librados context on shutdown (#8797 Dan Mick)
+* common: add $cctid config metavariable (#6228 Adam Crume)
+* crush: align rule and ruleset ids (#9675 Xiaoxi Chen)
+* crush: fix negative weight bug during create_or_move_item (#9998 Pawel Sadowski)
+* crush: fix potential buffer overflow in erasure rules (#9492 Johnu George)
+* debian: fix python-ceph -> ceph file movement (Sage Weil)
+* libcephfs,ceph-fuse: fix flush tid wraparound bug (#9869 Greg Farnum, Yan, Zheng)
+* libcephfs: close fd befure umount (#10415 Yan, Zheng)
+* librados: fix crash from C API when read timeout is enabled (#9582 Sage Weil)
+* librados: handle reply race with pool deletion (#10372 Sage Weil)
+* librbd: cap memory utilization for read requests (Jason Dillaman)
+* librbd: do not close a closed parent image on failure (#10030 Jason Dillaman)
+* librbd: fix diff tests (#10002 Josh Durgin)
+* librbd: protect list_children from invalid pools (#10123 Jason Dillaman)
+* make check improvemens (Loic Dachary)
+* mds: fix ctime updates (#9514 Greg Farnum)
+* mds: fix journal import tool (#10025 John Spray)
+* mds: fix rare NULL deref in cap flush handler (Greg Farnum)
+* mds: handle unknown lock messages (Yan, Zheng)
+* mds: store backtrace for straydir (Yan, Zheng)
+* mon: abort startup if disk is full (#9502 Joao Eduardo Luis)
+* mon: add paxos instrumentation (Sage Weil)
+* mon: fix double-free in rare OSD startup path (Sage Weil)
+* mon: fix osdmap trimming (#9987 Sage Weil)
+* mon: fix paxos corner cases (#9301 #9053 Sage Weil)
+* osd: cancel callback on blacklisted watchers (#8315 Samuel Just)
+* osd: cleanly abort set-alloc-hint operations during upgrade (#9419 David Zafman)
+* osd: clear rollback PG metadata on PG deletion (#9293 Samuel Just)
+* osd: do not abort deep scrub if hinfo is missing (#10018 Loic Dachary)
+* osd: erasure-code regression tests (Loic Dachary)
+* osd: fix distro metadata reporting for SUSE (#8654 Danny Al-Gaaf)
+* osd: fix full OSD checks during backfill (#9574 Samuel Just)
+* osd: fix ioprio parsing (#9677 Loic Dachary)
+* osd: fix journal direct-io shutdown (#9073 Mark Kirkwood, Ma Jianpeng, Somnath Roy)
+* osd: fix journal dump (Ma Jianpeng)
+* osd: fix occasional stall during peering or activation (Sage Weil)
+* osd: fix past_interval display bug (#9752 Loic Dachary)
+* osd: fix rare crash triggered by admin socket dump_ops_in_filght (#9916 Dong Lei)
+* osd: fix snap trimming performance issues (#9487 #9113 Samuel Just, Sage Weil, Dan van der Ster, Florian Haas)
+* osd: fix snapdir handling on cache eviction (#8629 Sage Weil)
+* osd: handle map gaps in map advance code (Sage Weil)
+* osd: handle undefined CRUSH results in interval check (#9718 Samuel Just)
+* osd: include shard in JSON dump of ghobject (#10063 Loic Dachary)
+* osd: make backfill reservation denial handling more robust (#9626 Samuel Just)
+* osd: make misdirected op checks handle EC + primary affinity (#9835 Samuel Just, Sage Weil)
+* osd: mount XFS with inode64 by default (Sage Weil)
+* osd: other misc bugs (#9821 #9875 Samuel Just)
+* rgw: add .log to default log path (#9353 Alexandre Marangone)
+* rgw: clean up fcgi request context (#10194 Yehuda Sadeh)
+* rgw: convet header underscores to dashes (#9206 Yehuda Sadeh)
+* rgw: copy object data if copy target is in different pool (#9039 Yehuda Sadeh)
+* rgw: don't try to authenticate CORS peflight request (#8718 Robert Hubbard, Yehuda Sadeh)
+* rgw: fix civetweb URL decoding (#8621 Yehuda Sadeh)
+* rgw: fix hash calculation during PUT (Yehuda Sadeh)
+* rgw: fix misc bugs (#9089 #9201 Yehuda Sadeh)
+* rgw: fix object tail test (#9226 Sylvain Munaut, Yehuda Sadeh)
+* rgw: make sysvinit script run rgw under systemd context as needed (#10125 Loic Dachary)
+* rgw: separate civetweb log from rgw log (Yehuda Sadeh)
+* rgw: set length for keystone token validations (#7796 Mark Kirkwood, Yehuda Sadeh)
+* rgw: subuser creation fixes (#8587 Yehuda Sadeh)
+* rpm: misc packaging improvements (Sandon Van Ness, Dan Mick, Erik Logthenberg, Boris Ranto)
+* rpm: use standard udev rules for CentOS7/RHEL7 (#9747 Loic Dachary)
+
+
+v0.80.7 Firefly
+===============
+
+This release fixes a few critical issues with v0.80.6, particularly
+with clusters running mixed versions.
+
+We recommend that all v0.80.x Firefly users upgrade to this release.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.7.txt>`.
+
+Notable Changes
+---------------
+
+* osd: fix invalid memory reference in log trimming (#9731 Samuel Just)
+* osd: fix use-after-free in cache tiering code (#7588 Sage Weil)
+* osd: remove bad backfill assertion for mixed-version clusters (#9696 Samuel Just)
+
+
+
+v0.80.6 Firefly
+===============
+
+This is a major bugfix release for firefly, fixing a range of issues
+in the OSD and monitor, particularly with cache tiering.  There are
+also important fixes in librados, with the watch/notify mechanism used
+by librbd, and in radosgw.
+
+A few pieces of new functionality of been backported, including improved
+'ceph df' output (view amount of writeable space per pool), support for
+non-default cluster names when using sysvinit or systemd, and improved
+(and fixed) support for dmcrypt.
+
+We recommend that all v0.80.x Firefly users upgrade to this release.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.6.txt>`.
+
+Notable Changes
+---------------
+
+* build: fix atomic64_t on i386 (#8969 Sage Weil)
+* build: fix build on alpha (Michael Cree, Dmitry Smirnov)
+* build: fix build on hppa (Dmitry Smirnov)
+* build: fix yasm detection on x32 arch (Sage Weil)
+* ceph-disk: fix 'list' function with dmcrypt (Sage Weil)
+* ceph-disk: fix dmcrypt support (Alfredo Deza)
+* ceph: allow non-default cluster to be specified (#8944)
+* common: fix dup log messages to mon (#9080 Sage Weil)
+* global: write pid file when -f is used (systemd, upstart) (Alexandre Oliva)
+* librados: fix crash when read timeout is enabled (#9362 Matthias Kiefer, Sage Weil)
+* librados: fix lock leaks in error paths (#9022 Pavan Rallabhandi)
+* librados: fix watch resend on PG acting set change (#9220 Samuel Just)
+* librados: python: fix aio_read handling with \0 (Mohammad Salehe)
+* librbd: add interface to invalidate cached data (Josh Durgin)
+* librbd: fix crash when using clone of flattened image (#8845 Josh Durgin)
+* librbd: fix error path cleanup on open (#8912 Josh Durgin)
+* librbd: fix null pointer check (Danny Al-Gaaf)
+* librbd: limit dirty object count (Haomai Wang)
+* mds: fix rstats for root and mdsdir (Yan, Zheng)
+* mon: add 'get' command for new cache tier pool properties (Joao Eduardo Luis)
+* mon: add 'osd pool get-quota' (#8523 Joao Eduardo Luis)
+* mon: add cluster fingerprint (Sage Weil)
+* mon: disallow nonsensical cache-mode transitions (#8155 Joao Eduardo Luis)
+* mon: fix cache tier rounding error on i386 (Sage Weil)
+* mon: fix occasional memory leak (#9176 Sage Weil)
+* mon: fix reported latency for 'osd perf' (#9269 Samuel Just)
+* mon: include 'max avail' in 'ceph df' output (Sage Weil, Xioaxi Chen)
+* mon: persistently mark pools where scrub may find incomplete clones (#8882 Sage Weil)
+* mon: preload erasure plugins (Loic Dachary)
+* mon: prevent cache-specific settings on non-tier pools (#8696 Joao Eduardo Luis)
+* mon: reduce log spam (Aanchal Agrawal, Sage Weil)
+* mon: warn when cache pools have no hit_sets enabled (Sage Weil)
+* msgr: fix trivial memory leak (Sage Weil)
+* osd: automatically scrub PGs with invalid stats (#8147 Sage Weil)
+* osd: avoid sharing PG metadata that is not durable (Samuel Just)
+* osd: cap hit_set size (#9339 Samuel Just)
+* osd: create default erasure profile if needed (#8601 Loic Dachary)
+* osd: dump tid as JSON int (not string)  where appropriate (Joao Eduardo Luis)
+* osd: encode blacklist in deterministic order (#9211 Sage Weil)
+* osd: fix behavior when cache tier has no hit_sets enabled (#8982 Sage Weil)
+* osd: fix cache tier flushing of snapshots (#9054 Samuel Just)
+* osd: fix cache tier op ordering when going from full to non-full (#8931 Sage Weil)
+* osd: fix crash on dup recovery reservation (#8863 Sage Weil)
+* osd: fix division by zero when pg_num adjusted with no OSDs (#9052 Sage Weil)
+* osd: fix hint crash in experimental keyvaluestore_dev backend (Hoamai Wang)
+* osd: fix leak in copyfrom cancellation (#8894 Samuel Just)
+* osd: fix locking for copyfrom finish (#8889 Sage Weil)
+* osd: fix long filename handling in backend (#8701 Sage Weil)
+* osd: fix min_size check with backfill (#9497 Samuel Just)
+* osd: fix mount/remount sync race (#9144 Sage Weil)
+* osd: fix object listing + erasure code bug (Guang Yang)
+* osd: fix race on reconnect to failed OSD (#8944 Greg Farnum)
+* osd: fix recovery reservation deadlock (Samuel Just)
+* osd: fix tiering agent arithmetic for negative values (#9082 Karan Singh)
+* osd: improve shutdown order (#9218 Sage Weil)
+* osd: improve subop discard logic (#9259 Samuel Just)
+* osd: introduce optional sleep, io priority for scrub and snap trim (Sage Weil)
+* osd: make scrub check for and remove stale erasure-coded objects (Samuel Just)
+* osd: misc fixes (#9481 #9482 #9179 Sameul Just)
+* osd: mix keyvaluestore_dev improvements (Haomai Wang)
+* osd: only require CRUSH features for rules that are used (#8963 Sage Weil)
+* osd: preload erasure plugins on startup (Loic Dachary)
+* osd: prevent PGs from falling behind when consuming OSDMaps (#7576 Sage Weil)
+* osd: prevent old clients from using tiered pools (#8714 Sage Weil)
+* osd: set min_size on erasure pools to data chunk count (Sage Weil)
+* osd: trim old erasure-coded objects more aggressively (Samuel Just)
+* rados: enforce erasure code alignment (Lluis Pamies-Juarez)
+* rgw: align object stripes with erasure pool alignment (#8442 Yehuda Sadeh)
+* rgw: don't send error body on HEAD for civetweb (#8539 Yehuda Sadeh)
+* rgw: fix crash in CORS preflight request (Yehuda Sadeh)
+* rgw: fix decoding of + in URL (#8702 Brian Rak)
+* rgw: fix object removal on object create (#8972 Patrycja Szabowska, Yehuda Sadeh)
+* systemd: use systemd-run when starting radosgw (JuanJose Galvez)
+* sysvinit: support non-default cluster name (Alfredo Deza)
+
+
+v0.80.5 Firefly
+===============
+
+This release fixes a few important bugs in the radosgw and fixes
+several packaging and environment issues, including OSD log rotation,
+systemd environments, and daemon restarts on upgrade.
+
+We recommend that all v0.80.x Firefly users upgrade, particularly if they
+are using upstart, systemd, or radosgw.
+
+Notable Changes
+---------------
+
+* ceph-dencoder: do not needlessly link to librgw, librados, etc. (Sage Weil)
+* do not needlessly link binaries to leveldb (Sage Weil)
+* mon: fix mon crash when no auth keys are present (#8851, Joao Eduardo Luis)
+* osd: fix cleanup (and avoid occasional crash) during shutdown (#7981, Sage Weil)
+* osd: fix log rotation under upstart (Sage Weil)
+* rgw: fix multipart upload when object has irregular size (#8846, Yehuda Sadeh, Sylvain Munaut)
+* rgw: improve bucket listing S3 compatibility (#8858, Yehuda Sadeh)
+* rgw: improve delimited bucket listing (Yehuda Sadeh)
+* rpm: do not restart daemons on upgrade (#8849, Alfredo Deza)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.5.txt>`.
+
+v0.80.4 Firefly
+===============
+
+This Firefly point release fixes an potential data corruption problem
+when ceph-osd daemons run on top of XFS and service Firefly librbd
+clients.  A recently added allocation hint that RBD utilizes triggers
+an XFS bug on some kernels (Linux 3.2, and likely others) that leads
+to data corruption and deep-scrub errors (and inconsistent PGs).  This
+release avoids the situation by disabling the allocation hint until we
+can validate which kernels are affected and/or are known to be safe to
+use the hint on.
+
+We recommend that all v0.80.x Firefly users urgently upgrade,
+especially if they are using RBD.
+
+Notable Changes
+---------------
+
+* osd: disable XFS extsize hint by default (#8830, Samuel Just)
+* rgw: fix extra data pool default name (Yehuda Sadeh)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.4.txt>`.
+
+
+v0.80.3 Firefly
+===============
+
+This is the third Firefly point release.  It includes a single fix
+for a radosgw regression that was discovered in v0.80.2 right after it
+was released.
+
+We recommend that all v0.80.x Firefly users upgrade.
+
+Notable Changes
+---------------
+
+* radosgw: fix regression in manifest decoding (#8804, Sage Weil)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.3.txt>`.
+
+
+v0.80.2 Firefly
+===============
+
+This is the second Firefly point release.  It contains a range of
+important fixes, including several bugs in the OSD cache tiering, some
+compatibility checks that affect upgrade situations, several radosgw
+bugs, and an irritating and unnecessary feature bit check that
+prevents older clients from communicating with a cluster with any
+erasure coded pools.
+
+One someone large change in this point release is that the ceph RPM
+package is separated into a ceph and ceph-common package, similar to
+Debian.  The ceph-common package contains just the client libraries
+without any of the server-side daemons.
+
+We recommend that all v0.80.x Firefly users skip this release and use
+v0.80.3.
+
+Notable Changes
+---------------
+
+* ceph-disk: better debug logging (Alfredo Deza)
+* ceph-disk: fix preparation of OSDs with dmcrypt (#6700, Stephen F Taylor)
+* ceph-disk: partprobe on prepare to fix dm-crypt (#6966, Eric Eastman)
+* do not require ERASURE_CODE feature from clients (#8556, Sage Weil)
+* libcephfs-java: build with older JNI headers (Greg Farnum)
+* libcephfs-java: fix build with gcj-jdk (Dmitry Smirnov)
+* librados: fix osd op tid for redirected ops (#7588, Samuel Just)
+* librados: fix rados_pool_list buffer bounds checks (#8447, Sage Weil)
+* librados: resend ops when pool overlay changes (#8305, Sage Weil)
+* librbd, ceph-fuse: reduce CPU overhead for clean object check in cache (Haomai Wang)
+* mon: allow deletion of cephfs pools (John Spray)
+* mon: fix default pool ruleset choice (#8373, John Spray)
+* mon: fix health summary for mon low disk warning (Sage Weil)
+* mon: fix 'osd pool set <pool> cache_target_full_ratio' (Geoffrey Hartz)
+* mon: fix quorum feature check (Greg Farnum)
+* mon: fix request forwarding in mixed firefly+dumpling clusters 9#8727, Joao Eduardo Luis)
+* mon: fix rule vs ruleset check in 'osd pool set ... crush_ruleset' command (John Spray)
+* mon: make osd 'down' count accurate (Sage Weil)
+* mon: set 'next commit' in primary-affinity reply (Ilya Dryomov)
+* mon: verify CRUSH features are supported by all mons (#8738, Greg Farnum)
+* msgr: fix sequence negotiation during connection reset (Guang Yang)
+* osd: block scrub on blocked objects (#8011, Samuel Just)
+* osd: call XFS hint ioctl less often (#8241, Ilya Dryomov)
+* osd: copy xattr spill out marker on clone (Haomai Wang)
+* osd: fix flush of snapped objects (#8334, Samuel Just)
+* osd: fix hashindex restart of merge operation (#8332, Samuel Just)
+* osd: fix osdmap subscription bug causing startup hang (Greg Farnum)
+* osd: fix potential null deref (#8328, Sage Weil)
+* osd: fix shutdown race (#8319, Sage Weil)
+* osd: handle 'none' in CRUSH results properly during peering (#8507, Samuel Just)
+* osd: set no spill out marker on new objects (Greg Farnum)
+* osd: skip op ordering debug checks on tiered pools (#8380, Sage Weil)
+* rados: enforce 'put' alignment (Lluis Pamies-Juarez)
+* rest-api: fix for 'rx' commands (Ailing Zhang)
+* rgw: calc user manifest etag and fix check (#8169, #8436, Yehuda Sadeh)
+* rgw: fetch attrs on multipart completion (#8452, Yehuda Sadeh, Sylvain Munaut)
+* rgw: fix buffer overflow for long instance ids (#8608, Yehuda Sadeh)
+* rgw: fix entity permission check on metadata put (#8428, Yehuda Sadeh)
+* rgw: fix multipart retry race (#8269, Yehuda Sadeh)
+* rpm: split ceph into ceph and ceph-common RPMs (Sandon Van Ness, Dan Mick)
+* sysvinit: continue startin daemons after failure doing mount (#8554, Sage Weil)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.2.txt>`.
+
+v0.80.1 Firefly
+===============
+
+This first Firefly point release fixes a few bugs, the most visible
+being a problem that prevents scrub from completing in some cases.
+
+Notable Changes
+---------------
+
+* osd: revert incomplete scrub fix (Samuel Just)
+* rgw: fix stripe calculation for manifest objects (Yehuda Sadeh)
+* rgw: improve handling, memory usage for abort reads (Yehuda Sadeh)
+* rgw: send Swift user manifest HTTP header (Yehuda Sadeh)
+* libcephfs, ceph-fuse: expose MDS session state via admin socket (Yan, Zheng)
+* osd: add simple throttle for snap trimming (Sage Weil)
+* monclient: fix possible hang from ill-timed monitor connection failure (Sage Weil)
+* osd: fix trimming of past HitSets (Sage Weil)
+* osd: fix whiteouts for non-writeback cache modes (Sage Weil)
+* osd: prevent divide by zero in tiering agent (David Zafman)
+* osd: prevent busy loop when tiering agent can do no work (David Zafman)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.80.1.txt>`.
+
+
+v0.80 Firefly
+=============
+
+This release will form the basis for our long-term supported release
+Firefly, v0.80.x.  The big new features are support for erasure coding
+and cache tiering, although a broad range of other features, fixes,
+and improvements have been made across the code base.  Highlights include:
+
+* *Erasure coding*: support for a broad range of erasure codes for lower
+  storage overhead and better data durability.
+* *Cache tiering*: support for creating 'cache pools' that store hot,
+  recently accessed objects with automatic demotion of colder data to
+  a base tier.  Typically the cache pool is backed by faster storage
+  devices like SSDs.
+* *Primary affinity*: Ceph now has the ability to skew selection of
+  OSDs as the "primary" copy, which allows the read workload to be
+  cheaply skewed away from parts of the cluster without migrating any
+  data.
+* *Key/value OSD backend* (experimental): An alternative storage backend
+  for Ceph OSD processes that puts all data in a key/value database like
+  leveldb.  This provides better performance for workloads dominated by
+  key/value operations (like radosgw bucket indices).
+* *Standalone radosgw* (experimental): The radosgw process can now run
+  in a standalone mode without an apache (or similar) web server or
+  fastcgi.  This simplifies deployment and can improve performance.
+
+We expect to maintain a series of stable releases based on v0.80
+Firefly for as much as a year.  In the meantime, development of Ceph
+continues with the next release, Giant, which will feature work on the
+CephFS distributed file system, more alternative storage backends
+(like RocksDB and f2fs), RDMA support, support for pyramid erasure
+codes, and additional functionality in the block device (RBD) like
+copy-on-read and multisite mirroring.
+
+
+Upgrade Sequencing
+------------------
+
+* If your existing cluster is running a version older than v0.67
+  Dumpling, please first upgrade to the latest Dumpling release before
+  upgrading to v0.80 Firefly.  Please refer to the :ref:`dumpling-upgrade`
+  documentation.
+
+* We recommend adding the following to the [mon] section of your
+  ceph.conf prior to upgrade::
+
+    mon warn on legacy crush tunables = false
+
+  This will prevent health warnings due to the use of legacy CRUSH
+  placement.  Although it is possible to rebalance existing data
+  across your cluster (see the upgrade notes below), we do not
+  normally recommend it for production environments as a large amount
+  of data will move and there is a significant performance impact from
+  the rebalancing.
+
+* Upgrade daemons in the following order:
+
+    #. Monitors
+    #. OSDs
+    #. MDSs and/or radosgw
+
+  If the ceph-mds daemon is restarted first, it will wait until all
+  OSDs have been upgraded before finishing its startup sequence.  If
+  the ceph-mon daemons are not restarted prior to the ceph-osd
+  daemons, they will not correctly register their new capabilities
+  with the cluster and new features may not be usable until they are
+  restarted a second time.
+
+* Upgrade radosgw daemons together.  There is a subtle change in behavior
+  for multipart uploads that prevents a multipart request that was initiated
+  with a new radosgw from being completed by an old radosgw.
+
+
+Upgrading from v0.79
+--------------------
+
+* OSDMap's json-formatted dump changed for keys 'full' and 'nearfull'.
+  What was previously being outputted as 'true' or 'false' strings are
+  now being outputted 'true' and 'false' booleans according to json syntax.
+
+* HEALTH_WARN on 'mon osd down out interval == 0'. Having this option set
+  to zero on the leader acts much like having the 'noout' flag set.  This
+  warning will only be reported if the monitor getting the 'health' or
+  'status' request has this option set to zero.
+
+* Monitor 'auth' commands now require the mon 'x' capability.  This matches
+  dumpling v0.67.x and earlier, but differs from emperor v0.72.x.
+
+* A librados WATCH operation on a non-existent object now returns ENOENT;
+  previously it did not.
+
+* Librados interface change:  As there are no partial writes, the rados_write()
+  and rados_append() operations now return 0 on success like rados_write_full()
+  always has.  This includes the C++ interface equivalents and AIO return
+  values for the aio variants.
+
+* The radosgw init script (sysvinit) how requires that the 'host = ...' line in
+  ceph.conf, if present, match the short hostname (the output of 'hostname -s'),
+  not the fully qualified hostname or the (occasionally non-short) output of
+  'hostname'.  Failure to adjust this when upgrading from emperor or dumpling
+  may prevent the radosgw daemon from starting.
+
+Upgrading from v0.72 Emperor
+----------------------------
+
+* See notes above.
+
+* The 'ceph -s' or 'ceph status' command's 'num_in_osds' field in the
+  JSON and XML output has been changed from a string to an int.
+
+* The recently added 'ceph mds set allow_new_snaps' command's syntax
+  has changed slightly; it is now 'ceph mds set allow_new_snaps true'.
+  The 'unset' command has been removed; instead, set the value to
+  'false'.
+
+* The syntax for allowing snapshots is now 'mds set allow_new_snaps
+  <true|false>' instead of 'mds <set,unset> allow_new_snaps'.
+
+* 'rbd ls' on a pool which never held rbd images now exits with code
+  0. It outputs nothing in plain format, or an empty list in
+  non-plain format. This is consistent with the behavior for a pool
+  which used to hold images, but contains none. Scripts relying on
+  this behavior should be updated.
+
+* The MDS requires a new OSD operation TMAP2OMAP, added in this release.  When
+  upgrading, be sure to upgrade and restart the ceph-osd daemons before the
+  ceph-mds daemon.  The MDS will refuse to start if any up OSDs do not support
+  the new feature.
+
+* The 'ceph mds set_max_mds N' command is now deprecated in favor of
+  'ceph mds set max_mds N'.
+
+* The 'osd pool create ...' syntax has changed for erasure pools.
+
+* The default CRUSH rules and layouts are now using the 'bobtail'
+  tunables and defaults.  Upgraded clusters using the old values will
+  now present with a health WARN state.  This can be disabled by
+  adding 'mon warn on legacy crush tunables = false' to ceph.conf and
+  restarting the monitors.  Alternatively, you can switch to the new
+  tunables with 'ceph osd crush tunables firefly,' but keep in mind
+  that this will involve moving a *significant* portion of the data
+  already stored in the cluster and in a large cluster may take
+  several days to complete.  We do not recommend adjusting tunables on a
+  production cluster.
+
+* We now default to the 'bobtail' CRUSH tunable values that are first supported
+  by Ceph clients in bobtail (v0.56) and Linux kernel version v3.9.  If you
+  plan to access a newly created Ceph cluster with an older kernel client, you
+  should use 'ceph osd crush tunables legacy' to switch back to the legacy
+  behavior.  Note that making that change will likely result in some data
+  movement in the system, so adjust the setting before populating the new
+  cluster with data.
+
+* We now set the HASHPSPOOL flag on newly created pools (and new
+  clusters) by default.  Support for this flag first appeared in
+  v0.64; v0.67 Dumpling is the first major release that supports it.
+  It is first supported by the Linux kernel version v3.9.  If you plan
+  to access a newly created Ceph cluster with an older kernel or
+  clients (e.g, librados, librbd) from a pre-dumpling Ceph release,
+  you should add 'osd pool default flag hashpspool = false' to the
+  '[global]' section of your 'ceph.conf' prior to creating your
+  monitors (e.g., after 'ceph-deploy new' but before 'ceph-deploy mon
+  create ...').
+
+* The configuration option 'osd pool default crush rule' is deprecated
+  and replaced with 'osd pool default crush replicated ruleset'. 'osd
+  pool default crush rule' takes precedence for backward compatibility
+  and a deprecation warning is displayed when it is used.
+
+* As part of fix for #6796, 'ceph osd pool set <pool> <var> <arg>' now
+  receives <arg> as an integer instead of a string.  This affects how
+  'hashpspool' flag is set/unset: instead of 'true' or 'false', it now
+  must be '0' or '1'.
+
+* The behavior of the CRUSH 'indep' choose mode has been changed.  No
+  ceph cluster should have been using this behavior unless someone has
+  manually extracted a crush map, modified a CRUSH rule to replace
+  'firstn' with 'indep', recompiled, and reinjected the new map into
+  the cluster.  If the 'indep' mode is currently in use on a cluster,
+  the rule should be modified to use 'firstn' instead, and the
+  administrator should wait until any data movement completes before
+  upgrading.
+
+* The 'osd dump' command now dumps pool snaps as an array instead of an
+  object.
+
+
+Upgrading from v0.67 Dumpling
+-----------------------------
+
+* See notes above.
+
+* ceph-fuse and radosgw now use the same default values for the admin
+  socket and log file paths that the other daemons (ceph-osd,
+  ceph-mon, etc.) do.  If you run these daemons as non-root, you may
+  need to adjust your ceph.conf to disable these options or to adjust
+  the permissions on /var/run/ceph and /var/log/ceph.
+
+* The MDS now disallows snapshots by default as they are not
+  considered stable.  The command 'ceph mds set allow_snaps' will
+  enable them.
+
+* For clusters that were created before v0.44 (pre-argonaut, Spring
+  2012) and store radosgw data, the auto-upgrade from TMAP to OMAP
+  objects has been disabled.  Before upgrading, make sure that any
+  buckets created on pre-argonaut releases have been modified (e.g.,
+  by PUTing and then DELETEing an object from each bucket).  Any
+  cluster created with argonaut (v0.48) or a later release or not
+  using radosgw never relied on the automatic conversion and is not
+  affected by this change.
+
+* Any direct users of the 'tmap' portion of the librados API should be
+  aware that the automatic tmap -> omap conversion functionality has
+  been removed.
+
+* Most output that used K or KB (e.g., for kilobyte) now uses a
+  lower-case k to match the official SI convention.  Any scripts that
+  parse output and check for an upper-case K will need to be modified.
+
+* librados::Rados::pool_create_async() and librados::Rados::pool_delete_async()
+  don't drop a reference to the completion object on error, caller needs to take
+  care of that. This has never really worked correctly and we were leaking an
+  object
+
+* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the
+  specified location, as that's a job for 'ceph osd crush add'.  It will
+  however continue to work just the same as long as the osd already exists
+  in the crush map.
+
+* The OSD now enforces that class write methods cannot both mutate an
+  object and return data.  The rbd.assign_bid method, the lone
+  offender, has been removed.  This breaks compatibility with
+  pre-bobtail librbd clients by preventing them from creating new
+  images.
+
+* librados now returns on commit instead of ack for synchronous calls.
+  This is a bit safer in the case where both OSDs and the client crash, and
+  is probably how it should have been acting from the beginning. Users are
+  unlikely to notice but it could result in lower performance in some
+  circumstances. Those who care should switch to using the async interfaces,
+  which let you specify safety semantics precisely.
+
+* The C++ librados AioComplete::get_version() method was incorrectly
+  returning an int (usually 32-bits).  To avoid breaking library
+  compatibility, a get_version64() method is added that returns the
+  full-width value.  The old method is deprecated and will be removed
+  in a future release.  Users of the C++ librados API that make use of
+  the get_version() method should modify their code to avoid getting a
+  value that is truncated from 64 to to 32 bits.
+
+
+Notable changes since v0.79
+---------------------------
+
+* ceph-fuse, libcephfs: fix several caching bugs (Yan, Zheng)
+* ceph-fuse: trim inodes in response to mds memory pressure (Yan, Zheng)
+* librados: fix inconsistencies in API error values (David Zafman)
+* librados: fix watch operations with cache pools (Sage Weil)
+* librados: new snap rollback operation (David Zafman)
+* mds: fix respawn (John Spray)
+* mds: misc bugs (Yan, Zheng)
+* mds: misc multi-mds fixes (Yan, Zheng)
+* mds: use shared_ptr for requests (Greg Farnum)
+* mon: fix peer feature checks (Sage Weil)
+* mon: require 'x' mon caps for auth operations (Joao Luis)
+* mon: shutdown when removed from mon cluster (Joao Luis)
+* msgr: fix locking bug in authentication (Josh Durgin)
+* osd: fix bug in journal replay/restart (Sage Weil)
+* osd: many many many bug fixes with cache tiering (Samuel Just)
+* osd: track omap and hit_set objects in pg stats (Samuel Just)
+* osd: warn if agent cannot enable due to invalid (post-split) stats (Sage Weil)
+* rados bench: track metadata for multiple runs separately (Guang Yang)
+* rgw: fixed subuser modify (Yehuda Sadeh)
+* rpm: fix redhat-lsb dependency (Sage Weil, Alfredo Deza)
+
+
+Notable changes since v0.72 Emperor
+-----------------------------------
+
+* buffer: some zero-copy groundwork (Josh Durgin)
+* build: misc improvements (Ken Dreyer)
+* ceph-conf: stop creating bogus log files (Josh Durgin, Sage Weil)
+* ceph-crush-location: new hook for setting CRUSH location of osd daemons on start)
+* ceph-disk: avoid fd0 (Loic Dachary)
+* ceph-disk: generalize path names, add tests (Loic Dachary)
+* ceph-disk: misc improvements for puppet (Loic Dachary)
+* ceph-disk: several bug fixes (Loic Dachary)
+* ceph-fuse: fix race for sync reads (Sage Weil)
+* ceph-fuse, libcephfs: fix several caching bugs (Yan, Zheng)
+* ceph-fuse: trim inodes in response to mds memory pressure (Yan, Zheng)
+* ceph-kvstore-tool: expanded command set and capabilities (Joao Eduardo Luis)
+* ceph.spec: fix build dependency (Loic Dachary)
+* common: bloom filter improvements (Sage Weil)
+* common: check preexisting admin socket for active daemon before removing (Loic Dachary)
+* common: fix aligned buffer allocation (Loic Dachary)
+* common: fix authentication on big-endian architectures (Dan Mick)
+* common: fix config variable substitution (Loic Dachary)
+* common: portability changes to support libc++ (Noah Watkins)
+* common: switch to unordered_map from hash_map (Noah Watkins)
+* config: recursive metavariable expansion (Loic Dachary)
+* crush: default to bobtail tunables (Sage Weil)
+* crush: fix off-by-one error in recent refactor (Sage Weil)
+* crush: many additional tests (Loic Dachary)
+* crush: misc fixes, cleanups (Loic Dachary)
+* crush: new rule steps to adjust retry attempts (Sage Weil)
+* crush, osd: s/rep/replicated/ for less confusion (Loic Dachary)
+* crush: refactor descend_once behavior; support set_choose*_tries for replicated rules (Sage Weil)
+* crush: usability and test improvements (Loic Dachary)
+* debian: change directory ownership between ceph and ceph-common (Sage Weil)
+* debian: integrate misc fixes from downstream packaging (James Page)
+* doc: big update to install docs (John Wilkins)
+* doc: many many install doc improvements (John Wilkins)
+* doc: many many updates (John Wilkins)
+* doc: misc fixes (David Moreau Simard, Kun Huang)
+* erasure-code: improve buffer alignment (Loic Dachary)
+* erasure-code: rewrite region-xor using vector operations (Andreas Peters)
+* init: fix startup ordering/timeout problem with OSDs (Dmitry Smirnov)
+* libcephfs: fix resource leak (Zheng Yan)
+* librados: add C API coverage for atomic write operations (Christian Marie)
+* librados: fix inconsistencies in API error values (David Zafman)
+* librados: fix throttle leak (and eventual deadlock) (Josh Durgin)
+* librados: fix watch operations with cache pools (Sage Weil)
+* librados: new snap rollback operation (David Zafman)
+* librados, osd: new TMAP2OMAP operation (Yan, Zheng)
+* librados: read directly into user buffer (Rutger ter Borg)
+* librbd: fix use-after-free aio completion bug #5426 (Josh Durgin)
+* librbd: localize/distribute parent reads (Sage Weil)
+* librbd: skip zeroes/holes when copying sparse images (Josh Durgin)
+* mailmap: affiliation updates (Loic Dachary)
+* mailmap updates (Loic Dachary)
+* many portability improvements (Noah Watkins)
+* many unit test improvements (Loic Dachary)
+* mds: always store backtrace in default pool (Yan, Zheng)
+* mds: cope with MDS failure during creation (John Spray)
+* mds: fix cap migration behavior (Yan, Zheng)
+* mds: fix client session flushing (Yan, Zheng)
+* mds: fix crash from client sleep/resume (Zheng Yan)
+* mds: fix many many multi-mds bugs (Yan, Zheng)
+* mds: fix readdir end check (Zheng Yan)
+* mds: fix Resetter locking (Alexandre Oliva)
+* mds: fix respawn (John Spray)
+* mds: inline data support (Li Wang, Yunchuan Wen)
+* mds: misc bugs (Yan, Zheng)
+* mds: misc fixes for directory fragments (Zheng Yan)
+* mds: misc fixes for larger directories (Zheng Yan)
+* mds: misc fixes for multiple MDSs (Zheng Yan)
+* mds: misc multi-mds fixes (Yan, Zheng)
+* mds: remove .ceph directory (John Spray)
+* mds: store directories in omap instead of tmap (Yan, Zheng)
+* mds: update old-format backtraces opportunistically (Zheng Yan)
+* mds: use shared_ptr for requests (Greg Farnum)
+* misc cleanups from coverity (Xing Lin)
+* misc coverity fixes, cleanups (Danny Al-Gaaf)
+* misc coverity fixes (Xing Lin, Li Wang, Danny Al-Gaaf)
+* misc portability fixes (Noah Watkins, Alan Somers)
+* misc portability fixes (Noah Watkins, Christophe Courtaut, Alan Somers, huanjun)
+* misc portability work (Noah Watkins)
+* mon: add erasure profiles and improve erasure pool creation (Loic Dachary)
+* mon: add 'mon getmap EPOCH' (Joao Eduardo Luis)
+* mon: allow adjustment of cephfs max file size via 'ceph mds set max_file_size' (Sage Weil)
+* mon: allow debug quorum_{enter,exit} commands via admin socket
+* mon: 'ceph osd pg-temp ...' and primary-temp commands (Ilya Dryomov)
+* mon: change mds allow_new_snaps syntax to be more consistent (Sage Weil)
+* mon: clean up initial crush rule creation (Loic Dachary)
+* mon: collect misc metadata about osd (os, kernel, etc.), new 'osd metadata' command (Sage Weil)
+* mon: do not create erasure rules by default (Sage Weil)
+* mon: do not generate spurious MDSMaps in certain cases (Sage Weil)
+* mon: do not use keyring if auth = none (Loic Dachary)
+* mon: fix peer feature checks (Sage Weil)
+* mon: fix pg_temp leaks (Joao Eduardo Luis)
+* mon: fix pool count in 'ceph -s' output (Sage Weil)
+* mon: handle more whitespace (newline, tab) in mon capabilities (Sage Weil)
+* mon: improve (replicate or erasure) pool creation UX (Loic Dachary)
+* mon: infrastructure to handle mixed-version mon cluster and cli/rest API (Greg Farnum)
+* mon: MForward tests (Loic Dachary)
+* mon: mkfs now idempotent (Loic Dachary)
+* mon: only seed new osdmaps to current OSDs (Sage Weil)
+* mon, osd: create erasure style crush rules (Loic Dachary, Sage Weil)
+* mon: 'osd crush show-tunables' (Sage Weil)
+* mon: 'osd dump' dumps pool snaps as array, not object (Dan Mick)
+* mon, osd: new 'erasure' pool type (still not fully supported)
+* mon: persist quorum features to disk (Greg Farnum)
+* mon: prevent extreme changes in pool pg_num (Greg Farnum)
+* mon: require 'x' mon caps for auth operations (Joao Luis)
+* mon: shutdown when removed from mon cluster (Joao Luis)
+* mon: take 'osd pool set ...' value as an int, not string (Joao Eduardo Luis)
+* mon: track osd features in OSDMap (Joao Luis, David Zafman)
+* mon: trim MDSMaps (Joao Eduardo Luis)
+* mon: warn if crush has non-optimal tunables (Sage Weil)
+* mount.ceph: add -n for autofs support (Steve Stock)
+* msgr: fix locking bug in authentication (Josh Durgin)
+* msgr: fix messenger restart race (Xihui He)
+* msgr: improve connection error detection between clients and monitors (Greg Farnum, Sage Weil)
+* osd: add/fix CPU feature detection for jerasure (Loic Dachary)
+* osd: add HitSet tracking for read ops (Sage Weil, Greg Farnum)
+* osd: avoid touching leveldb for some xattrs (Haomai Wang, Sage Weil)
+* osd: backfill to multiple targets (David Zafman)
+* osd: backfill to osds not in acting set (David Zafman)
+* osd: cache pool support for snapshots (Sage Weil)
+* osd: client IO path changes for EC (Samuel Just)
+* osd: default to 3x replication
+* osd: do not include backfill targets in acting set (David Zafman)
+* osd: enable new hashpspool layout by default (Sage Weil)
+* osd: erasure plugin benchmarking tool (Loic Dachary)
+* osd: fix and cleanup misc backfill issues (David Zafman)
+* osd: fix bug in journal replay/restart (Sage Weil)
+* osd: fix copy-get omap bug (Sage Weil)
+* osd: fix linux kernel version detection (Ilya Dryomov)
+* osd: fix memstore segv (Haomai Wang)
+* osd: fix object_info_t encoding bug from emperor (Sam Just)
+* osd: fix omap_clear operation to not zap xattrs (Sam Just, Yan, Zheng)
+* osd: fix several bugs with tier infrastructure
+* osd: fix throttle thread (Haomai Wang)
+* osd: fix XFS detection (Greg Farnum, Sushma Gurram)
+* osd: generalize scrubbing infrastructure to allow EC (David Zafman)
+* osd: handle more whitespace (newline, tab) in osd capabilities (Sage Weil)
+* osd: ignore num_objects_dirty on scrub for old pools (Sage Weil)
+* osd: improved scrub checks on clones (Sage Weil, Sam Just)
+* osd: improve locking in fd lookup cache (Samuel Just, Greg Farnum)
+* osd: include more info in pg query result (Sage Weil)
+* osd, librados: fix full cluster handling (Josh Durgin)
+* osd: many erasure fixes (Sam Just)
+* osd: many many many bug fixes with cache tiering (Samuel Just)
+* osd: move to jerasure2 library (Loic Dachary)
+* osd: new 'chassis' type in default crush hierarchy (Sage Weil)
+* osd: new keyvaluestore-dev backend based on leveldb (Haomai Wang)
+* osd: new OSDMap encoding (Greg Farnum)
+* osd: new tests for erasure pools (David Zafman)
+* osd: preliminary cache pool support (no snaps) (Greg Farnum, Sage Weil)
+* osd: reduce scrub lock contention (Guang Yang)
+* osd: requery unfound on stray notify (#6909) (Samuel Just)
+* osd: some PGBackend infrastructure (Samuel Just)
+* osd: support for new 'memstore' (memory-backed) backend (Sage Weil)
+* osd: track erasure compatibility (David Zafman)
+* osd: track omap and hit_set objects in pg stats (Samuel Just)
+* osd: warn if agent cannot enable due to invalid (post-split) stats (Sage Weil)
+* rados: add 'crush location', smart replica selection/balancing (Sage Weil)
+* rados bench: track metadata for multiple runs separately (Guang Yang)
+* rados: some performance optimizations (Yehuda Sadeh)
+* rados tool: fix listomapvals (Josh Durgin)
+* rbd: add 'rbdmap' init script for mapping rbd images on book (Adam Twardowski)
+* rbd: add rbdmap support for upstart (Laurent Barbe)
+* rbd: expose kernel rbd client options via 'rbd map' (Ilya Dryomov)
+* rbd: fix bench-write command (Hoamai Wang)
+* rbd: make 'rbd list' return empty list and success on empty pool (Josh Durgin)
+* rbd: prevent deletion of images with watchers (Ilya Dryomov)
+* rbd: support for 4096 mapped devices, up from ~250 (Ilya Dryomov)
+* rest-api: do not fail when no OSDs yet exist (Dan Mick)
+* rgw: add 'status' command to sysvinit script (David Moreau Simard)
+* rgw: allow multiple frontends (Yehuda Sadeh)
+* rgw: allow use of an erasure data pool (Yehuda Sadeh)
+* rgw: convert bucket info to new format on demand (Yehuda Sadeh)
+* rgw: fixed subuser modify (Yehuda Sadeh)
+* rgw: fix error setting empty owner on ACLs (Yehuda Sadeh)
+* rgw: fix fastcgi deadlock (do not return data from librados callback) (Yehuda Sadeh)
+* rgw: fix many-part multipart uploads (Yehuda Sadeh)
+* rgw: fix misc CORS bugs (Robin H. Johnson)
+* rgw: fix object placement read op (Yehuda Sadeh)
+* rgw: fix reading bucket policy (#6940)
+* rgw: fix read_user_buckets 'max' behavior (Yehuda Sadeh)
+* rgw: fix several CORS bugs (Robin H. Johnson)
+* rgw: fix use-after-free when releasing completion handle (Yehuda Sadeh)
+* rgw: improve swift temp URL support (Yehuda Sadeh)
+* rgw: make multi-object delete idempotent (Yehuda Sadeh)
+* rgw: optionally defer to bucket ACLs instead of object ACLs (Liam Monahan)
+* rgw: prototype mongoose frontend (Yehuda Sadeh)
+* rgw: several doc fixes (Alexandre Marangone)
+* rgw: support for password (instead of admin token) for keystone authentication (Christophe Courtaut)
+* rgw: switch from mongoose to civetweb (Yehuda Sadeh)
+* rgw: user quotas (Yehuda Sadeh)
+* rpm: fix redhat-lsb dependency (Sage Weil, Alfredo Deza)
+* specfile: fix RPM build on RHEL6 (Ken Dreyer, Derek Yarnell)
+* specfile: ship libdir/ceph (Key Dreyer)
+* sysvinit, upstart: prevent both init systems from starting the same daemons (Josh Durgin)
+
+
+Notable changes since v0.67 Dumpling
+------------------------------------
+
+* build cleanly under clang (Christophe Courtaut)
+* build: Makefile refactor (Roald J. van Loon)
+* build: fix [/usr]/sbin locations (Alan Somers)
+* ceph-disk: fix journal preallocation
+* ceph-fuse, radosgw: enable admin socket and logging by default
+* ceph-fuse: fix problem with readahead vs truncate race (Yan, Zheng)
+* ceph-fuse: trim deleted inodes from cache (Yan, Zheng)
+* ceph-fuse: use newer fuse api (Jianpeng Ma)
+* ceph-kvstore-tool: new tool for working with leveldb (copy, crc) (Joao Luis)
+* ceph-post-file: new command to easily share logs or other files with ceph devs
+* ceph: improve parsing of CEPH_ARGS (Benoit Knecht)
+* ceph: make -h behave when monitors are down
+* ceph: parse CEPH_ARGS env variable
+* common: bloom_filter improvements, cleanups
+* common: cache crc32c values where possible
+* common: correct SI is kB not KB (Dan Mick)
+* common: fix looping on BSD (Alan Somers)
+* common: migrate SharedPtrRegistry to use boost::shared_ptr<> (Loic Dachary)
+* common: misc portability fixes (Noah Watkins)
+* crc32c: fix optimized crc32c code (it now detects arch support properly)
+* crc32c: improved intel-optimized crc32c support (~8x faster on my laptop!)
+* crush: fix name caching
+* doc: erasure coding design notes (Loic Dachary)
+* hadoop: removed old version of shim to avoid confusing users (Noah Watkins)
+* librados, mon: ability to query/ping out-of-quorum monitor status (Joao Luis)
+* librados: fix async aio completion wakeup
+* librados: fix installed header #includes (Dan Mick)
+* librados: get_version64() method for C++ API
+* librados: hello_world example (Greg Farnum)
+* librados: sync calls now return on commit (instead of ack) (Greg Farnum)
+* librbd python bindings: fix parent image name limit (Josh Durgin)
+* librbd, ceph-fuse: avoid some sources of ceph-fuse, rbd cache stalls
+* mds: avoid leaking objects when deleting truncated files (Yan, Zheng)
+* mds: fix F_GETLK (Yan, Zheng)
+* mds: fix LOOKUPSNAP bug
+* mds: fix heap profiler commands (Joao Luis)
+* mds: fix locking deadlock (David Disseldorp)
+* mds: fix many bugs with stray (unlinked) inodes (Yan, Zheng)
+* mds: fix many directory fragmentation bugs (Yan, Zheng)
+* mds: fix mds rejoin with legacy parent backpointer xattrs (Alexandre Oliva)
+* mds: fix rare restart/failure race during fs creation
+* mds: fix standby-replay when we fall behind (Yan, Zheng)
+* mds: fix stray directory purging (Yan, Zheng)
+* mds: notify clients about deleted files (so they can release from their cache) (Yan, Zheng)
+* mds: several bug fixes with clustered mds (Yan, Zheng)
+* mon, osd: improve osdmap trimming logic (Samuel Just)
+* mon, osd: initial CLI for configuring tiering
+* mon: a few 'ceph mon add' races fixed (command is now idempotent) (Joao Luis)
+* mon: allow (un)setting HASHPSPOOL flag on existing pools (Joao Luis)
+* mon: allow cap strings with . to be unquoted
+* mon: allow logging level of cluster log (/var/log/ceph/ceph.log) to be adjusted
+* mon: avoid rewriting full osdmaps on restart (Joao Luis)
+* mon: continue to discover peer addr info during election phase
+* mon: disallow CephFS snapshots until 'ceph mds set allow_new_snaps' (Greg Farnum)
+* mon: do not expose uncommitted state from 'osd crush {add,set} ...' (Joao Luis)
+* mon: fix 'ceph osd crush reweight ...' (Joao Luis)
+* mon: fix 'osd crush move ...' command for buckets (Joao Luis)
+* mon: fix byte counts (off by factor of 4) (Dan Mick, Joao Luis)
+* mon: fix paxos corner case
+* mon: kv properties for pools to support EC (Loic Dachary)
+* mon: make 'osd pool rename' idempotent (Joao Luis)
+* mon: modify 'auth add' semantics to make a bit more sense (Joao Luis)
+* mon: new 'osd perf' command to dump recent performance information (Samuel Just)
+* mon: new and improved 'ceph -s' or 'ceph status' command (more info, easier to read)
+* mon: some auth check cleanups (Joao Luis)
+* mon: track per-pool stats (Joao Luis)
+* mon: warn about pools with bad pg_num
+* mon: warn when mon data stores grow very large (Joao Luis)
+* monc: fix small memory leak
+* new wireshark patches pulled into the tree (Kevin Jones)
+* objecter, librados: redirect requests based on cache tier config
+* objecter: fix possible hang when cluster is unpaused (Josh Durgin)
+* osd, librados: add new COPY_FROM rados operation
+* osd, librados: add new COPY_GET rados operations (used by COPY_FROM)
+* osd: 'osd recover clone overlap limit' option to limit cloning during recovery (Samuel Just)
+* osd: COPY_GET on-wire encoding improvements (Greg Farnum)
+* osd: add 'osd heartbeat min healthy ratio' configurable (was hard-coded at 33%)
+* osd: add option to disable pg log debug code (which burns CPU)
+* osd: allow cap strings with . to be unquoted
+* osd: automatically detect proper xattr limits (David Zafman)
+* osd: avoid extra copy in erasure coding reference implementation (Loic Dachary)
+* osd: basic cache pool redirects (Greg Farnum)
+* osd: basic whiteout, dirty flag support (not yet used)
+* osd: bloom_filter encodability, fixes, cleanups (Loic Dachary, Sage Weil)
+* osd: clean up and generalize copy-from code (Greg Farnum)
+* osd: cls_hello OSD class example
+* osd: erasure coding doc updates (Loic Dachary)
+* osd: erasure coding plugin infrastructure, tests (Loic Dachary)
+* osd: experiemental support for ZFS (zfsonlinux.org) (Yan, Zheng)
+* osd: fix RWORDER flags
+* osd: fix exponential backoff of slow request warnings (Loic Dachary)
+* osd: fix handling of racing read vs write (Samuel Just)
+* osd: fix version value returned by various operations (Greg Farnum)
+* osd: generalized temp object infrastructure
+* osd: ghobject_t infrastructure for EC (David Zafman)
+* osd: improvements for compatset support and storage (David Zafman)
+* osd: infrastructure to copy objects from other OSDs
+* osd: instrument peering states (David Zafman)
+* osd: misc copy-from improvements
+* osd: opportunistic crc checking on stored data (off by default)
+* osd: properly enforce RD/WR flags for rados classes
+* osd: reduce blocking on backing fs (Samuel Just)
+* osd: refactor recovery using PGBackend (Samuel Just)
+* osd: remove old magical tmap->omap conversion
+* osd: remove old pg log on upgrade (Samuel Just)
+* osd: revert xattr size limit (fixes large rgw uploads)
+* osd: use fdatasync(2) instead of fsync(2) to improve performance (Sam Just)
+* pybind: fix blacklisting nonce (Loic Dachary)
+* radosgw-agent: multi-region replication/DR
+* rgw: complete in-progress requests before shutting down
+* rgw: default log level is now more reasonable (Yehuda Sadeh)
+* rgw: fix S3 auth with response-* query string params (Sylvain Munaut, Yehuda Sadeh)
+* rgw: fix a few minor memory leaks (Yehuda Sadeh)
+* rgw: fix acl group check (Yehuda Sadeh)
+* rgw: fix inefficient use of std::list::size() (Yehuda Sadeh)
+* rgw: fix major CPU utilization bug with internal caching (Yehuda Sadeh, Mark Nelson)
+* rgw: fix ordering of write operations (preventing data loss on crash) (Yehuda Sadeh)
+* rgw: fix ordering of writes for mulitpart upload (Yehuda Sadeh)
+* rgw: fix various CORS bugs (Yehuda Sadeh)
+* rgw: fix/improve swift COPY support (Yehuda Sadeh)
+* rgw: improve help output (Christophe Courtaut)
+* rgw: misc fixes to support DR (Josh Durgin, Yehuda Sadeh)
+* rgw: per-bucket quota (Yehuda Sadeh)
+* rgw: validate S3 tokens against keystone (Roald J. van Loon)
+* rgw: wildcard support for keystone roles (Christophe Courtaut)
+* rpm: fix junit dependencies (Alan Grosskurth)
+* sysvinit radosgw: fix status return code (Danny Al-Gaaf)
+* sysvinit rbdmap: fix error 'service rbdmap stop' (Laurent Barbe)
+* sysvinit: add condrestart command (Dan van der Ster)
+* sysvinit: fix shutdown order (mons last) (Alfredo Deza)
+
+
+v0.79
+=====
+
+This release is intended to serve as a release candidate for firefly,
+which will hopefully be v0.80.  No changes are being made to the code
+base at this point except those that fix bugs.  Please test this
+release if you intend to make use of the new erasure-coded pools or
+cache tiers in firefly.
+
+This release fixes a range of bugs found in v0.78 and streamlines the
+user experience when creating erasure-coded pools.  There is also a
+raft of fixes for the MDS (multi-mds, directory fragmentation, and
+large directories).  The main notable new piece of functionality is a
+small change to allow radosgw to use an erasure-coded pool for object
+data.
+
+
+Upgrading
+---------
+* Erasure pools created with v0.78 will no longer function with v0.79.  You
+  will need to delete the old pool and create a new one.
+
+* A bug was fixed in the authentication handshake with big-endian
+  architectures that prevent authentication between big- and
+  little-endian machines in the same cluster.  If you have a cluster
+  that consists entirely of big-endian machines, you will need to
+  upgrade all daemons and clients and restart.
+
+* The 'ceph.file.layout' and 'ceph.dir.layout' extended attributes are
+  no longer included in the listxattr(2) results to prevent problems with
+  'cp -a' and similar tools.
+
+* Monitor 'auth' read-only commands now expect the user to have 'rx' caps.
+  This is the same behavior that was present in dumpling, but in emperor
+  and more recent development releases the 'r' cap was sufficient.  The
+  affected commands are::
+
+    ceph auth export
+    ceph auth get
+    ceph auth get-key
+    ceph auth print-key
+    ceph auth list
+
+Notable Changes
+---------------
+* ceph-conf: stop creating bogus log files (Josh Durgin, Sage Weil)
+* common: fix authentication on big-endian architectures (Dan Mick)
+* debian: change directory ownership between ceph and ceph-common (Sage Weil)
+* init: fix startup ordering/timeout problem with OSDs (Dmitry Smirnov)
+* librbd: skip zeroes/holes when copying sparse images (Josh Durgin)
+* mds: cope with MDS failure during creation (John Spray)
+* mds: fix crash from client sleep/resume (Zheng Yan)
+* mds: misc fixes for directory fragments (Zheng Yan)
+* mds: misc fixes for larger directories (Zheng Yan)
+* mds: misc fixes for multiple MDSs (Zheng Yan)
+* mds: remove .ceph directory (John Spray)
+* misc coverity fixes, cleanups (Danny Al-Gaaf)
+* mon: add erasure profiles and improve erasure pool creation (Loic Dachary)
+* mon: 'ceph osd pg-temp ...' and primary-temp commands (Ilya Dryomov)
+* mon: fix pool count in 'ceph -s' output (Sage Weil)
+* msgr: improve connection error detection between clients and monitors (Greg Farnum, Sage Weil)
+* osd: add/fix CPU feature detection for jerasure (Loic Dachary)
+* osd: improved scrub checks on clones (Sage Weil, Sam Just)
+* osd: many erasure fixes (Sam Just)
+* osd: move to jerasure2 library (Loic Dachary)
+* osd: new tests for erasure pools (David Zafman)
+* osd: reduce scrub lock contention (Guang Yang)
+* rgw: allow use of an erasure data pool (Yehuda Sadeh)
+
+
+v0.78
+=====
+
+This development release includes two key features: erasure coding and
+cache tiering.  A huge amount of code was merged for this release and
+several additional weeks were spent stabilizing the code base, and it
+is now in a state where it is ready to be tested by a broader user
+base.
+
+This is *not* the firefly release.  Firefly will be delayed for at
+least another sprint so that we can get some operational experience
+with the new code and do some additional testing before committing to
+long term support.
+
+.. note:: Please note that while it is possible to create and test
+          erasure coded pools in this release, the pools will not be
+          usable when you upgrade to v0.79 as the OSDMap encoding will
+          subtlely change.  Please do not populate your test pools
+          with important data that can't be reloaded.
+
+Upgrading
+---------
+
+* Upgrade daemons in the following order:
+
+    #. Monitors
+    #. OSDs
+    #. MDSs and/or radosgw
+
+  If the ceph-mds daemon is restarted first, it will wait until all
+  OSDs have been upgraded before finishing its startup sequence.  If
+  the ceph-mon daemons are not restarted prior to the ceph-osd
+  daemons, they will not correctly register their new capabilities
+  with the cluster and new features may not be usable until they are
+  restarted a second time.
+
+* Upgrade radosgw daemons together.  There is a subtle change in behavior
+  for multipart uploads that prevents a multipart request that was initiated
+  with a new radosgw from being completed by an old radosgw.
+
+* CephFS recently added support for a new 'backtrace' attribute on
+  file data objects that is used for lookup by inode number (i.e., NFS
+  reexport and hard links), and will later be used by fsck repair.
+  This replaces the existing anchor table mechanism that is used for
+  hard link resolution.  In order to completely phase that out, any
+  inode that has an outdated backtrace attribute will get updated when
+  the inode itself is modified.  This will result in some extra workload
+  after a legacy CephFS file system is upgraded.
+
+* The per-op return code in librados' ObjectWriteOperation interface
+  is now filled in.
+
+* The librados cmpxattr operation now handles xattrs containing null bytes as
+  data rather than null-terminated strings.
+
+* Compound operations in librados that create and then delete the same object
+  are now explicitly disallowed (they fail with -EINVAL).
+
+* The default leveldb cache size for the ceph-osd daemon has been
+  increased from 4 MB to 128 MB.  This will increase the memory
+  footprint of that process but tends to increase performance of omap
+  (key/value) objects (used for CephFS and the radosgw).  If memory in your
+  deployment is tight, you can preserve the old behavior by adding::
+
+    leveldb write buffer size = 0
+    leveldb cache size = 0
+
+  to your ceph.conf to get back the (leveldb) defaults.
+
+Notable Changes
+---------------
+* ceph-brag: new client and server tools (Sebastien Han, Babu Shanmugam)
+* ceph-disk: use partx on RHEL or CentOS instead of partprobe (Alfredo Deza)
+* ceph: fix combination of 'tell' and interactive mode (Joao Eduardo Luis)
+* ceph-fuse: fix bugs with inline data and multiple MDSs (Zheng Yan)
+* client: fix getcwd() to use new LOOKUPPARENT operation (Zheng Yan)
+* common: fall back to json-pretty for admin socket (Loic Dachary)
+* common: fix 'config dump' debug prefix (Danny Al-Gaaf)
+* common: misc coverity fixes (Danny Al-Gaaf)
+* common: throtller, shared_cache performance improvements, TrackedOp (Greg Farnum, Samuel Just)
+* crush: fix JSON schema for dump (John Spray)
+* crush: misc cleanups, tests (Loic Dachary)
+* crush: new vary_r tunable (Sage Weil)
+* crush: prevent invalid buckets of type 0 (Sage Weil)
+* keyvaluestore: add perfcounters, misc bug fixes (Haomai Wang)
+* keyvaluestore: portability improvements (Noah Watkins)
+* libcephfs: API changes to better support NFS reexport via Ganesha (Matt Benjamin, Adam Emerson, Andrey Kuznetsov, Casey Bodley, David Zafman)
+* librados: API documentation improvements (John Wilkins, Josh Durgin)
+* librados: fix object enumeration bugs; allow iterator assignment (Josh Durgin)
+* librados: streamline tests (Josh Durgin)
+* librados: support for atomic read and omap operations for C API (Josh Durgin)
+* librados: support for osd and mon command timeouts (Josh Durgin)
+* librbd: pass allocation hints to OSD (Ilya Dryomov)
+* logrotate: fix bug that prevented rotation for some daemons (Loic Dachary)
+* mds: avoid duplicated discovers during recovery (Zheng Yan)
+* mds: fix file lock owner checks (Zheng Yan)
+* mds: fix LOOKUPPARENT, new LOOKUPNAME ops for reliable NFS reexport (Zheng Yan)
+* mds: fix xattr handling on setxattr (Zheng Yan)
+* mds: fix xattrs in getattr replies (Sage Weil)
+* mds: force backtrace updates for old inodes on update (Zheng Yan)
+* mds: several multi-mds and dirfrag bug fixes (Zheng Yan)
+* mon: encode erasure stripe width in pool metadata (Loic Dachary)
+* mon: erasure code crush rule creation (Loic Dachary)
+* mon: erasure code plugin support (Loic Dachary)
+* mon: fix bugs in initial post-mkfs quorum creation (Sage Weil)
+* mon: fix error output to terminal during startup (Joao Eduardo Luis)
+* mon: fix legacy CRUSH tunables warning (Sage Weil)
+* mon: fix osd_epochs lower bound tracking for map trimming (Sage Weil)
+* mon: fix OSDMap encoding features (Sage Weil, Aaron Ten Clay)
+* mon: fix 'pg dump' JSON output (John Spray)
+* mon: include dirty stats in 'ceph df detail' (Sage Weil)
+* mon: list quorum member names in quorum order (Sage Weil)
+* mon: prevent addition of non-empty cache tier (Sage Weil)
+* mon: prevent deletion of CephFS pools (John Spray)
+* mon: warn when cache tier approaches 'full' (Sage Weil)
+* osd: allocation hint, with XFS support (Ilya Dryomov)
+* osd: erasure coded pool support (Samuel Just)
+* osd: fix bug causing slow/stalled recovery (#7706) (Samuel Just)
+* osd: fix bugs in log merging (Samuel Just)
+* osd: fix/clarify end-of-object handling on read (Loic Dachary)
+* osd: fix impolite mon session backoff, reconnect behavior (Greg Farnum)
+* osd: fix SnapContext cache id bug (Samuel Just)
+* osd: increase default leveldb cache size and write buffer (Sage Weil, Dmitry Smirnov)
+* osd: limit size of 'osd bench ...' arguments (Joao Eduardo Luis)
+* osdmaptool: new --test-map-pgs mode (Sage Weil, Ilya Dryomov)
+* osd, mon: add primary-affinity to adjust selection of primaries (Sage Weil)
+* osd: new 'status' admin socket command (Sage Weil)
+* osd: simple tiering agent (Sage Weil)
+* osd: store checksums for erasure coded object stripes (Samuel Just)
+* osd: tests for objectstore backends (Haomai Wang)
+* osd: various refactoring and bug fixes (Samuel Just, David Zafman)
+* rados: add 'set-alloc-hint' command (Ilya Dryomov)
+* rbd-fuse: fix enumerate_images overflow, memory leak (Ilya Dryomov)
+* rbdmap: fix upstart script (Stephan Renatus)
+* rgw: avoid logging system events to usage log (Yehuda Sadeh)
+* rgw: fix Swift range reponse (Yehuda Sadeh)
+* rgw: improve scalability for manifest objects (Yehuda Sadeh)
+* rgw: misc fixes for multipart objects, policies (Yehuda Sadeh)
+* rgw: support non-standard MultipartUpload command (Yehuda Sadeh)
+
+
+
+v0.77
+=====
+
+This is the final development release before the Firefly feature
+freeze.  The main items in this release include some additional
+refactoring work in the OSD IO path (include some locking
+improvements), per-user quotas for the radosgw, a switch to civetweb
+from mongoose for the prototype radosgw standalone mode, and a
+prototype leveldb-based backend for the OSD.  The C librados API also
+got support for atomic write operations (read side transactions will
+appear in v0.78).
+
+Upgrading
+---------
+
+* The 'ceph -s' or 'ceph status' command's 'num_in_osds' field in the
+  JSON and XML output has been changed from a string to an int.
+
+* The recently added 'ceph mds set allow_new_snaps' command's syntax
+  has changed slightly; it is now 'ceph mds set allow_new_snaps true'.
+  The 'unset' command has been removed; instead, set the value to
+  'false'.
+
+* The syntax for allowing snapshots is now 'mds set allow_new_snaps
+  <true|false>' instead of 'mds <set,unset> allow_new_snaps'.
+
+Notable Changes
+---------------
+
+* osd: client IO path changes for EC (Samuel Just)
+* common: portability changes to support libc++ (Noah Watkins)
+* common: switch to unordered_map from hash_map (Noah Watkins)
+* rgw: switch from mongoose to civetweb (Yehuda Sadeh)
+* osd: improve locking in fd lookup cache (Samuel Just, Greg Farnum)
+* doc: many many updates (John Wilkins)
+* rgw: user quotas (Yehuda Sadeh)
+* mon: persist quorum features to disk (Greg Farnum)
+* mon: MForward tests (Loic Dachary)
+* mds: inline data support (Li Wang, Yunchuan Wen)
+* rgw: fix many-part multipart uploads (Yehuda Sadeh)
+* osd: new keyvaluestore-dev backend based on leveldb (Haomai Wang)
+* rbd: prevent deletion of images with watchers (Ilya Dryomov)
+* osd: avoid touching leveldb for some xattrs (Haomai Wang, Sage Weil)
+* mailmap: affiliation updates (Loic Dachary)
+* osd: new OSDMap encoding (Greg Farnum)
+* osd: generalize scrubbing infrastructure to allow EC (David Zafman)
+* rgw: several doc fixes (Alexandre Marangone)
+* librados: add C API coverage for atomic write operations (Christian Marie)
+* rgw: improve swift temp URL support (Yehuda Sadeh)
+* rest-api: do not fail when no OSDs yet exist (Dan Mick)
+* common: check preexisting admin socket for active daemon before removing (Loic Dachary)
+* osd: handle more whitespace (newline, tab) in osd capabilities (Sage Weil)
+* mon: handle more whitespace (newline, tab) in mon capabilities (Sage Weil)
+* rgw: make multi-object delete idempotent (Yehuda Sadeh)
+* crush: fix off-by-one error in recent refactor (Sage Weil)
+* rgw: fix read_user_buckets 'max' behavior (Yehuda Sadeh)
+* mon: change mds allow_new_snaps syntax to be more consistent (Sage Weil)
+
+
+v0.76
+=====
+
+This release includes another batch of updates for firefly
+functionality.  Most notably, the cache pool infrastructure now
+support snapshots, the OSD backfill functionality has been generalized
+to include multiple targets (necessary for the coming erasure pools),
+and there were performance improvements to the erasure code plugin on
+capable processors.  The MDS now properly utilizes (and seamlessly
+migrates to) the OSD key/value interface (aka omap) for storing directory
+objects.  There continue to be many other fixes and improvements for
+usability and code portability across the tree.
+
+Upgrading
+---------
+
+* 'rbd ls' on a pool which never held rbd images now exits with code
+  0. It outputs nothing in plain format, or an empty list in
+  non-plain format. This is consistent with the behavior for a pool
+  which used to hold images, but contains none. Scripts relying on
+  this behavior should be updated.
+
+* The MDS requires a new OSD operation TMAP2OMAP, added in this release.  When
+  upgrading, be sure to upgrade and restart the ceph-osd daemons before the
+  ceph-mds daemon.  The MDS will refuse to start if any up OSDs do not support
+  the new feature.
+
+* The 'ceph mds set_max_mds N' command is now deprecated in favor of
+  'ceph mds set max_mds N'.
+
+Notable Changes
+---------------
+
+* build: misc improvements (Ken Dreyer)
+* ceph-disk: generalize path names, add tests (Loic Dachary)
+* ceph-disk: misc improvements for puppet (Loic Dachary)
+* ceph-disk: several bug fixes (Loic Dachary)
+* ceph-fuse: fix race for sync reads (Sage Weil)
+* config: recursive metavariable expansion (Loic Dachary)
+* crush: usability and test improvements (Loic Dachary)
+* doc: misc fixes (David Moreau Simard, Kun Huang)
+* erasure-code: improve buffer alignment (Loic Dachary)
+* erasure-code: rewrite region-xor using vector operations (Andreas Peters)
+* librados, osd: new TMAP2OMAP operation (Yan, Zheng)
+* mailmap updates (Loic Dachary)
+* many portability improvements (Noah Watkins)
+* many unit test improvements (Loic Dachary)
+* mds: always store backtrace in default pool (Yan, Zheng)
+* mds: store directories in omap instead of tmap (Yan, Zheng)
+* mon: allow adjustment of cephfs max file size via 'ceph mds set max_file_size' (Sage Weil)
+* mon: do not create erasure rules by default (Sage Weil)
+* mon: do not generate spurious MDSMaps in certain cases (Sage Weil)
+* mon: do not use keyring if auth = none (Loic Dachary)
+* mon: fix pg_temp leaks (Joao Eduardo Luis)
+* osd: backfill to multiple targets (David Zafman)
+* osd: cache pool support for snapshots (Sage Weil)
+* osd: fix and cleanup misc backfill issues (David Zafman)
+* osd: fix omap_clear operation to not zap xattrs (Sam Just, Yan, Zheng)
+* osd: ignore num_objects_dirty on scrub for old pools (Sage Weil)
+* osd: include more info in pg query result (Sage Weil)
+* osd: track erasure compatibility (David Zafman)
+* rbd: make 'rbd list' return empty list and success on empty pool (Josh Durgin)
+* rgw: fix object placement read op (Yehuda Sadeh)
+* rgw: fix several CORS bugs (Robin H. Johnson)
+* specfile: fix RPM build on RHEL6 (Ken Dreyer, Derek Yarnell)
+* specfile: ship libdir/ceph (Key Dreyer)
+
+
+v0.75
+=====
+
+This is a big release, with lots of infrastructure going in for
+firefly.  The big items include a prototype standalone frontend for
+radosgw (which does not require apache or fastcgi), tracking for read
+activity on the osds (to inform tiering decisions), preliminary cache
+pool support (no snapshots yet), and lots of bug fixes and other work
+across the tree to get ready for the next batch of erasure coding
+patches.
+
+For comparison, here are the diff stats for the last few versions::
+
+ v0.75 291 files changed, 82713 insertions(+), 33495 deletions(-)
+ v0.74 192 files changed, 17980 insertions(+), 1062 deletions(-)
+ v0.73 148 files changed, 4464 insertions(+), 2129 deletions(-)
+
+Upgrading
+---------
+
+- The 'osd pool create ...' syntax has changed for erasure pools.
+
+- The default CRUSH rules and layouts are now using the latest and
+  greatest tunables and defaults.  Clusters using the old values will
+  now present with a health WARN state.  This can be disabled by
+  adding 'mon warn on legacy crush tunables = false' to ceph.conf.
+
+
+Notable Changes
+---------------
+
+* common: bloom filter improvements (Sage Weil)
+* common: fix config variable substitution (Loic Dachary)
+* crush, osd: s/rep/replicated/ for less confusion (Loic Dachary)
+* crush: refactor descend_once behavior; support set_choose*_tries for replicated rules (Sage Weil)
+* librados: fix throttle leak (and eventual deadlock) (Josh Durgin)
+* librados: read directly into user buffer (Rutger ter Borg)
+* librbd: fix use-after-free aio completion bug #5426 (Josh Durgin)
+* librbd: localize/distribute parent reads (Sage Weil)
+* mds: fix Resetter locking (Alexandre Oliva)
+* mds: fix cap migration behavior (Yan, Zheng)
+* mds: fix client session flushing (Yan, Zheng)
+* mds: fix many many multi-mds bugs (Yan, Zheng)
+* misc portability work (Noah Watkins)
+* mon, osd: create erasure style crush rules (Loic Dachary, Sage Weil)
+* mon: 'osd crush show-tunables' (Sage Weil)
+* mon: clean up initial crush rule creation (Loic Dachary)
+* mon: improve (replicate or erasure) pool creation UX (Loic Dachary)
+* mon: infrastructure to handle mixed-version mon cluster and cli/rest API (Greg Farnum)
+* mon: mkfs now idempotent (Loic Dachary)
+* mon: only seed new osdmaps to current OSDs (Sage Weil)
+* mon: track osd features in OSDMap (Joao Luis, David Zafman)
+* mon: warn if crush has non-optimal tunables (Sage Weil)
+* mount.ceph: add -n for autofs support (Steve Stock)
+* msgr: fix messenger restart race (Xihui He)
+* osd, librados: fix full cluster handling (Josh Durgin)
+* osd: add HitSet tracking for read ops (Sage Weil, Greg Farnum)
+* osd: backfill to osds not in acting set (David Zafman)
+* osd: enable new hashpspool layout by default (Sage Weil)
+* osd: erasure plugin benchmarking tool (Loic Dachary)
+* osd: fix XFS detection (Greg Farnum, Sushma Gurram)
+* osd: fix copy-get omap bug (Sage Weil)
+* osd: fix linux kernel version detection (Ilya Dryomov)
+* osd: fix memstore segv (Haomai Wang)
+* osd: fix several bugs with tier infrastructure
+* osd: fix throttle thread (Haomai Wang)
+* osd: preliminary cache pool support (no snaps) (Greg Farnum, Sage Weil)
+* rados tool: fix listomapvals (Josh Durgin)
+* rados: add 'crush location', smart replica selection/balancing (Sage Weil)
+* rados: some performance optimizations (Yehuda Sadeh)
+* rbd: add rbdmap support for upstart (Laurent Barbe)
+* rbd: expose kernel rbd client options via 'rbd map' (Ilya Dryomov)
+* rbd: fix bench-write command (Hoamai Wang)
+* rbd: support for 4096 mapped devices, up from ~250 (Ilya Dryomov)
+* rgw: allow multiple frontends (Yehuda Sadeh)
+* rgw: convert bucket info to new format on demand (Yehuda Sadeh)
+* rgw: fix misc CORS bugs (Robin H. Johnson)
+* rgw: prototype mongoose frontend (Yehuda Sadeh)
+
+
+
+v0.74
+=====
+
+This release includes a few substantial pieces for Firefly, including
+a long-overdue switch to 3x replication by default and a switch to the
+"new" CRUSH tunables by default (supported since bobtail).  There is
+also a fix for a long-standing radosgw bug (stalled GET) that has
+already been backported to emperor and dumpling.
+
+Upgrading
+---------
+
+* We now default to the 'bobtail' CRUSH tunable values that are first supported
+  by Ceph clients in bobtail (v0.56) and Linux kernel version v3.9.  If you
+  plan to access a newly created Ceph cluster with an older kernel client, you
+  should use 'ceph osd crush tunables legacy' to switch back to the legacy
+  behavior.  Note that making that change will likely result in some data
+  movement in the system, so adjust the setting before populating the new
+  cluster with data.
+
+* We now set the HASHPSPOOL flag on newly created pools (and new
+  clusters) by default.  Support for this flag first appeared in
+  v0.64; v0.67 Dumpling is the first major release that supports it.
+  It is first supported by the Linux kernel version v3.9.  If you plan
+  to access a newly created Ceph cluster with an older kernel or
+  clients (e.g, librados, librbd) from a pre-dumpling Ceph release,
+  you should add 'osd pool default flag hashpspool = false' to the
+  '[global]' section of your 'ceph.conf' prior to creating your
+  monitors (e.g., after 'ceph-deploy new' but before 'ceph-deploy mon
+  create ...').
+
+* The configuration option 'osd pool default crush rule' is deprecated
+  and replaced with 'osd pool default crush replicated ruleset'. 'osd
+  pool default crush rule' takes precedence for backward compatibility
+  and a deprecation warning is displayed when it is used.
+
+Notable Changes
+---------------
+
+* buffer: some zero-copy groundwork (Josh Durgin)
+* ceph-disk: avoid fd0 (Loic Dachary)
+* crush: default to bobtail tunables (Sage Weil)
+* crush: many additional tests (Loic Dachary)
+* crush: misc fixes, cleanups (Loic Dachary)
+* crush: new rule steps to adjust retry attempts (Sage Weil)
+* debian: integrate misc fixes from downstream packaging (James Page)
+* doc: big update to install docs (John Wilkins)
+* libcephfs: fix resource leak (Zheng Yan)
+* misc coverity fixes (Xing Lin, Li Wang, Danny Al-Gaaf)
+* misc portability fixes (Noah Watkins, Alan Somers)
+* mon, osd: new 'erasure' pool type (still not fully supported)
+* mon: add 'mon getmap EPOCH' (Joao Eduardo Luis)
+* mon: collect misc metadata about osd (os, kernel, etc.), new 'osd metadata' command (Sage Weil)
+* osd: default to 3x replication
+* osd: do not include backfill targets in acting set (David Zafman)
+* osd: new 'chassis' type in default crush hierarchy (Sage Weil)
+* osd: requery unfound on stray notify (#6909) (Samuel Just)
+* osd: some PGBackend infrastructure (Samuel Just)
+* osd: support for new 'memstore' (memory-backed) backend (Sage Weil)
+* rgw: fix fastcgi deadlock (do not return data from librados callback) (Yehuda Sadeh)
+* rgw: fix reading bucket policy (#6940)
+* rgw: fix use-after-free when releasing completion handle (Yehuda Sadeh)
+
+
+v0.73
+=====
+
+This release, the first development release after emperor, includes
+many bug fixes and a few additional pieces of functionality.  The
+first batch of larger changes will be landing in the next version,
+v0.74.
+
+Upgrading
+---------
+
+- As part of fix for #6796, 'ceph osd pool set <pool> <var> <arg>' now
+  receives <arg> as an integer instead of a string.  This affects how
+  'hashpspool' flag is set/unset: instead of 'true' or 'false', it now
+  must be '0' or '1'.
+
+- The behavior of the CRUSH 'indep' choose mode has been changed.  No
+  ceph cluster should have been using this behavior unless someone has
+  manually extracted a crush map, modified a CRUSH rule to replace
+  'firstn' with 'indep', recompiled, and reinjected the new map into
+  the cluster.  If the 'indep' mode is currently in use on a cluster,
+  the rule should be modified to use 'firstn' instead, and the
+  administrator should wait until any data movement completes before
+  upgrading.
+
+- The 'osd dump' command now dumps pool snaps as an array instead of an
+  object.
+
+- The radosgw init script (sysvinit) how requires that the 'host = ...' line in
+  ceph.conf, if present, match the short hostname (the output of 'hostname -s'),
+  not the fully qualified hostname or the (occasionally non-short) output of
+  'hostname'.  Failure to adjust this when upgrading from emperor or dumpling
+  may prevent the radosgw daemon from starting.
+
+
+Notable Changes
+---------------
+
+* ceph-crush-location: new hook for setting CRUSH location of osd daemons on start
+* ceph-kvstore-tool: expanded command set and capabilities (Joao Eduardo Luis)
+* ceph.spec: fix build dependency (Loic Dachary)
+* common: fix aligned buffer allocation (Loic Dachary)
+* doc: many many install doc improvements (John Wilkins)
+* mds: fix readdir end check (Zheng Yan)
+* mds: update old-format backtraces opportunistically (Zheng Yan)
+* misc cleanups from coverity (Xing Lin)
+* misc portability fixes (Noah Watkins, Christophe Courtaut, Alan Somers, huanjun)
+* mon: 'osd dump' dumps pool snaps as array, not object (Dan Mick)
+* mon: allow debug quorum_{enter,exit} commands via admin socket
+* mon: prevent extreme changes in pool pg_num (Greg Farnum)
+* mon: take 'osd pool set ...' value as an int, not string (Joao Eduardo Luis)
+* mon: trim MDSMaps (Joao Eduardo Luis)
+* osd: fix object_info_t encoding bug from emperor (Sam Just)
+* rbd: add 'rbdmap' init script for mapping rbd images on book (Adam Twardowski)
+* rgw: add 'status' command to sysvinit script (David Moreau Simard)
+* rgw: fix error setting empty owner on ACLs (Yehuda Sadeh)
+* rgw: optionally defer to bucket ACLs instead of object ACLs (Liam Monahan)
+* rgw: support for password (instead of admin token) for keystone authentication (Christophe Courtaut)
+* sysvinit, upstart: prevent both init systems from starting the same daemons (Josh Durgin)
diff --git a/doc/releases/general.rst b/doc/releases/general.rst
new file mode 100644
index 000000000..b7fdbb387
--- /dev/null
+++ b/doc/releases/general.rst
@@ -0,0 +1,78 @@
+.. _ceph-releases-general:
+
+=======================
+Ceph Releases (general)
+=======================
+
+.. toctree::
+    :maxdepth: 1
+
+Understanding the release cycle
+-------------------------------
+
+Starting with the Nautilus release (14.2.0), there is a new stable release cycle
+every year, targeting the month of March. Each stable release series will receive a name
+(e.g., 'Mimic') and a major release number (e.g., 13 for Mimic because 'M' is
+the 13th letter of the alphabet).
+
+Releases are named after a species of cephalopod (usually the common
+name, since the latin names are harder to remember or pronounce).
+
+Version numbers have three components, *x.y.z*.  *x* identifies the release
+cycle (e.g., 13 for Mimic).  *y* identifies the release type:
+
+* x.0.z - development versions
+* x.1.z - release candidates (for test clusters, brave users)
+* x.2.z - stable/bugfix releases (for users)
+
+Release candidates (x.1.z)
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There is a feature freeze roughly two months prior to the planned
+initial stable release, after which focus shifts to stabilization and
+bug fixes only.
+
+* Release candidate release every 1-2 weeks
+* Intended for final testing and validation of the upcoming stable release
+
+Stable releases (x.2.z)
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Once the initial stable release is made (x.2.0), there are
+semi-regular bug-fix point releases with bug fixes and (occasionally)
+feature backports.  Bug fixes are accumulated and included in
+the next point release.
+
+* Stable point release every 4 to 6 weeks
+* Intended for production deployments
+* Bug fix backports for 2 full release cycles (2 years).
+* Online, rolling upgrade support and testing from the last two (2)
+  stable release(s) (starting from Luminous).
+* Online, rolling upgrade support and testing from prior stable point
+  releases
+
+For each stable release:
+
+* `Integration and upgrade tests
+  <https://github.com/ceph/ceph/tree/master/qa/suites/>`_ are run on a regular basis
+  and `their results <http://pulpito.ceph.com/>`_ analyzed by Ceph
+  developers.
+* `Issues <http://tracker.ceph.com/projects/ceph/issues?query_id=27>`_
+  fixed in the development branch (master) are scheduled to be backported.
+* When an issue found in the stable release is `reported
+  <http://tracker.ceph.com/projects/ceph/issues/new>`_, it is
+  triaged by Ceph developers.
+* The `stable releases and backport team <http://tracker.ceph.com/projects/ceph-releases/wiki>`_
+  publishes ``point releases`` including fixes that have been backported to the stable release.
+
+Lifetime of stable releases
+---------------------------
+
+The lifetime of a stable release series is calculated to be approximately 24
+months (i.e., two 12 month release cycles) after the month of the first release.
+For example, Mimic (13.2.z) will reach end of life (EOL) shortly after Octopus
+(15.2.0) is released. The lifetime of a release may vary because it depends on
+how quickly the stable releases are published.
+
+Detailed information on all releases, past and present, can be found at :ref:`ceph-releases-index`
+
diff --git a/doc/releases/giant.rst b/doc/releases/giant.rst
new file mode 100644
index 000000000..6b2182adb
--- /dev/null
+++ b/doc/releases/giant.rst
@@ -0,0 +1,1286 @@
+=====
+Giant
+=====
+
+Giant is the 7th stable release of Ceph.  It is named after the giant
+squid (Architeuthis dux).
+
+
+v0.87.2 Giant
+=============
+
+This is the second (and possibly final) point release for Giant.
+
+We recommend all v0.87.x Giant users upgrade to this release.
+
+Notable Changes
+---------------
+
+* ceph-objectstore-tool: only output unsupported features when incompatible (#11176 David Zafman)
+* common: do not implicitly unlock rwlock on destruction (Federico Simoncelli)
+* common: make wait timeout on empty queue configurable (#10818 Samuel Just)
+* crush: pick ruleset id that matches and rule id (Xiaoxi Chen)
+* crush: set_choose_tries = 100 for new erasure code rulesets (#10353 Loic Dachary)
+* librados: check initialized atomic safely (#9617 Josh Durgin)
+* librados: fix failed tick_event assert (#11183 Zhiqiang Wang)
+* librados: fix looping on skipped maps (#9986 Ding Dinghua)
+* librados: fix op submit with timeout (#10340 Samuel Just)
+* librados: pybind: fix memory leak (#10723 Billy Olsen)
+* librados: pybind: keep reference to callbacks (#10775 Josh Durgin)
+* librados: translate operation flags from C APIs (Matthew Richards)
+* libradosstriper: fix write_full on ENOENT (#10758 Sebastien Ponce)
+* libradosstriper: use strtoll instead of strtol (Dongmao Zhang)
+* mds: fix assertion caused by system time moving backwards (#11053 Yan, Zheng)
+* mon: allow injection of random delays on writes (Joao Eduardo Luis)
+* mon: do not trust small osd epoch cache values (#10787 Sage Weil)
+* mon: fail non-blocking flush if object is being scrubbed (#8011 Samuel Just)
+* mon: fix division by zero in stats dump (Joao Eduardo Luis)
+* mon: fix get_rule_avail when no osds (#10257 Joao Eduardo Luis)
+* mon: fix timeout rounds period (#10546 Joao Eduardo Luis)
+* mon: ignore osd failures before up_from (#10762 Dan van der Ster, Sage Weil)
+* mon: paxos: reset accept timeout before writing to store (#10220 Joao Eduardo Luis)
+* mon: return if fs exists on 'fs new' (Joao Eduardo Luis)
+* mon: use EntityName when expanding profiles (#10844 Joao Eduardo Luis)
+* mon: verify cross-service proposal preconditions (#10643 Joao Eduardo Luis)
+* mon: wait for osdmon to be writeable when requesting proposal (#9794 Joao Eduardo Luis)
+* mount.ceph: avoid spurious error message about /etc/mtab (#10351 Yan, Zheng)
+* msg/simple: allow RESETSESSION when we forget an endpoint (#10080 Greg Farnum)
+* msg/simple: discard delay queue before incoming queue (#9910 Sage Weil)
+* osd: clear_primary_state when leaving Primary (#10059 Samuel Just)
+* osd: do not ignore deleted pgs on startup (#10617 Sage Weil)
+* osd: fix FileJournal wrap to get header out first (#10883 David Zafman)
+* osd: fix PG leak in SnapTrimWQ (#10421 Kefu Chai)
+* osd: fix journalq population in do_read_entry (#6003 Samuel Just)
+* osd: fix operator== for op_queue_age_hit and fs_perf_stat (#10259 Samuel Just)
+* osd: fix rare assert after split (#10430 David Zafman)
+* osd: get pgid ancestor from last_map when building past intervals (#10430 David Zafman)
+* osd: include rollback_info_trimmed_to in {read,write}_log (#10157 Samuel Just)
+* osd: lock header_lock in DBObjectMap::sync (#9891 Samuel Just)
+* osd: requeue blocked op before flush it was blocked on (#10512 Sage Weil)
+* osd: tolerate missing object between list and attr get on backfill (#10150 Samuel Just)
+* osd: use correct atime for eviction decision (Xinze Chi)
+* rgw: flush XML header on get ACL request (#10106 Yehuda Sadeh)
+* rgw: index swift keys appropriately (#10471 Hemant Bruman, Yehuda Sadeh)
+* rgw: send cancel for bucket index pending ops (#10770 Baijiaruo, Yehuda Sadeh)
+* rgw: swift: support X_Remove_Container-Meta-{key} (#01475 Dmytro Iurchenko)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.87.2.txt>`.
+
+v0.87.1 Giant
+=============
+
+This is the first (and possibly final) point release for Giant.  Our focus
+on stability fixes will be directed towards Hammer and Firefly.
+
+We recommend that all v0.87 Giant users upgrade to this release.
+
+Upgrading
+---------
+
+* Due to a change in the Linux kernel version 3.18 and the limits of the FUSE
+  interface, ceph-fuse needs be mounted as root on at least some systems. See
+  issues #9997, #10277, and #10542 for details.
+
+Notable Changes
+---------------
+
+* build: disable stack-execute bit on assembler objects (#10114 Dan Mick)
+* build: support boost 1.57.0 (#10688 Ken Dreyer)
+* ceph-disk: fix dmcrypt file permissions (#9785 Loic Dachary)
+* ceph-disk: run partprobe after zap, behave with partx or partprobe (#9665 #9721 Loic Dachary)
+* cephfs-journal-tool: fix import for aged journals (#9977 John Spray)
+* cephfs-journal-tool: fix journal import (#10025 John Spray)
+* ceph-fuse: use remount to trim kernel dcache (#10277 Yan, Zheng)
+* common: add cctid meta variable (#6228 Adam Crume)
+* common: fix dump of shard for ghobject_t (#10063 Loic Dachary)
+* crush: fix bucket weight underflow (#9998 Pawel Sadowski)
+* erasure-code: enforce chunk size alignment (#10211 Loic Dachary)
+* erasure-code: regression test suite (#9420 Loic Dachary)
+* erasure-code: relax caucy w restrictions (#10325 Loic Dachary)
+* libcephfs,ceph-fuse: allow xattr caps on inject_release_failure (#9800 John Spray)
+* libcephfs,ceph-fuse: fix cap flush tid comparison (#9869 Greg Farnum)
+* libcephfs,ceph-fuse: new flag to indicated sorted dcache (#9178 Yan, Zheng)
+* libcephfs,ceph-fuse: prune cache before reconnecting to MDS (Yan, Zheng)
+* librados: limit number of in-flight read requests (#9854 Jason Dillaman)
+* libradospy: fix thread shutdown (#8797 Dan Mick)
+* libradosstriper: fix locking issue in truncate (#10129 Sebastien Ponce)
+* librbd: complete pending ops before closing mage (#10299 Jason Dillaman)
+* librbd: fix error path on image open failure (#10030 Jason Dillaman)
+* librbd: gracefully handle deleted/renamed pools (#10270 Jason Dillaman)
+* librbd: handle errors when creating ioctx while listing children (#10123 Jason Dillaman)
+* mds: fix compat version in MClientSession (#9945 John Spray)
+* mds: fix journaler write error handling (#10011 John Spray)
+* mds: fix locking for file size recovery (#10229 Yan, Zheng)
+* mds: handle heartbeat_reset during shutdown (#10382 John Spray)
+* mds: store backtrace for straydir (Yan, Zheng)
+* mon: allow tiers for FS pools (#10135 John Spray)
+* mon: fix caching of last_epoch_clean, osdmap trimming (#9987 Sage Weil)
+* mon: fix 'fs ls' on peons (#10288 John Spray)
+* mon: fix MDS health status from peons (#10151 John Spray)
+* mon: fix paxos off-by-one (#9301 Sage Weil)
+* msgr: simple: do not block on takeover while holding global lock (#9921 Greg Farnum)
+* osd: deep scrub must not abort if hinfo is missing (#10018 Loic Dachary)
+* osd: fix misdirected op detection (#9835 Sage Weil)
+* osd: fix past_interval display for acting (#9752 Loic Dachary)
+* osd: fix PG peering backoff when behind on osdmaps (#10431 Sage Weil)
+* osd: handle no-op write with snapshot case (#10262 Ssage Weil)
+* osd: use fast-dispatch (Sage Weil, Greg Farnum)
+* rados: fix write to /dev/null (Loic Dachary)
+* radosgw-admin: create subuser when needed (#10103 Yehuda Sadeh)
+* rbd: avoid invalidating aio_write buffer during image import (#10590 Jason Dillaman)
+* rbd: fix export with images > 2GB (Vicente Cheng)
+* rgw: change multipart upload id magic (#10271 Georgios Dimitrakakis, Yehuda Sadeh)
+* rgw: check keystone auth for S3 POST (#10062 Abhishek Lekshmanan)
+* rgw: check timestamp for S3 keystone auth (#10062 Abhishek Lekshmanan)
+* rgw: fix partial GET with swift (#10553 Yehuda Sadeh)
+* rgw: fix quota disable (#9907 Dong Lei)
+* rgw: fix rare corruption of object metadata on put (#9576 Yehuda Sadeh)
+* rgw: fix S3 object copy content-type (#9478 Yehuda Sadeh)
+* rgw: headers end with \r\n (#9254 Benedikt Fraunhofer, Yehuda Sadeh)
+* rgw: remove swift user manifest DLO hash calculation (#9973 Yehuda Sadeh)
+* rgw: return correct len when len is 0 (#9877 Yehuda Sadeh)
+* rgw: return X-Timestamp field (#8911 Yehuda Sadeh)
+* rgw: run radosgw as apache with systemd (#10125)
+* rgw: sent ETag on S3 object copy (#9479 Yehuda Sadeh)
+* rgw: sent HTTP status reason explicitly in fastcgi (Yehuda Sadeh)
+* rgw: set length for keystone token validation (#7796 Mark Kirkwood, Yehuda Sadeh)
+* rgw: set ulimit -n on sysvinit before starting daemon (#9587 Sage Weil)
+* rgw: update bucket index on set_attrs (#5595 Yehuda Sadeh)
+* rgw: update swift subuser permission masks when authenticating (#9918 Yehuda Sadeh)
+* rgw: URL decode HTTP query params correction (#10271 Georgios Dimitrakakis, Yehuda Sadeh)
+* rgw: use cached attrs while reading object attrs (#10307 Yehuda Sadeh)
+* rgw: use strict_strtoll for content length (#10701 Axel Dunkel, Yehuda Sadeh)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.87.1.txt>`.
+
+
+
+v0.87 Giant
+===========
+
+This release will form the basis for the stable release Giant,
+v0.87.x.  Highlights for Giant include:
+
+* *RADOS Performance*: a range of improvements have been made in the
+  OSD and client-side librados code that improve the throughput on
+  flash backends and improve parallelism and scaling on fast machines.
+* *CephFS*: we have fixed a raft of bugs in CephFS and built some
+  basic journal recovery and diagnostic tools.  Stability and
+  performance of single-MDS systems is vastly improved in Giant.
+  Although we do not yet recommend CephFS for production deployments,
+  we do encourage testing for non-critical workloads so that we can
+  better gauge the feature, usability, performance, and stability
+  gaps.
+* *Local Recovery Codes*: the OSDs now support an erasure-coding scheme
+  that stores some additional data blocks to reduce the IO required to
+  recover from single OSD failures.
+* *Degraded vs misplaced*: the Ceph health reports from 'ceph -s' and
+  related commands now make a distinction between data that is
+  degraded (there are fewer than the desired number of copies) and
+  data that is misplaced (stored in the wrong location in the
+  cluster).  The distinction is important because the latter does not
+  compromise data safety.
+* *Tiering improvements*: we have made several improvements to the
+  cache tiering implementation that improve performance.  Most
+  notably, objects are not promoted into the cache tier by a single
+  read; they must be found to be sufficiently hot before that happens.
+* *Monitor performance*: the monitors now perform writes to the local
+  data store asynchronously, improving overall responsiveness.
+* *Recovery tools*: the ceph_objectstore_tool is greatly expanded to
+  allow manipulation of an individual OSDs data store for debugging
+  and repair purposes.  This is most heavily used by our QA
+  infrastructure to exercise recovery code.
+
+Upgrade Sequencing
+------------------
+
+* If your existing cluster is running a version older than v0.80.x
+  Firefly, please first upgrade to the latest Firefly release before
+  moving on to Giant.  We have not tested upgrades directly from
+  Emperor, Dumpling, or older releases.
+
+  We *have* tested:
+
+   * Firefly to Giant
+   * Dumpling to Firefly to Giant
+
+* Please upgrade daemons in the following order:
+
+   #. Monitors
+   #. OSDs
+   #. MDSs and/or radosgw
+
+  Note that the relative ordering of OSDs and monitors should not matter, but
+  we primarily tested upgrading monitors first.
+
+Upgrading from v0.80x Firefly
+-----------------------------
+
+* The client-side caching for librbd is now enabled by default (rbd
+  cache = true).  A safety option (rbd cache writethrough until flush
+  = true) is also enabled so that writeback caching is not used until
+  the library observes a 'flush' command, indicating that the librbd
+  users is passing that operation through from the guest VM.  This
+  avoids potential data loss when used with older versions of qemu
+  that do not support flush.
+
+    leveldb_write_buffer_size = 8*1024*1024  = 33554432   // 8MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+* The 'rados getxattr ...' command used to add a gratuitous newline to the attr
+  value; it now does not.
+
+* The ``*_kb perf`` counters on the monitor have been removed.  These are
+  replaced with a new set of ``*_bytes`` counters (e.g., ``cluster_osd_kb`` is
+  replaced by ``cluster_osd_bytes``).
+
+* The ``rd_kb`` and ``wr_kb`` fields in the JSON dumps for pool stats (accessed
+  via the ``ceph df detail -f json-pretty`` and related commands) have been
+  replaced with corresponding ``*_bytes`` fields.  Similarly, the
+  ``total_space``, ``total_used``, and ``total_avail`` fields are replaced with
+  ``total_bytes``, ``total_used_bytes``,  and ``total_avail_bytes`` fields.
+
+* The ``rados df --format=json`` output ``read_bytes`` and ``write_bytes``
+  fields were incorrectly reporting ops; this is now fixed.
+
+* The ``rados df --format=json`` output previously included ``read_kb`` and
+  ``write_kb`` fields; these have been removed.  Please use ``read_bytes`` and
+  ``write_bytes`` instead (and divide by 1024 if appropriate).
+
+* The experimental keyvaluestore-dev OSD backend had an on-disk format
+  change that prevents existing OSD data from being upgraded.  This
+  affects developers and testers only.
+
+* mon-specific and osd-specific leveldb options have been removed.
+  From this point onward users should use the `leveldb_*` generic
+  options and add the options in the appropriate sections of their
+  configuration files.  Monitors will still maintain the following
+  monitor-specific defaults:
+
+    leveldb_write_buffer_size = 8*1024*1024  = 33554432   // 8MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+* CephFS support for the legacy anchor table has finally been removed.
+  Users with file systems created before firefly should ensure that inodes
+  with multiple hard links are modified *prior* to the upgrade to ensure that
+  the backtraces are written properly.  For example::
+
+    sudo find /mnt/cephfs -type f -links +1 -exec touch \{\} \;
+
+* We disallow nonsensical 'tier cache-mode' transitions.  From this point
+  onward, 'writeback' can only transition to 'forward' and 'forward'
+  can transition to 1) 'writeback' if there are dirty objects, or 2) any if
+  there are no dirty objects.
+
+Notable Changes since v0.86
+---------------------------
+
+* ceph-disk: use new udev rules for centos7/rhel7 (#9747 Loic Dachary)
+* libcephfs-java: fix fstat mode (Noah Watkins)
+* librados: fix deadlock when listing PG contents (Guang Yang)
+* librados: misc fixes to the new threading model (#9582 #9706 #9845 #9873 Sage Weil)
+* mds: fix inotable initialization (Henry C Chang)
+* mds: gracefully handle unknown lock type in flock requests (Yan, Zheng)
+* mon: add read-only, read-write, and role-definer rols (Joao Eduardo Luis)
+* mon: fix mon cap checks (Joao Eduardo Luis)
+* mon: misc fixes for new paxos async writes (#9635 Sage Weil)
+* mon: set scrub timestamps on PG creation (#9496 Joao Eduardo Luis)
+* osd: erasure code: fix buffer alignment (Janne Grunau, Loic Dachary)
+* osd: fix alloc hint induced crashes on mixed clusters (#9419 David Zafman)
+* osd: fix backfill reservation release on rejection (#9626, Samuel Just)
+* osd: fix ioprio option parsing (#9676 #9677 Loic Dachary)
+* osd: fix memory leak during snap trimming (#9113 Samuel Just)
+* osd: misc peering and recovery fixes (#9614 #9696 #9731 #9718 #9821 #9875 Samuel Just, Guang Yang)
+
+Notable Changes since v0.80.x Firefly
+-------------------------------------
+
+* bash completion improvements (Wido den Hollander)
+* brag: fixes, improvements (Loic Dachary)
+* buffer: improve rebuild_page_aligned (Ma Jianpeng)
+* build: fix build on alpha (Michael Cree, Dmitry Smirnov)
+* build: fix CentOS 5 (Gerben Meijer)
+* build: fix yasm check for x32 (Daniel Schepler, Sage Weil)
+* ceph-brag: add tox tests (Alfredo Deza)
+* ceph-conf: flush log on exit (Sage Weil)
+* ceph.conf: update sample (Sebastien Han)
+* ceph-dencoder: refactor build a bit to limit dependencies (Sage Weil, Dan Mick)
+* ceph-disk: add Scientific Linux support (Dan van der Ster)
+* ceph-disk: do not inadvertantly create directories (Owne Synge)
+* ceph-disk: fix dmcrypt support (Sage Weil)
+* ceph-disk: fix dmcrypt support (Stephen Taylor)
+* ceph-disk: handle corrupt volumes (Stuart Longlang)
+* ceph-disk: linter cleanup, logging improvements (Alfredo Deza)
+* ceph-disk: partprobe as needed (Eric Eastman)
+* ceph-disk: show information about dmcrypt in 'ceph-disk list' output (Sage Weil)
+* ceph-disk: use partition type UUIDs and blkid (Sage Weil)
+* ceph: fix for non-default cluster names (#8944, Dan Mick)
+* ceph-fuse, libcephfs: asok hooks for handling session resets, timeouts (Yan, Zheng)
+* ceph-fuse, libcephfs: fix crash in trim_caps (John Spray)
+* ceph-fuse, libcephfs: improve cap trimming (John Spray)
+* ceph-fuse, libcephfs: improve traceless reply handling (Sage Weil)
+* ceph-fuse, libcephfs: virtual xattrs for rstat (Yan, Zheng)
+* ceph_objectstore_tool: vastly improved and extended tool for working offline with OSD data stores (David Zafman)
+* ceph.spec: many fixes (Erik Logtenberg, Boris Ranto, Dan Mick, Sandon Van Ness)
+* ceph.spec: split out ceph-common package, other fixes (Sandon Van Ness)
+* ceph_test_librbd_fsx: fix RNG, make deterministic (Ilya Dryomov)
+* cephtool: fix help (Yilong Zhao)
+* cephtool: refactor and improve CLI tests (Joao Eduardo Luis)
+* cephtool: test cleanup (Joao Eduardo Luis)
+* clang build fixes (John Spray, Danny Al-Gaaf)
+* client: improved MDS session dumps (John Spray)
+* common: add config diff admin socket command (Joao Eduardo Luis)
+* common: add rwlock assertion checks (Yehuda Sadeh)
+* common: fix dup log messages (#9080, Sage Weil)
+* common: perfcounters now use atomics and go faster (Sage Weil)
+* config: support G, M, K, etc. suffixes (Joao Eduardo Luis)
+* coverity cleanups (Danny Al-Gaaf)
+* crush: clean up CrushWrapper interface (Xioaxi Chen)
+* crush: include new tunables in dump (Sage Weil)
+* crush: make ruleset ids unique (Xiaoxi Chen, Loic Dachary)
+* crush: only require rule features if the rule is used (#8963, Sage Weil)
+* crushtool: send output to stdout, not stderr (Wido den Hollander)
+* doc: cache tiering (John Wilkins)
+* doc: CRUSH updates (John Wilkins)
+* doc: document new upstream wireshark dissector (Kevin Cox)
+* doc: improve manual install docs (Francois Lafont)
+* doc: keystone integration docs (John Wilkins)
+* doc: librados example fixes (Kevin Dalley)
+* doc: many doc updates (John Wilkins)
+* doc: many install doc updates (John Wilkins)
+* doc: misc updates (John Wilkins, Loic Dachary, David Moreau Simard, Wido den Hollander. Volker Voigt, Alfredo Deza, Stephen Jahl, Dan van der Ster)
+* doc: osd primary affinity (John Wilkins)
+* doc: pool quotas (John Wilkins)
+* doc: pre-flight doc improvements (Kevin Dalley)
+* doc: switch to an unencumbered font (Ross Turk)
+* doc: updated simple configuration guides (John Wilkins)
+* doc: update erasure docs (Loic Dachary, Venky Shankar)
+* doc: update openstack docs (Josh Durgin)
+* filestore: disable use of XFS hint (buggy on old kernels) (Samuel Just)
+* filestore: fix xattr spillout (Greg Farnum, Haomai Wang)
+* fix hppa arch build (Dmitry Smirnov)
+* fix i386 builds (Sage Weil)
+* fix struct vs class inconsistencies (Thorsten Behrens)
+* global: write pid file even when running in foreground (Alexandre Oliva)
+* hadoop: improve tests (Huamin Chen, Greg Farnum, John Spray)
+* hadoop: update hadoop tests for Hadoop 2.0 (Haumin Chen)
+* init-ceph: continue starting other daemons on crush or mount failure (#8343, Sage Weil)
+* journaler: fix locking (Zheng, Yan)
+* keyvaluestore: fix hint crash (#8381, Haomai Wang)
+* keyvaluestore: header cache (Haomai Wang)
+* libcephfs-java: build against older JNI headers (Greg Farnum)
+* libcephfs-java: fix gcj-jdk build (Dmitry Smirnov)
+* librados: fix crash on read op timeout (#9362 Matthias Kiefer, Sage Weil)
+* librados: fix lock leaks in error paths (#9022, Paval Rallabhandi)
+* librados: fix pool existence check (#8835, Pavan Rallabhandi)
+* librados: fix rados_pool_list bounds checks (Sage Weil)
+* librados: fix shutdown race (#9130 Sage Weil)
+* librados: fix watch/notify test (#7934 David Zafman)
+* librados: fix watch reregistration on acting set change (#9220 Samuel Just)
+* librados: give Objecter fine-grained locks (Yehuda Sadeh, Sage Weil, John Spray)
+* librados: lttng tracepoitns (Adam Crume)
+* librados, osd: return ETIMEDOUT on failed notify (Sage Weil)
+* librados: pybind: fix reads when \0 is present (#9547 Mohammad Salehe)
+* librados_striper: striping library for librados (Sebastien Ponce)
+* librbd, ceph-fuse: reduce cache flush overhead (Haomai Wang)
+* librbd: check error code on cache invalidate (Josh Durgin)
+* librbd: enable caching by default (Sage Weil)
+* librbd: enforce cache size on read requests (Jason Dillaman)
+* librbd: fix crash using clone of flattened image (#8845, Josh Durgin)
+* librbd: fix error path when opening image (#8912, Josh Durgin)
+* librbd: handle blacklisting during shutdown (#9105 John Spray)
+* librbd: lttng tracepoints (Adam Crume)
+* librbd: new libkrbd library for kernel map/unmap/showmapped (Ilya Dryomov)
+* librbd: store and retrieve snapshot metadata based on id (Josh Durgin)
+* libs3: update to latest (Danny Al-Gaaf)
+* log: fix derr level (Joao Eduardo Luis)
+* logrotate: fix osd log rotation on ubuntu (Sage Weil)
+* lttng: tracing infrastructure (Noah Watkins, Adam Crume)
+* mailmap: many updates (Loic Dachary)
+* mailmap: updates (Loic Dachary, Abhishek Lekshmanan, M Ranga Swami Reddy)
+* Makefile: fix out of source builds (Stefan Eilemann)
+* many many coverity fixes, cleanups (Danny Al-Gaaf)
+* mds: adapt to new Objecter locking, give types to all Contexts (John Spray)
+* mds: add file system name, enabled flag (John Spray)
+* mds: add internal health checks (John Spray)
+* mds: add min/max UID for snapshot creation/deletion (#9029, Wido den Hollander)
+* mds: avoid tight mon reconnect loop (#9428 Sage Weil)
+* mds: boot refactor, cleanup (John Spray)
+* mds: cephfs-journal-tool (John Spray)
+* mds: fix crash killing sessions (#9173 John Spray)
+* mds: fix ctime updates (#9514 Greg Farnum)
+* mds: fix journal conversion with standby-replay (John Spray)
+* mds: fix replay locking (Yan, Zheng)
+* mds: fix standby-replay cache trimming (#8648 Zheng, Yan)
+* mds: fix xattr bug triggered by ACLs (Yan, Zheng)
+* mds: give perfcounters meaningful names (Sage Weil)
+* mds: improve health reporting to monitor (John Spray)
+* mds: improve Journaler on-disk format (John Spray)
+* mds: improve journal locking (Zheng, Yan)
+* mds, libcephfs: use client timestamp for mtime/ctime (Sage Weil)
+* mds: make max file recoveries tunable (Sage Weil)
+* mds: misc encoding improvements (John Spray)
+* mds: misc fixes for multi-mds (Yan, Zheng)
+* mds: multi-mds fixes (Yan, Zheng)
+* mds: OPTracker integration, dump_ops_in_flight (Greg Farnum)
+* mds: prioritize file recovery when appropriate (Sage Weil)
+* mds: refactor beacon, improve reliability (John Spray)
+* mds: remove legacy anchor table (Yan, Zheng)
+* mds: remove legacy discover ino (Yan, Zheng)
+* mds: restart on EBLACKLISTED (John Spray)
+* mds: separate inode recovery queue (John Spray)
+* mds: session ls, evict commands (John Spray)
+* mds: submit log events in async thread (Yan, Zheng)
+* mds: track RECALL progress, report failure (#9284 John Spray)
+* mds: update segment references during journal write (John Spray, Greg Farnum)
+* mds: use client-provided timestamp for user-visible file metadata (Yan, Zheng)
+* mds: use meaningful names for clients (John Spray)
+* mds: validate journal header on load and save (John Spray)
+* mds: warn clients which aren't revoking caps (Zheng, Yan, John Spray)
+* misc build errors/warnings for Fedora 20 (Boris Ranto)
+* misc build fixes for OS X (John Spray)
+* misc cleanup (Christophe Courtaut)
+* misc integer size cleanups (Kevin Cox)
+* misc memory leaks, cleanups, fixes (Danny Al-Gaaf, Sahid Ferdjaoui)
+* misc suse fixes (Danny Al-Gaaf)
+* misc word size fixes (Kevin Cox)
+* mon: add audit log for all admin commands (Joao Eduardo Luis)
+* mon: add cluster fingerprint (Sage Weil)
+* mon: add get-quota commands (Joao Eduardo Luis)
+* mon: add 'osd blocked-by' command to easily see which OSDs are blocking peering progress (Sage Weil)
+* mon: add 'osd reweight-by-pg' command (Sage Weil, Guang Yang)
+* mon: add perfcounters for paxos operations (Sage Weil)
+* mon: avoid creating unnecessary rule on pool create (#9304 Loic Dachary)
+* monclient: fix hang (Sage Weil)
+* mon: create default EC profile if needed (Loic Dachary)
+* mon: do not create file system by default (John Spray)
+* mon: do not spam log (Aanchal Agrawal, Sage Weil)
+* mon: drop mon- and osd- specific leveldb options (Joao Eduardo Luis)
+* mon: ec pool profile fixes (Loic Dachary)
+* mon: fix bug when no auth keys are present (#8851, Joao Eduardo Luis)
+* mon: fix 'ceph df' output for available space (Xiaoxi Chen)
+* mon: fix compat version for MForward (Joao Eduardo Luis)
+* mon: fix crash on loopback messages and paxos timeouts (#9062, Sage Weil)
+* mon: fix default replication pool ruleset choice (#8373, John Spray)
+* mon: fix divide by zero when pg_num is adjusted before OSDs are added (#9101, Sage Weil)
+* mon: fix double-free of old MOSDBoot (Sage Weil)
+* mon: fix health down messages (Sage Weil)
+* mon: fix occasional memory leak after session reset (#9176, Sage Weil)
+* mon: fix op write latency perfcounter (#9217 Xinxin Shu)
+* mon: fix 'osd perf' reported latency (#9269 Samuel Just)
+* mon: fix quorum feature check (#8738, Greg Farnum)
+* mon: fix ruleset/ruleid bugs (#9044, Loic Dachary)
+* mon: fix set cache_target_full_ratio (#8440, Geoffrey Hartz)
+* mon: fix store check on startup (Joao Eduardo Luis)
+* mon: include per-pool 'max avail' in df output (Sage Weil)
+* mon: make paxos transaction commits asynchronous (Sage Weil)
+* mon: make usage dumps in terms of bytes, not kB (Sage Weil)
+* mon: 'osd crush reweight-subtree ...' (Sage Weil)
+* mon, osd: relax client EC support requirements (Sage Weil)
+* mon: preload erasure plugins (#9153 Loic Dachary)
+* mon: prevent cache pools from being used directly by CephFS (#9435 John Spray)
+* mon: prevent EC pools from being used with cephfs (Joao Eduardo Luis)
+* mon: prevent implicit destruction of OSDs with 'osd setmaxosd ...' (#8865, Anand Bhat)
+* mon: prevent nonsensical cache-mode transitions (Joao Eduardo Luis)
+* mon: restore original weight when auto-marked out OSDs restart (Sage Weil)
+* mon: restrict some pool properties to tiered pools (Joao Eduardo Luis)
+* mon: some instrumentation (Sage Weil)
+* mon: use msg header tid for MMonGetVersionReply (Ilya Dryomov)
+* mon: use user-provided ruleset for replicated pool (Xiaoxi Chen)
+* mon: verify all quorum members are contiguous at end of Paxos round (#9053, Sage Weil)
+* mon: verify available disk space on startup (#9502 Joao Eduardo Luis)
+* mon: verify erasure plugin version on load (Loic Dachary)
+* msgr: avoid big lock when sending (most) messages (Greg Farnum)
+* msgr: fix logged address (Yongyue Sun)
+* msgr: misc locking fixes for fast dispatch (#8891, Sage Weil)
+* msgr: refactor to cleanly separate SimpleMessenger implemenetation, move toward Connection-based calls (Matt Benjamin, Sage Wei)
+* objecter: flag operations that are redirected by caching (Sage Weil)
+* objectstore: clean up KeyValueDB interface for key/value backends (Sage Weil)
+* osd: account for hit_set_archive bytes (Sage Weil)
+* osd: add ability to prehash filestore directories (Guang Yang)
+* osd: add 'dump_reservations' admin socket command (Sage Weil)
+* osd: add feature bit for erasure plugins (Loic Dachary)
+* osd: add header cache for KeyValueStore (Haomai Wang)
+* osd: add ISA erasure plugin table cache (Andreas-Joachim Peters)
+* osd: add local_mtime for use by cache agent (Zhiqiang Wang)
+* osd: add local recovery code (LRC) erasure plugin (Loic Dachary)
+* osd: add prototype KineticStore based on Seagate Kinetic (Josh Durgin)
+* osd: add READFORWARD caching mode (Luis Pabon)
+* osd: add superblock for KeyValueStore backend (Haomai Wang)
+* osd: add support for Intel ISA-L erasure code library (Andreas-Joachim Peters)
+* osd: allow map cache size to be adjusted at runtime (Sage Weil)
+* osd: avoid refcounting overhead by passing a few things by ref (Somnath Roy)
+* osd: avoid sharing PG info that is not durable (Samuel Just)
+* osd: bound osdmap epoch skew between PGs (Sage Weil)
+* osd: cache tier flushing fixes for snapped objects (Samuel Just)
+* osd: cap hit_set size (#9339 Samuel Just)
+* osd: clean up shard_id_t, shard_t (Loic Dachary)
+* osd: clear FDCache on unlink (#8914 Loic Dachary)
+* osd: clear slow request latency info on osd up/down (Sage Weil)
+* osd: do not evict blocked objects (#9285 Zhiqiang Wang)
+* osd: do not skip promote for write-ordered reads (#9064, Samuel Just)
+* osd: fix agent early finish looping (David Zafman)
+* osd: fix ambigous encoding order for blacklisted clients (#9211, Sage Weil)
+* osd: fix bogus assert during OSD shutdown (Sage Weil)
+* osd: fix bug with long object names and rename (#8701, Sage Weil)
+* osd: fix cache flush corner case for snapshotted objects (#9054, Samuel Just)
+* osd: fix cache full -> not full requeueing (#8931, Sage Weil)
+* osd: fix clone deletion case (#8334, Sam Just)
+* osd: fix clone vs cache_evict bug (#8629 Sage Weil)
+* osd: fix connection reconnect race (Greg Farnum)
+* osd: fix crash from duplicate backfill reservation (#8863 Sage Weil)
+* osd: fix dead peer connection checks (#9295 Greg Farnum, Sage Weil)
+* osd: fix discard of old/obsolete subop replies (#9259, Samuel Just)
+* osd: fix discard of peer messages from previous intervals (Greg Farnum)
+* osd: fix dump of open fds on EMFILE (Sage Weil)
+* osd: fix dumps (Joao Eduardo Luis)
+* osd: fix erasure-code lib initialization (Loic Dachary)
+* osd: fix extent normalization (Adam Crume)
+* osd: fix filestore removal corner case (#8332, Sam Just)
+* osd: fix flush vs OpContext (Samuel Just)
+* osd: fix gating of messages from old OSD instances (Greg Farnum)
+* osd: fix hang waiting for osdmap (#8338, Greg Farnum)
+* osd: fix interval check corner case during peering (#8104, Sam Just)
+* osd: fix ISA erasure alignment (Loic Dachary, Andreas-Joachim Peters)
+* osd: fix journal dump (Ma Jianpeng)
+* osd: fix journal-less operation (Sage Weil)
+* osd: fix keyvaluestore scrub (#8589 Haomai Wang)
+* osd: fix keyvaluestore upgrade (Haomai Wang)
+* osd: fix loopback msgr issue (Ma Jianpeng)
+* osd: fix LSB release parsing (Danny Al-Gaaf)
+* osd: fix MarkMeDown and other shutdown races (Sage Weil)
+* osd: fix memstore bugs with collection_move_rename, lock ordering (Sage Weil)
+* osd: fix min_read_recency_for_promote default on upgrade (Zhiqiang Wang)
+* osd: fix mon feature bit requirements bug and resulting log spam (Sage Weil)
+* osd: fix mount/remount sync race (#9144 Sage Weil)
+* osd: fix PG object listing/ordering bug (Guang Yang)
+* osd: fix PG stat errors with tiering (#9082, Sage Weil)
+* osd: fix purged_snap initialization on backfill (Sage Weil, Samuel Just, Dan van der Ster, Florian Haas)
+* osd: fix race condition on object deletion (#9480 Somnath Roy)
+* osd: fix recovery chunk size usage during EC recovery (Ma Jianpeng)
+* osd: fix recovery reservation deadlock for EC pools (Samuel Just)
+* osd: fix removal of old xattrs when overwriting chained xattrs (Ma Jianpeng)
+* osd: fix requesting queueing on PG split (Samuel Just)
+* osd: fix scrub vs cache bugs (Samuel Just)
+* osd: fix snap object writeback from cache tier (#9054 Samuel Just)
+* osd: fix trim of hitsets (Sage Weil)
+* osd: force new xattrs into leveldb if fs returns E2BIG (#7779, Sage Weil)
+* osd: implement alignment on chunk sizes (Loic Dachary)
+* osd: improved backfill priorities (Sage Weil)
+* osd: improve journal shutdown (Ma Jianpeng, Mark Kirkwood)
+* osd: improve locking for KeyValueStore (Haomai Wang)
+* osd: improve locking in OpTracker (Pavan Rallabhandi, Somnath Roy)
+* osd: improve prioritization of recovery of degraded over misplaced objects (Sage Weil)
+* osd: improve tiering agent arithmetic (Zhiqiang Wang, Sage Weil, Samuel Just)
+* osd: include backend information in metadata reported to mon (Sage Weil)
+* osd: locking, sharding, caching improvements in FileStore's FDCache (Somnath Roy, Greg Farnum)
+* osd: lttng tracepoints for filestore (Noah Watkins)
+* osd: make blacklist encoding deterministic (#9211 Sage Weil)
+* osd: make tiering behave if hit_sets aren't enabled (Sage Weil)
+* osd: many important bug fixes (Samuel Just)
+* osd: many many core fixes (Samuel Just)
+* osd: many many important fixes (#8231 #8315 #9113 #9179 #9293 #9294 #9326 #9453 #9481 #9482 #9497 #9574 Samuel Just)
+* osd: mark pools with incomplete clones (Sage Weil)
+* osd: misc erasure code plugin fixes (Loic Dachary)
+* osd: misc locking fixes for fast dispatch (Samuel Just, Ma Jianpeng)
+* osd, mon: add rocksdb support (Xinxin Shu, Sage Weil)
+* osd, mon: config sanity checks on start (Sage Weil, Joao Eduardo Luis)
+* osd, mon: distinguish between "misplaced" and "degraded" objects in cluster health and PG state reporting (Sage Weil)
+* osd, msgr: fast-dispatch of OSD ops (Greg Farnum, Samuel Just)
+* osd, objecter: resend ops on last_force_op_resend barrier; fix cache overlay op ordering (Sage Weil)
+* osd: preload erasure plugins (#9153 Loic Dachary)
+* osd: prevent old rados clients from using tiered pools (#8714, Sage Weil)
+* osd: reduce OpTracker overhead (Somnath Roy)
+* osd: refactor some ErasureCode functionality into command parent class (Loic Dachary)
+* osd: remove obsolete classic scrub code (David Zafman)
+* osd: scrub PGs with invalid stats (Sage Weil)
+* osd: set configurable hard limits on object and xattr names (Sage Weil, Haomai Wang)
+* osd: set rollback_info_completed on create (#8625, Samuel Just)
+* osd: sharded threadpool to improve parallelism (Somnath Roy)
+* osd: shard OpTracker to improve performance (Somnath Roy)
+* osd: simple io prioritization for scrub (Sage Weil)
+* osd: simple scrub throttling (Sage Weil)
+* osd: simple snap trimmer throttle (Sage Weil)
+* osd: tests for bench command (Loic Dachary)
+* osd: trim old EC objects quickly; verify on scrub (Samuel Just)
+* osd: use FIEMAP to inform copy_range (Haomai Wang)
+* osd: use local time for tiering decisions (Zhiqiang Wang)
+* osd: use xfs hint less frequently (Ilya Dryomov)
+* osd: verify erasure plugin version on load (Loic Dachary)
+* osd: work around GCC 4.8 bug in journal code (Matt Benjamin)
+* pybind/rados: fix small timeouts (John Spray)
+* qa: xfstests updates (Ilya Dryomov)
+* rados: allow setxattr value to be read from stdin (Sage Weil)
+* rados bench: fix arg order (Kevin Dalley)
+* rados: drop gratuitous \n from getxattr command (Sage Weil)
+* rados: fix bench write arithmetic (Jiangheng)
+* rados: fix {read,write}_ops values for df output (Sage Weil)
+* rbd: add rbdmap pre- and post post- hooks, fix misc bugs (Dmitry Smirnov)
+* rbd-fuse: allow exposing single image (Stephen Taylor)
+* rbd-fuse: fix unlink (Josh Durgin)
+* rbd: improve option default behavior (Josh Durgin)
+* rbd: parallelize rbd import, export (Jason Dillaman)
+* rbd: rbd-replay utility to replay captured rbd workload traces (Adam Crume)
+* rbd: use write-back (not write-through) when caching is enabled (Jason Dillaman)
+* removed mkcephfs (deprecated since dumpling)
+* rest-api: fix help (Ailing Zhang)
+* rgw: add civetweb as default frontent on port 7490 (#9013 Yehuda Sadeh)
+* rgw: add --min-rewrite-stripe-size for object restriper (Yehuda Sadeh)
+* rgw: add powerdns hook for dynamic DNS for global clusters (Wido den Hollander)
+* rgw: add S3 bucket get location operation (Abhishek Lekshmanan)
+* rgw: allow : in S3 access key (Roman Haritonov)
+* rgw: automatically align writes to EC pool (#8442, Yehuda Sadeh)
+* rgw: bucket link uses instance id (Yehuda Sadeh)
+* rgw: cache bucket info (Yehuda Sadeh)
+* rgw: cache decoded user info (Yehuda Sadeh)
+* rgw: check entity permission for put_metadata (#8428, Yehuda Sadeh)
+* rgw: copy object data is target bucket is in a different pool (#9039, Yehuda Sadeh)
+* rgw: do not try to authenticate CORS preflight requests (#8718, Robert Hubbard, Yehuda Sadeh)
+* rgw: fix admin create user op (#8583 Ray Lv)
+* rgw: fix civetweb URL decoding (#8621, Yehuda Sadeh)
+* rgw: fix crash on swift CORS preflight request (#8586, Yehuda Sadeh)
+* rgw: fix log filename suffix (#9353 Alexandre Marangone)
+* rgw: fix memory leak following chunk read error (Yehuda Sadeh)
+* rgw: fix memory leaks (Andrey Kuznetsov)
+* rgw: fix multipart object attr regression (#8452, Yehuda Sadeh)
+* rgw: fix multipart upload (#8846, Silvain Munaut, Yehuda Sadeh)
+* rgw: fix radosgw-admin 'show log' command (#8553, Yehuda Sadeh)
+* rgw: fix removal of objects during object creation (Patrycja Szablowska, Yehuda Sadeh)
+* rgw: fix striping for copied objects (#9089, Yehuda Sadeh)
+* rgw: fix test for identify whether an object has a tail (#9226, Yehuda Sadeh)
+* rgw: fix URL decoding (#8702, Brian Rak)
+* rgw: fix URL escaping (Yehuda Sadeh)
+* rgw: fix usage (Abhishek Lekshmanan)
+* rgw: fix user manifest (Yehuda Sadeh)
+* rgw: fix when stripe size is not a multiple of chunk size (#8937, Yehuda Sadeh)
+* rgw: handle empty extra pool name (Yehuda Sadeh)
+* rgw: improve civetweb logging (Yehuda Sadeh)
+* rgw: improve delimited listing of bucket, misc fixes (Yehuda Sadeh)
+* rgw: improve -h (Abhishek Lekshmanan)
+* rgw: many fixes for civetweb (Yehuda Sadeh)
+* rgw: misc civetweb fixes (Yehuda Sadeh)
+* rgw: misc civetweb frontend fixes (Yehuda Sadeh)
+* rgw: object and bucket rewrite functions to allow restriping old objects (Yehuda Sadeh)
+* rgw: powerdns backend for global namespaces (Wido den Hollander)
+* rgw: prevent multiobject PUT race (Yehuda Sadeh)
+* rgw: send user manifest header (Yehuda Sadeh)
+* rgw: subuser creation fixes (#8587 Yehuda Sadeh)
+* rgw: use systemd-run from sysvinit script (JuanJose Galvez)
+* rpm: do not restart daemons on upgrade (Alfredo Deza)
+* rpm: misc packaging fixes for rhel7 (Sandon Van Ness)
+* rpm: split ceph-common from ceph (Sandon Van Ness)
+* systemd: initial systemd config files (Federico Simoncelli)
+* systemd: wrap started daemons in new systemd environment (Sage Weil, Dan Mick)
+* sysvinit: add support for non-default cluster names (Alfredo Deza)
+* sysvinit: less sensitive to failures (Sage Weil)
+* test_librbd_fsx: test krbd as well as librbd (Ilya Dryomov)
+* unit test improvements (Loic Dachary)
+* upstart: increase max open files limit (Sage Weil)
+* vstart.sh: fix/improve rgw support (Luis Pabon, Abhishek Lekshmanan)
+
+
+
+
+v0.86
+=====
+
+This is a release candidate for Giant, which will hopefully be out
+in another week or two.  We did a feature freeze about a month ago
+and since then have been doing only stabilization and bug fixing (and
+a handful on low-risk enhancements).  A fair bit of new functionality
+went into the final sprint, but it's baked for quite a while now and
+we're feeling pretty good about it.
+
+Major items include:
+
+* librados locking refactor to improve scaling and client performance
+* local recovery code (LRC) erasure code plugin to trade some
+  additional storage overhead for improved recovery performance
+* LTTNG tracing framework, with initial tracepoints in librados,
+  librbd, and the OSD FileStore backend
+* separate monitor audit log for all administrative commands
+* asynchronous monitor transaction commits to reduce the impact on
+  monitor read requests while processing updates
+* low-level tool for working with individual OSD data stores for
+  debugging, recovery, and testing
+* many MDS improvements (bug fixes, health reporting)
+
+There are still a handful of known bugs in this release, but nothing
+severe enough to prevent a release.  By and large we are pretty
+pleased with the stability and expect the final Giant release to be
+quite reliable.
+
+Please try this out on your non-production clusters for a preview
+
+Notable Changes
+---------------
+
+* buffer: improve rebuild_page_aligned (Ma Jianpeng)
+* build: fix CentOS 5 (Gerben Meijer)
+* build: fix build on alpha (Michael Cree, Dmitry Smirnov)
+* build: fix yasm check for x32 (Daniel Schepler, Sage Weil)
+* ceph-disk: add Scientific Linux support (Dan van der Ster)
+* ceph-fuse, libcephfs: fix crash in trim_caps (John Spray)
+* ceph-fuse, libcephfs: improve cap trimming (John Spray)
+* ceph-fuse, libcephfs: virtual xattrs for rstat (Yan, Zheng)
+* ceph.conf: update sample (Sebastien Han)
+* ceph.spec: many fixes (Erik Logtenberg, Boris Ranto, Dan Mick, Sandon Van Ness)
+* ceph_objectstore_tool: vastly improved and extended tool for working offline with OSD data stores (David Zafman)
+* common: add config diff admin socket command (Joao Eduardo Luis)
+* common: add rwlock assertion checks (Yehuda Sadeh)
+* crush: clean up CrushWrapper interface (Xioaxi Chen)
+* crush: make ruleset ids unique (Xiaoxi Chen, Loic Dachary)
+* doc: improve manual install docs (Francois Lafont)
+* doc: misc updates (John Wilkins, Loic Dachary, David Moreau Simard, Wido den Hollander. Volker Voigt, Alfredo Deza, Stephen Jahl, Dan van der Ster)
+* global: write pid file even when running in foreground (Alexandre Oliva)
+* hadoop: improve tests (Huamin Chen, Greg Farnum, John Spray)
+* journaler: fix locking (Zheng, Yan)
+* librados, osd: return ETIMEDOUT on failed notify (Sage Weil)
+* librados: fix crash on read op timeout (#9362 Matthias Kiefer, Sage Weil)
+* librados: fix shutdown race (#9130 Sage Weil)
+* librados: fix watch reregistration on acting set change (#9220 Samuel Just)
+* librados: fix watch/notify test (#7934 David Zafman)
+* librados: give Objecter fine-grained locks (Yehuda Sadeh, Sage Weil, John Spray)
+* librados: lttng tracepoitns (Adam Crume)
+* librados: pybind: fix reads when \0 is present (#9547 Mohammad Salehe)
+* librbd: enforce cache size on read requests (Jason Dillaman)
+* librbd: handle blacklisting during shutdown (#9105 John Spray)
+* librbd: lttng tracepoints (Adam Crume)
+* lttng: tracing infrastructure (Noah Watkins, Adam Crume)
+* mailmap: updates (Loic Dachary, Abhishek Lekshmanan, M Ranga Swami Reddy)
+* many many coverity fixes, cleanups (Danny Al-Gaaf)
+* mds: adapt to new Objecter locking, give types to all Contexts (John Spray)
+* mds: add internal health checks (John Spray)
+* mds: avoid tight mon reconnect loop (#9428 Sage Weil)
+* mds: fix crash killing sessions (#9173 John Spray)
+* mds: fix ctime updates (#9514 Greg Farnum)
+* mds: fix replay locking (Yan, Zheng)
+* mds: fix standby-replay cache trimming (#8648 Zheng, Yan)
+* mds: give perfcounters meaningful names (Sage Weil)
+* mds: improve health reporting to monitor (John Spray)
+* mds: improve journal locking (Zheng, Yan)
+* mds: make max file recoveries tunable (Sage Weil)
+* mds: prioritize file recovery when appropriate (Sage Weil)
+* mds: refactor beacon, improve reliability (John Spray)
+* mds: restart on EBLACKLISTED (John Spray)
+* mds: track RECALL progress, report failure (#9284 John Spray)
+* mds: update segment references during journal write (John Spray, Greg Farnum)
+* mds: use meaningful names for clients (John Spray)
+* mds: warn clients which aren't revoking caps (Zheng, Yan, John Spray)
+* mon: add 'osd reweight-by-pg' command (Sage Weil, Guang Yang)
+* mon: add audit log for all admin commands (Joao Eduardo Luis)
+* mon: add cluster fingerprint (Sage Weil)
+* mon: avoid creating unnecessary rule on pool create (#9304 Loic Dachary)
+* mon: do not spam log (Aanchal Agrawal, Sage Weil)
+* mon: fix 'osd perf' reported latency (#9269 Samuel Just)
+* mon: fix double-free of old MOSDBoot (Sage Weil)
+* mon: fix op write latency perfcounter (#9217 Xinxin Shu)
+* mon: fix store check on startup (Joao Eduardo Luis)
+* mon: make paxos transaction commits asynchronous (Sage Weil)
+* mon: preload erasure plugins (#9153 Loic Dachary)
+* mon: prevent cache pools from being used directly by CephFS (#9435 John Spray)
+* mon: use user-provided ruleset for replicated pool (Xiaoxi Chen)
+* mon: verify available disk space on startup (#9502 Joao Eduardo Luis)
+* mon: verify erasure plugin version on load (Loic Dachary)
+* msgr: fix logged address (Yongyue Sun)
+* osd: account for hit_set_archive bytes (Sage Weil)
+* osd: add ISA erasure plugin table cache (Andreas-Joachim Peters)
+* osd: add ability to prehash filestore directories (Guang Yang)
+* osd: add feature bit for erasure plugins (Loic Dachary)
+* osd: add local recovery code (LRC) erasure plugin (Loic Dachary)
+* osd: cap hit_set size (#9339 Samuel Just)
+* osd: clear FDCache on unlink (#8914 Loic Dachary)
+* osd: do not evict blocked objects (#9285 Zhiqiang Wang)
+* osd: fix ISA erasure alignment (Loic Dachary, Andreas-Joachim Peters)
+* osd: fix clone vs cache_evict bug (#8629 Sage Weil)
+* osd: fix crash from duplicate backfill reservation (#8863 Sage Weil)
+* osd: fix dead peer connection checks (#9295 Greg Farnum, Sage Weil)
+* osd: fix keyvaluestore scrub (#8589 Haomai Wang)
+* osd: fix keyvaluestore upgrade (Haomai Wang)
+* osd: fix min_read_recency_for_promote default on upgrade (Zhiqiang Wang)
+* osd: fix mount/remount sync race (#9144 Sage Weil)
+* osd: fix purged_snap initialization on backfill (Sage Weil, Samuel Just, Dan van der Ster, Florian Haas)
+* osd: fix race condition on object deletion (#9480 Somnath Roy)
+* osd: fix snap object writeback from cache tier (#9054 Samuel Just)
+* osd: improve journal shutdown (Ma Jianpeng, Mark Kirkwood)
+* osd: improve locking in OpTracker (Pavan Rallabhandi, Somnath Roy)
+* osd: improve tiering agent arithmetic (Zhiqiang Wang, Sage Weil, Samuel Just)
+* osd: lttng tracepoints for filestore (Noah Watkins)
+* osd: make blacklist encoding deterministic (#9211 Sage Weil)
+* osd: many many important fixes (#8231 #8315 #9113 #9179 #9293 #9294 #9326 #9453 #9481 #9482 #9497 #9574 Samuel Just)
+* osd: misc erasure code plugin fixes (Loic Dachary)
+* osd: preload erasure plugins (#9153 Loic Dachary)
+* osd: shard OpTracker to improve performance (Somnath Roy)
+* osd: use local time for tiering decisions (Zhiqiang Wang)
+* osd: verify erasure plugin version on load (Loic Dachary)
+* rados: fix bench write arithmetic (Jiangheng)
+* rbd: parallelize rbd import, export (Jason Dillaman)
+* rbd: rbd-replay utility to replay captured rbd workload traces (Adam Crume)
+* rbd: use write-back (not write-through) when caching is enabled (Jason Dillaman)
+* rgw: add S3 bucket get location operation (Abhishek Lekshmanan)
+* rgw: add civetweb as default frontent on port 7490 (#9013 Yehuda Sadeh)
+* rgw: allow : in S3 access key (Roman Haritonov)
+* rgw: fix admin create user op (#8583 Ray Lv)
+* rgw: fix log filename suffix (#9353 Alexandre Marangone)
+* rgw: fix usage (Abhishek Lekshmanan)
+* rgw: many fixes for civetweb (Yehuda Sadeh)
+* rgw: subuser creation fixes (#8587 Yehuda Sadeh)
+* rgw: use systemd-run from sysvinit script (JuanJose Galvez)
+* unit test improvements (Loic Dachary)
+* vstart.sh: fix/improve rgw support (Luis Pabon, Abhishek Lekshmanan)
+
+v0.85
+=====
+
+This is the second-to-last development release before Giant that
+contains new functionality.  The big items to land during this cycle
+are the messenger refactoring from Matt Benjamin that lays some
+groundwork for RDMA support, a performance improvement series from
+SanDisk that improves performance on SSDs, lots of improvements to our
+new standalone civetweb-based RGW frontend, and a new 'osd blocked-by'
+mon command that allows admins to easily identify which OSDs are
+blocking peering progress.  The other big change is that the OSDs and
+Monitors now distinguish between "misplaced" and "degraded" objects:
+the latter means there are fewer copies than we'd like, while the
+former simply means the are not stored in the locations where we want
+them to be.
+
+Also of note is a change to librbd that enables client-side caching by
+default.  This is coupled with another option that makes the cache
+write-through until a "flush" operations is observed: this implies
+that the librbd user (usually a VM guest OS) supports barriers and
+flush and that it is safe for the cache to switch into writeback mode
+without compromising data safety or integrity.  It has long been
+recommended practice that these options be enabled (e.g., in OpenStack
+environments) but until now it has not been the default.
+
+We have frozen the tree for the looming Giant release, and the next
+development release will be a release candidate with a final batch of
+new functionality.
+
+Upgrading
+---------
+
+* The client-side caching for librbd is now enabled by default (rbd
+  cache = true).  A safety option (rbd cache writethrough until flush
+  = true) is also enabled so that writeback caching is not used until
+  the library observes a 'flush' command, indicating that the librbd
+  users is passing that operation through from the guest VM.  This
+  avoids potential data loss when used with older versions of qemu
+  that do not support flush.
+
+    leveldb_write_buffer_size = 32*1024*1024  = 33554432  // 32MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+* The 'rados getxattr ...' command used to add a gratuitous newline to the attr
+  value; it now does not.
+
+Notable Changes
+---------------
+
+* ceph-disk: do not inadvertantly create directories (Owne Synge)
+* ceph-disk: fix dmcrypt support (Sage Weil)
+* ceph-disk: linter cleanup, logging improvements (Alfredo Deza)
+* ceph-disk: show information about dmcrypt in 'ceph-disk list' output (Sage Weil)
+* ceph-disk: use partition type UUIDs and blkid (Sage Weil)
+* ceph: fix for non-default cluster names (#8944, Dan Mick)
+* doc: document new upstream wireshark dissector (Kevin Cox)
+* doc: many install doc updates (John Wilkins)
+* librados: fix lock leaks in error paths (#9022, Paval Rallabhandi)
+* librados: fix pool existence check (#8835, Pavan Rallabhandi)
+* librbd: enable caching by default (Sage Weil)
+* librbd: fix crash using clone of flattened image (#8845, Josh Durgin)
+* librbd: store and retrieve snapshot metadata based on id (Josh Durgin)
+* mailmap: many updates (Loic Dachary)
+* mds: add min/max UID for snapshot creation/deletion (#9029, Wido den Hollander)
+* misc build errors/warnings for Fedora 20 (Boris Ranto)
+* mon: add 'osd blocked-by' command to easily see which OSDs are blocking peering progress (Sage Weil)
+* mon: add perfcounters for paxos operations (Sage Weil)
+* mon: create default EC profile if needed (Loic Dachary)
+* mon: fix crash on loopback messages and paxos timeouts (#9062, Sage Weil)
+* mon: fix divide by zero when pg_num is adjusted before OSDs are added (#9101, Sage Weil)
+* mon: fix occasional memory leak after session reset (#9176, Sage Weil)
+* mon: fix ruleset/ruleid bugs (#9044, Loic Dachary)
+* mon: make usage dumps in terms of bytes, not kB (Sage Weil)
+* mon: prevent implicit destruction of OSDs with 'osd setmaxosd ...' (#8865, Anand Bhat)
+* mon: verify all quorum members are contiguous at end of Paxos round (#9053, Sage Weil)
+* msgr: refactor to cleanly separate SimpleMessenger implemenetation, move toward Connection-based calls (Matt Benjamin, Sage Wei)
+* objectstore: clean up KeyValueDB interface for key/value backends (Sage Weil)
+* osd: add local_mtime for use by cache agent (Zhiqiang Wang)
+* osd: add superblock for KeyValueStore backend (Haomai Wang)
+* osd: add support for Intel ISA-L erasure code library (Andreas-Joachim Peters)
+* osd: do not skip promote for write-ordered reads (#9064, Samuel Just)
+* osd: fix ambigous encoding order for blacklisted clients (#9211, Sage Weil)
+* osd: fix cache flush corner case for snapshotted objects (#9054, Samuel Just)
+* osd: fix discard of old/obsolete subop replies (#9259, Samuel Just)
+* osd: fix discard of peer messages from previous intervals (Greg Farnum)
+* osd: fix dump of open fds on EMFILE (Sage Weil)
+* osd: fix journal dump (Ma Jianpeng)
+* osd: fix mon feature bit requirements bug and resulting log spam (Sage Weil)
+* osd: fix recovery chunk size usage during EC recovery (Ma Jianpeng)
+* osd: fix recovery reservation deadlock for EC pools (Samuel Just)
+* osd: fix removal of old xattrs when overwriting chained xattrs (Ma Jianpeng)
+* osd: fix requesting queueing on PG split (Samuel Just)
+* osd: force new xattrs into leveldb if fs returns E2BIG (#7779, Sage Weil)
+* osd: implement alignment on chunk sizes (Loic Dachary)
+* osd: improve prioritization of recovery of degraded over misplaced objects (Sage Weil)
+* osd: locking, sharding, caching improvements in FileStore's FDCache (Somnath Roy, Greg Farnum)
+* osd: many important bug fixes (Samuel Just)
+* osd, mon: add rocksdb support (Xinxin Shu, Sage Weil)
+* osd, mon: distinguish between "misplaced" and "degraded" objects in cluster health and PG state reporting (Sage Weil)
+* osd: refactor some ErasureCode functionality into command parent class (Loic Dachary)
+* osd: set rollback_info_completed on create (#8625, Samuel Just)
+* rados: allow setxattr value to be read from stdin (Sage Weil)
+* rados: drop gratuitous \n from getxattr command (Sage Weil)
+* rgw: add --min-rewrite-stripe-size for object restriper (Yehuda Sadeh)
+* rgw: add powerdns hook for dynamic DNS for global clusters (Wido den Hollander)
+* rgw: copy object data is target bucket is in a different pool (#9039, Yehuda Sadeh)
+* rgw: do not try to authenticate CORS preflight requests (#8718, Robert Hubbard, Yehuda Sadeh)
+* rgw: fix civetweb URL decoding (#8621, Yehuda Sadeh)
+* rgw: fix removal of objects during object creation (Patrycja Szablowska, Yehuda Sadeh)
+* rgw: fix striping for copied objects (#9089, Yehuda Sadeh)
+* rgw: fix test for identify whether an object has a tail (#9226, Yehuda Sadeh)
+* rgw: fix when stripe size is not a multiple of chunk size (#8937, Yehuda Sadeh)
+* rgw: improve civetweb logging (Yehuda Sadeh)
+* rgw: misc civetweb frontend fixes (Yehuda Sadeh)
+* sysvinit: add support for non-default cluster names (Alfredo Deza)
+
+
+v0.84
+=====
+
+The next Ceph development release is here!  This release contains
+several meaty items, including some MDS improvements for journaling,
+the ability to remove the CephFS file system (and name it), several
+mon cleanups with tiered pools, several OSD performance branches, a
+new "read forward" RADOS caching mode, a prototype Kinetic OSD
+backend, and various radosgw improvements (especially with the new
+standalone civetweb frontend).  And there are a zillion OSD bug
+fixes. Things are looking pretty good for the Giant release that is
+coming up in the next month.
+
+Upgrading
+---------
+
+* The ``*_kb perf`` counters on the monitor have been removed.  These are
+  replaced with a new set of ``*_bytes`` counters (e.g., ``cluster_osd_kb`` is
+  replaced by ``cluster_osd_bytes``).
+
+* The ``rd_kb`` and ``wr_kb`` fields in the JSON dumps for pool stats (accessed
+  via the ``ceph df detail -f json-pretty`` and related commands) have been
+  replaced with corresponding ``*_bytes`` fields.  Similarly, the
+  ``total_space``, ``total_used``, and ``total_avail`` fields are replaced with
+  ``total_bytes``, ``total_used_bytes``,  and ``total_avail_bytes`` fields.
+
+* The ``rados df --format=json`` output ``read_bytes`` and ``write_bytes``
+  fields were incorrectly reporting ops; this is now fixed.
+
+* The ``rados df --format=json`` output previously included ``read_kb`` and
+  ``write_kb`` fields; these have been removed.  Please use ``read_bytes`` and
+  ``write_bytes`` instead (and divide by 1024 if appropriate).
+
+Notable Changes
+---------------
+
+* ceph-conf: flush log on exit (Sage Weil)
+* ceph-dencoder: refactor build a bit to limit dependencies (Sage Weil, Dan Mick)
+* ceph.spec: split out ceph-common package, other fixes (Sandon Van Ness)
+* ceph_test_librbd_fsx: fix RNG, make deterministic (Ilya Dryomov)
+* cephtool: refactor and improve CLI tests (Joao Eduardo Luis)
+* client: improved MDS session dumps (John Spray)
+* common: fix dup log messages (#9080, Sage Weil)
+* crush: include new tunables in dump (Sage Weil)
+* crush: only require rule features if the rule is used (#8963, Sage Weil)
+* crushtool: send output to stdout, not stderr (Wido den Hollander)
+* fix i386 builds (Sage Weil)
+* fix struct vs class inconsistencies (Thorsten Behrens)
+* hadoop: update hadoop tests for Hadoop 2.0 (Haumin Chen)
+* librbd, ceph-fuse: reduce cache flush overhead (Haomai Wang)
+* librbd: fix error path when opening image (#8912, Josh Durgin)
+* mds: add file system name, enabled flag (John Spray)
+* mds: boot refactor, cleanup (John Spray)
+* mds: fix journal conversion with standby-replay (John Spray)
+* mds: separate inode recovery queue (John Spray)
+* mds: session ls, evict commands (John Spray)
+* mds: submit log events in async thread (Yan, Zheng)
+* mds: use client-provided timestamp for user-visible file metadata (Yan, Zheng)
+* mds: validate journal header on load and save (John Spray)
+* misc build fixes for OS X (John Spray)
+* misc integer size cleanups (Kevin Cox)
+* mon: add get-quota commands (Joao Eduardo Luis)
+* mon: do not create file system by default (John Spray)
+* mon: fix 'ceph df' output for available space (Xiaoxi Chen)
+* mon: fix bug when no auth keys are present (#8851, Joao Eduardo Luis)
+* mon: fix compat version for MForward (Joao Eduardo Luis)
+* mon: restrict some pool properties to tiered pools (Joao Eduardo Luis)
+* msgr: misc locking fixes for fast dispatch (#8891, Sage Weil)
+* osd: add 'dump_reservations' admin socket command (Sage Weil)
+* osd: add READFORWARD caching mode (Luis Pabon)
+* osd: add header cache for KeyValueStore (Haomai Wang)
+* osd: add prototype KineticStore based on Seagate Kinetic (Josh Durgin)
+* osd: allow map cache size to be adjusted at runtime (Sage Weil)
+* osd: avoid refcounting overhead by passing a few things by ref (Somnath Roy)
+* osd: avoid sharing PG info that is not durable (Samuel Just)
+* osd: clear slow request latency info on osd up/down (Sage Weil)
+* osd: fix PG object listing/ordering bug (Guang Yang)
+* osd: fix PG stat errors with tiering (#9082, Sage Weil)
+* osd: fix bug with long object names and rename (#8701, Sage Weil)
+* osd: fix cache full -> not full requeueing (#8931, Sage Weil)
+* osd: fix gating of messages from old OSD instances (Greg Farnum)
+* osd: fix memstore bugs with collection_move_rename, lock ordering (Sage Weil)
+* osd: improve locking for KeyValueStore (Haomai Wang)
+* osd: make tiering behave if hit_sets aren't enabled (Sage Weil)
+* osd: mark pools with incomplete clones (Sage Weil)
+* osd: misc locking fixes for fast dispatch (Samuel Just, Ma Jianpeng)
+* osd: prevent old rados clients from using tiered pools (#8714, Sage Weil)
+* osd: reduce OpTracker overhead (Somnath Roy)
+* osd: set configurable hard limits on object and xattr names (Sage Weil, Haomai Wang)
+* osd: trim old EC objects quickly; verify on scrub (Samuel Just)
+* osd: work around GCC 4.8 bug in journal code (Matt Benjamin)
+* rados bench: fix arg order (Kevin Dalley)
+* rados: fix {read,write}_ops values for df output (Sage Weil)
+* rbd: add rbdmap pre- and post post- hooks, fix misc bugs (Dmitry Smirnov)
+* rbd: improve option default behavior (Josh Durgin)
+* rgw: automatically align writes to EC pool (#8442, Yehuda Sadeh)
+* rgw: fix crash on swift CORS preflight request (#8586, Yehuda Sadeh)
+* rgw: fix memory leaks (Andrey Kuznetsov)
+* rgw: fix multipart upload (#8846, Silvain Munaut, Yehuda Sadeh)
+* rgw: improve -h (Abhishek Lekshmanan)
+* rgw: improve delimited listing of bucket, misc fixes (Yehuda Sadeh)
+* rgw: misc civetweb fixes (Yehuda Sadeh)
+* rgw: powerdns backend for global namespaces (Wido den Hollander)
+* systemd: initial systemd config files (Federico Simoncelli)
+
+
+v0.83
+=====
+
+Another Ceph development release!  This has been a longer cycle, so
+there has been quite a bit of bug fixing and stabilization in this
+round.  There is also a bunch of packaging fixes for RPM distros
+(RHEL/CentOS, Fedora, and SUSE) and for systemd.  We've also added a new
+librados-striper library from Sebastien Ponce that provides a generic
+striping API for applications to code to.
+
+Upgrading
+---------
+
+* The experimental keyvaluestore-dev OSD backend had an on-disk format
+  change that prevents existing OSD data from being upgraded.  This
+  affects developers and testers only.
+
+* mon-specific and osd-specific leveldb options have been removed.
+  From this point onward users should use the `leveldb_*` generic
+  options and add the options in the appropriate sections of their
+  configuration files.  Monitors will still maintain the following
+  monitor-specific defaults:
+
+    leveldb_write_buffer_size = 32*1024*1024  = 33554432  // 32MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+Notable Changes
+---------------
+
+* ceph-disk: fix dmcrypt support (Stephen Taylor)
+* cephtool: fix help (Yilong Zhao)
+* cephtool: test cleanup (Joao Eduardo Luis)
+* doc: librados example fixes (Kevin Dalley)
+* doc: many doc updates (John Wilkins)
+* doc: update erasure docs (Loic Dachary, Venky Shankar)
+* filestore: disable use of XFS hint (buggy on old kernels) (Samuel Just)
+* filestore: fix xattr spillout (Greg Farnum, Haomai Wang)
+* keyvaluestore: header cache (Haomai Wang)
+* librados_striper: striping library for librados (Sebastien Ponce)
+* libs3: update to latest (Danny Al-Gaaf)
+* log: fix derr level (Joao Eduardo Luis)
+* logrotate: fix osd log rotation on ubuntu (Sage Weil)
+* mds: fix xattr bug triggered by ACLs (Yan, Zheng)
+* misc memory leaks, cleanups, fixes (Danny Al-Gaaf, Sahid Ferdjaoui)
+* misc suse fixes (Danny Al-Gaaf)
+* misc word size fixes (Kevin Cox)
+* mon: drop mon- and osd- specific leveldb options (Joao Eduardo Luis)
+* mon: ec pool profile fixes (Loic Dachary)
+* mon: fix health down messages (Sage Weil)
+* mon: fix quorum feature check (#8738, Greg Farnum)
+* mon: 'osd crush reweight-subtree ...' (Sage Weil)
+* mon, osd: relax client EC support requirements (Sage Weil)
+* mon: some instrumentation (Sage Weil)
+* objecter: flag operations that are redirected by caching (Sage Weil)
+* osd: clean up shard_id_t, shard_t (Loic Dachary)
+* osd: fix connection reconnect race (Greg Farnum)
+* osd: fix dumps (Joao Eduardo Luis)
+* osd: fix erasure-code lib initialization (Loic Dachary)
+* osd: fix extent normalization (Adam Crume)
+* osd: fix loopback msgr issue (Ma Jianpeng)
+* osd: fix LSB release parsing (Danny Al-Gaaf)
+* osd: improved backfill priorities (Sage Weil)
+* osd: many many core fixes (Samuel Just)
+* osd, mon: config sanity checks on start (Sage Weil, Joao Eduardo Luis)
+* osd: sharded threadpool to improve parallelism (Somnath Roy)
+* osd: simple io prioritization for scrub (Sage Weil)
+* osd: simple scrub throttling (Sage Weil)
+* osd: tests for bench command (Loic Dachary)
+* osd: use xfs hint less frequently (Ilya Dryomov)
+* pybind/rados: fix small timeouts (John Spray)
+* qa: xfstests updates (Ilya Dryomov)
+* rgw: cache bucket info (Yehuda Sadeh)
+* rgw: cache decoded user info (Yehuda Sadeh)
+* rgw: fix multipart object attr regression (#8452, Yehuda Sadeh)
+* rgw: fix radosgw-admin 'show log' command (#8553, Yehuda Sadeh)
+* rgw: fix URL decoding (#8702, Brian Rak)
+* rgw: handle empty extra pool name (Yehuda Sadeh)
+* rpm: do not restart daemons on upgrade (Alfredo Deza)
+* rpm: misc packaging fixes for rhel7 (Sandon Van Ness)
+* rpm: split ceph-common from ceph (Sandon Van Ness)
+* systemd: wrap started daemons in new systemd environment (Sage Weil, Dan Mick)
+* sysvinit: less sensitive to failures (Sage Weil)
+* upstart: increase max open files limit (Sage Weil)
+
+v0.82
+=====
+
+This is the second post-firefly development release.  It includes a range
+of bug fixes and some usability improvements.  There are some MDS debugging
+and diagnostic tools, an improved 'ceph df', and some OSD backend refactoring
+and cleanup.
+
+Notable Changes
+---------------
+
+* ceph-brag: add tox tests (Alfredo Deza)
+* common: perfcounters now use atomics and go faster (Sage Weil)
+* doc: CRUSH updates (John Wilkins)
+* doc: osd primary affinity (John Wilkins)
+* doc: pool quotas (John Wilkins)
+* doc: pre-flight doc improvements (Kevin Dalley)
+* doc: switch to an unencumbered font (Ross Turk)
+* doc: update openstack docs (Josh Durgin)
+* fix hppa arch build (Dmitry Smirnov)
+* init-ceph: continue starting other daemons on crush or mount failure (#8343, Sage Weil)
+* keyvaluestore: fix hint crash (#8381, Haomai Wang)
+* libcephfs-java: build against older JNI headers (Greg Farnum)
+* librados: fix rados_pool_list bounds checks (Sage Weil)
+* mds: cephfs-journal-tool (John Spray)
+* mds: improve Journaler on-disk format (John Spray)
+* mds, libcephfs: use client timestamp for mtime/ctime (Sage Weil)
+* mds: misc encoding improvements (John Spray)
+* mds: misc fixes for multi-mds (Yan, Zheng)
+* mds: OPTracker integration, dump_ops_in_flight (Greg Farnum)
+* misc cleanup (Christophe Courtaut)
+* mon: fix default replication pool ruleset choice (#8373, John Spray)
+* mon: fix set cache_target_full_ratio (#8440, Geoffrey Hartz)
+* mon: include per-pool 'max avail' in df output (Sage Weil)
+* mon: prevent EC pools from being used with cephfs (Joao Eduardo Luis)
+* mon: restore original weight when auto-marked out OSDs restart (Sage Weil)
+* mon: use msg header tid for MMonGetVersionReply (Ilya Dryomov)
+* osd: fix bogus assert during OSD shutdown (Sage Weil)
+* osd: fix clone deletion case (#8334, Sam Just)
+* osd: fix filestore removal corner case (#8332, Sam Just)
+* osd: fix hang waiting for osdmap (#8338, Greg Farnum)
+* osd: fix interval check corner case during peering (#8104, Sam Just)
+* osd: fix journal-less operation (Sage Weil)
+* osd: include backend information in metadata reported to mon (Sage Weil)
+* rest-api: fix help (Ailing Zhang)
+* rgw: check entity permission for put_metadata (#8428, Yehuda Sadeh)
+
+
+v0.81
+=====
+
+This is the first development release since Firefly.  It includes a
+lot of work that we delayed merging while stabilizing things.  Lots of
+new functionality, as well as several fixes that are baking a bit before
+getting backported.
+
+Upgrading
+---------
+
+* CephFS support for the legacy anchor table has finally been removed.
+  Users with file systems created before firefly should ensure that inodes
+  with multiple hard links are modified *prior* to the upgrade to ensure that
+  the backtraces are written properly.  For example::
+
+    sudo find /mnt/cephfs -type f -links +1 -exec touch \{\} \;
+
+* Disallow nonsensical 'tier cache-mode' transitions.  From this point
+  onward, 'writeback' can only transition to 'forward' and 'forward'
+  can transition to 1) 'writeback' if there are dirty objects, or 2) any if
+  there are no dirty objects.
+
+Notable Changes
+---------------
+
+* bash completion improvements (Wido den Hollander)
+* brag: fixes, improvements (Loic Dachary)
+* ceph-disk: handle corrupt volumes (Stuart Longlang)
+* ceph-disk: partprobe as needed (Eric Eastman)
+* ceph-fuse, libcephfs: asok hooks for handling session resets, timeouts (Yan, Zheng)
+* ceph-fuse, libcephfs: improve traceless reply handling (Sage Weil)
+* clang build fixes (John Spray, Danny Al-Gaaf)
+* config: support G, M, K, etc. suffixes (Joao Eduardo Luis)
+* coverity cleanups (Danny Al-Gaaf)
+* doc: cache tiering (John Wilkins)
+* doc: keystone integration docs (John Wilkins)
+* doc: updated simple configuration guides (John Wilkins)
+* libcephfs-java: fix gcj-jdk build (Dmitry Smirnov)
+* librbd: check error code on cache invalidate (Josh Durgin)
+* librbd: new libkrbd library for kernel map/unmap/showmapped (Ilya Dryomov)
+* Makefile: fix out of source builds (Stefan Eilemann)
+* mds: multi-mds fixes (Yan, Zheng)
+* mds: remove legacy anchor table (Yan, Zheng)
+* mds: remove legacy discover ino (Yan, Zheng)
+* monclient: fix hang (Sage Weil)
+* mon: prevent nonsensical cache-mode transitions (Joao Eduardo Luis)
+* msgr: avoid big lock when sending (most) messages (Greg Farnum)
+* osd: bound osdmap epoch skew between PGs (Sage Weil)
+* osd: cache tier flushing fixes for snapped objects (Samuel Just)
+* osd: fix agent early finish looping (David Zafman)
+* osd: fix flush vs OpContext (Samuel Just)
+* osd: fix MarkMeDown and other shutdown races (Sage Weil)
+* osd: fix scrub vs cache bugs (Samuel Just)
+* osd: fix trim of hitsets (Sage Weil)
+* osd, msgr: fast-dispatch of OSD ops (Greg Farnum, Samuel Just)
+* osd, objecter: resend ops on last_force_op_resend barrier; fix cache overlay op ordering (Sage Weil)
+* osd: remove obsolete classic scrub code (David Zafman)
+* osd: scrub PGs with invalid stats (Sage Weil)
+* osd: simple snap trimmer throttle (Sage Weil)
+* osd: use FIEMAP to inform copy_range (Haomai Wang)
+* rbd-fuse: allow exposing single image (Stephen Taylor)
+* rbd-fuse: fix unlink (Josh Durgin)
+* removed mkcephfs (deprecated since dumpling)
+* rgw: bucket link uses instance id (Yehuda Sadeh)
+* rgw: fix memory leak following chunk read error (Yehuda Sadeh)
+* rgw: fix URL escaping (Yehuda Sadeh)
+* rgw: fix user manifest (Yehuda Sadeh)
+* rgw: object and bucket rewrite functions to allow restriping old objects (Yehuda Sadeh)
+* rgw: prevent multiobject PUT race (Yehuda Sadeh)
+* rgw: send user manifest header (Yehuda Sadeh)
+* test_librbd_fsx: test krbd as well as librbd (Ilya Dryomov)
diff --git a/doc/releases/hammer.rst b/doc/releases/hammer.rst
new file mode 100644
index 000000000..b433f7061
--- /dev/null
+++ b/doc/releases/hammer.rst
@@ -0,0 +1,2325 @@
+======
+Hammer
+======
+
+Hammer is the 8th stable release of Ceph.  It is named after the
+hammer octopus (Octopus australis).
+
+v0.94.10 Hammer
+===============
+
+This Hammer point release fixes several bugs and adds two new features.
+
+We recommend that all hammer v0.94.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.10.txt>`.
+
+New Features
+------------
+
+ceph-objectstore-tool and ceph-monstore-tool now enable user to
+rebuild the monitor database from OSDs. (This feature is especially useful when
+all monitors fail to boot due to leveldb corruption.)
+
+In RADOS Gateway, it is now possible to reshard an existing bucket's index
+using an off-line tool.
+
+Usage:
+
+$ radosgw-admin bucket reshard --bucket=<bucket_name> --num_shards=<num_shards>
+
+This will create a new linked bucket instance that points to the newly created
+index objects. The old bucket instance still exists and currently it's up to
+the user to manually remove the old bucket index objects. (Note that bucket
+resharding currently requires that all IO (especially writes) to the specific
+bucket is quiesced.)
+
+Other Notable Changes
+---------------------
+
+* build/ops: ceph-create-keys loops forever (`issue#17753 <http://tracker.ceph.com/issues/17753>`_, `pr#12805 <http://github.com/ceph/ceph/pull/12805>`_, Alfredo Deza)
+* build/ops: improve ceph.in error message (`issue#11101 <http://tracker.ceph.com/issues/11101>`_, `pr#10905 <http://github.com/ceph/ceph/pull/10905>`_, Kefu Chai)
+* build/ops: make stop.sh more portable (`issue#16918 <http://tracker.ceph.com/issues/16918>`_, `pr#10569 <http://github.com/ceph/ceph/pull/10569>`_, Mykola Golub)
+* build/ops: remove SYSTEMD_RUN from initscript (`issue#16440 <http://tracker.ceph.com/issues/16440>`_, `issue#7627 <http://tracker.ceph.com/issues/7627>`_, `pr#9873 <http://github.com/ceph/ceph/pull/9873>`_, Vladislav Odintsov)
+* cephx: Fix multiple segfaults due to attempts to encrypt or decrypt (`issue#16266 <http://tracker.ceph.com/issues/16266>`_, `pr#11930 <http://github.com/ceph/ceph/pull/11930>`_, Brad Hubbard)
+* common: SIGABRT in TrackedOp::dump() via dump_ops_in_flight() (`issue#8885 <http://tracker.ceph.com/issues/8885>`_, `pr#12121 <http://github.com/ceph/ceph/pull/12121>`_, Jianpeng Ma, Zhiqiang Wang, David Zafman)
+* common: os/ObjectStore: fix _update_op for split dest_cid (`issue#15345 <http://tracker.ceph.com/issues/15345>`_, `pr#12071 <http://github.com/ceph/ceph/pull/12071>`_, Sage Weil)
+* crush: reset bucket->h.items[i] when removing tree item (`issue#16525 <http://tracker.ceph.com/issues/16525>`_, `pr#10724 <http://github.com/ceph/ceph/pull/10724>`_, Kefu Chai)
+* doc: add "Upgrading to Hammer" section (`issue#17386 <http://tracker.ceph.com/issues/17386>`_, `pr#11372 <http://github.com/ceph/ceph/pull/11372>`_, Kefu Chai)
+* doc: add orphan options to radosgw-admin --help and man page (`issue#17281 <http://tracker.ceph.com/issues/17281>`_, `issue#17280 <http://tracker.ceph.com/issues/17280>`_, `pr#11140 <http://github.com/ceph/ceph/pull/11140>`_, Abhishek Lekshmanan, Casey Bodley, Ken Dreyer, Thomas Serlin)
+* doc: clarify that RGW bucket object versioning is supported (`issue#16574 <http://tracker.ceph.com/issues/16574>`_, `pr#10437 <http://github.com/ceph/ceph/pull/10437>`_, Yuan Zhou, shawn chen)
+* librados: bad flags can crash the osd (`issue#16012 <http://tracker.ceph.com/issues/16012>`_, `pr#11936 <http://github.com/ceph/ceph/pull/11936>`_, Jianpeng Ma, Sage Weil)
+* librbd: ceph 10.2.2 rbd status on image format 2 returns "(2) No such file or directory" (`issue#16887 <http://tracker.ceph.com/issues/16887>`_, `pr#10987 <http://github.com/ceph/ceph/pull/10987>`_, Jason Dillaman)
+* librbd: diffs to clone's first snapshot should include parent diffs (`issue#18068 <http://tracker.ceph.com/issues/18068>`_, `pr#12446 <http://github.com/ceph/ceph/pull/12446>`_, Jason Dillaman)
+* librbd: image.stat() call in librbdpy fails sometimes (`issue#17310 <http://tracker.ceph.com/issues/17310>`_, `pr#11949 <http://github.com/ceph/ceph/pull/11949>`_, Jason Dillaman)
+* librbd: request exclusive lock if current owner cannot execute op (`issue#16171 <http://tracker.ceph.com/issues/16171>`_, `pr#12018 <http://github.com/ceph/ceph/pull/12018>`_, Mykola Golub)
+* mds: fix cephfs-java ftruncate unit test failure (`issue#11258 <http://tracker.ceph.com/issues/11258>`_, `pr#11939 <http://github.com/ceph/ceph/pull/11939>`_, Yan, Zheng)
+* mon: %USED of ceph df is wrong (`issue#16933 <http://tracker.ceph.com/issues/16933>`_, `pr#11934 <http://github.com/ceph/ceph/pull/11934>`_, Kefu Chai)
+* mon: MonmapMonitor should return success when MON will be removed (`issue#17725 <http://tracker.ceph.com/issues/17725>`_, `pr#12006 <http://github.com/ceph/ceph/pull/12006>`_, Joao Eduardo Luis)
+* mon: OSDMonitor: Missing nearfull flag set (`issue#17390 <http://tracker.ceph.com/issues/17390>`_, `pr#11273 <http://github.com/ceph/ceph/pull/11273>`_, Igor Podoski)
+* mon: OSDs marked OUT wrongly after monitor failover (`issue#17719 <http://tracker.ceph.com/issues/17719>`_, `pr#11946 <http://github.com/ceph/ceph/pull/11946>`_, Dong Wu)
+* mon: fix memory leak in prepare_beacon (`issue#17285 <http://tracker.ceph.com/issues/17285>`_, `pr#10238 <http://github.com/ceph/ceph/pull/10238>`_, Igor Podoski)
+* mon: osd flag health message is misleading (`issue#18175 <http://tracker.ceph.com/issues/18175>`_, `pr#12687 <http://github.com/ceph/ceph/pull/12687>`_, Sage Weil)
+* mon: prepare_pgtemp needs to only update up_thru if newer than the existing one (`issue#16185 <http://tracker.ceph.com/issues/16185>`_, `pr#11937 <http://github.com/ceph/ceph/pull/11937>`_, Samuel Just)
+* mon: return size_t from MonitorDBStore::Transaction::size() (`issue#14217 <http://tracker.ceph.com/issues/14217>`_, `pr#10904 <http://github.com/ceph/ceph/pull/10904>`_, Kefu Chai)
+* mon: send updated monmap to its subscribers (`issue#17558 <http://tracker.ceph.com/issues/17558>`_, `pr#11457 <http://github.com/ceph/ceph/pull/11457>`_, Kefu Chai)
+* msgr: OpTracker needs to release the message throttle in _unregistered (`issue#14248 <http://tracker.ceph.com/issues/14248>`_, `pr#11938 <http://github.com/ceph/ceph/pull/11938>`_, Samuel Just)
+* msgr: simple/Pipe: error decoding addr (`issue#18072 <http://tracker.ceph.com/issues/18072>`_, `pr#12266 <http://github.com/ceph/ceph/pull/12266>`_, Sage Weil)
+* osd: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items (`issue#16998 <http://tracker.ceph.com/issues/16998>`_, `pr#11933 <http://github.com/ceph/ceph/pull/11933>`_, Samuel Just)
+* osd: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (\*this == hobject_t(hobject_t::get_max()))) (`issue#13967 <http://tracker.ceph.com/issues/13967>`_, `pr#11932 <http://github.com/ceph/ceph/pull/11932>`_, Tao Chang)
+* osd: ReplicatedBackend::build_push_op: add a second config to limit omap entries/chunk independently of object data (`issue#16128 <http://tracker.ceph.com/issues/16128>`_, `pr#12417 <http://github.com/ceph/ceph/pull/12417>`_, Wanlong Gao)
+* osd: crash on EIO during deep-scrubbing (`issue#16034 <http://tracker.ceph.com/issues/16034>`_, `pr#11935 <http://github.com/ceph/ceph/pull/11935>`_, Nathan Cutler)
+* osd: filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE (`issue#18446 <http://tracker.ceph.com/issues/18446>`_, `pr#13041 <http://github.com/ceph/ceph/pull/13041>`_, xinxin shu)
+* osd: fix cached_removed_snaps bug in PGPool::update after map gap (`issue#18628 <http://tracker.ceph.com/issues/18628>`_, `issue#15943 <http://tracker.ceph.com/issues/15943>`_, `pr#12906 <http://github.com/ceph/ceph/pull/12906>`_, Samuel Just)
+* osd: fix collection_list shadow return value (`issue#17713 <http://tracker.ceph.com/issues/17713>`_, `pr#11927 <http://github.com/ceph/ceph/pull/11927>`_, Haomai Wang)
+* osd: fix fiemap issue in xfs when #extents > 1364 (`issue#17610 <http://tracker.ceph.com/issues/17610>`_, `pr#11615 <http://github.com/ceph/ceph/pull/11615>`_, Kefu Chai, Ning Yao)
+* osd: update PGPool to detect map gaps and reset cached_removed_snaps (`issue#15943 <http://tracker.ceph.com/issues/15943>`_, `pr#11676 <http://github.com/ceph/ceph/pull/11676>`_, Samuel Just)
+* rbd: export diff should open image as read-only (`issue#17671 <http://tracker.ceph.com/issues/17671>`_, `pr#11948 <http://github.com/ceph/ceph/pull/11948>`_, liyankun)
+* rbd: fix parameter check (`issue#18237 <http://tracker.ceph.com/issues/18237>`_, `pr#12312 <http://github.com/ceph/ceph/pull/12312>`_, Yankun Li)
+* rbd: fix possible rbd data corruption (`issue#16002 <http://tracker.ceph.com/issues/16002>`_, `pr#11618 <http://github.com/ceph/ceph/pull/11618>`_, Yan, Zheng, Greg Farnum)
+* rgw: Anonymous user is able to read bucket with authenticated read ACL (`issue#13207 <http://tracker.ceph.com/issues/13207>`_, `pr#11045 <http://github.com/ceph/ceph/pull/11045>`_, rahul.1aggarwal@gmail.com)
+* rgw: COPY broke multipart files uploaded under dumpling (`issue#16435 <http://tracker.ceph.com/issues/16435>`_, `pr#11950 <http://github.com/ceph/ceph/pull/11950>`_, Yehuda Sadeh)
+* rgw: TempURL in radosgw behaves now like its Swift's counterpart.  (`issue#18316 <http://tracker.ceph.com/issues/18316>`_, `pr#12619 <http://github.com/ceph/ceph/pull/12619>`_, Radoslaw Zarzynski)
+* rgw: default quota fixes (`issue#16410 <http://tracker.ceph.com/issues/16410>`_, `pr#10839 <http://github.com/ceph/ceph/pull/10839>`_, Pavan Rallabhandi, Daniel Gryniewicz)
+* rgw: do not abort when accept a CORS request with short origin (`issue#18187 <http://tracker.ceph.com/issues/18187>`_, `pr#12398 <http://github.com/ceph/ceph/pull/12398>`_, LiuYang)
+* rgw: do not omap_getvals with (u64)-1 max (`issue#17985 <http://tracker.ceph.com/issues/17985>`_, `pr#12418 <http://github.com/ceph/ceph/pull/12418>`_, Yehuda Sadeh, Sage Weil)
+* rgw: fix crash when client posts object with null condition (`issue#17635 <http://tracker.ceph.com/issues/17635>`_, `pr#11809 <http://github.com/ceph/ceph/pull/11809>`_, Yehuda Sadeh)
+* rgw: fix inconsistent uid/email handling in radosgw-admin (`issue#13598 <http://tracker.ceph.com/issues/13598>`_, `pr#11952 <http://github.com/ceph/ceph/pull/11952>`_, Matt Benjamin)
+* rgw: implement offline resharding command (`issue#17745 <http://tracker.ceph.com/issues/17745>`_, `pr#12227 <http://github.com/ceph/ceph/pull/12227>`_, Yehuda Sadeh, Orit Wasserman, weiqiaomiao)
+* rgw: swift: ranged request on a DLO provides wrong values in Content-Range HTTP header (`issue#13452 <http://tracker.ceph.com/issues/13452>`_, `pr#11951 <http://github.com/ceph/ceph/pull/11951>`_, Radoslaw Zarzynski)
+* rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt (`issue#17598 <http://tracker.ceph.com/issues/17598>`_, `pr#11899 <http://github.com/ceph/ceph/pull/11899>`_, weiqiaomiao)
+* tests: Cannot clone ceph/s3-tests.git (missing branch) (`issue#18384 <http://tracker.ceph.com/issues/18384>`_, `pr#12744 <http://github.com/ceph/ceph/pull/12744>`_, Orit Wasserman)
+* tests: Cannot reserve CentOS 7.2 smithi machines (`issue#18401 <http://tracker.ceph.com/issues/18401>`_, `pr#12762 <http://github.com/ceph/ceph/pull/12762>`_, Nathan Cutler)
+* tests: OSDs commit suicide in rbd suite when testing on btrfs (`issue#18397 <http://tracker.ceph.com/issues/18397>`_, `pr#12758 <http://github.com/ceph/ceph/pull/12758>`_, Nathan Cutler)
+* tests: Workunits needlessly wget from git.ceph.com (`issue#18336 <http://tracker.ceph.com/issues/18336>`_, `issue#18271 <http://tracker.ceph.com/issues/18271>`_, `issue#18388 <http://tracker.ceph.com/issues/18388>`_, `pr#12685 <http://github.com/ceph/ceph/pull/12685>`_, Sage Weil, Nathan Cutler)
+* tests: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa) (`issue#18574 <http://tracker.ceph.com/issues/18574>`_, `pr#13022 <http://github.com/ceph/ceph/pull/13022>`_, John Spray)
+* tests: merge ceph-qa-suite (`pr#12455 <http://github.com/ceph/ceph/pull/12455>`_, Sage Weil)
+* tests: objecter_requests workunit fails on wip branches (`issue#18393 <http://tracker.ceph.com/issues/18393>`_, `pr#12759 <http://github.com/ceph/ceph/pull/12759>`_, Sage Weil)
+* tests: populate mnt_point in qa/tasks/ceph.py (`issue#18383 <http://tracker.ceph.com/issues/18383>`_, `pr#12743 <http://github.com/ceph/ceph/pull/12743>`_, Nathan Cutler)
+* tests: qemu/tests/qemu-iotests/077 fails in dumpling, hammer, and jewel (`issue#10773 <http://tracker.ceph.com/issues/10773>`_, `pr#12423 <http://github.com/ceph/ceph/pull/12423>`_, Jason Dillaman)
+* tests: run fs/thrash on xfs instead of btrfs (`issue#17151 <http://tracker.ceph.com/issues/17151>`_, `pr#13039 <http://github.com/ceph/ceph/pull/13039>`_, Nathan Cutler)
+* tests: update Ubuntu image url after ceph.com refactor (`issue#18542 <http://tracker.ceph.com/issues/18542>`_, `pr#12957 <http://github.com/ceph/ceph/pull/12957>`_, Jason Dillaman)
+* tests: update rbd/singleton/all/formatted-output.yaml to support ceph-ci * (`issue#18440 <http://tracker.ceph.com/issues/18440>`_, `pr#12824 * <http://github.com/ceph/ceph/pull/12824>`_, Venky Shankar, Nathan Cutler)
+* tools: add a tool to rebuild mon store from OSD (`issue#17179 <http://tracker.ceph.com/issues/17179>`_, `issue#17400 <http://tracker.ceph.com/issues/17400>`_, `pr#11125 <http://github.com/ceph/ceph/pull/11125>`_, Kefu Chai, xie xingguo)
+* tools: ceph-objectstore-tool crashes if --journal-path <a-directory> (`issue#17307 <http://tracker.ceph.com/issues/17307>`_, `pr#11929 <http://github.com/ceph/ceph/pull/11929>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: add a way to split filestore directories offline (`issue#17220 <http://tracker.ceph.com/issues/17220>`_, `pr#11253 <http://github.com/ceph/ceph/pull/11253>`_, Josh Durgin)
+* tools: crushtool --compile generates output despite missing item (`issue#17306 <http://tracker.ceph.com/issues/17306>`_, `pr#11931 <http://github.com/ceph/ceph/pull/11931>`_, Kefu Chai)
+
+
+v0.94.9 Hammer
+==============
+
+This Hammer point release fixes a build issue present in 0.94.8 that prevented us
+from generating packages for Ubuntu Precise and CentOS 6.x.
+
+We recommend all users of v0.94.7 or older upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.9.txt>`.
+
+Notable Changes
+---------------
+
+* build/ops: revert: boost uuid makes valgrind complain (`pr#10913 <http://github.com/ceph/ceph/pull/10913>`_, Sage Weil)
+
+
+v0.94.8 Hammer
+==============
+
+This Hammer point release fixes several bugs.
+
+We recommend that all hammer v0.94.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.8.txt>`.
+
+Notable Changes
+---------------
+
+* build/ops: rocksdb do not link against tcmalloc if it's disabled (`issue#14799 <http://tracker.ceph.com/issues/14799>`_, `pr#10750 <http://github.com/ceph/ceph/pull/10750>`_, Sage Weil, Kefu Chai)
+* build/ops: Add -D_LARGEFILE64_SOURCE to Linux build. (`issue#16611 <http://tracker.ceph.com/issues/16611>`_, `pr#10182 <http://github.com/ceph/ceph/pull/10182>`_, Ira Cooper)
+* build/ops: boost uuid makes valgrind complain (`issue#12736 <http://tracker.ceph.com/issues/12736>`_, `pr#9741 <http://github.com/ceph/ceph/pull/9741>`_, Sage Weil, Rohan Mars)
+* build/ops: ceph-disk s/by-parttype-uuid/by-parttypeuuid/ (`issue#15867 <http://tracker.ceph.com/issues/15867>`_, `pr#9107 <http://github.com/ceph/ceph/pull/9107>`_, Nathan Cutler)
+* common: add units to rados bench output and clean up formatting (`issue#12248 <http://tracker.ceph.com/issues/12248>`_, `pr#8960 <http://github.com/ceph/ceph/pull/8960>`_, Dmitry Yatsushkevich, Brad Hubbard, Gu Zhongyan)
+* common: config set with negative value results in "error setting 'filestore_merge_threshold' to '-40': (22) Invalid argument" (`issue#13829 <http://tracker.ceph.com/issues/13829>`_, `pr#10291 <http://github.com/ceph/ceph/pull/10291>`_, Brad Hubbard, Kefu Chai)
+* common: linking to -lrbd causes process startup times to balloon (`issue#15225 <http://tracker.ceph.com/issues/15225>`_, `pr#8538 <http://github.com/ceph/ceph/pull/8538>`_, Richard W.M. Jones)
+* doc: fix by-parttypeuuid in ceph-disk(8) nroff (`issue#15867 <http://tracker.ceph.com/issues/15867>`_, `pr#10699 <http://github.com/ceph/ceph/pull/10699>`_, Ken Dreyer)
+* fs: double decreased the count to trim caps which will cause failing to respond to cache pressure (`issue#14319 <http://tracker.ceph.com/issues/14319>`_, `pr#8804 <http://github.com/ceph/ceph/pull/8804>`_, Zhi Zhang)
+* log: do not repeat errors to stderr (`issue#14616 <http://tracker.ceph.com/issues/14616>`_, `pr#10227 <http://github.com/ceph/ceph/pull/10227>`_, Sage Weil)
+* mds: failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7 (`issue#16013 <http://tracker.ceph.com/issues/16013>`_, `pr#10198 <http://github.com/ceph/ceph/pull/10198>`_, Yan, Zheng)
+* mds: fix stray purging in 'stripe_count > 1' case (`issue#15050 <http://tracker.ceph.com/issues/15050>`_, `pr#8042 <http://github.com/ceph/ceph/pull/8042>`_, Yan, Zheng)
+* mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs (`issue#15702 <http://tracker.ceph.com/issues/15702>`_, `pr#9404 <http://github.com/ceph/ceph/pull/9404>`_, Zhi Zhang)
+* mon: LibRadosMiscConnectFailure.ConnectFailure (not so intermittent) failure in upgrade/hammer-x  (`issue#13992 <http://tracker.ceph.com/issues/13992>`_, `pr#8806 <http://github.com/ceph/ceph/pull/8806>`_, Sage Weil)
+* mon: Monitor: validate prefix on handle_command() (`issue#16297 <http://tracker.ceph.com/issues/16297>`_, `pr#10038 <http://github.com/ceph/ceph/pull/10038>`_, You Ji)
+* mon: drop pg temps from not the current primary in OSDMonitor (`issue#16127 <http://tracker.ceph.com/issues/16127>`_, `pr#9893 <http://github.com/ceph/ceph/pull/9893>`_, Samuel Just)
+* mon: fix calculation of %USED (`issue#15641 <http://tracker.ceph.com/issues/15641>`_, `pr#9125 <http://github.com/ceph/ceph/pull/9125>`_, Ruifeng Yang, David Zafman)
+* mon: improve reweight_by_utilization() logic (`issue#15686 <http://tracker.ceph.com/issues/15686>`_, `pr#9416 <http://github.com/ceph/ceph/pull/9416>`_, xie xingguo)
+* mon: pool quota alarm is not in effect  (`issue#15478 <http://tracker.ceph.com/issues/15478>`_, `pr#8593 <http://github.com/ceph/ceph/pull/8593>`_, Danny Al-Gaaf)
+* mon: wrong ceph get mdsmap assertion (`issue#14681 <http://tracker.ceph.com/issues/14681>`_, `pr#7542 <http://github.com/ceph/ceph/pull/7542>`_, Vicente Cheng)
+* msgr: ceph-osd valgrind invalid reads/writes (`issue#15870 <http://tracker.ceph.com/issues/15870>`_, `pr#9238 <http://github.com/ceph/ceph/pull/9238>`_, Samuel Just)
+* objecter: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv (`issue#15760 <http://tracker.ceph.com/issues/15760>`_, `pr#9400 <http://github.com/ceph/ceph/pull/9400>`_, Sage Weil)
+* osd: OSD reporting ENOTEMPTY and crashing (`issue#14766 <http://tracker.ceph.com/issues/14766>`_, `pr#9277 <http://github.com/ceph/ceph/pull/9277>`_, Samuel Just)
+* osd: When generating past intervals due to an import end at pg epoch and fix build_past_intervals_parallel (`issue#12387 <http://tracker.ceph.com/issues/12387>`_, `issue#14438 <http://tracker.ceph.com/issues/14438>`_, `pr#8464 <http://github.com/ceph/ceph/pull/8464>`_, David Zafman)
+* osd: acting_primary not updated on split (`issue#15523 <http://tracker.ceph.com/issues/15523>`_, `pr#9001 <http://github.com/ceph/ceph/pull/9001>`_, Sage Weil)
+* osd: assert(!actingbackfill.empty()): old watch timeout tries to queue repop on replica (`issue#15391 <http://tracker.ceph.com/issues/15391>`_, `pr#8665 <http://github.com/ceph/ceph/pull/8665>`_, Sage Weil)
+* osd: assert(rollback_info_trimmed_to == head) in PGLog (`issue#13965 <http://tracker.ceph.com/issues/13965>`_, `pr#8849 <http://github.com/ceph/ceph/pull/8849>`_, Samuel Just)
+* osd: delete one of the repeated op->mark_started in ReplicatedBackend::sub_op_modify_impl (`issue#16572 <http://tracker.ceph.com/issues/16572>`_, `pr#9977 <http://github.com/ceph/ceph/pull/9977>`_, shun-s)
+* osd: fix omap digest compare when scrub (`issue#16000 <http://tracker.ceph.com/issues/16000>`_, `pr#9271 <http://github.com/ceph/ceph/pull/9271>`_, Xinze Chi)
+* osd: is_split crash in handle_pg_create (`issue#15426 <http://tracker.ceph.com/issues/15426>`_, `pr#8805 <http://github.com/ceph/ceph/pull/8805>`_, Kefu Chai)
+* osd: objects unfound after repair (fixed by repeering the pg) (`issue#15006 <http://tracker.ceph.com/issues/15006>`_, `pr#7961 <http://github.com/ceph/ceph/pull/7961>`_, Jianpeng Ma, Loic Dachary, Kefu Chai)
+* osd: rados cppool omap to ec pool crashes osd (`issue#14695 <http://tracker.ceph.com/issues/14695>`_, `pr#8845 <http://github.com/ceph/ceph/pull/8845>`_, Jianpeng Ma)
+* osd: remove all stale osdmaps in handle_osd_map() (`issue#13990 <http://tracker.ceph.com/issues/13990>`_, `pr#9090 <http://github.com/ceph/ceph/pull/9090>`_, Kefu Chai)
+* osd: send write and read sub ops on behalf of client ops at normal priority in ECBackend (`issue#14313 <http://tracker.ceph.com/issues/14313>`_, `pr#8573 <http://github.com/ceph/ceph/pull/8573>`_, Samuel Just)
+* rbd: snap rollback: restore the link to parent (`issue#14512 <http://tracker.ceph.com/issues/14512>`_, `pr#8535 <http://github.com/ceph/ceph/pull/8535>`_, Alexey Sheplyakov)
+* rgw: S3: set EncodingType in ListBucketResult (`issue#15896 <http://tracker.ceph.com/issues/15896>`_, `pr#8987 <http://github.com/ceph/ceph/pull/8987>`_, Victor Makarov, Robin H. Johnson)
+* rgw: backport rgwx-copy-if-newer for radosgw-agent (`issue#16262 <http://tracker.ceph.com/issues/16262>`_, `pr#9671 <http://github.com/ceph/ceph/pull/9671>`_, Yehuda Sadeh)
+* rgw: bucket listing following object delete is partial (`issue#14826 <http://tracker.ceph.com/issues/14826>`_, `pr#10555 <http://github.com/ceph/ceph/pull/10555>`_, Orit Wasserman)
+* rgw: convert plain object to versioned (with null version) when removing (`issue#15243 <http://tracker.ceph.com/issues/15243>`_, `pr#8755 <http://github.com/ceph/ceph/pull/8755>`_, Yehuda Sadeh)
+* rgw: fix multi-delete query param parsing. (`issue#16618 <http://tracker.ceph.com/issues/16618>`_, `pr#10189 <http://github.com/ceph/ceph/pull/10189>`_, Robin H. Johnson)
+* rgw: have a flavor of bucket deletion to bypass GC and to trigger (`issue#15557 <http://tracker.ceph.com/issues/15557>`_, `pr#10509 <http://github.com/ceph/ceph/pull/10509>`_, Pavan Rallabhandi)
+* rgw: keep track of written_objs correctly (`issue#15886 <http://tracker.ceph.com/issues/15886>`_, `pr#9240 <http://github.com/ceph/ceph/pull/9240>`_, Yehuda Sadeh)
+* rgw: multipart ListPartsResult has missing quotes on ETag (`issue#15334 <http://tracker.ceph.com/issues/15334>`_, `pr#8475 <http://github.com/ceph/ceph/pull/8475>`_, xie xingguo, Robin H. Johnson)
+* rgw: no Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest (`issue#15812 <http://tracker.ceph.com/issues/15812>`_, `pr#9402 <http://github.com/ceph/ceph/pull/9402>`_, Radoslaw Zarzynski)
+* rgw: radosgw server abort when user passed bad parameters to set quota (`issue#14190 <http://tracker.ceph.com/issues/14190>`_, `issue#14191 <http://tracker.ceph.com/issues/14191>`_, `pr#8313 <http://github.com/ceph/ceph/pull/8313>`_, Dunrong Huang)
+* rgw: radosgw-admin region-map set is not reporting the bucket quota correctly (`issue#16815 <http://tracker.ceph.com/issues/16815>`_, `pr#10554 <http://github.com/ceph/ceph/pull/10554>`_, Yehuda Sadeh, Orit Wasserman)
+* rgw: refrain from sending Content-Type/Content-Length for 304 responses (`issue#16327 <http://tracker.ceph.com/issues/16327>`_, `issue#13582 <http://tracker.ceph.com/issues/13582>`_, `issue#15119 <http://tracker.ceph.com/issues/15119>`_, `issue#14005 <http://tracker.ceph.com/issues/14005>`_, `pr#8379 <http://github.com/ceph/ceph/pull/8379>`_, Yehuda Sadeh, Nathan Cutler, Wido den Hollander)
+* rgw: remove bucket index objects when deleting the bucket (`issue#16412 <http://tracker.ceph.com/issues/16412>`_, `pr#10530 <http://github.com/ceph/ceph/pull/10530>`_, Orit Wasserman)
+* rgw: set Access-Control-Allow-Origin to an asterisk if allowed in a rule (`issue#15348 <http://tracker.ceph.com/issues/15348>`_, `pr#8528 <http://github.com/ceph/ceph/pull/8528>`_, Wido den Hollander)
+* rgw: subset of uploaded objects via radosgw are unretrievable when using EC pool (`issue#15745 <http://tracker.ceph.com/issues/15745>`_, `pr#9407 <http://github.com/ceph/ceph/pull/9407>`_, Yehuda Sadeh)
+* rgw: subuser rm fails with status 125 (`issue#14375 <http://tracker.ceph.com/issues/14375>`_, `pr#9961 <http://github.com/ceph/ceph/pull/9961>`_, Orit Wasserman)
+* rgw: the swift key remains after removing a subuser (`issue#12890 <http://tracker.ceph.com/issues/12890>`_, `issue#14375 <http://tracker.ceph.com/issues/14375>`_, `pr#10718 <http://github.com/ceph/ceph/pull/10718>`_, Orit Wasserman, Sangdi Xu)
+* rgw: user quota may not adjust on bucket removal (`issue#14507 <http://tracker.ceph.com/issues/14507>`_, `pr#8113 <http://github.com/ceph/ceph/pull/8113>`_, Edward Yang)
+* tests: be more generous with test timeout (`issue#15403 <http://tracker.ceph.com/issues/15403>`_, `pr#8470 <http://github.com/ceph/ceph/pull/8470>`_, Loic Dachary)
+* tests: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable (`issue#16289 <http://tracker.ceph.com/issues/16289>`_, `pr#9722 <http://github.com/ceph/ceph/pull/9722>`_, Mykola Golub)
+
+v0.94.7 Hammer
+==============
+
+This Hammer point release fixes several minor bugs.  It also includes
+a backport of an improved 'ceph osd reweight-by-utilization' command
+for handling OSDs with higher-than-average utilizations.
+
+We recommend that all hammer v0.94.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.7.txt>`.
+
+Notable Changes
+---------------
+
+* auth: keyring permisions for mon deamon (`issue#14950 <http://tracker.ceph.com/issues/14950>`_, `pr#8049 <http://github.com/ceph/ceph/pull/8049>`_, Owen Synge)
+* auth: PK11_DestroyContext() is called twice if PK11_DigestFinal() fails (`issue#14958 <http://tracker.ceph.com/issues/14958>`_, `pr#7922 <http://github.com/ceph/ceph/pull/7922>`_, Brad Hubbard, Dunrong Huang)
+* auth: use libnss more safely (`issue#14620 <http://tracker.ceph.com/issues/14620>`_, `pr#7488 <http://github.com/ceph/ceph/pull/7488>`_, Sage Weil)
+* ceph-disk: use blkid instead of sgdisk -i (`issue#14080 <http://tracker.ceph.com/issues/14080>`_, `issue#14094 <http://tracker.ceph.com/issues/14094>`_, `pr#7475 <http://github.com/ceph/ceph/pull/7475>`_, Ilya Dryomov, Loic Dachary)
+* ceph-fuse: fix ceph-fuse writing to stale log file after log rotation (`issue#12350 <http://tracker.ceph.com/issues/12350>`_, `pr#7110 <http://github.com/ceph/ceph/pull/7110>`_, Zhi Zhang)
+* ceph init script unconditionally sources /lib/lsb/init-functions (`issue#14402 <http://tracker.ceph.com/issues/14402>`_, `pr#7797 <http://github.com/ceph/ceph/pull/7797>`_, Yan, Zheng)
+* ceph.in: Notify user that 'tell' can't be used in interactive mode (`issue#14773 <http://tracker.ceph.com/issues/14773>`_, `pr#7656 <http://github.com/ceph/ceph/pull/7656>`_, David Zafman)
+* ceph-objectstore-tool, osd: Fix import handling (`issue#10794 <http://tracker.ceph.com/issues/10794>`_, `issue#13382 <http://tracker.ceph.com/issues/13382>`_, `pr#7917 <http://github.com/ceph/ceph/pull/7917>`_, Sage Weil, David Zafman)
+* client: added permission check based on getgrouplist (`issue#13268 <http://tracker.ceph.com/issues/13268>`_, `pr#6604 <http://github.com/ceph/ceph/pull/6604>`_, Yan, Zheng, Danny Al-Gaaf)
+* client: inoderef (`issue#13729 <http://tracker.ceph.com/issues/13729>`_, `pr#6551 <http://github.com/ceph/ceph/pull/6551>`_, Yan, Zheng)
+* common: clock skew report is incorrect by ceph health detail command (`issue#14175 <http://tracker.ceph.com/issues/14175>`_, `pr#8051 <http://github.com/ceph/ceph/pull/8051>`_, Joao Eduardo Luis)
+* global/pidfile: do not start two daemons with a single pid-file (`issue#13422 <http://tracker.ceph.com/issues/13422>`_, `pr#7671 <http://github.com/ceph/ceph/pull/7671>`_, Loic Dachary, shun song)
+* librados: segfault in Objecter::handle_watch_notify (`issue#13805 <http://tracker.ceph.com/issues/13805>`_, `pr#7992 <http://github.com/ceph/ceph/pull/7992>`_, Sage Weil)
+* librbd: flattening an rbd image with active IO can lead to hang (`issue#14092 <http://tracker.ceph.com/issues/14092>`_, `issue#14483 <http://tracker.ceph.com/issues/14483>`_, `pr#7485 <http://github.com/ceph/ceph/pull/7485>`_, Jason Dillaman)
+* librbd: possible QEMU deadlock after creating image snapshots (`issue#14988 <http://tracker.ceph.com/issues/14988>`_, `pr#8011 <http://github.com/ceph/ceph/pull/8011>`_, Jason Dillaman)
+* mon: Bucket owner isn't changed after unlink/link (`issue#11076 <http://tracker.ceph.com/issues/11076>`_, `pr#8583 <http://github.com/ceph/ceph/pull/8583>`_, Zengran Zhang)
+* monclient: avoid key renew storm on clock skew (`issue#12065 <http://tracker.ceph.com/issues/12065>`_, `pr#8398 <http://github.com/ceph/ceph/pull/8398>`_, Alexey Sheplyakov)
+* mon: implement reweight-by-utilization feature (`issue#15054 <http://tracker.ceph.com/issues/15054>`_, `pr#8026 <http://github.com/ceph/ceph/pull/8026>`_, Kefu Chai, Dan van der Ster, Sage Weil)
+* mon/LogMonitor: use the configured facility if log to syslog (`issue#13748 <http://tracker.ceph.com/issues/13748>`_, `pr#7648 <http://github.com/ceph/ceph/pull/7648>`_, Kefu Chai)
+* mon: mon sync does not copy config-key (`issue#14577 <http://tracker.ceph.com/issues/14577>`_, `pr#7576 <http://github.com/ceph/ceph/pull/7576>`_, Xiaowei Chen)
+* mon/OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1 (`issue#15655 <http://tracker.ceph.com/issues/15655>`_, `pr#8979 <http://github.com/ceph/ceph/pull/8979>`_, Samuel Just)
+* osd: consume_maps clearing of waiting_for_pg needs to check the spg_t shard for acting set membership (`issue#14278 <http://tracker.ceph.com/issues/14278>`_, `pr#7577 <http://github.com/ceph/ceph/pull/7577>`_, Samuel Just)
+* osd: log inconsistent shard sizes (`issue#14009 <http://tracker.ceph.com/issues/14009>`_, `pr#6946 <http://github.com/ceph/ceph/pull/6946>`_, Loic Dachary)
+* osd: OSD coredumps with leveldb compact on mount = true (`issue#14748 <http://tracker.ceph.com/issues/14748>`_, `pr#7645 <http://github.com/ceph/ceph/pull/7645>`_, Xiaoxi Chen)
+* osd/OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from (`issue#14686 <http://tracker.ceph.com/issues/14686>`_, `pr#7590 <http://github.com/ceph/ceph/pull/7590>`_, Xinze Chi)
+* osd: Protect against excessively large object map sizes (`issue#15121 <http://tracker.ceph.com/issues/15121>`_, `pr#8401 <http://github.com/ceph/ceph/pull/8401>`_, Jason Dillaman)
+* osd/ReplicatedPG: do not proxy read *and* process op locally (`issue#15171 <http://tracker.ceph.com/issues/15171>`_, `pr#8187 <http://github.com/ceph/ceph/pull/8187>`_, Sage Weil)
+* osd: scrub bogus results when missing a clone (`issue#14875 <http://tracker.ceph.com/issues/14875>`_, `issue#14874 <http://tracker.ceph.com/issues/14874>`_, `issue#14877 <http://tracker.ceph.com/issues/14877>`_, `issue#10098 <http://tracker.ceph.com/issues/10098>`_, `issue#14878 <http://tracker.ceph.com/issues/14878>`_, `issue#14881 <http://tracker.ceph.com/issues/14881>`_, `issue#14882 <http://tracker.ceph.com/issues/14882>`_, `issue#14883 <http://tracker.ceph.com/issues/14883>`_, `issue#14879 <http://tracker.ceph.com/issues/14879>`_, `issue#10290 <http://tracker.ceph.com/issues/10290>`_, `issue#12740 <http://tracker.ceph.com/issues/12740>`_, `issue#12738 <http://tracker.ceph.com/issues/12738>`_, `issue#14880 <http://tracker.ceph.com/issues/14880>`_, `issue#11135 <http://tracker.ceph.com/issues/11135>`_, `issue#14876 <http://tracker.ceph.com/issues/14876>`_, `issue#10809 <http://tracker.ceph.com/issues/10809>`_, `issue#12193 <http://tracker.ceph.com/issues/12193>`_, `issue#11237 <http://tracker.ceph.com/issues/11237>`_, `pr#7702 <http://github.com/ceph/ceph/pull/7702>`_, Xinze Chi, Sage Weil, John Spray, Kefu Chai, Mykola Golub, David Zafman)
+* osd: Unable to bring up OSD's after dealing with FULL cluster (OSD assert with /include/interval_set.h: 386: FAILED assert(_size >= 0)) (`issue#14428 <http://tracker.ceph.com/issues/14428>`_, `pr#7415 <http://github.com/ceph/ceph/pull/7415>`_, Alexey Sheplyakov)
+* osd: use GMT time for the object name of hitsets (`issue#13192 <http://tracker.ceph.com/issues/13192>`_, `issue#9732 <http://tracker.ceph.com/issues/9732>`_, `issue#12968 <http://tracker.ceph.com/issues/12968>`_, `pr#7883 <http://github.com/ceph/ceph/pull/7883>`_, Kefu Chai, David Zafman)
+* qa/workunits/post-file.sh: sudo (`issue#14586 <http://tracker.ceph.com/issues/14586>`_, `pr#7456 <http://github.com/ceph/ceph/pull/7456>`_, Sage Weil)
+* qa/workunits: remove 'mds setmap' from workunits (`pr#8123 <http://github.com/ceph/ceph/pull/8123>`_, Sage Weil)
+* rgw: default quota params (`issue#12997 <http://tracker.ceph.com/issues/12997>`_, `pr#7188 <http://github.com/ceph/ceph/pull/7188>`_, Daniel Gryniewicz)
+* rgw: make rgw_fronends more forgiving of whitespace (`issue#12038 <http://tracker.ceph.com/issues/12038>`_, `pr#7414 <http://github.com/ceph/ceph/pull/7414>`_, Matt Benjamin)
+* rgw: radosgw-admin bucket check --fix not work (`issue#14215 <http://tracker.ceph.com/issues/14215>`_, `pr#7185 <http://github.com/ceph/ceph/pull/7185>`_, Weijun Duan)
+* rpm package building fails if the build machine has lttng and babeltrace development packages installed locally (`issue#14844 <http://tracker.ceph.com/issues/14844>`_, `pr#8440 <http://github.com/ceph/ceph/pull/8440>`_, Kefu Chai)
+* rpm: redhat-lsb-core dependency was dropped, but is still needed (`issue#14906 <http://tracker.ceph.com/issues/14906>`_, `pr#7876 <http://github.com/ceph/ceph/pull/7876>`_, Nathan Cutler)
+* test_bit_vector.cc uses magic numbers against #defines that vary (`issue#14747 <http://tracker.ceph.com/issues/14747>`_, `pr#7672 <http://github.com/ceph/ceph/pull/7672>`_, Jason Dillaman)
+* test/librados/tier.cc doesn't completely clean up EC pools (`issue#13878 <http://tracker.ceph.com/issues/13878>`_, `pr#8052 <http://github.com/ceph/ceph/pull/8052>`_, Loic Dachary, Dan Mick)
+* tests: bufferlist: do not expect !is_page_aligned() after unaligned rebuild (`issue#15305 <http://tracker.ceph.com/issues/15305>`_, `pr#8272 <http://github.com/ceph/ceph/pull/8272>`_, Kefu Chai)
+* tools: fix race condition in seq/rand bench (part 1) (`issue#14968 <http://tracker.ceph.com/issues/14968>`_, `issue#14873 <http://tracker.ceph.com/issues/14873>`_, `pr#7896 <http://github.com/ceph/ceph/pull/7896>`_, Alexey Sheplyakov, Piotr Dałek)
+* tools: fix race condition in seq/rand bench (part 2) (`issue#14873 <http://tracker.ceph.com/issues/14873>`_, `pr#7817 <http://github.com/ceph/ceph/pull/7817>`_, Alexey Sheplyakov)
+* tools/rados: add bench smoke tests (`issue#14971 <http://tracker.ceph.com/issues/14971>`_, `pr#7903 <http://github.com/ceph/ceph/pull/7903>`_, Piotr Dałek)
+* tools, test: Add ceph-objectstore-tool to operate on the meta collection (`issue#14977 <http://tracker.ceph.com/issues/14977>`_, `pr#7911 <http://github.com/ceph/ceph/pull/7911>`_, David Zafman)
+* unittest_crypto: benchmark 100,000 CryptoKey::encrypt() calls (`issue#14863 <http://tracker.ceph.com/issues/14863>`_, `pr#7801 <http://github.com/ceph/ceph/pull/7801>`_, Sage Weil)
+
+
+v0.94.6 Hammer
+======================
+
+This Hammer point release fixes a range of bugs, most notably a fix
+for unbounded growth of the monitor's leveldb store, and a workaround
+in the OSD to keep most xattrs small enough to be stored inline in XFS
+inodes.
+
+We recommend that all hammer v0.94.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.6.txt>`.
+
+Notable Changes
+---------------
+* build/ops: Ceph daemon failed to start, because the service name was already used. (`issue#13474 <http://tracker.ceph.com/issues/13474>`_, `pr#6832 <http://github.com/ceph/ceph/pull/6832>`_, Chuanhong Wang)
+* build/ops: LTTng-UST tracing should be dynamically enabled (`issue#13274 <http://tracker.ceph.com/issues/13274>`_, `pr#6415 <http://github.com/ceph/ceph/pull/6415>`_, Jason Dillaman)
+* build/ops: ceph upstart script rbdmap.conf incorrectly processes parameters (`issue#13214 <http://tracker.ceph.com/issues/13214>`_, `pr#6159 <http://github.com/ceph/ceph/pull/6159>`_, Sage Weil)
+* build/ops: ceph.spec.in License line does not reflect COPYING (`issue#12935 <http://tracker.ceph.com/issues/12935>`_, `pr#6680 <http://github.com/ceph/ceph/pull/6680>`_, Nathan Cutler)
+* build/ops: ceph.spec.in libcephfs_jni1 has no %post and %postun  (`issue#12927 <http://tracker.ceph.com/issues/12927>`_, `pr#5789 <http://github.com/ceph/ceph/pull/5789>`_, Owen Synge)
+* build/ops: configure.ac: no use to add "+" before ac_ext=c (`issue#14330 <http://tracker.ceph.com/issues/14330>`_, `pr#6973 <http://github.com/ceph/ceph/pull/6973>`_, Kefu Chai, Robin H. Johnson)
+* build/ops: deb: strip tracepoint libraries from Wheezy/Precise builds (`issue#14801 <http://tracker.ceph.com/issues/14801>`_, `pr#7316 <http://github.com/ceph/ceph/pull/7316>`_, Jason Dillaman)
+* build/ops: init script reload doesn't work on EL7 (`issue#13709 <http://tracker.ceph.com/issues/13709>`_, `pr#7187 <http://github.com/ceph/ceph/pull/7187>`_, Hervé Rousseau)
+* build/ops: init-rbdmap uses distro-specific functions (`issue#12415 <http://tracker.ceph.com/issues/12415>`_, `pr#6528 <http://github.com/ceph/ceph/pull/6528>`_, Boris Ranto)
+* build/ops: logrotate reload error on Ubuntu 14.04 (`issue#11330 <http://tracker.ceph.com/issues/11330>`_, `pr#5787 <http://github.com/ceph/ceph/pull/5787>`_, Sage Weil)
+* build/ops: miscellaneous spec file fixes (`issue#12931 <http://tracker.ceph.com/issues/12931>`_, `issue#12994 <http://tracker.ceph.com/issues/12994>`_, `issue#12924 <http://tracker.ceph.com/issues/12924>`_, `issue#12360 <http://tracker.ceph.com/issues/12360>`_, `pr#5790 <http://github.com/ceph/ceph/pull/5790>`_, Boris Ranto, Nathan Cutler, Owen Synge, Travis Rhoden, Ken Dreyer)
+* build/ops: pass tcmalloc env through to ceph-os (`issue#14802 <http://tracker.ceph.com/issues/14802>`_, `pr#7365 <http://github.com/ceph/ceph/pull/7365>`_, Sage Weil)
+* build/ops: rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well (`issue#13785 <http://tracker.ceph.com/issues/13785>`_, `pr#6580 <http://github.com/ceph/ceph/pull/6580>`_, Loic Dachary)
+* build/ops: unknown argument --quiet in udevadm settle (`issue#13560 <http://tracker.ceph.com/issues/13560>`_, `pr#6530 <http://github.com/ceph/ceph/pull/6530>`_, Jason Dillaman)
+* common: Objecter: pool op callback may hang forever. (`issue#13642 <http://tracker.ceph.com/issues/13642>`_, `pr#6588 <http://github.com/ceph/ceph/pull/6588>`_, xie xingguo)
+* common: Objecter: potential null pointer access when do pool_snap_list. (`issue#13639 <http://tracker.ceph.com/issues/13639>`_, `pr#6839 <http://github.com/ceph/ceph/pull/6839>`_, xie xingguo)
+* common: ThreadPool add/remove work queue methods not thread safe (`issue#12662 <http://tracker.ceph.com/issues/12662>`_, `pr#5889 <http://github.com/ceph/ceph/pull/5889>`_, Jason Dillaman)
+* common: auth/cephx: large amounts of log are produced by osd (`issue#13610 <http://tracker.ceph.com/issues/13610>`_, `pr#6835 <http://github.com/ceph/ceph/pull/6835>`_, Qiankun Zheng)
+* common: client nonce collision due to unshared pid namespaces (`issue#13032 <http://tracker.ceph.com/issues/13032>`_, `pr#6151 <http://github.com/ceph/ceph/pull/6151>`_, Josh Durgin)
+* common: common/Thread:pthread_attr_destroy(thread_attr) when done with it (`issue#12570 <http://tracker.ceph.com/issues/12570>`_, `pr#6157 <http://github.com/ceph/ceph/pull/6157>`_, Piotr Dałek)
+* common: log: Log.cc: Assign LOG_DEBUG priority to syslog calls (`issue#13993 <http://tracker.ceph.com/issues/13993>`_, `pr#6994 <http://github.com/ceph/ceph/pull/6994>`_, Brad Hubbard)
+* common: objecter: cancellation bugs (`issue#13071 <http://tracker.ceph.com/issues/13071>`_, `pr#6155 <http://github.com/ceph/ceph/pull/6155>`_, Jianpeng Ma)
+* common: pure virtual method called (`issue#13636 <http://tracker.ceph.com/issues/13636>`_, `pr#6587 <http://github.com/ceph/ceph/pull/6587>`_, Jason Dillaman)
+* common: small probability sigabrt when setting rados_osd_op_timeout (`issue#13208 <http://tracker.ceph.com/issues/13208>`_, `pr#6143 <http://github.com/ceph/ceph/pull/6143>`_, Ruifeng Yang)
+* common: wrong conditional for boolean function KeyServer::get_auth() (`issue#9756 <http://tracker.ceph.com/issues/9756>`_, `issue#13424 <http://tracker.ceph.com/issues/13424>`_, `pr#6213 <http://github.com/ceph/ceph/pull/6213>`_, Nathan Cutler)
+* crush: crash if we see CRUSH_ITEM_NONE in early rule step (`issue#13477 <http://tracker.ceph.com/issues/13477>`_, `pr#6430 <http://github.com/ceph/ceph/pull/6430>`_, Sage Weil)
+* doc: man: document listwatchers cmd in "rados" manpage (`issue#14556 <http://tracker.ceph.com/issues/14556>`_, `pr#7434 <http://github.com/ceph/ceph/pull/7434>`_, Kefu Chai)
+* doc: regenerate man pages, add orphans commands to radosgw-admin(8) (`issue#14637 <http://tracker.ceph.com/issues/14637>`_, `pr#7524 <http://github.com/ceph/ceph/pull/7524>`_, Ken Dreyer)
+* fs: CephFS restriction on removing cache tiers is overly strict (`issue#11504 <http://tracker.ceph.com/issues/11504>`_, `pr#6402 <http://github.com/ceph/ceph/pull/6402>`_, John Spray)
+* fs: fsstress.sh fails (`issue#12710 <http://tracker.ceph.com/issues/12710>`_, `pr#7454 <http://github.com/ceph/ceph/pull/7454>`_, Yan, Zheng)
+* librados: LibRadosWatchNotify.WatchNotify2Timeout (`issue#13114 <http://tracker.ceph.com/issues/13114>`_, `pr#6336 <http://github.com/ceph/ceph/pull/6336>`_, Sage Weil)
+* librbd: ImageWatcher shouldn't block the notification thread (`issue#14373 <http://tracker.ceph.com/issues/14373>`_, `pr#7407 <http://github.com/ceph/ceph/pull/7407>`_, Jason Dillaman)
+* librbd: diff_iterate needs to handle holes in parent images (`issue#12885 <http://tracker.ceph.com/issues/12885>`_, `pr#6097 <http://github.com/ceph/ceph/pull/6097>`_, Jason Dillaman)
+* librbd: fix merge-diff for >2GB diff-files (`issue#14030 <http://tracker.ceph.com/issues/14030>`_, `pr#6980 <http://github.com/ceph/ceph/pull/6980>`_, Jason Dillaman)
+* librbd: invalidate object map on error even w/o holding lock (`issue#13372 <http://tracker.ceph.com/issues/13372>`_, `pr#6289 <http://github.com/ceph/ceph/pull/6289>`_, Jason Dillaman)
+* librbd: reads larger than cache size hang (`issue#13164 <http://tracker.ceph.com/issues/13164>`_, `pr#6354 <http://github.com/ceph/ceph/pull/6354>`_, Lu Shi)
+* mds: ceph mds add_data_pool check for EC pool is wrong (`issue#12426 <http://tracker.ceph.com/issues/12426>`_, `pr#5766 <http://github.com/ceph/ceph/pull/5766>`_, John Spray)
+* mon: MonitorDBStore: get_next_key() only if prefix matches (`issue#11786 <http://tracker.ceph.com/issues/11786>`_, `pr#5361 <http://github.com/ceph/ceph/pull/5361>`_, Joao Eduardo Luis)
+* mon: OSDMonitor: do not assume a session exists in send_incremental() (`issue#14236 <http://tracker.ceph.com/issues/14236>`_, `pr#7150 <http://github.com/ceph/ceph/pull/7150>`_, Joao Eduardo Luis)
+* mon: check for store writeablility before participating in election (`issue#13089 <http://tracker.ceph.com/issues/13089>`_, `pr#6144 <http://github.com/ceph/ceph/pull/6144>`_, Sage Weil)
+* mon: compact full epochs also (`issue#14537 <http://tracker.ceph.com/issues/14537>`_, `pr#7446 <http://github.com/ceph/ceph/pull/7446>`_, Kefu Chai)
+* mon: include min_last_epoch_clean as part of PGMap::print_summary and PGMap::dump (`issue#13198 <http://tracker.ceph.com/issues/13198>`_, `pr#6152 <http://github.com/ceph/ceph/pull/6152>`_, Guang Yang)
+* mon: map_cache can become inaccurate if osd does not receive the osdmaps (`issue#10930 <http://tracker.ceph.com/issues/10930>`_, `pr#5773 <http://github.com/ceph/ceph/pull/5773>`_, Kefu Chai)
+* mon: should not set isvalid = true when cephx_verify_authorizer return false (`issue#13525 <http://tracker.ceph.com/issues/13525>`_, `pr#6391 <http://github.com/ceph/ceph/pull/6391>`_, Ruifeng Yang)
+* osd: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0 (`issue#13840 <http://tracker.ceph.com/issues/13840>`_, `pr#6834 <http://github.com/ceph/ceph/pull/6834>`_, Chengyuan Li)
+* osd: FileStore calls syncfs(2) even it is not supported (`issue#12512 <http://tracker.ceph.com/issues/12512>`_, `pr#5530 <http://github.com/ceph/ceph/pull/5530>`_, Kefu Chai)
+* osd: FileStore: potential memory leak if getattrs fails. (`issue#13597 <http://tracker.ceph.com/issues/13597>`_, `pr#6420 <http://github.com/ceph/ceph/pull/6420>`_, xie xingguo)
+* osd: IO error on kvm/rbd with an erasure coded pool tier (`issue#12012 <http://tracker.ceph.com/issues/12012>`_, `pr#5897 <http://github.com/ceph/ceph/pull/5897>`_, Kefu Chai)
+* osd: OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval. (`issue#13471 <http://tracker.ceph.com/issues/13471>`_, `pr#6398 <http://github.com/ceph/ceph/pull/6398>`_, xiexingguo)
+* osd: ReplicatedBackend: populate recovery_info.size for clone (bug symptom is size mismatch on replicated backend on a clone in scrub) (`issue#12828 <http://tracker.ceph.com/issues/12828>`_, `pr#6153 <http://github.com/ceph/ceph/pull/6153>`_, Samuel Just)
+* osd: ReplicatedPG: wrong result code checking logic during sparse_read (`issue#14151 <http://tracker.ceph.com/issues/14151>`_, `pr#7179 <http://github.com/ceph/ceph/pull/7179>`_, xie xingguo)
+* osd: ReplicatedPG::hit_set_trim osd/ReplicatedPG.cc: 11006: FAILED assert(obc) (`issue#13192 <http://tracker.ceph.com/issues/13192>`_, `issue#9732 <http://tracker.ceph.com/issues/9732>`_, `issue#12968 <http://tracker.ceph.com/issues/12968>`_, `pr#5825 <http://github.com/ceph/ceph/pull/5825>`_, Kefu Chai, Zhiqiang Wang, Samuel Just, David Zafman)
+* osd: avoid multi set osd_op.outdata in tier pool (`issue#12540 <http://tracker.ceph.com/issues/12540>`_, `pr#6060 <http://github.com/ceph/ceph/pull/6060>`_, Xinze Chi)
+* osd: bug with cache/tiering and snapshot reads (`issue#12748 <http://tracker.ceph.com/issues/12748>`_, `pr#6589 <http://github.com/ceph/ceph/pull/6589>`_, Kefu Chai)
+* osd: ceph osd pool stats broken in hammer (`issue#13843 <http://tracker.ceph.com/issues/13843>`_, `pr#7180 <http://github.com/ceph/ceph/pull/7180>`_, BJ Lougee)
+* osd: ceph-disk prepare fails if device is a symlink (`issue#13438 <http://tracker.ceph.com/issues/13438>`_, `pr#7176 <http://github.com/ceph/ceph/pull/7176>`_, Joe Julian)
+* osd: check for full before changing the cached obc (hammer) (`issue#13098 <http://tracker.ceph.com/issues/13098>`_, `pr#6918 <http://github.com/ceph/ceph/pull/6918>`_, Alexey Sheplyakov)
+* osd: config_opts: increase suicide timeout to 300 to match recovery (`issue#14376 <http://tracker.ceph.com/issues/14376>`_, `pr#7236 <http://github.com/ceph/ceph/pull/7236>`_, Samuel Just)
+* osd: disable filestore_xfs_extsize by default (`issue#14397 <http://tracker.ceph.com/issues/14397>`_, `pr#7411 <http://github.com/ceph/ceph/pull/7411>`_, Ken Dreyer)
+* osd: do not cache unused memory in attrs (`issue#12565 <http://tracker.ceph.com/issues/12565>`_, `pr#6499 <http://github.com/ceph/ceph/pull/6499>`_, Xinze Chi, Ning Yao)
+* osd: dumpling incrementals do not work properly on hammer and newer (`issue#13234 <http://tracker.ceph.com/issues/13234>`_, `pr#6132 <http://github.com/ceph/ceph/pull/6132>`_, Samuel Just)
+* osd: filestore: fix peek_queue for OpSequencer (`issue#13209 <http://tracker.ceph.com/issues/13209>`_, `pr#6145 <http://github.com/ceph/ceph/pull/6145>`_, Xinze Chi)
+* osd: hit set clear repops fired in same epoch as map change -- segfault since they fall into the new interval even though the repops are cleared (`issue#12809 <http://tracker.ceph.com/issues/12809>`_, `pr#5890 <http://github.com/ceph/ceph/pull/5890>`_, Samuel Just)
+* osd: object_info_t::decode() has wrong version (`issue#13462 <http://tracker.ceph.com/issues/13462>`_, `pr#6335 <http://github.com/ceph/ceph/pull/6335>`_, David Zafman)
+* osd: osd/OSD.cc: 2469: FAILED assert(pg_stat_queue.empty()) on shutdown (`issue#14212 <http://tracker.ceph.com/issues/14212>`_, `pr#7178 <http://github.com/ceph/ceph/pull/7178>`_, Sage Weil)
+* osd: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started) (`issue#14015 <http://tracker.ceph.com/issues/14015>`_, `pr#7177 <http://github.com/ceph/ceph/pull/7177>`_, David Zafman)
+* osd: osd/PG.cc: 3837: FAILED assert(0 == "Running incompatible OSD") (`issue#11661 <http://tracker.ceph.com/issues/11661>`_, `pr#7206 <http://github.com/ceph/ceph/pull/7206>`_, David Zafman)
+* osd: osd/ReplicatedPG: Recency fix (`issue#14320 <http://tracker.ceph.com/issues/14320>`_, `pr#7207 <http://github.com/ceph/ceph/pull/7207>`_, Sage Weil, Robert LeBlanc)
+* osd: pg stuck in replay (`issue#13116 <http://tracker.ceph.com/issues/13116>`_, `pr#6401 <http://github.com/ceph/ceph/pull/6401>`_, Sage Weil)
+* osd: race condition detected during send_failures (`issue#13821 <http://tracker.ceph.com/issues/13821>`_, `pr#6755 <http://github.com/ceph/ceph/pull/6755>`_, Sage Weil)
+* osd: randomize scrub times (`issue#10973 <http://tracker.ceph.com/issues/10973>`_, `pr#6199 <http://github.com/ceph/ceph/pull/6199>`_, Kefu Chai)
+* osd: requeue_scrub when kick_object_context_blocked (`issue#12515 <http://tracker.ceph.com/issues/12515>`_, `pr#5891 <http://github.com/ceph/ceph/pull/5891>`_, Xinze Chi)
+* osd: revert: use GMT time for hitsets (`issue#13812 <http://tracker.ceph.com/issues/13812>`_, `pr#6644 <http://github.com/ceph/ceph/pull/6644>`_, Loic Dachary)
+* osd: segfault in agent_work (`issue#13199 <http://tracker.ceph.com/issues/13199>`_, `pr#6146 <http://github.com/ceph/ceph/pull/6146>`_, Samuel Just)
+* osd: should recalc the min_last_epoch_clean when decode PGMap (`issue#13112 <http://tracker.ceph.com/issues/13112>`_, `pr#6154 <http://github.com/ceph/ceph/pull/6154>`_, Kefu Chai)
+* osd: smaller object_info_t xattrs (`issue#14803 <http://tracker.ceph.com/issues/14803>`_, `pr#6544 <http://github.com/ceph/ceph/pull/6544>`_, Sage Weil)
+* osd: we do not ignore notify from down osds (`issue#12990 <http://tracker.ceph.com/issues/12990>`_, `pr#6158 <http://github.com/ceph/ceph/pull/6158>`_, Samuel Just)
+* rbd: QEMU hangs after creating snapshot and stopping VM (`issue#13726 <http://tracker.ceph.com/issues/13726>`_, `pr#6586 <http://github.com/ceph/ceph/pull/6586>`_, Jason Dillaman)
+* rbd: TaskFinisher::cancel should remove event from SafeTimer (`issue#14476 <http://tracker.ceph.com/issues/14476>`_, `pr#7417 <http://github.com/ceph/ceph/pull/7417>`_, Douglas Fuller)
+* rbd: avoid re-writing old-format image header on resize (`issue#13674 <http://tracker.ceph.com/issues/13674>`_, `pr#6585 <http://github.com/ceph/ceph/pull/6585>`_, Jason Dillaman)
+* rbd: fix bench-write (`issue#14225 <http://tracker.ceph.com/issues/14225>`_, `pr#7183 <http://github.com/ceph/ceph/pull/7183>`_, Sage Weil)
+* rbd: rbd-replay does not check for EOF and goes to endless loop (`issue#14452 <http://tracker.ceph.com/issues/14452>`_, `pr#7416 <http://github.com/ceph/ceph/pull/7416>`_, Mykola Golub)
+* rbd: rbd-replay-prep and rbd-replay improvements (`issue#13221 <http://tracker.ceph.com/issues/13221>`_, `issue#13220 <http://tracker.ceph.com/issues/13220>`_, `issue#13378 <http://tracker.ceph.com/issues/13378>`_, `pr#6286 <http://github.com/ceph/ceph/pull/6286>`_, Jason Dillaman)
+* rbd: verify self-managed snapshot functionality on image create (`issue#13633 <http://tracker.ceph.com/issues/13633>`_, `pr#7182 <http://github.com/ceph/ceph/pull/7182>`_, Jason Dillaman)
+* rgw: Make RGW_MAX_PUT_SIZE configurable (`issue#6999 <http://tracker.ceph.com/issues/6999>`_, `pr#7441 <http://github.com/ceph/ceph/pull/7441>`_, Vladislav Odintsov, Yuan Zhou)
+* rgw: Setting ACL on Object removes ETag (`issue#12955 <http://tracker.ceph.com/issues/12955>`_, `pr#6620 <http://github.com/ceph/ceph/pull/6620>`_, Brian Felton)
+* rgw: backport content-type casing (`issue#12939 <http://tracker.ceph.com/issues/12939>`_, `pr#5910 <http://github.com/ceph/ceph/pull/5910>`_, Robin H. Johnson)
+* rgw: bucket listing hangs on versioned buckets (`issue#12913 <http://tracker.ceph.com/issues/12913>`_, `pr#6352 <http://github.com/ceph/ceph/pull/6352>`_, Yehuda Sadeh)
+* rgw: fix wrong etag calculation during POST on S3 bucket. (`issue#11241 <http://tracker.ceph.com/issues/11241>`_, `pr#7442 <http://github.com/ceph/ceph/pull/7442>`_, Vladislav Odintsov, Radoslaw Zarzynski)
+* rgw: get bucket location returns region name, not region api name (`issue#13458 <http://tracker.ceph.com/issues/13458>`_, `pr#6349 <http://github.com/ceph/ceph/pull/6349>`_, Yehuda Sadeh)
+* rgw: missing handling of encoding-type=url when listing keys in bucket (`issue#12735 <http://tracker.ceph.com/issues/12735>`_, `pr#6527 <http://github.com/ceph/ceph/pull/6527>`_, Jeff Weber)
+* rgw: orphan tool should be careful about removing head objects (`issue#12958 <http://tracker.ceph.com/issues/12958>`_, `pr#6351 <http://github.com/ceph/ceph/pull/6351>`_, Yehuda Sadeh)
+* rgw: orphans finish segfaults (`issue#13824 <http://tracker.ceph.com/issues/13824>`_, `pr#7186 <http://github.com/ceph/ceph/pull/7186>`_, Igor Fedotov)
+* rgw: rgw-admin: document orphans commands in usage (`issue#14516 <http://tracker.ceph.com/issues/14516>`_, `pr#7526 <http://github.com/ceph/ceph/pull/7526>`_, Yehuda Sadeh)
+* rgw: swift API returns more than real object count and bytes used when retrieving account metadata (`issue#13140 <http://tracker.ceph.com/issues/13140>`_, `pr#6512 <http://github.com/ceph/ceph/pull/6512>`_, Sangdi Xu)
+* rgw: swift use Civetweb ssl can not get right url (`issue#13628 <http://tracker.ceph.com/issues/13628>`_, `pr#6491 <http://github.com/ceph/ceph/pull/6491>`_, Weijun Duan)
+* rgw: value of Swift API's X-Object-Manifest header is not url_decoded during segment look up (`issue#12728 <http://tracker.ceph.com/issues/12728>`_, `pr#6353 <http://github.com/ceph/ceph/pull/6353>`_, Radoslaw Zarzynski)
+* tests: fixed broken Makefiles after integration of ttng into rados (`issue#13210 <http://tracker.ceph.com/issues/13210>`_, `pr#6322 <http://github.com/ceph/ceph/pull/6322>`_, Sebastien Ponce)
+* tests: fsx failed to compile (`issue#14384 <http://tracker.ceph.com/issues/14384>`_, `pr#7501 <http://github.com/ceph/ceph/pull/7501>`_, Greg Farnum)
+* tests: notification slave needs to wait for master (`issue#13810 <http://tracker.ceph.com/issues/13810>`_, `pr#7226 <http://github.com/ceph/ceph/pull/7226>`_, Jason Dillaman)
+* tests: qa: remove legacy OS support from rbd/qemu-iotests (`issue#13483 <http://tracker.ceph.com/issues/13483>`_, `issue#14385 <http://tracker.ceph.com/issues/14385>`_, `pr#7252 <http://github.com/ceph/ceph/pull/7252>`_, Vasu Kulkarni, Jason Dillaman)
+* tests: testprofile must be removed before it is re-created (`issue#13664 <http://tracker.ceph.com/issues/13664>`_, `pr#6450 <http://github.com/ceph/ceph/pull/6450>`_, Loic Dachary)
+* tools: ceph-monstore-tool must do out_store.close() (`issue#10093 <http://tracker.ceph.com/issues/10093>`_, `pr#7347 <http://github.com/ceph/ceph/pull/7347>`_, huangjun)
+* tools: heavy memory shuffling in rados bench (`issue#12946 <http://tracker.ceph.com/issues/12946>`_, `pr#5810 <http://github.com/ceph/ceph/pull/5810>`_, Piotr Dałek)
+* tools: race condition in rados bench (`issue#12947 <http://tracker.ceph.com/issues/12947>`_, `pr#6791 <http://github.com/ceph/ceph/pull/6791>`_, Piotr Dałek)
+* tools: tool for artificially inflate the leveldb of the mon store for testing purposes  (`issue#10093 <http://tracker.ceph.com/issues/10093>`_, `issue#11815 <http://tracker.ceph.com/issues/11815>`_, `issue#14217 <http://tracker.ceph.com/issues/14217>`_, `pr#7412 <http://github.com/ceph/ceph/pull/7412>`_, Cilang Zhao, Bo Cai, Kefu Chai, huangjun, Joao Eduardo Luis)
+
+v0.94.5 Hammer
+==============
+
+This Hammer point release fixes a critical regression in librbd that can cause
+QEMU/KVM to crash when caching is enabled on images that have been cloned.
+
+All v0.94.4 Hammer users are strongly encouraged to upgrade.
+
+Notable Changes
+---------------
+* librbd: potential assertion failure during cache read (`issue#13559 <http://tracker.ceph.com/issues/13559>`_, `pr#6348 <http://github.com/ceph/ceph/pull/6348>`_, Jason Dillaman)
+* osd: osd/ReplicatedPG: remove stray debug line (`issue#13455 <http://tracker.ceph.com/issues/13455>`_, `pr#6362 <http://github.com/ceph/ceph/pull/6362>`_, Sage Weil)
+* tests: qemu workunit refers to apt-mirror.front.sepia.ceph.com (`issue#13420 <http://tracker.ceph.com/issues/13420>`_, `pr#6330 <http://github.com/ceph/ceph/pull/6330>`_, Yuan Zhou)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.5.txt>`.
+
+v0.94.4 Hammer
+==============
+
+This Hammer point release fixes several important bugs in Hammer, as well as
+fixing interoperability issues that are required before an upgrade to
+Infernalis. That is, all users of earlier version of Hammer or any
+version of Firefly will first need to upgrade to hammer v0.94.4 or
+later before upgrading to Infernalis (or future releases).
+
+All v0.94.x Hammer users are strongly encouraged to upgrade.
+
+Notable Changes
+---------------
+* build/ops: ceph.spec.in: 50-rbd.rules conditional is wrong (`issue#12166 <http://tracker.ceph.com/issues/12166>`_, `pr#5207 <http://github.com/ceph/ceph/pull/5207>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: ceph-common needs python-argparse on older distros, but doesn't require it (`issue#12034 <http://tracker.ceph.com/issues/12034>`_, `pr#5216 <http://github.com/ceph/ceph/pull/5216>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: radosgw requires apache for SUSE only -- makes no sense (`issue#12358 <http://tracker.ceph.com/issues/12358>`_, `pr#5411 <http://github.com/ceph/ceph/pull/5411>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: rpm: cephfs_java not fully conditionalized (`issue#11991 <http://tracker.ceph.com/issues/11991>`_, `pr#5202 <http://github.com/ceph/ceph/pull/5202>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: rpm: not possible to turn off Java (`issue#11992 <http://tracker.ceph.com/issues/11992>`_, `pr#5203 <http://github.com/ceph/ceph/pull/5203>`_, Owen Synge)
+* build/ops: ceph.spec.in: running fdupes unnecessarily (`issue#12301 <http://tracker.ceph.com/issues/12301>`_, `pr#5223 <http://github.com/ceph/ceph/pull/5223>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: snappy-devel for all supported distros (`issue#12361 <http://tracker.ceph.com/issues/12361>`_, `pr#5264 <http://github.com/ceph/ceph/pull/5264>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: SUSE/openSUSE builds need libbz2-devel (`issue#11629 <http://tracker.ceph.com/issues/11629>`_, `pr#5204 <http://github.com/ceph/ceph/pull/5204>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: useless %py_requires breaks SLE11-SP3 build (`issue#12351 <http://tracker.ceph.com/issues/12351>`_, `pr#5412 <http://github.com/ceph/ceph/pull/5412>`_, Nathan Cutler)
+* build/ops: error in ext_mime_map_init() when /etc/mime.types is missing (`issue#11864 <http://tracker.ceph.com/issues/11864>`_, `pr#5385 <http://github.com/ceph/ceph/pull/5385>`_, Ken Dreyer)
+* build/ops: upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s) (`issue#11798 <http://tracker.ceph.com/issues/11798>`_, `pr#5930 <http://github.com/ceph/ceph/pull/5930>`_, Sage Weil)
+* build/ops: With root as default user, unable to have multiple RGW instances running (`issue#10927 <http://tracker.ceph.com/issues/10927>`_, `pr#6161 <http://github.com/ceph/ceph/pull/6161>`_, Sage Weil)
+* build/ops: With root as default user, unable to have multiple RGW instances running (`issue#11140 <http://tracker.ceph.com/issues/11140>`_, `pr#6161 <http://github.com/ceph/ceph/pull/6161>`_, Sage Weil)
+* build/ops: With root as default user, unable to have multiple RGW instances running (`issue#11686 <http://tracker.ceph.com/issues/11686>`_, `pr#6161 <http://github.com/ceph/ceph/pull/6161>`_, Sage Weil)
+* build/ops: With root as default user, unable to have multiple RGW instances running (`issue#12407 <http://tracker.ceph.com/issues/12407>`_, `pr#6161 <http://github.com/ceph/ceph/pull/6161>`_, Sage Weil)
+* cli: ceph: cli throws exception on unrecognized errno (`issue#11354 <http://tracker.ceph.com/issues/11354>`_, `pr#5368 <http://github.com/ceph/ceph/pull/5368>`_, Kefu Chai)
+* cli: ceph tell: broken error message / misleading hinting (`issue#11101 <http://tracker.ceph.com/issues/11101>`_, `pr#5371 <http://github.com/ceph/ceph/pull/5371>`_, Kefu Chai)
+* common: arm: all programs that link to librados2 hang forever on startup (`issue#12505 <http://tracker.ceph.com/issues/12505>`_, `pr#5366 <http://github.com/ceph/ceph/pull/5366>`_, Boris Ranto)
+* common: buffer: critical bufferlist::zero bug (`issue#12252 <http://tracker.ceph.com/issues/12252>`_, `pr#5365 <http://github.com/ceph/ceph/pull/5365>`_, Haomai Wang)
+* common: ceph-object-corpus: add 0.94.2-207-g88e7ee7 hammer objects (`issue#13070 <http://tracker.ceph.com/issues/13070>`_, `pr#5551 <http://github.com/ceph/ceph/pull/5551>`_, Sage Weil)
+* common: do not insert emtpy ptr when rebuild emtpy bufferlist (`issue#12775 <http://tracker.ceph.com/issues/12775>`_, `pr#5764 <http://github.com/ceph/ceph/pull/5764>`_, Xinze Chi)
+* common: [  FAILED  ] TestLibRBD.BlockingAIO (`issue#12479 <http://tracker.ceph.com/issues/12479>`_, `pr#5768 <http://github.com/ceph/ceph/pull/5768>`_, Jason Dillaman)
+* common: LibCephFS.GetPoolId failure (`issue#12598 <http://tracker.ceph.com/issues/12598>`_, `pr#5887 <http://github.com/ceph/ceph/pull/5887>`_, Yan, Zheng)
+* common: Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy (`issue#11762 <http://tracker.ceph.com/issues/11762>`_, `pr#5378 <http://github.com/ceph/ceph/pull/5378>`_, Ketor Meng)
+* common: object_map_update fails with -EINVAL return code (`issue#12611 <http://tracker.ceph.com/issues/12611>`_, `pr#5559 <http://github.com/ceph/ceph/pull/5559>`_, Jason Dillaman)
+* common: Pipe: Drop connect_seq increase line (`issue#13093 <http://tracker.ceph.com/issues/13093>`_, `pr#5908 <http://github.com/ceph/ceph/pull/5908>`_, Haomai Wang)
+* common: recursive lock of md_config_t (0) (`issue#12614 <http://tracker.ceph.com/issues/12614>`_, `pr#5759 <http://github.com/ceph/ceph/pull/5759>`_, Josh Durgin)
+* crush: ceph osd crush reweight-subtree does not reweight parent node (`issue#11855 <http://tracker.ceph.com/issues/11855>`_, `pr#5374 <http://github.com/ceph/ceph/pull/5374>`_, Sage Weil)
+* doc: update docs to point to download.ceph.com (`issue#13162 <http://tracker.ceph.com/issues/13162>`_, `pr#6156 <http://github.com/ceph/ceph/pull/6156>`_, Alfredo Deza)
+* fs: ceph-fuse 0.94.2-1trusty segfaults / aborts (`issue#12297 <http://tracker.ceph.com/issues/12297>`_, `pr#5381 <http://github.com/ceph/ceph/pull/5381>`_, Greg Farnum)
+* fs: segfault launching ceph-fuse with bad --name (`issue#12417 <http://tracker.ceph.com/issues/12417>`_, `pr#5382 <http://github.com/ceph/ceph/pull/5382>`_, John Spray)
+* librados: Change radosgw pools default crush ruleset (`issue#11640 <http://tracker.ceph.com/issues/11640>`_, `pr#5754 <http://github.com/ceph/ceph/pull/5754>`_, Yuan Zhou)
+* librbd: correct issues discovered via lockdep / helgrind (`issue#12345 <http://tracker.ceph.com/issues/12345>`_, `pr#5296 <http://github.com/ceph/ceph/pull/5296>`_, Jason Dillaman)
+* librbd: Crash during TestInternal.MultipleResize (`issue#12664 <http://tracker.ceph.com/issues/12664>`_, `pr#5769 <http://github.com/ceph/ceph/pull/5769>`_, Jason Dillaman)
+* librbd: deadlock during cooperative exclusive lock transition (`issue#11537 <http://tracker.ceph.com/issues/11537>`_, `pr#5319 <http://github.com/ceph/ceph/pull/5319>`_, Jason Dillaman)
+* librbd: Possible crash while concurrently writing and shrinking an image (`issue#11743 <http://tracker.ceph.com/issues/11743>`_, `pr#5318 <http://github.com/ceph/ceph/pull/5318>`_, Jason Dillaman)
+* mon: add a cache layer over MonitorDBStore (`issue#12638 <http://tracker.ceph.com/issues/12638>`_, `pr#5697 <http://github.com/ceph/ceph/pull/5697>`_, Kefu Chai)
+* mon: fix crush testing for new pools (`issue#13400 <http://tracker.ceph.com/issues/13400>`_, `pr#6192 <http://github.com/ceph/ceph/pull/6192>`_, Sage Weil)
+* mon: get pools health'info have error (`issue#12402 <http://tracker.ceph.com/issues/12402>`_, `pr#5369 <http://github.com/ceph/ceph/pull/5369>`_, renhwztetecs)
+* mon: implicit erasure code crush ruleset is not validated (`issue#11814 <http://tracker.ceph.com/issues/11814>`_, `pr#5276 <http://github.com/ceph/ceph/pull/5276>`_, Loic Dachary)
+* mon: PaxosService: call post_refresh() instead of post_paxos_update() (`issue#11470 <http://tracker.ceph.com/issues/11470>`_, `pr#5359 <http://github.com/ceph/ceph/pull/5359>`_, Joao Eduardo Luis)
+* mon: pgmonitor: wrong at/near target max“ reporting (`issue#12401 <http://tracker.ceph.com/issues/12401>`_, `pr#5370 <http://github.com/ceph/ceph/pull/5370>`_, huangjun)
+* mon: register_new_pgs() should check ruleno instead of its index (`issue#12210 <http://tracker.ceph.com/issues/12210>`_, `pr#5377 <http://github.com/ceph/ceph/pull/5377>`_, Xinze Chi)
+* mon: Show osd as NONE in ceph osd map <pool> <object>  output (`issue#11820 <http://tracker.ceph.com/issues/11820>`_, `pr#5376 <http://github.com/ceph/ceph/pull/5376>`_, Shylesh Kumar)
+* mon: the output is wrong when runing ceph osd reweight (`issue#12251 <http://tracker.ceph.com/issues/12251>`_, `pr#5372 <http://github.com/ceph/ceph/pull/5372>`_, Joao Eduardo Luis)
+* osd: allow peek_map_epoch to return an error (`issue#13060 <http://tracker.ceph.com/issues/13060>`_, `pr#5892 <http://github.com/ceph/ceph/pull/5892>`_, Sage Weil)
+* osd: cache agent is idle although one object is left in the cache (`issue#12673 <http://tracker.ceph.com/issues/12673>`_, `pr#5765 <http://github.com/ceph/ceph/pull/5765>`_, Loic Dachary)
+* osd: copy-from doesn't preserve truncate_{seq,size} (`issue#12551 <http://tracker.ceph.com/issues/12551>`_, `pr#5885 <http://github.com/ceph/ceph/pull/5885>`_, Samuel Just)
+* osd: crash creating/deleting pools (`issue#12429 <http://tracker.ceph.com/issues/12429>`_, `pr#5527 <http://github.com/ceph/ceph/pull/5527>`_, John Spray)
+* osd: fix repair when recorded digest is wrong (`issue#12577 <http://tracker.ceph.com/issues/12577>`_, `pr#5468 <http://github.com/ceph/ceph/pull/5468>`_, Sage Weil)
+* osd: include/ceph_features: define HAMMER_0_94_4 feature (`issue#13026 <http://tracker.ceph.com/issues/13026>`_, `pr#5687 <http://github.com/ceph/ceph/pull/5687>`_, Sage Weil)
+* osd: is_new_interval() fixes (`issue#10399 <http://tracker.ceph.com/issues/10399>`_, `pr#5691 <http://github.com/ceph/ceph/pull/5691>`_, Jason Dillaman)
+* osd: is_new_interval() fixes (`issue#11771 <http://tracker.ceph.com/issues/11771>`_, `pr#5691 <http://github.com/ceph/ceph/pull/5691>`_, Jason Dillaman)
+* osd: long standing slow requests: connection->session->waiting_for_map->connection ref cycle (`issue#12338 <http://tracker.ceph.com/issues/12338>`_, `pr#5761 <http://github.com/ceph/ceph/pull/5761>`_, Samuel Just)
+* osd: Mutex Assert from PipeConnection::try_get_pipe (`issue#12437 <http://tracker.ceph.com/issues/12437>`_, `pr#5758 <http://github.com/ceph/ceph/pull/5758>`_, David Zafman)
+* osd: pg_interval_t::check_new_interval - for ec pool, should not rely on min_size to determine if the PG was active at the interval (`issue#12162 <http://tracker.ceph.com/issues/12162>`_, `pr#5373 <http://github.com/ceph/ceph/pull/5373>`_, Guang G Yang)
+* osd: PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size()) (`issue#12652 <http://tracker.ceph.com/issues/12652>`_, `pr#5763 <http://github.com/ceph/ceph/pull/5763>`_, Sage Weil)
+* osd: PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out (`issue#11358 <http://tracker.ceph.com/issues/11358>`_, `pr#5380 <http://github.com/ceph/ceph/pull/5380>`_, Samuel Just)
+* osd: read on chunk-aligned xattr not handled (`issue#12309 <http://tracker.ceph.com/issues/12309>`_, `pr#5367 <http://github.com/ceph/ceph/pull/5367>`_, Sage Weil)
+* osd: suicide timeout during peering - search for missing objects (`issue#12523 <http://tracker.ceph.com/issues/12523>`_, `pr#5762 <http://github.com/ceph/ceph/pull/5762>`_, Guang G Yang)
+* osd: WBThrottle::clear_object: signal on cond when we reduce throttle values (`issue#12223 <http://tracker.ceph.com/issues/12223>`_, `pr#5757 <http://github.com/ceph/ceph/pull/5757>`_, Samuel Just)
+* rbd: crash during shutdown after writeback blocked by IO errors (`issue#12597 <http://tracker.ceph.com/issues/12597>`_, `pr#5767 <http://github.com/ceph/ceph/pull/5767>`_, Jianpeng Ma)
+* rgw: add delimiter to prefix only when path is specified (`issue#12960 <http://tracker.ceph.com/issues/12960>`_, `pr#5860 <http://github.com/ceph/ceph/pull/5860>`_, Sylvain Baubeau)
+* rgw: create a tool for orphaned objects cleanup (`issue#9604 <http://tracker.ceph.com/issues/9604>`_, `pr#5717 <http://github.com/ceph/ceph/pull/5717>`_, Yehuda Sadeh)
+* rgw: don't preserve acls when copying object (`issue#11563 <http://tracker.ceph.com/issues/11563>`_, `pr#6039 <http://github.com/ceph/ceph/pull/6039>`_, Yehuda Sadeh)
+* rgw: don't preserve acls when copying object (`issue#12370 <http://tracker.ceph.com/issues/12370>`_, `pr#6039 <http://github.com/ceph/ceph/pull/6039>`_, Yehuda Sadeh)
+* rgw: don't preserve acls when copying object (`issue#13015 <http://tracker.ceph.com/issues/13015>`_, `pr#6039 <http://github.com/ceph/ceph/pull/6039>`_, Yehuda Sadeh)
+* rgw: Ensure that swift keys don't include backslashes (`issue#7647 <http://tracker.ceph.com/issues/7647>`_, `pr#5716 <http://github.com/ceph/ceph/pull/5716>`_, Yehuda Sadeh)
+* rgw: GWWatcher::handle_error -> common/Mutex.cc: 95: FAILED assert(r == 0) (`issue#12208 <http://tracker.ceph.com/issues/12208>`_, `pr#6164 <http://github.com/ceph/ceph/pull/6164>`_, Yehuda Sadeh)
+* rgw: HTTP return code is not being logged by CivetWeb  (`issue#12432 <http://tracker.ceph.com/issues/12432>`_, `pr#5498 <http://github.com/ceph/ceph/pull/5498>`_, Yehuda Sadeh)
+* rgw: init_rados failed leads to repeated delete (`issue#12978 <http://tracker.ceph.com/issues/12978>`_, `pr#6165 <http://github.com/ceph/ceph/pull/6165>`_, Xiaowei Chen)
+* rgw: init some manifest fields when handling explicit objs (`issue#11455 <http://tracker.ceph.com/issues/11455>`_, `pr#5732 <http://github.com/ceph/ceph/pull/5732>`_, Yehuda Sadeh)
+* rgw: Keystone Fernet tokens break auth (`issue#12761 <http://tracker.ceph.com/issues/12761>`_, `pr#6162 <http://github.com/ceph/ceph/pull/6162>`_, Abhishek Lekshmanan)
+* rgw: region data still exist in region-map after region-map update (`issue#12964 <http://tracker.ceph.com/issues/12964>`_, `pr#6163 <http://github.com/ceph/ceph/pull/6163>`_, dwj192)
+* rgw: remove trailing :port from host for purposes of subdomain matching (`issue#12353 <http://tracker.ceph.com/issues/12353>`_, `pr#6042 <http://github.com/ceph/ceph/pull/6042>`_, Yehuda Sadeh)
+* rgw: rest-bench common/WorkQueue.cc: 54: FAILED assert(_threads.empty()) (`issue#3896 <http://tracker.ceph.com/issues/3896>`_, `pr#5383 <http://github.com/ceph/ceph/pull/5383>`_, huangjun)
+* rgw: returns requested bucket name raw in Bucket response header (`issue#12537 <http://tracker.ceph.com/issues/12537>`_, `pr#5715 <http://github.com/ceph/ceph/pull/5715>`_, Yehuda Sadeh)
+* rgw: segmentation fault when rgw_gc_max_objs > HASH_PRIME (`issue#12630 <http://tracker.ceph.com/issues/12630>`_, `pr#5719 <http://github.com/ceph/ceph/pull/5719>`_, Ruifeng Yang)
+* rgw: segments are read during HEAD on Swift DLO (`issue#12780 <http://tracker.ceph.com/issues/12780>`_, `pr#6160 <http://github.com/ceph/ceph/pull/6160>`_, Yehuda Sadeh)
+* rgw: setting max number of buckets for user via ceph.conf option  (`issue#12714 <http://tracker.ceph.com/issues/12714>`_, `pr#6166 <http://github.com/ceph/ceph/pull/6166>`_, Vikhyat Umrao)
+* rgw: Swift API: X-Trans-Id header is wrongly formatted (`issue#12108 <http://tracker.ceph.com/issues/12108>`_, `pr#5721 <http://github.com/ceph/ceph/pull/5721>`_, Radoslaw Zarzynski)
+* rgw: testGetContentType and testHead failed (`issue#11091 <http://tracker.ceph.com/issues/11091>`_, `pr#5718 <http://github.com/ceph/ceph/pull/5718>`_, Radoslaw Zarzynski)
+* rgw: testGetContentType and testHead failed (`issue#11438 <http://tracker.ceph.com/issues/11438>`_, `pr#5718 <http://github.com/ceph/ceph/pull/5718>`_, Radoslaw Zarzynski)
+* rgw: testGetContentType and testHead failed (`issue#12157 <http://tracker.ceph.com/issues/12157>`_, `pr#5718 <http://github.com/ceph/ceph/pull/5718>`_, Radoslaw Zarzynski)
+* rgw: testGetContentType and testHead failed (`issue#12158 <http://tracker.ceph.com/issues/12158>`_, `pr#5718 <http://github.com/ceph/ceph/pull/5718>`_, Radoslaw Zarzynski)
+* rgw: testGetContentType and testHead failed (`issue#12363 <http://tracker.ceph.com/issues/12363>`_, `pr#5718 <http://github.com/ceph/ceph/pull/5718>`_, Radoslaw Zarzynski)
+* rgw: the arguments 'domain' should not be assigned when return false (`issue#12629 <http://tracker.ceph.com/issues/12629>`_, `pr#5720 <http://github.com/ceph/ceph/pull/5720>`_, Ruifeng Yang)
+* tests: qa/workunits/cephtool/test.sh: don't assume crash_replay_interval=45 (`issue#13406 <http://tracker.ceph.com/issues/13406>`_, `pr#6172 <http://github.com/ceph/ceph/pull/6172>`_, Sage Weil)
+* tests: TEST_crush_rule_create_erasure consistently fails on i386 builder (`issue#12419 <http://tracker.ceph.com/issues/12419>`_, `pr#6201 <http://github.com/ceph/ceph/pull/6201>`_, Loic Dachary)
+* tools: ceph-disk zap should ensure block device (`issue#11272 <http://tracker.ceph.com/issues/11272>`_, `pr#5755 <http://github.com/ceph/ceph/pull/5755>`_, Loic Dachary)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.4.txt>`.
+
+
+v0.94.3 Hammer
+==============
+
+This Hammer point release fixes a critical (though rare) data
+corruption bug that could be triggered when logs are rotated via
+SIGHUP.  It also fixes a range of other important bugs in the OSD,
+monitor, RGW, RGW, and CephFS.
+
+All v0.94.x Hammer users are strongly encouraged to upgrade.
+
+Upgrading
+---------
+
+* The ``pg ls-by-{pool,primary,osd}`` commands and ``pg ls`` now take
+  the argument ``recovering`` instead of ``recovery`` in order to
+  include the recovering pgs in the listed pgs.
+
+Notable Changes
+---------------
+* librbd: aio calls may block (`issue#11770 <http://tracker.ceph.com/issues/11770>`_, `pr#4875 <http://github.com/ceph/ceph/pull/4875>`_, Jason Dillaman)
+* osd: make the all osd/filestore thread pool suicide timeouts separately configurable (`issue#11701 <http://tracker.ceph.com/issues/11701>`_, `pr#5159 <http://github.com/ceph/ceph/pull/5159>`_, Samuel Just)
+* mon: ceph fails to compile with boost 1.58 (`issue#11982 <http://tracker.ceph.com/issues/11982>`_, `pr#5122 <http://github.com/ceph/ceph/pull/5122>`_, Kefu Chai)
+* tests: TEST_crush_reject_empty must not run a mon (`issue#12285,11975 <http://tracker.ceph.com/issues/12285,11975>`_, `pr#5208 <http://github.com/ceph/ceph/pull/5208>`_, Kefu Chai)
+* osd: FAILED assert(!old_value.deleted()) in upgrade:giant-x-hammer-distro-basic-multi run (`issue#11983 <http://tracker.ceph.com/issues/11983>`_, `pr#5121 <http://github.com/ceph/ceph/pull/5121>`_, Samuel Just)
+* build/ops: linking ceph to tcmalloc causes segfault on SUSE SLE11-SP3 (`issue#12368 <http://tracker.ceph.com/issues/12368>`_, `pr#5265 <http://github.com/ceph/ceph/pull/5265>`_, Thorsten Behrens)
+* common: utf8 and old gcc breakage on RHEL6.5 (`issue#7387 <http://tracker.ceph.com/issues/7387>`_, `pr#4687 <http://github.com/ceph/ceph/pull/4687>`_, Kefu Chai)
+* crush: take crashes due to invalid arg (`issue#11740 <http://tracker.ceph.com/issues/11740>`_, `pr#4891 <http://github.com/ceph/ceph/pull/4891>`_, Sage Weil)
+* rgw: need conversion tool to handle fixes following #11974 (`issue#12502 <http://tracker.ceph.com/issues/12502>`_, `pr#5384 <http://github.com/ceph/ceph/pull/5384>`_, Yehuda Sadeh)
+* rgw: Swift API: support for 202 Accepted response code on container creation (`issue#12299 <http://tracker.ceph.com/issues/12299>`_, `pr#5214 <http://github.com/ceph/ceph/pull/5214>`_, Radoslaw Zarzynski)
+* common: Log::reopen_log_file: take m_flush_mutex (`issue#12520 <http://tracker.ceph.com/issues/12520>`_, `pr#5405 <http://github.com/ceph/ceph/pull/5405>`_, Samuel Just)
+* rgw: Properly respond to the Connection header with Civetweb (`issue#12398 <http://tracker.ceph.com/issues/12398>`_, `pr#5284 <http://github.com/ceph/ceph/pull/5284>`_, Wido den Hollander)
+* rgw: multipart list part response returns incorrect field (`issue#12399 <http://tracker.ceph.com/issues/12399>`_, `pr#5285 <http://github.com/ceph/ceph/pull/5285>`_, Henry Chang)
+* build/ops: ceph.spec.in: 95-ceph-osd.rules, mount.ceph, and mount.fuse.ceph not installed properly on SUSE (`issue#12397 <http://tracker.ceph.com/issues/12397>`_, `pr#5283 <http://github.com/ceph/ceph/pull/5283>`_, Nathan Cutler)
+* rgw: radosgw-admin dumps user info twice (`issue#12400 <http://tracker.ceph.com/issues/12400>`_, `pr#5286 <http://github.com/ceph/ceph/pull/5286>`_, guce)
+* doc: fix doc build (`issue#12180 <http://tracker.ceph.com/issues/12180>`_, `pr#5095 <http://github.com/ceph/ceph/pull/5095>`_, Kefu Chai)
+* tests: backport 11493 fixes, and test, preventing ec cache pools (`issue#12314 <http://tracker.ceph.com/issues/12314>`_, `pr#4961 <http://github.com/ceph/ceph/pull/4961>`_, Samuel Just)
+* rgw: does not send Date HTTP header when civetweb frontend is used (`issue#11872 <http://tracker.ceph.com/issues/11872>`_, `pr#5228 <http://github.com/ceph/ceph/pull/5228>`_, Radoslaw Zarzynski)
+* mon: pg ls is broken (`issue#11910 <http://tracker.ceph.com/issues/11910>`_, `pr#5160 <http://github.com/ceph/ceph/pull/5160>`_, Kefu Chai)
+* librbd: A client opening an image mid-resize can result in the object map being invalidated (`issue#12237 <http://tracker.ceph.com/issues/12237>`_, `pr#5279 <http://github.com/ceph/ceph/pull/5279>`_, Jason Dillaman)
+* doc: missing man pages for ceph-create-keys, ceph-disk-* (`issue#11862 <http://tracker.ceph.com/issues/11862>`_, `pr#4846 <http://github.com/ceph/ceph/pull/4846>`_, Nathan Cutler)
+* tools: ceph-post-file fails on rhel7 (`issue#11876 <http://tracker.ceph.com/issues/11876>`_, `pr#5038 <http://github.com/ceph/ceph/pull/5038>`_, Sage Weil)
+* build/ops: rcceph script is buggy (`issue#12090 <http://tracker.ceph.com/issues/12090>`_, `pr#5028 <http://github.com/ceph/ceph/pull/5028>`_, Owen Synge)
+* rgw: Bucket header is enclosed by quotes (`issue#11874 <http://tracker.ceph.com/issues/11874>`_, `pr#4862 <http://github.com/ceph/ceph/pull/4862>`_, Wido den Hollander)
+* build/ops: packaging: add SuSEfirewall2 service files (`issue#12092 <http://tracker.ceph.com/issues/12092>`_, `pr#5030 <http://github.com/ceph/ceph/pull/5030>`_, Tim Serong)
+* rgw: Keystone PKI token expiration is not enforced (`issue#11722 <http://tracker.ceph.com/issues/11722>`_, `pr#4884 <http://github.com/ceph/ceph/pull/4884>`_, Anton Aksola)
+* build/ops: debian/control: ceph-common (>> 0.94.2) must be >= 0.94.2-2 (`issue#12529,11998 <http://tracker.ceph.com/issues/12529,11998>`_, `pr#5417 <http://github.com/ceph/ceph/pull/5417>`_, Loic Dachary)
+* mon: Clock skew causes missing summary and confuses Calamari (`issue#11879 <http://tracker.ceph.com/issues/11879>`_, `pr#4868 <http://github.com/ceph/ceph/pull/4868>`_, Thorsten Behrens)
+* rgw: rados objects wronly deleted (`issue#12099 <http://tracker.ceph.com/issues/12099>`_, `pr#5117 <http://github.com/ceph/ceph/pull/5117>`_, wuxingyi)
+* tests: kernel_untar_build fails on EL7 (`issue#12098 <http://tracker.ceph.com/issues/12098>`_, `pr#5119 <http://github.com/ceph/ceph/pull/5119>`_, Greg Farnum)
+* fs: Fh ref count will leak if readahead does not need to do read from osd (`issue#12319 <http://tracker.ceph.com/issues/12319>`_, `pr#5427 <http://github.com/ceph/ceph/pull/5427>`_, Zhi Zhang)
+* mon: OSDMonitor: allow addition of cache pool with non-empty snaps with co… (`issue#12595 <http://tracker.ceph.com/issues/12595>`_, `pr#5252 <http://github.com/ceph/ceph/pull/5252>`_, Samuel Just)
+* mon: MDSMonitor: handle MDSBeacon messages properly (`issue#11979 <http://tracker.ceph.com/issues/11979>`_, `pr#5123 <http://github.com/ceph/ceph/pull/5123>`_, Kefu Chai)
+* tools: ceph-disk: get_partition_type fails on /dev/cciss... (`issue#11760 <http://tracker.ceph.com/issues/11760>`_, `pr#4892 <http://github.com/ceph/ceph/pull/4892>`_, islepnev)
+* build/ops: max files open limit for OSD daemon is too low (`issue#12087 <http://tracker.ceph.com/issues/12087>`_, `pr#5026 <http://github.com/ceph/ceph/pull/5026>`_, Owen Synge)
+* mon: add an "osd crush tree" command (`issue#11833 <http://tracker.ceph.com/issues/11833>`_, `pr#5248 <http://github.com/ceph/ceph/pull/5248>`_, Kefu Chai)
+* mon: mon crashes when "ceph osd tree 85 --format json" (`issue#11975 <http://tracker.ceph.com/issues/11975>`_, `pr#4936 <http://github.com/ceph/ceph/pull/4936>`_, Kefu Chai)
+* build/ops: ceph / ceph-dbg steal ceph-objecstore-tool from ceph-test / ceph-test-dbg (`issue#11806 <http://tracker.ceph.com/issues/11806>`_, `pr#5069 <http://github.com/ceph/ceph/pull/5069>`_, Loic Dachary)
+* rgw: DragonDisk fails to create directories via S3: MissingContentLength (`issue#12042 <http://tracker.ceph.com/issues/12042>`_, `pr#5118 <http://github.com/ceph/ceph/pull/5118>`_, Yehuda Sadeh)
+* build/ops: /usr/bin/ceph from ceph-common is broken without installing ceph (`issue#11998 <http://tracker.ceph.com/issues/11998>`_, `pr#5206 <http://github.com/ceph/ceph/pull/5206>`_, Ken Dreyer)
+* build/ops: systemd: Increase max files open limit for OSD daemon (`issue#11964 <http://tracker.ceph.com/issues/11964>`_, `pr#5040 <http://github.com/ceph/ceph/pull/5040>`_, Owen Synge)
+* build/ops: rgw/logrotate.conf calls service with wrong init script name (`issue#12044 <http://tracker.ceph.com/issues/12044>`_, `pr#5055 <http://github.com/ceph/ceph/pull/5055>`_, wuxingyi)
+* common: OPT_INT option interprets 3221225472 as -1073741824, and crashes in Throttle::Throttle() (`issue#11738 <http://tracker.ceph.com/issues/11738>`_, `pr#4889 <http://github.com/ceph/ceph/pull/4889>`_, Kefu Chai)
+* doc: doc/release-notes: v0.94.2 (`issue#11492 <http://tracker.ceph.com/issues/11492>`_, `pr#4934 <http://github.com/ceph/ceph/pull/4934>`_, Sage Weil)
+* common: admin_socket: close socket descriptor in destructor (`issue#11706 <http://tracker.ceph.com/issues/11706>`_, `pr#4657 <http://github.com/ceph/ceph/pull/4657>`_, Jon Bernard)
+* rgw: Object copy bug (`issue#11755 <http://tracker.ceph.com/issues/11755>`_, `pr#4885 <http://github.com/ceph/ceph/pull/4885>`_, Javier M. Mellid)
+* rgw: empty json response when getting user quota (`issue#12245 <http://tracker.ceph.com/issues/12245>`_, `pr#5237 <http://github.com/ceph/ceph/pull/5237>`_, wuxingyi)
+* fs: cephfs Dumper tries to load whole journal into memory at once (`issue#11999 <http://tracker.ceph.com/issues/11999>`_, `pr#5120 <http://github.com/ceph/ceph/pull/5120>`_, John Spray)
+* rgw: Fix tool for #11442 does not correctly fix objects created via multipart uploads (`issue#12242 <http://tracker.ceph.com/issues/12242>`_, `pr#5229 <http://github.com/ceph/ceph/pull/5229>`_, Yehuda Sadeh)
+* rgw: Civetweb RGW appears to report full size of object as downloaded when only partially downloaded (`issue#12243 <http://tracker.ceph.com/issues/12243>`_, `pr#5231 <http://github.com/ceph/ceph/pull/5231>`_, Yehuda Sadeh)
+* osd: stuck incomplete (`issue#12362 <http://tracker.ceph.com/issues/12362>`_, `pr#5269 <http://github.com/ceph/ceph/pull/5269>`_, Samuel Just)
+* osd: start_flush: filter out removed snaps before determining snapc's (`issue#11911 <http://tracker.ceph.com/issues/11911>`_, `pr#4899 <http://github.com/ceph/ceph/pull/4899>`_, Samuel Just)
+* librbd: internal.cc: 1967: FAILED assert(watchers.size() == 1) (`issue#12239 <http://tracker.ceph.com/issues/12239>`_, `pr#5243 <http://github.com/ceph/ceph/pull/5243>`_, Jason Dillaman)
+* librbd: new QA client upgrade tests (`issue#12109 <http://tracker.ceph.com/issues/12109>`_, `pr#5046 <http://github.com/ceph/ceph/pull/5046>`_, Jason Dillaman)
+* librbd: [  FAILED  ] TestLibRBD.ExclusiveLockTransition (`issue#12238 <http://tracker.ceph.com/issues/12238>`_, `pr#5241 <http://github.com/ceph/ceph/pull/5241>`_, Jason Dillaman)
+* rgw: Swift API: XML document generated in response for GET on account does not contain account name (`issue#12323 <http://tracker.ceph.com/issues/12323>`_, `pr#5227 <http://github.com/ceph/ceph/pull/5227>`_, Radoslaw Zarzynski)
+* rgw: keystone does not support chunked input (`issue#12322 <http://tracker.ceph.com/issues/12322>`_, `pr#5226 <http://github.com/ceph/ceph/pull/5226>`_, Hervé Rousseau)
+* mds: MDS is crashed (mds/CDir.cc: 1391: FAILED assert(!is_complete())) (`issue#11737 <http://tracker.ceph.com/issues/11737>`_, `pr#4886 <http://github.com/ceph/ceph/pull/4886>`_, Yan, Zheng)
+* cli: ceph: cli interactive mode does not understand quotes (`issue#11736 <http://tracker.ceph.com/issues/11736>`_, `pr#4776 <http://github.com/ceph/ceph/pull/4776>`_, Kefu Chai)
+* librbd: add valgrind memory checks for unit tests (`issue#12384 <http://tracker.ceph.com/issues/12384>`_, `pr#5280 <http://github.com/ceph/ceph/pull/5280>`_, Zhiqiang Wang)
+* build/ops: admin/build-doc: script fails silently under certain circumstances (`issue#11902 <http://tracker.ceph.com/issues/11902>`_, `pr#4877 <http://github.com/ceph/ceph/pull/4877>`_, John Spray)
+* osd: Fixes for rados ops with snaps (`issue#11908 <http://tracker.ceph.com/issues/11908>`_, `pr#4902 <http://github.com/ceph/ceph/pull/4902>`_, Samuel Just)
+* build/ops: ceph.spec.in: ceph-common subpackage def needs tweaking for SUSE/openSUSE (`issue#12308 <http://tracker.ceph.com/issues/12308>`_, `pr#4883 <http://github.com/ceph/ceph/pull/4883>`_, Nathan Cutler)
+* fs: client: reference counting 'struct Fh' (`issue#12088 <http://tracker.ceph.com/issues/12088>`_, `pr#5222 <http://github.com/ceph/ceph/pull/5222>`_, Yan, Zheng)
+* build/ops: ceph.spec: update OpenSUSE BuildRequires  (`issue#11611 <http://tracker.ceph.com/issues/11611>`_, `pr#4667 <http://github.com/ceph/ceph/pull/4667>`_, Loic Dachary)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.3.txt>`.
+
+
+
+v0.94.2 Hammer
+==============
+
+This Hammer point release fixes a few critical bugs in RGW that can
+prevent objects starting with underscore from behaving properly and
+that prevent garbage collection of deleted objects when using the
+Civetweb standalone mode.
+
+All v0.94.x Hammer users are strongly encouraged to upgrade, and to
+make note of the repair procedure below if RGW is in use.
+
+Upgrading from previous Hammer release
+--------------------------------------
+
+Bug #11442 introduced a change that made rgw objects that start with underscore
+incompatible with previous versions. The fix to that bug reverts to the
+previous behavior. In order to be able to access objects that start with an
+underscore and were created in prior Hammer releases, following the upgrade it
+is required to run (for each affected bucket)::
+
+    $ radosgw-admin bucket check --check-head-obj-locator \
+                                 --bucket=<bucket> [--fix]
+
+Notable changes
+---------------
+
+* build: compilation error: No high-precision counter available  (armhf, powerpc..) (#11432, James Page)
+* ceph-dencoder links to libtcmalloc, and shouldn't (#10691, Boris Ranto)
+* ceph-disk: disk zap sgdisk invocation (#11143, Owen Synge)
+* ceph-disk: use a new disk as journal disk,ceph-disk prepare fail (#10983, Loic Dachary)
+* ceph-objectstore-tool should be in the ceph server package (#11376, Ken Dreyer)
+* librados: can get stuck in redirect loop if osdmap epoch == last_force_op_resend (#11026, Jianpeng Ma)
+* librbd: A retransmit of proxied flatten request can result in -EINVAL (Jason Dillaman)
+* librbd: ImageWatcher should cancel in-flight ops on watch error (#11363, Jason Dillaman)
+* librbd: Objectcacher setting max object counts too low (#7385, Jason Dillaman)
+* librbd: Periodic failure of TestLibRBD.DiffIterateStress (#11369, Jason Dillaman)
+* librbd: Queued AIO reference counters not properly updated (#11478, Jason Dillaman)
+* librbd: deadlock in image refresh (#5488, Jason Dillaman)
+* librbd: notification race condition on snap_create (#11342, Jason Dillaman)
+* mds: Hammer uclient checking (#11510, John Spray)
+* mds: remove caps from revoking list when caps are voluntarily released (#11482, Yan, Zheng)
+* messenger: double clear of pipe in reaper (#11381, Haomai Wang)
+* mon: Total size of OSDs is a maginitude less than it is supposed to be. (#11534, Zhe Zhang)
+* osd: don't check order in finish_proxy_read (#11211, Zhiqiang Wang)
+* osd: handle old semi-deleted pgs after upgrade (#11429, Samuel Just)
+* osd: object creation by write cannot use an offset on an erasure coded pool (#11507, Jianpeng Ma)
+* rgw: Improve rgw HEAD request by avoiding read the body of the first chunk (#11001, Guang Yang)
+* rgw: civetweb is hitting a limit (number of threads 1024) (#10243, Yehuda Sadeh)
+* rgw: civetweb should use unique request id (#10295, Orit Wasserman)
+* rgw: critical fixes for hammer (#11447, #11442, Yehuda Sadeh)
+* rgw: fix swift COPY headers (#10662, #10663, #11087, #10645, Radoslaw Zarzynski)
+* rgw: improve performance for large object  (multiple chunks) GET (#11322, Guang Yang)
+* rgw: init-radosgw: run RGW as root (#11453, Ken Dreyer)
+* rgw: keystone token cache does not work correctly (#11125, Yehuda Sadeh)
+* rgw: make quota/gc thread configurable for starting (#11047, Guang Yang)
+* rgw: make swift responses of RGW return last-modified, content-length, x-trans-id headers.(#10650, Radoslaw Zarzynski)
+* rgw: merge manifests correctly when there's prefix override (#11622, Yehuda Sadeh)
+* rgw: quota not respected in POST object (#11323, Sergey Arkhipov)
+* rgw: restore buffer of multipart upload after EEXIST (#11604, Yehuda Sadeh)
+* rgw: shouldn't need to disable rgw_socket_path if frontend is configured (#11160, Yehuda Sadeh)
+* rgw: swift: Response header of GET request for container does not contain X-Container-Object-Count, X-Container-Bytes-Used and x-trans-id headers (#10666, Dmytro Iurchenko)
+* rgw: swift: Response header of POST request for object does not contain content-length and x-trans-id headers (#10661, Radoslaw Zarzynski)
+* rgw: swift: response for GET/HEAD on container does not contain the X-Timestamp header (#10938, Radoslaw Zarzynski)
+* rgw: swift: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used (#11036, #10971, Radoslaw Zarzynski)
+* rgw: swift: wrong handling of empty metadata on Swift container (#11088, Radoslaw Zarzynski)
+* tests: TestFlatIndex.cc races with TestLFNIndex.cc (#11217, Xinze Chi)
+* tests: ceph-helpers kill_daemons fails when kill fails (#11398, Loic Dachary)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.2.txt>`.
+
+
+v0.94.1 Hammer
+==============
+
+This bug fix release fixes a few critical issues with CRUSH.  The most
+important addresses a bug in feature bit enforcement that may prevent
+pre-hammer clients from communicating with the cluster during an
+upgrade.  This only manifests in some cases (for example, when the
+'rack' type is in use in the CRUSH map, and possibly other cases), but for
+safety we strongly recommend that all users use 0.94.1 instead of 0.94 when
+upgrading.
+
+There is also a fix in the new straw2 buckets when OSD weights are 0.
+
+We recommend that all v0.94 users upgrade.
+
+Notable changes
+---------------
+
+* crush: fix divide-by-0 in straw2 (#11357 Sage Weil)
+* crush: fix has_v4_buckets (#11364 Sage Weil)
+* osd: fix negative degraded objects during backfilling (#7737 Guang Yang)
+
+For more detailed information, see :download:`the complete changelog <../changelog/v0.94.1.txt>`.
+
+
+v0.94 Hammer
+============
+
+This major release is expected to form the basis of the next long-term
+stable series.  It is intended to supersede v0.80.x Firefly.
+
+Highlights since Giant include:
+
+* *RADOS Performance*: a range of improvements have been made in the
+  OSD and client-side librados code that improve the throughput on
+  flash backends and improve parallelism and scaling on fast machines.
+* *Simplified RGW deployment*: the ceph-deploy tool now has a new
+  'ceph-deploy rgw create HOST' command that quickly deploys a
+  instance of the S3/Swift gateway using the embedded Civetweb server.
+  This is vastly simpler than the previous Apache-based deployment.
+  There are a few rough edges (e.g., around SSL support) but we
+  encourage users to try `the new method`_.
+* *RGW object versioning*: RGW now supports the S3 object versioning
+  API, which preserves old version of objects instead of overwriting
+  them.
+* *RGW bucket sharding*: RGW can now shard the bucket index for large
+  buckets across, improving performance for very large buckets.
+* *RBD object maps*: RBD now has an object map function that tracks
+  which parts of the image are allocating, improving performance for
+  clones and for commands like export and delete.
+* *RBD mandatory locking*: RBD has a new mandatory locking framework
+  (still disabled by default) that adds additional safeguards to
+  prevent multiple clients from using the same image at the same time.
+* *RBD copy-on-read*: RBD now supports copy-on-read for image clones,
+  improving performance for some workloads.
+* *CephFS snapshot improvements*: Many many bugs have been fixed with
+  CephFS snapshots.  Although they are still disabled by default,
+  stability has improved significantly.
+* *CephFS Recovery tools*: We have built some journal recovery and
+  diagnostic tools. Stability and performance of single-MDS systems is
+  vastly improved in Giant, and more improvements have been made now
+  in Hammer.  Although we still recommend caution when storing
+  important data in CephFS, we do encourage testing for non-critical
+  workloads so that we can better gauge the feature, usability,
+  performance, and stability gaps.
+* *CRUSH improvements*: We have added a new straw2 bucket algorithm
+  that reduces the amount of data migration required when changes are
+  made to the cluster.
+* *Shingled erasure codes (SHEC)*: The OSDs now have experimental
+  support for shingled erasure codes, which allow a small amount of
+  additional storage to be traded for improved recovery performance.
+* *RADOS cache tiering*: A series of changes have been made in the
+  cache tiering code that improve performance and reduce latency.
+* *RDMA support*: There is now experimental support the RDMA via the
+  Accelio (libxio) library.
+* *New administrator commands*: The 'ceph osd df' command shows
+  pertinent details on OSD disk utilizations.  The 'ceph pg ls ...'
+  command makes it much simpler to query PG states while diagnosing
+  cluster issues.
+
+.. _the new method: ../start/quick-ceph-deploy/#add-an-rgw-instance
+
+Other highlights since Firefly include:
+
+* *CephFS*: we have fixed a raft of bugs in CephFS and built some
+  basic journal recovery and diagnostic tools.  Stability and
+  performance of single-MDS systems is vastly improved in Giant.
+  Although we do not yet recommend CephFS for production deployments,
+  we do encourage testing for non-critical workloads so that we can
+  better gauge the feature, usability, performance, and stability
+  gaps.
+* *Local Recovery Codes*: the OSDs now support an erasure-coding scheme
+  that stores some additional data blocks to reduce the IO required to
+  recover from single OSD failures.
+* *Degraded vs misplaced*: the Ceph health reports from 'ceph -s' and
+  related commands now make a distinction between data that is
+  degraded (there are fewer than the desired number of copies) and
+  data that is misplaced (stored in the wrong location in the
+  cluster).  The distinction is important because the latter does not
+  compromise data safety.
+* *Tiering improvements*: we have made several improvements to the
+  cache tiering implementation that improve performance.  Most
+  notably, objects are not promoted into the cache tier by a single
+  read; they must be found to be sufficiently hot before that happens.
+* *Monitor performance*: the monitors now perform writes to the local
+  data store asynchronously, improving overall responsiveness.
+* *Recovery tools*: the ceph-objectstore-tool is greatly expanded to
+  allow manipulation of an individual OSDs data store for debugging
+  and repair purposes.  This is most heavily used by our QA
+  infrastructure to exercise recovery code.
+
+I would like to take this opportunity to call out the amazing growth
+in contributors to Ceph beyond the core development team from Inktank.
+Hammer features major new features and improvements from Intel, Fujitsu,
+UnitedStack, Yahoo, UbuntuKylin, CohortFS, Mellanox, CERN, Deutsche
+Telekom, Mirantis, and SanDisk.
+
+Dedication
+----------
+
+This release is dedicated in memoriam to Sandon Van Ness, aka
+Houkouonchi, who unexpectedly passed away a few weeks ago.  Sandon was
+responsible for maintaining the large and complex Sepia lab that
+houses the Ceph project's build and test infrastructure.  His efforts
+have made an important impact on our ability to reliably test Ceph
+with a relatively small group of people.  He was a valued member of
+the team and we will miss him.  H is also for Houkouonchi.
+
+Upgrading
+---------
+
+* If your existing cluster is running a version older than v0.80.x
+  Firefly, please first upgrade to the latest Firefly release before
+  moving on to Giant.  We have not tested upgrades directly from
+  Emperor, Dumpling, or older releases.
+
+  We *have* tested:
+
+   * Firefly to Hammer
+   * Giant to Hammer
+   * Dumpling to Firefly to Hammer
+
+* Please upgrade daemons in the following order:
+
+   #. Monitors
+   #. OSDs
+   #. MDSs and/or radosgw
+
+  Note that the relative ordering of OSDs and monitors should not matter, but
+  we primarily tested upgrading monitors first.
+
+* The ceph-osd daemons will perform a disk-format upgrade improve the
+  PG metadata layout and to repair a minor bug in the on-disk format.
+  It may take a minute or two for this to complete, depending on how
+  many objects are stored on the node; do not be alarmed if they do
+  not marked "up" by the cluster immediately after starting.
+
+* If upgrading from v0.93, set
+   osd enable degraded writes = false
+
+  on all osds prior to upgrading.  The degraded writes feature has
+  been reverted due to 11155.
+
+* The LTTNG tracing in librbd and librados is disabled in the release packages
+  until we find a way to avoid violating distro security policies when linking
+  libust.
+
+Upgrading from v0.87.x Giant
+----------------------------
+
+* librbd and librados include lttng tracepoints on distros with
+  liblttng 2.4 or later (only Ubuntu Trusty for the ceph.com
+  packages). When running a daemon that uses these libraries, i.e. an
+  application that calls fork(2) or clone(2) without exec(3), you must
+  set LD_PRELOAD=liblttng-ust-fork.so.0 to prevent a crash in the
+  lttng atexit handler when the process exits. The only ceph tool that
+  requires this is rbd-fuse.
+
+* If rgw_socket_path is defined and rgw_frontends defines a
+  socket_port and socket_host, we now allow the rgw_frontends settings
+  to take precedence.  This change should only affect users who have
+  made non-standard changes to their radosgw configuration.
+
+* If you are upgrading specifically from v0.92, you must stop all OSD
+  daemons and flush their journals (``ceph-osd -i NNN
+  --flush-journal``) before upgrading.  There was a transaction
+  encoding bug in v0.92 that broke compatibility.  Upgrading from v0.93,
+  v0.91, or anything earlier is safe.
+
+* The experimental 'keyvaluestore-dev' OSD backend has been renamed
+  'keyvaluestore' (for simplicity) and marked as experimental.  To
+  enable this untested feature and acknowledge that you understand
+  that it is untested and may destroy data, you need to add the
+  following to your ceph.conf::
+
+    enable experimental unrecoverable data corrupting features = keyvaluestore
+
+* The following librados C API function calls take a 'flags' argument whose value
+  is now correctly interpreted:
+
+     rados_write_op_operate()
+     rados_aio_write_op_operate()
+     rados_read_op_operate()
+     rados_aio_read_op_operate()
+
+  The flags were not correctly being translated from the librados constants to the
+  internal values.  Now they are.  Any code that is passing flags to these methods
+  should be audited to ensure that they are using the correct LIBRADOS_OP_FLAG_*
+  constants.
+
+* The 'rados' CLI 'copy' and 'cppool' commands now use the copy-from operation,
+  which means the latest CLI cannot run these commands against pre-firefly OSDs.
+
+* The librados watch/notify API now includes a watch_flush() operation to flush
+  the async queue of notify operations.  This should be called by any watch/notify
+  user prior to rados_shutdown().
+
+* The 'category' field for objects has been removed.  This was originally added
+  to track PG stat summations over different categories of objects for use by
+  radosgw.  It is no longer has any known users and is prone to abuse because it
+  can lead to a pg_stat_t structure that is unbounded.  The librados API calls
+  that accept this field now ignore it, and the OSD no longer tracks the
+  per-category summations.
+
+* The output for 'rados df' has changed.  The 'category' level has been
+  eliminated, so there is now a single stat object per pool.  The structure of
+  the JSON output is different, and the plaintext output has one less column.
+
+* The 'rados create <objectname> [category]' optional category argument is no
+  longer supported or recognized.
+
+* rados.py's Rados class no longer has a __del__ method; it was causing
+  problems on interpreter shutdown and use of threads.  If your code has
+  Rados objects with limited lifetimes and you're concerned about locked
+  resources, call Rados.shutdown() explicitly.
+
+* There is a new version of the librados watch/notify API with vastly
+  improved semantics.  Any applications using this interface are
+  encouraged to migrate to the new API.  The old API calls are marked
+  as deprecated and will eventually be removed.
+
+* The librados rados_unwatch() call used to be safe to call on an
+  invalid handle.  The new version has undefined behavior when passed
+  a bogus value (for example, when rados_watch() returns an error and
+  handle is not defined).
+
+* The structure of the formatted 'pg stat' command is changed for the
+  portion that counts states by name to avoid using the '+' character
+  (which appears in state names) as part of the XML token (it is not
+  legal).
+
+* Previously, the formatted output of 'ceph pg stat -f ...' was a full
+  pg dump that included all metadata about all PGs in the system.  It
+  is now a concise summary of high-level PG stats, just like the
+  unformatted 'ceph pg stat' command.
+
+* All JSON dumps of floating point values were incorrecting surrounding the
+  value with quotes.  These quotes have been removed.  Any consumer of structured
+  JSON output that was consuming the floating point values was previously having
+  to interpret the quoted string and will most likely need to be fixed to take
+  the unquoted number.
+
+* New ability to list all objects from all namespaces that can fail or
+  return incomplete results when not all OSDs have been upgraded.
+  Features rados --all ls, rados cppool, rados export, rados
+  cache-flush-evict-all and rados cache-try-flush-evict-all can also
+  fail or return incomplete results.
+
+* Due to a change in the Linux kernel version 3.18 and the limits of the FUSE
+  interface, ceph-fuse needs be mounted as root on at least some systems. See
+  issues #9997, #10277, and #10542 for details.
+
+Upgrading from v0.80x Firefly (additional notes)
+------------------------------------------------
+
+* The client-side caching for librbd is now enabled by default (rbd
+  cache = true).  A safety option (rbd cache writethrough until flush
+  = true) is also enabled so that writeback caching is not used until
+  the library observes a 'flush' command, indicating that the librbd
+  users is passing that operation through from the guest VM.  This
+  avoids potential data loss when used with older versions of qemu
+  that do not support flush.
+
+    leveldb_write_buffer_size = 8*1024*1024  = 33554432   // 8MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+* The 'rados getxattr ...' command used to add a gratuitous newline to the attr
+  value; it now does not.
+
+* The ``*_kb perf`` counters on the monitor have been removed.  These are
+  replaced with a new set of ``*_bytes`` counters (e.g., ``cluster_osd_kb`` is
+  replaced by ``cluster_osd_bytes``).
+
+* The ``rd_kb`` and ``wr_kb`` fields in the JSON dumps for pool stats (accessed
+  via the ``ceph df detail -f json-pretty`` and related commands) have been
+  replaced with corresponding ``*_bytes`` fields.  Similarly, the
+  ``total_space``, ``total_used``, and ``total_avail`` fields are replaced with
+  ``total_bytes``, ``total_used_bytes``,  and ``total_avail_bytes`` fields.
+
+* The ``rados df --format=json`` output ``read_bytes`` and ``write_bytes``
+  fields were incorrectly reporting ops; this is now fixed.
+
+* The ``rados df --format=json`` output previously included ``read_kb`` and
+  ``write_kb`` fields; these have been removed.  Please use ``read_bytes`` and
+  ``write_bytes`` instead (and divide by 1024 if appropriate).
+
+* The experimental keyvaluestore-dev OSD backend had an on-disk format
+  change that prevents existing OSD data from being upgraded.  This
+  affects developers and testers only.
+
+* mon-specific and osd-specific leveldb options have been removed.
+  From this point onward users should use the `leveldb_*` generic
+  options and add the options in the appropriate sections of their
+  configuration files.  Monitors will still maintain the following
+  monitor-specific defaults:
+
+    leveldb_write_buffer_size = 8*1024*1024  = 33554432   // 8MB
+    leveldb_cache_size        = 512*1024*1204 = 536870912 // 512MB
+    leveldb_block_size        = 64*1024       = 65536     // 64KB
+    leveldb_compression       = false
+    leveldb_log               = ""
+
+  OSDs will still maintain the following osd-specific defaults:
+
+    leveldb_log               = ""
+
+* CephFS support for the legacy anchor table has finally been removed.
+  Users with file systems created before firefly should ensure that inodes
+  with multiple hard links are modified *prior* to the upgrade to ensure that
+  the backtraces are written properly.  For example::
+
+    sudo find /mnt/cephfs -type f -links +1 -exec touch \{\} \;
+
+* We disallow nonsensical 'tier cache-mode' transitions.  From this point
+  onward, 'writeback' can only transition to 'forward' and 'forward'
+  can transition to 1) 'writeback' if there are dirty objects, or 2) any if
+  there are no dirty objects.
+
+
+Notable changes since v0.93
+---------------------------
+
+* build: a few cmake fixes (Matt Benjamin)
+* build: fix build on RHEL/CentOS 5.9 (Rohan Mars)
+* build: reorganize Makefile to allow modular builds (Boris Ranto)
+* ceph-fuse: be more forgiving on remount (#10982 Greg Farnum)
+* ceph: improve CLI parsing (#11093 David Zafman)
+* common: fix cluster logging to default channel (#11177 Sage Weil)
+* crush: fix parsing of straw2 buckets (#11015 Sage Weil)
+* doc: update man pages (David Zafman)
+* librados: fix leak in C_TwoContexts (Xiong Yiliang)
+* librados: fix leak in watch/notify path (Sage Weil)
+* librbd: fix and improve AIO cache invalidation (#10958 Jason Dillaman)
+* librbd: fix memory leak (Jason Dillaman)
+* librbd: fix ordering/queueing of resize operations (Jason Dillaman)
+* librbd: validate image is r/w on resize/flatten (Jason Dillaman)
+* librbd: various internal locking fixes (Jason Dillaman)
+* lttng: tracing is disabled until we streamline dependencies (Josh Durgin)
+* mon: add bootstrap-rgw profile (Sage Weil)
+* mon: do not pollute mon dir with CSV files from CRUSH check (Loic Dachary)
+* mon: fix clock drift time check interval (#10546 Joao Eduardo Luis)
+* mon: fix units in store stats (Joao Eduardo Luis)
+* mon: improve error handling on erasure code profile set (#10488, #11144 Loic Dachary)
+* mon: set {read,write}_tier on 'osd tier add-cache ...' (Jianpeng Ma)
+* ms: xio: fix misc bugs (Matt Benjamin, Vu Pham)
+* osd: DBObjectMap: fix locking to prevent rare crash (#9891 Samuel Just)
+* osd: fix and document last_epoch_started semantics (Samuel Just)
+* osd: fix divergent entry handling on PG split (Samuel Just)
+* osd: fix leak on shutdown (Kefu Chai)
+* osd: fix recording of digest on scrub (Samuel Just)
+* osd: fix whiteout handling (Sage Weil)
+* rbd: allow v2 striping parameters for clones and imports (Jason Dillaman)
+* rbd: fix formatted output of image features (Jason Dillaman)
+* rbd: updat eman page (Ilya Dryomov)
+* rgw: don't overwrite bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: enable IPv6 for civetweb (#10965 Yehuda Sadeh)
+* rgw: fix sysvinit script when rgw_socket_path is not defined (#11159 Yehuda Sadeh, Dan Mick)
+* rgw: pass civetweb configurables through (#10907 Yehuda Sadeh)
+* rgw: use new watch/notify API (Yehuda Sadeh, Sage Weil)
+* osd: reverted degraded writes feature due to 11155
+
+Notable changes since v0.87.x Giant
+-----------------------------------
+
+* add experimental features option (Sage Weil)
+* arch: fix NEON feaeture detection (#10185 Loic Dachary)
+* asyncmsgr: misc fixes (Haomai Wang)
+* buffer: add 'shareable' construct (Matt Benjamin)
+* buffer: add list::get_contiguous (Sage Weil)
+* buffer: avoid rebuild if buffer already contiguous (Jianpeng Ma)
+* build: CMake support (Ali Maredia, Casey Bodley, Adam Emerson, Marcus Watts, Matt Benjamin)
+* build: a few cmake fixes (Matt Benjamin)
+* build: aarch64 build fixes (Noah Watkins, Haomai Wang)
+* build: adjust build deps for yasm, virtualenv (Jianpeng Ma)
+* build: fix 'make check' races (#10384 Loic Dachary)
+* build: fix build on RHEL/CentOS 5.9 (Rohan Mars)
+* build: fix pkg names when libkeyutils is missing (Pankag Garg, Ken Dreyer)
+* build: improve build dependency tooling (Loic Dachary)
+* build: reorganize Makefile to allow modular builds (Boris Ranto)
+* build: support for jemalloc (Shishir Gowda)
+* ceph-disk: Scientific Linux support (Dan van der Ster)
+* ceph-disk: allow journal partition re-use (#10146 Loic Dachary, Dav van der Ster)
+* ceph-disk: call partx/partprobe consistency (#9721 Loic Dachary)
+* ceph-disk: do not re-use partition if encryption is required (Loic Dachary)
+* ceph-disk: fix dmcrypt key permissions (Loic Dachary)
+* ceph-disk: fix umount race condition (#10096 Blaine Gardner)
+* ceph-disk: improved systemd support (Owen Synge)
+* ceph-disk: init=none option (Loic Dachary)
+* ceph-disk: misc fixes (Christos Stavrakakis)
+* ceph-disk: respect --statedir for keyring (Loic Dachary)
+* ceph-disk: set guid if reusing journal partition (Dan van der Ster)
+* ceph-disk: support LUKS for encrypted partitions (Andrew Bartlett, Loic Dachary)
+* ceph-fuse, libcephfs: POSIX file lock support (Yan, Zheng)
+* ceph-fuse, libcephfs: allow xattr caps in inject_release_failure (#9800 John Spray)
+* ceph-fuse, libcephfs: fix I_COMPLETE_ORDERED checks (#9894 Yan, Zheng)
+* ceph-fuse, libcephfs: fix cap flush overflow (Greg Farnum, Yan, Zheng)
+* ceph-fuse, libcephfs: fix root inode xattrs (Yan, Zheng)
+* ceph-fuse, libcephfs: preserve dir ordering (#9178 Yan, Zheng)
+* ceph-fuse, libcephfs: trim inodes before reconnecting to MDS (Yan, Zheng)
+* ceph-fuse,libcephfs: add support for O_NOFOLLOW and O_PATH (Greg Farnum)
+* ceph-fuse,libcephfs: resend requests before completing cap reconnect (#10912 Yan, Zheng)
+* ceph-fuse: be more forgiving on remount (#10982 Greg Farnum)
+* ceph-fuse: fix dentry invalidation on 3.18+ kernels (#9997 Yan, Zheng)
+* ceph-fuse: fix kernel cache trimming (#10277 Yan, Zheng)
+* ceph-fuse: select kernel cache invalidation mechanism based on kernel version (Greg Farnum)
+* ceph-monstore-tool: fix shutdown (#10093 Loic Dachary)
+* ceph-monstore-tool: fix/improve CLI (Joao Eduardo Luis)
+* ceph-objectstore-tool: fix import (#10090 David Zafman)
+* ceph-objectstore-tool: improved import (David Zafman)
+* ceph-objectstore-tool: many improvements and tests (David Zafman)
+* ceph-objectstore-tool: many many improvements (David Zafman)
+* ceph-objectstore-tool: misc improvements, fixes (#9870 #9871 David Zafman)
+* ceph.spec: package rbd-replay-prep (Ken Dreyer)
+* ceph: add 'ceph osd df [tree]' command (#10452 Mykola Golub)
+* ceph: do not parse injectargs twice (Loic Dachary)
+* ceph: fix 'ceph tell ...' command validation (#10439 Joao Eduardo Luis)
+* ceph: improve 'ceph osd tree' output (Mykola Golub)
+* ceph: improve CLI parsing (#11093 David Zafman)
+* ceph: make 'ceph -s' output more readable (Sage Weil)
+* ceph: make 'ceph -s' show PG state counts in sorted order (Sage Weil)
+* ceph: make 'ceph tell mon.* version' work (Mykola Golub)
+* ceph: new 'ceph tell mds.$name_or_rank_or_gid' (John Spray)
+* ceph: show primary-affinity in 'ceph osd tree' (Mykola Golub)
+* ceph: test robustness (Joao Eduardo Luis)
+* ceph_objectstore_tool: behave with sharded flag (#9661 David Zafman)
+* cephfs-journal-tool: add recover_dentries function (#9883 John Spray)
+* cephfs-journal-tool: fix journal import (#10025 John Spray)
+* cephfs-journal-tool: skip up to expire_pos (#9977 John Spray)
+* cleanup rados.h definitions with macros (Ilya Dryomov)
+* common: add 'perf reset ...' admin command (Jianpeng Ma)
+* common: add TableFormatter (Andreas Peters)
+* common: add newline to flushed json output (Sage Weil)
+* common: check syncfs() return code (Jianpeng Ma)
+* common: do not unlock rwlock on destruction (Federico Simoncelli)
+* common: filtering for 'perf dump' (John Spray)
+* common: fix Formatter factory breakage (#10547 Loic Dachary)
+* common: fix block device discard check (#10296 Sage Weil)
+* common: make json-pretty output prettier (Sage Weil)
+* common: remove broken CEPH_LOCKDEP optoin (Kefu Chai)
+* common: shared_cache unit tests (Cheng Cheng)
+* common: support new gperftools header locations (Key Dreyer)
+* config: add $cctid meta variable (Adam Crume)
+* crush: fix buffer overrun for poorly formed rules (#9492 Johnu George)
+* crush: fix detach_bucket (#10095 Sage Weil)
+* crush: fix parsing of straw2 buckets (#11015 Sage Weil)
+* crush: fix several bugs in adjust_item_weight (Rongze Zhu)
+* crush: fix tree bucket behavior (Rongze Zhu)
+* crush: improve constness (Loic Dachary)
+* crush: new and improved straw2 bucket type (Sage Weil, Christina Anderson, Xiaoxi Chen)
+* crush: straw bucket weight calculation fixes (#9998 Sage Weil)
+* crush: update tries stats for indep rules (#10349 Loic Dachary)
+* crush: use larger choose_tries value for erasure code rulesets (#10353 Loic Dachary)
+* crushtool: add --location <id> command (Sage Weil, Loic Dachary)
+* debian,rpm: move RBD udev rules to ceph-common (#10864 Ken Dreyer)
+* debian: split python-ceph into python-{rbd,rados,cephfs} (Boris Ranto)
+* default to libnss instead of crypto++ (Federico Gimenez)
+* doc: CephFS disaster recovery guidance (John Spray)
+* doc: CephFS for early adopters (John Spray)
+* doc: add build-doc guidlines for Fedora and CentOS/RHEL (Nilamdyuti Goswami)
+* doc: add dumpling to firefly upgrade section (#7679 John Wilkins)
+* doc: ceph osd reweight vs crush weight (Laurent Guerby)
+* doc: do not suggest dangerous XFS nobarrier option (Dan van der Ster)
+* doc: document erasure coded pool operations (#9970 Loic Dachary)
+* doc: document the LRC per-layer plugin configuration (Yuan Zhou)
+* doc: enable rbd cache on openstack deployments (Sebastien Han)
+* doc: erasure code doc updates (Loic Dachary)
+* doc: file system osd config settings (Kevin Dalley)
+* doc: fix OpenStack Glance docs (#10478 Sebastien Han)
+* doc: improved installation nots on CentOS/RHEL installs (John Wilkins)
+* doc: key/value store config reference (John Wilkins)
+* doc: misc cleanups (Adam Spiers, Sebastien Han, Nilamdyuti Goswami, Ken Dreyer, John Wilkins)
+* doc: misc improvements (Nilamdyuti Goswami, John Wilkins, Chris Holcombe)
+* doc: misc updates (#9793 #9922 #10204 #10203 Travis Rhoden, Hazem, Ayari, Florian Coste, Andy Allan, Frank Yu, Baptiste Veuillez-Mainard, Yuan Zhou, Armando Segnini, Robert Jansen, Tyler Brekke, Viktor Suprun)
+* doc: misc updates (Alfredo Deza, VRan Liu)
+* doc: misc updates (Nilamdyuti Goswami, John Wilkins)
+* doc: new man pages (Nilamdyuti Goswami)
+* doc: preflight doc fixes (John Wilkins)
+* doc: replace cloudfiles with swiftclient Python Swift example (Tim Freund)
+* doc: update PG count guide (Gerben Meijer, Laurent Guerby, Loic Dachary)
+* doc: update man pages (David Zafman)
+* doc: update openstack docs for Juno (Sebastien Han)
+* doc: update release descriptions (Ken Dreyer)
+* doc: update sepia hardware inventory (Sandon Van Ness)
+* erasure-code: add mSHEC erasure code support (Takeshi Miyamae)
+* erasure-code: improved docs (#10340 Loic Dachary)
+* erasure-code: set max_size to 20 (#10363 Loic Dachary)
+* fix cluster logging from non-mon daemons (Sage Weil)
+* init-ceph: check for systemd-run before using it (Boris Ranto)
+* install-deps.sh: do not require sudo when root (Loic Dachary)
+* keyvaluestore: misc fixes (Haomai Wang)
+* keyvaluestore: performance improvements (Haomai Wang)
+* libcephfs,ceph-fuse: add 'status' asok (John Spray)
+* libcephfs,ceph-fuse: fix getting zero-length xattr (#10552 Yan, Zheng)
+* libcephfs: fix dirfrag trimming (#10387 Yan, Zheng)
+* libcephfs: fix mount timeout (#10041 Yan, Zheng)
+* libcephfs: fix test (#10415 Yan, Zheng)
+* libcephfs: fix use-afer-free on umount (#10412 Yan, Zheng)
+* libcephfs: include ceph and git version in client metadata (Sage Weil)
+* librados, osd: new watch/notify implementation (Sage Weil)
+* librados: add blacklist_add convenience method (Jason Dillaman)
+* librados: add rados_pool_get_base_tier() call (Adam Crume)
+* librados: add watch_flush() operation (Sage Weil, Haomai Wang)
+* librados: avoid memcpy on getxattr, read (Jianpeng Ma)
+* librados: cap buffer length (Loic Dachary)
+* librados: create ioctx by pool id (Jason Dillaman)
+* librados: do notify completion in fast-dispatch (Sage Weil)
+* librados: drop 'category' feature (Sage Weil)
+* librados: expose rados_{read|write}_op_assert_version in C API (Kim Vandry)
+* librados: fix infinite loop with skipped map epochs (#9986 Ding Dinghua)
+* librados: fix iterator operator= bugs (#10082 David Zafman, Yehuda Sadeh)
+* librados: fix leak in C_TwoContexts (Xiong Yiliang)
+* librados: fix leak in watch/notify path (Sage Weil)
+* librados: fix null deref when pool DNE (#9944 Sage Weil)
+* librados: fix objecter races (#9617 Josh Durgin)
+* librados: fix pool deletion handling (#10372 Sage Weil)
+* librados: fix pool name caching (#10458 Radoslaw Zarzynski)
+* librados: fix resource leak, misc bugs (#10425 Radoslaw Zarzynski)
+* librados: fix some watch/notify locking (Jason Dillaman, Josh Durgin)
+* librados: fix timer race from recent refactor (Sage Weil)
+* librados: new fadvise API (Ma Jianpeng)
+* librados: only export public API symbols (Jason Dillaman)
+* librados: remove shadowed variable (Kefu Chain)
+* librados: translate op flags from C APIs (Matthew Richards)
+* libradosstriper: fix remove() (Dongmao Zhang)
+* libradosstriper: fix shutdown hang (Dongmao Zhang)
+* libradosstriper: fix stat strtoll (Dongmao Zhang)
+* libradosstriper: fix trunc method (#10129 Sebastien Ponce)
+* libradosstriper: fix write_full when ENOENT (#10758 Sebastien Ponce)
+* libradosstriper: misc fixes (Sebastien Ponce)
+* librbd: CRC protection for RBD image map (Jason Dillaman)
+* librbd: add missing python docstrings (Jason Dillaman)
+* librbd: add per-image object map for improved performance (Jason Dillaman)
+* librbd: add readahead (Adam Crume)
+* librbd: add support for an "object map" indicating which objects exist (Jason Dillaman)
+* librbd: adjust internal locking (Josh Durgin, Jason Dillaman)
+* librbd: better handling of watch errors (Jason Dillaman)
+* librbd: complete pending ops before closing image (#10299 Josh Durgin)
+* librbd: coordinate maint operations through lock owner (Jason Dillaman)
+* librbd: copy-on-read (Min Chen, Li Wang, Yunchuan Wen, Cheng Cheng, Jason Dillaman)
+* librbd: differentiate between R/O vs R/W features (Jason Dillaman)
+* librbd: don't close a closed parent in failure path (#10030 Jason Dillaman)
+* librbd: enforce write ordering with a snapshot (Jason Dillaman)
+* librbd: exclusive image locking (Jason Dillaman)
+* librbd: fadvise API (Ma Jianpeng)
+* librbd: fadvise-style hints; add misc hints for certain operations (Jianpeng Ma)
+* librbd: fix and improve AIO cache invalidation (#10958 Jason Dillaman)
+* librbd: fix cache tiers in list_children and snap_unprotect (Adam Crume)
+* librbd: fix coverity false-positives (Jason Dillaman)
+* librbd: fix diff test (#10002 Josh Durgin)
+* librbd: fix list_children from invalid pool ioctxs (#10123 Jason Dillaman)
+* librbd: fix locking for readahead (#10045 Jason Dillaman)
+* librbd: fix memory leak (Jason Dillaman)
+* librbd: fix ordering/queueing of resize operations (Jason Dillaman)
+* librbd: fix performance regression in ObjectCacher (#9513 Adam Crume)
+* librbd: fix snap create races (Jason Dillaman)
+* librbd: fix write vs import race (#10590 Jason Dillaman)
+* librbd: flush AIO operations asynchronously (#10714 Jason Dillaman)
+* librbd: gracefully handle deleted/renamed pools (#10270 Jason Dillaman)
+* librbd: lttng tracepoints (Adam Crume)
+* librbd: make async versions of long-running maint operations (Jason Dillaman)
+* librbd: misc fixes (Xinxin Shu, Jason Dillaman)
+* librbd: mock tests (Jason Dillaman)
+* librbd: only export public API symbols (Jason Dillaman)
+* librbd: optionally blacklist clients before breaking locks (#10761 Jason Dillaman)
+* librbd: prevent copyup during shrink (Jason Dillaman)
+* librbd: refactor unit tests to use fixtures (Jason Dillaman)
+* librbd: validate image is r/w on resize/flatten (Jason Dillaman)
+* librbd: various internal locking fixes (Jason Dillaman)
+* many coverity fixes (Danny Al-Gaaf)
+* many many coverity cleanups (Danny Al-Gaaf)
+* mds: 'flush journal' admin command (John Spray)
+* mds: ENOSPC and OSDMap epoch barriers (#7317 John Spray)
+* mds: a whole bunch of initial scrub infrastructure (Greg Farnum)
+* mds: add cephfs-table-tool (John Spray)
+* mds: asok command for fetching subtree map (John Spray)
+* mds: avoid sending traceless replies in most cases (Yan, Zheng)
+* mds: constify MDSCacheObjects (John Spray)
+* mds: dirfrag buf fix (Yan, Zheng)
+* mds: disallow most commands on inactive MDS's (Greg Farnum)
+* mds: drop dentries, leases on deleted directories (#10164 Yan, Zheng)
+* mds: export dir asok command (John Spray)
+* mds: fix MDLog IO callback deadlock (John Spray)
+* mds: fix compat_version for MClientSession (#9945 John Spray)
+* mds: fix deadlock during journal probe vs purge (#10229 Yan, Zheng)
+* mds: fix race trimming log segments (Yan, Zheng)
+* mds: fix reply snapbl (Yan, Zheng)
+* mds: fix sessionmap lifecycle bugs (Yan, Zheng)
+* mds: fix stray/purge perfcounters (#10388 John Spray)
+* mds: handle heartbeat_reset during shutdown (#10382 John Spray)
+* mds: handle zero-size xattr (#10335 Yan, Zheng)
+* mds: initialize root inode xattr version (Yan, Zheng)
+* mds: introduce auth caps (John Spray)
+* mds: many many snapshot-related fixes (Yan, Zheng)
+* mds: misc bugs (Greg Farnum, John Spray, Yan, Zheng, Henry Change)
+* mds: refactor, improve Session storage (John Spray)
+* mds: store backtrace for stray dir (Yan, Zheng)
+* mds: subtree quota support (Yunchuan Wen)
+* mds: verify backtrace when fetching dirfrag (#9557 Yan, Zheng)
+* memstore: free space tracking (John Spray)
+* misc cleanup (Danny Al-Gaaf, David Anderson)
+* misc coverity fixes (Danny Al-Gaaf)
+* misc coverity fixes (Danny Al-Gaaf)
+* misc: various valgrind fixes and cleanups (Danny Al-Gaaf)
+* mon: 'osd crush reweight-all' command (Sage Weil)
+* mon: add 'ceph osd rename-bucket ...' command (Loic Dachary)
+* mon: add bootstrap-rgw profile (Sage Weil)
+* mon: add max pgs per osd warning (Sage Weil)
+* mon: add noforward flag for some mon commands (Mykola Golub)
+* mon: allow adding tiers to fs pools (#10135 John Spray)
+* mon: allow full flag to be manually cleared (#9323 Sage Weil)
+* mon: clean up auth list output (Loic Dachary)
+* mon: delay failure injection (Joao Eduardo Luis)
+* mon: disallow empty pool names (#10555 Wido den Hollander)
+* mon: do not deactivate last mds (#10862 John Spray)
+* mon: do not pollute mon dir with CSV files from CRUSH check (Loic Dachary)
+* mon: drop old ceph_mon_store_converter (Sage Weil)
+* mon: fix 'ceph pg dump_stuck degraded' (Xinxin Shu)
+* mon: fix 'mds fail' for standby MDSs (John Spray)
+* mon: fix 'osd crush link' id resolution (John Spray)
+* mon: fix 'profile osd' use of config-key function on mon (#10844 Joao Eduardo Luis)
+* mon: fix *_ratio* units and types (Sage Weil)
+* mon: fix JSON dumps to dump floats as flots and not strings (Sage Weil)
+* mon: fix MDS health status from peons (#10151 John Spray)
+* mon: fix caching for min_last_epoch_clean (#9987 Sage Weil)
+* mon: fix clock drift time check interval (#10546 Joao Eduardo Luis)
+* mon: fix compatset initalization during mkfs (Joao Eduardo Luis)
+* mon: fix error output for add_data_pool (#9852 Joao Eduardo Luis)
+* mon: fix feature tracking during elections (Joao Eduardo Luis)
+* mon: fix formatter 'pg stat' command output (Sage Weil)
+* mon: fix mds gid/rank/state parsing (John Spray)
+* mon: fix misc error paths (Joao Eduardo Luis)
+* mon: fix paxos off-by-one corner case (#9301 Sage Weil)
+* mon: fix paxos timeouts (#10220 Joao Eduardo Luis)
+* mon: fix stashed monmap encoding (#5203 Xie Rui)
+* mon: fix units in store stats (Joao Eduardo Luis)
+* mon: get canonical OSDMap from leader (#10422 Sage Weil)
+* mon: ignore failure reports from before up_from (#10762 Dan van der Ster, Sage Weil)
+* mon: implement 'fs reset' command (John Spray)
+* mon: improve error handling on erasure code profile set (#10488, #11144 Loic Dachary)
+* mon: improved corrupt CRUSH map detection (Joao Eduardo Luis)
+* mon: include entity name in audit log for forwarded requests (#9913 Joao Eduardo Luis)
+* mon: include pg_temp count in osdmap summary (Sage Weil)
+* mon: log health summary to cluster log (#9440 Joao Eduardo Luis)
+* mon: make 'mds fail' idempotent (John Spray)
+* mon: make pg dump {sum,pgs,pgs_brief} work for format=plain (#5963 #6759 Mykola Golub)
+* mon: new 'ceph pool ls [detail]' command (Sage Weil)
+* mon: new pool safety flags nodelete, nopgchange, nosizechange (#9792 Mykola Golub)
+* mon: new, friendly 'ceph pg ls ...' command (Xinxin Shu)
+* mon: paxos: allow reads while proposing (#9321 #9322 Joao Eduardo Luis)
+* mon: prevent MDS transition from STOPPING (#10791 Greg Farnum)
+* mon: propose all pending work in one transaction (Sage Weil)
+* mon: remove pg_temps for nonexistent pools (Joao Eduardo Luis)
+* mon: require mon_allow_pool_delete option to remove pools (Sage Weil)
+* mon: respect down flag when promoting standbys (John Spray)
+* mon: set globalid prealloc to larger value (Sage Weil)
+* mon: set {read,write}_tier on 'osd tier add-cache ...' (Jianpeng Ma)
+* mon: skip zeroed osd stats in get_rule_avail (#10257 Joao Eduardo Luis)
+* mon: validate min_size range (Jianpeng Ma)
+* mon: wait for writeable before cross-proposing (#9794 Joao Eduardo Luis)
+* mount.ceph: fix suprious error message (#10351 Yan, Zheng)
+* ms: xio: fix misc bugs (Matt Benjamin, Vu Pham)
+* msgr: async: bind threads to CPU cores, improved poll (Haomai Wang)
+* msgr: async: many fixes, unit tests (Haomai Wang)
+* msgr: async: several fixes (Haomai Wang)
+* msgr: asyncmessenger: add kqueue support (#9926 Haomai Wang)
+* msgr: avoid useless new/delete (Haomai Wang)
+* msgr: fix RESETSESSION bug (#10080 Greg Farnum)
+* msgr: fix crc configuration (Mykola Golub)
+* msgr: fix delay injection bug (#9910 Sage Weil, Greg Farnum)
+* msgr: misc unit tests (Haomai Wang)
+* msgr: new AsymcMessenger alternative implementation (Haomai Wang)
+* msgr: prefetch data when doing recv (Yehuda Sadeh)
+* msgr: simple: fix rare deadlock (Greg Farnum)
+* msgr: simple: retry binding to port on failure (#10029 Wido den Hollander)
+* msgr: xio: XioMessenger RDMA support (Casey Bodley, Vu Pham, Matt Benjamin)
+* objectstore: deprecate collection attrs (Sage Weil)
+* osd, librados: fadvise-style librados hints (Jianpeng Ma)
+* osd, librados: fix xattr_cmp_u64 (Dongmao Zhang)
+* osd, librados: revamp PG listing API to handle namespaces (#9031 #9262 #9438 David Zafman)
+* osd, mds: 'ops' as shorthand for 'dump_ops_in_flight' on asok (Sage Weil)
+* osd, mon: add checksums to all OSDMaps (Sage Weil)
+* osd, mon: send intiial pg create time from mon to osd (#9887 David Zafman)
+* osd,mon: add 'norebalance' flag (Kefu Chai)
+* osd,mon: specify OSD features explicitly in MOSDBoot (#10911 Sage Weil)
+* osd: DBObjectMap: fix locking to prevent rare crash (#9891 Samuel Just)
+* osd: EIO on whole-object reads when checksum is wrong (Sage Weil)
+* osd: add erasure code corpus (Loic Dachary)
+* osd: add fadvise flags to ObjectStore API (Jianpeng Ma)
+* osd: add get_latest_osdmap asok command (#9483 #9484 Mykola Golub)
+* osd: add misc tests (Loic Dachary, Danny Al-Gaaf)
+* osd: add option to prioritize heartbeat network traffic (Jian Wen)
+* osd: add support for the SHEC erasure-code algorithm (Takeshi Miyamae, Loic Dachary)
+* osd: allow deletion of objects with watcher (#2339 Sage Weil)
+* osd: allow recovery while below min_size (Samuel Just)
+* osd: allow recovery with fewer than min_size OSDs (Samuel Just)
+* osd: allow sparse read for Push/Pull (Haomai Wang)
+* osd: allow whiteout deletion in cache pool (Sage Weil)
+* osd: allow writes to degraded objects (Samuel Just)
+* osd: allow writes to degraded objects (Samuel Just)
+* osd: avoid publishing unchanged PG stats (Sage Weil)
+* osd: batch pg log trim (Xinze Chi)
+* osd: cache pool: ignore min flush age when cache is full (Xinze Chi)
+* osd: cache recent ObjectContexts (Dong Yuan)
+* osd: cache reverse_nibbles hash value (Dong Yuan)
+* osd: clean up internal ObjectStore interface (Sage Weil)
+* osd: cleanup boost optionals (William Kennington)
+* osd: clear cache on interval change (Samuel Just)
+* osd: do no proxy reads unless target OSDs are new (#10788 Sage Weil)
+* osd: do not abort deep scrub on missing hinfo (#10018 Loic Dachary)
+* osd: do not update digest on inconsistent object (#10524 Samuel Just)
+* osd: don't record digests for snapdirs (#10536 Samuel Just)
+* osd: drop upgrade support for pre-dumpling (Sage Weil)
+* osd: enable and use posix_fadvise (Sage Weil)
+* osd: erasure coding: allow bench.sh to test ISA backend (Yuan Zhou)
+* osd: erasure-code: encoding regression tests, corpus (#9420 Loic Dachary)
+* osd: erasure-code: enforce chunk size alignment (#10211 Loic Dachary)
+* osd: erasure-code: jerasure support for NEON (Loic Dachary)
+* osd: erasure-code: relax cauchy w restrictions (#10325 David Zhang, Loic Dachary)
+* osd: erasure-code: update gf-complete to latest upstream (Loic Dachary)
+* osd: expose non-journal backends via ceph-osd CLI (Hoamai Wang)
+* osd: filejournal: don't cache journal when not using direct IO (Jianpeng Ma)
+* osd: fix JSON output for stray OSDs (Loic Dachary)
+* osd: fix OSDCap parser on old (el6) boost::spirit (#10757 Kefu Chai)
+* osd: fix OSDCap parsing on el6 (#10757 Kefu Chai)
+* osd: fix ObjectStore::Transaction encoding version (#10734 Samuel Just)
+* osd: fix WBTHrottle perf counters (Haomai Wang)
+* osd: fix and document last_epoch_started semantics (Samuel Just)
+* osd: fix auth object selection during repair (#10524 Samuel Just)
+* osd: fix backfill bug (#10150 Samuel Just)
+* osd: fix bug in pending digest updates (#10840 Samuel Just)
+* osd: fix cancel_proxy_read_ops (Sage Weil)
+* osd: fix cleanup of interrupted pg deletion (#10617 Sage Weil)
+* osd: fix divergent entry handling on PG split (Samuel Just)
+* osd: fix ghobject_t formatted output to include shard (#10063 Loic Dachary)
+* osd: fix ioprio option (Mykola Golub)
+* osd: fix ioprio options (Loic Dachary)
+* osd: fix journal shutdown race (Sage Weil)
+* osd: fix journal wrapping bug (#10883 David Zafman)
+* osd: fix leak in SnapTrimWQ (#10421 Kefu Chai)
+* osd: fix leak on shutdown (Kefu Chai)
+* osd: fix memstore free space calculation (Xiaoxi Chen)
+* osd: fix mixed-version peering issues (Samuel Just)
+* osd: fix object age eviction (Zhiqiang Wang)
+* osd: fix object atime calculation (Xinze Chi)
+* osd: fix object digest update bug (#10840 Samuel Just)
+* osd: fix occasional peering stalls (#10431 Sage Weil)
+* osd: fix ordering issue with new transaction encoding (#10534 Dong Yuan)
+* osd: fix osd peer check on scrub messages (#9555 Sage Weil)
+* osd: fix past_interval display bug (#9752 Loic Dachary)
+* osd: fix past_interval generation (#10427 #10430 David Zafman)
+* osd: fix pgls filter ops (#9439 David Zafman)
+* osd: fix recording of digest on scrub (Samuel Just)
+* osd: fix scrub delay bug (#10693 Samuel Just)
+* osd: fix scrub vs try-flush bug (#8011 Samuel Just)
+* osd: fix short read handling on push (#8121 David Zafman)
+* osd: fix stderr with -f or -d (Dan Mick)
+* osd: fix transaction accounting (Jianpeng Ma)
+* osd: fix watch reconnect race (#10441 Sage Weil)
+* osd: fix watch timeout cache state update (#10784 David Zafman)
+* osd: fix whiteout handling (Sage Weil)
+* osd: flush snapshots from cache tier immediately (Sage Weil)
+* osd: force promotion of watch/notify ops (Zhiqiang Wang)
+* osd: handle no-op write with snapshot (#10262 Sage Weil)
+* osd: improve idempotency detection across cache promotion/demotion (#8935 Sage Weil, Samuel Just)
+* osd: include activating peers in blocked_by (#10477 Sage Weil)
+* osd: jerasure and gf-complete updates from upstream (#10216 Loic Dachary)
+* osd: journal: check fsync/fdatasync result (Jianpeng Ma)
+* osd: journal: fix alignment checks, avoid useless memmove (Jianpeng Ma)
+* osd: journal: fix hang on shutdown (#10474 David Zafman)
+* osd: journal: fix header.committed_up_to (Xinze Chi)
+* osd: journal: fix journal zeroing when direct IO is enabled (Xie Rui)
+* osd: journal: initialize throttle (Ning Yao)
+* osd: journal: misc bug fixes (#6003 David Zafman, Samuel Just)
+* osd: journal: update committed_thru after replay (#6756 Samuel Just)
+* osd: keyvaluestore: cleanup dead code (Ning Yao)
+* osd: keyvaluestore: fix getattr semantics (Haomai Wang)
+* osd: keyvaluestore: fix key ordering (#10119 Haomai Wang)
+* osd: keyvaluestore_dev: optimization (Chendi Xue)
+* osd: limit in-flight read requests (Jason Dillaman)
+* osd: log when scrub or repair starts (Loic Dachary)
+* osd: make misdirected op checks robust for EC pools (#9835 Sage Weil)
+* osd: memstore: fix size limit (Xiaoxi Chen)
+* osd: misc FIEMAP fixes (Ma Jianpeng)
+* osd: misc cleanup (Xinze Chi, Yongyue Sun)
+* osd: misc optimizations (Xinxin Shu, Zhiqiang Wang, Xinze Chi)
+* osd: misc scrub fixes (#10017 Loic Dachary)
+* osd: new 'activating' state between peering and active (Sage Weil)
+* osd: new optimized encoding for ObjectStore::Transaction (Dong Yuan)
+* osd: optimize Finisher (Xinze Chi)
+* osd: optimize WBThrottle map with unordered_map (Ning Yao)
+* osd: optimize filter_snapc (Ning Yao)
+* osd: preserve reqids for idempotency checks for promote/demote (Sage Weil, Zhiqiang Wang, Samuel Just)
+* osd: proxy read support (Zhiqiang Wang)
+* osd: proxy reads during cache promote (Zhiqiang Wang)
+* osd: remove dead locking code (Xinxin Shu)
+* osd: remove legacy classic scrub code (Sage Weil)
+* osd: remove unused fields in MOSDSubOp (Xiaoxi Chen)
+* osd: removed some dead code (Xinze Chi)
+* osd: replace MOSDSubOp messages with simpler, optimized MOSDRepOp (Xiaoxi Chen)
+* osd: restrict scrub to certain times of day (Xinze Chi)
+* osd: rocksdb: fix shutdown (Hoamai Wang)
+* osd: store PG metadata in per-collection objects for better concurrency (Sage Weil)
+* osd: store whole-object checksums on scrub, write_full (Sage Weil)
+* osd: support for discard for journal trim (Jianpeng Ma)
+* osd: use FIEMAP_FLAGS_SYNC instead of fsync (Jianpeng Ma)
+* osd: verify kernel is new enough before using XFS extsize ioctl, enable by default (#9956 Sage Weil)
+* pybind: fix memory leak in librados bindings (Billy Olsen)
+* pyrados: add object lock support (#6114 Mehdi Abaakouk)
+* pyrados: fix misnamed wait_* routings (#10104 Dan Mick)
+* pyrados: misc cleanups (Kefu Chai)
+* qa: add large auth ticket tests (Ilya Dryomov)
+* qa: fix mds tests (#10539 John Spray)
+* qa: fix osd create dup tests (#10083 Loic Dachary)
+* qa: ignore duplicates in rados ls (Josh Durgin)
+* qa: improve hadoop tests (Noah Watkins)
+* qa: many 'make check' improvements (Loic Dachary)
+* qa: misc tests (Loic Dachary, Yan, Zheng)
+* qa: parallelize make check (Loic Dachary)
+* qa: reorg fs quota tests (Greg Farnum)
+* qa: tolerate nearly-full disk for make check (Loic Dachary)
+* rados: fix put of /dev/null (Loic Dachary)
+* rados: fix usage (Jianpeng Ma)
+* rados: parse command-line arguments more strictly (#8983 Adam Crume)
+* rados: use copy-from operation for copy, cppool (Sage Weil)
+* radosgw-admin: add replicalog update command (Yehuda Sadeh)
+* rbd-fuse: clean up on shutdown (Josh Durgin)
+* rbd-fuse: fix memory leak (Adam Crume)
+* rbd-replay-many (Adam Crume)
+* rbd-replay: --anonymize flag to rbd-replay-prep (Adam Crume)
+* rbd: add 'merge-diff' function (MingXin Liu, Yunchuan Wen, Li Wang)
+* rbd: allow v2 striping parameters for clones and imports (Jason Dillaman)
+* rbd: fix 'rbd diff' for non-existent objects (Adam Crume)
+* rbd: fix buffer handling on image import (#10590 Jason Dillaman)
+* rbd: fix error when striping with format 1 (Sebastien Han)
+* rbd: fix export for image sizes over 2GB (Vicente Cheng)
+* rbd: fix formatted output of image features (Jason Dillaman)
+* rbd: leave exclusive lockin goff by default (Jason Dillaman)
+* rbd: updat eman page (Ilya Dryomov)
+* rbd: update init-rbdmap to fix dup mount point (Karel Striegel)
+* rbd: use IO hints for import, export, and bench operations (#10462 Jason Dillaman)
+* rbd: use rolling average for rbd bench-write throughput (Jason Dillaman)
+* rbd_recover_tool: RBD image recovery tool (Min Chen)
+* rgw: S3-style object versioning support (Yehuda Sadeh)
+* rgw: add location header when object is in another region (VRan Liu)
+* rgw: change multipart upload id magic (#10271 Yehuda Sadeh)
+* rgw: check keystone auth for S3 POST requests (#10062 Abhishek Lekshmanan)
+* rgw: check timestamp on s3 keystone auth (#10062 Abhishek Lekshmanan)
+* rgw: conditional PUT on ETag (#8562 Ray Lv)
+* rgw: create subuser if needed when creating user (#10103 Yehuda Sadeh)
+* rgw: decode http query params correction (#10271 Yehuda Sadeh)
+* rgw: don't overwrite bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: enable IPv6 for civetweb (#10965 Yehuda Sadeh)
+* rgw: extend replica log API (purge-all) (Yehuda Sadeh)
+* rgw: fail S3 POST if keystone not configured (#10688 Valery Tschopp, Yehuda Sadeh)
+* rgw: fix If-Modified-Since (VRan Liu)
+* rgw: fix XML header on get ACL request (#10106 Yehuda Sadeh)
+* rgw: fix bucket removal with data purge (Yehuda Sadeh)
+* rgw: fix content length check (#10701 Axel Dunkel, Yehuda Sadeh)
+* rgw: fix content-length update (#9576 Yehuda Sadeh)
+* rgw: fix disabling of max_size quota (#9907 Dong Lei)
+* rgw: fix error codes (#10334 #10329 Yehuda Sadeh)
+* rgw: fix incorrect len when len is 0 (#9877 Yehuda Sadeh)
+* rgw: fix object copy content type (#9478 Yehuda Sadeh)
+* rgw: fix partial GET in swift (#10553 Yehuda Sadeh)
+* rgw: fix replica log indexing (#8251 Yehuda Sadeh)
+* rgw: fix shutdown (#10472 Yehuda Sadeh)
+* rgw: fix swift metadata header name (Dmytro Iurchenko)
+* rgw: fix sysvinit script when rgw_socket_path is not defined (#11159 Yehuda Sadeh, Dan Mick)
+* rgw: fix user stags in get-user-info API (#9359 Ray Lv)
+* rgw: include XML ns on get ACL request (#10106 Yehuda Sadeh)
+* rgw: index swift keys appropriately (#10471 Yehuda Sadeh)
+* rgw: make sysvinit script set ulimit -n properly (Sage Weil)
+* rgw: misc fixes (#10307 Yehuda Sadeh)
+* rgw: only track cleanup for objects we write (#10311 Yehuda Sadeh)
+* rgw: pass civetweb configurables through (#10907 Yehuda Sadeh)
+* rgw: prevent illegal bucket policy that doesn't match placement rule (Yehuda Sadeh)
+* rgw: remove multipart entries from bucket index on abort (#10719 Yehuda Sadeh)
+* rgw: remove swift user manifest (DLO) hash calculation (#9973 Yehuda Sadeh)
+* rgw: respond with 204 to POST on containers (#10667 Yuan Zhou)
+* rgw: return timestamp on GET/HEAD (#8911 Yehuda Sadeh)
+* rgw: reuse fcgx connection struct (#10194 Yehuda Sadeh)
+* rgw: run radosgw as apache with systemd (#10125 Loic Dachary)
+* rgw: send explicit HTTP status string (Yehuda Sadeh)
+* rgw: set ETag on object copy (#9479 Yehuda Sadeh)
+* rgw: set length for keystone token validation request (#7796 Yehuda Sadeh, Mark Kirkwood)
+* rgw: support X-Storage-Policy header for Swift storage policy compat (Yehuda Sadeh)
+* rgw: support multiple host names (#7467 Yehuda Sadeh)
+* rgw: swift: dump container's custom metadata (#10665 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: swift: support Accept header for response format (#10746 Dmytro Iurchenko)
+* rgw: swift: support for X-Remove-Container-Meta-{key} (#10475 Dmytro Iurchenko)
+* rgw: tweak error codes (#10329 #10334 Yehuda Sadeh)
+* rgw: update bucket index on attr changes, for multi-site sync (#5595 Yehuda Sadeh)
+* rgw: use \r\n for http headers (#9254 Yehuda Sadeh)
+* rgw: use gc for multipart abort (#10445 Aaron Bassett, Yehuda Sadeh)
+* rgw: use new watch/notify API (Yehuda Sadeh, Sage Weil)
+* rpm: misc fixes (Key Dreyer)
+* rpm: move rgw logrotate to radosgw subpackage (Ken Dreyer)
+* systemd: better systemd unit files (Owen Synge)
+* sysvinit: fix race in 'stop' (#10389 Loic Dachary)
+* test: fix bufferlist tests (Jianpeng Ma)
+* tests: ability to run unit tests under docker (Loic Dachary)
+* tests: centos-6 dockerfile (#10755 Loic Dachary)
+* tests: improve docker-based tests (Loic Dachary)
+* tests: unit tests for shared_cache (Dong Yuan)
+* udev: fix rules for CentOS7/RHEL7 (Loic Dachary)
+* use clock_gettime instead of gettimeofday (Jianpeng Ma)
+* vstart.sh: set up environment for s3-tests (Luis Pabon)
+* vstart.sh: work with cmake (Yehuda Sadeh)
+
+
+
+
+
+
+v0.93
+=====
+
+This is the first release candidate for Hammer, and includes all of
+the features that will be present in the final release.  We welcome
+and encourage any and all testing in non-production clusters to identify
+any problems with functionality, stability, or performance before the
+final Hammer release.
+
+We suggest some caution in one area: librbd.  There is a lot of new
+functionality around object maps and locking that is disabled by
+default but may still affect stability for existing images.  We are
+continuing to shake out those bugs so that the final Hammer release
+(probably v0.94) will be rock solid.
+
+Major features since Giant include:
+
+* cephfs: journal scavenger repair tool (John Spray)
+* crush: new and improved straw2 bucket type (Sage Weil, Christina Anderson, Xiaoxi Chen)
+* doc: improved guidance for CephFS early adopters (John Spray)
+* librbd: add per-image object map for improved performance (Jason Dillaman)
+* librbd: copy-on-read (Min Chen, Li Wang, Yunchuan Wen, Cheng Cheng)
+* librados: fadvise-style IO hints (Jianpeng Ma)
+* mds: many many snapshot-related fixes (Yan, Zheng)
+* mon: new 'ceph osd df' command (Mykola Golub)
+* mon: new 'ceph pg ls ...' command (Xinxin Shu)
+* osd: improved performance for high-performance backends
+* osd: improved recovery behavior (Samuel Just)
+* osd: improved cache tier behavior with reads (Zhiqiang Wang)
+* rgw: S3-compatible bucket versioning support (Yehuda Sadeh)
+* rgw: large bucket index sharding (Guang Yang, Yehuda Sadeh)
+* RDMA "xio" messenger support (Matt Benjamin, Vu Pham)
+
+Upgrading
+---------
+
+* If you are upgrading from v0.92, you must stop all OSD daemons and flush their
+  journals (``ceph-osd -i NNN --flush-journal``) before upgrading.  There was
+  a transaction encoding bug in v0.92 that broke compatibility.  Upgrading from
+  v0.91 or anything earlier is safe.
+
+* No special restrictions when upgrading from firefly or giant.
+
+Notable Changes
+---------------
+
+* build: CMake support (Ali Maredia, Casey Bodley, Adam Emerson, Marcus Watts, Matt Benjamin)
+* ceph-disk: do not re-use partition if encryption is required (Loic Dachary)
+* ceph-disk: support LUKS for encrypted partitions (Andrew Bartlett, Loic Dachary)
+* ceph-fuse,libcephfs: add support for O_NOFOLLOW and O_PATH (Greg Farnum)
+* ceph-fuse,libcephfs: resend requests before completing cap reconnect (#10912 Yan, Zheng)
+* ceph-fuse: select kernel cache invalidation mechanism based on kernel version (Greg Farnum)
+* ceph-objectstore-tool: improved import (David Zafman)
+* ceph-objectstore-tool: misc improvements, fixes (#9870 #9871 David Zafman)
+* ceph: add 'ceph osd df [tree]' command (#10452 Mykola Golub)
+* ceph: fix 'ceph tell ...' command validation (#10439 Joao Eduardo Luis)
+* ceph: improve 'ceph osd tree' output (Mykola Golub)
+* cephfs-journal-tool: add recover_dentries function (#9883 John Spray)
+* common: add newline to flushed json output (Sage Weil)
+* common: filtering for 'perf dump' (John Spray)
+* common: fix Formatter factory breakage (#10547 Loic Dachary)
+* common: make json-pretty output prettier (Sage Weil)
+* crush: new and improved straw2 bucket type (Sage Weil, Christina Anderson, Xiaoxi Chen)
+* crush: update tries stats for indep rules (#10349 Loic Dachary)
+* crush: use larger choose_tries value for erasure code rulesets (#10353 Loic Dachary)
+* debian,rpm: move RBD udev rules to ceph-common (#10864 Ken Dreyer)
+* debian: split python-ceph into python-{rbd,rados,cephfs} (Boris Ranto)
+* doc: CephFS disaster recovery guidance (John Spray)
+* doc: CephFS for early adopters (John Spray)
+* doc: fix OpenStack Glance docs (#10478 Sebastien Han)
+* doc: misc updates (#9793 #9922 #10204 #10203 Travis Rhoden, Hazem, Ayari, Florian Coste, Andy Allan, Frank Yu, Baptiste Veuillez-Mainard, Yuan Zhou, Armando Segnini, Robert Jansen, Tyler Brekke, Viktor Suprun)
+* doc: replace cloudfiles with swiftclient Python Swift example (Tim Freund)
+* erasure-code: add mSHEC erasure code support (Takeshi Miyamae)
+* erasure-code: improved docs (#10340 Loic Dachary)
+* erasure-code: set max_size to 20 (#10363 Loic Dachary)
+* libcephfs,ceph-fuse: fix getting zero-length xattr (#10552 Yan, Zheng)
+* librados: add blacklist_add convenience method (Jason Dillaman)
+* librados: expose rados_{read|write}_op_assert_version in C API (Kim Vandry)
+* librados: fix pool name caching (#10458 Radoslaw Zarzynski)
+* librados: fix resource leak, misc bugs (#10425 Radoslaw Zarzynski)
+* librados: fix some watch/notify locking (Jason Dillaman, Josh Durgin)
+* libradosstriper: fix write_full when ENOENT (#10758 Sebastien Ponce)
+* librbd: CRC protection for RBD image map (Jason Dillaman)
+* librbd: add per-image object map for improved performance (Jason Dillaman)
+* librbd: add support for an "object map" indicating which objects exist (Jason Dillaman)
+* librbd: adjust internal locking (Josh Durgin, Jason Dillaman)
+* librbd: better handling of watch errors (Jason Dillaman)
+* librbd: coordinate maint operations through lock owner (Jason Dillaman)
+* librbd: copy-on-read (Min Chen, Li Wang, Yunchuan Wen, Cheng Cheng, Jason Dillaman)
+* librbd: enforce write ordering with a snapshot (Jason Dillaman)
+* librbd: fadvise-style hints; add misc hints for certain operations (Jianpeng Ma)
+* librbd: fix coverity false-positives (Jason Dillaman)
+* librbd: fix snap create races (Jason Dillaman)
+* librbd: flush AIO operations asynchronously (#10714 Jason Dillaman)
+* librbd: make async versions of long-running maint operations (Jason Dillaman)
+* librbd: mock tests (Jason Dillaman)
+* librbd: optionally blacklist clients before breaking locks (#10761 Jason Dillaman)
+* librbd: prevent copyup during shrink (Jason Dillaman)
+* mds: add cephfs-table-tool (John Spray)
+* mds: avoid sending traceless replies in most cases (Yan, Zheng)
+* mds: export dir asok command (John Spray)
+* mds: fix stray/purge perfcounters (#10388 John Spray)
+* mds: handle heartbeat_reset during shutdown (#10382 John Spray)
+* mds: many many snapshot-related fixes (Yan, Zheng)
+* mds: refactor, improve Session storage (John Spray)
+* misc coverity fixes (Danny Al-Gaaf)
+* mon: add noforward flag for some mon commands (Mykola Golub)
+* mon: disallow empty pool names (#10555 Wido den Hollander)
+* mon: do not deactivate last mds (#10862 John Spray)
+* mon: drop old ceph_mon_store_converter (Sage Weil)
+* mon: fix 'ceph pg dump_stuck degraded' (Xinxin Shu)
+* mon: fix 'profile osd' use of config-key function on mon (#10844 Joao Eduardo Luis)
+* mon: fix compatset initalization during mkfs (Joao Eduardo Luis)
+* mon: fix feature tracking during elections (Joao Eduardo Luis)
+* mon: fix mds gid/rank/state parsing (John Spray)
+* mon: ignore failure reports from before up_from (#10762 Dan van der Ster, Sage Weil)
+* mon: improved corrupt CRUSH map detection (Joao Eduardo Luis)
+* mon: include pg_temp count in osdmap summary (Sage Weil)
+* mon: log health summary to cluster log (#9440 Joao Eduardo Luis)
+* mon: make 'mds fail' idempotent (John Spray)
+* mon: make pg dump {sum,pgs,pgs_brief} work for format=plain (#5963 #6759 Mykola Golub)
+* mon: new pool safety flags nodelete, nopgchange, nosizechange (#9792 Mykola Golub)
+* mon: new, friendly 'ceph pg ls ...' command (Xinxin Shu)
+* mon: prevent MDS transition from STOPPING (#10791 Greg Farnum)
+* mon: propose all pending work in one transaction (Sage Weil)
+* mon: remove pg_temps for nonexistent pools (Joao Eduardo Luis)
+* mon: require mon_allow_pool_delete option to remove pools (Sage Weil)
+* mon: set globalid prealloc to larger value (Sage Weil)
+* mon: skip zeroed osd stats in get_rule_avail (#10257 Joao Eduardo Luis)
+* mon: validate min_size range (Jianpeng Ma)
+* msgr: async: bind threads to CPU cores, improved poll (Haomai Wang)
+* msgr: fix crc configuration (Mykola Golub)
+* msgr: misc unit tests (Haomai Wang)
+* msgr: xio: XioMessenger RDMA support (Casey Bodley, Vu Pham, Matt Benjamin)
+* osd, librados: fadvise-style librados hints (Jianpeng Ma)
+* osd, librados: fix xattr_cmp_u64 (Dongmao Zhang)
+* osd,mon: add 'norebalance' flag (Kefu Chai)
+* osd,mon: specify OSD features explicitly in MOSDBoot (#10911 Sage Weil)
+* osd: add option to prioritize heartbeat network traffic (Jian Wen)
+* osd: add support for the SHEC erasure-code algorithm (Takeshi Miyamae, Loic Dachary)
+* osd: allow recovery while below min_size (Samuel Just)
+* osd: allow recovery with fewer than min_size OSDs (Samuel Just)
+* osd: allow writes to degraded objects (Samuel Just)
+* osd: allow writes to degraded objects (Samuel Just)
+* osd: avoid publishing unchanged PG stats (Sage Weil)
+* osd: cache recent ObjectContexts (Dong Yuan)
+* osd: clear cache on interval change (Samuel Just)
+* osd: do no proxy reads unless target OSDs are new (#10788 Sage Weil)
+* osd: do not update digest on inconsistent object (#10524 Samuel Just)
+* osd: don't record digests for snapdirs (#10536 Samuel Just)
+* osd: fix OSDCap parser on old (el6) boost::spirit (#10757 Kefu Chai)
+* osd: fix OSDCap parsing on el6 (#10757 Kefu Chai)
+* osd: fix ObjectStore::Transaction encoding version (#10734 Samuel Just)
+* osd: fix auth object selection during repair (#10524 Samuel Just)
+* osd: fix bug in pending digest updates (#10840 Samuel Just)
+* osd: fix cancel_proxy_read_ops (Sage Weil)
+* osd: fix cleanup of interrupted pg deletion (#10617 Sage Weil)
+* osd: fix journal wrapping bug (#10883 David Zafman)
+* osd: fix leak in SnapTrimWQ (#10421 Kefu Chai)
+* osd: fix memstore free space calculation (Xiaoxi Chen)
+* osd: fix mixed-version peering issues (Samuel Just)
+* osd: fix object digest update bug (#10840 Samuel Just)
+* osd: fix ordering issue with new transaction encoding (#10534 Dong Yuan)
+* osd: fix past_interval generation (#10427 #10430 David Zafman)
+* osd: fix short read handling on push (#8121 David Zafman)
+* osd: fix watch timeout cache state update (#10784 David Zafman)
+* osd: force promotion of watch/notify ops (Zhiqiang Wang)
+* osd: improve idempotency detection across cache promotion/demotion (#8935 Sage Weil, Samuel Just)
+* osd: include activating peers in blocked_by (#10477 Sage Weil)
+* osd: jerasure and gf-complete updates from upstream (#10216 Loic Dachary)
+* osd: journal: check fsync/fdatasync result (Jianpeng Ma)
+* osd: journal: fix hang on shutdown (#10474 David Zafman)
+* osd: journal: fix header.committed_up_to (Xinze Chi)
+* osd: journal: initialize throttle (Ning Yao)
+* osd: journal: misc bug fixes (#6003 David Zafman, Samuel Just)
+* osd: misc cleanup (Xinze Chi, Yongyue Sun)
+* osd: new 'activating' state between peering and active (Sage Weil)
+* osd: preserve reqids for idempotency checks for promote/demote (Sage Weil, Zhiqiang Wang, Samuel Just)
+* osd: remove dead locking code (Xinxin Shu)
+* osd: restrict scrub to certain times of day (Xinze Chi)
+* osd: rocksdb: fix shutdown (Hoamai Wang)
+* pybind: fix memory leak in librados bindings (Billy Olsen)
+* qa: fix mds tests (#10539 John Spray)
+* qa: ignore duplicates in rados ls (Josh Durgin)
+* qa: improve hadoop tests (Noah Watkins)
+* qa: reorg fs quota tests (Greg Farnum)
+* rados: fix usage (Jianpeng Ma)
+* radosgw-admin: add replicalog update command (Yehuda Sadeh)
+* rbd-fuse: clean up on shutdown (Josh Durgin)
+* rbd: add 'merge-diff' function (MingXin Liu, Yunchuan Wen, Li Wang)
+* rbd: fix buffer handling on image import (#10590 Jason Dillaman)
+* rbd: leave exclusive lockin goff by default (Jason Dillaman)
+* rbd: update init-rbdmap to fix dup mount point (Karel Striegel)
+* rbd: use IO hints for import, export, and bench operations (#10462 Jason Dillaman)
+* rbd_recover_tool: RBD image recovery tool (Min Chen)
+* rgw: S3-style object versioning support (Yehuda Sadeh)
+* rgw: check keystone auth for S3 POST requests (#10062 Abhishek Lekshmanan)
+* rgw: extend replica log API (purge-all) (Yehuda Sadeh)
+* rgw: fail S3 POST if keystone not configured (#10688 Valery Tschopp, Yehuda Sadeh)
+* rgw: fix XML header on get ACL request (#10106 Yehuda Sadeh)
+* rgw: fix bucket removal with data purge (Yehuda Sadeh)
+* rgw: fix replica log indexing (#8251 Yehuda Sadeh)
+* rgw: fix swift metadata header name (Dmytro Iurchenko)
+* rgw: remove multipart entries from bucket index on abort (#10719 Yehuda Sadeh)
+* rgw: respond with 204 to POST on containers (#10667 Yuan Zhou)
+* rgw: reuse fcgx connection struct (#10194 Yehuda Sadeh)
+* rgw: support multiple host names (#7467 Yehuda Sadeh)
+* rgw: swift: dump container's custom metadata (#10665 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: swift: support Accept header for response format (#10746 Dmytro Iurchenko)
+* rgw: swift: support for X-Remove-Container-Meta-{key} (#10475 Dmytro Iurchenko)
+* rpm: move rgw logrotate to radosgw subpackage (Ken Dreyer)
+* tests: centos-6 dockerfile (#10755 Loic Dachary)
+* tests: unit tests for shared_cache (Dong Yuan)
+* vstart.sh: work with cmake (Yehuda Sadeh)
+
+
+
+v0.92
+=====
+
+This is the second-to-last chunk of new stuff before Hammer.  Big items
+include additional checksums on OSD objects, proxied reads in the
+cache tier, image locking in RBD, optimized OSD Transaction and
+replication messages, and a big pile of RGW and MDS bug fixes.
+
+Upgrading
+---------
+
+* The experimental 'keyvaluestore-dev' OSD backend has been renamed
+  'keyvaluestore' (for simplicity) and marked as experimental.  To
+  enable this untested feature and acknowledge that you understand
+  that it is untested and may destroy data, you need to add the
+  following to your ceph.conf::
+
+    enable experimental unrecoverable data corrupting features = keyvaluestore
+
+* The following librados C API function calls take a 'flags' argument whose value
+  is now correctly interpreted:
+
+     rados_write_op_operate()
+     rados_aio_write_op_operate()
+     rados_read_op_operate()
+     rados_aio_read_op_operate()
+
+  The flags were not correctly being translated from the librados constants to the
+  internal values.  Now they are.  Any code that is passing flags to these methods
+  should be audited to ensure that they are using the correct LIBRADOS_OP_FLAG_*
+  constants.
+
+* The 'rados' CLI 'copy' and 'cppool' commands now use the copy-from operation,
+  which means the latest CLI cannot run these commands against pre-firefly OSDs.
+
+* The librados watch/notify API now includes a watch_flush() operation to flush
+  the async queue of notify operations.  This should be called by any watch/notify
+  user prior to rados_shutdown().
+
+Notable Changes
+---------------
+
+* add experimental features option (Sage Weil)
+* build: fix 'make check' races (#10384 Loic Dachary)
+* build: fix pkg names when libkeyutils is missing (Pankag Garg, Ken Dreyer)
+* ceph: make 'ceph -s' show PG state counts in sorted order (Sage Weil)
+* ceph: make 'ceph tell mon.* version' work (Mykola Golub)
+* ceph-monstore-tool: fix/improve CLI (Joao Eduardo Luis)
+* ceph: show primary-affinity in 'ceph osd tree' (Mykola Golub)
+* common: add TableFormatter (Andreas Peters)
+* common: check syncfs() return code (Jianpeng Ma)
+* doc: do not suggest dangerous XFS nobarrier option (Dan van der Ster)
+* doc: misc updates (Nilamdyuti Goswami, John Wilkins)
+* install-deps.sh: do not require sudo when root (Loic Dachary)
+* libcephfs: fix dirfrag trimming (#10387 Yan, Zheng)
+* libcephfs: fix mount timeout (#10041 Yan, Zheng)
+* libcephfs: fix test (#10415 Yan, Zheng)
+* libcephfs: fix use-afer-free on umount (#10412 Yan, Zheng)
+* libcephfs: include ceph and git version in client metadata (Sage Weil)
+* librados: add watch_flush() operation (Sage Weil, Haomai Wang)
+* librados: avoid memcpy on getxattr, read (Jianpeng Ma)
+* librados: create ioctx by pool id (Jason Dillaman)
+* librados: do notify completion in fast-dispatch (Sage Weil)
+* librados: remove shadowed variable (Kefu Chain)
+* librados: translate op flags from C APIs (Matthew Richards)
+* librbd: differentiate between R/O vs R/W features (Jason Dillaman)
+* librbd: exclusive image locking (Jason Dillaman)
+* librbd: fix write vs import race (#10590 Jason Dillaman)
+* librbd: gracefully handle deleted/renamed pools (#10270 Jason Dillaman)
+* mds: asok command for fetching subtree map (John Spray)
+* mds: constify MDSCacheObjects (John Spray)
+* misc: various valgrind fixes and cleanups (Danny Al-Gaaf)
+* mon: fix 'mds fail' for standby MDSs (John Spray)
+* mon: fix stashed monmap encoding (#5203 Xie Rui)
+* mon: implement 'fs reset' command (John Spray)
+* mon: respect down flag when promoting standbys (John Spray)
+* mount.ceph: fix suprious error message (#10351 Yan, Zheng)
+* msgr: async: many fixes, unit tests (Haomai Wang)
+* msgr: simple: retry binding to port on failure (#10029 Wido den Hollander)
+* osd: add fadvise flags to ObjectStore API (Jianpeng Ma)
+* osd: add get_latest_osdmap asok command (#9483 #9484 Mykola Golub)
+* osd: EIO on whole-object reads when checksum is wrong (Sage Weil)
+* osd: filejournal: don't cache journal when not using direct IO (Jianpeng Ma)
+* osd: fix ioprio option (Mykola Golub)
+* osd: fix scrub delay bug (#10693 Samuel Just)
+* osd: fix watch reconnect race (#10441 Sage Weil)
+* osd: handle no-op write with snapshot (#10262 Sage Weil)
+* osd: journal: fix journal zeroing when direct IO is enabled (Xie Rui)
+* osd: keyvaluestore: cleanup dead code (Ning Yao)
+* osd, mds: 'ops' as shorthand for 'dump_ops_in_flight' on asok (Sage Weil)
+* osd: memstore: fix size limit (Xiaoxi Chen)
+* osd: misc scrub fixes (#10017 Loic Dachary)
+* osd: new optimized encoding for ObjectStore::Transaction (Dong Yuan)
+* osd: optimize filter_snapc (Ning Yao)
+* osd: optimize WBThrottle map with unordered_map (Ning Yao)
+* osd: proxy reads during cache promote (Zhiqiang Wang)
+* osd: proxy read support (Zhiqiang Wang)
+* osd: remove legacy classic scrub code (Sage Weil)
+* osd: remove unused fields in MOSDSubOp (Xiaoxi Chen)
+* osd: replace MOSDSubOp messages with simpler, optimized MOSDRepOp (Xiaoxi Chen)
+* osd: store whole-object checksums on scrub, write_full (Sage Weil)
+* osd: verify kernel is new enough before using XFS extsize ioctl, enable by default (#9956 Sage Weil)
+* rados: use copy-from operation for copy, cppool (Sage Weil)
+* rgw: change multipart upload id magic (#10271 Yehuda Sadeh)
+* rgw: decode http query params correction (#10271 Yehuda Sadeh)
+* rgw: fix content length check (#10701 Axel Dunkel, Yehuda Sadeh)
+* rgw: fix partial GET in swift (#10553 Yehuda Sadeh)
+* rgw: fix shutdown (#10472 Yehuda Sadeh)
+* rgw: include XML ns on get ACL request (#10106 Yehuda Sadeh)
+* rgw: misc fixes (#10307 Yehuda Sadeh)
+* rgw: only track cleanup for objects we write (#10311 Yehuda Sadeh)
+* rgw: tweak error codes (#10329 #10334 Yehuda Sadeh)
+* rgw: use gc for multipart abort (#10445 Aaron Bassett, Yehuda Sadeh)
+* sysvinit: fix race in 'stop' (#10389 Loic Dachary)
+* test: fix bufferlist tests (Jianpeng Ma)
+* tests: improve docker-based tests (Loic Dachary)
+
+
+v0.91
+=====
+
+We are quickly approaching the Hammer feature freeze but have a few
+more dev releases to go before we get there.  The headline items are
+subtree-based quota support in CephFS (ceph-fuse/libcephfs client
+support only for now), a rewrite of the watch/notify librados API used
+by RBD and RGW, OSDMap checksums to ensure that maps are always
+consistent inside the cluster, new API calls in librados and librbd
+for IO hinting modeled after posix_fadvise, and improved storage of
+per-PG state.
+
+We expect two more releases before the Hammer feature freeze (v0.93).
+
+Upgrading
+---------
+
+* The 'category' field for objects has been removed.  This was originally added
+  to track PG stat summations over different categories of objects for use by
+  radosgw.  It is no longer has any known users and is prone to abuse because it
+  can lead to a pg_stat_t structure that is unbounded.  The librados API calls
+  that accept this field now ignore it, and the OSD no longer tracks the
+  per-category summations.
+
+* The output for 'rados df' has changed.  The 'category' level has been
+  eliminated, so there is now a single stat object per pool.  The structure of
+  the JSON output is different, and the plaintext output has one less column.
+
+* The 'rados create <objectname> [category]' optional category argument is no
+  longer supported or recognized.
+
+* rados.py's Rados class no longer has a __del__ method; it was causing
+  problems on interpreter shutdown and use of threads.  If your code has
+  Rados objects with limited lifetimes and you're concerned about locked
+  resources, call Rados.shutdown() explicitly.
+
+* There is a new version of the librados watch/notify API with vastly
+  improved semantics.  Any applications using this interface are
+  encouraged to migrate to the new API.  The old API calls are marked
+  as deprecated and will eventually be removed.
+
+* The librados rados_unwatch() call used to be safe to call on an
+  invalid handle.  The new version has undefined behavior when passed
+  a bogus value (for example, when rados_watch() returns an error and
+  handle is not defined).
+
+* The structure of the formatted 'pg stat' command is changed for the
+  portion that counts states by name to avoid using the '+' character
+  (which appears in state names) as part of the XML token (it is not
+  legal).
+
+Notable Changes
+---------------
+
+* asyncmsgr: misc fixes (Haomai Wang)
+* buffer: add 'shareable' construct (Matt Benjamin)
+* build: aarch64 build fixes (Noah Watkins, Haomai Wang)
+* build: support for jemalloc (Shishir Gowda)
+* ceph-disk: allow journal partition re-use (#10146 Loic Dachary, Dav van der Ster)
+* ceph-disk: misc fixes (Christos Stavrakakis)
+* ceph-fuse: fix kernel cache trimming (#10277 Yan, Zheng)
+* ceph-objectstore-tool: many many improvements (David Zafman)
+* common: support new gperftools header locations (Key Dreyer)
+* crush: straw bucket weight calculation fixes (#9998 Sage Weil)
+* doc: misc improvements (Nilamdyuti Goswami, John Wilkins, Chris Holcombe)
+* libcephfs,ceph-fuse: add 'status' asok (John Spray)
+* librados, osd: new watch/notify implementation (Sage Weil)
+* librados: drop 'category' feature (Sage Weil)
+* librados: fix pool deletion handling (#10372 Sage Weil)
+* librados: new fadvise API (Ma Jianpeng)
+* libradosstriper: fix remove() (Dongmao Zhang)
+* librbd: complete pending ops before closing image (#10299 Josh Durgin)
+* librbd: fadvise API (Ma Jianpeng)
+* mds: ENOSPC and OSDMap epoch barriers (#7317 John Spray)
+* mds: dirfrag buf fix (Yan, Zheng)
+* mds: disallow most commands on inactive MDS's (Greg Farnum)
+* mds: drop dentries, leases on deleted directories (#10164 Yan, Zheng)
+* mds: handle zero-size xattr (#10335 Yan, Zheng)
+* mds: subtree quota support (Yunchuan Wen)
+* memstore: free space tracking (John Spray)
+* misc cleanup (Danny Al-Gaaf, David Anderson)
+* mon: 'osd crush reweight-all' command (Sage Weil)
+* mon: allow full flag to be manually cleared (#9323 Sage Weil)
+* mon: delay failure injection (Joao Eduardo Luis)
+* mon: fix paxos timeouts (#10220 Joao Eduardo Luis)
+* mon: get canonical OSDMap from leader (#10422 Sage Weil)
+* msgr: fix RESETSESSION bug (#10080 Greg Farnum)
+* objectstore: deprecate collection attrs (Sage Weil)
+* osd, mon: add checksums to all OSDMaps (Sage Weil)
+* osd: allow deletion of objects with watcher (#2339 Sage Weil)
+* osd: allow sparse read for Push/Pull (Haomai Wang)
+* osd: cache reverse_nibbles hash value (Dong Yuan)
+* osd: drop upgrade support for pre-dumpling (Sage Weil)
+* osd: enable and use posix_fadvise (Sage Weil)
+* osd: erasure-code: enforce chunk size alignment (#10211 Loic Dachary)
+* osd: erasure-code: jerasure support for NEON (Loic Dachary)
+* osd: erasure-code: relax cauchy w restrictions (#10325 David Zhang, Loic Dachary)
+* osd: erasure-code: update gf-complete to latest upstream (Loic Dachary)
+* osd: fix WBTHrottle perf counters (Haomai Wang)
+* osd: fix backfill bug (#10150 Samuel Just)
+* osd: fix occasional peering stalls (#10431 Sage Weil)
+* osd: fix scrub vs try-flush bug (#8011 Samuel Just)
+* osd: fix stderr with -f or -d (Dan Mick)
+* osd: misc FIEMAP fixes (Ma Jianpeng)
+* osd: optimize Finisher (Xinze Chi)
+* osd: store PG metadata in per-collection objects for better concurrency (Sage Weil)
+* pyrados: add object lock support (#6114 Mehdi Abaakouk)
+* pyrados: fix misnamed wait_* routings (#10104 Dan Mick)
+* pyrados: misc cleanups (Kefu Chai)
+* qa: add large auth ticket tests (Ilya Dryomov)
+* qa: many 'make check' improvements (Loic Dachary)
+* qa: misc tests (Loic Dachary, Yan, Zheng)
+* rgw: conditional PUT on ETag (#8562 Ray Lv)
+* rgw: fix error codes (#10334 #10329 Yehuda Sadeh)
+* rgw: index swift keys appropriately (#10471 Yehuda Sadeh)
+* rgw: prevent illegal bucket policy that doesn't match placement rule (Yehuda Sadeh)
+* rgw: run radosgw as apache with systemd (#10125 Loic Dachary)
+* rgw: support X-Storage-Policy header for Swift storage policy compat (Yehuda Sadeh)
+* rgw: use \r\n for http headers (#9254 Yehuda Sadeh)
+* rpm: misc fixes (Key Dreyer)
+
+
+v0.90
+=====
+
+This is the last development release before Christmas.  There are some
+API cleanups for librados and librbd, and lots of bug fixes across the
+board for the OSD, MDS, RGW, and CRUSH.  The OSD also gets support for
+discard (potentially helpful on SSDs, although it is off by default), and there
+are several improvements to ceph-disk.
+
+The next two development releases will be getting a slew of new
+functionality for hammer.  Stay tuned!
+
+Upgrading
+---------
+
+* Previously, the formatted output of 'ceph pg stat -f ...' was a full
+  pg dump that included all metadata about all PGs in the system.  It
+  is now a concise summary of high-level PG stats, just like the
+  unformatted 'ceph pg stat' command.
+
+* All JSON dumps of floating point values were incorrecting surrounding the
+  value with quotes.  These quotes have been removed.  Any consumer of structured
+  JSON output that was consuming the floating point values was previously having
+  to interpret the quoted string and will most likely need to be fixed to take
+  the unquoted number.
+
+Notable Changes
+---------------
+
+* arch: fix NEON feaeture detection (#10185 Loic Dachary)
+* build: adjust build deps for yasm, virtualenv (Jianpeng Ma)
+* build: improve build dependency tooling (Loic Dachary)
+* ceph-disk: call partx/partprobe consistency (#9721 Loic Dachary)
+* ceph-disk: fix dmcrypt key permissions (Loic Dachary)
+* ceph-disk: fix umount race condition (#10096 Blaine Gardner)
+* ceph-disk: init=none option (Loic Dachary)
+* ceph-monstore-tool: fix shutdown (#10093 Loic Dachary)
+* ceph-objectstore-tool: fix import (#10090 David Zafman)
+* ceph-objectstore-tool: many improvements and tests (David Zafman)
+* ceph.spec: package rbd-replay-prep (Ken Dreyer)
+* common: add 'perf reset ...' admin command (Jianpeng Ma)
+* common: do not unlock rwlock on destruction (Federico Simoncelli)
+* common: fix block device discard check (#10296 Sage Weil)
+* common: remove broken CEPH_LOCKDEP optoin (Kefu Chai)
+* crush: fix tree bucket behavior (Rongze Zhu)
+* doc: add build-doc guidlines for Fedora and CentOS/RHEL (Nilamdyuti Goswami)
+* doc: enable rbd cache on openstack deployments (Sebastien Han)
+* doc: improved installation nots on CentOS/RHEL installs (John Wilkins)
+* doc: misc cleanups (Adam Spiers, Sebastien Han, Nilamdyuti Goswami, Ken Dreyer, John Wilkins)
+* doc: new man pages (Nilamdyuti Goswami)
+* doc: update release descriptions (Ken Dreyer)
+* doc: update sepia hardware inventory (Sandon Van Ness)
+* librados: only export public API symbols (Jason Dillaman)
+* libradosstriper: fix stat strtoll (Dongmao Zhang)
+* libradosstriper: fix trunc method (#10129 Sebastien Ponce)
+* librbd: fix list_children from invalid pool ioctxs (#10123 Jason Dillaman)
+* librbd: only export public API symbols (Jason Dillaman)
+* many coverity fixes (Danny Al-Gaaf)
+* mds: 'flush journal' admin command (John Spray)
+* mds: fix MDLog IO callback deadlock (John Spray)
+* mds: fix deadlock during journal probe vs purge (#10229 Yan, Zheng)
+* mds: fix race trimming log segments (Yan, Zheng)
+* mds: store backtrace for stray dir (Yan, Zheng)
+* mds: verify backtrace when fetching dirfrag (#9557 Yan, Zheng)
+* mon: add max pgs per osd warning (Sage Weil)
+* mon: fix *_ratio* units and types (Sage Weil)
+* mon: fix JSON dumps to dump floats as flots and not strings (Sage Weil)
+* mon: fix formatter 'pg stat' command output (Sage Weil)
+* msgr: async: several fixes (Haomai Wang)
+* msgr: simple: fix rare deadlock (Greg Farnum)
+* osd: batch pg log trim (Xinze Chi)
+* osd: clean up internal ObjectStore interface (Sage Weil)
+* osd: do not abort deep scrub on missing hinfo (#10018 Loic Dachary)
+* osd: fix ghobject_t formatted output to include shard (#10063 Loic Dachary)
+* osd: fix osd peer check on scrub messages (#9555 Sage Weil)
+* osd: fix pgls filter ops (#9439 David Zafman)
+* osd: flush snapshots from cache tier immediately (Sage Weil)
+* osd: keyvaluestore: fix getattr semantics (Haomai Wang)
+* osd: keyvaluestore: fix key ordering (#10119 Haomai Wang)
+* osd: limit in-flight read requests (Jason Dillaman)
+* osd: log when scrub or repair starts (Loic Dachary)
+* osd: support for discard for journal trim (Jianpeng Ma)
+* qa: fix osd create dup tests (#10083 Loic Dachary)
+* rgw: add location header when object is in another region (VRan Liu)
+* rgw: check timestamp on s3 keystone auth (#10062 Abhishek Lekshmanan)
+* rgw: make sysvinit script set ulimit -n properly (Sage Weil)
+* systemd: better systemd unit files (Owen Synge)
+* tests: ability to run unit tests under docker (Loic Dachary)
+
+
+v0.89
+=====
+
+This is the second development release since Giant.  The big items
+include the first batch of scrub patches from Greg for CephFS, a rework
+in the librados object listing API to properly handle namespaces, and
+a pile of bug fixes for RGW.  There are also several smaller issues
+fixed up in the performance area with buffer alignment and memory
+copies, osd cache tiering agent, and various CephFS fixes.
+
+Upgrading
+---------
+
+* New ability to list all objects from all namespaces can fail or
+  return incomplete results when not all OSDs have been upgraded.
+  Features rados --all ls, rados cppool, rados export, rados
+  cache-flush-evict-all and rados cache-try-flush-evict-all can also
+  fail or return incomplete results.
+
+Notable Changes
+---------------
+
+* buffer: add list::get_contiguous (Sage Weil)
+* buffer: avoid rebuild if buffer already contiguous (Jianpeng Ma)
+* ceph-disk: improved systemd support (Owen Synge)
+* ceph-disk: set guid if reusing journal partition (Dan van der Ster)
+* ceph-fuse, libcephfs: allow xattr caps in inject_release_failure (#9800 John Spray)
+* ceph-fuse, libcephfs: fix I_COMPLETE_ORDERED checks (#9894 Yan, Zheng)
+* ceph-fuse: fix dentry invalidation on 3.18+ kernels (#9997 Yan, Zheng)
+* crush: fix detach_bucket (#10095 Sage Weil)
+* crush: fix several bugs in adjust_item_weight (Rongze Zhu)
+* doc: add dumpling to firefly upgrade section (#7679 John Wilkins)
+* doc: document erasure coded pool operations (#9970 Loic Dachary)
+* doc: file system osd config settings (Kevin Dalley)
+* doc: key/value store config reference (John Wilkins)
+* doc: update openstack docs for Juno (Sebastien Han)
+* fix cluster logging from non-mon daemons (Sage Weil)
+* init-ceph: check for systemd-run before using it (Boris Ranto)
+* librados: fix infinite loop with skipped map epochs (#9986 Ding Dinghua)
+* librados: fix iterator operator= bugs (#10082 David Zafman, Yehuda Sadeh)
+* librados: fix null deref when pool DNE (#9944 Sage Weil)
+* librados: fix timer race from recent refactor (Sage Weil)
+* libradosstriper: fix shutdown hang (Dongmao Zhang)
+* librbd: don't close a closed parent in failure path (#10030 Jason Dillaman)
+* librbd: fix diff test (#10002 Josh Durgin)
+* librbd: fix locking for readahead (#10045 Jason Dillaman)
+* librbd: refactor unit tests to use fixtures (Jason Dillaman)
+* many many coverity cleanups (Danny Al-Gaaf)
+* mds: a whole bunch of initial scrub infrastructure (Greg Farnum)
+* mds: fix compat_version for MClientSession (#9945 John Spray)
+* mds: fix reply snapbl (Yan, Zheng)
+* mon: allow adding tiers to fs pools (#10135 John Spray)
+* mon: fix MDS health status from peons (#10151 John Spray)
+* mon: fix caching for min_last_epoch_clean (#9987 Sage Weil)
+* mon: fix error output for add_data_pool (#9852 Joao Eduardo Luis)
+* mon: include entity name in audit log for forwarded requests (#9913 Joao Eduardo Luis)
+* mon: paxos: allow reads while proposing (#9321 #9322 Joao Eduardo Luis)
+* msgr: asyncmessenger: add kqueue support (#9926 Haomai Wang)
+* osd, librados: revamp PG listing API to handle namespaces (#9031 #9262 #9438 David Zafman)
+* osd, mon: send intiial pg create time from mon to osd (#9887 David Zafman)
+* osd: allow whiteout deletion in cache pool (Sage Weil)
+* osd: cache pool: ignore min flush age when cache is full (Xinze Chi)
+* osd: erasure coding: allow bench.sh to test ISA backend (Yuan Zhou)
+* osd: erasure-code: encoding regression tests, corpus (#9420 Loic Dachary)
+* osd: fix journal shutdown race (Sage Weil)
+* osd: fix object age eviction (Zhiqiang Wang)
+* osd: fix object atime calculation (Xinze Chi)
+* osd: fix past_interval display bug (#9752 Loic Dachary)
+* osd: journal: fix alignment checks, avoid useless memmove (Jianpeng Ma)
+* osd: journal: update committed_thru after replay (#6756 Samuel Just)
+* osd: keyvaluestore_dev: optimization (Chendi Xue)
+* osd: make misdirected op checks robust for EC pools (#9835 Sage Weil)
+* osd: removed some dead code (Xinze Chi)
+* qa: parallelize make check (Loic Dachary)
+* qa: tolerate nearly-full disk for make check (Loic Dachary)
+* rgw: create subuser if needed when creating user (#10103 Yehuda Sadeh)
+* rgw: fix If-Modified-Since (VRan Liu)
+* rgw: fix content-length update (#9576 Yehuda Sadeh)
+* rgw: fix disabling of max_size quota (#9907 Dong Lei)
+* rgw: fix incorrect len when len is 0 (#9877 Yehuda Sadeh)
+* rgw: fix object copy content type (#9478 Yehuda Sadeh)
+* rgw: fix user stags in get-user-info API (#9359 Ray Lv)
+* rgw: remove swift user manifest (DLO) hash calculation (#9973 Yehuda Sadeh)
+* rgw: return timestamp on GET/HEAD (#8911 Yehuda Sadeh)
+* rgw: set ETag on object copy (#9479 Yehuda Sadeh)
+* rgw: update bucket index on attr changes, for multi-site sync (#5595 Yehuda Sadeh)
+
+
+v0.88
+=====
+
+This is the first development release after Giant.  The two main
+features merged this round are the new AsyncMessenger (an alternative
+implementation of the network layer) from Haomai Wang at UnitedStack,
+and support for POSIX file locks in ceph-fuse and libcephfs from Yan,
+Zheng.  There is also a big pile of smaller items that re merged while
+we were stabilizing Giant, including a range of smaller performance
+and bug fixes and some new tracepoints for LTTNG.
+
+Notable Changes
+---------------
+
+* ceph-disk: Scientific Linux support (Dan van der Ster)
+* ceph-disk: respect --statedir for keyring (Loic Dachary)
+* ceph-fuse, libcephfs: POSIX file lock support (Yan, Zheng)
+* ceph-fuse, libcephfs: fix cap flush overflow (Greg Farnum, Yan, Zheng)
+* ceph-fuse, libcephfs: fix root inode xattrs (Yan, Zheng)
+* ceph-fuse, libcephfs: preserve dir ordering (#9178 Yan, Zheng)
+* ceph-fuse, libcephfs: trim inodes before reconnecting to MDS (Yan, Zheng)
+* ceph: do not parse injectargs twice (Loic Dachary)
+* ceph: make 'ceph -s' output more readable (Sage Weil)
+* ceph: new 'ceph tell mds.$name_or_rank_or_gid' (John Spray)
+* ceph: test robustness (Joao Eduardo Luis)
+* ceph_objectstore_tool: behave with sharded flag (#9661 David Zafman)
+* cephfs-journal-tool: fix journal import (#10025 John Spray)
+* cephfs-journal-tool: skip up to expire_pos (#9977 John Spray)
+* cleanup rados.h definitions with macros (Ilya Dryomov)
+* common: shared_cache unit tests (Cheng Cheng)
+* config: add $cctid meta variable (Adam Crume)
+* crush: fix buffer overrun for poorly formed rules (#9492 Johnu George)
+* crush: improve constness (Loic Dachary)
+* crushtool: add --location <id> command (Sage Weil, Loic Dachary)
+* default to libnss instead of crypto++ (Federico Gimenez)
+* doc: ceph osd reweight vs crush weight (Laurent Guerby)
+* doc: document the LRC per-layer plugin configuration (Yuan Zhou)
+* doc: erasure code doc updates (Loic Dachary)
+* doc: misc updates (Alfredo Deza, VRan Liu)
+* doc: preflight doc fixes (John Wilkins)
+* doc: update PG count guide (Gerben Meijer, Laurent Guerby, Loic Dachary)
+* keyvaluestore: misc fixes (Haomai Wang)
+* keyvaluestore: performance improvements (Haomai Wang)
+* librados: add rados_pool_get_base_tier() call (Adam Crume)
+* librados: cap buffer length (Loic Dachary)
+* librados: fix objecter races (#9617 Josh Durgin)
+* libradosstriper: misc fixes (Sebastien Ponce)
+* librbd: add missing python docstrings (Jason Dillaman)
+* librbd: add readahead (Adam Crume)
+* librbd: fix cache tiers in list_children and snap_unprotect (Adam Crume)
+* librbd: fix performance regression in ObjectCacher (#9513 Adam Crume)
+* librbd: lttng tracepoints (Adam Crume)
+* librbd: misc fixes (Xinxin Shu, Jason Dillaman)
+* mds: fix sessionmap lifecycle bugs (Yan, Zheng)
+* mds: initialize root inode xattr version (Yan, Zheng)
+* mds: introduce auth caps (John Spray)
+* mds: misc bugs (Greg Farnum, John Spray, Yan, Zheng, Henry Change)
+* misc coverity fixes (Danny Al-Gaaf)
+* mon: add 'ceph osd rename-bucket ...' command (Loic Dachary)
+* mon: clean up auth list output (Loic Dachary)
+* mon: fix 'osd crush link' id resolution (John Spray)
+* mon: fix misc error paths (Joao Eduardo Luis)
+* mon: fix paxos off-by-one corner case (#9301 Sage Weil)
+* mon: new 'ceph pool ls [detail]' command (Sage Weil)
+* mon: wait for writeable before cross-proposing (#9794 Joao Eduardo Luis)
+* msgr: avoid useless new/delete (Haomai Wang)
+* msgr: fix delay injection bug (#9910 Sage Weil, Greg Farnum)
+* msgr: new AsymcMessenger alternative implementation (Haomai Wang)
+* msgr: prefetch data when doing recv (Yehuda Sadeh)
+* osd: add erasure code corpus (Loic Dachary)
+* osd: add misc tests (Loic Dachary, Danny Al-Gaaf)
+* osd: cleanup boost optionals (William Kennington)
+* osd: expose non-journal backends via ceph-osd CLI (Hoamai Wang)
+* osd: fix JSON output for stray OSDs (Loic Dachary)
+* osd: fix ioprio options (Loic Dachary)
+* osd: fix transaction accounting (Jianpeng Ma)
+* osd: misc optimizations (Xinxin Shu, Zhiqiang Wang, Xinze Chi)
+* osd: use FIEMAP_FLAGS_SYNC instead of fsync (Jianpeng Ma)
+* rados: fix put of /dev/null (Loic Dachary)
+* rados: parse command-line arguments more strictly (#8983 Adam Crume)
+* rbd-fuse: fix memory leak (Adam Crume)
+* rbd-replay-many (Adam Crume)
+* rbd-replay: --anonymize flag to rbd-replay-prep (Adam Crume)
+* rbd: fix 'rbd diff' for non-existent objects (Adam Crume)
+* rbd: fix error when striping with format 1 (Sebastien Han)
+* rbd: fix export for image sizes over 2GB (Vicente Cheng)
+* rbd: use rolling average for rbd bench-write throughput (Jason Dillaman)
+* rgw: send explicit HTTP status string (Yehuda Sadeh)
+* rgw: set length for keystone token validation request (#7796 Yehuda Sadeh, Mark Kirkwood)
+* udev: fix rules for CentOS7/RHEL7 (Loic Dachary)
+* use clock_gettime instead of gettimeofday (Jianpeng Ma)
+* vstart.sh: set up environment for s3-tests (Luis Pabon)
diff --git a/doc/releases/index.rst b/doc/releases/index.rst
new file mode 100644
index 000000000..47279c8c5
--- /dev/null
+++ b/doc/releases/index.rst
@@ -0,0 +1,279 @@
+.. _ceph-releases-index:
+
+=====================
+Ceph Releases (index)
+=====================
+
+.. toctree::
+   :maxdepth: 1
+
+.. ceph_releases_gantt:: releases.yml
+
+.. _active-releases:
+
+Active Releases
+---------------
+
+The following Ceph releases are actively maintained and receive periodic backports and
+security fixes.
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   Reef (v18.2.*) <reef>
+   Quincy (v17.2.*) <quincy>
+   Pacific (v16.2.*) <pacific>
+
+.. ceph_releases:: releases.yml current
+
+
+Archived Releases
+-----------------
+
+The following older Ceph releases are no longer maintained (do not
+receive bug fixes or backports).
+
+.. ceph_releases:: releases.yml eol
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   Octopus (v15.2.*) <octopus>
+   Nautilus (v14.2.*) <nautilus>
+   Mimic (v13.2.*) <mimic>
+   Luminous (v12.2.*) <luminous>
+   Kraken (v11.2.*) <kraken>
+   Jewel (v10.2.*) <jewel>
+   Infernalis (v9.2.*) <infernalis>
+   Hammer (v0.94.*) <hammer>
+   Giant (v0.87.*) <giant>
+   Firefly (v0.80.*) <firefly>
+   Emperor (v0.72.*) <emperor>
+   Dumpling (v0.67.*) <dumpling>
+   Cuttlefish (v0.61.*) <cuttlefish>
+   Bobtail (v0.56.*) <bobtail>
+   Argonaut (v0.48.*) <argonaut>
+
+Release timeline
+----------------
+
+.. ceph_timeline_gantt:: releases.yml reef quincy 
+.. ceph_timeline:: releases.yml reef quincy 
+
+.. _Reef: reef
+.. _18.2.0: reef#v18-2-0-reef
+
+.. _Quincy: quincy
+.. _17.2.0: quincy#v17-2-0-quincy
+.. _17.2.1: quincy#v17-2-1-quincy
+.. _17.2.2: quincy#v17-2-2-quincy
+.. _17.2.3: quincy#v17-2-3-quincy
+.. _17.2.4: quincy#v17-2-4-quincy
+.. _17.2.5: quincy#v17-2-5-quincy
+
+.. _Pacific: pacific
+.. _16.2.11: pacific#v16-2-11-pacific
+.. _16.2.10: pacific#v16-2-10-pacific
+.. _16.2.9: pacific#v16-2-9-pacific
+.. _16.2.8: pacific#v16-2-8-pacific
+.. _16.2.7: pacific#v16-2-7-pacific
+.. _16.2.6: pacific#v16-2-6-pacific
+.. _16.2.5: pacific#v16-2-5-pacific
+.. _16.2.4: pacific#v16-2-4-pacific
+.. _16.2.3: pacific#v16-2-3-pacific
+.. _16.2.2: pacific#v16-2-2-pacific
+.. _16.2.1: pacific#v16-2-1-pacific
+.. _16.2.0: pacific#v16-2-0-pacific
+
+.. _Octopus: octopus
+.. _15.2.17: octopus#v15-2-17-octopus
+.. _15.2.16: octopus#v15-2-16-octopus
+.. _15.2.15: octopus#v15-2-15-octopus
+.. _15.2.14: octopus#v15-2-14-octopus
+.. _15.2.13: octopus#v15-2-13-octopus
+.. _15.2.12: octopus#v15-2-12-octopus
+.. _15.2.11: octopus#v15-2-11-octopus
+.. _15.2.10: octopus#v15-2-10-octopus
+.. _15.2.9: octopus#v15-2-9-octopus
+.. _15.2.8: octopus#v15-2-8-octopus
+.. _15.2.7: octopus#v15-2-7-octopus
+.. _15.2.6: octopus#v15-2-6-octopus
+.. _15.2.5: octopus#v15-2-5-octopus
+.. _15.2.4: octopus#v15-2-4-octopus
+.. _15.2.3: octopus#v15-2-3-octopus
+.. _15.2.2: octopus#v15-2-2-octopus
+.. _15.2.1: octopus#v15-2-1-octopus
+.. _15.2.0: octopus#v15-2-0-octopus
+
+.. _Nautilus: nautilus
+.. _14.2.22: nautilus#v14-2-22-nautilus
+.. _14.2.21: nautilus#v14-2-21-nautilus
+.. _14.2.20: nautilus#v14-2-20-nautilus
+.. _14.2.19: nautilus#v14-2-19-nautilus
+.. _14.2.18: nautilus#v14-2-18-nautilus
+.. _14.2.17: nautilus#v14-2-17-nautilus
+.. _14.2.16: nautilus#v14-2-16-nautilus
+.. _14.2.15: nautilus#v14-2-15-nautilus
+.. _14.2.14: nautilus#v14-2-14-nautilus
+.. _14.2.13: nautilus#v14-2-13-nautilus
+.. _14.2.12: nautilus#v14-2-12-nautilus
+.. _14.2.11: nautilus#v14-2-11-nautilus
+.. _14.2.10: nautilus#v14-2-10-nautilus
+.. _14.2.9: nautilus#v14-2-9-nautilus
+.. _14.2.8: nautilus#v14-2-8-nautilus
+.. _14.2.7: nautilus#v14-2-7-nautilus
+.. _14.2.6: nautilus#v14-2-6-nautilus
+.. _14.2.5: nautilus#v14-2-5-nautilus
+.. _14.2.4: nautilus#v14-2-4-nautilus
+.. _14.2.3: nautilus#v14-2-3-nautilus
+.. _14.2.2: nautilus#v14-2-2-nautilus
+.. _14.2.1: nautilus#v14-2-1-nautilus
+.. _14.2.0: nautilus#v14-2-0-nautilus
+
+.. _Mimic: mimic
+.. _13.2.10: mimic#v13-2-10-mimic
+.. _13.2.9: mimic#v13-2-9-mimic
+.. _13.2.8: mimic#v13-2-8-mimic
+.. _13.2.7: mimic#v13-2-7-mimic
+.. _13.2.6: mimic#v13-2-6-mimic
+.. _13.2.5: mimic#v13-2-5-mimic
+.. _13.2.4: mimic#v13-2-4-mimic
+.. _13.2.3: mimic#v13-2-3-mimic
+.. _13.2.2: mimic#v13-2-2-mimic
+.. _13.2.1: mimic#v13-2-1-mimic
+.. _13.2.0: mimic#v13-2-0-mimic
+
+.. _Luminous: luminous
+.. _12.2.13: luminous#v12-2-13-luminous
+.. _12.2.12: luminous#v12-2-12-luminous
+.. _12.2.11: luminous#v12-2-11-luminous
+.. _12.2.10: luminous#v12-2-10-luminous
+.. _12.2.9: luminous#v12-2-9-luminous
+.. _12.2.8: luminous#v12-2-8-luminous
+.. _12.2.7: luminous#v12-2-7-luminous
+.. _12.2.6: luminous#v12-2-6-luminous
+.. _12.2.5: luminous#v12-2-5-luminous
+.. _12.2.4: luminous#v12-2-4-luminous
+.. _12.2.3: luminous#v12-2-3-luminous
+.. _12.2.2: luminous#v12-2-2-luminous
+.. _12.2.1: luminous#v12-2-1-luminous
+.. _12.2.0: luminous#v12-2-0-luminous
+
+.. _Kraken: kraken
+.. _11.2.1: kraken#v11-2-1-kraken
+.. _11.2.0: kraken#v11-2-0-kraken
+.. _11.0.2: kraken#v11-0-2-kraken
+
+.. _Jewel: jewel
+.. _10.2.11: jewel#v10-2-11-jewel
+.. _10.2.10: jewel#v10-2-10-jewel
+.. _10.2.9: jewel#v10-2-9-jewel
+.. _10.2.8: jewel#v10-2-8-jewel
+.. _10.2.7: jewel#v10-2-7-jewel
+.. _10.2.6: jewel#v10-2-6-jewel
+.. _10.2.5: jewel#v10-2-5-jewel
+.. _10.2.4: jewel#v10-2-4-jewel
+.. _10.2.3: jewel#v10-2-3-jewel
+.. _10.2.2: jewel#v10-2-2-jewel
+.. _10.2.1: jewel#v10-2-1-jewel
+.. _10.2.0: jewel#v10-2-0-jewel
+
+.. _10.1.2: jewel#v10-1-2-jewel-release-candidate
+.. _10.1.1: jewel#v10-1-1-jewel-release-candidate
+.. _10.1.0: jewel#v10-1-0-jewel-release-candidate
+.. _10.0.5: jewel#v10-0-5
+.. _10.0.3: jewel#v10-0-3
+.. _10.0.2: jewel#v10-0-2
+.. _10.0.1: jewel#v10-0-1
+.. _10.0.0: jewel#v10-0-0
+
+.. _Infernalis: infernalis
+.. _9.2.1: infernalis#v9-2-1-infernalis
+.. _9.2.0: infernalis#v9-2-0-infernalis
+
+.. _9.1.0: infernalis#v9-1-0
+.. _9.0.3: infernalis#v9-0-3
+.. _9.0.2: infernalis#v9-0-2
+.. _9.0.1: infernalis#v9-0-1
+.. _9.0.0: infernalis#v9-0-0
+
+.. _0.94.10: hammer#v0-94-10-hammer
+.. _0.94.9: hammer#v0-94-9-hammer
+.. _0.94.8: hammer#v0-94-8-hammer
+.. _0.94.7: hammer#v0-94-7-hammer
+.. _0.94.6: hammer#v0-94-6-hammer
+.. _0.94.5: hammer#v0-94-5-hammer
+.. _0.94.4: hammer#v0-94-4-hammer
+.. _0.94.3: hammer#v0-94-3-hammer
+.. _0.94.2: hammer#v0-94-2-hammer
+.. _0.94.1: hammer#v0-94-1-hammer
+.. _0.94: hammer#v0-94-hammer
+.. _Hammer: hammer
+
+.. _0.93: hammer#v0-93
+.. _0.92: hammer#v0-92
+.. _0.91: hammer#v0-91
+.. _0.90: hammer#v0-90
+.. _0.89: hammer#v0-89
+.. _0.88: hammer#v0-88
+
+.. _0.87.2: giant#v0-87-2-giant
+.. _0.87.1: giant#v0-87-1-giant
+.. _0.87: giant#v0-87-giant
+.. _Giant: giant
+
+.. _0.86: giant#v0-86
+.. _0.85: giant#v0-85
+.. _0.84: giant#v0-84
+.. _0.83: giant#v0-83
+.. _0.82: giant#v0-82
+.. _0.81: giant#v0-81
+
+.. _0.80.11: firefly#v0-80-11-firefly
+.. _0.80.10: firefly#v0-80-10-firefly
+.. _0.80.9: firefly#v0-80-9-firefly
+.. _0.80.8: firefly#v0-80-8-firefly
+.. _0.80.7: firefly#v0-80-7-firefly
+.. _0.80.6: firefly#v0-80-6-firefly
+.. _0.80.5: firefly#v0-80-5-firefly
+.. _0.80.4: firefly#v0-80-4-firefly
+.. _0.80.3: firefly#v0-80-3-firefly
+.. _0.80.2: firefly#v0-80-2-firefly
+.. _0.80.1: firefly#v0-80-1-firefly
+.. _0.80: firefly#v0-80-firefly
+.. _Firefly: firefly
+
+.. _0.79: firefly#v0-79
+.. _0.78: firefly#v0-78
+.. _0.77: firefly#v0-77
+.. _0.76: firefly#v0-76
+.. _0.75: firefly#v0-75
+.. _0.74: firefly#v0-74
+.. _0.73: firefly#v0-73
+
+.. _0.72.2: emperor#v0-72-2-emperor
+.. _0.72.1: emperor#v0-72-1-emperor
+.. _0.72: emperor#v0-72-emperor
+.. _Emperor: emperor
+
+.. _0.71: dumpling#v0-71
+.. _0.70: dumpling#v0-70
+.. _0.69: dumpling#v0-69
+.. _0.68: dumpling#v0-68
+
+.. _0.67.11: dumpling#v0-67-11-dumpling
+.. _0.67.10: dumpling#v0-67-10-dumpling
+.. _0.67.9: dumpling#v0-67-9-dumpling
+.. _0.67.8: dumpling#v0-67-8-dumpling
+.. _0.67.7: dumpling#v0-67-7-dumpling
+.. _0.67.6: dumpling#v0-67-6-dumpling
+.. _0.67.5: dumpling#v0-67-5-dumpling
+.. _0.67.4: dumpling#v0-67-4-dumpling
+.. _0.67.3: dumpling#v0-67-3-dumpling
+.. _0.67.2: dumpling#v0-67-2-dumpling
+.. _0.67.1: dumpling#v0-67-1-dumpling
+.. _0.67: dumpling#v0-67-dumpling
+.. _Dumpling:  dumpling
+
diff --git a/doc/releases/infernalis.rst b/doc/releases/infernalis.rst
new file mode 100644
index 000000000..cd222f697
--- /dev/null
+++ b/doc/releases/infernalis.rst
@@ -0,0 +1,1534 @@
+.. _infernalis-release-notes:
+
+==========
+Infernalis
+==========
+
+Infernalis is the 9th stable release of Ceph.  It is named after the
+vampire squid (Vampyroteuthis infernalis, the "vampire squid from
+hell").
+
+v9.2.1 Infernalis
+=================
+
+This Infernalis point release fixes several packagins and init script
+issues, enables the librbd objectmap feature by default, a few librbd
+bugs, and a range of miscellaneous bug fixes across the system.
+
+We recommend that all infernalis v9.2.0 users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v9.2.1.txt>`.
+
+Upgrading
+---------
+
+* Some symbols wrongly exposed by the C++ interface for librados in
+  v9.1.0 and v9.2.0 were removed.  If you compiled your own
+  application against librados shipped with these releases, it is very
+  likely referencing these removed symbols. So you will need to
+  recompile it.
+
+
+Notable Changes
+---------------
+* build/ops: Ceph daemon failed to start, because the service name was already used. (`issue#13474 <http://tracker.ceph.com/issues/13474>`_, `pr#6833 <http://github.com/ceph/ceph/pull/6833>`_, Chuanhong Wang)
+* build/ops: ceph upstart script rbdmap.conf incorrectly processes parameters (`issue#13214 <http://tracker.ceph.com/issues/13214>`_, `pr#6396 <http://github.com/ceph/ceph/pull/6396>`_, Sage Weil)
+* build/ops: libunwind package missing on CentOS 7 (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#6845 <http://github.com/ceph/ceph/pull/6845>`_, Loic Dachary)
+* build/ops: rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well (`issue#13785 <http://tracker.ceph.com/issues/13785>`_, `pr#6628 <http://github.com/ceph/ceph/pull/6628>`_, Loic Dachary)
+* build/ops: systemd/ceph-disk@.service assumes /bin/flock (`issue#13975 <http://tracker.ceph.com/issues/13975>`_, `pr#6852 <http://github.com/ceph/ceph/pull/6852>`_, Loic Dachary)
+* build/ops: systemd: no rbdmap systemd unit file (`issue#13374 <http://tracker.ceph.com/issues/13374>`_, `pr#6500 <http://github.com/ceph/ceph/pull/6500>`_, Boris Ranto)
+* common: auth/cephx: large amounts of log are produced by osd (`issue#13610 <http://tracker.ceph.com/issues/13610>`_, `pr#6836 <http://github.com/ceph/ceph/pull/6836>`_, Qiankun Zheng)
+* common: log: Log.cc: Assign LOG_DEBUG priority to syslog calls (`issue#13993 <http://tracker.ceph.com/issues/13993>`_, `pr#6993 <http://github.com/ceph/ceph/pull/6993>`_, Brad Hubbard)
+* crush: crash if we see CRUSH_ITEM_NONE in early rule step (`issue#13477 <http://tracker.ceph.com/issues/13477>`_, `pr#6626 <http://github.com/ceph/ceph/pull/6626>`_, Sage Weil)
+* fs: Ceph file system is not freeing space (`issue#13777 <http://tracker.ceph.com/issues/13777>`_, `pr#7431 <http://github.com/ceph/ceph/pull/7431>`_, Yan, Zheng, John Spray)
+* fs: Ceph-fuse won't start correctly when the option log_max_new in ceph.conf set to zero (`issue#13443 <http://tracker.ceph.com/issues/13443>`_, `pr#6395 <http://github.com/ceph/ceph/pull/6395>`_, Wenjun Huang)
+* fs: Segmentation fault accessing file using fuse mount (`issue#13714 <http://tracker.ceph.com/issues/13714>`_, `pr#6853 <http://github.com/ceph/ceph/pull/6853>`_, Yan, Zheng)
+* librbd: Avoid re-writing old-format image header on resize (`issue#13674 <http://tracker.ceph.com/issues/13674>`_, `pr#6630 <http://github.com/ceph/ceph/pull/6630>`_, Jason Dillaman)
+* librbd: ImageWatcher shouldn't block the notification thread (`issue#14373 <http://tracker.ceph.com/issues/14373>`_, `pr#7406 <http://github.com/ceph/ceph/pull/7406>`_, Jason Dillaman)
+* librbd: QEMU hangs after creating snapshot and stopping VM (`issue#13726 <http://tracker.ceph.com/issues/13726>`_, `pr#6632 <http://github.com/ceph/ceph/pull/6632>`_, Jason Dillaman)
+* librbd: Verify self-managed snapshot functionality on image create (`issue#13633 <http://tracker.ceph.com/issues/13633>`_, `pr#7080 <http://github.com/ceph/ceph/pull/7080>`_, Jason Dillaman)
+* librbd: [ FAILED ] TestLibRBD.SnapRemoveViaLockOwner (`issue#14164 <http://tracker.ceph.com/issues/14164>`_, `pr#7079 <http://github.com/ceph/ceph/pull/7079>`_, Jason Dillaman)
+* librbd: enable feature objectmap (`issue#13558 <http://tracker.ceph.com/issues/13558>`_, `pr#6477 <http://github.com/ceph/ceph/pull/6477>`_, xinxin shu)
+* librbd: fix merge-diff for >2GB diff-files (`issue#14030 <http://tracker.ceph.com/issues/14030>`_, `pr#6981 <http://github.com/ceph/ceph/pull/6981>`_, Jason Dillaman)
+* librbd: flattening an rbd image with active IO can lead to hang (`issue#14092 <http://tracker.ceph.com/issues/14092>`_, `issue#14483 <http://tracker.ceph.com/issues/14483>`_, `pr#7484 <http://github.com/ceph/ceph/pull/7484>`_, Jason Dillaman)
+* mds: fix client capabilities during reconnect (client.XXXX isn't responding to mclientcaps warning) (`issue#11482 <http://tracker.ceph.com/issues/11482>`_, `pr#6752 <http://github.com/ceph/ceph/pull/6752>`_, Yan, Zheng)
+* mon: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0 (`issue#13840 <http://tracker.ceph.com/issues/13840>`_, `pr#6907 <http://github.com/ceph/ceph/pull/6907>`_, Chengyuan Li)
+* mon: should not set isvalid = true when cephx_verify_authorizer retur... (`issue#13525 <http://tracker.ceph.com/issues/13525>`_, `pr#6392 <http://github.com/ceph/ceph/pull/6392>`_, Ruifeng Yang)
+* objecter: pool op callback may hang forever. (`issue#13642 <http://tracker.ceph.com/issues/13642>`_, `pr#6627 <http://github.com/ceph/ceph/pull/6627>`_, xie xingguo)
+* objecter: potential null pointer access when do pool_snap_list. (`issue#13639 <http://tracker.ceph.com/issues/13639>`_, `pr#6840 <http://github.com/ceph/ceph/pull/6840>`_, xie xingguo)
+* osd: FileStore: potential memory leak if getattrs fails. (`issue#13597 <http://tracker.ceph.com/issues/13597>`_, `pr#6846 <http://github.com/ceph/ceph/pull/6846>`_, xie xingguo)
+* osd: OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval. (`issue#13471 <http://tracker.ceph.com/issues/13471>`_, `pr#6397 <http://github.com/ceph/ceph/pull/6397>`_, xiexingguo)
+* osd: call on_new_interval on newly split child PG (`issue#13962 <http://tracker.ceph.com/issues/13962>`_, `pr#6849 <http://github.com/ceph/ceph/pull/6849>`_, Sage Weil)
+* osd: ceph-disk list fails on /dev/cciss!c0d0 (`issue#13970 <http://tracker.ceph.com/issues/13970>`_, `issue#14230 <http://tracker.ceph.com/issues/14230>`_, `pr#6880 <http://github.com/ceph/ceph/pull/6880>`_, Loic Dachary)
+* osd: ceph-disk: use blkid instead of sgdisk -i (`issue#14080 <http://tracker.ceph.com/issues/14080>`_, `pr#7001 <http://github.com/ceph/ceph/pull/7001>`_, Loic Dachary, Ilya Dryomov)
+* osd: fix race condition during send_failures (`issue#13821 <http://tracker.ceph.com/issues/13821>`_, `pr#6694 <http://github.com/ceph/ceph/pull/6694>`_, Sage Weil)
+* osd: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started) (`issue#14015 <http://tracker.ceph.com/issues/14015>`_, `pr#6851 <http://github.com/ceph/ceph/pull/6851>`_, David Zafman)
+* osd: pgs stuck inconsistent after infernalis upgrade (`issue#13862 <http://tracker.ceph.com/issues/13862>`_, `pr#7421 <http://github.com/ceph/ceph/pull/7421>`_, David Zafman)
+* rbd: TaskFinisher::cancel should remove event from SafeTimer (`issue#14476 <http://tracker.ceph.com/issues/14476>`_, `pr#7426 <http://github.com/ceph/ceph/pull/7426>`_, Douglas Fuller)
+* rbd: cls_rbd: object_map_save should enable checksums (`issue#14280 <http://tracker.ceph.com/issues/14280>`_, `pr#7428 <http://github.com/ceph/ceph/pull/7428>`_, Douglas Fuller)
+* rbd: misdirected op in rbd balance-reads test (`issue#13491 <http://tracker.ceph.com/issues/13491>`_, `pr#6629 <http://github.com/ceph/ceph/pull/6629>`_, Jason Dillaman)
+* rbd: pure virtual method called (`issue#13636 <http://tracker.ceph.com/issues/13636>`_, `pr#6633 <http://github.com/ceph/ceph/pull/6633>`_, Jason Dillaman)
+* rbd: rbd clone issue (`issue#13553 <http://tracker.ceph.com/issues/13553>`_, `pr#6474 <http://github.com/ceph/ceph/pull/6474>`_, xinxin shu)
+* rbd: rbd-replay does not check for EOF and goes to endless loop (`issue#14452 <http://tracker.ceph.com/issues/14452>`_, `pr#7427 <http://github.com/ceph/ceph/pull/7427>`_, Mykola Golub)
+* rbd: unknown argument --quiet in udevadm settle (`issue#13560 <http://tracker.ceph.com/issues/13560>`_, `pr#6634 <http://github.com/ceph/ceph/pull/6634>`_, Jason Dillaman)
+* rgw: init script reload doesn't work on EL7 (`issue#13709 <http://tracker.ceph.com/issues/13709>`_, `pr#6650 <http://github.com/ceph/ceph/pull/6650>`_, Hervé Rousseau)
+* rgw: radosgw-admin --help doesn't show the orphans find command (`issue#14516 <http://tracker.ceph.com/issues/14516>`_, `pr#7543 <http://github.com/ceph/ceph/pull/7543>`_, Yehuda Sadeh)
+* tests: ceph-disk workunit uses configobj (`issue#14004 <http://tracker.ceph.com/issues/14004>`_, `pr#6828 <http://github.com/ceph/ceph/pull/6828>`_, Loic Dachary)
+* tests: fsx failed to compile (`issue#14384 <http://tracker.ceph.com/issues/14384>`_, `pr#7429 <http://github.com/ceph/ceph/pull/7429>`_, Greg Farnum)
+* tests: notification slave needs to wait for master (`issue#13810 <http://tracker.ceph.com/issues/13810>`_, `pr#7225 <http://github.com/ceph/ceph/pull/7225>`_, Jason Dillaman)
+* tests: rebuild exclusive lock test should acquire exclusive lock (`issue#14121 <http://tracker.ceph.com/issues/14121>`_, `pr#7038 <http://github.com/ceph/ceph/pull/7038>`_, Jason Dillaman)
+* tests: testprofile must be removed before it is re-created (`issue#13664 <http://tracker.ceph.com/issues/13664>`_, `pr#6449 <http://github.com/ceph/ceph/pull/6449>`_, Loic Dachary)
+* tests: verify it is possible to reuse an OSD id (`issue#13988 <http://tracker.ceph.com/issues/13988>`_, `pr#6882 <http://github.com/ceph/ceph/pull/6882>`_, Loic Dachary)
+
+v9.2.0 Infernalis
+=================
+
+This major release will be the foundation for the next stable series.
+There have been some major changes since v0.94.x Hammer, and the
+upgrade process is non-trivial.  Please read these release notes carefully.
+
+Major Changes from Hammer
+-------------------------
+
+- *General*:
+
+  * Ceph daemons are now managed via systemd (with the exception of
+    Ubuntu Trusty, which still uses upstart).
+  * Ceph daemons run as 'ceph' user instead root.
+  * On Red Hat distros, there is also an SELinux policy.
+
+- *RADOS*:
+
+  * The RADOS cache tier can now proxy write operations to the base
+    tier, allowing writes to be handled without forcing migration of
+    an object into the cache.
+  * The SHEC erasure coding support is no longer flagged as
+    experimental. SHEC trades some additional storage space for faster
+    repair.
+  * There is now a unified queue (and thus prioritization) of client
+    IO, recovery, scrubbing, and snapshot trimming.
+  * There have been many improvements to low-level repair tooling
+    (ceph-objectstore-tool).
+  * The internal ObjectStore API has been significantly cleaned up in order
+    to facilitate new storage backends like NewStore.
+
+- *RGW*:
+
+  * The Swift API now supports object expiration.
+  * There are many Swift API compatibility improvements.
+
+- *RBD*:
+
+  * The ``rbd du`` command shows actual usage (quickly, when
+    object-map is enabled).
+  * The object-map feature has seen many stability improvements.
+  * Object-map and exclusive-lock features can be enabled or disabled
+    dynamically.
+  * You can now store user metadata and set persistent librbd options
+    associated with individual images.
+  * The new deep-flatten features allows flattening of a clone and all
+    of its snapshots.  (Previously snapshots could not be flattened.)
+  * The export-diff command is now faster (it uses aio).  There is also
+    a new fast-diff feature.
+  * The --size argument can be specified with a suffix for units
+    (e.g., ``--size 64G``).
+  * There is a new ``rbd status`` command that, for now, shows who has
+    the image open/mapped.
+
+- *CephFS*:
+
+  * You can now rename snapshots.
+  * There have been ongoing improvements around administration, diagnostics,
+    and the check and repair tools.
+  * The caching and revocation of client cache state due to unused
+    inodes has been dramatically improved.
+  * The ceph-fuse client behaves better on 32-bit hosts.
+
+Distro compatibility
+--------------------
+
+We have decided to drop support for many older distributions so that we can
+move to a newer compiler toolchain (e.g., C++11).  Although it is still possible
+to build Ceph on older distributions by installing backported development tools,
+we are not building and publishing release packages for ceph.com.
+
+We now build packages for:
+
+* CentOS 7 or later.  We have dropped support for CentOS 6 (and other
+  RHEL 6 derivatives, like Scientific Linux 6).
+* Debian Jessie 8.x or later.  Debian Wheezy 7.x's g++ has incomplete
+  support for C++11 (and no systemd).
+* Ubuntu Trusty 14.04 or later.  Ubuntu Precise 12.04 is no longer
+  supported.
+* Fedora 22 or later.
+
+Upgrading from Firefly
+----------------------
+
+Upgrading directly from Firefly v0.80.z is not recommended.  It is
+possible to do a direct upgrade, but not without downtime.  We
+recommend that clusters are first upgraded to Hammer v0.94.4 or a
+later v0.94.z release; only then is it possible to upgrade to
+Infernalis 9.2.z for an online upgrade (see below).
+
+To do an offline upgrade directly from Firefly, all Firefly OSDs must
+be stopped and marked down before any Infernalis OSDs will be allowed
+to start up.  This fencing is enforced by the Infernalis monitor, so
+use an upgrade procedure like:
+
+#. Upgrade Ceph on monitor hosts
+#. Restart all ceph-mon daemons
+#. Upgrade Ceph on all OSD hosts
+#. Stop all ceph-osd daemons
+#. Mark all OSDs down with something like::
+
+     ceph osd down `seq 0 1000`
+
+#. Start all ceph-osd daemons
+#. Upgrade and restart remaining daemons (ceph-mds, radosgw)
+
+Upgrading from Hammer
+---------------------
+
+* All cluster nodes must first upgrade to Hammer v0.94.4 or a later v0.94.z release; only
+  then is it possible to upgrade to Infernalis 9.2.z.
+
+* For all distributions that support systemd (CentOS 7, Fedora, Debian
+  Jessie 8.x, OpenSUSE), ceph daemons are now managed using native systemd
+  files instead of the legacy sysvinit scripts.  For example::
+
+    systemctl start ceph.target       # start all daemons
+    systemctl status ceph-osd@12      # check status of osd.12
+
+  The main notable distro that is *not* yet using systemd is Ubuntu trusty
+  14.04.  (The next Ubuntu LTS, 16.04, will use systemd instead of upstart.)
+
+* Ceph daemons now run as user and group ``ceph`` by default.  The
+  ceph user has a static UID assigned by Fedora and Debian (also used
+  by derivative distributions like RHEL/CentOS and Ubuntu).  On SUSE
+  the ceph user will currently get a dynamically assigned UID when the
+  user is created.
+
+  If your systems already have a ceph user, upgrading the package will cause
+  problems.  We suggest you first remove or rename the existing 'ceph' user
+  and 'ceph' group before upgrading.
+
+  When upgrading, administrators have two options:
+
+   #. Add the following line to ``ceph.conf`` on all hosts::
+
+        setuser match path = /var/lib/ceph/$type/$cluster-$id
+
+      This will make the Ceph daemons run as root (i.e., not drop
+      privileges and switch to user ceph) if the daemon's data
+      directory is still owned by root.  Newly deployed daemons will
+      be created with data owned by user ceph and will run with
+      reduced privileges, but upgraded daemons will continue to run as
+      root.
+
+   #. Fix the data ownership during the upgrade.  This is the
+      preferred option, but it is more work and can be very time
+      consuming.  The process for each host is to:
+
+   #. Upgrade the ceph package.  This creates the ceph user and group.  For
+      example::
+
+        ceph-deploy install --stable infernalis HOST
+
+      #. Stop the daemon(s)::
+
+           service ceph stop           # fedora, centos, rhel, debian
+           stop ceph-all               # ubuntu
+
+      #. Fix the ownership::
+
+           chown -R ceph:ceph /var/lib/ceph
+           chown -R ceph:ceph /var/log/ceph
+
+      #. Restart the daemon(s)::
+
+           start ceph-all                # ubuntu
+           systemctl start ceph.target   # debian, centos, fedora, rhel
+
+      Alternatively, the same process can be done with a single daemon
+      type, for example by stopping only monitors and chowning only
+      ``/var/lib/ceph/mon``.
+
+* The on-disk format for the experimental KeyValueStore OSD backend has
+  changed.  You will need to remove any OSDs using that backend before you
+  upgrade any test clusters that use it.
+
+* When a pool quota is reached, librados operations now block indefinitely,
+  the same way they do when the cluster fills up.  (Previously they would return
+  -ENOSPC).  By default, a full cluster or pool will now block.  If your
+  librados application can handle ENOSPC or EDQUOT errors gracefully, you can
+  get error returns instead by using the new librados OPERATION_FULL_TRY flag.
+
+* The return code for librbd's rbd_aio_read and Image::aio_read API methods no
+  longer returns the number of bytes read upon success.  Instead, it returns 0
+  upon success and a negative value upon failure.
+
+* 'ceph scrub', 'ceph compact' and 'ceph sync force are now DEPRECATED.  Users
+  should instead use 'ceph mon scrub', 'ceph mon compact' and
+  'ceph mon sync force'.
+
+* 'ceph mon_metadata' should now be used as 'ceph mon metadata'. There is no
+  need to deprecate this command (same major release since it was first
+  introduced).
+
+* The `--dump-json` option of "osdmaptool" is replaced by `--dump json`.
+
+* The commands of "pg ls-by-{pool,primary,osd}" and "pg ls" now take "recovering"
+  instead of "recovery", to include the recovering pgs in the listed pgs.
+
+Notable Changes since Hammer
+----------------------------
+
+* aarch64: add optimized version of crc32c (Yazen Ghannam, Steve Capper)
+* auth: cache/reuse crypto lib key objects, optimize msg signature check (Sage Weil)
+* auth: reinit NSS after fork() (#11128 Yan, Zheng)
+* autotools: fix out of tree build (Krxysztof Kosinski)
+* autotools: improve make check output (Loic Dachary)
+* buffer: add invalidate_crc() (Piotr Dalek)
+* buffer: fix zero bug (#12252 Haomai Wang)
+* buffer: some cleanup (Michal Jarzabek)
+* build: allow tcmalloc-minimal (Thorsten Behrens)
+* build: C++11 now supported
+* build: cmake: fix nss linking (Danny Al-Gaaf)
+* build: cmake: misc fixes (Orit Wasserman, Casey Bodley)
+* build: disable LTTNG by default (#11333 Josh Durgin)
+* build: do not build ceph-dencoder with tcmalloc (#10691 Boris Ranto)
+* build: fix junit detection on Fedora 22 (Ira Cooper)
+* build: fix pg ref disabling (William A. Kennington III)
+* build: fix ppc build (James Page)
+* build: install-deps: misc fixes (Loic Dachary)
+* build: install-deps.sh improvements (Loic Dachary)
+* build: install-deps: support OpenSUSE (Loic Dachary)
+* build: make_dist_tarball.sh (Sage Weil)
+* build: many cmake improvements
+* build: misc cmake fixes (Matt Benjamin)
+* build: misc fixes (Boris Ranto, Ken Dreyer, Owen Synge)
+* build: OSX build fixes (Yan, Zheng)
+* build: remove rest-bench
+* ceph-authtool: fix return code on error (Gerhard Muntingh)
+* ceph-detect-init: added Linux Mint (Michal Jarzabek)
+* ceph-detect-init: robust init system detection (Owen Synge)
+* ceph-disk: ensure 'zap' only operates on a full disk (#11272 Loic Dachary)
+* ceph-disk: fix zap sgdisk invocation (Owen Synge, Thorsten Behrens)
+* ceph-disk: follow ceph-osd hints when creating journal (#9580 Sage Weil)
+* ceph-disk: handle re-using existing partition (#10987 Loic Dachary)
+* ceph-disk: improve parted output parsing (#10983 Loic Dachary)
+* ceph-disk: install pip > 6.1 (#11952 Loic Dachary)
+* ceph-disk: make suppression work for activate-all and activate-journal (Dan van der Ster)
+* ceph-disk: many fixes (Loic Dachary, Alfredo Deza)
+* ceph-disk: fixes to respect init system (Loic Dachary, Owen Synge)
+* ceph-disk: pass --cluster arg on prepare subcommand (Kefu Chai)
+* ceph-disk: support for multipath devices (Loic Dachary)
+* ceph-disk: support NVMe device partitions (#11612 Ilja Slepnev)
+* ceph: fix 'df' units (Zhe Zhang)
+* ceph: fix parsing in interactive cli mode (#11279 Kefu Chai)
+* cephfs-data-scan: many additions, improvements (John Spray)
+* ceph-fuse: do not require successful remount when unmounting (#10982 Greg Farnum)
+* ceph-fuse, libcephfs: don't clear COMPLETE when trimming null (Yan, Zheng)
+* ceph-fuse, libcephfs: drop inode when rmdir finishes (#11339 Yan, Zheng)
+* ceph-fuse,libcephfs: fix uninline (#11356 Yan, Zheng)
+* ceph-fuse, libcephfs: hold exclusive caps on dirs we "own" (#11226 Greg Farnum)
+* ceph-fuse: mostly behave on 32-bit hosts (Yan, Zheng)
+* ceph: improve error output for 'tell' (#11101 Kefu Chai)
+* ceph-monstore-tool: fix store-copy (Huangjun)
+* ceph: new 'ceph daemonperf' command (John Spray, Mykola Golub)
+* ceph-objectstore-tool: many many improvements (David Zafman)
+* ceph-objectstore-tool: refactoring and cleanup (John Spray)
+* ceph-post-file: misc fixes (Joey McDonald, Sage Weil)
+* ceph_test_rados: test pipelined reads (Zhiqiang Wang)
+* client: avoid sending unnecessary FLUSHSNAP messages (Yan, Zheng)
+* client: exclude setfilelock when calculating oldest tid (Yan, Zheng)
+* client: fix error handling in check_pool_perm (John Spray)
+* client: fsync waits only for inode's caps to flush (Yan, Zheng)
+* client: invalidate kernel dcache when cache size exceeds limits (Yan, Zheng)
+* client: make fsync wait for unsafe dir operations (Yan, Zheng)
+* client: pin lookup dentry to avoid inode being freed (Yan, Zheng)
+* common: add descriptions to perfcounters (Kiseleva Alyona)
+* common: add perf counter descriptions (Alyona Kiseleva)
+* common: bufferlist performance tuning (Piotr Dalek, Sage Weil)
+* common: detect overflow of int config values (#11484 Kefu Chai)
+* common: fix bit_vector extent calc (#12611 Jason Dillaman)
+* common: fix json parsing of utf8 (#7387 Tim Serong)
+* common: fix leak of pthread_mutexattr (#11762 Ketor Meng)
+* common: fix LTTNG vs fork issue (Josh Durgin)
+* common: fix throttle max change (Henry Chang)
+* common: make mutex more efficient
+* common: make work queue addition/removal thread safe (#12662 Jason Dillaman)
+* common: optracker improvements (Zhiqiang Wang, Jianpeng Ma)
+* common: PriorityQueue tests (Kefu Chai)
+* common: some async compression infrastructure (Haomai Wang)
+* crush: add --check to validate dangling names, max osd id (Kefu Chai)
+* crush: cleanup, sync with kernel (Ilya Dryomov)
+* crush: fix crash from invalid 'take' argument (#11602 Shiva Rkreddy, Sage Weil)
+* crush: fix divide-by-2 in straw2 (#11357 Yann Dupont, Sage Weil)
+* crush: fix has_v4_buckets (#11364 Sage Weil)
+* crush: fix subtree base weight on adjust_subtree_weight (#11855 Sage Weil)
+* crush: respect default replicated ruleset config on map creation (Ilya Dryomov)
+* crushtool: fix order of operations, usage (Sage Weil)
+* crypto: fix NSS leak (Jason Dillaman)
+* crypto: fix unbalanced init/shutdown (#12598 Zheng Yan)
+* deb: fix rest-bench-dbg and ceph-test-dbg dependendies (Ken Dreyer)
+* debian: minor package reorg (Ken Dreyer)
+* deb, rpm: move ceph-objectstore-tool to ceph (Ken Dreyer)
+* doc: docuemnt object corpus generation (#11099 Alexis Normand)
+* doc: document region hostnames (Robin H. Johnson)
+* doc: fix gender neutrality (Alexandre Maragone)
+* doc: fix install doc (#10957 Kefu Chai)
+* doc: fix sphinx issues (Kefu Chai)
+* doc: man page updates (Kefu Chai)
+* doc: mds data structure docs (Yan, Zheng)
+* doc: misc updates (Francois Lafont, Ken Dreyer, Kefu Chai, Owen Synge, Gael Fenet-Garde, Loic Dachary, Yannick Atchy-Dalama, Jiaying Ren, Kevin Caradant, Robert Maxime, Nicolas Yong, Germain Chipaux, Arthur Gorjux, Gabriel Sentucq, Clement Lebrun, Jean-Remi Deveaux, Clair Massot, Robin Tang, Thomas Laumondais, Jordan Dorne, Yuan Zhou, Valentin Thomas, Pierre Chaumont, Benjamin Troquereau, Benjamin Sesia, Vikhyat Umrao, Nilamdyuti Goswami, Vartika Rai, Florian Haas, Loic Dachary, Simon Guinot, Andy Allan, Alistair Israel, Ken Dreyer, Robin Rehu, Lee Revell, Florian Marsylle, Thomas Johnson, Bosse Klykken, Travis Rhoden, Ian Kelling)
+* doc: swift tempurls (#10184 Abhishek Lekshmanan)
+* doc: switch doxygen integration back to breathe (#6115 Kefu Chai)
+* doc: update release schedule docs (Loic Dachary)
+* erasure-code: cleanup (Kefu Chai)
+* erasure-code: improve tests (Loic Dachary)
+* erasure-code: shec: fix recovery bugs (Takanori Nakao, Shotaro Kawaguchi)
+* erasure-code: update ISA-L to 2.13 (Yuan Zhou)
+* gmock: switch to submodule (Danny Al-Gaaf, Loic Dachary)
+* hadoop: add terasort test (Noah Watkins)
+* init-radosgw: merge with sysv version; fix enumeration (Sage Weil)
+* java: fix libcephfs bindings (Noah Watkins)
+* libcephfs: add pread, pwrite (Jevon Qiao)
+* libcephfs,ceph-fuse: cache cleanup (Zheng Yan)
+* libcephfs,ceph-fuse: fix request resend on cap reconnect (#10912 Yan, Zheng)
+* librados: add config observer (Alistair Strachan)
+* librados: add FULL_TRY and FULL_FORCE flags for dealing with full clusters or pools (Sage Weil)
+* librados: add src_fadvise_flags for copy-from (Jianpeng Ma)
+* librados: define C++ flags from C constants (Josh Durgin)
+* librados: fadvise flags per op (Jianpeng Ma)
+* librados: fix last_force_resent handling (#11026 Jianpeng Ma)
+* librados: fix memory leak from C_TwoContexts (Xiong Yiliang)
+* librados: fix notify completion race (#13114 Sage Weil)
+* librados: fix striper when stripe_count = 1 and stripe_unit != object_size (#11120 Yan, Zheng)
+* librados, libcephfs: randomize client nonces (Josh Durgin)
+* librados: op perf counters (John Spray)
+* librados: pybind: fix binary omap values (Robin H. Johnson)
+* librados: pybind: fix write() method return code (Javier Guerra)
+* librados: respect default_crush_ruleset on pool_create (#11640 Yuan Zhou)
+* libradosstriper: fix leak (Danny Al-Gaaf)
+* librbd: add const for single-client-only features (Josh Durgin)
+* librbd: add deep-flatten operation (Jason Dillaman)
+* librbd: add purge_on_error cache behavior (Jianpeng Ma)
+* librbd: allow additional metadata to be stored with the image (Haomai Wang)
+* librbd: avoid blocking aio API methods (#11056 Jason Dillaman)
+* librbd: better handling for dup flatten requests (#11370 Jason Dillaman)
+* librbd: cancel in-flight ops on watch error (#11363 Jason Dillaman)
+* librbd: default new images to format 2 (#11348 Jason Dillaman)
+* librbd: fadvise for copy, export, import (Jianpeng Ma)
+* librbd: fast diff implementation that leverages object map (Jason Dillaman)
+* librbd: fix fast diff bugs (#11553 Jason Dillaman)
+* librbd: fix image format detection (Zhiqiang Wang)
+* librbd: fix lock ordering issue (#11577 Jason Dillaman)
+* librbd: fix reads larger than the cache size (Lu Shi)
+* librbd: fix snapshot creation when other snap is active (#11475 Jason Dillaman)
+* librbd: flatten/copyup fixes (Jason Dillaman)
+* librbd: handle NOCACHE fadvise flag (Jinapeng Ma)
+* librbd: lockdep, helgrind validation (Jason Dillaman, Josh Durgin)
+* librbd: metadata filter fixes (Haomai Wang)
+* librbd: misc aio fixes (#5488 Jason Dillaman)
+* librbd: misc rbd fixes (#11478 #11113 #11342 #11380 Jason Dillaman, Zhiqiang Wang)
+* librbd: new diff_iterate2 API (Jason Dillaman)
+* librbd: object map rebuild support (Jason Dillaman)
+* librbd: only update image flags while hold exclusive lock (#11791 Jason Dillaman)
+* librbd: optionally disable allocation hint (Haomai Wang)
+* librbd: prevent race between resize requests (#12664 Jason Dillaman)
+* librbd: readahead fixes (Zhiqiang Wang)
+* librbd: return result code from close (#12069 Jason Dillaman)
+* librbd: store metadata, including config options, in image (Haomai Wang)
+* librbd: tolerate old osds when getting image metadata (#11549 Jason Dillaman)
+* librbd: use write_full when possible (Zhiqiang Wang)
+* log: fix data corruption race resulting from log rotation (#12465 Samuel Just)
+* logrotate.d: prefer service over invoke-rc.d (#11330 Win Hierman, Sage Weil)
+* mds: add 'damaged' state to MDSMap (John Spray)
+* mds: add nicknames for perfcounters (John Spray)
+* mds: avoid emitting cap warnigns before evicting session (John Spray)
+* mds: avoid getting stuck in XLOCKDONE (#11254 Yan, Zheng)
+* mds: disable problematic rstat propagation into snap parents (Yan, Zheng)
+* mds: do not add snapped items to bloom filter (Yan, Zheng)
+* mds: expose frags via asok (John Spray)
+* mds: fix expected holes in journal objects (#13167 Yan, Zheng)
+* mds: fix handling for missing mydir dirfrag (#11641 John Spray)
+* mds: fix integer truncateion on large client ids (Henry Chang)
+* mds: fix mydir replica issue with shutdown (#10743 John Spray)
+* mds: fix out-of-order messages (#11258 Yan, Zheng)
+* mds: fix rejoin (Yan, Zheng)
+* mds: fix setting entire file layout in one setxattr (John Spray)
+* mds: fix shutdown (John Spray)
+* mds: fix shutdown with strays (#10744 John Spray)
+* mds: fix SnapServer crash on deleted pool (John Spray)
+* mds: fix snapshot bugs (Yan, Zheng)
+* mds: fix stray reintegration (Yan, Zheng)
+* mds: fix stray handling (John Spray)
+* mds: fix suicide beacon (John Spray)
+* mds: flush immediately in do_open_truncate (#11011 John Spray)
+* mds: handle misc corruption issues (John Spray)
+* mds: improve dump methods (John Spray)
+* mds: many fixes (Yan, Zheng, John Spray, Greg Farnum)
+* mds: many snapshot and stray fixes (Yan, Zheng)
+* mds: misc fixes (Jianpeng Ma, Dan van der Ster, Zhang Zhi)
+* mds: misc journal cleanups and fixes (#10368 John Spray)
+* mds: misc repair improvements (John Spray)
+* mds: misc snap fixes (Zheng Yan)
+* mds: misc snapshot fixes (Yan, Zheng)
+* mds: new SessionMap storage using omap (#10649 John Spray)
+* mds: persist completed_requests reliably (#11048 John Spray)
+* mds: reduce memory consumption (Yan, Zheng)
+* mds: respawn instead of suicide on blacklist (John Spray)
+* mds: separate safe_pos in Journaler (#10368 John Spray)
+* mds: snapshot rename support (#3645 Yan, Zheng)
+* mds: store layout on header object (#4161 John Spray)
+* mds: throttle purge stray operations (#10390 John Spray)
+* mds: tolerate clock jumping backwards (#11053 Yan, Zheng)
+* mds: warn when clients fail to advance oldest_client_tid (#10657 Yan, Zheng)
+* misc cleanups and fixes (Danny Al-Gaaf)
+* misc coverity fixes (Danny Al-Gaaf)
+* misc performance and cleanup (Nathan Cutler, Xinxin Shu)
+* mon: add cache over MonitorDBStore (Kefu Chai)
+* mon: add 'mon_metadata <id>' command (Kefu Chai)
+* mon: add 'node ls ...' command (Kefu Chai)
+* mon: add NOFORWARD, OBSOLETE, DEPRECATE flags for mon commands (Joao Eduardo Luis)
+* mon: add PG count to 'ceph osd df' output (Michal Jarzabek)
+* mon: 'ceph osd metadata' can dump all osds (Haomai Wang)
+* mon: clean up, reorg some mon commands (Joao Eduardo Luis)
+* monclient: flush_log (John Spray)
+* mon: detect kv backend failures (Sage Weil)
+* mon: disallow >2 tiers (#11840 Kefu Chai)
+* mon: disallow ec pools as tiers (#11650 Samuel Just)
+* mon: do not deactivate last mds (#10862 John Spray)
+* mon: fix average utilization calc for 'osd df' (Mykola Golub)
+* mon: fix CRUSH map test for new pools (Sage Weil)
+* mon: fix log dump crash when debugging (Mykola Golub)
+* mon: fix mds beacon replies (#11590 Kefu Chai)
+* mon: fix metadata update race (Mykola Golub)
+* mon: fix min_last_epoch_clean tracking (Kefu Chai)
+* mon: fix 'pg ls' sort order, state names (#11569 Kefu Chai)
+* mon: fix refresh (#11470 Joao Eduardo Luis)
+* mon: fix variance calc in 'osd df' (Sage Weil)
+* mon: improve callout to crushtool (Mykola Golub)
+* mon: make blocked op messages more readable (Jianpeng Ma)
+* mon: make osd get pool 'all' only return applicable fields (#10891 Michal Jarzabek)
+* mon: misc scaling fixes (Sage Weil)
+* mon: normalize erasure-code profile for storage and comparison (Loic Dachary)
+* mon: only send mon metadata to supporting peers (Sage Weil)
+* mon: optionally specify osd id on 'osd create' (Mykola Golub)
+* mon: 'osd tree' fixes (Kefu Chai)
+* mon: periodic background scrub (Joao Eduardo Luis)
+* mon: prevent bucket deletion when referenced by a crush rule (#11602 Sage Weil)
+* mon: prevent pgp_num > pg_num (#12025 Xinxin Shu)
+* mon: prevent pool with snapshot state from being used as a tier (#11493 Sage Weil)
+* mon: prime pg_temp when CRUSH map changes (Sage Weil)
+* mon: refine check_remove_tier checks (#11504 John Spray)
+* mon: reject large max_mds values (#12222 John Spray)
+* mon: remove spurious who arg from 'mds rm ...' (John Spray)
+* mon: streamline session handling, fix memory leaks (Sage Weil)
+* mon: upgrades must pass through hammer (Sage Weil)
+* mon: warn on bogus cache tier config (Jianpeng Ma)
+* msgr: add ceph_perf_msgr tool (Hoamai Wang)
+* msgr: async: fix seq handling (Haomai Wang)
+* msgr: async: many many fixes (Haomai Wang)
+* msgr: simple: fix clear_pipe (#11381 Haomai Wang)
+* msgr: simple: fix connect_seq assert (Haomai Wang)
+* msgr: xio: fastpath improvements (Raju Kurunkad)
+* msgr: xio: fix ip and nonce (Raju Kurunkad)
+* msgr: xio: improve lane assignment (Vu Pham)
+* msgr: xio: sync with accellio v1.4 (Vu Pham)
+* msgr: xio: misc fixes (#10735 Matt Benjamin, Kefu Chai, Danny Al-Gaaf, Raju Kurunkad, Vu Pham, Casey Bodley)
+* msg: unit tests (Haomai Wang)
+* objectcacher: misc bug fixes (Jianpeng Ma)
+* osd: add latency perf counters for tier operations (Xinze Chi)
+* osd: add misc perfcounters (Xinze Chi)
+* osd: add simple sleep injection in recovery (Sage Weil)
+* osd: allow SEEK_HOLE/SEEK_DATA for sparse read (Zhiqiang Wang)
+* osd: avoid dup omap sets for in pg metadata (Sage Weil)
+* osd: avoid multiple hit set insertions (Zhiqiang Wang)
+* osd: avoid transaction append in some cases (Sage Weil)
+* osd: break PG removal into multiple iterations (#10198 Guang Yang)
+* osd: cache proxy-write support (Zhiqiang Wang, Samuel Just)
+* osd: check scrub state when handling map (Jianpeng Ma)
+* osd: clean up some constness, privateness (Kefu Chai)
+* osd: clean up temp object if promotion fails (Jianpeng Ma)
+* osd: configure promotion based on write recency (Zhiqiang Wang)
+* osd: constrain collections to meta and PGs (normal and temp) (Sage Weil)
+* osd: don't send dup MMonGetOSDMap requests (Sage Weil, Kefu Chai)
+* osd: EIO injection (David Zhang)
+* osd: elminiate txn apend, ECSubWrite copy (Samuel Just)
+* osd: erasure-code: drop entries according to LRU (Andreas-Joachim Peters)
+* osd: erasure-code: fix SHEC floating point bug (#12936 Loic Dachary)
+* osd: erasure-code: update to ISA-L 2.14 (Yuan Zhou)
+* osd: filejournal: cleanup (David Zafman)
+* osd: filestore: clone using splice (Jianpeng Ma)
+* osd: filestore: fix recursive lock (Xinxin Shu)
+* osd: fix check_for_full (Henry Chang)
+* osd: fix dirty accounting in make_writeable (Zhiqiang Wang)
+* osd: fix dup promotion lost op bug (Zhiqiang Wang)
+* osd: fix endless repair when object is unrecoverable (Jianpeng Ma, Kefu Chai)
+* osd: fix hitset object naming to use GMT (Kefu Chai)
+* osd: fix misc memory leaks (Sage Weil)
+* osd: fix negative degraded stats during backfill (Guang Yang)
+* osd: fix osdmap dump of blacklist items (John Spray)
+* osd: fix peek_queue locking in FileStore (Xinze Chi)
+* osd: fix pg resurrection (#11429 Samuel Just)
+* osd: fix promotion vs full cache tier (Samuel Just)
+* osd: fix replay requeue when pg is still activating (#13116 Samuel Just)
+* osd: fix scrub stat bugs (Sage Weil, Samuel Just)
+* osd: fix snap flushing from cache tier (again) (#11787 Samuel Just)
+* osd: fix snap handling on promotion (#11296 Sam Just)
+* osd: fix temp-clearing (David Zafman)
+* osd: force promotion for ops EC can't handle (Zhiqiang Wang)
+* osd: handle log split with overlapping entries (#11358 Samuel Just)
+* osd: ignore non-existent osds in unfound calc (#10976 Mykola Golub)
+* osd: improve behavior on machines with large memory pages (Steve Capper)
+* osd: include a temp namespace within each collection/pgid (Sage Weil)
+* osd: increase default max open files (Owen Synge)
+* osd: keyvaluestore: misc fixes (Varada Kari)
+* osd: low and high speed flush modes (Mingxin Liu)
+* osd: make suicide timeouts individually configurable (Samuel Just)
+* osd: merge multiple setattr calls into a setattrs call (Xinxin Shu)
+* osd: misc fixes (Ning Yao, Kefu Chai, Xinze Chi, Zhiqiang Wang, Jianpeng Ma)
+* osd: move scrub in OpWQ (Samuel Just)
+* osd: newstore prototype (Sage Weil)
+* osd: ObjectStore internal API refactor (Sage Weil)
+* osd: peer_features includes self (David Zafman)
+* osd: pool size change triggers new interval (#11771 Samuel Just)
+* osd: prepopulate needs_recovery_map when only one peer has missing (#9558 Guang Yang)
+* osd: randomize scrub times (#10973 Kefu Chai)
+* osd: recovery, peering fixes (#11687 Samuel Just)
+* osd: refactor scrub and digest recording (Sage Weil)
+* osd: refuse first write to EC object at non-zero offset (Jianpeng Ma)
+* osd: relax reply order on proxy read (#11211 Zhiqiang Wang)
+* osd: require firefly features (David Zafman)
+* osd: set initial crush weight with more precision (Sage Weil)
+* osd: SHEC no longer experimental
+* osd: skip promotion for flush/evict op (Zhiqiang Wang)
+* osd: stripe over small xattrs to fit in XFS's 255 byte inline limit (Sage Weil, Ning Yao)
+* osd: sync object_map on syncfs (Samuel Just)
+* osd: take excl lock of op is rw (Samuel Just)
+* osd: throttle evict ops (Yunchuan Wen)
+* osd: upgrades must pass through hammer (Sage Weil)
+* osd: use a temp object for recovery (Sage Weil)
+* osd: use blkid to collection partition information (Joseph Handzik)
+* osd: use SEEK_HOLE / SEEK_DATA for sparse copy (Xinxin Shu)
+* osd: WBThrottle cleanups (Jianpeng Ma)
+* osd: write journal header on clean shutdown (Xinze Chi)
+* osdc/Objecter: allow per-pool calls to op_cancel_writes (John Spray)
+* os/filestore: enlarge getxattr buffer size (Jianpeng Ma)
+* pybind: pep8 cleanups (Danny Al-Gaaf)
+* pycephfs: many fixes for bindings (Haomai Wang)
+* qa: fix filelock_interrupt.py test (Yan, Zheng)
+* qa: improve ceph-disk tests (Loic Dachary)
+* qa: improve docker build layers (Loic Dachary)
+* qa: run-make-check.sh script (Loic Dachary)
+* rados: add --striper option to use libradosstriper (#10759 Sebastien Ponce)
+* rados: bench: add --no-verify option to improve performance (Piotr Dalek)
+* rados bench: misc fixes (Dmitry Yatsushkevich)
+* rados: fix error message on failed pool removal (Wido den Hollander)
+* radosgw-admin: add 'bucket check' function to repair bucket index (Yehuda Sadeh)
+* radosgw-admin: fix subuser modify output (#12286 Guce)
+* rados: handle --snapid arg properly (Abhishek Lekshmanan)
+* rados: improve bench buffer handling, performance (Piotr Dalek)
+* rados: misc bench fixes (Dmitry Yatsushkevich)
+* rados: new pool import implementation (John Spray)
+* rados: translate errno to string in CLI (#10877 Kefu Chai)
+* rbd: accept map options config option (Ilya Dryomov)
+* rbd: add disk usage tool (#7746 Jason Dillaman)
+* rbd: allow unmapping by spec (Ilya Dryomov)
+* rbd: cli: fix arg parsing with --io-pattern (Dmitry Yatsushkevich)
+* rbd: deprecate --new-format option (Jason Dillman)
+* rbd: fix error messages (#2862 Rajesh Nambiar)
+* rbd: fix link issues (Jason Dillaman)
+* rbd: improve CLI arg parsing, usage (Ilya Dryomov)
+* rbd: rbd-replay-prep and rbd-replay improvements (Jason Dillaman)
+* rbd: recognize queue_depth kernel option (Ilya Dryomov)
+* rbd: support G and T units for CLI (Abhishek Lekshmanan)
+* rbd: update rbd man page (Ilya Dryomov)
+* rbd: update xfstests tests (Douglas Fuller)
+* rbd: use image-spec and snap-spec in help (Vikhyat Umrao, Ilya Dryomov)
+* rest-bench: misc fixes (Shawn Chen)
+* rest-bench: support https (#3968 Yuan Zhou)
+* rgw: add max multipart upload parts (#12146 Abshishek Dixit)
+* rgw: add missing headers to Swift container details (#10666 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: add stats to headers for account GET (#10684 Yuan Zhou)
+* rgw: add Trasnaction-Id to response (Abhishek Dixit)
+* rgw: add X-Timestamp for Swift containers (#10938 Radoslaw Zarzynski)
+* rgw: always check if token is expired (#11367 Anton Aksola, Riku Lehto)
+* rgw: conversion tool to repair broken multipart objects (#12079 Yehuda Sadeh)
+* rgw: document layout of pools and objects (Pete Zaitcev)
+* rgw: do not enclose bucket header in quotes (#11860 Wido den Hollander)
+* rgw: do not prefetch data for HEAD requests (Guang Yang)
+* rgw: do not preserve ACLs when copying object (#12370 Yehuda Sadeh)
+* rgw: do not set content-type if length is 0 (#11091 Orit Wasserman)
+* rgw: don't clobber bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: don't use end_marker for namespaced object listing (#11437 Yehuda Sadeh)
+* rgw: don't use rgw_socket_path if frontend is configured (#11160 Yehuda Sadeh)
+* rgw: enforce Content-Length for POST on Swift cont/obj (#10661 Radoslaw Zarzynski)
+* rgw: error out if frontend did not send all data (#11851 Yehuda Sadeh)
+* rgw: expose the number of unhealthy workers through admin socket (Guang Yang)
+* rgw: fail if parts not specified on multipart upload (#11435 Yehuda Sadeh)
+* rgw: fix assignment of copy obj attributes (#11563 Yehuda Sadeh)
+* rgw: fix broken stats in container listing (#11285 Radoslaw Zarzynski)
+* rgw: fix bug in domain/subdomain splitting (Robin H. Johnson)
+* rgw: fix casing of Content-Type header (Robin H. Johnson)
+* rgw: fix civetweb max threads (#10243 Yehuda Sadeh)
+* rgw: fix Connection: header handling (#12298 Wido den Hollander)
+* rgw: fix copy metadata, support X-Copied-From for swift (#10663 Radoslaw Zarzynski)
+* rgw: fix data corruptions race condition (#11749 Wuxingyi)
+* rgw: fix decoding of X-Object-Manifest from GET on Swift DLO (Radslow Rzarzynski)
+* rgw: fix GET on swift account when limit == 0 (#10683 Radoslaw Zarzynski)
+* rgw: fix handling empty metadata items on Swift container (#11088 Radoslaw Zarzynski)
+* rgw: fix JSON response when getting user quota (#12117 Wuxingyi)
+* rgw: fix locator for objects starting with _ (#11442 Yehuda Sadeh)
+* rgw: fix log rotation (Wuxingyi)
+* rgw: fix mulitipart upload in retry path (#11604 Yehuda Sadeh)
+* rgw: fix quota enforcement on POST (#11323 Sergey Arkhipov)
+* rgw: fix reset_loc (#11974 Yehuda Sadeh)
+* rgw: fix return code on missing upload (#11436 Yehuda Sadeh)
+* rgw: fix sysvinit script
+* rgw: fix sysvinit script w/ multiple instances (Sage Weil, Pavan Rallabhandi)
+* rgw: force content_type for swift bucket stats requests (#12095 Orit Wasserman)
+* rgw: force content type header on responses with no body (#11438 Orit Wasserman)
+* rgw: generate Date header for civetweb (#10873 Radoslaw Zarzynski)
+* rgw: generate new object tag when setting attrs (#11256 Yehuda Sadeh)
+* rgw: improve content-length env var handling (#11419 Robin H. Johnson)
+* rgw: improved support for swift account metadata (Radoslaw Zarzynski)
+* rgw: improve handling of already removed buckets in expirer (Radoslaw Rzarzynski)
+* rgw: issue aio for first chunk before flush cached data (#11322 Guang Yang)
+* rgw: log to /var/log/ceph instead of /var/log/radosgw
+* rgw: make init script wait for radosgw to stop (#11140 Dmitry Yatsushkevich)
+* rgw: make max put size configurable (#6999 Yuan Zhou)
+* rgw: make quota/gc threads configurable (#11047 Guang Yang)
+* rgw: make read user buckets backward compat (#10683 Radoslaw Zarzynski)
+* rgw: merge manifests properly with prefix override (#11622 Yehuda Sadeh)
+* rgw: only scan for objects not in a namespace (#11984 Yehuda Sadeh)
+* rgw: orphan detection tool (Yehuda Sadeh)
+* rgw: pass in civetweb configurables (#10907 Yehuda Sadeh)
+* rgw: rectify 202 Accepted in PUT response (#11148 Radoslaw Zarzynski)
+* rgw: remove meta file after deleting bucket (#11149 Orit Wasserman)
+* rgw: remove trailing :port from HTTP_HOST header (Sage Weil)
+* rgw: return 412 on bad limit when listing buckets (#11613 Yehuda Sadeh)
+* rgw: rework X-Trans-Id header to conform with Swift API (Radoslaw Rzarzynski)
+* rgw: s3 encoding-type for get bucket (Jeff Weber)
+* rgw: send ETag, Last-Modified for swift (#11087 Radoslaw Zarzynski)
+* rgw: set content length on container GET, PUT, DELETE, HEAD (#10971, #11036 Radoslaw Zarzynski)
+* rgw: set max buckets per user in ceph.conf (Vikhyat Umrao)
+* rgw: shard work over multiple librados instances (Pavan Rallabhandi)
+* rgw: support end marker on swift container GET (#10682 Radoslaw Zarzynski)
+* rgw: support for Swift expiration API (Radoslaw Rzarzynski, Yehuda Sadeh)
+* rgw: swift: allow setting attributes with COPY (#10662 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: swift: do not override sent content type (#12363 Orit Wasserman)
+* rgw: swift: enforce Content-Type in response (#12157 Radoslaw Zarzynski)
+* rgw: swift: fix account listing (#11501 Radoslaw Zarzynski)
+* rgw: swift: fix metadata handling on copy (#10645 Radoslaw Zarzynski)
+* rgw: swift: send Last-Modified header (#10650 Radoslaw Zarzynski)
+* rgw: swift: set Content-Length for account GET (#12158 Radoslav Zarzynski)
+* rgw: swift: set content-length on keystone tokens (#11473 Herv Rousseau)
+* rgw: update keystone cache with token info (#11125 Yehuda Sadeh)
+* rgw: update to latest civetweb, enable config for IPv6 (#10965 Yehuda Sadeh)
+* rgw: use attrs from source bucket on copy (#11639 Javier M. Mellid)
+* rgw: use correct oid for gc chains (#11447 Yehuda Sadeh)
+* rgw: user rm is idempotent (Orit Wasserman)
+* rgw: use unique request id for civetweb (#10295 Orit Wasserman)
+* rocksdb: add perf counters for get/put latency (Xinxin Shu)
+* rocksdb, leveldb: fix compact_on_mount (Xiaoxi Chen)
+* rocksdb: pass options as single string (Xiaoxi Chen)
+* rocksdb: update to latest (Xiaoxi Chen)
+* rpm: add suse firewall files (Tim Serong)
+* rpm: always rebuild and install man pages for rpm (Owen Synge)
+* rpm: loosen ceph-test dependencies (Ken Dreyer)
+* rpm: many spec file fixes (Owen Synge, Ken Dreyer)
+* rpm: misc fixes (Boris Ranto, Owen Synge, Ken Dreyer, Ira Cooper)
+* rpm: misc systemd and SUSE fixes (Owen Synge, Nathan Cutler)
+* selinux policy (Boris Ranto, Milan Broz)
+* systemd: logrotate fixes (Tim Serong, Lars Marowsky-Bree, Nathan Cutler)
+* systemd: many fixes (Sage Weil, Owen Synge, Boris Ranto, Dan van der Ster)
+* systemd: run daemons as user ceph
+* sysvinit compat: misc fixes (Owen Synge)
+* test: misc fs test improvements (John Spray, Loic Dachary)
+* test: python tests, linter cleanup (Alfredo Deza)
+* tests: fixes for rbd xstests (Douglas Fuller)
+* tests: fix tiering health checks (Loic Dachary)
+* tests for low-level performance (Haomai Wang)
+* tests: many ec non-regression improvements (Loic Dachary)
+* tests: many many ec test improvements (Loic Dachary)
+* upstart: throttle restarts (#11798 Sage Weil, Greg Farnum)
+
+
+v9.1.0 Infernalis release candidate
+===================================
+
+This is the first Infernalis release candidate.  There have been some
+major changes since Hammer, and the upgrade process is non-trivial.
+Please read carefully.
+
+Getting the release candidate
+-----------------------------
+
+The v9.1.0 packages are pushed to the development release repositories::
+
+  http://download.ceph.com/rpm-testing
+  http://download.ceph.com/debian-testing
+
+For for info, see::
+
+  http://docs.ceph.com/docs/master/install/get-packages/
+
+Or install with ceph-deploy via::
+
+  ceph-deploy install --testing HOST
+
+
+Known issues
+------------
+
+* librbd and librados ABI compatibility is broken.  Be careful
+  installing this RC on client machines (e.g., those running qemu).
+  It will be fixed in the final v9.2.0 release.
+
+
+Major Changes from Hammer
+-------------------------
+
+- *General*:
+
+  * Ceph daemons are now managed via systemd (with the exception of
+    Ubuntu Trusty, which still uses upstart).
+  * Ceph daemons run as 'ceph' user instead of root.
+  * On Red Hat distros, there is also an SELinux policy.
+
+- *RADOS*:
+
+  * The RADOS cache tier can now proxy write operations to the base
+    tier, allowing writes to be handled without forcing migration of
+    an object into the cache.
+  * The SHEC erasure coding support is no longer flagged as
+    experimental. SHEC trades some additional storage space for faster
+    repair.
+  * There is now a unified queue (and thus prioritization) of client
+    IO, scrubbing, and snapshot trimming.
+  * There have been many improvements to low-level repair tooling
+    (ceph-objectstore-tool).
+  * The internal ObjectStore API has been significantly cleaned up in order
+    to facilitate new storage backends like NewStore.
+
+- *RGW*:
+
+  * The Swift API now supports object expiration.
+  * There are many Swift API compatibility improvements.
+
+- *RBD*:
+
+  * The ``rbd du`` command shows actual usage (quickly, when
+    object-map is enabled).
+  * The object-map feature has seen many stability improvements.
+  * Object-map and exclusive-lock features can be enabled or disabled
+    dynamically.
+  * You can now store user metadata and set persistent librbd options
+    associated with individual images.
+  * The new deep-flatten features allows flattening of a clone and all
+    of its snapshots.  (Previously snapshots could not be flattened.)
+  * The export-diff command is now faster (it uses aio).  There is also
+    a new fast-diff feature.
+  * The --size argument can be specified with a suffix for units
+    (e.g., ``--size 64G``).
+  * There is a new ``rbd status`` command that, for now, shows who has
+    the image open/mapped.
+
+- *CephFS*:
+
+  * You can now rename snapshots.
+  * There have been ongoing improvements around administration, diagnostics,
+    and the check and repair tools.
+  * The caching and revocation of client cache state due to unused
+    inodes has been dramatically improved.
+  * The ceph-fuse client behaves better on 32-bit hosts.
+
+Distro compatibility
+--------------------
+
+We have decided to drop support for many older distributions so that we can
+move to a newer compiler toolchain (e.g., C++11).  Although it is still possible
+to build Ceph on older distributions by installing backported development tools,
+we are not building and publishing release packages for them on ceph.com.
+
+In particular,
+
+* CentOS 7 or later; we have dropped support for CentOS 6 (and other
+  RHEL 6 derivatives, like Scientific Linux 6).
+* Debian Jessie 8.x or later; Debian Wheezy 7.x's g++ has incomplete
+  support for C++11 (and no systemd).
+* Ubuntu Trusty 14.04 or later; Ubuntu Precise 12.04 is no longer
+  supported.
+* Fedora 22 or later.
+
+Upgrading from Firefly
+----------------------
+
+Upgrading directly from Firefly v0.80.z is not possible.  All clusters
+must first upgrade to Hammer v0.94.4 or a later v0.94.z release; only
+then is it possible to do online upgrade to Infernalis 9.2.z.
+
+User can upgrade to latest hammer v0.94.z
+from gitbuilder with(also refer the hammer release notes for more details)::
+
+  ceph-deploy install --release hammer HOST
+
+
+Upgrading from Hammer
+---------------------
+
+* All cluster nodes must first upgrade to Hammer v0.94.4 or a later v0.94.z release; only
+  then is it possible to do online upgrade to Infernalis 9.2.z.
+
+* For all distributions that support systemd (CentOS 7, Fedora, Debian
+  Jessie 8.x, OpenSUSE), ceph daemons are now managed using native systemd
+  files instead of the legacy sysvinit scripts.  For example::
+
+    systemctl start ceph.target       # start all daemons
+    systemctl status ceph-osd@12      # check status of osd.12
+
+  The main notable distro that is *not* yet using systemd is Ubuntu trusty
+  14.04.  (The next Ubuntu LTS, 16.04, will use systemd instead of upstart.)
+
+* Ceph daemons now run as user and group ``ceph`` by default.  The
+  ceph user has a static UID assigned by Fedora and Debian (also used
+  by derivative distributions like RHEL/CentOS and Ubuntu).  On SUSE
+  the ceph user will currently get a dynamically assigned UID when the
+  user is created.
+
+  If your systems already have a ceph user, the package upgrade
+  process will usually fail with an error.  We suggest you first
+  remove or rename the existing 'ceph' user and then upgrade.
+
+  When upgrading, administrators have two options:
+
+   #. Add the following line to ``ceph.conf`` on all hosts::
+
+        setuser match path = /var/lib/ceph/$type/$cluster-$id
+
+      This will make the Ceph daemons run as root (i.e., not drop
+      privileges and switch to user ceph) if the daemon's data
+      directory is still owned by root.  Newly deployed daemons will
+      be created with data owned by user ceph and will run with
+      reduced privileges, but upgraded daemons will continue to run as
+      root.
+
+   #. Fix the data ownership during the upgrade.  This is the preferred option,
+      but is more work.  The process for each host would be to:
+
+      #. Upgrade the ceph package.  This creates the ceph user and group.  For
+         example::
+
+           ceph-deploy install --stable infernalis HOST
+
+      #. Stop the daemon(s)::
+
+           service ceph stop           # fedora, centos, rhel, debian
+           stop ceph-all               # ubuntu
+
+      #. Fix the ownership::
+
+           chown -R ceph:ceph /var/lib/ceph
+           chown -R ceph:ceph /var/log/ceph
+
+      #. Restart the daemon(s)::
+
+           start ceph-all                # ubuntu
+           systemctl start ceph.target   # debian, centos, fedora, rhel
+
+* The on-disk format for the experimental KeyValueStore OSD backend has
+  changed.  You will need to remove any OSDs using that backend before you
+  upgrade any test clusters that use it.
+
+Upgrade notes
+-------------
+
+* When a pool quota is reached, librados operations now block indefinitely,
+  the same way they do when the cluster fills up.  (Previously they would return
+  -ENOSPC).  By default, a full cluster or pool will now block.  If your
+  librados application can handle ENOSPC or EDQUOT errors gracefully, you can
+  get error returns instead by using the new librados OPERATION_FULL_TRY flag.
+
+Notable changes
+---------------
+
+NOTE: These notes are somewhat abbreviated while we find a less
+time-consuming process for generating them.
+
+* build: C++11 now supported
+* build: many cmake improvements
+* build: OSX build fixes (Yan, Zheng)
+* build: remove rest-bench
+* ceph-disk: many fixes (Loic Dachary)
+* ceph-disk: support for multipath devices (Loic Dachary)
+* ceph-fuse: mostly behave on 32-bit hosts (Yan, Zheng)
+* ceph-objectstore-tool: many improvements (David Zafman)
+* common: bufferlist performance tuning (Piotr Dalek, Sage Weil)
+* common: make mutex more efficient
+* common: some async compression infrastructure (Haomai Wang)
+* librados: add FULL_TRY and FULL_FORCE flags for dealing with full clusters or pools (Sage Weil)
+* librados: fix notify completion race (#13114 Sage Weil)
+* librados, libcephfs: randomize client nonces (Josh Durgin)
+* librados: pybind: fix binary omap values (Robin H. Johnson)
+* librbd: fix reads larger than the cache size (Lu Shi)
+* librbd: metadata filter fixes (Haomai Wang)
+* librbd: use write_full when possible (Zhiqiang Wang)
+* mds: avoid emitting cap warnigns before evicting session (John Spray)
+* mds: fix expected holes in journal objects (#13167 Yan, Zheng)
+* mds: fix SnapServer crash on deleted pool (John Spray)
+* mds: many fixes (Yan, Zheng, John Spray, Greg Farnum)
+* mon: add cache over MonitorDBStore (Kefu Chai)
+* mon: 'ceph osd metadata' can dump all osds (Haomai Wang)
+* mon: detect kv backend failures (Sage Weil)
+* mon: fix CRUSH map test for new pools (Sage Weil)
+* mon: fix min_last_epoch_clean tracking (Kefu Chai)
+* mon: misc scaling fixes (Sage Weil)
+* mon: streamline session handling, fix memory leaks (Sage Weil)
+* mon: upgrades must pass through hammer (Sage Weil)
+* msg/async: many fixes (Haomai Wang)
+* osd: cache proxy-write support (Zhiqiang Wang, Samuel Just)
+* osd: configure promotion based on write recency (Zhiqiang Wang)
+* osd: don't send dup MMonGetOSDMap requests (Sage Weil, Kefu Chai)
+* osd: erasure-code: fix SHEC floating point bug (#12936 Loic Dachary)
+* osd: erasure-code: update to ISA-L 2.14 (Yuan Zhou)
+* osd: fix hitset object naming to use GMT (Kefu Chai)
+* osd: fix misc memory leaks (Sage Weil)
+* osd: fix peek_queue locking in FileStore (Xinze Chi)
+* osd: fix promotion vs full cache tier (Samuel Just)
+* osd: fix replay requeue when pg is still activating (#13116 Samuel Just)
+* osd: fix scrub stat bugs (Sage Weil, Samuel Just)
+* osd: force promotion for ops EC can't handle (Zhiqiang Wang)
+* osd: improve behavior on machines with large memory pages (Steve Capper)
+* osd: merge multiple setattr calls into a setattrs call (Xinxin Shu)
+* osd: newstore prototype (Sage Weil)
+* osd: ObjectStore internal API refactor (Sage Weil)
+* osd: SHEC no longer experimental
+* osd: throttle evict ops (Yunchuan Wen)
+* osd: upgrades must pass through hammer (Sage Weil)
+* osd: use SEEK_HOLE / SEEK_DATA for sparse copy (Xinxin Shu)
+* rbd: rbd-replay-prep and rbd-replay improvements (Jason Dillaman)
+* rgw: expose the number of unhealthy workers through admin socket (Guang Yang)
+* rgw: fix casing of Content-Type header (Robin H. Johnson)
+* rgw: fix decoding of X-Object-Manifest from GET on Swift DLO (Radslow Rzarzynski)
+* rgw: fix sysvinit script
+* rgw: fix sysvinit script w/ multiple instances (Sage Weil, Pavan Rallabhandi)
+* rgw: improve handling of already removed buckets in expirer (Radoslaw Rzarzynski)
+* rgw: log to /var/log/ceph instead of /var/log/radosgw
+* rgw: rework X-Trans-Id header to be conform with Swift API (Radoslaw Rzarzynski)
+* rgw: s3 encoding-type for get bucket (Jeff Weber)
+* rgw: set max buckets per user in ceph.conf (Vikhyat Umrao)
+* rgw: support for Swift expiration API (Radoslaw Rzarzynski, Yehuda Sadeh)
+* rgw: user rm is idempotent (Orit Wasserman)
+* selinux policy (Boris Ranto, Milan Broz)
+* systemd: many fixes (Sage Weil, Owen Synge, Boris Ranto, Dan van der Ster)
+* systemd: run daemons as user ceph
+
+
+v9.0.3
+======
+
+This is the second to last batch of development work for the
+Infernalis cycle.  The most intrusive change is an internal (non
+user-visible) change to the OSD's ObjectStore interface.  Many fixes and
+improvements elsewhere across RGW, RBD, and another big pile of CephFS
+scrub/repair improvements.
+
+Upgrading
+---------
+
+* The return code for librbd's rbd_aio_read and Image::aio_read API methods no
+  longer returns the number of bytes read upon success.  Instead, it returns 0
+  upon success and a negative value upon failure.
+
+* 'ceph scrub', 'ceph compact' and 'ceph sync force' are now deprecated.  Users
+  should instead use 'ceph mon scrub', 'ceph mon compact' and
+  'ceph mon sync force'.
+
+* 'ceph mon_metadata' should now be used as 'ceph mon metadata'.
+
+* The `--dump-json` option of "osdmaptool" is replaced by `--dump json`.
+
+* The commands of 'pg ls-by-{pool,primary,osd}' and 'pg ls' now take 'recovering'
+  instead of 'recovery' to include the recovering pgs in the listed pgs.
+
+
+Notable Changes
+---------------
+
+  * autotools: fix out of tree build (Krxysztof Kosinski)
+  * autotools: improve make check output (Loic Dachary)
+  * buffer: add invalidate_crc() (Piotr Dalek)
+  * buffer: fix zero bug (#12252 Haomai Wang)
+  * build: fix junit detection on Fedora 22 (Ira Cooper)
+  * ceph-disk: install pip > 6.1 (#11952 Loic Dachary)
+  * cephfs-data-scan: many additions, improvements (John Spray)
+  * ceph: improve error output for 'tell' (#11101 Kefu Chai)
+  * ceph-objectstore-tool: misc improvements (David Zafman)
+  * ceph-objectstore-tool: refactoring and cleanup (John Spray)
+  * ceph_test_rados: test pipelined reads (Zhiqiang Wang)
+  * common: fix bit_vector extent calc (#12611 Jason Dillaman)
+  * common: make work queue addition/removal thread safe (#12662 Jason Dillaman)
+  * common: optracker improvements (Zhiqiang Wang, Jianpeng Ma)
+  * crush: add --check to validate dangling names, max osd id (Kefu Chai)
+  * crush: cleanup, sync with kernel (Ilya Dryomov)
+  * crush: fix subtree base weight on adjust_subtree_weight (#11855 Sage Weil)
+  * crypo: fix NSS leak (Jason Dillaman)
+  * crypto: fix unbalanced init/shutdown (#12598 Zheng Yan)
+  * doc: misc updates (Kefu Chai, Owen Synge, Gael Fenet-Garde, Loic Dachary, Yannick Atchy-Dalama, Jiaying Ren, Kevin Caradant, Robert Maxime, Nicolas Yong, Germain Chipaux, Arthur Gorjux, Gabriel Sentucq, Clement Lebrun, Jean-Remi Deveaux, Clair Massot, Robin Tang, Thomas Laumondais, Jordan Dorne, Yuan Zhou, Valentin Thomas, Pierre Chaumont, Benjamin Troquereau, Benjamin Sesia, Vikhyat Umrao)
+  * erasure-code: cleanup (Kefu Chai)
+  * erasure-code: improve tests (Loic Dachary)
+  * erasure-code: shec: fix recovery bugs (Takanori Nakao, Shotaro Kawaguchi)
+  * libcephfs: add pread, pwrite (Jevon Qiao)
+  * libcephfs,ceph-fuse: cache cleanup (Zheng Yan)
+  * librados: add src_fadvise_flags for copy-from (Jianpeng Ma)
+  * librados: respect default_crush_ruleset on pool_create (#11640 Yuan Zhou)
+  * librbd: fadvise for copy, export, import (Jianpeng Ma)
+  * librbd: handle NOCACHE fadvise flag (Jinapeng Ma)
+  * librbd: optionally disable allocation hint (Haomai Wang)
+  * librbd: prevent race between resize requests (#12664 Jason Dillaman)
+  * log: fix data corruption race resulting from log rotation (#12465 Samuel Just)
+  * mds: expose frags via asok (John Spray)
+  * mds: fix setting entire file layout in one setxattr (John Spray)
+  * mds: fix shutdown (John Spray)
+  * mds: handle misc corruption issues (John Spray)
+  * mds: misc fixes (Jianpeng Ma, Dan van der Ster, Zhang Zhi)
+  * mds: misc snap fixes (Zheng Yan)
+  * mds: store layout on header object (#4161 John Spray)
+  * misc performance and cleanup (Nathan Cutler, Xinxin Shu)
+  * mon: add NOFORWARD, OBSOLETE, DEPRECATE flags for mon commands (Joao Eduardo Luis)
+  * mon: add PG count to 'ceph osd df' output (Michal Jarzabek)
+  * mon: clean up, reorg some mon commands (Joao Eduardo Luis)
+  * mon: disallow >2 tiers (#11840 Kefu Chai)
+  * mon: fix log dump crash when debugging (Mykola Golub)
+  * mon: fix metadata update race (Mykola Golub)
+  * mon: fix refresh (#11470 Joao Eduardo Luis)
+  * mon: make blocked op messages more readable (Jianpeng Ma)
+  * mon: only send mon metadata to supporting peers (Sage Weil)
+  * mon: periodic background scrub (Joao Eduardo Luis)
+  * mon: prevent pgp_num > pg_num (#12025 Xinxin Shu)
+  * mon: reject large max_mds values (#12222 John Spray)
+  * msgr: add ceph_perf_msgr tool (Hoamai Wang)
+  * msgr: async: fix seq handling (Haomai Wang)
+  * msgr: xio: fastpath improvements (Raju Kurunkad)
+  * msgr: xio: sync with accellio v1.4 (Vu Pham)
+  * osd: clean up temp object if promotion fails (Jianpeng Ma)
+  * osd: constrain collections to meta and PGs (normal and temp) (Sage Weil)
+  * osd: filestore: clone using splice (Jianpeng Ma)
+  * osd: filestore: fix recursive lock (Xinxin Shu)
+  * osd: fix dup promotion lost op bug (Zhiqiang Wang)
+  * osd: fix temp-clearing (David Zafman)
+  * osd: include a temp namespace within each collection/pgid (Sage Weil)
+  * osd: low and high speed flush modes (Mingxin Liu)
+  * osd: peer_features includes self (David Zafman)
+  * osd: recovery, peering fixes (#11687 Samuel Just)
+  * osd: require firefly features (David Zafman)
+  * osd: set initial crush weight with more precision (Sage Weil)
+  * osd: use a temp object for recovery (Sage Weil)
+  * osd: use blkid to collection partition information (Joseph Handzik)
+  * rados: add --striper option to use libradosstriper (#10759 Sebastien Ponce)
+  * radosgw-admin: fix subuser modify output (#12286 Guce)
+  * rados: handle --snapid arg properly (Abhishek Lekshmanan)
+  * rados: improve bench buffer handling, performance (Piotr Dalek)
+  * rados: new pool import implementation (John Spray)
+  * rbd: fix link issues (Jason Dillaman)
+  * rbd: improve CLI arg parsing, usage (Ilya Dryomov)
+  * rbd: recognize queue_depth kernel option (Ilya Dryomov)
+  * rbd: support G and T units for CLI (Abhishek Lekshmanan)
+  * rbd: use image-spec and snap-spec in help (Vikhyat Umrao, Ilya Dryomov)
+  * rest-bench: misc fixes (Shawn Chen)
+  * rest-bench: support https (#3968 Yuan Zhou)
+  * rgw: add max multipart upload parts (#12146 Abshishek Dixit)
+  * rgw: add Trasnaction-Id to response (Abhishek Dixit)
+  * rgw: document layout of pools and objects (Pete Zaitcev)
+  * rgw: do not preserve ACLs when copying object (#12370 Yehuda Sadeh)
+  * rgw: fix Connection: header handling (#12298 Wido den Hollander)
+  * rgw: fix data corruptions race condition (#11749 Wuxingyi)
+  * rgw: fix JSON response when getting user quota (#12117 Wuxingyi)
+  * rgw: force content_type for swift bucket stats requests (#12095 Orit Wasserman)
+  * rgw: improved support for swift account metadata (Radoslaw Zarzynski)
+  * rgw: make max put size configurable (#6999 Yuan Zhou)
+  * rgw: orphan detection tool (Yehuda Sadeh)
+  * rgw: swift: do not override sent content type (#12363 Orit Wasserman)
+  * rgw: swift: set Content-Length for account GET (#12158 Radoslav Zarzynski)
+  * rpm: always rebuild and install man pages for rpm (Owen Synge)
+  * rpm: misc fixes (Boris Ranto, Owen Synge, Ken Dreyer, Ira Cooper)
+  * systemd: logrotate fixes (Tim Seron, Lars Marowsky-Bree, Nathan Cutler)
+  * sysvinit compat: misc fixes (Owen Synge)
+  * test: misc fs test improvements (John Spray, Loic Dachary)
+  * test: python tests, linter cleanup (Alfredo Deza)
+
+
+v9.0.2
+======
+
+This development release features more of the OSD work queue
+unification, randomized osd scrub times, a huge pile of librbd fixes,
+more MDS repair and snapshot fixes, and a significant amount of work
+on the tests and build infrastructure.
+
+Notable Changes
+---------------
+
+* buffer: some cleanup (Michal Jarzabek)
+* build: cmake: fix nss linking (Danny Al-Gaaf)
+* build: cmake: misc fixes (Orit Wasserman, Casey Bodley)
+* build: install-deps: misc fixes (Loic Dachary)
+* build: make_dist_tarball.sh (Sage Weil)
+* ceph-detect-init: added Linux Mint (Michal Jarzabek)
+* ceph-detect-init: robust init system detection (Owen Synge, Loic Dachary)
+* ceph-disk: ensure 'zap' only operates on a full disk (#11272 Loic Dachary)
+* ceph-disk: misc fixes to respect init system (Loic Dachary, Owen Synge)
+* ceph-disk: support NVMe device partitions (#11612 Ilja Slepnev)
+* ceph: fix 'df' units (Zhe Zhang)
+* ceph: fix parsing in interactive cli mode (#11279 Kefu Chai)
+* ceph-objectstore-tool: many many changes (David Zafman)
+* ceph-post-file: misc fixes (Joey McDonald, Sage Weil)
+* client: avoid sending unnecessary FLUSHSNAP messages (Yan, Zheng)
+* client: exclude setfilelock when calculating oldest tid (Yan, Zheng)
+* client: fix error handling in check_pool_perm (John Spray)
+* client: fsync waits only for inode's caps to flush (Yan, Zheng)
+* client: invalidate kernel dcache when cache size exceeds limits (Yan, Zheng)
+* client: make fsync wait for unsafe dir operations (Yan, Zheng)
+* client: pin lookup dentry to avoid inode being freed (Yan, Zheng)
+* common: detect overflow of int config values (#11484 Kefu Chai)
+* common: fix json parsing of utf8 (#7387 Tim Serong)
+* common: fix leak of pthread_mutexattr (#11762 Ketor Meng)
+* crush: respect default replicated ruleset config on map creation (Ilya Dryomov)
+* deb, rpm: move ceph-objectstore-tool to ceph (Ken Dreyer)
+* doc: man page updates (Kefu Chai)
+* doc: misc updates (#11396 Nilamdyuti, Francois Lafont, Ken Dreyer, Kefu Chai)
+* init-radosgw: merge with sysv version; fix enumeration (Sage Weil)
+* librados: add config observer (Alistair Strachan)
+* librbd: add const for single-client-only features (Josh Durgin)
+* librbd: add deep-flatten operation (Jason Dillaman)
+* librbd: avoid blocking aio API methods (#11056 Jason Dillaman)
+* librbd: fix fast diff bugs (#11553 Jason Dillaman)
+* librbd: fix image format detection (Zhiqiang Wang)
+* librbd: fix lock ordering issue (#11577 Jason Dillaman)
+* librbd: flatten/copyup fixes (Jason Dillaman)
+* librbd: lockdep, helgrind validation (Jason Dillaman, Josh Durgin)
+* librbd: only update image flags while hold exclusive lock (#11791 Jason Dillaman)
+* librbd: return result code from close (#12069 Jason Dillaman)
+* librbd: tolerate old osds when getting image metadata (#11549 Jason Dillaman)
+* mds: do not add snapped items to bloom filter (Yan, Zheng)
+* mds: fix handling for missing mydir dirfrag (#11641 John Spray)
+* mds: fix rejoin (Yan, Zheng)
+* mds: fix stra reintegration (Yan, Zheng)
+* mds: fix suicide beason (John Spray)
+* mds: misc repair improvements (John Spray)
+* mds: misc snapshot fixes (Yan, Zheng)
+* mds: respawn instead of suicide on blacklist (John Spray)
+* misc coverity fixes (Danny Al-Gaaf)
+* mon: add 'mon_metadata <id>' command (Kefu Chai)
+* mon: add 'node ls ...' command (Kefu Chai)
+* mon: disallow ec pools as tiers (#11650 Samuel Just)
+* mon: fix mds beacon replies (#11590 Kefu Chai)
+* mon: fix 'pg ls' sort order, state names (#11569 Kefu Chai)
+* mon: normalize erasure-code profile for storage and comparison (Loic Dachary)
+* mon: optionally specify osd id on 'osd create' (Mykola Golub)
+* mon: 'osd tree' fixes (Kefu Chai)
+* mon: prevent pool with snapshot state from being used as a tier (#11493 Sage Weil)
+* mon: refine check_remove_tier checks (#11504 John Spray)
+* mon: remove spurious who arg from 'mds rm ...' (John Spray)
+* msgr: async: misc fixes (Haomai Wang)
+* msgr: xio: fix ip and nonce (Raju Kurunkad)
+* msgr: xio: improve lane assignment (Vu Pham)
+* msgr: xio: misc fixes (Vu Pham, Cosey Bodley)
+* osd: avoid transaction append in some cases (Sage Weil)
+* osdc/Objecter: allow per-pool calls to op_cancel_writes (John Spray)
+* osd: elminiate txn apend, ECSubWrite copy (Samuel Just)
+* osd: filejournal: cleanup (David Zafman)
+* osd: fix check_for_full (Henry Chang)
+* osd: fix dirty accounting in make_writeable (Zhiqiang Wang)
+* osd: fix osdmap dump of blacklist items (John Spray)
+* osd: fix snap flushing from cache tier (again) (#11787 Samuel Just)
+* osd: fix snap handling on promotion (#11296 Sam Just)
+* osd: handle log split with overlapping entries (#11358 Samuel Just)
+* osd: keyvaluestore: misc fixes (Varada Kari)
+* osd: make suicide timeouts individually configurable (Samuel Just)
+* osd: move scrub in OpWQ (Samuel Just)
+* osd: pool size change triggers new interval (#11771 Samuel Just)
+* osd: randomize scrub times (#10973 Kefu Chai)
+* osd: refactor scrub and digest recording (Sage Weil)
+* osd: refuse first write to EC object at non-zero offset (Jianpeng Ma)
+* osd: stripe over small xattrs to fit in XFS's 255 byte inline limit (Sage Weil, Ning Yao)
+* osd: sync object_map on syncfs (Samuel Just)
+* osd: take excl lock of op is rw (Samuel Just)
+* osd: WBThrottle cleanups (Jianpeng Ma)
+* pycephfs: many fixes for bindings (Haomai Wang)
+* rados: bench: add --no-verify option to improve performance (Piotr Dalek)
+* rados: misc bench fixes (Dmitry Yatsushkevich)
+* rbd: add disk usage tool (#7746 Jason Dillaman)
+* rgw: alwasy check if token is expired (#11367 Anton Aksola, Riku Lehto)
+* rgw: conversion tool to repair broken multipart objects (#12079 Yehuda Sadeh)
+* rgw: do not enclose bucket header in quotes (#11860 Wido den Hollander)
+* rgw: error out if frontend did not send all data (#11851 Yehuda Sadeh)
+* rgw: fix assignment of copy obj attributes (#11563 Yehuda Sadeh)
+* rgw: fix reset_loc (#11974 Yehuda Sadeh)
+* rgw: improve content-length env var handling (#11419 Robin H. Johnson)
+* rgw: only scan for objects not in a namespace (#11984 Yehuda Sadeh)
+* rgw: remove trailing :port from HTTP_HOST header (Sage Weil)
+* rgw: shard work over multiple librados instances (Pavan Rallabhandi)
+* rgw: swift: enforce Content-Type in response (#12157 Radoslaw Zarzynski)
+* rgw: use attrs from source bucket on copy (#11639 Javier M. Mellid)
+* rocksdb: pass options as single string (Xiaoxi Chen)
+* rpm: many spec file fixes (Owen Synge, Ken Dreyer)
+* tests: fixes for rbd xstests (Douglas Fuller)
+* tests: fix tiering health checks (Loic Dachary)
+* tests for low-level performance (Haomai Wang)
+* tests: many ec non-regression improvements (Loic Dachary)
+* tests: many many ec test improvements (Loic Dachary)
+* upstart: throttle restarts (#11798 Sage Weil, Greg Farnum)
+
+
+v9.0.1
+======
+
+This development release is delayed a bit due to tooling changes in the build
+environment.  As a result the next one (v9.0.2) will have a bit more work than
+is usual.
+
+Highlights here include lots of RGW Swift fixes, RBD feature work
+surrounding the new object map feature, more CephFS snapshot fixes,
+and a few important CRUSH fixes.
+
+Notable Changes
+---------------
+
+* auth: cache/reuse crypto lib key objects, optimize msg signature check (Sage Weil)
+* build: allow tcmalloc-minimal (Thorsten Behrens)
+* build: do not build ceph-dencoder with tcmalloc (#10691 Boris Ranto)
+* build: fix pg ref disabling (William A. Kennington III)
+* build: install-deps.sh improvements (Loic Dachary)
+* build: misc fixes (Boris Ranto, Ken Dreyer, Owen Synge)
+* ceph-authtool: fix return code on error (Gerhard Muntingh)
+* ceph-disk: fix zap sgdisk invocation (Owen Synge, Thorsten Behrens)
+* ceph-disk: pass --cluster arg on prepare subcommand (Kefu Chai)
+* ceph-fuse, libcephfs: drop inode when rmdir finishes (#11339 Yan, Zheng)
+* ceph-fuse,libcephfs: fix uninline (#11356 Yan, Zheng)
+* ceph-monstore-tool: fix store-copy (Huangjun)
+* common: add perf counter descriptions (Alyona Kiseleva)
+* common: fix throttle max change (Henry Chang)
+* crush: fix crash from invalid 'take' argument (#11602 Shiva Rkreddy, Sage Weil)
+* crush: fix divide-by-2 in straw2 (#11357 Yann Dupont, Sage Weil)
+* deb: fix rest-bench-dbg and ceph-test-dbg dependendies (Ken Dreyer)
+* doc: document region hostnames (Robin H. Johnson)
+* doc: update release schedule docs (Loic Dachary)
+* init-radosgw: run radosgw as root (#11453 Ken Dreyer)
+* librados: fadvise flags per op (Jianpeng Ma)
+* librbd: allow additional metadata to be stored with the image (Haomai Wang)
+* librbd: better handling for dup flatten requests (#11370 Jason Dillaman)
+* librbd: cancel in-flight ops on watch error (#11363 Jason Dillaman)
+* librbd: default new images to format 2 (#11348 Jason Dillaman)
+* librbd: fast diff implementation that leverages object map (Jason Dillaman)
+* librbd: fix snapshot creation when other snap is active (#11475 Jason Dillaman)
+* librbd: new diff_iterate2 API (Jason Dillaman)
+* librbd: object map rebuild support (Jason Dillaman)
+* logrotate.d: prefer service over invoke-rc.d (#11330 Win Hierman, Sage Weil)
+* mds: avoid getting stuck in XLOCKDONE (#11254 Yan, Zheng)
+* mds: fix integer truncateion on large client ids (Henry Chang)
+* mds: many snapshot and stray fixes (Yan, Zheng)
+* mds: persist completed_requests reliably (#11048 John Spray)
+* mds: separate safe_pos in Journaler (#10368 John Spray)
+* mds: snapshot rename support (#3645 Yan, Zheng)
+* mds: warn when clients fail to advance oldest_client_tid (#10657 Yan, Zheng)
+* misc cleanups and fixes (Danny Al-Gaaf)
+* mon: fix average utilization calc for 'osd df' (Mykola Golub)
+* mon: fix variance calc in 'osd df' (Sage Weil)
+* mon: improve callout to crushtool (Mykola Golub)
+* mon: prevent bucket deletion when referenced by a crush rule (#11602 Sage Weil)
+* mon: prime pg_temp when CRUSH map changes (Sage Weil)
+* monclient: flush_log (John Spray)
+* msgr: async: many many fixes (Haomai Wang)
+* msgr: simple: fix clear_pipe (#11381 Haomai Wang)
+* osd: add latency perf counters for tier operations (Xinze Chi)
+* osd: avoid multiple hit set insertions (Zhiqiang Wang)
+* osd: break PG removal into multiple iterations (#10198 Guang Yang)
+* osd: check scrub state when handling map (Jianpeng Ma)
+* osd: fix endless repair when object is unrecoverable (Jianpeng Ma, Kefu Chai)
+* osd: fix pg resurrection (#11429 Samuel Just)
+* osd: ignore non-existent osds in unfound calc (#10976 Mykola Golub)
+* osd: increase default max open files (Owen Synge)
+* osd: prepopulate needs_recovery_map when only one peer has missing (#9558 Guang Yang)
+* osd: relax reply order on proxy read (#11211 Zhiqiang Wang)
+* osd: skip promotion for flush/evict op (Zhiqiang Wang)
+* osd: write journal header on clean shutdown (Xinze Chi)
+* qa: run-make-check.sh script (Loic Dachary)
+* rados bench: misc fixes (Dmitry Yatsushkevich)
+* rados: fix error message on failed pool removal (Wido den Hollander)
+* radosgw-admin: add 'bucket check' function to repair bucket index (Yehuda Sadeh)
+* rbd: allow unmapping by spec (Ilya Dryomov)
+* rbd: deprecate --new-format option (Jason Dillman)
+* rgw: do not set content-type if length is 0 (#11091 Orit Wasserman)
+* rgw: don't use end_marker for namespaced object listing (#11437 Yehuda Sadeh)
+* rgw: fail if parts not specified on multipart upload (#11435 Yehuda Sadeh)
+* rgw: fix GET on swift account when limit == 0 (#10683 Radoslaw Zarzynski)
+* rgw: fix broken stats in container listing (#11285 Radoslaw Zarzynski)
+* rgw: fix bug in domain/subdomain splitting (Robin H. Johnson)
+* rgw: fix civetweb max threads (#10243 Yehuda Sadeh)
+* rgw: fix copy metadata, support X-Copied-From for swift (#10663 Radoslaw Zarzynski)
+* rgw: fix locator for objects starting with _ (#11442 Yehuda Sadeh)
+* rgw: fix mulitipart upload in retry path (#11604 Yehuda Sadeh)
+* rgw: fix quota enforcement on POST (#11323 Sergey Arkhipov)
+* rgw: fix return code on missing upload (#11436 Yehuda Sadeh)
+* rgw: force content type header on responses with no body (#11438 Orit Wasserman)
+* rgw: generate new object tag when setting attrs (#11256 Yehuda Sadeh)
+* rgw: issue aio for first chunk before flush cached data (#11322 Guang Yang)
+* rgw: make read user buckets backward compat (#10683 Radoslaw Zarzynski)
+* rgw: merge manifests properly with prefix override (#11622 Yehuda Sadeh)
+* rgw: return 412 on bad limit when listing buckets (#11613 Yehuda Sadeh)
+* rgw: send ETag, Last-Modified for swift (#11087 Radoslaw Zarzynski)
+* rgw: set content length on container GET, PUT, DELETE, HEAD (#10971, #11036 Radoslaw Zarzynski)
+* rgw: support end marker on swift container GET (#10682 Radoslaw Zarzynski)
+* rgw: swift: fix account listing (#11501 Radoslaw Zarzynski)
+* rgw: swift: set content-length on keystone tokens (#11473 Herv Rousseau)
+* rgw: use correct oid for gc chains (#11447 Yehuda Sadeh)
+* rgw: use unique request id for civetweb (#10295 Orit Wasserman)
+* rocksdb, leveldb: fix compact_on_mount (Xiaoxi Chen)
+* rocksdb: add perf counters for get/put latency (Xinxin Shu)
+* rpm: add suse firewall files (Tim Serong)
+* rpm: misc systemd and suse fixes (Owen Synge, Nathan Cutler)
+
+
+
+v9.0.0
+======
+
+This is the first development release for the Infernalis cycle, and
+the first Ceph release to sport a version number from the new
+numbering scheme.  The "9" indicates this is the 9th release cycle--I
+(for Infernalis) is the 9th letter.  The first "0" indicates this is a
+development release ("1" will mean release candidate and "2" will mean
+stable release), and the final "0" indicates this is the first such
+development release.
+
+A few highlights include:
+
+* a new 'ceph daemonperf' command to watch perfcounter stats in realtime
+* reduced MDS memory usage
+* many MDS snapshot fixes
+* librbd can now store options in the image itself
+* many fixes for RGW Swift API support
+* OSD performance improvements
+* many doc updates and misc bug fixes
+
+Notable Changes
+---------------
+
+* aarch64: add optimized version of crc32c (Yazen Ghannam, Steve Capper)
+* auth: reinit NSS after fork() (#11128 Yan, Zheng)
+* build: disable LTTNG by default (#11333 Josh Durgin)
+* build: fix ppc build (James Page)
+* build: install-deps: support OpenSUSE (Loic Dachary)
+* build: misc cmake fixes (Matt Benjamin)
+* ceph-disk: follow ceph-osd hints when creating journal (#9580 Sage Weil)
+* ceph-disk: handle re-using existing partition (#10987 Loic Dachary)
+* ceph-disk: improve parted output parsing (#10983 Loic Dachary)
+* ceph-disk: make suppression work for activate-all and activate-journal (Dan van der Ster)
+* ceph-disk: misc fixes (Alfredo Deza)
+* ceph-fuse, libcephfs: don't clear COMPLETE when trimming null (Yan, Zheng)
+* ceph-fuse, libcephfs: hold exclusive caps on dirs we "own" (#11226 Greg Farnum)
+* ceph-fuse: do not require successful remount when unmounting (#10982 Greg Farnum)
+* ceph: new 'ceph daemonperf' command (John Spray, Mykola Golub)
+* common: PriorityQueue tests (Kefu Chai)
+* common: add descriptions to perfcounters (Kiseleva Alyona)
+* common: fix LTTNG vs fork issue (Josh Durgin)
+* crush: fix has_v4_buckets (#11364 Sage Weil)
+* crushtool: fix order of operations, usage (Sage Weil)
+* debian: minor package reorg (Ken Dreyer)
+* doc: docuemnt object corpus generation (#11099 Alexis Normand)
+* doc: fix gender neutrality (Alexandre Maragone)
+* doc: fix install doc (#10957 Kefu Chai)
+* doc: fix sphinx issues (Kefu Chai)
+* doc: mds data structure docs (Yan, Zheng)
+* doc: misc updates (Nilamdyuti Goswami, Vartika Rai, Florian Haas, Loic Dachary, Simon Guinot, Andy Allan, Alistair Israel, Ken Dreyer, Robin Rehu, Lee Revell, Florian Marsylle, Thomas Johnson, Bosse Klykken, Travis Rhoden, Ian Kelling)
+* doc: swift tempurls (#10184 Abhishek Lekshmanan)
+* doc: switch doxygen integration back to breathe (#6115 Kefu Chai)
+* erasure-code: update ISA-L to 2.13 (Yuan Zhou)
+* gmock: switch to submodule (Danny Al-Gaaf, Loic Dachary)
+* hadoop: add terasort test (Noah Watkins)
+* java: fix libcephfs bindings (Noah Watkins)
+* libcephfs,ceph-fuse: fix request resend on cap reconnect (#10912 Yan, Zheng)
+* librados: define C++ flags from C constants (Josh Durgin)
+* librados: fix last_force_resent handling (#11026 Jianpeng Ma)
+* librados: fix memory leak from C_TwoContexts (Xiong Yiliang)
+* librados: fix striper when stripe_count = 1 and stripe_unit != object_size (#11120 Yan, Zheng)
+* librados: op perf counters (John Spray)
+* librados: pybind: fix write() method return code (Javier Guerra)
+* libradosstriper: fix leak (Danny Al-Gaaf)
+* librbd: add purge_on_error cache behavior (Jianpeng Ma)
+* librbd: misc aio fixes (#5488 Jason Dillaman)
+* librbd: misc rbd fixes (#11478 #11113 #11342 #11380 Jason Dillaman, Zhiqiang Wang)
+* librbd: readahead fixes (Zhiqiang Wang)
+* librbd: store metadata, including config options, in image (Haomai Wang)
+* mds: add 'damaged' state to MDSMap (John Spray)
+* mds: add nicknames for perfcounters (John Spray)
+* mds: disable problematic rstat propagation into snap parents (Yan, Zheng)
+* mds: fix mydir replica issue with shutdown (#10743 John Spray)
+* mds: fix out-of-order messages (#11258 Yan, Zheng)
+* mds: fix shutdown with strays (#10744 John Spray)
+* mds: fix snapshot fixes (Yan, Zheng)
+* mds: fix stray handling (John Spray)
+* mds: flush immediately in do_open_truncate (#11011 John Spray)
+* mds: improve dump methods (John Spray)
+* mds: misc journal cleanups and fixes (#10368 John Spray)
+* mds: new SessionMap storage using omap (#10649 John Spray)
+* mds: reduce memory consumption (Yan, Zheng)
+* mds: throttle purge stray operations (#10390 John Spray)
+* mds: tolerate clock jumping backwards (#11053 Yan, Zheng)
+* misc coverity fixes (Danny Al-Gaaf)
+* mon: do not deactivate last mds (#10862 John Spray)
+* mon: make osd get pool 'all' only return applicable fields (#10891 Michal Jarzabek)
+* mon: warn on bogus cache tier config (Jianpeng Ma)
+* msg/async: misc bug fixes and updates (Haomai Wang)
+* msg/simple: fix connect_seq assert (Haomai Wang)
+* msg/xio: misc fixes (#10735 Matt Benjamin, Kefu Chai, Danny Al-Gaaf, Raju Kurunkad, Vu Pham)
+* msg: unit tests (Haomai Wang)
+* objectcacher: misc bug fixes (Jianpeng Ma)
+* os/filestore: enlarge getxattr buffer size (Jianpeng Ma)
+* osd: EIO injection (David Zhang)
+* osd: add misc perfcounters (Xinze Chi)
+* osd: add simple sleep injection in recovery (Sage Weil)
+* osd: allow SEEK_HOLE/SEEK_DATA for sparse read (Zhiqiang Wang)
+* osd: avoid dup omap sets for in pg metadata (Sage Weil)
+* osd: clean up some constness, privateness (Kefu Chai)
+* osd: erasure-code: drop entries according to LRU (Andreas-Joachim Peters)
+* osd: fix negative degraded stats during backfill (Guang Yang)
+* osd: misc fixes (Ning Yao, Kefu Chai, Xinze Chi, Zhiqiang Wang, Jianpeng Ma)
+* pybind: pep8 cleanups (Danny Al-Gaaf)
+* qa: fix filelock_interrupt.py test (Yan, Zheng)
+* qa: improve ceph-disk tests (Loic Dachary)
+* qa: improve docker build layers (Loic Dachary)
+* rados: translate erno to string in CLI (#10877 Kefu Chai)
+* rbd: accept map options config option (Ilya Dryomov)
+* rbd: cli: fix arg parsing with --io-pattern (Dmitry Yatsushkevich)
+* rbd: fix error messages (#2862 Rajesh Nambiar)
+* rbd: update rbd man page (Ilya Dryomov)
+* rbd: update xfstests tests (Douglas Fuller)
+* rgw: add X-Timestamp for Swift containers (#10938 Radoslaw Zarzynski)
+* rgw: add missing headers to Swift container details (#10666 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: add stats to headers for account GET (#10684 Yuan Zhou)
+* rgw: do not prefecth data for HEAD requests (Guang Yang)
+* rgw: don't clobber bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: don't use rgw_socket_path if frontend is configured (#11160 Yehuda Sadeh)
+* rgw: enforce Content-Lenth for POST on Swift cont/obj (#10661 Radoslaw Zarzynski)
+* rgw: fix handling empty metadata items on Swift container (#11088 Radoslaw Zarzynski)
+* rgw: fix log rotation (Wuxingyi)
+* rgw: generate Date header for civetweb (#10873 Radoslaw Zarzynski)
+* rgw: make init script wait for radosgw to stop (#11140 Dmitry Yatsushkevich)
+* rgw: make quota/gc threads configurable (#11047 Guang Yang)
+* rgw: pass in civetweb configurables (#10907 Yehuda Sadeh)
+* rgw: rectify 202 Accepted in PUT response (#11148 Radoslaw Zarzynski)
+* rgw: remove meta file after deleting bucket (#11149 Orit Wasserman)
+* rgw: swift: allow setting attributes with COPY (#10662 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: swift: fix metadata handling on copy (#10645 Radoslaw Zarzynski)
+* rgw: swift: send Last-Modified header (#10650 Radoslaw Zarzynski)
+* rgw: update keystone cache with token info (#11125 Yehuda Sadeh)
+* rgw: update to latest civetweb, enable config for IPv6 (#10965 Yehuda Sadeh)
+* rocksdb: update to latest (Xiaoxi Chen)
+* rpm: loosen ceph-test dependencies (Ken Dreyer)
+
diff --git a/doc/releases/jewel.rst b/doc/releases/jewel.rst
new file mode 100644
index 000000000..e76106013
--- /dev/null
+++ b/doc/releases/jewel.rst
@@ -0,0 +1,3384 @@
+=====
+Jewel
+=====
+
+Jewel is the 10th stable release of Ceph. It is named after the jewel
+squid (Histioteuthis reversa).
+
+
+v10.2.11 Jewel
+==============
+
+This point releases brings a number of important bugfixes and has a few
+important security fixes. This is expected to be the last Jewel release. We
+recommend all Jewel 10.2.x users to upgrade.
+
+Notable Changes
+---------------
+
+* CVE 2018-1128: auth: cephx authorizer subject to replay attack (`issue#24836 <http://tracker.ceph.com/issues/24836>`_, Sage Weil)
+
+* CVE 2018-1129: auth: cephx signature check is weak (`issue#24837 <http://tracker.ceph.com/issues/24837>`_, Sage Weil)
+
+* CVE 2018-10861: mon: auth checks not correct for pool ops (`issue#24838 <http://tracker.ceph.com/issues/24838>`_, Jason Dillaman)
+
+* The RBD C API's rbd_discard method and the C++ API's Image::discard method
+  now enforce a maximum length of 2GB. This restriction prevents overflow of
+  the result code.
+
+* New OSDs will now use rocksdb for omap data by default, rather than
+  leveldb. omap is used by RGW bucket indexes and CephFS directories,
+  and when a single leveldb grows to 10s of GB with a high write or
+  delete workload, it can lead to high latency when leveldb's
+  single-threaded compaction cannot keep up. rocksdb supports multiple
+  threads for compaction, which avoids this problem.
+
+* The CephFS client now catches failures to clear dentries during startup
+  and refuses to start as consistency and untrimmable cache issues may
+  develop. The new option client_die_on_failed_dentry_invalidate (default:
+  true) may be turned off to allow the client to proceed (dangerous!).
+
+* In 10.2.10 and earlier releases, keyring caps were not checked for validity,
+  so the caps string could be anything. As of 10.2.11, caps strings are
+  validated and providing a keyring with an invalid caps string to, e.g.,
+  "ceph auth add" will result in an error.
+
+
+Changelog
+---------
+
+* admin: bump sphinx to 1.6 (`issue#21717 <http://tracker.ceph.com/issues/21717>`_, `pr#18166 <https://github.com/ceph/ceph/pull/18166>`_, Kefu Chai, Alfredo Deza)
+* auth: ceph auth add does not sanity-check caps (`issue#22525 <http://tracker.ceph.com/issues/22525>`_, `pr#21367 <https://github.com/ceph/ceph/pull/21367>`_, Jing Li, Nathan Cutler, Kefu Chai, Sage Weil)
+* build/ops: rpm: bump epoch ahead of ceph-common in RHEL base (`issue#20508 <http://tracker.ceph.com/issues/20508>`_, `pr#21190 <https://github.com/ceph/ceph/pull/21190>`_, Ken Dreyer)
+* build/ops: upstart: radosgw-all does not start on boot if ceph-base is not installed (`issue#18313 <http://tracker.ceph.com/issues/18313>`_, `pr#16294 <https://github.com/ceph/ceph/pull/16294>`_, Ken Dreyer)
+* ceph_authtool: add mode option (`issue#23513 <http://tracker.ceph.com/issues/23513>`_, `pr#21197 <https://github.com/ceph/ceph/pull/21197>`_, Sébastien Han)
+* ceph-disk: factor out the retry logic into a decorator (`issue#21728 <http://tracker.ceph.com/issues/21728>`_, `pr#18169 <https://github.com/ceph/ceph/pull/18169>`_, Kefu Chai)
+* ceph-disk: fix --runtime omission when enabling ceph-osd@$ID.service units for device-backed OSDs (`issue#21498 <http://tracker.ceph.com/issues/21498>`_, `pr#17942 <https://github.com/ceph/ceph/pull/17942>`_, Carl Xiong)
+* ceph-disk flake8 test fails on very old, and very new, versions of flake8 (`issue#22207 <http://tracker.ceph.com/issues/22207>`_, `pr#19153 <https://github.com/ceph/ceph/pull/19153>`_, Nathan Cutler)
+* cephfs: ceph.in: pass RADOS inst to LibCephFS (`issue#21406 <http://tracker.ceph.com/issues/21406>`_, `issue#21967 <http://tracker.ceph.com/issues/21967>`_, `pr#19907 <https://github.com/ceph/ceph/pull/19907>`_, Patrick Donnelly)
+* cephfs: client::mkdirs not handle well when two clients send mkdir request for a same dir (`issue#20592 <http://tracker.ceph.com/issues/20592>`_, `pr#20271 <https://github.com/ceph/ceph/pull/20271>`_, dongdong tao)
+* cephfs: client: prevent fallback to remount when dentry_invalidate_cb is true but root->dir is NULL (`issue#23211 <http://tracker.ceph.com/issues/23211>`_, `pr#21189 <https://github.com/ceph/ceph/pull/21189>`_, Zhi Zhang)
+* cephfs: fix tmap_upgrade crash (`issue#23529 <http://tracker.ceph.com/issues/23529>`_, `pr#21208 <https://github.com/ceph/ceph/pull/21208>`_, "Yan, Zheng")
+* cephfs: fuse client: ::rmdir() uses a deleted memory structure of dentry leads … (`issue#22536 <http://tracker.ceph.com/issues/22536>`_, `pr#19993 <https://github.com/ceph/ceph/pull/19993>`_, YunfeiGuan)
+* cephfs-journal-tool: add "set pool_id" option (`issue#22631 <http://tracker.ceph.com/issues/22631>`_, `pr#20111 <https://github.com/ceph/ceph/pull/20111>`_, dongdong tao)
+* cephfs-journal-tool: move shutdown to the deconstructor of MDSUtility (`issue#22734 <http://tracker.ceph.com/issues/22734>`_, `pr#20333 <https://github.com/ceph/ceph/pull/20333>`_, dongdong tao)
+* cephfs: osdc: "FAILED assert(bh->last_write_tid > tid)" in powercycle-wip-yuri-master-1.19.18-distro-basic-smithi (`issue#22741 <http://tracker.ceph.com/issues/22741>`_, `pr#20312 <https://github.com/ceph/ceph/pull/20312>`_, "Yan, Zheng")
+* cephfs: osdc/Journaler: make sure flush() writes enough data (`issue#22824 <http://tracker.ceph.com/issues/22824>`_, `pr#20435 <https://github.com/ceph/ceph/pull/20435>`_, "Yan, Zheng")
+* cephfs: Processes stuck waiting for write with ceph-fuse (`issue#22008 <http://tracker.ceph.com/issues/22008>`_, `issue#22207 <http://tracker.ceph.com/issues/22207>`_, `pr#19141 <https://github.com/ceph/ceph/pull/19141>`_, "Yan, Zheng")
+* ceph-fuse: failure to remount in startup test does not handle client_die_on_failed_remount properly (`issue#22269 <http://tracker.ceph.com/issues/22269>`_, `pr#21162 <https://github.com/ceph/ceph/pull/21162>`_, Patrick Donnelly)
+* ceph.in: bypass codec when writing raw binary data (`issue#23185 <http://tracker.ceph.com/issues/23185>`_, `pr#20763 <https://github.com/ceph/ceph/pull/20763>`_, Oleh Prypin)
+* ceph-objectstore-tool command to trim the pg log (`issue#23242 <http://tracker.ceph.com/issues/23242>`_, `pr#20882 <https://github.com/ceph/ceph/pull/20882>`_, Josh Durgin, David Zafman)
+* ceph-objectstore-tool: "$OBJ get-omaphdr" and "$OBJ list-omap" scan all pgs instead of using specific pg (`issue#21327 <http://tracker.ceph.com/issues/21327>`_, `pr#20284 <https://github.com/ceph/ceph/pull/20284>`_, David Zafman)
+* ceph.restart + ceph_manager.wait_for_clean is racy (`issue#15778 <http://tracker.ceph.com/issues/15778>`_, `pr#20508 <https://github.com/ceph/ceph/pull/20508>`_, Warren Usui, Sage Weil)
+* ceph_volume_client: fix setting caps for IDs (`issue#21501 <http://tracker.ceph.com/issues/21501>`_, `pr#18084 <https://github.com/ceph/ceph/pull/18084>`_, Ramana Raja)
+* class rbd.Image discard----OSError: [errno 2147483648] error discarding region (`issue#16465 <http://tracker.ceph.com/issues/16465>`_, `issue#21966 <http://tracker.ceph.com/issues/21966>`_, `pr#20287 <https://github.com/ceph/ceph/pull/20287>`_, Nathan Cutler, Huan Zhang, Jason Dillaman)
+* cli/crushtools/build.t sometimes fails in jenkins' make check run (`issue#21758 <http://tracker.ceph.com/issues/21758>`_, `pr#21158 <https://github.com/ceph/ceph/pull/21158>`_, Kefu Chai)
+* client reconnect gather race (`issue#22263 <http://tracker.ceph.com/issues/22263>`_, `pr#21163 <https://github.com/ceph/ceph/pull/21163>`_, "Yan, Zheng")
+* client: release revoking Fc after invalidate cache (`issue#22652 <http://tracker.ceph.com/issues/22652>`_, `pr#19975 <https://github.com/ceph/ceph/pull/19975>`_, "Yan, Zheng")
+* client: set client_try_dentry_invalidate to false by default (`issue#21423 <http://tracker.ceph.com/issues/21423>`_, `pr#17925 <https://github.com/ceph/ceph/pull/17925>`_, "Yan, Zheng")
+* [cli] rename of non-existent image results in seg fault (`issue#21248 <http://tracker.ceph.com/issues/21248>`_, `pr#20280 <https://github.com/ceph/ceph/pull/20280>`_, Jason Dillaman)
+* CLI unit formatting tests are broken (`issue#24733 <http://tracker.ceph.com/issues/24733>`_, `pr#22913 <https://github.com/ceph/ceph/pull/22913>`_, Jason Dillaman)
+* common: compute SimpleLRU's size with contents.size() instead of lru.… (`issue#22613 <http://tracker.ceph.com/issues/22613>`_, `pr#19978 <https://github.com/ceph/ceph/pull/19978>`_, Xuehan Xu)
+* common/config: set rocksdb_cache_size to OPT_U64 (`issue#22104 <http://tracker.ceph.com/issues/22104>`_, `pr#18850 <https://github.com/ceph/ceph/pull/18850>`_, Vikhyat Umrao, liuhongtong)
+* common: fix typo in rados bench write JSON output (`issue#24199 <http://tracker.ceph.com/issues/24199>`_, `pr#22407 <https://github.com/ceph/ceph/pull/22407>`_, Sandor Zeestraten)
+* config: lower default omap entries recovered at once (`issue#21897 <http://tracker.ceph.com/issues/21897>`_, `pr#19927 <https://github.com/ceph/ceph/pull/19927>`_, Josh Durgin)
+* core: Addition of online osd 'omap'compaction command (`issue#19592 <http://tracker.ceph.com/issues/19592>`_, `pr#17101 <https://github.com/ceph/ceph/pull/17101>`_, liuchang0812, Sage Weil)
+* core: global/signal_handler.cc: fix typo (`issue#21432 <http://tracker.ceph.com/issues/21432>`_, `pr#17883 <https://github.com/ceph/ceph/pull/17883>`_, Kefu Chai)
+* core: librados: Double free in rados_getxattrs_next (`issue#22042 <http://tracker.ceph.com/issues/22042>`_, `pr#20381 <https://github.com/ceph/ceph/pull/20381>`_, Gu Zhongyan)
+* core: Objecter::C_ObjectOperation_sparse_read throws/catches exceptions on -ENOENT (`issue#21844 <http://tracker.ceph.com/issues/21844>`_, `pr#18743 <https://github.com/ceph/ceph/pull/18743>`_, Jason Dillaman)
+* Deleting a pool with active notify linger ops can result in seg fault (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#22188 <https://github.com/ceph/ceph/pull/22188>`_, Kefu Chai, Jason Dillaman)
+* doc: clarify Path Restriction instructions (`issue#16906 <http://tracker.ceph.com/issues/16906>`_, `pr#19795 <https://github.com/ceph/ceph/pull/19795>`_, huanwen ren)
+* doc: clarify Path Restriction instructions (`issue#16906 <http://tracker.ceph.com/issues/16906>`_, `pr#19840 <https://github.com/ceph/ceph/pull/19840>`_, Drunkard Zhang)
+* doc: remove region from INSTALL CEPH OBJECT GATEWAY (`issue#21610 <http://tracker.ceph.com/issues/21610>`_, `pr#18303 <https://github.com/ceph/ceph/pull/18303>`_, Orit Wasserman)
+* Filestore rocksdb compaction readahead option not set by default (`issue#21505 <http://tracker.ceph.com/issues/21505>`_, `pr#20446 <https://github.com/ceph/ceph/pull/20446>`_, Mark Nelson)
+* follow-on: osd: be_select_auth_object() sanity check oi soid (`issue#20471 <http://tracker.ceph.com/issues/20471>`_, `pr#20622 <https://github.com/ceph/ceph/pull/20622>`_, David Zafman)
+* HashIndex: randomize split threshold by a configurable amount (`issue#15835 <http://tracker.ceph.com/issues/15835>`_, `pr#19906 <https://github.com/ceph/ceph/pull/19906>`_, Josh Durgin)
+* include/fs_types: fix unsigned integer overflow (`issue#22494 <http://tracker.ceph.com/issues/22494>`_, `pr#19611 <https://github.com/ceph/ceph/pull/19611>`_, runsisi)
+* install-deps.sh: point gcc to the one shipped by distro (`issue#22220 <http://tracker.ceph.com/issues/22220>`_, `pr#19461 <https://github.com/ceph/ceph/pull/19461>`_, Kefu Chai)
+* install-deps.sh: readlink /usr/bin/gcc not /usr/bin/x86_64-linux-gnu-gcc (`issue#22220 <http://tracker.ceph.com/issues/22220>`_, `pr#19521 <https://github.com/ceph/ceph/pull/19521>`_, Kefu Chai)
+* install-deps.sh: update g++ symlink also (`issue#22220 <http://tracker.ceph.com/issues/22220>`_, `pr#19656 <https://github.com/ceph/ceph/pull/19656>`_, Kefu Chai)
+* journal: Message too long error when appending journal (`issue#23526 <http://tracker.ceph.com/issues/23526>`_, `pr#21215 <https://github.com/ceph/ceph/pull/21215>`_, Mykola Golub)
+* [journal] tags are not being expired if no other clients are registered (`issue#21960 <http://tracker.ceph.com/issues/21960>`_, `pr#20282 <https://github.com/ceph/ceph/pull/20282>`_, Jason Dillaman)
+* legal: remove doc license ambiguity (`issue#23336 <http://tracker.ceph.com/issues/23336>`_, `pr#20999 <https://github.com/ceph/ceph/pull/20999>`_, Nathan Cutler)
+* librados: copy out data to users' buffer for xio (`issue#20616 <http://tracker.ceph.com/issues/20616>`_, `pr#17594 <https://github.com/ceph/ceph/pull/17594>`_, Vu Pham)
+* librbd: cannot clone all image-metas if we have more than 64 key/value pairs (`issue#21814 <http://tracker.ceph.com/issues/21814>`_, `pr#21228 <https://github.com/ceph/ceph/pull/21228>`_, PCzhangPC)
+* librbd: cannot copy all image-metas if we have more than 64 key/value pairs (`issue#21815 <http://tracker.ceph.com/issues/21815>`_, `pr#21203 <https://github.com/ceph/ceph/pull/21203>`_, PCzhangPC)
+* librbd: create+truncate for whole-object layered discards (`issue#23285 <http://tracker.ceph.com/issues/23285>`_, `pr#21219 <https://github.com/ceph/ceph/pull/21219>`_, Jason Dillaman)
+* librbd: list_children should not attempt to refresh image (`issue#21670 <http://tracker.ceph.com/issues/21670>`_, `pr#21224 <https://github.com/ceph/ceph/pull/21224>`_, Jason Dillaman)
+* librbd: object map batch update might cause OSD suicide timeout (`issue#22716 <http://tracker.ceph.com/issues/22716>`_, `issue#21797 <http://tracker.ceph.com/issues/21797>`_, `pr#21220 <https://github.com/ceph/ceph/pull/21220>`_, Song Shun, Jason Dillaman)
+* librbd: set deleted parent pointer to null (`issue#22158 <http://tracker.ceph.com/issues/22158>`_, `pr#19098 <https://github.com/ceph/ceph/pull/19098>`_, Jason Dillaman)
+* log: Fix AddressSanitizer: new-delete-type-mismatch (`issue#23324 <http://tracker.ceph.com/issues/23324>`_, `pr#21084 <https://github.com/ceph/ceph/pull/21084>`_, Brad Hubbard)
+* mds: FAILED assert(get_version() < pv) in CDir::mark_dirty (`issue#21584 <http://tracker.ceph.com/issues/21584>`_, `pr#21156 <https://github.com/ceph/ceph/pull/21156>`_, Yan, Zheng, "Yan, Zheng")
+* mds: fix dump last_sent (`issue#22562 <http://tracker.ceph.com/issues/22562>`_, `pr#19961 <https://github.com/ceph/ceph/pull/19961>`_, dongdong tao)
+* mds: fix integer overflow (`issue#21067 <http://tracker.ceph.com/issues/21067>`_, `pr#17188 <https://github.com/ceph/ceph/pull/17188>`_, Henry Chang)
+* mds: fix scrub crash (`issue#22730 <http://tracker.ceph.com/issues/22730>`_, `pr#20335 <https://github.com/ceph/ceph/pull/20335>`_, dongdong tao)
+* mds: session reference leak (`issue#22821 <http://tracker.ceph.com/issues/22821>`_, `pr#21175 <https://github.com/ceph/ceph/pull/21175>`_, Nathan Cutler, "Yan, Zheng")
+* mds: unbalanced auth_pin/auth_unpin in RecoveryQueue code (`issue#22647 <http://tracker.ceph.com/issues/22647>`_, `pr#20067 <https://github.com/ceph/ceph/pull/20067>`_, "Yan, Zheng")
+* mds: underwater dentry check in CDir::_omap_fetched is racy (`issue#23032 <http://tracker.ceph.com/issues/23032>`_, `pr#21185 <https://github.com/ceph/ceph/pull/21185>`_, Yan, Zheng)
+* mon/LogMonitor: call no_reply() on ignored log message (`issue#24180 <http://tracker.ceph.com/issues/24180>`_, `pr#22431 <https://github.com/ceph/ceph/pull/22431>`_, Sage Weil)
+* mon/MDSMonitor: no_reply on MMDSLoadTargets (`issue#23769 <http://tracker.ceph.com/issues/23769>`_, `pr#22189 <https://github.com/ceph/ceph/pull/22189>`_, Sage Weil)
+* mon/OSDMonitor.cc: fix expected_num_objects interpret error (`issue#22530 <http://tracker.ceph.com/issues/22530>`_, `pr#22050 <https://github.com/ceph/ceph/pull/22050>`_, Yang Honggang)
+* mon/OSDMonitor: fix dividing by zero in OSDUtilizationDumper (`issue#22662 <http://tracker.ceph.com/issues/22662>`_, `pr#20344 <https://github.com/ceph/ceph/pull/20344>`_, Mingxin Liu)
+* ObjectStore/StoreTest.FiemapHoles/3 fails with kstore (`issue#21716 <http://tracker.ceph.com/issues/21716>`_, `pr#20143 <https://github.com/ceph/ceph/pull/20143>`_, Kefu Chai, Ning Yao)
+* osd: also check the exsistence of clone obc for "CEPH_SNAPDIR" requests (`issue#17445 <http://tracker.ceph.com/issues/17445>`_, `pr#17707 <https://github.com/ceph/ceph/pull/17707>`_, Xuehan Xu)
+* osdc/Objecter: prevent double-invocation of linger op callback (`issue#23872 <http://tracker.ceph.com/issues/23872>`_, `pr#21754 <https://github.com/ceph/ceph/pull/21754>`_, Jason Dillaman)
+* osd: objecter sends out of sync with pg epochs for proxied ops (`issue#22123 <http://tracker.ceph.com/issues/22123>`_, `pr#20518 <https://github.com/ceph/ceph/pull/20518>`_, Sage Weil)
+* osd ops (sent and?) arrive at osd out of order (`issue#19133 <http://tracker.ceph.com/issues/19133>`_, `issue#19139 <http://tracker.ceph.com/issues/19139>`_, `pr#17893 <https://github.com/ceph/ceph/pull/17893>`_, Jianpeng Ma, Sage Weil)
+* osd: OSDMap cache assert on shutdown (`issue#21737 <http://tracker.ceph.com/issues/21737>`_, `pr#21184 <https://github.com/ceph/ceph/pull/21184>`_, Greg Farnum)
+* osd: osd_scrub_during_recovery only considers primary, not replicas (`issue#18206 <http://tracker.ceph.com/issues/18206>`_, `pr#17815 <https://github.com/ceph/ceph/pull/17815>`_, David Zafman)
+* osd/PrimaryLogPG: dump snap_trimq size (`issue#22448 <http://tracker.ceph.com/issues/22448>`_, `pr#21200 <https://github.com/ceph/ceph/pull/21200>`_, Piotr Dałek)
+* osd: recover_replicas: object added to missing set for backfill, but is not in recovering, error! (`issue#18162 <http://tracker.ceph.com/issues/18162>`_, `issue#14513 <http://tracker.ceph.com/issues/14513>`_, `pr#18690 <https://github.com/ceph/ceph/pull/18690>`_, huangjun, Adam C. Emerson, David Zafman)
+* osd: replica read can trigger cache promotion (`issue#20919 <http://tracker.ceph.com/issues/20919>`_, `pr#21199 <https://github.com/ceph/ceph/pull/21199>`_, Sage Weil)
+* osd: update heartbeat peers when a new OSD is added (`issue#18004 <http://tracker.ceph.com/issues/18004>`_, `pr#20108 <https://github.com/ceph/ceph/pull/20108>`_, Pan Liu)
+* performance: Only scan for omap corruption once (`issue#21328 <http://tracker.ceph.com/issues/21328>`_, `pr#18951 <https://github.com/ceph/ceph/pull/18951>`_, David Zafman)
+* qa: failures from pjd fstest (`issue#21383 <http://tracker.ceph.com/issues/21383>`_, `pr#21152 <https://github.com/ceph/ceph/pull/21152>`_, "Yan, Zheng")
+* qa: src/test/libcephfs/test.cc:376: Expected: (len) > (0), actual: -34 vs 0 (`issue#22221 <http://tracker.ceph.com/issues/22221>`_, `pr#21172 <https://github.com/ceph/ceph/pull/21172>`_, Patrick Donnelly)
+* qa: use xfs instead of btrfs w/ filestore (`issue#20169 <http://tracker.ceph.com/issues/20169>`_, `issue#20911 <http://tracker.ceph.com/issues/20911>`_, `pr#18165 <https://github.com/ceph/ceph/pull/18165>`_, Sage Weil)
+* qa: use xfs instead of btrfs w/ filestore (`issue#21481 <http://tracker.ceph.com/issues/21481>`_, `pr#17847 <https://github.com/ceph/ceph/pull/17847>`_, Patrick Donnelly)
+* radosgw: fix awsv4 header line sort order (`issue#21607 <http://tracker.ceph.com/issues/21607>`_, `pr#18080 <https://github.com/ceph/ceph/pull/18080>`_, Marcus Watts)
+* rbd: clean up warnings when mirror commands used on non-setup pool (`issue#21319 <http://tracker.ceph.com/issues/21319>`_, `pr#21227 <https://github.com/ceph/ceph/pull/21227>`_, Jason Dillaman)
+* rbd: disk usage on empty pool no longer returns an error message (`issue#22200 <http://tracker.ceph.com/issues/22200>`_, `pr#19186 <https://github.com/ceph/ceph/pull/19186>`_, Jason Dillaman)
+* [rbd] image-meta list does not return all entries (`issue#21179 <http://tracker.ceph.com/issues/21179>`_, `pr#20281 <https://github.com/ceph/ceph/pull/20281>`_, Jason Dillaman)
+* rbd: is_qemu_running in qemu_rebuild_object_map.sh and qemu_dynamic_features.sh may return false positive (`issue#23502 <http://tracker.ceph.com/issues/23502>`_, `pr#21207 <https://github.com/ceph/ceph/pull/21207>`_, Mykola Golub)
+* rbd: [journal] allocating a new tag after acquiring the lock should use on-disk committed position (`issue#22945 <http://tracker.ceph.com/issues/22945>`_, `pr#21206 <https://github.com/ceph/ceph/pull/21206>`_, Jason Dillaman)
+* rbd: librbd: filter out potential race with image rename (`issue#18435 <http://tracker.ceph.com/issues/18435>`_, `pr#19855 <https://github.com/ceph/ceph/pull/19855>`_, Jason Dillaman)
+* rbd ls -l crashes with SIGABRT (`issue#21558 <http://tracker.ceph.com/issues/21558>`_, `pr#19801 <https://github.com/ceph/ceph/pull/19801>`_, Jason Dillaman)
+* rbd-mirror: cluster watcher should ensure it has latest OSD map (`issue#22461 <http://tracker.ceph.com/issues/22461>`_, `pr#19644 <https://github.com/ceph/ceph/pull/19644>`_, Jason Dillaman)
+* rbd-mirror: fix potential infinite loop when formatting status message (`issue#22932 <http://tracker.ceph.com/issues/22932>`_, `pr#20418 <https://github.com/ceph/ceph/pull/20418>`_, Mykola Golub)
+* rbd-mirror: ignore permission errors on rbd_mirroring object (`issue#20571 <http://tracker.ceph.com/issues/20571>`_, `pr#21225 <https://github.com/ceph/ceph/pull/21225>`_, Jason Dillaman)
+* rbd-mirror: strip environment/CLI overrides for remote cluster (`issue#21894 <http://tracker.ceph.com/issues/21894>`_, `pr#21223 <https://github.com/ceph/ceph/pull/21223>`_, Jason Dillaman)
+* [rbd-nbd] Fedora does not register resize events (`issue#22131 <http://tracker.ceph.com/issues/22131>`_, `pr#19115 <https://github.com/ceph/ceph/pull/19115>`_, Jason Dillaman)
+* rbd-nbd: fix ebusy when do map (`issue#23528 <http://tracker.ceph.com/issues/23528>`_, `pr#21232 <https://github.com/ceph/ceph/pull/21232>`_, Li Wang)
+* rbd: possible deadlock in various maintenance operations (`issue#22120 <http://tracker.ceph.com/issues/22120>`_, `pr#20285 <https://github.com/ceph/ceph/pull/20285>`_, Jason Dillaman)
+* rbd: rbd crashes during map (`issue#21808 <http://tracker.ceph.com/issues/21808>`_, `pr#18843 <https://github.com/ceph/ceph/pull/18843>`_, Peter Keresztes Schmidt)
+* rbd: rbd-mirror split brain test case can have a false-positive failure until teuthology (`issue#22485 <http://tracker.ceph.com/issues/22485>`_, `pr#21205 <https://github.com/ceph/ceph/pull/21205>`_, Jason Dillaman)
+* rbd: TestLibRBD.RenameViaLockOwner may still fail with -ENOENT (`issue#23068 <http://tracker.ceph.com/issues/23068>`_, `pr#20627 <https://github.com/ceph/ceph/pull/20627>`_, Mykola Golub)
+* repair_test fails due to race with osd start (`issue#20705 <http://tracker.ceph.com/issues/20705>`_, `pr#20146 <https://github.com/ceph/ceph/pull/20146>`_, Sage Weil)
+* rgw: 15912 15673 (Fix duplicate tag removal during GC, cls/refcount: store and use list of retired tags) (`issue#20107 <http://tracker.ceph.com/issues/20107>`_, `pr#16708 <https://github.com/ceph/ceph/pull/16708>`_, Jens Rosenboom)
+* rgw: abort in listing mapped nbd devices when running in a container (`issue#22012 <http://tracker.ceph.com/issues/22012>`_, `issue#22011 <http://tracker.ceph.com/issues/22011>`_, `pr#20286 <https://github.com/ceph/ceph/pull/20286>`_, Li Wang, Pan Liu)
+* rgw: add ability to sync user stats from admin api (`issue#21301 <http://tracker.ceph.com/issues/21301>`_, `pr#20179 <https://github.com/ceph/ceph/pull/20179>`_, Nathan Johnson)
+* rgw: add cors header rule check in cors option request (`issue#22002 <http://tracker.ceph.com/issues/22002>`_, `pr#19057 <https://github.com/ceph/ceph/pull/19057>`_, yuliyang)
+* rgw: add radosgw-admin sync error trim to trim sync error log (`issue#23287 <http://tracker.ceph.com/issues/23287>`_, `pr#21210 <https://github.com/ceph/ceph/pull/21210>`_, fang yuxiang)
+* rgw: add xml output header in RGWCopyObj_ObjStore_S3 response msg (`issue#22416 <http://tracker.ceph.com/issues/22416>`_, `pr#19887 <https://github.com/ceph/ceph/pull/19887>`_, Enming Zhang)
+* rgw: automated trimming of datalog and mdlog (`issue#18227 <http://tracker.ceph.com/issues/18227>`_, `pr#20061 <https://github.com/ceph/ceph/pull/20061>`_, Casey Bodley)
+* rgw: bi list entry count incremented on error, distorting error code (`issue#21205 <http://tracker.ceph.com/issues/21205>`_, `pr#18207 <https://github.com/ceph/ceph/pull/18207>`_, Nathan Cutler)
+* rgw: boto3 v4 SignatureDoesNotMatch failure due to sorting of sse-kms headers (`issue#21832 <http://tracker.ceph.com/issues/21832>`_, `pr#18772 <https://github.com/ceph/ceph/pull/18772>`_, Nathan Cutler)
+* rgw: bucket resharding should not update bucket ACL or user stats (`issue#22124 <http://tracker.ceph.com/issues/22124>`_, `pr#20421 <https://github.com/ceph/ceph/pull/20421>`_, Orit Wasserman)
+* rgw: copying part without http header x-amz-copy-source-range will be mistaken for copying object (`issue#22729 <http://tracker.ceph.com/issues/22729>`_, `pr#21294 <https://github.com/ceph/ceph/pull/21294>`_, Malcolm Lee)
+* rgw: core dump, recursive lock of RGWKeystoneTokenCache (`issue#23171 <http://tracker.ceph.com/issues/23171>`_, `pr#20639 <https://github.com/ceph/ceph/pull/20639>`_, Mark Kogan, Adam Kupczyk)
+* rgw: data sync of versioned objects, note updating bi marker (`issue#18885 <http://tracker.ceph.com/issues/18885>`_, `pr#21213 <https://github.com/ceph/ceph/pull/21213>`_, Yehuda Sadeh)
+* rgw: dont log EBUSY errors in 'sync error list' (`issue#22473 <http://tracker.ceph.com/issues/22473>`_, `pr#19908 <https://github.com/ceph/ceph/pull/19908>`_, Casey Bodley)
+* rgw: ECANCELED in rgw_get_system_obj() leads to infinite loop (`issue#17996 <http://tracker.ceph.com/issues/17996>`_, `pr#20561 <https://github.com/ceph/ceph/pull/20561>`_, Yehuda Sadeh)
+* rgw: file deadlock on lru evicting (`issue#22736 <http://tracker.ceph.com/issues/22736>`_, `pr#20076 <https://github.com/ceph/ceph/pull/20076>`_, Matt Benjamin)
+* rgw: file write error (`issue#21455 <http://tracker.ceph.com/issues/21455>`_, `pr#18304 <https://github.com/ceph/ceph/pull/18304>`_, Yao Zongyou)
+* rgw: fix chained cache invalidation to prevent cache size growth (`issue#22410 <http://tracker.ceph.com/issues/22410>`_, `pr#19469 <https://github.com/ceph/ceph/pull/19469>`_, Mark Kogan)
+* rgw: fix doubled underscore with s3/swift server-side copy (`issue#22529 <http://tracker.ceph.com/issues/22529>`_, `pr#19747 <https://github.com/ceph/ceph/pull/19747>`_, Matt Benjamin)
+* rgw: fix GET website response error code (`issue#22272 <http://tracker.ceph.com/issues/22272>`_, `pr#19488 <https://github.com/ceph/ceph/pull/19488>`_, Dmitry Plyakin)
+* rgw: fix index update in dir_suggest_changes (`issue#24280 <http://tracker.ceph.com/issues/24280>`_, `pr#22677 <https://github.com/ceph/ceph/pull/22677>`_, Tianshan Qu)
+* rgw: fix marker encoding problem (`issue#20463 <http://tracker.ceph.com/issues/20463>`_, `pr#17731 <https://github.com/ceph/ceph/pull/17731>`_, Orit Wasserman, Marcus Watts)
+* rgw: fix swift anonymous access (`issue#22259 <http://tracker.ceph.com/issues/22259>`_, `pr#19194 <https://github.com/ceph/ceph/pull/19194>`_, Marcus Watts)
+* rgw: Fix swift object expiry not deleting objects (`issue#22084 <http://tracker.ceph.com/issues/22084>`_, `pr#18925 <https://github.com/ceph/ceph/pull/18925>`_, Pavan Rallabhandi)
+* rgw: fix the bug that part's index can't be removed after completing (`issue#19604 <http://tracker.ceph.com/issues/19604>`_, `pr#16763 <https://github.com/ceph/ceph/pull/16763>`_, Zhang Shaowen, Matt Benjamin)
+* rgw: fix the max-uploads parameter not work (`issue#22825 <http://tracker.ceph.com/issues/22825>`_, `pr#20479 <https://github.com/ceph/ceph/pull/20479>`_, Xin Liao)
+* rgw: inefficient buffer usage for PUTs (`issue#23207 <http://tracker.ceph.com/issues/23207>`_, `pr#21098 <https://github.com/ceph/ceph/pull/21098>`_, Marcus Watts)
+* rgw: libcurl & ssl fixes (`issue#22951 <http://tracker.ceph.com/issues/22951>`_, `issue#23203 <http://tracker.ceph.com/issues/23203>`_, `issue#23162 <http://tracker.ceph.com/issues/23162>`_, `pr#20749 <https://github.com/ceph/ceph/pull/20749>`_, Marcus Watts, Abhishek Lekshmanan, Jesse Williamson)
+* rgw: list bucket which enable versioning get wrong result when user marker (`issue#21500 <http://tracker.ceph.com/issues/21500>`_, `pr#20291 <https://github.com/ceph/ceph/pull/20291>`_, yuliyang)
+* rgw: log includes zero byte sometimes (`issue#20037 <http://tracker.ceph.com/issues/20037>`_, `pr#17151 <https://github.com/ceph/ceph/pull/17151>`_, Abhishek Lekshmanan)
+* rgw:  make init env methods return an error (`issue#23039 <http://tracker.ceph.com/issues/23039>`_, `pr#20800 <https://github.com/ceph/ceph/pull/20800>`_, Abhishek Lekshmanan)
+* RGW: Multipart upload may double the quota (`issue#21586 <http://tracker.ceph.com/issues/21586>`_, `pr#18121 <https://github.com/ceph/ceph/pull/18121>`_, Sibei Gao, Matt Benjamin)
+* rgw: multisite: data sync status advances despite failure in RGWListBucketIndexesCR (`issue#21735 <http://tracker.ceph.com/issues/21735>`_, `pr#20269 <https://github.com/ceph/ceph/pull/20269>`_, Casey Bodley)
+* rgw: multisite: Get bucket location which is located in another zonegroup, will return 301 Moved Permanently (`issue#21125 <http://tracker.ceph.com/issues/21125>`_, `pr#18305 <https://github.com/ceph/ceph/pull/18305>`_, Shasha Lu, lvshuhua, Jiaying Ren)
+* rgw: null instance mtime incorrect when enable versioning (`issue#21743 <http://tracker.ceph.com/issues/21743>`_, `pr#20262 <https://github.com/ceph/ceph/pull/20262>`_, Shasha Lu)
+* rgw: radosgw-admin: add an option to reset user stats (`issue#23335 <http://tracker.ceph.com/issues/23335>`_, `issue#23322 <http://tracker.ceph.com/issues/23322>`_, `pr#20877 <https://github.com/ceph/ceph/pull/20877>`_, Abhishek Lekshmanan)
+* rgw: release cls lock if taken in RGWCompleteMultipart (`issue#21596 <http://tracker.ceph.com/issues/21596>`_, `issue#22368 <http://tracker.ceph.com/issues/22368>`_, `pr#18116 <https://github.com/ceph/ceph/pull/18116>`_, Casey Bodley, Matt Benjamin)
+* rgw: resharding needs to set back the bucket ACL after link (`issue#22742 <http://tracker.ceph.com/issues/22742>`_, `pr#20039 <https://github.com/ceph/ceph/pull/20039>`_, Orit Wasserman)
+* rgw: resolve Random 500 errors in Swift PutObject (22517) (`issue#22517 <http://tracker.ceph.com/issues/22517>`_, `issue#21560 <http://tracker.ceph.com/issues/21560>`_, `pr#19769 <https://github.com/ceph/ceph/pull/19769>`_, Adam C. Emerson, Matt Benjamin)
+* rgw: rgw_file: recursive lane lock can occur in LRU drain (`issue#20374 <http://tracker.ceph.com/issues/20374>`_, `pr#17149 <https://github.com/ceph/ceph/pull/17149>`_, Matt Benjamin)
+* rgw: S3 POST policy should not require Content-Type (`issue#20201 <http://tracker.ceph.com/issues/20201>`_, `pr#19635 <https://github.com/ceph/ceph/pull/19635>`_, Matt Benjamin)
+* rgw: s3website error handler uses original object name (`issue#23201 <http://tracker.ceph.com/issues/23201>`_, `issue#20307 <http://tracker.ceph.com/issues/20307>`_, `pr#21100 <https://github.com/ceph/ceph/pull/21100>`_, liuhong, Casey Bodley)
+* rgw: segfaults after running radosgw-admin data sync init (`issue#22083 <http://tracker.ceph.com/issues/22083>`_, `pr#19783 <https://github.com/ceph/ceph/pull/19783>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: segmentation fault when starting radosgw after reverting .rgw.root (`issue#21996 <http://tracker.ceph.com/issues/21996>`_, `pr#20292 <https://github.com/ceph/ceph/pull/20292>`_, Orit Wasserman, Casey Bodley)
+* rgw: stale bucket index entry remains after object deletion (`issue#22555 <http://tracker.ceph.com/issues/22555>`_, `pr#20293 <https://github.com/ceph/ceph/pull/20293>`_, J. Eric Ivancich)
+* rgw: system user can't delete bucket completely (`issue#22248 <http://tracker.ceph.com/issues/22248>`_, `pr#21212 <https://github.com/ceph/ceph/pull/21212>`_, Casey Bodley)
+* rgw: tcmalloc (`issue#23469 <http://tracker.ceph.com/issues/23469>`_, `pr#21073 <https://github.com/ceph/ceph/pull/21073>`_, Matt Benjamin)
+* rgw: upldate the max-buckets when the quota is uploaded (`issue#22745 <http://tracker.ceph.com/issues/22745>`_, `pr#20496 <https://github.com/ceph/ceph/pull/20496>`_, zhaokun)
+* rgw: user creation can overwrite existing user even if different uid is given (`issue#21685 <http://tracker.ceph.com/issues/21685>`_, `pr#20074 <https://github.com/ceph/ceph/pull/20074>`_, Casey Bodley)
+* RHEL 7.3 Selinux denials at OSD start (`issue#19200 <http://tracker.ceph.com/issues/19200>`_, `pr#18780 <https://github.com/ceph/ceph/pull/18780>`_, Boris Ranto)
+* scrub errors not cleared on replicas can cause inconsistent pg state when replica takes over primary (`issue#23267 <http://tracker.ceph.com/issues/23267>`_, `pr#21194 <https://github.com/ceph/ceph/pull/21194>`_, David Zafman)
+* snapset xattr corruption propagated from primary to other shards (`issue#20186 <http://tracker.ceph.com/issues/20186>`_, `issue#18409 <http://tracker.ceph.com/issues/18409>`_, `issue#21907 <http://tracker.ceph.com/issues/21907>`_, `pr#20331 <https://github.com/ceph/ceph/pull/20331>`_, David Zafman)
+* systemd: Add explicit Before=ceph.target (`issue#21477 <http://tracker.ceph.com/issues/21477>`_, `pr#17841 <https://github.com/ceph/ceph/pull/17841>`_, Tim Serong)
+* table of contents doesn't render for luminous/jewel docs (`issue#23780 <http://tracker.ceph.com/issues/23780>`_, `pr#21503 <https://github.com/ceph/ceph/pull/21503>`_, Alfredo Deza)
+* test: Adjust for Jewel quirk caused of differences with master (`issue#23006 <http://tracker.ceph.com/issues/23006>`_, `pr#20463 <https://github.com/ceph/ceph/pull/20463>`_, David Zafman)
+* test/CMakeLists: disable test_pidfile.sh (`issue#20975 <http://tracker.ceph.com/issues/20975>`_, `pr#20557 <https://github.com/ceph/ceph/pull/20557>`_, Sage Weil)
+* test_health_warnings.sh can fail (`issue#21121 <http://tracker.ceph.com/issues/21121>`_, `pr#20289 <https://github.com/ceph/ceph/pull/20289>`_, Sage Weil)
+* test/librbd: fixed metadata tests under upgrade scenarios (`issue#21911 <http://tracker.ceph.com/issues/21911>`_, `pr#18548 <https://github.com/ceph/ceph/pull/18548>`_, Jason Dillaman)
+* test/librbd: utilize unique pool for cache tier testing (`issue#11502 <http://tracker.ceph.com/issues/11502>`_, `pr#20524 <https://github.com/ceph/ceph/pull/20524>`_, Jason Dillaman)
+* tests: rbd_mirror_helpers.sh request_resync_image function saves image id to wrong variable (`issue#21663 <http://tracker.ceph.com/issues/21663>`_, `pr#19804 <https://github.com/ceph/ceph/pull/19804>`_, Jason Dillaman)
+* tests: test_admin_socket.sh may fail on wait_for_clean (`issue#23499 <http://tracker.ceph.com/issues/23499>`_, `pr#21125 <https://github.com/ceph/ceph/pull/21125>`_, Mykola Golub)
+* tests: tests/librbd: updated test_notify to handle new release lock semantics (`issue#21912 <http://tracker.ceph.com/issues/21912>`_, `pr#18560 <https://github.com/ceph/ceph/pull/18560>`_, Jason Dillaman)
+* tests: unittest_pglog timeout (`issue#23504 <http://tracker.ceph.com/issues/23504>`_, `issue#18030 <http://tracker.ceph.com/issues/18030>`_, `pr#21135 <https://github.com/ceph/ceph/pull/21135>`_, Nathan Cutler, Loic Dachary)
+* tools: ceph-objectstore-tool set-size should clear data-digest (`issue#22112 <http://tracker.ceph.com/issues/22112>`_, `pr#20070 <https://github.com/ceph/ceph/pull/20070>`_, David Zafman)
+* Ubuntu amd64 client can not discover the ubuntu arm64 ceph cluster (`issue#19705 <http://tracker.ceph.com/issues/19705>`_, `pr#18294 <https://github.com/ceph/ceph/pull/18294>`_, Kefu Chai)
+
+
+
+v10.2.10 Jewel
+==============
+
+This point release brings a number of important bugfixes in all major
+components of Ceph, we recommend all Jewel 10.2.x users to upgrade.
+
+For a detailed list of changes refer to :download: `the complete changelog <../changelog/v10.2.10txt>`
+
+
+Notable Changes
+-----------------
+* build/ops: Add fix subcommand to ceph-disk, fix SELinux denials, and speed up upgrade from non-SELinux enabled ceph to an SELinux enabled one (`issue#20077 <http://tracker.ceph.com/issues/20077>`_, `issue#20184 <http://tracker.ceph.com/issues/20184>`_, `issue#19545 <http://tracker.ceph.com/issues/19545>`_, `pr#14346 <https://github.com/ceph/ceph/pull/14346>`_, Boris Ranto)
+* build/ops: deb: Fix logrotate packaging (`issue#19938 <http://tracker.ceph.com/issues/19938>`_, `pr#15428 <https://github.com/ceph/ceph/pull/15428>`_, Nathan Cutler)
+* build/ops: extended, customizable systemd ceph-disk timeout (`issue#18740 <http://tracker.ceph.com/issues/18740>`_, `pr#15051 <https://github.com/ceph/ceph/pull/15051>`_, Alexey Sheplyakov)
+* build/ops: rpm: fix python-Sphinx package name for SUSE (`issue#19924 <http://tracker.ceph.com/issues/19924>`_, `pr#15196 <https://github.com/ceph/ceph/pull/15196>`_, Nathan Cutler, Jan Matejek)
+* build/ops: rpm: set subman cron attributes in spec file (`issue#20074 <http://tracker.ceph.com/issues/20074>`_, `pr#15473 <https://github.com/ceph/ceph/pull/15473>`_, Thomas Serlin)
+* cephfs: ceph-fuse segfaults at mount time, assert in ceph::log::Log::stop (`issue#18157 <http://tracker.ceph.com/issues/18157>`_, `pr#16963 <https://github.com/ceph/ceph/pull/16963>`_, Greg Farnum)
+* cephfs: df reports negative disk "used" value when quota exceed (`issue#20178 <http://tracker.ceph.com/issues/20178>`_, `pr#16151 <https://github.com/ceph/ceph/pull/16151>`_, John Spray)
+* cephfs: get_quota_root sends lookupname op for every buffered write (`issue#20945 <http://tracker.ceph.com/issues/20945>`_, `pr#17396 <https://github.com/ceph/ceph/pull/17396>`_, Dan van der Ster)
+* cephfs: osdc/Filer: truncate large file party by party (`issue#19755 <http://tracker.ceph.com/issues/19755>`_, `pr#15442 <https://github.com/ceph/ceph/pull/15442>`_, "Yan, Zheng")
+* core: an OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed (`issue#20544 <http://tracker.ceph.com/issues/20544>`_, `issue#16878 <http://tracker.ceph.com/issues/16878>`_, `issue#19733 <http://tracker.ceph.com/issues/19733>`_, `issue#15912 <http://tracker.ceph.com/issues/15912>`_, `pr#15050 <https://github.com/ceph/ceph/pull/15050>`_, Sage Weil, David Zafman)
+* core: disable skewed utilization warning by default (`issue#20730 <http://tracker.ceph.com/issues/20730>`_, `pr#17210 <https://github.com/ceph/ceph/pull/17210>`_, David Zafman)
+* core: interval_set: optimize intersect_of insert operations (`issue#21229 <http://tracker.ceph.com/issues/21229>`_, `pr#17514 <https://github.com/ceph/ceph/pull/17514>`_, Zac Medico)
+* core: kv: let ceph_logger destructed after db reset (`issue#21336 <http://tracker.ceph.com/issues/21336>`_, `pr#17626 <https://github.com/ceph/ceph/pull/17626>`_, wumingqiao)
+* core: test_envlibrados_for_rocksdb.yaml fails on crypto restart (`issue#19741 <http://tracker.ceph.com/issues/19741>`_, `pr#16293 <https://github.com/ceph/ceph/pull/16293>`_, Kefu Chai)
+* libradosstriper silently fails to delete empty objects in jewel (`issue#20325 <http://tracker.ceph.com/issues/20325>`_, `pr#15760 <https://github.com/ceph/ceph/pull/15760>`_, Stan K)
+* librbd: fail IO request when exclusive lock cannot be obtained (`issue#20168 <http://tracker.ceph.com/issues/20168>`_, `issue#21251 <http://tracker.ceph.com/issues/21251>`_, `pr#17402 <https://github.com/ceph/ceph/pull/17402>`_, Jason Dillaman)
+* librbd: prevent self-blacklisting during break lock (`issue#18666 <http://tracker.ceph.com/issues/18666>`_, `pr#17412 <https://github.com/ceph/ceph/pull/17412>`_, Jason Dillaman)
+* librbd: reacquire lock should update lock owner client id (`issue#19929 <http://tracker.ceph.com/issues/19929>`_, `pr#17385 <https://github.com/ceph/ceph/pull/17385>`_, Jason Dillaman)
+* mds: damage reporting by ino number is useless (`issue#18509 <http://tracker.ceph.com/issues/18509>`_, `issue#16016 <http://tracker.ceph.com/issues/16016>`_, `pr#14699 <https://github.com/ceph/ceph/pull/14699>`_, John Spray, Michal Jarzabek)
+* mds: log rotation doesn't work if mds has respawned (`issue#19291 <http://tracker.ceph.com/issues/19291>`_, `pr#14673 <https://github.com/ceph/ceph/pull/14673>`_, Patrick Donnelly)
+* mds: save  projected path into inode_t::stray_prior_path (`issue#20340 <http://tracker.ceph.com/issues/20340>`_, `pr#16150 <https://github.com/ceph/ceph/pull/16150>`_, "Yan, Zheng")
+* mon: crash on shutdown, lease_ack_timeout event (`issue#19825 <http://tracker.ceph.com/issues/19825>`_, `pr#15083 <https://github.com/ceph/ceph/pull/15083>`_, Kefu Chai, Michal Jarzabek, Alexey Sheplyakov)
+* mon: Disallow enabling 'hashpspool' option to a pool without some kind of --i-understand-this-will-remap-all-pgs flag (`issue#18468 <http://tracker.ceph.com/issues/18468>`_, `pr#13507 <https://github.com/ceph/ceph/pull/13507>`_, Vikhyat Umrao)
+* mon: factor mon_osd_full_ratio into MAX AVAIL calc (`issue#18522 <http://tracker.ceph.com/issues/18522>`_, `pr#15236 <https://github.com/ceph/ceph/pull/15236>`_, Sage Weil)
+* mon: fail to form large quorum; msg/async busy loop (`issue#20230 <http://tracker.ceph.com/issues/20230>`_, `pr#15726 <https://github.com/ceph/ceph/pull/15726>`_, Haomai Wang, Michal Jarzabek)
+* mon: fix force_pg_create pg stuck in creating bug (`issue#18298 <http://tracker.ceph.com/issues/18298>`_, `pr#17008 <https://github.com/ceph/ceph/pull/17008>`_, Alexey Sheplyakov)
+* mon: osd crush set crushmap need sanity check (`issue#19302 <http://tracker.ceph.com/issues/19302>`_, `pr#16144 <https://github.com/ceph/ceph/pull/16144>`_, Loic Dachary)
+* osd: Add heartbeat message for Jumbo Frames (MTU 9000) (`issue#20087 <http://tracker.ceph.com/issues/20087>`_, `issue#20323 <http://tracker.ceph.com/issues/20323>`_, `pr#16059 <https://github.com/ceph/ceph/pull/16059>`_, Piotr Dałek, Sage Weil, Greg Farnum)
+* osd: fix infinite loops in fiemap (`issue#19996 <http://tracker.ceph.com/issues/19996>`_, `pr#15189 <https://github.com/ceph/ceph/pull/15189>`_, Sage Weil, Ning Yao)
+* osd: leaked MOSDMap (`issue#18293 <http://tracker.ceph.com/issues/18293>`_, `pr#14943 <https://github.com/ceph/ceph/pull/14943>`_, Sage Weil)
+* osd: objecter full_try behavior not consistent with osd (`issue#19430 <http://tracker.ceph.com/issues/19430>`_, `pr#15474 <https://github.com/ceph/ceph/pull/15474>`_, Sage Weil)
+* osd: omap threadpool heartbeat is only reset every 100 values (`issue#20375 <http://tracker.ceph.com/issues/20375>`_, `pr#16167 <https://github.com/ceph/ceph/pull/16167>`_, Josh Durgin)
+* osd: osd_internal_types: wake snaptrimmer on put_read lock, too (`issue#19131 <http://tracker.ceph.com/issues/19131>`_, `pr#16015 <https://github.com/ceph/ceph/pull/16015>`_, Sage Weil)
+* osd: PrimaryLogPG: do not call on_shutdown() if (pg.deleting) (`issue#19902 <http://tracker.ceph.com/issues/19902>`_, `pr#15065 <https://github.com/ceph/ceph/pull/15065>`_, Kefu Chai)
+* osd: rados ls on pool with no access returns no error (`issue#20043 <http://tracker.ceph.com/issues/20043>`_, `issue#19790 <http://tracker.ceph.com/issues/19790>`_, `pr#16473 <https://github.com/ceph/ceph/pull/16473>`_, Nathan Cutler, Kefu Chai, John Spray, Sage Weil, Brad Hubbard)
+* osd: ReplicatedPG: solve cache tier osd high memory consumption (`issue#20464 <http://tracker.ceph.com/issues/20464>`_, `pr#16169 <https://github.com/ceph/ceph/pull/16169>`_, Peng Xie)
+* osd: Reset() snaptrimmer on shutdown and do not default-abort on leaked pg refs (`issue#19931 <http://tracker.ceph.com/issues/19931>`_, `pr#15322 <https://github.com/ceph/ceph/pull/15322>`_, Greg Farnum)
+* osd: scrub_to specifies clone ver, but transaction include head write ver (`issue#20041 <http://tracker.ceph.com/issues/20041>`_, `pr#16405 <https://github.com/ceph/ceph/pull/16405>`_, David Zafman)
+* osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal (`issue#20427 <http://tracker.ceph.com/issues/20427>`_, `pr#15947 <https://github.com/ceph/ceph/pull/15947>`_, Alexey Sheplyakov)
+* qa: add a sleep after restarting osd before "tell"ing it (`issue#16239 <http://tracker.ceph.com/issues/16239>`_, `pr#15475 <https://github.com/ceph/ceph/pull/15475>`_, Kefu Chai)
+* rbd: api: is_exclusive_lock_owner shouldn't return -EBUSY (`issue#20182 <http://tracker.ceph.com/issues/20182>`_, `pr#16296 <https://github.com/ceph/ceph/pull/16296>`_, Jason Dillaman)
+* rbd: cli: ensure positional arguments exist before casting (`issue#20185 <http://tracker.ceph.com/issues/20185>`_, `pr#16295 <https://github.com/ceph/ceph/pull/16295>`_, Jason Dillaman)
+* rbd: cli: map with cephx disabled results in error message (`issue#19035 <http://tracker.ceph.com/issues/19035>`_, `pr#16297 <https://github.com/ceph/ceph/pull/16297>`_, Jason Dillaman)
+* rbd: default features should be negotiated with the OSD (`issue#17010 <http://tracker.ceph.com/issues/17010>`_, `pr#14874 <https://github.com/ceph/ceph/pull/14874>`_, Mykola Golub, Jason Dillaman)
+* rbd: Enabling mirroring for a pool with clones may fail (`issue#19798 <http://tracker.ceph.com/issues/19798>`_, `issue#19130 <http://tracker.ceph.com/issues/19130>`_, `pr#14663 <https://github.com/ceph/ceph/pull/14663>`_, Mykola Golub, Jason Dillaman)
+* rbd-mirror: image sync should send NOCACHE advise flag (`issue#17127 <http://tracker.ceph.com/issues/17127>`_, `pr#16285 <https://github.com/ceph/ceph/pull/16285>`_, Mykola Golub)
+* rbd: object-map: batch updates during trim operation (`issue#17356 <http://tracker.ceph.com/issues/17356>`_, `pr#15460 <https://github.com/ceph/ceph/pull/15460>`_, Mykola Golub, Venky Shankar, Nathan Cutler)
+* rbd: Potential IO hang if image is flattened while read request is in-flight (`issue#19832 <http://tracker.ceph.com/issues/19832>`_, `pr#15464 <https://github.com/ceph/ceph/pull/15464>`_, Jason Dillaman)
+* rbd: rbd_clone_copy_on_read ineffective with exclusive-lock (`issue#18888 <http://tracker.ceph.com/issues/18888>`_, `pr#16124 <https://github.com/ceph/ceph/pull/16124>`_, Nathan Cutler, Venky Shankar, Jason Dillaman)
+* rbd: rbd-mirror: ensure missing images are re-synced when detected (`issue#19811 <http://tracker.ceph.com/issues/19811>`_, `pr#15488 <https://github.com/ceph/ceph/pull/15488>`_, Jason Dillaman)
+* rbd: rbd-mirror: failover and failback of unmodified image results in split-brain (`issue#19858 <http://tracker.ceph.com/issues/19858>`_, `pr#14977 <https://github.com/ceph/ceph/pull/14977>`_, Jason Dillaman)
+* rbd: rbd-nbd: kernel reported invalid device size (0, expected 1073741824) (`issue#19871 <http://tracker.ceph.com/issues/19871>`_, `pr#15463 <https://github.com/ceph/ceph/pull/15463>`_, Mykola Golub)
+* rgw: add the remove-x-delete feature to cancel swift object expiration (`issue#19074 <http://tracker.ceph.com/issues/19074>`_, `pr#14659 <https://github.com/ceph/ceph/pull/14659>`_, Jing Wenjun)
+* rgw: aws4: add rgw_s3_auth_aws4_force_boto2_compat conf option (`issue#16463 <http://tracker.ceph.com/issues/16463>`_, `pr#17009 <https://github.com/ceph/ceph/pull/17009>`_, Javier M. Mellid)
+* rgw: bucket index check in radosgw-admin removes valid index (`issue#18470 <http://tracker.ceph.com/issues/18470>`_, `pr#16856 <https://github.com/ceph/ceph/pull/16856>`_, Zhang Shaowen, Pavan Rallabhandi)
+* rgw: cls: ceph::timespan tag_timeout wrong units (`issue#20380 <http://tracker.ceph.com/issues/20380>`_, `pr#16289 <https://github.com/ceph/ceph/pull/16289>`_, Matt Benjamin)
+* rgw: Custom data header support (`issue#19644 <http://tracker.ceph.com/issues/19644>`_, `pr#15966 <https://github.com/ceph/ceph/pull/15966>`_, Pavan Rallabhandi)
+* rgw: datalog trim can't work as expected (`issue#20190 <http://tracker.ceph.com/issues/20190>`_, `pr#16299 <https://github.com/ceph/ceph/pull/16299>`_, Zhang Shaowen)
+* rgw: Delete non-empty bucket in slave zonegroup (`issue#19313 <http://tracker.ceph.com/issues/19313>`_, `pr#15477 <https://github.com/ceph/ceph/pull/15477>`_, Zhang Shaowen)
+* rgw: Do not decrement stats cache when the cache values are zero (`issue#20661 <http://tracker.ceph.com/issues/20661>`_, `issue#20934 <http://tracker.ceph.com/issues/20934>`_, `pr#16720 <https://github.com/ceph/ceph/pull/16720>`_, Aleksei Gutikov, Pavan Rallabhandi)
+* rgw: fix crash caused by shard id out of range when listing data log (`issue#19732 <http://tracker.ceph.com/issues/19732>`_, `pr#15465 <https://github.com/ceph/ceph/pull/15465>`_, redickwang)
+* rgw: fix hangs in RGWRealmReloader::reload on SIGHUP (`issue#20686 <http://tracker.ceph.com/issues/20686>`_, `pr#17281 <https://github.com/ceph/ceph/pull/17281>`_, fang.yuxiang)
+* rgw: fix infinite loop in rest api for log list (`issue#20386 <http://tracker.ceph.com/issues/20386>`_, `pr#15988 <https://github.com/ceph/ceph/pull/15988>`_, xierui, Casey Bodley)
+* rgw: fix race in RGWCompleteMultipart (`issue#20861 <http://tracker.ceph.com/issues/20861>`_, `pr#16767 <https://github.com/ceph/ceph/pull/16767>`_, Abhishek Varshney, Matt Benjamin)
+* rgw: Fix up to 1000 entries at a time in check_bad_index_multipart (`issue#20772 <http://tracker.ceph.com/issues/20772>`_, `pr#16880 <https://github.com/ceph/ceph/pull/16880>`_, Orit Wasserman, Matt Benjamin)
+* rgw: folders starting with _ underscore are not in bucket index (`issue#19432 <http://tracker.ceph.com/issues/19432>`_, `pr#16276 <https://github.com/ceph/ceph/pull/16276>`_, Giovani Rinaldi, Orit Wasserman)
+* rgw: 'gc list --include-all' command infinite loop the first 1000 items (`issue#19978 <http://tracker.ceph.com/issues/19978>`_, `pr#15719 <https://github.com/ceph/ceph/pull/15719>`_, Shasha Lu, fang yuxiang)
+* rgw: meta sync thread crash at RGWMetaSyncShardCR (`issue#20251 <http://tracker.ceph.com/issues/20251>`_, `pr#16711 <https://github.com/ceph/ceph/pull/16711>`_, fang yuxiang, Nathan Cutler)
+* rgw: multipart copy-part remove '/' for s3 java sdk request header (`issue#20075 <http://tracker.ceph.com/issues/20075>`_, `pr#16266 <https://github.com/ceph/ceph/pull/16266>`_, donglingpeng)
+* rgw: multipart parts on versioned bucket create versioned bucket index entries (`issue#19604 <http://tracker.ceph.com/issues/19604>`_, `issue#17964 <http://tracker.ceph.com/issues/17964>`_, `pr#17278 <https://github.com/ceph/ceph/pull/17278>`_, Zhang Shaowen)
+* rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards (`issue#19745 <http://tracker.ceph.com/issues/19745>`_, `pr#15450 <https://github.com/ceph/ceph/pull/15450>`_, Shasha Lu)
+* rgw: multisite: bucket zonegroup redirect not working (`issue#19488 <http://tracker.ceph.com/issues/19488>`_, `pr#15448 <https://github.com/ceph/ceph/pull/15448>`_, Casey Bodley)
+* rgw: multisite: fixes for meta sync across periods (`issue#18639 <http://tracker.ceph.com/issues/18639>`_, `pr#15556 <https://github.com/ceph/ceph/pull/15556>`_, Casey Bodley)
+* rgw: multisite: lock is not released when RGWMetaSyncShardCR::full_sync() fails to write marker (`issue#18077 <http://tracker.ceph.com/issues/18077>`_, `pr#17155 <https://github.com/ceph/ceph/pull/17155>`_, Zhang Shaowen)
+* rgw: multisite: log_meta on secondary zone causes continuous loop of metadata sync (`issue#20357 <http://tracker.ceph.com/issues/20357>`_, `issue#20244 <http://tracker.ceph.com/issues/20244>`_, `pr#17148 <https://github.com/ceph/ceph/pull/17148>`_, Orit Wasserman, Casey Bodley)
+* rgw: multisite: memory leak on failed lease in RGWDataSyncShardCR (`issue#19861 <http://tracker.ceph.com/issues/19861>`_, `issue#19834 <http://tracker.ceph.com/issues/19834>`_, `issue#19446 <http://tracker.ceph.com/issues/19446>`_, `pr#15457 <https://github.com/ceph/ceph/pull/15457>`_, Casey Bodley, weiqiaomiao)
+* rgw: multisite: operating bucket's acl&cors is not restricted on slave zone (`issue#16888 <http://tracker.ceph.com/issues/16888>`_, `pr#15453 <https://github.com/ceph/ceph/pull/15453>`_, Casey Bodley, Shasha Lu, Guo Zhandong)
+* rgw: multisite: realm rename does not propagate to other clusters (`issue#19746 <http://tracker.ceph.com/issues/19746>`_, `pr#15454 <https://github.com/ceph/ceph/pull/15454>`_, Casey Bodley)
+* rgw: multisite: rest api fails to decode large period on "period commit" (`issue#19505 <http://tracker.ceph.com/issues/19505>`_, `pr#15447 <https://github.com/ceph/ceph/pull/15447>`_, Casey Bodley)
+* rgw: multisite: RGWPeriodPuller does not call RGWPeriod::reflect() on new period (`issue#19816 <http://tracker.ceph.com/issues/19816>`_, `issue#19817 <http://tracker.ceph.com/issues/19817>`_, `pr#17167 <https://github.com/ceph/ceph/pull/17167>`_, Casey Bodley)
+* rgw: multisite: RGWRadosRemoveOmapKeysCR::request_complete return val is wrong (`issue#20539 <http://tracker.ceph.com/issues/20539>`_, `pr#17156 <https://github.com/ceph/ceph/pull/17156>`_, Shasha Lu)
+* rgw: not initialized pointer cause rgw crash with ec data pool (`issue#20542 <http://tracker.ceph.com/issues/20542>`_, `pr#17164 <https://github.com/ceph/ceph/pull/17164>`_, Aleksei Gutikov, fang yuxiang)
+* rgw: radosgw-admin: bucket rm with --bypass-gc and without --purge-data doesn't throw error message (`issue#20688 <http://tracker.ceph.com/issues/20688>`_, `pr#17159 <https://github.com/ceph/ceph/pull/17159>`_, Abhishek Varshney)
+* rgw: radosgw-admin data sync run crash (`issue#20423 <http://tracker.ceph.com/issues/20423>`_, `pr#17165 <https://github.com/ceph/ceph/pull/17165>`_, Shasha Lu)
+* rgw: radosgw-admin: fix bucket limit check argparse, div(0) (`issue#20966 <http://tracker.ceph.com/issues/20966>`_, `pr#16952 <https://github.com/ceph/ceph/pull/16952>`_, Matt Benjamin)
+* rgw: reduce log level of 'storing entry at' in cls_log (`issue#19835 <http://tracker.ceph.com/issues/19835>`_, `pr#15455 <https://github.com/ceph/ceph/pull/15455>`_, Willem Jan Withagen)
+* rgw: remove unnecessary 'error in read_id for object name: default' (`issue#19922 <http://tracker.ceph.com/issues/19922>`_, `pr#15197 <https://github.com/ceph/ceph/pull/15197>`_, weiqiaomiao)
+* rgw: replace '+' with "%20" in canonical query string for s3 v4 auth (`issue#20501 <http://tracker.ceph.com/issues/20501>`_, `pr#16951 <https://github.com/ceph/ceph/pull/16951>`_, Zhang Shaowen, Matt Benjamin)
+* rgw: rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get (`issue#16072 <http://tracker.ceph.com/issues/16072>`_, `pr#16268 <https://github.com/ceph/ceph/pull/16268>`_, zhao kun)
+* rgw: rgw_file: cannot delete bucket w/uxattrs (`issue#20061 <http://tracker.ceph.com/issues/20061>`_, `issue#20047 <http://tracker.ceph.com/issues/20047>`_, `issue#19214 <http://tracker.ceph.com/issues/19214>`_, `issue#20045 <http://tracker.ceph.com/issues/20045>`_, `pr#15459 <https://github.com/ceph/ceph/pull/15459>`_, Matt Benjamin)
+* rgw: rgw_file: fix size and (c|m)time unix attrs in write_finish (`issue#19653 <http://tracker.ceph.com/issues/19653>`_, `pr#15449 <https://github.com/ceph/ceph/pull/15449>`_, Matt Benjamin)
+* rgw: rgw_file:  incorrect lane lock behavior in evict_block() (`issue#21141 <http://tracker.ceph.com/issues/21141>`_, `pr#17597 <https://github.com/ceph/ceph/pull/17597>`_, Matt Benjamin)
+* rgw: rgw_file: prevent conflict of mkdir between restarts (`issue#20275 <http://tracker.ceph.com/issues/20275>`_, `pr#17147 <https://github.com/ceph/ceph/pull/17147>`_, Gui Hecheng)
+* rgw: rgw_file:  v3 write timer does not close open handles (`issue#19932 <http://tracker.ceph.com/issues/19932>`_, `pr#15456 <https://github.com/ceph/ceph/pull/15456>`_, Matt Benjamin)
+* rgw: Segmentation fault when exporting rgw bucket in nfs-ganesha (`issue#20663 <http://tracker.ceph.com/issues/20663>`_, `pr#17285 <https://github.com/ceph/ceph/pull/17285>`_, Matt Benjamin)
+* rgw: send data-log list infinitely (`issue#20951 <http://tracker.ceph.com/issues/20951>`_, `pr#17287 <https://github.com/ceph/ceph/pull/17287>`_, fang.yuxiang)
+* rgw: set latest object's acl failed (`issue#18649 <http://tracker.ceph.com/issues/18649>`_, `pr#15451 <https://github.com/ceph/ceph/pull/15451>`_, Zhang Shaowen)
+* rgw: Truncated objects (`issue#20107 <http://tracker.ceph.com/issues/20107>`_, `pr#17166 <https://github.com/ceph/ceph/pull/17166>`_, Yehuda Sadeh)
+* rgw: uninitialized memory is accessed during creation of bucket's metadata (`issue#20774 <http://tracker.ceph.com/issues/20774>`_, `pr#17280 <https://github.com/ceph/ceph/pull/17280>`_, Radoslaw Zarzynski)
+* rgw: usage logging on tenated buckets causes invalid memory reads (`issue#20779 <http://tracker.ceph.com/issues/20779>`_, `pr#17279 <https://github.com/ceph/ceph/pull/17279>`_, Radoslaw Zarzynski)
+* rgw: user quota did not work well on multipart upload (`issue#19285 <http://tracker.ceph.com/issues/19285>`_, `issue#19602 <http://tracker.ceph.com/issues/19602>`_, `pr#17277 <https://github.com/ceph/ceph/pull/17277>`_, Zhang Shaowen)
+* rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions (`issue#19886 <http://tracker.ceph.com/issues/19886>`_, `pr#16316 <https://github.com/ceph/ceph/pull/16316>`_, Zhang Shaowen)
+* rgw: when uploading objects continuously into a versioned bucket, some objects will not sync (`issue#18208 <http://tracker.ceph.com/issues/18208>`_, `pr#15452 <https://github.com/ceph/ceph/pull/15452>`_, lvshuhua)
+* tools: ceph cli: Rados object in state configuring race (`issue#16477 <http://tracker.ceph.com/issues/16477>`_, `pr#15762 <https://github.com/ceph/ceph/pull/15762>`_, Loic Dachary)
+* tools: ceph-disk: dmcrypt cluster must default to ceph (`issue#20893 <http://tracker.ceph.com/issues/20893>`_, `pr#16870 <https://github.com/ceph/ceph/pull/16870>`_, Loic Dachary)
+* tools: ceph-disk: don't activate suppressed journal devices (`issue#19489 <http://tracker.ceph.com/issues/19489>`_, `pr#16703 <https://github.com/ceph/ceph/pull/16703>`_, David Disseldorp)
+* tools: ceph-disk: separate ceph-osd --check-needs-\* logs (`issue#19888 <http://tracker.ceph.com/issues/19888>`_, `pr#15503 <https://github.com/ceph/ceph/pull/15503>`_, Loic Dachary)
+* tools: ceph-disk: systemd unit timesout too quickly (`issue#20229 <http://tracker.ceph.com/issues/20229>`_, `pr#17133 <https://github.com/ceph/ceph/pull/17133>`_, Loic Dachary)
+* tools: ceph-disk: Use stdin for 'config-key put' command (`issue#21059 <http://tracker.ceph.com/issues/21059>`_, `pr#17084 <https://github.com/ceph/ceph/pull/17084>`_, Brad Hubbard, Loic Dachary, Sage Weil)
+* tools: libradosstriper processes arbitrary printf placeholders in user input (`issue#20240 <http://tracker.ceph.com/issues/20240>`_, `pr#17574 <https://github.com/ceph/ceph/pull/17574>`_, Stan K)
+
+
+v10.2.9 Jewel
+=============
+
+This point release fixes a regression introduced in v10.2.8.
+
+We recommend that all Jewel users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.9.txt>`.
+
+
+Notable Changes
+---------------
+
+* cephfs: Damaged MDS with 10.2.8 (`issue#20599 <http://tracker.ceph.com/issues/20599>`_, `pr#16282 <https://github.com/ceph/ceph/pull/16282>`_, Nathan Cutler)
+
+
+
+v10.2.8 Jewel
+=============
+
+This point release brought a number of important bugfixes in all major
+components of Ceph. However, it also introduced a regression that could cause
+MDS damage, and a new release, v10.2.9, was published to address this.
+Therefore, Jewel users should *not* upgrade to this version - instead, we
+recommend upgrading directly to v10.2.9.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.8.txt>`.
+
+OSD Removal Caveat
+------------------
+
+There was a bug introduced in Jewel (#19119) that broke the mapping behavior
+when an "out" OSD that still existed in the CRUSH map was removed with 'osd rm'.
+This could result in 'misdirected op' and other errors.  The bug is now fixed,
+but the fix itself introduces the same risk because the behavior may vary between
+clients and OSDs.  To avoid problems, please ensure that all OSDs are removed
+from the CRUSH map before deleting them.  That is, be sure to do::
+
+    ceph osd crush rm osd.123
+
+before::
+
+    ceph osd rm osd.123
+
+Snap Trimmer Improvements
+-------------------------
+
+This release greatly improves control and throttling of the snap trimmer. It
+introduces the "osd max trimming pgs" option (defaulting to 2), which limits
+how many PGs on an OSD can be trimming snapshots at a time. And it restores
+the safe use of the "osd snap trim sleep" option, which defaults to 0 but
+otherwise adds the given number of seconds in delay between every dispatch
+of trim operations to the underlying system.
+
+Other Notable Changes
+---------------------
+
+* build/ops: "osd marked itself down" will not recognised if host runs mon + osd on shutdown/reboot (`issue#18516 <http://tracker.ceph.com/issues/18516>`_, `pr#13492 <https://github.com/ceph/ceph/pull/13492>`_, Boris Ranto)
+* build/ops: ceph-base package missing dependency for psmisc (`issue#19129 <http://tracker.ceph.com/issues/19129>`_, `pr#13786 <https://github.com/ceph/ceph/pull/13786>`_, Nathan Cutler)
+* build/ops: enable build of ceph-resource-agents package on rpm-based os (`issue#17613 <http://tracker.ceph.com/issues/17613>`_, `issue#19546 <http://tracker.ceph.com/issues/19546>`_, `pr#13606 <https://github.com/ceph/ceph/pull/13606>`_, Nathan Cutler)
+* build/ops: rbdmap.service not included in debian packaging (jewel-only) (`issue#19547 <http://tracker.ceph.com/issues/19547>`_, `pr#14383 <https://github.com/ceph/ceph/pull/14383>`_, Ken Dreyer)
+* cephfs: Journaler may execute on_safe contexts prematurely (`issue#20055 <http://tracker.ceph.com/issues/20055>`_, `pr#15468 <https://github.com/ceph/ceph/pull/15468>`_, "Yan, Zheng")
+* cephfs: MDS assert failed when shutting down (`issue#19204 <http://tracker.ceph.com/issues/19204>`_, `pr#14683 <https://github.com/ceph/ceph/pull/14683>`_, John Spray)
+* cephfs: MDS goes readonly writing backtrace for a file whose data pool has been removed (`issue#19401 <http://tracker.ceph.com/issues/19401>`_, `pr#14682 <https://github.com/ceph/ceph/pull/14682>`_, John Spray)
+* cephfs: MDS server crashes due to inconsistent metadata (`issue#19406 <http://tracker.ceph.com/issues/19406>`_, `pr#14676 <https://github.com/ceph/ceph/pull/14676>`_, John Spray)
+* cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command (`issue#16709 <http://tracker.ceph.com/issues/16709>`_, `pr#14674 <https://github.com/ceph/ceph/pull/14674>`_, John Spray)
+* cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) (`issue#18914 <http://tracker.ceph.com/issues/18914>`_, `pr#14685 <https://github.com/ceph/ceph/pull/14685>`_, "Yan, Zheng")
+* cephfs: Test failure: test_open_inode (`issue#18661 <http://tracker.ceph.com/issues/18661>`_, `pr#14669 <https://github.com/ceph/ceph/pull/14669>`_, John Spray)
+* cephfs: The mount point break off when mds switch hanppened (`issue#19437 <http://tracker.ceph.com/issues/19437>`_, `pr#14679 <https://github.com/ceph/ceph/pull/14679>`_, Guan yunfei)
+* cephfs: ceph-fuse does not recover after lost connection to MDS (`issue#16743 <http://tracker.ceph.com/issues/16743>`_, `issue#18757 <http://tracker.ceph.com/issues/18757>`_, `pr#14698 <https://github.com/ceph/ceph/pull/14698>`_, Kefu Chai, Henrik Korkuc, Patrick Donnelly)
+* cephfs: client: fix the cross-quota rename boundary check conditions (`issue#18699 <http://tracker.ceph.com/issues/18699>`_, `pr#14667 <https://github.com/ceph/ceph/pull/14667>`_, Greg Farnum)
+* cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file (`issue#19033 <http://tracker.ceph.com/issues/19033>`_, `pr#14684 <https://github.com/ceph/ceph/pull/14684>`_, Yang Honggang)
+* cephfs: non-local quota changes not visible until some IO is done (`issue#17939 <http://tracker.ceph.com/issues/17939>`_, `pr#15466 <https://github.com/ceph/ceph/pull/15466>`_, John Spray, Nathan Cutler)
+* cephfs: normalize file open flags internally used by cephfs (`issue#18872 <http://tracker.ceph.com/issues/18872>`_, `issue#19890 <http://tracker.ceph.com/issues/19890>`_, `pr#15000 <https://github.com/ceph/ceph/pull/15000>`_, Jan Fajerski, "Yan, Zheng")
+* common: monitor creation with IPv6 public network segfaults (`issue#19371 <http://tracker.ceph.com/issues/19371>`_, `pr#14324 <https://github.com/ceph/ceph/pull/14324>`_, Fabian Grünbichler)
+* common: radosstriper: protect aio_write API from calls with 0 bytes (`issue#14609 <http://tracker.ceph.com/issues/14609>`_, `pr#13254 <https://github.com/ceph/ceph/pull/13254>`_, Sebastien Ponce)
+* core: Objecter::epoch_barrier isn't respected in _op_submit() (`issue#19396 <http://tracker.ceph.com/issues/19396>`_, `pr#14332 <https://github.com/ceph/ceph/pull/14332>`_, Ilya Dryomov)
+* core: clear divergent_priors set off disk (`issue#17916 <http://tracker.ceph.com/issues/17916>`_, `pr#14596 <https://github.com/ceph/ceph/pull/14596>`_, Greg Farnum)
+* core: improve snap trimming, enable restriction of parallelism (`issue#19241 <http://tracker.ceph.com/issues/19241>`_, `pr#14492 <https://github.com/ceph/ceph/pull/14492>`_, Samuel Just, Greg Farnum)
+* core: os/filestore/HashIndex: be loud about splits (`issue#18235 <http://tracker.ceph.com/issues/18235>`_, `pr#13788 <https://github.com/ceph/ceph/pull/13788>`_, Dan van der Ster)
+* core: os/filestore: fix clang static check warn use-after-free (`issue#19311 <http://tracker.ceph.com/issues/19311>`_, `pr#14044 <https://github.com/ceph/ceph/pull/14044>`_, liuchang0812, yaoning)
+* core: transient jerasure unit test failures (`issue#18070 <http://tracker.ceph.com/issues/18070>`_, `issue#17762 <http://tracker.ceph.com/issues/17762>`_, `issue#18128 <http://tracker.ceph.com/issues/18128>`_, `issue#17951 <http://tracker.ceph.com/issues/17951>`_, `pr#14701 <https://github.com/ceph/ceph/pull/14701>`_, Kefu Chai, Pan Liu, Loic Dachary, Jason Dillaman)
+* core: two instances of omap_digest mismatch (`issue#18533 <http://tracker.ceph.com/issues/18533>`_, `pr#14204 <https://github.com/ceph/ceph/pull/14204>`_, Samuel Just, David Zafman)
+* doc: Improvements to crushtool manpage (`issue#19649 <http://tracker.ceph.com/issues/19649>`_, `pr#14635 <https://github.com/ceph/ceph/pull/14635>`_, Loic Dachary, Nathan Cutler)
+* doc: PendingReleaseNotes: note about 19119 (`issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#13732 <https://github.com/ceph/ceph/pull/13732>`_, Sage Weil)
+* doc: admin ops: fix the quota section (`issue#19397 <http://tracker.ceph.com/issues/19397>`_, `pr#14654 <https://github.com/ceph/ceph/pull/14654>`_, Chu, Hua-Rong)
+* doc: radosgw-admin: add the 'object stat' command to usage (`issue#19013 <http://tracker.ceph.com/issues/19013>`_, `pr#13872 <https://github.com/ceph/ceph/pull/13872>`_, Pavan Rallabhandi)
+* doc: rgw S3 create bucket should not do response in json (`issue#18889 <http://tracker.ceph.com/issues/18889>`_, `pr#13874 <https://github.com/ceph/ceph/pull/13874>`_, Abhishek Lekshmanan)
+* fs: Invalid error code returned by MDS is causing a kernel client WARNING (`issue#19205 <http://tracker.ceph.com/issues/19205>`_, `pr#13831 <https://github.com/ceph/ceph/pull/13831>`_, Jan Fajerski, xie xingguo)
+* librbd: Incomplete declaration for ContextWQ in librbd/Journal.h (`issue#18862 <http://tracker.ceph.com/issues/18862>`_, `pr#14152 <https://github.com/ceph/ceph/pull/14152>`_, Boris Ranto)
+* librbd: Issues with C API image metadata retrieval functions (`issue#19588 <http://tracker.ceph.com/issues/19588>`_, `pr#14666 <https://github.com/ceph/ceph/pull/14666>`_, Mykola Golub)
+* librbd: Possible deadlock performing a synchronous API action while refresh in-progress (`issue#18419 <http://tracker.ceph.com/issues/18419>`_, `pr#13154 <https://github.com/ceph/ceph/pull/13154>`_, Jason Dillaman)
+* librbd: is_exclusive_lock_owner API should ping OSD (`issue#19287 <http://tracker.ceph.com/issues/19287>`_, `pr#14481 <https://github.com/ceph/ceph/pull/14481>`_, Jason Dillaman)
+* librbd: remove image header lock assertions (`issue#18244 <http://tracker.ceph.com/issues/18244>`_, `pr#13809 <https://github.com/ceph/ceph/pull/13809>`_, Jason Dillaman)
+* mds: C_MDSInternalNoop::complete doesn't free itself (`issue#19501 <http://tracker.ceph.com/issues/19501>`_, `pr#14677 <https://github.com/ceph/ceph/pull/14677>`_, "Yan, Zheng")
+* mds: Too many stat ops when trying to probe a large file (`issue#19955 <http://tracker.ceph.com/issues/19955>`_, `pr#15472 <https://github.com/ceph/ceph/pull/15472>`_, "Yan, Zheng")
+* mds: avoid reusing deleted inode in StrayManager::_purge_stray_logged (`issue#18877 <http://tracker.ceph.com/issues/18877>`_, `pr#14670 <https://github.com/ceph/ceph/pull/14670>`_, Zhi Zhang)
+* mds: enable start when session ino info is corrupt (`issue#19708 <http://tracker.ceph.com/issues/19708>`_, `issue#16842 <http://tracker.ceph.com/issues/16842>`_, `pr#14700 <https://github.com/ceph/ceph/pull/14700>`_, John Spray)
+* mds: fragment space check can cause replayed request fail (`issue#18660 <http://tracker.ceph.com/issues/18660>`_, `pr#14668 <https://github.com/ceph/ceph/pull/14668>`_, "Yan, Zheng")
+* mds: heartbeat timeout during rejoin, when working with large amount of caps/inodes (`issue#19118 <http://tracker.ceph.com/issues/19118>`_, `pr#14672 <https://github.com/ceph/ceph/pull/14672>`_, John Spray)
+* mds: issue new caps when sending reply to client (`issue#19635 <http://tracker.ceph.com/issues/19635>`_, `pr#15438 <https://github.com/ceph/ceph/pull/15438>`_, "Yan, Zheng")
+* mon: OSDMonitor: make 'osd crush move ...' work on osds (`issue#18587 <http://tracker.ceph.com/issues/18587>`_, `pr#13261 <https://github.com/ceph/ceph/pull/13261>`_, Sage Weil)
+* mon: fix 'sortbitwise' warning on jewel (`issue#20578 <http://tracker.ceph.com/issues/20578>`_, `pr#15208 <https://github.com/ceph/ceph/pull/15208>`_, huanwen ren, Sage Weil)
+* mon: make get_mon_log_message() atomic (`issue#19427 <http://tracker.ceph.com/issues/19427>`_, `pr#14587 <https://github.com/ceph/ceph/pull/14587>`_, Kefu Chai)
+* mon: remove bad rocksdb option (`issue#19392 <http://tracker.ceph.com/issues/19392>`_, `pr#14236 <https://github.com/ceph/ceph/pull/14236>`_, Sage Weil)
+* msg: IPv6 Heartbeat packets are not marked with DSCP QoS - simple messenger (`issue#18887 <http://tracker.ceph.com/issues/18887>`_, `pr#13450 <https://github.com/ceph/ceph/pull/13450>`_, Yan Jun, Robin H. Johnson)
+* msg: set close on exec flag (`issue#16390 <http://tracker.ceph.com/issues/16390>`_, `pr#13585 <https://github.com/ceph/ceph/pull/13585>`_, Kefu Chai)
+* osd: --flush-journal: sporadic segfaults on exit (`issue#18820 <http://tracker.ceph.com/issues/18820>`_, `pr#13477 <https://github.com/ceph/ceph/pull/13477>`_, Alexey Sheplyakov)
+* osd: Give requested scrubs a higher priority (`issue#15789 <http://tracker.ceph.com/issues/15789>`_, `pr#14686 <https://github.com/ceph/ceph/pull/14686>`_, David Zafman)
+* osd: Implement asynchronous scrub sleep (`issue#19986 <http://tracker.ceph.com/issues/19986>`_, `issue#19497 <http://tracker.ceph.com/issues/19497>`_, `pr#15529 <https://github.com/ceph/ceph/pull/15529>`_, Brad Hubbard)
+* osd: Object level shard errors are tracked and used if no auth available (`issue#20089 <http://tracker.ceph.com/issues/20089>`_, `pr#15416 <https://github.com/ceph/ceph/pull/15416>`_, David Zafman)
+* osd: ReplicatedPG: try with pool's use-gmt setting if hitset archive not found (`issue#19185 <http://tracker.ceph.com/issues/19185>`_, `pr#13827 <https://github.com/ceph/ceph/pull/13827>`_, Kefu Chai)
+* osd: allow client throttler to be adjusted on-fly, without restart (`issue#18791 <http://tracker.ceph.com/issues/18791>`_, `pr#13214 <https://github.com/ceph/ceph/pull/13214>`_, Piotr Dałek)
+* osd: bypass readonly ops when osd full (`issue#19394 <http://tracker.ceph.com/issues/19394>`_, `pr#14181 <https://github.com/ceph/ceph/pull/14181>`_, Jianpeng Ma, yaoning)
+* osd: degraded and misplaced status output inaccurate (`issue#18619 <http://tracker.ceph.com/issues/18619>`_, `pr#14325 <https://github.com/ceph/ceph/pull/14325>`_, David Zafman)
+* osd: new added OSD always down when full flag is set (`issue#15025 <http://tracker.ceph.com/issues/15025>`_, `pr#14326 <https://github.com/ceph/ceph/pull/14326>`_, Mingxin Liu)
+* osd: pg_pool_t::encode(): be compatible with Hammer <= 0.94.6 (`issue#19508 <http://tracker.ceph.com/issues/19508>`_, `pr#14392 <https://github.com/ceph/ceph/pull/14392>`_, Alexey Sheplyakov)
+* osd: pre-jewel "osd rm" incrementals are misinterpreted (`issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#13884 <https://github.com/ceph/ceph/pull/13884>`_, Ilya Dryomov)
+* osd: preserve allocation hint attribute during recovery (`issue#19083 <http://tracker.ceph.com/issues/19083>`_, `pr#13647 <https://github.com/ceph/ceph/pull/13647>`_, yaoning)
+* osd: promote throttle parameters are reversed (`issue#19773 <http://tracker.ceph.com/issues/19773>`_, `pr#14791 <https://github.com/ceph/ceph/pull/14791>`_, Mark Nelson)
+* osd: reindex properly on pg log split (`issue#18975 <http://tracker.ceph.com/issues/18975>`_, `pr#14047 <https://github.com/ceph/ceph/pull/14047>`_, Alexey Sheplyakov)
+* osd: restrict want_acting to up+acting on recovery completion (`issue#18929 <http://tracker.ceph.com/issues/18929>`_, `pr#13541 <https://github.com/ceph/ceph/pull/13541>`_, Sage Weil)
+* rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly (`issue#18335 <http://tracker.ceph.com/issues/18335>`_, `pr#13932 <https://github.com/ceph/ceph/pull/13932>`_, Mykola Golub, Alexey Sheplyakov)
+* rbd: [api] temporarily restrict (rbd\_)mirror_peer_add from adding multiple peers (`issue#19256 <http://tracker.ceph.com/issues/19256>`_, `pr#14664 <https://github.com/ceph/ceph/pull/14664>`_, Jason Dillaman)
+* rbd: qemu crash triggered by network issues (`issue#18436 <http://tracker.ceph.com/issues/18436>`_, `pr#13244 <https://github.com/ceph/ceph/pull/13244>`_, Jason Dillaman)
+* rbd: rbd --pool=x rename y z does not work (`issue#18326 <http://tracker.ceph.com/issues/18326>`_, `pr#14148 <https://github.com/ceph/ceph/pull/14148>`_, Gaurav Kumar Garg)
+* rbd: systemctl stop rbdmap unmaps all rbds and not just the ones in /etc/ceph/rbdmap (`issue#18884 <http://tracker.ceph.com/issues/18884>`_, `issue#18262 <http://tracker.ceph.com/issues/18262>`_, `pr#14083 <https://github.com/ceph/ceph/pull/14083>`_, David Disseldorp, Nathan Cutler)
+* rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log (`issue#18980 <http://tracker.ceph.com/issues/18980>`_, `pr#14064 <https://github.com/ceph/ceph/pull/14064>`_, Casey Bodley)
+* rgw: 'radosgw-admin sync status' on master zone of non-master zonegroup (`issue#18091 <http://tracker.ceph.com/issues/18091>`_, `pr#13779 <https://github.com/ceph/ceph/pull/13779>`_, Jing Wenjun)
+* rgw: Change loglevel to 20 for 'System already converted' message (`issue#18919 <http://tracker.ceph.com/issues/18919>`_, `pr#13834 <https://github.com/ceph/ceph/pull/13834>`_, Vikhyat Umrao)
+* rgw: Use decoded URI when verifying TempURL (`issue#18590 <http://tracker.ceph.com/issues/18590>`_, `pr#13724 <https://github.com/ceph/ceph/pull/13724>`_, Alexey Sheplyakov)
+* rgw: a few cases where rgw_obj is incorrectly initialized (`issue#19096 <http://tracker.ceph.com/issues/19096>`_, `pr#13842 <https://github.com/ceph/ceph/pull/13842>`_, Yehuda Sadeh)
+* rgw: add apis to support ragweed suite (`issue#19804 <http://tracker.ceph.com/issues/19804>`_, `pr#14851 <https://github.com/ceph/ceph/pull/14851>`_, Yehuda Sadeh)
+* rgw: add bucket size limit check to radosgw-admin (`issue#17925 <http://tracker.ceph.com/issues/17925>`_, `pr#14787 <https://github.com/ceph/ceph/pull/14787>`_, Matt Benjamin)
+* rgw: allow system users to read SLO parts (`issue#19027 <http://tracker.ceph.com/issues/19027>`_, `pr#14752 <https://github.com/ceph/ceph/pull/14752>`_, Casey Bodley)
+* rgw: don't return skew time in pre-signed url (`issue#18828 <http://tracker.ceph.com/issues/18828>`_, `issue#18829 <http://tracker.ceph.com/issues/18829>`_, `pr#14605 <https://github.com/ceph/ceph/pull/14605>`_, liuchang0812)
+* rgw: failure to create s3 type subuser from admin rest api (`issue#16682 <http://tracker.ceph.com/issues/16682>`_, `pr#14815 <https://github.com/ceph/ceph/pull/14815>`_, snakeAngel2015)
+* rgw: fix break inside of yield in RGWFetchAllMetaCR (`issue#17655 <http://tracker.ceph.com/issues/17655>`_, `pr#14066 <https://github.com/ceph/ceph/pull/14066>`_, Casey Bodley)
+* rgw: fix failed to create bucket if a non-master zonegroup has a single zone (`issue#19756 <http://tracker.ceph.com/issues/19756>`_, `pr#14766 <https://github.com/ceph/ceph/pull/14766>`_, weiqiaomiao)
+* rgw: health check errors out incorrectly (`issue#19025 <http://tracker.ceph.com/issues/19025>`_, `pr#13865 <https://github.com/ceph/ceph/pull/13865>`_, Pavan Rallabhandi)
+* rgw: list_plain_entries() stops before bi_log entries (`issue#19876 <http://tracker.ceph.com/issues/19876>`_, `pr#15383 <https://github.com/ceph/ceph/pull/15383>`_, Casey Bodley)
+* rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote (`issue#19599 <http://tracker.ceph.com/issues/19599>`_, `pr#14607 <https://github.com/ceph/ceph/pull/14607>`_, Zhang Shaowen, Casey Bodley)
+* rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync() (`issue#18076 <http://tracker.ceph.com/issues/18076>`_, `pr#13837 <https://github.com/ceph/ceph/pull/13837>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: only append zonegroups to rest params if not empty (`issue#20078 <http://tracker.ceph.com/issues/20078>`_, `pr#15312 <https://github.com/ceph/ceph/pull/15312>`_, Yehuda Sadeh, Karol Mroz)
+* rgw: pullup civet chunked (`issue#19736 <http://tracker.ceph.com/issues/19736>`_, `pr#14776 <https://github.com/ceph/ceph/pull/14776>`_, Matt Benjamin)
+* rgw: rgw_file: fix event expire check, don't expire directories being read (`issue#19623 <http://tracker.ceph.com/issues/19623>`_, `issue#19270 <http://tracker.ceph.com/issues/19270>`_, `issue#19625 <http://tracker.ceph.com/issues/19625>`_, `issue#19624 <http://tracker.ceph.com/issues/19624>`_, `issue#19634 <http://tracker.ceph.com/issues/19634>`_, `issue#19435 <http://tracker.ceph.com/issues/19435>`_, `pr#14653 <https://github.com/ceph/ceph/pull/14653>`_, Gui Hecheng, Matt Benjamin)
+* rgw: swift: disable revocation thread under certain circumstances (`issue#19499 <http://tracker.ceph.com/issues/19499>`_, `issue#9493 <http://tracker.ceph.com/issues/9493>`_, `pr#14789 <https://github.com/ceph/ceph/pull/14789>`_, Marcus Watts)
+* rgw: the swift container acl does not support field .ref (`issue#18484 <http://tracker.ceph.com/issues/18484>`_, `pr#13833 <https://github.com/ceph/ceph/pull/13833>`_, Jing Wenjun)
+* rgw: typo in rgw_admin.cc (`issue#19026 <http://tracker.ceph.com/issues/19026>`_, `pr#13863 <https://github.com/ceph/ceph/pull/13863>`_, Ronak Jain)
+* rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response() (`issue#19249 <http://tracker.ceph.com/issues/19249>`_, `pr#14661 <https://github.com/ceph/ceph/pull/14661>`_, Yehuda Sadeh)
+* rgw: upgrade to multisite v2 fails if there is a zone without zone info (`issue#19231 <http://tracker.ceph.com/issues/19231>`_, `pr#14136 <https://github.com/ceph/ceph/pull/14136>`_, Danny Al-Gaaf, Orit Wasserman)
+* rgw: use separate http_manager for read_sync_status (`issue#19236 <http://tracker.ceph.com/issues/19236>`_, `pr#14195 <https://github.com/ceph/ceph/pull/14195>`_, Casey Bodley, Shasha Lu)
+* rgw: when converting region_map we need to use rgw_zone_root_pool (`issue#19195 <http://tracker.ceph.com/issues/19195>`_, `pr#14143 <https://github.com/ceph/ceph/pull/14143>`_, Orit Wasserman)
+* rgw: zonegroupmap set does not work (`issue#19498 <http://tracker.ceph.com/issues/19498>`_, `issue#18725 <http://tracker.ceph.com/issues/18725>`_, `pr#14660 <https://github.com/ceph/ceph/pull/14660>`_, Orit Wasserman, Casey Bodley)
+* rgw:fix memory leaks in data/md sync (`issue#20088 <http://tracker.ceph.com/issues/20088>`_, `pr#15382 <https://github.com/ceph/ceph/pull/15382>`_, weiqiaomiao)
+* tests: 'ceph auth import -i' overwrites caps, should alert user before overwrite (`issue#18932 <http://tracker.ceph.com/issues/18932>`_, `pr#13544 <https://github.com/ceph/ceph/pull/13544>`_, Vikhyat Umrao)
+* tests: New upgrade test for #19508 (`issue#19829 <http://tracker.ceph.com/issues/19829>`_, `issue#19508 <http://tracker.ceph.com/issues/19508>`_, `pr#14930 <https://github.com/ceph/ceph/pull/14930>`_, Nathan Cutler)
+* tests: [  FAILED  ] TestLibRBD.ImagePollIO in upgrade:client-upgrade-kraken-distro-basic-smithi (`issue#18617 <http://tracker.ceph.com/issues/18617>`_, `pr#13107 <https://github.com/ceph/ceph/pull/13107>`_, Jason Dillaman)
+* tests: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value (`issue#19597 <http://tracker.ceph.com/issues/19597>`_, `pr#14665 <https://github.com/ceph/ceph/pull/14665>`_, Jason Dillaman)
+* tests: additional rbd-mirror test stability improvements (`issue#18935 <http://tracker.ceph.com/issues/18935>`_, `pr#14154 <https://github.com/ceph/ceph/pull/14154>`_, Jason Dillaman)
+* tests: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure (`issue#15368 <http://tracker.ceph.com/issues/15368>`_, `pr#14763 <https://github.com/ceph/ceph/pull/14763>`_, Sage Weil)
+* tests: buffer overflow in test LibCephFS.DirLs (`issue#18941 <http://tracker.ceph.com/issues/18941>`_, `pr#14671 <https://github.com/ceph/ceph/pull/14671>`_, "Yan, Zheng")
+* tests: clone workunit using the branch specified by task (`issue#19429 <http://tracker.ceph.com/issues/19429>`_, `pr#14371 <https://github.com/ceph/ceph/pull/14371>`_, Kefu Chai, Dan Mick)
+* tests: drop upgrade/hammer-jewel-x (`issue#20574 <http://tracker.ceph.com/issues/20574>`_, `pr#15933 <https://github.com/ceph/ceph/pull/15933>`_, Nathan Cutler)
+* tests: dummy suite fails in OpenStack (`issue#18259 <http://tracker.ceph.com/issues/18259>`_, `pr#14070 <https://github.com/ceph/ceph/pull/14070>`_, Nathan Cutler)
+* tests: eliminate race condition in Thrasher constructor (`issue#18799 <http://tracker.ceph.com/issues/18799>`_, `pr#13608 <https://github.com/ceph/ceph/pull/13608>`_, Nathan Cutler)
+* tests: enable quotas for pre-luminous quota tests (`issue#20412 <http://tracker.ceph.com/issues/20412>`_, `pr#15936 <https://github.com/ceph/ceph/pull/15936>`_, Patrick Donnelly)
+* tests: fix oversight in yaml comment (`issue#20581 <http://tracker.ceph.com/issues/20581>`_, `pr#14449 <https://github.com/ceph/ceph/pull/14449>`_, Nathan Cutler)
+* tests: move swift.py task from teuthology to ceph, phase one (jewel) (`issue#20392 <http://tracker.ceph.com/issues/20392>`_, `pr#15870 <https://github.com/ceph/ceph/pull/15870>`_, Nathan Cutler, Sage Weil, Warren Usui, Greg Farnum, Ali Maredia, Tommi Virtanen, Zack Cerza, Sam Lang, Yehuda Sadeh, Joe Buck, Josh Durgin)
+* tests: qa/Fixed upgrade sequence to 10.2.0 -> 10.2.7 -> latest -x (10.2.8) (`issue#20572 <http://tracker.ceph.com/issues/20572>`_, `pr#16089 <https://github.com/ceph/ceph/pull/16089>`_, Yuri Weinstein)
+* tests: qa/suites/upgrade/hammer-x: set "sortbitwise" for jewel clusters (`issue#20342 <http://tracker.ceph.com/issues/20342>`_, `pr#15842 <https://github.com/ceph/ceph/pull/15842>`_, Nathan Cutler)
+* tests: qa/workunits/rados/test-upgrade-\*: whitelist tests for master (part 1) (`issue#20577 <http://tracker.ceph.com/issues/20577>`_, `pr#15360 <https://github.com/ceph/ceph/pull/15360>`_, Sage Weil)
+* tests: qa/workunits/rados/test-upgrade-\*: whitelist tests for master (part 2) (`issue#20576 <http://tracker.ceph.com/issues/20576>`_, `pr#15778 <https://github.com/ceph/ceph/pull/15778>`_, Kefu Chai)
+* tests: qa/workunits/rados/test-upgrade-\*: whitelist tests the right way (`issue#20575 <http://tracker.ceph.com/issues/20575>`_, `pr#15824 <https://github.com/ceph/ceph/pull/15824>`_, Kefu Chai)
+* tests: rados: sleep before ceph tell osd.0 flush_pg_stats after restart (`issue#16239 <http://tracker.ceph.com/issues/16239>`_, `issue#20489 <http://tracker.ceph.com/issues/20489>`_, `pr#14710 <https://github.com/ceph/ceph/pull/14710>`_, Kefu Chai, Nathan Cutler)
+* tests: run upgrade/client-upgrade on latest CentOS 7.3 (`issue#20573 <http://tracker.ceph.com/issues/20573>`_, `pr#16088 <https://github.com/ceph/ceph/pull/16088>`_, Nathan Cutler)
+* tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap (`issue#17447 <http://tracker.ceph.com/issues/17447>`_, `pr#14150 <https://github.com/ceph/ceph/pull/14150>`_, Jason Dillaman)
+* tests: systemd test backport to jewel (`issue#19717 <http://tracker.ceph.com/issues/19717>`_, `pr#14694 <https://github.com/ceph/ceph/pull/14694>`_, Vasu Kulkarni)
+* tests: test/librados/tmap_migrate: g_ceph_context->put() upon return (`issue#20579 <http://tracker.ceph.com/issues/20579>`_, `pr#14809 <https://github.com/ceph/ceph/pull/14809>`_, Kefu Chai)
+* tests: test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2 (`issue#19692 <http://tracker.ceph.com/issues/19692>`_, `pr#14680 <https://github.com/ceph/ceph/pull/14680>`_, Jason Dillaman)
+* tests: upgrade/hammer-x failing with OSD has the store locked when Thrasher runs ceph-objectstore-tool on down PG (`issue#19556 <http://tracker.ceph.com/issues/19556>`_, `pr#14416 <https://github.com/ceph/ceph/pull/14416>`_, Nathan Cutler)
+* tests: upgrade:hammer-x/stress-split-erasure-code-x86_64 fails in 10.2.8 integration testing (`issue#20413 <http://tracker.ceph.com/issues/20413>`_, `pr#15904 <https://github.com/ceph/ceph/pull/15904>`_, Nathan Cutler)
+* tools: brag fails to count "in" mds (`issue#19192 <http://tracker.ceph.com/issues/19192>`_, `pr#14112 <https://github.com/ceph/ceph/pull/14112>`_, Oleh Prypin, Peng Zhang)
+* tools: ceph-disk does not support cluster names different than 'ceph' (`issue#17821 <http://tracker.ceph.com/issues/17821>`_, `pr#14765 <https://github.com/ceph/ceph/pull/14765>`_, Loic Dachary)
+* tools: ceph-disk: Racing between partition creation and device node creation (`issue#19428 <http://tracker.ceph.com/issues/19428>`_, `pr#14329 <https://github.com/ceph/ceph/pull/14329>`_, Erwan Velu)
+* tools: ceph-disk: bluestore --setgroup incorrectly set with user (`issue#18955 <http://tracker.ceph.com/issues/18955>`_, `pr#13489 <https://github.com/ceph/ceph/pull/13489>`_, craigchi)
+* tools: ceph-disk: ceph-disk list reports mount error for OSD having mount options with SELinux context (`issue#17331 <http://tracker.ceph.com/issues/17331>`_, `pr#14402 <https://github.com/ceph/ceph/pull/14402>`_, Brad Hubbard)
+* tools: ceph-disk: do not setup_statedir on trigger (`issue#19941 <http://tracker.ceph.com/issues/19941>`_, `pr#15504 <https://github.com/ceph/ceph/pull/15504>`_, Loic Dachary)
+* tools: ceph-disk: enable directory backed OSD at boot time (`issue#19628 <http://tracker.ceph.com/issues/19628>`_, `pr#14602 <https://github.com/ceph/ceph/pull/14602>`_, Loic Dachary)
+* tools: rados: RadosImport::import should return an error if Rados::connect fails (`issue#19319 <http://tracker.ceph.com/issues/19319>`_, `pr#14113 <https://github.com/ceph/ceph/pull/14113>`_, Brad Hubbard)
+
+
+v10.2.7 Jewel
+=============
+
+This point release fixes several important bugs in RBD mirroring, librbd &  RGW.
+
+We recommend that all v10.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.7.txt>`.
+
+
+Notable Changes
+---------------
+
+* librbd: possible race in ExclusiveLock handle_peer_notification (`issue#19368 <http://tracker.ceph.com/issues/19368>`_, `pr#14233 <https://github.com/ceph/ceph/pull/14233>`_, Mykola Golub)
+* osd: Increase priority for inactive PGs backfill (`issue#18350 <http://tracker.ceph.com/issues/18350>`_, `pr#13232 <https://github.com/ceph/ceph/pull/13232>`_, Bartłomiej Święcki)
+* osd: Scrub improvements and other fixes (`issue#17857 <http://tracker.ceph.com/issues/17857>`_, `issue#18114 <http://tracker.ceph.com/issues/18114>`_, `issue#13937 <http://tracker.ceph.com/issues/13937>`_, `issue#18113 <http://tracker.ceph.com/issues/18113>`_, `pr#13146 <https://github.com/ceph/ceph/pull/13146>`_, Kefu Chai, David Zafman)
+* osd: fix OSD network address in OSD heartbeat_check log message (`issue#18657 <http://tracker.ceph.com/issues/18657>`_, `pr#13108 <https://github.com/ceph/ceph/pull/13108>`_, Vikhyat Umrao)
+* rbd-mirror: deleting a snapshot during sync can result in read errors (`issue#18990 <http://tracker.ceph.com/issues/18990>`_, `pr#13596 <https://github.com/ceph/ceph/pull/13596>`_, Jason Dillaman)
+* rgw: 'period update' does not remove short_zone_ids of deleted zones (`issue#15618 <http://tracker.ceph.com/issues/15618>`_, `pr#14140 <https://github.com/ceph/ceph/pull/14140>`_, Casey Bodley)
+* rgw: DUMPABLE flag is cleared by setuid preventing coredumps (`issue#19089 <http://tracker.ceph.com/issues/19089>`_, `pr#13844 <https://github.com/ceph/ceph/pull/13844>`_, Brad Hubbard)
+* rgw: clear data_sync_cr if RGWDataSyncControlCR fails (`issue#17569 <http://tracker.ceph.com/issues/17569>`_, `pr#13886 <https://github.com/ceph/ceph/pull/13886>`_, Casey Bodley)
+* rgw: fix openssl (`issue#11239 <http://tracker.ceph.com/issues/11239>`_, `issue#19098 <http://tracker.ceph.com/issues/19098>`_, `issue#16535 <http://tracker.ceph.com/issues/16535>`_, `pr#14215 <https://github.com/ceph/ceph/pull/14215>`_, Marcus Watts)
+* rgw: fix swift cannot disable object versioning with empty X-Versions-Location (`issue#18852 <http://tracker.ceph.com/issues/18852>`_, `pr#13823 <https://github.com/ceph/ceph/pull/13823>`_, Jing Wenjun)
+* rgw: librgw: RGWLibFS::setattr fails on directories (`issue#18808 <http://tracker.ceph.com/issues/18808>`_, `pr#13778 <https://github.com/ceph/ceph/pull/13778>`_, Matt Benjamin)
+* rgw: make sending Content-Length in 204 and 304 controllable (`issue#16602 <http://tracker.ceph.com/issues/16602>`_, `pr#13503 <https://github.com/ceph/ceph/pull/13503>`_, Radoslaw Zarzynski, Matt Benjamin)
+* rgw: multipart uploads copy part support (`issue#12790 <http://tracker.ceph.com/issues/12790>`_, `pr#13219 <https://github.com/ceph/ceph/pull/13219>`_, Yehuda Sadeh, Javier M. Mellid, Matt Benjamin)
+* rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO (`issue#19019 <http://tracker.ceph.com/issues/19019>`_, `pr#13867 <https://github.com/ceph/ceph/pull/13867>`_, Casey Bodley)
+* rgw: radosgw/swift: clean up flush / newline behavior (`issue#18473 <http://tracker.ceph.com/issues/18473>`_, `pr#14100 <https://github.com/ceph/ceph/pull/14100>`_, Nathan Cutler, Marcus Watts, Matt Benjamin)
+* rgw: radosgw/swift: clean up flush / newline behavior. (`issue#18473 <http://tracker.ceph.com/issues/18473>`_, `pr#13143 <https://github.com/ceph/ceph/pull/13143>`_, Marcus Watts, Matt Benjamin)
+* rgw: rgw_fh: RGWFileHandle dtor must also cond-unlink from FHCache (`issue#19112 <http://tracker.ceph.com/issues/19112>`_, `pr#14231 <https://github.com/ceph/ceph/pull/14231>`_, Matt Benjamin)
+* rgw: rgw_file:  avoid interning .. in FHCache table and don't ref for them (`issue#19036 <http://tracker.ceph.com/issues/19036>`_, `pr#13848 <https://github.com/ceph/ceph/pull/13848>`_, Matt Benjamin)
+* rgw: rgw_file:  interned RGWFileHandle objects need parent refs (`issue#18650 <http://tracker.ceph.com/issues/18650>`_, `pr#13583 <https://github.com/ceph/ceph/pull/13583>`_, Matt Benjamin)
+* rgw: rgw_file:  restore (corrected) fix for dir partial match (return of FLAG_EXACT_MATCH) (`issue#19060 <http://tracker.ceph.com/issues/19060>`_, `issue#18992 <http://tracker.ceph.com/issues/18992>`_, `issue#19059 <http://tracker.ceph.com/issues/19059>`_, `pr#13858 <https://github.com/ceph/ceph/pull/13858>`_, Matt Benjamin)
+* rgw: rgw_file: FHCache residence check should be exhaustive (`issue#19111 <http://tracker.ceph.com/issues/19111>`_, `pr#14169 <https://github.com/ceph/ceph/pull/14169>`_, Matt Benjamin)
+* rgw: rgw_file: ensure valid_s3_object_name for directories, too (`issue#19066 <http://tracker.ceph.com/issues/19066>`_, `pr#13717 <https://github.com/ceph/ceph/pull/13717>`_, Matt Benjamin)
+* rgw: rgw_file: fix marker computation (`issue#19018 <http://tracker.ceph.com/issues/19018>`_, `issue#18989 <http://tracker.ceph.com/issues/18989>`_, `issue#18992 <http://tracker.ceph.com/issues/18992>`_, `issue#18991 <http://tracker.ceph.com/issues/18991>`_, `pr#13869 <https://github.com/ceph/ceph/pull/13869>`_, Matt Benjamin)
+* rgw: rgw_file: wip dir orphan (`issue#18992 <http://tracker.ceph.com/issues/18992>`_, `issue#18989 <http://tracker.ceph.com/issues/18989>`_, `issue#19018 <http://tracker.ceph.com/issues/19018>`_, `issue#18991 <http://tracker.ceph.com/issues/18991>`_, `pr#14205 <https://github.com/ceph/ceph/pull/14205>`_, Gui Hecheng, Matt Benjamin)
+* rgw: rgw_file: various fixes (`pr#14206 <https://github.com/ceph/ceph/pull/14206>`_, Matt Benjamin)
+* rgw: rgw_file: expand argv (`pr#14230 <https://github.com/ceph/ceph/pull/14230>`_, Matt Benjamin)
+
+
+v10.2.6 Jewel
+=============
+
+This point release fixes several important bugs in RBD mirroring, RGW multi-site, CephFS, and RADOS.
+
+We recommend that all v10.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.6.txt>`.
+
+OSDs No Longer Send ENXIO by Default
+------------------------------------
+
+In previous versions, if a client sent an op to the wrong OSD, the OSD
+would reply with ENXIO.  The rationale here is that the client or OSD is
+clearly buggy and we want to surface the error as clearly as possible.
+We now only send the ENXIO reply if the osd_enxio_on_misdirected_op option
+is enabled (it's off by default).  This means that a VM using librbd that
+previously would have gotten an EIO and gone read-only will now see a
+blocked/hung IO instead.
+
+Other Notable Changes
+---------------------
+
+* build/ops: add hostname sanity check to run-{c}make-check.sh (`issue#18134 <http://tracker.ceph.com/issues/18134>`_, `pr#12302 <http://github.com/ceph/ceph/pull/12302>`_, Nathan Cutler)
+* build/ops: add ldap lib to rgw lib deps based on build config (`issue#17313 <http://tracker.ceph.com/issues/17313>`_, `pr#13183 <http://github.com/ceph/ceph/pull/13183>`_, Nathan Cutler)
+* build/ops: ceph-create-keys loops forever (`issue#17753 <http://tracker.ceph.com/issues/17753>`_, `pr#11884 <http://github.com/ceph/ceph/pull/11884>`_, Alfredo Deza)
+* build/ops: ceph daemons DUMPABLE flag is cleared by setuid preventing coredumps (`issue#17650 <http://tracker.ceph.com/issues/17650>`_, `pr#11736 <http://github.com/ceph/ceph/pull/11736>`_, Patrick Donnelly)
+* build/ops: fixed compilation error when --with-radowsgw=no (`issue#18512 <http://tracker.ceph.com/issues/18512>`_, `pr#12729 <http://github.com/ceph/ceph/pull/12729>`_, Pan Liu)
+* build/ops: fixed the issue when --disable-server, compilation fails. (`issue#18120 <http://tracker.ceph.com/issues/18120>`_, `pr#12239 <http://github.com/ceph/ceph/pull/12239>`_, Pan Liu)
+* build/ops: fix undefined crypto references with --with-xio (`issue#18133 <http://tracker.ceph.com/issues/18133>`_, `pr#12296 <http://github.com/ceph/ceph/pull/12296>`_, Nathan Cutler)
+* build/ops: install-deps.sh based on /etc/os-release (`issue#18466 <http://tracker.ceph.com/issues/18466>`_, `issue#18198 <http://tracker.ceph.com/issues/18198>`_, `pr#12405 <http://github.com/ceph/ceph/pull/12405>`_, Jan Fajerski, Nitin A Kamble, Nathan Cutler)
+* build/ops: Remove the runtime dependency on lsb_release (`issue#17425 <http://tracker.ceph.com/issues/17425>`_, `pr#11875 <http://github.com/ceph/ceph/pull/11875>`_, John Coyle, Brad Hubbard)
+* build/ops: rpm: /etc/ceph/rbdmap is packaged with executable access rights (`issue#17395 <http://tracker.ceph.com/issues/17395>`_, `pr#11855 <http://github.com/ceph/ceph/pull/11855>`_, Ken Dreyer)
+* build/ops: selinux: Allow ceph to manage tmp files (`issue#17436 <http://tracker.ceph.com/issues/17436>`_, `pr#13048 <http://github.com/ceph/ceph/pull/13048>`_, Boris Ranto)
+* build/ops: systemd: Restart Mon after 10s in case of failure (`issue#18635 <http://tracker.ceph.com/issues/18635>`_, `pr#13058 <http://github.com/ceph/ceph/pull/13058>`_, Wido den Hollander)
+* build/ops: systemd restarts Ceph Mon to quickly after failing to start (`issue#18635 <http://tracker.ceph.com/issues/18635>`_, `pr#13184 <http://github.com/ceph/ceph/pull/13184>`_, Wido den Hollander)
+* ceph-disk: fix flake8 errors (`issue#17898 <http://tracker.ceph.com/issues/17898>`_, `pr#11976 <http://github.com/ceph/ceph/pull/11976>`_, Ken Dreyer)
+* cephfs: fuse client crash when adding a new osd (`issue#17270 <http://tracker.ceph.com/issues/17270>`_, `pr#11860 <http://github.com/ceph/ceph/pull/11860>`_, John Spray)
+* cli: ceph-disk: convert none str to str before printing it (`issue#18371 <http://tracker.ceph.com/issues/18371>`_, `pr#13187 <http://github.com/ceph/ceph/pull/13187>`_, Kefu Chai)
+* client: Fix lookup of "/.." in jewel (`issue#18408 <http://tracker.ceph.com/issues/18408>`_, `pr#12766 <http://github.com/ceph/ceph/pull/12766>`_, Jeff Layton)
+* client: fix stale entries in command table (`issue#17974 <http://tracker.ceph.com/issues/17974>`_, `pr#12137 <http://github.com/ceph/ceph/pull/12137>`_, John Spray)
+* client: populate metadata during mount (`issue#18361 <http://tracker.ceph.com/issues/18361>`_, `pr#13085 <http://github.com/ceph/ceph/pull/13085>`_, John Spray)
+* cli: implement functionality for adding, editing and removing omap values with binary keys (`issue#18123 <http://tracker.ceph.com/issues/18123>`_, `pr#12755 <http://github.com/ceph/ceph/pull/12755>`_, Jason Dillaman)
+* common: Improve linux dcache hash algorithm (`issue#17599 <http://tracker.ceph.com/issues/17599>`_, `pr#11529 <http://github.com/ceph/ceph/pull/11529>`_, Yibo Cai)
+* common: utime.h: fix timezone issue in round_to_* funcs.  (`issue#14862 <http://tracker.ceph.com/issues/14862>`_, `pr#11508 <http://github.com/ceph/ceph/pull/11508>`_, Zhao Chao)
+* doc: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh (`issue#17746 <http://tracker.ceph.com/issues/17746>`_, `pr#13043 <http://github.com/ceph/ceph/pull/13043>`_, Ronak Jain)
+* librbd: allow to open an image without opening parent image (`issue#18325 <http://tracker.ceph.com/issues/18325>`_, `pr#13130 <http://github.com/ceph/ceph/pull/13130>`_, Ricardo Dias)
+* librbd: metadata_set API operation should not change global config setting (`issue#18465 <http://tracker.ceph.com/issues/18465>`_, `pr#13168 <http://github.com/ceph/ceph/pull/13168>`_, Mykola Golub)
+* librbd: new API method to force break a peer's exclusive lock (`issue#15632 <http://tracker.ceph.com/issues/15632>`_, `issue#16773 <http://tracker.ceph.com/issues/16773>`_, `issue#17188 <http://tracker.ceph.com/issues/17188>`_, `issue#16988 <http://tracker.ceph.com/issues/16988>`_, `issue#17210 <http://tracker.ceph.com/issues/17210>`_, `issue#17251 <http://tracker.ceph.com/issues/17251>`_, `issue#18429 <http://tracker.ceph.com/issues/18429>`_, `issue#17227 <http://tracker.ceph.com/issues/17227>`_, `issue#18327 <http://tracker.ceph.com/issues/18327>`_, `issue#17015 <http://tracker.ceph.com/issues/17015>`_, `pr#12890 <http://github.com/ceph/ceph/pull/12890>`_, Danny Al-Gaaf, Mykola Golub, Jason Dillaman)
+* librbd: properly order concurrent updates to the object map (`issue#16176 <http://tracker.ceph.com/issues/16176>`_, `pr#12909 <http://github.com/ceph/ceph/pull/12909>`_, Jason Dillaman)
+* librbd: restore journal access when force disabling mirroring (`issue#17588 <http://tracker.ceph.com/issues/17588>`_, `pr#11916 <http://github.com/ceph/ceph/pull/11916>`_, Mykola Golub)
+* mds: Cannot create deep directories when caps contain path=/somepath (`issue#17858 <http://tracker.ceph.com/issues/17858>`_, `pr#12154 <http://github.com/ceph/ceph/pull/12154>`_, Patrick Donnelly)
+* mds: cephfs metadata pool: deep-scrub error omap_digest != best guess omap_digest (`issue#17177 <http://tracker.ceph.com/issues/17177>`_, `pr#12380 <http://github.com/ceph/ceph/pull/12380>`_, Yan, Zheng)
+* mds: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa) (`issue#18574 <http://tracker.ceph.com/issues/18574>`_, `pr#13023 <http://github.com/ceph/ceph/pull/13023>`_, John Spray)
+* mds: ceph-fuse crash during snapshot tests (`issue#18460 <http://tracker.ceph.com/issues/18460>`_, `pr#13120 <http://github.com/ceph/ceph/pull/13120>`_, Yan, Zheng)
+* mds: ceph_volume_client: fix recovery from partial auth update  (`issue#17216 <http://tracker.ceph.com/issues/17216>`_, `pr#11656 <http://github.com/ceph/ceph/pull/11656>`_, Ramana Raja)
+* mds: ceph_volume_client.py : Error: Can't handle arrays of non-strings (`issue#17800 <http://tracker.ceph.com/issues/17800>`_, `pr#12325 <http://github.com/ceph/ceph/pull/12325>`_, Ramana Raja)
+* mds: Cleanly reject session evict command when in replay (`issue#17801 <http://tracker.ceph.com/issues/17801>`_, `pr#12153 <http://github.com/ceph/ceph/pull/12153>`_, Yan, Zheng)
+* mds: client segfault on ceph_rmdir path / (`issue#9935 <http://tracker.ceph.com/issues/9935>`_, `pr#13029 <http://github.com/ceph/ceph/pull/13029>`_, Michal Jarzabek)
+* mds: Clients without pool-changing caps shouldn't be allowed to change pool_namespace (`issue#17798 <http://tracker.ceph.com/issues/17798>`_, `pr#12155 <http://github.com/ceph/ceph/pull/12155>`_, John Spray)
+* mds: Decode errors on backtrace will crash MDS (`issue#18311 <http://tracker.ceph.com/issues/18311>`_, `pr#12836 <http://github.com/ceph/ceph/pull/12836>`_, Nathan Cutler, John Spray)
+* mds: false failing to respond to cache pressure warning (`issue#17611 <http://tracker.ceph.com/issues/17611>`_, `pr#11861 <http://github.com/ceph/ceph/pull/11861>`_, Yan, Zheng)
+* mds: finish clientreplay requests before requesting active state (`issue#18461 <http://tracker.ceph.com/issues/18461>`_, `pr#13113 <http://github.com/ceph/ceph/pull/13113>`_, Yan, Zheng)
+* mds: fix incorrect assertion in Server::_dir_is_nonempty() (`issue#18578 <http://tracker.ceph.com/issues/18578>`_, `pr#13459 <http://github.com/ceph/ceph/pull/13459>`_, Yan, Zheng)
+* mds: fix MDSMap upgrade decoding (`issue#17837 <http://tracker.ceph.com/issues/17837>`_, `pr#13139 <http://github.com/ceph/ceph/pull/13139>`_, John Spray, Patrick Donnelly)
+* mds: fix missing ll_get for ll_walk (`issue#18086 <http://tracker.ceph.com/issues/18086>`_, `pr#13125 <http://github.com/ceph/ceph/pull/13125>`_, Gui Hecheng)
+* mds: Fix mount root for ceph_mount users and change tarball format (`issue#18312 <http://tracker.ceph.com/issues/18312>`_, `issue#18254 <http://tracker.ceph.com/issues/18254>`_, `pr#12592 <http://github.com/ceph/ceph/pull/12592>`_, Jeff Layton)
+* mds: fix null pointer dereference in Locker::handle_client_caps (`issue#18306 <http://tracker.ceph.com/issues/18306>`_, `pr#13060 <http://github.com/ceph/ceph/pull/13060>`_, Yan, Zheng)
+* mds: lookup of /.. in  returns -ENOENT (`issue#18408 <http://tracker.ceph.com/issues/18408>`_, `pr#12783 <http://github.com/ceph/ceph/pull/12783>`_, Jeff Layton)
+* mds: MDS crashes on missing metadata object (`issue#18179 <http://tracker.ceph.com/issues/18179>`_, `pr#13119 <http://github.com/ceph/ceph/pull/13119>`_, Yan, Zheng)
+* mds: mds fails to respawn if executable has changed (`issue#17531 <http://tracker.ceph.com/issues/17531>`_, `pr#11873 <http://github.com/ceph/ceph/pull/11873>`_, Patrick Donnelly)
+* mds: MDS: false failing to respond to cache pressure warning (`issue#17716 <http://tracker.ceph.com/issues/17716>`_, `pr#11856 <http://github.com/ceph/ceph/pull/11856>`_, Yan, Zheng)
+* mds: MDS goes damaged on blacklist (failed to read JournalPointer: -108 ((108) Cannot send after transport endpoint shutdown) (`issue#17236 <http://tracker.ceph.com/issues/17236>`_, `pr#11413 <http://github.com/ceph/ceph/pull/11413>`_, John Spray)
+* mds: MDS long-time blocked ops. ceph-fuse locks up with getattr of file (`issue#17275 <http://tracker.ceph.com/issues/17275>`_, `pr#11858 <http://github.com/ceph/ceph/pull/11858>`_, Yan, Zheng)
+* mds: speed up readdir by skipping unwanted dn (`issue#18519 <http://tracker.ceph.com/issues/18519>`_, `pr#12921 <http://github.com/ceph/ceph/pull/12921>`_, Xiaoxi Chen)
+* mds: standby-replay daemons can sometimes miss events (`issue#17954 <http://tracker.ceph.com/issues/17954>`_, `pr#13126 <http://github.com/ceph/ceph/pull/13126>`_, John Spray)
+* mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version) (`issue#18366 <http://tracker.ceph.com/issues/18366>`_, `pr#13115 <http://github.com/ceph/ceph/pull/13115>`_, Sage Weil)
+* mon: ceph osd down detection behaviour (`issue#18104 <http://tracker.ceph.com/issues/18104>`_, `pr#12677 <http://github.com/ceph/ceph/pull/12677>`_, xie xingguo)
+* mon: Error EINVAL: removing mon.a at 172.21.15.16:6789/0, there will be 1 monitors (`issue#17725 <http://tracker.ceph.com/issues/17725>`_, `pr#11999 <http://github.com/ceph/ceph/pull/11999>`_, Joao Eduardo Luis)
+* mon: health does not report pgs stuck in more than one state (`issue#17515 <http://tracker.ceph.com/issues/17515>`_, `pr#11660 <http://github.com/ceph/ceph/pull/11660>`_, Sage Weil)
+* mon: monitor assertion failure when deactivating mds in (invalid) fscid 0 (`issue#17518 <http://tracker.ceph.com/issues/17518>`_, `pr#11862 <http://github.com/ceph/ceph/pull/11862>`_, Patrick Donnelly)
+* mon: monitor cannot start because of FAILED assert(info.state == MDSMap::STATE_STANDBY) (`issue#18166 <http://tracker.ceph.com/issues/18166>`_, `pr#13123 <http://github.com/ceph/ceph/pull/13123>`_, John Spray, Patrick Donnelly)
+* mon: osd flag health message is misleading (`issue#18175 <http://tracker.ceph.com/issues/18175>`_, `pr#13117 <http://github.com/ceph/ceph/pull/13117>`_, Sage Weil)
+* mon: OSDMonitor: clear jewel+ feature bits when talking to Hammer OSD (`issue#18582 <http://tracker.ceph.com/issues/18582>`_, `pr#13131 <http://github.com/ceph/ceph/pull/13131>`_, Piotr Dałek)
+* mon: OSDs marked OUT wrongly after monitor failover (`issue#17719 <http://tracker.ceph.com/issues/17719>`_, `pr#11947 <http://github.com/ceph/ceph/pull/11947>`_, Dong Wu)
+* mon: peon wrongly delete routed pg stats op before receive pg stats ack (`issue#18458 <http://tracker.ceph.com/issues/18458>`_, `pr#13045 <http://github.com/ceph/ceph/pull/13045>`_, Mingxin Liu)
+* mon: send updated monmap to its subscribers (`issue#17558 <http://tracker.ceph.com/issues/17558>`_, `pr#11743 <http://github.com/ceph/ceph/pull/11743>`_, Kefu Chai)
+* msgr: don't truncate message sequence to 32-bits (`issue#16122 <http://tracker.ceph.com/issues/16122>`_, `pr#12416 <http://github.com/ceph/ceph/pull/12416>`_, Yan, Zheng)
+* msgr: msg/simple: clear_pipe when wait() is mopping up pipes (`issue#15784 <http://tracker.ceph.com/issues/15784>`_, `pr#13062 <http://github.com/ceph/ceph/pull/13062>`_, Sage Weil)
+* msgr: msg/simple/Pipe: error decoding addr (`issue#18072 <http://tracker.ceph.com/issues/18072>`_, `pr#12291 <http://github.com/ceph/ceph/pull/12291>`_, Sage Weil)
+* osd: Add config option to disable new scrubs during recovery (`issue#17866 <http://tracker.ceph.com/issues/17866>`_, `pr#11944 <http://github.com/ceph/ceph/pull/11944>`_, Wido den Hollander)
+* osd: collection_list shadow return value # (`issue#17713 <http://tracker.ceph.com/issues/17713>`_, `pr#11737 <http://github.com/ceph/ceph/pull/11737>`_, Haomai Wang)
+* osd: do not send ENXIO on misdirected op by default (`issue#18751 <http://tracker.ceph.com/issues/18751>`_, `pr#13255 <http://github.com/ceph/ceph/pull/13255>`_, Sage Weil)
+* osd: FileStore: fiemap cannot be totally retrieved in xfs when the number of extents > 1364  (`issue#17610 <http://tracker.ceph.com/issues/17610>`_, `pr#11998 <http://github.com/ceph/ceph/pull/11998>`_, Kefu Chai, Ning Yao)
+* osd: leveldb corruption leads to Operation not permitted not handled and assert (`issue#18037 <http://tracker.ceph.com/issues/18037>`_, `pr#12789 <http://github.com/ceph/ceph/pull/12789>`_, Nathan Cutler)
+* osd: limit omap data in push op (`issue#16128 <http://tracker.ceph.com/issues/16128>`_, `pr#11991 <http://github.com/ceph/ceph/pull/11991>`_, Wanlong Gao)
+* osd: osd crashes when radosgw-admin bi list --max-entries=1 command runing (`issue#17745 <http://tracker.ceph.com/issues/17745>`_, `pr#11758 <http://github.com/ceph/ceph/pull/11758>`_, weiqiaomiao)
+* osd: osd_max_backfills default has changed, documentation should reflect that. (`issue#17701 <http://tracker.ceph.com/issues/17701>`_, `pr#11735 <http://github.com/ceph/ceph/pull/11735>`_, huangjun)
+* osd: OSDMonitor: only reject MOSDBoot based on up_from if inst matches (`issue#17899 <http://tracker.ceph.com/issues/17899>`_, `pr#12868 <http://github.com/ceph/ceph/pull/12868>`_, Samuel Just)
+* osd: osd/PG: publish PG stats when backfill-related states change (`issue#18369 <http://tracker.ceph.com/issues/18369>`_, `pr#12875 <http://github.com/ceph/ceph/pull/12875>`_, Alexey Sheplyakov, Sage Weil)
+* osd: Remove extra call to reg_next_scrub() during splits (`issue#16474 <http://tracker.ceph.com/issues/16474>`_, `pr#11606 <http://github.com/ceph/ceph/pull/11606>`_, David Zafman)
+* osd: Revert "Merge pull request #12978 from asheplyakov/jewel-18581" (`issue#18809 <http://tracker.ceph.com/issues/18809>`_, `pr#13280 <http://github.com/ceph/ceph/pull/13280>`_, Samuel Just)
+* osd: update_log_missing does not order correctly with osd_ops (`issue#17789 <http://tracker.ceph.com/issues/17789>`_, `pr#11997 <http://github.com/ceph/ceph/pull/11997>`_, Samuel Just)
+* qa/tasks: backport rbd_fio fixes to jewel (`issue#13512 <http://tracker.ceph.com/issues/13512>`_, `pr#13104 <http://github.com/ceph/ceph/pull/13104>`_, Ilya Dryomov)
+* qa/tasks/workunits: backport misc fixes to jewel (`issue#18336 <http://tracker.ceph.com/issues/18336>`_, `pr#12912 <http://github.com/ceph/ceph/pull/12912>`_, Sage Weil)
+* rados: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas (part 2) (`issue#15943 <http://tracker.ceph.com/issues/15943>`_, `issue#18504 <http://tracker.ceph.com/issues/18504>`_, `pr#12791 <http://github.com/ceph/ceph/pull/12791>`_, Samuel Just)
+* rados: Memory leaks in object_list_begin and object_list_end (`issue#18252 <http://tracker.ceph.com/issues/18252>`_, `pr#13118 <http://github.com/ceph/ceph/pull/13118>`_, Brad Hubbard)
+* rados: The request lock RPC message might be incorrectly ignored  (`issue#17030 <http://tracker.ceph.com/issues/17030>`_, `pr#10865 <http://github.com/ceph/ceph/pull/10865>`_, Jason Dillaman)
+* rbd: add image id block name prefix APIs (`issue#18270 <http://tracker.ceph.com/issues/18270>`_, `pr#12529 <http://github.com/ceph/ceph/pull/12529>`_, Jason Dillaman)
+* rbd: add max_part and nbds_max options in rbd nbd map, in order to keep consistent with   (`issue#18186 <http://tracker.ceph.com/issues/18186>`_, `pr#12426 <http://github.com/ceph/ceph/pull/12426>`_, Pan Liu)
+* rbd: Attempting to remove an image w/ incompatible features results in partial removal (`issue#18315 <http://tracker.ceph.com/issues/18315>`_, `pr#13156 <http://github.com/ceph/ceph/pull/13156>`_, Dongsheng Yang)
+* rbd: bench-write will crash if --io-size is 4G (`issue#18422 <http://tracker.ceph.com/issues/18422>`_, `pr#13129 <http://github.com/ceph/ceph/pull/13129>`_, Gaurav Kumar Garg)
+* rbd: diff calculate can hide parent extents when examining first snapshot in clone (`issue#18068 <http://tracker.ceph.com/issues/18068>`_, `pr#12322 <http://github.com/ceph/ceph/pull/12322>`_, Jason Dillaman)
+* rbd: Exclusive lock improperly initialized on read-only image when using snap_set API (`issue#17618 <http://tracker.ceph.com/issues/17618>`_, `pr#11852 <http://github.com/ceph/ceph/pull/11852>`_, Jason Dillaman)
+* rbd: FAILED assert(m_processing == 0) while running test_lock_fence.sh (`issue#17973 <http://tracker.ceph.com/issues/17973>`_, `pr#12323 <http://github.com/ceph/ceph/pull/12323>`_, Venky Shankar)
+* rbd: Improve error reporting from rbd feature enable/disable (`issue#16985 <http://tracker.ceph.com/issues/16985>`_, `pr#13157 <http://github.com/ceph/ceph/pull/13157>`_, Gaurav Kumar Garg)
+* rbd: JournalMetadata flooding with errors when being blacklisted (`issue#18243 <http://tracker.ceph.com/issues/18243>`_, `pr#12739 <http://github.com/ceph/ceph/pull/12739>`_, Jason Dillaman)
+* rbd: librbd: use proper snapshot when computing diff parent overlap (`issue#18200 <http://tracker.ceph.com/issues/18200>`_, `pr#12649 <http://github.com/ceph/ceph/pull/12649>`_, Xiaoxi Chen)
+* rbd: partition func should be enabled When load nbd.ko for rbd-nbd (`issue#18115 <http://tracker.ceph.com/issues/18115>`_, `pr#12754 <http://github.com/ceph/ceph/pull/12754>`_, Pan Liu)
+* rbd: Potential race when removing two-way mirroring image (`issue#18447 <http://tracker.ceph.com/issues/18447>`_, `pr#13233 <http://github.com/ceph/ceph/pull/13233>`_, Mykola Golub)
+* rbd: [qa] crash in journal-enabled fsx run (`issue#18618 <http://tracker.ceph.com/issues/18618>`_, `pr#13128 <http://github.com/ceph/ceph/pull/13128>`_, Jason Dillaman)
+* rbd: 'rbd du' of missing image does not return error (`issue#16987 <http://tracker.ceph.com/issues/16987>`_, `pr#11854 <http://github.com/ceph/ceph/pull/11854>`_, Dongsheng Yang)
+* rbd: rbd-mirror: gmock warnings in bootstrap request unit tests (`issue#18048 <http://tracker.ceph.com/issues/18048>`_, `issue#18012 <http://tracker.ceph.com/issues/18012>`_, `issue#18156 <http://tracker.ceph.com/issues/18156>`_, `issue#16991 <http://tracker.ceph.com/issues/16991>`_, `issue#18051 <http://tracker.ceph.com/issues/18051>`_, `pr#12425 <http://github.com/ceph/ceph/pull/12425>`_, Mykola Golub)
+* rbd: rbd-mirror: image sync object map reload logs message (`issue#16179 <http://tracker.ceph.com/issues/16179>`_, `pr#12753 <http://github.com/ceph/ceph/pull/12753>`_, runsisi)
+* rbd: rbd-mirror: snap protect of non-layered image results in split-brain (`issue#16962 <http://tracker.ceph.com/issues/16962>`_, `pr#11869 <http://github.com/ceph/ceph/pull/11869>`_, Mykola Golub)
+* rbd: [rbd-mirror] sporadic image replayer shut down failure (`issue#18441 <http://tracker.ceph.com/issues/18441>`_, `pr#13155 <http://github.com/ceph/ceph/pull/13155>`_, Jason Dillaman)
+* rbd: rbd-nbd: disallow mapping images >2TB in size (`issue#17219 <http://tracker.ceph.com/issues/17219>`_, `pr#11870 <http://github.com/ceph/ceph/pull/11870>`_, Mykola Golub)
+* rbd: rbd-nbd: invalid error code for "failed to read nbd request" messages (`issue#18242 <http://tracker.ceph.com/issues/18242>`_, `pr#12756 <http://github.com/ceph/ceph/pull/12756>`_, Mykola Golub)
+* rbd: status json format has duplicated/overwritten key (`issue#18261 <http://tracker.ceph.com/issues/18261>`_, `pr#12741 <http://github.com/ceph/ceph/pull/12741>`_, Mykola Golub)
+* rbd: TestLibRBD.DiscardAfterWrite doesn't handle rbd_skip_partial_discard = true (`issue#17750 <http://tracker.ceph.com/issues/17750>`_, `pr#11853 <http://github.com/ceph/ceph/pull/11853>`_, Jason Dillaman)
+* rbd: truncate can cause unflushed snapshot data lose (`issue#17193 <http://tracker.ceph.com/issues/17193>`_, `pr#12324 <http://github.com/ceph/ceph/pull/12324>`_, Yan, Zheng)
+* : ReplicatedBackend: take read locks for clone sources during recovery (`issue#17831 <http://tracker.ceph.com/issues/17831>`_, `issue#18583 <http://tracker.ceph.com/issues/18583>`_, `pr#12978 <http://github.com/ceph/ceph/pull/12978>`_, Samuel Just)
+* rgw: add option to log custom HTTP headers (rgw_log_http_headers) (`issue#18891 <http://tracker.ceph.com/issues/18891>`_, `pr#12490 <http://github.com/ceph/ceph/pull/12490>`_, Matt Benjamin)
+* rgw: add suport for Swift-at-root dependent features of Swift API  (`issue#18526 <http://tracker.ceph.com/issues/18526>`_, `issue#16673 <http://tracker.ceph.com/issues/16673>`_, `pr#11497 <http://github.com/ceph/ceph/pull/11497>`_, Pritha Srivastava, Radoslaw Zarzynski, Pete Zaitcev, Abhishek Lekshmanan)
+* rgw: add support for the prefix parameter in account listing of Swift API (`issue#17931 <http://tracker.ceph.com/issues/17931>`_, `pr#12258 <http://github.com/ceph/ceph/pull/12258>`_, Radoslaw Zarzynski)
+* rgw: Add workaround for upgrade issues for older jewel versions (`issue#17820 <http://tracker.ceph.com/issues/17820>`_, `pr#12316 <http://github.com/ceph/ceph/pull/12316>`_, Orit Wasserman)
+* rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion (`issue#18364 <http://tracker.ceph.com/issues/18364>`_, `issue#16355 <http://tracker.ceph.com/issues/16355>`_, `pr#13276 <http://github.com/ceph/ceph/pull/13276>`_, Radoslaw Zarzynski)
+* rgw: bucket check remove _multipart_ prefix (`issue#13724 <http://tracker.ceph.com/issues/13724>`_, `pr#11470 <http://github.com/ceph/ceph/pull/11470>`_, Weijun Duan)
+* rgw: bucket resharding (`issue#17549 <http://tracker.ceph.com/issues/17549>`_, `issue#17550 <http://tracker.ceph.com/issues/17550>`_, `pr#13341 <http://github.com/ceph/ceph/pull/13341>`_, Yehuda Sadeh, Robin H. Johnson)
+* rgw: disable virtual hosting of buckets when no hostnames are configured (`issue#17440 <http://tracker.ceph.com/issues/17440>`_, `issue#15975 <http://tracker.ceph.com/issues/15975>`_, `issue#17136 <http://tracker.ceph.com/issues/17136>`_, `pr#11760 <http://github.com/ceph/ceph/pull/11760>`_, Casey Bodley, Robin H. Johnson)
+* rgw: do not abort when accept a CORS request with short origin (`issue#18187 <http://tracker.ceph.com/issues/18187>`_, `pr#12397 <http://github.com/ceph/ceph/pull/12397>`_, LiuYang)
+* rgw: don't store empty chains in gc (`issue#17897 <http://tracker.ceph.com/issues/17897>`_, `pr#12174 <http://github.com/ceph/ceph/pull/12174>`_, Yehuda Sadeh)
+* rgw:fix for deleting objects name beginning and ending with underscores of one bucket using POST method of js sdk.  (`issue#17888 <http://tracker.ceph.com/issues/17888>`_, `pr#12320 <http://github.com/ceph/ceph/pull/12320>`_, Casey Bodley)
+* rgw: fix period update crash (`issue#18631 <http://tracker.ceph.com/issues/18631>`_, `pr#13273 <http://github.com/ceph/ceph/pull/13273>`_, Orit Wasserman)
+* rgw: fix put_acls for objects starting and ending with underscore (`issue#17625 <http://tracker.ceph.com/issues/17625>`_, `pr#11675 <http://github.com/ceph/ceph/pull/11675>`_, Orit Wasserman)
+* rgw: fix use of marker in List::list_objects() (`issue#18331 <http://tracker.ceph.com/issues/18331>`_, `pr#13358 <http://github.com/ceph/ceph/pull/13358>`_, Yehuda Sadeh)
+* rgw: for the create_bucket api, if the input creation_time is zero, we … (`issue#16597 <http://tracker.ceph.com/issues/16597>`_, `pr#11990 <http://github.com/ceph/ceph/pull/11990>`_, weiqiaomiao)
+* rgw: Have a flavor of bucket deletion in radosgw-admin to bypass garbage collection (`issue#15557 <http://tracker.ceph.com/issues/15557>`_, `pr#10661 <http://github.com/ceph/ceph/pull/10661>`_, Pavan Rallabhandi)
+* rgw: json encode/decode of RGWBucketInfo missing index_type field (`issue#17755 <http://tracker.ceph.com/issues/17755>`_, `pr#11759 <http://github.com/ceph/ceph/pull/11759>`_, Yehuda Sadeh)
+* rgw: ldap: enforce simple_bind w/LDAPv3 redux (`issue#18339 <http://tracker.ceph.com/issues/18339>`_, `pr#12678 <http://github.com/ceph/ceph/pull/12678>`_, Weibing Zhang)
+* rgw: leak from RGWMetaSyncShardCR::incremental_sync (`issue#18412 <http://tracker.ceph.com/issues/18412>`_, `issue#18300 <http://tracker.ceph.com/issues/18300>`_, `pr#13004 <http://github.com/ceph/ceph/pull/13004>`_, Casey Bodley, Sage Weil)
+* rgw: leak in RGWFetchAllMetaCR (`issue#17812 <http://tracker.ceph.com/issues/17812>`_, `pr#11872 <http://github.com/ceph/ceph/pull/11872>`_, Casey Bodley)
+* rgw: librgw: objects created from s3 apis are not visible from nfs mount point  (`issue#18651 <http://tracker.ceph.com/issues/18651>`_, `pr#13177 <http://github.com/ceph/ceph/pull/13177>`_, Matt Benjamin)
+* rgw: log name instead of id for SystemMetaObj on failure (`issue#15776 <http://tracker.ceph.com/issues/15776>`_, `pr#12622 <http://github.com/ceph/ceph/pull/12622>`_, Wido den Hollander, Abhishek Lekshmanan)
+* rgw: multimds: mds entering up:replay and processing down mds aborts (`issue#17670 <http://tracker.ceph.com/issues/17670>`_, `pr#11857 <http://github.com/ceph/ceph/pull/11857>`_, Patrick Donnelly)
+* rgw: multipart upload copy (`issue#12790 <http://tracker.ceph.com/issues/12790>`_, `pr#13068 <http://github.com/ceph/ceph/pull/13068>`_, Yehuda Sadeh, Javier M. Mellid, Matt Benjamin)
+* rgw: multisite: after finishing full sync on a bucket, incremental sync starts over from the beginning (`issue#17661 <http://tracker.ceph.com/issues/17661>`_, `issue#17624 <http://tracker.ceph.com/issues/17624>`_, `pr#11864 <http://github.com/ceph/ceph/pull/11864>`_, Zengran Zhang, Casey Bodley)
+* rgw: multisite: assert(next) failed in RGWMetaSyncCR (`issue#17044 <http://tracker.ceph.com/issues/17044>`_, `pr#11477 <http://github.com/ceph/ceph/pull/11477>`_, Casey Bodley)
+* rgw: multisite: coroutine deadlock assertion on error in FetchAllMetaCR (`issue#17571 <http://tracker.ceph.com/issues/17571>`_, `pr#11866 <http://github.com/ceph/ceph/pull/11866>`_, Casey Bodley)
+* rgw: multisite: coroutine deadlock in RGWMetaSyncCR after ECANCELED errors (`issue#17465 <http://tracker.ceph.com/issues/17465>`_, `pr#12738 <http://github.com/ceph/ceph/pull/12738>`_, Casey Bodley)
+* rgw: multisite doesn't retry RGWFetchAllMetaCR on failed lease (`issue#17047 <http://tracker.ceph.com/issues/17047>`_, `pr#11476 <http://github.com/ceph/ceph/pull/11476>`_, Casey Bodley)
+* rgw: multisite: ECANCELED & 500 error on bucket delete (`issue#17698 <http://tracker.ceph.com/issues/17698>`_, `pr#12044 <http://github.com/ceph/ceph/pull/12044>`_, Casey Bodley)
+* rgw: multisite: failed assertion in 'radosgw-admin bucket sync status' (`issue#18083 <http://tracker.ceph.com/issues/18083>`_, `pr#12314 <http://github.com/ceph/ceph/pull/12314>`_, Casey Bodley)
+* rgw: multisite: fix ref counting of completions (`issue#17792 <http://tracker.ceph.com/issues/17792>`_, `issue#18414 <http://tracker.ceph.com/issues/18414>`_, `issue#17793 <http://tracker.ceph.com/issues/17793>`_, `issue#18407 <http://tracker.ceph.com/issues/18407>`_, `pr#13001 <http://github.com/ceph/ceph/pull/13001>`_, Casey Bodley)
+* rgw: multisite: metadata master can get the wrong value for 'oldest_log_period' (`issue#16894 <http://tracker.ceph.com/issues/16894>`_, `pr#11868 <http://github.com/ceph/ceph/pull/11868>`_, Casey Bodley)
+* rgw: multisite: obsolete 'radosgw-admin period prepare' command (`issue#17387 <http://tracker.ceph.com/issues/17387>`_, `pr#11574 <http://github.com/ceph/ceph/pull/11574>`_, Gaurav Kumar Garg)
+* rgw: multisite: race between ReadSyncStatus and InitSyncStatus leads to EIO errors (`issue#17568 <http://tracker.ceph.com/issues/17568>`_, `pr#11865 <http://github.com/ceph/ceph/pull/11865>`_, Casey Bodley)
+* rgw: multisite requests failing with '400 Bad Request' with civetweb 1.8 (`issue#17822 <http://tracker.ceph.com/issues/17822>`_, `pr#12313 <http://github.com/ceph/ceph/pull/12313>`_, Casey Bodley)
+* rgw: multisite: segfault after changing value of rgw_data_log_num_shards (`issue#18488 <http://tracker.ceph.com/issues/18488>`_, `pr#13180 <http://github.com/ceph/ceph/pull/13180>`_, Casey Bodley)
+* rgw: multisite: sync status reports master is on a different period (`issue#18064 <http://tracker.ceph.com/issues/18064>`_, `pr#13175 <http://github.com/ceph/ceph/pull/13175>`_, Abhishek Lekshmanan)
+* rgw: multisite upgrade from hammer -> jewel ignores rgw_region_root_pool (`issue#17963 <http://tracker.ceph.com/issues/17963>`_, `pr#12156 <http://github.com/ceph/ceph/pull/12156>`_, Casey Bodley)
+* rgw: radosgw-admin period update reverts deleted zonegroup  (`issue#17239 <http://tracker.ceph.com/issues/17239>`_, `pr#13171 <http://github.com/ceph/ceph/pull/13171>`_, Orit Wasserman)
+* rgw: Realm set does not create a new period (`issue#18333 <http://tracker.ceph.com/issues/18333>`_, `pr#13182 <http://github.com/ceph/ceph/pull/13182>`_, Orit Wasserman)
+* rgw: remove spurious mount entries for RGW buckets (`issue#17850 <http://tracker.ceph.com/issues/17850>`_, `pr#12045 <http://github.com/ceph/ceph/pull/12045>`_, Matt Benjamin)
+* rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth. (`issue#17076 <http://tracker.ceph.com/issues/17076>`_, `pr#12542 <http://github.com/ceph/ceph/pull/12542>`_, Pritha Srivastava)
+* rgw: rgw-admin: missing command to modify placement targets (`issue#18078 <http://tracker.ceph.com/issues/18078>`_, `pr#12428 <http://github.com/ceph/ceph/pull/12428>`_, Yehuda Sadeh, Casey Bodley)
+* rgw: RGWRados::get_system_obj() sends unnecessary stat request before read (`issue#17580 <http://tracker.ceph.com/issues/17580>`_, `pr#11867 <http://github.com/ceph/ceph/pull/11867>`_, Casey Bodley)
+* rgw: rgw_rest_s3:  apply missed base64 try-catch (`issue#17663 <http://tracker.ceph.com/issues/17663>`_, `pr#11672 <http://github.com/ceph/ceph/pull/11672>`_, Matt Benjamin)
+* rgw: RGW will not list Argonaut-era bucket via HTTP (but radosgw-admin works) (`issue#17372 <http://tracker.ceph.com/issues/17372>`_, `pr#11863 <http://github.com/ceph/ceph/pull/11863>`_, Yehuda Sadeh)
+* rgw: sends omap_getvals with (u64)-1 limit (`issue#17985 <http://tracker.ceph.com/issues/17985>`_, `pr#12419 <http://github.com/ceph/ceph/pull/12419>`_, Yehuda Sadeh, Sage Weil)
+* rgw: slave zonegroup cannot enable the bucket versioning (`issue#18003 <http://tracker.ceph.com/issues/18003>`_, `pr#13173 <http://github.com/ceph/ceph/pull/13173>`_, Orit Wasserman)
+* rgw: TempURL properly handles accounts created with the implicit tenant (`issue#17961 <http://tracker.ceph.com/issues/17961>`_, `pr#12079 <http://github.com/ceph/ceph/pull/12079>`_, Radoslaw Zarzynski)
+* rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt (`issue#17598 <http://tracker.ceph.com/issues/17598>`_, `pr#11876 <http://github.com/ceph/ceph/pull/11876>`_, weiqiaomiao)
+* rgw: Unable to commit period zonegroup change (`issue#17364 <http://tracker.ceph.com/issues/17364>`_, `pr#12315 <http://github.com/ceph/ceph/pull/12315>`_, Orit Wasserman)
+* rgw: valgrind "invalid read size 4" RGWGetObj (`issue#18071 <http://tracker.ceph.com/issues/18071>`_, `pr#12997 <http://github.com/ceph/ceph/pull/12997>`_, Matt Benjamin)
+* rgw: work around curl_multi_wait bug with non-blocking reads (`issue#15915 <http://tracker.ceph.com/issues/15915>`_, `issue#16368 <http://tracker.ceph.com/issues/16368>`_, `issue#16695 <http://tracker.ceph.com/issues/16695>`_, `pr#11627 <http://github.com/ceph/ceph/pull/11627>`_, John Coyle, Casey Bodley)
+* tests: add require_jewel_osds before upgrading last hammer node (`issue#18719 <http://tracker.ceph.com/issues/18719>`_, `pr#13161 <http://github.com/ceph/ceph/pull/13161>`_, Nathan Cutler)
+* tests: add require_jewel_osds to upgrade/hammer-x/tiering (`issue#18920 <http://tracker.ceph.com/issues/18920>`_, `pr#13404 <http://github.com/ceph/ceph/pull/13404>`_, Nathan Cutler)
+* tests: assertion failure in a radosgw-admin related task (`issue#17167 <http://tracker.ceph.com/issues/17167>`_, `pr#12764 <http://github.com/ceph/ceph/pull/12764>`_, Orit Wasserman)
+* tests: Cannot reserve CentOS 7.2 smithi machines (`issue#18416 <http://tracker.ceph.com/issues/18416>`_, `issue#18401 <http://tracker.ceph.com/issues/18401>`_, `pr#13050 <http://github.com/ceph/ceph/pull/13050>`_, Nathan Cutler, Sage Weil, Yuri Weinstein)
+* tests: ignore bogus ceph-objectstore-tool error in ceph_manager (`issue#16263 <http://tracker.ceph.com/issues/16263>`_, `pr#13240 <http://github.com/ceph/ceph/pull/13240>`_, Nathan Cutler, Kefu Chai)
+* tests: objecter_requests workunit fails on wip branches (`issue#18393 <http://tracker.ceph.com/issues/18393>`_, `pr#12761 <http://github.com/ceph/ceph/pull/12761>`_, Sage Weil)
+* tests: qa/suites/upgrade/hammer-x: break stress split ec symlinks (`issue#19006 <http://tracker.ceph.com/issues/19006>`_, `pr#13533 <http://github.com/ceph/ceph/pull/13533>`_, Nathan Cutler)
+* tests: qa/suites/upgrade/hammer-x/stress-split: finish thrashing before final upgrade (`issue#19004 <http://tracker.ceph.com/issues/19004>`_, `pr#13222 <http://github.com/ceph/ceph/pull/13222>`_, Sage Weil)
+* tests: qa/tasks/ceph_deploy.py: use dev option (`issue#18736 <http://tracker.ceph.com/issues/18736>`_, `pr#13106 <http://github.com/ceph/ceph/pull/13106>`_, Vasu Kulkarni)
+* tests: qa/workunits/rbd: use more recent qemu-iotests that support Xenial (`issue#18149 <http://tracker.ceph.com/issues/18149>`_, `issue#10773 <http://tracker.ceph.com/issues/10773>`_, `pr#13103 <http://github.com/ceph/ceph/pull/13103>`_, Jason Dillaman)
+* tests: remove qa/suites/buildpackages (`issue#18846 <http://tracker.ceph.com/issues/18846>`_, `pr#13299 <http://github.com/ceph/ceph/pull/13299>`_, Loic Dachary)
+* tests: SUSE yaml facets in qa/distros/all are out of date (`issue#18856 <http://tracker.ceph.com/issues/18856>`_, `issue#18846 <http://tracker.ceph.com/issues/18846>`_, `pr#13331 <http://github.com/ceph/ceph/pull/13331>`_, Nathan Cutler)
+* tests: update rbd/singleton/all/formatted-output.yaml to support ceph-ci (`issue#18440 <http://tracker.ceph.com/issues/18440>`_, `pr#12822 <http://github.com/ceph/ceph/pull/12822>`_, Nathan Cutler, Venky Shankar)
+* tests: update Ubuntu image url after ceph.com refactor (`issue#18542 <http://tracker.ceph.com/issues/18542>`_, `pr#12959 <http://github.com/ceph/ceph/pull/12959>`_, Jason Dillaman)
+* tests: upgrade:hammer-x: install firefly only on Ubuntu 14.04 (`issue#18089 <http://tracker.ceph.com/issues/18089>`_, `pr#13153 <http://github.com/ceph/ceph/pull/13153>`_, Nathan Cutler)
+* tests: use ceph-jewel branch for s3tests (`issue#18384 <http://tracker.ceph.com/issues/18384>`_, `pr#12745 <http://github.com/ceph/ceph/pull/12745>`_, Nathan Cutler)
+* tests: Workunits needlessly wget from git.ceph.com (`issue#18336 <http://tracker.ceph.com/issues/18336>`_, `issue#18271 <http://tracker.ceph.com/issues/18271>`_, `issue#18388 <http://tracker.ceph.com/issues/18388>`_, `pr#12686 <http://github.com/ceph/ceph/pull/12686>`_, Nathan Cutler, Sage Weil)
+* test: temporarily disable fork()'ing tests (`issue#16556 <http://tracker.ceph.com/issues/16556>`_, `issue#17832 <http://tracker.ceph.com/issues/17832>`_, `pr#11953 <http://github.com/ceph/ceph/pull/11953>`_, John Spray)
+* test: test fails due to The UNIX domain socket path (`issue#16014 <http://tracker.ceph.com/issues/16014>`_, `pr#12151 <http://github.com/ceph/ceph/pull/12151>`_, Loic Dachary)
+* tools: ceph-disk: ceph-disk@.service races with ceph-osd@.service (`issue#17889 <http://tracker.ceph.com/issues/17889>`_, `issue#17813 <http://tracker.ceph.com/issues/17813>`_, `pr#12147 <http://github.com/ceph/ceph/pull/12147>`_, Loic Dachary)
+* tools: ceph-disk --dmcrypt create must not require admin key (`issue#17849 <http://tracker.ceph.com/issues/17849>`_, `pr#12033 <http://github.com/ceph/ceph/pull/12033>`_, Loic Dachary)
+* tools: ceph-disk prepare writes osd log 0 with root owner (`issue#18538 <http://tracker.ceph.com/issues/18538>`_, `pr#13025 <http://github.com/ceph/ceph/pull/13025>`_, Samuel Matzek)
+* tools: crushtool --compile is create output despite of missing item (`issue#17306 <http://tracker.ceph.com/issues/17306>`_, `pr#11410 <http://github.com/ceph/ceph/pull/11410>`_, Kefu Chai)
+* tools: rados bench seq must verify the hostname (`issue#17526 <http://tracker.ceph.com/issues/17526>`_, `pr#13049 <http://github.com/ceph/ceph/pull/13049>`_, Loic Dachary)
+* tools: snapshotted RBD extent objects can't be manually evicted from a cache tier (`issue#17896 <http://tracker.ceph.com/issues/17896>`_, `pr#11968 <http://github.com/ceph/ceph/pull/11968>`_, Mingxin Liu)
+* tools: systemd/ceph-disk: reduce ceph-disk flock contention (`issue#18049 <http://tracker.ceph.com/issues/18049>`_, `issue#13160 <http://tracker.ceph.com/issues/13160>`_, `pr#12210 <http://github.com/ceph/ceph/pull/12210>`_, David Disseldorp)
+
+
+v10.2.5 Jewel
+=============
+
+This point release fixes an important `regression introduced in v10.2.4 <http://tracker.ceph.com/issues/18185>`_.
+
+We recommend that all v10.2.x users upgrade.
+
+Notable Changes
+---------------
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.5.txt>`.
+
+* msg/simple/Pipe: avoid returning 0 on poll timeout (`issue#18185 <http://tracker.ceph.com/issues/18185>`_, `pr#12376 <https://github.com/ceph/ceph/pull/12376>`_, Sage Weil)
+
+v10.2.4 Jewel
+=============
+
+This point release fixes several important bugs in RBD mirroring, RGW multi-site, CephFS, and RADOS.
+
+We recommend that all v10.2.x users upgrade. Also note the following when upgrading from hammer
+
+Upgrading from hammer
+---------------------
+
+When the last hammer OSD in a cluster containing jewel MONs is
+upgraded to jewel, as of 10.2.4 the jewel MONs will issue this
+warning: "all OSDs are running jewel or later but the
+'require_jewel_osds' osdmap flag is not set" and change the
+cluster health status to HEALTH_WARN.
+
+This is a signal for the admin to do "ceph osd set require_jewel_osds" - by
+doing this, the upgrade path is complete and no more pre-Jewel OSDs may be added
+to the cluster.
+
+
+Notable Changes
+---------------
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.4.txt>`.
+
+* build/ops: aarch64: Compiler-based detection of crc32 extended CPU type is broken (`issue#17516 <http://tracker.ceph.com/issues/17516>`_, `pr#11492 <http://github.com/ceph/ceph/pull/11492>`_, Alexander Graf)
+* build/ops: allow building RGW with LDAP disabled (`issue#17312 <http://tracker.ceph.com/issues/17312>`_, `pr#11478 <http://github.com/ceph/ceph/pull/11478>`_, Daniel Gryniewicz)
+* build/ops: backport 'logrotate: Run as root/ceph' (`issue#17381 <http://tracker.ceph.com/issues/17381>`_, `pr#11201 <http://github.com/ceph/ceph/pull/11201>`_, Boris Ranto)
+* build/ops: ceph installs stuff in %_udevrulesdir but does not own that directory (`issue#16949 <http://tracker.ceph.com/issues/16949>`_, `pr#10862 <http://github.com/ceph/ceph/pull/10862>`_, Nathan Cutler)
+* build/ops: ceph-osd-prestart.sh fails confusingly when data directory does not exist (`issue#17091 <http://tracker.ceph.com/issues/17091>`_, `pr#10812 <http://github.com/ceph/ceph/pull/10812>`_, Nathan Cutler)
+* build/ops: disable LTTng-UST in openSUSE builds (`issue#16937 <http://tracker.ceph.com/issues/16937>`_, `pr#10794 <http://github.com/ceph/ceph/pull/10794>`_, Michel Normand)
+* build/ops: i386 tarball gitbuilder failure on master (`issue#16398 <http://tracker.ceph.com/issues/16398>`_, `pr#10855 <http://github.com/ceph/ceph/pull/10855>`_, Vikhyat Umrao, Kefu Chai)
+* build/ops: include more files in "make dist" tarball (`issue#17560 <http://tracker.ceph.com/issues/17560>`_, `pr#11431 <http://github.com/ceph/ceph/pull/11431>`_, Ken Dreyer)
+* build/ops: incorrect value of CINIT_FLAG_DEFER_DROP_PRIVILEGES (`issue#16663 <http://tracker.ceph.com/issues/16663>`_, `pr#10278 <http://github.com/ceph/ceph/pull/10278>`_, Casey Bodley)
+* build/ops: remove SYSTEMD_RUN from initscript (`issue#7627 <http://tracker.ceph.com/issues/7627>`_, `issue#16441 <http://tracker.ceph.com/issues/16441>`_, `issue#16440 <http://tracker.ceph.com/issues/16440>`_, `pr#9872 <http://github.com/ceph/ceph/pull/9872>`_, Vladislav Odintsov)
+* build/ops: systemd: add install section to rbdmap.service file (`issue#17541 <http://tracker.ceph.com/issues/17541>`_, `pr#11158 <http://github.com/ceph/ceph/pull/11158>`_, Jelle vd Kooij)
+* common: Enable/Disable of features is allowed even the features are already enabled/disabled (`issue#16079 <http://tracker.ceph.com/issues/16079>`_, `pr#11460 <http://github.com/ceph/ceph/pull/11460>`_, Lu Shi)
+* common: Log.cc: Assign LOG_INFO priority to syslog calls (`issue#15808 <http://tracker.ceph.com/issues/15808>`_, `pr#11231 <http://github.com/ceph/ceph/pull/11231>`_, Brad Hubbard)
+* common: Proxied operations shouldn't result in error messages if replayed (`issue#16130 <http://tracker.ceph.com/issues/16130>`_, `pr#11461 <http://github.com/ceph/ceph/pull/11461>`_, Vikhyat Umrao)
+* common: Request exclusive lock if owner sends -ENOTSUPP for proxied maintenance op (`issue#16171 <http://tracker.ceph.com/issues/16171>`_, `pr#10784 <http://github.com/ceph/ceph/pull/10784>`_, Jason Dillaman)
+* common: msgr/async: Messenger thread long time lock hold risk (`issue#15758 <http://tracker.ceph.com/issues/15758>`_, `pr#10761 <http://github.com/ceph/ceph/pull/10761>`_, Wei Jin)
+* doc: fix description for rsize and rasize (`issue#17357 <http://tracker.ceph.com/issues/17357>`_, `pr#11171 <http://github.com/ceph/ceph/pull/11171>`_, Andreas Gerstmayr)
+* filestore: can get stuck in an unbounded loop during scrub (`issue#17859 <http://tracker.ceph.com/issues/17859>`_, `pr#12001 <http://github.com/ceph/ceph/pull/12001>`_, Sage Weil)
+* fs: Failure in snaptest-git-ceph.sh (`issue#17172 <http://tracker.ceph.com/issues/17172>`_, `pr#11419 <http://github.com/ceph/ceph/pull/11419>`_, Yan, Zheng)
+* fs: Log path as well as ino when detecting metadata damage (`issue#16973 <http://tracker.ceph.com/issues/16973>`_, `pr#11418 <http://github.com/ceph/ceph/pull/11418>`_, John Spray)
+* fs: client: FAILED assert(root_ancestor->qtree == __null) (`issue#16066 <http://tracker.ceph.com/issues/16066>`_, `issue#16067 <http://tracker.ceph.com/issues/16067>`_, `pr#10107 <http://github.com/ceph/ceph/pull/10107>`_, Yan, Zheng)
+* fs: client: add missing client_lock for get_root (`issue#17197 <http://tracker.ceph.com/issues/17197>`_, `pr#10921 <http://github.com/ceph/ceph/pull/10921>`_, Patrick Donnelly)
+* fs: client: fix shutdown with open inodes (`issue#16764 <http://tracker.ceph.com/issues/16764>`_, `pr#10958 <http://github.com/ceph/ceph/pull/10958>`_, John Spray)
+* fs: client: nlink count is not maintained correctly (`issue#16668 <http://tracker.ceph.com/issues/16668>`_, `pr#10877 <http://github.com/ceph/ceph/pull/10877>`_, Jeff Layton)
+* fs: multimds: allow_multimds not required when max_mds is set in ceph.conf at startup (`issue#17105 <http://tracker.ceph.com/issues/17105>`_, `pr#10997 <http://github.com/ceph/ceph/pull/10997>`_, Patrick Donnelly)
+* librados: memory leaks from ceph::crypto (WITH_NSS) (`issue#17205 <http://tracker.ceph.com/issues/17205>`_, `pr#11409 <http://github.com/ceph/ceph/pull/11409>`_, Casey Bodley)
+* librados: modify Pipe::connect() to return the error code (`issue#15308 <http://tracker.ceph.com/issues/15308>`_, `pr#11193 <http://github.com/ceph/ceph/pull/11193>`_, Vikhyat Umrao)
+* librados: remove new setxattr overload to avoid breaking the C++ ABI (`issue#18058 <http://tracker.ceph.com/issues/18058>`_, `pr#12207 <http://github.com/ceph/ceph/pull/12207>`_, Josh Durgin)
+* librbd: cannot disable journaling or remove non-mirrored, non-primary image (`issue#16740 <http://tracker.ceph.com/issues/16740>`_, `pr#11337 <http://github.com/ceph/ceph/pull/11337>`_, Jason Dillaman)
+* librbd: discard after write can result in assertion failure (`issue#17695 <http://tracker.ceph.com/issues/17695>`_, `pr#11644 <http://github.com/ceph/ceph/pull/11644>`_, Jason Dillaman)
+* librbd::Operations: update notification failed: (2) No such file or directory (`issue#17549 <http://tracker.ceph.com/issues/17549>`_, `pr#11420 <http://github.com/ceph/ceph/pull/11420>`_, Jason Dillaman)
+* mds: Crash in Client::_invalidate_kernel_dcache when reconnecting during unmount (`issue#17253 <http://tracker.ceph.com/issues/17253>`_, `pr#11414 <http://github.com/ceph/ceph/pull/11414>`_, Yan, Zheng)
+* mds: Duplicate damage table entries (`issue#17173 <http://tracker.ceph.com/issues/17173>`_, `pr#11412 <http://github.com/ceph/ceph/pull/11412>`_, John Spray)
+* mds: Failure in dirfrag.sh (`issue#17286 <http://tracker.ceph.com/issues/17286>`_, `pr#11416 <http://github.com/ceph/ceph/pull/11416>`_, Yan, Zheng)
+* mds: Failure in snaptest-git-ceph.sh (`issue#17271 <http://tracker.ceph.com/issues/17271>`_, `pr#11415 <http://github.com/ceph/ceph/pull/11415>`_, Yan, Zheng)
+* mon: Ceph Status - Segmentation Fault (`issue#16266 <http://tracker.ceph.com/issues/16266>`_, `pr#11408 <http://github.com/ceph/ceph/pull/11408>`_, Brad Hubbard)
+* mon: Display full flag in ceph status if full flag is set (`issue#15809 <http://tracker.ceph.com/issues/15809>`_, `pr#9388 <http://github.com/ceph/ceph/pull/9388>`_, Vikhyat Umrao)
+* mon: Error EINVAL: removing mon.a at 172.21.15.16:6789/0, there will be 1 monitors (`issue#17725 <http://tracker.ceph.com/issues/17725>`_, `pr#12267 <http://github.com/ceph/ceph/pull/12267>`_, Joao Eduardo Luis)
+* mon: OSDMonitor: only reject MOSDBoot based on up_from if inst matches (`issue#17899 <http://tracker.ceph.com/issues/17899>`_, `pr#12067 <http://github.com/ceph/ceph/pull/12067>`_, Samuel Just)
+* mon: OSDMonitor: Missing nearfull flag set (`issue#17390 <http://tracker.ceph.com/issues/17390>`_, `pr#11272 <http://github.com/ceph/ceph/pull/11272>`_, Igor Podoski)
+* mon: Upgrading 0.94.6 -> 0.94.9 saturating mon node networking (`issue#17365 <http://tracker.ceph.com/issues/17365>`_, `issue#17386 <http://tracker.ceph.com/issues/17386>`_, `pr#11679 <http://github.com/ceph/ceph/pull/11679>`_, Sage Weil, xie xingguo)
+* mon: ceph mon Segmentation fault after set crush_ruleset ceph 10.2.2 (`issue#16653 <http://tracker.ceph.com/issues/16653>`_, `pr#10861 <http://github.com/ceph/ceph/pull/10861>`_, song baisen)
+* mon: crash: crush/CrushWrapper.h: 940: FAILED assert(successful_detach) (`issue#16525 <http://tracker.ceph.com/issues/16525>`_, `pr#10496 <http://github.com/ceph/ceph/pull/10496>`_, Kefu Chai)
+* mon: don't crash on invalid standby_for_fscid (`issue#17466 <http://tracker.ceph.com/issues/17466>`_, `pr#11389 <http://github.com/ceph/ceph/pull/11389>`_, John Spray)
+* mon: fix missing osd metadata (again) (`issue#17685 <http://tracker.ceph.com/issues/17685>`_, `pr#11642 <http://github.com/ceph/ceph/pull/11642>`_, John Spray)
+* mon: osdmonitor: decouple adjust_heartbeat_grace and min_down_reporters (`issue#17055 <http://tracker.ceph.com/issues/17055>`_, `pr#10757 <http://github.com/ceph/ceph/pull/10757>`_, Zengran Zhang)
+* mon: the %USED of ceph df is wrong (`issue#16933 <http://tracker.ceph.com/issues/16933>`_, `pr#10860 <http://github.com/ceph/ceph/pull/10860>`_, Kefu Chai)
+* osd: condition OSDMap encoding on features (`issue#18015 <http://tracker.ceph.com/issues/18015>`_, `pr#12167 <http://github.com/ceph/ceph/pull/12167>`_, Sage Weil)
+* osd: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items (`issue#16998 <http://tracker.ceph.com/issues/16998>`_, `pr#10883 <http://github.com/ceph/ceph/pull/10883>`_, Samuel Just)
+* osd: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (\*this == hobject_t(hobject_t::get_max()))) (`issue#13967 <http://tracker.ceph.com/issues/13967>`_, `pr#10885 <http://github.com/ceph/ceph/pull/10885>`_, Tao Chang)
+* osd: Potential crash during journal::Replay shut down (`issue#16433 <http://tracker.ceph.com/issues/16433>`_, `pr#10645 <http://github.com/ceph/ceph/pull/10645>`_, Jason Dillaman)
+* osd: add peer_addr in heartbeat_check log message (`issue#15762 <http://tracker.ceph.com/issues/15762>`_, `pr#9739 <http://github.com/ceph/ceph/pull/9739>`_, Vikhyat Umrao, Sage Weil)
+* osd: adjust scrub boundary to object without SnapSet (`issue#17470 <http://tracker.ceph.com/issues/17470>`_, `pr#11311 <http://github.com/ceph/ceph/pull/11311>`_, Samuel Just)
+* osd: ceph osd df does not show summarized info correctly if one or more OSDs are out (`issue#16706 <http://tracker.ceph.com/issues/16706>`_, `pr#10759 <http://github.com/ceph/ceph/pull/10759>`_, xie xingguo)
+* osd: journal: do not prematurely flag object recorder as closed (`issue#17590 <http://tracker.ceph.com/issues/17590>`_, `pr#11634 <http://github.com/ceph/ceph/pull/11634>`_, Jason Dillaman)
+* osd: mark_all_unfound_lost() leaves unapplied changes (`issue#16156 <http://tracker.ceph.com/issues/16156>`_, `pr#10886 <http://github.com/ceph/ceph/pull/10886>`_, Samuel Just)
+* osd: segfault in ObjectCacher::FlusherThread (`issue#16610 <http://tracker.ceph.com/issues/16610>`_, `pr#10864 <http://github.com/ceph/ceph/pull/10864>`_, Yan, Zheng)
+* qa: remove EnumerateObjects from librados upgrade tests (`pr#11728 <https://github.com/ceph/ceph/pull/11728>`_, Josh Durgin)
+* rbd: Disabling pool mirror mode with registered peers results orphaned mirrored images (`issue#16984 <http://tracker.ceph.com/issues/16984>`_, `pr#10857 <http://github.com/ceph/ceph/pull/10857>`_, Jason Dillaman)
+* rbd: ImageWatcher: use after free within C_UnwatchAndFlush (`issue#17289 <http://tracker.ceph.com/issues/17289>`_, `issue#17254 <http://tracker.ceph.com/issues/17254>`_, `pr#11466 <http://github.com/ceph/ceph/pull/11466>`_, Jason Dillaman)
+* rbd: Prevent the creation of a clone from a non-primary mirrored image (`issue#16449 <http://tracker.ceph.com/issues/16449>`_, `pr#10650 <http://github.com/ceph/ceph/pull/10650>`_, Mykola Golub)
+* rbd: RBD should restrict mirror enable/disable actions on parents/clones (`issue#16056 <http://tracker.ceph.com/issues/16056>`_, `pr#11459 <http://github.com/ceph/ceph/pull/11459>`_, zhuangzeqiang)
+* rbd: TestJournalReplay: sporadic assert(m_state == STATE_READY || m_state == STATE_STOPPING) failure (`issue#17566 <http://tracker.ceph.com/issues/17566>`_, `pr#11590 <http://github.com/ceph/ceph/pull/11590>`_, Jason Dillaman)
+* rbd: bench io-size should not be larger than image size (`issue#16967 <http://tracker.ceph.com/issues/16967>`_, `pr#10796 <http://github.com/ceph/ceph/pull/10796>`_, Jason Dillaman)
+* rbd: ceph 10.2.2 rbd status on image format 2 returns (2) No such file or directory (`issue#16887 <http://tracker.ceph.com/issues/16887>`_, `pr#10652 <http://github.com/ceph/ceph/pull/10652>`_, Jason Dillaman)
+* rbd: helgrind: TestLibRBD.TestIOPP potential deadlock closing an image with read-ahead enabled (`issue#17198 <http://tracker.ceph.com/issues/17198>`_, `pr#11463 <http://github.com/ceph/ceph/pull/11463>`_, Jason Dillaman)
+* rbd: image.stat() call in librbdpy fails sometimes (`issue#17310 <http://tracker.ceph.com/issues/17310>`_, `pr#11464 <http://github.com/ceph/ceph/pull/11464>`_, Jason Dillaman)
+* rbd: krbd qa scripts and concurrent.sh test fix (`issue#17223 <http://tracker.ceph.com/issues/17223>`_, `pr#11018 <http://github.com/ceph/ceph/pull/11018>`_, Ilya Dryomov)
+* rbd: krbd-related CLI patches (`issue#17554 <http://tracker.ceph.com/issues/17554>`_, `pr#11400 <http://github.com/ceph/ceph/pull/11400>`_, Ilya Dryomov)
+* rbd: mirror: improve resiliency of stress test case (`issue#16855 <http://tracker.ceph.com/issues/16855>`_, `issue#16555 <http://tracker.ceph.com/issues/16555>`_, `issue#14738 <http://tracker.ceph.com/issues/14738>`_, `issue#15259 <http://tracker.ceph.com/issues/15259>`_, `issue#17446 <http://tracker.ceph.com/issues/17446>`_, `issue#17355 <http://tracker.ceph.com/issues/17355>`_, `issue#16538 <http://tracker.ceph.com/issues/16538>`_, `issue#16974 <http://tracker.ceph.com/issues/16974>`_, `issue#17283 <http://tracker.ceph.com/issues/17283>`_, `issue#17317 <http://tracker.ceph.com/issues/17317>`_, `issue#17416 <http://tracker.ceph.com/issues/17416>`_, `issue#16227 <http://tracker.ceph.com/issues/16227>`_, `pr#11433 <http://github.com/ceph/ceph/pull/11433>`_, Mykola Golub, Ricardo Dias, Jason Dillaman)
+* rbd: rbd-nbd IO hang (`issue#16921 <http://tracker.ceph.com/issues/16921>`_, `pr#11467 <http://github.com/ceph/ceph/pull/11467>`_, Jason Dillaman)
+* rbd: update_features API needs to support backwards/forward compatibility (`issue#17330 <http://tracker.ceph.com/issues/17330>`_, `pr#11462 <http://github.com/ceph/ceph/pull/11462>`_, Jason Dillaman)
+* rgw: COPY broke multipart files uploaded under dumpling (`issue#16435 <http://tracker.ceph.com/issues/16435>`_, `pr#10866 <http://github.com/ceph/ceph/pull/10866>`_, Yehuda Sadeh)
+* rgw: Config parameter rgw keystone make new tenants in radosgw multitenancy does not work (`issue#17293 <http://tracker.ceph.com/issues/17293>`_, `pr#11473 <http://github.com/ceph/ceph/pull/11473>`_, SirishaGuduru)
+* rgw: Do not archive metadata by default (`issue#17256 <http://tracker.ceph.com/issues/17256>`_, `pr#11321 <http://github.com/ceph/ceph/pull/11321>`_, Pavan Rallabhandi, Matt Benjamin)
+* rgw: ERROR: got unexpected error when trying to read object: -2 (`issue#17111 <http://tracker.ceph.com/issues/17111>`_, `pr#11472 <http://github.com/ceph/ceph/pull/11472>`_, Yang Honggang)
+* rgw: Modification for TEST S3 ACCESS section in INSTALL CEPH OBJECT GATEWAY page (`issue#15603 <http://tracker.ceph.com/issues/15603>`_, `pr#11475 <http://github.com/ceph/ceph/pull/11475>`_, la-sguduru)
+* rgw: RGW loses realm/period/zonegroup/zone data: period overwritten if somewhere in the cluster is still running Hammer (`issue#17371 <http://tracker.ceph.com/issues/17371>`_, `pr#11519 <http://github.com/ceph/ceph/pull/11519>`_, Orit Wasserman)
+* rgw: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR (`issue#17073 <http://tracker.ceph.com/issues/17073>`_, `pr#11330 <http://github.com/ceph/ceph/pull/11330>`_, Casey Bodley)
+* rgw: S3 object versioning fails when applied on a non-master zone (`issue#16494 <http://tracker.ceph.com/issues/16494>`_, `pr#11367 <http://github.com/ceph/ceph/pull/11367>`_, Yehuda Sadeh)
+* rgw: add orphan options to radosgw-admin --help and man page (`issue#17281 <http://tracker.ceph.com/issues/17281>`_, `issue#17280 <http://tracker.ceph.com/issues/17280>`_, `pr#11139 <http://github.com/ceph/ceph/pull/11139>`_, Ken Dreyer, Thomas Serlin)
+* rgw: back off bucket sync on failures, don't store marker (`issue#16742 <http://tracker.ceph.com/issues/16742>`_, `pr#11021 <http://github.com/ceph/ceph/pull/11021>`_, Yehuda Sadeh)
+* rgw: combined LDAP backports (`issue#17544 <http://tracker.ceph.com/issues/17544>`_, `issue#17185 <http://tracker.ceph.com/issues/17185>`_, `pr#11332 <http://github.com/ceph/ceph/pull/11332>`_, Harald Klein, Matt Benjamin)
+* rgw: cors auto memleak (`issue#16564 <http://tracker.ceph.com/issues/16564>`_, `pr#10656 <http://github.com/ceph/ceph/pull/10656>`_, Yan Jun)
+* rgw: default quota fixes (`issue#16410 <http://tracker.ceph.com/issues/16410>`_, `pr#10832 <http://github.com/ceph/ceph/pull/10832>`_, Pavan Rallabhandi, Daniel Gryniewicz)
+* rgw: doc: description of multipart part entity is wrong (`issue#17504 <http://tracker.ceph.com/issues/17504>`_, `pr#11342 <http://github.com/ceph/ceph/pull/11342>`_, weiqiaomiao)
+* rgw: don't loop forever when reading data from 0 sized segment. (`issue#17692 <http://tracker.ceph.com/issues/17692>`_, `pr#11626 <http://github.com/ceph/ceph/pull/11626>`_, Marcus Watts)
+* rgw: fix put_acls for objects starting and ending with underscore (`issue#17625 <http://tracker.ceph.com/issues/17625>`_, `pr#11669 <http://github.com/ceph/ceph/pull/11669>`_, Orit Wasserman)
+* rgw: fix regression with handling double underscore (`issue#17443 <http://tracker.ceph.com/issues/17443>`_, `issue#16856 <http://tracker.ceph.com/issues/16856>`_, `pr#11563 <http://github.com/ceph/ceph/pull/11563>`_, Yehuda Sadeh, Orit Wasserman)
+* rgw: handle empty POST condition (`issue#17635 <http://tracker.ceph.com/issues/17635>`_, `pr#11662 <http://github.com/ceph/ceph/pull/11662>`_, Yehuda Sadeh)
+* rgw: metadata sync can skip markers for failed/incomplete entries (`issue#16759 <http://tracker.ceph.com/issues/16759>`_, `pr#10657 <http://github.com/ceph/ceph/pull/10657>`_, Yehuda Sadeh)
+* rgw: nfs backports (`issue#17393 <http://tracker.ceph.com/issues/17393>`_, `issue#17311 <http://tracker.ceph.com/issues/17311>`_, `issue#17367 <http://tracker.ceph.com/issues/17367>`_, `issue#17319 <http://tracker.ceph.com/issues/17319>`_, `issue#17321 <http://tracker.ceph.com/issues/17321>`_, `issue#17322 <http://tracker.ceph.com/issues/17322>`_, `issue#17323 <http://tracker.ceph.com/issues/17323>`_, `issue#17325 <http://tracker.ceph.com/issues/17325>`_, `issue#17326 <http://tracker.ceph.com/issues/17326>`_, `issue#17327 <http://tracker.ceph.com/issues/17327>`_, `pr#11335 <http://github.com/ceph/ceph/pull/11335>`_, Min Chen, Yan Jun, Weibing Zhang, Matt Benjamin)
+* rgw: period commit loses zonegroup changes: region_map converted repeatedly (`issue#17051 <http://tracker.ceph.com/issues/17051>`_, `pr#10890 <http://github.com/ceph/ceph/pull/10890>`_, Casey Bodley)
+* rgw: period commit return error when the current period has a zonegroup which doesn't have a master zone (`issue#17110 <http://tracker.ceph.com/issues/17110>`_, `pr#10867 <http://github.com/ceph/ceph/pull/10867>`_, weiqiaomiao)
+* rgw: radosgw daemon core when reopen logs (`issue#17036 <http://tracker.ceph.com/issues/17036>`_, `pr#10868 <http://github.com/ceph/ceph/pull/10868>`_, weiqiaomiao)
+* rgw: rgw file uses too much CPU in gc/idle thread (`issue#16976 <http://tracker.ceph.com/issues/16976>`_, `pr#10889 <http://github.com/ceph/ceph/pull/10889>`_, Matt Benjamin)
+* rgw: s3tests-test-readwrite failing with 500 (`issue#16930 <http://tracker.ceph.com/issues/16930>`_, `pr#11471 <http://github.com/ceph/ceph/pull/11471>`_, Yehuda Sadeh)
+* rgw: upgrade from old multisite to new multisite fails (`issue#16751 <http://tracker.ceph.com/issues/16751>`_, `pr#10891 <http://github.com/ceph/ceph/pull/10891>`_, Orit Wasserman)
+* rgw:response information is error when geting token of swift account (`issue#15195 <http://tracker.ceph.com/issues/15195>`_, `pr#11474 <http://github.com/ceph/ceph/pull/11474>`_, Qiankun Zheng)
+* rgw:user email can modify to empty when it has values (`issue#13286 <http://tracker.ceph.com/issues/13286>`_, `pr#11469 <http://github.com/ceph/ceph/pull/11469>`_, Yehuda Sadeh, Weijun Duan)
+* tests: ceph-disk must ignore debug monc (`issue#17607 <http://tracker.ceph.com/issues/17607>`_, `pr#11548 <http://github.com/ceph/ceph/pull/11548>`_, Loic Dachary)
+* tests: fix TestClsRbd.mirror_image failure in upgrade:jewel-x-master-distro-basic-vps (`issue#16529 <http://tracker.ceph.com/issues/16529>`_, `pr#10888 <http://github.com/ceph/ceph/pull/10888>`_, Jason Dillaman)
+* tests: scsi_debug fails /dev/disk/by-partuuid (`issue#17100 <http://tracker.ceph.com/issues/17100>`_, `pr#11411 <http://github.com/ceph/ceph/pull/11411>`_, Loic Dachary)
+* tests: test/ceph_test_msgr: do not use Message::middle for holding transient… (`issue#17365 <http://tracker.ceph.com/issues/17365>`_, `issue#17728 <http://tracker.ceph.com/issues/17728>`_, `issue#16955 <http://tracker.ceph.com/issues/16955>`_, `pr#11742 <http://github.com/ceph/ceph/pull/11742>`_, Haomai Wang, Kefu Chai, Michal Jarzabek, Sage Weil)
+* tools: Missing comma in ceph-create-keys causes concatenation of arguments (`issue#17815 <http://tracker.ceph.com/issues/17815>`_, `pr#11822 <http://github.com/ceph/ceph/pull/11822>`_, Patrick Donnelly)
+* tools: add a tool to rebuild mon store from OSD (`issue#17179 <http://tracker.ceph.com/issues/17179>`_, `issue#17400 <http://tracker.ceph.com/issues/17400>`_, `pr#11126 <http://github.com/ceph/ceph/pull/11126>`_, Kefu Chai, xie xingguo)
+* tools: ceph-create-keys: sometimes blocks forever if mds allow is set (`issue#16255 <http://tracker.ceph.com/issues/16255>`_, `pr#11417 <http://github.com/ceph/ceph/pull/11417>`_, John Spray)
+* tools: ceph-disk should timeout when a lock cannot be acquired (`issue#16580 <http://tracker.ceph.com/issues/16580>`_, `pr#10758 <http://github.com/ceph/ceph/pull/10758>`_, Loic Dachary)
+* tools: ceph-disk: expected systemd unit failures are confusing (`issue#15990 <http://tracker.ceph.com/issues/15990>`_, `pr#10884 <http://github.com/ceph/ceph/pull/10884>`_, Boris Ranto)
+* tools: ceph-disk: using a regular file as a journal fails (`issue#16280 <http://tracker.ceph.com/issues/16280>`_, `issue#17662 <http://tracker.ceph.com/issues/17662>`_, `pr#11657 <http://github.com/ceph/ceph/pull/11657>`_, Jayashree Candadai, Anirudha Bose, Loic Dachary, Shylesh Kumar)
+* tools: ceph-objectstore-tool crashes if --journal-path <a-directory> (`issue#17307 <http://tracker.ceph.com/issues/17307>`_, `pr#11407 <http://github.com/ceph/ceph/pull/11407>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: add a way to split filestore directories offline (`issue#17220 <http://tracker.ceph.com/issues/17220>`_, `pr#11252 <http://github.com/ceph/ceph/pull/11252>`_, Josh Durgin)
+* tools: ceph-post-file: use new ssh key (`issue#14267 <http://tracker.ceph.com/issues/14267>`_, `pr#11746 <http://github.com/ceph/ceph/pull/11746>`_, David Galloway)
+
+
+v10.2.3 Jewel
+=============
+
+This point release fixes several important bugs in RBD mirroring, RGW multi-site, CephFS, and RADOS.
+
+We recommend that all v10.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.3.txt>`.
+
+Notable Changes
+---------------
+
+* build/ops: 60-ceph-partuuid-workaround-rules still needed by debian jessie (udev 215-17) (`issue#16351 <http://tracker.ceph.com/issues/16351>`_, `pr#10653 <http://github.com/ceph/ceph/pull/10653>`_, runsisi, Loic Dachary)
+* build/ops: ceph Resource Agent does not work with systemd (`issue#14828 <http://tracker.ceph.com/issues/14828>`_, `pr#9917 <http://github.com/ceph/ceph/pull/9917>`_, Nathan Cutler)
+* build/ops: ceph-base requires parted (`issue#16095 <http://tracker.ceph.com/issues/16095>`_, `pr#10008 <http://github.com/ceph/ceph/pull/10008>`_, Ken Dreyer)
+* build/ops: ceph-osd-prestart.sh contains Upstart-specific code (`issue#15984 <http://tracker.ceph.com/issues/15984>`_, `pr#10364 <http://github.com/ceph/ceph/pull/10364>`_, Nathan Cutler)
+* build/ops: mount.ceph: move from ceph-base to ceph-common and add symlink in /sbin for SUSE (`issue#16598 <http://tracker.ceph.com/issues/16598>`_, `issue#16645 <http://tracker.ceph.com/issues/16645>`_, `pr#10357 <http://github.com/ceph/ceph/pull/10357>`_, Nathan Cutler, Dan Horák, Ricardo Dias, Kefu Chai)
+* build/ops: need rocksdb commit 7ca731b12ce for ppc64le build (`issue#17092 <http://tracker.ceph.com/issues/17092>`_, `pr#10816 <http://github.com/ceph/ceph/pull/10816>`_, Nathan Cutler)
+* build/ops: rpm: OBS needs ExclusiveArch (`issue#16936 <http://tracker.ceph.com/issues/16936>`_, `pr#10614 <http://github.com/ceph/ceph/pull/10614>`_, Michel Normand)
+* cli: ceph command line tool chokes on ceph –w (the dash is unicode 'en dash' &ndash, copy-paste to reproduce) (`issue#12287 <http://tracker.ceph.com/issues/12287>`_, `pr#10420 <http://github.com/ceph/ceph/pull/10420>`_, Oleh Prypin, Kefu Chai)
+* common: expose buffer const_iterator symbols (`issue#16899 <http://tracker.ceph.com/issues/16899>`_, `pr#10552 <http://github.com/ceph/ceph/pull/10552>`_, Noah Watkins)
+* common: global-init: fixup chown of the run directory along with log and asok files (`issue#15607 <http://tracker.ceph.com/issues/15607>`_, `pr#8754 <http://github.com/ceph/ceph/pull/8754>`_, Karol Mroz)
+* fs: ceph-fuse: link to libtcmalloc or jemalloc (`issue#16655 <http://tracker.ceph.com/issues/16655>`_, `pr#10303 <http://github.com/ceph/ceph/pull/10303>`_, Yan, Zheng)
+* fs: client: crash in unmount when fuse_use_invalidate_cb is enabled (`issue#16137 <http://tracker.ceph.com/issues/16137>`_, `pr#10106 <http://github.com/ceph/ceph/pull/10106>`_, Yan, Zheng)
+* fs: client: fstat cap release (`issue#15723 <http://tracker.ceph.com/issues/15723>`_, `pr#9562 <http://github.com/ceph/ceph/pull/9562>`_, Yan, Zheng, Noah Watkins)
+* fs: essential backports for OpenStack Manila (`issue#15406 <http://tracker.ceph.com/issues/15406>`_, `issue#15614 <http://tracker.ceph.com/issues/15614>`_, `issue#15615 <http://tracker.ceph.com/issues/15615>`_, `pr#10453 <http://github.com/ceph/ceph/pull/10453>`_, John Spray, Ramana Raja, Xiaoxi Chen)
+* fs: fix double-unlock on shutdown (`issue#17126 <http://tracker.ceph.com/issues/17126>`_, `pr#10847 <http://github.com/ceph/ceph/pull/10847>`_, Greg Farnum)
+* fs: fix mdsmap print_summary with standby replays (`issue#15705 <http://tracker.ceph.com/issues/15705>`_, `pr#9547 <http://github.com/ceph/ceph/pull/9547>`_, John Spray)
+* fs: fuse mounted file systems fails SAMBA CTDB ping_pong rw test with v9.0.2 (`issue#12653 <http://tracker.ceph.com/issues/12653>`_, `issue#15634 <http://tracker.ceph.com/issues/15634>`_, `pr#10108 <http://github.com/ceph/ceph/pull/10108>`_, Yan, Zheng)
+* librados: Missing export for rados_aio_get_version in src/include/rados/librados.h (`issue#15535 <http://tracker.ceph.com/issues/15535>`_, `pr#9574 <http://github.com/ceph/ceph/pull/9574>`_, Jim Wright)
+* librados: osd: bad flags can crash the osd (`issue#16012 <http://tracker.ceph.com/issues/16012>`_, `pr#9997 <http://github.com/ceph/ceph/pull/9997>`_, Sage Weil)
+* librbd: Close journal and object map before flagging exclusive lock as released (`issue#16450 <http://tracker.ceph.com/issues/16450>`_, `pr#10053 <http://github.com/ceph/ceph/pull/10053>`_, Jason Dillaman)
+* librbd: Crash when utilizing advisory locking API functions (`issue#16364 <http://tracker.ceph.com/issues/16364>`_, `pr#10051 <http://github.com/ceph/ceph/pull/10051>`_, Jason Dillaman)
+* librbd: ExclusiveLock object leaked when switching to snapshot (`issue#16446 <http://tracker.ceph.com/issues/16446>`_, `pr#10054 <http://github.com/ceph/ceph/pull/10054>`_, Jason Dillaman)
+* librbd: FAILED assert(object_no < m_object_map.size()) (`issue#16561 <http://tracker.ceph.com/issues/16561>`_, `pr#10647 <http://github.com/ceph/ceph/pull/10647>`_, Jason Dillaman)
+* librbd: Image removal doesn't necessarily clean up all rbd_mirroring entries (`issue#16471 <http://tracker.ceph.com/issues/16471>`_, `pr#10009 <http://github.com/ceph/ceph/pull/10009>`_, Jason Dillaman)
+* librbd: Object map/fast-diff invalidated if journal replays the same snap remove event (`issue#16350 <http://tracker.ceph.com/issues/16350>`_, `pr#10010 <http://github.com/ceph/ceph/pull/10010>`_, Jason Dillaman)
+* librbd: Timeout sending mirroring notification shouldn't result in failure (`issue#16470 <http://tracker.ceph.com/issues/16470>`_, `pr#10052 <http://github.com/ceph/ceph/pull/10052>`_, Jason Dillaman)
+* librbd: Whitelist EBUSY error from snap unprotect for journal replay (`issue#16445 <http://tracker.ceph.com/issues/16445>`_, `pr#10055 <http://github.com/ceph/ceph/pull/10055>`_, Jason Dillaman)
+* librbd: cancel all tasks should wait until finisher is done (`issue#16517 <http://tracker.ceph.com/issues/16517>`_, `pr#9752 <http://github.com/ceph/ceph/pull/9752>`_, Haomai Wang)
+* librbd: delay acquiring lock if image watch has failed (`issue#16923 <http://tracker.ceph.com/issues/16923>`_, `pr#10827 <http://github.com/ceph/ceph/pull/10827>`_, Jason Dillaman)
+* librbd: fix missing return statement if failed to get mirror image state (`issue#16600 <http://tracker.ceph.com/issues/16600>`_, `pr#10144 <http://github.com/ceph/ceph/pull/10144>`_, runsisi)
+* librbd: flag image as updated after proxying maintenance op (`issue#16404 <http://tracker.ceph.com/issues/16404>`_, `pr#9883 <http://github.com/ceph/ceph/pull/9883>`_, Jason Dillaman)
+* librbd: mkfs.xfs slow performance with discards and object map (`issue#16707 <http://tracker.ceph.com/issues/16707>`_, `issue#16689 <http://tracker.ceph.com/issues/16689>`_, `pr#10649 <http://github.com/ceph/ceph/pull/10649>`_, Jason Dillaman)
+* librbd: potential use after free on refresh error (`issue#16519 <http://tracker.ceph.com/issues/16519>`_, `pr#9952 <http://github.com/ceph/ceph/pull/9952>`_, Mykola Golub)
+* librbd: rbd-nbd does not properly handle resize notifications (`issue#15715 <http://tracker.ceph.com/issues/15715>`_, `pr#10679 <http://github.com/ceph/ceph/pull/10679>`_, Mykola Golub)
+* librbd: the option 'rbd_cache_writethrough_until_flush=true' dosn't work (`issue#16740 <http://tracker.ceph.com/issues/16740>`_, `issue#16386 <http://tracker.ceph.com/issues/16386>`_, `issue#16708 <http://tracker.ceph.com/issues/16708>`_, `issue#16654 <http://tracker.ceph.com/issues/16654>`_, `issue#16478 <http://tracker.ceph.com/issues/16478>`_, `pr#10797 <http://github.com/ceph/ceph/pull/10797>`_, Mykola Golub, xinxin shu, Xiaowei Chen, Jason Dillaman)
+* mds:  tell command blocks forever with async messenger (TestVolumeClient.test_evict_client failure) (`issue#16288 <http://tracker.ceph.com/issues/16288>`_, `pr#10501 <http://github.com/ceph/ceph/pull/10501>`_, Douglas Fuller)
+* mds: Confusing MDS log message when shut down with stalled journaler reads (`issue#15689 <http://tracker.ceph.com/issues/15689>`_, `pr#9557 <http://github.com/ceph/ceph/pull/9557>`_, John Spray)
+* mds: Deadlock on shutdown active rank while busy with metadata IO (`issue#16042 <http://tracker.ceph.com/issues/16042>`_, `pr#10502 <http://github.com/ceph/ceph/pull/10502>`_, Patrick Donnelly)
+* mds: Failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7 (`issue#16013 <http://tracker.ceph.com/issues/16013>`_, `pr#10199 <http://github.com/ceph/ceph/pull/10199>`_, Yan, Zheng)
+* mds: Fix shutting down mds timed-out due to deadlock (`issue#16396 <http://tracker.ceph.com/issues/16396>`_, `pr#10500 <http://github.com/ceph/ceph/pull/10500>`_, Zhi Zhang)
+* mds: MDSMonitor fixes (`issue#16136 <http://tracker.ceph.com/issues/16136>`_, `pr#9561 <http://github.com/ceph/ceph/pull/9561>`_, xie xingguo)
+* mds: MDSMonitor::check_subs() is very buggy (`issue#16022 <http://tracker.ceph.com/issues/16022>`_, `pr#10103 <http://github.com/ceph/ceph/pull/10103>`_, Yan, Zheng)
+* mds: Session::check_access() is buggy (`issue#16358 <http://tracker.ceph.com/issues/16358>`_, `pr#10105 <http://github.com/ceph/ceph/pull/10105>`_, Yan, Zheng)
+* mds: StrayManager.cc: 520: FAILED assert(dnl->is_primary()) (`issue#15920 <http://tracker.ceph.com/issues/15920>`_, `pr#9559 <http://github.com/ceph/ceph/pull/9559>`_, Yan, Zheng)
+* mds: enforce a dirfrag limit on entries (`issue#16164 <http://tracker.ceph.com/issues/16164>`_, `pr#10104 <http://github.com/ceph/ceph/pull/10104>`_, Patrick Donnelly)
+* mds: fix SnapRealm::have_past_parents_open() (`issue#16299 <http://tracker.ceph.com/issues/16299>`_, `pr#10499 <http://github.com/ceph/ceph/pull/10499>`_, Yan, Zheng)
+* mds: fix getattr starve setattr (`issue#16154 <http://tracker.ceph.com/issues/16154>`_, `pr#9560 <http://github.com/ceph/ceph/pull/9560>`_, Yan, Zheng)
+* mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs (`issue#15702 <http://tracker.ceph.com/issues/15702>`_, `pr#9405 <http://github.com/ceph/ceph/pull/9405>`_, Zhi Zhang)
+* mon: "mon metadata" fails when only one monitor exists (`issue#15866 <http://tracker.ceph.com/issues/15866>`_, `pr#10654 <http://github.com/ceph/ceph/pull/10654>`_, John Spray, Kefu Chai)
+* mon: Monitor: validate prefix on handle_command() (`issue#16297 <http://tracker.ceph.com/issues/16297>`_, `pr#10036 <http://github.com/ceph/ceph/pull/10036>`_, You Ji)
+* mon: OSDMonitor: drop pg temps from not the current primary (`issue#16127 <http://tracker.ceph.com/issues/16127>`_, `pr#9998 <http://github.com/ceph/ceph/pull/9998>`_, Samuel Just)
+* mon: prepare_pgtemp needs to only update up_thru if newer than the existing one (`issue#16185 <http://tracker.ceph.com/issues/16185>`_, `pr#10001 <http://github.com/ceph/ceph/pull/10001>`_, Samuel Just)
+* msgr: AsyncConnection::lockmsg/async lockdep cycle: AsyncMessenger::lock, MDSDaemon::mds_lock, AsyncConnection::lock (`issue#16237 <http://tracker.ceph.com/issues/16237>`_, `pr#10004 <http://github.com/ceph/ceph/pull/10004>`_, Haomai Wang)
+* msgr: async messenger mon crash (`issue#16378 <http://tracker.ceph.com/issues/16378>`_, `issue#16418 <http://tracker.ceph.com/issues/16418>`_, `pr#9996 <http://github.com/ceph/ceph/pull/9996>`_, Haomai Wang)
+* msgr: backports of all asyncmsgr fixes to jewel (`issue#15503 <http://tracker.ceph.com/issues/15503>`_, `issue#15372 <http://tracker.ceph.com/issues/15372>`_, `pr#9633 <http://github.com/ceph/ceph/pull/9633>`_, Yan Jun, Haomai Wang, Piotr Dałek)
+* msgr: msg/async: connection race hang (`issue#15849 <http://tracker.ceph.com/issues/15849>`_, `pr#10003 <http://github.com/ceph/ceph/pull/10003>`_, Haomai Wang)
+* osd: FileStore: umount hang because sync thread doesn't exit (`issue#15695 <http://tracker.ceph.com/issues/15695>`_, `pr#9105 <http://github.com/ceph/ceph/pull/9105>`_, Kefu Chai)
+* osd: Fixes for list-inconsistent-* (`issue#15766 <http://tracker.ceph.com/issues/15766>`_, `issue#16192 <http://tracker.ceph.com/issues/16192>`_, `issue#15719 <http://tracker.ceph.com/issues/15719>`_, `pr#9565 <http://github.com/ceph/ceph/pull/9565>`_, David Zafman)
+* osd: New pools have bogus stuck inactive/unclean HEALTH_ERR messages until they are first active and clean (`issue#14952 <http://tracker.ceph.com/issues/14952>`_, `pr#10007 <http://github.com/ceph/ceph/pull/10007>`_, Sage Weil)
+* osd: OSD crash with Hammer to Jewel Upgrade: void FileStore::init_temp_collections() (`issue#16672 <http://tracker.ceph.com/issues/16672>`_, `pr#10561 <http://github.com/ceph/ceph/pull/10561>`_, David Zafman)
+* osd: OSD failed to subscribe skipped osdmaps after ceph osd pause (`issue#17023 <http://tracker.ceph.com/issues/17023>`_, `pr#10804 <http://github.com/ceph/ceph/pull/10804>`_, Kefu Chai)
+* osd: ObjectCacher split BufferHead read fix (`issue#16002 <http://tracker.ceph.com/issues/16002>`_, `pr#10074 <http://github.com/ceph/ceph/pull/10074>`_, Greg Farnum)
+* osd: ReplicatedBackend doesn't increment stats on pull, only push (`issue#16277 <http://tracker.ceph.com/issues/16277>`_, `pr#10421 <http://github.com/ceph/ceph/pull/10421>`_, Kefu Chai)
+* osd: Scrub error: 0/1 pinned (`issue#15952 <http://tracker.ceph.com/issues/15952>`_, `pr#9576 <http://github.com/ceph/ceph/pull/9576>`_, Samuel Just)
+* osd: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas (`issue#15943 <http://tracker.ceph.com/issues/15943>`_, `pr#9575 <http://github.com/ceph/ceph/pull/9575>`_, Samuel Just)
+* osd: partprobe intermittent issues during ceph-disk prepare (`issue#15176 <http://tracker.ceph.com/issues/15176>`_, `pr#10497 <http://github.com/ceph/ceph/pull/10497>`_, Marius Vollmer, Loic Dachary)
+* osd: saw valgrind issues in ReplicatedPG::new_repop (`issue#16801 <http://tracker.ceph.com/issues/16801>`_, `pr#10760 <http://github.com/ceph/ceph/pull/10760>`_, Kefu Chai)
+* osd: sparse_read on ec pool should return extends with correct offset (`issue#16138 <http://tracker.ceph.com/issues/16138>`_, `pr#10006 <http://github.com/ceph/ceph/pull/10006>`_, kofiliu)
+* osd:sched_time not actually randomized (`issue#15890 <http://tracker.ceph.com/issues/15890>`_, `pr#9578 <http://github.com/ceph/ceph/pull/9578>`_, xie xingguo)
+* rbd: ImageReplayer::is_replaying does not include flush state  (`issue#16970 <http://tracker.ceph.com/issues/16970>`_, `pr#10790 <http://github.com/ceph/ceph/pull/10790>`_, Jason Dillaman)
+* rbd: Journal duplicate op detection can cause lockdep error (`issue#16363 <http://tracker.ceph.com/issues/16363>`_, `pr#10044 <http://github.com/ceph/ceph/pull/10044>`_, Jason Dillaman)
+* rbd: Journal needs to handle duplicate maintenance op tids (`issue#16362 <http://tracker.ceph.com/issues/16362>`_, `pr#10045 <http://github.com/ceph/ceph/pull/10045>`_, Jason Dillaman)
+* rbd: Unable to disable journaling feature if in unexpected mirror state (`issue#16348 <http://tracker.ceph.com/issues/16348>`_, `pr#10042 <http://github.com/ceph/ceph/pull/10042>`_, Jason Dillaman)
+* rbd: bashism in src/rbdmap  (`issue#16608 <http://tracker.ceph.com/issues/16608>`_, `pr#10786 <http://github.com/ceph/ceph/pull/10786>`_, Jason Dillaman)
+* rbd: doc: format 2 now is the default image format (`issue#17026 <http://tracker.ceph.com/issues/17026>`_, `pr#10732 <http://github.com/ceph/ceph/pull/10732>`_, Chengwei Yang)
+* rbd: hen journaling is enabled, a flush request shouldn't flush the cache (`issue#15761 <http://tracker.ceph.com/issues/15761>`_, `pr#10041 <http://github.com/ceph/ceph/pull/10041>`_, Yuan Zhou)
+* rbd: possible race condition during journal transition from replay to ready (`issue#16198 <http://tracker.ceph.com/issues/16198>`_, `pr#10047 <http://github.com/ceph/ceph/pull/10047>`_, Jason Dillaman)
+* rbd: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable (`issue#16289 <http://tracker.ceph.com/issues/16289>`_, `pr#9721 <http://github.com/ceph/ceph/pull/9721>`_, Mykola Golub)
+* rbd: rbd-mirror should disable proxied maintenance ops for non-primary image (`issue#16411 <http://tracker.ceph.com/issues/16411>`_, `pr#10050 <http://github.com/ceph/ceph/pull/10050>`_, Jason Dillaman)
+* rbd: rbd-mirror: FAILED assert(m_local_image_ctx->object_map != nullptr) (`issue#16558 <http://tracker.ceph.com/issues/16558>`_, `pr#10646 <http://github.com/ceph/ceph/pull/10646>`_, Jason Dillaman)
+* rbd: rbd-mirror: FAILED assert(m_on_update_status_finish == nullptr) (`issue#16956 <http://tracker.ceph.com/issues/16956>`_, `pr#10792 <http://github.com/ceph/ceph/pull/10792>`_, Jason Dillaman)
+* rbd: rbd-mirror: FAILED assert(m_state == STATE_STOPPING) (`issue#16980 <http://tracker.ceph.com/issues/16980>`_, `pr#10791 <http://github.com/ceph/ceph/pull/10791>`_, Jason Dillaman)
+* rbd: rbd-mirror: ensure replay status formatter has completed before stopping replay (`issue#16352 <http://tracker.ceph.com/issues/16352>`_, `pr#10043 <http://github.com/ceph/ceph/pull/10043>`_, Jason Dillaman)
+* rbd: rbd-mirror: include local pool id in resync throttle unique key (`issue#16536 <http://tracker.ceph.com/issues/16536>`_, `issue#15239 <http://tracker.ceph.com/issues/15239>`_, `issue#16488 <http://tracker.ceph.com/issues/16488>`_, `issue#16491 <http://tracker.ceph.com/issues/16491>`_, `issue#16329 <http://tracker.ceph.com/issues/16329>`_, `issue#15108 <http://tracker.ceph.com/issues/15108>`_, `issue#15670 <http://tracker.ceph.com/issues/15670>`_, `pr#10678 <http://github.com/ceph/ceph/pull/10678>`_, Ricardo Dias, Jason Dillaman)
+* rbd: rbd-mirror: potential race condition accessing local image journal  (`issue#16230 <http://tracker.ceph.com/issues/16230>`_, `pr#10046 <http://github.com/ceph/ceph/pull/10046>`_, Jason Dillaman)
+* rbd: rbd-mirror: reduce memory footprint during journal replay (`issue#16321 <http://tracker.ceph.com/issues/16321>`_, `issue#16489 <http://tracker.ceph.com/issues/16489>`_, `issue#16622 <http://tracker.ceph.com/issues/16622>`_, `issue#16539 <http://tracker.ceph.com/issues/16539>`_, `issue#16223 <http://tracker.ceph.com/issues/16223>`_, `issue#16349 <http://tracker.ceph.com/issues/16349>`_, `pr#10684 <http://github.com/ceph/ceph/pull/10684>`_, Mykola Golub, Jason Dillaman)
+* rgw: A query on a static large object fails with 404 error (`issue#16015 <http://tracker.ceph.com/issues/16015>`_, `pr#9544 <http://github.com/ceph/ceph/pull/9544>`_, Radoslaw Zarzynski)
+* rgw: Add zone rename to radosgw_admin (`issue#16934 <http://tracker.ceph.com/issues/16934>`_, `pr#10663 <http://github.com/ceph/ceph/pull/10663>`_, Shilpa Jagannath)
+* rgw: Bucket index shards orphaned after bucket delete (`issue#16412 <http://tracker.ceph.com/issues/16412>`_, `pr#10525 <http://github.com/ceph/ceph/pull/10525>`_, Orit Wasserman)
+* rgw: Bug when using port 443s in rgw. (`issue#16548 <http://tracker.ceph.com/issues/16548>`_, `pr#10664 <http://github.com/ceph/ceph/pull/10664>`_, Pritha Srivastava)
+* rgw: Fallback to Host header for bucket name. (`issue#15975 <http://tracker.ceph.com/issues/15975>`_, `pr#10693 <http://github.com/ceph/ceph/pull/10693>`_, Robin H. Johnson)
+* rgw: Fix civetweb IPv6 (`issue#16928 <http://tracker.ceph.com/issues/16928>`_, `pr#10580 <http://github.com/ceph/ceph/pull/10580>`_, Robin H. Johnson)
+* rgw: Increase log level for messages occuring while running rgw admin command (`issue#16935 <http://tracker.ceph.com/issues/16935>`_, `pr#10765 <http://github.com/ceph/ceph/pull/10765>`_, Shilpa Jagannath)
+* rgw: No Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest (`issue#15812 <http://tracker.ceph.com/issues/15812>`_, `pr#9265 <http://github.com/ceph/ceph/pull/9265>`_, Radoslaw Zarzynski)
+* rgw: RGWPeriodPuller tries to pull from itself (`issue#16939 <http://tracker.ceph.com/issues/16939>`_, `pr#10764 <http://github.com/ceph/ceph/pull/10764>`_, Casey Bodley)
+* rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule (`issue#15348 <http://tracker.ceph.com/issues/15348>`_, `pr#9453 <http://github.com/ceph/ceph/pull/9453>`_, Wido den Hollander)
+* rgw: Swift API returns double space usage and objects of account metadata (`issue#16188 <http://tracker.ceph.com/issues/16188>`_, `pr#10148 <http://github.com/ceph/ceph/pull/10148>`_, Albert Tu)
+* rgw: account/container metadata not actually present in a request are deleted during POST through Swift API (`issue#15977 <http://tracker.ceph.com/issues/15977>`_, `issue#15779 <http://tracker.ceph.com/issues/15779>`_, `pr#9542 <http://github.com/ceph/ceph/pull/9542>`_, Radoslaw Zarzynski)
+* rgw: add socket backlog setting for via ceph.conf (`issue#16406 <http://tracker.ceph.com/issues/16406>`_, `pr#10216 <http://github.com/ceph/ceph/pull/10216>`_, Feng Guo)
+* rgw: add tenant support to multisite sync (`issue#16469 <http://tracker.ceph.com/issues/16469>`_, `issue#16121 <http://tracker.ceph.com/issues/16121>`_, `issue#16665 <http://tracker.ceph.com/issues/16665>`_, `pr#10845 <http://github.com/ceph/ceph/pull/10845>`_, Yehuda Sadeh, Josh Durgin, Casey Bodley, Pritha Srivastava)
+* rgw: add_zone only clears master_zone if --master=false (`issue#15901 <http://tracker.ceph.com/issues/15901>`_, `pr#9327 <http://github.com/ceph/ceph/pull/9327>`_, Casey Bodley)
+* rgw: aws4 parsing issue (`issue#15940 <http://tracker.ceph.com/issues/15940>`_, `issue#15939 <http://tracker.ceph.com/issues/15939>`_, `pr#9545 <http://github.com/ceph/ceph/pull/9545>`_, Yehuda Sadeh)
+* rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support (`issue#16146 <http://tracker.ceph.com/issues/16146>`_, `pr#10167 <http://github.com/ceph/ceph/pull/10167>`_, Radoslaw Zarzynski, Javier M. Mellid)
+* rgw: backport merge of static sites fixes (`issue#15555 <http://tracker.ceph.com/issues/15555>`_, `issue#15532 <http://tracker.ceph.com/issues/15532>`_, `issue#15531 <http://tracker.ceph.com/issues/15531>`_, `pr#9568 <http://github.com/ceph/ceph/pull/9568>`_, Robin H. Johnson)
+* rgw: can set negative max_buckets on RGWUserInfo (`issue#14534 <http://tracker.ceph.com/issues/14534>`_, `pr#10655 <http://github.com/ceph/ceph/pull/10655>`_, Yehuda Sadeh)
+* rgw: cleanup radosgw-admin temp command as it was deprecated (`issue#16023 <http://tracker.ceph.com/issues/16023>`_, `pr#9390 <http://github.com/ceph/ceph/pull/9390>`_, Vikhyat Umrao)
+* rgw: comparing return code to ERR_NOT_MODIFIED in rgw_rest_s3.cc (needs minus sign) (`issue#16327 <http://tracker.ceph.com/issues/16327>`_, `pr#9790 <http://github.com/ceph/ceph/pull/9790>`_, Nathan Cutler)
+* rgw: custom metadata aren't camelcased in Swift's responses (`issue#15902 <http://tracker.ceph.com/issues/15902>`_, `pr#9267 <http://github.com/ceph/ceph/pull/9267>`_, Radoslaw Zarzynski)
+* rgw: data sync stops after getting error in all data log sync shards (`issue#16530 <http://tracker.ceph.com/issues/16530>`_, `pr#10073 <http://github.com/ceph/ceph/pull/10073>`_, Yehuda Sadeh)
+* rgw: default zone and zonegroup cannot be added to a realm (`issue#16839 <http://tracker.ceph.com/issues/16839>`_, `pr#10658 <http://github.com/ceph/ceph/pull/10658>`_, Casey Bodley)
+* rgw: document multi tenancy (`issue#16635 <http://tracker.ceph.com/issues/16635>`_, `pr#10217 <http://github.com/ceph/ceph/pull/10217>`_, Pete Zaitcev)
+* rgw: don't unregister request if request is not connected to manager (`issue#15911 <http://tracker.ceph.com/issues/15911>`_, `pr#9242 <http://github.com/ceph/ceph/pull/9242>`_, Yehuda Sadeh)
+* rgw: failed to create bucket after upgrade from hammer to jewel (`issue#16627 <http://tracker.ceph.com/issues/16627>`_, `pr#10524 <http://github.com/ceph/ceph/pull/10524>`_, Orit Wasserman)
+* rgw: fix ldap bindpw parsing (`issue#16286 <http://tracker.ceph.com/issues/16286>`_, `pr#10518 <http://github.com/ceph/ceph/pull/10518>`_, Matt Benjamin)
+* rgw: fix multi-delete query param parsing. (`issue#16618 <http://tracker.ceph.com/issues/16618>`_, `pr#10188 <http://github.com/ceph/ceph/pull/10188>`_, Robin H. Johnson)
+* rgw: improve support for Swift's object versioning. (`issue#15925 <http://tracker.ceph.com/issues/15925>`_, `pr#10710 <http://github.com/ceph/ceph/pull/10710>`_, Radoslaw Zarzynski)
+* rgw: initial slashes are not properly handled in Swift's BulkDelete (`issue#15948 <http://tracker.ceph.com/issues/15948>`_, `pr#9316 <http://github.com/ceph/ceph/pull/9316>`_, Radoslaw Zarzynski)
+* rgw: master: build failures with boost > 1.58 (`issue#16392 <http://tracker.ceph.com/issues/16392>`_, `issue#16391 <http://tracker.ceph.com/issues/16391>`_, `pr#10026 <http://github.com/ceph/ceph/pull/10026>`_, Abhishek Lekshmanan)
+* rgw: multisite segfault on ~RGWRealmWatcher if realm was deleted (`issue#16817 <http://tracker.ceph.com/issues/16817>`_, `pr#10660 <http://github.com/ceph/ceph/pull/10660>`_, Casey Bodley)
+* rgw: multisite sync races with deletes (`issue#16222 <http://tracker.ceph.com/issues/16222>`_, `issue#16464 <http://tracker.ceph.com/issues/16464>`_, `issue#16220 <http://tracker.ceph.com/issues/16220>`_, `issue#16143 <http://tracker.ceph.com/issues/16143>`_, `pr#10293 <http://github.com/ceph/ceph/pull/10293>`_, Yehuda Sadeh, Casey Bodley)
+* rgw: multisite: preserve zone's extra pool (`issue#16712 <http://tracker.ceph.com/issues/16712>`_, `pr#10537 <http://github.com/ceph/ceph/pull/10537>`_, Abhishek Lekshmanan)
+* rgw: object expirer's hints might be trimmed without processing in some circumstances (`issue#16705 <http://tracker.ceph.com/issues/16705>`_, `issue#16684 <http://tracker.ceph.com/issues/16684>`_, `pr#10763 <http://github.com/ceph/ceph/pull/10763>`_, Radoslaw Zarzynski)
+* rgw: radosgw-admin failure for user create after upgrade from hammer to jewel (`issue#15937 <http://tracker.ceph.com/issues/15937>`_, `pr#9294 <http://github.com/ceph/ceph/pull/9294>`_, Orit Wasserman, Abhishek Lekshmanan)
+* rgw: radosgw-admin: EEXIST messages for create operations (`issue#15720 <http://tracker.ceph.com/issues/15720>`_, `pr#9268 <http://github.com/ceph/ceph/pull/9268>`_, Abhishek Lekshmanan)
+* rgw: radosgw-admin: inconsistency in uid/email handling (`issue#13598 <http://tracker.ceph.com/issues/13598>`_, `pr#10520 <http://github.com/ceph/ceph/pull/10520>`_, Matt Benjamin)
+* rgw: realm pull fails when using apache frontend (`issue#15846 <http://tracker.ceph.com/issues/15846>`_, `pr#9266 <http://github.com/ceph/ceph/pull/9266>`_, Orit Wasserman)
+* rgw: retry on bucket sync errors (`issue#16108 <http://tracker.ceph.com/issues/16108>`_, `pr#9425 <http://github.com/ceph/ceph/pull/9425>`_, Yehuda Sadeh)
+* rgw: s3website: x-amz-website-redirect-location header returns malformed HTTP response (`issue#15531 <http://tracker.ceph.com/issues/15531>`_, `pr#9099 <http://github.com/ceph/ceph/pull/9099>`_, Robin H. Johnson)
+* rgw: segfault in RGWOp_MDLog_Notify (`issue#16666 <http://tracker.ceph.com/issues/16666>`_, `pr#10662 <http://github.com/ceph/ceph/pull/10662>`_, Casey Bodley)
+* rgw: segmentation fault on error_repo in data sync (`issue#16603 <http://tracker.ceph.com/issues/16603>`_, `pr#10523 <http://github.com/ceph/ceph/pull/10523>`_, Casey Bodley)
+* rgw: selinux denials in RGW (`issue#16126 <http://tracker.ceph.com/issues/16126>`_, `pr#10519 <http://github.com/ceph/ceph/pull/10519>`_, Boris Ranto)
+* rgw: support size suffixes for --max-size in radosgw-admin command (`issue#16004 <http://tracker.ceph.com/issues/16004>`_, `pr#9743 <http://github.com/ceph/ceph/pull/9743>`_, Vikhyat Umrao)
+* rgw: updating CORS/ACLs might not work in some circumstances (`issue#15976 <http://tracker.ceph.com/issues/15976>`_, `pr#9543 <http://github.com/ceph/ceph/pull/9543>`_, Radoslaw Zarzynski)
+* rgw: use zone endpoints instead of zonegroup endpoints (`issue#16834 <http://tracker.ceph.com/issues/16834>`_, `pr#10659 <http://github.com/ceph/ceph/pull/10659>`_, Casey Bodley)
+* tests: improve rbd-mirror test case coverage (`issue#16197 <http://tracker.ceph.com/issues/16197>`_, `pr#9631 <http://github.com/ceph/ceph/pull/9631>`_, Mykola Golub, Jason Dillaman)
+* tests: rados/test.sh workunit timesout on OpenStack (`issue#15403 <http://tracker.ceph.com/issues/15403>`_, `pr#8904 <http://github.com/ceph/ceph/pull/8904>`_, Loic Dachary)
+* tools: ceph-disk: Accept bcache devices as data disks (`issue#13278 <http://tracker.ceph.com/issues/13278>`_, `pr#8497 <http://github.com/ceph/ceph/pull/8497>`_, Peter Sabaini)
+* tools: rados: Add cleanup message with time to rados bench output (`issue#15704 <http://tracker.ceph.com/issues/15704>`_, `pr#9740 <http://github.com/ceph/ceph/pull/9740>`_, Vikhyat Umrao)
+* tools: src/script/subman fails with KeyError: 'nband' (`issue#16961 <http://tracker.ceph.com/issues/16961>`_, `pr#10625 <http://github.com/ceph/ceph/pull/10625>`_, Loic Dachary, Ali Maredia)
+
+
+v10.2.2 Jewel
+=============
+
+This point release fixes several important bugs in RBD mirroring, RGW multi-site, CephFS, and RADOS.
+
+We recommend that all v10.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.2.txt>`.
+
+Notable Changes
+---------------
+
+* ceph: cli: exception when pool name has non-ascii characters (`issue#15913 <http://tracker.ceph.com/issues/15913>`_, `pr#9320 <http://github.com/ceph/ceph/pull/9320>`_, Ricardo Dias)
+* ceph-disk: workaround gperftool hang (`issue#13522 <http://tracker.ceph.com/issues/13522>`_, `issue#16103 <http://tracker.ceph.com/issues/16103>`_, `pr#9427 <http://github.com/ceph/ceph/pull/9427>`_, Loic Dachary)
+* cephfs: backports needed for Manila (`issue#15599 <http://tracker.ceph.com/issues/15599>`_, `issue#15417 <http://tracker.ceph.com/issues/15417>`_, `issue#15045 <http://tracker.ceph.com/issues/15045>`_, `pr#9430 <http://github.com/ceph/ceph/pull/9430>`_, John Spray, Ramana Raja, Xiaoxi Chen)
+* ceph.spec.in: drop support for RHEL<7 and SUSE<1210 in jewel and above (`issue#15725 <http://tracker.ceph.com/issues/15725>`_, `issue#15627 <http://tracker.ceph.com/issues/15627>`_, `issue#13445 <http://tracker.ceph.com/issues/13445>`_, `issue#15822 <http://tracker.ceph.com/issues/15822>`_, `issue#15472 <http://tracker.ceph.com/issues/15472>`_, `issue#15987 <http://tracker.ceph.com/issues/15987>`_, `issue#15516 <http://tracker.ceph.com/issues/15516>`_, `issue#15549 <http://tracker.ceph.com/issues/15549>`_, `pr#8938 <http://github.com/ceph/ceph/pull/8938>`_, Boris Ranto, Sage Weil, Nathan Cutler, Lars Marowsky-Bree)
+* ceph_test_librbd_fsx crashes during journal replay shut down (`issue#16123 <http://tracker.ceph.com/issues/16123>`_, `pr#9556 <http://github.com/ceph/ceph/pull/9556>`_, Jason Dillaman)
+* client: fix bugs accidentally disabling readahead (`issue#16024 <http://tracker.ceph.com/issues/16024>`_, `pr#9656 <http://github.com/ceph/ceph/pull/9656>`_, Patrick Donnelly, Greg Farnum)
+* cls_journal: initialize empty commit position upon client register (`issue#15757 <http://tracker.ceph.com/issues/15757>`_, `pr#9376 <http://github.com/ceph/ceph/pull/9376>`_, runsisi, Venky Shankar)
+* cls::rbd: mirror_image_status_list returned max 64 items (`pr#9069 <http://github.com/ceph/ceph/pull/9069>`_, Mykola Golub)
+* cls_rbd: mirror image status summary should read full directory (`issue#16178 <http://tracker.ceph.com/issues/16178>`_, `pr#9608 <http://github.com/ceph/ceph/pull/9608>`_, Jason Dillaman)
+* common: BackoffThrottle spins unnecessarily with very small backoff while the throttle is full (`issue#15953 <http://tracker.ceph.com/issues/15953>`_, `pr#9579 <http://github.com/ceph/ceph/pull/9579>`_, Samuel Just)
+* common: Do not link lttng into libglobal (`pr#9194 <http://github.com/ceph/ceph/pull/9194>`_, Karol Mroz)
+* debian: install systemd target files (`issue#15573 <http://tracker.ceph.com/issues/15573>`_, `pr#8815 <http://github.com/ceph/ceph/pull/8815>`_, Kefu Chai, Sage Weil)
+* doc: update mirroring guide to include pool/image status commands (`issue#15746 <http://tracker.ceph.com/issues/15746>`_, `pr#9180 <http://github.com/ceph/ceph/pull/9180>`_, Mykola Golub)
+* librbd: Disabling journaling feature results in "Transport endpoint is not connected" error (`issue#15863 <http://tracker.ceph.com/issues/15863>`_, `pr#9548 <http://github.com/ceph/ceph/pull/9548>`_, Yuan Zhou)
+* librbd: do not shut down exclusive lock while acquiring' (`issue#16291 <http://tracker.ceph.com/issues/16291>`_, `issue#16260 <http://tracker.ceph.com/issues/16260>`_, `pr#9691 <http://github.com/ceph/ceph/pull/9691>`_, Jason Dillaman)
+* librbd: Initial python APIs to support mirroring (`issue#15656 <http://tracker.ceph.com/issues/15656>`_, `pr#9550 <http://github.com/ceph/ceph/pull/9550>`_, Mykola Golub)
+* librbd: journal IO error results in failed assertion in AioCompletion (`issue#16077 <http://tracker.ceph.com/issues/16077>`_, `issue#15034 <http://tracker.ceph.com/issues/15034>`_, `issue#15791 <http://tracker.ceph.com/issues/15791>`_, `pr#9611 <http://github.com/ceph/ceph/pull/9611>`_, Hector Martin, Jason Dillaman)
+* librbd: journal: live replay might skip entries from previous object set (`issue#15864 <http://tracker.ceph.com/issues/15864>`_, `issue#15665 <http://tracker.ceph.com/issues/15665>`_, `pr#9217 <http://github.com/ceph/ceph/pull/9217>`_, Jason Dillaman)
+* librbd: journal: support asynchronous shutdown (`issue#15949 <http://tracker.ceph.com/issues/15949>`_, `issue#14530 <http://tracker.ceph.com/issues/14530>`_, `issue#15993 <http://tracker.ceph.com/issues/15993>`_, `pr#9373 <http://github.com/ceph/ceph/pull/9373>`_, Jason Dillaman)
+* librbd: Metadata config overrides are applied synchronously (`issue#15928 <http://tracker.ceph.com/issues/15928>`_, `pr#9318 <http://github.com/ceph/ceph/pull/9318>`_, Jason Dillaman)
+* librbd: Object Map is showing as invalid, even when Object Map is disabled for that Image. (`issue#16076 <http://tracker.ceph.com/issues/16076>`_, `pr#9555 <http://github.com/ceph/ceph/pull/9555>`_, xinxin shu)
+* librbd: prevent error messages when journal externally disabled (`issue#16114 <http://tracker.ceph.com/issues/16114>`_, `pr#9610 <http://github.com/ceph/ceph/pull/9610>`_, Zhiqiang Wang, Jason Dillaman)
+* librbd: recursive lock possible when disabling journaling (`issue#16235 <http://tracker.ceph.com/issues/16235>`_, `pr#9654 <http://github.com/ceph/ceph/pull/9654>`_, Jason Dillaman)
+* librbd: refresh image if needed in mirror functions (`issue#16096 <http://tracker.ceph.com/issues/16096>`_, `pr#9609 <http://github.com/ceph/ceph/pull/9609>`_, Jon Bernard)
+* librbd: remove should ignore mirror errors from older OSDs (`issue#16268 <http://tracker.ceph.com/issues/16268>`_, `pr#9692 <http://github.com/ceph/ceph/pull/9692>`_, Jason Dillaman)
+* librbd: reuse ImageCtx::finisher and SafeTimer for lots of images case (`issue#13938 <http://tracker.ceph.com/issues/13938>`_, `pr#9580 <http://github.com/ceph/ceph/pull/9580>`_, Haomai Wang)
+* librbd: validate image metadata configuration overrides (`issue#15522 <http://tracker.ceph.com/issues/15522>`_, `pr#9554 <http://github.com/ceph/ceph/pull/9554>`_, zhuangzeqiang)
+* mds: order directories by hash and fix simultaneous readdir races (`issue#15508 <http://tracker.ceph.com/issues/15508>`_, `pr#9655 <http://github.com/ceph/ceph/pull/9655>`_, Yan, Zheng, Greg Farnum)
+* mon: Hammer (0.94.3) OSD does not delete old OSD Maps in a timely fashion (maybe at all?) (`issue#13990 <http://tracker.ceph.com/issues/13990>`_, `pr#9100 <http://github.com/ceph/ceph/pull/9100>`_, Kefu Chai)
+* mon/Monitor: memory leak on Monitor::handle_ping() (`issue#15793 <http://tracker.ceph.com/issues/15793>`_, `pr#9270 <http://github.com/ceph/ceph/pull/9270>`_, xie xingguo)
+* osd: acting_primary not updated on split (`issue#15523 <http://tracker.ceph.com/issues/15523>`_, `pr#8968 <http://github.com/ceph/ceph/pull/8968>`_, Sage Weil)
+* osd: boot race with noup being set (`issue#15678 <http://tracker.ceph.com/issues/15678>`_, `pr#9101 <http://github.com/ceph/ceph/pull/9101>`_, Sage Weil)
+* osd: deadlock in OSD::_committed_osd_maps (`issue#15701 <http://tracker.ceph.com/issues/15701>`_, `pr#9103 <http://github.com/ceph/ceph/pull/9103>`_, Xinze Chi)
+* osd: hobject_t::get_max() vs is_max() discrepancy (`issue#16113 <http://tracker.ceph.com/issues/16113>`_, `pr#9614 <http://github.com/ceph/ceph/pull/9614>`_, Samuel Just)
+* osd: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv (`issue#15760 <http://tracker.ceph.com/issues/15760>`_, `pr#9104 <http://github.com/ceph/ceph/pull/9104>`_, Sage Weil)
+* osd: remove reliance on FLAG_OMAP for reads (`pr#9638 <http://github.com/ceph/ceph/pull/9638>`_, Samuel Just)
+* osd valgrind invalid reads/writes (`issue#15870 <http://tracker.ceph.com/issues/15870>`_, `pr#9237 <http://github.com/ceph/ceph/pull/9237>`_, Samuel Just)
+* pybind: rbd API should default features parameter to None (`issue#15982 <http://tracker.ceph.com/issues/15982>`_, `pr#9553 <http://github.com/ceph/ceph/pull/9553>`_, Mykola Golub)
+* qa: dynamic_features.sh races with image deletion (`issue#15500 <http://tracker.ceph.com/issues/15500>`_, `pr#9552 <http://github.com/ceph/ceph/pull/9552>`_, Mykola Golub)
+* qa/workunits: ensure replay has started before checking position (`issue#16248 <http://tracker.ceph.com/issues/16248>`_, `pr#9674 <http://github.com/ceph/ceph/pull/9674>`_, Jason Dillaman)
+* qa/workunits/rbd: fixed rbd_mirror teuthology runtime errors (`pr#9232 <http://github.com/ceph/ceph/pull/9232>`_, Jason Dillaman)
+* radosgw-admin: fix 'period push' handling of --url (`issue#15926 <http://tracker.ceph.com/issues/15926>`_, `pr#9210 <http://github.com/ceph/ceph/pull/9210>`_, Casey Bodley)
+* rbd-mirror: Delete local image mirror when remote image mirroring is disabled (`issue#15916 <http://tracker.ceph.com/issues/15916>`_, `issue#14421 <http://tracker.ceph.com/issues/14421>`_, `pr#9372 <http://github.com/ceph/ceph/pull/9372>`_, runsisi, Mykola Golub, Ricardo Dias)
+* rbd-mirror: do not propagate deletions when pool unavailable (`issue#16229 <http://tracker.ceph.com/issues/16229>`_, `pr#9630 <http://github.com/ceph/ceph/pull/9630>`_, Jason Dillaman)
+* rbd-mirror: do not re-use image id from mirror directory if creating image (`issue#16253 <http://tracker.ceph.com/issues/16253>`_, `pr#9673 <http://github.com/ceph/ceph/pull/9673>`_, Jason Dillaman)
+* rbd-mirror: FAILED assert(!m_status_watcher) (`issue#16245 <http://tracker.ceph.com/issues/16245>`_, `issue#16290 <http://tracker.ceph.com/issues/16290>`_, `pr#9690 <http://github.com/ceph/ceph/pull/9690>`_, Mykola Golub)
+* rbd-mirror: fix deletion propagation edge cases (`issue#16226 <http://tracker.ceph.com/issues/16226>`_, `pr#9629 <http://github.com/ceph/ceph/pull/9629>`_, Jason Dillaman)
+* rbd-mirror: fix journal shut down ordering (`issue#16165 <http://tracker.ceph.com/issues/16165>`_, `pr#9628 <http://github.com/ceph/ceph/pull/9628>`_, Jason Dillaman)
+* rbd-mirror: potential crash during image status update (`issue#15909 <http://tracker.ceph.com/issues/15909>`_, `pr#9226 <http://github.com/ceph/ceph/pull/9226>`_, Mykola Golub, Jason Dillaman)
+* rbd-mirror: refresh image after creating sync point (`issue#16196 <http://tracker.ceph.com/issues/16196>`_, `pr#9627 <http://github.com/ceph/ceph/pull/9627>`_, Jason Dillaman)
+* rbd-mirror: replicate cloned images (`issue#14937 <http://tracker.ceph.com/issues/14937>`_, `pr#9423 <http://github.com/ceph/ceph/pull/9423>`_, Jason Dillaman)
+* rbd-mirror should disable the rbd cache for local images (`issue#15930 <http://tracker.ceph.com/issues/15930>`_, `pr#9317 <http://github.com/ceph/ceph/pull/9317>`_, Jason Dillaman)
+* rbd-mirror: support bootstrap canceling (`issue#16201 <http://tracker.ceph.com/issues/16201>`_, `pr#9612 <http://github.com/ceph/ceph/pull/9612>`_, Mykola Golub)
+* rbd-mirror: support multiple replicated pools (`issue#16045 <http://tracker.ceph.com/issues/16045>`_, `pr#9409 <http://github.com/ceph/ceph/pull/9409>`_, Jason Dillaman)
+* rgw: fix manager selection when APIs customized (`issue#15974 <http://tracker.ceph.com/issues/15974>`_, `issue#15973 <http://tracker.ceph.com/issues/15973>`_, `pr#9245 <http://github.com/ceph/ceph/pull/9245>`_, Robin H. Johnson)
+* rgw: keep track of written_objs correctly (`issue#15886 <http://tracker.ceph.com/issues/15886>`_, `pr#9239 <http://github.com/ceph/ceph/pull/9239>`_, Yehuda Sadeh)
+* rpm: ceph gid mismatch on upgrade from hammer with pre-existing ceph user (SUSE) (`issue#15869 <http://tracker.ceph.com/issues/15869>`_, `pr#9424 <http://github.com/ceph/ceph/pull/9424>`_, Nathan Cutler)
+* systemd: ceph-{mds,mon,osd,radosgw} systemd unit files need wants=time-sync.target (`issue#15419 <http://tracker.ceph.com/issues/15419>`_, `pr#8802 <http://github.com/ceph/ceph/pull/8802>`_, Nathan Cutler)
+* test: failure in journal.sh workunit test (`issue#16011 <http://tracker.ceph.com/issues/16011>`_, `pr#9377 <http://github.com/ceph/ceph/pull/9377>`_, Mykola Golub)
+* tests: rm -fr /tmp/*virtualenv* (`issue#16087 <http://tracker.ceph.com/issues/16087>`_, `pr#9403 <http://github.com/ceph/ceph/pull/9403>`_, Loic Dachary)
+
+v10.2.1 Jewel
+=============
+
+This is the first bugfix release for Jewel.  It contains several annoying
+packaging and init system fixes and a range of important bugfixes across
+RBD, RGW, and CephFS.
+
+We recommend that all v10.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v10.2.1.txt>`.
+
+Notable Changes
+---------------
+
+* cephfs: CephFSVolumeClient should isolate volumes by RADOS namespace (`issue#15400 <http://tracker.ceph.com/issues/15400>`_, `pr#8787 <http://github.com/ceph/ceph/pull/8787>`_, Xiaoxi Chen)
+* cephfs: handle standby-replay nodes properly in upgrades (`issue#15591 <http://tracker.ceph.com/issues/15591>`_, `pr#8971 <http://github.com/ceph/ceph/pull/8971>`_, John Spray)
+* ceph-{mds,mon,osd} packages need scriptlets with systemd code (`issue#14941 <http://tracker.ceph.com/issues/14941>`_, `pr#8801 <http://github.com/ceph/ceph/pull/8801>`_, Boris Ranto, Nathan Cutler)
+* ceph_test_keyvaluedb: fix (`issue#15435 <http://tracker.ceph.com/issues/15435>`_, `pr#9051 <http://github.com/ceph/ceph/pull/9051>`_, Allen Samuels, Sage Weil)
+* cmake: add missing source file to rbd_mirror/image_replayer (`pr#9052 <http://github.com/ceph/ceph/pull/9052>`_, Casey Bodley)
+* cmake: fix rbd compile errors (`pr#9076 <http://github.com/ceph/ceph/pull/9076>`_, runsisi, Jason Dillaman)
+* journal: incorrectly computed object offset within set (`issue#15765 <http://tracker.ceph.com/issues/15765>`_, `pr#9038 <http://github.com/ceph/ceph/pull/9038>`_, Jason Dillaman)
+* librbd: client-side handling for incompatible object map sizes (`issue#15642 <http://tracker.ceph.com/issues/15642>`_, `pr#9039 <http://github.com/ceph/ceph/pull/9039>`_, Jason Dillaman)
+* librbd: constrain size of AioWriteEvent journal entries (`issue#15750 <http://tracker.ceph.com/issues/15750>`_, `pr#9048 <http://github.com/ceph/ceph/pull/9048>`_, Jason Dillaman)
+* librbd: does not crash if image header is too short (`pr#9044 <http://github.com/ceph/ceph/pull/9044>`_, Kefu Chai)
+* librbd: Errors encountered disabling object-map while flatten is in-progress (`issue#15572 <http://tracker.ceph.com/issues/15572>`_, `pr#8869 <http://github.com/ceph/ceph/pull/8869>`_, Jason Dillaman)
+* librbd: fix get/list mirror image status API (`issue#15771 <http://tracker.ceph.com/issues/15771>`_, `pr#9036 <http://github.com/ceph/ceph/pull/9036>`_, Mykola Golub)
+* librbd: Parent image is closed twice if error encountered while opening (`issue#15574 <http://tracker.ceph.com/issues/15574>`_, `pr#8867 <http://github.com/ceph/ceph/pull/8867>`_, Jason Dillaman)
+* librbd: possible double-free of object map invalidation request upon error (`issue#15643 <http://tracker.ceph.com/issues/15643>`_, `pr#8865 <http://github.com/ceph/ceph/pull/8865>`_, runsisi)
+* librbd: possible race condition leads to use-after-free (`issue#15690 <http://tracker.ceph.com/issues/15690>`_, `pr#9009 <http://github.com/ceph/ceph/pull/9009>`_, Jason Dillaman)
+* librbd: potential concurrent event processing during journal replay (`issue#15755 <http://tracker.ceph.com/issues/15755>`_, `pr#9040 <http://github.com/ceph/ceph/pull/9040>`_, Jason Dillaman)
+* librbd: Potential double free of SetSnapRequest instance (`issue#15571 <http://tracker.ceph.com/issues/15571>`_, `pr#8803 <http://github.com/ceph/ceph/pull/8803>`_, runsisi)
+* librbd: put the validation of image snap context earlier (`pr#9046 <http://github.com/ceph/ceph/pull/9046>`_, runsisi)
+* librbd: reduce log level for image format 1 warning (`issue#15577 <http://tracker.ceph.com/issues/15577>`_, `pr#9003 <http://github.com/ceph/ceph/pull/9003>`_, Jason Dillaman)
+* mds/MDSAuthCap parse no longer fails on paths with hyphens (`issue#15465 <http://tracker.ceph.com/issues/15465>`_, `pr#8969 <http://github.com/ceph/ceph/pull/8969>`_, John Spray)
+* mds: MDS incarnation no longer gets lost after remove filesystem (`issue#15399 <http://tracker.ceph.com/issues/15399>`_, `pr#8970 <http://github.com/ceph/ceph/pull/8970>`_, John Spray)
+* mon/OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1 (`issue#15655 <http://tracker.ceph.com/issues/15655>`_, `pr#9006 <http://github.com/ceph/ceph/pull/9006>`_, Samuel Just)
+* python: clone operation will fail if config overridden with "rbd default format = 1" (`issue#15685 <http://tracker.ceph.com/issues/15685>`_, `pr#8972 <http://github.com/ceph/ceph/pull/8972>`_, Jason Dillaman)
+* radosgw-admin: add missing --zonegroup-id to usage (`issue#15650 <http://tracker.ceph.com/issues/15650>`_, `pr#9019 <http://github.com/ceph/ceph/pull/9019>`_, Casey Bodley)
+* radosgw-admin: update usage for zone[group] modify (`issue#15651 <http://tracker.ceph.com/issues/15651>`_, `pr#9016 <http://github.com/ceph/ceph/pull/9016>`_, Casey Bodley)
+* radosgw-admin: zonegroup remove command (`issue#15684 <http://tracker.ceph.com/issues/15684>`_, `pr#9015 <http://github.com/ceph/ceph/pull/9015>`_, Casey Bodley)
+* rbd CLI to retrieve rbd mirror state for a pool / specific image (`issue#15144 <http://tracker.ceph.com/issues/15144>`_, `issue#14420 <http://tracker.ceph.com/issues/14420>`_, `pr#8868 <http://github.com/ceph/ceph/pull/8868>`_, Mykola Golub)
+* rbd disk-usage CLI command should support calculating full image usage (`issue#14540 <http://tracker.ceph.com/issues/14540>`_, `pr#8870 <http://github.com/ceph/ceph/pull/8870>`_, Jason Dillaman)
+* rbd: helpful error message on map failure (`issue#15721 <http://tracker.ceph.com/issues/15721>`_, `pr#9041 <http://github.com/ceph/ceph/pull/9041>`_, Venky Shankar)
+* rbd: help message distinction between commands and aliases (`issue#15521 <http://tracker.ceph.com/issues/15521>`_, `pr#9004 <http://github.com/ceph/ceph/pull/9004>`_, Yongqiang He)
+* rbd-mirror: admin socket commands to start/stop/restart mirroring (`issue#15718 <http://tracker.ceph.com/issues/15718>`_, `pr#9010 <http://github.com/ceph/ceph/pull/9010>`_, Mykola Golub, Josh Durgin)
+* rbd-mirror can crash if start up is interrupted (`issue#15630 <http://tracker.ceph.com/issues/15630>`_, `pr#8866 <http://github.com/ceph/ceph/pull/8866>`_, Jason Dillaman)
+* rbd-mirror: image sync needs to handle snapshot size and protection status (`issue#15110 <http://tracker.ceph.com/issues/15110>`_, `pr#9050 <http://github.com/ceph/ceph/pull/9050>`_, Jason Dillaman)
+* rbd-mirror: lockdep error during bootstrap (`issue#15664 <http://tracker.ceph.com/issues/15664>`_, `pr#9008 <http://github.com/ceph/ceph/pull/9008>`_, Jason Dillaman)
+* rbd-nbd: fix rbd-nbd aio callback error handling (`issue#15604 <http://tracker.ceph.com/issues/15604>`_, `pr#9005 <http://github.com/ceph/ceph/pull/9005>`_, Chang-Yi Lee)
+* rgw: add AWS4 completion support for RGW_OP_SET_BUCKET_WEBSITE (`issue#15626 <http://tracker.ceph.com/issues/15626>`_, `pr#9018 <http://github.com/ceph/ceph/pull/9018>`_, Javier M. Mellid)
+* rgw admin output (`issue#15747 <http://tracker.ceph.com/issues/15747>`_, `pr#9054 <http://github.com/ceph/ceph/pull/9054>`_, Casey Bodley)
+* rgw: fix issue #15597 (`issue#15597 <http://tracker.ceph.com/issues/15597>`_, `pr#9020 <http://github.com/ceph/ceph/pull/9020>`_, Yehuda Sadeh)
+* rgw: fix printing wrong X-Storage-Url in Swift's TempAuth. (`issue#15667 <http://tracker.ceph.com/issues/15667>`_, `pr#9021 <http://github.com/ceph/ceph/pull/9021>`_, Radoslaw Zarzynski)
+* rgw: handle stripe transition when flushing final pending_data_bl (`issue#15745 <http://tracker.ceph.com/issues/15745>`_, `pr#9053 <http://github.com/ceph/ceph/pull/9053>`_, Yehuda Sadeh)
+* rgw: leak fixes (`issue#15792 <http://tracker.ceph.com/issues/15792>`_, `pr#9022 <http://github.com/ceph/ceph/pull/9022>`_, Yehuda Sadeh)
+* rgw: multisite: Issues with Deleting Buckets (`issue#15540 <http://tracker.ceph.com/issues/15540>`_, `pr#8930 <http://github.com/ceph/ceph/pull/8930>`_, Abhishek Lekshmanan)
+* rgw: period commit fix (`issue#15828 <http://tracker.ceph.com/issues/15828>`_, `pr#9081 <http://github.com/ceph/ceph/pull/9081>`_, Casey Bodley)
+* rgw: period delete fixes (`issue#15469 <http://tracker.ceph.com/issues/15469>`_, `pr#9047 <http://github.com/ceph/ceph/pull/9047>`_, Casey Bodley)
+* rgw: radosgw-admin zone set cuts pool names short if name starts with a period (`issue#15598 <http://tracker.ceph.com/issues/15598>`_, `pr#9029 <http://github.com/ceph/ceph/pull/9029>`_, Yehuda Sadeh)
+* rgw: segfault at RGWAsyncGetSystemObj (`issue#15565 <http://tracker.ceph.com/issues/15565>`_, `issue#15625 <http://tracker.ceph.com/issues/15625>`_, `pr#9017 <http://github.com/ceph/ceph/pull/9017>`_, Yehuda Sadeh)
+* several backports (`issue#15588 <http://tracker.ceph.com/issues/15588>`_, `issue#15655 <http://tracker.ceph.com/issues/15655>`_, `pr#8853 <http://github.com/ceph/ceph/pull/8853>`_, Alexandre Derumier, xie xingguo, Alfredo Deza)
+* systemd: fix typo in preset file (`pr#8843 <http://github.com/ceph/ceph/pull/8843>`_, Nathan Cutler)
+* tests: make check fails on ext4 (`issue#15837 <http://tracker.ceph.com/issues/15837>`_, `pr#9063 <http://github.com/ceph/ceph/pull/9063>`_, Loic Dachary, Sage Weil)
+
+
+v10.2.0 Jewel
+=============
+
+This major release of Ceph is the foundation for the next
+long-term stable release series.  There have been many major changes since
+the Infernalis (9.2.x) and Hammer (0.94.x) releases, and the upgrade
+process is non-trivial. Please read these release notes carefully.
+
+Major Changes from Infernalis
+-----------------------------
+
+- *CephFS*:
+
+  * This is the first release in which CephFS is declared stable!
+    Several features are disabled by default, including snapshots and
+    multiple active MDS servers.
+  * The repair and disaster recovery tools are now feature-complete.
+  * A new cephfs-volume-manager module is included that provides a
+    high-level interface for creating "shares" for OpenStack Manila
+    and similar projects.
+  * There is now experimental support for multiple CephFS file systems
+    within a single cluster.
+
+- *RGW*:
+
+  * The multisite feature has been almost completely rearchitected and
+    rewritten to support any number of clusters/sites, bidirectional
+    fail-over, and active/active configurations.
+  * You can now access radosgw buckets via NFS (experimental).
+  * The AWS4 authentication protocol is now supported.
+  * There is now support for S3 request payer buckets.
+  * The new multitenancy infrastructure improves compatibility with
+    Swift, which provides a separate container namespace for each
+    user/tenant.
+  * The OpenStack Keystone v3 API is now supported.  There are a range
+    of other small Swift API features and compatibility improvements
+    as well, including bulk delete and SLO (static large objects).
+
+- *RBD*:
+
+  * There is new support for mirroring (asynchronous replication) of
+    RBD images across clusters.  This is implemented as a per-RBD
+    image journal that can be streamed across a WAN to another site,
+    and a new rbd-mirror daemon that performs the cross-cluster
+    replication.
+  * The exclusive-lock, object-map, fast-diff, and journaling features
+    can be enabled or disabled dynamically. The deep-flatten features
+    can be disabled dynamically but not re-enabled.
+  * The RBD CLI has been rewritten to provide command-specific help
+    and full bash completion support.
+  * RBD snapshots can now be renamed.
+
+- *RADOS*:
+
+  * BlueStore, a new OSD backend, is included as an experimental
+    feature.  The plan is for it to become the default backend in the
+    K or L release.
+  * The OSD now persists scrub results and provides a librados API to
+    query results in detail.
+  * We have revised our documentation to recommend *against* using
+    ext4 as the underlying filesystem for Ceph OSD daemons due to
+    problems supporting our long object name handling.
+
+Major Changes from Hammer
+-------------------------
+
+- *General*:
+
+  * Ceph daemons are now managed via systemd (with the exception of
+    Ubuntu Trusty, which still uses upstart).
+  * Ceph daemons run as 'ceph' user instead of 'root'.
+  * On Red Hat distros, there is also an SELinux policy.
+
+- *RADOS*:
+
+  * The RADOS cache tier can now proxy write operations to the base
+    tier, allowing writes to be handled without forcing migration of
+    an object into the cache.
+  * The SHEC erasure coding support is no longer flagged as
+    experimental. SHEC trades some additional storage space for faster
+    repair.
+  * There is now a unified queue (and thus prioritization) of client
+    IO, recovery, scrubbing, and snapshot trimming.
+  * There have been many improvements to low-level repair tooling
+    (ceph-objectstore-tool).
+  * The internal ObjectStore API has been significantly cleaned up in order
+    to facilitate new storage backends like BlueStore.
+
+- *RGW*:
+
+  * The Swift API now supports object expiration.
+  * There are many Swift API compatibility improvements.
+
+- *RBD*:
+
+  * The ``rbd du`` command shows actual usage (quickly, when
+    object-map is enabled).
+  * The object-map feature has seen many stability improvements.
+  * The object-map and exclusive-lock features can be enabled or disabled
+    dynamically.
+  * You can now store user metadata and set persistent librbd options
+    associated with individual images.
+  * The new deep-flatten features allow flattening of a clone and all
+    of its snapshots.  (Previously snapshots could not be flattened.)
+  * The export-diff command is now faster (it uses aio).  There is also
+    a new fast-diff feature.
+  * The --size argument can be specified with a suffix for units
+    (e.g., ``--size 64G``).
+  * There is a new ``rbd status`` command that, for now, shows who has
+    the image open/mapped.
+
+- *CephFS*:
+
+  * You can now rename snapshots.
+  * There have been ongoing improvements around administration, diagnostics,
+    and the check and repair tools.
+  * The caching and revocation of client cache state due to unused
+    inodes has been dramatically improved.
+  * The ceph-fuse client behaves better on 32-bit hosts.
+
+Distro compatibility
+--------------------
+
+Starting with Infernalis, we have dropped support for many older
+distributions so that we can move to a newer compiler toolchain (e.g.,
+C++11).  Although it is still possible to build Ceph on older
+distributions by installing backported development tools, we are not
+building and publishing release packages for ceph.com.
+
+We now build packages for the following distributions and architectures:
+
+- x86_64:
+
+  * CentOS 7.x.  We have dropped support for CentOS 6 (and other RHEL 6
+    derivatives, like Scientific Linux 6).
+  * Debian Jessie 8.x.  Debian Wheezy 7.x's g++ has incomplete support
+    for C++11 (and no systemd).
+  * Ubuntu Xenial 16.04 and Trusty 14.04.  Ubuntu Precise 12.04 is no
+    longer supported.
+  * Fedora 22 or later.
+
+- aarch64 / arm64:
+
+  * Ubuntu Xenial 16.04.
+
+Upgrading from Infernalis or Hammer
+-----------------------------------
+
+* We now recommend against using ``ext4`` as the underlying file
+  system for Ceph OSDs, especially when RGW or other users of long
+  RADOS object names are used.  For more information about why, please
+  see `Filesystem Recommendations`_.
+
+  If you have an existing cluster that uses ext4 for the OSDs but uses only
+  RBD and/or CephFS, then the ext4 limitations will not affect you.  Before
+  upgrading, be sure add the following to ``ceph.conf`` to allow the OSDs to
+  start::
+
+    osd max object name len = 256
+    osd max object namespace len = 64
+
+  Keep in mind that if you set these lower object name limits and
+  later decide to use RGW on this cluster, it will have problems
+  storing S3/Swift objects with long names.  This startup check can also be
+  disabled via the below option, although this is not recommended::
+
+    osd check max object name len on startup = false
+
+.. _Filesystem Recommendations: ../configuration/filesystem-recommendations
+
+* There are no major compatibility changes since Infernalis.  Simply
+  upgrading the daemons on each host and restarting all daemons is
+  sufficient.
+
+* The rbd CLI no longer accepts the deprecated '--image-features' option
+  during create, import, and clone operations.  The '--image-feature'
+  option should be used instead.
+
+* The rbd legacy image format (version 1) is deprecated with the Jewel release.
+  Attempting to create a new version 1 RBD image will result in a warning.
+  Future releases of Ceph will remove support for version 1 RBD images.
+
+* The 'send_pg_creates' and 'map_pg_creates' mon CLI commands are
+  obsolete and no longer supported.
+
+* A new configure option 'mon_election_timeout' is added to specifically
+  limit max waiting time of monitor election process, which was previously
+  restricted by 'mon_lease'.
+
+* CephFS filesystems created using versions older than Firefly (0.80) must
+  use the new 'cephfs-data-scan tmap_upgrade' command after upgrading to
+  Jewel.  See 'Upgrading' in the CephFS documentation for more information.
+
+* The 'ceph mds setmap' command has been removed.
+
+* The default RBD image features for new images have been updated to
+  enable the following: exclusive lock, object map, fast-diff, and
+  deep-flatten. These features are not currently supported by the RBD
+  kernel driver nor older RBD clients. They can be disabled on a per-image
+  basis via the RBD CLI, or the default features can be updated to the
+  pre-Jewel setting by adding the following to the client section of the Ceph
+  configuration file::
+
+    rbd default features = 1
+
+* The rbd legacy image format (version 1) is deprecated with the Jewel
+  release.
+
+* After upgrading, users should set the 'sortbitwise' flag to enable the new
+  internal object sort order::
+
+    ceph osd set sortbitwise
+
+  This flag is important for the new object enumeration API and for
+  new backends like BlueStore.
+
+* The rbd CLI no longer permits creating images and snapshots with potentially
+  ambiguous names (e.g. the '/' and '@' characters are disallowed). The
+  validation can be temporarily disabled by adding "--rbd-validate-names=false"
+  to the rbd CLI when creating an image or snapshot. It can also be disabled
+  by adding the following to the client section of the Ceph configuration file::
+
+    rbd validate names = false
+
+Upgrading from Hammer
+---------------------
+
+* All cluster nodes must first upgrade to Hammer v0.94.4 or a later
+  v0.94.z release; only then is it possible to upgrade to Jewel
+  10.2.z.
+
+* For all distributions that support systemd (CentOS 7, Fedora, Debian
+  Jessie 8.x, OpenSUSE), ceph daemons are now managed using native systemd
+  files instead of the legacy sysvinit scripts.  For example::
+
+    systemctl start ceph.target       # start all daemons
+    systemctl status ceph-osd@12      # check status of osd.12
+
+  The main notable distro that is *not* yet using systemd is Ubuntu trusty
+  14.04.  (The next Ubuntu LTS, 16.04, will use systemd instead of upstart.)
+
+* Ceph daemons now run as user and group ``ceph`` by default.  The
+  ceph user has a static UID assigned by Fedora and Debian (also used by
+  derivative distributions like RHEL/CentOS and Ubuntu).  On SUSE the same
+  UID/GID as in Fedora and Debian will be used, *provided it is not already
+  assigned*. In the unlikely event the preferred UID or GID is assigned to a
+  different user/group, ceph will get a dynamically assigned UID/GID.
+
+  If your systems already have a ceph user, upgrading the package will cause
+  problems.  We suggest you first remove or rename the existing 'ceph' user
+  and 'ceph' group before upgrading.
+
+  When upgrading, administrators have two options:
+
+   #. Add the following line to ``ceph.conf`` on all hosts::
+
+        setuser match path = /var/lib/ceph/$type/$cluster-$id
+
+      This will make the Ceph daemons run as root (i.e., not drop
+      privileges and switch to user ceph) if the daemon's data
+      directory is still owned by root.  Newly deployed daemons will
+      be created with data owned by user ceph and will run with
+      reduced privileges, but upgraded daemons will continue to run as
+      root.
+
+   #. Fix the data ownership during the upgrade.  This is the
+      preferred option, but it is more work and can be very time
+      consuming.  The process for each host is to:
+
+      #. Upgrade the ceph package.  This creates the ceph user and group.  For
+         example::
+
+           ceph-deploy install --stable jewel HOST
+
+      #. Stop the daemon(s)::
+
+           service ceph stop           # fedora, centos, rhel, debian
+           stop ceph-all               # ubuntu
+
+      #. Fix the ownership::
+
+           chown -R ceph:ceph /var/lib/ceph
+           chown -R ceph:ceph /var/log/ceph
+
+      #. Restart the daemon(s)::
+
+           start ceph-all                # ubuntu
+           systemctl start ceph.target   # debian, centos, fedora, rhel
+
+      Alternatively, the same process can be done with a single daemon
+      type, for example by stopping only monitors and chowning only
+      ``/var/lib/ceph/mon``.
+
+* The on-disk format for the experimental KeyValueStore OSD backend has
+  changed.  You will need to remove any OSDs using that backend before you
+  upgrade any test clusters that use it.
+
+* When a pool quota is reached, librados operations now block indefinitely,
+  the same way they do when the cluster fills up.  (Previously they would return
+  -ENOSPC.)  By default, a full cluster or pool will now block.  If your
+  librados application can handle ENOSPC or EDQUOT errors gracefully, you can
+  get error returns instead by using the new librados OPERATION_FULL_TRY flag.
+
+* The return code for librbd's rbd_aio_read and Image::aio_read API methods no
+  longer returns the number of bytes read upon success.  Instead, it returns 0
+  upon success and a negative value upon failure.
+
+* 'ceph scrub', 'ceph compact' and 'ceph sync force' are now DEPRECATED.  Users
+  should instead use 'ceph mon scrub', 'ceph mon compact' and
+  'ceph mon sync force'.
+
+* 'ceph mon_metadata' should now be used as 'ceph mon metadata'. There is no
+  need to deprecate this command (same major release since it was first
+  introduced).
+
+* The `--dump-json` option of "osdmaptool" is replaced by `--dump json`.
+
+* The commands of "pg ls-by-{pool,primary,osd}" and "pg ls" now take "recovering"
+  instead of "recovery", to include the recovering pgs in the listed pgs.
+
+Upgrading from Firefly
+----------------------
+
+Upgrading directly from Firefly v0.80.z is not recommended.  It is
+possible to do a direct upgrade, but not without downtime, as all OSDs
+must be stopped, upgraded, and then restarted.  We recommend that
+clusters be first upgraded to Hammer v0.94.6 or a later v0.94.z
+release; only then is it possible to upgrade to Jewel 10.2.z for an
+online upgrade (see below).
+
+To do an offline upgrade directly from Firefly, all Firefly OSDs must
+be stopped and marked down before any Jewel OSDs will be allowed
+to start up.  This fencing is enforced by the Jewel monitor, so
+you should use an upgrade procedure like:
+
+  #. Upgrade Ceph on monitor hosts
+  #. Restart all ceph-mon daemons
+  #. Set noout::
+       ceph osd set noout
+  #. Upgrade Ceph on all OSD hosts
+  #. Stop all ceph-osd daemons
+  #. Mark all OSDs down with something like::
+       ceph osd down `seq 0 1000`
+  #. Start all ceph-osd daemons
+  #. Let the cluster settle and then unset noout::
+       ceph osd unset noout
+  #. Upgrade and restart any remaining daemons (ceph-mds, radosgw)
+
+Notable Changes since Infernalis
+--------------------------------
+
+* aarch64: add optimized version of crc32c (Yazen Ghannam, Steve Capper)
+* Adding documentation on how to use new dynamic throttle scheme (`pr#8069 <http://github.com/ceph/ceph/pull/8069>`_, Somnath Roy)
+* admin/build-doc: depend on zlib1g-dev and graphviz (`pr#7522 <http://github.com/ceph/ceph/pull/7522>`_, Ken Dreyer)
+* auth: cache/reuse crypto lib key objects, optimize msg signature check (Sage Weil)
+* auth: fail if rotating key is missing (do not spam log) (`pr#6473 <http://github.com/ceph/ceph/pull/6473>`_, Qiankun Zheng)
+* auth: fix crash when bad keyring is passed (`pr#6698 <http://github.com/ceph/ceph/pull/6698>`_, Dunrong Huang)
+* auth: make keyring without mon entity type return -EACCES (`pr#5734 <http://github.com/ceph/ceph/pull/5734>`_, Xiaowei Chen)
+* AUTHORS: update email (`pr#7854 <http://github.com/ceph/ceph/pull/7854>`_, Yehuda Sadeh)
+* auth: reinit NSS after fork() (#11128 Yan, Zheng)
+* authtool: update --help and manpage to match code. (`pr#8456 <http://github.com/ceph/ceph/pull/8456>`_, Robin H. Johnson)
+* autotools: fix out of tree build (Krxysztof Kosinski)
+* autotools: improve make check output (Loic Dachary)
+* Be more careful about directory fragmentation and scrubbing (`issue#15167 <http://tracker.ceph.com/issues/15167>`_, `pr#8180 <http://github.com/ceph/ceph/pull/8180>`_, Yan, Zheng)
+* bluestore: latest and greatest (`issue#14210 <http://tracker.ceph.com/issues/14210>`_, `issue#13801 <http://tracker.ceph.com/issues/13801>`_, `pr#6896 <http://github.com/ceph/ceph/pull/6896>`_, xie.xingguo, Jianpeng Ma, YiQiang Chen, Sage Weil, Ning Yao)
+* buffer: add invalidate_crc() (Piotr Dalek)
+* buffer: add symmetry operator==() and operator!=() (`pr#7974 <http://github.com/ceph/ceph/pull/7974>`_, Kefu Chai)
+* buffer: fix internal iterator invalidation on rebuild, get_contiguous (`pr#6962 <http://github.com/ceph/ceph/pull/6962>`_, Sage Weil)
+* buffer: fix zero bug (#12252 Haomai Wang)
+* buffer: hide iterator_impl symbols (`issue#14788 <http://tracker.ceph.com/issues/14788>`_, `pr#7688 <http://github.com/ceph/ceph/pull/7688>`_, Kefu Chai)
+* buffer: increment history alloc as well in raw_combined (`issue#14955 <http://tracker.ceph.com/issues/14955>`_, `pr#7910 <http://github.com/ceph/ceph/pull/7910>`_, Samuel Just)
+* buffer: make usable outside of ceph source again (`pr#6863 <http://github.com/ceph/ceph/pull/6863>`_, Josh Durgin)
+* buffer: raw_combined allocations buffer and ref count together (`pr#7612 <http://github.com/ceph/ceph/pull/7612>`_, Sage Weil)
+* buffer: some cleanup (Michal Jarzabek)
+* buffer: use move construct to append/push_back/push_front (`pr#7455 <http://github.com/ceph/ceph/pull/7455>`_, Haomai Wang)
+* build: Adding build requires (`pr#7742 <http://github.com/ceph/ceph/pull/7742>`_, Erwan Velu)
+* build: a few armhf (32-bit build) fixes (`pr#7999 <http://github.com/ceph/ceph/pull/7999>`_, Eric Lee, Sage Weil)
+* build: allow jemalloc with rocksdb-static (`pr#7368 <http://github.com/ceph/ceph/pull/7368>`_, Somnath Roy)
+* build: allow tcmalloc-minimal (Thorsten Behrens)
+* build: build internal plugins and classes as modules (`pr#6462 <http://github.com/ceph/ceph/pull/6462>`_, James Page)
+* build: C++11 now supported
+* build: cmake check fixes (`pr#6787 <http://github.com/ceph/ceph/pull/6787>`_, Orit Wasserman)
+* build: cmake: fix nss linking (Danny Al-Gaaf)
+* build: cmake: misc fixes (Orit Wasserman, Casey Bodley)
+* build: cmake tweaks (`pr#6254 <http://github.com/ceph/ceph/pull/6254>`_, John Spray)
+* build: disable LTTNG by default (#11333 Josh Durgin)
+* build: do not build ceph-dencoder with tcmalloc (#10691 Boris Ranto)
+* build: fix a few warnings (`pr#6847 <http://github.com/ceph/ceph/pull/6847>`_, Orit Wasserman)
+* build: fix bz2-dev dependency (`pr#6948 <http://github.com/ceph/ceph/pull/6948>`_, Samuel Just)
+* build: fix compiling warnings (`pr#8366 <http://github.com/ceph/ceph/pull/8366>`_, Dongsheng Yang)
+* build: Fixing BTRFS issue at 'make check' (`pr#7805 <http://github.com/ceph/ceph/pull/7805>`_, Erwan Velu)
+* build: fix Jenkins make check errors due to deep-scrub randomization (`pr#6671 <http://github.com/ceph/ceph/pull/6671>`_, David Zafman)
+* build: fix junit detection on Fedora 22 (Ira Cooper)
+* build: fix pg ref disabling (William A. Kennington III)
+* build: fix ppc build (James Page)
+* build: fix the autotools and cmake build (the new fusestore needs libfuse) (`pr#7393 <http://github.com/ceph/ceph/pull/7393>`_, Kefu Chai)
+* build: fix warnings (`pr#7197 <http://github.com/ceph/ceph/pull/7197>`_, Kefu Chai, xie xingguo)
+* build: fix warnings (`pr#7315 <http://github.com/ceph/ceph/pull/7315>`_, Kefu Chai)
+* build: FreeBSD related fixes (`pr#7170 <http://github.com/ceph/ceph/pull/7170>`_, Mykola Golub)
+* build: Gentoo: _FORTIFY_SOURCE fix. (`issue#13920 <http://tracker.ceph.com/issues/13920>`_, `pr#6739 <http://github.com/ceph/ceph/pull/6739>`_, Robin H. Johnson)
+* build: install-deps: misc fixes (Loic Dachary)
+* build: install-deps.sh improvements (Loic Dachary)
+* build: install-deps: support OpenSUSE (Loic Dachary)
+* build: kill warnings (`pr#7397 <http://github.com/ceph/ceph/pull/7397>`_, Kefu Chai)
+* build: make_dist_tarball.sh (Sage Weil)
+* build: many cmake improvements
+* build: misc cmake fixes (Matt Benjamin)
+* build: misc fixes (Boris Ranto, Ken Dreyer, Owen Synge)
+* build: misc make check fixes (`pr#7153 <http://github.com/ceph/ceph/pull/7153>`_, Sage Weil)
+* build: more CMake package check fixes (`pr#6108 <http://github.com/ceph/ceph/pull/6108>`_, Daniel Gryniewicz)
+* build: move libexec scripts to standardize across distros (`issue#14687 <http://tracker.ceph.com/issues/14687>`_, `issue#14705 <http://tracker.ceph.com/issues/14705>`_, `issue#14723 <http://tracker.ceph.com/issues/14723>`_, `pr#7636 <http://github.com/ceph/ceph/pull/7636>`_, Nathan Cutler, Kefu Chai)
+* build/ops: enable CR in CentOS 7 (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#6844 <http://github.com/ceph/ceph/pull/6844>`_, Loic Dachary)
+* build/ops: rbd-replay moved from ceph-test-dbg to ceph-common-dbg (`issue#13785 <http://tracker.ceph.com/issues/13785>`_, `pr#6578 <http://github.com/ceph/ceph/pull/6578>`_, Loic Dachary)
+* build/ops: systemd ceph-disk unit must not assume /bin/flock (`issue#13975 <http://tracker.ceph.com/issues/13975>`_, `pr#6803 <http://github.com/ceph/ceph/pull/6803>`_, Loic Dachary)
+* build: OSX build fixes (Yan, Zheng)
+* build: Refrain from versioning and packaging EC testing plugins (`issue#14756 <http://tracker.ceph.com/issues/14756>`_, `issue#14723 <http://tracker.ceph.com/issues/14723>`_, `pr#7637 <http://github.com/ceph/ceph/pull/7637>`_, Nathan Cutler, Kefu Chai)
+* build: remove rest-bench
+* build: Respect TMPDIR for virtualenv. (`pr#8457 <http://github.com/ceph/ceph/pull/8457>`_, Robin H. Johnson)
+* build: spdk submodule; cmake (`pr#7503 <http://github.com/ceph/ceph/pull/7503>`_, Kefu Chai)
+* build: workaround an automake bug for "make check" (`issue#14723 <http://tracker.ceph.com/issues/14723>`_, `pr#7626 <http://github.com/ceph/ceph/pull/7626>`_, Kefu Chai)
+* ceph-authtool: fix return code on error (Gerhard Muntingh)
+* ceph: bash auto complete for CLI based on mon command descriptions (`pr#7693 <http://github.com/ceph/ceph/pull/7693>`_, Adam Kupczyk)
+* ceph_daemon.py: Resolved ImportError to work with python3 (`pr#7937 <http://github.com/ceph/ceph/pull/7937>`_, Sarthak Munshi)
+* ceph-detect-init: add debian/jessie test (`pr#8074 <http://github.com/ceph/ceph/pull/8074>`_, Kefu Chai)
+* ceph-detect-init: added Linux Mint (Michal Jarzabek)
+* ceph-detect-init: add missing test case (`pr#8105 <http://github.com/ceph/ceph/pull/8105>`_, Nathan Cutler)
+* ceph-detect-init: fix py3 test (`pr#7025 <http://github.com/ceph/ceph/pull/7025>`_, Kefu Chai)
+* ceph-detect-init: fix py3 test (`pr#7243 <http://github.com/ceph/ceph/pull/7243>`_, Kefu Chai)
+* ceph_detect_init/__init__.py: remove shebang (`pr#7731 <http://github.com/ceph/ceph/pull/7731>`_, Nathan Cutler)
+* ceph-detect-init: return correct value on recent SUSE distros (`issue#14770 <http://tracker.ceph.com/issues/14770>`_, `pr#7909 <http://github.com/ceph/ceph/pull/7909>`_, Nathan Cutler)
+* ceph-detect-init: robust init system detection (Owen Synge)
+* ceph-detect-init/run-tox.sh: FreeBSD: No init detect (`pr#8373 <http://github.com/ceph/ceph/pull/8373>`_, Willem Jan Withagen)
+* ceph-detect-init: Ubuntu >= 15.04 uses systemd (`pr#6873 <http://github.com/ceph/ceph/pull/6873>`_, James Page)
+* ceph-disk: Add destroy and deactivate option (`issue#7454 <http://tracker.ceph.com/issues/7454>`_, `pr#5867 <http://github.com/ceph/ceph/pull/5867>`_, Vicente Cheng)
+* ceph-disk: add -f flag for btrfs mkfs (`pr#7222 <http://github.com/ceph/ceph/pull/7222>`_, Darrell Enns)
+* ceph-disk: Add --setuser and --setgroup options for ceph-disk (`pr#7351 <http://github.com/ceph/ceph/pull/7351>`_, Mike Shuey)
+* ceph-disk: ceph-disk list fails on /dev/cciss!c0d0 (`issue#13970 <http://tracker.ceph.com/issues/13970>`_, `issue#14233 <http://tracker.ceph.com/issues/14233>`_, `issue#14230 <http://tracker.ceph.com/issues/14230>`_, `pr#6879 <http://github.com/ceph/ceph/pull/6879>`_, Loic Dachary)
+* ceph-disk: compare parted output with the dereferenced path (`issue#13438 <http://tracker.ceph.com/issues/13438>`_, `pr#6219 <http://github.com/ceph/ceph/pull/6219>`_, Joe Julian)
+* ceph-disk: deactivate / destroy PATH arg are optional (`pr#7756 <http://github.com/ceph/ceph/pull/7756>`_, Loic Dachary)
+* ceph-disk: do not always fail when re-using a partition (`pr#8508 <http://github.com/ceph/ceph/pull/8508>`_, You Ji)
+* ceph-disk: ensure 'zap' only operates on a full disk (#11272 Loic Dachary)
+* ceph-disk: fixes to respect init system (Loic Dachary, Owen Synge)
+* ceph-disk: fix failures when preparing disks with udev > 214 (`issue#14080 <http://tracker.ceph.com/issues/14080>`_, `issue#14094 <http://tracker.ceph.com/issues/14094>`_, `pr#6926 <http://github.com/ceph/ceph/pull/6926>`_, Loic Dachary, Ilya Dryomov)
+* ceph-disk: fix prepare --help (`pr#7758 <http://github.com/ceph/ceph/pull/7758>`_, Loic Dachary)
+* ceph-disk: Fix trivial typo (`pr#7472 <http://github.com/ceph/ceph/pull/7472>`_, Brad Hubbard)
+* ceph-disk: fix zap sgdisk invocation (Owen Synge, Thorsten Behrens)
+* ceph-disk: flake8 fixes (`pr#7646 <http://github.com/ceph/ceph/pull/7646>`_, Loic Dachary)
+* ceph-disk: follow ceph-osd hints when creating journal (#9580 Sage Weil)
+* ceph-disk: get Nonetype when ceph-disk list with --format plain on single device. (`pr#6410 <http://github.com/ceph/ceph/pull/6410>`_, Vicente Cheng)
+* ceph-disk: handle re-using existing partition (#10987 Loic Dachary)
+* ceph-disk: improve parted output parsing (#10983 Loic Dachary)
+* ceph-disk: Improving 'make check' for ceph-disk (`pr#7762 <http://github.com/ceph/ceph/pull/7762>`_, Erwan Velu)
+* ceph-disk: install pip > 6.1 (#11952 Loic Dachary)
+* ceph-disk: key management support (`issue#14669 <http://tracker.ceph.com/issues/14669>`_, `pr#7552 <http://github.com/ceph/ceph/pull/7552>`_, Loic Dachary)
+* ceph-disk: make some arguments as required if necessary (`pr#7687 <http://github.com/ceph/ceph/pull/7687>`_, Dongsheng Yang)
+* ceph-disk: make suppression work for activate-all and activate-journal (Dan van der Ster)
+* ceph-disk: many fixes (Loic Dachary, Alfredo Deza)
+* ceph-disk: pass --cluster arg on prepare subcommand (Kefu Chai)
+* ceph-disk: s/dmcrpyt/dmcrypt/ (`issue#14838 <http://tracker.ceph.com/issues/14838>`_, `pr#7744 <http://github.com/ceph/ceph/pull/7744>`_, Loic Dachary, Frode Sandholtbraaten)
+* ceph-disk: support bluestore (`issue#13422 <http://tracker.ceph.com/issues/13422>`_, `pr#7218 <http://github.com/ceph/ceph/pull/7218>`_, Loic Dachary, Sage Weil)
+* ceph-disk: support for multipath devices (Loic Dachary)
+* ceph-disk: support NVMe device partitions (#11612 Ilja Slepnev)
+* ceph-disk/test: fix test_prepare.py::TestPrepare tests (`pr#7549 <http://github.com/ceph/ceph/pull/7549>`_, Kefu Chai)
+* ceph-disk: warn for prepare partitions with bad GUIDs (`issue#13943 <http://tracker.ceph.com/issues/13943>`_, `pr#6760 <http://github.com/ceph/ceph/pull/6760>`_, David Disseldorp)
+* ceph: fix 'df' units (Zhe Zhang)
+* ceph: fix parsing in interactive cli mode (#11279 Kefu Chai)
+* ceph: fix tell behavior (`pr#6329 <http://github.com/ceph/ceph/pull/6329>`_, David Zafman)
+* cephfs-data-scan: many additions, improvements (John Spray)
+* cephfs-data-scan: scan_frags (`pr#5941 <http://github.com/ceph/ceph/pull/5941>`_, John Spray)
+* cephfs-data-scan: scrub tag filtering (#12133 and #12145) (`issue#12133 <http://tracker.ceph.com/issues/12133>`_, `issue#12145 <http://tracker.ceph.com/issues/12145>`_, `pr#5685 <http://github.com/ceph/ceph/pull/5685>`_, John Spray)
+* ceph-fuse: add process to ceph-fuse --help (`pr#6821 <http://github.com/ceph/ceph/pull/6821>`_, Wei Feng)
+* ceph-fuse: do not require successful remount when unmounting (#10982 Greg Farnum)
+* ceph-fuse: fix double decreasing the count to trim caps (`issue#14319 <http://tracker.ceph.com/issues/14319>`_, `pr#7229 <http://github.com/ceph/ceph/pull/7229>`_, Zhi Zhang)
+* ceph-fuse: fix double free of args (`pr#7015 <http://github.com/ceph/ceph/pull/7015>`_, Ilya Shipitsin)
+* ceph-fuse: fix fsync() (`pr#6388 <http://github.com/ceph/ceph/pull/6388>`_, Yan, Zheng)
+* ceph-fuse: Fix potential filehandle ref leak at umount (`issue#14800 <http://tracker.ceph.com/issues/14800>`_, `pr#7686 <http://github.com/ceph/ceph/pull/7686>`_, Zhi Zhang)
+* ceph-fuse, libcephfs: don't clear COMPLETE when trimming null (Yan, Zheng)
+* ceph-fuse, libcephfs: drop inode when rmdir finishes (#11339 Yan, Zheng)
+* ceph-fuse,libcephfs: Fix client handling of "lost" open directories on shutdown (`issue#14996 <http://tracker.ceph.com/issues/14996>`_, `pr#7994 <http://github.com/ceph/ceph/pull/7994>`_, Yan, Zheng)
+* ceph-fuse,libcephfs: fix free fds being exhausted eventually because freed fds are never put back (`issue#14798 <http://tracker.ceph.com/issues/14798>`_, `pr#7685 <http://github.com/ceph/ceph/pull/7685>`_, Zhi Zhang)
+* ceph-fuse,libcephfs: fix uninline (#11356 Yan, Zheng)
+* ceph-fuse, libcephfs: hold exclusive caps on dirs we "own" (#11226 Greg Farnum)
+* ceph-fuse: mostly behave on 32-bit hosts (Yan, Zheng)
+* ceph-fuse:print usage information when no parameter specified (`pr#6868 <http://github.com/ceph/ceph/pull/6868>`_, Bo Cai)
+* ceph-fuse: rotate log file (`pr#8485 <http://github.com/ceph/ceph/pull/8485>`_, Sage Weil)
+* ceph-fuse: While starting ceph-fuse, start the log thread first (`issue#13443 <http://tracker.ceph.com/issues/13443>`_, `pr#6224 <http://github.com/ceph/ceph/pull/6224>`_, Wenjun Huang)
+* ceph: improve error output for 'tell' (#11101 Kefu Chai)
+* ceph: improve the error message (`issue#11101 <http://tracker.ceph.com/issues/11101>`_, `pr#7106 <http://github.com/ceph/ceph/pull/7106>`_, Kefu Chai)
+* ceph.in: avoid a broken pipe error when use ceph command (`issue#14354 <http://tracker.ceph.com/issues/14354>`_, `pr#7212 <http://github.com/ceph/ceph/pull/7212>`_, Bo Cai)
+* ceph.in: correct dev python path for automake builds (`pr#8360 <http://github.com/ceph/ceph/pull/8360>`_, Josh Durgin)
+* ceph.in: fix python libpath for automake as well (`pr#8362 <http://github.com/ceph/ceph/pull/8362>`_, Josh Durgin)
+* ceph.in: Minor python3 specific changes (`pr#7947 <http://github.com/ceph/ceph/pull/7947>`_, Sarthak Munshi)
+* ceph-kvstore-tool: handle bad out file on command line (`pr#6093 <http://github.com/ceph/ceph/pull/6093>`_, Kefu Chai)
+* ceph-mds:add --help/-h (`pr#6850 <http://github.com/ceph/ceph/pull/6850>`_, Cilang Zhao)
+* ceph-monstore-tool: fix store-copy (Huangjun)
+* ceph: new 'ceph daemonperf' command (John Spray, Mykola Golub)
+* ceph_objectstore_bench: fix race condition, bugs (`issue#13516 <http://tracker.ceph.com/issues/13516>`_, `pr#6681 <http://github.com/ceph/ceph/pull/6681>`_, Igor Fedotov)
+* ceph-objectstore-tool: fix --dry-run for many ceph-objectstore-tool operations (`pr#6545 <http://github.com/ceph/ceph/pull/6545>`_, David Zafman)
+* ceph-objectstore-tool: many many improvements (David Zafman)
+* ceph-objectstore-tool: refactoring and cleanup (John Spray)
+* ceph-post-file: misc fixes (Joey McDonald, Sage Weil)
+* ceph-rest-api: fix fs/flag/set (`pr#8428 <http://github.com/ceph/ceph/pull/8428>`_, Sage Weil)
+* ceph.spec.in: add BuildRequires: systemd (`issue#13860 <http://tracker.ceph.com/issues/13860>`_, `pr#6692 <http://github.com/ceph/ceph/pull/6692>`_, Nathan Cutler)
+* ceph.spec.in: add copyright notice (`issue#14694 <http://tracker.ceph.com/issues/14694>`_, `pr#7569 <http://github.com/ceph/ceph/pull/7569>`_, Nathan Cutler)
+* ceph.spec.in: add license declaration (`pr#7574 <http://github.com/ceph/ceph/pull/7574>`_, Nathan Cutler)
+* ceph.spec.in: disable lttng and babeltrace explicitly (`issue#14844 <http://tracker.ceph.com/issues/14844>`_, `pr#7857 <http://github.com/ceph/ceph/pull/7857>`_, Kefu Chai)
+* ceph.spec.in: do not install Ceph RA on systemd platforms (`issue#14828 <http://tracker.ceph.com/issues/14828>`_, `pr#7894 <http://github.com/ceph/ceph/pull/7894>`_, Nathan Cutler)
+* ceph.spec.in: fix openldap and openssl build dependencies for SUSE (`issue#15138 <http://tracker.ceph.com/issues/15138>`_, `pr#8120 <http://github.com/ceph/ceph/pull/8120>`_, Nathan Cutler)
+* ceph.spec.in: limit _smp_mflags when lowmem_builder is set in SUSE's OBS (`issue#13858 <http://tracker.ceph.com/issues/13858>`_, `pr#6691 <http://github.com/ceph/ceph/pull/6691>`_, Nathan Cutler)
+* ceph_test_libcephfs: tolerate duplicated entries in readdir (`issue#14377 <http://tracker.ceph.com/issues/14377>`_, `pr#7246 <http://github.com/ceph/ceph/pull/7246>`_, Yan, Zheng)
+* ceph_test_msgr: reduce test size to fix memory size (`pr#8127 <http://github.com/ceph/ceph/pull/8127>`_, Haomai Wang)
+* ceph_test_msgr: Use send_message instead of keepalive to wakeup connection (`pr#6605 <http://github.com/ceph/ceph/pull/6605>`_, Haomai Wang)
+* ceph_test_rados_misc: shorten mount timeout (`pr#8209 <http://github.com/ceph/ceph/pull/8209>`_, Sage Weil)
+* ceph_test_rados: test pipelined reads (Zhiqiang Wang)
+* check-generated.sh: can't source bash from sh (`pr#8521 <http://github.com/ceph/ceph/pull/8521>`_, Michal Jarzabek)
+* cleanup (`pr#8058 <http://github.com/ceph/ceph/pull/8058>`_, Yehuda Sadeh, Orit Wasserman)
+* cleanup: remove misc dead code (`pr#7201 <http://github.com/ceph/ceph/pull/7201>`_, Erwan Velu)
+* client: a better check for MDS availability (`pr#6253 <http://github.com/ceph/ceph/pull/6253>`_, John Spray)
+* client: add option to control how directory size is calculated (`pr#7323 <http://github.com/ceph/ceph/pull/7323>`_, Yan, Zheng)
+* client: avoid creating orphan object in Client::check_pool_perm() (`issue#13782 <http://tracker.ceph.com/issues/13782>`_, `pr#6603 <http://github.com/ceph/ceph/pull/6603>`_, Yan, Zheng)
+* client: avoid sending unnecessary FLUSHSNAP messages (Yan, Zheng)
+* client: check if Fh is readable when processing a read (`issue#11517 <http://tracker.ceph.com/issues/11517>`_, `pr#7209 <http://github.com/ceph/ceph/pull/7209>`_, Yan, Zheng)
+* client: close mds sessions in shutdown() (`pr#6269 <http://github.com/ceph/ceph/pull/6269>`_, John Spray)
+* client: don't invalidate page cache when inode is no longer used (`pr#6380 <http://github.com/ceph/ceph/pull/6380>`_, Yan, Zheng)
+* client: don't mark_down on command reply (`pr#6204 <http://github.com/ceph/ceph/pull/6204>`_, John Spray)
+* client: drop prefix from ints (`pr#6275 <http://github.com/ceph/ceph/pull/6275>`_, John Coyle)
+* client: exclude setfilelock when calculating oldest tid (Yan, Zheng)
+* client: fix error handling in check_pool_perm (John Spray)
+* client: flush kernel pagecache before creating snapshot (`issue#10436 <http://tracker.ceph.com/issues/10436>`_, `pr#7495 <http://github.com/ceph/ceph/pull/7495>`_, Yan, Zheng)
+* client: fsync waits only for inode's caps to flush (Yan, Zheng)
+* client: invalidate kernel dcache when cache size exceeds limits (Yan, Zheng)
+* client: make fsync wait for unsafe dir operations (Yan, Zheng)
+* client: modify a word in log (`pr#6906 <http://github.com/ceph/ceph/pull/6906>`_, YongQiang He)
+* client: pin lookup dentry to avoid inode being freed (Yan, Zheng)
+* client: properly trim unlinked inode (`issue#13903 <http://tracker.ceph.com/issues/13903>`_, `pr#7297 <http://github.com/ceph/ceph/pull/7297>`_, Yan, Zheng)
+* client: removed unused Mutex from MetaRequest (`pr#7655 <http://github.com/ceph/ceph/pull/7655>`_, Greg Farnum)
+* client: sys/file.h includes for flock operations (`pr#6282 <http://github.com/ceph/ceph/pull/6282>`_, John Coyle)
+* client: use null snapc to check pool permission (`issue#13714 <http://tracker.ceph.com/issues/13714>`_, `pr#6497 <http://github.com/ceph/ceph/pull/6497>`_, Yan, Zheng)
+* cls/cls_rbd.cc: fix misused metadata_name_from_key (`issue#13922 <http://tracker.ceph.com/issues/13922>`_, `pr#6661 <http://github.com/ceph/ceph/pull/6661>`_, Xiaoxi Chen)
+* cls/cls_rbd: pass string by reference (`pr#7232 <http://github.com/ceph/ceph/pull/7232>`_, Jeffrey Lu)
+* cls_hello: Fix grammatical error in description comment (`pr#7951 <http://github.com/ceph/ceph/pull/7951>`_, Brad Hubbard)
+* cls_journal: fix -EEXIST checking (`pr#8413 <http://github.com/ceph/ceph/pull/8413>`_, runsisi)
+* cls_rbd: add guards for error cases (`issue#14316 <http://tracker.ceph.com/issues/14316>`_, `issue#14317 <http://tracker.ceph.com/issues/14317>`_, `pr#7165 <http://github.com/ceph/ceph/pull/7165>`_, xie xingguo)
+* cls_rbd: change object_map_update to return 0 on success, add logging (`pr#6467 <http://github.com/ceph/ceph/pull/6467>`_, Douglas Fuller)
+* cls_rbd: enable object map checksums for object_map_save (`issue#14280 <http://tracker.ceph.com/issues/14280>`_, `pr#7149 <http://github.com/ceph/ceph/pull/7149>`_, Douglas Fuller)
+* cls_rbd: fix -EEXIST checking in cls::rbd::image_set (`pr#8371 <http://github.com/ceph/ceph/pull/8371>`_, runsisi)
+* cls_rbd: fix the test for ceph-dencoder (`pr#7793 <http://github.com/ceph/ceph/pull/7793>`_, Kefu Chai)
+* cls_rbd: mirror_image_list should return global image id (`pr#8297 <http://github.com/ceph/ceph/pull/8297>`_, Jason Dillaman)
+* cls_rbd: mirroring directory (`issue#14419 <http://tracker.ceph.com/issues/14419>`_, `pr#7620 <http://github.com/ceph/ceph/pull/7620>`_, Josh Durgin)
+* cls_rbd: pass WILLNEED fadvise flags during object map update (`issue#15332 <http://tracker.ceph.com/issues/15332>`_, `pr#8380 <http://github.com/ceph/ceph/pull/8380>`_, Jason Dillaman)
+* cls_rbd: protect against excessively large object maps (`issue#15121 <http://tracker.ceph.com/issues/15121>`_, `pr#8099 <http://github.com/ceph/ceph/pull/8099>`_, Jason Dillaman)
+* cls_rbd: read_peers: update last_read on next cls_cxx_map_get_vals (`pr#8374 <http://github.com/ceph/ceph/pull/8374>`_, Mykola Golub)
+* cls/rgw: fix FTBFS (`pr#8142 <http://github.com/ceph/ceph/pull/8142>`_, Kefu Chai)
+* cls/rgw: fix use of timespan (`issue#15181 <http://tracker.ceph.com/issues/15181>`_, `pr#8212 <http://github.com/ceph/ceph/pull/8212>`_, Yehuda Sadeh)
+* cmake: add common/fs_types.cc to libcommon (`pr#7898 <http://github.com/ceph/ceph/pull/7898>`_, Orit Wasserman)
+* cmake: Add common/PluginRegistry.cc to CMakeLists.txt (`pr#6805 <http://github.com/ceph/ceph/pull/6805>`_, Pete Zaitcev)
+* cmake: Added new unittests to make check (`pr#7572 <http://github.com/ceph/ceph/pull/7572>`_, Ali Maredia)
+* cmake: Add ENABLE_GIT_VERSION to avoid rebuilding (`pr#7171 <http://github.com/ceph/ceph/pull/7171>`_, Kefu Chai)
+* cmake: add ErasureCode.cc to jerasure plugins (`pr#7808 <http://github.com/ceph/ceph/pull/7808>`_, Casey Bodley)
+* cmake: add FindOpenSSL.cmake (`pr#8106 <http://github.com/ceph/ceph/pull/8106>`_, Marcus Watts, Matt Benjamin)
+* cmake: add KernelDevice.cc to libos_srcs (`pr#7507 <http://github.com/ceph/ceph/pull/7507>`_, Kefu Chai)
+* cmake: add missing check for HAVE_EXECINFO_H (`pr#7270 <http://github.com/ceph/ceph/pull/7270>`_, Casey Bodley)
+* cmake: add missing librbd image_watcher sources (`issue#14823 <http://tracker.ceph.com/issues/14823>`_, `pr#7717 <http://github.com/ceph/ceph/pull/7717>`_, Casey Bodley)
+* cmake: add missing librbd/MirrorWatcher.cc and librd/ObjectWatcher.cc (`pr#8399 <http://github.com/ceph/ceph/pull/8399>`_, Orit Wasserman)
+* cmake: add nss as a suffix for pk11pub.h (`pr#6556 <http://github.com/ceph/ceph/pull/6556>`_, Samuel Just)
+* cmake: add rgw_basic_types.cc to librgw.a (`pr#6786 <http://github.com/ceph/ceph/pull/6786>`_, Orit Wasserman)
+* cmake: add StandardPolicy.cc to librbd (`pr#8368 <http://github.com/ceph/ceph/pull/8368>`_, Kefu Chai)
+* cmake: add TracepointProvider.cc to libcommon (`pr#6823 <http://github.com/ceph/ceph/pull/6823>`_, Orit Wasserman)
+* cmake: avoid false-positive LDAP header detect (`pr#8100 <http://github.com/ceph/ceph/pull/8100>`_, Matt Benjamin)
+* cmake: Build cython modules and change paths to bin/, lib/ (`pr#8351 <http://github.com/ceph/ceph/pull/8351>`_, John Spray, Ali Maredia)
+* cmake: check for libsnappy in default path also (`pr#7366 <http://github.com/ceph/ceph/pull/7366>`_, Kefu Chai)
+* cmake: cleanups and more features from automake (`pr#7103 <http://github.com/ceph/ceph/pull/7103>`_, Casey Bodley, Ali Maredia)
+* cmake: define STRERROR_R_CHAR_P for GNU-specific strerror_r (`pr#6751 <http://github.com/ceph/ceph/pull/6751>`_, Ilya Dryomov)
+* cmake: detect bzip2 and lz4 (`pr#7126 <http://github.com/ceph/ceph/pull/7126>`_, Kefu Chai)
+* cmake: feb5 (`pr#7541 <http://github.com/ceph/ceph/pull/7541>`_, Matt Benjamin)
+* cmake: fix build with bluestore (`pr#7099 <http://github.com/ceph/ceph/pull/7099>`_, John Spray)
+* cmake: fix files list (`pr#6539 <http://github.com/ceph/ceph/pull/6539>`_, Yehuda Sadeh)
+* cmake: fix mrun to handle cmake build structure (`pr#8237 <http://github.com/ceph/ceph/pull/8237>`_, Orit Wasserman)
+* cmake: fix paths to various EC source files (`pr#7748 <http://github.com/ceph/ceph/pull/7748>`_, Ali Maredia, Matt Benjamin)
+* cmake: fix the build of test_rados_api_list (`pr#8438 <http://github.com/ceph/ceph/pull/8438>`_, Kefu Chai)
+* cmake: fix the build of tests (`pr#7523 <http://github.com/ceph/ceph/pull/7523>`_, Kefu Chai)
+* cmake: fix the build on trusty (`pr#7249 <http://github.com/ceph/ceph/pull/7249>`_, Kefu Chai)
+* cmake: For CMake version <= 2.8.11, use LINK_PRIVATE and LINK_PUBLIC (`pr#7474 <http://github.com/ceph/ceph/pull/7474>`_, Tao Chang)
+* CMake: For CMake version <= 2.8.11, use LINK_PRIVATE (`pr#8422 <http://github.com/ceph/ceph/pull/8422>`_, Haomai Wang)
+* cmake: let ceph-client-debug link with tcmalloc (`pr#7314 <http://github.com/ceph/ceph/pull/7314>`_, Kefu Chai)
+* cmake: librbd and libjournal build fixes (`pr#6557 <http://github.com/ceph/ceph/pull/6557>`_, Ilya Dryomov)
+* cmake: made rocksdb an imported library (`pr#7131 <http://github.com/ceph/ceph/pull/7131>`_, Ali Maredia)
+* cmake: no need to run configure from run-cmake-check.sh (`pr#6959 <http://github.com/ceph/ceph/pull/6959>`_, Orit Wasserman)
+* cmake (`pr#7849 <http://github.com/ceph/ceph/pull/7849>`_, Ali Maredia)
+* cmake: Remove duplicate find_package libcurl line. (`pr#7972 <http://github.com/ceph/ceph/pull/7972>`_, Brad Hubbard)
+* cmake: support ccache via a WITH_CCACHE build option (`pr#6875 <http://github.com/ceph/ceph/pull/6875>`_, John Coyle)
+* cmake: test_build_libcephfs needs ${ALLOC_LIBS} (`pr#7300 <http://github.com/ceph/ceph/pull/7300>`_, Ali Maredia)
+* cmake: update for recent librbd changes (`pr#6715 <http://github.com/ceph/ceph/pull/6715>`_, John Spray)
+* cmake: update for recent rbd changes (`pr#6818 <http://github.com/ceph/ceph/pull/6818>`_, Mykola Golub)
+* cmake: Use uname instead of arch. (`pr#6358 <http://github.com/ceph/ceph/pull/6358>`_, John Coyle)
+* coc: fix typo in the apt-get command (`pr#6659 <http://github.com/ceph/ceph/pull/6659>`_, Chris Holcombe)
+* common: add descriptions to perfcounters (Kiseleva Alyona)
+* common: add generic plugin infrastructure (`pr#6696 <http://github.com/ceph/ceph/pull/6696>`_, Sage Weil)
+* common: add latency perf counter for finisher (`pr#6175 <http://github.com/ceph/ceph/pull/6175>`_, Xinze Chi)
+* common: add perf counter descriptions (Alyona Kiseleva)
+* common/address_help.cc: fix the leak in entity_addr_from_url() (`issue#14132 <http://tracker.ceph.com/issues/14132>`_, `pr#6987 <http://github.com/ceph/ceph/pull/6987>`_, Qiankun Zheng)
+* common: add thread names (`pr#5882 <http://github.com/ceph/ceph/pull/5882>`_, Igor Podoski)
+* common: add zlib compression plugin (`pr#7437 <http://github.com/ceph/ceph/pull/7437>`_, Alyona Kiseleva, Kiseleva Alyona)
+* common: admin socket commands for tcmalloc heap get/set operations (`pr#7512 <http://github.com/ceph/ceph/pull/7512>`_, Samuel Just)
+* common: ake ceph_time clocks work under BSD (`pr#7340 <http://github.com/ceph/ceph/pull/7340>`_, Adam C. Emerson)
+* common: allow enable/disable of optracker at runtime (`pr#5168 <http://github.com/ceph/ceph/pull/5168>`_, Jianpeng Ma)
+* common: Allow OPT_INT settings with negative values (`issue#13829 <http://tracker.ceph.com/issues/13829>`_, `pr#7390 <http://github.com/ceph/ceph/pull/7390>`_, Brad Hubbard, Kefu Chai)
+* common: assert: abort() rather than throw (`pr#6804 <http://github.com/ceph/ceph/pull/6804>`_, Adam C. Emerson)
+* common: assert: __STRING macro is not defined by musl libc. (`pr#6210 <http://github.com/ceph/ceph/pull/6210>`_, John Coyle)
+* common/bit_vector: use hard-coded value for block size (`issue#14747 <http://tracker.ceph.com/issues/14747>`_, `pr#7610 <http://github.com/ceph/ceph/pull/7610>`_, Jason Dillaman)
+* common: buffer: add cached_crc and cached_crc_adjust counts to perf dump (`pr#6535 <http://github.com/ceph/ceph/pull/6535>`_, Ning Yao)
+* common: buffer/assert minor fixes (`pr#6990 <http://github.com/ceph/ceph/pull/6990>`_, Matt Benjamin)
+* common: bufferlist performance tuning (Piotr Dalek, Sage Weil)
+* common: buffer: put a guard for stat() syscall during read_file (`pr#7956 <http://github.com/ceph/ceph/pull/7956>`_, xie xingguo)
+* common: buffer: remove unneeded list destructor (`pr#6456 <http://github.com/ceph/ceph/pull/6456>`_, Michal Jarzabek)
+* common/buffer: replace RWLock with spinlocks (`pr#7294 <http://github.com/ceph/ceph/pull/7294>`_, Piotr Dałek)
+* common/ceph_context.cc:fix order of initialisers (`pr#6838 <http://github.com/ceph/ceph/pull/6838>`_, Michal Jarzabek)
+* common: change the type of counter total/unhealthy_workers (`pr#7254 <http://github.com/ceph/ceph/pull/7254>`_, Guang Yang)
+* common: default cluster name to config file prefix (`pr#7364 <http://github.com/ceph/ceph/pull/7364>`_, Javen Wu)
+* common: Deprecate or free up a bunch of feature bits (`pr#8214 <http://github.com/ceph/ceph/pull/8214>`_, Samuel Just)
+* common: detect overflow of int config values (#11484 Kefu Chai)
+* common: Do not use non-portable constants in mutex_debug (`pr#7766 <http://github.com/ceph/ceph/pull/7766>`_, Adam C. Emerson)
+* common: don't reverse hobject_t hash bits when zero (`pr#6653 <http://github.com/ceph/ceph/pull/6653>`_, Piotr Dałek)
+* common: fix bit_vector extent calc (#12611 Jason Dillaman)
+* common: fix json parsing of utf8 (#7387 Tim Serong)
+* common: fix leak of pthread_mutexattr (#11762 Ketor Meng)
+* common: fix LTTNG vs fork issue (Josh Durgin)
+* common: fix OpTracker age histogram calculation (`pr#5065 <http://github.com/ceph/ceph/pull/5065>`_, Zhiqiang Wang)
+* common: fix race during optracker switches between enabled/disabled mode (`pr#8330 <http://github.com/ceph/ceph/pull/8330>`_, xie xingguo)
+* common: fix reset max in Throttle using perf reset command (`issue#13517 <http://tracker.ceph.com/issues/13517>`_, `pr#6300 <http://github.com/ceph/ceph/pull/6300>`_, Xinze Chi)
+* common: fix throttle max change (Henry Chang)
+* common: fix time_t cast in decode (`issue#15330 <http://tracker.ceph.com/issues/15330>`_, `pr#8419 <http://github.com/ceph/ceph/pull/8419>`_, Adam C. Emerson)
+* common/Formatter: avoid newline if there is no output (`pr#5351 <http://github.com/ceph/ceph/pull/5351>`_, Aran85)
+* common: improve shared_cache and simple_cache efficiency with hash table (`pr#6909 <http://github.com/ceph/ceph/pull/6909>`_, Ning Yao)
+* common/lockdep: increase max lock names (`pr#6961 <http://github.com/ceph/ceph/pull/6961>`_, Sage Weil)
+* common: log: Assign LOG_DEBUG priority to syslog calls (`issue#13993 <http://tracker.ceph.com/issues/13993>`_, `pr#6815 <http://github.com/ceph/ceph/pull/6815>`_, Brad Hubbard)
+* common: log: predict log message buffer allocation size (`pr#6641 <http://github.com/ceph/ceph/pull/6641>`_, Adam Kupczyk)
+* common: make mutex more efficient
+* common: make work queue addition/removal thread safe (#12662 Jason Dillaman)
+* common/MemoryModel: Added explicit feature check for mallinfo(). (`pr#6252 <http://github.com/ceph/ceph/pull/6252>`_, John Coyle)
+* common: new timekeeping common code, and Objecter conversion (`pr#5782 <http://github.com/ceph/ceph/pull/5782>`_, Adam C. Emerson)
+* common/obj_bencher.cc: bump the precision of bandwidth field (`pr#8021 <http://github.com/ceph/ceph/pull/8021>`_, Piotr Dałek)
+* common/obj_bencher.cc: faster object name generation (`pr#7863 <http://github.com/ceph/ceph/pull/7863>`_, Piotr Dałek)
+* common/obj_bencher.cc: fix verification crashing when there's no objects (`pr#5853 <http://github.com/ceph/ceph/pull/5853>`_, Piotr Dałek)
+* common/obj_bencher.cc: make verify error fatal (`issue#14971 <http://tracker.ceph.com/issues/14971>`_, `pr#7897 <http://github.com/ceph/ceph/pull/7897>`_, Piotr Dałek)
+* common: optimize debug logging code (`pr#6441 <http://github.com/ceph/ceph/pull/6441>`_, Adam Kupczyk)
+* common: optimize debug logging (`pr#6307 <http://github.com/ceph/ceph/pull/6307>`_, Adam Kupczyk)
+* common: optracker improvements (Zhiqiang Wang, Jianpeng Ma)
+* common/page.cc: _page_mask has too many bits (`pr#7588 <http://github.com/ceph/ceph/pull/7588>`_, Dan Mick)
+* common: perf counter for bufferlist history total alloc (`pr#6198 <http://github.com/ceph/ceph/pull/6198>`_, Xinze Chi)
+* common: PriorityQueue tests (Kefu Chai)
+* common: reduce CPU usage by making stringstream in stringify function thread local (`pr#6543 <http://github.com/ceph/ceph/pull/6543>`_, Evgeniy Firsov)
+* common: re-enable backtrace support (`pr#6771 <http://github.com/ceph/ceph/pull/6771>`_, Jason Dillaman)
+* common: set thread name from correct thread (`pr#7845 <http://github.com/ceph/ceph/pull/7845>`_, Igor Podoski)
+* common: signal_handler: added support for using reentrant strsignal() implementations vs. sys_siglist[] (`pr#6796 <http://github.com/ceph/ceph/pull/6796>`_, John Coyle)
+* common: snappy decompressor may assert when handling segmented input bufferlist (`issue#14400 <http://tracker.ceph.com/issues/14400>`_, `pr#7268 <http://github.com/ceph/ceph/pull/7268>`_, Igor Fedotov)
+* common: some async compression infrastructure (Haomai Wang)
+* common/str_map: cleanup: replaced get_str_map() function overloading by using default parameters for delimiters (`pr#7266 <http://github.com/ceph/ceph/pull/7266>`_, Sahithi R V)
+* common/strtol.cc: fix the coverity warnings (`pr#7967 <http://github.com/ceph/ceph/pull/7967>`_, Kefu Chai)
+* common: SubProcess: Avoid buffer corruption when calling err() (`issue#15011 <http://tracker.ceph.com/issues/15011>`_, `pr#8054 <http://github.com/ceph/ceph/pull/8054>`_, Erwan Velu)
+* common: SubProcess: fix multiple definition bug (`pr#6790 <http://github.com/ceph/ceph/pull/6790>`_, Yunchuan Wen)
+* common: Thread: move copy constructor and assignment op (`pr#5133 <http://github.com/ceph/ceph/pull/5133>`_, Michal Jarzabek)
+* common: time: have skewing-now call non-skewing now (`pr#7466 <http://github.com/ceph/ceph/pull/7466>`_, Adam C. Emerson)
+* common/TrackedOp: fix inaccurate counting for slow requests (`issue#14804 <http://tracker.ceph.com/issues/14804>`_, `pr#7690 <http://github.com/ceph/ceph/pull/7690>`_, xie xingguo)
+* common: unit test for interval_set implementations (`pr#6 <http://github.com/ceph/ceph/pull/6>`_, Igor Fedotov)
+* common: use namespace instead of subclasses for buffer (`pr#6686 <http://github.com/ceph/ceph/pull/6686>`_, Michal Jarzabek)
+* common: various fixes from SCA runs (`pr#7680 <http://github.com/ceph/ceph/pull/7680>`_, Danny Al-Gaaf)
+* common: WorkQueue: new PointerWQ base class for ContextWQ (`issue#13636 <http://tracker.ceph.com/issues/13636>`_, `pr#6525 <http://github.com/ceph/ceph/pull/6525>`_, Jason Dillaman)
+* compat: use prefixed typeof extension (`pr#6216 <http://github.com/ceph/ceph/pull/6216>`_, John Coyle)
+* config: add $data_dir/config to config search path (`pr#7377 <http://github.com/ceph/ceph/pull/7377>`_, Sage Weil)
+* config: complains when a setting is not tracked (`issue#11692 <http://tracker.ceph.com/issues/11692>`_, `pr#7085 <http://github.com/ceph/ceph/pull/7085>`_, Kefu Chai)
+* config: fix osd_crush_initial_weight (`pr#7975 <http://github.com/ceph/ceph/pull/7975>`_, You Ji)
+* config: increase default async op threads (`pr#7802 <http://github.com/ceph/ceph/pull/7802>`_, Piotr Dałek)
+* config_opts: disable filestore throttle soft backoff by default (`pr#8265 <http://github.com/ceph/ceph/pull/8265>`_, Samuel Just)
+* configure.ac: boost_iostreams is required, not optional (`pr#7816 <http://github.com/ceph/ceph/pull/7816>`_, Hector Martin)
+* configure.ac: macro fix (`pr#6769 <http://github.com/ceph/ceph/pull/6769>`_, Igor Podoski)
+* configure.ac: make "--with-librocksdb-static" default to 'check' (`issue#14463 <http://tracker.ceph.com/issues/14463>`_, `pr#7317 <http://github.com/ceph/ceph/pull/7317>`_, Dan Mick)
+* configure.ac: update help strings for cython (`pr#7856 <http://github.com/ceph/ceph/pull/7856>`_, Josh Durgin)
+* configure: Add -D_LARGEFILE64_SOURCE to Linux build. (`pr#8402 <http://github.com/ceph/ceph/pull/8402>`_, Ira Cooper)
+* configure: detect bz2 and lz4 (`issue#13850 <http://tracker.ceph.com/issues/13850>`_, `issue#13981 <http://tracker.ceph.com/issues/13981>`_, `pr#7030 <http://github.com/ceph/ceph/pull/7030>`_, Kefu Chai)
+* correct radosgw-admin command (`pr#7006 <http://github.com/ceph/ceph/pull/7006>`_, YankunLi)
+* crush: add --check to validate dangling names, max osd id (Kefu Chai)
+* crush: add chooseleaf_stable tunable (`pr#6572 <http://github.com/ceph/ceph/pull/6572>`_, Sangdi Xu, Sage Weil)
+* crush: add safety assert (`issue#14496 <http://tracker.ceph.com/issues/14496>`_, `pr#7344 <http://github.com/ceph/ceph/pull/7344>`_, songbaisen)
+* crush: cleanup, sync with kernel (Ilya Dryomov)
+* crush: clean up whitespace removal (`issue#14302 <http://tracker.ceph.com/issues/14302>`_, `pr#7157 <http://github.com/ceph/ceph/pull/7157>`_, songbaisen)
+* crush/CrushTester: check for overlapped rules (`pr#7139 <http://github.com/ceph/ceph/pull/7139>`_, Kefu Chai)
+* crush/CrushTester: workaround a bug in boost::icl (`pr#7560 <http://github.com/ceph/ceph/pull/7560>`_, Kefu Chai)
+* crush: fix cli tests for new crush tunables (`pr#8107 <http://github.com/ceph/ceph/pull/8107>`_, Sage Weil)
+* crush: fix crash from invalid 'take' argument (#11602 Shiva Rkreddy, Sage Weil)
+* crush: fix divide-by-2 in straw2 (#11357 Yann Dupont, Sage Weil)
+* crush: fix error log (`pr#8430 <http://github.com/ceph/ceph/pull/8430>`_, Wei Jin)
+* crush: fix has_v4_buckets (#11364 Sage Weil)
+* crush: fix subtree base weight on adjust_subtree_weight (#11855 Sage Weil)
+* crush: fix typo (`pr#8518 <http://github.com/ceph/ceph/pull/8518>`_, Wei Jin)
+* crush: reply quickly from get_immediate_parent (`issue#14334 <http://tracker.ceph.com/issues/14334>`_, `pr#7181 <http://github.com/ceph/ceph/pull/7181>`_, song baisen)
+* crush: respect default replicated ruleset config on map creation (Ilya Dryomov)
+* crushtool: Don't crash when called on a file that isn't a crushmap (`issue#8286 <http://tracker.ceph.com/issues/8286>`_, `pr#8038 <http://github.com/ceph/ceph/pull/8038>`_, Brad Hubbard)
+* crushtool: fix order of operations, usage (Sage Weil)
+* crushtool: improve usage/tip messages (`pr#7142 <http://github.com/ceph/ceph/pull/7142>`_, xie xingguo)
+* crushtool: set type 0 name "device" for --build option (`pr#6824 <http://github.com/ceph/ceph/pull/6824>`_, Sangdi Xu)
+* crush: update tunable docs.  change default profile to jewel (`pr#7964 <http://github.com/ceph/ceph/pull/7964>`_, Sage Weil)
+* crush: validate bucket id before indexing buckets array (`issue#13477 <http://tracker.ceph.com/issues/13477>`_, `pr#6246 <http://github.com/ceph/ceph/pull/6246>`_, Sage Weil)
+* crypto: fix NSS leak (Jason Dillaman)
+* crypto: fix unbalanced init/shutdown (#12598 Zheng Yan)
+* deb: fix rest-bench-dbg and ceph-test-dbg dependendies (Ken Dreyer)
+* debian/changelog: Remove stray 'v' in version (`pr#7936 <http://github.com/ceph/ceph/pull/7936>`_, Dan Mick)
+* debian/changelog: Remove stray 'v' in version (`pr#7938 <http://github.com/ceph/ceph/pull/7938>`_, Dan Mick)
+* debian: include cpio in build-requiers (`pr#7533 <http://github.com/ceph/ceph/pull/7533>`_, Rémi BUISSON)
+* debian: minor package reorg (Ken Dreyer)
+* debian: package librgw_file* tests (`pr#7930 <http://github.com/ceph/ceph/pull/7930>`_, Ken Dreyer)
+* debian: packaging fixes for jewel (`pr#7807 <http://github.com/ceph/ceph/pull/7807>`_, Ken Dreyer, Ali Maredia)
+* debian/rpm split servers (`issue#10587 <http://tracker.ceph.com/issues/10587>`_, `pr#7746 <http://github.com/ceph/ceph/pull/7746>`_, Ken Dreyer)
+* debian/rules: put init-ceph in /etc/init.d/ceph, not ceph-base (`issue#15329 <http://tracker.ceph.com/issues/15329>`_, `pr#8406 <http://github.com/ceph/ceph/pull/8406>`_, Dan Mick)
+* deb, rpm: move ceph-objectstore-tool to ceph (Ken Dreyer)
+* doc: add ceph-detect-init(8) source to dist tarball (`pr#7933 <http://github.com/ceph/ceph/pull/7933>`_, Ken Dreyer)
+* doc: add cinder backend section to rbd-openstack.rst (`pr#7923 <http://github.com/ceph/ceph/pull/7923>`_, RustShen)
+* doc: adding "--allow-shrink" in decreasing the size of the rbd block to distinguish from the increasing option (`pr#7020 <http://github.com/ceph/ceph/pull/7020>`_, Yehua)
+* doc: add orphans commands to radosgw-admin(8) (`issue#14637 <http://tracker.ceph.com/issues/14637>`_, `pr#7518 <http://github.com/ceph/ceph/pull/7518>`_, Ken Dreyer)
+* doc: add v0.80.11 to the release timeline (`pr#6658 <http://github.com/ceph/ceph/pull/6658>`_, Loic Dachary)
+* doc: admin/build-doc: add lxml dependencies on debian (`pr#6610 <http://github.com/ceph/ceph/pull/6610>`_, Ken Dreyer)
+* doc: admin/build-doc: make paths absolute (`pr#7119 <http://github.com/ceph/ceph/pull/7119>`_, Dan Mick)
+* doc: amend Fixes instructions in SubmittingPatches (`pr#8312 <http://github.com/ceph/ceph/pull/8312>`_, Nathan Cutler)
+* doc: amend the rados.8 (`pr#7251 <http://github.com/ceph/ceph/pull/7251>`_, Kefu Chai)
+* doc/architecture.rst: remove redundant word "across" (`pr#8179 <http://github.com/ceph/ceph/pull/8179>`_, Zhao Junwang)
+* doc/cephfs/posix: update (`pr#6922 <http://github.com/ceph/ceph/pull/6922>`_, Sage Weil)
+* doc: Clarify usage on starting single osd/mds/mon. (`pr#7641 <http://github.com/ceph/ceph/pull/7641>`_, Patrick Donnelly)
+* doc: CodingStyle: fix broken URLs (`pr#6733 <http://github.com/ceph/ceph/pull/6733>`_, Kefu Chai)
+* doc: correct typo 'restared' to 'restarted' (`pr#6734 <http://github.com/ceph/ceph/pull/6734>`_, Yilong Zhao)
+* doc: detailed description of bugfixing workflow (`pr#7941 <http://github.com/ceph/ceph/pull/7941>`_, Nathan Cutler)
+* doc/dev: add "Deploy a cluster for manual testing" section (`issue#15218 <http://tracker.ceph.com/issues/15218>`_, `pr#8228 <http://github.com/ceph/ceph/pull/8228>`_, Nathan Cutler)
+* doc/dev: add section on interrupting a running suite (`pr#8116 <http://github.com/ceph/ceph/pull/8116>`_, Nathan Cutler)
+* doc/dev: continue writing Testing in the cloud chapter (`pr#7960 <http://github.com/ceph/ceph/pull/7960>`_, Nathan Cutler)
+* doc: dev: document ceph-qa-suite (`pr#6955 <http://github.com/ceph/ceph/pull/6955>`_, Loic Dachary)
+* doc/dev/index: refactor/reorg (`pr#6792 <http://github.com/ceph/ceph/pull/6792>`_, Nathan Cutler)
+* doc/dev/index.rst: begin writing Contributing to Ceph (`pr#6727 <http://github.com/ceph/ceph/pull/6727>`_, Nathan Cutler)
+* doc/dev/index.rst: fix headings (`pr#6780 <http://github.com/ceph/ceph/pull/6780>`_, Nathan Cutler)
+* doc/dev: integrate testing into the narrative (`pr#7946 <http://github.com/ceph/ceph/pull/7946>`_, Nathan Cutler)
+* doc: dev: introduction to tests (`pr#6910 <http://github.com/ceph/ceph/pull/6910>`_, Loic Dachary)
+* doc/dev: various refinements (`pr#7954 <http://github.com/ceph/ceph/pull/7954>`_, Nathan Cutler)
+* doc: docuemnt object corpus generation (#11099 Alexis Normand)
+* doc: document "readforward" and "readproxy" cache mode (`pr#7023 <http://github.com/ceph/ceph/pull/7023>`_, Kefu Chai)
+* doc: document region hostnames (Robin H. Johnson)
+* doc: download GPG key from download.ceph.com (`issue#13603 <http://tracker.ceph.com/issues/13603>`_, `pr#6384 <http://github.com/ceph/ceph/pull/6384>`_, Ken Dreyer)
+* doc: draft notes for jewel (`pr#8211 <http://github.com/ceph/ceph/pull/8211>`_, Loic Dachary, Sage Weil)
+* doc: file must be empty when writing layout fields of file use "setfattr" (`pr#6848 <http://github.com/ceph/ceph/pull/6848>`_, Cilang Zhao)
+* doc: fix 0.94.4 and 0.94.5 ordering (`pr#7763 <http://github.com/ceph/ceph/pull/7763>`_, Loic Dachary)
+* doc: Fixed incorrect name of a "List Multipart Upload Parts" Response Entity (`issue#14003 <http://tracker.ceph.com/issues/14003>`_, `pr#6829 <http://github.com/ceph/ceph/pull/6829>`_, Lenz Grimmer)
+* doc: Fixes a CRUSH map step take argument (`pr#7327 <http://github.com/ceph/ceph/pull/7327>`_, Ivan Grcic)
+* doc: Fixes a spelling error (`pr#6705 <http://github.com/ceph/ceph/pull/6705>`_, Jeremy Qian)
+* doc: Fixes headline different font size and type (`pr#8328 <http://github.com/ceph/ceph/pull/8328>`_, scienceluo)
+* doc: fix gender neutrality (Alexandre Maragone)
+* doc: fixing image in section ERASURE CODING (`pr#7298 <http://github.com/ceph/ceph/pull/7298>`_, Rachana Patel)
+* doc: fix install doc (#10957 Kefu Chai)
+* doc: fix misleading configuration guide on cache tiering (`pr#7000 <http://github.com/ceph/ceph/pull/7000>`_, Yuan Zhou)
+* doc: fix "mon osd down out subtree limit" option name (`pr#7164 <http://github.com/ceph/ceph/pull/7164>`_, François Lafont)
+* doc: fix outdated content in cache tier (`pr#6272 <http://github.com/ceph/ceph/pull/6272>`_, Yuan Zhou)
+* doc: fix S3 C# example (`pr#7027 <http://github.com/ceph/ceph/pull/7027>`_, Dunrong Huang)
+* doc: fix sphinx issues (Kefu Chai)
+* doc: fix typo, duplicated content etc. for Jewel release notes (`pr#8342 <http://github.com/ceph/ceph/pull/8342>`_, xie xingguo)
+* doc: fix typo in cephfs/quota (`pr#6745 <http://github.com/ceph/ceph/pull/6745>`_, Drunkard Zhang)
+* doc: fix typo, indention etc. (`pr#7829 <http://github.com/ceph/ceph/pull/7829>`_, xie xingguo)
+* doc: fix typo in developer guide (`pr#6943 <http://github.com/ceph/ceph/pull/6943>`_, Nathan Cutler)
+* doc: fix typo (`pr#7004 <http://github.com/ceph/ceph/pull/7004>`_, tianqing)
+* doc: fix wrong type of hyphen (`pr#8252 <http://github.com/ceph/ceph/pull/8252>`_, xie xingguo)
+* doc: initial draft of RBD mirroring admin documentation (`issue#15041 <http://tracker.ceph.com/issues/15041>`_, `pr#8169 <http://github.com/ceph/ceph/pull/8169>`_, Jason Dillaman)
+* doc: INSTALL redirect to online documentation (`pr#6749 <http://github.com/ceph/ceph/pull/6749>`_, Loic Dachary)
+* doc: little improvements for troubleshooting scrub issues (`pr#6827 <http://github.com/ceph/ceph/pull/6827>`_, Mykola Golub)
+* doc: man page updates (Kefu Chai)
+* doc: mds data structure docs (Yan, Zheng)
+* doc: misc updates (Francois Lafont, Ken Dreyer, Kefu Chai, Owen Synge, Gael Fenet-Garde, Loic Dachary, Yannick Atchy-Dalama, Jiaying Ren, Kevin Caradant, Robert Maxime, Nicolas Yong, Germain Chipaux, Arthur Gorjux, Gabriel Sentucq, Clement Lebrun, Jean-Remi Deveaux, Clair Massot, Robin Tang, Thomas Laumondais, Jordan Dorne, Yuan Zhou, Valentin Thomas, Pierre Chaumont, Benjamin Troquereau, Benjamin Sesia, Vikhyat Umrao, Nilamdyuti Goswami, Vartika Rai, Florian Haas, Loic Dachary, Simon Guinot, Andy Allan, Alistair Israel, Ken Dreyer, Robin Rehu, Lee Revell, Florian Marsylle, Thomas Johnson, Bosse Klykken, Travis Rhoden, Ian Kelling)
+* doc: Modified a note section in rbd-snapshot doc. (`pr#6908 <http://github.com/ceph/ceph/pull/6908>`_, Nilamdyuti Goswami)
+* doc: note that cephfs auth stuff is new in jewel (`pr#6858 <http://github.com/ceph/ceph/pull/6858>`_, John Spray)
+* doc: osd-config Add Configuration Options for op queue. (`pr#7837 <http://github.com/ceph/ceph/pull/7837>`_, Robert LeBlanc)
+* doc: osd: s/schedued/scheduled/ (`pr#6872 <http://github.com/ceph/ceph/pull/6872>`_, Loic Dachary)
+* doc/rados/api/librados-intro.rst: fix typo (`pr#7879 <http://github.com/ceph/ceph/pull/7879>`_, xie xingguo)
+* doc/rados/operations/crush: fix the formatting (`pr#8306 <http://github.com/ceph/ceph/pull/8306>`_, Kefu Chai)
+* doc: release-notes: draft v0.80.11 release notes (`pr#6374 <http://github.com/ceph/ceph/pull/6374>`_, Loic Dachary)
+* doc: release-notes: draft v10.0.0 release notes (`pr#6666 <http://github.com/ceph/ceph/pull/6666>`_, Loic Dachary)
+* doc/release-notes: fix indents (`pr#8345 <http://github.com/ceph/ceph/pull/8345>`_, Kefu Chai)
+* doc/release-notes: v9.1.0 (`pr#6281 <http://github.com/ceph/ceph/pull/6281>`_, Loic Dachary)
+* doc/releases-notes: fix build error (`pr#6483 <http://github.com/ceph/ceph/pull/6483>`_, Kefu Chai)
+* doc: Remove Ceph Monitors do lots of fsync() (`issue#15288 <http://tracker.ceph.com/issues/15288>`_, `pr#8327 <http://github.com/ceph/ceph/pull/8327>`_, Vikhyat Umrao)
+* doc: remove redundant space in ceph-authtool/monmaptool doc (`pr#7244 <http://github.com/ceph/ceph/pull/7244>`_, Jiaying Ren)
+* doc: remove toctree items under Create CephFS (`pr#6241 <http://github.com/ceph/ceph/pull/6241>`_, Jevon Qiao)
+* doc: remove unnecessary period in headline (`pr#6775 <http://github.com/ceph/ceph/pull/6775>`_, Marc Koderer)
+* doc: rename the "Create a Ceph User" section and add verbage about… (`issue#13502 <http://tracker.ceph.com/issues/13502>`_, `pr#6297 <http://github.com/ceph/ceph/pull/6297>`_, ritz303)
+* doc: revise SubmittingPatches (`pr#7292 <http://github.com/ceph/ceph/pull/7292>`_, Kefu Chai)
+* doc: rgw admin uses "region list" not "regions list" (`pr#8517 <http://github.com/ceph/ceph/pull/8517>`_, Kris Jurka)
+* doc: rgw explain keystone's verify ssl switch (`pr#7862 <http://github.com/ceph/ceph/pull/7862>`_, Abhishek Lekshmanan)
+* doc: rgw: port changes from downstream to upstream (`pr#7264 <http://github.com/ceph/ceph/pull/7264>`_, Bara Ancincova)
+* doc: rgw_region_root_pool option should be in [global] (`issue#15244 <http://tracker.ceph.com/issues/15244>`_, `pr#8271 <http://github.com/ceph/ceph/pull/8271>`_, Vikhyat Umrao)
+* doc: rst style fix for pools document (`pr#6816 <http://github.com/ceph/ceph/pull/6816>`_, Drunkard Zhang)
+* doc: script and guidelines for mirroring Ceph (`pr#7384 <http://github.com/ceph/ceph/pull/7384>`_, Wido den Hollander)
+* docs: Fix styling of newly added mirror docs (`pr#6127 <http://github.com/ceph/ceph/pull/6127>`_, Wido den Hollander)
+* doc: small fixes (`pr#7813 <http://github.com/ceph/ceph/pull/7813>`_, xiexingguo)
+* doc: standardize @param (not @parma, @parmam, @params) (`pr#7714 <http://github.com/ceph/ceph/pull/7714>`_, Nathan Cutler)
+* doc: SubmittingPatches: there is no next; only jewel (`pr#6811 <http://github.com/ceph/ceph/pull/6811>`_, Nathan Cutler)
+* doc: swift tempurls (#10184 Abhishek Lekshmanan)
+* doc: switch doxygen integration back to breathe (#6115 Kefu Chai)
+* doc, tests: update all http://ceph.com/ to download.ceph.com (`pr#6435 <http://github.com/ceph/ceph/pull/6435>`_, Alfredo Deza)
+* doc: Update ceph-disk manual page with new feature deactivate/destroy. (`pr#6637 <http://github.com/ceph/ceph/pull/6637>`_, Vicente Cheng)
+* doc: Updated CloudStack RBD documentation (`pr#8308 <http://github.com/ceph/ceph/pull/8308>`_, Wido den Hollander)
+* doc: update doc for with new pool settings (`pr#5951 <http://github.com/ceph/ceph/pull/5951>`_, Guang Yang)
+* doc: Updated the rados command man page to include the --run-name opt… (`issue#12899 <http://tracker.ceph.com/issues/12899>`_, `pr#5900 <http://github.com/ceph/ceph/pull/5900>`_, ritz303)
+* doc: update infernalis release notes (`pr#6575 <http://github.com/ceph/ceph/pull/6575>`_, vasukulkarni)
+* doc: Update list of admin/build-doc dependencies (`issue#14070 <http://tracker.ceph.com/issues/14070>`_, `pr#6934 <http://github.com/ceph/ceph/pull/6934>`_, Nathan Cutler)
+* doc: update radosgw-admin example (`pr#6256 <http://github.com/ceph/ceph/pull/6256>`_, YankunLi)
+* doc: update release schedule docs (Loic Dachary)
+* doc: update the OS recommendations for newer Ceph releases (`pr#6355 <http://github.com/ceph/ceph/pull/6355>`_, ritz303)
+* doc: use 'ceph auth get-or-create' for creating RGW keyring (`pr#6930 <http://github.com/ceph/ceph/pull/6930>`_, Wido den Hollander)
+* doc: very basic doc on mstart (`pr#8207 <http://github.com/ceph/ceph/pull/8207>`_, Abhishek Lekshmanan)
+* drop envz.h includes (`pr#6285 <http://github.com/ceph/ceph/pull/6285>`_, John Coyle)
+* erasure-code: cleanup (Kefu Chai)
+* erasure-code: improve tests (Loic Dachary)
+* erasure-code: shec: fix recovery bugs (Takanori Nakao, Shotaro Kawaguchi)
+* erasure-code: update ISA-L to 2.13 (Yuan Zhou)
+* fix FTBFS introduced by d0af316 (`pr#7792 <http://github.com/ceph/ceph/pull/7792>`_, Kefu Chai)
+* fix: use right init_flags to finish CephContext (`pr#6549 <http://github.com/ceph/ceph/pull/6549>`_, Yunchuan Wen)
+* fs: be more careful about the "mds setmap" command to prevent breakage (`issue#14380 <http://tracker.ceph.com/issues/14380>`_, `pr#7262 <http://github.com/ceph/ceph/pull/7262>`_, Yan, Zheng)
+* ghobject_t: use # instead of ! as a separator (`pr#8055 <http://github.com/ceph/ceph/pull/8055>`_, Sage Weil)
+* global: do not start two daemons with a single pid-file (`issue#13422 <http://tracker.ceph.com/issues/13422>`_, `pr#7075 <http://github.com/ceph/ceph/pull/7075>`_, shun song)
+* global: do not start two daemons with a single pid-file (part 2) (`issue#13422 <http://tracker.ceph.com/issues/13422>`_, `pr#7463 <http://github.com/ceph/ceph/pull/7463>`_, Loic Dachary)
+* global/global_init: expand metavariables in setuser_match_path (`issue#15365 <http://tracker.ceph.com/issues/15365>`_, `pr#8433 <http://github.com/ceph/ceph/pull/8433>`_, Sage Weil)
+* global/signal_handler: print thread name in signal handler (`pr#8177 <http://github.com/ceph/ceph/pull/8177>`_, Jianpeng Ma)
+* gmock: switch to submodule (Danny Al-Gaaf, Loic Dachary)
+* hadoop: add terasort test (Noah Watkins)
+* helgrind: additional race conditionslibrbd: journal replay should honor inter-event dependencies (`pr#7274 <http://github.com/ceph/ceph/pull/7274>`_, Jason Dillaman)
+* helgrind: fix real (and imaginary) race conditions (`issue#14163 <http://tracker.ceph.com/issues/14163>`_, `pr#7208 <http://github.com/ceph/ceph/pull/7208>`_, Jason Dillaman)
+* include/encoding: do not try to be clever with list encoding (`pr#7913 <http://github.com/ceph/ceph/pull/7913>`_, Sage Weil)
+* init-ceph: do umount when the path exists. (`pr#6866 <http://github.com/ceph/ceph/pull/6866>`_, Xiaoxi Chen)
+* init-ceph.in: allow case-insensitive true in `osd crush update on start' (`pr#7943 <http://github.com/ceph/ceph/pull/7943>`_, Eric Cook)
+* init-ceph.in: skip ceph-disk if it is not present (`issue#10587 <http://tracker.ceph.com/issues/10587>`_, `pr#7286 <http://github.com/ceph/ceph/pull/7286>`_, Ken Dreyer)
+* init-ceph: use getopt to make option processing more flexible (`issue#3015 <http://tracker.ceph.com/issues/3015>`_, `pr#6089 <http://github.com/ceph/ceph/pull/6089>`_, Nathan Cutler)
+* init-radosgw: merge with sysv version; fix enumeration (Sage Weil)
+* java: fix libcephfs bindings (Noah Watkins)
+* journal: async methods to (un)register and update client (`pr#7832 <http://github.com/ceph/ceph/pull/7832>`_, Mykola Golub)
+* journal: disconnect watch after watch error (`issue#14168 <http://tracker.ceph.com/issues/14168>`_, `pr#7113 <http://github.com/ceph/ceph/pull/7113>`_, Jason Dillaman)
+* journal: fire replay complete event after reading last object (`issue#13924 <http://tracker.ceph.com/issues/13924>`_, `pr#6762 <http://github.com/ceph/ceph/pull/6762>`_, Jason Dillaman)
+* journal: fix final result for JournalTrimmer::C_RemoveSet (`pr#8516 <http://github.com/ceph/ceph/pull/8516>`_, runsisi)
+* journal: fix race condition between Future and journal shutdown (`issue#15364 <http://tracker.ceph.com/issues/15364>`_, `pr#8477 <http://github.com/ceph/ceph/pull/8477>`_, Jason Dillaman)
+* journal: flush commit position on metadata shutdown (`pr#7385 <http://github.com/ceph/ceph/pull/7385>`_, Mykola Golub)
+* journal: improve commit position tracking (`pr#7776 <http://github.com/ceph/ceph/pull/7776>`_, Jason Dillaman)
+* journal: incremental improvements and fixes (`pr#6552 <http://github.com/ceph/ceph/pull/6552>`_, Mykola Golub)
+* journal: prevent race injecting new records into overflowed object (`issue#15202 <http://tracker.ceph.com/issues/15202>`_, `pr#8220 <http://github.com/ceph/ceph/pull/8220>`_, Jason Dillaman)
+* journal: reset commit_position_task_ctx pointer after task complete (`pr#7480 <http://github.com/ceph/ceph/pull/7480>`_, Mykola Golub)
+* journal: re-use common threads between journalers (`pr#7906 <http://github.com/ceph/ceph/pull/7906>`_, Jason Dillaman)
+* journal: support replaying beyond skipped splay objects (`pr#6687 <http://github.com/ceph/ceph/pull/6687>`_, Jason Dillaman)
+* krbd: remove deprecated --quiet param from udevadm (`issue#13560 <http://tracker.ceph.com/issues/13560>`_, `pr#6394 <http://github.com/ceph/ceph/pull/6394>`_, Jason Dillaman)
+* kv: fix bug in kv key optimization (`pr#6511 <http://github.com/ceph/ceph/pull/6511>`_, Sage Weil)
+* kv: implement value_as_ptr() and use it in .get() (`pr#7052 <http://github.com/ceph/ceph/pull/7052>`_, Piotr Dałek)
+* kv/KineticStore: fix broken split_key (`pr#6574 <http://github.com/ceph/ceph/pull/6574>`_, Haomai Wang)
+* kv: optimize and clean up internal key/value interface (`pr#6312 <http://github.com/ceph/ceph/pull/6312>`_, Piotr Dałek, Sage Weil)
+* libcephfs: add pread, pwrite (Jevon Qiao)
+* libcephfs,ceph-fuse: cache cleanup (Zheng Yan)
+* libcephfs,ceph-fuse: fix request resend on cap reconnect (#10912 Yan, Zheng)
+* libcephfs: fix python tests and fix getcwd on missing dir (`pr#7901 <http://github.com/ceph/ceph/pull/7901>`_, John Spray)
+* libcephfs: Improve portability by replacing loff_t type usage with off_t (`pr#6301 <http://github.com/ceph/ceph/pull/6301>`_, John Coyle)
+* libcephfs: only check file offset on glibc platforms (`pr#6288 <http://github.com/ceph/ceph/pull/6288>`_, John Coyle)
+* libcephfs: update LIBCEPHFS_VERSION to indicate the interface was changed (`pr#7551 <http://github.com/ceph/ceph/pull/7551>`_, Jevon Qiao)
+* librados: add config observer (Alistair Strachan)
+* librados: add c++ style osd/pg command interface (`pr#6893 <http://github.com/ceph/ceph/pull/6893>`_, Yunchuan Wen)
+* librados: add FULL_TRY and FULL_FORCE flags for dealing with full clusters or pools (Sage Weil)
+* librados: add src_fadvise_flags for copy-from (Jianpeng Ma)
+* librados: aix gcc librados port (`pr#6675 <http://github.com/ceph/ceph/pull/6675>`_, Rohan Mars)
+* librados: avoid malloc(0) (which can return NULL on some platforms) (`issue#13944 <http://tracker.ceph.com/issues/13944>`_, `pr#6779 <http://github.com/ceph/ceph/pull/6779>`_, Dan Mick)
+* librados: cancel aio notification linger op upon completion (`pr#8102 <http://github.com/ceph/ceph/pull/8102>`_, Jason Dillaman)
+* librados: check connection state in rados_monitor_log (`issue#14499 <http://tracker.ceph.com/issues/14499>`_, `pr#7350 <http://github.com/ceph/ceph/pull/7350>`_, David Disseldorp)
+* librados: clean up Objecter.h (`pr#6731 <http://github.com/ceph/ceph/pull/6731>`_, Jie Wang)
+* librados: define C++ flags from C constants (Josh Durgin)
+* librados: detect laggy ops with objecter_timeout, not osd_timeout (`pr#7629 <http://github.com/ceph/ceph/pull/7629>`_, Greg Farnum)
+* librados: do cleanup (`pr#6488 <http://github.com/ceph/ceph/pull/6488>`_, xie xingguo)
+* librados: do not clear handle for aio_watch() (`pr#7771 <http://github.com/ceph/ceph/pull/7771>`_, xie xingguo)
+* librados: fadvise flags per op (Jianpeng Ma)
+* librados: fix examples/librados/Makefile error. (`pr#6320 <http://github.com/ceph/ceph/pull/6320>`_, You Ji)
+* librados: fix last_force_resent handling (#11026 Jianpeng Ma)
+* librados: fix memory leak from C_TwoContexts (Xiong Yiliang)
+* librados: fix notify completion race (#13114 Sage Weil)
+* librados: fix pool alignment API overflow issue (`issue#13715 <http://tracker.ceph.com/issues/13715>`_, `pr#6489 <http://github.com/ceph/ceph/pull/6489>`_, xie xingguo)
+* librados: fix potential null pointer access when do pool_snap_list (`issue#13639 <http://tracker.ceph.com/issues/13639>`_, `pr#6422 <http://github.com/ceph/ceph/pull/6422>`_, xie xingguo)
+* librados: fix PromoteOn2ndRead test for EC (`pr#6373 <http://github.com/ceph/ceph/pull/6373>`_, Sage Weil)
+* librados: fix rare race where pool op callback may hang forever (`issue#13642 <http://tracker.ceph.com/issues/13642>`_, `pr#6426 <http://github.com/ceph/ceph/pull/6426>`_, xie xingguo)
+* librados: fix several flaws introduced by the enumeration_objects API (`issue#14299 <http://tracker.ceph.com/issues/14299>`_, `issue#14301 <http://tracker.ceph.com/issues/14301>`_, `issue#14300 <http://tracker.ceph.com/issues/14300>`_, `pr#7156 <http://github.com/ceph/ceph/pull/7156>`_, xie xingguo)
+* librados: fix striper when stripe_count = 1 and stripe_unit != object_size (#11120 Yan, Zheng)
+* librados: fix test failure with new aio watch/unwatch API  (`pr#7824 <http://github.com/ceph/ceph/pull/7824>`_, Jason Dillaman)
+* librados: implement async watch/unwatch (`pr#7649 <http://github.com/ceph/ceph/pull/7649>`_, Haomai Wang)
+* librados: include/rados/librados.h: fix typo (`pr#6741 <http://github.com/ceph/ceph/pull/6741>`_, Nathan Cutler)
+* librados: init crush_location from config file. (`issue#13473 <http://tracker.ceph.com/issues/13473>`_, `pr#6243 <http://github.com/ceph/ceph/pull/6243>`_, Wei Luo)
+* librados, libcephfs: randomize client nonces (Josh Durgin)
+* librados: mix lock cycle (un)registering asok commands (`pr#7581 <http://github.com/ceph/ceph/pull/7581>`_, John Spray)
+* librados: move to c++11 concurrency types (`pr#5931 <http://github.com/ceph/ceph/pull/5931>`_, Adam C. Emerson)
+* librados: new style (sharded) object listing (`pr#6405 <http://github.com/ceph/ceph/pull/6405>`_, John Spray, Sage Weil)
+* librados: op perf counters (John Spray)
+* librados: potential null pointer access in list_(n)objects (`issue#13822 <http://tracker.ceph.com/issues/13822>`_, `pr#6639 <http://github.com/ceph/ceph/pull/6639>`_, xie xingguo)
+* librados: pybind: fix binary omap values (Robin H. Johnson)
+* librados: pybind: fix write() method return code (Javier Guerra)
+* librados: race condition on aio_notify completion handling (`pr#7864 <http://github.com/ceph/ceph/pull/7864>`_, Jason Dillaman)
+* librados: remove duplicate definitions for rados pool_stat_t and cluster_stat_t (`pr#7330 <http://github.com/ceph/ceph/pull/7330>`_, Igor Fedotov)
+* librados: respect default_crush_ruleset on pool_create (#11640 Yuan Zhou)
+* librados: Revert "rados: Add new field flags for ceph_osd_op.copy_get." (`pr#8486 <http://github.com/ceph/ceph/pull/8486>`_, Sage Weil)
+* librados: shutdown finisher in a more graceful way (`pr#7519 <http://github.com/ceph/ceph/pull/7519>`_, xie xingguo)
+* librados: Solaris port (`pr#6416 <http://github.com/ceph/ceph/pull/6416>`_, Rohan Mars)
+* librados: stat2 with higher time precision (`pr#7915 <http://github.com/ceph/ceph/pull/7915>`_, Yehuda Sadeh, Matt Benjamin)
+* librados: Striper: Fix incorrect push_front -> append_zero change (`pr#7578 <http://github.com/ceph/ceph/pull/7578>`_, Haomai Wang)
+* libradosstriper: fix leak (Danny Al-Gaaf)
+* librados_test_stub: protect against notify/unwatch race (`pr#7540 <http://github.com/ceph/ceph/pull/7540>`_, Jason Dillaman)
+* librados: wrongly passed in argument for stat command (`issue#13703 <http://tracker.ceph.com/issues/13703>`_, `pr#6476 <http://github.com/ceph/ceph/pull/6476>`_, xie xingguo)
+* librbd: add const for single-client-only features (Josh Durgin)
+* librbd: add deep-flatten operation (Jason Dillaman)
+* librbd: add purge_on_error cache behavior (Jianpeng Ma)
+* librbd: allocate new journal tag after acquiring exclusive lock (`pr#7884 <http://github.com/ceph/ceph/pull/7884>`_, Jason Dillaman)
+* librbd: allow additional metadata to be stored with the image (Haomai Wang)
+* librbd: API: async open and close (`issue#14264 <http://tracker.ceph.com/issues/14264>`_, `pr#7259 <http://github.com/ceph/ceph/pull/7259>`_, Mykola Golub)
+* librbd: automatically flush IO after blocking write operations (`issue#13913 <http://tracker.ceph.com/issues/13913>`_, `pr#6742 <http://github.com/ceph/ceph/pull/6742>`_, Jason Dillaman)
+* librbd: avoid blocking aio API methods (#11056 Jason Dillaman)
+* librbd: Avoid create two threads per image (`pr#7400 <http://github.com/ceph/ceph/pull/7400>`_, Haomai Wang)
+* librbd: avoid throwing error if mirroring is unsupported (`pr#8417 <http://github.com/ceph/ceph/pull/8417>`_, Jason Dillaman)
+* librbd: better handling for dup flatten requests (#11370 Jason Dillaman)
+* librbd: better handling of exclusive lock transition period (`pr#7204 <http://github.com/ceph/ceph/pull/7204>`_, Jason Dillaman)
+* librbd: block maintenance ops until after journal is ready (`issue#14510 <http://tracker.ceph.com/issues/14510>`_, `pr#7382 <http://github.com/ceph/ceph/pull/7382>`_, Jason Dillaman)
+* librbd: block read requests until journal replayed (`pr#7627 <http://github.com/ceph/ceph/pull/7627>`_, Jason Dillaman)
+* librbd: cancel in-flight ops on watch error (#11363 Jason Dillaman)
+* librbd: check for presence of journal before attempting to remove  (`issue#13912 <http://tracker.ceph.com/issues/13912>`_, `pr#6737 <http://github.com/ceph/ceph/pull/6737>`_, Jason Dillaman)
+* librbd: clear error when older OSD doesn't support image flags (`issue#14122 <http://tracker.ceph.com/issues/14122>`_, `pr#7035 <http://github.com/ceph/ceph/pull/7035>`_, Jason Dillaman)
+* librbd: correct include guard in RenameRequest.h (`pr#7143 <http://github.com/ceph/ceph/pull/7143>`_, Jason Dillaman)
+* librbd: correct issues discovered during teuthology testing (`issue#14108 <http://tracker.ceph.com/issues/14108>`_, `issue#14107 <http://tracker.ceph.com/issues/14107>`_, `pr#6974 <http://github.com/ceph/ceph/pull/6974>`_, Jason Dillaman)
+* librbd: correct issues discovered via valgrind memcheck (`pr#8132 <http://github.com/ceph/ceph/pull/8132>`_, Jason Dillaman)
+* librbd: correct issues discovered when cache is disabled (`issue#14123 <http://tracker.ceph.com/issues/14123>`_, `pr#6979 <http://github.com/ceph/ceph/pull/6979>`_, Jason Dillaman)
+* librbd: correct race conditions discovered during unit testing (`issue#14060 <http://tracker.ceph.com/issues/14060>`_, `pr#6923 <http://github.com/ceph/ceph/pull/6923>`_, Jason Dillaman)
+* librbd: deadlock while attempting to flush AIO requests (`issue#13726 <http://tracker.ceph.com/issues/13726>`_, `pr#6508 <http://github.com/ceph/ceph/pull/6508>`_, Jason Dillaman)
+* librbd: default new images to format 2 (#11348 Jason Dillaman)
+* librbd: differentiate journal replay flush vs shut down (`pr#7698 <http://github.com/ceph/ceph/pull/7698>`_, Jason Dillaman)
+* librbd: disable copy-on-read when not exclusive lock owner (`issue#14167 <http://tracker.ceph.com/issues/14167>`_, `pr#7129 <http://github.com/ceph/ceph/pull/7129>`_, Jason Dillaman)
+* librbd: disable image mirroring when image is removed (`issue#15265 <http://tracker.ceph.com/issues/15265>`_, `pr#8375 <http://github.com/ceph/ceph/pull/8375>`_, Ricardo Dias)
+* librbd: disallow unsafe rbd_op_threads values (`issue#15034 <http://tracker.ceph.com/issues/15034>`_, `pr#8459 <http://github.com/ceph/ceph/pull/8459>`_, Josh Durgin)
+* librbd: do not ignore self-managed snapshot release result (`issue#14170 <http://tracker.ceph.com/issues/14170>`_, `pr#7043 <http://github.com/ceph/ceph/pull/7043>`_, Jason Dillaman)
+* librbd: enable/disable image mirroring automatically for pool mode (`issue#15143 <http://tracker.ceph.com/issues/15143>`_, `pr#8204 <http://github.com/ceph/ceph/pull/8204>`_, Ricardo Dias)
+* librbd: ensure copy-on-read requests are complete prior to closing parent image  (`pr#6740 <http://github.com/ceph/ceph/pull/6740>`_, Jason Dillaman)
+* librbd: ensure librados callbacks are flushed prior to destroying (`issue#14092 <http://tracker.ceph.com/issues/14092>`_, `pr#7040 <http://github.com/ceph/ceph/pull/7040>`_, Jason Dillaman)
+* librbd: exit if parent's snap is gone during clone (`issue#14118 <http://tracker.ceph.com/issues/14118>`_, `pr#6968 <http://github.com/ceph/ceph/pull/6968>`_, xie xingguo)
+* librbd: fadvise for copy, export, import (Jianpeng Ma)
+* librbd: fast diff implementation that leverages object map (Jason Dillaman)
+* librbd: fix enable objectmap feature issue (`issue#13558 <http://tracker.ceph.com/issues/13558>`_, `pr#6339 <http://github.com/ceph/ceph/pull/6339>`_, xinxin shu)
+* librbd: fix fast diff bugs (#11553 Jason Dillaman)
+* librbd: fix image format detection (Zhiqiang Wang)
+* librbd: fix internal handling of dynamic feature updates (`pr#7299 <http://github.com/ceph/ceph/pull/7299>`_, Jason Dillaman)
+* librbd: fix journal iohint (`pr#6917 <http://github.com/ceph/ceph/pull/6917>`_, Jianpeng Ma)
+* librbd: fix known test case race condition failures (`issue#13969 <http://tracker.ceph.com/issues/13969>`_, `pr#6800 <http://github.com/ceph/ceph/pull/6800>`_, Jason Dillaman)
+* librbd: fix lock ordering issue (#11577 Jason Dillaman)
+* librbd: fix merge-diff for >2GB diff-files (`issue#14030 <http://tracker.ceph.com/issues/14030>`_, `pr#6889 <http://github.com/ceph/ceph/pull/6889>`_, Yunchuan Wen)
+* librbd: fix potential memory leak (`issue#14332 <http://tracker.ceph.com/issues/14332>`_, `issue#14333 <http://tracker.ceph.com/issues/14333>`_, `pr#7174 <http://github.com/ceph/ceph/pull/7174>`_, xie xingguo)
+* librbd: fix reads larger than the cache size (Lu Shi)
+* librbd: fix snap_exists API return code overflow (`issue#14129 <http://tracker.ceph.com/issues/14129>`_, `pr#6986 <http://github.com/ceph/ceph/pull/6986>`_, xie xingguo)
+* librbd: fix snapshot creation when other snap is active (#11475 Jason Dillaman)
+* librbd: fix state machine race conditions during shut down (`pr#7761 <http://github.com/ceph/ceph/pull/7761>`_, Jason Dillaman)
+* librbd: fix test case race condition for journaling ops (`pr#6877 <http://github.com/ceph/ceph/pull/6877>`_, Jason Dillaman)
+* librbd: fix tracepoint parameter in diff_iterate (`pr#6892 <http://github.com/ceph/ceph/pull/6892>`_, Yunchuan Wen)
+* librbd: flatten/copyup fixes (Jason Dillaman)
+* librbd: flush and invalidate cache via admin socket (`issue#2468 <http://tracker.ceph.com/issues/2468>`_, `pr#6453 <http://github.com/ceph/ceph/pull/6453>`_, Mykola Golub)
+* librbd: handle NOCACHE fadvise flag (Jinapeng Ma)
+* librbd: handle unregistering the image watcher when disconnected (`pr#8094 <http://github.com/ceph/ceph/pull/8094>`_, Jason Dillaman)
+* librbd: image refresh code paths converted to async state machines (`pr#6859 <http://github.com/ceph/ceph/pull/6859>`_, Jason Dillaman)
+* librbd: include missing header for bool type (`pr#6798 <http://github.com/ceph/ceph/pull/6798>`_, Mykola Golub)
+* librbd: initial collection of state machine unit tests (`pr#6703 <http://github.com/ceph/ceph/pull/6703>`_, Jason Dillaman)
+* librbd: integrate journaling for maintenance operations (`pr#6625 <http://github.com/ceph/ceph/pull/6625>`_, Jason Dillaman)
+* librbd: integrate journaling support for IO operations (`pr#6541 <http://github.com/ceph/ceph/pull/6541>`_, Jason Dillaman)
+* librbd: integrate journal replay with fsx testing (`pr#7583 <http://github.com/ceph/ceph/pull/7583>`_, Jason Dillaman)
+* librbd: journal framework for tracking exclusive lock transitions (`issue#13298 <http://tracker.ceph.com/issues/13298>`_, `pr#7529 <http://github.com/ceph/ceph/pull/7529>`_, Jason Dillaman)
+* librbd: journaling-related lock dependency cleanup (`pr#6777 <http://github.com/ceph/ceph/pull/6777>`_, Jason Dillaman)
+* librbd: journal replay needs to support re-executing maintenance ops (`issue#14822 <http://tracker.ceph.com/issues/14822>`_, `pr#7785 <http://github.com/ceph/ceph/pull/7785>`_, Jason Dillaman)
+* librbd: journal replay should honor inter-event dependencies (`pr#7019 <http://github.com/ceph/ceph/pull/7019>`_, Jason Dillaman)
+* librbd: journal shut down flush race condition (`issue#14434 <http://tracker.ceph.com/issues/14434>`_, `pr#7302 <http://github.com/ceph/ceph/pull/7302>`_, Jason Dillaman)
+* librbd: lockdep, helgrind validation (Jason Dillaman, Josh Durgin)
+* librbd: metadata filter fixes (Haomai Wang)
+* librbd: misc aio fixes (#5488 Jason Dillaman)
+* librbd: misc rbd fixes (#11478 #11113 #11342 #11380 Jason Dillaman, Zhiqiang Wang)
+* librbd: new diff_iterate2 API (Jason Dillaman)
+* librbd: not necessary to hold owner_lock while releasing snap id (`issue#13914 <http://tracker.ceph.com/issues/13914>`_, `pr#6736 <http://github.com/ceph/ceph/pull/6736>`_, Jason Dillaman)
+* librbd: object map rebuild support (Jason Dillaman)
+* librbd: only send signal when AIO completions queue empty (`pr#6729 <http://github.com/ceph/ceph/pull/6729>`_, Jianpeng Ma)
+* librbd: only update image flags while hold exclusive lock (#11791 Jason Dillaman)
+* librbd: optionally disable allocation hint (Haomai Wang)
+* librbd: optionally validate new RBD pools for snapshot support (`issue#13633 <http://tracker.ceph.com/issues/13633>`_, `pr#6925 <http://github.com/ceph/ceph/pull/6925>`_, Jason Dillaman)
+* librbd: partial revert of commit 9b0e359 (`issue#13969 <http://tracker.ceph.com/issues/13969>`_, `pr#6789 <http://github.com/ceph/ceph/pull/6789>`_, Jason Dillaman)
+* librbd: perf counters might not be initialized on error (`issue#13740 <http://tracker.ceph.com/issues/13740>`_, `pr#6523 <http://github.com/ceph/ceph/pull/6523>`_, Jason Dillaman)
+* librbd: perf section name: use hyphen to separate components (`issue#13719 <http://tracker.ceph.com/issues/13719>`_, `pr#6516 <http://github.com/ceph/ceph/pull/6516>`_, Mykola Golub)
+* librbd: prevent race between resize requests (#12664 Jason Dillaman)
+* librbd: properly handle replay of snap remove RPC message (`issue#14164 <http://tracker.ceph.com/issues/14164>`_, `pr#7042 <http://github.com/ceph/ceph/pull/7042>`_, Jason Dillaman)
+* librbd: readahead fixes (Zhiqiang Wang)
+* librbd: reduce mem copies to user-buffer during read (`pr#7548 <http://github.com/ceph/ceph/pull/7548>`_, Jianpeng Ma)
+* librbd: reduce verbosity of common error condition logging (`issue#14234 <http://tracker.ceph.com/issues/14234>`_, `pr#7114 <http://github.com/ceph/ceph/pull/7114>`_, Jason Dillaman)
+* librbd: refresh image if required before replaying journal ops (`issue#14908 <http://tracker.ceph.com/issues/14908>`_, `pr#7978 <http://github.com/ceph/ceph/pull/7978>`_, Jason Dillaman)
+* librbd: remove canceled tasks from timer thread (`issue#14476 <http://tracker.ceph.com/issues/14476>`_, `pr#7329 <http://github.com/ceph/ceph/pull/7329>`_, Douglas Fuller)
+* librbd: remove duplicate read_only test in librbd::async_flatten (`pr#5856 <http://github.com/ceph/ceph/pull/5856>`_, runsisi)
+* librbd: remove last synchronous librados calls from open/close state machine (`pr#7839 <http://github.com/ceph/ceph/pull/7839>`_, Jason Dillaman)
+* librbd: replaying a journal op post-refresh requires locking (`pr#8028 <http://github.com/ceph/ceph/pull/8028>`_, Jason Dillaman)
+* librbd: resize should only update image size within header (`issue#13674 <http://tracker.ceph.com/issues/13674>`_, `pr#6447 <http://github.com/ceph/ceph/pull/6447>`_, Jason Dillaman)
+* librbd: retrieve image name when opening by id (`pr#7736 <http://github.com/ceph/ceph/pull/7736>`_, Mykola Golub)
+* librbd: return error if we fail to delete object_map head object (`issue#14098 <http://tracker.ceph.com/issues/14098>`_, `pr#6958 <http://github.com/ceph/ceph/pull/6958>`_, xie xingguo)
+* librbd: return result code from close (#12069 Jason Dillaman)
+* librbd: Revert "librbd: use task finisher per CephContext" (`issue#14780 <http://tracker.ceph.com/issues/14780>`_, `pr#7667 <http://github.com/ceph/ceph/pull/7667>`_, Josh Durgin)
+* librbd: send notifications for mirroring status updates (`pr#8355 <http://github.com/ceph/ceph/pull/8355>`_, Jason Dillaman)
+* librbd: several race conditions discovered under single CPU environment (`pr#7653 <http://github.com/ceph/ceph/pull/7653>`_, Jason Dillaman)
+* librbd: simplify IO method signatures for 32bit environments (`pr#6700 <http://github.com/ceph/ceph/pull/6700>`_, Jason Dillaman)
+* librbd: small fixes for error messages and readahead counter (`issue#14127 <http://tracker.ceph.com/issues/14127>`_, `pr#6983 <http://github.com/ceph/ceph/pull/6983>`_, xie xingguo)
+* librbd: start perf counters after id is initialized (`issue#13720 <http://tracker.ceph.com/issues/13720>`_, `pr#6494 <http://github.com/ceph/ceph/pull/6494>`_, Mykola Golub)
+* librbd: store metadata, including config options, in image (Haomai Wang)
+* librbd: support eventfd for AIO completion notifications (`pr#5465 <http://github.com/ceph/ceph/pull/5465>`_, Haomai Wang)
+* librbd: tolerate old osds when getting image metadata (#11549 Jason Dillaman)
+* librbd: truncate does not need to mark the object as existing in the object map (`issue#14789 <http://tracker.ceph.com/issues/14789>`_, `pr#7772 <http://github.com/ceph/ceph/pull/7772>`_, xinxin shu)
+* librbd: uninitialized state in snap remove state machine (`pr#6982 <http://github.com/ceph/ceph/pull/6982>`_, Jason Dillaman)
+* librbd: update of mirror pool mode and mirror peer handling (`pr#7718 <http://github.com/ceph/ceph/pull/7718>`_, Jason Dillaman)
+* librbd: use async librados notifications (`pr#7668 <http://github.com/ceph/ceph/pull/7668>`_, Jason Dillaman)
+* librbd: use write_full when possible (Zhiqiang Wang)
+* log: do not repeat errors to stderr (`issue#14616 <http://tracker.ceph.com/issues/14616>`_, `pr#7983 <http://github.com/ceph/ceph/pull/7983>`_, Sage Weil)
+* log: fix data corruption race resulting from log rotation (#12465 Samuel Just)
+* log: fix stack overflow when flushing large log lines (`issue#14707 <http://tracker.ceph.com/issues/14707>`_, `pr#7599 <http://github.com/ceph/ceph/pull/7599>`_, Igor Fedotov)
+* logrotate.d: prefer service over invoke-rc.d (#11330 Win Hierman, Sage Weil)
+* log: segv in a portable way (`issue#14856 <http://tracker.ceph.com/issues/14856>`_, `pr#7790 <http://github.com/ceph/ceph/pull/7790>`_, Kefu Chai)
+* log: use delete[] (`pr#7904 <http://github.com/ceph/ceph/pull/7904>`_, Sage Weil)
+* mailmap: add UMCloud affiliation (`pr#6820 <http://github.com/ceph/ceph/pull/6820>`_, Jiaying Ren)
+* mailmap for 10.0.4 (`pr#7932 <http://github.com/ceph/ceph/pull/7932>`_, Abhishek Lekshmanan)
+* mailmap: hange organization for Dongmao Zhang (`pr#7173 <http://github.com/ceph/ceph/pull/7173>`_, Dongmao Zhang)
+* mailmap: Igor Podoski affiliation (`pr#7219 <http://github.com/ceph/ceph/pull/7219>`_, Igor Podoski)
+* mailmap: Jewel updates (`pr#6750 <http://github.com/ceph/ceph/pull/6750>`_, Abhishek Lekshmanan)
+* mailmap: modify member info  (`pr#6468 <http://github.com/ceph/ceph/pull/6468>`_, Xiaowei Chen)
+* mailmap: revise organization (`pr#6519 <http://github.com/ceph/ceph/pull/6519>`_, Li Wang)
+* mailmap: Ubuntu Kylin name changed to Kylin Cloud (`pr#6532 <http://github.com/ceph/ceph/pull/6532>`_, Loic Dachary)
+* mailmap: update .organizationmap (`pr#6565 <http://github.com/ceph/ceph/pull/6565>`_, chenji-kael)
+* mailmap update (`pr#7210 <http://github.com/ceph/ceph/pull/7210>`_, M Ranga Swami Reddy)
+* mailmap update (`pr#8522 <http://github.com/ceph/ceph/pull/8522>`_, M Ranga Swami Reddy)
+* mailmap: updates for infernalis. (`pr#6495 <http://github.com/ceph/ceph/pull/6495>`_, Yann Dupont)
+* mailmap: updates (`pr#6258 <http://github.com/ceph/ceph/pull/6258>`_, M Ranga Swami Reddy)
+* mailmap: updates (`pr#6594 <http://github.com/ceph/ceph/pull/6594>`_, chenji-kael)
+* mailmap updates (`pr#6992 <http://github.com/ceph/ceph/pull/6992>`_, Loic Dachary)
+* mailmap updates (`pr#7189 <http://github.com/ceph/ceph/pull/7189>`_, Loic Dachary)
+* mailmap updates (`pr#7528 <http://github.com/ceph/ceph/pull/7528>`_, Yann Dupont)
+* mailmap updates (`pr#8256 <http://github.com/ceph/ceph/pull/8256>`_, Loic Dachary)
+* mailmap: Xie Xingguo affiliation (`pr#6409 <http://github.com/ceph/ceph/pull/6409>`_, Loic Dachary)
+* Makefile-env.am: set a default for CEPH_BUILD_VIRTUALENV (part 2) (`pr#8320 <http://github.com/ceph/ceph/pull/8320>`_, Loic Dachary)
+* makefile: fix rbdmap manpage (`pr#8310 <http://github.com/ceph/ceph/pull/8310>`_, Kefu Chai)
+* makefile: remove libedit from libclient.la (`pr#7284 <http://github.com/ceph/ceph/pull/7284>`_, Kefu Chai)
+* makefiles: remove bz2-dev from dependencies (`issue#13981 <http://tracker.ceph.com/issues/13981>`_, `pr#6939 <http://github.com/ceph/ceph/pull/6939>`_, Piotr Dałek)
+* man/8/ceph-disk: fix formatting issue (`pr#8003 <http://github.com/ceph/ceph/pull/8003>`_, Sage Weil)
+* man/8/ceph-disk: fix formatting issue (`pr#8012 <http://github.com/ceph/ceph/pull/8012>`_, Sage Weil)
+* man: document listwatchers cmd in "rados" manpage (`pr#7021 <http://github.com/ceph/ceph/pull/7021>`_, Kefu Chai)
+* mdsa: A few more snapshot fixes, mostly around snapshotted inode/dentry tracking (`pr#7798 <http://github.com/ceph/ceph/pull/7798>`_, Yan, Zheng)
+* mds: Add cmapv to ESessions default constructor initializer list (`pr#8403 <http://github.com/ceph/ceph/pull/8403>`_, John Coyle)
+* mds: add 'damaged' state to MDSMap (John Spray)
+* mds: add nicknames for perfcounters (John Spray)
+* mds: add 'p' flag in auth caps to control setting pool in layout (`pr#6567 <http://github.com/ceph/ceph/pull/6567>`_, John Spray)
+* mds: advance clientreplay when replying (`issue#14357 <http://tracker.ceph.com/issues/14357>`_, `pr#7216 <http://github.com/ceph/ceph/pull/7216>`_, John Spray)
+* mds: allow client to request caps when opening file (`issue#14360 <http://tracker.ceph.com/issues/14360>`_, `pr#7952 <http://github.com/ceph/ceph/pull/7952>`_, Yan, Zheng)
+* mds: avoid emitting cap warnigns before evicting session (John Spray)
+* mds: avoid getting stuck in XLOCKDONE (#11254 Yan, Zheng)
+* mds, client: add namespace to file_layout_t (previously ceph_file_layout) (`pr#7098 <http://github.com/ceph/ceph/pull/7098>`_, Yan, Zheng, Sage Weil)
+* mds, client: fix locking around handle_conf_change (`issue#14365 <http://tracker.ceph.com/issues/14365>`_, `issue#14374 <http://tracker.ceph.com/issues/14374>`_, `pr#7312 <http://github.com/ceph/ceph/pull/7312>`_, John Spray)
+* mds: disable problematic rstat propagation into snap parents (Yan, Zheng)
+* mds: do not add snapped items to bloom filter (Yan, Zheng)
+* mds: don't double-shutdown the timer when suiciding (`issue#14697 <http://tracker.ceph.com/issues/14697>`_, `pr#7616 <http://github.com/ceph/ceph/pull/7616>`_, Greg Farnum)
+* mds: expose frags via asok (John Spray)
+* mds: expose state of recovery to status ASOK command (`issue#14146 <http://tracker.ceph.com/issues/14146>`_, `pr#7068 <http://github.com/ceph/ceph/pull/7068>`_, Yan, Zheng)
+* mds: Extend the existing pool access checking to include specific RADOS namespacse. (`pr#8444 <https://github.com/ceph/ceph/pull/8444>`_, Yan, Zheng)
+* mds: filelock deadlock (`pr#7713 <http://github.com/ceph/ceph/pull/7713>`_, Yan, Zheng)
+* mds: fix client capabilities during reconnect (client.XXXX isn't responding to mclientcaps(revoke)) (`issue#11482 <http://tracker.ceph.com/issues/11482>`_, `pr#6432 <http://github.com/ceph/ceph/pull/6432>`_, Yan, Zheng)
+* mds: fix client cap/message replay order on restart (`issue#14254 <http://tracker.ceph.com/issues/14254>`_, `issue#13546 <http://tracker.ceph.com/issues/13546>`_, `pr#7199 <http://github.com/ceph/ceph/pull/7199>`_, Yan, Zheng)
+* mds: fix expected holes in journal objects (#13167 Yan, Zheng)
+* mds: fix file_layout_t legacy encoding snafu (`pr#8455 <http://github.com/ceph/ceph/pull/8455>`_, Sage Weil)
+* mds: fix fsmap decode (`pr#8063 <http://github.com/ceph/ceph/pull/8063>`_, Greg Farnum)
+* mds: fix FSMap upgrade with daemons in the map (`pr#8073 <http://github.com/ceph/ceph/pull/8073>`_, John Spray, Greg Farnum)
+* mds: fix handling for missing mydir dirfrag (#11641 John Spray)
+* mds: fix inode_t::compare() (`issue#15038 <http://tracker.ceph.com/issues/15038>`_, `pr#8014 <http://github.com/ceph/ceph/pull/8014>`_, Yan, Zheng)
+* mds: fix integer truncateion on large client ids (Henry Chang)
+* mds: fix mydir replica issue with shutdown (#10743 John Spray)
+* mds: fix out-of-order messages (#11258 Yan, Zheng)
+* mds: fix rejoin (Yan, Zheng)
+* mds: fix scrub_path (`pr#6684 <http://github.com/ceph/ceph/pull/6684>`_, John Spray)
+* mds: fix setting entire file layout in one setxattr (John Spray)
+* mds: fix setvxattr (broken in a536d114) (`issue#14029 <http://tracker.ceph.com/issues/14029>`_, `pr#6941 <http://github.com/ceph/ceph/pull/6941>`_, John Spray)
+* mds: fix shutdown (John Spray)
+* mds: fix shutdown with strays (#10744 John Spray)
+* mds: fix SnapServer crash on deleted pool (John Spray)
+* mds: fix snapshot bugs (Yan, Zheng)
+* mds: fix standby replay thread creation (`issue#14144 <http://tracker.ceph.com/issues/14144>`_, `pr#7132 <http://github.com/ceph/ceph/pull/7132>`_, John Spray)
+* mds: fix stray handling (John Spray)
+* mds: fix stray purging in 'stripe_count > 1' case (`issue#15050 <http://tracker.ceph.com/issues/15050>`_, `pr#8040 <http://github.com/ceph/ceph/pull/8040>`_, Yan, Zheng)
+* mds: fix stray reintegration (Yan, Zheng)
+* mds: fix suicide beacon (John Spray)
+* mds: flush immediately in do_open_truncate (#11011 John Spray)
+* mds: function parameter 'df' should be passed by reference (`pr#7490 <http://github.com/ceph/ceph/pull/7490>`_, Na Xie)
+* mds: handle misc corruption issues (John Spray)
+* mds: implement snapshot rename (`pr#5645 <http://github.com/ceph/ceph/pull/5645>`_, xinxin shu)
+* mds: improve dump methods (John Spray)
+* mds: judgment added to avoid the risk of visiting the NULL pointer (`pr#7358 <http://github.com/ceph/ceph/pull/7358>`_, Kongming Wu)
+* mds: many fixes (Yan, Zheng, John Spray, Greg Farnum)
+* mds: many snapshot and stray fixes (Yan, Zheng)
+* mds: messages/MOSDOp: cast in assert to eliminate warnings (`issue#13625 <http://tracker.ceph.com/issues/13625>`_, `pr#6414 <http://github.com/ceph/ceph/pull/6414>`_, David Zafman)
+* mds: misc fixes (Jianpeng Ma, Dan van der Ster, Zhang Zhi)
+* mds: misc journal cleanups and fixes (#10368 John Spray)
+* mds: misc repair improvements (John Spray)
+* mds: misc snap fixes (Zheng Yan)
+* mds: misc snapshot fixes (Yan, Zheng)
+* mds: Multi-filesystem support (`issue#14952 <http://tracker.ceph.com/issues/14952>`_, `pr#6953 <http://github.com/ceph/ceph/pull/6953>`_, John Spray, Sage Weil)
+* mds: new filtered MDS tell commands for sessions (`pr#6180 <http://github.com/ceph/ceph/pull/6180>`_, John Spray)
+* mds: new SessionMap storage using omap (#10649 John Spray)
+* mds: persist completed_requests reliably (#11048 John Spray)
+* mds: properly set STATE_STRAY/STATE_ORPHAN for stray dentry/inode (`issue#13777 <http://tracker.ceph.com/issues/13777>`_, `pr#6553 <http://github.com/ceph/ceph/pull/6553>`_, Yan, Zheng)
+* mds: Protect a number of unstable/experimental features behind durable flags (`pr#8383 <https://github.com/ceph/ceph/pull/8383>`_, Greg Farnum)
+* mds: reduce memory consumption (Yan, Zheng)
+* mds: repair the command option "--hot-standby" (`pr#6454 <http://github.com/ceph/ceph/pull/6454>`_, Wei Feng)
+* mds: respawn instead of suicide on blacklist (John Spray)
+* mds: ScrubStack and "tag path" command (`pr#5662 <http://github.com/ceph/ceph/pull/5662>`_, Yan, Zheng, John Spray, Greg Farnum)
+* mds: separate safe_pos in Journaler (#10368 John Spray)
+* mds/Session: use projected parent for auth path check (`issue#13364 <http://tracker.ceph.com/issues/13364>`_, `pr#6200 <http://github.com/ceph/ceph/pull/6200>`_, Sage Weil)
+* mds: snapshot rename support (#3645 Yan, Zheng)
+* mds: store layout on header object (#4161 John Spray)
+* mds: tear down connections from `tell` commands (`issue#14048 <http://tracker.ceph.com/issues/14048>`_, `pr#6933 <http://github.com/ceph/ceph/pull/6933>`_, John Spray)
+* mds: throttle purge stray operations (#10390 John Spray)
+* mds: tolerate clock jumping backwards (#11053 Yan, Zheng)
+* mds: warn when clients fail to advance oldest_client_tid (#10657 Yan, Zheng)
+* mds: we should wait messenger when MDSDaemon suicide (`pr#6996 <http://github.com/ceph/ceph/pull/6996>`_, Wei Feng)
+* messages/MOSDOp: clear reqid inc for v6 encoding (`issue#15230 <http://tracker.ceph.com/issues/15230>`_, `pr#8299 <http://github.com/ceph/ceph/pull/8299>`_, Sage Weil)
+* Minor fixes around data scan in some scenarios (`pr#8115 <http://github.com/ceph/ceph/pull/8115>`_, Yan, Zheng)
+* mirrors: Change contact e-mail address for se.ceph.com (`pr#8007 <http://github.com/ceph/ceph/pull/8007>`_, Wido den Hollander)
+* mirrors: Updated scripts and documentation for mirrors (`pr#7847 <http://github.com/ceph/ceph/pull/7847>`_, Wido den Hollander)
+* misc cleanups and fixes (Danny Al-Gaaf)
+* misc coverity fixes (Danny Al-Gaaf)
+* misc performance and cleanup (Nathan Cutler, Xinxin Shu)
+* misc: use make_shared while creating shared_ptr (`pr#7769 <http://github.com/ceph/ceph/pull/7769>`_, Somnath Roy)
+* mon: add an independent option for max election time (`pr#7245 <http://github.com/ceph/ceph/pull/7245>`_, Sangdi Xu)
+* mon: add cache over MonitorDBStore (Kefu Chai)
+* mon: add 'mon_metadata <id>' command (Kefu Chai)
+* mon: add 'node ls ...' command (Kefu Chai)
+* mon: add NOFORWARD, OBSOLETE, DEPRECATE flags for mon commands (Joao Eduardo Luis)
+* mon: add `osd blacklist clear` (`pr#6945 <http://github.com/ceph/ceph/pull/6945>`_, John Spray)
+* mon: add PG count to 'ceph osd df' output (Michal Jarzabek)
+* mon: add RAW USED column to ceph df detail (`pr#7087 <http://github.com/ceph/ceph/pull/7087>`_, Ruifeng Yang)
+* mon: block 'ceph osd pg-temp ...' if pg_temp update is already pending (`pr#6704 <http://github.com/ceph/ceph/pull/6704>`_, Sage Weil)
+* mon: 'ceph osd metadata' can dump all osds (Haomai Wang)
+* mon: clean up, reorg some mon commands (Joao Eduardo Luis)
+* mon: cleanup set-quota error msg (`pr#7371 <http://github.com/ceph/ceph/pull/7371>`_, Abhishek Lekshmanan)
+* monclient: avoid key renew storm on clock skew (`issue#12065 <http://tracker.ceph.com/issues/12065>`_, `pr#8258 <http://github.com/ceph/ceph/pull/8258>`_, Alexey Sheplyakov)
+* monclient: flush_log (John Spray)
+* mon: compact full epochs also (`issue#14537 <http://tracker.ceph.com/issues/14537>`_, `pr#7396 <http://github.com/ceph/ceph/pull/7396>`_, Kefu Chai)
+* mon: consider pool size when creating pool (`issue#14509 <http://tracker.ceph.com/issues/14509>`_, `pr#7359 <http://github.com/ceph/ceph/pull/7359>`_, songbaisen)
+* mon: consider the pool size when setting pool crush rule (`issue#14495 <http://tracker.ceph.com/issues/14495>`_, `pr#7341 <http://github.com/ceph/ceph/pull/7341>`_, song baisen)
+* mon: degrade a log message to level 2 (`pr#6929 <http://github.com/ceph/ceph/pull/6929>`_, Kongming Wu)
+* mon: detect kv backend failures (Sage Weil)
+* mon: disallow >2 tiers (#11840 Kefu Chai)
+* mon: disallow ec pools as tiers (#11650 Samuel Just)
+* mon: do not deactivate last mds (#10862 John Spray)
+* mon: do not send useless pg_create messages for split pgs (`pr#8247 <http://github.com/ceph/ceph/pull/8247>`_, Sage Weil)
+* mon: don't require OSD W for MRemoveSnaps (`issue#13777 <http://tracker.ceph.com/issues/13777>`_, `pr#6601 <http://github.com/ceph/ceph/pull/6601>`_, John Spray)
+* mon: drop useless rank init assignment (`issue#14508 <http://tracker.ceph.com/issues/14508>`_, `pr#7321 <http://github.com/ceph/ceph/pull/7321>`_, huanwen ren)
+* mon: enable 'mon osd prime pg temp' by default (`pr#7838 <http://github.com/ceph/ceph/pull/7838>`_, Robert LeBlanc)
+* mon: fix average utilization calc for 'osd df' (Mykola Golub)
+* mon: fix calculation of %USED (`pr#7881 <http://github.com/ceph/ceph/pull/7881>`_, Adam Kupczyk)
+* mon: fix ceph df pool available calculation for 0-weighted OSDs (`pr#6660 <http://github.com/ceph/ceph/pull/6660>`_, Chengyuan Li)
+* mon: fix coding-style on PG related Monitor files (`pr#6881 <http://github.com/ceph/ceph/pull/6881>`_, Wido den Hollander)
+* mon: fix CRUSH map test for new pools (Sage Weil)
+* mon: fixes related to mondbstore->get() changes (`pr#6564 <http://github.com/ceph/ceph/pull/6564>`_, Piotr Dałek)
+* mon: fix keyring permissions (`issue#14950 <http://tracker.ceph.com/issues/14950>`_, `pr#7880 <http://github.com/ceph/ceph/pull/7880>`_, Owen Synge)
+* mon: fix locking in preinit error paths (`issue#14473 <http://tracker.ceph.com/issues/14473>`_, `pr#7353 <http://github.com/ceph/ceph/pull/7353>`_, huanwen ren)
+* mon: fix log dump crash when debugging (Mykola Golub)
+* mon: fix mds beacon replies (#11590 Kefu Chai)
+* mon: fix metadata update race (Mykola Golub)
+* mon: fix min_last_epoch_clean tracking (Kefu Chai)
+* mon: fix monmap creation stamp (`pr#7459 <http://github.com/ceph/ceph/pull/7459>`_, duanweijun)
+* mon: fix 'pg ls' sort order, state names (#11569 Kefu Chai)
+* mon: fix refresh (#11470 Joao Eduardo Luis)
+* mon: fix reuse of osd ids (clear osd info on osd deletion) (`issue#13988 <http://tracker.ceph.com/issues/13988>`_, `pr#6900 <http://github.com/ceph/ceph/pull/6900>`_, Loic Dachary, Sage Weil)
+* mon: fix routed_request_tids leak (`pr#6102 <http://github.com/ceph/ceph/pull/6102>`_, Ning Yao)
+* mon: fix sync of config-key data (`pr#7363 <http://github.com/ceph/ceph/pull/7363>`_, Xiaowei Chen)
+* mon: fix the can't change subscribe level bug in monitoring log (`pr#7031 <http://github.com/ceph/ceph/pull/7031>`_, Zhiqiang Wang)
+* mon: fix variance calc in 'osd df' (Sage Weil)
+* mon: go into ERR state if multiple PGs are stuck inactive (`issue#13923 <http://tracker.ceph.com/issues/13923>`_, `pr#7253 <http://github.com/ceph/ceph/pull/7253>`_, Wido den Hollander)
+* mon: improve callout to crushtool (Mykola Golub)
+* mon: initialize last_* timestamps on new pgs to creation time (`issue#14952 <http://tracker.ceph.com/issues/14952>`_, `pr#7980 <http://github.com/ceph/ceph/pull/7980>`_, Sage Weil)
+* mon: initialize recorded election epoch properly even when standalone (`issue#13627 <http://tracker.ceph.com/issues/13627>`_, `pr#6407 <http://github.com/ceph/ceph/pull/6407>`_, huanwen ren)
+* mon: make blocked op messages more readable (Jianpeng Ma)
+* mon: make clock skew checks sane (`issue#14175 <http://tracker.ceph.com/issues/14175>`_, `pr#7141 <http://github.com/ceph/ceph/pull/7141>`_, Joao Eduardo Luis)
+* mon: make osd get pool 'all' only return applicable fields (#10891 Michal Jarzabek)
+* mon: mark_down_pgs in lockstep with pg_map's osdmap epoch (`pr#8208 <http://github.com/ceph/ceph/pull/8208>`_, Sage Weil)
+* mon/MDSMonitor: add confirmation to "ceph mds rmfailed" (`issue#14379 <http://tracker.ceph.com/issues/14379>`_, `pr#7248 <http://github.com/ceph/ceph/pull/7248>`_, Yan, Zheng)
+* mon/MDSMonitor.cc: properly note beacon when health metrics changes (`issue#14684 <http://tracker.ceph.com/issues/14684>`_, `pr#7757 <http://github.com/ceph/ceph/pull/7757>`_, Yan, Zheng)
+* mon: misc scaling fixes (Sage Weil)
+* mon: modify a dout level in OSDMonitor.cc (`pr#6928 <http://github.com/ceph/ceph/pull/6928>`_, Yongqiang He)
+* mon/MonClient: avoid null pointer error when configured incorrectly (`issue#14405 <http://tracker.ceph.com/issues/14405>`_, `pr#7276 <http://github.com/ceph/ceph/pull/7276>`_, Bo Cai)
+* mon/MonClient: fix shutdown race (`issue#13992 <http://tracker.ceph.com/issues/13992>`_, `pr#8335 <http://github.com/ceph/ceph/pull/8335>`_, Sage Weil)
+* mon/monitor: some clean up (`pr#7520 <http://github.com/ceph/ceph/pull/7520>`_, huanwen ren)
+* mon: MonmapMonitor: don't expose uncommitted state to client (`pr#6854 <http://github.com/ceph/ceph/pull/6854>`_, Joao Eduardo Luis)
+* mon: normalize erasure-code profile for storage and comparison (Loic Dachary)
+* mon: only send mon metadata to supporting peers (Sage Weil)
+* mon: optionally specify osd id on 'osd create' (Mykola Golub)
+* mon/OSDMonitor: osdmap laggy set a maximum limit for interval (`pr#7109 <http://github.com/ceph/ceph/pull/7109>`_, Zengran Zhang)
+* mon: osd [test-]reweight-by-{pg,utilization} command updates (`pr#7890 <http://github.com/ceph/ceph/pull/7890>`_, Dan van der Ster, Sage Weil)
+* mon: 'osd tree' fixes (Kefu Chai)
+* mon: paxos is_recovering calc error (`pr#7227 <http://github.com/ceph/ceph/pull/7227>`_, Weijun Duan)
+* mon: periodic background scrub (Joao Eduardo Luis)
+* mon/PGMap: show rd/wr iops separately in status reports (`pr#7072 <http://github.com/ceph/ceph/pull/7072>`_, Cilang Zhao)
+* mon: PGMonitor: acting primary diff with cur_stat, should not set pg to stale (`pr#7083 <http://github.com/ceph/ceph/pull/7083>`_, Xiaowei Chen)
+* mon/PGMonitor: reliably mark PGs state (`pr#8089 <http://github.com/ceph/ceph/pull/8089>`_, Sage Weil)
+* mon: PG Monitor should report waiting for backfill (`issue#12744 <http://tracker.ceph.com/issues/12744>`_, `pr#7398 <http://github.com/ceph/ceph/pull/7398>`_, Abhishek Lekshmanan)
+* mon/pgmonitor: use appropriate forced conversions in get_rule_avail (`pr#7705 <http://github.com/ceph/ceph/pull/7705>`_, huanwen ren)
+* mon: prevent bucket deletion when referenced by a crush rule (#11602 Sage Weil)
+* mon: prevent pgp_num > pg_num (#12025 Xinxin Shu)
+* mon: prevent pool with snapshot state from being used as a tier (#11493 Sage Weil)
+* mon: prime pg_temp when CRUSH map changes (Sage Weil)
+* mon: reduce CPU and memory manager pressure of pg health check (`pr#7482 <http://github.com/ceph/ceph/pull/7482>`_, Piotr Dałek)
+* mon: refine check_remove_tier checks (#11504 John Spray)
+* mon: reject large max_mds values (#12222 John Spray)
+* mon: remove 'mds setmap' (`issue#15136 <http://tracker.ceph.com/issues/15136>`_, `pr#8121 <http://github.com/ceph/ceph/pull/8121>`_, Sage Weil)
+* mon: remove remove_legacy_versions() (`pr#8324 <http://github.com/ceph/ceph/pull/8324>`_, Kefu Chai)
+* mon: remove spurious who arg from 'mds rm ...' (John Spray)
+* mon: remove unnecessary comment for update_from_paxos (`pr#8400 <http://github.com/ceph/ceph/pull/8400>`_, Qinghua Jin)
+* mon: remove unused variable (`issue#15292 <http://tracker.ceph.com/issues/15292>`_, `pr#8337 <http://github.com/ceph/ceph/pull/8337>`_, Javier M. Mellid)
+* mon: revert MonitorDBStore's WholeStoreIteratorImpl::get (`issue#13742 <http://tracker.ceph.com/issues/13742>`_, `pr#6522 <http://github.com/ceph/ceph/pull/6522>`_, Piotr Dałek)
+* mon: should not set isvalid = true when cephx_verify_authorizer return false (`issue#13525 <http://tracker.ceph.com/issues/13525>`_, `pr#6306 <http://github.com/ceph/ceph/pull/6306>`_, Ruifeng Yang)
+* mon: show the pool quota info on ceph df detail command (`issue#14216 <http://tracker.ceph.com/issues/14216>`_, `pr#7094 <http://github.com/ceph/ceph/pull/7094>`_, song baisen)
+* mon: some cleanup in MonmapMonitor.cc (`pr#7418 <http://github.com/ceph/ceph/pull/7418>`_, huanwen ren)
+* mon: standardize Ceph removal commands (`pr#7939 <http://github.com/ceph/ceph/pull/7939>`_, Dongsheng Yang)
+* mon: streamline session handling, fix memory leaks (Sage Weil)
+* mon: support min_down_reporter by subtree level (default by host) (`pr#6709 <http://github.com/ceph/ceph/pull/6709>`_, Xiaoxi Chen)
+* mon: unconfuse object count skew message (`pr#7882 <http://github.com/ceph/ceph/pull/7882>`_, Piotr Dałek)
+* mon: unregister command on shutdown (`pr#7504 <http://github.com/ceph/ceph/pull/7504>`_, huanwen ren)
+* mon: upgrades must pass through hammer (Sage Weil)
+* mon: warn if pg(s) not scrubbed (`issue#13142 <http://tracker.ceph.com/issues/13142>`_, `pr#6440 <http://github.com/ceph/ceph/pull/6440>`_, Michal Jarzabek)
+* mon: warn on bogus cache tier config (Jianpeng Ma)
+* mount.ceph: memory leaks (`pr#6905 <http://github.com/ceph/ceph/pull/6905>`_, Qiankun Zheng)
+* mount.fuse.ceph: better parsing of arguments passed to mount.fuse.ceph by mount command (`issue#14735 <http://tracker.ceph.com/issues/14735>`_, `pr#7607 <http://github.com/ceph/ceph/pull/7607>`_, Florent Bautista)
+* mrun: update path to cmake binaries (`pr#8447 <http://github.com/ceph/ceph/pull/8447>`_, Casey Bodley)
+* msg: add override to virutal methods (`pr#6977 <http://github.com/ceph/ceph/pull/6977>`_, Michal Jarzabek)
+* msg: add thread safety for "random" Messenger + fix wrong usage of random functions (`pr#7650 <http://github.com/ceph/ceph/pull/7650>`_, Avner BenHanoch)
+* msg/async: AsyncConnection: avoid debug log in cleanup_handler (`pr#7547 <http://github.com/ceph/ceph/pull/7547>`_, Haomai Wang)
+* msg/async: AsyncMessenger: fix several bugs (`pr#7831 <http://github.com/ceph/ceph/pull/7831>`_, Haomai Wang)
+* msg/async: AsyncMessenger: fix valgrind leak (`pr#7725 <http://github.com/ceph/ceph/pull/7725>`_, Haomai Wang)
+* msg/async: avoid log spam on throttle (`issue#15031 <http://tracker.ceph.com/issues/15031>`_, `pr#8263 <http://github.com/ceph/ceph/pull/8263>`_, Kefu Chai)
+* msg/async: bunch of fixes (`pr#7379 <http://github.com/ceph/ceph/pull/7379>`_, Piotr Dałek)
+* msg/async: cleanup dead connection and misc things (`pr#7158 <http://github.com/ceph/ceph/pull/7158>`_, Haomai Wang)
+* msg/async: don't calculate msg header crc when not needed (`pr#7815 <http://github.com/ceph/ceph/pull/7815>`_, Piotr Dałek)
+* msg/async: don't use shared_ptr to manage EventCallback (`pr#7028 <http://github.com/ceph/ceph/pull/7028>`_, Haomai Wang)
+* msg/async: Event: fix clock skew problem (`pr#7949 <http://github.com/ceph/ceph/pull/7949>`_, Wei Jin)
+* msg/async: fix array boundary (`pr#7451 <http://github.com/ceph/ceph/pull/7451>`_, Wei Jin)
+* msg: async: fix perf counter description and simplify _send_keepalive_or_ack (`pr#8046 <http://github.com/ceph/ceph/pull/8046>`_, xie xingguo)
+* msg/async: fix potential race condition (`pr#7453 <http://github.com/ceph/ceph/pull/7453>`_, Haomai Wang)
+* msg/async: fix send closed local_connection message problem (`pr#7255 <http://github.com/ceph/ceph/pull/7255>`_, Haomai Wang)
+* msg/async: let receiver ack message ASAP (`pr#6478 <http://github.com/ceph/ceph/pull/6478>`_, Haomai Wang)
+* msg/async: reduce extra tcp packet for message ack (`pr#7380 <http://github.com/ceph/ceph/pull/7380>`_, Haomai Wang)
+* msg/async: remove experiment feature (`pr#7820 <http://github.com/ceph/ceph/pull/7820>`_, Haomai Wang)
+* msg: async: small cleanups (`pr#7871 <http://github.com/ceph/ceph/pull/7871>`_, xie xingguo)
+* msg/async: smarter MSG_MORE (`pr#7625 <http://github.com/ceph/ceph/pull/7625>`_, Piotr Dałek)
+* msg: async: start over after failing to bind a port in specified range (`issue#14928 <http://tracker.ceph.com/issues/14928>`_, `issue#13002 <http://tracker.ceph.com/issues/13002>`_, `pr#7852 <http://github.com/ceph/ceph/pull/7852>`_, xie xingguo)
+* msg/async: support of non-block connect in async messenger (`issue#12802 <http://tracker.ceph.com/issues/12802>`_, `pr#5848 <http://github.com/ceph/ceph/pull/5848>`_, Jianhui Yuan)
+* msg/async: _try_send trim already sent for outcoming_bl more efficient (`pr#7970 <http://github.com/ceph/ceph/pull/7970>`_, Yan Jun)
+* msg/async: will crash if enabling async msg because of an assertion (`pr#6640 <http://github.com/ceph/ceph/pull/6640>`_, Zhi Zhang)
+* msg: filter out lo addr when bind osd addr (`pr#7012 <http://github.com/ceph/ceph/pull/7012>`_, Ji Chen)
+* msgr: add ceph_perf_msgr tool (Hoamai Wang)
+* msgr: async: fix seq handling (Haomai Wang)
+* msgr: async: many many fixes (Haomai Wang)
+* msg: removed unneeded includes from Dispatcher (`pr#6814 <http://github.com/ceph/ceph/pull/6814>`_, Michal Jarzabek)
+* msg: remove duplicated code - local_delivery will now call 'enqueue' (`pr#7948 <http://github.com/ceph/ceph/pull/7948>`_, Avner BenHanoch)
+* msg: remove unneeded inline (`pr#6989 <http://github.com/ceph/ceph/pull/6989>`_, Michal Jarzabek)
+* msgr:  fix large message data content length causing overflow (`pr#6809 <http://github.com/ceph/ceph/pull/6809>`_, Jun Huang, Haomai Wang)
+* msgr: simple: fix clear_pipe (#11381 Haomai Wang)
+* msgr: simple: fix connect_seq assert (Haomai Wang)
+* msgr: xio: fastpath improvements (Raju Kurunkad)
+* msgr: xio: fix ip and nonce (Raju Kurunkad)
+* msgr: xio: improve lane assignment (Vu Pham)
+* msgr: xio: misc fixes (#10735 Matt Benjamin, Kefu Chai, Danny Al-Gaaf, Raju Kurunkad, Vu Pham, Casey Bodley)
+* msgr: xio: sync with accellio v1.4 (Vu Pham)
+* msg: significantly reduce minimal memory usage of connections (`pr#7567 <http://github.com/ceph/ceph/pull/7567>`_, Piotr Dałek)
+* msg/simple: pipe: memory leak when signature check failed (`pr#7096 <http://github.com/ceph/ceph/pull/7096>`_, Ruifeng Yang)
+* msg/simple: remove unneeded friend declarations (`pr#6924 <http://github.com/ceph/ceph/pull/6924>`_, Michal Jarzabek)
+* msg: unit tests (Haomai Wang)
+* msg/xio: fix compilation (`pr#7479 <http://github.com/ceph/ceph/pull/7479>`_, Roi Dayan)
+* msg/xio: fixes (`pr#7603 <http://github.com/ceph/ceph/pull/7603>`_, Roi Dayan)
+* mstart: start rgw on different ports as well (`pr#8167 <http://github.com/ceph/ceph/pull/8167>`_, Abhishek Lekshmanan)
+* nfs for rgw (Matt Benjamin, Orit Wasserman) (`pr#7634 <http://github.com/ceph/ceph/pull/7634>`_, Yehuda Sadeh, Matt Benjamin)
+* objectcacher: misc bug fixes (Jianpeng Ma)
+* objecter: avoid recursive lock of Objecter::rwlock (`pr#7343 <http://github.com/ceph/ceph/pull/7343>`_, Yan, Zheng)
+* organizationmap: modify org mail info. (`pr#7240 <http://github.com/ceph/ceph/pull/7240>`_, Xiaowei Chen)
+* os/bluestore: a few fixes (`pr#8193 <http://github.com/ceph/ceph/pull/8193>`_, Sage Weil)
+* os/bluestore/BlueFS: Before reap ioct, it should wait io complete (`pr#8178 <http://github.com/ceph/ceph/pull/8178>`_, Jianpeng Ma)
+* os/bluestore/BlueStore: Don't leak trim overlay data before write. (`pr#7895 <http://github.com/ceph/ceph/pull/7895>`_, Jianpeng Ma)
+* os/bluestore: ceph-bluefs-tool fixes (`issue#15261 <http://tracker.ceph.com/issues/15261>`_, `pr#8292 <http://github.com/ceph/ceph/pull/8292>`_, Venky Shankar)
+* os/bluestore: clone overlay data (`pr#7860 <http://github.com/ceph/ceph/pull/7860>`_, Jianpeng Ma)
+* os/bluestore: fix assert (`issue#14436 <http://tracker.ceph.com/issues/14436>`_, `pr#7293 <http://github.com/ceph/ceph/pull/7293>`_, xie xingguo)
+* os/bluestore: fix a typo in SPDK path parsing (`pr#7601 <http://github.com/ceph/ceph/pull/7601>`_, Jianjian Huo)
+* os/bluestore: fix bluestore_wal_transaction_t encoding test (`pr#7342 <http://github.com/ceph/ceph/pull/7342>`_, Kefu Chai)
+* os/bluestore: fix bluestore_wal_transaction_t encoding test (`pr#7419 <http://github.com/ceph/ceph/pull/7419>`_, Kefu Chai, Brad Hubbard)
+* os/bluestore: insert new onode to the front position of onode LRU (`pr#7492 <http://github.com/ceph/ceph/pull/7492>`_, Jianjian Huo)
+* os/bluestore/KernelDevice: force block size (`pr#8006 <http://github.com/ceph/ceph/pull/8006>`_, Sage Weil)
+* os/bluestore: make bluestore_sync_transaction = true can work. (`pr#7674 <http://github.com/ceph/ceph/pull/7674>`_, Jianpeng Ma)
+* os/bluestore/NVMEDevice: make IO thread using dpdk launch (`pr#8160 <http://github.com/ceph/ceph/pull/8160>`_, Haomai Wang)
+* os/bluestore/NVMEDevice: refactor probe/attach codes and support zero command (`pr#7647 <http://github.com/ceph/ceph/pull/7647>`_, Haomai Wang)
+* os/bluestore: revamp BlueFS bdev management and add perfcounters (`issue#15376 <http://tracker.ceph.com/issues/15376>`_, `pr#8431 <http://github.com/ceph/ceph/pull/8431>`_, Sage Weil)
+* os/bluestore: small fixes in bluestore StupidAllocator (`pr#8101 <http://github.com/ceph/ceph/pull/8101>`_, Jianjian Huo)
+* os/bluestore: use intrusive_ptr for Dir (`pr#7247 <http://github.com/ceph/ceph/pull/7247>`_, Igor Fedotov)
+* osd: add cache hint when pushing raw clone during recovery (`pr#7069 <http://github.com/ceph/ceph/pull/7069>`_, Zhiqiang Wang)
+* osd: Add config option osd_read_ec_check_for_errors for testing (`pr#5865 <http://github.com/ceph/ceph/pull/5865>`_, David Zafman)
+* osd: add latency perf counters for tier operations (Xinze Chi)
+* osd: add misc perfcounters (Xinze Chi)
+* osd: add missing newline to usage message (`pr#7613 <http://github.com/ceph/ceph/pull/7613>`_, Willem Jan Withagen)
+* osd: add osd op queue latency perfcounter (`pr#5793 <http://github.com/ceph/ceph/pull/5793>`_, Haomai Wang)
+* osd: add pin/unpin support to cache tier (11066) (`pr#6326 <http://github.com/ceph/ceph/pull/6326>`_, Zhiqiang Wang)
+* osd: add 'proxy' cache mode (`issue#12814 <http://tracker.ceph.com/issues/12814>`_, `pr#8210 <http://github.com/ceph/ceph/pull/8210>`_, Sage Weil)
+* osd: add scrub persist/query API (`issue#13505 <http://tracker.ceph.com/issues/13505>`_, `pr#6898 <http://github.com/ceph/ceph/pull/6898>`_, Kefu Chai, Samuel Just)
+* osd: add simple sleep injection in recovery (Sage Weil)
+* osd: add the support of per pool scrub priority (`pr#7062 <http://github.com/ceph/ceph/pull/7062>`_, Zhiqiang Wang)
+* osd: a fix for HeartbeatDispatcher and cleanups (`pr#7550 <http://github.com/ceph/ceph/pull/7550>`_, Kefu Chai)
+* osd: Allow repair of history.last_epoch_started using config (`pr#6793 <http://github.com/ceph/ceph/pull/6793>`_, David Zafman)
+* osd: allow SEEK_HOLE/SEEK_DATA for sparse read (Zhiqiang Wang)
+* osd: auto repair EC pool (`issue#12754 <http://tracker.ceph.com/issues/12754>`_, `pr#6196 <http://github.com/ceph/ceph/pull/6196>`_, Guang Yang)
+* osd: avoid calculating crush mapping for most ops (`pr#6371 <http://github.com/ceph/ceph/pull/6371>`_, Sage Weil)
+* osd: avoid debug std::string initialization in PG::get/put (`pr#7117 <http://github.com/ceph/ceph/pull/7117>`_, Evgeniy Firsov)
+* osd: avoid double-check for replaying and can_checkpoint() in FileStore::_check_replay_guard (`pr#6471 <http://github.com/ceph/ceph/pull/6471>`_, Ning Yao)
+* osd: avoid duplicate op->mark_started in ReplicatedBackend (`pr#6689 <http://github.com/ceph/ceph/pull/6689>`_, Jacek J. �akis)
+* osd: avoid dup omap sets for in pg metadata (Sage Weil)
+* osd: avoid FORCE updating digest been overwritten by MAYBE when comparing scrub map (`pr#7051 <http://github.com/ceph/ceph/pull/7051>`_, Zhiqiang Wang)
+* osd: avoid multiple hit set insertions (Zhiqiang Wang)
+* osd: avoid osd_op_thread suicide because osd_scrub_sleep (`pr#7009 <http://github.com/ceph/ceph/pull/7009>`_, Jianpeng Ma)
+* osd: avoid transaction append in some cases (Sage Weil)
+* osd: bail out of _committed_osd_maps if we are shutting down (`pr#8267 <http://github.com/ceph/ceph/pull/8267>`_, Samuel Just)
+* osd: blockdevice: avoid implicit cast and add guard (`pr#7460 <http://github.com/ceph/ceph/pull/7460>`_, xie xingguo)
+* osd: bluefs: fix alignment for odd page sizes (`pr#7900 <http://github.com/ceph/ceph/pull/7900>`_, Dan Mick)
+* osd: bluestore: add 'override' to virtual functions (`pr#7886 <http://github.com/ceph/ceph/pull/7886>`_, Michal Jarzabek)
+* osd: bluestore: allow _dump_onode dynamic accept log level (`pr#7995 <http://github.com/ceph/ceph/pull/7995>`_, Jianpeng Ma)
+* osd: bluestore/blockdevice: use std::mutex et al (`pr#7568 <http://github.com/ceph/ceph/pull/7568>`_, Sage Weil)
+* osd: bluestore: bluefs: fix several small bugs (`issue#14344 <http://tracker.ceph.com/issues/14344>`_, `issue#14343 <http://tracker.ceph.com/issues/14343>`_, `pr#7200 <http://github.com/ceph/ceph/pull/7200>`_, xie xingguo)
+* osd: bluestore/BlueFS: initialize super block_size earlier in mkfs (`pr#7535 <http://github.com/ceph/ceph/pull/7535>`_, Sage Weil)
+* osd: bluestore: don't include when building without libaio (`issue#14207 <http://tracker.ceph.com/issues/14207>`_, `pr#7169 <http://github.com/ceph/ceph/pull/7169>`_, Mykola Golub)
+* osd: bluestore: fix bluestore onode_t attr leak (`pr#7125 <http://github.com/ceph/ceph/pull/7125>`_, Ning Yao)
+* osd: bluestore: fix bluestore_wal_transaction_t encoding test (`pr#7168 <http://github.com/ceph/ceph/pull/7168>`_, Kefu Chai)
+* osd: bluestore: fix check for write falling within the same extent (`issue#14954 <http://tracker.ceph.com/issues/14954>`_, `pr#7892 <http://github.com/ceph/ceph/pull/7892>`_, Jianpeng Ma)
+* osd: BlueStore: fix fsck and blockdevice read-relevant issue (`pr#7362 <http://github.com/ceph/ceph/pull/7362>`_, xie xingguo)
+* osd: BlueStore: fix null pointer access (`issue#14561 <http://tracker.ceph.com/issues/14561>`_, `pr#7435 <http://github.com/ceph/ceph/pull/7435>`_, xie xingguo)
+* osd: bluestore: fix several bugs (`issue#14259 <http://tracker.ceph.com/issues/14259>`_, `issue#14353 <http://tracker.ceph.com/issues/14353>`_, `issue#14260 <http://tracker.ceph.com/issues/14260>`_, `issue#14261 <http://tracker.ceph.com/issues/14261>`_, `pr#7122 <http://github.com/ceph/ceph/pull/7122>`_, xie xingguo)
+* osd: bluestore: fix space rebalancing, collection split, buffered reads (`pr#7196 <http://github.com/ceph/ceph/pull/7196>`_, Sage Weil)
+* osd: bluestore: for overwrite a extent, allocate new extent on min_alloc_size write (`pr#7996 <http://github.com/ceph/ceph/pull/7996>`_, Jianpeng Ma)
+* osd: bluestore: improve fs-type verification and tidy up (`pr#7651 <http://github.com/ceph/ceph/pull/7651>`_, xie xingguo)
+* osd: bluestore, kstore: fix nid overwritten logic (`issue#14407 <http://tracker.ceph.com/issues/14407>`_, `issue#14433 <http://tracker.ceph.com/issues/14433>`_, `pr#7283 <http://github.com/ceph/ceph/pull/7283>`_, xie xingguo)
+* osd: bluestore: misc fixes (`pr#7658 <http://github.com/ceph/ceph/pull/7658>`_, Jianpeng Ma)
+* osd: bluestore: more fixes (`pr#7130 <http://github.com/ceph/ceph/pull/7130>`_, Sage Weil)
+* osd: BlueStore/NVMEDevice: fix compiling and fd leak (`pr#7496 <http://github.com/ceph/ceph/pull/7496>`_, xie xingguo)
+* osd: bluestore: NVMEDevice: fix error handling (`pr#7799 <http://github.com/ceph/ceph/pull/7799>`_, xie xingguo)
+* osd: bluestore: remove unneeded includes (`pr#7870 <http://github.com/ceph/ceph/pull/7870>`_, Michal Jarzabek)
+* osd: bluestore: Revert NVMEDevice task cstor and refresh interface changes (`pr#7729 <http://github.com/ceph/ceph/pull/7729>`_, Haomai Wang)
+* osd: bluestore updates, scrub fixes (`pr#8035 <http://github.com/ceph/ceph/pull/8035>`_, Sage Weil)
+* osd: bluestore: use btree_map for allocator (`pr#7269 <http://github.com/ceph/ceph/pull/7269>`_, Igor Fedotov, Sage Weil)
+* osd: break PG removal into multiple iterations (#10198 Guang Yang)
+* osd: cache proxy-write support (Zhiqiang Wang, Samuel Just)
+* osd: cache tier: add config option for eviction check list size (`pr#6997 <http://github.com/ceph/ceph/pull/6997>`_, Yuan Zhou)
+* osd: call on_new_interval on newly split child PG (`issue#13962 <http://tracker.ceph.com/issues/13962>`_, `pr#6778 <http://github.com/ceph/ceph/pull/6778>`_, Sage Weil)
+* osd: cancel failure reports if we fail to rebind network (`pr#6278 <http://github.com/ceph/ceph/pull/6278>`_, Xinze Chi)
+* osdc: Fix race condition with tick_event and shutdown (`issue#14256 <http://tracker.ceph.com/issues/14256>`_, `pr#7151 <http://github.com/ceph/ceph/pull/7151>`_, Adam C. Emerson)
+* osd: change mutex to spinlock to optimize thread context switch. (`pr#6492 <http://github.com/ceph/ceph/pull/6492>`_, Xiaowei Chen)
+* osd: check do_shutdown before do_restart (`pr#6547 <http://github.com/ceph/ceph/pull/6547>`_, Xiaoxi Chen)
+* osd: check health state before pre_booting (`issue#14181 <http://tracker.ceph.com/issues/14181>`_, `pr#7053 <http://github.com/ceph/ceph/pull/7053>`_, Xiaoxi Chen)
+* osd: check scrub state when handling map (Jianpeng Ma)
+* osd: clarify the scrub result report (`pr#6534 <http://github.com/ceph/ceph/pull/6534>`_, Li Wang)
+* osd/ClassHandler: only dlclose() the classes not missing (`pr#8354 <http://github.com/ceph/ceph/pull/8354>`_, Kefu Chai)
+* osd: clean up CMPXATTR checks (`pr#5961 <http://github.com/ceph/ceph/pull/5961>`_, Jianpeng Ma)
+* osd: clean up some constness, privateness (Kefu Chai)
+* osd: clean up temp object if copy-from fails (`pr#8487 <http://github.com/ceph/ceph/pull/8487>`_, Sage Weil)
+* osd: clean up temp object if promotion fails (Jianpeng Ma)
+* osd: clear pg_stat_queue after stopping pgs (`issue#14212 <http://tracker.ceph.com/issues/14212>`_, `pr#7091 <http://github.com/ceph/ceph/pull/7091>`_, Sage Weil)
+* osdc/Objecter: allow per-pool calls to op_cancel_writes (John Spray)
+* osdc/Objecter: dout log after assign tid (`pr#8202 <http://github.com/ceph/ceph/pull/8202>`_, Xinze Chi)
+* osdc/Objecter: fix narrow race with tid assignment (`issue#14364 <http://tracker.ceph.com/issues/14364>`_, `pr#7981 <http://github.com/ceph/ceph/pull/7981>`_, Sage Weil)
+* osdc/Objecter: use full pgid hash in PGNLS ops (`pr#8378 <http://github.com/ceph/ceph/pull/8378>`_, Sage Weil)
+* osd: configure promotion based on write recency (Zhiqiang Wang)
+* osd: consider high/low mode when putting agent to sleep (`issue#14752 <http://tracker.ceph.com/issues/14752>`_, `pr#7631 <http://github.com/ceph/ceph/pull/7631>`_, Sage Weil)
+* osd: constrain collections to meta and PGs (normal and temp) (Sage Weil)
+* osd: correctly handle small osd_scrub_interval_randomize_ratio (`pr#7147 <http://github.com/ceph/ceph/pull/7147>`_, Samuel Just)
+* osd: defer decoding of MOSDRepOp/MOSDRepOpReply (`pr#6503 <http://github.com/ceph/ceph/pull/6503>`_, Xinze Chi)
+* osd: delay populating in-memory PG log hashmaps (`pr#6425 <http://github.com/ceph/ceph/pull/6425>`_, Piotr Dałek)
+* osd: disable filestore_xfs_extsize by default (`issue#14397 <http://tracker.ceph.com/issues/14397>`_, `pr#7265 <http://github.com/ceph/ceph/pull/7265>`_, Ken Dreyer)
+* osd: do not keep ref of old osdmap in pg (`issue#13990 <http://tracker.ceph.com/issues/13990>`_, `pr#7007 <http://github.com/ceph/ceph/pull/7007>`_, Kefu Chai)
+* osd: don't do random deep scrubs for user initiated scrubs (`pr#6673 <http://github.com/ceph/ceph/pull/6673>`_, David Zafman)
+* osd: don't send dup MMonGetOSDMap requests (Sage Weil, Kefu Chai)
+* osd: don't update epoch and rollback_info objects attrs if there is no need (`pr#6555 <http://github.com/ceph/ceph/pull/6555>`_, Ning Yao)
+* osd: drop deprecated removal pg type (`pr#6970 <http://github.com/ceph/ceph/pull/6970>`_, Igor Podoski)
+* osd: drop fiemap len=0 logic (`pr#7267 <http://github.com/ceph/ceph/pull/7267>`_, Sage Weil)
+* osd: drop the interim set from load_pgs() (`pr#6277 <http://github.com/ceph/ceph/pull/6277>`_, Piotr Dałek)
+* osd: dump number of missing objects for each peer with pg query (`pr#6058 <http://github.com/ceph/ceph/pull/6058>`_, Guang Yang)
+* osd: duplicated clear for peer_missing (`pr#8315 <http://github.com/ceph/ceph/pull/8315>`_, Ning Yao)
+* osd: EIO injection (David Zhang)
+* osd: elminiate txn apend, ECSubWrite copy (Samuel Just)
+* osd: enable perfcounters on sharded work queue mutexes (`pr#6455 <http://github.com/ceph/ceph/pull/6455>`_, Jacek J. �akis)
+* osd: ensure new osdmaps commit before publishing them to pgs (`issue#15073 <http://tracker.ceph.com/issues/15073>`_, `pr#8096 <http://github.com/ceph/ceph/pull/8096>`_, Sage Weil)
+* osd: erasure-code: drop entries according to LRU (Andreas-Joachim Peters)
+* osd: erasure-code: fix SHEC floating point bug (#12936 Loic Dachary)
+* osd: erasure-code: update to ISA-L 2.14 (Yuan Zhou)
+* osd: filejournal: cleanup (David Zafman)
+* osd: FileJournal: _fdump wrongly returns if journal is currently unreadable. (`issue#13626 <http://tracker.ceph.com/issues/13626>`_, `pr#6406 <http://github.com/ceph/ceph/pull/6406>`_, xie xingguo)
+* osd: FileJournal: fix return code of create method (`issue#14134 <http://tracker.ceph.com/issues/14134>`_, `pr#6988 <http://github.com/ceph/ceph/pull/6988>`_, xie xingguo)
+* osd: FileJournal: reduce locking scope in write_aio_bl (`issue#12789 <http://tracker.ceph.com/issues/12789>`_, `pr#5670 <http://github.com/ceph/ceph/pull/5670>`_, Zhi Zhang)
+* osd: filejournal: report journal entry count (`pr#7643 <http://github.com/ceph/ceph/pull/7643>`_, tianqing)
+* osd: FileJournal: support batch peak and pop from writeq (`pr#6701 <http://github.com/ceph/ceph/pull/6701>`_, Xinze Chi)
+* osd: FileStore: add a field indicate xattr only one chunk for set xattr. (`pr#6244 <http://github.com/ceph/ceph/pull/6244>`_, Jianpeng Ma)
+* osd: FileStore: Added O_DSYNC write scheme (`pr#7752 <http://github.com/ceph/ceph/pull/7752>`_, Somnath Roy)
+* osd: FileStore: add error check for object_map->sync() (`pr#7281 <http://github.com/ceph/ceph/pull/7281>`_, Chendi Xue)
+* osd: FileStore: cleanup: remove obsolete option "filestore_xattr_use_omap" (`issue#14356 <http://tracker.ceph.com/issues/14356>`_, `pr#7217 <http://github.com/ceph/ceph/pull/7217>`_, Vikhyat Umrao)
+* osd: filestore: clone using splice (Jianpeng Ma)
+* osd: FileStore: conditional collection of drive metadata (`pr#6956 <http://github.com/ceph/ceph/pull/6956>`_, Somnath Roy)
+* osd: filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE (`pr#7768 <http://github.com/ceph/ceph/pull/7768>`_, xinxin shu)
+* osd: filestore: fast abort if statfs encounters ENOENT (`pr#7703 <http://github.com/ceph/ceph/pull/7703>`_, xie xingguo)
+* osd: FileStore: fix initialization order for m_disable_wbthrottle (`pr#8067 <http://github.com/ceph/ceph/pull/8067>`_, Samuel Just)
+* osd: filestore: fix race condition with split vs collection_move_rename and long object names (`issue#14766 <http://tracker.ceph.com/issues/14766>`_, `pr#8136 <http://github.com/ceph/ceph/pull/8136>`_, Samuel Just)
+* osd: filestore: fix recursive lock (Xinxin Shu)
+* osd: filestore: fix result code overwritten for clone (`issue#14817 <http://tracker.ceph.com/issues/14817>`_, `issue#14827 <http://tracker.ceph.com/issues/14827>`_, `pr#7711 <http://github.com/ceph/ceph/pull/7711>`_, xie xingguo)
+* osd: filestore: fix wrong scope of result code for error cases during mkfs (`issue#14814 <http://tracker.ceph.com/issues/14814>`_, `pr#7704 <http://github.com/ceph/ceph/pull/7704>`_, xie xingguo)
+* osd: filestore: fix wrong scope of result code for error cases during mount (`issue#14815 <http://tracker.ceph.com/issues/14815>`_, `pr#7707 <http://github.com/ceph/ceph/pull/7707>`_, xie xingguo)
+* osd: FileStore: LFNIndex: remove redundant local variable 'obj'. (`issue#13552 <http://tracker.ceph.com/issues/13552>`_, `pr#6333 <http://github.com/ceph/ceph/pull/6333>`_, xiexingguo)
+* osd: FileStore: modify the format of colon (`pr#7333 <http://github.com/ceph/ceph/pull/7333>`_, Donghai Xu)
+* osd: FileStore:: optimize lfn_unlink (`pr#6649 <http://github.com/ceph/ceph/pull/6649>`_, Jianpeng Ma)
+* osd: FileStore: potential memory leak if _fgetattrs fails (`issue#13597 <http://tracker.ceph.com/issues/13597>`_, `pr#6377 <http://github.com/ceph/ceph/pull/6377>`_, xie xingguo)
+* osd: FileStore: print file name before osd assert if read file failed (`pr#7111 <http://github.com/ceph/ceph/pull/7111>`_, Ji Chen)
+* osd: FileStore: remove __SWORD_TYPE dependency (`pr#6263 <http://github.com/ceph/ceph/pull/6263>`_, John Coyle)
+* osd: FileStore: remove unused local variable 'handle' (`pr#6381 <http://github.com/ceph/ceph/pull/6381>`_, xie xingguo)
+* osd: filestore: restructure journal and op queue throttling (`pr#7767 <http://github.com/ceph/ceph/pull/7767>`_, Samuel Just)
+* osd: FileStore: support multiple ondisk finish and apply finishers (`pr#6486 <http://github.com/ceph/ceph/pull/6486>`_, Xinze Chi, Haomai Wang)
+* osd: FileStore: use pwritev instead of lseek+writev (`pr#7349 <http://github.com/ceph/ceph/pull/7349>`_, Haomai Wang, Tao Chang)
+* osd: fix bogus scrub results when missing a clone (`issue#12738 <http://tracker.ceph.com/issues/12738>`_, `issue#12740 <http://tracker.ceph.com/issues/12740>`_, `pr#5783 <http://github.com/ceph/ceph/pull/5783>`_, David Zafman)
+* osd: fix broken balance / localized read handling (`issue#13491 <http://tracker.ceph.com/issues/13491>`_, `pr#6364 <http://github.com/ceph/ceph/pull/6364>`_, Jason Dillaman)
+* osd: fix bug in last_* PG state timestamps (`pr#6517 <http://github.com/ceph/ceph/pull/6517>`_, Li Wang)
+* osd: fix bugs for omap ops (`pr#8230 <http://github.com/ceph/ceph/pull/8230>`_, Jianpeng Ma)
+* osd: fix check_for_full (Henry Chang)
+* osd: fix ClassHandler::ClassData::get_filter() (`pr#6747 <http://github.com/ceph/ceph/pull/6747>`_, Yan, Zheng)
+* osd: fix/clean up full map request handling (`pr#8446 <http://github.com/ceph/ceph/pull/8446>`_, Sage Weil)
+* osd: fix debug message in OSD::is_healthy (`pr#6226 <http://github.com/ceph/ceph/pull/6226>`_, Xiaoxi Chen)
+* osd: fix dirty accounting in make_writeable (Zhiqiang Wang)
+* osd: fix dirtying info without correctly setting drity_info field (`pr#8275 <http://github.com/ceph/ceph/pull/8275>`_, xie xingguo)
+* osd: fix dump_ops_in_flight races (`issue#8885 <http://tracker.ceph.com/issues/8885>`_, `pr#8044 <http://github.com/ceph/ceph/pull/8044>`_, David Zafman)
+* osd: fix dup promotion lost op bug (Zhiqiang Wang)
+* osd: fix endless repair when object is unrecoverable (Jianpeng Ma, Kefu Chai)
+* osd: fix epoch check in handle_pg_create (`pr#8382 <http://github.com/ceph/ceph/pull/8382>`_, Samuel Just)
+* osd: fixes for several cases where op result code was not checked or set (`issue#13566 <http://tracker.ceph.com/issues/13566>`_, `pr#6347 <http://github.com/ceph/ceph/pull/6347>`_, xie xingguo)
+* osd: fix failure report handling during ms_handle_connect() (`pr#8348 <http://github.com/ceph/ceph/pull/8348>`_, xie xingguo)
+* osd: fix FileStore::_destroy_collection error return code (`pr#6612 <http://github.com/ceph/ceph/pull/6612>`_, Ruifeng Yang)
+* osd: fix forced prmootion for CALL ops (`issue#14745 <http://tracker.ceph.com/issues/14745>`_, `pr#7617 <http://github.com/ceph/ceph/pull/7617>`_, Sage Weil)
+* osd: fix fusestore hanging during stop/quit (`issue#14786 <http://tracker.ceph.com/issues/14786>`_, `pr#7677 <http://github.com/ceph/ceph/pull/7677>`_, xie xingguo)
+* osd: fix hitset object naming to use GMT (Kefu Chai)
+* osd: fix inaccurate counter and skip over queueing an empty transaction (`pr#7754 <http://github.com/ceph/ceph/pull/7754>`_, xie xingguo)
+* osd: fix incorrect throttle in WBThrottle (`pr#6713 <http://github.com/ceph/ceph/pull/6713>`_, Zhang Huan)
+* osd: fix invalid list traversal in process_copy_chunk (`pr#7511 <http://github.com/ceph/ceph/pull/7511>`_, Samuel Just)
+* osd: fix lack of object unblock when flush fails (`issue#14511 <http://tracker.ceph.com/issues/14511>`_, `pr#7584 <http://github.com/ceph/ceph/pull/7584>`_, Igor Fedotov)
+* osd: fix log info (`pr#8273 <http://github.com/ceph/ceph/pull/8273>`_, Wei Jin)
+* osd: fix misc memory leaks (Sage Weil)
+* osd: fix MOSDOp encoding (`pr#6174 <http://github.com/ceph/ceph/pull/6174>`_, Sage Weil)
+* osd: fix MOSDRepScrub reference counter in replica_scrub (`pr#6730 <http://github.com/ceph/ceph/pull/6730>`_, Jie Wang)
+* osd: fix negative degraded stats during backfill (Guang Yang)
+* osd: fix null pointer access and race condition (`issue#14072 <http://tracker.ceph.com/issues/14072>`_, `pr#6916 <http://github.com/ceph/ceph/pull/6916>`_, xie xingguo)
+* osd: fix osdmap dump of blacklist items (John Spray)
+* osd: fix overload of '==' operator for pg_stat_t (`issue#14921 <http://tracker.ceph.com/issues/14921>`_, `pr#7842 <http://github.com/ceph/ceph/pull/7842>`_, xie xingguo)
+* osd: fix peek_queue locking in FileStore (Xinze Chi)
+* osd: fix pg resurrection (#11429 Samuel Just)
+* osd: fix promotion vs full cache tier (Samuel Just)
+* osd: fix race condition for heartbeat_need_update (`issue#14387 <http://tracker.ceph.com/issues/14387>`_, `pr#7739 <http://github.com/ceph/ceph/pull/7739>`_, xie xingguo)
+* osd: fix reactivate (check OSDSuperblock in mkfs() when we already have the superblock) (`issue#13586 <http://tracker.ceph.com/issues/13586>`_, `pr#6385 <http://github.com/ceph/ceph/pull/6385>`_, Vicente Cheng)
+* osd: fix reference count, rare race condition etc. (`pr#8254 <http://github.com/ceph/ceph/pull/8254>`_, xie xingguo)
+* osd: fix replay requeue when pg is still activating (#13116 Samuel Just)
+* osd: fix return value from maybe_handle_cache_detail() (`pr#7593 <http://github.com/ceph/ceph/pull/7593>`_, Igor Fedotov)
+* osd: fix rollback_info_trimmed_to before index() (`issue#13965 <http://tracker.ceph.com/issues/13965>`_, `pr#6801 <http://github.com/ceph/ceph/pull/6801>`_, Samuel Just)
+* osd: fix scrub start hobject (`pr#7467 <http://github.com/ceph/ceph/pull/7467>`_, Sage Weil)
+* osd: fix scrub stat bugs (Sage Weil, Samuel Just)
+* osd: fix snap flushing from cache tier (again) (#11787 Samuel Just)
+* osd: fix snap handling on promotion (#11296 Sam Just)
+* osd: fix sparse-read result code checking logic (`issue#14151 <http://tracker.ceph.com/issues/14151>`_, `pr#7016 <http://github.com/ceph/ceph/pull/7016>`_, xie xingguo)
+* osd: fix temp-clearing (David Zafman)
+* osd: fix temp object removal after upgrade (`issue#13862 <http://tracker.ceph.com/issues/13862>`_, `pr#6976 <http://github.com/ceph/ceph/pull/6976>`_, David Zafman)
+* osd: fix tick relevant issues (`pr#8369 <http://github.com/ceph/ceph/pull/8369>`_, xie xingguo)
+* osd: fix trivial scrub bug (`pr#6533 <http://github.com/ceph/ceph/pull/6533>`_, Li Wang)
+* osd: fix two scrub relevant issues (`pr#8462 <http://github.com/ceph/ceph/pull/8462>`_, xie xingguo)
+* osd: fix unnecessary object promotion when deleting from cache pool (`issue#13894 <http://tracker.ceph.com/issues/13894>`_, `pr#7537 <http://github.com/ceph/ceph/pull/7537>`_, Igor Fedotov)
+* osd: fix wip (l_osd_op_wip) perf counter and remove repop_map (`pr#7077 <http://github.com/ceph/ceph/pull/7077>`_, Xinze Chi)
+* osd: fix wrongly placed assert and some cleanups (`pr#6766 <http://github.com/ceph/ceph/pull/6766>`_, xiexingguo, xie xingguo)
+* osd: fix wrong return type of find_osd_on_ip() (`issue#14872 <http://tracker.ceph.com/issues/14872>`_, `pr#7812 <http://github.com/ceph/ceph/pull/7812>`_, xie xingguo)
+* osd: fix wrong use of right parenthesis in localized read logic (`pr#6566 <http://github.com/ceph/ceph/pull/6566>`_, Jie Wang)
+* osd: force promotion for ops EC can't handle (Zhiqiang Wang)
+* osd: ghobject_t: use ! instead of @ as a separator (`pr#7595 <http://github.com/ceph/ceph/pull/7595>`_, Sage Weil)
+* osd: handle dup pg_create that races with pg deletion (`pr#8033 <http://github.com/ceph/ceph/pull/8033>`_, Sage Weil)
+* osd: handle log split with overlapping entries (#11358 Samuel Just)
+* osd: ignore non-existent osds in unfound calc (#10976 Mykola Golub)
+* osd: improve behavior on machines with large memory pages (Steve Capper)
+* osd: improve temperature calculation for cache tier agent (`pr#4737 <http://github.com/ceph/ceph/pull/4737>`_, MingXin Liu)
+* osd: include a temp namespace within each collection/pgid (Sage Weil)
+* osd: increase default max open files (Owen Synge)
+* osd: initialize last_recalibrate field at construction (`pr#8071 <http://github.com/ceph/ceph/pull/8071>`_, xie xingguo)
+* osd: init started to 0 (`issue#13206 <http://tracker.ceph.com/issues/13206>`_, `pr#6107 <http://github.com/ceph/ceph/pull/6107>`_, Sage Weil)
+* osd: KeyValueStore: don't queue NULL context (`pr#6783 <http://github.com/ceph/ceph/pull/6783>`_, Haomai Wang)
+* osd: KeyValueStore: fix return code of mkfs (`pr#7036 <http://github.com/ceph/ceph/pull/7036>`_, xie xingguo)
+* osd: KeyValueStore: fix the name's typo of keyvaluestore_default_strip_size (`pr#6375 <http://github.com/ceph/ceph/pull/6375>`_, Zhi Zhang)
+* osd: KeyValueStore: fix wrongly placed assert (`issue#14176 <http://tracker.ceph.com/issues/14176>`_, `issue#14178 <http://tracker.ceph.com/issues/14178>`_, `pr#7047 <http://github.com/ceph/ceph/pull/7047>`_, xie xingguo)
+* osd: keyvaluestore: misc fixes (Varada Kari)
+* osd: kstore: fix a race condition in _txc_finish() (`pr#7804 <http://github.com/ceph/ceph/pull/7804>`_, Jianjian Huo)
+* osd: kstore: latency breakdown (`pr#7850 <http://github.com/ceph/ceph/pull/7850>`_, James Liu)
+* osd: kstore: several small fixes (`issue#14351 <http://tracker.ceph.com/issues/14351>`_, `issue#14352 <http://tracker.ceph.com/issues/14352>`_, `pr#7213 <http://github.com/ceph/ceph/pull/7213>`_, xie xingguo)
+* osd: kstore: small fixes to kstore (`issue#14204 <http://tracker.ceph.com/issues/14204>`_, `pr#7095 <http://github.com/ceph/ceph/pull/7095>`_, xie xingguo)
+* osd: kstore: sync up kstore with recent bluestore updates (`pr#7681 <http://github.com/ceph/ceph/pull/7681>`_, Jianjian Huo)
+* osd: low and high speed flush modes (Mingxin Liu)
+* osd: make backend and block device code a bit more generic (`pr#6759 <http://github.com/ceph/ceph/pull/6759>`_, Sage Weil)
+* osd: make list_missing query missing_loc.needs_recovery_map (`pr#6298 <http://github.com/ceph/ceph/pull/6298>`_, Guang Yang)
+* osd: make suicide timeouts individually configurable (Samuel Just)
+* osdmap: remove unused local variables (`pr#6864 <http://github.com/ceph/ceph/pull/6864>`_, luo kexue)
+* osdmap: rm nonused variable (`pr#8423 <http://github.com/ceph/ceph/pull/8423>`_, Wei Jin)
+* osd: memstore: fix alignment of Page for test_pageset (`pr#7587 <http://github.com/ceph/ceph/pull/7587>`_, Casey Bodley)
+* osd: memstore: fix two bugs (`pr#6963 <http://github.com/ceph/ceph/pull/6963>`_, Casey Bodley, Sage Weil)
+* osd: merge local_t and op_t txn to single one (`pr#6439 <http://github.com/ceph/ceph/pull/6439>`_, Xinze Chi)
+* osd: merge multiple setattr calls into a setattrs call (Xinxin Shu)
+* osd: min_write_recency_for_promote & min_read_recency_for_promote are tiering only (`pr#8081 <http://github.com/ceph/ceph/pull/8081>`_, huanwen ren)
+* osd: misc FileStore fixes (`issue#14192 <http://tracker.ceph.com/issues/14192>`_, `issue#14188 <http://tracker.ceph.com/issues/14188>`_, `issue#14194 <http://tracker.ceph.com/issues/14194>`_, `issue#14187 <http://tracker.ceph.com/issues/14187>`_, `issue#14186 <http://tracker.ceph.com/issues/14186>`_, `pr#7059 <http://github.com/ceph/ceph/pull/7059>`_, xie xingguo)
+* osd: misc fixes (Ning Yao, Kefu Chai, Xinze Chi, Zhiqiang Wang, Jianpeng Ma)
+* osd: misc optimization for map utilization (`pr#6950 <http://github.com/ceph/ceph/pull/6950>`_, Ning Yao)
+* osd, mon: fix exit issue (`pr#7420 <http://github.com/ceph/ceph/pull/7420>`_, Jiaying Ren)
+* osd,mon: log leveldb and rocksdb to ceph log (`pr#6921 <http://github.com/ceph/ceph/pull/6921>`_, Sage Weil)
+* osd: more fixes for incorrectly dirtying info; resend reply for duplicated scrub-reserve req (`pr#8291 <http://github.com/ceph/ceph/pull/8291>`_, xie xingguo)
+* osd: move newest decode version of MOSDOp and MOSDOpReply to the front (`pr#6642 <http://github.com/ceph/ceph/pull/6642>`_, Jacek J. �akis)
+* osd: move scrub in OpWQ (Samuel Just)
+* osd: new and delete ObjectStore::Transaction in a function is not necessary (`pr#6299 <http://github.com/ceph/ceph/pull/6299>`_, Ruifeng Yang)
+* osd: newstore: misc updates (including kv and os/fs stuff) (`pr#6609 <http://github.com/ceph/ceph/pull/6609>`_, Sage Weil)
+* osd: newstore prototype (Sage Weil)
+* osd: note down the number of missing clones (`pr#6654 <http://github.com/ceph/ceph/pull/6654>`_, Kefu Chai)
+* osd: ObjectStore internal API refactor (Sage Weil)
+* osd: Omap small bugs adapted (`pr#6669 <http://github.com/ceph/ceph/pull/6669>`_, Jianpeng Ma, David Zafman)
+* osd: optimize clone write path if object-map is enabled (`pr#6403 <http://github.com/ceph/ceph/pull/6403>`_, xinxin shu)
+* osd: optimize get_object_context (`pr#6305 <http://github.com/ceph/ceph/pull/6305>`_, Jianpeng Ma)
+* osd: optimize MOSDOp/do_op/handle_op (`pr#5211 <http://github.com/ceph/ceph/pull/5211>`_, Jacek J. Lakis)
+* osd: optimize scrub subset_last_update calculation (`pr#6518 <http://github.com/ceph/ceph/pull/6518>`_, Li Wang)
+* osd: optimize the session_handle_reset function (`issue#14182 <http://tracker.ceph.com/issues/14182>`_, `pr#7054 <http://github.com/ceph/ceph/pull/7054>`_, songbaisen)
+* osd: os/chain_xattr: On linux use linux/limits.h for XATTR_NAME_MAX. (`pr#6343 <http://github.com/ceph/ceph/pull/6343>`_, John Coyle)
+* osd/OSD.cc: finish full_map_request every MOSDMap message. (`issue#15130 <http://tracker.ceph.com/issues/15130>`_, `pr#8147 <http://github.com/ceph/ceph/pull/8147>`_, Xiaoxi Chen)
+* osd/OSD: fix build_past_intervals_parallel (`pr#8215 <http://github.com/ceph/ceph/pull/8215>`_, David Zafman)
+* osd/OSDMap: fix typo in summarize_mapping_stats (`pr#8088 <http://github.com/ceph/ceph/pull/8088>`_, Sage Weil)
+* osd: OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from (`issue#14686 <http://tracker.ceph.com/issues/14686>`_, `pr#7553 <http://github.com/ceph/ceph/pull/7553>`_, Xinze Chi)
+* osd: OSDService: Fix typo in osdmap comment (`pr#7275 <http://github.com/ceph/ceph/pull/7275>`_, Brad Hubbard)
+* osd: os: skip checking pg_meta object existance in FileStore (`pr#6870 <http://github.com/ceph/ceph/pull/6870>`_, Ning Yao)
+* osd: partial revert of "ReplicatedPG: result code not correctly set in some cases." (`issue#13796 <http://tracker.ceph.com/issues/13796>`_, `pr#6622 <http://github.com/ceph/ceph/pull/6622>`_, Sage Weil)
+* osd: peer_features includes self (David Zafman)
+* osd: PG::activate(): handle unexpected cached_removed_snaps more gracefully (`issue#14428 <http://tracker.ceph.com/issues/14428>`_, `pr#7309 <http://github.com/ceph/ceph/pull/7309>`_, Alexey Sheplyakov)
+* osd/PG: indicate in pg query output whether ignore_history_les would help (`pr#8156 <http://github.com/ceph/ceph/pull/8156>`_, Sage Weil)
+* osd: PGLog: clean up read_log (`pr#7092 <http://github.com/ceph/ceph/pull/7092>`_, Jie Wang)
+* osd/PGLog: fix warning (`pr#8057 <http://github.com/ceph/ceph/pull/8057>`_, Sage Weil)
+* osd: pg_pool_t: add dictionary for pool options (`issue#13077 <http://tracker.ceph.com/issues/13077>`_, `pr#6081 <http://github.com/ceph/ceph/pull/6081>`_, Mykola Golub)
+* osd/PG: set epoch_created and parent_split_bits for child pg (`issue#15426 <http://tracker.ceph.com/issues/15426>`_, `pr#8552 <http://github.com/ceph/ceph/pull/8552>`_, Kefu Chai)
+* osd: pool size change triggers new interval (#11771 Samuel Just)
+* osd: prepopulate needs_recovery_map when only one peer has missing (#9558 Guang Yang)
+* osd: prevent osd_recovery_sleep from causing recovery-thread suicide (`pr#7065 <http://github.com/ceph/ceph/pull/7065>`_, Jianpeng Ma)
+* osd: probabilistic cache tier promotion throttling (`pr#7465 <http://github.com/ceph/ceph/pull/7465>`_, Sage Weil)
+* osd: randomize deep scrubbing (`pr#6550 <http://github.com/ceph/ceph/pull/6550>`_, Dan van der Ster, Herve Rousseau)
+* osd: randomize scrub times (#10973 Kefu Chai)
+* osd: recovery, peering fixes (#11687 Samuel Just)
+* osd: reduce memory consumption of some structs (`pr#6475 <http://github.com/ceph/ceph/pull/6475>`_, Piotr Dałek)
+* osd: reduce string use in coll_t::calc_str() (`pr#6505 <http://github.com/ceph/ceph/pull/6505>`_, Igor Podoski)
+* osd: refactor scrub and digest recording (Sage Weil)
+* osd: refuse first write to EC object at non-zero offset (Jianpeng Ma)
+* osd: relax reply order on proxy read (#11211 Zhiqiang Wang)
+* osd: release related sources when scrub is interrupted (`pr#6744 <http://github.com/ceph/ceph/pull/6744>`_, Jianpeng Ma)
+* osd: release the message throttle when OpRequest unregistered (`issue#14248 <http://tracker.ceph.com/issues/14248>`_, `pr#7148 <http://github.com/ceph/ceph/pull/7148>`_, Samuel Just)
+* osd: remove __SWORD_TYPE dependency (`pr#6262 <http://github.com/ceph/ceph/pull/6262>`_, John Coyle)
+* osd: remove unused OSDMap::set_weightf() (`issue#14369 <http://tracker.ceph.com/issues/14369>`_, `pr#7231 <http://github.com/ceph/ceph/pull/7231>`_, huanwen ren)
+* osd: remove up_thru_pending field, which is never used (`pr#7991 <http://github.com/ceph/ceph/pull/7991>`_, xie xingguo)
+* osd: reorder bool fields in PGLog struct (`pr#6279 <http://github.com/ceph/ceph/pull/6279>`_, Piotr Dałek)
+* osd: Replace snprintf with faster implementation in eversion_t::get_key_name (`pr#7121 <http://github.com/ceph/ceph/pull/7121>`_, Evgeniy Firsov)
+* osd/ReplicatedPG: be more careful about calling publish_stats_to_osd() (`issue#14962 <http://tracker.ceph.com/issues/14962>`_, `pr#8039 <http://github.com/ceph/ceph/pull/8039>`_, Greg Farnum)
+* osd: replicatedpg: break out loop if we encounter fatal error during do_pg_op() (`issue#14922 <http://tracker.ceph.com/issues/14922>`_, `pr#7844 <http://github.com/ceph/ceph/pull/7844>`_, xie xingguo)
+* osd: ReplicatedPG: clean up unused function (`pr#7211 <http://github.com/ceph/ceph/pull/7211>`_, Xiaowei Chen)
+* osd/ReplicatedPG: clear watches on change after applying repops (`issue#15151 <http://tracker.ceph.com/issues/15151>`_, `pr#8163 <http://github.com/ceph/ceph/pull/8163>`_, Sage Weil)
+* osd/ReplicatedPG: fix promotion recency logic (`issue#14320 <http://tracker.ceph.com/issues/14320>`_, `pr#6702 <http://github.com/ceph/ceph/pull/6702>`_, Sage Weil)
+* osd: ReplicatedPG: remove unused local variables (`issue#13575 <http://tracker.ceph.com/issues/13575>`_, `pr#6360 <http://github.com/ceph/ceph/pull/6360>`_, xiexingguo)
+* osd/ReplicatedPG::_rollback_to: update the OMAP flag (`issue#14777 <http://tracker.ceph.com/issues/14777>`_, `pr#8495 <http://github.com/ceph/ceph/pull/8495>`_, Samuel Just)
+* osd: repop and lost-unfound overhaul (`pr#7765 <http://github.com/ceph/ceph/pull/7765>`_, Samuel Just)
+* osd: require firefly features (David Zafman)
+* osd: reset primary and up_primary when building a new past_interval. (`issue#13471 <http://tracker.ceph.com/issues/13471>`_, `pr#6240 <http://github.com/ceph/ceph/pull/6240>`_, xiexingguo)
+* osd: resolve boot vs NOUP set + clear race (`pr#7483 <http://github.com/ceph/ceph/pull/7483>`_, Sage Weil)
+* osd: scrub: do not assign value if read error (`pr#6568 <http://github.com/ceph/ceph/pull/6568>`_, Li Wang)
+* osd/ScrubStore: remove unused function (`pr#8045 <http://github.com/ceph/ceph/pull/8045>`_, Kefu Chai)
+* osd: set initial crush weight with more precision (Sage Weil)
+* osd: several small cleanups (`pr#7055 <http://github.com/ceph/ceph/pull/7055>`_, xie xingguo)
+* osd: SHEC no longer experimental
+* osd: shut down if we flap too many times in a short period (`pr#6708 <http://github.com/ceph/ceph/pull/6708>`_, Xiaoxi Chen)
+* osd: skip promote for writefull w/ FADVISE_DONTNEED/NOCACHE (`pr#7010 <http://github.com/ceph/ceph/pull/7010>`_, Jianpeng Ma)
+* osd: skip promotion for flush/evict op (Zhiqiang Wang)
+* osd: slightly reduce actual size of pg_log_entry_t (`pr#6690 <http://github.com/ceph/ceph/pull/6690>`_, Piotr Dałek)
+* osd: small fixes to memstore (`issue#14228 <http://tracker.ceph.com/issues/14228>`_, `issue#14229 <http://tracker.ceph.com/issues/14229>`_, `issue#14227 <http://tracker.ceph.com/issues/14227>`_, `pr#7107 <http://github.com/ceph/ceph/pull/7107>`_, xie xingguo)
+* osd: stripe over small xattrs to fit in XFS's 255 byte inline limit (Sage Weil, Ning Yao)
+* osd: support pool level recovery_priority and recovery_op_priority (`pr#5953 <http://github.com/ceph/ceph/pull/5953>`_, Guang Yang)
+* osd: sync object_map on syncfs (Samuel Just)
+* osd: take excl lock of op is rw (Samuel Just)
+* osd: throttle evict ops (Yunchuan Wen)
+* osd: try evicting after flushing is done (`pr#5630 <http://github.com/ceph/ceph/pull/5630>`_, Zhiqiang Wang)
+* osd: upgrades must pass through hammer (Sage Weil)
+* osd: use a temp object for recovery (Sage Weil)
+* osd: use atomic to generate ceph_tid (`pr#7017 <http://github.com/ceph/ceph/pull/7017>`_, Evgeniy Firsov)
+* osd: use blkid to collection partition information (Joseph Handzik)
+* osd: use optimized is_zero in object_stat_sum_t.is_zero() (`pr#7203 <http://github.com/ceph/ceph/pull/7203>`_, Piotr Dałek)
+* osd: use pg id (without shard) when referring the PG (`pr#6236 <http://github.com/ceph/ceph/pull/6236>`_, Guang Yang)
+* osd: use SEEK_HOLE / SEEK_DATA for sparse copy (Xinxin Shu)
+* osd: utime_t, eversion_t, osd_stat_sum_t encoding optimization (`pr#6902 <http://github.com/ceph/ceph/pull/6902>`_, Xinze Chi)
+* osd: WBThrottle cleanups (Jianpeng Ma)
+* osd: WeightedPriorityQueue: move to intrusive containers (`pr#7654 <http://github.com/ceph/ceph/pull/7654>`_, Robert LeBlanc)
+* osd: write file journal optimization (`pr#6484 <http://github.com/ceph/ceph/pull/6484>`_, Xinze Chi)
+* osd: write journal header on clean shutdown (Xinze Chi)
+* os/filestore: enlarge getxattr buffer size (Jianpeng Ma)
+* os/filestore/FileJournal: set block size via config option (`pr#7628 <http://github.com/ceph/ceph/pull/7628>`_, Sage Weil)
+* os/filestore: fix punch hole usage in _zero (`pr#8050 <http://github.com/ceph/ceph/pull/8050>`_, Sage Weil)
+* os/filestore: fix result handling logic of destroy_collection (`pr#7721 <http://github.com/ceph/ceph/pull/7721>`_, xie xingguo)
+* os/filestore: force lfn attrs to be written atomically, restructure name length limits (`pr#8496 <http://github.com/ceph/ceph/pull/8496>`_, Samuel Just)
+* os/filestore: require offset == length == 0 for full object read; add test (`pr#7957 <http://github.com/ceph/ceph/pull/7957>`_, Jianpeng Ma)
+* os/fs: fix io_getevents argument (`pr#7355 <http://github.com/ceph/ceph/pull/7355>`_, Jingkai Yuan)
+* os/fusestore: add error handling (`pr#7395 <http://github.com/ceph/ceph/pull/7395>`_, xie xingguo)
+* os/keyvaluestore: kill KeyValueStore (`pr#7320 <http://github.com/ceph/ceph/pull/7320>`_, Haomai Wang)
+* os/kstore: insert new onode to the front position of onode LRU (`pr#7505 <http://github.com/ceph/ceph/pull/7505>`_, xie xingguo)
+* os/ObjectStore: add custom move operations for ObjectStore::Transaction (`pr#7303 <http://github.com/ceph/ceph/pull/7303>`_, Casey Bodley)
+* os/ObjectStore: add noexcept to ensure move ctor is used (`pr#8421 <http://github.com/ceph/ceph/pull/8421>`_, Kefu Chai)
+* os/ObjectStore: fix _update_op for split dest_cid (`pr#8364 <http://github.com/ceph/ceph/pull/8364>`_, Sage Weil)
+* os/ObjectStore: implement more efficient get_encoded_bytes()  (`pr#7775 <http://github.com/ceph/ceph/pull/7775>`_, Piotr Dałek)
+* os/ObjectStore: make device uuid probe output something friendly (`pr#8418 <http://github.com/ceph/ceph/pull/8418>`_, Sage Weil)
+* os/ObjectStore: try_move_rename in transaction append and add coverage to store_test (`issue#15205 <http://tracker.ceph.com/issues/15205>`_, `pr#8359 <http://github.com/ceph/ceph/pull/8359>`_, Samuel Just)
+* packaging: add build dependency on python devel package (`pr#7205 <http://github.com/ceph/ceph/pull/7205>`_, Josh Durgin)
+* packaging: make infernalis -> jewel upgrade work (`issue#15047 <http://tracker.ceph.com/issues/15047>`_, `pr#8034 <http://github.com/ceph/ceph/pull/8034>`_, Nathan Cutler)
+* packaging: move cephfs repair tools to ceph-common (`issue#15145 <http://tracker.ceph.com/issues/15145>`_, `pr#8133 <http://github.com/ceph/ceph/pull/8133>`_, Boris Ranto, Ken Dreyer)
+* PG: pg down state blocked by osd.x, lost osd.x cannot solve peering stuck (`issue#13531 <http://tracker.ceph.com/issues/13531>`_, `pr#6317 <http://github.com/ceph/ceph/pull/6317>`_, Xiaowei Chen)
+* pybind: add ceph_volume_client interface for Manila and similar frameworks (`pr#6205 <http://github.com/ceph/ceph/pull/6205>`_, John Spray)
+* pybind: add flock to libcephfs python bindings (`pr#7902 <http://github.com/ceph/ceph/pull/7902>`_, John Spray)
+* pybind/cephfs: add symlink and its unit test (`pr#6323 <http://github.com/ceph/ceph/pull/6323>`_, Shang Ding)
+* pybind: decode empty string in conf_parse_argv() correctly (`pr#6711 <http://github.com/ceph/ceph/pull/6711>`_, Josh Durgin)
+* pybind: Ensure correct python flags are passed (`pr#7663 <http://github.com/ceph/ceph/pull/7663>`_, James Page)
+* pybind: fix build failure, remove extraneous semicolon in method (`issue#14371 <http://tracker.ceph.com/issues/14371>`_, `pr#7235 <http://github.com/ceph/ceph/pull/7235>`_, Abhishek Lekshmanan)
+* pybind: flag an RBD image as closed regardless of result code (`pr#8005 <http://github.com/ceph/ceph/pull/8005>`_, Jason Dillaman)
+* pybind: Implementation of rados_ioctx_snapshot_rollback (`pr#6878 <http://github.com/ceph/ceph/pull/6878>`_, Florent Manens)
+* pybind/Makefile.am: Prevent race creating CYTHON_BUILD_DIR (`issue#15276 <http://tracker.ceph.com/issues/15276>`_, `pr#8356 <http://github.com/ceph/ceph/pull/8356>`_, Dan Mick)
+* pybind: move cephfs to Cython (`pr#7745 <http://github.com/ceph/ceph/pull/7745>`_, John Spray, Mehdi Abaakouk)
+* pybind: pep8 cleanups (Danny Al-Gaaf)
+* pybind: port the rbd bindings to Cython (`issue#13115 <http://tracker.ceph.com/issues/13115>`_, `pr#6768 <http://github.com/ceph/ceph/pull/6768>`_, Hector Martin)
+* pybind/rados: fix object lifetime issues and other bugs in aio (`pr#7778 <http://github.com/ceph/ceph/pull/7778>`_, Hector Martin)
+* pybind/rados: python3 fix (`pr#8331 <http://github.com/ceph/ceph/pull/8331>`_, Mehdi Abaakouk)
+* pybind/rados: use __dealloc__ since __del__ is ignored by cython (`pr#7692 <http://github.com/ceph/ceph/pull/7692>`_, Mehdi Abaakouk)
+* pybind: remove next() on iterators (`pr#7706 <http://github.com/ceph/ceph/pull/7706>`_, Mehdi Abaakouk)
+* pybind: replace __del__ with __dealloc__ for rbd (`pr#7708 <http://github.com/ceph/ceph/pull/7708>`_, Josh Durgin)
+* pybind: support ioctx:exec (`pr#6795 <http://github.com/ceph/ceph/pull/6795>`_, Noah Watkins)
+* pybind/test_rbd: fix test_create_defaults (`issue#14279 <http://tracker.ceph.com/issues/14279>`_, `pr#7155 <http://github.com/ceph/ceph/pull/7155>`_, Josh Durgin)
+* pybind: use correct subdir for rados install-exec rule (`pr#7684 <http://github.com/ceph/ceph/pull/7684>`_, Josh Durgin)
+* pycephfs: many fixes for bindings (Haomai Wang)
+* python binding of librados with cython (`pr#7621 <http://github.com/ceph/ceph/pull/7621>`_, Mehdi Abaakouk)
+* python: use pip instead of python setup.py (`pr#7605 <http://github.com/ceph/ceph/pull/7605>`_, Loic Dachary)
+* qa: add workunit to run ceph_test_rbd_mirror (`pr#8221 <http://github.com/ceph/ceph/pull/8221>`_, Josh Durgin)
+* qa: disable rbd/qemu-iotests test case 055 on RHEL/CentOSlibrbd: journal replay should honor inter-event dependencies (`issue#14385 <http://tracker.ceph.com/issues/14385>`_, `pr#7272 <http://github.com/ceph/ceph/pull/7272>`_, Jason Dillaman)
+* qa: erasure-code benchmark plugin selection (`pr#6685 <http://github.com/ceph/ceph/pull/6685>`_, Loic Dachary)
+* qa: fix filelock_interrupt.py test (Yan, Zheng)
+* qa: improve ceph-disk tests (Loic Dachary)
+* qa: improve docker build layers (Loic Dachary)
+* qa/krbd: Expunge generic/247 (`pr#6831 <http://github.com/ceph/ceph/pull/6831>`_, Douglas Fuller)
+* qa: run-make-check.sh script (Loic Dachary)
+* qa: update rest test cephfs calls (`issue#15309 <http://tracker.ceph.com/issues/15309>`_, `pr#8372 <http://github.com/ceph/ceph/pull/8372>`_, John Spray)
+* qa: update rest test cephfs calls (part 2) (`issue#15309 <http://tracker.ceph.com/issues/15309>`_, `pr#8393 <http://github.com/ceph/ceph/pull/8393>`_, John Spray)
+* qa/workunits/cephtool/test.sh: false positive fail on /tmp/obj1. (`pr#6837 <http://github.com/ceph/ceph/pull/6837>`_, Robin H. Johnson)
+* qa/workunits/cephtool/test.sh: no ./ (`pr#6748 <http://github.com/ceph/ceph/pull/6748>`_, Sage Weil)
+* qa/workunits/cephtool/test.sh: wait longer in ceph_watch_start() (`issue#14910 <http://tracker.ceph.com/issues/14910>`_, `pr#7861 <http://github.com/ceph/ceph/pull/7861>`_, Kefu Chai)
+* qa/workunits: merge_diff shouldn't attempt to use striping (`issue#14165 <http://tracker.ceph.com/issues/14165>`_, `pr#7041 <http://github.com/ceph/ceph/pull/7041>`_, Jason Dillaman)
+* qa/workunits/rados/test.sh: capture stderr too (`pr#8004 <http://github.com/ceph/ceph/pull/8004>`_, Sage Weil)
+* qa/workunits/rados/test.sh: test tmap_migrate (`pr#8114 <http://github.com/ceph/ceph/pull/8114>`_, Sage Weil)
+* qa/workunits/rbd: do not use object map during read flag testing (`pr#8104 <http://github.com/ceph/ceph/pull/8104>`_, Jason Dillaman)
+* qa/workunits/rbd: new online maintenance op tests (`pr#8216 <http://github.com/ceph/ceph/pull/8216>`_, Jason Dillaman)
+* qa/workunits/rbd: rbd-nbd test should use sudo for map/unmap ops (`issue#14221 <http://tracker.ceph.com/issues/14221>`_, `pr#7101 <http://github.com/ceph/ceph/pull/7101>`_, Jason Dillaman)
+* qa/workunits/rbd: use POSIX function definition (`issue#15104 <http://tracker.ceph.com/issues/15104>`_, `pr#8068 <http://github.com/ceph/ceph/pull/8068>`_, Nathan Cutler)
+* qa/workunits/rest/test.py: add confirmation to 'mds setmap' (`issue#14606 <http://tracker.ceph.com/issues/14606>`_, `pr#7982 <http://github.com/ceph/ceph/pull/7982>`_, Sage Weil)
+* qa/workunits/rest/test.py: don't use newfs (`pr#8191 <http://github.com/ceph/ceph/pull/8191>`_, Sage Weil)
+* qa/workunits/snaps: move snap tests into fs sub-directory (`pr#6496 <http://github.com/ceph/ceph/pull/6496>`_, Yan, Zheng)
+* rados: add ceph:: namespace to bufferlist type (`pr#8059 <http://github.com/ceph/ceph/pull/8059>`_, Noah Watkins)
+* rados: add --striper option to use libradosstriper (#10759 Sebastien Ponce)
+* rados: bench: add --no-verify option to improve performance (Piotr Dalek)
+* rados: bench: fix off-by-one to avoid writing past object_size (`pr#6677 <http://github.com/ceph/ceph/pull/6677>`_, Tao Chang)
+* rados bench: misc fixes (Dmitry Yatsushkevich)
+* rados: fix bug for write bench (`pr#7851 <http://github.com/ceph/ceph/pull/7851>`_, James Liu)
+* rados: fix error message on failed pool removal (Wido den Hollander)
+* radosgw-admin: add 'bucket check' function to repair bucket index (Yehuda Sadeh)
+* radosgw-admin: allow (`pr#8529 <http://github.com/ceph/ceph/pull/8529>`_, Orit Wasserman)
+* radosgw-admin: Checking the legality of the parameters (`issue#13018 <http://tracker.ceph.com/issues/13018>`_, `pr#5879 <http://github.com/ceph/ceph/pull/5879>`_, Qiankun Zheng)
+* radosgw-admin: Create --secret-key alias for --secret (`issue#5821 <http://tracker.ceph.com/issues/5821>`_, `pr#5335 <http://github.com/ceph/ceph/pull/5335>`_, Yuan Zhou)
+* radosgw-admin: fix for 'realm pull' (`pr#8404 <http://github.com/ceph/ceph/pull/8404>`_, Casey Bodley)
+* radosgw-admin: fix subuser modify output (#12286 Guce)
+* radosgw-admin: metadata list user should return an empty list when user pool is empty (`issue#13596 <http://tracker.ceph.com/issues/13596>`_, `pr#6465 <http://github.com/ceph/ceph/pull/6465>`_, Orit Wasserman)
+* radosgw-admin: 'period commit' supplies user-readable error messages (`pr#8264 <http://github.com/ceph/ceph/pull/8264>`_, Casey Bodley)
+* rados: handle --snapid arg properly (Abhishek Lekshmanan)
+* rados: implement rm --force option to force remove when full (`pr#6202 <http://github.com/ceph/ceph/pull/6202>`_, Xiaowei Chen)
+* rados: improve bench buffer handling, performance (Piotr Dalek)
+* rados: misc bench fixes (Dmitry Yatsushkevich)
+* rados: new options for write benchmark (`pr#6340 <http://github.com/ceph/ceph/pull/6340>`_, Joaquim Rocha)
+* rados: new pool import implementation (John Spray)
+* rados: translate errno to string in CLI (#10877 Kefu Chai)
+* rbd: accept map options config option (Ilya Dryomov)
+* rbd: accept --user, refuse -i command-line optionals (`pr#6590 <http://github.com/ceph/ceph/pull/6590>`_, Ilya Dryomov)
+* rbd: add disk usage tool (#7746 Jason Dillaman)
+* rbd: additional validation for striping parameters (`pr#6914 <http://github.com/ceph/ceph/pull/6914>`_, Na Xie)
+* rbd: add missing command aliases to refactored CLI (`issue#13806 <http://tracker.ceph.com/issues/13806>`_, `pr#6606 <http://github.com/ceph/ceph/pull/6606>`_, Jason Dillaman)
+* rbd: add --object-size option, deprecate --order (`issue#12112 <http://tracker.ceph.com/issues/12112>`_, `pr#6830 <http://github.com/ceph/ceph/pull/6830>`_, Vikhyat Umrao)
+* rbd: add pool name to disambiguate rbd admin socket commands (`pr#6904 <http://github.com/ceph/ceph/pull/6904>`_, wuxiangwei)
+* rbd: add RBD pool mirroring configuration API + CLI (`pr#6129 <http://github.com/ceph/ceph/pull/6129>`_, Jason Dillaman)
+* rbd: add support for mirror image promotion/demotion/resync (`pr#8138 <http://github.com/ceph/ceph/pull/8138>`_, Jason Dillaman)
+* rbd: allow librados to prune the command-line for config overrides (`issue#15250 <http://tracker.ceph.com/issues/15250>`_, `pr#8282 <http://github.com/ceph/ceph/pull/8282>`_, Jason Dillaman)
+* rbd: allow unmapping by spec (Ilya Dryomov)
+* rbd: cli: fix arg parsing with --io-pattern (Dmitry Yatsushkevich)
+* rbd: clone operation should default to image format 2 (`pr#8119 <http://github.com/ceph/ceph/pull/8119>`_, Jason Dillaman)
+* rbd: correct an output string for merge-diff (`pr#7046 <http://github.com/ceph/ceph/pull/7046>`_, Kongming Wu)
+* rbd: deprecate image format 1 (`pr#7841 <http://github.com/ceph/ceph/pull/7841>`_, Jason Dillaman)
+* rbd: deprecate --new-format option (Jason Dillman)
+* rbd: dynamically generated bash completion (`issue#13494 <http://tracker.ceph.com/issues/13494>`_, `pr#6316 <http://github.com/ceph/ceph/pull/6316>`_, Jason Dillaman)
+* rbd: fix build with "--without-rbd" (`issue#14058 <http://tracker.ceph.com/issues/14058>`_, `pr#6899 <http://github.com/ceph/ceph/pull/6899>`_, Piotr Dałek)
+* rbd: fix clone isssue (`issue#13553 <http://tracker.ceph.com/issues/13553>`_, `pr#6334 <http://github.com/ceph/ceph/pull/6334>`_, xinxin shu)
+* rbd: fix error messages (#2862 Rajesh Nambiar)
+* rbd: fixes for refactored CLI and related tests (`pr#6738 <http://github.com/ceph/ceph/pull/6738>`_, Ilya Dryomov)
+* rbd: fix init-rbdmap CMDPARAMS (`issue#13214 <http://tracker.ceph.com/issues/13214>`_, `pr#6109 <http://github.com/ceph/ceph/pull/6109>`_, Sage Weil)
+* rbd: fix link issues (Jason Dillaman)
+* rbd: fix static initialization ordering issues (`pr#6978 <http://github.com/ceph/ceph/pull/6978>`_, Mykola Golub)
+* rbd-fuse: image name can not include snap name (`pr#7044 <http://github.com/ceph/ceph/pull/7044>`_, Yongqiang He)
+* rbd-fuse: implement mv operation (`pr#6938 <http://github.com/ceph/ceph/pull/6938>`_, wuxiangwei)
+* rbd: improve CLI arg parsing, usage (Ilya Dryomov)
+* rbd: journal: configuration via conf, cli, api and some fixes (`pr#6665 <http://github.com/ceph/ceph/pull/6665>`_, Mykola Golub)
+* rbd: journal reset should disable/re-enable journaling feature (`issue#15097 <http://tracker.ceph.com/issues/15097>`_, `pr#8490 <http://github.com/ceph/ceph/pull/8490>`_, Jason Dillaman)
+* rbd: make config changes actually apply (`pr#6520 <http://github.com/ceph/ceph/pull/6520>`_, Mykola Golub)
+* rbdmap: add manpage (`issue#15212 <http://tracker.ceph.com/issues/15212>`_, `pr#8224 <http://github.com/ceph/ceph/pull/8224>`_, Nathan Cutler)
+* rbdmap: systemd support (`issue#13374 <http://tracker.ceph.com/issues/13374>`_, `pr#6479 <http://github.com/ceph/ceph/pull/6479>`_, Boris Ranto)
+* rbd: merge_diff test should use new --object-size parameter instead of --order (`issue#14106 <http://tracker.ceph.com/issues/14106>`_, `pr#6972 <http://github.com/ceph/ceph/pull/6972>`_, Na Xie, Jason Dillaman)
+* rbd-mirror: asok commands to get status and flush on Mirror and Replayer level (`pr#8235 <http://github.com/ceph/ceph/pull/8235>`_, Mykola Golub)
+* rbd-mirror: enabling/disabling pool mirroring should update the mirroring directory (`issue#15217 <http://tracker.ceph.com/issues/15217>`_, `pr#8261 <http://github.com/ceph/ceph/pull/8261>`_, Ricardo Dias)
+* rbd-mirror: fix image replay test failures (`pr#8158 <http://github.com/ceph/ceph/pull/8158>`_, Jason Dillaman)
+* rbd-mirror: fix long termination due to 30sec wait in main loop (`pr#8185 <http://github.com/ceph/ceph/pull/8185>`_, Mykola Golub)
+* rbd-mirror: fix missing increment for iterators (`pr#8352 <http://github.com/ceph/ceph/pull/8352>`_, runsisi)
+* rbd-mirror: ImageReplayer async start/stop (`pr#7944 <http://github.com/ceph/ceph/pull/7944>`_, Mykola Golub)
+* rbd-mirror: ImageReplayer improvements (`pr#7759 <http://github.com/ceph/ceph/pull/7759>`_, Mykola Golub)
+* rbd-mirror: implement ImageReplayer (`pr#7614 <http://github.com/ceph/ceph/pull/7614>`_, Mykola Golub)
+* rbd-mirror: initial failover / failback support (`pr#8287 <http://github.com/ceph/ceph/pull/8287>`_, Jason Dillaman)
+* rbd-mirror: integrate with image sync state machine (`pr#8079 <http://github.com/ceph/ceph/pull/8079>`_, Jason Dillaman)
+* rbd-mirror: make remote context respect env and argv config params (`pr#8182 <http://github.com/ceph/ceph/pull/8182>`_, Mykola Golub)
+* rbd-mirror: minor fix-ups for initial skeleton implementation (`pr#7958 <http://github.com/ceph/ceph/pull/7958>`_, Mykola Golub)
+* rbd-mirror: prevent enabling/disabling an image's mirroring when not in image mode (`issue#15267 <http://tracker.ceph.com/issues/15267>`_, `pr#8332 <http://github.com/ceph/ceph/pull/8332>`_, Ricardo Dias)
+* rbd-mirror: remote to local cluster image sync (`pr#7979 <http://github.com/ceph/ceph/pull/7979>`_, Jason Dillaman)
+* rbd-mirror: switch fsid over to mirror uuid (`issue#15238 <http://tracker.ceph.com/issues/15238>`_, `pr#8280 <http://github.com/ceph/ceph/pull/8280>`_, Ricardo Dias)
+* rbd-mirror: use pool/image names in asok commands (`pr#8159 <http://github.com/ceph/ceph/pull/8159>`_, Mykola Golub)
+* rbd-mirror: use the mirroring directory to detect candidate images (`issue#15142 <http://tracker.ceph.com/issues/15142>`_, `pr#8162 <http://github.com/ceph/ceph/pull/8162>`_, Ricardo Dias)
+* rbd-mirror: workaround for intermingled lockdep singletons (`pr#8476 <http://github.com/ceph/ceph/pull/8476>`_, Jason Dillaman)
+* rbd: must specify both of stripe-unit and stripe-count when specifying stripingv2 feature (`pr#7026 <http://github.com/ceph/ceph/pull/7026>`_, Donghai Xu)
+* rbd-nbd: add copyright (`pr#7166 <http://github.com/ceph/ceph/pull/7166>`_, Li Wang)
+* rbd-nbd: fix up return code handling (`pr#7215 <http://github.com/ceph/ceph/pull/7215>`_, Mykola Golub)
+* rbd-nbd: network block device (NBD) support for RBD  (`pr#6657 <http://github.com/ceph/ceph/pull/6657>`_, Yunchuan Wen, Li Wang)
+* rbd-nbd: small improvements in logging and forking (`pr#7127 <http://github.com/ceph/ceph/pull/7127>`_, Mykola Golub)
+* rbd: output formatter may not be closed upon error (`issue#13711 <http://tracker.ceph.com/issues/13711>`_, `pr#6706 <http://github.com/ceph/ceph/pull/6706>`_, xie xingguo)
+* rbd: rbdmap improvements (`pr#6445 <http://github.com/ceph/ceph/pull/6445>`_, Boris Ranto)
+* rbd: rbd order will be place in 22, when set to 0 in the config_opt (`issue#14139 <http://tracker.ceph.com/issues/14139>`_, `issue#14047 <http://tracker.ceph.com/issues/14047>`_, `pr#6886 <http://github.com/ceph/ceph/pull/6886>`_, huanwen ren)
+* rbd: rbd-replay-prep and rbd-replay improvements (Jason Dillaman)
+* rbd: recognize queue_depth kernel option (Ilya Dryomov)
+* rbd: refactor cli command handling (`pr#5987 <http://github.com/ceph/ceph/pull/5987>`_, Jason Dillaman)
+* rbd/run_cli_tests.sh: Reflect test failures (`issue#14825 <http://tracker.ceph.com/issues/14825>`_, `pr#7781 <http://github.com/ceph/ceph/pull/7781>`_, Zack Cerza)
+* rbd: stripe unit/count set incorrectly from config (`pr#6593 <http://github.com/ceph/ceph/pull/6593>`_, Mykola Golub)
+* rbd: striping parameters should support 64bit integers (`pr#6942 <http://github.com/ceph/ceph/pull/6942>`_, Na Xie)
+* rbd: support for enabling/disabling mirroring on specific images (`issue#13296 <http://tracker.ceph.com/issues/13296>`_, `pr#8056 <http://github.com/ceph/ceph/pull/8056>`_, Ricardo Dias)
+* rbd: support G and T units for CLI (Abhishek Lekshmanan)
+* rbd: support negative boolean command-line optionals (`issue#13784 <http://tracker.ceph.com/issues/13784>`_, `pr#6607 <http://github.com/ceph/ceph/pull/6607>`_, Jason Dillaman)
+* rbd: unbreak rbd map + cephx_sign_messages option (`pr#6583 <http://github.com/ceph/ceph/pull/6583>`_, Ilya Dryomov)
+* rbd: update default image features (`pr#7846 <http://github.com/ceph/ceph/pull/7846>`_, Jason Dillaman)
+* rbd: update rbd man page (Ilya Dryomov)
+* rbd: update xfstests tests (Douglas Fuller)
+* rbd: use default order from configuration when not specified (`pr#6965 <http://github.com/ceph/ceph/pull/6965>`_, Yunchuan Wen)
+* rbd: use image-spec and snap-spec in help (Vikhyat Umrao, Ilya Dryomov)
+* release-notes: draft v0.94.4 release notes (`pr#5907 <http://github.com/ceph/ceph/pull/5907>`_, Loic Dachary)
+* release-notes: draft v0.94.4 release notes (`pr#6195 <http://github.com/ceph/ceph/pull/6195>`_, Loic Dachary)
+* release-notes: draft v0.94.4 release notes (`pr#6238 <http://github.com/ceph/ceph/pull/6238>`_, Loic Dachary)
+* release-notes: draft v0.94.6 release notes (`issue#13356 <http://tracker.ceph.com/issues/13356>`_, `pr#7689 <http://github.com/ceph/ceph/pull/7689>`_, Abhishek Varshney, Loic Dachary)
+* release-notes: draft v10.0.3 release notes (`pr#7592 <http://github.com/ceph/ceph/pull/7592>`_, Loic Dachary)
+* release-notes: draft v10.0.4 release notes (`pr#7966 <http://github.com/ceph/ceph/pull/7966>`_, Loic Dachary)
+* release-notes: draft v9.2.1 release notes (`issue#13750 <http://tracker.ceph.com/issues/13750>`_, `pr#7694 <http://github.com/ceph/ceph/pull/7694>`_, Abhishek Varshney)
+* releases: what is merged where and when ? (`pr#8358 <http://github.com/ceph/ceph/pull/8358>`_, Loic Dachary)
+* rest-bench: misc fixes (Shawn Chen)
+* rest-bench: support https (#3968 Yuan Zhou)
+* rgw: accept data only at the first time in response to a request (`pr#8084 <http://github.com/ceph/ceph/pull/8084>`_, sunspot)
+* rgw: add a few more help options in admin interface (`pr#8410 <http://github.com/ceph/ceph/pull/8410>`_, Abhishek Lekshmanan)
+* rgw: add a method to purge all associate keys when removing a subuser (`issue#12890 <http://tracker.ceph.com/issues/12890>`_, `pr#6002 <http://github.com/ceph/ceph/pull/6002>`_, Sangdi Xu)
+* rgw: add a missing cap type (`pr#6774 <http://github.com/ceph/ceph/pull/6774>`_, Yehuda Sadeh)
+* rgw: add an inspection to the field of type when assigning user caps (`pr#6051 <http://github.com/ceph/ceph/pull/6051>`_, Kongming Wu)
+* rgw: add bucket request payment feature usage statistics integration (`issue#13834 <http://tracker.ceph.com/issues/13834>`_, `pr#6656 <http://github.com/ceph/ceph/pull/6656>`_, Javier M. Mellid)
+* rgw: add compat header for TEMP_FAILURE_RETRY (`pr#6294 <http://github.com/ceph/ceph/pull/6294>`_, John Coyle)
+* rgw: add default quota config (`pr#6400 <http://github.com/ceph/ceph/pull/6400>`_, Daniel Gryniewicz)
+* rgw: add LifeCycle feature (`pr#6331 <http://github.com/ceph/ceph/pull/6331>`_, Ji Chen)
+* rgw: add max multipart upload parts (#12146 Abshishek Dixit)
+* rgw: add missing error code for admin op API (`pr#7037 <http://github.com/ceph/ceph/pull/7037>`_, Dunrong Huang)
+* rgw: add missing headers to Swift container details (#10666 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: add stats to headers for account GET (#10684 Yuan Zhou)
+* rgw: adds the radosgw-admin sync status command that gives a human readable status of the sync process at a specific zone (`pr#8030 <http://github.com/ceph/ceph/pull/8030>`_, Yehuda Sadeh)
+* rgw: add support for caching of Keystone admin token. (`pr#7630 <http://github.com/ceph/ceph/pull/7630>`_, Radoslaw Zarzynski)
+* rgw: add support for "end_marker" parameter for GET on Swift account. (`issue#10682 <http://tracker.ceph.com/issues/10682>`_, `pr#4216 <http://github.com/ceph/ceph/pull/4216>`_, Radoslaw Zarzynski)
+* rgw: add support for getting Swift's DLO without manifest handling (`pr#6206 <http://github.com/ceph/ceph/pull/6206>`_, Radoslaw Zarzynski)
+* rgw: add support for metadata upload during PUT on Swift container. (`pr#8002 <http://github.com/ceph/ceph/pull/8002>`_, Radoslaw Zarzynski)
+* rgw: add support for Static Large Objects of Swift API (`issue#12886 <http://tracker.ceph.com/issues/12886>`_, `issue#13452 <http://tracker.ceph.com/issues/13452>`_, `pr#6643 <http://github.com/ceph/ceph/pull/6643>`_, Yehuda Sadeh, Radoslaw Zarzynski)
+* rgw: add support for system requests over Swift API (`pr#7666 <http://github.com/ceph/ceph/pull/7666>`_, Radoslaw Zarzynski)
+* rgw: add Trasnaction-Id to response (Abhishek Dixit)
+* rgw: add X-Timestamp for Swift containers (#10938 Radoslaw Zarzynski)
+* rgw: add zone delete to rgw-admin help (`pr#8184 <http://github.com/ceph/ceph/pull/8184>`_, Abhishek Lekshmanan)
+* rgw: adjust error code when bucket does not exist in copy operation (`issue#14975 <http://tracker.ceph.com/issues/14975>`_, `pr#7916 <http://github.com/ceph/ceph/pull/7916>`_, Yehuda Sadeh)
+* rgw: adjust the request_uri to support absoluteURI of http request (`issue#12917 <http://tracker.ceph.com/issues/12917>`_, `pr#7675 <http://github.com/ceph/ceph/pull/7675>`_, Wenjun Huang)
+* rgw: admin api for retrieving usage info (Ji Chen) (`pr#8031 <http://github.com/ceph/ceph/pull/8031>`_, Yehuda Sadeh, Ji Chen)
+* rgw_admin: orphans finish segfaults (`pr#6652 <http://github.com/ceph/ceph/pull/6652>`_, Igor Fedotov)
+* rgw-admin: remove unused iterator and fix error message (`pr#8507 <http://github.com/ceph/ceph/pull/8507>`_, Karol Mroz)
+* rgw_admin: remove unused parent_period arg (`pr#8411 <http://github.com/ceph/ceph/pull/8411>`_, Abhishek Lekshmanan)
+* rgw: Allow an implicit tenant in case of Keystone (`pr#8139 <http://github.com/ceph/ceph/pull/8139>`_, Pete Zaitcev)
+* rgw: allow authentication keystone with self signed certs  (`issue#14853 <http://tracker.ceph.com/issues/14853>`_, `issue#13422 <http://tracker.ceph.com/issues/13422>`_, `pr#7777 <http://github.com/ceph/ceph/pull/7777>`_, Abhishek Lekshmanan)
+* rgw: always check if token is expired (#11367 Anton Aksola, Riku Lehto)
+* rgw: approximate AmazonS3 HostId error field. (`pr#7444 <http://github.com/ceph/ceph/pull/7444>`_, Robin H. Johnson)
+* rgw: aws4 subdomain calling bugfix (`issue#15369 <http://tracker.ceph.com/issues/15369>`_, `pr#8472 <http://github.com/ceph/ceph/pull/8472>`_, Javier M. Mellid)
+* rgw:bucket link now set the bucket.instance acl (bug fix) (`issue#11076 <http://tracker.ceph.com/issues/11076>`_, `pr#8037 <http://github.com/ceph/ceph/pull/8037>`_, Zengran Zhang)
+* rgw: bucket request payment support (`issue#13427 <http://tracker.ceph.com/issues/13427>`_, `pr#6214 <http://github.com/ceph/ceph/pull/6214>`_, Javier M. Mellid)
+* rgw: Bug fix for mtime anomalies in RadosGW and other places (`pr#7328 <http://github.com/ceph/ceph/pull/7328>`_, Adam C. Emerson, Casey Bodley)
+* rgw: build-related fixes (`pr#8076 <http://github.com/ceph/ceph/pull/8076>`_, Yehuda Sadeh, Matt Benjamin)
+* rgw: calculate payload hash in RGWPutObj_ObjStore only when necessary. (`pr#7869 <http://github.com/ceph/ceph/pull/7869>`_, Radoslaw Zarzynski)
+* [rgw] Check return code in RGWFileHandle::write (`pr#7875 <http://github.com/ceph/ceph/pull/7875>`_, Brad Hubbard)
+* rgw: check the return value when call fe->run() (`issue#14585 <http://tracker.ceph.com/issues/14585>`_, `pr#7457 <http://github.com/ceph/ceph/pull/7457>`_, wei qiaomiao)
+* rgw: clarify the error message when trying to create an existed user (`pr#5938 <http://github.com/ceph/ceph/pull/5938>`_, Zeqiang Zhuang)
+* rgw: cleanups to comments and messages (`pr#7633 <http://github.com/ceph/ceph/pull/7633>`_, Pete Zaitcev)
+* rgw: content length (`issue#13582 <http://tracker.ceph.com/issues/13582>`_, `pr#6975 <http://github.com/ceph/ceph/pull/6975>`_, Yehuda Sadeh)
+* rgw: conversion tool to repair broken multipart objects (#12079 Yehuda Sadeh)
+* rgw: convert plain object to versioned (with null version) when removing (`issue#15243 <http://tracker.ceph.com/issues/15243>`_, `pr#8268 <http://github.com/ceph/ceph/pull/8268>`_, Yehuda Sadeh)
+* rgw: delete default zone (`pr#7005 <http://github.com/ceph/ceph/pull/7005>`_, YankunLi)
+* rgw: document layout of pools and objects (Pete Zaitcev)
+* rgw: do not abort radowgw server when using admin op API with bad parameters  (`issue#14190 <http://tracker.ceph.com/issues/14190>`_, `issue#14191 <http://tracker.ceph.com/issues/14191>`_, `pr#7063 <http://github.com/ceph/ceph/pull/7063>`_, Dunrong Huang)
+* rgw: do not enclose bucket header in quotes (#11860 Wido den Hollander)
+* rgw: do not prefetch data for HEAD requests (Guang Yang)
+* rgw: do not preserve ACLs when copying object (#12370 Yehuda Sadeh)
+* rgw: Do not send a Content-Type on a '304 Not Modified' response (`issue#15119 <http://tracker.ceph.com/issues/15119>`_, `pr#8253 <http://github.com/ceph/ceph/pull/8253>`_, Wido den Hollander)
+* rgw: do not set content-type if length is 0 (#11091 Orit Wasserman)
+* rgw: don't clobber bucket/object owner when setting ACLs (#10978 Yehuda Sadeh)
+* rgw: don't use end_marker for namespaced object listing (#11437 Yehuda Sadeh)
+* rgw: don't use rgw_socket_path if frontend is configured (#11160 Yehuda Sadeh)
+* rgw: don't use s->bucket for metadata api path entry (`issue#14549 <http://tracker.ceph.com/issues/14549>`_, `pr#7408 <http://github.com/ceph/ceph/pull/7408>`_, Yehuda Sadeh)
+* rgw: Drop a debugging message (`pr#7280 <http://github.com/ceph/ceph/pull/7280>`_, Pete Zaitcev)
+* rgw: drop permissions of rgw/civetweb after startup (`issue#13600 <http://tracker.ceph.com/issues/13600>`_, `pr#8019 <http://github.com/ceph/ceph/pull/8019>`_, Karol Mroz)
+* rgw: Drop unused usage_exit from rgw_admin.cc (`pr#7632 <http://github.com/ceph/ceph/pull/7632>`_, Pete Zaitcev)
+* rgw: enforce Content-Length for POST on Swift cont/obj (#10661 Radoslaw Zarzynski)
+* rgw: error out if frontend did not send all data (#11851 Yehuda Sadeh)
+* rgw: expose the number of unhealthy workers through admin socket (Guang Yang)
+* rgw: extend rgw_extended_http_attrs to affect Swift accounts and containers as well (`pr#5969 <http://github.com/ceph/ceph/pull/5969>`_, Radoslaw Zarzynski)
+* rgw: fail if parts not specified on multipart upload (#11435 Yehuda Sadeh)
+* rgw: fcgi should include acconfig (`pr#7760 <http://github.com/ceph/ceph/pull/7760>`_, Abhishek Lekshmanan)
+* rgw_file: set owner uid, gid, and Unix mode on new objects (`pr#8321 <http://github.com/ceph/ceph/pull/8321>`_, Matt Benjamin)
+* rgw: fix a glaring syntax error (`pr#6888 <http://github.com/ceph/ceph/pull/6888>`_, Pavan Rallabhandi)
+* rgw: fix assignment of copy obj attributes (#11563 Yehuda Sadeh)
+* rgw: fix a typo in error message (`pr#8434 <http://github.com/ceph/ceph/pull/8434>`_, Abhishek Lekshmanan)
+* rgw: fix a typo in init-radosgw (`pr#6817 <http://github.com/ceph/ceph/pull/6817>`_, Zhi Zhang)
+* rgw: fix broken stats in container listing (#11285 Radoslaw Zarzynski)
+* rgw: fix bug in domain/subdomain splitting (Robin H. Johnson)
+* rgw: fix casing of Content-Type header (Robin H. Johnson)
+* rgw: fix civetweb max threads (#10243 Yehuda Sadeh)
+* rgw: fix compilation warning (`pr#7160 <http://github.com/ceph/ceph/pull/7160>`_, Yehuda Sadeh)
+* rgw: fix compiling error (`pr#8394 <http://github.com/ceph/ceph/pull/8394>`_, xie xingguo)
+* rgw: fix Connection: header handling (#12298 Wido den Hollander)
+* rgw: fix copy metadata, support X-Copied-From for swift (#10663 Radoslaw Zarzynski)
+* rgw: fix data corruptions race condition (#11749 Wuxingyi)
+* rgw: fix decoding of X-Object-Manifest from GET on Swift DLO (Radslow Rzarzynski)
+* rgw: fixes for per-period metadata logs (`pr#7827 <http://github.com/ceph/ceph/pull/7827>`_, Casey Bodley)
+* rgw: fix GET on swift account when limit == 0 (#10683 Radoslaw Zarzynski)
+* rgw: fix handling empty metadata items on Swift container (#11088 Radoslaw Zarzynski)
+* rgw: fix JSON response when getting user quota (#12117 Wuxingyi)
+* rgw: fix locator for objects starting with _ (#11442 Yehuda Sadeh)
+* rgw: fix lockdep false positive (`pr#8284 <http://github.com/ceph/ceph/pull/8284>`_, Yehuda Sadeh)
+* rgw: fix log rotation (Wuxingyi)
+* rgw: fix mdlog (`pr#8183 <http://github.com/ceph/ceph/pull/8183>`_, Orit Wasserman)
+* rgw: fix mulitipart upload in retry path (#11604 Yehuda Sadeh)
+* rgw: fix objects can not be displayed which object name does not cont… (`issue#12963 <http://tracker.ceph.com/issues/12963>`_, `pr#5738 <http://github.com/ceph/ceph/pull/5738>`_, Weijun Duan)
+* rgw: fix openssl linkage (`pr#6513 <http://github.com/ceph/ceph/pull/6513>`_, Yehuda Sadeh)
+* rgw: fix partial read issue in rgw_admin and rgw_tools (`pr#6761 <http://github.com/ceph/ceph/pull/6761>`_, Jiaying Ren)
+* rgw: fix problem deleting objects begining with double underscores (`issue#15318 <http://tracker.ceph.com/issues/15318>`_, `pr#8488 <http://github.com/ceph/ceph/pull/8488>`_, Orit Wasserman)
+* rgw: fix quota enforcement on POST (#11323 Sergey Arkhipov)
+* rgw: fix reload on non Debian systems. (`pr#6482 <http://github.com/ceph/ceph/pull/6482>`_, Hervé Rousseau)
+* rgw: fix reset_loc (#11974 Yehuda Sadeh)
+* rgw: fix response of delete expired objects (`issue#13469 <http://tracker.ceph.com/issues/13469>`_, `pr#6228 <http://github.com/ceph/ceph/pull/6228>`_, Yuan Zhou)
+* rgw: fix return code on missing upload (#11436 Yehuda Sadeh)
+* rgw: Fix subuser harder with tenants (`pr#7618 <http://github.com/ceph/ceph/pull/7618>`_, Pete Zaitcev)
+* rgw: fix swift API returning incorrect account metadata (`issue#13140 <http://tracker.ceph.com/issues/13140>`_, `pr#6047 <http://github.com/ceph/ceph/pull/6047>`_, Sangdi Xu)
+* rgw: fix sysvinit script
+* rgw: fix sysvinit script w/ multiple instances (Sage Weil, Pavan Rallabhandi)
+* rgw: fix the build failure (`pr#6927 <http://github.com/ceph/ceph/pull/6927>`_, Kefu Chai)
+* rgw: fix typo in RGWHTTPClient::process error message (`pr#6424 <http://github.com/ceph/ceph/pull/6424>`_, Brad Hubbard)
+* rgw: fix wrong check for parse() return (`pr#6797 <http://github.com/ceph/ceph/pull/6797>`_, Dunrong Huang)
+* rgw: fix wrong etag calculation during POST on S3 bucket. (`issue#11241 <http://tracker.ceph.com/issues/11241>`_, `pr#6030 <http://github.com/ceph/ceph/pull/6030>`_, Radoslaw Zarzynski)
+* rgw: fix wrong handling of limit=0 during listing of Swift account. (`issue#14903 <http://tracker.ceph.com/issues/14903>`_, `pr#7821 <http://github.com/ceph/ceph/pull/7821>`_, Radoslaw Zarzynski)
+* rgw: force content_type for swift bucket stats requests (#12095 Orit Wasserman)
+* rgw: force content type header on responses with no body (#11438 Orit Wasserman)
+* rgw: generate Date header for civetweb (#10873 Radoslaw Zarzynski)
+* rgw: generate new object tag when setting attrs (#11256 Yehuda Sadeh)
+* rgw: highres time stamps (`pr#8108 <http://github.com/ceph/ceph/pull/8108>`_, Yehuda Sadeh, Adam C. Emerson, Matt Benjamin)
+* rgw: improve content-length env var handling (#11419 Robin H. Johnson)
+* rgw: improved support for swift account metadata (Radoslaw Zarzynski)
+* rgw: improve error handling in S3/Keystone integration (`pr#7597 <http://github.com/ceph/ceph/pull/7597>`_, Radoslaw Zarzynski)
+* rgw: improve handling of already removed buckets in expirer (Radoslaw Rzarzynski)
+* rgw: increase verbosity level on RGWObjManifest line (`pr#7285 <http://github.com/ceph/ceph/pull/7285>`_, magicrobotmonkey)
+* rgw: indexless (`pr#7786 <http://github.com/ceph/ceph/pull/7786>`_, Yehuda Sadeh)
+* rgw: issue aio for first chunk before flush cached data (#11322 Guang Yang)
+* rgw: Jewel nfs fixes 3 (`pr#8460 <http://github.com/ceph/ceph/pull/8460>`_, Matt Benjamin)
+* rgw: keystone v3 (`pr#7719 <http://github.com/ceph/ceph/pull/7719>`_, Mark Barnes, Radoslaw Zarzynski)
+* rgw: ldap fixes (`pr#8168 <http://github.com/ceph/ceph/pull/8168>`_, Matt Benjamin)
+* rgw_ldap: make ldap.h inclusion conditional (`pr#8500 <http://github.com/ceph/ceph/pull/8500>`_, Matt Benjamin)
+* rgw: ldap (Matt Benjamin) (`pr#7985 <http://github.com/ceph/ceph/pull/7985>`_, Matt Benjamin)
+* rgw: let radosgw-admin bucket stats return a standard josn (`pr#7029 <http://github.com/ceph/ceph/pull/7029>`_, Ruifeng Yang)
+* rgw: link against system openssl (instead of dlopen at runtime) (`pr#6419 <http://github.com/ceph/ceph/pull/6419>`_, Sage Weil)
+* rgw: link civetweb with openssl (Sage, Marcus Watts) (`pr#7825 <http://github.com/ceph/ceph/pull/7825>`_, Marcus Watts, Sage Weil)
+* rgw: link payer info to usage logging (`pr#7918 <http://github.com/ceph/ceph/pull/7918>`_, Yehuda Sadeh, Javier M. Mellid)
+* rgw: log to /var/log/ceph instead of /var/log/radosgw
+* rgw: make init script wait for radosgw to stop (#11140 Dmitry Yatsushkevich)
+* rgw: make max put size configurable (#6999 Yuan Zhou)
+* rgw: make quota/gc threads configurable (#11047 Guang Yang)
+* rgw: make read user buckets backward compat (#10683 Radoslaw Zarzynski)
+* rgw: mdlog trim add usage prompt (`pr#6059 <http://github.com/ceph/ceph/pull/6059>`_, Weijun Duan)
+* rgw: merge manifests properly with prefix override (#11622 Yehuda Sadeh)
+* rgw: modify command stucking when operating radosgw-admin metadata list user (`pr#7032 <http://github.com/ceph/ceph/pull/7032>`_, Peiyang Liu)
+* rgw: modify documents and help infos' descriptions to the usage of option date when executing command "log show" (`pr#6080 <http://github.com/ceph/ceph/pull/6080>`_, Kongming Wu)
+* rgw: modify the conditional statement in parse_metadata_key method. (`pr#5875 <http://github.com/ceph/ceph/pull/5875>`_, Zengran Zhang)
+* rgw: move signal.h dependency from rgw_front.h (`pr#7678 <http://github.com/ceph/ceph/pull/7678>`_, Matt Benjamin)
+* rgw: Multipart ListPartsResult ETag quotes (`issue#15334 <http://tracker.ceph.com/issues/15334>`_, `pr#8387 <http://github.com/ceph/ceph/pull/8387>`_, Robin H. Johnson)
+* rgw: multiple improvements regarding etag calculation for SLO/DLO of Swift API. (`pr#7764 <http://github.com/ceph/ceph/pull/7764>`_, Radoslaw Zarzynski)
+* rgw: multiple Swift API compliance improvements for TempURL (Radoslaw Zarzynsk) (`issue#14806 <http://tracker.ceph.com/issues/14806>`_, `issue#11163 <http://tracker.ceph.com/issues/11163>`_, `pr#7891 <http://github.com/ceph/ceph/pull/7891>`_, Radoslaw Zarzynski)
+* rgw: multisite fixes (`pr#8013 <http://github.com/ceph/ceph/pull/8013>`_, Yehuda Sadeh)
+* rgw: multitenancy support (`pr#6784 <http://github.com/ceph/ceph/pull/6784>`_, Yehuda Sadeh, Pete Zaitcev)
+* rgw: new multisite merge (`issue#14549 <http://tracker.ceph.com/issues/14549>`_, `pr#7709 <http://github.com/ceph/ceph/pull/7709>`_, Yehuda Sadeh, Orit Wasserman, Casey Bodley, Daniel Gryniewicz)
+* rgw: only scan for objects not in a namespace (#11984 Yehuda Sadeh)
+* rgw: orphan detection tool (Yehuda Sadeh)
+* rgw: Parse --subuser better (`pr#7279 <http://github.com/ceph/ceph/pull/7279>`_, Pete Zaitcev)
+* rgw: pass in civetweb configurables (#10907 Yehuda Sadeh)
+* rgw: prevent anonymous user from reading bucket with authenticated read ACL (`issue#13207 <http://tracker.ceph.com/issues/13207>`_, `pr#6057 <http://github.com/ceph/ceph/pull/6057>`_, root)
+* rgw: radosgw-admin bucket check --fix not work (`pr#7093 <http://github.com/ceph/ceph/pull/7093>`_, Weijun Duan)
+* rgw: rectify 202 Accepted in PUT response (#11148 Radoslaw Zarzynski)
+* rgw: refuse to calculate digest when the s3 secret key is empty (`issue#13133 <http://tracker.ceph.com/issues/13133>`_, `pr#6045 <http://github.com/ceph/ceph/pull/6045>`_, Sangdi Xu)
+* rgw: remove duplicated code in RGWRados::get_bucket_info() (`pr#7413 <http://github.com/ceph/ceph/pull/7413>`_, liyankun)
+* rgw: remove extra check in RGWGetObj::execute (`issue#12352 <http://tracker.ceph.com/issues/12352>`_, `pr#5262 <http://github.com/ceph/ceph/pull/5262>`_, Javier M. Mellid)
+* rgw: remove meta file after deleting bucket (#11149 Orit Wasserman)
+* rgw: remove trailing :port from HTTP_HOST header (Sage Weil)
+* rgw: Remove unused code in PutMetadataAccount:execute (`pr#6668 <http://github.com/ceph/ceph/pull/6668>`_, Pete Zaitcev)
+* rgw: remove unused variable in RGWPutMetadataBucket::execute (`pr#6735 <http://github.com/ceph/ceph/pull/6735>`_, Radoslaw Zarzynski)
+* rgw: remove unused vector (`pr#7990 <http://github.com/ceph/ceph/pull/7990>`_, Na Xie)
+* rgw: reset return code in when iterating over the bucket the objects (`issue#14826 <http://tracker.ceph.com/issues/14826>`_, `pr#7803 <http://github.com/ceph/ceph/pull/7803>`_, Orit Wasserman)
+* rgw: retry RGWRemoteMetaLog::read_log_info() while master is down (`pr#8453 <http://github.com/ceph/ceph/pull/8453>`_, Casey Bodley)
+* rgw: return 412 on bad limit when listing buckets (#11613 Yehuda Sadeh)
+* rgw: Revert "rgw ldap" (`pr#8075 <http://github.com/ceph/ceph/pull/8075>`_, Yehuda Sadeh)
+* rgw: rework X-Trans-Id header to conform with Swift API (Radoslaw Rzarzynski)
+* rgw/rgw_admin:fix bug about list and stats command (`pr#8200 <http://github.com/ceph/ceph/pull/8200>`_, Qiankun Zheng)
+* rgw/rgw_common.h: fix the RGWBucketInfo decoding (`pr#8154 <http://github.com/ceph/ceph/pull/8154>`_, Kefu Chai)
+* rgw/rgw_common.h: fix the RGWBucketInfo decoding (`pr#8165 <http://github.com/ceph/ceph/pull/8165>`_, Kefu Chai)
+* rgw: RGWLib::env is not used so remove it (`pr#7874 <http://github.com/ceph/ceph/pull/7874>`_, Brad Hubbard)
+* rgw/rgw_orphan: check the return value of save_state (`pr#7544 <http://github.com/ceph/ceph/pull/7544>`_, Boris Ranto)
+* rgw/rgw_resolve: fallback to res_query when res_nquery not implemented (`pr#6292 <http://github.com/ceph/ceph/pull/6292>`_, John Coyle)
+* rgw: RGWZoneParams::create should not handle -EEXIST error (`pr#7927 <http://github.com/ceph/ceph/pull/7927>`_, Orit Wasserman)
+* rgw: s3 encoding-type for get bucket (Jeff Weber)
+* rgw: S3: set EncodingType in ListBucketResult (`pr#7712 <http://github.com/ceph/ceph/pull/7712>`_, Victor Makarov)
+* rgw: send ETag, Last-Modified for swift (#11087 Radoslaw Zarzynski)
+* rgw: set content length on container GET, PUT, DELETE, HEAD (#10971, #11036 Radoslaw Zarzynski)
+* rgw: set max buckets per user in ceph.conf (Vikhyat Umrao)
+* rgw: shard work over multiple librados instances (Pavan Rallabhandi)
+* rgw: signature mismatch with escaped characters in url query portion (`issue#15358 <http://tracker.ceph.com/issues/15358>`_, `pr#8445 <http://github.com/ceph/ceph/pull/8445>`_, Javier M. Mellid)
+* rgw: static large objects (Radoslaw Zarzynski, Yehuda Sadeh)
+* rgw: store system object meta in cache when creating it (`issue#14678 <http://tracker.ceph.com/issues/14678>`_, `pr#7615 <http://github.com/ceph/ceph/pull/7615>`_, Yehuda Sadeh)
+* rgw: support core file limit for radosgw daemon (`pr#6346 <http://github.com/ceph/ceph/pull/6346>`_, Guang Yang)
+* rgw: support end marker on swift container GET (#10682 Radoslaw Zarzynski)
+* rgw: support for aws authentication v4 (Javier M. Mellid) (`issue#10333 <http://tracker.ceph.com/issues/10333>`_, `pr#7720 <http://github.com/ceph/ceph/pull/7720>`_, Yehuda Sadeh, Javier M. Mellid)
+* rgw: support for Swift expiration API (Radoslaw Rzarzynski, Yehuda Sadeh)
+* rgw: support json format for admin policy API (Dunrong Huang) (`issue#14090 <http://tracker.ceph.com/issues/14090>`_, `pr#8036 <http://github.com/ceph/ceph/pull/8036>`_, Yehuda Sadeh, Dunrong Huang)
+* rgw: swift: allow setting attributes with COPY (#10662 Ahmad Faheem, Dmytro Iurchenko)
+* rgw: swift bulk delete (Radoslaw Zarzynski)
+* rgw: swift: do not override sent content type (#12363 Orit Wasserman)
+* rgw: swift: enforce Content-Type in response (#12157 Radoslaw Zarzynski)
+* rgw: swift: fix account listing (#11501 Radoslaw Zarzynski)
+* rgw: swift: fix metadata handling on copy (#10645 Radoslaw Zarzynski)
+* rgw: swift: send Last-Modified header (#10650 Radoslaw Zarzynski)
+* rgw: swift: set Content-Length for account GET (#12158 Radoslav Zarzynski)
+* rgw: swift: set content-length on keystone tokens (#11473 Herv Rousseau)
+* rgw: swift use Civetweb ssl can not get right url (`issue#13628 <http://tracker.ceph.com/issues/13628>`_, `pr#6408 <http://github.com/ceph/ceph/pull/6408>`_, Weijun Duan)
+* rgw: swift versioning disabled (`pr#8066 <http://github.com/ceph/ceph/pull/8066>`_, Yehuda Sadeh, Radoslaw Zarzynski)
+* rgw: sync fixes 3 (`pr#8170 <http://github.com/ceph/ceph/pull/8170>`_, Yehuda Sadeh)
+* rgw: sync fixes 4 (`pr#8190 <http://github.com/ceph/ceph/pull/8190>`_, Yehuda Sadeh)
+* rgw sync fixes (`pr#8095 <http://github.com/ceph/ceph/pull/8095>`_, Yehuda Sadeh)
+* rgw: the map 'headers' is assigned a wrong value (`pr#8481 <http://github.com/ceph/ceph/pull/8481>`_, weiqiaomiao)
+* rgw: try to parse Keystone token in order appropriate to configuration. (`pr#7822 <http://github.com/ceph/ceph/pull/7822>`_, Radoslaw Zarzynski)
+* rgw: update keystone cache with token info (#11125 Yehuda Sadeh)
+* rgw: update to latest civetweb, enable config for IPv6 (#10965 Yehuda Sadeh)
+* rgw: use attrs from source bucket on copy (#11639 Javier M. Mellid)
+* rgw: use correct oid for gc chains (#11447 Yehuda Sadeh)
+* rgw:Use count fn in RGWUserBuckets for quota check (`pr#8294 <http://github.com/ceph/ceph/pull/8294>`_, Abhishek Lekshmanan)
+* rgw: use pimpl pattern for RGWPeriodHistory (`pr#7809 <http://github.com/ceph/ceph/pull/7809>`_, Casey Bodley)
+* rgw: user quota may not adjust on bucket removal (`issue#14507 <http://tracker.ceph.com/issues/14507>`_, `pr#7586 <http://github.com/ceph/ceph/pull/7586>`_, root)
+* rgw: user rm is idempotent (Orit Wasserman)
+* rgw: use smart pointer for C_Reinitwatch (`pr#6767 <http://github.com/ceph/ceph/pull/6767>`_, Orit Wasserman)
+* rgw: use unique request id for civetweb (#10295 Orit Wasserman)
+* rgw: warn on suspicious civetweb frontend parameters (`pr#6944 <http://github.com/ceph/ceph/pull/6944>`_, Matt Benjamin)
+* rocksdb: add perf counters for get/put latency (Xinxin Shu)
+* rocksdb: build with PORTABLE=1 (`pr#6311 <http://github.com/ceph/ceph/pull/6311>`_, Sage Weil)
+* rocksdb, leveldb: fix compact_on_mount (Xiaoxi Chen)
+* rocksdb: pass options as single string (Xiaoxi Chen)
+* rocksdb: remove rdb source files from dist tarball (`issue#13554 <http://tracker.ceph.com/issues/13554>`_, `pr#6379 <http://github.com/ceph/ceph/pull/6379>`_, Kefu Chai)
+* rocksdb: remove rdb sources from dist tarball (`issue#13554 <http://tracker.ceph.com/issues/13554>`_, `pr#7105 <http://github.com/ceph/ceph/pull/7105>`_, Venky Shankar)
+* rocksdb: update to latest (Xiaoxi Chen)
+* rocksdb: use native rocksdb makefile (and our autotools) (`pr#6290 <http://github.com/ceph/ceph/pull/6290>`_, Sage Weil)
+* rpm: add suse firewall files (Tim Serong)
+* rpm: always rebuild and install man pages for rpm (Owen Synge)
+* rpm: ceph.spec.in: correctly declare systemd dependency for SLE/openSUSE (`pr#6114 <http://github.com/ceph/ceph/pull/6114>`_, Nathan Cutler)
+* rpm: ceph.spec.in: fix libs-compat / devel-compat conditional (`issue#12315 <http://tracker.ceph.com/issues/12315>`_, `pr#5219 <http://github.com/ceph/ceph/pull/5219>`_, Ken Dreyer)
+* rpm,deb: remove conditional BuildRequires for btrfs-progs (`issue#15042 <http://tracker.ceph.com/issues/15042>`_, `pr#8016 <http://github.com/ceph/ceph/pull/8016>`_, Erwan Velu)
+* rpm: loosen ceph-test dependencies (Ken Dreyer)
+* rpm: many spec file fixes (Owen Synge, Ken Dreyer)
+* rpm: misc fixes (Boris Ranto, Owen Synge, Ken Dreyer, Ira Cooper)
+* rpm: misc systemd and SUSE fixes (Owen Synge, Nathan Cutler)
+* rpm: move %post(un) ldconfig calls to ceph-base (`issue#14940 <http://tracker.ceph.com/issues/14940>`_, `pr#7867 <http://github.com/ceph/ceph/pull/7867>`_, Nathan Cutler)
+* rpm: move runtime dependencies to ceph-base and fix other packaging issues (`issue#14864 <http://tracker.ceph.com/issues/14864>`_, `pr#7826 <http://github.com/ceph/ceph/pull/7826>`_, Nathan Cutler)
+* rpm: prefer UID/GID 167 when creating ceph user/group (`issue#15246 <http://tracker.ceph.com/issues/15246>`_, `pr#8277 <http://github.com/ceph/ceph/pull/8277>`_, Nathan Cutler)
+* rpm: remove sub-package dependencies on "ceph" (`issue#15146 <http://tracker.ceph.com/issues/15146>`_, `pr#8137 <http://github.com/ceph/ceph/pull/8137>`_, Ken Dreyer)
+* rpm: rhel 5.9 librados compile fix, moved blkid to RBD check/compilation (`issue#13177 <http://tracker.ceph.com/issues/13177>`_, `pr#5954 <http://github.com/ceph/ceph/pull/5954>`_, Rohan Mars)
+* script: add missing stop_rgw variable to stop.sh script (`pr#7959 <http://github.com/ceph/ceph/pull/7959>`_, Karol Mroz)
+* scripts: adjust mstart and mstop script to run with cmake build (`pr#6920 <http://github.com/ceph/ceph/pull/6920>`_, Orit Wasserman)
+* scripts: release_notes can track original issue (`pr#6009 <http://github.com/ceph/ceph/pull/6009>`_, Abhishek Lekshmanan)
+* script: subscription-manager support (`issue#14972 <http://tracker.ceph.com/issues/14972>`_, `pr#7907 <http://github.com/ceph/ceph/pull/7907>`_, Loic Dachary)
+* selinux: allow log files to be located in /var/log/radosgw (`pr#7604 <http://github.com/ceph/ceph/pull/7604>`_, Boris Ranto)
+* selinux policy (Boris Ranto, Milan Broz)
+* selinux: Update policy to grant additional access (`issue#14870 <http://tracker.ceph.com/issues/14870>`_, `pr#7971 <http://github.com/ceph/ceph/pull/7971>`_, Boris Ranto)
+* set 128MB tcmalloc cache size by bytes (`pr#8427 <http://github.com/ceph/ceph/pull/8427>`_, Star Guo)
+* sstring.hh: return type from str_len(...) need not be const (`pr#7679 <http://github.com/ceph/ceph/pull/7679>`_, Matt Benjamin)
+* stringify outputted error code and fix unmatched parentheses. (`pr#6998 <http://github.com/ceph/ceph/pull/6998>`_, xie.xingguo, xie xingguo)
+* Striper: reduce assemble_result log level (`pr#8426 <http://github.com/ceph/ceph/pull/8426>`_, Jason Dillaman)
+* submodules: revert an accidental change (`pr#7929 <http://github.com/ceph/ceph/pull/7929>`_, Yehuda Sadeh)
+* systemd: correctly escape block device paths (`issue#14706 <http://tracker.ceph.com/issues/14706>`_, `pr#7579 <http://github.com/ceph/ceph/pull/7579>`_, James Page)
+* systemd: drop any systemd imposed process/thread limits (`pr#8450 <http://github.com/ceph/ceph/pull/8450>`_, James Page)
+* systemd: fix typos (`pr#6679 <http://github.com/ceph/ceph/pull/6679>`_, Tobias Suckow)
+* systemd: logrotate fixes (Tim Serong, Lars Marowsky-Bree, Nathan Cutler)
+* systemd: many fixes (Sage Weil, Owen Synge, Boris Ranto, Dan van der Ster)
+* systemd: run daemons as user ceph
+* systemd: set up environment in rbdmap unit file (`issue#14984 <http://tracker.ceph.com/issues/14984>`_, `pr#8222 <http://github.com/ceph/ceph/pull/8222>`_, Nathan Cutler)
+* systemd: start/stop/restart ceph services by daemon type (`issue#13497 <http://tracker.ceph.com/issues/13497>`_, `pr#6276 <http://github.com/ceph/ceph/pull/6276>`_, Zhi Zhang)
+* sysvinit: allow custom cluster names (`pr#6732 <http://github.com/ceph/ceph/pull/6732>`_, Richard Chan)
+* sysvinit compat: misc fixes (Owen Synge)
+* test: add missing shut_down mock method (`pr#8125 <http://github.com/ceph/ceph/pull/8125>`_, Jason Dillaman)
+* test/bufferlist: Avoid false-positive tests (`pr#7955 <http://github.com/ceph/ceph/pull/7955>`_, Erwan Velu)
+* test: ceph_test_rados: use less CPU (`pr#7513 <http://github.com/ceph/ceph/pull/7513>`_, Samuel Just)
+* test/cli-integration/rbd: disable progress output (`issue#14931 <http://tracker.ceph.com/issues/14931>`_, `pr#7858 <http://github.com/ceph/ceph/pull/7858>`_, Josh Durgin)
+* test: correct librbd errors discovered with unoptimized cmake build (`pr#7914 <http://github.com/ceph/ceph/pull/7914>`_, Jason Dillaman)
+* test: create pools for rbd tests with different prefix (`pr#7738 <http://github.com/ceph/ceph/pull/7738>`_, Mykola Golub)
+* test: enable test for bug #2339 which has been resolved. (`pr#7743 <http://github.com/ceph/ceph/pull/7743>`_, You Ji)
+* test/encoding/readable.sh fix (`pr#6714 <http://github.com/ceph/ceph/pull/6714>`_, Igor Podoski)
+* Test exit values on test.sh, fix tier.cc (`issue#15165 <http://tracker.ceph.com/issues/15165>`_, `pr#8266 <http://github.com/ceph/ceph/pull/8266>`_, Samuel Just)
+* test: fix issues discovered via the rbd permissions test case (`pr#8129 <http://github.com/ceph/ceph/pull/8129>`_, Jason Dillaman)
+* test: fix osd-scrub-snaps.sh (`pr#6697 <http://github.com/ceph/ceph/pull/6697>`_, Xinze Chi)
+* test: Fix test to run with btrfs which has snap_### dirs (`issue#15347 <http://tracker.ceph.com/issues/15347>`_, `pr#8420 <http://github.com/ceph/ceph/pull/8420>`_, David Zafman)
+* test: fixup and improvements for rbd-mirror test (`pr#8090 <http://github.com/ceph/ceph/pull/8090>`_, Mykola Golub)
+* test: fix ut test failure caused by lfn change (`issue#15464 <http://tracker.ceph.com/issues/15464>`_, `pr#8544 <http://github.com/ceph/ceph/pull/8544>`_, xie xingguo)
+* test: fix valgrind memcheck issues for rbd-mirror test cases (`issue#15354 <http://tracker.ceph.com/issues/15354>`_, `pr#8493 <http://github.com/ceph/ceph/pull/8493>`_, Jason Dillaman)
+* test: handle exception thrown from close during rbd lock test (`pr#8124 <http://github.com/ceph/ceph/pull/8124>`_, Jason Dillaman)
+* test/libcephfs/flock: add sys/file.h include for flock operations (`pr#6310 <http://github.com/ceph/ceph/pull/6310>`_, John Coyle)
+* test/librados/test.cc: clean up EC pools' crush rules too (`issue#13878 <http://tracker.ceph.com/issues/13878>`_, `pr#6788 <http://github.com/ceph/ceph/pull/6788>`_, Loic Dachary, Dan Mick)
+* test/librbd/fsx: Use c++11 std::mt19937 generator instead of random_r() (`pr#6332 <http://github.com/ceph/ceph/pull/6332>`_, John Coyle)
+* test: misc fs test improvements (John Spray, Loic Dachary)
+* test/mon/osd-erasure-code-profile: pick new mon port (`pr#7161 <http://github.com/ceph/ceph/pull/7161>`_, Sage Weil)
+* test: more debug logging for TestWatchNotify (`pr#7737 <http://github.com/ceph/ceph/pull/7737>`_, Mykola Golub)
+* test: new librbd flatten test case (`pr#7609 <http://github.com/ceph/ceph/pull/7609>`_, Jason Dillaman)
+* test/osd: Relax the timing intervals in osd-markdown.sh (`pr#7899 <http://github.com/ceph/ceph/pull/7899>`_, Dan Mick)
+* test_pool_create.sh: put test files in the test dir so they are cleaned up (`pr#8219 <http://github.com/ceph/ceph/pull/8219>`_, Josh Durgin)
+* test/pybind/test_ceph_argparse: fix reweight-by-utilization tests (`pr#8027 <http://github.com/ceph/ceph/pull/8027>`_, Kefu Chai, Sage Weil)
+* test: python tests, linter cleanup (Alfredo Deza)
+* test/radosgw-admin: update the expected usage outputs (`pr#7723 <http://github.com/ceph/ceph/pull/7723>`_, Kefu Chai)
+* test: rbd-mirror: add "switch to the next tag" test (`pr#8149 <http://github.com/ceph/ceph/pull/8149>`_, Mykola Golub)
+* test: rbd-mirror: compare positions using all fields (`pr#8172 <http://github.com/ceph/ceph/pull/8172>`_, Mykola Golub)
+* test: rbd-mirror: script improvements for manual testing (`pr#8325 <http://github.com/ceph/ceph/pull/8325>`_, Mykola Golub)
+* test: reproducer for writeback CoW deadlock (`pr#8009 <http://github.com/ceph/ceph/pull/8009>`_, Jason Dillaman)
+* test/rgw: add multisite test for meta sync across periods (`pr#7887 <http://github.com/ceph/ceph/pull/7887>`_, Casey Bodley)
+* test_rgw_admin: use freopen for output redirection. (`pr#6303 <http://github.com/ceph/ceph/pull/6303>`_, John Coyle)
+* tests: add const for ec test (`pr#6911 <http://github.com/ceph/ceph/pull/6911>`_, Michal Jarzabek)
+* tests: add Ubuntu 16.04 xenial dockerfile (`pr#8519 <http://github.com/ceph/ceph/pull/8519>`_, Loic Dachary)
+* tests: allow docker-test.sh to run under root (`issue#13355 <http://tracker.ceph.com/issues/13355>`_, `pr#6173 <http://github.com/ceph/ceph/pull/6173>`_, Loic Dachary)
+* tests: allow object corpus readable test to skip specific incompat instances (`pr#6932 <http://github.com/ceph/ceph/pull/6932>`_, Igor Podoski)
+* tests: centos7 needs the Continuous Release (CR) Repository enabled for (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#6842 <http://github.com/ceph/ceph/pull/6842>`_, Brad Hubbard)
+* tests: ceph-disk.sh: should use "readlink -f" instead (`pr#7594 <http://github.com/ceph/ceph/pull/7594>`_, Kefu Chai)
+* tests: ceph-disk.sh: use "readlink -f" instead for fullpath (`pr#7606 <http://github.com/ceph/ceph/pull/7606>`_, Kefu Chai)
+* tests: ceph-disk workunit uses configobj  (`pr#6342 <http://github.com/ceph/ceph/pull/6342>`_, Loic Dachary)
+* tests: ceph-helpers assert success getting backfills (`pr#6699 <http://github.com/ceph/ceph/pull/6699>`_, Loic Dachary)
+* tests: ceph_test_keyvaluedb_iterators: fix broken test (`pr#6597 <http://github.com/ceph/ceph/pull/6597>`_, Haomai Wang)
+* tests: concatenate test_rados_test_tool from src and qa (`issue#13691 <http://tracker.ceph.com/issues/13691>`_, `pr#6464 <http://github.com/ceph/ceph/pull/6464>`_, Loic Dachary)
+* tests: configure with rocksdb by default (`issue#14220 <http://tracker.ceph.com/issues/14220>`_, `pr#7100 <http://github.com/ceph/ceph/pull/7100>`_, Loic Dachary)
+* tests: destroy testprofile before creating one (`issue#13664 <http://tracker.ceph.com/issues/13664>`_, `pr#6446 <http://github.com/ceph/ceph/pull/6446>`_, Loic Dachary)
+* tests: fix a few build warnings (`pr#7608 <http://github.com/ceph/ceph/pull/7608>`_, Sage Weil)
+* tests: fixes for rbd xstests (Douglas Fuller)
+* tests: fix failure for osd-scrub-snap.sh (`issue#13986 <http://tracker.ceph.com/issues/13986>`_, `pr#6890 <http://github.com/ceph/ceph/pull/6890>`_, Loic Dachary, Ning Yao)
+* tests: Fix for make check. (`pr#7102 <http://github.com/ceph/ceph/pull/7102>`_, David Zafman)
+* tests: Fixing broken test/cephtool-test-mon.sh test (`pr#8429 <http://github.com/ceph/ceph/pull/8429>`_, Erwan Velu)
+* tests: fix race condition testing auto scrub (`issue#13592 <http://tracker.ceph.com/issues/13592>`_, `pr#6724 <http://github.com/ceph/ceph/pull/6724>`_, Xinze Chi, Loic Dachary)
+* tests: fix test_rados_tools.sh rados lookup (`issue#13691 <http://tracker.ceph.com/issues/13691>`_, `pr#6502 <http://github.com/ceph/ceph/pull/6502>`_, Loic Dachary)
+* tests: fix tiering health checks (Loic Dachary)
+* tests: fix typo in TestClsRbd.snapshots test case (`issue#13727 <http://tracker.ceph.com/issues/13727>`_, `pr#6504 <http://github.com/ceph/ceph/pull/6504>`_, Jason Dillaman)
+* tests: flush op work queue prior to destroying MockImageCtx (`issue#14092 <http://tracker.ceph.com/issues/14092>`_, `pr#7002 <http://github.com/ceph/ceph/pull/7002>`_, Jason Dillaman)
+* tests for low-level performance (Haomai Wang)
+* tests: ignore test-suite.log (`pr#6584 <http://github.com/ceph/ceph/pull/6584>`_, Loic Dachary)
+* tests: Improving 'make check' execution time (`pr#8131 <http://github.com/ceph/ceph/pull/8131>`_, Erwan Velu)
+* tests: many ec non-regression improvements (Loic Dachary)
+* tests: many many ec test improvements (Loic Dachary)
+* tests: notification slave needs to wait for master (`issue#13810 <http://tracker.ceph.com/issues/13810>`_, `pr#7220 <http://github.com/ceph/ceph/pull/7220>`_, Jason Dillaman)
+* tests: --osd-scrub-load-threshold=2000 for more consistency (`issue#14027 <http://tracker.ceph.com/issues/14027>`_, `pr#6871 <http://github.com/ceph/ceph/pull/6871>`_, Loic Dachary)
+* tests: osd-scrub-snaps.sh to display full osd logs on error (`issue#13986 <http://tracker.ceph.com/issues/13986>`_, `pr#6857 <http://github.com/ceph/ceph/pull/6857>`_, Loic Dachary)
+* tests: port uniqueness reminder (`pr#6387 <http://github.com/ceph/ceph/pull/6387>`_, Loic Dachary)
+* tests: restore run-cli-tests (`pr#6571 <http://github.com/ceph/ceph/pull/6571>`_, Loic Dachary, Sage Weil, Jason Dillaman)
+* tests: snap rename and rebuild object map in client update test (`pr#7224 <http://github.com/ceph/ceph/pull/7224>`_, Jason Dillaman)
+* tests: sync ceph-erasure-code-corpus for mktemp -d (`pr#7596 <http://github.com/ceph/ceph/pull/7596>`_, Loic Dachary)
+* tests: test/librados/test.cc must create profile (`issue#13664 <http://tracker.ceph.com/issues/13664>`_, `pr#6452 <http://github.com/ceph/ceph/pull/6452>`_, Loic Dachary)
+* tests: test_pidfile.sh lingering processes (`issue#14834 <http://tracker.ceph.com/issues/14834>`_, `pr#7734 <http://github.com/ceph/ceph/pull/7734>`_, Loic Dachary)
+* tests: unittest_bufferlist: fix hexdump test (`pr#7152 <http://github.com/ceph/ceph/pull/7152>`_, Sage Weil)
+* tests: unittest_ipaddr: fix segv (`pr#7154 <http://github.com/ceph/ceph/pull/7154>`_, Sage Weil)
+* test/system/rados_list_parallel: print oid if rados_write fails (`issue#15240 <http://tracker.ceph.com/issues/15240>`_, `pr#8309 <http://github.com/ceph/ceph/pull/8309>`_, Kefu Chai)
+* test/system/\*: use dynamically generated pool name (`issue#15240 <http://tracker.ceph.com/issues/15240>`_, `pr#8318 <http://github.com/ceph/ceph/pull/8318>`_, Kefu Chai)
+* test/test-erasure-code.sh: disable pg_temp priming (`issue#15211 <http://tracker.ceph.com/issues/15211>`_, `pr#8260 <http://github.com/ceph/ceph/pull/8260>`_, Sage Weil)
+* test: TestMirroringWatcher test cases were not closing images (`pr#8435 <http://github.com/ceph/ceph/pull/8435>`_, Jason Dillaman)
+* test/TestPGLog: fix the FTBFS (`issue#14930 <http://tracker.ceph.com/issues/14930>`_, `pr#7855 <http://github.com/ceph/ceph/pull/7855>`_, Kefu Chai)
+* test/test_pool_create.sh: fix port (`pr#8361 <http://github.com/ceph/ceph/pull/8361>`_, Sage Weil)
+* test/time: no need to abs(uint64_t) for comparing (`pr#7726 <http://github.com/ceph/ceph/pull/7726>`_, Kefu Chai)
+* test: update rbd integration cram tests for new default features (`pr#8001 <http://github.com/ceph/ceph/pull/8001>`_, Jason Dillaman)
+* test: use sequential journal_tid for object cacher test (`issue#13877 <http://tracker.ceph.com/issues/13877>`_, `pr#6710 <http://github.com/ceph/ceph/pull/6710>`_, Josh Durgin)
+* tools: add cephfs-table-tool 'take_inos' (`pr#6655 <http://github.com/ceph/ceph/pull/6655>`_, John Spray)
+* tools/cephfs: add tmap_upgrade (`pr#7003 <http://github.com/ceph/ceph/pull/7003>`_, John Spray)
+* tools/cephfs: fix overflow writing header to fixed size buffer (#13816) (`pr#6617 <http://github.com/ceph/ceph/pull/6617>`_, John Spray)
+* tools/cephfs: fix tmap_upgrade (`issue#15135 <http://tracker.ceph.com/issues/15135>`_, `pr#8128 <http://github.com/ceph/ceph/pull/8128>`_, John Spray)
+* tools: ceph_monstore_tool: add inflate-pgmap command (`issue#14217 <http://tracker.ceph.com/issues/14217>`_, `pr#7097 <http://github.com/ceph/ceph/pull/7097>`_, Kefu Chai)
+* tools: ceph-monstore-update-crush: add "--test" when testing crushmap (`pr#6418 <http://github.com/ceph/ceph/pull/6418>`_, Kefu Chai)
+* tools: Fix layout handing in cephfs-data-scan (#13898) (`pr#6719 <http://github.com/ceph/ceph/pull/6719>`_, John Spray)
+* tools: monstore: add 'show-versions' command. (`pr#7073 <http://github.com/ceph/ceph/pull/7073>`_, Cilang Zhao)
+* tools/rados: reduce "rados put" memory usage by op_size (`pr#7928 <http://github.com/ceph/ceph/pull/7928>`_, Piotr Dałek)
+* tools:remove duplicate references (`pr#5917 <http://github.com/ceph/ceph/pull/5917>`_, Bo Cai)
+* tools: support printing part cluster map in readable fashion (`issue#13079 <http://tracker.ceph.com/issues/13079>`_, `pr#5921 <http://github.com/ceph/ceph/pull/5921>`_, Bo Cai)
+* unittest_compression_zlib: do not assume buffer will be null terminated (`pr#8064 <http://github.com/ceph/ceph/pull/8064>`_, Sage Weil)
+* unittest_erasure_code_plugin: fix deadlock (Alpine) (`pr#8314 <http://github.com/ceph/ceph/pull/8314>`_, John Coyle)
+* unittest_osdmap: default crush tunables now firefly (`pr#8098 <http://github.com/ceph/ceph/pull/8098>`_, Sage Weil)
+* upstart: throttle restarts (#11798 Sage Weil, Greg Farnum)
+* vstart: fix up cmake paths when VSTART_DEST is given (`pr#8363 <http://github.com/ceph/ceph/pull/8363>`_, Casey Bodley)
+* vstart: grant full access to Swift testing account (`pr#6239 <http://github.com/ceph/ceph/pull/6239>`_, Yuan Zhou)
+* vstart: make -k with optional mon_num. (`pr#8251 <http://github.com/ceph/ceph/pull/8251>`_, Jianpeng Ma)
+* vstart: set cephfs root uid/gid to caller (`pr#6255 <http://github.com/ceph/ceph/pull/6255>`_, John Spray)
+* vstart.sh: add mstart, mstop, mrun wrappers for running multiple vstart-style test clusters out of src tree (`pr#6901 <http://github.com/ceph/ceph/pull/6901>`_, Yehuda Sadeh)
+* vstart.sh: avoid race condition starting rgw via vstart.sh (`issue#14829 <http://tracker.ceph.com/issues/14829>`_, `pr#7727 <http://github.com/ceph/ceph/pull/7727>`_, Javier M. Mellid)
+* vstart.sh: silence a harmless msg where btrfs is not found (`pr#7640 <http://github.com/ceph/ceph/pull/7640>`_, Patrick Donnelly)
+* xio: add prefix to xio msgr logs (`pr#8148 <http://github.com/ceph/ceph/pull/8148>`_, Roi Dayan)
+* xio: fix compilation against latest accelio (`pr#8022 <http://github.com/ceph/ceph/pull/8022>`_, Roi Dayan)
+* xio: fix incorrect ip being assigned in case of multiple RDMA ports (`pr#7747 <http://github.com/ceph/ceph/pull/7747>`_, Subramanyam Varanasi)
+* xio: remove duplicate assignment of peer addr (`pr#8025 <http://github.com/ceph/ceph/pull/8025>`_, Roi Dayan)
+* xio: remove redundant magic methods (`pr#7773 <http://github.com/ceph/ceph/pull/7773>`_, Roi Dayan)
+* xio: remove unused variable (`pr#8023 <http://github.com/ceph/ceph/pull/8023>`_, Roi Dayan)
+* xio: xio_init needs to be called before any other xio function (`pr#8227 <http://github.com/ceph/ceph/pull/8227>`_, Roi Dayan)
+* xxhash: use clone of xxhash.git; add .gitignore (`pr#7986 <http://github.com/ceph/ceph/pull/7986>`_, Sage Weil)
diff --git a/doc/releases/kraken.rst b/doc/releases/kraken.rst
new file mode 100644
index 000000000..3d39e3293
--- /dev/null
+++ b/doc/releases/kraken.rst
@@ -0,0 +1,2337 @@
+======
+Kraken
+======
+
+Kraken is the 11th stable release of Ceph.  It is named after the
+mythical kraken, a legendary sea monster in Scandinavian folklore with
+cephalopod-like appearance.
+
+v11.2.1 Kraken
+==============
+
+This is the first bugfix release for Kraken, and probably the last release of
+the Kraken series (Kraken will be declared "End Of Life" (EOL) when Luminous
+is declared stable). It contains a large number of bugfixes across all Ceph
+components.
+
+We recommend that all v11.2.x users upgrade.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v11.2.1.txt>`.
+
+Notable Changes
+---------------
+
+* In previous versions, if a client sent an op to the wrong OSD, the OSD
+  would reply with ENXIO.  The rationale here is that the client or OSD is
+  clearly buggy and we want to surface the error as clearly as possible.
+  We now only send the ENXIO reply if the osd_enxio_on_misdirected_op option
+  is enabled (it's off by default).  This means that a VM using librbd that
+  previously would have gotten an EIO and gone read-only will now see a
+  blocked/hung IO instead.
+
+* There was a bug introduced in Jewel (#19119) that broke the mapping behavior
+  when an "out" OSD that still existed in the CRUSH map was removed with 'osd rm'.
+  This could result in 'misdirected op' and other errors.  The bug is now fixed,
+  but the fix itself introduces the same risk because the behavior may vary between
+  clients and OSDs.  To avoid problems, please ensure that all OSDs are removed
+  from the CRUSH map before deleting them.  That is, be sure to do::
+
+     ceph osd crush rm osd.123
+
+  before::
+
+     ceph osd rm osd.123
+
+* This release greatly improves control and throttling of the snap trimmer. It
+  introduces the "osd max trimming pgs" option (defaulting to 2), which limits
+  how many PGs on an OSD can be trimming snapshots at a time. And it restores
+  the safe use of the "osd snap trim sleep" option, which defaults to 0 but
+  otherwise adds the given number of seconds in delay between every dispatch
+  of trim operations to the underlying system.
+
+Other Notable Changes
+---------------------
+
+* build/ops: ceph-base missing dependency for psmisc in Ubuntu Xenial (`issue#19129 <http://tracker.ceph.com/issues/19129>`_, `issue#19564 <http://tracker.ceph.com/issues/19564>`_, `pr#14425 <https://github.com/ceph/ceph/pull/14425>`_, Nathan Cutler)
+* build/ops: logrotate is missing from debian package (kraken, master) (`issue#19670 <http://tracker.ceph.com/issues/19670>`_, `issue#19390 <http://tracker.ceph.com/issues/19390>`_, `pr#14734 <https://github.com/ceph/ceph/pull/14734>`_, Kefu Chai)
+* build/ops: selinux: Do parallel relabel on package install (`issue#20077 <http://tracker.ceph.com/issues/20077>`_, `issue#20184 <http://tracker.ceph.com/issues/20184>`_, `issue#20191 <http://tracker.ceph.com/issues/20191>`_, `issue#20193 <http://tracker.ceph.com/issues/20193>`_, `pr#15509 <https://github.com/ceph/ceph/pull/15509>`_, Boris Ranto)
+* build/ops: spec file mentions non-existent ceph-create-keys systemd unit file, causing ceph-mon units to not be enabled via preset (`issue#19460 <http://tracker.ceph.com/issues/19460>`_, `pr#14315 <https://github.com/ceph/ceph/pull/14315>`_, Sébastien Han)
+* build/ops: systemd restarts Ceph Mon to quickly after failing to start (`issue#18635 <http://tracker.ceph.com/issues/18635>`_, `issue#18721 <http://tracker.ceph.com/issues/18721>`_, `pr#13185 <https://github.com/ceph/ceph/pull/13185>`_, Wido den Hollander)
+* build/ops: systemd: Start OSDs after MONs (`issue#18907 <http://tracker.ceph.com/issues/18907>`_, `issue#18516 <http://tracker.ceph.com/issues/18516>`_, `pr#13494 <https://github.com/ceph/ceph/pull/13494>`_, Boris Ranto)
+* ceph-disk: Add fix subcommand kraken back-port (`issue#19544 <http://tracker.ceph.com/issues/19544>`_, `pr#14345 <https://github.com/ceph/ceph/pull/14345>`_, Boris Ranto)
+* ceph-disk: does not support cluster names different than 'ceph' (`issue#18973 <http://tracker.ceph.com/issues/18973>`_, `issue#17821 <http://tracker.ceph.com/issues/17821>`_, `pr#13497 <https://github.com/ceph/ceph/pull/13497>`_, Loic Dachary)
+* ceph-disk: enable directory backed OSD at boot time (`issue#19628 <http://tracker.ceph.com/issues/19628>`_, `issue#19647 <http://tracker.ceph.com/issues/19647>`_, `pr#14604 <https://github.com/ceph/ceph/pull/14604>`_, Loic Dachary)
+* ceph-disk: error on _bytes2str (`issue#18431 <http://tracker.ceph.com/issues/18431>`_, `issue#18371 <http://tracker.ceph.com/issues/18371>`_, `pr#13501 <https://github.com/ceph/ceph/pull/13501>`_, Kefu Chai)
+* ceph-disk: fails if OSD udev rule triggers prior to mount of /var (`issue#20150 <http://tracker.ceph.com/issues/20150>`_, `issue#19941 <http://tracker.ceph.com/issues/19941>`_, `pr#16092 <https://github.com/ceph/ceph/pull/16092>`_, Loic Dachary)
+* ceph-disk: Fix getting wrong group name when --setgroup in bluestore (`issue#18956 <http://tracker.ceph.com/issues/18956>`_, `pr#13488 <https://github.com/ceph/ceph/pull/13488>`_, craigchi)
+* ceph-disk list reports mount error for OSD having mount options with SELinux context (`issue#19537 <http://tracker.ceph.com/issues/19537>`_, `issue#17331 <http://tracker.ceph.com/issues/17331>`_, `pr#14403 <https://github.com/ceph/ceph/pull/14403>`_, Brad Hubbard)
+* ceph-disk prepare get wrong group name in bluestore (`issue#18997 <http://tracker.ceph.com/issues/18997>`_, `pr#13543 <https://github.com/ceph/ceph/pull/13543>`_, craigchi)
+* ceph-disk: Racing between partition creation & device node creation (`issue#20034 <http://tracker.ceph.com/issues/20034>`_, `pr#16138 <https://github.com/ceph/ceph/pull/16138>`_, Erwan Velu)
+* ceph-disk: separate ceph-osd --check-needs-\* logs (`issue#20010 <http://tracker.ceph.com/issues/20010>`_, `issue#19888 <http://tracker.ceph.com/issues/19888>`_, `pr#16135 <https://github.com/ceph/ceph/pull/16135>`_, Loic Dachary)
+* cephfs: buffer overflow in test LibCephFS.DirLs (`issue#18941 <http://tracker.ceph.com/issues/18941>`_, `issue#19045 <http://tracker.ceph.com/issues/19045>`_, `pr#14571 <https://github.com/ceph/ceph/pull/14571>`_, "Yan, Zheng")
+* cephfs: ceph-fuse crash during snapshot tests (`issue#18552 <http://tracker.ceph.com/issues/18552>`_, `issue#18460 <http://tracker.ceph.com/issues/18460>`_, `pr#14563 <https://github.com/ceph/ceph/pull/14563>`_, Yan, Zheng)
+* cephfs: ceph-fuse does not recover after lost connection to MDS (`issue#19678 <http://tracker.ceph.com/issues/19678>`_, `issue#18757 <http://tracker.ceph.com/issues/18757>`_, `pr#16105 <https://github.com/ceph/ceph/pull/16105>`_, Henrik Korkuc)
+* cephfs: client: fix the cross-quota rename boundary check conditions (`issue#18700 <http://tracker.ceph.com/issues/18700>`_, `pr#14567 <https://github.com/ceph/ceph/pull/14567>`_, Greg Farnum)
+* cephfs: Deadlock on two ceph-fuse clients accessing the same file (`issue#20028 <http://tracker.ceph.com/issues/20028>`_, `issue#19635 <http://tracker.ceph.com/issues/19635>`_, `pr#16191 <https://github.com/ceph/ceph/pull/16191>`_, "Yan, Zheng")
+* cephfs: fragment space check can cause replayed request fail (`issue#18660 <http://tracker.ceph.com/issues/18660>`_, `issue#18706 <http://tracker.ceph.com/issues/18706>`_, `pr#14568 <https://github.com/ceph/ceph/pull/14568>`_, "Yan, Zheng")
+* cephfs: MDS crashes on missing metadata object (`issue#18179 <http://tracker.ceph.com/issues/18179>`_, `issue#18566 <http://tracker.ceph.com/issues/18566>`_, `pr#14565 <https://github.com/ceph/ceph/pull/14565>`_, Yan, Zheng)
+* cephfs: MDS heartbeat timeout during rejoin, when working with large amount of caps/inodes (`issue#19118 <http://tracker.ceph.com/issues/19118>`_, `issue#19335 <http://tracker.ceph.com/issues/19335>`_, `pr#14572 <https://github.com/ceph/ceph/pull/14572>`_, John Spray)
+* cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file (`issue#19674 <http://tracker.ceph.com/issues/19674>`_, `issue#19033 <http://tracker.ceph.com/issues/19033>`_, `pr#16103 <https://github.com/ceph/ceph/pull/16103>`_, Yang Honggang)
+* cephfs: MDS server crashes due to inconsistent metadata (`issue#19406 <http://tracker.ceph.com/issues/19406>`_, `issue#19620 <http://tracker.ceph.com/issues/19620>`_, `pr#14574 <https://github.com/ceph/ceph/pull/14574>`_, John Spray)
+* cephfs: mds/StrayManager: avoid reusing deleted inode in StrayManager::_purge_stray_logged (`issue#18950 <http://tracker.ceph.com/issues/18950>`_, `pr#14570 <https://github.com/ceph/ceph/pull/14570>`_, Zhi Zhang)
+* cephfs: mount point break off problem after mds switch (`issue#19667 <http://tracker.ceph.com/issues/19667>`_, `issue#19437 <http://tracker.ceph.com/issues/19437>`_, `pr#16100 <https://github.com/ceph/ceph/pull/16100>`_, Guan yunfei, Sage Weil)
+* cephfs: non-local quota changes not visible until some IO is done (`issue#17939 <http://tracker.ceph.com/issues/17939>`_, `issue#19763 <http://tracker.ceph.com/issues/19763>`_, `pr#16108 <https://github.com/ceph/ceph/pull/16108>`_, John Spray)
+* cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command (`issue#19483 <http://tracker.ceph.com/issues/19483>`_, `issue#16709 <http://tracker.ceph.com/issues/16709>`_, `pr#14573 <https://github.com/ceph/ceph/pull/14573>`_, John Spray)
+* cephfs: normalize file open flags internally used by cephfs (`issue#19845 <http://tracker.ceph.com/issues/19845>`_, `pr#14998 <https://github.com/ceph/ceph/pull/14998>`_, Jan Fajerski)
+* cephfs: segfault in handle_client_caps (`issue#18306 <http://tracker.ceph.com/issues/18306>`_, `issue#18616 <http://tracker.ceph.com/issues/18616>`_, `pr#14566 <https://github.com/ceph/ceph/pull/14566>`_, Yan, Zheng)
+* cephfs: speed up readdir by skipping unwanted dn (`issue#18531 <http://tracker.ceph.com/issues/18531>`_, `pr#13028 <https://github.com/ceph/ceph/pull/13028>`_, Xiaoxi Chen)
+* cephfs: src/test/pybind/test_cephfs.py fails (`issue#20500 <http://tracker.ceph.com/issues/20500>`_, `issue#19890 <http://tracker.ceph.com/issues/19890>`_, `pr#16114 <https://github.com/ceph/ceph/pull/16114>`_, "Yan, Zheng")
+* cephfs: test_client_recovery.TestClientRecovery fails (`issue#18562 <http://tracker.ceph.com/issues/18562>`_, `issue#18396 <http://tracker.ceph.com/issues/18396>`_, `pr#14564 <https://github.com/ceph/ceph/pull/14564>`_, Yan, Zheng)
+* cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa) (`issue#18574 <http://tracker.ceph.com/issues/18574>`_, `issue#18604 <http://tracker.ceph.com/issues/18604>`_, `pr#13024 <https://github.com/ceph/ceph/pull/13024>`_, John Spray)
+* cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) (`issue#18914 <http://tracker.ceph.com/issues/18914>`_, `issue#19676 <http://tracker.ceph.com/issues/19676>`_, `pr#16104 <https://github.com/ceph/ceph/pull/16104>`_, "Yan, Zheng")
+* cephfs: test_open_inode fails (`issue#18899 <http://tracker.ceph.com/issues/18899>`_, `issue#18661 <http://tracker.ceph.com/issues/18661>`_, `pr#14569 <https://github.com/ceph/ceph/pull/14569>`_, John Spray)
+* client: populate metadata during mount (`issue#18361 <http://tracker.ceph.com/issues/18361>`_, `issue#18540 <http://tracker.ceph.com/issues/18540>`_, `pr#12951 <https://github.com/ceph/ceph/pull/12951>`_, John Spray)
+* client: segfault on ceph_rmdir path / (`issue#18612 <http://tracker.ceph.com/issues/18612>`_, `issue#9935 <http://tracker.ceph.com/issues/9935>`_, `pr#13030 <https://github.com/ceph/ceph/pull/13030>`_, Michal Jarzabek)
+* cls_rbd: default initialize snapshot namespace for legacy clients (`issue#19413 <http://tracker.ceph.com/issues/19413>`_, `issue#19833 <http://tracker.ceph.com/issues/19833>`_, `pr#14934 <https://github.com/ceph/ceph/pull/14934>`_, Jason Dillaman)
+* cls/rgw: list_plain_entries() stops before bi_log entries (`issue#19876 <http://tracker.ceph.com/issues/19876>`_, `issue#20015 <http://tracker.ceph.com/issues/20015>`_, `pr#15384 <https://github.com/ceph/ceph/pull/15384>`_, Casey Bodley)
+* common: monitor creation with IPv6 public network segfaults (`issue#19465 <http://tracker.ceph.com/issues/19465>`_, `issue#19371 <http://tracker.ceph.com/issues/19371>`_, `pr#14323 <https://github.com/ceph/ceph/pull/14323>`_, Fabian Grünbichler)
+* common: possible lockdep false alarm for ThreadPool lock (`issue#18819 <http://tracker.ceph.com/issues/18819>`_, `issue#18894 <http://tracker.ceph.com/issues/18894>`_, `pr#13487 <https://github.com/ceph/ceph/pull/13487>`_, Mykola Golub)
+* core: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure (`issue#19561 <http://tracker.ceph.com/issues/19561>`_, `issue#15368 <http://tracker.ceph.com/issues/15368>`_, `pr#14733 <https://github.com/ceph/ceph/pull/14733>`_, Sage Weil)
+* core: bluestore bdev: flush no-op optimization is racy (`issue#20495 <http://tracker.ceph.com/issues/20495>`_, `issue#19326 <http://tracker.ceph.com/issues/19326>`_, `issue#19327 <http://tracker.ceph.com/issues/19327>`_, `issue#19250 <http://tracker.ceph.com/issues/19250>`_, `issue#19251 <http://tracker.ceph.com/issues/19251>`_, `pr#14736 <https://github.com/ceph/ceph/pull/14736>`_, Sage Weil)
+* core: improve control and throttling of the snap trimmer (`issue#19329 <http://tracker.ceph.com/issues/19329>`_, `issue#19931 <http://tracker.ceph.com/issues/19931>`_, `pr#14597 <https://github.com/ceph/ceph/pull/14597>`_, Samuel Just, Greg Farnum)
+* core: two instances of omap_digest mismatch (`issue#19391 <http://tracker.ceph.com/issues/19391>`_, `pr#14200 <https://github.com/ceph/ceph/pull/14200>`_, Samuel Just, David Zafman)
+* doc: PendingReleaseNotes: warning about 'osd rm ...' and #13733 (`issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#14506 <https://github.com/ceph/ceph/pull/14506>`_, Sage Weil)
+* doc: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh (`issue#17746 <http://tracker.ceph.com/issues/17746>`_, `issue#18571 <http://tracker.ceph.com/issues/18571>`_, `pr#13044 <https://github.com/ceph/ceph/pull/13044>`_, Ronak Jain)
+* doc: rgw: admin ops: fix the quota section (`issue#19397 <http://tracker.ceph.com/issues/19397>`_, `issue#19462 <http://tracker.ceph.com/issues/19462>`_, `pr#14521 <https://github.com/ceph/ceph/pull/14521>`_, Chu, Hua-Rong)
+* fix: rgw crashed caused by shard id out of range when listing data log (`issue#20156 <http://tracker.ceph.com/issues/20156>`_, `issue#19732 <http://tracker.ceph.com/issues/19732>`_, `pr#16173 <https://github.com/ceph/ceph/pull/16173>`_, redickwang)
+* fuse: TestVolumeClient.test_evict_client failure creating pidfile (`issue#18439 <http://tracker.ceph.com/issues/18439>`_, `issue#18309 <http://tracker.ceph.com/issues/18309>`_, `pr#12813 <https://github.com/ceph/ceph/pull/12813>`_, Nathan Cutler)
+* librbd: allow to open an image without opening parent image (`issue#18609 <http://tracker.ceph.com/issues/18609>`_, `issue#18325 <http://tracker.ceph.com/issues/18325>`_, `pr#13132 <https://github.com/ceph/ceph/pull/13132>`_, Ricardo Dias)
+* librbd: corrected resize RPC message backwards compatibility (`issue#19636 <http://tracker.ceph.com/issues/19636>`_, `issue#19659 <http://tracker.ceph.com/issues/19659>`_, `pr#14620 <https://github.com/ceph/ceph/pull/14620>`_, Jason Dillaman)
+* librbd: Incomplete declaration for ContextWQ in librbd/Journal.h (`issue#18862 <http://tracker.ceph.com/issues/18862>`_, `issue#18892 <http://tracker.ceph.com/issues/18892>`_, `pr#14153 <https://github.com/ceph/ceph/pull/14153>`_, Boris Ranto)
+* librbd: is_exclusive_lock_owner API should ping OSD (`issue#19467 <http://tracker.ceph.com/issues/19467>`_, `issue#19287 <http://tracker.ceph.com/issues/19287>`_, `pr#14480 <https://github.com/ceph/ceph/pull/14480>`_, Jason Dillaman)
+* librbd: possible race in ExclusiveLock handle_peer_notification (`issue#19368 <http://tracker.ceph.com/issues/19368>`_, `pr#14163 <https://github.com/ceph/ceph/pull/14163>`_, Mykola Golub)
+* librbd: prevent self-blacklisting during break lock (`issue#18703 <http://tracker.ceph.com/issues/18703>`_, `issue#18666 <http://tracker.ceph.com/issues/18666>`_, `pr#13201 <https://github.com/ceph/ceph/pull/13201>`_, Jason Dillaman)
+* make check fails with Error EIO: load dlopen(build/lib/libec_FAKE.so): build/lib/libec_FAKE.so: cannot open shared object file: No such file or directory (`issue#20487 <http://tracker.ceph.com/issues/20487>`_, `issue#20345 <http://tracker.ceph.com/issues/20345>`_, `issue#18876 <http://tracker.ceph.com/issues/18876>`_, `pr#16069 <https://github.com/ceph/ceph/pull/16069>`_, Kefu Chai, Kyr Shatskyy)
+* mds: assert fail when shutting down (`issue#19672 <http://tracker.ceph.com/issues/19672>`_, `issue#19204 <http://tracker.ceph.com/issues/19204>`_, `pr#16102 <https://github.com/ceph/ceph/pull/16102>`_, John Spray)
+* mds: C_MDSInternalNoop::complete doesn't free itself (`issue#19664 <http://tracker.ceph.com/issues/19664>`_, `issue#19501 <http://tracker.ceph.com/issues/19501>`_, `pr#16099 <https://github.com/ceph/ceph/pull/16099>`_, "Yan, Zheng")
+* mds: daemon goes readonly writing backtrace for a file whose data pool has been removed (`issue#19669 <http://tracker.ceph.com/issues/19669>`_, `issue#19401 <http://tracker.ceph.com/issues/19401>`_, `pr#16101 <https://github.com/ceph/ceph/pull/16101>`_, John Spray)
+* mds: damage reporting by ino number is useless (`issue#18509 <http://tracker.ceph.com/issues/18509>`_, `issue#19680 <http://tracker.ceph.com/issues/19680>`_, `pr#16106 <https://github.com/ceph/ceph/pull/16106>`_, John Spray)
+* mds: Decode errors on backtrace will crash MDS (`issue#18311 <http://tracker.ceph.com/issues/18311>`_, `issue#18463 <http://tracker.ceph.com/issues/18463>`_, `pr#12835 <https://github.com/ceph/ceph/pull/12835>`_, John Spray)
+* mds: enable daemon to start when session ino info is corrupt (`issue#19710 <http://tracker.ceph.com/issues/19710>`_, `issue#16842 <http://tracker.ceph.com/issues/16842>`_, `pr#16107 <https://github.com/ceph/ceph/pull/16107>`_, John Spray)
+* mds: failed filelock.can_read(-1) assertion in Server::_dir_is_nonempty (`issue#18707 <http://tracker.ceph.com/issues/18707>`_, `issue#18578 <http://tracker.ceph.com/issues/18578>`_, `pr#13555 <https://github.com/ceph/ceph/pull/13555>`_, Yan, Zheng)
+* mds: finish clientreplay requests before requesting active state (`issue#18678 <http://tracker.ceph.com/issues/18678>`_, `issue#18461 <http://tracker.ceph.com/issues/18461>`_, `pr#13112 <https://github.com/ceph/ceph/pull/13112>`_, Yan, Zheng)
+* mds: unresponsive when truncating a very large file (`issue#19755 <http://tracker.ceph.com/issues/19755>`_, `issue#20026 <http://tracker.ceph.com/issues/20026>`_, `pr#16190 <https://github.com/ceph/ceph/pull/16190>`_, "Yan, Zheng")
+* mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version) (`issue#18366 <http://tracker.ceph.com/issues/18366>`_, `issue#18403 <http://tracker.ceph.com/issues/18403>`_, `pr#13116 <https://github.com/ceph/ceph/pull/13116>`_, Sage Weil)
+* mon crash on shutdown, lease_ack_timeout event (`issue#19928 <http://tracker.ceph.com/issues/19928>`_, `issue#19825 <http://tracker.ceph.com/issues/19825>`_, `pr#15084 <https://github.com/ceph/ceph/pull/15084>`_, Kefu Chai, Alexey Sheplyakov)
+* mon: fail to form large quorum; msg/async busy loop (`issue#20230 <http://tracker.ceph.com/issues/20230>`_, `issue#20315 <http://tracker.ceph.com/issues/20315>`_, `pr#15729 <https://github.com/ceph/ceph/pull/15729>`_, Haomai Wang)
+* mon: force_create_pg could leave pg stuck in creating state (`issue#19181 <http://tracker.ceph.com/issues/19181>`_, `issue#18298 <http://tracker.ceph.com/issues/18298>`_, `pr#13790 <https://github.com/ceph/ceph/pull/13790>`_, Adam C. Emerson, Sage Weil)
+* mon/MonClient: make get_mon_log_message() atomic (`issue#19618 <http://tracker.ceph.com/issues/19618>`_, `issue#19427 <http://tracker.ceph.com/issues/19427>`_, `pr#14588 <https://github.com/ceph/ceph/pull/14588>`_, Kefu Chai)
+* mon: 'osd crush move ...' doesnt work on osds (`issue#18682 <http://tracker.ceph.com/issues/18682>`_, `issue#18587 <http://tracker.ceph.com/issues/18587>`_, `pr#13500 <https://github.com/ceph/ceph/pull/13500>`_, Sage Weil)
+* mon: osd crush set crushmap need sanity check (`issue#19302 <http://tracker.ceph.com/issues/19302>`_, `issue#20365 <http://tracker.ceph.com/issues/20365>`_, `pr#16143 <https://github.com/ceph/ceph/pull/16143>`_, Loic Dachary)
+* mon: peon wrongly delete routed pg stats op before receive pg stats ack (`issue#18554 <http://tracker.ceph.com/issues/18554>`_, `issue#18458 <http://tracker.ceph.com/issues/18458>`_, `pr#13046 <https://github.com/ceph/ceph/pull/13046>`_, Mingxin Liu)
+* mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc (`issue#18522 <http://tracker.ceph.com/issues/18522>`_, `issue#20035 <http://tracker.ceph.com/issues/20035>`_, `pr#15237 <https://github.com/ceph/ceph/pull/15237>`_, Sage Weil)
+* msg/simple/SimpleMessenger.cc: 239: FAILED assert(!cleared) (`issue#15784 <http://tracker.ceph.com/issues/15784>`_, `issue#18378 <http://tracker.ceph.com/issues/18378>`_, `pr#16133 <https://github.com/ceph/ceph/pull/16133>`_, Sage Weil)
+* multisite: rest api fails to decode large period on 'period commit' (`issue#19505 <http://tracker.ceph.com/issues/19505>`_, `issue#19616 <http://tracker.ceph.com/issues/19616>`_, `issue#19614 <http://tracker.ceph.com/issues/19614>`_, `issue#20244 <http://tracker.ceph.com/issues/20244>`_, `issue#19488 <http://tracker.ceph.com/issues/19488>`_, `issue#19776 <http://tracker.ceph.com/issues/19776>`_, `issue#20293 <http://tracker.ceph.com/issues/20293>`_, `issue#19746 <http://tracker.ceph.com/issues/19746>`_, `pr#16161 <https://github.com/ceph/ceph/pull/16161>`_, Casey Bodley, Abhishek Lekshmanan)
+* objecter: full_try behavior not consistent with osd (`issue#19560 <http://tracker.ceph.com/issues/19560>`_, `issue#19430 <http://tracker.ceph.com/issues/19430>`_, `pr#14732 <https://github.com/ceph/ceph/pull/14732>`_, Sage Weil)
+* ojecter: epoch_barrier isn't respected in _op_submit() (`issue#19396 <http://tracker.ceph.com/issues/19396>`_, `issue#19496 <http://tracker.ceph.com/issues/19496>`_, `pr#14331 <https://github.com/ceph/ceph/pull/14331>`_, Ilya Dryomov)
+* os/bluestore: deep decode onode value (`issue#20366 <http://tracker.ceph.com/issues/20366>`_, `pr#15792 <https://github.com/ceph/ceph/pull/15792>`_, Sage Weil)
+* os/bluestore: fix Allocator::allocate() int truncation (`issue#20884 <http://tracker.ceph.com/issues/20884>`_, `issue#18595 <http://tracker.ceph.com/issues/18595>`_, `pr#13011 <https://github.com/ceph/ceph/pull/13011>`_, Sage Weil)
+* osd: allow client throttler to be adjusted on-fly, without restart (`issue#18791 <http://tracker.ceph.com/issues/18791>`_, `issue#18793 <http://tracker.ceph.com/issues/18793>`_, `pr#13216 <https://github.com/ceph/ceph/pull/13216>`_, Piotr Dałek)
+* osd: An OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed (`issue#20544 <http://tracker.ceph.com/issues/20544>`_, `issue#16878 <http://tracker.ceph.com/issues/16878>`_, `issue#19340 <http://tracker.ceph.com/issues/19340>`_, `issue#19841 <http://tracker.ceph.com/issues/19841>`_, `issue#20672 <http://tracker.ceph.com/issues/20672>`_, `pr#16134 <https://github.com/ceph/ceph/pull/16134>`_, Sage Weil, David Zafman)
+* osd: bogus assert when checking acting set on recovery completion in rados/upgrade (`issue#18999 <http://tracker.ceph.com/issues/18999>`_, `pr#13542 <https://github.com/ceph/ceph/pull/13542>`_, Sage Weil)
+* osd: calc_clone_subsets misuses try_read_lock vs missing (`issue#18610 <http://tracker.ceph.com/issues/18610>`_, `issue#18583 <http://tracker.ceph.com/issues/18583>`_, `issue#18723 <http://tracker.ceph.com/issues/18723>`_, `issue#17831 <http://tracker.ceph.com/issues/17831>`_, `pr#14616 <https://github.com/ceph/ceph/pull/14616>`_, Samuel Just)
+* osd: ceph degraded and misplaced status output inaccurate (`issue#18619 <http://tracker.ceph.com/issues/18619>`_, `issue#19480 <http://tracker.ceph.com/issues/19480>`_, `pr#14322 <https://github.com/ceph/ceph/pull/14322>`_, David Zafman)
+* osd: condition object_info_t encoding on required (not up) features (`issue#18842 <http://tracker.ceph.com/issues/18842>`_, `issue#18831 <http://tracker.ceph.com/issues/18831>`_, `issue#18814 <http://tracker.ceph.com/issues/18814>`_, `pr#13485 <https://github.com/ceph/ceph/pull/13485>`_, Ilya Dryomov)
+* osd: do not send ENXIO on misdirected op by default (`issue#19622 <http://tracker.ceph.com/issues/19622>`_, `pr#13253 <https://github.com/ceph/ceph/pull/13253>`_, Sage Weil)
+* osd: FAILED assert(object_contexts.empty()) (live on master only from Jan-Feb 2017, all other instances are different) (`issue#20522 <http://tracker.ceph.com/issues/20522>`_, `issue#20523 <http://tracker.ceph.com/issues/20523>`_, `issue#18927 <http://tracker.ceph.com/issues/18927>`_, `issue#18809 <http://tracker.ceph.com/issues/18809>`_, `pr#16132 <https://github.com/ceph/ceph/pull/16132>`_, Samuel Just)
+* osd: --flush-journal: sporadic segfaults on exit (`issue#18952 <http://tracker.ceph.com/issues/18952>`_, `issue#18820 <http://tracker.ceph.com/issues/18820>`_, `pr#13490 <https://github.com/ceph/ceph/pull/13490>`_, Alexey Sheplyakov)
+* osd: Give requested scrubs a higher priority (`issue#19685 <http://tracker.ceph.com/issues/19685>`_, `issue#15789 <http://tracker.ceph.com/issues/15789>`_, `pr#14735 <https://github.com/ceph/ceph/pull/14735>`_, David Zafman)
+* osd: Implement asynchronous scrub sleep (`issue#20033 <http://tracker.ceph.com/issues/20033>`_, `issue#19986 <http://tracker.ceph.com/issues/19986>`_, `issue#20173 <http://tracker.ceph.com/issues/20173>`_, `issue#19497 <http://tracker.ceph.com/issues/19497>`_, `pr#15526 <https://github.com/ceph/ceph/pull/15526>`_, Brad Hubbard)
+* osd: leaked MOSDMap (`issue#19760 <http://tracker.ceph.com/issues/19760>`_, `issue#18293 <http://tracker.ceph.com/issues/18293>`_, `pr#14942 <https://github.com/ceph/ceph/pull/14942>`_, Sage Weil)
+* osd: leveldb corruption leads to Operation not permitted not handled and assert (`issue#18037 <http://tracker.ceph.com/issues/18037>`_, `issue#18418 <http://tracker.ceph.com/issues/18418>`_, `pr#12790 <https://github.com/ceph/ceph/pull/12790>`_, Nathan Cutler)
+* osd: metadata reports filestore when using bluestore (`issue#18677 <http://tracker.ceph.com/issues/18677>`_, `issue#18638 <http://tracker.ceph.com/issues/18638>`_, `pr#16083 <https://github.com/ceph/ceph/pull/16083>`_, Wido den Hollander)
+* osd: New added OSD always down when full flag is set (`issue#19485 <http://tracker.ceph.com/issues/19485>`_, `pr#14321 <https://github.com/ceph/ceph/pull/14321>`_, Mingxin Liu)
+* osd: Object level shard errors are tracked and used if no auth available (`issue#20089 <http://tracker.ceph.com/issues/20089>`_, `pr#15421 <https://github.com/ceph/ceph/pull/15421>`_, David Zafman)
+* osd: os/bluestore: fix statfs to not include DB partition in free space (`issue#18599 <http://tracker.ceph.com/issues/18599>`_, `issue#18722 <http://tracker.ceph.com/issues/18722>`_, `pr#13284 <https://github.com/ceph/ceph/pull/13284>`_, Sage Weil)
+* osd: osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting) (`issue#19902 <http://tracker.ceph.com/issues/19902>`_, `issue#19916 <http://tracker.ceph.com/issues/19916>`_, `pr#15066 <https://github.com/ceph/ceph/pull/15066>`_, Kefu Chai)
+* osd: pg log split does not rebuild index for parent or child (`issue#19315 <http://tracker.ceph.com/issues/19315>`_, `issue#18975 <http://tracker.ceph.com/issues/18975>`_, `pr#14048 <https://github.com/ceph/ceph/pull/14048>`_, Sage Weil)
+* osd: pglog: with config, don't assert in the presence of stale diverg… (`issue#17916 <http://tracker.ceph.com/issues/17916>`_, `issue#19702 <http://tracker.ceph.com/issues/19702>`_, `pr#14646 <https://github.com/ceph/ceph/pull/14646>`_, Greg Farnum)
+* osd: publish PG stats when backfill-related states change (`issue#18497 <http://tracker.ceph.com/issues/18497>`_, `issue#18369 <http://tracker.ceph.com/issues/18369>`_, `pr#13295 <https://github.com/ceph/ceph/pull/13295>`_, Sage Weil)
+* osd: Revert "PrimaryLogPG::failed_push: update missing as well" (`issue#18659 <http://tracker.ceph.com/issues/18659>`_, `pr#13091 <https://github.com/ceph/ceph/pull/13091>`_, David Zafman)
+* osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal (`issue#20443 <http://tracker.ceph.com/issues/20443>`_, `pr#15962 <https://github.com/ceph/ceph/pull/15962>`_, Alexey Sheplyakov)
+* pre-jewel "osd rm" incrementals are misinterpreted (`issue#19209 <http://tracker.ceph.com/issues/19209>`_, `issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#13883 <https://github.com/ceph/ceph/pull/13883>`_, Ilya Dryomov)
+* rbd: Add missing parameter feedback to 'rbd snap limit' (`issue#18601 <http://tracker.ceph.com/issues/18601>`_, `pr#14537 <https://github.com/ceph/ceph/pull/14537>`_, Tang Jin)
+* rbd: [api] is_exclusive_lock_owner shouldn't return -EBUSY (`issue#20266 <http://tracker.ceph.com/issues/20266>`_, `issue#20182 <http://tracker.ceph.com/issues/20182>`_, `pr#16187 <https://github.com/ceph/ceph/pull/16187>`_, Jason Dillaman)
+* rbd: [api] temporarily restrict (rbd\_)mirror_peer_add from adding multiple peers (`issue#19256 <http://tracker.ceph.com/issues/19256>`_, `issue#19324 <http://tracker.ceph.com/issues/19324>`_, `pr#14545 <https://github.com/ceph/ceph/pull/14545>`_, Jason Dillaman)
+* rbd: attempting to remove an image with incompatible features results in partial removal (`issue#18456 <http://tracker.ceph.com/issues/18456>`_, `issue#18315 <http://tracker.ceph.com/issues/18315>`_, `pr#13247 <https://github.com/ceph/ceph/pull/13247>`_, Dongsheng Yang)
+* rbd: [cli] ensure positional arguments exist before casting (`issue#20264 <http://tracker.ceph.com/issues/20264>`_, `issue#20185 <http://tracker.ceph.com/issues/20185>`_, `pr#16186 <https://github.com/ceph/ceph/pull/16186>`_, Jason Dillaman)
+* rbd: cli: map with cephx disabled results in error message (`issue#19035 <http://tracker.ceph.com/issues/19035>`_, `issue#20517 <http://tracker.ceph.com/issues/20517>`_, `pr#16298 <https://github.com/ceph/ceph/pull/16298>`_, Jason Dillaman)
+* rbd: [  FAILED  ] TestJournalTrimmer.RemoveObjectsWithOtherClient (`issue#18769 <http://tracker.ceph.com/issues/18769>`_, `issue#18738 <http://tracker.ceph.com/issues/18738>`_, `pr#14147 <https://github.com/ceph/ceph/pull/14147>`_, Jason Dillaman)
+* rbd: Improve compatibility between librbd + krbd for the data pool (`issue#18771 <http://tracker.ceph.com/issues/18771>`_, `issue#18653 <http://tracker.ceph.com/issues/18653>`_, `pr#14539 <https://github.com/ceph/ceph/pull/14539>`_, Jason Dillaman)
+* rbd: Issues with C API image metadata retrieval functions (`issue#19588 <http://tracker.ceph.com/issues/19588>`_, `issue#19611 <http://tracker.ceph.com/issues/19611>`_, `pr#15612 <https://github.com/ceph/ceph/pull/15612>`_, Mykola Golub)
+* rbd: 'metadata_set' API operation should not change global config setting (`issue#18465 <http://tracker.ceph.com/issues/18465>`_, `issue#18549 <http://tracker.ceph.com/issues/18549>`_, `pr#14534 <https://github.com/ceph/ceph/pull/14534>`_, Mykola Golub)
+* rbd-mirror: additional test stability improvements (`issue#18935 <http://tracker.ceph.com/issues/18935>`_, `issue#18947 <http://tracker.ceph.com/issues/18947>`_, `pr#14155 <https://github.com/ceph/ceph/pull/14155>`_, Jason Dillaman)
+* rbd-mirror: deleting a snapshot during sync can result in read errors (`issue#19037 <http://tracker.ceph.com/issues/19037>`_, `issue#18990 <http://tracker.ceph.com/issues/18990>`_, `pr#14622 <https://github.com/ceph/ceph/pull/14622>`_, Jason Dillaman)
+* rbd-mirror: ensure missing images are re-synced when detected (`issue#20022 <http://tracker.ceph.com/issues/20022>`_, `issue#19811 <http://tracker.ceph.com/issues/19811>`_, `pr#15486 <https://github.com/ceph/ceph/pull/15486>`_, Jason Dillaman)
+* rbd-mirror: failover and failback of unmodified image results in split-brain (`issue#19872 <http://tracker.ceph.com/issues/19872>`_, `issue#19858 <http://tracker.ceph.com/issues/19858>`_, `pr#14974 <https://github.com/ceph/ceph/pull/14974>`_, Jason Dillaman)
+* rbd-mirror: potential race mirroring cloned image (`issue#18501 <http://tracker.ceph.com/issues/18501>`_, `issue#17993 <http://tracker.ceph.com/issues/17993>`_, `pr#14533 <https://github.com/ceph/ceph/pull/14533>`_, Jason Dillaman)
+* rbd-mirror: sporadic image replayer shut down failure (`issue#18493 <http://tracker.ceph.com/issues/18493>`_, `issue#18441 <http://tracker.ceph.com/issues/18441>`_, `pr#14531 <https://github.com/ceph/ceph/pull/14531>`_, Jason Dillaman)
+* rbd-nbd: add signal handler (`issue#19621 <http://tracker.ceph.com/issues/19621>`_, `issue#19349 <http://tracker.ceph.com/issues/19349>`_, `pr#16098 <https://github.com/ceph/ceph/pull/16098>`_, Kefu Chai, Pan Liu)
+* rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly (`issue#18970 <http://tracker.ceph.com/issues/18970>`_, `issue#17951 <http://tracker.ceph.com/issues/17951>`_, `issue#18910 <http://tracker.ceph.com/issues/18910>`_, `issue#18335 <http://tracker.ceph.com/issues/18335>`_, `pr#14540 <https://github.com/ceph/ceph/pull/14540>`_, Mykola Golub, Pan Liu)
+* rbd: Possible deadlock performing a synchronous API action while refresh in-progress (`issue#18495 <http://tracker.ceph.com/issues/18495>`_, `issue#18419 <http://tracker.ceph.com/issues/18419>`_, `pr#14532 <https://github.com/ceph/ceph/pull/14532>`_, Jason Dillaman)
+* rbd: Potential IO hang if image is flattened while read request is in-flight (`issue#19832 <http://tracker.ceph.com/issues/19832>`_, `issue#20154 <http://tracker.ceph.com/issues/20154>`_, `pr#16184 <https://github.com/ceph/ceph/pull/16184>`_, Jason Dillaman)
+* rbd: [qa] crash in journal-enabled fsx run (`issue#18618 <http://tracker.ceph.com/issues/18618>`_, `issue#18632 <http://tracker.ceph.com/issues/18632>`_, `pr#14538 <https://github.com/ceph/ceph/pull/14538>`_, Jason Dillaman)
+* rbd: qemu crash triggered by network issues (`issue#18776 <http://tracker.ceph.com/issues/18776>`_, `issue#18436 <http://tracker.ceph.com/issues/18436>`_, `pr#13245 <https://github.com/ceph/ceph/pull/13245>`_, Jason Dillaman)
+* rbd: 'rbd bench-write' will crash if --io-size is 4G (`issue#18422 <http://tracker.ceph.com/issues/18422>`_, `issue#18557 <http://tracker.ceph.com/issues/18557>`_, `pr#14536 <https://github.com/ceph/ceph/pull/14536>`_, Gaurav Kumar Garg)
+* rbd: rbd_clone_copy_on_read ineffective with exclusive-lock (`issue#19173 <http://tracker.ceph.com/issues/19173>`_, `issue#18888 <http://tracker.ceph.com/issues/18888>`_, `pr#14543 <https://github.com/ceph/ceph/pull/14543>`_, Venky Shankar)
+* rbd: rbd --pool=x rename y z does not work (`issue#18777 <http://tracker.ceph.com/issues/18777>`_, `issue#18326 <http://tracker.ceph.com/issues/18326>`_, `pr#14149 <https://github.com/ceph/ceph/pull/14149>`_, Gaurav Kumar Garg)
+* rbd: refuse to use an ec pool that doesn't support overwrites (`issue#19081 <http://tracker.ceph.com/issues/19081>`_, `issue#19336 <http://tracker.ceph.com/issues/19336>`_, `pr#16096 <https://github.com/ceph/ceph/pull/16096>`_, Jason Dillaman)
+* rgw: add apis to support ragweed suite (`issue#19809 <http://tracker.ceph.com/issues/19809>`_, `pr#14852 <https://github.com/ceph/ceph/pull/14852>`_, Yehuda Sadeh)
+* rgw: add the remove-x-delete feature to cancel swift object expiration (`issue#19472 <http://tracker.ceph.com/issues/19472>`_, `issue#19074 <http://tracker.ceph.com/issues/19074>`_, `pr#14522 <https://github.com/ceph/ceph/pull/14522>`_, Jing Wenjun)
+* rgw: a few cases where rgw_obj is incorrectly initialized (`issue#19146 <http://tracker.ceph.com/issues/19146>`_, `issue#19096 <http://tracker.ceph.com/issues/19096>`_, `pr#13843 <https://github.com/ceph/ceph/pull/13843>`_, Yehuda Sadeh)
+* rgw: anonymous user error code of getting object is not consistent with SWIFT (`issue#18806 <http://tracker.ceph.com/issues/18806>`_, `issue#19178 <http://tracker.ceph.com/issues/19178>`_, `pr#13877 <https://github.com/ceph/ceph/pull/13877>`_, Jing Wenjun)
+* rgw: civetweb frontend segfaults in Luminous (`issue#19749 <http://tracker.ceph.com/issues/19749>`_, `issue#19840 <http://tracker.ceph.com/issues/19840>`_, `pr#16166 <https://github.com/ceph/ceph/pull/16166>`_, Abhishek Lekshmanan, Jesse Williamson)
+* rgw: civetweb: move to post 1.8 version (`issue#19704 <http://tracker.ceph.com/issues/19704>`_, `pr#14960 <https://github.com/ceph/ceph/pull/14960>`_, Yehuda Sadeh)
+* rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log (`issue#19212 <http://tracker.ceph.com/issues/19212>`_, `issue#18980 <http://tracker.ceph.com/issues/18980>`_, `pr#14065 <https://github.com/ceph/ceph/pull/14065>`_, Casey Bodley)
+* rgw: crash when updating period with placement group (`issue#18772 <http://tracker.ceph.com/issues/18772>`_, `issue#18631 <http://tracker.ceph.com/issues/18631>`_, `pr#14511 <https://github.com/ceph/ceph/pull/14511>`_, Orit Wasserman)
+* rgw: Custom data header support (`issue#19843 <http://tracker.ceph.com/issues/19843>`_, `pr#15985 <https://github.com/ceph/ceph/pull/15985>`_, Pavan Rallabhandi)
+* rgw: datalog trim can't work as expected (`issue#20263 <http://tracker.ceph.com/issues/20263>`_, `issue#20190 <http://tracker.ceph.com/issues/20190>`_, `pr#16175 <https://github.com/ceph/ceph/pull/16175>`_, Zhang Shaowen)
+* rgw: DUMPABLE flag is cleared by setuid preventing coredumps (`issue#19147 <http://tracker.ceph.com/issues/19147>`_, `issue#19089 <http://tracker.ceph.com/issues/19089>`_, `pr#13845 <https://github.com/ceph/ceph/pull/13845>`_, Brad Hubbard)
+* rgw: Error parsing xml when get bucket lifecycle (`issue#19363 <http://tracker.ceph.com/issues/19363>`_, `issue#19534 <http://tracker.ceph.com/issues/19534>`_, `pr#14528 <https://github.com/ceph/ceph/pull/14528>`_, liuchang0812)
+* rgw: first write also tries to read object (`issue#18904 <http://tracker.ceph.com/issues/18904>`_, `issue#18622 <http://tracker.ceph.com/issues/18622>`_, `issue#18623 <http://tracker.ceph.com/issues/18623>`_, `issue#18621 <http://tracker.ceph.com/issues/18621>`_, `pr#14515 <https://github.com/ceph/ceph/pull/14515>`_, Yehuda Sadeh)
+* rgw: fix break inside of yield in RGWFetchAllMetaCR (`issue#19322 <http://tracker.ceph.com/issues/19322>`_, `issue#17655 <http://tracker.ceph.com/issues/17655>`_, `pr#14067 <https://github.com/ceph/ceph/pull/14067>`_, Casey Bodley)
+* rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT (`issue#18476 <http://tracker.ceph.com/issues/18476>`_, `pr#13006 <https://github.com/ceph/ceph/pull/13006>`_, Radoslaw Zarzynski)
+* rgw: fix RadosGW hang during multi-chunk upload of AWSv4 (`issue#19837 <http://tracker.ceph.com/issues/19837>`_, `issue#19754 <http://tracker.ceph.com/issues/19754>`_, `pr#14939 <https://github.com/ceph/ceph/pull/14939>`_, Radoslaw Zarzynski)
+* rgw: fix use of marker in List::list_objects() (`issue#19047 <http://tracker.ceph.com/issues/19047>`_, `issue#18331 <http://tracker.ceph.com/issues/18331>`_, `pr#14517 <https://github.com/ceph/ceph/pull/14517>`_, Yehuda Sadeh)
+* rgw: 'gc list --include-all' command infinite loop the first 1000 items (`issue#19978 <http://tracker.ceph.com/issues/19978>`_, `issue#20147 <http://tracker.ceph.com/issues/20147>`_, `pr#16139 <https://github.com/ceph/ceph/pull/16139>`_, Shasha Lu, fang yuxiang)
+* rgw: get wrong content when download object with specific range when compression was enabled (`issue#20100 <http://tracker.ceph.com/issues/20100>`_, `issue#20268 <http://tracker.ceph.com/issues/20268>`_, `pr#16178 <https://github.com/ceph/ceph/pull/16178>`_, fang yuxiang)
+* rgw: health check errors out incorrectly (`issue#19025 <http://tracker.ceph.com/issues/19025>`_, `issue#19157 <http://tracker.ceph.com/issues/19157>`_, `pr#13866 <https://github.com/ceph/ceph/pull/13866>`_, Pavan Rallabhandi)
+* rgw: Lifecycle thread will still handle the bucket even if it has been removed (`issue#20285 <http://tracker.ceph.com/issues/20285>`_, `issue#20405 <http://tracker.ceph.com/issues/20405>`_, `pr#16183 <https://github.com/ceph/ceph/pull/16183>`_, Zhang Shaowen)
+* rgw: make sending Content-Length in 204 and 304 controllable (`issue#18985 <http://tracker.ceph.com/issues/18985>`_, `issue#16602 <http://tracker.ceph.com/issues/16602>`_, `pr#13514 <https://github.com/ceph/ceph/pull/13514>`_, Radoslaw Zarzynski)
+* rgw: meta sync thread crash at RGWMetaSyncShardCR (`issue#20251 <http://tracker.ceph.com/issues/20251>`_, `issue#20347 <http://tracker.ceph.com/issues/20347>`_, `pr#16180 <https://github.com/ceph/ceph/pull/16180>`_, Fang Yuxiang, Nathan Cutler)
+* rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards (`issue#19745 <http://tracker.ceph.com/issues/19745>`_, `issue#19759 <http://tracker.ceph.com/issues/19759>`_, `pr#16290 <https://github.com/ceph/ceph/pull/16290>`_, Shasha Lu)
+* rgw: multisite: EPERM when trying to read SLO objects as system/admin user (`issue#19027 <http://tracker.ceph.com/issues/19027>`_, `issue#19475 <http://tracker.ceph.com/issues/19475>`_, `pr#14523 <https://github.com/ceph/ceph/pull/14523>`_, Casey Bodley)
+* rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote (`issue#19608 <http://tracker.ceph.com/issues/19608>`_, `pr#14606 <https://github.com/ceph/ceph/pull/14606>`_, Zhang Shaowen, Casey Bodley)
+* rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO (`issue#19160 <http://tracker.ceph.com/issues/19160>`_, `issue#19019 <http://tracker.ceph.com/issues/19019>`_, `pr#13868 <https://github.com/ceph/ceph/pull/13868>`_, Casey Bodley)
+* rgw: multisite: segfault after changing value of rgw_data_log_num_shards (`issue#18488 <http://tracker.ceph.com/issues/18488>`_, `issue#18548 <http://tracker.ceph.com/issues/18548>`_, `pr#13181 <https://github.com/ceph/ceph/pull/13181>`_, Casey Bodley)
+* rgw: multisite: some 'radosgw-admin data sync' commands hang (`issue#19236 <http://tracker.ceph.com/issues/19236>`_, `issue#19354 <http://tracker.ceph.com/issues/19354>`_, `pr#14142 <https://github.com/ceph/ceph/pull/14142>`_, Shasha Lu)
+* rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync() (`issue#19049 <http://tracker.ceph.com/issues/19049>`_, `issue#18076 <http://tracker.ceph.com/issues/18076>`_, `pr#13838 <https://github.com/ceph/ceph/pull/13838>`_, Casey Bodley)
+* rgw: multisite: sync status reports master is on a different period (`issue#18709 <http://tracker.ceph.com/issues/18709>`_, `issue#18064 <http://tracker.ceph.com/issues/18064>`_, `pr#13176 <https://github.com/ceph/ceph/pull/13176>`_, Abhishek Lekshmanan)
+* rgw: no http referer info in container metadata dump in swift API (`issue#18665 <http://tracker.ceph.com/issues/18665>`_, `issue#18898 <http://tracker.ceph.com/issues/18898>`_, `pr#13829 <https://github.com/ceph/ceph/pull/13829>`_, Jing Wenjun)
+* rgw: "period update" does not remove short_zone_ids of deleted zones (`issue#15618 <http://tracker.ceph.com/issues/15618>`_, `issue#19342 <http://tracker.ceph.com/issues/19342>`_, `pr#14141 <https://github.com/ceph/ceph/pull/14141>`_, Casey Bodley)
+* rgw: radosgw-admin: add the 'object stat' command to usage (`issue#19164 <http://tracker.ceph.com/issues/19164>`_, `issue#19013 <http://tracker.ceph.com/issues/19013>`_, `pr#13873 <https://github.com/ceph/ceph/pull/13873>`_, Pavan Rallabhandi)
+* rgw: radosgw-admin period update reverts deleted zonegroup (`issue#18713 <http://tracker.ceph.com/issues/18713>`_, `issue#17239 <http://tracker.ceph.com/issues/17239>`_, `pr#13172 <https://github.com/ceph/ceph/pull/13172>`_, Orit Wasserman)
+* rgw: 'radosgw-admin usage show' listing 0 bytes_sent/received (`issue#20261 <http://tracker.ceph.com/issues/20261>`_, `pr#16174 <https://github.com/ceph/ceph/pull/16174>`_, Pritha Srivastava)
+* rgw: 'radosgw-admin zone create' command with specified zone-id creates a zone with different id (`issue#19524 <http://tracker.ceph.com/issues/19524>`_, `issue#19498 <http://tracker.ceph.com/issues/19498>`_, `pr#14526 <https://github.com/ceph/ceph/pull/14526>`_, Orit Wasserman)
+* rgw: Realm set does not create a new period (`issue#18333 <http://tracker.ceph.com/issues/18333>`_, `issue#18499 <http://tracker.ceph.com/issues/18499>`_, `pr#14509 <https://github.com/ceph/ceph/pull/14509>`_, Orit Wasserman)
+* rgw: reduce log level of 'storing entry at' in cls_log (`issue#19835 <http://tracker.ceph.com/issues/19835>`_, `issue#19839 <http://tracker.ceph.com/issues/19839>`_, `pr#16165 <https://github.com/ceph/ceph/pull/16165>`_, Willem Jan Withagen)
+* rgw: Response header of swift API returned by radosgw does not contain x-openstack-request-id. But Swift returns it (`issue#19443 <http://tracker.ceph.com/issues/19443>`_, `issue#19573 <http://tracker.ceph.com/issues/19573>`_, `pr#14529 <https://github.com/ceph/ceph/pull/14529>`_, tone-zhang)
+* rgw: rgw_file: fix marker computation (`issue#20158 <http://tracker.ceph.com/issues/20158>`_, `issue#19526 <http://tracker.ceph.com/issues/19526>`_, `issue#18989 <http://tracker.ceph.com/issues/18989>`_, `issue#19470 <http://tracker.ceph.com/issues/19470>`_, `issue#19471 <http://tracker.ceph.com/issues/19471>`_, `issue#18651 <http://tracker.ceph.com/issues/18651>`_, `issue#20195 <http://tracker.ceph.com/issues/20195>`_, `issue#19059 <http://tracker.ceph.com/issues/19059>`_, `issue#19112 <http://tracker.ceph.com/issues/19112>`_, `issue#19018 <http://tracker.ceph.com/issues/19018>`_, `issue#19036 <http://tracker.ceph.com/issues/19036>`_, `issue#19154 <http://tracker.ceph.com/issues/19154>`_, `issue#19170 <http://tracker.ceph.com/issues/19170>`_, `issue#19663 <http://tracker.ceph.com/issues/19663>`_, `issue#19661 <http://tracker.ceph.com/issues/19661>`_, `issue#19111 <http://tracker.ceph.com/issues/19111>`_, `issue#18992 <http://tracker.ceph.com/issues/18992>`_, `issue#18650 <http://tracker.ceph.com/issues/18650>`_, `issue#18991 <http://tracker.ceph.com/issues/18991>`_, `issue#19623 <http://tracker.ceph.com/issues/19623>`_, `issue#19149 <http://tracker.ceph.com/issues/19149>`_, `issue#19270 <http://tracker.ceph.com/issues/19270>`_, `issue#19723 <http://tracker.ceph.com/issues/19723>`_, `issue#19625 <http://tracker.ceph.com/issues/19625>`_, `issue#19624 <http://tracker.ceph.com/issues/19624>`_, `issue#19060 <http://tracker.ceph.com/issues/19060>`_, `issue#19166 <http://tracker.ceph.com/issues/19166>`_, `issue#18810 <http://tracker.ceph.com/issues/18810>`_, `issue#19168 <http://tracker.ceph.com/issues/19168>`_, `issue#19162 <http://tracker.ceph.com/issues/19162>`_, `issue#19066 <http://tracker.ceph.com/issues/19066>`_, `issue#18808 <http://tracker.ceph.com/issues/18808>`_, `issue#19634 <http://tracker.ceph.com/issues/19634>`_, `issue#19435 <http://tracker.ceph.com/issues/19435>`_, `issue#19144 <http://tracker.ceph.com/issues/19144>`_, `issue#19229 <http://tracker.ceph.com/issues/19229>`_, `issue#18902 <http://tracker.ceph.com/issues/18902>`_, `pr#13871 <https://github.com/ceph/ceph/pull/13871>`_, Gui Hecheng, Matt Benjamin)
+* rgw: S3 create bucket should not do response in json (`issue#19172 <http://tracker.ceph.com/issues/19172>`_, `issue#18889 <http://tracker.ceph.com/issues/18889>`_, `pr#13875 <https://github.com/ceph/ceph/pull/13875>`_, Abhishek Lekshmanan)
+* rgw: S3 v4 authentication issue with X-Amz-Expires (`issue#19477 <http://tracker.ceph.com/issues/19477>`_, `issue#18828 <http://tracker.ceph.com/issues/18828>`_, `pr#14524 <https://github.com/ceph/ceph/pull/14524>`_, liuchang0812)
+* rgw: S3 v4 authentication issue with X-Amz-Expires (`issue#19725 <http://tracker.ceph.com/issues/19725>`_, `issue#18828 <http://tracker.ceph.com/issues/18828>`_, `pr#16162 <https://github.com/ceph/ceph/pull/16162>`_, liuchang0812)
+* rgw: should parse the url to http host to compare with the container referer acl (`issue#18896 <http://tracker.ceph.com/issues/18896>`_, `issue#18685 <http://tracker.ceph.com/issues/18685>`_, `pr#13780 <https://github.com/ceph/ceph/pull/13780>`_, Jing Wenjun)
+* rgw: slave zonegroup cannot enable the bucket versioning (`issue#18711 <http://tracker.ceph.com/issues/18711>`_, `issue#18003 <http://tracker.ceph.com/issues/18003>`_, `pr#13174 <https://github.com/ceph/ceph/pull/13174>`_, Orit Wasserman)
+* rgw: Swift API: spurious newline after http body causes weird errors (`issue#18780 <http://tracker.ceph.com/issues/18780>`_, `issue#18473 <http://tracker.ceph.com/issues/18473>`_, `pr#13224 <https://github.com/ceph/ceph/pull/13224>`_, Marcus Watts, Matt Benjamin)
+* rgw: swift API: cannot disable object versioning with empty X-Versions-Location (`issue#18852 <http://tracker.ceph.com/issues/18852>`_, `issue#19175 <http://tracker.ceph.com/issues/19175>`_, `pr#14519 <https://github.com/ceph/ceph/pull/14519>`_, Jing Wenjun)
+* rgw: swift: disable revocation thread under certain circumstances (`issue#19499 <http://tracker.ceph.com/issues/19499>`_, `issue#9493 <http://tracker.ceph.com/issues/9493>`_, `issue#19777 <http://tracker.ceph.com/issues/19777>`_, `pr#16164 <https://github.com/ceph/ceph/pull/16164>`_, Marcus Watts)
+* rgw: Swift's at-root features (/crossdomain.xml, /info, /healthcheck) are broken (`issue#20031 <http://tracker.ceph.com/issues/20031>`_, `issue#19520 <http://tracker.ceph.com/issues/19520>`_, `pr#16168 <https://github.com/ceph/ceph/pull/16168>`_, Radoslaw Zarzynski)
+* rgw: the swift container acl does not support field .ref (`issue#18909 <http://tracker.ceph.com/issues/18909>`_, `issue#19180 <http://tracker.ceph.com/issues/19180>`_, `issue#18484 <http://tracker.ceph.com/issues/18484>`_, `issue#18796 <http://tracker.ceph.com/issues/18796>`_, `pr#14516 <https://github.com/ceph/ceph/pull/14516>`_, Jing Wenjun, Radoslaw Zarzynski)
+* rgw: typo in rgw_admin.cc (`issue#19156 <http://tracker.ceph.com/issues/19156>`_, `issue#19026 <http://tracker.ceph.com/issues/19026>`_, `pr#13864 <https://github.com/ceph/ceph/pull/13864>`_, Ronak Jain)
+* rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response() (`issue#19574 <http://tracker.ceph.com/issues/19574>`_, `issue#19249 <http://tracker.ceph.com/issues/19249>`_, `pr#14530 <https://github.com/ceph/ceph/pull/14530>`_, Yehuda Sadeh)
+* rgw: upgrade to multisite v2 fails if there is a zone without zone info (`issue#19331 <http://tracker.ceph.com/issues/19331>`_, `issue#19231 <http://tracker.ceph.com/issues/19231>`_, `pr#14137 <https://github.com/ceph/ceph/pull/14137>`_, Danny Al-Gaaf, Orit Wasserman)
+* rgw: usage stats and quota are not operational for multi-tenant users (`issue#18364 <http://tracker.ceph.com/issues/18364>`_, `issue#18843 <http://tracker.ceph.com/issues/18843>`_, `issue#16355 <http://tracker.ceph.com/issues/16355>`_, `pr#14513 <https://github.com/ceph/ceph/pull/14513>`_, Radoslaw Zarzynski)
+* rgw: Use decoded URI when verifying TempURL (`issue#18590 <http://tracker.ceph.com/issues/18590>`_, `issue#18627 <http://tracker.ceph.com/issues/18627>`_, `pr#12986 <https://github.com/ceph/ceph/pull/12986>`_, Michal Koutný)
+* rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions (`issue#20363 <http://tracker.ceph.com/issues/20363>`_, `issue#19886 <http://tracker.ceph.com/issues/19886>`_, `pr#16181 <https://github.com/ceph/ceph/pull/16181>`_, Zhang Shaowen)
+* rgw: when converting region_map we need to use rgw_zone_root_pool (`issue#19195 <http://tracker.ceph.com/issues/19195>`_, `issue#19356 <http://tracker.ceph.com/issues/19356>`_, `pr#14144 <https://github.com/ceph/ceph/pull/14144>`_, Orit Wasserman)
+* rgw: when uploading the objects continuesly in the versioned bucket, some objects will not sync (`issue#19766 <http://tracker.ceph.com/issues/19766>`_, `issue#18208 <http://tracker.ceph.com/issues/18208>`_, `pr#16163 <https://github.com/ceph/ceph/pull/16163>`_, lvshuhua)
+* rgw: wrong object size after copy of uncompressed multipart objects (`issue#20269 <http://tracker.ceph.com/issues/20269>`_, `issue#20071 <http://tracker.ceph.com/issues/20071>`_, `pr#16179 <https://github.com/ceph/ceph/pull/16179>`_, fang yuxiang)
+* rgw: zonegroupmap set does not work (`issue#18725 <http://tracker.ceph.com/issues/18725>`_, `issue#19479 <http://tracker.ceph.com/issues/19479>`_, `pr#14525 <https://github.com/ceph/ceph/pull/14525>`_, Casey Bodley)
+* tests: AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool' (`issue#19064 <http://tracker.ceph.com/issues/19064>`_, `issue#18799 <http://tracker.ceph.com/issues/18799>`_, `pr#13609 <https://github.com/ceph/ceph/pull/13609>`_, Nathan Cutler)
+* tests: backport Sage's fixes to qa/suites/upgrade/jewel-x (`issue#19651 <http://tracker.ceph.com/issues/19651>`_, `pr#14612 <https://github.com/ceph/ceph/pull/14612>`_, Sage Weil)
+* tests: ceph-object-corpus: kraken objects (`issue#20878 <http://tracker.ceph.com/issues/20878>`_, `pr#14983 <https://github.com/ceph/ceph/pull/14983>`_, Sage Weil)
+* tests: CMakeLists.txt: disable memstore make check test (`issue#17743 <http://tracker.ceph.com/issues/17743>`_, `pr#16215 <https://github.com/ceph/ceph/pull/16215>`_, Sage Weil)
+* tests: HEALTH_WARN pool rbd pg_num 244 > pgp_num 224 during upgrade (`issue#19771 <http://tracker.ceph.com/issues/19771>`_, `issue#20024 <http://tracker.ceph.com/issues/20024>`_, `pr#16137 <https://github.com/ceph/ceph/pull/16137>`_, Kefu Chai)
+* tests: ignore bogus ceph-objectstore-tool error in ceph_manager (`issue#18805 <http://tracker.ceph.com/issues/18805>`_, `issue#16263 <http://tracker.ceph.com/issues/16263>`_, `pr#13239 <https://github.com/ceph/ceph/pull/13239>`_, Nathan Cutler, Kefu Chai)
+* tests: insufficient timeout in radosbench task (`issue#20497 <http://tracker.ceph.com/issues/20497>`_, `pr#16111 <https://github.com/ceph/ceph/pull/16111>`_, Sage Weil)
+* tests: LibRadosMiscConnectFailure.ConnectFailure hang (`issue#20271 <http://tracker.ceph.com/issues/20271>`_, `issue#19901 <http://tracker.ceph.com/issues/19901>`_, `pr#16140 <https://github.com/ceph/ceph/pull/16140>`_, Sage Weil)
+* tests: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value (`issue#19597 <http://tracker.ceph.com/issues/19597>`_, `issue#19609 <http://tracker.ceph.com/issues/19609>`_, `pr#16097 <https://github.com/ceph/ceph/pull/16097>`_, Jason Dillaman)
+* tests: move swift.py task from teuthology to ceph, phase one (kraken) (`issue#20392 <http://tracker.ceph.com/issues/20392>`_, `pr#15869 <https://github.com/ceph/ceph/pull/15869>`_, Nathan Cutler, Sage Weil, Warren Usui, Greg Farnum, Ali Maredia, Tommi Virtanen, Zack Cerza, Sam Lang, Yehuda Sadeh, Joe Buck, Josh Durgin)
+* tests: ObjectStore/StoreTest.OnodeSizeTracking/2 fails on bluestore (`issue#20499 <http://tracker.ceph.com/issues/20499>`_, `pr#16112 <https://github.com/ceph/ceph/pull/16112>`_, xie xingguo)
+* tests: qa: ceph-ansible test tweaks (`issue#20882 <http://tracker.ceph.com/issues/20882>`_, `pr#12984 <https://github.com/ceph/ceph/pull/12984>`_, `pr#13618 <https://github.com/ceph/ceph/pull/13618>`_, Tamil Muthamizhan, Yuri Weinstein)
+* tests: qa/suites/upgrade: add tiering test to hammer-jewel-x (`issue#20879 <http://tracker.ceph.com/issues/20879>`_, `issue#19185 <http://tracker.ceph.com/issues/19185>`_, `pr#14692 <https://github.com/ceph/ceph/pull/14692>`_, Kefu Chai)
+* tests: qa/tasks: misc systemd updates (`issue#19719 <http://tracker.ceph.com/issues/19719>`_, `pr#14702 <https://github.com/ceph/ceph/pull/14702>`_, Vasu Kulkarni)
+* tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode (`issue#20638 <http://tracker.ceph.com/issues/20638>`_, `issue#20630 <http://tracker.ceph.com/issues/20630>`_, `issue#20634 <http://tracker.ceph.com/issues/20634>`_, `pr#16342 <https://github.com/ceph/ceph/pull/16342>`_, Jason Dillaman)
+* tests: qa/tasks: set pgp = pg num on thrashing finish (`issue#20881 <http://tracker.ceph.com/issues/20881>`_, `pr#13757 <https://github.com/ceph/ceph/pull/13757>`_, Kefu Chai)
+* tests: qa/tasks/workunit: Backport repo fixes from master (`issue#19429 <http://tracker.ceph.com/issues/19429>`_, `issue#19531 <http://tracker.ceph.com/issues/19531>`_, `pr#14487 <https://github.com/ceph/ceph/pull/14487>`_, Kefu Chai, Dan Mick)
+* tests: remove hard-coded image name from TestLibRBD.Mirror (`issue#18555 <http://tracker.ceph.com/issues/18555>`_, `issue#19130 <http://tracker.ceph.com/issues/19130>`_, `issue#19227 <http://tracker.ceph.com/issues/19227>`_, `issue#18447 <http://tracker.ceph.com/issues/18447>`_, `issue#19807 <http://tracker.ceph.com/issues/19807>`_, `issue#19798 <http://tracker.ceph.com/issues/19798>`_, `pr#16113 <https://github.com/ceph/ceph/pull/16113>`_, Mykola Golub, Jason Dillaman)
+* tests: remove qa/suites/buildpackages (`issue#18849 <http://tracker.ceph.com/issues/18849>`_, `issue#18846 <http://tracker.ceph.com/issues/18846>`_, `pr#13298 <https://github.com/ceph/ceph/pull/13298>`_, Loic Dachary)
+* tests: run certain upgrade/jewel-x tests on Xenial only (`issue#20877 <http://tracker.ceph.com/issues/20877>`_, `pr#16493 <https://github.com/ceph/ceph/pull/16493>`_, Nathan Cutler)
+* tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap (`issue#18822 <http://tracker.ceph.com/issues/18822>`_, `issue#17447 <http://tracker.ceph.com/issues/17447>`_, `pr#14151 <https://github.com/ceph/ceph/pull/14151>`_, Jason Dillaman)
+* tests: SUSE yaml facets in qa/distros/all are out of date (`issue#18849 <http://tracker.ceph.com/issues/18849>`_, `issue#18870 <http://tracker.ceph.com/issues/18870>`_, `issue#18846 <http://tracker.ceph.com/issues/18846>`_, `issue#18856 <http://tracker.ceph.com/issues/18856>`_, `pr#13330 <https://github.com/ceph/ceph/pull/13330>`_, Nathan Cutler)
+* tests: swift.py: clone the ceph-kraken branch (`issue#20520 <http://tracker.ceph.com/issues/20520>`_, `pr#16131 <https://github.com/ceph/ceph/pull/16131>`_, Nathan Cutler)
+* tests: test/librbd: decouple ceph_test_librbd_api from libceph-common (`issue#20175 <http://tracker.ceph.com/issues/20175>`_, `issue#20351 <http://tracker.ceph.com/issues/20351>`_, `pr#16195 <https://github.com/ceph/ceph/pull/16195>`_, Kefu Chai)
+* tests: test_notify.py: assert(not image.is_exclusive_lock_owner()) on line 147 (`issue#19716 <http://tracker.ceph.com/issues/19716>`_, `issue#19794 <http://tracker.ceph.com/issues/19794>`_, `pr#14833 <https://github.com/ceph/ceph/pull/14833>`_, Mykola Golub)
+* tests: test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2 (`issue#19692 <http://tracker.ceph.com/issues/19692>`_, `issue#19693 <http://tracker.ceph.com/issues/19693>`_, `pr#14641 <https://github.com/ceph/ceph/pull/14641>`_, Jason Dillaman)
+* tests: use ceph-kraken branch for s3tests (`issue#18387 <http://tracker.ceph.com/issues/18387>`_, `pr#12746 <https://github.com/ceph/ceph/pull/12746>`_, Nathan Cutler)
+* tests: use librados API to retrieve config params (`issue#18668 <http://tracker.ceph.com/issues/18668>`_, `issue#18617 <http://tracker.ceph.com/issues/18617>`_, `pr#13102 <https://github.com/ceph/ceph/pull/13102>`_, Jason Dillaman)
+* tests: various OpenStack tweaks (`issue#20882 <http://tracker.ceph.com/issues/20882>`_, `pr#13707 <https://github.com/ceph/ceph/pull/13707>`_, `pr#13641 <https://github.com/ceph/ceph/pull/13641>`_, `pr#13635 <https://github.com/ceph/ceph/pull/13635>`_, `pr#13633 <https://github.com/ceph/ceph/pull/13633>`_, `pr#13613 <https://github.com/ceph/ceph/pull/13613>`_, `pr#13283 <https://github.com/ceph/ceph/pull/13283>`_, `pr#13673 <https://github.com/ceph/ceph/pull/13673>`_, `pr#13638 <https://github.com/ceph/ceph/pull/13638>`_, `pr#14485 <https://github.com/ceph/ceph/pull/14485>`_, Zack Cerza)
+* tools: ceph-brag fails to count "in" mds (`issue#19333 <http://tracker.ceph.com/issues/19333>`_, `issue#19192 <http://tracker.ceph.com/issues/19192>`_, `pr#14098 <https://github.com/ceph/ceph/pull/14098>`_, Peng Zhang)
+* tools: ceph-disk prepare writes osd log 0 with root owner (`issue#18538 <http://tracker.ceph.com/issues/18538>`_, `issue#18606 <http://tracker.ceph.com/issues/18606>`_, `pr#13026 <https://github.com/ceph/ceph/pull/13026>`_, Samuel Matzek)
+* tools: RadosImport::import should return an error if Rados::connect fails (`issue#19351 <http://tracker.ceph.com/issues/19351>`_, `issue#19319 <http://tracker.ceph.com/issues/19319>`_, `pr#14095 <https://github.com/ceph/ceph/pull/14095>`_, Brad Hubbard)
+
+v11.2.0 Kraken
+==============
+
+This is the first release of the Kraken series.  It is a stable
+release that will be maintained with bugfixes and backports until the
+next stable release, Luminous, is completed in the Spring of 2017.
+
+Major Changes from Jewel
+------------------------
+
+- *RADOS*:
+
+  * The new *BlueStore* backend now has a stable disk format and is
+    passing our failure and stress testing. Although the backend is
+    still flagged as experimental, we encourage users to try it out
+    for non-production clusters and non-critical data sets.
+  * RADOS now has experimental support for *overwrites on
+    erasure-coded* pools. Because the disk format and implementation
+    are not yet finalized, there is a special pool option that must be
+    enabled to test the new feature.  Enabling this option on a cluster
+    will permanently bar that cluster from being upgraded to future
+    versions.
+  * We now default to the AsyncMessenger (``ms type = async``) instead
+    of the legacy SimpleMessenger.  The most noticeable difference is
+    that we now use a fixed sized thread pool for network connections
+    (instead of two threads per socket with SimpleMessenger).
+  * Some OSD failures are now detected almost immediately, whereas
+    previously the heartbeat timeout (which defaults to 20 seconds)
+    had to expire.  This prevents IO from blocking for an extended
+    period for failures where the host remains up but the ceph-osd
+    process is no longer running.
+  * There is a new ``ceph-mgr`` daemon.  It is currently collocated with
+    the monitors by default, and is not yet used for much, but the basic
+    infrastructure is now in place.
+  * The size of encoded OSDMaps has been reduced.
+  * The OSDs now quiesce scrubbing when recovery or rebalancing is in progress.
+
+- *RGW*:
+
+  * RGW now supports a new zone type that can be used for metadata indexing
+    via ElasticSearch.
+  * RGW now supports the S3 multipart object copy-part API.
+  * It is possible now to reshard an existing bucket. Note that bucket
+    resharding currently requires that all IO (especially writes) to
+    the specific bucket is quiesced.
+  * RGW now supports data compression for objects.
+  * Civetweb version has been upgraded to 1.8
+  * The Swift static website API is now supported (S3 support has been added
+    previously).
+  * S3 bucket lifecycle API has been added. Note that currently it only supports
+    object expiration.
+  * Support for custom search filters has been added to the LDAP auth
+    implementation.
+  * Support for NFS version 3 has been added to the RGW NFS gateway.
+  * A Python binding has been created for librgw.
+
+- *RBD*:
+
+  * RBD now supports images stored in an *erasure-coded* RADOS pool
+    using the new (experimental) overwrite support. Images must be
+    created using the new rbd CLI "--data-pool <ec pool>" option to
+    specify the EC pool where the backing data objects are
+    stored. Attempting to create an image directly on an EC pool will
+    not be successful since the image's backing metadata is only
+    supported on a replicated pool.
+  * The rbd-mirror daemon now supports replicating dynamic image
+    feature updates and image metadata key/value pairs from the
+    primary image to the non-primary image.
+  * The number of image snapshots can be optionally restricted to a
+    configurable maximum.
+  * The rbd Python API now supports asynchronous IO operations.
+
+- *CephFS*:
+
+  * libcephfs function definitions have been changed to enable proper
+    uid/gid control.  The library version has been increased to reflect the
+    interface change.
+  * Standby replay MDS daemons now consume less memory on workloads
+    doing deletions.
+  * Scrub now repairs backtrace, and populates `damage ls` with
+    discovered errors.
+  * A new `pg_files` subcommand to `cephfs-data-scan` can identify
+    files affected by a damaged or lost RADOS PG.
+  * The false-positive "failing to respond to cache pressure" warnings have
+    been fixed.
+
+
+Upgrading from Kraken release candidate 11.1.0
+----------------------------------------------
+
+* The new *BlueStore* backend had an on-disk format change after 11.1.0.
+  Any BlueStore OSDs created with 11.1.0 will need to be destroyed and
+  recreated.
+
+Upgrading from Jewel
+--------------------
+
+* All clusters must first be upgraded to Jewel 10.2.z before upgrading
+  to Kraken 11.2.z (or, eventually, Luminous 12.2.z).
+
+* The ``sortbitwise`` flag must be set on the Jewel cluster before upgrading
+  to Kraken.  The latest Jewel (10.2.8+) releases issue a health warning if
+  the flag is not set, so this is probably already set.  If it is not, Kraken
+  OSDs will refuse to start and will print and error message in their log.
+
+* You may upgrade OSDs, Monitors, and MDSs in any order.  RGW daemons
+  should be upgraded last.
+
+* When upgrading, new ceph-mgr daemon instances will be created automatically
+  alongside any monitors.  This will be true for Jewel to Kraken and Jewel to
+  Luminous upgrades, but likely not be true for future upgrades beyond
+  Luminous.  You are, of course, free to create new ceph-mgr daemon instances
+  and destroy the auto-created ones if you do not with them to be colocated
+  with the ceph-mon daemons.
+
+
+BlueStore
+---------
+
+BlueStore is a new backend for managing data stored by each OSD on the directly
+hard disk or SSD.  Unlike the existing FileStore implementation, which makes
+use of an XFS file system to store objects as files, BlueStore manages the
+underlying block device directly.  Implements its own file system-like on-disk
+structure the is designed specifically for Ceph OSD workloads.  Key features
+of BlueStore include:
+
+ * Checksums on all data written to disk, with checksum verifications on all
+   reads, enabled by default.
+ * Inline compression support, which can be enabled on a per-pool or per-object
+   basis via pool properties or client hints, respectively.
+ * Efficient journaling.  Unlike FileStore, which writes *all* data to
+   its journal device, BlueStore only journals metadata and (in some
+   cases) small writes, reducing the size and throughput requirements
+   for its journal.  As with FileStore, the journal can be colocated
+   on the same device as other data or allocated on a smaller,
+   high-performance device (e.g., an SSD or NVMe device).  BlueStore
+   journals are only 512 MB by default.
+
+The BlueStore on-disk format is expected to continue to evolve.  However, we
+will provide support in the OSD to migrate to the new format on upgrade.
+
+.. note: BlueStore is still marked "experimental" in Kraken.  We
+   recommend its use for proof-of-concept and test environments, or
+   other cases where data loss can be tolerated.  Although it is
+   stable in our testing environment, the code is new and bugs are
+   inevitable.  We hope that with user feedback from Kraken
+   deployments we will have sufficient confidence to mark it stable
+   (and the default) in the next major release (Luminous).
+
+In order to enable BlueStore, add the following to ceph.conf::
+
+  enable experimental unrecoverable data corrupting features = bluestore
+
+To create a BlueStore OSD, pass the --bluestore option to ceph-disk or
+ceph-deploy during OSD creation.
+
+
+
+Upgrade notes
+-------------
+
+* The OSDs now avoid starting new scrubs while recovery is in progress.  To
+  revert to the old behavior (and do not let recovery activity affect the
+  scrub scheduling) you can set the following option::
+
+    osd scrub during recovery = true
+
+* The list of monitor hosts/addresses for building the monmap can now be
+  obtained from DNS SRV records. The service name used when querying the DNS
+  is defined in the "mon_dns_srv_name" config option, which defaults to
+  "ceph-mon".
+
+* The 'osd class load list' config option is a list of object class names that
+  the OSD is permitted to load (or '*' for all classes). By default it
+  contains all existing in-tree classes for backwards compatibility.
+
+* The 'osd class default list' config option is a list of object class
+  names (or '*' for all classes) that clients may invoke having only
+  the '*', 'x', 'class-read', or 'class-write' capabilities. By
+  default it contains all existing in-tree classes for backwards
+  compatibility. Invoking classes not listed in 'osd class default
+  list' requires a capability naming the class (e.g. 'allow class
+  foo').
+
+* The 'rgw rest getusage op compat' config option allows you to dump
+  (or not dump) the description of user stats in the S3 GetUsage
+  API. This option defaults to false.  If the value is true, the
+  response data for GetUsage looks like::
+
+    "stats": {
+                "TotalBytes": 516,
+                "TotalBytesRounded": 1024,
+                "TotalEntries": 1
+             }
+
+  If the value is false, the response for GetUsage looks as it did before::
+
+    {
+         516,
+         1024,
+         1
+    }
+
+* The 'osd out ...' and 'osd in ...' commands now preserve the OSD
+  weight.  That is, after marking an OSD out and then in, the weight
+  will be the same as before (instead of being reset to 1.0).
+  Previously the mons would only preserve the weight if the mon
+  automatically marked and OSD out and then in, but not when an admin
+  did so explicitly.
+
+* The 'ceph osd perf' command will display 'commit_latency(ms)' and
+  'apply_latency(ms)'. Previously, the names of these two columns are
+  'fs_commit_latency(ms)' and 'fs_apply_latency(ms)'. We remove the
+  prefix 'fs\_', because they are not filestore specific.
+
+* Monitors will no longer allow pools to be removed by default.  The
+  setting mon_allow_pool_delete has to be set to true (defaults to
+  false) before they allow pools to be removed.  This is a additional
+  safeguard against pools being removed by accident.
+
+* If you have manually specified the monitor user rocksdb via the
+  ``mon keyvaluedb = rocksdb`` option, you will need to manually add a
+  file to the mon data directory to preserve this option::
+
+     echo rocksdb > /var/lib/ceph/mon/ceph-`hostname`/kv_backend
+
+  New monitors will now use rocksdb by default, but if that file is
+  not present, existing monitors will use leveldb.  The ``mon
+  keyvaluedb`` option now only affects the backend chosen when a
+  monitor is created.
+
+* The 'osd crush initial weight' option allows you to specify a CRUSH
+  weight for a newly added OSD.  Previously a value of 0 (the default)
+  meant that we should use the size of the OSD's store to weight the
+  new OSD.  Now, a value of 0 means it should have a weight of 0, and
+  a negative value (the new default) means we should automatically
+  weight the OSD based on its size.  If your configuration file
+  explicitly specifies a value of 0 for this option you will need to
+  change it to a negative value (e.g., -1) to preserve the current
+  behavior.
+
+* The ``osd crush location`` config option is no longer supported.  Please
+  update your ceph.conf to use the ``crush location`` option instead.
+
+* The static libraries are no longer included by the debian
+  development packages (lib*-dev) as it is not required per debian
+  packaging policy.  The shared (.so) versions are packaged as before.
+
+* The libtool pseudo-libraries (.la files) are no longer included by
+  the debian development packages (lib*-dev) as they are not required
+  per https://wiki.debian.org/ReleaseGoals/LAFileRemoval and
+  https://www.debian.org/doc/manuals/maint-guide/advanced.en.html.
+
+* The jerasure and shec plugins can now detect SIMD instruction at
+  runtime and no longer need to be explicitly configured for different
+  processors.  The following plugins are now deprecated:
+  jerasure_generic, jerasure_sse3, jerasure_sse4, jerasure_neon,
+  shec_generic, shec_sse3, shec_sse4, and shec_neon. If you use any of
+  these plugins directly you will see a warning in the mon log file.
+  Please switch to using just 'jerasure' or 'shec'.
+
+* The librados omap get_keys and get_vals operations include a start key and a
+  limit on the number of keys to return.  The OSD now imposes a configurable
+  limit on the number of keys and number of total bytes it will respond with,
+  which means that a librados user might get fewer keys than they asked for.
+  This is necessary to prevent careless users from requesting an unreasonable
+  amount of data from the cluster in a single operation.  The new limits are
+  configured with ``osd_max_omap_entries_per_request``, defaulting to 131,072, and
+  ``osd_max_omap_bytes_per_request``, defaulting to 4MB.
+
+* Calculation of recovery priorities has been updated.
+  This could lead to unintuitive recovery prioritization
+  during cluster upgrade. In case of such recovery, OSDs
+  in the old version would operate on different priority ranges
+  than new ones. Once upgraded, the cluster will operate on
+  consistent values.
+
+
+Notable Changes
+---------------
+
+* bluestore: add counter to trace blob splitting (`pr#11718 <http://github.com/ceph/ceph/pull/11718>`_, xie xingguo)
+* bluestore: a few more cleanups (`pr#11780 <http://github.com/ceph/ceph/pull/11780>`_, xie xingguo)
+* bluestore: avoid polluting shard info if need resharding (`pr#11439 <http://github.com/ceph/ceph/pull/11439>`_, xie xingguo)
+* bluestore: avoid unnecessary call to init_csum() (`pr#12015 <http://github.com/ceph/ceph/pull/12015>`_, xie xingguo)
+* bluestore: ceph-disk: adjust bluestore default device sizes (`pr#12530 <http://github.com/ceph/ceph/pull/12530>`_, Sage Weil)
+* bluestore: ceph_test_objectstore: smaller device (`pr#11591 <http://github.com/ceph/ceph/pull/11591>`_, Sage Weil)
+* bluestore: clean up Allocator::dump (`issue#18054 <http://tracker.ceph.com/issues/18054>`_, `pr#12282 <http://github.com/ceph/ceph/pull/12282>`_, Sage Weil)
+* bluestore: clear extent map on object removal (`pr#11603 <http://github.com/ceph/ceph/pull/11603>`_, Sage Weil)
+* bluestore: compressor/ZLibCompressor: fix broken isal-l (`pr#11445 <http://github.com/ceph/ceph/pull/11445>`_, Igor Fedotov)
+* bluestore: dedup if space overlap truly exists (`pr#11986 <http://github.com/ceph/ceph/pull/11986>`_, xie xingguo)
+* bluestore: dedup omap_head, reuse nid instead (`pr#12275 <http://github.com/ceph/ceph/pull/12275>`_, xie xingguo)
+* bluestore: deep fsck (`pr#11724 <http://github.com/ceph/ceph/pull/11724>`_, Sage Weil)
+* bluestore: default bluestore_clone_cow=true (`pr#11540 <http://github.com/ceph/ceph/pull/11540>`_, Sage Weil)
+* bluestore: drop inline_dirty from struct ExtentMap (`pr#11377 <http://github.com/ceph/ceph/pull/11377>`_, xie xingguo)
+* bluestore: drop member "space" from Onode (`pr#12185 <http://github.com/ceph/ceph/pull/12185>`_, xie xingguo)
+* bluestore: fix alloc release timing on sync submits (`pr#11983 <http://github.com/ceph/ceph/pull/11983>`_, Sage Weil)
+* bluestore: fix bufferspace stats leak due to blob splitting (`pr#12039 <http://github.com/ceph/ceph/pull/12039>`_, xie xingguo)
+* bluestore: fix collection_list end bound off-by-one (`pr#11771 <http://github.com/ceph/ceph/pull/11771>`_, Sage Weil)
+* bluestore: fix compiler warnings (`pr#11905 <http://github.com/ceph/ceph/pull/11905>`_, xie xingguo)
+* bluestore: fixes and cleanups (`pr#11761 <http://github.com/ceph/ceph/pull/11761>`_, xie xingguo)
+* bluestore: fix escaping of chars > 0x80 (`pr#11502 <http://github.com/ceph/ceph/pull/11502>`_, Sage Weil)
+* bluestore: fix extent shard span check (`pr#11725 <http://github.com/ceph/ceph/pull/11725>`_, Sage Weil)
+* bluestore: fix has_aios (`pr#11317 <http://github.com/ceph/ceph/pull/11317>`_, Sage Weil)
+* bluestore: Fix invalid compression statfs caused by clone op (`pr#11351 <http://github.com/ceph/ceph/pull/11351>`_, Igor Fedotov)
+* bluestore: fix lack of resharding (`pr#11597 <http://github.com/ceph/ceph/pull/11597>`_, Igor Fedotov)
+* bluestore: fix latency calculation (`pr#12040 <http://github.com/ceph/ceph/pull/12040>`_, Pan Liu)
+* bluestore: fix onode vs extent key suffix (`pr#11452 <http://github.com/ceph/ceph/pull/11452>`_, Sage Weil)
+* bluestore: fix potential memory leak (`pr#11893 <http://github.com/ceph/ceph/pull/11893>`_, xie xingguo)
+* bluestore: fix race condtion during blob spliting (`pr#11422 <http://github.com/ceph/ceph/pull/11422>`_, xiexingguo, xie xingguo)
+* bluestore: fix remove_collection to properly detect collection e… (`pr#11398 <http://github.com/ceph/ceph/pull/11398>`_, Igor Fedotov)
+* bluestore: fix _split_collections race with osr_reap (`pr#11748 <http://github.com/ceph/ceph/pull/11748>`_, Sage Weil)
+* bluestore: fix up compression tests and debug output (`pr#11350 <http://github.com/ceph/ceph/pull/11350>`_, Sage Weil)
+* bluestore: fix writes that span existing shard boundaries (`pr#11451 <http://github.com/ceph/ceph/pull/11451>`_, Sage Weil)
+* bluestore: flush before enumerating omap values (`issue#18140 <http://tracker.ceph.com/issues/18140>`_, `pr#12328 <http://github.com/ceph/ceph/pull/12328>`_, Sage Weil)
+* bluestore: formatting nits (`pr#11514 <http://github.com/ceph/ceph/pull/11514>`_, xie xingguo)
+* bluestore: fsck: fix omap_head check (`pr#11726 <http://github.com/ceph/ceph/pull/11726>`_, Sage Weil)
+* bluestore: GC infra refactor, more UTs and GC range calculation fixes (`pr#11482 <http://github.com/ceph/ceph/pull/11482>`_, Igor Fedotov)
+* bluestore: KernelDevice: fix race in aio_thread vs aio_wait (`issue#17824 <http://tracker.ceph.com/issues/17824>`_, `pr#12204 <http://github.com/ceph/ceph/pull/12204>`_, Sage Weil)
+* bluestore: kv: dump rocksdb stats (`pr#12287 <http://github.com/ceph/ceph/pull/12287>`_, Varada Kari, Jianpeng Ma, Sage Weil)
+* bluestore: kv/rocksdb: enable rocksdb write path breakdown (`pr#11696 <http://github.com/ceph/ceph/pull/11696>`_, Haodong Tang)
+* bluestore: kv/RocksDBStore: rename option (`pr#11769 <http://github.com/ceph/ceph/pull/11769>`_, Sage Weil)
+* bluestore: less code redundancy (`pr#11740 <http://github.com/ceph/ceph/pull/11740>`_, xie xingguo)
+* bluestore: make 2q cache kin/kout size tunable (`pr#11599 <http://github.com/ceph/ceph/pull/11599>`_, Haodong Tang)
+* bluestore: mark ops that can't tolerate ENOENT (`pr#12114 <http://github.com/ceph/ceph/pull/12114>`_, Sage Weil)
+* bluestore: mempool: changes for bitmap allocator (`pr#11922 <http://github.com/ceph/ceph/pull/11922>`_, Ramesh Chander)
+* bluestore: misc. fixes and cleanups (`pr#11964 <http://github.com/ceph/ceph/pull/11964>`_, xie xingguo)
+* bluestore: move bluefs into its own mempool (`pr#11834 <http://github.com/ceph/ceph/pull/11834>`_, Sage Weil)
+* bluestore: no garbage collection for uncompressed blobs (`pr#11539 <http://github.com/ceph/ceph/pull/11539>`_, Roushan Ali, Sage Weil)
+* bluestore: optional debug mode to identify aio stalls (`pr#11818 <http://github.com/ceph/ceph/pull/11818>`_, Sage Weil)
+* bluestore: os/bluestore: a few cleanups (`pr#11483 <http://github.com/ceph/ceph/pull/11483>`_, Sage Weil)
+* bluestore: os/bluestore: avoid resharding if the last shard size fall below shar… (`pr#12447 <http://github.com/ceph/ceph/pull/12447>`_, Igor Fedotov)
+* bluestore: os/bluestore: bitmap allocator dump functionality (`pr#12298 <http://github.com/ceph/ceph/pull/12298>`_, Ramesh Chander)
+* bluestore: os/bluestore: bluestore_sync_submit_transaction = false (`pr#12367 <http://github.com/ceph/ceph/pull/12367>`_, Sage Weil)
+* bluestore: os/bluestore: cleanup around Blob::ref_map (`pr#11896 <http://github.com/ceph/ceph/pull/11896>`_, Igor Fedotov)
+* bluestore: os/bluestore: clear omap flag if parent has none (`pr#12351 <http://github.com/ceph/ceph/pull/12351>`_, xie xingguo)
+* bluestore: os/bluestore: don't implicitly create the source object for clone (`pr#12353 <http://github.com/ceph/ceph/pull/12353>`_, xie xingguo)
+* bluestore: os/bluestore: drop old bluestore preconditioning; replace with wal preextension of file size (`pr#12265 <http://github.com/ceph/ceph/pull/12265>`_, Sage Weil)
+* bluestore: os/bluestore: fix global commit latency (`pr#12356 <http://github.com/ceph/ceph/pull/12356>`_, xie xingguo)
+* bluestore: os/bluestore: fix ondisk encoding for blobs (`pr#12488 <http://github.com/ceph/ceph/pull/12488>`_, Varada Kari, Sage Weil)
+* bluestore: os/bluestore: fix potential csum_order overflow (`pr#12333 <http://github.com/ceph/ceph/pull/12333>`_, xie xingguo)
+* bluestore: os/bluestore: fix target_buffer value overflow in Cache::trim() (`pr#12507 <http://github.com/ceph/ceph/pull/12507>`_, Igor Fedotov)
+* bluestore: os/bluestore: include modified objects in flush list even if onode unchanged (`pr#12541 <http://github.com/ceph/ceph/pull/12541>`_, Sage Weil)
+* bluestore: os/bluestore: kill dead gc-related counters (`pr#12065 <http://github.com/ceph/ceph/pull/12065>`_, xie xingguo)
+* bluestore: os/bluestore: kill overlay related options (`pr#11557 <http://github.com/ceph/ceph/pull/11557>`_, xie xingguo)
+* bluestore: os/bluestore: misc coverity fixes/cleanups (`pr#12202 <http://github.com/ceph/ceph/pull/12202>`_, Sage Weil)
+* bluestore: os/bluestore: preserve source collection cache during split (`pr#12574 <http://github.com/ceph/ceph/pull/12574>`_, Sage Weil)
+* bluestore: os/bluestore: remove 'extents' from shard_info (`pr#12629 <http://github.com/ceph/ceph/pull/12629>`_, Sage Weil)
+* bluestore: os/bluestore: simplified allocator interfaces to single apis (`pr#12355 <http://github.com/ceph/ceph/pull/12355>`_, Ramesh Chander)
+* bluestore: os/bluestore: simplify allocator release flow (`pr#12343 <http://github.com/ceph/ceph/pull/12343>`_, Sage Weil)
+* bluestore: os/bluestore: simplify can_split_at()  (`pr#11607 <http://github.com/ceph/ceph/pull/11607>`_, xie xingguo)
+* bluestore: os/bluestore: use iterator for erase() method directly (`pr#11490 <http://github.com/ceph/ceph/pull/11490>`_, xie xingguo)
+* bluestore: os/kstore: rmcoll fix to satisfy store_test (`pr#11533 <http://github.com/ceph/ceph/pull/11533>`_, Igor Fedotov)
+* bluestore: os: make filestore_blackhole -> objectstore_blackhole (`pr#11788 <http://github.com/ceph/ceph/pull/11788>`_, Sage Weil)
+* bluestore: os: move_ranges_destroy_src (`pr#11237 <http://github.com/ceph/ceph/pull/11237>`_, Manali Kulkarni, Sage Weil)
+* bluestore: readability improvements and doxygen fix (`pr#11895 <http://github.com/ceph/ceph/pull/11895>`_, xie xingguo)
+* bluestore: reap collection after all pending ios done (`pr#11797 <http://github.com/ceph/ceph/pull/11797>`_, Haomai Wang)
+* bluestore: reap ioc when stopping aio_thread. (`pr#11811 <http://github.com/ceph/ceph/pull/11811>`_, Haodong Tang)
+* bluestore: refactor _do_write(); move initializaiton of csum out of loop (`pr#11823 <http://github.com/ceph/ceph/pull/11823>`_, xie xingguo)
+* bluestore: remove duplicated namespace of tx state (`pr#11845 <http://github.com/ceph/ceph/pull/11845>`_, xie xingguo)
+* bluestore: remove garbage collector staff (`pr#12042 <http://github.com/ceph/ceph/pull/12042>`_, Igor Fedotov)
+* bluestore: set next object as ghobject_t::get_max() when start.hobj.i… (`pr#11495 <http://github.com/ceph/ceph/pull/11495>`_, Xinze Chi, Haomai Wang)
+* bluestore: simplify blob status checking for small writes (`pr#11366 <http://github.com/ceph/ceph/pull/11366>`_, xie xingguo)
+* bluestore: some more cleanups (`pr#11910 <http://github.com/ceph/ceph/pull/11910>`_, xie xingguo)
+* bluestore: spdk: a few fixes (`pr#11882 <http://github.com/ceph/ceph/pull/11882>`_, Yehuda Sadeh)
+* bluestore: speed up omap-key generation for same onode (`pr#11807 <http://github.com/ceph/ceph/pull/11807>`_, xie xingguo)
+* bluestore: traverse buffer_map in reverse order when spliting BufferSpace (`pr#11468 <http://github.com/ceph/ceph/pull/11468>`_, xie xingguo)
+* bluestore: update cache logger after 'trim_cache' operation (`pr#11695 <http://github.com/ceph/ceph/pull/11695>`_, Haodong Tang)
+* bluestore: use bitmap allocator for bluefs (`pr#12285 <http://github.com/ceph/ceph/pull/12285>`_, Sage Weil)
+* bluestore: use std::unordered_map for SharedBlob lookup (`pr#11394 <http://github.com/ceph/ceph/pull/11394>`_, Sage Weil)
+* build/ops: AArch64: Detect crc32 extension support from assembler (`issue#17516 <http://tracker.ceph.com/issues/17516>`_, `pr#11391 <http://github.com/ceph/ceph/pull/11391>`_, Alexander Graf)
+* build/ops: boost: embedded (`pr#11817 <http://github.com/ceph/ceph/pull/11817>`_, Sage Weil, Matt Benjamin)
+* build/ops: build: dump env during build (`issue#18084 <http://tracker.ceph.com/issues/18084>`_, `pr#12284 <http://github.com/ceph/ceph/pull/12284>`_, Sage Weil)
+* build/ops: ceph-detect-init: FreeBSD introduction of bsdrc (`pr#11906 <http://github.com/ceph/ceph/pull/11906>`_, Willem Jan Withagen, Kefu Chai)
+* build/ops: ceph-disk: enable --runtime ceph-osd systemd units (`issue#17889 <http://tracker.ceph.com/issues/17889>`_, `pr#12241 <http://github.com/ceph/ceph/pull/12241>`_, Loic Dachary)
+* build/ops: ceph.spec: add pybind rgwfile (`pr#11847 <http://github.com/ceph/ceph/pull/11847>`_, Haomai Wang)
+* build/ops,cleanup,bluestore: os/bluestore: remove build warning in a better way (`pr#11920 <http://github.com/ceph/ceph/pull/11920>`_, Igor Fedotov)
+* build/ops: CMakeLists: add vstart-base target (`pr#12476 <http://github.com/ceph/ceph/pull/12476>`_, Sage Weil)
+* build/ops: CMakeLists.txt: enable LTTNG by default (`pr#11500 <http://github.com/ceph/ceph/pull/11500>`_, Sage Weil)
+* build/ops: common/buffer.cc: raw_pipe depends on splice(2) (`pr#11967 <http://github.com/ceph/ceph/pull/11967>`_, Willem Jan Withagen)
+* build/ops,common: common/str_list.h: fix clang warning about std::move (`pr#12570 <http://github.com/ceph/ceph/pull/12570>`_, Willem Jan Withagen)
+* build/ops,core: xio: fix build (`pr#11768 <http://github.com/ceph/ceph/pull/11768>`_, Matt Benjamin)
+* build/ops: deb: add python dependencies where needed (`issue#17579 <http://tracker.ceph.com/issues/17579>`_, `pr#11507 <http://github.com/ceph/ceph/pull/11507>`_, Nathan Cutler, Kefu Chai)
+* build/ops: deb: add python-rgw packages (`pr#11832 <http://github.com/ceph/ceph/pull/11832>`_, Sage Weil)
+* build/ops: debian: apply dh_python to python-rgw also (`pr#12260 <http://github.com/ceph/ceph/pull/12260>`_, Kefu Chai)
+* build/ops: deb: update python-rgw dependencies to librgw2 (`pr#11885 <http://github.com/ceph/ceph/pull/11885>`_, Casey Bodley)
+* build/ops: do_freebsd.sh: Build with SYSTEM Boost on FreeBSD (`pr#11942 <http://github.com/ceph/ceph/pull/11942>`_, Willem Jan Withagen)
+* build/ops: do_freebsd.sh: Do not use LTTNG on FreeBSD (`pr#11551 <http://github.com/ceph/ceph/pull/11551>`_, Willem Jan Withagen)
+* build/ops: do_freebsd.sh: Set options for debug building. (`pr#11443 <http://github.com/ceph/ceph/pull/11443>`_, Willem Jan Withagen)
+* build/ops: FreeBSD: do_freebsd.sh (`pr#12090 <http://github.com/ceph/ceph/pull/12090>`_, Willem Jan Withagen)
+* build/ops: FreeBSD:test/encoding/readable.sh": fix nproc and ls -v calls (`pr#11522 <http://github.com/ceph/ceph/pull/11522>`_, Willem Jan Withagen)
+* build/ops: FreeBSD: update require packages (`pr#11512 <http://github.com/ceph/ceph/pull/11512>`_, Willem Jan Withagen)
+* build/ops: git-archive-all.sh: use an actually unique tmp dir (`pr#12011 <http://github.com/ceph/ceph/pull/12011>`_, Dan Mick)
+* build/ops: include/enc: make clang happy (`pr#11638 <http://github.com/ceph/ceph/pull/11638>`_, Kefu Chai, Sage Weil)
+* build/ops: install-deps.sh: allow building on SLES systems (`pr#11708 <http://github.com/ceph/ceph/pull/11708>`_, Nitin A Kamble)
+* build/ops: install-deps.sh: JQ is needed in one script (`pr#12080 <http://github.com/ceph/ceph/pull/12080>`_, Willem Jan Withagen)
+* build/ops: Log: Replace namespace log with logging (`pr#11650 <http://github.com/ceph/ceph/pull/11650>`_, Willem Jan Withagen)
+* build/ops: Merging before make check because it clearly breaks the build and the build part is done (`pr#11924 <http://github.com/ceph/ceph/pull/11924>`_, Sage Weil)
+* build/ops: ok, w/upstream acks, merging--jenkins build did succeed (this is a build-only change) (`pr#12008 <http://github.com/ceph/ceph/pull/12008>`_, Matt Benjamin)
+* build/ops: qa: Add ceph-ansible installer. (`issue#16770 <http://tracker.ceph.com/issues/16770>`_, `pr#10402 <http://github.com/ceph/ceph/pull/10402>`_, Warren Usui)
+* build/ops: rocksdb: do not build with --march=native (`pr#11677 <http://github.com/ceph/ceph/pull/11677>`_, Kefu Chai)
+* build/ops: rocksdb: update to latest (`pr#12100 <http://github.com/ceph/ceph/pull/12100>`_, Kefu Chai)
+* build/ops: rpm: Remove trailing whitespace in usermod command (SUSE) (`pr#10707 <http://github.com/ceph/ceph/pull/10707>`_, Tim Serong)
+* build/ops: scripts/release-notes: allow title guesses from gh tags & description update (`pr#11399 <http://github.com/ceph/ceph/pull/11399>`_, Abhishek Lekshmanan)
+* build/ops: systemd: Fix startup of ceph-mgr on Debian 8 (`pr#12555 <http://github.com/ceph/ceph/pull/12555>`_, Mark Korenberg)
+* build/ops: tracing/objectstore.tp: add missing move_ranges\_... tp (`pr#11484 <http://github.com/ceph/ceph/pull/11484>`_, Sage Weil)
+* build/ops: upstart: fix ceph-crush-location default (`issue#6698 <http://tracker.ceph.com/issues/6698>`_, `pr#803 <http://github.com/ceph/ceph/pull/803>`_, Jason Dillaman)
+* build/ops: upstart: start ceph-all after static-network-up (`issue#17689 <http://tracker.ceph.com/issues/17689>`_, `pr#11631 <http://github.com/ceph/ceph/pull/11631>`_, Billy Olsen)
+* cephfs: add gid to asok status (`pr#11487 <http://github.com/ceph/ceph/pull/11487>`_, Patrick Donnelly)
+* cephfs: API cleanup for libcephfs interfaces (`issue#17911 <http://tracker.ceph.com/issues/17911>`_, `pr#12106 <http://github.com/ceph/ceph/pull/12106>`_, Jeff Layton)
+* cephfs: ceph-fuse: start up log on parent process before shutdown (`issue#18157 <http://tracker.ceph.com/issues/18157>`_, `pr#12347 <http://github.com/ceph/ceph/pull/12347>`_, Greg Farnum)
+* cephfs: ceph_fuse: use sizeof get the buf length (`pr#11176 <http://github.com/ceph/ceph/pull/11176>`_, LeoZhang)
+* cephfs,cleanup: ceph-fuse: start up log on parent process before shutdown (`issue#18157 <http://tracker.ceph.com/issues/18157>`_, `pr#12358 <http://github.com/ceph/ceph/pull/12358>`_, Kefu Chai)
+* cephfs: client: add pid to metadata (`issue#17276 <http://tracker.ceph.com/issues/17276>`_, `pr#11359 <http://github.com/ceph/ceph/pull/11359>`_, Patrick Donnelly)
+* cephfs: client: Client.cc: remove duplicated op type checking against CEPH_MD… (`pr#11608 <http://github.com/ceph/ceph/pull/11608>`_, Weibing Zhang)
+* cephfs: client: don't take extra target inode reference in ll_link (`pr#11440 <http://github.com/ceph/ceph/pull/11440>`_, Jeff Layton)
+* cephfs: client: fix mutex name typos (`pr#12401 <http://github.com/ceph/ceph/pull/12401>`_, Yunchuan Wen)
+* cephfs: client: get caller's uid/gid on every libcephfs operation (`issue#17591 <http://tracker.ceph.com/issues/17591>`_, `pr#11526 <http://github.com/ceph/ceph/pull/11526>`_, Yan, Zheng)
+* cephfs: client: get gid from MonClient (`pr#11486 <http://github.com/ceph/ceph/pull/11486>`_, Patrick Donnelly)
+* cephfs: client: improve failure messages/debugging (`pr#12110 <http://github.com/ceph/ceph/pull/12110>`_, Patrick Donnelly)
+* cephfs: client/mds: Clear setuid bits when writing or truncating (`issue#18131 <http://tracker.ceph.com/issues/18131>`_, `pr#12412 <http://github.com/ceph/ceph/pull/12412>`_, Jeff Layton)
+* cephfs: client: put CapSnap not ptr in cap_snaps map (`pr#12111 <http://github.com/ceph/ceph/pull/12111>`_, Patrick Donnelly)
+* cephfs: client: remove redundant initialization (`pr#12028 <http://github.com/ceph/ceph/pull/12028>`_, Patrick Donnelly)
+* cephfs:  client: remove unnecessary bufferptr[] for writev (`pr#11836 <http://github.com/ceph/ceph/pull/11836>`_, Patrick Donnelly)
+* cephfs: client: remove unneeded layout on MClientCaps (`pr#11790 <http://github.com/ceph/ceph/pull/11790>`_, John Spray)
+* cephfs: client: set metadata["root"] from mount method when it's called with … (`pr#12505 <http://github.com/ceph/ceph/pull/12505>`_, Jeff Layton)
+* cephfs: client: trim_caps() do not dereference cap if it's removed (`pr#12145 <http://github.com/ceph/ceph/pull/12145>`_, Kefu Chai)
+* cephfs: client: use unique_ptr (`pr#11837 <http://github.com/ceph/ceph/pull/11837>`_, Patrick Donnelly)
+* cephfs: common/ceph_string: add ceph string constants for CEPH_SESSION_FORCE_RO (`pr#11516 <http://github.com/ceph/ceph/pull/11516>`_, Zhi Zhang)
+* cephfs: Fix #17562 (backtrace check fails when scrubbing directory created by fsstress) (`issue#17562 <http://tracker.ceph.com/issues/17562>`_, `pr#11517 <http://github.com/ceph/ceph/pull/11517>`_, Yan, Zheng)
+* cephfs: fix missing ll_get for ll_walk (`issue#18086 <http://tracker.ceph.com/issues/18086>`_, `pr#12061 <http://github.com/ceph/ceph/pull/12061>`_, Gui Hecheng)
+* cephfs: get new fsmap after marking clusters down (`issue#7271 <http://tracker.ceph.com/issues/7271>`_, `issue#17894 <http://tracker.ceph.com/issues/17894>`_, `pr#1262 <http://github.com/ceph/ceph/pull/1262>`_, Patrick Donnelly)
+* cephfs: Have ceph clear setuid/setgid bits on chown (`issue#18131 <http://tracker.ceph.com/issues/18131>`_, `pr#12331 <http://github.com/ceph/ceph/pull/12331>`_, Jeff Layton)
+* cephfs: libcephfs: add ceph_fsetattr&&ceph_lchmod&&ceph_lutime (`pr#11191 <http://github.com/ceph/ceph/pull/11191>`_, huanwen ren)
+* cephfs: libcephfs: add readlink function in cephfs.pyx (`pr#12384 <http://github.com/ceph/ceph/pull/12384>`_, huanwen ren)
+* cephfs: libcephfs and test suite fixes (`issue#18013 <http://tracker.ceph.com/issues/18013>`_, `issue#17982 <http://tracker.ceph.com/issues/17982>`_, `pr#12228 <http://github.com/ceph/ceph/pull/12228>`_, Jeff Layton)
+* cephfs: libcephfs client API overhaul and update (`pr#11647 <http://github.com/ceph/ceph/pull/11647>`_, Jeff Layton)
+* cephfs: lua: use simpler lua_next traversal structure (`pr#11958 <http://github.com/ceph/ceph/pull/11958>`_, Patrick Donnelly)
+* cephfs: mds/Beacon: move C_MDS_BeaconSender class to .cc (`pr#10940 <http://github.com/ceph/ceph/pull/10940>`_, Michal Jarzabek)
+* cephfs: mds/CDir.cc: remove unneeded use of count (`pr#11613 <http://github.com/ceph/ceph/pull/11613>`_, Michal Jarzabek)
+* cephfs: mds/CInode.h: remove unneeded use of count (`pr#11371 <http://github.com/ceph/ceph/pull/11371>`_, Michal Jarzabek)
+* cephfs: mds/DamageTable.cc: move shared ptrs (`pr#11435 <http://github.com/ceph/ceph/pull/11435>`_, Michal Jarzabek)
+* cephfs: mds/DamageTable.cc: remove unneeded use of count (`pr#11625 <http://github.com/ceph/ceph/pull/11625>`_, Michal Jarzabek)
+* cephfs: mds/DamageTable: move classes to .cc file (`pr#11450 <http://github.com/ceph/ceph/pull/11450>`_, Michal Jarzabek)
+* cephfs: mds/flock: add const to member functions (`pr#11692 <http://github.com/ceph/ceph/pull/11692>`_, Michal Jarzabek)
+* cephfs: mds/FSMap.cc: remove unneeded use of count (`pr#11402 <http://github.com/ceph/ceph/pull/11402>`_, Michal Jarzabek)
+* cephfs: mds/FSMapUser.h: remove copy ctr and assign op (`pr#11509 <http://github.com/ceph/ceph/pull/11509>`_, Michal Jarzabek)
+* cephfs: mds/InfoTable.h: add override to virtual functs (`pr#11496 <http://github.com/ceph/ceph/pull/11496>`_, Michal Jarzabek)
+* cephfs: mds/InoTable.h: add override to virtual functs (`pr#11604 <http://github.com/ceph/ceph/pull/11604>`_, Michal Jarzabek)
+* cephfs: mds/Mantle.h: include correct header files (`pr#11886 <http://github.com/ceph/ceph/pull/11886>`_, Michal Jarzabek)
+* cephfs: mds/Mantle: pass parameters by const ref (`pr#11713 <http://github.com/ceph/ceph/pull/11713>`_, Michal Jarzabek)
+* cephfs: mds/MDCache.h: remove unneeded call to clear func (`pr#11954 <http://github.com/ceph/ceph/pull/11954>`_, Michal Jarzabek)
+* cephfs: mds/MDCache.h: remove unused functions (`pr#11908 <http://github.com/ceph/ceph/pull/11908>`_, Michal Jarzabek)
+* cephfs: mds/MDLog: add const to member functions (`pr#11663 <http://github.com/ceph/ceph/pull/11663>`_, Michal Jarzabek)
+* cephfs: mds/MDSMap.h: add const to member functions (`pr#11511 <http://github.com/ceph/ceph/pull/11511>`_, Michal Jarzabek)
+* cephfs: mds/MDSRank: add const to member functions (`pr#11752 <http://github.com/ceph/ceph/pull/11752>`_, Michal Jarzabek)
+* cephfs: mds/MDSRank.h: add override to virtual function (`pr#11727 <http://github.com/ceph/ceph/pull/11727>`_, Michal Jarzabek)
+* cephfs: mds/MDSRank.h: make destructor protected (`pr#11651 <http://github.com/ceph/ceph/pull/11651>`_, Michal Jarzabek)
+* cephfs: mds/MDSTableClient.h: add const to member funct (`pr#11681 <http://github.com/ceph/ceph/pull/11681>`_, Michal Jarzabek)
+* cephfs: mds/Migrator.cc: remove unneeded use of count (`pr#11523 <http://github.com/ceph/ceph/pull/11523>`_, Michal Jarzabek)
+* cephfs: mds/Migrator.h: add const to member functions (`pr#11819 <http://github.com/ceph/ceph/pull/11819>`_, Michal Jarzabek)
+* cephfs: mds/Migrator.h: remove unneeded use of count (`pr#11833 <http://github.com/ceph/ceph/pull/11833>`_, Michal Jarzabek)
+* cephfs: mds/Mutation.h: add const to member functions (`pr#11670 <http://github.com/ceph/ceph/pull/11670>`_, Michal Jarzabek)
+* cephfs: mds/Mutation.h: simplify constructors (`pr#11455 <http://github.com/ceph/ceph/pull/11455>`_, Michal Jarzabek)
+* cephfs: MDS: reduce usage of context wrapper (`pr#11560 <http://github.com/ceph/ceph/pull/11560>`_, Yan, Zheng)
+* cephfs: mds/ScrubHeader.h: pass string by const reference (`pr#11904 <http://github.com/ceph/ceph/pull/11904>`_, Michal Jarzabek)
+* cephfs: mds/server: merge the snapshot request judgment (`pr#11150 <http://github.com/ceph/ceph/pull/11150>`_, huanwen ren)
+* cephfs: mds/SessionMap: add const to member functions (`pr#11541 <http://github.com/ceph/ceph/pull/11541>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.cc: avoid copying and add const (`pr#11297 <http://github.com/ceph/ceph/pull/11297>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.cc:put classes in unnamed namespace (`pr#11316 <http://github.com/ceph/ceph/pull/11316>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.cc: remove unneeded use of count (`pr#11338 <http://github.com/ceph/ceph/pull/11338>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.h: remove unneeded function (`pr#11565 <http://github.com/ceph/ceph/pull/11565>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.h: remove unneeded use of count (`pr#11358 <http://github.com/ceph/ceph/pull/11358>`_, Michal Jarzabek)
+* cephfs: mds/SnapRealm: remove unneeded use of count (`pr#11609 <http://github.com/ceph/ceph/pull/11609>`_, Michal Jarzabek)
+* cephfs: mds/SnapServer.h: add override to virtual functs (`pr#11380 <http://github.com/ceph/ceph/pull/11380>`_, Michal Jarzabek)
+* cephfs: mds/SnapServer.h: add override to virtual functs (`pr#11583 <http://github.com/ceph/ceph/pull/11583>`_, Michal Jarzabek)
+* cephfs: mon/MDSMonitor: fix iterating over mutated map (`issue#18166 <http://tracker.ceph.com/issues/18166>`_, `pr#12395 <http://github.com/ceph/ceph/pull/12395>`_, John Spray)
+* cephfs: multimds: fix state check in Migrator::find_stale_export_freeze() (`pr#12098 <http://github.com/ceph/ceph/pull/12098>`_, Yan, Zheng)
+* cephfs: osdc: After write try merge bh. (`issue#17270 <http://tracker.ceph.com/issues/17270>`_, `pr#11545 <http://github.com/ceph/ceph/pull/11545>`_, Jianpeng Ma)
+* cephfs: Partial organization of mds/ header sections (`pr#11959 <http://github.com/ceph/ceph/pull/11959>`_, Patrick Donnelly)
+* cephfs: Port/bootstrap (`pr#827 <http://github.com/ceph/ceph/pull/827>`_, Yan, Zheng)
+* cephfs: Revert "osdc: After write try merge bh." (`issue#17270 <http://tracker.ceph.com/issues/17270>`_, `pr#11262 <http://github.com/ceph/ceph/pull/11262>`_, John Spray)
+* cephfs: Small pile of random cephfs fixes and cleanup (`pr#11421 <http://github.com/ceph/ceph/pull/11421>`_, Jeff Layton)
+* cephfs: src/mds: fix MDSMap upgrade decoding (`issue#17837 <http://tracker.ceph.com/issues/17837>`_, `pr#12097 <http://github.com/ceph/ceph/pull/12097>`_, John Spray)
+* cephfs: systemd: add ceph-fuse service file (`pr#11542 <http://github.com/ceph/ceph/pull/11542>`_, Patrick Donnelly)
+* cephfs: test fragment size limit (`issue#16164 <http://tracker.ceph.com/issues/16164>`_, `pr#1069 <http://github.com/ceph/ceph/pull/1069>`_, Patrick Donnelly)
+* cephfs: test readahead is working (`issue#16024 <http://tracker.ceph.com/issues/16024>`_, `pr#1046 <http://github.com/ceph/ceph/pull/1046>`_, Patrick Donnelly)
+* cephfs: test: temporarily remove fork()ing flock tests (`issue#16556 <http://tracker.ceph.com/issues/16556>`_, `pr#11211 <http://github.com/ceph/ceph/pull/11211>`_, John Spray)
+* cephfs: tool/cephfs: displaying "list" in journal event mode (`pr#11236 <http://github.com/ceph/ceph/pull/11236>`_, huanwen ren)
+* cephfs: tools/cephfs: add pg_files command (`issue#17249 <http://tracker.ceph.com/issues/17249>`_, `pr#11026 <http://github.com/ceph/ceph/pull/11026>`_, John Spray)
+* cephfs: tools/cephfs: add scan_links command which fixes linkages errors (`pr#11446 <http://github.com/ceph/ceph/pull/11446>`_, Yan, Zheng)
+* cephfs: update tests to enable multimds when needed (`pr#933 <http://github.com/ceph/ceph/pull/933>`_, Greg Farnum)
+* cleanup: build: The Light Clangtastic (`pr#11921 <http://github.com/ceph/ceph/pull/11921>`_, Adam C. Emerson)
+* cleanup,common: common/blkdev: use realpath instead of readlink to resolve the recurs… (`pr#12462 <http://github.com/ceph/ceph/pull/12462>`_, Xinze Chi)
+* cleanup,common: common/throttle: simplify Throttle::_wait() (`pr#11165 <http://github.com/ceph/ceph/pull/11165>`_, xie xingguo)
+* cleanup,common: src/common: remove nonused config option (`pr#12311 <http://github.com/ceph/ceph/pull/12311>`_, Wei Jin)
+* cleanup: coverity fix: fixing few coverity issue (`pr#9624 <http://github.com/ceph/ceph/pull/9624>`_, Gaurav Kumar Garg)
+* cleanup: deprecate readdir_r() with readdir() (`pr#11805 <http://github.com/ceph/ceph/pull/11805>`_, Kefu Chai)
+* cleanup: erasure-code: fix gf-complete warning (`pr#12150 <http://github.com/ceph/ceph/pull/12150>`_, Kefu Chai)
+* cleanup: fix typos (`pr#12502 <http://github.com/ceph/ceph/pull/12502>`_, xianxiaxiao)
+* cleanup: mds/FSMap.cc: prevent unneeded copy of map entry (`pr#11798 <http://github.com/ceph/ceph/pull/11798>`_, Michal Jarzabek)
+* cleanup: mds/FSMap.h: add const and reference (`pr#11802 <http://github.com/ceph/ceph/pull/11802>`_, Michal Jarzabek)
+* cleanup: mds/FSMap: pass shared_ptr by const ref (`pr#11383 <http://github.com/ceph/ceph/pull/11383>`_, Michal Jarzabek)
+* cleanup: mds/SnapServer: add const to member function (`pr#11688 <http://github.com/ceph/ceph/pull/11688>`_, Michal Jarzabek)
+* cleanup: mon/MonCap.h: add std::move for std::string (`pr#10722 <http://github.com/ceph/ceph/pull/10722>`_, Michal Jarzabek)
+* cleanup: mon/OSDMonitor: only show interesting flags in health warning (`issue#18175 <http://tracker.ceph.com/issues/18175>`_, `pr#12365 <http://github.com/ceph/ceph/pull/12365>`_, Sage Weil)
+* cleanup: msg/async: assert(0) -> ceph_abort() (`pr#12339 <http://github.com/ceph/ceph/pull/12339>`_, Li Wang)
+* cleanup: msg/AsyncMessenger: remove unneeded include (`pr#9846 <http://github.com/ceph/ceph/pull/9846>`_, Michal Jarzabek)
+* cleanup: msg/async/rdma: fix disconnect log line (`pr#12254 <http://github.com/ceph/ceph/pull/12254>`_, Adir Lev)
+* cleanup: msg/async: remove unused member variable (`pr#12387 <http://github.com/ceph/ceph/pull/12387>`_, Kefu Chai)
+* cleanup: msg: fix format specifier for unsigned value id (`pr#11145 <http://github.com/ceph/ceph/pull/11145>`_, Weibing Zhang)
+* cleanup: msg/Pipe: move DelayedDelivery class to cc file (`pr#10447 <http://github.com/ceph/ceph/pull/10447>`_, Michal Jarzabek)
+* cleanup: msg/test: fix the guided compile-command to ceph_test_msgr (`pr#10490 <http://github.com/ceph/ceph/pull/10490>`_, Yan Jun)
+* cleanup: osd/PGBackend: build_push_op segment fault (`pr#9357 <http://github.com/ceph/ceph/pull/9357>`_, Zengran Zhang)
+* cleanup: osd/PG.h: change PGRecoveryStats struct to class (`pr#11178 <http://github.com/ceph/ceph/pull/11178>`_, Michal Jarzabek)
+* cleanup: osd/PG.h: remove unneeded forward declaration (`pr#12135 <http://github.com/ceph/ceph/pull/12135>`_, Li Wang)
+* cleanup: osd/ReplicatedPG: remove unneeded use of count (`pr#11251 <http://github.com/ceph/ceph/pull/11251>`_, Michal Jarzabek)
+* cleanup: os/filestore: clean filestore perfcounters (`pr#11524 <http://github.com/ceph/ceph/pull/11524>`_, Wei Jin)
+* cleanup: os/fs/FS.cc: condition on WITH_AIO for FreeBSD (`pr#11913 <http://github.com/ceph/ceph/pull/11913>`_, Willem Jan Withagen)
+* cleanup,rbd: cls_rbd: silence compiler warnings (`pr#11363 <http://github.com/ceph/ceph/pull/11363>`_, xiexingguo)
+* cleanup,rbd: journal: avoid logging an error when a watch is blacklisted (`issue#18243 <http://tracker.ceph.com/issues/18243>`_, `pr#12473 <http://github.com/ceph/ceph/pull/12473>`_, Jason Dillaman)
+* cleanup,rbd: journal: prevent repetitive error messages after being blacklisted (`issue#18243 <http://tracker.ceph.com/issues/18243>`_, `pr#12497 <http://github.com/ceph/ceph/pull/12497>`_, Jason Dillaman)
+* cleanup,rbd: librbd/ImageCtx: no need for virtual dtor (`pr#12220 <http://github.com/ceph/ceph/pull/12220>`_, Sage Weil)
+* cleanup,rbd: rbd-mirror: configuration overrides for hard coded timers (`pr#11840 <http://github.com/ceph/ceph/pull/11840>`_, Dongsheng Yang)
+* cleanup,rbd: rbd-mirror: set SEQUENTIAL and NOCACHE advise flags on image sync (`issue#17127 <http://tracker.ceph.com/issues/17127>`_, `pr#12280 <http://github.com/ceph/ceph/pull/12280>`_, Mykola Golub)
+* cleanup: remove unneeded forward declaration (`pr#12257 <http://github.com/ceph/ceph/pull/12257>`_, Li Wang, Yunchuan Wen)
+* cleanup: remove unused declaration (`pr#12466 <http://github.com/ceph/ceph/pull/12466>`_, Li Wang, Yunchuan Wen)
+* cleanup,rgw: rgw multisite: move lease up to RunBucketSync instead of child crs (`pr#11598 <http://github.com/ceph/ceph/pull/11598>`_, Casey Bodley)
+* cleanup,rgw: rgw/rest: don't print empty x-amz-request-id (`pr#10674 <http://github.com/ceph/ceph/pull/10674>`_, Marcus Watts)
+* cleanup,rgw: verified: f23 (`pr#12103 <http://github.com/ceph/ceph/pull/12103>`_, Radoslaw Zarzynski)
+* cleanup: src/common/perf_counters.h: fix wrong word (`pr#11690 <http://github.com/ceph/ceph/pull/11690>`_, zhang.zezhu)
+* cleanup: Wip ctypos (`pr#12495 <http://github.com/ceph/ceph/pull/12495>`_, xianxiaxiao)
+* cleanup: xio: provide dout_prefix for XioConnection (`pr#9444 <http://github.com/ceph/ceph/pull/9444>`_, Avner BenHanoch)
+* cleanup: yasm-wrapper: translate "-isystem $1" to "-i $1" (`pr#12093 <http://github.com/ceph/ceph/pull/12093>`_, Kefu Chai)
+* cmake: add -Wno-unknown-pragmas to CMAKE_CXX_FLAGS (`pr#12128 <http://github.com/ceph/ceph/pull/12128>`_, Kefu Chai)
+* cmake: check WITH_RADOSGW for fcgi and expat dependencies (`pr#11481 <http://github.com/ceph/ceph/pull/11481>`_, David Disseldorp)
+* cmake: compile C code with c99 (`pr#12369 <http://github.com/ceph/ceph/pull/12369>`_, Kefu Chai)
+* cmake: detect keyutils if WITH_LIBCEPHFS OR WITH_RBD (`pr#12359 <http://github.com/ceph/ceph/pull/12359>`_, Kefu Chai)
+* cmake: do not link erasure tests again libosd (`pr#11738 <http://github.com/ceph/ceph/pull/11738>`_, Kefu Chai)
+* cmake: find gperftools package for tcmalloc_minimal too (`pr#11403 <http://github.com/ceph/ceph/pull/11403>`_, Bassam Tabbara)
+* cmake: fix boost build on ubuntu 16.10 yakkety (`pr#12143 <http://github.com/ceph/ceph/pull/12143>`_, Bassam Tabbara)
+* cmake: Fix for cross compiling (`pr#11404 <http://github.com/ceph/ceph/pull/11404>`_, Bassam Tabbara)
+* cmake: fix git version string, cleanup (`pr#11661 <http://github.com/ceph/ceph/pull/11661>`_, Sage Weil)
+* cmake: librbd cleanup (`pr#11842 <http://github.com/ceph/ceph/pull/11842>`_, Kefu Chai)
+* cmake: link tests against static librados (`issue#17260 <http://tracker.ceph.com/issues/17260>`_, `pr#11575 <http://github.com/ceph/ceph/pull/11575>`_, Kefu Chai)
+* cmake: pass CMAKE_BUILD_TYPE down to rocksdb (`pr#11767 <http://github.com/ceph/ceph/pull/11767>`_, Kefu Chai)
+* cmake: remove include/Makefile.am (`pr#11666 <http://github.com/ceph/ceph/pull/11666>`_, Kefu Chai)
+* cmake: replace civetweb symlink w/file copy (`pr#11900 <http://github.com/ceph/ceph/pull/11900>`_, Matt Benjamin)
+* cmake: should link against ${ALLOC_LIBS} (`pr#11978 <http://github.com/ceph/ceph/pull/11978>`_, Kefu Chai)
+* cmake: src/test/CMakeLists.txt: Exclude test on HAVE_BLKID (`pr#12301 <http://github.com/ceph/ceph/pull/12301>`_, Willem Jan Withagen)
+* cmake: Support for embedding Ceph Daemons (`pr#11764 <http://github.com/ceph/ceph/pull/11764>`_, Bassam Tabbara)
+* cmake: use external project for rocksdb (`pr#11385 <http://github.com/ceph/ceph/pull/11385>`_, Bassam Tabbara)
+* common: Add throttle_get_started perf counter (`pr#12163 <http://github.com/ceph/ceph/pull/12163>`_, Bartłomiej Święcki)
+* common: assert(0) -> ceph_abort() (`pr#12031 <http://github.com/ceph/ceph/pull/12031>`_, Sage Weil)
+* common: auth: fix NULL pointer access when trying to delete CryptoAESKeyHandler instance (`pr#11614 <http://github.com/ceph/ceph/pull/11614>`_, runsisi)
+* common,bluestore: compressor: fixes and tests; disable zlib isal (it's broken) (`pr#11349 <http://github.com/ceph/ceph/pull/11349>`_, Sage Weil)
+* common,bluestore: mempool: mempool infrastructure, bluestore changes to use it (`pr#11331 <http://github.com/ceph/ceph/pull/11331>`_, Allen Samuels, Sage Weil)
+* common: buffer: add advance(unsigned) back (`issue#17809 <http://tracker.ceph.com/issues/17809>`_, `pr#11993 <http://github.com/ceph/ceph/pull/11993>`_, Kefu Chai)
+* common: buffer: add copy(unsigned, ptr) back (`issue#17809 <http://tracker.ceph.com/issues/17809>`_, `pr#12246 <http://github.com/ceph/ceph/pull/12246>`_, Kefu Chai)
+* common: client/Client.cc: fix/silence "logically dead code" CID-Error (`pr#291 <http://github.com/ceph/ceph/pull/291>`_, Yehuda Sadeh)
+* common: common/strtol.cc: Get error testing also to work on FreeBSD (`pr#12034 <http://github.com/ceph/ceph/pull/12034>`_, Willem Jan Withagen)
+* common: fix clang compilation error (`pr#12565 <http://github.com/ceph/ceph/pull/12565>`_, Mykola Golub)
+* common: FreeBSD/EventKqueue.{h,cc} Added code to restore events on (thread)fork (`pr#11430 <http://github.com/ceph/ceph/pull/11430>`_, Willem Jan Withagen)
+* common: log/LogClient: fill seq & who for syslog and graylog (`issue#16609 <http://tracker.ceph.com/issues/16609>`_, `pr#10196 <http://github.com/ceph/ceph/pull/10196>`_, Xiaoxi Chen)
+* common: make l_finisher_complete_lat more accurate (`pr#11637 <http://github.com/ceph/ceph/pull/11637>`_, Pan Liu)
+* common: msg/simple/Accepter.cc: replace shutdown() with selfpipe event in poll() (FreeBSD) (`pr#10720 <http://github.com/ceph/ceph/pull/10720>`_, Willem Jan Withagen)
+* common: osdc/Objecter: fix relock race (`issue#17942 <http://tracker.ceph.com/issues/17942>`_, `pr#12234 <http://github.com/ceph/ceph/pull/12234>`_, Sage Weil)
+* common: osdc/Objecter: handle race between calc_target and handle_osd_map (`issue#17942 <http://tracker.ceph.com/issues/17942>`_, `pr#12055 <http://github.com/ceph/ceph/pull/12055>`_, Sage Weil)
+* common: osd/osdmap: fix divide by zero error (`pr#12521 <http://github.com/ceph/ceph/pull/12521>`_, Yunchuan Wen)
+* common: release g_ceph_context before returns (`issue#17762 <http://tracker.ceph.com/issues/17762>`_, `pr#11733 <http://github.com/ceph/ceph/pull/11733>`_, Kefu Chai)
+* common: Remove the runtime dependency on lsb_release (`issue#17425 <http://tracker.ceph.com/issues/17425>`_, `pr#11365 <http://github.com/ceph/ceph/pull/11365>`_, Brad Hubbard)
+* common: test/fio: fix global CephContext life cycle (`pr#12245 <http://github.com/ceph/ceph/pull/12245>`_, Igor Fedotov)
+* core: auth: tolerate missing MGR keys during upgrade (`pr#11401 <http://github.com/ceph/ceph/pull/11401>`_, Sage Weil)
+* core,bluestore: os/bluestore: fix warning and uninit variable (`pr#12032 <http://github.com/ceph/ceph/pull/12032>`_, Sage Weil)
+* core,bluestore: os: fix offsets for move_ranges operation (`pr#11595 <http://github.com/ceph/ceph/pull/11595>`_, Sage Weil)
+* core,bluestore: os: remove move_ranges_destroy_src (`pr#11791 <http://github.com/ceph/ceph/pull/11791>`_, Sage Weil)
+* core: ceph-disk: allow using a regular file as a journal (`issue#17662 <http://tracker.ceph.com/issues/17662>`_, `pr#11619 <http://github.com/ceph/ceph/pull/11619>`_, Jayashree Candadai, Loic Dachary)
+* core: ceph-disk: resolve race conditions (`issue#17889 <http://tracker.ceph.com/issues/17889>`_, `issue#17813 <http://tracker.ceph.com/issues/17813>`_, `pr#12136 <http://github.com/ceph/ceph/pull/12136>`_, Loic Dachary)
+* core,cephfs: osdc/ObjectCacher: wake up dirty stat waiters after removing buffers (`issue#17275 <http://tracker.ceph.com/issues/17275>`_, `pr#11593 <http://github.com/ceph/ceph/pull/11593>`_, Yan, Zheng)
+* core: ceph.in: allow 'flags' to not be present in cmddescs (`issue#18297 <http://tracker.ceph.com/issues/18297>`_, `pr#12540 <http://github.com/ceph/ceph/pull/12540>`_, Dan Mick)
+* core,cleanup: ceph-disk: do not create bluestore wal/db partitions by default (`issue#18291 <http://tracker.ceph.com/issues/18291>`_, `pr#12531 <http://github.com/ceph/ceph/pull/12531>`_, Loic Dachary)
+* core,cleanup,common: common/TrackedOp: remove unused 'now' in _dump() (`pr#12007 <http://github.com/ceph/ceph/pull/12007>`_, John Spray)
+* core,cleanup: FileStore: Only verify split when it has been really done and done correctly (`pr#11731 <http://github.com/ceph/ceph/pull/11731>`_, Li Wang)
+* core,cleanup: kv: remove snapshot iterator (`pr#12049 <http://github.com/ceph/ceph/pull/12049>`_, Sage Weil)
+* core,cleanup: mon/MonClient.h: remove repeated searching of map (`pr#10601 <http://github.com/ceph/ceph/pull/10601>`_, Michal Jarzabek)
+* core,cleanup: msg: Fix typos in socket creation error message (`pr#11907 <http://github.com/ceph/ceph/pull/11907>`_, Brad Hubbard)
+* core,cleanup: osd/command tell: check pgid at the right time (`pr#11547 <http://github.com/ceph/ceph/pull/11547>`_, Javeme)
+* core,cleanup: osd/OSDMap.cc: fix duplicated assignment for new_blacklist_entries (`pr#11799 <http://github.com/ceph/ceph/pull/11799>`_, Ker Liu)
+* core,cleanup: osd/PG.cc: prevent repeated searching of map/set (`pr#11203 <http://github.com/ceph/ceph/pull/11203>`_, Michal Jarzabek)
+* core,cleanup: osd/ReplicatedPG: remove redundant check for balance/localize read (`pr#10209 <http://github.com/ceph/ceph/pull/10209>`_, runsisi)
+* core,cleanup: osd/ReplicatedPG: remove unneeded use of count (`pr#11242 <http://github.com/ceph/ceph/pull/11242>`_, Michal Jarzabek)
+* core,cleanup: os/filestore: handle EINTR returned by io_getevents() (`pr#11890 <http://github.com/ceph/ceph/pull/11890>`_, Pan Liu)
+* core,cleanup: os/ObjectStore: remove legacy tbl support (`pr#11770 <http://github.com/ceph/ceph/pull/11770>`_, Jianpeng Ma)
+* core,cleanup: scan build fixes (`pr#12148 <http://github.com/ceph/ceph/pull/12148>`_, Kefu Chai)
+* core,cleanup: src: rename ReplicatedPG to PrimaryLogPG (`pr#12487 <http://github.com/ceph/ceph/pull/12487>`_, Samuel Just)
+* core,cleanup: Wip scrub misc (`pr#11397 <http://github.com/ceph/ceph/pull/11397>`_, David Zafman)
+* core,common: buffer: put buffers in buffer_{data,meta} mempools (`pr#11839 <http://github.com/ceph/ceph/pull/11839>`_, Sage Weil)
+* core,common: msg: add entity_addr_t types; add new entity_addrvec_t type (`pr#9825 <http://github.com/ceph/ceph/pull/9825>`_, Zhao Junwang, Sage Weil)
+* core,common: msg/simple/Pipe: handle addr decode error (`issue#18072 <http://tracker.ceph.com/issues/18072>`_, `pr#12221 <http://github.com/ceph/ceph/pull/12221>`_, Sage Weil)
+* core: compress: Fix compilation failure from missing header (`pr#12108 <http://github.com/ceph/ceph/pull/12108>`_, Adam C. Emerson)
+* core: denc: don't pass null instances into encoder fns (`issue#17636 <http://tracker.ceph.com/issues/17636>`_, `pr#11577 <http://github.com/ceph/ceph/pull/11577>`_, John Spray)
+* core: erasure-code: synchronize with upstream gf-complete (`issue#18092 <http://tracker.ceph.com/issues/18092>`_, `pr#12382 <http://github.com/ceph/ceph/pull/12382>`_, Loic Dachary)
+* core: FreeBSD/OSD.cc: add client_messenger to the avoid_ports set. (`pr#12463 <http://github.com/ceph/ceph/pull/12463>`_, Willem Jan Withagen)
+* core: include/object: pass "snapid_t&" to bound_encode() (`pr#11552 <http://github.com/ceph/ceph/pull/11552>`_, Kefu Chai)
+* core: kv/RocksDBStore: Don't update rocksdb perf_context if rocksdb_perf di… (`pr#12064 <http://github.com/ceph/ceph/pull/12064>`_, Jianpeng Ma)
+* core: librados-dev: install inline_memory.h (`issue#17654 <http://tracker.ceph.com/issues/17654>`_, `pr#11730 <http://github.com/ceph/ceph/pull/11730>`_, Josh Durgin)
+* core: messages/MForward: reencode forwarded message if target has differing features (`pr#11610 <http://github.com/ceph/ceph/pull/11610>`_, Sage Weil)
+* core,mgr: messages: fix out of range assertion (`pr#11345 <http://github.com/ceph/ceph/pull/11345>`_, John Spray)
+* core: mon,ceph-disk: add lockbox permissions to bootstrap-osd (`issue#17849 <http://tracker.ceph.com/issues/17849>`_, `pr#11996 <http://github.com/ceph/ceph/pull/11996>`_, Loic Dachary)
+* core:  mon: make it more clearly to debug for paxos state (`pr#12438 <http://github.com/ceph/ceph/pull/12438>`_, song baisen)
+* core: mon/OSDMonitor: encode full osdmaps with features all OSDs can understand (`pr#11284 <http://github.com/ceph/ceph/pull/11284>`_, Sage Weil)
+* core: mon/OSDMonitor: encode OSDMap::Incremental with same features as OSDMap (`pr#11596 <http://github.com/ceph/ceph/pull/11596>`_, Sage Weil)
+* core: mon/OSDMonitor: newly created osd should not be wrongly marked in (`pr#11795 <http://github.com/ceph/ceph/pull/11795>`_, runsisi)
+* core: mon/OSDMonitor: remove duplicate jewel/kraken flag warning (`pr#11775 <http://github.com/ceph/ceph/pull/11775>`_, Josh Durgin)
+* core: mon/PGMap: PGs can be stuck more than one thing (`issue#17515 <http://tracker.ceph.com/issues/17515>`_, `pr#11339 <http://github.com/ceph/ceph/pull/11339>`_, Sage Weil)
+* core: mon: print the num_pools and num_objects in 'ceph -s -f json/json-p… (`issue#17703 <http://tracker.ceph.com/issues/17703>`_, `pr#11654 <http://github.com/ceph/ceph/pull/11654>`_, huangjun)
+* core: msg/async/AsyncConnection: dispatch write handler on keepalive2 (`issue#17664 <http://tracker.ceph.com/issues/17664>`_, `pr#11601 <http://github.com/ceph/ceph/pull/11601>`_, Ilya Dryomov)
+* core: msg/async: DPDKStack as AsyncMessenger backend (`pr#10748 <http://github.com/ceph/ceph/pull/10748>`_, Haomai Wang)
+* core: msg/async/rdma: change log level: 0 -> 1 (`pr#12334 <http://github.com/ceph/ceph/pull/12334>`_, Avner BenHanoch)
+* core: msg/async/rdma: don't use more buffers than what device capabilities … (`pr#12263 <http://github.com/ceph/ceph/pull/12263>`_, Avner BenHanoch)
+* core: msg/async/rdma: ensure CephContext existed (`pr#12068 <http://github.com/ceph/ceph/pull/12068>`_, Haomai Wang)
+* core: msg/async/rdma: event polling thread can block on event (`pr#12270 <http://github.com/ceph/ceph/pull/12270>`_, Haomai Wang)
+* core: msg/async/rdma: fixup memory free (`pr#12236 <http://github.com/ceph/ceph/pull/12236>`_, gongchuang)
+* core: msg/async/rdma: set correct value to memory manager (`pr#12299 <http://github.com/ceph/ceph/pull/12299>`_, Adir Lev)
+* core: msg/async: set nonce before starting the workers (`pr#12390 <http://github.com/ceph/ceph/pull/12390>`_, Kefu Chai)
+* core: msg: make loopback Connection feature accurate all the time (`pr#11183 <http://github.com/ceph/ceph/pull/11183>`_, Sage Weil)
+* core: msg: seed random engine used for ms_type="random" (`pr#11880 <http://github.com/ceph/ceph/pull/11880>`_, Casey Bodley)
+* core: msg/simple/Pipe: avoid returning 0 on poll timeout (`issue#18184 <http://tracker.ceph.com/issues/18184>`_, `pr#12375 <http://github.com/ceph/ceph/pull/12375>`_, Sage Weil)
+* core: msg/simple/Pipe::stop_and_wait: unlock pipe_lock for stop_fast_dispatching() (`issue#18042 <http://tracker.ceph.com/issues/18042>`_, `pr#12307 <http://github.com/ceph/ceph/pull/12307>`_, Samuel Just)
+* core: msg/simple: save the errno in case being changed by subsequent codes (`pr#10297 <http://github.com/ceph/ceph/pull/10297>`_, Yan Jun)
+* core: osd/ECTransaction: only write out the hinfo if not delete (`issue#17983 <http://tracker.ceph.com/issues/17983>`_, `pr#12141 <http://github.com/ceph/ceph/pull/12141>`_, Samuel Just)
+* core: OSDMonitor: only reject MOSDBoot based on up_from if inst matches (`issue#17899 <http://tracker.ceph.com/issues/17899>`_, `pr#12003 <http://github.com/ceph/ceph/pull/12003>`_, Samuel Just)
+* core: osd,mon: require sortbitwise flag to upgrade beyond jewel (`pr#11772 <http://github.com/ceph/ceph/pull/11772>`_, Sage Weil)
+* core: osd/osd_types: fix the osd_stat_t::decode() (`pr#12235 <http://github.com/ceph/ceph/pull/12235>`_, Kefu Chai)
+* core: osd/PG: add "down" pg state (distinct from down+peering) (`pr#12289 <http://github.com/ceph/ceph/pull/12289>`_, Sage Weil)
+* core: osd/PGLog::proc_replica_log,merge_log: fix bound for last_update (`issue#18127 <http://tracker.ceph.com/issues/18127>`_, `pr#12340 <http://github.com/ceph/ceph/pull/12340>`_, Samuel Just)
+* core: osd/ReplicatedPG: do_update_log_missing: take the pg lock in the callback (`issue#17789 <http://tracker.ceph.com/issues/17789>`_, `pr#11754 <http://github.com/ceph/ceph/pull/11754>`_, Samuel Just)
+* core: osd/ReplicatedPG::record_write_error: don't leak orig_reply on cancel (`issue#18180 <http://tracker.ceph.com/issues/18180>`_, `pr#12450 <http://github.com/ceph/ceph/pull/12450>`_, Samuel Just)
+* core: os/filestore: avoid to get the wrong hardlink number. (`pr#11841 <http://github.com/ceph/ceph/pull/11841>`_, huangjun)
+* core: os/filestore/chain_xattr.h:uses ENODATA, so include compat.h (`pr#12279 <http://github.com/ceph/ceph/pull/12279>`_, Willem Jan Withagen)
+* core: os/filestore: Fix erroneous WARNING: max attr too small (`issue#17420 <http://tracker.ceph.com/issues/17420>`_, `pr#11246 <http://github.com/ceph/ceph/pull/11246>`_, Brad Hubbard)
+* core: os/FileStore: fix fiemap issue in xfs when #extents > 1364  (`pr#11554 <http://github.com/ceph/ceph/pull/11554>`_, Ning Yao)
+* core: os/filestore: fix journal logger (`pr#12099 <http://github.com/ceph/ceph/pull/12099>`_, Wei Jin)
+* core: os/filestore: fix potential result code overwriting (`pr#11491 <http://github.com/ceph/ceph/pull/11491>`_, xie xingguo)
+* core: os/filestore/HashIndex: fix list_by_hash_* termination on reaching end (`issue#17859 <http://tracker.ceph.com/issues/17859>`_, `pr#11898 <http://github.com/ceph/ceph/pull/11898>`_, Sage Weil)
+* core: os/ObjectStore: properly clear object map when replaying OP_REMOVE (`issue#17177 <http://tracker.ceph.com/issues/17177>`_, `pr#11388 <http://github.com/ceph/ceph/pull/11388>`_, Yan, Zheng)
+* core,performance: msg/async: ibverbs/rdma support (`pr#11531 <http://github.com/ceph/ceph/pull/11531>`_, Haomai Wang, Zhi Wang)
+* core,performance: osd/OSDMap.cc: remove unneeded use of count (`pr#11221 <http://github.com/ceph/ceph/pull/11221>`_, Michal Jarzabek)
+* core,performance: osd/PrimaryLogPG: don't truncate if we don't have to for WRITEFULL (`pr#12534 <http://github.com/ceph/ceph/pull/12534>`_, Samuel Just)
+* core,performance: os/fs/FS: optimize aio::pwritev which make caller provide length. (`pr#9062 <http://github.com/ceph/ceph/pull/9062>`_, Jianpeng Ma)
+* core,pybind,common: python-rados: implement new aio_execute (`pr#12140 <http://github.com/ceph/ceph/pull/12140>`_, Iain Buclaw)
+* core,rbd,bluestore,rgw,performance,cephfs: fast denc encoding (`pr#11027 <http://github.com/ceph/ceph/pull/11027>`_, Sage Weil)
+* core: remove spurious executable permissions on source code files (`pr#1061 <http://github.com/ceph/ceph/pull/1061>`_, Samuel Just)
+* core: ReplicatedPG::failed_push: release read lock on failure (`issue#17857 <http://tracker.ceph.com/issues/17857>`_, `pr#11914 <http://github.com/ceph/ceph/pull/11914>`_, Kefu Chai)
+* core: rocksdb: update to latest, and make it the default for the mons (`pr#11354 <http://github.com/ceph/ceph/pull/11354>`_, Sage Weil)
+* core: set dumpable flag after setuid (`issue#17650 <http://tracker.ceph.com/issues/17650>`_, `pr#11582 <http://github.com/ceph/ceph/pull/11582>`_, Patrick Donnelly)
+* core: systemd/ceph-disk: reduce ceph-disk flock contention (`issue#18049 <http://tracker.ceph.com/issues/18049>`_, `issue#13160 <http://tracker.ceph.com/issues/13160>`_, `pr#12200 <http://github.com/ceph/ceph/pull/12200>`_, David Disseldorp)
+* core: tchaikov (`issue#17713 <http://tracker.ceph.com/issues/17713>`_, `pr#11382 <http://github.com/ceph/ceph/pull/11382>`_, Haomai Wang)
+* core,tests: ceph_test_rados_api_tier: dump hitset that we fail to decode (`issue#17945 <http://tracker.ceph.com/issues/17945>`_, `pr#12057 <http://github.com/ceph/ceph/pull/12057>`_, Sage Weil)
+* core,tests: common osd: Improve scrub analysis, list-inconsistent-obj output and osd-scrub-repair test (`issue#18114 <http://tracker.ceph.com/issues/18114>`_, `pr#9613 <http://github.com/ceph/ceph/pull/9613>`_, Kefu Chai, David Zafman)
+* core,tests: test,cmake: turn unit.h into unit.cc to speed up compilation (`pr#12194 <http://github.com/ceph/ceph/pull/12194>`_, Kefu Chai)
+* core,tests: test/rados/list.cc: Memory leak in ceph_test_rados_api_list (`issue#18250 <http://tracker.ceph.com/issues/18250>`_, `pr#12479 <http://github.com/ceph/ceph/pull/12479>`_, Brad Hubbard)
+* core,tests: workunits/ceph-helpers.sh: Fixes for FreeBSD (`pr#12085 <http://github.com/ceph/ceph/pull/12085>`_, Willem Jan Withagen)
+* core,tools: Added append functionality to rados tool. (`pr#11036 <http://github.com/ceph/ceph/pull/11036>`_, Tomy Cheru)
+* core,tools: Tested-by: Huawen Ren <ren.huanwen@zte.com.cn> (`issue#17400 <http://tracker.ceph.com/issues/17400>`_, `pr#11276 <http://github.com/ceph/ceph/pull/11276>`_, Kefu Chai)
+* core,tools: vstart: decrease pool size if <3 OSDs (`pr#11528 <http://github.com/ceph/ceph/pull/11528>`_, John Spray)
+* crush: make counting of choose_tries consistent (`issue#17229 <http://tracker.ceph.com/issues/17229>`_, `pr#10993 <http://github.com/ceph/ceph/pull/10993>`_, Vicente Cheng)
+* crush: remove the crush_lock (`pr#11830 <http://github.com/ceph/ceph/pull/11830>`_, Adam C. Emerson)
+* crush: Silence coverity warnings for test/crush/crush.cc (`pr#12436 <http://github.com/ceph/ceph/pull/12436>`_, Brad Hubbard)
+* doc: Add doc about osd scrub {during recovery|chunk {min|max}| sleep} (`pr#12176 <http://github.com/ceph/ceph/pull/12176>`_, Paweł Sadowski)
+* doc: Add docs about looking up Monitors through DNS (`issue#14527 <http://tracker.ceph.com/issues/14527>`_, `pr#10852 <http://github.com/ceph/ceph/pull/10852>`_, Wido den Hollander)
+* doc: add docs for raw compression (`pr#12244 <http://github.com/ceph/ceph/pull/12244>`_, Casey Bodley)
+* doc: Add documentation about mon_allow_pool_delete before pool remove (`pr#11943 <http://github.com/ceph/ceph/pull/11943>`_, Wido den Hollander)
+* doc: add infernalis EOL date (`pr#11925 <http://github.com/ceph/ceph/pull/11925>`_, Ken Dreyer)
+* doc: adding changelog for v10.2.4 (`pr#12346 <http://github.com/ceph/ceph/pull/12346>`_, Abhishek Lekshmanan)
+* doc: Add MON docs about pool flags and pool removal config settings (`pr#10853 <http://github.com/ceph/ceph/pull/10853>`_, Wido den Hollander)
+* doc: add python-rgw doc (`pr#11859 <http://github.com/ceph/ceph/pull/11859>`_, Kefu Chai)
+* doc: change the osd_max_backfills default to 1 (`issue#17701 <http://tracker.ceph.com/issues/17701>`_, `pr#11658 <http://github.com/ceph/ceph/pull/11658>`_, huangjun)
+* doc: clarify file deletion from OSD restricted pool behaviour (`issue#17937 <http://tracker.ceph.com/issues/17937>`_, `pr#12054 <http://github.com/ceph/ceph/pull/12054>`_, David Disseldorp)
+* doc: clarify mds deactivate purpose (`pr#11957 <http://github.com/ceph/ceph/pull/11957>`_, Patrick Donnelly)
+* doc: common/Throttle: fix typo for BackoffThrottle (`pr#12129 <http://github.com/ceph/ceph/pull/12129>`_, Wei Jin)
+* doc: correcting the object name (`pr#12354 <http://github.com/ceph/ceph/pull/12354>`_, Uday Mullangi)
+* doc: Correcting the sample python tempurl generation script. (`issue#15258 <http://tracker.ceph.com/issues/15258>`_, `pr#8712 <http://github.com/ceph/ceph/pull/8712>`_, Diwakar Goel)
+* doc: Coverity and SCA fixes (`pr#7784 <http://github.com/ceph/ceph/pull/7784>`_, Danny Al-Gaaf)
+* doc: doc/dev/osd_internals: add pgpool.rst (`pr#12500 <http://github.com/ceph/ceph/pull/12500>`_, Brad Hubbard)
+* doc: doc/dev/perf: a few notes on perf (`pr#12168 <http://github.com/ceph/ceph/pull/12168>`_, Sage Weil)
+* doc: doc/dev/perf: fix dittography (`pr#12317 <http://github.com/ceph/ceph/pull/12317>`_, xie xingguo)
+* doc: doc/man: avoid file builtin to solve build error (`pr#11984 <http://github.com/ceph/ceph/pull/11984>`_, Patrick Donnelly)
+* doc: doc/rados/configuration/ms-ref.rst: document a few async msgr options (`pr#12126 <http://github.com/ceph/ceph/pull/12126>`_, Piotr Dałek)
+* doc: doc/rados/configuration/osd-config-ref.rst: document the fast mark down (`pr#12124 <http://github.com/ceph/ceph/pull/12124>`_, Piotr Dałek)
+* doc: doc/release-notes: kraken release notes (draft) (`pr#12338 <http://github.com/ceph/ceph/pull/12338>`_, Sage Weil)
+* doc: doc/releases: add links to kraken and v10.2.4 (`pr#12409 <http://github.com/ceph/ceph/pull/12409>`_, Kefu Chai)
+* doc: doc/start/hardware-recommentdations: cosmetic (`pr#10585 <http://github.com/ceph/ceph/pull/10585>`_, Zhao Junwang)
+* doc: Documentation syntax cleanup (`pr#11784 <http://github.com/ceph/ceph/pull/11784>`_, John Spray)
+* doc: document osd tell bench (`issue#5431 <http://tracker.ceph.com/issues/5431>`_, `pr#16 <http://github.com/ceph/ceph/pull/16>`_, Sage Weil)
+* doc: drop --journal-check from ceph-mds man page (`issue#17747 <http://tracker.ceph.com/issues/17747>`_, `pr#11912 <http://github.com/ceph/ceph/pull/11912>`_, Nathan Cutler)
+* doc: explain rgw_fcgi_socket_backlog in rgw/config-ref.rst (`pr#12548 <http://github.com/ceph/ceph/pull/12548>`_, liuchang0812)
+* doc: final additions to 11.1.0-rc release notes (`pr#12448 <http://github.com/ceph/ceph/pull/12448>`_, Abhishek Lekshmanan)
+* doc: Fix broken link for caps (`issue#17587 <http://tracker.ceph.com/issues/17587>`_, `pr#11546 <http://github.com/ceph/ceph/pull/11546>`_, Uday Mullangi)
+* doc: fix broken links (`issue#17587 <http://tracker.ceph.com/issues/17587>`_, `pr#11518 <http://github.com/ceph/ceph/pull/11518>`_, Uday Mullangi)
+* doc: fix dead link "Hardware Recommendations" (`pr#11379 <http://github.com/ceph/ceph/pull/11379>`_, Leo Zhang)
+* doc: fix dead link of "os-recommendations" in troubleshooting-osd (`pr#11454 <http://github.com/ceph/ceph/pull/11454>`_, Leo Zhang)
+* doc: Fixed mapping error in legacy mds command (`pr#11668 <http://github.com/ceph/ceph/pull/11668>`_, Malte Fiala)
+* doc: Fix for worker arguments to cephfs-data-scan tool (`pr#12360 <http://github.com/ceph/ceph/pull/12360>`_, Wido den Hollander)
+* doc: fix grammar/spelling in RGW sections (`pr#12329 <http://github.com/ceph/ceph/pull/12329>`_, Ken Dreyer)
+* doc: Fixing the broken hyperlinks by pointing to correct documentation. (`pr#11617 <http://github.com/ceph/ceph/pull/11617>`_, Uday Mullangi)
+* doc: fix librados example programs (`pr#11302 <http://github.com/ceph/ceph/pull/11302>`_, Alexey Sheplyakov)
+* doc: fix mgr literal block rST syntax (`pr#11652 <http://github.com/ceph/ceph/pull/11652>`_, Ken Dreyer)
+* doc: fix start development cluster operation in index.rst (`pr#11233 <http://github.com/ceph/ceph/pull/11233>`_, Leo Zhang)
+* doc: fix the script for rebuild monitor db (`pr#11962 <http://github.com/ceph/ceph/pull/11962>`_, Kefu Chai)
+* doc: fix typos (`pr#8751 <http://github.com/ceph/ceph/pull/8751>`_, Li Peng)
+* doc: Flag deprecated mds commands and omit deprecated mon commands in help output (`pr#11434 <http://github.com/ceph/ceph/pull/11434>`_, Patrick Donnelly)
+* doc: mailmap: change personal info (`pr#12310 <http://github.com/ceph/ceph/pull/12310>`_, Wei Jin)
+* doc: mailmap updates sept (`pr#10955 <http://github.com/ceph/ceph/pull/10955>`_, Yann Dupont)
+* doc: mds: fixup "mds bal mode" Description (`pr#12127 <http://github.com/ceph/ceph/pull/12127>`_, huanwen ren)
+* doc: mention corresponding libvirt section in nova.conf (`pr#12584 <http://github.com/ceph/ceph/pull/12584>`_, Marc Koderer)
+* doc: Modify documentation for mon_osd_down_out_interval (`pr#12408 <http://github.com/ceph/ceph/pull/12408>`_, Brad Hubbard)
+* doc: network-protocol typos (`pr#9837 <http://github.com/ceph/ceph/pull/9837>`_, Zhao Junwang)
+* doc: openstack glance mitaka uses show_multiple_locations (`pr#12020 <http://github.com/ceph/ceph/pull/12020>`_, Sébastien Han)
+* doc: README.FreeBSD: update to match the bimonthly FreeBSD status report (`pr#11442 <http://github.com/ceph/ceph/pull/11442>`_, Willem Jan Withagen)
+* doc: README: hint at where to look to diagnose test failures (`pr#11903 <http://github.com/ceph/ceph/pull/11903>`_, Dan Mick)
+* doc: reformat SubmittingPatches with more rst syntax (`pr#11570 <http://github.com/ceph/ceph/pull/11570>`_, Kefu Chai)
+* doc: release notes for 10.2.4 (`pr#12053 <http://github.com/ceph/ceph/pull/12053>`_, Abhishek Lekshmanan)
+* doc: release notes for 10.2.5 (`issue#18207 <http://tracker.ceph.com/issues/18207>`_, `pr#12410 <http://github.com/ceph/ceph/pull/12410>`_, Loic Dachary)
+* doc: release notes for 11.0.2 (`pr#11369 <http://github.com/ceph/ceph/pull/11369>`_, Abhishek Lekshmanan)
+* doc: Remove duplicate command for Ubuntu (`pr#12186 <http://github.com/ceph/ceph/pull/12186>`_, chrone)
+* doc: reviewed-by: John Wilkins <jowilkin@redhat.com> (`issue#17526 <http://tracker.ceph.com/issues/17526>`_, `pr#11352 <http://github.com/ceph/ceph/pull/11352>`_, Loic Dachary)
+* doc: reviewed-by: John Wilkins <jowilkin@redhat.com> (`issue#17665 <http://tracker.ceph.com/issues/17665>`_, `pr#11602 <http://github.com/ceph/ceph/pull/11602>`_, Jason Dillaman)
+* doc: rgw: fix a typo in S3 java api example (`pr#11762 <http://github.com/ceph/ceph/pull/11762>`_, Weibing Zhang)
+* doc: rm "type=rpm-md" from yum repositories (`pr#10248 <http://github.com/ceph/ceph/pull/10248>`_, Ken Dreyer)
+* doc: Small styling fix to mirror documentation (`pr#9714 <http://github.com/ceph/ceph/pull/9714>`_, Wido den Hollander)
+* doc: src/doc: fix class names in exports.txt (`pr#12000 <http://github.com/ceph/ceph/pull/12000>`_, John Spray)
+* doc: standardize EPEL instructions (`pr#11653 <http://github.com/ceph/ceph/pull/11653>`_, Ken Dreyer)
+* doc: update cinder key permissions for mitaka (`pr#12211 <http://github.com/ceph/ceph/pull/12211>`_, Sébastien Han)
+* doc: Update crush-map.rst, fix a typo mistake (`pr#11785 <http://github.com/ceph/ceph/pull/11785>`_, whu_liuchang)
+* doc: Update filestore xattr config documentation. (`pr#11826 <http://github.com/ceph/ceph/pull/11826>`_, Bartłomiej Święcki)
+* doc: Update install-ceph-gateway.rst (`pr#11432 <http://github.com/ceph/ceph/pull/11432>`_, Hans van den Bogert)
+* doc: Update keystone doc about v3 options (`pr#11392 <http://github.com/ceph/ceph/pull/11392>`_, Proskurin Kirill)
+* doc: Update layout.rst, move commands to CODE block (`pr#11987 <http://github.com/ceph/ceph/pull/11987>`_, liuchang0812)
+* doc: we can now run multiple MDS, so qualify warning (`issue#18040 <http://tracker.ceph.com/issues/18040>`_, `pr#12184 <http://github.com/ceph/ceph/pull/12184>`_, Nathan Cutler)
+* fs: add snapshot tests to mds thrashing (`pr#1073 <http://github.com/ceph/ceph/pull/1073>`_, Yan, Zheng)
+* fs: enable ceph-fuse permission checking for all pjd suites (`pr#1187 <http://github.com/ceph/ceph/pull/1187>`_, Greg Farnum)
+* fs: fix two frag_enable fragments (`issue#6143 <http://tracker.ceph.com/issues/6143>`_, `pr#656 <http://github.com/ceph/ceph/pull/656>`_, Sage Weil)
+* fs: fix up dd testing again (`issue#10861 <http://tracker.ceph.com/issues/10861>`_, `pr#373 <http://github.com/ceph/ceph/pull/373>`_, Greg Farnum)
+* fs: fuse_default_permissions = 0 for kernel build test (`pr#1109 <http://github.com/ceph/ceph/pull/1109>`_, Patrick Donnelly)
+* fs: Mantle: A Programmable Metadata Load Balancer (`pr#10887 <http://github.com/ceph/ceph/pull/10887>`_, Michael Sevilla)
+* fs: unify common parts of sub-suites (`issue#1737 <http://tracker.ceph.com/issues/1737>`_, `pr#1282 <http://github.com/ceph/ceph/pull/1282>`_, Patrick Donnelly)
+* librados: Add rados_aio_exec to the C API (`pr#11709 <http://github.com/ceph/ceph/pull/11709>`_, Iain Buclaw)
+* librados: add timeout to watch/notify (`pr#11378 <http://github.com/ceph/ceph/pull/11378>`_, Ryne Li)
+* librados: do not request osd ack if no completed completion is set (`pr#11204 <http://github.com/ceph/ceph/pull/11204>`_, Sage Weil)
+* librados:  For C-API, expose LIBRADOS_OPERATION_FULL_FORCE flag (`pr#9172 <http://github.com/ceph/ceph/pull/9172>`_, Jianpeng Ma)
+* librados: improvements async IO in librados and libradosstriper (`pr#10049 <http://github.com/ceph/ceph/pull/10049>`_, Sebastien Ponce)
+* librados: Memory leaks in object_list_begin and object_list_end (`issue#18252 <http://tracker.ceph.com/issues/18252>`_, `pr#12482 <http://github.com/ceph/ceph/pull/12482>`_, Brad Hubbard)
+* librados: postpone cct deletion (`pr#11659 <http://github.com/ceph/ceph/pull/11659>`_, Kefu Chai)
+* librados: remove new setxattr overload to avoid breaking the C++ ABI (`issue#18058 <http://tracker.ceph.com/issues/18058>`_, `pr#12206 <http://github.com/ceph/ceph/pull/12206>`_, Josh Durgin)
+* librados: remove unused bufferlist from rados_write_op_rmxattr (`pr#12030 <http://github.com/ceph/ceph/pull/12030>`_, Piotr Dałek)
+* librbd: add support for snapshot namespaces (`pr#11160 <http://github.com/ceph/ceph/pull/11160>`_, Victor Denisov)
+* librbd: API changes to support separate data pool (`pr#11353 <http://github.com/ceph/ceph/pull/11353>`_, Jason Dillaman)
+* librbd: batch object map updates during trim (`issue#17356 <http://tracker.ceph.com/issues/17356>`_, `pr#11510 <http://github.com/ceph/ceph/pull/11510>`_, Venky Shankar)
+* librbd: bug fixes for optional data pool support (`pr#11960 <http://github.com/ceph/ceph/pull/11960>`_, Venky Shankar)
+* librbd: cannot access non-primary image when mirroring force disabled  (`issue#16740 <http://tracker.ceph.com/issues/16740>`_, `issue#17588 <http://tracker.ceph.com/issues/17588>`_, `pr#11568 <http://github.com/ceph/ceph/pull/11568>`_, Jason Dillaman)
+* librbd: cls_rbd updates for separate data pool (`issue#17422 <http://tracker.ceph.com/issues/17422>`_, `pr#11327 <http://github.com/ceph/ceph/pull/11327>`_, Jason Dillaman)
+* librbd: default features should be negotiated with the OSD (`issue#17010 <http://tracker.ceph.com/issues/17010>`_, `pr#11808 <http://github.com/ceph/ceph/pull/11808>`_, Mykola Golub)
+* librbd: diffs to clone's first snapshot should include parent diffs (`issue#18068 <http://tracker.ceph.com/issues/18068>`_, `pr#12218 <http://github.com/ceph/ceph/pull/12218>`_, Jason Dillaman)
+* librbd: do not create empty object map object on image creation (`issue#17752 <http://tracker.ceph.com/issues/17752>`_, `pr#11704 <http://github.com/ceph/ceph/pull/11704>`_, Jason Dillaman)
+* librbd: enabling/disabling rbd feature should report missing dependency (`issue#16985 <http://tracker.ceph.com/issues/16985>`_, `pr#12238 <http://github.com/ceph/ceph/pull/12238>`_, Gaurav Kumar Garg)
+* librbd: ensure consistency groups will gracefully fail on older OSDs (`pr#11623 <http://github.com/ceph/ceph/pull/11623>`_, Jason Dillaman)
+* librbd: exclusive lock incorrectly initialized when switching to head revision (`issue#17618 <http://tracker.ceph.com/issues/17618>`_, `pr#11559 <http://github.com/ceph/ceph/pull/11559>`_, Jason Dillaman)
+* librbd: fix rollback if failed to disable mirroring for image (`pr#11260 <http://github.com/ceph/ceph/pull/11260>`_, runsisi)
+* librbd: ignore error when object map is already locked by current client (`issue#16179 <http://tracker.ceph.com/issues/16179>`_, `pr#12484 <http://github.com/ceph/ceph/pull/12484>`_, runsisi)
+* librbd: ignore notify errors on missing image header (`issue#17549 <http://tracker.ceph.com/issues/17549>`_, `pr#11395 <http://github.com/ceph/ceph/pull/11395>`_, Jason Dillaman)
+* librbd: keep rbd_default_features setting as bitmask (`issue#18247 <http://tracker.ceph.com/issues/18247>`_, `pr#12486 <http://github.com/ceph/ceph/pull/12486>`_, Jason Dillaman)
+* librbd: mark request as finished after failed refresh (`issue#17973 <http://tracker.ceph.com/issues/17973>`_, `pr#12160 <http://github.com/ceph/ceph/pull/12160>`_, Venky Shankar)
+* librbd: minor cleanup (`pr#12078 <http://github.com/ceph/ceph/pull/12078>`_, Dongsheng Yang)
+* librbd: new API method to force break a peer's exclusive lock (`issue#18429 <http://tracker.ceph.com/issues/18429>`_, `issue#16988 <http://tracker.ceph.com/issues/16988>`_, `issue#18327 <http://tracker.ceph.com/issues/18327>`_, `pr#12889 <http://github.com/ceph/ceph/pull/12889>`_, Jason Dillaman)
+* librbd: parse rbd_default_features config option as a string (`pr#11175 <http://github.com/ceph/ceph/pull/11175>`_, Alyona Kiseleva, Alexey Sheplyakov)
+* librbd: possible assert failure creating image when using data pool (`pr#11641 <http://github.com/ceph/ceph/pull/11641>`_, Venky Shankar)
+* librbd: proper check for get_data_pool compatibility (`issue#17791 <http://tracker.ceph.com/issues/17791>`_, `pr#11755 <http://github.com/ceph/ceph/pull/11755>`_, Mykola Golub)
+* librbd: properly order concurrent updates to the object map (`issue#16176 <http://tracker.ceph.com/issues/16176>`_, `pr#12420 <http://github.com/ceph/ceph/pull/12420>`_, Jason Dillaman)
+* librbd: release lock after demote (`issue#17880 <http://tracker.ceph.com/issues/17880>`_, `pr#11940 <http://github.com/ceph/ceph/pull/11940>`_, Mykola Golub)
+* librbd: remove consistency group rbd cli and API support (`issue#18231 <http://tracker.ceph.com/issues/18231>`_, `pr#12475 <http://github.com/ceph/ceph/pull/12475>`_, Jason Dillaman)
+* librbd: remove image header lock assertions (`issue#18244 <http://tracker.ceph.com/issues/18244>`_, `pr#12472 <http://github.com/ceph/ceph/pull/12472>`_, Jason Dillaman)
+* librbd: remove unused local variable (`pr#12388 <http://github.com/ceph/ceph/pull/12388>`_, Yunchuan Wen)
+* librbd: silence the unused variable warning (`pr#11678 <http://github.com/ceph/ceph/pull/11678>`_, Kefu Chai)
+* librbd: snap_get_limit compatibility check (`pr#11766 <http://github.com/ceph/ceph/pull/11766>`_, Mykola Golub)
+* librbd: update internals to use optional separate data pool (`pr#11356 <http://github.com/ceph/ceph/pull/11356>`_, Jason Dillaman)
+* librbd: use proper snapshot when computing diff parent overlap (`issue#18200 <http://tracker.ceph.com/issues/18200>`_, `pr#12396 <http://github.com/ceph/ceph/pull/12396>`_, Xiaoxi Chen)
+* log: optimize header file dependency (`pr#9768 <http://github.com/ceph/ceph/pull/9768>`_, Xiaowei Chen)
+* mds: add debug assertion for issue #17636 (`pr#11576 <http://github.com/ceph/ceph/pull/11576>`_, Yan, Zheng)
+* mds: add tests for mantle (programmable balancer) (`pr#1145 <http://github.com/ceph/ceph/pull/1145>`_, Michael Sevilla)
+* mds: check if down mds is known (`issue#17670 <http://tracker.ceph.com/issues/17670>`_, `pr#11611 <http://github.com/ceph/ceph/pull/11611>`_, Patrick Donnelly)
+* mds: don't access mdsmap from log submit thread (`issue#18047 <http://tracker.ceph.com/issues/18047>`_, `pr#12208 <http://github.com/ceph/ceph/pull/12208>`_, Yan, Zheng)
+* mds: don't maintain bloom filters in standby replay (`issue#16924 <http://tracker.ceph.com/issues/16924>`_, `pr#12133 <http://github.com/ceph/ceph/pull/12133>`_, John Spray)
+* mds: enable rmxattr on pool_namespace attrs  (`issue#17797 <http://tracker.ceph.com/issues/17797>`_, `pr#11783 <http://github.com/ceph/ceph/pull/11783>`_, John Spray)
+* mds: fix dropping events in standby replay (`issue#17954 <http://tracker.ceph.com/issues/17954>`_, `pr#12077 <http://github.com/ceph/ceph/pull/12077>`_, John Spray)
+* mds: fix EMetaBlob::fullbit xattr dump (`pr#11536 <http://github.com/ceph/ceph/pull/11536>`_, Sage Weil)
+* mds: fix false "failing to respond to cache pressure" warning (`pr#11373 <http://github.com/ceph/ceph/pull/11373>`_, Yan, Zheng)
+* mds: force client flush snap data before truncating objects (`issue#17193 <http://tracker.ceph.com/issues/17193>`_, `pr#11994 <http://github.com/ceph/ceph/pull/11994>`_, Yan, Zheng)
+* mds: handle bad standby_for_fscids in fsmap (`issue#17466 <http://tracker.ceph.com/issues/17466>`_, `pr#11281 <http://github.com/ceph/ceph/pull/11281>`_, John Spray)
+* mds: ignore 'session evict' when mds is replaying log (`issue#17801 <http://tracker.ceph.com/issues/17801>`_, `pr#11813 <http://github.com/ceph/ceph/pull/11813>`_, Yan, Zheng)
+* mds: include legacy client fsid in FSMap print (`pr#11283 <http://github.com/ceph/ceph/pull/11283>`_, John Spray)
+* mds: more deterministic timing on frag split/join (`issue#17853 <http://tracker.ceph.com/issues/17853>`_, `pr#12022 <http://github.com/ceph/ceph/pull/12022>`_, John Spray)
+* mds: more unique_pointer changes (`pr#11635 <http://github.com/ceph/ceph/pull/11635>`_, Patrick Donnelly)
+* mds: properly commit new dirfrag before splitting it (`issue#17990 <http://tracker.ceph.com/issues/17990>`_, `pr#12125 <http://github.com/ceph/ceph/pull/12125>`_, Yan, Zheng)
+* mds: release pool allocator memory after exceeding size limit (`issue#18225 <http://tracker.ceph.com/issues/18225>`_, `pr#12443 <http://github.com/ceph/ceph/pull/12443>`_, John Spray)
+* mds: remove duplicated log in handle_client_readdir (`pr#11806 <http://github.com/ceph/ceph/pull/11806>`_, Zhi Zhang)
+* mds: remove "--journal-check" help text (`issue#17747 <http://tracker.ceph.com/issues/17747>`_, `pr#11739 <http://github.com/ceph/ceph/pull/11739>`_, Nathan Cutler)
+* mds: remove unused EFragment::OP_ONESHOT (`pr#11887 <http://github.com/ceph/ceph/pull/11887>`_, John Spray)
+* mds: repair backtraces during scrub (`issue#17639 <http://tracker.ceph.com/issues/17639>`_, `pr#11578 <http://github.com/ceph/ceph/pull/11578>`_, John Spray)
+* mds: require MAY_SET_POOL to set pool_ns (`issue#17798 <http://tracker.ceph.com/issues/17798>`_, `pr#11789 <http://github.com/ceph/ceph/pull/11789>`_, John Spray)
+* mds: respawn using /proc/self/exe (`issue#17531 <http://tracker.ceph.com/issues/17531>`_, `pr#11362 <http://github.com/ceph/ceph/pull/11362>`_, Patrick Donnelly)
+* mds: revert "mds/Mutation: remove redundant _dump method" (`issue#17906 <http://tracker.ceph.com/issues/17906>`_, `pr#11985 <http://github.com/ceph/ceph/pull/11985>`_, Patrick Donnelly)
+* mds: use parse_filesystem in parse_role to handle exceptions and reuse parsing code (`issue#17518 <http://tracker.ceph.com/issues/17518>`_, `pr#11357 <http://github.com/ceph/ceph/pull/11357>`_, Patrick Donnelly)
+* mds: use projected path construction for access (`issue#17858 <http://tracker.ceph.com/issues/17858>`_, `pr#12063 <http://github.com/ceph/ceph/pull/12063>`_, Patrick Donnelly)
+* mds: use unique_ptr to simplify resource mgmt (`pr#11543 <http://github.com/ceph/ceph/pull/11543>`_, Patrick Donnelly)
+* mgr: doc/mgr: fix mgr how long to wait to failover (`pr#11550 <http://github.com/ceph/ceph/pull/11550>`_, huanwen ren)
+* mgr: init() return when connection daemons failed && add some err info (`pr#11424 <http://github.com/ceph/ceph/pull/11424>`_, huanwen ren)
+* mgr: misc minor changes (`issue#17455 <http://tracker.ceph.com/issues/17455>`_, `pr#11386 <http://github.com/ceph/ceph/pull/11386>`_, xie xingguo)
+* mgr: PyModules.cc: remove duplicated if condition for fs_map (`pr#11639 <http://github.com/ceph/ceph/pull/11639>`_, Weibing Zhang)
+* mgr: remove unnecessary C_StdFunction (`pr#11883 <http://github.com/ceph/ceph/pull/11883>`_, John Spray)
+* mon: add missing space in warning message (`pr#11361 <http://github.com/ceph/ceph/pull/11361>`_, Patrick Donnelly)
+* mon: clean legacy code (`pr#9643 <http://github.com/ceph/ceph/pull/9643>`_, Wei Jin)
+* mon: clear duplicated logic in MDSMonitor (`pr#11209 <http://github.com/ceph/ceph/pull/11209>`_, Zhi Zhang)
+* mon: Do not allow pools to be deleted by default (`pr#11665 <http://github.com/ceph/ceph/pull/11665>`_, Wido den Hollander)
+* mon: fix "OSDs marked OUT wrongly after monitor failover" (`issue#17719 <http://tracker.ceph.com/issues/17719>`_, `pr#11664 <http://github.com/ceph/ceph/pull/11664>`_, Dong Wu)
+* mon: Forbidden copy and assignment function in monoprequest (`pr#9513 <http://github.com/ceph/ceph/pull/9513>`_, song baisen)
+* mon: have mon-specific features & rework internal monmap structures (`pr#10907 <http://github.com/ceph/ceph/pull/10907>`_, Joao Eduardo Luis)
+* mon: if crushtool config is empty use internal crush test (`pr#11765 <http://github.com/ceph/ceph/pull/11765>`_, Bassam Tabbara)
+* mon: make MDSMonitor tolerant of slow mon elections (`issue#17308 <http://tracker.ceph.com/issues/17308>`_, `pr#11167 <http://github.com/ceph/ceph/pull/11167>`_, John Spray)
+* mon: MonmapMonitor: return success when monitor will be removed (`issue#17725 <http://tracker.ceph.com/issues/17725>`_, `pr#11747 <http://github.com/ceph/ceph/pull/11747>`_, Joao Eduardo Luis)
+* mon: move case CEPH_MSG_POOLOP to OSDs group (`pr#11848 <http://github.com/ceph/ceph/pull/11848>`_, Javeme)
+* mon: osdmap's epoch should be more than 0 (`pr#9859 <http://github.com/ceph/ceph/pull/9859>`_, Na Xie)
+* mon: OSDMonitor: fix the check error of pg creating (`issue#17169 <http://tracker.ceph.com/issues/17169>`_, `pr#10916 <http://github.com/ceph/ceph/pull/10916>`_, DesmondS)
+* mon:  paxos add the timeout function when peon recovery (`pr#10359 <http://github.com/ceph/ceph/pull/10359>`_, song baisen)
+* mon: preserve osd weight when marking osd out, then in (`pr#11293 <http://github.com/ceph/ceph/pull/11293>`_, Sage Weil)
+* mon: prevent post-jewel OSDs from booting if require_jewel_osds is not set (`pr#11498 <http://github.com/ceph/ceph/pull/11498>`_, Sage Weil)
+* mon: remove ceph-create-keys from mon startup (`issue#16036 <http://tracker.ceph.com/issues/16036>`_, `pr#9345 <http://github.com/ceph/ceph/pull/9345>`_, Owen Synge)
+* mon:  remove the redudant jugement in LogMonitor tick function (`pr#10474 <http://github.com/ceph/ceph/pull/10474>`_, song baisen)
+* mon: remove utime_t param in _dump (`pr#12029 <http://github.com/ceph/ceph/pull/12029>`_, Patrick Donnelly)
+* mon: send updated monmap to its subscribers (`issue#17558 <http://tracker.ceph.com/issues/17558>`_, `pr#11456 <http://github.com/ceph/ceph/pull/11456>`_, Kefu Chai)
+* mon: small change on the HealthMonitor start_epoch  function (`pr#10296 <http://github.com/ceph/ceph/pull/10296>`_, songbaisen)
+* mon: support for building without leveldb + mon mkfs bug fix (`pr#11800 <http://github.com/ceph/ceph/pull/11800>`_, Bassam Tabbara)
+* osd: add a pg _fastinfo attribute to reduce per-io metadata updates (`pr#11213 <http://github.com/ceph/ceph/pull/11213>`_, Sage Weil)
+* osd: Add config option to disable new scrubs during recovery (`issue#17866 <http://tracker.ceph.com/issues/17866>`_, `pr#11874 <http://github.com/ceph/ceph/pull/11874>`_, Wido den Hollander)
+* osd: a few fast dispatch optimizations (`pr#12052 <http://github.com/ceph/ceph/pull/12052>`_, Sage Weil)
+* osd: cleanup C_CompleteSplits::finish() (`pr#12094 <http://github.com/ceph/ceph/pull/12094>`_, Jie Wang)
+* osd: clean up PeeringWQ::_dequeue(), remove unnecessary variable (`pr#12117 <http://github.com/ceph/ceph/pull/12117>`_, Jie Wang)
+* osd: clean up process_peering_events (`pr#12009 <http://github.com/ceph/ceph/pull/12009>`_, Jie Wang)
+* osdc/Objecter: resend pg commands on interval change (`issue#18358 <http://tracker.ceph.com/issues/18358>`_, `pr#12910 <http://github.com/ceph/ceph/pull/12910>`_, Samuel Just)
+* osd: condition OSDMap encoding on features (`pr#12166 <http://github.com/ceph/ceph/pull/12166>`_, Sage Weil)
+* osd: default osd_scrub_during_recovery=false (`pr#12402 <http://github.com/ceph/ceph/pull/12402>`_, Sage Weil)
+* osd: do not open pgs when the pg is not in pg_map (`issue#17806 <http://tracker.ceph.com/issues/17806>`_, `pr#11803 <http://github.com/ceph/ceph/pull/11803>`_, Xinze Chi)
+* osd: drop stray debug message (`pr#11296 <http://github.com/ceph/ceph/pull/11296>`_, Sage Weil)
+* osd: EC Overwrites (`issue#17668 <http://tracker.ceph.com/issues/17668>`_, `pr#11701 <http://github.com/ceph/ceph/pull/11701>`_, Tomy Cheru, Samuel Just)
+* osd: enhance logging for osd network error (`pr#12458 <http://github.com/ceph/ceph/pull/12458>`_, liuchang0812)
+* osd: fix CEPH_OSD_FLAG_RWORDERED (`pr#12603 <http://github.com/ceph/ceph/pull/12603>`_, Sage Weil)
+* osd: fix duplicated id of incompat feature "fastinfo" (`pr#11588 <http://github.com/ceph/ceph/pull/11588>`_, xie xingguo)
+* osd: fix ec scrub errors (`issue#17999 <http://tracker.ceph.com/issues/17999>`_, `pr#12306 <http://github.com/ceph/ceph/pull/12306>`_, Samuel Just)
+* osd: fixes to make rbd on ec work (`pr#12305 <http://github.com/ceph/ceph/pull/12305>`_, Samuel Just)
+* osd: Fix map gaps again (bug 15943) (`issue#15943 <http://tracker.ceph.com/issues/15943>`_, `pr#12571 <http://github.com/ceph/ceph/pull/12571>`_, Samuel Just)
+* osd: fix memory leak from EC write workload (`issue#18093 <http://tracker.ceph.com/issues/18093>`_, `pr#12256 <http://github.com/ceph/ceph/pull/12256>`_, Sage Weil)
+* osd: fix rados write op hang (`pr#11143 <http://github.com/ceph/ceph/pull/11143>`_, Yunchuan Wen)
+* osd: Fix read error propogation in ECBackend (`issue#17966 <http://tracker.ceph.com/issues/17966>`_, `pr#12142 <http://github.com/ceph/ceph/pull/12142>`_, Samuel Just)
+* osd: fix scrub boundary to not include a SnapSet (`pr#11255 <http://github.com/ceph/ceph/pull/11255>`_, Samuel Just)
+* osd: fix signed/unsigned comparison warning (`pr#12400 <http://github.com/ceph/ceph/pull/12400>`_, Greg Farnum)
+* osd: fix typo in PG::clear_primary_state (`pr#11513 <http://github.com/ceph/ceph/pull/11513>`_, Brad Hubbard)
+* osd: Fix typos in PG::find_best_info (`pr#11515 <http://github.com/ceph/ceph/pull/11515>`_, Brad Hubbard)
+* osd: fix typos in "struct OSDOp" comments (`pr#12350 <http://github.com/ceph/ceph/pull/12350>`_, Chanyoung Park)
+* osd: Flush Journal on shutdown (`pr#11249 <http://github.com/ceph/ceph/pull/11249>`_, Wido den Hollander)
+* osd: force watch PING to be write ordered (`issue#18310 <http://tracker.ceph.com/issues/18310>`_, `pr#12590 <http://github.com/ceph/ceph/pull/12590>`_, Samuel Just)
+* osd: handle EC recovery read errors (`issue#13937 <http://tracker.ceph.com/issues/13937>`_, `pr#9304 <http://github.com/ceph/ceph/pull/9304>`_, David Zafman)
+* osd: heartbeat peers need to be updated when a new OSD added into an existed cluster (`issue#18004 <http://tracker.ceph.com/issues/18004>`_, `pr#12069 <http://github.com/ceph/ceph/pull/12069>`_, Pan Liu)
+* osd: Increase priority for inactive PGs backfill (`pr#12389 <http://github.com/ceph/ceph/pull/12389>`_, Bartłomiej Święcki)
+* osd: kill PG_STATE_SPLITTING (`pr#11824 <http://github.com/ceph/ceph/pull/11824>`_, xie xingguo)
+* osd: mark queued flag for op (`pr#12352 <http://github.com/ceph/ceph/pull/12352>`_, Yunchuan Wen)
+* osd: osdc: pass a string reference type to "osdmap->lookup_pg_pool_name" (`pr#12219 <http://github.com/ceph/ceph/pull/12219>`_, Leo Zhang)
+* osd: osd/OSDMonitor: accept 'osd pool set ...' value as string (`pr#911 <http://github.com/ceph/ceph/pull/911>`_, David Zafman)
+* osd: PGLog: initialize writeout_from in PGLog constructor (`issue#12973 <http://tracker.ceph.com/issues/12973>`_, `pr#558 <http://github.com/ceph/ceph/pull/558>`_, Sage Weil)
+* osd/PrimaryLogPG: don't update digests for objects with mismatched names (`issue#18409 <http://tracker.ceph.com/issues/18409>`_, `pr#12803 <http://github.com/ceph/ceph/pull/12803>`_, Samuel Just)
+* osd/PrimaryLogPG::failed_push: update missing as well (`issue#18165 <http://tracker.ceph.com/issues/18165>`_, `pr#12911 <http://github.com/ceph/ceph/pull/12911>`_, Samuel Just)
+* osd: print log when osd want to kill self (`pr#9288 <http://github.com/ceph/ceph/pull/9288>`_, Haomai Wang)
+* osd: Remove extra call to reg_next_scrub() during splits (`issue#16474 <http://tracker.ceph.com/issues/16474>`_, `pr#11206 <http://github.com/ceph/ceph/pull/11206>`_, David Zafman)
+* osd: remove redudant call of heartbeat_check (`pr#12130 <http://github.com/ceph/ceph/pull/12130>`_, Pan Liu)
+* osd: remove the lock heartbeat_update_lock, and change heatbeat_need\_… (`pr#12461 <http://github.com/ceph/ceph/pull/12461>`_, Pan Liu)
+* osd:  remove the redundant clear method in consume_map function (`pr#10553 <http://github.com/ceph/ceph/pull/10553>`_, song baisen)
+* osd: Remove unused '_lsb_release_' declarations (`pr#11364 <http://github.com/ceph/ceph/pull/11364>`_, Brad Hubbard)
+* osd: replace hb_out and hb_in with a single hb_peers (`issue#18057 <http://tracker.ceph.com/issues/18057>`_, `pr#12178 <http://github.com/ceph/ceph/pull/12178>`_, Pan Liu)
+* osd: ReplicatedPG: don't bless C_OSD_SendMessageOnConn (`issue#13304 <http://tracker.ceph.com/issues/13304>`_, `pr#669 <http://github.com/ceph/ceph/pull/669>`_, Jason Dillaman)
+* osd: set server-side limits on omap get operations (`pr#12059 <http://github.com/ceph/ceph/pull/12059>`_, Sage Weil)
+* osd: When deep-scrub errors present upgrade regular scrubs (`pr#12268 <http://github.com/ceph/ceph/pull/12268>`_, David Zafman)
+* performance,bluestore: kv/MemDB: making memdb code adapt to generic maps (`pr#11436 <http://github.com/ceph/ceph/pull/11436>`_, Ramesh Chander)
+* performance,bluestore: os/bluestore: allow default to buffered write (`pr#11301 <http://github.com/ceph/ceph/pull/11301>`_, Sage Weil)
+* performance,bluestore: os/bluestore: bluestore_cache_meta_ratio = .5 (`pr#11919 <http://github.com/ceph/ceph/pull/11919>`_, Sage Weil)
+* performance,bluestore: os/bluestore: reduce Onode in-memory footprint (`pr#12568 <http://github.com/ceph/ceph/pull/12568>`_, Igor Fedotov)
+* performance,bluestore: os/bluestore: refactor bluestore_sync_submit_transaction (`pr#11537 <http://github.com/ceph/ceph/pull/11537>`_, Sage Weil)
+* performance,bluestore: os/bluestore: speed up omap-key generation for same onode(the read paths) (`pr#11894 <http://github.com/ceph/ceph/pull/11894>`_, xie xingguo)
+* performance,bluestore: os/bluestore: speedup the performance of multi-replication flow by switc… (`pr#11844 <http://github.com/ceph/ceph/pull/11844>`_, Pan Liu)
+* performance,cephfs: Fix long stalls when calling ceph_fsync() (`issue#17563 <http://tracker.ceph.com/issues/17563>`_, `pr#11710 <http://github.com/ceph/ceph/pull/11710>`_, Jeff Layton)
+* performance,cleanup: Context: std::move the callback param in FunctionContext's ctor (`pr#11892 <http://github.com/ceph/ceph/pull/11892>`_, Kefu Chai)
+* performance,cleanup: osd/PG.h: move shared ptr instead of copying it (`pr#11154 <http://github.com/ceph/ceph/pull/11154>`_, Michal Jarzabek)
+* performance,common: common/config_opts.h: Optimized RocksDB WAL settings. (`pr#11530 <http://github.com/ceph/ceph/pull/11530>`_, Mark Nelson)
+* performance,common: osd/OSDMap: improve the performance of pg_to_acting_osds (`pr#12190 <http://github.com/ceph/ceph/pull/12190>`_, Pan Liu)
+* performance: msg/async: set ms_async_send_inline to false to improve small randread iops (`pr#11521 <http://github.com/ceph/ceph/pull/11521>`_, Mark Nelson)
+* performance,tools: rados: add hints to rados bench (`pr#12169 <http://github.com/ceph/ceph/pull/12169>`_, Sage Weil)
+* pybind: avoid "exception 'int' object is not iterable" (`pr#11532 <http://github.com/ceph/ceph/pull/11532>`_, Javeme)
+* pybind,cephfs: ceph_volume_client: fix recovery from partial auth update (`issue#17216 <http://tracker.ceph.com/issues/17216>`_, `pr#11304 <http://github.com/ceph/ceph/pull/11304>`_, Ramana Raja)
+* pybind,cephfs: ceph_volume_client: set an existing auth ID's default mon caps  (`issue#17800 <http://tracker.ceph.com/issues/17800>`_, `pr#11917 <http://github.com/ceph/ceph/pull/11917>`_, Ramana Raja)
+* pybind: ceph-rest-api: understand the new style entity_addr_t representation (`issue#17742 <http://tracker.ceph.com/issues/17742>`_, `pr#11686 <http://github.com/ceph/ceph/pull/11686>`_, Kefu Chai)
+* pybind: clean up mgr stuff for flake8 (`pr#11314 <http://github.com/ceph/ceph/pull/11314>`_, John Spray)
+* pybind: fix build failure of rgwfile binding (`pr#11825 <http://github.com/ceph/ceph/pull/11825>`_, Kefu Chai)
+* pybind: pybind/rados: add missing "length" requires for aio_execute() (`pr#12439 <http://github.com/ceph/ceph/pull/12439>`_, Kefu Chai)
+* pybind: pybind/rados: Add @requires for all aio methods (`pr#12327 <http://github.com/ceph/ceph/pull/12327>`_, Iain Buclaw)
+* qa: fixed distros links (`pr#12773 <http://github.com/ceph/ceph/pull/12773>`_, Yuri Weinstein)
+* qa: Fixed link to centos distro (`pr#12768 <http://github.com/ceph/ceph/pull/12768>`_, Yuri Weinstein)
+* qa/suites: switch from centos 7.2 to 7.x (`pr#12632 <http://github.com/ceph/ceph/pull/12632>`_, Sage Weil)
+* qa/tasks/peer: update task based on current peering behavior (`issue#18330 <http://tracker.ceph.com/issues/18330>`_, `pr#12614 <http://github.com/ceph/ceph/pull/12614>`_, Sage Weil)
+* qa/tasks/workunit: clear clone dir before retrying checkout (`issue#18336 <http://tracker.ceph.com/issues/18336>`_, `pr#12630 <http://github.com/ceph/ceph/pull/12630>`_, Sage Weil)
+* qa: update Ubuntu image url after ceph.com refactor (`issue#18542 <http://tracker.ceph.com/issues/18542>`_, `pr#12960 <http://github.com/ceph/ceph/pull/12960>`_, Jason Dillaman)
+* qa/workunits/rbd/test_lock_fence.sh fails (`issue#18388 <http://tracker.ceph.com/issues/18388>`_, `pr#12752 <http://github.com/ceph/ceph/pull/12752>`_, Nathan Cutler)
+* rbd: added rbd-nbd fsx test case (`pr#1049 <http://github.com/ceph/ceph/pull/1049>`_, Jason Dillaman)
+* rbd: add fsx journal replay test case (`pr#821 <http://github.com/ceph/ceph/pull/821>`_, Jason Dillaman)
+* rbd: add singleton to assert no rbdmap regression (`issue#14984 <http://tracker.ceph.com/issues/14984>`_, `pr#902 <http://github.com/ceph/ceph/pull/902>`_, Nathan Cutler)
+* rbd: add some missing workunits (`pr#870 <http://github.com/ceph/ceph/pull/870>`_, Josh Durgin)
+* rbd: add support for separate image data pool (`issue#17424 <http://tracker.ceph.com/issues/17424>`_, `pr#11355 <http://github.com/ceph/ceph/pull/11355>`_, Jason Dillaman)
+* rbd: expose rbd unmap options (`issue#17554 <http://tracker.ceph.com/issues/17554>`_, `pr#11370 <http://github.com/ceph/ceph/pull/11370>`_, Ilya Dryomov)
+* rbd: fix json formatting for image and journal status output (`issue#18261 <http://tracker.ceph.com/issues/18261>`_, `pr#12525 <http://github.com/ceph/ceph/pull/12525>`_, Mykola Golub)
+* rbd: fix parsing of group and image specific pools (`pr#11632 <http://github.com/ceph/ceph/pull/11632>`_, Victor Denisov)
+* rbd: journal: do not prematurely flag object recorder as closed (`issue#17590 <http://tracker.ceph.com/issues/17590>`_, `pr#11520 <http://github.com/ceph/ceph/pull/11520>`_, Jason Dillaman)
+* rbd: krbd: kernel client expects ip[:port], not an entity_addr_t (`pr#11902 <http://github.com/ceph/ceph/pull/11902>`_, Ilya Dryomov)
+* rbd: --max_part and --nbds_max options for nbd map (`issue#18186 <http://tracker.ceph.com/issues/18186>`_, `pr#12379 <http://github.com/ceph/ceph/pull/12379>`_, Pan Liu)
+* rbd: move nbd test workload to separate client host from OSDs (`pr#1170 <http://github.com/ceph/ceph/pull/1170>`_, Jason Dillaman)
+* rbd: provision volumes to format as XFS (`issue#6693 <http://tracker.ceph.com/issues/6693>`_, `pr#1028 <http://github.com/ceph/ceph/pull/1028>`_, Loic Dachary)
+* rbd: rbd-mirror: fix sparse read optimization in image sync (`issue#18146 <http://tracker.ceph.com/issues/18146>`_, `pr#12368 <http://github.com/ceph/ceph/pull/12368>`_, Mykola Golub)
+* rbd: rbd-mirror HA: move librbd::image_watcher::Notifier to librbd::object_watcher (`issue#17017 <http://tracker.ceph.com/issues/17017>`_, `pr#11290 <http://github.com/ceph/ceph/pull/11290>`_, Mykola Golub)
+* rbd: rbd-mirror: recovering after split-brain (`issue#16991 <http://tracker.ceph.com/issues/16991>`_, `issue#18051 <http://tracker.ceph.com/issues/18051>`_, `pr#12212 <http://github.com/ceph/ceph/pull/12212>`_, Mykola Golub)
+* rbd: rbd-mirror: snap protect of non-layered image results in split-brain (`issue#16962 <http://tracker.ceph.com/issues/16962>`_, `pr#11744 <http://github.com/ceph/ceph/pull/11744>`_, Mykola Golub)
+* rbd: rbd-nbd: disallow mapping images >2TB in size (`issue#17219 <http://tracker.ceph.com/issues/17219>`_, `pr#11741 <http://github.com/ceph/ceph/pull/11741>`_, Mykola Golub)
+* rbd: rbd-nbd: invalid error code for "failed to read nbd request" messages (`issue#18242 <http://tracker.ceph.com/issues/18242>`_, `pr#12483 <http://github.com/ceph/ceph/pull/12483>`_, Mykola Golub)
+* rbd: rbd-nbd: restart parent process logger after forking (`issue#18070 <http://tracker.ceph.com/issues/18070>`_, `pr#12222 <http://github.com/ceph/ceph/pull/12222>`_, Jason Dillaman)
+* rbd: rbd-nbd: support disabling auto-exclusive lock transition logic (`issue#17488 <http://tracker.ceph.com/issues/17488>`_, `pr#11438 <http://github.com/ceph/ceph/pull/11438>`_, Mykola Golub)
+* rbd: rbd-nbd: support partition for rbd-nbd mapped raw block device (`issue#18115 <http://tracker.ceph.com/issues/18115>`_, `pr#12259 <http://github.com/ceph/ceph/pull/12259>`_, Pan Liu)
+* rbd: tests with rbd_skip_partial_discard option enabled (`pr#1077 <http://github.com/ceph/ceph/pull/1077>`_, Mykola Golub)
+* rbd,tools: rbd : make option --stripe-unit w/ B/K/M work (`pr#12407 <http://github.com/ceph/ceph/pull/12407>`_, Jianpeng Ma)
+* rbd: updated tests to use new rbd default feature set (`pr#842 <http://github.com/ceph/ceph/pull/842>`_, Jason Dillaman)
+* rbd: use snap_remove implementation from internal (`pr#12035 <http://github.com/ceph/ceph/pull/12035>`_, Victor Denisov)
+* rgw: add default zone name (`issue#7009 <http://tracker.ceph.com/issues/7009>`_, `pr#954 <http://github.com/ceph/ceph/pull/954>`_, Orit Wasserman)
+* rgw: add documentation for upgrading with rgw_region_root_pool (`pr#12138 <http://github.com/ceph/ceph/pull/12138>`_, Orit Wasserman)
+* rgw: add option to log custom HTTP headers (rgw_log_http_headers) (`pr#7639 <http://github.com/ceph/ceph/pull/7639>`_, Matt Benjamin)
+* rgw: add recovery procedure for upgrade to older version of jewel (`issue#17820 <http://tracker.ceph.com/issues/17820>`_, `pr#11827 <http://github.com/ceph/ceph/pull/11827>`_, Orit Wasserman)
+* rgw: add rgw_compression_type=random for teuthology testing (`pr#11901 <http://github.com/ceph/ceph/pull/11901>`_, Casey Bodley)
+* rgw: add sleep to let the sync agent init (`pr#1136 <http://github.com/ceph/ceph/pull/1136>`_, Orit Wasserman)
+* rgw: add suport for creating S3 type subuser of admin rest api (`issue#16682 <http://tracker.ceph.com/issues/16682>`_, `pr#10325 <http://github.com/ceph/ceph/pull/10325>`_, snakeAngel2015)
+* rgw: add support for the prefix parameter in account listing of Swift API (`issue#17931 <http://tracker.ceph.com/issues/17931>`_, `pr#12047 <http://github.com/ceph/ceph/pull/12047>`_, Radoslaw Zarzynski)
+* rgw: allow fastcgi idle timeout to be adjusted (`pr#230 <http://github.com/ceph/ceph/pull/230>`_, Sage Weil)
+* rgw: also approve, passed teuthology (many false positives in several classes) (`issue#17985 <http://tracker.ceph.com/issues/17985>`_, `pr#12224 <http://github.com/ceph/ceph/pull/12224>`_, Yehuda Sadeh, Sage Weil)
+* rgw: Anonymous users shouldn't be able to access requester pays buckets. (`issue#17175 <http://tracker.ceph.com/issues/17175>`_, `pr#11719 <http://github.com/ceph/ceph/pull/11719>`_, Zhang Shaowen)
+* rgw: aws4: add presigned url bugfix in runtime (`issue#16463 <http://tracker.ceph.com/issues/16463>`_, `pr#10160 <http://github.com/ceph/ceph/pull/10160>`_, Javier M. Mellid)
+* rgw: bucket resharding (`issue#17550 <http://tracker.ceph.com/issues/17550>`_, `pr#11230 <http://github.com/ceph/ceph/pull/11230>`_, Yehuda Sadeh)
+* rgw:bugfix for deleting objects name beginning and ending with underscores of one bucket using POST method of AWS's js sdk. (`issue#17888 <http://tracker.ceph.com/issues/17888>`_, `pr#11982 <http://github.com/ceph/ceph/pull/11982>`_, root)
+* rgw: Class member cookie is not initialized correctly in some coroutine's constructor. (`pr#11673 <http://github.com/ceph/ceph/pull/11673>`_, Zhang Shaowen)
+* rgw: clean up RGWShardedOmapCRManager on early return (`issue#17571 <http://tracker.ceph.com/issues/17571>`_, `pr#11505 <http://github.com/ceph/ceph/pull/11505>`_, Casey Bodley)
+* rgw: clear data_sync_cr if RGWDataSyncControlCR fails (`issue#17569 <http://tracker.ceph.com/issues/17569>`_, `pr#11506 <http://github.com/ceph/ceph/pull/11506>`_, Casey Bodley)
+* rgw: compilation of the ASIO front-end is enabled by default. (`pr#12073 <http://github.com/ceph/ceph/pull/12073>`_, Radoslaw Zarzynski)
+* rgw: compression uses optional::emplace instead of in-place factories (`pr#12021 <http://github.com/ceph/ceph/pull/12021>`_, Radoslaw Zarzynski)
+* rgw: conform to the standard usage of string::find (`pr#10086 <http://github.com/ceph/ceph/pull/10086>`_, Yan Jun)
+* rgw: data_extra_pool is unique per zone (`issue#17025 <http://tracker.ceph.com/issues/17025>`_, `pr#1119 <http://github.com/ceph/ceph/pull/1119>`_, Orit Wasserman)
+* rgw: delete entries_index in RGWFetchAllMetaCR (`issue#17812 <http://tracker.ceph.com/issues/17812>`_, `pr#11816 <http://github.com/ceph/ceph/pull/11816>`_, Casey Bodley)
+* rgw: do not abort when accept a CORS request with short origin (`pr#12381 <http://github.com/ceph/ceph/pull/12381>`_, LiuYang)
+* rgw: do not enable both tcp and uds for fastcgi (`issue#5797 <http://tracker.ceph.com/issues/5797>`_, `pr#479 <http://github.com/ceph/ceph/pull/479>`_, Andrew Schoen)
+* rgw: don't error out on empty owner when setting acls (`issue#6892 <http://tracker.ceph.com/issues/6892>`_, `pr#877 <http://github.com/ceph/ceph/pull/877>`_, Loic Dachary, Nathan Cutler)
+* rgw: Don't loop forever when reading data from 0 sized segment. (`issue#17692 <http://tracker.ceph.com/issues/17692>`_, `pr#11567 <http://github.com/ceph/ceph/pull/11567>`_, Marcus Watts)
+* rgw: dont set CURLOPT_UPLOAD for GET requests (`issue#17822 <http://tracker.ceph.com/issues/17822>`_, `pr#12105 <http://github.com/ceph/ceph/pull/12105>`_, Casey Bodley)
+* rgw: don't store empty chains in gc (`issue#17897 <http://tracker.ceph.com/issues/17897>`_, `pr#11969 <http://github.com/ceph/ceph/pull/11969>`_, Yehuda Sadeh)
+* rgw: do quota tests on ubuntu (`issue#6382 <http://tracker.ceph.com/issues/6382>`_, `pr#635 <http://github.com/ceph/ceph/pull/635>`_, Sage Weil)
+* rgw: dump objects in RGWBucket::check_object_index() (`issue#14589 <http://tracker.ceph.com/issues/14589>`_, `pr#11324 <http://github.com/ceph/ceph/pull/11324>`_, Yehuda Sadeh)
+* rgw: dump remaining coroutines when cr deadlock is detected (`pr#11580 <http://github.com/ceph/ceph/pull/11580>`_, Casey Bodley)
+* rgw: extract host name from host:port string (`issue#17788 <http://tracker.ceph.com/issues/17788>`_, `pr#11751 <http://github.com/ceph/ceph/pull/11751>`_, Yehuda Sadeh)
+* rgw: Fixed problem with PUT with x-amz-copy-source when source object is compressed. (`pr#12253 <http://github.com/ceph/ceph/pull/12253>`_, Adam Kupczyk)
+* rgw: fixes for virtual hosting of buckets (`issue#17440 <http://tracker.ceph.com/issues/17440>`_, `issue#15975 <http://tracker.ceph.com/issues/15975>`_, `issue#17136 <http://tracker.ceph.com/issues/17136>`_, `pr#11280 <http://github.com/ceph/ceph/pull/11280>`_, Casey Bodley, Robin H. Johnson)
+* rgw: fix etag in multipart complete (`issue#17794 <http://tracker.ceph.com/issues/17794>`_, `issue#6830 <http://tracker.ceph.com/issues/6830>`_, `issue#16129 <http://tracker.ceph.com/issues/16129>`_, `issue#17872 <http://tracker.ceph.com/issues/17872>`_, `pr#1269 <http://github.com/ceph/ceph/pull/1269>`_, Casey Bodley, Orit Wasserman)
+* rgw: fix for bucket delete racing with mdlog sync (`issue#17698 <http://tracker.ceph.com/issues/17698>`_, `pr#11648 <http://github.com/ceph/ceph/pull/11648>`_, Casey Bodley)
+* rgw: fix for passing temporary in InitBucketSyncStatus (`issue#17661 <http://tracker.ceph.com/issues/17661>`_, `pr#11594 <http://github.com/ceph/ceph/pull/11594>`_, Casey Bodley)
+* rgw: fix for unsafe change of rgw_zonegroup (`issue#17962 <http://tracker.ceph.com/issues/17962>`_, `pr#12075 <http://github.com/ceph/ceph/pull/12075>`_, Casey Bodley)
+* rgw: fix indentation for cache_pools (`issue#8295 <http://tracker.ceph.com/issues/8295>`_, `pr#251 <http://github.com/ceph/ceph/pull/251>`_, Sage Weil)
+* rgw: fix missing master zone for a single zone zonegroup (`issue#17364 <http://tracker.ceph.com/issues/17364>`_, `pr#11965 <http://github.com/ceph/ceph/pull/11965>`_, Orit Wasserman)
+* rgw: fix osd crashes when execute "radosgw-admin bi list --max-entries=1" command (`issue#17745 <http://tracker.ceph.com/issues/17745>`_, `pr#11697 <http://github.com/ceph/ceph/pull/11697>`_, weiqiaomiao)
+* rgw: fix put_acls for objects starting and ending with underscore (`issue#17625 <http://tracker.ceph.com/issues/17625>`_, `pr#11566 <http://github.com/ceph/ceph/pull/11566>`_, Orit Wasserman)
+* rgw: fix RGWSimpleRadosLockCR set_description() (`pr#11961 <http://github.com/ceph/ceph/pull/11961>`_, Tianshan Qu)
+* rgw: fix the field 'total_time'  of log entry in log show opt (`issue#17598 <http://tracker.ceph.com/issues/17598>`_, `pr#11425 <http://github.com/ceph/ceph/pull/11425>`_, weiqiaomiao)
+* rgw: fix uncompressed object size deduction in RGWRados::copy_obj_data. (`issue#17803 <http://tracker.ceph.com/issues/17803>`_, `pr#11794 <http://github.com/ceph/ceph/pull/11794>`_, Radoslaw Zarzynski)
+* rgw: frontend subsystem rework (`pr#10767 <http://github.com/ceph/ceph/pull/10767>`_, Radoslaw Zarzynski, Casey Bodley, Matt Benjamin)
+* rgw: ftw (`issue#17888 <http://tracker.ceph.com/issues/17888>`_, `pr#12262 <http://github.com/ceph/ceph/pull/12262>`_, Casey Bodley)
+* rgw: get_system_obj does not use result of get_system_obj_state (`issue#17580 <http://tracker.ceph.com/issues/17580>`_, `pr#11444 <http://github.com/ceph/ceph/pull/11444>`_, Casey Bodley)
+* rgw: get_zonegroup() uses "default" zonegroup if empty (`issue#17372 <http://tracker.ceph.com/issues/17372>`_, `pr#11207 <http://github.com/ceph/ceph/pull/11207>`_, Yehuda Sadeh)
+* rgw: handle empty POST condition (`issue#17635 <http://tracker.ceph.com/issues/17635>`_, `pr#11581 <http://github.com/ceph/ceph/pull/11581>`_, Yehuda Sadeh)
+* rgw: handle Swift auth errors in a way compatible with new Tempests. (`issue#16590 <http://tracker.ceph.com/issues/16590>`_, `pr#10021 <http://github.com/ceph/ceph/pull/10021>`_, Radoslaw Zarzynski)
+* rgw: json encode/decode index_type, allow modification (`issue#17755 <http://tracker.ceph.com/issues/17755>`_, `pr#11707 <http://github.com/ceph/ceph/pull/11707>`_, Yehuda Sadeh)
+* rgw: loses realm/period/zonegroup/zone data: period overwritten if somewhere in the cluster is still running Hammer (`issue#17371 <http://tracker.ceph.com/issues/17371>`_, `pr#11426 <http://github.com/ceph/ceph/pull/11426>`_, Orit Wasserman)
+* rgw: make RGWLocalAuthApplier::is_admin_of() aware about system users. (`issue#18106 <http://tracker.ceph.com/issues/18106>`_, `pr#12283 <http://github.com/ceph/ceph/pull/12283>`_, Radoslaw Zarzynski)
+* rgw: metadata sync info should be shown at master zone of slave zoneg… (`issue#18091 <http://tracker.ceph.com/issues/18091>`_, `pr#12187 <http://github.com/ceph/ceph/pull/12187>`_, Jing Wenjun)
+* rgw: minor cleanup (`pr#10057 <http://github.com/ceph/ceph/pull/10057>`_, Yan Jun)
+* rgw: move compression config into zone placement (`pr#12113 <http://github.com/ceph/ceph/pull/12113>`_, Casey Bodley)
+* rgw: move xfs to a seperate directory (`pr#969 <http://github.com/ceph/ceph/pull/969>`_, Orit Wasserman)
+* rgw: multipart upload copy (`issue#12790 <http://tracker.ceph.com/issues/12790>`_, `pr#11269 <http://github.com/ceph/ceph/pull/11269>`_, Yehuda Sadeh, Javier M. Mellid)
+* rgw: need to close_section in lc list op (`pr#12232 <http://github.com/ceph/ceph/pull/12232>`_, weiqiaomiao)
+* rgw: policy acl format should be xml (`pr#946 <http://github.com/ceph/ceph/pull/946>`_, Orit Wasserman)
+* rgw: radosgw-admin: more on placement configuration (`issue#18078 <http://tracker.ceph.com/issues/18078>`_, `pr#12242 <http://github.com/ceph/ceph/pull/12242>`_, Casey Bodley)
+* rgw: region conversion respects pre-existing rgw_region_root_pool (`issue#17963 <http://tracker.ceph.com/issues/17963>`_, `pr#12076 <http://github.com/ceph/ceph/pull/12076>`_, Casey Bodley)
+* rgw: remove a redundant judgement when listng objects. (`pr#10849 <http://github.com/ceph/ceph/pull/10849>`_, zhangshaowen)
+* rgw: remove circular reference in RGWAsyncRadosRequest (`issue#17793 <http://tracker.ceph.com/issues/17793>`_, `issue#17792 <http://tracker.ceph.com/issues/17792>`_, `pr#11815 <http://github.com/ceph/ceph/pull/11815>`_, Casey Bodley)
+* rgw: remove suggestion to upgrade libcurl (`pr#11630 <http://github.com/ceph/ceph/pull/11630>`_, Casey Bodley)
+* rgw: remove unused variable "ostr" in rgw_b64.h and fix the comment (`pr#11329 <http://github.com/ceph/ceph/pull/11329>`_, Weibing Zhang)
+* rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth. (`issue#17076 <http://tracker.ceph.com/issues/17076>`_, `pr#10919 <http://github.com/ceph/ceph/pull/10919>`_, Pritha Srivastava)
+* rgw: revert unintentional change to civetweb (`pr#12004 <http://github.com/ceph/ceph/pull/12004>`_, Bassam Tabbara)
+* rgw: rgw-admin: new commands to control placement (`issue#18078 <http://tracker.ceph.com/issues/18078>`_, `pr#12230 <http://github.com/ceph/ceph/pull/12230>`_, Yehuda Sadeh)
+* rgw: RGWBucketSyncStatusManager uses existing async_rados (`issue#18083 <http://tracker.ceph.com/issues/18083>`_, `pr#12229 <http://github.com/ceph/ceph/pull/12229>`_, Casey Bodley)
+* rgw: rgw_file:  apply missed base64 try-catch (`issue#17663 <http://tracker.ceph.com/issues/17663>`_, `pr#11671 <http://github.com/ceph/ceph/pull/11671>`_, Matt Benjamin)
+* rgw: RGWHTTPArgs::get_str() - return argument string that was set. (`pr#10672 <http://github.com/ceph/ceph/pull/10672>`_, Marcus Watts)
+* rgw: rgw multisite: fix the increamtal bucket sync init (`issue#17624 <http://tracker.ceph.com/issues/17624>`_, `pr#11553 <http://github.com/ceph/ceph/pull/11553>`_, Zengran Zhang)
+* rgw: rgw multisite: use a rados lock to coordinate data log trimming (`pr#10546 <http://github.com/ceph/ceph/pull/10546>`_, Casey Bodley)
+* rgw: RGW Python bindings - use explicit array (`pr#11831 <http://github.com/ceph/ceph/pull/11831>`_, Daniel Gryniewicz)
+* rgw: rgw_rados.cc fix shard_num format for snprintf (`pr#11493 <http://github.com/ceph/ceph/pull/11493>`_, Weibing Zhang)
+* rgw: rgw/rgw_file.cc: Add compat.h to allow CLOCK_MONOTONE (`pr#12309 <http://github.com/ceph/ceph/pull/12309>`_, Willem Jan Withagen)
+* rgw: RGWSimpleRadosReadCR tolerates empty reads (`issue#17568 <http://tracker.ceph.com/issues/17568>`_, `pr#11504 <http://github.com/ceph/ceph/pull/11504>`_, Casey Bodley)
+* rgw: [RGW] Wip rgw compression (`pr#11494 <http://github.com/ceph/ceph/pull/11494>`_, Alyona Kiseleva, Adam Kupczyk, Casey Bodley)
+* rgw: set duration for lifecycle lease (`issue#17965 <http://tracker.ceph.com/issues/17965>`_, `pr#12231 <http://github.com/ceph/ceph/pull/12231>`_, Yehuda Sadeh)
+* rgw: should assign 'olh_bl" to state.attrset[RGW_ATTR_OLH_ID_TAG] instead of 'bl' (`pr#10239 <http://github.com/ceph/ceph/pull/10239>`_, weiqiaomiao)
+* rgw: skip empty http args in method parse() to avoid extra effort (`pr#11989 <http://github.com/ceph/ceph/pull/11989>`_, Guo Zhandong)
+* rgw: split osd's in 2 nodes (`issue#15612 <http://tracker.ceph.com/issues/15612>`_, `pr#1019 <http://github.com/ceph/ceph/pull/1019>`_, Vasu Kulkarni)
+* rgw: support for x-robots-tag header (`issue#17790 <http://tracker.ceph.com/issues/17790>`_, `pr#11753 <http://github.com/ceph/ceph/pull/11753>`_, Yehuda Sadeh)
+* rgw: sync modules, metadata search (`pr#10731 <http://github.com/ceph/ceph/pull/10731>`_, Yehuda Sadeh)
+* rgw: Update version of civetweb to 1.8 (`pr#11343 <http://github.com/ceph/ceph/pull/11343>`_, Marcus Watts)
+* rgw: use civetweb if no frontend was configured (`pr#958 <http://github.com/ceph/ceph/pull/958>`_, Orit Wasserman)
+* rgw: use explicit flag to cancel RGWCoroutinesManager::run() (`issue#17465 <http://tracker.ceph.com/issues/17465>`_, `pr#12452 <http://github.com/ceph/ceph/pull/12452>`_, Casey Bodley)
+* rgw: valgrind fixes for kraken (`issue#18414 <http://tracker.ceph.com/issues/18414>`_, `issue#18407 <http://tracker.ceph.com/issues/18407>`_, `issue#18412 <http://tracker.ceph.com/issues/18412>`_, `issue#18300 <http://tracker.ceph.com/issues/18300>`_, `pr#12949 <http://github.com/ceph/ceph/pull/12949>`_, Casey Bodley)
+* rgw: verified that failed check is in osd-scrub-repair.sh (`issue#17850 <http://tracker.ceph.com/issues/17850>`_, `pr#11881 <http://github.com/ceph/ceph/pull/11881>`_, Matt Benjamin)
+* rgw: we don't support btrfs any more (`pr#1132 <http://github.com/ceph/ceph/pull/1132>`_, Orit Wasserman)
+* rgw: Wip rgwfile pybind (`pr#11624 <http://github.com/ceph/ceph/pull/11624>`_, Haomai Wang)
+* tests,bluestore: os/bluestore: add UT for an estimation of Onode in-memory size (`pr#12532 <http://github.com/ceph/ceph/pull/12532>`_, Igor Fedotov)
+* tests,bluestore: os/test/store_test: fix legacy bluestore cache settings application (`pr#11915 <http://github.com/ceph/ceph/pull/11915>`_, Igor Fedotov)
+* tests: ceph-disk: force debug monc = 0 (`issue#17607 <http://tracker.ceph.com/issues/17607>`_, `pr#11534 <http://github.com/ceph/ceph/pull/11534>`_, Loic Dachary)
+* tests: ceph_objectstore_tool.py: Don't use btrfs on FreeBSD (`pr#10507 <http://github.com/ceph/ceph/pull/10507>`_, Willem Jan Withagen)
+* tests: ceph_test_objectstore: fix Rename test (`pr#12261 <http://github.com/ceph/ceph/pull/12261>`_, Sage Weil)
+* tests: check hostname --fqdn sanity before running make check (`issue#18134 <http://tracker.ceph.com/issues/18134>`_, `pr#12297 <http://github.com/ceph/ceph/pull/12297>`_, Nathan Cutler)
+* tests,cleanup,rbd: test/librbd: in test_notify set object-map and fast-diff features by default (`pr#11821 <http://github.com/ceph/ceph/pull/11821>`_, Mykola Golub)
+* tests,cleanup: test_bloom_filter.cc: Fix a mismatch for the random_seed parameter (`pr#11774 <http://github.com/ceph/ceph/pull/11774>`_, Willem Jan Withagen)
+* tests,cleanup: test/osd/osd-fast-mark-down.sh: remove unnecessary teardown() calls (`pr#12101 <http://github.com/ceph/ceph/pull/12101>`_, Kefu Chai)
+* tests,cleanup: test/osd-scrub-repair.sh: use repair() instead of "ceph pg repair" (`pr#12036 <http://github.com/ceph/ceph/pull/12036>`_, Kefu Chai)
+* tests,cleanup: test/rados: remove unused bufferlist variable (`pr#10221 <http://github.com/ceph/ceph/pull/10221>`_, Yan Jun)
+* tests,common: test: add perf-reset test in test/perf_counters.cc (`pr#8948 <http://github.com/ceph/ceph/pull/8948>`_, wangsongbo)
+* tests: disable failing tests (`issue#17561 <http://tracker.ceph.com/issues/17561>`_, `issue#17757 <http://tracker.ceph.com/issues/17757>`_, `pr#11714 <http://github.com/ceph/ceph/pull/11714>`_, Loic Dachary)
+* tests: disable the echo when running get_timeout_delays() (`pr#12180 <http://github.com/ceph/ceph/pull/12180>`_, Kefu Chai)
+* tests: do not use memstore.test_temp_dir in two tests (`issue#17743 <http://tracker.ceph.com/issues/17743>`_, `pr#12281 <http://github.com/ceph/ceph/pull/12281>`_, Loic Dachary)
+* tests: erasure-code: add k=2, m=2 for isa & jerasure (`issue#18188 <http://tracker.ceph.com/issues/18188>`_, `pr#12383 <http://github.com/ceph/ceph/pull/12383>`_, Loic Dachary)
+* tests: facilitate background process debug in ceph-helpers.sh (`issue#17830 <http://tracker.ceph.com/issues/17830>`_, `pr#12183 <http://github.com/ceph/ceph/pull/12183>`_, Loic Dachary)
+* tests: fix ceph-helpers.sh wait_for_clean delays (`issue#17830 <http://tracker.ceph.com/issues/17830>`_, `pr#12095 <http://github.com/ceph/ceph/pull/12095>`_, Loic Dachary)
+* tests: fix osd-scrub-repair.sh (`pr#12072 <http://github.com/ceph/ceph/pull/12072>`_, David Zafman)
+* tests: Fix racey test by setting noout flag (tracker 17757) (`issue#17757 <http://tracker.ceph.com/issues/17757>`_, `pr#11715 <http://github.com/ceph/ceph/pull/11715>`_, David Zafman)
+* tests: merge ceph-qa-suite
+* tests: Minor clean-ups (`pr#12048 <http://github.com/ceph/ceph/pull/12048>`_, David Zafman)
+* tests: minor make check cleanup (`pr#12146 <http://github.com/ceph/ceph/pull/12146>`_, David Zafman)
+* tests: no python3 tests for ceph-disk (`issue#17923 <http://tracker.ceph.com/issues/17923>`_, `pr#12025 <http://github.com/ceph/ceph/pull/12025>`_, Loic Dachary)
+* tests: osd-crush.sh must retry crush dump (`issue#17919 <http://tracker.ceph.com/issues/17919>`_, `pr#12016 <http://github.com/ceph/ceph/pull/12016>`_, Loic Dachary)
+* tests: osd-scrub-repair.sh abort if add_something fails (`pr#12172 <http://github.com/ceph/ceph/pull/12172>`_, Loic Dachary)
+* tests: os/memstore: fix a mem leak in MemStore::Collection::create_object() (`pr#12201 <http://github.com/ceph/ceph/pull/12201>`_, Kefu Chai)
+* tests: os/memstore, os/filestore: fix store_test's to satisfy rm_coll behavi… (`pr#11558 <http://github.com/ceph/ceph/pull/11558>`_, Igor Fedotov)
+* tests: paxos fixes (`issue#11913 <http://tracker.ceph.com/issues/11913>`_, `pr#457 <http://github.com/ceph/ceph/pull/457>`_, John Spray)
+* tests: pin flake8 to avoid behavior changes (`issue#17898 <http://tracker.ceph.com/issues/17898>`_, `pr#11971 <http://github.com/ceph/ceph/pull/11971>`_, Loic Dachary)
+* tests: qa: fixed script to schedule rados and other suites with --subset option (`pr#12587 <http://github.com/ceph/ceph/pull/12587>`_, Yuri Weinstein)
+* tests: qa/tasks/admin_socket: subst in repo name (`pr#12508 <http://github.com/ceph/ceph/pull/12508>`_, Sage Weil)
+* tests: qa/tasks/ceph_deploy: use dev option instead of dev-commit (`pr#12514 <http://github.com/ceph/ceph/pull/12514>`_, Vasu Kulkarni)
+* tests: qa/tasks/osd_backfill.py: wait for osd.[12] to start (`issue#18303 <http://tracker.ceph.com/issues/18303>`_, `pr#12577 <http://github.com/ceph/ceph/pull/12577>`_, Sage Weil)
+* tests: qa/workunits/cephtool/test.sh: FreeBSD has no distro. (`pr#11702 <http://github.com/ceph/ceph/pull/11702>`_, Willem Jan Withagen)
+* tests: qa/workunits: include extension for nose tests (`pr#12572 <http://github.com/ceph/ceph/pull/12572>`_, Sage Weil)
+* tests: qa/workunits/rados/test_envlibrados_for_rocksdb: force librados-dev install (`pr#11941 <http://github.com/ceph/ceph/pull/11941>`_, Sage Weil)
+* tests,rbd: qa/workunits/rbd: fix (`issue#18271 <http://tracker.ceph.com/issues/18271>`_, `pr#12511 <http://github.com/ceph/ceph/pull/12511>`_, Sage Weil)
+* tests,rbd: qa/workunits/rbd: removed qemu-iotest case 077 (`issue#10773 <http://tracker.ceph.com/issues/10773>`_, `pr#12366 <http://github.com/ceph/ceph/pull/12366>`_, Jason Dillaman)
+* tests,rbd: qa/workunits/rbd: simplify running nbd test under build env (`pr#11781 <http://github.com/ceph/ceph/pull/11781>`_, Mykola Golub)
+* tests,rbd: qa/workunits/rbd: use image id when probing for image presence (`issue#18048 <http://tracker.ceph.com/issues/18048>`_, `pr#12195 <http://github.com/ceph/ceph/pull/12195>`_, Mykola Golub)
+* tests,rbd: qa/workunits/rbd: use more recent qemu-iotests that support Xenial (`issue#18149 <http://tracker.ceph.com/issues/18149>`_, `pr#12371 <http://github.com/ceph/ceph/pull/12371>`_, Jason Dillaman)
+* tests,rbd: rbd-mirror: fix gmock warnings in bootstrap request unit tests (`issue#18156 <http://tracker.ceph.com/issues/18156>`_, `pr#12344 <http://github.com/ceph/ceph/pull/12344>`_, Mykola Golub)
+* tests,rbd: rbd-mirror: improve resiliency of stress test case (`issue#17416 <http://tracker.ceph.com/issues/17416>`_, `pr#11326 <http://github.com/ceph/ceph/pull/11326>`_, Jason Dillaman)
+* tests,rbd: test: new librbd discard after write test case (`pr#11645 <http://github.com/ceph/ceph/pull/11645>`_, Jason Dillaman)
+* tests,rbd: test: skip TestLibRBD.DiscardAfterWrite if skip partial discard enabled (`issue#17750 <http://tracker.ceph.com/issues/17750>`_, `pr#11703 <http://github.com/ceph/ceph/pull/11703>`_, Jason Dillaman)
+* tests,rbd: test: TestJournalReplay test cases need to wait for event commit (`issue#17566 <http://tracker.ceph.com/issues/17566>`_, `pr#11480 <http://github.com/ceph/ceph/pull/11480>`_, Jason Dillaman)
+* tests: remove TestPGLog EXPECT_DEATH tests (`issue#18030 <http://tracker.ceph.com/issues/18030>`_, `pr#12361 <http://github.com/ceph/ceph/pull/12361>`_, Loic Dachary)
+* tests: save 9 characters for asok paths (`issue#16014 <http://tracker.ceph.com/issues/16014>`_, `pr#12066 <http://github.com/ceph/ceph/pull/12066>`_, Loic Dachary)
+* tests: sync ceph-erasure-code-corpus for using 'arch' not 'uname -p' (`pr#12024 <http://github.com/ceph/ceph/pull/12024>`_, Kefu Chai)
+* tests: test/ceph_crypto: do not read ceph.conf in global_init() (`issue#18128 <http://tracker.ceph.com/issues/18128>`_, `pr#12318 <http://github.com/ceph/ceph/pull/12318>`_, Kefu Chai)
+* tests: test: ceph-objectstore-tool: should import platform before using it (`pr#12038 <http://github.com/ceph/ceph/pull/12038>`_, Kefu Chai)
+* tests: test/ceph_test_msgr: do not use Message::middle for holding transient… (`issue#17728 <http://tracker.ceph.com/issues/17728>`_, `pr#11680 <http://github.com/ceph/ceph/pull/11680>`_, Kefu Chai)
+* tests: test: disable osd-scrub-repair and test-erasure-eio (`issue#17830 <http://tracker.ceph.com/issues/17830>`_, `pr#12058 <http://github.com/ceph/ceph/pull/12058>`_, Loic Dachary, Dan Mick)
+* tests: test: disable osd-scrub-repair and test-erasure-eio (`pr#11979 <http://github.com/ceph/ceph/pull/11979>`_, Dan Mick)
+* tests: test: Don't write to a poolid that this test might not have created (`pr#12378 <http://github.com/ceph/ceph/pull/12378>`_, David Zafman)
+* tests: test: enable unittest_dns_resolve (`pr#12209 <http://github.com/ceph/ceph/pull/12209>`_, Kefu Chai)
+* tests: test/encoding/readable.sh: fix shell script warning (`pr#11527 <http://github.com/ceph/ceph/pull/11527>`_, Willem Jan Withagen)
+* tests: TestErasureCodePluginJerasure must stop the log thread (`issue#17561 <http://tracker.ceph.com/issues/17561>`_, `pr#11721 <http://github.com/ceph/ceph/pull/11721>`_, Loic Dachary)
+* tests: test: fix test-erasure-eio and osd-scrub-repair races (17830) (`pr#11926 <http://github.com/ceph/ceph/pull/11926>`_, David Zafman)
+* tests: test/osd-fast-mark-down.sh: wrong assumption on first subtest (`pr#12123 <http://github.com/ceph/ceph/pull/12123>`_, Piotr Dałek)
+* tests: test/osd/osd-fast-mark-down.sh: introduce large timeout (`issue#17918 <http://tracker.ceph.com/issues/17918>`_, `pr#12019 <http://github.com/ceph/ceph/pull/12019>`_, Piotr Dałek)
+* tests: test/osd-scrub-repair.sh: Use test case specific object names to help… (`pr#11449 <http://github.com/ceph/ceph/pull/11449>`_, David Zafman)
+* tests: test/store_test: fix errors on the whole test suite run caused by the… (`pr#11427 <http://github.com/ceph/ceph/pull/11427>`_, Igor Fedotov)
+* tests: test_subman.sh: Don't use --tmpdir (`pr#11384 <http://github.com/ceph/ceph/pull/11384>`_, Willem Jan Withagen)
+* tests: test: test-erasure-eio.sh fix recovery testing and enable it (`pr#12170 <http://github.com/ceph/ceph/pull/12170>`_, David Zafman)
+* tests: The default changed to disallow pool delete as of #11665; the tests assume it's allowed. (`pr#11897 <http://github.com/ceph/ceph/pull/11897>`_, Sage Weil)
+* tests: Turn off tests again due to Jenkins failures (`pr#12217 <http://github.com/ceph/ceph/pull/12217>`_, David Zafman)
+* tests: unittest_throttle avoid ASSERT_DEATH (`issue#18036 <http://tracker.ceph.com/issues/18036>`_, `pr#12393 <http://github.com/ceph/ceph/pull/12393>`_, Loic Dachary)
+* tests: update rbd/singleton/all/formatted-output.yaml to support ceph-ci (`issue#18440 <http://tracker.ceph.com/issues/18440>`_, `pr#12823 <http://github.com/ceph/ceph/pull/12823>`_, Nathan Cutler)
+* tests: use shorter directories for tests (`issue#16014 <http://tracker.ceph.com/issues/16014>`_, `pr#12046 <http://github.com/ceph/ceph/pull/12046>`_, Loic Dachary)
+* tests: vstart.sh: fix bashism in the script (`pr#11889 <http://github.com/ceph/ceph/pull/11889>`_, Mykola Golub)
+* tests: workunits/ceph-helpers.sh: FreeBSD returns a different errorstring. (`pr#12005 <http://github.com/ceph/ceph/pull/12005>`_, Willem Jan Withagen)
+* tools: Adding ceph-lazy tool (`pr#11055 <http://github.com/ceph/ceph/pull/11055>`_, gcharot)
+* tools: ceph-create-keys should not try forever to do things (`issue#17753 <http://tracker.ceph.com/issues/17753>`_, `issue#12649 <http://tracker.ceph.com/issues/12649>`_, `issue#16255 <http://tracker.ceph.com/issues/16255>`_, `pr#11749 <http://github.com/ceph/ceph/pull/11749>`_, Alfredo Deza)
+* tools: ceph_detect_init: add support for Alpine (`pr#8316 <http://github.com/ceph/ceph/pull/8316>`_, John Coyle)
+* tools: ceph-disk: fix flake8 errors (`issue#17898 <http://tracker.ceph.com/issues/17898>`_, `pr#11973 <http://github.com/ceph/ceph/pull/11973>`_, Ken Dreyer)
+* tools: ceph-disk: prevent unnecessary tracebacks from subprocess.check_call (`issue#16125 <http://tracker.ceph.com/issues/16125>`_, `pr#12414 <http://github.com/ceph/ceph/pull/12414>`_, Alfredo Deza)
+* tools: ceph-post-file: single command to upload a file to cephdrop (`pr#505 <http://github.com/ceph/ceph/pull/505>`_, Dan Mick, Travis Rhoden)
+* tools: cleanup phase of cephfs-data-scan (`pr#12337 <http://github.com/ceph/ceph/pull/12337>`_, Vishal Kanaujia)
+* tools: osdmaptool: additional tests (`pr#1196 <http://github.com/ceph/ceph/pull/1196>`_, Sage Weil)
+* tools: osdmaptool: fix divide by zero error (`pr#12561 <http://github.com/ceph/ceph/pull/12561>`_, Yunchuan Wen)
+* tools: rados: fix segfaults when run without --pool (`issue#17684 <http://tracker.ceph.com/issues/17684>`_, `pr#11633 <http://github.com/ceph/ceph/pull/11633>`_, David Disseldorp)
+* tools: rados: optionally support reading omap key from file (`issue#18123 <http://tracker.ceph.com/issues/18123>`_, `pr#12286 <http://github.com/ceph/ceph/pull/12286>`_, Jason Dillaman)
+* tools: script/run-coverity: update (`pr#12162 <http://github.com/ceph/ceph/pull/12162>`_, Sage Weil)
+* tools: script/sepia_bt.sh: a script to prepare for debugging on teuthology@sepia (`pr#12012 <http://github.com/ceph/ceph/pull/12012>`_, Kefu Chai)
+* tools: src/vstart.sh: Only execute btrfs if it is available (`pr#11683 <http://github.com/ceph/ceph/pull/11683>`_, Willem Jan Withagen)
+* tools: tools/ceph-monstore-update-crush.sh: FreeBSD getopt is not compatible… (`pr#11525 <http://github.com/ceph/ceph/pull/11525>`_, Willem Jan Withagen)
+
+
+
+v11.0.2 Kraken
+==============
+
+This development checkpoint release includes a lot of changes and improvements
+to Kraken. This is the first release introducing ceph-mgr, a new daemon which
+provides additional monitoring & interfaces to external monitoring/management
+systems. There are also many improvements to bluestore, RGW introduces sync
+modules, copy part for multipart uploads and metadata search via elastic search
+as a tech preview.
+
+Notable Changes
+---------------
+
+* bluestore: os/bluestore: misc fixes (`pr#10953 <http://github.com/ceph/ceph/pull/10953>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: do not op_file_update deleted files (`pr#10686 <http://github.com/ceph/ceph/pull/10686>`_, Sage Weil)
+* bluestore: bluestore/BitAllocator: Fix deadlock with musl libc (`pr#10634 <http://github.com/ceph/ceph/pull/10634>`_, John Coyle)
+* bluestore: bluestore/BlueFS: revert direct IO for WRITER_WAL (`pr#11059 <http://github.com/ceph/ceph/pull/11059>`_, Mark Nelson)
+* bluestore: ceph-disk: support creating block.db and block.wal with customized size for bluestore (`pr#10135 <http://github.com/ceph/ceph/pull/10135>`_, Zhi Zhang)
+* bluestore: compressor/zlib: switch to raw deflate (`pr#11122 <http://github.com/ceph/ceph/pull/11122>`_, Piotr Dałek)
+* bluestore: do not use freelist to track bluefs_extents (`pr#10698 <http://github.com/ceph/ceph/pull/10698>`_, Sage Weil)
+* bluestore: initialize csum_order properly (`pr#10728 <http://github.com/ceph/ceph/pull/10728>`_, xie xingguo)
+* bluestore: kv/rocksdb: dump transactions on error (`pr#11042 <http://github.com/ceph/ceph/pull/11042>`_, Somnath Roy)
+* bluestore: kv: In memory keyvalue db implementation (`pr#9933 <http://github.com/ceph/ceph/pull/9933>`_, Ramesh Chander)
+* bluestore: os/bluestore/BitAllocator: batch is_allocated bit checks (`pr#10704 <http://github.com/ceph/ceph/pull/10704>`_, Ramesh Chander)
+* bluestore: os/bluestore/BlueFS: For logs of rocksdb & bluefs only use directio. (`pr#11012 <http://github.com/ceph/ceph/pull/11012>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: async compaction (`pr#10717 <http://github.com/ceph/ceph/pull/10717>`_, Varada Kari, Sage Weil)
+* bluestore: os/bluestore/BlueFS: do not hold internal lock while waiting for IO (`pr#9898 <http://github.com/ceph/ceph/pull/9898>`_, Varada Kari, Sage Weil)
+* bluestore: os/bluestore/BlueFS: do not start racing async compaction (`pr#11010 <http://github.com/ceph/ceph/pull/11010>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: don't inc l_bluefs_files_written_wal if overwrite. (`pr#10143 <http://github.com/ceph/ceph/pull/10143>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: factor unflushed log into runway calculation (`pr#10966 <http://github.com/ceph/ceph/pull/10966>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: fix async compaction logging bug (`pr#10964 <http://github.com/ceph/ceph/pull/10964>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: log dirty files at sync time (`pr#11108 <http://github.com/ceph/ceph/pull/11108>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: only extend extent on same bdev (`pr#11023 <http://github.com/ceph/ceph/pull/11023>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: prevent concurrent async compaction (`pr#11095 <http://github.com/ceph/ceph/pull/11095>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: release completed aios (`pr#11268 <http://github.com/ceph/ceph/pull/11268>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: use StupidAllocator; fix async compaction bug (`pr#11087 <http://github.com/ceph/ceph/pull/11087>`_, Sage Weil)
+* bluestore: os/bluestore/bluefs: add file refs check (`pr#10863 <http://github.com/ceph/ceph/pull/10863>`_, xie xingguo)
+* bluestore: os/bluestore/bluefs: use map to track dirty files (`pr#10923 <http://github.com/ceph/ceph/pull/10923>`_, xie xingguo)
+* bluestore: os/bluestore/bluefs_types: fix extent operator<< (`pr#10685 <http://github.com/ceph/ceph/pull/10685>`_, Sage Weil)
+* bluestore: os/bluestore/bluestore_types: uint64_t for ref_map (`pr#11267 <http://github.com/ceph/ceph/pull/11267>`_, Sage Weil)
+* bluestore: os/bluestore: Hint based allocation in bitmap Allocator (`pr#10978 <http://github.com/ceph/ceph/pull/10978>`_, Ramesh Chander)
+* bluestore: os/bluestore: Remove bit alloc Woverloaded-virtual warnings (`pr#10082 <http://github.com/ceph/ceph/pull/10082>`_, Ramesh Chander)
+* bluestore: os/bluestore: a few cleanups (`pr#11192 <http://github.com/ceph/ceph/pull/11192>`_, xie xingguo)
+* bluestore: os/bluestore: a few fixes about  the global csum setting (`pr#11195 <http://github.com/ceph/ceph/pull/11195>`_, xie xingguo)
+* bluestore: os/bluestore: add assert to compress_extent_map (`pr#11240 <http://github.com/ceph/ceph/pull/11240>`_, Sage Weil)
+* bluestore: os/bluestore: add cache-related stats (`pr#10961 <http://github.com/ceph/ceph/pull/10961>`_, xie xingguo)
+* bluestore: os/bluestore: add checks and kill unreachable code (`pr#11077 <http://github.com/ceph/ceph/pull/11077>`_, xie xingguo)
+* bluestore: os/bluestore: add error injection (`pr#11151 <http://github.com/ceph/ceph/pull/11151>`_, Sage Weil)
+* bluestore: os/bluestore: add max blob size; fix compressed min blob size logic (`pr#11239 <http://github.com/ceph/ceph/pull/11239>`_, Sage Weil)
+* bluestore: os/bluestore: add multiple finishers to bluestore (`pr#10780 <http://github.com/ceph/ceph/pull/10780>`_, Ilsoo Byun)
+* bluestore: os/bluestore: add perf counters for compression effectiveness and space utilization measurements (`pr#10449 <http://github.com/ceph/ceph/pull/10449>`_, Igor Fedotov)
+* bluestore: os/bluestore: apply "small encoding" for onode_t::extents map (`pr#10018 <http://github.com/ceph/ceph/pull/10018>`_, Igor Fedotov)
+* bluestore: os/bluestore: avoid blob_t reencode when unchanged (`pr#10768 <http://github.com/ceph/ceph/pull/10768>`_, Sage Weil)
+* bluestore: os/bluestore: binary search specified shard (`pr#11245 <http://github.com/ceph/ceph/pull/11245>`_, xie xingguo)
+* bluestore: os/bluestore: change algorithm of compression header from string to int (`pr#10137 <http://github.com/ceph/ceph/pull/10137>`_, xie xingguo)
+* bluestore: os/bluestore: compaction fixes (`pr#11279 <http://github.com/ceph/ceph/pull/11279>`_, Sage Weil)
+* bluestore: os/bluestore: drop redundant call of get_blob (`pr#11275 <http://github.com/ceph/ceph/pull/11275>`_, xie xingguo)
+* bluestore: os/bluestore: drop unreferenced spanning blobs (`pr#11212 <http://github.com/ceph/ceph/pull/11212>`_, Sage Weil)
+* bluestore: os/bluestore: fix a few leaks (`pr#11068 <http://github.com/ceph/ceph/pull/11068>`_, Sage Weil)
+* bluestore: os/bluestore: fix a few memory utilization leaks and wasters (`pr#11011 <http://github.com/ceph/ceph/pull/11011>`_, Sage Weil)
+* bluestore: os/bluestore: fix crash in decode_some() (`pr#11312 <http://github.com/ceph/ceph/pull/11312>`_, Sage Weil)
+* bluestore: os/bluestore: fix decoding hash of bnode (`pr#10773 <http://github.com/ceph/ceph/pull/10773>`_, xie xingguo)
+* bluestore: os/bluestore: fix fsck() won't catch stray shard sometimes (`pr#11219 <http://github.com/ceph/ceph/pull/11219>`_, xie xingguo)
+* bluestore: os/bluestore: fix gc when blob extends past eof (`pr#11282 <http://github.com/ceph/ceph/pull/11282>`_, Sage Weil)
+* bluestore: os/bluestore: fix improper local var variable in collection_list meth… (`pr#10680 <http://github.com/ceph/ceph/pull/10680>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix incorrect pool decoding of bnode (`pr#10117 <http://github.com/ceph/ceph/pull/10117>`_, xie xingguo)
+* bluestore: os/bluestore: fix leak of result-checking of _fsck_check_extents (`pr#11040 <http://github.com/ceph/ceph/pull/11040>`_, xie xingguo)
+* bluestore: os/bluestore: fix leaks in our use of rocksdb (`pr#11250 <http://github.com/ceph/ceph/pull/11250>`_, Sage Weil)
+* bluestore: os/bluestore: fix memory leak during bit_alloc testing (`pr#9935 <http://github.com/ceph/ceph/pull/9935>`_, xie xingguo)
+* bluestore: os/bluestore: fix offset bug in _do_write_small. (`pr#11030 <http://github.com/ceph/ceph/pull/11030>`_, amoxic)
+* bluestore: os/bluestore: fix onode cache addition race (`pr#11300 <http://github.com/ceph/ceph/pull/11300>`_, Sage Weil)
+* bluestore: os/bluestore: fix potential access violation (`pr#10362 <http://github.com/ceph/ceph/pull/10362>`_, xie xingguo)
+* bluestore: os/bluestore: fix potential access violation during rename (`pr#11033 <http://github.com/ceph/ceph/pull/11033>`_, xie xingguo)
+* bluestore: os/bluestore: fix shard_info::dump() (`pr#11061 <http://github.com/ceph/ceph/pull/11061>`_, xie xingguo)
+* bluestore: os/bluestore: fix spanning blob leak from ~ExtentMap (`pr#11223 <http://github.com/ceph/ceph/pull/11223>`_, Somnath Roy)
+* bluestore: os/bluestore: fix statfs tests (`pr#10910 <http://github.com/ceph/ceph/pull/10910>`_, Sage Weil)
+* bluestore: os/bluestore: fix when block device is not a multiple of the block size (`pr#10844 <http://github.com/ceph/ceph/pull/10844>`_, Sage Weil)
+* bluestore: os/bluestore: fix write_big counter and some more cleanups (`pr#11344 <http://github.com/ceph/ceph/pull/11344>`_, xie xingguo)
+* bluestore: os/bluestore: fix/improve csum error message (`pr#10938 <http://github.com/ceph/ceph/pull/10938>`_, Sage Weil)
+* bluestore: os/bluestore: garbage collect partially overlapped blobs (`pr#11232 <http://github.com/ceph/ceph/pull/11232>`_, Roushan Ali)
+* bluestore: os/bluestore: get rid off "isa-l" type in ZLibCompressor ctor (`pr#10931 <http://github.com/ceph/ceph/pull/10931>`_, xie xingguo)
+* bluestore: os/bluestore: gifting bluefs more carefully (`pr#10950 <http://github.com/ceph/ceph/pull/10950>`_, xie xingguo)
+* bluestore: os/bluestore: honour allow-eio flag; use global compressor if possible (`pr#10970 <http://github.com/ceph/ceph/pull/10970>`_, xie xingguo)
+* bluestore: os/bluestore: improve required compression threshold (`pr#10080 <http://github.com/ceph/ceph/pull/10080>`_, xie xingguo)
+* bluestore: os/bluestore: include bluefs space in statfs result (`pr#10795 <http://github.com/ceph/ceph/pull/10795>`_, Sage Weil)
+* bluestore: os/bluestore: introduce power 2 macros for block alignment and rounding (`pr#10128 <http://github.com/ceph/ceph/pull/10128>`_, xie xingguo)
+* bluestore: os/bluestore: make assert conditional with macro for allocator (`pr#11014 <http://github.com/ceph/ceph/pull/11014>`_, Ramesh Chander)
+* bluestore: os/bluestore: make cache settings process-wide (`pr#11295 <http://github.com/ceph/ceph/pull/11295>`_, Sage Weil)
+* bluestore: os/bluestore: make clone_range copy-on-write (`pr#11106 <http://github.com/ceph/ceph/pull/11106>`_, Sage Weil)
+* bluestore: os/bluestore: make onode keys more efficient (and sort correctly) (`pr#11009 <http://github.com/ceph/ceph/pull/11009>`_, xie xingguo, Sage Weil)
+* bluestore: os/bluestore: make trim() of 2Q cache more fine-grained (`pr#9946 <http://github.com/ceph/ceph/pull/9946>`_, xie xingguo)
+* bluestore: os/bluestore: make zone/span size of bitmap-allocator configurable (`pr#10040 <http://github.com/ceph/ceph/pull/10040>`_, xie xingguo)
+* bluestore: os/bluestore: misc cleanup and test fixes (`pr#11346 <http://github.com/ceph/ceph/pull/11346>`_, Igor Fedotov)
+* bluestore: os/bluestore: misc cleanups (`pr#10201 <http://github.com/ceph/ceph/pull/10201>`_, xie xingguo)
+* bluestore: os/bluestore: misc cleanups (`pr#11197 <http://github.com/ceph/ceph/pull/11197>`_, Haomai Wang)
+* bluestore: os/bluestore: misc fixes (`pr#9999 <http://github.com/ceph/ceph/pull/9999>`_, xie xingguo)
+* bluestore: os/bluestore: misc fixes (`pr#10771 <http://github.com/ceph/ceph/pull/10771>`_, xie xingguo)
+* bluestore: os/bluestore: misc. fixes (`pr#11129 <http://github.com/ceph/ceph/pull/11129>`_, xie xingguo)
+* bluestore: os/bluestore: more cleanups (`pr#11235 <http://github.com/ceph/ceph/pull/11235>`_, xie xingguo)
+* bluestore: os/bluestore: more cleanups and fixes (`pr#11210 <http://github.com/ceph/ceph/pull/11210>`_, xie xingguo)
+* bluestore: os/bluestore: narrow condition of sanity check when get_object_key() (`pr#11149 <http://github.com/ceph/ceph/pull/11149>`_, xie xingguo)
+* bluestore: os/bluestore: narrow lock scope for cache trim() (`pr#10410 <http://github.com/ceph/ceph/pull/10410>`_, xie xingguo)
+* bluestore: os/bluestore: optimize intrusive sets for size. (`pr#11319 <http://github.com/ceph/ceph/pull/11319>`_, Mark Nelson)
+* bluestore: os/bluestore: pack a few more in-memory types (`pr#11328 <http://github.com/ceph/ceph/pull/11328>`_, Sage Weil)
+* bluestore: os/bluestore: precondition rocksdb/bluefs during mkfs (`pr#10814 <http://github.com/ceph/ceph/pull/10814>`_, Sage Weil)
+* bluestore: os/bluestore: prevent extent merging across shard boundaries (`pr#11216 <http://github.com/ceph/ceph/pull/11216>`_, Sage Weil)
+* bluestore: os/bluestore: print bluefs_extents in hex (`pr#10689 <http://github.com/ceph/ceph/pull/10689>`_, Sage Weil)
+* bluestore: os/bluestore: proper handling for csum enable/disable settings (`pr#10431 <http://github.com/ceph/ceph/pull/10431>`_, Igor Fedotov)
+* bluestore: os/bluestore: refactor dirty blob tracking along with some related fixes (`pr#10215 <http://github.com/ceph/ceph/pull/10215>`_, Igor Fedotov)
+* bluestore: os/bluestore: remove cmake warning from extent alloc functions (`issue#16766 <http://tracker.ceph.com/issues/16766>`_, `pr#10492 <http://github.com/ceph/ceph/pull/10492>`_, Ramesh Chander)
+* bluestore: os/bluestore: remove deferred_csum machinery (`pr#11243 <http://github.com/ceph/ceph/pull/11243>`_, Sage Weil)
+* bluestore: os/bluestore: remove some copy-pastes (`pr#11017 <http://github.com/ceph/ceph/pull/11017>`_, Igor Fedotov)
+* bluestore: os/bluestore: replace store with logger in Cache (`pr#10969 <http://github.com/ceph/ceph/pull/10969>`_, xie xingguo)
+* bluestore: os/bluestore: shard extent map (`pr#10963 <http://github.com/ceph/ceph/pull/10963>`_, Sage Weil)
+* bluestore: os/bluestore: simplify LRUCache::trim() (`pr#10109 <http://github.com/ceph/ceph/pull/10109>`_, xie xingguo)
+* bluestore: os/bluestore: simplify calculation of collection key range (`pr#11166 <http://github.com/ceph/ceph/pull/11166>`_, xie xingguo)
+* bluestore: os/bluestore: sloppy reshard boundaries to avoid spanning blobs (`pr#11263 <http://github.com/ceph/ceph/pull/11263>`_, Sage Weil)
+* bluestore: os/bluestore: still more cleanups (`pr#11274 <http://github.com/ceph/ceph/pull/11274>`_, xie xingguo)
+* bluestore: os/bluestore: switch spanning_blob_map to std::map (`pr#11336 <http://github.com/ceph/ceph/pull/11336>`_, Sage Weil)
+* bluestore: os/bluestore: trim cache on reads (`pr#10095 <http://github.com/ceph/ceph/pull/10095>`_, Sage Weil)
+* bluestore: os/bluestore: try to split blobs instead of spanning them (`pr#11264 <http://github.com/ceph/ceph/pull/11264>`_, Sage Weil)
+* bluestore: os/bluestore: upgrade compression settings to atomics (`pr#11244 <http://github.com/ceph/ceph/pull/11244>`_, xie xingguo)
+* bluestore: os/bluestore: use small encoding for bluefs extent and fnode (`pr#10375 <http://github.com/ceph/ceph/pull/10375>`_, xie xingguo)
+* bluestore: os/bluestore: yet another statfs test fix (`pr#10926 <http://github.com/ceph/ceph/pull/10926>`_, Igor Fedotov)
+* bluestore: os/bluestore:Fix size calculation in bitallocator (`pr#10377 <http://github.com/ceph/ceph/pull/10377>`_, Ramesh Chander)
+* bluestore: os/bluestore: fix error handling of posix_fallocate() (`pr#10277 <http://github.com/ceph/ceph/pull/10277>`_, xie xingguo)
+* bluestore: os/bluestore: use BE for gifting and reclaiming from bluefs (`pr#10294 <http://github.com/ceph/ceph/pull/10294>`_, xie xingguo)
+* bluestore: os/bluestore: get rid off blob's ref_map for non-shared objects (`pr#9988 <http://github.com/ceph/ceph/pull/9988>`_, Igor Fedotov)
+* bluestore: kv/MemDB: fix wrong output target and add sanity checks (`pr#10358 <http://github.com/ceph/ceph/pull/10358>`_, xie xingguo)
+* bluestore: os/bluestore: add a boundary check of cache read (`pr#10349 <http://github.com/ceph/ceph/pull/10349>`_, xie xingguo)
+* bluestore: os/bluestore: fix bitmap allocating failure if max_alloc_size is 0 (`pr#10379 <http://github.com/ceph/ceph/pull/10379>`_, xie xingguo)
+* bluestore: os/bluestore: misc fixes (`pr#10327 <http://github.com/ceph/ceph/pull/10327>`_, xie xingguo)
+* bluestore: kv/MemDB: misc fixes and cleanups (`pr#10295 <http://github.com/ceph/ceph/pull/10295>`_, xie xingguo)
+* bluestore: rocksdb: pull up to master (4.12 + a few patches) (`pr#11069 <http://github.com/ceph/ceph/pull/11069>`_, Sage Weil)
+* bluestore: test/store_test: extend Bluestore compression test to verify compress… (`pr#11080 <http://github.com/ceph/ceph/pull/11080>`_, Igor Fedotov)
+* bluestore: test/store_test: fix statfs results check to consider SSD min_alloc_size (`pr#11096 <http://github.com/ceph/ceph/pull/11096>`_, Igor Fedotov)
+* bluestore: unittest_bluestore_types: a few more types for sizeof (`pr#11323 <http://github.com/ceph/ceph/pull/11323>`_, Sage Weil)
+* bluestore: ceph_test_objectstore: test clone_range and fix a few bugs (`pr#11103 <http://github.com/ceph/ceph/pull/11103>`_, Sage Weil)
+* bluestore: kv: fix some bugs in memdb (`pr#10550 <http://github.com/ceph/ceph/pull/10550>`_, Haodong Tang)
+* bluestore: os/bluestore/BlueFS: disable buffered io (`pr#10766 <http://github.com/ceph/ceph/pull/10766>`_, Sage Weil)
+* build/ops,bluestore: test/objectstore/CMakeLists.txt: fix libaio conditional (`pr#11008 <http://github.com/ceph/ceph/pull/11008>`_, Sage Weil)
+* build/ops,cephfs: client: added def for ACCESSPERMS when undefined (`pr#9835 <http://github.com/ceph/ceph/pull/9835>`_, John Coyle)
+* build/ops,cephfs: deb: merge ceph-fs-common into ceph-common (`issue#16808 <http://tracker.ceph.com/issues/16808>`_, `pr#10433 <http://github.com/ceph/ceph/pull/10433>`_, Nathan Cutler)
+* build/ops,cephfs: man/Makefile-client.am: drop legacy cephfs tool (`pr#10444 <http://github.com/ceph/ceph/pull/10444>`_, Nathan Cutler)
+* build/ops,cephfs: test: break out librados-using cephfs test (`issue#16556 <http://tracker.ceph.com/issues/16556>`_, `pr#10452 <http://github.com/ceph/ceph/pull/10452>`_, John Spray)
+* build/ops,common: common/dns_resolve: use ns_name_uncompress instead of ns_name_ntop (`pr#9755 <http://github.com/ceph/ceph/pull/9755>`_, John Coyle)
+* build/ops,common: msg/async/net_handler.cc: make it more compatible with BSDs (`pr#10029 <http://github.com/ceph/ceph/pull/10029>`_, Willem Jan Withagen)
+* build/ops,pybind: Include Python 3 bindings into the cmake build and make packages for them (`pr#10208 <http://github.com/ceph/ceph/pull/10208>`_, Oleh Prypin)
+* build/ops,rbd: systemd: add install section to rbdmap.service file (`pr#10942 <http://github.com/ceph/ceph/pull/10942>`_, Jelle vd Kooij)
+* build/ops,rbd: test: fix rbd-mirror workunit test cases for cmake (`pr#10076 <http://github.com/ceph/ceph/pull/10076>`_, Jason Dillaman)
+* build/ops,rgw: rgw-ldap: add ldap lib to rgw lib deps based on build config (`pr#9852 <http://github.com/ceph/ceph/pull/9852>`_, John Coyle)
+* build/ops: .gitignore: Add .pyc files globally (`pr#11076 <http://github.com/ceph/ceph/pull/11076>`_, Brad Hubbard)
+* build/ops: Allow compressor build without YASM (`pr#10937 <http://github.com/ceph/ceph/pull/10937>`_, Daniel Gryniewicz)
+* build/ops: CMake - stop pip checking for updates (`pr#10161 <http://github.com/ceph/ceph/pull/10161>`_, Daniel Gryniewicz)
+* build/ops: CMakeList.txt: link ceph_objectstore_tool against fuse only if WITH_FUSE (`pr#10149 <http://github.com/ceph/ceph/pull/10149>`_, Willem Jan Withagen)
+* build/ops: Cmake: fix using CMAKE_DL_LIBS instead of dl (`pr#10317 <http://github.com/ceph/ceph/pull/10317>`_, Willem Jan Withagen)
+* build/ops: CmakeLists.txt: use LIB_RESOLV instead of resolv. (`pr#10972 <http://github.com/ceph/ceph/pull/10972>`_, Willem Jan Withagen)
+* build/ops: Enable builds without ceph-test subpackage (`issue#16776 <http://tracker.ceph.com/issues/16776>`_, `pr#10872 <http://github.com/ceph/ceph/pull/10872>`_, Ricardo Dias)
+* build/ops: Fix libatomic_ops-devel in SUSE and specfile cleanup (`issue#16645 <http://tracker.ceph.com/issues/16645>`_, `pr#10363 <http://github.com/ceph/ceph/pull/10363>`_, Nathan Cutler)
+* build/ops: FreeBSD: Define CLOCK_REALTIME_COARSE in compat.h (`pr#10506 <http://github.com/ceph/ceph/pull/10506>`_, Willem Jan Withagen)
+* build/ops: Gentoo support for ceph-disk / ceph-detect-init; pip speedup (`pr#8317 <http://github.com/ceph/ceph/pull/8317>`_, Robin H. Johnson)
+* build/ops: LTTng-UST disabled for openSUSE (`issue#16937 <http://tracker.ceph.com/issues/16937>`_, `pr#10592 <http://github.com/ceph/ceph/pull/10592>`_, Michel Normand)
+* build/ops: Port ceph-brag to Python 3 (+ small fixes) (`pr#10064 <http://github.com/ceph/ceph/pull/10064>`_, Oleh Prypin)
+* build/ops: Removes remaining reference to WITH_MDS (`pr#10286 <http://github.com/ceph/ceph/pull/10286>`_, J. Eric Ivancich)
+* build/ops: Stop hiding errors from run-tox.sh (`issue#17267 <http://tracker.ceph.com/issues/17267>`_, `pr#11071 <http://github.com/ceph/ceph/pull/11071>`_, Dan Mick)
+* build/ops: Wip kill warnings (`pr#10881 <http://github.com/ceph/ceph/pull/10881>`_, Kefu Chai)
+* build/ops: autogen: Fix rocksdb error when make dist (`pr#10988 <http://github.com/ceph/ceph/pull/10988>`_, tianqing)
+* build/ops: autotools: remove a few other remaining traces (`pr#11019 <http://github.com/ceph/ceph/pull/11019>`_, Sage Weil)
+* build/ops: build scripts: Enable dnf for Fedora >= 22 (`pr#11105 <http://github.com/ceph/ceph/pull/11105>`_, Brad Hubbard)
+* build/ops: build: drop dryrun of autogen.sh from run-cmake-check.sh script (`pr#11013 <http://github.com/ceph/ceph/pull/11013>`_, xie xingguo)
+* build/ops: ceph-disk tests: Let missing python interpreters be non-fatal (`pr#11072 <http://github.com/ceph/ceph/pull/11072>`_, Dan Mick)
+* build/ops: ceph-disk: Compatibility fixes for Python 3 (`pr#9936 <http://github.com/ceph/ceph/pull/9936>`_, Anirudha Bose)
+* build/ops: ceph-disk: do not activate device that is not ready (`issue#15990 <http://tracker.ceph.com/issues/15990>`_, `pr#9943 <http://github.com/ceph/ceph/pull/9943>`_, Boris Ranto)
+* build/ops: ceph-osd-prestart.sh: check existence of OSD data directory (`issue#17091 <http://tracker.ceph.com/issues/17091>`_, `pr#10809 <http://github.com/ceph/ceph/pull/10809>`_, Nathan Cutler)
+* build/ops: ceph-osd-prestart.sh: drop Upstart-specific code (`issue#15984 <http://tracker.ceph.com/issues/15984>`_, `pr#9667 <http://github.com/ceph/ceph/pull/9667>`_, Nathan Cutler)
+* build/ops: ceph-post-file replace DSA with RSA ssh key (`issue#14267 <http://tracker.ceph.com/issues/14267>`_, `pr#10800 <http://github.com/ceph/ceph/pull/10800>`_, David Galloway)
+* build/ops: ceph.spec.in: don't try to package __pycache__ for SUSE (`issue#17106 <http://tracker.ceph.com/issues/17106>`_, `pr#10805 <http://github.com/ceph/ceph/pull/10805>`_, Tim Serong)
+* build/ops: ceph.spec.in: fix rpm package building error (`pr#10115 <http://github.com/ceph/ceph/pull/10115>`_, runsisi)
+* build/ops: changes for Clang and yasm (`pr#10417 <http://github.com/ceph/ceph/pull/10417>`_, Willem Jan Withagen)
+* build/ops: cmake changes (`pr#10351 <http://github.com/ceph/ceph/pull/10351>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10059 <http://github.com/ceph/ceph/pull/10059>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10279 <http://github.com/ceph/ceph/pull/10279>`_, Kefu Chai)
+* build/ops: cmake changes (`issue#16804 <http://tracker.ceph.com/issues/16804>`_, `pr#10391 <http://github.com/ceph/ceph/pull/10391>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10361 <http://github.com/ceph/ceph/pull/10361>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10112 <http://github.com/ceph/ceph/pull/10112>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10489 <http://github.com/ceph/ceph/pull/10489>`_, Kefu Chai)
+* build/ops: cmake changes (`pr#10283 <http://github.com/ceph/ceph/pull/10283>`_, Kefu Chai)
+* build/ops: cmake changes (`issue#16504 <http://tracker.ceph.com/issues/16504>`_, `pr#9995 <http://github.com/ceph/ceph/pull/9995>`_, Kefu Chai, Sage Weil, Dan Mick)
+* build/ops: cmake changes (`pr#9975 <http://github.com/ceph/ceph/pull/9975>`_, Kefu Chai)
+* build/ops: cmake changes related to LTTng-UST (`pr#10917 <http://github.com/ceph/ceph/pull/10917>`_, Kefu Chai)
+* build/ops: common/compressor: add libcommon as a dependency for zlib and snappy p… (`pr#11083 <http://github.com/ceph/ceph/pull/11083>`_, Igor Fedotov)
+* build/ops: compat: add abstractions for non portable pthread name funcs (`pr#9763 <http://github.com/ceph/ceph/pull/9763>`_, John Coyle)
+* build/ops: configure.ac: Use uname instead of arch. (`pr#9766 <http://github.com/ceph/ceph/pull/9766>`_, John Coyle)
+* build/ops: configure.ac: add _LIBS variables for boost_system and boost_iostreams (`pr#9848 <http://github.com/ceph/ceph/pull/9848>`_, John Coyle)
+* build/ops: configure.ac: fix res_query detection (`pr#9820 <http://github.com/ceph/ceph/pull/9820>`_, John Coyle)
+* build/ops: debian and cmake cleanups  (`pr#10788 <http://github.com/ceph/ceph/pull/10788>`_, Kefu Chai)
+* build/ops: debian: bump compat to 9 (`issue#16744 <http://tracker.ceph.com/issues/16744>`_, `pr#10366 <http://github.com/ceph/ceph/pull/10366>`_, Kefu Chai)
+* build/ops: debian: python related changes (`pr#10322 <http://github.com/ceph/ceph/pull/10322>`_, Kefu Chai)
+* build/ops: debian: replace SysV rbdmap with systemd service (`pr#10435 <http://github.com/ceph/ceph/pull/10435>`_, Ken Dreyer)
+* build/ops: debian: set libexec dir to correct value as autotools did (`pr#10096 <http://github.com/ceph/ceph/pull/10096>`_, Daniel Gryniewicz)
+* build/ops: do_cmake.sh: set up initial plugin dir (`pr#10067 <http://github.com/ceph/ceph/pull/10067>`_, Sage Weil)
+* build/ops: fix /etc/os-release parsing in install-deps.sh (`pr#10981 <http://github.com/ceph/ceph/pull/10981>`_, Nathan Cutler)
+* build/ops: fix the rpm build for centos (`pr#10289 <http://github.com/ceph/ceph/pull/10289>`_, Oleh Prypin, Josh Durgin)
+* build/ops: force Python 3 packages to build in SUSE (`issue#17106 <http://tracker.ceph.com/issues/17106>`_, `pr#10894 <http://github.com/ceph/ceph/pull/10894>`_, Dominique Leuenberger, Nathan Cutler)
+* build/ops: install-deps.sh based on /etc/os-release (`issue#16522 <http://tracker.ceph.com/issues/16522>`_, `pr#10017 <http://github.com/ceph/ceph/pull/10017>`_, Jan Fajerski)
+* build/ops: install-deps: exit non-zero when we cannot match distro  (`pr#10941 <http://github.com/ceph/ceph/pull/10941>`_, Gregory Meno)
+* build/ops: isa-l: add isa-l library as a submodule (`pr#10066 <http://github.com/ceph/ceph/pull/10066>`_, Alyona Kiseleva)
+* build/ops: jerasure: include generic objects in neon jerasure lib (like sse3/4) (`pr#10879 <http://github.com/ceph/ceph/pull/10879>`_, Dan Mick)
+* build/ops: logrotate: Run as root/ceph (`pr#10587 <http://github.com/ceph/ceph/pull/10587>`_, Boris Ranto)
+* build/ops: lttng: build the tracepoint provider lib from .c files in repo (`pr#11196 <http://github.com/ceph/ceph/pull/11196>`_, Kefu Chai)
+* build/ops: make-dist: generate ceph.spec (`issue#16501 <http://tracker.ceph.com/issues/16501>`_, `pr#9986 <http://github.com/ceph/ceph/pull/9986>`_, Sage Weil)
+* build/ops: make-dist: set rpm_release correctly for release builds (`pr#11334 <http://github.com/ceph/ceph/pull/11334>`_, Dan Mick)
+* build/ops: make-srpm.sh: A simple script to make the srpm for ceph. (`pr#11064 <http://github.com/ceph/ceph/pull/11064>`_, Ira Cooper)
+* build/ops: makefile: change librgw_file_* as check_PROGRAMS (`issue#16646 <http://tracker.ceph.com/issues/16646>`_, `pr#10229 <http://github.com/ceph/ceph/pull/10229>`_, Brad Hubbard)
+* build/ops: remove autotools (`pr#11007 <http://github.com/ceph/ceph/pull/11007>`_, Sage Weil)
+* build/ops: rpm: Do not start targets on update (`pr#9968 <http://github.com/ceph/ceph/pull/9968>`_, Nathan Cutler, Boris Ranto)
+* build/ops: rpm: ExclusiveArch for suse_version (`issue#16936 <http://tracker.ceph.com/issues/16936>`_, `pr#10594 <http://github.com/ceph/ceph/pull/10594>`_, Michel Normand)
+* build/ops: rpm: Fix creation of mount.ceph symbolic link for SUSE distros (`pr#10353 <http://github.com/ceph/ceph/pull/10353>`_, Ricardo Dias)
+* build/ops: rpm: add udev BuildRequires to provide /usr/lib/udev directory (`issue#16949 <http://tracker.ceph.com/issues/16949>`_, `pr#10608 <http://github.com/ceph/ceph/pull/10608>`_, Nathan Cutler)
+* build/ops: rpm: build rpm with cmake (`pr#10016 <http://github.com/ceph/ceph/pull/10016>`_, Kefu Chai)
+* build/ops: rpm: drop obsolete libs-compat and python-ceph-compat metapackages (`issue#16353 <http://tracker.ceph.com/issues/16353>`_, `pr#9757 <http://github.com/ceph/ceph/pull/9757>`_, Nathan Cutler)
+* build/ops: rpm: fix permissions for /etc/ceph/rbdmap (`issue#17395 <http://tracker.ceph.com/issues/17395>`_, `pr#11217 <http://github.com/ceph/ceph/pull/11217>`_, Ken Dreyer)
+* build/ops: rpm: fix shared library devel package names and dependencies (`issue#16345 <http://tracker.ceph.com/issues/16345>`_, `issue#16346 <http://tracker.ceph.com/issues/16346>`_, `pr#9744 <http://github.com/ceph/ceph/pull/9744>`_, Nathan Cutler, Ken Dreyer)
+* build/ops: rpm: move mount.ceph from ceph-base to ceph-common and add symlink in /sbin for SUSE (`issue#16598 <http://tracker.ceph.com/issues/16598>`_, `pr#10147 <http://github.com/ceph/ceph/pull/10147>`_, Nathan Cutler)
+* build/ops: run-cmake-check.sh: Remove redundant calls (`pr#11116 <http://github.com/ceph/ceph/pull/11116>`_, Brad Hubbard)
+* build/ops: script: improve ceph-release-notes regex (`pr#10729 <http://github.com/ceph/ceph/pull/10729>`_, Nathan Cutler)
+* build/ops: src/CMakeLists.txt: remove double flag -Wno-invalid-offsetof (`pr#10443 <http://github.com/ceph/ceph/pull/10443>`_, Willem Jan Withagen)
+* build/ops: src/CMakeLists.txt: remove unneeded libraries from ceph-dencoder target (`pr#10478 <http://github.com/ceph/ceph/pull/10478>`_, Willem Jan Withagen)
+* build/ops: src/global/pidfile.cc: Assign elements in structures individually (`pr#10516 <http://github.com/ceph/ceph/pull/10516>`_, Willem Jan Withagen)
+* build/ops: src/kv/CMakeLists.txt: force rocksdb/include to first include directory (`pr#11194 <http://github.com/ceph/ceph/pull/11194>`_, Willem Jan Withagen)
+* build/ops: test/common/test_util.cc: FreeBSD does not have distro information (`pr#10547 <http://github.com/ceph/ceph/pull/10547>`_, Willem Jan Withagen)
+* build/ops: test: make check using cmake (`pr#10116 <http://github.com/ceph/ceph/pull/10116>`_, Kefu Chai, Sage Weil)
+* build/ops: verfied f23 (`pr#10222 <http://github.com/ceph/ceph/pull/10222>`_, Kefu Chai)
+* build/ops: yasm-wrapper: dont echo the yasm command line (`pr#10819 <http://github.com/ceph/ceph/pull/10819>`_, Casey Bodley)
+* build/ops: .gitignore: exclude coredumps, logfiles and temporary testresults (`pr#8150 <http://github.com/ceph/ceph/pull/8150>`_, Willem Jan Withagen)
+* build/ops: this fixes the broken build (`pr#9992 <http://github.com/ceph/ceph/pull/9992>`_, Haomai Wang)
+* build/ops: mrgw: search for cmake build dir. (`pr#10180 <http://github.com/ceph/ceph/pull/10180>`_, Abhishek Lekshmanan)
+* build/ops: mrun, mstart.sh, mstop.sh: search for cmake build directory (`pr#10097 <http://github.com/ceph/ceph/pull/10097>`_, Yehuda Sadeh)
+* build/ops: arm64 fixes(`pr#10438 <http://github.com/ceph/ceph/pull/10438>`_, Dan Mick)
+* build/ops: Wip kill warnings (`pr#10934 <http://github.com/ceph/ceph/pull/10934>`_, Kefu Chai)
+* build/ops: systemd: add osd id to service description (`pr#10091 <http://github.com/ceph/ceph/pull/10091>`_, Ruben Kerkhof)
+* build/ops: fix wrong indent caused compile warning (`pr#10014 <http://github.com/ceph/ceph/pull/10014>`_, Wanlong Gao)
+* build/ops: ceph-detect-init: fix the py3 test (`pr#10266 <http://github.com/ceph/ceph/pull/10266>`_, Kefu Chai)
+* build/ops: ceph.spec: fix ceph-mgr version requirement (`pr#11285 <http://github.com/ceph/ceph/pull/11285>`_, Sage Weil)
+* build/ops: make-dist/ceph.spec.in: Fix srpm build breakage. (`pr#10404 <http://github.com/ceph/ceph/pull/10404>`_, Ira Cooper)
+* build/ops: master: remove SYSTEMD_RUN from initscript (`issue#16440 <http://tracker.ceph.com/issues/16440>`_, `issue#7627 <http://tracker.ceph.com/issues/7627>`_, `pr#9871 <http://github.com/ceph/ceph/pull/9871>`_, Vladislav Odintsov)
+* build/ops: rocksdb: revert the change introduced by dc41731 (`pr#10595 <http://github.com/ceph/ceph/pull/10595>`_, Kefu Chai)
+* build/ops: do_freebsd*.sh: rename do_freebsd-cmake.sh to do_freebsd.sh (`pr#11088 <http://github.com/ceph/ceph/pull/11088>`_, Kefu Chai)
+* build/ops: gcc 6.1.1 complains about missing include: <random>. 4.8.3 does not c… (`pr#10747 <http://github.com/ceph/ceph/pull/10747>`_, Daniel Oliveira)
+* build/ops: selinux: Allow ceph to manage tmp files (`issue#17436 <http://tracker.ceph.com/issues/17436>`_, `pr#11259 <http://github.com/ceph/ceph/pull/11259>`_, Boris Ranto)
+* build/ops: selinux: allow read /proc/<pid>/cmdline (`issue#16675 <http://tracker.ceph.com/issues/16675>`_, `pr#10339 <http://github.com/ceph/ceph/pull/10339>`_, Kefu Chai)
+* cephfs,common: osdc/Journaler: move C_DelayFlush class to .cc (`pr#10744 <http://github.com/ceph/ceph/pull/10744>`_, Michal Jarzabek)
+* cephfs,core,rbd: ObjectCacher: fix bh_read_finish offset logic (`issue#16002 <http://tracker.ceph.com/issues/16002>`_, `pr#9606 <http://github.com/ceph/ceph/pull/9606>`_, Greg Farnum)
+* cephfs,core,rbd: osdc/ObjectCacher: move C_ReadFinish, C_RetryRead (`pr#10781 <http://github.com/ceph/ceph/pull/10781>`_, Michal Jarzabek)
+* cephfs: Add ceph_ll_setlk and ceph_ll_getlk (`pr#9566 <http://github.com/ceph/ceph/pull/9566>`_, Frank S. Filz)
+* cephfs: CephFS: misc. cleanups and remove legacy cephfs tool (`issue#16195 <http://tracker.ceph.com/issues/16195>`_, `issue#16035 <http://tracker.ceph.com/issues/16035>`_, `issue#15923 <http://tracker.ceph.com/issues/15923>`_, `pr#10243 <http://github.com/ceph/ceph/pull/10243>`_, John Spray)
+* cephfs: Clean up handling of "/.." in ceph client (`pr#10691 <http://github.com/ceph/ceph/pull/10691>`_, Jeff Layton)
+* cephfs: Client: fixup param type and return value (`pr#10463 <http://github.com/ceph/ceph/pull/10463>`_, gongchuang)
+* cephfs: Client: pass "UserPerm" struct everywhere for security checks (`issue#16367 <http://tracker.ceph.com/issues/16367>`_, `issue#17368 <http://tracker.ceph.com/issues/17368>`_, `pr#11218 <http://github.com/ceph/ceph/pull/11218>`_, Greg Farnum)
+* cephfs: First pile of statx patches (`pr#10922 <http://github.com/ceph/ceph/pull/10922>`_, Sage Weil, Jeff Layton)
+* cephfs: Fix attribute handling at lookup time (`issue#16668 <http://tracker.ceph.com/issues/16668>`_, `pr#10386 <http://github.com/ceph/ceph/pull/10386>`_, Jeff Layton)
+* cephfs: Inotable repair during forward scrub (`pr#10281 <http://github.com/ceph/ceph/pull/10281>`_, Vishal Kanaujia)
+* cephfs: Server: drop locks and auth pins if wait for pending truncate (`pr#9716 <http://github.com/ceph/ceph/pull/9716>`_, xie xingguo)
+* cephfs: Small interface cleanups for struct ceph_statx  (`pr#11093 <http://github.com/ceph/ceph/pull/11093>`_, Jeff Layton)
+* cephfs: build ceph-fuse on OSX (`pr#9371 <http://github.com/ceph/ceph/pull/9371>`_, Yan, Zheng)
+* cephfs: ceph-fuse: link to libtcmalloc or jemalloc (`issue#16655 <http://tracker.ceph.com/issues/16655>`_, `pr#10258 <http://github.com/ceph/ceph/pull/10258>`_, Yan, Zheng)
+* cephfs: ceph_volume_client: store authentication metadata (`issue#15406 <http://tracker.ceph.com/issues/15406>`_, `issue#15615 <http://tracker.ceph.com/issues/15615>`_, `pr#9864 <http://github.com/ceph/ceph/pull/9864>`_, John Spray, Ramana Raja)
+* cephfs: client/barrier: move C_Block_Sync class to .cc (`pr#11001 <http://github.com/ceph/ceph/pull/11001>`_, Michal Jarzabek)
+* cephfs: client/filer: cleanup the redundant judgments of _write&&_fallocate (`pr#10062 <http://github.com/ceph/ceph/pull/10062>`_, huanwen ren)
+* cephfs: client: add missing client_lock for get_root (`pr#10027 <http://github.com/ceph/ceph/pull/10027>`_, Patrick Donnelly)
+* cephfs: client: discard mds map if it is identical to ours (`pr#9774 <http://github.com/ceph/ceph/pull/9774>`_, xie xingguo)
+* cephfs: client: fast abort if underlying statsf() call failed; end scope of std::hex properly (`pr#9803 <http://github.com/ceph/ceph/pull/9803>`_, xie xingguo)
+* cephfs: client: fix access violation (`pr#9793 <http://github.com/ceph/ceph/pull/9793>`_, xie xingguo)
+* cephfs: client: fix readdir vs fragmentation race (`issue#17286 <http://tracker.ceph.com/issues/17286>`_, `pr#11147 <http://github.com/ceph/ceph/pull/11147>`_, Yan, Zheng)
+* cephfs: client: fix segment fault in Client::_invalidate_kernel_dcache(). (`issue#17253 <http://tracker.ceph.com/issues/17253>`_, `pr#11170 <http://github.com/ceph/ceph/pull/11170>`_, Yan, Zheng)
+* cephfs: client: fix shutdown with open inodes (`issue#16764 <http://tracker.ceph.com/issues/16764>`_, `pr#10419 <http://github.com/ceph/ceph/pull/10419>`_, John Spray)
+* cephfs: client: include COMPLETE and ORDERED states in cache dump (`pr#10485 <http://github.com/ceph/ceph/pull/10485>`_, Greg Farnum)
+* cephfs: client: kill compiling warning (`pr#9994 <http://github.com/ceph/ceph/pull/9994>`_, xie xingguo)
+* cephfs: client: misc fixes (`pr#9838 <http://github.com/ceph/ceph/pull/9838>`_, xie xingguo)
+* cephfs: client: move Inode specific cleanup to destructor (`pr#10168 <http://github.com/ceph/ceph/pull/10168>`_, Patrick Donnelly)
+* cephfs: client: note order of member init in cons (`pr#10169 <http://github.com/ceph/ceph/pull/10169>`_, Patrick Donnelly)
+* cephfs: client: properly set inode number of created inode in replay request (`issue#17172 <http://tracker.ceph.com/issues/17172>`_, `pr#10957 <http://github.com/ceph/ceph/pull/10957>`_, Yan, Zheng)
+* cephfs: client: protect InodeRef with client_lock (`issue#17392 <http://tracker.ceph.com/issues/17392>`_, `pr#11225 <http://github.com/ceph/ceph/pull/11225>`_, Yan, Zheng)
+* cephfs: doc/mds: fixup mds doc (`pr#10573 <http://github.com/ceph/ceph/pull/10573>`_, huanwen ren)
+* cephfs: fuse_ll: fix incorrect error settings of fuse_ll_mkdir() (`pr#9809 <http://github.com/ceph/ceph/pull/9809>`_, xie xingguo)
+* cephfs: include/ceph_fs.h: guard `#define CEPH_SETATTR_*` with #ifndef (`pr#10265 <http://github.com/ceph/ceph/pull/10265>`_, Kefu Chai)
+* cephfs: libcephfs: Fix the incorrect integer conversion in libcephfs_jni.cc (`pr#10640 <http://github.com/ceph/ceph/pull/10640>`_, wenjunhuang)
+* cephfs: libcephfs: add unmount function in cephfs.pyx (`pr#10774 <http://github.com/ceph/ceph/pull/10774>`_, huanwen ren)
+* cephfs: libcephfs: fix portability-related error settings (`pr#9794 <http://github.com/ceph/ceph/pull/9794>`_, xie xingguo)
+* cephfs: libcephfs: kill compiling warning (`pr#10622 <http://github.com/ceph/ceph/pull/10622>`_, xie xingguo)
+* cephfs: mds/CDir: remove the part of judgment for _next_dentry_on_set (`pr#10476 <http://github.com/ceph/ceph/pull/10476>`_, zhang.zezhu)
+* cephfs: mds/CInode: fix potential fin hanging (`pr#9773 <http://github.com/ceph/ceph/pull/9773>`_, xie xingguo)
+* cephfs: mds/MDBalancer: cleanup (`pr#10512 <http://github.com/ceph/ceph/pull/10512>`_, huanwen ren)
+* cephfs: mds/MDCache: kill a comipler warning (`pr#11254 <http://github.com/ceph/ceph/pull/11254>`_, xie xingguo)
+* cephfs: mds/MDSMap default metadata pool to -1 (was: output None instead of 0 when no fs present.) (`issue#16588 <http://tracker.ceph.com/issues/16588>`_, `pr#10202 <http://github.com/ceph/ceph/pull/10202>`_, Xiaoxi Chen)
+* cephfs: mds/MDSTable: add const to member functions (`pr#10846 <http://github.com/ceph/ceph/pull/10846>`_, Michal Jarzabek)
+* cephfs: mds/SessionMap.h: change statement to assertion (`pr#11289 <http://github.com/ceph/ceph/pull/11289>`_, Michal Jarzabek)
+* cephfs: mds/SnapRealm.h: add const to member functions (`pr#10878 <http://github.com/ceph/ceph/pull/10878>`_, Michal Jarzabek)
+* cephfs: mds/server: clean up handle_client_open() (`pr#11120 <http://github.com/ceph/ceph/pull/11120>`_, huanwen ren)
+* cephfs: mon/MDSMonitor: move C_Updated class to .cc file (`pr#10668 <http://github.com/ceph/ceph/pull/10668>`_, Michal Jarzabek)
+* cephfs: osdc/mds: fixup pos parameter in the journaler (`pr#10200 <http://github.com/ceph/ceph/pull/10200>`_, huanwen ren)
+* cephfs: reduce unnecessary mds log flush (`pr#10393 <http://github.com/ceph/ceph/pull/10393>`_, Yan, Zheng)
+* cephfs: tools/cephfs: Remove cephfs-data-scan tmap_upgrade (`issue#16144 <http://tracker.ceph.com/issues/16144>`_, `pr#10100 <http://github.com/ceph/ceph/pull/10100>`_, Douglas Fuller)
+* cephfs: ceph_fuse: use sizeof get the buf length (`pr#11176 <http://github.com/ceph/ceph/pull/11176>`_, LeoZhang)
+* cli: retry when the mon is not configured (`issue#16477 <http://tracker.ceph.com/issues/16477>`_, `pr#11089 <http://github.com/ceph/ceph/pull/11089>`_, Loic Dachary)
+* cmake: Add -pie to CMAKE_EXE_LINKER_FLAGS (`pr#10755 <http://github.com/ceph/ceph/pull/10755>`_, Tim Serong)
+* cmake: Fix FCGI include directory (`pr#9983 <http://github.com/ceph/ceph/pull/9983>`_, Tim Serong)
+* cmake: Fix mismatched librgw VERSION / SOVERSION (`pr#10754 <http://github.com/ceph/ceph/pull/10754>`_, Tim Serong)
+* cmake: FreeBSD specific excludes in CMakeLists.txt (`pr#10973 <http://github.com/ceph/ceph/pull/10973>`_, Willem Jan Withagen)
+* cmake: FreeBSD specific excludes in CMakeLists.txt files (`pr#10517 <http://github.com/ceph/ceph/pull/10517>`_, Willem Jan Withagen)
+* cmake: Really add FCGI_INCLUDE_DIR to include_directories for rgw (`pr#10139 <http://github.com/ceph/ceph/pull/10139>`_, Tim Serong)
+* cmake: Removed README.cmake.md, edited README.md (`pr#10028 <http://github.com/ceph/ceph/pull/10028>`_, Ali Maredia)
+* cmake: Support tcmalloc_minimal allocator (`pr#11111 <http://github.com/ceph/ceph/pull/11111>`_, Bassam Tabbara)
+* cmake: add dependency from ceph_smalliobenchrbd to cls libraries (`pr#10870 <http://github.com/ceph/ceph/pull/10870>`_, J. Eric Ivancich)
+* cmake: add_subdirectory(include) (`pr#10360 <http://github.com/ceph/ceph/pull/10360>`_, Kefu Chai)
+* cmake: ceph_test_rbd_mirror does not require librados_test_stub (`pr#10164 <http://github.com/ceph/ceph/pull/10164>`_, Jason Dillaman)
+* cmake: cleanup Findgperftools.cmake (`pr#10670 <http://github.com/ceph/ceph/pull/10670>`_, Kefu Chai)
+* cmake: correct ceph_test_librbd/ceph_test_rbd_mirror linkage (`issue#16882 <http://tracker.ceph.com/issues/16882>`_, `pr#10598 <http://github.com/ceph/ceph/pull/10598>`_, Jason Dillaman)
+* cmake: disable -fvar-tracking-assignments for ceph_dencoder.cc (`pr#10275 <http://github.com/ceph/ceph/pull/10275>`_, Kefu Chai)
+* cmake: disable unittest_async_compressor (`pr#10394 <http://github.com/ceph/ceph/pull/10394>`_, Kefu Chai)
+* cmake: do not link against unused objects or libraries (`pr#10837 <http://github.com/ceph/ceph/pull/10837>`_, Kefu Chai)
+* cmake: enable ccache for rocksdb too (`pr#11100 <http://github.com/ceph/ceph/pull/11100>`_, Bassam Tabbara)
+* cmake: exclude non-public symbols in shared libraries (`issue#16556 <http://tracker.ceph.com/issues/16556>`_, `pr#10472 <http://github.com/ceph/ceph/pull/10472>`_, Kefu Chai)
+* cmake: fix incorrect dependencies to librados (`pr#10145 <http://github.com/ceph/ceph/pull/10145>`_, Jason Dillaman)
+* cmake: fix the FTBFS introduced by dc8b3ba (`pr#10282 <http://github.com/ceph/ceph/pull/10282>`_, Kefu Chai)
+* cmake: fix the build of unittest_async_compressor (`pr#10400 <http://github.com/ceph/ceph/pull/10400>`_, Kefu Chai)
+* cmake: fix the tracing header dependencies (`pr#10906 <http://github.com/ceph/ceph/pull/10906>`_, Kefu Chai)
+* cmake: fix unittest_rbd_mirror failures under non-optimized builds (`pr#9990 <http://github.com/ceph/ceph/pull/9990>`_, Jason Dillaman)
+* cmake: fix wrong path introduced by bb163e9 (`pr#10643 <http://github.com/ceph/ceph/pull/10643>`_, Kefu Chai)
+* cmake: fixes (`pr#10092 <http://github.com/ceph/ceph/pull/10092>`_, Daniel Gryniewicz)
+* cmake: fixes for pypi changes (`pr#10204 <http://github.com/ceph/ceph/pull/10204>`_, Kefu Chai)
+* cmake: include(SIMDExt) in src/CMakeLists.txt (`pr#11003 <http://github.com/ceph/ceph/pull/11003>`_, Kefu Chai)
+* cmake: install ceph_test_cls_rgw (`pr#10025 <http://github.com/ceph/ceph/pull/10025>`_, Kefu Chai)
+* cmake: install ceph_test_rados_striper_api_* (`pr#10541 <http://github.com/ceph/ceph/pull/10541>`_, Kefu Chai)
+* cmake: install platlib into a subdir of build-base dir (`pr#10666 <http://github.com/ceph/ceph/pull/10666>`_, Kefu Chai)
+* cmake: make py3 a nice-to-have (`issue#17103 <http://tracker.ceph.com/issues/17103>`_, `pr#11015 <http://github.com/ceph/ceph/pull/11015>`_, Kefu Chai)
+* cmake: pass -DINTEL* to gf-complete cflags (`pr#10956 <http://github.com/ceph/ceph/pull/10956>`_, tone.zhang, Kefu Chai)
+* cmake: pass cmake's compiler and flags to compile RocksDB into build (`pr#10418 <http://github.com/ceph/ceph/pull/10418>`_, Willem Jan Withagen)
+* cmake: recompile erasure src for different variants (`pr#10772 <http://github.com/ceph/ceph/pull/10772>`_, Kefu Chai)
+* cmake: remove WITH_MDS option (`pr#10186 <http://github.com/ceph/ceph/pull/10186>`_, Ali Maredia)
+* cmake: remove more autotools hacks (`pr#11229 <http://github.com/ceph/ceph/pull/11229>`_, Sage Weil)
+* cmake: remove unnecessary linked libs from libcephfs (`issue#16556 <http://tracker.ceph.com/issues/16556>`_, `pr#10081 <http://github.com/ceph/ceph/pull/10081>`_, Kefu Chai)
+* cmake: rework NSS and SSL (`pr#9831 <http://github.com/ceph/ceph/pull/9831>`_, Matt Benjamin)
+* cmake: set ARM_CRC_FLAGS from the CRC test rather than ARM_NEON_FLAGS (`issue#17250 <http://tracker.ceph.com/issues/17250>`_, `pr#11028 <http://github.com/ceph/ceph/pull/11028>`_, Dan Mick)
+* cmake: specify distutils build path explicitly (`pr#10568 <http://github.com/ceph/ceph/pull/10568>`_, Kefu Chai)
+* cmake: supress more warnings (`pr#10469 <http://github.com/ceph/ceph/pull/10469>`_, Willem Jan Withagen)
+* cmake: use PERF_LOCAL_FLAGS only if defined (`issue#17104 <http://tracker.ceph.com/issues/17104>`_, `pr#10828 <http://github.com/ceph/ceph/pull/10828>`_, Michel Normand)
+* cmake: use stock Find* modules. (`pr#10178 <http://github.com/ceph/ceph/pull/10178>`_, Kefu Chai)
+* cmake: work to get inital FreeBSD stuff (`pr#10352 <http://github.com/ceph/ceph/pull/10352>`_, Willem Jan Withagen)
+* cmake: find GIT_VER variables if there is no .git dir (`pr#11499 <http://github.com/ceph/ceph/pull/11499>`_, Ali Maredia)
+* common,bluestore: Isa-l extention for zlib compression plugin (`pr#10158 <http://github.com/ceph/ceph/pull/10158>`_, Alyona Kiseleva, Dan Mick)
+* common,bluestore: compressor/zlib: zlib wrapper fix (`pr#11079 <http://github.com/ceph/ceph/pull/11079>`_, Igor Fedotov)
+* common: auth/cephx: misc fixes (`pr#9679 <http://github.com/ceph/ceph/pull/9679>`_, xie xingguo)
+* common: common/PluginRegistry: improve error output for shared library load fa… (`pr#11081 <http://github.com/ceph/ceph/pull/11081>`_, Igor Fedotov)
+* common: common/Throttle.h: remove unneeded class (`pr#10902 <http://github.com/ceph/ceph/pull/10902>`_, Michal Jarzabek)
+* common: common/Timer.h: delete copy constr and assign op (`pr#11046 <http://github.com/ceph/ceph/pull/11046>`_, Michal Jarzabek)
+* common: common/WorkQueue: add std move (`pr#9729 <http://github.com/ceph/ceph/pull/9729>`_, Michal Jarzabek)
+* common: compressor: zlib compressor plugin  cleanup (`pr#9782 <http://github.com/ceph/ceph/pull/9782>`_, Alyona Kiseleva)
+* common: erasure-code: Runtime detection of SIMD for jerasure and shec (`pr#11086 <http://github.com/ceph/ceph/pull/11086>`_, Bassam Tabbara)
+* common: global: log which process/command sent a signal (`pr#8964 <http://github.com/ceph/ceph/pull/8964>`_, song baisen)
+* common: include/assert: clean up ceph assertion macros (`pr#9969 <http://github.com/ceph/ceph/pull/9969>`_, Sage Weil)
+* common: instantiate strict_si_cast<long> not strict_si_cast<int64_t> (`issue#16398 <http://tracker.ceph.com/issues/16398>`_, `pr#9934 <http://github.com/ceph/ceph/pull/9934>`_, Kefu Chai)
+* common: lockdep: verbose even if no logging is set (`pr#10576 <http://github.com/ceph/ceph/pull/10576>`_, Willem Jan Withagen)
+* common: messages/MOSDMap: mark as enlighten OSDMap encoder (`pr#10843 <http://github.com/ceph/ceph/pull/10843>`_, Sage Weil)
+* common: mon/Monitor.cc:replce lock/unlock with Mutex:Lockr (`pr#9792 <http://github.com/ceph/ceph/pull/9792>`_, Michal Jarzabek)
+* common: msg/AsyncMessenger.cc: remove code duplication (`pr#10030 <http://github.com/ceph/ceph/pull/10030>`_, Michal Jarzabek)
+* common: msg/async: less verbose debug messages at debug_ms=1 (`pr#11205 <http://github.com/ceph/ceph/pull/11205>`_, Sage Weil)
+* common: msg/async: remove static member variable (`issue#16686 <http://tracker.ceph.com/issues/16686>`_, `pr#10440 <http://github.com/ceph/ceph/pull/10440>`_, Kefu Chai)
+* common: only call crypto::init once per CephContext (`issue#17205 <http://tracker.ceph.com/issues/17205>`_, `pr#10965 <http://github.com/ceph/ceph/pull/10965>`_, Casey Bodley)
+* common: osdc/ObjectCacher: change iterator to const_iterator and add const to member functions (`pr#9644 <http://github.com/ceph/ceph/pull/9644>`_, Michal Jarzabek)
+* common: preforker: prevent call to 'write' on an fd that was already closed (`pr#10949 <http://github.com/ceph/ceph/pull/10949>`_, Avner BenHanoch)
+* common: remove basename() dependency (`pr#9845 <http://github.com/ceph/ceph/pull/9845>`_, John Coyle)
+* common: src/common/buffer.cc fix judgment for lseek (`pr#10130 <http://github.com/ceph/ceph/pull/10130>`_, zhang.zezhu)
+* common: unknown hash type of judgment modification (`pr#9510 <http://github.com/ceph/ceph/pull/9510>`_, huanwen ren)
+* common: Timer.cc: replace long types with auto (`pr#11067 <http://github.com/ceph/ceph/pull/11067>`_, Michal Jarzabek)
+* common: TrackedOp: move ShardedTrackingData to .cc (`pr#10639 <http://github.com/ceph/ceph/pull/10639>`_, Michal Jarzabek)
+* common: config_opts: fix comment(radio -> ratio) (`pr#10783 <http://github.com/ceph/ceph/pull/10783>`_, xie xingguo)
+* common: src/common/dns_resolve.cc: reorder the includes (`pr#10505 <http://github.com/ceph/ceph/pull/10505>`_, Willem Jan Withagen)
+* common: global/signal_handler: use sig_str instead of sys_siglist (`pr#10633 <http://github.com/ceph/ceph/pull/10633>`_, John Coyle)
+* core,cephfs: Revert "osd/ReplicatedPG: for sync-read it don't cacl l_osd_op_r_prep… (`issue#16908 <http://tracker.ceph.com/issues/16908>`_, `pr#10875 <http://github.com/ceph/ceph/pull/10875>`_, Samuel Just)
+* core,cephfs: mon/mds: add err info when load_metadata is abnormal (`pr#10176 <http://github.com/ceph/ceph/pull/10176>`_, huanwen ren)
+* core,common: osd/OSD.cc: remove unneeded returns (`pr#11043 <http://github.com/ceph/ceph/pull/11043>`_, Michal Jarzabek)
+* core,pybind: python-rados: extends ReadOp/WriteOp API (`pr#9944 <http://github.com/ceph/ceph/pull/9944>`_, Mehdi Abaakouk)
+* core,pybind: python-rados: implement new aio_stat. (`pr#11006 <http://github.com/ceph/ceph/pull/11006>`_, Iain Buclaw)
+* core,pybind: qa/workunits/rados/test_python.sh: Allow specifying Python executable (`pr#10782 <http://github.com/ceph/ceph/pull/10782>`_, Oleh Prypin)
+* core:   os/filestore/LFNIndex: remove unused variable 'subdir_path' (`pr#8959 <http://github.com/ceph/ceph/pull/8959>`_, huangjun)
+* core: Create ceph-mgr (`pr#10328 <http://github.com/ceph/ceph/pull/10328>`_, John Spray, Tim Serong)
+* core: FileJournal: Remove obsolete _check_disk_write_cache function (`pr#11073 <http://github.com/ceph/ceph/pull/11073>`_, Brad Hubbard)
+* core: Lua object class support (`pr#7338 <http://github.com/ceph/ceph/pull/7338>`_, Noah Watkins)
+* core: OSD crash with Hammer to Jewel Upgrade: void FileStore::init_temp_collections() (`issue#16672 <http://tracker.ceph.com/issues/16672>`_, `pr#10565 <http://github.com/ceph/ceph/pull/10565>`_, David Zafman)
+* core: OSD.cc: remove unneeded return (`pr#9701 <http://github.com/ceph/ceph/pull/9701>`_, Michal Jarzabek)
+* core: OSD: avoid FileStore finisher deadlock in osd_lock when shutdown OSD (`pr#11052 <http://github.com/ceph/ceph/pull/11052>`_, Haomai Wang)
+* core: ObjectCacher: fix last_write check in bh_write_adjacencies() (`issue#16610 <http://tracker.ceph.com/issues/16610>`_, `pr#10304 <http://github.com/ceph/ceph/pull/10304>`_, Yan, Zheng)
+* core: ReplicatedPG: call op_applied for submit_log_entries based repops (`pr#9489 <http://github.com/ceph/ceph/pull/9489>`_, Samuel Just)
+* core: Wip 16998 (`issue#16998 <http://tracker.ceph.com/issues/16998>`_, `pr#10688 <http://github.com/ceph/ceph/pull/10688>`_, Samuel Just)
+* core: ceph-create-keys: add missing argument comma (`pr#11123 <http://github.com/ceph/ceph/pull/11123>`_, Patrick Donnelly)
+* core: ceph-create-keys: fix existing-but-different case (`issue#16255 <http://tracker.ceph.com/issues/16255>`_, `pr#10415 <http://github.com/ceph/ceph/pull/10415>`_, John Spray)
+* core: ceph-disk: partprobe should block udev induced BLKRRPART (`issue#15176 <http://tracker.ceph.com/issues/15176>`_, `pr#9330 <http://github.com/ceph/ceph/pull/9330>`_, Marius Vollmer, Loic Dachary)
+* core: ceph-disk: timeout ceph-disk to avoid blocking forever (`issue#16580 <http://tracker.ceph.com/issues/16580>`_, `pr#10262 <http://github.com/ceph/ceph/pull/10262>`_, Loic Dachary)
+* core: ceph-objectstore-tool: add a way to split filestore directories offline (`issue#17220 <http://tracker.ceph.com/issues/17220>`_, `pr#10776 <http://github.com/ceph/ceph/pull/10776>`_, Josh Durgin)
+* core: ceph.in: python 3 compatibility of the ceph CLI (`pr#9702 <http://github.com/ceph/ceph/pull/9702>`_, Oleh Prypin)
+* core: ceph_mon: use readdir() as readdir_r() is deprecated (`pr#11047 <http://github.com/ceph/ceph/pull/11047>`_, Kefu Chai)
+* core: cephx: Fix multiple segfaults due to attempts to encrypt or decrypt (`issue#16266 <http://tracker.ceph.com/issues/16266>`_, `pr#9703 <http://github.com/ceph/ceph/pull/9703>`_, Brad Hubbard)
+* core: https://github.com/ceph/ceph/pull/11052 (`pr#10371 <http://github.com/ceph/ceph/pull/10371>`_, Yan Jun)
+* core: include write error codes in the pg log (`issue#14468 <http://tracker.ceph.com/issues/14468>`_, `pr#10170 <http://github.com/ceph/ceph/pull/10170>`_, Josh Durgin)
+* core: kv/MemDB: fix assert triggerred by m_total_bytes underflow (`pr#10471 <http://github.com/ceph/ceph/pull/10471>`_, xie xingguo)
+* core: kv/RocksDB: add perfcounter for submit_transaction_sync operation (`pr#9770 <http://github.com/ceph/ceph/pull/9770>`_, Haodong Tang)
+* core: logmon: check is_leader() before doing any work on get_trim_to() (`pr#10342 <http://github.com/ceph/ceph/pull/10342>`_, song baisen)
+* core: memstore: clone zero-fills holes from source range (`pr#11157 <http://github.com/ceph/ceph/pull/11157>`_, Casey Bodley)
+* core: message: optimization for message priority strategy (`pr#8687 <http://github.com/ceph/ceph/pull/8687>`_, yaoning)
+* core: messages/MForward: fix encoding features (`issue#17365 <http://tracker.ceph.com/issues/17365>`_, `pr#11180 <http://github.com/ceph/ceph/pull/11180>`_, Sage Weil)
+* core: mgr/MgrClient: fix ms_handle_reset (`pr#11298 <http://github.com/ceph/ceph/pull/11298>`_, Sage Weil)
+* core: mgr/MgrMap: initialize all fields (`issue#17492 <http://tracker.ceph.com/issues/17492>`_, `pr#11308 <http://github.com/ceph/ceph/pull/11308>`_, Sage Weil)
+* core: mon/ConfigKeyService: pass strings by const ref (`pr#10618 <http://github.com/ceph/ceph/pull/10618>`_, Michal Jarzabek)
+* core: mon/LogMonitor: move C_Log struct to cc file (`pr#10721 <http://github.com/ceph/ceph/pull/10721>`_, Michal Jarzabek)
+* core: mon/MonClient.h: pass strings by const reference (`pr#10605 <http://github.com/ceph/ceph/pull/10605>`_, Michal Jarzabek)
+* core: mon/MonDBStore: fix assert which never fires (`pr#10706 <http://github.com/ceph/ceph/pull/10706>`_, xie xingguo)
+* core: mon/MonitorDBStore: do not use snapshot iterator; close on close (`pr#10102 <http://github.com/ceph/ceph/pull/10102>`_, Sage Weil)
+* core: mon/OSDMonitor.cc: remove use of boost assign (`pr#11060 <http://github.com/ceph/ceph/pull/11060>`_, Michal Jarzabek)
+* core: mon/PGMonitor: batch filter pg states; add sanity check (`pr#9394 <http://github.com/ceph/ceph/pull/9394>`_, xie xingguo)
+* core: mon/PGMonitor: calc the %USED of pool using used/(used+avail) (`issue#16933 <http://tracker.ceph.com/issues/16933>`_, `pr#10584 <http://github.com/ceph/ceph/pull/10584>`_, Kefu Chai)
+* core: mon/PGMonitor: move C_Stats struct to cc file (`pr#10719 <http://github.com/ceph/ceph/pull/10719>`_, Michal Jarzabek)
+* core: mon/PaxosService: make the return value type inconsistent (`pr#10231 <http://github.com/ceph/ceph/pull/10231>`_, zhang.zezhu)
+* core: mon/osdmonitor: fix incorrect output of "osd df" due to osd out (`issue#16706 <http://tracker.ceph.com/issues/16706>`_, `pr#10308 <http://github.com/ceph/ceph/pull/10308>`_, xie xingguo)
+* core: msg/AsyncMessenger: change return type to void (`pr#10230 <http://github.com/ceph/ceph/pull/10230>`_, Michal Jarzabek)
+* core: msg/Messenger: add const and override to function (`pr#10183 <http://github.com/ceph/ceph/pull/10183>`_, Michal Jarzabek)
+* core: msg/async/AsyncConnection: replace Mutex with std::mutex for peformance (`issue#16714 <http://tracker.ceph.com/issues/16714>`_, `issue#16715 <http://tracker.ceph.com/issues/16715>`_, `pr#10340 <http://github.com/ceph/ceph/pull/10340>`_, Haomai Wang)
+* core: msg/async/Event: ensure not refer to member variable which may destroyed (`issue#16714 <http://tracker.ceph.com/issues/16714>`_, `pr#10369 <http://github.com/ceph/ceph/pull/10369>`_, Haomai Wang)
+* core: msg/async/kqueue: avoid remove nonexist kqueue event (`pr#9869 <http://github.com/ceph/ceph/pull/9869>`_, Haomai Wang)
+* core: msg/async: Support close idle connection feature  (`issue#16366 <http://tracker.ceph.com/issues/16366>`_, `pr#9783 <http://github.com/ceph/ceph/pull/9783>`_, Haomai Wang)
+* core: msg/async: allow other async backend implementations (`pr#10264 <http://github.com/ceph/ceph/pull/10264>`_, Haomai Wang)
+* core: msg/async: avoid set out of range ms_async_op_threads option (`pr#11200 <http://github.com/ceph/ceph/pull/11200>`_, Haomai Wang)
+* core: msg/async: connect authorizer fix + recv_buf size (`pr#9784 <http://github.com/ceph/ceph/pull/9784>`_, Ilya Dryomov)
+* core: msg/async: harden error logic handle (`pr#9781 <http://github.com/ceph/ceph/pull/9781>`_, Haomai Wang)
+* core: msg/async: remove fd output in log prefix (`pr#11199 <http://github.com/ceph/ceph/pull/11199>`_, Haomai Wang)
+* core: msg/async: remove file event lock (`issue#16554 <http://tracker.ceph.com/issues/16554>`_, `issue#16552 <http://tracker.ceph.com/issues/16552>`_, `pr#10090 <http://github.com/ceph/ceph/pull/10090>`_, Haomai Wang)
+* core: msg/simple/Pipe: eliminating casts for the comparing of len and recv_max_prefetch (`pr#10273 <http://github.com/ceph/ceph/pull/10273>`_, zhang.zezhu)
+* core: msg/simple: fix wrong condition checking of writing TAG_CLOSE on closing (`pr#10343 <http://github.com/ceph/ceph/pull/10343>`_, xie xingguo)
+* core: msg/simple: wait dispatch_queue until all pipes closed (`issue#16472 <http://tracker.ceph.com/issues/16472>`_, `pr#9930 <http://github.com/ceph/ceph/pull/9930>`_, Haomai Wang)
+* core: msg: make async backend default (`pr#10746 <http://github.com/ceph/ceph/pull/10746>`_, Haomai Wang)
+* core: msg: mark daemons down on RST + ECONNREFUSED (`pr#8558 <http://github.com/ceph/ceph/pull/8558>`_, Piotr Dałek)
+* core: os/FuseStore: fix several FuseStore issues (`pr#10723 <http://github.com/ceph/ceph/pull/10723>`_, Sage Weil)
+* core: os/MemStore: move BufferlistObject to .cc file (`pr#10833 <http://github.com/ceph/ceph/pull/10833>`_, Michal Jarzabek)
+* core: os/ObjectStore: fix return code of collection_empty() method (`pr#11050 <http://github.com/ceph/ceph/pull/11050>`_, xie xingguo)
+* core: os/RocksDBStore: use effective Get API instead of iterator api (`pr#9411 <http://github.com/ceph/ceph/pull/9411>`_, Jianjian Huo, Haomai Wang, Mark Nelson)
+* core: os/filestore/FDCache: fix bug when filestore_fd_cache_shards = 0 (`pr#11048 <http://github.com/ceph/ceph/pull/11048>`_, jimifm)
+* core: os/filestore/FileJournal: error out if FileJournal is not a file (`issue#17307 <http://tracker.ceph.com/issues/17307>`_, `pr#11146 <http://github.com/ceph/ceph/pull/11146>`_, Kefu Chai)
+* core: os/filestore: add sanity checks and cleanups for mount() process (`pr#9734 <http://github.com/ceph/ceph/pull/9734>`_, xie xingguo)
+* core: os/filestore: disable use of splice by default (`pr#11113 <http://github.com/ceph/ceph/pull/11113>`_, Haomai Wang)
+* core: osd/OSD.cc: remove repeated searching of map (`pr#10986 <http://github.com/ceph/ceph/pull/10986>`_, Michal Jarzabek)
+* core: osd/OSD.cc: remove unneeded searching of maps (`pr#11039 <http://github.com/ceph/ceph/pull/11039>`_, Michal Jarzabek)
+* core: osd/OSD.h: add const to member functions (`pr#11114 <http://github.com/ceph/ceph/pull/11114>`_, Michal Jarzabek)
+* core: osd/OSD.h: move some members under private (`pr#11121 <http://github.com/ceph/ceph/pull/11121>`_, Michal Jarzabek)
+* core: osd/OSD.h: remove unneeded line (`pr#8980 <http://github.com/ceph/ceph/pull/8980>`_, Michal Jarzabek)
+* core: osd/OSDMonitor: misc. cleanups (`pr#10739 <http://github.com/ceph/ceph/pull/10739>`_, xie xingguo)
+* core: osd/OSDMonitor: misc. fixes (`pr#10491 <http://github.com/ceph/ceph/pull/10491>`_, xie xingguo)
+* core: osd/ReplicatedBackend: add sanity check during build_push_op() (`pr#9491 <http://github.com/ceph/ceph/pull/9491>`_, Yan Jun)
+* core: osd/ReplicatedPG: for sync-read it don't cacl l_osd_op_r_prepare_lat. (`pr#10365 <http://github.com/ceph/ceph/pull/10365>`_, Jianpeng Ma)
+* core: osd/ReplicatedPG: remove class redeclaration (`pr#11041 <http://github.com/ceph/ceph/pull/11041>`_, Michal Jarzabek)
+* core: osd/ReplicatedPG: remove unused param "op" from generate_subop() (`pr#10811 <http://github.com/ceph/ceph/pull/10811>`_, jimifm)
+* core: osd/Watch: add consts to member functions (`pr#10251 <http://github.com/ceph/ceph/pull/10251>`_, Michal Jarzabek)
+* core: osd/osd_type: check if pool is gone during check_new_interval()  (`pr#10859 <http://github.com/ceph/ceph/pull/10859>`_, xie xingguo)
+* core: osd/osdmonitor: pool of objects and bytes beyond quota should all be warn (`pr#9085 <http://github.com/ceph/ceph/pull/9085>`_, huanwen ren)
+* core: osdc/objecter: misc fixes (`pr#10826 <http://github.com/ceph/ceph/pull/10826>`_, xie xingguo)
+* core: pass string by const ref and add override to virtual function (`pr#9082 <http://github.com/ceph/ceph/pull/9082>`_, Michal Jarzabek)
+* core: qa/workunits/objectstore/test_fuse.sh: make test_fuse.sh work with filestore (`pr#11057 <http://github.com/ceph/ceph/pull/11057>`_, Sage Weil)
+* core: rados: add option to include clones when doing flush or evict (`pr#9698 <http://github.com/ceph/ceph/pull/9698>`_, Mingxin Liu)
+* core: subman: use replace instead of format (`issue#16961 <http://tracker.ceph.com/issues/16961>`_, `pr#10620 <http://github.com/ceph/ceph/pull/10620>`_, Loic Dachary)
+* core: test/common/Throttle.cc: fix race in shutdown (`pr#10094 <http://github.com/ceph/ceph/pull/10094>`_, Samuel Just)
+* core: test: add the necessary judgment (`pr#9694 <http://github.com/ceph/ceph/pull/9694>`_, huanwen ren)
+* core: tox.ini: remove extraneous coverage --omit option (`pr#10943 <http://github.com/ceph/ceph/pull/10943>`_, Josh Durgin)
+* core: udev: always populate /dev/disk/by-parttypeuuid (`issue#16351 <http://tracker.ceph.com/issues/16351>`_, `pr#9885 <http://github.com/ceph/ceph/pull/9885>`_, Loic Dachary)
+* core: os/FuseStore: remove unneeded header file (`pr#10799 <http://github.com/ceph/ceph/pull/10799>`_, Michal Jarzabek)
+* core: os/MemStore: move OmapIteratorImpl to cc file (`pr#10803 <http://github.com/ceph/ceph/pull/10803>`_, Michal Jarzabek)
+* core: os/Memstore.h: add override to virtual functions (`pr#10801 <http://github.com/ceph/ceph/pull/10801>`_, Michal Jarzabek)
+* core: os/Memstore: move PageSetObject class to .cc file (`pr#10817 <http://github.com/ceph/ceph/pull/10817>`_, Michal Jarzabek)
+* core: os/bluestore: remove unused head file. (`pr#11186 <http://github.com/ceph/ceph/pull/11186>`_, Jianpeng Ma)
+* core: safe_io: Improve portability by replacing loff_t type usage with off_t. (`pr#9767 <http://github.com/ceph/ceph/pull/9767>`_, John Coyle)
+* core: src/kv/MemDB.cc: the type of the parameter of push_back() does not match the ops's value_type (`pr#10455 <http://github.com/ceph/ceph/pull/10455>`_, Willem Jan Withagen)
+* core: msg/simple: apply prefetch policy more precisely (`pr#10344 <http://github.com/ceph/ceph/pull/10344>`_, xie xingguo)
+* core: CompatSet.h: remove unneeded inline (`pr#10071 <http://github.com/ceph/ceph/pull/10071>`_, Michal Jarzabek)
+* core: Objclass perm feedback (`pr#10313 <http://github.com/ceph/ceph/pull/10313>`_, Noah Watkins)
+* core: arch/arm.c: remove unnecessary variable read for simplicity (`pr#10821 <http://github.com/ceph/ceph/pull/10821>`_, Weibing Zhang)
+* crush: don't normalize input of crush_ln iteratively (`pr#10935 <http://github.com/ceph/ceph/pull/10935>`_, Piotr Dałek)
+* crush: reset bucket->h.items[i] when removing tree item (`issue#16525 <http://tracker.ceph.com/issues/16525>`_, `pr#10093 <http://github.com/ceph/ceph/pull/10093>`_, Kefu Chai)
+* crush: CrushCompiler.cc:884 (`pr#10952 <http://github.com/ceph/ceph/pull/10952>`_, xu biao)
+* crush: CrushCompiler: error out as long as parse fails (`issue#17306 <http://tracker.ceph.com/issues/17306>`_, `pr#11144 <http://github.com/ceph/ceph/pull/11144>`_, Kefu Chai)
+* doc: Add documentation about snapshots (`pr#10436 <http://github.com/ceph/ceph/pull/10436>`_, Greg Farnum)
+* doc: Add two options to radosgw-admin.rst manpage (`issue#17281 <http://tracker.ceph.com/issues/17281>`_, `pr#11134 <http://github.com/ceph/ceph/pull/11134>`_, Thomas Serlin)
+* doc: Changed config parameter "rgw keystone make new tenants" in radosgw multitenancy (`issue#17293 <http://tracker.ceph.com/issues/17293>`_, `pr#11127 <http://github.com/ceph/ceph/pull/11127>`_, SirishaGuduru)
+* doc: Modification for "TEST S3 ACCESS" section  in "INSTALL CEPH OBJECT GATEWAY" page (`pr#9089 <http://github.com/ceph/ceph/pull/9089>`_, la-sguduru)
+* doc: Update developer docs for cmake paths (`pr#11163 <http://github.com/ceph/ceph/pull/11163>`_, John Spray)
+* doc: add "--orphan-stale-secs" to radosgw-admin(8) (`issue#17280 <http://tracker.ceph.com/issues/17280>`_, `pr#11097 <http://github.com/ceph/ceph/pull/11097>`_, Ken Dreyer)
+* doc: add $pid metavar conf doc (`pr#11172 <http://github.com/ceph/ceph/pull/11172>`_, Patrick Donnelly)
+* doc: add Backporting section to Essentials chapter (`issue#15497 <http://tracker.ceph.com/issues/15497>`_, `pr#10457 <http://github.com/ceph/ceph/pull/10457>`_, Nathan Cutler)
+* doc: add Prepare tenant section to Testing in the cloud chapter (`pr#10413 <http://github.com/ceph/ceph/pull/10413>`_, Nathan Cutler)
+* doc: add Upload logs to archive server section... (`pr#10414 <http://github.com/ceph/ceph/pull/10414>`_, Nathan Cutler)
+* doc: add client config ref (`issue#16743 <http://tracker.ceph.com/issues/16743>`_, `pr#10434 <http://github.com/ceph/ceph/pull/10434>`_, Patrick Donnelly)
+* doc: add graphic for cap bit field (`pr#10897 <http://github.com/ceph/ceph/pull/10897>`_, Patrick Donnelly)
+* doc: add missing PR to hammer 0.94.8 release notes (`pr#10900 <http://github.com/ceph/ceph/pull/10900>`_, Nathan Cutler)
+* doc: add openSUSE instructions to quick-start-preflight (`pr#10454 <http://github.com/ceph/ceph/pull/10454>`_, Nathan Cutler)
+* doc: add rgw_enable_usage_log option in Rados Gateway admin guide (`issue#16604 <http://tracker.ceph.com/issues/16604>`_, `pr#10159 <http://github.com/ceph/ceph/pull/10159>`_, Mike Hackett)
+* doc: add troubleshooting steps for ceph-fuse (`pr#10374 <http://github.com/ceph/ceph/pull/10374>`_, Ken Dreyer)
+* doc: admin/build-doc: bypass sanity check if building doc (`issue#16940 <http://tracker.ceph.com/issues/16940>`_, `pr#10623 <http://github.com/ceph/ceph/pull/10623>`_, Kefu Chai)
+* doc: ceph-authtool man page option is --print-key not --print (`pr#9731 <http://github.com/ceph/ceph/pull/9731>`_, Brad Hubbard)
+* doc: ceph-deploy mon add doesn't take multiple nodes (`pr#10085 <http://github.com/ceph/ceph/pull/10085>`_, Chengwei Yang)
+* doc: clarify rbd size units (`pr#11303 <http://github.com/ceph/ceph/pull/11303>`_, Ilya Dryomov)
+* doc: cleanup outdated radosgw description (`pr#11248 <http://github.com/ceph/ceph/pull/11248>`_, Jiaying Ren)
+* doc: describe libvirt client logging (`pr#10542 <http://github.com/ceph/ceph/pull/10542>`_, Ken Dreyer)
+* doc: do not list all major versions in get-packages.rst (`pr#10899 <http://github.com/ceph/ceph/pull/10899>`_, Nathan Cutler)
+* doc: doc/cephfs: explain the various health messages (`pr#10244 <http://github.com/ceph/ceph/pull/10244>`_, John Spray)
+* doc: doc/dev: Fix missing code section due to no lexer for "none" (`pr#9083 <http://github.com/ceph/ceph/pull/9083>`_, Brad Hubbard)
+* doc: doc/radosgw: fix description of response elements 'Part' (`pr#10641 <http://github.com/ceph/ceph/pull/10641>`_, weiqiaomiao)
+* doc: doc/radosgw: rename config.rst to config-fcgi.rst (`pr#10381 <http://github.com/ceph/ceph/pull/10381>`_, Nathan Cutler)
+* doc: extend the CephFS troubleshooting guide (`pr#10458 <http://github.com/ceph/ceph/pull/10458>`_, Greg Farnum)
+* doc: fix broken link in SHEC erasure code plugin (`issue#16996 <http://tracker.ceph.com/issues/16996>`_, `pr#10675 <http://github.com/ceph/ceph/pull/10675>`_, Albert Tu)
+* doc: fix description for rsize and rasize (`pr#11101 <http://github.com/ceph/ceph/pull/11101>`_, Andreas Gerstmayr)
+* doc: fix rados/configuration/osd-config-ref.rst (`pr#10619 <http://github.com/ceph/ceph/pull/10619>`_, Chengwei Yang)
+* doc: fix singleton example in Developer Guide (`pr#10830 <http://github.com/ceph/ceph/pull/10830>`_, Nathan Cutler)
+* doc: fix some nits in release notes and releases table (`pr#10903 <http://github.com/ceph/ceph/pull/10903>`_, Nathan Cutler)
+* doc: fix standby replay config (`issue#16664 <http://tracker.ceph.com/issues/16664>`_, `pr#10268 <http://github.com/ceph/ceph/pull/10268>`_, Patrick Donnelly)
+* doc: fix wrong osdkeepalive name in mount.ceph manpage (`pr#10840 <http://github.com/ceph/ceph/pull/10840>`_, Zhi Zhang)
+* doc: fix/add changelog for 10.2.2, 0.94.7, 0.94.8 (`pr#10895 <http://github.com/ceph/ceph/pull/10895>`_, Sage Weil)
+* doc: format 2 now is the default image format (`pr#10705 <http://github.com/ceph/ceph/pull/10705>`_, Chengwei Yang)
+* doc: lgtm (build verified f23) (`pr#9745 <http://github.com/ceph/ceph/pull/9745>`_, weiqiaomiao)
+* doc: mailmap updates for upcoming 11.0.0 (`pr#9301 <http://github.com/ceph/ceph/pull/9301>`_, Yann Dupont)
+* doc: manual instructions to set up mds daemon (`pr#11115 <http://github.com/ceph/ceph/pull/11115>`_, Peter Maloney)
+* doc: missing "make vstart" in quick_guide.rst (`pr#11226 <http://github.com/ceph/ceph/pull/11226>`_, Leo Zhang)
+* doc: more details for pool deletion (`pr#10190 <http://github.com/ceph/ceph/pull/10190>`_, Ken Dreyer)
+* doc: peering.rst, fix typo (`pr#10131 <http://github.com/ceph/ceph/pull/10131>`_, Brad Hubbard)
+* doc: perf_counters.rst fix trivial typo (`pr#10292 <http://github.com/ceph/ceph/pull/10292>`_, Brad Hubbard)
+* doc: rbdmap: specify bash shell interpreter (`issue#16608 <http://tracker.ceph.com/issues/16608>`_, `pr#10733 <http://github.com/ceph/ceph/pull/10733>`_, Jason Dillaman)
+* doc: release-notes.rst: draft 0.94.8 release notes (`pr#10730 <http://github.com/ceph/ceph/pull/10730>`_, Nathan Cutler)
+* doc: remove btrfs contradiction (`pr#9758 <http://github.com/ceph/ceph/pull/9758>`_, Nathan Cutler)
+* doc: remove i386 from minimal hardware recommendations (`pr#10276 <http://github.com/ceph/ceph/pull/10276>`_, Kefu Chai)
+* doc: remove old references to inktank premium support (`pr#11182 <http://github.com/ceph/ceph/pull/11182>`_, Alfredo Deza)
+* doc: remove the description of deleted options (`issue#17041 <http://tracker.ceph.com/issues/17041>`_, `pr#10741 <http://github.com/ceph/ceph/pull/10741>`_, MinSheng Lin)
+* doc: rgw, doc: fix formatting around Keystone-related options. (`pr#10331 <http://github.com/ceph/ceph/pull/10331>`_, Radoslaw Zarzynski)
+* doc: rgw/doc: fix indent (`pr#10676 <http://github.com/ceph/ceph/pull/10676>`_, Yan Jun)
+* doc: rm SysV instructions, add systemd (`pr#10184 <http://github.com/ceph/ceph/pull/10184>`_, Ken Dreyer)
+* doc: silence sphinx warnings (`pr#10621 <http://github.com/ceph/ceph/pull/10621>`_, Kefu Chai)
+* doc: small standby doc edits (`pr#10479 <http://github.com/ceph/ceph/pull/10479>`_, Patrick Donnelly)
+* doc: update CephFS "early adopters" info (`pr#10068 <http://github.com/ceph/ceph/pull/10068>`_, John Spray)
+* doc: update canonical tarballs URL (`pr#9695 <http://github.com/ceph/ceph/pull/9695>`_, Ken Dreyer)
+* doc: update rbd glance configuration notes (`pr#10629 <http://github.com/ceph/ceph/pull/10629>`_, Jason Dillaman)
+* doc: update s3 static webiste feature support status (`pr#10223 <http://github.com/ceph/ceph/pull/10223>`_, Jiaying Ren)
+* doc: changelog: add v10.2.3 (`pr#11238 <http://github.com/ceph/ceph/pull/11238>`_, Abhishek Lekshmanan)
+* doc: install: Use https:// for download.ceph.com (`pr#10709 <http://github.com/ceph/ceph/pull/10709>`_, Colin Walters)
+* doc: release-notes: v0.94.9 (`pr#10927 <http://github.com/ceph/ceph/pull/10927>`_, Sage Weil)
+* doc: release-notes: v10.2.3 jewel (`pr#11234 <http://github.com/ceph/ceph/pull/11234>`_, Abhishek Lekshmanan)
+* doc: Add UK mirror and update copyright (`pr#10531 <http://github.com/ceph/ceph/pull/10531>`_, Patrick McGarry)
+* doc: README.md: replace package build instructions with tarball instructions (`pr#10829 <http://github.com/ceph/ceph/pull/10829>`_, Sage Weil)
+* doc: Removed reference about pool ownership based on BZ#1368528 (`pr#11063 <http://github.com/ceph/ceph/pull/11063>`_, Bara Ancincova)
+* librados: use bufferlist instead of buffer::list in public header (`pr#10632 <http://github.com/ceph/ceph/pull/10632>`_, Ryne Li)
+* librados: Rados-stripper: Flexible string matching for not found attributes (`pr#10577 <http://github.com/ceph/ceph/pull/10577>`_, Willem Jan Withagen)
+* librados: librados examples: link and include from current source tree by default. (`issue#15100 <http://tracker.ceph.com/issues/15100>`_, `pr#8189 <http://github.com/ceph/ceph/pull/8189>`_, Jesse Williamson)
+* librbd: API methods to directly acquire and release the exclusive lock (`issue#15632 <http://tracker.ceph.com/issues/15632>`_, `pr#9592 <http://github.com/ceph/ceph/pull/9592>`_, Mykola Golub)
+* librbd: add consistency groups operations with images (`pr#10034 <http://github.com/ceph/ceph/pull/10034>`_, Victor Denisov)
+* librbd: add explicit shrink check while resizing images (`pr#9878 <http://github.com/ceph/ceph/pull/9878>`_, Vaibhav Bhembre)
+* librbd: asynchronous v2 image creation (`issue#15321 <http://tracker.ceph.com/issues/15321>`_, `pr#9585 <http://github.com/ceph/ceph/pull/9585>`_, Venky Shankar)
+* librbd: backward/forward compatibility for update_features (`issue#17330 <http://tracker.ceph.com/issues/17330>`_, `pr#11155 <http://github.com/ceph/ceph/pull/11155>`_, Jason Dillaman)
+* librbd: block name prefix might overflow fixed size C-string (`issue#17310 <http://tracker.ceph.com/issues/17310>`_, `pr#11148 <http://github.com/ceph/ceph/pull/11148>`_, Jason Dillaman)
+* librbd: cache was not switching to writeback after first flush (`issue#16654 <http://tracker.ceph.com/issues/16654>`_, `pr#10762 <http://github.com/ceph/ceph/pull/10762>`_, Jason Dillaman)
+* librbd: corrected use-after-free in ImageWatcher (`issue#17289 <http://tracker.ceph.com/issues/17289>`_, `pr#11112 <http://github.com/ceph/ceph/pull/11112>`_, Jason Dillaman)
+* librbd: deadlock when replaying journal during image open (`issue#17188 <http://tracker.ceph.com/issues/17188>`_, `pr#10945 <http://github.com/ceph/ceph/pull/10945>`_, Jason Dillaman)
+* librbd: delay acquiring lock if image watch has failed (`issue#16923 <http://tracker.ceph.com/issues/16923>`_, `pr#10574 <http://github.com/ceph/ceph/pull/10574>`_, Jason Dillaman)
+* librbd: discard hangs when 'rbd_skip_partial_discard' is enabled (`issue#16386 <http://tracker.ceph.com/issues/16386>`_, `pr#10060 <http://github.com/ceph/ceph/pull/10060>`_, Mykola Golub)
+* librbd: extract group module from librbd/internal (`pr#11070 <http://github.com/ceph/ceph/pull/11070>`_, Victor Denisov)
+* librbd: failed assertion after shrinking a clone image twice (`issue#16561 <http://tracker.ceph.com/issues/16561>`_, `pr#10072 <http://github.com/ceph/ceph/pull/10072>`_, Jason Dillaman)
+* librbd: fix missing return statement if failed to get mirror image state (`pr#10136 <http://github.com/ceph/ceph/pull/10136>`_, runsisi)
+* librbd: fix possible inconsistent state when disabling mirroring fails (`issue#16984 <http://tracker.ceph.com/issues/16984>`_, `pr#10711 <http://github.com/ceph/ceph/pull/10711>`_, Jason Dillaman)
+* librbd: ignore partial refresh error when acquiring exclusive lock (`issue#17227 <http://tracker.ceph.com/issues/17227>`_, `pr#11044 <http://github.com/ceph/ceph/pull/11044>`_, Jason Dillaman)
+* librbd: initial hooks for client-side, image-extent cache in IO path (`pr#9121 <http://github.com/ceph/ceph/pull/9121>`_, Jason Dillaman)
+* librbd: interlock image refresh and exclusive lock operations (`issue#16773 <http://tracker.ceph.com/issues/16773>`_, `issue#17015 <http://tracker.ceph.com/issues/17015>`_, `pr#10770 <http://github.com/ceph/ceph/pull/10770>`_, Jason Dillaman)
+* librbd: memory leak in MirroringWatcher::notify_image_updated (`pr#11306 <http://github.com/ceph/ceph/pull/11306>`_, Mykola Golub)
+* librbd: optimize away unnecessary object map updates  (`issue#16707 <http://tracker.ceph.com/issues/16707>`_, `issue#16689 <http://tracker.ceph.com/issues/16689>`_, `pr#10332 <http://github.com/ceph/ceph/pull/10332>`_, Jason Dillaman)
+* librbd: optionally unregister "laggy" journal clients (`issue#14738 <http://tracker.ceph.com/issues/14738>`_, `pr#10378 <http://github.com/ceph/ceph/pull/10378>`_, Mykola Golub)
+* librbd: permit disabling journaling if in corrupt state (`issue#16740 <http://tracker.ceph.com/issues/16740>`_, `pr#10712 <http://github.com/ceph/ceph/pull/10712>`_, Jason Dillaman)
+* librbd: possible deadlock if cluster connection closed after image (`issue#17254 <http://tracker.ceph.com/issues/17254>`_, `pr#11037 <http://github.com/ceph/ceph/pull/11037>`_, Jason Dillaman)
+* librbd: potential deadlock closing image with in-flight readahead (`issue#17198 <http://tracker.ceph.com/issues/17198>`_, `pr#11152 <http://github.com/ceph/ceph/pull/11152>`_, Jason Dillaman)
+* librbd: potential double-unwatch of watch handle upon error (`issue#17210 <http://tracker.ceph.com/issues/17210>`_, `pr#10974 <http://github.com/ceph/ceph/pull/10974>`_, Jason Dillaman)
+* librbd: potential seg fault when blacklisting an image client (`issue#17251 <http://tracker.ceph.com/issues/17251>`_, `pr#11034 <http://github.com/ceph/ceph/pull/11034>`_, Jason Dillaman)
+* librbd: prevent creation of clone from non-primary mirrored image (`issue#16449 <http://tracker.ceph.com/issues/16449>`_, `pr#10123 <http://github.com/ceph/ceph/pull/10123>`_, Mykola Golub)
+* librbd: prevent creation of v2 image ids that are too large (`issue#16887 <http://tracker.ceph.com/issues/16887>`_, `pr#10581 <http://github.com/ceph/ceph/pull/10581>`_, Jason Dillaman)
+* mds: Add path filtering for dump cache (`issue#11171 <http://tracker.ceph.com/issues/11171>`_, `pr#9925 <http://github.com/ceph/ceph/pull/9925>`_, Douglas Fuller)
+* mds: Kill C_SaferCond in evict_sessions() (`issue#16288 <http://tracker.ceph.com/issues/16288>`_, `pr#9971 <http://github.com/ceph/ceph/pull/9971>`_, Douglas Fuller)
+* mds: Return "committing" rather than "committed" member in get_committing (`pr#10250 <http://github.com/ceph/ceph/pull/10250>`_, Greg Farnum)
+* mds: Set mds_snap_max_uid to 4294967294 (`pr#11016 <http://github.com/ceph/ceph/pull/11016>`_, Wido den Hollander)
+* mds: add assertion in handle_slave_rename_prep (`issue#16807 <http://tracker.ceph.com/issues/16807>`_, `pr#10429 <http://github.com/ceph/ceph/pull/10429>`_, John Spray)
+* mds: add assertions for standby_daemons invariant (`issue#16592 <http://tracker.ceph.com/issues/16592>`_, `pr#10316 <http://github.com/ceph/ceph/pull/10316>`_, Patrick Donnelly)
+* mds: add health warning for oversized cache (`issue#16570 <http://tracker.ceph.com/issues/16570>`_, `pr#10245 <http://github.com/ceph/ceph/pull/10245>`_, John Spray)
+* mds: add maximum fragment size constraint (`issue#16164 <http://tracker.ceph.com/issues/16164>`_, `pr#9789 <http://github.com/ceph/ceph/pull/9789>`_, Patrick Donnelly)
+* mds: add perf counters for MDLog replay and SessionMap (`pr#10539 <http://github.com/ceph/ceph/pull/10539>`_, John Spray)
+* mds: catch duplicates in DamageTable (`issue#17173 <http://tracker.ceph.com/issues/17173>`_, `pr#11137 <http://github.com/ceph/ceph/pull/11137>`_, John Spray)
+* mds: fix Session::check_access() (`issue#16358 <http://tracker.ceph.com/issues/16358>`_, `pr#9769 <http://github.com/ceph/ceph/pull/9769>`_, Yan, Zheng)
+* mds: fix daemon selection when starting ranks (`pr#10540 <http://github.com/ceph/ceph/pull/10540>`_, John Spray)
+* mds: fix shutting down mds timed-out due to deadlock (`issue#16396 <http://tracker.ceph.com/issues/16396>`_, `pr#9884 <http://github.com/ceph/ceph/pull/9884>`_, Zhi Zhang)
+* mds: fix up _dispatch ref-counting semantics (`pr#10533 <http://github.com/ceph/ceph/pull/10533>`_, Greg Farnum)
+* mds: fixup dump Formatter' type error; add path_ino and is_primary in the CDentry::dump() (`pr#10119 <http://github.com/ceph/ceph/pull/10119>`_, huanwen ren)
+* mds: handle blacklisting during journal recovery (`issue#17236 <http://tracker.ceph.com/issues/17236>`_, `pr#11138 <http://github.com/ceph/ceph/pull/11138>`_, John Spray)
+* mds: log path with CDir damage messages (`issue#16973 <http://tracker.ceph.com/issues/16973>`_, `pr#10996 <http://github.com/ceph/ceph/pull/10996>`_, John Spray)
+* mds: move Finisher to unlocked shutdown (`issue#16042 <http://tracker.ceph.com/issues/16042>`_, `pr#10142 <http://github.com/ceph/ceph/pull/10142>`_, Patrick Donnelly)
+* mds: populate DamageTable from scrub and log more quietly (`issue#16016 <http://tracker.ceph.com/issues/16016>`_, `pr#11136 <http://github.com/ceph/ceph/pull/11136>`_, John Spray)
+* mds: remove fail-safe queueing replay request (`issue#17271 <http://tracker.ceph.com/issues/17271>`_, `pr#11078 <http://github.com/ceph/ceph/pull/11078>`_, Yan, Zheng)
+* mds: remove max_mds config option (`issue#17105 <http://tracker.ceph.com/issues/17105>`_, `pr#10914 <http://github.com/ceph/ceph/pull/10914>`_, Patrick Donnelly)
+* mds: remove unused MDSDaemon::objecter (`pr#10566 <http://github.com/ceph/ceph/pull/10566>`_, Patrick Donnelly)
+* mds: snap failover fixes (`pr#9955 <http://github.com/ceph/ceph/pull/9955>`_, Yan, Zheng)
+* mds: trim null dentries proactively (`issue#16919 <http://tracker.ceph.com/issues/16919>`_, `pr#10606 <http://github.com/ceph/ceph/pull/10606>`_, John Spray)
+* mds: unuse Class and cleanup (`pr#10399 <http://github.com/ceph/ceph/pull/10399>`_, huanwen ren)
+* mds: use reference to avoid copy (`pr#10191 <http://github.com/ceph/ceph/pull/10191>`_, Patrick Donnelly)
+* mds: MDCache.h: remove unneeded access specifier (`pr#10901 <http://github.com/ceph/ceph/pull/10901>`_, Michal Jarzabek)
+* mds: MDSDaemon: move C_MDS_Tick class to .cc file (`pr#11220 <http://github.com/ceph/ceph/pull/11220>`_, Michal Jarzabek)
+* mgr: implement con reset handling (`pr#11299 <http://github.com/ceph/ceph/pull/11299>`_, Sage Weil)
+* mgr: squash compiler warnings (`pr#11307 <http://github.com/ceph/ceph/pull/11307>`_, John Spray)
+* mon: MonClient may hang on pinging an unresponsive monitor (`pr#9259 <http://github.com/ceph/ceph/pull/9259>`_, xie xingguo)
+* mon: Monitor: validate prefix on handle_command() (`issue#16297 <http://tracker.ceph.com/issues/16297>`_, `pr#9700 <http://github.com/ceph/ceph/pull/9700>`_, You Ji)
+* mon: OSDMonitor: Missing nearfull flag set (`pr#11082 <http://github.com/ceph/ceph/pull/11082>`_, Igor Podoski)
+* mon: change osdmap flags set and unset messages (`issue#15983 <http://tracker.ceph.com/issues/15983>`_, `pr#9252 <http://github.com/ceph/ceph/pull/9252>`_, Vikhyat Umrao)
+* mon: clear list in better way (`pr#9718 <http://github.com/ceph/ceph/pull/9718>`_, song baisen)
+* mon: do not recalculate 'to_remove' when it's known (`pr#9717 <http://github.com/ceph/ceph/pull/9717>`_, song baisen)
+* mon: misc cleanups (`pr#10591 <http://github.com/ceph/ceph/pull/10591>`_, xie xingguo)
+* mon: remove the redundant cancel_probe_timeout function (`pr#10261 <http://github.com/ceph/ceph/pull/10261>`_, song baisen)
+* mon: remove the redundant is_active judge in PaxosService (`pr#9749 <http://github.com/ceph/ceph/pull/9749>`_, song baisen)
+* mon: tear down standby replays on MDS rank stop (`issue#16909 <http://tracker.ceph.com/issues/16909>`_, `pr#10628 <http://github.com/ceph/ceph/pull/10628>`_, John Spray)
+* mon: use clearer code structure (`pr#10192 <http://github.com/ceph/ceph/pull/10192>`_, Patrick Donnelly)
+* mon: validate states transmitted in beacons (`issue#16592 <http://tracker.ceph.com/issues/16592>`_, `pr#10428 <http://github.com/ceph/ceph/pull/10428>`_, John Spray)
+* mon: wait 10m (not 5m) before marking down OSDs out (`pr#11184 <http://github.com/ceph/ceph/pull/11184>`_, Sage Weil)
+* mon: write fsid use the right return value (`pr#10197 <http://github.com/ceph/ceph/pull/10197>`_, song baisen)
+* mon: Elector:move C_ElectionExpire class to cc file (`pr#10416 <http://github.com/ceph/ceph/pull/10416>`_, Michal Jarzabek)
+* mon: HealthMonitor: add override to virtual functs (`pr#10549 <http://github.com/ceph/ceph/pull/10549>`_, Michal Jarzabek)
+* mon: HealthMonitor: remove unneeded include (`pr#10563 <http://github.com/ceph/ceph/pull/10563>`_, Michal Jarzabek)
+* mon: MonClient.h: delete copy constr and assing op (`pr#10599 <http://github.com/ceph/ceph/pull/10599>`_, Michal Jarzabek)
+* mon: MonClient: move C_CancelMonCommand to cc file (`pr#10392 <http://github.com/ceph/ceph/pull/10392>`_, Michal Jarzabek)
+* mon: MonClient: move C_Tick struct to cc file (`pr#10383 <http://github.com/ceph/ceph/pull/10383>`_, Michal Jarzabek)
+* mon: Monitor.h: add override to virtual functions (`pr#10515 <http://github.com/ceph/ceph/pull/10515>`_, Michal Jarzabek)
+* mon: Monitor: move C_Scrub, C_ScrubTimeout to .cc (`pr#10513 <http://github.com/ceph/ceph/pull/10513>`_, Michal Jarzabek)
+* mon: OSDMonitor.cc: remove unneeded casts (`pr#10575 <http://github.com/ceph/ceph/pull/10575>`_, Michal Jarzabek)
+* mon: Paxos: move classes to .cc file (`pr#11215 <http://github.com/ceph/ceph/pull/11215>`_, Michal Jarzabek)
+* mon: PaxosService: move classes to cc file (`pr#10529 <http://github.com/ceph/ceph/pull/10529>`_, Michal Jarzabek)
+* mon: remove the redundant list swap in paxos commit_proposal (`pr#10011 <http://github.com/ceph/ceph/pull/10011>`_, song baisen)
+* msgr: set close on exec flag (`issue#16390 <http://tracker.ceph.com/issues/16390>`_, `pr#9772 <http://github.com/ceph/ceph/pull/9772>`_, Kefu Chai)
+* msgr: Accepter.h: add override to virtual function (`pr#10422 <http://github.com/ceph/ceph/pull/10422>`_, Michal Jarzabek)
+* msgr: Accepter: move include to cc file (`pr#10441 <http://github.com/ceph/ceph/pull/10441>`_, Michal Jarzabek)
+* msgr: AsyncConnection: add const to mem functions (`pr#10302 <http://github.com/ceph/ceph/pull/10302>`_, Michal Jarzabek)
+* msgr: AsyncMessenger.cc: remove unneeded cast (`pr#10141 <http://github.com/ceph/ceph/pull/10141>`_, Michal Jarzabek)
+* msgr: AsyncMessenger: add const to function (`pr#10114 <http://github.com/ceph/ceph/pull/10114>`_, Michal Jarzabek)
+* msgr: AsyncMessenger: move C_handle_reap class to cc (`pr#10113 <http://github.com/ceph/ceph/pull/10113>`_, Michal Jarzabek)
+* msgr: AsyncMessenger: move C_processor_accept class (`pr#9991 <http://github.com/ceph/ceph/pull/9991>`_, Michal Jarzabek)
+* msgr: AsyncMessenger: remove unneeded include file (`pr#10195 <http://github.com/ceph/ceph/pull/10195>`_, Michal Jarzabek)
+* msgr: AsyncMessenger: remove unused function (`pr#10163 <http://github.com/ceph/ceph/pull/10163>`_, Michal Jarzabek)
+* msgr: EventKqueue.h: add override to virtual func (`pr#10318 <http://github.com/ceph/ceph/pull/10318>`_, Michal Jarzabek)
+* msgr: EventPoll.h: add override to virtual functions (`pr#10314 <http://github.com/ceph/ceph/pull/10314>`_, Michal Jarzabek)
+* msgr: EventSelect.h: add override to virtual funct (`pr#10321 <http://github.com/ceph/ceph/pull/10321>`_, Michal Jarzabek)
+* msgr: EventSelect: move includes to cc file (`pr#10333 <http://github.com/ceph/ceph/pull/10333>`_, Michal Jarzabek)
+* msgr: FastStrategy.h: add override to virtual funct (`pr#10482 <http://github.com/ceph/ceph/pull/10482>`_, Michal Jarzabek)
+* msgr: Message.h: add const to member function (`pr#10354 <http://github.com/ceph/ceph/pull/10354>`_, Michal Jarzabek)
+* msgr: Message.h: remove code duplication (`pr#10356 <http://github.com/ceph/ceph/pull/10356>`_, Michal Jarzabek)
+* msgr: QueueStrategy: add override to virtual functs (`pr#10503 <http://github.com/ceph/ceph/pull/10503>`_, Michal Jarzabek)
+* msgr: Stack.h: delete copy constr and assign op (`pr#11107 <http://github.com/ceph/ceph/pull/11107>`_, Michal Jarzabek)
+* msgr: async/Event.h: add const to member function (`pr#10224 <http://github.com/ceph/ceph/pull/10224>`_, Michal Jarzabek)
+* msgr: async: remove unused code. (`pr#11247 <http://github.com/ceph/ceph/pull/11247>`_, Jianpeng Ma)
+* osd: bail out if transaction size overflows (`issue#16982 <http://tracker.ceph.com/issues/16982>`_, `pr#10753 <http://github.com/ceph/ceph/pull/10753>`_, Kefu Chai)
+* osd: cleanup options and other redundancies (`pr#10450 <http://github.com/ceph/ceph/pull/10450>`_, xie xingguo)
+* osd: drop unused variables/methods (`pr#10559 <http://github.com/ceph/ceph/pull/10559>`_, xie xingguo)
+* osd: fix the mem leak of RepGather (`issue#16801 <http://tracker.ceph.com/issues/16801>`_, `pr#10423 <http://github.com/ceph/ceph/pull/10423>`_, Kefu Chai)
+* osd: fixups to explicitly persistenting missing sets (`pr#10405 <http://github.com/ceph/ceph/pull/10405>`_, Samuel Just)
+* osd: increment stats on recovery pull also (`issue#16277 <http://tracker.ceph.com/issues/16277>`_, `pr#10152 <http://github.com/ceph/ceph/pull/10152>`_, Kefu Chai)
+* osd: limit omap data in push op (`issue#16128 <http://tracker.ceph.com/issues/16128>`_, `pr#9894 <http://github.com/ceph/ceph/pull/9894>`_, Wanlong Gao)
+* osd: minor performance improvements (`pr#10470 <http://github.com/ceph/ceph/pull/10470>`_, xie xingguo)
+* osd: minor performance improvements and fixes (`pr#10526 <http://github.com/ceph/ceph/pull/10526>`_, xie xingguo)
+* osd: misc fixes and cleanups (`pr#10610 <http://github.com/ceph/ceph/pull/10610>`_, xie xingguo)
+* osd: miscellaneous fixes (`pr#10572 <http://github.com/ceph/ceph/pull/10572>`_, xie xingguo)
+* osd: more cleanups (`pr#10548 <http://github.com/ceph/ceph/pull/10548>`_, xie xingguo)
+* osd: object class loading and execution permissions (`pr#9972 <http://github.com/ceph/ceph/pull/9972>`_, Noah Watkins)
+* osd: pass shared_ptr by const reference (`pr#11266 <http://github.com/ceph/ceph/pull/11266>`_, Michal Jarzabek)
+* osd: persist the missing set explicitly (`pr#10334 <http://github.com/ceph/ceph/pull/10334>`_, Samuel Just)
+* osd: remove dispatch queue check since we don't queue hb message to this (`pr#9947 <http://github.com/ceph/ceph/pull/9947>`_, Haomai Wang)
+* osd: remove duplicated function (`pr#9117 <http://github.com/ceph/ceph/pull/9117>`_, Wei Jin)
+* osd: replace ceph:atomic_t with std::atomic in osd module. (`pr#9138 <http://github.com/ceph/ceph/pull/9138>`_, Xiaowei Chen)
+* osd: should not look up an empty pg (`issue#17380 <http://tracker.ceph.com/issues/17380>`_, `pr#11208 <http://github.com/ceph/ceph/pull/11208>`_, Kefu Chai, Loic Dachary)
+* osd: small cleanups (`pr#9980 <http://github.com/ceph/ceph/pull/9980>`_, Wanlong Gao)
+* osd: subscribe for old osdmaps when pause flag is set (`issue#17023 <http://tracker.ceph.com/issues/17023>`_, `pr#10725 <http://github.com/ceph/ceph/pull/10725>`_, Kefu Chai)
+* osd:preserve allocation hint attribute during recovery (`pr#9452 <http://github.com/ceph/ceph/pull/9452>`_, yaoning)
+* osd: osd-fast-mark-down.sh: fix typo in variable assignments (`pr#11224 <http://github.com/ceph/ceph/pull/11224>`_, Willem Jan Withagen)
+* osd: OSD.cc: initialise variable at definition (`pr#11099 <http://github.com/ceph/ceph/pull/11099>`_, Michal Jarzabek)
+* osd: OSD.cc: remove unneeded searching of map (`pr#11000 <http://github.com/ceph/ceph/pull/11000>`_, Michal Jarzabek)
+* osd: OSD.h: make some members private (`pr#11085 <http://github.com/ceph/ceph/pull/11085>`_, Michal Jarzabek)
+* osd: PG.cc: remove unneeded use of count (`pr#11228 <http://github.com/ceph/ceph/pull/11228>`_, Michal Jarzabek)
+* osd: PGBackend.h: move structs to .cc file (`pr#10975 <http://github.com/ceph/ceph/pull/10975>`_, Michal Jarzabek)
+* osd: ReplicatedBackend: move classes to cc file (`pr#10967 <http://github.com/ceph/ceph/pull/10967>`_, Michal Jarzabek)
+* osd: ReplicatedPG.h: add override to virtual funct (`pr#11271 <http://github.com/ceph/ceph/pull/11271>`_, Michal Jarzabek)
+* osd: ReplicatedPG: move classes to .cc file (`pr#10971 <http://github.com/ceph/ceph/pull/10971>`_, Michal Jarzabek)
+* osd: ReplicatedPG:move C_OSD_OnApplied class to cc (`pr#11288 <http://github.com/ceph/ceph/pull/11288>`_, Michal Jarzabek)
+* osd: Watch.h: remove unneeded forward declaration (`pr#10269 <http://github.com/ceph/ceph/pull/10269>`_, Michal Jarzabek)
+* osd: osdc/ObjectCacher.h: add const to member functions (`pr#9569 <http://github.com/ceph/ceph/pull/9569>`_, Michal Jarzabek)
+* osd: osdc/ObjectCacher.h: add const to member functions (`pr#9652 <http://github.com/ceph/ceph/pull/9652>`_, Michal Jarzabek)
+* osd: osdc/Objecter: move RequestStateHook class to .cc (`pr#10734 <http://github.com/ceph/ceph/pull/10734>`_, Michal Jarzabek)
+* pybind: Port Python-based tests and remaining Python bindings to Python 3 (`pr#10177 <http://github.com/ceph/ceph/pull/10177>`_, Oleh Prypin)
+* pybind: Rework cephfs/setup.py for PyPI (`pr#10315 <http://github.com/ceph/ceph/pull/10315>`_, Anirudha Bose)
+* pybind: Rework rbd/setup.py for PyPI (`issue#16940 <http://tracker.ceph.com/issues/16940>`_, `pr#10376 <http://github.com/ceph/ceph/pull/10376>`_, Anirudha Bose)
+* pybind: global/signal_handler: dump cmdline instead of arg[0] (`pr#10345 <http://github.com/ceph/ceph/pull/10345>`_, Kefu Chai)
+* pybind: make rados ready for PyPI (`pr#9833 <http://github.com/ceph/ceph/pull/9833>`_, Anirudha Bose)
+* pybind: pybind/ceph_argparse: handle non ascii unicode args (`issue#12287 <http://tracker.ceph.com/issues/12287>`_, `pr#8943 <http://github.com/ceph/ceph/pull/8943>`_, Kefu Chai)
+* pybind: Python 3 compatibility for workunits (`pr#10815 <http://github.com/ceph/ceph/pull/10815>`_, Anirudha Bose)
+* rbd: Allow user to remove snapshot with --force to auto flatten children (`pr#10087 <http://github.com/ceph/ceph/pull/10087>`_, Dongsheng Yang)
+* rbd: Reviewed-off-by: Ilya Dryomov <idryomov@gmail.com> (`issue#16171 <http://tracker.ceph.com/issues/16171>`_, `pr#10481 <http://github.com/ceph/ceph/pull/10481>`_, Jason Dillaman)
+* rbd: Reviewed-off-by: Ilya Dryomov <idryomov@gmail.com> (`issue#17030 <http://tracker.ceph.com/issues/17030>`_, `pr#10735 <http://github.com/ceph/ceph/pull/10735>`_, Jason Dillaman)
+* rbd: bench io-size should not be larger than image size (`issue#16967 <http://tracker.ceph.com/issues/16967>`_, `pr#10708 <http://github.com/ceph/ceph/pull/10708>`_, Jason Dillaman)
+* rbd: cleanup - Proxied operations shouldn't result in error messages if replayed (`issue#16130 <http://tracker.ceph.com/issues/16130>`_, `pr#9724 <http://github.com/ceph/ceph/pull/9724>`_, Vikhyat Umrao)
+* rbd: cls_rbd: clean up status from rbd-mirror if image removed (`pr#11142 <http://github.com/ceph/ceph/pull/11142>`_, Huan Zhang)
+* rbd: cls_rbd: set omap values in batch during image creation (`pr#9981 <http://github.com/ceph/ceph/pull/9981>`_, Dongsheng Yang)
+* rbd: inherit the parent image features when cloning an image (`issue#15388 <http://tracker.ceph.com/issues/15388>`_, `pr#9334 <http://github.com/ceph/ceph/pull/9334>`_, Dongsheng Yang)
+* rbd: journal: ensure in-flight ops are complete destroying journaler (`issue#17446 <http://tracker.ceph.com/issues/17446>`_, `pr#11257 <http://github.com/ceph/ceph/pull/11257>`_, Mykola Golub, Jason Dillaman)
+* rbd: journal: increase concurrency/parallelism of journal recorder (`issue#15259 <http://tracker.ceph.com/issues/15259>`_, `pr#10445 <http://github.com/ceph/ceph/pull/10445>`_, Ricardo Dias)
+* rbd: journal: move JournalTrimmer::C_RemoveSet struct (`pr#10912 <http://github.com/ceph/ceph/pull/10912>`_, Michal Jarzabek)
+* rbd: qa/workunits/rbd: before removing image make sure it is not bootstrapped (`issue#16555 <http://tracker.ceph.com/issues/16555>`_, `pr#10155 <http://github.com/ceph/ceph/pull/10155>`_, Mykola Golub)
+* rbd: qa/workunits/rbd: check status also in pool dir after asok commands (`pr#11291 <http://github.com/ceph/ceph/pull/11291>`_, Mykola Golub)
+* rbd: qa/workunits/rbd: set image-meta on primary image and wait it is replicated (`pr#11294 <http://github.com/ceph/ceph/pull/11294>`_, Mykola Golub)
+* rbd: qa/workunits/rbd: small fixup and improvements for rbd-mirror tests (`pr#10483 <http://github.com/ceph/ceph/pull/10483>`_, Mykola Golub)
+* rbd: qa/workunits/rbd: wait for image deleted before checking health (`pr#10545 <http://github.com/ceph/ceph/pull/10545>`_, Mykola Golub)
+* rbd: qa/workunits: support filtering cls_rbd unit test cases (`issue#16529 <http://tracker.ceph.com/issues/16529>`_, `pr#10714 <http://github.com/ceph/ceph/pull/10714>`_, Jason Dillaman)
+* rbd: rbd-mirror: 'wait_for_scheduled_deletion' callback might deadlock (`issue#16491 <http://tracker.ceph.com/issues/16491>`_, `pr#9964 <http://github.com/ceph/ceph/pull/9964>`_, Jason Dillaman)
+* rbd: rbd-mirror: Add sparse read for sync image (`issue#16780 <http://tracker.ceph.com/issues/16780>`_, `pr#11005 <http://github.com/ceph/ceph/pull/11005>`_, tianqing)
+* rbd: rbd-mirror: add additional test scenarios (`pr#10488 <http://github.com/ceph/ceph/pull/10488>`_, lande1234)
+* rbd: rbd-mirror: concurrent access of event might result in heap corruption (`issue#17283 <http://tracker.ceph.com/issues/17283>`_, `pr#11104 <http://github.com/ceph/ceph/pull/11104>`_, Jason Dillaman)
+* rbd: rbd-mirror: force-promoted image will remain R/O until rbd-mirror daemon restarted (`issue#16974 <http://tracker.ceph.com/issues/16974>`_, `pr#11090 <http://github.com/ceph/ceph/pull/11090>`_, Jason Dillaman)
+* rbd: rbd-mirror: gracefully fail if object map is unavailable (`issue#16558 <http://tracker.ceph.com/issues/16558>`_, `pr#10065 <http://github.com/ceph/ceph/pull/10065>`_, Jason Dillaman)
+* rbd: rbd-mirror: gracefully handle being blacklisted (`issue#16349 <http://tracker.ceph.com/issues/16349>`_, `pr#9970 <http://github.com/ceph/ceph/pull/9970>`_, Jason Dillaman)
+* rbd: rbd-mirror: image deleter should use pool id + global image uuid for key (`issue#16538 <http://tracker.ceph.com/issues/16538>`_, `issue#16227 <http://tracker.ceph.com/issues/16227>`_, `pr#10484 <http://github.com/ceph/ceph/pull/10484>`_, Jason Dillaman)
+* rbd: rbd-mirror: improve split-brain detection logic (`issue#16855 <http://tracker.ceph.com/issues/16855>`_, `pr#10703 <http://github.com/ceph/ceph/pull/10703>`_, Jason Dillaman)
+* rbd: rbd-mirror: include local pool id in resync throttle unique key (`issue#16536 <http://tracker.ceph.com/issues/16536>`_, `pr#10254 <http://github.com/ceph/ceph/pull/10254>`_, Jason Dillaman)
+* rbd: rbd-mirror: non-primary image is recording journal events during image sync (`pr#10462 <http://github.com/ceph/ceph/pull/10462>`_, Jason Dillaman)
+* rbd: rbd-mirror: potential IO stall when using asok flush request (`issue#16708 <http://tracker.ceph.com/issues/16708>`_, `pr#10432 <http://github.com/ceph/ceph/pull/10432>`_, Jason Dillaman)
+* rbd: rbd-mirror: potential assertion failure during error-induced shutdown (`issue#16956 <http://tracker.ceph.com/issues/16956>`_, `pr#10613 <http://github.com/ceph/ceph/pull/10613>`_, Jason Dillaman)
+* rbd: rbd-mirror: potential race condition during failure shutdown (`issue#16980 <http://tracker.ceph.com/issues/16980>`_, `pr#10667 <http://github.com/ceph/ceph/pull/10667>`_, Jason Dillaman)
+* rbd: rbd-mirror: quiesce in-flight event commits before shut down (`issue#17355 <http://tracker.ceph.com/issues/17355>`_, `pr#11185 <http://github.com/ceph/ceph/pull/11185>`_, Jason Dillaman)
+* rbd: rbd-mirror: reduce memory footprint during journal replay (`issue#16223 <http://tracker.ceph.com/issues/16223>`_, `pr#10341 <http://github.com/ceph/ceph/pull/10341>`_, Jason Dillaman)
+* rbd: rbd-mirror: remove ceph_test_rbd_mirror_image_replay test case (`issue#16539 <http://tracker.ceph.com/issues/16539>`_, `pr#10083 <http://github.com/ceph/ceph/pull/10083>`_, Mykola Golub)
+* rbd: rbd-mirror: replaying state should include flush action (`issue#16970 <http://tracker.ceph.com/issues/16970>`_, `pr#10627 <http://github.com/ceph/ceph/pull/10627>`_, Jason Dillaman)
+* rbd: rbd-mirror: replicate dynamic feature updates (`issue#16213 <http://tracker.ceph.com/issues/16213>`_, `pr#10980 <http://github.com/ceph/ceph/pull/10980>`_, Mykola Golub)
+* rbd: rbd-mirror: replicate image metadata settings (`issue#16212 <http://tracker.ceph.com/issues/16212>`_, `pr#11168 <http://github.com/ceph/ceph/pull/11168>`_, Mykola Golub)
+* rbd: rbd-mirror: snap rename does not properly replicate to peers (`issue#16622 <http://tracker.ceph.com/issues/16622>`_, `pr#10249 <http://github.com/ceph/ceph/pull/10249>`_, Jason Dillaman)
+* rbd: rbd-nbd does not properly handle resize notifications (`issue#15715 <http://tracker.ceph.com/issues/15715>`_, `pr#9291 <http://github.com/ceph/ceph/pull/9291>`_, Mykola Golub)
+* rbd: rbd-nbd: fix kernel deadlock during teuthology testing (`issue#16921 <http://tracker.ceph.com/issues/16921>`_, `pr#10985 <http://github.com/ceph/ceph/pull/10985>`_, Jason Dillaman)
+* rbd: recognize lock_on_read option (`pr#11313 <http://github.com/ceph/ceph/pull/11313>`_, Ilya Dryomov)
+* rbd: return error if we specified a wrong image name for rbd du (`issue#16987 <http://tracker.ceph.com/issues/16987>`_, `pr#11031 <http://github.com/ceph/ceph/pull/11031>`_, Dongsheng Yang)
+* rbd: test/librbd/fsx: enable exclusive-lock feature in krbd mode (`pr#10984 <http://github.com/ceph/ceph/pull/10984>`_, Ilya Dryomov)
+* rbd: test/rbd: fix possible mock journal race conditions (`issue#17317 <http://tracker.ceph.com/issues/17317>`_, `pr#11153 <http://github.com/ceph/ceph/pull/11153>`_, Jason Dillaman)
+* rbd: test: cmake related fixups for rbd tests (`pr#10124 <http://github.com/ceph/ceph/pull/10124>`_, Mykola Golub)
+* rbd: test: run-rbd-tests test cmake fixup (`pr#10134 <http://github.com/ceph/ceph/pull/10134>`_, Mykola Golub)
+* rbd: test: use wrapper that respects RBD_FEATURES when creating rbd image (`issue#16717 <http://tracker.ceph.com/issues/16717>`_, `pr#10348 <http://github.com/ceph/ceph/pull/10348>`_, Mykola Golub)
+* rbd: udev: add krbd readahead placeholder (`pr#10841 <http://github.com/ceph/ceph/pull/10841>`_, Nick Fisk)
+* rbd: rbd_mirror/ImageSynceThrottler: move struct to .cc (`pr#10928 <http://github.com/ceph/ceph/pull/10928>`_, Michal Jarzabek)
+* rgw: (build verified, provably unused/not aliased) (`pr#9993 <http://github.com/ceph/ceph/pull/9993>`_, weiqiaomiao)
+* rgw: Add documentation for the Multi-tenancy feature (`pr#9570 <http://github.com/ceph/ceph/pull/9570>`_, Pete Zaitcev)
+* rgw: Clean up lifecycle thread (`pr#10480 <http://github.com/ceph/ceph/pull/10480>`_, Daniel Gryniewicz)
+* rgw: Do not archive metadata by default (`issue#17256 <http://tracker.ceph.com/issues/17256>`_, `pr#11051 <http://github.com/ceph/ceph/pull/11051>`_, Pavan Rallabhandi)
+* rgw: Fix Host->bucket fallback logic inversion (`issue#15975 <http://tracker.ceph.com/issues/15975>`_, `issue#17136 <http://tracker.ceph.com/issues/17136>`_, `pr#10873 <http://github.com/ceph/ceph/pull/10873>`_, Robin H. Johnson)
+* rgw: Fix for using port 443 with pre-signed urls. (`issue#16548 <http://tracker.ceph.com/issues/16548>`_, `pr#10088 <http://github.com/ceph/ceph/pull/10088>`_, Pritha Srivastava)
+* rgw: Fix incorrect content length and range for zero sized objects during range requests (`issue#16388 <http://tracker.ceph.com/issues/16388>`_, `pr#10207 <http://github.com/ceph/ceph/pull/10207>`_, Pavan Rallabhandi)
+* rgw: Got rid of recursive mutex. (`pr#10562 <http://github.com/ceph/ceph/pull/10562>`_, Adam Kupczyk)
+* rgw: RGW : setting socket backlog for via ceph.conf (`issue#16406 <http://tracker.ceph.com/issues/16406>`_, `pr#9891 <http://github.com/ceph/ceph/pull/9891>`_, Feng Guo)
+* rgw: RGWMetaSyncCR holds refs to stacks instead of crs (`issue#16666 <http://tracker.ceph.com/issues/16666>`_, `pr#10301 <http://github.com/ceph/ceph/pull/10301>`_, Casey Bodley)
+* rgw: Reviewed by: Pritha Srivastava <prsrivas@redhat.com> (`issue#16188 <http://tracker.ceph.com/issues/16188>`_, `pr#9584 <http://github.com/ceph/ceph/pull/9584>`_, Albert Tu)
+* rgw: Rgw lifecycle testing (`pr#11131 <http://github.com/ceph/ceph/pull/11131>`_, Daniel Gryniewicz)
+* rgw: Rgw nfs 28 (`pr#10611 <http://github.com/ceph/ceph/pull/10611>`_, Matt Benjamin)
+* rgw: add configurables for {data,meta} sync error injection (`pr#10388 <http://github.com/ceph/ceph/pull/10388>`_, Yehuda Sadeh)
+* rgw: add deadlock detection to RGWCoroutinesManager::run() (`pr#10032 <http://github.com/ceph/ceph/pull/10032>`_, Casey Bodley)
+* rgw: add lc_pool when decode or encode struct RGWZoneParams (`pr#10439 <http://github.com/ceph/ceph/pull/10439>`_, weiqiaomiao)
+* rgw: add missing master_zone when running with old default region config (`issue#16627 <http://tracker.ceph.com/issues/16627>`_, `pr#10205 <http://github.com/ceph/ceph/pull/10205>`_, Orit Wasserman)
+* rgw: add pg_ver to tombstone_cache (`pr#9851 <http://github.com/ceph/ceph/pull/9851>`_, Casey Bodley)
+* rgw: add reinit/rebind logic (ldap) (`pr#10532 <http://github.com/ceph/ceph/pull/10532>`_, Matt Benjamin)
+* rgw: add return value checking to avoid possible subsequent `parse` exception (`pr#10241 <http://github.com/ceph/ceph/pull/10241>`_, Yan Jun)
+* rgw: add suport for Swift-at-root dependent features of Swift API (`issue#16673 <http://tracker.ceph.com/issues/16673>`_, `pr#10280 <http://github.com/ceph/ceph/pull/10280>`_, Pritha Srivastava, Radoslaw Zarzynski)
+* rgw: add support for Static Website of Swift API (`pr#9844 <http://github.com/ceph/ceph/pull/9844>`_, Radoslaw Zarzynski)
+* rgw: add tenant support to multisite sync (`issue#16469 <http://tracker.ceph.com/issues/16469>`_, `pr#10075 <http://github.com/ceph/ceph/pull/10075>`_, Casey Bodley)
+* rgw: back off bucket sync on failures, don't store marker (`issue#16742 <http://tracker.ceph.com/issues/16742>`_, `pr#10355 <http://github.com/ceph/ceph/pull/10355>`_, Yehuda Sadeh)
+* rgw: better error message when user has no bucket created yet (`issue#16444 <http://tracker.ceph.com/issues/16444>`_, `pr#10162 <http://github.com/ceph/ceph/pull/10162>`_, Gaurav Kumar Garg)
+* rgw: clean-up in the authentication infrastructure (`pr#10212 <http://github.com/ceph/ceph/pull/10212>`_, Radoslaw Zarzynski)
+* rgw: clear realm watch on failed watch_restart (`issue#16817 <http://tracker.ceph.com/issues/16817>`_, `pr#10446 <http://github.com/ceph/ceph/pull/10446>`_, Casey Bodley)
+* rgw: collect skips a specific coroutine stack (`issue#16665 <http://tracker.ceph.com/issues/16665>`_, `pr#10274 <http://github.com/ceph/ceph/pull/10274>`_, Yehuda Sadeh)
+* rgw: cosmetic changes only--build verified, f23 (`pr#9931 <http://github.com/ceph/ceph/pull/9931>`_, Yan Jun)
+* rgw: delete region map after upgrade to zonegroup map (`issue#17051 <http://tracker.ceph.com/issues/17051>`_, `pr#10831 <http://github.com/ceph/ceph/pull/10831>`_, Casey Bodley)
+* rgw: do not try to encode or decode time_t and fix compiling warnings (`pr#10751 <http://github.com/ceph/ceph/pull/10751>`_, Kefu Chai)
+* rgw: don't fail if lost race when setting acls (`issue#16930 <http://tracker.ceph.com/issues/16930>`_, `pr#11286 <http://github.com/ceph/ceph/pull/11286>`_, Yehuda Sadeh)
+* rgw: drop create_bucket in fwd_request log message (`pr#10214 <http://github.com/ceph/ceph/pull/10214>`_, Abhishek Lekshmanan)
+* rgw: eradicate dynamic memory allocation in RGWPostObj. (`pr#11054 <http://github.com/ceph/ceph/pull/11054>`_, Radoslaw Zarzynski)
+* rgw: file setattr (`pr#8618 <http://github.com/ceph/ceph/pull/8618>`_, Matt Benjamin)
+* rgw: finish error_repo cr in stop_spawned_services() (`issue#16530 <http://tracker.ceph.com/issues/16530>`_, `pr#10031 <http://github.com/ceph/ceph/pull/10031>`_, Yehuda Sadeh)
+* rgw: fix RGWAccessControlPolicy_SWIFT::create return value check error (`issue#17090 <http://tracker.ceph.com/issues/17090>`_, `pr#10727 <http://github.com/ceph/ceph/pull/10727>`_, weiqiaomiao)
+* rgw: fix compilation (`pr#10252 <http://github.com/ceph/ceph/pull/10252>`_, Josh Durgin)
+* rgw: fix decoding of creation_time and last_update. (`issue#17167 <http://tracker.ceph.com/issues/17167>`_, `pr#11132 <http://github.com/ceph/ceph/pull/11132>`_, Orit Wasserman)
+* rgw: fix error_repo segfault in data sync (`issue#16603 <http://tracker.ceph.com/issues/16603>`_, `pr#10157 <http://github.com/ceph/ceph/pull/10157>`_, Casey Bodley)
+* rgw: fix failed to create bucket if a non-master zonegroup has a single zone (`pr#10991 <http://github.com/ceph/ceph/pull/10991>`_, weiqiaomiao)
+* rgw: fix flush_read_list() error msg (`pr#10749 <http://github.com/ceph/ceph/pull/10749>`_, Jiaying Ren)
+* rgw: fix for issue 16494 (`issue#16494 <http://tracker.ceph.com/issues/16494>`_, `pr#10077 <http://github.com/ceph/ceph/pull/10077>`_, Yehuda Sadeh)
+* rgw: fix for s3tests failure when ldap auth is not applied (`pr#10669 <http://github.com/ceph/ceph/pull/10669>`_, Casey Bodley)
+* rgw: fix get object instance  returned NoSuchKey error (`issue#17111 <http://tracker.ceph.com/issues/17111>`_, `pr#10820 <http://github.com/ceph/ceph/pull/10820>`_, Yang Honggang)
+* rgw: fix is_admin handling in RGWLDAPAuthEngine and introduce acct_privilege_t (`pr#10687 <http://github.com/ceph/ceph/pull/10687>`_, Radoslaw Zarzynski)
+* rgw: fix issue 16435 (`issue#16435 <http://tracker.ceph.com/issues/16435>`_, `pr#10193 <http://github.com/ceph/ceph/pull/10193>`_, Yehuda Sadeh)
+* rgw: fix multi-delete query param parsing. (`issue#16618 <http://tracker.ceph.com/issues/16618>`_, `pr#10187 <http://github.com/ceph/ceph/pull/10187>`_, Robin H. Johnson)
+* rgw: fix period update --commit return error (`issue#17110 <http://tracker.ceph.com/issues/17110>`_, `pr#10836 <http://github.com/ceph/ceph/pull/10836>`_, weiqiaomiao)
+* rgw: fix radosgw daemon core when reopen logs (`issue#17036 <http://tracker.ceph.com/issues/17036>`_, `pr#10737 <http://github.com/ceph/ceph/pull/10737>`_, weiqiaomiao)
+* rgw: fix regression with handling double underscore (`issue#16856 <http://tracker.ceph.com/issues/16856>`_, `pr#10939 <http://github.com/ceph/ceph/pull/10939>`_, Orit Wasserman)
+* rgw: fix rgw_bucket_dir_entry decode v (`pr#10918 <http://github.com/ceph/ceph/pull/10918>`_, Tianshan Qu)
+* rgw: fix the error return variable in log message and cleanups (`pr#10138 <http://github.com/ceph/ceph/pull/10138>`_, Yan Jun)
+* rgw: fix the missing return value (`pr#10122 <http://github.com/ceph/ceph/pull/10122>`_, Yan Jun)
+* rgw: fix upgrade from old multisite to new multisite configuration (`issue#16751 <http://tracker.ceph.com/issues/16751>`_, `pr#10368 <http://github.com/ceph/ceph/pull/10368>`_, Orit Wasserman)
+* rgw: fix wrong  variable definition in cls_version_check func (`pr#10233 <http://github.com/ceph/ceph/pull/10233>`_, weiqiaomiao)
+* rgw: fix wrong variable definition in rgw_cls_lc_set_entry function (`pr#10408 <http://github.com/ceph/ceph/pull/10408>`_, weiqiaomiao)
+* rgw: for the create_bucket api, if the input creation_time is zero, we should set it to 'now" (`issue#16597 <http://tracker.ceph.com/issues/16597>`_, `pr#10118 <http://github.com/ceph/ceph/pull/10118>`_, weiqiaomiao)
+* rgw: kill a compile warning for rgw_sync (`pr#10425 <http://github.com/ceph/ceph/pull/10425>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: lgtm (`pr#9941 <http://github.com/ceph/ceph/pull/9941>`_, weiqiaomiao)
+* rgw: lgtm (build verified, f23) (`pr#9754 <http://github.com/ceph/ceph/pull/9754>`_, John Coyle)
+* rgw: lgtm, build verified f23 (`pr#10035 <http://github.com/ceph/ceph/pull/10035>`_, Yan Jun)
+* rgw: lgtm--build verified, f23 (`pr#10002 <http://github.com/ceph/ceph/pull/10002>`_, Yan Jun)
+* rgw: lgtm--build verified, f23 (`pr#9985 <http://github.com/ceph/ceph/pull/9985>`_, Yan Jun)
+* rgw: lgtm--should backport (`pr#9979 <http://github.com/ceph/ceph/pull/9979>`_, Yan Jun)
+* rgw: log mp upload failures due to parts mismatch (`pr#10424 <http://github.com/ceph/ceph/pull/10424>`_, Abhishek Lekshmanan)
+* rgw: merge setting flags operation together and cleanups (`pr#10203 <http://github.com/ceph/ceph/pull/10203>`_, Yan Jun)
+* rgw: miscellaneous cleanups (`pr#10299 <http://github.com/ceph/ceph/pull/10299>`_, Yan Jun)
+* rgw: multiple fixes for Swift's object expiration (`issue#16705 <http://tracker.ceph.com/issues/16705>`_, `issue#16684 <http://tracker.ceph.com/issues/16684>`_, `pr#10330 <http://github.com/ceph/ceph/pull/10330>`_, Radoslaw Zarzynski)
+* rgw: need to 'open_object_section' before dump stats in 'RGWGetUsage\_… (`issue#17499 <http://tracker.ceph.com/issues/17499>`_, `pr#11325 <http://github.com/ceph/ceph/pull/11325>`_, weiqiaomiao)
+* rgw: obsolete 'radosgw-admin period prepare' command (`issue#17387 <http://tracker.ceph.com/issues/17387>`_, `pr#11278 <http://github.com/ceph/ceph/pull/11278>`_, Gaurav Kumar Garg)
+* rgw: radosgw-admin: add "--orphan-stale-secs" to --help (`issue#17280 <http://tracker.ceph.com/issues/17280>`_, `pr#11098 <http://github.com/ceph/ceph/pull/11098>`_, Ken Dreyer)
+* rgw: radosgw-admin: zone[group] modify can change realm id (`issue#16839 <http://tracker.ceph.com/issues/16839>`_, `pr#10477 <http://github.com/ceph/ceph/pull/10477>`_, Casey Bodley)
+* rgw: raise log levels for common radosgw-admin errors (`issue#16935 <http://tracker.ceph.com/issues/16935>`_, `pr#10602 <http://github.com/ceph/ceph/pull/10602>`_, Shilpa Jagannath)
+* rgw: register the correct handler for cls_user_complete_stats (`issue#16624 <http://tracker.ceph.com/issues/16624>`_, `pr#10151 <http://github.com/ceph/ceph/pull/10151>`_, Orit Wasserman)
+* rgw: remove bucket index objects when deleting the bucket (`issue#16412 <http://tracker.ceph.com/issues/16412>`_, `pr#10120 <http://github.com/ceph/ceph/pull/10120>`_, Orit Wasserman)
+* rgw: remove possible duplicate setting (`pr#10110 <http://github.com/ceph/ceph/pull/10110>`_, Yan Jun)
+* rgw: remove the field ret from class RGWPutLC (`pr#10726 <http://github.com/ceph/ceph/pull/10726>`_, weiqiaomiao)
+* rgw: remove unused bufferlist variable (`pr#10194 <http://github.com/ceph/ceph/pull/10194>`_, Yan Jun)
+* rgw: remove unused realm from radosgw-admin zone modify (`issue#16632 <http://tracker.ceph.com/issues/16632>`_, `pr#10211 <http://github.com/ceph/ceph/pull/10211>`_, Orit Wasserman)
+* rgw: remove unused variables (`pr#10589 <http://github.com/ceph/ceph/pull/10589>`_, Yan Jun)
+* rgw: return "NoSuchLifecycleConfiguration" if lifecycle config does not exist (`pr#10442 <http://github.com/ceph/ceph/pull/10442>`_, weiqiaomiao)
+* rgw: revert a commit that broke s3 signature validation (`issue#17279 <http://tracker.ceph.com/issues/17279>`_, `pr#11102 <http://github.com/ceph/ceph/pull/11102>`_, Casey Bodley)
+* rgw: rgw file: remove busy-wait in RGWLibFS::gc() (`pr#10638 <http://github.com/ceph/ceph/pull/10638>`_, Matt Benjamin)
+* rgw: rgw ldap:  protect rgw::from_base64 from non-base64 input (`pr#10777 <http://github.com/ceph/ceph/pull/10777>`_, Matt Benjamin)
+* rgw: rgw ldap: enforce simple_bind w/LDAPv3 (`pr#10593 <http://github.com/ceph/ceph/pull/10593>`_, Matt Benjamin)
+* rgw: rgw multisite: RGWCoroutinesManager::run returns status of last cr (`issue#17047 <http://tracker.ceph.com/issues/17047>`_, `pr#10778 <http://github.com/ceph/ceph/pull/10778>`_, Casey Bodley)
+* rgw: rgw multisite: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR (`issue#17073 <http://tracker.ceph.com/issues/17073>`_, `pr#10779 <http://github.com/ceph/ceph/pull/10779>`_, Casey Bodley)
+* rgw: rgw multisite: fix for assertion in RGWMetaSyncCR (`issue#17044 <http://tracker.ceph.com/issues/17044>`_, `pr#10743 <http://github.com/ceph/ceph/pull/10743>`_, Casey Bodley)
+* rgw: rgw multisite: fixes for period puller (`issue#16939 <http://tracker.ceph.com/issues/16939>`_, `pr#10596 <http://github.com/ceph/ceph/pull/10596>`_, Casey Bodley)
+* rgw: rgw multisite: trim data logs as peer zones catch up (`pr#10372 <http://github.com/ceph/ceph/pull/10372>`_, Casey Bodley)
+* rgw: rgw nfs v3 completions (`pr#10745 <http://github.com/ceph/ceph/pull/10745>`_, Matt Benjamin)
+* rgw: rgw-admin: allow unsetting user's email (`issue#13286 <http://tracker.ceph.com/issues/13286>`_, `pr#11340 <http://github.com/ceph/ceph/pull/11340>`_, Yehuda Sadeh, Weijun Duan)
+* rgw: rgw/admin: fix some return values and indents (`pr#9170 <http://github.com/ceph/ceph/pull/9170>`_, Yan Jun)
+* rgw: rgw/rados: remove confused error printout (`pr#9351 <http://github.com/ceph/ceph/pull/9351>`_, Yan Jun)
+* rgw: rgw/rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get (`pr#9136 <http://github.com/ceph/ceph/pull/9136>`_, zhao kun)
+* rgw: rgw/rgw_lc.cc: fix sleep time according to the error message (`pr#10930 <http://github.com/ceph/ceph/pull/10930>`_, Weibing Zhang)
+* rgw: rgw/rgw_main: fix unnecessary variables defined (`pr#10475 <http://github.com/ceph/ceph/pull/10475>`_, zhang.zezhu)
+* rgw: rgw/swift: remove redundant assignment operation (`pr#11292 <http://github.com/ceph/ceph/pull/11292>`_, Yan Jun)
+* rgw: rgw_file:  pre-assign times (`issue#17367 <http://tracker.ceph.com/issues/17367>`_, `pr#11181 <http://github.com/ceph/ceph/pull/11181>`_, Matt Benjamin)
+* rgw: rgw_file: fix rename cases and unify unlink (`pr#10271 <http://github.com/ceph/ceph/pull/10271>`_, Matt Benjamin)
+* rgw: rgw_file: fix set_attrs operation (`pr#11159 <http://github.com/ceph/ceph/pull/11159>`_, Matt Benjamin)
+* rgw: rgw_file: refuse partial, out-of-order writes (`pr#10284 <http://github.com/ceph/ceph/pull/10284>`_, Matt Benjamin)
+* rgw: rgw_file: restore local definition of RGWLibFS gc interval (`pr#10756 <http://github.com/ceph/ceph/pull/10756>`_, Matt Benjamin)
+* rgw: rgw_file: unlock() must precede out label (`pr#10635 <http://github.com/ceph/ceph/pull/10635>`_, Matt Benjamin)
+* rgw: right parenthesis is missing in radosgw-admin help message on caps (`pr#10947 <http://github.com/ceph/ceph/pull/10947>`_, Weibing Zhang)
+* rgw: set correct instance on the object (`issue#17443 <http://tracker.ceph.com/issues/17443>`_, `pr#11270 <http://github.com/ceph/ceph/pull/11270>`_, Yehuda Sadeh)
+* rgw: store oldest mdlog period in rados (`issue#16894 <http://tracker.ceph.com/issues/16894>`_, `pr#10558 <http://github.com/ceph/ceph/pull/10558>`_, Casey Bodley)
+* rgw: test/multi.py add a destructive attr to tests (`pr#10401 <http://github.com/ceph/ceph/pull/10401>`_, Abhishek Lekshmanan)
+* rgw: test/rgw: add --gateways-per-zone to test_multi.py (`pr#10742 <http://github.com/ceph/ceph/pull/10742>`_, Casey Bodley)
+* rgw: test_multi.py avoid creating mds (`pr#10174 <http://github.com/ceph/ceph/pull/10174>`_, Abhishek Lekshmanan)
+* rgw: test_rgw_bencode: null terminate strings before checking (`issue#16861 <http://tracker.ceph.com/issues/16861>`_, `pr#10510 <http://github.com/ceph/ceph/pull/10510>`_, Yehuda Sadeh)
+* rgw: use endpoints from master zone instead of zonegroup (`issue#16834 <http://tracker.ceph.com/issues/16834>`_, `pr#10456 <http://github.com/ceph/ceph/pull/10456>`_, Casey Bodley)
+* rgw: use the standard usage of string.find (`pr#10226 <http://github.com/ceph/ceph/pull/10226>`_, Yan Jun)
+* rgw: verfied:  f23, subset of s3tests (`pr#10448 <http://github.com/ceph/ceph/pull/10448>`_, Pritha Srivastava)
+* rgw: verified (`pr#10000 <http://github.com/ceph/ceph/pull/10000>`_, weiqiaomiao)
+* rgw: verified non-regression (MS AD) (`pr#10597 <http://github.com/ceph/ceph/pull/10597>`_, Pritha Srivastava)
+* rgw: verified:  autobuild (`issue#16928 <http://tracker.ceph.com/issues/16928>`_, `pr#10579 <http://github.com/ceph/ceph/pull/10579>`_, Robin H. Johnson)
+* rgw: verified: MS AD (`pr#10307 <http://github.com/ceph/ceph/pull/10307>`_, Pritha Srivastava)
+* rgw: verified: f23 (`pr#10882 <http://github.com/ceph/ceph/pull/10882>`_, Michal Jarzabek)
+* rgw: verified: f23 (`pr#10858 <http://github.com/ceph/ceph/pull/10858>`_, Weibing Zhang)
+* rgw: verified: f23 (`pr#10822 <http://github.com/ceph/ceph/pull/10822>`_, Yan Jun)
+* rgw: verified: f23 (`pr#10929 <http://github.com/ceph/ceph/pull/10929>`_, Weibing Zhang)
+* rgw: wip: rgw multisite: preserve zone's extra pool (`issue#16712 <http://tracker.ceph.com/issues/16712>`_, `pr#10397 <http://github.com/ceph/ceph/pull/10397>`_, Abhishek Lekshmanan)
+* rgw: work around curl_multi_wait bug with non-blocking reads (`issue#15915 <http://tracker.ceph.com/issues/15915>`_, `issue#16695 <http://tracker.ceph.com/issues/16695>`_, `pr#10998 <http://github.com/ceph/ceph/pull/10998>`_, Casey Bodley)
+* rgw:add a s3 API of make torrent for a object (`pr#10396 <http://github.com/ceph/ceph/pull/10396>`_, zhouruisong)
+* rgw:add a s3 API of make torrent for a object (`pr#9589 <http://github.com/ceph/ceph/pull/9589>`_, zhouruisong)
+* rgw:bucket check remove _multipart_ prefix (`pr#6501 <http://github.com/ceph/ceph/pull/6501>`_, Weijun Duan)
+* rgw:clean unuse bufferlist (`pr#10232 <http://github.com/ceph/ceph/pull/10232>`_, weiqiaomiao)
+* rgw:fix rgw boot failed after upgrade to master latest version (`pr#10409 <http://github.com/ceph/ceph/pull/10409>`_, weiqiaomiao)
+* rgw:lifecycle feature [rebased] (`pr#9737 <http://github.com/ceph/ceph/pull/9737>`_, Ji Chen, Daniel Gryniewicz)
+* rgw: rgw/rgw_rados.h: remove unneeded class C_Tick (`pr#10954 <http://github.com/ceph/ceph/pull/10954>`_, Michal Jarzabek)
+* rgw: ext_mime_map_init add string describing for error number (`pr#9807 <http://github.com/ceph/ceph/pull/9807>`_, Yan Jun)
+* tests: Add test for global static non-POD segfault (`pr#10486 <http://github.com/ceph/ceph/pull/10486>`_, Brad Hubbard)
+* tests: populate /dev/disk/by-partuuid for scsi_debug (`issue#17100 <http://tracker.ceph.com/issues/17100>`_, `pr#10824 <http://github.com/ceph/ceph/pull/10824>`_, Loic Dachary)
+* tests: use a fixture for memstore clone testing (`pr#11190 <http://github.com/ceph/ceph/pull/11190>`_, Kefu Chai)
+* tests: run-\*make-check.sh: Make DRY_RUN actually mean a dry run (`pr#11074 <http://github.com/ceph/ceph/pull/11074>`_, Brad Hubbard)
+* tests: run-cmake-check.sh: Actually run the tests (`pr#11075 <http://github.com/ceph/ceph/pull/11075>`_, Brad Hubbard)
+* tests: run-cmake-check.sh: Init submodules (`pr#11091 <http://github.com/ceph/ceph/pull/11091>`_, Brad Hubbard)
+* tests: run-make-check.sh: Make DRY_RUN actually do a dry run (`pr#11092 <http://github.com/ceph/ceph/pull/11092>`_, Brad Hubbard)
+* tests: run-make-check.sh: pass args to do_cmake.sh (`pr#10701 <http://github.com/ceph/ceph/pull/10701>`_, John Coyle)
+* tests: unittest_chain_xattr: account for existing xattrs (`issue#16025 <http://tracker.ceph.com/issues/16025>`_, `pr#11109 <http://github.com/ceph/ceph/pull/11109>`_, Dan Mick)
+* tests: src/test/cli/* tests: POSIX Convert grep -P to grep -E (`pr#10319 <http://github.com/ceph/ceph/pull/10319>`_, Willem Jan Withagen)
+* test: ceph_test_msgr: fix circular locking dependency (`issue#16955 <http://tracker.ceph.com/issues/16955>`_, `pr#10612 <http://github.com/ceph/ceph/pull/10612>`_, Kefu Chai)
+* test: cli/crushtool: fix the test of compile-decompile-recompile.t (`issue#17306 <http://tracker.ceph.com/issues/17306>`_, `pr#11173 <http://github.com/ceph/ceph/pull/11173>`_, Kefu Chai)
+* test: libcephfs: fix gcc sys/fcntl.h warnings (`pr#10126 <http://github.com/ceph/ceph/pull/10126>`_, John Coyle)
+* test: librados: rados_connect() should succeed (`issue#17087 <http://tracker.ceph.com/issues/17087>`_, `pr#10806 <http://github.com/ceph/ceph/pull/10806>`_, Kefu Chai)
+* test: mds: add fs dump in test_ceph_argparse.py (`pr#10347 <http://github.com/ceph/ceph/pull/10347>`_, huanwen ren)
+* test: simple_dispatcher.cc: remove unused variable (`pr#9932 <http://github.com/ceph/ceph/pull/9932>`_, Michal Jarzabek)
+* test: store_test: tidy-up SyntheticWorkloadState class  (`pr#10775 <http://github.com/ceph/ceph/pull/10775>`_, xie xingguo)
+* test: More portable use of mmap(MAP_ANON) (`pr#10557 <http://github.com/ceph/ceph/pull/10557>`_, Willem Jan Withagen)
+* test: Removeall merged after print_function commit needs a fix (`pr#10535 <http://github.com/ceph/ceph/pull/10535>`_, David Zafman)
+* test: ceph-disk.sh do not kill all daemons (`issue#16729 <http://tracker.ceph.com/issues/16729>`_, `pr#10346 <http://github.com/ceph/ceph/pull/10346>`_, Kefu Chai)
+* test: cephtool/test.sh: fix expect_false() calls (`pr#10133 <http://github.com/ceph/ceph/pull/10133>`_, Kefu Chai)
+* test: fix usage info of omapbench (`pr#10089 <http://github.com/ceph/ceph/pull/10089>`_, Wanlong Gao)
+* test: remove ceph_test_rados_api_tmap_migrate (`issue#16144 <http://tracker.ceph.com/issues/16144>`_, `pr#10256 <http://github.com/ceph/ceph/pull/10256>`_, Kefu Chai)
+* test: test_{compression_plugin,async_compressor}: do not copy plugins (`pr#10153 <http://github.com/ceph/ceph/pull/10153>`_, Kefu Chai)
+* test: test_rados_tool.sh: Make script work under ctest (`pr#10166 <http://github.com/ceph/ceph/pull/10166>`_, Willem Jan Withagen)
+* test: qa/workunits/cephtool/test.sh: fix omission of ceph-command (`pr#10979 <http://github.com/ceph/ceph/pull/10979>`_, Willem Jan Withagen)
+* test: qa/workunits/cephtool/test.sh: s/TMPDIR/TEMP_DIR/ (`pr#10306 <http://github.com/ceph/ceph/pull/10306>`_, Kefu Chai)
+* test: qa/workunits/cephtool/test.sh: use absolute path for TEMP_DIR (`pr#10430 <http://github.com/ceph/ceph/pull/10430>`_, Kefu Chai)
+* tools: New "removeall" used to remove head with snapshots (`pr#10098 <http://github.com/ceph/ceph/pull/10098>`_, David Zafman)
+* tools: do not closed stdout ; fix overload of "<" operator (`pr#9290 <http://github.com/ceph/ceph/pull/9290>`_, xie xingguo)
+* tools: fix the core dump when get the crushmap do not exist (`pr#10451 <http://github.com/ceph/ceph/pull/10451>`_, song baisen)
+* tools: rebuild monstore (`issue#17179 <http://tracker.ceph.com/issues/17179>`_, `pr#10933 <http://github.com/ceph/ceph/pull/10933>`_, Kefu Chai)
+* tools: use TextTable for "rados df" plain output (`pr#9362 <http://github.com/ceph/ceph/pull/9362>`_, xie xingguo)
+* tools: fio engine for objectstore (`pr#10267 <http://github.com/ceph/ceph/pull/10267>`_, Casey Bodley, Igor Fedotov, Daniel Gollub)
+* tools: rados/client: fix typo (`pr#10493 <http://github.com/ceph/ceph/pull/10493>`_, Yan Jun)
+* tools: rados/client: fix waiting on the condition variable more efficient. (`pr#9939 <http://github.com/ceph/ceph/pull/9939>`_, Yan Jun)
+* tools: tools/rebuild_mondb: kill comipling warning and other fixes (`pr#11117 <http://github.com/ceph/ceph/pull/11117>`_, xie xingguo)
+* tools: authtool: Enhance argument combinations validation (`issue#2904 <http://tracker.ceph.com/issues/2904>`_, `pr#9704 <http://github.com/ceph/ceph/pull/9704>`_, Brad Hubbard)
+* tools: ceph-disk: change ownership of initfile to ceph:ceph (`issue#16280 <http://tracker.ceph.com/issues/16280>`_, `pr#9688 <http://github.com/ceph/ceph/pull/9688>`_, Shylesh Kumar)
+* test: ceph_test_rados_api_tmap_migrate: remove test for tmap_upgrade (`pr#10234 <http://github.com/ceph/ceph/pull/10234>`_, Kefu Chai)
diff --git a/doc/releases/luminous.rst b/doc/releases/luminous.rst
new file mode 100644
index 000000000..fca0b7d45
--- /dev/null
+++ b/doc/releases/luminous.rst
@@ -0,0 +1,5552 @@
+========
+Luminous
+========
+
+Luminous is the 12th stable release of Ceph.  It is named after the
+luminous squid (watasenia scintillans, aka firefly squid).
+
+v12.2.13 Luminous
+=================
+
+This is the 13th bug fix release of the Luminous v12.2.x long term stable
+release series. We recommend that all users upgrade to this release.
+
+Notable Changes
+---------------
+
+* Ceph now packages python bindings for python3.6 instead of
+  python3.4, because EPEL7 recently switched from python3.4 to
+  python3.6 as the native python3. see the `announcement <https://lists.fedoraproject.org/archives/list/epel-announce@lists.fedoraproject.org/message/EGUMKAIMPK2UD5VSHXM53BH2MBDGDWMO/>_`
+  for more details on the background of this change.
+
+* We now have telemetry support via a ceph-mgr module. The telemetry module is
+  absolutely on an opt-in basis, and is meant to collect generic cluster
+  information and push it to a central endpoint. By default, we're pushing it
+  to a project endpoint at https://telemetry.ceph.com/report, but this is
+  customizable using by setting the 'url' config option with::
+
+    ceph telemetry config-set url '<your url>'
+
+  You will have to opt-in on sharing your information with::
+
+    ceph telemetry on
+
+  You can view exactly what information will be reported first with::
+
+    ceph telemetry show
+
+  Should you opt-in, your information will be licensed under the
+  Community Data License Agreement - Sharing - Version 1.0, which you can
+  read at https://cdla.io/sharing-1-0/
+
+  The telemetry module reports information about CephFS file systems,
+  including:
+
+    - how many MDS daemons (in total and per file system)
+    - which features are (or have been) enabled
+    - how many data pools
+    - approximate file system age (year + month of creation)
+    - how much metadata is being cached per file system
+
+  As well as:
+
+    - whether IPv4 or IPv6 addresses are used for the monitors
+    - whether RADOS cache tiering is enabled (and which mode)
+    - whether pools are replicated or erasure coded, and
+      which erasure code profile plugin and parameters are in use
+    - how many RGW daemons, zones, and zonegroups are present; which RGW frontends are in use
+    - aggregate stats about the CRUSH map, like which algorithms are used, how
+      big buckets are, how many rules are defined, and what tunables are in use
+
+* A health warning is now generated if the average osd heartbeat ping
+  time exceeds a configurable threshold for any of the intervals
+  computed.  The OSD computes 1 minute, 5 minute and 15 minute
+  intervals with average, minimum and maximum values.  New configuration
+  option ``mon_warn_on_slow_ping_ratio`` specifies a percentage of
+  ``osd_heartbeat_grace`` to determine the threshold.  A value of zero
+  disables the warning.  New configuration option
+  ``mon_warn_on_slow_ping_time`` specified in milliseconds over-rides the
+  computed value, causes a warning
+  when OSD heartbeat pings take longer than the specified amount.
+  New admin command ``ceph daemon mgr.# dump_osd_network [threshold]`` command will
+  list all connections with a ping time longer than the specified threshold or
+  value determined by the config options, for the average for any of the 3 intervals.
+  New admin command ``ceph daemon osd.# dump_osd_network [threshold]`` will
+  do the same but only including heartbeats initiated by the specified OSD.
+
+* The configuration value ``osd_calc_pg_upmaps_max_stddev`` used for upmap
+  balancing has been removed. Instead use the mgr balancer config
+  ``upmap_max_deviation`` which now is an integer number of PGs of deviation
+  from the target PGs per OSD.  This can be set with a command like
+  ``ceph config set mgr mgr/balancer/upmap_max_deviation 2``.  The default
+  ``upmap_max_deviation`` is 1.  There are situations where crush rules
+  would not allow a pool to ever have completely balanced PGs.  For example, if
+  crush requires 1 replica on each of 3 racks, but there are fewer OSDs in 1 of
+  the racks.  In those cases, the configuration value can be increased.
+
+Changelog
+---------
+
+* bluestore: >2GB bluefs writes (`pr#28965 <https://github.com/ceph/ceph/pull/28965>`_, kungf, Kefu Chai, Sage Weil)
+* bluestore: Inspect allocations (`pr#29539 <https://github.com/ceph/ceph/pull/29539>`_, Neha Ojha, Adam Kupczyk)
+* bluestore: [AFTER: #28644] luminous: os/bluestore: default to bitmap allocator for bluestore/bluefs (`pr#28972 <https://github.com/ceph/ceph/pull/28972>`_, Igor Fedotov)
+* bluestore: add bluestore_ignore_data_csum option (`pr#26247 <https://github.com/ceph/ceph/pull/26247>`_, Sage Weil)
+* bluestore: apply shared_alloc_size to shared device with log level change (`pr#29910 <https://github.com/ceph/ceph/pull/29910>`_, Vikhyat Umrao, Josh Durgin, Igor Fedotov, Sage Weil)
+* bluestore: avoid length overflow in extents returned by Stupid Alloc (`issue#40703 <http://tracker.ceph.com/issues/40703>`_, `pr#29025 <https://github.com/ceph/ceph/pull/29025>`_, Igor Fedotov)
+* bluestore: call fault_range properly prior to looking for blob to … (`pr#27529 <https://github.com/ceph/ceph/pull/27529>`_, Igor Fedotov)
+* bluestore: common/options: Set concurrent bluestore rocksdb compactions to 2 (`pr#30149 <https://github.com/ceph/ceph/pull/30149>`_, Mark Nelson)
+* bluestore: dump before "no spanning blob id" abort (`pr#28030 <https://github.com/ceph/ceph/pull/28030>`_, Igor Fedotov)
+* bluestore: fix assertion in StupidAllocator::get_fragmentation (`pr#32523 <https://github.com/ceph/ceph/pull/32523>`_, Lei Liu, Igor Fedotov)
+* bluestore: fix duplicate allocations in bmap allocator (`issue#40080 <http://tracker.ceph.com/issues/40080>`_, `pr#28644 <https://github.com/ceph/ceph/pull/28644>`_, Igor Fedotov)
+* bluestore: fix improper setting of STATE_KV_SUBMITTED (`pr#31674 <https://github.com/ceph/ceph/pull/31674>`_, Igor Fedotov)
+* bluestore: fix length overflow (`issue#39247 <http://tracker.ceph.com/issues/39247>`_, `pr#27365 <https://github.com/ceph/ceph/pull/27365>`_, Jianpeng Ma)
+* bluestore: fix out-of-bound access in bmap allocator (`pr#27739 <https://github.com/ceph/ceph/pull/27739>`_, Igor Fedotov)
+* bluestore: load OSD all compression settings unconditionally (`issue#40480 <http://tracker.ceph.com/issues/40480>`_, `pr#28895 <https://github.com/ceph/ceph/pull/28895>`_, Igor Fedotov)
+* bluestore: os/bluestore/BitmapFreelistManager: disable bluestore_debug_freelist (`pr#27459 <https://github.com/ceph/ceph/pull/27459>`_, Sage Weil)
+* bluestore: os/bluestore_tool: bluefs-bdev-expand: indicate bypassed for main dev (`pr#27912 <https://github.com/ceph/ceph/pull/27912>`_, Igor Fedotov)
+* bluestore: test/store_test: fix/workaround for BlobReuseOnOverwriteUT and garbageCollection (`pr#27056 <https://github.com/ceph/ceph/pull/27056>`_, Igor Fedotov)
+* build/ops: admin/build-doc: use python3 (`pr#30665 <https://github.com/ceph/ceph/pull/30665>`_, Kefu Chai, Jason Dillaman)
+* build/ops: admin/build-doc: use python3 (follow-on fix) (`pr#30690 <https://github.com/ceph/ceph/pull/30690>`_, Nathan Cutler)
+* build/ops: backport miscellaneous install-deps.sh and ceph.spec.in fixes from master (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `issue#37707 <http://tracker.ceph.com/issues/37707>`_, `issue#18163 <http://tracker.ceph.com/issues/18163>`_, `issue#22998 <http://tracker.ceph.com/issues/22998>`_, `pr#30722 <https://github.com/ceph/ceph/pull/30722>`_, Yao Guotao, Tomasz Setkowski, Andrey Parfenov, Alfredo Deza, Kefu Chai, Nathan Cutler, Yunchuan Wen, Zack Cerza, Brad Hubbard, Loic Dachary)
+* build/ops: ceph-test RPM not built for SUSE (`pr#29736 <https://github.com/ceph/ceph/pull/29736>`_, Nathan Cutler)
+* build/ops: cmake: pass -march to detect compiler support of arm64 crc/crypto (`issue#36080 <http://tracker.ceph.com/issues/36080>`_, `issue#17516 <http://tracker.ceph.com/issues/17516>`_, `pr#24169 <https://github.com/ceph/ceph/pull/24169>`_, Kefu Chai)
+* build/ops: do_cmake.sh: source not found (`issue#40004 <http://tracker.ceph.com/issues/40004>`_, `issue#39981 <http://tracker.ceph.com/issues/39981>`_, `pr#28216 <https://github.com/ceph/ceph/pull/28216>`_, Nathan Cutler)
+* build/ops: install-deps.sh: Remove CR repo (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#30129 <https://github.com/ceph/ceph/pull/30129>`_, Brad Hubbard, Alfredo Deza)
+* build/ops: python-cephfs should depend on python-rados (`issue#37612 <http://tracker.ceph.com/issues/37612>`_, `issue#24918 <http://tracker.ceph.com/issues/24918>`_, `pr#27950 <https://github.com/ceph/ceph/pull/27950>`_, Kefu Chai)
+* build/ops: python3-cephfs should provide python36-cephfs (`pr#30981 <https://github.com/ceph/ceph/pull/30981>`_, Kefu Chai)
+* build/ops: rpm: Build with lttng on openSUSE (`issue#39332 <http://tracker.ceph.com/issues/39332>`_, `pr#27618 <https://github.com/ceph/ceph/pull/27618>`_, Nathan Cutler)
+* build/ops: rpm: explicitly declare python-tox build dependency (`pr#31934 <https://github.com/ceph/ceph/pull/31934>`_, Nathan Cutler)
+* ceph-volume: assume msgrV1 for all branches containing mimic (`pr#32796 <https://github.com/ceph/ceph/pull/32796>`_, Jan Fajerski)
+* ceph-volume: batch functional idempotency test fails since message is now on stderr (`pr#29791 <https://github.com/ceph/ceph/pull/29791>`_, Jan Fajerski)
+* ceph-volume: broken assertion errors after pytest changes (`pr#28929 <https://github.com/ceph/ceph/pull/28929>`_, Alfredo Deza)
+* ceph-volume: do not fail when trying to remove crypt mapper (`pr#30556 <https://github.com/ceph/ceph/pull/30556>`_, Guillaume Abrioux)
+* ceph-volume: does not recognize wal/db partitions created by ceph-disk (`pr#29462 <https://github.com/ceph/ceph/pull/29462>`_, Jan Fajerski)
+* ceph-volume: fix stderr failure to decode/encode when redirected (`pr#30299 <https://github.com/ceph/ceph/pull/30299>`_, Alfredo Deza)
+* ceph-volume: fix warnings raised by pytest (`pr#30677 <https://github.com/ceph/ceph/pull/30677>`_, Rishabh Dave)
+* ceph-volume: lvm list is O(n^2) (`pr#30094 <https://github.com/ceph/ceph/pull/30094>`_, Rishabh Dave)
+* ceph-volume: lvm.activate: Return an error if WAL/DB devices absent (`pr#29038 <https://github.com/ceph/ceph/pull/29038>`_, David Casier)
+* ceph-volume: lvm.zap fix cleanup for db partitions (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30302 <https://github.com/ceph/ceph/pull/30302>`_, Dominik Csapak)
+* ceph-volume: missing string substitution when reporting mounts (`issue#40978 <http://tracker.ceph.com/issues/40978>`_, `pr#29351 <https://github.com/ceph/ceph/pull/29351>`_, Shyukri Shyukriev)
+* ceph-volume: pre-install python-apt and its variants before test runs (`pr#30296 <https://github.com/ceph/ceph/pull/30296>`_, Alfredo Deza)
+* ceph-volume: prints errors to stdout with --format json (`issue#38548 <http://tracker.ceph.com/issues/38548>`_, `pr#29508 <https://github.com/ceph/ceph/pull/29508>`_, Jan Fajerski)
+* ceph-volume: prints log messages to stdout (`pr#29603 <https://github.com/ceph/ceph/pull/29603>`_, Jan Fajerski, Kefu Chai, Alfredo Deza)
+* ceph-volume: set a lvm_size property on the fakedevice fixture (`pr#30331 <https://github.com/ceph/ceph/pull/30331>`_, Andrew Schoen)
+* ceph-volume: simple: when 'type' file is not present activate fails (`pr#29415 <https://github.com/ceph/ceph/pull/29415>`_, Alfredo Deza)
+* ceph-volume: tests add a sleep in tox for slow OSDs after booting (`pr#28927 <https://github.com/ceph/ceph/pull/28927>`_, Alfredo Deza)
+* ceph-volume: tests set the noninteractive flag for Debian (`pr#29901 <https://github.com/ceph/ceph/pull/29901>`_, Alfredo Deza)
+* ceph-volume: update testing playbook 'deploy.yml' (`pr#29075 <https://github.com/ceph/ceph/pull/29075>`_, Andrew Schoen, Guillaume Abrioux)
+* ceph-volume: use the Device.rotational property instead of sys_api (`pr#28519 <https://github.com/ceph/ceph/pull/28519>`_, Andrew Schoen)
+* ceph-volume: use the OSD identifier when reporting success (`pr#29771 <https://github.com/ceph/ceph/pull/29771>`_, Alfredo Deza)
+* ceph-volume: zap always skips block.db, leaves them around (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30305 <https://github.com/ceph/ceph/pull/30305>`_, Alfredo Deza)
+* cephfs: client: _readdir_cache_cb() may use the readdir_cache already clear (`issue#41148 <http://tracker.ceph.com/issues/41148>`_, `pr#30934 <https://github.com/ceph/ceph/pull/30934>`_, huanwen ren)
+* cephfs: client: ceph.dir.rctime xattr value incorrectly prefixes 09 to the nanoseconds component (`issue#40166 <http://tracker.ceph.com/issues/40166>`_, `pr#28502 <https://github.com/ceph/ceph/pull/28502>`_, David Disseldorp)
+* cephfs: client: clean up error checking and return of _lookup_parent (`issue#40085 <http://tracker.ceph.com/issues/40085>`_, `pr#28437 <https://github.com/ceph/ceph/pull/28437>`_, Jeff Layton)
+* cephfs: client: return -EIO when sync file which unsafe reqs have been dropped (`issue#40877 <http://tracker.ceph.com/issues/40877>`_, `pr#30242 <https://github.com/ceph/ceph/pull/30242>`_, simon gao)
+* cephfs: client: unlink dentry for inode with llref=0 (`issue#40960 <http://tracker.ceph.com/issues/40960>`_, `pr#29830 <https://github.com/ceph/ceph/pull/29830>`_, Xiaoxi CHEN)
+* cephfs: kclient: nofail option not supported (`pr#28436 <https://github.com/ceph/ceph/pull/28436>`_, Kenneth Waegeman)
+* cephfs: mds/server: check directory split after rename (`issue#39198 <http://tracker.ceph.com/issues/39198>`_, `issue#38994 <http://tracker.ceph.com/issues/38994>`_, `pr#27801 <https://github.com/ceph/ceph/pull/27801>`_, Shen Hang)
+* cephfs: mds: add command that config individual client session (`issue#40811 <http://tracker.ceph.com/issues/40811>`_, `pr#31573 <https://github.com/ceph/ceph/pull/31573>`_, "Yan, Zheng")
+* cephfs: mds: add reference when setting Connection::priv to existing session (`pr#31049 <https://github.com/ceph/ceph/pull/31049>`_, "Yan, Zheng")
+* cephfs: mds: avoid trimming too many log segments after mds failover (`issue#40028 <http://tracker.ceph.com/issues/40028>`_, `pr#28543 <https://github.com/ceph/ceph/pull/28543>`_, simon gao)
+* cephfs: mds: better output of 'ceph health detail' (`issue#39266 <http://tracker.ceph.com/issues/39266>`_, `pr#27848 <https://github.com/ceph/ceph/pull/27848>`_, Shen Hang)
+* cephfs: mds: check dir fragment to split dir if mkdir makes it oversized (`pr#29829 <https://github.com/ceph/ceph/pull/29829>`_, Erqi Chen)
+* cephfs: mds: cleanup truncating inodes when standby replay mds trim log segments (`pr#31286 <https://github.com/ceph/ceph/pull/31286>`_, "Yan, Zheng")
+* cephfs: mds: dont print subtrees if they are too big or too many (`pr#27679 <https://github.com/ceph/ceph/pull/27679>`_, Rishabh Dave)
+* cephfs: mds: drop reconnect message from non-existent session (`issue#39191 <http://tracker.ceph.com/issues/39191>`_, `issue#39026 <http://tracker.ceph.com/issues/39026>`_, `pr#27737 <https://github.com/ceph/ceph/pull/27737>`_, Shen Hang)
+* cephfs: mds: fix corner case of replaying open sessions (`pr#28536 <https://github.com/ceph/ceph/pull/28536>`_, "Yan, Zheng")
+* cephfs: mds: initialize cap_revoke_eviction_timeout with conf (`issue#38844 <http://tracker.ceph.com/issues/38844>`_, `issue#39208 <http://tracker.ceph.com/issues/39208>`_, `pr#27840 <https://github.com/ceph/ceph/pull/27840>`_, simon gao)
+* cephfs: mds: msg weren't destroyed before handle_client_reconnect returned, if the reconnect msg was from non-existent session (`issue#40588 <http://tracker.ceph.com/issues/40588>`_, `issue#40807 <http://tracker.ceph.com/issues/40807>`_, `pr#29097 <https://github.com/ceph/ceph/pull/29097>`_, Shen Hang)
+* cephfs: mds: remove superfluous error in StrayManager::advance_delayed() (`issue#38679 <http://tracker.ceph.com/issues/38679>`_, `pr#28432 <https://github.com/ceph/ceph/pull/28432>`_, "Yan, Zheng")
+* cephfs: mds: reset heartbeat inside big loop (`pr#28544 <https://github.com/ceph/ceph/pull/28544>`_, "Yan, Zheng")
+* cephfs: mds: there is an assertion when calling Beacon::shutdown() (`issue#38822 <http://tracker.ceph.com/issues/38822>`_, `pr#28438 <https://github.com/ceph/ceph/pull/28438>`_, huanwen ren)
+* cephfs: mount: key parsing fail when doing a remount (`issue#40163 <http://tracker.ceph.com/issues/40163>`_, `pr#29226 <https://github.com/ceph/ceph/pull/29226>`_, Luis Henriques)
+* cephfs: pybind/ceph_volume_client: remove ceph mds calls in favor of ceph fs calls (`issue#22038 <http://tracker.ceph.com/issues/22038>`_, `issue#22524 <http://tracker.ceph.com/issues/22524>`_, `pr#28445 <https://github.com/ceph/ceph/pull/28445>`_, Patrick Donnelly, Ramana Raja)
+* cephfs: qa/cephfs: relax min_caps_per_client check (`issue#38270 <http://tracker.ceph.com/issues/38270>`_, `issue#38686 <http://tracker.ceph.com/issues/38686>`_, `pr#27040 <https://github.com/ceph/ceph/pull/27040>`_, "Yan, Zheng")
+* cephfs: qa: misc cache drop fixes (`issue#38340 <http://tracker.ceph.com/issues/38340>`_, `issue#38445 <http://tracker.ceph.com/issues/38445>`_, `pr#27342 <https://github.com/ceph/ceph/pull/27342>`_, Patrick Donnelly)
+* common/config: hold lock while accessing mutable container (`pr#30345 <https://github.com/ceph/ceph/pull/30345>`_, Jason Dillaman)
+* common: Keyrings created by ceph auth get are not suitable for ceph auth import (`issue#40548 <http://tracker.ceph.com/issues/40548>`_, `issue#22227 <http://tracker.ceph.com/issues/22227>`_, `pr#28742 <https://github.com/ceph/ceph/pull/28742>`_, Kefu Chai)
+* common: common/ceph_context: avoid unnecessary wait during service thread shutdown (`pr#31020 <https://github.com/ceph/ceph/pull/31020>`_, Jason Dillaman)
+* common: common/options.cc: Lower the default value of osd_deep_scrub_large_omap_object_key_threshold (`pr#29175 <https://github.com/ceph/ceph/pull/29175>`_, Neha Ojha)
+* common: common/util: handle long lines in /proc/cpuinfo (`issue#38296 <http://tracker.ceph.com/issues/38296>`_, `pr#32349 <https://github.com/ceph/ceph/pull/32349>`_, Sage Weil)
+* common: compressor/zstd: improvements (`pr#28647 <https://github.com/ceph/ceph/pull/28647>`_, Adam C. Emerson, Sage Weil)
+* common: data race in OutputDataSocket (`issue#40188 <http://tracker.ceph.com/issues/40188>`_, `issue#40266 <http://tracker.ceph.com/issues/40266>`_, `pr#29202 <https://github.com/ceph/ceph/pull/29202>`_, Casey Bodley)
+* core: ENOENT in collection_move_rename on EC backfill target (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `issue#38880 <http://tracker.ceph.com/issues/38880>`_, `pr#28110 <https://github.com/ceph/ceph/pull/28110>`_, Neha Ojha)
+* core: Health warnings on long network ping times (`issue#40586 <http://tracker.ceph.com/issues/40586>`_, `issue#40640 <http://tracker.ceph.com/issues/40640>`_, `pr#30230 <https://github.com/ceph/ceph/pull/30230>`_, xie xingguo, David Zafman)
+* core: Revert "crush: remove invalid upmap items" (`pr#32019 <https://github.com/ceph/ceph/pull/32019>`_, David Zafman)
+* core: backport recent messenger fixes (`issue#39243 <http://tracker.ceph.com/issues/39243>`_, `issue#38242 <http://tracker.ceph.com/issues/38242>`_, `issue#39448 <http://tracker.ceph.com/issues/39448>`_, `pr#27583 <https://github.com/ceph/ceph/pull/27583>`_, xie xingguo, Jason Dillaman)
+* core: ceph tell osd.xx bench help : gives wrong help (`issue#39006 <http://tracker.ceph.com/issues/39006>`_, `issue#39373 <http://tracker.ceph.com/issues/39373>`_, `pr#28112 <https://github.com/ceph/ceph/pull/28112>`_, Neha Ojha)
+* core: ceph-objectstore-tool: rename dump-import to dump-export (`issue#39343 <http://tracker.ceph.com/issues/39343>`_, `issue#39284 <http://tracker.ceph.com/issues/39284>`_, `pr#27636 <https://github.com/ceph/ceph/pull/27636>`_, David Zafman)
+* core: crc cache should be invalidated when posting preallocated rx buffers (`issue#38436 <http://tracker.ceph.com/issues/38436>`_, `pr#29248 <https://github.com/ceph/ceph/pull/29248>`_, Ilya Dryomov)
+* core: crush/CrushWrapper: ensure crush_choose_arg_map.size == max_buckets (`issue#38664 <http://tracker.ceph.com/issues/38664>`_, `issue#38719 <http://tracker.ceph.com/issues/38719>`_, `pr#27085 <https://github.com/ceph/ceph/pull/27085>`_, Sage Weil)
+* core: crush: remove invalid upmap items (`pr#31234 <https://github.com/ceph/ceph/pull/31234>`_, huangjun)
+* core: lazy omap stat collection (`pr#29190 <https://github.com/ceph/ceph/pull/29190>`_, Brad Hubbard)
+* core: mds,osd,mon,msg: use intrusive_ptr for holding Connection::priv (`issue#20924 <http://tracker.ceph.com/issues/20924>`_, `pr#29859 <https://github.com/ceph/ceph/pull/29859>`_, Shinobu Kinjo, Kefu Chai, Jianpeng Ma, Samuel Just)
+* core: mgr/localpool: pg_num is an int arg to 'osd pool create' (`pr#30446 <https://github.com/ceph/ceph/pull/30446>`_, Sage Weil)
+* core: mgr/prometheus: assign a value to osd_dev_node when obj_store is not filestore or bluestore (`pr#31587 <https://github.com/ceph/ceph/pull/31587>`_, jiahuizeng)
+* core: mon, osd: parallel clean_pg_upmaps (`issue#40229 <http://tracker.ceph.com/issues/40229>`_, `issue#40104 <http://tracker.ceph.com/issues/40104>`_, `pr#28594 <https://github.com/ceph/ceph/pull/28594>`_, xie xingguo)
+* core: mon,osd: limit MOSDMap messages by size as well as map count (`issue#38276 <http://tracker.ceph.com/issues/38276>`_, `pr#28640 <https://github.com/ceph/ceph/pull/28640>`_, Sage Weil)
+* core: mon/OSDMonitor: trim not-longer-exist failure reporters (`pr#30905 <https://github.com/ceph/ceph/pull/30905>`_, NancySu05)
+* core: mon: Error message displayed when mon_osd_max_split_count would be exceeded is not as user-friendly as it could be (`issue#39353 <http://tracker.ceph.com/issues/39353>`_, `issue#39563 <http://tracker.ceph.com/issues/39563>`_, `pr#27908 <https://github.com/ceph/ceph/pull/27908>`_, Nathan Cutler, Brad Hubbard)
+* core: mon: ensure prepare_failure() marks no_reply on op (`pr#30519 <https://github.com/ceph/ceph/pull/30519>`_, Joao Eduardo Luis)
+* core: mon: mon/AuthMonitor: don't validate fs caps on authorize (`pr#28666 <https://github.com/ceph/ceph/pull/28666>`_, Joao Eduardo Luis)
+* core: msg: output peer address when detecting bad CRCs (`issue#39367 <http://tracker.ceph.com/issues/39367>`_, `pr#27858 <https://github.com/ceph/ceph/pull/27858>`_, Greg Farnum)
+* core: osd/OSDMap.cc: don't output over/underfull messages to lderr (`pr#31598 <https://github.com/ceph/ceph/pull/31598>`_, Neha Ojha)
+* core: osd/OSDMap: Replace get_out_osds with get_out_existing_osds (`issue#39154 <http://tracker.ceph.com/issues/39154>`_, `issue#39420 <http://tracker.ceph.com/issues/39420>`_, `pr#27728 <https://github.com/ceph/ceph/pull/27728>`_, Brad Hubbard)
+* core: osd/OSDMap: do not trust partially simplified pg_upmap_item (`pr#30926 <https://github.com/ceph/ceph/pull/30926>`_, xie xingguo)
+* core: osd/PG: Add PG to large omap log message (`pr#30922 <https://github.com/ceph/ceph/pull/30922>`_, Brad Hubbard)
+* core: osd/PG: discover missing objects when an OSD peers and PG is degraded (`pr#27751 <https://github.com/ceph/ceph/pull/27751>`_, Jonas Jelten)
+* core: osd/PGLog: preserve original_crt to check rollbackability (`issue#38894 <http://tracker.ceph.com/issues/38894>`_, `issue#38905 <http://tracker.ceph.com/issues/38905>`_, `issue#36739 <http://tracker.ceph.com/issues/36739>`_, `issue#39042 <http://tracker.ceph.com/issues/39042>`_, `pr#27715 <https://github.com/ceph/ceph/pull/27715>`_, Neha Ojha)
+* core: osd/PeeringState: recover_got - add special handler for empty log (`pr#30896 <https://github.com/ceph/ceph/pull/30896>`_, xie xingguo)
+* core: osd/PrimaryLogPG: skip obcs that don't exist during backfill scan_range (`pr#31030 <https://github.com/ceph/ceph/pull/31030>`_, Sage Weil)
+* core: osd/ReplicatedBackend.cc: 1321: FAILED assert(get_parent()->get_log().get_log().objects.count(soid) && (get_parent()->get_log().get_log().objects.find(soid)->second->op == pg_log_entry_t::LOST_REVERT) && (get_parent()->get_log().get_log().object (`issue#39537 <http://tracker.ceph.com/issues/39537>`_, `issue#26958 <http://tracker.ceph.com/issues/26958>`_, `pr#28989 <https://github.com/ceph/ceph/pull/28989>`_, xie xingguo)
+* core: osd/ReplicatedBackend.cc: 1349: FAILED ceph_assert(peer_missing.count(fromshard)) (`pr#31855 <https://github.com/ceph/ceph/pull/31855>`_, Neha Ojha, xie xingguo)
+* core: osd/bluestore: Actually wait until completion in write_sync (`pr#29564 <https://github.com/ceph/ceph/pull/29564>`_, Vitaliy Filippov)
+* core: osd: Better error message when OSD count is less than osd_pool_default_size (`issue#38617 <http://tracker.ceph.com/issues/38617>`_, `issue#38585 <http://tracker.ceph.com/issues/38585>`_, `pr#30298 <https://github.com/ceph/ceph/pull/30298>`_, Vikhyat Umrao, Kefu Chai, Sage Weil, zjh)
+* core: osd: Diagnostic logging for upmap cleaning (`pr#32666 <https://github.com/ceph/ceph/pull/32666>`_, David Zafman)
+* core: osd: FAILED ceph_assert(attrs || !pg_log.get_missing().is_missing(soid) || (it_objects != pg_log.get_log().objects.end() && it_objects->second->op == pg_log_entry_t::LOST_REVERT)) in PrimaryLogPG::get_object_context() (`issue#39218 <http://tracker.ceph.com/issues/39218>`_, `issue#38931 <http://tracker.ceph.com/issues/38931>`_, `issue#38784 <http://tracker.ceph.com/issues/38784>`_, `pr#27878 <https://github.com/ceph/ceph/pull/27878>`_, xie xingguo)
+* core: osd: Fix for compatibility of encode/decode of osd_stat_t (`pr#31277 <https://github.com/ceph/ceph/pull/31277>`_, David Zafman)
+* core: osd: Include dups in copy_after() and copy_up_to() (`issue#39304 <http://tracker.ceph.com/issues/39304>`_, `pr#28185 <https://github.com/ceph/ceph/pull/28185>`_, David Zafman)
+* core: osd: Remove unused osdmap flags full, nearfull from output (`issue#22350 <http://tracker.ceph.com/issues/22350>`_, `pr#30902 <https://github.com/ceph/ceph/pull/30902>`_, Gu Zhongyan, David Zafman)
+* core: osd: add hdd, ssd and hybrid variants for osd_snap_trim_sleep (`pr#31857 <https://github.com/ceph/ceph/pull/31857>`_, Neha Ojha)
+* core: osd: clear PG_STATE_CLEAN when repair object (`pr#30271 <https://github.com/ceph/ceph/pull/30271>`_, Zengran Zhang)
+* core: osd: fix out of order caused by letting old msg from down osd be processed (`pr#31293 <https://github.com/ceph/ceph/pull/31293>`_, Mingxin Liu)
+* core: osd: merge replica log on primary need according to replica log's crt (`pr#30917 <https://github.com/ceph/ceph/pull/30917>`_, Zengran Zhang)
+* core: osd: refuse to start if we're > N+2 from recorded require_osd_release (`issue#38076 <http://tracker.ceph.com/issues/38076>`_, `pr#31858 <https://github.com/ceph/ceph/pull/31858>`_, Sage Weil)
+* core: osd: report omap/data/metadata usage (`issue#40638 <http://tracker.ceph.com/issues/40638>`_, `pr#28851 <https://github.com/ceph/ceph/pull/28851>`_, Sage Weil)
+* core: osd: rollforward may need to mark pglog dirty (`issue#40403 <http://tracker.ceph.com/issues/40403>`_, `pr#31036 <https://github.com/ceph/ceph/pull/31036>`_, Zengran Zhang)
+* core: osd: scrub error on big objects; make bluestore refuse to start on big objects (`pr#30785 <https://github.com/ceph/ceph/pull/30785>`_, Sage Weil, David Zafman)
+* core: osd: shutdown recovery_request_timer earlier (`issue#39204 <http://tracker.ceph.com/issues/39204>`_, `pr#27810 <https://github.com/ceph/ceph/pull/27810>`_, Zengran Zhang)
+* core: pybind: Rados.get_fsid() returning bytes in python3 (`issue#38873 <http://tracker.ceph.com/issues/38873>`_, `issue#38381 <http://tracker.ceph.com/issues/38381>`_, `pr#27674 <https://github.com/ceph/ceph/pull/27674>`_, Jason Dillaman)
+* core: should report EINVAL in ErasureCode::parse() if m<=0 (`issue#38682 <http://tracker.ceph.com/issues/38682>`_, `issue#38750 <http://tracker.ceph.com/issues/38750>`_, `pr#28111 <https://github.com/ceph/ceph/pull/28111>`_, Sage Weil)
+* doc: Minor rados related documentation fixes (`issue#38896 <http://tracker.ceph.com/issues/38896>`_, `issue#38902 <http://tracker.ceph.com/issues/38902>`_, `pr#27185 <https://github.com/ceph/ceph/pull/27185>`_, David Zafman)
+* doc: Missing Documentation for radosgw-admin reshard commands (man pages) (`issue#40092 <http://tracker.ceph.com/issues/40092>`_, `issue#21617 <http://tracker.ceph.com/issues/21617>`_, `pr#28329 <https://github.com/ceph/ceph/pull/28329>`_, Orit Wasserman)
+* doc: Update layout.rst (`pr#26381 <https://github.com/ceph/ceph/pull/26381>`_, ypdai)
+* doc: describe metadata_heap cleanup (`issue#18174 <http://tracker.ceph.com/issues/18174>`_, `pr#30071 <https://github.com/ceph/ceph/pull/30071>`_, Dan van der Ster)
+* doc: doc/rbd: s/guess/xml/ for codeblock lexer (`pr#31091 <https://github.com/ceph/ceph/pull/31091>`_, Kefu Chai)
+* doc: doc/rgw: document CreateBucketConfiguration for s3 PUT Bucket api (`issue#39597 <http://tracker.ceph.com/issues/39597>`_, `pr#31647 <https://github.com/ceph/ceph/pull/31647>`_, Casey Bodley)
+* doc: doc/rgw: document use of 'realm pull' instead of 'period pull' (`issue#39655 <http://tracker.ceph.com/issues/39655>`_, `pr#30132 <https://github.com/ceph/ceph/pull/30132>`_, Casey Bodley)
+* doc: fixed --read-only argument value in multisite doc (`pr#31655 <https://github.com/ceph/ceph/pull/31655>`_, Chenjiong Deng)
+* doc: osd_recovery_priority is not documented (but osd_recovery_op_priority is) (`pr#27471 <https://github.com/ceph/ceph/pull/27471>`_, David Zafman)
+* doc: update bluestore cache settings and clarify data fraction (`issue#39522 <http://tracker.ceph.com/issues/39522>`_, `pr#31257 <https://github.com/ceph/ceph/pull/31257>`_, Jan Fajerski)
+* doc: wrong datatype describing crush_rule (`pr#32267 <https://github.com/ceph/ceph/pull/32267>`_, Kefu Chai)
+* doc: wrong value of usage log default in logging section (`issue#37892 <http://tracker.ceph.com/issues/37892>`_, `issue#37856 <http://tracker.ceph.com/issues/37856>`_, `pr#29015 <https://github.com/ceph/ceph/pull/29015>`_, Abhishek Lekshmanan)
+* mgr: Change default upmap_max_deviation to 5 (`pr#32586 <https://github.com/ceph/ceph/pull/32586>`_, David Zafman)
+* mgr: Release GIL and Balancer fixes (`pr#31992 <https://github.com/ceph/ceph/pull/31992>`_, Kefu Chai, Noah Watkins, David Zafman)
+* mgr: mgr/BaseMgrModule: drop GIL in set_config (`issue#39040 <http://tracker.ceph.com/issues/39040>`_, `issue#36766 <http://tracker.ceph.com/issues/36766>`_, `pr#27808 <https://github.com/ceph/ceph/pull/27808>`_, John Spray, xie xingguo, Sage Weil)
+* mgr: mgr/balancer: blame if upmap won't actually work (`issue#38781 <http://tracker.ceph.com/issues/38781>`_, `pr#26498 <https://github.com/ceph/ceph/pull/26498>`_, xie xingguo)
+* mgr: mgr/balancer: python3 compatibility issue (`pr#31104 <https://github.com/ceph/ceph/pull/31104>`_, Mykola Golub)
+* mgr: mgr/prometheus: Cast collect_timeout (scrape_interval) to float (`pr#31107 <https://github.com/ceph/ceph/pull/31107>`_, Ben Meekhof)
+* mgr: mgr/prometheus: replace whitespaces in metrics' names (`pr#31105 <https://github.com/ceph/ceph/pull/31105>`_, Alfonso Martínez)
+* mgr: DaemonServer::handle_conf_change - broken locking (`issue#38899 <http://tracker.ceph.com/issues/38899>`_, `issue#38962 <http://tracker.ceph.com/issues/38962>`_, `pr#29213 <https://github.com/ceph/ceph/pull/29213>`_, xie xingguo)
+* mgr: pybind/mgr: Cancel output color control (`pr#31696 <https://github.com/ceph/ceph/pull/31696>`_, Zheng Yin)
+* mgr: restful: Query nodes_by_id for items (`pr#31272 <https://github.com/ceph/ceph/pull/31272>`_, Boris Ranto)
+* mgr: telemetry module for mgr (`issue#37976 <http://tracker.ceph.com/issues/37976>`_, `pr#32135 <https://github.com/ceph/ceph/pull/32135>`_, Joao Eduardo Luis, Wido den Hollander, Kefu Chai, Sage Weil, Dan Mick)
+* rbd: Reduce log level for cls/journal and cls/rbd expected errors (`issue#40865 <http://tracker.ceph.com/issues/40865>`_, `pr#30857 <https://github.com/ceph/ceph/pull/30857>`_, Jason Dillaman)
+* rbd: journal: properly advance read offset after skipping invalid range (`pr#28811 <https://github.com/ceph/ceph/pull/28811>`_, Mykola Golub)
+* rbd: krbd: avoid udev netlink socket overrun and retry on transient errors from udev_enumerate_scan_devices() (`issue#39089 <http://tracker.ceph.com/issues/39089>`_, `pr#31360 <https://github.com/ceph/ceph/pull/31360>`_, Zhi Zhang, Ilya Dryomov)
+* rbd: krbd: return -ETIMEDOUT in polling (`issue#38792 <http://tracker.ceph.com/issues/38792>`_, `issue#38975 <http://tracker.ceph.com/issues/38975>`_, `pr#27536 <https://github.com/ceph/ceph/pull/27536>`_, Dongsheng Yang)
+* rbd: librbd: add missing shutdown states to managed lock helper (`issue#38387 <http://tracker.ceph.com/issues/38387>`_, `issue#38508 <http://tracker.ceph.com/issues/38508>`_, `pr#28158 <https://github.com/ceph/ceph/pull/28158>`_, Jason Dillaman)
+* rbd: librbd: async open/close should free ImageCtx before issuing callback (`issue#39427 <http://tracker.ceph.com/issues/39427>`_, `issue#39031 <http://tracker.ceph.com/issues/39031>`_, `pr#28126 <https://github.com/ceph/ceph/pull/28126>`_, Jason Dillaman)
+* rbd: librbd: disable image mirroring when moving to trash (`pr#28149 <https://github.com/ceph/ceph/pull/28149>`_, Mykola Golub)
+* rbd: librbd: ensure compare-and-write doesn't skip compare after copyup (`issue#38440 <http://tracker.ceph.com/issues/38440>`_, `issue#38383 <http://tracker.ceph.com/issues/38383>`_, `pr#28134 <https://github.com/ceph/ceph/pull/28134>`_, Ilya Dryomov)
+* rbd: librbd: improve object map performance under high IOPS workloads (`issue#38674 <http://tracker.ceph.com/issues/38674>`_, `issue#38538 <http://tracker.ceph.com/issues/38538>`_, `pr#28137 <https://github.com/ceph/ceph/pull/28137>`_, Jason Dillaman)
+* rbd: librbd: properly track in-flight flush requests (`issue#40574 <http://tracker.ceph.com/issues/40574>`_, `pr#28773 <https://github.com/ceph/ceph/pull/28773>`_, Jason Dillaman)
+* rbd: librbd: race condition possible when validating RBD pool (`issue#38500 <http://tracker.ceph.com/issues/38500>`_, `issue#38564 <http://tracker.ceph.com/issues/38564>`_, `pr#28140 <https://github.com/ceph/ceph/pull/28140>`_, Jason Dillaman)
+* rbd: rbd-mirror: clear out bufferlist prior to listing mirror images (`issue#39460 <http://tracker.ceph.com/issues/39460>`_, `issue#39407 <http://tracker.ceph.com/issues/39407>`_, `pr#28124 <https://github.com/ceph/ceph/pull/28124>`_, Jason Dillaman)
+* rbd: rbd-mirror: don't overwrite status error returned by replay (`pr#29874 <https://github.com/ceph/ceph/pull/29874>`_, Mykola Golub)
+* rbd: rbd-mirror: handle duplicates in image sync throttler queue (`issue#40592 <http://tracker.ceph.com/issues/40592>`_, `issue#40519 <http://tracker.ceph.com/issues/40519>`_, `pr#28812 <https://github.com/ceph/ceph/pull/28812>`_, Mykola Golub)
+* rbd: rbd-mirror: ignore errors relating to parsing the cluster config file (`pr#30118 <https://github.com/ceph/ceph/pull/30118>`_, Jason Dillaman)
+* rbd: rbd-mirror: make logrotate work (`pr#32599 <https://github.com/ceph/ceph/pull/32599>`_, Mykola Golub)
+* rbd: rbd/action: fix error getting positional argument (`issue#40095 <http://tracker.ceph.com/issues/40095>`_, `pr#29295 <https://github.com/ceph/ceph/pull/29295>`_, songweibin)
+* rbd: tools/rbd-ggate: close log before running postfork (`pr#30858 <https://github.com/ceph/ceph/pull/30858>`_, Willem Jan Withagen)
+* rbd: use the ordered throttle for the export action (`issue#40435 <http://tracker.ceph.com/issues/40435>`_, `pr#30856 <https://github.com/ceph/ceph/pull/30856>`_, Jason Dillaman)
+* rgw: Adding tcp_nodelay option to Beast (`issue#38925 <http://tracker.ceph.com/issues/38925>`_, `pr#27424 <https://github.com/ceph/ceph/pull/27424>`_, Or Friedmann)
+* rgw: GetBucketCORS API returns Not Found error code when CORS configuration does not exist (`issue#38887 <http://tracker.ceph.com/issues/38887>`_, `issue#26964 <http://tracker.ceph.com/issues/26964>`_, `pr#27123 <https://github.com/ceph/ceph/pull/27123>`_, yuliyang, ashitakasam)
+* rgw: LC: handle resharded buckets (`pr#29122 <https://github.com/ceph/ceph/pull/29122>`_, Abhishek Lekshmanan)
+* rgw: RGWCoroutine::call(nullptr) sets retcode=0 (`pr#30329 <https://github.com/ceph/ceph/pull/30329>`_, Casey Bodley)
+* rgw: TempURL should not allow PUTs with the X-Object-Manifest (`issue#20797 <http://tracker.ceph.com/issues/20797>`_, `pr#31652 <https://github.com/ceph/ceph/pull/31652>`_, Radoslaw Zarzynski)
+* rgw: add list user admin OP API (`pr#30984 <https://github.com/ceph/ceph/pull/30984>`_, Oshyn Song)
+* rgw: allow radosgw-admin to list bucket w --allow-unordered (`pr#31220 <https://github.com/ceph/ceph/pull/31220>`_, J. Eric Ivancich)
+* rgw: civetweb frontend: response is buffered in memory if content length is not explicitly specified (`issue#39615 <http://tracker.ceph.com/issues/39615>`_, `issue#12713 <http://tracker.ceph.com/issues/12713>`_, `pr#28069 <https://github.com/ceph/ceph/pull/28069>`_, Robin H. Johnson)
+* rgw: cls/rgw: raise debug level of bi_log_iterate_entries output (`issue#40559 <http://tracker.ceph.com/issues/40559>`_, `pr#27974 <https://github.com/ceph/ceph/pull/27974>`_, Casey Bodley)
+* rgw: cls/user: cls_user_set_buckets_info overwrites creation_time (`issue#39635 <http://tracker.ceph.com/issues/39635>`_, `pr#31648 <https://github.com/ceph/ceph/pull/31648>`_, Casey Bodley)
+* rgw: conditionally allow builtin users with non-unique email addresses (`issue#40089 <http://tracker.ceph.com/issues/40089>`_, `issue#40506 <http://tracker.ceph.com/issues/40506>`_, `pr#28717 <https://github.com/ceph/ceph/pull/28717>`_, Matt Benjamin)
+* rgw: crypt: permit RGW-AUTO/default with SSE-S3 headers (`pr#31860 <https://github.com/ceph/ceph/pull/31860>`_, Matt Benjamin)
+* rgw: datalog/mdlog trim commands loop until done (`pr#29713 <https://github.com/ceph/ceph/pull/29713>`_, Casey Bodley)
+* rgw: delete_obj_index() takes mtime for bilog (`issue#24991 <http://tracker.ceph.com/issues/24991>`_, `pr#31649 <https://github.com/ceph/ceph/pull/31649>`_, Casey Bodley)
+* rgw: don't crash on missing /etc/mime.types (`issue#38920 <http://tracker.ceph.com/issues/38920>`_, `issue#38328 <http://tracker.ceph.com/issues/38328>`_, `pr#27332 <https://github.com/ceph/ceph/pull/27332>`_, Casey Bodley)
+* rgw: don't throw when accept errors are happening on frontend (`pr#30147 <https://github.com/ceph/ceph/pull/30147>`_, Yuval Lifshitz)
+* rgw: failed to pass test_bucket_create_naming_bad_punctuation in s3test (`issue#39360 <http://tracker.ceph.com/issues/39360>`_, `issue#39358 <http://tracker.ceph.com/issues/39358>`_, `issue#23587 <http://tracker.ceph.com/issues/23587>`_, `issue#26965 <http://tracker.ceph.com/issues/26965>`_, `pr#27668 <https://github.com/ceph/ceph/pull/27668>`_, yuliyang, Abhishek Lekshmanan)
+* rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR (`issue#39984 <http://tracker.ceph.com/issues/39984>`_, `issue#40149 <http://tracker.ceph.com/issues/40149>`_, `pr#28408 <https://github.com/ceph/ceph/pull/28408>`_, Tianshan Qu, Casey Bodley)
+* rgw: fix cls_bucket_list_unordered() partial results (`pr#30254 <https://github.com/ceph/ceph/pull/30254>`_, Mark Kogan)
+* rgw: fix drain handles error when deleting bucket with bypass-gc option (`pr#30198 <https://github.com/ceph/ceph/pull/30198>`_, dongdong tao)
+* rgw: fix issue for CreateBucket with BucketLocation param (`pr#29826 <https://github.com/ceph/ceph/pull/29826>`_, Enming Zhang, Matt Benjamin)
+* rgw: fix read not exists null version return wrong (`issue#38811 <http://tracker.ceph.com/issues/38811>`_, `issue#38908 <http://tracker.ceph.com/issues/38908>`_, `pr#27330 <https://github.com/ceph/ceph/pull/27330>`_, Tianshan Qu)
+* rgw: fix refcount tags to match and update object's idtag (`pr#30323 <https://github.com/ceph/ceph/pull/30323>`_, J. Eric Ivancich)
+* rgw: gc use aio (`issue#24592 <http://tracker.ceph.com/issues/24592>`_, `pr#28784 <https://github.com/ceph/ceph/pull/28784>`_, Yehuda Sadeh, Zhang Shaowen, Yao Zongyou, Jesse Williamson)
+* rgw: get or set realm zonegroup zone need check user's caps (`issue#37497 <http://tracker.ceph.com/issues/37497>`_, `pr#28332 <https://github.com/ceph/ceph/pull/28332>`_, yuliyang, Casey Bodley)
+* rgw: housekeeping of reset stats operation in radosgw-admin and cls back-end (`pr#30674 <https://github.com/ceph/ceph/pull/30674>`_, J. Eric Ivancich)
+* rgw: inefficient unordered bucket listing (`issue#39409 <http://tracker.ceph.com/issues/39409>`_, `issue#39393 <http://tracker.ceph.com/issues/39393>`_, `pr#28350 <https://github.com/ceph/ceph/pull/28350>`_, Casey Bodley)
+* rgw: lc: continue past get_obj_state() failure (`pr#32194 <https://github.com/ceph/ceph/pull/32194>`_, Matt Benjamin)
+* rgw: make dns hostnames matching case insensitive (`issue#40995 <http://tracker.ceph.com/issues/40995>`_, `pr#30375 <https://github.com/ceph/ceph/pull/30375>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: mitigate bucket list with max-entries excessively high (`pr#30666 <https://github.com/ceph/ceph/pull/30666>`_, J. Eric Ivancich)
+* rgw: multisite: 'radosgw-admin bucket sync status' should call syncs_from(source.name) instead of id (`issue#40022 <http://tracker.ceph.com/issues/40022>`_, `issue#40143 <http://tracker.ceph.com/issues/40143>`_, `pr#29271 <https://github.com/ceph/ceph/pull/29271>`_, Casey Bodley)
+* rgw: orphans find perf improvments (`issue#39180 <http://tracker.ceph.com/issues/39180>`_, `pr#28314 <https://github.com/ceph/ceph/pull/28314>`_, Abhishek Lekshmanan)
+* rgw: parse_copy_location defers url-decode (`issue#27217 <http://tracker.ceph.com/issues/27217>`_, `pr#31651 <https://github.com/ceph/ceph/pull/31651>`_, Casey Bodley)
+* rgw: policy fix for nonexistent objects (`issue#38638 <http://tracker.ceph.com/issues/38638>`_, `pr#29153 <https://github.com/ceph/ceph/pull/29153>`_, Pritha Srivastava)
+* rgw: remove_olh_pending_entries() does not limit the number of xattrs to remove (`issue#39118 <http://tracker.ceph.com/issues/39118>`_, `issue#39177 <http://tracker.ceph.com/issues/39177>`_, `pr#28349 <https://github.com/ceph/ceph/pull/28349>`_, Casey Bodley)
+* rgw: resolve bugs and clean up garbage collection code (`issue#38454 <http://tracker.ceph.com/issues/38454>`_, `pr#31664 <https://github.com/ceph/ceph/pull/31664>`_, Dan Hill, J. Eric Ivancich)
+* rgw: return ERR_NO_SUCH_BUCKET early while evaluating bucket policy (`issue#38420 <http://tracker.ceph.com/issues/38420>`_, `pr#31218 <https://github.com/ceph/ceph/pull/31218>`_, Abhishek Lekshmanan)
+* rgw: rgw-admin: fix data sync report for master zone (`issue#38958 <http://tracker.ceph.com/issues/38958>`_, `pr#27453 <https://github.com/ceph/ceph/pull/27453>`_, cfanz)
+* rgw: rgw-admin: object stat command output's delete_at not readable (`issue#39497 <http://tracker.ceph.com/issues/39497>`_, `pr#27991 <https://github.com/ceph/ceph/pull/27991>`_, Abhishek Lekshmanan)
+* rgw: rgw/OutputDataSocket: actually discard data on full buffer (`issue#40178 <http://tracker.ceph.com/issues/40178>`_, `pr#31654 <https://github.com/ceph/ceph/pull/31654>`_, Matt Benjamin)
+* rgw: rgw/multisite: Don't allow certain radosgw-admin commands to run on non-master zone (`issue#39548 <http://tracker.ceph.com/issues/39548>`_, `pr#30946 <https://github.com/ceph/ceph/pull/30946>`_, Danny Al-Gaaf, Shilpa Jagannath)
+* rgw: rgw_file: save etag and acl info in setattr (`issue#39227 <http://tracker.ceph.com/issues/39227>`_, `pr#27881 <https://github.com/ceph/ceph/pull/27881>`_, Tao Chen)
+* rgw: rgw_sync: drop ENOENT error logs from mdlog (`issue#40032 <http://tracker.ceph.com/issues/40032>`_, `issue#38748 <http://tracker.ceph.com/issues/38748>`_, `pr#27110 <https://github.com/ceph/ceph/pull/27110>`_, Abhishek Lekshmanan)
+* rgw: set null version object acl issues (`issue#36763 <http://tracker.ceph.com/issues/36763>`_, `pr#31653 <https://github.com/ceph/ceph/pull/31653>`_, Tianshan Qu)
+* rgw: the Multi-Object Delete operation of S3 API wrongly handles the Code response element (`issue#18241 <http://tracker.ceph.com/issues/18241>`_, `issue#40135 <http://tracker.ceph.com/issues/40135>`_, `pr#29269 <https://github.com/ceph/ceph/pull/29269>`_, Radoslaw Zarzynski)
+* rgw: unable to cancel reshard operations for buckets with tenants (`issue#39016 <http://tracker.ceph.com/issues/39016>`_, `pr#27992 <https://github.com/ceph/ceph/pull/27992>`_, Abhishek Lekshmanan)
+* rgw: update civetweb submodule to match version in mimic (`issue#24158 <http://tracker.ceph.com/issues/24158>`_, `pr#27982 <https://github.com/ceph/ceph/pull/27982>`_, Abhishek Lekshmanan)
+* rgw: update s3-test download code for s3-test tasks (`pr#32227 <https://github.com/ceph/ceph/pull/32227>`_, Ali Maredia)
+* rgw: when exclusive lock fails due existing lock, log add'l info (`issue#38397 <http://tracker.ceph.com/issues/38397>`_, `issue#38171 <http://tracker.ceph.com/issues/38171>`_, `pr#26554 <https://github.com/ceph/ceph/pull/26554>`_, J. Eric Ivancich)
+* rgw:send x-amz-version-id header when upload files (`issue#39572 <http://tracker.ceph.com/issues/39572>`_, `pr#27935 <https://github.com/ceph/ceph/pull/27935>`_, Xinying Song)
+* tools: Add clear-data-digest command to objectstore tool (`pr#29366 <https://github.com/ceph/ceph/pull/29366>`_, Li Yichao)
+* tools: platform.linux_distribution() is deprecated; stop using it (`issue#39277 <http://tracker.ceph.com/issues/39277>`_, `issue#18163 <http://tracker.ceph.com/issues/18163>`_, `pr#27557 <https://github.com/ceph/ceph/pull/27557>`_, Nathan Cutler)
+* tools: rados tools list objects in a pg (`issue#36732 <http://tracker.ceph.com/issues/36732>`_, `pr#30608 <https://github.com/ceph/ceph/pull/30608>`_, Li Wang, Vikhyat Umrao)
+
+
+v12.2.12 Luminous
+=================
+
+This is the twelfth bug fix release of the Luminous v12.2.x long term stable
+release series. We recommend that all users upgrade to this release.
+
+Notable Changes
+---------------
+
+* In 12.2.11 and earlier releases, keyring caps were not checked for validity,
+  so the caps string could be anything. As of 12.2.12, caps strings are
+  validated and providing a keyring with an invalid caps string to, e.g.,
+  `ceph auth add` will result in an error.
+
+Changelog
+---------
+
+* auth: ceph auth add does not sanity-check caps (`issue#22525 <https://tracker.ceph.com/issues/22525>`_, `pr#24906 <https://github.com/ceph/ceph/pull/24906>`_, Jing Li, Nathan Cutler, Sage Weil)
+* build/ops: Allow multi instances of "make tests" on the same machine (`issue#36737 <https://tracker.ceph.com/issues/36737>`_, `pr#26186 <https://github.com/ceph/ceph/pull/26186>`_, Kefu Chai)
+* build/ops: rpm: require ceph-base instead of ceph-common (`issue#37620 <https://tracker.ceph.com/issues/37620>`_, `pr#25810 <https://github.com/ceph/ceph/pull/25810>`_, Sébastien Han)
+* ceph-volume: add --all flag to simple activate (`pr#26656 <https://github.com/ceph/ceph/pull/26656>`_, Jan Fajerski)
+* ceph-volume: look for rotational data in lsblk (`pr#26989 <https://github.com/ceph/ceph/pull/26989>`_, Andrew Schoen)
+* ceph-volume: replace testinfra command with py.test (`pr#26824 <https://github.com/ceph/ceph/pull/26824>`_, Alfredo Deza)
+* ceph-volume: revert partition as disk (`issue#37506 <https://tracker.ceph.com/issues/37506>`_, `pr#26295 <https://github.com/ceph/ceph/pull/26295>`_, Jan Fajerski)
+* ceph-volume: `simple scan` will now scan all running ceph-disk OSDs (`pr#26857 <https://github.com/ceph/ceph/pull/26857>`_, Andrew Schoen)
+* ceph-volume: use our own testinfra suite for functional testing (`pr#26703 <https://github.com/ceph/ceph/pull/26703>`_, Andrew Schoen)
+* CLI: ability to change file ownership (`issue#38370 <https://tracker.ceph.com/issues/38370>`_, `pr#26758 <https://github.com/ceph/ceph/pull/26758>`_, Sébastien Han)
+* client: session flush does not cause cap release message flush (`issue#38009 <https://tracker.ceph.com/issues/38009>`_, `pr#26271 <https://github.com/ceph/ceph/pull/26271>`_, Patrick Donnelly)
+* common: ceph_timer: stop timer's thread when it is suspended (`issue#37766 <https://tracker.ceph.com/issues/37766>`_, `pr#26579 <https://github.com/ceph/ceph/pull/26579>`_, Peng Wang)
+* common: fix for broken rbdmap parameter parsing (`issue#36327 <https://tracker.ceph.com/issues/36327>`_, `pr#26000 <https://github.com/ceph/ceph/pull/26000>`_, Marc Schoechlin)
+* core: Objecter::calc_op_budget: Fix invalid access to extent union member (`issue#37932 <https://tracker.ceph.com/issues/37932>`_, `pr#26064 <https://github.com/ceph/ceph/pull/26064>`_, Simon Ruggier)
+* core: os/filestore: ceph_abort() on fsync(2) or fdatasync(2) failure (`issue#38258 <https://tracker.ceph.com/issues/38258>`_, `pr#26871 <https://github.com/ceph/ceph/pull/26871>`_, Sage Weil)
+* crypto: don't use PK11_ImportSymKey() in FIPS mode (`issue#38843 <https://tracker.ceph.com/issues/38843>`_, `pr#27104 <https://github.com/ceph/ceph/pull/27104>`_, Radoslaw Zarzynski)
+* Fix recovery and backfill priority handling (`issue#27985 <https://tracker.ceph.com/issues/27985>`_, `issue#38041 <https://tracker.ceph.com/issues/38041>`_, `pr#26793 <https://github.com/ceph/ceph/pull/26793>`_, Sage Weil, xie xingguo, David Zafman)
+* journal: max journal order is incorrectly set at 64 (`issue#37541 <https://tracker.ceph.com/issues/37541>`_, `pr#25955 <https://github.com/ceph/ceph/pull/25955>`_, Mykola Golub)
+* librgw: export multitenancy support (`issue#37928 <https://tracker.ceph.com/issues/37928>`_, `pr#25986 <https://github.com/ceph/ceph/pull/25986>`_, Tao Chen)
+* mds: broadcast quota message to client when disable quota (`issue#38054 <https://tracker.ceph.com/issues/38054>`_, `pr#26293 <https://github.com/ceph/ceph/pull/26293>`_, Junhui Tang)
+* mds: fix potential re-evaluate stray dentry in _unlink_local_finish (`issue#38263 <https://tracker.ceph.com/issues/38263>`_, `pr#26473 <https://github.com/ceph/ceph/pull/26473>`_, Zhi Zhang)
+* mds: handle fragment notify race (`issue#36035 <https://tracker.ceph.com/issues/36035>`_, `pr#25990 <https://github.com/ceph/ceph/pull/25990>`_, "Yan, Zheng")
+* mds: handle state change race (`issue#37594 <https://tracker.ceph.com/issues/37594>`_, `pr#26005 <https://github.com/ceph/ceph/pull/26005>`_, "Yan, Zheng")
+* mds: log evicted clients to clog/dbg (`issue#37639 <https://tracker.ceph.com/issues/37639>`_, `pr#25858 <https://github.com/ceph/ceph/pull/25858>`_, Patrick Donnelly)
+* mds: log new client sessions with various metadata (`issue#37678 <https://tracker.ceph.com/issues/37678>`_, `pr#26257 <https://github.com/ceph/ceph/pull/26257>`_, Patrick Donnelly)
+* mds: message invalid access (`issue#38488 <https://tracker.ceph.com/issues/38488>`_, `pr#26661 <https://github.com/ceph/ceph/pull/26661>`_, Patrick Donnelly)
+* MDSMonitor: do not assign standby-replay when degraded (`issue#36384 <https://tracker.ceph.com/issues/36384>`_, `pr#26642 <https://github.com/ceph/ceph/pull/26642>`_, Patrick Donnelly)
+* MDSMonitor: missing osdmon writeable check (`issue#37929 <https://tracker.ceph.com/issues/37929>`_, `pr#26065 <https://github.com/ceph/ceph/pull/26065>`_, Patrick Donnelly)
+* mds: optimize revoking stale caps (`issue#38043 <https://tracker.ceph.com/issues/38043>`_, `pr#26278 <https://github.com/ceph/ceph/pull/26278>`_, "Yan, Zheng")
+* mds: stopping MDS with a large cache (40+GB) causes it to miss heartbeats (`issue#37723 <https://tracker.ceph.com/issues/37723>`_, `issue#38022 <https://tracker.ceph.com/issues/38022>`_, `pr#26232 <https://github.com/ceph/ceph/pull/26232>`_, Patrick Donnelly)
+* mds: trim cache after journal flush (`issue#38010 <https://tracker.ceph.com/issues/38010>`_, `pr#26215 <https://github.com/ceph/ceph/pull/26215>`_, Patrick Donnelly)
+* mds: wait for client to release shared cap when re-acquiring xlock (`issue#38491 <https://tracker.ceph.com/issues/38491>`_, `pr#27024 <https://github.com/ceph/ceph/pull/27024>`_, "Yan, Zheng")
+* mds: wait shorter intervals if beacon not sent (`issue#36367 <https://tracker.ceph.com/issues/36367>`_, `pr#25979 <https://github.com/ceph/ceph/pull/25979>`_, Patrick Donnelly)
+* mgr: "balancer execute" only requires read permissions (`issue#25345 <https://tracker.ceph.com/issues/25345>`_, `pr#25768 <https://github.com/ceph/ceph/pull/25768>`_, John Spray)
+* mgr/balancer: restrict automatic balancing to specific weekdays (`pr#26501 <https://github.com/ceph/ceph/pull/26501>`_, xie xingguo)
+* mgr/BaseMgrModule: drop GIL for ceph_send_command (`issue#38537 <https://tracker.ceph.com/issues/38537>`_, `pr#26830 <https://github.com/ceph/ceph/pull/26830>`_, Sage Weil)
+* mgr/DaemonServer: log pgmap usage to cluster log (`issue#37886 <https://tracker.ceph.com/issues/37886>`_, `pr#26207 <https://github.com/ceph/ceph/pull/26207>`_, Neha Ojha)
+* mgr/dashboard: fix for using '::' on hosts without ipv6 (`issue#38575 <https://tracker.ceph.com/issues/38575>`_, `pr#26751 <https://github.com/ceph/ceph/pull/26751>`_, Noah Watkins)
+* mgr: deadlock: _check_auth_rotating possible clock skew, rotating keys expired way too early (`issue#23460 <https://tracker.ceph.com/issues/23460>`_, `pr#26427 <https://github.com/ceph/ceph/pull/26427>`_, Yan Jun)
+* mgr: drop GIL in StandbyPyModule::get_config (`issue#35985 <http://tracker.ceph.com/issues/35985>`_, `pr#26613 <https://github.com/ceph/ceph/pull/26613>`_, John Spray; `pr#27639 <https://github.com/ceph/ceph/pull/27639>`_, wumingqiao)
+* mgr/restful: fix py got exception when get osd info (`issue#38182 <https://tracker.ceph.com/issues/38182>`_, `pr#26199 <https://github.com/ceph/ceph/pull/26199>`_, Boris Ranto, zouaiguo)
+* mon: A PG with PG_STATE_REPAIR doesn't mean damaged data, PG_STATE_IN… (`issue#38070 <https://tracker.ceph.com/issues/38070>`_, `pr#26305 <https://github.com/ceph/ceph/pull/26305>`_, David Zafman)
+* mon/MgrStatMonitor: ensure only one copy of initial service map (`issue#38839 <https://tracker.ceph.com/issues/38839>`_, `pr#27207 <https://github.com/ceph/ceph/pull/27207>`_, Sage Weil)
+* mon: monstore tool rebuild does not generate creating_pgs (`issue#36306 <https://tracker.ceph.com/issues/36306>`_, `pr#25825 <https://github.com/ceph/ceph/pull/25825>`_, Sage Weil)
+* mon: scrub warning check incorrectly uses mon scrub interval (`issue#37264 <https://tracker.ceph.com/issues/37264>`_, `pr#26557 <https://github.com/ceph/ceph/pull/26557>`_, Zhi Zhang, Sage Weil, David Zafman)
+* msg/async: backport recent messenger fixes (`issue#36497 <https://tracker.ceph.com/issues/36497>`_, `issue#37778 <https://tracker.ceph.com/issues/37778>`_, `pr#25956 <https://github.com/ceph/ceph/pull/25956>`_, xie xingguo)
+* msg/msg_types: fix the dencoder of entity_addr_t (`issue#24676 <https://tracker.ceph.com/issues/24676>`_, `pr#26042 <https://github.com/ceph/ceph/pull/26042>`_, Kefu Chai)
+* msgr: should set EPOLLET flag on del_event() (`issue#38828 <https://tracker.ceph.com/issues/38828>`_, `pr#27226 <https://github.com/ceph/ceph/pull/27226>`_, Roman Penyaev)
+* msg: should set EPOLLET flag on del_event() (`issue#38857 <https://tracker.ceph.com/issues/38857>`_, `pr#27226 <https://github.com/ceph/ceph/pull/27226>`_, Roman Penyaev)
+* multisite: es sync null versioned object failed because of olh info (`issue#23842 <https://tracker.ceph.com/issues/23842>`_, `issue#23841 <https://tracker.ceph.com/issues/23841>`_, `pr#26358 <https://github.com/ceph/ceph/pull/26358>`_, Tianshan Qu, Shang Ding)
+* Object can still be deleted even if s3:DeleteObject policy is set (`issue#37403 <https://tracker.ceph.com/issues/37403>`_, `pr#26310 <https://github.com/ceph/ceph/pull/26310>`_, Enming.Zhang)
+* objecter: avoid race when reset down osd's session (`issue#24601 <https://tracker.ceph.com/issues/24601>`_, `pr#25853 <https://github.com/ceph/ceph/pull/25853>`_, Zengran Zhang)
+* os/bluestore: backport new bitmap allocator (`issue#24598 <https://tracker.ceph.com/issues/24598>`_, `pr#26979 <https://github.com/ceph/ceph/pull/26979>`_, Radoslaw Zarzynski, Jianpeng Ma, Igor Fedotov, Sage Weil)
+* os/bluestore: bitmap allocator might fail to return contiguous chunk despite having enough space (`issue#38761 <https://tracker.ceph.com/issues/38761>`_, `pr#27312 <https://github.com/ceph/ceph/pull/27312>`_, Igor Fedotov)
+* os/bluestore: do not assert on non-zero err codes from compress() call (`issue#37839 <https://tracker.ceph.com/issues/37839>`_, `pr#26544 <https://github.com/ceph/ceph/pull/26544>`_, Igor Fedotov)
+* os/bluestore: fix lack of onode ref during removal (`issue#38395 <https://tracker.ceph.com/issues/38395>`_, `pr#26540 <https://github.com/ceph/ceph/pull/26540>`_, Sage Weil)
+* os/bluestore: Fix problem with bluefs's freespace not being balanced when kv_sync_thread is sleeping (`issue#38574 <https://tracker.ceph.com/issues/38574>`_, `pr#26866 <https://github.com/ceph/ceph/pull/26866>`_, Adam Kupczyk)
+* os/bluestore: fixup access a destroy cond cause deadlock or undefine (`issue#37733 <https://tracker.ceph.com/issues/37733>`_, `pr#26261 <https://github.com/ceph/ceph/pull/26261>`_, linbing)
+* os/bluestore: KernelDevice::read() does the EIO mapping now (`issue#36455 <https://tracker.ceph.com/issues/36455>`_, `pr#25855 <https://github.com/ceph/ceph/pull/25855>`_, Radoslaw Zarzynski)
+* osd: backport recent upmap fixes (`issue#37968 <https://tracker.ceph.com/issues/37968>`_, `issue#37940 <https://tracker.ceph.com/issues/37940>`_, `issue#37881 <https://tracker.ceph.com/issues/37881>`_, `pr#26127 <https://github.com/ceph/ceph/pull/26127>`_, huangjun, xie xingguo)
+* osd: backport recent upmap fixes (`issue#38826 <https://tracker.ceph.com/issues/38826>`_, `issue#38897 <https://tracker.ceph.com/issues/38897>`_, `pr#27224 <https://github.com/ceph/ceph/pull/27224>`_, huangjun, xie xingguo)
+* osd/bluestore: deep fsck fails on inspecting very large onodes (`issue#38065 <https://tracker.ceph.com/issues/38065>`_, `pr#26387 <https://github.com/ceph/ceph/pull/26387>`_, Igor Fedotov)
+* OSD crashes in get_str_map while creating with ceph-volume (`issue#38329 <https://tracker.ceph.com/issues/38329>`_, `pr#26900 <https://github.com/ceph/ceph/pull/26900>`_, Sage Weil)
+* osd: keep using cache even if op will invalid cache (`issue#37593 <https://tracker.ceph.com/issues/37593>`_, `pr#26078 <https://github.com/ceph/ceph/pull/26078>`_, Zengran Zhang)
+* osd/PG.cc: account for missing set irrespective of last_complete (`issue#37919 <https://tracker.ceph.com/issues/37919>`_, `pr#26236 <https://github.com/ceph/ceph/pull/26236>`_, Neha Ojha)
+* osd/PrimaryLogPG: fix the extent length error of the sync read (`issue#37680 <https://tracker.ceph.com/issues/37680>`_, `pr#25711 <https://github.com/ceph/ceph/pull/25711>`_, Xiaofei Cui)
+* osd/PrimaryLogPG: handle object !exists in handle_watch_timeout (`issue#38432 <https://tracker.ceph.com/issues/38432>`_, `pr#26706 <https://github.com/ceph/ceph/pull/26706>`_, Sage Weil)
+* rbd-mirror: update mirror status when stopping (`issue#36659 <https://tracker.ceph.com/issues/36659>`_, `pr#25720 <https://github.com/ceph/ceph/pull/25720>`_, Jason Dillaman)
+* rgw: bucket full sync handles delete markers (`issue#38007 <https://tracker.ceph.com/issues/38007>`_, `pr#26192 <https://github.com/ceph/ceph/pull/26192>`_, Casey Bodley)
+* rgw: bucket limit check misbehaves for > max-entries buckets (usually 1000) (`issue#35973 <https://tracker.ceph.com/issues/35973>`_, `pr#26946 <https://github.com/ceph/ceph/pull/26946>`_, Matt Benjamin)
+* rgw: bug in versioning concurrent, list and get have consistency issue (`issue#38060 <https://tracker.ceph.com/issues/38060>`_, `pr#26548 <https://github.com/ceph/ceph/pull/26548>`_, Wang Hao)
+* rgw: check for non-existent bucket in RGWGetACLs (`issue#38116 <https://tracker.ceph.com/issues/38116>`_, `pr#26530 <https://github.com/ceph/ceph/pull/26530>`_, Matt Benjamin)
+* rgw: data sync drains lease stack on lease failure (`issue#38479 <https://tracker.ceph.com/issues/38479>`_, `pr#26761 <https://github.com/ceph/ceph/pull/26761>`_, Casey Bodley)
+* rgw: fails to start on Fedora 28 from default configuration (`issue#24228 <https://tracker.ceph.com/issues/24228>`_, `pr#26131 <https://github.com/ceph/ceph/pull/26131>`_, Matt Benjamin)
+* rgw: feature -- log successful bucket resharding events (`issue#37647 <https://tracker.ceph.com/issues/37647>`_, `pr#25738 <https://github.com/ceph/ceph/pull/25738>`_, J. Eric Ivancich)
+* rgw: fetch_remote_obj filters out olh attrs (`issue#37792 <https://tracker.ceph.com/issues/37792>`_, `pr#26191 <https://github.com/ceph/ceph/pull/26191>`_, Casey Bodley)
+* rgw: fix cls_bucket_head result order consistency (`issue#38410 <https://tracker.ceph.com/issues/38410>`_, `pr#26546 <https://github.com/ceph/ceph/pull/26546>`_, Tianshan Qu)
+* rgw: fix radosgw linkage with WITH_RADOSGW_BEAST_FRONTEND=OFF (`issue#23680 <https://tracker.ceph.com/issues/23680>`_, `pr#26332 <https://github.com/ceph/ceph/pull/26332>`_, Nathan Cutler)
+* rgw: fix rgw_data_sync_info::json_decode() (`issue#38373 <https://tracker.ceph.com/issues/38373>`_, `pr#26549 <https://github.com/ceph/ceph/pull/26549>`_, Casey Bodley)
+* rgw: handle S3 version 2 pre-signed urls with meta-data (`issue#23470 <https://tracker.ceph.com/issues/23470>`_, `pr#25901 <https://github.com/ceph/ceph/pull/25901>`_, Matt Benjamin)
+* rgw: ldap: fix LDAPAuthEngine::init() when uri !empty() (`issue#38699 <https://tracker.ceph.com/issues/38699>`_, `pr#27173 <https://github.com/ceph/ceph/pull/27173>`_, Matt Benjamin)
+* rgw: multiple es related fixes and improvements (`issue#22877 <https://tracker.ceph.com/issues/22877>`_, `issue#23655 <https://tracker.ceph.com/issues/23655>`_, `issue#38030 <https://tracker.ceph.com/issues/38030>`_, `issue#38028 <https://tracker.ceph.com/issues/38028>`_, `issue#36092 <https://tracker.ceph.com/issues/36092>`_, `pr#26516 <https://github.com/ceph/ceph/pull/26516>`_, Yehuda Sadeh, Abhishek Lekshmanan)
+* rgw multisite: data sync checks empty next_marker for datalog (`issue#39033 <https://tracker.ceph.com/issues/39033>`_, `pr#27299 <https://github.com/ceph/ceph/pull/27299>`_, Casey Bodley)
+* rgw: nfs: skip empty (non-POSIX) path segments (`issue#38744 <https://tracker.ceph.com/issues/38744>`_, `pr#27180 <https://github.com/ceph/ceph/pull/27180>`_, Matt Benjamin)
+* rgw: only update last_trim marker on ENODATA (`issue#38075 <https://tracker.ceph.com/issues/38075>`_, `pr#26619 <https://github.com/ceph/ceph/pull/26619>`_, Casey Bodley)
+* rgw: "radosgw-admin bucket rm ... --purge-objects" can hang (`issue#38007 <https://tracker.ceph.com/issues/38007>`_, `issue#38134 <https://tracker.ceph.com/issues/38134>`_, `pr#26263 <https://github.com/ceph/ceph/pull/26263>`_, J. Eric Ivancich)
+* rgw: rgw_file: only first subuser can be exported to nfs (`issue#37855 <https://tracker.ceph.com/issues/37855>`_, `pr#26677 <https://github.com/ceph/ceph/pull/26677>`_, MinSheng Lin)
+* rgw: rgwgc: process coredump in some special case (`issue#23199 <https://tracker.ceph.com/issues/23199>`_, `pr#25611 <https://github.com/ceph/ceph/pull/25611>`_, zhaokun)
+* rgw: sse-c-fixes (`issue#38700 <https://tracker.ceph.com/issues/38700>`_, `pr#27295 <https://github.com/ceph/ceph/pull/27295>`_, Adam Kupczyk, Casey Bodley, Abhishek Lekshmanan)
+* rgw: sync module: avoid printing attrs of objects in log (`issue#37646 <https://tracker.ceph.com/issues/37646>`_, `pr#27030 <https://github.com/ceph/ceph/pull/27030>`_, Abhishek Lekshmanan)
+* tools: ceph-objectstore-tool: Dump hashinfo (`issue#37597 <https://tracker.ceph.com/issues/37597>`_, `pr#25722 <https://github.com/ceph/ceph/pull/25722>`_, David Zafman)
+
+
+v12.2.11 Luminous
+=================
+
+This is the eleventh bug fix release of the Luminous v12.2.x long term stable
+release series. We recommend that all users upgrade to this release. Please note
+the following precautions while upgrading.
+
+Notable Changes
+---------------
+
+* This release fixes the pg log hard limit bug that was introduced in
+  12.2.9, https://tracker.ceph.com/issues/36686.  A flag called
+  `pglog_hardlimit` has been introduced, which is off by default. Enabling
+  this flag will limit the length of the pg log.  In order to enable
+  that, the flag must be set by running `ceph osd set pglog_hardlimit`
+  after completely upgrading to 12.2.11. Once the cluster has this flag
+  set, the length of the pg log will be capped by a hard limit. Once set,
+  this flag *must not* be unset anymore.
+  
+* There have been fixes to RGW dynamic and manual resharding, which no longer
+  leaves behind stale bucket instances to be removed manually. For finding and
+  cleaning up older instances from a reshard a radosgw-admin command `reshard
+  stale-instances list` and `reshard stale-instances rm` should do the necessary
+  cleanup.
+
+* `cephfs-journal-tool` makes rank argument (--rank) mandatory. Rank is
+  of format `filesystem:rank`, where `filesystem` is the CephFS filesystem
+  and `rank` is the MDS rank on which the operation is to be executed. To
+  operate on all ranks, use `all` or `*` as the rank specifier. Note that,
+  operations that dump journal information to file will now dump to per-rank
+  suffixed dump files. Importing journal information from dump files is
+  disallowed if operation is targetted for all ranks.
+
+* CVE-2018-14662: mon: limit caps allowed to access the config store
+
+* CVE-2018-16846: rgw: enforce bounds on max-keys/max-uploads/max-parts (`issue#35994 <http://tracker.ceph.com/issues/35994>`)
+
+* CVE-2018-16889: rgw: sanitize customer encryption keys from log output in v4 auth (`issue#37847 <http://tracker.ceph.com/issues/37847>`)
+
+Changelog
+---------
+
+* build/ops: cmake: link unittest_compression against gtest (`pr#24921 <https://github.com/ceph/ceph/pull/24921>`_, Willem Jan Withagen)
+* build/ops: run-make-check.sh ccache tweaks (`issue#24826 <http://tracker.ceph.com/issues/24826>`_, `issue#24817 <http://tracker.ceph.com/issues/24817>`_, `issue#24777 <http://tracker.ceph.com/issues/24777>`_, `pr#23902 <https://github.com/ceph/ceph/pull/23902>`_, Nathan Cutler, Erwan Velu)
+* ceph-bluestore-tool: fix set label functionality for specific keys (`pr#25187 <https://github.com/ceph/ceph/pull/25187>`_, Igor Fedotov)
+* ceph-create-keys: fix octal notation for Python 3 without losing compatibility with Python 2 (`issue#37643 <http://tracker.ceph.com/issues/37643>`_, `pr#25532 <https://github.com/ceph/ceph/pull/25532>`_, James Page)
+* cephfs: ceph-volume-client: allow setting mode of CephFS volumes (`pr#25407 <https://github.com/ceph/ceph/pull/25407>`_, Tom Barron)
+* cephfs-journal-tool: make --rank argument mandatory (`pr#24728 <https://github.com/ceph/ceph/pull/24728>`_, Venky Shankar)
+* cephfs: mgr/status: fix fs status subcommand did not show standby-replay MDS' perf info (`issue#36575 <http://tracker.ceph.com/issues/36575>`_, `issue#36399 <http://tracker.ceph.com/issues/36399>`_, `pr#25032 <https://github.com/ceph/ceph/pull/25032>`_, Zhi Zhang)
+* cephfs: race of updating wanted caps (`issue#37635 <http://tracker.ceph.com/issues/37635>`_, `issue#37464 <http://tracker.ceph.com/issues/37464>`_, `pr#25762 <https://github.com/ceph/ceph/pull/25762>`_, "Yan, Zheng")
+* ceph-volume: Adapt code to support Python3 (`pr#26030 <https://github.com/ceph/ceph/pull/26030>`_, Volker Theile)
+* ceph-volume add device_id to inventory listing (`pr#25350 <https://github.com/ceph/ceph/pull/25350>`_, Jan Fajerski)
+* ceph-volume: enable device discards (`issue#36532 <http://tracker.ceph.com/issues/36532>`_, `pr#25748 <https://github.com/ceph/ceph/pull/25748>`_, Jonas Jelten)
+* ceph-volume: fix Batch object in py3 environments (`pr#25552 <https://github.com/ceph/ceph/pull/25552>`_, Jan Fajerski)
+* ceph-volume: fix JSON output in `inventory` (`issue#37390 <http://tracker.ceph.com/issues/37390>`_, `pr#25922 <https://github.com/ceph/ceph/pull/25922>`_, Sebastian Wagner)
+* ceph-volume: Fix TypeError: join() takes exactly one argument (2 given) (`issue#37595 <http://tracker.ceph.com/issues/37595>`_, `pr#25772 <https://github.com/ceph/ceph/pull/25772>`_, Sebastian Wagner)
+* ceph-volume fix TypeError on dmcrypt when using Python3 (`pr#26114 <https://github.com/ceph/ceph/pull/26114>`_, Alfredo Deza)
+* ceph-volume: introduce class hierachy for strategies (`pr#25553 <https://github.com/ceph/ceph/pull/25553>`_, Jan Fajerski, Alfredo Deza)
+* ceph-volume: mark a device not available if it belongs to ceph-disk (`pr#26117 <https://github.com/ceph/ceph/pull/26117>`_, Andrew Schoen)
+* ceph-volume normalize comma to dot for string to int conversions (`issue#37442 <http://tracker.ceph.com/issues/37442>`_, `pr#25776 <https://github.com/ceph/ceph/pull/25776>`_, Alfredo Deza)
+* ceph-volume: set permissions right before prime-osd-dir (`issue#37486 <http://tracker.ceph.com/issues/37486>`_, `pr#25778 <https://github.com/ceph/ceph/pull/25778>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume tests/functional declare ceph-ansible roles instead of importing them (`issue#37805 <http://tracker.ceph.com/issues/37805>`_, `pr#25838 <https://github.com/ceph/ceph/pull/25838>`_, Alfredo Deza)
+* ceph-volume zap devices associated with an OSD ID and/or OSD FSID (`pr#26014 <https://github.com/ceph/ceph/pull/26014>`_, Alfredo Deza)
+* ceph-volume: zap: improve zapping to remove all partitions and all LVs, encrypted or not (`issue#37449 <http://tracker.ceph.com/issues/37449>`_, `pr#25352 <https://github.com/ceph/ceph/pull/25352>`_, Alfredo Deza)
+* cli: dump osd-fsid as part of osd find <id> (`issue#37966 <http://tracker.ceph.com/issues/37966>`_, `pr#26036 <https://github.com/ceph/ceph/pull/26036>`_, Noah Watkins)
+* client: do not move f->pos untill success write (`issue#37631 <http://tracker.ceph.com/issues/37631>`_, `pr#25684 <https://github.com/ceph/ceph/pull/25684>`_, Junhui Tang)
+* client: explicitly show blacklisted state via asok status command (`issue#36456 <http://tracker.ceph.com/issues/36456>`_, `issue#36352 <http://tracker.ceph.com/issues/36352>`_, `pr#24994 <https://github.com/ceph/ceph/pull/24994>`_, Jonathan Brielmaier, Zhi Zhang)
+* client: fix fuse client hang because its pipe to mds is not ok4 (`issue#37829 <http://tracker.ceph.com/issues/37829>`_, `issue#36079 <http://tracker.ceph.com/issues/36079>`_, `pr#25904 <https://github.com/ceph/ceph/pull/25904>`_, Guan yunfei)
+* client: request next osdmap for blacklisted client (`issue#36668 <http://tracker.ceph.com/issues/36668>`_, `issue#36691 <http://tracker.ceph.com/issues/36691>`_, `pr#24986 <https://github.com/ceph/ceph/pull/24986>`_, Zhi Zhang)
+* common: auth/AuthSessionHandler: no handler if no session key (`issue#37427 <http://tracker.ceph.com/issues/37427>`_, `issue#36443 <http://tracker.ceph.com/issues/36443>`_, `pr#25297 <https://github.com/ceph/ceph/pull/25297>`_, Sage Weil)
+* common/blkdev, ceph-volume: improve get_device_id (`pr#25752 <https://github.com/ceph/ceph/pull/25752>`_, Sage Weil)
+* common: fix memory leaks in WeightedPriorityQueue (`issue#37429 <http://tracker.ceph.com/issues/37429>`_, `issue#36248 <http://tracker.ceph.com/issues/36248>`_, `pr#25296 <https://github.com/ceph/ceph/pull/25296>`_, Radoslaw Zarzynski)
+* common: (mon) command sanitization accepts floats when Int type is defined resulting in exception fault in ceph-mon (`issue#26919 <http://tracker.ceph.com/issues/26919>`_, `pr#24374 <https://github.com/ceph/ceph/pull/24374>`_, Sage Weil)
+* common: shut up some warnings (`pr#24648 <https://github.com/ceph/ceph/pull/24648>`_, Kefu Chai)
+* config: drop config::lock when invoking config observer (`issue#37762 <http://tracker.ceph.com/issues/37762>`_, `pr#25833 <https://github.com/ceph/ceph/pull/25833>`_, Kefu Chai, Venky Shankar)
+* core: bluestore: rename does not old ref to replacement onode at old name (`issue#36541 <http://tracker.ceph.com/issues/36541>`_, `issue#36638 <http://tracker.ceph.com/issues/36638>`_, `pr#24989 <https://github.com/ceph/ceph/pull/24989>`_, Jonathan Brielmaier, Sage Weil)
+* core: enable the pg deletion process to be throttled (`issue#36321 <http://tracker.ceph.com/issues/36321>`_, `pr#24501 <https://github.com/ceph/ceph/pull/24501>`_, David Zafman)
+* core: mgr crash on scrub of unconnected osd (`issue#36110 <http://tracker.ceph.com/issues/36110>`_, `issue#36464 <http://tracker.ceph.com/issues/36464>`_, `pr#25030 <https://github.com/ceph/ceph/pull/25030>`_, Sage Weil)
+* core: mon osdmap cash too small during upgrade to mimic (`issue#36506 <http://tracker.ceph.com/issues/36506>`_, `pr#25021 <https://github.com/ceph/ceph/pull/25021>`_, Sage Weil)
+* core: Objecter: add ignore cache flag if got redirect reply (`issue#36657 <http://tracker.ceph.com/issues/36657>`_, `pr#25074 <https://github.com/ceph/ceph/pull/25074>`_, Iain Buclaw, Jonathan Brielmaier)
+* core: os/bluestore_tool: fix bluefs expand (`pr#25384 <https://github.com/ceph/ceph/pull/25384>`_, Igor Fedotov)
+* core: rados rm --force-full is blocked when cluster is in full status (`issue#36436 <http://tracker.ceph.com/issues/36436>`_, `pr#25018 <https://github.com/ceph/ceph/pull/25018>`_, Yang Honggang)
+* crushtool: add --reclassify operation to convert legacy crush maps to use device classes (`pr#25307 <https://github.com/ceph/ceph/pull/25307>`_, Sage Weil)
+* debian: correct ceph-common relationship with older radosgw package (`pr#24997 <https://github.com/ceph/ceph/pull/24997>`_, Matthew Vernon)
+* doc: broken link on troubleshooting-mon page (`pr#25500 <https://github.com/ceph/ceph/pull/25500>`_, James McClune)
+* doc: fix broken fstab url in cephfs/fuse (`issue#36286 <http://tracker.ceph.com/issues/36286>`_, `pr#24434 <https://github.com/ceph/ceph/pull/24434>`_, Jos Collin)
+* doc: Fix typo error on cephfs/fuse/ (`issue#36180 <http://tracker.ceph.com/issues/36180>`_, `issue#36309 <http://tracker.ceph.com/issues/36309>`_, `pr#24752 <https://github.com/ceph/ceph/pull/24752>`_, Karun Josy)
+* doc: Put command template into literal block (`pr#25001 <https://github.com/ceph/ceph/pull/25001>`_, Alexey Stupnikov)
+* doc/rados: update bluestore provisioning and autotuning docs (`issue#37341 <http://tracker.ceph.com/issues/37341>`_, `pr#25284 <https://github.com/ceph/ceph/pull/25284>`_, Mark Nelson)
+* doc: show edit on github links and version warnings (`pr#25267 <https://github.com/ceph/ceph/pull/25267>`_, Neha Ojha, Noah Watkins)
+* doc/user-management: Remove obsolete reset caps command (`issue#37663 <http://tracker.ceph.com/issues/37663>`_, `pr#25609 <https://github.com/ceph/ceph/pull/25609>`_, Brad Hubbard)
+* examples: fix link order in librados example Makefile (`issue#37795 <http://tracker.ceph.com/issues/37795>`_, `pr#25829 <https://github.com/ceph/ceph/pull/25829>`_, Mahati Chamarthy)
+* extend reconnect period when mds is busy (`issue#37739 <http://tracker.ceph.com/issues/37739>`_, `pr#25784 <https://github.com/ceph/ceph/pull/25784>`_, "Yan, Zheng")
+* fsck: cid is improperly matched to oid (`issue#36145 <http://tracker.ceph.com/issues/36145>`_, `issue#32731 <http://tracker.ceph.com/issues/32731>`_, `pr#24705 <https://github.com/ceph/ceph/pull/24705>`_, Sage Weil)
+* libcephfs: expose CEPH_SETATTR_MTIME_NOW and CEPH_SETATTR_ATIME_NOW (`issue#36206 <http://tracker.ceph.com/issues/36206>`_, `issue#35961 <http://tracker.ceph.com/issues/35961>`_, `pr#24465 <https://github.com/ceph/ceph/pull/24465>`_, Zhu Shangzhong)
+* librbd: fix missing unblock_writes if shrink is not allowed (`issue#37363 <http://tracker.ceph.com/issues/37363>`_, `issue#36778 <http://tracker.ceph.com/issues/36778>`_, `pr#25253 <https://github.com/ceph/ceph/pull/25253>`_, runsisi)
+* librbd: reset snaps in rbd_snap_list() (`issue#37535 <http://tracker.ceph.com/issues/37535>`_, `issue#37508 <http://tracker.ceph.com/issues/37508>`_, `pr#25458 <https://github.com/ceph/ceph/pull/25458>`_, Kefu Chai)
+* mds: add "drop cache" command (`issue#36695 <http://tracker.ceph.com/issues/36695>`_, `issue#36281 <http://tracker.ceph.com/issues/36281>`_, `pr#24468 <https://github.com/ceph/ceph/pull/24468>`_, Rishabh Dave, Patrick Donnelly, Venky Shankar)
+* mds: clean up log messages for standby-replay (`pr#25804 <https://github.com/ceph/ceph/pull/25804>`_, Patrick Donnelly)
+* mds: create heartbeat grace config option (`issue#37674 <http://tracker.ceph.com/issues/37674>`_, `issue#37820 <http://tracker.ceph.com/issues/37820>`_, `pr#25889 <https://github.com/ceph/ceph/pull/25889>`_, Patrick Donnelly)
+* mds: directories pinned keep being replicated back and forth between exporting mds and importing mds (`issue#37368 <http://tracker.ceph.com/issues/37368>`_, `issue#37606 <http://tracker.ceph.com/issues/37606>`_, `pr#25522 <https://github.com/ceph/ceph/pull/25522>`_, Xuehan Xu)
+* mds: disallow dumping huge caches to formatter (`issue#37608 <http://tracker.ceph.com/issues/37608>`_, `pr#25567 <https://github.com/ceph/ceph/pull/25567>`_, Venky Shankar)
+* mds: do not call Journaler::_trim twice (`issue#37566 <http://tracker.ceph.com/issues/37566>`_, `issue#37629 <http://tracker.ceph.com/issues/37629>`_, `pr#25562 <https://github.com/ceph/ceph/pull/25562>`_, Tang Junhui)
+* mds: fix bug filelock stuck at LOCK_XSYN leading client can't read data (`issue#37700 <http://tracker.ceph.com/issues/37700>`_, `issue#37333 <http://tracker.ceph.com/issues/37333>`_, `pr#25677 <https://github.com/ceph/ceph/pull/25677>`_, Guan yunfei)
+* mds: fix incorrect l_pq_executing_ops statistics when meet an invalid item in purge queue (`issue#37627 <http://tracker.ceph.com/issues/37627>`_, `issue#37567 <http://tracker.ceph.com/issues/37567>`_, `pr#25560 <https://github.com/ceph/ceph/pull/25560>`_, Junhui Tang)
+* mds: fix infinite loop in OpTracker::check_ops_in_flight (`issue#37977 <http://tracker.ceph.com/issues/37977>`_, `pr#26048 <https://github.com/ceph/ceph/pull/26048>`_, "Yan, Zheng")
+* mds: fix infinite loop in OpTracker::check_ops_in_flight (`issue#37977 <http://tracker.ceph.com/issues/37977>`_, `pr#26088 <https://github.com/ceph/ceph/pull/26088>`_, "Yan, Zheng")
+* mds: fix mds damaged due to unexpected journal length (`issue#36200 <http://tracker.ceph.com/issues/36200>`_, `pr#24440 <https://github.com/ceph/ceph/pull/24440>`_, Zhi Zhang)
+* mds: migrate strays part by part when shutdown mds (`issue#26926 <http://tracker.ceph.com/issues/26926>`_, `issue#32091 <http://tracker.ceph.com/issues/32091>`_, `pr#24324 <https://github.com/ceph/ceph/pull/24324>`_, "Yan, Zheng")
+* MDSMonitor: allow beacons from stopping MDS that was laggy (`issue#37737 <http://tracker.ceph.com/issues/37737>`_, `pr#25686 <https://github.com/ceph/ceph/pull/25686>`_, Patrick Donnelly)
+* mds: obsolete MDSMap option configs (`issue#37540 <http://tracker.ceph.com/issues/37540>`_, `pr#25431 <https://github.com/ceph/ceph/pull/25431>`_, Patrick Donnelly)
+* mds: purge queue recovery hangs during boot if PQ journal is damaged (`issue#37899 <http://tracker.ceph.com/issues/37899>`_, `issue#37543 <http://tracker.ceph.com/issues/37543>`_, `pr#25968 <https://github.com/ceph/ceph/pull/25968>`_, Patrick Donnelly)
+* mds: PurgeQueue write error handler does not handle EBLACKLISTED (`issue#37604 <http://tracker.ceph.com/issues/37604>`_, `pr#25524 <https://github.com/ceph/ceph/pull/25524>`_, Patrick Donnelly)
+* mds: rctime not set on system inode (root) at startup (`issue#36221 <http://tracker.ceph.com/issues/36221>`_, `issue#36460 <http://tracker.ceph.com/issues/36460>`_, `pr#25043 <https://github.com/ceph/ceph/pull/25043>`_, Patrick Donnelly)
+* mds: remove duplicated l_mdc_num_strays perfcounter set (`issue#37633 <http://tracker.ceph.com/issues/37633>`_, `issue#37516 <http://tracker.ceph.com/issues/37516>`_, `pr#25682 <https://github.com/ceph/ceph/pull/25682>`_, Zhi Zhang)
+* mds: severe internal fragment when decoding xattr_map from log event (`issue#37399 <http://tracker.ceph.com/issues/37399>`_, `issue#37602 <http://tracker.ceph.com/issues/37602>`_, `pr#25520 <https://github.com/ceph/ceph/pull/25520>`_, "Yan, Zheng")
+* mds: "src/mds/MDLog.cc: 281: FAILED ceph_assert(!capped)" during max_mds thrashing (`issue#36350 <http://tracker.ceph.com/issues/36350>`_, `issue#37092 <http://tracker.ceph.com/issues/37092>`_, `pr#25826 <https://github.com/ceph/ceph/pull/25826>`_, "Yan, Zheng")
+* mgr/balancer: add cmd to list all plans (`issue#37420 <http://tracker.ceph.com/issues/37420>`_, `pr#25259 <https://github.com/ceph/ceph/pull/25259>`_, Yang Honggang)
+* mgr/balancer: add crush_compat_metrics param (`issue#37413 <http://tracker.ceph.com/issues/37413>`_, `pr#25257 <https://github.com/ceph/ceph/pull/25257>`_, Dan van der Ster)
+* mgr: balancer: python 3 compat fixes (`issue#37416 <http://tracker.ceph.com/issues/37416>`_, `pr#25258 <https://github.com/ceph/ceph/pull/25258>`_, Noah Watkins)
+* mgr: fix crash due to multiple sessions from daemons with same name (`pr#25867 <https://github.com/ceph/ceph/pull/25867>`_, Mykola Golub)
+* mgr: hold lock while accessing the request list and submitting request (`pr#25047 <https://github.com/ceph/ceph/pull/25047>`_, Jerry Lee)
+* mgr: Module 'influx' has failed (`issue#25201 <http://tracker.ceph.com/issues/25201>`_, `pr#25184 <https://github.com/ceph/ceph/pull/25184>`_, Nathan Cutler, Wido den Hollander)
+* mgr: prometheus: added bluestore db and wal devices to ceph_disk_occupation metric.// (`issue#37362 <http://tracker.ceph.com/issues/37362>`_, `pr#25216 <https://github.com/ceph/ceph/pull/25216>`_, Konstantin Shalygin)
+* mgr: race between daemon state and service map in 'service status' (`issue#37478 <http://tracker.ceph.com/issues/37478>`_, `issue#36656 <http://tracker.ceph.com/issues/36656>`_, `pr#25369 <https://github.com/ceph/ceph/pull/25369>`_, Mykola Golub)
+* mgr: [restful] deep_scrub is not a valid OSD command (`issue#36720 <http://tracker.ceph.com/issues/36720>`_, `issue#36750 <http://tracker.ceph.com/issues/36750>`_, `pr#25041 <https://github.com/ceph/ceph/pull/25041>`_, Boris Ranto)
+* mon: mark REMOVE_SNAPS messages as no_reply (`issue#37568 <http://tracker.ceph.com/issues/37568>`_, `issue#37694 <http://tracker.ceph.com/issues/37694>`_, `pr#25779 <https://github.com/ceph/ceph/pull/25779>`_, "Yan, Zheng")
+* mon/OSDMonitor: do not populate void pg_temp into nextmap (`issue#37811 <http://tracker.ceph.com/issues/37811>`_, `pr#25845 <https://github.com/ceph/ceph/pull/25845>`_, Aleksei Zakharov)
+* mon: shutdown messenger early to avoid accessing deleted logger (`issue#37780 <http://tracker.ceph.com/issues/37780>`_, `issue#37813 <http://tracker.ceph.com/issues/37813>`_, `pr#25847 <https://github.com/ceph/ceph/pull/25847>`_, ningtao)
+* os/bluestore: avoid frequent allocator dump on bluefs rebalance failure (`pr#24543 <https://github.com/ceph/ceph/pull/24543>`_, Igor Fedotov)
+* os/bluestore/BlueStore.cc: 1025: FAILED assert(buffer_bytes >= b->length) from ObjectStore/StoreTest.ColSplitTest2/2 (`issue#26943 <http://tracker.ceph.com/issues/26943>`_, `issue#24439 <http://tracker.ceph.com/issues/24439>`_, `pr#24992 <https://github.com/ceph/ceph/pull/24992>`_, Jonathan Brielmaier, Sage Weil)
+* os/bluestore: handle spurious read errors (`issue#22464 <http://tracker.ceph.com/issues/22464>`_, `pr#24649 <https://github.com/ceph/ceph/pull/24649>`_, Paul Emmerich)
+* osd: backport recent upmap fixes (`pr#25418 <https://github.com/ceph/ceph/pull/25418>`_, ningtao, xie xingguo)
+* osdc/Objecter: update op_target_t::paused in _calc_target (`issue#37398 <http://tracker.ceph.com/issues/37398>`_, `issue#37553 <http://tracker.ceph.com/issues/37553>`_, `pr#25719 <https://github.com/ceph/ceph/pull/25719>`_, Song Shun, runsisi)
+* osdc: reduce ObjectCacher's memory fragments (`issue#36642 <http://tracker.ceph.com/issues/36642>`_, `issue#36192 <http://tracker.ceph.com/issues/36192>`_, `pr#24872 <https://github.com/ceph/ceph/pull/24872>`_, "Yan, Zheng")
+* osd: failed assert when osd_memory_target options mismatch (`issue#37697 <http://tracker.ceph.com/issues/37697>`_, `issue#37507 <http://tracker.ceph.com/issues/37507>`_, `pr#25604 <https://github.com/ceph/ceph/pull/25604>`_, xie xingguo)
+* osd/mon: pg log hard limit with upgrades fixed (`issue#37903 <http://tracker.ceph.com/issues/37903>`_, `issue#21416 <http://tracker.ceph.com/issues/21416>`_, `pr#25949 <https://github.com/ceph/ceph/pull/25949>`_, Neha Ojha, xie xingguo)
+* osd/OSD.cc: log slow requests in OSD logs (`pr#25824 <https://github.com/ceph/ceph/pull/25824>`_, Neha Ojha)
+* osd/OSDMap: cancel mapping if target osd is out (`issue#37501 <http://tracker.ceph.com/issues/37501>`_, `pr#25698 <https://github.com/ceph/ceph/pull/25698>`_, ningtao, xie xingguo)
+* osd: potential deadlock in PG::_scan_snaps when repairing snap mapper (`issue#36630 <http://tracker.ceph.com/issues/36630>`_, `pr#24833 <https://github.com/ceph/ceph/pull/24833>`_, Mykola Golub)
+* osd: Prioritize user specified scrubs (`issue#37343 <http://tracker.ceph.com/issues/37343>`_, `issue#37269 <http://tracker.ceph.com/issues/37269>`_, `pr#25514 <https://github.com/ceph/ceph/pull/25514>`_, kungf, David Zafman)
+* osd: race condition opening heartbeat connection (`issue#36602 <http://tracker.ceph.com/issues/36602>`_, `issue#36636 <http://tracker.ceph.com/issues/36636>`_, `pr#25035 <https://github.com/ceph/ceph/pull/25035>`_, Sage Weil)
+* osd: RBD client IOPS pool stats are incorrect (2x higher; includes IO hints as an op) (`issue#24909 <http://tracker.ceph.com/issues/24909>`_, `issue#36556 <http://tracker.ceph.com/issues/36556>`_, `pr#25025 <https://github.com/ceph/ceph/pull/25025>`_, Jason Dillaman)
+* pybind/mgr/status: fix ceph fs status in py3 environments (`issue#37573 <http://tracker.ceph.com/issues/37573>`_, `issue#37625 <http://tracker.ceph.com/issues/37625>`_, `pr#25695 <https://github.com/ceph/ceph/pull/25695>`_, Jan Fajerski)
+* rbd: pybind: added missing RBD_FLAG_FAST_DIFF_INVALID constant (`issue#36407 <http://tracker.ceph.com/issues/36407>`_, `pr#25006 <https://github.com/ceph/ceph/pull/25006>`_, Jason Dillaman)
+* rbd: [rbd-mirror] periodic mirror status timer might fail to be scheduled (`issue#36500 <http://tracker.ceph.com/issues/36500>`_, `issue#36554 <http://tracker.ceph.com/issues/36554>`_, `pr#24917 <https://github.com/ceph/ceph/pull/24917>`_, Nathan Cutler, Jason Dillaman)
+* rgw: add ssl support to beast frontend (`issue#22832 <http://tracker.ceph.com/issues/22832>`_, `issue#24358 <http://tracker.ceph.com/issues/24358>`_, `pr#24621 <https://github.com/ceph/ceph/pull/24621>`_, Casey Bodley)
+* rgw: apply quota config to users created via external auth (`issue#24595 <http://tracker.ceph.com/issues/24595>`_, `issue#36222 <http://tracker.ceph.com/issues/36222>`_, `pr#24547 <https://github.com/ceph/ceph/pull/24547>`_, Casey Bodley, Matt Benjamin)
+* rgw: beast frontend fails to parse ipv6 endpoints (`issue#36733 <http://tracker.ceph.com/issues/36733>`_, `issue#36662 <http://tracker.ceph.com/issues/36662>`_, `pr#25512 <https://github.com/ceph/ceph/pull/25512>`_, Casey Bodley)
+* rgw: bucket resharding fixes (`issue#37446 <http://tracker.ceph.com/issues/37446>`_, `issue#36688 <http://tracker.ceph.com/issues/36688>`_, `pr#25326 <https://github.com/ceph/ceph/pull/25326>`_, Orit Wasserman, Abhishek Lekshmanan, J. Eric Ivancich)
+* rgw: catch exceptions from librados::NObjectIterator (`issue#37091 <http://tracker.ceph.com/issues/37091>`_, `issue#37475 <http://tracker.ceph.com/issues/37475>`_, `pr#25289 <https://github.com/ceph/ceph/pull/25289>`_, Casey Bodley)
+* rgw: Don't treat colons specially in resource part of ARN (`issue#37482 <http://tracker.ceph.com/issues/37482>`_, `issue#23817 <http://tracker.ceph.com/issues/23817>`_, `pr#25387 <https://github.com/ceph/ceph/pull/25387>`_, Adam C. Emerson)
+* rgw: es fixes for working with nfs ganesha (`issue#37349 <http://tracker.ceph.com/issues/37349>`_, `issue#36233 <http://tracker.ceph.com/issues/36233>`_, `issue#22758 <http://tracker.ceph.com/issues/22758>`_, `pr#25444 <https://github.com/ceph/ceph/pull/25444>`_, Abhishek Lekshmanan)
+* rgw_file: user info never synced since librgw init (`issue#37549 <http://tracker.ceph.com/issues/37549>`_, `pr#25484 <https://github.com/ceph/ceph/pull/25484>`_, Tao Chen)
+* rgw: fixes for zone deletion (`issue#37328 <http://tracker.ceph.com/issues/37328>`_, `issue#37466 <http://tracker.ceph.com/issues/37466>`_, `pr#25320 <https://github.com/ceph/ceph/pull/25320>`_, Abhishek Lekshmanan)
+* rgw: fix max-size in radosgw-admin and REST Admin API (`issue#37519 <http://tracker.ceph.com/issues/37519>`_, `pr#25448 <https://github.com/ceph/ceph/pull/25448>`_, Nick Erdmann)
+* rgw: fix version bucket stats (`issue#37563 <http://tracker.ceph.com/issues/37563>`_, `issue#21429 <http://tracker.ceph.com/issues/21429>`_, `pr#25644 <https://github.com/ceph/ceph/pull/25644>`_, Shasha Lu)
+* rgw: librgw: crashes in multisite configuration (`issue#36302 <http://tracker.ceph.com/issues/36302>`_, `issue#36414 <http://tracker.ceph.com/issues/36414>`_, `pr#24909 <https://github.com/ceph/ceph/pull/24909>`_, Casey Bodley)
+* rgw: multisite: sync gets stuck retrying deletes that fail with ERR_PRECONDITION_FAILED (`issue#37551 <http://tracker.ceph.com/issues/37551>`_, `issue#37448 <http://tracker.ceph.com/issues/37448>`_, `pr#25506 <https://github.com/ceph/ceph/pull/25506>`_, Casey Bodley)
+* rgw: radosgw-admin: translate reshard status codes (trivial) (`issue#37284 <http://tracker.ceph.com/issues/37284>`_, `issue#36486 <http://tracker.ceph.com/issues/36486>`_, `pr#25195 <https://github.com/ceph/ceph/pull/25195>`_, Matt Benjamin)
+* rgw: rgw-admin: reshard add can add a non-existent bucket (`issue#36449 <http://tracker.ceph.com/issues/36449>`_, `issue#36757 <http://tracker.ceph.com/issues/36757>`_, `pr#25088 <https://github.com/ceph/ceph/pull/25088>`_, Jonathan Brielmaier, Abhishek Lekshmanan)
+* rgw: SSE encryption does not detect ssl termination in proxy (`issue#36644 <http://tracker.ceph.com/issues/36644>`_, `issue#27221 <http://tracker.ceph.com/issues/27221>`_, `pr#24944 <https://github.com/ceph/ceph/pull/24944>`_, Jonathan Brielmaier, Casey Bodley)
+* rpm: Use hardened LDFLAGS (`issue#36316 <http://tracker.ceph.com/issues/36316>`_, `issue#36391 <http://tracker.ceph.com/issues/36391>`_, `pr#25173 <https://github.com/ceph/ceph/pull/25173>`_, Boris Ranto)
+
+
+v12.2.10 Luminous
+=================
+
+This is the tenth bug fix release of the Luminous v12.2.x long term stable
+release series. The previous release, v12.2.9, introduced the PG hard-limit
+patches which were found to cause an issue in certain upgrade scenarios, and
+this release was expedited to revert those patches. If you already successfully
+upgraded to v12.2.9, you should **not** upgrade to v12.2.10, but rather **wait**
+for a release in which http://tracker.ceph.com/issues/36686 is addressed. All
+other users are encouraged to upgrade to this release.
+
+Notable Changes
+---------------
+
+OSD
+
+* This release reverts the PG hard-limit patches added in v12.2.9.
+
+Changelog
+---------
+
+* ceph-volume: add some choose_disk capabilities (`issue#36446 <http://tracker.ceph.com/issues/36446>`_, `pr#24783 <https://github.com/ceph/ceph/pull/24783>`_, Erwan Velu)
+* ceph-volume: remove version reporting from help menu (`issue#36386 <http://tracker.ceph.com/issues/36386>`_, `pr#24754 <https://github.com/ceph/ceph/pull/24754>`_, Alfredo Deza)
+* ceph-volume: systemd import main so console_scripts work for executable (`issue#36648 <http://tracker.ceph.com/issues/36648>`_, `pr#24853 <https://github.com/ceph/ceph/pull/24853>`_, Alfredo Deza)
+* ceph-volume: tests install ceph-ansible's requirements.txt dependencies (`issue#36672 <http://tracker.ceph.com/issues/36672>`_, `pr#24960 <https://github.com/ceph/ceph/pull/24960>`_, Alfredo Deza)
+* ceph-volume: util.encryption don't push stderr to terminal (`issue#36246 <http://tracker.ceph.com/issues/36246>`_, `pr#24827 <https://github.com/ceph/ceph/pull/24827>`_, Alfredo Deza)
+* ceph-volume: util.encryption robust blkid+lsblk detection of lockbox (`pr#24981 <https://github.com/ceph/ceph/pull/24981>`_, Alfredo Deza)
+* ceph-volume: use console_scripts (`issue#36601 <http://tracker.ceph.com/issues/36601>`_, `pr#24837 <https://github.com/ceph/ceph/pull/24837>`_, Mehdi Abaakouk)
+* OSDMapMapping does not handle active.size() > pool size (`issue#26866 <http://tracker.ceph.com/issues/26866>`_, `issue#35935 <http://tracker.ceph.com/issues/35935>`_, `pr#24432 <https://github.com/ceph/ceph/pull/24432>`_, Sage Weil)
+* PG: add custom_reaction Backfilled and release reservations (`issue#24333 <http://tracker.ceph.com/issues/24333>`_, `pr#23493 <https://github.com/ceph/ceph/pull/23493>`_, Neha Ojha)
+* Revert "PG: add custom_reaction Backfilled and release reservations after backfill (`pr#24902 <https://github.com/ceph/ceph/pull/24902>`_, Neha Ojha)
+* Revert pg log limit changes (`issue#36686 <http://tracker.ceph.com/issues/36686>`_, `pr#24903 <https://github.com/ceph/ceph/pull/24903>`_, Neha Ojha)
+* backport and other test fixes for osd-scrub-repair.sh (`issue#35845 <http://tracker.ceph.com/issues/35845>`_, `issue#36393 <http://tracker.ceph.com/issues/36393>`_, `pr#24532 <https://github.com/ceph/ceph/pull/24532>`_, Xinying Song, David Zafman)
+* ceph-volume tests.systemd update imports for systemd module (`issue#36704 <http://tracker.ceph.com/issues/36704>`_, `pr#24958 <https://github.com/ceph/ceph/pull/24958>`_, Alfredo Deza)
+* ceph-volume: adds a --prepare flag to `lvm batch` (`issue#36363 <http://tracker.ceph.com/issues/36363>`_, `pr#24759 <https://github.com/ceph/ceph/pull/24759>`_, Andrew Schoen)
+* cls/user: cls_user_remove_bucket writes modified header (`issue#36534 <http://tracker.ceph.com/issues/36534>`_, `issue#36496 <http://tracker.ceph.com/issues/36496>`_, `pr#24855 <https://github.com/ceph/ceph/pull/24855>`_, Casey Bodley)
+* core: by pass cache if performing deep scrub (`issue#35067 <http://tracker.ceph.com/issues/35067>`_, `pr#24802 <https://github.com/ceph/ceph/pull/24802>`_, Xiaoguang Wang)
+* crush/CrushWrapper: fix crush tree json dumper (`issue#36149 <http://tracker.ceph.com/issues/36149>`_, `pr#24482 <https://github.com/ceph/ceph/pull/24482>`_, Oshyn Song)
+* ec: src/common/interval_map.h: 161: FAILED assert(len > 0) (`issue#21931 <http://tracker.ceph.com/issues/21931>`_, `issue#22330 <http://tracker.ceph.com/issues/22330>`_, `pr#24582 <https://github.com/ceph/ceph/pull/24582>`_, Neha Ojha)
+* gperftools-libs-2.6.1-1 or newer required for binaries linked against corresponding version at build time (`issue#36552 <http://tracker.ceph.com/issues/36552>`_, `issue#23657 <http://tracker.ceph.com/issues/23657>`_, `issue#36558 <http://tracker.ceph.com/issues/36558>`_, `issue#36508 <http://tracker.ceph.com/issues/36508>`_, `pr#24706 <https://github.com/ceph/ceph/pull/24706>`_, Brad Hubbard)
+* osd: add creating to pg_string_state (`issue#36174 <http://tracker.ceph.com/issues/36174>`_, `issue#36297 <http://tracker.ceph.com/issues/36297>`_, `pr#24602 <https://github.com/ceph/ceph/pull/24602>`_, Dan van der Ster)
+* osd: cast 'whoami' to unsigned so it can be used as the seed for RNG (`issue#26890 <http://tracker.ceph.com/issues/26890>`_, `pr#24659 <https://github.com/ceph/ceph/pull/24659>`_, Kefu Chai)
+* osd: get loadavg per cpu for scrub load threshold check (`pr#24593 <https://github.com/ceph/ceph/pull/24593>`_, kungf)
+* osdc/Objecter: possible race condition with connection reset (`issue#36183 <http://tracker.ceph.com/issues/36183>`_, `issue#36295 <http://tracker.ceph.com/issues/36295>`_, `pr#24574 <https://github.com/ceph/ceph/pull/24574>`_, Jason Dillaman)
+* qa: add test that builds example librados programs (`issue#36229 <http://tracker.ceph.com/issues/36229>`_, `issue#15100 <http://tracker.ceph.com/issues/15100>`_, `pr#24538 <https://github.com/ceph/ceph/pull/24538>`_, Nathan Cutler)
+* qa/ceph-ansible: Specify stable-3.2 branch (`issue#37331 <https://tracker.ceph.com/issues/37331>`_, `pr#25170 <https://github.com/ceph/ceph/pull/25170>`_, Brad Hubbard)
+* rgw/beast: drop privileges after binding ports (`issue#36041 <http://tracker.ceph.com/issues/36041>`_, `pr#24454 <https://github.com/ceph/ceph/pull/24454>`_, Paul Emmerich)
+* rgw: RGWAsyncGetBucketInstanceInfo does not access coroutine memory (`issue#35812 <http://tracker.ceph.com/issues/35812>`_, `issue#36212 <http://tracker.ceph.com/issues/36212>`_, `pr#24507 <https://github.com/ceph/ceph/pull/24507>`_, Casey Bodley)
+* rgw: fix leak of curl handle on shutdown (`issue#35715 <http://tracker.ceph.com/issues/35715>`_, `issue#36214 <http://tracker.ceph.com/issues/36214>`_, `pr#24519 <https://github.com/ceph/ceph/pull/24519>`_, Casey Bodley)
+* rgw: list bucket can not show the object uploaded by RGWPostObj when enable bucket versioning (`pr#24570 <https://github.com/ceph/ceph/pull/24570>`_, yuliyang)
+* rgw: multisite: enforce spawn_window for data full sync (`issue#26897 <http://tracker.ceph.com/issues/26897>`_, `pr#24857 <https://github.com/ceph/ceph/pull/24857>`_, Casey Bodley)
+* rgw: set default objecter_inflight_ops = 24576 (`issue#36570 <http://tracker.ceph.com/issues/36570>`_, `issue#25109 <http://tracker.ceph.com/issues/25109>`_, `pr#24862 <https://github.com/ceph/ceph/pull/24862>`_, Matt Benjamin)
+* rgw: user stats account for resharded buckets (`pr#24854 <https://github.com/ceph/ceph/pull/24854>`_, Casey Bodley)
+* segv in BlueStore::OldExtent::create (`issue#36526 <http://tracker.ceph.com/issues/36526>`_, `issue#36591 <http://tracker.ceph.com/issues/36591>`_, `pr#24746 <https://github.com/ceph/ceph/pull/24746>`_, Sage Weil)
+* test/common: unittest_mclock_priority_queue builds with "make" command (`pr#24808 <https://github.com/ceph/ceph/pull/24808>`_, J. Eric Ivancich)
+
+
+v12.2.9 Luminous
+================
+
+This is the ninth bug fix release of the Luminous v12.2.x long term stable
+release series. Although this release contains several bugfixes across all the
+components, it also introduced the PG hard-limit patches which could cause
+problems during upgrade when not all PGs were active+clean. Therefore, users
+should not install this release. Instead, they should skip it and upgrade to
+12.2.10 directly.
+
+Notable Changes
+---------------
+
+OSD
+
+* 12.2.9 contains the pg hard limit patches (https://tracker.ceph.com/issues/23979).
+  A partial upgrade during recovery/backfill, can cause the osds on the
+  previous version, to fail with assert(trim_to <= info.last_complete). The
+  workaround for users is to upgrade and restart all OSDs to a version with the
+  pg hard limit, or only upgrade when all PGs are active+clean. This patch will
+  be reverted in 12.2.10, until a clean upgrade path is added to the pg log hard
+  limit patches.
+
+  See also: http://tracker.ceph.com/issues/36686
+
+* The bluestore_cache_* options are no longer needed. They are replaced
+  by osd_memory_target, defaulting to 4GB. BlueStore will expand
+  and contract its cache to attempt to stay within this
+  limit. Users upgrading should note this is a higher default
+  than the previous bluestore_cache_size of 1GB, so OSDs using
+  BlueStore will use more memory by default.
+
+  For more details, see `BlueStore docs <http://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#cache-size>_`
+
+Changelog
+---------
+
+* build/ops: add e2fsprogs runtime dependency (`pr#24663 <https://github.com/ceph/ceph/pull/24663>`_, Guillaume Abrioux, Alfredo Deza)
+* build/ops: deb: fix ceph-mgr .pyc files left behind (`issue#26883 <http://tracker.ceph.com/issues/26883>`_, `pr#23832 <https://github.com/ceph/ceph/pull/23832>`_, Dan Mick)
+* build/ops: deb: require fuse for ceph-fuse (`issue#21057 <http://tracker.ceph.com/issues/21057>`_, `pr#23693 <https://github.com/ceph/ceph/pull/23693>`_, Thomas Serlin)
+* build/ops: rpm: selinux-policy fixes (`pr#24136 <https://github.com/ceph/ceph/pull/24136>`_, Brad Hubbard)
+* build/ops: rpm: use updated gperftools (`issue#35969 <http://tracker.ceph.com/issues/35969>`_, `pr#24259 <https://github.com/ceph/ceph/pull/24259>`_, Kefu Chai)
+* ceph-volume:  activate option --auto-detect-objectstore respects --no-systemd (`issue#36249 <http://tracker.ceph.com/issues/36249>`_, `pr#24358 <https://github.com/ceph/ceph/pull/24358>`_, Alfredo Deza)
+* ceph-volume:  lsblk can fail to find PARTLABEL, must fallback to blkid (`issue#36098 <http://tracker.ceph.com/issues/36098>`_, `pr#24335 <https://github.com/ceph/ceph/pull/24335>`_, Alfredo Deza)
+* ceph-volume: add new ceph-handlers role from ceph-ansible (`issue#36251 <http://tracker.ceph.com/issues/36251>`_, `pr#24338 <https://github.com/ceph/ceph/pull/24338>`_, Alfredo Deza)
+* ceph-volume: batch carve out lvs for bluestore (`issue#34535 <http://tracker.ceph.com/issues/34535>`_, `pr#24075 <https://github.com/ceph/ceph/pull/24075>`_, Alfredo Deza)
+* ceph-volume: batch tests for mixed-type of devices (`issue#35535 <http://tracker.ceph.com/issues/35535>`_, `issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#23967 <https://github.com/ceph/ceph/pull/23967>`_, Alfredo Deza)
+* ceph-volume: batch: allow --osds-per-device, default it to 1 (`issue#35913 <http://tracker.ceph.com/issues/35913>`_, `pr#24080 <https://github.com/ceph/ceph/pull/24080>`_, Alfredo Deza)
+* ceph-volume: batch: allow journal+block.db sizing on the CLI (`issue#36088 <http://tracker.ceph.com/issues/36088>`_, `pr#24209 <https://github.com/ceph/ceph/pull/24209>`_, Alfredo Deza)
+* ceph-volume: custom cluster names fail on filestore trigger (`issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#24280 <https://github.com/ceph/ceph/pull/24280>`_, Alfredo Deza)
+* ceph-volume: do not send (lvm) stderr/stdout to the terminal, use the logfile (`issue#36492 <http://tracker.ceph.com/issues/36492>`_, `pr#24741 <https://github.com/ceph/ceph/pull/24741>`_, Alfredo Deza)
+* ceph-volume: earlier detection for --journal and --filestore flag requirements (`issue#24794 <http://tracker.ceph.com/issues/24794>`_, `pr#24206 <https://github.com/ceph/ceph/pull/24206>`_, Alfredo Deza)
+* ceph-volume: fix journal and filestore data size in `lvm batch --report` (`issue#36242 <http://tracker.ceph.com/issues/36242>`_, `pr#24307 <https://github.com/ceph/ceph/pull/24307>`_, Andrew Schoen)
+* ceph-volume: fix zap not working with LVs (`issue#35970 <http://tracker.ceph.com/issues/35970>`_, `pr#24082 <https://github.com/ceph/ceph/pull/24082>`_, Alfredo Deza)
+* ceph-volume: lvm.prepare update help to indicate partitions are needed, not devices (`issue#24795 <http://tracker.ceph.com/issues/24795>`_, `pr#24451 <https://github.com/ceph/ceph/pull/24451>`_, Jeffrey Zhang, Alfredo Deza)
+* ceph-volume: make `lvm batch` idempotent (`pr#24589 <https://github.com/ceph/ceph/pull/24589>`_, Andrew Schoen)
+* ceph-volume: remove version reporting from help menu (`issue#36386 <http://tracker.ceph.com/issues/36386>`_, `pr#24754 <https://github.com/ceph/ceph/pull/24754>`_, Alfredo Deza)
+* ceph-volume: skip processing devices that don't exist when scanning system disks (`issue#36247 <http://tracker.ceph.com/issues/36247>`_, `pr#24382 <https://github.com/ceph/ceph/pull/24382>`_, Alfredo Deza)
+* cephfs: MDSMonitor: consider raising priority of MMDSBeacons from MDS so they are processed before other client messages (`issue#26899 <http://tracker.ceph.com/issues/26899>`_, `pr#23554 <https://github.com/ceph/ceph/pull/23554>`_, Patrick Donnelly)
+* cephfs: MDSMonitor: lookup of gid in prepare_beacon that has been removed will cause exception (`issue#35848 <http://tracker.ceph.com/issues/35848>`_, `pr#23990 <https://github.com/ceph/ceph/pull/23990>`_, Patrick Donnelly)
+* cephfs: ceph-fuse: add SELinux policy (`issue#36103 <http://tracker.ceph.com/issues/36103>`_, `pr#24313 <https://github.com/ceph/ceph/pull/24313>`_, Patrick Donnelly)
+* cephfs: ceph_volume_client: allow atomic update of RADOS objects (`issue#24173 <http://tracker.ceph.com/issues/24173>`_, `pr#24084 <https://github.com/ceph/ceph/pull/24084>`_, Rishabh Dave)
+* cephfs: ceph_volume_client: delay required after adding data pool to MDSMap (`issue#25141 <http://tracker.ceph.com/issues/25141>`_, `pr#23726 <https://github.com/ceph/ceph/pull/23726>`_, Patrick Donnelly)
+* cephfs: ceph_volume_client: py3 compatible (`issue#17230 <http://tracker.ceph.com/issues/17230>`_, `pr#24083 <https://github.com/ceph/ceph/pull/24083>`_, Rishabh Dave, Patrick Donnelly)
+* cephfs: cephfs-data-scan: print the max used ino (`issue#26925 <http://tracker.ceph.com/issues/26925>`_, `pr#23881 <https://github.com/ceph/ceph/pull/23881>`_, "Yan, Zheng")
+* cephfs: cephfs-journal-tool: wrong layout info used (`issue#24644 <http://tracker.ceph.com/issues/24644>`_, `pr#24033 <https://github.com/ceph/ceph/pull/24033>`_, Gu Zhongyan)
+* cephfs: client: check for unmounted condition before printing debug output (`issue#25213 <http://tracker.ceph.com/issues/25213>`_, `pr#23617 <https://github.com/ceph/ceph/pull/23617>`_, Jeff Layton)
+* cephfs: client: drop null child dentries before try pruning inode's alias (`issue#22293 <http://tracker.ceph.com/issues/22293>`_, `pr#24119 <https://github.com/ceph/ceph/pull/24119>`_, "Yan, Zheng")
+* cephfs: client: fix choose_target_mds for requests that do name lookup (`issue#26860 <http://tracker.ceph.com/issues/26860>`_, `pr#23793 <https://github.com/ceph/ceph/pull/23793>`_, "Yan, Zheng")
+* cephfs: client: retry remount on dcache invalidation failure (`issue#27657 <http://tracker.ceph.com/issues/27657>`_, `pr#24303 <https://github.com/ceph/ceph/pull/24303>`_, Venky Shankar)
+* cephfs: client: statfs inode count odd (`issue#24849 <http://tracker.ceph.com/issues/24849>`_, `pr#24376 <https://github.com/ceph/ceph/pull/24376>`_, Rishabh Dave)
+* cephfs: client: two ceph-fuse clients, one can not list out files created by another (`issue#27051 <http://tracker.ceph.com/issues/27051>`_, `pr#24282 <https://github.com/ceph/ceph/pull/24282>`_, Peng Xie)
+* cephfs: client: update ctime when modifying file content (`issue#35945 <http://tracker.ceph.com/issues/35945>`_, `pr#24323 <https://github.com/ceph/ceph/pull/24323>`_, "Yan, Zheng")
+* common: get real hostname from container/pod environment (`pr#23915 <https://github.com/ceph/ceph/pull/23915>`_, Sage Weil)
+* core: PGPool::update optimizations (`pr#23969 <https://github.com/ceph/ceph/pull/23969>`_, Zac Medico)
+* core: ceph-disk: compatibility fix for python 3 (`issue#35906 <http://tracker.ceph.com/issues/35906>`_, `pr#24347 <https://github.com/ceph/ceph/pull/24347>`_, Tim Serong)
+* core: discover_all_missing() not always called during activating (`issue#22837 <http://tracker.ceph.com/issues/22837>`_, `pr#23817 <https://github.com/ceph/ceph/pull/23817>`_, Sage Weil, David Zafman)
+* core: kv/KeyValueDB: return const char\* from MergeOperator::name() (`issue#26875 <http://tracker.ceph.com/issues/26875>`_, `pr#23566 <https://github.com/ceph/ceph/pull/23566>`_, Sage Weil)
+* core: librados application's symbol could conflict with the libceph-common (`issue#25154 <http://tracker.ceph.com/issues/25154>`_, `pr#23483 <https://github.com/ceph/ceph/pull/23483>`_, Kefu Chai)
+* core: mgr/MgrClient: guard send_pgstats() with lock (`issue#23370 <http://tracker.ceph.com/issues/23370>`_, `pr#23791 <https://github.com/ceph/ceph/pull/23791>`_, Kefu Chai)
+* core: mgr/balancer: deepcopy best plan - otherwise we get latest (`issue#27000 <http://tracker.ceph.com/issues/27000>`_, `pr#23740 <https://github.com/ceph/ceph/pull/23740>`_, Stefan Priebe)
+* core: mgrc: enable disabling stats via mgr_stats_threshold (`issue#25197 <http://tracker.ceph.com/issues/25197>`_, `pr#23461 <https://github.com/ceph/ceph/pull/23461>`_, John Spray)
+* core: mon/OSDMonitor: invalidate max_failed_since on cancel_report (`issue#35860 <http://tracker.ceph.com/issues/35860>`_, `pr#24257 <https://github.com/ceph/ceph/pull/24257>`_, xie xingguo)
+* core: object errors found in be_select_auth_object() aren't logged the same (`issue#25108 <http://tracker.ceph.com/issues/25108>`_, `pr#23871 <https://github.com/ceph/ceph/pull/23871>`_, David Zafman)
+* core: os/bluestore: bluestore_buffer_hit_bytes perf counter doesn't reset (`pr#23773 <https://github.com/ceph/ceph/pull/23773>`_, Igor Fedotov)
+* core: os/bluestore: cache autotuning and memory limit (`pr#24065 <https://github.com/ceph/ceph/pull/24065>`_, Mark Nelson)
+* core: osd,mon: increase mon_max_pg_per_osd to 250 (`issue#25112 <http://tracker.ceph.com/issues/25112>`_, `pr#23862 <https://github.com/ceph/ceph/pull/23862>`_, Neha Ojha)
+* core: osd/PG: avoid choose_acting picking want with > pool size items (`issue#35924 <http://tracker.ceph.com/issues/35924>`_, `pr#24299 <https://github.com/ceph/ceph/pull/24299>`_, Sage Weil)
+* core: osdc/Objecter: fix split vs reconnect race (`issue#22544 <http://tracker.ceph.com/issues/22544>`_, `pr#24188 <https://github.com/ceph/ceph/pull/24188>`_, Sage Weil)
+* core: rados python bindings use prval from stack (`issue#25175 <http://tracker.ceph.com/issues/25175>`_, `pr#23864 <https://github.com/ceph/ceph/pull/23864>`_, Sage Weil)
+* doc: Fix broken urls (`issue#25185 <http://tracker.ceph.com/issues/25185>`_, `pr#23621 <https://github.com/ceph/ceph/pull/23621>`_, Jos Collin)
+* doc: remove deprecated 'scrubq' from ceph(8) (`issue#35813 <http://tracker.ceph.com/issues/35813>`_, `pr#24211 <https://github.com/ceph/ceph/pull/24211>`_, Ruben Kerkhof)
+* doc: rgw: ldap-auth: fixed option name 'rgw_ldap_searchfilter' (`issue#23081 <http://tracker.ceph.com/issues/23081>`_, `pr#23761 <https://github.com/ceph/ceph/pull/23761>`_, Konstantin Shalygin)
+* mds: MDBalancer::try_rebalance() may stop prematurely (`issue#26973 <http://tracker.ceph.com/issues/26973>`_, `pr#23884 <https://github.com/ceph/ceph/pull/23884>`_, "Yan, Zheng")
+* mds: allows client to create .. and . dirents (`issue#25113 <http://tracker.ceph.com/issues/25113>`_, `pr#24329 <https://github.com/ceph/ceph/pull/24329>`_, Venky Shankar)
+* mds: avoid using g_conf->get_val<...>(...) in hot path (`issue#24820 <http://tracker.ceph.com/issues/24820>`_, `pr#23408 <https://github.com/ceph/ceph/pull/23408>`_, "Yan, Zheng")
+* mds: calculate load by checking self CPU usage (`issue#26834 <http://tracker.ceph.com/issues/26834>`_, `pr#23505 <https://github.com/ceph/ceph/pull/23505>`_, "Yan, Zheng")
+* mds: configurable timeout for client eviction (`issue#25188 <http://tracker.ceph.com/issues/25188>`_, `pr#24086 <https://github.com/ceph/ceph/pull/24086>`_, Patrick Donnelly, Venky Shankar)
+* mds: crash when dumping ops in flight (`issue#26894 <http://tracker.ceph.com/issues/26894>`_, `pr#23677 <https://github.com/ceph/ceph/pull/23677>`_, "Yan, Zheng")
+* mds: curate priority of perf counters sent to mgr (`issue#22097 <http://tracker.ceph.com/issues/22097>`_, `issue#24004 <http://tracker.ceph.com/issues/24004>`_, `pr#24089 <https://github.com/ceph/ceph/pull/24089>`_, Guan yunfei, Venky Shankar)
+* mds: explain delayed client_request due to subtree migration (`issue#24840 <http://tracker.ceph.com/issues/24840>`_, `pr#23678 <https://github.com/ceph/ceph/pull/23678>`_, Yan, Zheng, "Yan, Zheng")
+* mds: health warning for slow metadata IO (`issue#24879 <http://tracker.ceph.com/issues/24879>`_, `pr#24171 <https://github.com/ceph/ceph/pull/24171>`_, "Yan, Zheng")
+* mds: internal op missing events time 'throttled', 'all_read', 'dispatched' (`issue#36114 <http://tracker.ceph.com/issues/36114>`_, `pr#24410 <https://github.com/ceph/ceph/pull/24410>`_, Yanhu Cao)
+* mds: mds got laggy because of MDSBeacon stuck in mqueue (`issue#23519 <http://tracker.ceph.com/issues/23519>`_, `pr#23556 <https://github.com/ceph/ceph/pull/23556>`_, "Yan, Zheng")
+* mds: optimize the way how max export size is enforced (`issue#25131 <http://tracker.ceph.com/issues/25131>`_, `pr#23789 <https://github.com/ceph/ceph/pull/23789>`_, "Yan, Zheng")
+* mds: prevent MDSRank::evict_client from blocking finisher thread (`issue#35720 <http://tracker.ceph.com/issues/35720>`_, `pr#23946 <https://github.com/ceph/ceph/pull/23946>`_, "Yan, Zheng")
+* mds: print is_laggy message once (`issue#35250 <http://tracker.ceph.com/issues/35250>`_, `pr#24138 <https://github.com/ceph/ceph/pull/24138>`_, Patrick Donnelly)
+* mds: rctime may go back (`issue#35916 <http://tracker.ceph.com/issues/35916>`_, `pr#24378 <https://github.com/ceph/ceph/pull/24378>`_, "Yan, Zheng")
+* mds: reset heartbeat map at potential time-consuming places (`issue#26858 <http://tracker.ceph.com/issues/26858>`_, `pr#23507 <https://github.com/ceph/ceph/pull/23507>`_, Yan, Zheng, "Yan, Zheng")
+* mds: runs out of file descriptors after several respawns (`issue#35850 <http://tracker.ceph.com/issues/35850>`_, `pr#24310 <https://github.com/ceph/ceph/pull/24310>`_, Patrick Donnelly)
+* mds: track average session uptime (`issue#25013 <http://tracker.ceph.com/issues/25013>`_, `pr#24421 <https://github.com/ceph/ceph/pull/24421>`_, Patrick Donnelly, Venky Shankar)
+* mds: use monotonic clock for beacon message timekeeping (`issue#26959 <http://tracker.ceph.com/issues/26959>`_, `pr#24311 <https://github.com/ceph/ceph/pull/24311>`_, Patrick Donnelly)
+* mgr: Sync the prometheus module (`pr#23216 <https://github.com/ceph/ceph/pull/23216>`_, Boris Ranto)
+* mon: Automatically set expected_num_objects for new pools with >=100 PGs per OSD (`issue#24687 <http://tracker.ceph.com/issues/24687>`_, `pr#24395 <https://github.com/ceph/ceph/pull/24395>`_, Douglas Fuller)
+* msg: "challenging authorizer" messages appear at debug_ms=0 (`issue#35251 <http://tracker.ceph.com/issues/35251>`_, `pr#23943 <https://github.com/ceph/ceph/pull/23943>`_, Patrick Donnelly)
+* msg: async: clean up local buffers on dispatch (`issue#35987 <http://tracker.ceph.com/issues/35987>`_, `pr#24387 <https://github.com/ceph/ceph/pull/24387>`_, Greg Farnum)
+* msg: ceph_abort() when there are enough accepter errors in msg server (`issue#23649 <http://tracker.ceph.com/issues/23649>`_, `pr#24419 <https://github.com/ceph/ceph/pull/24419>`_, penglaiyxy@gmail.com)
+* osd: EC: slow/hung ops in multimds suite test (`issue#23769 <http://tracker.ceph.com/issues/23769>`_, `pr#24393 <https://github.com/ceph/ceph/pull/24393>`_, Sage Weil)
+* osd: ECBackend: don't get result code of subchunk-read overwritten (`issue#21769 <http://tracker.ceph.com/issues/21769>`_, `pr#24342 <https://github.com/ceph/ceph/pull/24342>`_, songweibin)
+* osd: Limit pg log length during recovery/backfill so that we don't run out of memory (`issue#21416 <http://tracker.ceph.com/issues/21416>`_, `pr#23211 <https://github.com/ceph/ceph/pull/23211>`_, Neha Ojha, xie xingguo)
+* osd: OSDMap: fix apply upmap segfault (`issue#22056 <http://tracker.ceph.com/issues/22056>`_, `pr#23579 <https://github.com/ceph/ceph/pull/23579>`_, Brad Hubbard)
+* osd: PG: add custom_reaction Backfilled and release reservations after bac… (`issue#23614 <http://tracker.ceph.com/issues/23614>`_, `pr#23493 <https://github.com/ceph/ceph/pull/23493>`_, Neha Ojha)
+* osd: PrimaryLogPG: fix potential pg-log overtrimming (`pr#24308 <https://github.com/ceph/ceph/pull/24308>`_, xie xingguo)
+* osd: backport 'bench' and stdout changes (`issue#24022 <http://tracker.ceph.com/issues/24022>`_, `pr#23680 <https://github.com/ceph/ceph/pull/23680>`_, Коренберг Маркr, John Spray, Kefu Chai)
+* osd: read object attrs failed at EC recovery (`issue#24406 <http://tracker.ceph.com/issues/24406>`_, `pr#24327 <https://github.com/ceph/ceph/pull/24327>`_, xiaofei cui)
+* osd: scrub livelock (`issue#26890 <http://tracker.ceph.com/issues/26890>`_, `pr#24396 <https://github.com/ceph/ceph/pull/24396>`_, Sage Weil)
+* qa/suites/rados/upgrade/jewel-x-singleton: exclude python3-rados, python3-cephfs (`pr#24479 <https://github.com/ceph/ceph/pull/24479>`_, Neha Ojha)
+* rbd: [rbd-mirror] failed assertion when updating mirror status (`issue#36084 <http://tracker.ceph.com/issues/36084>`_, `pr#24320 <https://github.com/ceph/ceph/pull/24320>`_, Jason Dillaman)
+* rbd: fix error import when the input is a pipe (`issue#34536 <http://tracker.ceph.com/issues/34536>`_, `pr#24003 <https://github.com/ceph/ceph/pull/24003>`_, songweibin)
+* rbd: librbd: blacklisted client might not notice it lost the lock (`issue#34534 <http://tracker.ceph.com/issues/34534>`_, `pr#24405 <https://github.com/ceph/ceph/pull/24405>`_, Song Shun, Mykola Golub, Jason Dillaman)
+* rbd: librbd: discard should wait for in-flight cache writeback to complete (`issue#23548 <http://tracker.ceph.com/issues/23548>`_, `pr#23594 <https://github.com/ceph/ceph/pull/23594>`_, Jason Dillaman)
+* rbd: librbd: ensure exclusive lock acquired when removing sync point snaps… (`issue#24898 <http://tracker.ceph.com/issues/24898>`_, `pr#24123 <https://github.com/ceph/ceph/pull/24123>`_, Mykola Golub, Jason Dillaman)
+* rbd: librbd: fix refuse to release lock when cookie is the same at rewatch (`issue#27986 <http://tracker.ceph.com/issues/27986>`_, `pr#23758 <https://github.com/ceph/ceph/pull/23758>`_, Song Shun)
+* rbd: librbd: fixed assert when flattening clone with zero overlap (`issue#35702 <http://tracker.ceph.com/issues/35702>`_, `pr#24285 <https://github.com/ceph/ceph/pull/24285>`_, Jason Dillaman)
+* rbd: librbd: image create request should validate data pool for self-managed snapshot support (`issue#24675 <http://tracker.ceph.com/issues/24675>`_, `pr#24390 <https://github.com/ceph/ceph/pull/24390>`_, Mykola Golub)
+* rbd: librbd: journaling unable request can not be sent to remote lock owner (`issue#26939 <http://tracker.ceph.com/issues/26939>`_, `pr#24100 <https://github.com/ceph/ceph/pull/24100>`_, Mykola Golub)
+* rbd: librbd: object map improperly flagged as invalidated (`issue#24516 <http://tracker.ceph.com/issues/24516>`_, `pr#24415 <https://github.com/ceph/ceph/pull/24415>`_, Jason Dillaman)
+* rbd: librbd: potential race on image create request complete (`issue#24910 <http://tracker.ceph.com/issues/24910>`_, `pr#23892 <https://github.com/ceph/ceph/pull/23892>`_, Mykola Golub)
+* rgw: 'radosgw-admin sync error trim' only trims partially (`issue#24873 <http://tracker.ceph.com/issues/24873>`_, `pr#24054 <https://github.com/ceph/ceph/pull/24054>`_, Casey Bodley)
+* rgw: Fix log level of gc_iterate_entries (`issue#23801 <http://tracker.ceph.com/issues/23801>`_, `pr#23665 <https://github.com/ceph/ceph/pull/23665>`_, iliul)
+* rgw: Limit the number of lifecycle rules on one bucket (`issue#24572 <http://tracker.ceph.com/issues/24572>`_, `pr#23522 <https://github.com/ceph/ceph/pull/23522>`_, Zhang Shaowen)
+* rgw: The delete markers generated by object expiration should have owner (`issue#24568 <http://tracker.ceph.com/issues/24568>`_, `pr#23545 <https://github.com/ceph/ceph/pull/23545>`_, Zhang Shaowen)
+* rgw: abort_bucket_multiparts() ignores individual NoSuchUpload errors (`issue#35986 <http://tracker.ceph.com/issues/35986>`_, `pr#24389 <https://github.com/ceph/ceph/pull/24389>`_, Casey Bodley)
+* rgw: change default rgw_thread_pool_size to 512 (`issue#25214 <http://tracker.ceph.com/issues/25214>`_, `issue#24544 <http://tracker.ceph.com/issues/24544>`_, `pr#24034 <https://github.com/ceph/ceph/pull/24034>`_, Douglas Fuller, Casey Bodley)
+* rgw: cls/rgw: don't assert in decode_list_index_key() (`issue#24117 <http://tracker.ceph.com/issues/24117>`_, `pr#24391 <https://github.com/ceph/ceph/pull/24391>`_, Yehuda Sadeh)
+* rgw: cls/rgw: ready rgw_usage_log_entry for extraction via ceph-dencoder (`issue#34537 <http://tracker.ceph.com/issues/34537>`_, `pr#23974 <https://github.com/ceph/ceph/pull/23974>`_, Vaibhav Bhembre)
+* rgw: fix chunked-encoding for chunks >1MiB (`issue#35990 <http://tracker.ceph.com/issues/35990>`_, `pr#24361 <https://github.com/ceph/ceph/pull/24361>`_, Robin H. Johnson)
+* rgw: fix deadlock on RGWIndexCompletionManager::stop (`issue#26949 <http://tracker.ceph.com/issues/26949>`_, `pr#24069 <https://github.com/ceph/ceph/pull/24069>`_, Yao Zongyou)
+* rgw: incremental data sync uses truncated flag to detect end of listing (`issue#26952 <http://tracker.ceph.com/issues/26952>`_, `pr#24242 <https://github.com/ceph/ceph/pull/24242>`_, Casey Bodley)
+* rgw: multisite: data sync error repo processing does not back off on empty (`issue#26938 <http://tracker.ceph.com/issues/26938>`_, `pr#24318 <https://github.com/ceph/ceph/pull/24318>`_, Casey Bodley)
+* rgw: multisite: intermittent failures in test_bucket_sync_disable_enable (`issue#26895 <http://tracker.ceph.com/issues/26895>`_, `pr#24316 <https://github.com/ceph/ceph/pull/24316>`_, Casey Bodley)
+* rgw: multisite: intermittent test_bucket_index_log_trim failures (`issue#36034 <http://tracker.ceph.com/issues/36034>`_, `pr#24398 <https://github.com/ceph/ceph/pull/24398>`_, Casey Bodley)
+* rgw: multisite: object metadata operations are skipped by sync (`issue#24367 <http://tracker.ceph.com/issues/24367>`_, `pr#24056 <https://github.com/ceph/ceph/pull/24056>`_, Casey Bodley)
+* rgw: multisite: object name should be urlencoded when we put it into ES (`issue#23216 <http://tracker.ceph.com/issues/23216>`_, `pr#24424 <https://github.com/ceph/ceph/pull/24424>`_, Chang Liu)
+* rgw: multisite: out of order updates to sync status markers (`issue#35539 <http://tracker.ceph.com/issues/35539>`_, `pr#24317 <https://github.com/ceph/ceph/pull/24317>`_, Yehuda Sadeh)
+* rgw: multisite: segfault on shutdown/realm reload (`issue#35543 <http://tracker.ceph.com/issues/35543>`_, `pr#24231 <https://github.com/ceph/ceph/pull/24231>`_, Casey Bodley)
+* rgw: multisite: update index segfault on shutdown/realm reload (`issue#35905 <http://tracker.ceph.com/issues/35905>`_, `pr#24397 <https://github.com/ceph/ceph/pull/24397>`_, Tianshan Qu)
+* rgw: raise debug level on redundant data sync error messages (`issue#35830 <http://tracker.ceph.com/issues/35830>`_, `issue#36037 <http://tracker.ceph.com/issues/36037>`_, `pr#24135 <https://github.com/ceph/ceph/pull/24135>`_, Casey Bodley, Matt Benjamin)
+* rgw: raise default rgw_curl_low_speed_time to 300 seconds (`issue#27989 <http://tracker.ceph.com/issues/27989>`_, `pr#24046 <https://github.com/ceph/ceph/pull/24046>`_, Casey Bodley)
+* rgw: resharding produces invalid values of bucket stats (`issue#36290 <http://tracker.ceph.com/issues/36290>`_, `pr#24527 <https://github.com/ceph/ceph/pull/24527>`_, Abhishek Lekshmanan)
+* rgw: return x-amz-version-id: null when delete obj in versioning suspended bucket (`issue#35814 <http://tracker.ceph.com/issues/35814>`_, `pr#24190 <https://github.com/ceph/ceph/pull/24190>`_, yuliyang)
+* rgw: rgw_file: deep stat handling (`issue#24915 <http://tracker.ceph.com/issues/24915>`_, `pr#23499 <https://github.com/ceph/ceph/pull/23499>`_, Matt Benjamin)
+* tests: Excluded 'python34-cephfs' from the install tasks (`pr#24650 <https://github.com/ceph/ceph/pull/24650>`_, Yuri Weinstein)
+* tests: Use pids instead of jobspecs which were wrong (`issue#27056 <http://tracker.ceph.com/issues/27056>`_, `pr#23901 <https://github.com/ceph/ceph/pull/23901>`_, David Zafman)
+* tests: cephfs: multifs requires 4 mds but gets only 2 (`issue#24899 <http://tracker.ceph.com/issues/24899>`_, `pr#24328 <https://github.com/ceph/ceph/pull/24328>`_, Patrick Donnelly)
+* tests: cls_rgw test is only run in rados suite: add it to rgw suite as well (`issue#24815 <http://tracker.ceph.com/issues/24815>`_, `pr#24070 <https://github.com/ceph/ceph/pull/24070>`_, Casey Bodley, Sage Weil)
+* tests: librbd: not valid to have different parents between image snapshots (`issue#36097 <http://tracker.ceph.com/issues/36097>`_, `pr#24245 <https://github.com/ceph/ceph/pull/24245>`_, Jason Dillaman)
+* tests: move mds/client config to qa from teuthology ceph.conf.template (`issue#26900 <http://tracker.ceph.com/issues/26900>`_, `issue#24839 <http://tracker.ceph.com/issues/24839>`_, `pr#23877 <https://github.com/ceph/ceph/pull/23877>`_, Patrick Donnelly)
+* tests: qa/tasks: s3a fix mirror (`pr#24039 <https://github.com/ceph/ceph/pull/24039>`_, Vasu Kulkarni)
+* tests: qa/workunits: replace 'realpath' with 'readlink -f' in fsstress.sh (`issue#27211 <http://tracker.ceph.com/issues/27211>`_, `issue#36409 <http://tracker.ceph.com/issues/36409>`_, `pr#24620 <https://github.com/ceph/ceph/pull/24620>`_, Ilya Dryomov, Jason Dillaman)
+* tests: qa: add .qa helper link (`pr#24134 <https://github.com/ceph/ceph/pull/24134>`_, Patrick Donnelly)
+* tests: qa: added v12.2.8 to the mix (`issue#35541 <http://tracker.ceph.com/issues/35541>`_, `pr#23913 <https://github.com/ceph/ceph/pull/23913>`_, Yuri Weinstein)
+* tests: remove knfs qa suite from future releases (`issue#36075 <http://tracker.ceph.com/issues/36075>`_, `pr#24268 <https://github.com/ceph/ceph/pull/24268>`_, Yuri Weinstein)
+* tools: ceph-objectstore-tool: Allow target level as first positional parameter (`issue#35846 <http://tracker.ceph.com/issues/35846>`_, `pr#24115 <https://github.com/ceph/ceph/pull/24115>`_, David Zafman)
+
+
+v12.2.8 Luminous
+================
+
+This is the eighth bug fix release of the Luminous v12.2.x long term stable
+release series. This release contains several bugfixes across all the components
+and we recommend all users upgrade.
+
+Upgrade Notes from previous luminous releases
+---------------------------------------------
+
+When upgrading from v12.2.5 or v12.2.6 please note that upgrade caveats from
+12.2.5 will apply to any _newer_ luminous version including 12.2.8. Please read
+the notes at luminous-12-2-5-upgrades_ .
+
+For the cluster that installed the broken 12.2.6 release, 12.2.7 fixed the
+regression and introduced a workaround option `osd distrust data digest = true`,
+but 12.2.7 clusters still generated health warnings like ::
+
+  [ERR] 11.288 shard 207: soid  11:1155c332:::rbd_data.207dce238e1f29.0000000000000527:head data_digest 0xc8997a5b != data_digest 0x2ca15853
+
+
+12.2.8 improves the deep scrub code to automatically repair these
+inconsistencies. Once the entire cluster has been upgraded and then fully deep
+scrubbed, and all such inconsistencies are resolved, it will be safe to disable
+the `osd distrust data digest = true` workaround option.
+
+Notable Changes
+---------------
+
+- *OSD*
+
+  * Scrub repair is enhanced to handle data digest mismatch info on
+    replicas as long as all replicas' digests match each other.
+
+- *RGW*
+
+  * Options `rgw curl low speed limit` and `rgw curl low speed time`
+    are added to control the lower speed limits and times below which
+    the requests are considered too slow to be aborted and can help
+    mitigate data sync getting blocked during network issues
+
+  * Option `rgw s3 auth order` configurable added which takes a comma
+    separated list of order to try for s3 authentication when external
+    engines are involved.
+
+Changelog
+---------
+* bluestore: set correctly shard for existed Collection (`issue#24761 <http://tracker.ceph.com/issues/24761>`_, `pr#22860 <https://github.com/ceph/ceph/pull/22860>`_, Jianpeng Ma)
+* build/ops: Boost system library is no longer required to compile and link example librados program (`issue#25054 <http://tracker.ceph.com/issues/25054>`_, `pr#23202 <https://github.com/ceph/ceph/pull/23202>`_, Nathan Cutler)
+* build/ops: Bring back diff -y for non-FreeBSD (`issue#24396 <http://tracker.ceph.com/issues/24396>`_, `issue#21664 <http://tracker.ceph.com/issues/21664>`_, `pr#22848 <https://github.com/ceph/ceph/pull/22848>`_, Sage Weil, David Zafman)
+* build/ops: install-deps.sh fails on newest openSUSE Leap (`issue#25064 <http://tracker.ceph.com/issues/25064>`_, `pr#23179 <https://github.com/ceph/ceph/pull/23179>`_, Kyr Shatskyy)
+* build/ops: Mimic build fails with -DWITH_RADOSGW=0 (`issue#24437 <http://tracker.ceph.com/issues/24437>`_, `pr#22864 <https://github.com/ceph/ceph/pull/22864>`_, Dan Mick)
+* build/ops: order rbdmap.service before remote-fs-pre.target (`issue#24713 <http://tracker.ceph.com/issues/24713>`_, `pr#22844 <https://github.com/ceph/ceph/pull/22844>`_, Ilya Dryomov)
+* build/ops: rpm: silence osd block chown (`issue#25152 <http://tracker.ceph.com/issues/25152>`_, `pr#23313 <https://github.com/ceph/ceph/pull/23313>`_, Dan van der Ster)
+* cephfs-journal-tool: Fix purging when importing an zero-length journal (`issue#24239 <http://tracker.ceph.com/issues/24239>`_, `pr#22980 <https://github.com/ceph/ceph/pull/22980>`_, yupeng chen, zhongyan gu)
+* cephfs: MDSMonitor: uncommitted state exposed to clients/mdss (`issue#23768 <http://tracker.ceph.com/issues/23768>`_, `pr#23013 <https://github.com/ceph/ceph/pull/23013>`_, Patrick Donnelly)
+* ceph-fuse mount failed because no mds (`issue#22205 <http://tracker.ceph.com/issues/22205>`_, `pr#22895 <https://github.com/ceph/ceph/pull/22895>`_, liyan)
+* ceph-volume add a __release__ string, to help version-conditional calls (`issue#25170 <http://tracker.ceph.com/issues/25170>`_, `pr#23331 <https://github.com/ceph/ceph/pull/23331>`_, Alfredo Deza)
+* ceph-volume: adds test for `ceph-volume lvm list /dev/sda` (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `issue#24957 <http://tracker.ceph.com/issues/24957>`_, `pr#23350 <https://github.com/ceph/ceph/pull/23350>`_, Andrew Schoen)
+* ceph-volume: do not use stdin in luminous (`issue#25173 <http://tracker.ceph.com/issues/25173>`_, `issue#23260 <http://tracker.ceph.com/issues/23260>`_, `pr#23367 <https://github.com/ceph/ceph/pull/23367>`_, Alfredo Deza)
+* ceph-volume enable the ceph-osd during lvm activation (`issue#24152 <http://tracker.ceph.com/issues/24152>`_, `pr#23394 <https://github.com/ceph/ceph/pull/23394>`_, Dan van der Ster, Alfredo Deza)
+* ceph-volume expand on the LVM API to create multiple LVs at different sizes (`issue#24020 <http://tracker.ceph.com/issues/24020>`_, `pr#23395 <https://github.com/ceph/ceph/pull/23395>`_, Alfredo Deza)
+* ceph-volume lvm.activate conditional mon-config on prime-osd-dir (`issue#25216 <http://tracker.ceph.com/issues/25216>`_, `pr#23397 <https://github.com/ceph/ceph/pull/23397>`_, Alfredo Deza)
+* ceph-volume lvm.batch remove non-existent sys_api property (`issue#34310 <http://tracker.ceph.com/issues/34310>`_, `pr#23811 <https://github.com/ceph/ceph/pull/23811>`_, Alfredo Deza)
+* ceph-volume lvm.listing only include devices if they exist (`issue#24952 <http://tracker.ceph.com/issues/24952>`_, `pr#23150 <https://github.com/ceph/ceph/pull/23150>`_, Alfredo Deza)
+* ceph-volume: process.call with stdin in Python 3 fix (`issue#24993 <http://tracker.ceph.com/issues/24993>`_, `pr#23238 <https://github.com/ceph/ceph/pull/23238>`_, Alfredo Deza)
+* ceph-volume: PVolumes.get() should return one PV when using name or uuid (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `pr#23329 <https://github.com/ceph/ceph/pull/23329>`_, Andrew Schoen)
+* ceph-volume: refuse to zap mapper devices (`issue#24504 <http://tracker.ceph.com/issues/24504>`_, `pr#23374 <https://github.com/ceph/ceph/pull/23374>`_, Andrew Schoen)
+* ceph-volume: tests.functional inherit SSH_ARGS from ansible (`issue#34311 <http://tracker.ceph.com/issues/34311>`_, `pr#23813 <https://github.com/ceph/ceph/pull/23813>`_, Alfredo Deza)
+* ceph-volume tests/functional run lvm list after OSD provisioning (`issue#24961 <http://tracker.ceph.com/issues/24961>`_, `pr#23147 <https://github.com/ceph/ceph/pull/23147>`_, Alfredo Deza)
+* ceph-volume: unmount lvs correctly before zapping (`issue#24796 <http://tracker.ceph.com/issues/24796>`_, `pr#23128 <https://github.com/ceph/ceph/pull/23128>`_, Andrew Schoen)
+* ceph-volume: update batch documentation to explain filestore strategies (`issue#34309 <http://tracker.ceph.com/issues/34309>`_, `pr#23825 <https://github.com/ceph/ceph/pull/23825>`_, Alfredo Deza)
+* change default filestore_merge_threshold to -10 (`issue#24686 <http://tracker.ceph.com/issues/24686>`_, `pr#22814 <https://github.com/ceph/ceph/pull/22814>`_, Douglas Fuller)
+* client: add inst to asok status output (`issue#24724 <http://tracker.ceph.com/issues/24724>`_, `pr#23107 <https://github.com/ceph/ceph/pull/23107>`_, Patrick Donnelly)
+* client: fixup parallel calls to ceph_ll_lookup_inode() in NFS FASL (`issue#22683 <http://tracker.ceph.com/issues/22683>`_, `pr#23012 <https://github.com/ceph/ceph/pull/23012>`_, huanwen ren)
+* client: increase verbosity level for log messages in helper methods (`issue#21014 <http://tracker.ceph.com/issues/21014>`_, `pr#23014 <https://github.com/ceph/ceph/pull/23014>`_, Rishabh Dave)
+* client:  update inode fields according to issued caps (`issue#24269 <http://tracker.ceph.com/issues/24269>`_, `pr#22783 <https://github.com/ceph/ceph/pull/22783>`_, "Yan, Zheng")
+* common: Abort in OSDMap::decode() during qa/standalone/erasure-code/test-erasure-eio.sh (`issue#23492 <http://tracker.ceph.com/issues/23492>`_, `pr#23025 <https://github.com/ceph/ceph/pull/23025>`_, Sage Weil)
+* common/DecayCounter: set last_decay to current time when decoding decay counter (`issue#24440 <http://tracker.ceph.com/issues/24440>`_, `pr#22779 <https://github.com/ceph/ceph/pull/22779>`_, Zhi Zhang)
+* doc: ceph-bluestore-tool manpage not getting rendered correctly (`issue#24800 <http://tracker.ceph.com/issues/24800>`_, `pr#23177 <https://github.com/ceph/ceph/pull/23177>`_, Nathan Cutler)
+* filestore: add pgid in filestore pg dir split log message (`issue#24878 <http://tracker.ceph.com/issues/24878>`_, `pr#23454 <https://github.com/ceph/ceph/pull/23454>`_, Vikhyat Umrao)
+* let "ceph status" use base 10 when printing numbers not sizes (`issue#22095 <http://tracker.ceph.com/issues/22095>`_, `pr#22680 <https://github.com/ceph/ceph/pull/22680>`_, Jan Fajerski, Kefu Chai)
+* librados: fix buffer overflow for aio_exec python binding (`issue#23964 <http://tracker.ceph.com/issues/23964>`_, `pr#22708 <https://github.com/ceph/ceph/pull/22708>`_, Aleksei Gutikov)
+* librbd: force 'invalid object map' flag on-disk update (`issue#24434 <http://tracker.ceph.com/issues/24434>`_, `pr#22753 <https://github.com/ceph/ceph/pull/22753>`_, Mykola Golub)
+* librbd: utilize the journal disabled policy when removing images (`issue#23512 <http://tracker.ceph.com/issues/23512>`_, `pr#23595 <https://github.com/ceph/ceph/pull/23595>`_, Jason Dillaman)
+* mds: don't report slow request for blocked filelock request (`issue#22428 <http://tracker.ceph.com/issues/22428>`_, `pr#22782 <https://github.com/ceph/ceph/pull/22782>`_, "Yan, Zheng")
+* mds: dump recent events on respawn (`issue#24853 <http://tracker.ceph.com/issues/24853>`_, `pr#23213 <https://github.com/ceph/ceph/pull/23213>`_, Patrick Donnelly)
+* mds: handle discontinuous mdsmap (`issue#24856 <http://tracker.ceph.com/issues/24856>`_, `pr#23169 <https://github.com/ceph/ceph/pull/23169>`_, "Yan, Zheng")
+* mds: increase debug level for dropped client cap msg (`issue#24855 <http://tracker.ceph.com/issues/24855>`_, `pr#23214 <https://github.com/ceph/ceph/pull/23214>`_, Patrick Donnelly)
+* mds: low wrlock efficiency due to dirfrags traversal (`issue#24467 <http://tracker.ceph.com/issues/24467>`_, `pr#22885 <https://github.com/ceph/ceph/pull/22885>`_, Xuehan Xu)
+* mds: print mdsmap processed at low debug level (`issue#24852 <http://tracker.ceph.com/issues/24852>`_, `pr#23212 <https://github.com/ceph/ceph/pull/23212>`_, Patrick Donnelly)
+* mds: scrub doesn't always return JSON results (`issue#23958 <http://tracker.ceph.com/issues/23958>`_, `pr#23222 <https://github.com/ceph/ceph/pull/23222>`_, Venky Shankar)
+* mds: unset deleted vars in shutdown_pass (`issue#23766 <http://tracker.ceph.com/issues/23766>`_, `pr#23015 <https://github.com/ceph/ceph/pull/23015>`_, Patrick Donnelly)
+* mgr: add units to performance counters (`issue#22747 <http://tracker.ceph.com/issues/22747>`_, `pr#23266 <https://github.com/ceph/ceph/pull/23266>`_, Ernesto Puerta, Rubab Syed)
+* mgr: ceph osd safe-to-destroy crashes the mgr (`issue#23249 <http://tracker.ceph.com/issues/23249>`_, `pr#22806 <https://github.com/ceph/ceph/pull/22806>`_, Sage Weil)
+* mgr/MgrClient: Protect daemon_health_metrics (`issue#23352 <http://tracker.ceph.com/issues/23352>`_, `pr#23459 <https://github.com/ceph/ceph/pull/23459>`_, Kjetil Joergensen, Brad Hubbard)
+* mon: Add option to view IP addresses of clients in output of 'ceph features' (`issue#21315 <http://tracker.ceph.com/issues/21315>`_, `pr#22773 <https://github.com/ceph/ceph/pull/22773>`_, Paul Emmerich)
+* mon/HealthMonitor: do not send MMonHealthChecks to pre-luminous mon (`issue#24481 <http://tracker.ceph.com/issues/24481>`_, `pr#22655 <https://github.com/ceph/ceph/pull/22655>`_, Sage Weil)
+* os/bluestore: fix flush_commit locking (`issue#21480 <http://tracker.ceph.com/issues/21480>`_, `pr#22904 <https://github.com/ceph/ceph/pull/22904>`_, Sage Weil)
+* os/bluestore: fix incomplete faulty range marking when doing compression (`issue#21480 <http://tracker.ceph.com/issues/21480>`_, `pr#22909 <https://github.com/ceph/ceph/pull/22909>`_, Igor Fedotov)
+* os/bluestore: fix races on SharedBlob::coll in ~SharedBlob (`issue#24859 <http://tracker.ceph.com/issues/24859>`_, `pr#23064 <https://github.com/ceph/ceph/pull/23064>`_, Radoslaw Zarzynski)
+* osdc: Fix the wrong BufferHead offset (`issue#24484 <http://tracker.ceph.com/issues/24484>`_, `pr#22865 <https://github.com/ceph/ceph/pull/22865>`_, dongdong tao)
+* osd: do_sparse_read(): Verify checksum earlier so we will try to repair and missed backport (`issue#24875 <http://tracker.ceph.com/issues/24875>`_, `pr#23379 <https://github.com/ceph/ceph/pull/23379>`_, xie xingguo, David Zafman)
+* osd: eternal stuck PG in 'unfound_recovery' (`issue#24373 <http://tracker.ceph.com/issues/24373>`_, `pr#22546 <https://github.com/ceph/ceph/pull/22546>`_, Sage Weil)
+* osd: may get empty info at recovery (`issue#24588 <http://tracker.ceph.com/issues/24588>`_, `pr#22862 <https://github.com/ceph/ceph/pull/22862>`_, Sage Weil)
+* osd/OSDMap: CRUSH_TUNABLES5 added in jewel, not kraken (`issue#25057 <http://tracker.ceph.com/issues/25057>`_, `pr#23227 <https://github.com/ceph/ceph/pull/23227>`_, Sage Weil)
+* osd/Session: fix invalid iterator dereference in Sessoin::have_backoff() (`issue#24486 <http://tracker.ceph.com/issues/24486>`_, `pr#22729 <https://github.com/ceph/ceph/pull/22729>`_, Sage Weil)
+* pjd: cd: too many arguments (`issue#24307 <http://tracker.ceph.com/issues/24307>`_, `pr#22883 <https://github.com/ceph/ceph/pull/22883>`_, Neha Ojha)
+* PurgeQueue sometimes ignores Journaler errors (`issue#24533 <http://tracker.ceph.com/issues/24533>`_, `pr#22811 <https://github.com/ceph/ceph/pull/22811>`_, John Spray)
+* pybind: pybind/mgr/mgr_module: make rados handle available to all modules (`issue#24788 <http://tracker.ceph.com/issues/24788>`_, `issue#25102 <http://tracker.ceph.com/issues/25102>`_, `pr#23235 <https://github.com/ceph/ceph/pull/23235>`_, Ernesto Puerta, Sage Weil)
+* pybind: Python bindings use iteritems method which is not Python 3 compatible (`issue#24779 <http://tracker.ceph.com/issues/24779>`_, `pr#22918 <https://github.com/ceph/ceph/pull/22918>`_, Nathan Cutler, Kefu Chai)
+* pybind: rados.pyx: make all exceptions accept keyword arguments (`issue#24033 <http://tracker.ceph.com/issues/24033>`_, `pr#22979 <https://github.com/ceph/ceph/pull/22979>`_, Rishabh Dave)
+* rbd: fix issues in IEC unit handling (`issue#26927 <http://tracker.ceph.com/issues/26927>`_, `issue#26928 <http://tracker.ceph.com/issues/26928>`_, `pr#23776 <https://github.com/ceph/ceph/pull/23776>`_, Jason Dillaman)
+* repeated eviction of idle client until some IO happens (`issue#24052 <http://tracker.ceph.com/issues/24052>`_, `pr#22780 <https://github.com/ceph/ceph/pull/22780>`_, "Yan, Zheng")
+* rgw: add curl_low_speed_limit and curl_low_speed_time config to avoid the thread hangs in data sync (`issue#25019 <http://tracker.ceph.com/issues/25019>`_, `pr#23144 <https://github.com/ceph/ceph/pull/23144>`_, Mark Kogan, Zhang Shaowen)
+* rgw: add unit test for cls bi list command (`issue#24483 <http://tracker.ceph.com/issues/24483>`_, `pr#22846 <https://github.com/ceph/ceph/pull/22846>`_, Orit Wasserman, Xinying Song)
+* rgw: do not ignore EEXIST in RGWPutObj::execute (`issue#22790 <http://tracker.ceph.com/issues/22790>`_, `pr#23207 <https://github.com/ceph/ceph/pull/23207>`_, Matt Benjamin)
+* rgw: fail to recover index from crash luminous backport (`issue#24640 <http://tracker.ceph.com/issues/24640>`_, `issue#24280 <http://tracker.ceph.com/issues/24280>`_, `pr#23130 <https://github.com/ceph/ceph/pull/23130>`_, Tianshan Qu)
+* rgw: fix gc may cause a large number of read traffic (`issue#24767 <http://tracker.ceph.com/issues/24767>`_, `pr#22984 <https://github.com/ceph/ceph/pull/22984>`_, Xin Liao)
+* rgw: fix the bug of radowgw-admin zonegroup set requires realm (`issue#21583 <http://tracker.ceph.com/issues/21583>`_, `pr#22767 <https://github.com/ceph/ceph/pull/22767>`_, lvshanchun)
+* rgw: have a configurable authentication order (`issue#23089 <http://tracker.ceph.com/issues/23089>`_, `pr#23501 <https://github.com/ceph/ceph/pull/23501>`_, Abhishek Lekshmanan)
+* rgw: index complete miss zones_trace set (`issue#24590 <http://tracker.ceph.com/issues/24590>`_, `pr#22820 <https://github.com/ceph/ceph/pull/22820>`_, Tianshan Qu)
+* rgw: Invalid Access-Control-Request-Request may bypass validate_cors_rule_method (`issue#24223 <http://tracker.ceph.com/issues/24223>`_, `pr#22934 <https://github.com/ceph/ceph/pull/22934>`_, Jeegn Chen)
+* rgw: meta and data notify thread miss stop cr manager (`issue#24589 <http://tracker.ceph.com/issues/24589>`_, `pr#22822 <https://github.com/ceph/ceph/pull/22822>`_, Tianshan Qu)
+* rgw-multisite: endless loop in RGWBucketShardIncrementalSyncCR (`issue#24603 <http://tracker.ceph.com/issues/24603>`_, `pr#22817 <https://github.com/ceph/ceph/pull/22817>`_, cfanz)
+* rgw performance regression for luminous 12.2.4 (`issue#23379 <http://tracker.ceph.com/issues/23379>`_, `pr#22930 <https://github.com/ceph/ceph/pull/22930>`_, Mark Kogan)
+* rgw: radogw-admin reshard status command should print text for reshar… (`issue#23257 <http://tracker.ceph.com/issues/23257>`_, `pr#23019 <https://github.com/ceph/ceph/pull/23019>`_, Orit Wasserman)
+* rgw: "radosgw-admin objects expire" always returns ok even if the pro… (`issue#24592 <http://tracker.ceph.com/issues/24592>`_, `pr#23000 <https://github.com/ceph/ceph/pull/23000>`_, Zhang Shaowen)
+* rgw: require --yes-i-really-mean-it to run radosgw-admin orphans find (`issue#24146 <http://tracker.ceph.com/issues/24146>`_, `pr#22985 <https://github.com/ceph/ceph/pull/22985>`_, Matt Benjamin)
+* rgw: REST admin metadata API paging failure bucket & bucket.instance: InvalidArgument (`issue#23099 <http://tracker.ceph.com/issues/23099>`_, `pr#22932 <https://github.com/ceph/ceph/pull/22932>`_, Matt Benjamin)
+* rgw: set cr state if aio_read err return in RGWCloneMetaLogCoroutine (`issue#24566 <http://tracker.ceph.com/issues/24566>`_, `pr#22942 <https://github.com/ceph/ceph/pull/22942>`_, Tianshan Qu)
+* spdk: fix ceph-osd crash when activate SPDK (`issue#24371 <http://tracker.ceph.com/issues/24371>`_, `pr#22686 <https://github.com/ceph/ceph/pull/22686>`_, tone-zhang)
+* tools/ceph-objectstore-tool: split filestore directories offline to target hash level (`issue#21366 <http://tracker.ceph.com/issues/21366>`_, `pr#23418 <https://github.com/ceph/ceph/pull/23418>`_, Zhi Zhang)
+
+
+v12.2.7 Luminous
+================
+
+This is the seventh bugfix release of Luminous v12.2.x long term
+stable release series. This release contains several fixes for
+regressions in the v12.2.6 and v12.2.5 releases.  We recommend that
+all users upgrade.
+
+:note: The v12.2.6 release has serious known regressions.  If you installed this release, please see the upgrade procedure below.
+
+:note: The v12.2.5 release has a potential data corruption issue with erasure coded pools.  If you ran v12.2.5 with erasure coding, please see below.
+
+.. _luminous-12-2-5-upgrades:
+
+Upgrading from v12.2.6
+----------------------
+
+v12.2.6 included an incomplete backport of an optimization for
+BlueStore OSDs that avoids maintaining both the per-object checksum
+and the internal BlueStore checksum.  Due to the accidental omission
+of a critical follow-on patch, v12.2.6 corrupts (fails to update) the
+stored per-object checksum value for some objects.  This can result in
+an EIO error when trying to read those objects.
+
+#. If your cluster uses FileStore only, no special action is required.
+   This problem only affects clusters with BlueStore.
+
+#. If your cluster has only BlueStore OSDs (no FileStore), then you
+   should enable the following OSD option::
+
+     osd skip data digest = true
+
+   This will avoid setting and start ignoring the full-object digests
+   whenever the primary for a PG is BlueStore.
+
+#. If you have a mix of BlueStore and FileStore OSDs, then you should
+   enable the following OSD option::
+
+     osd distrust data digest = true
+
+   This will avoid setting and start ignoring the full-object digests
+   in all cases.  This weakens the data integrity checks for
+   FileStore (although those checks were always only opportunistic).
+
+If your cluster includes BlueStore OSDs and was affected, deep scrubs
+will generate errors about mismatched CRCs for affected objects.
+Currently the repair operation does not know how to correct them
+(since all replicas do not match the expected checksum it does not
+know how to proceed).  These warnings are harmless in the sense that
+IO is not affected and the replicas are all still in sync.  The number
+of affected objects is likely to drop (possibly to zero) on their own
+over time as those objects are modified.  We expect to include a scrub
+improvement in v12.2.8 to clean up any remaining objects.
+
+Additionally, see the notes below, which apply to both v12.2.5 and v12.2.6.
+
+Upgrading from v12.2.5 or v12.2.6
+---------------------------------
+
+If you used v12.2.5 or v12.2.6 in combination with erasure coded
+pools, there is a small risk of corruption under certain workloads.
+Specifically, when:
+
+* An erasure coded pool is in use
+* The pool is busy with successful writes
+* The pool is also busy with updates that result in an error result to
+  the librados user.  RGW garbage collection is the most common
+  example of this (it sends delete operations on objects that don't
+  always exist.)
+* Some OSDs are reasonably busy.  One known example of such load is
+  FileStore splitting, although in principle any load on the cluster
+  could also trigger the behavior.
+* One or more OSDs restarts.
+
+This combination can trigger an OSD crash and possibly leave PGs in a state
+where they fail to peer.
+
+Notably, upgrading a cluster involves OSD restarts and as such may
+increase the risk of encountering this bug.  For this reason, for
+clusters with erasure coded pools, we recommend the following upgrade
+procedure to minimize risk:
+
+#. Install the v12.2.7 packages.
+#. Temporarily quiesce IO to cluster::
+
+     ceph osd pause
+
+#. Restart all OSDs and wait for all PGs to become active.
+#. Resume IO::
+
+     ceph osd unpause
+
+This will cause an availability outage for the duration of the OSD
+restarts.  If this in unacceptable, an *more risky* alternative is to
+disable RGW garbage collection (the primary known cause of these rados
+operations) for the duration of the upgrade::
+
+#. Set ``rgw_enable_gc_threads = false`` in ceph.conf
+#. Restart all radosgw daemons
+#. Upgrade and restart all OSDs
+#. Remove ``rgw_enable_gc_threads = false`` from ceph.conf
+#. Restart all radosgw daemons
+
+Upgrading from other versions
+-----------------------------
+
+If your cluster did not run v12.2.5 or v12.2.6 then none of the above
+issues apply to you and you should upgrade normally.
+
+Notable Changes
+---------------
+
+* mon/AuthMonitor: improve error message (`issue#21765 <http://tracker.ceph.com/issues/21765>`_, `pr#22963 <https://github.com/ceph/ceph/pull/22963>`_, Douglas Fuller)
+* osd/PG: do not blindly roll forward to log.head (`issue#24597 <http://tracker.ceph.com/issues/24597>`_, `pr#22976 <https://github.com/ceph/ceph/pull/22976>`_, Sage Weil)
+* osd/PrimaryLogPG: rebuild attrs from clients (`issue#24768 <http://tracker.ceph.com/issues/24768>`_, `pr#22962 <https://github.com/ceph/ceph/pull/22962>`_, Sage Weil)
+* osd: work around data digest problems in 12.2.6 (version 2) (`issue#24922 <http://tracker.ceph.com/issues/24922>`_, `pr#23055 <https://github.com/ceph/ceph/pull/23055>`_, Sage Weil)
+* rgw: objects in cache never refresh after rgw_cache_expiry_interval (`issue#24346 <http://tracker.ceph.com/issues/24346>`_, `pr#22369 <https://github.com/ceph/ceph/pull/22369>`_, Casey Bodley, Matt Benjamin)
+
+
+v12.2.6 Luminous
+================
+
+:note: This is a broken release with serious known regressions.  Do not install it.
+
+This is the sixth bugfix release of Luminous v12.2.x long term stable release
+series. This release contains a range of bug fixes across all components of
+Ceph and a few security fixes.
+
+Notable Changes
+---------------
+
+- *Auth*:
+
+  * In 12.2.4 and earlier releases, keyring caps were not checked for validity,
+    so the caps string could be anything. As of 12.2.6, caps strings are
+    validated and providing a keyring with an invalid caps string to, e.g.,
+    "ceph auth add" will result in an error.
+  * CVE 2018-1128: auth: cephx authorizer subject to replay attack (`issue#24836 <http://tracker.ceph.com/issues/24836>`_, Sage Weil)
+  * CVE 2018-1129: auth: cephx signature check is weak (`issue#24837 <http://tracker.ceph.com/issues/24837>`_, Sage Weil)
+  * CVE 2018-10861: mon: auth checks not correct for pool ops (`issue#24838 <http://tracker.ceph.com/issues/24838>`_, Jason Dillaman)
+
+
+* The config-key interface can store arbitrary binary blobs but JSON
+  can only express printable strings.  If binary blobs are present,
+  the 'ceph config-key dump' command will show them as something like
+  ``<<< binary blob of length N >>>``.
+
+
+Other Notable Changes
+---------------------
+* build/ops: build-integration-branch script (`issue#24003 <http://tracker.ceph.com/issues/24003>`_, `pr#21919 <https://github.com/ceph/ceph/pull/21919>`_, Nathan Cutler, Kefu Chai, Sage Weil)
+* cephfs-journal-tool: wait prezero ops before destroying journal (`issue#20549 <http://tracker.ceph.com/issues/20549>`_, `pr#21874 <https://github.com/ceph/ceph/pull/21874>`_, "Yan, Zheng")
+* cephfs: MDSMonitor: cleanup and protect fsmap access (`issue#23762 <http://tracker.ceph.com/issues/23762>`_, `pr#21732 <https://github.com/ceph/ceph/pull/21732>`_, Patrick Donnelly)
+* cephfs: MDSMonitor: crash after assigning standby-replay daemon in multifs setup (`issue#23762 <http://tracker.ceph.com/issues/23762>`_, `issue#23658 <http://tracker.ceph.com/issues/23658>`_, `pr#22603 <https://github.com/ceph/ceph/pull/22603>`_, "Yan, Zheng")
+* cephfs: MDSMonitor: fix mds health printed in bad format (`issue#23582 <http://tracker.ceph.com/issues/23582>`_, `pr#21447 <https://github.com/ceph/ceph/pull/21447>`_, Patrick Donnelly)
+* cephfs: MDSMonitor: initialize new Filesystem epoch from pending (`issue#23764 <http://tracker.ceph.com/issues/23764>`_, `pr#21512 <https://github.com/ceph/ceph/pull/21512>`_, Patrick Donnelly)
+* ceph-fuse: missing dentries in readdir result (`issue#23894 <http://tracker.ceph.com/issues/23894>`_, `pr#22119 <https://github.com/ceph/ceph/pull/22119>`_, "Yan, Zheng")
+* ceph-fuse: return proper exit code (`issue#23665 <http://tracker.ceph.com/issues/23665>`_, `pr#21495 <https://github.com/ceph/ceph/pull/21495>`_, Patrick Donnelly)
+* ceph-fuse: trim ceph-fuse -V output (`issue#23248 <http://tracker.ceph.com/issues/23248>`_, `pr#21600 <https://github.com/ceph/ceph/pull/21600>`_, Jos Collin)
+* ceph_test_rados_api_aio: fix race with full pool and osdmap (`issue#23917 <http://tracker.ceph.com/issues/23917>`_, `issue#23876 <http://tracker.ceph.com/issues/23876>`_, `pr#21778 <https://github.com/ceph/ceph/pull/21778>`_, Sage Weil)
+* ceph-volume: error on commands that need ceph.conf to operate (`issue#23941 <http://tracker.ceph.com/issues/23941>`_, `pr#22746 <https://github.com/ceph/ceph/pull/22746>`_, Andrew Schoen)
+* ceph-volume: failed ceph-osd --mkfs command doesn't halt the OSD creation process (`issue#23874 <http://tracker.ceph.com/issues/23874>`_, `pr#21746 <https://github.com/ceph/ceph/pull/21746>`_, Alfredo Deza)
+* client: add ceph_ll_sync_inode (`issue#23291 <http://tracker.ceph.com/issues/23291>`_, `pr#21109 <https://github.com/ceph/ceph/pull/21109>`_, Jeff Layton)
+* client: add client option descriptions (`issue#22933 <http://tracker.ceph.com/issues/22933>`_, `pr#21589 <https://github.com/ceph/ceph/pull/21589>`_, Patrick Donnelly)
+* client: anchor dentries for trimming to make cap traversal safe (`issue#24137 <http://tracker.ceph.com/issues/24137>`_, `pr#22201 <https://github.com/ceph/ceph/pull/22201>`_, Patrick Donnelly)
+* client: avoid freeing inode when it contains TX buffer head (`issue#23837 <http://tracker.ceph.com/issues/23837>`_, `pr#22168 <https://github.com/ceph/ceph/pull/22168>`_, Guan yunfei, "Yan, Zheng", Jason Dillaman)
+* client: dirty caps may never get the chance to flush (`issue#22546 <http://tracker.ceph.com/issues/22546>`_, `pr#21278 <https://github.com/ceph/ceph/pull/21278>`_, dongdong tao)
+* client: fix issue of revoking non-auth caps (`issue#24172 <http://tracker.ceph.com/issues/24172>`_, `pr#22221 <https://github.com/ceph/ceph/pull/22221>`_, "Yan, Zheng")
+* client: fix request send_to_auth was never really used (`issue#23541 <http://tracker.ceph.com/issues/23541>`_, `pr#21354 <https://github.com/ceph/ceph/pull/21354>`_, Zhi Zhang)
+* client: Fix the gid_count check (`issue#23652 <http://tracker.ceph.com/issues/23652>`_, `pr#21596 <https://github.com/ceph/ceph/pull/21596>`_, Jos Collin)
+* client: flush the mdlog in _fsync before waiting on unstable reqs (`issue#23714 <http://tracker.ceph.com/issues/23714>`_, `pr#21542 <https://github.com/ceph/ceph/pull/21542>`_, Jeff Layton)
+* client: hangs on umount if it had an MDS session evicted (`issue#10915 <http://tracker.ceph.com/issues/10915>`_, `pr#22018 <https://github.com/ceph/ceph/pull/22018>`_, Rishabh Dave)
+* client: void sending mds request while holding cap reference (`issue#24369 <http://tracker.ceph.com/issues/24369>`_, `pr#22354 <https://github.com/ceph/ceph/pull/22354>`_, "Yan, Zheng")
+* cmake: fix the cepfs java binding build on Bionic (`issue#23458 <http://tracker.ceph.com/issues/23458>`_, `issue#24012 <http://tracker.ceph.com/issues/24012>`_, `pr#21872 <https://github.com/ceph/ceph/pull/21872>`_, Kefu Chai, Shengjing Zhu)
+* cmake/modules/BuildRocksDB.cmake: enable compressions for rocksdb (`issue#24025 <http://tracker.ceph.com/issues/24025>`_, `pr#22215 <https://github.com/ceph/ceph/pull/22215>`_, Kefu Chai)
+* common: ARMv8 feature detection broken, leading to illegal instruction crashes (`issue#23464 <http://tracker.ceph.com/issues/23464>`_, `pr#22567 <https://github.com/ceph/ceph/pull/22567>`_, Adam Kupczyk)
+* common: fix BoundedKeyCounter const_pointer_iterator (`issue#22139 <http://tracker.ceph.com/issues/22139>`_, `pr#21083 <https://github.com/ceph/ceph/pull/21083>`_, Casey Bodley)
+* common: fix typo in rados bench write JSON output (`issue#24199 <http://tracker.ceph.com/issues/24199>`_, `pr#22391 <https://github.com/ceph/ceph/pull/22391>`_, Sandor Zeestraten)
+* common: partially revert 95fc248 to make get_process_name work (`issue#24123 <http://tracker.ceph.com/issues/24123>`_, `pr#22290 <https://github.com/ceph/ceph/pull/22290>`_, Mykola Golub)
+* core: Deleting a pool with active notify linger ops can result in seg fault (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#22143 <https://github.com/ceph/ceph/pull/22143>`_, Kefu Chai, Jason Dillaman)
+* core: mon/MgrMonitor: change 'unresponsive' message to info level (`issue#24222 <http://tracker.ceph.com/issues/24222>`_, `pr#22331 <https://github.com/ceph/ceph/pull/22331>`_, Sage Weil)
+* core: Wip scrub omap (`issue#24366 <http://tracker.ceph.com/issues/24366>`_, `pr#22375 <https://github.com/ceph/ceph/pull/22375>`_, xie xingguo, David Zafman)
+* crush: fix device_class_clone for unpopulated/empty weight-sets (`issue#23386 <http://tracker.ceph.com/issues/23386>`_, `pr#22381 <https://github.com/ceph/ceph/pull/22381>`_, Sage Weil)
+* crush, osd: handle multiple parents properly when applying pg upmaps (`issue#23921 <http://tracker.ceph.com/issues/23921>`_, `pr#22115 <https://github.com/ceph/ceph/pull/22115>`_, xiexingguo)
+* doc: Fix -d description in ceph-fuse (`issue#23214 <http://tracker.ceph.com/issues/23214>`_, `pr#21616 <https://github.com/ceph/ceph/pull/21616>`_, Jos Collin)
+* doc:Update ceph-fuse doc (`issue#23084 <http://tracker.ceph.com/issues/23084>`_, `pr#21603 <https://github.com/ceph/ceph/pull/21603>`_, Jos Collin)
+* fuse: wire up fuse_ll_access (`issue#23509 <http://tracker.ceph.com/issues/23509>`_, `pr#21475 <https://github.com/ceph/ceph/pull/21475>`_, Jeff Layton)
+* kceph: umount on evicted client blocks forever (`issue#24053 <http://tracker.ceph.com/issues/24053>`_, `issue#24054 <http://tracker.ceph.com/issues/24054>`_, `pr#22208 <https://github.com/ceph/ceph/pull/22208>`_, Yan, Zheng, "Yan, Zheng")
+* librbd: commit IO as safe when complete if writeback cache is disabled (`issue#23516 <http://tracker.ceph.com/issues/23516>`_, `pr#22370 <https://github.com/ceph/ceph/pull/22370>`_, Jason Dillaman)
+* librbd: prevent watcher from unregistering with in-flight actions (`issue#23955 <http://tracker.ceph.com/issues/23955>`_, `pr#21938 <https://github.com/ceph/ceph/pull/21938>`_, Jason Dillaman)
+* lvm: when osd creation fails log the exception (`issue#24456 <http://tracker.ceph.com/issues/24456>`_, `pr#22641 <https://github.com/ceph/ceph/pull/22641>`_, Andrew Schoen)
+* mds: avoid calling rejoin_gather_finish() two times successively (`issue#24047 <http://tracker.ceph.com/issues/24047>`_, `pr#22171 <https://github.com/ceph/ceph/pull/22171>`_, "Yan, Zheng")
+* mds: broadcast quota to relevant clients when quota is explicitly set (`issue#24133 <http://tracker.ceph.com/issues/24133>`_, `pr#22271 <https://github.com/ceph/ceph/pull/22271>`_, Zhi Zhang)
+* mds: crash when failover (`issue#23518 <http://tracker.ceph.com/issues/23518>`_, `pr#21900 <https://github.com/ceph/ceph/pull/21900>`_, "Yan, Zheng")
+* mds: don't discover inode/dirfrag when mds is in 'starting' state (`issue#23812 <http://tracker.ceph.com/issues/23812>`_, `pr#21990 <https://github.com/ceph/ceph/pull/21990>`_, "Yan, Zheng")
+* mds: fix occasional dir rstat inconsistency between multi-MDSes (`issue#23538 <http://tracker.ceph.com/issues/23538>`_, `pr#21617 <https://github.com/ceph/ceph/pull/21617>`_, "Yan, Zheng", Zhi Zhang)
+* mds: fix some memory leak (`issue#24289 <http://tracker.ceph.com/issues/24289>`_, `pr#22310 <https://github.com/ceph/ceph/pull/22310>`_, "Yan, Zheng")
+* mds: fix unhealth heartbeat during rejoin (`issue#23530 <http://tracker.ceph.com/issues/23530>`_, `pr#21366 <https://github.com/ceph/ceph/pull/21366>`_, dongdong tao)
+* mds: handle imported session race (`issue#24072 <http://tracker.ceph.com/issues/24072>`_, `issue#24087 <http://tracker.ceph.com/issues/24087>`_, `pr#21989 <https://github.com/ceph/ceph/pull/21989>`_, Patrick Donnelly)
+* mds: include nfiles/nsubdirs of directory inode in MClientCaps (`issue#23855 <http://tracker.ceph.com/issues/23855>`_, `pr#22118 <https://github.com/ceph/ceph/pull/22118>`_, "Yan, Zheng")
+* mds: kick rdlock if waiting for dirfragtreelock (`issue#23919 <http://tracker.ceph.com/issues/23919>`_, `pr#21901 <https://github.com/ceph/ceph/pull/21901>`_, Patrick Donnelly)
+* mds: make rstat.rctime follow inodes' ctime (`issue#23380 <http://tracker.ceph.com/issues/23380>`_, `pr#21448 <https://github.com/ceph/ceph/pull/21448>`_, "Yan, Zheng")
+* mds: mark damaged if sessions' preallocated inos don't match inotable (`issue#23452 <http://tracker.ceph.com/issues/23452>`_, `pr#21372 <https://github.com/ceph/ceph/pull/21372>`_, "Yan, Zheng")
+* mds: mark new root inode dirty (`issue#23960 <http://tracker.ceph.com/issues/23960>`_, `pr#21922 <https://github.com/ceph/ceph/pull/21922>`_, Patrick Donnelly)
+* mds: mds shutdown fixes and optimization (`issue#23602 <http://tracker.ceph.com/issues/23602>`_, `pr#21346 <https://github.com/ceph/ceph/pull/21346>`_, "Yan, Zheng")
+* mds: misc load balancer fixes (`issue#21745 <http://tracker.ceph.com/issues/21745>`_, `pr#21412 <https://github.com/ceph/ceph/pull/21412>`_, "Yan, Zheng", Jianyu Li)
+* mds: properly check auth subtree count in MDCache::shutdown_pass() (`issue#23813 <http://tracker.ceph.com/issues/23813>`_, `pr#21844 <https://github.com/ceph/ceph/pull/21844>`_, "Yan, Zheng")
+* mds: properly dirty sessions opened by journal replay (`issue#23625 <http://tracker.ceph.com/issues/23625>`_, `pr#21441 <https://github.com/ceph/ceph/pull/21441>`_, "Yan, Zheng")
+* mds: properly trim log segments after scrub repairs something (`issue#23880 <http://tracker.ceph.com/issues/23880>`_, `pr#21840 <https://github.com/ceph/ceph/pull/21840>`_, "Yan, Zheng")
+* mds: set could_consume to false when no purge queue item actually exe… (`issue#24073 <http://tracker.ceph.com/issues/24073>`_, `pr#22176 <https://github.com/ceph/ceph/pull/22176>`_, Xuehan Xu)
+* mds: trim log during shutdown to clean metadata (`issue#23923 <http://tracker.ceph.com/issues/23923>`_, `pr#21899 <https://github.com/ceph/ceph/pull/21899>`_, Patrick Donnelly)
+* mds: underwater dentry check in CDir::_omap_fetched is racy (`issue#23032 <http://tracker.ceph.com/issues/23032>`_, `pr#21187 <https://github.com/ceph/ceph/pull/21187>`_, Yan, Zheng)
+* mg_read() call has wrong arguments (`issue#23596 <http://tracker.ceph.com/issues/23596>`_, `pr#21382 <https://github.com/ceph/ceph/pull/21382>`_, Nathan Cutler)
+* mgr/influx: Only split string on first occurence of dot (.) (`issue#23996 <http://tracker.ceph.com/issues/23996>`_, `pr#21965 <https://github.com/ceph/ceph/pull/21965>`_, Wido den Hollander)
+* mgr: Module 'balancer' has failed: could not find bucket -14 (`issue#24167 <http://tracker.ceph.com/issues/24167>`_, `pr#22308 <https://github.com/ceph/ceph/pull/22308>`_, Sage Weil)
+* mon: add  'ceph osd pool get erasure allow_ec_overwrites' command (`issue#23487 <http://tracker.ceph.com/issues/23487>`_, `pr#21378 <https://github.com/ceph/ceph/pull/21378>`_, Mykola Golub)
+* mon: enable level_compaction_dynamic_level_bytes for rocksdb (`issue#24361 <http://tracker.ceph.com/issues/24361>`_, `pr#22360 <https://github.com/ceph/ceph/pull/22360>`_, Kefu Chai)
+* mon: handle bad snapshot removal reqs gracefully (`issue#18746 <http://tracker.ceph.com/issues/18746>`_, `pr#21717 <https://github.com/ceph/ceph/pull/21717>`_, Paul Emmerich)
+* mon: High MON cpu usage when cluster is changing (`issue#23713 <http://tracker.ceph.com/issues/23713>`_, `pr#21968 <https://github.com/ceph/ceph/pull/21968>`_, Sage Weil, Xiaoxi CHEN)
+* mon/MDSMonitor: do not send redundant MDS health messages to cluster log (`issue#24308 <http://tracker.ceph.com/issues/24308>`_, `pr#22558 <https://github.com/ceph/ceph/pull/22558>`_, Sage Weil)
+* msg/async/AsyncConnection: Fix FPE in process_connection (`issue#23618 <http://tracker.ceph.com/issues/23618>`_, `pr#21376 <https://github.com/ceph/ceph/pull/21376>`_, Brad Hubbard)
+* os/bluestore: alter the allow_eio policy regarding kernel's error list (`issue#23333 <http://tracker.ceph.com/issues/23333>`_, `pr#21405 <https://github.com/ceph/ceph/pull/21405>`_, Radoslaw Zarzynski)
+* os/bluestore/bluefs_types: make block_mask 64-bit (`issue#23840 <http://tracker.ceph.com/issues/23840>`_, `pr#21740 <https://github.com/ceph/ceph/pull/21740>`_, Sage Weil)
+* os/bluestore: fix exceeding the max IO queue depth in KernelDevice (`issue#23246 <http://tracker.ceph.com/issues/23246>`_, `pr#21407 <https://github.com/ceph/ceph/pull/21407>`_, Radoslaw Zarzynski)
+* os/bluestore: fix SharedBlobSet refcounting race (`issue#24319 <http://tracker.ceph.com/issues/24319>`_, `pr#22650 <https://github.com/ceph/ceph/pull/22650>`_, Sage Weil)
+* os/bluestore: simplify and fix SharedBlob::put() (`issue#24211 <http://tracker.ceph.com/issues/24211>`_, `pr#22351 <https://github.com/ceph/ceph/pull/22351>`_, Sage Weil)
+* osdc/Objecter: fix recursive locking in _finish_command (`issue#23940 <http://tracker.ceph.com/issues/23940>`_, `pr#21939 <https://github.com/ceph/ceph/pull/21939>`_, Sage Weil)
+* osdc/Objecter: prevent double-invocation of linger op callback (`issue#23872 <http://tracker.ceph.com/issues/23872>`_, `pr#21752 <https://github.com/ceph/ceph/pull/21752>`_, Jason Dillaman)
+* osd: do not crash on empty snapset (`issue#23851 <http://tracker.ceph.com/issues/23851>`_, `pr#21638 <https://github.com/ceph/ceph/pull/21638>`_, Mykola Golub, Igor Fedotov)
+* osd: Don't evict even when preemption has restarted with smaller chunk (`issue#22881 <http://tracker.ceph.com/issues/22881>`_, `issue#23909 <http://tracker.ceph.com/issues/23909>`_, `issue#23646 <http://tracker.ceph.com/issues/23646>`_, `pr#22044 <https://github.com/ceph/ceph/pull/22044>`_, Sage Weil, fang yuxiang, Jianpeng Ma, kungf, xie xingguo, David Zafman)
+* osd/ECBackend: only check required shards when finishing recovery reads (`issue#23195 <http://tracker.ceph.com/issues/23195>`_, `pr#21911 <https://github.com/ceph/ceph/pull/21911>`_, Josh Durgin, Kefu Chai)
+* osd: increase default hard pg limit (`issue#24243 <http://tracker.ceph.com/issues/24243>`_, `pr#22592 <https://github.com/ceph/ceph/pull/22592>`_, Josh Durgin)
+* osd/OSDMap: check against cluster topology changing before applying pg upmaps (`issue#23878 <http://tracker.ceph.com/issues/23878>`_, `pr#21818 <https://github.com/ceph/ceph/pull/21818>`_, xiexingguo)
+* osd/PG: fix DeferRecovery vs AllReplicasRecovered race (`issue#23860 <http://tracker.ceph.com/issues/23860>`_, `pr#21964 <https://github.com/ceph/ceph/pull/21964>`_, Sage Weil)
+* osd/PG: fix uninit read in Incomplete::react(AdvMap&) (`issue#23980 <http://tracker.ceph.com/issues/23980>`_, `pr#21993 <https://github.com/ceph/ceph/pull/21993>`_, Sage Weil)
+* osd/PrimaryLogPG: avoid infinite loop when flush collides with write … (`issue#23664 <http://tracker.ceph.com/issues/23664>`_, `pr#21764 <https://github.com/ceph/ceph/pull/21764>`_, Sage Weil)
+* osd: publish osdmap to OSDService before starting wq threads (`issue#21977 <http://tracker.ceph.com/issues/21977>`_, `pr#21737 <https://github.com/ceph/ceph/pull/21737>`_, Sage Weil)
+* osd: Warn about objects with too many omap entries (`issue#23784 <http://tracker.ceph.com/issues/23784>`_, `pr#21518 <https://github.com/ceph/ceph/pull/21518>`_, Brad Hubbard)
+* qa: disable -Werror when compiling env_librados_test (`issue#23786 <http://tracker.ceph.com/issues/23786>`_, `pr#21655 <https://github.com/ceph/ceph/pull/21655>`_, Kefu Chai)
+* qa: fix blacklisted check for test_lifecycle (`issue#23975 <http://tracker.ceph.com/issues/23975>`_, `pr#21921 <https://github.com/ceph/ceph/pull/21921>`_, Patrick Donnelly)
+* qa: remove racy/buggy test_purge_queue_op_rate (`issue#23829 <http://tracker.ceph.com/issues/23829>`_, `pr#21841 <https://github.com/ceph/ceph/pull/21841>`_, Patrick Donnelly)
+* qa/suites/rbd/basic/msgr-failures: remove many.yaml (`issue#23789 <http://tracker.ceph.com/issues/23789>`_, `pr#22128 <https://github.com/ceph/ceph/pull/22128>`_, Sage Weil)
+* qa: wait longer for osd to flush pg stats (`issue#24321 <http://tracker.ceph.com/issues/24321>`_, `pr#22296 <https://github.com/ceph/ceph/pull/22296>`_, Kefu Chai)
+* qa/workunits/mon/test_mon_config_key.py fails on master (`issue#23622 <http://tracker.ceph.com/issues/23622>`_, `pr#21368 <https://github.com/ceph/ceph/pull/21368>`_, Sage Weil)
+* qa/workunits/rbd: adapt import_export test to handle multiple units (`issue#24733 <http://tracker.ceph.com/issues/24733>`_, `pr#22911 <https://github.com/ceph/ceph/pull/22911>`_, Jason Dillaman)
+* qa/workunits/rbd: potential race in mirror disconnect test (`issue#23938 <http://tracker.ceph.com/issues/23938>`_, `pr#21869 <https://github.com/ceph/ceph/pull/21869>`_, Mykola Golub)
+* radosgw-admin sync status improvements (`issue#20473 <http://tracker.ceph.com/issues/20473>`_, `pr#21908 <https://github.com/ceph/ceph/pull/21908>`_, lvshanchun, Casey Bodley)
+* rbd: improve 'import-diff' corrupt input error messages (`issue#18844 <http://tracker.ceph.com/issues/18844>`_, `issue#23038 <http://tracker.ceph.com/issues/23038>`_, `pr#21316 <https://github.com/ceph/ceph/pull/21316>`_, PCzhangPC, songweibin, Jason Dillaman)
+* rbd-mirror: ensure remote demotion is replayed locally (`issue#24009 <http://tracker.ceph.com/issues/24009>`_, `pr#22142 <https://github.com/ceph/ceph/pull/22142>`_, Jason Dillaman)
+* rbd-nbd can deadlock in logging thread (`issue#23143 <http://tracker.ceph.com/issues/23143>`_, `pr#21705 <https://github.com/ceph/ceph/pull/21705>`_, Sage Weil)
+* rbd: python bindings fixes and improvements (`issue#23609 <http://tracker.ceph.com/issues/23609>`_, `pr#21725 <https://github.com/ceph/ceph/pull/21725>`_, Ricardo Dias)
+* rbd: [rbd-mirror] asok hook for image replayer not re-registered after bootstrap (`issue#23888 <http://tracker.ceph.com/issues/23888>`_, `pr#21726 <https://github.com/ceph/ceph/pull/21726>`_, Jason Dillaman)
+* rbd: [rbd-mirror] local tag predecessor mirror uuid is incorrectly replaced with remote (`issue#23876 <http://tracker.ceph.com/issues/23876>`_, `pr#21741 <https://github.com/ceph/ceph/pull/21741>`_, Jason Dillaman)
+* rbd: [rbd-mirror] potential deadlock when running asok 'flush' command (`issue#24141 <http://tracker.ceph.com/issues/24141>`_, `pr#22180 <https://github.com/ceph/ceph/pull/22180>`_, Mykola Golub)
+* rbd: [rbd-mirror] potential races during PoolReplayer shut-down (`issue#24008 <http://tracker.ceph.com/issues/24008>`_, `pr#22172 <https://github.com/ceph/ceph/pull/22172>`_, Jason Dillaman)
+* rgw: add buffering filter to compression for fetch_remote_obj (`issue#23547 <http://tracker.ceph.com/issues/23547>`_, `pr#21758 <https://github.com/ceph/ceph/pull/21758>`_, Casey Bodley)
+* rgw: add configurable AWS-compat invalid range get behavior (`issue#24317 <http://tracker.ceph.com/issues/24317>`_, `pr#22302 <https://github.com/ceph/ceph/pull/22302>`_, Matt Benjamin)
+* rgw: admin rest api shouldn't return error when getting user's stats if (`issue#23821 <http://tracker.ceph.com/issues/23821>`_, `pr#21661 <https://github.com/ceph/ceph/pull/21661>`_, Zhang Shaowen)
+* rgw: Allow swift acls to be deleted (`issue#22897 <http://tracker.ceph.com/issues/22897>`_, `pr#22465 <https://github.com/ceph/ceph/pull/22465>`_, Marcus Watts)
+* rgw: aws4 auth supports PutBucketRequestPayment (`issue#23803 <http://tracker.ceph.com/issues/23803>`_, `pr#21660 <https://github.com/ceph/ceph/pull/21660>`_, Casey Bodley)
+* rgw: beast frontend can listen on multiple endpoints (`issue#22779 <http://tracker.ceph.com/issues/22779>`_, `pr#21568 <https://github.com/ceph/ceph/pull/21568>`_, Casey Bodley)
+* rgw: Bucket lifecycles stick around after buckets are deleted (`issue#19632 <http://tracker.ceph.com/issues/19632>`_, `pr#22551 <https://github.com/ceph/ceph/pull/22551>`_, Wei Qiaomiao)
+* rgw: Do not modify email if argument is not set (`issue#24142 <http://tracker.ceph.com/issues/24142>`_, `pr#22352 <https://github.com/ceph/ceph/pull/22352>`_, Volker Theile)
+* rgw: do not reflect period if not current (`issue#22844 <http://tracker.ceph.com/issues/22844>`_, `pr#21735 <https://github.com/ceph/ceph/pull/21735>`_, Tianshan Qu)
+* rgw: es module: set compression type correctly (`issue#22758 <http://tracker.ceph.com/issues/22758>`_, `pr#21736 <https://github.com/ceph/ceph/pull/21736>`_, Abhishek Lekshmanan)
+* rgw_file: conditionally unlink handles when direct deleted (`issue#23299 <http://tracker.ceph.com/issues/23299>`_, `pr#21438 <https://github.com/ceph/ceph/pull/21438>`_, Matt Benjamin)
+* rgw: fix bi_list to reset is_truncated flag if it skips entires (`issue#22721 <http://tracker.ceph.com/issues/22721>`_, `pr#21669 <https://github.com/ceph/ceph/pull/21669>`_, Orit Wasserman)
+* rgw: fix 'copy part' without 'x-amz-copy-source-range' when compressi… (`issue#23196 <http://tracker.ceph.com/issues/23196>`_, `pr#22438 <https://github.com/ceph/ceph/pull/22438>`_, fang yuxiang)
+* rgw: fix error handling for GET with ?torrent (`issue#23506 <http://tracker.ceph.com/issues/23506>`_, `pr#21674 <https://github.com/ceph/ceph/pull/21674>`_, Casey Bodley)
+* rgw: fix use of libcurl with empty header values (`issue#23663 <http://tracker.ceph.com/issues/23663>`_, `pr#21738 <https://github.com/ceph/ceph/pull/21738>`_, Casey Bodley)
+* rgw:lc: RGWPutLC return ERR_MALFORMED_XML  when missing <Rule> tag in… (`issue#21377 <http://tracker.ceph.com/issues/21377>`_, `pr#19884 <https://github.com/ceph/ceph/pull/19884>`_, Shasha Lu)
+* rgw: making implicit_tenants backwards compatible (`issue#24348 <http://tracker.ceph.com/issues/24348>`_, `pr#22363 <https://github.com/ceph/ceph/pull/22363>`_, Marcus Watts)
+* rgw: Misnamed S3 operation (`issue#24061 <http://tracker.ceph.com/issues/24061>`_, `pr#21917 <https://github.com/ceph/ceph/pull/21917>`_, xiangxiang)
+* rgw: move all pool creation into rgw_init_ioctx (`issue#23480 <http://tracker.ceph.com/issues/23480>`_, `pr#21675 <https://github.com/ceph/ceph/pull/21675>`_, Casey Bodley)
+* rgw: radosgw-admin should not use metadata cache for readonly commands (`issue#23468 <http://tracker.ceph.com/issues/23468>`_, `pr#21437 <https://github.com/ceph/ceph/pull/21437>`_, Orit Wasserman)
+* rgw: raise log level on coroutine shutdown errors (`issue#23974 <http://tracker.ceph.com/issues/23974>`_, `pr#21792 <https://github.com/ceph/ceph/pull/21792>`_, Casey Bodley)
+* rgw: return EINVAL if max_keys can not convert correctly (`issue#23586 <http://tracker.ceph.com/issues/23586>`_, `pr#21435 <https://github.com/ceph/ceph/pull/21435>`_, yuliyang)
+* rgw: rgw_statfs should report the correct stats (`issue#22202 <http://tracker.ceph.com/issues/22202>`_, `pr#21724 <https://github.com/ceph/ceph/pull/21724>`_, Supriti Singh)
+* rgw: trim all spaces inside a metadata value (`issue#23301 <http://tracker.ceph.com/issues/23301>`_, `pr#22177 <https://github.com/ceph/ceph/pull/22177>`_, Orit Wasserman)
+* slow mon ops from osd_failure (`issue#24322 <http://tracker.ceph.com/issues/24322>`_, `pr#22568 <https://github.com/ceph/ceph/pull/22568>`_, Sage Weil)
+* table of contents doesn't render for luminous/jewel docs (`issue#23780 <http://tracker.ceph.com/issues/23780>`_, `pr#21502 <https://github.com/ceph/ceph/pull/21502>`_, Alfredo Deza)
+* test/librados: increase pgp_num along with pg_num (`issue#23763 <http://tracker.ceph.com/issues/23763>`_, `pr#21556 <https://github.com/ceph/ceph/pull/21556>`_, Kefu Chai)
+* test/rgw: fix for bucket checkpoints (`issue#24212 <http://tracker.ceph.com/issues/24212>`_, `pr#22541 <https://github.com/ceph/ceph/pull/22541>`_, Casey Bodley)
+* tests: filestore journal replay does not guard omap operations (`issue#22920 <http://tracker.ceph.com/issues/22920>`_, `pr#21547 <https://github.com/ceph/ceph/pull/21547>`_, Sage Weil)
+* tools: ceph-disk: write log to /var/log/ceph not to /var/run/ceph (`issue#24041 <http://tracker.ceph.com/issues/24041>`_, `pr#21870 <https://github.com/ceph/ceph/pull/21870>`_, Kefu Chai)
+* tools: ceph-fuse: getgroups failure causes exception (`issue#23446 <http://tracker.ceph.com/issues/23446>`_, `pr#21687 <https://github.com/ceph/ceph/pull/21687>`_, Jeff Layton)
+
+
+v12.2.5 Luminous
+================
+
+This is the fifth bugfix release of Luminous v12.2.x long term stable
+release series. This release contains a range of bug fixes across all
+components of Ceph. We recommend all the users of 12.2.x series to
+update.
+
+Notable Changes
+---------------
+
+* MGR
+
+  The ceph-rest-api command-line tool included in the ceph-mon
+  package has been obsoleted by the MGR "restful" module. The
+  ceph-rest-api tool is hereby declared deprecated and will be dropped
+  in Mimic.
+
+  The MGR "restful" module provides similar functionality via a "pass through"
+  method. See http://docs.ceph.com/docs/luminous/mgr/restful for details.
+
+* CephFS
+
+  Upgrading an MDS cluster to 12.2.3+ will result in all active MDS
+  exiting due to feature incompatibilities once an upgraded MDS comes
+  online (even as standby). Operators may ignore the error messages
+  and continue upgrading/restarting or follow this upgrade sequence:
+
+  Reduce the number of ranks to 1 (`ceph fs set <fs_name> max_mds 1`),
+  wait for all other MDS to deactivate, leaving the one active MDS,
+  upgrade the single active MDS, then upgrade/start standbys. Finally,
+  restore the previous max_mds.
+
+  See also: https://tracker.ceph.com/issues/23172
+
+
+Other Notable Changes
+---------------------
+
+* add --add-bucket and --move options to crushtool (`issue#23472 <http://tracker.ceph.com/issues/23472>`_, `issue#23471 <http://tracker.ceph.com/issues/23471>`_, `pr#21079 <https://github.com/ceph/ceph/pull/21079>`_, Kefu Chai)
+* BlueStore.cc: _balance_bluefs_freespace: assert(0 == "allocate failed, wtf") (`issue#23063 <http://tracker.ceph.com/issues/23063>`_, `pr#21394 <https://github.com/ceph/ceph/pull/21394>`_, Igor Fedotov, xie xingguo, Sage Weil, Zac Medico)
+* bluestore: correctly check all block devices to decide if journal is\_… (`issue#23173 <http://tracker.ceph.com/issues/23173>`_, `issue#23141 <http://tracker.ceph.com/issues/23141>`_, `pr#20651 <https://github.com/ceph/ceph/pull/20651>`_, Greg Farnum)
+* bluestore: statfs available can go negative (`issue#23074 <http://tracker.ceph.com/issues/23074>`_, `pr#20554 <https://github.com/ceph/ceph/pull/20554>`_, Igor Fedotov, Sage Weil)
+* build Debian installation packages failure (`issue#22856 <http://tracker.ceph.com/issues/22856>`_, `issue#22828 <http://tracker.ceph.com/issues/22828>`_, `pr#20250 <https://github.com/ceph/ceph/pull/20250>`_, Tone Zhang)
+* build/ops: deb: move python-jinja2 dependency to mgr (`issue#22457 <http://tracker.ceph.com/issues/22457>`_, `pr#20748 <https://github.com/ceph/ceph/pull/20748>`_, Nathan Cutler)
+* build/ops: deb: move python-jinja2 dependency to mgr (`issue#22457 <http://tracker.ceph.com/issues/22457>`_, `pr#21233 <https://github.com/ceph/ceph/pull/21233>`_, Nathan Cutler)
+* build/ops: run-make-check.sh: fix SUSE support (`issue#22875 <http://tracker.ceph.com/issues/22875>`_, `issue#23178 <http://tracker.ceph.com/issues/23178>`_, `pr#20737 <https://github.com/ceph/ceph/pull/20737>`_, Nathan Cutler)
+* cephfs-journal-tool: Fix Dumper destroyed before shutdown (`issue#22862 <http://tracker.ceph.com/issues/22862>`_, `issue#22734 <http://tracker.ceph.com/issues/22734>`_, `pr#20251 <https://github.com/ceph/ceph/pull/20251>`_, dongdong tao)
+* ceph.in: print all matched commands if arg missing (`issue#22344 <http://tracker.ceph.com/issues/22344>`_, `issue#23186 <http://tracker.ceph.com/issues/23186>`_, `pr#20664 <https://github.com/ceph/ceph/pull/20664>`_, Luo Kexue, Kefu Chai)
+* ceph-objectstore-tool command to trim the pg log (`issue#23242 <http://tracker.ceph.com/issues/23242>`_, `pr#20803 <https://github.com/ceph/ceph/pull/20803>`_, Josh Durgin, David Zafman)
+* ceph osd force-create-pg cause all ceph-mon to crash and unable to come up again (`issue#22942 <http://tracker.ceph.com/issues/22942>`_, `pr#20399 <https://github.com/ceph/ceph/pull/20399>`_, Sage Weil)
+* ceph-volume: adds raw device support to 'lvm list' (`issue#23140 <http://tracker.ceph.com/issues/23140>`_, `pr#20647 <https://github.com/ceph/ceph/pull/20647>`_, Andrew Schoen)
+* ceph-volume: allow parallel creates (`issue#23757 <http://tracker.ceph.com/issues/23757>`_, `pr#21509 <https://github.com/ceph/ceph/pull/21509>`_, Theofilos Mouratidis)
+* ceph-volume: allow skipping systemd interactions on activate/create (`issue#23678 <http://tracker.ceph.com/issues/23678>`_, `pr#21538 <https://github.com/ceph/ceph/pull/21538>`_, Alfredo Deza)
+* ceph-volume: automatic VDO detection (`issue#23581 <http://tracker.ceph.com/issues/23581>`_, `pr#21505 <https://github.com/ceph/ceph/pull/21505>`_, Alfredo Deza)
+* ceph-volume be resilient to $PATH issues (`pr#20716 <https://github.com/ceph/ceph/pull/20716>`_, Alfredo Deza)
+* ceph-volume: fix action plugins path in tox (`pr#20923 <https://github.com/ceph/ceph/pull/20923>`_, Guillaume Abrioux)
+* ceph-volume Implement an 'activate all' to help with dense servers or migrating OSDs (`pr#21533 <https://github.com/ceph/ceph/pull/21533>`_, Alfredo Deza)
+* ceph-volume improve robustness when reloading vms in tests (`pr#21072 <https://github.com/ceph/ceph/pull/21072>`_, Alfredo Deza)
+* ceph-volume lvm.activate error if no bluestore OSDs are found (`issue#23644 <http://tracker.ceph.com/issues/23644>`_, `pr#21335 <https://github.com/ceph/ceph/pull/21335>`_, Alfredo Deza)
+* ceph-volume: Nits noticed while studying code (`pr#21565 <https://github.com/ceph/ceph/pull/21565>`_, Dan Mick)
+* ceph-volume tests alleviate libvirt timeouts when reloading (`issue#23163 <http://tracker.ceph.com/issues/23163>`_, `pr#20754 <https://github.com/ceph/ceph/pull/20754>`_, Alfredo Deza)
+* ceph-volume update man page for prepare/activate flags (`pr#21574 <https://github.com/ceph/ceph/pull/21574>`_, Alfredo Deza)
+* ceph-volume: Using --readonly for {vg|pv|lv}s commands (`pr#21519 <https://github.com/ceph/ceph/pull/21519>`_, Erwan Velu)
+* client: allow client to use caps that are revoked but not yet returned (`issue#23028 <http://tracker.ceph.com/issues/23028>`_, `issue#23314 <http://tracker.ceph.com/issues/23314>`_, `pr#20904 <https://github.com/ceph/ceph/pull/20904>`_, Jeff Layton)
+* : Client:Fix readdir bug (`issue#22936 <http://tracker.ceph.com/issues/22936>`_, `pr#20356 <https://github.com/ceph/ceph/pull/20356>`_, dongdong tao)
+* client: release revoking Fc after invalidate cache (`issue#22652 <http://tracker.ceph.com/issues/22652>`_, `pr#20342 <https://github.com/ceph/ceph/pull/20342>`_, "Yan, Zheng")
+* Client: setattr should drop "Fs" rather than "As" for mtime and size (`issue#22935 <http://tracker.ceph.com/issues/22935>`_, `pr#20354 <https://github.com/ceph/ceph/pull/20354>`_, dongdong tao)
+* client: use either dentry_invalidate_cb or remount_cb to invalidate k… (`issue#23355 <http://tracker.ceph.com/issues/23355>`_, `pr#20960 <https://github.com/ceph/ceph/pull/20960>`_, Zhi Zhang)
+* cls/rbd: group_image_list incorrectly flagged as RW (`issue#23407 <http://tracker.ceph.com/issues/23407>`_, `issue#23388 <http://tracker.ceph.com/issues/23388>`_, `pr#20967 <https://github.com/ceph/ceph/pull/20967>`_, Jason Dillaman)
+* cls/rgw: fix bi_log_iterate_entries return wrong truncated (`issue#22737 <http://tracker.ceph.com/issues/22737>`_, `issue#23225 <http://tracker.ceph.com/issues/23225>`_, `pr#21054 <https://github.com/ceph/ceph/pull/21054>`_, Tianshan Qu)
+* cmake: rbd resource agent needs to be executable (`issue#22980 <http://tracker.ceph.com/issues/22980>`_, `pr#20617 <https://github.com/ceph/ceph/pull/20617>`_, Tim Bishop)
+* common/dns_resolv.cc: Query for AAAA-record if ms_bind_ipv6 is True (`issue#23078 <http://tracker.ceph.com/issues/23078>`_, `issue#23174 <http://tracker.ceph.com/issues/23174>`_, `pr#20710 <https://github.com/ceph/ceph/pull/20710>`_, Wido den Hollander)
+* common/ipaddr: Do not select link-local IPv6 addresses (`issue#21813 <http://tracker.ceph.com/issues/21813>`_, `pr#21111 <https://github.com/ceph/ceph/pull/21111>`_, Willem Jan Withagen)
+* common: omit short option for id in help for clients (`issue#23156 <http://tracker.ceph.com/issues/23156>`_, `issue#23041 <http://tracker.ceph.com/issues/23041>`_, `pr#20654 <https://github.com/ceph/ceph/pull/20654>`_, Patrick Donnelly)
+* common: should not check for VERSION_ID (`issue#23477 <http://tracker.ceph.com/issues/23477>`_, `issue#23478 <http://tracker.ceph.com/issues/23478>`_, `pr#21090 <https://github.com/ceph/ceph/pull/21090>`_, Kefu Chai, Shengjing Zhu)
+* config: Change bluestore_cache_kv_max to type INT64 (`pr#20334 <https://github.com/ceph/ceph/pull/20334>`_, Zhi Zhang)
+* Couldn't init storage provider (RADOS) (`issue#23349 <http://tracker.ceph.com/issues/23349>`_, `issue#22351 <http://tracker.ceph.com/issues/22351>`_, `pr#20896 <https://github.com/ceph/ceph/pull/20896>`_, Brad Hubbard)
+* doc: Add missing pg states from doc (`issue#23113 <http://tracker.ceph.com/issues/23113>`_, `pr#20584 <https://github.com/ceph/ceph/pull/20584>`_, David Zafman)
+* doc: outline upgrade procedure for mds cluster (`issue#23634 <http://tracker.ceph.com/issues/23634>`_, `issue#23568 <http://tracker.ceph.com/issues/23568>`_, `pr#21352 <https://github.com/ceph/ceph/pull/21352>`_, Patrick Donnelly)
+* doc/rgw: add page for http frontend configuration (`issue#13523 <http://tracker.ceph.com/issues/13523>`_, `issue#22884 <http://tracker.ceph.com/issues/22884>`_, `pr#20242 <https://github.com/ceph/ceph/pull/20242>`_, Casey Bodley)
+* doc: rgw: mention the civetweb support for binding to multiple ports (`issue#20942 <http://tracker.ceph.com/issues/20942>`_, `issue#23317 <http://tracker.ceph.com/issues/23317>`_, `pr#20906 <https://github.com/ceph/ceph/pull/20906>`_, Abhishek Lekshmanan)
+* docs fix ceph-volume missing sub-commands (`pr#20691 <https://github.com/ceph/ceph/pull/20691>`_, Katie Holly, Yao Zongyou, David Galloway, Sage Weil, Alfredo Deza)
+* doc: update man page to explain ceph-volume support bluestore (`issue#23142 <http://tracker.ceph.com/issues/23142>`_, `issue#22663 <http://tracker.ceph.com/issues/22663>`_, `pr#20679 <https://github.com/ceph/ceph/pull/20679>`_, lijing)
+* Double free in rados_getxattrs_next (`issue#22940 <http://tracker.ceph.com/issues/22940>`_, `issue#22042 <http://tracker.ceph.com/issues/22042>`_, `pr#20358 <https://github.com/ceph/ceph/pull/20358>`_, Gu Zhongyan)
+* fixes for openssl & libcurl (`issue#23239 <http://tracker.ceph.com/issues/23239>`_, `issue#23245 <http://tracker.ceph.com/issues/23245>`_, `issue#22951 <http://tracker.ceph.com/issues/22951>`_, `issue#23221 <http://tracker.ceph.com/issues/23221>`_, `issue#23203 <http://tracker.ceph.com/issues/23203>`_, `pr#20722 <https://github.com/ceph/ceph/pull/20722>`_, Marcus Watts, Abhishek Lekshmanan, Jesse Williamson)
+* invalid JSON returned when querying pool parameters (`issue#23312 <http://tracker.ceph.com/issues/23312>`_, `issue#23200 <http://tracker.ceph.com/issues/23200>`_, `pr#20890 <https://github.com/ceph/ceph/pull/20890>`_, Chang Liu)
+* is_qemu_running in qemu_rebuild_object_map.sh and qemu_dynamic_features.sh may return false positive (`issue#23524 <http://tracker.ceph.com/issues/23524>`_, `pr#21192 <https://github.com/ceph/ceph/pull/21192>`_, Mykola Golub)
+* [journal] allocating a new tag after acquiring the lock should use on-disk committed position (`issue#23011 <http://tracker.ceph.com/issues/23011>`_, `issue#22945 <http://tracker.ceph.com/issues/22945>`_, `pr#20454 <https://github.com/ceph/ceph/pull/20454>`_, Jason Dillaman)
+* journal: Message too long error when appending journal (`issue#23545 <http://tracker.ceph.com/issues/23545>`_, `issue#23526 <http://tracker.ceph.com/issues/23526>`_, `pr#21216 <https://github.com/ceph/ceph/pull/21216>`_, Mykola Golub)
+* legal: remove doc license ambiguity (`issue#23410 <http://tracker.ceph.com/issues/23410>`_, `issue#23336 <http://tracker.ceph.com/issues/23336>`_, `pr#20988 <https://github.com/ceph/ceph/pull/20988>`_, Nathan Cutler)
+* librados: make OPERATION_FULL_FORCE the default for rados_remove() (`issue#23114 <http://tracker.ceph.com/issues/23114>`_, `issue#22413 <http://tracker.ceph.com/issues/22413>`_, `pr#20585 <https://github.com/ceph/ceph/pull/20585>`_, Kefu Chai)
+* librados/snap_set_diff: don't assert on empty snapset (`issue#23423 <http://tracker.ceph.com/issues/23423>`_, `pr#20991 <https://github.com/ceph/ceph/pull/20991>`_, Mykola Golub)
+* librbd: potential crash if object map check encounters error (`issue#22857 <http://tracker.ceph.com/issues/22857>`_, `issue#22819 <http://tracker.ceph.com/issues/22819>`_, `pr#20253 <https://github.com/ceph/ceph/pull/20253>`_, Jason Dillaman)
+* log: Fix AddressSanitizer: new-delete-type-mismatch (`issue#23324 <http://tracker.ceph.com/issues/23324>`_, `issue#23412 <http://tracker.ceph.com/issues/23412>`_, `pr#20998 <https://github.com/ceph/ceph/pull/20998>`_, Brad Hubbard)
+* mds: add uptime to MDS status (`issue#23150 <http://tracker.ceph.com/issues/23150>`_, `pr#20626 <https://github.com/ceph/ceph/pull/20626>`_, Patrick Donnelly)
+* mds: FAILED assert (p != active_requests.end()) in MDRequestRef MDCache::request_get(metareqid_t) (`issue#23154 <http://tracker.ceph.com/issues/23154>`_, `issue#23059 <http://tracker.ceph.com/issues/23059>`_, `pr#21176 <https://github.com/ceph/ceph/pull/21176>`_, "Yan, Zheng")
+* mds: fix session reference leak (`issue#22821 <http://tracker.ceph.com/issues/22821>`_, `issue#22969 <http://tracker.ceph.com/issues/22969>`_, `pr#20432 <https://github.com/ceph/ceph/pull/20432>`_, "Yan, Zheng")
+* mds: optimize getattr file size (`issue#23013 <http://tracker.ceph.com/issues/23013>`_, `issue#22925 <http://tracker.ceph.com/issues/22925>`_, `pr#20455 <https://github.com/ceph/ceph/pull/20455>`_, "Yan, Zheng")
+* mgr: Backport recent prometheus exporter changes (`pr#20642 <https://github.com/ceph/ceph/pull/20642>`_, Jan Fajerski, Boris Ranto)
+* mgr: Backport recent prometheus rgw changes (`pr#21492 <https://github.com/ceph/ceph/pull/21492>`_, Jan Fajerski, John Spray, Boris Ranto, Rubab-Syed)
+* mgr/balancer: pool-specific optimization support and bug fixes (`pr#20359 <https://github.com/ceph/ceph/pull/20359>`_, xie xingguo)
+* mgr: die on bind() failure (`issue#23175 <http://tracker.ceph.com/issues/23175>`_, `pr#20712 <https://github.com/ceph/ceph/pull/20712>`_, John Spray)
+* mgr: fix MSG_MGR_MAP handling (`issue#23409 <http://tracker.ceph.com/issues/23409>`_, `pr#20973 <https://github.com/ceph/ceph/pull/20973>`_, Gu Zhongyan)
+* mgr: prometheus: fix PG state names (`pr#21365 <https://github.com/ceph/ceph/pull/21365>`_, John Spray)
+* mgr: prometheus: set metadata metrics value to '1' (#22717) (`pr#20254 <https://github.com/ceph/ceph/pull/20254>`_, Konstantin Shalygin)
+* mgr: quieten logging on missing OSD stats (`issue#23224 <http://tracker.ceph.com/issues/23224>`_, `pr#21053 <https://github.com/ceph/ceph/pull/21053>`_, John Spray)
+* mgr/zabbix: Backports to Luminous (`pr#20781 <https://github.com/ceph/ceph/pull/20781>`_, Wido den Hollander)
+* mon: allow removal of tier of ec overwritable pool (`issue#22971 <http://tracker.ceph.com/issues/22971>`_, `issue#22754 <http://tracker.ceph.com/issues/22754>`_, `pr#20433 <https://github.com/ceph/ceph/pull/20433>`_, Patrick Donnelly)
+* mon: ops get stuck in "resend forwarded message to leader" (`issue#22114 <http://tracker.ceph.com/issues/22114>`_, `issue#23077 <http://tracker.ceph.com/issues/23077>`_, `pr#21016 <https://github.com/ceph/ceph/pull/21016>`_, Kefu Chai, Greg Farnum)
+* mon, osd: fix potential collided \*Up Set\* after PG remapping (`issue#23118 <http://tracker.ceph.com/issues/23118>`_, `pr#20829 <https://github.com/ceph/ceph/pull/20829>`_, xie xingguo)
+* mon/OSDMonitor.cc: fix expected_num_objects interpret error (`issue#22530 <http://tracker.ceph.com/issues/22530>`_, `issue#23315 <http://tracker.ceph.com/issues/23315>`_, `pr#20907 <https://github.com/ceph/ceph/pull/20907>`_, Yang Honggang)
+* mon: update PaxosService::cached_first_committed in PaxosService::may… (`issue#23626 <http://tracker.ceph.com/issues/23626>`_, `issue#11332 <http://tracker.ceph.com/issues/11332>`_, `pr#21328 <https://github.com/ceph/ceph/pull/21328>`_, Xuehan Xu, yupeng chen)
+* msg/async: size of EventCenter::file_events should be greater than fd (`issue#23253 <http://tracker.ceph.com/issues/23253>`_, `issue#23306 <http://tracker.ceph.com/issues/23306>`_, `pr#20867 <https://github.com/ceph/ceph/pull/20867>`_, Yupeng Chen)
+* Objecter: add ignore overlay flag if got redirect reply (`pr#20766 <https://github.com/ceph/ceph/pull/20766>`_, Ting Yi Lin)
+* os/bluestore: avoid overhead of std::function in blob_t (`pr#20674 <https://github.com/ceph/ceph/pull/20674>`_, Radoslaw Zarzynski)
+* os/bluestore: avoid unneeded BlobRefing in _do_read() (`pr#20675 <https://github.com/ceph/ceph/pull/20675>`_, Radoslaw Zarzynski)
+* os/bluestore: backport fixes around _reap_collection (`pr#20964 <https://github.com/ceph/ceph/pull/20964>`_, Jianpeng Ma)
+* os/bluestore: change the type of aio_t:res to long (`issue#23527 <http://tracker.ceph.com/issues/23527>`_, `issue#23544 <http://tracker.ceph.com/issues/23544>`_, `pr#21231 <https://github.com/ceph/ceph/pull/21231>`_, kungf)
+* os/bluestore: _dump_onode() don't prolongate Onode anymore (`pr#20676 <https://github.com/ceph/ceph/pull/20676>`_, Radoslaw Zarzynski)
+* os/bluestore: recalc_allocated() when decoding bluefs_fnode_t (`issue#23256 <http://tracker.ceph.com/issues/23256>`_, `issue#23212 <http://tracker.ceph.com/issues/23212>`_, `pr#20771 <https://github.com/ceph/ceph/pull/20771>`_, Jianpeng Ma, Igor Fedotov, Kefu Chai)
+* os/bluestore: trim cache every 50ms (instead of 200ms) (`issue#23226 <http://tracker.ceph.com/issues/23226>`_, `pr#21059 <https://github.com/ceph/ceph/pull/21059>`_, Sage Weil)
+* osd: add numpg_removing metric (`pr#20785 <https://github.com/ceph/ceph/pull/20785>`_, Sage Weil)
+* osdc/Journaler: make sure flush() writes enough data (`issue#22967 <http://tracker.ceph.com/issues/22967>`_, `issue#22824 <http://tracker.ceph.com/issues/22824>`_, `pr#20431 <https://github.com/ceph/ceph/pull/20431>`_, "Yan, Zheng")
+* osd: do not release_reserved_pushes when requeuing (`pr#21229 <https://github.com/ceph/ceph/pull/21229>`_, Sage Weil)
+* osd: Fix assert when checking missing version (`issue#21218 <http://tracker.ceph.com/issues/21218>`_, `issue#23024 <http://tracker.ceph.com/issues/23024>`_, `pr#20495 <https://github.com/ceph/ceph/pull/20495>`_, David Zafman)
+* osd: objecter sends out of sync with pg epochs for proxied ops (`issue#22123 <http://tracker.ceph.com/issues/22123>`_, `issue#23075 <http://tracker.ceph.com/issues/23075>`_, `pr#20609 <https://github.com/ceph/ceph/pull/20609>`_, Sage Weil)
+* osd/OSDMap: skip out/crush-out osds (`pr#20840 <https://github.com/ceph/ceph/pull/20840>`_, xie xingguo)
+* osd/osd_types: fix pg_pool_t encoding for hammer (`pr#21283 <https://github.com/ceph/ceph/pull/21283>`_, Sage Weil)
+* osd: remove cost from mclock op queues; cost not handled well in dmcl… (`pr#21426 <https://github.com/ceph/ceph/pull/21426>`_, J. Eric Ivancich)
+* osd: Remove partially created pg known as DNE (`issue#23160 <http://tracker.ceph.com/issues/23160>`_, `issue#21833 <http://tracker.ceph.com/issues/21833>`_, `pr#20668 <https://github.com/ceph/ceph/pull/20668>`_, David Zafman)
+* osd: resend osd_pgtemp if it's not acked (`issue#23610 <http://tracker.ceph.com/issues/23610>`_, `issue#23630 <http://tracker.ceph.com/issues/23630>`_, `pr#21330 <https://github.com/ceph/ceph/pull/21330>`_, Kefu Chai)
+* osd: treat successful and erroroneous writes the same for log trimming (`issue#23323 <http://tracker.ceph.com/issues/23323>`_, `issue#22050 <http://tracker.ceph.com/issues/22050>`_, `pr#20851 <https://github.com/ceph/ceph/pull/20851>`_, Josh Durgin)
+* os/filestore: fix do_copy_range replay bug (`issue#23351 <http://tracker.ceph.com/issues/23351>`_, `issue#23298 <http://tracker.ceph.com/issues/23298>`_, `pr#20957 <https://github.com/ceph/ceph/pull/20957>`_, Sage Weil)
+* parent blocks are still seen after a whole-object discard (`issue#23304 <http://tracker.ceph.com/issues/23304>`_, `issue#23285 <http://tracker.ceph.com/issues/23285>`_, `pr#20860 <https://github.com/ceph/ceph/pull/20860>`_, Ilya Dryomov, Jason Dillaman)
+* PendingReleaseNotes: add note about upgrading MDS (`issue#23414 <http://tracker.ceph.com/issues/23414>`_, `pr#21001 <https://github.com/ceph/ceph/pull/21001>`_, Patrick Donnelly)
+* : qa: adjust cephfs full test for kclient (`issue#22966 <http://tracker.ceph.com/issues/22966>`_, `issue#22886 <http://tracker.ceph.com/issues/22886>`_, `pr#20417 <https://github.com/ceph/ceph/pull/20417>`_, "Yan, Zheng")
+* qa: ignore io pause warnings in mds-full test (`issue#23062 <http://tracker.ceph.com/issues/23062>`_, `issue#22990 <http://tracker.ceph.com/issues/22990>`_, `pr#20525 <https://github.com/ceph/ceph/pull/20525>`_, Patrick Donnelly)
+* qa: ignore MON_DOWN while thrashing mons (`issue#23061 <http://tracker.ceph.com/issues/23061>`_, `pr#20523 <https://github.com/ceph/ceph/pull/20523>`_, Patrick Donnelly)
+* qa/rgw: remove some civetweb overrides for beast testing (`issue#23002 <http://tracker.ceph.com/issues/23002>`_, `issue#23176 <http://tracker.ceph.com/issues/23176>`_, `pr#20736 <https://github.com/ceph/ceph/pull/20736>`_, Casey Bodley)
+* qa: src/test/libcephfs/test.cc:376: Expected: (len) > (0), actual: -34 vs 0 (`issue#22383 <http://tracker.ceph.com/issues/22383>`_, `issue#22221 <http://tracker.ceph.com/issues/22221>`_, `pr#21173 <https://github.com/ceph/ceph/pull/21173>`_, Patrick Donnelly)
+* qa: synchronize kcephfs suites with fs/multimds (`issue#22891 <http://tracker.ceph.com/issues/22891>`_, `issue#22627 <http://tracker.ceph.com/issues/22627>`_, `pr#20302 <https://github.com/ceph/ceph/pull/20302>`_, Patrick Donnelly)
+* qa/tests - added tag: v12.2.2 to be used by client.1 (`pr#21452 <https://github.com/ceph/ceph/pull/21452>`_, Yuri Weinstein)
+* qa/tests - Change machine type from 'vps' to 'ovh' as 'vps' does not … (`pr#21031 <https://github.com/ceph/ceph/pull/21031>`_, Yuri Weinstein)
+* qa/workunits/rados/test-upgrade-to-mimic.sh: fix tee output (`pr#21506 <https://github.com/ceph/ceph/pull/21506>`_, Sage Weil)
+* qa/workunits/rbd: switch devstack tempest to 17.2.0 tag (`issue#23177 <http://tracker.ceph.com/issues/23177>`_, `issue#22961 <http://tracker.ceph.com/issues/22961>`_, `pr#20715 <https://github.com/ceph/ceph/pull/20715>`_, Jason Dillaman)
+* radosgw-admin data sync run crashes (`issue#23180 <http://tracker.ceph.com/issues/23180>`_, `pr#20762 <https://github.com/ceph/ceph/pull/20762>`_, lvshanchun)
+* rbd-mirror: fix potential infinite loop when formatting status message (`issue#22964 <http://tracker.ceph.com/issues/22964>`_, `issue#22932 <http://tracker.ceph.com/issues/22932>`_, `pr#20416 <https://github.com/ceph/ceph/pull/20416>`_, Mykola Golub)
+* rbd-nbd: fix ebusy when do map (`issue#23542 <http://tracker.ceph.com/issues/23542>`_, `issue#23528 <http://tracker.ceph.com/issues/23528>`_, `pr#21230 <https://github.com/ceph/ceph/pull/21230>`_, Li Wang)
+* rgw: add radosgw-admin sync error trim to trim sync error log (`issue#23302 <http://tracker.ceph.com/issues/23302>`_, `pr#20859 <https://github.com/ceph/ceph/pull/20859>`_, fang yuxiang)
+* rgw: add xml output header in RGWCopyObj_ObjStore_S3 response msg (`issue#22416 <http://tracker.ceph.com/issues/22416>`_, `issue#22635 <http://tracker.ceph.com/issues/22635>`_, `pr#19883 <https://github.com/ceph/ceph/pull/19883>`_, Enming Zhang)
+* rgw: Admin API Support for bucket quota change (`issue#23357 <http://tracker.ceph.com/issues/23357>`_, `issue#21811 <http://tracker.ceph.com/issues/21811>`_, `pr#20885 <https://github.com/ceph/ceph/pull/20885>`_, Jeegn Chen)
+* rgw: allow beast frontend to listen on specific IP address (`issue#22858 <http://tracker.ceph.com/issues/22858>`_, `issue#22778 <http://tracker.ceph.com/issues/22778>`_, `pr#20252 <https://github.com/ceph/ceph/pull/20252>`_, Yuan Zhou)
+* rgw: can't download object with range when compression enabled (`issue#23146 <http://tracker.ceph.com/issues/23146>`_, `issue#23179 <http://tracker.ceph.com/issues/23179>`_, `issue#22852 <http://tracker.ceph.com/issues/22852>`_, `pr#20741 <https://github.com/ceph/ceph/pull/20741>`_, fang yuxiang)
+* rgw: data sync of versioned objects, note updating bi marker (`issue#23025 <http://tracker.ceph.com/issues/23025>`_, `pr#21214 <https://github.com/ceph/ceph/pull/21214>`_, Yehuda Sadeh)
+* RGW doesn't check time skew in auth v4 http header request (`issue#23252 <http://tracker.ceph.com/issues/23252>`_, `issue#22766 <http://tracker.ceph.com/issues/22766>`_, `issue#22439 <http://tracker.ceph.com/issues/22439>`_, `issue#22418 <http://tracker.ceph.com/issues/22418>`_, `pr#20072 <https://github.com/ceph/ceph/pull/20072>`_, Bingyin Zhang, Casey Bodley)
+* rgw_file: avoid evaluating nullptr for readdir offset (`issue#22889 <http://tracker.ceph.com/issues/22889>`_, `pr#20345 <https://github.com/ceph/ceph/pull/20345>`_, Matt Benjamin)
+* rgw: fix crash with rgw_run_sync_thread false (`issue#23318 <http://tracker.ceph.com/issues/23318>`_, `issue#20448 <http://tracker.ceph.com/issues/20448>`_, `pr#20932 <https://github.com/ceph/ceph/pull/20932>`_, Orit Wasserman)
+* rgw: fix memory fragmentation problem reading data from client (`issue#23347 <http://tracker.ceph.com/issues/23347>`_, `pr#20953 <https://github.com/ceph/ceph/pull/20953>`_, Marcus Watts)
+* rgw: fix mutlisite read-write issues (`issue#23690 <http://tracker.ceph.com/issues/23690>`_, `issue#22804 <http://tracker.ceph.com/issues/22804>`_, `pr#21390 <https://github.com/ceph/ceph/pull/21390>`_, Niu Pengju)
+* rgw: fix the max-uploads parameter not work (`issue#23020 <http://tracker.ceph.com/issues/23020>`_, `issue#22825 <http://tracker.ceph.com/issues/22825>`_, `pr#20476 <https://github.com/ceph/ceph/pull/20476>`_, Xin Liao)
+* rgw_log, rgw_file: account for new required envvars (`issue#23192 <http://tracker.ceph.com/issues/23192>`_, `issue#21942 <http://tracker.ceph.com/issues/21942>`_, `pr#20672 <https://github.com/ceph/ceph/pull/20672>`_, Matt Benjamin)
+* rgw: log the right http status code in civetweb frontend's access log (`issue#22812 <http://tracker.ceph.com/issues/22812>`_, `issue#22538 <http://tracker.ceph.com/issues/22538>`_, `pr#20157 <https://github.com/ceph/ceph/pull/20157>`_, Yao Zongyou)
+* rgw: parse old rgw_obj with namespace correctly (`issue#23102 <http://tracker.ceph.com/issues/23102>`_, `issue#22982 <http://tracker.ceph.com/issues/22982>`_, `pr#20586 <https://github.com/ceph/ceph/pull/20586>`_, Yehuda Sadeh)
+* rgw recalculate stats option added (`issue#23691 <http://tracker.ceph.com/issues/23691>`_, `issue#23720 <http://tracker.ceph.com/issues/23720>`_, `issue#23335 <http://tracker.ceph.com/issues/23335>`_, `issue#23322 <http://tracker.ceph.com/issues/23322>`_, `pr#21393 <https://github.com/ceph/ceph/pull/21393>`_, Abhishek Lekshmanan)
+* rgw: reject encrypted object COPY before supported (`issue#23232 <http://tracker.ceph.com/issues/23232>`_, `issue#23346 <http://tracker.ceph.com/issues/23346>`_, `pr#20937 <https://github.com/ceph/ceph/pull/20937>`_, Jeegn Chen)
+* rgw: rgw: reshard cancel command should clear bucket resharding flag (`issue#21619 <http://tracker.ceph.com/issues/21619>`_, `pr#21389 <https://github.com/ceph/ceph/pull/21389>`_, Orit Wasserman)
+* rgw: s3website error handler uses original object name (`issue#23201 <http://tracker.ceph.com/issues/23201>`_, `issue#23310 <http://tracker.ceph.com/issues/23310>`_, `pr#20889 <https://github.com/ceph/ceph/pull/20889>`_, Casey Bodley)
+* rgw: upldate the max-buckets when the quota is uploaded (`issue#23022 <http://tracker.ceph.com/issues/23022>`_, `pr#20477 <https://github.com/ceph/ceph/pull/20477>`_, zhaokun)
+* rgw: usage log fixes (`issue#23686 <http://tracker.ceph.com/issues/23686>`_, `issue#23758 <http://tracker.ceph.com/issues/23758>`_, `pr#21388 <https://github.com/ceph/ceph/pull/21388>`_, Yehuda Sadeh, Greg Farnum, Robin H. Johnson)
+* rocksdb: incorporate the fix in RocksDB for no fast CRC32 path (`issue#22534 <http://tracker.ceph.com/issues/22534>`_, `pr#20825 <https://github.com/ceph/ceph/pull/20825>`_, Radoslaw Zarzynski)
+* scrub errors not cleared on replicas can cause inconsistent pg state when replica takes over primary (`issue#23267 <http://tracker.ceph.com/issues/23267>`_, `pr#21103 <https://github.com/ceph/ceph/pull/21103>`_, David Zafman)
+* snapmapper inconsistency, crash on luminous (`issue#23500 <http://tracker.ceph.com/issues/23500>`_, `pr#21118 <https://github.com/ceph/ceph/pull/21118>`_, Sage Weil)
+* Special scrub handling of hinfo_key errors (`issue#23654 <http://tracker.ceph.com/issues/23654>`_, `issue#23428 <http://tracker.ceph.com/issues/23428>`_, `issue#23364 <http://tracker.ceph.com/issues/23364>`_, `pr#21397 <https://github.com/ceph/ceph/pull/21397>`_, David Zafman)
+* src: s/--use-wheel// (`pr#21177 <https://github.com/ceph/ceph/pull/21177>`_, Kefu Chai)
+* systemd: Wait 10 seconds before restarting ceph-mgr (`issue#23083 <http://tracker.ceph.com/issues/23083>`_, `issue#23101 <http://tracker.ceph.com/issues/23101>`_, `pr#20604 <https://github.com/ceph/ceph/pull/20604>`_, Wido den Hollander)
+* test_admin_socket.sh may fail on wait_for_clean (`issue#23507 <http://tracker.ceph.com/issues/23507>`_, `pr#21124 <https://github.com/ceph/ceph/pull/21124>`_, Mykola Golub)
+* test/ceph-disk: specify the python used for creating venv (`issue#23281 <http://tracker.ceph.com/issues/23281>`_, `pr#20817 <https://github.com/ceph/ceph/pull/20817>`_, Kefu Chai)
+* TestLibRBD.RenameViaLockOwner may still fail with -ENOENT (`issue#23152 <http://tracker.ceph.com/issues/23152>`_, `issue#23068 <http://tracker.ceph.com/issues/23068>`_, `pr#20628 <https://github.com/ceph/ceph/pull/20628>`_, Mykola Golub)
+* test/librbd: utilize unique pool for cache tier testing (`issue#23064 <http://tracker.ceph.com/issues/23064>`_, `issue#11502 <http://tracker.ceph.com/issues/11502>`_, `pr#20550 <https://github.com/ceph/ceph/pull/20550>`_, Jason Dillaman)
+* test/pybind/test_rbd: allow v1 images for testing (`pr#21471 <https://github.com/ceph/ceph/pull/21471>`_, Sage Weil)
+* test: Replace bc command with printf command (`pr#21015 <https://github.com/ceph/ceph/pull/21015>`_, David Zafman)
+* tests: drop upgrade/jewel-x/point-to-point-x in luminous and master (`issue#23159 <http://tracker.ceph.com/issues/23159>`_, `issue#22888 <http://tracker.ceph.com/issues/22888>`_, `pr#20641 <https://github.com/ceph/ceph/pull/20641>`_, Nathan Cutler)
+* tests: ENGINE Error in 'start' listener <bound  in rados (`issue#23606 <http://tracker.ceph.com/issues/23606>`_, `pr#21307 <https://github.com/ceph/ceph/pull/21307>`_, John Spray)
+* tests: rgw: swift tests target ceph-luminous branch (`pr#21048 <https://github.com/ceph/ceph/pull/21048>`_, Nathan Cutler)
+* tests: unittest_pglog timeout (`issue#23522 <http://tracker.ceph.com/issues/23522>`_, `issue#23504 <http://tracker.ceph.com/issues/23504>`_, `pr#21134 <https://github.com/ceph/ceph/pull/21134>`_, Nathan Cutler)
+* Update mgr/restful documentation (`issue#23230 <http://tracker.ceph.com/issues/23230>`_, `pr#20725 <https://github.com/ceph/ceph/pull/20725>`_, Boris Ranto)
+
+v12.2.4 Luminous
+================
+
+This is the fourth bugfix release of Luminous v12.2.x long term stable release
+series. This was primarily intended to fix a few build, ceph-volume/ceph-disk
+and RGW issues. We recommend all the users of 12.2.x series to update.
+
+Notable Changes
+---------------
+
+* ceph-volume: adds support to zap encrypted devices (`issue#22878 <http://tracker.ceph.com/issues/22878>`_, `pr#20545 <https://github.com/ceph/ceph/pull/20545>`_, Andrew Schoen)
+* ceph-volume: log the current running command for easier debugging (`issue#23004 <http://tracker.ceph.com/issues/23004>`_, `pr#20597 <https://github.com/ceph/ceph/pull/20597>`_, Andrew Schoen)
+* ceph-volume: warn on mix of filestore and bluestore flags (`issue#23003 <http://tracker.ceph.com/issues/23003>`_, `pr#20568 <https://github.com/ceph/ceph/pull/20568>`_, Alfredo Deza)
+* cmake: check bootstrap.sh instead before downloading boost (`issue#23071 <http://tracker.ceph.com/issues/23071>`_, `pr#20515 <https://github.com/ceph/ceph/pull/20515>`_, Kefu Chai)
+* core: Backport of cache manipulation: issues #22603 and #22604 (`issue#22604 <http://tracker.ceph.com/issues/22604>`_, `issue#22603 <http://tracker.ceph.com/issues/22603>`_, `pr#20353 <https://github.com/ceph/ceph/pull/20353>`_, Adam C. Emerson)
+* core: last-stat-seq returns 0 because osd stats are cleared (`issue#23093 <http://tracker.ceph.com/issues/23093>`_, `pr#20548 <https://github.com/ceph/ceph/pull/20548>`_, Sage Weil, David Zafman)
+* core: Snapset inconsistency is detected with its own error (`issue#22996 <http://tracker.ceph.com/issues/22996>`_, `pr#20501 <https://github.com/ceph/ceph/pull/20501>`_, David Zafman)
+* rgw:  make init env methods return an error (`issue#23039 <http://tracker.ceph.com/issues/23039>`_, `pr#20564 <https://github.com/ceph/ceph/pull/20564>`_, Abhishek Lekshmanan)
+* rgw: parse old rgw_obj with namespace correctly (`issue#22982 <http://tracker.ceph.com/issues/22982>`_, `pr#20566 <https://github.com/ceph/ceph/pull/20566>`_, Yehuda Sadeh)
+* rgw: return valid Location element, CompleteMultipartUpload (`issue#22655 <http://tracker.ceph.com/issues/22655>`_, `pr#20266 <https://github.com/ceph/ceph/pull/20266>`_, Matt Benjamin)
+* rgw: URL-decode S3 and Swift object-copy URLs (`issue#22121 <http://tracker.ceph.com/issues/22121>`_, `issue#22729 <http://tracker.ceph.com/issues/22729>`_, `pr#20236 <https://github.com/ceph/ceph/pull/20236>`_, Malcolm Lee, Matt Benjamin)
+* rgw: use explicit index pool placement (`issue#22928 <http://tracker.ceph.com/issues/22928>`_, `pr#20565 <https://github.com/ceph/ceph/pull/20565>`_, Yehuda Sadeh)
+* tools: ceph-disk: v12.2.2 unable to create bluestore osd using ceph-disk (`issue#22354 <http://tracker.ceph.com/issues/22354>`_, `pr#20563 <https://github.com/ceph/ceph/pull/20563>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: "$OBJ get-omaphdr" and "$OBJ list-omap" scan all pgs instead of using specific pg (`issue#21327 <http://tracker.ceph.com/issues/21327>`_, `pr#20283 <https://github.com/ceph/ceph/pull/20283>`_, David Zafman)
+
+
+v12.2.3 Luminous
+================
+
+This is the third bugfix release of Luminous v12.2.x long term stable release
+series. It contains a range of bug fixes and a few features across Bluestore,
+CephFS, RBD & RGW. We recommend all the users of 12.2.x series update.
+
+Notable Changes
+---------------
+
+- *CephFS*:
+
+  * The CephFS client now checks for older kernels' inability to reliably clear
+    dentries from the kernel dentry cache. The new option
+    client_die_on_failed_dentry_invalidate (default: true) may be turned off to
+    allow the client to proceed (dangerous!).
+
+
+Other Notable Changes
+---------------------
+* bluestore: do not crash on over-large objects (`issue#22161 <http://tracker.ceph.com/issues/22161>`_, `pr#19630 <https://github.com/ceph/ceph/pull/19630>`_, Sage Weil)
+* bluestore: OSD crash on boot with assert caused by Bluefs on flush write (`issue#21932 <http://tracker.ceph.com/issues/21932>`_, `pr#19047 <https://github.com/ceph/ceph/pull/19047>`_, Jianpeng Ma)
+* build/ops: ceph-base symbols not stripped in debs (`issue#22640 <http://tracker.ceph.com/issues/22640>`_, `pr#19969 <https://github.com/ceph/ceph/pull/19969>`_, Sage Weil)
+* build/ops: ceph-conf: dump parsed config in plain text or as json (`issue#21862 <http://tracker.ceph.com/issues/21862>`_, `pr#18842 <https://github.com/ceph/ceph/pull/18842>`_, Piotr Dałek)
+* build/ops: ceph-mgr dashboard has dependency on python-jinja2 (`issue#22457 <http://tracker.ceph.com/issues/22457>`_, `pr#19865 <https://github.com/ceph/ceph/pull/19865>`_, John Spray)
+* build/ops: ceph-volume fails when centos7 image doesn't have lvm2 installed (`issue#22443 <http://tracker.ceph.com/issues/22443>`_, `issue#22217 <http://tracker.ceph.com/issues/22217>`_, `pr#20215 <https://github.com/ceph/ceph/pull/20215>`_, Nathan Cutler, Theofilos Mouratidis)
+* build/ops: Default kernel.pid_max is easily exceeded during recovery on high OSD-count system (`issue#21929 <http://tracker.ceph.com/issues/21929>`_, `pr#19133 <https://github.com/ceph/ceph/pull/19133>`_, David Disseldorp, Kefu Chai)
+* build/ops: install-deps.sh: revert gcc to the one shipped by distro (`issue#22220 <http://tracker.ceph.com/issues/22220>`_, `pr#19680 <https://github.com/ceph/ceph/pull/19680>`_, Kefu Chai)
+* build/ops: luminous build fails with --without-radosgw (`issue#22321 <http://tracker.ceph.com/issues/22321>`_, `pr#19483 <https://github.com/ceph/ceph/pull/19483>`_, Jason Dillaman)
+* build/ops: move ceph-\*-tool binaries out of ceph-test subpackage (`issue#22319 <http://tracker.ceph.com/issues/22319>`_, `issue#21762 <http://tracker.ceph.com/issues/21762>`_, `pr#19355 <https://github.com/ceph/ceph/pull/19355>`_, liuchang0812, Nathan Cutler, Kefu Chai, Sage Weil)
+* build.ops: rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move (`issue#22558 <http://tracker.ceph.com/issues/22558>`_, `pr#19839 <https://github.com/ceph/ceph/pull/19839>`_, Kefu Chai)
+* ceph: cluster [ERR] Unhandled exception from module 'balancer' while running on mgr.x: 'NoneType' object has no attribute 'iteritems'" in cluster log (`issue#22090 <http://tracker.ceph.com/issues/22090>`_, `pr#19023 <https://github.com/ceph/ceph/pull/19023>`_, Sage Weil)
+* cephfs: cephfs-journal-tool: add "set pool_id" option (`issue#22631 <http://tracker.ceph.com/issues/22631>`_, `pr#20085 <https://github.com/ceph/ceph/pull/20085>`_, dongdong tao)
+* cephfs: cephfs-journal-tool: tool would miss to report some invalid range (`issue#22459 <http://tracker.ceph.com/issues/22459>`_, `pr#19626 <https://github.com/ceph/ceph/pull/19626>`_, dongdong tao)
+* cephfs: cephfs: potential adjust failure in lru_expire (`issue#22458 <http://tracker.ceph.com/issues/22458>`_, `pr#19627 <https://github.com/ceph/ceph/pull/19627>`_, dongdong tao)
+* cephfs: "ceph tell mds" commands result in "File exists" errors on client admin socket (`issue#21406 <http://tracker.ceph.com/issues/21406>`_, `issue#21967 <http://tracker.ceph.com/issues/21967>`_, `pr#18831 <https://github.com/ceph/ceph/pull/18831>`_, Patrick Donnelly)
+* cephfs: client: anchor Inode while trimming caps (`issue#22157 <http://tracker.ceph.com/issues/22157>`_, `pr#19105 <https://github.com/ceph/ceph/pull/19105>`_, Patrick Donnelly)
+* cephfs: client: avoid recursive lock in ll_get_vino (`issue#22629 <http://tracker.ceph.com/issues/22629>`_, `pr#20086 <https://github.com/ceph/ceph/pull/20086>`_, dongdong tao)
+* cephfs: client: dual client segfault with racing ceph_shutdown (`issue#21512 <http://tracker.ceph.com/issues/21512>`_, `issue#20988 <http://tracker.ceph.com/issues/20988>`_, `pr#20082 <https://github.com/ceph/ceph/pull/20082>`_, Jeff Layton)
+* cephfs: client: implement delegation support in userland cephfs (`issue#18490 <http://tracker.ceph.com/issues/18490>`_, `pr#19480 <https://github.com/ceph/ceph/pull/19480>`_, Jeff Layton)
+* cephfs: client: quit on failed remount during dentry invalidate test #19218 (`issue#22269 <http://tracker.ceph.com/issues/22269>`_, `pr#19370 <https://github.com/ceph/ceph/pull/19370>`_, Patrick Donnelly)
+* cephfs: List of filesystems does not get refreshed after a filesystem deletion (`issue#21599 <http://tracker.ceph.com/issues/21599>`_, `pr#18730 <https://github.com/ceph/ceph/pull/18730>`_, John Spray)
+* cephfs: MDS : Avoid the assert failure when the inode for the cap_export from other… (`issue#22610 <http://tracker.ceph.com/issues/22610>`_, `pr#20300 <https://github.com/ceph/ceph/pull/20300>`_, Jianyu Li)
+* cephfs: MDSMonitor: monitor gives constant "is now active in filesystem cephfs as rank" cluster log info messages (`issue#21959 <http://tracker.ceph.com/issues/21959>`_, `pr#19055 <https://github.com/ceph/ceph/pull/19055>`_, Patrick Donnelly)
+* cephfs: racy is_mounted() checks in libcephfs (`issue#21025 <http://tracker.ceph.com/issues/21025>`_, `pr#17875 <https://github.com/ceph/ceph/pull/17875>`_, Jeff Layton)
+* cephfs: src/mds/MDCache.cc: 7421: FAILED assert(CInode::count() == inode_map.size() + snap_inode_map.size()) (`issue#21928 <http://tracker.ceph.com/issues/21928>`_, `pr#18912 <https://github.com/ceph/ceph/pull/18912>`_, "Yan, Zheng")
+* cephfs: vstart_runner: fixes for recent cephfs changes (`issue#22526 <http://tracker.ceph.com/issues/22526>`_, `pr#19829 <https://github.com/ceph/ceph/pull/19829>`_, Patrick Donnelly)
+* ceph-volume: adds a --destroy flag to ceph-volume lvm zap (`issue#22653 <http://tracker.ceph.com/issues/22653>`_, `pr#20240 <https://github.com/ceph/ceph/pull/20240>`_, Andrew Schoen)
+* ceph-volume: adds success messages for lvm prepare/activate/create (`issue#22307 <http://tracker.ceph.com/issues/22307>`_, `pr#20238 <https://github.com/ceph/ceph/pull/20238>`_, Andrew Schoen)
+* ceph-volume: dmcrypt support for lvm (`issue#22619 <http://tracker.ceph.com/issues/22619>`_, `pr#20241 <https://github.com/ceph/ceph/pull/20241>`_, Alfredo Deza)
+* ceph-volume dmcrypt support for simple (`issue#22620 <http://tracker.ceph.com/issues/22620>`_, `pr#20350 <https://github.com/ceph/ceph/pull/20350>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: do not use --key during mkfs (`issue#22283 <http://tracker.ceph.com/issues/22283>`_, `pr#20244 <https://github.com/ceph/ceph/pull/20244>`_, Kefu Chai, Sage Weil)
+* ceph-volume: fix usage of the --osd-id flag (`issue#22642 <http://tracker.ceph.com/issues/22642>`_, `issue#22836 <http://tracker.ceph.com/issues/22836>`_, `pr#20323 <https://github.com/ceph/ceph/pull/20323>`_, Andrew Schoen)
+* ceph-volume Format correctly when vg/lv cannot be used (`issue#22299 <http://tracker.ceph.com/issues/22299>`_, `pr#19527 <https://github.com/ceph/ceph/pull/19527>`_, Alfredo Deza)
+* ceph-volume handle inline comments in the ceph.conf file (`issue#22297 <http://tracker.ceph.com/issues/22297>`_, `pr#19532 <https://github.com/ceph/ceph/pull/19532>`_, Alfredo Deza)
+* ceph-volume: handle leading whitespace/tabs in ceph.conf (`issue#22280 <http://tracker.ceph.com/issues/22280>`_, `pr#19526 <https://github.com/ceph/ceph/pull/19526>`_, Alfredo Deza)
+* ceph-volume: lvm zap will unmount osd paths used by zapped devices (`issue#22876 <http://tracker.ceph.com/issues/22876>`_, `pr#20438 <https://github.com/ceph/ceph/pull/20438>`_, Andrew Schoen)
+* ceph-volume: removed the explicit use of sudo (`issue#22282 <http://tracker.ceph.com/issues/22282>`_, `pr#19525 <https://github.com/ceph/ceph/pull/19525>`_, Andrew Schoen)
+* ceph-volume rollback on failed OSD prepare/create (`issue#22281 <http://tracker.ceph.com/issues/22281>`_, `pr#20237 <https://github.com/ceph/ceph/pull/20237>`_, Alfredo Deza)
+* ceph-volume should be able to handle multiple LVM (VG/LV) tags (`issue#22305 <http://tracker.ceph.com/issues/22305>`_, `pr#19528 <https://github.com/ceph/ceph/pull/19528>`_, Alfredo Deza)
+* ceph-volume use realpath when checking mounts (`issue#22988 <http://tracker.ceph.com/issues/22988>`_, `pr#20429 <https://github.com/ceph/ceph/pull/20429>`_, Alfredo Deza)
+* ceph-volume: warn on missing ceph.conf file (`issue#22326 <http://tracker.ceph.com/issues/22326>`_, `pr#19530 <https://github.com/ceph/ceph/pull/19530>`_, Alfredo Deza)
+* common: compute SimpleLRU's size with contents.size() instead of lru.… (`issue#22613 <http://tracker.ceph.com/issues/22613>`_, `pr#19977 <https://github.com/ceph/ceph/pull/19977>`_, Xuehan Xu)
+* config: lower default omap entries recovered at once (`issue#21897 <http://tracker.ceph.com/issues/21897>`_, `pr#19928 <https://github.com/ceph/ceph/pull/19928>`_, Josh Durgin)
+* core: backoff causes out of order op (`issue#21407 <http://tracker.ceph.com/issues/21407>`_, `pr#18747 <https://github.com/ceph/ceph/pull/18747>`_, Sage Weil)
+* core: common/throttle: start using 64-bit values (`issue#22539 <http://tracker.ceph.com/issues/22539>`_, `pr#19995 <https://github.com/ceph/ceph/pull/19995>`_, Igor Fedotov)
+* core: fix broken use of streamstream::rdbuf() (`issue#22715 <http://tracker.ceph.com/issues/22715>`_, `pr#20042 <https://github.com/ceph/ceph/pull/20042>`_, Sage Weil)
+* core: possible deadlock in various maintenance operations (`issue#22120 <http://tracker.ceph.com/issues/22120>`_, `pr#19123 <https://github.com/ceph/ceph/pull/19123>`_, Jason Dillaman)
+* core: _read_bdev_label unable to decode label at offset (`issue#22285 <http://tracker.ceph.com/issues/22285>`_, `pr#20326 <https://github.com/ceph/ceph/pull/20326>`_, Sage Weil)
+* core: rocksdb: fixes early metadata spill over to slow device in (`issue#22264 <http://tracker.ceph.com/issues/22264>`_, `pr#19257 <https://github.com/ceph/ceph/pull/19257>`_, Igor Fedotov)
+* core: Various odd clog messages for mons (`issue#22082 <http://tracker.ceph.com/issues/22082>`_, `pr#19031 <https://github.com/ceph/ceph/pull/19031>`_, John Spray)
+* crush: balancer crush-compat sends "foo" command (`issue#22361 <http://tracker.ceph.com/issues/22361>`_, `pr#19555 <https://github.com/ceph/ceph/pull/19555>`_, John Spray)
+* doc: crush_ruleset is invalid command in luminous (`issue#20559 <http://tracker.ceph.com/issues/20559>`_, `pr#19446 <https://github.com/ceph/ceph/pull/19446>`_, Nathan Cutler)
+* doc: doc/rbd: tweaks for the LIO iSCSI gateway (`issue#21763 <http://tracker.ceph.com/issues/21763>`_, `pr#20213 <https://github.com/ceph/ceph/pull/20213>`_, Ashish Singh, Mike Christie, Jason Dillaman)
+* doc: man page for mount.fuse.ceph (`issue#21539 <http://tracker.ceph.com/issues/21539>`_, `issue#22595 <http://tracker.ceph.com/issues/22595>`_, `pr#19449 <https://github.com/ceph/ceph/pull/19449>`_, Jos Collin)
+* doc: misc fixes for CephFS best practices (`issue#22630 <http://tracker.ceph.com/issues/22630>`_, `pr#19858 <https://github.com/ceph/ceph/pull/19858>`_, Jos Collin)
+* doc: remove region from "INSTALL CEPH OBJECT GATEWAY" (`issue#21610 <http://tracker.ceph.com/issues/21610>`_, `pr#18865 <https://github.com/ceph/ceph/pull/18865>`_, Orit Wasserman)
+* doc: update Blacklisting and OSD epoch barrier (`issue#22542 <http://tracker.ceph.com/issues/22542>`_, `pr#19741 <https://github.com/ceph/ceph/pull/19741>`_, Jos Collin)
+* librbd: cannot clone all image-metas if we have more than 64 key/value pairs (`issue#21814 <http://tracker.ceph.com/issues/21814>`_, `pr#19503 <https://github.com/ceph/ceph/pull/19503>`_, PCzhangPC)
+* librbd: cannot copy all image-metas if we have more than 64 key/value pairs (`issue#21815 <http://tracker.ceph.com/issues/21815>`_, `pr#19504 <https://github.com/ceph/ceph/pull/19504>`_, PCzhangPC)
+* librbd: compare and write against a clone can result in failure (`issue#20789 <http://tracker.ceph.com/issues/20789>`_, `pr#20211 <https://github.com/ceph/ceph/pull/20211>`_, Mykola Golub, Jason Dillaman)
+* librbd: default to sparse-reads for any IO operation over 64K (`issue#21849 <http://tracker.ceph.com/issues/21849>`_, `pr#20208 <https://github.com/ceph/ceph/pull/20208>`_, Jason Dillaman)
+* librbd: fix snap create/rm may taking long time (`issue#22716 <http://tracker.ceph.com/issues/22716>`_, `pr#20153 <https://github.com/ceph/ceph/pull/20153>`_, Song Shun)
+* librbd: force removal of a snapshot cannot ignore dependent children (`issue#22791 <http://tracker.ceph.com/issues/22791>`_, `pr#20135 <https://github.com/ceph/ceph/pull/20135>`_, Jason Dillaman)
+* librbd: Image-meta should be dynamically refreshed (`issue#21529 <http://tracker.ceph.com/issues/21529>`_, `pr#19447 <https://github.com/ceph/ceph/pull/19447>`_, Dongsheng Yang, Jason Dillaman)
+* librbd: journal should ignore -EILSEQ errors from compare-and-write (`issue#21628 <http://tracker.ceph.com/issues/21628>`_, `pr#20206 <https://github.com/ceph/ceph/pull/20206>`_, Jason Dillaman)
+* librbd: refresh image after applying new/removing old metadata (`issue#21711 <http://tracker.ceph.com/issues/21711>`_, `pr#19485 <https://github.com/ceph/ceph/pull/19485>`_, Jason Dillaman)
+* librbd: set deleted parent pointer to null (`issue#22158 <http://tracker.ceph.com/issues/22158>`_, `pr#20210 <https://github.com/ceph/ceph/pull/20210>`_, Jason Dillaman)
+* luminous: ceph-fuse: ::rmdir() uses a deleted memory structure of dentry leads … (`issue#22536 <http://tracker.ceph.com/issues/22536>`_, `pr#19968 <https://github.com/ceph/ceph/pull/19968>`_, YunfeiGuan)
+* mds: check for CEPH_OSDMAP_FULL is now wrong; cluster full flag is obsolete (`issue#22483 <http://tracker.ceph.com/issues/22483>`_, `pr#19830 <https://github.com/ceph/ceph/pull/19830>`_, Patrick Donnelly)
+* mds: don't check gid when none specified in auth caps (`issue#22009 <http://tracker.ceph.com/issues/22009>`_, `pr#18835 <https://github.com/ceph/ceph/pull/18835>`_, Douglas Fuller)
+* mds: don't delay processing completed requests in replay queue (`issue#22163 <http://tracker.ceph.com/issues/22163>`_, `pr#19157 <https://github.com/ceph/ceph/pull/19157>`_, "Yan, Zheng")
+* mds: don't report repaired backtraces in damagetable, write back after repair, clean up scrub log (`issue#18743 <http://tracker.ceph.com/issues/18743>`_, `issue#22058 <http://tracker.ceph.com/issues/22058>`_, `pr#20341 <https://github.com/ceph/ceph/pull/20341>`_, "Yan, Zheng", John Spray)
+* mds: fix CDir::log_mark_dirty() (`issue#21584 <http://tracker.ceph.com/issues/21584>`_, `pr#18008 <https://github.com/ceph/ceph/pull/18008>`_, "Yan, Zheng")
+* mds: fix dump last_sent (`issue#22562 <http://tracker.ceph.com/issues/22562>`_, `pr#19959 <https://github.com/ceph/ceph/pull/19959>`_, dongdong tao)
+* mds: fix MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 definition (`issue#21985 <http://tracker.ceph.com/issues/21985>`_, `pr#18782 <https://github.com/ceph/ceph/pull/18782>`_, "Yan, Zheng")
+* mds: fix return value of MDCache::dump_cache (`issue#22798 <http://tracker.ceph.com/issues/22798>`_, `pr#20121 <https://github.com/ceph/ceph/pull/20121>`_, "Yan, Zheng")
+* mds: fix scrub crash (`issue#22730 <http://tracker.ceph.com/issues/22730>`_, `pr#20249 <https://github.com/ceph/ceph/pull/20249>`_, dongdong tao)
+* mds: fix StrayManager::truncate() (`issue#21091 <http://tracker.ceph.com/issues/21091>`_, `pr#18019 <https://github.com/ceph/ceph/pull/18019>`_, "Yan, Zheng")
+* mds: handle client reconnect gather race (`issue#22263 <http://tracker.ceph.com/issues/22263>`_, `pr#19326 <https://github.com/ceph/ceph/pull/19326>`_, "Yan, Zheng")
+* mds: handle client session messages when mds is stopping (`issue#22460 <http://tracker.ceph.com/issues/22460>`_, `pr#19585 <https://github.com/ceph/ceph/pull/19585>`_, "Yan, Zheng")
+* mds: handle 'inode gets queued for recovery multiple times' (`issue#22647 <http://tracker.ceph.com/issues/22647>`_, `pr#19982 <https://github.com/ceph/ceph/pull/19982>`_, "Yan, Zheng")
+* mds: ignore export pin for unlinked directory (`issue#22219 <http://tracker.ceph.com/issues/22219>`_, `pr#19360 <https://github.com/ceph/ceph/pull/19360>`_, "Yan, Zheng")
+* mds: limit size of subtree migration (`issue#21892 <http://tracker.ceph.com/issues/21892>`_, `pr#20339 <https://github.com/ceph/ceph/pull/20339>`_, "Yan, Zheng")
+* mds: no assertion on inode being purging in find_ino_peers() (`issue#21722 <http://tracker.ceph.com/issues/21722>`_, `pr#18869 <https://github.com/ceph/ceph/pull/18869>`_, Zhi Zhang)
+* mds: preserve order of requests during recovery of multimds cluster (`issue#21843 <http://tracker.ceph.com/issues/21843>`_, `pr#18871 <https://github.com/ceph/ceph/pull/18871>`_, "Yan, Zheng")
+* mds: prevent filelock from being stuck at XSYN state (`issue#22008 <http://tracker.ceph.com/issues/22008>`_, `pr#20340 <https://github.com/ceph/ceph/pull/20340>`_, "Yan, Zheng")
+* mds: properly eval locks after importing inode (`issue#22357 <http://tracker.ceph.com/issues/22357>`_, `pr#19646 <https://github.com/ceph/ceph/pull/19646>`_, "Yan, Zheng")
+* mds: reduce debugging level for balancer messages (`issue#21853 <http://tracker.ceph.com/issues/21853>`_, `pr#19827 <https://github.com/ceph/ceph/pull/19827>`_, Patrick Donnelly)
+* mds: respect mds_client_writeable_range_max_inc_objs config (`issue#22492 <http://tracker.ceph.com/issues/22492>`_, `pr#19776 <https://github.com/ceph/ceph/pull/19776>`_, "Yan, Zheng")
+* mds: set higher priority for some perf counters (`issue#22776 <http://tracker.ceph.com/issues/22776>`_, `pr#20299 <https://github.com/ceph/ceph/pull/20299>`_, Shangzhong Zhu)
+* mds: set PRIO_USEFUL on num_sessions counter (`issue#21927 <http://tracker.ceph.com/issues/21927>`_, `pr#18722 <https://github.com/ceph/ceph/pull/18722>`_, John Spray)
+* mds: tell session ls returns vanila EINVAL when MDS is not active (`issue#21991 <http://tracker.ceph.com/issues/21991>`_, `pr#19505 <https://github.com/ceph/ceph/pull/19505>`_, Jos Collin)
+* mds: track dirty dentries in separate list (`issue#19578 <http://tracker.ceph.com/issues/19578>`_, `pr#19775 <https://github.com/ceph/ceph/pull/19775>`_, "Yan, Zheng")
+* mds: trim 'N' log segments according to how many log segments are there (`issue#21975 <http://tracker.ceph.com/issues/21975>`_, `pr#18783 <https://github.com/ceph/ceph/pull/18783>`_, "Yan, Zheng")
+* mgr: ceph-mgr spuriously reloading OSD metadata on map changes (`issue#21159 <http://tracker.ceph.com/issues/21159>`_, `pr#18732 <https://github.com/ceph/ceph/pull/18732>`_, Yanhu Cao)
+* mgr: disconnect unregistered service daemon when report received (`issue#22286 <http://tracker.ceph.com/issues/22286>`_, `pr#20089 <https://github.com/ceph/ceph/pull/20089>`_, Jason Dillaman)
+* mgr: KeyError: ('name',) in balancer rm (`issue#22470 <http://tracker.ceph.com/issues/22470>`_, `pr#19624 <https://github.com/ceph/ceph/pull/19624>`_, Dan van der Ster)
+* mgr: Manager daemon x is unresponsive. No standby daemons available (`issue#21147 <http://tracker.ceph.com/issues/21147>`_, `pr#19501 <https://github.com/ceph/ceph/pull/19501>`_, Sage Weil)
+* mgr: mgr/balancer/upmap_max_iterations must be cast to integer (`issue#22429 <http://tracker.ceph.com/issues/22429>`_, `pr#19553 <https://github.com/ceph/ceph/pull/19553>`_, Dan van der Ster)
+* mgr: mgr/dashboard: added iSCSI IOPS/throughput metrics (`issue#21391 <http://tracker.ceph.com/issues/21391>`_, `pr#20209 <https://github.com/ceph/ceph/pull/20209>`_, Jason Dillaman)
+* mgr: mgr/dashboard: Fix PG status coloring (`issue#22615 <http://tracker.ceph.com/issues/22615>`_, `pr#19844 <https://github.com/ceph/ceph/pull/19844>`_, Wido den Hollander)
+* mgr: mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr pro… (`issue#22116 <http://tracker.ceph.com/issues/22116>`_, `pr#19929 <https://github.com/ceph/ceph/pull/19929>`_, Jan Fajerski, Peter Woodman)
+* mgr: mgr tests don't indicate failure if exception thrown from serve() (`issue#21999 <http://tracker.ceph.com/issues/21999>`_, `pr#18832 <https://github.com/ceph/ceph/pull/18832>`_, John Spray)
+* mgr: mgr[zabbix] float division by zero (osd['kb'] = 0) (`issue#21904 <http://tracker.ceph.com/issues/21904>`_, `pr#19048 <https://github.com/ceph/ceph/pull/19048>`_, Ilja Slepnev)
+* mgr: prometheus: added osd commit/apply latency metrics (#22718) (`issue#22718 <http://tracker.ceph.com/issues/22718>`_, `pr#20084 <https://github.com/ceph/ceph/pull/20084>`_, Konstantin Shalygin)
+* mgr: pybind/mgr/dashboard: fix duplicated slash in html href (`issue#22851 <http://tracker.ceph.com/issues/22851>`_, `pr#20325 <https://github.com/ceph/ceph/pull/20325>`_, Shengjing Zhu)
+* mgr: pybind/mgr/dashboard: fix reverse proxy support (`issue#22557 <http://tracker.ceph.com/issues/22557>`_, `pr#20182 <https://github.com/ceph/ceph/pull/20182>`_, Nick Erdmann, Kefu Chai)
+* mgr: pybind/mgr/prometheus: fix metric type undef -> untyped (`issue#22313 <http://tracker.ceph.com/issues/22313>`_, `pr#19834 <https://github.com/ceph/ceph/pull/19834>`_, Ilya Margolin)
+* mgr: restarting active ceph-mgr cause glitches in bps and iops metrics (`issue#21773 <http://tracker.ceph.com/issues/21773>`_, `pr#18735 <https://github.com/ceph/ceph/pull/18735>`_, Aleksei Gutikov, Kefu Chai)
+* mgr: Services reported with blank hostname (`issue#20887 <http://tracker.ceph.com/issues/20887>`_, `issue#21687 <http://tracker.ceph.com/issues/21687>`_, `pr#17869 <https://github.com/ceph/ceph/pull/17869>`_, liuchang0812, Chang Liu)
+* mon: do not use per_pool_sum_delta to show recovery summary (`issue#22727 <http://tracker.ceph.com/issues/22727>`_, `pr#20150 <https://github.com/ceph/ceph/pull/20150>`_, Chang Liu)
+* mon: fix mgr using auth_client_required policy (`issue#22096 <http://tracker.ceph.com/issues/22096>`_, `pr#20156 <https://github.com/ceph/ceph/pull/20156>`_, John Spray)
+* mon: MDSMonitor: reject misconfigured mds_blacklist_interval (`issue#21821 <http://tracker.ceph.com/issues/21821>`_, `pr#19871 <https://github.com/ceph/ceph/pull/19871>`_, John Spray)
+* mon/MgrMonitor: limit mgrmap history (`issue#22257 <http://tracker.ceph.com/issues/22257>`_, `pr#19187 <https://github.com/ceph/ceph/pull/19187>`_, Sage Weil)
+* mon: reenable timer to send digest when paxos is temporarily inactive (`issue#22142 <http://tracker.ceph.com/issues/22142>`_, `pr#19481 <https://github.com/ceph/ceph/pull/19481>`_, Jan Fajerski)
+* msg: msg/async/AsyncConnection.cc: 1835: FAILED assert(state == STATE_CLOSED) (`issue#21883 <http://tracker.ceph.com/issues/21883>`_, `pr#18746 <https://github.com/ceph/ceph/pull/18746>`_, Haomai Wang)
+* msg: msg/async: unregister connection failed when racing happened (`issue#22437 <http://tracker.ceph.com/issues/22437>`_, `pr#19552 <https://github.com/ceph/ceph/pull/19552>`_, Haomai Wang)
+* osdc: "FAILED assert(bh->last_write_tid > tid)" in powercycle-wip-yuri-master-1.19.18-distro-basic-smithi (`issue#22741 <http://tracker.ceph.com/issues/22741>`_, `pr#20256 <https://github.com/ceph/ceph/pull/20256>`_, "Yan, Zheng")
+* osdc/Journaler: add 'stopping' check to various finish callbacks (`issue#22360 <http://tracker.ceph.com/issues/22360>`_, `pr#19610 <https://github.com/ceph/ceph/pull/19610>`_, "Yan, Zheng")
+* osdc/Objecter: objecter op_send_bytes perf counter always 0 (`issue#21982 <http://tracker.ceph.com/issues/21982>`_, `pr#19046 <https://github.com/ceph/ceph/pull/19046>`_, Jianpeng Ma)
+* osd: do not check out-of-date osdmap for DESTROYED flag on start (`issue#22673 <http://tracker.ceph.com/issues/22673>`_, `pr#20068 <https://github.com/ceph/ceph/pull/20068>`_, Sage Weil)
+* osd,mgr: report pending creating pgs to mgr (`issue#22440 <http://tracker.ceph.com/issues/22440>`_, `pr#20204 <https://github.com/ceph/ceph/pull/20204>`_, Kefu Chai)
+* osd: miscounting degraded objects and PG stuck in recovery_unfound (`issue#22145 <http://tracker.ceph.com/issues/22145>`_, `pr#20055 <https://github.com/ceph/ceph/pull/20055>`_, Sage Weil, David Zafman)
+* osd: Objecter::C_ObjectOperation_sparse_read throws/catches exceptions on -ENOENT (`issue#21844 <http://tracker.ceph.com/issues/21844>`_, `pr#18744 <https://github.com/ceph/ceph/pull/18744>`_, Jason Dillaman)
+* osd: Objecter::_send_op unnecessarily constructs costly hobject_t (`issue#21845 <http://tracker.ceph.com/issues/21845>`_, `pr#18745 <https://github.com/ceph/ceph/pull/18745>`_, Jason Dillaman)
+* osd: On pg repair the primary is not favored as was intended (`issue#21907 <http://tracker.ceph.com/issues/21907>`_, `pr#19083 <https://github.com/ceph/ceph/pull/19083>`_, David Zafman)
+* osd: OSD crushes with FAILED assert(used_blocks.size() > count) during the first start after upgrade 12.2.1 -> 12.2.2 (`issue#22535 <http://tracker.ceph.com/issues/22535>`_, `pr#19888 <https://github.com/ceph/ceph/pull/19888>`_, Igor Fedotov)
+* osd: OSDMap cache assert on shutdown (`issue#21737 <http://tracker.ceph.com/issues/21737>`_, `pr#18749 <https://github.com/ceph/ceph/pull/18749>`_, Greg Farnum)
+* osd: OSDService::recovery_need_sleep read+updated without locking (`issue#21566 <http://tracker.ceph.com/issues/21566>`_, `pr#18753 <https://github.com/ceph/ceph/pull/18753>`_, Neha Ojha)
+* osd: "osd status" command exception if OSD not in pgmap stats (`issue#21707 <http://tracker.ceph.com/issues/21707>`_, `pr#19084 <https://github.com/ceph/ceph/pull/19084>`_, Yanhu Cao)
+* osd, pg, mgr: make snap trim queue problems visible (`issue#22448 <http://tracker.ceph.com/issues/22448>`_, `pr#20098 <https://github.com/ceph/ceph/pull/20098>`_, Piotr Dałek)
+* osd: Pool Compression type option doesn't apply to new OSDs (`issue#22419 <http://tracker.ceph.com/issues/22419>`_, `pr#20106 <https://github.com/ceph/ceph/pull/20106>`_, Kefu Chai)
+* osd: replica read can trigger cache promotion (`issue#20919 <http://tracker.ceph.com/issues/20919>`_, `pr#19499 <https://github.com/ceph/ceph/pull/19499>`_, Sage Weil)
+* osd/ReplicatedPG.cc: recover_replicas: object added to missing set for backfill, but is not in recovering, error! (`issue#21382 <http://tracker.ceph.com/issues/21382>`_, `issue#14513 <http://tracker.ceph.com/issues/14513>`_, `issue#18162 <http://tracker.ceph.com/issues/18162>`_, `pr#20081 <https://github.com/ceph/ceph/pull/20081>`_, David Zafman)
+* osd: subscribe osdmaps if any pending pgs (`issue#22113 <http://tracker.ceph.com/issues/22113>`_, `pr#19059 <https://github.com/ceph/ceph/pull/19059>`_, Kefu Chai)
+* osd: "sudo cp /var/lib/ceph/osd/ceph-0/fsid ..." fails (`issue#20736 <http://tracker.ceph.com/issues/20736>`_, `pr#19631 <https://github.com/ceph/ceph/pull/19631>`_, Patrick Donnelly)
+* os: fix 0-length zero semantics, test (`issue#21712 <http://tracker.ceph.com/issues/21712>`_, `pr#20049 <https://github.com/ceph/ceph/pull/20049>`_, Sage Weil)
+* qa/tests: Applied PR 20053 to stress-split tests (`issue#22665 <http://tracker.ceph.com/issues/22665>`_, `pr#20451 <https://github.com/ceph/ceph/pull/20451>`_, Yuri Weinstein)
+* rbd: abort in listing mapped nbd devices when running in a container (`issue#22012 <http://tracker.ceph.com/issues/22012>`_, `issue#22011 <http://tracker.ceph.com/issues/22011>`_, `pr#19051 <https://github.com/ceph/ceph/pull/19051>`_, Li Wang)
+* rbd: [api] compare-and-write methods not properly advertised (`issue#22036 <http://tracker.ceph.com/issues/22036>`_, `pr#18834 <https://github.com/ceph/ceph/pull/18834>`_, Jason Dillaman)
+* rbd: class rbd.Image discard----OSError: [errno 2147483648] error discarding region (`issue#21966 <http://tracker.ceph.com/issues/21966>`_, `pr#19058 <https://github.com/ceph/ceph/pull/19058>`_, Jason Dillaman)
+* rbd: cluster resource agent ocf:ceph:rbd - wrong permissions (`issue#22362 <http://tracker.ceph.com/issues/22362>`_, `pr#19554 <https://github.com/ceph/ceph/pull/19554>`_, Nathan Cutler)
+* rbd: disk usage on empty pool no longer returns an error message (`issue#22200 <http://tracker.ceph.com/issues/22200>`_, `pr#19107 <https://github.com/ceph/ceph/pull/19107>`_, Jason Dillaman)
+* rbd: fix crash during map (`issue#21808 <http://tracker.ceph.com/issues/21808>`_, `pr#18698 <https://github.com/ceph/ceph/pull/18698>`_, Peter Keresztes Schmidt)
+* rbd: [journal] tags are not being expired if no other clients are registered (`issue#21960 <http://tracker.ceph.com/issues/21960>`_, `pr#18840 <https://github.com/ceph/ceph/pull/18840>`_, Jason Dillaman)
+* rbd: librbd: filter out potential race with image rename (`issue#18435 <http://tracker.ceph.com/issues/18435>`_, `pr#19853 <https://github.com/ceph/ceph/pull/19853>`_, Jason Dillaman)
+* rbd-mirror: Allow a different data-pool to be used on the secondary cluster (`issue#21088 <http://tracker.ceph.com/issues/21088>`_, `pr#19305 <https://github.com/ceph/ceph/pull/19305>`_, Adam Wolfe Gordon)
+* rbd-mirror: primary image should register in remote, non-primary image's journal (`issue#21961 <http://tracker.ceph.com/issues/21961>`_, `issue#21561 <http://tracker.ceph.com/issues/21561>`_, `pr#20207 <https://github.com/ceph/ceph/pull/20207>`_, Jason Dillaman)
+* rbd-mirror: sync image metadata when transfering remote image (`issue#21535 <http://tracker.ceph.com/issues/21535>`_, `pr#19484 <https://github.com/ceph/ceph/pull/19484>`_, Jason Dillaman)
+* rbd: Python RBD metadata_get does not work (`issue#22306 <http://tracker.ceph.com/issues/22306>`_, `pr#19479 <https://github.com/ceph/ceph/pull/19479>`_, Mykola Golub)
+* rbd: rbd ls -l crashes with SIGABRT (`issue#21558 <http://tracker.ceph.com/issues/21558>`_, `pr#19800 <https://github.com/ceph/ceph/pull/19800>`_, Jason Dillaman)
+* rbd: [rbd-mirror] new pools might not be detected (`issue#22461 <http://tracker.ceph.com/issues/22461>`_, `pr#19625 <https://github.com/ceph/ceph/pull/19625>`_, Jason Dillaman)
+* rbd: [rbd-nbd] Fedora does not register resize events (`issue#22131 <http://tracker.ceph.com/issues/22131>`_, `pr#19066 <https://github.com/ceph/ceph/pull/19066>`_, Jason Dillaman)
+* rbd: [test] UpdateFeatures RPC message should be included in test_notify.py (`issue#21936 <http://tracker.ceph.com/issues/21936>`_, `pr#18838 <https://github.com/ceph/ceph/pull/18838>`_, Jason Dillaman)
+* Revert " luminous: msg/async: unregister connection failed when racing happened" (`issue#22231 <http://tracker.ceph.com/issues/22231>`_, `pr#20247 <https://github.com/ceph/ceph/pull/20247>`_, Sage Weil)
+* rgw: 501 is returned When init multipart is using V4 signature and chunk encoding (`issue#22129 <http://tracker.ceph.com/issues/22129>`_, `pr#19506 <https://github.com/ceph/ceph/pull/19506>`_, Jeegn Chen)
+* rgw: add cors header rule check in cors option request (`issue#22002 <http://tracker.ceph.com/issues/22002>`_, `pr#19053 <https://github.com/ceph/ceph/pull/19053>`_, yuliyang)
+* rgw: backport beast frontend and boost 1.66 update (`issue#22101 <http://tracker.ceph.com/issues/22101>`_, `issue#20935 <http://tracker.ceph.com/issues/20935>`_, `issue#21831 <http://tracker.ceph.com/issues/21831>`_, `issue#20048 <http://tracker.ceph.com/issues/20048>`_, `issue#22600 <http://tracker.ceph.com/issues/22600>`_, `issue#20971 <http://tracker.ceph.com/issues/20971>`_, `pr#19848 <https://github.com/ceph/ceph/pull/19848>`_, Casey Bodley, Jiaying Ren)
+* rgw: bucket index object not deleted after radosgw-admin bucket rm --purge-objects --bypass-gc (`issue#22122 <http://tracker.ceph.com/issues/22122>`_, `issue#19959 <http://tracker.ceph.com/issues/19959>`_, `pr#19085 <https://github.com/ceph/ceph/pull/19085>`_, Aleksei Gutikov)
+* rgw: bucket policy evaluation logical error (`issue#21901 <http://tracker.ceph.com/issues/21901>`_, `issue#21896 <http://tracker.ceph.com/issues/21896>`_, `pr#19810 <https://github.com/ceph/ceph/pull/19810>`_, Adam C. Emerson)
+* rgw: bucket resharding should not update bucket ACL or user stats (`issue#22742 <http://tracker.ceph.com/issues/22742>`_, `issue#22124 <http://tracker.ceph.com/issues/22124>`_, `pr#20327 <https://github.com/ceph/ceph/pull/20327>`_, Orit Wasserman)
+* rgw: check going_down() when lifecycle processing (`issue#22099 <http://tracker.ceph.com/issues/22099>`_, `pr#19088 <https://github.com/ceph/ceph/pull/19088>`_, Yao Zongyou)
+* rgw: Dynamic bucket indexing, resharding and tenants seems to be broken (`issue#22046 <http://tracker.ceph.com/issues/22046>`_, `pr#19050 <https://github.com/ceph/ceph/pull/19050>`_, Orit Wasserman)
+* rgw: file deadlock on lru evicting (`issue#22736 <http://tracker.ceph.com/issues/22736>`_, `pr#20075 <https://github.com/ceph/ceph/pull/20075>`_, Matt Benjamin)
+* rgw: fix chained cache invalidation to prevent cache size growth (`issue#22410 <http://tracker.ceph.com/issues/22410>`_, `pr#19785 <https://github.com/ceph/ceph/pull/19785>`_, Mark Kogan)
+* rgw: fix for empty query string in beast frontend (`issue#22797 <http://tracker.ceph.com/issues/22797>`_, `pr#20338 <https://github.com/ceph/ceph/pull/20338>`_, Casey Bodley)
+* rgw: fix GET website response error code (`issue#22272 <http://tracker.ceph.com/issues/22272>`_, `pr#19489 <https://github.com/ceph/ceph/pull/19489>`_, Dmitry Plyakin)
+* rgw: fix rewrite a versioning object create a new object bug (`issue#21984 <http://tracker.ceph.com/issues/21984>`_, `issue#22529 <http://tracker.ceph.com/issues/22529>`_, `pr#19787 <https://github.com/ceph/ceph/pull/19787>`_, Enming Zhang, Matt Benjamin)
+* rgw: Fix swift object expiry not deleting objects (`issue#22084 <http://tracker.ceph.com/issues/22084>`_, `pr#18972 <https://github.com/ceph/ceph/pull/18972>`_, Pavan Rallabhandi)
+* rgw: Fix swift object expiry not deleting objects (`issue#22084 <http://tracker.ceph.com/issues/22084>`_, `pr#19090 <https://github.com/ceph/ceph/pull/19090>`_, Pavan Rallabhandi)
+* rgw: librgw: fix shutdown error with resources uncleaned (`issue#22296 <http://tracker.ceph.com/issues/22296>`_, `pr#20073 <https://github.com/ceph/ceph/pull/20073>`_, Tao Chen)
+* rgw: log keystone errors at a higher level (`issue#22151 <http://tracker.ceph.com/issues/22151>`_, `pr#19077 <https://github.com/ceph/ceph/pull/19077>`_, Abhishek Lekshmanan)
+* rgw: make HTTP dechunking compatible with Amazon S3 (`issue#21015 <http://tracker.ceph.com/issues/21015>`_, `pr#19500 <https://github.com/ceph/ceph/pull/19500>`_, Radoslaw Zarzynski)
+* rgw: modify s3 type subuser access permission fail (`issue#21983 <http://tracker.ceph.com/issues/21983>`_, `pr#18766 <https://github.com/ceph/ceph/pull/18766>`_, yuliyang)
+* rgw: multisite: destination zone does not compress synced objects (`issue#21895 <http://tracker.ceph.com/issues/21895>`_, `pr#18867 <https://github.com/ceph/ceph/pull/18867>`_, Casey Bodley)
+* rgw: multisite: 'radosgw-admin sync error list' contains temporary EBUSY errors (`issue#22473 <http://tracker.ceph.com/issues/22473>`_, `pr#19799 <https://github.com/ceph/ceph/pull/19799>`_, Casey Bodley)
+* rgw: null instance mtime incorrect when enable versioning (`issue#21743 <http://tracker.ceph.com/issues/21743>`_, `pr#18870 <https://github.com/ceph/ceph/pull/18870>`_, Shasha Lu)
+* rgw: Policy parser may or may not dereference uninitialized boost::optional sometimes (`issue#21962 <http://tracker.ceph.com/issues/21962>`_, `pr#18868 <https://github.com/ceph/ceph/pull/18868>`_, Adam C. Emerson)
+* rgw: Possible deadlock in 'list_children' when refresh is required (`issue#21670 <http://tracker.ceph.com/issues/21670>`_, `pr#18564 <https://github.com/ceph/ceph/pull/18564>`_, Jason Dillaman)
+* rgw: put bucket policy panics RGW process (`issue#22541 <http://tracker.ceph.com/issues/22541>`_, `pr#19847 <https://github.com/ceph/ceph/pull/19847>`_, Bingyin Zhang)
+* rgw: radosgw-admin reshard command argument error (`issue#21723 <http://tracker.ceph.com/issues/21723>`_, `pr#19502 <https://github.com/ceph/ceph/pull/19502>`_, Yao Zongyou)
+* rgw: radosgw-admin zonegroup get and zone get should return defaults when there is no realm (`issue#21615 <http://tracker.ceph.com/issues/21615>`_, `pr#19086 <https://github.com/ceph/ceph/pull/19086>`_, lvshanchun)
+* rgw: Random 500 errors in Swift PutObject (needs cache fixes) (`issue#22517 <http://tracker.ceph.com/issues/22517>`_, `issue#21560 <http://tracker.ceph.com/issues/21560>`_, `pr#19788 <https://github.com/ceph/ceph/pull/19788>`_, Adam C. Emerson)
+* rgw: refuses upload when Content-Type missing from POST policy (`issue#20201 <http://tracker.ceph.com/issues/20201>`_, `pr#19867 <https://github.com/ceph/ceph/pull/19867>`_, Matt Benjamin)
+* rgw: revert PR #8765 (`issue#22364 <http://tracker.ceph.com/issues/22364>`_, `pr#19434 <https://github.com/ceph/ceph/pull/19434>`_, fang.yuxiang)
+* rgw: RGWCrashError: RGW will crash if a putting lc config request does not include an ID tag in the request xml (`issue#21980 <http://tracker.ceph.com/issues/21980>`_, `issue#22006 <http://tracker.ceph.com/issues/22006>`_, `pr#18765 <https://github.com/ceph/ceph/pull/18765>`_, Enming Zhang)
+* rgw: rgw multisite: automated trimming for bucket index logs (`issue#18229 <http://tracker.ceph.com/issues/18229>`_, `pr#20062 <https://github.com/ceph/ceph/pull/20062>`_, Casey Bodley)
+* rgw: RGW: S3 POST policy should not require Content-Type (`issue#20201 <http://tracker.ceph.com/issues/20201>`_, `pr#19784 <https://github.com/ceph/ceph/pull/19784>`_, Matt Benjamin)
+* rgw: rgw segfaults after running radosgw-admin data sync init (`issue#22083 <http://tracker.ceph.com/issues/22083>`_, `pr#19071 <https://github.com/ceph/ceph/pull/19071>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: rgw usage trim only trims a few entries (`issue#22234 <http://tracker.ceph.com/issues/22234>`_, `pr#19636 <https://github.com/ceph/ceph/pull/19636>`_, Abhishek Lekshmanan)
+* rgw: S3 API Policy Conditions IpAddress and NotIpAddress do not work (`issue#20931 <http://tracker.ceph.com/issues/20931>`_, `issue#20991 <http://tracker.ceph.com/issues/20991>`_, `pr#19819 <https://github.com/ceph/ceph/pull/19819>`_, John Gibson, yuliyang, Casey Bodley, Abhishek Lekshmanan, Jiaying Ren)
+* rgw: Segmentation fault when starting radosgw after reverting .rgw.root (`issue#21996 <http://tracker.ceph.com/issues/21996>`_, `pr#18764 <https://github.com/ceph/ceph/pull/18764>`_, Orit Wasserman, Casey Bodley)
+* rgw: set sync_from_all as true when no value is seen (`issue#22062 <http://tracker.ceph.com/issues/22062>`_, `pr#19038 <https://github.com/ceph/ceph/pull/19038>`_, Abhishek Lekshmanan)
+* rgw: unlink deleted bucket from bucket's owner (`issue#22248 <http://tracker.ceph.com/issues/22248>`_, `pr#20357 <https://github.com/ceph/ceph/pull/20357>`_, Casey Bodley)
+* rgw: user stats increased after bucket reshard (`issue#22124 <http://tracker.ceph.com/issues/22124>`_, `pr#19538 <https://github.com/ceph/ceph/pull/19538>`_, Orit Wasserman)
+* rgw: When a system object is created exclusively, do not distribute the (`issue#22792 <http://tracker.ceph.com/issues/22792>`_, `pr#20107 <https://github.com/ceph/ceph/pull/20107>`_, J. Eric Ivancich, Robin H. Johnson)
+* tests: ceph_test_cls_log failures related to cls_cxx_subop_version() (`issue#21964 <http://tracker.ceph.com/issues/21964>`_, `pr#18715 <https://github.com/ceph/ceph/pull/18715>`_, Casey Bodley)
+* tests: ceph_test_objectstore fails ObjectStore/StoreTest.Synthetic/1 (filestore) buffer content mismatch (`issue#21712 <http://tracker.ceph.com/issues/21712>`_, `issue#21818 <http://tracker.ceph.com/issues/21818>`_, `pr#18742 <https://github.com/ceph/ceph/pull/18742>`_, Sage Weil)
+* tests: configure zabbix properly before selftest (`issue#22514 <http://tracker.ceph.com/issues/22514>`_, `pr#19831 <https://github.com/ceph/ceph/pull/19831>`_, John Spray)
+* tests: do not configure ec data pool with memstore (`issue#22436 <http://tracker.ceph.com/issues/22436>`_, `pr#19628 <https://github.com/ceph/ceph/pull/19628>`_, Patrick Donnelly)
+* tests: force backfill test can conflict with pool removal (`issue#22614 <http://tracker.ceph.com/issues/22614>`_, `pr#19966 <https://github.com/ceph/ceph/pull/19966>`_, Sage Weil)
+* tests: full flag not set on osdmap for tasks.cephfs.test_full (`issue#22475 <http://tracker.ceph.com/issues/22475>`_, `pr#19962 <https://github.com/ceph/ceph/pull/19962>`_, Patrick Donnelly)
+* tests: increase osd count for ec testing (`issue#22646 <http://tracker.ceph.com/issues/22646>`_, `pr#19976 <https://github.com/ceph/ceph/pull/19976>`_, Patrick Donnelly)
+* tests - Initial checkin for luminous point-to-point upgrade (`issue#22048 <http://tracker.ceph.com/issues/22048>`_, `pr#18771 <https://github.com/ceph/ceph/pull/18771>`_, Yuri Weinstein)
+* tests: qa/workunits/rbd: simplify split-brain test to avoid potential race (`issue#22485 <http://tracker.ceph.com/issues/22485>`_, `pr#20205 <https://github.com/ceph/ceph/pull/20205>`_, Jason Dillaman)
+* tests: qa/workunits/rbd: switch devstack to pike release (`issue#22786 <http://tracker.ceph.com/issues/22786>`_, `pr#20136 <https://github.com/ceph/ceph/pull/20136>`_, Jason Dillaman)
+* tests: rbd_mirror_helpers.sh request_resync_image function saves image id to wrong variable (`issue#21663 <http://tracker.ceph.com/issues/21663>`_, `pr#19802 <https://github.com/ceph/ceph/pull/19802>`_, Jason Dillaman)
+* tools/ceph_monstore_tool: include mgrmap in initial paxos epoch (`issue#22266 <http://tracker.ceph.com/issues/22266>`_, `pr#20116 <https://github.com/ceph/ceph/pull/20116>`_, Kefu Chai)
+* tools: ceph-monstore-tool --readable mode doesn't understand FSMap, MgrMap (`issue#21577 <http://tracker.ceph.com/issues/21577>`_, `pr#18754 <https://github.com/ceph/ceph/pull/18754>`_, John Spray)
+* tools: ceph-objectstore-tool: Add option dump-import to examine an export (`issue#22086 <http://tracker.ceph.com/issues/22086>`_, `pr#19487 <https://github.com/ceph/ceph/pull/19487>`_, David Zafman)
+* tools: ceph_objectstore_tool: no flush before collection_empty() calls; ObjectStore/StoreTest.SimpleAttrTest/2 fails (`issue#22409 <http://tracker.ceph.com/issues/22409>`_, `pr#19967 <https://github.com/ceph/ceph/pull/19967>`_, Igor Fedotov)
+* tools: ceph-objectstore-tool set-size should clear data-digest (`issue#22112 <http://tracker.ceph.com/issues/22112>`_, `pr#20069 <https://github.com/ceph/ceph/pull/20069>`_, David Zafman)
+* tools/crushtool: skip device id if no name exists (`issue#22117 <http://tracker.ceph.com/issues/22117>`_, `pr#19039 <https://github.com/ceph/ceph/pull/19039>`_, Jan Fajerski)
+
+
+v12.2.2 Luminous
+================
+
+This is the second bugfix release of Luminous v12.2.x long term stable release
+series. It contains a range of bug fixes and a few features across Bluestore,
+CephFS, RBD & RGW. We recommend all the users of 12.2.x series update.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v12.2.2.txt>`.
+
+
+Notable Changes
+---------------
+
+* Standby ceph-mgr daemons now redirect requests to the active messenger, easing
+  configuration for tools & users accessing the web dashboard, restful API, or
+  other ceph-mgr module services.
+* The prometheus module has several significant updates and improvements.
+* The new balancer module enables automatic optimization of CRUSH weights to
+  balance data across the cluster.
+* The ceph-volume tool has been updated to include support for BlueStore as well
+  as FileStore. The only major missing ceph-volume feature is dm-crypt support.
+* RGW's dynamic bucket index resharding is disabled in multisite environments,
+  as it can cause inconsistencies in replication of bucket indexes to remote
+  sites.
+
+Other Notable Changes
+---------------------
+
+* build/ops: bump sphinx to 1.6 (`issue#21717 <http://tracker.ceph.com/issues/21717>`_, `pr#18167 <https://github.com/ceph/ceph/pull/18167>`_, Kefu Chai, Alfredo Deza)
+* build/ops: macros expanding in spec file comment (`issue#22250 <http://tracker.ceph.com/issues/22250>`_, `pr#19173 <https://github.com/ceph/ceph/pull/19173>`_, Ken Dreyer)
+* build/ops: python-numpy-devel build dependency for SUSE (`issue#21176 <http://tracker.ceph.com/issues/21176>`_, `pr#17692 <https://github.com/ceph/ceph/pull/17692>`_, Nathan Cutler)
+* build/ops: selinux: Allow getattr on lnk sysfs files (`issue#21492 <http://tracker.ceph.com/issues/21492>`_, `pr#18650 <https://github.com/ceph/ceph/pull/18650>`_, Boris Ranto)
+* build/ops: Ubuntu amd64 client can not discover the ubuntu arm64 ceph cluster (`issue#19705 <http://tracker.ceph.com/issues/19705>`_, `pr#18293 <https://github.com/ceph/ceph/pull/18293>`_, Kefu Chai)
+* core: buffer: fix ABI breakage by removing list _mempool member (`issue#21573 <http://tracker.ceph.com/issues/21573>`_, `pr#18491 <https://github.com/ceph/ceph/pull/18491>`_, Sage Weil)
+* core: Daemons(OSD, Mon...) exit abnormally at injectargs command (`issue#21365 <http://tracker.ceph.com/issues/21365>`_, `pr#17864 <https://github.com/ceph/ceph/pull/17864>`_, Yan Jun)
+* core: Disable messenger logging (debug ms = 0/0) for clients unless overridden (`issue#21860 <http://tracker.ceph.com/issues/21860>`_, `pr#18529 <https://github.com/ceph/ceph/pull/18529>`_, Jason Dillaman)
+* core: Improve OSD startup time by only scanning for omap corruption once (`issue#21328 <http://tracker.ceph.com/issues/21328>`_, `pr#17889 <https://github.com/ceph/ceph/pull/17889>`_, Luo Kexue, David Zafman)
+* core: upmap does not respect osd reweights (`issue#21538 <http://tracker.ceph.com/issues/21538>`_, `pr#18699 <https://github.com/ceph/ceph/pull/18699>`_, Theofilos Mouratidis)
+* dashboard: barfs on nulls where it expects numbers (`issue#21570 <http://tracker.ceph.com/issues/21570>`_, `pr#18728 <https://github.com/ceph/ceph/pull/18728>`_, John Spray)
+* dashboard: OSD list has servers and osds in arbitrary order (`issue#21572 <http://tracker.ceph.com/issues/21572>`_, `pr#18736 <https://github.com/ceph/ceph/pull/18736>`_, John Spray)
+* dashboard: the dashboard uses absolute links for filesystems and clients (`issue#20568 <http://tracker.ceph.com/issues/20568>`_, `pr#18737 <https://github.com/ceph/ceph/pull/18737>`_, Nick Erdmann)
+* filestore: set default readahead and compaction threads for rocksdb (`issue#21505 <http://tracker.ceph.com/issues/21505>`_, `pr#18234 <https://github.com/ceph/ceph/pull/18234>`_, Josh Durgin, Mark Nelson)
+* librbd: object map batch update might cause OSD suicide timeout (`issue#21797 <http://tracker.ceph.com/issues/21797>`_, `pr#18416 <https://github.com/ceph/ceph/pull/18416>`_, Jason Dillaman)
+* librbd: snapshots should be created/removed against data pool (`issue#21567 <http://tracker.ceph.com/issues/21567>`_, `pr#18336 <https://github.com/ceph/ceph/pull/18336>`_, Jason Dillaman)
+* mds: make sure snap inode's last matches its parent dentry's last (`issue#21337 <http://tracker.ceph.com/issues/21337>`_, `pr#17994 <https://github.com/ceph/ceph/pull/17994>`_, "Yan, Zheng")
+* mds: sanitize mdsmap of removed pools (`issue#21945 <http://tracker.ceph.com/issues/21945>`_, `issue#21568 <http://tracker.ceph.com/issues/21568>`_, `pr#18628 <https://github.com/ceph/ceph/pull/18628>`_, Patrick Donnelly)
+* mgr: bulk backport of ceph-mgr improvements (`issue#21594 <http://tracker.ceph.com/issues/21594>`_, `issue#17460 <http://tracker.ceph.com/issues/17460>`_, `issue#21197 <http://tracker.ceph.com/issues/21197>`_, `issue#21158 <http://tracker.ceph.com/issues/21158>`_, `issue#21593 <http://tracker.ceph.com/issues/21593>`_, `pr#18675 <https://github.com/ceph/ceph/pull/18675>`_, Benjeman Meekhof, Sage Weil, Jan Fajerski, John Spray, Kefu Chai, My Do, Spandan Kumar Sahu)
+* mgr: ceph-mgr gets process called "exe" after respawn (`issue#21404 <http://tracker.ceph.com/issues/21404>`_, `pr#18738 <https://github.com/ceph/ceph/pull/18738>`_, John Spray)
+* mgr: fix crashable DaemonStateIndex::get calls (`issue#17737 <http://tracker.ceph.com/issues/17737>`_, `pr#18412 <https://github.com/ceph/ceph/pull/18412>`_, John Spray)
+* mgr: key mismatch for mgr after upgrade from jewel to luminous(dev) (`issue#20950 <http://tracker.ceph.com/issues/20950>`_, `pr#18727 <https://github.com/ceph/ceph/pull/18727>`_, John Spray)
+* mgr: mgr status module uses base 10 units (`issue#21189 <http://tracker.ceph.com/issues/21189>`_, `issue#21752 <http://tracker.ceph.com/issues/21752>`_, `pr#18257 <https://github.com/ceph/ceph/pull/18257>`_, John Spray, Yanhu Cao)
+* mgr: mgr[zabbix] float division by zero (`issue#21518 <http://tracker.ceph.com/issues/21518>`_, `pr#18734 <https://github.com/ceph/ceph/pull/18734>`_, John Spray)
+* mgr: Prometheus crash when update (`issue#21253 <http://tracker.ceph.com/issues/21253>`_, `pr#17867 <https://github.com/ceph/ceph/pull/17867>`_, John Spray)
+* mgr: prometheus module generates invalid output when counter names contain non-alphanum characters (`issue#20899 <http://tracker.ceph.com/issues/20899>`_, `pr#17868 <https://github.com/ceph/ceph/pull/17868>`_, John Spray, Jeremy H Austin)
+* mgr: Quieten scary RuntimeError from restful module on startup (`issue#21292 <http://tracker.ceph.com/issues/21292>`_, `pr#17866 <https://github.com/ceph/ceph/pull/17866>`_, John Spray)
+* mgr: Spurious ceph-mgr failovers during mon elections (`issue#20629 <http://tracker.ceph.com/issues/20629>`_, `pr#18726 <https://github.com/ceph/ceph/pull/18726>`_, John Spray)
+* mon: Client client.admin marked osd.2 out, after it was down for 1504627577 seconds (`issue#21249 <http://tracker.ceph.com/issues/21249>`_, `pr#17862 <https://github.com/ceph/ceph/pull/17862>`_, John Spray)
+* mon: DNS SRV default service name not used anymore (`issue#21204 <http://tracker.ceph.com/issues/21204>`_, `pr#17863 <https://github.com/ceph/ceph/pull/17863>`_, Kefu Chai)
+* mon/MgrMonitor: handle cmd descs to/from disk in the absence of active mgr (`issue#21300 <http://tracker.ceph.com/issues/21300>`_, `pr#18038 <https://github.com/ceph/ceph/pull/18038>`_, Joao Eduardo Luis)
+* mon/mgr: sync "mgr_command_descs","osd_metadata" and "mgr_metadata" prefixes to new mons (`issue#21527 <http://tracker.ceph.com/issues/21527>`_, `pr#18620 <https://github.com/ceph/ceph/pull/18620>`_, huanwen ren)
+* mon: osd feature checks with 0 up osds (`issue#21471 <http://tracker.ceph.com/issues/21471>`_, `issue#20751 <http://tracker.ceph.com/issues/20751>`_, `pr#18364 <https://github.com/ceph/ceph/pull/18364>`_, Brad Hubbard, Sage Weil)
+* mon,osd: fix "pg ls {forced_backfill, backfilling}" (`issue#21609 <http://tracker.ceph.com/issues/21609>`_, `pr#18236 <https://github.com/ceph/ceph/pull/18236>`_, Kefu Chai)
+* mon/OSDMonitor: add option to fix up ruleset-\* to crush-\* for ec profiles (`issue#22128 <http://tracker.ceph.com/issues/22128>`_, `pr#18945 <https://github.com/ceph/ceph/pull/18945>`_, Sage Weil)
+* mon, osd: per pool space-full flag support (`issue#21409 <http://tracker.ceph.com/issues/21409>`_, `pr#17730 <https://github.com/ceph/ceph/pull/17730>`_, xie xingguo)
+* mon/PGMap: Fix %USED calculation (`issue#22247 <http://tracker.ceph.com/issues/22247>`_, `pr#19230 <https://github.com/ceph/ceph/pull/19230>`_, Xiaoxi Chen)
+* mon: update get_store_prefixes implementations (`issue#21534 <http://tracker.ceph.com/issues/21534>`_, `pr#18621 <https://github.com/ceph/ceph/pull/18621>`_, John Spray, huanwen ren)
+* msgr: messages/MOSDMap: do compat reencode of crush map, too (`issue#21882 <http://tracker.ceph.com/issues/21882>`_, `pr#18456 <https://github.com/ceph/ceph/pull/18456>`_, Sage Weil)
+* msgr: src/messages/MOSDMap: reencode OSDMap for older clients (`issue#21660 <http://tracker.ceph.com/issues/21660>`_, `pr#18140 <https://github.com/ceph/ceph/pull/18140>`_, Sage Weil)
+* os/bluestore/BlueFS: fix race with log flush during async log compaction (`issue#21878 <http://tracker.ceph.com/issues/21878>`_, `pr#18503 <https://github.com/ceph/ceph/pull/18503>`_, Sage Weil)
+* os/bluestore: fix another aio stall/deadlock (`issue#21470 <http://tracker.ceph.com/issues/21470>`_, `pr#18127 <https://github.com/ceph/ceph/pull/18127>`_, Sage Weil)
+* os/bluestore: fix SharedBlob unregistration (`issue#22039 <http://tracker.ceph.com/issues/22039>`_, `pr#18983 <https://github.com/ceph/ceph/pull/18983>`_, Sage Weil)
+* os/bluestore: handle compressed extents in blob unsharing checks (`issue#21766 <http://tracker.ceph.com/issues/21766>`_, `pr#18501 <https://github.com/ceph/ceph/pull/18501>`_, Sage Weil)
+* os/bluestore: replace 21089 repair with something online (instead of fsck) (`issue#21089 <http://tracker.ceph.com/issues/21089>`_, `pr#17734 <https://github.com/ceph/ceph/pull/17734>`_, Sage Weil)
+* os/bluestore: set bitmap freelist resolution to min_alloc_size (`issue#21408 <http://tracker.ceph.com/issues/21408>`_, `pr#18050 <https://github.com/ceph/ceph/pull/18050>`_, Sage Weil)
+* os/blueStore::umount will crash when the BlueStore is opened by start_kv_only() (`issue#21624 <http://tracker.ceph.com/issues/21624>`_, `pr#18750 <https://github.com/ceph/ceph/pull/18750>`_, Chang Liu)
+* osd: additional protection for out-of-bounds EC reads (`issue#21629 <http://tracker.ceph.com/issues/21629>`_, `pr#18413 <https://github.com/ceph/ceph/pull/18413>`_, Jason Dillaman)
+* osd: allow recovery preemption (`issue#21613 <http://tracker.ceph.com/issues/21613>`_, `pr#18025 <https://github.com/ceph/ceph/pull/18025>`_, Sage Weil)
+* osd: build_past_intervals_parallel: Ignore new partially created PGs (`issue#21833 <http://tracker.ceph.com/issues/21833>`_, `pr#18673 <https://github.com/ceph/ceph/pull/18673>`_, David Zafman)
+* osd: dump bluestore debug on shutdown if debug option is set (`issue#21259 <http://tracker.ceph.com/issues/21259>`_, `pr#18103 <https://github.com/ceph/ceph/pull/18103>`_, Sage Weil)
+* osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL (`issue#21981 <http://tracker.ceph.com/issues/21981>`_, `pr#18723 <https://github.com/ceph/ceph/pull/18723>`_, Yao Zongyou)
+* osd: make the PG's SORTBITWISE assert a more generous shutdown (`issue#20416 <http://tracker.ceph.com/issues/20416>`_, `pr#18132 <https://github.com/ceph/ceph/pull/18132>`_, Greg Farnum)
+* osd: OSD metadata 'backend_filestore_dev_node' is unknown even for simple deployment (`issue#20944 <http://tracker.ceph.com/issues/20944>`_, `pr#17865 <https://github.com/ceph/ceph/pull/17865>`_, Sage Weil)
+* rbd: [cli] mirror getter commands will fail if mirroring has never been enabled (`issue#21319 <http://tracker.ceph.com/issues/21319>`_, `pr#17861 <https://github.com/ceph/ceph/pull/17861>`_, Jason Dillaman)
+* rbd: cls/journal: fixed possible infinite loop in expire_tags (`issue#21956 <http://tracker.ceph.com/issues/21956>`_, `pr#18626 <https://github.com/ceph/ceph/pull/18626>`_, Jason Dillaman)
+* rbd: cls/journal: possible infinite loop within tag_list class method (`issue#21771 <http://tracker.ceph.com/issues/21771>`_, `pr#18417 <https://github.com/ceph/ceph/pull/18417>`_, Jason Dillaman)
+* rbd: [rbd-mirror] asok hook names not updated when image is renamed (`issue#20860 <http://tracker.ceph.com/issues/20860>`_, `pr#17860 <https://github.com/ceph/ceph/pull/17860>`_, Mykola Golub)
+* rbd: [rbd-mirror] forced promotion can result in incorrect status (`issue#21559 <http://tracker.ceph.com/issues/21559>`_, `pr#18337 <https://github.com/ceph/ceph/pull/18337>`_, Jason Dillaman)
+* rbd: [rbd-mirror] peer cluster connections should filter out command line optionals (`issue#21894 <http://tracker.ceph.com/issues/21894>`_, `pr#18566 <https://github.com/ceph/ceph/pull/18566>`_, Jason Dillaman)
+* rgw: add support for Swift's per storage policy statistics (`issue#17932 <http://tracker.ceph.com/issues/17932>`_, `issue#21506 <http://tracker.ceph.com/issues/21506>`_, `pr#17835 <https://github.com/ceph/ceph/pull/17835>`_, Radoslaw Zarzynski, Casey Bodley)
+* rgw: add support for Swift's reversed account listings (`issue#21148 <http://tracker.ceph.com/issues/21148>`_, `pr#17834 <https://github.com/ceph/ceph/pull/17834>`_, Radoslaw Zarzynski)
+* rgw: avoid logging keystone revocation failures when no keystone is configured (`issue#21400 <http://tracker.ceph.com/issues/21400>`_, `pr#18441 <https://github.com/ceph/ceph/pull/18441>`_, Abhishek Lekshmanan)
+* rgw: disable dynamic resharding in multisite enviorment (`issue#21725 <http://tracker.ceph.com/issues/21725>`_, `pr#18432 <https://github.com/ceph/ceph/pull/18432>`_, Orit Wasserman)
+* rgw: encryption: PutObj response does not include sse-kms headers (`issue#21576 <http://tracker.ceph.com/issues/21576>`_, `pr#18442 <https://github.com/ceph/ceph/pull/18442>`_, Casey Bodley)
+* rgw: encryption: reject requests that don't provide all expected headers (`issue#21581 <http://tracker.ceph.com/issues/21581>`_, `pr#18429 <https://github.com/ceph/ceph/pull/18429>`_, Enming Zhang)
+* rgw: expose --sync-stats via admin api (`issue#21301 <http://tracker.ceph.com/issues/21301>`_, `pr#18439 <https://github.com/ceph/ceph/pull/18439>`_, Nathan Johnson)
+* rgw: failed CompleteMultipartUpload request does not release lock (`issue#21596 <http://tracker.ceph.com/issues/21596>`_, `pr#18430 <https://github.com/ceph/ceph/pull/18430>`_, Matt Benjamin)
+* rgw_file:  set s->obj_size from bytes_written (`issue#21940 <http://tracker.ceph.com/issues/21940>`_, `pr#18599 <https://github.com/ceph/ceph/pull/18599>`_, Matt Benjamin)
+* rgw: fix a bug about inconsistent unit of comparison (`issue#21590 <http://tracker.ceph.com/issues/21590>`_, `pr#18438 <https://github.com/ceph/ceph/pull/18438>`_, gaosibei)
+* rgw: fix bilog entries on multipart complete (`issue#21772 <http://tracker.ceph.com/issues/21772>`_, `pr#18334 <https://github.com/ceph/ceph/pull/18334>`_, Casey Bodley)
+* rgw: fix error handling in ListBucketIndexesCR (`issue#21735 <http://tracker.ceph.com/issues/21735>`_, `pr#18591 <https://github.com/ceph/ceph/pull/18591>`_, Casey Bodley)
+* rgw: fix refcnt issues (`issue#21819 <http://tracker.ceph.com/issues/21819>`_, `pr#18539 <https://github.com/ceph/ceph/pull/18539>`_, baixueyu)
+* rgw: lc process only schdule the first item of lc objects (`issue#21022 <http://tracker.ceph.com/issues/21022>`_, `pr#17859 <https://github.com/ceph/ceph/pull/17859>`_, Shasha Lu)
+* rgw: list bucket which enable versioning get wrong result when user marker (`issue#21500 <http://tracker.ceph.com/issues/21500>`_, `pr#18569 <https://github.com/ceph/ceph/pull/18569>`_, yuliyang)
+* rgw: list_objects() honors end_marker regardless of namespace (`issue#18977 <http://tracker.ceph.com/issues/18977>`_, `pr#17832 <https://github.com/ceph/ceph/pull/17832>`_, Radoslaw Zarzynski)
+* rgw: Multipart upload may double the quota (`issue#21586 <http://tracker.ceph.com/issues/21586>`_, `pr#18435 <https://github.com/ceph/ceph/pull/18435>`_, Sibei Gao)
+* rgw: multisite: Get bucket location which is located in another zonegroup, will return 301 Moved Permanently (`issue#21125 <http://tracker.ceph.com/issues/21125>`_, `pr#17857 <https://github.com/ceph/ceph/pull/17857>`_, Shasha Lu)
+* rgw: multisite: race between sync of bucket and bucket instance metadata (`issue#21990 <http://tracker.ceph.com/issues/21990>`_, `pr#18767 <https://github.com/ceph/ceph/pull/18767>`_, Casey Bodley)
+* rgw: policy checks missing from Get/SetRequestPayment operations (`issue#21389 <http://tracker.ceph.com/issues/21389>`_, `pr#18440 <https://github.com/ceph/ceph/pull/18440>`_, Adam C. Emerson)
+* rgw: radosgw-admin usage show loops indefinitly (`issue#21196 <http://tracker.ceph.com/issues/21196>`_, `pr#18437 <https://github.com/ceph/ceph/pull/18437>`_, Mark Kogan)
+* rgw: rgw_file: explicit NFSv3 open() emulation (`issue#21854 <http://tracker.ceph.com/issues/21854>`_, `pr#18446 <https://github.com/ceph/ceph/pull/18446>`_, Matt Benjamin)
+* rgw: rgw_file: fix write error when the write offset overlaps (`issue#21455 <http://tracker.ceph.com/issues/21455>`_, `pr#18004 <https://github.com/ceph/ceph/pull/18004>`_, Yao Zongyou)
+* rgw: rgw file write error (`issue#21455 <http://tracker.ceph.com/issues/21455>`_, `pr#18433 <https://github.com/ceph/ceph/pull/18433>`_, Yao Zongyou)
+* rgw: s3:GetBucketCORS/s3:PutBucketCORS policy fails with 403 (`issue#21578 <http://tracker.ceph.com/issues/21578>`_, `pr#18444 <https://github.com/ceph/ceph/pull/18444>`_, Adam C. Emerson)
+* rgw:  s3:GetBucketLocation bucket policy fails with 403 (`issue#21582 <http://tracker.ceph.com/issues/21582>`_, `pr#18443 <https://github.com/ceph/ceph/pull/18443>`_, Adam C. Emerson)
+* rgw: s3:GetBucketWebsite/PutBucketWebsite fails with 403 (`issue#21597 <http://tracker.ceph.com/issues/21597>`_, `pr#18445 <https://github.com/ceph/ceph/pull/18445>`_, Adam C. Emerson)
+* rgw: setxattrs call leads to different mtimes for bucket index and object (`issue#21200 <http://tracker.ceph.com/issues/21200>`_, `pr#17856 <https://github.com/ceph/ceph/pull/17856>`_, Abhishek Lekshmanan)
+* rgw: stop/join TokenCache revoke thread only if started (`issue#21666 <http://tracker.ceph.com/issues/21666>`_, `pr#18138 <https://github.com/ceph/ceph/pull/18138>`_, Karol Mroz)
+* rgw: string_view instance points to expired memory in PrefixableSignatureHelper (`issue#21085 <http://tracker.ceph.com/issues/21085>`_, `pr#17858 <https://github.com/ceph/ceph/pull/17858>`_, Radoslaw Zarzynski)
+* rgw: user creation can overwrite existing user even if different uid is given (`issue#21685 <http://tracker.ceph.com/issues/21685>`_, `pr#18436 <https://github.com/ceph/ceph/pull/18436>`_, Casey Bodley)
+* rgw: We cant't get torrents if  objects are encrypted using SSE-C (`issue#21720 <http://tracker.ceph.com/issues/21720>`_, `pr#18431 <https://github.com/ceph/ceph/pull/18431>`_, Zhang Shaowen)
+* rgw: wrong error message is returned when putting container with a name that is too long (`issue#17938 <http://tracker.ceph.com/issues/17938>`_, `issue#21169 <http://tracker.ceph.com/issues/21169>`_, `issue#17935 <http://tracker.ceph.com/issues/17935>`_, `issue#17934 <http://tracker.ceph.com/issues/17934>`_, `issue#17936 <http://tracker.ceph.com/issues/17936>`_, `pr#17811 <https://github.com/ceph/ceph/pull/17811>`_, Radoslaw Zarzynski)
+* rgw: zone compression type is not validated (`issue#21775 <http://tracker.ceph.com/issues/21775>`_, `pr#18434 <https://github.com/ceph/ceph/pull/18434>`_, Casey Bodley)
+* tools: ceph-disk create deprecation warnings (`issue#22154 <http://tracker.ceph.com/issues/22154>`_, `pr#18989 <https://github.com/ceph/ceph/pull/18989>`_, Alfredo Deza)
+* tools: ceph-disk: fix '--runtime' omission for ceph-osd service (`issue#21498 <http://tracker.ceph.com/issues/21498>`_, `pr#17914 <https://github.com/ceph/ceph/pull/17914>`_, Carl Xiong)
+* tools: ceph-disk flake8 test fails on very old, and very new, versions of flake8 (`issue#22207 <http://tracker.ceph.com/issues/22207>`_, `pr#19152 <https://github.com/ceph/ceph/pull/19152>`_, Nathan Cutler)
+* tools: ceph-disk: retry on OSError (`issue#21728 <http://tracker.ceph.com/issues/21728>`_, `pr#18189 <https://github.com/ceph/ceph/pull/18189>`_, Kefu Chai)
+* tools: ceph-disk: unlocks dmcrypted partitions when activating them (`issue#20488 <http://tracker.ceph.com/issues/20488>`_, `pr#18625 <https://github.com/ceph/ceph/pull/18625>`_, Kefu Chai, Felix Winterhalter)
+* tools: ceph-kvstore-tool does not call bluestore's umount when exit (`issue#21625 <http://tracker.ceph.com/issues/21625>`_, `pr#18751 <https://github.com/ceph/ceph/pull/18751>`_, Chang Liu)
+* tools: ceph_monstore_tool: rebuild initial mgrmap also (`issue#22266 <http://tracker.ceph.com/issues/22266>`_, `pr#19240 <https://github.com/ceph/ceph/pull/19240>`_, Kefu Chai)
+* tools: ceph-objectstore-tool and ceph-bluestore-tool: backports from master (`issue#21272 <http://tracker.ceph.com/issues/21272>`_, `pr#17896 <https://github.com/ceph/ceph/pull/17896>`_, Sage Weil, David Zafman)
+* tools: ceph_volume_client: add get, put, and delete object interfaces (`issue#21601 <http://tracker.ceph.com/issues/21601>`_, `pr#18037 <https://github.com/ceph/ceph/pull/18037>`_, Ramana Raja)
+* tools: cli/crushtools/build.t sometimes fails in jenkins' make check run (`issue#21758 <http://tracker.ceph.com/issues/21758>`_, `pr#18398 <https://github.com/ceph/ceph/pull/18398>`_, Kefu Chai, Sage Weil)
+
+
+
+v12.2.1 Luminous
+================
+
+This is the first bugfix release of Luminous v12.2.x long term stable
+release series. It contains a range of bug fixes and a few features
+across CephFS, RBD & RGW. We recommend all the users of 12.2.x series
+update.
+
+For more detailed information, see :download:`the complete changelog <../changelog/v12.2.1.txt>`.
+
+
+Notable Changes
+---------------
+
+* Dynamic resharding is now enabled by default for RGW, RGW will now
+  automatically reshard there bucket index once the index grows beyond
+  `rgw_max_objs_per_shard`
+
+* Limiting MDS cache via a memory limit is now supported using the new
+  mds_cache_memory_limit config option (1GB by default).  A cache reservation
+  can also be specified using mds_cache_reservation as a percentage of the
+  limit (5% by default). Limits by inode count are still supported using
+  mds_cache_size. Setting mds_cache_size to 0 (the default) disables the
+  inode limit.
+
+* The maximum number of PGs per OSD before the monitor issues a
+  warning has been reduced from 300 to 200 PGs.  200 is still twice
+  the generally recommended target of 100 PGs per OSD.  This limit can
+  be adjusted via the ``mon_max_pg_per_osd`` option on the
+  monitors.  The older ``mon_pg_warn_max_per_osd`` option has been removed.
+
+* Creating pools or adjusting pg_num will now fail if the change would
+  make the number of PGs per OSD exceed the configured
+  ``mon_max_pg_per_osd`` limit.  The option can be adjusted if it
+  is really necessary to create a pool with more PGs.
+
+* There was a bug in the PG mapping behavior of the new *upmap*
+  feature. If you made use of this feature (e.g., via the `ceph osd
+  pg-upmap-items` command), we recommend that all mappings be removed (via
+  the `ceph osd rm-pg-upmap-items` command) before upgrading to this
+  point release.
+
+* A stall in BlueStore IO submission that was affecting many users has
+  been resolved.
+
+Other Notable Changes
+---------------------
+
+* bluestore: asyn cdeferred_try_submit deadlock (`issue#21207 <http://tracker.ceph.com/issues/21207>`_, `pr#17494 <https://github.com/ceph/ceph/pull/17494>`_, Sage Weil)
+* bluestore: fix deferred write deadlock, aio short return handling (`issue#21171 <http://tracker.ceph.com/issues/21171>`_, `pr#17601 <https://github.com/ceph/ceph/pull/17601>`_, Sage Weil)
+* bluestore: osd crash when change option bluestore_csum_type from none to CRC32 (`issue#21175 <http://tracker.ceph.com/issues/21175>`_, `pr#17497 <https://github.com/ceph/ceph/pull/17497>`_, xie xingguo)
+* bluestore: os/bluestore/BlueFS.cc: 1255: FAILED assert(!log_file->fnode.extents.empty()) (`issue#21250 <http://tracker.ceph.com/issues/21250>`_, `pr#17562 <https://github.com/ceph/ceph/pull/17562>`_, Sage Weil)
+* build/ops: ceph-fuse RPM should require fusermount (`issue#21057 <http://tracker.ceph.com/issues/21057>`_, `pr#17470 <https://github.com/ceph/ceph/pull/17470>`_, Ken Dreyer)
+* build/ops: RHEL 7.3 Selinux denials at OSD start (`issue#19200 <http://tracker.ceph.com/issues/19200>`_, `pr#17468 <https://github.com/ceph/ceph/pull/17468>`_, Boris Ranto)
+* build/ops: rocksdb,cmake:  build portable binaries (`issue#20529 <http://tracker.ceph.com/issues/20529>`_, `pr#17745 <https://github.com/ceph/ceph/pull/17745>`_, Kefu Chai)
+* cephfs: client/mds has wrong check to clear S_ISGID on chown (`issue#21004 <http://tracker.ceph.com/issues/21004>`_, `pr#17471 <https://github.com/ceph/ceph/pull/17471>`_, Patrick Donnelly)
+* cephfs: get_quota_root sends lookupname op for every buffered write (`issue#20945 <http://tracker.ceph.com/issues/20945>`_, `pr#17473 <https://github.com/ceph/ceph/pull/17473>`_, Dan van der Ster)
+* cephfs: MDCache::try_subtree_merge() may print N^2 lines of debug message (`issue#21221 <http://tracker.ceph.com/issues/21221>`_, `pr#17712 <https://github.com/ceph/ceph/pull/17712>`_, Patrick Donnelly)
+* cephfs: MDS rank add/remove log messages say wrong number of ranks (`issue#21421 <http://tracker.ceph.com/issues/21421>`_, `pr#17887 <https://github.com/ceph/ceph/pull/17887>`_, John Spray)
+* cephfs: MDS: standby-replay mds should avoid initiating subtree export (`issue#21378 <http://tracker.ceph.com/issues/21378>`_, `issue#21222 <http://tracker.ceph.com/issues/21222>`_, `pr#17714 <https://github.com/ceph/ceph/pull/17714>`_, "Yan, Zheng", Jianyu Li)
+* cephfs: the standbys are not updated via ceph tell mds.\* command (`issue#21230 <http://tracker.ceph.com/issues/21230>`_, `pr#17565 <https://github.com/ceph/ceph/pull/17565>`_, Kefu Chai)
+* common: adding line break at end of some cli results (`issue#21019 <http://tracker.ceph.com/issues/21019>`_, `pr#17467 <https://github.com/ceph/ceph/pull/17467>`_, songweibin)
+* core: [cls] metadata_list API function does not honor `max_return` parameter (`issue#21247 <http://tracker.ceph.com/issues/21247>`_, `pr#17558 <https://github.com/ceph/ceph/pull/17558>`_, Jason Dillaman)
+* core: incorrect erasure-code space in command ceph df (`issue#21243 <http://tracker.ceph.com/issues/21243>`_, `pr#17724 <https://github.com/ceph/ceph/pull/17724>`_, liuchang0812)
+* core: interval_set: optimize intersect_of insert operations (`issue#21229 <http://tracker.ceph.com/issues/21229>`_, `pr#17487 <https://github.com/ceph/ceph/pull/17487>`_, Zac Medico)
+* core: osd crush rule rename not idempotent (`issue#21162 <http://tracker.ceph.com/issues/21162>`_, `pr#17481 <https://github.com/ceph/ceph/pull/17481>`_, xie xingguo)
+* core: osd/PGLog: write only changed dup entries (`issue#21026 <http://tracker.ceph.com/issues/21026>`_, `pr#17378 <https://github.com/ceph/ceph/pull/17378>`_, Josh Durgin)
+* doc: doc/rbd: iSCSI Gateway Documentation (`issue#20437 <http://tracker.ceph.com/issues/20437>`_, `pr#17381 <https://github.com/ceph/ceph/pull/17381>`_, Aron Gunn, Jason Dillaman)
+* mds: fix 'dirfrag end' check in Server::handle_client_readdir (`issue#21070 <http://tracker.ceph.com/issues/21070>`_, `pr#17686 <https://github.com/ceph/ceph/pull/17686>`_, "Yan, Zheng")
+* mds: support limiting cache by memory (`issue#20594 <http://tracker.ceph.com/issues/20594>`_, `pr#17711 <https://github.com/ceph/ceph/pull/17711>`_, "Yan, Zheng", Patrick Donnelly)
+* mgr: 500 error when attempting to view filesystem data (`issue#20692 <http://tracker.ceph.com/issues/20692>`_, `pr#17477 <https://github.com/ceph/ceph/pull/17477>`_, John Spray)
+* mgr: ceph mgr versions shows active mgr as Unknown (`issue#21260 <http://tracker.ceph.com/issues/21260>`_, `pr#17635 <https://github.com/ceph/ceph/pull/17635>`_, John Spray)
+* mgr: Crash in MonCommandCompletion (`issue#21157 <http://tracker.ceph.com/issues/21157>`_, `pr#17483 <https://github.com/ceph/ceph/pull/17483>`_, John Spray)
+* mon: mon/OSDMonitor: deleting pool while pgs are being created leads to assert(p != pools.end) in update_creating_pgs() (`issue#21309 <http://tracker.ceph.com/issues/21309>`_, `pr#17634 <https://github.com/ceph/ceph/pull/17634>`_, Joao Eduardo Luis)
+* mon: OSDMonitor: osd pool application get support (`issue#20976 <http://tracker.ceph.com/issues/20976>`_, `pr#17472 <https://github.com/ceph/ceph/pull/17472>`_, xie xingguo)
+* mon: rate limit on health check update logging (`issue#20888 <http://tracker.ceph.com/issues/20888>`_, `pr#17500 <https://github.com/ceph/ceph/pull/17500>`_, John Spray)
+* osd: build_initial_pg_history doesn't update up/acting/etc (`issue#21203 <http://tracker.ceph.com/issues/21203>`_, `pr#17496 <https://github.com/ceph/ceph/pull/17496>`_, w11979, Sage Weil)
+* osd: osd/PG: discard msgs from down peers (`issue#19605 <http://tracker.ceph.com/issues/19605>`_, `pr#17501 <https://github.com/ceph/ceph/pull/17501>`_, Kefu Chai)
+* osd/PrimaryLogPG: request osdmap update in the right block (`issue#21428 <http://tracker.ceph.com/issues/21428>`_, `pr#17829 <https://github.com/ceph/ceph/pull/17829>`_, Josh Durgin)
+* osd: PrimaryLogPG: sparse read won't trigger repair correctly (`issue#21123 <http://tracker.ceph.com/issues/21123>`_, `pr#17475 <https://github.com/ceph/ceph/pull/17475>`_, xie xingguo)
+* osd: request new map from PG when needed (`issue#21428 <http://tracker.ceph.com/issues/21428>`_, `pr#17796 <https://github.com/ceph/ceph/pull/17796>`_, Josh Durgin)
+* osd: Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items" (`issue#21410 <http://tracker.ceph.com/issues/21410>`_, `pr#17812 <https://github.com/ceph/ceph/pull/17812>`_, Sage Weil)
+* osd: subscribe to new osdmap while waiting_for_healthy (`issue#21121 <http://tracker.ceph.com/issues/21121>`_, `pr#17498 <https://github.com/ceph/ceph/pull/17498>`_, Sage Weil)
+* osd: update info only if new_interval (`issue#21203 <http://tracker.ceph.com/issues/21203>`_, `pr#17622 <https://github.com/ceph/ceph/pull/17622>`_, Kefu Chai)
+* pybind: dashboard usage graph getting bigger and bigger (`issue#20746 <http://tracker.ceph.com/issues/20746>`_, `pr#17486 <https://github.com/ceph/ceph/pull/17486>`_, Yixing Yan)
+* rbd: image-meta list does not return all entries (`issue#21179 <http://tracker.ceph.com/issues/21179>`_, `pr#17561 <https://github.com/ceph/ceph/pull/17561>`_, Jason Dillaman)
+* rbd: some generic options can not be passed by rbd-nbd (`issue#20426 <http://tracker.ceph.com/issues/20426>`_, `pr#17557 <https://github.com/ceph/ceph/pull/17557>`_, Pan Liu)
+* rbd: switch to new config option getter methods (`issue#20737 <http://tracker.ceph.com/issues/20737>`_, `pr#17464 <https://github.com/ceph/ceph/pull/17464>`_, Jason Dillaman)
+* rbd: TestMirroringWatcher.ModeUpdated: periodic failure due to injected message failures (`issue#21029 <http://tracker.ceph.com/issues/21029>`_, `pr#17465 <https://github.com/ceph/ceph/pull/17465>`_, Jason Dillaman)
+* rgw: bucket index sporadically reshards to 65521 shards (`issue#20934 <http://tracker.ceph.com/issues/20934>`_, `pr#17476 <https://github.com/ceph/ceph/pull/17476>`_, Aleksei Gutikov)
+* rgw: bytes_send and bytes_recv in the msg of usage show returning is 0 in master branch (`issue#19870 <http://tracker.ceph.com/issues/19870>`_, `pr#17444 <https://github.com/ceph/ceph/pull/17444>`_, Marcus Watts)
+* rgw: data encryption sometimes fails to follow AWS settings (`issue#21349 <http://tracker.ceph.com/issues/21349>`_, `pr#17642 <https://github.com/ceph/ceph/pull/17642>`_, hechuang)
+* rgw: memory leak in MetadataHandlers (`issue#21214 <http://tracker.ceph.com/issues/21214>`_, `pr#17570 <https://github.com/ceph/ceph/pull/17570>`_, Luo Kexue, Jos Collin)
+* rgw: multisite: objects encrypted with SSE-KMS are stored unencrypted in target zone (`issue#20668 <http://tracker.ceph.com/issues/20668>`_, `issue#20671 <http://tracker.ceph.com/issues/20671>`_, `pr#17446 <https://github.com/ceph/ceph/pull/17446>`_, Casey Bodley)
+* rgw: need to stream metadata full sync init (`issue#18079 <http://tracker.ceph.com/issues/18079>`_, `pr#17448 <https://github.com/ceph/ceph/pull/17448>`_, Yehuda Sadeh)
+* rgw: object copied from remote src acl permission become full-control issue (`issue#20658 <http://tracker.ceph.com/issues/20658>`_, `pr#17478 <https://github.com/ceph/ceph/pull/17478>`_, Enming Zhang)
+* rgw: put lifecycle configuration fails if Prefix is not set (`issue#19587 <http://tracker.ceph.com/issues/19587>`_, `issue#20872 <http://tracker.ceph.com/issues/20872>`_, `pr#17479 <https://github.com/ceph/ceph/pull/17479>`_, Shasha Lu, Abhishek Lekshmanan)
+* rgw: rgw_file:  incorrect lane lock behavior in evict_block() (`issue#21141 <http://tracker.ceph.com/issues/21141>`_, `pr#17485 <https://github.com/ceph/ceph/pull/17485>`_, Matt Benjamin)
+* rgw: send data-log list infinitely (`issue#20951 <http://tracker.ceph.com/issues/20951>`_, `pr#17445 <https://github.com/ceph/ceph/pull/17445>`_, fang.yuxiang)
+* rgw: shadow objects are sometimes not removed (`issue#20234 <http://tracker.ceph.com/issues/20234>`_, `pr#17555 <https://github.com/ceph/ceph/pull/17555>`_, Yehuda Sadeh)
+* rgw: usage of --inconsistent-index should require user confirmation and print a warning (`issue#20777 <http://tracker.ceph.com/issues/20777>`_, `pr#17488 <https://github.com/ceph/ceph/pull/17488>`_, Orit Wasserman)
+* tools: [cli] rename of non-existent image results in seg fault (`issue#21248 <http://tracker.ceph.com/issues/21248>`_, `pr#17556 <https://github.com/ceph/ceph/pull/17556>`_, Jason Dillaman)
+
+
+v12.2.0 Luminous
+================
+
+This is the first release of Luminous v12.2.x long term stable release
+series.  There have been major changes since Kraken (v11.2.z) and
+Jewel (v10.2.z), and the upgrade process is non-trivial. Please read
+these release notes carefully.
+
+Major Changes from Kraken
+-------------------------
+
+- *General*:
+
+  * Ceph now has a simple, `built-in web-based dashboard
+    <../mgr/dashboard>`_ for monitoring cluster status.
+
+- *RADOS*:
+
+  * *BlueStore*:
+
+    - The new *BlueStore* backend for *ceph-osd* is now stable and the
+      new default for newly created OSDs.  BlueStore manages data
+      stored by each OSD by directly managing the physical HDDs or
+      SSDs without the use of an intervening file system like XFS.
+      This provides greater performance and features. See
+      :doc:`/rados/configuration/storage-devices` and
+      :doc:`/rados/configuration/bluestore-config-ref`.
+    - BlueStore supports `full data and metadata checksums
+      <../rados/configuration/bluestore-config-ref/#checksums>`_ of all
+      data stored by Ceph.
+    - BlueStore supports `inline compression
+      <../rados/configuration/bluestore-config-ref/#inline-compression>`_ using
+      zlib, snappy, or LZ4. (Ceph also supports zstd for `RGW compression
+      <../man/8/radosgw-admin/#options>`_ but zstd is not recommended for
+      BlueStore for performance reasons.)
+
+  * *Erasure coded* pools now have `full support for overwrites
+    <../rados/operations/erasure-code/#erasure-coding-with-overwrites>`_,
+    allowing them to be used with RBD and CephFS.
+
+  * *ceph-mgr*:
+
+    - There is a new daemon, *ceph-mgr*, which is a required part of
+      any Ceph deployment.  Although IO can continue when *ceph-mgr*
+      is down, metrics will not refresh and some metrics-related calls
+      (e.g., ``ceph df``) may block.  We recommend deploying several
+      instances of *ceph-mgr* for reliability.  See the notes on
+      `Upgrading`_ below.
+    - The *ceph-mgr* daemon includes a `REST-based management API
+      <../mgr/restful>`_. The API is still experimental and somewhat
+      limited but will form the basis for API-based management of Ceph
+      going forward.
+    - *ceph-mgr* also includes a `Prometheus exporter
+      <../mgr/prometheus>`_ plugin, which can provide Ceph
+      perfcounters to Prometheus.
+    - ceph-mgr now has a `Zabbix <../mgr/zabbix>`_ plugin. Using
+      zabbix_sender it sends trapper events to a Zabbix server
+      containing high-level information of the Ceph cluster. This
+      makes it easy to monitor a Ceph cluster's status and send out
+      notifications in case of a malfunction.
+
+  * The overall *scalability* of the cluster has improved. We have
+    successfully tested clusters with up to 10,000 OSDs.
+  * Each OSD can now have a `device class
+    <../rados/operations/crush-map/#device-classes>`_ associated with
+    it (e.g., `hdd` or `ssd`), allowing CRUSH rules to trivially map
+    data to a subset of devices in the system.  Manually writing CRUSH
+    rules or manual editing of the CRUSH is normally not required.
+  * There is a new `upmap <../rados/operations/upmap>`_ exception
+    mechanism that allows individual PGs to be moved around to achieve
+    a *perfect distribution* (this requires luminous clients).
+  * Each OSD now adjusts its default configuration based on whether the
+    backing device is an HDD or SSD.  Manual tuning generally not required.
+  * The prototype `mClock QoS queueing algorithm
+    <../rados/configuration/osd-config-ref/#qos-based-on-mclock>`_ is now
+    available.
+  * There is now a *backoff* mechanism that prevents OSDs from being
+    overloaded by requests to objects or PGs that are not currently able to
+    process IO.
+  * There is a simplified `OSD replacement process
+    <../rados/operations/add-or-rm-osds/#replacing-an-osd>`_ that is more
+    robust.
+  * You can query the supported features and (apparent) releases of
+    all connected daemons and clients with `ceph features
+    <../man/8/ceph#features>`_.
+  * You can configure the oldest Ceph client version you wish to allow to
+    connect to the cluster via ``ceph osd set-require-min-compat-client`` and
+    Ceph will prevent you from enabling features that will break compatibility
+    with those clients.
+  * Several `sleep` settings, include ``osd_recovery_sleep``,
+    ``osd_snap_trim_sleep``, and ``osd_scrub_sleep`` have been
+    reimplemented to work efficiently.  (These are used in some cases
+    to work around issues throttling background work.)
+  * Pools are now expected to be associated with the application using them.
+    Upon completing the upgrade to Luminous, the cluster will attempt to associate
+    existing pools to known applications (i.e. CephFS, RBD, and RGW). In-use pools
+    that are not associated to an application will generate a health warning. Any
+    unassociated pools can be manually associated using the new
+    ``ceph osd pool application enable`` command. For more details see
+    `associate pool to application <../rados/operations/pools/#associate-pool-to-application>`_
+    in the documentation.
+
+- *RGW*:
+
+  * RGW *metadata search* backed by ElasticSearch now supports end
+    user requests service via RGW itself, and also supports custom
+    metadata fields. A query language a set of RESTful APIs were
+    created for users to be able to search objects by their
+    metadata. New APIs that allow control of custom metadata fields
+    were also added.
+  * RGW now supports *dynamic bucket index sharding*. This has to be enabled via
+    the `rgw dynamic resharding` configurable. As the number of objects in a
+    bucket grows, RGW will automatically reshard the bucket index in response.
+    No user intervention or bucket size capacity planning is required.
+  * RGW introduces *server side encryption* of uploaded objects with
+    three options for the management of encryption keys: automatic
+    encryption (only recommended for test setups), customer provided
+    keys similar to Amazon SSE-C specification, and through the use of
+    an external key management service (Openstack Barbican) similar
+    to Amazon SSE-KMS specification. :doc:`/radosgw/encryption`
+  * RGW now has preliminary AWS-like bucket policy API support.  For
+    now, policy is a means to express a range of new authorization
+    concepts.  In the future it will be the foundation for additional
+    auth capabilities such as STS and group policy. :doc:`/radosgw/bucketpolicy`
+  * RGW has consolidated the several metadata index pools via the use of rados
+    namespaces. :doc:`/radosgw/pools`
+  * S3 Object Tagging API has been added; while APIs are
+    supported for GET/PUT/DELETE object tags and in PUT object
+    API, there is no support for tags on Policies & Lifecycle yet
+  * RGW multisite now supports for enabling or disabling sync at a
+    bucket level.
+
+- *RBD*:
+
+  * RBD now has full, stable support for *erasure coded pools* via the new
+    ``--data-pool`` option to ``rbd create``.
+  * RBD mirroring's rbd-mirror daemon is now highly available. We
+    recommend deploying several instances of rbd-mirror for
+    reliability.
+  * RBD mirroring's rbd-mirror daemon should utilize unique Ceph user
+    IDs per instance to support the new mirroring dashboard.
+  * The default 'rbd' pool is no longer created automatically during
+    cluster creation. Additionally, the name of the default pool used
+    by the rbd CLI when no pool is specified can be overridden via a
+    new ``rbd default pool = <pool name>`` configuration option.
+  * Initial support for deferred image deletion via new ``rbd
+    trash`` CLI commands. Images, even ones actively in-use by
+    clones, can be moved to the trash and deleted at a later time.
+  * New pool-level ``rbd mirror pool promote`` and ``rbd mirror pool
+    demote`` commands to batch promote/demote all mirrored images
+    within a pool.
+  * Mirroring now optionally supports a configurable replication delay
+    via the ``rbd mirroring replay delay = <seconds>`` configuration
+    option.
+  * Improved discard handling when the object map feature is enabled.
+  * rbd CLI ``import`` and ``copy`` commands now detect sparse and
+    preserve sparse regions.
+  * Images and Snapshots will now include a creation timestamp.
+  * Specifying user authorization capabilities for RBD clients has been
+    simplified. The general syntax for using RBD capability profiles is
+    "mon 'profile rbd' osd 'profile rbd[-read-only][ pool={pool-name}[, ...]]'".
+    For more details see :doc:`/rados/operations/user-management`
+    in the documentation.
+
+- *CephFS*:
+
+  * *Multiple active MDS daemons* is now considered stable.  The number
+    of active MDS servers may be adjusted up or down on an active CephFS file
+    system.
+  * CephFS *directory fragmentation* is now stable and enabled by
+    default on new filesystems.  To enable it on existing filesystems
+    use "ceph fs set <fs_name> allow_dirfrags".  Large or very busy
+    directories are sharded and (potentially) distributed across
+    multiple MDS daemons automatically.
+  * Directory subtrees can be explicitly pinned to specific MDS daemons in
+    cases where the automatic load balancing is not desired or effective.
+  * Client keys can now be created using the new ``ceph fs authorize`` command
+    to create keys with access to the given CephFS file system and all of its
+    data pools.
+  * When running 'df' on a CephFS filesystem comprising exactly one data pool,
+    the result now reflects the file storage space used and available in that
+    data pool (fuse client only).
+
+- *Miscellaneous*:
+
+  * Release packages are now being built for *Debian Stretch*.  Note
+    that QA is limited to CentOS and Ubuntu (xenial and trusty).  The
+    distributions we build for now include:
+
+    - CentOS 7 (x86_64 and aarch64)
+    - Debian 8 Jessie (x86_64)
+    - Debian 9 Stretch (x86_64)
+    - Ubuntu 16.04 Xenial (x86_64 and aarch64)
+    - Ubuntu 14.04 Trusty (x86_64)
+
+  * A first release of Ceph for FreeBSD is available which contains a full set
+    of features, other than Bluestore. It will run everything needed to build a
+    storage cluster. For clients, all access methods are available, albeit
+    CephFS is only accessible through a Fuse implementation. RBD images can be
+    mounted on FreeBSD systems through rbd-ggate
+    Ceph versions are released through the regular FreeBSD ports and packages
+    system. The most current version is available as: net/ceph-devel. Once
+    Luminous goes into official release, this version will be available as
+    net/ceph. Future development releases will be available via net/ceph-devel
+    More details about this port are in: `README.FreeBSD <https://github.com/ceph/ceph/blob/master/README.FreeBSD>`_
+
+  * *CLI changes*:
+
+    - The ``ceph -s`` or ``ceph status`` command has a fresh look.
+    - ``ceph mgr metadata`` will dump metadata associated with each mgr
+      daemon.
+    - ``ceph versions`` or ``ceph {osd,mds,mon,mgr} versions``
+      summarize versions of running daemons.
+    - ``ceph {osd,mds,mon,mgr} count-metadata <property>`` similarly
+      tabulates any other daemon metadata visible via the ``ceph
+      {osd,mds,mon,mgr} metadata`` commands.
+    - ``ceph features`` summarizes features and releases of connected
+      clients and daemons.
+    - ``ceph osd require-osd-release <release>`` replaces the old
+      ``require_RELEASE_osds`` flags.
+    - ``ceph osd pg-upmap``, ``ceph osd rm-pg-upmap``, ``ceph osd
+      pg-upmap-items``, ``ceph osd rm-pg-upmap-items`` can explicitly
+      manage `upmap` items (see :doc:`/rados/operations/upmap`).
+    - ``ceph osd getcrushmap`` returns a crush map version number on
+      stderr, and ``ceph osd setcrushmap [version]`` will only inject
+      an updated crush map if the version matches.  This allows crush
+      maps to be updated offline and then reinjected into the cluster
+      without fear of clobbering racing changes (e.g., by newly added
+      osds or changes by other administrators).
+    - ``ceph osd create`` has been replaced by ``ceph osd new``.  This
+      should be hidden from most users by user-facing tools like
+      `ceph-disk`.
+    - ``ceph osd destroy`` will mark an OSD destroyed and remove its
+      cephx and lockbox keys.  However, the OSD id and CRUSH map entry
+      will remain in place, allowing the id to be reused by a
+      replacement device with minimal data rebalancing.
+    - ``ceph osd purge`` will remove all traces of an OSD from the
+      cluster, including its cephx encryption keys, dm-crypt lockbox
+      keys, OSD id, and crush map entry.
+    - ``ceph osd ls-tree <name>`` will output a list of OSD ids under
+      the given CRUSH name (like a host or rack name).  This is useful
+      for applying changes to entire subtrees.  For example, ``ceph
+      osd down `ceph osd ls-tree rack1```.
+    - ``ceph osd {add,rm}-{noout,noin,nodown,noup}`` allow the
+      `noout`, `noin`, `nodown`, and `noup` flags to be applied to
+      specific OSDs.
+    - ``ceph osd safe-to-destroy <osd(s)>`` will report whether it is safe to
+      remove or destroy OSD(s) without reducing data durability or redundancy.
+    - ``ceph osd ok-to-stop <osd(s)>`` will report whether it is okay to stop
+      OSD(s) without immediately compromising availability (i.e., all PGs
+      should remain active but may be degraded).
+    - ``ceph log last [n]`` will output the last *n* lines of the cluster
+      log.
+    - ``ceph mgr dump`` will dump the MgrMap, including the currently active
+      ceph-mgr daemon and any standbys.
+    - ``ceph mgr module ls`` will list active ceph-mgr modules.
+    - ``ceph mgr module {enable,disable} <name>`` will enable or
+      disable the named mgr module.  The module must be present in the
+      configured `mgr_module_path` on the host(s) where `ceph-mgr` is
+      running.
+    - ``ceph osd crush ls <node>`` will list items (OSDs or other CRUSH nodes)
+      directly beneath a given CRUSH node.
+    - ``ceph osd crush swap-bucket <src> <dest>`` will swap the
+      contents of two CRUSH buckets in the hierarchy while preserving
+      the buckets' ids.  This allows an entire subtree of devices to
+      be replaced (e.g., to replace an entire host of FileStore OSDs
+      with newly-imaged BlueStore OSDs) without disrupting the
+      distribution of data across neighboring devices.
+    - ``ceph osd set-require-min-compat-client <release>`` configures
+      the oldest client release the cluster is required to support.
+      Other changes, like CRUSH tunables, will fail with an error if
+      they would violate this setting.  Changing this setting also
+      fails if clients older than the specified release are currently
+      connected to the cluster.
+    - ``ceph config-key dump`` dumps config-key entries and their
+      contents.  (The existing ``ceph config-key list`` only dumps the key
+      names, not the values.)
+    - ``ceph config-key list`` is deprecated in favor of ``ceph config-key ls``.
+    - ``ceph config-key put`` is deprecated in favor of ``ceph config-key set``.
+    - ``ceph auth list`` is deprecated in favor of ``ceph auth ls``.
+    - ``ceph osd crush rule list`` is deprecated in favor of ``ceph osd crush rule ls``.
+    - ``ceph osd set-{full,nearfull,backfillfull}-ratio`` sets the
+      cluster-wide ratio for various full thresholds (when the cluster
+      refuses IO, when the cluster warns about being close to full,
+      when an OSD will defer rebalancing a PG to itself,
+      respectively).
+    - ``ceph osd reweightn`` will specify the `reweight` values for
+      multiple OSDs in a single command.  This is equivalent to a series of
+      ``ceph osd reweight`` commands.
+    - ``ceph osd crush {set,rm}-device-class`` manage the new
+      CRUSH *device class* feature. Note that manually creating or deleting
+      a device class name is generally not necessary as it will be smart
+      enough to be self-managed. ``ceph osd crush class ls`` and
+      ``ceph osd crush class ls-osd`` will output all existing device classes
+      and a list of OSD ids under the given device class respectively.
+    - ``ceph osd crush rule create-replicated`` replaces the old
+      ``ceph osd crush rule create-simple`` command to create a CRUSH
+      rule for a replicated pool.  Notably it takes a `class` argument
+      for the *device class* the rule should target (e.g., `ssd` or
+      `hdd`).
+    - ``ceph mon feature ls`` will list monitor features recorded in the
+      MonMap.  ``ceph mon feature set`` will set an optional feature (none of
+      these exist yet).
+    - ``ceph tell <daemon> help`` will now return a usage summary.
+    - ``ceph fs authorize`` creates a new client key with caps automatically
+      set to access the given CephFS file system.
+    - The ``ceph health`` structured output (JSON or XML) no longer contains
+      'timechecks' section describing the time sync status.  This
+      information is now available via the 'ceph time-sync-status'
+      command.
+    - Certain extra fields in the ``ceph health`` structured output that
+      used to appear if the mons were low on disk space (which duplicated
+      the information in the normal health warning messages) are now gone.
+    - The ``ceph -w`` output no longer contains audit log entries by default.
+      Add a ``--watch-channel=audit`` or ``--watch-channel=*`` to see them.
+    - New "ceph -w" behavior - the "ceph -w" output no longer contains
+      I/O rates, available space, pg info, etc. because these are no
+      longer logged to the central log (which is what ``ceph -w``
+      shows). The same information can be obtained by running ``ceph pg
+      stat``; alternatively, I/O rates per pool can be determined using
+      ``ceph osd pool stats``. Although these commands do not
+      self-update like ``ceph -w`` did, they do have the ability to
+      return formatted output by providing a ``--format=<format>``
+      option.
+    - Added new commands ``pg force-recovery`` and
+      ``pg-force-backfill``. Use them to boost recovery or backfill
+      priority of specified pgs, so they're recovered/backfilled
+      before any other. Note that these commands don't interrupt
+      ongoing recovery/backfill, but merely queue specified pgs before
+      others so they're recovered/backfilled as soon as possible. New
+      commands ``pg cancel-force-recovery`` and ``pg
+      cancel-force-backfill`` restore default recovery/backfill
+      priority of previously forced pgs.
+
+
+Major Changes from Jewel
+------------------------
+
+- *RADOS*:
+
+  * We now default to the AsyncMessenger (``ms type = async``) instead
+    of the legacy SimpleMessenger.  The most noticeable difference is
+    that we now use a fixed sized thread pool for network connections
+    (instead of two threads per socket with SimpleMessenger).
+  * Some OSD failures are now detected almost immediately, whereas
+    previously the heartbeat timeout (which defaults to 20 seconds)
+    had to expire.  This prevents IO from blocking for an extended
+    period for failures where the host remains up but the ceph-osd
+    process is no longer running.
+  * The size of encoded OSDMaps has been reduced.
+  * The OSDs now quiesce scrubbing when recovery or rebalancing is in progress.
+
+- *RGW*:
+
+  * RGW now supports the S3 multipart object copy-part API.
+  * It is possible now to reshard an existing bucket offline. Offline
+    bucket resharding currently requires that all IO (especially
+    writes) to the specific bucket is quiesced.  (For automatic online
+    resharding, see the new feature in Luminous above.)
+  * RGW now supports data compression for objects.
+  * Civetweb version has been upgraded to 1.8
+  * The Swift static website API is now supported (S3 support has been added
+    previously).
+  * S3 bucket lifecycle API has been added. Note that currently it only supports
+    object expiration.
+  * Support for custom search filters has been added to the LDAP auth
+    implementation.
+  * Support for NFS version 3 has been added to the RGW NFS gateway.
+  * A Python binding has been created for librgw.
+
+- *RBD*:
+
+  * The rbd-mirror daemon now supports replicating dynamic image
+    feature updates and image metadata key/value pairs from the
+    primary image to the non-primary image.
+  * The number of image snapshots can be optionally restricted to a
+    configurable maximum.
+  * The rbd Python API now supports asynchronous IO operations.
+
+- *CephFS*:
+
+  * libcephfs function definitions have been changed to enable proper
+    uid/gid control.  The library version has been increased to reflect the
+    interface change.
+  * Standby replay MDS daemons now consume less memory on workloads
+    doing deletions.
+  * Scrub now repairs backtrace, and populates `damage ls` with
+    discovered errors.
+  * A new `pg_files` subcommand to `cephfs-data-scan` can identify
+    files affected by a damaged or lost RADOS PG.
+  * The false-positive "failing to respond to cache pressure" warnings have
+    been fixed.
+
+
+Upgrade from Jewel or Kraken
+----------------------------
+.. _Upgrading:
+
+#. Ensure that the ``sortbitwise`` flag is enabled::
+
+     # ceph osd set sortbitwise
+
+#. Make sure your cluster is stable and healthy (no down or
+   recovering OSDs).  (Optional, but recommended.)
+
+#. Do not create any new erasure-code pools while upgrading the monitors.
+
+#. You can monitor the progress of your upgrade at each stage with the
+   ``ceph versions`` command, which will tell you what ceph version is
+   running for each type of daemon.
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional
+   but recommended.)::
+
+     # ceph osd set noout
+
+#. Verify that all RBD client users have sufficient caps to blacklist
+   other client users. RBD client users with only ``"allow r"``
+   monitor caps should be updated as follows::
+
+     # ceph auth caps client.<ID> mon 'allow r, allow command "osd blacklist"' osd '<existing OSD caps for user>'
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.  Note that, unlike prior releases, the ceph-mon
+   daemons *must* be upgraded first::
+
+     # systemctl restart ceph-mon.target
+
+   Verify the monitor upgrade is complete once all monitors are up by
+   looking for the ``luminous`` feature string in the mon map.  For
+   example::
+
+     # ceph mon feature ls
+
+   should include `luminous` under persistent features::
+
+     on current monmap (epoch NNN)
+        persistent: [kraken,luminous]
+        required: [kraken,luminous]
+
+#. Add or restart ``ceph-mgr`` daemons.  If you are upgrading from
+   kraken, upgrade packages and restart ceph-mgr daemons with::
+
+     # systemctl restart ceph-mgr.target
+
+   If you are upgrading from kraken, you may already have ceph-mgr
+   daemons deployed.  If not, or if you are upgrading from jewel, you
+   can deploy new daemons with tools like ceph-deploy or ceph-ansible.
+   For example::
+
+     # ceph-deploy mgr create HOST
+
+   Verify the ceph-mgr daemons are running by checking ``ceph -s``::
+
+     # ceph -s
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all hosts::
+
+     # systemctl restart ceph-osd.target
+
+   You can monitor the progress of the OSD upgrades with the new
+   ``ceph versions`` or ``ceph osd versions`` command::
+
+     # ceph osd versions
+     {
+        "ceph version 12.2.0 (...) luminous (stable)": 12,
+        "ceph version 10.2.6 (...)": 3,
+     }
+
+#. Upgrade all CephFS daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart ceph-mds.target
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart radosgw.target
+
+#. Complete the upgrade by disallowing pre-luminous OSDs and enabling
+   all new Luminous-only functionality::
+
+     # ceph osd require-osd-release luminous
+
+   If you set ``noout`` at the beginning, be sure to clear it with::
+
+     # ceph osd unset noout
+
+#. Verify the cluster is healthy with ``ceph health``.
+
+
+Upgrading from pre-Jewel releases (like Hammer)
+-----------------------------------------------
+
+You *must* first upgrade to Jewel (10.2.z) before attempting an
+upgrade to Luminous.
+
+Upgrade compatibility notes, Jewel to Kraken
+--------------------------------------------
+
+These changes occurred between the Jewel and Kraken releases and will affect
+upgrades from Jewel to Luminous.
+
+* The ``osd crush location`` config option is no longer supported.  Please
+  update your ceph.conf to use the ``crush location`` option instead.  Be sure
+  to update your config file to avoid any movement of OSDs from your customized
+  location back to the default one.
+
+* The OSDs now avoid starting new scrubs while recovery is in progress.  To
+  revert to the old behavior (and do not let recovery activity affect the
+  scrub scheduling) you can set the following option::
+
+    osd scrub during recovery = true
+
+* The list of monitor hosts/addresses for building the monmap can now be
+  obtained from DNS SRV records. The service name used in when querying the DNS
+  is defined in the "mon_dns_srv_name" config option, which defaults to
+  "ceph-mon".
+
+* The 'osd class load list' config option is a list of object class names that
+  the OSD is permitted to load (or '*' for all classes). By default it
+  contains all existing in-tree classes for backwards compatibility.
+
+* The 'osd class default list' config option is a list of object class
+  names (or '*' for all classes) that clients may invoke having only
+  the '*', 'x', 'class-read', or 'class-write' capabilities. By
+  default it contains all existing in-tree classes for backwards
+  compatibility. Invoking classes not listed in 'osd class default
+  list' requires a capability naming the class (e.g. 'allow class
+  foo').
+
+* The 'rgw rest getusage op compat' config option allows you to dump
+  (or not dump) the description of user stats in the S3 GetUsage
+  API. This option defaults to false.  If the value is true, the
+  response data for GetUsage looks like::
+
+    "stats": {
+                "TotalBytes": 516,
+                "TotalBytesRounded": 1024,
+                "TotalEntries": 1
+             }
+
+  If the value is false, the response for GetUsage looks as it did before::
+
+    {
+         516,
+         1024,
+         1
+    }
+
+* The 'osd out ...' and 'osd in ...' commands now preserve the OSD
+  weight.  That is, after marking an OSD out and then in, the weight
+  will be the same as before (instead of being reset to 1.0).
+  Previously the mons would only preserve the weight if the mon
+  automatically marked an OSD out and then in, but not when an admin
+  did so explicitly.
+
+* The 'ceph osd perf' command will display 'commit_latency(ms)' and
+  'apply_latency(ms)'. Previously, the names of these two columns were
+  'fs_commit_latency(ms)' and 'fs_apply_latency(ms)'. We removed the
+  prefix 'fs\_' because the values are not filestore-specific.
+
+* Monitors will no longer allow pools to be removed by default.  The
+  setting mon_allow_pool_delete has to be set to true (defaults to
+  false) before they allow pools to be removed.  This is a additional
+  safeguard against pools being removed by accident.
+
+* If you have manually specified the monitor user rocksdb via the
+  ``mon keyvaluedb = rocksdb`` option, you will need to manually add a
+  file to the mon data directory to preserve this option::
+
+     echo rocksdb > /var/lib/ceph/mon/ceph-`hostname`/kv_backend
+
+  New monitors will now use rocksdb by default, but if that file is
+  not present, existing monitors will use leveldb.  The ``mon
+  keyvaluedb`` option now only affects the backend chosen when a
+  monitor is created.
+
+* The 'osd crush initial weight' option allows you to specify a CRUSH
+  weight for a newly added OSD.  Previously a value of 0 (the default)
+  meant that we should use the size of the OSD's store to weight the
+  new OSD.  Now, a value of 0 means it should have a weight of 0, and
+  a negative value (the new default) means we should automatically
+  weight the OSD based on its size.  If your configuration file
+  explicitly specifies a value of 0 for this option you will need to
+  change it to a negative value (e.g., -1) to preserve the current
+  behavior.
+
+* The static libraries are no longer included by the debian
+  development packages (lib*-dev) as it is not required per debian
+  packaging policy.  The shared (.so) versions are packaged as before.
+
+* The libtool pseudo-libraries (.la files) are no longer included by
+  the debian development packages (lib*-dev) as they are not required
+  per https://wiki.debian.org/ReleaseGoals/LAFileRemoval and
+  https://www.debian.org/doc/manuals/maint-guide/advanced.en.html.
+
+* The jerasure and shec plugins can now detect SIMD instruction at
+  runtime and no longer need to be explicitly configured for different
+  processors.  The following plugins are now deprecated:
+  jerasure_generic, jerasure_sse3, jerasure_sse4, jerasure_neon,
+  shec_generic, shec_sse3, shec_sse4, and shec_neon. If you use any of
+  these plugins directly you will see a warning in the mon log file.
+  Please switch to using just 'jerasure' or 'shec'.
+
+* The librados omap get_keys and get_vals operations include a start key and a
+  limit on the number of keys to return.  The OSD now imposes a configurable
+  limit on the number of keys and number of total bytes it will respond with,
+  which means that a librados user might get fewer keys than they asked for.
+  This is necessary to prevent careless users from requesting an unreasonable
+  amount of data from the cluster in a single operation.  The new limits are
+  configured with ``osd_max_omap_entries_per_request``, defaulting to 131,072, and
+  ``osd_max_omap_bytes_per_request``, defaulting to 4MB.
+
+* Calculation of recovery priorities has been updated.
+  This could lead to unintuitive recovery prioritization
+  during cluster upgrade. In case of such recovery, OSDs
+  in old version would operate on different priority ranges
+  than new ones. Once upgraded, cluster will operate on
+  consistent values.
+
+
+Upgrade compatibility notes, Kraken to Luminous
+-----------------------------------------------
+
+* The configuration option ``osd pool erasure code stripe width`` has
+  been replaced by ``osd pool erasure code stripe unit``, and given
+  the ability to be overridden by the erasure code profile setting
+  ``stripe_unit``. For more details see
+  :ref:`erasure-code-profiles`.
+
+* rbd and cephfs can use erasure coding with bluestore. This may be
+  enabled by setting ``allow_ec_overwrites`` to ``true`` for a pool. Since
+  this relies on bluestore's checksumming to do deep scrubbing,
+  enabling this on a pool stored on filestore is not allowed.
+
+* The ``rados df`` JSON output now prints numeric values as numbers instead of
+  strings.
+
+* The ``mon_osd_max_op_age`` option has been renamed to
+  ``mon_osd_warn_op_age`` (default: 32 seconds), to indicate we
+  generate a warning at this age.  There is also a new
+  ``mon_osd_err_op_age_ratio`` that is a expressed as a multiple of
+  ``mon_osd_warn_op_age`` (default: 128, for roughly 60 minutes) to
+  control when an error is generated.
+
+* The default maximum size for a single RADOS object has been reduced from
+  100GB to 128MB.  The 100GB limit was completely impractical in practice
+  while the 128MB limit is a bit high but not unreasonable.  If you have an
+  application written directly to librados that is using objects larger than
+  128MB you may need to adjust ``osd_max_object_size``.
+
+* The semantics of the ``rados ls`` and librados object listing
+  operations have always been a bit confusing in that "whiteout"
+  objects (which logically don't exist and will return ENOENT if you
+  try to access them) are included in the results.  Previously
+  whiteouts only occurred in cache tier pools.  In luminous, logically
+  deleted but snapshotted objects now result in a whiteout object, and
+  as a result they will appear in ``rados ls`` results, even though
+  trying to read such an object will result in ENOENT.  The ``rados
+  listsnaps`` operation can be used in such a case to enumerate which
+  snapshots are present.
+  This may seem a bit strange, but is less strange than having a
+  deleted-but-snapshotted object not appear at all and be completely
+  hidden from librados's ability to enumerate objects.  Future
+  versions of Ceph will likely include an alternative object
+  enumeration interface that makes it more natural and efficient to
+  enumerate all objects along with their snapshot and clone metadata.
+
+* The deprecated ``crush_ruleset`` property has finally been removed;
+  please use  ``crush_rule`` instead for the ``osd pool get ...`` and ``osd
+  pool set ...`` commands.
+
+* The ``osd pool default crush replicated ruleset`` option has been
+  removed and replaced by the ``osd pool default crush rule`` option.
+  By default it is -1, which means the mon will pick the first type
+  replicated rule in the CRUSH map for replicated pools.  Erasure
+  coded pools have rules that are automatically created for them if
+  they are not specified at pool creation time.
+
+* We no longer test the FileStore ceph-osd backend in combination with
+  btrfs.  We recommend against using btrfs.  If you are using
+  btrfs-based OSDs and want to upgrade to luminous you will need to
+  add the following to your ceph.conf::
+
+    enable experimental unrecoverable data corrupting features = btrfs
+
+  The code is mature and unlikely to change, but we are only
+  continuing to test the Jewel stable branch against btrfs.  We
+  recommend moving these OSDs to FileStore with XFS or BlueStore.
+* The ``ruleset-*`` properties for the erasure code profiles have been
+  renamed to ``crush-*`` to move away from the obsolete 'ruleset'
+  term and to be more clear about their purpose.  There is also a new
+  optional ``crush-device-class`` property to specify a CRUSH device
+  class to use for the erasure coded pool.  Existing erasure code
+  profiles will be converted automatically when upgrade completes
+  (when the ``ceph osd require-osd-release luminous`` command is run)
+  but any provisioning tools that create erasure coded pools may need
+  to be updated.
+* The structure of the XML output for ``osd crush tree`` has changed
+  slightly to better match the ``osd tree`` output.  The top level
+  structure is now ``nodes`` instead of ``crush_map_roots``.
+* When assigning a network to the public network and not to
+  the cluster network the network specification of the public
+  network will be used for the cluster network as well.
+  In older versions this would lead to cluster services
+  being bound to 0.0.0.0:<port>, thus making the
+  cluster service even more publicly available than the
+  public services. When only specifying a cluster network it
+  will still result in the public services binding to 0.0.0.0.
+
+* In previous versions, if a client sent an op to the wrong OSD, the OSD
+  would reply with ENXIO.  The rationale here is that the client or OSD is
+  clearly buggy and we want to surface the error as clearly as possible.
+  We now only send the ENXIO reply if the osd_enxio_on_misdirected_op option
+  is enabled (it's off by default).  This means that a VM using librbd that
+  previously would have gotten an EIO and gone read-only will now see a
+  blocked/hung IO instead.
+
+* The "journaler allow split entries" config setting has been removed.
+
+* The 'mon_warn_osd_usage_min_max_delta' config option has been
+  removed and the associated health warning has been disabled because
+  it does not address clusters undergoing recovery or CRUSH rules that do
+  not target all devices in the cluster.
+
+* Added new configuration "public bind addr" to support dynamic
+  environments like Kubernetes. When set the Ceph MON daemon could
+  bind locally to an IP address and advertise a different IP address
+  ``public addr`` on the network.
+
+* The crush ``choose_args`` encoding has been changed to make it
+  architecture-independent. If you deployed Luminous dev releases or
+  12.1.0 rc release and made use of the CRUSH choose_args feature, you
+  need to remove all choose_args mappings from your CRUSH map before
+  starting the upgrade.
+
+
+- *librados*:
+
+  * Some variants of the omap_get_keys and omap_get_vals librados
+    functions have been deprecated in favor of omap_get_vals2 and
+    omap_get_keys2.  The new methods include an output argument
+    indicating whether there are additional keys left to fetch.
+    Previously this had to be inferred from the requested key count vs
+    the number of keys returned, but this breaks with new OSD-side
+    limits on the number of keys or bytes that can be returned by a
+    single omap request.  These limits were introduced by kraken but
+    are effectively disabled by default (by setting a very large limit
+    of 1 GB) because users of the newly deprecated interface cannot
+    tell whether they should fetch more keys or not.  In the case of
+    the standalone calls in the C++ interface
+    (IoCtx::get_omap_{keys,vals}), librados has been updated to loop on
+    the client side to provide a correct result via multiple calls to
+    the OSD.  In the case of the methods used for building
+    multi-operation transactions, however, client-side looping is not
+    practical, and the methods have been deprecated.  Note that use of
+    either the IoCtx methods on older librados versions or the
+    deprecated methods on any version of librados will lead to
+    incomplete results if/when the new OSD limits are enabled.
+
+  * The original librados rados_objects_list_open (C) and objects_begin
+    (C++) object listing API, deprecated in Hammer, has finally been
+    removed.  Users of this interface must update their software to use
+    either the rados_nobjects_list_open (C) and nobjects_begin (C++) API or
+    the new rados_object_list_begin (C) and object_list_begin (C++) API
+    before updating the client-side librados library to Luminous.
+    Object enumeration (via any API) with the latest librados version
+    and pre-Hammer OSDs is no longer supported.  Note that no in-tree
+    Ceph services rely on object enumeration via the deprecated APIs, so
+    only external librados users might be affected.
+    The newest (and recommended) rados_object_list_begin (C) and
+    object_list_begin (C++) API is only usable on clusters with the
+    SORTBITWISE flag enabled (Jewel and later).  (Note that this flag is
+    required to be set before upgrading beyond Jewel.)
+
+- *CephFS*:
+
+  * When configuring ceph-fuse mounts in /etc/fstab, a new syntax is
+    available that uses "ceph.<arg>=<val>" in the options column, instead
+    of putting configuration in the device column.  The old style syntax
+    still works.  See the documentation page "Mount CephFS in your
+    file systems table" for details.
+  * CephFS clients without the 'p' flag in their authentication capability
+    string will no longer be able to set quotas or any layout fields.  This
+    flag previously only restricted modification of the pool and namespace
+    fields in layouts.
+  * CephFS will generate a health warning if you have fewer standby daemons
+    than it thinks you wanted.  By default this will be 1 if you ever had
+    a standby, and 0 if you did not.  You can customize this using
+    ``ceph fs set <fs> standby_count_wanted <number>``.  Setting it
+    to zero will effectively disable the health check.
+  * The "ceph mds tell ..." command has been removed.  It is superseded
+    by "ceph tell mds.<id> ..."
+  * The ``apply`` mode of cephfs-journal-tool has been removed
+
+
+Other Notable Changes
+---------------------
+* async: Fixed compilation error when enable -DWITH_DPDK (`pr#12660 <https://github.com/ceph/ceph/pull/12660>`_, Pan Liu)
+* async: fixed coredump when enable dpdk (`pr#12854 <https://github.com/ceph/ceph/pull/12854>`_, Pan Liu)
+* async: fixed the error "Cause: Cannot create lock on '/var/run/.rte_c… (`pr#12860 <https://github.com/ceph/ceph/pull/12860>`_, Pan Liu)
+* bluestore: avoid unnecessary copy with coll_t (`pr#12576 <https://github.com/ceph/ceph/pull/12576>`_, Yunchuan Wen)
+* bluestore: BitAllocator: delete useless codes (`pr#13619 <https://github.com/ceph/ceph/pull/13619>`_, Jie Wang)
+* bluestore: bluestore/BlueFS: pass string as const ref (`pr#16600 <https://github.com/ceph/ceph/pull/16600>`_, dingdangzhang)
+* bluestore: bluestore, NVMEDEVICE: Specify the max io completion in conf (`pr#13799 <https://github.com/ceph/ceph/pull/13799>`_, optimistyzy)
+* bluestore: bluestore/NVMEDEVICE: update SPDK to version 17.03 (`pr#14585 <https://github.com/ceph/ceph/pull/14585>`_, optimistyzy)
+* bluestore: bluestore, NVMeDevice: use task' own lock for (random) read (`pr#14094 <https://github.com/ceph/ceph/pull/14094>`_, optimistyzy)
+* bluestore,build/ops,performance: os/bluestore: enable SSE-assisted CRC32 calculations in RocksDB (`pr#13741 <https://github.com/ceph/ceph/pull/13741>`_, Radoslaw Zarzynski)
+* bluestore: ceph-disk: add --filestore argument, default to --bluestore (`pr#15437 <https://github.com/ceph/ceph/pull/15437>`_, Loic Dachary, Sage Weil)
+* bluestore: common/config: set rocksdb_cache_size to OPT_U64 (`pr#13995 <https://github.com/ceph/ceph/pull/13995>`_, liuhongtong)
+* bluestore: common/options: make "blue{fs,store}_allocator" LEVEL_DEV (`issue#20660 <http://tracker.ceph.com/issues/20660>`_, `pr#16645 <https://github.com/ceph/ceph/pull/16645>`_, Kefu Chai)
+* bluestore,common,performance: common/Finisher: Using queue(list<context\*>) instead queue(context\*) (`pr#8942 <https://github.com/ceph/ceph/pull/8942>`_, Jianpeng Ma)
+* bluestore,common,performance: isa-l: update isa-l to v2.18 (`pr#15895 <https://github.com/ceph/ceph/pull/15895>`_, Ganesh Mahalingam, Tushar Gohad)
+* bluestore,core: os/bluestore: fix statfs to not include DB partition in free space (`issue#18599 <http://tracker.ceph.com/issues/18599>`_, `pr#13140 <https://github.com/ceph/ceph/pull/13140>`_, Sage Weil)
+* bluestore,core: os/bluestore: fix warning (`pr#15435 <https://github.com/ceph/ceph/pull/15435>`_, Sage Weil)
+* bluestore,core: os/bluestore: improve mempool usage (`pr#15402 <https://github.com/ceph/ceph/pull/15402>`_, Sage Weil)
+* bluestore,core: os/bluestore: write "mkfs_done" into disk only if we pass fsck() tests (`pr#15238 <https://github.com/ceph/ceph/pull/15238>`_, xie xingguo)
+* bluestore,core: osd/OSDMap: should update input param if osd dne (`pr#14863 <https://github.com/ceph/ceph/pull/14863>`_, Kefu Chai)
+* bluestore,core: os: remove experimental status for BlueStore (`pr#15177 <https://github.com/ceph/ceph/pull/15177>`_, Sage Weil)
+* bluestore: fixed compilation error when enable spdk (`pr#12672 <https://github.com/ceph/ceph/pull/12672>`_, Pan Liu)
+* bluestore: include/intarith: templatize ctz/clz/cbits helpers (`pr#14862 <https://github.com/ceph/ceph/pull/14862>`_, Kefu Chai)
+* bluestore: luminous: os/bluestore: compensate for bad freelistmanager size/blocks metadata (`issue#21089 <http://tracker.ceph.com/issues/21089>`_, `pr#17273 <https://github.com/ceph/ceph/pull/17273>`_, Sage Weil)
+* bluestore: NVMEDevice: add the spdk core mask check (`pr#14068 <https://github.com/ceph/ceph/pull/14068>`_, optimistyzy)
+* bluestore: NVMEDevice: cleanup the logic in data_buf_next_sge (`pr#13056 <https://github.com/ceph/ceph/pull/13056>`_, optimistyzy)
+* bluestore: NVMeDevice: fix the core id for rte_remote_launch (`pr#13896 <https://github.com/ceph/ceph/pull/13896>`_, optimistyzy)
+* bluestore: NVMEDevice: fix bug in data_buf_next_sge (`pr#12812 <https://github.com/ceph/ceph/pull/12812>`_, optimistyzy)
+* bluestore: NVMEDevice: minor error for get slave core (`pr#14012 <https://github.com/ceph/ceph/pull/14012>`_, Ziye Yang)
+* bluestore: NVMEDevice: optimize sector_size usage (`pr#12780 <https://github.com/ceph/ceph/pull/12780>`_, optimistyzy)
+* bluestore: NVMEDevice: remove unnessary dpdk header file (`pr#14650 <https://github.com/ceph/ceph/pull/14650>`_, optimistyzy)
+* bluestore: NVMEDevice: fix the I/O logic for read (`pr#13971 <https://github.com/ceph/ceph/pull/13971>`_, optimistyzy)
+* bluestore: os/bluestore: add a debug option to bypass block device writes for bl… (`pr#12464 <https://github.com/ceph/ceph/pull/12464>`_, Igor Fedotov)
+* bluestore: os/bluestore: Add bluestore pextent vector to mempool (`pr#12946 <https://github.com/ceph/ceph/pull/12946>`_, Igor Fedotvo, Igor Fedotov)
+* bluestore: os/bluestore: add flush_store_cache cmd (`pr#13428 <https://github.com/ceph/ceph/pull/13428>`_, xie xingguo)
+* bluestore: os/bluestore: add more perf_counters to BlueStore (`pr#13274 <https://github.com/ceph/ceph/pull/13274>`_, Igor Fedotov)
+* bluestore: os/bluestore: add new garbage collector (`pr#12144 <https://github.com/ceph/ceph/pull/12144>`_, Igor Fedotov)
+* bluestore: os/bluestore: add perf variable for throttle info in bluestore (`pr#12583 <https://github.com/ceph/ceph/pull/12583>`_, Pan Liu)
+* bluestore: os/bluestore: add "_" prefix for internal methods (`pr#13409 <https://github.com/ceph/ceph/pull/13409>`_, xie xingguo)
+* bluestore: os/bluestore: align reclaim size to bluefs_alloc_size (`pr#14744 <https://github.com/ceph/ceph/pull/14744>`_, Haomai Wang)
+* bluestore: os/bluestore/Allocator: drop unused return value in release function (`pr#13913 <https://github.com/ceph/ceph/pull/13913>`_, wangzhengyong)
+* bluestore: os/bluestore: allow multiple DeferredBatches in flight at once (`issue#20295 <http://tracker.ceph.com/issues/20295>`_, `pr#16769 <https://github.com/ceph/ceph/pull/16769>`_, Nathan Cutler, Sage Weil)
+* bluestore: os/bluestore: allow multiple SPDK BlueStore OSD instances (`issue#16966 <http://tracker.ceph.com/issues/16966>`_, `pr#12604 <https://github.com/ceph/ceph/pull/12604>`_, Orlando Moreno)
+* bluestore: os/bluestore: assert blob map returns success (`pr#14473 <https://github.com/ceph/ceph/pull/14473>`_, shiqi)
+* bluestore: os/bluestore: avoid nullptr in bluestore_extent_ref_map_t::bound_encode (`pr#14073 <https://github.com/ceph/ceph/pull/14073>`_, Sage Weil)
+* bluestore: os/bluestore: avoid unnecessary memory copy, use variable reference in BlockDevice::Open (`pr#12942 <https://github.com/ceph/ceph/pull/12942>`_, liuchang0812)
+* bluestore: os/bluestore: better debug output on unsharing blobs (`issue#20227 <http://tracker.ceph.com/issues/20227>`_, `pr#15746 <https://github.com/ceph/ceph/pull/15746>`_, Sage Weil)
+* bluestore: os/bluestore/BitAllocator: fix bug of checking required blocks (`pr#13470 <https://github.com/ceph/ceph/pull/13470>`_, wangzhengyong)
+* bluestore: os/bluestore/BitMapAllocator: rm unused variable (`pr#13599 <https://github.com/ceph/ceph/pull/13599>`_, Jie Wang)
+* bluestore: os/bluestore/BitmapFreelistManager: readability improvements (`pr#12719 <https://github.com/ceph/ceph/pull/12719>`_, xie xingguo)
+* bluestore: os/bluestore/BlockDevice: support pmem device as bluestore backend (`pr#15102 <https://github.com/ceph/ceph/pull/15102>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: clean up log_writer aios from compaction (`issue#20454 <http://tracker.ceph.com/issues/20454>`_, `pr#16017 <https://github.com/ceph/ceph/pull/16017>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: clear current log entrys before dump all fnode (`pr#15973 <https://github.com/ceph/ceph/pull/15973>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: fix reclaim_blocks (`issue#18368 <http://tracker.ceph.com/issues/18368>`_, `pr#12725 <https://github.com/ceph/ceph/pull/12725>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: Rebuild memcopy for bufferlist::page_aligned_app… (`pr#15728 <https://github.com/ceph/ceph/pull/15728>`_, Jianpeng Ma, Sage Weil)
+* bluestore: os/bluestore/BlueFS: .slow should be compared with dirname (`pr#15595 <https://github.com/ceph/ceph/pull/15595>`_, zhanglei)
+* bluestore: os/bluestore/BlueStore: Avoid double counting state_kv_queued_lat (`pr#16374 <https://github.com/ceph/ceph/pull/16374>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueStore.cc:remove unuse code in _open_bdev() (`pr#13553 <https://github.com/ceph/ceph/pull/13553>`_, yonghengdexin735)
+* bluestore: os/bluestore/BlueStore.cc: remove unused variable (`pr#12703 <https://github.com/ceph/ceph/pull/12703>`_, Li Wang)
+* bluestore: os/bluestore/BlueStore: no device no symlink (`pr#15721 <https://github.com/ceph/ceph/pull/15721>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueStore: remove unused code (`pr#16522 <https://github.com/ceph/ceph/pull/16522>`_, Jianpeng Ma)
+* bluestore: os/bluestore: cleanup BitAllocator (`pr#12661 <https://github.com/ceph/ceph/pull/12661>`_, xie xingguo)
+* bluestore: os/bluestore: cleanup bluestore_types (`pr#15680 <https://github.com/ceph/ceph/pull/15680>`_, xie xingguo)
+* bluestore: os/bluestore: clean up flush logic (`pr#14162 <https://github.com/ceph/ceph/pull/14162>`_, Jianpeng Ma)
+* bluestore: os/bluestore: cleanup, got rid of table reference of 1<<x (`pr#13718 <https://github.com/ceph/ceph/pull/13718>`_, Adam Kupczyk)
+* bluestore: os/bluestore: clean up Invalid return value judgment (`pr#14219 <https://github.com/ceph/ceph/pull/14219>`_, shiqi)
+* bluestore: os/bluestore: cleanup min_alloc_size; some formatting nits (`pr#15826 <https://github.com/ceph/ceph/pull/15826>`_, xie xingguo)
+* bluestore: os/bluestore: clear result in BlueRocksEnv::getChildren (`issue#20857 <http://tracker.ceph.com/issues/20857>`_, `pr#16683 <https://github.com/ceph/ceph/pull/16683>`_, liuchang0812)
+* bluestore: os/bluestore: clear up redundant size assignment in KerenelDevice (`pr#16121 <https://github.com/ceph/ceph/pull/16121>`_, Shasha Lu)
+* bluestore: os/bluestore: conditionally load crr option (`pr#12877 <https://github.com/ceph/ceph/pull/12877>`_, xie xingguo)
+* bluestore: os/bluestore: configure rocksdb cache via bluestore_cache_kv_ratio (`pr#15580 <https://github.com/ceph/ceph/pull/15580>`_, Sage Weil)
+* bluestore: os/bluestore: default journal media to store media if bluefs is disabled (`pr#16844 <https://github.com/ceph/ceph/pull/16844>`_, xie xingguo)
+* bluestore: os/bluestore: _do_remove: dirty shard individually as each blob is unshared (`issue#20849 <http://tracker.ceph.com/issues/20849>`_, `pr#16822 <https://github.com/ceph/ceph/pull/16822>`_, Sage Weil)
+* bluestore: os/blueStore: Failure retry for opening file (`pr#16237 <https://github.com/ceph/ceph/pull/16237>`_, Yankun Li)
+* bluestore: os/bluestore: fix a bug in small write handling on sharded extents (`pr#13728 <https://github.com/ceph/ceph/pull/13728>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix Allocator::allocate() int truncation (`issue#18595 <http://tracker.ceph.com/issues/18595>`_, `pr#13010 <https://github.com/ceph/ceph/pull/13010>`_, Sage Weil)
+* bluestore: os/bluestore: fix a typo about bleustore (`pr#15357 <https://github.com/ceph/ceph/pull/15357>`_, Dongsheng Yang)
+* bluestore: os/bluestore: fix BitMapAllocator assert on out-of-bound hint value (`pr#15289 <https://github.com/ceph/ceph/pull/15289>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix buffers pinned by indefinitely deferred writes (`pr#15398 <https://github.com/ceph/ceph/pull/15398>`_, Sage Weil)
+* bluestore: os/bluestore: fix bug for calc extent_avg in reshard function (`pr#13931 <https://github.com/ceph/ceph/pull/13931>`_, wangzhengyong)
+* bluestore: os/bluestore: fix bug in aio_read() (`pr#13511 <https://github.com/ceph/ceph/pull/13511>`_, tangwenjun)
+* bluestore: os/bluestore: fix bug in _open_alloc() (`pr#13577 <https://github.com/ceph/ceph/pull/13577>`_, yonghengdexin735)
+* bluestore: os/bluestore: fix bug in _open_super_meta() (`pr#13559 <https://github.com/ceph/ceph/pull/13559>`_, Taeksang Kim)
+* bluestore: os/bluestore: fix bugs in bluefs and bdev flush (`issue#19250 <http://tracker.ceph.com/issues/19250>`_, `issue#19251 <http://tracker.ceph.com/issues/19251>`_, `pr#13911 <https://github.com/ceph/ceph/pull/13911>`_, Sage Weil)
+* bluestore: os/bluestore: fix coredump in register_ctrlr() (`pr#13556 <https://github.com/ceph/ceph/pull/13556>`_, tangwenjun)
+* bluestore: os/bluestore: fix deferred_aio deadlock (`pr#16051 <https://github.com/ceph/ceph/pull/16051>`_, Sage Weil)
+* bluestore: os/bluestore: fix deferred write race (`issue#19880 <http://tracker.ceph.com/issues/19880>`_, `pr#15004 <https://github.com/ceph/ceph/pull/15004>`_, Sage Weil)
+* bluestore: os/bluestore: fix deferred writes vs collection split race (`issue#19379 <http://tracker.ceph.com/issues/19379>`_, `pr#14157 <https://github.com/ceph/ceph/pull/14157>`_, Sage Weil)
+* bluestore: os/bluestore: fix dirty_range on _do_clone_range (`issue#20810 <http://tracker.ceph.com/issues/20810>`_, `pr#16738 <https://github.com/ceph/ceph/pull/16738>`_, Sage Weil)
+* bluestore: os/bluestore: fix false assert in IOContext::aio_wake (`pr#15268 <https://github.com/ceph/ceph/pull/15268>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix false asserts in Cache::trim_all() (`pr#15470 <https://github.com/ceph/ceph/pull/15470>`_, xie xingguo)
+* bluestore: os/bluestore: fix fsck deferred_replay (`pr#15295 <https://github.com/ceph/ceph/pull/15295>`_, Sage Weil)
+* bluestore: os/bluestore: fix min_alloc_size at mkfs time (`pr#13192 <https://github.com/ceph/ceph/pull/13192>`_, Sage Weil)
+* bluestore: os/bluestore: fix narrow osr->flush() race (`pr#14489 <https://github.com/ceph/ceph/pull/14489>`_, Sage Weil)
+* bluestore: os/bluestore: fix NVMEDevice::open failure if serial number ends with a … (`pr#12956 <https://github.com/ceph/ceph/pull/12956>`_, Hongtong Liu)
+* bluestore: os/bluestore: fix OnodeSizeTracking testing (`issue#20498 <http://tracker.ceph.com/issues/20498>`_, `pr#12684 <https://github.com/ceph/ceph/pull/12684>`_, xie xingguo)
+* bluestore: os/bluestore: fix perf counters (`pr#13965 <https://github.com/ceph/ceph/pull/13965>`_, Sage Weil)
+* bluestore: os/bluestore: fix possible out of order shard(offset == 0); add sanity check (`pr#15658 <https://github.com/ceph/ceph/pull/15658>`_, xie xingguo)
+* bluestore: os/bluestore: fix potential access violation (`pr#15657 <https://github.com/ceph/ceph/pull/15657>`_, xie xingguo)
+* bluestore: os/bluestore: fix potential assert in cache _trim method (`pr#13234 <https://github.com/ceph/ceph/pull/13234>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix reclaim_blocks and clean up Allocator interface (`issue#18573 <http://tracker.ceph.com/issues/18573>`_, `pr#12963 <https://github.com/ceph/ceph/pull/12963>`_, Sage Weil)
+* bluestore: os/bluestore: fix typo(s/trasnaction/transaction/) (`pr#14890 <https://github.com/ceph/ceph/pull/14890>`_, xie xingguo)
+* bluestore: os/bluestore: fix unsharing blob dirty_range args (`issue#20227 <http://tracker.ceph.com/issues/20227>`_, `pr#15766 <https://github.com/ceph/ceph/pull/15766>`_, Sage Weil)
+* bluestore: os/bluestore: fix use after free race with aio_wait (`pr#14956 <https://github.com/ceph/ceph/pull/14956>`_, Sage Weil)
+* bluestore: os/bluestore: fix wal-queue bytes-counter to keep pace with others (`pr#13382 <https://github.com/ceph/ceph/pull/13382>`_, xie xingguo)
+* bluestore: os/bluestore: fsck: verify blob.unused field (`pr#14316 <https://github.com/ceph/ceph/pull/14316>`_, Sage Weil)
+* bluestore: os/bluestore: handle rounding error in cache ratios (`pr#15672 <https://github.com/ceph/ceph/pull/15672>`_, Sage Weil)
+* bluestore: os/bluestore: implement collect_metadata (`pr#14115 <https://github.com/ceph/ceph/pull/14115>`_, Sage Weil)
+* bluestore: os/bluestore: include logical object offset in crc error (`pr#13074 <https://github.com/ceph/ceph/pull/13074>`_, Sage Weil)
+* bluestore: os/bluestore: initialize finishers properly (`pr#15666 <https://github.com/ceph/ceph/pull/15666>`_, xie xingguo)
+* bluestore: os/bluestore/KernelDevice: fix comments (`pr#15264 <https://github.com/ceph/ceph/pull/15264>`_, xie xingguo)
+* bluestore: os/bluestore/KernelDevice: fix debug message (`pr#13135 <https://github.com/ceph/ceph/pull/13135>`_, Sage Weil)
+* bluestore: os/bluestore/KernelDevice: helpful warning when aio limit exhausted (`pr#15116 <https://github.com/ceph/ceph/pull/15116>`_, Sage Weil)
+* bluestore: os/bluestore/KernelDevice: kill zeros (`pr#12856 <https://github.com/ceph/ceph/pull/12856>`_, xie xingguo)
+* bluestore: os/bluestore: kill BufferSpace.empty() (`pr#12871 <https://github.com/ceph/ceph/pull/12871>`_, xie xingguo)
+* bluestore: os/bluestore: kill orphan declaration of do_write_check_depth() (`pr#12853 <https://github.com/ceph/ceph/pull/12853>`_, xie xingguo)
+* bluestore: os/bluestore: leverage the type knowledge in BitMapAreaLeaf (`pr#13736 <https://github.com/ceph/ceph/pull/13736>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: Make BitmapFreelistManager kv itereator short lived (`pr#16243 <https://github.com/ceph/ceph/pull/16243>`_, Mark Nelson)
+* bluestore: os/bluestore: make live changes for BlueStore throttle config work like initial config (`pr#14225 <https://github.com/ceph/ceph/pull/14225>`_, J. Eric Ivancich)
+* bluestore: os/bluestore: miscellaneous fixes to BitAllocator (`pr#12696 <https://github.com/ceph/ceph/pull/12696>`_, xie xingguo)
+* bluestore: os/bluestore: misc fix and cleanups (`pr#16315 <https://github.com/ceph/ceph/pull/16315>`_, Jianpeng Ma)
+* bluestore: os/bluestore: misc fixes (`pr#14333 <https://github.com/ceph/ceph/pull/14333>`_, Sage Weil)
+* bluestore: os/bluestore: move aio.h/cc from fs dir to bluestore dir (`pr#16409 <https://github.com/ceph/ceph/pull/16409>`_, Pan Liu)
+* bluestore: os/bluestore: move object exist in assign nid (`pr#16117 <https://github.com/ceph/ceph/pull/16117>`_, Jianpeng Ma)
+* bluestore: os/bluestore: move sharedblob to new collection in same shard (`issue#20358 <http://tracker.ceph.com/issues/20358>`_, `pr#15783 <https://github.com/ceph/ceph/pull/15783>`_, Sage Weil)
+* bluestore: os/bluestore: narrow cache lock range; make sure min_alloc_size p2 aligned (`pr#15911 <https://github.com/ceph/ceph/pull/15911>`_, xie xingguo)
+* bluestore: os/bluestore: "noid" is not always necessary in clone op (`pr#13769 <https://github.com/ceph/ceph/pull/13769>`_, wangzhengyong)
+* bluestore: os/bluestore: nullptr in OmapIteratorImpl::valid (`pr#12900 <https://github.com/ceph/ceph/pull/12900>`_, Xinze Chi)
+* bluestore: os/bluestore/NVMEDevice: Add multiple thread support for SPDK I/O thread (`pr#14420 <https://github.com/ceph/ceph/pull/14420>`_, Ziye Yang)
+* bluestore: os/bluestore/NVMEDevice.cc: fix the random read issue (`pr#13055 <https://github.com/ceph/ceph/pull/13055>`_, optimistyzy)
+* bluestore: os/bluestore/NVMEDevice: fix the compilation issue for collect_metadata (`pr#14455 <https://github.com/ceph/ceph/pull/14455>`_, optimistyzy)
+* bluestore: os/bluestore/NVMEdevice: fix the unrelease segs issue (`pr#12862 <https://github.com/ceph/ceph/pull/12862>`_, optimistyzy)
+* bluestore: os/bluestore: only submit deferred if there is any (`pr#16269 <https://github.com/ceph/ceph/pull/16269>`_, Sage Weil)
+* bluestore: os/bluestore: preallocate object[extent_shard] key to avoid reallocate (`pr#12644 <https://github.com/ceph/ceph/pull/12644>`_, xie xingguo)
+* bluestore: os/bluestore: pre-calculate number of ghost buffers to evict (`pr#15029 <https://github.com/ceph/ceph/pull/15029>`_, xie xingguo)
+* bluestore: os/bluestore: put strings in mempool (`pr#12651 <https://github.com/ceph/ceph/pull/12651>`_, Allen Samuels, Sage Weil)
+* bluestore: os/bluestore: Record l_bluestore_state_kv_queued_lat for sync_submit (`pr#14448 <https://github.com/ceph/ceph/pull/14448>`_, Jianpeng Ma)
+* bluestore: os/bluestore: reduce some overhead for _do_clone_range() and _do_remove() (`pr#15944 <https://github.com/ceph/ceph/pull/15944>`_, xie xingguo)
+* bluestore: os/bluestore: refactor BlueStore::_do_write;  kill dead ExtentMap::find_lextent() method (`pr#15750 <https://github.com/ceph/ceph/pull/15750>`_, xie xingguo)
+* bluestore: os/bluestore: refactor ExtentMap::update to avoid preceeding db updat… (`pr#12394 <https://github.com/ceph/ceph/pull/12394>`_, Igor Fedotov)
+* bluestore: os/bluestore: remove a never read value (`pr#12618 <https://github.com/ceph/ceph/pull/12618>`_, liuchang0812)
+* bluestore: os/bluestore: Remove ExtentFreeListManager (`pr#14772 <https://github.com/ceph/ceph/pull/14772>`_, Jianpeng Ma)
+* bluestore: os/bluestore: remove intermediate key var to avoid string copy (`pr#12643 <https://github.com/ceph/ceph/pull/12643>`_, xie xingguo)
+* bluestore: os/bluestore: remove no use parameter in bluestore_blob_t::map_bl (`pr#13013 <https://github.com/ceph/ceph/pull/13013>`_, wangzhengyong)
+* bluestore: os/bluestore: remove unneeded indirection in BitMapZone (`pr#13743 <https://github.com/ceph/ceph/pull/13743>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: remove unused condition variable (`pr#14973 <https://github.com/ceph/ceph/pull/14973>`_, Igor Fedotov)
+* bluestore: os/bluestore: remove unused local variable "pos" (`pr#13715 <https://github.com/ceph/ceph/pull/13715>`_, wangzhengyong)
+* bluestore: os/bluestore: remove unused variables (`pr#15718 <https://github.com/ceph/ceph/pull/15718>`_, zhanglei)
+* bluestore: os/bluestore: rename/fix throttle options (`pr#14717 <https://github.com/ceph/ceph/pull/14717>`_, Sage Weil)
+* bluestore: os/bluestore: roundoff bluefs allocs to bluefs_alloc_size (`pr#14876 <https://github.com/ceph/ceph/pull/14876>`_, Ramesh Chander)
+* bluestore: os/bluestore: shrink buffer_map key into uint32_t (`pr#12850 <https://github.com/ceph/ceph/pull/12850>`_, xie xingguo)
+* bluestore: os/bluestore: slightly refactor Blob::try_reuse_blob (`pr#15836 <https://github.com/ceph/ceph/pull/15836>`_, xie xingguo)
+* bluestore: os/bluestore: some cleanup (`pr#13390 <https://github.com/ceph/ceph/pull/13390>`_, liuchang0812)
+* bluestore: os/bluestore: space between func and contents (`pr#16804 <https://github.com/ceph/ceph/pull/16804>`_, xie xingguo)
+* bluestore: os/bluestore: stop calculating bound if we must reshard; narrow shard combination condition (`pr#15631 <https://github.com/ceph/ceph/pull/15631>`_, xie xingguo)
+* bluestore: os/bluestore/StupidAllocator: rounded down len to an align boundary (`issue#20660 <http://tracker.ceph.com/issues/20660>`_, `pr#16593 <https://github.com/ceph/ceph/pull/16593>`_, Zhu Shangzhong)
+* bluestore: os/bluestore: target_bytes should scale with meta/data ratios (`pr#15708 <https://github.com/ceph/ceph/pull/15708>`_, Mark Nelson)
+* bluestore: os/bluestore: _txc_release_alloc when do wal cleaning (`pr#12692 <https://github.com/ceph/ceph/pull/12692>`_, Xinze Chi)
+* bluestore: os/bluestore: use bufferlist functions whenever possible (`pr#16158 <https://github.com/ceph/ceph/pull/16158>`_, Jianpeng Ma)
+* bluestore: os/bluestore: use correct bound encode size for unused (`pr#14731 <https://github.com/ceph/ceph/pull/14731>`_, Haomai Wang)
+* bluestore: os/bluestore: use reference to avoid string copy (`pr#16364 <https://github.com/ceph/ceph/pull/16364>`_, Pan Liu)
+* bluestore: os: extend ObjectStore interface to dump store's performance counters (`pr#13203 <https://github.com/ceph/ceph/pull/13203>`_, Igor Fedotov)
+* bluestore,performance: common/config_opts.h: compaction readahead for bluestore/rocksdb (`pr#14932 <https://github.com/ceph/ceph/pull/14932>`_, Mark Nelson)
+* bluestore,performance: kv/RocksDBStore: implement rm_range_keys operator interface and test (`pr#13855 <https://github.com/ceph/ceph/pull/13855>`_, Haomai Wang)
+* bluestore,performance: os/aio: remove the redundant memset(struct iocb) (`pr#13662 <https://github.com/ceph/ceph/pull/13662>`_, Jianpeng Ma)
+* bluestore,performance: os/bluestore: add bluestore_prefer_wal_size option (`pr#13217 <https://github.com/ceph/ceph/pull/13217>`_, Sage Weil)
+* bluestore,performance: os/bluestore: avoid overloading extents during reshard; atomic deferred_batch_ops (`pr#15502 <https://github.com/ceph/ceph/pull/15502>`_, xie xingguo)
+* bluestore,performance: os/bluestore: avoid the VTABLE-related burden in BitMapAllocator's hotspot (`pr#14348 <https://github.com/ceph/ceph/pull/14348>`_, Radoslaw Zarzynski)
+* bluestore,performance: os/bluestore: batch throttle (`pr#15284 <https://github.com/ceph/ceph/pull/15284>`_, Jianpeng Ma)
+* bluestore,performance: os/bluestore/BlueFS: add bluefs_sync_write option (`pr#14510 <https://github.com/ceph/ceph/pull/14510>`_, Sage Weil)
+* bluestore,performance: os/bluestore/BlueFS: optimize get_allocated (`pr#14121 <https://github.com/ceph/ceph/pull/14121>`_, Jianpeng Ma)
+* bluestore,performance: os/bluestore/BlueFS: tune flushing of writes (`pr#13032 <https://github.com/ceph/ceph/pull/13032>`_, Sage Weil)
+* bluestore,performance: os/bluestore/bluestore_types: drop std::bitset for blob unused (`pr#12569 <https://github.com/ceph/ceph/pull/12569>`_, Sage Weil)
+* bluestore,performance: os/bluestore: cap rocksdb cache size (`pr#15786 <https://github.com/ceph/ceph/pull/15786>`_, Mark Nelson)
+* bluestore,performance: os/bluestore: default 16KB min_alloc_size on ssd (`pr#14076 <https://github.com/ceph/ceph/pull/14076>`_, Sage Weil)
+* bluestore,performance: os/bluestore: default cache size of 3gb (`pr#15976 <https://github.com/ceph/ceph/pull/15976>`_, Sage Weil)
+* bluestore,performance: os/bluestore: differ default cache size for hdd/ssd backends (`pr#16157 <https://github.com/ceph/ceph/pull/16157>`_, xie xingguo)
+* bluestore,performance: os/bluestore: do not balance bluefs on every kv_sync_thread iteration (`pr#14557 <https://github.com/ceph/ceph/pull/14557>`_, Sage Weil)
+* bluestore,performance: os/bluestore: do not cache shard keys (`pr#12634 <https://github.com/ceph/ceph/pull/12634>`_, Sage Weil)
+* bluestore,performance: os/bluestore: eliminate some excessive stuff (`pr#14675 <https://github.com/ceph/ceph/pull/14675>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore: fix deferred writes; improve flush (`pr#13888 <https://github.com/ceph/ceph/pull/13888>`_, Sage Weil)
+* bluestore,performance: os/bluestore: generate same onode extent-shard keys in a more efficient way (`pr#12681 <https://github.com/ceph/ceph/pull/12681>`_, xie xingguo)
+* bluestore,performance: os/bluestore: get rid off excessive lock at BitMapAllocator (`pr#14749 <https://github.com/ceph/ceph/pull/14749>`_, Igor Fedotov)
+* bluestore,performance: os/blueStore: In osd_tp_thread, call _txc_finalize_kv (`pr#14709 <https://github.com/ceph/ceph/pull/14709>`_, Jianpeng Ma)
+* bluestore,performance: os/bluestore: keep statfs replica in RAM to avoid expensive KV retrieval (`pr#15309 <https://github.com/ceph/ceph/pull/15309>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore/KernelDevice: batch aio submit (`pr#16032 <https://github.com/ceph/ceph/pull/16032>`_, Haodong Tang)
+* bluestore,performance: os/bluestore/KernelDevice: fix sync write vs flush (`pr#15034 <https://github.com/ceph/ceph/pull/15034>`_, Sage Weil)
+* bluestore,performance: os/bluestore: kvdb histogram (`pr#12620 <https://github.com/ceph/ceph/pull/12620>`_, Varada Kari)
+* bluestore,performance: os/bluestore: make bluestore_max_blob_size parameter hdd/ssd case dependant (`pr#14434 <https://github.com/ceph/ceph/pull/14434>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore: memory and dereference clean-up in the BitAllocator (`pr#13811 <https://github.com/ceph/ceph/pull/13811>`_, Radoslaw Zarzynski)
+* bluestore,performance: os/bluestore: move cache_trim into MempoolThread (`pr#15380 <https://github.com/ceph/ceph/pull/15380>`_, xie xingguo)
+* bluestore,performance: os/bluestore: optimize blob usage when doing appends/overwrites (`pr#13337 <https://github.com/ceph/ceph/pull/13337>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore: optimized (encode|decode)_escaped (`pr#15759 <https://github.com/ceph/ceph/pull/15759>`_, Piotr Dałek)
+* bluestore,performance: os/bluestore: partial reshard support (`pr#13162 <https://github.com/ceph/ceph/pull/13162>`_, Sage Weil)
+* bluestore,performance: os/bluestore: prevent lock for almost "flush" calls (`pr#12524 <https://github.com/ceph/ceph/pull/12524>`_, Haomai Wang)
+* bluestore,performance: os/bluestore: put bluefs in the middle of the shared device (`pr#14873 <https://github.com/ceph/ceph/pull/14873>`_, Sage Weil)
+* bluestore,performance: os/bluestore: refactor small write handling to reuse blob more effect… (`pr#14399 <https://github.com/ceph/ceph/pull/14399>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore: remove CephContext\* from BmapEntry (`pr#13651 <https://github.com/ceph/ceph/pull/13651>`_, Radoslaw Zarzynski)
+* bluestore,performance: os/bluestore: replace Blob ref_map with reference counting (`pr#12904 <https://github.com/ceph/ceph/pull/12904>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore: rewrite deferred write handling (`issue#16644 <http://tracker.ceph.com/issues/16644>`_, `pr#14491 <https://github.com/ceph/ceph/pull/14491>`_, Sage Weil)
+* bluestore,performance: os/bluestore: separate kv_sync_thread into two parts (`pr#14035 <https://github.com/ceph/ceph/pull/14035>`_, Jianpeng Ma, Igor Fedotov, Sage Weil)
+* bluestore,performance: os/bluestore: set cache meta ratio to .9 (`pr#12635 <https://github.com/ceph/ceph/pull/12635>`_, Sage Weil)
+* bluestore,performance: os/bluestore: the exhausted check in BitMapZone can be lock-less (`pr#13653 <https://github.com/ceph/ceph/pull/13653>`_, Radoslaw Zarzynski)
+* bluestore,performance: os/bluestore: try to unshare blobs for EC overwrite workload (`pr#14239 <https://github.com/ceph/ceph/pull/14239>`_, Sage Weil)
+* bluestore,performance: os/bluestore: tune deferred_batch_ops separately for hdd and ssd (`pr#14435 <https://github.com/ceph/ceph/pull/14435>`_, Sage Weil)
+* bluestore,performance: os/bluestore: unify throttling model (`issue#19542 <http://tracker.ceph.com/issues/19542>`_, `pr#14306 <https://github.com/ceph/ceph/pull/14306>`_, Sage Weil)
+* bluestore,performance: os/bluestore: use aio for reads (`issue#19030 <http://tracker.ceph.com/issues/19030>`_, `pr#13066 <https://github.com/ceph/ceph/pull/13066>`_, Sage Weil)
+* bluestore,performance: os/bluestore: use Best-Effort policy when evicting onode from cache (`pr#12876 <https://github.com/ceph/ceph/pull/12876>`_, xie xingguo)
+* bluestore,performance: os/bluestore: use denc for varint encoding (`pr#14911 <https://github.com/ceph/ceph/pull/14911>`_, Piotr Dałek)
+* bluestore,performance: os/bluestore: various onode changes to reduce its in-memory footprint (`pr#12700 <https://github.com/ceph/ceph/pull/12700>`_, Igor Fedotov)
+* bluestore,performance: os/fs/aio: use small_vector for aio_t; clean up header location (`pr#14853 <https://github.com/ceph/ceph/pull/14853>`_, Sage Weil)
+* bluestore: rocksdb: add option: writable_file_max_buffer_size = 0 (`pr#12562 <https://github.com/ceph/ceph/pull/12562>`_, Jianpeng Ma)
+* bluestore,tests: ceph-dencoder: enable bluestore types (`pr#13595 <https://github.com/ceph/ceph/pull/13595>`_, Willem Jan Withagen, Kefu Chai)
+* bluestore,tests: ceph_test_objectstore: match clone_range src and dst offset (`pr#13211 <https://github.com/ceph/ceph/pull/13211>`_, Sage Weil)
+* bluestore,tests: qa/objectstore/bluestore\*: fsck on mount (`pr#15785 <https://github.com/ceph/ceph/pull/15785>`_, Sage Weil)
+* bluestore,tests: test/ceph-test-objectstore: Don't always include BlueStore code (`pr#13516 <https://github.com/ceph/ceph/pull/13516>`_, Willem Jan Withagen)
+* bluestore,tests: test/objectstore/store_test_fixture.cc: Exclude bluestore code if required (`pr#14085 <https://github.com/ceph/ceph/pull/14085>`_, Willem Jan Withagen)
+* bluestore,tests: test/store_test: add deferred test case setup to support explicit min… (`issue#18857 <http://tracker.ceph.com/issues/18857>`_, `pr#13415 <https://github.com/ceph/ceph/pull/13415>`_, Igor Fedotov)
+* bluestore,tests: test/store_test: fix bluestore test cases disablement (`pr#14228 <https://github.com/ceph/ceph/pull/14228>`_, Igor Fedotov)
+* bluestore,tests: test/unittest_bluefs: check whether add_block_device success (`pr#14013 <https://github.com/ceph/ceph/pull/14013>`_, shiqi)
+* bluestore,tests: test/unittest_bluefs: When fsync ret is less than 0, fsync can not be… (`pr#15365 <https://github.com/ceph/ceph/pull/15365>`_, shiqi)
+* bluestore,tests: unittest_alloc: add test_alloc_big (`issue#16662 <http://tracker.ceph.com/issues/16662>`_, `pr#14844 <https://github.com/ceph/ceph/pull/14844>`_, Sage Weil)
+* bluestore,tools: ceph-bluestore-tool: rename from bluefs-tool; improve usage (`pr#14258 <https://github.com/ceph/ceph/pull/14258>`_, Sage Weil)
+* bluestore,tools: ceph-kvstore-tool: allow 'bluestore-kv' as kvdb type; add escaping, compaction (`pr#14718 <https://github.com/ceph/ceph/pull/14718>`_, Sage Weil)
+* bluestore: wrap blob id when it reaches maximum value of int16_t (`issue#19555 <http://tracker.ceph.com/issues/19555>`_, `pr#15654 <https://github.com/ceph/ceph/pull/15654>`_, Xiaoyan Li)
+* build/ops: 12.0.3 (`pr#15600 <https://github.com/ceph/ceph/pull/15600>`_, Jenkins Build Slave User)
+* build/ops: add 12.0.1 release tag in master (`pr#14690 <https://github.com/ceph/ceph/pull/14690>`_, Jenkins Build Slave User)
+* build/ops: add psmisc dependency to ceph-base (deb and rpm) (`issue#19129 <http://tracker.ceph.com/issues/19129>`_, `pr#13744 <https://github.com/ceph/ceph/pull/13744>`_, Nathan Cutler)
+* build/ops: add sanity checks to run-make-check.sh (`pr#12683 <https://github.com/ceph/ceph/pull/12683>`_, Nathan Cutler)
+* build/ops: alpine: add alpine linux dev support (`pr#9853 <https://github.com/ceph/ceph/pull/9853>`_, John Coyle)
+* build/ops: arch: fix build on PowerPC with FreeBSD (`pr#14378 <https://github.com/ceph/ceph/pull/14378>`_, Andrew Solomon)
+* build/ops: arch: fix cmake's ARM CRC intrinsics test to handle duplicitous gcc 4.8.5 (`issue#19386 <http://tracker.ceph.com/issues/19386>`_, `pr#14132 <https://github.com/ceph/ceph/pull/14132>`_, Dan Mick)
+* build/ops: arch: use __get_cpuid instead of do_cpuid (`issue#7869 <http://tracker.ceph.com/issues/7869>`_, `pr#14857 <https://github.com/ceph/ceph/pull/14857>`_, Jos Collin)
+* build/ops: auth: Let's not use the deprecated cephx option (`pr#12721 <https://github.com/ceph/ceph/pull/12721>`_, Dave Chen)
+* build/ops: build: Add Virtuozzo Linux support (`pr#14301 <https://github.com/ceph/ceph/pull/14301>`_, Andrey Parfenov)
+* build/ops: build: build erasure-code isa lib without versions (`pr#16205 <https://github.com/ceph/ceph/pull/16205>`_, James Page)
+* build/ops: build/cmake:  provide asan, tsan, ubsan builds (`pr#12615 <https://github.com/ceph/ceph/pull/12615>`_, Matt Benjamin)
+* build/ops: build: execute dh_systemd_{enable,start} after dh_install (`issue#19585 <http://tracker.ceph.com/issues/19585>`_, `pr#16218 <https://github.com/ceph/ceph/pull/16218>`_, James Page)
+* build/ops: build: move bash_completion.d/ceph to ceph-common (`pr#15148 <https://github.com/ceph/ceph/pull/15148>`_, Leo Zhang)
+* build/ops: build: remove ceph-disk-udev entirely (`pr#15259 <https://github.com/ceph/ceph/pull/15259>`_, Leo Zhang)
+* build/ops: build: remove ceph-qa-suite directory (`pr#13880 <https://github.com/ceph/ceph/pull/13880>`_, Casey Bodley)
+* build/ops: build: revert -Wvla from #15342 (`pr#15469 <https://github.com/ceph/ceph/pull/15469>`_, Willem Jan Withagen)
+* build/ops: builds with dpdk v16.07 (`pr#12707 <https://github.com/ceph/ceph/pull/12707>`_, Kefu Chai)
+* build/ops: build: Use .S suffix for ppc64le assembly files (`issue#20106 <http://tracker.ceph.com/issues/20106>`_, `pr#15373 <https://github.com/ceph/ceph/pull/15373>`_, Andrew Solomon)
+* build/ops: ceph-disk: ability to use a different cluster name with dmcrypt (`issue#17821 <http://tracker.ceph.com/issues/17821>`_, `pr#11786 <https://github.com/ceph/ceph/pull/11786>`_, Sébastien Han, Erwan Velu)
+* build/ops: ceph-disk: don't activate suppressed journal devices (`issue#19489 <http://tracker.ceph.com/issues/19489>`_, `pr#16123 <https://github.com/ceph/ceph/pull/16123>`_, David Disseldorp)
+* build/ops: ceph.in: allow developer mode from outside build tree (`issue#20472 <http://tracker.ceph.com/issues/20472>`_, `pr#16055 <https://github.com/ceph/ceph/pull/16055>`_, Dan Mick)
+* build/ops: ceph_release: we are in the 'rc' phase (12.1.z) (`pr#15957 <https://github.com/ceph/ceph/pull/15957>`_, Sage Weil)
+* build/ops: ceph.spec.in, debian/control: Add bc to build dependencies (`issue#18876 <http://tracker.ceph.com/issues/18876>`_, `pr#13338 <https://github.com/ceph/ceph/pull/13338>`_, Kyr Shatskyy)
+* build/ops: Clean up make check for persistent test nodes (like arm64) (`pr#16773 <https://github.com/ceph/ceph/pull/16773>`_, Dan Mick)
+* build/ops: cmake,crc32c: conditionalize crc32c on different archs (`pr#14289 <https://github.com/ceph/ceph/pull/14289>`_, Kefu Chai)
+* build/ops: CMakeLists.txt: boost_python.so requires libpython.\*.so on FreeBSD (`pr#12763 <https://github.com/ceph/ceph/pull/12763>`_, Willem Jan Withagen)
+* build/ops: CMakeLists.txt: don't do crypto/isa-l if not Intel (`pr#14721 <https://github.com/ceph/ceph/pull/14721>`_, Dan Mick)
+* build/ops: CMakeLists.txt: suppress unneeded warning about jemalloc (`pr#13377 <https://github.com/ceph/ceph/pull/13377>`_, Willem Jan Withagen)
+* build/ops,common: build: Adds C++ warning flag for C Variable-Length Arrays (`pr#15342 <https://github.com/ceph/ceph/pull/15342>`_, Jesse Williamson)
+* build/ops,common: common/blkdev.cc: propagate get_device_by_fd to different OSes (`pr#15547 <https://github.com/ceph/ceph/pull/15547>`_, Willem Jan Withagen)
+* build/ops: common/module.c: do not use strerror_r the GNU way (`pr#12363 <https://github.com/ceph/ceph/pull/12363>`_, Willem Jan Withagen)
+* build/ops: compressor/zstd: add zstd to embedded ceph (`pr#13159 <https://github.com/ceph/ceph/pull/13159>`_, Bassam Tabbara)
+* build/ops: conditionalize rgw Beast frontend so it isn't built on s390x architecture (`issue#20048 <http://tracker.ceph.com/issues/20048>`_, `pr#15225 <https://github.com/ceph/ceph/pull/15225>`_, Willem Jan Withagen, Nathan Cutler, Kefu Chai, Tim Serong, Casey Bodley)
+* build/ops,core: build: let FreeBSD build ceph-fuse (`pr#14282 <https://github.com/ceph/ceph/pull/14282>`_, Willem Jan Withagen)
+* build/ops,core: ceph-disk: use correct user in check_journal_req (`issue#18538 <http://tracker.ceph.com/issues/18538>`_, `pr#12947 <https://github.com/ceph/ceph/pull/12947>`_, Samuel Matzek)
+* build/ops,core: common/freebsd_errno.cc: fix missing (`pr#15741 <https://github.com/ceph/ceph/pull/15741>`_, Willem Jan Withagen)
+* build/ops,core: erasure-code: update ec_isa version + add missing AVX512 ISA-L sources (`pr#15636 <https://github.com/ceph/ceph/pull/15636>`_, Ganesh Mahalingam, Tushar Gohad)
+* build/ops,core: os: allow offline conversion of filestore -> bluestore (or anything else) (`pr#14210 <https://github.com/ceph/ceph/pull/14210>`_, Sage Weil)
+* build/ops,core: osd/OSD: auto class on osd start up (`pr#16014 <https://github.com/ceph/ceph/pull/16014>`_, xie xingguo)
+* build/ops,core: osd/Pool: Disallow enabling 'hashpspool' option to a pool without  '--yes-i-really-mean-it' (`issue#18468 <http://tracker.ceph.com/issues/18468>`_, `pr#13406 <https://github.com/ceph/ceph/pull/13406>`_, Vikhyat Umrao)
+* build/ops,core,tests: osd/dmclock/testing: reorganize testing, building now optional (`issue#19987 <http://tracker.ceph.com/issues/19987>`_, `pr#15375 <https://github.com/ceph/ceph/pull/15375>`_, J. Eric Ivancich)
+* build/ops: debian: Add missing tp files in deb packaging (`pr#13526 <https://github.com/ceph/ceph/pull/13526>`_, Ganesh Mahalingam)
+* build/ops: debian: ceph-mgr: fix package description (`pr#15513 <https://github.com/ceph/ceph/pull/15513>`_, Fabian Grünbichler)
+* build/ops: debian/control: add ceph-base-dbg (`pr#13796 <https://github.com/ceph/ceph/pull/13796>`_, Sage Weil)
+* build/ops: debian: drop boost build dependencies (`pr#13524 <https://github.com/ceph/ceph/pull/13524>`_, Kefu Chai)
+* build/ops: debian: package ceph.logroate properly (`issue#19390 <http://tracker.ceph.com/issues/19390>`_, `pr#14600 <https://github.com/ceph/ceph/pull/14600>`_, Kefu Chai)
+* build/ops: debian: package crypto plugin only on amd64 (`pr#14820 <https://github.com/ceph/ceph/pull/14820>`_, Kefu Chai)
+* build/ops: debian/rpm: move radosgw-admin to ceph-common (`issue#19577 <http://tracker.ceph.com/issues/19577>`_, `pr#14940 <https://github.com/ceph/ceph/pull/14940>`_, Ali Maredia)
+* build/ops: debian/rules, ceph.spec.in: invoke cmake with -DBOOST_J (`pr#14114 <https://github.com/ceph/ceph/pull/14114>`_, Dan Mick)
+* build/ops: debian: sync logrotate packaging with downstream (`issue#19938 <http://tracker.ceph.com/issues/19938>`_, `pr#15567 <https://github.com/ceph/ceph/pull/15567>`_, Fabian Grünbichler)
+* build/ops: debian: workaround the bug in dpkg-maintscript-helper (`issue#20453 <http://tracker.ceph.com/issues/20453>`_, `pr#16072 <https://github.com/ceph/ceph/pull/16072>`_, Kefu Chai)
+* build/ops: debian: wrap-and-sort all files (`pr#16110 <https://github.com/ceph/ceph/pull/16110>`_, James Page)
+* build/ops: dmclock: error: ‘function’ in namespace ‘std’ does not name a template type (`pr#14909 <https://github.com/ceph/ceph/pull/14909>`_, Jos Collin)
+* build/ops: dmclock: include missing <functional> header (`pr#14923 <https://github.com/ceph/ceph/pull/14923>`_, Jos Collin)
+* build/ops: dmclock: initial commit of dmclock QoS library (`pr#14330 <https://github.com/ceph/ceph/pull/14330>`_, J. Eric Ivancich)
+* build/ops: do_cmake.sh: enable ccache if installed (`pr#15274 <https://github.com/ceph/ceph/pull/15274>`_, Sage Weil)
+* build/ops: do_cmake.sh: fix syntax for /bin/sh (doesn't have +=) (`pr#16433 <https://github.com/ceph/ceph/pull/16433>`_, Dan Mick)
+* build/ops: do_freebsd.sh: Remove ENODATA requirement (`pr#13626 <https://github.com/ceph/ceph/pull/13626>`_, Willem Jan Withagen)
+* build/ops: drop libfcgi build dependency (`pr#15285 <https://github.com/ceph/ceph/pull/15285>`_, Nathan Cutler)
+* build/ops: gitignore: Ignore rejects by patch (`pr#14405 <https://github.com/ceph/ceph/pull/14405>`_, Willem Jan Withagen)
+* build/ops: include/assert: test c++ before using static_cast<> (`pr#16424 <https://github.com/ceph/ceph/pull/16424>`_, Kefu Chai)
+* build/ops: init-ceph: add ceph libraries path to environment (`pr#14693 <https://github.com/ceph/ceph/pull/14693>`_, Mohamad Gebai)
+* build/ops: init-ceph: fix ceph user args (`pr#13467 <https://github.com/ceph/ceph/pull/13467>`_, Sage Weil)
+* build/ops: init-ceph: Make init-ceph work under FreeBSD for init-system (`pr#13209 <https://github.com/ceph/ceph/pull/13209>`_, Willem Jan Withagen)
+* build/ops: install-deps.sh: add missing dependencies for FreeBSD (`pr#16545 <https://github.com/ceph/ceph/pull/16545>`_, Alan Somers)
+* build/ops: install-deps.sh: workaround setuptools' dependency on six (`pr#15406 <https://github.com/ceph/ceph/pull/15406>`_, Kefu Chai)
+* build/ops: mailmap: Update OVH contributors (`pr#13063 <https://github.com/ceph/ceph/pull/13063>`_, Bartłomiej Święcki)
+* build/ops: make package groups comply with openSUSE guidelines (`issue#19184 <http://tracker.ceph.com/issues/19184>`_, `pr#13781 <https://github.com/ceph/ceph/pull/13781>`_, Nathan Cutler)
+* build/ops: make-srpm: Pass first parameter to make-dist for building SRPM (`pr#13480 <https://github.com/ceph/ceph/pull/13480>`_, Wido den Hollander)
+* build/ops: merge v12.0.2 release tag (`pr#15091 <https://github.com/ceph/ceph/pull/15091>`_, Jenkins Build Slave User)
+* build/ops,mgr: debian/ceph-base.dirs: create bootstrap-mgr dirs (`pr#14838 <https://github.com/ceph/ceph/pull/14838>`_, Sage Weil)
+* build/ops: miscellaneous cleanups and fixes (run-make-check.sh, ceph.spec.in) (`issue#20091 <http://tracker.ceph.com/issues/20091>`_, `issue#20127 <http://tracker.ceph.com/issues/20127>`_, `pr#15399 <https://github.com/ceph/ceph/pull/15399>`_, Nathan Cutler)
+* build/ops,mon: mon/ConfigKeyService: add 'config-key dump' to show keys and vals (`pr#14858 <https://github.com/ceph/ceph/pull/14858>`_, Dan Mick)
+* build/ops,mon: systemd: Restart Mon after 10s in case of failure (`issue#18635 <http://tracker.ceph.com/issues/18635>`_, `pr#13057 <https://github.com/ceph/ceph/pull/13057>`_, Wido den Hollander)
+* build/ops: msg/async/rdma: compile with rdma as default (`pr#13901 <https://github.com/ceph/ceph/pull/13901>`_, DanielBar-On)
+* build/ops: os/bluestore: fix build errors when spdk is on (`pr#16118 <https://github.com/ceph/ceph/pull/16118>`_, Ilsoo Byun)
+* build/ops: packaging: install libceph-common.so\* not libceph-common.so.\* (`issue#18692 <http://tracker.ceph.com/issues/18692>`_, `pr#13148 <https://github.com/ceph/ceph/pull/13148>`_, Kefu Chai)
+* build/ops,performance: crc32c: Add crc32c function optimized for ppc architecture (`pr#13909 <https://github.com/ceph/ceph/pull/13909>`_, Andrew Solomon)
+* build/ops,performance,rbd: byteorder: use gcc intrinsics for byteswap (`pr#15012 <https://github.com/ceph/ceph/pull/15012>`_, Kefu Chai)
+* build/ops,rbd,rgw: CMakeLists: trim rbd/rgw forced dependencies (`pr#16574 <https://github.com/ceph/ceph/pull/16574>`_, Patrick Donnelly)
+* build/ops,rbd,tests: test/librbd: decouple ceph_test_librbd_api from libceph-common (`issue#20175 <http://tracker.ceph.com/issues/20175>`_, `pr#15611 <https://github.com/ceph/ceph/pull/15611>`_, Kefu Chai)
+* build/ops,rbd,tests: test/librbd: re-enable internal tests in ceph_test_librbd (`pr#16255 <https://github.com/ceph/ceph/pull/16255>`_, Mykola Golub)
+* build/ops,rbd,tests: test: Need to exclude the fsx executable also on FreeBSD (`pr#13686 <https://github.com/ceph/ceph/pull/13686>`_, Willem Jan Withagen)
+* build/ops: Revert "msg/async: increase worker reference with local listen table enabled backend" (`issue#20603 <http://tracker.ceph.com/issues/20603>`_, `pr#16323 <https://github.com/ceph/ceph/pull/16323>`_, Haomai Wang)
+* build/ops: Revert "msg/async/rdma: Debug prints for ibv (`pr#14245 <https://github.com/ceph/ceph/pull/14245>`_, Kefu Chai)
+* build/ops,rgw: rgw_file: radosgw-admin can be built under FreeBSD (`pr#12191 <https://github.com/ceph/ceph/pull/12191>`_, Willem Jan Withagen)
+* build/ops,rgw,tests,tools: vstart: allow to start multiple radosgw when RGW=x (`pr#15632 <https://github.com/ceph/ceph/pull/15632>`_, Adam Kupczyk)
+* build/ops,rgw,tools: vstart: add --rgw_compression to set rgw compression plugin (`pr#15929 <https://github.com/ceph/ceph/pull/15929>`_, Casey Bodley)
+* build/ops: rocksdb: build with ppc64 (`pr#12908 <https://github.com/ceph/ceph/pull/12908>`_, Kefu Chai)
+* build/ops: rocksdb: sync with upstream (`pr#14456 <https://github.com/ceph/ceph/pull/14456>`_, Kefu Chai)
+* build/ops: rocksdb: sync with upstream (`pr#14818 <https://github.com/ceph/ceph/pull/14818>`_, Nathan Cutler, Kefu Chai)
+* build/ops: rpm: apply epoch only if %epoch macro is defined (`pr#15286 <https://github.com/ceph/ceph/pull/15286>`_, Nathan Cutler)
+* build/ops: rpm: build ceph-resource-agents by default (`issue#17613 <http://tracker.ceph.com/issues/17613>`_, `pr#13515 <https://github.com/ceph/ceph/pull/13515>`_, Nathan Cutler)
+* build/ops: rpm: bump epoch ahead of RHEL base (`issue#20508 <http://tracker.ceph.com/issues/20508>`_, `pr#16126 <https://github.com/ceph/ceph/pull/16126>`_, Ken Dreyer)
+* build/ops: rpm,deb: fix ceph-volume (`issue#20915 <http://tracker.ceph.com/issues/20915>`_, `pr#16832 <https://github.com/ceph/ceph/pull/16832>`_, Sage Weil)
+* build/ops: rpm: disable dwz to speed up valgrind (`issue#19099 <http://tracker.ceph.com/issues/19099>`_, `pr#13748 <https://github.com/ceph/ceph/pull/13748>`_, Kefu Chai)
+* build/ops: rpm: drop boost build dependencies (`pr#13519 <https://github.com/ceph/ceph/pull/13519>`_, Nathan Cutler)
+* build/ops: rpm: Drop legacy libxio support (`pr#16449 <https://github.com/ceph/ceph/pull/16449>`_, Nathan Cutler)
+* build/ops: rpm: fix python-Sphinx package name for SUSE (`pr#15015 <https://github.com/ceph/ceph/pull/15015>`_, Nathan Cutler, Jan Matejek)
+* build/ops: rpm: fix typo WTIH_BABELTRACE (`pr#16366 <https://github.com/ceph/ceph/pull/16366>`_, Nathan Cutler)
+* build/ops: rpm: Fix undefined FIRST_ARG (`issue#20077 <http://tracker.ceph.com/issues/20077>`_, `pr#16208 <https://github.com/ceph/ceph/pull/16208>`_, Boris Ranto)
+* build/ops: rpm: gperftools-devel >= 2.4 (`issue#13522 <http://tracker.ceph.com/issues/13522>`_, `pr#14870 <https://github.com/ceph/ceph/pull/14870>`_, Nathan Cutler)
+* build/ops: rpm: make librbd1 %post scriptlet depend on coreutils (`issue#20052 <http://tracker.ceph.com/issues/20052>`_, `pr#15231 <https://github.com/ceph/ceph/pull/15231>`_, Giacomo Comes, Nathan Cutler)
+* build/ops: rpm: move _epoch_prefix below Epoch definition (`pr#15417 <https://github.com/ceph/ceph/pull/15417>`_, Nathan Cutler)
+* build/ops: rpm: move RDMA and python-prettytables build dependencies to distro-conditional section (`pr#15200 <https://github.com/ceph/ceph/pull/15200>`_, Nathan Cutler)
+* build/ops: rpm: obsolete libcephfs1 (`pr#16074 <https://github.com/ceph/ceph/pull/16074>`_, Nathan Cutler)
+* build/ops: rpm: package COPYING, move sample ceph.conf to ceph-common (`pr#15596 <https://github.com/ceph/ceph/pull/15596>`_, Nathan Cutler)
+* build/ops: rpm: package crypto on x86_64 only (`pr#14779 <https://github.com/ceph/ceph/pull/14779>`_, Nathan Cutler)
+* build/ops: rpm: put mgr python build dependencies in make_check bcond (`issue#20425 <http://tracker.ceph.com/issues/20425>`_, `pr#15940 <https://github.com/ceph/ceph/pull/15940>`_, Nathan Cutler, Tim Serong)
+* build/ops: rpm: sane packaging of %{_docdir}/ceph directory (`pr#15900 <https://github.com/ceph/ceph/pull/15900>`_, Nathan Cutler)
+* build/ops: script: adding contributor credits script (`pr#13251 <https://github.com/ceph/ceph/pull/13251>`_, Patrick McGarry)
+* build/ops: script: drop the -x arg for credits script (`pr#14296 <https://github.com/ceph/ceph/pull/14296>`_, Abhishek Lekshmanan)
+* build/ops: script/sepia_bt.sh: download packages from shaman not gitbuilder (`pr#12799 <https://github.com/ceph/ceph/pull/12799>`_, Kefu Chai)
+* build/ops: script/sepia_bt.sh: get sha1,release from t.log if it's not in core (`pr#13620 <https://github.com/ceph/ceph/pull/13620>`_, Kefu Chai)
+* build/ops: script/sepia_bt.sh: support xenial (`pr#13292 <https://github.com/ceph/ceph/pull/13292>`_, Kefu Chai)
+* build/ops: selinux: Allow ceph daemons to read net stats (`issue#19254 <http://tracker.ceph.com/issues/19254>`_, `pr#13945 <https://github.com/ceph/ceph/pull/13945>`_, Boris Ranto)
+* build/ops: selinux: Allow read on var_run_t (`issue#16674 <http://tracker.ceph.com/issues/16674>`_, `pr#15523 <https://github.com/ceph/ceph/pull/15523>`_, Boris Ranto)
+* build/ops: selinux: Do parallel relabel on package install (`issue#20077 <http://tracker.ceph.com/issues/20077>`_, `pr#14871 <https://github.com/ceph/ceph/pull/14871>`_, Boris Ranto)
+* build/ops: selinux: Install ceph-base before ceph-selinux (`issue#20184 <http://tracker.ceph.com/issues/20184>`_, `pr#15490 <https://github.com/ceph/ceph/pull/15490>`_, Boris Ranto)
+* build/ops: Set subman cron attributes in spec file (`issue#20074 <http://tracker.ceph.com/issues/20074>`_, `pr#15270 <https://github.com/ceph/ceph/pull/15270>`_, Thomas Serlin)
+* build/ops: spdk: upgrade spdk to v16.12 (`pr#12734 <https://github.com/ceph/ceph/pull/12734>`_, Pan Liu)
+* build/ops: src/CMakeLists.txt: disable -Werror on rocksdb (`pr#12560 <https://github.com/ceph/ceph/pull/12560>`_, Willem Jan Withagen)
+* build/ops: src/CMakeLists.txt: Move parse_secret_objs setting within definition block (`pr#12785 <https://github.com/ceph/ceph/pull/12785>`_, Willem Jan Withagen)
+* build/ops: src/init-ceph.in:  allow one((re)?start|stop) as commands (`pr#14560 <https://github.com/ceph/ceph/pull/14560>`_, Willem Jan Withagen)
+* build/ops: sync luminous tag back to master (`pr#16758 <https://github.com/ceph/ceph/pull/16758>`_, Jenkins Build Slave User)
+* build/ops: systemd: Add explicit Before=ceph.target (`pr#15835 <https://github.com/ceph/ceph/pull/15835>`_, Tim Serong)
+* build/ops: systemd/ceph-disk: make it possible to customize timeout (`issue#18740 <http://tracker.ceph.com/issues/18740>`_, `pr#13197 <https://github.com/ceph/ceph/pull/13197>`_, Alexey Sheplyakov)
+* build/ops: systemd/ceph-mgr: remove automagic mgr creation hack (`issue#19994 <http://tracker.ceph.com/issues/19994>`_, `pr#16023 <https://github.com/ceph/ceph/pull/16023>`_, Sage Weil)
+* build/ops: systemd: remove ceph-create-keys from presets (`pr#14226 <https://github.com/ceph/ceph/pull/14226>`_, Sébastien Han)
+* build/ops: systemd: Start OSDs after MONs (`issue#18516 <http://tracker.ceph.com/issues/18516>`_, `pr#13097 <https://github.com/ceph/ceph/pull/13097>`_, Boris Ranto)
+* build/ops: test/fio_ceph_objectstore: fix fio plugin build failure caused by rec… (`pr#12655 <https://github.com/ceph/ceph/pull/12655>`_, Igor Fedotov)
+* build/ops,tests: qa: make run-standalone work on FreeBSD (`pr#16595 <https://github.com/ceph/ceph/pull/16595>`_, Willem Jan Withagen)
+* build/ops,tests: test/osd/CMakeLists.txt: osd-dup.sh require BlueStore/AIO (`pr#14387 <https://github.com/ceph/ceph/pull/14387>`_, Willem Jan Withagen)
+* build/ops,tests: test/osd/osd-dup.sh: warn on low open file limit (`pr#14637 <https://github.com/ceph/ceph/pull/14637>`_, Piotr Dałek)
+* build/ops,tests,tools: vstart.sh: Work around mgr restfull not available (`pr#15877 <https://github.com/ceph/ceph/pull/15877>`_, Willem Jan Withagen)
+* build/ops: The Clangtastic Mr. Clocks (`pr#15186 <https://github.com/ceph/ceph/pull/15186>`_, Adam C. Emerson)
+* build/ops: tool: add some ceph relate processes to ps-ceph.pl (`pr#12406 <https://github.com/ceph/ceph/pull/12406>`_, songbaisen)
+* build/ops: tools/scripts:"FreeBSD getopt is not compatible, use the one from packages" (`pr#13260 <https://github.com/ceph/ceph/pull/13260>`_, Willem Jan Withagen)
+* build/ops: tracing: Fix error in including all files in osd_tp (`pr#12501 <https://github.com/ceph/ceph/pull/12501>`_, Ganesh Mahalingam)
+* build/ops: upstart: start radosgw-all according to runlevel (`issue#18313 <http://tracker.ceph.com/issues/18313>`_, `pr#12586 <https://github.com/ceph/ceph/pull/12586>`_, Ken Dreyer)
+* build/ops: vstart: clean up usage a bit (`pr#13138 <https://github.com/ceph/ceph/pull/13138>`_, Sage Weil)
+* build/ops: vstart: do not start mgr if not start_all (`pr#13974 <https://github.com/ceph/ceph/pull/13974>`_, Kefu Chai)
+* build/ops: yasm-wrapper: filter -pthread (`pr#15249 <https://github.com/ceph/ceph/pull/15249>`_, Alessandro Barbieri)
+* build/ops: yasm-wrapper: strip -E (stops ccache trashing source files) (`pr#14633 <https://github.com/ceph/ceph/pull/14633>`_, Tim Serong)
+* cephfs: #11950: Persistent purge queue (`issue#11950 <http://tracker.ceph.com/issues/11950>`_, `pr#12786 <https://github.com/ceph/ceph/pull/12786>`_, John Spray)
+* cephfs: #17980: MDS client blacklisting and blacklist on eviction (`issue#17980 <http://tracker.ceph.com/issues/17980>`_, `issue#9754 <http://tracker.ceph.com/issues/9754>`_, `pr#14610 <https://github.com/ceph/ceph/pull/14610>`_, John Spray)
+* cephfs: #18600: Clear out tasks that don't make sense from multimds suite (`issue#18600 <http://tracker.ceph.com/issues/18600>`_, `pr#13089 <https://github.com/ceph/ceph/pull/13089>`_, John Spray)
+* cephfs: ceph_fuse: fix daemonization when pid file is non-empty (`pr#13532 <https://github.com/ceph/ceph/pull/13532>`_, "Yan, Zheng")
+* cephfs: ceph_fuse: pid_file default to empty (`issue#18309 <http://tracker.ceph.com/issues/18309>`_, `pr#12628 <https://github.com/ceph/ceph/pull/12628>`_, Nathan Cutler)
+* cephfs: ceph-fuse: use user space permission check by default (`issue#19820 <http://tracker.ceph.com/issues/19820>`_, `pr#14907 <https://github.com/ceph/ceph/pull/14907>`_, "Yan, Zheng")
+* cephfs: ceph: simplify CInode::maybe_export_pin() (`pr#15106 <https://github.com/ceph/ceph/pull/15106>`_, "Yan, Zheng")
+* cephfs: client: avoid returning negative space available (`issue#20178 <http://tracker.ceph.com/issues/20178>`_, `pr#15481 <https://github.com/ceph/ceph/pull/15481>`_, John Spray)
+* cephfs: client: call the lru_remove() twice,when trim cache (`pr#15662 <https://github.com/ceph/ceph/pull/15662>`_, huanwen ren)
+* cephfs: client: check for luminous MDS before sending FLUSH_MDLOG (`pr#15805 <https://github.com/ceph/ceph/pull/15805>`_, John Spray)
+* cephfs: client/Client.cc: after reset session from MDS - reconnect (`issue#18757 <http://tracker.ceph.com/issues/18757>`_, `pr#13522 <https://github.com/ceph/ceph/pull/13522>`_, Henrik Korkuc)
+* cephfs: client/Client.cc: prevent segfaulting (`issue#9935 <http://tracker.ceph.com/issues/9935>`_, `pr#12550 <https://github.com/ceph/ceph/pull/12550>`_, Michal Jarzabek)
+* cephfs: client: client_quota no longer optional (`pr#14978 <https://github.com/ceph/ceph/pull/14978>`_, Dan van der Ster)
+* cephfs: client: don't request lookup parent if ino is root (`pr#12478 <https://github.com/ceph/ceph/pull/12478>`_, huanwen ren)
+* cephfs: client: drop cap snaps when auth mds session gets closed (`issue#19022 <http://tracker.ceph.com/issues/19022>`_, `pr#13579 <https://github.com/ceph/ceph/pull/13579>`_, "Yan, Zheng")
+* cephfs: client: fix clang warn of "argument is an uninitialized value" (`pr#12580 <https://github.com/ceph/ceph/pull/12580>`_, liuchang0812)
+* cephfs: 	client: fix Client::handle_cap_flushsnap_ack() crash (`issue#18460 <http://tracker.ceph.com/issues/18460>`_, `pr#12859 <https://github.com/ceph/ceph/pull/12859>`_, Yan, Zheng)
+* cephfs: client: fix Dentry::dump (`pr#15779 <https://github.com/ceph/ceph/pull/15779>`_, huanwen ren)
+* cephfs: client: fix display ino in the ldout (`pr#15314 <https://github.com/ceph/ceph/pull/15314>`_, huanwen ren)
+* cephfs: client: fix potential buffer overflow (`pr#12515 <https://github.com/ceph/ceph/pull/12515>`_, Yunchuan Wen)
+* cephfs: client: fix the cross-quota rename boundary check conditions (`pr#12489 <https://github.com/ceph/ceph/pull/12489>`_, Greg Farnum)
+* cephfs: client: fix UserPerm::gid_in_group() (`issue#19903 <http://tracker.ceph.com/issues/19903>`_, `pr#15039 <https://github.com/ceph/ceph/pull/15039>`_, "Yan, Zheng")
+* cephfs: client: getattr before returning quota/layout xattrs (`issue#17939 <http://tracker.ceph.com/issues/17939>`_, `pr#14018 <https://github.com/ceph/ceph/pull/14018>`_, John Spray)
+* cephfs: client/inode: fix the dump type of Inode::dump() (`pr#15198 <https://github.com/ceph/ceph/pull/15198>`_, huanwen ren)
+* cephfs: client: populate metadata during mount (`issue#18361 <http://tracker.ceph.com/issues/18361>`_, `pr#12915 <https://github.com/ceph/ceph/pull/12915>`_, John Spray)
+* cephfs: client: priority to verify the correctness of the "flag" (`pr#12897 <https://github.com/ceph/ceph/pull/12897>`_, huanwen ren)
+* cephfs: client: refine fsync/close writeback error handling (`pr#14589 <https://github.com/ceph/ceph/pull/14589>`_, John Spray)
+* cephfs: client: remove dead log code (`pr#13093 <https://github.com/ceph/ceph/pull/13093>`_, Patrick Donnelly)
+* cephfs: client: remove request from session->requests when handling forward (`issue#18675 <http://tracker.ceph.com/issues/18675>`_, `pr#13124 <https://github.com/ceph/ceph/pull/13124>`_, "Yan, Zheng")
+* cephfs: client: simplify remove_cap interface (`pr#12161 <https://github.com/ceph/ceph/pull/12161>`_, John Spray)
+* cephfs: client: specify inode in get_caps log message (`pr#13966 <https://github.com/ceph/ceph/pull/13966>`_, John Spray)
+* cephfs: client: wait for lastest osdmap when handling set file/dir layout (`issue#18914 <http://tracker.ceph.com/issues/18914>`_, `pr#13580 <https://github.com/ceph/ceph/pull/13580>`_, "Yan, Zheng")
+* cephfs,common: common/MemoryModel: Bump int to long and drop mallinfo (`pr#13453 <https://github.com/ceph/ceph/pull/13453>`_, Xiaoxi Chen)
+* cephfs,common,core: librados,osdc: kill ack vs commit distinction (`pr#12607 <https://github.com/ceph/ceph/pull/12607>`_, Sage Weil)
+* cephfs,common: include/fs_types: fix unsigned integer overflow (`pr#12440 <https://github.com/ceph/ceph/pull/12440>`_, runsisi)
+* cephfs,common,rbd: blkin: librbd trace hooks (`pr#15053 <https://github.com/ceph/ceph/pull/15053>`_, Victor Araujo, Jason Dillaman)
+* cephfs,common,rbd: osdc: cache should ignore error bhs during trim (`issue#18436 <http://tracker.ceph.com/issues/18436>`_, `pr#12966 <https://github.com/ceph/ceph/pull/12966>`_, Jason Dillaman)
+* cephfs,core: Add test for is_hacky_ecoverwrites in cephfs pool checks (`pr#13466 <https://github.com/ceph/ceph/pull/13466>`_, John Spray)
+* cephfs,core: cleanup: use std::make_shared to replace new (`pr#12276 <https://github.com/ceph/ceph/pull/12276>`_, Yunchuan Wen)
+* cephfs,core,mon: mon/MDSMonitor: fix segv when multiple MDSs raise same alert (`pr#16302 <https://github.com/ceph/ceph/pull/16302>`_, Sage Weil)
+* cephfs: fix mount point break off problem after mds switch occured (`issue#19437 <http://tracker.ceph.com/issues/19437>`_, `pr#14267 <https://github.com/ceph/ceph/pull/14267>`_, Guan yunfei)
+* cephfs: fix write_buf's _len overflow problem (`issue#19033 <http://tracker.ceph.com/issues/19033>`_, `pr#13587 <https://github.com/ceph/ceph/pull/13587>`_, Yang Honggang)
+* cephfs: fs/ceph-fuse: normalize file open flags on the wire (`pr#14822 <https://github.com/ceph/ceph/pull/14822>`_, Jan Fajerski)
+* cephfs: libcephfs.cc: fix memory leak (`pr#12557 <https://github.com/ceph/ceph/pull/12557>`_, Michal Jarzabek)
+* cephfs: libcephfs: cleanups (`pr#12830 <https://github.com/ceph/ceph/pull/12830>`_, huanwen ren)
+* cephfs: libcephfs: fix cct refcount constructing from rados (`pr#12831 <https://github.com/ceph/ceph/pull/12831>`_, John Spray)
+* cephfs: mds/MDBalancer: remove useless check_targets and hit_targets logic from MDS balancer (`issue#20131 <http://tracker.ceph.com/issues/20131>`_, `pr#15407 <https://github.com/ceph/ceph/pull/15407>`_, Zhi Zhang)
+* cephfs: mds/MDLog.cc Fix perf counter type for jlat (`pr#13449 <https://github.com/ceph/ceph/pull/13449>`_, Xiaoxi Chen)
+* cephfs: mds/Server.cc: Don't evict a slow client if (`issue#17855 <http://tracker.ceph.com/issues/17855>`_, `pr#12935 <https://github.com/ceph/ceph/pull/12935>`_, Michal Jarzabek)
+* cephfs: mds/StrayManager: avoid reusing deleted inode in StrayManager::_purge_stray_logged (`issue#18877 <http://tracker.ceph.com/issues/18877>`_, `pr#13347 <https://github.com/ceph/ceph/pull/13347>`_, Zhi Zhang)
+* cephfs,mgr: pybind/mgr/fsstatus: use mds_mem.dn as dentry counter (`pr#15255 <https://github.com/ceph/ceph/pull/15255>`_, Zhi Zhang)
+* cephfs: Mitigation for #16842, validate sessions after load (`issue#16842 <http://tracker.ceph.com/issues/16842>`_, `pr#14164 <https://github.com/ceph/ceph/pull/14164>`_, John Spray)
+* cephfs: mon/FSCommand: fix indentation (`pr#15423 <https://github.com/ceph/ceph/pull/15423>`_, Sage Weil)
+* cephfs: mon/MDSMonitor.cc:refuse fs new on pools with obj (`issue#11124 <http://tracker.ceph.com/issues/11124>`_, `pr#12825 <https://github.com/ceph/ceph/pull/12825>`_, Michal Jarzabek)
+* cephfs: mon/MDSMonitor: respect mds_standby_for_rank config (`pr#15129 <https://github.com/ceph/ceph/pull/15129>`_, "Yan, Zheng")
+* cephfs: mount: do not print "unknown" option to kclient (`issue#18159 <http://tracker.ceph.com/issues/18159>`_, `pr#12465 <https://github.com/ceph/ceph/pull/12465>`_, John Spray)
+* cephfs: osdc/Filer: truncate large file party by party (`issue#19755 <http://tracker.ceph.com/issues/19755>`_, `pr#14769 <https://github.com/ceph/ceph/pull/14769>`_, "Yan, Zheng")
+* cephfs: osdc/Journaler: avoid executing on_safe contexts prematurely (`issue#20055 <http://tracker.ceph.com/issues/20055>`_, `pr#15240 <https://github.com/ceph/ceph/pull/15240>`_, "Yan, Zheng")
+* cephfs: osdc/Journaler: fix memory leak in Journaler::_issue_read() (`issue#20338 <http://tracker.ceph.com/issues/20338>`_, `pr#15776 <https://github.com/ceph/ceph/pull/15776>`_, "Yan, Zheng")
+* cephfs: osdc/Objecter: fix inflight_ops update (`pr#15768 <https://github.com/ceph/ceph/pull/15768>`_, "Yan, Zheng")
+* cephfs: osdc: remove journaler_allow_split_entries option (`issue#19691 <http://tracker.ceph.com/issues/19691>`_, `pr#14636 <https://github.com/ceph/ceph/pull/14636>`_, John Spray)
+* cephfs,performance: client: make seeky readdir more efficiency (`issue#19306 <http://tracker.ceph.com/issues/19306>`_, `pr#14317 <https://github.com/ceph/ceph/pull/14317>`_, "Yan, Zheng")
+* cephfs,performance: mds/server: skip unwanted dn in handle_client_readdir (`pr#12870 <https://github.com/ceph/ceph/pull/12870>`_, Xiaoxi Chen)
+* cephfs: Permit recovering metadata into a new RADOS pool (`issue#15069 <http://tracker.ceph.com/issues/15069>`_, `issue#15068 <http://tracker.ceph.com/issues/15068>`_, `pr#10636 <https://github.com/ceph/ceph/pull/10636>`_, Douglas Fuller)
+* cephfs: qa/cephfs: disable mds_bal_frag for TestStrays.test_purge_queue_op_rate (`issue#19892 <http://tracker.ceph.com/issues/19892>`_, `pr#15105 <https://github.com/ceph/ceph/pull/15105>`_, "Yan, Zheng")
+* cephfs: qa/cephfs: Fix for test_data_scan (`issue#19893 <http://tracker.ceph.com/issues/19893>`_, `pr#15094 <https://github.com/ceph/ceph/pull/15094>`_, Douglas Fuller)
+* cephfs: qa: fix race in Mount.open_background (`issue#18661 <http://tracker.ceph.com/issues/18661>`_, `pr#13137 <https://github.com/ceph/ceph/pull/13137>`_, John Spray)
+* cephfs: qa/suites/fs: reserve more space for mds in full tests (`issue#19891 <http://tracker.ceph.com/issues/19891>`_, `pr#15026 <https://github.com/ceph/ceph/pull/15026>`_, "Yan, Zheng")
+* cephfs: qa/tasks/cephfs: use getattr to guarantee inode is in client cache (`issue#19912 <http://tracker.ceph.com/issues/19912>`_, `pr#15062 <https://github.com/ceph/ceph/pull/15062>`_, "Yan, Zheng")
+* cephfs: qa: unpin knfs from ubuntu (`issue#16397 <http://tracker.ceph.com/issues/16397>`_, `pr#13088 <https://github.com/ceph/ceph/pull/13088>`_, John Spray)
+* cephfs: qa: update log whitelists for kcephfs suite (`pr#14922 <https://github.com/ceph/ceph/pull/14922>`_, "Yan, Zheng")
+* cephfs: qa: update remaining ceph.com to download.ceph.com (`issue#18574 <http://tracker.ceph.com/issues/18574>`_, `pr#12964 <https://github.com/ceph/ceph/pull/12964>`_, John Spray)
+* cephfs: qa: whitelist new fullness messages in fs tests (`issue#19253 <http://tracker.ceph.com/issues/19253>`_, `pr#13915 <https://github.com/ceph/ceph/pull/13915>`_, John Spray)
+* cephfs: Remove "experimental" warnings from multimds (`pr#15154 <https://github.com/ceph/ceph/pull/15154>`_, John Spray, "Yan, Zheng")
+* cephfs: Rewrite mount.fuse.ceph (to python) and move ceph-fuse options to fs_mntops (`pr#11448 <https://github.com/ceph/ceph/pull/11448>`_, Edgaras Lukosevicius)
+* cephfs: tasks/cephfs: fix kernel force umount (`issue#18396 <http://tracker.ceph.com/issues/18396>`_, `pr#12833 <https://github.com/ceph/ceph/pull/12833>`_, Yan, Zheng)
+* cephfs: test/libcephfs: avoid buffer overflow when testing ceph_getdents() (`issue#18941 <http://tracker.ceph.com/issues/18941>`_, `pr#13429 <https://github.com/ceph/ceph/pull/13429>`_, "Yan, Zheng")
+* cephfs,tests: Add multimds:thrash sub-suite and fix bugs in thrasher for multimds (`issue#18690 <http://tracker.ceph.com/issues/18690>`_, `issue#10792 <http://tracker.ceph.com/issues/10792>`_, `pr#13262 <https://github.com/ceph/ceph/pull/13262>`_, Patrick Donnelly)
+* cephfs,tests: ceph-object-corpus: mark MMDSSlaveRequest incompat change (`pr#15730 <https://github.com/ceph/ceph/pull/15730>`_, Sage Weil)
+* cephfs,tests: Improve vstart_runner to (optionally) create its own cluster (`pr#12800 <https://github.com/ceph/ceph/pull/12800>`_, John Spray)
+* cephfs,tests: qa: fix float parse error in test_fragment (`pr#15122 <https://github.com/ceph/ceph/pull/15122>`_, Patrick Donnelly)
+* cephfs,tests: qa: fix test_standby_for_invalid_fscid with vstart_runner (`pr#14272 <https://github.com/ceph/ceph/pull/14272>`_, John Spray)
+* cephfs,tests: qa: handle SSHException in logrotate (`pr#13359 <https://github.com/ceph/ceph/pull/13359>`_, John Spray)
+* cephfs,tests: qa, mds: add checks for fragmentation, and enable it by default (`issue#16523 <http://tracker.ceph.com/issues/16523>`_, `pr#13862 <https://github.com/ceph/ceph/pull/13862>`_, john Spray, John Spray)
+* cephfs,tests: qa: misc cephfs test improvements (`issue#20131 <http://tracker.ceph.com/issues/20131>`_, `pr#15411 <https://github.com/ceph/ceph/pull/15411>`_, John Spray)
+* cephfs,tests: qa: re-enable ENOSPC tests for kclient (`issue#19550 <http://tracker.ceph.com/issues/19550>`_, `pr#14396 <https://github.com/ceph/ceph/pull/14396>`_, John Spray)
+* cephfs,tests: qa: silence spurious insufficient standby health warnings (`pr#15035 <https://github.com/ceph/ceph/pull/15035>`_, Patrick Donnelly)
+* cephfs,tests: qa: silence upgrade test failure (`issue#19934 <http://tracker.ceph.com/issues/19934>`_, `pr#15126 <https://github.com/ceph/ceph/pull/15126>`_, Patrick Donnelly)
+* cephfs,tests: qa: simplify TestJournalRepair (`pr#15096 <https://github.com/ceph/ceph/pull/15096>`_, John Spray)
+* cephfs,tests: qa/tasks: force umount during kclient teardown (`issue#18663 <http://tracker.ceph.com/issues/18663>`_, `pr#13099 <https://github.com/ceph/ceph/pull/13099>`_, John Spray)
+* cephfs,tests: qa: Tidy up fs/ suite (`pr#14575 <https://github.com/ceph/ceph/pull/14575>`_, John Spray)
+* cephfs,tests: qa/vstart_runner: amend ps invocation (`pr#14254 <https://github.com/ceph/ceph/pull/14254>`_, Ilya Dryomov)
+* cephfs,tests: qa: whitelist another fullness log message (`issue#19253 <http://tracker.ceph.com/issues/19253>`_, `pr#14221 <https://github.com/ceph/ceph/pull/14221>`_, John Spray)
+* cephfs,tests: tasks/cephfs: tear down on mount() failure (`pr#13282 <https://github.com/ceph/ceph/pull/13282>`_, John Spray)
+* cephfs: tools/cephfs: remove `apply` mode of cephfs-journal-tool (`pr#15715 <https://github.com/ceph/ceph/pull/15715>`_, John Spray)
+* cephfs: tools/cephfs: set dir_layout when injecting inodes (`issue#19406 <http://tracker.ceph.com/issues/19406>`_, `pr#14234 <https://github.com/ceph/ceph/pull/14234>`_, John Spray)
+* ceph-volume: use unique logical volumes (`pr#17208 <https://github.com/ceph/ceph/pull/17208>`_, Alfredo Deza)
+* cleanup: .gitignore: exclude rpm files (`pr#15745 <https://github.com/ceph/ceph/pull/15745>`_, Leo Zhang)
+* cleanup: Move code from .h into .cc (`pr#12737 <https://github.com/ceph/ceph/pull/12737>`_, Amir Vadai)
+* cleanup: resolve compiler warnings (`pr#13236 <https://github.com/ceph/ceph/pull/13236>`_, Adam C. Emerson)
+* cleanup: src: put-to operator function - const input cleanup (`issue#3977 <http://tracker.ceph.com/issues/3977>`_, `pr#15364 <https://github.com/ceph/ceph/pull/15364>`_, Jos Collin)
+* cmake: add "container" to required boost components (`pr#14850 <https://github.com/ceph/ceph/pull/14850>`_, Kefu Chai)
+* cmake: Add -finstrument-functions flag to OSD code (`pr#15055 <https://github.com/ceph/ceph/pull/15055>`_, Mohamad Gebai)
+* cmake: add RGW and MDS to libcephd (`pr#12345 <https://github.com/ceph/ceph/pull/12345>`_, Bassam Tabbara)
+* cmake: Add simple recursive ctags target for Ceph source only (`pr#14334 <https://github.com/ceph/ceph/pull/14334>`_, Kefu Chai, Dan Mick)
+* cmake: align cmake names of library packages (`issue#19853 <http://tracker.ceph.com/issues/19853>`_, `pr#14951 <https://github.com/ceph/ceph/pull/14951>`_, Nathan Cutler)
+* cmake: Allow tests to build without NSS (`pr#13315 <https://github.com/ceph/ceph/pull/13315>`_, Daniel Gryniewicz)
+* cmake: build boost as an external project (`pr#15376 <https://github.com/ceph/ceph/pull/15376>`_, Kefu Chai)
+* cmake: build tracepoint libraries for vstart target (`pr#14354 <https://github.com/ceph/ceph/pull/14354>`_, Mohamad Gebai)
+* cmake: check the existence of gperf before using it (`pr#15164 <https://github.com/ceph/ceph/pull/15164>`_, Kefu Chai)
+* cmake: cleanup the use of udev and blkid in target_link_lib() (`pr#12811 <https://github.com/ceph/ceph/pull/12811>`_, Willem Jan Withagen)
+* cmake: disable -fvar-tracking-assignments for config.cc (`pr#16695 <https://github.com/ceph/ceph/pull/16695>`_, Kefu Chai)
+* cmake: disable mallinfo for jemalloc (`pr#12469 <https://github.com/ceph/ceph/pull/12469>`_, Bassam Tabbara)
+* cmake: do not add dependencies to INTERFACE library on cmake < 3.3 (`pr#15813 <https://github.com/ceph/ceph/pull/15813>`_, Kefu Chai)
+* cmake: do not compile crush twice (`pr#14725 <https://github.com/ceph/ceph/pull/14725>`_, Kefu Chai)
+* cmake: do not link libcommon against some libs (`pr#15340 <https://github.com/ceph/ceph/pull/15340>`_, Willem Jan Withagen)
+* cmake: do not try to add submodule to exclude list if .git is not around (`pr#14495 <https://github.com/ceph/ceph/pull/14495>`_, Kefu Chai)
+* cmake: enable cross-compilation of boost (`issue#18938 <http://tracker.ceph.com/issues/18938>`_, `pr#14881 <https://github.com/ceph/ceph/pull/14881>`_, Kefu Chai)
+* cmake: exclude \*.css while generating ctags (`pr#15663 <https://github.com/ceph/ceph/pull/15663>`_, Leo Zhang)
+* cmake: explictly call find_package(PythonInterp) first to fix build err (`pr#12385 <https://github.com/ceph/ceph/pull/12385>`_, Yixun Lan)
+* cmake: fix boost components for WITH_SYSTEM_BOOST (`pr#15160 <https://github.com/ceph/ceph/pull/15160>`_, Bassam Tabbara)
+* cmake: Fix broken async/rdma compilation since move to libceph-common (`pr#13122 <https://github.com/ceph/ceph/pull/13122>`_, Oren Duer)
+* cmake: fix broken RDMA compilation after merge PR #12878 (`pr#13186 <https://github.com/ceph/ceph/pull/13186>`_, Oren Duer)
+* cmake: fix hard coded boost python lib (`pr#12480 <https://github.com/ceph/ceph/pull/12480>`_, John Coyle)
+* cmake: fix rpath on shared libraries and binaries targets (`pr#12927 <https://github.com/ceph/ceph/pull/12927>`_, Ricardo Dias)
+* cmake: fix the build with -DWITH_ZFS=ON (`pr#15907 <https://github.com/ceph/ceph/pull/15907>`_, Kefu Chai)
+* cmake: fix the linked lib reference of unittest_rgw_crypto (`pr#14869 <https://github.com/ceph/ceph/pull/14869>`_, Willem Jan Withagen)
+* cmake: improved build speed by 5x when using ccache (`pr#15147 <https://github.com/ceph/ceph/pull/15147>`_, Bassam Tabbara)
+* cmake: kill duplicated cmake commands (`pr#14948 <https://github.com/ceph/ceph/pull/14948>`_, liuchang0812)
+* cmake: link against fcgi only if enabled (`pr#15425 <https://github.com/ceph/ceph/pull/15425>`_, Yao Zongyou)
+* cmake: link ceph-{mgr,mon,mds,osd} against libcommon statically (`pr#12878 <https://github.com/ceph/ceph/pull/12878>`_, Kefu Chai)
+* cmake: link consumers of libclient with libcommon (`issue#18838 <http://tracker.ceph.com/issues/18838>`_, `pr#13394 <https://github.com/ceph/ceph/pull/13394>`_, Kefu Chai)
+* cmake: misc fixes for build on i386 (`pr#15516 <https://github.com/ceph/ceph/pull/15516>`_, James Page)
+* cmake: pass -d0 to b2 if not CMAKE_VERBOSE_MAKEFILE (`pr#14651 <https://github.com/ceph/ceph/pull/14651>`_, Kefu Chai)
+* cmake: remove Findpciaccess.cmake (`pr#12776 <https://github.com/ceph/ceph/pull/12776>`_, optimistyzy)
+* cmake: Rewrite HAVE_BABELTRACE  option to WITH (`pr#15305 <https://github.com/ceph/ceph/pull/15305>`_, Willem Jan Withagen)
+* cmake: rgw: do not link against boost in a wholesale (`pr#15347 <https://github.com/ceph/ceph/pull/15347>`_, Nathan Cutler, Kefu Chai)
+* cmake: search for Keyutils in default paths (`pr#12769 <https://github.com/ceph/ceph/pull/12769>`_, Pascal Bach)
+* cmake: search for nspr include files for both suffixes: nspr4 and nspr (`issue#18535 <http://tracker.ceph.com/issues/18535>`_, `pr#12939 <https://github.com/ceph/ceph/pull/12939>`_, John Lin)
+* cmake: should not compile crc32c_ppc.c on intel arch (`pr#14423 <https://github.com/ceph/ceph/pull/14423>`_, Kefu Chai)
+* cmake: simplify find_package jemalloc (`pr#12468 <https://github.com/ceph/ceph/pull/12468>`_, Bassam Tabbara)
+* cmake: support for external rocksdb (`pr#12467 <https://github.com/ceph/ceph/pull/12467>`_, Bassam Tabbara)
+* cmake: support optional argument for overriding default ctag excludes (`pr#14379 <https://github.com/ceph/ceph/pull/14379>`_, Kefu Chai)
+* cmake: turn libcommon into a shared library (`pr#12840 <https://github.com/ceph/ceph/pull/12840>`_, Kefu Chai)
+* cmake: use CMAKE_INSTALL_INCLUDEDIR (`pr#16483 <https://github.com/ceph/ceph/pull/16483>`_, David Disseldorp)
+* cmake: workaound ccache issue with .S assembly files (`pr#15142 <https://github.com/ceph/ceph/pull/15142>`_, Bassam Tabbara)
+* common: add ceph::size() (`pr#15181 <https://github.com/ceph/ceph/pull/15181>`_, Kefu Chai)
+* common: add override in common and misc (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13443 <https://github.com/ceph/ceph/pull/13443>`_, liuchang0812)
+* common: add override in header file (`pr#13774 <https://github.com/ceph/ceph/pull/13774>`_, liuchang0812)
+* common: add override in msg subsystem (`pr#13771 <https://github.com/ceph/ceph/pull/13771>`_, liuchang0812)
+* common: auth: Enhancement for the supported auth methods (`pr#12937 <https://github.com/ceph/ceph/pull/12937>`_, Dave Chen)
+* common: auth/RotatingKeyRing: use std::move() to set secrets (`pr#15866 <https://github.com/ceph/ceph/pull/15866>`_, Kefu Chai)
+* common: avoid statically allocating configuration options (`issue#20869 <http://tracker.ceph.com/issues/20869>`_, `pr#16735 <https://github.com/ceph/ceph/pull/16735>`_, Jason Dillaman)
+* common: Better handling for missing/inaccessible ceph.conf files (`issue#19658 <http://tracker.ceph.com/issues/19658>`_, `pr#14757 <https://github.com/ceph/ceph/pull/14757>`_, Dan Mick)
+* common: bufferlist: cleanup semantical wrong for bufferlist::append (`pr#12247 <https://github.com/ceph/ceph/pull/12247>`_, Yankun Li)
+* common: buffer: silence unused var warning on FreeBSD (`pr#16452 <https://github.com/ceph/ceph/pull/16452>`_, Willem Jan Withagen)
+* common: ceph_osd: remove client message cap limit (`pr#14944 <https://github.com/ceph/ceph/pull/14944>`_, Haomai Wang)
+* common: ceph: wait for maps before doing 'ceph tell ... help' (`issue#20113 <http://tracker.ceph.com/issues/20113>`_, `pr#16756 <https://github.com/ceph/ceph/pull/16756>`_, Sage Weil)
+* common: cls/log/cls_log.cc: reduce logging noise (`issue#19835 <http://tracker.ceph.com/issues/19835>`_, `pr#14879 <https://github.com/ceph/ceph/pull/14879>`_, Willem Jan Withagen)
+* common: cls: optimize header file dependency (`pr#15165 <https://github.com/ceph/ceph/pull/15165>`_, Brad Hubbard, Xiaowei Chen)
+* common: cmdparse: more constness (`pr#15023 <https://github.com/ceph/ceph/pull/15023>`_, Kefu Chai)
+* common: common/admin_socket: add config for admin socket permission bits (`pr#11684 <https://github.com/ceph/ceph/pull/11684>`_, runsisi)
+* common: common/admin-socket: fix potential buffer overflow (`pr#12518 <https://github.com/ceph/ceph/pull/12518>`_, Yunchuan Wen)
+* common: common/auth: add override in headers (`pr#13692 <https://github.com/ceph/ceph/pull/13692>`_, liuchang0812)
+* common: common/BackTrace: add operator<< (`pr#9028 <https://github.com/ceph/ceph/pull/9028>`_, Kefu Chai)
+* common: common/BackTrace: demangle on FreeBSD also (`pr#12992 <https://github.com/ceph/ceph/pull/12992>`_, Kefu Chai)
+* common: common/buffer: close pipe fd if set nonblocking fails (`pr#12828 <https://github.com/ceph/ceph/pull/12828>`_, donglinpeng)
+* common: common/buffer: off-by-one error in max iov length blocking (`issue#20907 <http://tracker.ceph.com/issues/20907>`_, `pr#16803 <https://github.com/ceph/ceph/pull/16803>`_, Dan Mick)
+* common: common/ceph_context.cc: Use CEPH_DEV to reduce logfile noise (`pr#10384 <https://github.com/ceph/ceph/pull/10384>`_, Willem Jan Withagen)
+* common: common/ceph_context: 'config diff get' option added (`pr#10736 <https://github.com/ceph/ceph/pull/10736>`_, Daniel Oliveira)
+* common: common/ceph_context: fewer warnings about experimental features (`pr#14170 <https://github.com/ceph/ceph/pull/14170>`_, Sage Weil)
+* common: common/ceph_context: fix leak of registered commands on exit (`pr#15302 <https://github.com/ceph/ceph/pull/15302>`_, xie xingguo)
+* common: common/ceph_context: Show clear message if all features are enabled (`pr#12676 <https://github.com/ceph/ceph/pull/12676>`_, Dave Chen)
+* common: common/cmdparse.cc: remove unused variable 'argnum' in dump_cmd_to_json() (`pr#16862 <https://github.com/ceph/ceph/pull/16862>`_, Luo Kexue)
+* common: common/common_init: disable default dout logging for UTILITY_NODOUT too (`issue#20771 <http://tracker.ceph.com/issues/20771>`_, `pr#16578 <https://github.com/ceph/ceph/pull/16578>`_, Sage Weil)
+* common: common/config: Add /usr/local/etc/ceph to default paths (`pr#14797 <https://github.com/ceph/ceph/pull/14797>`_, Willem Jan Withagen)
+* common: common/config: eliminate config_t::set_val unsafe option (`issue#19106 <http://tracker.ceph.com/issues/19106>`_, `pr#13687 <https://github.com/ceph/ceph/pull/13687>`_, liuchang0812)
+* common: common/config: fix return type of string::find and use string::npos (`pr#9924 <https://github.com/ceph/ceph/pull/9924>`_, Yan Jun)
+* common: common,config: OPT_FLOAT and OPT_DOUBLE output format in config show (`issue#20104 <http://tracker.ceph.com/issues/20104>`_, `pr#15647 <https://github.com/ceph/ceph/pull/15647>`_, Yanhu Cao)
+* common: common/config_opt: remove unused config (`pr#15874 <https://github.com/ceph/ceph/pull/15874>`_, alex.wu)
+* common: common/config_opts: drop unused opt (`pr#15876 <https://github.com/ceph/ceph/pull/15876>`_, Yanhu Cao)
+* common: common/config_opts.h: FreeBSD timing changed due to no SO_REUSEADDR (`pr#12594 <https://github.com/ceph/ceph/pull/12594>`_, Willem Jan Withagen)
+* common: common/config_opts.h: Remove deprecated osd_compact_leveldb_on_mount option (`issue#19318 <http://tracker.ceph.com/issues/19318>`_, `pr#14059 <https://github.com/ceph/ceph/pull/14059>`_, Vikhyat Umrao)
+* common: common/config_opts.h: remove obsolete configuration option (`pr#12659 <https://github.com/ceph/ceph/pull/12659>`_, Li Wang)
+* common: common/config_opts: Set the HDD throttle cost to 1.5M (`pr#14808 <https://github.com/ceph/ceph/pull/14808>`_, Mark Nelson)
+* common: common/EventTrace: fix compiler warning (`pr#13659 <https://github.com/ceph/ceph/pull/13659>`_, Jianpeng Ma)
+* common: common/Finisher: fix uninitialized variable warning (`pr#14958 <https://github.com/ceph/ceph/pull/14958>`_, Piotr Dałek)
+* common: common/freebsd_errno.cc: fixed again a stupid typo (`pr#15742 <https://github.com/ceph/ceph/pull/15742>`_, Willem Jan Withagen)
+* common: common/interval_set: return int64_t for size() (`pr#12898 <https://github.com/ceph/ceph/pull/12898>`_, Xinze Chi)
+* common: common/iso_8601.cc: Make return expression Clang compatible (`pr#15336 <https://github.com/ceph/ceph/pull/15336>`_, Willem Jan Withagen)
+* common: common/LogEntry: include EntityName in log entries (`pr#15395 <https://github.com/ceph/ceph/pull/15395>`_, Sage Weil)
+* common: common/Mutex.cc: fixed the error in comment (`pr#16214 <https://github.com/ceph/ceph/pull/16214>`_, Pan Liu)
+* common: common/options: refactors to set the properties in a more structured way (`pr#16482 <https://github.com/ceph/ceph/pull/16482>`_, Kefu Chai)
+* common: common,osdc: remove atomic_t completely (`pr#15562 <https://github.com/ceph/ceph/pull/15562>`_, Kefu Chai)
+* common: common/perf_counters: add average time for PERFCOUNTER_TIME (`pr#15478 <https://github.com/ceph/ceph/pull/15478>`_, xie xingguo)
+* common: common/perf_counters: fix race condition with atomic variables (`pr#14227 <https://github.com/ceph/ceph/pull/14227>`_, J. Eric Ivancich)
+* common: common/perf_counters: make schema more friendly and update docs (`pr#14933 <https://github.com/ceph/ceph/pull/14933>`_, Sage Weil)
+* common: common/perf_counters.: Remove unnecessary judgment (`pr#10407 <https://github.com/ceph/ceph/pull/10407>`_, zhang.zezhu)
+* common: common/simple_spin: use __ppc_yield() on all powerpc archs (`pr#14310 <https://github.com/ceph/ceph/pull/14310>`_, Kefu Chai)
+* common: common,test: migrate atomic_t to std::atomic (`pr#14866 <https://github.com/ceph/ceph/pull/14866>`_, Jesse Williamson)
+* common: common/Timer: do not add event if already shutdown (`issue#20432 <http://tracker.ceph.com/issues/20432>`_, `pr#16201 <https://github.com/ceph/ceph/pull/16201>`_, Kefu Chai)
+* common:   common/WorkQueue: use threadpoolname + threadaddr for heartbeat_han… (`pr#16563 <https://github.com/ceph/ceph/pull/16563>`_, huangjun)
+* common: common/xmlformatter: turn on underscored and add unittest (`pr#12916 <https://github.com/ceph/ceph/pull/12916>`_, liuchang0812)
+* common: compressor/zlib: remove g_ceph_context/g_conf from compressor plugin (`pr#16245 <https://github.com/ceph/ceph/pull/16245>`_, Casey Bodley)
+* common: compressor/zstd: add zstd compression plugin (`pr#13075 <https://github.com/ceph/ceph/pull/13075>`_, Kefu Chai, Sage Weil)
+* common: config: Improve warning for unobserved value (`issue#18424 <http://tracker.ceph.com/issues/18424>`_, `pr#12855 <https://github.com/ceph/ceph/pull/12855>`_, Brad Hubbard)
+* common: config_opt: use bool instead of int for the default value of filestore_debug_omap_check (`pr#15651 <https://github.com/ceph/ceph/pull/15651>`_, Leo Zhang)
+* common,core: ceph_test_rados_api_misc: fix LibRadosMiscConnectFailure.ConnectFailure retry (`issue#19901 <http://tracker.ceph.com/issues/19901>`_, `pr#15522 <https://github.com/ceph/ceph/pull/15522>`_, Sage Weil)
+* common: core/common: Fix ENODATA for FreeBSD with compat.h (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15685 <https://github.com/ceph/ceph/pull/15685>`_, Willem Jan Withagen)
+* common,core: common, osd, tools: Add histograms to performance counters (`pr#12829 <https://github.com/ceph/ceph/pull/12829>`_, Bartłomiej Święcki)
+* common,core: common/pick_address.cc: Copy public_netw to cluster_netw if cluster empty (`pr#12929 <https://github.com/ceph/ceph/pull/12929>`_, Willem Jan Withagen)
+* common,core: common/TracepointProvider: add assert if dlopen error (`pr#13430 <https://github.com/ceph/ceph/pull/13430>`_, Jianpeng Ma)
+* common,core: common/TrackedOp: make TrackedOp::reset_desc() safe (`issue#19110 <http://tracker.ceph.com/issues/19110>`_, `pr#13702 <https://github.com/ceph/ceph/pull/13702>`_, Sage Weil)
+* common,core: mempool: put bloom_filter in mempool (`pr#13009 <https://github.com/ceph/ceph/pull/13009>`_, Sage Weil)
+* common,core: osd,mds,mgr: do not dereference null rotating_keys (`issue#20667 <http://tracker.ceph.com/issues/20667>`_, `pr#16455 <https://github.com/ceph/ceph/pull/16455>`_, Sage Weil)
+* common,core: osd,osdc: pg and osd-based backoff (`pr#12342 <https://github.com/ceph/ceph/pull/12342>`_, Sage Weil)
+* common,core: osd/OSDMap: make osd_state 32 bits wide (`pr#15390 <https://github.com/ceph/ceph/pull/15390>`_, Sage Weil)
+* common,core: osd/OSDMap: replace require_osds flags with a single require_osd_release field (`pr#15068 <https://github.com/ceph/ceph/pull/15068>`_, Sage Weil)
+* common,core: osd/OSDMap: replace string-based min_compat_client with a CEPH_RELEASE uint8_t (`pr#15351 <https://github.com/ceph/ceph/pull/15351>`_, Sage Weil)
+* common,core: osd/osd_types: add flag name (IGNORE_REDIRECT) (`pr#15795 <https://github.com/ceph/ceph/pull/15795>`_, Myoungwon Oh)
+* common,core: rados:  we need to get the latest osdmap when pool does not exists (`pr#13289 <https://github.com/ceph/ceph/pull/13289>`_, song baisen)
+* common,core,tests: Wip cppcheck errors (`pr#14446 <https://github.com/ceph/ceph/pull/14446>`_, Brad Hubbard)
+* common: crc32c: include acconfig.h to fix ceph_crc32c_aarch64() (`pr#15515 <https://github.com/ceph/ceph/pull/15515>`_, Kefu Chai)
+* common: crush/CrushWrapper: fix has_incompat_choose_args (`pr#15218 <https://github.com/ceph/ceph/pull/15218>`_, Sage Weil)
+* common: crush/CrushWrapper: fix has_incompat_choose_args() (`pr#15244 <https://github.com/ceph/ceph/pull/15244>`_, Sage Weil)
+* common: crypto: cleanup NSPR in main thread (`pr#14801 <https://github.com/ceph/ceph/pull/14801>`_, Kefu Chai)
+* common: delete unused conf "filestore_debug_disable_sharded_check" (`pr#13051 <https://github.com/ceph/ceph/pull/13051>`_, Chuanhong Wang)
+* common: denc: add encode/decode for basic_sstring (`pr#15135 <https://github.com/ceph/ceph/pull/15135>`_, Kefu Chai, Casey Bodley)
+* common: do not print error when asok is closed (`pr#14022 <https://github.com/ceph/ceph/pull/14022>`_, Patrick Donnelly)
+* common: fix building against libcryptopp (`pr#14949 <https://github.com/ceph/ceph/pull/14949>`_, Shengjing Zhu)
+* common: Fix clang compilation (`pr#13335 <https://github.com/ceph/ceph/pull/13335>`_, Bartłomiej Święcki)
+* common: Fix heap buffer overflow in do_request (`issue#19393 <http://tracker.ceph.com/issues/19393>`_, `pr#14173 <https://github.com/ceph/ceph/pull/14173>`_, Brad Hubbard)
+* common: fix lockdep vs recursive mutexes (`pr#9940 <https://github.com/ceph/ceph/pull/9940>`_, Adam Kupczyk)
+* common: fix log warnings (`pr#16056 <https://github.com/ceph/ceph/pull/16056>`_, xie xingguo)
+* common: fix Option set_long_description (`pr#16668 <https://github.com/ceph/ceph/pull/16668>`_, Yan Jun)
+* common: fix segfault in public IPv6 addr picking (`issue#19371 <http://tracker.ceph.com/issues/19371>`_, `pr#14124 <https://github.com/ceph/ceph/pull/14124>`_, Fabian Grünbichler)
+* common: fix that $host always expands to localhost instead of actual hostname (`issue#11081 <http://tracker.ceph.com/issues/11081>`_, `pr#12998 <https://github.com/ceph/ceph/pull/12998>`_, liuchang0812)
+* common: fix typo in option of rados_mon_op_timeout's comment (`pr#15681 <https://github.com/ceph/ceph/pull/15681>`_, Leo Zhang)
+* common: Fix unused variable references warnings (`pr#14790 <https://github.com/ceph/ceph/pull/14790>`_, Willem Jan Withagen)
+* common: follow up to new options infrastructure (`pr#16527 <https://github.com/ceph/ceph/pull/16527>`_, John Spray)
+    common: Forward-declare container I/O overloads
+* common: get_process_name: use getprogname on bsd systems (`pr#15338 <https://github.com/ceph/ceph/pull/15338>`_, Mykola Golub)
+* common: get rid of "warning: ignoring return value of ‘strerror_r’" (`pr#12775 <https://github.com/ceph/ceph/pull/12775>`_, xie xingguo)
+* common: global: we need to handle the init_on_startup return value when global_init (`pr#13018 <https://github.com/ceph/ceph/pull/13018>`_, song baisen)
+* common: Implements simple_spin_t in terms of std::atomic_flag (`pr#14370 <https://github.com/ceph/ceph/pull/14370>`_, Jesse Williamson)
+* common: Improved CRC calculation for zero buffers (`pr#11966 <https://github.com/ceph/ceph/pull/11966>`_, Adam Kupczyk)
+* common: include/ceph_features.h uses uint64_t, which is in sys/types.h (`pr#13339 <https://github.com/ceph/ceph/pull/13339>`_, Willem Jan Withagen)
+* common: include/denc: improvements (`pr#12626 <https://github.com/ceph/ceph/pull/12626>`_, Adam C. Emerson)
+* common: include/denc, kv: silence gcc warnings (`pr#13458 <https://github.com/ceph/ceph/pull/13458>`_, Kefu Chai)
+* common: include/denc: remove nullptr runtime magic boundedness check (`pr#13889 <https://github.com/ceph/ceph/pull/13889>`_, Sage Weil)
+* common: include/lru.h: add const to member functions (`pr#15408 <https://github.com/ceph/ceph/pull/15408>`_, yonghengdexin735)
+* common: include/mempool: fix typo in comments (`pr#12772 <https://github.com/ceph/ceph/pull/12772>`_, huangjun)
+* common: include/rados: Fix typo in rados_ioctx_cct() doc (`pr#15220 <https://github.com/ceph/ceph/pull/15220>`_, Jos Collin)
+* common: include: Redo some includes for FreeBSD (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15337 <https://github.com/ceph/ceph/pull/15337>`_, Willem Jan Withagen)
+* common: initialize array in struct BackTrace (`pr#15864 <https://github.com/ceph/ceph/pull/15864>`_, Jos Collin)
+* common: initialize _hash in LogEntryKey() (`pr#15615 <https://github.com/ceph/ceph/pull/15615>`_, Jos Collin)
+* common: int_types.h: remove hacks to workaround old systems (`pr#15069 <https://github.com/ceph/ceph/pull/15069>`_, Kefu Chai)
+* common: kv: resolve a crash issue in ~LevelDBStore() (`pr#16553 <https://github.com/ceph/ceph/pull/16553>`_, wumingqiao)
+* common: librados,libradosstriper,test: migrate atomic_t to std::atomic (baragon) (`pr#14658 <https://github.com/ceph/ceph/pull/14658>`_, Jesse Williamson)
+* common: librados, osd: clang fixes (`pr#13768 <https://github.com/ceph/ceph/pull/13768>`_, Kefu Chai)
+* common: libradosstriper: Add example code (`pr#15350 <https://github.com/ceph/ceph/pull/15350>`_, Logan Blyth)
+* common: libradosstriper: fix format injection vulnerability (`issue#20240 <http://tracker.ceph.com/issues/20240>`_, `pr#15674 <https://github.com/ceph/ceph/pull/15674>`_, Stan K)
+* common: libradosstriper: fix MultiAioCompletion leaks on failure (`pr#15471 <https://github.com/ceph/ceph/pull/15471>`_, Kefu Chai)
+* common: make attempts of auth rotating configurable (`pr#12563 <https://github.com/ceph/ceph/pull/12563>`_, xie xingguo)
+* common: Make spinlock delay more conventional (`pr#14248 <https://github.com/ceph/ceph/pull/14248>`_, Brad Hubbard)
+* common: mempool: improve dump; fix buffer accounting bugs (`pr#15403 <https://github.com/ceph/ceph/pull/15403>`_, Sage Weil)
+* common: messages: fix return type name of MOSDMap (`pr#14382 <https://github.com/ceph/ceph/pull/14382>`_, Leo Zhang)
+* common: mgr/PyFormatter: implement dump_format_va (`pr#15634 <https://github.com/ceph/ceph/pull/15634>`_, Sage Weil)
+* common: misc cleanups in common, global, os, osd submodules (`pr#16321 <https://github.com/ceph/ceph/pull/16321>`_, Yan Jun)
+* common: misc fixes detected by crypto shutdown assert (`pr#12925 <https://github.com/ceph/ceph/pull/12925>`_, Sage Weil)
+* common,mon: crush,mon: add weight-set introspection and manipulation commands (`pr#16326 <https://github.com/ceph/ceph/pull/16326>`_, Sage Weil)
+* common,mon: messenger,client,compressor: migrate atomic_t to std::atomic (`pr#14657 <https://github.com/ceph/ceph/pull/14657>`_, Jesse Williamson)
+* common: mon/MonClient: scale backoff interval down when we have a healthy mon session (`issue#20371 <http://tracker.ceph.com/issues/20371>`_, `pr#16576 <https://github.com/ceph/ceph/pull/16576>`_, Kefu Chai, Sage Weil)
+* common,mon: mon,crush: add 'osd crush swap-bucket' command (`pr#15072 <https://github.com/ceph/ceph/pull/15072>`_, Sage Weil)
+* common: msg/async: add assert of ms_async_op_threads > 0 (`pr#15629 <https://github.com/ceph/ceph/pull/15629>`_, linbing)
+* common: msg/async: assert if compiled code doesn't support the configured ms (`pr#12559 <https://github.com/ceph/ceph/pull/12559>`_, Avner BenHanoch)
+* common: msg/async: fix crash that writing char to nonblock-fd gets EAGAIN in EventCenter::wakeup (`pr#13822 <https://github.com/ceph/ceph/pull/13822>`_, liuchang0812)
+* common: msg/async: make recv_stamp more precise (`pr#15810 <https://github.com/ceph/ceph/pull/15810>`_, Pan Liu)
+* common: msg/async/rdma: Add fork safe on RDMA (`pr#13740 <https://github.com/ceph/ceph/pull/13740>`_, Sarit Zubakov)
+* common: msg/async/rdma: clean line endings (`pr#12688 <https://github.com/ceph/ceph/pull/12688>`_, Adir Lev)
+* common: msg/async/rdma: Remove compilation warning (`pr#13142 <https://github.com/ceph/ceph/pull/13142>`_, Sarit Zubakov)
+* common: msg/async/rdma: rename chunk_size to buffer_size (`pr#13666 <https://github.com/ceph/ceph/pull/13666>`_, Adir Lev)
+* common: msg/async/rdma: Support for RoCE v2 and SL (`pr#12556 <https://github.com/ceph/ceph/pull/12556>`_, Oren Duer)
+* common: msg/async/rdma: Update fix broken compilation (`pr#13940 <https://github.com/ceph/ceph/pull/13940>`_, Sarit Zubakov)
+* common: msg/async: return right away in NetHandler::set_priority() if not supported (`pr#14795 <https://github.com/ceph/ceph/pull/14795>`_, Kefu Chai)
+* common: msg/simple: call clear_pipe in wait() shutdown path (`issue#15784 <http://tracker.ceph.com/issues/15784>`_, `pr#12633 <https://github.com/ceph/ceph/pull/12633>`_, Sage Weil)
+* common: msg/SimpleMessenger: error out misplace in set_socket_options (`pr#13961 <https://github.com/ceph/ceph/pull/13961>`_, wangzhengyong)
+* common: msg/simple/Pipe: support IPv6 QoS (`issue#18887 <http://tracker.ceph.com/issues/18887>`_, `pr#13370 <https://github.com/ceph/ceph/pull/13370>`_, Robin H. Johnson)
+* common: .organizationmap: Updated authors (`pr#14360 <https://github.com/ceph/ceph/pull/14360>`_, Jos Collin)
+* common: osdc: fix osdc_osd_seesion perf counter (`pr#13478 <https://github.com/ceph/ceph/pull/13478>`_, Xiaoxi Chen)
+* common: osdc/Objecter: fix bugs in explicit naming of op spg_t (`pr#13534 <https://github.com/ceph/ceph/pull/13534>`_, Sage Weil)
+* common: osdc/Objecter: fix pool dne corner case (`issue#19552 <http://tracker.ceph.com/issues/19552>`_, `pr#14901 <https://github.com/ceph/ceph/pull/14901>`_, Sage Weil)
+* common: osdc/Objecter: handle command target that goes down (`issue#19452 <http://tracker.ceph.com/issues/19452>`_, `pr#14302 <https://github.com/ceph/ceph/pull/14302>`_, Sage Weil)
+* common: osdc/Objecter: release message if it's not handled (`issue#19741 <http://tracker.ceph.com/issues/19741>`_, `pr#15890 <https://github.com/ceph/ceph/pull/15890>`_, Kefu Chai)
+* common: osdc/Objecter: resend RWORDERED ops on full (`issue#19133 <http://tracker.ceph.com/issues/19133>`_, `pr#13759 <https://github.com/ceph/ceph/pull/13759>`_, Sage Weil)
+* common: osd/OSDMap: fix feature commit comment (`pr#15056 <https://github.com/ceph/ceph/pull/15056>`_, Sage Weil)
+* common: osd/OSDMap: get_previous_up_osd_before() may run into endless loop (`pr#12976 <https://github.com/ceph/ceph/pull/12976>`_, Mingxin Liu)
+* common: osd/OSDMap: print require_osd_release (`pr#15974 <https://github.com/ceph/ceph/pull/15974>`_, Sage Weil)
+* common: osd/osd_types: clean up OSDOp printers (`pr#12980 <https://github.com/ceph/ceph/pull/12980>`_, Sage Weil)
+* common: Passing null pointer option_name to operator << in md_config_t::parse_option() (`pr#15881 <https://github.com/ceph/ceph/pull/15881>`_, Jos Collin)
+* common,performance: buffer: allow buffers to be accounted in arbitrary mempools (`pr#15352 <https://github.com/ceph/ceph/pull/15352>`_, Sage Weil)
+* common,performance: common/Finisher: batch handle perfcounter && only send signal when waiter existed (`pr#14363 <https://github.com/ceph/ceph/pull/14363>`_, Jianpeng Ma)
+* common,performance: crc32c: Add ppc64le fast zero optimized assembly (`pr#15100 <https://github.com/ceph/ceph/pull/15100>`_, Andrew Solomon)
+* common,performance: inline_memory: optimized mem_is_zero for non-x64 (`pr#15307 <https://github.com/ceph/ceph/pull/15307>`_, Piotr Dałek)
+* common,performance: kv/rocksdb: supports SliceParts interface (`pr#15058 <https://github.com/ceph/ceph/pull/15058>`_, Haomai Wang)
+* common,performance: osd/OSDMap: make pg_temp more efficient (`pr#15291 <https://github.com/ceph/ceph/pull/15291>`_, Sage Weil)
+* common: possible lockdep false alarm for ThreadPool lock (`issue#18819 <http://tracker.ceph.com/issues/18819>`_, `pr#13258 <https://github.com/ceph/ceph/pull/13258>`_, Mykola Golub)
+* common: prevent unset_dumpable from generating warnings (`pr#16462 <https://github.com/ceph/ceph/pull/16462>`_, Willem Jan Withagen)
+* common: rados: allow "rados purge" to delete objects when osd is full (`pr#13814 <https://github.com/ceph/ceph/pull/13814>`_, Pan Liu)
+* common: rados: more info added to pool deletion error (`issue#19400 <http://tracker.ceph.com/issues/19400>`_, `pr#14235 <https://github.com/ceph/ceph/pull/14235>`_, Vedant Nanda)
+* common,rbd: osdc/Objecter: unify disparate EAGAIN handling paths into one (`pr#16627 <https://github.com/ceph/ceph/pull/16627>`_, Sage Weil)
+* common,rbd,rgw: common/escape: do not escape / in json (`pr#14130 <https://github.com/ceph/ceph/pull/14130>`_, Sage Weil)
+* common,rbd,rgw: common/rgw/rbd: remove some unused variables (`pr#16690 <https://github.com/ceph/ceph/pull/16690>`_, Luo Kexue)
+* common,rdma: msg/async/rdma: automatically set RDMAV_HUGEPAGES_SAFE according to conf (`pr#15755 <https://github.com/ceph/ceph/pull/15755>`_, DanielBar-On)
+* common,rdma: msg/async/rdma: check ulimit (`pr#13655 <https://github.com/ceph/ceph/pull/13655>`_, Sarit Zubakov, Adir Lev)
+* common,rdma: msg/async/rdma: Introduce Device.{cc,h} (`pr#14001 <https://github.com/ceph/ceph/pull/14001>`_, Amir Vadai)
+* common,rdma: msg/async/rdma: Introduce RDMAConnMgr + Debug prints (`pr#14201 <https://github.com/ceph/ceph/pull/14201>`_, Amir Vadai)
+* common,rdma: msg/async/rdma: Move resource handling to Device (`pr#14088 <https://github.com/ceph/ceph/pull/14088>`_, Sarit Zubakov, Amir Vadai)
+* common,rdma: msg/async/rdma: RDMA-CM Initialize device on first connect (`pr#14179 <https://github.com/ceph/ceph/pull/14179>`_, Amir Vadai)
+* common,rdma: msg/async/rdma: reduce number of rdma rx/tx buffers (`pr#13190 <https://github.com/ceph/ceph/pull/13190>`_, Adir Lev)
+* common,rdma: msg/async/rdma: use lists properly (`pr#15908 <https://github.com/ceph/ceph/pull/15908>`_, Adir lev, Adir Lev)
+* common: remove config opt conversion utility (`pr#16480 <https://github.com/ceph/ceph/pull/16480>`_, John Spray)
+* common: remove \n on clog messages (`pr#13794 <https://github.com/ceph/ceph/pull/13794>`_, Sage Weil)
+* common: Remove redundant includes - 2 (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15169 <https://github.com/ceph/ceph/pull/15169>`_, Jos Collin)
+* common: Remove redundant includes - 3 (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15204 <https://github.com/ceph/ceph/pull/15204>`_, Jos Collin)
+* common: Remove redundant includes - 4 (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15251 <https://github.com/ceph/ceph/pull/15251>`_, Jos Collin)
+* common: Remove redundant includes - 5 (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15267 <https://github.com/ceph/ceph/pull/15267>`_, Jos Collin)
+* common: Remove redundant includes - 6 (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15299 <https://github.com/ceph/ceph/pull/15299>`_, Jos Collin)
+* common: Remove redundant includes (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15003 <https://github.com/ceph/ceph/pull/15003>`_, Brad Hubbard)
+* common: Remove redundant includes (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15019 <https://github.com/ceph/ceph/pull/15019>`_, Brad Hubbard)
+* common: Remove redundant includes (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15042 <https://github.com/ceph/ceph/pull/15042>`_, Brad Hubbard)
+* common: Remove redundant includes (`issue#19883 <http://tracker.ceph.com/issues/19883>`_, `pr#15086 <https://github.com/ceph/ceph/pull/15086>`_, Jos Collin)
+* common: remove useless parameter (`pr#14096 <https://github.com/ceph/ceph/pull/14096>`_, baiyanchun)
+* common: Revamp config option definitions (`issue#20627 <http://tracker.ceph.com/issues/20627>`_, `pr#16211 <https://github.com/ceph/ceph/pull/16211>`_, John Spray, Kefu Chai, Sage Weil)
+* common,rgw: cls/refcount: store and use list of retired tags (`issue#20107 <http://tracker.ceph.com/issues/20107>`_, `pr#15673 <https://github.com/ceph/ceph/pull/15673>`_, Yehuda Sadeh)
+* common: src/common/ceph_string: stringify new osd states (`pr#15751 <https://github.com/ceph/ceph/pull/15751>`_, xie xingguo)
+* common: src/common: change last_work_queue to next_work_queue (`pr#14738 <https://github.com/ceph/ceph/pull/14738>`_, Pan Liu)
+* common: support s390 and unknown architectures in spin-wait loop (`issue#19492 <http://tracker.ceph.com/issues/19492>`_, `pr#14337 <https://github.com/ceph/ceph/pull/14337>`_, Nathan Cutler)
+* common,tests: ceph_test_rados_api_c_read_operations: do not assert per-op rval is correct (`issue#19518 <http://tracker.ceph.com/issues/19518>`_, `pr#16196 <https://github.com/ceph/ceph/pull/16196>`_, Sage Weil)
+* common,tests: ceph_test_rados_api_list: more fix LibRadosListNP.ListObjectsError (`issue#19963 <http://tracker.ceph.com/issues/19963>`_, `pr#15138 <https://github.com/ceph/ceph/pull/15138>`_, Sage Weil)
+* common,tests: test: Make screencandy optional for FreeBSD (`pr#15444 <https://github.com/ceph/ceph/pull/15444>`_, Willem Jan Withagen)
+* common: the latency dumped by "ceph osd perf" is not real (`issue#20749 <http://tracker.ceph.com/issues/20749>`_, `pr#16512 <https://github.com/ceph/ceph/pull/16512>`_, Pan Liu)
+* common,tools: osdmaptool: show all the pg map to osds info (`pr#9419 <https://github.com/ceph/ceph/pull/9419>`_, song baisen)
+* common: tracing: Fix handle leak in TracepointProvider (`pr#12652 <https://github.com/ceph/ceph/pull/12652>`_, Brad Hubbard)
+* common: tracing: fix segv (`issue#18576 <http://tracker.ceph.com/issues/18576>`_, `pr#14304 <https://github.com/ceph/ceph/pull/14304>`_, Anjaneya Chagam)
+* common: Update the error string when res_nsearch() or res_search() fails (`pr#15878 <https://github.com/ceph/ceph/pull/15878>`_, huanwen ren)
+* common: use ref to avoid unnecessary memory copy (`issue#19107 <http://tracker.ceph.com/issues/19107>`_, `pr#13689 <https://github.com/ceph/ceph/pull/13689>`_, liuchang0812)
+* common: use std::move() for better performance (`pr#16620 <https://github.com/ceph/ceph/pull/16620>`_, Xinying Song)
+* common: xio: migrate atomic_t to std::atomic<> (`pr#15230 <https://github.com/ceph/ceph/pull/15230>`_, Jesse Williamson)
+* compressor: conditionalize on HAVE_LZ4 (`pr#17174 <https://github.com/ceph/ceph/pull/17174>`_, Kefu Chai)
+* compressor: fix Mutex::Locker used is not correct (`pr#13935 <https://github.com/ceph/ceph/pull/13935>`_, hechuang)
+* compressor: zlib: fix plugin for non-Intel arches (`pr#14947 <https://github.com/ceph/ceph/pull/14947>`_, Dan Mick)
+* core: auth: 'ceph auth import -i' overwrites caps, if caps are not specified (`issue#18932 <http://tracker.ceph.com/issues/18932>`_, `pr#13468 <https://github.com/ceph/ceph/pull/13468>`_, Vikhyat Umrao)
+* core: auth: Remove unused function in AuthSessionHandler (`pr#16666 <https://github.com/ceph/ceph/pull/16666>`_, Luo Kexue)
+* core: ceph: allow '-' with -i and -o for stdin/stdout (`pr#16359 <https://github.com/ceph/ceph/pull/16359>`_, Sage Weil)
+* core: ceph-create-keys: Add connection timeouts (`pr#11995 <https://github.com/ceph/ceph/pull/11995>`_, Owen Synge)
+* core: ceph-dencoder: Silence coverity CID 1412579 (`pr#15744 <https://github.com/ceph/ceph/pull/15744>`_, Brad Hubbard)
+* core: ceph-detect-init: Add docker detection (`pr#13218 <https://github.com/ceph/ceph/pull/13218>`_, Guillaume Abrioux)
+* core: ceph-disk: Adding retry loop in get_partition_dev() (`pr#14275 <https://github.com/ceph/ceph/pull/14275>`_, Erwan Velu)
+* core: ceph-disk/ceph_disk/main.py: fix calling of the bsdrc init scripts (`pr#14476 <https://github.com/ceph/ceph/pull/14476>`_, Willem Jan Withagen)
+* core: ceph-disk/ceph_disk/main.py: Replace ST_ISBLK() test by is_diskdevice() (`pr#15587 <https://github.com/ceph/ceph/pull/15587>`_, Willem Jan Withagen)
+* core: ceph-disk: ceph-disk on FreeBSD should not use mpath-code (`pr#14837 <https://github.com/ceph/ceph/pull/14837>`_, Willem Jan Withagen)
+* core: ceph-disk: dmcrypt activate must use the same cluster as prepare (`issue#17821 <http://tracker.ceph.com/issues/17821>`_, `pr#13573 <https://github.com/ceph/ceph/pull/13573>`_, Loic Dachary)
+* core: ceph-disk: dmcrypt cluster must default to ceph (`issue#20893 <http://tracker.ceph.com/issues/20893>`_, `pr#16776 <https://github.com/ceph/ceph/pull/16776>`_, Loic Dachary)
+* core: ceph-disk: do not setup_statedir on trigger (`issue#19941 <http://tracker.ceph.com/issues/19941>`_, `pr#15410 <https://github.com/ceph/ceph/pull/15410>`_, Loic Dachary)
+* core: ceph-disk: enable directory backed OSD at boot time (`issue#19628 <http://tracker.ceph.com/issues/19628>`_, `pr#14546 <https://github.com/ceph/ceph/pull/14546>`_, Loic Dachary)
+* core: ceph-disk: Fix getting wrong group name when --setgroup in bluestore (`issue#18955 <http://tracker.ceph.com/issues/18955>`_, `pr#13457 <https://github.com/ceph/ceph/pull/13457>`_, craigchi)
+* core: ceph-disk: FreeBSD changes to get it working and passing tests (`pr#12086 <https://github.com/ceph/ceph/pull/12086>`_, Willem Jan Withagen)
+* core: ceph-disk: implement prepare --no-locking (`pr#14728 <https://github.com/ceph/ceph/pull/14728>`_, Dan van der Ster, Loic Dachary)
+* core: ceph_disk/main.py: Allow FreeBSD zap a OSD disk (`pr#15642 <https://github.com/ceph/ceph/pull/15642>`_, Willem Jan Withagen)
+* core: ceph-disk,osd: add support for crush device classes (`issue#19513 <http://tracker.ceph.com/issues/19513>`_, `pr#14436 <https://github.com/ceph/ceph/pull/14436>`_, Loic Dachary)
+* core: ceph-disk: Populate mount options when running "list" (`issue#17331 <http://tracker.ceph.com/issues/17331>`_, `pr#14293 <https://github.com/ceph/ceph/pull/14293>`_, Brad Hubbard)
+* core: ceph-disk: Reporting /sys directory in get_partition_dev() (`pr#14080 <https://github.com/ceph/ceph/pull/14080>`_, Erwan Velu)
+* core: ceph-disk: Revert "Revert "change get_dmcrypt_key test to support different cluster name"" (`pr#13600 <https://github.com/ceph/ceph/pull/13600>`_, Loic Dachary)
+* core: ceph-disk: separate ceph-osd --check-needs-\* logs (`issue#19888 <http://tracker.ceph.com/issues/19888>`_, `pr#15016 <https://github.com/ceph/ceph/pull/15016>`_, Loic Dachary)
+* core: ceph-disk: set the default systemd unit timeout to 3h (`issue#20229 <http://tracker.ceph.com/issues/20229>`_, `pr#15585 <https://github.com/ceph/ceph/pull/15585>`_, Loic Dachary)
+* core: ceph-disk: support osd new (`pr#15432 <https://github.com/ceph/ceph/pull/15432>`_, Loic Dachary, Sage Weil)
+* core: ceph-disk: Write 10M to all partitions before zapping (`issue#18962 <http://tracker.ceph.com/issues/18962>`_, `pr#13766 <https://github.com/ceph/ceph/pull/13766>`_, Wido den Hollander)
+* core: ceph: do not throw TypeError on connection failure (`pr#13268 <https://github.com/ceph/ceph/pull/13268>`_, Kefu Chai)
+* core: ceph.in: Fix couple of minor issues on the messages (`pr#12797 <https://github.com/ceph/ceph/pull/12797>`_, Dave Chen)
+* core: ceph-objectstore-tool: do not populate snapmapper with missing clones (`issue#19943 <http://tracker.ceph.com/issues/19943>`_, `pr#15787 <https://github.com/ceph/ceph/pull/15787>`_, Sage Weil)
+* core: ceph-osd: fix auto detect which objectstore is currently running (`issue#20865 <http://tracker.ceph.com/issues/20865>`_, `pr#16717 <https://github.com/ceph/ceph/pull/16717>`_, Yanhu Cao)
+* core: ceph-osd: --flush-journal: sporadic segfaults on exit (`issue#18820 <http://tracker.ceph.com/issues/18820>`_, `pr#13311 <https://github.com/ceph/ceph/pull/13311>`_, Alexey Sheplyakov)
+* core: client/SyntheticClient.cc: Fix warning in random_walk (`issue#19445 <http://tracker.ceph.com/issues/19445>`_, `pr#14308 <https://github.com/ceph/ceph/pull/14308>`_, Brad Hubbard)
+* core: cls/timeindex: clean up cls_timeindex_client.h|cc (`pr#13987 <https://github.com/ceph/ceph/pull/13987>`_, Shinobu Kinjo)
+* core: common/options: remove mon_warn_osd_usage_min_max_delta from options.cc too (`pr#16488 <https://github.com/ceph/ceph/pull/16488>`_, Sage Weil)
+* core: common/TrackedOp: allow dumping historic ops sorted by duration (`pr#14050 <https://github.com/ceph/ceph/pull/14050>`_, Piotr Dałek)
+* core: compressor: add LZ4 support (`pr#15434 <https://github.com/ceph/ceph/pull/15434>`_, Haomai Wang)
+* core: compressor: optimize header file dependency (`pr#15187 <https://github.com/ceph/ceph/pull/15187>`_, Brad Hubbard, Xiaowei Chen)
+* core: Context: C_ContextsBase: delete enclosed contexts in dtor (`issue#20432 <http://tracker.ceph.com/issues/20432>`_, `pr#16159 <https://github.com/ceph/ceph/pull/16159>`_, Kefu Chai)
+* core: crush/CrushWrapper: chooseargs encoding fix (`pr#15984 <https://github.com/ceph/ceph/pull/15984>`_, Ilya Dryomov)
+* core: crush/CrushWrapper: make get_immediate_parent[_id] ignore per-class shadow hierarchy (`issue#20546 <http://tracker.ceph.com/issues/20546>`_, `pr#16221 <https://github.com/ceph/ceph/pull/16221>`_, Sage Weil)
+* core: crush, mon: make jewel the lower bound for client/crush compat for new clusters (`pr#15370 <https://github.com/ceph/ceph/pull/15370>`_, Sage Weil)
+* core: erasure-code: optimize header file dependency (`pr#15172 <https://github.com/ceph/ceph/pull/15172>`_, Brad Hubbard, Xiaowei Chen)
+* core: erasure-code: Remove duplicate of isa-l files (`pr#15372 <https://github.com/ceph/ceph/pull/15372>`_, Ganesh Mahalingam)
+* core: erasure-code: sync jerasure/gf-complete submodules (`pr#14424 <https://github.com/ceph/ceph/pull/14424>`_, Loic Dachary)
+* core: filestore: migrate atomic_t to std::atomic<> (`pr#15228 <https://github.com/ceph/ceph/pull/15228>`_, Jesse Williamson)
+* core: Give requested scrub work a higher priority (`issue#15789 <http://tracker.ceph.com/issues/15789>`_, `pr#14488 <https://github.com/ceph/ceph/pull/14488>`_, David Zafman)
+* core: global: start removing g_ceph_context (`pr#12149 <https://github.com/ceph/ceph/pull/12149>`_, Adam C. Emerson)
+* core: HashIndex.cc: add compat.h for ENODATA (`pr#16697 <https://github.com/ceph/ceph/pull/16697>`_, Willem Jan Withagen)
+* core: include/denc: add {encode,decode}_nohead for denc_traits<basic_string> (`issue#18938 <http://tracker.ceph.com/issues/18938>`_, `pr#14099 <https://github.com/ceph/ceph/pull/14099>`_, Kefu Chai)
+* core: include/mempool.h: fix Clangs complaint about types (`pr#13523 <https://github.com/ceph/ceph/pull/13523>`_, Willem Jan Withagen)
+* core: include/types.h, introduce host_to_ceph_errno (`pr#15496 <https://github.com/ceph/ceph/pull/15496>`_, Willem Jan Withagen)
+* core: Install Pecan for FreeBSD (`pr#15610 <https://github.com/ceph/ceph/pull/15610>`_, Willem Jan Withagen)
+* core: introduce (and fix) code to pass errno to other OSes (`pr#15495 <https://github.com/ceph/ceph/pull/15495>`_, Willem Jan Withagen)
+* core: introduce DirectMessenger (`pr#14755 <https://github.com/ceph/ceph/pull/14755>`_, Casey Bodley, Matt Benjamin)
+* core: kv/RocksDBStore: abort if rocksdb EIO, don't return incorrect result (`pr#15862 <https://github.com/ceph/ceph/pull/15862>`_, Haomai Wang)
+* core: kv/RocksDBStore: use vector instead of VLA for holding slices (`pr#16615 <https://github.com/ceph/ceph/pull/16615>`_, Kefu Chai)
+* core: libradosstriper: Initialize member variable m_writeRc in WriteCompletionData (`pr#16780 <https://github.com/ceph/ceph/pull/16780>`_, amitkuma)
+* core: luminous: Improve size scrub error handling and ignore system attrs in xattr checking (`issue#21051 <http://tracker.ceph.com/issues/21051>`_, `issue#18836 <http://tracker.ceph.com/issues/18836>`_, `issue#20243 <http://tracker.ceph.com/issues/20243>`_, `pr#17196 <https://github.com/ceph/ceph/pull/17196>`_, David Zafman)
+* core: luminous: Include front/back interface names in OSD metadata (`issue#21048 <http://tracker.ceph.com/issues/21048>`_, `issue#20956 <http://tracker.ceph.com/issues/20956>`_, `pr#17193 <https://github.com/ceph/ceph/pull/17193>`_, John Spray)
+* core: luminous: mon: bug in functon reweight_by_utilization (`issue#21079 <http://tracker.ceph.com/issues/21079>`_, `issue#20970 <http://tracker.ceph.com/issues/20970>`_, `pr#17198 <https://github.com/ceph/ceph/pull/17198>`_, xie xingguo)
+* core: luminous: mon: "ceph osd crush rule rename" support (`pr#17260 <https://github.com/ceph/ceph/pull/17260>`_, xie xingguo)
+* core: luminous: multisite: FAILED assert(prev_iter != pos_to_prev.end()) in RGWMetaSyncShardCR::collect_children() (`issue#21097 <http://tracker.ceph.com/issues/21097>`_, `issue#20906 <http://tracker.ceph.com/issues/20906>`_, `pr#17234 <https://github.com/ceph/ceph/pull/17234>`_, Casey Bodley)
+* core: luminous: osd: osd_scrub_during_recovery only considers primary, not replicas (`issue#18206 <http://tracker.ceph.com/issues/18206>`_, `issue#21077 <http://tracker.ceph.com/issues/21077>`_, `pr#17195 <https://github.com/ceph/ceph/pull/17195>`_, David Zafman)
+* core: luminous: src/common/LogClient.cc: 310: FAILED assert(num_unsent <= log_queue.size()) (`issue#20965 <http://tracker.ceph.com/issues/20965>`_, `issue#18209 <http://tracker.ceph.com/issues/18209>`_, `pr#17197 <https://github.com/ceph/ceph/pull/17197>`_, Sage Weil)
+* core: make the conversion from wire error to host OS work (`pr#15780 <https://github.com/ceph/ceph/pull/15780>`_, Willem Jan Withagen)
+* core: Merge pull request #16755 from ivancich/wip-pull-new-dmclock (`pr#16922 <https://github.com/ceph/ceph/pull/16922>`_, Gregory Farnum)
+* core: messages: default-initialize MOSDPGRecoveryDelete[Reply] members (`pr#16584 <https://github.com/ceph/ceph/pull/16584>`_, Greg Farnum)
+* core: messages: Initialize members in MMDSTableRequest (`pr#16810 <https://github.com/ceph/ceph/pull/16810>`_, amitkuma)
+* core: messages: Initialize member variables (`pr#16819 <https://github.com/ceph/ceph/pull/16819>`_, amitkuma)
+* core: messages: Initialize member variables (`pr#16839 <https://github.com/ceph/ceph/pull/16839>`_, amitkuma)
+* core: messages: Initializing member variable in MMDSCacheRejoin (`pr#16791 <https://github.com/ceph/ceph/pull/16791>`_, amitkuma)
+* core: messages/MOSDOp: fix pg_t decoding for version <7 decoding (`issue#19005 <http://tracker.ceph.com/issues/19005>`_, `pr#13537 <https://github.com/ceph/ceph/pull/13537>`_, Sage Weil)
+* core: messages/MOSDPGTrim: add the missed HEAD_VERSION AND COMPAT_VERSION (`issue#18266 <http://tracker.ceph.com/issues/18266>`_, `pr#12517 <https://github.com/ceph/ceph/pull/12517>`_, huangjun)
+* core: messages/MOSDPing.h: drop unused fields (`pr#15843 <https://github.com/ceph/ceph/pull/15843>`_, Piotr Dałek)
+* core: messages/MOSDPing: initialize MOSDPing padding (`issue#20323 <http://tracker.ceph.com/issues/20323>`_, `pr#15714 <https://github.com/ceph/ceph/pull/15714>`_, Sage Weil)
+* core: messages/MOSDSubOp: Make encode_payload can be reentrant (`pr#12654 <https://github.com/ceph/ceph/pull/12654>`_, Haomai Wang)
+* core: messages: remove compat cruft (`pr#14475 <https://github.com/ceph/ceph/pull/14475>`_, Sage Weil)
+* core: mgr/MgrClient: do not attempt to access a global variable for config (`pr#16544 <https://github.com/ceph/ceph/pull/16544>`_, Jason Dillaman)
+* core: mgr/MgrClient: use unique_ptr for MgrClient::session (`issue#19097 <http://tracker.ceph.com/issues/19097>`_, `pr#13685 <https://github.com/ceph/ceph/pull/13685>`_, Kefu Chai)
+* core,mgr: mgr/DaemonServer: stop spamming log with pg stats (`pr#15487 <https://github.com/ceph/ceph/pull/15487>`_, Sage Weil)
+* core,mgr: mgr,librados: service map (`pr#15858 <https://github.com/ceph/ceph/pull/15858>`_, Yehuda Sadeh, John Spray, Sage Weil)
+* core,mgr,mon: mgr,mon: enable/disable mgr modules via 'ceph mgr module ...' commands (`pr#15958 <https://github.com/ceph/ceph/pull/15958>`_, Sage Weil)
+* core,mgr,mon: mon,mgr: tag some commands for ceph-mgr (`pr#13617 <https://github.com/ceph/ceph/pull/13617>`_, Sage Weil)
+* core,mgr,mon: mon/PGMap: fix osd_epoch update when removing osd_stat (`issue#20208 <http://tracker.ceph.com/issues/20208>`_, `pr#15573 <https://github.com/ceph/ceph/pull/15573>`_, Sage Weil)
+* core,mgr: mon/PGMap: slightly better debugging around pgmap updates (`pr#15820 <https://github.com/ceph/ceph/pull/15820>`_, Sage Weil)
+* core,mgr,tests: qa: flush out monc's dropped msgs on msgr failure injection (`issue#20371 <http://tracker.ceph.com/issues/20371>`_, `pr#16484 <https://github.com/ceph/ceph/pull/16484>`_, Joao Eduardo Luis)
+* core,mgr,tests: qa/suites/rados/rest: test restful mgr module (`pr#15604 <https://github.com/ceph/ceph/pull/15604>`_, Sage Weil)
+* core: misc: SCA fixes (`pr#14426 <https://github.com/ceph/ceph/pull/14426>`_, Danny Al-Gaaf)
+* core,mon: crush, mon: simplify device class manipulation commands (`pr#16388 <https://github.com/ceph/ceph/pull/16388>`_, xie xingguo)
+* core: mon,mgr: fix "ceph osd df", add some tools to find untested commands (`issue#20256 <http://tracker.ceph.com/issues/20256>`_, `pr#15675 <https://github.com/ceph/ceph/pull/15675>`_, Greg Farnum)
+* core: mon/MonClient: discard stray messages from non-acitve conns (`issue#19015 <http://tracker.ceph.com/issues/19015>`_, `pr#13656 <https://github.com/ceph/ceph/pull/13656>`_, Kefu Chai)
+* core: mon/MonClient: don't return zero global_id (`issue#19134 <http://tracker.ceph.com/issues/19134>`_, `pr#13853 <https://github.com/ceph/ceph/pull/13853>`_, "Yan, Zheng", Kefu Chai)
+* core: mon/MonClient: hunt monitors in parallel (`issue#16091 <http://tracker.ceph.com/issues/16091>`_, `pr#11128 <https://github.com/ceph/ceph/pull/11128>`_, Steven Dieffenbach, Kefu Chai)
+* core: mon/MonClient: persist global_id across re-connecting (`issue#18968 <http://tracker.ceph.com/issues/18968>`_, `pr#13550 <https://github.com/ceph/ceph/pull/13550>`_, Kefu Chai)
+* core: mon/MonClient: respect the priority in SRV RR (`issue#5249 <http://tracker.ceph.com/issues/5249>`_, `pr#15964 <https://github.com/ceph/ceph/pull/15964>`_, Kefu Chai)
+* core,mon: mon/LogMonitor: 'log last' command (`pr#15497 <https://github.com/ceph/ceph/pull/15497>`_, Sage Weil)
+* core: mon/MonmapMonitor: use `__func__` instead of hard code function name (`pr#16037 <https://github.com/ceph/ceph/pull/16037>`_, Yanhu Cao)
+* core,mon: mon/MgrStatMonitor: avoid dup health warnings during luminous upgrade (`issue#20435 <http://tracker.ceph.com/issues/20435>`_, `pr#15986 <https://github.com/ceph/ceph/pull/15986>`_, Sage Weil)
+* core,mon: mon/MgrStatMonitor: keep mgrstat version ahead of pgmon (`issue#20219 <http://tracker.ceph.com/issues/20219>`_, `pr#15584 <https://github.com/ceph/ceph/pull/15584>`_, Sage Weil)
+* core,mon: mon,osd: add crush_version to OSDMap, and allow crush map updates to gate on crush_version (`pr#15533 <https://github.com/ceph/ceph/pull/15533>`_, Sage Weil)
+* core,mon: mon,osd: decouple creating pgs from pgmap (`pr#13999 <https://github.com/ceph/ceph/pull/13999>`_, Kefu Chai)
+* core,mon: mon, osd: misc fixes (`pr#16078 <https://github.com/ceph/ceph/pull/16078>`_, xie xingguo)
+* core,mon: mon/OSDMonitor: cancel mapping job from update_from_paxos (`issue#20067 <http://tracker.ceph.com/issues/20067>`_, `pr#15320 <https://github.com/ceph/ceph/pull/15320>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: make 'osd crush move ...' work on osds (`issue#18587 <http://tracker.ceph.com/issues/18587>`_, `pr#12981 <https://github.com/ceph/ceph/pull/12981>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: make snaps on tier pool should not be allowed (`pr#9348 <https://github.com/ceph/ceph/pull/9348>`_, Mingxin Liu)
+* core,mon: mon/OSDMonitor: use up set instead of acting set in reweight_by_utilization (`pr#13802 <https://github.com/ceph/ceph/pull/13802>`_, Mingxin Liu)
+* core,mon: mon,osd: new mechanism for managing full and nearfull OSDs for luminous (`pr#13615 <https://github.com/ceph/ceph/pull/13615>`_, Sage Weil)
+* core,mon: mon/PGMap: call blocked requests ERR not WARN (`pr#15501 <https://github.com/ceph/ceph/pull/15501>`_, Sage Weil)
+* core: mon,osd: add require_min_compat_client setting to enforce and clarify client compatibility (`pr#14959 <https://github.com/ceph/ceph/pull/14959>`_, Sage Weil)
+* core: mon,osd: luminous feature bits, require flags, upgrade gates (`pr#13278 <https://github.com/ceph/ceph/pull/13278>`_, Sage Weil)
+* core: mon, osd: misc fixes and cleanups (`pr#16160 <https://github.com/ceph/ceph/pull/16160>`_, xie xingguo)
+* core: mon, osd: misc fixes (`pr#16283 <https://github.com/ceph/ceph/pull/16283>`_, xie xingguo)
+* core: mon/OSDMonitor: _apply_remap -> _apply_upmap; less code redundancy (`pr#15846 <https://github.com/ceph/ceph/pull/15846>`_, xie xingguo)
+* core: mon/OSDMonitor: batch noup/noin osds support (`pr#15725 <https://github.com/ceph/ceph/pull/15725>`_, xie xingguo)
+* core: mon/OSDMonitor: batch OSDs nodown/noout support (`pr#15381 <https://github.com/ceph/ceph/pull/15381>`_, xie xingguo)
+* core: mon/OSDMonitor: change info in 'osd failed' messages (`pr#15321 <https://github.com/ceph/ceph/pull/15321>`_, Sage Weil)
+* core: mon/OSDMonitor: do not allow crush device classes until luminous (`pr#16188 <https://github.com/ceph/ceph/pull/16188>`_, Sage Weil)
+* core: mon/OSDMonitor: fixup sortbitwise flag warning (`pr#12682 <https://github.com/ceph/ceph/pull/12682>`_, huanwen ren)
+* core: mon/OSDMonitor: make mapping job behave if mon_osd_prime_pg_temp = false (`issue#19020 <http://tracker.ceph.com/issues/19020>`_, `pr#13574 <https://github.com/ceph/ceph/pull/13574>`_, Sage Weil)
+* core: mon/OSDMonitor: osd crush set-device-class (`issue#19307 <http://tracker.ceph.com/issues/19307>`_, `pr#14039 <https://github.com/ceph/ceph/pull/14039>`_, Loic Dachary)
+* core: mon/OSDMonitor: set last_force_op_resend on overlay pool too (`issue#18366 <http://tracker.ceph.com/issues/18366>`_, `pr#12712 <https://github.com/ceph/ceph/pull/12712>`_, Sage Weil)
+* core: mon/OSDMonitor: should propose osdmap update when cluster addr changed (`pr#11065 <https://github.com/ceph/ceph/pull/11065>`_, Mingxin Liu)
+* core: mon/OSDMonitor: skip prime_pg_temp if mapping is prior to osdmap (`pr#14826 <https://github.com/ceph/ceph/pull/14826>`_, Kefu Chai)
+* core: mon,osd/OSDMap: a couple pg-upmap fixes (`pr#15319 <https://github.com/ceph/ceph/pull/15319>`_, Sage Weil)
+* core: mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc (`issue#18522 <http://tracker.ceph.com/issues/18522>`_, `pr#12923 <https://github.com/ceph/ceph/pull/12923>`_, Sage Weil)
+* core: mon/PGMonitor: fix wrongly report "pg stuck in inactive" (`pr#14391 <https://github.com/ceph/ceph/pull/14391>`_, Mingxin Liu)
+* core,mon,rbd: mon,osd: new rbd-based cephx cap profiles (`pr#15991 <https://github.com/ceph/ceph/pull/15991>`_, Jason Dillaman)
+* core: msg/async/AsyncConnection: keepalive objecter ping connection to avoid timeout (`pr#14009 <https://github.com/ceph/ceph/pull/14009>`_, Haomai Wang)
+* core: msg/async/AsyncConnection: socket's fd can be zero, avoid false assert (`pr#13080 <https://github.com/ceph/ceph/pull/13080>`_, Haomai Wang)
+* core: msg/async: avoid requeue racing with handle_write (`issue#20093 <http://tracker.ceph.com/issues/20093>`_, `pr#15324 <https://github.com/ceph/ceph/pull/15324>`_, Haomai Wang)
+* core: msg/async/dpdk: fix compile errors (`pr#12698 <https://github.com/ceph/ceph/pull/12698>`_, Haomai Wang)
+* core: msg/async: fix deleted_conn is out of sync with conns (`issue#20230 <http://tracker.ceph.com/issues/20230>`_, `pr#15645 <https://github.com/ceph/ceph/pull/15645>`_, Haomai Wang)
+* core: msg/async: fix the bug of inaccurate calculation of l_msgr_send_bytes (`pr#16526 <https://github.com/ceph/ceph/pull/16526>`_, Jin Cai)
+* core: msg/async/rdma: add log to show correct destruct queuepair (`pr#13412 <https://github.com/ceph/ceph/pull/13412>`_, Haomai Wang)
+* core: msg/async/rdma: add perf counters to RDMA backend (`pr#13484 <https://github.com/ceph/ceph/pull/13484>`_, Haomai Wang)
+* core: msg/async/rdma: destroy QueuePair if needed (`pr#13810 <https://github.com/ceph/ceph/pull/13810>`_, Haomai Wang)
+* core: msg/async/rdma: don't need to delete event when tcp connection isn't … (`pr#13528 <https://github.com/ceph/ceph/pull/13528>`_, Haomai Wang)
+* core: msg/async/rdma: fix ceph_clock_now calls (`pr#12711 <https://github.com/ceph/ceph/pull/12711>`_, Haomai Wang)
+* core: msg/async/rdma: fix potential racing connection usage (`pr#13738 <https://github.com/ceph/ceph/pull/13738>`_, Haomai Wang)
+* core: msg/async/rdma: make Infiniband can be forkable (`pr#13525 <https://github.com/ceph/ceph/pull/13525>`_, Haomai Wang)
+* core: msg/async/rdm: fix leak when existing failure in ip network (`pr#13435 <https://github.com/ceph/ceph/pull/13435>`_, Haomai Wang)
+* core: msg/async: set thread name for msgr worker (`pr#13699 <https://github.com/ceph/ceph/pull/13699>`_, Haomai Wang)
+* core: msg/async/Stack.cc: use of pthread_setname_np() needs compat.h (`pr#13825 <https://github.com/ceph/ceph/pull/13825>`_, Willem Jan Withagen)
+* core: msg/async: support IPv6 QoS (`issue#18887 <http://tracker.ceph.com/issues/18887>`_, `issue#18928 <http://tracker.ceph.com/issues/18928>`_, `pr#13418 <https://github.com/ceph/ceph/pull/13418>`_, Robin H. Johnson)
+* core: msg/simple: fix missing unlock when already bind (`pr#13267 <https://github.com/ceph/ceph/pull/13267>`_, Haomai Wang)
+* core: msg/simple/Pipe:the returned value for do_recv unequal to zero (`pr#10272 <https://github.com/ceph/ceph/pull/10272>`_, zhang.zezhu)
+* core: objclass: modify omap_get_{keys,vals} api (`pr#16667 <https://github.com/ceph/ceph/pull/16667>`_, Yehuda Sadeh, Casey Bodley)
+* core: objclass-sdk: use namespace ceph for bufferlist (`pr#15581 <https://github.com/ceph/ceph/pull/15581>`_, Neha Ojha)
+* core: os/bluestore: do not use nullptr to calc the size of bluestore_pextent_t (`pr#14030 <https://github.com/ceph/ceph/pull/14030>`_, Kefu Chai)
+* core: os/bluestore rm unused variable in aio_read() (`pr#13530 <https://github.com/ceph/ceph/pull/13530>`_, tangwenjun)
+* core: os/bluestore: silence gcc warning (`pr#14028 <https://github.com/ceph/ceph/pull/14028>`_, Kefu Chai)
+* core: osdc: clean up osd_command/start_mon_command interfaces (`pr#13727 <https://github.com/ceph/ceph/pull/13727>`_, John Spray)
+* core: osdc/Objecter: fix possible OSDSession leak on wrong connection (`pr#13365 <https://github.com/ceph/ceph/pull/13365>`_, xie xingguo)
+* core: osdc/Objecter: resend pg commands on interval change (`issue#18358 <http://tracker.ceph.com/issues/18358>`_, `pr#12869 <https://github.com/ceph/ceph/pull/12869>`_, Samuel Just)
+* core: osdc/Objecter: respect epoch barrier in _op_submit() (`issue#19396 <http://tracker.ceph.com/issues/19396>`_, `pr#14190 <https://github.com/ceph/ceph/pull/14190>`_, Ilya Dryomov)
+* core: osd/: don't leak context for Blessed\*Context or RecoveryQueueAsync (`issue#18809 <http://tracker.ceph.com/issues/18809>`_, `pr#13342 <https://github.com/ceph/ceph/pull/13342>`_, Samuel Just)
+* core: OSD: drop parameter t from merge_log() (`pr#13923 <https://github.com/ceph/ceph/pull/13923>`_, xie xingguo)
+* core: osd/ECBackend: cleanup for unnecessary copy with pg_stat_t (`pr#12564 <https://github.com/ceph/ceph/pull/12564>`_, Yunchuan Wen)
+* core: osd/ECBackend: drop duplicated pending_commit field from << operator (`pr#13665 <https://github.com/ceph/ceph/pull/13665>`_, xie xingguo)
+* core: osd/ECBackend: only need check missing_loc when doing recovery (`pr#12526 <https://github.com/ceph/ceph/pull/12526>`_, huangjun)
+* core:   osd/ECBackend: remove unused variable "ReadCB" (`pr#12543 <https://github.com/ceph/ceph/pull/12543>`_, huangjun)
+* core: osd/ECTransaction: cleanup the redundant check which works in overwrite IO context (`pr#15765 <https://github.com/ceph/ceph/pull/15765>`_, tang.jin)
+* core: osd/ECTransaction: only read partial stripes when below \*original\* object size (`issue#19882 <http://tracker.ceph.com/issues/19882>`_, `pr#15712 <https://github.com/ceph/ceph/pull/15712>`_, Sage Weil)
+* core: osd/filestore: Revert "os/filestore: move ondisk in front (`issue#20524 <http://tracker.ceph.com/issues/20524>`_, `pr#16156 <https://github.com/ceph/ceph/pull/16156>`_, Kefu Chai)
+* core: osd,librados: add manifest, redirect (`pr#15325 <https://github.com/ceph/ceph/pull/15325>`_, Sage Weil)
+* core: osd,librados: cmpext support (`pr#14715 <https://github.com/ceph/ceph/pull/14715>`_, Zhengyong Wang, David Disseldorp, Mike Christie)
+* core: osd,librados: remove clone_range and associated multi-object cruft (`pr#13008 <https://github.com/ceph/ceph/pull/13008>`_, Samuel Just)
+* core: osd, messages/MOSDPing: bunch of fixes related to ping inflation (`pr#15727 <https://github.com/ceph/ceph/pull/15727>`_, Piotr Dałek)
+* core: osd/mon/mds: fix `config set` tell command (`issue#20803 <http://tracker.ceph.com/issues/20803>`_, `pr#16700 <https://github.com/ceph/ceph/pull/16700>`_, John Spray)
+* core: osd,mon: misc full fixes and cleanups (`pr#13968 <https://github.com/ceph/ceph/pull/13968>`_, David Zafman)
+* core: osd/OpRequest: dump both name and addr for the client op (`pr#12691 <https://github.com/ceph/ceph/pull/12691>`_, runsisi)
+* core: osd/OSD: bump up current version; conditionally encoding manifest into oi (`pr#15687 <https://github.com/ceph/ceph/pull/15687>`_, xie xingguo)
+* core: osd/osd_internal_types: wake snaptrimmer on put_read lock, too (`issue#19131 <http://tracker.ceph.com/issues/19131>`_, `pr#13755 <https://github.com/ceph/ceph/pull/13755>`_, Sage Weil)
+* core: osd/OSDMap: bump encoding version for require_min_compat_client (`pr#15046 <https://github.com/ceph/ceph/pull/15046>`_, "Yan, Zheng")
+* core: osd/OSDMap: Change \*pg_to_\* to return void (`pr#15684 <https://github.com/ceph/ceph/pull/15684>`_, Brad Hubbard)
+* core: osd/OSDMap: don't set weight to IN when OSD is destroyed (`issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#13730 <https://github.com/ceph/ceph/pull/13730>`_, Ilya Dryomov)
+* core: osd/OSDMap: hide require_osd and sortbitwise flags (`pr#14440 <https://github.com/ceph/ceph/pull/14440>`_, Sage Weil)
+* core: osd/OSDMap: improve upmap calculation (`issue#19818 <http://tracker.ceph.com/issues/19818>`_, `pr#14902 <https://github.com/ceph/ceph/pull/14902>`_, Sage Weil)
+* core: osd/OSDMap: Uncomment code to enable private default constructors (`pr#12597 <https://github.com/ceph/ceph/pull/12597>`_, Brad Hubbard)
+* core: osd/OSD: tolerate any 'set-device-class' error on OSD startup (`pr#16812 <https://github.com/ceph/ceph/pull/16812>`_, xie xingguo)
+* core: osd/osd_type: Fix logging output (`pr#12778 <https://github.com/ceph/ceph/pull/12778>`_, Brad Hubbard)
+* core: osd/osd_types: Move comment to more relevant position (`pr#12779 <https://github.com/ceph/ceph/pull/12779>`_, Brad Hubbard)
+* core: osd/osd_types: print notify-ack op properly (`pr#12585 <https://github.com/ceph/ceph/pull/12585>`_, Sage Weil)
+* core: osd/PG: add new have_unfound() function in MissingLoc (`pr#12668 <https://github.com/ceph/ceph/pull/12668>`_, huangjun)
+* core: osd/PG: Add two new mClock implementations of the PG sharded operator queue (`pr#14997 <https://github.com/ceph/ceph/pull/14997>`_, J. Eric Ivancich)
+* core: osd/PG.cc: Optimistic estimation on PG.last_active (`pr#14799 <https://github.com/ceph/ceph/pull/14799>`_, Xiaoxi Chen)
+* core: osd/PG.cc: unify the call of checking whether lock is held (`pr#15013 <https://github.com/ceph/ceph/pull/15013>`_, Jin Cai)
+* core: osd/PG: check the connection first in fulfill_log (`pr#12579 <https://github.com/ceph/ceph/pull/12579>`_, huangjun)
+* core: osd/PG: conditionally retry on receiving pg-notify when Primary is Incomplete (`pr#13942 <https://github.com/ceph/ceph/pull/13942>`_, xie xingguo)
+* core: osd/PG: drop pre-firefly compat_mode for choose_acting (`pr#15057 <https://github.com/ceph/ceph/pull/15057>`_, Sage Weil)
+* core: osd/PG: fix lost unfound + delete when there are no missing objects (`issue#20904 <http://tracker.ceph.com/issues/20904>`_, `pr#16809 <https://github.com/ceph/ceph/pull/16809>`_, Josh Durgin)
+* core: osd/PG: fix possible overflow on unfound objects (`pr#12669 <https://github.com/ceph/ceph/pull/12669>`_, huangjun)
+* core: osd/PG: fix warning so we discard_event() on a no-op state change (`pr#16655 <https://github.com/ceph/ceph/pull/16655>`_, Sage Weil)
+* core: osd/PG: ignore CancelRecovery in NotRecovering (`issue#20804 <http://tracker.ceph.com/issues/20804>`_, `pr#16638 <https://github.com/ceph/ceph/pull/16638>`_, Sage Weil)
+* core: osd/PGLog: avoid infinite loop if missing version is corrupted (`pr#16798 <https://github.com/ceph/ceph/pull/16798>`_, Josh Durgin)
+* core: osd/PGLog: fix inaccurate missing assert (`issue#20753 <http://tracker.ceph.com/issues/20753>`_, `pr#16539 <https://github.com/ceph/ceph/pull/16539>`_, Josh Durgin)
+* core: osd/PGLog: fix index for parent and child log on split (`issue#18975 <http://tracker.ceph.com/issues/18975>`_, `pr#13493 <https://github.com/ceph/ceph/pull/13493>`_, Sage Weil)
+* core: osd/pglog: remove loop through empty collection (`pr#15121 <https://github.com/ceph/ceph/pull/15121>`_, J. Eric Ivancich)
+* core: osd/PGLog: skip ERROR entires in _merge_object_divergent_entries (`issue#20843 <http://tracker.ceph.com/issues/20843>`_, `pr#16675 <https://github.com/ceph/ceph/pull/16675>`_, Jeegn Chen)
+* core: osd/PG: make non-empty PastIntervals non-fatal (`issue#20167 <http://tracker.ceph.com/issues/20167>`_, `pr#15639 <https://github.com/ceph/ceph/pull/15639>`_, Sage Weil)
+* core: osd/PG: only correct filestore collection bits on load (`issue#19541 <http://tracker.ceph.com/issues/19541>`_, `pr#14397 <https://github.com/ceph/ceph/pull/14397>`_, Sage Weil)
+* core: osd/PG: publish PG stats when backfill-related states change (`issue#18369 <http://tracker.ceph.com/issues/18369>`_, `pr#12727 <https://github.com/ceph/ceph/pull/12727>`_, Sage Weil)
+* core: osd/PG: reset the missing set when restarting backfill (`issue#19191 <http://tracker.ceph.com/issues/19191>`_, `pr#14053 <https://github.com/ceph/ceph/pull/14053>`_, Josh Durgin)
+* core: osd/PG: restrict want_acting to up+acting on recovery completion (`issue#18929 <http://tracker.ceph.com/issues/18929>`_, `pr#13420 <https://github.com/ceph/ceph/pull/13420>`_, Sage Weil)
+* core: osd/PG: set clean when last_epoch_clean is updated (`issue#19023 <http://tracker.ceph.com/issues/19023>`_, `pr#15555 <https://github.com/ceph/ceph/pull/15555>`_, Samuel Just)
+* core: osd/PG: simplify the logic of backfill_targets checking (`pr#12519 <https://github.com/ceph/ceph/pull/12519>`_, huangjun)
+* core: osd/PG: some minor cleanups (`pr#14133 <https://github.com/ceph/ceph/pull/14133>`_, runsisi)
+* core: osd/PrimaryLogPG: clear oi from trim_object() (`issue#19947 <http://tracker.ceph.com/issues/19947>`_, `pr#15519 <https://github.com/ceph/ceph/pull/15519>`_, Sage Weil)
+* core: osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting) (`issue#19902 <http://tracker.ceph.com/issues/19902>`_, `pr#15040 <https://github.com/ceph/ceph/pull/15040>`_, Kefu Chai)
+* core: osd/PrimaryLogPG: do not expect FULL_TRY ops to get resent (`issue#19430 <http://tracker.ceph.com/issues/19430>`_, `pr#14255 <https://github.com/ceph/ceph/pull/14255>`_, Sage Weil)
+* core: osd/PrimaryLogPG::failed_push: update missing as well (`issue#18165 <http://tracker.ceph.com/issues/18165>`_, `pr#12888 <https://github.com/ceph/ceph/pull/12888>`_, Samuel Just)
+* core: osd/PrimaryLogPG: fix oi reset during trim_object (`issue#19947 <http://tracker.ceph.com/issues/19947>`_, `pr#15696 <https://github.com/ceph/ceph/pull/15696>`_, Sage Weil)
+* core:   osd/PrimaryLogPG: fix recovering hang when have unfound objects (`pr#16558 <https://github.com/ceph/ceph/pull/16558>`_, huangjun)
+* core: osd/PrimaryLogPG: optimal pick_newest_available (`pr#12695 <https://github.com/ceph/ceph/pull/12695>`_, huangjun)
+* core: osd/PrimaryLogPG: record prior_version for DELETE events (`issue#20274 <http://tracker.ceph.com/issues/20274>`_, `pr#15649 <https://github.com/ceph/ceph/pull/15649>`_, Sage Weil)
+* core: osd/PrimaryLogPG: remove duplicated code (`pr#13894 <https://github.com/ceph/ceph/pull/13894>`_, Jianpeng Ma)
+* core: osd/PrimaryLogPG: set return value if sparse read failed (`pr#14093 <https://github.com/ceph/ceph/pull/14093>`_, huangjun)
+* core: osd/PrimaryLogPG: skip deleted missing objects in pg[n]ls (`issue#20739 <http://tracker.ceph.com/issues/20739>`_, `pr#16490 <https://github.com/ceph/ceph/pull/16490>`_, Josh Durgin)
+* core: osd/PrimaryLogPG solve cache tier osd high memory consumption (`issue#20464 <http://tracker.ceph.com/issues/20464>`_, `pr#16011 <https://github.com/ceph/ceph/pull/16011>`_, Peng Xie)
+* core: osd/PrimaryLogPG::try_lock_for_read: give up if missing (`issue#18583 <http://tracker.ceph.com/issues/18583>`_, `pr#13087 <https://github.com/ceph/ceph/pull/13087>`_, Samuel Just)
+* core: osd/PrimaryLogPG: unify the access to primary pg (`pr#12527 <https://github.com/ceph/ceph/pull/12527>`_, huangjun)
+* core: osd/PrimayLogPG: update modified range to include the whole object size for write_full op (`pr#15021 <https://github.com/ceph/ceph/pull/15021>`_, runsisi)
+* core: osd/ReplicatedBackend: clear pull source once we are done with it (`issue#19076 <http://tracker.ceph.com/issues/19076>`_, `pr#13879 <https://github.com/ceph/ceph/pull/13879>`_, Samuel Just)
+* core: osd/ReplicatedBackend: remove MOSDSubOp cruft from repop_applied (`pr#14358 <https://github.com/ceph/ceph/pull/14358>`_, Jianpeng Ma)
+* core: osd/ReplicatedBackend: reset thread heartbeat after every omap entry … (`issue#20375 <http://tracker.ceph.com/issues/20375>`_, `pr#15823 <https://github.com/ceph/ceph/pull/15823>`_, Josh Durgin)
+* core: osd/ReplicatedBackend: take read locks for clone sources during recovery (`issue#17831 <http://tracker.ceph.com/issues/17831>`_, `pr#12844 <https://github.com/ceph/ceph/pull/12844>`_, Samuel Just)
+* core: os/filestore: call committed_thru when no journal entries are replayed (`pr#15781 <https://github.com/ceph/ceph/pull/15781>`_, Kuan-Kai Chiu)
+* core: os/filestore: debug which omap keys are set (`issue#19067 <http://tracker.ceph.com/issues/19067>`_, `pr#13671 <https://github.com/ceph/ceph/pull/13671>`_, Sage Weil)
+* core: os/filestore: do not free event if not added (`pr#16235 <https://github.com/ceph/ceph/pull/16235>`_, Kefu Chai)
+* core: os/filestore/FileJournal: bufferlist rebuild (`pr#13980 <https://github.com/ceph/ceph/pull/13980>`_, Jianpeng Ma)
+* core: os/filestore/FileJournal: FileJournal::open() close journal file before return error (`issue#20504 <http://tracker.ceph.com/issues/20504>`_, `pr#16120 <https://github.com/ceph/ceph/pull/16120>`_, Yang Honggang)
+* core: os/filestore/FileStore.cc: remove a redundant judgement when get max latency (`pr#15961 <https://github.com/ceph/ceph/pull/15961>`_, Jianpeng Ma)
+* core: os/filestore/FileStore.cc: remove unneeded loop (`pr#12177 <https://github.com/ceph/ceph/pull/12177>`_, Li Wang)
+* core: os/filestore: fix clang static check warn "use-after-free“ (`pr#12581 <https://github.com/ceph/ceph/pull/12581>`_, liuchang0812)
+* core: os/filestore: fix infinit loops in fiemap() (`pr#14367 <https://github.com/ceph/ceph/pull/14367>`_, Ning Yao)
+* core: os/filestore: handle error returned from write_fd() (`pr#10146 <https://github.com/ceph/ceph/pull/10146>`_, yonghengdexin735)
+* core: os/filestore/HashIndex: be loud about splits (`issue#18235 <http://tracker.ceph.com/issues/18235>`_, `pr#12421 <https://github.com/ceph/ceph/pull/12421>`_, Dan van der Ster)
+* core: os/filestore/JournalingObjectStore cleanup (`pr#12528 <https://github.com/ceph/ceph/pull/12528>`_, Li Wang)
+* core: os/filestore: require experimental flag for btrfs (`pr#16086 <https://github.com/ceph/ceph/pull/16086>`_, Sage Weil)
+* core: os/filestore: version will be uninitialized varible if store_version doesn't exist (`pr#12582 <https://github.com/ceph/ceph/pull/12582>`_, liuchang0812)
+* core: os/fs/FS.cc: remove the redundant code (`pr#14362 <https://github.com/ceph/ceph/pull/14362>`_, Jianpeng Ma)
+* core: os/FuseStore: include <functional> header in src/os/FuseStore.h for gcc 7.x (`pr#13454 <https://github.com/ceph/ceph/pull/13454>`_, Jos Collin)
+* core,performance: common/config_opts: improve rdma buffer size to 128k (`pr#13510 <https://github.com/ceph/ceph/pull/13510>`_, Haomai Wang)
+* core,performance: common/TrackedOp: various cleanups and optimizations (`pr#12537 <https://github.com/ceph/ceph/pull/12537>`_, Sage Weil)
+* core,performance: kv/RocksDBStore: Table options for indexing and filtering (`pr#16450 <https://github.com/ceph/ceph/pull/16450>`_, Mark Nelson)
+* core,performance: mon,osd: explicitly remap some pgs (`pr#13984 <https://github.com/ceph/ceph/pull/13984>`_, Sage Weil)
+* core,performance: msg/async: avoid lossy connection sending ack message (`pr#13700 <https://github.com/ceph/ceph/pull/13700>`_, Haomai Wang)
+* core,performance: msg/async/rdma: cleanup (`pr#13509 <https://github.com/ceph/ceph/pull/13509>`_, Haomai Wang)
+* core,performance: msg/async/rdma: refactor tx handle flow to get rid of locks (`pr#13680 <https://github.com/ceph/ceph/pull/13680>`_, Haomai Wang)
+* core,performance: msg/async: reduce write_lock contention (`pr#15092 <https://github.com/ceph/ceph/pull/15092>`_, Haomai Wang)
+* core,performance: osd/ECBackend: Send write message to peers first, then do local write (`pr#12522 <https://github.com/ceph/ceph/pull/12522>`_, huangjun)
+* core,performance: osd/OSD.h: requeue the scrub job with higher priority to shorten the blocking time of related requests (`pr#15552 <https://github.com/ceph/ceph/pull/15552>`_, Jin Cai)
+* core,performance: osd, os: reduce fiemap burden (`pr#14640 <https://github.com/ceph/ceph/pull/14640>`_, Piotr Dałek)
+* core,performance: osd/pg: bound the portion of the log we request in GetLog::GetLog() (`pr#12233 <https://github.com/ceph/ceph/pull/12233>`_, Jie Wang)
+* core,performance: osd/PG: make prioritized recovery possible (`pr#13723 <https://github.com/ceph/ceph/pull/13723>`_, Piotr Dałek)
+* core,performance: os/filestore: avoid unnecessary copy in filestore::_do_transaction (`pr#12578 <https://github.com/ceph/ceph/pull/12578>`_, Yunchuan Wen)
+* core,performance: os/filestore/HashIndex: randomize split threshold by a configurable amount (`issue#15835 <http://tracker.ceph.com/issues/15835>`_, `pr#15689 <https://github.com/ceph/ceph/pull/15689>`_, Josh Durgin)
+* core,performance: os/filestore: queue ondisk completion before apply work (`pr#13918 <https://github.com/ceph/ceph/pull/13918>`_, Pan Liu)
+* core,performance: os/filestore: use new sleep strategy when io_submit gets EAGAIN (`pr#14860 <https://github.com/ceph/ceph/pull/14860>`_, Pan Liu)
+* core,performance: os/kstore: Added rocksdb bloom filter settings (`pr#13053 <https://github.com/ceph/ceph/pull/13053>`_, Ted-Chang)
+* core,performance: src/OSD: add more useful perf counters for performance tuning (`pr#15915 <https://github.com/ceph/ceph/pull/15915>`_, Pan Liu)
+* core: PGLog: store extra duplicate ops beyond the normal log entries (`pr#16172 <https://github.com/ceph/ceph/pull/16172>`_, Josh Durgin, J. Eric Ivancich)
+* core: Prefix /proc/ with FreeBSD emulation (`pr#14290 <https://github.com/ceph/ceph/pull/14290>`_, Willem Jan Withagen)
+* core: PrimaryLogPG: don't update digests for objects with mismatched names (`issue#18409 <http://tracker.ceph.com/issues/18409>`_, `pr#12788 <https://github.com/ceph/ceph/pull/12788>`_, Samuel Just)
+* core: print more information when run ceph-osd cmd with 'check options' (`pr#16678 <https://github.com/ceph/ceph/pull/16678>`_, mychoxin)
+* core: qa: do not restrict valgrind runs to centos (`issue#18126 <http://tracker.ceph.com/issues/18126>`_, `pr#15389 <https://github.com/ceph/ceph/pull/15389>`_, Greg Farnum)
+* core,rbd: mon,osd: do not create rbd pool by default (`pr#15894 <https://github.com/ceph/ceph/pull/15894>`_, Greg Farnum, Sage Weil, David Zafman)
+* core: ReplicatedBackend: don't queue Context outside of ObjectStore with obc (`issue#18927 <http://tracker.ceph.com/issues/18927>`_, `pr#13569 <https://github.com/ceph/ceph/pull/13569>`_, Samuel Just)
+* core: Revert "PrimaryLogPG::failed_push: update missing as well" (`issue#18624 <http://tracker.ceph.com/issues/18624>`_, `pr#13090 <https://github.com/ceph/ceph/pull/13090>`_, David Zafman)
+* core,rgw: misc: SCA and Coverity Fixes (`pr#13208 <https://github.com/ceph/ceph/pull/13208>`_, Danny Al-Gaaf)
+* core,rgw: qa: Removed all 'default_idle_timeout' due to chnage in rwg task (`pr#15420 <https://github.com/ceph/ceph/pull/15420>`_, Yuri Weinstein)
+* core,rgw,tests: qa/rgw_snaps: move default_idle_timeout config under the client (`issue#20128 <http://tracker.ceph.com/issues/20128>`_, `pr#15400 <https://github.com/ceph/ceph/pull/15400>`_, Yehuda Sadeh)
+* core,rgw,tests: qa/suits/rados/basic/tasks/rgw_snaps: wait for pools to be created (`pr#16509 <https://github.com/ceph/ceph/pull/16509>`_, Sage Weil)
+* core: rocksdb: sync with upstream (`issue#18464 <http://tracker.ceph.com/issues/18464>`_, `pr#13306 <https://github.com/ceph/ceph/pull/13306>`_, Kefu Chai)
+* core: src/ceph.in: Use env(CEPH_DEV) to suppress noise from ceph (`pr#14746 <https://github.com/ceph/ceph/pull/14746>`_, Willem Jan Withagen)
+* core: src/vstart.sh: kill dead upmap option (`pr#15848 <https://github.com/ceph/ceph/pull/15848>`_, xie xingguo)
+* core:" Stringify needs access to << before reference" src/include/stringify.h (`pr#16334 <https://github.com/ceph/ceph/pull/16334>`_, Willem Jan Withagen)
+* core: test, osd: fix some coverity issues (`pr#13293 <https://github.com/ceph/ceph/pull/13293>`_, liuchang0812)
+* core: test/pybind/test_rados.py: tolerate TimedOut in test_ping_monitor (`issue#18529 <http://tracker.ceph.com/issues/18529>`_, `pr#12934 <https://github.com/ceph/ceph/pull/12934>`_, Samuel Just)
+* core,tests: ceph-disk: sensible default for block.db (`pr#15576 <https://github.com/ceph/ceph/pull/15576>`_, Loic Dachary)
+* core,tests: ceph-disk/tests: Certain partition types do not work on FreeBSD (`pr#13560 <https://github.com/ceph/ceph/pull/13560>`_, Willem Jan Withagen)
+* core,tests: ceph-disk/tests/test_main.py: FreeBSD does not do multipath (`pr#13847 <https://github.com/ceph/ceph/pull/13847>`_, Willem Jan Withagen)
+* core,tests: ceph_test_librados_api_misc: fix stupid LibRadosMiscConnectFailure.ConnectFailure test (`issue#15368 <http://tracker.ceph.com/issues/15368>`_, `pr#14261 <https://github.com/ceph/ceph/pull/14261>`_, Sage Weil)
+* core,tests: ceph_test_rados_api_misc: avoid livelock from PoolCreationRace (`pr#13565 <https://github.com/ceph/ceph/pull/13565>`_, Sage Weil)
+* core,tests: ceph_test_rados_api_misc: Fix trivial memory leak (`pr#12680 <https://github.com/ceph/ceph/pull/12680>`_, Brad Hubbard)
+* core,tests: ceph_test_rados_api: wait for snap trim on ENOENT during cleanup (`issue#19948 <http://tracker.ceph.com/issues/19948>`_, `pr#15638 <https://github.com/ceph/ceph/pull/15638>`_, Sage Weil)
+* core,tests: ceph_test_rados_api_watch_notify: flush after unwatch (`issue#20105 <http://tracker.ceph.com/issues/20105>`_, `pr#16402 <https://github.com/ceph/ceph/pull/16402>`_, Sage Weil)
+* core,tests: ceph_test_rados_api_watch_notify: make LibRadosWatchNotify.Watch3Timeout tolerate thrashing (`issue#19433 <http://tracker.ceph.com/issues/19433>`_, `pr#14899 <https://github.com/ceph/ceph/pull/14899>`_, Sage Weil)
+* core,tests: ceph_test_rados: max_stride_size must be more than min_stride_size (`issue#20775 <http://tracker.ceph.com/issues/20775>`_, `pr#16590 <https://github.com/ceph/ceph/pull/16590>`_, Lianne Wang)
+* core,tests: c_write_operations.cc: Fix trivial memory leak (`pr#12663 <https://github.com/ceph/ceph/pull/12663>`_, Brad Hubbard)
+* core,tests: do all valgrind runs on centos (`issue#20360 <http://tracker.ceph.com/issues/20360>`_, `issue#18126 <http://tracker.ceph.com/issues/18126>`_, `pr#16046 <https://github.com/ceph/ceph/pull/16046>`_, Sage Weil)
+* core,tests: os: allow 'osd objectstore = random' to pick either filestore or bluestore (`pr#13754 <https://github.com/ceph/ceph/pull/13754>`_, Sage Weil)
+* core,tests: qa: avoid map-gap tests for k=2 m=1 (`issue#20844 <http://tracker.ceph.com/issues/20844>`_, `pr#16789 <https://github.com/ceph/ceph/pull/16789>`_, Sage Weil)
+* core,tests: qa: move ceph-helpers-based make check tests to qa/standalone; run via teuthology (`pr#16513 <https://github.com/ceph/ceph/pull/16513>`_, Sage Weil)
+* core,tests: qa/objectstore/filestore-btrfs: test btrfs on trusty only (`issue#20169 <http://tracker.ceph.com/issues/20169>`_, `pr#15814 <https://github.com/ceph/ceph/pull/15814>`_, Sage Weil)
+* core,tests: qa/objectstore: test bluestore with aggressive compression (`pr#14623 <https://github.com/ceph/ceph/pull/14623>`_, Sage Weil)
+* core,tests: qa/rados/upgrade/jewel-x-singleton: run luminous.yaml at the end (`pr#13378 <https://github.com/ceph/ceph/pull/13378>`_, Sage Weil)
+* core,tests: qa: stop testing btrfs (`issue#20169 <http://tracker.ceph.com/issues/20169>`_, `pr#16044 <https://github.com/ceph/ceph/pull/16044>`_, Sage Weil)
+* core,tests: qa/suites/powercycle/osd/tasks/radosbench: consume less space (`issue#20302 <http://tracker.ceph.com/issues/20302>`_, `pr#15821 <https://github.com/ceph/ceph/pull/15821>`_, Sage Weil)
+* core,tests: qa/suites/rados: at-end: ignore PG_{AVAILABILITY,DEGRADED} (`issue#20693 <http://tracker.ceph.com/issues/20693>`_, `pr#16575 <https://github.com/ceph/ceph/pull/16575>`_, Sage Weil)
+* core,tests: qa/suites/rados/\*/at-end: wait for healthy before scrubbing (`pr#15245 <https://github.com/ceph/ceph/pull/15245>`_, Sage Weil)
+* core,tests: qa/suites/rados/basic: set low omap limit for rgw workload (`pr#13071 <https://github.com/ceph/ceph/pull/13071>`_, Sage Weil)
+* core,tests: qa/suites/rados/basic/tasks/rados_python: POOL_APP_NOT_ENABLED (`pr#16827 <https://github.com/ceph/ceph/pull/16827>`_, Sage Weil)
+* core,tests: qa/suites/rados/mgr/tasks/failover: whitelist (`pr#16795 <https://github.com/ceph/ceph/pull/16795>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton/all/reg11184: whitelist health warnings (`pr#16306 <https://github.com/ceph/ceph/pull/16306>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton-nomsg/health-warnings: behave on ext4 (`issue#20043 <http://tracker.ceph.com/issues/20043>`_, `pr#15207 <https://github.com/ceph/ceph/pull/15207>`_, Sage Weil)
+* core,tests: qa/suites/rados: temporarily remove scrub_test from basic/ until post-luminous (`issue#19935 <http://tracker.ceph.com/issues/19935>`_, `pr#15202 <https://github.com/ceph/ceph/pull/15202>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash/workload/\*: enable rados.py cache tiering ops (`issue#11793 <http://tracker.ceph.com/issues/11793>`_, `pr#16244 <https://github.com/ceph/ceph/pull/16244>`_, Sage Weil)
+* core,tests: qa/suites/upgrade/kraken-x: enable experimental for bluestore (`pr#15359 <https://github.com/ceph/ceph/pull/15359>`_, Sage Weil)
+* core,tests: qa/tasks/ceph: enable rbd on rbd pool (`pr#16794 <https://github.com/ceph/ceph/pull/16794>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: get osds all in after thrashing (`pr#15784 <https://github.com/ceph/ceph/pull/15784>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: wait for osd to start after objectstore-tool sequence (`issue#20705 <http://tracker.ceph.com/issues/20705>`_, `pr#16454 <https://github.com/ceph/ceph/pull/16454>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: wait longer for pg stats to flush (`pr#16322 <https://github.com/ceph/ceph/pull/16322>`_, Sage Weil)
+* core,tests: qa/tasks/ceph: osd_scrub_pgs: reissue scrub requests in loop (`issue#20326 <http://tracker.ceph.com/issues/20326>`_, `pr#15747 <https://github.com/ceph/ceph/pull/15747>`_, Sage Weil)
+* core,tests: qa/tasks/ceph.py: no osd id to 'osd create' command (`issue#20548 <http://tracker.ceph.com/issues/20548>`_, `pr#16233 <https://github.com/ceph/ceph/pull/16233>`_, Sage Weil)
+* core,tests: qa/tasks/ceph.py: tolerate active+clean+something (`pr#15717 <https://github.com/ceph/ceph/pull/15717>`_, Sage Weil)
+* core,tests: qa/tasks/ceph: simplify ceph deployment slightly (`pr#15853 <https://github.com/ceph/ceph/pull/15853>`_, Sage Weil)
+* core,tests: qa/tasks/ceph: wait for mgr to activate and pg stats to flush in health() (`issue#20744 <http://tracker.ceph.com/issues/20744>`_, `pr#16514 <https://github.com/ceph/ceph/pull/16514>`_, Sage Weil)
+* core,tests: qa/tasks/dump_stuck: fix dump_stuck test bug (`pr#16559 <https://github.com/ceph/ceph/pull/16559>`_, huangjun)
+* core,tests: qa/tasks/dump_stuck: fix for active+clean+remapped (`issue#20431 <http://tracker.ceph.com/issues/20431>`_, `pr#15955 <https://github.com/ceph/ceph/pull/15955>`_, Sage Weil)
+* core,tests: qa/tasks/radosbench: longer timeout (`pr#16213 <https://github.com/ceph/ceph/pull/16213>`_, Sage Weil)
+* core,tests: qa/workunits/cephtool/test.sh: add sudo for daemon compact (`pr#16500 <https://github.com/ceph/ceph/pull/16500>`_, Sage Weil)
+* core,tests: qa/workunits/cephtool/test.sh: fix osd full health detail grep (`issue#20187 <http://tracker.ceph.com/issues/20187>`_, `pr#15494 <https://github.com/ceph/ceph/pull/15494>`_, Sage Weil)
+* core,tests: qa/workunits/rados/test_health_warning: misc fixes (`issue#19990 <http://tracker.ceph.com/issues/19990>`_, `pr#15201 <https://github.com/ceph/ceph/pull/15201>`_, Sage Weil)
+* core,tests: qa/workunits/rest: use unique pool names for cephfs test (`pr#13188 <https://github.com/ceph/ceph/pull/13188>`_, Sage Weil)
+* core,tests: Revert "qa: do not restrict valgrind runs to centos" (`issue#20360 <http://tracker.ceph.com/issues/20360>`_, `pr#15791 <https://github.com/ceph/ceph/pull/15791>`_, Sage Weil)
+* core,tests: test: add separate ceph-helpers-based smoke test (`pr#16572 <https://github.com/ceph/ceph/pull/16572>`_, Sage Weil)
+* core,tests: test/librados/cmd.cc: Fix trivial memory leaks (`pr#12671 <https://github.com/ceph/ceph/pull/12671>`_, Brad Hubbard)
+* core,tests: test/librados/c_read_operations.cc: Fix trivial memory leak (`pr#12656 <https://github.com/ceph/ceph/pull/12656>`_, Brad Hubbard)
+* core,tests: test/librados/c_read_operations.cc: Fix valgrind errors (`issue#18354 <http://tracker.ceph.com/issues/18354>`_, `pr#12657 <https://github.com/ceph/ceph/pull/12657>`_, Brad Hubbard)
+* core,tests: test/librados: Silence Coverity memory leak warnings (`pr#12442 <https://github.com/ceph/ceph/pull/12442>`_, Brad Hubbard, Samuel Just)
+* core,tests: test/librados/snapshots.cc: Fix memory leak (`pr#12690 <https://github.com/ceph/ceph/pull/12690>`_, Brad Hubbard)
+* core,tests: test/librados/tier.cc: Fix valgrind errors (`issue#18360 <http://tracker.ceph.com/issues/18360>`_, `pr#12705 <https://github.com/ceph/ceph/pull/12705>`_, Brad Hubbard)
+* core,tests: test/osd/TestRados.cc: run set-redirect test after finishing setup (`issue#20114 <http://tracker.ceph.com/issues/20114>`_, `pr#15385 <https://github.com/ceph/ceph/pull/15385>`_, Myoungwon Oh)
+* core,tests: test_rados_watch_notify: Fix trivial memory leaks (`pr#12713 <https://github.com/ceph/ceph/pull/12713>`_, Brad Hubbard)
+* core,tests,tools: Fixes: http://tracker.ceph.com/issues/18533 (`pr#13423 <https://github.com/ceph/ceph/pull/13423>`_, Samuel Just, David Zafman)
+* core,tests: upgrade/jewel-x: a few fixes (`pr#16830 <https://github.com/ceph/ceph/pull/16830>`_, Sage Weil)
+* core: throttle: Minimal destructor fix for Luminous (`pr#16661 <https://github.com/ceph/ceph/pull/16661>`_, Adam C. Emerson)
+* core,tools: ceph: perfcounter priorities and daemonperf updates to use them (`pr#14793 <https://github.com/ceph/ceph/pull/14793>`_, Sage Weil, Dan Mick)
+* core,tools: kv: move 'bluestore-kv' hackery out of KeyValueDB into ceph-kvstore-tool (`issue#19778 <http://tracker.ceph.com/issues/19778>`_, `pr#14895 <https://github.com/ceph/ceph/pull/14895>`_, Sage Weil)
+* core,tools: osdmaptool: require --upmap-save before modifying input osdmap (`pr#15247 <https://github.com/ceph/ceph/pull/15247>`_, Sage Weil)
+* core: vstart.sh: start mgr after mon, before osds (`pr#16613 <https://github.com/ceph/ceph/pull/16613>`_, Sage Weil)
+* core: Wip 20985 divergent handling luminous (`issue#20985 <http://tracker.ceph.com/issues/20985>`_, `pr#17001 <https://github.com/ceph/ceph/pull/17001>`_, Greg Farnum)
+* create the ceph-volume and ceph-volume-systemd man pages (`pr#17158 <https://github.com/ceph/ceph/pull/17158>`_, Alfredo Deza)
+* crush: a couple of weight-set fixes (`pr#16623 <https://github.com/ceph/ceph/pull/16623>`_, xie xingguo)
+* crush: add devices class that rules can use as a filter (`issue#18943 <http://tracker.ceph.com/issues/18943>`_, `pr#13444 <https://github.com/ceph/ceph/pull/13444>`_, Loic Dachary)
+* crush: add --dump to crushtool (`pr#13726 <https://github.com/ceph/ceph/pull/13726>`_, Loic Dachary)
+* crush: add missing tunable in tests (`pr#15412 <https://github.com/ceph/ceph/pull/15412>`_, Loic Dachary)
+* crush: allow uniform buckets with no items (`pr#13521 <https://github.com/ceph/ceph/pull/13521>`_, Loic Dachary)
+* crush: API documentation (`pr#13205 <https://github.com/ceph/ceph/pull/13205>`_, Loic Dachary)
+* crush: bucket: crush_add_uniform_bucket_item should check for uniformity (`pr#14208 <https://github.com/ceph/ceph/pull/14208>`_, Sahid Orentino Ferdjaoui)
+* crush: builder: clean the arguments of crush_reweight\* methods (`pr#14110 <https://github.com/ceph/ceph/pull/14110>`_, Sahid Orentino Ferdjaoui)
+* crush: builder: creating crush map with optimal configurations (`pr#14209 <https://github.com/ceph/ceph/pull/14209>`_, Sahid Orentino Ferdjaoui)
+* crush: builder: legacy has chooseleaf_stable = 0 (`pr#14695 <https://github.com/ceph/ceph/pull/14695>`_, Loic Dachary)
+* crush: crush_init_workspace starts with struct crush_work (`pr#14696 <https://github.com/ceph/ceph/pull/14696>`_, Loic Dachary)
+* crush: detect and (usually) fix ruleset != rule id (`pr#13683 <https://github.com/ceph/ceph/pull/13683>`_, Sage Weil)
+* crush: document tunables and rule step set (`pr#13722 <https://github.com/ceph/ceph/pull/13722>`_, Loic Dachary)
+* crush: do is_out test only if we do not collide (`pr#13326 <https://github.com/ceph/ceph/pull/13326>`_, xie xingguo)
+* crush: encode can override weights with weight set (`issue#19836 <http://tracker.ceph.com/issues/19836>`_, `pr#15002 <https://github.com/ceph/ceph/pull/15002>`_, Loic Dachary)
+* crush: enforce buckets-before-rules rule (`pr#16453 <https://github.com/ceph/ceph/pull/16453>`_, Sage Weil)
+* crush: fix CrushCompiler won't compile maps with empty shadow tree (`pr#17228 <https://github.com/ceph/ceph/pull/17228>`_, xie xingguo)
+* crush: fix dprintk compilation (`pr#13424 <https://github.com/ceph/ceph/pull/13424>`_, Loic Dachary)
+* crush: force rebuilding shadow hierarchy after swapping buckets (`pr#17229 <https://github.com/ceph/ceph/pull/17229>`_, xie xingguo)
+* crush: misc changes/fixes for device classes (`issue#20845 <http://tracker.ceph.com/issues/20845>`_, `pr#16805 <https://github.com/ceph/ceph/pull/16805>`_, Kefu Chai, xie xingguo, Sage Weil)
+* crush: more class fixes (`pr#16837 <https://github.com/ceph/ceph/pull/16837>`_, xie xingguo)
+* crush: only encode class info if SERVER_LUMINOUS (`issue#19361 <http://tracker.ceph.com/issues/19361>`_, `pr#14131 <https://github.com/ceph/ceph/pull/14131>`_, Sage Weil)
+* crush: optimize header file dependency (`pr#9307 <https://github.com/ceph/ceph/pull/9307>`_, Xiaowei Chen)
+* crush: silence warning from -Woverflow (`pr#16329 <https://github.com/ceph/ceph/pull/16329>`_, Jos Collin)
+* crush: s/ruleset/id/ in decompiled output; prevent compilation when ruleset != id (`pr#16400 <https://github.com/ceph/ceph/pull/16400>`_, Sage Weil)
+* crush: update choose_args when items are added/removed (`pr#15311 <https://github.com/ceph/ceph/pull/15311>`_, Loic Dachary)
+* crush: update documentation for negative choose step (`pr#14970 <https://github.com/ceph/ceph/pull/14970>`_, Loic Dachary)
+* crush: various weight-set fixes (`pr#17214 <https://github.com/ceph/ceph/pull/17214>`_, xie xingguo)
+* crush: verify weights is influenced by the number of replicas (`issue#15653 <http://tracker.ceph.com/issues/15653>`_, `pr#13083 <https://github.com/ceph/ceph/pull/13083>`_, Adam C. Emerson, Loic Dachary)
+* crush: weight_set and id remapping (`issue#15653 <http://tracker.ceph.com/issues/15653>`_, `pr#14486 <https://github.com/ceph/ceph/pull/14486>`_, Loic Dachary)
+* crush:  when osd_location_hook does not exist, we should exit error (`pr#12961 <https://github.com/ceph/ceph/pull/12961>`_, song baisen)
+* doc: 12.1.0/release notes 2 (`pr#15627 <https://github.com/ceph/ceph/pull/15627>`_, Abhishek Lekshmanan)
+* doc: 12.1.1 & 12.1.2 release notes (`pr#16377 <https://github.com/ceph/ceph/pull/16377>`_, Abhishek Lekshmanan)
+* doc: add 0.94.10 and hammer EOL to releases.rst (`pr#13069 <https://github.com/ceph/ceph/pull/13069>`_, Nathan Cutler)
+* doc: add 12.0.1 release notes (`pr#14106 <https://github.com/ceph/ceph/pull/14106>`_, Abhishek Lekshmanan)
+* doc: Add amitkumar50 affiliation to .organizationmap (`pr#16475 <https://github.com/ceph/ceph/pull/16475>`_, Amit Kumar)
+* doc add ceph-volume and ceph-volume-systemd man pages to CMakeLists file (`pr#17170 <https://github.com/ceph/ceph/pull/17170>`_, Alfredo Deza)
+* doc: add changelog for v0.94.10 (`pr#13572 <https://github.com/ceph/ceph/pull/13572>`_, Abhishek Lekshmanan)
+* doc: add changelog for v10.2.6 Jewel release (`pr#13839 <https://github.com/ceph/ceph/pull/13839>`_, Abhishek Lekshmanan)
+* doc: add changelog for v10.2.7 (`pr#14441 <https://github.com/ceph/ceph/pull/14441>`_, Abhishek Lekshmanan)
+* doc: add descriptions for mon/mgr options (`pr#15032 <https://github.com/ceph/ceph/pull/15032>`_, Kefu Chai)
+* doc: add doc requirements on PR submitters (`pr#16394 <https://github.com/ceph/ceph/pull/16394>`_, John Spray)
+* doc: added mgr caps to manual deployment documentation (`pr#16660 <https://github.com/ceph/ceph/pull/16660>`_, Nick Erdmann)
+* doc: add FreeBSD manual install (`pr#14941 <https://github.com/ceph/ceph/pull/14941>`_, Willem Jan Withagen)
+* doc: add instructions for replacing an OSD (`pr#16314 <https://github.com/ceph/ceph/pull/16314>`_, Kefu Chai)
+* doc: add new cn ceph mirror to doc and mirroring (`pr#15089 <https://github.com/ceph/ceph/pull/15089>`_, Shengjing Zhu)
+* doc: add optional argument for build-doc (`pr#14058 <https://github.com/ceph/ceph/pull/14058>`_, Kefu Chai)
+* doc: add rados xattr commands to manpage (`pr#15362 <https://github.com/ceph/ceph/pull/15362>`_, Andreas Gerstmayr)
+* doc: add rbd new trash cli and cleanups  in release-notes.rst (`issue#20702 <http://tracker.ceph.com/issues/20702>`_, `pr#16498 <https://github.com/ceph/ceph/pull/16498>`_, songweibin)
+* doc: add README to dmclock subdir to inform developers it's a git subtree (`pr#15386 <https://github.com/ceph/ceph/pull/15386>`_, J. Eric Ivancich)
+* doc: add RGW ldap auth documentation (`pr#14339 <https://github.com/ceph/ceph/pull/14339>`_, Harald Klein)
+* doc: add some undocumented options to rbd-nbd (`pr#14134 <https://github.com/ceph/ceph/pull/14134>`_, wangzhengyong)
+* doc: add verbiage to rbdmap manpage (`issue#18262 <http://tracker.ceph.com/issues/18262>`_, `pr#12509 <https://github.com/ceph/ceph/pull/12509>`_, Nathan Cutler)
+* doc: Add Zabbix ceph-mgr plugin to PendingReleaseNotes (`pr#16412 <https://github.com/ceph/ceph/pull/16412>`_, Wido den Hollander)
+* doc: AUTHORS: update CephFS PTL (`pr#16399 <https://github.com/ceph/ceph/pull/16399>`_, Patrick Donnelly)
+* doc: AUTHORS: update tech leads (`pr#14350 <https://github.com/ceph/ceph/pull/14350>`_, Patrick Donnelly)
+* doc: AUTHORS: update with release manager, backport team (`pr#15391 <https://github.com/ceph/ceph/pull/15391>`_, Sage Weil)
+* doc: build/install-deps.sh: Add sphinx package for building docs on FreeBSD (`pr#13223 <https://github.com/ceph/ceph/pull/13223>`_, Willem Jan Withagen)
+* doc: ceph-disk: use '-' for feeding ceph cli with stdin (`pr#16362 <https://github.com/ceph/ceph/pull/16362>`_, Kefu Chai)
+* doc: change osd_op_thread_timeout default value to 15 (`pr#14199 <https://github.com/ceph/ceph/pull/14199>`_, Andreas Gerstmayr)
+* doc: Change the default values of some OSD options (`issue#20199 <http://tracker.ceph.com/issues/20199>`_, `pr#15566 <https://github.com/ceph/ceph/pull/15566>`_, Bara Ancincova)
+* doc: clarify "ceph quorum" syntax (`issue#17802 <http://tracker.ceph.com/issues/17802>`_, `pr#11787 <https://github.com/ceph/ceph/pull/11787>`_, Nathan Cutler)
+* doc: clarify SubmittingPatches.rst (`pr#12988 <https://github.com/ceph/ceph/pull/12988>`_, Nathan Cutler)
+* doc: clarify that "ms bind ipv6" disables IPv4 (`pr#13317 <https://github.com/ceph/ceph/pull/13317>`_, Ken Dreyer)
+* doc: clarify the path restriction mds cap example (`pr#12993 <https://github.com/ceph/ceph/pull/12993>`_, John Spray)
+* doc: common/options.cc: document bluestore config options (`pr#16489 <https://github.com/ceph/ceph/pull/16489>`_, Sage Weil)
+* doc: correct and improve add user capability section (`pr#14055 <https://github.com/ceph/ceph/pull/14055>`_, Chu, Hua-Rong)
+* doc: correct arguments for ceph tell osd.N bench (`pr#14462 <https://github.com/ceph/ceph/pull/14462>`_, Patrick Dinnen)
+* doc: Correcting the remove bucket example and adding bucket link/unlink examples (`pr#12460 <https://github.com/ceph/ceph/pull/12460>`_, Uday Mullangi)
+* doc: correct S3 lifecycle support explain (`issue#18459 <http://tracker.ceph.com/issues/18459>`_, `pr#12827 <https://github.com/ceph/ceph/pull/12827>`_, liuchang0812)
+* doc: correct the quota section (`issue#19397 <http://tracker.ceph.com/issues/19397>`_, `pr#14122 <https://github.com/ceph/ceph/pull/14122>`_, Chu, Hua-Rong)
+* doc: crush: API documentation fixes (`pr#13589 <https://github.com/ceph/ceph/pull/13589>`_, Loic Dachary)
+* doc: crush typo in algorithm description (`pr#13661 <https://github.com/ceph/ceph/pull/13661>`_, Loic Dachary)
+* doc: deletes duplicated word and clarifies an example (`pr#13746 <https://github.com/ceph/ceph/pull/13746>`_, Tahia Khan)
+* doc: describe CephFS max_file_size (`pr#15287 <https://github.com/ceph/ceph/pull/15287>`_, Ken Dreyer)
+* doc: describe mark_events logging available via the OSD's OpTracker (`pr#15095 <https://github.com/ceph/ceph/pull/15095>`_, Greg Farnum)
+* doc: Describe mClock's use within Ceph in great detail (`pr#16707 <https://github.com/ceph/ceph/pull/16707>`_, J. Eric Ivancich)
+* doc: dev add a note about ccache (`pr#14478 <https://github.com/ceph/ceph/pull/14478>`_, Abhishek Lekshmanan)
+* doc: dev: add notes on PR make check validation test (`pr#16079 <https://github.com/ceph/ceph/pull/16079>`_, Nathan Cutler)
+* doc: dev guide: how to run s3-tests locally against vstart (`pr#14508 <https://github.com/ceph/ceph/pull/14508>`_, Nathan Cutler, Abhishek Lekshmanan)
+* doc: dev improve the s3tests doc to reflect current scripts (`pr#15180 <https://github.com/ceph/ceph/pull/15180>`_, Abhishek Lekshmanan)
+* doc: doc/cephfs: mention RADOS object size limit (`pr#15550 <https://github.com/ceph/ceph/pull/15550>`_, John Spray)
+* doc: doc/dev: update log_based_pg.rst, fix some display problem (`pr#12730 <https://github.com/ceph/ceph/pull/12730>`_, liuchang0812)
+* doc: Doc:Fixes Python Swift client commands (`issue#17746 <http://tracker.ceph.com/issues/17746>`_, `pr#12887 <https://github.com/ceph/ceph/pull/12887>`_, Ronak Jain)
+* doc: doc/install/manual-deployment: update osd creation steps (`pr#16573 <https://github.com/ceph/ceph/pull/16573>`_, Sage Weil)
+* doc: doc/mgr/dashboard: update dashboard docs to reflect new defaults (`pr#16241 <https://github.com/ceph/ceph/pull/16241>`_, Sage Weil)
+* doc: doc/mon: fix ceph-authtool command in rebuild mon's sample (`pr#16503 <https://github.com/ceph/ceph/pull/16503>`_, huanwen ren)
+* doc: doc/qa: cover `config help` command (`pr#16727 <https://github.com/ceph/ceph/pull/16727>`_, John Spray)
+* doc: doc/rados.8: add offset option for put command (`pr#16155 <https://github.com/ceph/ceph/pull/16155>`_, Jianpeng Ma)
+* doc: doc/rados: add page for health checks and update monitoring.rst (`pr#16566 <https://github.com/ceph/ceph/pull/16566>`_, John Spray)
+* doc: doc/rados/configuration: document bluestore (`pr#16765 <https://github.com/ceph/ceph/pull/16765>`_, Sage Weil)
+* doc: doc/radosgw/s3/cpp.rst: update usage of libs3 APIs to make the examples work (`pr#10851 <https://github.com/ceph/ceph/pull/10851>`_, Weibing Zhang)
+* doc: doc/rados/operations/health-checks: osd section (`pr#16611 <https://github.com/ceph/ceph/pull/16611>`_, Sage Weil)
+* doc: doc/release-notes: add Images creation timestamp note (`pr#15963 <https://github.com/ceph/ceph/pull/15963>`_, clove)
+* doc: doc/release-notes: avoid 'production-ready' in describing kraken (`pr#13675 <https://github.com/ceph/ceph/pull/13675>`_, Sage Weil)
+* doc: doc/release-notes: final kraken notes (`pr#12968 <https://github.com/ceph/ceph/pull/12968>`_, Sage Weil)
+* doc: doc/release-notes: fix bluestore links (`pr#16787 <https://github.com/ceph/ceph/pull/16787>`_, Sage Weil)
+* doc: doc/release-notes: fix links, formatting; add crush device class docs (`pr#16741 <https://github.com/ceph/ceph/pull/16741>`_, Sage Weil)
+* doc: doc/release-notes: fix upmap and osd replacement links; add fixme (`pr#16730 <https://github.com/ceph/ceph/pull/16730>`_, Sage Weil)
+* doc: doc/release-notes: Luminous release notes typo fixes  "ceph config-key ls"->"ceph config-key list" (`pr#16330 <https://github.com/ceph/ceph/pull/16330>`_, scienceluo)
+* doc: doc/release-notes: Luminous release notes typo fixes (`pr#16338 <https://github.com/ceph/ceph/pull/16338>`_, Luo Kexue)
+* doc: doc/release-notes: sort release note changes into the right section (`pr#16764 <https://github.com/ceph/ceph/pull/16764>`_, Sage Weil)
+* doc: doc/release-notes: update device class cli (`pr#16851 <https://github.com/ceph/ceph/pull/16851>`_, xie xingguo)
+* doc: doc/release-notes: update luminous notes (`pr#15851 <https://github.com/ceph/ceph/pull/15851>`_, Sage Weil)
+* doc: doc/release-notes: update which jewel version does sortbitwise warning (`pr#15209 <https://github.com/ceph/ceph/pull/15209>`_, Sage Weil)
+* doc: doc/releases: Update releases from Feb 2017 to July 2017 (`pr#16303 <https://github.com/ceph/ceph/pull/16303>`_, Bryan Stillwell)
+* doc: doc/rgw: instructions for changing multisite master zone (`pr#14089 <https://github.com/ceph/ceph/pull/14089>`_, Casey Bodley)
+* doc: doc/rgw: remove fastcgi page and sample configs (`pr#15133 <https://github.com/ceph/ceph/pull/15133>`_, Casey Bodley)
+* doc: doc/rgw: remove Federated Configuration, clean up multisite (`issue#19504 <http://tracker.ceph.com/issues/19504>`_, `issue#18082 <http://tracker.ceph.com/issues/18082>`_, `pr#15132 <https://github.com/ceph/ceph/pull/15132>`_, Casey Bodley)
+* doc: docs: Clarify the relationship of min_size to EC pool recovery (`pr#14419 <https://github.com/ceph/ceph/pull/14419>`_, Brad Hubbard)
+* doc: docs: Fix problems with example code (`pr#14007 <https://github.com/ceph/ceph/pull/14007>`_, Brad Hubbard)
+* doc: docs: mgr dashboard (`pr#15920 <https://github.com/ceph/ceph/pull/15920>`_, Wido den Hollander)
+* doc: [docs/quick-start]: update quick start to add a note for mgr create command for luminous+ builds (`pr#16350 <https://github.com/ceph/ceph/pull/16350>`_, Vasu Kulkarni)
+* doc: Documentation Fixes for http://tracker.ceph.com/issues/19879 (`issue#20057 <http://tracker.ceph.com/issues/20057>`_, `issue#19879 <http://tracker.ceph.com/issues/19879>`_, `pr#15606 <https://github.com/ceph/ceph/pull/15606>`_, Sameer Tiwari)
+* doc: Documentation updates for July 2017 releases (`pr#16401 <https://github.com/ceph/ceph/pull/16401>`_, Bryan Stillwell)
+* doc: document bluestore compression settings (`pr#16747 <https://github.com/ceph/ceph/pull/16747>`_, Kefu Chai)
+* doc: document mClock related options (`pr#16552 <https://github.com/ceph/ceph/pull/16552>`_, Kefu Chai)
+* doc: document `osd-agent-{max,low}-ops` options (`pr#13648 <https://github.com/ceph/ceph/pull/13648>`_, Kefu Chai)
+* doc: document perf historgrams (`pr#15150 <https://github.com/ceph/ceph/pull/15150>`_, Piotr Dałek)
+* doc: document "rados cleanup" in rados manpage (`issue#20894 <http://tracker.ceph.com/issues/20894>`_, `pr#16777 <https://github.com/ceph/ceph/pull/16777>`_, Nathan Cutler)
+* doc: document repair/scrub features (`issue#15786 <http://tracker.ceph.com/issues/15786>`_, `pr#9032 <https://github.com/ceph/ceph/pull/9032>`_, Kefu Chai, David Zafman)
+* doc: Document RGW quota cache options (`issue#18747 <http://tracker.ceph.com/issues/18747>`_, `pr#13395 <https://github.com/ceph/ceph/pull/13395>`_, Daniel Gryniewicz)
+* doc: Document that osd_heartbeat_grace applies to MON and OSD (`pr#13098 <https://github.com/ceph/ceph/pull/13098>`_, Wido den Hollander)
+* doc: document the setup of restful and dashboard plugins (`issue#20239 <http://tracker.ceph.com/issues/20239>`_, `pr#15707 <https://github.com/ceph/ceph/pull/15707>`_, Kefu Chai)
+* doc: explain about logging levels (`pr#12920 <https://github.com/ceph/ceph/pull/12920>`_, liuchang0812)
+* doc: fio: update README.md so only the fio ceph engine is built (`pr#15081 <https://github.com/ceph/ceph/pull/15081>`_, Kefu Chai)
+* doc: fix a typo (`pr#13930 <https://github.com/ceph/ceph/pull/13930>`_, Drunkard Zhang)
+* doc: fix broken link in erasure-code.rst (`issue#19972 <http://tracker.ceph.com/issues/19972>`_, `pr#15143 <https://github.com/ceph/ceph/pull/15143>`_, MinSheng Lin)
+* doc: fix document about rados mon (`pr#12662 <https://github.com/ceph/ceph/pull/12662>`_, liuchang0812)
+* doc: Fixed a typo in yum repo filename script (`pr#16431 <https://github.com/ceph/ceph/pull/16431>`_, Jeff Green)
+* doc: fixes a broken hyperlink to RADOS paper in architecture (`pr#13682 <https://github.com/ceph/ceph/pull/13682>`_, Tahia Khan)
+* doc: Fixes a typo (`pr#13985 <https://github.com/ceph/ceph/pull/13985>`_, Edwin F. Boza)
+* doc: Fixes parameter name in rbd configuration on openstack havana/icehouse (`issue#17978 <http://tracker.ceph.com/issues/17978>`_, `pr#13403 <https://github.com/ceph/ceph/pull/13403>`_, Michael Eischer)
+* doc: Fixes radosgw-admin ex: in swift auth section (`issue#16687 <http://tracker.ceph.com/issues/16687>`_, `pr#12646 <https://github.com/ceph/ceph/pull/12646>`_, SirishaGuduru)
+* doc:  fixes to silence sphinx-build (`pr#13997 <https://github.com/ceph/ceph/pull/13997>`_, Kefu Chai)
+* doc: fix factual inaccuracy in doc/architecture.rst (`pr#15235 <https://github.com/ceph/ceph/pull/15235>`_, Nathan Cutler, Sage Weil)
+* doc: fixing an error in 12.0.3 release notes (`pr#15195 <https://github.com/ceph/ceph/pull/15195>`_, Abhishek Lekshmanan)
+* doc: fix link for ceph-mgr cephx authorization (`pr#16246 <https://github.com/ceph/ceph/pull/16246>`_, Greg Farnum)
+* doc: fix link that pointed to a nonexistent file (`pr#14740 <https://github.com/ceph/ceph/pull/14740>`_, Peter Maloney)
+* doc: fix syntax on code snippets in cephfs/multimds (`pr#15499 <https://github.com/ceph/ceph/pull/15499>`_, John Spray)
+* doc: fix the librados c api can not compile problem (`pr#9396 <https://github.com/ceph/ceph/pull/9396>`_, song baisen)
+* doc: fix the links to http://ceph.com/docs (`issue#19090 <http://tracker.ceph.com/issues/19090>`_, `pr#13976 <https://github.com/ceph/ceph/pull/13976>`_, Kefu Chai)
+* doc: Fix typo and grammar in RGW config reference (`pr#13356 <https://github.com/ceph/ceph/pull/13356>`_, Ruben Kerkhof)
+* doc: fix typo in config.rst (`pr#16721 <https://github.com/ceph/ceph/pull/16721>`_, Jos Collin)
+* doc: fix typos in config.rst (`pr#16681 <https://github.com/ceph/ceph/pull/16681>`_, Song Shun)
+* doc: fix typos in radosgw-admin usage (`pr#13936 <https://github.com/ceph/ceph/pull/13936>`_, Enming Zhang)
+* doc: freshen mgr docs (`pr#15690 <https://github.com/ceph/ceph/pull/15690>`_, John Spray)
+* doc: hammer 0.94.10 release notes (`pr#13152 <https://github.com/ceph/ceph/pull/13152>`_, Nathan Cutler)
+* doc: Have install put manpages in the FreeBSD correct location (`pr#13301 <https://github.com/ceph/ceph/pull/13301>`_, Willem Jan Withagen)
+* doc: how to specify filesystem for cephfs clients (`pr#14087 <https://github.com/ceph/ceph/pull/14087>`_, John Spray)
+* doc: improve firewalld instructions (`pr#13360 <https://github.com/ceph/ceph/pull/13360>`_, Ken Dreyer)
+* doc: Indicate how to add multiple admin capbabilies (`pr#13956 <https://github.com/ceph/ceph/pull/13956>`_, Chu, Hua-Rong)
+* doc: instructions and guidance for multimds (`issue#19135 <http://tracker.ceph.com/issues/19135>`_, `pr#13830 <https://github.com/ceph/ceph/pull/13830>`_, John Spray)
+* doc: instructions for provisioning OpenStack VMs ad hoc (`pr#13368 <https://github.com/ceph/ceph/pull/13368>`_, Nathan Cutler)
+* doc: Jewel 10.2.6 release notes (`pr#13835 <https://github.com/ceph/ceph/pull/13835>`_, Abhishek Lekshmanan)
+* doc: Jewel v10.2.8 release notes (`pr#16274 <https://github.com/ceph/ceph/pull/16274>`_, Nathan Cutler)
+* doc: Jewel v10.2.9 release notes (`pr#16318 <https://github.com/ceph/ceph/pull/16318>`_, Nathan Cutler)
+* doc: kernel client os-recommendations update (`pr#13369 <https://github.com/ceph/ceph/pull/13369>`_, John Spray, Ilya Dryomov)
+* doc: kill some broken links (`pr#15203 <https://github.com/ceph/ceph/pull/15203>`_, liuchang0812)
+* doc: kill sphinx warnings (`pr#16198 <https://github.com/ceph/ceph/pull/16198>`_, Kefu Chai)
+* doc: luminous: doc: update rbd-mirroring documentation (`issue#20701 <http://tracker.ceph.com/issues/20701>`_, `pr#16912 <https://github.com/ceph/ceph/pull/16912>`_, Jason Dillaman)
+* doc: Luminous release notes typo fixes (`pr#15899 <https://github.com/ceph/ceph/pull/15899>`_, Abhishek Lekshmanan)
+* doc: mailmap: add affiliation for Zhu Shangzhong (`pr#16537 <https://github.com/ceph/ceph/pull/16537>`_, Zhu Shangzhong)
+* doc: mailmap: add Alibaba into organization map (`pr#14900 <https://github.com/ceph/ceph/pull/14900>`_, James Liu)
+* doc: mailmap: add Myoungwon Oh's mailmap and affiliation (`pr#15934 <https://github.com/ceph/ceph/pull/15934>`_, Myoungwon Oh)
+* doc: mailmap for v12.0.2 (`pr#14753 <https://github.com/ceph/ceph/pull/14753>`_, Abhishek Lekshmanan)
+* doc: mailmap: Michal Koutny affiliation (`pr#13036 <https://github.com/ceph/ceph/pull/13036>`_, Nathan Cutler)
+* doc: mailmap, organizationmap: add affiliation for Tushar Gohad (`pr#16081 <https://github.com/ceph/ceph/pull/16081>`_, Tushar Gohad)
+* doc:  .mailmap, .organizationmap: Update Fan Yang information and affiliation (`pr#16067 <https://github.com/ceph/ceph/pull/16067>`_, Fan Yang)
+* doc: .mailmap, .organizationmap: Update Song Weibin information and affiliation (`pr#16311 <https://github.com/ceph/ceph/pull/16311>`_, songweibin)
+* doc: .mailmap, .organizationmap: Update ztczll affiliation (`pr#16038 <https://github.com/ceph/ceph/pull/16038>`_, zhanglei)
+* doc: mailmap updates for v11.1.0 (`pr#12335 <https://github.com/ceph/ceph/pull/12335>`_, Abhishek Lekshmanan)
+* doc: mailmap updates (`pr#13309 <https://github.com/ceph/ceph/pull/13309>`_, Loic Dachary)
+* doc: mailmap: V12.0.1 credits (`pr#14479 <https://github.com/ceph/ceph/pull/14479>`_, M Ranga Swami Reddy)
+* doc: mailmap: Willem Jan Withagen affiliation (`pr#13034 <https://github.com/ceph/ceph/pull/13034>`_, Willem Jan Withagen)
+* doc: mailmap: ztczll affiliation (`pr#15079 <https://github.com/ceph/ceph/pull/15079>`_, zhanglei)
+* doc: man/8/ceph-disk: fix formatting (`pr#13969 <https://github.com/ceph/ceph/pull/13969>`_, Kefu Chai)
+* doc: mention certain conf vars should be in global (`pr#15119 <https://github.com/ceph/ceph/pull/15119>`_, Ali Maredia)
+* doc: mention ENXIO change in the 10.2.6 release notes (`pr#13878 <https://github.com/ceph/ceph/pull/13878>`_, Nathan Cutler)
+* doc: mention --show-mappings in crushtool manpage (`issue#19649 <http://tracker.ceph.com/issues/19649>`_, `pr#14599 <https://github.com/ceph/ceph/pull/14599>`_, Nathan Cutler, Loic Dachary)
+* doc: mention teuthology-worker security group (`pr#14748 <https://github.com/ceph/ceph/pull/14748>`_, Nathan Cutler)
+* doc: Merge pull request from stiwari/wip-19879 (`issue#19879 <http://tracker.ceph.com/issues/19879>`_, `pr#15609 <https://github.com/ceph/ceph/pull/15609>`_, Sameer Tiwari)
+* doc: mgr/restful: bind to :: and update docs (`pr#16267 <https://github.com/ceph/ceph/pull/16267>`_, Sage Weil)
+* doc: minor changes in fuse client config reference (`pr#13065 <https://github.com/ceph/ceph/pull/13065>`_, Barbora An�incová)
+* doc: minor change to a cloud testing paragraph (`pr#13277 <https://github.com/ceph/ceph/pull/13277>`_, Jan Fajerski)
+* doc: minor fixes in radosgw/ (`pr#15103 <https://github.com/ceph/ceph/pull/15103>`_, Drunkard Zhang)
+* doc: min_size advice is not helpful (`pr#12936 <https://github.com/ceph/ceph/pull/12936>`_, Brad Hubbard)
+* doc: misc minor fixes (`pr#13713 <https://github.com/ceph/ceph/pull/13713>`_, Drunkard Zhang)
+* doc: Modify Configuring Cinder section (`issue#18840 <http://tracker.ceph.com/issues/18840>`_, `pr#13400 <https://github.com/ceph/ceph/pull/13400>`_, Shinobu Kinjo)
+* doc: op queue and mclock related options (`pr#16662 <https://github.com/ceph/ceph/pull/16662>`_, J. Eric Ivancich)
+* doc: organizationmap: add Xianxia Xiao to Kylin Cloud team (`pr#12718 <https://github.com/ceph/ceph/pull/12718>`_, Yunchuan Wen)
+* doc: PendingReleaseNotes: "ceph -w" behavior has changed drastically (`pr#16425 <https://github.com/ceph/ceph/pull/16425>`_, Joao Eduardo Luis, Nathan Cutler)
+* doc: PendingReleaseNotes: notes on whiteouts vs pgnls (`pr#15575 <https://github.com/ceph/ceph/pull/15575>`_, Sage Weil)
+* doc: PendingReleaseNotes: note the fuse fstab format change (`pr#13259 <https://github.com/ceph/ceph/pull/13259>`_, John Spray)
+* doc: PendingReleaseNotes: recent cephfs changes (`pr#14196 <https://github.com/ceph/ceph/pull/14196>`_, John Spray)
+* doc: PendingReleaseNotes: warning about 'osd rm ...' and #19119 (`issue#19119 <http://tracker.ceph.com/issues/19119>`_, `pr#13731 <https://github.com/ceph/ceph/pull/13731>`_, Sage Weil)
+* doc: peoplemap: add pdonnell alias (`pr#14352 <https://github.com/ceph/ceph/pull/14352>`_, Patrick Donnelly)
+* doc: radosgw-admin: new 'global quota' commands update period config (`issue#19409 <http://tracker.ceph.com/issues/19409>`_, `pr#14252 <https://github.com/ceph/ceph/pull/14252>`_, Casey Bodley)
+* doc: README.FreeBSD: update current status (`pr#12096 <https://github.com/ceph/ceph/pull/12096>`_, Willem Jan Withagen)
+* doc: README.FreeBSD: Update the status (`pr#14406 <https://github.com/ceph/ceph/pull/14406>`_, Willem Jan Withagen)
+* doc: README.md: fix build instructions inconsistent (`pr#14555 <https://github.com/ceph/ceph/pull/14555>`_, Yao Zongyou)
+* doc: README.md: use github heading syntax to mark the headings (`pr#14591 <https://github.com/ceph/ceph/pull/14591>`_, Kefu Chai)
+* doc: release-notes clarify about rgw encryption (`pr#14800 <https://github.com/ceph/ceph/pull/14800>`_, Abhishek Lekshmanan)
+* doc: release notes for v10.2.7 Jewel (`pr#14295 <https://github.com/ceph/ceph/pull/14295>`_, Abhishek Lekshmanan)
+* doc: release notes for v11.1.1 (`pr#12642 <https://github.com/ceph/ceph/pull/12642>`_, Abhishek Lekshmanan)
+* doc: release notes for v12.0.3 (dev) (`pr#15090 <https://github.com/ceph/ceph/pull/15090>`_, Abhishek Lekshmanan)
+* doc: releases update the luminous, hammer, jewel release dates (`pr#13584 <https://github.com/ceph/ceph/pull/13584>`_, Abhishek Lekshmanan)
+* doc: remove deprecated subcommand in man/8/ceph.rst (`pr#14928 <https://github.com/ceph/ceph/pull/14928>`_, Drunkard Zhang)
+* doc: remove docs on non-existant command (`pr#16616 <https://github.com/ceph/ceph/pull/16616>`_, Luo Kexue, Kefu Chai)
+* doc: remove duplicated references (`pr#13396 <https://github.com/ceph/ceph/pull/13396>`_, Kefu Chai)
+* doc: remove mentions about mon_osd_min_down_reports (`issue#19016 <http://tracker.ceph.com/issues/19016>`_, `pr#13558 <https://github.com/ceph/ceph/pull/13558>`_, Barbora An�incová)
+* doc: remove some non-existent and fix the default value according to … (`pr#15664 <https://github.com/ceph/ceph/pull/15664>`_, Leo Zhang)
+* doc: Remove "splitting" state (`pr#12636 <https://github.com/ceph/ceph/pull/12636>`_, Brad Hubbard)
+* doc: reword mds deactivate docs; add optional fs_name argument (`issue#20607 <http://tracker.ceph.com/issues/20607>`_, `pr#16471 <https://github.com/ceph/ceph/pull/16471>`_, Jan Fajerski)
+* doc: Re-word the warnings about using git subtrees (`pr#14999 <https://github.com/ceph/ceph/pull/14999>`_, J. Eric Ivancich)
+* doc: rgw clarify limitations when creating tenant names (`pr#16418 <https://github.com/ceph/ceph/pull/16418>`_, Abhishek Lekshmanan)
+* doc: rgw: Clean up create subuser parameters (`pr#14335 <https://github.com/ceph/ceph/pull/14335>`_, hrchu)
+* doc: rgw: correct get usage parameter default value (`pr#14372 <https://github.com/ceph/ceph/pull/14372>`_, hrchu)
+* doc: rgw: Get user usage needs to specify user (`pr#14804 <https://github.com/ceph/ceph/pull/14804>`_, hrchu)
+* doc: rgw: make a note abt system users vs normal users (`issue#18889 <http://tracker.ceph.com/issues/18889>`_, `pr#13461 <https://github.com/ceph/ceph/pull/13461>`_, Abhishek Lekshmanan)
+* doc: rgw: note rgw_enable_usage_log option in adminops guide (`pr#14803 <https://github.com/ceph/ceph/pull/14803>`_, hrchu)
+* doc: rgw: remove mention of megabytes for quotas (`pr#14413 <https://github.com/ceph/ceph/pull/14413>`_, Hans van den Bogert)
+* doc: rgw: Rewrite Java swift examples (`pr#14268 <https://github.com/ceph/ceph/pull/14268>`_, Chu, Hua-Rong)
+* doc: rgw: Rewrite the key management (`pr#14384 <https://github.com/ceph/ceph/pull/14384>`_, hrchu)
+* doc: rgw server-side encryption and barbican (`pr#13483 <https://github.com/ceph/ceph/pull/13483>`_, Adam Kupczyk, Casey Bodley)
+* doc: script: build-doc/serve-doc fixes (`pr#14438 <https://github.com/ceph/ceph/pull/14438>`_, Abhishek Lekshmanan)
+* doc: script: ceph-release-notes: use https instead of http (`pr#14103 <https://github.com/ceph/ceph/pull/14103>`_, Kefu Chai)
+* docs: doc/cephfs/troubleshooting: fix broken bullet list (`pr#12894 <https://github.com/ceph/ceph/pull/12894>`_, Dan Mick)
+* docs: doc/dev: add some info about FreeBSD (`pr#14503 <https://github.com/ceph/ceph/pull/14503>`_, Willem Jan Withagen)
+* docs: doc/release-notes: fix ceph-deploy command (`pr#15987 <https://github.com/ceph/ceph/pull/15987>`_, Sage Weil)
+* docs: doc/release-note: update release-note (`pr#15748 <https://github.com/ceph/ceph/pull/15748>`_, liuchang0812)
+* docs: document "osd recovery max single start" setting (`issue#17396 <http://tracker.ceph.com/issues/17396>`_, `pr#15275 <https://github.com/ceph/ceph/pull/15275>`_, Ken Dreyer)
+* docs: mailmap: fix Zhao Chao affiliation (`pr#13413 <https://github.com/ceph/ceph/pull/13413>`_, Zhao Chao)
+* docs: mailmap: Leo Zhang infomation and affiliation (`pr#15145 <https://github.com/ceph/ceph/pull/15145>`_, Leo Zhang)
+* docs: mailmap: Liu Yang affiliation (`pr#13427 <https://github.com/ceph/ceph/pull/13427>`_, LiuYang)
+* docs: mailmap: shiqi affiliation (`pr#14361 <https://github.com/ceph/ceph/pull/14361>`_, shiqi)
+* docs: mailmap: update organization info (`pr#14747 <https://github.com/ceph/ceph/pull/14747>`_, liuchang0812)
+* docs: mailmap: Weibing Zhang mailmap affiliation (`pr#15076 <https://github.com/ceph/ceph/pull/15076>`_, Weibing Zhang)
+* docs: PendingReleaseNotes: mention forced recovery (`pr#16775 <https://github.com/ceph/ceph/pull/16775>`_, Piotr Dałek)
+* docs: Remove contractions from the documentation (`pr#16629 <https://github.com/ceph/ceph/pull/16629>`_, John Wilkins)
+* doc: style fix for doc/cephfs/client-config-ref.rst (`pr#14840 <https://github.com/ceph/ceph/pull/14840>`_, Drunkard Zhang)
+* doc: tools/cephfs: fix cephfs-journal-tool --help (`pr#15614 <https://github.com/ceph/ceph/pull/15614>`_, John Spray)
+* doc: two minor fixes (`pr#14494 <https://github.com/ceph/ceph/pull/14494>`_, Drunkard Zhang)
+* doc: typo fixes on hyperlink/words (`pr#15144 <https://github.com/ceph/ceph/pull/15144>`_, Drunkard Zhang)
+* doc: typo fix in s3_compliance (`pr#12598 <https://github.com/ceph/ceph/pull/12598>`_, LiuYang)
+* doc: typo in hit_set_search_last_n (`pr#14108 <https://github.com/ceph/ceph/pull/14108>`_, Sven Seeberg)
+* doc: Update adminops.rst (`pr#13893 <https://github.com/ceph/ceph/pull/13893>`_, Chu, Hua-Rong)
+* doc: update ceph(8) man page with new sub-commands (`pr#16437 <https://github.com/ceph/ceph/pull/16437>`_, Kefu Chai)
+* doc: Update CephFS disaster recovery documentation (`pr#12370 <https://github.com/ceph/ceph/pull/12370>`_, Wido den Hollander)
+* doc: Update disk thread section to reflect that scrubbing is no longe… (`pr#12621 <https://github.com/ceph/ceph/pull/12621>`_, Nick Fisk)
+* doc: update intro, quick start docs (`pr#16224 <https://github.com/ceph/ceph/pull/16224>`_, Sage Weil)
+* doc: Update keystone.rst (`pr#12717 <https://github.com/ceph/ceph/pull/12717>`_, Chu, Hua-Rong)
+* doc: update links to point to ceph/qa instead of ceph-qa-suite (`pr#13397 <https://github.com/ceph/ceph/pull/13397>`_, Jan Fajerski, Nathan Cutler)
+* doc: Update .organizationmap (`pr#16507 <https://github.com/ceph/ceph/pull/16507>`_, luokexue)
+* doc: update packages mentioned by build-doc and related doc (`pr#14649 <https://github.com/ceph/ceph/pull/14649>`_, Yu Shengzuo)
+* doc: Update sample.ceph.conf (`pr#13751 <https://github.com/ceph/ceph/pull/13751>`_, Saumay Agrawal)
+* doc: update sample explaning "%" operator in test suites (`pr#15511 <https://github.com/ceph/ceph/pull/15511>`_, Kefu Chai)
+* doc: Update some RGW documentation (`pr#15175 <https://github.com/ceph/ceph/pull/15175>`_, Jens Rosenboom)
+* doc: update the pool names created by vstart.sh by default (`pr#16652 <https://github.com/ceph/ceph/pull/16652>`_, Zhu Shangzhong)
+* doc: update the rados namespace docs (`pr#15838 <https://github.com/ceph/ceph/pull/15838>`_, Abhishek Lekshmanan)
+* doc: update the support status of swift static website (`pr#13824 <https://github.com/ceph/ceph/pull/13824>`_, Jing Wenjun)
+* doc: update the usage of 'ceph-deploy purge' (`pr#15080 <https://github.com/ceph/ceph/pull/15080>`_, Yu Shengzuo)
+* doc: update to new ceph fs commands (`pr#13346 <https://github.com/ceph/ceph/pull/13346>`_, Patrick Donnelly)
+* doc: upmap docs; various missing links for release notes (`pr#16637 <https://github.com/ceph/ceph/pull/16637>`_, Sage Weil)
+* doc: use do_cmake.sh instead of `cmake ..` (`pr#15110 <https://github.com/ceph/ceph/pull/15110>`_, Kefu Chai)
+* doc: v12.0.0 release notes (`pr#13281 <https://github.com/ceph/ceph/pull/13281>`_, Abhishek Lekshmanan)
+* doc: v12.0.2 (dev) release notes (`pr#14625 <https://github.com/ceph/ceph/pull/14625>`_, Abhishek Lekshmanan)
+* doc: v12.1.0 release notes notable changes addition again (`pr#15857 <https://github.com/ceph/ceph/pull/15857>`_, Abhishek Lekshmanan)
+* doc: various fixes (`pr#16723 <https://github.com/ceph/ceph/pull/16723>`_, Kefu Chai)
+* doc: vstart: add --help documentation for rgw_num (`pr#13817 <https://github.com/ceph/ceph/pull/13817>`_, Ali Maredia)
+* doc: wip-doc-multisite ports downstream multisite document upstream (`pr#14259 <https://github.com/ceph/ceph/pull/14259>`_, John Wilkins)
+* doc: Wip osd discussion docs (`pr#13344 <https://github.com/ceph/ceph/pull/13344>`_, Greg Farnum)
+* filestore: os/filestore: Exclude BTRFS on FreeBSD (`pr#16171 <https://github.com/ceph/ceph/pull/16171>`_, Willem Jan Withagen)
+* filestore: os/filestore/FileJournal: Fix typo in the comment (`pr#14493 <https://github.com/ceph/ceph/pull/14493>`_, Zhou Zhengping)
+* filestore: os/filestore: use existing variable for same func (`pr#13742 <https://github.com/ceph/ceph/pull/13742>`_, Pan Liu)
+* filestore: os/filestore: when print log, use `__func__` instead of hard code function name (`pr#15261 <https://github.com/ceph/ceph/pull/15261>`_, mychoxin)
+* filestore: os/filestore: zfs add get_name() (`pr#15650 <https://github.com/ceph/ceph/pull/15650>`_, Yanhu Cao)
+    Fix full testing in cephtool/test.sh when used by rados suite
+    @Jing-Scott  updated, addressing @rzarzynski's change request
+* librados: add log channel to rados_monitor_log2 callback (`pr#15926 <https://github.com/ceph/ceph/pull/15926>`_, Sage Weil)
+* librados: add missing implementations for C service daemon API methods (`pr#16543 <https://github.com/ceph/ceph/pull/16543>`_, Jason Dillaman)
+* librados: add override for librados (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13442 <https://github.com/ceph/ceph/pull/13442>`_, liuchang0812)
+* librados: add override in headers (`pr#13775 <https://github.com/ceph/ceph/pull/13775>`_, liuchang0812)
+* librados: asynchronous selfmanaged_snap_create/selfmanaged_snap_remove APIs (`issue#16180 <http://tracker.ceph.com/issues/16180>`_, `pr#12050 <https://github.com/ceph/ceph/pull/12050>`_, Jason Dillaman)
+* librados: do not expose non-public symbols (`pr#13265 <https://github.com/ceph/ceph/pull/13265>`_, Kefu Chai)
+* librados: fix compile errors from simplified aio completions (`pr#12849 <https://github.com/ceph/ceph/pull/12849>`_, xie xingguo)
+* librados: fix rados_pool_list when buf is null (`pr#14859 <https://github.com/ceph/ceph/pull/14859>`_, Sage Weil)
+* librados: redirect balanced reads to acting primary when targeting object isn't recovered (`issue#17968 <http://tracker.ceph.com/issues/17968>`_, `pr#15489 <https://github.com/ceph/ceph/pull/15489>`_, Xuehan Xu)
+* librados: remove legacy object listing API, clean up newer api (`pr#13149 <https://github.com/ceph/ceph/pull/13149>`_, Sage Weil)
+* librados: replace the var name from onack to complete (`pr#13857 <https://github.com/ceph/ceph/pull/13857>`_, Pan Liu)
+* librados: set the flag CEPH_OSD_FLAG_FULL_TRY of Op in the right place (`pr#14193 <https://github.com/ceph/ceph/pull/14193>`_, Pan Liu)
+* librados: use cursor for nobjects listing (`pr#13323 <https://github.com/ceph/ceph/pull/13323>`_, Yehuda Sadeh, Sage Weil)
+* librbd: add compare and write API (`pr#14868 <https://github.com/ceph/ceph/pull/14868>`_, Zhengyong Wang, Jason Dillaman)
+* librbd: add create timestamp metadata for image (`pr#15757 <https://github.com/ceph/ceph/pull/15757>`_, runsisi)
+* librbd: added rbd_flatten_with_progress to API (`issue#15824 <http://tracker.ceph.com/issues/15824>`_, `pr#12905 <https://github.com/ceph/ceph/pull/12905>`_, Ricardo Dias)
+* librbd: add LIBRBD_SUPPORTS_WRITESAME support (`pr#16583 <https://github.com/ceph/ceph/pull/16583>`_, Xiubo Li)
+* librbd: add override keyword in header files (`issue#19012 <http://tracker.ceph.com/issues/19012>`_, `pr#13536 <https://github.com/ceph/ceph/pull/13536>`_, liuchang0812)
+* librbd: add SnapshotNamespace to ImageCtx (`pr#12970 <https://github.com/ceph/ceph/pull/12970>`_, Victor Denisov)
+* librbd: add writesame API (`pr#12645 <https://github.com/ceph/ceph/pull/12645>`_, Mingxin Liu, Gui Hecheng)
+* librbd: allow to open an image without opening the parent image (`issue#18325 <http://tracker.ceph.com/issues/18325>`_, `pr#12885 <https://github.com/ceph/ceph/pull/12885>`_, Ricardo Dias)
+* librbd: asynchronous clone state machine (`pr#12041 <https://github.com/ceph/ceph/pull/12041>`_, Dongsheng Yang)
+* librbd: asynchronous image removal state machine (`pr#12102 <https://github.com/ceph/ceph/pull/12102>`_, Dongsheng Yang, Venky Shankar)
+* librbd: avoid possible recursive lock when racing acquire lock (`issue#17447 <http://tracker.ceph.com/issues/17447>`_, `pr#12991 <https://github.com/ceph/ceph/pull/12991>`_, Jason Dillaman)
+* librbd: changed the return type of ImageRequestWQ::discard() (`issue#18511 <http://tracker.ceph.com/issues/18511>`_, `pr#14032 <https://github.com/ceph/ceph/pull/14032>`_, Jos Collin)
+* librbd: cleanup logging code under librbd/io (`pr#14975 <https://github.com/ceph/ceph/pull/14975>`_, runsisi)
+* librbd: corrected resize RPC message backwards compatibility (`issue#19636 <http://tracker.ceph.com/issues/19636>`_, `pr#14615 <https://github.com/ceph/ceph/pull/14615>`_, Jason Dillaman)
+* librbd: create fewer empty objects during copyup (`issue#15028 <http://tracker.ceph.com/issues/15028>`_, `pr#12326 <https://github.com/ceph/ceph/pull/12326>`_, Douglas Fuller, Venky Shankar)
+* librbd: deferred image deletion (`issue#18481 <http://tracker.ceph.com/issues/18481>`_, `pr#13105 <https://github.com/ceph/ceph/pull/13105>`_, Ricardo Dias)
+* librbd: delay mirror registration when creating clones (`issue#17993 <http://tracker.ceph.com/issues/17993>`_, `pr#12839 <https://github.com/ceph/ceph/pull/12839>`_, Jason Dillaman)
+* librbd: discard related IO should skip op if object non-existent (`issue#19962 <http://tracker.ceph.com/issues/19962>`_, `pr#15239 <https://github.com/ceph/ceph/pull/15239>`_, Mykola Golub)
+* librbd: do not instantiate templates while building tests (`issue#18938 <http://tracker.ceph.com/issues/18938>`_, `pr#14891 <https://github.com/ceph/ceph/pull/14891>`_, Kefu Chai)
+* librbd: do not raise an error if trash list returns -ENOENT (`pr#15085 <https://github.com/ceph/ceph/pull/15085>`_, runsisi)
+* librbd: don't continue to remove an image w/ incompatible features (`issue#18315 <http://tracker.ceph.com/issues/18315>`_, `pr#12638 <https://github.com/ceph/ceph/pull/12638>`_, Dongsheng Yang)
+* librbd: eliminate compiler warnings (`pr#13729 <https://github.com/ceph/ceph/pull/13729>`_, Jason Dillaman)
+* librbd: fail IO request when exclusive lock cannot be obtained (`pr#15860 <https://github.com/ceph/ceph/pull/15860>`_, Jason Dillaman)
+* librbd: filter expected error codes from is_exclusive_lock_owner (`issue#20182 <http://tracker.ceph.com/issues/20182>`_, `pr#15483 <https://github.com/ceph/ceph/pull/15483>`_, Jason Dillaman)
+* librbd: fix clang compilation error (`issue#19260 <http://tracker.ceph.com/issues/19260>`_, `pr#13926 <https://github.com/ceph/ceph/pull/13926>`_, Mykola Golub)
+* librbd: fixed initializer list ordering (`pr#13042 <https://github.com/ceph/ceph/pull/13042>`_, Jason Dillaman)
+* librbd: fix issues with image removal state machine (`pr#15734 <https://github.com/ceph/ceph/pull/15734>`_, Jason Dillaman)
+* librbd: fix rbd_metadata_list and rbd_metadata_get (`issue#19588 <http://tracker.ceph.com/issues/19588>`_, `pr#14471 <https://github.com/ceph/ceph/pull/14471>`_, Mykola Golub)
+* librbd: fix segfault on EOPNOTSUPP returned while fetching snapshot timestamp (`issue#18839 <http://tracker.ceph.com/issues/18839>`_, `pr#13287 <https://github.com/ceph/ceph/pull/13287>`_, Gui Hecheng)
+* librbd: fix valgrind errors and ensure tests detect future leaks (`pr#15415 <https://github.com/ceph/ceph/pull/15415>`_, Jason Dillaman)
+* librbd: fix valid coverity warnings (`pr#14023 <https://github.com/ceph/ceph/pull/14023>`_, Jason Dillaman)
+* librbd: image create validates that pool supports overwrites (`issue#19081 <http://tracker.ceph.com/issues/19081>`_, `pr#13986 <https://github.com/ceph/ceph/pull/13986>`_, Jason Dillaman)
+* librbd: image-extent cache needs to clip out-of-bounds read buffers (`pr#13679 <https://github.com/ceph/ceph/pull/13679>`_, Jason Dillaman)
+* librbd: Include WorkQueue.h since we use it (`issue#18862 <http://tracker.ceph.com/issues/18862>`_, `pr#13322 <https://github.com/ceph/ceph/pull/13322>`_, Boris Ranto)
+* librbd: initialize diff parent overlap to zero (`pr#13077 <https://github.com/ceph/ceph/pull/13077>`_, Gu Zhongyan)
+* librbd: introduce new constants for tracking max block name prefix (`issue#18653 <http://tracker.ceph.com/issues/18653>`_, `pr#13141 <https://github.com/ceph/ceph/pull/13141>`_, Jason Dillaman)
+* librbd: is_exclusive_lock_owner API should ping OSD (`issue#19287 <http://tracker.ceph.com/issues/19287>`_, `pr#14003 <https://github.com/ceph/ceph/pull/14003>`_, Jason Dillaman)
+* librbd: managed lock refactoring (`pr#12922 <https://github.com/ceph/ceph/pull/12922>`_, Mykola Golub)
+* librbd: metadata_set API operation should not change global config setting (`issue#18465 <http://tracker.ceph.com/issues/18465>`_, `pr#12843 <https://github.com/ceph/ceph/pull/12843>`_, Mykola Golub)
+* librbd: minor fixes for image trash move (`pr#14834 <https://github.com/ceph/ceph/pull/14834>`_, runsisi)
+* librbd: new API method to force break a peer's exclusive lock (`issue#18429 <http://tracker.ceph.com/issues/18429>`_, `issue#16988 <http://tracker.ceph.com/issues/16988>`_, `issue#18327 <http://tracker.ceph.com/issues/18327>`_, `pr#12639 <https://github.com/ceph/ceph/pull/12639>`_, Jason Dillaman)
+* librbd: Notifier::notify API improvement (`pr#14072 <https://github.com/ceph/ceph/pull/14072>`_, Mykola Golub)
+* librbd: optimize copy-up to add hints only once to object op (`issue#19875 <http://tracker.ceph.com/issues/19875>`_, `pr#15037 <https://github.com/ceph/ceph/pull/15037>`_, Mykola Golub)
+* librbd: pass an uint64_t to clip_io() as the third param (`issue#18938 <http://tracker.ceph.com/issues/18938>`_, `pr#14159 <https://github.com/ceph/ceph/pull/14159>`_, Kefu Chai)
+* librbd: permit removal of image being bootstrapped by rbd-mirror (`issue#16555 <http://tracker.ceph.com/issues/16555>`_, `pr#12549 <https://github.com/ceph/ceph/pull/12549>`_, Mykola Golub)
+* librbd: possible deadlock with flush if refresh in-progress (`issue#18419 <http://tracker.ceph.com/issues/18419>`_, `pr#12838 <https://github.com/ceph/ceph/pull/12838>`_, Jason Dillaman)
+* librbd: potential read IO hang when image is flattened (`issue#19832 <http://tracker.ceph.com/issues/19832>`_, `pr#15234 <https://github.com/ceph/ceph/pull/15234>`_, Jason Dillaman)
+* librbd: potential use of uninitialised value in ImageWatcher (`pr#14091 <https://github.com/ceph/ceph/pull/14091>`_, Mykola Golub)
+* librbd: prevent self-blacklisting during break lock (`issue#18666 <http://tracker.ceph.com/issues/18666>`_, `pr#13110 <https://github.com/ceph/ceph/pull/13110>`_, Jason Dillaman)
+* librbd: race initializing exclusive lock and configuring IO path (`pr#13086 <https://github.com/ceph/ceph/pull/13086>`_, Jason Dillaman)
+* librbd: random unit test failures due to shut down race (`issue#19389 <http://tracker.ceph.com/issues/19389>`_, `pr#14166 <https://github.com/ceph/ceph/pull/14166>`_, Jason Dillaman)
+* librbd: rbd ack cleanup (`pr#13791 <https://github.com/ceph/ceph/pull/13791>`_, runsisi)
+* librbd: reacquire lock should update lock owner client id (`issue#19929 <http://tracker.ceph.com/issues/19929>`_, `pr#15093 <https://github.com/ceph/ceph/pull/15093>`_, Jason Dillaman)
+* librbd: reduce potential of erroneous blacklisting on image close (`issue#19970 <http://tracker.ceph.com/issues/19970>`_, `pr#15162 <https://github.com/ceph/ceph/pull/15162>`_, Jason Dillaman)
+* librbd: refactor exclusive lock support into generic managed lock (`issue#17016 <http://tracker.ceph.com/issues/17016>`_, `pr#12846 <https://github.com/ceph/ceph/pull/12846>`_, Ricardo Dias, Jason Dillaman)
+* librbd: relax "is parent mirrored" check when enabling mirroring for pool (`issue#19130 <http://tracker.ceph.com/issues/19130>`_, `pr#13752 <https://github.com/ceph/ceph/pull/13752>`_, Mykola Golub)
+* librbd: remove redundant check for image id emptiness (`pr#14830 <https://github.com/ceph/ceph/pull/14830>`_, runsisi)
+* librbd: remove unnecessary dependencies of ManagedLock (`pr#12982 <https://github.com/ceph/ceph/pull/12982>`_, Jason Dillaman)
+* librbd: remove unused rbd_image_options_t ostream operator (`pr#15443 <https://github.com/ceph/ceph/pull/15443>`_, Mykola Golub)
+* librbd: resolve static analyser warnings (`pr#12863 <https://github.com/ceph/ceph/pull/12863>`_, Jason Dillaman)
+* librbd: scatter/gather support for the C API (`issue#13025 <http://tracker.ceph.com/issues/13025>`_, `pr#13447 <https://github.com/ceph/ceph/pull/13447>`_, Jason Dillaman)
+* librbd: silence -Wunused-variable warning (`pr#14953 <https://github.com/ceph/ceph/pull/14953>`_, Kefu Chai)
+* librbd: simplify image open/close semantics (`pr#13701 <https://github.com/ceph/ceph/pull/13701>`_, Jason Dillaman)
+* librbd: support for shared locking in ManagedLock (`pr#12886 <https://github.com/ceph/ceph/pull/12886>`_, Ricardo Dias)
+* librbd: support to list snapshot timestamp (`issue#808 <http://tracker.ceph.com/issues/808>`_, `pr#12817 <https://github.com/ceph/ceph/pull/12817>`_, Pan Liu)
+* librbd: Uninitialized variable used handle_refresh() (`pr#16724 <https://github.com/ceph/ceph/pull/16724>`_, amitkuma)
+* librbd: use 'override' keyword instead of 'virtual' (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13437 <https://github.com/ceph/ceph/pull/13437>`_, liuchang0812)
+* librbd: warning message for mirroring pool option (`issue#18125 <http://tracker.ceph.com/issues/18125>`_, `pr#12319 <https://github.com/ceph/ceph/pull/12319>`_, Gaurav Kumar Garg)
+* log: use one write system call per message (`pr#11955 <https://github.com/ceph/ceph/pull/11955>`_, Patrick Donnelly)
+* mds: add authority check for delay dirfrag split (`issue#18487 <http://tracker.ceph.com/issues/18487>`_, `pr#12994 <https://github.com/ceph/ceph/pull/12994>`_, "Yan, Zheng")
+* mds: add override in headers (`pr#13691 <https://github.com/ceph/ceph/pull/13691>`_, liuchang0812)
+* mds: add override in mds subsystem (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13438 <https://github.com/ceph/ceph/pull/13438>`_, liuchang0812)
+* mds: add perf counters for file system operations (`pr#14938 <https://github.com/ceph/ceph/pull/14938>`_, Michael Sevilla)
+* mds: automate MDS object count tracking (`pr#13591 <https://github.com/ceph/ceph/pull/13591>`_, Patrick Donnelly)
+* mds: bump client_reply debug to match client_req (`pr#14036 <https://github.com/ceph/ceph/pull/14036>`_, Patrick Donnelly)
+* mds: change_attr++ and set ctime for set_vxattr (`issue#19583 <http://tracker.ceph.com/issues/19583>`_, `pr#14726 <https://github.com/ceph/ceph/pull/14726>`_, Patrick Donnelly)
+* mds: change the type of data_pools (`pr#15278 <https://github.com/ceph/ceph/pull/15278>`_, Vicente Cheng)
+* mds: check export pin during replay (`issue#20039 <http://tracker.ceph.com/issues/20039>`_, `pr#15205 <https://github.com/ceph/ceph/pull/15205>`_, Patrick Donnelly)
+* mds: check for errors decoding backtraces (`issue#18311 <http://tracker.ceph.com/issues/18311>`_, `pr#12588 <https://github.com/ceph/ceph/pull/12588>`_, John Spray)
+* mds:  Client syncfs is slow (waits for next MDS tick) (`issue#20129 <http://tracker.ceph.com/issues/20129>`_, `pr#15544 <https://github.com/ceph/ceph/pull/15544>`_, dongdong tao)
+* mds: don't assert on read errors in RecoveryQueue (`issue#19282 <http://tracker.ceph.com/issues/19282>`_, `pr#14017 <https://github.com/ceph/ceph/pull/14017>`_, John Spray)
+* mds: don't modify inode that is not projected (`issue#16768 <http://tracker.ceph.com/issues/16768>`_, `pr#13052 <https://github.com/ceph/ceph/pull/13052>`_, "Yan, Zheng")
+* mds: drop partial entry and adjust write_pos when opening PurgeQueue (`issue#19450 <http://tracker.ceph.com/issues/19450>`_, `pr#14447 <https://github.com/ceph/ceph/pull/14447>`_, "Yan, Zheng")
+* mds: explicitly output error msg for dump cache asok command (`pr#15592 <https://github.com/ceph/ceph/pull/15592>`_, Zhi Zhang)
+* mds: extend 'p' auth cap to cover all vxattr stuff (`issue#19075 <http://tracker.ceph.com/issues/19075>`_, `pr#13628 <https://github.com/ceph/ceph/pull/13628>`_, John Spray)
+* mds: finish clientreplay requests before requesting active state (`issue#18461 <http://tracker.ceph.com/issues/18461>`_, `pr#12852 <https://github.com/ceph/ceph/pull/12852>`_, Yan, Zheng)
+* mds: fix bad iterator dereference reported by coverity (`issue#18830 <http://tracker.ceph.com/issues/18830>`_, `pr#13272 <https://github.com/ceph/ceph/pull/13272>`_, John Spray)
+* mds: fix CDir::merge() for mds_debug_auth_pins (`issue#19946 <http://tracker.ceph.com/issues/19946>`_, `pr#15130 <https://github.com/ceph/ceph/pull/15130>`_, "Yan, Zheng")
+* mds: fix client ID truncation (`pr#15258 <https://github.com/ceph/ceph/pull/15258>`_, Henry Chang)
+* mds: fix handling very fast delete ops (`issue#19245 <http://tracker.ceph.com/issues/19245>`_, `pr#13899 <https://github.com/ceph/ceph/pull/13899>`_, John Spray)
+* mds: fix hangs involving re-entrant calls to journaler (`issue#20165 <http://tracker.ceph.com/issues/20165>`_, `pr#15430 <https://github.com/ceph/ceph/pull/15430>`_, John Spray)
+* mds: fix incorrect assertion in Server::_dir_is_nonempty() (`issue#18578 <http://tracker.ceph.com/issues/18578>`_, `pr#12973 <https://github.com/ceph/ceph/pull/12973>`_, Yan, Zheng)
+* mds: fix IO error handling in SessionMap (`pr#13464 <https://github.com/ceph/ceph/pull/13464>`_, John Spray)
+* mds: fix mantle script to not fail for last rank (`issue#19589 <http://tracker.ceph.com/issues/19589>`_, `pr#14704 <https://github.com/ceph/ceph/pull/14704>`_, Patrick Donnelly)
+* mds: fix mgrc shutdown (`issue#19566 <http://tracker.ceph.com/issues/19566>`_, `pr#14505 <https://github.com/ceph/ceph/pull/14505>`_, John Spray)
+* mds: fix null pointer dereference in Locker::handle_client_caps (`issue#18306 <http://tracker.ceph.com/issues/18306>`_, `pr#12808 <https://github.com/ceph/ceph/pull/12808>`_, Yan, Zheng)
+* mds: fix stray creation/removal notification (`issue#19630 <http://tracker.ceph.com/issues/19630>`_, `pr#14554 <https://github.com/ceph/ceph/pull/14554>`_, "Yan, Zheng")
+* mds: fix use-after-free in Locker::file_update_finish() (`issue#19828 <http://tracker.ceph.com/issues/19828>`_, `pr#14991 <https://github.com/ceph/ceph/pull/14991>`_, "Yan, Zheng")
+* mds: ignore ENOENT on writing backtrace (`issue#19401 <http://tracker.ceph.com/issues/19401>`_, `pr#14207 <https://github.com/ceph/ceph/pull/14207>`_, John Spray)
+* mds: ignore fs full check for CEPH_MDS_OP_SETFILELOCK (`issue#18953 <http://tracker.ceph.com/issues/18953>`_, `pr#13455 <https://github.com/ceph/ceph/pull/13455>`_, "Yan, Zheng")
+* mds: improvements for stray reintegration (`pr#15548 <https://github.com/ceph/ceph/pull/15548>`_, "Yan, Zheng")
+* mds: include advisory `path` field in damage (`issue#18509 <http://tracker.ceph.com/issues/18509>`_, `pr#14104 <https://github.com/ceph/ceph/pull/14104>`_, John Spray)
+* mds: issue new caps when sending reply to client (`issue#19635 <http://tracker.ceph.com/issues/19635>`_, `pr#14743 <https://github.com/ceph/ceph/pull/14743>`_, "Yan, Zheng")
+* mds: limit client writable range increment (`issue#19955 <http://tracker.ceph.com/issues/19955>`_, `pr#15131 <https://github.com/ceph/ceph/pull/15131>`_, "Yan, Zheng")
+* mds: make C_MDSInternalNoop::complete() delete 'this' (`issue#19501 <http://tracker.ceph.com/issues/19501>`_, `pr#14347 <https://github.com/ceph/ceph/pull/14347>`_, "Yan, Zheng")
+* mds: mds perf item 'l_mdl_expos' always behind journaler (`pr#15621 <https://github.com/ceph/ceph/pull/15621>`_, redickwang)
+* mds: miscellaneous fixes (`issue#18646 <http://tracker.ceph.com/issues/18646>`_, `pr#12974 <https://github.com/ceph/ceph/pull/12974>`_, Yan, Zheng, "Yan, Zheng")
+* mds: miscellaneous multimds fixes (`issue#19022 <http://tracker.ceph.com/issues/19022>`_, `pr#13698 <https://github.com/ceph/ceph/pull/13698>`_, "Yan, Zheng")
+* mds: miscellaneous multimds fixes part2 (`pr#15125 <https://github.com/ceph/ceph/pull/15125>`_, "Yan, Zheng")
+* mds: miscellaneous multimds fixes (`pr#14550 <https://github.com/ceph/ceph/pull/14550>`_, "Yan, Zheng")
+* mds: misc multimds fixes (`issue#18717 <http://tracker.ceph.com/issues/18717>`_, `issue#18754 <http://tracker.ceph.com/issues/18754>`_, `pr#13227 <https://github.com/ceph/ceph/pull/13227>`_, "Yan, Zheng")
+* mds: misc multimds fixes part2 (`pr#12794 <https://github.com/ceph/ceph/pull/12794>`_, Yan, Zheng)
+* mds: misc multimds fixes (`pr#12274 <https://github.com/ceph/ceph/pull/12274>`_, Yan, Zheng)
+* mds,mon: Clean issues detected by cppcheck (`pr#13199 <https://github.com/ceph/ceph/pull/13199>`_, Ilya Shipitsin)
+* mds: multimds flock fixes (`pr#15440 <https://github.com/ceph/ceph/pull/15440>`_, "Yan, Zheng")
+* mds: Pass empty string to clear mantle balancer (`issue#20076 <http://tracker.ceph.com/issues/20076>`_, `pr#15282 <https://github.com/ceph/ceph/pull/15282>`_, Zhi Zhang)
+* mds: pretty json from `tell` commands (`pr#14105 <https://github.com/ceph/ceph/pull/14105>`_, John Spray)
+* mds: print rank as int (`issue#19201 <http://tracker.ceph.com/issues/19201>`_, `pr#13816 <https://github.com/ceph/ceph/pull/13816>`_, Patrick Donnelly)
+* mds: propagate error encountered during opening inode by number (`issue#18179 <http://tracker.ceph.com/issues/18179>`_, `pr#12749 <https://github.com/ceph/ceph/pull/12749>`_, Yan, Zheng)
+* mds: properly create aux subtrees for pinned directory (`issue#20083 <http://tracker.ceph.com/issues/20083>`_, `pr#15300 <https://github.com/ceph/ceph/pull/15300>`_, "Yan, Zheng")
+* mds: relocate PTRWAITER put near get (`pr#14921 <https://github.com/ceph/ceph/pull/14921>`_, Patrick Donnelly)
+* mds: remove boost::pool usage and use tcmalloc directly (`issue#18425 <http://tracker.ceph.com/issues/18425>`_, `pr#12792 <https://github.com/ceph/ceph/pull/12792>`_, Zhi Zhang)
+* mds: remove legacy "mds tell" command (`issue#19288 <http://tracker.ceph.com/issues/19288>`_, `pr#14015 <https://github.com/ceph/ceph/pull/14015>`_, John Spray)
+* mds: remove "mds log" config option (`issue#18816 <http://tracker.ceph.com/issues/18816>`_, `pr#14652 <https://github.com/ceph/ceph/pull/14652>`_, John Spray)
+* mds: remove some redundant object counters (`pr#13704 <https://github.com/ceph/ceph/pull/13704>`_, Patrick Donnelly)
+* mds: replace C_VoidFn in MDSDaemon with lambdas (`pr#13465 <https://github.com/ceph/ceph/pull/13465>`_, John Spray)
+* mds: Return error message instead of asserting (`pr#14469 <https://github.com/ceph/ceph/pull/14469>`_, Brad Hubbard)
+* mds: save  projected path into inode_t::stray_prior_path (`issue#20340 <http://tracker.ceph.com/issues/20340>`_, `pr#15800 <https://github.com/ceph/ceph/pull/15800>`_, "Yan, Zheng")
+* mds: set ceph-mds name uncond for external tools (`issue#19291 <http://tracker.ceph.com/issues/19291>`_, `pr#14021 <https://github.com/ceph/ceph/pull/14021>`_, Patrick Donnelly)
+* mds: shut down finisher before objecter (`issue#19204 <http://tracker.ceph.com/issues/19204>`_, `pr#13859 <https://github.com/ceph/ceph/pull/13859>`_, John Spray)
+* mds: skip fragment space check for replayed request (`issue#18660 <http://tracker.ceph.com/issues/18660>`_, `pr#13095 <https://github.com/ceph/ceph/pull/13095>`_, "Yan, Zheng")
+* mds: support export pinning on directories (`issue#17834 <http://tracker.ceph.com/issues/17834>`_, `pr#14598 <https://github.com/ceph/ceph/pull/14598>`_, "Yan, Zheng", Patrick Donnelly)
+* mds: try to avoid false positive heartbeat timeouts (`issue#19118 <http://tracker.ceph.com/issues/19118>`_, `pr#13807 <https://github.com/ceph/ceph/pull/13807>`_, John Spray)
+* mds: use debug_mds for most subsys (`issue#19734 <http://tracker.ceph.com/issues/19734>`_, `pr#15052 <https://github.com/ceph/ceph/pull/15052>`_, Patrick Donnelly)
+* mds: use same inode count in health check as in trim (`issue#19395 <http://tracker.ceph.com/issues/19395>`_, `pr#14197 <https://github.com/ceph/ceph/pull/14197>`_, John Spray)
+* mds: warn if insufficient standbys exist (`issue#17604 <http://tracker.ceph.com/issues/17604>`_, `pr#12074 <https://github.com/ceph/ceph/pull/12074>`_, Patrick Donnelly)
+* mgr: add a get_version to the python interface (`pr#13669 <https://github.com/ceph/ceph/pull/13669>`_, John Spray)
+* mgr: add machinery for python modules to send MCommands to daemons (`pr#14920 <https://github.com/ceph/ceph/pull/14920>`_, John Spray)
+* mgr: add mgr allow \* to client.admin (`pr#14864 <https://github.com/ceph/ceph/pull/14864>`_, huanwen ren)
+* mgr: add override in headers (`pr#13772 <https://github.com/ceph/ceph/pull/13772>`_, liuchang0812)
+* mgr: add override in mgr subsystem (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13436 <https://github.com/ceph/ceph/pull/13436>`_, liuchang0812)
+* mgr: add per-DaemonState lock (`pr#16432 <https://github.com/ceph/ceph/pull/16432>`_, Sage Weil)
+* mgr: always free allocated MgrPyModule (`issue#19590 <http://tracker.ceph.com/issues/19590>`_, `pr#14507 <https://github.com/ceph/ceph/pull/14507>`_, Kefu Chai)
+* mgr: ceph-create-keys: update client.admin if it already exists (`issue#19940 <http://tracker.ceph.com/issues/19940>`_, `pr#15112 <https://github.com/ceph/ceph/pull/15112>`_, John Spray)
+* mgr: ceph: introduce "tell x help" subcommand (`issue#19885 <http://tracker.ceph.com/issues/19885>`_, `pr#15111 <https://github.com/ceph/ceph/pull/15111>`_, liuchang0812)
+* mgr: ceph-mgr: Implement new pecan-based rest api (`pr#14457 <https://github.com/ceph/ceph/pull/14457>`_, Boris Ranto)
+* mgr: ceph-mgr: rotate logs on sighup (`issue#19568 <http://tracker.ceph.com/issues/19568>`_, `pr#14437 <https://github.com/ceph/ceph/pull/14437>`_, Dan van der Ster)
+* mgr: clean up daemon start process (`issue#20383 <http://tracker.ceph.com/issues/20383>`_, `pr#16020 <https://github.com/ceph/ceph/pull/16020>`_, John Spray)
+* mgr: clean up fsstatus module (`pr#15925 <https://github.com/ceph/ceph/pull/15925>`_, John Spray)
+* mgr: cleanup, stop clients sending in perf counters (`pr#15578 <https://github.com/ceph/ceph/pull/15578>`_, John Spray)
+* mgr: cluster log message on plugin load error (`pr#15927 <https://github.com/ceph/ceph/pull/15927>`_, John Spray)
+* mgr: dashboard code cleanup (`pr#15577 <https://github.com/ceph/ceph/pull/15577>`_, John Spray)
+* mgr: dashboard GUI module (`pr#14946 <https://github.com/ceph/ceph/pull/14946>`_, John Spray, Dan Mick)
+* mgr: dashboard improvements (`pr#16043 <https://github.com/ceph/ceph/pull/16043>`_, John Spray)
+* mgr: do shutdown using finisher so we can do it in the right order (`issue#19743 <http://tracker.ceph.com/issues/19743>`_, `pr#14835 <https://github.com/ceph/ceph/pull/14835>`_, Kefu Chai)
+* mgr: do the shutdown in the right order (`issue#19813 <http://tracker.ceph.com/issues/19813>`_, `pr#14952 <https://github.com/ceph/ceph/pull/14952>`_, Kefu Chai)
+* mgr: drop repeated log info. and unnecessary write permission (`pr#15896 <https://github.com/ceph/ceph/pull/15896>`_, Yan Jun)
+* mgr: enable ceph_send_command() to send pg command (`pr#15865 <https://github.com/ceph/ceph/pull/15865>`_, Kefu Chai)
+* mgr: fix bugs in init, beacons (`issue#19516 <http://tracker.ceph.com/issues/19516>`_, `issue#19502 <http://tracker.ceph.com/issues/19502>`_, `pr#14374 <https://github.com/ceph/ceph/pull/14374>`_, Sage Weil)
+* mgr: fix crash on missing 'ceph_version' in daemon metadata (fixes #18764) (`issue#18764 <http://tracker.ceph.com/issues/18764>`_, `pr#14129 <https://github.com/ceph/ceph/pull/14129>`_, Tim Serong)
+* mgr: fix crash on set_config from python module with insufficient caps (`issue#19629 <http://tracker.ceph.com/issues/19629>`_, `pr#14706 <https://github.com/ceph/ceph/pull/14706>`_, Tim Serong)
+* mgr: fix lock cycle (`pr#16508 <https://github.com/ceph/ceph/pull/16508>`_, Sage Weil)
+* mgr: fix metadata handling from old MDS daemons (`pr#14161 <https://github.com/ceph/ceph/pull/14161>`_, John Spray)
+* mgr: fix MgrStandby eating messages (`pr#15716 <https://github.com/ceph/ceph/pull/15716>`_, John Spray)
+* mgr: fix python module teardown & add tests (`issue#19407 <http://tracker.ceph.com/issues/19407>`_, `issue#19412 <http://tracker.ceph.com/issues/19412>`_, `issue#19258 <http://tracker.ceph.com/issues/19258>`_, `pr#14232 <https://github.com/ceph/ceph/pull/14232>`_, John Spray)
+* mgr: fix session leak (`issue#19591 <http://tracker.ceph.com/issues/19591>`_, `pr#14720 <https://github.com/ceph/ceph/pull/14720>`_, Sage Weil)
+* mgr: fix several init/re-init bugs (`issue#19491 <http://tracker.ceph.com/issues/19491>`_, `pr#14328 <https://github.com/ceph/ceph/pull/14328>`_, Sage Weil)
+* mgr: handle "module.set_config(.., None)" correctly (`pr#16749 <https://github.com/ceph/ceph/pull/16749>`_, Kefu Chai)
+* mgr: increase debug level for ticks 0 -> 10 (`pr#16301 <https://github.com/ceph/ceph/pull/16301>`_, Dan Mick)
+* mgr: load modules in separate python sub-interpreters (`pr#14971 <https://github.com/ceph/ceph/pull/14971>`_, Tim Serong)
+* mgr: luminous: mgr: add missing call to pick_addresses (`issue#20955 <http://tracker.ceph.com/issues/20955>`_, `issue#21049 <http://tracker.ceph.com/issues/21049>`_, `pr#17173 <https://github.com/ceph/ceph/pull/17173>`_, John Spray)
+* mgr: Make stats period configurable (`issue#17449 <http://tracker.ceph.com/issues/17449>`_, `pr#12732 <https://github.com/ceph/ceph/pull/12732>`_, liuchang0812)
+* mgr: Mark session connections down on shutdown (`issue#19900 <http://tracker.ceph.com/issues/19900>`_, `pr#15192 <https://github.com/ceph/ceph/pull/15192>`_, Brad Hubbard)
+* mgr: mgr/ClusterState: do not mangle PGMap outside of Incremental (`issue#20208 <http://tracker.ceph.com/issues/20208>`_, `pr#16262 <https://github.com/ceph/ceph/pull/16262>`_, Sage Weil)
+* mgr: mgr/DaemonServer.cc: log daemon type string as well as id (`pr#15560 <https://github.com/ceph/ceph/pull/15560>`_, Dan Mick)
+* mgr: mgr/dashboard: add OSD list view (`pr#16373 <https://github.com/ceph/ceph/pull/16373>`_, John Spray)
+* mgr: mgr/dashboard: fix type error in get_rate function (`issue#20276 <http://tracker.ceph.com/issues/20276>`_, `pr#15668 <https://github.com/ceph/ceph/pull/15668>`_, liuchang0812)
+* mgr: mgr/dashboard: load log lines on startup, split out audit log (`pr#15709 <https://github.com/ceph/ceph/pull/15709>`_, John Spray)
+* mgr: mgr/MgrClient: fix reconnect event leak (`issue#19580 <http://tracker.ceph.com/issues/19580>`_, `pr#14431 <https://github.com/ceph/ceph/pull/14431>`_, Sage Weil)
+* mgr: mgr/MgrStandby: prevent use-after-free on just-shut-down Mgr (`issue#19595 <http://tracker.ceph.com/issues/19595>`_, `pr#15297 <https://github.com/ceph/ceph/pull/15297>`_, Sage Weil)
+* mgr: mgr/MgrStandby: respawn when deactivated (`issue#19595 <http://tracker.ceph.com/issues/19595>`_, `issue#19549 <http://tracker.ceph.com/issues/19549>`_, `pr#15557 <https://github.com/ceph/ceph/pull/15557>`_, Sage Weil)
+* mgr: mgr_module interface to report health alerts (`pr#16487 <https://github.com/ceph/ceph/pull/16487>`_, Sage Weil)
+* mgr: mgr,osd:  ceph-mgr --help, unify usage text of other daemons (`pr#15176 <https://github.com/ceph/ceph/pull/15176>`_, Tim Serong)
+* mgr: mgr/PyState: shut up about get_config on nonexistent keys (`pr#16641 <https://github.com/ceph/ceph/pull/16641>`_, Sage Weil)
+* mgr: mgr/status: row has incorrect number of values (`issue#20750 <http://tracker.ceph.com/issues/20750>`_, `pr#16529 <https://github.com/ceph/ceph/pull/16529>`_, liuchang0812)
+* mgr: Misc. bug fixes (`issue#18994 <http://tracker.ceph.com/issues/18994>`_, `pr#14883 <https://github.com/ceph/ceph/pull/14883>`_, John Spray)
+* mgr: mkdir bootstrap-mgr (`pr#14824 <https://github.com/ceph/ceph/pull/14824>`_, huanwen ren)
+* mgr: mon/mgr: add detail error infomation (`pr#16048 <https://github.com/ceph/ceph/pull/16048>`_, Yan Jun)
+* mgr,mon: mgr,mon: debug init and mgrdigest subscriptions (`issue#20633 <http://tracker.ceph.com/issues/20633>`_, `pr#16351 <https://github.com/ceph/ceph/pull/16351>`_, Sage Weil)
+* mgr: mon/MgrMonitor: fix standby addition to mgrmap (`issue#20647 <http://tracker.ceph.com/issues/20647>`_, `pr#16397 <https://github.com/ceph/ceph/pull/16397>`_, Sage Weil)
+* mgr,mon: mon/AuthMonitor: generate bootstrap-mgr key on upgrade (`issue#20666 <http://tracker.ceph.com/issues/20666>`_, `pr#16395 <https://github.com/ceph/ceph/pull/16395>`_, Joao Eduardo Luis)
+* mgr,mon: mon,mgr: extricate PGmap from monitor (`issue#20067 <http://tracker.ceph.com/issues/20067>`_, `issue#20174 <http://tracker.ceph.com/issues/20174>`_, `issue#20050 <http://tracker.ceph.com/issues/20050>`_, `pr#15073 <https://github.com/ceph/ceph/pull/15073>`_, Kefu Chai, Sage Weil, Greg Farnum)
+* mgr,mon: mon/MgrMonitor: add 'mgr dump [epoch]' command (`pr#15158 <https://github.com/ceph/ceph/pull/15158>`_, Sage Weil)
+* mgr,mon: mon/MgrMonitor: only propose if we updated (`pr#14645 <https://github.com/ceph/ceph/pull/14645>`_, Sage Weil)
+* mgr,mon: mon/MgrMonitor: reset mgrdigest timer with new subscription (`issue#20633 <http://tracker.ceph.com/issues/20633>`_, `pr#16582 <https://github.com/ceph/ceph/pull/16582>`_, Sage Weil)
+* mgr,mon: mon,mgr: move reweight-by-\* to mgr (`pr#14404 <https://github.com/ceph/ceph/pull/14404>`_, Kefu Chai)
+* mgr,mon: mon,mgr: print pgmap reports to debug (not cluster) log (`pr#15740 <https://github.com/ceph/ceph/pull/15740>`_, Sage Weil)
+* mgr,mon: mon,mgr: trim osdmap without the help of pgmap (`pr#14504 <https://github.com/ceph/ceph/pull/14504>`_, Kefu Chai)
+* mgr: move 'osd perf' and 'osd blocked-by' to mgr (`pr#14303 <https://github.com/ceph/ceph/pull/14303>`_, Sage Weil)
+* mgr: move "osd pool stats" to mgr (`pr#14365 <https://github.com/ceph/ceph/pull/14365>`_, Kefu Chai)
+* mgr: optimization some judgment and adjust the debug remove value in register_new_pgs (`pr#14046 <https://github.com/ceph/ceph/pull/14046>`_, song baisen)
+* mgr: optimize DaemonStateIndex::cull() a little bit (`pr#14967 <https://github.com/ceph/ceph/pull/14967>`_, Kefu Chai)
+* mgr: pass through cluster log to plugins (`pr#13690 <https://github.com/ceph/ceph/pull/13690>`_, John Spray)
+* mgr: perf schema fns/change notification and Prometheus plugin (`pr#16406 <https://github.com/ceph/ceph/pull/16406>`_, Dan Mick)
+* mgr: print a more helpful error message for when users lack mgr ceph caps (`issue#20296 <http://tracker.ceph.com/issues/20296>`_, `pr#15697 <https://github.com/ceph/ceph/pull/15697>`_, Greg Farnum)
+* mgr,pybind: luminous: mgr/dashboard: fix duplicate images listed on iSCSI status page (`issue#21017 <http://tracker.ceph.com/issues/21017>`_, `pr#17282 <https://github.com/ceph/ceph/pull/17282>`_, Jason Dillaman)
+* mgr: pybind/mgr/dashboard: bind to :: by default (`pr#16223 <https://github.com/ceph/ceph/pull/16223>`_, Sage Weil)
+* mgr: pybind/mgr/dashboard: monkeypatch os.exit to stop cherrypy from taking down mgr (`issue#20216 <http://tracker.ceph.com/issues/20216>`_, `pr#15588 <https://github.com/ceph/ceph/pull/15588>`_, Sage Weil)
+* mgr: pybind/mgr: Delete `rest` module (`pr#15429 <https://github.com/ceph/ceph/pull/15429>`_, John Spray)
+* mgr: pybind/mgr/rest: completely terminate cherrypy in shutdown (`pr#14995 <https://github.com/ceph/ceph/pull/14995>`_, Tim Serong)
+* mgr: pybind/mgr/rest: don't set timezone to Chicago (`pr#14184 <https://github.com/ceph/ceph/pull/14184>`_, Tim Serong)
+* mgr: pybind/mgr/restful: improve cert handling; work with vstart (`pr#15405 <https://github.com/ceph/ceph/pull/15405>`_, Sage Weil)
+* mgr: pybind/mgr/zabbix: fix health in non-compat mode (`issue#20767 <http://tracker.ceph.com/issues/20767>`_, `pr#16580 <https://github.com/ceph/ceph/pull/16580>`_, Sage Weil)
+* mgr,pybind,rbd: mgr/dashboard: show rbd image features (`pr#16468 <https://github.com/ceph/ceph/pull/16468>`_, Yanhu Cao)
+* mgr: raise python exception on failure in send_command() (`pr#15704 <https://github.com/ceph/ceph/pull/15704>`_, Kefu Chai)
+* mgr,rbd: mgr/dashboard: RBD iSCSI daemon status page (`pr#16547 <https://github.com/ceph/ceph/pull/16547>`_, Jason Dillaman)
+* mgr,rbd: mgr/dashboard: rbd mirroring status page (`pr#16360 <https://github.com/ceph/ceph/pull/16360>`_, Jason Dillaman)
+* mgr,rbd: pybind/mgr/dashboard: initial block integration (`pr#15521 <https://github.com/ceph/ceph/pull/15521>`_, Jason Dillaman)
+* mgr: redirect python stdout,stderr to ceph log (`pr#14189 <https://github.com/ceph/ceph/pull/14189>`_, Kefu Chai, Tim Serong, Dan Mick)
+* mgr: release allocated PyString (`pr#14716 <https://github.com/ceph/ceph/pull/14716>`_, Kefu Chai)
+* mgr: remove default cert; disable both restful and dashboard by default (`pr#15601 <https://github.com/ceph/ceph/pull/15601>`_, Boris Ranto, Sage Weil)
+* mgr: remove non-existent MDS daemons from FSMap (`issue#17453 <http://tracker.ceph.com/issues/17453>`_, `pr#14937 <https://github.com/ceph/ceph/pull/14937>`_, Spandan Kumar Sahu)
+* mgr: remove unused function declarations (`pr#14366 <https://github.com/ceph/ceph/pull/14366>`_, Wei Jin)
+* mgr: rm nonused main function (`pr#14313 <https://github.com/ceph/ceph/pull/14313>`_, Wei Jin)
+* mgr: shutdown py_modules in Mgr::shutdown() (`issue#19258 <http://tracker.ceph.com/issues/19258>`_, `pr#14078 <https://github.com/ceph/ceph/pull/14078>`_, Kefu Chai)
+* mgr,tests: qa/suites: move mgr tests into rados suite (`pr#14687 <https://github.com/ceph/ceph/pull/14687>`_, John Spray)
+* mgr,tests: qa/upgrade/jewel-x/point-to-point: add a mgr during final upgrade (`pr#15637 <https://github.com/ceph/ceph/pull/15637>`_, Sage Weil)
+* mgr: use unique_ptr for MgrStandby::active_mgr (`pr#13667 <https://github.com/ceph/ceph/pull/13667>`_, John Spray)
+* mgr: various cleanups (`pr#14802 <https://github.com/ceph/ceph/pull/14802>`_, Kefu Chai)
+* mgr: vstart.sh: fix mgr vs restful command startup race (`pr#16564 <https://github.com/ceph/ceph/pull/16564>`_, Sage Weil)
+* mgr: Zabbix monitoring module (`pr#16019 <https://github.com/ceph/ceph/pull/16019>`_, Wido den Hollander)
+* misc: fix code typos in header files (`pr#12716 <https://github.com/ceph/ceph/pull/12716>`_, Xianxia Xiao)
+* misc: kill clang warnings (`pr#14549 <https://github.com/ceph/ceph/pull/14549>`_, Kefu Chai)
+* misc: Warning Elimination (`pr#14439 <https://github.com/ceph/ceph/pull/14439>`_, Adam C. Emerson)
+* mon: add crush type down health warnings (`pr#14914 <https://github.com/ceph/ceph/pull/14914>`_, Neha Ojha)
+* mon: added bootstrap-rbd auth profile (`pr#16633 <https://github.com/ceph/ceph/pull/16633>`_, Jason Dillaman)
+* mon: add force-create-pg back (`issue#20605 <http://tracker.ceph.com/issues/20605>`_, `pr#16353 <https://github.com/ceph/ceph/pull/16353>`_, Kefu Chai)
+* mon: add mgr metdata commands, and overall 'versions' command for all daemon versions (`pr#16460 <https://github.com/ceph/ceph/pull/16460>`_, Sage Weil)
+* mon: add mon_debug_no_require_luminous (`pr#14490 <https://github.com/ceph/ceph/pull/14490>`_, Sage Weil)
+* mon: Add override for FsNewHandler::handle() (`pr#15331 <https://github.com/ceph/ceph/pull/15331>`_, yonghengdexin735)
+* mon: add override in headers (`pr#13693 <https://github.com/ceph/ceph/pull/13693>`_, liuchang0812)
+* mon: add override in mon subsystem (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13440 <https://github.com/ceph/ceph/pull/13440>`_, liuchang0812)
+* mon: add support public_bind_addr option (`pr#16189 <https://github.com/ceph/ceph/pull/16189>`_, Bassam Tabbara)
+* mon: add warn info for osds were removed from osdmap but still kept in crushmap (`pr#12273 <https://github.com/ceph/ceph/pull/12273>`_, song baisen)
+* mon: a few health fixes (`pr#16415 <https://github.com/ceph/ceph/pull/16415>`_, xie xingguo)
+* mon: a few more upmap (and other) fixes (`pr#16239 <https://github.com/ceph/ceph/pull/16239>`_, xie xingguo)
+* mon: avoid segfault in wait_auth_rotating (`issue#19566 <http://tracker.ceph.com/issues/19566>`_, `pr#14430 <https://github.com/ceph/ceph/pull/14430>`_, John Spray)
+* mon: avoid start election twice when quorum enter (`pr#10150 <https://github.com/ceph/ceph/pull/10150>`_, song baisen)
+* mon: check is_shutdown() in timer callbacks (`issue#19825 <http://tracker.ceph.com/issues/19825>`_, `pr#14919 <https://github.com/ceph/ceph/pull/14919>`_, Kefu Chai)
+* mon: clean up in ceph_mon.cc (`pr#14102 <https://github.com/ceph/ceph/pull/14102>`_, huanwen ren)
+* mon: clean up some osdmon/pgmon interactions (`pr#12403 <https://github.com/ceph/ceph/pull/12403>`_, Sage Weil)
+* mon: cleanups (`pr#15272 <https://github.com/ceph/ceph/pull/15272>`_, Kefu Chai)
+* mon: collect mon metdata as part of the election (`issue#20434 <http://tracker.ceph.com/issues/20434>`_, `pr#16148 <https://github.com/ceph/ceph/pull/16148>`_, Sage Weil)
+* mon: common/config_opts.h: kill mon_pg_create_interval (`pr#13800 <https://github.com/ceph/ceph/pull/13800>`_, xie xingguo)
+* mon: 'config-key put' -> 'config-key set' (`pr#16569 <https://github.com/ceph/ceph/pull/16569>`_, Sage Weil)
+* mon:  crush straw_calc_version value is 0 or 1 not 0 to 2 (`pr#13554 <https://github.com/ceph/ceph/pull/13554>`_, song baisen)
+* mon: debug session feature tracking (`issue#20475 <http://tracker.ceph.com/issues/20475>`_, `pr#16128 <https://github.com/ceph/ceph/pull/16128>`_, Sage Weil)
+* mon: delete unused config opts of mon_sync_fs_threshold (`pr#15676 <https://github.com/ceph/ceph/pull/15676>`_, linbing)
+* mon: delete useless function definition (`pr#15188 <https://github.com/ceph/ceph/pull/15188>`_, shiqi)
+* mon: detect existing fs and duplicate name earlier (`issue#18964 <http://tracker.ceph.com/issues/18964>`_, `pr#13471 <https://github.com/ceph/ceph/pull/13471>`_, Patrick Donnelly)
+* mon: DIVIDE_BY_ZERO in PGMapDigest::dump_pool_stats_full() (`pr#15622 <https://github.com/ceph/ceph/pull/15622>`_, Jos Collin)
+* mon: Division by zero in PGMapDigest::dump_pool_stats_full() (`pr#15901 <https://github.com/ceph/ceph/pull/15901>`_, Jos Collin)
+* mon: do crushtool test with fork and timeout, but w/o exec of crushtool (`issue#19964 <http://tracker.ceph.com/issues/19964>`_, `pr#16025 <https://github.com/ceph/ceph/pull/16025>`_, Sage Weil)
+* mon: do not dereference empty mgr_commands (`pr#16501 <https://github.com/ceph/ceph/pull/16501>`_, Sage Weil)
+* mon: do not prime_pg_temp creating pgs; clean up pg create conditions (`issue#19826 <http://tracker.ceph.com/issues/19826>`_, `pr#14913 <https://github.com/ceph/ceph/pull/14913>`_, Sage Weil)
+* mon: don't call propose_pending in prepare_update() (`issue#19738 <http://tracker.ceph.com/issues/19738>`_, `pr#14711 <https://github.com/ceph/ceph/pull/14711>`_, John Spray)
+* mon: don't kill MDSs unless some beacons are getting through (`issue#19706 <http://tracker.ceph.com/issues/19706>`_, `pr#15308 <https://github.com/ceph/ceph/pull/15308>`_, John Spray)
+* mon: don't prefix mgr summary with epoch number (`pr#15512 <https://github.com/ceph/ceph/pull/15512>`_, John Spray)
+* mon: don't set last_osd_report when the pg stats msg is ignored (`pr#12975 <https://github.com/ceph/ceph/pull/12975>`_, Zhiqiang Wang)
+* mon: drop useless assignment statements (`pr#13958 <https://github.com/ceph/ceph/pull/13958>`_, wangzhengyong)
+* mon: emit cluster log messages on MDS health changes (`issue#19551 <http://tracker.ceph.com/issues/19551>`_, `pr#14398 <https://github.com/ceph/ceph/pull/14398>`_, John Spray)
+* mon: enable luminous monmap feature on full quorum (`pr#13379 <https://github.com/ceph/ceph/pull/13379>`_, Joao Eduardo Luis)
+* mon: extensible output format for health checks (`pr#16701 <https://github.com/ceph/ceph/pull/16701>`_, John Spray)
+* mon: Filter `log last` output by severity and channel (`pr#15924 <https://github.com/ceph/ceph/pull/15924>`_, John Spray)
+* mon: fix accesing pending_fsmap from peon (`issue#20040 <http://tracker.ceph.com/issues/20040>`_, `pr#15213 <https://github.com/ceph/ceph/pull/15213>`_, John Spray)
+* mon: fix a few bugs with the osd health reporting (`pr#15179 <https://github.com/ceph/ceph/pull/15179>`_, Sage Weil)
+* mon: fix a few nits (`pr#12670 <https://github.com/ceph/ceph/pull/12670>`_, Sage Weil)
+* mon: Fix deep_age copy paste error (`pr#16434 <https://github.com/ceph/ceph/pull/16434>`_, Brad Hubbard)
+* mon: Fixed typo in function comment blocks and in other comments (`pr#15304 <https://github.com/ceph/ceph/pull/15304>`_, linbing)
+* mon: Fixed typo in @post of _active() (`pr#15191 <https://github.com/ceph/ceph/pull/15191>`_, Linbing)
+* mon: fix force_pg_create pg stuck in creating bug (`issue#18298 <http://tracker.ceph.com/issues/18298>`_, `pr#12539 <https://github.com/ceph/ceph/pull/12539>`_, Sage Weil)
+* mon: fix hang on deprecated/removed 'pg set_full_ratio' commands (`issue#20600 <http://tracker.ceph.com/issues/20600>`_, `pr#16300 <https://github.com/ceph/ceph/pull/16300>`_, Sage Weil)
+* mon: fix hiding mdsmonitor informative strings (`issue#16709 <http://tracker.ceph.com/issues/16709>`_, `pr#13904 <https://github.com/ceph/ceph/pull/13904>`_, John Spray)
+* mon: fix kvstore type in mon compact command (`pr#15954 <https://github.com/ceph/ceph/pull/15954>`_, liuchang0812)
+* mon: fix legacy health checks in 'ceph status' during upgrade; fix jewel-x upgrade combo (`pr#17176 <https://github.com/ceph/ceph/pull/17176>`_, Sage Weil)
+* mon: fix mon_keyvaluedb application (`pr#15059 <https://github.com/ceph/ceph/pull/15059>`_, Sage Weil)
+* mon: Fix output text and doc (`pr#16367 <https://github.com/ceph/ceph/pull/16367>`_, Yan Jun)
+* mon: fix prime_pg_temp overrun (`issue#19874 <http://tracker.ceph.com/issues/19874>`_, `pr#14979 <https://github.com/ceph/ceph/pull/14979>`_, Sage Weil)
+* mon: Fix status output warning for mon_warn_osd_usage_min_max_delta (`issue#20544 <http://tracker.ceph.com/issues/20544>`_, `pr#16220 <https://github.com/ceph/ceph/pull/16220>`_, David Zafman)
+* mon: fix synchronise pgmap with others (`pr#14418 <https://github.com/ceph/ceph/pull/14418>`_, song baisen, z09440)
+* mon: fix wrongly delete routed pgstats op (`issue#18458 <http://tracker.ceph.com/issues/18458>`_, `pr#12784 <https://github.com/ceph/ceph/pull/12784>`_, Mingxin Liu)
+* mon: fix wrong mon-num counting logic of 'ceph features' command (`pr#17172 <https://github.com/ceph/ceph/pull/17172>`_, xie xingguo)
+* mon: handle cases where store->get() may return error (`issue#19601 <http://tracker.ceph.com/issues/19601>`_, `pr#14678 <https://github.com/ceph/ceph/pull/14678>`_, Jos Collin)
+* mon: include device class in tree view; hide shadow hierarchy (`pr#16016 <https://github.com/ceph/ceph/pull/16016>`_, Sage Weil)
+* mon: Incorrect expression in PGMap::get_health() (`pr#15648 <https://github.com/ceph/ceph/pull/15648>`_, Jos Collin)
+* mon: in output of "ceph osd df tree", display "-", not "0",  for pg amount of a bucket (`pr#13015 <https://github.com/ceph/ceph/pull/13015>`_, Chuanhong Hong)
+* mon: it's no need to get pg action_primary osd twice in pg scrub (`pr#15313 <https://github.com/ceph/ceph/pull/15313>`_, linbing)
+* mon: '\* list' -> '\* ls' (`pr#16423 <https://github.com/ceph/ceph/pull/16423>`_, Sage Weil)
+* mon: load mgr commands at runtime (`pr#16028 <https://github.com/ceph/ceph/pull/16028>`_, John Spray, Sage Weil)
+* mon: logclient: use the seq id of the 1st log entry when resetting session (`issue#19427 <http://tracker.ceph.com/issues/19427>`_, `pr#14927 <https://github.com/ceph/ceph/pull/14927>`_, Kefu Chai)
+* mon: Log errors at startup (`issue#14088 <http://tracker.ceph.com/issues/14088>`_, `pr#15723 <https://github.com/ceph/ceph/pull/15723>`_, Brad Hubbard)
+* mon: luminous: mon/MonCommands: fix copy-and-paste error (`pr#17274 <https://github.com/ceph/ceph/pull/17274>`_, xie xingguo)
+* mon: maintain the "cluster" PerfCounters when using ceph-mgr (`issue#20562 <http://tracker.ceph.com/issues/20562>`_, `pr#16249 <https://github.com/ceph/ceph/pull/16249>`_, Greg Farnum)
+* mon: mark `osd create` as deprecated (`pr#15641 <https://github.com/ceph/ceph/pull/15641>`_, Joao Eduardo Luis)
+* mon: mon,crush: create crush rules using device classes for replicated and ec pools via cli (`pr#16027 <https://github.com/ceph/ceph/pull/16027>`_, Sage Weil)
+* mon: mon/HealthMonitor: avoid sending unnecessary MMonHealthChecks to leader (`pr#16478 <https://github.com/ceph/ceph/pull/16478>`_, xie xingguo)
+* mon: mon/HealthMonitor: trigger a proposal if stat updated (`pr#16477 <https://github.com/ceph/ceph/pull/16477>`_, Kefu Chai)
+* mon: mon/LogMonitor: don't read list's end() for log last (`pr#16376 <https://github.com/ceph/ceph/pull/16376>`_, Joao Eduardo Luis)
+* mon: mon/MDSMonitor: close object section of formatter (`pr#16516 <https://github.com/ceph/ceph/pull/16516>`_, Chang Liu)
+* mon: mon/MDSMonitor: remove create_new_fs from header (`pr#14019 <https://github.com/ceph/ceph/pull/14019>`_, Henrik Korkuc)
+* mon: mon/MgrMonitor: only induce mgr epoch shortly after mkfs (`pr#16356 <https://github.com/ceph/ceph/pull/16356>`_, Sage Weil)
+* mon: mon/MgrMonitor: send digests only if is_active() (`pr#15109 <https://github.com/ceph/ceph/pull/15109>`_, Kefu Chai)
+* mon: mon/MgrStatMonitor: do not crash on luminous dev version upgrades (`pr#16287 <https://github.com/ceph/ceph/pull/16287>`_, Sage Weil)
+* mon: mon/MonClient: cancel pending commands on shutdown (`issue#20051 <http://tracker.ceph.com/issues/20051>`_, `pr#15227 <https://github.com/ceph/ceph/pull/15227>`_, Kefu Chai, Sage Weil)
+* mon: mon/MonClient: make get_mon_log_message() atomic (`issue#19427 <http://tracker.ceph.com/issues/19427>`_, `pr#14422 <https://github.com/ceph/ceph/pull/14422>`_, Kefu Chai)
+* mon: mon/MonClient: random all ranks then pick first_n (`pr#13479 <https://github.com/ceph/ceph/pull/13479>`_, Mingxin Liu)
+* mon: mon/Monitor.h: add const to member function (`pr#10412 <https://github.com/ceph/ceph/pull/10412>`_, Michal Jarzabek)
+* mon: mon/Monitor: recreate mon session if features changed (`issue#20433 <http://tracker.ceph.com/issues/20433>`_, `pr#16230 <https://github.com/ceph/ceph/pull/16230>`_, Joao Eduardo Luis)
+* mon: {mon,osd,mds} {versions,count-metadata} (`pr#15436 <https://github.com/ceph/ceph/pull/15436>`_, Sage Weil)
+* mon: mon/OSDMonitor: a couple of upmap and other fixes (`pr#15917 <https://github.com/ceph/ceph/pull/15917>`_, xie xingguo)
+* mon: mon/OSDMonitor: check get()'s return value instead of bl's length (`pr#14805 <https://github.com/ceph/ceph/pull/14805>`_, Kefu Chai)
+* mon: mon/OSDMonitor: check last_osd_report only when the whole cluster is lu… (`pr#14294 <https://github.com/ceph/ceph/pull/14294>`_, Kefu Chai)
+* mon: mon/OSDMonitor: Clean up: delete extra S signature for plural (`pr#14174 <https://github.com/ceph/ceph/pull/14174>`_, Shinobu Kinjo)
+* mon: mon/OSDMonitor: cleanup pending_created_pgs after done with it (`pr#14898 <https://github.com/ceph/ceph/pull/14898>`_, Kefu Chai)
+* mon: mon/OSDMonitor: do not alter the "created" epoch of a pg (`issue#19787 <http://tracker.ceph.com/issues/19787>`_, `pr#14849 <https://github.com/ceph/ceph/pull/14849>`_, Kefu Chai)
+* mon: mon/OSDMonitor: ensure UP is not set for newly-created OSDs (`issue#20751 <http://tracker.ceph.com/issues/20751>`_, `pr#16534 <https://github.com/ceph/ceph/pull/16534>`_, Sage Weil)
+* mon: mon/OSDMonitor: fix dividing by zero in OSDUtilizationDumper (`pr#13531 <https://github.com/ceph/ceph/pull/13531>`_, Mingxin Liu)
+* mon: mon/OSDMonitor: fix output func name in can_mark_out (`pr#14758 <https://github.com/ceph/ceph/pull/14758>`_, xie xingguo)
+* mon: mon/OSDMonitor: fix process osd failure (`pr#12938 <https://github.com/ceph/ceph/pull/12938>`_, Mingxin Liu)
+* mon: mon/OSDMonitor: guard 'osd crush set-device-class' (`pr#16217 <https://github.com/ceph/ceph/pull/16217>`_, Sage Weil)
+* mon: mon/OSDMonitor: increase last_epoch_clean's lower bound if possible (`pr#14855 <https://github.com/ceph/ceph/pull/14855>`_, Kefu Chai)
+* mon: mon/OSDMonitor: issue pool application related warning (`pr#16520 <https://github.com/ceph/ceph/pull/16520>`_, xie xingguo)
+* mon: mon/OSDMonitor: "osd crush class rename" support (`pr#15875 <https://github.com/ceph/ceph/pull/15875>`_, xie xingguo)
+* mon: mon/OSDMonitor: remove trivial PGMap dependency for 'osd primary-temp' command (`pr#13616 <https://github.com/ceph/ceph/pull/13616>`_, Sage Weil)
+* mon: mon/OSDMonitor: remove zeroed new_state updates (`issue#20751 <http://tracker.ceph.com/issues/20751>`_, `pr#16518 <https://github.com/ceph/ceph/pull/16518>`_, Sage Weil)
+* mon: mon/OSDMonitor: sanity check osd before performing 'osd purge' (`pr#16838 <https://github.com/ceph/ceph/pull/16838>`_, xie xingguo)
+* mon: mon/OSDMonitor: some cleanup for reweight-by-pg (`pr#13462 <https://github.com/ceph/ceph/pull/13462>`_, Haodong Tang)
+* mon: mon/OSDMonitor: spinlock -> std::mutex (`pr#14269 <https://github.com/ceph/ceph/pull/14269>`_, Sage Weil)
+* mon: mon/OSDMonitor: tolerate upgrade from post-kraken dev cluster (`pr#14442 <https://github.com/ceph/ceph/pull/14442>`_, Sage Weil)
+* mon: mon/OSDMonitor: transit creating_pgs from pgmap when upgrading (`issue#19584 <http://tracker.ceph.com/issues/19584>`_, `pr#14551 <https://github.com/ceph/ceph/pull/14551>`_, Kefu Chai)
+* mon: mon/OSDMonitor: two pool opts related fix (`pr#15968 <https://github.com/ceph/ceph/pull/15968>`_, xie xingguo)
+* mon: mon/OSDMonitor: update creating epoch if target osd changed (`issue#19515 <http://tracker.ceph.com/issues/19515>`_, `pr#14386 <https://github.com/ceph/ceph/pull/14386>`_, Kefu Chai)
+* mon: mon/OSDMonitor: update creating_pgs using pending_creatings (`issue#19814 <http://tracker.ceph.com/issues/19814>`_, `pr#14897 <https://github.com/ceph/ceph/pull/14897>`_, Kefu Chai)
+* mon: mon/OSDMonitor: update pg_creatings even the new acting set is empty (`issue#19744 <http://tracker.ceph.com/issues/19744>`_, `pr#14730 <https://github.com/ceph/ceph/pull/14730>`_, Kefu Chai)
+* mon: mon/PaxosService: use `__func__` instead of hard code function name (`pr#15863 <https://github.com/ceph/ceph/pull/15863>`_, Yanhu Cao)
+* mon: mon/PGMap: add up_primary pg number field for pg-dump cmd (`pr#13451 <https://github.com/ceph/ceph/pull/13451>`_, xie xingguo)
+* mon: mon/PGMap.cc: fix "osd_epochs" section of dump_basic (`pr#14996 <https://github.com/ceph/ceph/pull/14996>`_, xie xingguo)
+* mon: mon/PGMap: make si units more readable in PGMap summary (`pr#14185 <https://github.com/ceph/ceph/pull/14185>`_, liuhong)
+* mon: mon/PGMap: remove skewed utilizatoin warning (`issue#20730 <http://tracker.ceph.com/issues/20730>`_, `pr#16461 <https://github.com/ceph/ceph/pull/16461>`_, Sage Weil)
+* mon: mon/PGMap: show %used in formatted output (`issue#20123 <http://tracker.ceph.com/issues/20123>`_, `pr#15387 <https://github.com/ceph/ceph/pull/15387>`_, Joao Eduardo Luis)
+* mon: mon/PGMonitor: clean up min/max span warning (`pr#14611 <https://github.com/ceph/ceph/pull/14611>`_, Sage Weil)
+* mon: mon/PGMonitor: fix description for ceph pg ls (`pr#12807 <https://github.com/ceph/ceph/pull/12807>`_, runsisi)
+* mon: mon/PGMonitor: rm nonused function (`pr#14033 <https://github.com/ceph/ceph/pull/14033>`_, Wei Jin)
+* mon: move 'pg map' to OSDMonitor (`pr#14559 <https://github.com/ceph/ceph/pull/14559>`_, Sage Weil)
+* mon: no delay for single message MSG_ALIVE and MSG_PGTEMP (`pr#12107 <https://github.com/ceph/ceph/pull/12107>`_, yaoning)
+* mon: optracker's initiated_at timestamp should not be NULL (`pr#12826 <https://github.com/ceph/ceph/pull/12826>`_, Mingxin Liu)
+* mon: osd crush set crushmap need sanity check (`issue#19302 <http://tracker.ceph.com/issues/19302>`_, `pr#14029 <https://github.com/ceph/ceph/pull/14029>`_, Loic Dachary)
+* mon: OSDMonitor add check only concern our self cluster command (`pr#10309 <https://github.com/ceph/ceph/pull/10309>`_, song baisen)
+* mon/OSDMonitor: add plain output for "crush class ls-osd" command (`pr#17230 <https://github.com/ceph/ceph/pull/17230>`_, xie xingguo)
+* mon/OSDMonitor: check creating_pgs.last_scan_epoch instead when sending creates (`issue#20785 <http://tracker.ceph.com/issues/20785>`_, `pr#17257 <https://github.com/ceph/ceph/pull/17257>`_, Kefu Chai)
+* mon: OSDMonitor: check mon_max_pool_pg_num when set pool pg_num (`pr#16511 <https://github.com/ceph/ceph/pull/16511>`_, chenhg)
+* mon/OSDMonitor: do not send_pg_creates with stale info (`issue#20785 <http://tracker.ceph.com/issues/20785>`_, `pr#17191 <https://github.com/ceph/ceph/pull/17191>`_, Kefu Chai)
+* mon/OSDMonitor: fix improper input/testing range of crush somke testing (`pr#17232 <https://github.com/ceph/ceph/pull/17232>`_, xie xingguo)
+* mon: osd/PGMonitor: always update pgmap with latest osdmap (`issue#19398 <http://tracker.ceph.com/issues/19398>`_, `pr#14777 <https://github.com/ceph/ceph/pull/14777>`_, Kefu Chai)
+* mon/pgmap: add objects prefix for unfound type (`issue#21127 <http://tracker.ceph.com/issues/21127>`_, `pr#17264 <https://github.com/ceph/ceph/pull/17264>`_, huanwen ren, Sage Weil)
+* mon/PGMap: fix "0 stuck requests are blocked > 4096 sec" warn (`pr#17215 <https://github.com/ceph/ceph/pull/17215>`_, xie xingguo)
+* mon: PGMonitor add check only concern our self cluster command (`pr#9976 <https://github.com/ceph/ceph/pull/9976>`_, song baisen)
+* mon: post-jewel cleanups (`pr#13150 <https://github.com/ceph/ceph/pull/13150>`_, Kefu Chai)
+* mon: prime pg_temp and a few health warning fixes (`pr#16530 <https://github.com/ceph/ceph/pull/16530>`_, xie xingguo)
+* mon: refactor MDSMonitor command handling (`pr#13581 <https://github.com/ceph/ceph/pull/13581>`_, John Spray)
+* mon: Removed unnecessary function declaration in MDSMonitor.h (`pr#15374 <https://github.com/ceph/ceph/pull/15374>`_, yonghengdexin735)
+* mon: remove the redudant jugement in paxosservice is_writeable function (`pr#10240 <https://github.com/ceph/ceph/pull/10240>`_, song baisen)
+* mon: remove unnecessary function declaration (`pr#13762 <https://github.com/ceph/ceph/pull/13762>`_, liuchang0812)
+* mon: replace osds with `osd destroy` and `osd new` (`pr#14074 <https://github.com/ceph/ceph/pull/14074>`_, Joao Eduardo Luis, Sage Weil)
+* mon: restructure prime_pg_temp around a full pg mapping calculated on multiple CPUs (`pr#13207 <https://github.com/ceph/ceph/pull/13207>`_, Sage Weil)
+* mon: revamp health check/warning system (`pr#15643 <https://github.com/ceph/ceph/pull/15643>`_, John Spray, Sage Weil)
+* mon: revise "ceph status" output (`pr#15396 <https://github.com/ceph/ceph/pull/15396>`_, John Spray)
+* mon: show class in 'osd crush tree' output; sort output (`pr#16740 <https://github.com/ceph/ceph/pull/16740>`_, Sage Weil)
+* mon: show destroyed status in tree view; do not auto-out destroyed osds (`pr#16446 <https://github.com/ceph/ceph/pull/16446>`_, xie xingguo)
+* mon: show inactive % in ceph status (`pr#14810 <https://github.com/ceph/ceph/pull/14810>`_, Sage Weil)
+* mon: show io status quickly if no update in a long period (`pr#14176 <https://github.com/ceph/ceph/pull/14176>`_, Mingxin Liu)
+* mon: show the leader info on mon stat command (`pr#14178 <https://github.com/ceph/ceph/pull/14178>`_, song baisen)
+* mon: skip crush smoke test when running under valgrind (`issue#20602 <http://tracker.ceph.com/issues/20602>`_, `pr#16346 <https://github.com/ceph/ceph/pull/16346>`_, Sage Weil)
+* mon: smooth io/recovery stats over longer period (`pr#13249 <https://github.com/ceph/ceph/pull/13249>`_, Sage Weil)
+* mon: stop issuing not-[deep]-scrubbed warnings if disabled (`pr#16465 <https://github.com/ceph/ceph/pull/16465>`_, xie xingguo)
+* mon: support pool application metadata key/values (`pr#15763 <https://github.com/ceph/ceph/pull/15763>`_, Jason Dillaman)
+* mon,tests: qa/suites: add test exercising workunits/mon/auth_caps.sh (`pr#15754 <https://github.com/ceph/ceph/pull/15754>`_, Kefu Chai)
+* mon,tests: test: Initialize pointer msg in MonClientHelper (`pr#16784 <https://github.com/ceph/ceph/pull/16784>`_, amitkuma)
+* mon: Tidy up removal of debug mon features (`pr#14467 <https://github.com/ceph/ceph/pull/14467>`_, Brad Hubbard)
+* mon: track features from connect clients, and use it to gate set-require-min-compat-client (`pr#15371 <https://github.com/ceph/ceph/pull/15371>`_, Sage Weil)
+* mon: trim the creating_pgs after updating it with pgmap (`issue#20067 <http://tracker.ceph.com/issues/20067>`_, `pr#15318 <https://github.com/ceph/ceph/pull/15318>`_, Kefu Chai)
+* mon: update mgrmap when active goes offline (`issue#19407 <http://tracker.ceph.com/issues/19407>`_, `pr#14220 <https://github.com/ceph/ceph/pull/14220>`_, John Spray)
+* mon: Update OSDMon.cc comments (`pr#13750 <https://github.com/ceph/ceph/pull/13750>`_, Saumay Agrawal)
+* mon: warn about using osd new instead of osd create (`pr#17302 <https://github.com/ceph/ceph/pull/17302>`_, Neha Ojha)
+* msg: allow different ms type for cluster network and public network (`pr#12023 <https://github.com/ceph/ceph/pull/12023>`_, Haomai Wang)
+* msg: always set header.version in encode_payload() (`issue#19939 <http://tracker.ceph.com/issues/19939>`_, `pr#16421 <https://github.com/ceph/ceph/pull/16421>`_, Kefu Chai)
+* msg: client bind (`pr#12901 <https://github.com/ceph/ceph/pull/12901>`_, Zengran Zhang, Haomai Wang)
+* msg: do not enable client-side binding by default (`issue#20049 <http://tracker.ceph.com/issues/20049>`_, `pr#15392 <https://github.com/ceph/ceph/pull/15392>`_, Jason Dillaman)
+* msg: don't set msgr addr when disabing client bind (`pr#15243 <https://github.com/ceph/ceph/pull/15243>`_, Haomai Wang)
+* msg: end parameter in entity_addr_t::parse is optional (`pr#13650 <https://github.com/ceph/ceph/pull/13650>`_, Mykola Golub)
+* msg: Fix calls to Messenger::create with new parameter (`pr#13329 <https://github.com/ceph/ceph/pull/13329>`_, Sarit Zubakov)
+* msg: Increase loglevels on some messages (`pr#14707 <https://github.com/ceph/ceph/pull/14707>`_, Willem Jan Withagen)
+* msg: Initialize member variables in Infiniband (`pr#16781 <https://github.com/ceph/ceph/pull/16781>`_, amitkuma)
+* msg: Initializing uninitialized members MMonGetVersion (`pr#16811 <https://github.com/ceph/ceph/pull/16811>`_, amitkuma)
+* msg: Initializing uninitialized members MMonGetVersionReply (`pr#16813 <https://github.com/ceph/ceph/pull/16813>`_, amitkuma)
+* msg: Initializing uninitialized members MMonPaxos (`pr#16814 <https://github.com/ceph/ceph/pull/16814>`_, amitkuma)
+* msg: Initializing uninitialized members MMonProbe (`pr#16815 <https://github.com/ceph/ceph/pull/16815>`_, amitkuma)
+* msg: Initializing uninitialized members module messages (`pr#16817 <https://github.com/ceph/ceph/pull/16817>`_, amitkuma)
+* msg: Initializing uninitialized members MOSDAlive (`pr#16816 <https://github.com/ceph/ceph/pull/16816>`_, amitkuma)
+* msg: make listen backlog an option, increase from 128 to 512 (`issue#20330 <http://tracker.ceph.com/issues/20330>`_, `pr#15743 <https://github.com/ceph/ceph/pull/15743>`_, Haomai Wang)
+* msg: messages: coverity fixes (`pr#13473 <https://github.com/ceph/ceph/pull/13473>`_, Kefu Chai)
+* msg: msg/async: avoid atomic variable overhead (`pr#12809 <https://github.com/ceph/ceph/pull/12809>`_, Wei Jin)
+* msg: msg/async: cleanup code (`pr#13304 <https://github.com/ceph/ceph/pull/13304>`_, Jianpeng Ma)
+* msg: msg/async: cleanups (`pr#12832 <https://github.com/ceph/ceph/pull/12832>`_, Wei Jin)
+* msg: msg/async: fix file description leak in NetHandler (`pr#13271 <https://github.com/ceph/ceph/pull/13271>`_, liuchang0812)
+* msg: msg/async: increase worker reference with local listen table enabled backend (`issue#20390 <http://tracker.ceph.com/issues/20390>`_, `pr#15897 <https://github.com/ceph/ceph/pull/15897>`_, Haomai Wang)
+* msg: msg/async: Lower down the AsyncMessenger's standby warning from debug (`pr#15242 <https://github.com/ceph/ceph/pull/15242>`_, Pan Liu)
+* msg: msg/AsyncMessenger: remove unused method (`pr#10125 <https://github.com/ceph/ceph/pull/10125>`_, Michal Jarzabek)
+* msg: msg/async/net_handler: errno should be stored before calling next function (`pr#14985 <https://github.com/ceph/ceph/pull/14985>`_, Zhou Zhengping)
+* msg: msg/async/rdma: check if exp verbs avail (`pr#13391 <https://github.com/ceph/ceph/pull/13391>`_, Oren Duer, Adir Lev)
+* msg: msg/async/rdma: check if fin message completed (`pr#15624 <https://github.com/ceph/ceph/pull/15624>`_, Alexander Mikheev, Adir Lev)
+* msg: msg/async/rdma: Data path fixes (`pr#15903 <https://github.com/ceph/ceph/pull/15903>`_, Adir lev)
+* msg: msg/async/rdma: Debug prints for ibv* (`pr#14249 <https://github.com/ceph/ceph/pull/14249>`_, Amir Vadai)
+* msg: msg/async/rdma: Device::last_poll_dev must be positive (`pr#14250 <https://github.com/ceph/ceph/pull/14250>`_, Amir Vadai)
+* msg: msg/async/rdma: Fix broken compilation (`pr#13603 <https://github.com/ceph/ceph/pull/13603>`_, Sarit Zubakov)
+* msg: msg/async/rdma: Fix memory leak of OSD (`pr#13101 <https://github.com/ceph/ceph/pull/13101>`_, Sarit Zubakov)
+* msg: msg/async/rdma: fix outstanding queuepair when destruct RDMAStack (`pr#13905 <https://github.com/ceph/ceph/pull/13905>`_, Haomai Wang)
+* msg: msg/async/rdma: fix RoCE v2 deafult value (`pr#12648 <https://github.com/ceph/ceph/pull/12648>`_, Adir Lev, Oren Duer)
+* msg: msg/async/rdma: Fix small memory leaks detected by valgrind (`pr#14288 <https://github.com/ceph/ceph/pull/14288>`_, Amir Vadai)
+* msg: msg/async/rdma: handle buffers after close msg (`pr#15749 <https://github.com/ceph/ceph/pull/15749>`_, DanielBar-On, Alexander Mikheev, Adir Lev)
+* msg: msg/async/rdma: move active_queue_pairs perf counter dec to polling (`pr#13716 <https://github.com/ceph/ceph/pull/13716>`_, DanielBar-On)
+* msg: msg/async/rdma: Print error only on ENOMEM (`pr#13538 <https://github.com/ceph/ceph/pull/13538>`_, Sarit Zubakov)
+* msg: msg/async/rdma: RDMA-CM, Pass specific ConnMgr info in constructor (`pr#14409 <https://github.com/ceph/ceph/pull/14409>`_, Amir Vadai)
+* msg: msg/async/rdma: register buffer as continuous (`pr#15967 <https://github.com/ceph/ceph/pull/15967>`_, Adir Lev)
+* msg: msg/async/rdma: remove assert from ibv_dealloc_pd in ProtectionDomain (`pr#15832 <https://github.com/ceph/ceph/pull/15832>`_, DanielBar-On)
+* msg: msg/async/rdma: update destructor message (`pr#13539 <https://github.com/ceph/ceph/pull/13539>`_, Sarit Zubakov)
+* msg: msg/async/rdma: zero wqe inline (`pr#13392 <https://github.com/ceph/ceph/pull/13392>`_, Adir Lev)
+* msg: msg/async: remove false alert "assert" (`pr#15288 <https://github.com/ceph/ceph/pull/15288>`_, Haomai Wang)
+* msg: msg/async: remove useless close function (`pr#13286 <https://github.com/ceph/ceph/pull/13286>`_, liuchang0812)
+* msg: msg/async: rm nonused thread variable in posixworker (`pr#12777 <https://github.com/ceph/ceph/pull/12777>`_, Wei Jin)
+* msg: msg/async: use auto iterator having more simple code and good performance (`pr#16524 <https://github.com/ceph/ceph/pull/16524>`_, dingdangzhang)
+* msg: msg/Messenger.cc: add std::move (`pr#9760 <https://github.com/ceph/ceph/pull/9760>`_, Michal Jarzabek)
+* msg: msg/MOSDOpReply: fix missing trace decode (`pr#15999 <https://github.com/ceph/ceph/pull/15999>`_, Yan Jun)
+* msg: msg/RDMA: Fix broken compilation due to new argument in net.connect() (`pr#13096 <https://github.com/ceph/ceph/pull/13096>`_, Amir Vadai)
+* msg: msg/simple: Remove dead code in pipe.cc (`issue#12684 <http://tracker.ceph.com/issues/12684>`_, `pr#12601 <https://github.com/ceph/ceph/pull/12601>`_, Rishabh Kumar)
+* msg: msg/simple: use my addr when setting sock priority (`issue#19801 <http://tracker.ceph.com/issues/19801>`_, `pr#14878 <https://github.com/ceph/ceph/pull/14878>`_, Kefu Chai)
+* msg: no need to pass supported features to Messenger::Policy ctor (`pr#13785 <https://github.com/ceph/ceph/pull/13785>`_, Sage Weil)
+* msg: QueueStrategy::wait() joins all threads (`issue#20534 <http://tracker.ceph.com/issues/20534>`_, `pr#16194 <https://github.com/ceph/ceph/pull/16194>`_, Casey Bodley)
+* msg: Remove unused variable perf_counter in RDMAStack (`pr#16783 <https://github.com/ceph/ceph/pull/16783>`_, amitkuma)
+* msg: Revert the change from assert(0)-> ceph_abort() where is not applicable (`pr#12930 <https://github.com/ceph/ceph/pull/12930>`_, Dave Chen)
+* msg: src/msg/async/AsyncConnect.cc: Use of sizeof() on a Pointer Type (`pr#14773 <https://github.com/ceph/ceph/pull/14773>`_, Svyatoslav)
+* msg: src/msg/async: Update fix broken compilation for Posix (`pr#14336 <https://github.com/ceph/ceph/pull/14336>`_, Sarit Zubakov)
+* msg: src/msg/simple/Pipe.cc: Fix the inclusion of '}' (`pr#14843 <https://github.com/ceph/ceph/pull/14843>`_, Willem Jan Withagen)
+* os/bluestore: print leaked extents to debug output (`pr#17303 <https://github.com/ceph/ceph/pull/17303>`_, Sage Weil)
+* osd: add asock command to dump the scrub queue (`issue#17861 <http://tracker.ceph.com/issues/17861>`_, `pr#12728 <https://github.com/ceph/ceph/pull/12728>`_, liuchang0812)
+* osd: add default_device_class to metadata (`pr#16634 <https://github.com/ceph/ceph/pull/16634>`_, Neha Ojha)
+* osd: add dump filter for tracked ops (`pr#16561 <https://github.com/ceph/ceph/pull/16561>`_, Yan Jun)
+* osd: add "heap \*" admin command (`issue#15475 <http://tracker.ceph.com/issues/15475>`_, `pr#13073 <https://github.com/ceph/ceph/pull/13073>`_, Jesse Williamson)
+* osd: adding PerfCounters for backoff throttle (`pr#13017 <https://github.com/ceph/ceph/pull/13017>`_, Chuanhong Wang)
+* osd:  add is_split check before _start_split (`pr#13307 <https://github.com/ceph/ceph/pull/13307>`_, song baisen)
+* osd: add override in headers files (`pr#13962 <https://github.com/ceph/ceph/pull/13962>`_, liuchang0812)
+* osd: add override in osd subsystem (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13439 <https://github.com/ceph/ceph/pull/13439>`_, liuchang0812)
+* osd: Add recovery sleep configuration option for HDDs and SSDs (`pr#16328 <https://github.com/ceph/ceph/pull/16328>`_, Neha Ojha)
+* osd: add snap trim reservation and re-implement osd_snap_trim_sleep (`pr#13594 <https://github.com/ceph/ceph/pull/13594>`_, Samuel Just)
+* osd: adjust osd_min_pg_log_entries (`issue#21026 <http://tracker.ceph.com/issues/21026>`_, `pr#17202 <https://github.com/ceph/ceph/pull/17202>`_, J. Eric Ivancich)
+* osd: allow client throttler to be adjusted on-fly, without restart (`issue#18791 <http://tracker.ceph.com/issues/18791>`_, `pr#13213 <https://github.com/ceph/ceph/pull/13213>`_, Piotr Dałek)
+* osd: bail from _committed_osd_maps inside osd_lock (`issue#20273 <http://tracker.ceph.com/issues/20273>`_, `pr#15710 <https://github.com/ceph/ceph/pull/15710>`_, Sage Weil)
+* osd: Calculate degraded and misplaced more accurately (`issue#18619 <http://tracker.ceph.com/issues/18619>`_, `pr#13031 <https://github.com/ceph/ceph/pull/13031>`_, David Zafman)
+* osd: change a few messages at level 0 and 1; change default level to 1/5 (`pr#13407 <https://github.com/ceph/ceph/pull/13407>`_, Sage Weil)
+* osd: Check for and automatically repair object info soid during scrub (`issue#20471 <http://tracker.ceph.com/issues/20471>`_, `pr#16052 <https://github.com/ceph/ceph/pull/16052>`_, David Zafman)
+* osd: check fsid is normal before osd mkfs (`pr#13898 <https://github.com/ceph/ceph/pull/13898>`_, song baisen)
+* osd: check queue_transaction return value (`pr#15873 <https://github.com/ceph/ceph/pull/15873>`_, zhanglei)
+* osd: Check snapset for validity when selecting authoritative shard (`issue#20186 <http://tracker.ceph.com/issues/20186>`_, `issue#18409 <http://tracker.ceph.com/issues/18409>`_, `pr#15559 <https://github.com/ceph/ceph/pull/15559>`_, David Zafman)
+* osd: Check whether journal is rotational or not (`pr#16614 <https://github.com/ceph/ceph/pull/16614>`_, Neha Ojha)
+* osd: clarify REQUIRE_LUMINOUS error message (`pr#13363 <https://github.com/ceph/ceph/pull/13363>`_, Josh Durgin)
+* osd: clean nonused work queue (`pr#14990 <https://github.com/ceph/ceph/pull/14990>`_, Wei Jin)
+* osd: Cleanup-Updated OSDMap.cc with C++11 style range-for loops (`pr#14381 <https://github.com/ceph/ceph/pull/14381>`_, Jos Collin)
+* osd: cleanup: use string & to avoid unnecessary copy (`pr#12336 <https://github.com/ceph/ceph/pull/12336>`_, Yunchuan Wen)
+* osd: clear_queued_recovery() in on_shutdown() (`issue#20432 <http://tracker.ceph.com/issues/20432>`_, `pr#16093 <https://github.com/ceph/ceph/pull/16093>`_, Kefu Chai)
+* osd: cmpext operator should ignore -ENOENT on read (`pr#16622 <https://github.com/ceph/ceph/pull/16622>`_, Jason Dillaman)
+* osd: combine conditional statements (`pr#16391 <https://github.com/ceph/ceph/pull/16391>`_, Yan Jun)
+* osd: combine unstable stats with info.stats when publish stats to osd (`pr#14060 <https://github.com/ceph/ceph/pull/14060>`_, Mingxin Liu)
+* osd: compact osd feature (`issue#19592 <http://tracker.ceph.com/issues/19592>`_, `pr#16045 <https://github.com/ceph/ceph/pull/16045>`_, liuchang0812)
+* osd: condition object_info_t encoding on required (not up) features (`issue#18644 <http://tracker.ceph.com/issues/18644>`_, `pr#13114 <https://github.com/ceph/ceph/pull/13114>`_, Sage Weil)
+* osd: constify OpRequest::get_req(); fix a few cases of operator<< vs mutated message races (`pr#13545 <https://github.com/ceph/ceph/pull/13545>`_, Sage Weil)
+* osd: correct comment of perfcounter cached_crc in code (`pr#13256 <https://github.com/ceph/ceph/pull/13256>`_, lvshuhua)
+* osd: correct epoch setting of osd boot msg (`pr#12623 <https://github.com/ceph/ceph/pull/12623>`_, Mingxin Liu)
+* osd: correct the func name in execute_ctx() log messages (`pr#13582 <https://github.com/ceph/ceph/pull/13582>`_, Gu Zhongyan)
+* osd: Corrupt objects stop snaptrim and mark pg snaptrim_error (`issue#13837 <http://tracker.ceph.com/issues/13837>`_, `pr#15635 <https://github.com/ceph/ceph/pull/15635>`_, David Zafman)
+* osd: debug con in ms_handle_connect (`pr#13540 <https://github.com/ceph/ceph/pull/13540>`_, Sage Weil)
+* osd: do not send ENXIO on misdirected op by default (`issue#18751 <http://tracker.ceph.com/issues/18751>`_, `pr#13206 <https://github.com/ceph/ceph/pull/13206>`_, Sage Weil)
+* osd: do not send pg_created unless luminous (`issue#20785 <http://tracker.ceph.com/issues/20785>`_, `pr#16677 <https://github.com/ceph/ceph/pull/16677>`_, Kefu Chai)
+* osd: do not try to boot until we've seen the first osdmap (`pr#15732 <https://github.com/ceph/ceph/pull/15732>`_, Sage Weil)
+* osd: do not try to set device class before luminous (`issue#20850 <http://tracker.ceph.com/issues/20850>`_, `pr#16706 <https://github.com/ceph/ceph/pull/16706>`_, Josh Durgin)
+* osd: don't leak pgrefs or reservations in SnapTrimmer (`issue#19931 <http://tracker.ceph.com/issues/19931>`_, `pr#15214 <https://github.com/ceph/ceph/pull/15214>`_, Greg Farnum)
+* osd: don't share osdmap with objecter when preboot (`issue#15025 <http://tracker.ceph.com/issues/15025>`_, `pr#13946 <https://github.com/ceph/ceph/pull/13946>`_, Mingxin Liu)
+* osd: don't use ORDERSNAP for flush; always request/send ondisk ack (`issue#18961 <http://tracker.ceph.com/issues/18961>`_, `pr#13570 <https://github.com/ceph/ceph/pull/13570>`_, Samuel Just)
+* osd: drop support for listing objects at a given snap (`pr#13398 <https://github.com/ceph/ceph/pull/13398>`_, Sage Weil)
+* osd: dump the field name of object watchers  and cleanups (`pr#15946 <https://github.com/ceph/ceph/pull/15946>`_, Yan Jun)
+* osd: EC read handling: don't grab an objectstore error to use as the read error (`pr#16663 <https://github.com/ceph/ceph/pull/16663>`_, David Zafman)
+* osd: eliminate snapdir objects and move clone snaps vector into SnapSet (`pr#13610 <https://github.com/ceph/ceph/pull/13610>`_, Sage Weil)
+* osd: Execute crush_location_hook when configured in ceph.conf (`pr#15951 <https://github.com/ceph/ceph/pull/15951>`_, Wido den Hollander)
+* osd: _exit() intead of exit() for failure injection (`issue#18372 <http://tracker.ceph.com/issues/18372>`_, `pr#12726 <https://github.com/ceph/ceph/pull/12726>`_, Sage Weil)
+* osd: extend OMAP_GETKEYS and GETVALS to include a 'more' output field (`pr#12950 <https://github.com/ceph/ceph/pull/12950>`_, Sage Weil)
+* osd: fall back to failsafe threshold if osdmap doesn't set [near]full (`pr#14004 <https://github.com/ceph/ceph/pull/14004>`_, Sage Weil)
+* osd: faster dispatch (`pr#13343 <https://github.com/ceph/ceph/pull/13343>`_, Sage Weil)
+* osd: fix a couple bugs with persisting the missing set when it contains deletes (`issue#20704 <http://tracker.ceph.com/issues/20704>`_, `pr#16459 <https://github.com/ceph/ceph/pull/16459>`_, Josh Durgin)
+* osd: fix argument-dependent lookup of swap() (`pr#15124 <https://github.com/ceph/ceph/pull/15124>`_, Casey Bodley)
+* osd: fix a signed/unsigned warning in PG (`pr#13922 <https://github.com/ceph/ceph/pull/13922>`_, Greg Farnum)
+* osd: fix comments about pg refs and lock (`pr#14279 <https://github.com/ceph/ceph/pull/14279>`_, tang.jin)
+* osd: fix coverity warning for uninitialized members (`pr#12724 <https://github.com/ceph/ceph/pull/12724>`_, Li Wang)
+* osd: fix func name in log produced by handle_pg_peering_evt() (`pr#13801 <https://github.com/ceph/ceph/pull/13801>`_, xie xingguo)
+* osd: fix occasional MOSDMap leak (`issue#18293 <http://tracker.ceph.com/issues/18293>`_, `pr#14558 <https://github.com/ceph/ceph/pull/14558>`_, Sage Weil)
+* osd: fix OpRequest and tracked op dump information (`pr#16504 <https://github.com/ceph/ceph/pull/16504>`_, Yan Jun)
+* osd: fix past_intervals base case by adding epoch_pool_created to pg_history_t (`issue#19877 <http://tracker.ceph.com/issues/19877>`_, `pr#14989 <https://github.com/ceph/ceph/pull/14989>`_, Sage Weil)
+* osd: fix pg ref leaks when osd shutdown (`issue#20684 <http://tracker.ceph.com/issues/20684>`_, `pr#16408 <https://github.com/ceph/ceph/pull/16408>`_, Yang Honggang)
+* osd: fix some osd beacon bugs (`pr#14274 <https://github.com/ceph/ceph/pull/14274>`_, Sage Weil)
+* osd: fix stat sum update of recovery pushing (`pr#13328 <https://github.com/ceph/ceph/pull/13328>`_, Zhiqiang Wang)
+* osd: fix the setting of soid in sub_op_push (`pr#13353 <https://github.com/ceph/ceph/pull/13353>`_, Zhiqiang Wang)
+* osd: fix typo in comment (`pr#13061 <https://github.com/ceph/ceph/pull/13061>`_, Gu Zhongyan)
+* osd: Fix useless MAX(0, unsigned) to prevent out of wack misplaced (`issue#18718 <http://tracker.ceph.com/issues/18718>`_, `pr#13164 <https://github.com/ceph/ceph/pull/13164>`_, David Zafman)
+* osd: have clients resend ops on pg split (`pr#13235 <https://github.com/ceph/ceph/pull/13235>`_, Sage Weil)
+* osd: hdd vs ssd defaults for osd op thread pool (`pr#15422 <https://github.com/ceph/ceph/pull/15422>`_, Sage Weil)
+* osd: heartbeat with packets large enough to require working jumbo frames (`issue#20087 <http://tracker.ceph.com/issues/20087>`_, `pr#15535 <https://github.com/ceph/ceph/pull/15535>`_, Greg Farnum)
+* osd: Implement asynchronous recovery sleep (`pr#15212 <https://github.com/ceph/ceph/pull/15212>`_, Neha Ojha)
+* osd: Implement asynchronous scrub sleep (`issue#19497 <http://tracker.ceph.com/issues/19497>`_, `pr#14886 <https://github.com/ceph/ceph/pull/14886>`_, Brad Hubbard)
+* osd: Implement peering state timing (`pr#14627 <https://github.com/ceph/ceph/pull/14627>`_, Brad Hubbard)
+* osd: improve error message when FileStore op fails due to EPERM (`issue#18037 <http://tracker.ceph.com/issues/18037>`_, `pr#12181 <https://github.com/ceph/ceph/pull/12181>`_, Nathan Cutler)
+* osd: initialize waiting_for_pg_osdmap on startup (`issue#20748 <http://tracker.ceph.com/issues/20748>`_, `pr#16535 <https://github.com/ceph/ceph/pull/16535>`_, Sage Weil)
+* osd: kill all remaining MOSDSubOp users (`pr#13401 <https://github.com/ceph/ceph/pull/13401>`_, Sage Weil)
+* osd: kill sortbitwise (`pr#13321 <https://github.com/ceph/ceph/pull/13321>`_, Sage Weil)
+* osd: Log audit (`pr#16281 <https://github.com/ceph/ceph/pull/16281>`_, Brad Hubbard)
+* osd: make ec overwrites ready to use (`pr#14496 <https://github.com/ceph/ceph/pull/14496>`_, Josh Durgin)
+* osd: moved OpFinisher logic from OSDOp to OpContext (`issue#20783 <http://tracker.ceph.com/issues/20783>`_, `pr#16617 <https://github.com/ceph/ceph/pull/16617>`_, Jason Dillaman)
+* osd: Move scrub sleep timer to osdservice (`issue#19986 <http://tracker.ceph.com/issues/19986>`_, `pr#15217 <https://github.com/ceph/ceph/pull/15217>`_, Brad Hubbard)
+* osd: never send rados ack (only commit) (`pr#12451 <https://github.com/ceph/ceph/pull/12451>`_, Sage Weil)
+* osd: new op for calculating an extent checksum (`pr#14256 <https://github.com/ceph/ceph/pull/14256>`_, Jason Dillaman)
+* osd: objclass sdk (`pr#14723 <https://github.com/ceph/ceph/pull/14723>`_, Neha Ojha)
+* osd: Object level shard errors are tracked and used if no auth available (`issue#20089 <http://tracker.ceph.com/issues/20089>`_, `pr#15397 <https://github.com/ceph/ceph/pull/15397>`_, David Zafman)
+* osd: On EIO from read recover the primary replica from another copy (`issue#18165 <http://tracker.ceph.com/issues/18165>`_, `pr#14760 <https://github.com/ceph/ceph/pull/14760>`_, David Zafman)
+* osd: osdc/ObjectCacher: use state instead of get_state() (`pr#12544 <https://github.com/ceph/ceph/pull/12544>`_, huangjun)
+* osd: osdc/Objecter: more constness (`pr#14819 <https://github.com/ceph/ceph/pull/14819>`_, Kefu Chai)
+* osd: osdc: silence warning from `-Wsign-compare` (`pr#14729 <https://github.com/ceph/ceph/pull/14729>`_, Jos Collin)
+* osd: osd does not using MPing Messages,do not include unused include (`pr#15833 <https://github.com/ceph/ceph/pull/15833>`_, linbing)
+* osd: osd/OSDMap.cc: check if osd is out in subtree_type_is_down (`issue#19989 <http://tracker.ceph.com/issues/19989>`_, `pr#15250 <https://github.com/ceph/ceph/pull/15250>`_, Neha Ojha)
+* osd: osd/OSDMap: require OSD features only of OSDs (`issue#18831 <http://tracker.ceph.com/issues/18831>`_, `pr#13275 <https://github.com/ceph/ceph/pull/13275>`_, Ilya Dryomov)
+* osd: osd/PrimaryLogPG: nullptr not NULL (`pr#13973 <https://github.com/ceph/ceph/pull/13973>`_, Shinobu Kinjo)
+* osd: 'osd tree in|out|up|down' to filter tree results (`pr#15294 <https://github.com/ceph/ceph/pull/15294>`_, Sage Weil)
+* osd: os/kstore: some error handling (`pr#13960 <https://github.com/ceph/ceph/pull/13960>`_, wangzhengyong)
+* osd/PGBackend: delete reply if fails to complete delete request (`issue#20913 <http://tracker.ceph.com/issues/20913>`_, `pr#17233 <https://github.com/ceph/ceph/pull/17233>`_, Kefu Chai)
+* osd: pg: be more careful with locking around forced pg recovery (`issue#20808 <http://tracker.ceph.com/issues/20808>`_, `pr#16712 <https://github.com/ceph/ceph/pull/16712>`_, Greg Farnum)
+* osd: pglog trimming fixes (`pr#12882 <https://github.com/ceph/ceph/pull/12882>`_, Zhiqiang Wang)
+* osd: pglog: with config, don't assert in the presence of stale diverg… (`issue#17916 <http://tracker.ceph.com/issues/17916>`_, `pr#14648 <https://github.com/ceph/ceph/pull/14648>`_, Greg Farnum)
+* osd: pg-remap -> pg-upmap (`pr#14556 <https://github.com/ceph/ceph/pull/14556>`_, Sage Weil)
+* osd: populate last_epoch_split during build_initial_pg_history (`issue#20754 <http://tracker.ceph.com/issues/20754>`_, `pr#16519 <https://github.com/ceph/ceph/pull/16519>`_, Sage Weil)
+* osd: Preserve OSDOp information for historic ops (`pr#15265 <https://github.com/ceph/ceph/pull/15265>`_, Guo-Fu Tseng)
+* osd: PrimaryLogPG, PGBackend: complete callback even if interval changes (`issue#20747 <http://tracker.ceph.com/issues/20747>`_, `pr#16536 <https://github.com/ceph/ceph/pull/16536>`_, Josh Durgin)
+* osd: print pg_info_t::purged_snaps as array, not string (`issue#18584 <http://tracker.ceph.com/issues/18584>`_, `pr#14217 <https://github.com/ceph/ceph/pull/14217>`_, liuchang0812)
+* osd: process deletes during recovery instead of peering (`issue#19971 <http://tracker.ceph.com/issues/19971>`_, `pr#15952 <https://github.com/ceph/ceph/pull/15952>`_, Josh Durgin)
+* osd: put osdmap in mempool (`pr#14780 <https://github.com/ceph/ceph/pull/14780>`_, Sage Weil)
+* osd: reduce buffer pinning from EC entries (`pr#15120 <https://github.com/ceph/ceph/pull/15120>`_, Sage Weil)
+* osd: reduce map cache size (`pr#15292 <https://github.com/ceph/ceph/pull/15292>`_, Sage Weil)
+* osd: reduce rados_max_object_size from 100 GB -> 128 MB (`pr#15520 <https://github.com/ceph/ceph/pull/15520>`_, Sage Weil)
+* osd: remove copy-get-classic (`pr#13547 <https://github.com/ceph/ceph/pull/13547>`_, Sage Weil)
+* osd: remove sortbitwise thrashing (`pr#13296 <https://github.com/ceph/ceph/pull/13296>`_, Sage Weil)
+* osd: renamed the new vector name in OSDMap::build_simple_crush_map_from_conf (`pr#14583 <https://github.com/ceph/ceph/pull/14583>`_, Jos Collin)
+* osd: rename osd -> osd_pglog; include pglog-related bufferlists (`pr#15531 <https://github.com/ceph/ceph/pull/15531>`_, Sage Weil)
+* osd: rephrase "wrongly marked me down" clog message (`pr#16365 <https://github.com/ceph/ceph/pull/16365>`_, John Spray)
+* osd: replace object_info_t::operator=() with decode() (`pr#13938 <https://github.com/ceph/ceph/pull/13938>`_, tang.jin)
+* osd: ReplicatedBackend::prep_push() remove redundant variable assignments (`pr#14817 <https://github.com/ceph/ceph/pull/14817>`_, Jin Cai)
+* osd: restart boot process if waiting for luminous mons (`issue#20631 <http://tracker.ceph.com/issues/20631>`_, `pr#16341 <https://github.com/ceph/ceph/pull/16341>`_, Sage Weil)
+* osd: Return correct osd_objectstore in OSD metadata (`issue#18638 <http://tracker.ceph.com/issues/18638>`_, `pr#13072 <https://github.com/ceph/ceph/pull/13072>`_, Wido den Hollander)
+* osd: Return early on shutdown (`issue#19900 <http://tracker.ceph.com/issues/19900>`_, `pr#15345 <https://github.com/ceph/ceph/pull/15345>`_, Brad Hubbard)
+* osd: Reverse order of op_has_sufficient_caps and do_pg_op (`issue#19790 <http://tracker.ceph.com/issues/19790>`_, `pr#15354 <https://github.com/ceph/ceph/pull/15354>`_, Brad Hubbard)
+* osd: sched_scrub() lock pg only if all scrubbing conditions are fulfilled (`pr#14968 <https://github.com/ceph/ceph/pull/14968>`_, Jin Cai)
+* osd: scrub_to specifies clone ver, but transaction include head write… (`issue#20041 <http://tracker.ceph.com/issues/20041>`_, `pr#16404 <https://github.com/ceph/ceph/pull/16404>`_, David Zafman)
+* osd: silence warning from -Wint-in-bool-context (`pr#16744 <https://github.com/ceph/ceph/pull/16744>`_, Jos Collin)
+* osd: simplify past_intervals representation (`pr#14444 <https://github.com/ceph/ceph/pull/14444>`_, Samuel Just, Sage Weil)
+* osd: small clear up and optimize on _recover_now and should_share_map function (`pr#13476 <https://github.com/ceph/ceph/pull/13476>`_, song baisen)
+* osd: stop mgrc earlier in shutdown() (`issue#19638 <http://tracker.ceph.com/issues/19638>`_, `pr#14904 <https://github.com/ceph/ceph/pull/14904>`_, Kefu Chai)
+* osd: stop MgrClient callbacks on shutdown (`issue#19638 <http://tracker.ceph.com/issues/19638>`_, `pr#14896 <https://github.com/ceph/ceph/pull/14896>`_, Sage Weil)
+* osd: strip pglog op name (`pr#14764 <https://github.com/ceph/ceph/pull/14764>`_, liuchang0812)
+* osd: support cmpext operation on EC-backed pools (`pr#15693 <https://github.com/ceph/ceph/pull/15693>`_, Zhengyong Wang, Jason Dillaman)
+* osd: support dumping long ops (`pr#13019 <https://github.com/ceph/ceph/pull/13019>`_, Zhiqiang Wang)
+* osd: switch filestore to default to rocksdb (`pr#14814 <https://github.com/ceph/ceph/pull/14814>`_, Neha Ojha)
+* osd: tag fast dispatch messages with min_epoch (`pr#13681 <https://github.com/ceph/ceph/pull/13681>`_, Sage Weil)
+* osd: take PGRef for recovery sleep wakeup event (`issue#20226 <http://tracker.ceph.com/issues/20226>`_, `pr#15582 <https://github.com/ceph/ceph/pull/15582>`_, Sage Weil)
+* osd: the condition of last epoch <= superblock.newest_map epoch has been check twice (`pr#15590 <https://github.com/ceph/ceph/pull/15590>`_, linbing)
+* osd: the osd should not share map with others when it is in stopping state (`pr#13668 <https://github.com/ceph/ceph/pull/13668>`_, song baisen)
+* osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid miss… (`pr#15891 <https://github.com/ceph/ceph/pull/15891>`_, Ming Lin)
+* osd: use append(bufferlist &) to avoid unnecessary copy (`pr#12272 <https://github.com/ceph/ceph/pull/12272>`_, Yunchuan Wen)
+* osd: use separate waitlist for scrub (`pr#13136 <https://github.com/ceph/ceph/pull/13136>`_, Sage Weil)
+* osd: various changes for preventing internal ENOSPC condition (`issue#16878 <http://tracker.ceph.com/issues/16878>`_, `pr#13425 <https://github.com/ceph/ceph/pull/13425>`_, David Zafman)
+* osd: we know the definite epoch of marking down (`pr#13121 <https://github.com/ceph/ceph/pull/13121>`_, Mingxin Liu)
+* osd: when osd in not in failure_pending, we don't need to get osd inst from osdmap (`pr#15558 <https://github.com/ceph/ceph/pull/15558>`_, linbing)
+* osd: When scrub finds an attr error mark shard inconsistent (`issue#20089 <http://tracker.ceph.com/issues/20089>`_, `pr#15368 <https://github.com/ceph/ceph/pull/15368>`_, David Zafman)
+* osd: zipkin tracing (`pr#14305 <https://github.com/ceph/ceph/pull/14305>`_, Sage Weil, Marios-Evaggelos Kogias, Victor Araujo, Casey Bodley, Andrew Shewmaker, Chendi.Xue)
+* performance: buffer, osd: add missing crc cache miss perf counter (`pr#14957 <https://github.com/ceph/ceph/pull/14957>`_, Piotr Dałek)
+* performance: common/config_opts.h: Lower HDD throttle cost (`pr#15485 <https://github.com/ceph/ceph/pull/15485>`_, Mark Nelson)
+* performance: crc32c: optimize aarch64 crc32c implementation (`pr#12977 <https://github.com/ceph/ceph/pull/12977>`_, wei xiao)
+* performance: denc: add need_contiguous to denc_traits (`pr#15224 <https://github.com/ceph/ceph/pull/15224>`_, Kefu Chai)
+* performance: osd, messenger, librados: lttng oid tracing (`pr#12492 <https://github.com/ceph/ceph/pull/12492>`_, Anjaneya Chagam)
+* performance: osd/PG.cc: loop invariant code motion (`pr#12720 <https://github.com/ceph/ceph/pull/12720>`_, Li Wang)
+* performance: osd/ReplicatedBackend: do not set omap header if it is empty (`pr#12612 <https://github.com/ceph/ceph/pull/12612>`_, fang yuxiang)
+* performance,rgw: rgw_file: permit dirent offset computation (`pr#16275 <https://github.com/ceph/ceph/pull/16275>`_, Matt Benjamin)
+* pybind: better error msg (`pr#14497 <https://github.com/ceph/ceph/pull/14497>`_, Kefu Chai)
+* pybind: cephfs should be built without librados / python-rados (`pr#13431 <https://github.com/ceph/ceph/pull/13431>`_, Kefu Chai)
+* pybind: ceph.in: Check return value when connecting (`pr#16130 <https://github.com/ceph/ceph/pull/16130>`_, Douglas Fuller)
+* pybind: ceph-rest-api: Various REST API fixes (`pr#15910 <https://github.com/ceph/ceph/pull/15910>`_, Wido den Hollander)
+* pybind: conditional compile the linux specific constant (`pr#12198 <https://github.com/ceph/ceph/pull/12198>`_, Kefu Chai)
+* pybind: fix docstring for librbd Python binding (`pr#13977 <https://github.com/ceph/ceph/pull/13977>`_, runsisi)
+* pybind: fix open flags calculation (`issue#19890 <http://tracker.ceph.com/issues/19890>`_, `pr#15018 <https://github.com/ceph/ceph/pull/15018>`_, "Yan, Zheng")
+* pybind: pybind/ceph_argparse: fix empty string check (`issue#20135 <http://tracker.ceph.com/issues/20135>`_, `pr#15500 <https://github.com/ceph/ceph/pull/15500>`_, Sage Weil)
+* pybind: pybind/ceph_daemon.py: fix Termsize.update (`pr#15253 <https://github.com/ceph/ceph/pull/15253>`_, Kefu Chai)
+* pybind: pybind/ceph_daemon: use small chunk for recv (`pr#13804 <https://github.com/ceph/ceph/pull/13804>`_, Xiaoxi Chen)
+* pybind: pybind/mgr/dashboard: fix get kernel_version error (`pr#16094 <https://github.com/ceph/ceph/pull/16094>`_, Peng Zhang)
+* pybind: pybind/mgr/restful: fix typo (`pr#16560 <https://github.com/ceph/ceph/pull/16560>`_, Nick Erdmann)
+* pybind: pybind/mgr/restful: use list to pass hooks to create a `Pecan` instance (`issue#20258 <http://tracker.ceph.com/issues/20258>`_, `pr#15646 <https://github.com/ceph/ceph/pull/15646>`_, Kefu Chai)
+* pybind: pybind/rados: avoid call free() on invalid pointer (`pr#15159 <https://github.com/ceph/ceph/pull/15159>`_, Mingxin Liu)
+* pybind: pybind/rados: use new APIs instead of deprecated ones (`pr#16684 <https://github.com/ceph/ceph/pull/16684>`_, Kefu Chai)
+* pybind,rbd: pybind/rbd: OSError should be picklable (`issue#20223 <http://tracker.ceph.com/issues/20223>`_, `pr#15574 <https://github.com/ceph/ceph/pull/15574>`_, Jason Dillaman)
+* pybind: restore original API for backwards compatibility (`issue#20421 <http://tracker.ceph.com/issues/20421>`_, `pr#15932 <https://github.com/ceph/ceph/pull/15932>`_, Jason Dillaman)
+* pybind: support mon target in pybind (`pr#15409 <https://github.com/ceph/ceph/pull/15409>`_, liuchang0812)
+* qa: fix POOL_APP_NOT_ENABLED warning in krbd:unmap suite (`pr#17192 <https://github.com/ceph/ceph/pull/17192>`_, Ilya Dryomov)
+* rbd: add default note info to size (create and resize) (`pr#15561 <https://github.com/ceph/ceph/pull/15561>`_, Zheng Yin)
+* rbd: add error prompt when input command 'snap set limit' is incomplete (`pr#12945 <https://github.com/ceph/ceph/pull/12945>`_, Tang Jin)
+* rbd: additional validation for 'bench' optional parameters (`pr#12697 <https://github.com/ceph/ceph/pull/12697>`_, Yunchuan Wen)
+* rbd: bench-write should return error if io-size >= 4G (`issue#18422 <http://tracker.ceph.com/issues/18422>`_, `pr#12864 <https://github.com/ceph/ceph/pull/12864>`_, Gaurav Kumar Garg)
+* rbd: cleanup: fix the typo in namespace comment (`pr#12858 <https://github.com/ceph/ceph/pull/12858>`_, Dongsheng Yang)
+* rbd: cleanup: rbd: fix a typo in comment (`pr#14049 <https://github.com/ceph/ceph/pull/14049>`_, Dongsheng Yang)
+* rbd: cls_rbd: default initialize snapshot namespace for legacy clients (`issue#19413 <http://tracker.ceph.com/issues/19413>`_, `pr#14903 <https://github.com/ceph/ceph/pull/14903>`_, Jason Dillaman)
+* rbd: cls/rbd: silence warning from -Wunused-variable (`pr#16670 <https://github.com/ceph/ceph/pull/16670>`_, Yan Jun)
+* rbd: cls/rbd: trash_list should be iterable (`issue#20643 <http://tracker.ceph.com/issues/20643>`_, `pr#16372 <https://github.com/ceph/ceph/pull/16372>`_, Jason Dillaman)
+* rbd: common/bit_vector: utilize deep-copy during data decode (`issue#19863 <http://tracker.ceph.com/issues/19863>`_, `pr#15017 <https://github.com/ceph/ceph/pull/15017>`_, Jason Dillaman)
+* rbd: correct coverity warnings (`pr#12954 <https://github.com/ceph/ceph/pull/12954>`_, Jason Dillaman)
+* rbd: correct issues with image importing (`pr#14401 <https://github.com/ceph/ceph/pull/14401>`_, Jason Dillaman)
+* rbd: demote/promote all mirrored images in a pool (`issue#18748 <http://tracker.ceph.com/issues/18748>`_, `pr#13758 <https://github.com/ceph/ceph/pull/13758>`_, Jason Dillaman)
+* rbd: destination pool should be source pool if it is not specified (`issue#18326 <http://tracker.ceph.com/issues/18326>`_, `pr#13189 <https://github.com/ceph/ceph/pull/13189>`_, Gaurav Kumar Garg)
+* rbd: do not attempt to load key if auth is disabled (`issue#19035 <http://tracker.ceph.com/issues/19035>`_, `pr#16024 <https://github.com/ceph/ceph/pull/16024>`_, Jason Dillaman)
+* rbd: Drop unused member variable reopen in C_OpenComplete (`pr#16729 <https://github.com/ceph/ceph/pull/16729>`_, amitkuma)
+* rbd: enable rbd on FreeBSD (without KRBD) (`pr#12798 <https://github.com/ceph/ceph/pull/12798>`_, Willem Jan Withagen)
+* rbd: error out if import image format failed (`pr#13957 <https://github.com/ceph/ceph/pull/13957>`_, wangzhengyong)
+* rbd: fixed coverity 'Argument cannot be negative' warning (`pr#16686 <https://github.com/ceph/ceph/pull/16686>`_, amitkuma)
+* rbd: fix typo in Kernel.cc (`issue#19273 <http://tracker.ceph.com/issues/19273>`_, `pr#13983 <https://github.com/ceph/ceph/pull/13983>`_, Gaurav Kumar Garg)
+* rbd: 'image-meta remove' for missing key does not return error (`issue#16990 <http://tracker.ceph.com/issues/16990>`_, `pr#16393 <https://github.com/ceph/ceph/pull/16393>`_, PCzhangPC)
+* rbd: import-diff should discard any zeroed extents (`pr#14445 <https://github.com/ceph/ceph/pull/14445>`_, Jason Dillaman)
+* rbd: import needs to sanity check auto-generated image name (`issue#19128 <http://tracker.ceph.com/issues/19128>`_, `pr#14754 <https://github.com/ceph/ceph/pull/14754>`_, Mykola Golub)
+* rbd: import real thin-provision image (`issue#15648 <http://tracker.ceph.com/issues/15648>`_, `pr#12883 <https://github.com/ceph/ceph/pull/12883>`_, yaoning, Ning Yao)
+* rbd: info command should indicate if parent is in trash (`pr#14875 <https://github.com/ceph/ceph/pull/14875>`_, Jason Dillaman)
+* rbd: introduce v2 format for rbd export/import (`issue#13186 <http://tracker.ceph.com/issues/13186>`_, `pr#10487 <https://github.com/ceph/ceph/pull/10487>`_, Dongsheng Yang)
+* rbd: journal: don't hold future lock during assignment (`issue#18618 <http://tracker.ceph.com/issues/18618>`_, `pr#13033 <https://github.com/ceph/ceph/pull/13033>`_, Jason Dillaman)
+* rbd: journal: stop processing removal after error (`issue#18738 <http://tracker.ceph.com/issues/18738>`_, `pr#13193 <https://github.com/ceph/ceph/pull/13193>`_, Jason Dillaman)
+* rbd: luminous: librbd: default localize parent reads to false (`issue#20941 <http://tracker.ceph.com/issues/20941>`_, `pr#16899 <https://github.com/ceph/ceph/pull/16899>`_, Jason Dillaman)
+* rbd: luminous: librbd: remove consistency group rbd cli and API support (`pr#16875 <https://github.com/ceph/ceph/pull/16875>`_, Jason Dillaman)
+* rbd: luminous: rbd-ggate: tool to map images on FreeBSD via GEOM Gate (`pr#16895 <https://github.com/ceph/ceph/pull/16895>`_, Mykola Golub)
+* rbd: luminous: rbd-mirror: align use of uint64_t in service_daemon::AttributeType (`pr#16948 <https://github.com/ceph/ceph/pull/16948>`_, James Page)
+* rbd: luminous: rbd-mirror: simplify notifications for image assignment (`issue#15764 <http://tracker.ceph.com/issues/15764>`_, `pr#16878 <https://github.com/ceph/ceph/pull/16878>`_, Jason Dillaman)
+* rbd: luminous: rbd: parallelize rbd ls -l (`pr#16921 <https://github.com/ceph/ceph/pull/16921>`_, Piotr Dałek)
+* rbd: make it more understandable when adding peer returns error (`pr#16313 <https://github.com/ceph/ceph/pull/16313>`_, songweibin)
+* rbd-mirror: add support for active/passive daemon instances (`issue#17018 <http://tracker.ceph.com/issues/17018>`_, `issue#17019 <http://tracker.ceph.com/issues/17019>`_, `issue#17020 <http://tracker.ceph.com/issues/17020>`_, `pr#12948 <https://github.com/ceph/ceph/pull/12948>`_, Mykola Golub)
+* rbd-mirror: assertion failure during pool replayer shut down (`issue#20644 <http://tracker.ceph.com/issues/20644>`_, `pr#16704 <https://github.com/ceph/ceph/pull/16704>`_, Jason Dillaman)
+* rbd-mirror: avoid processing new events after stop requested (`issue#18441 <http://tracker.ceph.com/issues/18441>`_, `pr#12837 <https://github.com/ceph/ceph/pull/12837>`_, Jason Dillaman)
+* rbd-mirror: check remote image mirroring state when bootstrapping (`issue#18447 <http://tracker.ceph.com/issues/18447>`_, `pr#12820 <https://github.com/ceph/ceph/pull/12820>`_, Mykola Golub)
+* rbd-mirror: coordinate image syncs with leader (`issue#18789 <http://tracker.ceph.com/issues/18789>`_, `pr#14745 <https://github.com/ceph/ceph/pull/14745>`_, Mykola Golub)
+* rbd-mirror: delayed replication support (`issue#15371 <http://tracker.ceph.com/issues/15371>`_, `pr#11879 <https://github.com/ceph/ceph/pull/11879>`_, Mykola Golub)
+* rbd-mirror: deleting a snapshot during sync can result in read errors (`issue#18990 <http://tracker.ceph.com/issues/18990>`_, `pr#13568 <https://github.com/ceph/ceph/pull/13568>`_, Jason Dillaman)
+* rbd-mirror: ensure missing images are re-synced when detected (`issue#19811 <http://tracker.ceph.com/issues/19811>`_, `pr#14945 <https://github.com/ceph/ceph/pull/14945>`_, Jason Dillaman)
+* rbd-mirror: failover and failback of unmodified image results in split-brain (`issue#19858 <http://tracker.ceph.com/issues/19858>`_, `pr#14963 <https://github.com/ceph/ceph/pull/14963>`_, Jason Dillaman)
+* rbd-mirror: guard the deletion of non-primary images (`pr#16398 <https://github.com/ceph/ceph/pull/16398>`_, Jason Dillaman)
+* rbd-mirror: ignore permission errors on rbd_mirroring object (`issue#20571 <http://tracker.ceph.com/issues/20571>`_, `pr#16264 <https://github.com/ceph/ceph/pull/16264>`_, Jason Dillaman)
+* rbd-mirror: image deletions should be handled by assigned instance (`pr#14832 <https://github.com/ceph/ceph/pull/14832>`_, Jason Dillaman)
+* rbd-mirror: initialize timer context pointer to null (`pr#16603 <https://github.com/ceph/ceph/pull/16603>`_, Jason Dillaman)
+* rbd-mirror: InstanceWatcher watch/notify stub for leader/follower RPC (`issue#18783 <http://tracker.ceph.com/issues/18783>`_, `pr#13312 <https://github.com/ceph/ceph/pull/13312>`_, Mykola Golub)
+* rbd-mirror: lock loss during sync should wait for in-flight copies (`pr#15532 <https://github.com/ceph/ceph/pull/15532>`_, Jason Dillaman)
+* rbd-mirror: permit release of local image exclusive lock after force promotion (`issue#18963 <http://tracker.ceph.com/issues/18963>`_, `pr#15140 <https://github.com/ceph/ceph/pull/15140>`_, Jason Dillaman)
+* rbd-mirror: pool watcher should track mirror uuid (`pr#14240 <https://github.com/ceph/ceph/pull/14240>`_, Jason Dillaman)
+* rbd-mirror: remove tracking of image names from pool watcher (`pr#14712 <https://github.com/ceph/ceph/pull/14712>`_, Jason Dillaman)
+* rbd-mirror: replace remote pool polling with add/remove notifications (`issue#15029 <http://tracker.ceph.com/issues/15029>`_, `pr#12364 <https://github.com/ceph/ceph/pull/12364>`_, Jason Dillaman)
+* rbd-mirror: resolve admin socket path names collision (`issue#19907 <http://tracker.ceph.com/issues/19907>`_, `pr#15048 <https://github.com/ceph/ceph/pull/15048>`_, Mykola Golub)
+* rbd-mirror: separate ImageReplayer handling from Replayer (`issue#18785 <http://tracker.ceph.com/issues/18785>`_, `pr#13803 <https://github.com/ceph/ceph/pull/13803>`_, Mykola Golub)
+* rbd-mirror: Set the data pool correctly when creating images (`issue#20567 <http://tracker.ceph.com/issues/20567>`_, `pr#17023 <https://github.com/ceph/ceph/pull/17023>`_, Adam Wolfe Gordon)
+* rbd-mirror: track images via global image id (`pr#13416 <https://github.com/ceph/ceph/pull/13416>`_, Jason Dillaman)
+* rbd: modified some commands' description into imperative sentence (`pr#16694 <https://github.com/ceph/ceph/pull/16694>`_, songweibin)
+* rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly (`issue#18335 <http://tracker.ceph.com/issues/18335>`_, `pr#13229 <https://github.com/ceph/ceph/pull/13229>`_, Mykola Golub)
+* rbd-nbd: clean up the doc and help information (`pr#14146 <https://github.com/ceph/ceph/pull/14146>`_, Pan Liu)
+* rbd-nbd: create admin socket only for map command (`issue#17951 <http://tracker.ceph.com/issues/17951>`_, `pr#12433 <https://github.com/ceph/ceph/pull/12433>`_, Pan Liu)
+* rbd-nbd: display pool/image/snap information in list output (`pr#15317 <https://github.com/ceph/ceph/pull/15317>`_, Pan Liu)
+* rbd-nbd: don't ignore --read-only option in BLKROSET ioctl (`pr#13944 <https://github.com/ceph/ceph/pull/13944>`_, Pan Liu)
+* rbd-nbd: ensure unmap returns error code (`pr#15593 <https://github.com/ceph/ceph/pull/15593>`_, guojiannan, chenfangxian)
+* rbd-nbd: fix a typo "moudle" (`pr#13652 <https://github.com/ceph/ceph/pull/13652>`_, Pan Liu)
+* rbd-nbd: fix typo in comment (`pr#14034 <https://github.com/ceph/ceph/pull/14034>`_, Pan Liu)
+* rbd-nbd: no need to check image format any more (`pr#13389 <https://github.com/ceph/ceph/pull/13389>`_, Mykola Golub)
+* rbd-nbd: relax size check for newer kernel versions (`issue#19871 <http://tracker.ceph.com/issues/19871>`_, `pr#14976 <https://github.com/ceph/ceph/pull/14976>`_, Mykola Golub)
+* rbd-nbd: remove debug messages from do_unmap (`pr#14253 <https://github.com/ceph/ceph/pull/14253>`_, Pan Liu)
+* rbd-nbd: s/cpp_error/cpp_strerror/ to fix FTBFS (`pr#14223 <https://github.com/ceph/ceph/pull/14223>`_, Kefu Chai)
+* rbd-nbd: support signal handle for SIGHUP, SIGINT and SIGTERM (`issue#19349 <http://tracker.ceph.com/issues/19349>`_, `pr#14079 <https://github.com/ceph/ceph/pull/14079>`_, Pan Liu)
+* rbd-nbd: update size only when NBD_SET_SIZE successful (`pr#14005 <https://github.com/ceph/ceph/pull/14005>`_, Pan Liu)
+* rbd-nbd: warn when kernel parameters are ignored (`issue#19108 <http://tracker.ceph.com/issues/19108>`_, `pr#13694 <https://github.com/ceph/ceph/pull/13694>`_, Pan Liu)
+* rbd: prevent adding multiple mirror peers to a single pool (`issue#19256 <http://tracker.ceph.com/issues/19256>`_, `pr#13919 <https://github.com/ceph/ceph/pull/13919>`_, Jason Dillaman)
+* rbd: properly decode features when using image name optional (`issue#20185 <http://tracker.ceph.com/issues/20185>`_, `pr#15492 <https://github.com/ceph/ceph/pull/15492>`_, Jason Dillaman)
+* rbd:  pybind/rbd: add image metadata methods (`issue#19451 <http://tracker.ceph.com/issues/19451>`_, `pr#14463 <https://github.com/ceph/ceph/pull/14463>`_, Mykola Golub)
+* rbd: pybind/rbd: fix crash if more than 1024 images in trash bin (`pr#15134 <https://github.com/ceph/ceph/pull/15134>`_, runsisi)
+* rbd: rbd/bench: add notes of default values, it's easy to use (`pr#14762 <https://github.com/ceph/ceph/pull/14762>`_, Zheng Yin)
+* rbd: rbd/bench: fix write gaps when doing sequential writes with io-threads > 1 (`pr#15206 <https://github.com/ceph/ceph/pull/15206>`_, Igor Fedotov)
+* rbd: rbd, librbd: migrate atomic_t to std::atomic (`pr#14656 <https://github.com/ceph/ceph/pull/14656>`_, Jesse Williamson)
+* rbd: rbd-mirror A/A: leader should track up/down rbd-mirror instances (`issue#18784 <http://tracker.ceph.com/issues/18784>`_, `pr#13571 <https://github.com/ceph/ceph/pull/13571>`_, Mykola Golub)
+* rbd: rbd-mirror A/A: proxy InstanceReplayer APIs via InstanceWatcher RPC (`issue#18787 <http://tracker.ceph.com/issues/18787>`_, `pr#13978 <https://github.com/ceph/ceph/pull/13978>`_, Mykola Golub)
+* rbd: recognize exclusive option (`pr#14785 <https://github.com/ceph/ceph/pull/14785>`_, Ilya Dryomov)
+* rbd: removed hardcoded default pool (`pr#15518 <https://github.com/ceph/ceph/pull/15518>`_, Jason Dillaman)
+* rbd: remove direct linking to static boost libraries (`pr#12962 <https://github.com/ceph/ceph/pull/12962>`_, Jason Dillaman)
+* rbd: removed spurious error message from mirror pool commands (`pr#14935 <https://github.com/ceph/ceph/pull/14935>`_, Jason Dillaman)
+* rbd: remove unused condition within group action handler (`pr#12723 <https://github.com/ceph/ceph/pull/12723>`_, Gaurav Kumar Garg)
+* rbd,rgw,tools: tools/rbd, rgw: Removed unreachable returns (`pr#16308 <https://github.com/ceph/ceph/pull/16308>`_, Jos Collin)
+* rbd: spell out image features unsupported by the kernel (`issue#19095 <http://tracker.ceph.com/issues/19095>`_, `pr#13812 <https://github.com/ceph/ceph/pull/13812>`_, Ilya Dryomov)
+* rbd: stop indefinite thread waiting in krbd udev handling (`issue#17195 <http://tracker.ceph.com/issues/17195>`_, `pr#14051 <https://github.com/ceph/ceph/pull/14051>`_, Spandan Kumar Sahu)
+* rbd: test: fix rbd unit test cases w/ striping feature (`issue#18888 <http://tracker.ceph.com/issues/18888>`_, `pr#13196 <https://github.com/ceph/ceph/pull/13196>`_, Venky Shankar)
+* rbd,tests: luminous: qa/workunits/rbd: use command line option to specify watcher asok (`issue#20954 <http://tracker.ceph.com/issues/20954>`_, `pr#16946 <https://github.com/ceph/ceph/pull/16946>`_, Mykola Golub)
+* rbd,tests: luminous: test/librbd: fix race condition with OSD map refresh (`issue#20918 <http://tracker.ceph.com/issues/20918>`_, `pr#16903 <https://github.com/ceph/ceph/pull/16903>`_, Jason Dillaman)
+* rbd,tests: qa: add workunit to test krbd data-pool support (`pr#13482 <https://github.com/ceph/ceph/pull/13482>`_, Ilya Dryomov)
+* rbd,tests: qa: integrate OpenStack 'gate-tempest-dsvm-full-devstack-plugin-ceph' (`issue#18594 <http://tracker.ceph.com/issues/18594>`_, `pr#13158 <https://github.com/ceph/ceph/pull/13158>`_, Jason Dillaman)
+* rbd,tests: qa: krbd_data_pool.sh: account for rbd_info metadata object (`pr#14631 <https://github.com/ceph/ceph/pull/14631>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd discard/zeroout tests (`pr#15388 <https://github.com/ceph/ceph/pull/15388>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd write-after-checksum tests (`pr#14836 <https://github.com/ceph/ceph/pull/14836>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/krbd: unmap subsuite needs straw buckets (`pr#15290 <https://github.com/ceph/ceph/pull/15290>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/rbd: restrict python memcheck validation to CentOS (`pr#15923 <https://github.com/ceph/ceph/pull/15923>`_, Jason Dillaman)
+* rbd,tests: qa/tasks/qemu: update default image url after ceph.com redesign (`issue#18542 <http://tracker.ceph.com/issues/18542>`_, `pr#12953 <https://github.com/ceph/ceph/pull/12953>`_, Jason Dillaman)
+* rbd,tests: qa/tasks/rbd_fio: bump default fio version to 2.21 (`pr#16656 <https://github.com/ceph/ceph/pull/16656>`_, Ilya Dryomov)
+* rbd,tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode (`issue#20630 <http://tracker.ceph.com/issues/20630>`_, `pr#16340 <https://github.com/ceph/ceph/pull/16340>`_, Jason Dillaman)
+* rbd,tests: qa: thrash tests for backoff and upmap (`pr#16428 <https://github.com/ceph/ceph/pull/16428>`_, Ilya Dryomov)
+* rbd,tests: qa: update krbd_data_pool.sh to match the new rados ls behavior (`pr#15594 <https://github.com/ceph/ceph/pull/15594>`_, Ilya Dryomov)
+* rbd,tests: qa/workunits: adjust path to ceph-helpers.sh (`pr#16599 <https://github.com/ceph/ceph/pull/16599>`_, Sage Weil)
+* rbd,tests: qa/workunits: corrected issues with RBD cli test (`pr#14460 <https://github.com/ceph/ceph/pull/14460>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: diff.sh failed removing nonexistent file (`pr#14482 <https://github.com/ceph/ceph/pull/14482>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: increased trash deferment period (`pr#14846 <https://github.com/ceph/ceph/pull/14846>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: resolve potential rbd-mirror race conditions (`issue#18935 <http://tracker.ceph.com/issues/18935>`_, `pr#13421 <https://github.com/ceph/ceph/pull/13421>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: test data pool is mirrored correctly (`pr#17077 <https://github.com/ceph/ceph/pull/17077>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: tweak rbd-mirror config to spead up testing (`pr#13228 <https://github.com/ceph/ceph/pull/13228>`_, Mykola Golub)
+* rbd,tests: qa/workunits: switch to OpenStack Ocata release for RBD testing (`pr#14465 <https://github.com/ceph/ceph/pull/14465>`_, Jason Dillaman)
+* rbd,tests: test: correct language mode in file headers (`pr#12924 <https://github.com/ceph/ceph/pull/12924>`_, Jason Dillaman)
+* rbd,tests: test: fix compile warning in ceph_test_cls_rbd (`pr#15919 <https://github.com/ceph/ceph/pull/15919>`_, Jason Dillaman)
+* rbd,tests: test: fix failing rbd devstack teuthology test (`pr#15956 <https://github.com/ceph/ceph/pull/15956>`_, Jason Dillaman)
+* rbd,tests: test/librados_test_stub: fixed cls_cxx_map_get_keys/vals return value (`issue#19597 <http://tracker.ceph.com/issues/19597>`_, `pr#14484 <https://github.com/ceph/ceph/pull/14484>`_, Jason Dillaman)
+* rbd,tests: test/librbd: add break_lock test (`pr#12842 <https://github.com/ceph/ceph/pull/12842>`_, Mykola Golub)
+* rbd,tests: test/librbd/CMakeLists.txt: ceph_test_librbd_fsx requires linux includes/libs (`pr#13630 <https://github.com/ceph/ceph/pull/13630>`_, Willem Jan Withagen)
+* rbd,tests: test/librbd/fsx: Add break in case OP_WRITESAME and OP_COMPARE_AND_WRITE (`pr#16742 <https://github.com/ceph/ceph/pull/16742>`_, Luo Kexue)
+* rbd,tests: test/librbd: move tests using non-public api to internal (`pr#13806 <https://github.com/ceph/ceph/pull/13806>`_, Venky Shankar)
+* rbd,tests: test/librbd/test_librbd.cc: set \*features even if RBD_FEATURES is unset (`issue#19865 <http://tracker.ceph.com/issues/19865>`_, `pr#14965 <https://github.com/ceph/ceph/pull/14965>`_, Dan Mick)
+* rbd,tests: test/librbd/test_notify.py: don't disable feature in slave (`issue#19716 <http://tracker.ceph.com/issues/19716>`_, `pr#14751 <https://github.com/ceph/ceph/pull/14751>`_, Mykola Golub)
+* rbd,tests: test/librbd: unit tests cleanup (`pr#15113 <https://github.com/ceph/ceph/pull/15113>`_, Mykola Golub)
+* rbd,tests: test: rbd master/slave notify test should test active features (`issue#19692 <http://tracker.ceph.com/issues/19692>`_, `pr#14638 <https://github.com/ceph/ceph/pull/14638>`_, Jason Dillaman)
+* rbd,tests: test/rbd_mirror: race in TestMockInstanceWatcher on destroy (`pr#14453 <https://github.com/ceph/ceph/pull/14453>`_, Mykola Golub)
+* rbd,tests: test/rbd_mirror: race in TestMockLeaderWatcher.AcquireError (`issue#19405 <http://tracker.ceph.com/issues/19405>`_, `pr#14741 <https://github.com/ceph/ceph/pull/14741>`_, Mykola Golub)
+* rbd,tests: test: remove hard-coded image name from RBD metadata test (`issue#19798 <http://tracker.ceph.com/issues/19798>`_, `pr#14848 <https://github.com/ceph/ceph/pull/14848>`_, Jason Dillaman)
+* rbd,tests: test: support blacklisting within librados_test_stub (`pr#13737 <https://github.com/ceph/ceph/pull/13737>`_, Jason Dillaman)
+* rbd,tests: test/unittest_librbd: remove unused variables (`pr#15720 <https://github.com/ceph/ceph/pull/15720>`_, shiqi)
+* rbd,tests: test: use librados API to retrieve config params (`issue#18617 <http://tracker.ceph.com/issues/18617>`_, `pr#13076 <https://github.com/ceph/ceph/pull/13076>`_, Jason Dillaman)
+* rbd,tools: rbdmap: consider /etc/ceph/rbdmap when unmapping images (`issue#18884 <http://tracker.ceph.com/issues/18884>`_, `pr#13361 <https://github.com/ceph/ceph/pull/13361>`_, David Disseldorp)
+* rbd,tools: tools/rbd_mirror: initialize non-static class member m_do_resync in ImageReplayer (`pr#15889 <https://github.com/ceph/ceph/pull/15889>`_, Jos Collin)
+* rbd,tools: tools/rbd_nbd: add --version show support (`pr#16254 <https://github.com/ceph/ceph/pull/16254>`_, Jin Cai)
+* rbd: use concurrent writes for imports (`issue#19034 <http://tracker.ceph.com/issues/19034>`_, `pr#13782 <https://github.com/ceph/ceph/pull/13782>`_, Venky Shankar)
+* rbd: use min<uint64_t>() explicitly (`issue#18938 <http://tracker.ceph.com/issues/18938>`_, `pr#14202 <https://github.com/ceph/ceph/pull/14202>`_, Kefu Chai)
+* rbd: validate pool and snap name optionals (`issue#14535 <http://tracker.ceph.com/issues/14535>`_, `pr#13836 <https://github.com/ceph/ceph/pull/13836>`_, Gaurav Kumar Garg)
+* rbd: warning, ‘devno’ may be used uninitialized in this function (`pr#14271 <https://github.com/ceph/ceph/pull/14271>`_, Jos Collin)
+* rbd: When Ceph cluster becomes full, should allow user to remove rbd … (`pr#12627 <https://github.com/ceph/ceph/pull/12627>`_, Pan Liu)
+* rdma: msg/async: Postpone bind if network stack is not ready (`pr#14414 <https://github.com/ceph/ceph/pull/14414>`_, Amir Vadai, Haomai Wang)
+* rdma: msg/async/rdma: Add DSCP support (`pr#15484 <https://github.com/ceph/ceph/pull/15484>`_, Sarit Zubakov)
+* rdma: msg/async/rdma: add inqueue rx chunks perf counter (`pr#14782 <https://github.com/ceph/ceph/pull/14782>`_, Haomai Wang)
+* rdma: msg/async/rdma: fix log line spacing (`pr#13263 <https://github.com/ceph/ceph/pull/13263>`_, Adir Lev)
+* rdma: msg/async/rdma: Make poll_blocking() poll for async events in additio… (`pr#14320 <https://github.com/ceph/ceph/pull/14320>`_, Amir Vadai)
+* rdma: msg/async/rdma: Make port number an attribute of the Connection not o… (`pr#14297 <https://github.com/ceph/ceph/pull/14297>`_, Amir Vadai)
+* rdma: msg/async/rdma: RDMA-CM, get_device() by ibv_context (`pr#14410 <https://github.com/ceph/ceph/pull/14410>`_, Amir Vadai)
+* rdma: msg/async: Revert RDMA-CM (`pr#15262 <https://github.com/ceph/ceph/pull/15262>`_, Amir Vadai)
+    Replace using sleep with new wait_for_health() bash function
+* rgw: abort early when s->length empty during putobj (`pr#15682 <https://github.com/ceph/ceph/pull/15682>`_, Jiaying Ren)
+* rgw: AbortMultipart request returns NoSuchUpload error if the meta obj doesn't exist (`pr#12793 <https://github.com/ceph/ceph/pull/12793>`_, Zhang Shaowen)
+* rgw: acl grants num limit (`pr#16291 <https://github.com/ceph/ceph/pull/16291>`_, Enming Zhang)
+* rgw: add a new error code for non-existed subuser (`pr#16095 <https://github.com/ceph/ceph/pull/16095>`_, Zhao Chao)
+* rgw: add a new error code for non-existed user (`issue#20468 <http://tracker.ceph.com/issues/20468>`_, `pr#16033 <https://github.com/ceph/ceph/pull/16033>`_, Zhao Chao)
+* rgw: add apis to support ragweed (`pr#13645 <https://github.com/ceph/ceph/pull/13645>`_, Yehuda Sadeh)
+* rgw: add a separate configuration for data notify interval (`pr#16551 <https://github.com/ceph/ceph/pull/16551>`_, fang yuxiang)
+* rgw: add bucket size limit check to radosgw-admin (`issue#17925 <http://tracker.ceph.com/issues/17925>`_, `pr#11796 <https://github.com/ceph/ceph/pull/11796>`_, Matt Benjamin)
+* rgw: Added a globbing method for AWS Policies (`pr#12445 <https://github.com/ceph/ceph/pull/12445>`_, Pritha Srivastava)
+* rgw: Added code for REST APIs for AWS Roles (`pr#12104 <https://github.com/ceph/ceph/pull/12104>`_, Pritha Srivastava)
+* rgw: Added code to correctly account for bytes sent/ received during a 'PUT' operation (`pr#14042 <https://github.com/ceph/ceph/pull/14042>`_, Pritha Srivastava)
+* rgw: Adding code to create tenanted user for s3 bucket policy tests (`pr#15028 <https://github.com/ceph/ceph/pull/15028>`_, Pritha Srivastava)
+* rgw: add lifecycle validation according to S3 (`issue#18394 <http://tracker.ceph.com/issues/18394>`_, `pr#12750 <https://github.com/ceph/ceph/pull/12750>`_, Zhang Shaowen)
+* rgw: add missing RGWPeriod::reflect() based on new atomic update_latest_epoch() (`issue#19816 <http://tracker.ceph.com/issues/19816>`_, `issue#19817 <http://tracker.ceph.com/issues/19817>`_, `pr#14915 <https://github.com/ceph/ceph/pull/14915>`_, Casey Bodley)
+* rgw: add --num-zonegroups option for multi test (`pr#14216 <https://github.com/ceph/ceph/pull/14216>`_, lvshuhua)
+* rgw: add override in header files mostly (`pr#13586 <https://github.com/ceph/ceph/pull/13586>`_, liuchang0812)
+* rgw: add override in rgw subsystem (`issue#18922 <http://tracker.ceph.com/issues/18922>`_, `pr#13441 <https://github.com/ceph/ceph/pull/13441>`_, liuchang0812)
+* rgw: add pool namespace to cache's key so that system obj can have unique key (`issue#19372 <http://tracker.ceph.com/issues/19372>`_, `pr#14125 <https://github.com/ceph/ceph/pull/14125>`_, Zhang Shaowen)
+* rgw: add radosclient finisher to perf counter (`issue#19011 <http://tracker.ceph.com/issues/19011>`_, `pr#13535 <https://github.com/ceph/ceph/pull/13535>`_, lvshuhua)
+* rgw: add "rgw_verify_ssl" config (`pr#15301 <https://github.com/ceph/ceph/pull/15301>`_, Shasha Lu)
+* rgw: add 'state==SyncState::IncrementalSync' condition when add item … (`pr#14552 <https://github.com/ceph/ceph/pull/14552>`_, Shasha Lu)
+* rgw: add support container and object levels of swift bulkupload (`pr#14775 <https://github.com/ceph/ceph/pull/14775>`_, Jing Wenjun)
+* rgw: add support for delete marker expiration in s3 lifecycle (`issue#19730 <http://tracker.ceph.com/issues/19730>`_, `pr#14703 <https://github.com/ceph/ceph/pull/14703>`_, Zhang Shaowen)
+* rgw: add support for FormPost of Swift API (`issue#17273 <http://tracker.ceph.com/issues/17273>`_, `pr#11179 <https://github.com/ceph/ceph/pull/11179>`_, Radoslaw Zarzynski, Orit Wasserman)
+* rgw: add support for multipart upload expiration (`issue#19088 <http://tracker.ceph.com/issues/19088>`_, `pr#13622 <https://github.com/ceph/ceph/pull/13622>`_, Zhang Shaowen)
+* rgw: add support for noncurrentversion expiration in s3 lifecycle (`issue#18916 <http://tracker.ceph.com/issues/18916>`_, `pr#13385 <https://github.com/ceph/ceph/pull/13385>`_, Zhang Shaowen)
+* rgw: add support for Swift's TempURLs with prefix-based scope (`issue#20398 <http://tracker.ceph.com/issues/20398>`_, `pr#16370 <https://github.com/ceph/ceph/pull/16370>`_, Radoslaw Zarzynski)
+* rgw: add support for the BulkUpload of Swift API (`pr#12243 <https://github.com/ceph/ceph/pull/12243>`_, Radoslaw Zarzynski)
+* rgw: add the remove-x-delete feature to cancel swift object expiration (`issue#19074 <http://tracker.ceph.com/issues/19074>`_, `pr#13621 <https://github.com/ceph/ceph/pull/13621>`_, Jing Wenjun)
+* rgw: add the Vim's modeline into rgw_orphan.cc (`pr#15431 <https://github.com/ceph/ceph/pull/15431>`_, Radoslaw Zarzynski)
+* rgw: add variadic string join for s3 signature generation (`pr#15678 <https://github.com/ceph/ceph/pull/15678>`_, Casey Bodley)
+* rgw: Add --zonegroup-new-name  in usage (`pr#12084 <https://github.com/ceph/ceph/pull/12084>`_, Hans van den Bogert)
+* rgw: allow larger payload for period commit (`issue#19505 <http://tracker.ceph.com/issues/19505>`_, `pr#14355 <https://github.com/ceph/ceph/pull/14355>`_, Casey Bodley)
+* rgw: allow system users to read SLO parts (`issue#19027 <http://tracker.ceph.com/issues/19027>`_, `pr#13561 <https://github.com/ceph/ceph/pull/13561>`_, Casey Bodley)
+* rgw: auto reshard old buckets (`pr#15665 <https://github.com/ceph/ceph/pull/15665>`_, Orit Wasserman)
+* rgw: avoid listing user buckets for rgw_delete_user (`pr#13991 <https://github.com/ceph/ceph/pull/13991>`_, liuchang0812)
+* rgw: avoid using null pointer in rgw_file.cc (`pr#14474 <https://github.com/ceph/ceph/pull/14474>`_, lihongjie)
+* rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion (`issue#18364 <http://tracker.ceph.com/issues/18364>`_, `issue#16355 <http://tracker.ceph.com/issues/16355>`_, `pr#13220 <https://github.com/ceph/ceph/pull/13220>`_, Radoslaw Zarzynski)
+* rgw: bucket index check in radosgw-admin removes valid index (`issue#18470 <http://tracker.ceph.com/issues/18470>`_, `pr#12851 <https://github.com/ceph/ceph/pull/12851>`_, Zhang Shaowen)
+* rgw: bucket stats display bucket index type (`pr#14466 <https://github.com/ceph/ceph/pull/14466>`_, fang yuxiang)
+* rgw: change default chunk size to 4MB (`issue#18621 <http://tracker.ceph.com/issues/18621>`_, `issue#18622 <http://tracker.ceph.com/issues/18622>`_, `issue#18623 <http://tracker.ceph.com/issues/18623>`_, `pr#13035 <https://github.com/ceph/ceph/pull/13035>`_, Yehuda Sadeh)
+* rgw: change loglevel to 20 for 'System already converted' message (`issue#18919 <http://tracker.ceph.com/issues/18919>`_, `pr#13399 <https://github.com/ceph/ceph/pull/13399>`_, Vikhyat Umrao)
+* rgw: change loglevel to 5 in user's quota sync (`issue#18921 <http://tracker.ceph.com/issues/18921>`_, `pr#13408 <https://github.com/ceph/ceph/pull/13408>`_, Zhang Shaowen)
+* rgw: Changes for s3test config file, to add user under a tenant (`pr#15753 <https://github.com/ceph/ceph/pull/15753>`_, Pritha Srivastava)
+* rgw: check placement existence when create bucket (`pr#16385 <https://github.com/ceph/ceph/pull/16385>`_, Jiaying Ren)
+* rgw: check placement target existence during bucket creation (`pr#16384 <https://github.com/ceph/ceph/pull/16384>`_, Jiaying Ren)
+* rgw: civetweb don't go past the array index while calling mg_start (`issue#19749 <http://tracker.ceph.com/issues/19749>`_, `pr#14750 <https://github.com/ceph/ceph/pull/14750>`_, Abhishek Lekshmanan, Jesse Williamson)
+* rgw: clean redundant code (`pr#13302 <https://github.com/ceph/ceph/pull/13302>`_, Yankun Li)
+* rgw: clean unuse code in cls_statelog_check_state (`pr#10260 <https://github.com/ceph/ceph/pull/10260>`_, weiqiaomiao)
+* rgw: clean-up error mapping in Swift's authentication strategy (`pr#15756 <https://github.com/ceph/ceph/pull/15756>`_, Radoslaw Zarzynski)
+* rgw: cleanup: fix variable name in RGWRados::create_pool() declaration (`pr#14547 <https://github.com/ceph/ceph/pull/14547>`_, Nathan Cutler)
+* rgw: cleanup lc continuation (`pr#14906 <https://github.com/ceph/ceph/pull/14906>`_, Jiaying Ren)
+* rgw: cleanup lifecycle managament (`pr#13820 <https://github.com/ceph/ceph/pull/13820>`_, Jiaying Ren)
+* rgw: cleanup rgw-admin duplicated judge during OLH GET/READLOG (`pr#15700 <https://github.com/ceph/ceph/pull/15700>`_, Jiaying Ren)
+* rgw: clean up the redundant assignment in last_entry_in_listing (`pr#13387 <https://github.com/ceph/ceph/pull/13387>`_, Jing Wenjun)
+* rgw: clean up the unneeded rgw::io::ChunkingFilter::has_content_length (`pr#13504 <https://github.com/ceph/ceph/pull/13504>`_, Radoslaw Zarzynski)
+* rgw: cleanup unused codes in rgw_admin.cc (`pr#15771 <https://github.com/ceph/ceph/pull/15771>`_, fang yuxiang)
+* rgw: cleanup unused var in rgw/rgw_rest_s3.cc (`pr#13434 <https://github.com/ceph/ceph/pull/13434>`_, Jiaying Ren)
+* rgw: clear master_zonegroup when reseting RGWPeriodMap (`issue#17239 <http://tracker.ceph.com/issues/17239>`_, `pr#12658 <https://github.com/ceph/ceph/pull/12658>`_, Orit Wasserman)
+* rgw: clear old zone short ids on period update (`issue#15618 <http://tracker.ceph.com/issues/15618>`_, `pr#13949 <https://github.com/ceph/ceph/pull/13949>`_, Casey Bodley)
+* rgw: cls: ceph::timespan tag_timeout wrong units (`issue#20380 <http://tracker.ceph.com/issues/20380>`_, `pr#16026 <https://github.com/ceph/ceph/pull/16026>`_, Matt Benjamin)
+* rgw: cls/rgw: Clean up the "magic string" usage in the cls layer for RGW (`pr#12536 <https://github.com/ceph/ceph/pull/12536>`_, Ira Cooper)
+* rgw: cls/rgw: list_plain_entries() stops before bi_log entries (`issue#19876 <http://tracker.ceph.com/issues/19876>`_, `pr#14981 <https://github.com/ceph/ceph/pull/14981>`_, Casey Bodley)
+* rgw: cls/user: cls_user_bucket backward compatibility (`issue#19367 <http://tracker.ceph.com/issues/19367>`_, `pr#14128 <https://github.com/ceph/ceph/pull/14128>`_, Yehuda Sadeh)
+* rgw: cls_user don't clobber existing bucket stats when creating bucket (`issue#16357 <http://tracker.ceph.com/issues/16357>`_, `pr#10121 <https://github.com/ceph/ceph/pull/10121>`_, Abhishek Lekshmanan)
+* rgw: complete versioning enablement after sending it to meta master (`issue#18003 <http://tracker.ceph.com/issues/18003>`_, `pr#12444 <https://github.com/ceph/ceph/pull/12444>`_, Orit Wasserman)
+* rgw: Compress crash bug refactor (`issue#20098 <http://tracker.ceph.com/issues/20098>`_, `pr#15569 <https://github.com/ceph/ceph/pull/15569>`_, Adam Kupczyk)
+* rgw: continuation of the auth rework -- AWSv4 (`issue#18800 <http://tracker.ceph.com/issues/18800>`_, `pr#14885 <https://github.com/ceph/ceph/pull/14885>`_, Radoslaw Zarzynski, Javier M. Mellid)
+* rgw: continuation of the auth rework (`pr#12893 <https://github.com/ceph/ceph/pull/12893>`_, Radoslaw Zarzynski, Matt Benjamin)
+* rgw: Correcting the condition in ceph_assert while parsing an AWS Principal (`pr#15997 <https://github.com/ceph/ceph/pull/15997>`_, Pritha Srivastava)
+* rgw: correct the debug info when unlink instance failed (`pr#13761 <https://github.com/ceph/ceph/pull/13761>`_, Zhang Shaowen)
+* rgw: Correct the return codes for the health check feature (`issue#19025 <http://tracker.ceph.com/issues/19025>`_, `pr#13557 <https://github.com/ceph/ceph/pull/13557>`_, Pavan Rallabhandi)
+* rgw: custom user data header (`issue#19644 <http://tracker.ceph.com/issues/19644>`_, `pr#14592 <https://github.com/ceph/ceph/pull/14592>`_, Pavan Rallabhandi)
+* rgw: datalog trim and mdlog trim handles the result returned by osd incorrectly (`issue#20190 <http://tracker.ceph.com/issues/20190>`_, `pr#15507 <https://github.com/ceph/ceph/pull/15507>`_, Zhang Shaowen)
+* rgw: data sync includes instance in rgw_obj_index_key (`pr#13948 <https://github.com/ceph/ceph/pull/13948>`_, Casey Bodley)
+* rgw: deduplicate variants of rgw_make_bucket_entry_name() (`pr#14299 <https://github.com/ceph/ceph/pull/14299>`_, Radoslaw Zarzynski)
+* rgw: delete non-empty buckets in slave zonegroup works not well (`issue#19313 <http://tracker.ceph.com/issues/19313>`_, `pr#14043 <https://github.com/ceph/ceph/pull/14043>`_, Zhang Shaowen)
+* rgw: delete object in error path (`issue#20620 <http://tracker.ceph.com/issues/20620>`_, `pr#16324 <https://github.com/ceph/ceph/pull/16324>`_, Yehuda Sadeh)
+* rgw: disable dynamic reshading for 1st L point release (`pr#16969 <https://github.com/ceph/ceph/pull/16969>`_, Matt Benjamin)
+* rgw: display more info when using  radosgw-admin bucket stats (`pr#15256 <https://github.com/ceph/ceph/pull/15256>`_, fang.yuxiang)
+* rgw: Do not decrement stats cache when the cache values are zero (`issue#20661 <http://tracker.ceph.com/issues/20661>`_, `pr#16389 <https://github.com/ceph/ceph/pull/16389>`_, Pavan Rallabhandi)
+* rgw: Do not fetch bucket stats by default upon bucket listing (`issue#20377 <http://tracker.ceph.com/issues/20377>`_, `pr#15834 <https://github.com/ceph/ceph/pull/15834>`_, Pavan Rallabhandi)
+* rgw: do not log debug output at level 0 (`pr#15633 <https://github.com/ceph/ceph/pull/15633>`_, Jens Rosenboom)
+* rgw: don't do unneccesary write if buffer with zero length (`pr#14925 <https://github.com/ceph/ceph/pull/14925>`_, fang yuxiang)
+* rgw: don't init rgw_obj from rgw_obj_key when it's incorrect to do so (`issue#19096 <http://tracker.ceph.com/issues/19096>`_, `pr#13676 <https://github.com/ceph/ceph/pull/13676>`_, Yehuda Sadeh)
+* rgw: don't log the env_map twice (`pr#13481 <https://github.com/ceph/ceph/pull/13481>`_, Abhishek Lekshmanan)
+* rgw: don't read all user input for a few param requests (`pr#13815 <https://github.com/ceph/ceph/pull/13815>`_, Abhishek Lekshmanan)
+* rgw: don't return skew time error in pre-signed url (`issue#18828 <http://tracker.ceph.com/issues/18828>`_, `pr#13354 <https://github.com/ceph/ceph/pull/13354>`_, liuchang0812)
+* rgw: dont spawn error_repo until lease is acquired (`issue#19446 <http://tracker.ceph.com/issues/19446>`_, `pr#14714 <https://github.com/ceph/ceph/pull/14714>`_, Casey Bodley)
+* rgw: don't specify a length when converting bl -> string (`issue#20037 <http://tracker.ceph.com/issues/20037>`_, `pr#15599 <https://github.com/ceph/ceph/pull/15599>`_, Abhishek Lekshmanan)
+* rgw: don't use strlen in constexprs to not brake Clang builds (`pr#15688 <https://github.com/ceph/ceph/pull/15688>`_, Radoslaw Zarzynski)
+* rgw: drop asio/{yield,coroutine}.hpp replacements (`pr#15413 <https://github.com/ceph/ceph/pull/15413>`_, Kefu Chai)
+* rgw: Drop dump_usage_bucket_info() to silence warning from -Wunused-function (`pr#16497 <https://github.com/ceph/ceph/pull/16497>`_, Wei Qiaomiao)
+* rgw: drop unused find_replacement() and some function docs (`pr#16386 <https://github.com/ceph/ceph/pull/16386>`_, Jiaying Ren)
+* rgw: drop unused function RGWRemoteDataLog::get_shard_info() (`pr#16236 <https://github.com/ceph/ceph/pull/16236>`_, Shasha Lu)
+* rgw: drop unused param "bucket" from select_bucket_placement (`pr#14390 <https://github.com/ceph/ceph/pull/14390>`_, Shasha Lu)
+* rgw: drop unused port var (`pr#14412 <https://github.com/ceph/ceph/pull/14412>`_, Jiaying Ren)
+* rgw: drop unused rgw_pool parameter, local variables and member variable (`pr#16154 <https://github.com/ceph/ceph/pull/16154>`_, Jiaying Ren)
+* rgw: drop unused var header_ended (`pr#15686 <https://github.com/ceph/ceph/pull/15686>`_, Jiaying Ren)
+* rgw: drop using std ns in header files and other cleanups (`pr#15137 <https://github.com/ceph/ceph/pull/15137>`_, Abhishek Lekshmanan)
+* rgw: dynamic resharding (`pr#15493 <https://github.com/ceph/ceph/pull/15493>`_, Yehuda Sadeh, Orit Wasserman)
+* rgw: enable to update acl of bucket created in slave zonegroup (`issue#16888 <http://tracker.ceph.com/issues/16888>`_, `pr#14082 <https://github.com/ceph/ceph/pull/14082>`_, Guo Zhandong)
+* rgw: error_code in error log is not right when data sync fails (`issue#18437 <http://tracker.ceph.com/issues/18437>`_, `pr#12810 <https://github.com/ceph/ceph/pull/12810>`_, Zhang Shaowen)
+* rgw: error more verbosely in RGWRados::create_pool (`pr#14642 <https://github.com/ceph/ceph/pull/14642>`_, Matt Benjamin)
+* rgw: external auth engines of S3 honor rgw_keystone_implicit_tenants (`issue#17779 <http://tracker.ceph.com/issues/17779>`_, `pr#15572 <https://github.com/ceph/ceph/pull/15572>`_, Radoslaw Zarzynski)
+* rgw: Fix a bug that multipart upload may exceed the quota (`issue#19602 <http://tracker.ceph.com/issues/19602>`_, `pr#12010 <https://github.com/ceph/ceph/pull/12010>`_, Zhang Shaowen)
+* rgw: fix asctime when logging in rgw_lc (`pr#16422 <https://github.com/ceph/ceph/pull/16422>`_, Abhishek Lekshmanan)
+* rgw: fix break inside of yield in RGWFetchAllMetaCR (`issue#17655 <http://tracker.ceph.com/issues/17655>`_, `pr#11586 <https://github.com/ceph/ceph/pull/11586>`_, Casey Bodley)
+* rgw: fix broken /crossdomain.xml, /info and /healthcheck of Swift API (`issue#19520 <http://tracker.ceph.com/issues/19520>`_, `pr#14373 <https://github.com/ceph/ceph/pull/14373>`_, Radoslaw Zarzynski)
+* rgw: fix build of conflict after auth rework (`pr#14203 <https://github.com/ceph/ceph/pull/14203>`_, Casey Bodley)
+* rgw: fix configurable write obj window size (`pr#13934 <https://github.com/ceph/ceph/pull/13934>`_, hechuang)
+* rgw: fix constexpr for string_size in clang (`pr#15738 <https://github.com/ceph/ceph/pull/15738>`_, Adam C. Emerson, Casey Bodley)
+* rgw: fix disabling Swift's object versioning through empty X-Versions-Location (`issue#18852 <http://tracker.ceph.com/issues/18852>`_, `pr#13303 <https://github.com/ceph/ceph/pull/13303>`_, Jing Wenjun)
+* rgw: Fix duplicate tag removal during GC (`issue#20107 <http://tracker.ceph.com/issues/20107>`_, `pr#15912 <https://github.com/ceph/ceph/pull/15912>`_, Jens Rosenboom)
+* rgw: fix error code of inexistence of versions location in swift api (`issue#18880 <http://tracker.ceph.com/issues/18880>`_, `pr#13350 <https://github.com/ceph/ceph/pull/13350>`_, Jing Wenjun)
+* rgw: fix error handling in get_params() of RGWPostObj_ObjStore_S3 (`pr#15670 <https://github.com/ceph/ceph/pull/15670>`_, Radoslaw Zarzynski)
+* rgw: fix error handling in the link() method of RGWBucket (`issue#20279 <http://tracker.ceph.com/issues/20279>`_, `pr#15669 <https://github.com/ceph/ceph/pull/15669>`_, Radoslaw Zarzynski)
+* rgw: fix error message in removing bucket with --bypass-gc flag (`issue#20688 <http://tracker.ceph.com/issues/20688>`_, `pr#16419 <https://github.com/ceph/ceph/pull/16419>`_, Abhishek Varshney)
+* rgw: fix err when copy object in bucket with specified placement rule (`issue#20378 <http://tracker.ceph.com/issues/20378>`_, `pr#15837 <https://github.com/ceph/ceph/pull/15837>`_, fang yuxiang)
+* rgw: fixes for AWSBrowserUploadAbstractor auth (`issue#20372 <http://tracker.ceph.com/issues/20372>`_, `pr#15882 <https://github.com/ceph/ceph/pull/15882>`_, Radoslaw Zarzynski, Casey Bodley)
+* rgw:Fixes typo in rgw_admin.cc (`issue#19026 <http://tracker.ceph.com/issues/19026>`_, `pr#13576 <https://github.com/ceph/ceph/pull/13576>`_, Ronak Jain)
+* rgw: fix for broken yields in RGWMetaSyncShardCR (`issue#18076 <http://tracker.ceph.com/issues/18076>`_, `pr#12223 <https://github.com/ceph/ceph/pull/12223>`_, Casey Bodley)
+* rgw: fix for EINVAL errors on forwarded bucket put_acl requests (`pr#14376 <https://github.com/ceph/ceph/pull/14376>`_, Casey Bodley)
+* rgw: fix for null version_id in fetch_remote_obj() (`pr#14375 <https://github.com/ceph/ceph/pull/14375>`_, Casey Bodley)
+* rgw: Fix for Policy Parse exception in case of multiple statements (`pr#16689 <https://github.com/ceph/ceph/pull/16689>`_, Pritha Srivastava)
+* rgw: fix forward request for bulkupload to be applied in multisite (`issue#19645 <http://tracker.ceph.com/issues/19645>`_, `pr#14601 <https://github.com/ceph/ceph/pull/14601>`_, Jing Wenjun)
+* rgw: fix 'gc list --include-all' command infinite loop the first items (`issue#19978 <http://tracker.ceph.com/issues/19978>`_, `pr#12774 <https://github.com/ceph/ceph/pull/12774>`_, Shasha Lu, fang yuxiang)
+* rgw: fix get bucket policy s3 compatible issue (`pr#15280 <https://github.com/ceph/ceph/pull/15280>`_, Enming Zhang)
+* rgw: fix handling of --remote in radosgw-admin period commands (`issue#19554 <http://tracker.ceph.com/issues/19554>`_, `pr#14407 <https://github.com/ceph/ceph/pull/14407>`_, Casey Bodley)
+* rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT (`issue#18476 <http://tracker.ceph.com/issues/18476>`_, `pr#12865 <https://github.com/ceph/ceph/pull/12865>`_, Radoslaw Zarzynski)
+* rgw: fix infinite loop in rest api for log list (`issue#20386 <http://tracker.ceph.com/issues/20386>`_, `pr#15983 <https://github.com/ceph/ceph/pull/15983>`_, xierui, Casey Bodley)
+* rgw: fix init_bucket_for_sync retcode (`pr#13684 <https://github.com/ceph/ceph/pull/13684>`_, Shasha Lu)
+* rgw: fix lc list failure when shards not be all created (`issue#19898 <http://tracker.ceph.com/issues/19898>`_, `pr#15025 <https://github.com/ceph/ceph/pull/15025>`_, Jiaying Ren)
+* rgw: fix leaks with incomplete multiparts (`issue#17164 <http://tracker.ceph.com/issues/17164>`_, `pr#15630 <https://github.com/ceph/ceph/pull/15630>`_, Abhishek Varshney)
+* rgw: fix marker encoding problem (`issue#20463 <http://tracker.ceph.com/issues/20463>`_, `pr#15998 <https://github.com/ceph/ceph/pull/15998>`_, Marcus Watts)
+* rgw: fix memory leak in copy_obj_to_remote_dest (`pr#9974 <https://github.com/ceph/ceph/pull/9974>`_, weiqiaomiao)
+* rgw: fix memory leak in delete_obj_aio (`pr#13998 <https://github.com/ceph/ceph/pull/13998>`_, wangzhengyong)
+* rgw: fix memory leak in RGWGetObjLayout (`pr#14014 <https://github.com/ceph/ceph/pull/14014>`_, liuchang0812)
+* rgw: fix memory leaks during Swift Static Website's error handling (`issue#20757 <http://tracker.ceph.com/issues/20757>`_, `pr#16531 <https://github.com/ceph/ceph/pull/16531>`_, Radoslaw Zarzynski)
+* rgw: fix not initialized vars which cause rgw crash with ec data pool (`issue#20542 <http://tracker.ceph.com/issues/20542>`_, `pr#16177 <https://github.com/ceph/ceph/pull/16177>`_, Aleksei Gutikov)
+* rgw: fix off-by-one in RGWDataChangesLog::get_info (`issue#18488 <http://tracker.ceph.com/issues/18488>`_, `pr#12884 <https://github.com/ceph/ceph/pull/12884>`_, Casey Bodley)
+* rgw: fix parse/eval of policy conditions with IfExists (`issue#20708 <http://tracker.ceph.com/issues/20708>`_, `pr#16463 <https://github.com/ceph/ceph/pull/16463>`_, Casey Bodley)
+* rgw: fix period update crash (`issue#18631 <http://tracker.ceph.com/issues/18631>`_, `pr#13054 <https://github.com/ceph/ceph/pull/13054>`_, Orit Wasserman)
+* rgw: fix potential null pointer dereference in rgw_admin (`pr#15667 <https://github.com/ceph/ceph/pull/15667>`_, Radoslaw Zarzynski)
+* rgw: fix radosgw-admin data sync run crash (`issue#20423 <http://tracker.ceph.com/issues/20423>`_, `pr#15938 <https://github.com/ceph/ceph/pull/15938>`_, Shasha Lu)
+* rgw: fix radosgw-admin retcode (`pr#15257 <https://github.com/ceph/ceph/pull/15257>`_, Shasha Lu)
+* rgw: fix RadosGW hang during multi-chunk upload of AWSv4 (`issue#19754 <http://tracker.ceph.com/issues/19754>`_, `pr#14770 <https://github.com/ceph/ceph/pull/14770>`_, Radoslaw Zarzynski)
+* rgw: fix radosgw will crash when service is restarted during lifecycl… (`issue#20756 <http://tracker.ceph.com/issues/20756>`_, `pr#16495 <https://github.com/ceph/ceph/pull/16495>`_, Wei Qiaomiao)
+* rgw: fix response header of Swift API (`issue#19443 <http://tracker.ceph.com/issues/19443>`_, `pr#14280 <https://github.com/ceph/ceph/pull/14280>`_, tone-zhang)
+* rgw: fix rest client's order of args in get_v2_signature (`pr#15731 <https://github.com/ceph/ceph/pull/15731>`_, Casey Bodley)
+* rgw: fix rgw bucket policy IfExists position (`issue#20248 <http://tracker.ceph.com/issues/20248>`_, `pr#15607 <https://github.com/ceph/ceph/pull/15607>`_, yuliyang)
+* rgw: fix rgw hang when do RGWRealmReloader::reload after go SIGHUP (`issue#20686 <http://tracker.ceph.com/issues/20686>`_, `pr#16417 <https://github.com/ceph/ceph/pull/16417>`_, fang.yuxiang)
+* rgw: fix RGWPutBucketPolicy error when set BucketPolicy again without delete pre set Policy (`issue#20252 <http://tracker.ceph.com/issues/20252>`_, `pr#15617 <https://github.com/ceph/ceph/pull/15617>`_, yuliyang)
+* rgw: fix s3 object uploads with chunked transfers and v4 signatures (`issue#20447 <http://tracker.ceph.com/issues/20447>`_, `pr#15965 <https://github.com/ceph/ceph/pull/15965>`_, Marcus Watts)
+* rgw: fix segfault in RevokeThread during its shutdown procedure (`issue#19831 <http://tracker.ceph.com/issues/19831>`_, `pr#15033 <https://github.com/ceph/ceph/pull/15033>`_, Radoslaw Zarzynski)
+* rgw: fix slave zonegroup cannot enable the bucket versioning (`issue#18003 <http://tracker.ceph.com/issues/18003>`_, `pr#12175 <https://github.com/ceph/ceph/pull/12175>`_, lvshuhua)
+* rgw: fix SLO/DLO range requests (`pr#15060 <https://github.com/ceph/ceph/pull/15060>`_, Shasha Lu)
+    rgw: fix swift cannot disable object versioning
+* rgw: fix swift default auth error after auth strategy refactoring (`pr#15711 <https://github.com/ceph/ceph/pull/15711>`_, Casey Bodley)
+* rgw: fix test_multi.py default config file path (`pr#15306 <https://github.com/ceph/ceph/pull/15306>`_, Jiaying Ren)
+* rgw: fix the bug that part's index can't be removed after completing multipart upload (`issue#19604 <http://tracker.ceph.com/issues/19604>`_, `pr#14500 <https://github.com/ceph/ceph/pull/14500>`_, Zhang Shaowen)
+* rgw: fix the signature mismatch of FormPost in swift API (`issue#20220 <http://tracker.ceph.com/issues/20220>`_, `pr#15564 <https://github.com/ceph/ceph/pull/15564>`_, Jing Wenjun)
+* rgw: fix the UTF8 check on bucket entry name in rgw_log_op() (`issue#20779 <http://tracker.ceph.com/issues/20779>`_, `pr#16604 <https://github.com/ceph/ceph/pull/16604>`_, Radoslaw Zarzynski)
+* rgw: fix transition from full to incremental meta sync (`pr#13920 <https://github.com/ceph/ceph/pull/13920>`_, Casey Bodley)
+* rgw: fix typo in comment (`pr#13578 <https://github.com/ceph/ceph/pull/13578>`_, liuchang0812)
+* rgw: fix uninitialized fields (`pr#14120 <https://github.com/ceph/ceph/pull/14120>`_, wangzhengyong)
+* rgw: fix upgrate from hammer when zone doesn't have zoneparams (`issue#19231 <http://tracker.ceph.com/issues/19231>`_, `pr#13900 <https://github.com/ceph/ceph/pull/13900>`_, Orit Wasserman)
+* rgw: Fix up to 1000 entries at a time in check_bad_index_multipart (`issue#20772 <http://tracker.ceph.com/issues/20772>`_, `pr#16692 <https://github.com/ceph/ceph/pull/16692>`_, Orit Wasserman)
+* rgw: fix use of marker in List::list_objects() (`issue#18331 <http://tracker.ceph.com/issues/18331>`_, `pr#13147 <https://github.com/ceph/ceph/pull/13147>`_, Yehuda Sadeh)
+* rgw: fix versioned bucket data sync fail when upload is busy (`issue#18208 <http://tracker.ceph.com/issues/18208>`_, `pr#12357 <https://github.com/ceph/ceph/pull/12357>`_, lvshuhua)
+* rgw: fix wrong error code for expired Swift TempURL's links (`issue#20384 <http://tracker.ceph.com/issues/20384>`_, `pr#15850 <https://github.com/ceph/ceph/pull/15850>`_, Radoslaw Zarzynski)
+* rgw: fix X-Object-Meta-Static-Large-Object in SLO download (`issue#19951 <http://tracker.ceph.com/issues/19951>`_, `pr#15045 <https://github.com/ceph/ceph/pull/15045>`_, Shasha Lu)
+* rgw: fix zone did't update realm_id when added to zonegroup (`issue#17995 <http://tracker.ceph.com/issues/17995>`_, `pr#12139 <https://github.com/ceph/ceph/pull/12139>`_, Tianshan Qu)
+* rgw: forward RGWPutBucketPolicy to meta master (`issue#20297 <http://tracker.ceph.com/issues/20297>`_, `pr#15736 <https://github.com/ceph/ceph/pull/15736>`_, Casey Bodley)
+* rgw: get torrent request's parameter is not the same as amazon s3 (`issue#19136 <http://tracker.ceph.com/issues/19136>`_, `pr#13760 <https://github.com/ceph/ceph/pull/13760>`_, Zhang Shaowen)
+* rgw: get wrong content when download object with specific range with compression (`issue#20100 <http://tracker.ceph.com/issues/20100>`_, `pr#15323 <https://github.com/ceph/ceph/pull/15323>`_, fang yuxiang)
+* rgw: handle error return value in build_linked_oids_index (`pr#13955 <https://github.com/ceph/ceph/pull/13955>`_, wangzhengyong)
+* rgw: http_client clarify the debug msg function call (`pr#13688 <https://github.com/ceph/ceph/pull/13688>`_, Abhishek Lekshmanan)
+* rgw: if user.email is empty, dont try to delete (`issue#18980 <http://tracker.ceph.com/issues/18980>`_, `pr#13783 <https://github.com/ceph/ceph/pull/13783>`_, Casey Bodley)
+* rgw: implement get/put object tags for S3 (`pr#13753 <https://github.com/ceph/ceph/pull/13753>`_, Abhishek Lekshmanan)
+* rgw: improve handling of illformed Swift's container ACLs (`issue#18796 <http://tracker.ceph.com/issues/18796>`_, `pr#13248 <https://github.com/ceph/ceph/pull/13248>`_, Radoslaw Zarzynski)
+* rgw: /info claims we do support Swift's accounts ACLs (`issue#20394 <http://tracker.ceph.com/issues/20394>`_, `pr#15887 <https://github.com/ceph/ceph/pull/15887>`_, Radoslaw Zarzynski)
+* rgw: initialize non-static class members in ESQueryCompiler (`pr#15884 <https://github.com/ceph/ceph/pull/15884>`_, Jos Collin)
+* rgw: initialize Non-static class member val in ESQueryNodeLeafVal_Int (`pr#15888 <https://github.com/ceph/ceph/pull/15888>`_, Jos Collin)
+* rgw: initialize Non-static class member worker in RGWReshard (`pr#15886 <https://github.com/ceph/ceph/pull/15886>`_, Jos Collin)
+* rgw: Initialize of member variable admin_specified in RGWUserAdminOpState (`pr#16847 <https://github.com/ceph/ceph/pull/16847>`_, amitkuma)
+* rgw: Initialize pointer fields (`pr#16021 <https://github.com/ceph/ceph/pull/16021>`_, Jos Collin)
+* rgw: LCWorker's worktime is not the same as config rgw_lifecycle_work_time (`issue#18087 <http://tracker.ceph.com/issues/18087>`_, `pr#11963 <https://github.com/ceph/ceph/pull/11963>`_, Zhang Shaowen)
+* rgw: ldap: simple_bind() should set ldap version option on tldap (`pr#12616 <https://github.com/ceph/ceph/pull/12616>`_, Weibing Zhang)
+* rgw: lease_stack: use reset method instead of assignment (`pr#16185 <https://github.com/ceph/ceph/pull/16185>`_, Nathan Cutler)
+* rgw: Let the object stat command be shown in the usage (`issue#19013 <http://tracker.ceph.com/issues/19013>`_, `pr#13291 <https://github.com/ceph/ceph/pull/13291>`_, Pavan Rallabhandi)
+* rgw: librgw shut (`issue#18585 <http://tracker.ceph.com/issues/18585>`_, `pr#12972 <https://github.com/ceph/ceph/pull/12972>`_, Matt Benjamin)
+* rgw: lifecycle thread shouldn't process the bucket which has been deleted (`issue#20285 <http://tracker.ceph.com/issues/20285>`_, `pr#15677 <https://github.com/ceph/ceph/pull/15677>`_, Zhang Shaowen)
+* rgw: lock is not released when set sync marker is failed (`issue#18077 <http://tracker.ceph.com/issues/18077>`_, `pr#12197 <https://github.com/ceph/ceph/pull/12197>`_, Zhang Shaowen)
+* rgw: log_meta only for more than one zone (`issue#20357 <http://tracker.ceph.com/issues/20357>`_, `pr#15777 <https://github.com/ceph/ceph/pull/15777>`_, Orit Wasserman, Leo Zhang)
+* rgw: lower some log's level in gc process (`pr#15426 <https://github.com/ceph/ceph/pull/15426>`_, Zhang Shaowen)
+* rgw: luminous: rgw: Fix rgw not responding occasionally when receiving SIGHUP signal (`issue#20962 <http://tracker.ceph.com/issues/20962>`_, `pr#17113 <https://github.com/ceph/ceph/pull/17113>`_, Yao Zongyou)
+* rgw: luminous: RGW: Get Bucket ACL does not honor the s3:GetBucketACL action (`issue#21013 <http://tracker.ceph.com/issues/21013>`_, `issue#21056 <http://tracker.ceph.com/issues/21056>`_, `pr#17117 <https://github.com/ceph/ceph/pull/17117>`_, Abhishek Lekshmanan)
+* rgw: luminous: rgw: GetObject Tagging needs to exit earlier if the object has no attributes (`issue#21054 <http://tracker.ceph.com/issues/21054>`_, `issue#21010 <http://tracker.ceph.com/issues/21010>`_, `pr#17118 <https://github.com/ceph/ceph/pull/17118>`_, Abhishek Lekshmanan)
+* rgw: luminous: rgw_lc: support for AWSv4 authentication (`pr#16914 <https://github.com/ceph/ceph/pull/16914>`_, Abhishek Lekshmanan)
+* rgw: luminous: rgw: S3 v4 auth fails when query string contains (`issue#21000 <http://tracker.ceph.com/issues/21000>`_, `issue#21003 <http://tracker.ceph.com/issues/21003>`_, `issue#20501 <http://tracker.ceph.com/issues/20501>`_, `issue#21043 <http://tracker.ceph.com/issues/21043>`_, `pr#17114 <https://github.com/ceph/ceph/pull/17114>`_, Zhang Shaowen, Marcus Watts)
+* rgw: luminous: rgw: Use namespace for lc_pool and roles_pool (`issue#20177 <http://tracker.ceph.com/issues/20177>`_, `issue#20967 <http://tracker.ceph.com/issues/20967>`_, `pr#16943 <https://github.com/ceph/ceph/pull/16943>`_, Orit Wasserman)
+* rgw: make RGWEnv return a const ref. to its map (`pr#15269 <https://github.com/ceph/ceph/pull/15269>`_, Abhishek Lekshmanan)
+* rgw: make sending Content-Length in 204 and 304 responses controllable (`issue#16602 <http://tracker.ceph.com/issues/16602>`_, `pr#10156 <https://github.com/ceph/ceph/pull/10156>`_, Radoslaw Zarzynski)
+* rgw: make sync thread name clear (`issue#18860 <http://tracker.ceph.com/issues/18860>`_, `pr#13324 <https://github.com/ceph/ceph/pull/13324>`_, lvshuhua)
+* rgw: match wildcards in StringLike policy conditions (`issue#20308 <http://tracker.ceph.com/issues/20308>`_, `pr#16491 <https://github.com/ceph/ceph/pull/16491>`_, Casey Bodley)
+* rgw: metadata search part 2 (`pr#14351 <https://github.com/ceph/ceph/pull/14351>`_, Yehuda Sadeh)
+* rgw: meta sync thread crash at RGWMetaSyncShardCR (`issue#20251 <http://tracker.ceph.com/issues/20251>`_, `pr#15660 <https://github.com/ceph/ceph/pull/15660>`_, fang.yuxiang)
+* rgw: migrate atomic_t to std::atomic<> (ebirah) (`pr#14839 <https://github.com/ceph/ceph/pull/14839>`_, Jesse Williamson)
+* rgw: migrate atomic_t to std::atomic<> (`pr#15001 <https://github.com/ceph/ceph/pull/15001>`_, Jesse Williamson)
+* rgw: modify email to empty by admin RESTful api doesn't work (`pr#16309 <https://github.com/ceph/ceph/pull/16309>`_, fang.yuxiang)
+* rgw: move the S3 anonymous auth handling to a dedicated engine (`pr#16485 <https://github.com/ceph/ceph/pull/16485>`_, Radoslaw Zarzynski)
+* rgw: multipart copy-part remove '/' for s3 java sdk request header (`issue#20075 <http://tracker.ceph.com/issues/20075>`_, `pr#15283 <https://github.com/ceph/ceph/pull/15283>`_, root)
+* rgw: multisite enabled over multiple clusters (`pr#12535 <https://github.com/ceph/ceph/pull/12535>`_, Ali Maredia)
+* rgw: multisite: fixes for zonegroup redirect (`issue#19488 <http://tracker.ceph.com/issues/19488>`_, `pr#14319 <https://github.com/ceph/ceph/pull/14319>`_, Casey Bodley)
+* rgw:multisite: fix RGWRadosRemoveOmapKeysCR and change cn to intrusive_ptr (`issue#20539 <http://tracker.ceph.com/issues/20539>`_, `pr#16197 <https://github.com/ceph/ceph/pull/16197>`_, Shasha Lu)
+* rgw: never let http_redirect_code of RGWRedirectInfo to stay uninitialized (`issue#20774 <http://tracker.ceph.com/issues/20774>`_, `pr#16601 <https://github.com/ceph/ceph/pull/16601>`_, Radoslaw Zarzynski)
+* rgw: omit X-Account-Access-Control if there is no grant to serialize (`issue#20395 <http://tracker.ceph.com/issues/20395>`_, `pr#15883 <https://github.com/ceph/ceph/pull/15883>`_, Radoslaw Zarzynski)
+* rgw: only log metadata on metadata master zone (`issue#20244 <http://tracker.ceph.com/issues/20244>`_, `pr#15613 <https://github.com/ceph/ceph/pull/15613>`_, Casey Bodley)
+* rgw: optimize data sync. Add zones_trace in log to avoid needless sync (`issue#19219 <http://tracker.ceph.com/issues/19219>`_, `pr#13851 <https://github.com/ceph/ceph/pull/13851>`_, Zhang Shaowen)
+* rgw: optimize generating torrent file. Object data won't stay in memory  now (`pr#15153 <https://github.com/ceph/ceph/pull/15153>`_, Zhang Shaowen)
+* rgw: orphan: fix error messages (`pr#12782 <https://github.com/ceph/ceph/pull/12782>`_, Weibing Zhang)
+* rgw: pass authentication domain to civetweb (`issue#17657 <http://tracker.ceph.com/issues/17657>`_, `pr#12861 <https://github.com/ceph/ceph/pull/12861>`_, Abhishek Lekshmanan)
+* rgw: polymorphic error codes (`pr#10690 <https://github.com/ceph/ceph/pull/10690>`_, Pritha Srivastava, Marcus Watts)
+* rgw: print is_admin as int instead of __u8 (`pr#12264 <https://github.com/ceph/ceph/pull/12264>`_, Casey Bodley)
+* rgw: put object's acl can't work well on the latest object (`issue#18649 <http://tracker.ceph.com/issues/18649>`_, `pr#13078 <https://github.com/ceph/ceph/pull/13078>`_, Zhang Shaowen)
+* rgw: radosgw-admin: use zone id when creating a zone (`issue#19498 <http://tracker.ceph.com/issues/19498>`_, `pr#14340 <https://github.com/ceph/ceph/pull/14340>`_, Orit Wasserman)
+* rgw: radosgw-admin: warn that 'realm rename' does not update other clusters (`issue#19746 <http://tracker.ceph.com/issues/19746>`_, `pr#14722 <https://github.com/ceph/ceph/pull/14722>`_, Casey Bodley)
+* rgw: radosgw, crypto: simplified code in handle_data functions (`pr#15598 <https://github.com/ceph/ceph/pull/15598>`_, Adam Kupczyk)
+* rgw: radosgw: fix compilation with cryptopp (`pr#15960 <https://github.com/ceph/ceph/pull/15960>`_, Adam Kupczyk)
+* rgw: raise debug level of meta sync logging (`pr#15524 <https://github.com/ceph/ceph/pull/15524>`_, Casey Bodley)
+* rgw: raise debug level of RGWPostObj_ObjStore_S3::get_policy (`pr#16203 <https://github.com/ceph/ceph/pull/16203>`_, Shasha Lu)
+* rgw: reject request if decoded URI contains \0 in the middle (`issue#20418 <http://tracker.ceph.com/issues/20418>`_, `pr#15953 <https://github.com/ceph/ceph/pull/15953>`_, Radoslaw Zarzynski)
+* rgw: remove a redundant judgement in rgw_rados.cc:delete_obj (`pr#11124 <https://github.com/ceph/ceph/pull/11124>`_, Zhang Shaowen)
+* rgw: Removed Unwanted headers (`pr#14183 <https://github.com/ceph/ceph/pull/14183>`_, Jos Collin)
+* rgw: remove duplicate flush formatter (`pr#12437 <https://github.com/ceph/ceph/pull/12437>`_, Guo Zhandong)
+* rgw: remove extra RGWMPObj in rgw_multi.h (`pr#14619 <https://github.com/ceph/ceph/pull/14619>`_, Casey Bodley)
+* rgw: remove fastcgi from default rgw frontends (`pr#15098 <https://github.com/ceph/ceph/pull/15098>`_, Casey Bodley)
+* rgw: remove invalid read size4 (`issue#18071 <http://tracker.ceph.com/issues/18071>`_, `pr#12767 <https://github.com/ceph/ceph/pull/12767>`_, Matt Benjamin)
+    rgw: Remove pessimizing move
+* rgw: remove redundant codes in rgw_cache.h (`pr#13902 <https://github.com/ceph/ceph/pull/13902>`_, lihongjie)
+* rgw: Remove spurious XML header for GetBucketPolicy (`issue#20247 <http://tracker.ceph.com/issues/20247>`_, `pr#15586 <https://github.com/ceph/ceph/pull/15586>`_, Adam C. Emerson)
+* rgw: remove the useless output when listing zonegroups (`pr#16331 <https://github.com/ceph/ceph/pull/16331>`_, Zhang Shaowen)
+* rgw: remove unused func in rgw_file.h (`pr#15698 <https://github.com/ceph/ceph/pull/15698>`_, lihongjie)
+* rgw: remove useless --tier_type in radosgw-admin (`pr#13856 <https://github.com/ceph/ceph/pull/13856>`_, Zhang Shaowen)
+* rgw: rename s3_code to err_code for swift (`pr#12300 <https://github.com/ceph/ceph/pull/12300>`_, Guo Zhandong)
+* rgw: Replace get_zonegroup().is_master_zonegroup() with is_meta_master() in RGWBulkDelete::Deleter::delete_single() (`pr#16062 <https://github.com/ceph/ceph/pull/16062>`_, Fan Yang)
+* rgw: req xml params size limitation error msg (`pr#16310 <https://github.com/ceph/ceph/pull/16310>`_, Enming Zhang)
+* rgw: respect Swift's negative, HTTP referer-based ACL grants (`issue#18841 <http://tracker.ceph.com/issues/18841>`_, `pr#14344 <https://github.com/ceph/ceph/pull/14344>`_, Radoslaw Zarzynski)
+* rgw: restore admin socket path in mrgw.sh (`pr#16540 <https://github.com/ceph/ceph/pull/16540>`_, Casey Bodley)
+* rgw: return the version id in get object and object metadata request (`issue#19370 <http://tracker.ceph.com/issues/19370>`_, `pr#14117 <https://github.com/ceph/ceph/pull/14117>`_, Zhang Shaowen)
+* rgw: rgw-admin: fix bucket limit check argparse, div(0) (`pr#15316 <https://github.com/ceph/ceph/pull/15316>`_, Matt Benjamin)
+* rgw: rgw-admin: remove deprecated regionmap commands (`issue#18725 <http://tracker.ceph.com/issues/18725>`_, `pr#13963 <https://github.com/ceph/ceph/pull/13963>`_, Casey Bodley)
+* rgw: rgw_common: use string::npos for the results of str.find (`pr#14341 <https://github.com/ceph/ceph/pull/14341>`_, Abhishek Lekshmanan)
+* rgw: rgw_crypt: log error messages during failures (`pr#16726 <https://github.com/ceph/ceph/pull/16726>`_, Abhishek Lekshmanan)
+* rgw: rgw_file: add compression interop to RGW NFS (`issue#20462 <http://tracker.ceph.com/issues/20462>`_, `pr#15989 <https://github.com/ceph/ceph/pull/15989>`_, Matt Benjamin)
+* rgw: rgw_file: add lock protection for readdir against gc (`issue#20121 <http://tracker.ceph.com/issues/20121>`_, `pr#15329 <https://github.com/ceph/ceph/pull/15329>`_, Gui Hecheng)
+* rgw: rgw_file:  add service map registration (`pr#16251 <https://github.com/ceph/ceph/pull/16251>`_, Matt Benjamin)
+* rgw: rgw_file: add timed namespace invalidation (`issue#18651 <http://tracker.ceph.com/issues/18651>`_, `pr#13038 <https://github.com/ceph/ceph/pull/13038>`_, Matt Benjamin)
+* rgw: rgw_file: avoid a recursive lane lock in LRU drain (`issue#20374 <http://tracker.ceph.com/issues/20374>`_, `pr#15819 <https://github.com/ceph/ceph/pull/15819>`_, Matt Benjamin)
+* rgw: rgw_file: avoid stranding invalid-name bucket handles in fhcache (`issue#19036 <http://tracker.ceph.com/issues/19036>`_, `pr#13590 <https://github.com/ceph/ceph/pull/13590>`_, Matt Benjamin)
+* rgw: rgw_file cleanup names (`pr#15568 <https://github.com/ceph/ceph/pull/15568>`_, Gui Hecheng)
+* rgw: rgw_file: cleanup virtual keyword on derived functions (`pr#14908 <https://github.com/ceph/ceph/pull/14908>`_, Gui Hecheng)
+* rgw: rgw_file: ensure valid_s3_object_name for directories, too (`issue#19066 <http://tracker.ceph.com/issues/19066>`_, `pr#13614 <https://github.com/ceph/ceph/pull/13614>`_, Matt Benjamin)
+* rgw: rgw_file: fix assert upon setattr on bucket (`issue#20287 <http://tracker.ceph.com/issues/20287>`_, `pr#15679 <https://github.com/ceph/ceph/pull/15679>`_, Gui Hecheng)
+* rgw: rgw_file: fix double unref on rgw_fh for rename (`pr#13988 <https://github.com/ceph/ceph/pull/13988>`_, Gui Hecheng)
+* rgw: rgw_file: fix flags set on unsuccessful unlink (`pr#15222 <https://github.com/ceph/ceph/pull/15222>`_, Gui Hecheng)
+* rgw: rgw_file:  fix fs_inst progression (`issue#19214 <http://tracker.ceph.com/issues/19214>`_, `pr#13832 <https://github.com/ceph/ceph/pull/13832>`_, Matt Benjamin)
+* rgw: rgw_file: fix missing unlock in unlink (`issue#19435 <http://tracker.ceph.com/issues/19435>`_, `pr#14262 <https://github.com/ceph/ceph/pull/14262>`_, Gui Hecheng)
+* rgw: rgw_file: fix misuse of make_key_name before make_fhk (`pr#15108 <https://github.com/ceph/ceph/pull/15108>`_, Gui Hecheng)
+* rgw: rgw_file: fix non-negative return code for open operation (`pr#14045 <https://github.com/ceph/ceph/pull/14045>`_, Gui Hecheng)
+* rgw: rgw_file: fix non-posix errcode EINVAL to ENAMETOOLONG (`pr#13764 <https://github.com/ceph/ceph/pull/13764>`_, Gui Hecheng)
+* rgw: rgw_file: fix readdir after dirent-change (`issue#19634 <http://tracker.ceph.com/issues/19634>`_, `pr#14561 <https://github.com/ceph/ceph/pull/14561>`_, Matt Benjamin)
+* rgw: rgw_file: fix reversed return value of getattr (`pr#13895 <https://github.com/ceph/ceph/pull/13895>`_, Gui Hecheng)
+* rgw: rgw_file:  fix RGWLibFS::setattr for directory objects (`issue#18808 <http://tracker.ceph.com/issues/18808>`_, `pr#13252 <https://github.com/ceph/ceph/pull/13252>`_, Matt Benjamin)
+* rgw: rgw_file: fix up potential race condition (`pr#14553 <https://github.com/ceph/ceph/pull/14553>`_, Gui Hecheng)
+* rgw: rgw_file: implement reliable has-children check (unlink dir) (`issue#19270 <http://tracker.ceph.com/issues/19270>`_, `pr#13953 <https://github.com/ceph/ceph/pull/13953>`_, Matt Benjamin)
+* rgw: rgw_file:  interned RGWFileHandle objects need parent refs (`issue#18650 <http://tracker.ceph.com/issues/18650>`_, `pr#13084 <https://github.com/ceph/ceph/pull/13084>`_, Matt Benjamin)
+* rgw: rgw_file: posix style atime,ctime,mtime (`pr#13765 <https://github.com/ceph/ceph/pull/13765>`_, Gui Hecheng)
+* rgw: rgw_file:  pre-compute unix attrs in write_finish() (`issue#19653 <http://tracker.ceph.com/issues/19653>`_, `pr#14609 <https://github.com/ceph/ceph/pull/14609>`_, Matt Benjamin)
+* rgw: rgw_file: prevent conflict of mkdir between restarts (`issue#20275 <http://tracker.ceph.com/issues/20275>`_, `pr#15655 <https://github.com/ceph/ceph/pull/15655>`_, Gui Hecheng)
+* rgw: rgw_file: properly & or flags (`issue#20663 <http://tracker.ceph.com/issues/20663>`_, `pr#16448 <https://github.com/ceph/ceph/pull/16448>`_, Matt Benjamin)
+* rgw: rgw_file: release rgw_fh lock and ref on ENOTEMPTY (`issue#20061 <http://tracker.ceph.com/issues/20061>`_, `pr#15246 <https://github.com/ceph/ceph/pull/15246>`_, Matt Benjamin)
+* rgw: rgw_file: removed extra rele() on fs in rgw_umount() (`pr#15152 <https://github.com/ceph/ceph/pull/15152>`_, Gui Hecheng)
+* rgw: rgw_file: remove hidden uxattr objects from buckets on delete (`issue#20045 <http://tracker.ceph.com/issues/20045>`_, `pr#15210 <https://github.com/ceph/ceph/pull/15210>`_, Matt Benjamin)
+* rgw: rgw_file:  remove post-unlink lookup check (`issue#20047 <http://tracker.ceph.com/issues/20047>`_, `pr#15216 <https://github.com/ceph/ceph/pull/15216>`_, Matt Benjamin)
+* rgw: rgw_file: replace raw fs->fh_lru.unref with predefined fs->unref (`pr#15541 <https://github.com/ceph/ceph/pull/15541>`_, Gui Hecheng)
+* rgw: rgw_file: RGWFileHandle dtor must also cond-unlink from FHCache (`issue#19112 <http://tracker.ceph.com/issues/19112>`_, `pr#13712 <https://github.com/ceph/ceph/pull/13712>`_, Matt Benjamin)
+* rgw: rgw_file skip policy read for virtual components (`pr#16034 <https://github.com/ceph/ceph/pull/16034>`_, Gui Hecheng)
+* rgw: rgw_file:  split last argv on ws, if provided (`pr#12965 <https://github.com/ceph/ceph/pull/12965>`_, Matt Benjamin)
+* rgw: rgw_file: store bucket uxattrs on the bucket (`issue#20082 <http://tracker.ceph.com/issues/20082>`_, `pr#15293 <https://github.com/ceph/ceph/pull/15293>`_, Matt Benjamin)
+* rgw: rgw_file: support readdir cb type hints (plus fixes) (`issue#19623 <http://tracker.ceph.com/issues/19623>`_, `issue#19625 <http://tracker.ceph.com/issues/19625>`_, `issue#19624 <http://tracker.ceph.com/issues/19624>`_, `pr#14458 <https://github.com/ceph/ceph/pull/14458>`_, Matt Benjamin)
+* rgw: rgw_file:  use fh_hook::is_linked() to check residence (`issue#19111 <http://tracker.ceph.com/issues/19111>`_, `pr#13703 <https://github.com/ceph/ceph/pull/13703>`_, Matt Benjamin)
+* rgw: rgw_file: v3: fix write-timer action (`issue#19932 <http://tracker.ceph.com/issues/19932>`_, `pr#15097 <https://github.com/ceph/ceph/pull/15097>`_, Matt Benjamin)
+* rgw: rgw : fix race in RGWCompleteMultipart (`issue#20861 <http://tracker.ceph.com/issues/20861>`_, `pr#16732 <https://github.com/ceph/ceph/pull/16732>`_, Abhishek Varshney)
+* rgw: rgw：fix s3 aws v2 signature priority between header['X-Amz-Date'] and header['Date'] (`issue#20176 <http://tracker.ceph.com/issues/20176>`_, `pr#15467 <https://github.com/ceph/ceph/pull/15467>`_, yuliyang)
+* rgw:  rgw: fix the subdir without slash of s3 website url (`issue#20307 <http://tracker.ceph.com/issues/20307>`_, `pr#15703 <https://github.com/ceph/ceph/pull/15703>`_, liuhong)
+* rgw: rgw_lc: drop a bunch of unused headers (`pr#14342 <https://github.com/ceph/ceph/pull/14342>`_, Abhishek Lekshmanan)
+* rgw: rgw_ldap: log the ldap err in case of bind failure (`pr#14781 <https://github.com/ceph/ceph/pull/14781>`_, Abhishek Lekshmanan)
+* rgw: rgw/lifecycle: do not send lifecycle rules when GetLifeCycle failed (`issue#19363 <http://tracker.ceph.com/issues/19363>`_, `pr#14160 <https://github.com/ceph/ceph/pull/14160>`_, liuchang0812)
+* rgw: RGWMetaSyncShardControlCR retries with backoff on all error codes (`issue#19019 <http://tracker.ceph.com/issues/19019>`_, `pr#13546 <https://github.com/ceph/ceph/pull/13546>`_, Casey Bodley)
+* rgw: RGWMetaSyncShardCR drops stack refs on destruction (`issue#18412 <http://tracker.ceph.com/issues/18412>`_, `issue#18300 <http://tracker.ceph.com/issues/18300>`_, `pr#12605 <https://github.com/ceph/ceph/pull/12605>`_, Casey Bodley)
+* rgw: rgw multisite: automated mdlog trimming (`pr#13111 <https://github.com/ceph/ceph/pull/13111>`_, Casey Bodley)
+* rgw: rgw multisite: feature of bucket sync enable/disable (`pr#15801 <https://github.com/ceph/ceph/pull/15801>`_, Zhang Shaowen, Casey Bodley, Zengran Zhang)
+* rgw: rgw multisite: fixes for meta sync across periods (`issue#18639 <http://tracker.ceph.com/issues/18639>`_, `pr#13070 <https://github.com/ceph/ceph/pull/13070>`_, Casey Bodley)
+* rgw: rgw multisite: fix ref counting of completions (`issue#18414 <http://tracker.ceph.com/issues/18414>`_, `issue#18407 <http://tracker.ceph.com/issues/18407>`_, `pr#12841 <https://github.com/ceph/ceph/pull/12841>`_, Casey Bodley)
+* rgw: rgw-multisite: fix the problem of rgw website configure 'RedirectAllRequestsTo' failed to sync to slave zone (`pr#15036 <https://github.com/ceph/ceph/pull/15036>`_, yuliyang)
+* rgw: rgw-multisite: fix the problem of rgw website configure request not redirect to metadata master (`pr#15082 <https://github.com/ceph/ceph/pull/15082>`_, yuliyang)
+* rgw: rgw multisite: remove the redundant post in OPT_ZONEGROUP_MODIFY (`pr#14359 <https://github.com/ceph/ceph/pull/14359>`_, Jing Wenjun)
+* rgw: rgw/multisite: validate bucket location during bucket creation (`pr#15333 <https://github.com/ceph/ceph/pull/15333>`_, Jiaying Ren)
+* rgw: RGW NFS: add nfs.rst to doc/radosgw (`pr#15789 <https://github.com/ceph/ceph/pull/15789>`_, Matt Benjamin)
+* rgw: rgw_op: remove unused variable iter (`pr#14276 <https://github.com/ceph/ceph/pull/14276>`_, Weibing Zhang)
+* rgw: RGWPeriodPusher spawns http thread before cr thread (`issue#19834 <http://tracker.ceph.com/issues/19834>`_, `pr#14936 <https://github.com/ceph/ceph/pull/14936>`_, Casey Bodley)
+* rgw: rgw_rados: create sync module instances only if run_sync_thread is set (`issue#19830 <http://tracker.ceph.com/issues/19830>`_, `pr#14994 <https://github.com/ceph/ceph/pull/14994>`_, Abhishek Lekshmanan)
+* rgw: rgw_rados drop deprecated global var (`pr#14411 <https://github.com/ceph/ceph/pull/14411>`_, Jiaying Ren)
+* rgw: rgw_rados: initialize cur_shard (`pr#15735 <https://github.com/ceph/ceph/pull/15735>`_, Abhishek Lekshmanan)
+* rgw: rgw realm set fixes (`issue#18333 <http://tracker.ceph.com/issues/18333>`_, `pr#12731 <https://github.com/ceph/ceph/pull/12731>`_, Orit Wasserman)
+* rgw: rgw/rgw_frontend.h: Return negative value for empty uid in RGWLoadGenFrontend::init() (`pr#16204 <https://github.com/ceph/ceph/pull/16204>`_, jimifm)
+* rgw: rgw/rgw_main.cc: fix parenteses and function result (`pr#12295 <https://github.com/ceph/ceph/pull/12295>`_, Willem Jan Withagen)
+* rgw: rgw/rgw_op:Prevents memory leaks when calling func swift_versioning_copy() fails (`pr#15328 <https://github.com/ceph/ceph/pull/15328>`_, jimifm)
+* rgw: rgw/rgw_rados: Remove duplicate calls in RGWRados::finalize() (`pr#15281 <https://github.com/ceph/ceph/pull/15281>`_, jimifm)
+* rgw: rgw/rgw_string.h: FreeBSD would like errno.h included (`pr#15737 <https://github.com/ceph/ceph/pull/15737>`_, Willem Jan Withagen)
+* rgw: rgw/rgw_swift_auth.cc: using string::back() instead as the C++11 recommend (`pr#14827 <https://github.com/ceph/ceph/pull/14827>`_, liuyuhong)
+* rgw: rgw structures rework (`issue#17996 <http://tracker.ceph.com/issues/17996>`_, `issue#19249 <http://tracker.ceph.com/issues/19249>`_, `pr#11485 <https://github.com/ceph/ceph/pull/11485>`_, Yehuda Sadeh)
+* rgw: rgw,test: fix rgw placement rule pool config option (`pr#16084 <https://github.com/ceph/ceph/pull/16084>`_, Jiaying Ren)
+* rgw: S3 lifecycle now supports expiration date (`pr#15807 <https://github.com/ceph/ceph/pull/15807>`_, Zhang Shaowen)
+* rgw: s3 server-side encryption (SSE-C, SSE-KMS) (`pr#11049 <https://github.com/ceph/ceph/pull/11049>`_, Adam Kupczyk, Casey Bodley, Radoslaw Zarzynski)
+* rgw: segment fault when shard id out of range (`issue#19732 <http://tracker.ceph.com/issues/19732>`_, `pr#14389 <https://github.com/ceph/ceph/pull/14389>`_, redickwang)
+* rgw: set dumpable flag after setuid post ff0e521 (`issue#19089 <http://tracker.ceph.com/issues/19089>`_, `pr#13657 <https://github.com/ceph/ceph/pull/13657>`_, Brad Hubbard)
+* rgw: set FCGI_INCLUDE_DIR for cephd_rgw_base (`issue#18918 <http://tracker.ceph.com/issues/18918>`_, `pr#13393 <https://github.com/ceph/ceph/pull/13393>`_, David Disseldorp)
+* rgw: set object accounted size correctly (`issue#20071 <http://tracker.ceph.com/issues/20071>`_, `pr#14950 <https://github.com/ceph/ceph/pull/14950>`_, fang yuxiang)
+* rgw: set placement rule properly (`pr#15221 <https://github.com/ceph/ceph/pull/15221>`_, fang.yuxiang)
+* rgw: should delete in_stream_req  if conn->get_obj(...) return not zero value (`pr#9950 <https://github.com/ceph/ceph/pull/9950>`_, weiqiaomiao)
+* rgw: should not restrict location_constraint same when user not provide (`pr#16770 <https://github.com/ceph/ceph/pull/16770>`_, Tianshan Qu)
+* rgw: should unlock when reshard_log->update() reture non-zero in RGWB… (`pr#16502 <https://github.com/ceph/ceph/pull/16502>`_, Wei Qiaomiao)
+* rgw: silence compile warning from -Wmaybe-uninitialized (`pr#15996 <https://github.com/ceph/ceph/pull/15996>`_, Jiaying Ren)
+* rgw: silence warning from -Wmaybe-uninitialized (`pr#15949 <https://github.com/ceph/ceph/pull/15949>`_, Jos Collin)
+* rgw: stat requests skip compression, manifest handling, etc (`pr#14109 <https://github.com/ceph/ceph/pull/14109>`_, Casey Bodley)
+* rgw: Support certain archaic and antiquated distributions(`pr#15498 <https://github.com/ceph/ceph/pull/15498>`_, Adam C. Emerson)
+* rgw: swift: ability to update swift read and write acls separately (`issue#19289 <http://tracker.ceph.com/issues/19289>`_, `pr#14499 <https://github.com/ceph/ceph/pull/14499>`_, Marcus Watts)
+* rgw: swift: disable revocation thread if sleep == 0 (`issue#19499 <http://tracker.ceph.com/issues/19499>`_, `issue#9493 <http://tracker.ceph.com/issues/9493>`_, `pr#14501 <https://github.com/ceph/ceph/pull/14501>`_, Marcus Watts)
+* rgw: swift: fix anonymous user's error code of getting object (`issue#18806 <http://tracker.ceph.com/issues/18806>`_, `pr#13242 <https://github.com/ceph/ceph/pull/13242>`_, Jing Wenjun)
+* rgw: swift: the http referer acl in swift API should be shown (`issue#18665 <http://tracker.ceph.com/issues/18665>`_, `pr#13003 <https://github.com/ceph/ceph/pull/13003>`_, Jing Wenjun)
+* rgw: swift: The http referer should be parsed to compare in swift API (`issue#18685 <http://tracker.ceph.com/issues/18685>`_, `pr#13005 <https://github.com/ceph/ceph/pull/13005>`_, Jing Wenjun)
+* rgw: switch from "timegm()" to "internal_timegm()" for better portability (`issue#12863 <http://tracker.ceph.com/issues/12863>`_, `pr#14327 <https://github.com/ceph/ceph/pull/14327>`_, Rishabh Kumar)
+* rgw: switch to std::array in RGWBulkUploadOp due to C++11 and FreeBSD (`pr#14314 <https://github.com/ceph/ceph/pull/14314>`_, Radoslaw Zarzynski)
+* rgw: sync status compares the current master period (`issue#18064 <http://tracker.ceph.com/issues/18064>`_, `pr#12907 <https://github.com/ceph/ceph/pull/12907>`_, Abhishek Lekshmanan)
+* rgw: test,rgw: fix rgw placement rule pool config option (`pr#16380 <https://github.com/ceph/ceph/pull/16380>`_, Jiaying Ren)
+* rgw,tests: luminous: qa/rgw: use 'ceph osd pool application enable' on created pools (`pr#17259 <https://github.com/ceph/ceph/pull/17259>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add cluster name to path when s3tests scans rgw log (`pr#14845 <https://github.com/ceph/ceph/pull/14845>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add configuration for server-side encryption tests (`pr#13597 <https://github.com/ceph/ceph/pull/13597>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add encryption config for s3tests under thrash (`pr#15694 <https://github.com/ceph/ceph/pull/15694>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add multisite suite to configure and run multisite tests (`pr#14688 <https://github.com/ceph/ceph/pull/14688>`_, Casey Bodley)
+* rgw,tests: qa/rgw: disable lifecycle tests because of expiration failures (`pr#16760 <https://github.com/ceph/ceph/pull/16760>`_, Casey Bodley)
+* rgw,tests: qa/rgw: don't scan radosgw logs for encryption keys on jewel upgrade test (`pr#14697 <https://github.com/ceph/ceph/pull/14697>`_, Casey Bodley)
+* rgw,tests: qa/rgw: fix assertions in radosgw_admin task (`pr#14842 <https://github.com/ceph/ceph/pull/14842>`_, Casey Bodley)
+* rgw,tests: qa/rgw: remove apache/fastcgi and radosgw-agent tests (`pr#15184 <https://github.com/ceph/ceph/pull/15184>`_, Casey Bodley)
+* rgw,tests: qa/suites/rgw/thrash: add osd thrashing tests (`pr#13445 <https://github.com/ceph/ceph/pull/13445>`_, Sage Weil)
+* rgw,tests: qa/tasks: S3A hadoop task to test s3a with Ceph (`pr#14624 <https://github.com/ceph/ceph/pull/14624>`_, Vasu Kulkarni)
+* rgw,tests: test/rgw: add bucket acl and versioning tests to test_multi.py (`pr#12449 <https://github.com/ceph/ceph/pull/12449>`_, Casey Bodley)
+* rgw,tests: test/rgw: add test for versioned object sync (`pr#12474 <https://github.com/ceph/ceph/pull/12474>`_, Casey Bodley)
+* rgw,tests: test/rgw: fixes for test_multi_period_incremental_sync() (`pr#13067 <https://github.com/ceph/ceph/pull/13067>`_, Casey Bodley)
+* rgw,tests: test/rgw: fix for empty lists as default arguments (`pr#14816 <https://github.com/ceph/ceph/pull/14816>`_, Casey Bodley)
+* rgw,tests: test/rgw: refactor test_multi.py for use in qa suite (`pr#14433 <https://github.com/ceph/ceph/pull/14433>`_, Casey Bodley)
+* rgw,tests: test/rgw: test_bucket_delete_notempty in test_multi.py (`pr#14090 <https://github.com/ceph/ceph/pull/14090>`_, Casey Bodley)
+* rgw,tests: vstart: add rgw configuration needed to pass all s3tests (`pr#15782 <https://github.com/ceph/ceph/pull/15782>`_, Casey Bodley)
+* rgw,tests: vstart: remove rgw_enable_static_website (`pr#15856 <https://github.com/ceph/ceph/pull/15856>`_, Casey Bodley)
+* rgw: the swift container acl should support field .ref (`issue#18484 <http://tracker.ceph.com/issues/18484>`_, `pr#12874 <https://github.com/ceph/ceph/pull/12874>`_, Jing Wenjun)
+* rgw: Turn off fcgi as a frontend (`issue#16784 <http://tracker.ceph.com/issues/16784>`_, `pr#15070 <https://github.com/ceph/ceph/pull/15070>`_, Thomas Serlin)
+* rgw: Uninitialized member in LCRule (`pr#15827 <https://github.com/ceph/ceph/pull/15827>`_, Jos Collin)
+* rgw: update Beast for streaming reads in asio frontend (`pr#14273 <https://github.com/ceph/ceph/pull/14273>`_, Casey Bodley)
+* rgw: update bucket cors in secondary zonegroup should forward to master (`issue#16888 <http://tracker.ceph.com/issues/16888>`_, `pr#15260 <https://github.com/ceph/ceph/pull/15260>`_, Shasha Lu)
+* rgw: update function doc in rgw_rados.h  and rgw_rados.cc (`pr#15803 <https://github.com/ceph/ceph/pull/15803>`_, Jiaying Ren)
+* rgw: update is_truncated in function rgw_read_user_buckets (`issue#19365 <http://tracker.ceph.com/issues/19365>`_, `pr#14343 <https://github.com/ceph/ceph/pull/14343>`_, liuchang0812)
+* rgw: usage (`issue#16191 <http://tracker.ceph.com/issues/16191>`_, `pr#14287 <https://github.com/ceph/ceph/pull/14287>`_, Ji Chen, Orit Wasserman)
+* rgw: use 64-bit offsets for compression (`issue#20231 <http://tracker.ceph.com/issues/20231>`_, `pr#15656 <https://github.com/ceph/ceph/pull/15656>`_, Adam Kupczyk, fang yuxiang)
+* rgw: use a namespace for rgw reshard pool for upgrades as well (`issue#20289 <http://tracker.ceph.com/issues/20289>`_, `pr#16368 <https://github.com/ceph/ceph/pull/16368>`_, Karol Mroz, Abhishek Lekshmanan)
+* rgw: Use comparison instead of assignment (`pr#16653 <https://github.com/ceph/ceph/pull/16653>`_, amitkuma)
+* rgw: Use decoded URI when verifying TempURL (`issue#18590 <http://tracker.ceph.com/issues/18590>`_, `pr#13007 <https://github.com/ceph/ceph/pull/13007>`_, Michal Koutný)
+* rgw: use get_data_extra_pool() when get extra pool (`issue#20064 <http://tracker.ceph.com/issues/20064>`_, `pr#15219 <https://github.com/ceph/ceph/pull/15219>`_, fang yuxiang)
+* rgw: use pre-defined calls to replace raw flag operation (`pr#15107 <https://github.com/ceph/ceph/pull/15107>`_, Gui Hecheng)
+* rgw: use rgw_zone_root_pool for region_map like is done in hammer (`issue#19195 <http://tracker.ceph.com/issues/19195>`_, `pr#13928 <https://github.com/ceph/ceph/pull/13928>`_, Orit Wasserman)
+* rgw: use separate http_manager for read_sync_status (`issue#19236 <http://tracker.ceph.com/issues/19236>`_, `pr#13660 <https://github.com/ceph/ceph/pull/13660>`_, Shasha Lu)
+* rgw: use uncompressed size for range_to_ofs() in slo/dlo (`pr#15931 <https://github.com/ceph/ceph/pull/15931>`_, Casey Bodley)
+* rgw: using RGW_OBJ_NS_MULTIPART in check_bad_index_multipart (`pr#15774 <https://github.com/ceph/ceph/pull/15774>`_, Shasha Lu)
+* rgw: using the same bucket num_shards as master zg when create bucket in secondary zg (`issue#19745 <http://tracker.ceph.com/issues/19745>`_, `pr#14388 <https://github.com/ceph/ceph/pull/14388>`_, Shasha Lu)
+* rgw: validate tenant names during user create (`pr#16442 <https://github.com/ceph/ceph/pull/16442>`_, Abhishek Lekshmanan)
+* rgw: verify md5 in post obj (`issue#19739 <http://tracker.ceph.com/issues/19739>`_, `pr#14961 <https://github.com/ceph/ceph/pull/14961>`_, Yehuda Sadeh)
+* rgw: version id doesn't work in fetch_remote_obj (`pr#14010 <https://github.com/ceph/ceph/pull/14010>`_, Zhang Shaowen)
+* rgw: VersionIdMarker and NextVersionIdMarker should be returned when listing object versions (`issue#19886 <http://tracker.ceph.com/issues/19886>`_, `pr#15014 <https://github.com/ceph/ceph/pull/15014>`_, Zhang Shaowen)
+* rgw: warning, output may be truncated before the last format character (`pr#14194 <https://github.com/ceph/ceph/pull/14194>`_, Jos Collin)
+* rgw: when create_bucket use the same num_shards with info.num_shards (`issue#19745 <http://tracker.ceph.com/issues/19745>`_, `pr#15010 <https://github.com/ceph/ceph/pull/15010>`_, Shasha Lu)
+* rgw: wip dir orphan (`issue#18992 <http://tracker.ceph.com/issues/18992>`_, `issue#18989 <http://tracker.ceph.com/issues/18989>`_, `issue#19018 <http://tracker.ceph.com/issues/19018>`_, `issue#18991 <http://tracker.ceph.com/issues/18991>`_, `pr#13529 <https://github.com/ceph/ceph/pull/13529>`_, Matt Benjamin)
+* rgw: Wip librgw refcnt (`pr#13405 <https://github.com/ceph/ceph/pull/13405>`_, Matt Benjamin)
+* rgw: wip parentref (`issue#19060 <http://tracker.ceph.com/issues/19060>`_, `issue#19059 <http://tracker.ceph.com/issues/19059>`_, `pr#13607 <https://github.com/ceph/ceph/pull/13607>`_, Matt Benjamin)
+* rgw: Wip rgw fix prefix list (`issue#19432 <http://tracker.ceph.com/issues/19432>`_, `pr#15916 <https://github.com/ceph/ceph/pull/15916>`_, Giovani Rinaldi, Orit Wasserman)
+* rgw: Wip rgw openssl 7 (`issue#11239 <http://tracker.ceph.com/issues/11239>`_, `issue#16535 <http://tracker.ceph.com/issues/16535>`_, `pr#11776 <https://github.com/ceph/ceph/pull/11776>`_, Yehuda Sadeh, Marcus Watts)
+* rgw: wip: rgw: rest_admin/user avoid double checking input args (`pr#13460 <https://github.com/ceph/ceph/pull/13460>`_, Abhishek Lekshmanan)
+* tests: Add integration tests for admin socket output (`pr#15223 <https://github.com/ceph/ceph/pull/15223>`_, Brad Hubbard)
+* tests: add MGR=1 so 'pg dump' won't be blocked (`pr#14266 <https://github.com/ceph/ceph/pull/14266>`_, Kefu Chai)
+* tests: Add openstack requirements to smoke suite (`pr#12913 <https://github.com/ceph/ceph/pull/12913>`_, Zack Cerza)
+* tests: add setup/teardown for asok dir (`pr#16523 <https://github.com/ceph/ceph/pull/16523>`_, Kefu Chai)
+* tests: buildpackages: remove because it does not belong (`issue#18846 <http://tracker.ceph.com/issues/18846>`_, `pr#13297 <https://github.com/ceph/ceph/pull/13297>`_, Loic Dachary)
+* tests: ceph-disk: add setting for external py-modules for tox-testing (`pr#15433 <https://github.com/ceph/ceph/pull/15433>`_, Willem Jan Withagen)
+* tests: ceph-disk: use communicate() instead of wait() for output (`pr#16347 <https://github.com/ceph/ceph/pull/16347>`_, Kefu Chai)
+* tests: ceph-helpers.sh reduce get_timeout_delays() verbosity (`pr#13257 <https://github.com/ceph/ceph/pull/13257>`_, Kefu Chai)
+* tests: ceph_objectstore_tool.py: kill all daemons (`pr#14428 <https://github.com/ceph/ceph/pull/14428>`_, Kefu Chai)
+* tests: ceph_test_objectstore: tolerate fsck EOPNOTSUPP too (`pr#13325 <https://github.com/ceph/ceph/pull/13325>`_, Sage Weil)
+* tests: ceph_test_rados_api_tier: tolerate ENOENT from 'pg scrub' (`pr#14807 <https://github.com/ceph/ceph/pull/14807>`_, Sage Weil)
+* tests: ceph_test_rados_api_watch_notify: move global variables into test class (`issue#18395 <http://tracker.ceph.com/issues/18395>`_, `pr#12751 <https://github.com/ceph/ceph/pull/12751>`_, Kefu Chai)
+* tests: ceph_test_rados_api_watch_notify: test timeout using rados_wat… (`issue#19312 <http://tracker.ceph.com/issues/19312>`_, `pr#14061 <https://github.com/ceph/ceph/pull/14061>`_, Kefu Chai)
+* tests: cephtool/test.sh error on full tests (`issue#19698 <http://tracker.ceph.com/issues/19698>`_, `pr#14647 <https://github.com/ceph/ceph/pull/14647>`_, Willem Jan Withagen, David Zafman)
+* tests: cephtool/test.sh: Only delete a test pool when no longer needed (`pr#16443 <https://github.com/ceph/ceph/pull/16443>`_, Willem Jan Withagen)
+* tests: cls_lock: move lock_info_t definition to cls_lock_types.h (`pr#16091 <https://github.com/ceph/ceph/pull/16091>`_, runsisi)
+* tests: config_opts: drop unused opts (`pr#15031 <https://github.com/ceph/ceph/pull/15031>`_, Kefu Chai)
+* tests: Decreased amount of jobs on master, kraken, luminous runs (`pr#17074 <https://github.com/ceph/ceph/pull/17074>`_, Yuri Weinstein)
+* tests: Don't dump core when using EXPECT_DEATH (`pr#14821 <https://github.com/ceph/ceph/pull/14821>`_, Kefu Chai, Brad Hubbard)
+* tests: drop buildpackages.py (`issue#18846 <http://tracker.ceph.com/issues/18846>`_, `pr#13319 <https://github.com/ceph/ceph/pull/13319>`_, Nathan Cutler)
+* tests: drop obsolete Perl scripts (`pr#13951 <https://github.com/ceph/ceph/pull/13951>`_, Nathan Cutler)
+* tests: drop rbd_cli_tests.pl and RbdLib.pm (`issue#14825 <http://tracker.ceph.com/issues/14825>`_, `pr#12821 <https://github.com/ceph/ceph/pull/12821>`_, Nathan Cutler)
+* tests: drop unused rbd_functional_tests.pl script (`issue#14825 <http://tracker.ceph.com/issues/14825>`_, `pr#12818 <https://github.com/ceph/ceph/pull/12818>`_, Nathan Cutler)
+* tests: fio_ceph_objectstore: fixes improper write request data lifetime (`pr#14338 <https://github.com/ceph/ceph/pull/14338>`_, Adam Kupczyk)
+* tests: fix broken links in upgrade/hammer-jewel-x/stress-split (`issue#19793 <http://tracker.ceph.com/issues/19793>`_, `pr#14831 <https://github.com/ceph/ceph/pull/14831>`_, Nathan Cutler)
+* tests: fix NULL references to be acceptable by Clang (`pr#12880 <https://github.com/ceph/ceph/pull/12880>`_, Willem Jan Withagen)
+* tests: fix rados/upgrade/jewel-x-singleton and make workunit task handle repo URLs not ending in ".git" (`issue#20554 <http://tracker.ceph.com/issues/20554>`_, `issue#20368 <http://tracker.ceph.com/issues/20368>`_, `pr#16228 <https://github.com/ceph/ceph/pull/16228>`_, Nathan Cutler, Sage Weil)
+* tests: fix regression in qa/tasks/ceph_master.py (`issue#16263 <http://tracker.ceph.com/issues/16263>`_, `pr#13279 <https://github.com/ceph/ceph/pull/13279>`_, Nathan Cutler, Kefu Chai)
+* tests: fix template specialization of PromoteRequest class (`pr#12815 <https://github.com/ceph/ceph/pull/12815>`_, Ricardo Dias)
+* tests: ignore bogus ceph-objectstore-tool error in ceph_manager (`issue#16263 <http://tracker.ceph.com/issues/16263>`_, `pr#13194 <https://github.com/ceph/ceph/pull/13194>`_, Nathan Cutler)
+* tests: include/denc: support ENCODE_DUMP (`pr#14962 <https://github.com/ceph/ceph/pull/14962>`_, Sage Weil)
+* tests: libradosstriper: do not assign garbage to returned value (`pr#15009 <https://github.com/ceph/ceph/pull/15009>`_, Kefu Chai)
+* tests: luminous: tests: qa/standalone: misc fixes (`issue#20465 <http://tracker.ceph.com/issues/20465>`_, `issue#20921 <http://tracker.ceph.com/issues/20921>`_, `issue#20979 <http://tracker.ceph.com/issues/20979>`_, `pr#16985 <https://github.com/ceph/ceph/pull/16985>`_, David Zafman)
+* tests: mgr,os,test: kill clang analyzer warnings (`pr#16227 <https://github.com/ceph/ceph/pull/16227>`_, Kefu Chai)
+* tests: move swift.py task from teuthology to ceph, phase one (master) (`issue#20392 <http://tracker.ceph.com/issues/20392>`_, `pr#15859 <https://github.com/ceph/ceph/pull/15859>`_, Nathan Cutler, Sage Weil, Warren Usui, Greg Farnum, Ali Maredia, Tommi Virtanen, Zack Cerza, Sam Lang, Yehuda Sadeh, Joe Buck, Josh Durgin)
+* tests: nosetests: use /usr/bin/env to find nosetests (`pr#12091 <https://github.com/ceph/ceph/pull/12091>`_, Willem Jan Withagen)
+* tests: os: Argument cannot be negative (`pr#16688 <https://github.com/ceph/ceph/pull/16688>`_, amitkuma)
+* tests: os/bluestore,test/ceph_test_objectstore: silence gcc warnings (`pr#13924 <https://github.com/ceph/ceph/pull/13924>`_, Kefu Chai)
+* tests: osd-scrub-repair.sh disable scrub backoff in test (`pr#13334 <https://github.com/ceph/ceph/pull/13334>`_, Kefu Chai)
+* tests: qa: Added luminous to the mix in schedule_subset.sh (`pr#16430 <https://github.com/ceph/ceph/pull/16430>`_, Yuri Weinstein)
+* tests: qa/added overrides (`pr#14917 <https://github.com/ceph/ceph/pull/14917>`_, Yuri Weinstein)
+* tests: qa: Add reboot case for systemd test (`issue#19717 <http://tracker.ceph.com/issues/19717>`_, `pr#14229 <https://github.com/ceph/ceph/pull/14229>`_, Vasu Kulkarni)
+* tests: qa: add supported distros for ceph-ansible (`pr#13711 <https://github.com/ceph/ceph/pull/13711>`_, Tamil Muthamizhan)
+* tests: qa: add task for dnsmasq configuration (`pr#15071 <https://github.com/ceph/ceph/pull/15071>`_, Casey Bodley)
+* tests: [qa/ceph-deploy]:  run create mgr nodes as well (`pr#16216 <https://github.com/ceph/ceph/pull/16216>`_, Vasu Kulkarni)
+* tests: qa: Cleaned up distros to use `latest` versions (`pr#12804 <https://github.com/ceph/ceph/pull/12804>`_, Yuri Weinstein)
+* tests: qa/distros: make centos_latest 7.3 (`pr#12944 <https://github.com/ceph/ceph/pull/12944>`_, Sage Weil)
+* tests: qa,doc: document and fix tests for pool application warnings (`pr#16568 <https://github.com/ceph/ceph/pull/16568>`_, Sage Weil)
+* tests: qa: do not mention ceph branch explicitly (`pr#13225 <https://github.com/ceph/ceph/pull/13225>`_, Tamil Muthamizhan)
+* tests: qa: do not restrict valgrind runs to centos (`issue#18126 <http://tracker.ceph.com/issues/18126>`_, `pr#15893 <https://github.com/ceph/ceph/pull/15893>`_, Greg Farnum)
+* tests: qa/erasure-code: override min_size to 2 (`issue#19770 <http://tracker.ceph.com/issues/19770>`_, `pr#14872 <https://github.com/ceph/ceph/pull/14872>`_, Kefu Chai)
+* tests: qa: fixed distros links (`pr#12770 <https://github.com/ceph/ceph/pull/12770>`_, Yuri Weinstein)
+* tests: qa/run-standalone.sh: fix the find option to be compatible with GNU find (`pr#16646 <https://github.com/ceph/ceph/pull/16646>`_, Kefu Chai)
+* tests: qa: specify client for fs workunit (`pr#12914 <https://github.com/ceph/ceph/pull/12914>`_, Tamil Muthamizhan)
+* tests: qa: split test_tiering into smaller pieces (`pr#15146 <https://github.com/ceph/ceph/pull/15146>`_, Kefu Chai)
+* tests: qa/suite: Added a smoke suite for ceph-ansible (`pr#12610 <https://github.com/ceph/ceph/pull/12610>`_, Tamil Muthamizhan)
+* tests: qa/suite: replace reference to fs/xfs.yaml (`pr#14756 <https://github.com/ceph/ceph/pull/14756>`_, Yehuda Sadeh)
+* tests: qa/suites/ceph-ansible: removing fs workunit (`pr#12928 <https://github.com/ceph/ceph/pull/12928>`_, Tamil Muthamizhan)
+* tests: qa/suites/{ceph-ansible,rest}: OpenStack volumes (`pr#13672 <https://github.com/ceph/ceph/pull/13672>`_, Zack Cerza)
+* tests: qa/suites/ceph-deploy: Drop OpenStack volume count (`pr#13706 <https://github.com/ceph/ceph/pull/13706>`_, Zack Cerza)
+* tests: qa/suites: drop 'fs' facet, and add 'objectstore' facet where missing (`pr#14198 <https://github.com/ceph/ceph/pull/14198>`_, Sage Weil)
+* tests: qa/suites: escape the parenthesis of the whitelist text (`pr#16722 <https://github.com/ceph/ceph/pull/16722>`_, Kefu Chai)
+* tests: qa/suites: fix upgrade tests vs cluster full thrashing (`pr#13852 <https://github.com/ceph/ceph/pull/13852>`_, Sage Weil)
+* tests: qa/suites/fs: Add openstack volume configuration (`pr#13640 <https://github.com/ceph/ceph/pull/13640>`_, Zack Cerza)
+* tests: qa/suites/jewel-x/point-to-point: don't scane for keys on second s3tests either (`pr#14788 <https://github.com/ceph/ceph/pull/14788>`_, Sage Weil)
+* tests: qa/suites/kcephfs: Openstack volume configuration (`pr#13634 <https://github.com/ceph/ceph/pull/13634>`_, Zack Cerza)
+* tests: qa/suites/{knfs,hadoop,samba}: OpenStack volume configuration (`pr#13637 <https://github.com/ceph/ceph/pull/13637>`_, Zack Cerza)
+* tests: qa/suites/krbd: Add openstack volume configuration (`pr#13631 <https://github.com/ceph/ceph/pull/13631>`_, Zack Cerza)
+* tests: qa/suites/powercycle/osd/whitelist_health: whitelist more (`pr#17306 <https://github.com/ceph/ceph/pull/17306>`_, Sage Weil)
+* tests: qa/suites/powercycle: whitelist health for thrashing (`pr#16759 <https://github.com/ceph/ceph/pull/16759>`_, Sage Weil)
+* tests: qa/suites/rados: a bit more whitelisting (`pr#16820 <https://github.com/ceph/ceph/pull/16820>`_, Sage Weil)
+* tests: qa/suites/rados: fix ec thrashing (`pr#15087 <https://github.com/ceph/ceph/pull/15087>`_, Sage Weil)
+* tests: qa/suites/rados/objectstore: enable experimental features for testing bluestore (`pr#13456 <https://github.com/ceph/ceph/pull/13456>`_, Kefu Chai, Dan Mick)
+* tests: qa/suites/rados/singleton/all/erasure-code-nonregression: fix typo (`pr#16579 <https://github.com/ceph/ceph/pull/16579>`_, Sage Weil)
+* tests: qa/suites/rados/singleton/all/mon-auth-caps: more osds so we can go clean (`pr#16225 <https://github.com/ceph/ceph/pull/16225>`_, Sage Weil)
+* tests: qa/suites/rados/singleton-bluestore: concat settings (`pr#14884 <https://github.com/ceph/ceph/pull/14884>`_, Kefu Chai)
+* tests: qa/suites/rados/singleton-nomsgr/all/multi-backfill-reject: sleep longer (`pr#16739 <https://github.com/ceph/ceph/pull/16739>`_, Sage Weil)
+* tests: qa/suites/rados/singleton-nomsgr: fix syntax (`pr#15276 <https://github.com/ceph/ceph/pull/15276>`_, Sage Weil)
+* tests: qa/suites/rados/thrash: make sure osds have map before legacy scrub (`pr#15117 <https://github.com/ceph/ceph/pull/15117>`_, Sage Weil)
+* tests: qa/suites/rados/upgrade: restart mds (`pr#15517 <https://github.com/ceph/ceph/pull/15517>`_, Sage Weil)
+* tests: qa/suites: Reduce fs combination tests for smoke, use bluestore (`pr#14854 <https://github.com/ceph/ceph/pull/14854>`_, Vasu Kulkarni)
+* tests: qa/suites: Revert "qa/suites: add mon-reweight-min-pgs-per-osd = 4" (`pr#14584 <https://github.com/ceph/ceph/pull/14584>`_, Kefu Chai)
+* tests: qa/suites/rgw: Add openstack volume configuration (`pr#13611 <https://github.com/ceph/ceph/pull/13611>`_, Zack Cerza)
+* tests: qa/suites/upgarde/jewel-x/parallel: more whitelisting (`pr#16849 <https://github.com/ceph/ceph/pull/16849>`_, Sage Weil)
+* tests: qa/suites/upgrade: add tiering test to hammer-jewel-x (`issue#19185 <http://tracker.ceph.com/issues/19185>`_, `pr#13805 <https://github.com/ceph/ceph/pull/13805>`_, Kefu Chai)
+* tests: qa/suites/upgrade/hammer-jewel-x: add luminous.yaml (`issue#20342 <http://tracker.ceph.com/issues/20342>`_, `pr#15764 <https://github.com/ceph/ceph/pull/15764>`_, Kefu Chai)
+* tests: qa/suites/upgrade/jewel-x: add mgr.x role (`pr#14689 <https://github.com/ceph/ceph/pull/14689>`_, Sage Weil)
+* tests: qa/suites/upgrade/jewel-x: misc fixes for new health checks (`pr#16429 <https://github.com/ceph/ceph/pull/16429>`_, Sage Weil)
+* tests: qa/suites/upgrade/kraken-x: do not thrash cluster full during upgrade (`issue#19232 <http://tracker.ceph.com/issues/19232>`_, `pr#13892 <https://github.com/ceph/ceph/pull/13892>`_, Dan Mick)
+* tests: qa/suites/upgrade/kraken-x: misc fixes (`pr#14887 <https://github.com/ceph/ceph/pull/14887>`_, Sage Weil)
+* tests: qa/suites/upgrade/kraken-x (`pr#13517 <https://github.com/ceph/ceph/pull/13517>`_, Sage Weil, Yuri Weinstein)
+* tests: qa/suites/upgrade: set "sortbitwise" for jewel clusters (`pr#15661 <https://github.com/ceph/ceph/pull/15661>`_, Kefu Chai)
+* tests: qa/suite/upgrade/jewel-x: various fixes (`pr#13734 <https://github.com/ceph/ceph/pull/13734>`_, Sage Weil)
+* tests: qa/tasks: assert on pg status with a timeout (`issue#19594 <http://tracker.ceph.com/issues/19594>`_, `pr#14608 <https://github.com/ceph/ceph/pull/14608>`_, Kefu Chai)
+* tests: qa/tasks/ceph: debug osd setup (`pr#16841 <https://github.com/ceph/ceph/pull/16841>`_, Sage Weil)
+* tests: qa/tasks/ceph-deploy: create-keys explicitly (`pr#12867 <https://github.com/ceph/ceph/pull/12867>`_, Vasu Kulkarni)
+* tests: qa/tasks/ceph-deploy: Fix bluestore options for ceph-deploy (`pr#16571 <https://github.com/ceph/ceph/pull/16571>`_, Vasu Kulkarni)
+* tests: qa/tasks/ceph-deploy: use the new create option during instantiation (`pr#12892 <https://github.com/ceph/ceph/pull/12892>`_, Vasu Kulkarni)
+* tests: qa/tasks/ceph: don't hard-code cluster name when copying fsid (`pr#16212 <https://github.com/ceph/ceph/pull/16212>`_, Jason Dillaman)
+* tests: qa/tasks/ceph_manager: always fix pgp_num when done with thrashosd task (`issue#19771 <http://tracker.ceph.com/issues/19771>`_, `pr#14931 <https://github.com/ceph/ceph/pull/14931>`_, Kefu Chai)
+* tests: qa/tasks/ceph_manager: 'ceph $service tell ...' is obsolete (`pr#15252 <https://github.com/ceph/ceph/pull/15252>`_, Sage Weil)
+* tests: qa/tasks/ceph.py: debug which pgs aren't scrubbing (`pr#13649 <https://github.com/ceph/ceph/pull/13649>`_, Sage Weil)
+* tests: qa/tasks/ceph: raise exceptions if scrubbing fails or cannot proceed (`pr#15310 <https://github.com/ceph/ceph/pull/15310>`_, Sage Weil)
+* tests: qa/tasks/ceph: should be "Waiting for all PGs", not "all osds" (`pr#16122 <https://github.com/ceph/ceph/pull/16122>`_, Kefu Chai)
+* tests: qa/tasks/ceph: wait longer for scrub (`pr#16824 <https://github.com/ceph/ceph/pull/16824>`_, Sage Weil)
+* tests: qa/tasks: few fixes to get ceph-deploy 1node to working state (`pr#14400 <https://github.com/ceph/ceph/pull/14400>`_, Vasu Kulkarni)
+* tests: qa/tasks/radosbench: increase timeout (`pr#15885 <https://github.com/ceph/ceph/pull/15885>`_, Sage Weil)
+* tests: qa/tasks/rebuild_mondb: grant "mgr:allow \*" to client.admin (`issue#19439 <http://tracker.ceph.com/issues/19439>`_, `pr#14284 <https://github.com/ceph/ceph/pull/14284>`_, Kefu Chai)
+* tests: qa/tasks/reg11184: use literal 'foo' instead pool_name (`pr#16451 <https://github.com/ceph/ceph/pull/16451>`_, Kefu Chai)
+* tests: qa/tasks/repair_test: unset flags we set (`pr#15296 <https://github.com/ceph/ceph/pull/15296>`_, Sage Weil)
+* tests: qa/tasks/rgw.py: start Apache before RadosGW (`pr#13846 <https://github.com/ceph/ceph/pull/13846>`_, Radoslaw Zarzynski)
+* tests: qa/tasks/thrashosds-health.yaml: ignore MON_DOWN (`issue#20910 <http://tracker.ceph.com/issues/20910>`_, `pr#17003 <https://github.com/ceph/ceph/pull/17003>`_, Sage Weil)
+* tests: qa/tasks: use sudo to check ceph health for systemd test (`pr#14464 <https://github.com/ceph/ceph/pull/14464>`_, Vasu Kulkarni)
+* tests: qa/tasks/workunit.py: use "overrides" as the default settings of workunit (`issue#19429 <http://tracker.ceph.com/issues/19429>`_, `pr#14281 <https://github.com/ceph/ceph/pull/14281>`_, Kefu Chai)
+* tests: qa/tasks/workunit: use ceph.git as an alternative of ceph-ci.git for cloning workunit (`pr#13663 <https://github.com/ceph/ceph/pull/13663>`_, Kefu Chai)
+* tests: qa/tasks/workunit: use the suite repo for cloning workunit (`pr#13452 <https://github.com/ceph/ceph/pull/13452>`_, Kefu Chai)
+* tests: qa/tasks/workunit: use the suite repo for cloning workunit (`pr#13625 <https://github.com/ceph/ceph/pull/13625>`_, Kefu Chai)
+* tests: qa/test_rados_tool.sh: POSIX dd only accepts 'k' as multiplier (`pr#12699 <https://github.com/ceph/ceph/pull/12699>`_, Willem Jan Withagen)
+* tests: qa: timeout when waiting for mgr to be available in healthy() (`pr#16797 <https://github.com/ceph/ceph/pull/16797>`_, Josh Durgin)
+* tests: qa: Using centos 7.2 for `latest` version (`pr#12806 <https://github.com/ceph/ceph/pull/12806>`_, Yuri Weinstein)
+* tests: qa/workunits/ceph-helpers: display rejected string (`issue#20344 <http://tracker.ceph.com/issues/20344>`_, `pr#14468 <https://github.com/ceph/ceph/pull/14468>`_, Kefu Chai)
+* tests: qa/workunits/ceph-helpers: enable experimental features for osd (`pr#16319 <https://github.com/ceph/ceph/pull/16319>`_, Kefu Chai)
+* tests: qa/workunits/ceph-helpers.sh: use syntax understood by jq 1.3 (`pr#15530 <https://github.com/ceph/ceph/pull/15530>`_, Kefu Chai)
+* tests: qa/workunits/ceph-helpers: test wait_for_health_ok differently (`pr#16317 <https://github.com/ceph/ceph/pull/16317>`_, Kefu Chai)
+* tests: qa/workunits/ceph-helpers: wait_for_clean() races with pg creation (`pr#12866 <https://github.com/ceph/ceph/pull/12866>`_, David Zafman)
+* tests: qa/workunits/cephtool/test.sh: Be more liberal in testing health-output (`pr#14614 <https://github.com/ceph/ceph/pull/14614>`_, Willem Jan Withagen)
+* tests: qa/workunits/cephtool/test.sh: "ceph osd stat" output changed, update accordingly (`pr#16444 <https://github.com/ceph/ceph/pull/16444>`_, Willem Jan Withagen, Kefu Chai)
+* tests: qa/workunits/cephtool/test.sh: disable 'fs status' until bug is fixed (`issue#20761 <http://tracker.ceph.com/issues/20761>`_, `pr#16541 <https://github.com/ceph/ceph/pull/16541>`_, Sage Weil)
+* tests: qa/workunits/cephtool/test.sh: fix test to watch audit channel (`pr#16470 <https://github.com/ceph/ceph/pull/16470>`_, Sage Weil)
+* tests: qa/workunits/cephtool/test.sh: only include last line for epoch (`issue#20477 <http://tracker.ceph.com/issues/20477>`_, `pr#15770 <https://github.com/ceph/ceph/pull/15770>`_, Kefu Chai)
+* tests: qa/workunits/rados/test.sh: print test name when it fails (`pr#13264 <https://github.com/ceph/ceph/pull/13264>`_, Kefu Chai)
+* tests: rados: move cephtool.yaml to new singleton/bluestore subsuite (`issue#19797 <http://tracker.ceph.com/issues/19797>`_, `pr#14847 <https://github.com/ceph/ceph/pull/14847>`_, Nathan Cutler)
+* tests: rbd/test_lock_fence.sh: fix rbdrw.py relative path (`issue#18388 <http://tracker.ceph.com/issues/18388>`_, `pr#12747 <https://github.com/ceph/ceph/pull/12747>`_, Nathan Cutler)
+* tests: re-enable cephfs python tests on kclient (`issue#17193 <http://tracker.ceph.com/issues/17193>`_, `issue#18161 <http://tracker.ceph.com/issues/18161>`_, `pr#13200 <https://github.com/ceph/ceph/pull/13200>`_, Nathan Cutler)
+* tests: remove temporary file (`pr#12919 <https://github.com/ceph/ceph/pull/12919>`_, Kefu Chai)
+* tests: Revert "dummy: reduce run time, run user.yaml playbook" (`issue#18259 <http://tracker.ceph.com/issues/18259>`_, `pr#12506 <https://github.com/ceph/ceph/pull/12506>`_, Nathan Cutler)
+* tests: Revert "qa/tasks/workunit: use the suite repo for cloning workunit" (`pr#13495 <https://github.com/ceph/ceph/pull/13495>`_, Sage Weil)
+* tests: rgw.py: put client roles in a separate list (`issue#20417 <http://tracker.ceph.com/issues/20417>`_, `pr#15913 <https://github.com/ceph/ceph/pull/15913>`_, Nathan Cutler)
+* tests: rgw/singleton: drop duplicate filestore-xfs.yaml (`pr#15959 <https://github.com/ceph/ceph/pull/15959>`_, Nathan Cutler)
+* tests: set -x in suites/iozone.sh workunit (`issue#19740 <http://tracker.ceph.com/issues/19740>`_, `pr#14713 <https://github.com/ceph/ceph/pull/14713>`_, Nathan Cutler)
+* tests: src/test/test_denc.cc: Fix errors in buffer overflow (`pr#12653 <https://github.com/ceph/ceph/pull/12653>`_, Willem Jan Withagen)
+* tests: subst repo and branch in git.ceph.com URL in qa/tasks/cram.py and qa/tasks/qemu.py (`issue#18440 <http://tracker.ceph.com/issues/18440>`_, `pr#12816 <https://github.com/ceph/ceph/pull/12816>`_, Nathan Cutler)
+* tests: tasks/workunit.py: when cloning, use --depth=1 (`pr#14214 <https://github.com/ceph/ceph/pull/14214>`_, Dan Mick)
+* tests: test: add explicit braces to avoid ambiguous ‘else’ and to silence warnings (`pr#14472 <https://github.com/ceph/ceph/pull/14472>`_, Jos Collin)
+* tests: test: add override in test submodule (`pr#13773 <https://github.com/ceph/ceph/pull/13773>`_, liuchang0812)
+* tests: test: ceph osd stat out has changed, fix tests for that (`pr#16403 <https://github.com/ceph/ceph/pull/16403>`_, Willem Jan Withagen)
+* tests: test:Check make_writeable() return value (`pr#15266 <https://github.com/ceph/ceph/pull/15266>`_, zhanglei)
+* tests: test: clean up unused variable (`pr#12873 <https://github.com/ceph/ceph/pull/12873>`_, liuchang0812)
+* tests: test/CMakeLists: disable test_pidfile.sh (`issue#20975 <http://tracker.ceph.com/issues/20975>`_, `pr#17241 <https://github.com/ceph/ceph/pull/17241>`_, Sage Weil)
+* tests: test/compressor: disable isal tests if not available (`pr#14929 <https://github.com/ceph/ceph/pull/14929>`_, Kefu Chai)
+* tests: test: c_read_operations.cc: silence warning from -Wsign-compare (`pr#14888 <https://github.com/ceph/ceph/pull/14888>`_, Jos Collin)
+* tests: test: create asok files in a temp directory under $TMPDIR (`issue#16895 <http://tracker.ceph.com/issues/16895>`_, `pr#16445 <https://github.com/ceph/ceph/pull/16445>`_, Kefu Chai)
+* tests: test/crush: silence warnings from -Walloc-size-larger-than= and -Wstringop-overflow (`pr#15173 <https://github.com/ceph/ceph/pull/15173>`_, Jos Collin)
+* tests: test: c_write_operations.cc: silence warning from -Wsign-compare (`pr#14889 <https://github.com/ceph/ceph/pull/14889>`_, Jos Collin)
+* tests: test: Division by zero in Legacy::encode_n() (`pr#15902 <https://github.com/ceph/ceph/pull/15902>`_, Jos Collin)
+* tests: test/encoding: fix readable.sh bugs; fix ceph-object-corpus (`pr#13678 <https://github.com/ceph/ceph/pull/13678>`_, Sage Weil)
+* tests: test/fio_ceph_objectstore: fix fio plugin build failure by engine_data (`pr#15044 <https://github.com/ceph/ceph/pull/15044>`_, lisali)
+* tests: test/fio: Fix assert in set_cache_shards in bluestore fio (`pr#15659 <https://github.com/ceph/ceph/pull/15659>`_, Xiaoyan Li)
+* tests: test/fio: fix lack of setting for Sequencer::shard_hint (`pr#15571 <https://github.com/ceph/ceph/pull/15571>`_, Igor Fedotov)
+* tests: test/fio: print all perfcounters rather than objectstore itself (`pr#16339 <https://github.com/ceph/ceph/pull/16339>`_, Jianpeng Ma)
+* tests: test/fio: remove experimental option for bluestore & rocksdb (`pr#16263 <https://github.com/ceph/ceph/pull/16263>`_, Pan Liu)
+* tests: test: Fixes for test_pidfile (`issue#20770 <http://tracker.ceph.com/issues/20770>`_, `pr#16587 <https://github.com/ceph/ceph/pull/16587>`_, David Zafman)
+* tests: test: fixing assert that creates warning: comparison between signed and unsigned integer expressions (`pr#14794 <https://github.com/ceph/ceph/pull/14794>`_, Jos Collin)
+* tests: test: Fix mismatched sign comparison in histogram test (`pr#13362 <https://github.com/ceph/ceph/pull/13362>`_, Adam C. Emerson)
+* tests: test: Fix reg11184 test to remove extraneous pg (`pr#16265 <https://github.com/ceph/ceph/pull/16265>`_, David Zafman)
+* tests: test: fix test_pidfile (`pr#13646 <https://github.com/ceph/ceph/pull/13646>`_, yaoning)
+* tests: test/fsx:  Remove the dead code associated with aio backend (`pr#14905 <https://github.com/ceph/ceph/pull/14905>`_, Zhou Zhengping)
+* tests: test: Initialize pointer variables in TestMemIoCtxImpl (`pr#16785 <https://github.com/ceph/ceph/pull/16785>`_, amitkuma)
+* tests: test/librados: Initialize member variables in aio.cc (`pr#16845 <https://github.com/ceph/ceph/pull/16845>`_, amitkuma)
+* tests: test: librados_test_stub: tmap_update: return -ENOENT when removing nonexisent key (`pr#12667 <https://github.com/ceph/ceph/pull/12667>`_, Mykola Golub)
+* tests: test: migrate atomic_t to std::atomic (`pr#14655 <https://github.com/ceph/ceph/pull/14655>`_, Jesse Williamson)
+* tests: test,mon,msg: kill clang analyzer warnings (`pr#16320 <https://github.com/ceph/ceph/pull/16320>`_, Kefu Chai)
+* tests: test/mon: silence warnings from -Wreorder (`pr#15692 <https://github.com/ceph/ceph/pull/15692>`_, Jos Collin)
+* tests: test/msgr: fixed the hang issue for perf_msg_client (`pr#16358 <https://github.com/ceph/ceph/pull/16358>`_, Pan Liu)
+* tests: test/msgr: silence warnings from -Wsign-compare (`pr#15356 <https://github.com/ceph/ceph/pull/15356>`_, Jos Collin)
+* tests: test/msgr: silence warnings from -Wsign-compare (`pr#15570 <https://github.com/ceph/ceph/pull/15570>`_, Jos Collin)
+* tests: test: objectstore: chain_xattr: fix wrong memset usage to fill buf (`pr#14277 <https://github.com/ceph/ceph/pull/14277>`_, Weibing Zhang)
+* tests: test/objectstore: Check apply_transaction() return values (`pr#15171 <https://github.com/ceph/ceph/pull/15171>`_, zhanglei)
+* tests: test/objectstore/: Check put_ref return value (`pr#15007 <https://github.com/ceph/ceph/pull/15007>`_, zhanglei)
+* tests: test/old: Removed commented code (`pr#15366 <https://github.com/ceph/ceph/pull/15366>`_, Jos Collin)
+* tests: test/osdc: fix comparison error and silence warning from -Wunused-value (`pr#15353 <https://github.com/ceph/ceph/pull/15353>`_, Willem Jan Withagen)
+* tests: test/osd: kill compile warning (`pr#16669 <https://github.com/ceph/ceph/pull/16669>`_, Yan Jun)
+* tests: test/osd/osd-dup.sh: lower wb fd throttle limits (`pr#14984 <https://github.com/ceph/ceph/pull/14984>`_, Dan Mick)
+* tests: test/osd/osd-dup.sh: use wait_for_clean (`pr#15722 <https://github.com/ceph/ceph/pull/15722>`_, Dan Mick)
+* tests: test/osd/osd-scrub-repair.sh: disable ec_overwrite tests on FreeBSD (`pr#15445 <https://github.com/ceph/ceph/pull/15445>`_, Willem Jan Withagen)
+* tests: test/osd/osd-scrub-repair.sh: Fix diff options on FreeBSD (`pr#15914 <https://github.com/ceph/ceph/pull/15914>`_, Willem Jan Withagen)
+* tests: test/osd: Removed Commented Code - 2 (`issue#20207 <http://tracker.ceph.com/issues/20207>`_, `pr#15540 <https://github.com/ceph/ceph/pull/15540>`_, Jos Collin)
+* tests: test: osd/TestOSDMap.cc: fix Clang complain about promotion (`pr#15525 <https://github.com/ceph/ceph/pull/15525>`_, Willem Jan Withagen)
+* tests: test/rados: fix wrong parameter order of RETURN1_IF_NOT_VAL (`pr#16589 <https://github.com/ceph/ceph/pull/16589>`_, Yan Jun)
+* tests: test: reg11184 might not always find pg 2.0 prior to import (`pr#16610 <https://github.com/ceph/ceph/pull/16610>`_, David Zafman)
+* tests: test: Rename FileJournal object to distinguish (`pr#15279 <https://github.com/ceph/ceph/pull/15279>`_, Jos Collin)
+* tests: test: replace hard-code binary names with varibles (`pr#12675 <https://github.com/ceph/ceph/pull/12675>`_, liuchang0812)
+* tests: test: sed on FreeBSD requires "-i extension", so use gsed (`pr#13903 <https://github.com/ceph/ceph/pull/13903>`_, Willem Jan Withagen)
+* tests: test: s/osd_objectstore_type/osd_objectstore (`pr#16469 <https://github.com/ceph/ceph/pull/16469>`_, xie xingguo)
+* tests: test: test_denc.cc: silence warning from -Wsign-compare (`pr#15355 <https://github.com/ceph/ceph/pull/15355>`_, Jos Collin)
+* tests: test: Test fix for SnapSet change (`pr#15161 <https://github.com/ceph/ceph/pull/15161>`_, David Zafman)
+* tests: test: test_pidfile running 2nd mon has unreliable log output (`pr#16635 <https://github.com/ceph/ceph/pull/16635>`_, David Zafman)
+* tests: test: Thrasher: do not update pools_to_fix_pgp_num if nothing happens (`pr#13518 <https://github.com/ceph/ceph/pull/13518>`_, Kefu Chai)
+* tests: test: Thrasher: update pgp_num of all expanded pools if not yet (`pr#13367 <https://github.com/ceph/ceph/pull/13367>`_, Kefu Chai)
+* tests: test/unittest_bluefs: check whether mounted success (`pr#14988 <https://github.com/ceph/ceph/pull/14988>`_, shiqi)
+* tests: test/unittest_bluefs: remove unused variable (`pr#14006 <https://github.com/ceph/ceph/pull/14006>`_, shiqi)
+* tests: test: unittest_hostname compile error on freebsd (`pr#13739 <https://github.com/ceph/ceph/pull/13739>`_, liuchang0812)
+* tests: test: update test_rados_tool.sh, use POOL and OBJ var (`pr#12706 <https://github.com/ceph/ceph/pull/12706>`_, liuchang0812)
+* tests: test: use 7130 for crush-classes.sh (`pr#14783 <https://github.com/ceph/ceph/pull/14783>`_, Loic Dachary)
+* tests: test/vstart_wrapper.sh: display_log on test failure (`pr#15620 <https://github.com/ceph/ceph/pull/15620>`_, Kefu Chai)
+* tests: test: warning: comparison between signed and unsigned integer expressions (`pr#14705 <https://github.com/ceph/ceph/pull/14705>`_, Jos Collin)
+* tests: Thrasher: eliminate a race between kill_osd and __init__ (`issue#18799 <http://tracker.ceph.com/issues/18799>`_, `pr#13237 <https://github.com/ceph/ceph/pull/13237>`_, Nathan Cutler)
+* tests: Thrasher: handle "OSD has the store locked" gracefully (`issue#19556 <http://tracker.ceph.com/issues/19556>`_, `pr#14415 <https://github.com/ceph/ceph/pull/14415>`_, Nathan Cutler)
+* tests,tools: script/find_dups_in_pg_log: scrip to find dup requests due to short pg logs (`pr#13417 <https://github.com/ceph/ceph/pull/13417>`_, Sage Weil)
+* tests,tools: test, ceph-osdomap-tool: kill clang warnings (`pr#15905 <https://github.com/ceph/ceph/pull/15905>`_, Kefu Chai)
+* tests,tools: test: kill warnings (`pr#14892 <https://github.com/ceph/ceph/pull/14892>`_, Kefu Chai)
+* tests: update SUSE yaml facets in qa/distros/all (`issue#18856 <http://tracker.ceph.com/issues/18856>`_, `pr#13313 <https://github.com/ceph/ceph/pull/13313>`_, Nathan Cutler)
+* tests: workunit: request branch when cloning (`pr#14260 <https://github.com/ceph/ceph/pull/14260>`_, Kefu Chai, Dan Mick)
+* tools: add override in tool submodule (`pr#13776 <https://github.com/ceph/ceph/pull/13776>`_, liuchang0812)
+* tools: brag: count the number of mds in fsmap not in mdsmap (`issue#19192 <http://tracker.ceph.com/issues/19192>`_, `pr#13798 <https://github.com/ceph/ceph/pull/13798>`_, Peng Zhang)
+* tools: ceph_common.sh: fix syntax error (`issue#17826 <http://tracker.ceph.com/issues/17826>`_, `pr#13419 <https://github.com/ceph/ceph/pull/13419>`_, Dan Mick)
+* tools: ceph-conf: fix typo in usage: 'mon add' should be 'mon addr' (`pr#15935 <https://github.com/ceph/ceph/pull/15935>`_, Peng Zhang)
+* tools: ceph-create-keys: add an argument to override default 10-minute timeout (`pr#16049 <https://github.com/ceph/ceph/pull/16049>`_, Douglas Fuller)
+* tools: ceph-detect-init: adding Arch Linux support (`pr#12787 <https://github.com/ceph/ceph/pull/12787>`_, Jamin W. Collins)
+* tools: ceph-detect-init: Adds Oracle Linux Server and Oracle VM Server detect (`pr#13917 <https://github.com/ceph/ceph/pull/13917>`_, Nikita Gerasimov)
+* tools: ceph-detect-init: detect init system by poking the system (`issue#19884 <http://tracker.ceph.com/issues/19884>`_, `pr#15043 <https://github.com/ceph/ceph/pull/15043>`_, Kefu Chai)
+* tools: ceph-disk: Add fix subcommand (`pr#13310 <https://github.com/ceph/ceph/pull/13310>`_, Boris Ranto)
+* tools: ceph-disk: change the lockbox partition number to 5 (`issue#20556 <http://tracker.ceph.com/issues/20556>`_, `pr#16247 <https://github.com/ceph/ceph/pull/16247>`_, Shangzhong Zhu)
+* tools: ceph-disk: command invocation needs all fields separate (`pr#15733 <https://github.com/ceph/ceph/pull/15733>`_, Willem Jan Withagen)
+* tools: ceph-disk: convert none str to str before printing it (`issue#18371 <http://tracker.ceph.com/issues/18371>`_, `pr#12760 <https://github.com/ceph/ceph/pull/12760>`_, Kefu Chai)
+* tools: ceph-disk: do not remove mount point if deactive --once (`pr#16474 <https://github.com/ceph/ceph/pull/16474>`_, Song Shun)
+* tools: ceph-disk: Fix for missing 'not' in \*_is_diskdevice checks (`issue#20706 <http://tracker.ceph.com/issues/20706>`_, `pr#16481 <https://github.com/ceph/ceph/pull/16481>`_, Nikita Gerasimov)
+* tools: ceph_disk/main.py: FreeBSD root has wheel for group (`pr#16609 <https://github.com/ceph/ceph/pull/16609>`_, Willem Jan Withagen)
+* tools: ceph-disk: s/ceph_osd_mkfs/command_check_call/ (`issue#20685 <http://tracker.ceph.com/issues/20685>`_, `pr#16427 <https://github.com/ceph/ceph/pull/16427>`_, Zhu Shangzhong)
+* tools: ceph.in: add help for locally-handled commands (`pr#13288 <https://github.com/ceph/ceph/pull/13288>`_, Dan Mick)
+* tools: ceph.in: adjust usage width according to user's tty (`pr#15190 <https://github.com/ceph/ceph/pull/15190>`_, Kefu Chai)
+* tools: ceph.in: assert(state==connected) before help_for_target() (`pr#15156 <https://github.com/ceph/ceph/pull/15156>`_, Kefu Chai)
+* tools: ceph.in: drop the compatiiblity to handle non json commands (`pr#15508 <https://github.com/ceph/ceph/pull/15508>`_, Kefu Chai)
+* tools: ceph.in: filter out audit from ceph -w (`pr#16345 <https://github.com/ceph/ceph/pull/16345>`_, John Spray)
+* tools: ceph.in, mgr: misc cleanups (`pr#16229 <https://github.com/ceph/ceph/pull/16229>`_, liuchang0812)
+* tools: ceph.in: print return code when json_command failed (`pr#15378 <https://github.com/ceph/ceph/pull/15378>`_, liuchang0812)
+* tools: ceph-objectstore-tool: Handle object names that are also valid json (`pr#12848 <https://github.com/ceph/ceph/pull/12848>`_, David Zafman)
+* tools: ceph-release-notes: escape asterisks not for inline emphasis (`pr#16199 <https://github.com/ceph/ceph/pull/16199>`_, Kefu Chai)
+* tools: ceph-release-notes: escape _ for unintended links (`issue#17499 <http://tracker.ceph.com/issues/17499>`_, `pr#16528 <https://github.com/ceph/ceph/pull/16528>`_, Kefu Chai)
+* tools: ceph-release-notes: handle an edge case (`pr#16277 <https://github.com/ceph/ceph/pull/16277>`_, Nathan Cutler)
+* tools: ceph-release-notes: ignore low-numbered PRs (`issue#18695 <http://tracker.ceph.com/issues/18695>`_, `pr#13151 <https://github.com/ceph/ceph/pull/13151>`_, Nathan Cutler)
+* tools: ceph-release-notes: port it to py3 (`pr#16261 <https://github.com/ceph/ceph/pull/16261>`_, Kefu Chai)
+* tools: ceph-release-notes: prefixes and pep8 compliance (`pr#14156 <https://github.com/ceph/ceph/pull/14156>`_, Nathan Cutler)
+* tools: ceph-release-notes: refactor and fix regressions (`pr#16411 <https://github.com/ceph/ceph/pull/16411>`_, Nathan Cutler)
+* tools: ceph-release-notes: strip trailing punctuation (`pr#14385 <https://github.com/ceph/ceph/pull/14385>`_, Nathan Cutler)
+* tools: ceph-rest-api: be more tolerant on network failure (`issue#20115 <http://tracker.ceph.com/issues/20115>`_, `pr#15706 <https://github.com/ceph/ceph/pull/15706>`_, Kefu Chai)
+* tools: ceph-volume: adds functional CI testing #16919 (`pr#16970 <https://github.com/ceph/ceph/pull/16970>`_, Andrew Schoen, Alfredo Deza)
+* tools: ceph-volume docs (`pr#17124 <https://github.com/ceph/ceph/pull/17124>`_, Alfredo Deza)
+* tools: ceph-volume: initial take on ceph-volume CLI tool (`pr#16632 <https://github.com/ceph/ceph/pull/16632>`_, Dan Mick, Alfredo Deza)
+* tools: ceph-volume: Use a delimited CLI output parser instead of JSON (`pr#17123 <https://github.com/ceph/ceph/pull/17123>`_, Alfredo Deza)
+* tools: change compare_exchange_weak to compare_exchange_strong (`pr#15030 <https://github.com/ceph/ceph/pull/15030>`_, Jesse Williamson)
+* tools: Cleanup dead code in ceph-objectstore-tool (`pr#15812 <https://github.com/ceph/ceph/pull/15812>`_, David Zafman)
+* tools: fio_ceph_objectstore: Print db_statistics when rocksdb_perf is enabled (`pr#15796 <https://github.com/ceph/ceph/pull/15796>`_, Xiaoyan Li)
+* tools: init-ceph: print trailing n in "status" output (`pr#13351 <https://github.com/ceph/ceph/pull/13351>`_, Kefu Chai)
+* tools: init-ceph: should have a space before "]" (`pr#14796 <https://github.com/ceph/ceph/pull/14796>`_, Kefu Chai)
+* tools: os/bluestore/bluestore_tool: add sanity check to get rid of occasionally crash (`pr#16013 <https://github.com/ceph/ceph/pull/16013>`_, xie xingguo)
+* tools: rados: check for negative return value of rados_create_with_context() as its comment put (`pr#10893 <https://github.com/ceph/ceph/pull/10893>`_, zhang.zezhu)
+* tools: rados: fix typo in 'df' column name (`pr#15603 <https://github.com/ceph/ceph/pull/15603>`_, Ilya Dryomov)
+* tools: rados: out json 'df' values as numbers, not strings (`issue#15546 <http://tracker.ceph.com/issues/15546>`_, `pr#14644 <https://github.com/ceph/ceph/pull/14644>`_, Sage Weil)
+* tools: script: add docker core dump debugger (`pr#16375 <https://github.com/ceph/ceph/pull/16375>`_, Patrick Donnelly)
+* tools: script: ceph-release-notes check orig. issue only for backports (`pr#12979 <https://github.com/ceph/ceph/pull/12979>`_, Abhishek Lekshmanan)
+* tools: script/sepia_bt.sh: no need to pass version and sha1 anymore (`pr#13380 <https://github.com/ceph/ceph/pull/13380>`_, Kefu Chai)
+* tools: src/ceph-disk/ceph_disk/main.py: Make 'ceph-disk list' work on FreeBSD (`pr#14483 <https://github.com/ceph/ceph/pull/14483>`_, Willem Jan Withagen)
+* tools: stop.sh: boilerplate error (don't stop mon when stopping mgr) (`pr#14461 <https://github.com/ceph/ceph/pull/14461>`_, Dan Mick)
+* tools: support hammer in rbd_recover_tool (`pr#12413 <https://github.com/ceph/ceph/pull/12413>`_, Bartłomiej Święcki)
+* tools: tools/ceph_kvstore_tool: add "bluestore-kv" to usage (`pr#15326 <https://github.com/ceph/ceph/pull/15326>`_, xie xingguo)
+* tools: tools/crushtool: replicated-rule API support (`pr#15011 <https://github.com/ceph/ceph/pull/15011>`_, xie xingguo)
+* tools: tools/rados: add a parameter "--offset" to rados put command (`pr#12674 <https://github.com/ceph/ceph/pull/12674>`_, liuchang0812)
+* tools: tools/rados: Check return value of connect (`issue#19319 <http://tracker.ceph.com/issues/19319>`_, `pr#14057 <https://github.com/ceph/ceph/pull/14057>`_, Brad Hubbard)
+* tools: tools/rados: fixed typo in help information (`pr#15618 <https://github.com/ceph/ceph/pull/15618>`_, Pan Liu)
+* tools: tools/rados: remove useless function declaration (`pr#12566 <https://github.com/ceph/ceph/pull/12566>`_, liuchang0812)
+* tools: tools/rados: some cleanups (`pr#16147 <https://github.com/ceph/ceph/pull/16147>`_, Yan Jun)
+* tools: vstart: "debug_ms=1" for mgr by default (`pr#15127 <https://github.com/ceph/ceph/pull/15127>`_, Kefu Chai)
+* tools: vstart: don't configure rgw_dns_name (`pr#13411 <https://github.com/ceph/ceph/pull/13411>`_, Yehuda Sadeh)
+* tools: vstart: don't create cluster by default (`pr#13891 <https://github.com/ceph/ceph/pull/13891>`_, Yehuda Sadeh)
+* tools: vstart: print "start osd.$id" instead of "start osd$id" (`pr#15427 <https://github.com/ceph/ceph/pull/15427>`_, Kefu Chai)
+* tools: vstart.sh: bind restful, dashboard to ::, not 127.0.0.1 (`pr#16349 <https://github.com/ceph/ceph/pull/16349>`_, Sage Weil)
+* tools: vstart.sh: do not add host for mgr.\* section if not $overwrite_conf (`pr#13767 <https://github.com/ceph/ceph/pull/13767>`_, Kefu Chai)
+* tools: warning, ‘%.16x’ directive output truncated writing 16 bytes into a region of size 9 (`pr#14292 <https://github.com/ceph/ceph/pull/14292>`_, Jos Collin)
+* tracing: don't include oid when tracing at dequeue_op() (`pr#13410 <https://github.com/ceph/ceph/pull/13410>`_, Yehuda Sadeh)
diff --git a/doc/releases/mimic.rst b/doc/releases/mimic.rst
new file mode 100644
index 000000000..c811556e3
--- /dev/null
+++ b/doc/releases/mimic.rst
@@ -0,0 +1,4475 @@
+=====
+Mimic
+=====
+
+Mimic is the 13th stable release of Ceph.  It is named after the mimic
+octopus (thaumoctopus mimicus).
+
+v13.2.10 Mimic
+==============
+
+This is the tenth bugfix release of Ceph Mimic, this release fixes a RGW
+vulnerability affecting mimic, and we recommend that all mimic users upgrade.
+
+Notable Changes
+---------------
+* CVE 2020 12059: Fix an issue with Post Object Requests with Tagging
+  (`issue#44967 <http://tracker.ceph.com/issues/44967>`_, Lei Cao, Abhishek
+  Lekshmanan)
+
+v13.2.9 Mimic
+=============
+
+This is the ninth and very likely the last stable release in the Ceph Mimic
+stable release series. This release fixes bugs across all components and also
+contains a RGW security fix. We recommend all mimic users to upgrade to this
+version.
+
+Notable Changes
+---------------
+
+* CVE-2020-1760: Fixed XSS due to RGW GetObject header-splitting
+
+* The configuration value ``osd_calc_pg_upmaps_max_stddev`` used for upmap
+  balancing has been removed. Instead use the mgr balancer config
+  ``upmap_max_deviation`` which now is an integer number of PGs of deviation
+  from the target PGs per OSD.  This can be set with a command like
+  ``ceph config set mgr mgr/balancer/upmap_max_deviation 2``.  The default
+  ``upmap_max_deviation`` is 1.  There are situations where crush rules
+  would not allow a pool to ever have completely balanced PGs.  For example, if
+  crush requires 1 replica on each of 3 racks, but there are fewer OSDs in 1 of
+  the racks.  In those cases, the configuration value can be increased.
+
+* The ``cephfs-data-scan scan_links`` command now automatically repair inotables
+  and snaptable.
+
+
+Changelog
+---------
+
+* bluestore: os/bluestore: fix improper setting of STATE_KV_SUBMITTED (`pr#31673 <https://github.com/ceph/ceph/pull/31673>`_, Igor Fedotov)
+* ceph-volume/batch: check lvs list before access (`pr#34479 <https://github.com/ceph/ceph/pull/34479>`_, Jan Fajerski)
+* ceph-volume/batch: fail on filtered devices when non-interactive (`pr#33201 <https://github.com/ceph/ceph/pull/33201>`_, Jan Fajerski)
+* ceph-volume/batch: return success when all devices are filtered (`pr#34476 <https://github.com/ceph/ceph/pull/34476>`_, Jan Fajerski)
+* ceph-volume/lvm/activate.py: clarify error message: fsid refers to osd_fsid (`pr#32865 <https://github.com/ceph/ceph/pull/32865>`_, Yaniv Kaul)
+* ceph-volume/test: patch VolumeGroups (`pr#32559 <https://github.com/ceph/ceph/pull/32559>`_, Jan Fajerski)
+* ceph-volume: Dereference symlink in lvm list (`pr#32876 <https://github.com/ceph/ceph/pull/32876>`_, Benoît Knecht)
+* ceph-volume: add db and wal support to raw mode (`pr#33622 <https://github.com/ceph/ceph/pull/33622>`_, Sébastien Han)
+* ceph-volume: add methods to pass filters to pvs, vgs and lvs commands (`pr#33215 <https://github.com/ceph/ceph/pull/33215>`_, Rishabh Dave)
+* ceph-volume: add proper size attribute to partitions (`pr#32529 <https://github.com/ceph/ceph/pull/32529>`_, Jan Fajerski)
+* ceph-volume: add raw mode (`pr#33580 <https://github.com/ceph/ceph/pull/33580>`_, Jan Fajerski, Sage Weil, Guillaume Abrioux)
+* ceph-volume: add sizing arguments to prepare (`pr#33578 <https://github.com/ceph/ceph/pull/33578>`_, Jan Fajerski)
+* ceph-volume: add utility functions (`pr#32544 <https://github.com/ceph/ceph/pull/32544>`_, Mohamad Gebai)
+* ceph-volume: allow raw block devices everywhere (`pr#32869 <https://github.com/ceph/ceph/pull/32869>`_, Jan Fajerski)
+* ceph-volume: allow to skip restorecon calls (`pr#32530 <https://github.com/ceph/ceph/pull/32530>`_, Alfredo Deza)
+* ceph-volume: avoid calling zap_lv with a LV-less VG (`pr#33610 <https://github.com/ceph/ceph/pull/33610>`_, Jan Fajerski)
+* ceph-volume: batch bluestore fix create_lvs call (`pr#33579 <https://github.com/ceph/ceph/pull/33579>`_, Jan Fajerski)
+* ceph-volume: batch bluestore fix create_lvs call (`pr#33623 <https://github.com/ceph/ceph/pull/33623>`_, Jan Fajerski)
+* ceph-volume: check if we run in an selinux environment (`pr#32866 <https://github.com/ceph/ceph/pull/32866>`_, Jan Fajerski)
+* ceph-volume: check if we run in an selinux environment, now also in py2 (`pr#32867 <https://github.com/ceph/ceph/pull/32867>`_, Jan Fajerski)
+* ceph-volume: devices/simple/scan: Fix string in log statement (`pr#34444 <https://github.com/ceph/ceph/pull/34444>`_, Jan Fajerski)
+* ceph-volume: don't create osd['block.db'] by default (`pr#33626 <https://github.com/ceph/ceph/pull/33626>`_, Jan Fajerski)
+* ceph-volume: don't remove vg twice when zapping filestore (`pr#33615 <https://github.com/ceph/ceph/pull/33615>`_, Jan Fajerski)
+* ceph-volume: finer grained availability notion in inventory (`pr#33606 <https://github.com/ceph/ceph/pull/33606>`_, Jan Fajerski)
+* ceph-volume: fix is_ceph_device for lvm batch (`pr#33608 <https://github.com/ceph/ceph/pull/33608>`_, Jan Fajerski, Dimitri Savineau)
+* ceph-volume: fix the integer overflow (`pr#32872 <https://github.com/ceph/ceph/pull/32872>`_, dongdong tao)
+* ceph-volume: import mock.mock instead of unittest.mock (py2) (`pr#32871 <https://github.com/ceph/ceph/pull/32871>`_, Jan Fajerski)
+* ceph-volume: lvm deactivate command (`pr#33208 <https://github.com/ceph/ceph/pull/33208>`_, Jan Fajerski)
+* ceph-volume: lvm/deactivate: add unit tests, remove --all (`pr#32862 <https://github.com/ceph/ceph/pull/32862>`_, Jan Fajerski)
+* ceph-volume: lvm: get_device_vgs() filter by provided prefix (`pr#33617 <https://github.com/ceph/ceph/pull/33617>`_, Jan Fajerski, Yehuda Sadeh)
+* ceph-volume: make get_devices fs location independent (`pr#33124 <https://github.com/ceph/ceph/pull/33124>`_, Jan Fajerski)
+* ceph-volume: minor clean-up of "simple scan" subcommand help (`pr#32557 <https://github.com/ceph/ceph/pull/32557>`_, Michael Fritch)
+* ceph-volume: mokeypatch calls to lvm related binaries (`pr#31406 <https://github.com/ceph/ceph/pull/31406>`_, Jan Fajerski)
+* ceph-volume: pass journal_size as Size not string (`pr#33611 <https://github.com/ceph/ceph/pull/33611>`_, Jan Fajerski)
+* ceph-volume: rearrange api/lvm.py (`pr#31407 <https://github.com/ceph/ceph/pull/31407>`_, Rishabh Dave)
+* ceph-volume: refactor listing.py + fixes (`pr#33603 <https://github.com/ceph/ceph/pull/33603>`_, Jan Fajerski, Rishabh Dave, Theofilos Mouratidis, Guillaume Abrioux)
+* ceph-volume: reject disks smaller then 5GB in inventory (`issue#40776 <http://tracker.ceph.com/issues/40776>`_, `pr#32528 <https://github.com/ceph/ceph/pull/32528>`_, Jan Fajerski)
+* ceph-volume: silence 'ceph-bluestore-tool' failures (`pr#33605 <https://github.com/ceph/ceph/pull/33605>`_, Sébastien Han)
+* ceph-volume: skip missing interpreters when running tox tests (`pr#33489 <https://github.com/ceph/ceph/pull/33489>`_, Andrew Schoen)
+* ceph-volume: skip osd creation when already done (`pr#33607 <https://github.com/ceph/ceph/pull/33607>`_, Guillaume Abrioux)
+* ceph-volume: strip _dmcrypt suffix in simple scan json output (`pr#33618 <https://github.com/ceph/ceph/pull/33618>`_, Jan Fajerski)
+* ceph-volume: use correct extents if using db-devices and >1 osds_per_device (`pr#32875 <https://github.com/ceph/ceph/pull/32875>`_, Fabian Niepelt)
+* ceph-volume: use fsync for dd command (`pr#31552 <https://github.com/ceph/ceph/pull/31552>`_, Rishabh Dave)
+* ceph-volume: use get_device_vgs in has_common_vg (`pr#33609 <https://github.com/ceph/ceph/pull/33609>`_, Jan Fajerski)
+* ceph-volume: util: look for executable in $PATH (`pr#32861 <https://github.com/ceph/ceph/pull/32861>`_, Shyukri Shyukriev)
+* cephfs: cephfs: osdc/objecter: Fix last_sent in scientific format and add age to ops (`pr#31384 <https://github.com/ceph/ceph/pull/31384>`_, Varsha Rao)
+* cephfs: cephfs: test_volume_client: declare only one default for python version (`issue#40460 <http://tracker.ceph.com/issues/40460>`_, `pr#30110 <https://github.com/ceph/ceph/pull/30110>`_, Rishabh Dave)
+* cephfs: client: more precise CEPH_CLIENT_CAPS_PENDING_CAPSNAP (`pr#31283 <https://github.com/ceph/ceph/pull/31283>`_, "Yan, Zheng")
+* cephfs: client: remove Inode.dir_contacts field and handle bad whence value to llseek gracefully (`pr#31380 <https://github.com/ceph/ceph/pull/31380>`_, Jeff Layton)
+* cephfs: mds: avoid calling clientreplay_done() prematurely (`pr#31282 <https://github.com/ceph/ceph/pull/31282>`_, "Yan, Zheng")
+* cephfs: mds: fix assert(omap_num_objs <= MAX_OBJECTS) of OpenFileTable (`pr#32757 <https://github.com/ceph/ceph/pull/32757>`_, "Yan, Zheng")
+* cephfs: mds: fix infinite loop in Locker::file_update_finish (`pr#31284 <https://github.com/ceph/ceph/pull/31284>`_, "Yan, Zheng")
+* cephfs: mds: mds returns -5(EIO) error when the deleted file does not exist (`pr#31381 <https://github.com/ceph/ceph/pull/31381>`_, huanwen ren)
+* cephfs: mds: split the dir if the op makes it oversized, because some ops maybe in flight (`pr#31379 <https://github.com/ceph/ceph/pull/31379>`_, simon gao)
+* cephfs: tools/cephfs: make 'cephfs-data-scan scan_links' reconstruct snaptable (`pr#31281 <https://github.com/ceph/ceph/pull/31281>`_, "Yan, Zheng")
+* common/config: parse --log-early option (`pr#33130 <https://github.com/ceph/ceph/pull/33130>`_, Sage Weil)
+* common: common/admin_socket: Increase socket timeouts (`pr#33323 <https://github.com/ceph/ceph/pull/33323>`_, Brad Hubbard)
+* common: common/config: update values when they are removed via mon (`pr#33327 <https://github.com/ceph/ceph/pull/33327>`_, Sage Weil)
+* common: common/util: use ifstream to read from /proc files (`pr#32902 <https://github.com/ceph/ceph/pull/32902>`_, Kefu Chai, songweibin)
+* core,mgr,tests: mgr: Release GIL and Balancer fixes (`pr#31957 <https://github.com/ceph/ceph/pull/31957>`_, Neha Ojha, Kefu Chai, Noah Watkins, David Zafman)
+* core,mgr: mgr/prometheus: assign a value to osd_dev_node when obj_store is not filestore or bluestore (`pr#31557 <https://github.com/ceph/ceph/pull/31557>`_, jiahuizeng)
+* core,tests: qa/tasks/cbt: install python3 deps (`pr#34193 <https://github.com/ceph/ceph/pull/34193>`_, Sage Weil)
+* core: mon/OSDMonitor: fix format error ceph osd stat --format json (`pr#33322 <https://github.com/ceph/ceph/pull/33322>`_, Zheng Yin)
+* core: mon: Don't put session during feature change (`pr#33154 <https://github.com/ceph/ceph/pull/33154>`_, Brad Hubbard)
+* core: osd/PeeringState.cc: don't let num_objects become negative (`pr#33331 <https://github.com/ceph/ceph/pull/33331>`_, Neha Ojha)
+* core: osd/PeeringState.cc: skip peer_purged when discovering all missing (`pr#33329 <https://github.com/ceph/ceph/pull/33329>`_, Neha Ojha)
+* core: osd/PeeringState.h: ignore MLogRec in Peering/GetInfo (`pr#33594 <https://github.com/ceph/ceph/pull/33594>`_, Neha Ojha)
+* core: osd/PeeringState: do not exclude up from acting_recovery_backfill (`pr#33324 <https://github.com/ceph/ceph/pull/33324>`_, Nathan Cutler, xie xingguo)
+* core: osd: Allow 64-char hostname to be added as the "host" in CRUSH (`pr#33145 <https://github.com/ceph/ceph/pull/33145>`_, Michal Skalski)
+* core: osd: Diagnostic logging for upmap cleaning (`pr#32717 <https://github.com/ceph/ceph/pull/32717>`_, David Zafman)
+* core: osd: backfill_toofull seen on cluster where the most full OSD is at 1% (`pr#32361 <https://github.com/ceph/ceph/pull/32361>`_, David Zafman)
+* core: osd: set collection pool opts on collection create, pg load (`pr#32125 <https://github.com/ceph/ceph/pull/32125>`_, Sage Weil)
+* core: selinux: Allow ceph to read udev db (`pr#32258 <https://github.com/ceph/ceph/pull/32258>`_, Boris Ranto)
+* core: selinux: Allow ceph-mgr access to httpd dir (`pr#34458 <https://github.com/ceph/ceph/pull/34458>`_, Brad Hubbard)
+* doc: remove invalid option mon_pg_warn_max_per_osd (`pr#31875 <https://github.com/ceph/ceph/pull/31875>`_, zhang daolong)
+* doc: doc/_templates/page.html: redirect to etherpad (`pr#32249 <https://github.com/ceph/ceph/pull/32249>`_, Neha Ojha)
+* doc: doc/cephfs/client-auth: description and example are inconsistent (`pr#32782 <https://github.com/ceph/ceph/pull/32782>`_, Ilya Dryomov)
+* doc: wrong datatype describing crush_rule (`pr#32255 <https://github.com/ceph/ceph/pull/32255>`_, Kefu Chai)
+* mgr,pybind: mgr/prometheus: report per-pool pg states (`pr#33158 <https://github.com/ceph/ceph/pull/33158>`_, Aleksei Zakharov)
+* mgr,pybind: mgr/telemetry: check get_metadata return val (`pr#33096 <https://github.com/ceph/ceph/pull/33096>`_, Yaarit Hatuka)
+* mount.ceph: give a hint message when no mds is up or cluster is laggy (`pr#32911 <https://github.com/ceph/ceph/pull/32911>`_, Xiubo Li)
+* pybind: pybind/mgr: Cancel output color control (`pr#31805 <https://github.com/ceph/ceph/pull/31805>`_, Zheng Yin)
+* qa: get rid of iterkeys for py3 compatibility (`pr#33999 <https://github.com/ceph/ceph/pull/33999>`_, Kyr Shatskyy)
+* rbd:  creating thick-provision image progress percent info exceeds 100% (`pr#33318 <https://github.com/ceph/ceph/pull/33318>`_, Xiangdong Mu)
+* rbd: librbd: diff iterate with fast-diff now correctly includes parent (`pr#32470 <https://github.com/ceph/ceph/pull/32470>`_, Jason Dillaman)
+* rbd: librbd: don't call refresh from mirror::GetInfoRequest state machine (`pr#32952 <https://github.com/ceph/ceph/pull/32952>`_, Mykola Golub)
+* rbd: librbd: fix rbd_open_by_id, rbd_open_by_id_read_only (`pr#33315 <https://github.com/ceph/ceph/pull/33315>`_, yangjun)
+* rbd: nautilus: rbd-mirror: fix 'rbd mirror status' asok command output (`pr#32714 <https://github.com/ceph/ceph/pull/32714>`_, Mykola Golub)
+* rbd: rbd-mirror: clone v2 mirroring improvements (`pr#31520 <https://github.com/ceph/ceph/pull/31520>`_, Mykola Golub)
+* rbd: rbd-mirror: improve detection of blacklisted state (`pr#33598 <https://github.com/ceph/ceph/pull/33598>`_, Mykola Golub)
+* rbd: rbd-mirror: make logrotate work (`pr#32598 <https://github.com/ceph/ceph/pull/32598>`_, Mykola Golub)
+* rgw: add bucket permission verify when copy obj (`pr#31377 <https://github.com/ceph/ceph/pull/31377>`_, NancySu05)
+* rgw: add list user admin OP API (`pr#31754 <https://github.com/ceph/ceph/pull/31754>`_, Oshyn Song)
+* rgw: add missing admin property when sync user info (`pr#30804 <https://github.com/ceph/ceph/pull/30804>`_, zhang Shaowen)
+* rgw: add num_shards to radosgw-admin bucket stats (`pr#31183 <https://github.com/ceph/ceph/pull/31183>`_, Paul Emmerich)
+* rgw: adding mfa code validation when bucket versioning status is changed (`pr#33303 <https://github.com/ceph/ceph/pull/33303>`_, Pritha Srivastava)
+* rgw: allow reshard log entries for non-existent buckets to be cancelled (`pr#33302 <https://github.com/ceph/ceph/pull/33302>`_, J. Eric Ivancich)
+* rgw: auto-clean reshard queue entries for non-existent buckets (`pr#33300 <https://github.com/ceph/ceph/pull/33300>`_, J. Eric Ivancich)
+* rgw: change the "rgw admin status" 'num_shards' output to signed int (`issue#37645 <http://tracker.ceph.com/issues/37645>`_, `pr#33305 <https://github.com/ceph/ceph/pull/33305>`_, Mark Kogan)
+* rgw: crypt: permit RGW-AUTO/default with SSE-S3 headers (`pr#31861 <https://github.com/ceph/ceph/pull/31861>`_, Matt Benjamin)
+* rgw: find oldest period and update RGWMetadataLogHistory() (`pr#33309 <https://github.com/ceph/ceph/pull/33309>`_, Shilpa Jagannath)
+* rgw: fix a bug that bucket instance obj can't be removed after resharding completed (`pr#33306 <https://github.com/ceph/ceph/pull/33306>`_, zhang Shaowen)
+* rgw: fix bad user stats on versioned bucket after reshard (`pr#33304 <https://github.com/ceph/ceph/pull/33304>`_, J. Eric Ivancich)
+* rgw: fix memory growth while deleting objects with (`pr#31378 <https://github.com/ceph/ceph/pull/31378>`_, Mark Kogan)
+* rgw: get barbican secret key request maybe return error code (`pr#33966 <https://github.com/ceph/ceph/pull/33966>`_, Richard Bai(白学余))
+* rgw: make max_connections configurable in beast (`pr#33341 <https://github.com/ceph/ceph/pull/33341>`_, Tiago Pasqualini)
+* rgw: making implicit_tenants backwards compatible (`issue#24348 <http://tracker.ceph.com/issues/24348>`_, `pr#33748 <https://github.com/ceph/ceph/pull/33748>`_, Marcus Watts)
+* rgw: maybe coredump when reload operator happened (`pr#33313 <https://github.com/ceph/ceph/pull/33313>`_, Richard Bai(白学余))
+* rgw: move forward marker even in case of many rgw.none indexes (`pr#33311 <https://github.com/ceph/ceph/pull/33311>`_, Ilsoo Byun)
+* rgw: prevent bucket reshard scheduling if bucket is resharding (`pr#31299 <https://github.com/ceph/ceph/pull/31299>`_, J. Eric Ivancich)
+* rgw: update the hash source for multipart entries during resharding (`pr#33312 <https://github.com/ceph/ceph/pull/33312>`_, dongdong tao)
+
+
+v13.2.8 Mimic
+=============
+
+This is the eighth release in the Ceph Mimic stable release series. Its sole
+purpose is to fix a regression that found its way into the previous release.
+
+Notable Changes
+---------------
+
+* Due to a missed backport, clusters in the process of being upgraded from
+  13.2.6 to 13.2.7 might suffer an OSD crash in build_incremental_map_msg.
+  This regression was reported in https://tracker.ceph.com/issues/43106
+  and is fixed in 13.2.8 (this release). Users of 13.2.6 can upgrade to 13.2.8
+  directly - i.e., skip 13.2.7 - to avoid this.
+
+Changelog
+---------
+
+* osd: fix sending incremental map messages (more) (`issue#43106 <https://tracker.ceph.com/issues/43106>`_, `pr#32000 <https://github.com/ceph/ceph/pull/32000>`_, Sage Weil)
+* tests: added missing point release versions (`pr#32087 <https://github.com/ceph/ceph/pull/32087>`_, Yuri Weinstein)
+* tests: rgw: add missing force-branch: ceph-mimic for swift tasks (`pr#32033 <https://github.com/ceph/ceph/pull/32033>`_, Casey Bodley)
+
+
+v13.2.7 Mimic
+=============
+
+This is the seventh bugfix release of the Mimic v13.2.x long-term stable release
+series. All Mimic users are advised to upgrade.
+
+Notable Changes
+---------------
+
+MDS:
+
+* Cache trimming is now throttled. Dropping the MDS cache via the "ceph tell
+  mds.<foo> cache drop" command or large reductions in the cache size will no
+  longer cause service unavailability.
+
+* Behavior with recalling caps has been significantly improved to not attempt
+  recalling too many caps at once, leading to instability. MDS with a large
+  cache (64GB+) should be more stable.
+
+* MDS now provides a config option "mds_max_caps_per_client" (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
+* The "mds_recall_state_timeout" config parameter has been removed. Late client
+  recall warnings are now generated based on the number of caps the MDS has
+  recalled which have not been released. The new config parameters
+  "mds_recall_warning_threshold" (default: 32K) and "mds_recall_warning_decay_rate"
+  (default: 60s) set the threshold for this warning.
+
+* The "cache drop" admin socket command has been removed. The "ceph tell mds.X
+  cache drop" remains.
+
+OSD:
+
+* A health warning is now generated if the average osd heartbeat ping time exceeds
+  a configurable threshold for any of the intervals computed. The OSD computes
+  1 minute, 5 minute and 15 minute intervals with average, minimum and maximum
+  values.  New configuration option "mon_warn_on_slow_ping_ratio" specifies
+  a percentage of "osd_heartbeat_grace" to determine the threshold.  A value of
+  zero disables the warning. A new configuration option "mon_warn_on_slow_ping_time",
+  specified in milliseconds, overrides the computed value, causing a warning
+  when OSD heartbeat pings take longer than the specified amount.  A new admin
+  command "ceph daemon mgr.# dump_osd_network [threshold]" lists all connections
+  with a ping time longer than the specified threshold or value determined by
+  the config options, for the average for any of the 3 intervals.  A new admin
+  command "ceph daemon osd.# dump_osd_network [threshold]" does the same but only
+  including heartbeats initiated by the specified OSD.
+
+* The default value of the "osd_deep_scrub_large_omap_object_key_threshold"
+  parameter has been lowered to detect an object with large number of omap keys
+  more easily.
+  
+RGW:
+
+* radosgw-admin introduces two subcommands that allow the managing of expire-stale
+  objects that might be left behind after a bucket reshard in earlier versions
+  of RGW. One subcommand lists such objects and the other deletes them. Read the
+  troubleshooting section of the dynamic resharding docs for details.
+
+Changelog
+---------
+
+* bluestore: 50-100% iops lost due to bluefs_preextend_wal_files = false (`issue#40280 <http://tracker.ceph.com/issues/40280>`_, `pr#28574 <https://github.com/ceph/ceph/pull/28574>`_, Vitaliy Filippov)
+* bluestore: Change default for bluestore_fsck_on_mount_deep as false (`pr#29699 <https://github.com/ceph/ceph/pull/29699>`_, Neha Ojha)
+* bluestore: _txc_add_transaction error (39) Directory not empty not handled on operation 21 (op 1, counting from 0) (`issue#39692 <http://tracker.ceph.com/issues/39692>`_, `pr#29217 <https://github.com/ceph/ceph/pull/29217>`_, Sage Weil)
+* bluestore: apply shared_alloc_size to shared device with log level change (`pr#30219 <https://github.com/ceph/ceph/pull/30219>`_, Vikhyat Umrao, Josh Durgin, Sage Weil, Igor Fedotov)
+* bluestore: avoid length overflow in extents returned by Stupid Allocator (`issue#40758 <http://tracker.ceph.com/issues/40758>`_, `issue#40703 <http://tracker.ceph.com/issues/40703>`_, `pr#29024 <https://github.com/ceph/ceph/pull/29024>`_, Igor Fedotov)
+* bluestore: common/options: Set concurrent bluestore rocksdb compactions to 2 (`pr#30150 <https://github.com/ceph/ceph/pull/30150>`_, Mark Nelson)
+* bluestore: default to bitmap allocator for bluestore/bluefs (`pr#28970 <https://github.com/ceph/ceph/pull/28970>`_, Igor Fedotov)
+* bluestore: dump before "no-spanning blob id" abort (`pr#28029 <https://github.com/ceph/ceph/pull/28029>`_, Igor Fedotov)
+* bluestore: fix >2GB bluefs writes (`pr#28967 <https://github.com/ceph/ceph/pull/28967>`_, Sage Weil, kungf)
+* bluestore: fix duplicate allocations in bmap allocator (`issue#40080 <http://tracker.ceph.com/issues/40080>`_, `pr#28645 <https://github.com/ceph/ceph/pull/28645>`_, Igor Fedotov)
+* bluestore: load OSD all compression settings unconditionally (`issue#40480 <http://tracker.ceph.com/issues/40480>`_, `pr#28894 <https://github.com/ceph/ceph/pull/28894>`_, Igor Fedotov)
+* build/ops: Cython 0.29 removed support for subinterpreters: raises ImportError: Interpreter change detected (`issue#39593 <http://tracker.ceph.com/issues/39593>`_, `issue#39592 <http://tracker.ceph.com/issues/39592>`_, `pr#27971 <https://github.com/ceph/ceph/pull/27971>`_, Kefu Chai, Tim Serong)
+* build/ops: admin/build-doc: use python3 (`pr#30663 <https://github.com/ceph/ceph/pull/30663>`_, Kefu Chai)
+* build/ops: admin/build-doc: use python3 (follow-on fix) (`pr#30687 <https://github.com/ceph/ceph/pull/30687>`_, Nathan Cutler)
+* build/ops: backport miscellaneous install-deps.sh and ceph.spec.in fixes from master (`issue#37707 <http://tracker.ceph.com/issues/37707>`_, `pr#30718 <https://github.com/ceph/ceph/pull/30718>`_, Jeff Layton, Kefu Chai, Nathan Cutler, Brad Hubbard, Changcheng Liu, Sebastian Wagner, Yunchuan Wen, Tomasz Setkowski, Zack Cerza)
+* build/ops: ceph.spec.in: reserve 2500MB per build job (`pr#30355 <https://github.com/ceph/ceph/pull/30355>`_, Dan van der Ster)
+* build/ops: cmake,run-make-check.sh: disable SPDK by default (`pr#30183 <https://github.com/ceph/ceph/pull/30183>`_, Kefu Chai)
+* build/ops: cmake: detect armv8 crc and crypto feature using CHECK_C_COMPILER_FLAG (`issue#17516 <http://tracker.ceph.com/issues/17516>`_, `pr#30713 <https://github.com/ceph/ceph/pull/30713>`_, Kefu Chai)
+* build/ops: do_cmake.sh: source not found (`issue#39981 <http://tracker.ceph.com/issues/39981>`_, `issue#40005 <http://tracker.ceph.com/issues/40005>`_, `pr#28217 <https://github.com/ceph/ceph/pull/28217>`_, Nathan Cutler)
+* build/ops: fix build fail related to PYTHON_EXECUTABLE variable (`pr#30260 <https://github.com/ceph/ceph/pull/30260>`_, Ilsoo Byun)
+* build/ops: install-deps.sh: Remove CR repo (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#30128 <https://github.com/ceph/ceph/pull/30128>`_, Alfredo Deza, Brad Hubbard)
+* build/ops: install-deps.sh: install `python\*-devel` for python\*rpm-macros (`pr#30244 <https://github.com/ceph/ceph/pull/30244>`_, Kefu Chai)
+* build/ops: make "patch" build dependency explicit (`issue#40269 <http://tracker.ceph.com/issues/40269>`_, `issue#40175 <http://tracker.ceph.com/issues/40175>`_, `pr#29150 <https://github.com/ceph/ceph/pull/29150>`_, Nathan Cutler)
+* build/ops: python3-cephfs should provide python36-cephfs (`pr#30982 <https://github.com/ceph/ceph/pull/30982>`_, Kefu Chai)
+* build/ops: rpm: always build ceph-test package (`pr#30188 <https://github.com/ceph/ceph/pull/30188>`_, Nathan Cutler)
+* ceph-volume: PVolumes.filter shouldn't purge itself (`pr#30806 <https://github.com/ceph/ceph/pull/30806>`_, Rishabh Dave)
+* ceph-volume: VolumeGroups.filter shouldn't purge itself (`pr#30808 <https://github.com/ceph/ceph/pull/30808>`_, Rishabh Dave)
+* ceph-volume: add Ceph's device id to inventory (`pr#31211 <https://github.com/ceph/ceph/pull/31211>`_, Sebastian Wagner)
+* ceph-volume: api/lvm: check if list of LVs is empty (`pr#31229 <https://github.com/ceph/ceph/pull/31229>`_, Rishabh Dave)
+* ceph-volume: assume msgrV1 for all branches containing mimic (`pr#31615 <https://github.com/ceph/ceph/pull/31615>`_, Jan Fajerski)
+* ceph-volume: batch functional idempotency test fails since message is now on stderr (`pr#29688 <https://github.com/ceph/ceph/pull/29688>`_, Jan Fajerski)
+* ceph-volume: broken assertion errors after pytest changes (`pr#28948 <https://github.com/ceph/ceph/pull/28948>`_, Alfredo Deza)
+* ceph-volume: do not fail when trying to remove crypt mapper (`pr#30555 <https://github.com/ceph/ceph/pull/30555>`_, Guillaume Abrioux)
+* ceph-volume: does not recognize wal/db partitions created by ceph-disk (`pr#29463 <https://github.com/ceph/ceph/pull/29463>`_, Jan Fajerski)
+* ceph-volume: ensure device lists are disjoint (`pr#30334 <https://github.com/ceph/ceph/pull/30334>`_, Jan Fajerski)
+* ceph-volume: extend batch (`issue#40919 <http://tracker.ceph.com/issues/40919>`_, `pr#29243 <https://github.com/ceph/ceph/pull/29243>`_, Andrew Schoen, Jan Fajerski, Sébastien Han, Volker Theile)
+* ceph-volume: fix stderr failure to decode/encode when redirected (`pr#30301 <https://github.com/ceph/ceph/pull/30301>`_, Alfredo Deza)
+* ceph-volume: fix warnings raised by pytest (`pr#30678 <https://github.com/ceph/ceph/pull/30678>`_, Rishabh Dave)
+* ceph-volume: implement __format__ in Size to format sizes in py3 (`pr#30333 <https://github.com/ceph/ceph/pull/30333>`_, Jan Fajerski)
+* ceph-volume: look for rotational data in lsblk (`pr#26991 <https://github.com/ceph/ceph/pull/26991>`_, Andrew Schoen)
+* ceph-volume: lvm.activate: Return an error if WAL/DB devices absent (`pr#29039 <https://github.com/ceph/ceph/pull/29039>`_, David Casier)
+* ceph-volume: lvm.zap fix cleanup for db partitions (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30303 <https://github.com/ceph/ceph/pull/30303>`_, Dominik Csapak)
+* ceph-volume: minor optimizations related to class Volumes's use (`pr#30096 <https://github.com/ceph/ceph/pull/30096>`_, Rishabh Dave)
+* ceph-volume: miscellaneous backports (`pr#31227 <https://github.com/ceph/ceph/pull/31227>`_, Mohamad Gebai, Andrew Schoen)
+* ceph-volume: missing string substitution when reporting mounts (`issue#40977 <http://tracker.ceph.com/issues/40977>`_, `pr#29350 <https://github.com/ceph/ceph/pull/29350>`_, Shyukri Shyukriev)
+* ceph-volume: more mimic backports (`pr#29631 <https://github.com/ceph/ceph/pull/29631>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: more missing mimic backports (`pr#31362 <https://github.com/ceph/ceph/pull/31362>`_, Mohamad Gebai, Kefu Chai)
+* ceph-volume: pre-install python-apt and its variants before test runs (`pr#30295 <https://github.com/ceph/ceph/pull/30295>`_, Alfredo Deza)
+* ceph-volume: prints errors to stdout with --format json (`issue#38548 <http://tracker.ceph.com/issues/38548>`_, `pr#29507 <https://github.com/ceph/ceph/pull/29507>`_, Jan Fajerski)
+* ceph-volume: prints log messages to stdout (`pr#29602 <https://github.com/ceph/ceph/pull/29602>`_, Jan Fajerski, Alfredo Deza, Kefu Chai)
+* ceph-volume: replace testinfra command with py.test (`pr#28930 <https://github.com/ceph/ceph/pull/28930>`_, Alfredo Deza)
+* ceph-volume: simple functional tests drop test for lvm zap (`pr#29661 <https://github.com/ceph/ceph/pull/29661>`_, Jan Fajerski)
+* ceph-volume: simple: when 'type' file is not present activate fails (`pr#29417 <https://github.com/ceph/ceph/pull/29417>`_, Jan Fajerski, Alfredo Deza)
+* ceph-volume: tests add a sleep in tox for slow OSDs after booting (`pr#28947 <https://github.com/ceph/ceph/pull/28947>`_, Alfredo Deza)
+* ceph-volume: tests set the noninteractive flag for Debian (`pr#29900 <https://github.com/ceph/ceph/pull/29900>`_, Alfredo Deza)
+* ceph-volume: update testing playbook 'deploy.yml' (`pr#29074 <https://github.com/ceph/ceph/pull/29074>`_, Andrew Schoen, Guillaume Abrioux)
+* ceph-volume: use the OSD identifier when reporting success (`pr#29770 <https://github.com/ceph/ceph/pull/29770>`_, Alfredo Deza)
+* ceph-volume: zap always skips block.db, leaves them around (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30306 <https://github.com/ceph/ceph/pull/30306>`_, Alfredo Deza)
+* ceph_detect_init: Add support for ALT Linux (`pr#27028 <https://github.com/ceph/ceph/pull/27028>`_, Andrey Bychkov)
+* cephfs: MDSTableServer.cc: 83: FAILED assert(version == tid) (`issue#39212 <http://tracker.ceph.com/issues/39212>`_, `issue#38835 <http://tracker.ceph.com/issues/38835>`_, `pr#29222 <https://github.com/ceph/ceph/pull/29222>`_, "Yan, Zheng")
+* cephfs: avoid map been inserted by mistake (`pr#29833 <https://github.com/ceph/ceph/pull/29833>`_, XiaoGuoDong2019)
+* cephfs: ceph-fuse: client hang because its bad session PipeConnection to mds (`issue#39305 <http://tracker.ceph.com/issues/39305>`_, `issue#39685 <http://tracker.ceph.com/issues/39685>`_, `pr#29200 <https://github.com/ceph/ceph/pull/29200>`_, Guan yunfei)
+* cephfs: client: EINVAL may be returned when offset is 0 (`pr#30932 <https://github.com/ceph/ceph/pull/30932>`_, wenpengLi)
+* cephfs: client: _readdir_cache_cb() may use the readdir_cache already clear (`issue#41148 <http://tracker.ceph.com/issues/41148>`_, `pr#30933 <https://github.com/ceph/ceph/pull/30933>`_, huanwen ren)
+* cephfs: client: add procession of SEEK_HOLE and SEEK_DATA in lseek (`pr#30918 <https://github.com/ceph/ceph/pull/30918>`_, Shen Hang)
+* cephfs: client: bump ll_ref from int32 to uint64_t (`pr#29187 <https://github.com/ceph/ceph/pull/29187>`_, Xiaoxi CHEN)
+* cephfs: client: ceph.dir.rctime xattr value incorrectly prefixes 09 to the nanoseconds component (`issue#40168 <http://tracker.ceph.com/issues/40168>`_, `pr#28501 <https://github.com/ceph/ceph/pull/28501>`_, David Disseldorp)
+* cephfs: client: fix bad error handling in _lookup_parent (`issue#40085 <http://tracker.ceph.com/issues/40085>`_, `pr#29609 <https://github.com/ceph/ceph/pull/29609>`_, Jeff Layton)
+* cephfs: client: nfs-ganesha with cephfs client, removing dir reports not empty (`issue#40746 <http://tracker.ceph.com/issues/40746>`_, `pr#30443 <https://github.com/ceph/ceph/pull/30443>`_, Peng Xie)
+* cephfs: client: return -EIO when sync file which unsafe reqs have been dropped (`issue#40877 <http://tracker.ceph.com/issues/40877>`_, `pr#30241 <https://github.com/ceph/ceph/pull/30241>`_, simon gao)
+* cephfs: client: set snapdir's link count to 1 (`pr#30108 <https://github.com/ceph/ceph/pull/30108>`_, "Yan, Zheng")
+* cephfs: client: support the fallocate() when fuse version >= 2.9 (`issue#40615 <http://tracker.ceph.com/issues/40615>`_, `pr#30228 <https://github.com/ceph/ceph/pull/30228>`_, huanwen ren)
+* cephfs: client: unlink dentry for inode with llref=0 (`issue#40960 <http://tracker.ceph.com/issues/40960>`_, `pr#29479 <https://github.com/ceph/ceph/pull/29479>`_, Xiaoxi CHEN)
+* cephfs: fix a memory leak (`pr#29915 <https://github.com/ceph/ceph/pull/29915>`_, XiaoGuoDong2019)
+* cephfs: getattr on snap inode stuck (`issue#40437 <http://tracker.ceph.com/issues/40437>`_, `pr#29230 <https://github.com/ceph/ceph/pull/29230>`_, "Yan, Zheng")
+* cephfs: kcephfs TestClientLimits.test_client_pin fails with client caps fell below min (`issue#38270 <http://tracker.ceph.com/issues/38270>`_, `issue#38687 <http://tracker.ceph.com/issues/38687>`_, `pr#29211 <https://github.com/ceph/ceph/pull/29211>`_, "Yan, Zheng")
+* cephfs: mds: Fix duplicate client entries in eviction list (`pr#30950 <https://github.com/ceph/ceph/pull/30950>`_, Sidharth Anupkrishnan)
+* cephfs: mds: avoid sending too many osd requests at once after mds restarts (`issue#40042 <http://tracker.ceph.com/issues/40042>`_, `issue#40028 <http://tracker.ceph.com/issues/40028>`_, `pr#28650 <https://github.com/ceph/ceph/pull/28650>`_, simon gao)
+* cephfs: mds: behind on trimming and [dentry] was purgeable but no longer is! (`issue#39223 <http://tracker.ceph.com/issues/39223>`_, `issue#38679 <http://tracker.ceph.com/issues/38679>`_, `pr#29224 <https://github.com/ceph/ceph/pull/29224>`_, "Yan, Zheng")
+* cephfs: mds: cannot switch mds state from standby-replay to active (`issue#40213 <http://tracker.ceph.com/issues/40213>`_, `pr#29232 <https://github.com/ceph/ceph/pull/29232>`_, "Yan, Zheng", simon gao)
+* cephfs: mds: change how mds revoke stale caps (`issue#38043 <http://tracker.ceph.com/issues/38043>`_, `issue#17854 <http://tracker.ceph.com/issues/17854>`_, `pr#28585 <https://github.com/ceph/ceph/pull/28585>`_, "Yan, Zheng", Rishabh Dave)
+* cephfs: mds: check dir fragment to split dir if mkdir makes it oversized (`issue#39689 <http://tracker.ceph.com/issues/39689>`_, `pr#28381 <https://github.com/ceph/ceph/pull/28381>`_, Erqi Chen)
+* cephfs: mds: cleanup unneeded client_snap_caps when splitting snap inode (`issue#39987 <http://tracker.ceph.com/issues/39987>`_, `pr#30234 <https://github.com/ceph/ceph/pull/30234>`_, "Yan, Zheng")
+* cephfs: mds: delay exporting directory whose pin value exceeds max rank id (`issue#40603 <http://tracker.ceph.com/issues/40603>`_, `pr#29940 <https://github.com/ceph/ceph/pull/29940>`_, Zhi Zhang)
+* cephfs: mds: destroy reconnect msg when it is from non-existent session to avoid memory leak (`issue#40588 <http://tracker.ceph.com/issues/40588>`_, `pr#28796 <https://github.com/ceph/ceph/pull/28796>`_, Shen Hang)
+* cephfs: mds: evict an unresponsive client only when another client wants its caps (`pr#30239 <https://github.com/ceph/ceph/pull/30239>`_, Rishabh Dave)
+* cephfs: mds: fix SnapRealm::resolve_snapname for long name (`issue#39472 <http://tracker.ceph.com/issues/39472>`_, `pr#28186 <https://github.com/ceph/ceph/pull/28186>`_, "Yan, Zheng")
+* cephfs: mds: fix corner case of replaying open sessions (`pr#28579 <https://github.com/ceph/ceph/pull/28579>`_, "Yan, Zheng")
+* cephfs: mds: high debug logging with many subtrees is slow (`issue#38875 <http://tracker.ceph.com/issues/38875>`_, `pr#29219 <https://github.com/ceph/ceph/pull/29219>`_, Rishabh Dave)
+* cephfs: mds: make MDSIOContextBase delete itself when shutting down (`pr#30417 <https://github.com/ceph/ceph/pull/30417>`_, Xuehan Xu)
+* cephfs: mds: mds_cap_revoke_eviction_timeout is not used to initialize Server::cap_revoke_eviction_timeout (`issue#38844 <http://tracker.ceph.com/issues/38844>`_, `issue#39210 <http://tracker.ceph.com/issues/39210>`_, `pr#29220 <https://github.com/ceph/ceph/pull/29220>`_, simon gao)
+* cephfs: mds: output lock state in format dump (`issue#39669 <http://tracker.ceph.com/issues/39669>`_, `issue#39645 <http://tracker.ceph.com/issues/39645>`_, `pr#28274 <https://github.com/ceph/ceph/pull/28274>`_, Zhi Zhang)
+* cephfs: mds: remove cache drop admin socket command (`issue#38020 <http://tracker.ceph.com/issues/38020>`_, `issue#38099 <http://tracker.ceph.com/issues/38099>`_, `pr#29210 <https://github.com/ceph/ceph/pull/29210>`_, Patrick Donnelly)
+* cephfs: mds: reset heartbeat during long-running loops in recovery (`issue#40222 <http://tracker.ceph.com/issues/40222>`_, `pr#28918 <https://github.com/ceph/ceph/pull/28918>`_, "Yan, Zheng")
+* cephfs: mds: stopping MDS with a large cache (40+GB) causes it to miss heartbeats (`issue#38022 <http://tracker.ceph.com/issues/38022>`_, `issue#38129 <http://tracker.ceph.com/issues/38129>`_, `issue#37723 <http://tracker.ceph.com/issues/37723>`_, `issue#38131 <http://tracker.ceph.com/issues/38131>`_, `pr#28452 <https://github.com/ceph/ceph/pull/28452>`_, Patrick Donnelly)
+* cephfs: mds: there is an assertion when calling Beacon::shutdown() (`issue#39215 <http://tracker.ceph.com/issues/39215>`_, `issue#38822 <http://tracker.ceph.com/issues/38822>`_, `pr#29223 <https://github.com/ceph/ceph/pull/29223>`_, huanwen ren)
+* cephfs: mount.ceph.c: do not pass nofail to the kernel (`issue#39233 <http://tracker.ceph.com/issues/39233>`_, `pr#28090 <https://github.com/ceph/ceph/pull/28090>`_, Kenneth Waegeman)
+* cephfs: mount.ceph: properly handle -o strictatime (`pr#30240 <https://github.com/ceph/ceph/pull/30240>`_, Jeff Layton)
+* cephfs: mount: key parsing fail when doing a remount (`issue#40165 <http://tracker.ceph.com/issues/40165>`_, `pr#29225 <https://github.com/ceph/ceph/pull/29225>`_, Luis Henriques)
+* cephfs: pybind: added lseek() (`issue#39679 <http://tracker.ceph.com/issues/39679>`_, `pr#28337 <https://github.com/ceph/ceph/pull/28337>`_, Xiaowei Chu)
+* cephfs: test_volume_client: fix test_put_object_versioned() (`issue#39405 <http://tracker.ceph.com/issues/39405>`_, `issue#39510 <http://tracker.ceph.com/issues/39510>`_, `pr#30236 <https://github.com/ceph/ceph/pull/30236>`_, Rishabh Dave)
+* common/ceph_context: avoid unnecessary wait during service thread shutdown (`pr#31096 <https://github.com/ceph/ceph/pull/31096>`_, Jason Dillaman)
+* common/options.cc: Lower the default value of osd_deep_scrub_large_omap_object_key_threshold (`pr#29174 <https://github.com/ceph/ceph/pull/29174>`_, Neha Ojha)
+* common/util: handle long lines in /proc/cpuinfo (`issue#39475 <http://tracker.ceph.com/issues/39475>`_, `issue#38296 <http://tracker.ceph.com/issues/38296>`_, `pr#28206 <https://github.com/ceph/ceph/pull/28206>`_, Sage Weil)
+* common: Keyrings created by ceph auth get are not suitable for ceph auth import (`issue#22227 <http://tracker.ceph.com/issues/22227>`_, `issue#40547 <http://tracker.ceph.com/issues/40547>`_, `pr#28741 <https://github.com/ceph/ceph/pull/28741>`_, Kefu Chai)
+* common: data race in OutputDataSocket (`issue#40268 <http://tracker.ceph.com/issues/40268>`_, `issue#40188 <http://tracker.ceph.com/issues/40188>`_, `pr#29201 <https://github.com/ceph/ceph/pull/29201>`_, Casey Bodley)
+* common: parse ISO 8601 datetime format (`issue#40088 <http://tracker.ceph.com/issues/40088>`_, `pr#28326 <https://github.com/ceph/ceph/pull/28326>`_, Sage Weil)
+* core: .mgrstat failed to decode mgrstat state; luminous dev version? (`issue#38852 <http://tracker.ceph.com/issues/38852>`_, `issue#38839 <http://tracker.ceph.com/issues/38839>`_, `pr#29249 <https://github.com/ceph/ceph/pull/29249>`_, Sage Weil)
+* core: Better default value for osd_snap_trim_sleep (`pr#29732 <https://github.com/ceph/ceph/pull/29732>`_, Neha Ojha)
+* core: Health warnings on long network ping times (`issue#40640 <http://tracker.ceph.com/issues/40640>`_, `issue#40586 <http://tracker.ceph.com/issues/40586>`_, `pr#30225 <https://github.com/ceph/ceph/pull/30225>`_, xie xingguo, David Zafman)
+* core: ceph daemon mon.a config set mon_health_to_clog false cause leader mon assert (`issue#39625 <http://tracker.ceph.com/issues/39625>`_, `pr#29741 <https://github.com/ceph/ceph/pull/29741>`_, huangjun)
+* core: crc cache should be invalidated when posting preallocated rx buffers (`issue#38437 <http://tracker.ceph.com/issues/38437>`_, `pr#29247 <https://github.com/ceph/ceph/pull/29247>`_, Ilya Dryomov)
+* core: lazy omap stat collection (`pr#29189 <https://github.com/ceph/ceph/pull/29189>`_, Brad Hubbard)
+* core: mon, osd: parallel clean_pg_upmaps (`issue#40104 <http://tracker.ceph.com/issues/40104>`_, `issue#40230 <http://tracker.ceph.com/issues/40230>`_, `pr#28619 <https://github.com/ceph/ceph/pull/28619>`_, xie xingguo)
+* core: mon,osd: limit MOSDMap messages by size as well as map count (`issue#38277 <http://tracker.ceph.com/issues/38277>`_, `issue#38040 <http://tracker.ceph.com/issues/38040>`_, `pr#29242 <https://github.com/ceph/ceph/pull/29242>`_, Sage Weil)
+* core: mon/AuthMonitor: fix initial creation of rotating keys (`issue#40634 <http://tracker.ceph.com/issues/40634>`_, `pr#30181 <https://github.com/ceph/ceph/pull/30181>`_, Sage Weil)
+* core: mon/MDSMonitor: use stringstream instead of dout for mds repaired (`issue#40472 <http://tracker.ceph.com/issues/40472>`_, `pr#30235 <https://github.com/ceph/ceph/pull/30235>`_, Zhi Zhang)
+* core: mon/MgrMonitor: fix null deref when invalid formatter is specified (`pr#29593 <https://github.com/ceph/ceph/pull/29593>`_, Sage Weil)
+* core: mon/OSDMonitor.cc: better error message about min_size (`pr#29618 <https://github.com/ceph/ceph/pull/29618>`_, Neha Ojha)
+* core: mon/OSDMonitor: trim not-longer-exist failure reporters (`pr#30903 <https://github.com/ceph/ceph/pull/30903>`_, NancySu05)
+* core: mon: C_AckMarkedDown has not handled the Callback Arguments (`pr#30213 <https://github.com/ceph/ceph/pull/30213>`_, NancySu05)
+* core: mon: ensure prepare_failure() marks no_reply on op (`pr#30481 <https://github.com/ceph/ceph/pull/30481>`_, Joao Eduardo Luis)
+* core: mon: paxos: introduce new reset_pending_committing_finishers for safety (`issue#39744 <http://tracker.ceph.com/issues/39744>`_, `issue#39484 <http://tracker.ceph.com/issues/39484>`_, `pr#28540 <https://github.com/ceph/ceph/pull/28540>`_, Greg Farnum)
+* core: mon: show pool id in pool ls command (`issue#40287 <http://tracker.ceph.com/issues/40287>`_, `pr#30485 <https://github.com/ceph/ceph/pull/30485>`_, Chang Liu)
+* core: osd beacon sometimes has empty pg list (`issue#40464 <http://tracker.ceph.com/issues/40464>`_, `issue#40377 <http://tracker.ceph.com/issues/40377>`_, `pr#29253 <https://github.com/ceph/ceph/pull/29253>`_, Sage Weil)
+* core: osd/OSD.cc: make osd bench description consistent with parameters (`issue#39374 <http://tracker.ceph.com/issues/39374>`_, `issue#39006 <http://tracker.ceph.com/issues/39006>`_, `pr#28097 <https://github.com/ceph/ceph/pull/28097>`_, Neha Ojha)
+* core: osd/OSDCap: Check for empty namespace (`issue#40835 <http://tracker.ceph.com/issues/40835>`_, `pr#30214 <https://github.com/ceph/ceph/pull/30214>`_, Brad Hubbard)
+* core: osd/OSDMap: Replace get_out_osds with get_out_existing_osds (`issue#39422 <http://tracker.ceph.com/issues/39422>`_, `issue#39154 <http://tracker.ceph.com/issues/39154>`_, `pr#28142 <https://github.com/ceph/ceph/pull/28142>`_, Brad Hubbard)
+* core: osd/OSDMap: do not trust partially simplified pg_upmap_item (`pr#30898 <https://github.com/ceph/ceph/pull/30898>`_, xie xingguo)
+* core: osd/PG: Add PG to large omap log message (`pr#30924 <https://github.com/ceph/ceph/pull/30924>`_, Brad Hubbard)
+* core: osd/PG: fix last_complete re-calculation on splitting (`issue#39538 <http://tracker.ceph.com/issues/39538>`_, `issue#26958 <http://tracker.ceph.com/issues/26958>`_, `pr#28259 <https://github.com/ceph/ceph/pull/28259>`_, xie xingguo)
+* core: osd/PeeringState: do not complain about past_intervals constrained by oldest epoch (`pr#30222 <https://github.com/ceph/ceph/pull/30222>`_, Sage Weil)
+* core: osd/PeeringState: recover_got - add special handler for empty log (`pr#30895 <https://github.com/ceph/ceph/pull/30895>`_, xie xingguo)
+* core: osd/PrimaryLogPG: Avoid accessing destroyed references in finish_degr… (`pr#30291 <https://github.com/ceph/ceph/pull/30291>`_, Tao Ning)
+* core: osd/PrimaryLogPG: skip obcs that don't exist during backfill scan_range (`pr#31029 <https://github.com/ceph/ceph/pull/31029>`_, Sage Weil)
+* core: osd/PrimaryLogPG: update oi.size on write op implicitly truncating ob… (`pr#30275 <https://github.com/ceph/ceph/pull/30275>`_, xie xingguo)
+* core: osd: Better error message when OSD count is less than osd_pool_default_size (`issue#38617 <http://tracker.ceph.com/issues/38617>`_, `pr#30180 <https://github.com/ceph/ceph/pull/30180>`_, Kefu Chai, Sage Weil, zjh)
+* core: osd: Don't evict after a flush if intersecting scrub range (`issue#38840 <http://tracker.ceph.com/issues/38840>`_, `issue#39518 <http://tracker.ceph.com/issues/39518>`_, `pr#28232 <https://github.com/ceph/ceph/pull/28232>`_, David Zafman)
+* core: osd: Don't include user changeable flag in snaptrim related assert (`issue#38124 <http://tracker.ceph.com/issues/38124>`_, `issue#39698 <http://tracker.ceph.com/issues/39698>`_, `pr#28202 <https://github.com/ceph/ceph/pull/28202>`_, David Zafman)
+* core: osd: Fix for compatibility of encode/decode of osd_stat_t (`pr#31275 <https://github.com/ceph/ceph/pull/31275>`_, Kefu Chai, David Zafman)
+* core: osd: Include dups in copy_after() and copy_up_to() (`issue#39304 <http://tracker.ceph.com/issues/39304>`_, `pr#28089 <https://github.com/ceph/ceph/pull/28089>`_, David Zafman)
+* core: osd: Output Base64 encoding of CRC header if binary data present (`issue#39737 <http://tracker.ceph.com/issues/39737>`_, `pr#28503 <https://github.com/ceph/ceph/pull/28503>`_, David Zafman)
+* core: osd: Remove unused osdmap flags full, nearfull from output (`pr#30901 <https://github.com/ceph/ceph/pull/30901>`_, David Zafman)
+* core: osd: clear PG_STATE_CLEAN when repair object (`pr#30243 <https://github.com/ceph/ceph/pull/30243>`_, Zengran Zhang)
+* core: osd: fix build_incremental_map_msg (`issue#38282 <http://tracker.ceph.com/issues/38282>`_, `pr#31236 <https://github.com/ceph/ceph/pull/31236>`_, Sage Weil)
+* core: osd: make project_pg_history handle concurrent osdmap publish (`issue#26970 <http://tracker.ceph.com/issues/26970>`_, `pr#29976 <https://github.com/ceph/ceph/pull/29976>`_, Sage Weil)
+* core: osd: merge replica log on primary need according to replica log's crt (`pr#30916 <https://github.com/ceph/ceph/pull/30916>`_, Zengran Zhang)
+* core: osd: pg stuck in backfill_wait with plenty of disk space (`issue#38034 <http://tracker.ceph.com/issues/38034>`_, `pr#28201 <https://github.com/ceph/ceph/pull/28201>`_, xie xingguo, David Zafman)
+* core: osd: report omap/data/metadata usage (`issue#40639 <http://tracker.ceph.com/issues/40639>`_, `pr#28852 <https://github.com/ceph/ceph/pull/28852>`_, Sage Weil)
+* core: osd: rollforward may need to mark pglog dirty (`issue#40403 <http://tracker.ceph.com/issues/40403>`_, `pr#31035 <https://github.com/ceph/ceph/pull/31035>`_, Zengran Zhang)
+* core: osd: scrub error on big objects; make bluestore refuse to start on big objects (`pr#30784 <https://github.com/ceph/ceph/pull/30784>`_, David Zafman, Sage Weil)
+* core: osd: take heartbeat_lock when calling heartbeat() (`issue#39513 <http://tracker.ceph.com/issues/39513>`_, `issue#39439 <http://tracker.ceph.com/issues/39439>`_, `pr#28220 <https://github.com/ceph/ceph/pull/28220>`_, Sage Weil)
+* core: osds allows to partially start more than N+2 (`issue#38206 <http://tracker.ceph.com/issues/38206>`_, `issue#38076 <http://tracker.ceph.com/issues/38076>`_, `pr#29241 <https://github.com/ceph/ceph/pull/29241>`_, Sage Weil)
+* core: should report EINVAL in ErasureCode::parse() if m<=0 (`issue#38682 <http://tracker.ceph.com/issues/38682>`_, `issue#38751 <http://tracker.ceph.com/issues/38751>`_, `pr#28995 <https://github.com/ceph/ceph/pull/28995>`_, Sage Weil)
+* core: should set EPOLLET flag on del_event() (`issue#38856 <http://tracker.ceph.com/issues/38856>`_, `pr#29250 <https://github.com/ceph/ceph/pull/29250>`_, Roman Penyaev)
+* doc/ceph-fuse: mention -k option in ceph-fuse man page (`pr#30936 <https://github.com/ceph/ceph/pull/30936>`_, Rishabh Dave)
+* doc/rbd: s/guess/xml/ for codeblock lexer (`pr#31090 <https://github.com/ceph/ceph/pull/31090>`_, Kefu Chai)
+* doc/rgw: document use of 'realm pull' instead of 'period pull' (`issue#39655 <http://tracker.ceph.com/issues/39655>`_, `pr#30131 <https://github.com/ceph/ceph/pull/30131>`_, Casey Bodley)
+* doc: Document behaviour of fsync-after-close (`issue#24641 <http://tracker.ceph.com/issues/24641>`_, `pr#29765 <https://github.com/ceph/ceph/pull/29765>`_, Jos Collin, Jeff Layton)
+* doc: Object Gateway multisite document read-only argument error (`issue#40497 <http://tracker.ceph.com/issues/40497>`_, `pr#29289 <https://github.com/ceph/ceph/pull/29289>`_, Chenjiong Deng)
+* doc: default values for mon_health_to_clog\_\* were flipped (`pr#30227 <https://github.com/ceph/ceph/pull/30227>`_, James McClune)
+* doc: describe metadata_heap cleanup (`issue#18174 <http://tracker.ceph.com/issues/18174>`_, `pr#30070 <https://github.com/ceph/ceph/pull/30070>`_, Dan van der Ster)
+* doc: fix rgw_ldap_dnattr username token (`pr#30099 <https://github.com/ceph/ceph/pull/30099>`_, Thomas Kriechbaumer)
+* doc: rgw: CreateBucketConfiguration for s3 PUT Bucket request (`issue#39602 <http://tracker.ceph.com/issues/39602>`_, `issue#39597 <http://tracker.ceph.com/issues/39597>`_, `pr#29257 <https://github.com/ceph/ceph/pull/29257>`_, Casey Bodley)
+* doc: update bluestore cache settings and clarify data fraction (`issue#39522 <http://tracker.ceph.com/issues/39522>`_, `pr#31258 <https://github.com/ceph/ceph/pull/31258>`_, Jan Fajerski)
+* doc: wrong value of usage log default in logging section (`issue#37891 <http://tracker.ceph.com/issues/37891>`_, `issue#37856 <http://tracker.ceph.com/issues/37856>`_, `pr#29014 <https://github.com/ceph/ceph/pull/29014>`_, Abhishek Lekshmanan)
+* filestore: assure sufficient leaves in pre-split (`issue#39390 <http://tracker.ceph.com/issues/39390>`_, `pr#30182 <https://github.com/ceph/ceph/pull/30182>`_, Jeegn Chen)
+* krbd: avoid udev netlink socket overrun and retry on transient errors from udev_enumerate_scan_devices() (`pr#31322 <https://github.com/ceph/ceph/pull/31322>`_, Ilya Dryomov, Adam C. Emerson)
+* krbd: fix rbd map hang due to udev return subsystem unordered (`issue#39089 <http://tracker.ceph.com/issues/39089>`_, `pr#30176 <https://github.com/ceph/ceph/pull/30176>`_, Zhi Zhang)
+* mgr/balancer: fix fudge (`pr#28399 <https://github.com/ceph/ceph/pull/28399>`_, xie xingguo)
+* mgr/balancer: python3 compatibility issue (`pr#31013 <https://github.com/ceph/ceph/pull/31013>`_, Mykola Golub)
+* mgr/balancer: restrict automatic balancing to specific weekdays (`pr#26499 <https://github.com/ceph/ceph/pull/26499>`_, xie xingguo)
+* mgr/crash: fix python3 invalid syntax problems (`pr#29029 <https://github.com/ceph/ceph/pull/29029>`_, Ricardo Dias)
+* mgr/dashboard: Fix run-frontend-e2e-tests.sh (`issue#40707 <http://tracker.ceph.com/issues/40707>`_, `pr#28954 <https://github.com/ceph/ceph/pull/28954>`_, Kiefer Chang, Tiago Melo)
+* mgr/dashboard: Fix various RGW issues (`pr#28210 <https://github.com/ceph/ceph/pull/28210>`_, Volker Theile)
+* mgr/dashboard: RGW proxy can't handle self-signed SSL certificates (`pr#30543 <https://github.com/ceph/ceph/pull/30543>`_, Volker Theile)
+* mgr/dashboard: cephfs multimds graphs stack together (`issue#40660 <http://tracker.ceph.com/issues/40660>`_, `pr#28911 <https://github.com/ceph/ceph/pull/28911>`_, Kiefer Chang)
+* mgr/localpool: pg_num is an int arg to 'osd pool create' (`pr#30447 <https://github.com/ceph/ceph/pull/30447>`_, Sage Weil)
+* mgr/prometheus: Cast collect_timeout (scrape_interval) to float (`pr#31108 <https://github.com/ceph/ceph/pull/31108>`_, Ben Meekhof)
+* mgr/prometheus: replace whitespaces in metrics' names (`issue#39458 <http://tracker.ceph.com/issues/39458>`_, `pr#28165 <https://github.com/ceph/ceph/pull/28165>`_, Alfonso Martínez)
+* mgr/telemetry: Ignore crashes in report when module not enabled (`pr#30846 <https://github.com/ceph/ceph/pull/30846>`_, Wido den Hollander)
+* mgr: DaemonServer::handle_conf_change - broken locking (`issue#38899 <http://tracker.ceph.com/issues/38899>`_, `issue#38963 <http://tracker.ceph.com/issues/38963>`_, `pr#29197 <https://github.com/ceph/ceph/pull/29197>`_, xie xingguo)
+* mgr: deadlock (`issue#39040 <http://tracker.ceph.com/issues/39040>`_, `issue#39426 <http://tracker.ceph.com/issues/39426>`_, `pr#28161 <https://github.com/ceph/ceph/pull/28161>`_, xie xingguo)
+* mgr: do not reset reported if a new metric is not collected (`pr#30391 <https://github.com/ceph/ceph/pull/30391>`_, Ilsoo Byun)
+* radosgw-admin: bucket sync status not 'caught up' during full sync (`issue#40806 <http://tracker.ceph.com/issues/40806>`_, `pr#30170 <https://github.com/ceph/ceph/pull/30170>`_, Casey Bodley)
+* rbd-mirror: cannot restore deferred deletion mirrored images (`pr#30828 <https://github.com/ceph/ceph/pull/30828>`_, Jason Dillaman, Mykola Golub)
+* rbd-mirror: clear out bufferlist prior to listing mirror images (`issue#39461 <http://tracker.ceph.com/issues/39461>`_, `issue#39407 <http://tracker.ceph.com/issues/39407>`_, `pr#28123 <https://github.com/ceph/ceph/pull/28123>`_, Jason Dillaman)
+* rbd-mirror: don't overwrite status error returned by replay (`pr#29872 <https://github.com/ceph/ceph/pull/29872>`_, Mykola Golub)
+* rbd-mirror: handle duplicates in image sync throttler queue (`issue#40519 <http://tracker.ceph.com/issues/40519>`_, `issue#40593 <http://tracker.ceph.com/issues/40593>`_, `pr#28815 <https://github.com/ceph/ceph/pull/28815>`_, Mykola Golub)
+* rbd-mirror: ignore errors relating to parsing the cluster config file (`pr#30117 <https://github.com/ceph/ceph/pull/30117>`_, Jason Dillaman)
+* rbd/action: fix error getting positional argument (`issue#40095 <http://tracker.ceph.com/issues/40095>`_, `pr#29294 <https://github.com/ceph/ceph/pull/29294>`_, songweibin)
+* rbd/tests: avoid hexdump skip and length options in krbd test (`pr#30569 <https://github.com/ceph/ceph/pull/30569>`_, Ilya Dryomov)
+* rbd: Reduce log level for cls/journal and cls/rbd expected errors (`issue#40865 <http://tracker.ceph.com/issues/40865>`_, `pr#29565 <https://github.com/ceph/ceph/pull/29565>`_, Jason Dillaman)
+* rbd: filter out group/trash snapshots from snap_list (`issue#38538 <http://tracker.ceph.com/issues/38538>`_, `issue#39186 <http://tracker.ceph.com/issues/39186>`_, `pr#28138 <https://github.com/ceph/ceph/pull/28138>`_, songweibin, Jason Dillaman)
+* rbd: journal: properly advance read offset after skipping invalid range (`pr#28814 <https://github.com/ceph/ceph/pull/28814>`_, Mykola Golub)
+* rbd: librbd: add missing shutdown states to managed lock helper (`issue#38387 <http://tracker.ceph.com/issues/38387>`_, `issue#38509 <http://tracker.ceph.com/issues/38509>`_, `pr#28151 <https://github.com/ceph/ceph/pull/28151>`_, Jason Dillaman)
+* rbd: librbd: async open/close should free ImageCtx before issuing callback (`issue#39429 <http://tracker.ceph.com/issues/39429>`_, `issue#39031 <http://tracker.ceph.com/issues/39031>`_, `pr#28125 <https://github.com/ceph/ceph/pull/28125>`_, Jason Dillaman)
+* rbd: librbd: avoid dereferencing an empty container during deep-copy (`issue#40368 <http://tracker.ceph.com/issues/40368>`_, `pr#30177 <https://github.com/ceph/ceph/pull/30177>`_, Jason Dillaman)
+* rbd: librbd: disable image mirroring when moving to trash (`pr#28150 <https://github.com/ceph/ceph/pull/28150>`_, Mykola Golub)
+* rbd: librbd: ensure compare-and-write doesn't skip compare after copyup (`issue#38383 <http://tracker.ceph.com/issues/38383>`_, `issue#38441 <http://tracker.ceph.com/issues/38441>`_, `pr#28133 <https://github.com/ceph/ceph/pull/28133>`_, Ilya Dryomov)
+* rbd: librbd: properly handle potential object map failures (`issue#39952 <http://tracker.ceph.com/issues/39952>`_, `issue#36074 <http://tracker.ceph.com/issues/36074>`_, `pr#30796 <https://github.com/ceph/ceph/pull/30796>`_, Jason Dillaman, Mykola Golub)
+* rbd: librbd: properly track in-flight flush requests (`issue#40573 <http://tracker.ceph.com/issues/40573>`_, `pr#28770 <https://github.com/ceph/ceph/pull/28770>`_, Jason Dillaman)
+* rbd: librbd: race condition possible when validating RBD pool (`issue#38500 <http://tracker.ceph.com/issues/38500>`_, `issue#38563 <http://tracker.ceph.com/issues/38563>`_, `pr#28139 <https://github.com/ceph/ceph/pull/28139>`_, Jason Dillaman)
+* rbd: use the ordered throttle for the export action (`issue#40435 <http://tracker.ceph.com/issues/40435>`_, `pr#30178 <https://github.com/ceph/ceph/pull/30178>`_, Jason Dillaman)
+* restful: Query nodes_by_id for items (`pr#31273 <https://github.com/ceph/ceph/pull/31273>`_, Boris Ranto)
+* rgw admin: disable stale instance delete in a multiste env (`pr#30340 <https://github.com/ceph/ceph/pull/30340>`_, Abhishek Lekshmanan)
+* rgw/OutputDataSocket: append_output(buffer::list&) says it will (but does not) discard output at data_max_backlog (`issue#40178 <http://tracker.ceph.com/issues/40178>`_, `issue#40351 <http://tracker.ceph.com/issues/40351>`_, `pr#29279 <https://github.com/ceph/ceph/pull/29279>`_, Matt Benjamin)
+* rgw/cls: keep issuing bilog trim ops after reset (`issue#40187 <http://tracker.ceph.com/issues/40187>`_, `pr#30074 <https://github.com/ceph/ceph/pull/30074>`_, Casey Bodley)
+* rgw/multisite: Don't allow certain radosgw-admin commands to run on non-master zone (`issue#39548 <http://tracker.ceph.com/issues/39548>`_, `pr#30133 <https://github.com/ceph/ceph/pull/30133>`_, Shilpa Jagannath)
+* rgw/rgw_op: Remove get_val from hotpath via legacy options (`pr#30141 <https://github.com/ceph/ceph/pull/30141>`_, Mark Nelson)
+* rgw: Add support for --bypass-gc flag of radosgw-admin bucket rm command in RGW Multi-site (`issue#39748 <http://tracker.ceph.com/issues/39748>`_, `issue#24991 <http://tracker.ceph.com/issues/24991>`_, `pr#29262 <https://github.com/ceph/ceph/pull/29262>`_, Casey Bodley)
+* rgw: Don't crash on copy when metadata directive not supplied (`issue#40416 <http://tracker.ceph.com/issues/40416>`_, `pr#29500 <https://github.com/ceph/ceph/pull/29500>`_, Adam C. Emerson)
+* rgw: Fix bucket versioning vs. swift metadata bug (`pr#30140 <https://github.com/ceph/ceph/pull/30140>`_, Marcus Watts)
+* rgw: Fix rgw decompression log-print (`pr#30156 <https://github.com/ceph/ceph/pull/30156>`_, Han Fengzhe)
+* rgw: Multisite sync corruption for large multipart obj (`issue#40144 <http://tracker.ceph.com/issues/40144>`_, `pr#29273 <https://github.com/ceph/ceph/pull/29273>`_, Casey Bodley, Tianshan Qu, Xiaoxi CHEN)
+* rgw: RGWCoroutine::call(nullptr) sets retcode=0 (`pr#30159 <https://github.com/ceph/ceph/pull/30159>`_, Casey Bodley)
+* rgw: Return tenant field in bucket_stats function (`issue#40038 <http://tracker.ceph.com/issues/40038>`_, `pr#28209 <https://github.com/ceph/ceph/pull/28209>`_, Volker Theile)
+* rgw: S3 policy evaluated incorrectly (`issue#38638 <http://tracker.ceph.com/issues/38638>`_, `issue#39274 <http://tracker.ceph.com/issues/39274>`_, `pr#29255 <https://github.com/ceph/ceph/pull/29255>`_, Pritha Srivastava)
+* rgw: Save an unnecessary copy of RGWEnv (`pr#29483 <https://github.com/ceph/ceph/pull/29483>`_, Mark Kogan)
+* rgw: Swift interface: server side copy fails if object name contains '?' (`issue#27217 <http://tracker.ceph.com/issues/27217>`_, `issue#40128 <http://tracker.ceph.com/issues/40128>`_, `pr#29267 <https://github.com/ceph/ceph/pull/29267>`_, Casey Bodley)
+* rgw: TempURL should not allow PUTs with the X-Object-Manifest (`issue#40133 <http://tracker.ceph.com/issues/40133>`_, `issue#20797 <http://tracker.ceph.com/issues/20797>`_, `pr#28711 <https://github.com/ceph/ceph/pull/28711>`_, Radoslaw Zarzynski)
+* rgw: abort multipart fix (`pr#29016 <https://github.com/ceph/ceph/pull/29016>`_, J. Eric Ivancich)
+* rgw: asio: check the remote endpoint before processing requests (`pr#30977 <https://github.com/ceph/ceph/pull/30977>`_, Abhishek Lekshmanan)
+* rgw: conditionally allow builtin users with non-unique email addresses (`issue#40089 <http://tracker.ceph.com/issues/40089>`_, `issue#40507 <http://tracker.ceph.com/issues/40507>`_, `pr#28716 <https://github.com/ceph/ceph/pull/28716>`_, Matt Benjamin)
+* rgw: data/bilogs are trimmed when no peers are reading them (`issue#39487 <http://tracker.ceph.com/issues/39487>`_, `pr#30130 <https://github.com/ceph/ceph/pull/30130>`_, Casey Bodley)
+* rgw: datalog/mdlog trim commands loop until done (`pr#30868 <https://github.com/ceph/ceph/pull/30868>`_, Casey Bodley)
+* rgw: do necessary checking of website configuration (`issue#40678 <http://tracker.ceph.com/issues/40678>`_, `pr#30980 <https://github.com/ceph/ceph/pull/30980>`_, Enming Zhang)
+* rgw: don't throw when accept errors are happening on frontend (`pr#30154 <https://github.com/ceph/ceph/pull/30154>`_, Yuval Lifshitz)
+* rgw: fix CreateBucket with BucketLocation parameter failed under default zonegroup (`pr#30171 <https://github.com/ceph/ceph/pull/30171>`_, Enming Zhang)
+* rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR (`issue#40147 <http://tracker.ceph.com/issues/40147>`_, `issue#39984 <http://tracker.ceph.com/issues/39984>`_, `pr#28409 <https://github.com/ceph/ceph/pull/28409>`_, Casey Bodley, Tianshan Qu)
+* rgw: fix cls_bucket_list_unordered() partial results (`pr#30253 <https://github.com/ceph/ceph/pull/30253>`_, Mark Kogan)
+* rgw: fix data sync start delay if remote haven't init data_log (`pr#30510 <https://github.com/ceph/ceph/pull/30510>`_, Tianshan Qu)
+* rgw: fix drain handles error when deleting bucket with bypass-gc option (`pr#29984 <https://github.com/ceph/ceph/pull/29984>`_, dongdong tao)
+* rgw: fix list bucket with delimiter wrongly skip some special keys (`issue#40905 <http://tracker.ceph.com/issues/40905>`_, `pr#30168 <https://github.com/ceph/ceph/pull/30168>`_, Tianshan Qu)
+* rgw: fix list versions starts with version_id=null (`pr#30775 <https://github.com/ceph/ceph/pull/30775>`_, Tianshan Qu)
+* rgw: fix potential realm watch lost (`issue#40991 <http://tracker.ceph.com/issues/40991>`_, `pr#30167 <https://github.com/ceph/ceph/pull/30167>`_, Tianshan Qu)
+* rgw: fix race b/w bucket reshard and ops waiting on reshard completion (`pr#29139 <https://github.com/ceph/ceph/pull/29139>`_, J. Eric Ivancich)
+* rgw: fix refcount tags to match and update object's idtag (`pr#30891 <https://github.com/ceph/ceph/pull/30891>`_, J. Eric Ivancich)
+* rgw: fixed "unrecognized arg" error when using "radosgw-admin zone rm" (`pr#30172 <https://github.com/ceph/ceph/pull/30172>`_, Hongang Chen)
+* rgw: gc remove tag after all sub io finish (`issue#40903 <http://tracker.ceph.com/issues/40903>`_, `pr#30173 <https://github.com/ceph/ceph/pull/30173>`_, Tianshan Qu)
+* rgw: housekeeping of reset stats operation in radosgw-admin and cls back-end (`pr#30165 <https://github.com/ceph/ceph/pull/30165>`_, J. Eric Ivancich)
+* rgw: increase beast parse buffer size to 64k (`pr#30450 <https://github.com/ceph/ceph/pull/30450>`_, Casey Bodley)
+* rgw: ldap auth: S3 auth failure should return InvalidAccessKeyId (`pr#30652 <https://github.com/ceph/ceph/pull/30652>`_, Matt Benjamin)
+* rgw: make dns hostnames matching case insensitive (`issue#40995 <http://tracker.ceph.com/issues/40995>`_, `pr#30166 <https://github.com/ceph/ceph/pull/30166>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: mitigate bucket list with max-entries excessively high (`pr#30134 <https://github.com/ceph/ceph/pull/30134>`_, J. Eric Ivancich)
+* rgw: multisite: 'radosgw-admin bucket sync status' should call syncs_from(source.name) instead of id (`issue#40022 <http://tracker.ceph.com/issues/40022>`_, `issue#40141 <http://tracker.ceph.com/issues/40141>`_, `pr#29270 <https://github.com/ceph/ceph/pull/29270>`_, Casey Bodley)
+* rgw: multisite: RGWListBucketIndexesCR for data full sync needs pagination (`issue#39551 <http://tracker.ceph.com/issues/39551>`_, `issue#40354 <http://tracker.ceph.com/issues/40354>`_, `pr#29284 <https://github.com/ceph/ceph/pull/29284>`_, Shilpa Jagannath)
+* rgw: multisite: data sync loops back to the start of the datalog after reaching the end (`issue#39033 <http://tracker.ceph.com/issues/39033>`_, `issue#39074 <http://tracker.ceph.com/issues/39074>`_, `pr#29021 <https://github.com/ceph/ceph/pull/29021>`_, Casey Bodley)
+* rgw: multisite: mismatch of bucket creation times from List Buckets (`issue#39635 <http://tracker.ceph.com/issues/39635>`_, `issue#39734 <http://tracker.ceph.com/issues/39734>`_, `pr#28483 <https://github.com/ceph/ceph/pull/28483>`_, Casey Bodley)
+* rgw: multisite: overwrites in versioning-suspended buckets fail to sync (`issue#38080 <http://tracker.ceph.com/issues/38080>`_, `issue#37792 <http://tracker.ceph.com/issues/37792>`_, `pr#29017 <https://github.com/ceph/ceph/pull/29017>`_, Casey Bodley)
+* rgw: multisite: period pusher gets 403 Forbidden against other zonegroups (`issue#39415 <http://tracker.ceph.com/issues/39415>`_, `issue#39287 <http://tracker.ceph.com/issues/39287>`_, `pr#29256 <https://github.com/ceph/ceph/pull/29256>`_, Casey Bodley)
+* rgw: non-existent mdlog failures logged at level 0 (`issue#38747 <http://tracker.ceph.com/issues/38747>`_, `issue#40033 <http://tracker.ceph.com/issues/40033>`_, `pr#28757 <https://github.com/ceph/ceph/pull/28757>`_, Abhishek Lekshmanan)
+* rgw: perfcounters: add gc retire counter (`pr#30073 <https://github.com/ceph/ceph/pull/30073>`_, Matt Benjamin)
+* rgw: permit rgw-admin to populate user info by access-key (`pr#30105 <https://github.com/ceph/ceph/pull/30105>`_, Matt Benjamin, Marc Koderer)
+* rgw: provide admin-friendly reshard status output (`issue#37615 <http://tracker.ceph.com/issues/37615>`_, `issue#40357 <http://tracker.ceph.com/issues/40357>`_, `pr#29285 <https://github.com/ceph/ceph/pull/29285>`_, Mark Kogan)
+* rgw: remove_olh_pending_entries() does not limit the number of xattrs to remove (`issue#39179 <http://tracker.ceph.com/issues/39179>`_, `issue#39118 <http://tracker.ceph.com/issues/39118>`_, `pr#28348 <https://github.com/ceph/ceph/pull/28348>`_, Casey Bodley)
+* rgw: resharding of a versioned bucket causes a bucket stats discrepancy (`issue#39532 <http://tracker.ceph.com/issues/39532>`_, `pr#28249 <https://github.com/ceph/ceph/pull/28249>`_, J. Eric Ivancich)
+* rgw: return ERR_NO_SUCH_BUCKET early while evaluating bucket policy (`issue#38420 <http://tracker.ceph.com/issues/38420>`_, `issue#39697 <http://tracker.ceph.com/issues/39697>`_, `pr#28422 <https://github.com/ceph/ceph/pull/28422>`_, Abhishek Lekshmanan)
+* rgw: rgw_file: all directories are virtual with respect to contents (`issue#40262 <http://tracker.ceph.com/issues/40262>`_, `issue#40204 <http://tracker.ceph.com/issues/40204>`_, `pr#28887 <https://github.com/ceph/ceph/pull/28887>`_, Matt Benjamin)
+* rgw: set null version object issues (`issue#36763 <http://tracker.ceph.com/issues/36763>`_, `issue#40360 <http://tracker.ceph.com/issues/40360>`_, `pr#29288 <https://github.com/ceph/ceph/pull/29288>`_, Tianshan Qu)
+* rgw: support delimiter longer then one symbol (`issue#39989 <http://tracker.ceph.com/issues/39989>`_, `issue#38776 <http://tracker.ceph.com/issues/38776>`_, `pr#29018 <https://github.com/ceph/ceph/pull/29018>`_, Tianshan Qu, Matt Benjamin)
+* rgw: swift object expiry fails when a bucket reshards (`issue#39741 <http://tracker.ceph.com/issues/39741>`_, `pr#29258 <https://github.com/ceph/ceph/pull/29258>`_, Casey Bodley, Abhishek Lekshmanan, J. Eric Ivancich)
+* rgw: swift: refrain from corrupting static large objects when using nginx as a GET cache (`pr#30135 <https://github.com/ceph/ceph/pull/30135>`_, Andrey Groshev)
+* rgw: the Multi-Object Delete operation of S3 API wrongly handles the Code response element (`issue#18241 <http://tracker.ceph.com/issues/18241>`_, `issue#40136 <http://tracker.ceph.com/issues/40136>`_, `pr#29268 <https://github.com/ceph/ceph/pull/29268>`_, Radoslaw Zarzynski)
+* rgw: update resharding documentation (`issue#39047 <http://tracker.ceph.com/issues/39047>`_, `pr#29020 <https://github.com/ceph/ceph/pull/29020>`_, J. Eric Ivancich)
+* rgw_file:  fix invalidation of top-level directories (`issue#40215 <http://tracker.ceph.com/issues/40215>`_, `pr#29276 <https://github.com/ceph/ceph/pull/29276>`_, Matt Benjamin)
+* rgw_file: advance_mtime() should consider namespace expiration (`issue#40415 <http://tracker.ceph.com/issues/40415>`_, `pr#30660 <https://github.com/ceph/ceph/pull/30660>`_, Matt Benjamin)
+* rgw_file: fix readdir eof() calc--caller stop implies !eof and introduce fast S3 Unix stats (immutable) (`issue#40375 <http://tracker.ceph.com/issues/40375>`_, `issue#40456 <http://tracker.ceph.com/issues/40456>`_, `pr#30077 <https://github.com/ceph/ceph/pull/30077>`_, Matt Benjamin)
+* rgw_file: include tenant when hashing bucket names (`issue#40225 <http://tracker.ceph.com/issues/40225>`_, `issue#40118 <http://tracker.ceph.com/issues/40118>`_, `pr#29277 <https://github.com/ceph/ceph/pull/29277>`_, Matt Benjamin)
+* rgw_file: readdir: do not construct markers w/leading '/' (`pr#30157 <https://github.com/ceph/ceph/pull/30157>`_, Matt Benjamin)
+* rgw_file: save etag and acl info in setattr (`issue#39229 <http://tracker.ceph.com/issues/39229>`_, `pr#28073 <https://github.com/ceph/ceph/pull/28073>`_, Tao Chen)
+* rpm: missing dependency on python34-ceph-argparse from python34-cephfs (and others?) (`issue#24918 <http://tracker.ceph.com/issues/24918>`_, `issue#24919 <http://tracker.ceph.com/issues/24919>`_, `issue#37613 <http://tracker.ceph.com/issues/37613>`_, `pr#27949 <https://github.com/ceph/ceph/pull/27949>`_, Kefu Chai)
+* tests: cls_rbd: removed mirror peer pool test cases (`pr#31485 <https://github.com/ceph/ceph/pull/31485>`_, Jason Dillaman)
+* tests: librbd: set nbd timeout due to newer kernels defaulting it on (`pr#30424 <https://github.com/ceph/ceph/pull/30424>`_, Jason Dillaman)
+* tests: ceph-disk: use a Python2.7 compatible version of pytest (`pr#31254 <https://github.com/ceph/ceph/pull/31254>`_, Alfredo Deza)
+* tests: rgw: don't use ceph-ansible in s3a-hadoop suite (`issue#39706 <http://tracker.ceph.com/issues/39706>`_, `pr#30069 <https://github.com/ceph/ceph/pull/30069>`_, Casey Bodley)
+* tests/workunits/rbd: wait for rbd-nbd unmap to complete (`issue#39598 <http://tracker.ceph.com/issues/39598>`_, `issue#39674 <http://tracker.ceph.com/issues/39674>`_, `pr#28310 <https://github.com/ceph/ceph/pull/28310>`_, Jason Dillaman)
+* tests: fix issues in vstart runner (`pr#28208 <https://github.com/ceph/ceph/pull/28208>`_, Volker Theile)
+* tests: limit loops waiting for force-backfill/force-recovery to happen (`issue#38351 <http://tracker.ceph.com/issues/38351>`_, `issue#38309 <http://tracker.ceph.com/issues/38309>`_, `pr#29245 <https://github.com/ceph/ceph/pull/29245>`_, David Zafman)
+* tests: remove s3tests ! (`pr#31640 <https://github.com/ceph/ceph/pull/31640>`_, Yuri Weinstein)
+* tests: cephfs: TestMisc.test_evict_client fails (`issue#40219 <http://tracker.ceph.com/issues/40219>`_, `pr#29228 <https://github.com/ceph/ceph/pull/29228>`_, "Yan, Zheng")
+* tests: do not take ceph.conf.template from ceph/teuthology.git (`pr#30841 <https://github.com/ceph/ceph/pull/30841>`_, Sage Weil)
+* tests: ignore expected MDS_CLIENT_LATE_RELEASE warning (`issue#40968 <http://tracker.ceph.com/issues/40968>`_, `pr#29812 <https://github.com/ceph/ceph/pull/29812>`_, Patrick Donnelly)
+* tests: install python3-cephfs for fs suite (`pr#31285 <https://github.com/ceph/ceph/pull/31285>`_, Kefu Chai)
+* tests: kclient unmount hangs after file system goes down (`issue#38709 <http://tracker.ceph.com/issues/38709>`_, `issue#38677 <http://tracker.ceph.com/issues/38677>`_, `pr#29218 <https://github.com/ceph/ceph/pull/29218>`_, Patrick Donnelly)
+* tests: krbd_msgr_segments.t: filter lvcreate output (`pr#31324 <https://github.com/ceph/ceph/pull/31324>`_, Ilya Dryomov)
+* tests: make get_mon_status use mon addr (`pr#31461 <https://github.com/ceph/ceph/pull/31461>`_, Sage Weil, Nathan Cutler)
+* tests: make: \*\*\* [hello_world_cpp] Error 127 in rados (`issue#40320 <http://tracker.ceph.com/issues/40320>`_, `pr#29203 <https://github.com/ceph/ceph/pull/29203>`_, Kefu Chai)
+* tests: qa/standalone/scrub/osd-scrub-snaps.sh sometimes fails (`issue#40179 <http://tracker.ceph.com/issues/40179>`_, `issue#40078 <http://tracker.ceph.com/issues/40078>`_, `pr#29251 <https://github.com/ceph/ceph/pull/29251>`_, David Zafman)
+* tests: qa/tasks/ceph.py: pass cluster_name to get_mons (`pr#31424 <https://github.com/ceph/ceph/pull/31424>`_, Nathan Cutler)
+* tests: qa/workunits/rbd: stress test "rbd mirror pool status --verbose" (`pr#29873 <https://github.com/ceph/ceph/pull/29873>`_, Mykola Golub)
+* tests: remove "1node" and "systemd" tests as ceph-deploy is not actively developed (`pr#28457 <https://github.com/ceph/ceph/pull/28457>`_, Yuri Weinstein)
+* tests: sleep briefly after resetting kclient (`pr#29751 <https://github.com/ceph/ceph/pull/29751>`_, Patrick Donnelly)
+* tests: test_volume_client: print python version correctly (`issue#40317 <http://tracker.ceph.com/issues/40317>`_, `issue#40184 <http://tracker.ceph.com/issues/40184>`_, `pr#29208 <https://github.com/ceph/ceph/pull/29208>`_, Lianne)
+* tests: use curl in wait_for_radosgw() in util/rgw.py (`pr#28668 <https://github.com/ceph/ceph/pull/28668>`_, Ali Maredia)
+* tests: use hard_reset to reboot kclient (`issue#37681 <http://tracker.ceph.com/issues/37681>`_, `pr#30233 <https://github.com/ceph/ceph/pull/30233>`_, Patrick Donnelly)
+* tests: whitelisted  'application not enabled' (`pr#28389 <https://github.com/ceph/ceph/pull/28389>`_, Yuri Weinstein)
+* tools/rados: list objects in a pg (`issue#36732 <http://tracker.ceph.com/issues/36732>`_, `pr#30893 <https://github.com/ceph/ceph/pull/30893>`_, Vikhyat Umrao, Li Wang)
+* tools/rbd-ggate: close log before running postfork (`pr#30121 <https://github.com/ceph/ceph/pull/30121>`_, Willem Jan Withagen)
+* tools: Add clear-data-digest command to objectstore tool (`issue#37749 <http://tracker.ceph.com/issues/37749>`_, `pr#29196 <https://github.com/ceph/ceph/pull/29196>`_, Li Yichao)
+* tools: ceph-objectstore-tool can't remove head with bad snapset (`pr#30081 <https://github.com/ceph/ceph/pull/30081>`_, David Zafman)
+* tools: ceph-objectstore-tool: return 0 if incmap is sane (`pr#31659 <https://github.com/ceph/ceph/pull/31659>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: update-mon-db: do not fail if incmap is missing (`pr#30979 <https://github.com/ceph/ceph/pull/30979>`_, Kefu Chai)
+* tools: crushtool crash on Fedora 28 and newer (`issue#39174 <http://tracker.ceph.com/issues/39174>`_, `issue#39311 <http://tracker.ceph.com/issues/39311>`_, `pr#27986 <https://github.com/ceph/ceph/pull/27986>`_, Brad Hubbard)
+
+
+v13.2.6 Mimic
+=============
+
+This is the sixth bugfix release of the Mimic v13.2.x long term
+stable release series. We recommend all Mimic users upgrade.
+
+Notable Changes
+---------------
+* Ceph v13.2.6 now packages python bindings for python3.6 instead of
+  python3.4, because EPEL7 recently switched from python3.4 to
+  python3.6 as the native python3. See the `announcement <https://lists.fedoraproject.org/archives/list/epel-announce@lists.fedoraproject.org/message/EGUMKAIMPK2UD5VSHXM53BH2MBDGDWMO/>_`
+  for more details on the background of this change.
+
+Changelog
+---------
+* cephfs: MDSMonitor: do not assign standby-replay when degraded (`issue#36384 <http://tracker.ceph.com/issues/36384>`_, `pr#26643 <https://github.com/ceph/ceph/pull/26643>`_, Patrick Donnelly)
+* ceph-volume: add --all flag to simple activate (`pr#26655 <https://github.com/ceph/ceph/pull/26655>`_, Jan Fajerski)
+* ceph-volume: use our own testinfra suite for functional testing (`pr#26702 <https://github.com/ceph/ceph/pull/26702>`_, Andrew Schoen)
+* cli: ability to change file ownership (`issue#38370 <http://tracker.ceph.com/issues/38370>`_, `pr#26760 <https://github.com/ceph/ceph/pull/26760>`_, Sébastien Han)
+* cli: better output of 'ceph health detail' (`issue#39266 <http://tracker.ceph.com/issues/39266>`_, `pr#27847 <https://github.com/ceph/ceph/pull/27847>`_, Shen Hang)
+* cls/rgw: raise debug level of bi_log_iterate_entries output (`pr#27973 <https://github.com/ceph/ceph/pull/27973>`_, Casey Bodley)
+* common: ceph_timer: stop timer's thread when it is suspended (`issue#37766 <http://tracker.ceph.com/issues/37766>`_, `pr#26583 <https://github.com/ceph/ceph/pull/26583>`_, Peng Wang)
+* common/str_map: fix trim() on empty string (`issue#38329 <http://tracker.ceph.com/issues/38329>`_, `pr#26810 <https://github.com/ceph/ceph/pull/26810>`_, Sage Weil)
+* core: ENOENT in collection_move_rename on EC backfill target (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `pr#27943 <https://github.com/ceph/ceph/pull/27943>`_, Neha Ojha)
+* core: Fix recovery and backfill priority handling (`issue#38041 <http://tracker.ceph.com/issues/38041>`_, `pr#27081 <https://github.com/ceph/ceph/pull/27081>`_, David Zafman)
+* crush: add root_bucket to identify underfull buckets (`issue#38826 <http://tracker.ceph.com/issues/38826>`_, `pr#27257 <https://github.com/ceph/ceph/pull/27257>`_, huangjun)
+* crush: backport recent upmap fixes (`issue#37968 <http://tracker.ceph.com/issues/37968>`_, `issue#38897 <http://tracker.ceph.com/issues/38897>`_, `issue#37940 <http://tracker.ceph.com/issues/37940>`_, `pr#27963 <https://github.com/ceph/ceph/pull/27963>`_, xie xingguo)
+* crush/CrushWrapper: ensure crush_choose_arg_map.size == max_buckets (`issue#38664 <http://tracker.ceph.com/issues/38664>`_, `pr#27082 <https://github.com/ceph/ceph/pull/27082>`_, Sage Weil)
+* doc: Fix incorrect mention of 'osd_deep_mon_scrub_interval' (`pr#26860 <https://github.com/ceph/ceph/pull/26860>`_, Ashish Singh)
+* doc: Minor rados related documentation fixes (`issue#38896 <http://tracker.ceph.com/issues/38896>`_, `pr#27188 <https://github.com/ceph/ceph/pull/27188>`_, David Zafman)
+* doc: osd_recovery_priority is not documented (but osd_recovery_op_priority is) (`issue#23999 <http://tracker.ceph.com/issues/23999>`_, `pr#26901 <https://github.com/ceph/ceph/pull/26901>`_, David Zafman)
+* doc/radosgw: Document mappings of S3 Operations to ACL grants (`issue#38523 <http://tracker.ceph.com/issues/38523>`_, `pr#26968 <https://github.com/ceph/ceph/pull/26968>`_, Adam C. Emerson)
+* doc/rgw: document placement target configuration (`issue#24508 <http://tracker.ceph.com/issues/24508>`_, `pr#27032 <https://github.com/ceph/ceph/pull/27032>`_, Casey Bodley)
+* doc: Update bluestore config docs - fix typo (as -> has) (`pr#27845 <https://github.com/ceph/ceph/pull/27845>`_, Yaniv Kaul)
+* doc: updated reference link for log based PG (`issue#38465 <http://tracker.ceph.com/issues/38465>`_, `pr#26829 <https://github.com/ceph/ceph/pull/26829>`_, James McClune)
+* include/intarith: enforce the same type for p2\*() arguments (`pr#27318 <https://github.com/ceph/ceph/pull/27318>`_, Ilya Dryomov)
+* librbd: avoid aggregate-initializing any static_visitor (`issue#38659 <http://tracker.ceph.com/issues/38659>`_, `pr#27041 <https://github.com/ceph/ceph/pull/27041>`_, Willem Jan Withagen)
+* librbd: avoid aggregate-initializing IsWriteOpVisitor (`issue#38660 <http://tracker.ceph.com/issues/38660>`_, `pr#27039 <https://github.com/ceph/ceph/pull/27039>`_, Willem Jan Withagen)
+* mds: drop reconnect message from non-existent session (`issue#39026 <http://tracker.ceph.com/issues/39026>`_, `pr#27916 <https://github.com/ceph/ceph/pull/27916>`_, Shen Hang)
+* mds: inode filtering on 'dump cache' asok (`issue#11172 <http://tracker.ceph.com/issues/11172>`_, `pr#27058 <https://github.com/ceph/ceph/pull/27058>`_, dongdong tao)
+* mds/server: check directory split after rename (`issue#38994 <http://tracker.ceph.com/issues/38994>`_, `pr#27917 <https://github.com/ceph/ceph/pull/27917>`_, Shen Hang)
+* mds: wait for client to release shared cap when re-acquiring xlock (`issue#38491 <http://tracker.ceph.com/issues/38491>`_, `pr#27023 <https://github.com/ceph/ceph/pull/27023>`_, "Yan, Zheng")
+* mgr/balancer: blame if upmap won't actually work (`issue#38780 <http://tracker.ceph.com/issues/38780>`_, `pr#26497 <https://github.com/ceph/ceph/pull/26497>`_, xie xingguo)
+* mgr/BaseMgrModule: drop GIL for ceph_send_command (`issue#38537 <http://tracker.ceph.com/issues/38537>`_, `pr#26833 <https://github.com/ceph/ceph/pull/26833>`_, Sage Weil)
+* mgr: crashdump feature backport (`pr#24639 <https://github.com/ceph/ceph/pull/24639>`_, Noah Watkins, Sage Weil, Dan Mick)
+* mgr/dashboard: fix for using '::' on hosts without ipv6 (`issue#38575 <http://tracker.ceph.com/issues/38575>`_, `pr#26750 <https://github.com/ceph/ceph/pull/26750>`_, Noah Watkins)
+* mgr/dashboard: Manager should complain about wrong dashboard certificate (`issue#24453 <http://tracker.ceph.com/issues/24453>`_, `pr#27747 <https://github.com/ceph/ceph/pull/27747>`_, Volker Theile, Ricardo Dias)
+* mgr/dashboard: Search broken for entries with null values (`issue#38583 <http://tracker.ceph.com/issues/38583>`_, `pr#26944 <https://github.com/ceph/ceph/pull/26944>`_, Patrick Nawracay)
+* mgr/dashboard: show I/O stats in Pool list (`pr#27053 <https://github.com/ceph/ceph/pull/27053>`_, Alfonso Martínez)
+* mgr/dashboard: Update npm packages (`issue#39080 <http://tracker.ceph.com/issues/39080>`_, `pr#26670 <https://github.com/ceph/ceph/pull/26670>`_, Tiago Melo)
+* mgr/dashboard: Use human readable units on the OSD I/O graphs (`issue#25075 <http://tracker.ceph.com/issues/25075>`_, `pr#27558 <https://github.com/ceph/ceph/pull/27558>`_, Tiago Melo)
+* mgr: drop GIL in get_config (`pr#26612 <https://github.com/ceph/ceph/pull/26612>`_, John Spray)
+* mgr: enable inter-module calls (`pr#27638 <https://github.com/ceph/ceph/pull/27638>`_, John Spray)
+* mgr/prometheus: add interface and objectstore to osd metadata (`pr#26537 <https://github.com/ceph/ceph/pull/26537>`_, Jan Fajerski, Konstantin Shalygin)
+* mgr/PyModule: put mgr_module_path first in sys.path (`issue#38469 <http://tracker.ceph.com/issues/38469>`_, `pr#26777 <https://github.com/ceph/ceph/pull/26777>`_, Tim Serong)
+* mon/OSDMonitor: fix osd boot check (`pr#27351 <https://github.com/ceph/ceph/pull/27351>`_, Sage Weil)
+* mon/OSDMonitor: further improve prepare_command_pool_set E2BIG error message (`issue#39353 <http://tracker.ceph.com/issues/39353>`_, `pr#27647 <https://github.com/ceph/ceph/pull/27647>`_, Nathan Cutler)
+* msg: output peer address when detecting bad CRCs (`issue#39367 <http://tracker.ceph.com/issues/39367>`_, `pr#27860 <https://github.com/ceph/ceph/pull/27860>`_, Greg Farnum)
+* multisite: bucket full sync does not handle delete markers (`issue#38007 <http://tracker.ceph.com/issues/38007>`_, `pr#26194 <https://github.com/ceph/ceph/pull/26194>`_, Casey Bodley)
+* multisite: rgw_data_sync_status json decode failure breaks automated datalog trimming (`issue#38373 <http://tracker.ceph.com/issues/38373>`_, `pr#26615 <https://github.com/ceph/ceph/pull/26615>`_, Casey Bodley)
+* os/bluestore: backport new bitmap allocator (`pr#26983 <https://github.com/ceph/ceph/pull/26983>`_, Igor Fedotov, Sage Weil)
+* os/bluestore: bitmap allocator might fail to return contiguous chunk despite having enough space (`pr#27298 <https://github.com/ceph/ceph/pull/27298>`_, Igor Fedotov)
+* os/bluestore: call fault_range properly prior to looking for blob to … (`pr#27570 <https://github.com/ceph/ceph/pull/27570>`_, Igor Fedotov)
+* os/bluestore: fix improper backport for p2 macros for bmap allocator (`pr#27606 <https://github.com/ceph/ceph/pull/27606>`_, Igor Fedotov)
+* os/bluestore: fix length overflow (`issue#39245 <http://tracker.ceph.com/issues/39245>`_, `pr#27366 <https://github.com/ceph/ceph/pull/27366>`_, Jianpeng Ma)
+* os/bluestore: fix out-of-bound access in bmap allocator (`pr#27738 <https://github.com/ceph/ceph/pull/27738>`_, Igor Fedotov)
+* os/bluestore_tool: bluefs-bdev-expand: indicate bypassed for main dev (`pr#27447 <https://github.com/ceph/ceph/pull/27447>`_, Igor Fedotov)
+* osd: FAILED ceph_assert(attrs || !pg_log.get_missing().is_missing(soid) || (it_objects != pg_log.get_log().objects.end() && it_objects->second->op == pg_log_entry_t::LOST_REVERT)) in PrimaryLogPG::get_object_context() (`issue#38931 <http://tracker.ceph.com/issues/38931>`_, `issue#38784 <http://tracker.ceph.com/issues/38784>`_, `pr#27940 <https://github.com/ceph/ceph/pull/27940>`_, xie xingguo)
+* osd: fixup OpTracker destruct assert, waiting_for_osdmap take ref      with OpRequest (`issue#38377 <http://tracker.ceph.com/issues/38377>`_, `pr#26862 <https://github.com/ceph/ceph/pull/26862>`_, linbing)
+* osd/PG: discover missing objects when an OSD peers and PG is degraded (`pr#27745 <https://github.com/ceph/ceph/pull/27745>`_, Jonas Jelten)
+* osd/PGLog.h: print olog_can_rollback_to before deciding to rollback (`issue#38894 <http://tracker.ceph.com/issues/38894>`_, `pr#27284 <https://github.com/ceph/ceph/pull/27284>`_, Neha Ojha)
+* osd/PGLog: preserve original_crt to check rollbackability (`issue#39023 <http://tracker.ceph.com/issues/39023>`_, `issue#36739 <http://tracker.ceph.com/issues/36739>`_, `pr#27629 <https://github.com/ceph/ceph/pull/27629>`_, Neha Ojha)
+* osd/PrimaryLogPG: handle object !exists in handle_watch_timeout (`issue#38432 <http://tracker.ceph.com/issues/38432>`_, `pr#26709 <https://github.com/ceph/ceph/pull/26709>`_, Sage Weil)
+* osd: process_copy_chunk remove obc ref before pg unlock (`issue#38842 <http://tracker.ceph.com/issues/38842>`_, `pr#27587 <https://github.com/ceph/ceph/pull/27587>`_, Zengran Zhang)
+* osd: shutdown recovery_request_timer earlier (`issue#38945 <http://tracker.ceph.com/issues/38945>`_, `pr#27938 <https://github.com/ceph/ceph/pull/27938>`_, Zengran Zhang)
+* pybind/rados: fixed Python3 string conversion issue on get_fsid (`issue#38381 <http://tracker.ceph.com/issues/38381>`_, `pr#27259 <https://github.com/ceph/ceph/pull/27259>`_, Jason Dillaman)
+* rbd: API list_images() Segmentation fault (`issue#38468 <http://tracker.ceph.com/issues/38468>`_, `pr#26707 <https://github.com/ceph/ceph/pull/26707>`_, songweibin)
+* rbd: krbd: return -ETIMEDOUT in polling (`issue#38792 <http://tracker.ceph.com/issues/38792>`_, `pr#27588 <https://github.com/ceph/ceph/pull/27588>`_, Dongsheng Yang)
+* rbd_mirror: don't report error if image replay canceled (`pr#26140 <https://github.com/ceph/ceph/pull/26140>`_, Mykola Golub)
+* rgw: Adding tcp_nodelay option to Beast (`issue#34308 <http://tracker.ceph.com/issues/34308>`_, `pr#27367 <https://github.com/ceph/ceph/pull/27367>`_, Or Friedmann)
+* rgw admin: add tenant argument to reshard cancel (`issue#38214 <http://tracker.ceph.com/issues/38214>`_, `pr#27603 <https://github.com/ceph/ceph/pull/27603>`_, Abhishek Lekshmanan)
+* rgw-admin: fix data sync report for master zone (`issue#38938 <http://tracker.ceph.com/issues/38938>`_, `pr#27421 <https://github.com/ceph/ceph/pull/27421>`_, cfanz)
+* rgw: admin: handle delete_at attr in object stat output (`pr#27828 <https://github.com/ceph/ceph/pull/27828>`_, Abhishek Lekshmanan)
+* rgw: allow radosgw-admin to list bucket w --allow-unordered (`pr#28096 <https://github.com/ceph/ceph/pull/28096>`_, J. Eric Ivancich)
+* rgw: beast: set a default port for endpoints (`issue#39000 <http://tracker.ceph.com/issues/39000>`_, `pr#27661 <https://github.com/ceph/ceph/pull/27661>`_, Abhishek Lekshmanan)
+* rgw: bucket limit check misbehaves for > max-entries buckets (usually 1000) (`pr#26945 <https://github.com/ceph/ceph/pull/26945>`_, Matt Benjamin)
+* rgw: bug in versioning concurrent, list and get have consistency issue (`issue#38060 <http://tracker.ceph.com/issues/38060>`_, `pr#26664 <https://github.com/ceph/ceph/pull/26664>`_, Wang Hao)
+* rgw: check for non-existent bucket in RGWGetACLs (`issue#38116 <http://tracker.ceph.com/issues/38116>`_, `pr#26529 <https://github.com/ceph/ceph/pull/26529>`_, Matt Benjamin)
+* rgw: cls_bucket_list_unordered lists a single shard (`issue#39393 <http://tracker.ceph.com/issues/39393>`_, `pr#28086 <https://github.com/ceph/ceph/pull/28086>`_, Casey Bodley)
+* rgw: data sync drains lease stack on lease failure (`issue#38479 <http://tracker.ceph.com/issues/38479>`_, `pr#26762 <https://github.com/ceph/ceph/pull/26762>`_, Casey Bodley)
+* rgw: don't crash on missing /etc/mime.types (`issue#38328 <http://tracker.ceph.com/issues/38328>`_, `pr#27354 <https://github.com/ceph/ceph/pull/27354>`_, Casey Bodley)
+* rgw: failed to pass test_bucket_create_naming_bad_punctuation in s3test (`issue#23587 <http://tracker.ceph.com/issues/23587>`_, `issue#26965 <http://tracker.ceph.com/issues/26965>`_, `pr#27666 <https://github.com/ceph/ceph/pull/27666>`_, yuliyang, Abhishek Lekshmanan)
+* rgw: fix bug of apply default quota, for this create new a user may core using beast (`issue#38847 <http://tracker.ceph.com/issues/38847>`_, `pr#27335 <https://github.com/ceph/ceph/pull/27335>`_, liaoxin01)
+* rgw: fix read not exists null version return wrong (`issue#38811 <http://tracker.ceph.com/issues/38811>`_, `pr#27304 <https://github.com/ceph/ceph/pull/27304>`_, Tianshan Qu)
+* rgw: Fix S3 compatibility bug when CORS is not found (`issue#37945 <http://tracker.ceph.com/issues/37945>`_, `pr#27356 <https://github.com/ceph/ceph/pull/27356>`_, Nick Janus)
+* rgw: GetBucketCORS API returns Not Found error code when CORS configuration does not exist (`issue#26964 <http://tracker.ceph.com/issues/26964>`_, `pr#27122 <https://github.com/ceph/ceph/pull/27122>`_, yuliyang, ashitakasam)
+* rgw: get or set realm zonegroup zone should check user's caps for security (`issue#37352 <http://tracker.ceph.com/issues/37352>`_, `pr#27948 <https://github.com/ceph/ceph/pull/27948>`_, yuliyang, Casey Bodley)
+* rgw: ldap: fix LDAPAuthEngine::init() when uri !empty() (`issue#38699 <http://tracker.ceph.com/issues/38699>`_, `pr#27174 <https://github.com/ceph/ceph/pull/27174>`_, Matt Benjamin)
+* rgw: multiple es related fixes and improvements (`issue#38028 <http://tracker.ceph.com/issues/38028>`_, `issue#22877 <http://tracker.ceph.com/issues/22877>`_, `issue#36233 <http://tracker.ceph.com/issues/36233>`_, `issue#38030 <http://tracker.ceph.com/issues/38030>`_, `issue#36092 <http://tracker.ceph.com/issues/36092>`_, `pr#26517 <https://github.com/ceph/ceph/pull/26517>`_, Yehuda Sadeh, Abhishek Lekshmanan, Willem Jan Withagen)
+* rgw: nfs: skip empty (non-POSIX) path segments (`issue#38744 <http://tracker.ceph.com/issues/38744>`_, `pr#27179 <https://github.com/ceph/ceph/pull/27179>`_, Matt Benjamin)
+* rgw: only update last_trim marker on ENODATA (`issue#38075 <http://tracker.ceph.com/issues/38075>`_, `pr#26641 <https://github.com/ceph/ceph/pull/26641>`_, Casey Bodley)
+* rgw: resolve bugs and clean up garbage collection code (`issue#38454 <http://tracker.ceph.com/issues/38454>`_, `pr#27796 <https://github.com/ceph/ceph/pull/27796>`_, J. Eric Ivancich)
+* rgw: rgw_file: use correct secret key to check auth (`issue#37855 <http://tracker.ceph.com/issues/37855>`_, `pr#26687 <https://github.com/ceph/ceph/pull/26687>`_, MinSheng Lin)
+* rgw: sse c fixes (`issue#38700 <http://tracker.ceph.com/issues/38700>`_, `pr#27297 <https://github.com/ceph/ceph/pull/27297>`_, Adam Kupczyk, Casey Bodley, Abhishek Lekshmanan)
+* rgw: sync module: avoid printing attrs of objects in log (`issue#37646 <http://tracker.ceph.com/issues/37646>`_, `pr#27029 <https://github.com/ceph/ceph/pull/27029>`_, Abhishek Lekshmanan)
+* rgw: use chunked encoding to get partial results out faster (`issue#12713 <http://tracker.ceph.com/issues/12713>`_, `pr#28014 <https://github.com/ceph/ceph/pull/28014>`_, Robin H. Johnson)
+* rgw: when exclusive lock fails due existing lock, log add'l info (`issue#38171 <http://tracker.ceph.com/issues/38171>`_, `pr#26553 <https://github.com/ceph/ceph/pull/26553>`_, J. Eric Ivancich)
+* rgw: when using nfs-ganesha to upload file, rgw es sync module get failed (`issue#36233 <http://tracker.ceph.com/issues/36233>`_, `pr#27972 <https://github.com/ceph/ceph/pull/27972>`_, Abhishek Lekshmanan)
+* run-standalone.sh: Need double-quotes to handle | in core_pattern on all distributions (`issue#38325 <http://tracker.ceph.com/issues/38325>`_, `pr#26811 <https://github.com/ceph/ceph/pull/26811>`_, David Zafman)
+* spdk: update to latest spdk-18.05 branch (`pr#27451 <https://github.com/ceph/ceph/pull/27451>`_, Kefu Chai)
+* test: run-standalone.sh set local library location so mgr can find li… (`issue#38262 <http://tracker.ceph.com/issues/38262>`_, `pr#26495 <https://github.com/ceph/ceph/pull/26495>`_, David Zafman)
+* test/store_test: fix/workaround for BlobReuseOnOverwriteUT and garbageCollection (`pr#27055 <https://github.com/ceph/ceph/pull/27055>`_, Igor Fedotov)
+* test: Verify a log trim trims the dup_index (`pr#26578 <https://github.com/ceph/ceph/pull/26578>`_, Brad Hubbard)
+* tools: ceph-disk/tests: use random unused port for CEPH_MON (`issue#39066 <http://tracker.ceph.com/issues/39066>`_, `pr#27228 <https://github.com/ceph/ceph/pull/27228>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: rename dump-import to dump-export (`issue#39284 <http://tracker.ceph.com/issues/39284>`_, `pr#27635 <https://github.com/ceph/ceph/pull/27635>`_, David Zafman)
+
+
+v13.2.5 Mimic
+=============
+
+This is the fifth bugfix release of the Mimic v13.2.x long term
+stable release series. We recommend all Mimic users upgrade.
+
+Notable Changes
+---------------
+
+* This release fixes the pg log hard limit bug that was introduced in
+  13.2.2, https://tracker.ceph.com/issues/36686. A flag called
+  `pglog_hardlimit` has been introduced, which is off by default. Enabling
+  this flag will limit the length of the pg log. In order to enable
+  that, the flag must be set by running `ceph osd set pglog_hardlimit`
+  after completely upgrading to 13.2.2. Once the cluster has this flag
+  set, the length of the pg log will be capped by a hard limit. Once set,
+  this flag *must not* be unset anymore. In luminous, this feature was
+  introduced in 12.2.11. Users who are running 12.2.11, and want to
+  continue to use this feature, should upgrade to 13.2.5 or later.
+
+* This release also fixes a CVE on civetweb, CVE-2019-3821 where SSL file
+  descriptors were not closed in civetweb in case the initial negotiation fails.
+
+* There have been fixes to RGW dynamic and manual resharding, which no longer
+  leaves behind stale bucket instances to be removed manually. For finding and
+  cleaning up older instances from a reshard a radosgw-admin command `reshard
+  stale-instances list` and `reshard stale-instances rm` should do the necessary
+  cleanup. These commands should *not* be used on a multisite setup as the stale
+  instances may be unlikely to be from a reshard and can have consequences. In
+  the next version the admin CLI will prevent this command to be run on a
+  multisite cluster, however for the current release users are urged not to
+  use the delete command on a multisite cluster.
+
+Changelog
+---------
+
+* build/ops: Destruction of basic_string _GLIBCXX_USE_CXX11_ABI=0 and C++17 mode results in invalid delete (`issue#38177 <http://tracker.ceph.com/issues/38177>`_, `pr#26593 <https://github.com/ceph/ceph/pull/26593>`_, Kefu Chai, Jason Dillaman)
+* build/ops: rpm: require ceph-base instead of ceph-common (`issue#37620 <http://tracker.ceph.com/issues/37620>`_, `pr#25809 <https://github.com/ceph/ceph/pull/25809>`_, Sébastien Han)
+* build/ops: run-make-check.sh ccache tweaks (`issue#24817 <http://tracker.ceph.com/issues/24817>`_, `issue#24777 <http://tracker.ceph.com/issues/24777>`_, `pr#25153 <https://github.com/ceph/ceph/pull/25153>`_, Nathan Cutler, Jonathan Brielmaier, Erwan Velu)
+* ceph-create-keys: fix octal notation for Python 3 without losing compatibility with Python 2 (`issue#37641 <http://tracker.ceph.com/issues/37641>`_, `pr#25531 <https://github.com/ceph/ceph/pull/25531>`_, James Page)
+* cephfs: MDCache::finish_snaprealm_reconnect() create and drop MClientSnap message (`issue#38285 <http://tracker.ceph.com/issues/38285>`_, `pr#26472 <https://github.com/ceph/ceph/pull/26472>`_, "Yan, Zheng")
+* cephfs: mgr/status: fix fs status subcommand did not show standby-replay MDS' perf info (`issue#36399 <http://tracker.ceph.com/issues/36399>`_, `pr#25031 <https://github.com/ceph/ceph/pull/25031>`_, Zhi Zhang)
+* ceph-objectstore-tool: Dump hashinfo (`issue#37597 <http://tracker.ceph.com/issues/37597>`_, `pr#25721 <https://github.com/ceph/ceph/pull/25721>`_, David Zafman)
+* ceph-volume-client: allow setting mode of CephFS volumes (`issue#36651 <http://tracker.ceph.com/issues/36651>`_, `pr#25413 <https://github.com/ceph/ceph/pull/25413>`_, Tom Barron)
+* ceph-volume: enable device discards (`issue#36532 <http://tracker.ceph.com/issues/36532>`_, `pr#25749 <https://github.com/ceph/ceph/pull/25749>`_, Jonas Jelten)
+* ceph-volume: fix JSON output in `inventory` (`issue#37390 <http://tracker.ceph.com/issues/37390>`_, `pr#25923 <https://github.com/ceph/ceph/pull/25923>`_, Sebastian Wagner)
+* ceph-volume: Fix TypeError: join() takes exactly one argument (2 given) (`issue#37595 <http://tracker.ceph.com/issues/37595>`_, `pr#25771 <https://github.com/ceph/ceph/pull/25771>`_, Sebastian Wagner)
+* ceph-volume normalize comma to dot for string to int conversions (`issue#37442 <http://tracker.ceph.com/issues/37442>`_, `pr#25775 <https://github.com/ceph/ceph/pull/25775>`_, Alfredo Deza)
+* ceph-volume: revert partition as disk (`issue#37506 <http://tracker.ceph.com/issues/37506>`_, `pr#26294 <https://github.com/ceph/ceph/pull/26294>`_, Jan Fajerski)
+* ceph-volume: set permissions right before prime-osd-dir (`issue#37486 <http://tracker.ceph.com/issues/37486>`_, `pr#25777 <https://github.com/ceph/ceph/pull/25777>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume tests/functional declare ceph-ansible roles instead of importing them (`issue#37805 <http://tracker.ceph.com/issues/37805>`_, `pr#25837 <https://github.com/ceph/ceph/pull/25837>`_, Alfredo Deza)
+* ceph-volume  zap: improve zapping to remove all partitions and all LVs, encrypted or not (`issue#37449 <http://tracker.ceph.com/issues/37449>`_, `pr#25351 <https://github.com/ceph/ceph/pull/25351>`_, Alfredo Deza)
+* cli: dump osd-fsid as part of osd find <id> (`issue#37966 <http://tracker.ceph.com/issues/37966>`_, `pr#26035 <https://github.com/ceph/ceph/pull/26035>`_, Noah Watkins)
+* client: do not move f->pos untill success write (`issue#37546 <http://tracker.ceph.com/issues/37546>`_, `pr#25683 <https://github.com/ceph/ceph/pull/25683>`_, Junhui Tang)
+* client: fix failure in quota size limitation when using samba (`issue#37547 <http://tracker.ceph.com/issues/37547>`_, `pr#25678 <https://github.com/ceph/ceph/pull/25678>`_, Junhui Tang)
+* client: fix fuse client hang because its pipe to mds is not ok (`issue#36079 <http://tracker.ceph.com/issues/36079>`_, `pr#25903 <https://github.com/ceph/ceph/pull/25903>`_, Guan yunfei)
+*  client: retry remount on dcache invalidation failure (`issue#27657 <http://tracker.ceph.com/issues/27657>`_, `pr#24695 <https://github.com/ceph/ceph/pull/24695>`_, Venky Shankar)
+* client: session flush does not cause cap release message flush (`issue#38009 <http://tracker.ceph.com/issues/38009>`_, `pr#26424 <https://github.com/ceph/ceph/pull/26424>`_, Patrick Donnelly)
+* cmake: do not pass -B{symbolic,symbolic-functions} to linker on FreeBSD (`issue#36717 <http://tracker.ceph.com/issues/36717>`_, `pr#25525 <https://github.com/ceph/ceph/pull/25525>`_, Willem Jan Withagen)
+* common: fix memory leaks in WeightedPriorityQueue (`issue#36248 <http://tracker.ceph.com/issues/36248>`_, `pr#25295 <https://github.com/ceph/ceph/pull/25295>`_, Radoslaw Zarzynski)
+* common: fix missing include boost/noncopyable.hpp (`issue#38178 <http://tracker.ceph.com/issues/38178>`_, `pr#26277 <https://github.com/ceph/ceph/pull/26277>`_, Willem Jan Withagen)
+* core: list-inconsistent-obj output truncated, causing osd-scrub-repair.sh failure (`issue#37653 <http://tracker.ceph.com/issues/37653>`_, `pr#25603 <https://github.com/ceph/ceph/pull/25603>`_, David Zafman)
+* core: luminous->(mimic,nautilus): PGMapDigest decode error on luminous end (`issue#38295 <http://tracker.ceph.com/issues/38295>`_, `pr#26451 <https://github.com/ceph/ceph/pull/26451>`_, Sage Weil)
+* core: Objecter::calc_op_budget: Fix invalid access to extent union member (`issue#37932 <http://tracker.ceph.com/issues/37932>`_, `pr#26066 <https://github.com/ceph/ceph/pull/26066>`_, Simon Ruggier)
+* core: scrub warning check incorrectly uses mon scrub interval (`issue#37264 <http://tracker.ceph.com/issues/37264>`_, `pr#26493 <https://github.com/ceph/ceph/pull/26493>`_, David Zafman)
+* deep fsck fails on inspecting very large onodes (`issue#38065 <http://tracker.ceph.com/issues/38065>`_, `pr#26291 <https://github.com/ceph/ceph/pull/26291>`_, Igor Fedotov)
+* doc: pin the version for "breathe" to 4.1.11 (`issue#38229 <http://tracker.ceph.com/issues/38229>`_, `pr#26333 <https://github.com/ceph/ceph/pull/26333>`_, Alfredo Deza)
+* doc: rados/configuration: refresh osdmap section (`issue#38051 <http://tracker.ceph.com/issues/38051>`_, `pr#26373 <https://github.com/ceph/ceph/pull/26373>`_, Ilya Dryomov)
+* doc: updated Ceph documentation links (`issue#37793 <http://tracker.ceph.com/issues/37793>`_, `pr#26180 <https://github.com/ceph/ceph/pull/26180>`_, James McClune)
+* doc/user-management: Remove obsolete reset caps command (`issue#37663 <http://tracker.ceph.com/issues/37663>`_, `pr#25607 <https://github.com/ceph/ceph/pull/25607>`_, Brad Hubbard)
+* journal: max journal order is incorrectly set at 64 (`issue#37541 <http://tracker.ceph.com/issues/37541>`_, `pr#25957 <https://github.com/ceph/ceph/pull/25957>`_, Mykola Golub)
+* librbd: fix missing unblock_writes if shrink is not allowed (`issue#36778 <http://tracker.ceph.com/issues/36778>`_, `pr#25252 <https://github.com/ceph/ceph/pull/25252>`_, runsisi)
+* librbd: reset snaps in rbd_snap_list() (`issue#37508 <http://tracker.ceph.com/issues/37508>`_, `pr#25459 <https://github.com/ceph/ceph/pull/25459>`_, Kefu Chai)
+* mds: broadcast quota message to client when disable quota (`issue#38054 <http://tracker.ceph.com/issues/38054>`_, `pr#26292 <https://github.com/ceph/ceph/pull/26292>`_, Junhui Tang)
+* mds: create separate config for heartbeat timeout (`issue#37674 <http://tracker.ceph.com/issues/37674>`_, `pr#26010 <https://github.com/ceph/ceph/pull/26010>`_, Patrick Donnelly)
+* mds: directories pinned keep being replicated back and forth between exporting mds and importing mds (`issue#37368 <http://tracker.ceph.com/issues/37368>`_, `pr#25521 <https://github.com/ceph/ceph/pull/25521>`_, Xuehan Xu)
+* mds: disallow dumping huge caches to formatter (`issue#36703 <http://tracker.ceph.com/issues/36703>`_, `pr#25642 <https://github.com/ceph/ceph/pull/25642>`_, Venky Shankar)
+* mds: do not call Journaler::_trim twice (`issue#37566 <http://tracker.ceph.com/issues/37566>`_, `pr#25561 <https://github.com/ceph/ceph/pull/25561>`_, Tang Junhui)
+* mds: fix bug filelock stuck at LOCK_XSYN leading client can't read data (`issue#37333 <http://tracker.ceph.com/issues/37333>`_, `pr#25676 <https://github.com/ceph/ceph/pull/25676>`_, Guan yunfei)
+* mds: fix incorrect l_pq_executing_ops statistics when meet an invalid item in purge queue (`issue#37567 <http://tracker.ceph.com/issues/37567>`_, `pr#25559 <https://github.com/ceph/ceph/pull/25559>`_, Junhui Tang)
+* mds: fix potential re-evaluate stray dentry in _unlink_local_finish (`issue#38263 <http://tracker.ceph.com/issues/38263>`_, `pr#26474 <https://github.com/ceph/ceph/pull/26474>`_, Zhi Zhang)
+* mds: fix races of updating wanted caps (`issue#37464 <http://tracker.ceph.com/issues/37464>`_, `pr#25680 <https://github.com/ceph/ceph/pull/25680>`_, "Yan, Zheng")
+* mds: handle fragment notify race (`issue#36035 <http://tracker.ceph.com/issues/36035>`_, `pr#26252 <https://github.com/ceph/ceph/pull/26252>`_, "Yan, Zheng")
+* mds: handle state change race (`issue#37594 <http://tracker.ceph.com/issues/37594>`_, `pr#26051 <https://github.com/ceph/ceph/pull/26051>`_, "Yan, Zheng")
+* mds: log evicted clients to clog/dbg (`issue#37639 <http://tracker.ceph.com/issues/37639>`_, `pr#25857 <https://github.com/ceph/ceph/pull/25857>`_, Patrick Donnelly)
+* MDSMonitor: allow beacons from stopping MDS that was laggy (`issue#37724 <http://tracker.ceph.com/issues/37724>`_, `pr#25685 <https://github.com/ceph/ceph/pull/25685>`_, Patrick Donnelly)
+* MDSMonitor: missing osdmon writeable check (`issue#37929 <http://tracker.ceph.com/issues/37929>`_, `pr#26069 <https://github.com/ceph/ceph/pull/26069>`_, Patrick Donnelly)
+* mds: purge queue recovery hangs during boot if PQ journal is damaged (`issue#37543 <http://tracker.ceph.com/issues/37543>`_, `pr#26055 <https://github.com/ceph/ceph/pull/26055>`_, Patrick Donnelly)
+* mds: PurgeQueue write error handler does not handle EBLACKLISTED (`issue#37394 <http://tracker.ceph.com/issues/37394>`_, `pr#25523 <https://github.com/ceph/ceph/pull/25523>`_, Patrick Donnelly)
+* mds: remove duplicated l_mdc_num_strays perfcounter set (`issue#37516 <http://tracker.ceph.com/issues/37516>`_, `pr#25681 <https://github.com/ceph/ceph/pull/25681>`_, Zhi Zhang)
+* mds: remove wrong assertion in Locker::snapflush_nudge (`issue#37721 <http://tracker.ceph.com/issues/37721>`_, `pr#25885 <https://github.com/ceph/ceph/pull/25885>`_, "Yan, Zheng")
+* mds: runs out of file descriptors after several respawns (`issue#35850 <http://tracker.ceph.com/issues/35850>`_, `pr#25822 <https://github.com/ceph/ceph/pull/25822>`_, Patrick Donnelly)
+* mds: severe internal fragment when decoding xattr_map from log event (`issue#37399 <http://tracker.ceph.com/issues/37399>`_, `pr#25519 <https://github.com/ceph/ceph/pull/25519>`_, "Yan, Zheng")
+* mds: trim cache after journal flush (`issue#38010 <http://tracker.ceph.com/issues/38010>`_, `pr#26214 <https://github.com/ceph/ceph/pull/26214>`_, Patrick Donnelly)
+* mds: wait shorter intervals if beacon not sent (`issue#36367 <http://tracker.ceph.com/issues/36367>`_, `pr#25980 <https://github.com/ceph/ceph/pull/25980>`_, Patrick Donnelly)
+* mgr: add get_latest_counter() to C++ -> Python interface (`issue#38138 <http://tracker.ceph.com/issues/38138>`_, `pr#26074 <https://github.com/ceph/ceph/pull/26074>`_, Jan Fajerski)
+* mgr/balancer: add cmd to list all plans (`issue#37418 <http://tracker.ceph.com/issues/37418>`_, `pr#25293 <https://github.com/ceph/ceph/pull/25293>`_, Yang Honggang)
+* mgr/balancer: add crush_compat_metrics param to change optimization keys (`issue#37412 <http://tracker.ceph.com/issues/37412>`_, `pr#25291 <https://github.com/ceph/ceph/pull/25291>`_, Dan van der Ster)
+* mgr/dashboard: Set mirror_mode to None (`issue#37870 <http://tracker.ceph.com/issues/37870>`_, `pr#26009 <https://github.com/ceph/ceph/pull/26009>`_, Sebastian Wagner)
+* mgr: deadlock: _check_auth_rotating possible clock skew, rotating keys expired way too early (`issue#23460 <http://tracker.ceph.com/issues/23460>`_, `pr#26426 <https://github.com/ceph/ceph/pull/26426>`_, Yan Jun)
+* mgr: prometheus: added bluestore db and wal devices to ceph_disk_occupation metric (`issue#36627 <http://tracker.ceph.com/issues/36627>`_, `pr#25218 <https://github.com/ceph/ceph/pull/25218>`_, Konstantin Shalygin)
+* mgr: race between daemon state and service map in 'service status' (`issue#36656 <http://tracker.ceph.com/issues/36656>`_, `pr#25368 <https://github.com/ceph/ceph/pull/25368>`_, Mykola Golub)
+* mgr/restful: fix py got exception when get osd info (`issue#38182 <http://tracker.ceph.com/issues/38182>`_, `pr#26200 <https://github.com/ceph/ceph/pull/26200>`_, Boris Ranto, zouaiguo)
+* mgr: various python3 fixes (`issue#37415 <http://tracker.ceph.com/issues/37415>`_, `pr#25292 <https://github.com/ceph/ceph/pull/25292>`_, Noah Watkins)
+* mgr will refuse connection from the monitor who starts behind it (`issue#37753 <http://tracker.ceph.com/issues/37753>`_, `pr#26235 <https://github.com/ceph/ceph/pull/26235>`_, Xinying Song)
+* mgr/zabbix: Send more PG information to Zabbix (`issue#38180 <http://tracker.ceph.com/issues/38180>`_, `pr#25944 <https://github.com/ceph/ceph/pull/25944>`_, Wido den Hollander)
+* mon: A PG with PG_STATE_REPAIR doesn't mean damaged data, PG_STATE_IN… (`issue#38070 <http://tracker.ceph.com/issues/38070>`_, `pr#26304 <https://github.com/ceph/ceph/pull/26304>`_, David Zafman)
+* mon: log last command skips latest entry (`issue#36679 <http://tracker.ceph.com/issues/36679>`_, `pr#25526 <https://github.com/ceph/ceph/pull/25526>`_, John Spray)
+* mon: mark REMOVE_SNAPS messages as no_reply (`issue#37568 <http://tracker.ceph.com/issues/37568>`_, `pr#25782 <https://github.com/ceph/ceph/pull/25782>`_, "Yan, Zheng")
+* mon/OSDMonitor: do not populate void pg_temp into nextmap (`issue#37784 <http://tracker.ceph.com/issues/37784>`_, `pr#25844 <https://github.com/ceph/ceph/pull/25844>`_, Aleksei Zakharov)
+* mon: shutdown messenger early to avoid accessing deleted logger (`issue#37780 <http://tracker.ceph.com/issues/37780>`_, `pr#25846 <https://github.com/ceph/ceph/pull/25846>`_, ningtao)
+* msg/async: backport recent messenger fixes (`issue#36497 <http://tracker.ceph.com/issues/36497>`_, `issue#37778 <http://tracker.ceph.com/issues/37778>`_, `pr#25958 <https://github.com/ceph/ceph/pull/25958>`_, xie xingguo)
+* msg/async: crashes when authenticator provided by verify_authorizer not implemented (`issue#36443 <http://tracker.ceph.com/issues/36443>`_, `pr#25299 <https://github.com/ceph/ceph/pull/25299>`_, Sage Weil)
+* multisite: es sync null versioned object failed because of olh info (`issue#23842 <http://tracker.ceph.com/issues/23842>`_, `issue#23841 <http://tracker.ceph.com/issues/23841>`_, `pr#25578 <https://github.com/ceph/ceph/pull/25578>`_, Tianshan Qu, Shang Ding)
+* os/bluestore: fixup access a destroy cond cause deadlock or undefine (`issue#37733 <http://tracker.ceph.com/issues/37733>`_, `pr#26260 <https://github.com/ceph/ceph/pull/26260>`_, linbing)
+* os/bluestore: KernelDevice::read() does the EIO mapping now (`issue#36455 <http://tracker.ceph.com/issues/36455>`_, `pr#25854 <https://github.com/ceph/ceph/pull/25854>`_, Radoslaw Zarzynski)
+* os/bluestore: rename does not old ref to replacement onode at old name (`issue#36541 <http://tracker.ceph.com/issues/36541>`_, `pr#25313 <https://github.com/ceph/ceph/pull/25313>`_, Sage Weil)
+* osd: Add support for osd_delete_sleep configuration value (`issue#36474 <http://tracker.ceph.com/issues/36474>`_, `pr#25507 <https://github.com/ceph/ceph/pull/25507>`_, Jianpeng Ma, David Zafman)
+* osd-backfill-stats.sh fails in rados/standalone/osd.yaml (`issue#37393 <http://tracker.ceph.com/issues/37393>`_, `issue#35982 <http://tracker.ceph.com/issues/35982>`_, `pr#26329 <https://github.com/ceph/ceph/pull/26329>`_, Sage Weil, David Zafman)
+* osd: backport recent upmap fixes (`issue#37940 <http://tracker.ceph.com/issues/37940>`_, `issue#37881 <http://tracker.ceph.com/issues/37881>`_, `pr#26128 <https://github.com/ceph/ceph/pull/26128>`_, huangjun, xie xingguo)
+* osdc/Objecter: update op_target_t::paused in _calc_target (`issue#37398 <http://tracker.ceph.com/issues/37398>`_, `pr#25718 <https://github.com/ceph/ceph/pull/25718>`_, Song Shun, runsisi)
+* osd: failed assert when osd_memory_target options mismatch (`issue#37507 <http://tracker.ceph.com/issues/37507>`_, `pr#25605 <https://github.com/ceph/ceph/pull/25605>`_, xie xingguo)
+* osd: force-backfill sets forced_recovery instead of forced_backfill in 13.2.1 (`issue#27985 <http://tracker.ceph.com/issues/27985>`_, `pr#26324 <https://github.com/ceph/ceph/pull/26324>`_, xie xingguo)
+* osd/mon: fix upgrades for pg log hard limit (`issue#36686 <http://tracker.ceph.com/issues/36686>`_, `pr#26206 <https://github.com/ceph/ceph/pull/26206>`_, Neha Ojha)
+* osd/OSDMap: cancel mapping if target osd is out (`issue#37501 <http://tracker.ceph.com/issues/37501>`_, `pr#25699 <https://github.com/ceph/ceph/pull/25699>`_, ningtao, xie xingguo)
+* osd/OSD: OSD::mkfs asserts when reusing disk with existing superblock (`issue#37404 <http://tracker.ceph.com/issues/37404>`_, `pr#25385 <https://github.com/ceph/ceph/pull/25385>`_, Igor Fedotov)
+* osd/PG.cc: account for missing set irrespective of last_complete (`issue#37919 <http://tracker.ceph.com/issues/37919>`_, `pr#26239 <https://github.com/ceph/ceph/pull/26239>`_, Neha Ojha)
+* osd/PrimaryLogPG: fix the extent length error of the sync read (`issue#37680 <http://tracker.ceph.com/issues/37680>`_, `pr#25708 <https://github.com/ceph/ceph/pull/25708>`_, Xiaofei Cui)
+* osd: Prioritize user specified scrubs (`issue#37269 <http://tracker.ceph.com/issues/37269>`_, `pr#25513 <https://github.com/ceph/ceph/pull/25513>`_, David Zafman)
+* os/filestore: ceph_abort() on fsync(2) or fdatasync(2) failure (`issue#38258 <http://tracker.ceph.com/issues/38258>`_, `pr#26438 <https://github.com/ceph/ceph/pull/26438>`_, Sage Weil)
+* pybind/mgr: drop unnecessary iterkeys usage to make py-3 compatible (`issue#37581 <http://tracker.ceph.com/issues/37581>`_, `pr#25759 <https://github.com/ceph/ceph/pull/25759>`_, Mykola Golub)
+* pybind/mgr/status: fix ceph fs status in py3 environments (`issue#37573 <http://tracker.ceph.com/issues/37573>`_, `pr#25694 <https://github.com/ceph/ceph/pull/25694>`_, Jan Fajerski)
+* qa: pjd test appears to require more than 3h timeout for some configurations (`issue#36594 <http://tracker.ceph.com/issues/36594>`_, `pr#25557 <https://github.com/ceph/ceph/pull/25557>`_, Patrick Donnelly)
+* qa/rados/upgrade: align thrashing with upgrade suite, don't import/export pgs (`issue#37665 <http://tracker.ceph.com/issues/37665>`_, `pr#25856 <https://github.com/ceph/ceph/pull/25856>`_, Sage Weil)
+* qa/tasks/radosbench: default to 64k writes (`issue#37797 <http://tracker.ceph.com/issues/37797>`_, `pr#26354 <https://github.com/ceph/ceph/pull/26354>`_, Sage Weil)
+* qa: test_damage needs to silence MDS_READ_ONLY (`issue#37944 <http://tracker.ceph.com/issues/37944>`_, `pr#26072 <https://github.com/ceph/ceph/pull/26072>`_, Patrick Donnelly)
+* qa: test_damage performs truncate test on same object repeatedly (`issue#37836 <http://tracker.ceph.com/issues/37836>`_, `issue#37837 <http://tracker.ceph.com/issues/37837>`_, `pr#26047 <https://github.com/ceph/ceph/pull/26047>`_, Patrick Donnelly)
+* qa: teuthology may hang on diagnostic commands for fuse mount (`issue#36390 <http://tracker.ceph.com/issues/36390>`_, `pr#25515 <https://github.com/ceph/ceph/pull/25515>`_, Patrick Donnelly)
+* qa: whitelist cap revoke warning (`issue#25188 <http://tracker.ceph.com/issues/25188>`_, `pr#26496 <https://github.com/ceph/ceph/pull/26496>`_, Patrick Donnelly)
+* qa/workunits/rados/test_health_warnings: prevent out osds (`issue#37776 <http://tracker.ceph.com/issues/37776>`_, `pr#25850 <https://github.com/ceph/ceph/pull/25850>`_, Sage Weil)
+* qa: wrong setting for msgr failures (`issue#36676 <http://tracker.ceph.com/issues/36676>`_, `pr#25517 <https://github.com/ceph/ceph/pull/25517>`_, Patrick Donnelly)
+* rbd: fix delay time calculation for trash move (`issue#37861 <http://tracker.ceph.com/issues/37861>`_, `pr#25954 <https://github.com/ceph/ceph/pull/25954>`_, Mykola Golub)
+* rgw: debug logging for v4 auth does not sanitize encryption keys (`issue#37847 <http://tracker.ceph.com/issues/37847>`_, `pr#26003 <https://github.com/ceph/ceph/pull/26003>`_, Casey Bodley)
+* rgw: Don't treat colons specially in resource part of ARN (`issue#23817 <http://tracker.ceph.com/issues/23817>`_, `pr#25386 <https://github.com/ceph/ceph/pull/25386>`_, Adam C. Emerson)
+* rgw: fails to start on Fedora 28 from default configuration (`issue#24228 <http://tracker.ceph.com/issues/24228>`_, `pr#26129 <https://github.com/ceph/ceph/pull/26129>`_, Matt Benjamin)
+* rgw: feature -- log successful bucket resharding events (`issue#37647 <http://tracker.ceph.com/issues/37647>`_, `pr#25740 <https://github.com/ceph/ceph/pull/25740>`_, J. Eric Ivancich)
+* rgw_file: user info never synced since librgw init (`issue#37527 <http://tracker.ceph.com/issues/37527>`_, `pr#25485 <https://github.com/ceph/ceph/pull/25485>`_, Tao Chen)
+* rgw: fix max-size in radosgw-admin and REST Admin API (`issue#37517 <http://tracker.ceph.com/issues/37517>`_, `pr#25449 <https://github.com/ceph/ceph/pull/25449>`_, Nick Erdmann)
+* rgw: fix version bucket stats (`issue#21429 <http://tracker.ceph.com/issues/21429>`_, `pr#25643 <https://github.com/ceph/ceph/pull/25643>`_, Shasha Lu)
+* rgw: handle S3 version 2 pre-signed urls with meta-data (`issue#23470 <http://tracker.ceph.com/issues/23470>`_, `pr#25899 <https://github.com/ceph/ceph/pull/25899>`_, Matt Benjamin)
+* rgw: master zone deletion without a zonegroup rm would break rgw rados init (`issue#37328 <http://tracker.ceph.com/issues/37328>`_, `pr#25511 <https://github.com/ceph/ceph/pull/25511>`_, Abhishek Lekshmanan)
+* rgw: multisite: sync gets stuck retrying deletes that fail with ERR_PRECONDITION_FAILED (`issue#37448 <http://tracker.ceph.com/issues/37448>`_, `pr#25505 <https://github.com/ceph/ceph/pull/25505>`_, Casey Bodley)
+* rgw: Object can still be deleted even if s3:DeleteObject policy is set (`issue#37403 <http://tracker.ceph.com/issues/37403>`_, `pr#26309 <https://github.com/ceph/ceph/pull/26309>`_, Enming.Zhang)
+* rgw: "radosgw-admin bucket rm ... --purge-objects" can hang (`issue#38134 <http://tracker.ceph.com/issues/38134>`_, `pr#26266 <https://github.com/ceph/ceph/pull/26266>`_, J. Eric Ivancich)
+* rgw: radosgw-admin: translate reshard status codes (trivial) (`issue#36486 <http://tracker.ceph.com/issues/36486>`_, `pr#25198 <https://github.com/ceph/ceph/pull/25198>`_, Matt Benjamin)
+* rgw: rgwgc: process coredump in some special case (`issue#23199 <http://tracker.ceph.com/issues/23199>`_, `pr#25624 <https://github.com/ceph/ceph/pull/25624>`_, zhaokun)
+* rpm: Use hardened LDFLAGS (`issue#36316 <http://tracker.ceph.com/issues/36316>`_, `pr#25171 <https://github.com/ceph/ceph/pull/25171>`_, Boris Ranto)
+
+v13.2.4 Mimic
+=============
+
+This is the fourth bugfix release of the Mimic v13.2.x long term
+stable release series.  This release includes two security fixes that
+were tested but inadvertently excluded from the final v13.2.3 release
+build.
+
+Changelog
+---------
+
+* CVE-2018-16846: rgw: enforce bounds on max-keys/max-uploads/max-parts (`issue#35994 <http://tracker.ceph.com/issues/35994>`_)
+* CVE-2018-14662: mon: limit caps allowed to access the config store
+
+v13.2.3 Mimic
+=============
+
+This is the third bugfix release of the Mimic v13.2.x long term stable release
+series. This release contains many fixes across all components of Ceph.
+We recommend that all users upgrade.
+
+* The default memory utilization for the mons has been increased
+  somewhat.  Rocksdb now uses 512 MB of RAM by default, which should
+  be sufficient for small to medium-sized clusters; large clusters
+  should tune this up.  Also, the `mon_osd_cache_size` has been
+  increase from 10 OSDMaps to 500, which will translate to an
+  additional 500 MB to 1 GB of RAM for large clusters, and much less
+  for small clusters.
+
+* Ceph v13.2.2 includes a wrong backport, which may cause mds to go into
+  'damaged' state when upgrading Ceph cluster from previous version.
+  The bug is fixed in v13.2.3. If you are already running v13.2.2,
+  upgrading to v13.2.3 does not require special action.
+
+* The bluestore_cache_* options are no longer needed. They are replaced
+  by osd_memory_target, defaulting to 4GB. BlueStore will expand
+  and contract its cache to attempt to stay within this
+  limit. Users upgrading should note this is a higher default
+  than the previous bluestore_cache_size of 1GB, so OSDs using
+  BlueStore will use more memory by default.
+  For more details, see the `BlueStore docs <http://docs.ceph.com/docs/mimic/rados/configuration/bluestore-config-ref/#automatic-cache-sizing>`_.
+
+* This version contains an upgrade bug, http://tracker.ceph.com/issues/36686,
+  due to which upgrading during recovery/backfill can cause OSDs to fail. This
+  bug can be worked around, either by restarting all the OSDs after the upgrade,
+  or by upgrading when all PGs are in "active+clean" state. If you have already
+  successfully upgraded to 13.2.2, this issue should not impact you. Going
+  forward, we are working on a clean upgrade path for this feature.
+
+Changelog
+---------
+
+* build/ops: Can't compile Ceph on Fedora 29 as it doesn't recognize python\*3\*-tox as an install Tox (`issue#18163 <http://tracker.ceph.com/issues/18163>`_, `issue#37301 <http://tracker.ceph.com/issues/37301>`_, `issue#37422 <http://tracker.ceph.com/issues/37422>`_, `pr#25294 <https://github.com/ceph/ceph/pull/25294>`_, Nathan Cutler, Brad Hubbard)
+* build/ops: debian: correct ceph-common relationship with older radosgw package (`pr#25115 <https://github.com/ceph/ceph/pull/25115>`_, Matthew Vernon)
+* ceph-bluestore-tool: fix set label functionality for specific keys (`pr#24352 <https://github.com/ceph/ceph/pull/24352>`_, Igor Fedotov)
+* ceph fs add_data_pool applies pool application metadata incorrectly (`issue#36203 <http://tracker.ceph.com/issues/36203>`_, `issue#36028 <http://tracker.ceph.com/issues/36028>`_, `pr#24470 <https://github.com/ceph/ceph/pull/24470>`_, John Spray)
+* cephfs: client: explicitly show blacklisted state via asok status command (`issue#36457 <http://tracker.ceph.com/issues/36457>`_, `issue#36352 <http://tracker.ceph.com/issues/36352>`_, `pr#24993 <https://github.com/ceph/ceph/pull/24993>`_, Jonathan Brielmaier, Zhi Zhang)
+* cephfs: client: request next osdmap for blacklisted client (`issue#36668 <http://tracker.ceph.com/issues/36668>`_, `issue#36690 <http://tracker.ceph.com/issues/36690>`_, `pr#24987 <https://github.com/ceph/ceph/pull/24987>`_, Zhi Zhang)
+* cephfs-journal-tool: wrong layout info used (`issue#24933 <http://tracker.ceph.com/issues/24933>`_, `issue#24644 <http://tracker.ceph.com/issues/24644>`_, `pr#24583 <https://github.com/ceph/ceph/pull/24583>`_, Gu Zhongyan)
+* cephfs: some tool commands silently operate on only rank 0, even if multiple ranks exist (`issue#36218 <http://tracker.ceph.com/issues/36218>`_, `pr#25036 <https://github.com/ceph/ceph/pull/25036>`_, Venky Shankar)
+* ceph-fuse: add to selinux profile (`issue#36103 <http://tracker.ceph.com/issues/36103>`_, `issue#36197 <http://tracker.ceph.com/issues/36197>`_, `pr#24439 <https://github.com/ceph/ceph/pull/24439>`_, Patrick Donnelly)
+* ceph-volume:  activate option --auto-detect-objectstore respects --no-systemd (`issue#36249 <http://tracker.ceph.com/issues/36249>`_, `pr#24357 <https://github.com/ceph/ceph/pull/24357>`_, Alfredo Deza)
+* ceph-volume add device_id to inventory listing (`pr#25349 <https://github.com/ceph/ceph/pull/25349>`_, Jan Fajerski)
+* ceph-volume: add inventory command (`issue#24972 <http://tracker.ceph.com/issues/24972>`_, `pr#25013 <https://github.com/ceph/ceph/pull/25013>`_, Jan Fajerski)
+* ceph-volume Additional work on ceph-volume to add some choose_disk capabilities (`issue#36446 <http://tracker.ceph.com/issues/36446>`_, `pr#24782 <https://github.com/ceph/ceph/pull/24782>`_, Erwan Velu)
+* ceph-volume add new ceph-handlers role from ceph-ansible (`issue#36251 <http://tracker.ceph.com/issues/36251>`_, `pr#24337 <https://github.com/ceph/ceph/pull/24337>`_, Alfredo Deza)
+* ceph-volume: adds a --prepare flag to `lvm batch` (`issue#36363 <http://tracker.ceph.com/issues/36363>`_, `pr#24760 <https://github.com/ceph/ceph/pull/24760>`_, Andrew Schoen)
+* ceph-volume: allow to specify --cluster-fsid instead of reading from ceph.conf (`issue#26953 <http://tracker.ceph.com/issues/26953>`_, `pr#25116 <https://github.com/ceph/ceph/pull/25116>`_, Alfredo Deza)
+* ceph_volume_client: py3 compatible (`issue#26850 <http://tracker.ceph.com/issues/26850>`_, `issue#17230 <http://tracker.ceph.com/issues/17230>`_, `pr#24443 <https://github.com/ceph/ceph/pull/24443>`_, Rishabh Dave, Patrick Donnelly)
+* ceph-volume custom cluster names fail on filestore trigger (`issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#24279 <https://github.com/ceph/ceph/pull/24279>`_, Alfredo Deza)
+* ceph-volume: do not send (lvm) stderr/stdout to the terminal, use the logfile (`issue#36492 <http://tracker.ceph.com/issues/36492>`_, `pr#24740 <https://github.com/ceph/ceph/pull/24740>`_, Alfredo Deza)
+* ceph-volume enable  --no-systemd flag for simple sub-command (`issue#36470 <http://tracker.ceph.com/issues/36470>`_, `pr#25011 <https://github.com/ceph/ceph/pull/25011>`_, Alfredo Deza)
+* ceph-volume: fix journal and filestore data size in `lvm batch --report` (`issue#36242 <http://tracker.ceph.com/issues/36242>`_, `pr#24306 <https://github.com/ceph/ceph/pull/24306>`_, Andrew Schoen)
+* ceph-volume:  lsblk can fail to find PARTLABEL, must fallback to blkid (`issue#36098 <http://tracker.ceph.com/issues/36098>`_, `pr#24334 <https://github.com/ceph/ceph/pull/24334>`_, Alfredo Deza)
+* ceph-volume lvm.prepare update help to indicate partitions are needed, not devices (`issue#24795 <http://tracker.ceph.com/issues/24795>`_, `pr#24449 <https://github.com/ceph/ceph/pull/24449>`_, Alfredo Deza)
+* ceph-volume: make `lvm batch` idempotent (`pr#24588 <https://github.com/ceph/ceph/pull/24588>`_, Andrew Schoen)
+* ceph-volume: patch Device when testing (`issue#36768 <http://tracker.ceph.com/issues/36768>`_, `pr#25066 <https://github.com/ceph/ceph/pull/25066>`_, Alfredo Deza)
+* ceph-volume: reject devices that have existing GPT headers (`issue#27062 <http://tracker.ceph.com/issues/27062>`_, `pr#25103 <https://github.com/ceph/ceph/pull/25103>`_, Andrew Schoen)
+* ceph-volume: remove LVs when using zap --destroy (`pr#25100 <https://github.com/ceph/ceph/pull/25100>`_, Alfredo Deza)
+* ceph-volume remove version reporting from help menu (`issue#36386 <http://tracker.ceph.com/issues/36386>`_, `pr#24753 <https://github.com/ceph/ceph/pull/24753>`_, Alfredo Deza)
+* ceph-volume: rename Device property valid to available (`issue#36701 <http://tracker.ceph.com/issues/36701>`_, `pr#25133 <https://github.com/ceph/ceph/pull/25133>`_, Jan Fajerski)
+* ceph-volume: skip processing devices that don't exist when scanning system disks (`issue#36247 <http://tracker.ceph.com/issues/36247>`_, `pr#24381 <https://github.com/ceph/ceph/pull/24381>`_, Alfredo Deza)
+* ceph-volume systemd import main so console_scripts work for executable (`issue#36648 <http://tracker.ceph.com/issues/36648>`_, `pr#24852 <https://github.com/ceph/ceph/pull/24852>`_, Alfredo Deza)
+* ceph-volume tests install ceph-ansible's requirements.txt dependencies (`issue#36672 <http://tracker.ceph.com/issues/36672>`_, `pr#24959 <https://github.com/ceph/ceph/pull/24959>`_, Alfredo Deza)
+* ceph-volume tests.systemd update imports for systemd module (`issue#36704 <http://tracker.ceph.com/issues/36704>`_, `pr#24957 <https://github.com/ceph/ceph/pull/24957>`_, Alfredo Deza)
+* ceph-volume: use console_scripts (`issue#36601 <http://tracker.ceph.com/issues/36601>`_, `pr#24838 <https://github.com/ceph/ceph/pull/24838>`_, Mehdi Abaakouk)
+* ceph-volume util.encryption don't push stderr to terminal (`issue#36246 <http://tracker.ceph.com/issues/36246>`_, `pr#24826 <https://github.com/ceph/ceph/pull/24826>`_, Alfredo Deza)
+* ceph-volume util.encryption robust blkid+lsblk detection of lockbox (`pr#24980 <https://github.com/ceph/ceph/pull/24980>`_, Alfredo Deza)
+* client: fix use-after-free in Client::link() (`issue#35841 <http://tracker.ceph.com/issues/35841>`_, `issue#24557 <http://tracker.ceph.com/issues/24557>`_, `pr#24187 <https://github.com/ceph/ceph/pull/24187>`_, "Yan, Zheng")
+* client: statfs inode count odd (`issue#35940 <http://tracker.ceph.com/issues/35940>`_, `issue#24849 <http://tracker.ceph.com/issues/24849>`_, `pr#24377 <https://github.com/ceph/ceph/pull/24377>`_, Rishabh Dave)
+* client:two ceph-fuse client, one can not list out files created by an… (`issue#27051 <http://tracker.ceph.com/issues/27051>`_, `issue#35934 <http://tracker.ceph.com/issues/35934>`_, `pr#24295 <https://github.com/ceph/ceph/pull/24295>`_, Peng Xie)
+* client: update ctime when modifying file content (`issue#35945 <http://tracker.ceph.com/issues/35945>`_, `issue#36134 <http://tracker.ceph.com/issues/36134>`_, `pr#24385 <https://github.com/ceph/ceph/pull/24385>`_, "Yan, Zheng")
+* common: get real hostname from container/pod environment (`pr#23916 <https://github.com/ceph/ceph/pull/23916>`_, Sage Weil)
+* core: _aio_log_start inflight overlap of 0x10000~1000 with [65536~4096] (`issue#36754 <http://tracker.ceph.com/issues/36754>`_, `issue#36625 <http://tracker.ceph.com/issues/36625>`_, `pr#25062 <https://github.com/ceph/ceph/pull/25062>`_, Jonathan Brielmaier, Yang Honggang)
+* core: FAILED assert(osdmap_manifest.pinned.empty()) in OSDMonitor::prune_init() (`issue#24612 <http://tracker.ceph.com/issues/24612>`_, `issue#35071 <http://tracker.ceph.com/issues/35071>`_, `pr#24918 <https://github.com/ceph/ceph/pull/24918>`_, Joao Eduardo Luis)
+* core: Interactive mode CLI prints no output since Mimic (`issue#36358 <http://tracker.ceph.com/issues/36358>`_, `issue#36432 <http://tracker.ceph.com/issues/36432>`_, `pr#24971 <https://github.com/ceph/ceph/pull/24971>`_, John Spray, Mohamad Gebai)
+* core: mgr crash on scrub of unconnected osd (`issue#36110 <http://tracker.ceph.com/issues/36110>`_, `issue#36465 <http://tracker.ceph.com/issues/36465>`_, `pr#25029 <https://github.com/ceph/ceph/pull/25029>`_, Sage Weil)
+* core: mon osdmap cash too small during upgrade to mimic (`issue#36505 <http://tracker.ceph.com/issues/36505>`_, `pr#25019 <https://github.com/ceph/ceph/pull/25019>`_, Sage Weil)
+* core: monstore tool rebuild does not generate creating_pgs (`issue#36306 <http://tracker.ceph.com/issues/36306>`_, `issue#36433 <http://tracker.ceph.com/issues/36433>`_, `pr#25016 <https://github.com/ceph/ceph/pull/25016>`_, Sage Weil)
+* core: Objecter: add ignore cache flag if got redirect reply (`issue#36658 <http://tracker.ceph.com/issues/36658>`_, `pr#25075 <https://github.com/ceph/ceph/pull/25075>`_, Iain Buclaw, Jonathan Brielmaier)
+* core: objecter cannot resend split-dropped op when racing with con reset (`issue#22544 <http://tracker.ceph.com/issues/22544>`_, `issue#35843 <http://tracker.ceph.com/issues/35843>`_, `pr#24970 <https://github.com/ceph/ceph/pull/24970>`_, Sage Weil)
+* core: os/bluestore: cache autotuning and memory limit (`issue#37340 <http://tracker.ceph.com/issues/37340>`_, `pr#25283 <https://github.com/ceph/ceph/pull/25283>`_, Josh Durgin, Mark Nelson)
+* core: rados rm --force-full is blocked when cluster is in full status (`issue#36435 <http://tracker.ceph.com/issues/36435>`_, `pr#25017 <https://github.com/ceph/ceph/pull/25017>`_, Yang Honggang)
+* crush/CrushWrapper: fix crush tree json dumper (`issue#36150 <http://tracker.ceph.com/issues/36150>`_, `pr#24481 <https://github.com/ceph/ceph/pull/24481>`_, Oshyn Song)
+* debian/control: require fuse for ceph-fuse (`issue#21057 <http://tracker.ceph.com/issues/21057>`_, `pr#24037 <https://github.com/ceph/ceph/pull/24037>`_, Thomas Serlin)
+* doc: add ceph-volume inventory sections (`pr#25130 <https://github.com/ceph/ceph/pull/25130>`_, Jan Fajerski)
+* doc: fix broken fstab url in cephfs/fuse (`issue#36286 <http://tracker.ceph.com/issues/36286>`_, `issue#36313 <http://tracker.ceph.com/issues/36313>`_, `pr#24441 <https://github.com/ceph/ceph/pull/24441>`_, Jos Collin)
+* doc: Put command template into literal block (`pr#25000 <https://github.com/ceph/ceph/pull/25000>`_, Alexey Stupnikov)
+* doc: remove deprecated 'scrubq' from ceph(8) (`issue#35813 <http://tracker.ceph.com/issues/35813>`_, `issue#35855 <http://tracker.ceph.com/issues/35855>`_, `pr#24210 <https://github.com/ceph/ceph/pull/24210>`_, Ruben Kerkhof)
+* docs: backport edit on github changes (`pr#25362 <https://github.com/ceph/ceph/pull/25362>`_, Neha Ojha, Noah Watkins)
+* doc: Typo error on cephfs/fuse/ (`issue#36180 <http://tracker.ceph.com/issues/36180>`_, `issue#36308 <http://tracker.ceph.com/issues/36308>`_, `pr#24420 <https://github.com/ceph/ceph/pull/24420>`_, Karun Josy)
+* ec: src/common/interval_map.h: 161: FAILED assert(len > 0) (`issue#21931 <http://tracker.ceph.com/issues/21931>`_, `issue#22330 <http://tracker.ceph.com/issues/22330>`_, `pr#24581 <https://github.com/ceph/ceph/pull/24581>`_, Neha Ojha)
+* fsck: cid is improperly matched to oid (`issue#36146 <http://tracker.ceph.com/issues/36146>`_, `issue#36551 <http://tracker.ceph.com/issues/36551>`_, `issue#36099 <http://tracker.ceph.com/issues/36099>`_, `issue#32731 <http://tracker.ceph.com/issues/32731>`_, `pr#24480 <https://github.com/ceph/ceph/pull/24480>`_, Kefu Chai, Sage Weil)
+* kernel_untar_build.sh: bison: command not found (`issue#36121 <http://tracker.ceph.com/issues/36121>`_, `pr#24241 <https://github.com/ceph/ceph/pull/24241>`_, Neha Ojha)
+* libcephfs: expose CEPH_SETATTR_MTIME_NOW and CEPH_SETATTR_ATIME_NOW (`issue#36205 <http://tracker.ceph.com/issues/36205>`_, `issue#35961 <http://tracker.ceph.com/issues/35961>`_, `pr#24464 <https://github.com/ceph/ceph/pull/24464>`_, Zhu Shangzhong)
+* librados application's symbol could conflict with the libceph-common (`issue#26839 <http://tracker.ceph.com/issues/26839>`_, `issue#25154 <http://tracker.ceph.com/issues/25154>`_, `pr#24708 <https://github.com/ceph/ceph/pull/24708>`_, Kefu Chai)
+* librbd: blacklisted client might not notice it lost the lock (`issue#34534 <http://tracker.ceph.com/issues/34534>`_, `pr#24401 <https://github.com/ceph/ceph/pull/24401>`_, Jason Dillaman)
+* librbd: ensure exclusive lock acquired when removing sync point snaps… (`issue#35714 <http://tracker.ceph.com/issues/35714>`_, `issue#24898 <http://tracker.ceph.com/issues/24898>`_, `pr#24137 <https://github.com/ceph/ceph/pull/24137>`_, Mykola Golub)
+* librbd: fixed assert when flattening clone with zero overlap (`issue#35957 <http://tracker.ceph.com/issues/35957>`_, `issue#35702 <http://tracker.ceph.com/issues/35702>`_, `pr#24356 <https://github.com/ceph/ceph/pull/24356>`_, Jason Dillaman)
+* librbd: journaling unable request can not be sent to remote lock owner (`issue#26939 <http://tracker.ceph.com/issues/26939>`_, `issue#35712 <http://tracker.ceph.com/issues/35712>`_, `pr#24122 <https://github.com/ceph/ceph/pull/24122>`_, Mykola Golub)
+* librbd: object map improperly flagged as invalidated (`issue#24516 <http://tracker.ceph.com/issues/24516>`_, `issue#36225 <http://tracker.ceph.com/issues/36225>`_, `pr#24413 <https://github.com/ceph/ceph/pull/24413>`_, Jason Dillaman)
+* librgw: crashes in multisite configuration (`issue#36302 <http://tracker.ceph.com/issues/36302>`_, `issue#36415 <http://tracker.ceph.com/issues/36415>`_, `pr#24908 <https://github.com/ceph/ceph/pull/24908>`_, Casey Bodley)
+* mds: allows client to create .. and . dirents (`issue#32104 <http://tracker.ceph.com/issues/32104>`_, `pr#24384 <https://github.com/ceph/ceph/pull/24384>`_, Venky Shankar)
+* mds: curate priority of perf counters sent to mgr (`issue#35938 <http://tracker.ceph.com/issues/35938>`_, `issue#26991 <http://tracker.ceph.com/issues/26991>`_, `issue#32090 <http://tracker.ceph.com/issues/32090>`_, `issue#35837 <http://tracker.ceph.com/issues/35837>`_, `pr#24467 <https://github.com/ceph/ceph/pull/24467>`_, Patrick Donnelly, Venky Shankar)
+* mds: evict cap revoke non-responding clients (`pr#24661 <https://github.com/ceph/ceph/pull/24661>`_, Venky Shankar)
+* mimic:mds: fix mds damaged due to unexpected journal length (`issue#36199 <http://tracker.ceph.com/issues/36199>`_, `pr#24463 <https://github.com/ceph/ceph/pull/24463>`_, Zhi Zhang)
+* mds: internal op missing events time 'throttled', 'all_read', 'dispatched' (`issue#36114 <http://tracker.ceph.com/issues/36114>`_, `issue#36195 <http://tracker.ceph.com/issues/36195>`_, `pr#24411 <https://github.com/ceph/ceph/pull/24411>`_, Yanhu Cao)
+* mds: migrate strays part by part when shutdown mds (`issue#26926 <http://tracker.ceph.com/issues/26926>`_, `issue#32092 <http://tracker.ceph.com/issues/32092>`_, `pr#24435 <https://github.com/ceph/ceph/pull/24435>`_, "Yan, Zheng")
+* mds: optimize the way how max export size is enforced (`issue#25131 <http://tracker.ceph.com/issues/25131>`_, `pr#23952 <https://github.com/ceph/ceph/pull/23952>`_, "Yan, Zheng")
+* mds: print is_laggy message once (`issue#35250 <http://tracker.ceph.com/issues/35250>`_, `issue#35719 <http://tracker.ceph.com/issues/35719>`_, `pr#24161 <https://github.com/ceph/ceph/pull/24161>`_, Patrick Donnelly)
+* mds: rctime may go back (`issue#35916 <http://tracker.ceph.com/issues/35916>`_, `issue#36136 <http://tracker.ceph.com/issues/36136>`_, `pr#24379 <https://github.com/ceph/ceph/pull/24379>`_, "Yan, Zheng")
+* mds: rctime not set on system inode (root) at startup (`issue#36221 <http://tracker.ceph.com/issues/36221>`_, `issue#36461 <http://tracker.ceph.com/issues/36461>`_, `pr#25042 <https://github.com/ceph/ceph/pull/25042>`_, Patrick Donnelly)
+* mds: reset heartbeat map at potential time-consuming places (`issue#26858 <http://tracker.ceph.com/issues/26858>`_, `pr#23506 <https://github.com/ceph/ceph/pull/23506>`_, Yan, Zheng, "Yan, Zheng")
+* mds: src/mds/MDLog.cc: 281: FAILED ceph_assert(!capped) during max_mds thrashing (`issue#36350 <http://tracker.ceph.com/issues/36350>`_, `issue#37093 <http://tracker.ceph.com/issues/37093>`_, `pr#25095 <https://github.com/ceph/ceph/pull/25095>`_, "Yan, Zheng", Jonathan Brielmaier)
+* mgr/DaemonServer: fix Session leak (`pr#24233 <https://github.com/ceph/ceph/pull/24233>`_, Sage Weil)
+* mgr/dashboard: Add http support to dashboard (`issue#36069 <http://tracker.ceph.com/issues/36069>`_, `pr#24734 <https://github.com/ceph/ceph/pull/24734>`_, Boris Ranto, Wido den Hollander)
+* mgr/dashboard: Add support for URI encode (`issue#24621 <http://tracker.ceph.com/issues/24621>`_, `issue#26856 <http://tracker.ceph.com/issues/26856>`_, `issue#24907 <http://tracker.ceph.com/issues/24907>`_, `pr#24488 <https://github.com/ceph/ceph/pull/24488>`_, Tiago Melo)
+* mgr/dashboard: Progress bar does not stop in TableKeyValueComponent (`issue#35925 <http://tracker.ceph.com/issues/35925>`_, `pr#24258 <https://github.com/ceph/ceph/pull/24258>`_, Volker Theile)
+* mgr/dashboard: Remove fieldsets when using CdTable (`issue#27851 <http://tracker.ceph.com/issues/27851>`_, `issue#26999 <http://tracker.ceph.com/issues/26999>`_, `pr#24478 <https://github.com/ceph/ceph/pull/24478>`_, Tiago Melo)
+* mgr: hold lock while accessing the request list and submittin request (`pr#25113 <https://github.com/ceph/ceph/pull/25113>`_, Jerry Lee)
+* mgr: [restful] deep_scrub is not a valid OSD command (`issue#36720 <http://tracker.ceph.com/issues/36720>`_, `issue#36749 <http://tracker.ceph.com/issues/36749>`_, `pr#25040 <https://github.com/ceph/ceph/pull/25040>`_, Boris Ranto)
+* mon: mgr options not parse propertly (`issue#35076 <http://tracker.ceph.com/issues/35076>`_, `issue#35836 <http://tracker.ceph.com/issues/35836>`_, `pr#24176 <https://github.com/ceph/ceph/pull/24176>`_, Sage Weil)
+* mon/OSDMonitor: invalidate max_failed_since on cancel_report (`issue#35930 <http://tracker.ceph.com/issues/35930>`_, `issue#35860 <http://tracker.ceph.com/issues/35860>`_, `pr#24281 <https://github.com/ceph/ceph/pull/24281>`_, xie xingguo)
+* mon: test if gid exists in pending for prepare_beacon (`issue#35848 <http://tracker.ceph.com/issues/35848>`_, `pr#24272 <https://github.com/ceph/ceph/pull/24272>`_, Patrick Donnelly)
+* msg/async: clean up local buffers on dispatch (`issue#36127 <http://tracker.ceph.com/issues/36127>`_, `issue#35987 <http://tracker.ceph.com/issues/35987>`_, `pr#24386 <https://github.com/ceph/ceph/pull/24386>`_, Greg Farnum)
+* msg: ceph_abort() when there are enough accepter errors in msg server (`issue#36219 <http://tracker.ceph.com/issues/36219>`_, `pr#25045 <https://github.com/ceph/ceph/pull/25045>`_, penglaiyxy@gmail.com)
+* msg: challenging authorizer messages appear at debug_ms=0 (`issue#35251 <http://tracker.ceph.com/issues/35251>`_, `issue#35717 <http://tracker.ceph.com/issues/35717>`_, `pr#24113 <https://github.com/ceph/ceph/pull/24113>`_, Patrick Donnelly)
+* multisite: data full sync does not limit concurrent bucket sync (`issue#26897 <http://tracker.ceph.com/issues/26897>`_, `issue#36216 <http://tracker.ceph.com/issues/36216>`_, `pr#24536 <https://github.com/ceph/ceph/pull/24536>`_, Casey Bodley)
+* multisite: data sync error repo processing does not back off on empty (`issue#35979 <http://tracker.ceph.com/issues/35979>`_, `issue#26938 <http://tracker.ceph.com/issues/26938>`_, `pr#24319 <https://github.com/ceph/ceph/pull/24319>`_, Casey Bodley)
+* multisite: incremental data sync makes unnecessary call to RGWReadRemoteDataLogShardInfoCR (`issue#35977 <http://tracker.ceph.com/issues/35977>`_, `issue#26952 <http://tracker.ceph.com/issues/26952>`_, `pr#24710 <https://github.com/ceph/ceph/pull/24710>`_, Casey Bodley)
+* multisite: intermittent test_bucket_index_log_trim failures (`issue#36201 <http://tracker.ceph.com/issues/36201>`_, `issue#36034 <http://tracker.ceph.com/issues/36034>`_, `pr#24400 <https://github.com/ceph/ceph/pull/24400>`_, Casey Bodley)
+* multisite: invalid read in RGWCloneMetaLogCoroutine (`issue#36208 <http://tracker.ceph.com/issues/36208>`_, `issue#35851 <http://tracker.ceph.com/issues/35851>`_, `pr#24414 <https://github.com/ceph/ceph/pull/24414>`_, Casey Bodley)
+* multisite: segfault on shutdown/realm reload (`issue#35857 <http://tracker.ceph.com/issues/35857>`_, `issue#35543 <http://tracker.ceph.com/issues/35543>`_, `pr#24235 <https://github.com/ceph/ceph/pull/24235>`_, Casey Bodley)
+* os/bluestore: fix bloom filter num entry miscalculation in repairer (`issue#25001 <http://tracker.ceph.com/issues/25001>`_, `pr#24339 <https://github.com/ceph/ceph/pull/24339>`_, Igor Fedotov)
+* os/bluestore: handle spurious read errors (`issue#22464 <http://tracker.ceph.com/issues/22464>`_, `pr#24647 <https://github.com/ceph/ceph/pull/24647>`_, Paul Emmerich)
+* osd: add creating to pg_string_state (`issue#36174 <http://tracker.ceph.com/issues/36174>`_, `issue#36298 <http://tracker.ceph.com/issues/36298>`_, `pr#24601 <https://github.com/ceph/ceph/pull/24601>`_, Dan van der Ster)
+* osd: backport recent upmap fixes (`pr#25419 <https://github.com/ceph/ceph/pull/25419>`_, ningtao, xie xingguo)
+* osdc/Objecter: possible race condition with connection reset (`issue#36183 <http://tracker.ceph.com/issues/36183>`_, `issue#36296 <http://tracker.ceph.com/issues/36296>`_, `pr#24600 <https://github.com/ceph/ceph/pull/24600>`_, Jason Dillaman)
+* osd: crash in OpTracker::unregister_inflight_op via OSD::get_health_metrics (`issue#24889 <http://tracker.ceph.com/issues/24889>`_, `pr#23026 <https://github.com/ceph/ceph/pull/23026>`_, Radoslaw Zarzynski)
+* osdc: reduce ObjectCacher's memory fragments (`issue#36192 <http://tracker.ceph.com/issues/36192>`_, `issue#36643 <http://tracker.ceph.com/issues/36643>`_, `pr#24873 <https://github.com/ceph/ceph/pull/24873>`_, "Yan, Zheng")
+* osd/ECBackend: don't get result code of subchunk-read overwritten (`issue#35959 <http://tracker.ceph.com/issues/35959>`_, `issue#21769 <http://tracker.ceph.com/issues/21769>`_, `pr#24298 <https://github.com/ceph/ceph/pull/24298>`_, songweibin)
+* OSDMapMapping does not handle active.size() > pool size (`issue#26866 <http://tracker.ceph.com/issues/26866>`_, `issue#35936 <http://tracker.ceph.com/issues/35936>`_, `pr#24431 <https://github.com/ceph/ceph/pull/24431>`_, Sage Weil)
+* osd/PG: avoid choose_acting picking want with > pool size items (`issue#35963 <http://tracker.ceph.com/issues/35963>`_, `issue#35924 <http://tracker.ceph.com/issues/35924>`_, `pr#24344 <https://github.com/ceph/ceph/pull/24344>`_, Sage Weil)
+* osd/PrimaryLogPG: fix potential pg-log overtrimming (`pr#24309 <https://github.com/ceph/ceph/pull/24309>`_, xie xingguo)
+* osd: race condition opening heartbeat connection (`issue#36637 <http://tracker.ceph.com/issues/36637>`_, `issue#36602 <http://tracker.ceph.com/issues/36602>`_, `pr#25026 <https://github.com/ceph/ceph/pull/25026>`_, Sage Weil)
+* osd: RBD client IOPS pool stats are incorrect (2x higher; includes IO hints as an op) (`issue#24909 <http://tracker.ceph.com/issues/24909>`_, `issue#36557 <http://tracker.ceph.com/issues/36557>`_, `pr#25024 <https://github.com/ceph/ceph/pull/25024>`_, Jason Dillaman)
+* osd: Remove old bft= which has been superceded by backfill (`issue#36292 <http://tracker.ceph.com/issues/36292>`_, `issue#36170 <http://tracker.ceph.com/issues/36170>`_, `pr#24573 <https://github.com/ceph/ceph/pull/24573>`_, David Zafman)
+* qa: add test that builds example librados programs (`issue#36228 <http://tracker.ceph.com/issues/36228>`_, `issue#15100 <http://tracker.ceph.com/issues/15100>`_, `pr#24537 <https://github.com/ceph/ceph/pull/24537>`_, Nathan Cutler)
+* qa/ceph-ansible: Specify stable-3.2 branch (`pr#25191 <https://github.com/ceph/ceph/pull/25191>`_, Brad Hubbard)
+* qa: extend timeout for SessionMap flush (`issue#36156 <http://tracker.ceph.com/issues/36156>`_, `pr#24438 <https://github.com/ceph/ceph/pull/24438>`_, Patrick Donnelly)
+* qa: fsstress workunit does not execute in parallel on same host without clobbering files (`issue#36278 <http://tracker.ceph.com/issues/36278>`_, `issue#24177 <http://tracker.ceph.com/issues/24177>`_, `issue#36323 <http://tracker.ceph.com/issues/36323>`_, `issue#36184 <http://tracker.ceph.com/issues/36184>`_, `issue#36165 <http://tracker.ceph.com/issues/36165>`_, `issue#36153 <http://tracker.ceph.com/issues/36153>`_, `pr#24408 <https://github.com/ceph/ceph/pull/24408>`_, Patrick Donnelly)
+* qa: increase rm timeout for workunit cleanup (`issue#36501 <http://tracker.ceph.com/issues/36501>`_, `issue#36365 <http://tracker.ceph.com/issues/36365>`_, `pr#24684 <https://github.com/ceph/ceph/pull/24684>`_, Patrick Donnelly)
+* qa: install dependencies for rbd_workunit_kernel_untar_build (`issue#35074 <http://tracker.ceph.com/issues/35074>`_, `issue#35077 <http://tracker.ceph.com/issues/35077>`_, `pr#24240 <https://github.com/ceph/ceph/pull/24240>`_, Ilya Dryomov)
+* qa: remove knfs site from future releases (`issue#36075 <http://tracker.ceph.com/issues/36075>`_, `issue#36102 <http://tracker.ceph.com/issues/36102>`_, `pr#24269 <https://github.com/ceph/ceph/pull/24269>`_, Yuri Weinstein)
+* qa/suites/rados/thrash-old-clients: exclude packages for hammer, jewel (`pr#25193 <https://github.com/ceph/ceph/pull/25193>`_, Neha Ojha)
+* qa/suites/rgw/verify/tasks/cls_rgw: test cls_rgw (`issue#25024 <http://tracker.ceph.com/issues/25024>`_, `pr#23197 <https://github.com/ceph/ceph/pull/23197>`_, Casey Bodley, Sage Weil)
+* qa/tasks/qemu: use unique clone directory to avoid race with workunit (`issue#36542 <http://tracker.ceph.com/issues/36542>`_, `issue#36569 <http://tracker.ceph.com/issues/36569>`_, `pr#24811 <https://github.com/ceph/ceph/pull/24811>`_, Jason Dillaman)
+* qa: test_recovery_pool tries asok on wrong node (`issue#24928 <http://tracker.ceph.com/issues/24928>`_, `issue#24858 <http://tracker.ceph.com/issues/24858>`_, `pr#23087 <https://github.com/ceph/ceph/pull/23087>`_, Patrick Donnelly)
+* qa: tolerate failed rank while waiting for state (`issue#36280 <http://tracker.ceph.com/issues/36280>`_, `issue#35828 <http://tracker.ceph.com/issues/35828>`_, `pr#24572 <https://github.com/ceph/ceph/pull/24572>`_, Patrick Donnelly)
+* qa/workunits: replace 'realpath' with 'readlink -f' in fsstress.sh (`issue#36409 <http://tracker.ceph.com/issues/36409>`_, `issue#36430 <http://tracker.ceph.com/issues/36430>`_, `issue#35538 <http://tracker.ceph.com/issues/35538>`_, `pr#24622 <https://github.com/ceph/ceph/pull/24622>`_, Ilya Dryomov, Jason Dillaman)
+* RADOS: probably missing clone location for async_recovery_targets (`issue#35964 <http://tracker.ceph.com/issues/35964>`_, `issue#35546 <http://tracker.ceph.com/issues/35546>`_, `pr#24345 <https://github.com/ceph/ceph/pull/24345>`_, xie xingguo)
+* mimic:rbd: fix error import when the input is a pipe (`issue#35705 <http://tracker.ceph.com/issues/35705>`_, `issue#34536 <http://tracker.ceph.com/issues/34536>`_, `pr#24002 <https://github.com/ceph/ceph/pull/24002>`_, songweibin)
+* [rbd-mirror] failed assertion when updating mirror status (`issue#36084 <http://tracker.ceph.com/issues/36084>`_, `issue#36120 <http://tracker.ceph.com/issues/36120>`_, `pr#24321 <https://github.com/ceph/ceph/pull/24321>`_, Jason Dillaman)
+* rbd: [rbd-mirror] forced promotion after killing remote cluster results in stuck state (`issue#36659 <http://tracker.ceph.com/issues/36659>`_, `issue#36693 <http://tracker.ceph.com/issues/36693>`_, `pr#24952 <https://github.com/ceph/ceph/pull/24952>`_, Jonathan Brielmaier, Jason Dillaman)
+* rbd: [rbd-mirror] periodic mirror status timer might fail to be scheduled (`issue#36500 <http://tracker.ceph.com/issues/36500>`_, `issue#36555 <http://tracker.ceph.com/issues/36555>`_, `pr#24916 <https://github.com/ceph/ceph/pull/24916>`_, Jason Dillaman)
+* rbd: rbd-nbd: do not ceph_abort() after print the usages (`issue#36660 <http://tracker.ceph.com/issues/36660>`_, `issue#36713 <http://tracker.ceph.com/issues/36713>`_, `pr#24988 <https://github.com/ceph/ceph/pull/24988>`_, Shiyang Ruan)
+* rbd: TokenBucketThrottle: use reference to m_blockers.front() and then update it (`issue#36529 <http://tracker.ceph.com/issues/36529>`_, `issue#36475 <http://tracker.ceph.com/issues/36475>`_, `pr#24915 <https://github.com/ceph/ceph/pull/24915>`_, Dongsheng Yang)
+* Revert "mimic: cephfs-journal-tool: enable purge_queue journal's event commands" (`issue#36346 <http://tracker.ceph.com/issues/36346>`_, `issue#24604 <http://tracker.ceph.com/issues/24604>`_, `pr#24485 <https://github.com/ceph/ceph/pull/24485>`_, Xuehan Xu, "Yan, Zheng")
+* rgw: abort_bucket_multiparts() ignores individual NoSuchUpload errors (`issue#36129 <http://tracker.ceph.com/issues/36129>`_, `issue#35986 <http://tracker.ceph.com/issues/35986>`_, `pr#24388 <https://github.com/ceph/ceph/pull/24388>`_, Casey Bodley)
+* rgw-admin: reshard add can add a non existant bucket (`issue#36449 <http://tracker.ceph.com/issues/36449>`_, `issue#36756 <http://tracker.ceph.com/issues/36756>`_, `pr#25087 <https://github.com/ceph/ceph/pull/25087>`_, Jonathan Brielmaier, Abhishek Lekshmanan)
+* rgw: async sync_object and remove_object does not access coroutine me… (`issue#36138 <http://tracker.ceph.com/issues/36138>`_, `issue#35905 <http://tracker.ceph.com/issues/35905>`_, `pr#24417 <https://github.com/ceph/ceph/pull/24417>`_, Tianshan Qu)
+* rgw/beast: drop privileges after binding ports (`issue#36041 <http://tracker.ceph.com/issues/36041>`_, `pr#24436 <https://github.com/ceph/ceph/pull/24436>`_, Paul Emmerich)
+* rgw: beast frontend fails to parse ipv6 endpoints (`issue#36662 <http://tracker.ceph.com/issues/36662>`_, `issue#36734 <http://tracker.ceph.com/issues/36734>`_, `pr#25079 <https://github.com/ceph/ceph/pull/25079>`_, Jonathan Brielmaier, Casey Bodley)
+* rgw: cls_user_remove_bucket does not write the modified cls_user_stats (`issue#36496 <http://tracker.ceph.com/issues/36496>`_, `issue#36533 <http://tracker.ceph.com/issues/36533>`_, `pr#24910 <https://github.com/ceph/ceph/pull/24910>`_, Casey Bodley)
+* rgw: default quota not set in radosgw for Openstack users (`issue#24595 <http://tracker.ceph.com/issues/24595>`_, `issue#36223 <http://tracker.ceph.com/issues/36223>`_, `pr#24907 <https://github.com/ceph/ceph/pull/24907>`_, Casey Bodley)
+* mimic:rgw: fix chunked-encoding for chunks >1MiB (`issue#36125 <http://tracker.ceph.com/issues/36125>`_, `issue#35990 <http://tracker.ceph.com/issues/35990>`_, `pr#24363 <https://github.com/ceph/ceph/pull/24363>`_, Robin H. Johnson)
+* rgw: fix deadlock on RGWIndexCompletionManager::stop (`issue#26949 <http://tracker.ceph.com/issues/26949>`_, `issue#35710 <http://tracker.ceph.com/issues/35710>`_, `pr#24101 <https://github.com/ceph/ceph/pull/24101>`_, Yao Zongyou)
+* mimic:rgw: fix leak of curl handle on shutdown (`issue#35715 <http://tracker.ceph.com/issues/35715>`_, `issue#36213 <http://tracker.ceph.com/issues/36213>`_, `pr#24518 <https://github.com/ceph/ceph/pull/24518>`_, Casey Bodley)
+* mimic:rgw: list bucket can not show the object uploaded by RGWPostObj when enable bucket versioning (`pr#24571 <https://github.com/ceph/ceph/pull/24571>`_, yuliyang)
+* rgw: radosgw-admin user stats are incorrect when dynamic re-sharding is enabled (`issue#36535 <http://tracker.ceph.com/issues/36535>`_, `pr#24911 <https://github.com/ceph/ceph/pull/24911>`_, Casey Bodley)
+* rgw: raise debug level on redundant data sync error messages (`issue#35830 <http://tracker.ceph.com/issues/35830>`_, `issue#36140 <http://tracker.ceph.com/issues/36140>`_, `pr#24418 <https://github.com/ceph/ceph/pull/24418>`_, Casey Bodley)
+* rgw: raise default rgw_curl_low_speed_time to 300 seconds (`issue#35708 <http://tracker.ceph.com/issues/35708>`_, `issue#27989 <http://tracker.ceph.com/issues/27989>`_, `pr#24071 <https://github.com/ceph/ceph/pull/24071>`_, Casey Bodley)
+* rgw: renew resharding locks to prevent expiration (`issue#36687 <http://tracker.ceph.com/issues/36687>`_, `issue#27219 <http://tracker.ceph.com/issues/27219>`_, `issue#34307 <http://tracker.ceph.com/issues/34307>`_, `pr#24899 <https://github.com/ceph/ceph/pull/24899>`_, Orit Wasserman, J. Eric Ivancich)
+* rgw: resharding produces invalid values of bucket stats (`issue#36290 <http://tracker.ceph.com/issues/36290>`_, `issue#36381 <http://tracker.ceph.com/issues/36381>`_, `pr#24526 <https://github.com/ceph/ceph/pull/24526>`_, Abhishek Lekshmanan)
+* mimic:rgw: return x-amz-version-id: null when delete obj in versioning (`issue#35814 <http://tracker.ceph.com/issues/35814>`_, `pr#24189 <https://github.com/ceph/ceph/pull/24189>`_, yuliyang)
+* rgw: RGWAsyncGetBucketInstanceInfo does not access coroutine memory (`issue#36211 <http://tracker.ceph.com/issues/36211>`_, `issue#35812 <http://tracker.ceph.com/issues/35812>`_, `pr#24516 <https://github.com/ceph/ceph/pull/24516>`_, Casey Bodley)
+* rgw: set default objecter_inflight_ops = 24576 (`issue#36571 <http://tracker.ceph.com/issues/36571>`_, `issue#25109 <http://tracker.ceph.com/issues/25109>`_, `pr#24860 <https://github.com/ceph/ceph/pull/24860>`_, Jonathan Brielmaier, Matt Benjamin)
+* rgw: support server-side encryption when SSL is terminated in a proxy (`issue#36645 <http://tracker.ceph.com/issues/36645>`_, `issue#27221 <http://tracker.ceph.com/issues/27221>`_, `pr#24931 <https://github.com/ceph/ceph/pull/24931>`_, Jonathan Brielmaier, Casey Bodley)
+* rgw: use-after-free from RGWRadosGetOmapKeysCR::~RGWRadosGetOmapKeysCR (`issue#21154 <http://tracker.ceph.com/issues/21154>`_, `issue#36537 <http://tracker.ceph.com/issues/36537>`_, `issue#36539 <http://tracker.ceph.com/issues/36539>`_, `pr#24912 <https://github.com/ceph/ceph/pull/24912>`_, Casey Bodley, Sage Weil)
+* rpm: use updated gperftools (`issue#36508 <http://tracker.ceph.com/issues/36508>`_, `issue#35969 <http://tracker.ceph.com/issues/35969>`_, `pr#24260 <https://github.com/ceph/ceph/pull/24260>`_, Brad Hubbard, Kefu Chai)
+* segv in BlueStore::OldExtent::create (`issue#36592 <http://tracker.ceph.com/issues/36592>`_, `issue#36526 <http://tracker.ceph.com/issues/36526>`_, `pr#24745 <https://github.com/ceph/ceph/pull/24745>`_, Sage Weil)
+* test/librbd: not valid to have different parents between image snapshots (`issue#36117 <http://tracker.ceph.com/issues/36117>`_, `pr#24244 <https://github.com/ceph/ceph/pull/24244>`_, Jason Dillaman)
+* [test] periodic seg faults within unittest_librbd (`issue#36220 <http://tracker.ceph.com/issues/36220>`_, `issue#36238 <http://tracker.ceph.com/issues/36238>`_, `pr#24711 <https://github.com/ceph/ceph/pull/24711>`_, Jason Dillaman)
+* test/rbd_mirror: race in WaitingOnLeaderReleaseLeader (`issue#36236 <http://tracker.ceph.com/issues/36236>`_, `issue#36276 <http://tracker.ceph.com/issues/36276>`_, `pr#24551 <https://github.com/ceph/ceph/pull/24551>`_, Mykola Golub)
+* tests: ceph-admin-commands.sh workunit does not log what it's doing (`issue#37153 <http://tracker.ceph.com/issues/37153>`_, `issue#37089 <http://tracker.ceph.com/issues/37089>`_, `pr#25085 <https://github.com/ceph/ceph/pull/25085>`_, Nathan Cutler)
+* tests: librados api aio tests race condition (`issue#24587 <http://tracker.ceph.com/issues/24587>`_, `issue#36647 <http://tracker.ceph.com/issues/36647>`_, `pr#25027 <https://github.com/ceph/ceph/pull/25027>`_, Josh Durgin)
+* tests: make readable.sh fail if it doesn't run anything (`pr#25050 <https://github.com/ceph/ceph/pull/25050>`_, Greg Farnum)
+* tests: rbd: move OpenStack devstack test to rocky release (`issue#36410 <http://tracker.ceph.com/issues/36410>`_, `issue#36428 <http://tracker.ceph.com/issues/36428>`_, `pr#24913 <https://github.com/ceph/ceph/pull/24913>`_, Jason Dillaman)
+* tests: unittest_rbd_mirror: TestMockImageMap.AddInstancePingPongImageTest: Value of: it != peer_ack_ctxs->end() (`issue#36683 <http://tracker.ceph.com/issues/36683>`_, `issue#36689 <http://tracker.ceph.com/issues/36689>`_, `pr#24946 <https://github.com/ceph/ceph/pull/24946>`_, Mykola Golub, Jonathan Brielmaier)
+* tests: use timeout for fs asok operations (`issue#36335 <http://tracker.ceph.com/issues/36335>`_, `issue#36503 <http://tracker.ceph.com/issues/36503>`_, `pr#25332 <https://github.com/ceph/ceph/pull/25332>`_, Patrick Donnelly)
+* tests: /usr/bin/ld: cannot find -lradospp in rados mimic (`issue#37396 <http://tracker.ceph.com/issues/37396>`_, `pr#25285 <https://github.com/ceph/ceph/pull/25285>`_, Nathan Cutler)
+* test: Use a grep pattern that works across releases (`issue#35845 <http://tracker.ceph.com/issues/35845>`_, `issue#35909 <http://tracker.ceph.com/issues/35909>`_, `pr#24017 <https://github.com/ceph/ceph/pull/24017>`_, David Zafman)
+* tools: ceph-objectstore-tool: Allow target level as first positional … (`issue#35846 <http://tracker.ceph.com/issues/35846>`_, `issue#35992 <http://tracker.ceph.com/issues/35992>`_, `pr#24116 <https://github.com/ceph/ceph/pull/24116>`_, David Zafman)
+
+v13.2.2 Mimic
+=============
+
+This is the second bugfix release of the Mimic v13.2.x long term stable release
+series. This release contains many fixes across all components of Ceph.
+We recommend that all users upgrade.
+
+* This version contains an upgrade bug, http://tracker.ceph.com/issues/36686,
+  due to which upgrading during recovery/backfill can cause OSDs to fail. This
+  bug can be worked around, either by restarting all the OSDs after the upgrade,
+  or by upgrading when all PGs are in "active+clean" state.
+
+  If you have successfully upgraded to 13.2.2, this issue should not impact
+  you. Going forward, we are working on a clean upgrade path for this feature.
+
+Changelog
+---------
+
+* build/ops: Boost system library is no longer required to compile and link example librados program (`issue#25073 <http://tracker.ceph.com/issues/25073>`_, `issue#25054 <http://tracker.ceph.com/issues/25054>`_, `pr#23201 <https://github.com/ceph/ceph/pull/23201>`_, Nathan Cutler)
+* build/ops: debian/rules: fix ceph-mgr .pyc files left behind (`issue#27059 <http://tracker.ceph.com/issues/27059>`_, `issue#26883 <http://tracker.ceph.com/issues/26883>`_, `pr#23831 <https://github.com/ceph/ceph/pull/23831>`_, Dan Mick)
+* build/ops: mimic 13.2.0 doesn't build in Fedora rawhide (`issue#24449 <http://tracker.ceph.com/issues/24449>`_, `issue#24905 <http://tracker.ceph.com/issues/24905>`_, `pr#23885 <https://github.com/ceph/ceph/pull/23885>`_, Kefu Chai)
+* ceph-disk: compatibility fix for python 3 (`pr#24008 <https://github.com/ceph/ceph/pull/24008>`_, Tim Serong)
+* ceph-disk: return a list instead of an iterator (`pr#23392 <https://github.com/ceph/ceph/pull/23392>`_, Alexander Graul)
+* cephfs-journal-tool: enable purge_queue journal's event commands (`issue#24604 <http://tracker.ceph.com/issues/24604>`_, `issue#26989 <http://tracker.ceph.com/issues/26989>`_, `pr#23818 <https://github.com/ceph/ceph/pull/23818>`_, Xuehan Xu)
+* ceph tell osd.x bench writes resulting JSON to stderr instead of stdout (`issue#35942 <http://tracker.ceph.com/issues/35942>`_, `issue#24022 <http://tracker.ceph.com/issues/24022>`_, `pr#24041 <https://github.com/ceph/ceph/pull/24041>`_, Коренберг Маркr, John Spray, Kefu Chai)
+* ceph-volume add a __release__ string, to help version-conditional calls (`issue#25169 <http://tracker.ceph.com/issues/25169>`_, `pr#23333 <https://github.com/ceph/ceph/pull/23333>`_, Alfredo Deza)
+* ceph-volume: adds test for `ceph-volume lvm list /dev/sda` (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `issue#24957 <http://tracker.ceph.com/issues/24957>`_, `pr#23349 <https://github.com/ceph/ceph/pull/23349>`_, Andrew Schoen)
+* ceph-volume: an OSD ID must exist and be destroyed before reuse (`pr#23101 <https://github.com/ceph/ceph/pull/23101>`_, Andrew Schoen, Ron Allred)
+* ceph-volume:  batch: allow journal+block.db sizing on the CLI (`issue#36088 <http://tracker.ceph.com/issues/36088>`_, `pr#24208 <https://github.com/ceph/ceph/pull/24208>`_, Alfredo Deza)
+* ceph-volume batch: allow --osds-per-device, default it to 1 (`issue#35913 <http://tracker.ceph.com/issues/35913>`_, `pr#24079 <https://github.com/ceph/ceph/pull/24079>`_, Alfredo Deza)
+* ceph-volume batch carve out lvs for bluestore (`issue#34535 <http://tracker.ceph.com/issues/34535>`_, `pr#24074 <https://github.com/ceph/ceph/pull/24074>`_, Alfredo Deza)
+* ceph-volume batch command (`pr#23777 <https://github.com/ceph/ceph/pull/23777>`_, Alfredo Deza)
+* ceph-volume: batch tests for mixed-type of devices (`issue#35535 <http://tracker.ceph.com/issues/35535>`_, `issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#23966 <https://github.com/ceph/ceph/pull/23966>`_, Alfredo Deza)
+* ceph_volume_client: allow atomic update of RADOS objects (`issue#24173 <http://tracker.ceph.com/issues/24173>`_, `issue#24863 <http://tracker.ceph.com/issues/24863>`_, `pr#23878 <https://github.com/ceph/ceph/pull/23878>`_, Rishabh Dave)
+* CephVolumeClient: delay required after adding data pool to MDSMap (`issue#25206 <http://tracker.ceph.com/issues/25206>`_, `pr#23725 <https://github.com/ceph/ceph/pull/23725>`_, Patrick Donnelly)
+* ceph-volume: do not use stdin in luminous (`issue#25173 <http://tracker.ceph.com/issues/25173>`_, `pr#23368 <https://github.com/ceph/ceph/pull/23368>`_, Alfredo Deza)
+* ceph-volume: earlier detection for --journal and --filestore flag requirements (`issue#24794 <http://tracker.ceph.com/issues/24794>`_, `pr#24205 <https://github.com/ceph/ceph/pull/24205>`_, Alfredo Deza)
+* ceph-volume enable the ceph-osd during lvm activation (`issue#24152 <http://tracker.ceph.com/issues/24152>`_, `pr#23393 <https://github.com/ceph/ceph/pull/23393>`_, Dan van der Ster, Alfredo Deza)
+* ceph-volume expand auto engine for multiple devices on filestore (`pr#23807 <https://github.com/ceph/ceph/pull/23807>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: expand auto engine for single type devices on filestore (`pr#23786 <https://github.com/ceph/ceph/pull/23786>`_, Alfredo Deza)
+* ceph-volume fix zap not working with LVs (`issue#35970 <http://tracker.ceph.com/issues/35970>`_, `pr#24081 <https://github.com/ceph/ceph/pull/24081>`_, Alfredo Deza)
+* ceph-volume lvm.activate conditional mon-config on prime-osd-dir (`issue#25216 <http://tracker.ceph.com/issues/25216>`_, `pr#23400 <https://github.com/ceph/ceph/pull/23400>`_, Alfredo Deza)
+* ceph-volume: `lvm batch` allow extra flags (like dmcrypt) for bluestore (`pr#23780 <https://github.com/ceph/ceph/pull/23780>`_, Alfredo Deza)
+* ceph-volume: `lvm batch` documentation and man page updates (`pr#23756 <https://github.com/ceph/ceph/pull/23756>`_, Alfredo Deza)
+* ceph-volume lvm.batch remove non-existent sys_api property (`issue#34310 <http://tracker.ceph.com/issues/34310>`_, `pr#23810 <https://github.com/ceph/ceph/pull/23810>`_, Alfredo Deza)
+* ceph-volume lvm.listing only include devices if they exist (`issue#24952 <http://tracker.ceph.com/issues/24952>`_, `pr#23149 <https://github.com/ceph/ceph/pull/23149>`_, Alfredo Deza)
+* ceph-volume: process.call with stdin in Python 3 fix (`issue#24993 <http://tracker.ceph.com/issues/24993>`_, `pr#23239 <https://github.com/ceph/ceph/pull/23239>`_, Alfredo Deza)
+* ceph-volume: PVolumes.get() should return one PV when using name or uuid (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `pr#23327 <https://github.com/ceph/ceph/pull/23327>`_, Andrew Schoen)
+* ceph-volume: refuse to zap mapper devices (`issue#24504 <http://tracker.ceph.com/issues/24504>`_, `pr#22965 <https://github.com/ceph/ceph/pull/22965>`_, Andrew Schoen)
+* ceph-volume: Restore SELinux context (`pr#23295 <https://github.com/ceph/ceph/pull/23295>`_, Boris Ranto)
+* ceph-volume: run tests without waiting on ceph repos (`pr#23806 <https://github.com/ceph/ceph/pull/23806>`_, Andrew Schoen)
+* ceph-volume tests/functional add mgrs daemons to lvm tests (`pr#23784 <https://github.com/ceph/ceph/pull/23784>`_, Alfredo Deza)
+* ceph-volume: tests.functional inherit SSH_ARGS from ansible (`pr#23812 <https://github.com/ceph/ceph/pull/23812>`_, Alfredo Deza)
+* ceph-volume: update batch documentation to explain filestore strategies (`issue#34309 <http://tracker.ceph.com/issues/34309>`_, `pr#23826 <https://github.com/ceph/ceph/pull/23826>`_, Alfredo Deza)
+* ceph-volume: update version of ansible to 2.6.x for simple tests (`pr#23269 <https://github.com/ceph/ceph/pull/23269>`_, Andrew Schoen)
+* client: add inst to asok status output (`issue#24724 <http://tracker.ceph.com/issues/24724>`_, `issue#24931 <http://tracker.ceph.com/issues/24931>`_, `pr#23109 <https://github.com/ceph/ceph/pull/23109>`_, Patrick Donnelly)
+* client: check for unmounted condition before printing debug output (`issue#25213 <http://tracker.ceph.com/issues/25213>`_, `issue#26914 <http://tracker.ceph.com/issues/26914>`_, `pr#23603 <https://github.com/ceph/ceph/pull/23603>`_, Jeff Layton)
+* client: requests that do name lookup may be sent to wrong mds (`issue#26984 <http://tracker.ceph.com/issues/26984>`_, `issue#26860 <http://tracker.ceph.com/issues/26860>`_, `pr#23700 <https://github.com/ceph/ceph/pull/23700>`_, "Yan, Zheng")
+* cls/rgw: add rgw_usage_log_entry type to ceph-dencoder (`issue#35070 <http://tracker.ceph.com/issues/35070>`_, `pr#23857 <https://github.com/ceph/ceph/pull/23857>`_, Vaibhav Bhembre)
+* common: check completion condition before waiting (`issue#25007 <http://tracker.ceph.com/issues/25007>`_, `issue#25222 <http://tracker.ceph.com/issues/25222>`_, `pr#23435 <https://github.com/ceph/ceph/pull/23435>`_, Patrick Donnelly)
+* core: deep scrub cannot find the bitrot if the object is cached (`issue#35068 <http://tracker.ceph.com/issues/35068>`_, `pr#23873 <https://github.com/ceph/ceph/pull/23873>`_, Adam C. Emerson, Xiaoguang Wang)
+* core: Fix 25085 and 24949 (`pr#23272 <https://github.com/ceph/ceph/pull/23272>`_, David Zafman)
+* core: force-create-pg broken (`issue#34532 <http://tracker.ceph.com/issues/34532>`_, `issue#26940 <http://tracker.ceph.com/issues/26940>`_, `pr#23872 <https://github.com/ceph/ceph/pull/23872>`_, Sage Weil)
+* core: Limit pg log length during recovery/backfill so that we don't run out of memory (`issue#21416 <http://tracker.ceph.com/issues/21416>`_, `pr#23403 <https://github.com/ceph/ceph/pull/23403>`_, Neha Ojha)
+* doc: broken bash example in bluestore migration (`issue#35078 <http://tracker.ceph.com/issues/35078>`_, `pr#23854 <https://github.com/ceph/ceph/pull/23854>`_, Alfredo Deza)
+* doc: Fix broken urls (`issue#25185 <http://tracker.ceph.com/issues/25185>`_, `issue#26916 <http://tracker.ceph.com/issues/26916>`_, `pr#23607 <https://github.com/ceph/ceph/pull/23607>`_, Jos Collin)
+* doc: http://docs.ceph.com/docs/mimic/rados/operations/pg-states/ (`issue#25055 <http://tracker.ceph.com/issues/25055>`_, `pr#23163 <https://github.com/ceph/ceph/pull/23163>`_, Jan Fajerski, Nathan Cutler)
+* docs: radosgw: ldap-auth: fixed option name 'rgw_ldap_searchfilter' (`issue#32129 <http://tracker.ceph.com/issues/32129>`_, `pr#23956 <https://github.com/ceph/ceph/pull/23956>`_, Konstantin Shalygin)
+* filestore: add pgid in filestore pg dir split log message (`issue#25225 <http://tracker.ceph.com/issues/25225>`_, `pr#23453 <https://github.com/ceph/ceph/pull/23453>`_, Vikhyat Umrao)
+* kv: MergeOperator name() returns string, and caller calls c_str() on the temporary (`issue#26907 <http://tracker.ceph.com/issues/26907>`_, `issue#26875 <http://tracker.ceph.com/issues/26875>`_, `pr#23865 <https://github.com/ceph/ceph/pull/23865>`_, Sage Weil)
+* libradosstriper conditional compile (`issue#27213 <http://tracker.ceph.com/issues/27213>`_, `pr#23869 <https://github.com/ceph/ceph/pull/23869>`_, Kefu Chai, Jesse Williamson)
+* librbd: deep-copy should not write to objects that cannot exist (`issue#25000 <http://tracker.ceph.com/issues/25000>`_, `issue#25083 <http://tracker.ceph.com/issues/25083>`_, `pr#23358 <https://github.com/ceph/ceph/pull/23358>`_, Jason Dillaman)
+* librbd: validate data pool for self-managed snapshot support (`issue#24945 <http://tracker.ceph.com/issues/24945>`_, `pr#23560 <https://github.com/ceph/ceph/pull/23560>`_, Mykola Golub)
+* link against libstdc++ statically (`issue#26880 <http://tracker.ceph.com/issues/26880>`_, `issue#25209 <http://tracker.ceph.com/issues/25209>`_, `pr#23490 <https://github.com/ceph/ceph/pull/23490>`_, Kefu Chai)
+* mds: avoid using g_conf->get_val<...>(...) in hot path (`issue#24820 <http://tracker.ceph.com/issues/24820>`_, `pr#23407 <https://github.com/ceph/ceph/pull/23407>`_, "Yan, Zheng")
+* mds: calculate load by checking self CPU usage (`issue#26834 <http://tracker.ceph.com/issues/26834>`_, `issue#26888 <http://tracker.ceph.com/issues/26888>`_, `pr#23503 <https://github.com/ceph/ceph/pull/23503>`_, "Yan, Zheng")
+* mds: crash when dumping ops in flight (`issue#26894 <http://tracker.ceph.com/issues/26894>`_, `issue#26982 <http://tracker.ceph.com/issues/26982>`_, `pr#23672 <https://github.com/ceph/ceph/pull/23672>`_, "Yan, Zheng")
+* mds: dump recent events on respawn (`issue#25040 <http://tracker.ceph.com/issues/25040>`_, `pr#23275 <https://github.com/ceph/ceph/pull/23275>`_, Patrick Donnelly)
+* mds: explain delayed client_request due to subtree migration (`issue#26988 <http://tracker.ceph.com/issues/26988>`_, `issue#24840 <http://tracker.ceph.com/issues/24840>`_, `pr#23792 <https://github.com/ceph/ceph/pull/23792>`_, Yan, Zheng, "Yan, Zheng")
+* mds: handle discontinuous mdsmap (`issue#24856 <http://tracker.ceph.com/issues/24856>`_, `pr#23180 <https://github.com/ceph/ceph/pull/23180>`_, "Yan, Zheng")
+* mds: health warning for slow metadata IO (`issue#24879 <http://tracker.ceph.com/issues/24879>`_, `issue#25045 <http://tracker.ceph.com/issues/25045>`_, `pr#23343 <https://github.com/ceph/ceph/pull/23343>`_, "Yan, Zheng")
+* mds: increase debug level for dropped client cap msg (`issue#25042 <http://tracker.ceph.com/issues/25042>`_, `pr#23309 <https://github.com/ceph/ceph/pull/23309>`_, Patrick Donnelly)
+* mds: introduce cephfs' own feature bits (`issue#14456 <http://tracker.ceph.com/issues/14456>`_, `issue#24914 <http://tracker.ceph.com/issues/24914>`_, `pr#23105 <https://github.com/ceph/ceph/pull/23105>`_, Yan, Zheng, "Yan, Zheng", Patrick Donnelly)
+* mds: mark beacons as high priority (`issue#26905 <http://tracker.ceph.com/issues/26905>`_, `issue#26899 <http://tracker.ceph.com/issues/26899>`_, `pr#23565 <https://github.com/ceph/ceph/pull/23565>`_, Patrick Donnelly)
+* mds: MDBalancer::try_rebalance() may stop prematurely (`issue#32086 <http://tracker.ceph.com/issues/32086>`_, `issue#26973 <http://tracker.ceph.com/issues/26973>`_, `pr#23883 <https://github.com/ceph/ceph/pull/23883>`_, "Yan, Zheng")
+* MDSMonitor: note ignored beacons/map changes at higher debug level (`issue#26898 <http://tracker.ceph.com/issues/26898>`_, `issue#26929 <http://tracker.ceph.com/issues/26929>`_, `pr#23704 <https://github.com/ceph/ceph/pull/23704>`_, Patrick Donnelly)
+* mds,osd,mon,msg: use intrusive_ptr for holding Connection::priv (`issue#20924 <http://tracker.ceph.com/issues/20924>`_, `pr#22339 <https://github.com/ceph/ceph/pull/22339>`_, "Yan, Zheng", Kefu Chai)
+* mds: print mdsmap processed at low debug level (`issue#25035 <http://tracker.ceph.com/issues/25035>`_, `pr#23196 <https://github.com/ceph/ceph/pull/23196>`_, Patrick Donnelly)
+* mds: scrub doesn't always return JSON results (`issue#23958 <http://tracker.ceph.com/issues/23958>`_, `issue#25037 <http://tracker.ceph.com/issues/25037>`_, `pr#23225 <https://github.com/ceph/ceph/pull/23225>`_, Venky Shankar)
+* mds: use fast dispatch to handle MDSBeacon (`issue#23519 <http://tracker.ceph.com/issues/23519>`_, `issue#26923 <http://tracker.ceph.com/issues/26923>`_, `pr#23703 <https://github.com/ceph/ceph/pull/23703>`_, "Yan, Zheng")
+* mgr balancer does not save optimized plan but latest (`issue#32082 <http://tracker.ceph.com/issues/32082>`_, `issue#27000 <http://tracker.ceph.com/issues/27000>`_, `pr#23782 <https://github.com/ceph/ceph/pull/23782>`_, Stefan Priebe)
+* mgr: "balancer execute" only requires read permissions (`issue#26912 <http://tracker.ceph.com/issues/26912>`_, `issue#25345 <http://tracker.ceph.com/issues/25345>`_, `pr#23583 <https://github.com/ceph/ceph/pull/23583>`_, John Spray)
+* mgrc: enable disabling stats via mgr_stats_threshold (`issue#25197 <http://tracker.ceph.com/issues/25197>`_, `issue#26837 <http://tracker.ceph.com/issues/26837>`_, `pr#23463 <https://github.com/ceph/ceph/pull/23463>`_, John Spray)
+* mgr/dashboard: Display RGW user/bucket quota max size in human readable form (`issue#35706 <http://tracker.ceph.com/issues/35706>`_, `pr#24047 <https://github.com/ceph/ceph/pull/24047>`_, Volker Theile)
+* mgr/dashboard: Escape regex pattern in DeletionModalComponent (`issue#24902 <http://tracker.ceph.com/issues/24902>`_, `issue#26920 <http://tracker.ceph.com/issues/26920>`_, `pr#23669 <https://github.com/ceph/ceph/pull/23669>`_, Tiago Melo)
+* mgr/dashboard: Prevent RGW API user deletion (`pr#22670 <https://github.com/ceph/ceph/pull/22670>`_, Volker Theile)
+* mgr/dashboard: RestClient can't handle ProtocolError exceptions (`pr#23875 <https://github.com/ceph/ceph/pull/23875>`_, Volker Theile)
+* mgr/dashboard: RGW is not working if an URL prefix is defined (`pr#23203 <https://github.com/ceph/ceph/pull/23203>`_, Volker Theile)
+* mgr/dashboard: URL prefix is not working (`issue#25120 <http://tracker.ceph.com/issues/25120>`_, `pr#23874 <https://github.com/ceph/ceph/pull/23874>`_, Ricardo Marques)
+* mgr: Ignore daemon if no metadata was returned (`pr#23356 <https://github.com/ceph/ceph/pull/23356>`_, Wido den Hollander)
+* mgr/MgrClient: Protect daemon_health_metrics (`issue#23352 <http://tracker.ceph.com/issues/23352>`_, `pr#23458 <https://github.com/ceph/ceph/pull/23458>`_, Kjetil Joergensen, Brad Hubbard)
+* mgr: Sync the prometheus module (`pr#23215 <https://github.com/ceph/ceph/pull/23215>`_, Boris Ranto)
+* mon: add purge-new (`pr#23259 <https://github.com/ceph/ceph/pull/23259>`_, Sage Weil)
+* mon: Automatically set expected_num_objects for new pools with >=100 PGs per OSD (`issue#24687 <http://tracker.ceph.com/issues/24687>`_, `issue#25144 <http://tracker.ceph.com/issues/25144>`_, `pr#23860 <https://github.com/ceph/ceph/pull/23860>`_, Douglas Fuller)
+* multisite: intermittent failures in test_bucket_sync_disable_enable (`issue#26895 <http://tracker.ceph.com/issues/26895>`_, `issue#26980 <http://tracker.ceph.com/issues/26980>`_, `pr#23856 <https://github.com/ceph/ceph/pull/23856>`_, Casey Bodley)
+* multisite: object metadata operations are skipped by sync (`issue#24367 <http://tracker.ceph.com/issues/24367>`_, `issue#24986 <http://tracker.ceph.com/issues/24986>`_, `pr#23172 <https://github.com/ceph/ceph/pull/23172>`_, Casey Bodley)
+* object errors found in be_select_auth_object() aren't logged the same (`issue#32108 <http://tracker.ceph.com/issues/32108>`_, `issue#25108 <http://tracker.ceph.com/issues/25108>`_, `pr#23870 <https://github.com/ceph/ceph/pull/23870>`_, David Zafman)
+* os/bluestore: bluestore_buffer_hit_bytes perf counter doesn't reset (`pr#23772 <https://github.com/ceph/ceph/pull/23772>`_, Igor Fedotov)
+* os/bluestore/BlueStore.cc: 1025: FAILED assert(buffer_bytes >= b->length) from ObjectStore/StoreTest.ColSplitTest2/2 (`issue#24439 <http://tracker.ceph.com/issues/24439>`_, `issue#26944 <http://tracker.ceph.com/issues/26944>`_, `pr#23748 <https://github.com/ceph/ceph/pull/23748>`_, Sage Weil)
+* os/bluestore: fix assertion in StupidAllocator::get_fragmentation (`pr#23676 <https://github.com/ceph/ceph/pull/23676>`_, Igor Fedotov)
+* osd: do_sparse_read(): Verify checksum earlier so we will try to repair (`issue#24875 <http://tracker.ceph.com/issues/24875>`_, `pr#23378 <https://github.com/ceph/ceph/pull/23378>`_, David Zafman)
+* osd,mon: increase mon_max_pg_per_osd to 300 (`issue#25176 <http://tracker.ceph.com/issues/25176>`_, `pr#23861 <https://github.com/ceph/ceph/pull/23861>`_, Neha Ojha)
+* osd/OSDMap: CRUSH_TUNABLES5 added in jewel, not kraken (`issue#25057 <http://tracker.ceph.com/issues/25057>`_, `issue#25101 <http://tracker.ceph.com/issues/25101>`_, `pr#23226 <https://github.com/ceph/ceph/pull/23226>`_, Sage Weil)
+* osd/PrimaryLogPG: avoid dereferencing invalid complete_to (`pr#23951 <https://github.com/ceph/ceph/pull/23951>`_, xie xingguo)
+* osd: segv in OSDMap::calc_pg_upmaps from balancer (`issue#22056 <http://tracker.ceph.com/issues/22056>`_, `issue#26933 <http://tracker.ceph.com/issues/26933>`_, `pr#23888 <https://github.com/ceph/ceph/pull/23888>`_, Brad Hubbard)
+* qa: cfuse_workunit_kernel_untar_build fails on Ubuntu 18.04 (`issue#26956 <http://tracker.ceph.com/issues/26956>`_, `issue#26967 <http://tracker.ceph.com/issues/26967>`_, `issue#24679 <http://tracker.ceph.com/issues/24679>`_, `pr#23769 <https://github.com/ceph/ceph/pull/23769>`_, Patrick Donnelly)
+* qa: fix ceph-disk suite and add coverage for ceph-detect-init (`pr#23337 <https://github.com/ceph/ceph/pull/23337>`_, Nathan Cutler)
+* qa/rgw: patch keystone requirements.txt (`issue#26946 <http://tracker.ceph.com/issues/26946>`_, `issue#23659 <http://tracker.ceph.com/issues/23659>`_, `pr#23771 <https://github.com/ceph/ceph/pull/23771>`_, Casey Bodley)
+* qa/suites/rados: move valgrind test to singleton-flat (`issue#24992 <http://tracker.ceph.com/issues/24992>`_, `pr#23744 <https://github.com/ceph/ceph/pull/23744>`_, Sage Weil)
+* qa/tasks: s3a fix mirror (`pr#24038 <https://github.com/ceph/ceph/pull/24038>`_, Vasu Kulkarni)
+* qa/tests:  added OBJECT_MISPLACED to the whitelist (`pr#23301 <https://github.com/ceph/ceph/pull/23301>`_, Yuri Weinstein)
+* qa/tests: added v13.2.1 to the mix (`pr#23218 <https://github.com/ceph/ceph/pull/23218>`_, Yuri Weinstein)
+* qa/tests: update ansible version to 2.5 (`pr#24091 <https://github.com/ceph/ceph/pull/24091>`_, Yuri Weinstein)
+* rados: not all exceptions accept keyargs (`issue#25178 <http://tracker.ceph.com/issues/25178>`_, `issue#24033 <http://tracker.ceph.com/issues/24033>`_, `pr#23335 <https://github.com/ceph/ceph/pull/23335>`_, Rishabh Dave)
+* rados python bindings use prval from stack (`issue#25204 <http://tracker.ceph.com/issues/25204>`_, `issue#25175 <http://tracker.ceph.com/issues/25175>`_, `pr#23863 <https://github.com/ceph/ceph/pull/23863>`_, Sage Weil)
+* rbd: improved trash snapshot namespace handling (`issue#25121 <http://tracker.ceph.com/issues/25121>`_, `issue#23398 <http://tracker.ceph.com/issues/23398>`_, `issue#25114 <http://tracker.ceph.com/issues/25114>`_, `pr#23559 <https://github.com/ceph/ceph/pull/23559>`_, Mykola Golub, Jason Dillaman)
+* rgw: add curl_low_speed_limit and curl_low_speed_time config to avoid (`issue#25021 <http://tracker.ceph.com/issues/25021>`_, `pr#23173 <https://github.com/ceph/ceph/pull/23173>`_, Mark Kogan, Zhang Shaowen)
+* rgw: change default rgw_thread_pool_size to 512 (`issue#25214 <http://tracker.ceph.com/issues/25214>`_, `issue#25088 <http://tracker.ceph.com/issues/25088>`_, `issue#25218 <http://tracker.ceph.com/issues/25218>`_, `issue#24544 <http://tracker.ceph.com/issues/24544>`_, `pr#23383 <https://github.com/ceph/ceph/pull/23383>`_, Douglas Fuller, Casey Bodley)
+* rgw: civetweb fails on urls with control characters (`issue#26849 <http://tracker.ceph.com/issues/26849>`_, `issue#24158 <http://tracker.ceph.com/issues/24158>`_, `pr#23855 <https://github.com/ceph/ceph/pull/23855>`_, Abhishek Lekshmanan)
+* rgw: civetweb: use poll instead of select while waiting on sockets (`issue#35954 <http://tracker.ceph.com/issues/35954>`_, `pr#24058 <https://github.com/ceph/ceph/pull/24058>`_, Abhishek Lekshmanan)
+* rgw: do not ignore EEXIST in RGWPutObj::execute (`issue#25078 <http://tracker.ceph.com/issues/25078>`_, `issue#22790 <http://tracker.ceph.com/issues/22790>`_, `pr#23206 <https://github.com/ceph/ceph/pull/23206>`_, Matt Benjamin)
+* rgw: fail to recover index from crash mimic backport (`issue#24640 <http://tracker.ceph.com/issues/24640>`_, `issue#24629 <http://tracker.ceph.com/issues/24629>`_, `issue#24280 <http://tracker.ceph.com/issues/24280>`_, `pr#23118 <https://github.com/ceph/ceph/pull/23118>`_, Tianshan Qu)
+* rgw_file: deep stat handling (`issue#26842 <http://tracker.ceph.com/issues/26842>`_, `issue#24915 <http://tracker.ceph.com/issues/24915>`_, `pr#23498 <https://github.com/ceph/ceph/pull/23498>`_, Matt Benjamin)
+* rgw: Fix log level of gc_iterate_entries (`issue#23801 <http://tracker.ceph.com/issues/23801>`_, `issue#26921 <http://tracker.ceph.com/issues/26921>`_, `pr#23686 <https://github.com/ceph/ceph/pull/23686>`_, iliul)
+* rgw: Limit the number of lifecycle rules on one bucket (`issue#26845 <http://tracker.ceph.com/issues/26845>`_, `issue#24572 <http://tracker.ceph.com/issues/24572>`_, `pr#23521 <https://github.com/ceph/ceph/pull/23521>`_, Zhang Shaowen)
+* rgw: radosgw-admin: 'sync error trim' loops until complete (`issue#24873 <http://tracker.ceph.com/issues/24873>`_, `issue#24984 <http://tracker.ceph.com/issues/24984>`_, `pr#23140 <https://github.com/ceph/ceph/pull/23140>`_, Casey Bodley)
+* rgw: The delete markers generated by object expiration should have owner (`issue#24568 <http://tracker.ceph.com/issues/24568>`_, `issue#26847 <http://tracker.ceph.com/issues/26847>`_, `pr#23541 <https://github.com/ceph/ceph/pull/23541>`_, Zhang Shaowen)
+* rpm: should change ceph-mgr package depency from py-bcrypt to python2-bcrypt (`issue#27212 <http://tracker.ceph.com/issues/27212>`_, `pr#23868 <https://github.com/ceph/ceph/pull/23868>`_, Konstantin Sakhinov)
+* rpm: silence osd block chown (`issue#25152 <http://tracker.ceph.com/issues/25152>`_, `pr#23324 <https://github.com/ceph/ceph/pull/23324>`_, Dan van der Ster)
+* run-rbd-unit-tests.sh test fails to finish in jenkin's make check run (`issue#27060 <http://tracker.ceph.com/issues/27060>`_, `issue#24910 <http://tracker.ceph.com/issues/24910>`_, `pr#23858 <https://github.com/ceph/ceph/pull/23858>`_, Mykola Golub)
+* scrub livelock (`issue#26931 <http://tracker.ceph.com/issues/26931>`_, `issue#26890 <http://tracker.ceph.com/issues/26890>`_, `pr#23722 <https://github.com/ceph/ceph/pull/23722>`_, Sage Weil)
+* spdk: compile with -march=core2 instead of -march=native (`issue#25032 <http://tracker.ceph.com/issues/25032>`_, `pr#23175 <https://github.com/ceph/ceph/pull/23175>`_, Nathan Cutler)
+* tests: cluster [WRN] 25 slow requests in powercycle (`issue#25119 <http://tracker.ceph.com/issues/25119>`_, `pr#23886 <https://github.com/ceph/ceph/pull/23886>`_, Neha Ojha)
+* test: Use pids instead of jobspecs which were wrong (`issue#32079 <http://tracker.ceph.com/issues/32079>`_, `issue#27056 <http://tracker.ceph.com/issues/27056>`_, `pr#23893 <https://github.com/ceph/ceph/pull/23893>`_, David Zafman)
+* tools/ceph-detect-init: support RHEL as a platform (`issue#18163 <http://tracker.ceph.com/issues/18163>`_, `pr#23303 <https://github.com/ceph/ceph/pull/23303>`_, Nathan Cutler)
+* tools: ceph-detect-init: support SLED (`issue#18163 <http://tracker.ceph.com/issues/18163>`_, `pr#23111 <https://github.com/ceph/ceph/pull/23111>`_, Nathan Cutler)
+* tools: cephfs-data-scan: print the max used ino (`issue#26978 <http://tracker.ceph.com/issues/26978>`_, `issue#26925 <http://tracker.ceph.com/issues/26925>`_, `pr#23880 <https://github.com/ceph/ceph/pull/23880>`_, "Yan, Zheng")
+
+
+v13.2.1 Mimic
+=============
+
+This is the first bugfix release of the Mimic v13.2.x long term stable release
+series. This release contains many fixes across all components of Ceph,
+including a few security fixes. We recommend that all users upgrade.
+
+Notable Changes
+---------------
+
+* CVE 2018-1128: auth: cephx authorizer subject to replay attack (`issue#24836 <http://tracker.ceph.com/issues/24836>`_, Sage Weil)
+* CVE 2018-1129: auth: cephx signature check is weak (`issue#24837 <http://tracker.ceph.com/issues/24837>`_, Sage Weil)
+* CVE 2018-10861: mon: auth checks not correct for pool ops (`issue#24838 <http://tracker.ceph.com/issues/24838>`_, Jason Dillaman)
+
+Changelog
+---------
+
+* bluestore:  common/hobject: improved hash calculation for hobject_t etc (`pr#22777 <https://github.com/ceph/ceph/pull/22777>`_, Adam Kupczyk, Sage Weil)
+* bluestore,core: mimic: os/bluestore: don't store/use path_block.{db,wal} from meta (`pr#22477 <https://github.com/ceph/ceph/pull/22477>`_, Sage Weil, Alfredo Deza)
+* bluestore: os/bluestore: backport 24319 and 24550 (`issue#24550 <http://tracker.ceph.com/issues/24550>`_, `issue#24502 <http://tracker.ceph.com/issues/24502>`_, `issue#24319 <http://tracker.ceph.com/issues/24319>`_, `issue#24581 <http://tracker.ceph.com/issues/24581>`_, `pr#22649 <https://github.com/ceph/ceph/pull/22649>`_, Sage Weil)
+* bluestore: os/bluestore: fix incomplete faulty range marking when doing compression (`pr#22910 <https://github.com/ceph/ceph/pull/22910>`_, Igor Fedotov)
+* bluestore: spdk: fix ceph-osd crash when activate SPDK (`issue#24472 <http://tracker.ceph.com/issues/24472>`_, `issue#24371 <http://tracker.ceph.com/issues/24371>`_, `pr#22684 <https://github.com/ceph/ceph/pull/22684>`_, tone-zhang)
+* build/ops: build/ops: ceph.git has two different versions of dpdk in the source tree (`issue#24942 <http://tracker.ceph.com/issues/24942>`_, `issue#24032 <http://tracker.ceph.com/issues/24032>`_, `pr#23070 <https://github.com/ceph/ceph/pull/23070>`_, Kefu Chai)
+* build/ops: build/ops: install-deps.sh fails on newest openSUSE Leap (`issue#25065 <http://tracker.ceph.com/issues/25065>`_, `pr#23178 <https://github.com/ceph/ceph/pull/23178>`_, Kyr Shatskyy)
+* build/ops: build/ops: Mimic build fails with -DWITH_RADOSGW=0 (`issue#24766 <http://tracker.ceph.com/issues/24766>`_, `pr#22851 <https://github.com/ceph/ceph/pull/22851>`_, Dan Mick)
+* build/ops: cmake: enable RTTI for both debug and release RocksDB builds (`pr#22299 <https://github.com/ceph/ceph/pull/22299>`_, Igor Fedotov)
+* build/ops: deb/rpm: add python-six as build-time and run-time dependency (`issue#24885 <http://tracker.ceph.com/issues/24885>`_, `pr#22948 <https://github.com/ceph/ceph/pull/22948>`_, Nathan Cutler, Kefu Chai)
+* build/ops: deb,rpm: fix block.db symlink ownership (`pr#23246 <https://github.com/ceph/ceph/pull/23246>`_, Sage Weil)
+* build/ops: include: fix build with older clang (OSX target) (`pr#23049 <https://github.com/ceph/ceph/pull/23049>`_, Christopher Blum)
+* build/ops: include: fix build with older clang (`pr#23034 <https://github.com/ceph/ceph/pull/23034>`_, Kefu Chai)
+* build/ops,rbd: build/ops: order rbdmap.service before remote-fs-pre.target (`issue#24713 <http://tracker.ceph.com/issues/24713>`_, `issue#24734 <http://tracker.ceph.com/issues/24734>`_, `pr#22843 <https://github.com/ceph/ceph/pull/22843>`_, Ilya Dryomov)
+* cephfs: cephfs: allow prohibiting user snapshots in CephFS (`issue#24705 <http://tracker.ceph.com/issues/24705>`_, `issue#24284 <http://tracker.ceph.com/issues/24284>`_, `pr#22812 <https://github.com/ceph/ceph/pull/22812>`_, "Yan, Zheng")
+* cephfs: cephfs-journal-tool: Fix purging when importing an zero-length journal (`issue#24861 <http://tracker.ceph.com/issues/24861>`_, `pr#22981 <https://github.com/ceph/ceph/pull/22981>`_, yupeng chen, zhongyan gu)
+* cephfs: client: fix bug #24491 _ll_drop_pins may access invalid iterator (`issue#24534 <http://tracker.ceph.com/issues/24534>`_, `pr#22791 <https://github.com/ceph/ceph/pull/22791>`_, Liu Yangkuan)
+* cephfs: client: update inode fields according to issued caps (`issue#24539 <http://tracker.ceph.com/issues/24539>`_, `issue#24269 <http://tracker.ceph.com/issues/24269>`_, `pr#22819 <https://github.com/ceph/ceph/pull/22819>`_, "Yan, Zheng")
+* cephfs: common/DecayCounter: set last_decay to current time when decoding dec… (`issue#24440 <http://tracker.ceph.com/issues/24440>`_, `issue#24537 <http://tracker.ceph.com/issues/24537>`_, `pr#22816 <https://github.com/ceph/ceph/pull/22816>`_, Zhi Zhang)
+* cephfs,core: mon/MDSMonitor: do not send redundant MDS health messages to cluster log (`issue#24308 <http://tracker.ceph.com/issues/24308>`_, `issue#24330 <http://tracker.ceph.com/issues/24330>`_, `pr#22265 <https://github.com/ceph/ceph/pull/22265>`_, Sage Weil)
+* cephfs: mds: add magic to header of open file table (`issue#24541 <http://tracker.ceph.com/issues/24541>`_, `issue#24240 <http://tracker.ceph.com/issues/24240>`_, `pr#22841 <https://github.com/ceph/ceph/pull/22841>`_, "Yan, Zheng")
+* cephfs: mds: low wrlock efficiency due to dirfrags traversal (`issue#24704 <http://tracker.ceph.com/issues/24704>`_, `issue#24467 <http://tracker.ceph.com/issues/24467>`_, `pr#22884 <https://github.com/ceph/ceph/pull/22884>`_, Xuehan Xu)
+* cephfs: PurgeQueue sometimes ignores Journaler errors (`issue#24533 <http://tracker.ceph.com/issues/24533>`_, `issue#24703 <http://tracker.ceph.com/issues/24703>`_, `pr#22810 <https://github.com/ceph/ceph/pull/22810>`_, John Spray)
+* cephfs,rbd: osdc: Fix the wrong BufferHead offset (`issue#24583 <http://tracker.ceph.com/issues/24583>`_, `pr#22869 <https://github.com/ceph/ceph/pull/22869>`_, dongdong tao)
+* cephfs: repeated eviction of idle client until some IO happens (`issue#24052 <http://tracker.ceph.com/issues/24052>`_, `issue#24296 <http://tracker.ceph.com/issues/24296>`_, `pr#22550 <https://github.com/ceph/ceph/pull/22550>`_, "Yan, Zheng")
+* cephfs: test gets ENOSPC from bluestore block device (`issue#24238 <http://tracker.ceph.com/issues/24238>`_, `issue#24913 <http://tracker.ceph.com/issues/24913>`_, `issue#24899 <http://tracker.ceph.com/issues/24899>`_, `issue#24758 <http://tracker.ceph.com/issues/24758>`_, `pr#22835 <https://github.com/ceph/ceph/pull/22835>`_, Patrick Donnelly, Sage Weil)
+* cephfs,tests: pjd: cd: too many arguments (`issue#24310 <http://tracker.ceph.com/issues/24310>`_, `pr#22882 <https://github.com/ceph/ceph/pull/22882>`_, Neha Ojha)
+* cephfs,tests: qa: client socket inaccessible without sudo (`issue#24872 <http://tracker.ceph.com/issues/24872>`_, `issue#24904 <http://tracker.ceph.com/issues/24904>`_, `pr#23030 <https://github.com/ceph/ceph/pull/23030>`_, Patrick Donnelly)
+* cephfs,tests: qa: fix ffsb cd argument (`issue#24719 <http://tracker.ceph.com/issues/24719>`_, `issue#24829 <http://tracker.ceph.com/issues/24829>`_, `issue#24680 <http://tracker.ceph.com/issues/24680>`_, `issue#24579 <http://tracker.ceph.com/issues/24579>`_, `pr#22956 <https://github.com/ceph/ceph/pull/22956>`_, Yan, Zheng, Patrick Donnelly)
+* cephfs,tests: qa/suites: Add supported-random-distro$ links (`issue#24706 <http://tracker.ceph.com/issues/24706>`_, `issue#24138 <http://tracker.ceph.com/issues/24138>`_, `pr#22700 <https://github.com/ceph/ceph/pull/22700>`_, Warren Usui)
+* ceph-volume describe better the options for migrating away from ceph-disk (`pr#22514 <https://github.com/ceph/ceph/pull/22514>`_, Alfredo Deza)
+* ceph-volume dmcrypt and activate --all documentation updates (`pr#22529 <https://github.com/ceph/ceph/pull/22529>`_, Alfredo Deza)
+* ceph-volume: error on commands that need ceph.conf to operate (`issue#23941 <http://tracker.ceph.com/issues/23941>`_, `pr#22747 <https://github.com/ceph/ceph/pull/22747>`_, Andrew Schoen)
+* ceph-volume expand on the LVM API to create multiple LVs at different sizes (`pr#22508 <https://github.com/ceph/ceph/pull/22508>`_, Alfredo Deza)
+* ceph-volume initial take on auto sub-command (`pr#22515 <https://github.com/ceph/ceph/pull/22515>`_, Alfredo Deza)
+* ceph-volume lvm.activate Do not search for a MON configuration (`pr#22398 <https://github.com/ceph/ceph/pull/22398>`_, Wido den Hollander)
+* ceph-volume lvm.common use destroy-new, doesn't need admin keyring (`issue#24585 <http://tracker.ceph.com/issues/24585>`_, `pr#22900 <https://github.com/ceph/ceph/pull/22900>`_, Alfredo Deza)
+* ceph-volume: provide a nice errror message when missing ceph.conf (`pr#22832 <https://github.com/ceph/ceph/pull/22832>`_, Andrew Schoen)
+* ceph-volume tests destroy osds on monitor hosts (`pr#22507 <https://github.com/ceph/ceph/pull/22507>`_, Alfredo Deza)
+* ceph-volume tests do not include admin keyring in OSD nodes (`pr#22425 <https://github.com/ceph/ceph/pull/22425>`_, Alfredo Deza)
+* ceph-volume tests.functional install new ceph-ansible dependencies (`pr#22535 <https://github.com/ceph/ceph/pull/22535>`_, Alfredo Deza)
+* ceph-volume: tests/functional run lvm list after OSD provisioning (`issue#24961 <http://tracker.ceph.com/issues/24961>`_, `pr#23148 <https://github.com/ceph/ceph/pull/23148>`_, Alfredo Deza)
+* ceph-volume tests/functional use Ansible 2.6 (`pr#23244 <https://github.com/ceph/ceph/pull/23244>`_, Alfredo Deza)
+* ceph-volume: unmount lvs correctly before zapping (`issue#24796 <http://tracker.ceph.com/issues/24796>`_, `pr#23127 <https://github.com/ceph/ceph/pull/23127>`_, Andrew Schoen)
+* cmake: bump up the required boost version to 1.67 (`pr#22412 <https://github.com/ceph/ceph/pull/22412>`_, Kefu Chai)
+* common: common: Abort in OSDMap::decode() during qa/standalone/erasure-code/test-erasure-eio.sh (`issue#24865 <http://tracker.ceph.com/issues/24865>`_, `issue#23492 <http://tracker.ceph.com/issues/23492>`_, `pr#23024 <https://github.com/ceph/ceph/pull/23024>`_, Sage Weil)
+* common: common: fix typo in rados bench write JSON output (`issue#24292 <http://tracker.ceph.com/issues/24292>`_, `issue#24199 <http://tracker.ceph.com/issues/24199>`_, `pr#22406 <https://github.com/ceph/ceph/pull/22406>`_, Sandor Zeestraten)
+* common,core: common: partially revert 95fc248 to make get_process_name work (`issue#24123 <http://tracker.ceph.com/issues/24123>`_, `issue#24215 <http://tracker.ceph.com/issues/24215>`_, `pr#22311 <https://github.com/ceph/ceph/pull/22311>`_, Mykola Golub)
+* common: osd: Change osd_skip_data_digest default to false and make it LEVEL_DEV (`pr#23084 <https://github.com/ceph/ceph/pull/23084>`_, Sage Weil, David Zafman)
+* common: tell ... config rm <foo> not idempotent (`issue#24468 <http://tracker.ceph.com/issues/24468>`_, `issue#24408 <http://tracker.ceph.com/issues/24408>`_, `pr#22552 <https://github.com/ceph/ceph/pull/22552>`_, Sage Weil)
+* core: bluestore: flush_commit is racy (`issue#24261 <http://tracker.ceph.com/issues/24261>`_, `issue#21480 <http://tracker.ceph.com/issues/21480>`_, `pr#22382 <https://github.com/ceph/ceph/pull/22382>`_, Sage Weil)
+* core: ceph osd safe-to-destroy crashes the mgr (`issue#24708 <http://tracker.ceph.com/issues/24708>`_, `issue#23249 <http://tracker.ceph.com/issues/23249>`_, `pr#22805 <https://github.com/ceph/ceph/pull/22805>`_, Sage Weil)
+* core: change default filestore_merge_threshold to -10 (`issue#24686 <http://tracker.ceph.com/issues/24686>`_, `issue#24747 <http://tracker.ceph.com/issues/24747>`_, `pr#22813 <https://github.com/ceph/ceph/pull/22813>`_, Douglas Fuller)
+* core: common/hobject: improved hash calculation (`pr#22722 <https://github.com/ceph/ceph/pull/22722>`_, Adam Kupczyk)
+* core: cosbench stuck at booting cosbench driver (`issue#24473 <http://tracker.ceph.com/issues/24473>`_, `pr#22887 <https://github.com/ceph/ceph/pull/22887>`_, Neha Ojha)
+* core: librados: fix buffer overflow for aio_exec python binding (`issue#24475 <http://tracker.ceph.com/issues/24475>`_, `pr#22707 <https://github.com/ceph/ceph/pull/22707>`_, Aleksei Gutikov)
+* core: mon: enable level_compaction_dynamic_level_bytes for rocksdb (`issue#24375 <http://tracker.ceph.com/issues/24375>`_, `issue#24361 <http://tracker.ceph.com/issues/24361>`_, `pr#22361 <https://github.com/ceph/ceph/pull/22361>`_, Kefu Chai)
+* core: mon/MgrMonitor: change 'unresponsive' message to info level (`issue#24246 <http://tracker.ceph.com/issues/24246>`_, `issue#24222 <http://tracker.ceph.com/issues/24222>`_, `pr#22333 <https://github.com/ceph/ceph/pull/22333>`_, Sage Weil)
+* core: mon/OSDMonitor: no_reply on MOSDFailure messages (`issue#24322 <http://tracker.ceph.com/issues/24322>`_, `issue#24350 <http://tracker.ceph.com/issues/24350>`_, `pr#22297 <https://github.com/ceph/ceph/pull/22297>`_, Sage Weil)
+* core: os/bluestore: firstly delete db then delete bluefs if open db met error (`pr#22525 <https://github.com/ceph/ceph/pull/22525>`_, Jianpeng Ma)
+* core: os/bluestore: fix races on SharedBlob::coll in ~SharedBlob (`issue#24859 <http://tracker.ceph.com/issues/24859>`_, `issue#24887 <http://tracker.ceph.com/issues/24887>`_, `pr#23065 <https://github.com/ceph/ceph/pull/23065>`_, Radoslaw Zarzynski)
+* core: osd: choose_acting loop (`issue#24383 <http://tracker.ceph.com/issues/24383>`_, `issue#24618 <http://tracker.ceph.com/issues/24618>`_, `pr#22889 <https://github.com/ceph/ceph/pull/22889>`_, Neha Ojha)
+* core: osd: do not blindly roll forward to log.head (`issue#24597 <http://tracker.ceph.com/issues/24597>`_, `pr#22997 <https://github.com/ceph/ceph/pull/22997>`_, Sage Weil)
+* core: osd: eternal stuck PG in 'unfound_recovery' (`issue#24500 <http://tracker.ceph.com/issues/24500>`_, `issue#24373 <http://tracker.ceph.com/issues/24373>`_, `pr#22545 <https://github.com/ceph/ceph/pull/22545>`_, Sage Weil)
+* core: osd: fix deep scrub with osd_skip_data_digest=true (default) and blue… (`issue#24922 <http://tracker.ceph.com/issues/24922>`_, `issue#24958 <http://tracker.ceph.com/issues/24958>`_, `pr#23094 <https://github.com/ceph/ceph/pull/23094>`_, Sage Weil)
+* core: osd: fix getting osd maps on initial osd startup (`pr#22651 <https://github.com/ceph/ceph/pull/22651>`_, Paul Emmerich)
+* core: osd: increase default hard pg limit (`issue#24355 <http://tracker.ceph.com/issues/24355>`_, `pr#22621 <https://github.com/ceph/ceph/pull/22621>`_, Josh Durgin)
+* core: osd: may get empty info at recovery (`issue#24771 <http://tracker.ceph.com/issues/24771>`_, `issue#24588 <http://tracker.ceph.com/issues/24588>`_, `pr#22861 <https://github.com/ceph/ceph/pull/22861>`_, Sage Weil)
+* core: osd/PrimaryLogPG: rebuild attrs from clients (`issue#24768 <http://tracker.ceph.com/issues/24768>`_, `issue#24805 <http://tracker.ceph.com/issues/24805>`_, `pr#22960 <https://github.com/ceph/ceph/pull/22960>`_, Sage Weil)
+* core: osd: retry to read object attrs at EC recovery (`issue#24406 <http://tracker.ceph.com/issues/24406>`_, `pr#22394 <https://github.com/ceph/ceph/pull/22394>`_, xiaofei cui)
+* core: osd/Session: fix invalid iterator dereference in Sessoin::have_backoff() (`issue#24486 <http://tracker.ceph.com/issues/24486>`_, `issue#24494 <http://tracker.ceph.com/issues/24494>`_, `pr#22730 <https://github.com/ceph/ceph/pull/22730>`_, Sage Weil)
+* core: PG: add custom_reaction Backfilled and release reservations after bac… (`issue#24332 <http://tracker.ceph.com/issues/24332>`_, `pr#22559 <https://github.com/ceph/ceph/pull/22559>`_, Neha Ojha)
+* core: set correctly shard for existed Collection (`issue#24769 <http://tracker.ceph.com/issues/24769>`_, `issue#24761 <http://tracker.ceph.com/issues/24761>`_, `pr#22859 <https://github.com/ceph/ceph/pull/22859>`_, Jianpeng Ma)
+* core,tests: Bring back diff -y for non-FreeBSD (`issue#24738 <http://tracker.ceph.com/issues/24738>`_, `issue#24470 <http://tracker.ceph.com/issues/24470>`_, `pr#22826 <https://github.com/ceph/ceph/pull/22826>`_, Sage Weil, David Zafman)
+* core,tests: ceph_test_rados_api_misc: fix LibRadosMiscPool.PoolCreationRace (`issue#24204 <http://tracker.ceph.com/issues/24204>`_, `issue#24150 <http://tracker.ceph.com/issues/24150>`_, `pr#22291 <https://github.com/ceph/ceph/pull/22291>`_, Sage Weil)
+* core,tests: qa/workunits/suites/blogbench.sh: use correct dir name (`pr#22775 <https://github.com/ceph/ceph/pull/22775>`_, Neha Ojha)
+* core,tests: Wip scrub omap (`issue#24366 <http://tracker.ceph.com/issues/24366>`_, `issue#24381 <http://tracker.ceph.com/issues/24381>`_, `pr#22374 <https://github.com/ceph/ceph/pull/22374>`_, David Zafman)
+* core,tools: ceph-detect-init: stop using platform.linux_distribution (`issue#18163 <http://tracker.ceph.com/issues/18163>`_, `pr#21523 <https://github.com/ceph/ceph/pull/21523>`_, Nathan Cutler)
+* core: ValueError: too many values to unpack due to lack of subdir (`issue#24617 <http://tracker.ceph.com/issues/24617>`_, `pr#22888 <https://github.com/ceph/ceph/pull/22888>`_, Neha Ojha)
+* doc: ceph-bluestore-tool manpage not getting rendered correctly (`issue#25062 <http://tracker.ceph.com/issues/25062>`_, `issue#24800 <http://tracker.ceph.com/issues/24800>`_, `pr#23176 <https://github.com/ceph/ceph/pull/23176>`_, Nathan Cutler)
+* doc: doc: update experimental features - snapshots (`pr#22803 <https://github.com/ceph/ceph/pull/22803>`_, Jos Collin)
+* doc: fix the links in releases/schedule.rst (`pr#22372 <https://github.com/ceph/ceph/pull/22372>`_, Kefu Chai)
+* doc: [mimic] doc/cephfs: remove lingering "experimental" note about multimds (`pr#22854 <https://github.com/ceph/ceph/pull/22854>`_, John Spray)
+* lvm: when osd creation fails log the exception (`issue#24456 <http://tracker.ceph.com/issues/24456>`_, `pr#22640 <https://github.com/ceph/ceph/pull/22640>`_, Andrew Schoen)
+* mgr/dashboard: Fix bug when creating S3 keys (`pr#22468 <https://github.com/ceph/ceph/pull/22468>`_, Volker Theile)
+* mgr/dashboard: fix lint error caused by codelyzer update (`pr#22713 <https://github.com/ceph/ceph/pull/22713>`_, Tiago Melo)
+* mgr/dashboard: Fix some datatable CSS issues (`pr#22274 <https://github.com/ceph/ceph/pull/22274>`_, Volker Theile)
+* mgr/dashboard: Float numbers incorrectly formatted (`issue#24081 <http://tracker.ceph.com/issues/24081>`_, `issue#24707 <http://tracker.ceph.com/issues/24707>`_, `pr#22886 <https://github.com/ceph/ceph/pull/22886>`_, Stephan Müller, Tiago Melo)
+* mgr/dashboard: Missing breadcrumb on monitor performance counters page (`issue#24764 <http://tracker.ceph.com/issues/24764>`_, `pr#22849 <https://github.com/ceph/ceph/pull/22849>`_, Ricardo Marques, Tiago Melo)
+* mgr/dashboard: Replace Pool with Pools (`issue#24699 <http://tracker.ceph.com/issues/24699>`_, `pr#22807 <https://github.com/ceph/ceph/pull/22807>`_, Lenz Grimmer)
+* mgr: mgr/dashboard: Listen on port 8443 by default and not 8080 (`pr#22449 <https://github.com/ceph/ceph/pull/22449>`_, Wido den Hollander)
+* mgr,mon: exception for dashboard in config-key warning (`pr#22770 <https://github.com/ceph/ceph/pull/22770>`_, John Spray)
+* mgr,pybind: Python bindings use iteritems method which is not Python 3 compatible (`issue#24803 <http://tracker.ceph.com/issues/24803>`_, `issue#24779 <http://tracker.ceph.com/issues/24779>`_, `pr#22917 <https://github.com/ceph/ceph/pull/22917>`_, Nathan Cutler)
+* mgr: Sync up ceph-mgr prometheus related changes (`pr#22341 <https://github.com/ceph/ceph/pull/22341>`_, Boris Ranto)
+* mon: don't require CEPHX_V2 from mons until nautilus (`pr#23233 <https://github.com/ceph/ceph/pull/23233>`_, Sage Weil)
+* mon/OSDMonitor: Respect paxos_propose_interval (`pr#22268 <https://github.com/ceph/ceph/pull/22268>`_, Xiaoxi CHEN)
+* osd: forward-port osd_distrust_data_digest from luminous (`pr#23184 <https://github.com/ceph/ceph/pull/23184>`_, Sage Weil)
+* osd/OSDMap: fix CEPHX_V2 osd requirement to nautilus, not mimic (`pr#23250 <https://github.com/ceph/ceph/pull/23250>`_, Sage Weil)
+* qa/rgw: disable testing on ec-cache pools (`issue#23965 <http://tracker.ceph.com/issues/23965>`_, `pr#23096 <https://github.com/ceph/ceph/pull/23096>`_, Casey Bodley)
+* qa/suites/upgrade/mimic-p2p: allow target version to apply (`pr#23262 <https://github.com/ceph/ceph/pull/23262>`_, Sage Weil)
+* qa/tests: added supported distro for powercycle suite (`pr#22224 <https://github.com/ceph/ceph/pull/22224>`_, Yuri Weinstein)
+* qa/tests: changed distro symlink to point to new way using supported OSes (`pr#22653 <https://github.com/ceph/ceph/pull/22653>`_, Yuri Weinstein)
+* rbd: librbd: deep_copy: resize head object map if needed (`issue#24499 <http://tracker.ceph.com/issues/24499>`_, `issue#24399 <http://tracker.ceph.com/issues/24399>`_, `pr#22768 <https://github.com/ceph/ceph/pull/22768>`_, Mykola Golub)
+* rbd: librbd: fix crash when opening nonexistent snapshot (`issue#24637 <http://tracker.ceph.com/issues/24637>`_, `issue#24698 <http://tracker.ceph.com/issues/24698>`_, `pr#22943 <https://github.com/ceph/ceph/pull/22943>`_, Mykola Golub)
+* rbd: librbd: force 'invalid object map' flag on-disk update (`issue#24496 <http://tracker.ceph.com/issues/24496>`_, `issue#24434 <http://tracker.ceph.com/issues/24434>`_, `pr#22754 <https://github.com/ceph/ceph/pull/22754>`_, Mykola Golub)
+* rbd: librbd: utilize the journal disabled policy when removing images (`issue#24388 <http://tracker.ceph.com/issues/24388>`_, `issue#23512 <http://tracker.ceph.com/issues/23512>`_, `pr#22662 <https://github.com/ceph/ceph/pull/22662>`_, Jason Dillaman)
+* rbd: Prevent the use of internal feature bits from outside cls/rbd (`issue#24165 <http://tracker.ceph.com/issues/24165>`_, `issue#24203 <http://tracker.ceph.com/issues/24203>`_, `pr#22222 <https://github.com/ceph/ceph/pull/22222>`_, Jason Dillaman)
+* rbd: rbd-mirror daemon failed to stop on active/passive test case (`issue#24390 <http://tracker.ceph.com/issues/24390>`_, `pr#22667 <https://github.com/ceph/ceph/pull/22667>`_, Jason Dillaman)
+* rbd: [rbd-mirror] entries_behind_master will not be zero after mirror over (`issue#24391 <http://tracker.ceph.com/issues/24391>`_, `issue#23516 <http://tracker.ceph.com/issues/23516>`_, `pr#22549 <https://github.com/ceph/ceph/pull/22549>`_, Jason Dillaman)
+* rbd: rbd-mirror simple image map policy doesn't always level-load instances (`issue#24519 <http://tracker.ceph.com/issues/24519>`_, `issue#24161 <http://tracker.ceph.com/issues/24161>`_, `pr#22892 <https://github.com/ceph/ceph/pull/22892>`_, Venky Shankar)
+* rbd: rbd trash purge --threshold should support data pool (`issue#24476 <http://tracker.ceph.com/issues/24476>`_, `issue#22872 <http://tracker.ceph.com/issues/22872>`_, `pr#22891 <https://github.com/ceph/ceph/pull/22891>`_, Mahati Chamarthy)
+* rbd,tests: qa: krbd_exclusive_option.sh: bump lock_timeout to 60 seconds (`issue#25081 <http://tracker.ceph.com/issues/25081>`_, `pr#23209 <https://github.com/ceph/ceph/pull/23209>`_, Ilya Dryomov)
+* rbd: yet another case when deep copying a clone may result in invalid object map (`issue#24596 <http://tracker.ceph.com/issues/24596>`_, `issue#24545 <http://tracker.ceph.com/issues/24545>`_, `pr#22894 <https://github.com/ceph/ceph/pull/22894>`_, Mykola Golub)
+* rgw: cls_bucket_list fails causes cascading osd crashes (`issue#24631 <http://tracker.ceph.com/issues/24631>`_, `issue#24117 <http://tracker.ceph.com/issues/24117>`_, `pr#22927 <https://github.com/ceph/ceph/pull/22927>`_, Yehuda Sadeh)
+* rgw: multisite: RGWSyncTraceNode released twice and crashed in reload (`issue#24432 <http://tracker.ceph.com/issues/24432>`_, `issue#24619 <http://tracker.ceph.com/issues/24619>`_, `pr#22926 <https://github.com/ceph/ceph/pull/22926>`_, Tianshan Qu)
+* rgw: objects in cache never refresh after rgw_cache_expiry_interval (`issue#24346 <http://tracker.ceph.com/issues/24346>`_, `issue#24385 <http://tracker.ceph.com/issues/24385>`_, `pr#22643 <https://github.com/ceph/ceph/pull/22643>`_, Casey Bodley)
+* rgw: add configurable AWS-compat invalid range get behavior (`issue#24317 <http://tracker.ceph.com/issues/24317>`_, `issue#24352 <http://tracker.ceph.com/issues/24352>`_, `pr#22590 <https://github.com/ceph/ceph/pull/22590>`_, Matt Benjamin)
+* rgw: Admin OPS Api overwrites email when user is modified (`issue#24253 <http://tracker.ceph.com/issues/24253>`_, `pr#22523 <https://github.com/ceph/ceph/pull/22523>`_, Volker Theile)
+* rgw: fix gc may cause a large number of read traffic (`issue#24807 <http://tracker.ceph.com/issues/24807>`_, `issue#24767 <http://tracker.ceph.com/issues/24767>`_, `pr#22941 <https://github.com/ceph/ceph/pull/22941>`_, Xin Liao)
+* rgw: have a configurable authentication order (`issue#23089 <http://tracker.ceph.com/issues/23089>`_, `issue#24547 <http://tracker.ceph.com/issues/24547>`_, `pr#22842 <https://github.com/ceph/ceph/pull/22842>`_, Abhishek Lekshmanan)
+* rgw: index complete miss zones_trace set (`issue#24701 <http://tracker.ceph.com/issues/24701>`_, `issue#24590 <http://tracker.ceph.com/issues/24590>`_, `pr#22818 <https://github.com/ceph/ceph/pull/22818>`_, Tianshan Qu)
+* rgw: Invalid Access-Control-Request-Request may bypass validate_cors_rule_method (`issue#24809 <http://tracker.ceph.com/issues/24809>`_, `issue#24223 <http://tracker.ceph.com/issues/24223>`_, `pr#22935 <https://github.com/ceph/ceph/pull/22935>`_, Jeegn Chen)
+* rgw: meta and data notify thread miss stop cr manager (`issue#24702 <http://tracker.ceph.com/issues/24702>`_, `issue#24589 <http://tracker.ceph.com/issues/24589>`_, `pr#22821 <https://github.com/ceph/ceph/pull/22821>`_, Tianshan Qu)
+* rgw:-multisite: endless loop in RGWBucketShardIncrementalSyncCR (`issue#24700 <http://tracker.ceph.com/issues/24700>`_, `issue#24603 <http://tracker.ceph.com/issues/24603>`_, `pr#22815 <https://github.com/ceph/ceph/pull/22815>`_, cfanz)
+* rgw: performance regression for luminous 12.2.4 (`issue#23379 <http://tracker.ceph.com/issues/23379>`_, `issue#24633 <http://tracker.ceph.com/issues/24633>`_, `pr#22929 <https://github.com/ceph/ceph/pull/22929>`_, Mark Kogan)
+* rgw: radogw-admin reshard status command should print text for reshar… (`issue#24834 <http://tracker.ceph.com/issues/24834>`_, `issue#23257 <http://tracker.ceph.com/issues/23257>`_, `pr#23021 <https://github.com/ceph/ceph/pull/23021>`_, Orit Wasserman)
+* rgw: "radosgw-admin objects expire" always returns ok even if the pro… (`issue#24831 <http://tracker.ceph.com/issues/24831>`_, `issue#24592 <http://tracker.ceph.com/issues/24592>`_, `pr#23001 <https://github.com/ceph/ceph/pull/23001>`_, Zhang Shaowen)
+* rgw: require --yes-i-really-mean-it to run radosgw-admin orphans find (`issue#24146 <http://tracker.ceph.com/issues/24146>`_, `issue#24843 <http://tracker.ceph.com/issues/24843>`_, `pr#22986 <https://github.com/ceph/ceph/pull/22986>`_, Matt Benjamin)
+* rgw: REST admin metadata API paging failure bucket & bucket.instance: InvalidArgument (`issue#23099 <http://tracker.ceph.com/issues/23099>`_, `issue#24813 <http://tracker.ceph.com/issues/24813>`_, `pr#22933 <https://github.com/ceph/ceph/pull/22933>`_, Matt Benjamin)
+* rgw: set cr state if aio_read err return in RGWCloneMetaLogCoroutine:state_send_rest_request (`issue#24566 <http://tracker.ceph.com/issues/24566>`_, `issue#24783 <http://tracker.ceph.com/issues/24783>`_, `pr#22880 <https://github.com/ceph/ceph/pull/22880>`_, Tianshan Qu)
+* rgw: test/rgw: fix for bucket checkpoints (`issue#24212 <http://tracker.ceph.com/issues/24212>`_, `issue#24313 <http://tracker.ceph.com/issues/24313>`_, `pr#22466 <https://github.com/ceph/ceph/pull/22466>`_, Casey Bodley)
+* rgw,tests: add unit test for cls bi list command (`issue#24736 <http://tracker.ceph.com/issues/24736>`_, `issue#24483 <http://tracker.ceph.com/issues/24483>`_, `pr#22845 <https://github.com/ceph/ceph/pull/22845>`_, Orit Wasserman)
+* tests: mimic - qa/tests: Set ansible-version: 2.4 (`issue#24926 <http://tracker.ceph.com/issues/24926>`_, `pr#23122 <https://github.com/ceph/ceph/pull/23122>`_, Yuri Weinstein)
+* tests: osd sends op_reply out of order (`issue#25010 <http://tracker.ceph.com/issues/25010>`_, `pr#23136 <https://github.com/ceph/ceph/pull/23136>`_, Neha Ojha)
+* tests: qa/tests - added overrides stanza to allow runs on ovh on rhel OS (`pr#23156 <https://github.com/ceph/ceph/pull/23156>`_, Yuri Weinstein)
+* tests: qa/tests - added skeleton for mimic point to point upgrades testing (`pr#22697 <https://github.com/ceph/ceph/pull/22697>`_, Yuri Weinstein)
+* tests: qa/tests: fix supported distro lists for ceph-deploy (`pr#23017 <https://github.com/ceph/ceph/pull/23017>`_, Vasu Kulkarni)
+* tests: qa: wait longer for osd to flush pg stats (`issue#24321 <http://tracker.ceph.com/issues/24321>`_, `pr#22492 <https://github.com/ceph/ceph/pull/22492>`_, Kefu Chai)
+* tests: tests: Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) in powercycle (`issue#25034 <http://tracker.ceph.com/issues/25034>`_, `pr#23154 <https://github.com/ceph/ceph/pull/23154>`_, Neha Ojha)
+* tests: tests: make test_ceph_argparse.py pass on py3-only systems (`issue#24825 <http://tracker.ceph.com/issues/24825>`_, `issue#24816 <http://tracker.ceph.com/issues/24816>`_, `pr#22988 <https://github.com/ceph/ceph/pull/22988>`_, Nathan Cutler)
+* tests: upgrade/luminous-x: whitelist REQUEST_SLOW for rados_mon_thrash (`issue#25056 <http://tracker.ceph.com/issues/25056>`_, `issue#25051 <http://tracker.ceph.com/issues/25051>`_, `pr#23164 <https://github.com/ceph/ceph/pull/23164>`_, Nathan Cutler)
+
+v13.2.0 Mimic
+=============
+
+This is the first stable release of Mimic, the next long term release series.
+
+Major Changes from Luminous
+---------------------------
+
+- *Dashboard*:
+
+  * The (read-only) Ceph manager dashboard introduced in Ceph Luminous has been
+    replaced with a new implementation inspired by and derived from the
+    `openATTIC <https://openattic.org>`_ Ceph management tool, providing a
+    drop-in replacement offering a :ref:`number of additional management
+    features <mgr-dashboard>`.
+
+- *RADOS*:
+
+  * Config options can now be centrally stored and managed by the monitor.
+  * The monitor daemon uses significantly less disk space when undergoing
+    recovery or rebalancing operations.
+  * An *async recovery* feature reduces the tail latency of requests
+    when the OSDs are recovering from a recent failure.
+  * OSD preemption of scrub by conflicting requests reduces tail latency.
+
+- *RGW*:
+
+  * RGW can now replicate a zone (or a subset of buckets) to an
+    external cloud storage service like S3.
+  * RGW now supports the S3 multi-factor authentication API on
+    versioned buckets.
+  * The Beast frontend is no longer experimental, and is considered
+    stable and ready for use.
+
+- *CephFS*:
+
+  * Snapshots are now stable when combined with multiple MDS daemons.
+
+- *RBD*:
+
+  * Image clones no longer require explicit *protect* and *unprotect*
+    steps.
+  * Images can be deep-copied (including any clone linkage to a parent
+    image and associated snapshots) to new pools or with altered data
+    layouts.
+
+- *Misc*:
+
+  * We have dropped the Debian builds for the Mimic release due to the
+    lack of GCC 8 in Stretch.  We expect Debian builds to return with
+    the release of Buster in early 2019, and hope to build a final
+    Luminous release (and possibly later Mimic point releases) once
+    Buster is available.
+
+
+Upgrading from Luminous
+-----------------------
+
+Notes
+~~~~~
+
+* We recommend you avoid creating any RADOS pools while the upgrade is
+  in process.
+
+* You can monitor the progress of your upgrade at each stage with the
+  ``ceph versions`` command, which will tell you what ceph version(s) are
+  running for each type of daemon.
+
+Instructions
+~~~~~~~~~~~~
+
+#. If your cluster was originally installed with a version prior to
+   Luminous, ensure that it has completed at least one full scrub of
+   all PGs while running Luminous.  Failure to do so will cause your
+   monitor daemons to refuse to join the quorum on start, leaving them
+   non-functional.
+
+   If you are unsure whether or not your Luminous cluster has
+   completed a full scrub of all PGs, you can check your cluster's
+   state by running::
+
+     # ceph osd dump | grep ^flags
+
+   In order to be able to proceed to Mimic, your OSD map must include
+   the ``recovery_deletes`` and ``purged_snapdirs`` flags.
+
+   If your OSD map does not contain both these flags, you can simply
+   wait for approximately 24-48 hours, which in a standard cluster
+   configuration should be ample time for all your placement groups to
+   be scrubbed at least once, and then repeat the above process to
+   recheck.
+
+   However, if you have just completed an upgrade to Luminous and want
+   to proceed to Mimic in short order, you can force a scrub on all
+   placement groups with a one-line shell command, like::
+
+     # ceph pg dump pgs_brief | cut -d " " -f 1 | xargs -n1 ceph pg scrub
+
+   You should take into consideration that this forced scrub may
+   possibly have a negative impact on your Ceph clients' performance.
+
+#. Make sure your cluster is stable and healthy (no down or
+   recovering OSDs).  (Optional, but recommended.)
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional,
+   but recommended.)::
+
+     # ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.::
+
+     # systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is
+   complete by looking for the ``mimic`` feature string in the mon
+   map.  For example::
+
+     # ceph mon feature ls
+
+   should include `mimic` under persistent features::
+
+     on current monmap (epoch NNN)
+        persistent: [kraken,luminous,mimic]
+        required: [kraken,luminous,mimic]
+
+#. Upgrade ``ceph-mgr`` daemons by installing the new packages and
+   restarting with::
+
+     # systemctl restart ceph-mgr.target
+
+   Verify the ``ceph-mgr`` daemons are running by checking ``ceph
+   -s``::
+
+     # ceph -s
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all hosts::
+
+     # systemctl restart ceph-osd.target
+
+   You can monitor the progress of the OSD upgrades with the new
+   ``ceph versions`` or ``ceph osd versions`` command::
+
+     # ceph osd versions
+     {
+        "ceph version 12.2.5 (...) luminous (stable)": 12,
+        "ceph version 13.2.0 (...) mimic (stable)": 22,
+     }
+
+#. Upgrade all CephFS MDS daemons.  For each CephFS file system,
+
+   #. Reduce the number of ranks to 1.  (Make note of the original
+      number of MDS daemons first if you plan to restore it later.)::
+
+	# ceph status
+	# ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by
+      periodically checking the status::
+
+	# ceph status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with::
+
+	# systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS::
+
+	# ceph status
+
+   #. Upgrade the last remaining MDS daemon by installing the new
+      packages and restarting the daemon::
+
+        # systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline::
+
+	# systemctl start ceph-mds.target
+
+   #. Restore the original value of ``max_mds`` for the volume::
+
+	# ceph fs set <fs_name> max_mds <original_max_mds>
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart radosgw.target
+
+#. Complete the upgrade by disallowing pre-Mimic OSDs and enabling
+   all new Mimic-only functionality::
+
+     # ceph osd require-osd-release mimic
+
+#. If you set ``noout`` at the beginning, be sure to clear it with::
+
+     # ceph osd unset noout
+
+#. Verify the cluster is healthy with ``ceph health``.
+
+Upgrading from pre-Luminous releases (like Jewel)
+-------------------------------------------------
+
+You *must* first upgrade to Luminous (12.2.z) before attempting an
+upgrade to Mimic.  In addition, your cluster must have completed at
+least one scrub of all PGs while running Luminous, setting the
+``recovery_deletes`` and ``purged_snapdirs`` flags in the OSD map.
+
+Upgrade compatibility notes
+---------------------------
+
+These changes occurred between the Luminous and Mimic releases.
+
+* *core*:
+
+  - The ``pg force-recovery`` command will not work for erasure-coded PGs when a
+    Luminous monitor is running along with a Mimic OSD. Please use the
+    recommended upgrade order of monitors before OSDs to avoid this issue.
+
+  - The sample ``crush-location-hook`` script has been removed. Its output is
+    equivalent to the built-in default behavior, so it has been replaced with an
+    example in the CRUSH documentation.
+
+  - The ``-f`` option of the rados tool now means ``--format`` instead
+    of ``--force``, for consistency with the ceph tool.
+
+  - The format of the ``config diff`` output via the admin socket has changed. It
+    now reflects the source of each config option (e.g., default, config file,
+    command line) as well as the final (active) value.
+
+  - Commands variously marked as `del`, `delete`, `remove`
+    etc. should now all be normalized as `rm`. Commands already
+    supporting alternatives to `rm` remain backward-compatible. This
+    changeset applies to the ``radosgw-admin`` tool as well.
+
+  - Monitors will now prune on-disk full maps if the number of maps grows above
+    a certain number (mon_osdmap_full_prune_min, default: 10000), thus
+    preventing unbounded growth of the monitor data store. This feature is
+    enabled by default, and can be disabled by setting
+    `mon_osdmap_full_prune_enabled` to false.
+
+  - *rados list-inconsistent-obj format changes:*
+
+    + Various error strings have been improved.  For example, the "oi"
+      or "oi_attr" in errors which stands for object info is now
+      "info" (e.g. oi_attr_missing is now info_missing).
+
+    + The object's "selected_object_info" is now in json format
+      instead of string.
+
+    + The attribute errors (attr_value_mismatch, attr_name_mismatch)
+      only apply to user attributes.  Only user attributes are output
+      and have the internal leading underscore stripped.
+
+    + If there are hash information errors (hinfo_missing, hinfo_corrupted,
+      hinfo_inconsistency) then "hashinfo" is added with the json format of the
+      information. If the information is corrupt then "hashinfo" is a string
+      containing the value.
+
+    + If there are snapset errors (snapset_missing, snapset_corrupted,
+      snapset_inconsistency) then "snapset" is added with the json format of the
+      information. If the information is corrupt then "snapset" is a string
+      containing the value.
+
+    + If there are object information errors (info_missing, info_corrupted,
+      obj_size_info_mismatch, object_info_inconsistency) then "object_info" is
+      added with the json format of the information instead of a string. If the
+      information is corrupt then "object_info" is a string containing the
+      value.
+
+  - *rados list-inconsistent-snapset format changes:*
+
+    + Various error strings have been improved. For example, the "ss_attr" in
+      errors which stands for snapset info is now "snapset" (e.g.
+      ss_attr_missing is now snapset_missing). The error snapset_mismatch has
+      been renamed to snapset_error to better reflect what it means.
+
+    + The head snapset information is output in json format as "snapset." This
+      means that even when there are no head errors, the head object will be
+      output when any shard has an error. This head object is there to show the
+      snapset that was used in determining errors.
+
+  - The `osd_mon_report_interval_min` option has been renamed to
+    `osd_mon_report_interval`, and the `osd_mon_report_interval_max`
+    (unused) has been eliminated. If this value has been customized on your
+    cluster then your configuration should be adjusted in order to avoid
+    reverting to the default value.
+
+  - The config-key interface can store arbitrary binary blobs but JSON can only
+    express printable strings. If binary blobs are present, the 'ceph config-key
+    dump' command will show them as something like `<<< binary blob of length N
+    >>>`.
+
+  - Bootstrap auth keys will now be generated automatically on a fresh
+    deployment; these keys will also be generated, if missing, during upgrade.
+
+  - The ``osd force-create-pg`` command now requires a force option to proceed
+    because the command is dangerous: it declares that data loss is permanent
+    and instructs the cluster to proceed with an empty PG in its place, without
+    making any further efforts to find the missing data.
+
+  *CephFS*:
+
+  - Upgrading an MDS cluster to 12.2.3+ will result in all active MDS
+    exiting due to feature incompatibilities once an upgraded MDS comes online
+    (even as standby). Operators may ignore the error messages and continue
+    upgrading/restarting or follow this upgrade sequence:
+
+    After upgrading the monitors to Mimic, reduce the number of ranks to 1
+    (`ceph fs set <fs_name> max_mds 1`), wait for all other MDS to deactivate,
+    leaving the one active MDS, stop all standbys, upgrade the single active
+    MDS, then upgrade/start standbys. Finally, restore the previous max_mds.
+
+    !! NOTE: see release notes on snapshots in CephFS if you have ever enabled
+    snapshots on your file system.
+
+    See also: https://tracker.ceph.com/issues/23172
+
+  - Several ``ceph mds ...`` commands have been obsoleted and replaced by
+    equivalent ``ceph fs ...`` commands:
+
+    + ``mds dump`` -> ``fs dump``
+    + ``mds getmap`` -> ``fs dump``
+    + ``mds stop`` -> ``mds deactivate``
+    + ``mds set_max_mds`` -> ``fs set max_mds``
+    + ``mds set`` -> ``fs set``
+    + ``mds cluster_down`` -> ``fs set cluster_down true``
+    + ``mds cluster_up`` -> ``fs set cluster_down false``
+    + ``mds add_data_pool`` -> ``fs add_data_pool``
+    + ``mds remove_data_pool`` -> ``fs rm_data_pool``
+    + ``mds rm_data_pool`` -> ``fs rm_data_pool``
+
+  - New CephFS file system attributes session_timeout and
+    session_autoclose are configurable via ``ceph fs set``. The MDS
+    config options `mds_session_timeout`, `mds_session_autoclose`, and
+    `mds_max_file_size` are now obsolete.
+
+  - As the multiple MDS feature is now standard, it is now enabled by
+    default. ``ceph fs set allow_multimds`` is now deprecated and will be
+    removed in a future release.
+
+  - As the directory fragmentation feature is now standard, it is now
+    enabled by default. ``ceph fs set allow_dirfrags`` is now deprecated and
+    will be removed in a future release.
+
+  - MDS daemons now activate and deactivate based on the value of
+    `max_mds`. Accordingly, ``ceph mds deactivate`` has been deprecated as it
+    is now redundant.
+
+  - Taking a CephFS cluster down is now done by setting the down flag which
+    deactivates all MDS. For example: `ceph fs set cephfs down true`.
+
+  - Preventing standbys from joining as new actives (formerly the now
+    deprecated cluster_down flag) on a file system is now accomplished by
+    setting the joinable flag.  This is useful mostly for testing so that a
+    file system may be quickly brought down and deleted.
+
+  - New CephFS file system attributes session_timeout and session_autoclose
+    are configurable via `ceph fs set`. The MDS config options
+    mds_session_timeout, mds_session_autoclose, and mds_max_file_size are now
+    obsolete.
+
+  - Each mds rank now maintains a table that tracks open files and their
+    ancestor directories. Recovering MDS can quickly get open files' paths,
+    significantly reducing the time of loading inodes for open files. MDS
+    creates the table automatically if it does not exist.
+
+  - CephFS snapshot is now stable and enabled by default on new filesystems.
+    To enable snapshot on existing filesystems, use the command::
+
+      ceph fs set <fs_name> allow_new_snaps
+
+    The on-disk format of snapshot metadata has changed. The old format
+    metadata can not be properly handled in multiple active MDS configuration.
+    To guarantee all snapshot metadata on existing filesystems get updated,
+    perform the sequence of upgrading the MDS cluster strictly.
+
+    See http://docs.ceph.com/docs/mimic/cephfs/upgrading/
+
+    For filesystems that have ever enabled snapshots, the multiple-active MDS
+    feature is disabled by the mimic monitor daemon. This will cause the "restore
+    previous max_mds" step in above URL to fail. To re-enable the feature,
+    either delete all old snapshots or scrub the whole filesystem:
+
+      - ``ceph daemon <mds of rank 0> scrub_path / force recursive repair``
+      - ``ceph daemon <mds of rank 0> scrub_path '~mdsdir' force recursive repair``
+
+  - Support has been added in Mimic for quotas in the Linux kernel client as of v4.17.
+
+    See http://docs.ceph.com/docs/mimic/cephfs/quota/
+
+  - Many fixes have been made to the MDS metadata balancer which distributes
+    load across MDS. It is expected that the automatic balancing should work
+    well for most use-cases. In Luminous, subtree pinning was advised as a
+    manual workaround for poor balancer behavior. This may no longer be
+    necessary so it is recommended to try experimentally disabling pinning as a
+    form of load balancing to see if the built-in balancer adequately works for
+    you. Please report any poor behavior post-upgrade.
+
+  - NFS-Ganesha is an NFS userspace server that can export shares from multiple
+    file systems, including CephFS. Support for this CephFS client has improved
+    significantly in Mimic. In particular, delegations are now supported through
+    the libcephfs library so that Ganesha may issue delegations to its NFS clients
+    allowing for safe write buffering and coherent read caching. Documentation
+    is also now available: http://docs.ceph.com/docs/mimic/cephfs/nfs/
+
+  - MDS uptime is now available in the output of the MDS admin socket ``status`` command.
+
+  - MDS performance counters for client requests now include average latency as well as the count.
+
+
+* *RBD*
+
+  - The RBD C API's `rbd_discard` method now enforces a maximum length of
+    2GB to match the C++ API's `Image::discard` method. This restriction
+    prevents overflow of the result code.
+
+  - The rbd CLI's ``lock list`` JSON and XML output has changed.
+
+  - The rbd CLI's ``showmapped`` JSON and XML output has changed.
+
+  - RBD now optionally supports simplified image clone semantics where
+    non-protected snapshots can be cloned; and snapshots with linked clones
+    can be removed and the space automatically reclaimed once all remaining
+    linked clones are detached. This feature is enabled by default if
+    the OSD "require-min-compat-client" flag is set to mimic or later; or can be
+    overridden via the "rbd_default_clone_format" configuration option.
+
+  - RBD now supports deep copy of images that preserves snapshot history.
+
+* *RGW*
+
+  - The RGW Beast frontend is now declared stable and ready for production use.
+    :ref:`rgw_frontends` for details.
+
+  - Civetweb frontend has been updated to the latest 1.10 release.
+
+  - The S3 API now has support for multi-factor authentication. Refer to
+    :ref:`rgw_mfa` for details.
+
+  - RGW now has a sync plugin to sync to AWS and clouds with S3-like APIs.
+
+* *MGR*
+
+  - The (read-only) Ceph manager dashboard introduced in Ceph Luminous has been
+    replaced with a new implementation, providing a drop-in replacement offering
+    a number of additional management features. To access the new dashboard, you
+    first need to define a username and password and create an SSL certificate.
+    See the :ref:`mgr-dashboard` for a feature overview and installation
+    instructions.
+
+  - The ``ceph-rest-api`` command-line tool (obsoleted by the MGR
+    `restful` module and deprecated since v12.2.5) has been dropped.
+
+    There is a MGR module called `restful` which provides similar functionality
+    via a "pass through" method. See http://docs.ceph.com/docs/master/mgr/restful
+    for details.
+
+  - New command to track throughput and IOPS statistics, also available in
+    ``ceph -s`` and previously in ``ceph -w``. To use this command, enable
+    the ``iostat`` Manager module and invoke it using ``ceph iostat``. See the
+    :ref:`iostat documentation <mgr-iostat-overview>` for details.
+
+* *build/packaging*
+
+  - The ``rcceph`` script (``systemd/ceph`` in the source code tree, shipped as
+    ``/usr/sbin/rcceph`` in the ceph-base package for CentOS and SUSE) has been
+    dropped. This script was used to perform admin operations (start, stop,
+    restart, etc.) on all OSD and/or MON daemons running on a given machine. This
+    functionality is provided by the systemd target units (``ceph-osd.target``,
+    ``ceph-mon.target``, etc.).
+
+  - The python-ceph-compat package is declared deprecated, and will be dropped
+    when all supported distros have completed the move to Python 3. It has
+    already been dropped from those supported distros where Python 3 is
+    standard and Python 2 is optional (currently only SUSE).
+
+  - Ceph codebase has now moved to the C++-17 standard.
+
+  - The Ceph LZ4 compression plugin is now enabled by default, and introduces a
+    new build dependency.
+
+
+Detailed Changelog
+------------------
+
+* arch/arm: set ceph_arch_aarch64_crc32 only if the build host supports crc32cx (`issue#19705 <http://tracker.ceph.com/issues/19705>`_, `pr#17420 <https://github.com/ceph/ceph/pull/17420>`_, Kefu Chai)
+* assert(false)->ceph_abort() (`pr#18072 <https://github.com/ceph/ceph/pull/18072>`_, Li Wang)
+* auth: keep /dev/urandom open for get_random_bytes (`issue#21401 <http://tracker.ceph.com/issues/21401>`_, `pr#17972 <https://github.com/ceph/ceph/pull/17972>`_, Casey Bodley)
+* bluestore: BlueStore::ExtentMap::dup impl (`pr#19719 <https://github.com/ceph/ceph/pull/19719>`_, Shinobu Kinjo)
+* bluestore: bluestore/NVMEDevice: accurate the latency perf counter of queue latency (`pr#17435 <https://github.com/ceph/ceph/pull/17435>`_, Ziye Yang, Pan Liu)
+* bluestore: bluestore/NVMEDevice: convert the legacy config opt related with SPDK (`pr#18502 <https://github.com/ceph/ceph/pull/18502>`_, Ziye Yang)
+* bluestore: bluestore/NVMEDevice: do not deference a dangling pointer (`pr#19067 <https://github.com/ceph/ceph/pull/19067>`_, Kefu Chai)
+* bluestore: bluestore/NVMEDevice: fix the bug in write function (`pr#17086 <https://github.com/ceph/ceph/pull/17086>`_, Ziye Yang, Pan Liu)
+* bluestore: bluestore/NVMeDevice: update NVMeDevice code due to SPDK upgrade (`pr#16927 <https://github.com/ceph/ceph/pull/16927>`_, Ziye Yang)
+* bluestore,build/ops: bluestore,cmake: enable building bluestore without aio (`pr#19017 <https://github.com/ceph/ceph/pull/19017>`_, Kefu Chai)
+* bluestore,build/ops: Build: create a proper WITH_BLUESTORE option (`pr#18357 <https://github.com/ceph/ceph/pull/18357>`_, Alan Somers)
+* bluestore,build/ops: ceph.spec.in,debian/rules: change aio-max-nr to 1048576 (`pr#17894 <https://github.com/ceph/ceph/pull/17894>`_, chenliuzhong)
+* bluestore,build/ops,tests: os: add compile option to build libbluefs.so (`pr#16733 <https://github.com/ceph/ceph/pull/16733>`_, Pan Liu)
+* bluestore,build/ops,tests: test/fio: fix build failure caused by sequencer replacement (`pr#20387 <https://github.com/ceph/ceph/pull/20387>`_, Igor Fedotov)
+* bluestore: ceph-bluestore-tool: better fsck/repair, bluefs-bdev-{expand,sizes} (`pr#17709 <https://github.com/ceph/ceph/pull/17709>`_, Sage Weil)
+* bluestore: ceph-bluestore-tool: check if bdev is empty on 'bluefs-bdev-expand' (`pr#17874 <https://github.com/ceph/ceph/pull/17874>`_, WANG Guoqin)
+* bluestore: ceph-bluestore-tool: link target shouldn't ending with "\n" (`pr#18585 <https://github.com/ceph/ceph/pull/18585>`_, Yao Zongyou)
+* bluestore,common: intarith: get rid of  P2\* and ROUND_UP\* macros (`pr#21085 <https://github.com/ceph/ceph/pull/21085>`_, xie xingguo)
+* bluestore:  comp_min_blob_size init error (`pr#18318 <https://github.com/ceph/ceph/pull/18318>`_, linbing)
+* bluestore: config: Change bluestore_cache_kv_max to type INT64 (`pr#20255 <https://github.com/ceph/ceph/pull/20255>`_, Zhi Zhang)
+* bluestore,core: ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing (`pr#18565 <https://github.com/ceph/ceph/pull/18565>`_, Sage Weil)
+* bluestore,core: common/options: bluefs_buffered_io=true by default (`pr#20542 <https://github.com/ceph/ceph/pull/20542>`_, Sage Weil)
+* bluestore,core: os/bluestore: compensate for bad freelistmanager size/blocks metadata (`issue#21089 <http://tracker.ceph.com/issues/21089>`_, `pr#17268 <https://github.com/ceph/ceph/pull/17268>`_, Sage Weil)
+* bluestore,core: os/bluestore: fix data read error injection in bluestore (`pr#19866 <https://github.com/ceph/ceph/pull/19866>`_, Sage Weil)
+* bluestore,core: os/bluestore: kv_max -> kv_min (`pr#20544 <https://github.com/ceph/ceph/pull/20544>`_, Sage Weil)
+* bluestore,core: os/bluestore: switch default allocator to stupid; test both bitmap and stupid in qa (`pr#16906 <https://github.com/ceph/ceph/pull/16906>`_, Sage Weil)
+* bluestore,core: src/bluestore/NVMEDevice: make all read use aio_submit (`pr#17655 <https://github.com/ceph/ceph/pull/17655>`_, Ziye Yang, Pan Liu)
+* bluestore,core,tests: test/unittest_bluefs: check whether rmdir success (`pr#15363 <https://github.com/ceph/ceph/pull/15363>`_, shiqi)
+* bluestore,core: tool: ceph-kvstore-tool doesn't umount BlueStore properly (`issue#21625 <http://tracker.ceph.com/issues/21625>`_, `pr#18083 <https://github.com/ceph/ceph/pull/18083>`_, Chang Liu)
+* bluestore: define default value of LoglevelV only once (3 templates) (`pr#20727 <https://github.com/ceph/ceph/pull/20727>`_, Matt Benjamin)
+* bluestore: drop unused friend class in SharedDriverQueueData (`pr#16894 <https://github.com/ceph/ceph/pull/16894>`_, Pan Liu)
+* bluestore: fix aio_t::rval  type (`issue#23527 <http://tracker.ceph.com/issues/23527>`_, `pr#21136 <https://github.com/ceph/ceph/pull/21136>`_, kungf)
+* bluestore: fix build on armhf (`pr#20951 <https://github.com/ceph/ceph/pull/20951>`_, Kefu Chai)
+* bluestore: fixed compilation error when enable spdk with gcc 4.8.5 (`pr#16945 <https://github.com/ceph/ceph/pull/16945>`_, Ziye Yang, Pan Liu)
+* bluestore: kv/RocksDBStore: extract common code to a new function (`pr#16532 <https://github.com/ceph/ceph/pull/16532>`_, Pan Liu)
+* bluestore/NVMEDevice: code cleanup (`pr#17284 <https://github.com/ceph/ceph/pull/17284>`_, Ziye Yang, Pan Liu)
+* bluestore: os/bluestore: add bluestore_prefer_deferred_size_hdd/ssd to tracked keys (`pr#17459 <https://github.com/ceph/ceph/pull/17459>`_, xie xingguo)
+* bluestore: os/bluestore: add discard method for ssd's performance (`pr#14727 <https://github.com/ceph/ceph/pull/14727>`_, Taeksang Kim)
+* bluestore: os/bluestore: Add lat record of deferred_queued and deferred_aio_wait (`pr#17015 <https://github.com/ceph/ceph/pull/17015>`_, lisali)
+* bluestore: os/bluestore: Add missing __func__ in dout (`pr#17903 <https://github.com/ceph/ceph/pull/17903>`_, lisali)
+* bluestore: os/bluestore: add perf counter for allocator fragmentation (`pr#21377 <https://github.com/ceph/ceph/pull/21377>`_, Igor Fedotov)
+* bluestore: os/bluestore: allocate entire write in one go (`pr#17698 <https://github.com/ceph/ceph/pull/17698>`_, Sage Weil)
+* bluestore: os/bluestore: allow reconstruction of osd data dir from bluestore bdev label (`pr#18256 <https://github.com/ceph/ceph/pull/18256>`_, Sage Weil)
+* bluestore: os/bluestore: alter the allow_eio policy regarding kernel's error list (`issue#23333 <http://tracker.ceph.com/issues/23333>`_, `pr#21306 <https://github.com/ceph/ceph/pull/21306>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: avoid excessive ops in _txc_release_alloc (`pr#18854 <https://github.com/ceph/ceph/pull/18854>`_, Igor Fedotov)
+* bluestore: os/bluestore: avoid omit cache for remove-collection (`pr#18785 <https://github.com/ceph/ceph/pull/18785>`_, Jianpeng Ma)
+* bluestore: os/bluestore: avoid overhead of std::function in blob_t (`pr#20294 <https://github.com/ceph/ceph/pull/20294>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: avoid unneeded BlobRefing in _do_read() (`pr#19864 <https://github.com/ceph/ceph/pull/19864>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: be more verbose when hitting unloaded shard in extent map (`pr#21245 <https://github.com/ceph/ceph/pull/21245>`_, Igor Fedotov)
+* bluestore: os/bluestore/BlueFS: compact log even when sync_metadata sees no work (`pr#17354 <https://github.com/ceph/ceph/pull/17354>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: Don't call debug related code under any condition (`pr#17627 <https://github.com/ceph/ceph/pull/17627>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: don't need wait for aio when using _sync_write (`pr#16066 <https://github.com/ceph/ceph/pull/16066>`_, Haodong Tang)
+* bluestore: os/bluestore/BlueFS: fix race with log flush during async log compaction (`issue#21878 <http://tracker.ceph.com/issues/21878>`_, `pr#18428 <https://github.com/ceph/ceph/pull/18428>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: move release unused extents work in _flush_and_syn_log (`pr#17684 <https://github.com/ceph/ceph/pull/17684>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: prevent _compact_log_async reentry (`issue#21250 <http://tracker.ceph.com/issues/21250>`_, `pr#17503 <https://github.com/ceph/ceph/pull/17503>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: Reduce unnecessary operations in collect_metadata (`pr#17995 <https://github.com/ceph/ceph/pull/17995>`_, Luo Kexue)
+* bluestore: os/bluestore/BlueFS: sanity check that alloc->allocate() won't return 0 (`pr#18259 <https://github.com/ceph/ceph/pull/18259>`_, xie xingguo)
+* bluestore: os/bluestore/BlueFS: several cleanups (`pr#17966 <https://github.com/ceph/ceph/pull/17966>`_, xie xingguo)
+* bluestore: os/bluestore/bluefs_types: make block_mask 64-bit (`pr#21629 <https://github.com/ceph/ceph/pull/21629>`_, Sage Weil)
+* bluestore: os/bluestore/BlueStore: ASAP wake up _kv_finalize_thread (`pr#18203 <https://github.com/ceph/ceph/pull/18203>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueStore: narrow deferred_lock in _deferred_submit_unlock (`pr#17628 <https://github.com/ceph/ceph/pull/17628>`_, Jianpeng Ma)
+* bluestore: os/bluestore: bluestore repair should use interval_set::union_insert (`pr#20900 <https://github.com/ceph/ceph/pull/20900>`_, Igor Fedotov)
+* bluestore: os/bluestore: cleanup around ExtentList, AllocExtent and bluestore_extent_t classes (`pr#20360 <https://github.com/ceph/ceph/pull/20360>`_, Igor Fedotov)
+* bluestore: os/bluestore:  clearer comments, not slower code (`pr#16872 <https://github.com/ceph/ceph/pull/16872>`_, Mark Nelson)
+* bluestore: os/bluestore: correctly check all block devices to decide if journal is_rotational (`issue#23141 <http://tracker.ceph.com/issues/23141>`_, `pr#20602 <https://github.com/ceph/ceph/pull/20602>`_, Greg Farnum)
+* bluestore: os/bluestore: delete redundant header file in KernelDevice.cc (`pr#18631 <https://github.com/ceph/ceph/pull/18631>`_, Jing Li)
+* bluestore: os/bluestore: do not assert if BlueFS rebalance is unable to allocate sufficient space (`pr#18494 <https://github.com/ceph/ceph/pull/18494>`_, Igor Fedotov)
+* bluestore: os/bluestore: do not core dump when BlueRocksEnv gets EEXIST error (`issue#20871 <http://tracker.ceph.com/issues/20871>`_, `pr#17357 <https://github.com/ceph/ceph/pull/17357>`_, liuchang0812)
+* bluestore: os/bluestore: do not core dump when we try to open kvstore twice (`pr#18161 <https://github.com/ceph/ceph/pull/18161>`_, Chang Liu)
+* bluestore: os/bluestore: do not release empty bluefs_extents_reclaiming (`pr#18671 <https://github.com/ceph/ceph/pull/18671>`_, Igor Fedotov)
+* bluestore: os/bluestore: do not segv on kraken upgrade debug print (`issue#20977 <http://tracker.ceph.com/issues/20977>`_, `pr#16992 <https://github.com/ceph/ceph/pull/16992>`_, Sage Weil)
+* bluestore: os/bluestore: don't re-initialize csum-setting for existing blobs (`issue#21175 <http://tracker.ceph.com/issues/21175>`_, `pr#17398 <https://github.com/ceph/ceph/pull/17398>`_, xie xingguo)
+* bluestore: os/bluestore: do SSD discard on mkfs (`pr#20897 <https://github.com/ceph/ceph/pull/20897>`_, Igor Fedotov)
+* bluestore: os/bluestore: drop deferred_submit_lock, fix aio leak (`issue#21171 <http://tracker.ceph.com/issues/21171>`_, `pr#17352 <https://github.com/ceph/ceph/pull/17352>`_, Sage Weil)
+* bluestore: os/bluestore: drop unused function declaration (`pr#18075 <https://github.com/ceph/ceph/pull/18075>`_, Li Wang)
+* bluestore: os/bluestore: drop unused param "what" in apply() (`pr#17251 <https://github.com/ceph/ceph/pull/17251>`_, songweibin)
+* bluestore: os/bluestore: _dump_onode() don't prolongate Onode anymore (`pr#19841 <https://github.com/ceph/ceph/pull/19841>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: dynamic CF configuration; put pglog omap in separate CF (`pr#18224 <https://github.com/ceph/ceph/pull/18224>`_, Sage Weil)
+* bluestore: os/bluestore: enlarege aligned_size avoid too many vector(> IOV_MAX) (`issue#21932 <http://tracker.ceph.com/issues/21932>`_, `pr#18828 <https://github.com/ceph/ceph/pull/18828>`_, Jianpeng Ma)
+* bluestore: os/bluestore: ExtentMap::reshard - fix wrong shard length (`pr#17334 <https://github.com/ceph/ceph/pull/17334>`_, chenliuzhong)
+* bluestore: os/bluestore: fail early on very large objects (`issue#20923 <http://tracker.ceph.com/issues/20923>`_, `pr#16924 <https://github.com/ceph/ceph/pull/16924>`_, Sage Weil)
+* bluestore: os/bluestore: fix another aio stall/deadlock (`issue#21470 <http://tracker.ceph.com/issues/21470>`_, `pr#18118 <https://github.com/ceph/ceph/pull/18118>`_, Sage Weil)
+* bluestore: os/bluestore: fix broken cap in _balance_bluefs_freespace() (`pr#21097 <https://github.com/ceph/ceph/pull/21097>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix clone dirty_range again (`issue#20983 <http://tracker.ceph.com/issues/20983>`_, `pr#16994 <https://github.com/ceph/ceph/pull/16994>`_, Sage Weil)
+* bluestore: os/bluestore: fix dirty_shard off-by-one (`pr#16850 <https://github.com/ceph/ceph/pull/16850>`_, Sage Weil)
+* bluestore: os/bluestore: fix exceeding the max IO queue depth in KernelDevice (`issue#23246 <http://tracker.ceph.com/issues/23246>`_, `pr#20996 <https://github.com/ceph/ceph/pull/20996>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: fix potential assert when splitting collection (`pr#19519 <https://github.com/ceph/ceph/pull/19519>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix SharedBlob unregistration (`issue#22039 <http://tracker.ceph.com/issues/22039>`_, `pr#18805 <https://github.com/ceph/ceph/pull/18805>`_, Sage Weil)
+* bluestore: os/bluestore: fix some code formatting (`pr#21037 <https://github.com/ceph/ceph/pull/21037>`_, Gu Zhongyan)
+* bluestore: os/bluestore: fix the allocate in bluefs (`pr#19030 <https://github.com/ceph/ceph/pull/19030>`_, tangwenjun)
+* bluestore: os/bluestore: fix the demotion in StupidAllocator::init_rm_free (`pr#20430 <https://github.com/ceph/ceph/pull/20430>`_, Kefu Chai)
+* bluestore: os/bluestore: fix the wrong usage for map_any (`pr#18939 <https://github.com/ceph/ceph/pull/18939>`_, Jianpeng Ma)
+* bluestore: os/bluestore: fix wrong usage for BlueFS::_allocate (`pr#20708 <https://github.com/ceph/ceph/pull/20708>`_, Jianpeng Ma)
+* bluestore: os/bluestore: free the spdk qpair resource correctly in destructor of SharedDriverQueueData (`pr#20929 <https://github.com/ceph/ceph/pull/20929>`_, Jianyu Li)
+* bluestore: os/bluestore: handle small main device properly (`pr#17416 <https://github.com/ceph/ceph/pull/17416>`_, xie xingguo)
+* bluestore: os/bluestore: ignore 0x2000~2000 extent oddity from luminous upgrade (`issue#21408 <http://tracker.ceph.com/issues/21408>`_, `pr#17845 <https://github.com/ceph/ceph/pull/17845>`_, Sage Weil)
+* bluestore: os/bluestore: implement BlueStore repair (`pr#19843 <https://github.com/ceph/ceph/pull/19843>`_, Igor Fedotov)
+* bluestore: os/bluestore: make bluefs behave better near enospc (`pr#18120 <https://github.com/ceph/ceph/pull/18120>`_, Sage Weil)
+* bluestore: os/bluestore: mark derivatives of AioContext as final (`pr#20227 <https://github.com/ceph/ceph/pull/20227>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: move aio_callback{,_priv} to base class BlockDevice (`pr#17002 <https://github.com/ceph/ceph/pull/17002>`_, mychoxin)
+* bluestore: os/bluestore: move assert of read/write to base class (`pr#17033 <https://github.com/ceph/ceph/pull/17033>`_, mychoxin)
+* bluestore: os/bluestore: move size and block_size to the base class BlockDevice (`pr#16886 <https://github.com/ceph/ceph/pull/16886>`_, Pan Liu)
+* bluestore: os/bluestore: no need to fsync when failed to write label (`pr#20092 <https://github.com/ceph/ceph/pull/20092>`_, tangwenjun)
+* bluestore: os/bluestore: no trim debug noise if there is no trimming to be done (`pr#20684 <https://github.com/ceph/ceph/pull/20684>`_, Sage Weil)
+* bluestore: os/bluestore/NVMEDevice: change write_bl to bl (`pr#17145 <https://github.com/ceph/ceph/pull/17145>`_, Ziye Yang, Pan Liu)
+* bluestore: os/bluestore/NVMEDevice: fix the nvme queue depth issue (`pr#17200 <https://github.com/ceph/ceph/pull/17200>`_, Ziye Yang, Pan Liu)
+* bluestore: os/bluestore/NVMEDevice: Remove using dpdk thread (`pr#17769 <https://github.com/ceph/ceph/pull/17769>`_, Ziye Yang, Pan Liu)
+* bluestore: os/bluestore: OpSequencer: reduce kv_submitted_waiters if _is_all_kv_submitted() return true (`pr#18622 <https://github.com/ceph/ceph/pull/18622>`_, Jianpeng Ma)
+* bluestore: os/bluestore: optimize _collection_list (`pr#18777 <https://github.com/ceph/ceph/pull/18777>`_, Jianpeng Ma)
+* bluestore: os/bluestore: pass strict flag to bluestore_blob_use_tracker_t::equal() (`pr#15705 <https://github.com/ceph/ceph/pull/15705>`_, xie xingguo)
+* bluestore: os/bluestore: Prealloc memory avoid realloc in list_collection (`pr#18804 <https://github.com/ceph/ceph/pull/18804>`_, Jianpeng Ma)
+* bluestore: os/bluestore: prevent mount if osd_max_object_size >= 4G (`pr#19043 <https://github.com/ceph/ceph/pull/19043>`_, Sage Weil)
+* bluestore: os/bluestore: print aio in batch (`pr#18873 <https://github.com/ceph/ceph/pull/18873>`_, Kefu Chai)
+* bluestore: os/bluestore: print leaked extents to debug output (`pr#17225 <https://github.com/ceph/ceph/pull/17225>`_, Sage Weil)
+* bluestore: os/bluestore: propagate read-EIO to high level callers (`pr#17744 <https://github.com/ceph/ceph/pull/17744>`_, xie xingguo)
+* bluestore: os/bluestore: put cached attrs in correct mempool (`issue#21417 <http://tracker.ceph.com/issues/21417>`_, `pr#18001 <https://github.com/ceph/ceph/pull/18001>`_, Sage Weil)
+* bluestore: os/bluestore: recalc_allocated() when decoding bluefs_fnode_t (`issue#23212 <http://tracker.ceph.com/issues/23212>`_, `pr#20701 <https://github.com/ceph/ceph/pull/20701>`_, Jianpeng Ma, Kefu Chai)
+* bluestore: os/bluestore: reduce meaningless flush (`pr#19027 <https://github.com/ceph/ceph/pull/19027>`_, tangwenjun)
+* bluestore: os/bluestore: refactor FreeListManager to get clearer view on the number (`issue#22535 <http://tracker.ceph.com/issues/22535>`_, `pr#19718 <https://github.com/ceph/ceph/pull/19718>`_, Igor Fedotov)
+* bluestore: os/bluestore: release disk extents in bulky manner (`pr#17913 <https://github.com/ceph/ceph/pull/17913>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: remove ineffective BlueFS fnode extent calculation (`pr#18905 <https://github.com/ceph/ceph/pull/18905>`_, Igor Fedotov)
+* bluestore: os/bluestore: remove unused parameters (`pr#18635 <https://github.com/ceph/ceph/pull/18635>`_, Jianpeng Ma)
+* bluestore: os/bluestore: remove unused variable (`pr#21063 <https://github.com/ceph/ceph/pull/21063>`_, Gu Zhongyan)
+* bluestore: os/bluestore: remove useless function submit (`pr#17537 <https://github.com/ceph/ceph/pull/17537>`_, mychoxin)
+* bluestore: os/bluestore: reorder members of bluefs_extent_t for space efficiency (`pr#21034 <https://github.com/ceph/ceph/pull/21034>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: replace dout with ldout in StupidAllocator (`pr#17404 <https://github.com/ceph/ceph/pull/17404>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: report error and quit correctly when disk error happens (`issue#21263 <http://tracker.ceph.com/issues/21263>`_, `pr#17522 <https://github.com/ceph/ceph/pull/17522>`_, Pan Liu)
+* bluestore: os/bluestore: Revert "os/bluestore: allow multiple DeferredBatches in flight at once" (`issue#20925 <http://tracker.ceph.com/issues/20925>`_, `issue#20295 <http://tracker.ceph.com/issues/20295>`_, `pr#16900 <https://github.com/ceph/ceph/pull/16900>`_, Sage Weil)
+* bluestore: os/bluestore: s/bluefs_total/bluefs_free/ (`pr#21036 <https://github.com/ceph/ceph/pull/21036>`_, xie xingguo)
+* bluestore: os/bluestore: separate finisher for deferred_try_submit (`issue#21207 <http://tracker.ceph.com/issues/21207>`_, `pr#17409 <https://github.com/ceph/ceph/pull/17409>`_, Sage Weil)
+* bluestore: os/bluestore: set bitmap freelist resolution to min_alloc_size (`pr#17610 <https://github.com/ceph/ceph/pull/17610>`_, Sage Weil)
+* bluestore: os/bluestore: shrink aio submit size to pending value (`pr#17588 <https://github.com/ceph/ceph/pull/17588>`_, kungf)
+* bluestore: os/bluestore: silence -Wreturn-type warning (`pr#18286 <https://github.com/ceph/ceph/pull/18286>`_, Kefu Chai)
+* bluestore: os/bluestore: support calculate cost when using spdk (`pr#17091 <https://github.com/ceph/ceph/pull/17091>`_, Ziye Yang, Pan Liu)
+* bluestore: os/bluestore: synchronous on_applied completions (`pr#18196 <https://github.com/ceph/ceph/pull/18196>`_, Sage Weil)
+* bluestore: os/bluestore: trim cache every 50ms (instead of 200ms) (`pr#20498 <https://github.com/ceph/ceph/pull/20498>`_, Sage Weil)
+* bluestore: os/bluestore: update description for bluestore_compression\_[min|max|_blob_size options (`pr#21244 <https://github.com/ceph/ceph/pull/21244>`_, Igor Fedotov)
+* bluestore: os/bluestore: using macro OBJECT_MAX_SIZE to check osd_max_object_size (`pr#19622 <https://github.com/ceph/ceph/pull/19622>`_, Jianpeng Ma)
+* bluestore: osd/bluestore: delete unused variable in KernelDevice (`pr#20857 <https://github.com/ceph/ceph/pull/20857>`_, Leo Zhang)
+* bluestore: osd,os/bluestore: Display current size of osd_max_object_size (`pr#19725 <https://github.com/ceph/ceph/pull/19725>`_, Shinobu Kinjo)
+* bluestore: Revert "os/bluestore: pass strict flag to bluestore_blob_use_tracker_t::equal()" (`issue#21293 <http://tracker.ceph.com/issues/21293>`_, `pr#17569 <https://github.com/ceph/ceph/pull/17569>`_, Sage Weil)
+* bluestore,rgw: rgw,unittest_bit_alloc: silence clang analyzer warning (`pr#17294 <https://github.com/ceph/ceph/pull/17294>`_, Kefu Chai)
+* bluestore,tests: objectstore/store_test: fix lack of flush prior to collection_empty()… (`issue#22409 <http://tracker.ceph.com/issues/22409>`_, `pr#19764 <https://github.com/ceph/ceph/pull/19764>`_, Igor Fedotov)
+* bluestore,tests: Revert "bluestore/fio: Fixed problem with all objects having the same hash (`pr#18352 <https://github.com/ceph/ceph/pull/18352>`_, Radoslaw Zarzynski)
+* bluestore,tools: ceph-bluestore-tool: create out_dir before create full path  of kvdb (`pr#18367 <https://github.com/ceph/ceph/pull/18367>`_, Leo Zhang)
+* bluestore,tools: os/bluestore/bluestore_tool: add log-dump command to dump bluefs's log (`pr#18535 <https://github.com/ceph/ceph/pull/18535>`_, Yang Honggang)
+* build: fix dpdk build error (`pr#18087 <https://github.com/ceph/ceph/pull/18087>`_, chunmei)
+* build mimic-dev1 with gcc 7 (`issue#22438 <http://tracker.ceph.com/issues/22438>`_, `pr#19548 <https://github.com/ceph/ceph/pull/19548>`_, Kefu Chai)
+* build/ops: automake: remove files required by automake (`pr#17937 <https://github.com/ceph/ceph/pull/17937>`_, Kefu Chai)
+* build/ops: blkin: link against lttng-ust-fork (`pr#17673 <https://github.com/ceph/ceph/pull/17673>`_, Mohamad Gebai)
+* build/ops: boost: remove boost submodule (`pr#17405 <https://github.com/ceph/ceph/pull/17405>`_, Kefu Chai)
+* build/ops: build: do_cmake: allow ARGS to be overridden (`pr#19876 <https://github.com/ceph/ceph/pull/19876>`_, Abhishek Lekshmanan)
+* build/ops: build: remove PGMap.cc from libcommon (`pr#18496 <https://github.com/ceph/ceph/pull/18496>`_, Sage Weil)
+* build/ops: ceph-disk activate unlocks bluestore data partition (`issue#20488 <http://tracker.ceph.com/issues/20488>`_, `pr#16357 <https://github.com/ceph/ceph/pull/16357>`_, Felix Winterhalter)
+* build/ops: ceph_disk: allow "no fsid" on activate (`pr#18991 <https://github.com/ceph/ceph/pull/18991>`_, Dan Mick)
+* build/ops,cephfs: ceph-object-corpus: update to fix make check (`pr#21261 <https://github.com/ceph/ceph/pull/21261>`_, Patrick Donnelly)
+* build/ops,cephfs: cmake, test/fs, client: fix build with clang (`pr#20392 <https://github.com/ceph/ceph/pull/20392>`_, Adam C. Emerson)
+* build/ops: ceph.spec: use devtoolset-6-gcc-c++ on aarch64 (`issue#22301 <http://tracker.ceph.com/issues/22301>`_, `pr#19341 <https://github.com/ceph/ceph/pull/19341>`_, Kefu Chai)
+* build/ops: ceph-volume: Require lvm2, move to osd package (`issue#22443 <http://tracker.ceph.com/issues/22443>`_, `pr#19529 <https://github.com/ceph/ceph/pull/19529>`_, Theofilos Mouratidis)
+* build/ops: ceph-volume: tests add tests for the is_mounted utility (`pr#16962 <https://github.com/ceph/ceph/pull/16962>`_, Alfredo Deza)
+* build/ops: change WITH_SYSTEMD default to ON (`pr#20404 <https://github.com/ceph/ceph/pull/20404>`_, Nathan Cutler)
+* build/ops: cmake/BuildBoost: fixes to ready seastar (`pr#20616 <https://github.com/ceph/ceph/pull/20616>`_, Kefu Chai, Casey Bodley)
+* build/ops: cmake,deb: install system units using cmake (`pr#20618 <https://github.com/ceph/ceph/pull/20618>`_, Kefu Chai)
+* build/ops:  cmake: link libcommon with libstdc++ statically if WITH_STATIC_LIBSTDCXX (`issue#22438 <http://tracker.ceph.com/issues/22438>`_, `pr#19515 <https://github.com/ceph/ceph/pull/19515>`_, Kefu Chai)
+* build/ops: cmake,make-dist: bump up boost version to 1.67 (`pr#21572 <https://github.com/ceph/ceph/pull/21572>`_, Kefu Chai)
+* build/ops: cmake,mds: detect std::map::merge() before using it (`pr#21211 <https://github.com/ceph/ceph/pull/21211>`_, Willem Jan Withagen, Kefu Chai)
+* build/ops: cmake/mgr: use Python 3 virtualenv if mgr subinterpreter is Python 3 (`pr#21446 <https://github.com/ceph/ceph/pull/21446>`_, Nathan Cutler)
+* build/ops,common: cmake, common: silence cmake and gcc warnings (`issue#23774 <http://tracker.ceph.com/issues/23774>`_, `pr#21484 <https://github.com/ceph/ceph/pull/21484>`_, Kefu Chai)
+* build/ops: common/time: add time.h for Alpine build (`pr#19863 <https://github.com/ceph/ceph/pull/19863>`_, huanwen ren)
+* build/ops,common: Update C++ standard to 14 and clean up (`pr#19490 <https://github.com/ceph/ceph/pull/19490>`_, Adam C. Emerson)
+* build/ops,core: ceph-crush-location: remove (`pr#19881 <https://github.com/ceph/ceph/pull/19881>`_, Sage Weil)
+* build/ops,core: ceph-volume: do not use --key during mkfs (`issue#22283 <http://tracker.ceph.com/issues/22283>`_, `pr#19276 <https://github.com/ceph/ceph/pull/19276>`_, Kefu Chai, Sage Weil)
+* build/ops,core: /etc/sysconfig/ceph: remove jemalloc option (`issue#20557 <http://tracker.ceph.com/issues/20557>`_, `pr#18487 <https://github.com/ceph/ceph/pull/18487>`_, Sage Weil)
+* build/ops,core: mimic: cmake,common,filestore: silence gcc-8 warnings/errors (`pr#21862 <https://github.com/ceph/ceph/pull/21862>`_, Kefu Chai)
+* build/ops,core: mimic: cmake: do not check for aligned_alloc() anymore (`issue#23653 <http://tracker.ceph.com/issues/23653>`_, `pr#22048 <https://github.com/ceph/ceph/pull/22048>`_, Kefu Chai)
+* build/ops,core: msg/async: update to work with dpdk shipped with spdk v17.10 (`pr#19470 <https://github.com/ceph/ceph/pull/19470>`_, Kefu Chai)
+* build/ops,core: zstd: Upgrade to v1.3.2 (`pr#18407 <https://github.com/ceph/ceph/pull/18407>`_, Adam C. Emerson)
+* build/ops: debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move (`issue#22319 <http://tracker.ceph.com/issues/22319>`_, `pr#19328 <https://github.com/ceph/ceph/pull/19328>`_, Sage Weil)
+* build/ops: debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move (`issue#22319 <http://tracker.ceph.com/issues/22319>`_, `pr#19395 <https://github.com/ceph/ceph/pull/19395>`_, Kefu Chai, Sage Weil)
+* build/ops: debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move (`pr#19356 <https://github.com/ceph/ceph/pull/19356>`_, Kefu Chai)
+* build/ops: debian: fix package relationships after 40caf6a6 (`issue#21762 <http://tracker.ceph.com/issues/21762>`_, `pr#18474 <https://github.com/ceph/ceph/pull/18474>`_, Kefu Chai)
+* build/ops: debian: lock ceph user during purge (`pr#15118 <https://github.com/ceph/ceph/pull/15118>`_, Caleb Boylan)
+* build/ops: debian/rules: no more ChangeLog (`pr#18023 <https://github.com/ceph/ceph/pull/18023>`_, Sage Weil)
+* build/ops: debian/rules: strip ceph-base libraries (`issue#22640 <http://tracker.ceph.com/issues/22640>`_, `pr#19870 <https://github.com/ceph/ceph/pull/19870>`_, Sage Weil)
+* build/ops: do\_{cmake,freebsd}: Don't invoke nproc(1) on FreeBSD (`pr#17949 <https://github.com/ceph/ceph/pull/17949>`_, Alan Somers)
+* build/ops: dpdk: remove redundant dpdk submodule (`pr#18712 <https://github.com/ceph/ceph/pull/18712>`_, chunmei)
+* build/ops: EventKqueue: Clang want realloc return to be typed (`pr#21550 <https://github.com/ceph/ceph/pull/21550>`_, Willem Jan Withagen)
+* build/ops: filestore,rgw: fix types/casts making clang on 32-Bit working (`pr#21055 <https://github.com/ceph/ceph/pull/21055>`_, Daniel Glaser)
+* build/ops: Fix ppc64 support for ceph (`pr#16753 <https://github.com/ceph/ceph/pull/16753>`_, Boris Ranto)
+* build/ops: Fix two dpdk assert happened in dpdk library (`pr#18409 <https://github.com/ceph/ceph/pull/18409>`_, chunmei)
+* build/ops: FreeBSD: add new required packages to be installed (`pr#21349 <https://github.com/ceph/ceph/pull/21349>`_, Willem Jan Withagen)
+* build/ops: githubmap: add some known Ceph reviewers (`pr#17507 <https://github.com/ceph/ceph/pull/17507>`_, Patrick Donnelly)
+* build/ops: .githubmap: Add wjwithagen as a known Ceph reviewer (`pr#17518 <https://github.com/ceph/ceph/pull/17518>`_, Willem Jan Withagen)
+* build/ops: .githubmap: Update (`pr#18230 <https://github.com/ceph/ceph/pull/18230>`_, Sage Weil)
+* build/ops: .gitignore: allow debian .patch files (`pr#17577 <https://github.com/ceph/ceph/pull/17577>`_, Ken Dreyer)
+* build/ops: include: compat.h, fix the return result of pthread_set_name() (`pr#20474 <https://github.com/ceph/ceph/pull/20474>`_, Willem Jan Withagen)
+* build/ops: install-deps: Add support for 'opensuse-tumbleweed' (`pr#21650 <https://github.com/ceph/ceph/pull/21650>`_, Ricardo Marques)
+* build/ops: install-deps.sh: avoid re-installing g++-7 (`pr#19468 <https://github.com/ceph/ceph/pull/19468>`_, Kefu Chai)
+* build/ops: install-deps.sh, cmake: use GCC-7 on xenial also (`pr#19418 <https://github.com/ceph/ceph/pull/19418>`_, Kefu Chai)
+* build/ops: install-deps.sh: install new gcc as the default the right way (`pr#19417 <https://github.com/ceph/ceph/pull/19417>`_, Kefu Chai)
+* build/ops: install-deps.sh: pass --no-recommends to zypper (`issue#22998 <http://tracker.ceph.com/issues/22998>`_, `pr#20434 <https://github.com/ceph/ceph/pull/20434>`_, Nathan Cutler)
+* build/ops: install-deps.sh: set python2 %bcond by environment (`issue#22999 <http://tracker.ceph.com/issues/22999>`_, `pr#20436 <https://github.com/ceph/ceph/pull/20436>`_, Nathan Cutler)
+* build/ops: install-deps.sh: use DTS on centos if GCC is too old (`pr#19398 <https://github.com/ceph/ceph/pull/19398>`_, Kefu Chai)
+* build/ops: install-deps.sh: use tee for writing a file (`pr#19516 <https://github.com/ceph/ceph/pull/19516>`_, Kefu Chai)
+* build/ops: install-deps: use DTS-7 on aarch64 and only download mirrored package indexes (`pr#19645 <https://github.com/ceph/ceph/pull/19645>`_, Kefu Chai, Songbo Wang)
+* build/ops: libmpem: Revert "submodule: make libmpem as a submodule." (`pr#18414 <https://github.com/ceph/ceph/pull/18414>`_, Jianpeng Ma)
+* build/ops: logrotate: add systemd reload in logrotate in case of centos minimal without killall (`pr#16586 <https://github.com/ceph/ceph/pull/16586>`_, Tianshan Qu)
+* build/ops: make-dist,cmake: avoid re-downloading boost (`pr#19124 <https://github.com/ceph/ceph/pull/19124>`_, Kefu Chai)
+* build/ops: make-dist,cmake: move boost tarball location to download.ceph.com (`pr#17980 <https://github.com/ceph/ceph/pull/17980>`_, Sage Weil)
+* build/ops: make-dist,cmake: Try multiple URLs to download boost before failing (`pr#18048 <https://github.com/ceph/ceph/pull/18048>`_, Brad Hubbard)
+* build/ops: make-dist: fall back to python3 (`pr#21127 <https://github.com/ceph/ceph/pull/21127>`_, Nathan Cutler)
+* build/ops,mgr: mgr/dashboard: build tweaks (`pr#20752 <https://github.com/ceph/ceph/pull/20752>`_, John Spray)
+* build/ops,mgr: mgr/dashboard: remove node/npm system installation (`pr#20898 <https://github.com/ceph/ceph/pull/20898>`_, Tiago Melo)
+* build/ops,mgr: packaging: explicit jinja2 dependency for dashboard (`issue#22457 <http://tracker.ceph.com/issues/22457>`_, `pr#19598 <https://github.com/ceph/ceph/pull/19598>`_, John Spray)
+* build/ops,mgr,tests: mgr/dashboard: replace dashboard with dashboard_v2 (`pr#20912 <https://github.com/ceph/ceph/pull/20912>`_, Ricardo Dias)
+* build/ops: mimic: cmake: use javac -h for creating JNI native headers (`issue#24012 <http://tracker.ceph.com/issues/24012>`_, `pr#21824 <https://github.com/ceph/ceph/pull/21824>`_, Kefu Chai)
+* build/ops: mimic: silence various warnings to enable GCC-8 build (`pr#22081 <https://github.com/ceph/ceph/pull/22081>`_, Adam C. Emerson, Kefu Chai)
+* build/ops: mon,osd: do not use crush_device_class file to initalize class for new osds (`pr#19939 <https://github.com/ceph/ceph/pull/19939>`_, Sage Weil)
+* build/ops: mstart.sh: support read CLUSTERS_LIST from env var (`pr#16988 <https://github.com/ceph/ceph/pull/16988>`_, Jiaying Ren)
+* build/ops: os/CMakeLists: fix link errro when enable WITH_PMEM=ON (`pr#20658 <https://github.com/ceph/ceph/pull/20658>`_, Jianpeng Ma)
+* build/ops: osdc,os,osd: fix build on osx (`pr#20029 <https://github.com/ceph/ceph/pull/20029>`_, Kefu Chai)
+* build/ops: python-numpy-devel build dependency for SUSE (`issue#21176 <http://tracker.ceph.com/issues/21176>`_, `pr#17366 <https://github.com/ceph/ceph/pull/17366>`_, Nathan Cutler)
+* build/ops: qa/tests - added for the suites with subset be able to use 'testing' … (`pr#21454 <https://github.com/ceph/ceph/pull/21454>`_, Yuri Weinstein)
+* build/ops,rbd: ceph-dencoder: moved RBD types outside of RGW preprocessor guard (`issue#22321 <http://tracker.ceph.com/issues/22321>`_, `pr#19343 <https://github.com/ceph/ceph/pull/19343>`_, Jason Dillaman)
+* build/ops: rbdmap: fix umount when multiple mounts use the same RBD (`pr#17978 <https://github.com/ceph/ceph/pull/17978>`_, Alexandre Marangone)
+* build/ops: Revert "make-dist: add OBS-specific release suffix on SUSE" (`pr#20813 <https://github.com/ceph/ceph/pull/20813>`_, Nathan Cutler)
+* build/ops,rgw: radosgw: Make compilation with CryptoPP possible (`pr#14955 <https://github.com/ceph/ceph/pull/14955>`_, Adam Kupczyk)
+* build/ops: rocksdb: do not use aligned_alloc (`issue#23653 <http://tracker.ceph.com/issues/23653>`_, `pr#21632 <https://github.com/ceph/ceph/pull/21632>`_, Kefu Chai)
+* build/ops: rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move (`issue#22558 <http://tracker.ceph.com/issues/22558>`_, `pr#19777 <https://github.com/ceph/ceph/pull/19777>`_, Kefu Chai)
+* build/ops: rpm: build-depends on "cunit-devel" for suse (`pr#18997 <https://github.com/ceph/ceph/pull/18997>`_, Kefu Chai)
+* build/ops: rpm: conditionalize Python 2 availability to enable Ceph build on Python 3-only system (`pr#20018 <https://github.com/ceph/ceph/pull/20018>`_, Nathan Cutler)
+* build/ops: rpm,debian: Ensure all ceph-disk runtime dependencies are declared for ceph-base (`issue#23657 <http://tracker.ceph.com/issues/23657>`_, `pr#21356 <https://github.com/ceph/ceph/pull/21356>`_, Nathan Cutler)
+* build/ops: rpm,deb: package ceph-kvstore-tool man page (`pr#17387 <https://github.com/ceph/ceph/pull/17387>`_, Sage Weil)
+* build/ops: rpm: drop legacy librbd.so.1 symlink in /usr/lib64/qemu (`pr#17324 <https://github.com/ceph/ceph/pull/17324>`_, Nathan Cutler)
+* build/ops: rpm: fix _defined_if_python2_absent conditional (`pr#20166 <https://github.com/ceph/ceph/pull/20166>`_, Nathan Cutler)
+* build/ops: rpm: fix systemd macros for ceph-volume@.service (`issue#22217 <http://tracker.ceph.com/issues/22217>`_, `pr#19081 <https://github.com/ceph/ceph/pull/19081>`_, Nathan Cutler)
+* build/ops: rpm: move ceph-\*-tool binaries out of ceph-test subpackage (`issue#21762 <http://tracker.ceph.com/issues/21762>`_, `pr#18289 <https://github.com/ceph/ceph/pull/18289>`_, Nathan Cutler)
+* build/ops: rpm: Python 3-only ceph-disk and ceph-volume (`pr#20140 <https://github.com/ceph/ceph/pull/20140>`_, Nathan Cutler)
+* build/ops: rpm: recommend chrony instead of ntp-daemon (`pr#20138 <https://github.com/ceph/ceph/pull/20138>`_, Nathan Cutler)
+* build/ops: rpm: recommend python-influxdb with ceph-mgr (`pr#18511 <https://github.com/ceph/ceph/pull/18511>`_, Nathan Cutler, Tim Serong)
+* build/ops: rpm: Revert "ceph.spec: work around build.opensuse.org" (`pr#21716 <https://github.com/ceph/ceph/pull/21716>`_, Nathan Cutler)
+* build/ops: rpm: rip out rcceph script (`pr#19899 <https://github.com/ceph/ceph/pull/19899>`_, Nathan Cutler)
+* build/ops: rpm: selinux-policy fixes (`pr#19026 <https://github.com/ceph/ceph/pull/19026>`_, Brad Hubbard)
+* build/ops: rpm: set build parallelism based on available memory (`pr#19122 <https://github.com/ceph/ceph/pull/19122>`_, Nathan Cutler, Richard Brown)
+* build/ops: rpm: set permissions 0755 on rbd resource agent (`issue#22362 <http://tracker.ceph.com/issues/22362>`_, `pr#19494 <https://github.com/ceph/ceph/pull/19494>`_, Nathan Cutler)
+* build/ops: run-make-check.sh: fix SUSE support (`issue#22875 <http://tracker.ceph.com/issues/22875>`_, `pr#20234 <https://github.com/ceph/ceph/pull/20234>`_, Nathan Cutler)
+* build/ops: run-make-check.sh: handle Python 2 absence (`issue#23035 <http://tracker.ceph.com/issues/23035>`_, `pr#20480 <https://github.com/ceph/ceph/pull/20480>`_, Nathan Cutler)
+* build/ops: run-make-check.sh: run ulimit without sudo (`pr#17361 <https://github.com/ceph/ceph/pull/17361>`_, yang.wang)
+* build/ops: script/build-integration-branch: print pr url list with titles (`pr#17426 <https://github.com/ceph/ceph/pull/17426>`_, Sage Weil)
+* build/ops: selinux: Allow nvme devices (`issue#19200 <http://tracker.ceph.com/issues/19200>`_, `pr#15597 <https://github.com/ceph/ceph/pull/15597>`_, Boris Ranto)
+* build/ops: setup-virtualenv.sh: do not hardcode python binary (`issue#23437 <http://tracker.ceph.com/issues/23437>`_, `pr#21002 <https://github.com/ceph/ceph/pull/21002>`_, Nathan Cutler)
+* build/ops: spdk: update SPDK to fix the build failure on aarch64 (`pr#20134 <https://github.com/ceph/ceph/pull/20134>`_, Tone Zhang, Kefu Chai)
+* build/ops:  spdk: update SPDK to v17.10 (`pr#19208 <https://github.com/ceph/ceph/pull/19208>`_, Kefu Chai)
+* build/ops: spdk: update submodule to more recent upstream (`pr#20077 <https://github.com/ceph/ceph/pull/20077>`_, Nathan Cutler)
+* build/ops: specs: require of e2fsprogs (`pr#21345 <https://github.com/ceph/ceph/pull/21345>`_, Guillaume Abrioux)
+* build/ops: src/script/build-integration-branch (`pr#17382 <https://github.com/ceph/ceph/pull/17382>`_, Sage Weil)
+* build/ops: src: s/pip --use-wheel/pip/ (`pr#21159 <https://github.com/ceph/ceph/pull/21159>`_, Kefu Chai)
+* build/ops: submodule: make libmpem as a submodule (`pr#17036 <https://github.com/ceph/ceph/pull/17036>`_, Jianpeng Ma)
+* build/ops: sysctl.d: set kernel.pid_max=4194304 (`issue#21929 <http://tracker.ceph.com/issues/21929>`_, `pr#18544 <https://github.com/ceph/ceph/pull/18544>`_, David Disseldorp)
+* build/ops: systemd: rbd-mirror does not start on reboot (`pr#17969 <https://github.com/ceph/ceph/pull/17969>`_, Sébastien Han)
+* build/ops: test: delete asok directories correctly (`pr#21023 <https://github.com/ceph/ceph/pull/21023>`_, Chang Liu)
+* build/ops: test/fio: enable objectstore FIO plugin building without the need to install and build FIO source code (`pr#20535 <https://github.com/ceph/ceph/pull/20535>`_, Igor Fedotov)
+* build/ops,tests: common,test,cmake: various changes to re-enable build on osx (`pr#18888 <https://github.com/ceph/ceph/pull/18888>`_, Kefu Chai)
+* build/ops,tests: qa/tests: Changed rhel7.4 to rhel7.5 (`pr#21336 <https://github.com/ceph/ceph/pull/21336>`_, Yuri Weinstein)
+* build/ops,tests: test/fio: fix fio objectstore plugin building broken by (`pr#20514 <https://github.com/ceph/ceph/pull/20514>`_, Igor Fedotov)
+* build/ops: udev: Fix typo in udev OSD rules file (`pr#18976 <https://github.com/ceph/ceph/pull/18976>`_, Mitch Birti)
+* build/ops: use devtoolset-7 on centos/rhel-7 (`pr#18863 <https://github.com/ceph/ceph/pull/18863>`_, Kefu Chai)
+* cephfs: Client:Fix readdir bug (`pr#18784 <https://github.com/ceph/ceph/pull/18784>`_, dongdong tao)
+* cephfs: Client: setattr should drop "Fs" rather than "As" for mtime and size (`pr#18786 <https://github.com/ceph/ceph/pull/18786>`_, dongdong tao)
+* cephfs,common,rbd: common/common_init: disable ms subsystem log gathering for clients (`issue#21860 <http://tracker.ceph.com/issues/21860>`_, `pr#18418 <https://github.com/ceph/ceph/pull/18418>`_, Jason Dillaman)
+* cephfs,common,rbd: Various fixes for SCA issues (`pr#21708 <https://github.com/ceph/ceph/pull/21708>`_, Danny Al-Gaaf)
+* cephfs,core: mon/OSDMonitor: set FLAG_SELFMANAGED_SNAPS on cephfs snap removal (`issue#23949 <http://tracker.ceph.com/issues/23949>`_, `pr#21756 <https://github.com/ceph/ceph/pull/21756>`_, Sage Weil)
+* cephfs: MDS: add null check before we push_back "onfinish" (`pr#18892 <https://github.com/ceph/ceph/pull/18892>`_, dongdong tao)
+* cephfs: MDS: correct the error msg when init mon client (`pr#18836 <https://github.com/ceph/ceph/pull/18836>`_, dongdong tao)
+* cephfs: MDS: make popular counter decay at proper rate (`pr#18776 <https://github.com/ceph/ceph/pull/18776>`_, Jianyu Li)
+* cephfs: MDS: make rebalancer evaluate the overload state of each mds with the same criterion (`pr#19255 <https://github.com/ceph/ceph/pull/19255>`_, Jianyu Li)
+* cephfs: messages: Initialization of is_primary (`pr#16897 <https://github.com/ceph/ceph/pull/16897>`_, amitkuma)
+* cephfs: messages: Initialization of member variables (`pr#16898 <https://github.com/ceph/ceph/pull/16898>`_, amitkuma)
+* cephfs: mimic: MDSMonitor: clean up use of pending fsmap in uncommitted ops (`issue#23768 <http://tracker.ceph.com/issues/23768>`_, `pr#22005 <https://github.com/ceph/ceph/pull/22005>`_, Patrick Donnelly)
+* cephfs: mon/MDSMonitor: wait for readable OSDMap before sanitizing (`issue#21945 <http://tracker.ceph.com/issues/21945>`_, `pr#18603 <https://github.com/ceph/ceph/pull/18603>`_, Patrick Donnelly)
+* cephfs,mon: mon/MDSMonitor: fix a bug at preprocess_beacon (`pr#17415 <https://github.com/ceph/ceph/pull/17415>`_, wangshuguang)
+* cephfs: osdc/Journaler: use new style options (`pr#17806 <https://github.com/ceph/ceph/pull/17806>`_, Kefu Chai)
+* cephfs: qa: check pool full flags (`issue#22475 <http://tracker.ceph.com/issues/22475>`_, `pr#19588 <https://github.com/ceph/ceph/pull/19588>`_, Patrick Donnelly)
+* cephfs: qa: fix typo in test_full (`issue#23643 <http://tracker.ceph.com/issues/23643>`_, `pr#21334 <https://github.com/ceph/ceph/pull/21334>`_, Patrick Donnelly)
+* cephfs: Revert "ceph_context: re-expand admin_socket metavariables in child process" (`pr#18545 <https://github.com/ceph/ceph/pull/18545>`_, Patrick Donnelly)
+* cephfs,tests: qa/suites/powercycle/osd/whitelist_health: whitelist slow trimming (`pr#17307 <https://github.com/ceph/ceph/pull/17307>`_, Sage Weil)
+* cephfs,tests: qa/workunits/cephtool/test.sh: fix test_mon_mds() (`pr#21579 <https://github.com/ceph/ceph/pull/21579>`_, Kefu Chai)
+* cephfs,tools: mount.fuse.ceph: Fix typo (`pr#19128 <https://github.com/ceph/ceph/pull/19128>`_, Jos Collin)
+* cephfs: vstart_runner: fixes for recent cephfs changes (`pr#19533 <https://github.com/ceph/ceph/pull/19533>`_, Patrick Donnelly)
+* ceph-volume: add ANSIBLE_SSH_RETRIES=5 to functional tests (`pr#20592 <https://github.com/ceph/ceph/pull/20592>`_, Andrew Schoen)
+* ceph-volume add functional tests for simple, rearrange lvm tests (`pr#18882 <https://github.com/ceph/ceph/pull/18882>`_, Alfredo Deza)
+* ceph-volume: Add linesep/newline at end of JSON file when writing (`pr#19458 <https://github.com/ceph/ceph/pull/19458>`_, Wido den Hollander)
+* ceph-volume: adds a --destroy flag to ceph-volume lvm zap (`issue#22653 <http://tracker.ceph.com/issues/22653>`_, `pr#20010 <https://github.com/ceph/ceph/pull/20010>`_, Andrew Schoen)
+* ceph-volume: adds --crush-device-class flag for lvm prepare and create (`pr#19949 <https://github.com/ceph/ceph/pull/19949>`_, Andrew Schoen)
+* ceph-volume: adds custom cluster name support to simple (`pr#20367 <https://github.com/ceph/ceph/pull/20367>`_, Andrew Schoen)
+* ceph-volume: adds functional CI testing (`pr#16919 <https://github.com/ceph/ceph/pull/16919>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: adds functional testing for bluestore (`pr#18656 <https://github.com/ceph/ceph/pull/18656>`_, Andrew Schoen)
+* ceph-volume: adds raw device support to 'lvm list' (`issue#23140 <http://tracker.ceph.com/issues/23140>`_, `pr#20620 <https://github.com/ceph/ceph/pull/20620>`_, Andrew Schoen)
+* ceph-volume: adds success messages for lvm prepare/activate/create (`issue#22307 <http://tracker.ceph.com/issues/22307>`_, `pr#19875 <https://github.com/ceph/ceph/pull/19875>`_, Andrew Schoen)
+* ceph-volume: adds support to zap encrypted devices (`issue#22878 <http://tracker.ceph.com/issues/22878>`_, `pr#20537 <https://github.com/ceph/ceph/pull/20537>`_, Andrew Schoen)
+* ceph-volume: adds the ceph-volume lvm zap subcommand (`pr#18513 <https://github.com/ceph/ceph/pull/18513>`_, Andrew Schoen)
+* ceph-volume allow filtering by `uuid`, do not require osd id (`pr#17606 <https://github.com/ceph/ceph/pull/17606>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: allow parallel creates (`issue#23757 <http://tracker.ceph.com/issues/23757>`_, `pr#21489 <https://github.com/ceph/ceph/pull/21489>`_, Theofilos Mouratidis)
+* ceph-volume: allow skipping systemd interactions on activate/create (`issue#23678 <http://tracker.ceph.com/issues/23678>`_, `pr#21496 <https://github.com/ceph/ceph/pull/21496>`_, Alfredo Deza)
+* ceph-volume: allow using a device or partition for `lvm --data` (`pr#18924 <https://github.com/ceph/ceph/pull/18924>`_, Alfredo Deza)
+* ceph-volume be resilient to $PATH issues (`pr#20650 <https://github.com/ceph/ceph/pull/20650>`_, Alfredo Deza)
+* ceph-volume consume mount/format options from ceph.conf (`pr#20408 <https://github.com/ceph/ceph/pull/20408>`_, Alfredo Deza)
+* ceph-volume: correctly fallback to bluestore when no objectstore is specified (`pr#19213 <https://github.com/ceph/ceph/pull/19213>`_, Alfredo Deza)
+* ceph-volume correctly normalize mount flags (`pr#20543 <https://github.com/ceph/ceph/pull/20543>`_, Alfredo Deza)
+* ceph-volume: create the ceph-volume and ceph-volume-systemd man pages (`issue#21030 <http://tracker.ceph.com/issues/21030>`_, `pr#17152 <https://github.com/ceph/ceph/pull/17152>`_, Alfredo Deza)
+* ceph-volume: dmcrypt support for lvm (`issue#22619 <http://tracker.ceph.com/issues/22619>`_, `pr#20054 <https://github.com/ceph/ceph/pull/20054>`_, Alfredo Deza)
+* ceph-volume dmcrypt support for simple (`issue#22620 <http://tracker.ceph.com/issues/22620>`_, `pr#20264 <https://github.com/ceph/ceph/pull/20264>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume/doc: add missing subcommand in examples (`pr#19381 <https://github.com/ceph/ceph/pull/19381>`_, Guillaume Abrioux)
+* ceph-volume: ensure correct --filestore/--bluestore behavior (`pr#18518 <https://github.com/ceph/ceph/pull/18518>`_, Alfredo Deza)
+* ceph-volume  failed ceph-osd --mkfs command doesn't halt the OSD creation process (`issue#23874 <http://tracker.ceph.com/issues/23874>`_, `pr#21685 <https://github.com/ceph/ceph/pull/21685>`_, Alfredo Deza)
+* ceph-volume: fix action plugins path in tests (`pr#20910 <https://github.com/ceph/ceph/pull/20910>`_, Guillaume Abrioux)
+* ceph-volume fix filestore OSD creation after mon-config changes (`issue#23260 <http://tracker.ceph.com/issues/23260>`_, `pr#20787 <https://github.com/ceph/ceph/pull/20787>`_, Alfredo Deza)
+* ceph-volume: fix typo in ceph-volume lvm prepare help (`pr#21196 <https://github.com/ceph/ceph/pull/21196>`_, Jeffrey Zhang)
+* ceph-volume: fix usage of the --osd-id flag (`issue#22642 <http://tracker.ceph.com/issues/22642>`_, `issue#22836 <http://tracker.ceph.com/issues/22836>`_, `pr#20203 <https://github.com/ceph/ceph/pull/20203>`_, Andrew Schoen)
+* ceph-volume Format correctly when vg/lv cannot be used (`issue#22299 <http://tracker.ceph.com/issues/22299>`_, `pr#19285 <https://github.com/ceph/ceph/pull/19285>`_, Alfredo Deza)
+* ceph-volume  handle inline comments in the ceph.conf file (`issue#22297 <http://tracker.ceph.com/issues/22297>`_, `pr#19319 <https://github.com/ceph/ceph/pull/19319>`_, Alfredo Deza)
+* ceph-volume: handle leading whitespace/tabs in ceph.conf (`issue#22280 <http://tracker.ceph.com/issues/22280>`_, `pr#19259 <https://github.com/ceph/ceph/pull/19259>`_, Alfredo Deza)
+* ceph-volume  Implement an 'activate all' to help with dense servers or migrating OSDs (`pr#21130 <https://github.com/ceph/ceph/pull/21130>`_, Alfredo Deza)
+* ceph-volume improve robustness when reloading vms in tests (`pr#21070 <https://github.com/ceph/ceph/pull/21070>`_, Alfredo Deza)
+* ceph-volume: log the current running command for easier debugging (`issue#23004 <http://tracker.ceph.com/issues/23004>`_, `pr#20594 <https://github.com/ceph/ceph/pull/20594>`_, Andrew Schoen)
+* ceph-volume lvm api refactor/move (`pr#18110 <https://github.com/ceph/ceph/pull/18110>`_, Alfredo Deza)
+* ceph-volume lvm list (`pr#18095 <https://github.com/ceph/ceph/pull/18095>`_, Alfredo Deza)
+* ceph-volume lvm.prepare update to use create_osd_path (`pr#18514 <https://github.com/ceph/ceph/pull/18514>`_, Alfredo Deza)
+* ceph-volume: lvm zap will unmount osd paths used by zapped devices (`issue#22876 <http://tracker.ceph.com/issues/22876>`_, `pr#20265 <https://github.com/ceph/ceph/pull/20265>`_, Andrew Schoen)
+* ceph-volume: Nits noticed while studying code (`pr#21455 <https://github.com/ceph/ceph/pull/21455>`_, Dan Mick)
+* ceph-volume Persist non-lv devices for journals (`pr#17403 <https://github.com/ceph/ceph/pull/17403>`_, Alfredo Deza)
+* ceph-volume process the abspath of the executable first (`issue#23259 <http://tracker.ceph.com/issues/23259>`_, `pr#20824 <https://github.com/ceph/ceph/pull/20824>`_, Alfredo Deza)
+* ceph-volume: removed the explicit use of sudo (`issue#22282 <http://tracker.ceph.com/issues/22282>`_, `pr#19363 <https://github.com/ceph/ceph/pull/19363>`_, Andrew Schoen)
+* ceph-volume: remove extra space (`pr#21140 <https://github.com/ceph/ceph/pull/21140>`_, Sébastien Han)
+* ceph-volume rollback on failed OSD prepare/create (`issue#22281 <http://tracker.ceph.com/issues/22281>`_, `pr#19351 <https://github.com/ceph/ceph/pull/19351>`_, Alfredo Deza)
+* ceph-volume should be able to handle multiple LVM (VG/LV) tags (`issue#22305 <http://tracker.ceph.com/issues/22305>`_, `pr#19321 <https://github.com/ceph/ceph/pull/19321>`_, Alfredo Deza)
+* ceph-volume: support GPT and other deployed OSDs (`pr#18823 <https://github.com/ceph/ceph/pull/18823>`_, Alfredo Deza)
+* ceph-volume tests add optional flags for vagrant (`pr#20849 <https://github.com/ceph/ceph/pull/20849>`_, Alfredo Deza)
+* ceph-volume tests alleviate libvirt timeouts when reloading (`issue#23163 <http://tracker.ceph.com/issues/23163>`_, `pr#20718 <https://github.com/ceph/ceph/pull/20718>`_, Alfredo Deza)
+* ceph-volume tests.devices.lvm prepare isn't bluestore specific anymore (`pr#18984 <https://github.com/ceph/ceph/pull/18984>`_, Alfredo Deza)
+* ceph-volume tests remove unused import (`pr#20459 <https://github.com/ceph/ceph/pull/20459>`_, Alfredo Deza)
+* ceph-volume tests use granular env vars for vagrant (`pr#20864 <https://github.com/ceph/ceph/pull/20864>`_, Alfredo Deza)
+* ceph-volume: Try to cast OSD metadata to int while scanning directory (`pr#19477 <https://github.com/ceph/ceph/pull/19477>`_, Wido den Hollander)
+* ceph-volume update man page for prepare/activate flags (`pr#21570 <https://github.com/ceph/ceph/pull/21570>`_, Alfredo Deza)
+* ceph-volume use realpath when checking mounts (`issue#22988 <http://tracker.ceph.com/issues/22988>`_, `pr#20427 <https://github.com/ceph/ceph/pull/20427>`_, Alfredo Deza)
+* ceph-volume: use unique logical volumes (`pr#17207 <https://github.com/ceph/ceph/pull/17207>`_, Alfredo Deza)
+* ceph-volume: Using --readonly for {vg|pv|lv}s commands (`pr#21409 <https://github.com/ceph/ceph/pull/21409>`_, Erwan Velu)
+* ceph-volume: warn on missing ceph.conf file (`issue#22326 <http://tracker.ceph.com/issues/22326>`_, `pr#19347 <https://github.com/ceph/ceph/pull/19347>`_, Alfredo Deza)
+* ceph-volume warn on mix of filestore and bluestore flags (`issue#23003 <http://tracker.ceph.com/issues/23003>`_, `pr#20513 <https://github.com/ceph/ceph/pull/20513>`_, Alfredo Deza)
+* cleanup: Replacing MIN,MAX with std::min,std::max (`pr#18124 <https://github.com/ceph/ceph/pull/18124>`_, Amit Kumar)
+* cli: rados: support for high precision time using stat2 (`issue#21199 <http://tracker.ceph.com/issues/21199>`_, `pr#17395 <https://github.com/ceph/ceph/pull/17395>`_, Abhishek Lekshmanan)
+* cls_acl/_crypto: Add modeline (`pr#19010 <https://github.com/ceph/ceph/pull/19010>`_, Shinobu Kinjo)
+* cmake: add chrono to BOOST_COMPONENTS (`issue#23424 <http://tracker.ceph.com/issues/23424>`_, `pr#20977 <https://github.com/ceph/ceph/pull/20977>`_, Nathan Cutler)
+* cmake: add cython_rbd as a dependency to vstart target (`pr#18382 <https://github.com/ceph/ceph/pull/18382>`_, Ali Maredia)
+* cmake: bail out if GCC version is less than 5.1 (`pr#19344 <https://github.com/ceph/ceph/pull/19344>`_, Kefu Chai)
+* cmake: BuildBoost.cmake: use specified compiler for building boost (`pr#19898 <https://github.com/ceph/ceph/pull/19898>`_, Kefu Chai)
+* cmake: bump target jdk to 1.7 (`issue#23458 <http://tracker.ceph.com/issues/23458>`_, `pr#21082 <https://github.com/ceph/ceph/pull/21082>`_, Shengjing Zhu)
+* cmake: bump up required cmake version to 2.8.12 (`pr#18285 <https://github.com/ceph/ceph/pull/18285>`_, Kefu Chai)
+* cmake: changes of BuildBoost.cmake to ready seastar (`pr#21404 <https://github.com/ceph/ceph/pull/21404>`_, Kefu Chai)
+* cmake: check for aligned_alloc() instead of checking tcmalloc version (`pr#18557 <https://github.com/ceph/ceph/pull/18557>`_, Kefu Chai)
+* cmake: check gcc version not release date for libstdc++ saneness (`pr#18938 <https://github.com/ceph/ceph/pull/18938>`_, Kefu Chai)
+* cmake: check version of boost in src/boost (`pr#19914 <https://github.com/ceph/ceph/pull/19914>`_, Kefu Chai)
+* cmake: cleanups (`pr#18597 <https://github.com/ceph/ceph/pull/18597>`_, Kefu Chai)
+* cmake,common: changes to port part of ceph to osx (`pr#17615 <https://github.com/ceph/ceph/pull/17615>`_, Kefu Chai)
+* cmake: compile nvml as an external project (`pr#17462 <https://github.com/ceph/ceph/pull/17462>`_, Jianpeng Ma)
+* cmake: define HAVE_STDLIB_MAP_SPLICING for both libstdc++ and libc++ (`pr#21284 <https://github.com/ceph/ceph/pull/21284>`_, Kefu Chai)
+* cmake: disable DOWNLOAD_NO_PROGRESS if cmake ver is lower than 3.1 (`pr#20492 <https://github.com/ceph/ceph/pull/20492>`_, Kefu Chai)
+* cmake: disable FAIL_ON_WARNINGS for rocksdb (`pr#19426 <https://github.com/ceph/ceph/pull/19426>`_, Kefu Chai)
+* cmake: disable VTA on options.cc (`pr#17393 <https://github.com/ceph/ceph/pull/17393>`_, Kefu Chai)
+* cmake: do not find bzip2/lz4 for rocksdb (`pr#19963 <https://github.com/ceph/ceph/pull/19963>`_, runsisi)
+* cmake: do not link against librados.a (`pr#18576 <https://github.com/ceph/ceph/pull/18576>`_, Kefu Chai)
+* cmake: do not link against unused or duplicated libraries (`pr#18092 <https://github.com/ceph/ceph/pull/18092>`_, Kefu Chai)
+* cmake: enabled py3 only build (`pr#20064 <https://github.com/ceph/ceph/pull/20064>`_, Kefu Chai)
+* cmake: enable LZ4 by default (`pr#21332 <https://github.com/ceph/ceph/pull/21332>`_, Grant Slater, Casey Bodley)
+* cmake: enable new policies to silence cmake warnings (`pr#21662 <https://github.com/ceph/ceph/pull/21662>`_, Kefu Chai)
+* cmake: fix building without mgr module (`pr#21591 <https://github.com/ceph/ceph/pull/21591>`_, Yuan Zhou)
+* cmake: fix frontend cmake build (`pr#21449 <https://github.com/ceph/ceph/pull/21449>`_, Ricardo Dias)
+* cmake: fix libcephfs-test.jar build failure (`issue#22828 <http://tracker.ceph.com/issues/22828>`_, `pr#20175 <https://github.com/ceph/ceph/pull/20175>`_, Tone Zhang)
+* cmake: fix the include dir for building boost::python (`pr#20324 <https://github.com/ceph/ceph/pull/20324>`_, Kefu Chai)
+* cmake: fix typo in status message (`pr#21464 <https://github.com/ceph/ceph/pull/21464>`_, Lenz Grimmer)
+* cmake: hide symbols import from other libraries in libcls\_\* (`issue#23517 <http://tracker.ceph.com/issues/23517>`_, `pr#21571 <https://github.com/ceph/ceph/pull/21571>`_, Kefu Chai)
+* cmake: identify the possible incompatibility of rocksdb and tcmalloc (`issue#21422 <http://tracker.ceph.com/issues/21422>`_, `pr#17788 <https://github.com/ceph/ceph/pull/17788>`_, Kefu Chai)
+* cmake: in case of bad "ALLOCATOR" selected issue warning (`pr#17422 <https://github.com/ceph/ceph/pull/17422>`_, Adam Kupczyk)
+* cmake: include frontend build in 'all' target (`pr#21466 <https://github.com/ceph/ceph/pull/21466>`_, John Spray)
+* cmake: let "tests" depend on "mgr-dashboard-frontend-build" (`pr#21468 <https://github.com/ceph/ceph/pull/21468>`_, Kefu Chai)
+* cmake: 'make check' builds radosgw and its cls dependencies (`pr#20422 <https://github.com/ceph/ceph/pull/20422>`_, Casey Bodley)
+* cmake: mgr: exclude .gitignore (`pr#19174 <https://github.com/ceph/ceph/pull/19174>`_, Nathan Cutler)
+* cmake/modules/BuildRocksDB.cmake: enable compressions for rocksdb (`issue#24025 <http://tracker.ceph.com/issues/24025>`_, `pr#22183 <https://github.com/ceph/ceph/pull/22183>`_, Kefu Chai)
+* cmake: only create sysctl file on linux (`pr#19029 <https://github.com/ceph/ceph/pull/19029>`_, Kefu Chai)
+* cmake: pass static linkflags to the linker who links libcommon (`pr#19763 <https://github.com/ceph/ceph/pull/19763>`_, Kefu Chai)
+* cmake: s/boost_256/boost_sha256/ (`pr#21573 <https://github.com/ceph/ceph/pull/21573>`_, Kefu Chai)
+* cmake: set supported language the right way (`pr#18216 <https://github.com/ceph/ceph/pull/18216>`_, Kefu Chai)
+* cmake: should use the value of GPERFTOOLS_LIBRARIES as REQUIRED_VARS (`pr#18645 <https://github.com/ceph/ceph/pull/18645>`_, Kefu Chai)
+* cmake: s/sysconf/sysconfig/ (`pr#20631 <https://github.com/ceph/ceph/pull/20631>`_, Kefu Chai)
+* cmake: sync nvml submodule to latest code (`pr#20411 <https://github.com/ceph/ceph/pull/20411>`_, Jianpeng Ma)
+* cmake: System Includes to silence warnings from submodules and libraries! (`pr#18711 <https://github.com/ceph/ceph/pull/18711>`_, Adam C. Emerson)
+* cmake: typo fix when npm is not found (`pr#20801 <https://github.com/ceph/ceph/pull/20801>`_, Abhishek Lekshmanan)
+* cmake: update minimum boost version to 1.66 (`issue#20048 <http://tracker.ceph.com/issues/20048>`_, `issue#22600 <http://tracker.ceph.com/issues/22600>`_, `pr#19808 <https://github.com/ceph/ceph/pull/19808>`_, Casey Bodley)
+* cmake: update the error message for gperftools bug (`pr#17901 <https://github.com/ceph/ceph/pull/17901>`_, Kefu Chai)
+* cmake: warn if libstdc++ older than 5.1.0 is used (`pr#18837 <https://github.com/ceph/ceph/pull/18837>`_, Kefu Chai)
+* cmake: WITH_SPDK=ON by default (`pr#18944 <https://github.com/ceph/ceph/pull/18944>`_, Liu-Chunmei, Kefu Chai, wanjun.lp, Ziye Yang)
+* common: adding line break at end of some cli results (`issue#21019 <http://tracker.ceph.com/issues/21019>`_, `pr#16687 <https://github.com/ceph/ceph/pull/16687>`_, songweibin)
+* common: add line break for "ceph daemon TYPE.ID version" (`pr#17146 <https://github.com/ceph/ceph/pull/17146>`_, Zhu Shangzhong)
+* common: Add metadata with only Ceph version number and release (`pr#21095 <https://github.com/ceph/ceph/pull/21095>`_, Wido den Hollander)
+* common: Add min/max of ms_async_op_threads (`pr#19942 <https://github.com/ceph/ceph/pull/19942>`_, Shinobu Kinjo)
+* common: Add noreturn attribute to silence uninitialized warning (`pr#19348 <https://github.com/ceph/ceph/pull/19348>`_, Adam C. Emerson)
+* common: auth: add err reason for log info in load function (`pr#17256 <https://github.com/ceph/ceph/pull/17256>`_, Luo Kexue)
+* common: bench test fall into dead loop when <seconds>=0 (`pr#16382 <https://github.com/ceph/ceph/pull/16382>`_, PC)
+* common: buffer: avoid changing bufferlist ABI by removing new _mempool field (`issue#21573 <http://tracker.ceph.com/issues/21573>`_, `pr#18408 <https://github.com/ceph/ceph/pull/18408>`_, Sage Weil)
+* common: by default, do not assert on leaks in the shared_cache code (`issue#21737 <http://tracker.ceph.com/issues/21737>`_, `pr#18201 <https://github.com/ceph/ceph/pull/18201>`_, Greg Farnum)
+* common: ceph: add the right bracket to watch-channel argument in the help message (`pr#19698 <https://github.com/ceph/ceph/pull/19698>`_, Chang Liu)
+* common: ceph.in: execv using the same python (`pr#17713 <https://github.com/ceph/ceph/pull/17713>`_, Kefu Chai)
+* common: ceph_release: s/rc/stable/ (`pr#22264 <https://github.com/ceph/ceph/pull/22264>`_, Sage Weil)
+* common: change routines to public access (`pr#20003 <https://github.com/ceph/ceph/pull/20003>`_, Willem Jan Withagen)
+* common: Check this->data.op_size before use (`pr#18816 <https://github.com/ceph/ceph/pull/18816>`_, Amit Kumar)
+* common: cleanup address_helper (`pr#19643 <https://github.com/ceph/ceph/pull/19643>`_, Shinobu Kinjo)
+* common: cmake,common/RWLock: check for libpthread extensions (`pr#19202 <https://github.com/ceph/ceph/pull/19202>`_, Kefu Chai)
+* common:  common: add for_each_substr() for cheap string split (`pr#18798 <https://github.com/ceph/ceph/pull/18798>`_, Casey Bodley)
+* common:  common: add streaming interfaces for json/xml escaping (`pr#19806 <https://github.com/ceph/ceph/pull/19806>`_, Casey Bodley)
+* common: common/admin_socket: validate command json before feeding it to hook (`pr#20437 <https://github.com/ceph/ceph/pull/20437>`_, Kefu Chai)
+* common: common/blkdev: fix build in FreeBSD environment (`pr#19316 <https://github.com/ceph/ceph/pull/19316>`_, Mykola Golub)
+* common: common/buffer: cleanups (`pr#18312 <https://github.com/ceph/ceph/pull/18312>`_, Shinobu Kinjo)
+* common: common/buffer: switch crc cache to single pair instead of map (`pr#18906 <https://github.com/ceph/ceph/pull/18906>`_, Piotr Dałek)
+* common: common/config: add units to options (`issue#22747 <http://tracker.ceph.com/issues/22747>`_, `pr#20419 <https://github.com/ceph/ceph/pull/20419>`_, Kefu Chai)
+* common:  common/config: limit calls to normalize_key_name (`pr#20318 <https://github.com/ceph/ceph/pull/20318>`_, Piotr Dałek)
+* common: common/config: make internal_safe_to_start_threads internal (`pr#18884 <https://github.com/ceph/ceph/pull/18884>`_, Sage Weil)
+* common: common/ConfUtils: check key before actually normalizing (`pr#20370 <https://github.com/ceph/ceph/pull/20370>`_, Piotr Dałek)
+* common: common/dns_resolv.cc: Query for AAAA-record if ms_bind_ipv6 is True (`issue#23078 <http://tracker.ceph.com/issues/23078>`_, `pr#20530 <https://github.com/ceph/ceph/pull/20530>`_, Wido den Hollander)
+* common: common/dns_resolve: fix memory leak (`pr#19649 <https://github.com/ceph/ceph/pull/19649>`_, Yao Zongyou)
+* common: common/event_socket.h: include <errno.h> to use errno (`pr#18351 <https://github.com/ceph/ceph/pull/18351>`_, Kefu Chai)
+* common: common/Formatter: fix string_view usage for {json,xml}_stream_escaper (`issue#23622 <http://tracker.ceph.com/issues/23622>`_, `pr#21317 <https://github.com/ceph/ceph/pull/21317>`_, Sage Weil)
+* common: common/hobject: compare two objects' key directly (`pr#21062 <https://github.com/ceph/ceph/pull/21062>`_, xie xingguo)
+* common: common/hobject: preserve the order of hobject (`pr#21217 <https://github.com/ceph/ceph/pull/21217>`_, Kefu Chai)
+* common: common/ipaddr: Do not select link-local IPv6 addresses (`issue#21813 <http://tracker.ceph.com/issues/21813>`_, `pr#20862 <https://github.com/ceph/ceph/pull/20862>`_, Wido den Hollander)
+* common: common/lockdep: drop hash<pthread_t> specialization (`pr#20574 <https://github.com/ceph/ceph/pull/20574>`_, Kefu Chai)
+* common: common/LogClient: assign seq and queue atomically (`issue#18209 <http://tracker.ceph.com/issues/18209>`_, `pr#16828 <https://github.com/ceph/ceph/pull/16828>`_, Sage Weil)
+* common: common/log: Speed improvement for log (`pr#19100 <https://github.com/ceph/ceph/pull/19100>`_, Adam Kupczyk, Kefu Chai)
+* common: common/OpHistory: move insert/cleanup into separate thread (`pr#20540 <https://github.com/ceph/ceph/pull/20540>`_, Piotr Dałek)
+* common: common/options: drop unused options (`pr#20895 <https://github.com/ceph/ceph/pull/20895>`_, Kefu Chai)
+* common: common/options: long description for log_stderr_prefix (`pr#19869 <https://github.com/ceph/ceph/pull/19869>`_, Sage Weil)
+* common: common/options: pass by reference and use user-literals for size (`pr#18034 <https://github.com/ceph/ceph/pull/18034>`_, Kefu Chai)
+* common: common/options: use user-defined literals for default values (`pr#17180 <https://github.com/ceph/ceph/pull/17180>`_, Kefu Chai)
+* common: common/perf_counters: remove unused parameter (`pr#19805 <https://github.com/ceph/ceph/pull/19805>`_, Kefu Chai)
+* common: common/pick_address.cc: Cleanup (`pr#19707 <https://github.com/ceph/ceph/pull/19707>`_, Shinobu Kinjo)
+* common: common/pick_address: wrong prefix_len in pick_iface() (`pr#20128 <https://github.com/ceph/ceph/pull/20128>`_, Gu Zhongyan)
+* common: common/str_list: s/boost::string_view/std::string_view (`pr#20475 <https://github.com/ceph/ceph/pull/20475>`_, Kefu Chai)
+* common: common/strtol: fix strict_strtoll() so it accepts hex starting with 0x (`pr#21521 <https://github.com/ceph/ceph/pull/21521>`_, Kefu Chai)
+* common: common/strtoll: remove superfluous const modifier (`pr#21560 <https://github.com/ceph/ceph/pull/21560>`_, Jan Fajerski)
+* common: common/throttle: start using 64-bit values (`issue#22539 <http://tracker.ceph.com/issues/22539>`_, `pr#19759 <https://github.com/ceph/ceph/pull/19759>`_, Igor Fedotov)
+* common: common/types: make numbers a bit nicer when displaying space usage (`pr#17126 <https://github.com/ceph/ceph/pull/17126>`_, xie xingguo)
+* common: common/util: do not print error if VERSION_ID is missing (`pr#17787 <https://github.com/ceph/ceph/pull/17787>`_, Kefu Chai)
+* common: compressor: use generate_random_number() for type="random" (`pr#18272 <https://github.com/ceph/ceph/pull/18272>`_, Casey Bodley)
+* common: compressor/zstd: improvements (`pr#18879 <https://github.com/ceph/ceph/pull/18879>`_, Sage Weil)
+* common: compute SimpleLRU's size with contents.size() instead of lru.size() (`issue#22613 <http://tracker.ceph.com/issues/22613>`_, `pr#19813 <https://github.com/ceph/ceph/pull/19813>`_, Xuehan Xu)
+* common: config: expand tilde for ~/.ceph/$cluster.conf (`issue#23215 <http://tracker.ceph.com/issues/23215>`_, `pr#20774 <https://github.com/ceph/ceph/pull/20774>`_, Rishabh Dave)
+* common: config: notify config observers on set_mon_vals() (`pr#21161 <https://github.com/ceph/ceph/pull/21161>`_, Casey Bodley)
+* common: config: Remove _get_val (`pr#18222 <https://github.com/ceph/ceph/pull/18222>`_, Adam C. Emerson)
+* common/config: use with_val() for better performance (`pr#19056 <https://github.com/ceph/ceph/pull/19056>`_, Adam C. Emerson)
+* common: consolidate spinlocks (`pr#15816 <https://github.com/ceph/ceph/pull/15816>`_, Jesse Williamson)
+* common,core: common, osd: various cleanups (`pr#18149 <https://github.com/ceph/ceph/pull/18149>`_, Kefu Chai)
+* common,core: common/pick_address: add {public,cluster}_network_interface option (`pr#18028 <https://github.com/ceph/ceph/pull/18028>`_, Sage Weil)
+* common,core: common/Throttle: Clean up (`pr#16618 <https://github.com/ceph/ceph/pull/16618>`_, Adam C. Emerson)
+* common,core: fix broken use of streamstream::rdbuf() (`issue#22715 <http://tracker.ceph.com/issues/22715>`_, `pr#19998 <https://github.com/ceph/ceph/pull/19998>`_, Sage Weil)
+* common,core: include/ceph_features: deprecate a bunch of features (`pr#18546 <https://github.com/ceph/ceph/pull/18546>`_, Sage Weil)
+* common,core: include,messages,rbd: Initialize counter,group_pool (`pr#17774 <https://github.com/ceph/ceph/pull/17774>`_, Amit Kumar)
+* common,core: options: Do not use linked lists of pointers! (`pr#17984 <https://github.com/ceph/ceph/pull/17984>`_, Adam C. Emerson)
+* common,core: osdc/Objecter: take budgets across a LingerOp instead of on child Ops (`issue#22882 <http://tracker.ceph.com/issues/22882>`_, `pr#20519 <https://github.com/ceph/ceph/pull/20519>`_, Greg Farnum)
+* common,core: osd/OpRequest: reduce overhead when disabling tracking (`pr#18470 <https://github.com/ceph/ceph/pull/18470>`_, Haomai Wang)
+* common,core: rados: Prefer templates to macros (`pr#19913 <https://github.com/ceph/ceph/pull/19913>`_, Adam C. Emerson)
+* common,core,rbd,rgw: common,osd,rgw: Fixes for issues found during SCA (`pr#21419 <https://github.com/ceph/ceph/pull/21419>`_, Danny Al-Gaaf)
+* common,core,rbd,tests,tools: common,mds,mgr,mon,osd: store event only if it's added (`pr#16312 <https://github.com/ceph/ceph/pull/16312>`_, Kefu Chai)
+* common,core: Revert "msg/async/AsyncConnection: unregister connection when racing happened" (`issue#22231 <http://tracker.ceph.com/issues/22231>`_, `pr#19586 <https://github.com/ceph/ceph/pull/19586>`_, Sage Weil)
+* common,core: Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items" (`issue#21410 <http://tracker.ceph.com/issues/21410>`_, `pr#17760 <https://github.com/ceph/ceph/pull/17760>`_, Sage Weil)
+* common: Coverity and SCA fixes (`pr#17431 <https://github.com/ceph/ceph/pull/17431>`_, Danny Al-Gaaf)
+* common/crc/aarch64: Added cpu feature pmull and make aarch64 specific… (`pr#22184 <https://github.com/ceph/ceph/pull/22184>`_, Adam Kupczyk)
+* common: crush/CrushWrapper: fix out of bounds access (`issue#20926 <http://tracker.ceph.com/issues/20926>`_, `pr#16869 <https://github.com/ceph/ceph/pull/16869>`_, Sage Weil)
+* common: crypto: remove cryptopp library (`pr#20015 <https://github.com/ceph/ceph/pull/20015>`_, Casey Bodley)
+* common: denc cleanups and other fixes (`pr#19877 <https://github.com/ceph/ceph/pull/19877>`_, Adam C. Emerson)
+* common: denc: support enum with underlying type (`pr#18701 <https://github.com/ceph/ceph/pull/18701>`_, Kefu Chai)
+* common: Destroy attr of RWLock after initialized (`pr#17103 <https://github.com/ceph/ceph/pull/17103>`_, Wen Zhang)
+* common: dmclock: update mClockPriorityQueue with changes in subtree (`pr#20992 <https://github.com/ceph/ceph/pull/20992>`_, Casey Bodley)
+* common: dout: DoutPrefixProvider operates directly on stream (`pr#21608 <https://github.com/ceph/ceph/pull/21608>`_, Casey Bodley)
+* common: drop namespace using directives for std (`pr#19159 <https://github.com/ceph/ceph/pull/19159>`_, Shinobu Kinjo)
+* common: drop unused variables "bluestore_csum\_\*_block" in opts (`pr#17394 <https://github.com/ceph/ceph/pull/17394>`_, songweibin)
+* common: encoding: reset optional<> if it is uninitialized (`pr#17599 <https://github.com/ceph/ceph/pull/17599>`_, Kefu Chai)
+* common: Extends random.h: numeric types relaxed to compatible types (with (`pr#20670 <https://github.com/ceph/ceph/pull/20670>`_, Jesse Williamson)
+* common: fix BoundedKeyCounter const_pointer_iterator (`issue#22139 <http://tracker.ceph.com/issues/22139>`_, `pr#18953 <https://github.com/ceph/ceph/pull/18953>`_, Casey Bodley)
+* common: fix daemon abnormal exit at parsing invalid arguments (`issue#21365 <http://tracker.ceph.com/issues/21365>`_, `issue#21338 <http://tracker.ceph.com/issues/21338>`_, `pr#17664 <https://github.com/ceph/ceph/pull/17664>`_, Yan Jun)
+* common: fix potential memory leak in HTMLFormatter (`pr#20699 <https://github.com/ceph/ceph/pull/20699>`_, Yao Zongyou)
+* common: fix typo deamon in comments (`pr#17687 <https://github.com/ceph/ceph/pull/17687>`_, yonghengdexin735)
+* common: fix typo in options.cc (`pr#20549 <https://github.com/ceph/ceph/pull/20549>`_, songweibin)
+* common: FreeBSD wants the correct struct selection for ipv6 (`issue#21813 <http://tracker.ceph.com/issues/21813>`_, `pr#21143 <https://github.com/ceph/ceph/pull/21143>`_, Willem Jan Withagen)
+* common: global: output usage on -h, --help, or no args before contacting mons (`pr#20812 <https://github.com/ceph/ceph/pull/20812>`_, Sage Weil)
+* common: hint the main branch of dout() accordingly to default verbosity (`pr#21259 <https://github.com/ceph/ceph/pull/21259>`_, Radoslaw Zarzynski)
+* common: implement random number generator (following N3551) (`issue#18873 <http://tracker.ceph.com/issues/18873>`_, `pr#15341 <https://github.com/ceph/ceph/pull/15341>`_, Jesse Williamson)
+* common: Improving message sent to user when getting signals (`issue#23320 <http://tracker.ceph.com/issues/23320>`_, `pr#21000 <https://github.com/ceph/ceph/pull/21000>`_, Erwan Velu)
+* common: include/encoding: fix compat version error message (`pr#19660 <https://github.com/ceph/ceph/pull/19660>`_, Xinying Song)
+* common: include/interval_set: parameterize by map type and kill btree_interval_set.h (`pr#18611 <https://github.com/ceph/ceph/pull/18611>`_, Sage Weil)
+* common: include/rados: fix typo in librados.h (`pr#17988 <https://github.com/ceph/ceph/pull/17988>`_, wumingqiao)
+* common: include: Remove unused header, ciso646 (`pr#18320 <https://github.com/ceph/ceph/pull/18320>`_, Shinobu Kinjo)
+* common: include/types: format decimal numbers with decimal factor (`issue#22095 <http://tracker.ceph.com/issues/22095>`_, `pr#19117 <https://github.com/ceph/ceph/pull/19117>`_, Jan Fajerski)
+* common: include: xlist: Fix Clang error for missing string (`pr#19367 <https://github.com/ceph/ceph/pull/19367>`_, Willem Jan Withagen)
+* common: interval_set: kill subset_of() (`pr#21108 <https://github.com/ceph/ceph/pull/21108>`_, xie xingguo)
+* common: interval_set: optimize intersect_of insert operations (`issue#21229 <http://tracker.ceph.com/issues/21229>`_, `pr#17265 <https://github.com/ceph/ceph/pull/17265>`_, Zac Medico)
+* common: introduce md_config_cacher_t (`pr#20320 <https://github.com/ceph/ceph/pull/20320>`_, Radoslaw Zarzynski)
+* common: kick off mimic (`pr#16993 <https://github.com/ceph/ceph/pull/16993>`_, Sage Weil)
+* common: lockdep fixes (`issue#20988 <http://tracker.ceph.com/issues/20988>`_, `pr#17738 <https://github.com/ceph/ceph/pull/17738>`_, Jeff Layton)
+* common:  log: clear thread-local stream's ios flags on reuse (`pr#20174 <https://github.com/ceph/ceph/pull/20174>`_, Casey Bodley)
+* common: logically dead code inside shunique_lock.h (`pr#17341 <https://github.com/ceph/ceph/pull/17341>`_, Amit Kumar)
+* common: make ceph_clock_now() inlineable (`pr#20443 <https://github.com/ceph/ceph/pull/20443>`_, Radoslaw Zarzynski)
+* common: Make code to invoke assert() smaller (`pr#20445 <https://github.com/ceph/ceph/pull/20445>`_, Adam Kupczyk)
+* common: make some message informative, instead of error (`pr#16594 <https://github.com/ceph/ceph/pull/16594>`_, Willem Jan Withagen)
+* common: mark events of TrackedOp outside its constructor (`issue#22608 <http://tracker.ceph.com/issues/22608>`_, `pr#19828 <https://github.com/ceph/ceph/pull/19828>`_, Xuehan Xu)
+* common: mgr/dashboard_v2: Fix test_cluster_configuration test (`issue#23265 <http://tracker.ceph.com/issues/23265>`_, `pr#20782 <https://github.com/ceph/ceph/pull/20782>`_, Sebastian Wagner)
+* common: mimic: include/types: space between number and units (`pr#22107 <https://github.com/ceph/ceph/pull/22107>`_, Sage Weil)
+* common,mon: crush,mon: fix weight-set vs crush device classes (`issue#20939 <http://tracker.ceph.com/issues/20939>`_, `pr#16883 <https://github.com/ceph/ceph/pull/16883>`_, Sage Weil)
+* common,mon,osd,pybind: silence warning and remove executable mode bit (`pr#17512 <https://github.com/ceph/ceph/pull/17512>`_, Kefu Chai)
+* common: msg/async/AsyncConnection: less noisy debug (`pr#20600 <https://github.com/ceph/ceph/pull/20600>`_, Sage Weil)
+* common: msg/async: execute on core specified by core_id not its index (`pr#20659 <https://github.com/ceph/ceph/pull/20659>`_, Kefu Chai)
+* common: msg/msg_types: fix the entity_addr_t's decoder (`pr#17699 <https://github.com/ceph/ceph/pull/17699>`_, Kefu Chai)
+* common: msg/simple: s/ceph::size/std::size/ (`pr#19896 <https://github.com/ceph/ceph/pull/19896>`_, Kefu Chai)
+* common/options.cc: cleanup readable literals for default sizes (`pr#18425 <https://github.com/ceph/ceph/pull/18425>`_, Enming Zhang)
+* common/options.cc: Set Filestore rocksdb compaction readahead option (`issue#21505 <http://tracker.ceph.com/issues/21505>`_, `pr#17900 <https://github.com/ceph/ceph/pull/17900>`_, Mark Nelson)
+* common: OpTracker doesn't visit TrackedOp when nref == 0 (`issue#24037 <http://tracker.ceph.com/issues/24037>`_, `pr#22160 <https://github.com/ceph/ceph/pull/22160>`_, Radoslaw Zarzynski)
+* common: osdc/Objecter: fix warning (`pr#21757 <https://github.com/ceph/ceph/pull/21757>`_, Sage Weil)
+* common: osdc/Objecter: record correctly value for l_osdc_op_send_bytes (`issue#21982 <http://tracker.ceph.com/issues/21982>`_, `pr#18810 <https://github.com/ceph/ceph/pull/18810>`_, Jianpeng Ma)
+* common: osd/PrimaryLogPG: send requests to primary on cache miss (`issue#20919 <http://tracker.ceph.com/issues/20919>`_, `pr#16884 <https://github.com/ceph/ceph/pull/16884>`_, Sage Weil)
+* common: osd_types: define max in eversion_t::max() to static (`pr#17453 <https://github.com/ceph/ceph/pull/17453>`_, yang.wang)
+* common,os: initialize commit_data,cmount,iocb (`pr#17766 <https://github.com/ceph/ceph/pull/17766>`_, Amit Kumar)
+* common: posix_fallocate on ZFS returns EINVAL (`pr#20398 <https://github.com/ceph/ceph/pull/20398>`_, Willem Jan Withagen)
+* common: rados: clean up rados_getxattrs() and rados_striper_getxattrs() (`pr#20259 <https://github.com/ceph/ceph/pull/20259>`_, Gu Zhongyan)
+* common:  RAII-styled mechanism for updating PerfCounters (`pr#19149 <https://github.com/ceph/ceph/pull/19149>`_, Radoslaw Zarzynski)
+* common: random: revert change from boost::optional to std::optional (`issue#23778 <http://tracker.ceph.com/issues/23778>`_, `pr#21567 <https://github.com/ceph/ceph/pull/21567>`_, Casey Bodley)
+* common: Remove ceph_clock_gettime, extern keyword (`pr#19353 <https://github.com/ceph/ceph/pull/19353>`_, Shinobu Kinjo)
+* common: retry_sys_call no need take address of a function pointer (`pr#21281 <https://github.com/ceph/ceph/pull/21281>`_, Leo Zhang)
+* common: Revert "common/config: return const reference instead of a copy" (`pr#18934 <https://github.com/ceph/ceph/pull/18934>`_, Kefu Chai)
+* common: Revert "core: hint the dout()'s message crafting as a cold code." (`issue#23169 <http://tracker.ceph.com/issues/23169>`_, `pr#20636 <https://github.com/ceph/ceph/pull/20636>`_, Kefu Chai)
+* common,rgw: rgw,common,rbd: s/boost::regex/std::regex/ (`pr#19393 <https://github.com/ceph/ceph/pull/19393>`_, Kefu Chai)
+* common,rgw: rgw,common: remove already included header files (`pr#19390 <https://github.com/ceph/ceph/pull/19390>`_, Yao Zongyou)
+* common: silence jenkins's buiding warning in obj_bencher.cc (`pr#17272 <https://github.com/ceph/ceph/pull/17272>`_, Luo Kexue)
+* common: src/common: update some ms\_\* options to be more consistent (`pr#20652 <https://github.com/ceph/ceph/pull/20652>`_, shangfufei)
+* common: src/msg/async/rdma: decrease cpu usage by rdtsc instruction (`pr#16965 <https://github.com/ceph/ceph/pull/16965>`_, Jin Cai)
+* common: Static Pointer (`pr#19079 <https://github.com/ceph/ceph/pull/19079>`_, Adam C. Emerson)
+* common: strict_strtol INT_MAX and INT_MIN is valid (`pr#18574 <https://github.com/ceph/ceph/pull/18574>`_, Shasha Lu)
+* common: s/unique_lock/lock_guard/, if manual lock/unlock are not necessary (`pr#19770 <https://github.com/ceph/ceph/pull/19770>`_, Shinobu Kinjo)
+* common: Switch singletons to use immobile_any and cleanups (`pr#20273 <https://github.com/ceph/ceph/pull/20273>`_, Adam C. Emerson)
+* common: test: fix unittest memory leak to silence valgrind (`pr#19654 <https://github.com/ceph/ceph/pull/19654>`_, Yao Zongyou)
+* common,tests: test/common: unittest_mclock_priority_queue builds with "make" command (`pr#17582 <https://github.com/ceph/ceph/pull/17582>`_, J. Eric Ivancich)
+* common,tests: test/librados: create unique lock names (`issue#20798 <http://tracker.ceph.com/issues/20798>`_, `pr#16953 <https://github.com/ceph/ceph/pull/16953>`_, Neha Ojha)
+* common: tools/crushtool: skip device id if no name exists (`issue#22117 <http://tracker.ceph.com/issues/22117>`_, `pr#18901 <https://github.com/ceph/ceph/pull/18901>`_, Jan Fajerski)
+* common: use mono clock for HeartbeatMap (`pr#17827 <https://github.com/ceph/ceph/pull/17827>`_, Xinze Chi, Kefu Chai)
+* common: use move instead of copy in build_options() (`pr#18003 <https://github.com/ceph/ceph/pull/18003>`_, Casey Bodley)
+* common: utime： fix __32u sec time overflow (`pr#21113 <https://github.com/ceph/ceph/pull/21113>`_, kungf)
+* compressor: add zstd back (`pr#21106 <https://github.com/ceph/ceph/pull/21106>`_, Kefu Chai)
+* compressor: conditionalize on HAVE_LZ4 (`pr#17059 <https://github.com/ceph/ceph/pull/17059>`_, Kefu Chai)
+* compressor: kill AsyncCompressor which is broken (`pr#18472 <https://github.com/ceph/ceph/pull/18472>`_, Haomai Wang)
+* core: blkin: Fix unconditional tracing in OSD (`pr#19156 <https://github.com/ceph/ceph/pull/19156>`_, Yingxin)
+* core: ceph-debug-docker.sh: add ceph-osd-dbg package (`pr#17947 <https://github.com/ceph/ceph/pull/17947>`_, Patrick Donnelly)
+* core: ceph.in: Add blocking mode for scrub and deep-scrub (`pr#19793 <https://github.com/ceph/ceph/pull/19793>`_, Brad Hubbard)
+* core: ceph.in: do not panic at control+d in interactive mode (`pr#18374 <https://github.com/ceph/ceph/pull/18374>`_, Kefu Chai)
+* core: ceph.in: print all matched commands if arg missing (`issue#22344 <http://tracker.ceph.com/issues/22344>`_, `pr#19547 <https://github.com/ceph/ceph/pull/19547>`_, Kefu Chai)
+* core: ceph.in: use a different variable for holding thrown exception (`pr#20663 <https://github.com/ceph/ceph/pull/20663>`_, Kefu Chai)
+* core: ceph-kvstore-tool: copy to different store type and cleanup properly (`pr#18029 <https://github.com/ceph/ceph/pull/18029>`_, Josh Durgin)
+* core: ceph-mgr: exit after usage (`issue#23482 <http://tracker.ceph.com/issues/23482>`_, `pr#21401 <https://github.com/ceph/ceph/pull/21401>`_, Sage Weil)
+* core: ceph_osd.cc: Drop legacy or redundant code (`pr#18718 <https://github.com/ceph/ceph/pull/18718>`_, Shinobu Kinjo)
+* core: ceph-osd: some flags are not documented in the help output (`issue#20057 <http://tracker.ceph.com/issues/20057>`_, `pr#15565 <https://github.com/ceph/ceph/pull/15565>`_, Yanhu Cao)
+* core: ceph: print output of "status" as string not as bytes (`pr#21297 <https://github.com/ceph/ceph/pull/21297>`_, Kefu Chai)
+* core: ceph-rest-api: when port=0 use the DEFAULT_PORT instead (`pr#17443 <https://github.com/ceph/ceph/pull/17443>`_, You Ji)
+* core: ceph_test_objectstore: disable filestore_fiemap for tests (`issue#21880 <http://tracker.ceph.com/issues/21880>`_, `pr#18452 <https://github.com/ceph/ceph/pull/18452>`_, Sage Weil)
+* core: ceph_test_objectstore: do not change model for 0-length zero (`issue#21712 <http://tracker.ceph.com/issues/21712>`_, `pr#18519 <https://github.com/ceph/ceph/pull/18519>`_, Sage Weil)
+* core: ceph_test_rados_api_aio: fix race with full pool and osdmap (`issue#23916 <http://tracker.ceph.com/issues/23916>`_, `issue#23917 <http://tracker.ceph.com/issues/23917>`_, `pr#21709 <https://github.com/ceph/ceph/pull/21709>`_, Sage Weil)
+* core: ceph_test_rados_api_tier: add ListSnap test (`pr#17706 <https://github.com/ceph/ceph/pull/17706>`_, Xuehan Xu)
+* core: client,osd,test: Initialize fuse_req_key,snap,who,seq (`pr#17772 <https://github.com/ceph/ceph/pull/17772>`_, Amit Kumar)
+* core: common/admin_socket: various cleanups (`pr#20274 <https://github.com/ceph/ceph/pull/20274>`_, Adam C. Emerson)
+* core: common/config: cleanup remove some unused macros (`pr#19599 <https://github.com/ceph/ceph/pull/19599>`_, Yao Zongyou)
+* core: common,mds,osd: Explicitly delete copy ctor if noncopyable (`pr#19465 <https://github.com/ceph/ceph/pull/19465>`_, Shinobu Kinjo)
+* core: common/options: enable multiple rocksdb compaction threads for filestore (`pr#18232 <https://github.com/ceph/ceph/pull/18232>`_, Josh Durgin)
+* core: common, osd: duplicated "start" event in OpTracker, improve OpTracker::dump_ops (`pr#21119 <https://github.com/ceph/ceph/pull/21119>`_, Chang Liu)
+* core: compressor: Add Brotli Compressor (`pr#19549 <https://github.com/ceph/ceph/pull/19549>`_, BI SHUN KE)
+* core: config: lower default omap entries recovered at once (`issue#21897 <http://tracker.ceph.com/issues/21897>`_, `pr#19910 <https://github.com/ceph/ceph/pull/19910>`_, Josh Durgin)
+* core:  crush/CrushWrapper: fix potential invalid use of iterator (`pr#21325 <https://github.com/ceph/ceph/pull/21325>`_, xie xingguo)
+* core: dmclock: Delivery of the dmclock delta, rho and phase parameter + Enabling the client service tracker (`pr#16369 <https://github.com/ceph/ceph/pull/16369>`_, Byungsu Park, Taewoong Kim)
+* core: erasure-code: refactor the interfaces to hide internals from public (`pr#18683 <https://github.com/ceph/ceph/pull/18683>`_, Kefu Chai)
+* core: erasure-code: use jerasure_free_schedule to properly free a schedule (`pr#19650 <https://github.com/ceph/ceph/pull/19650>`_, Yao Zongyou)
+* core: erasure-code: use std::count() instead (`pr#19428 <https://github.com/ceph/ceph/pull/19428>`_, Kefu Chai)
+* core: etc/default/ceph: remove jemalloc option (`issue#20557 <http://tracker.ceph.com/issues/20557>`_, `pr#18486 <https://github.com/ceph/ceph/pull/18486>`_, Sage Weil)
+* core: filestore: include <linux/falloc.h> (`pr#20415 <https://github.com/ceph/ceph/pull/20415>`_, wumingqiao)
+* core: Fix a dead lock when doing rdma performance test by fio (`pr#17016 <https://github.com/ceph/ceph/pull/17016>`_, Wang Chuanhong)
+* core: Fix asserts caused by DNE pgs left behind after lots of OSD restarts (`issue#21833 <http://tracker.ceph.com/issues/21833>`_, `pr#20571 <https://github.com/ceph/ceph/pull/20571>`_, David Zafman)
+* core: include: kill MIN and MAX macros (`pr#20886 <https://github.com/ceph/ceph/pull/20886>`_, Sage Weil)
+* core: interval_set: optimize intersection_of (`pr#17088 <https://github.com/ceph/ceph/pull/17088>`_, Zac Medico)
+* core: kv/KeyValueDB: add column family (`pr#18049 <https://github.com/ceph/ceph/pull/18049>`_, Jianjian Huo, Adam C. Emerson, Sage Weil)
+* core: kv/RocksDB: get index and filter blocks memory usage (`pr#19934 <https://github.com/ceph/ceph/pull/19934>`_, Zhi Zhang)
+* core: kv/RocksDBStore: fix rocksdb error when block cache is disabled (`issue#23816 <http://tracker.ceph.com/issues/23816>`_, `pr#21583 <https://github.com/ceph/ceph/pull/21583>`_, Yang Honggang)
+* core:  librados: add OPERATION_ORDERSNAP flag and yet another aio_operate method (`pr#20343 <https://github.com/ceph/ceph/pull/20343>`_, Mykola Golub)
+* core: librados.h:  add LIBRADOS_SUPPORTS_APP_METADATA (`pr#16542 <https://github.com/ceph/ceph/pull/16542>`_, Matt Benjamin)
+* core: libradosstriper: fix the function declaration of rados_striper_trunc (`pr#20301 <https://github.com/ceph/ceph/pull/20301>`_, yuelongguang)
+* core: libradosstriper: silence warning from -Wreorder (`pr#16890 <https://github.com/ceph/ceph/pull/16890>`_, songweibin)
+* core: make the main dout() paths faster and more maintanable (`pr#20290 <https://github.com/ceph/ceph/pull/20290>`_, Radoslaw Zarzynski)
+* core: messages: Initialization of variable beat (`pr#17641 <https://github.com/ceph/ceph/pull/17641>`_, Amit Kumar)
+* core: messages: Initialize member variables (`pr#16846 <https://github.com/ceph/ceph/pull/16846>`_, amitkuma)
+* core: messages: initialize variable tid in MMDSFindIno (`pr#16793 <https://github.com/ceph/ceph/pull/16793>`_, amitkuma)
+* core: messages: Initializing members in MOSDPGUpdateLogMissing (`pr#16928 <https://github.com/ceph/ceph/pull/16928>`_, amitkuma)
+* core: messages: Initializing variable ceph_mds_reply_head (`pr#17090 <https://github.com/ceph/ceph/pull/17090>`_, amitkuma)
+* core: messages,journal: Initialization of stats_period,m_active_set (`pr#17792 <https://github.com/ceph/ceph/pull/17792>`_, Amit Kumar)
+* core: messages/MOSDMap: do compat reencode of crush map, too (`issue#21882 <http://tracker.ceph.com/issues/21882>`_, `pr#18454 <https://github.com/ceph/ceph/pull/18454>`_, Sage Weil)
+* core: messages/MOSDOp: a fixes of encode_payload (`pr#16836 <https://github.com/ceph/ceph/pull/16836>`_, Ying He)
+* core: messages: Silence uninitialized member warnings (`pr#17596 <https://github.com/ceph/ceph/pull/17596>`_, Amit Kumar)
+* core: mgr/DaemonServer.cc: add 'is_valid=false' when decode caps error (`issue#20990 <http://tracker.ceph.com/issues/20990>`_, `pr#16978 <https://github.com/ceph/ceph/pull/16978>`_, Yanhu Cao)
+* core,mgr: mgr/balancer: improve error message (`issue#22814 <http://tracker.ceph.com/issues/22814>`_, `pr#21427 <https://github.com/ceph/ceph/pull/21427>`_, Sage Weil)
+* core,mgr: osd,mgrclient: pass daemon_status by rvalue ref and other cleanups (`pr#18509 <https://github.com/ceph/ceph/pull/18509>`_, Kefu Chai)
+* core,mgr: osd,mgr: report slow requests and pending creating pgs to mgr (`pr#18614 <https://github.com/ceph/ceph/pull/18614>`_, Kefu Chai)
+* core: mimic: crush: update choose_args on bucket removal (`issue#24167 <http://tracker.ceph.com/issues/24167>`_, `pr#22120 <https://github.com/ceph/ceph/pull/22120>`_, Sage Weil)
+* core: mimic: osdc: guard op->on_notify_finish with lock (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#21834 <https://github.com/ceph/ceph/pull/21834>`_, Kefu Chai)
+* core: mimic: osd: clean up smart probe (`issue#23899 <http://tracker.ceph.com/issues/23899>`_, `issue#24104 <http://tracker.ceph.com/issues/24104>`_, `pr#21959 <https://github.com/ceph/ceph/pull/21959>`_, Sage Weil, Gu Zhongyan)
+* core: mimic: osd: Don't evict even when preemption has restarted with smaller chunk (`pr#22041 <https://github.com/ceph/ceph/pull/22041>`_, David Zafman)
+* core: mimic: osd/PrimaryLogPG: fix try_flush_mark_clean write contention case (`issue#24200 <http://tracker.ceph.com/issues/24200>`_, `issue#24174 <http://tracker.ceph.com/issues/24174>`_, `pr#22113 <https://github.com/ceph/ceph/pull/22113>`_, Sage Weil)
+* core: mon/ConfigKeyService: dump: print placeholder value for binary blobs (`issue#23622 <http://tracker.ceph.com/issues/23622>`_, `pr#21329 <https://github.com/ceph/ceph/pull/21329>`_, Sage Weil)
+* core,mon: crush, mon: bump up map version only if we truly created a weight-set (`pr#20178 <https://github.com/ceph/ceph/pull/20178>`_, xie xingguo)
+* core: mon/LogMonitor: separate out summary by channel (`pr#21395 <https://github.com/ceph/ceph/pull/21395>`_, Sage Weil)
+* core,mon: mon/AuthMonitor: create bootstrap keys on create_initial() (`pr#21236 <https://github.com/ceph/ceph/pull/21236>`_, Joao Eduardo Luis)
+* core,mon: mon/LogMonitor: do not crash on log sub w/ no messages (`pr#21469 <https://github.com/ceph/ceph/pull/21469>`_, Sage Weil)
+* core,mon: mon,osd,crush: misc cleanup (`pr#20687 <https://github.com/ceph/ceph/pull/20687>`_, songweibin)
+* core,mon: mon/OSDMonitor: Comment out unused function (`pr#20275 <https://github.com/ceph/ceph/pull/20275>`_, Brad Hubbard)
+* core,mon: mon/OSDMonitor: don't create pgs if pool was deleted (`issue#21309 <http://tracker.ceph.com/issues/21309>`_, `pr#17600 <https://github.com/ceph/ceph/pull/17600>`_, Joao Eduardo Luis)
+* core,mon: mon/OSDMonitor: implement cluster pg limit (`pr#17427 <https://github.com/ceph/ceph/pull/17427>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: list osd tree in named bucket (`pr#19564 <https://github.com/ceph/ceph/pull/19564>`_, kungf)
+* core: mon, osd: add create-time for pool (`pr#21690 <https://github.com/ceph/ceph/pull/21690>`_, xie xingguo)
+* core: mon, osd: fix potential collided \*Up Set\* after PG remapping (`issue#23118 <http://tracker.ceph.com/issues/23118>`_, `pr#20653 <https://github.com/ceph/ceph/pull/20653>`_, xie xingguo)
+* core,mon: osd,mon: add max-pg-per-osd limit (`pr#18358 <https://github.com/ceph/ceph/pull/18358>`_, Kefu Chai)
+* core: mon/OSDMonitor: filter out pgs that shouldn't exist from force-create-pg (`pr#20267 <https://github.com/ceph/ceph/pull/20267>`_, Sage Weil)
+* core: mon/OSDMonitor: fix min_size default for replicated pools (`pr#20555 <https://github.com/ceph/ceph/pull/20555>`_, Josh Durgin)
+* core: mon/OSDMonitor: Fix OSDMonitor error message outputs (`issue#22351 <http://tracker.ceph.com/issues/22351>`_, `pr#20022 <https://github.com/ceph/ceph/pull/20022>`_, Brad Hubbard)
+* core: mon/OSDMonitor: make 'osd crush class rename' idempotent (`pr#17330 <https://github.com/ceph/ceph/pull/17330>`_, xie xingguo)
+* core: mon/OSDMonitor: rename outer name declaration to avoid shadowing (`pr#20032 <https://github.com/ceph/ceph/pull/20032>`_, Sage Weil)
+* core: mon/OSDMonitor: require --yes-i-really-mean-it for force-create-pg (`pr#21619 <https://github.com/ceph/ceph/pull/21619>`_, Sage Weil)
+* core: mon,osd,osdc: refactor snap trimming (phase 1) (`pr#18276 <https://github.com/ceph/ceph/pull/18276>`_, Sage Weil)
+* core: mon, osd: per pool space-full flag support (`pr#17371 <https://github.com/ceph/ceph/pull/17371>`_, xie xingguo)
+* core: mon, osd: turn down non-error scrub message severity (`issue#20947 <http://tracker.ceph.com/issues/20947>`_, `pr#16916 <https://github.com/ceph/ceph/pull/16916>`_, John Spray)
+* core: mon/PGMap: fix PGMapDigest decode (`pr#22099 <https://github.com/ceph/ceph/pull/22099>`_, Sage Weil)
+* core: mon/PGMap: Fix %USED calculation bug (`issue#22247 <http://tracker.ceph.com/issues/22247>`_, `pr#19165 <https://github.com/ceph/ceph/pull/19165>`_, Xiaoxi Chen)
+* core: mon/PGMap: remove or narrow columns 'pg ls' output (`pr#20945 <https://github.com/ceph/ceph/pull/20945>`_, Sage Weil)
+* core: mon/PGMap: 'unclean' does not imply damaged (`pr#18493 <https://github.com/ceph/ceph/pull/18493>`_, Sage Weil)
+* core: MOSDPGRecoveryDelete[Reply]: bump header version (`pr#17585 <https://github.com/ceph/ceph/pull/17585>`_, Josh Durgin)
+* core: msg/asyc/rmda: fix the bug of assert when Infiniband::recv_msg receives disconnect message (`pr#17688 <https://github.com/ceph/ceph/pull/17688>`_, Jin Cai)
+* core: msg/async/AsyncConnection: combine multi alloc into one (`pr#18833 <https://github.com/ceph/ceph/pull/18833>`_, Haomai Wang)
+* core: msg/async/AsyncConnection: Fix FPE in process_connection (`issue#23618 <http://tracker.ceph.com/issues/23618>`_, `pr#21314 <https://github.com/ceph/ceph/pull/21314>`_, Brad Hubbard)
+* core: msg/async/AsyncConnection: state will be NONE if replacing by another one (`issue#21883 <http://tracker.ceph.com/issues/21883>`_, `pr#18467 <https://github.com/ceph/ceph/pull/18467>`_, Haomai Wang)
+* core: msg/async/AsyncConnection: unregister connection when racing happened (`pr#19013 <https://github.com/ceph/ceph/pull/19013>`_, Haomai Wang)
+* core: msg/async: batch handle numevents (`pr#18321 <https://github.com/ceph/ceph/pull/18321>`_, Jianpeng Ma)
+* core: msg/async: don't kill connection if replacing (`issue#21143 <http://tracker.ceph.com/issues/21143>`_, `pr#17288 <https://github.com/ceph/ceph/pull/17288>`_, Haomai Wang)
+* core: msg/async: don't stuck into resetsession/retrysession loop (`pr#17276 <https://github.com/ceph/ceph/pull/17276>`_, Haomai Wang)
+* core: msg/async: fix bug of data type conversion when uint64_t -> int -> uint64_t (`pr#18210 <https://github.com/ceph/ceph/pull/18210>`_, shangfufei)
+* core: msg/async: print error log if add_event fail (`pr#17102 <https://github.com/ceph/ceph/pull/17102>`_, mychoxin)
+* core: msg/async/rdma: fix multi cephcontext confllicting (`pr#16893 <https://github.com/ceph/ceph/pull/16893>`_, Haomai Wang)
+* core: msg/async/rdma: fix the bug that rdma polling thread uses the same thread name with msg worker (`pr#16936 <https://github.com/ceph/ceph/pull/16936>`_, Jin Cai)
+* core: msg/async/rdma: improves RX buffer management (`pr#16693 <https://github.com/ceph/ceph/pull/16693>`_, Alex Mikheev)
+* core: msg/async/rdma: uninitialized variable fix (`pr#18091 <https://github.com/ceph/ceph/pull/18091>`_, Vasily Philipov)
+* core: msg/DispatchQueue: clear queue after wait() (`issue#18351 <http://tracker.ceph.com/issues/18351>`_, `pr#20374 <https://github.com/ceph/ceph/pull/20374>`_, Sage Weil)
+* core: msgr/simple: set Pipe::out_seq to in_seq of the connecting side (`issue#23807 <http://tracker.ceph.com/issues/23807>`_, `pr#21585 <https://github.com/ceph/ceph/pull/21585>`_, Xuehan Xu)
+* core: os/bluestore: debug bluestore cache shutdown (`issue#21259 <http://tracker.ceph.com/issues/21259>`_, `pr#17844 <https://github.com/ceph/ceph/pull/17844>`_, Sage Weil)
+* core: os/bluestore: disable on_applied sync_complete (`issue#22668 <http://tracker.ceph.com/issues/22668>`_, `pr#20169 <https://github.com/ceph/ceph/pull/20169>`_, Sage Weil)
+* core: os/bluestore: make bdev label parsing error more meaningful and less noisy (`pr#20090 <https://github.com/ceph/ceph/pull/20090>`_, Sage Weil)
+* core: os/bluestore: make BlueStore opened by start_kv_only umountable (`issue#21624 <http://tracker.ceph.com/issues/21624>`_, `pr#18082 <https://github.com/ceph/ceph/pull/18082>`_, Chang Liu)
+* core: os/bluestore: use db->rm_range_keys to delete range of keys (`pr#18279 <https://github.com/ceph/ceph/pull/18279>`_, Xiaoyan Li)
+* core: OSD/admin_socket: add get_mapped_pools command (`pr#19112 <https://github.com/ceph/ceph/pull/19112>`_, Xiaoxi Chen)
+* core: osdc, class_api: kill implicit string conversions (`pr#16648 <https://github.com/ceph/ceph/pull/16648>`_, Piotr Dałek)
+* core: osdc: dec num_in_flight for pool_dne case (`pr#21110 <https://github.com/ceph/ceph/pull/21110>`_, Jianpeng Ma)
+* core: osdc: Do not use lock_guard as unique_lock (`pr#19756 <https://github.com/ceph/ceph/pull/19756>`_, Shinobu Kinjo)
+* core: osdc: invoke notify finish context on linger commit failure (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#21786 <https://github.com/ceph/ceph/pull/21786>`_, Jason Dillaman)
+* core: osdc/Objecter: add ignore overlay flag if got redirect reply (`pr#21275 <https://github.com/ceph/ceph/pull/21275>`_, Ting Yi Lin)
+* core: osdc/Objecter: delay initialization of hobject_t in _send_op (`issue#21845 <http://tracker.ceph.com/issues/21845>`_, `pr#18427 <https://github.com/ceph/ceph/pull/18427>`_, Jason Dillaman)
+* core: osdc/Objecter: fix recursive locking in _finish_command (`issue#23940 <http://tracker.ceph.com/issues/23940>`_, `pr#21742 <https://github.com/ceph/ceph/pull/21742>`_, Sage Weil)
+* core: osdc/Objecter: misc cleanups (`pr#18476 <https://github.com/ceph/ceph/pull/18476>`_, Jianpeng Ma)
+* core: osdc/Objecter: prevent double-invocation of linger op callback (`issue#23872 <http://tracker.ceph.com/issues/23872>`_, `pr#21649 <https://github.com/ceph/ceph/pull/21649>`_, Jason Dillaman)
+* core: osdc/Objecter: skip sparse-read result decode if bufferlist is empty (`issue#21844 <http://tracker.ceph.com/issues/21844>`_, `pr#18400 <https://github.com/ceph/ceph/pull/18400>`_, Jason Dillaman)
+* core: osd,compressor: Expose compression algorithms via MOSDBoot (`issue#22420 <http://tracker.ceph.com/issues/22420>`_, `pr#20558 <https://github.com/ceph/ceph/pull/20558>`_, Jesse Williamson)
+* core: osdc: remove unused function (`pr#21081 <https://github.com/ceph/ceph/pull/21081>`_, Jianpeng Ma)
+* core: osd,dmclock: use pointer to ClientInfo instead of a copy of it (`pr#18387 <https://github.com/ceph/ceph/pull/18387>`_, Kefu Chai)
+* core:  osd: do not forget pg_stat acks which failed to send (`pr#16702 <https://github.com/ceph/ceph/pull/16702>`_, huangjun)
+* core: OSD: drop unsed parameter passed to check_osdmap_features (`pr#18466 <https://github.com/ceph/ceph/pull/18466>`_, Leo Zhang)
+* core: osd/ECBackend: inject sleep during deep scrub (`pr#20531 <https://github.com/ceph/ceph/pull/20531>`_, xie xingguo)
+* core: osd/ECBackend: only check required shards when finishing recovery reads (`issue#23195 <http://tracker.ceph.com/issues/23195>`_, `pr#21273 <https://github.com/ceph/ceph/pull/21273>`_, Josh Durgin)
+* core: osd/ECBackend: update misleading comment about EIO handling (`pr#21686 <https://github.com/ceph/ceph/pull/21686>`_, Josh Durgin)
+* core: osd/ECBackend: wait for apply for luminous peers (`pr#21604 <https://github.com/ceph/ceph/pull/21604>`_, Sage Weil)
+* core: osd/ECMsgTypes: fix ECSubRead compat decode (`pr#20948 <https://github.com/ceph/ceph/pull/20948>`_, Sage Weil)
+* core: osd, librados: add a rados op (TIER_PROMOTE) (`pr#19362 <https://github.com/ceph/ceph/pull/19362>`_, Myoungwon Oh)
+* core: osd,librados: add manifest, operations for chunked object (`pr#15482 <https://github.com/ceph/ceph/pull/15482>`_, Myoungwon Oh)
+* core: osd,messages: Initialize read_length,options,send_reply (`pr#17799 <https://github.com/ceph/ceph/pull/17799>`_, Amit Kumar)
+* core: osd/OSD: batch-list objects to reduce memory consumption (`pr#20767 <https://github.com/ceph/ceph/pull/20767>`_, xie xingguo)
+* core: osd/OSD.cc: add 'isvalid=false' when failed to parse caps (`pr#16888 <https://github.com/ceph/ceph/pull/16888>`_, Yanhu Cao)
+* core: osd/OSD.cc: use option 'osd_scrub_cost' instead (`pr#18479 <https://github.com/ceph/ceph/pull/18479>`_, Liao Weizhong)
+* core: osd/OSDMap: add osdmap epoch info when printing info summary (`pr#20184 <https://github.com/ceph/ceph/pull/20184>`_, shun-s)
+* core: osd/OSDMap: fix HAVE_FEATURE logic in encode() (`pr#20922 <https://github.com/ceph/ceph/pull/20922>`_, Ilya Dryomov)
+* core: osd/OSDMap: ignore PGs from pools of failure-domain OSD (`pr#20703 <https://github.com/ceph/ceph/pull/20703>`_, xie xingguo)
+* core: osd/OSDMap: misleading message in print_oneline_summary() (`issue#22350 <http://tracker.ceph.com/issues/22350>`_, `pr#20313 <https://github.com/ceph/ceph/pull/20313>`_, Gu Zhongyan)
+* core: osd/OSDMap: more pg upmap fixes (`issue#23878 <http://tracker.ceph.com/issues/23878>`_, `pr#21670 <https://github.com/ceph/ceph/pull/21670>`_, xiexingguo)
+* core: osd/OSDMap: remove the unnecessary checks for null (`pr#18636 <https://github.com/ceph/ceph/pull/18636>`_, Kefu Chai)
+* core: osd/OSDMap: skip out/crush-out osds (`pr#20655 <https://github.com/ceph/ceph/pull/20655>`_, xie xingguo)
+* core: osd/OSDMap: upmap should respect the osd reweights (`issue#21538 <http://tracker.ceph.com/issues/21538>`_, `pr#17944 <https://github.com/ceph/ceph/pull/17944>`_, Theofilos Mouratidis)
+* core: osd/osd_type: get_clone_bytes - inline size() for overlapping size (`pr#17823 <https://github.com/ceph/ceph/pull/17823>`_, xie xingguo)
+* core: osd/osd_types.cc: copy extents map too while making clone (`pr#18396 <https://github.com/ceph/ceph/pull/18396>`_, xie xingguo)
+* core: osd/osd_types: fix ideal lower bound object-id of pg (`pr#21235 <https://github.com/ceph/ceph/pull/21235>`_, xie xingguo)
+* core: osd/osd_types: fix object_stat_sum_t decode (`pr#18551 <https://github.com/ceph/ceph/pull/18551>`_, Sage Weil)
+* core: osd/osd_types: fix pg_pool_t encoding for hammer (`pr#21282 <https://github.com/ceph/ceph/pull/21282>`_, Sage Weil)
+* core: osd/osd_types: kill preferred field in pg_t (`pr#20567 <https://github.com/ceph/ceph/pull/20567>`_, Sage Weil)
+* core: osd/osd_types: object_info_t: remove unused function (`pr#17905 <https://github.com/ceph/ceph/pull/17905>`_, Kefu Chai)
+* core: osd/osd_types: pg_pool_t: remove crash_replay_interval member (`pr#18379 <https://github.com/ceph/ceph/pull/18379>`_, Sage Weil)
+* core: osd/osd_types: remove backlog type for pg_log_entry_t (`pr#20887 <https://github.com/ceph/ceph/pull/20887>`_, Sage Weil)
+* core: osd/OSD: Using Wait rather than WaitInterval to wait queue.is_empty() (`pr#17659 <https://github.com/ceph/ceph/pull/17659>`_, Jianpeng Ma)
+* core: osd/PG: allow scrub preemption (`pr#18971 <https://github.com/ceph/ceph/pull/18971>`_, Sage Weil)
+* core: osd/PGBackend: delete reply if fails to complete delete request (`issue#20913 <http://tracker.ceph.com/issues/20913>`_, `pr#17183 <https://github.com/ceph/ceph/pull/17183>`_, Kefu Chai)
+* core: osd/PGBackend: drop input "snapid_t" from objects_list_range() (`pr#21151 <https://github.com/ceph/ceph/pull/21151>`_, xie xingguo)
+* core: osd/PGBackend: fix large_omap_objects checking (`pr#21150 <https://github.com/ceph/ceph/pull/21150>`_, xie xingguo)
+* core: osd/PGBackend: release a msg using msg->put() not delete (`issue#20913 <http://tracker.ceph.com/issues/20913>`_, `pr#17246 <https://github.com/ceph/ceph/pull/17246>`_, Kefu Chai)
+* core: osd/PG: const cleanup for recoverable/readable predicates (`pr#18982 <https://github.com/ceph/ceph/pull/18982>`_, Neha Ojha)
+* core: osd/PG: decay scrub_chunk_max too if scrub is preempted (`pr#20552 <https://github.com/ceph/ceph/pull/20552>`_, xie xingguo)
+* core: osd/PG: discard msgs from down peers (`issue#19605 <http://tracker.ceph.com/issues/19605>`_, `pr#17217 <https://github.com/ceph/ceph/pull/17217>`_, Kefu Chai)
+* core: osd/PG: drop unused variable "oldest_update" in PG.h (`pr#17142 <https://github.com/ceph/ceph/pull/17142>`_, songweibin)
+* core: osd/PG:  extend pg state bits to fix pg ls commands error (`issue#21609 <http://tracker.ceph.com/issues/21609>`_, `pr#18058 <https://github.com/ceph/ceph/pull/18058>`_, Yan Jun)
+* core: osd/PG: fix calc of misplaced objects (`pr#18528 <https://github.com/ceph/ceph/pull/18528>`_, Kefu Chai)
+* core: osd/PG: fix DeferRecovery vs AllReplicasRecovered race (`issue#23860 <http://tracker.ceph.com/issues/23860>`_, `pr#21706 <https://github.com/ceph/ceph/pull/21706>`_, Sage Weil)
+* core: osd/PG: fix objects degraded higher than 100% (`issue#21803 <http://tracker.ceph.com/issues/21803>`_, `issue#21898 <http://tracker.ceph.com/issues/21898>`_, `pr#18297 <https://github.com/ceph/ceph/pull/18297>`_, Sage Weil, David Zafman)
+* core: osd/PG: fix out of order priority for PG deletion (`pr#21613 <https://github.com/ceph/ceph/pull/21613>`_, xie xingguo)
+* core: osd/PG: fix recovery op leak (`pr#18524 <https://github.com/ceph/ceph/pull/18524>`_, Sage Weil)
+* core: osd/PG: fix uninit read in Incomplete::react(AdvMap&) (`issue#23980 <http://tracker.ceph.com/issues/23980>`_, `pr#21798 <https://github.com/ceph/ceph/pull/21798>`_, Sage Weil)
+* core: osd/PG: force rebuild of missing set on jewel upgrade (`issue#20958 <http://tracker.ceph.com/issues/20958>`_, `pr#16950 <https://github.com/ceph/ceph/pull/16950>`_, Sage Weil)
+* core: osd/PG: include primary in PG operator<< for ec pools (`pr#19453 <https://github.com/ceph/ceph/pull/19453>`_, Sage Weil)
+* core: osd/PGLog: assert out on performing overflowed log trimming (`pr#21580 <https://github.com/ceph/ceph/pull/21580>`_, xie xingguo)
+* core: osd/PGLog: cleanup unused function revise_have (`pr#19329 <https://github.com/ceph/ceph/pull/19329>`_, Enming Zhang)
+* core: osd/PGLog: fix sanity check against \*\*complete-to\*\* iter (`pr#21612 <https://github.com/ceph/ceph/pull/21612>`_, songweibin)
+* core: osd/PGLog: get rid of ineffective container operations (`pr#19161 <https://github.com/ceph/ceph/pull/19161>`_, xie xingguo)
+* core: osd/PGLog: write only changed dup entries (`issue#21026 <http://tracker.ceph.com/issues/21026>`_, `pr#17245 <https://github.com/ceph/ceph/pull/17245>`_, Josh Durgin)
+* core: osd, pg, mgr: make snap trim queue problems visible (`issue#22448 <http://tracker.ceph.com/issues/22448>`_, `pr#19520 <https://github.com/ceph/ceph/pull/19520>`_, Piotr Dałek)
+* core: osd/PG: misc cleanups (`pr#18340 <https://github.com/ceph/ceph/pull/18340>`_, Yan Jun)
+* core: osd/PG: miscellaneous choose acting changes and cleanups (`pr#18481 <https://github.com/ceph/ceph/pull/18481>`_, xie xingguo)
+* core: osd/PG: pass scrub priority to replica (`pr#20317 <https://github.com/ceph/ceph/pull/20317>`_, Sage Weil)
+* core: osd/PG: perfer async_recovery_targets in reverse order of cost (`pr#21578 <https://github.com/ceph/ceph/pull/21578>`_, xie xingguo)
+* core: osd/PG: perfer EC async_recovery_targets in reverse order of cost (`pr#21588 <https://github.com/ceph/ceph/pull/21588>`_, xie xingguo)
+* core: osd/PG: PGPool::update: avoid expensive union_of (`pr#17239 <https://github.com/ceph/ceph/pull/17239>`_, Zac Medico)
+* core: osd/PGPool::update: optimize with subset_of (`pr#17820 <https://github.com/ceph/ceph/pull/17820>`_, Zac Medico)
+* core: osd/PG: reduce some overhead on operating MissingLoc (`pr#18186 <https://github.com/ceph/ceph/pull/18186>`_, xie xingguo)
+* core: osd/PG: remote recovery preemption, and new feature bit to condition it on (`pr#18553 <https://github.com/ceph/ceph/pull/18553>`_, Sage Weil)
+* core: osd/PG: remove unused parameter in calc_ec_acting (`pr#17304 <https://github.com/ceph/ceph/pull/17304>`_, yang.wang)
+* core: osd/PG: restart recovery if NotRecovering and unfound found (`issue#22145 <http://tracker.ceph.com/issues/22145>`_, `pr#18974 <https://github.com/ceph/ceph/pull/18974>`_, Sage Weil)
+* core: osd/PG: revert approx size (`issue#22654 <http://tracker.ceph.com/issues/22654>`_, `pr#18755 <https://github.com/ceph/ceph/pull/18755>`_, Adam Kupczyk)
+* core: osd/PG: re-write of _update_calc_stats and improve pg degraded state (`issue#20059 <http://tracker.ceph.com/issues/20059>`_, `pr#19850 <https://github.com/ceph/ceph/pull/19850>`_, David Zafman)
+* core: osd/PG: some cleanups && add should_gather filter for loop logging (`pr#19546 <https://github.com/ceph/ceph/pull/19546>`_, Enming Zhang)
+* core: osd/PG: two cleanups (`pr#17171 <https://github.com/ceph/ceph/pull/17171>`_, xie xingguo)
+* core: osd/PG: use osd_backfill_retry_interval for schedule_backfill_retry() (`pr#18686 <https://github.com/ceph/ceph/pull/18686>`_, xie xingguo)
+* core: osd/PrimaryLogPG: add condition "is_chunky_scrub_active" to check object in chunky_scrub (`pr#18506 <https://github.com/ceph/ceph/pull/18506>`_, Jianpeng Ma)
+* core: osd/PrimaryLogPG: arrange recovery order by number of missing objects (`pr#18292 <https://github.com/ceph/ceph/pull/18292>`_, xie xingguo)
+* core: osd/PrimaryLogPG: avoid infinite loop when flush collides with write lock (`pr#21653 <https://github.com/ceph/ceph/pull/21653>`_, Sage Weil)
+* core: osd/PrimaryLogPG: calc clone_overlap size in a more efficient and concise way (`pr#17928 <https://github.com/ceph/ceph/pull/17928>`_, xie xingguo)
+* core: osd/PrimaryLogPG: cleanup do_sub_op && do_sub_op_reply and define soid in prepare_transaction more appropriate (`pr#19495 <https://github.com/ceph/ceph/pull/19495>`_, Enming Zhang)
+* core: osd/PrimaryLogPG: clear data digest on WRITEFULL if skip_data_digest (`pr#21676 <https://github.com/ceph/ceph/pull/21676>`_, Sage Weil)
+* core: osd/PrimaryLogPG: clear pin_stats_invalid bit properly on scrub-repair completion (`pr#18052 <https://github.com/ceph/ceph/pull/18052>`_, xie xingguo)
+* core: osd/PrimaryLogPG: defer evict if head \*or\* object intersect scrub interval (`issue#23646 <http://tracker.ceph.com/issues/23646>`_, `pr#21628 <https://github.com/ceph/ceph/pull/21628>`_, Sage Weil)
+* core: osd/PrimaryLogPG: do not pull-up snapc to snapset (`pr#18713 <https://github.com/ceph/ceph/pull/18713>`_, Sage Weil)
+* core: osd/PrimaryLogPG: do not set data digest for bluestore (`pr#17515 <https://github.com/ceph/ceph/pull/17515>`_, xie xingguo)
+* core: osd/PrimaryLogPG: do not set data/omap digest blindly (`pr#18061 <https://github.com/ceph/ceph/pull/18061>`_, xie xingguo)
+* core: osd/PrimaryLogPG: do not use approx_size() for log trimming (`pr#18338 <https://github.com/ceph/ceph/pull/18338>`_, xie xingguo)
+* core: osd/PrimaryLogPG: do_osd_ops - propagate EAGAIN/EINPROGRESS on failok (`pr#17222 <https://github.com/ceph/ceph/pull/17222>`_, xie xingguo)
+* core: osd/PrimaryLogPG: drop unused parameters (`pr#18581 <https://github.com/ceph/ceph/pull/18581>`_, Liao Weizhong)
+* core: osd/PrimaryLogPG: fix dup stat for async read (`pr#18693 <https://github.com/ceph/ceph/pull/18693>`_, Xinze Chi)
+* core: osd/PrimaryLogPG: Fix log messages (`pr#21639 <https://github.com/ceph/ceph/pull/21639>`_, Gu Zhongyan)
+* core: osd/PrimaryLogPG: fix sparse read won't trigger repair correctly (`pr#17221 <https://github.com/ceph/ceph/pull/17221>`_, xie xingguo)
+* core: osd/PrimaryLogPG: fix the oi size mismatch with real object size (`issue#23701 <http://tracker.ceph.com/issues/23701>`_, `pr#21408 <https://github.com/ceph/ceph/pull/21408>`_, Peng Xie)
+* core: osd/PrimaryLogPG: kick off recovery on backoffing a degraded object (`pr#17987 <https://github.com/ceph/ceph/pull/17987>`_, xie xingguo)
+* core: osd/PrimaryLogPG: kill add_interval_usage (`pr#17807 <https://github.com/ceph/ceph/pull/17807>`_, xie xingguo)
+* core: osd/PrimaryLogPG: maybe_handle_manifest_detail - sanity check obc existence (`pr#17298 <https://github.com/ceph/ceph/pull/17298>`_, xie xingguo)
+* core: osd/PrimaryLogPG: misc cleanups (`pr#17830 <https://github.com/ceph/ceph/pull/17830>`_, Yan Jun)
+* core: osd/PrimaryLogPG: more oi.extents fixes (`pr#18616 <https://github.com/ceph/ceph/pull/18616>`_, xie xingguo)
+* core: osd/PrimaryLogPG: prepare_transaction - fix EDQUOT vs ENOSPC (`pr#17808 <https://github.com/ceph/ceph/pull/17808>`_, xie xingguo)
+* core: osd/PrimaryLogPG: request osdmap update in the right block (`issue#21428 <http://tracker.ceph.com/issues/21428>`_, `pr#17828 <https://github.com/ceph/ceph/pull/17828>`_, Josh Durgin)
+* core: osd/PrimaryLogPG: several oi.extents fixes (`pr#18527 <https://github.com/ceph/ceph/pull/18527>`_, xie xingguo)
+* core: osd/PrimaryLogPG: trigger auto-repair on full-object-size CRC error (`pr#18353 <https://github.com/ceph/ceph/pull/18353>`_, xie xingguo)
+* core: osd/ReplicatedBackend: clean up code (`pr#20127 <https://github.com/ceph/ceph/pull/20127>`_, Jianpeng Ma)
+* core: osd/ReplicatedBackend: 'osd_deep_scrub_keys' doesn't work (`pr#20221 <https://github.com/ceph/ceph/pull/20221>`_, fang yuxiang)
+* core: osd/ReplicatedPG: add omap write bytes to delta_stats (`pr#18471 <https://github.com/ceph/ceph/pull/18471>`_, Haomai Wang)
+* core: osd_types.cc: reorder fields in serialized pg_stat_t (`pr#19965 <https://github.com/ceph/ceph/pull/19965>`_, Piotr Dałek)
+* core: os/filestore: disable rocksdb compression (`pr#18707 <https://github.com/ceph/ceph/pull/18707>`_, Sage Weil)
+* core: os/filestore/FileStore: Initialized by nullptr, NULL or 0 instead (`pr#18980 <https://github.com/ceph/ceph/pull/18980>`_, Shinobu Kinjo)
+* core: os/filestore: fix device/partition metadata detection (`issue#20944 <http://tracker.ceph.com/issues/20944>`_, `pr#16913 <https://github.com/ceph/ceph/pull/16913>`_, Sage Weil)
+* core: os/filestore: fix do_copy_range replay bug (`issue#23298 <http://tracker.ceph.com/issues/23298>`_, `pr#20832 <https://github.com/ceph/ceph/pull/20832>`_, Sage Weil)
+* core: os/Filestore: fix wbthrottle assert (`pr#14213 <https://github.com/ceph/ceph/pull/14213>`_, Xiaoxi Chen)
+* core: os/filestore: print out the error if do_read_entry() fails (`pr#18346 <https://github.com/ceph/ceph/pull/18346>`_, Kefu Chai)
+* core: os: FileStore, Using stl min | max, MIN | MAX macros instead (`pr#19832 <https://github.com/ceph/ceph/pull/19832>`_, Shinobu Kinjo)
+* core: os: fix 0-length zero semantics, add tests (`issue#21712 <http://tracker.ceph.com/issues/21712>`_, `pr#18159 <https://github.com/ceph/ceph/pull/18159>`_, Sage Weil)
+* core: os/FuseStore: fix incorrect used space statistics for fuse's statfs interface (`pr#19033 <https://github.com/ceph/ceph/pull/19033>`_, Zhi Zhang)
+* core: os/kstore: Drop unused function declaration (`pr#18077 <https://github.com/ceph/ceph/pull/18077>`_, Jos Collin)
+* core: os/kstore: fix statfs problem and add vstart.sh support (`issue#23590 <http://tracker.ceph.com/issues/23590>`_, `pr#21287 <https://github.com/ceph/ceph/pull/21287>`_, Yang Honggang)
+* core: os/memstore: Fix wrong use of lock_guard (`pr#20914 <https://github.com/ceph/ceph/pull/20914>`_, Shen-Ta Hsieh)
+* core: os/ObjectStore: fix get_data_alignment return -1 causing problem in msg header (`pr#18475 <https://github.com/ceph/ceph/pull/18475>`_, Haomai Wang)
+* core: os/ObjectStore.h: fix mistake in comment TRANSACTION ISOLATION (`pr#16840 <https://github.com/ceph/ceph/pull/16840>`_, mychoxin)
+* core: os,osd: initial work to drop onreadable/onapplied callbacks (`issue#23029 <http://tracker.ceph.com/issues/23029>`_, `pr#20177 <https://github.com/ceph/ceph/pull/20177>`_, Sage Weil)
+* core: os: unify Sequencer and CollectionHandle (`pr#20173 <https://github.com/ceph/ceph/pull/20173>`_, Sage Weil)
+* core: PG: fix name of WaitActingChange (`pr#18768 <https://github.com/ceph/ceph/pull/18768>`_, wumingqiao)
+* core: pg: handle MNotifyRec event in down state (`pr#20959 <https://github.com/ceph/ceph/pull/20959>`_, Mingxin Liu)
+* core: PGPool::update: optimize removed_snaps comparison when possible (`pr#17410 <https://github.com/ceph/ceph/pull/17410>`_, Zac Medico)
+* core: PGPool::update: optimize with interval_set.swap (`pr#17121 <https://github.com/ceph/ceph/pull/17121>`_, Zac Medico)
+* core: PG: primary should not be in the peer_info, skip if it is (`pr#20189 <https://github.com/ceph/ceph/pull/20189>`_, Neha Ojha)
+* core: ptl-tool: checkout branch after creation (`pr#18157 <https://github.com/ceph/ceph/pull/18157>`_, Patrick Donnelly)
+* core: ptl-tool: load labeled PRs (`pr#18231 <https://github.com/ceph/ceph/pull/18231>`_, Patrick Donnelly)
+* core: ptl-tool: make branch name configurable (`pr#18499 <https://github.com/ceph/ceph/pull/18499>`_, Patrick Donnelly)
+* core: ptl-tool: print bzs/tickets cited in commit msgs/comments (`pr#18547 <https://github.com/ceph/ceph/pull/18547>`_, Patrick Donnelly)
+* core: pybind/ceph_argparse: fix cli output info (`pr#17667 <https://github.com/ceph/ceph/pull/17667>`_, Luo Kexue)
+* core: pybind/ceph_argparse: Fix UnboundLocalError if command doesn't validate (`pr#21342 <https://github.com/ceph/ceph/pull/21342>`_, Tim Serong)
+* core:  pybind/ceph_argparse.py:'timeout' must in kwargs when call run_in_thread (`pr#21659 <https://github.com/ceph/ceph/pull/21659>`_, yangdeliu)
+* core,pybind: pybind/ceph_argparse: accept flexible req (`pr#20791 <https://github.com/ceph/ceph/pull/20791>`_, Gu Zhongyan)
+* core,pybind: pybind/rados: add alignment getter to IoCtx (`pr#21222 <https://github.com/ceph/ceph/pull/21222>`_, Bruce Flynn)
+* core,pybind: pybind/rados: add rados_service\_\*() (`pr#18812 <https://github.com/ceph/ceph/pull/18812>`_, Kefu Chai)
+* core,pybind: pybind/rados: add support open_ioctx2 API (`pr#17710 <https://github.com/ceph/ceph/pull/17710>`_, songweibin)
+* core,pybind: rados: support python API of "set_osdmap_full_try" (`pr#17418 <https://github.com/ceph/ceph/pull/17418>`_, songweibin)
+* core: qa: fix the potential delay of pg state change (`pr#17253 <https://github.com/ceph/ceph/pull/17253>`_, huangjun)
+* core: qa/standalone/osd/repro_long_log: no-mon-config for cot (`pr#20919 <https://github.com/ceph/ceph/pull/20919>`_, Sage Weil)
+* core: qa/standalone/scrub/osd-scrub-repair: no -y to diff (`issue#21618 <http://tracker.ceph.com/issues/21618>`_, `pr#18079 <https://github.com/ceph/ceph/pull/18079>`_, Sage Weil)
+* core: qa/suite/rados: fix balancer vs firefly tunables failures (`pr#18826 <https://github.com/ceph/ceph/pull/18826>`_, Sage Weil)
+* core: qa/suites/rados: fewer msgr failures (`pr#20918 <https://github.com/ceph/ceph/pull/20918>`_, Sage Weil)
+* core: qa/suites/rados/perf: whitelist health warnings (`pr#18878 <https://github.com/ceph/ceph/pull/18878>`_, Sage Weil)
+* core: qa/suites/rados/rest/mgr: provision openstack volumes (`pr#20573 <https://github.com/ceph/ceph/pull/20573>`_, Sage Weil)
+* core: qa/suites/rados/singleton/all/mon-seesaw: whitelist MON_DOWN (`pr#18246 <https://github.com/ceph/ceph/pull/18246>`_, Sage Weil)
+* core: qa/suites/rados/singleton/all/recover-preemption: handle slow starting osd (`pr#18078 <https://github.com/ceph/ceph/pull/18078>`_, Sage Weil)
+* core: qa/suites/rados/singleton/all/recovery_preemption: whitelist SLOW_OPS (`pr#21250 <https://github.com/ceph/ceph/pull/21250>`_, Sage Weil)
+* core: qa/suites/rados/singleton/diverget_priors\*: broaden whitelist (`pr#17379 <https://github.com/ceph/ceph/pull/17379>`_, Sage Weil)
+* core: qa/suites/rados/thrash: extend mgr beacon grace when many msgr failures injected (`issue#21147 <http://tracker.ceph.com/issues/21147>`_, `pr#19242 <https://github.com/ceph/ceph/pull/19242>`_, Sage Weil)
+* core: qa/suites/rados/verify/tasks/rados_api_tests: whitelist OBJECT_MISPLACED (`pr#21646 <https://github.com/ceph/ceph/pull/21646>`_, Sage Weil)
+* core: qa/workunits/rest/test.py: stop trying to test obsolte cluster_up/down (`pr#18552 <https://github.com/ceph/ceph/pull/18552>`_, Sage Weil)
+* core: rados/objclass.h: fix build define CEPH_CLS_API in all cases (`pr#21606 <https://github.com/ceph/ceph/pull/21606>`_, Danny Al-Gaaf)
+* core: rados: use WaitInterval()'s return value instead of manual timing (`pr#20028 <https://github.com/ceph/ceph/pull/20028>`_, Mohamad Gebai)
+* core,rbd: common,rbd-nbd: fix up prefork behavior vs AsyncMessenger singletons (`issue#23143 <http://tracker.ceph.com/issues/23143>`_, `pr#20681 <https://github.com/ceph/ceph/pull/20681>`_, Sage Weil)
+* core,rbd: librbd,os: address coverity false positives (`pr#17793 <https://github.com/ceph/ceph/pull/17793>`_, Amit Kumar)
+* core,rbd: mgr,osd,kv: Fix various warnings for Clang and GCC7 (`pr#17976 <https://github.com/ceph/ceph/pull/17976>`_, Adam C. Emerson)
+* core,rbd: vstart.sh: fix mstart variables (`pr#20826 <https://github.com/ceph/ceph/pull/20826>`_, Sage Weil)
+* core: rdma: Assign instead of compare (`pr#16664 <https://github.com/ceph/ceph/pull/16664>`_, amitkuma)
+* core: remove startsync (`issue#20604 <http://tracker.ceph.com/issues/20604>`_, `pr#16396 <https://github.com/ceph/ceph/pull/16396>`_, Amit Kumar)
+* core: rocksdb: sync with upstream (`issue#20529 <http://tracker.ceph.com/issues/20529>`_, `pr#17388 <https://github.com/ceph/ceph/pull/17388>`_, Kefu Chai)
+* core: rocksdb: sync with upstream (`pr#21320 <https://github.com/ceph/ceph/pull/21320>`_, Kefu Chai)
+* core: scrub errors not cleared on replicas can cause inconsistent pg state when replica takes over primary (`issue#23267 <http://tracker.ceph.com/issues/23267>`_, `pr#21101 <https://github.com/ceph/ceph/pull/21101>`_, David Zafman)
+* core: Snapset inconsistency is detected with its own error (`issue#22996 <http://tracker.ceph.com/issues/22996>`_, `pr#20450 <https://github.com/ceph/ceph/pull/20450>`_, David Zafman)
+* core: src/messages/MOSDMap: reencode OSDMap for older clients (`issue#21660 <http://tracker.ceph.com/issues/21660>`_, `pr#18134 <https://github.com/ceph/ceph/pull/18134>`_, Sage Weil)
+* core: src/osd/PG.cc: 6455: FAILED assert(0 == "we got a bad state machine event") (`pr#20933 <https://github.com/ceph/ceph/pull/20933>`_, David Zafman)
+* core: src/test/osd: add two pool test for manifest objects (`pr#20096 <https://github.com/ceph/ceph/pull/20096>`_, Myoungwon Oh)
+* core: test/cli/osdmaptool/test-map-pgs.t: remove nondetermimistic test (`pr#20872 <https://github.com/ceph/ceph/pull/20872>`_, Sage Weil)
+* core: test/objectstore_bench: Don't forget judging whether call usage (`pr#21369 <https://github.com/ceph/ceph/pull/21369>`_, Jianpeng Ma)
+* core,tests: ceph_test_filestore_idempotent_sequence: many fixes (`issue#22920 <http://tracker.ceph.com/issues/22920>`_, `pr#20279 <https://github.com/ceph/ceph/pull/20279>`_, Sage Weil)
+* core,tests: ceph_test_objectstore: drop expect regex (`pr#16968 <https://github.com/ceph/ceph/pull/16968>`_, Sage Weil)
+* core,tests: Erasure code read test and code cleanup (`issue#14513 <http://tracker.ceph.com/issues/14513>`_, `pr#17703 <https://github.com/ceph/ceph/pull/17703>`_, David Zafman)
+* core,tests: Erasure code recovery should send additional reads if necessary (`issue#21382 <http://tracker.ceph.com/issues/21382>`_, `pr#17920 <https://github.com/ceph/ceph/pull/17920>`_, David Zafman)
+* core,tests: osd,dmclock: fix dmclock test simulator change (`pr#20270 <https://github.com/ceph/ceph/pull/20270>`_, J. Eric Ivancich)
+* core,tests: os: kstore fix unittest for FiemapHole (`pr#17313 <https://github.com/ceph/ceph/pull/17313>`_, Ning Yao)
+* core,tests: os/memstore: memstore_page_set=false (`issue#20738 <http://tracker.ceph.com/issues/20738>`_, `pr#16995 <https://github.com/ceph/ceph/pull/16995>`_, Sage Weil)
+* core,tests: qa/ceph_manager: check pg state again before timedout (`issue#21294 <http://tracker.ceph.com/issues/21294>`_, `pr#17810 <https://github.com/ceph/ceph/pull/17810>`_, huangjun)
+* core,tests: qa/clusters/fixed-[23]: 4 osds per node, not 3 (`pr#16799 <https://github.com/ceph/ceph/pull/16799>`_, Sage Weil)
+* core,tests: qa: modify rgw default pool names (`pr#21630 <https://github.com/ceph/ceph/pull/21630>`_, Neha Ojha)
+* core,tests: qa/objectstore/bluestore\*: less debug output (`issue#20910 <http://tracker.ceph.com/issues/20910>`_, `pr#17505 <https://github.com/ceph/ceph/pull/17505>`_, Sage Weil)
+* core,tests: qa/overrides/2-size-2-min-size: whitelist REQUEST_STUCK (`pr#17243 <https://github.com/ceph/ceph/pull/17243>`_, Sage Weil)
+* core,tests: qa/standalone/ceph-helpers: pass --verbose to ceph-disk (`pr#19456 <https://github.com/ceph/ceph/pull/19456>`_, Sage Weil)
+* core,tests: qa/standalone/scrub/osd-scrub-repair: fix grep pattern (`issue#21127 <http://tracker.ceph.com/issues/21127>`_, `pr#17258 <https://github.com/ceph/ceph/pull/17258>`_, Sage Weil)
+* core,tests: qa/standalone/scrub/osd-scrub-snaps: adjust test for lack of snapdir objects (`pr#17927 <https://github.com/ceph/ceph/pull/17927>`_, Sage Weil)
+* core,tests: qa/suites/rados/monthrash: tolerate PG_AVAILABILITY during mon thrashing (`pr#18122 <https://github.com/ceph/ceph/pull/18122>`_, Sage Weil)
+* core,tests: qa/suites/rados/monthrash: whitelist SLOW_OPS (`pr#21331 <https://github.com/ceph/ceph/pull/21331>`_, Sage Weil)
+* core,tests: qa/suites/rados/objectstore: logs (`issue#20738 <http://tracker.ceph.com/issues/20738>`_, `pr#16923 <https://github.com/ceph/ceph/pull/16923>`_, Sage Weil)
+* core,tests: qa/suites/rados/perf: create pool with lower pg_num (`pr#17819 <https://github.com/ceph/ceph/pull/17819>`_, Neha Ojha)
+* core,tests: qa/suites/rados/rest/mgr-restful: whitelist more health (`pr#18457 <https://github.com/ceph/ceph/pull/18457>`_, Sage Weil)
+* core,tests: qa/suites/rados/rest: move rest_test from qa/suites/rest/ (`pr#19175 <https://github.com/ceph/ceph/pull/19175>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash: fix thrashing with ec vs map discon (`pr#16842 <https://github.com/ceph/ceph/pull/16842>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: add hammer clients (`pr#21703 <https://github.com/ceph/ceph/pull/21703>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: add rbd tests (`pr#21704 <https://github.com/ceph/ceph/pull/21704>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: do some thrashing with jewel and luminous clients (`pr#21679 <https://github.com/ceph/ceph/pull/21679>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: only centos and 16.04 (`pr#22125 <https://github.com/ceph/ceph/pull/22125>`_, Sage Weil)
+* core,tests: qa/suites/upgrade/jewel-x/stress-split: tolerate sloppy past_intervals (`pr#17226 <https://github.com/ceph/ceph/pull/17226>`_, Sage Weil)
+* core,tests: qa/suites/upgrade/luminous-x/stress-split: avoid enospc (`pr#21753 <https://github.com/ceph/ceph/pull/21753>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: revive osds before doing final rerr reset (`issue#21206 <http://tracker.ceph.com/issues/21206>`_, `pr#17406 <https://github.com/ceph/ceph/pull/17406>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: tolerate tell osd.\* error (`pr#19365 <https://github.com/ceph/ceph/pull/19365>`_, Sage Weil)
+* core,tests: qa/tasks/ceph.py: tolerate flush pg stats exception (`pr#16905 <https://github.com/ceph/ceph/pull/16905>`_, Sage Weil)
+* core,tests: qa/tasks/filestore_idempotent: shorter test (`pr#20509 <https://github.com/ceph/ceph/pull/20509>`_, Sage Weil)
+* core,tests: qa/tasks/thrashosds: set min_in default to 4 (`issue#21997 <http://tracker.ceph.com/issues/21997>`_, `pr#18670 <https://github.com/ceph/ceph/pull/18670>`_, Sage Weil)
+* core,tests: qa/tests: run ceph-ansible task on installer.0 role/node (`pr#19605 <https://github.com/ceph/ceph/pull/19605>`_, Yuri Weinstein)
+* core,tests: qa: tolerate failure to force backfill (`issue#22614 <http://tracker.ceph.com/issues/22614>`_, `pr#19765 <https://github.com/ceph/ceph/pull/19765>`_, Sage Weil)
+* core,tests: qa/workunits/rados/test_rados_tool: fix stray ``|``, race (`issue#22676 <http://tracker.ceph.com/issues/22676>`_, `pr#19946 <https://github.com/ceph/ceph/pull/19946>`_, Sage Weil)
+* core,tests: qa/workunits/rados/test.sh: ensure tee output is valid filename (`pr#21507 <https://github.com/ceph/ceph/pull/21507>`_, Sage Weil)
+* core,tests: rados: Initialization of alignment (`pr#17723 <https://github.com/ceph/ceph/pull/17723>`_, Amit Kumar)
+* core,tests: rados: Initializing members of librados/TestCase.h (`pr#16896 <https://github.com/ceph/ceph/pull/16896>`_, amitkuma)
+* core,tests: test: Checking fd for negative before closing (`pr#17190 <https://github.com/ceph/ceph/pull/17190>`_, amitkuma)
+* core,tests: test: Check to avoid divide by zero (`pr#17220 <https://github.com/ceph/ceph/pull/17220>`_, amitkuma)
+* core: tool: change default objectstore from filestore to bluestore (`pr#18066 <https://github.com/ceph/ceph/pull/18066>`_, Song Shun)
+* core: tool: misc cleanup of ceph-kvstore-tool (`issue#22092 <http://tracker.ceph.com/issues/22092>`_, `pr#18815 <https://github.com/ceph/ceph/pull/18815>`_, Chang Liu)
+* core,tools: Add export and remove ceph-objectstore-tool command option (`issue#21272 <http://tracker.ceph.com/issues/21272>`_, `pr#17538 <https://github.com/ceph/ceph/pull/17538>`_, David Zafman)
+* core,tools: ceph-objectstore-tool: fix import of post-split pg from pre-split ancestor (`issue#21753 <http://tracker.ceph.com/issues/21753>`_, `pr#18229 <https://github.com/ceph/ceph/pull/18229>`_, Sage Weil)
+* core: tools/ceph-objectstore-tool: split filestore directories offline to target hash level (`issue#21366 <http://tracker.ceph.com/issues/21366>`_, `pr#17666 <https://github.com/ceph/ceph/pull/17666>`_, Zhi Zhang)
+* core,tools: common, tool: update kvstore-tool to repair key/value database (`issue#17730 <http://tracker.ceph.com/issues/17730>`_, `issue#21744 <http://tracker.ceph.com/issues/21744>`_, `pr#16745 <https://github.com/ceph/ceph/pull/16745>`_, liuchang0812, Chang Liu)
+* core,tools: osd,os/bluestore: kill clang analyzer warnings (`pr#18015 <https://github.com/ceph/ceph/pull/18015>`_, Kefu Chai)
+* core: tools/rados: add touch command to change object modification time (`pr#18913 <https://github.com/ceph/ceph/pull/18913>`_, Yao Zongyou)
+* core,tools: scripts: add ptl-tool for scripting merges (`pr#17926 <https://github.com/ceph/ceph/pull/17926>`_, Patrick Donnelly)
+* core: vstart.sh: drop .ceph_port and use randomly selected available port (`pr#19268 <https://github.com/ceph/ceph/pull/19268>`_, Shinobu Kinjo)
+* core: vstart.sh: drop --{mon,osd,mds,rgw,mgr}_num options (`pr#18648 <https://github.com/ceph/ceph/pull/18648>`_, Kefu Chai)
+* core: vstart.sh: Remove duplicate global section (`pr#17543 <https://github.com/ceph/ceph/pull/17543>`_, iliul)
+* crush: cleanup update_device_class() log messages (`pr#21174 <https://github.com/ceph/ceph/pull/21174>`_, Gu Zhongyan)
+* crush: fix CrushCompiler won't compile maps with empty shadow tree (`pr#17058 <https://github.com/ceph/ceph/pull/17058>`_, xie xingguo)
+* crush: fix device_class_clone for unpopulated/empty weight-sets (`issue#23386 <http://tracker.ceph.com/issues/23386>`_, `pr#22169 <https://github.com/ceph/ceph/pull/22169>`_, Sage Weil)
+* crush: fix fast rule lookup when uniform (`pr#17510 <https://github.com/ceph/ceph/pull/17510>`_, Sage Weil)
+* crush: force rebuilding shadow hierarchy after swapping buckets (`pr#17083 <https://github.com/ceph/ceph/pull/17083>`_, xie xingguo)
+* crush: improve straw2 algorithm's readability (`pr#20196 <https://github.com/ceph/ceph/pull/20196>`_, Yao Zongyou)
+* crush: "osd crush class rename" support (`pr#16961 <https://github.com/ceph/ceph/pull/16961>`_, xie xingguo)
+* crush, osd: handle multiple parents properly when applying pg upmaps (`issue#23921 <http://tracker.ceph.com/issues/23921>`_, `pr#21835 <https://github.com/ceph/ceph/pull/21835>`_, xiexingguo)
+* crush: safe check for 'ceph osd crush swap-bucket' (`pr#17335 <https://github.com/ceph/ceph/pull/17335>`_, Carudy)
+* crush: various CrushWrapper cleanups (`pr#17360 <https://github.com/ceph/ceph/pull/17360>`_, Kefu Chai)
+* crush: various weight-set fixes (`pr#17014 <https://github.com/ceph/ceph/pull/17014>`_, xie xingguo)
+* denc: should check element's type not 'size_t' (`pr#19986 <https://github.com/ceph/ceph/pull/19986>`_, Kefu Chai)
+* denc: use constexpr-if to replace some SFINAE impls (`pr#19662 <https://github.com/ceph/ceph/pull/19662>`_, Kefu Chai)
+* doc: 12.1.3 release notes (`pr#16975 <https://github.com/ceph/ceph/pull/16975>`_, Abhishek Lekshmanan)
+* doc: 12.2.0 major release announcements (`pr#16915 <https://github.com/ceph/ceph/pull/16915>`_, Abhishek Lekshmanan)
+* doc: 12.2.1 release notes (`pr#18014 <https://github.com/ceph/ceph/pull/18014>`_, Abhishek Lekshmanan)
+* doc: 12.2.4 release notes (`pr#20619 <https://github.com/ceph/ceph/pull/20619>`_, Abhishek Lekshmanan)
+* doc: add 12.2.2 release notes (`pr#19264 <https://github.com/ceph/ceph/pull/19264>`_, Abhishek Lekshmanan)
+* doc: add allow_multimds and fs_name parameter (`pr#15847 <https://github.com/ceph/ceph/pull/15847>`_, Jan Fajerski)
+* doc: add ceph-kvstore-tool's man (`pr#17092 <https://github.com/ceph/ceph/pull/17092>`_, liuchang0812)
+* doc: add changelog for 12.2.1 (`pr#18020 <https://github.com/ceph/ceph/pull/18020>`_, Abhishek Lekshmanan)
+* doc: add changelog for v11.2.1 (`pr#16956 <https://github.com/ceph/ceph/pull/16956>`_, Abhishek Lekshmanan)
+* doc: add changelog for v12.2.2 (`pr#19284 <https://github.com/ceph/ceph/pull/19284>`_, Abhishek Lekshmanan)
+* doc: Added CHAP configuration instructions for iSCSI (`pr#18423 <https://github.com/ceph/ceph/pull/18423>`_, Ashish Singh)
+* doc: add example of setting pool in cephfs layout (`pr#17372 <https://github.com/ceph/ceph/pull/17372>`_, John Spray)
+* doc: Adding changelog for 10.2.10 (`pr#18151 <https://github.com/ceph/ceph/pull/18151>`_, Abhishek Lekshmanan)
+* doc: Add introduction about different way to run rbd-mirror (`pr#19692 <https://github.com/ceph/ceph/pull/19692>`_, Yu Shengzuo)
+* doc: add --max-buckets to radosgw-admin(8) (`pr#17439 <https://github.com/ceph/ceph/pull/17439>`_, Clément Pellegrini)
+* doc: add missing blank line (`pr#18724 <https://github.com/ceph/ceph/pull/18724>`_, iliul)
+* doc: Add missing pg states from doc (`pr#20504 <https://github.com/ceph/ceph/pull/20504>`_, David Zafman)
+* doc: add mount.fuse.ceph to index (`issue#22595 <http://tracker.ceph.com/issues/22595>`_, `pr#19792 <https://github.com/ceph/ceph/pull/19792>`_, Jos Collin)
+* doc: Add newbie-friendly updates to Helm start doc (`pr#18886 <https://github.com/ceph/ceph/pull/18886>`_, Blaine Gardner)
+* doc: add osd_max_object_size in osd configuration (`pr#18115 <https://github.com/ceph/ceph/pull/18115>`_, Mohamad Gebai)
+* doc: build-doc: Upgrade ceph python libraries (`pr#20726 <https://github.com/ceph/ceph/pull/20726>`_, Boris Ranto)
+* doc: ceph-disk: create deprecation warnings (`issue#22154 <http://tracker.ceph.com/issues/22154>`_, `pr#18988 <https://github.com/ceph/ceph/pull/18988>`_, Alfredo Deza)
+* doc: ceph-volume: automatic VDO detection (`issue#23581 <http://tracker.ceph.com/issues/23581>`_, `pr#21451 <https://github.com/ceph/ceph/pull/21451>`_, Alfredo Deza)
+* doc: ceph-volume docs (`pr#17068 <https://github.com/ceph/ceph/pull/17068>`_, Alfredo Deza)
+* doc: ceph-volume document multipath support (`pr#20878 <https://github.com/ceph/ceph/pull/20878>`_, Alfredo Deza)
+* doc: ceph-volume doc updates (`pr#20758 <https://github.com/ceph/ceph/pull/20758>`_, Alfredo Deza)
+* doc: ceph-volume include physical devices associated with an LV when listing (`pr#21645 <https://github.com/ceph/ceph/pull/21645>`_, Alfredo Deza)
+* doc: ceph-volume lvm bluestore support (`pr#18448 <https://github.com/ceph/ceph/pull/18448>`_, Alfredo Deza)
+* doc/ceph-volume OSD use the fsid file, not the osd_fsid (`issue#22427 <http://tracker.ceph.com/issues/22427>`_, `pr#20059 <https://github.com/ceph/ceph/pull/20059>`_, Alfredo Deza)
+* doc: change boolean option default value from zero to false (`pr#17733 <https://github.com/ceph/ceph/pull/17733>`_, Yao Zongyou)
+* doc: change cn mirror to ustc domain (`pr#18081 <https://github.com/ceph/ceph/pull/18081>`_, Shengjing Zhu)
+* doc: changelog for v12.2.3 (`pr#20503 <https://github.com/ceph/ceph/pull/20503>`_, Abhishek Lekshmanan)
+* doc: cleanup erasure coded pool doc on cephfs use (`pr#20572 <https://github.com/ceph/ceph/pull/20572>`_, Patrick Donnelly)
+* doc: CodingStyle: add python and javascript/typescript (`pr#20186 <https://github.com/ceph/ceph/pull/20186>`_, Joao Eduardo Luis)
+* doc: common/options: document filestore and filejournal options (`pr#17739 <https://github.com/ceph/ceph/pull/17739>`_, Sage Weil)
+* doc: common/options: document objecter, filer, and journal options (`pr#17740 <https://github.com/ceph/ceph/pull/17740>`_, Sage Weil)
+* doc: complete and update the subsystem logging level info table (`pr#18500 <https://github.com/ceph/ceph/pull/18500>`_, Luo Kexue)
+* doc: correcting typos in bluestore-config-ref and bluestore-migration (`pr#19154 <https://github.com/ceph/ceph/pull/19154>`_, Katie Holly)
+* doc: correct wrong bluestore config types (`pr#18205 <https://github.com/ceph/ceph/pull/18205>`_, Yao Zongyou)
+* doc: delete duplicate words (`pr#17104 <https://github.com/ceph/ceph/pull/17104>`_, iliul)
+* doc: dev description of async recovery (`pr#21051 <https://github.com/ceph/ceph/pull/21051>`_, Neha Ojha, Josh Durgin)
+* doc: doc/bluestore: add SPDK usage for bluestore (`pr#17654 <https://github.com/ceph/ceph/pull/17654>`_, Haomai Wang)
+* doc:  doc/cephfs/experimental-features: kernel client snapshots limit (`pr#18579 <https://github.com/ceph/ceph/pull/18579>`_, Ilya Dryomov)
+* doc: doc/cephfs/posix: remove stale information for seekdir (`pr#17658 <https://github.com/ceph/ceph/pull/17658>`_, "Yan, Zheng")
+* doc: doc/conf.py: do not set html_use_smartypants explicitly (`pr#17127 <https://github.com/ceph/ceph/pull/17127>`_, Kefu Chai)
+* doc: doc/dev: add a brief guide to serialization (`pr#20131 <https://github.com/ceph/ceph/pull/20131>`_, John Spray)
+* doc: doc/dev/cxx: add C++11 ABI related doc (`pr#20030 <https://github.com/ceph/ceph/pull/20030>`_, Kefu Chai)
+* doc: doc/dev/iana: document our official IANA numbers (`pr#16910 <https://github.com/ceph/ceph/pull/16910>`_, Sage Weil)
+* doc: doc/dev/index: update rados lead (`pr#16911 <https://github.com/ceph/ceph/pull/16911>`_, Sage Weil)
+* doc: doc/dev/macos: add doc for building on MacOS (`pr#20031 <https://github.com/ceph/ceph/pull/20031>`_, Kefu Chai)
+* doc: doc/dev/msgr2.rst: a few notes on protocol goals (`pr#20083 <https://github.com/ceph/ceph/pull/20083>`_, Sage Weil)
+* doc: doc/dev/perf: add doc on disabling -fomit-frame-pointer (`pr#17358 <https://github.com/ceph/ceph/pull/17358>`_, Kefu Chai)
+* doc: doc for mount.fuse.ceph (`issue#21539 <http://tracker.ceph.com/issues/21539>`_, `pr#19172 <https://github.com/ceph/ceph/pull/19172>`_, Jos Collin)
+* doc: doc/man remove deprecation of ceph-disk man page title (`pr#19325 <https://github.com/ceph/ceph/pull/19325>`_, Alfredo Deza)
+* doc: doc/mgr: Add limitations section to plugin guide (`pr#21347 <https://github.com/ceph/ceph/pull/21347>`_, Tim Serong)
+* doc: doc/mgr: add "local pool" plugin to toc (`pr#17961 <https://github.com/ceph/ceph/pull/17961>`_, Kefu Chai)
+* doc: doc/mgr/balancer: document (`issue#22789 <http://tracker.ceph.com/issues/22789>`_, `pr#21421 <https://github.com/ceph/ceph/pull/21421>`_, Sage Weil)
+* doc: doc/mgr: document facilities methods using `automethod` directive (`pr#18680 <https://github.com/ceph/ceph/pull/18680>`_, Kefu Chai)
+* doc: doc/mgr/plugins: add note about distinction between config and kv store (`pr#21671 <https://github.com/ceph/ceph/pull/21671>`_, Jan Fajerski)
+* doc: doc/mgr: remove non user-facing code from doc (`pr#20372 <https://github.com/ceph/ceph/pull/20372>`_, Kefu Chai)
+* doc: doc,os,osdc: drop and modify comments (`pr#17732 <https://github.com/ceph/ceph/pull/17732>`_, songweibin)
+* doc: doc/rados: Add explanation of straw2 (`pr#19247 <https://github.com/ceph/ceph/pull/19247>`_, Shinobu Kinjo)
+* doc: doc/rados/operations/bluestore-migration: document bluestore migration process (`pr#16918 <https://github.com/ceph/ceph/pull/16918>`_, Sage Weil)
+* doc: doc/rados/operations/bluestore-migration: update docs a bit (`pr#17011 <https://github.com/ceph/ceph/pull/17011>`_, Sage Weil)
+* doc: doc raise exceptions with a base class (`pr#18152 <https://github.com/ceph/ceph/pull/18152>`_, Alfredo Deza)
+* doc: doc/rbd: add info for rbd group (`pr#17633 <https://github.com/ceph/ceph/pull/17633>`_, yonghengdexin735)
+* doc:  doc/rbd: add missing several commands in rbd CLI man page (`issue#14539 <http://tracker.ceph.com/issues/14539>`_, `issue#16999 <http://tracker.ceph.com/issues/16999>`_, `pr#19659 <https://github.com/ceph/ceph/pull/19659>`_, songweibin)
+* doc: doc/rbd: correct the path of librbd python APIs (`pr#19690 <https://github.com/ceph/ceph/pull/19690>`_, songweibin)
+* doc: doc/rbd: fix typo s/morror/mirror (`pr#19997 <https://github.com/ceph/ceph/pull/19997>`_, songweibin)
+* doc: doc/rbd: iSCSI Gateway Documentation (`issue#20437 <http://tracker.ceph.com/issues/20437>`_, `pr#17376 <https://github.com/ceph/ceph/pull/17376>`_, Aron Gunn, Jason Dillaman)
+* doc: doc/rbd: specify additional ESX prerequisites (`pr#18517 <https://github.com/ceph/ceph/pull/18517>`_, Jason Dillaman)
+* doc: doc/rbd: tweaks for the LIO iSCSI gateway (`issue#21763 <http://tracker.ceph.com/issues/21763>`_, `pr#18250 <https://github.com/ceph/ceph/pull/18250>`_, Jason Dillaman)
+* doc: doc/rbd: tweaks to the Windows iSCSI initiator directions (`pr#18704 <https://github.com/ceph/ceph/pull/18704>`_, Jason Dillaman)
+* doc: doc/release-notes: add jewel->kraken notes (`pr#18482 <https://github.com/ceph/ceph/pull/18482>`_, Sage Weil)
+* doc: doc/release-notes: clarify purpose of require-osd-release (`pr#17270 <https://github.com/ceph/ceph/pull/17270>`_, Sage Weil)
+* doc: doc/release-notes: clarify that you need to keep your existing OSD caps (`pr#18825 <https://github.com/ceph/ceph/pull/18825>`_, Jason Dillaman)
+* doc: doc/release-notes: ensure RBD users can blacklist prior to upgrade (`issue#21353 <http://tracker.ceph.com/issues/21353>`_, `pr#17755 <https://github.com/ceph/ceph/pull/17755>`_, Jason Dillaman)
+* doc: doc/release-notes: fix typo 'psd' to 'osd' (`pr#18695 <https://github.com/ceph/ceph/pull/18695>`_, wangsongbo)
+* doc: doc/releases: the Kraken sleepeth, faintest sunlights flee (`pr#17424 <https://github.com/ceph/ceph/pull/17424>`_, Abhishek Lekshmanan)
+* doc: doc/releases: update release cycle docs (`pr#18117 <https://github.com/ceph/ceph/pull/18117>`_, Sage Weil)
+* doc: doc/rgw: add page for http frontend configuration (`issue#13523 <http://tracker.ceph.com/issues/13523>`_, `pr#20058 <https://github.com/ceph/ceph/pull/20058>`_, Casey Bodley)
+* doc: doc/scripts: py3 compatible (`pr#17640 <https://github.com/ceph/ceph/pull/17640>`_, Kefu Chai)
+* doc: docs: Do not use "min size = 1" as an example (`pr#17912 <https://github.com/ceph/ceph/pull/17912>`_, Alfredo Deza)
+* doc: docs fix ceph-volume missing sub-commands (`issue#23148 <http://tracker.ceph.com/issues/23148>`_, `pr#20673 <https://github.com/ceph/ceph/pull/20673>`_, Alfredo Deza)
+* doc: doc/start/os-recommendations.rst: bump krbd kernels (`pr#21478 <https://github.com/ceph/ceph/pull/21478>`_, Ilya Dryomov)
+* doc: docs update ceph-deploy reference to reflect ceph-volume API (`pr#20510 <https://github.com/ceph/ceph/pull/20510>`_, Alfredo Deza)
+* doc: document ceph-disk prepare class hierarchy (`pr#17019 <https://github.com/ceph/ceph/pull/17019>`_, Loic Dachary)
+* doc: document include/ipaddr.h (`issue#12056 <http://tracker.ceph.com/issues/12056>`_, `pr#17613 <https://github.com/ceph/ceph/pull/17613>`_, Nathan Cutler)
+* doc: drop duplicate line in ceph-bluestore-tool man page (`pr#19169 <https://github.com/ceph/ceph/pull/19169>`_, Xiaojun Liao)
+* doc: eliminate useless cat statement (`pr#17154 <https://github.com/ceph/ceph/pull/17154>`_, Ken Dreyer)
+* doc: examples: add new librbd example (`pr#18314 <https://github.com/ceph/ceph/pull/18314>`_, Mahati Chamarthy)
+* doc: expand developer documentation of unit tests (`pr#19594 <https://github.com/ceph/ceph/pull/19594>`_, Nathan Cutler)
+* doc: Fix a grammar error in rbd-snapshot.rst (`pr#21470 <https://github.com/ceph/ceph/pull/21470>`_, Zeqing Tyler Qi)
+* doc: fix CFLAGS in doc/dev/cpu-profiler.rst (`pr#19752 <https://github.com/ceph/ceph/pull/19752>`_, Chang Liu)
+* doc: fix desc of option "mon cluster log file" (`pr#18770 <https://github.com/ceph/ceph/pull/18770>`_, Kefu Chai)
+* doc: fix doc/radosgw/admin.rst typos (`pr#17397 <https://github.com/ceph/ceph/pull/17397>`_, Enming Zhang)
+* doc: Fix dynamic resharding doc formatting (`pr#20970 <https://github.com/ceph/ceph/pull/20970>`_, Ashish Singh)
+* doc: fix error in osd scrub load threshold (`pr#21678 <https://github.com/ceph/ceph/pull/21678>`_, Dirk Sarpe)
+* doc: Fixes a spelling error and a broken hyperlink (`pr#20442 <https://github.com/ceph/ceph/pull/20442>`_, Jordan Hus)
+* doc: Fixes rbd snapshot flatten example (`issue#17723 <http://tracker.ceph.com/issues/17723>`_, `pr#17436 <https://github.com/ceph/ceph/pull/17436>`_, Ashish Singh)
+* doc: fixes syntax in osd-config-ref (`issue#21733 <http://tracker.ceph.com/issues/21733>`_, `pr#18188 <https://github.com/ceph/ceph/pull/18188>`_, Joshua Schmid)
+* doc: Fixes the name of the CephFS snapshot directory (`pr#18710 <https://github.com/ceph/ceph/pull/18710>`_, Jordan Rodgers)
+* doc: fix hyper link to radosgw/config-ref (`pr#17986 <https://github.com/ceph/ceph/pull/17986>`_, Kefu Chai)
+* doc: fix librbdpy example (`pr#20019 <https://github.com/ceph/ceph/pull/20019>`_, Yuan Zhou)
+* doc: fix order of options in osd new (`issue#21023 <http://tracker.ceph.com/issues/21023>`_, `pr#17326 <https://github.com/ceph/ceph/pull/17326>`_, Neha Ojha)
+* doc: fix sphinx build warnings and errors (`pr#17025 <https://github.com/ceph/ceph/pull/17025>`_, Alfredo Deza)
+* doc: fix the desc of "osd max pg per osd hard ratio" (`pr#18373 <https://github.com/ceph/ceph/pull/18373>`_, Kefu Chai)
+* doc: Fix typo and URL (`pr#18040 <https://github.com/ceph/ceph/pull/18040>`_, Jos Collin)
+* doc: fix typo e.g,. => e.g (`pr#18607 <https://github.com/ceph/ceph/pull/18607>`_, Yao Zongyou)
+* doc: fix typo in bluestore-migration.rst (`pr#18389 <https://github.com/ceph/ceph/pull/18389>`_, Yao Zongyou)
+* doc: Fix typo in mount.fuse.ceph (`pr#19215 <https://github.com/ceph/ceph/pull/19215>`_, Jos Collin)
+* doc: fix typo in php.rst (`pr#17762 <https://github.com/ceph/ceph/pull/17762>`_, Yao Zongyou)
+* doc: fix typo in radosgw/dynamicresharding.rst (`pr#18651 <https://github.com/ceph/ceph/pull/18651>`_, Alexander Ermolaev)
+* doc: fix typo on specify db block device (`pr#17590 <https://github.com/ceph/ceph/pull/17590>`_, Xiaoxi Chen)
+* doc: Fix typo s/applicatoin/application/ (`pr#20720 <https://github.com/ceph/ceph/pull/20720>`_, Francois Deppierraz)
+* doc: Fix typos in placement-groups.rst (`pr#17973 <https://github.com/ceph/ceph/pull/17973>`_, Matt Boyle)
+* doc: Fix typos in release notes (`pr#18950 <https://github.com/ceph/ceph/pull/18950>`_, Stefan Knorr)
+* doc: .githubmap: Add cbodley (`pr#18946 <https://github.com/ceph/ceph/pull/18946>`_, Jos Collin)
+* doc: githubmap: add map for GitHub contributor lookup (`pr#17457 <https://github.com/ceph/ceph/pull/17457>`_, Patrick Donnelly)
+* doc: .githubmap, .mailmap, .organizationmap, .peoplemap: update Igor (`pr#19314 <https://github.com/ceph/ceph/pull/19314>`_, Igor Fedotov)
+* doc: globally change CRUSH ruleset to CRUSH rule (`issue#20559 <http://tracker.ceph.com/issues/20559>`_, `pr#19435 <https://github.com/ceph/ceph/pull/19435>`_, Nathan Cutler)
+* doc: Improved dashboard documentation (`pr#21443 <https://github.com/ceph/ceph/pull/21443>`_, Lenz Grimmer)
+* doc: Improved hitset parameters description (`pr#19691 <https://github.com/ceph/ceph/pull/19691>`_, Alexey Stupnikov)
+* doc: improve links in doc/releases.rst (`pr#18155 <https://github.com/ceph/ceph/pull/18155>`_, Nathan Cutler)
+* doc: Improve mgr/restful module documentation (`pr#20717 <https://github.com/ceph/ceph/pull/20717>`_, Boris Ranto)
+* doc: Improve the ceph fs set max_mds command (`issue#21007 <http://tracker.ceph.com/issues/21007>`_, `pr#17044 <https://github.com/ceph/ceph/pull/17044>`_, Bara Ancincova)
+* doc: include ceph-disk and ceph-disk-volume man pages in index (`pr#17168 <https://github.com/ceph/ceph/pull/17168>`_, Alfredo Deza)
+* doc: init flags to 0 in rados example (`pr#20671 <https://github.com/ceph/ceph/pull/20671>`_, Patrick Donnelly)
+* doc: Kube + Helm installation (`pr#18520 <https://github.com/ceph/ceph/pull/18520>`_, Alexandre Marangone)
+* doc: legal: remove doc license ambiguity (`issue#23336 <http://tracker.ceph.com/issues/23336>`_, `pr#20876 <https://github.com/ceph/ceph/pull/20876>`_, Nathan Cutler)
+* doc: lock_timeout is a per mapping option (`pr#21563 <https://github.com/ceph/ceph/pull/21563>`_, Ilya Dryomov)
+* doc: log-and-debug: fix default value of "log max recent" (`pr#20316 <https://github.com/ceph/ceph/pull/20316>`_, Nathan Cutler)
+* doc: mailmap: Add Sibei, XueYu Affiliation (`pr#18395 <https://github.com/ceph/ceph/pull/18395>`_, Sibei Gao)
+* doc: mailmap: Fixed maintenance guide URL (`pr#18076 <https://github.com/ceph/ceph/pull/18076>`_, Jos Collin)
+* doc: mailmap, organizationmap: add Dongsheng, Liuzhong, Pengcheng, Yang Affiliation (`pr#17548 <https://github.com/ceph/ceph/pull/17548>`_, Dongsheng Yang)
+* doc: .mailmap, .organizationmap: add Fufei, Mingqiao and Ying Affiliation (`pr#17540 <https://github.com/ceph/ceph/pull/17540>`_, Ying He)
+* doc: .mailmap, .organizationmap: Add Liu Lei's mailmap and affiliation (`pr#17105 <https://github.com/ceph/ceph/pull/17105>`_, iliul)
+* doc: .mailmap, .organizationmap: update JingChen, ZongyouYao, ShanchunLv's… (`pr#18960 <https://github.com/ceph/ceph/pull/18960>`_, Chang Liu)
+* doc: mailmap: update affiliation for Mykola Golub (`pr#18069 <https://github.com/ceph/ceph/pull/18069>`_, Mykola Golub)
+* doc: mailmap: update affiliation for Mykola Golub (`pr#19667 <https://github.com/ceph/ceph/pull/19667>`_, Mykola Golub)
+* doc: mailmap: Update umcloud affiliation (`pr#17441 <https://github.com/ceph/ceph/pull/17441>`_, Yixing Yan)
+* doc: make the commands in README.md properly aligned (`pr#18639 <https://github.com/ceph/ceph/pull/18639>`_, Yao Zongyou)
+* doc/man: add "ls" to "ceph osd" command's subcommands list (`pr#19382 <https://github.com/ceph/ceph/pull/19382>`_, Rishabh Dave)
+* doc: "mds blacklist interval" vs manually blacklisting (`pr#18195 <https://github.com/ceph/ceph/pull/18195>`_, Ken Dreyer)
+* doc: mgr/dashboard.rst: mention ceph.conf and ceph mgr services (`pr#20961 <https://github.com/ceph/ceph/pull/20961>`_, Nathan Cutler)
+* doc/mgr/plugins: mgr accessor during init causes exception (`pr#16973 <https://github.com/ceph/ceph/pull/16973>`_, Jan Fajerski)
+* doc: mimic: doc: Updated dashboard documentation (features, SSL config) (`pr#22079 <https://github.com/ceph/ceph/pull/22079>`_, Lenz Grimmer)
+* doc: misc fix spell errors in osd/OSD and doc (`pr#17107 <https://github.com/ceph/ceph/pull/17107>`_, songweibin)
+* doc: misc: fix various spelling errors (`pr#20831 <https://github.com/ceph/ceph/pull/20831>`_, Shengjing Zhu)
+* doc: Misc iSCSI doc updates (`pr#19931 <https://github.com/ceph/ceph/pull/19931>`_, Mike Christie)
+* doc: move glance_api_version option to the right place (`pr#17337 <https://github.com/ceph/ceph/pull/17337>`_, Luo Kexue)
+* doc: options.cc: document rgw config options (`pr#18007 <https://github.com/ceph/ceph/pull/18007>`_, Yehuda Sadeh)
+* doc: organizationmap: Add Adam Wolfe Gordon's affiliation (`pr#18295 <https://github.com/ceph/ceph/pull/18295>`_, Adam Wolfe Gordon)
+* doc: organizationmap: Add Ashish Singh affiliation (`pr#17109 <https://github.com/ceph/ceph/pull/17109>`_, Ashish Singh)
+* doc: .organizationmap: add Xin Yuan and Yichao Li's affiliation (`pr#21170 <https://github.com/ceph/ceph/pull/21170>`_, Li Wang)
+* doc: PendingReleaseNotes: Added note about Dashboard v2, fixed typo (`pr#21597 <https://github.com/ceph/ceph/pull/21597>`_, Lenz Grimmer)
+* doc: PendingReleaseNotes:Announce FreeBSD availability (`pr#16782 <https://github.com/ceph/ceph/pull/16782>`_, Willem Jan Withagen)
+* doc: PendingReleaseNotes: mention some monitor changes (`pr#21474 <https://github.com/ceph/ceph/pull/21474>`_, Joao Eduardo Luis)
+* doc: PendingReleaseNotes: note about upmap mapping change in luminous release notes (`pr#17813 <https://github.com/ceph/ceph/pull/17813>`_, Sage Weil)
+* doc: qa,doc: drop support of ubuntu trusty (`pr#19307 <https://github.com/ceph/ceph/pull/19307>`_, Kefu Chai)
+* doc/rados/operations/bluestore-migration: typos and whitespace (`pr#16991 <https://github.com/ceph/ceph/pull/16991>`_, Sage Weil)
+* doc/rados/operations/bluestore-migration: typos (`pr#17581 <https://github.com/ceph/ceph/pull/17581>`_, Sage Weil)
+* doc: README: Improve vstart.sh usage (`pr#17644 <https://github.com/ceph/ceph/pull/17644>`_, Fabian Vogt)
+* doc: README.md: bump up cmake to 2.8.12 (`pr#18348 <https://github.com/ceph/ceph/pull/18348>`_, Yan Jun)
+* doc: redundant "cephfs" when set the "allow_multimds" (`pr#20045 <https://github.com/ceph/ceph/pull/20045>`_, Shangzhong Zhu)
+* doc: release notes: fix grammar/style nits (`pr#18876 <https://github.com/ceph/ceph/pull/18876>`_, Nathan Cutler)
+* doc: release notes for 12.2.3 (`pr#20500 <https://github.com/ceph/ceph/pull/20500>`_, Abhishek Lekshmanan)
+* doc: release notes for v12.1.4 Luminous (`pr#17037 <https://github.com/ceph/ceph/pull/17037>`_, Abhishek Lekshmanan)
+* doc/release-notes: remove mention of crush weight optimization (`pr#16974 <https://github.com/ceph/ceph/pull/16974>`_, Sage Weil)
+* doc: release-notes.rst: add Kraken v11.2.1 and update releases.rst (`pr#16879 <https://github.com/ceph/ceph/pull/16879>`_, Nathan Cutler)
+* doc: release notes update for 10.2.10 (`pr#18148 <https://github.com/ceph/ceph/pull/18148>`_, Abhishek Lekshmanan)
+* doc/releases: drop LTS/stable line from second chart (`pr#18153 <https://github.com/ceph/ceph/pull/18153>`_, Sage Weil)
+* doc: Remove additional arguments when replacing OSD (`pr#18345 <https://github.com/ceph/ceph/pull/18345>`_, Wido den Hollander)
+* doc: remove duplicated --max-buckets option desc (`pr#19737 <https://github.com/ceph/ceph/pull/19737>`_, Kefu Chai)
+* doc: remove references to unversioned repository addresses (`pr#21357 <https://github.com/ceph/ceph/pull/21357>`_, Greg Farnum)
+* doc: remove unused config: "osd op threads" (`pr#21319 <https://github.com/ceph/ceph/pull/21319>`_, Jianpeng Ma)
+* doc: rename changelog with a .txt extension (`pr#18156 <https://github.com/ceph/ceph/pull/18156>`_, Abhishek Lekshmanan)
+* doc: reorganize releases (`pr#20784 <https://github.com/ceph/ceph/pull/20784>`_, Abhishek Lekshmanan)
+* doc: replace injectargs usage with "config set" (`pr#18789 <https://github.com/ceph/ceph/pull/18789>`_, John Spray)
+* doc: replace region with zonegroup in configure bucket sharding section (`issue#21610 <http://tracker.ceph.com/issues/21610>`_, `pr#18063 <https://github.com/ceph/ceph/pull/18063>`_, Orit Wasserman)
+* doc: restructure bluestore migration insructions (`pr#17603 <https://github.com/ceph/ceph/pull/17603>`_, Sage Weil)
+* doc: Revise the Example of Bucket Policy (`pr#17362 <https://github.com/ceph/ceph/pull/17362>`_, zhangwen)
+* doc: rgw: add a note for resharding in 12.2.1 docs (`pr#17675 <https://github.com/ceph/ceph/pull/17675>`_, Abhishek Lekshmanan)
+* doc: rgw add some basic documentation for sync plugins & ES (`pr#15849 <https://github.com/ceph/ceph/pull/15849>`_, Abhishek Lekshmanan)
+* doc: rgw adminops binding libraries (`pr#19164 <https://github.com/ceph/ceph/pull/19164>`_, hrchu)
+* doc: rgw mention about tagging & bucket policies in s3api (`pr#16907 <https://github.com/ceph/ceph/pull/16907>`_, Abhishek Lekshmanan)
+* doc: rgw: mention the civetweb support for binding to multiple ports (`issue#20942 <http://tracker.ceph.com/issues/20942>`_, `pr#17141 <https://github.com/ceph/ceph/pull/17141>`_, Abhishek Lekshmanan)
+* doc: rm stray ")" character from mds config ref (`pr#18228 <https://github.com/ceph/ceph/pull/18228>`_, Ken Dreyer)
+* docs: ceph-volume CLI updates (`pr#17425 <https://github.com/ceph/ceph/pull/17425>`_, Alfredo Deza)
+* doc: s/deamon/daemon/ (`pr#20931 <https://github.com/ceph/ceph/pull/20931>`_, ashitakasam)
+* doc: some improvements to ceph-conf.rst (`pr#21268 <https://github.com/ceph/ceph/pull/21268>`_, Nathan Cutler)
+* doc: Specify mount details in ceph-fuse (`pr#20071 <https://github.com/ceph/ceph/pull/20071>`_, Jos Collin)
+* doc: SubmittingPatches: clarify PR title section (`pr#17143 <https://github.com/ceph/ceph/pull/17143>`_, Nathan Cutler)
+* doc/templates update toctree call to include hidden entries (`pr#17076 <https://github.com/ceph/ceph/pull/17076>`_, Alfredo Deza)
+* doc: the client inputs the pool name instead of pool ID (`pr#17672 <https://github.com/ceph/ceph/pull/17672>`_, Frank Yu)
+* doc: typo fix (`pr#21077 <https://github.com/ceph/ceph/pull/21077>`_, Ashita Dashottar)
+* doc: update Blacklisting and OSD epoch barrier (`issue#22542 <http://tracker.ceph.com/issues/22542>`_, `pr#19701 <https://github.com/ceph/ceph/pull/19701>`_, Jos Collin)
+* doc: update ceph-disk with a state-transition diagram (`pr#17639 <https://github.com/ceph/ceph/pull/17639>`_, Kefu Chai)
+* doc: update ceph iscsi kernel and package info (`pr#20020 <https://github.com/ceph/ceph/pull/20020>`_, Mike Christie)
+* doc: Update commands and options in radosgw-admin (`pr#18267 <https://github.com/ceph/ceph/pull/18267>`_, Jos Collin)
+* doc: update Component Technical Leads and maintainers to canonical location (`pr#18376 <https://github.com/ceph/ceph/pull/18376>`_, Patrick McGarry)
+* doc: Update config file search paths to reflect reality (`pr#19882 <https://github.com/ceph/ceph/pull/19882>`_, Adam Wolfe Gordon)
+* doc: updated add primary storage documentation for latest CloudStack release (4.11) (`pr#21050 <https://github.com/ceph/ceph/pull/21050>`_, James McClune, John Wilkins)
+* doc: Update dashboard feature list (added RGW management) (`pr#21781 <https://github.com/ceph/ceph/pull/21781>`_, Lenz Grimmer)
+* doc: updated dashboard feature list (added new RGW details, Pools) (`pr#21562 <https://github.com/ceph/ceph/pull/21562>`_, Lenz Grimmer)
+* doc: Updated dashboard feature list (`pr#21693 <https://github.com/ceph/ceph/pull/21693>`_, Lenz Grimmer)
+* doc: Updated dashboard v2 feature list (`pr#20755 <https://github.com/ceph/ceph/pull/20755>`_, Lenz Grimmer)
+* doc: Updated documentation for Zabbix Mgr module (`pr#18356 <https://github.com/ceph/ceph/pull/18356>`_, Wido den Hollander)
+* doc: update default value of option mon_sync_timeout (`pr#17802 <https://github.com/ceph/ceph/pull/17802>`_, Yao Guotao)
+* doc: update default value of parameter mon_subscribe_interval (`pr#17669 <https://github.com/ceph/ceph/pull/17669>`_, yaoguotao)
+* doc: Update docs to remove gitbuilder and add shaman references (`pr#17022 <https://github.com/ceph/ceph/pull/17022>`_, Alfredo Deza)
+* doc: updated the dashboard feature list (`pr#21531 <https://github.com/ceph/ceph/pull/21531>`_, Lenz Grimmer)
+* doc: Updated the get-packages.rst to luminous (`pr#20815 <https://github.com/ceph/ceph/pull/20815>`_, Kai Wagner)
+* doc: update firewall doc to mention ceph-mgr (`pr#17974 <https://github.com/ceph/ceph/pull/17974>`_, John Spray)
+* doc: update iSCSI upstream kernel to 4.16 (`pr#20695 <https://github.com/ceph/ceph/pull/20695>`_, Mike Christie)
+* doc: update link to placing-different-pools (`pr#17833 <https://github.com/ceph/ceph/pull/17833>`_, Mohamad Gebai)
+* doc: update Li Wang Affiliation (`pr#18060 <https://github.com/ceph/ceph/pull/18060>`_, Li Wang)
+* doc: update man page to explain ceph-volume support bluestore (`issue#22663 <http://tracker.ceph.com/issues/22663>`_, `pr#19960 <https://github.com/ceph/ceph/pull/19960>`_, lijing)
+* doc: Update manual deployment (`issue#20309 <http://tracker.ceph.com/issues/20309>`_, `pr#15811 <https://github.com/ceph/ceph/pull/15811>`_, Jens Rosenboom)
+* doc: update mgr/dashboard doc about standbys (`pr#19879 <https://github.com/ceph/ceph/pull/19879>`_, John Spray)
+* doc: Update mgr doc on how to enable Zabbix module (`pr#16861 <https://github.com/ceph/ceph/pull/16861>`_, Wido den Hollander)
+* doc: update mgr related auth settings (`pr#20126 <https://github.com/ceph/ceph/pull/20126>`_, Kefu Chai)
+* doc: Update monitoring.rst (`pr#20630 <https://github.com/ceph/ceph/pull/20630>`_, Jos Collin)
+* doc: update rbd-mirroring documentation (`issue#20701 <http://tracker.ceph.com/issues/20701>`_, `pr#16908 <https://github.com/ceph/ceph/pull/16908>`_, Jason Dillaman)
+* doc: update references to use ceph-volume (`pr#19241 <https://github.com/ceph/ceph/pull/19241>`_, Alfredo Deza)
+* doc: update releases to the current state (`pr#17364 <https://github.com/ceph/ceph/pull/17364>`_, Abhishek Lekshmanan)
+* doc: Updates to bluestore migration doc (`pr#17602 <https://github.com/ceph/ceph/pull/17602>`_, David Galloway)
+* doc: v12.2.5 luminous release notes (`pr#21621 <https://github.com/ceph/ceph/pull/21621>`_, Abhishek Lekshmanan)
+* doc: various cleanups (`pr#18480 <https://github.com/ceph/ceph/pull/18480>`_, Kefu Chai)
+* examples: fix link order in librados example Makefile (`pr#17842 <https://github.com/ceph/ceph/pull/17842>`_, Mahati Chamarthy)
+* Fix ceph-mgr restarts (`pr#22051 <https://github.com/ceph/ceph/pull/22051>`_, Boris Ranto)
+* follow-up fixups for atomic_t spinlocks (`pr#17611 <https://github.com/ceph/ceph/pull/17611>`_, Jesse Williamson)
+* githubmap: Add ktdreyer (`pr#19209 <https://github.com/ceph/ceph/pull/19209>`_, Jos Collin)
+* include/buffer.h: fix typo in comment (`pr#17489 <https://github.com/ceph/ceph/pull/17489>`_, mychoxin)
+* include/ceph_features: fix OS_PERF_STAT_NS's incarnation (`pr#21467 <https://github.com/ceph/ceph/pull/21467>`_, Kefu Chai)
+* install-deps.sh: fix an error condition expression (`pr#20819 <https://github.com/ceph/ceph/pull/20819>`_, Yao Guotao)
+* java/native: fix milliseconds to mtime/atime conversion (`pr#17460 <https://github.com/ceph/ceph/pull/17460>`_, dengquan)
+* java/native: s/jni: lstat/jni: stat in native_ceph_stat (`pr#20142 <https://github.com/ceph/ceph/pull/20142>`_, Shangzhong Zhu)
+* KStore: statfs needs extra includes on FreeBSD (`pr#21429 <https://github.com/ceph/ceph/pull/21429>`_, Willem Jan Withagen)
+* kv/leveldb: fix deadlock when close db (`pr#16643 <https://github.com/ceph/ceph/pull/16643>`_, Zengran)
+* kv: unify {create_and\_,}open() methods (`pr#18177 <https://github.com/ceph/ceph/pull/18177>`_, Kefu Chai)
+* librados: add async interfaces for use with Networking TS (`pr#19054 <https://github.com/ceph/ceph/pull/19054>`_, Casey Bodley)
+* librados: block MgrClient::start_command until mgrmap (`pr#21832 <https://github.com/ceph/ceph/pull/21832>`_, John Spray, Kefu Chai)
+* librados: extend C API for so it accepts keys with NUL chars (`pr#20314 <https://github.com/ceph/ceph/pull/20314>`_, Piotr Dałek)
+* librados: Fix a potential risk of buffer::list::claim_prepend(list& b… (`issue#21338 <http://tracker.ceph.com/issues/21338>`_, `pr#17661 <https://github.com/ceph/ceph/pull/17661>`_, Guan yunfei)
+* librados: fix potential race condition if notify immediately fails (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#21859 <https://github.com/ceph/ceph/pull/21859>`_, Jason Dillaman)
+* librados: getter for min compatible client versions (`pr#20080 <https://github.com/ceph/ceph/pull/20080>`_, Jason Dillaman)
+* librados: invalid free() in rados_getxattrs_next() (`issue#22042 <http://tracker.ceph.com/issues/22042>`_, `pr#20260 <https://github.com/ceph/ceph/pull/20260>`_, Gu Zhongyan)
+* librados: make OPERATION_FULL_FORCE the default for rados_remove() (`issue#22413 <http://tracker.ceph.com/issues/22413>`_, `pr#20534 <https://github.com/ceph/ceph/pull/20534>`_, Kefu Chai)
+* librbd: abstract hard-coded journal and cache hooks on IO path (`pr#20682 <https://github.com/ceph/ceph/pull/20682>`_, Jason Dillaman)
+* librbd: Add a function to list image watchers (`pr#19188 <https://github.com/ceph/ceph/pull/19188>`_, Adam Wolfe Gordon)
+* librbd: add API function to get image name (`pr#20935 <https://github.com/ceph/ceph/pull/20935>`_, Mykola Golub)
+* librbd: added preprocessor macro for detecting compare-and-write support (`issue#22036 <http://tracker.ceph.com/issues/22036>`_, `pr#18708 <https://github.com/ceph/ceph/pull/18708>`_, Jason Dillaman)
+* librbd: add eventtrace support (`pr#19251 <https://github.com/ceph/ceph/pull/19251>`_, Mahati Chamarthy)
+* librbd: add preliminary support for new operation feature bit (`pr#19903 <https://github.com/ceph/ceph/pull/19903>`_, Jason Dillaman)
+* librbd: address coverity false positives (`pr#17696 <https://github.com/ceph/ceph/pull/17696>`_, Amit Kumar)
+* librbd: address coverity false positives (`pr#17721 <https://github.com/ceph/ceph/pull/17721>`_, Amit Kumar)
+* librbd: auto-remove trash snapshots when image is deleted (`issue#22873 <http://tracker.ceph.com/issues/22873>`_, `pr#20376 <https://github.com/ceph/ceph/pull/20376>`_, Jason Dillaman)
+* librbd: by default use new format for deep copy destination (`pr#20222 <https://github.com/ceph/ceph/pull/20222>`_, Mykola Golub)
+* librbd: cache last index position to accelerate snap create/rm (`issue#22716 <http://tracker.ceph.com/issues/22716>`_, `pr#19974 <https://github.com/ceph/ceph/pull/19974>`_, Song Shun)
+* librbd: cannot clone all image-metas if we have more than 64 key/value pairs (`pr#18327 <https://github.com/ceph/ceph/pull/18327>`_, PCzhangPC)
+* librbd: cannot copy all image-metas if we have more than 64 key/value pairs (`pr#18328 <https://github.com/ceph/ceph/pull/18328>`_, PCzhangPC)
+* librbd: clean up ManagedLock log prefix (`pr#20159 <https://github.com/ceph/ceph/pull/20159>`_, shun-s)
+* librbd: compare and write against a clone can result in failure (`issue#20789 <http://tracker.ceph.com/issues/20789>`_, `pr#18887 <https://github.com/ceph/ceph/pull/18887>`_, Jason Dillaman)
+* librbd: deep_copy: don't create snapshots above snap_id_end (`pr#19383 <https://github.com/ceph/ceph/pull/19383>`_, Mykola Golub)
+* librbd: default localize parent reads to false (`issue#20941 <http://tracker.ceph.com/issues/20941>`_, `pr#16882 <https://github.com/ceph/ceph/pull/16882>`_, Jason Dillaman)
+* librbd: default to sparse-reads for any IO operation over 64K (`issue#21849 <http://tracker.ceph.com/issues/21849>`_, `pr#18405 <https://github.com/ceph/ceph/pull/18405>`_, Jason Dillaman)
+* librbd: disable ENOENT tracking within the object cacher (`issue#23597 <http://tracker.ceph.com/issues/23597>`_, `pr#21308 <https://github.com/ceph/ceph/pull/21308>`_, Jason Dillaman)
+* librbd: disallow creation of v1 image format (`pr#20460 <https://github.com/ceph/ceph/pull/20460>`_, Julien COLLET, Julien Collet)
+* librbd: don't read metadata twice on image open (`pr#18542 <https://github.com/ceph/ceph/pull/18542>`_, Mykola Golub)
+* librbd: drop redundant check for null ImageCtx (`pr#18265 <https://github.com/ceph/ceph/pull/18265>`_, Jianpeng Ma)
+* librbd: filter out potential race with image rename (`issue#18435 <http://tracker.ceph.com/issues/18435>`_, `pr#19618 <https://github.com/ceph/ceph/pull/19618>`_, Jason Dillaman)
+* librbd: fix coverity warning for uninitialized member (`pr#18129 <https://github.com/ceph/ceph/pull/18129>`_, Li Wang)
+* librbd: fix deep copy a child-image (`pr#20099 <https://github.com/ceph/ceph/pull/20099>`_, songweibin)
+* librbd: fix don't send get_stripe_unit_count if striping is not enabled (`issue#21360 <http://tracker.ceph.com/issues/21360>`_, `pr#17660 <https://github.com/ceph/ceph/pull/17660>`_, Yanhu Cao)
+* librbd: fix issues discovered in clone v2 during upgrade tests (`issue#22979 <http://tracker.ceph.com/issues/22979>`_, `pr#20406 <https://github.com/ceph/ceph/pull/20406>`_, Jason Dillaman)
+* librbd: fix missing return in NotifyMessage::get_notify_op (`pr#20656 <https://github.com/ceph/ceph/pull/20656>`_, Yao Zongyou)
+* librbd: fix rbd close race with rewatch (`pr#21141 <https://github.com/ceph/ceph/pull/21141>`_, Song Shun)
+* librbd: fix refuse to release lock when cookie is the same at rewatch (`pr#20868 <https://github.com/ceph/ceph/pull/20868>`_, Song Shun)
+* librbd: fix structure size check in rbd_mirror_image_get_info/status (`pr#20478 <https://github.com/ceph/ceph/pull/20478>`_, Mykola Golub)
+* librbd: force removal of a snapshot cannot ignore dependent children (`issue#22791 <http://tracker.ceph.com/issues/22791>`_, `pr#20105 <https://github.com/ceph/ceph/pull/20105>`_, Jason Dillaman)
+* librbd: generalized deep copy function (`pr#16238 <https://github.com/ceph/ceph/pull/16238>`_, Mykola Golub)
+* librbd: group and snapshot cleanup (`pr#19990 <https://github.com/ceph/ceph/pull/19990>`_, Jason Dillaman)
+* librbd: group snapshots (`pr#11544 <https://github.com/ceph/ceph/pull/11544>`_, Victor Denisov, Jason Dillaman)
+* librbd: hold cache_lock while clearing cache nonexistence flags (`issue#21558 <http://tracker.ceph.com/issues/21558>`_, `pr#17992 <https://github.com/ceph/ceph/pull/17992>`_, Jason Dillaman)
+* librbd: image-meta config overrides should be dynamically refreshed (`issue#21529 <http://tracker.ceph.com/issues/21529>`_, `pr#18042 <https://github.com/ceph/ceph/pull/18042>`_, Dongsheng Yang, Jason Dillaman)
+* librbd: initial hooks for clone v2 support (`pr#20176 <https://github.com/ceph/ceph/pull/20176>`_, Jason Dillaman)
+* librbd: initialization of state member variables (`pr#16866 <https://github.com/ceph/ceph/pull/16866>`_, amitkuma)
+* librbd: Initializing members image,operation,journal (`pr#16934 <https://github.com/ceph/ceph/pull/16934>`_, amitkuma)
+* librbd: Initializing member variables (`pr#16867 <https://github.com/ceph/ceph/pull/16867>`_, amitkuma)
+* librbd: journal should ignore -EILSEQ errors from compare-and-write (`issue#21628 <http://tracker.ceph.com/issues/21628>`_, `pr#18099 <https://github.com/ceph/ceph/pull/18099>`_, Jason Dillaman)
+* librbd,librados: do not include stdbool.h in C++ headers (`pr#19945 <https://github.com/ceph/ceph/pull/19945>`_, Kefu Chai)
+* librbd: list_children should not attempt to refresh image (`issue#21670 <http://tracker.ceph.com/issues/21670>`_, `pr#18114 <https://github.com/ceph/ceph/pull/18114>`_, Jason Dillaman)
+* librbd: minor cleanup of the IO pathway (`pr#20560 <https://github.com/ceph/ceph/pull/20560>`_, Jason Dillaman)
+* librbd: minor code cleanup (`pr#21165 <https://github.com/ceph/ceph/pull/21165>`_, songweibin)
+* librbd: missing 'return' in deep_copy::ObjectCopyRequest::send_read_object (`pr#21493 <https://github.com/ceph/ceph/pull/21493>`_, Mykola Golub)
+* librbd: new tag should use on-disk committed position (`issue#22945 <http://tracker.ceph.com/issues/22945>`_, `pr#20423 <https://github.com/ceph/ceph/pull/20423>`_, Jason Dillaman)
+* librbd: object map batch update might cause OSD suicide timeout (`issue#21797 <http://tracker.ceph.com/issues/21797>`_, `pr#18315 <https://github.com/ceph/ceph/pull/18315>`_, Jason Dillaman)
+* librbd: possible deadlock with synchronous maintenance operations (`issue#22120 <http://tracker.ceph.com/issues/22120>`_, `pr#18909 <https://github.com/ceph/ceph/pull/18909>`_, Jason Dillaman)
+* librbd: potential crash if object map check encounters error (`issue#22819 <http://tracker.ceph.com/issues/22819>`_, `pr#20214 <https://github.com/ceph/ceph/pull/20214>`_, Jason Dillaman)
+* librbd: potential race between discard and writeback (`pr#21248 <https://github.com/ceph/ceph/pull/21248>`_, Jason Dillaman)
+* librbd: potential race in RewatchRequest when resetting watch_handle (`pr#20420 <https://github.com/ceph/ceph/pull/20420>`_, Mykola Golub)
+* librbd: prefer templates to macros (`pr#19912 <https://github.com/ceph/ceph/pull/19912>`_, Adam C. Emerson)
+* librbd: prevent overflow of discard API result code (`issue#21966 <http://tracker.ceph.com/issues/21966>`_, `pr#18923 <https://github.com/ceph/ceph/pull/18923>`_, Jason Dillaman)
+* librbd: prevent watcher from unregistering with in-flight actions (`issue#23955 <http://tracker.ceph.com/issues/23955>`_, `pr#21763 <https://github.com/ceph/ceph/pull/21763>`_, Jason Dillaman)
+* librbd: refresh image after applying new metadata (`issue#21711 <http://tracker.ceph.com/issues/21711>`_, `pr#18158 <https://github.com/ceph/ceph/pull/18158>`_, Jason Dillaman)
+* librbd: release lock executing deep copy progress callback (`issue#23929 <http://tracker.ceph.com/issues/23929>`_, `pr#21727 <https://github.com/ceph/ceph/pull/21727>`_, Mykola Golub)
+* librbd: remove unused member in FlattenRequest (`pr#19416 <https://github.com/ceph/ceph/pull/19416>`_, Mykola Golub)
+* librbd: remove unused variables from ReadResult refactor (`pr#18277 <https://github.com/ceph/ceph/pull/18277>`_, Jason Dillaman)
+* librbd: rename of non-existent image results in seg fault (`issue#21248 <http://tracker.ceph.com/issues/21248>`_, `pr#17502 <https://github.com/ceph/ceph/pull/17502>`_, Jason Dillaman)
+* librbd: set deleted parent pointer to null (`issue#22158 <http://tracker.ceph.com/issues/22158>`_, `pr#19003 <https://github.com/ceph/ceph/pull/19003>`_, Jason Dillaman)
+* librbd: should not set self as remote peer (`pr#17300 <https://github.com/ceph/ceph/pull/17300>`_, songweibin)
+* librbd: small cleanup for recently merged code (`pr#20578 <https://github.com/ceph/ceph/pull/20578>`_, Mykola Golub)
+* librbd: snapshots should be created/removed against data pool (`issue#21567 <http://tracker.ceph.com/issues/21567>`_, `pr#18043 <https://github.com/ceph/ceph/pull/18043>`_, Jason Dillaman)
+* librbd: speed up object map disk usage and resize (`pr#20218 <https://github.com/ceph/ceph/pull/20218>`_, shun-s)
+* librbd: speed up sparse copy when object map is available (`pr#18967 <https://github.com/ceph/ceph/pull/18967>`_, Song Shun)
+* librbd: update mirror::EnableRequest diagram according to code (`pr#19130 <https://github.com/ceph/ceph/pull/19130>`_, Mykola Golub)
+* librbd: use steady clock to measure elapsed time in AioCompletion (`pr#20007 <https://github.com/ceph/ceph/pull/20007>`_, Mohamad Gebai)
+* librbd: validate if dst group snap name is the same with src (`pr#20395 <https://github.com/ceph/ceph/pull/20395>`_, songweibin)
+* log: Fix AddressSanitizer: new-delete-type-mismatch (`issue#23324 <http://tracker.ceph.com/issues/23324>`_, `pr#20930 <https://github.com/ceph/ceph/pull/20930>`_, Brad Hubbard)
+* log: fix build on osx (`pr#18213 <https://github.com/ceph/ceph/pull/18213>`_, Kefu Chai)
+* log: silence warning from -Wsign-compare (`pr#18326 <https://github.com/ceph/ceph/pull/18326>`_, Jos Collin)
+* log: Use the coarse real time clock in log timestamps (`pr#18141 <https://github.com/ceph/ceph/pull/18141>`_, Adam C. Emerson)
+* mds: check metadata pool not cluster is full (`issue#22483 <http://tracker.ceph.com/issues/22483>`_, `pr#19602 <https://github.com/ceph/ceph/pull/19602>`_, Patrick Donnelly)
+* mds: fix CEPH_STAT_RSTAT definition (`pr#21633 <https://github.com/ceph/ceph/pull/21633>`_, "Yan, Zheng")
+* mds: get rid of the "if" check which is unnecessary inside a loop (`pr#18904 <https://github.com/ceph/ceph/pull/18904>`_, dongdong tao)
+* mds: Remove redundant null pointer check (`pr#19750 <https://github.com/ceph/ceph/pull/19750>`_, Brad Hubbard)
+* mds: simplify the code logic in replay_alloc_ids (`pr#18893 <https://github.com/ceph/ceph/pull/18893>`_, dongdong tao)
+* mempool: fix lack of pool names in mempool:dump output for JSON format (`pr#18329 <https://github.com/ceph/ceph/pull/18329>`_, Igor Fedotov)
+* messages: Initialization of uninitialized members various classes (`pr#16848 <https://github.com/ceph/ceph/pull/16848>`_, amitkuma)
+* messages/MDentryLink: add const to member function (`pr#15479 <https://github.com/ceph/ceph/pull/15479>`_, yonghengdexin735)
+* messages,test,msg: initialize h,reply_type,owner (`pr#17767 <https://github.com/ceph/ceph/pull/17767>`_, Amit Kumar)
+* mgr: add mgr daemon to DaemonStateIndex with metadata (hostname) (`issue#23286 <http://tracker.ceph.com/issues/23286>`_, `pr#20875 <https://github.com/ceph/ceph/pull/20875>`_, Jan Fajerski)
+* mgr: add missing call to pick_addresses (`issue#20955 <http://tracker.ceph.com/issues/20955>`_, `pr#16940 <https://github.com/ceph/ceph/pull/16940>`_, John Spray)
+* mgr: add the ip addr of standbys (`pr#16476 <https://github.com/ceph/ceph/pull/16476>`_, huanwen ren)
+* mgr: add units to performance counters (`issue#22747 <http://tracker.ceph.com/issues/22747>`_, `pr#20152 <https://github.com/ceph/ceph/pull/20152>`_, Rubab Syed)
+* mgr: allow service daemons to unregister from ServiceMap (`pr#20761 <https://github.com/ceph/ceph/pull/20761>`_, Sage Weil)
+* mgr: apply a threshold to perf counter prios (`pr#16699 <https://github.com/ceph/ceph/pull/16699>`_, John Spray)
+* mgr: balancer: fixed mistype "AttributeError: 'Logger' object has no attribute 'err'" (`pr#20130 <https://github.com/ceph/ceph/pull/20130>`_, Konstantin Shalygin)
+* mgr: centralized setting/getting of mgr configs (`pr#21442 <https://github.com/ceph/ceph/pull/21442>`_, John Spray, Rubab Syed)
+* mgr: ceph-mgr: can not change prometheus port for mgr (`pr#17746 <https://github.com/ceph/ceph/pull/17746>`_, wujian)
+* mgr: common interface for TSDB modules (`pr#17735 <https://github.com/ceph/ceph/pull/17735>`_, Jan Fajerski, John Spray, My Do)
+* mgr/dashboard: Adapt help text if server_addr is not set (`pr#21640 <https://github.com/ceph/ceph/pull/21640>`_, Volker Theile)
+* mgr/dashboard: Adapt RBD form to new application_metadata type (`pr#21602 <https://github.com/ceph/ceph/pull/21602>`_, Volker Theile)
+* mgr/dashboard: Add Api module (`pr#21126 <https://github.com/ceph/ceph/pull/21126>`_, Tiago Melo)
+* mgr/dashboard: Add 'autofocus' directive (`pr#21559 <https://github.com/ceph/ceph/pull/21559>`_, Volker Theile)
+* mgr/dashboard: Add CdDatePipe (`pr#21087 <https://github.com/ceph/ceph/pull/21087>`_, Ricardo Marques)
+* mgr/dashboard: Add 'cd-error-panel' component to display error messages (`pr#21558 <https://github.com/ceph/ceph/pull/21558>`_, Volker Theile)
+* mgr/dashboard: Add 'cd-loading-panel' component (`pr#21618 <https://github.com/ceph/ceph/pull/21618>`_, Volker Theile)
+* mgr/dashboard: Add custom validators (`pr#21041 <https://github.com/ceph/ceph/pull/21041>`_, Volker Theile)
+* mgr/dashboard: Add DimlessBinaryDirective (`pr#20972 <https://github.com/ceph/ceph/pull/20972>`_, Ricardo Marques)
+* mgr/dashboard: Add ErasureCodeProfile controller (`issue#23345 <http://tracker.ceph.com/issues/23345>`_, `pr#20920 <https://github.com/ceph/ceph/pull/20920>`_, Sebastian Wagner, Stephan Müller)
+* mgr/dashboard: Add 'forceIdentifier' attribute to datatable (`pr#21497 <https://github.com/ceph/ceph/pull/21497>`_, Volker Theile)
+* mgr/dashboard: Add helper component (`pr#20971 <https://github.com/ceph/ceph/pull/20971>`_, Ricardo Marques)
+* mgr/dashboard: additional fixes to block pages (`pr#20941 <https://github.com/ceph/ceph/pull/20941>`_, Jason Dillaman)
+* mgr/dashboard: Add minimalistic browsable API (`pr#20873 <https://github.com/ceph/ceph/pull/20873>`_, Sebastian Wagner)
+* mgr/dashboard: Add notification service/component (`pr#21078 <https://github.com/ceph/ceph/pull/21078>`_, Tiago Melo)
+* mgr/dashboard: Add Pool-create to the backend (`issue#23345 <http://tracker.ceph.com/issues/23345>`_, `pr#20865 <https://github.com/ceph/ceph/pull/20865>`_, Sebastian Wagner)
+* mgr/dashboard: Add RGW user and bucket management features (`pr#21351 <https://github.com/ceph/ceph/pull/21351>`_, Volker Theile)
+* mgr/dashboard: Adds reusable deletion dialog (`pr#20899 <https://github.com/ceph/ceph/pull/20899>`_, Stephan Müller, Tiago Melo)
+* mgr/dashboard: Add submit button component (`pr#21011 <https://github.com/ceph/ceph/pull/21011>`_, Tiago Melo)
+* mgr/dashboard: Add usage bar component (`pr#21128 <https://github.com/ceph/ceph/pull/21128>`_, Ricardo Marques)
+* mgr/dashboard: Angular modules cleanup (`pr#21402 <https://github.com/ceph/ceph/pull/21402>`_, Tiago Melo)
+* mgr/dashboard: Asynchronous tasks (frontend) (`pr#20962 <https://github.com/ceph/ceph/pull/20962>`_, Ricardo Marques)
+* mgr/dashboard: awsauth: fix python3 string decode problem (`pr#21875 <https://github.com/ceph/ceph/pull/21875>`_, Ricardo Dias)
+* mgr/dashboard: Change font-family of checkbox (`pr#21787 <https://github.com/ceph/ceph/pull/21787>`_, Tiago Melo)
+* mgr/dashboard: Clean up Pylint warnings (`pr#21694 <https://github.com/ceph/ceph/pull/21694>`_, Sebastian Wagner)
+* mgr/dashboard: Convert floating values to bytes (`pr#21677 <https://github.com/ceph/ceph/pull/21677>`_, Stephan Müller)
+* mgr/dashboard: Convert the RBD feature names to a list of strings (`pr#21024 <https://github.com/ceph/ceph/pull/21024>`_, Tatjana Dehler)
+* mgr/dashboard: Deletion dialog falsely executes deletion when pressing 'Cancel' (`pr#22032 <https://github.com/ceph/ceph/pull/22032>`_, Volker Theile)
+* mgr/dashboard: Display notification if RGW is not configured (`pr#21977 <https://github.com/ceph/ceph/pull/21977>`_, Volker Theile)
+* mgr/dashboard: Display RBD form errors on submission (`pr#21529 <https://github.com/ceph/ceph/pull/21529>`_, Ricardo Marques)
+* mgr/dashboard: Enable object rendering in KV-table (`pr#21701 <https://github.com/ceph/ceph/pull/21701>`_, Stephan Müller)
+* mgr/dashboard: fix 500 error on block device iSCSI status page (`pr#20928 <https://github.com/ceph/ceph/pull/20928>`_, Jason Dillaman)
+* mgr/dashboard: fix dashboard python 3 support (`pr#21007 <https://github.com/ceph/ceph/pull/21007>`_, Ricardo Dias)
+* mgr/dashboard: Fix data race and use-before-assignment (`pr#21590 <https://github.com/ceph/ceph/pull/21590>`_, Sebastian Wagner)
+* mgr/dashboard: fixed password generation in Auth controller (`issue#23404 <http://tracker.ceph.com/issues/23404>`_, `pr#21006 <https://github.com/ceph/ceph/pull/21006>`_, Ricardo Dias)
+* mgr/dashboard: Fixes documentation link- to open in new tab (`pr#22262 <https://github.com/ceph/ceph/pull/22262>`_, Kanika Murarka)
+* mgr/dashboard: Fixes type error in RBD form (`pr#21681 <https://github.com/ceph/ceph/pull/21681>`_, Stephan Müller)
+* mgr/dashboard: fix frontend e2e tests (`pr#20943 <https://github.com/ceph/ceph/pull/20943>`_, Tiago Melo)
+* mgr/dashboard: fix FS status on old MDS daemons (`issue#20692 <http://tracker.ceph.com/issues/20692>`_, `pr#16960 <https://github.com/ceph/ceph/pull/16960>`_, John Spray)
+* mgr/dashboard: fix linting problem (`pr#22277 <https://github.com/ceph/ceph/pull/22277>`_, Tiago Melo)
+* mgr/dashboard: Fix missing $event on deletion modal (`pr#21667 <https://github.com/ceph/ceph/pull/21667>`_, Ricardo Marques)
+* mgr/dashboard: Fix moment.js deprecation warning (`pr#22052 <https://github.com/ceph/ceph/pull/22052>`_, Tiago Melo)
+* mgr/dashboard: Fix objects named `default` are inaccessible (`pr#20976 <https://github.com/ceph/ceph/pull/20976>`_, Sebastian Wagner)
+* mgr/dashboard: Fix RBD task metadata (`pr#22152 <https://github.com/ceph/ceph/pull/22152>`_, Tiago Melo)
+* mgr/dashboard: Fix table without fetchData (`pr#21086 <https://github.com/ceph/ceph/pull/21086>`_, Ricardo Marques)
+* mgr/dashboard: Fix the data table action selector (`pr#21270 <https://github.com/ceph/ceph/pull/21270>`_, Stephan Müller)
+* mgr/dashboard: fix two type errors found by mypy (`pr#21774 <https://github.com/ceph/ceph/pull/21774>`_, Sebastian Wagner)
+* mgr/dashboard: Handle errors during deletion (`pr#22029 <https://github.com/ceph/ceph/pull/22029>`_, Volker Theile)
+* mgr/dashboard: Implement a RGW proxy (`pr#21258 <https://github.com/ceph/ceph/pull/21258>`_, Volker Theile, Patrick Nawracay)
+* mgr/dashboard: Improve background tasks style (`pr#21462 <https://github.com/ceph/ceph/pull/21462>`_, Ricardo Marques)
+* mgr/dashboard: improve error handling (`pr#18182 <https://github.com/ceph/ceph/pull/18182>`_, Nick Erdmann)
+* mgr/dashboard: Improve error panel (`pr#21978 <https://github.com/ceph/ceph/pull/21978>`_, Volker Theile)
+* mgr/dashboard: Improve `npm start` script (`pr#20989 <https://github.com/ceph/ceph/pull/20989>`_, Ricardo Marques)
+* mgr/dashboard: Improve table search (`pr#20807 <https://github.com/ceph/ceph/pull/20807>`_, Stephan Müller)
+* mgr/dashboard: Load the datatable content on component initialization (`pr#21595 <https://github.com/ceph/ceph/pull/21595>`_, Volker Theile)
+* mgr/dashboard: Navbar dropdown button does not respond for mobile browsers (`pr#21979 <https://github.com/ceph/ceph/pull/21979>`_, Volker Theile)
+* mgr/dashboard: Notification improvements (`pr#21350 <https://github.com/ceph/ceph/pull/21350>`_, Tiago Melo)
+* mgr/dashboard: pool: fix python3 dict_keys error (`pr#21636 <https://github.com/ceph/ceph/pull/21636>`_, Ricardo Dias)
+* mgr/dashboard: Pool listing (`pr#21353 <https://github.com/ceph/ceph/pull/21353>`_, Stephan Müller)
+* mgr/dashboard: rbd: add @AuthRequired to snapshots controller (`pr#21517 <https://github.com/ceph/ceph/pull/21517>`_, Ricardo Dias)
+* mgr/dashboard: RBD copy, RBD flatten and snapshot clone (frontend) (`pr#21526 <https://github.com/ceph/ceph/pull/21526>`_, Ricardo Marques, Ricardo Dias)
+* mgr/dashboard: RBD management (frontend) (`pr#21385 <https://github.com/ceph/ceph/pull/21385>`_, Ricardo Marques)
+* mgr/dashboard: Refactor multiple duplicates of `get_rate()` (`pr#21022 <https://github.com/ceph/ceph/pull/21022>`_, Sebastian Wagner)
+* mgr/dashboard: Refactor RGW backend (`pr#21855 <https://github.com/ceph/ceph/pull/21855>`_, Volker Theile)
+* mgr/dashboard: Rename and refactor ApiInterceptorService class (`pr#21386 <https://github.com/ceph/ceph/pull/21386>`_, Volker Theile)
+* mgr/dashboard: Replace font-awesome with fork-awesome (`pr#21327 <https://github.com/ceph/ceph/pull/21327>`_, Lenz Grimmer)
+* mgr/dashboard: restcontroller: fix detection of id args in element requests (`pr#21290 <https://github.com/ceph/ceph/pull/21290>`_, Ricardo Dias)
+* mgr/dashboard: RESTController improvements (`pr#21516 <https://github.com/ceph/ceph/pull/21516>`_, Ricardo Dias)
+* mgr/dashboard: run-tox: pass CEPH_BUILD_DIR value into tox script (`pr#21445 <https://github.com/ceph/ceph/pull/21445>`_, Ricardo Dias)
+* mgr: dashboard: show per pool IOPS on health page (#22495) (`issue#22495 <http://tracker.ceph.com/issues/22495>`_, `pr#19981 <https://github.com/ceph/ceph/pull/19981>`_, Konstantin Shalygin)
+* mgr/dashboard: Support aditional info on 'cd-view-cache' (`pr#21060 <https://github.com/ceph/ceph/pull/21060>`_, Ricardo Marques)
+* mgr/dashboard: TaskManager bug fixes (`pr#21240 <https://github.com/ceph/ceph/pull/21240>`_, Ricardo Dias)
+* mgr/dashboard: Update selected items on table refresh (`pr#21099 <https://github.com/ceph/ceph/pull/21099>`_, Ricardo Marques)
+* mgr/dashboard: Use Bootstrap CSS (`pr#21780 <https://github.com/ceph/ceph/pull/21780>`_, Volker Theile)
+* mgr/dashboard: using RoutesDispatcher as HTTP request dispatcher (`pr#21239 <https://github.com/ceph/ceph/pull/21239>`_, Ricardo Dias)
+* mgr/dashboard_v2: add mgr to the list of perf counters (`pr#20783 <https://github.com/ceph/ceph/pull/20783>`_, Tiago Melo)
+* mgr/dashboard_v2: add mocked service provider for TcmuIscsiService (`pr#20775 <https://github.com/ceph/ceph/pull/20775>`_, Tiago Melo)
+* mgr/dashboard_v2: Add toggle able columns (`pr#20806 <https://github.com/ceph/ceph/pull/20806>`_, Stephan Müller)
+* mgr/dashboard_v2: Configuration settings support (`pr#20743 <https://github.com/ceph/ceph/pull/20743>`_, Ricardo Dias)
+* mgr/dashboard_v2: fix and improve table details (`pr#20811 <https://github.com/ceph/ceph/pull/20811>`_, Tiago Melo)
+* mgr/dashboard_v2: Fix cephfs template table usage (`pr#20804 <https://github.com/ceph/ceph/pull/20804>`_, Stephan Müller)
+* mgr/dashboard_v2: fix cluster configuration page (`pr#20821 <https://github.com/ceph/ceph/pull/20821>`_, Tiago Melo)
+* mgr/dashboard_v2: Improve charts tooltips (`pr#20757 <https://github.com/ceph/ceph/pull/20757>`_, Tiago Melo)
+* mgr/dashboard_v2: Pool controller (`pr#20823 <https://github.com/ceph/ceph/pull/20823>`_, Ricardo Dias)
+* mgr/dashboard_v2: Rotate the refresh icon on load (`pr#20805 <https://github.com/ceph/ceph/pull/20805>`_, Stephan Müller)
+* mgr: die on bind() failure (`pr#20595 <https://github.com/ceph/ceph/pull/20595>`_, John Spray)
+* mgr: disconnect unregistered service daemon when report received (`issue#22286 <http://tracker.ceph.com/issues/22286>`_, `pr#19261 <https://github.com/ceph/ceph/pull/19261>`_, Jason Dillaman)
+* mgr: emit cluster log message on serve() exception (`issue#21999 <http://tracker.ceph.com/issues/21999>`_, `pr#18672 <https://github.com/ceph/ceph/pull/18672>`_, John Spray)
+* mgr: Expose rgw perf counters (`pr#21269 <https://github.com/ceph/ceph/pull/21269>`_, Boris Ranto)
+* mgr: fix "access denied" message (`pr#19518 <https://github.com/ceph/ceph/pull/19518>`_, John Spray)
+* mgr: fix crashable DaemonStateIndex::get calls (`issue#17737 <http://tracker.ceph.com/issues/17737>`_, `pr#17933 <https://github.com/ceph/ceph/pull/17933>`_, John Spray)
+* mgr: fix crash in MonCommandCompletion (`issue#21157 <http://tracker.ceph.com/issues/21157>`_, `pr#17308 <https://github.com/ceph/ceph/pull/17308>`_, John Spray)
+* mgr: fixes python error handling (`issue#23406 <http://tracker.ceph.com/issues/23406>`_, `pr#21005 <https://github.com/ceph/ceph/pull/21005>`_, Ricardo Dias)
+* mgr: fix MSG_MGR_MAP handling (`pr#20892 <https://github.com/ceph/ceph/pull/20892>`_, Gu Zhongyan)
+* mgr: fix "osd status" command exception if OSD not in pgmap stats (`issue#21707 <http://tracker.ceph.com/issues/21707>`_, `pr#18173 <https://github.com/ceph/ceph/pull/18173>`_, Yanhu Cao)
+* mgr: fix py3 support (`issue#22880 <http://tracker.ceph.com/issues/22880>`_, `pr#20362 <https://github.com/ceph/ceph/pull/20362>`_, Kefu Chai)
+* mgr: fix py calls for dne service perf counters (`issue#21253 <http://tracker.ceph.com/issues/21253>`_, `pr#17605 <https://github.com/ceph/ceph/pull/17605>`_, John Spray)
+* mgr: implement completion of osd MetadataUpdate (`issue#21159 <http://tracker.ceph.com/issues/21159>`_, `pr#16925 <https://github.com/ceph/ceph/pull/16925>`_, Yanhu Cao)
+* mgr: implement 'osd safe-to-destroy' and 'osd ok-to-stop' commands (`pr#16976 <https://github.com/ceph/ceph/pull/16976>`_, Sage Weil)
+* mgr: improved module loading for error reporting etc (`issue#21999 <http://tracker.ceph.com/issues/21999>`_, `issue#21683 <http://tracker.ceph.com/issues/21683>`_, `issue#21502 <http://tracker.ceph.com/issues/21502>`_, `pr#19235 <https://github.com/ceph/ceph/pull/19235>`_, John Spray)
+* mgr: improve reporting on unloadable modules (`issue#23358 <http://tracker.ceph.com/issues/23358>`_, `pr#20921 <https://github.com/ceph/ceph/pull/20921>`_, John Spray)
+* mgr: increase time resolution of Commit/Apply OSD latencies (`pr#19232 <https://github.com/ceph/ceph/pull/19232>`_, Коренберг Марк)
+* mgr: initialize PyModuleRegistry sooner (`issue#22918 <http://tracker.ceph.com/issues/22918>`_, `pr#20321 <https://github.com/ceph/ceph/pull/20321>`_, John Spray)
+* mgr: In plugins 'module' classes need not to be called "Module" anymore (`issue#17454 <http://tracker.ceph.com/issues/17454>`_, `pr#18526 <https://github.com/ceph/ceph/pull/18526>`_, Kefu Chai, bhavishyagopesh)
+* mgr: locking fixes (`issue#21158 <http://tracker.ceph.com/issues/21158>`_, `pr#17309 <https://github.com/ceph/ceph/pull/17309>`_, John Spray)
+* mgr: mgr/balancer: cast config vals to int or float (`issue#22429 <http://tracker.ceph.com/issues/22429>`_, `pr#19493 <https://github.com/ceph/ceph/pull/19493>`_, Dan van der Ster)
+* mgr: mgr/balancer: don't use 'foo' tags on commands (`issue#22361 <http://tracker.ceph.com/issues/22361>`_, `pr#19482 <https://github.com/ceph/ceph/pull/19482>`_, John Spray)
+* mgr: mgr/balancer: fix KeyError in balancer rm (`issue#22470 <http://tracker.ceph.com/issues/22470>`_, `pr#19578 <https://github.com/ceph/ceph/pull/19578>`_, Dan van der Ster)
+* mgr: mgr/balancer: fix OPTIONS definition (`pr#21620 <https://github.com/ceph/ceph/pull/21620>`_, John Spray)
+* mgr: mgr/balancer: fix upmap; default balancer module enabled (`pr#18691 <https://github.com/ceph/ceph/pull/18691>`_, Sage Weil)
+* mgr: mgr/balancer: make crush-compat mode work (`pr#17983 <https://github.com/ceph/ceph/pull/17983>`_, Sage Weil)
+* mgr: mgr/balancer: mgr module to automatically balance PGs across OSDs (`pr#16272 <https://github.com/ceph/ceph/pull/16272>`_, Spandan Kumar Sahu, Sage Weil)
+* mgr: mgr/balancer: more pool-specific enhancements (`pr#20225 <https://github.com/ceph/ceph/pull/20225>`_, xie xingguo)
+* mgr: mgr/balancer: pool-specific optimization support and bug fixes (`pr#20154 <https://github.com/ceph/ceph/pull/20154>`_, xie xingguo)
+* mgr:  mgr/balancer: replace magic value of -1 for DEFAULT_CHOOSE_ARGS (`pr#20258 <https://github.com/ceph/ceph/pull/20258>`_, Kefu Chai)
+* mgr: mgr/balancer: skip CRUSH_ITEM_NONE (`pr#18894 <https://github.com/ceph/ceph/pull/18894>`_, Sage Weil)
+* mgr: mgr/balancer: two more fixes (`pr#20180 <https://github.com/ceph/ceph/pull/20180>`_, xie xingguo)
+* mgr: mgrc: free MMgrClose in handle_mgr_close (`issue#23846 <http://tracker.ceph.com/issues/23846>`_, `pr#21626 <https://github.com/ceph/ceph/pull/21626>`_, Casey Bodley)
+* mgr: mgr/DaemonServer: add overrides value to 'config show' (`pr#21093 <https://github.com/ceph/ceph/pull/21093>`_, Gu Zhongyan)
+* mgr: mgr/DaemonServer.cc: [Cleanup] Change to using  get_val template function (`pr#18717 <https://github.com/ceph/ceph/pull/18717>`_, Shinobu Kinjo)
+* mgr: mgr/DaemonServer: [Cleanup] Remove redundant code (`pr#18716 <https://github.com/ceph/ceph/pull/18716>`_, Shinobu Kinjo)
+* mgr: mgr/dashboard: add configuration setting browser (`issue#22522 <http://tracker.ceph.com/issues/22522>`_, `pr#20043 <https://github.com/ceph/ceph/pull/20043>`_, Rubab Syed)
+* mgr: mgr/dashboard: add image id to mgr rbd info instead of block_name_prefix (`pr#20884 <https://github.com/ceph/ceph/pull/20884>`_, zouaiguo)
+* mgr: mgr/dashboard: Add monitor list (`pr#19632 <https://github.com/ceph/ceph/pull/19632>`_, Rubab Syed)
+* mgr: mgr/dashboard: Add RGW user and bucket lists (read-only) (`pr#20869 <https://github.com/ceph/ceph/pull/20869>`_, Volker Theile)
+* mgr: mgr/dashboard: add TLS (`pr#21627 <https://github.com/ceph/ceph/pull/21627>`_, John Spray)
+* mgr: mgr/dashboard: Add toBytes() method to FormatterService (`pr#20978 <https://github.com/ceph/ceph/pull/20978>`_, Volker Theile)
+* mgr: mgr/dashboard: asynchronous task support (`pr#20870 <https://github.com/ceph/ceph/pull/20870>`_, Ricardo Dias)
+* mgr: mgr/dashboard: change raw usage chart's color depending on usage (`pr#17421 <https://github.com/ceph/ceph/pull/17421>`_, Nick Erdmann)
+* mgr: mgr/dashboard: fix audit log loading (`pr#18848 <https://github.com/ceph/ceph/pull/18848>`_, John Spray)
+* mgr: mgr/dashboard: Fix backend tests for newer CherryPy versions (`pr#20778 <https://github.com/ceph/ceph/pull/20778>`_, Patrick Nawracay)
+* mgr: mgr/dashboard: Fix PG status coloring (`pr#19431 <https://github.com/ceph/ceph/pull/19431>`_, Wido den Hollander)
+* mgr: mgr/dashboard: format tooltip's label as user friendly string (`pr#18769 <https://github.com/ceph/ceph/pull/18769>`_, Yao Zongyou)
+* mgr: mgr/dashboard: handle null in format_number (`issue#21570 <http://tracker.ceph.com/issues/21570>`_, `pr#17991 <https://github.com/ceph/ceph/pull/17991>`_, John Spray)
+* mgr: mgr/dashboard: HTTP request logging (`pr#20797 <https://github.com/ceph/ceph/pull/20797>`_, Ricardo Dias)
+* mgr: mgr/dashboard: Improve auth interceptor (`pr#20847 <https://github.com/ceph/ceph/pull/20847>`_, Volker Theile)
+* mgr: mgr/dashboard: performance counter browsers (`issue#22521 <http://tracker.ceph.com/issues/22521>`_, `pr#19922 <https://github.com/ceph/ceph/pull/19922>`_, Rubab-Syed)
+* mgr: mgr/dashboard: RBD management (backend) (`pr#21360 <https://github.com/ceph/ceph/pull/21360>`_, Ricardo Dias)
+* mgr: mgr/dashboard: Remove unused code (`pr#21045 <https://github.com/ceph/ceph/pull/21045>`_, Volker Theile)
+* mgr: mgr/dashboard: Remove useless code (`pr#20958 <https://github.com/ceph/ceph/pull/20958>`_, Volker Theile)
+* mgr: mgr/dashboard: show warnings if data is out of date or mons are down (`pr#18847 <https://github.com/ceph/ceph/pull/18847>`_, John Spray)
+* mgr: mgr/dashboard: sort servers and OSDs in OSD list (`issue#21572 <http://tracker.ceph.com/issues/21572>`_, `pr#17993 <https://github.com/ceph/ceph/pull/17993>`_, John Spray)
+* mgr: mgr/dashboard: use rel="icon" for favicon (`pr#18013 <https://github.com/ceph/ceph/pull/18013>`_, Kefu Chai)
+* mgr: mgr/dashboard v2: Add CSS class for required form fields (`pr#20747 <https://github.com/ceph/ceph/pull/20747>`_, Volker Theile)
+* mgr: mgr/dashboard_v2: Add RBD create functionality to the Python backend (`pr#20751 <https://github.com/ceph/ceph/pull/20751>`_, Tatjana Dehler)
+* mgr: mgr/dashboard v2: Add units to performance counters (`pr#20742 <https://github.com/ceph/ceph/pull/20742>`_, Volker Theile)
+* mgr: mgr/dashboard v2: Display loading indicator in datatables during first load (`pr#20744 <https://github.com/ceph/ceph/pull/20744>`_, Volker Theile)
+* mgr: mgr/dashboard v2: Don't show details if multiple OSDs are selected (`pr#20772 <https://github.com/ceph/ceph/pull/20772>`_, Volker Theile)
+* mgr: mgr/dashboard v2: implement can_run method (`pr#20728 <https://github.com/ceph/ceph/pull/20728>`_, John Spray)
+* mgr: mgr/dashboard_v2: Initial submission of a web-based management UI (replacement for the existing dashboard) (`pr#20103 <https://github.com/ceph/ceph/pull/20103>`_, Stephan Müller, Lenz Grimmer, Tiago Melo, Ricardo Marques, Sebastian Wagner, Patrick Nawracay, Ricardo Dias, Volker Theile, Kai Wagner, Tatjana Dehler)
+* mgr: mgr/dashboard v2: Introduce CdTableSelection model (`pr#20746 <https://github.com/ceph/ceph/pull/20746>`_, Volker Theile)
+* mgr: mgr/dashboard_v2: Removed unused `tools.detail_route()` (`pr#20765 <https://github.com/ceph/ceph/pull/20765>`_, Sebastian Wagner)
+* mgr: mgr/influx: Added Additional Stats (`pr#21424 <https://github.com/ceph/ceph/pull/21424>`_, mhdo2)
+* mgr: mgr/influx: Add InfluxDB SSL Option (`pr#19374 <https://github.com/ceph/ceph/pull/19374>`_, Tobias Gall)
+* mgr: mgr/influx: Only split string on first occurence of dot (.) (`issue#23996 <http://tracker.ceph.com/issues/23996>`_, `pr#21795 <https://github.com/ceph/ceph/pull/21795>`_, Wido den Hollander)
+* mgr: mgr/influx: PEP-8 and other fixes to Influx module (`pr#19229 <https://github.com/ceph/ceph/pull/19229>`_, Wido den Hollander)
+* mgr: mgr/influx: Various fixes and improvements (`pr#20187 <https://github.com/ceph/ceph/pull/20187>`_, Wido den Hollander)
+* mgr: mgr/influx: Various time fixes (`pr#20494 <https://github.com/ceph/ceph/pull/20494>`_, Wido den Hollander)
+* mgr: mgr/localpool: default to 3x; allow min_size adjustment (`pr#18089 <https://github.com/ceph/ceph/pull/18089>`_, Sage Weil)
+* mgr: mgr/MgrClient: guard send_pgstats() with lock (`issue#23370 <http://tracker.ceph.com/issues/23370>`_, `pr#20909 <https://github.com/ceph/ceph/pull/20909>`_, Kefu Chai)
+* mgr: mgr/MgrClient: service registration filtered by service name instead of daemon name (`pr#21459 <https://github.com/ceph/ceph/pull/21459>`_, runsisi)
+* mgr: mgr/PGMap: drop REQUEST\_{SLOW,STUCK} HEALTH_WARNs (`pr#19114 <https://github.com/ceph/ceph/pull/19114>`_, Kefu Chai)
+* mgr: mgr/prometheus: add ceph_disk_occupation series (`issue#21594 <http://tracker.ceph.com/issues/21594>`_, `pr#18021 <https://github.com/ceph/ceph/pull/18021>`_, John Spray)
+* mgr: mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr prometheus plugin (`issue#22116 <http://tracker.ceph.com/issues/22116>`_, `pr#18890 <https://github.com/ceph/ceph/pull/18890>`_, Peter Woodman)
+* mgr: mgr/prometheus: Fix for MDS metrics (`issue#20899 <http://tracker.ceph.com/issues/20899>`_, `pr#17318 <https://github.com/ceph/ceph/pull/17318>`_, John Spray, Jeremy H Austin)
+* mgr: mgr/prometheus: fix PG state names (`pr#21288 <https://github.com/ceph/ceph/pull/21288>`_, John Spray)
+* mgr: mgr/prometheus: Skip bogus entries (`pr#20456 <https://github.com/ceph/ceph/pull/20456>`_, Boris Ranto)
+* mgr: mgr/prometheus: skip OSD output if missing from CRUSH devices (`pr#20644 <https://github.com/ceph/ceph/pull/20644>`_, John Spray)
+* mgr: mgr/restful: A couple of restful fixes (`pr#18649 <https://github.com/ceph/ceph/pull/18649>`_, Boris Ranto)
+* mgr: mgr/restful: cleaner message when not configured (`issue#21292 <http://tracker.ceph.com/issues/21292>`_, `pr#17573 <https://github.com/ceph/ceph/pull/17573>`_, John Spray)
+* mgr: mgr/smart: fix python3 module loading (`pr#21047 <https://github.com/ceph/ceph/pull/21047>`_, Ricardo Dias)
+* mgr: mgr/status: fix ceph fs status returns error (`issue#21752 <http://tracker.ceph.com/issues/21752>`_, `pr#18233 <https://github.com/ceph/ceph/pull/18233>`_, Yanhu Cao)
+* mgr: mgr/status: format byte quantities in base 2 multiples (`issue#21189 <http://tracker.ceph.com/issues/21189>`_, `pr#17380 <https://github.com/ceph/ceph/pull/17380>`_, John Spray)
+* mgr: mgr/telemetry: Add Ceph Telemetry module to send reports back to project (`pr#21970 <https://github.com/ceph/ceph/pull/21970>`_, Wido den Hollander)
+* mgr: mgr/zabbix: fix div by zero (`issue#21518 <http://tracker.ceph.com/issues/21518>`_, `pr#17931 <https://github.com/ceph/ceph/pull/17931>`_, John Spray)
+* mgr: mgr/zabbix: ignore osd with 0 kb capacity (`issue#21904 <http://tracker.ceph.com/issues/21904>`_, `pr#18809 <https://github.com/ceph/ceph/pull/18809>`_, Ilja Slepnev)
+* mgr: mgr/zabbix: Implement health checks (`pr#20198 <https://github.com/ceph/ceph/pull/20198>`_, Wido den Hollander)
+* mgr: mgr/zabbix: Send max, min and avg PGs of OSDs to Zabbix (`pr#21043 <https://github.com/ceph/ceph/pull/21043>`_, Wido den Hollander)
+* mgr: mgr/Zabbix: Various fixes to Zabbix module (`pr#19452 <https://github.com/ceph/ceph/pull/19452>`_, Wido den Hollander)
+* mgr: mimic: mgr/telegraf: Telegraf module for Ceph Mgr (`pr#22013 <https://github.com/ceph/ceph/pull/22013>`_, Wido den Hollander)
+* mgr: Modify mgr-influx module database check to not require admin privileges (`pr#18102 <https://github.com/ceph/ceph/pull/18102>`_, Benjeman Meekhof)
+* mgr: mon,mgr: improve 'mgr module disable' cmd (`pr#21188 <https://github.com/ceph/ceph/pull/21188>`_, Gu Zhongyan)
+* mgr: mon, mgr: move "osd pool stats" command to mgr and mgr python module (`pr#19985 <https://github.com/ceph/ceph/pull/19985>`_, Chang Liu)
+* mgr: mon/MgrStatMonitor: fix formatting of pending_digest (`issue#22991 <http://tracker.ceph.com/issues/22991>`_, `pr#20426 <https://github.com/ceph/ceph/pull/20426>`_, Patrick Donnelly)
+* mgr,mon: mon/MgrMonitor: read cmd descs if empty on update_from_paxos() (`issue#21300 <http://tracker.ceph.com/issues/21300>`_, `pr#17846 <https://github.com/ceph/ceph/pull/17846>`_, Joao Eduardo Luis)
+* mgr,mon: mon,mgr: remove single wildcard '\*' from ceph comand line description (`pr#21139 <https://github.com/ceph/ceph/pull/21139>`_, Gu Zhongyan)
+* mgr,mon: mon/mgr: sync "mgr_command_descs","osd_metadata" and "mgr_metadata" prefixes to new mons (`issue#21527 <http://tracker.ceph.com/issues/21527>`_, `pr#17929 <https://github.com/ceph/ceph/pull/17929>`_, huanwen ren)
+* mgr,mon: mon/MonCommands: mgr metadata - improve parameter naming consistency (`issue#23330 <http://tracker.ceph.com/issues/23330>`_, `pr#20866 <https://github.com/ceph/ceph/pull/20866>`_, Jan Fajerski)
+* mgr: preventing blank hostname in DaemonState (`issue#20887 <http://tracker.ceph.com/issues/20887>`_, `issue#21060 <http://tracker.ceph.com/issues/21060>`_, `pr#17138 <https://github.com/ceph/ceph/pull/17138>`_, liuchang0812)
+* mgr: prometheus: added osd commit/apply latency metrics (#22718) (`issue#22718 <http://tracker.ceph.com/issues/22718>`_, `pr#19980 <https://github.com/ceph/ceph/pull/19980>`_, Konstantin Shalygin)
+* mgr: prometheus: Don't crash on OSDs without metadata (`pr#20539 <https://github.com/ceph/ceph/pull/20539>`_, Christopher Blum)
+* mgr: prometheus fix metadata labels (`pr#21557 <https://github.com/ceph/ceph/pull/21557>`_, Jan Fajerski)
+* mgr: prometheus: set metadata metrics value to '1' (#22717) (`issue#22717 <http://tracker.ceph.com/issues/22717>`_, `pr#19979 <https://github.com/ceph/ceph/pull/19979>`_, Konstantin Shalygin)
+* mgr: pybind/mgr/balancer: add sanity check against empty adjusted_map (`pr#20836 <https://github.com/ceph/ceph/pull/20836>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: fix pool-deletion vs auto-optimization race (`pr#20706 <https://github.com/ceph/ceph/pull/20706>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: fix sanity check against empty weight-set (`pr#20278 <https://github.com/ceph/ceph/pull/20278>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: increase bad_steps properly (`pr#20194 <https://github.com/ceph/ceph/pull/20194>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: load weight-set from ms (`pr#20197 <https://github.com/ceph/ceph/pull/20197>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: more specific command outputs (`pr#20305 <https://github.com/ceph/ceph/pull/20305>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: remove optimization plan properly (`pr#20224 <https://github.com/ceph/ceph/pull/20224>`_, xie xingguo)
+* mgr: pybind/mgr/balancer: two more fixes (`pr#20788 <https://github.com/ceph/ceph/pull/20788>`_, xie xingguo)
+* mgr: pybind/mgr/dashboard: add url_prefix (`issue#20568 <http://tracker.ceph.com/issues/20568>`_, `pr#17119 <https://github.com/ceph/ceph/pull/17119>`_, Nick Erdmann)
+* mgr: pybind/mgr/dashboard: fix duplicated slash in html href (`issue#22851 <http://tracker.ceph.com/issues/22851>`_, `pr#20229 <https://github.com/ceph/ceph/pull/20229>`_, Shengjing Zhu)
+* mgr,pybind: mgr/dashboard: fix pool size base conversion (`pr#16771 <https://github.com/ceph/ceph/pull/16771>`_, Yixing Yan)
+* mgr: pybind/mgr/dashboard: fix reverse proxy support (`issue#22557 <http://tracker.ceph.com/issues/22557>`_, `pr#19758 <https://github.com/ceph/ceph/pull/19758>`_, Nick Erdmann)
+* mgr,pybind: mgr/iostat: print output as a table (`pr#21338 <https://github.com/ceph/ceph/pull/21338>`_, Mohamad Gebai)
+* mgr: pybind/mgr/localpool: module to automagically create localized pools (`pr#17528 <https://github.com/ceph/ceph/pull/17528>`_, Sage Weil)
+* mgr: pybind/mgr/mgr_module: add default param for MgrStandbyModule.get_con… (`pr#19948 <https://github.com/ceph/ceph/pull/19948>`_, Kefu Chai)
+* mgr: pybind/mgr/mgr_module: make rados handle available to all modules (`pr#19972 <https://github.com/ceph/ceph/pull/19972>`_, Sage Weil)
+* mgr: pybind/mgr_module: move PRIO\_\* and PERFCOUNTER\_\* to MgrModule class (`pr#18251 <https://github.com/ceph/ceph/pull/18251>`_, Jan Fajerski)
+* mgr: pybind/mgr: new 'hello world' mgr module skeleton (`pr#19491 <https://github.com/ceph/ceph/pull/19491>`_, Yaarit Hatuka)
+* mgr: pybind/mgr/prometheus: add file_sd_config command (`pr#21061 <https://github.com/ceph/ceph/pull/21061>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: add osd_in/out metric; make osd_weight a metric (`pr#18243 <https://github.com/ceph/ceph/pull/18243>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: add StandbyModule and handle failed MON cluster (`pr#19744 <https://github.com/ceph/ceph/pull/19744>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: don't crash when encountering an unknown PG state (`pr#18903 <https://github.com/ceph/ceph/pull/18903>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: don't export metrics for dead daemon; new metrics (`pr#20506 <https://github.com/ceph/ceph/pull/20506>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: fix creation of osd_metadata metric (`pr#21530 <https://github.com/ceph/ceph/pull/21530>`_, Jan Fajerski)
+* mgr: pybind/mgr/prometheus: fix metric type undef -> untyped (`issue#22313 <http://tracker.ceph.com/issues/22313>`_, `pr#19524 <https://github.com/ceph/ceph/pull/19524>`_, Ilya Margolin)
+* mgr: pybind/mgr/prometheus: fix metric type undef -> untyped (`pr#18208 <https://github.com/ceph/ceph/pull/18208>`_, Jan Fajerski)
+* mgr,pybind: pybing/mgr/prometheus: return default port if config-key get returns … (`pr#21696 <https://github.com/ceph/ceph/pull/21696>`_, Jan Fajerski)
+* mgr: python interface rework + enable modules to run in standby mode (`issue#21593 <http://tracker.ceph.com/issues/21593>`_, `issue#17460 <http://tracker.ceph.com/issues/17460>`_, `pr#16651 <https://github.com/ceph/ceph/pull/16651>`_, John Spray, Sage Weil)
+* mgr: quieten logging on missing OSD stats (`pr#20485 <https://github.com/ceph/ceph/pull/20485>`_, John Spray)
+* mgr,rbd: mgr/dashboard: added iSCSI IOPS/throughput metrics (`issue#21391 <http://tracker.ceph.com/issues/21391>`_, `pr#18653 <https://github.com/ceph/ceph/pull/18653>`_, Jason Dillaman)
+* mgr,rbd: mgr/dashboard: fix duplicate images listed on iSCSI status page (`issue#21017 <http://tracker.ceph.com/issues/21017>`_, `pr#17055 <https://github.com/ceph/ceph/pull/17055>`_, Jason Dillaman)
+* mgr: reconcile can_run checks and selftest (`pr#21607 <https://github.com/ceph/ceph/pull/21607>`_, John Spray, Kefu Chai)
+* mgr: remove a few junk lines (`pr#20005 <https://github.com/ceph/ceph/pull/20005>`_, John Spray)
+* mgr: remove unused static files from dashboard module (`pr#16762 <https://github.com/ceph/ceph/pull/16762>`_, John Spray)
+* mgr: request daemon's metadata when receiving a report from an unknown server (`issue#21687 <http://tracker.ceph.com/issues/21687>`_, `pr#18484 <https://github.com/ceph/ceph/pull/18484>`_, Chang Liu)
+* mgr,rgw: mgr/dashboard: RGW page (`pr#19512 <https://github.com/ceph/ceph/pull/19512>`_, Chang Liu)
+* mgr,rgw: prometheus: Implement rgw_metadata metric (`pr#21383 <https://github.com/ceph/ceph/pull/21383>`_, Boris Ranto)
+* mgr: safety checks on pyThreadState usage (`pr#18093 <https://github.com/ceph/ceph/pull/18093>`_, John Spray)
+* mgr: set explicit thread name (`issue#21404 <http://tracker.ceph.com/issues/21404>`_, `pr#17756 <https://github.com/ceph/ceph/pull/17756>`_, John Spray)
+* mgr: silence warning from -Wsign-compare (`pr#17881 <https://github.com/ceph/ceph/pull/17881>`_, Jos Collin)
+* mgr: skip first non-zero incremental in PGMap::apply_incremental() (`issue#21773 <http://tracker.ceph.com/issues/21773>`_, `pr#18347 <https://github.com/ceph/ceph/pull/18347>`_, Aleksei Gutikov)
+* mgr/status: output to stdout, not stderr (`issue#24175 <http://tracker.ceph.com/issues/24175>`_, `pr#22135 <https://github.com/ceph/ceph/pull/22135>`_, John Spray)
+* mgr: store declared_types in MgrSession (`issue#21197 <http://tracker.ceph.com/issues/21197>`_, `pr#17932 <https://github.com/ceph/ceph/pull/17932>`_, John Spray)
+* mgr: systemd: Wait 10 seconds before restarting ceph-mgr (`issue#23083 <http://tracker.ceph.com/issues/23083>`_, `pr#20533 <https://github.com/ceph/ceph/pull/20533>`_, Wido den Hollander)
+* mgr,tests: mgr/dashboard: skip data pool testcase for none-bluestore clusters (`pr#21004 <https://github.com/ceph/ceph/pull/21004>`_, Tatjana Dehler)
+* mgr,tests: mgr/dashboard_v2: fix test_perf_counters_mgr_get (`pr#20916 <https://github.com/ceph/ceph/pull/20916>`_, Tiago Melo)
+* mgr,tests: qa: add new prometheus test to rados/mgr suite (`pr#20047 <https://github.com/ceph/ceph/pull/20047>`_, John Spray)
+* mgr,tests: qa: configure zabbix properly before selftest (`issue#22514 <http://tracker.ceph.com/issues/22514>`_, `pr#19634 <https://github.com/ceph/ceph/pull/19634>`_, John Spray)
+* mgr,tests: qa: fix mgr _load_module helper (`pr#18685 <https://github.com/ceph/ceph/pull/18685>`_, John Spray)
+* mgr,tools: mgr/iostat: implement 'ceph iostat' as a mgr plugin (`pr#20100 <https://github.com/ceph/ceph/pull/20100>`_, Mohamad Gebai)
+* mgr: use new style config opts + add metadata (`pr#17374 <https://github.com/ceph/ceph/pull/17374>`_, John Spray)
+* mgr/zabbix: Fix wrong log message (`pr#21237 <https://github.com/ceph/ceph/pull/21237>`_, Gu Zhongyan)
+* mgr/zabbix: monitoring template improvements (`pr#19901 <https://github.com/ceph/ceph/pull/19901>`_, Marc Schoechlin)
+* mon: Add `ceph osd get-require-min-compat-client` command (`pr#19015 <https://github.com/ceph/ceph/pull/19015>`_, hansbogert)
+* mon: add  'ceph osd pool get erasure allow_ec_overwrites' command (`pr#21102 <https://github.com/ceph/ceph/pull/21102>`_, Mykola Golub)
+* mon: add MMonHealth back (`issue#22462 <http://tracker.ceph.com/issues/22462>`_, `pr#20528 <https://github.com/ceph/ceph/pull/20528>`_, Kefu Chai)
+* mon: add mon_health_preluminous_compat_warning (`pr#16902 <https://github.com/ceph/ceph/pull/16902>`_, Sage Weil)
+* mon: a few conversions to monotonic clock (`pr#18595 <https://github.com/ceph/ceph/pull/18595>`_, Patrick Donnelly)
+* mon: align lspools output (`pr#19597 <https://github.com/ceph/ceph/pull/19597>`_, Jos Collin)
+* mon: allow cluter and debug logs to go to stderr, with appropriate prefix (`pr#19385 <https://github.com/ceph/ceph/pull/19385>`_, Sage Weil)
+* mon: cache reencoded osdmaps (`issue#23713 <http://tracker.ceph.com/issues/23713>`_, `pr#21605 <https://github.com/ceph/ceph/pull/21605>`_, Sage Weil, Xiaoxi CHEN)
+* mon: centralized config (`pr#20172 <https://github.com/ceph/ceph/pull/20172>`_, Sage Weil)
+* mon: "ceph osd crush rule rename" support (`pr#17029 <https://github.com/ceph/ceph/pull/17029>`_, xie xingguo)
+* mon: check monitor address configuration (`pr#18073 <https://github.com/ceph/ceph/pull/18073>`_, Li Wang)
+* mon: clean up cluster logging on mon events (`issue#22082 <http://tracker.ceph.com/issues/22082>`_, `pr#18822 <https://github.com/ceph/ceph/pull/18822>`_, John Spray)
+* mon: cleanups to optracker code (`pr#21371 <https://github.com/ceph/ceph/pull/21371>`_, John Spray)
+* mon: cleanup unused option mon_health_data_update_interval (`pr#17728 <https://github.com/ceph/ceph/pull/17728>`_, Yao Guotao)
+* mon: common/options: set max_background_jobs instead of max_background_compactions (`pr#18397 <https://github.com/ceph/ceph/pull/18397>`_, Kefu Chai)
+* mon: Compress the warnings of pgs not scrubbed or deep-scrubbed (`pr#17295 <https://github.com/ceph/ceph/pull/17295>`_, Zhi Zhang)
+* mon: do not use per_pool_sum_delta to show recovery summary (`issue#22727 <http://tracker.ceph.com/issues/22727>`_, `pr#20009 <https://github.com/ceph/ceph/pull/20009>`_, Chang Liu)
+* mon: don't blow away bootstrap-mgr on upgrades (`issue#20950 <http://tracker.ceph.com/issues/20950>`_, `pr#18399 <https://github.com/ceph/ceph/pull/18399>`_, John Spray)
+* mon: double mon_mgr_mkfs_grace from 60s -> 120s (`pr#20955 <https://github.com/ceph/ceph/pull/20955>`_, Sage Weil)
+* mon: Drop redundant access specifier, etc (cleanup) (`pr#19028 <https://github.com/ceph/ceph/pull/19028>`_, Shinobu Kinjo)
+* mon: dump percent_used PGMap field as float (`pr#20439 <https://github.com/ceph/ceph/pull/20439>`_, John Spray)
+* mon: dump servicemap along with MgrStatMonitor dump info (`pr#18760 <https://github.com/ceph/ceph/pull/18760>`_, Zhi Zhang)
+* mon: expand cap validity check for mgr, osd, mds (`issue#22525 <http://tracker.ceph.com/issues/22525>`_, `pr#21311 <https://github.com/ceph/ceph/pull/21311>`_, Jing Li, Sage Weil)
+* mon: final luminous compatset feature and osdmap flag (`pr#17333 <https://github.com/ceph/ceph/pull/17333>`_, Sage Weil)
+* mon: fix commands advertised during mon cluster upgrade (`pr#16871 <https://github.com/ceph/ceph/pull/16871>`_, Sage Weil)
+* mon: fix dropping mgr metadata for active mgr (#21260) (`issue#21260 <http://tracker.ceph.com/issues/21260>`_, `pr#17571 <https://github.com/ceph/ceph/pull/17571>`_, John Spray)
+* mon: fix "fs new" pool metadata update, tests (`issue#20959 <http://tracker.ceph.com/issues/20959>`_, `pr#16954 <https://github.com/ceph/ceph/pull/16954>`_, Greg Farnum)
+* mon: fix legacy health checks in 'ceph status' during upgrade; fix jewel-x upgrade combo (`pr#16967 <https://github.com/ceph/ceph/pull/16967>`_, Sage Weil)
+* mon: fix mgr using auth_client_required policy (`pr#20048 <https://github.com/ceph/ceph/pull/20048>`_, John Spray)
+* mon: fix `osd out` clog message (`issue#21249 <http://tracker.ceph.com/issues/21249>`_, `pr#17525 <https://github.com/ceph/ceph/pull/17525>`_, John Spray)
+* mon: fix slow op warning on mon, improve slow op warnings (`issue#23769 <http://tracker.ceph.com/issues/23769>`_, `pr#21684 <https://github.com/ceph/ceph/pull/21684>`_, Sage Weil)
+* mon: fix structure of 'features' command (`pr#20115 <https://github.com/ceph/ceph/pull/20115>`_, Sage Weil)
+* mon: fix two stray legacy get_health() callers (`pr#17269 <https://github.com/ceph/ceph/pull/17269>`_, Sage Weil)
+* mon: fix wrong mon-num counting  logic of 'ceph features' command (`pr#16887 <https://github.com/ceph/ceph/pull/16887>`_, xie xingguo)
+* mon: handle bad snapshot removal reqs gracefully (`issue#18746 <http://tracker.ceph.com/issues/18746>`_, `pr#20835 <https://github.com/ceph/ceph/pull/20835>`_, Paul Emmerich)
+* mon: handle monitor lag when killing mgrs (`issue#20629 <http://tracker.ceph.com/issues/20629>`_, `pr#18268 <https://github.com/ceph/ceph/pull/18268>`_, John Spray)
+* mon: incorrect MAX AVAIL in "ceph df" (`issue#21243 <http://tracker.ceph.com/issues/21243>`_, `pr#17513 <https://github.com/ceph/ceph/pull/17513>`_, liuchang0812)
+* mon: invalid JSON returned when querying pool parameters (`issue#23200 <http://tracker.ceph.com/issues/23200>`_, `pr#20745 <https://github.com/ceph/ceph/pull/20745>`_, Chang Liu)
+* mon/LogMonitor: call no_reply() on ignored log message (`pr#22104 <https://github.com/ceph/ceph/pull/22104>`_, Sage Weil)
+* mon: mark mgr reports as no_reply (`issue#22114 <http://tracker.ceph.com/issues/22114>`_, `pr#21057 <https://github.com/ceph/ceph/pull/21057>`_, Kefu Chai)
+* mon: mark mon_allow_pool_delete as observed (`pr#18125 <https://github.com/ceph/ceph/pull/18125>`_, Dan van der Ster)
+* mon: mark OSD beacons and pg_create messages as no_reply (`issue#22114 <http://tracker.ceph.com/issues/22114>`_, `pr#20517 <https://github.com/ceph/ceph/pull/20517>`_, Greg Farnum)
+* mon: mon/AuthMonitor: don't validate `fs authorize` caps with `valid_caps()` (`pr#21418 <https://github.com/ceph/ceph/pull/21418>`_, Joao Eduardo Luis)
+* mon: mon/ConfigMonitor: clean up prepare_command() (`pr#20911 <https://github.com/ceph/ceph/pull/20911>`_, Gu Zhongyan)
+* mon: mon/Elector: force election epoch bump on start (`issue#20949 <http://tracker.ceph.com/issues/20949>`_, `pr#16944 <https://github.com/ceph/ceph/pull/16944>`_, Sage Weil)
+* mon: mon/Elector: remove unused member fields start_stamp and ack_stamp (`pr#21091 <https://github.com/ceph/ceph/pull/21091>`_, runsisi)
+* mon: mon/LogMonitor: "log last" should return up to n entries (`pr#18759 <https://github.com/ceph/ceph/pull/18759>`_, Kefu Chai)
+* mon: mon/MDSMonitor: fix clang build failure (`pr#20637 <https://github.com/ceph/ceph/pull/20637>`_, Willem Jan Withagen)
+* mon: mon,mgr: make  osd_metric more popular and report slow ops to mgr (`issue#23045 <http://tracker.ceph.com/issues/23045>`_, `pr#20660 <https://github.com/ceph/ceph/pull/20660>`_, lvshanchun)
+* mon: mon/MgrMonitor: limit mgrmap history (`issue#22257 <http://tracker.ceph.com/issues/22257>`_, `pr#19185 <https://github.com/ceph/ceph/pull/19185>`_, Sage Weil)
+* mon: mon/MonCommands: fix copy-and-paste error (`pr#17271 <https://github.com/ceph/ceph/pull/17271>`_, xie xingguo)
+* mon: mon,option: set default value for mon_dns_srv_name (`issue#21204 <http://tracker.ceph.com/issues/21204>`_, `pr#17539 <https://github.com/ceph/ceph/pull/17539>`_, Kefu Chai)
+* mon: mon/OSDMonitor: add location option for "crush add-bucket" command (`pr#17125 <https://github.com/ceph/ceph/pull/17125>`_, xie xingguo)
+* mon: mon/OSDMonitor: add 'osd crush set-all-straw-buckets-to-straw2' (`pr#18460 <https://github.com/ceph/ceph/pull/18460>`_, Sage Weil)
+* mon: mon/OSDMonitor: add plain output for "crush class ls-osd" command (`pr#17034 <https://github.com/ceph/ceph/pull/17034>`_, xie xingguo)
+* mon: mon/OSDMonitor: add space after __func__ in log msg (`pr#19127 <https://github.com/ceph/ceph/pull/19127>`_, Kefu Chai)
+* mon: mon/OSDMonitor: Better prepare_command_pool_set E2BIG error message (`pr#19944 <https://github.com/ceph/ceph/pull/19944>`_, Brad Hubbard)
+* mon: mon/OSDMonitor.cc: fix expected_num_objects interpret error (`issue#22530 <http://tracker.ceph.com/issues/22530>`_, `pr#19651 <https://github.com/ceph/ceph/pull/19651>`_, Yang Honggang)
+* mon: mon/OSDMonitor.cc : set erasure-code-profile to "" when create replicated pools (`pr#19673 <https://github.com/ceph/ceph/pull/19673>`_, zouaiguo)
+* mon: mon/OSDMonitor: check last_scan_epoch instead when sending creates (`issue#20785 <http://tracker.ceph.com/issues/20785>`_, `pr#17248 <https://github.com/ceph/ceph/pull/17248>`_, Kefu Chai)
+* mon: mon/OSDMonitor: clean up cmd 'osd tree-from' (`pr#20839 <https://github.com/ceph/ceph/pull/20839>`_, Gu Zhongyan)
+* mon: mon/OSDMonitor: do not send_pg_creates with stale info (`issue#20785 <http://tracker.ceph.com/issues/20785>`_, `pr#17065 <https://github.com/ceph/ceph/pull/17065>`_, Kefu Chai)
+* mon: mon/OSDMonitor: error out if setting ruleset-\* ec profile property (`pr#17848 <https://github.com/ceph/ceph/pull/17848>`_, Sage Weil)
+* mon: mon/OSDMonitor: fix improper input/testing range of crush somke testing (`pr#17179 <https://github.com/ceph/ceph/pull/17179>`_, xie xingguo)
+* mon: mon/OSDMonitor: fix 'osd pg temp' unable to cleanup pg-temp (`pr#16892 <https://github.com/ceph/ceph/pull/16892>`_, xie xingguo)
+* mon: mon/OSDMonitor: implement 'osd crush ls <node>' (`pr#16920 <https://github.com/ceph/ceph/pull/16920>`_, Sage Weil)
+* mon: mon/OSDMonitor: kill pending upmap changes too if pool is gone (`pr#20704 <https://github.com/ceph/ceph/pull/20704>`_, xie xingguo)
+* mon: mon/OSDMonitor: logging non-active osd id when handling osd beacon (`pr#21092 <https://github.com/ceph/ceph/pull/21092>`_, runsisi)
+* mon: mon/OSDMonitor: make 'osd crush rule rename' idempotent (`issue#21162 <http://tracker.ceph.com/issues/21162>`_, `pr#17329 <https://github.com/ceph/ceph/pull/17329>`_, xie xingguo)
+* mon: mon/OSDMonitor: "osd pool application get" support (`issue#20976 <http://tracker.ceph.com/issues/20976>`_, `pr#16955 <https://github.com/ceph/ceph/pull/16955>`_, xie xingguo)
+* mon: mon/OSDMonitor: txsize should be greater or eq to prune_interval - 1 (`pr#21430 <https://github.com/ceph/ceph/pull/21430>`_, Kefu Chai)
+* mon: mon/PGMap: drop DISK LOG column (`pr#17617 <https://github.com/ceph/ceph/pull/17617>`_, Sage Weil)
+* mon: mon/PGMap: fix "0 stuck requests are blocked > 4096 sec" warn (`pr#17099 <https://github.com/ceph/ceph/pull/17099>`_, xie xingguo)
+* mon: mon/PGMap: nice numbers for 'data' section of 'ceph df' command (`pr#17368 <https://github.com/ceph/ceph/pull/17368>`_, xie xingguo)
+* mon: mon/PGMap: Remove unnecessary header (`pr#18343 <https://github.com/ceph/ceph/pull/18343>`_, Shinobu Kinjo)
+* mon: mon/PGMap: reweight::by_utilization - skip DNE osds (`issue#20970 <http://tracker.ceph.com/issues/20970>`_, `pr#17064 <https://github.com/ceph/ceph/pull/17064>`_, xie xingguo)
+* mon: mon/pgmap: update pool nearfull display (`pr#17043 <https://github.com/ceph/ceph/pull/17043>`_, huanwen ren)
+* mon: more aggressively convert crush rulesets -> distinct rules (`pr#17508 <https://github.com/ceph/ceph/pull/17508>`_, John Spray, Sage Weil)
+* mon: more constness (`pr#17748 <https://github.com/ceph/ceph/pull/17748>`_, Kefu Chai)
+* mon: node ls improvement (`pr#20820 <https://github.com/ceph/ceph/pull/20820>`_, Gu Zhongyan)
+* mon: 'node ls' mgr support (`pr#20711 <https://github.com/ceph/ceph/pull/20711>`_, Gu Zhongyan)
+* mon: NULL check of logger before use (`pr#18788 <https://github.com/ceph/ceph/pull/18788>`_, Amit Kumar)
+* mon,osd: dump "compression_algorithms" in "mon metadata" (`issue#24135 <http://tracker.ceph.com/issues/24135>`_, `issue#22420 <http://tracker.ceph.com/issues/22420>`_, `pr#22004 <https://github.com/ceph/ceph/pull/22004>`_, Kefu Chai, Casey Bodley)
+* mon: osd feature checks with 0 up osds (`issue#21471 <http://tracker.ceph.com/issues/21471>`_, `issue#20751 <http://tracker.ceph.com/issues/20751>`_, `pr#17831 <https://github.com/ceph/ceph/pull/17831>`_, Brad Hubbard, Sage Weil)
+* mon: osdmap prune (`pr#19331 <https://github.com/ceph/ceph/pull/19331>`_, Joao Eduardo Luis)
+* mon/OSDMonitor: cleanup: move bufferlist before use (`pr#18258 <https://github.com/ceph/ceph/pull/18258>`_, Shinobu Kinjo)
+* mon/OSDMonitor: use new style options (`pr#18209 <https://github.com/ceph/ceph/pull/18209>`_, Kefu Chai)
+* mon: osd/OSDMap.h: toss osd out if it has no more pending states (`pr#19642 <https://github.com/ceph/ceph/pull/19642>`_, xie xingguo)
+* mon: paxos cleanup (`pr#20078 <https://github.com/ceph/ceph/pull/20078>`_, huanwen ren)
+* mon/PGMap: let pg_string_state() return boost::optional<> (`issue#21609 <http://tracker.ceph.com/issues/21609>`_, `pr#18218 <https://github.com/ceph/ceph/pull/18218>`_, Kefu Chai)
+* mon/PGMap: use new-style options and cleanup (`pr#18647 <https://github.com/ceph/ceph/pull/18647>`_, Kefu Chai)
+* mon: post-luminous cleanup (part 3 of ?) (`pr#17607 <https://github.com/ceph/ceph/pull/17607>`_, Sage Weil)
+* mon: rate limit on health check update logging (`issue#20888 <http://tracker.ceph.com/issues/20888>`_, `pr#16942 <https://github.com/ceph/ceph/pull/16942>`_, John Spray)
+* mon: reenable timer to send digest when paxos is temporarily inactive (`issue#22142 <http://tracker.ceph.com/issues/22142>`_, `pr#19404 <https://github.com/ceph/ceph/pull/19404>`_, Jan Fajerski)
+* mon: remove health service (`pr#20119 <https://github.com/ceph/ceph/pull/20119>`_, Chang Liu)
+* mon: remove_is_write_ready() (`pr#19191 <https://github.com/ceph/ceph/pull/19191>`_, Kefu Chai)
+* mon: remove pre-luminous compat cruft (2 of many) (`pr#17322 <https://github.com/ceph/ceph/pull/17322>`_, Sage Weil)
+* mon: remove unused waiting_for_commit (`pr#18617 <https://github.com/ceph/ceph/pull/18617>`_, Kefu Chai)
+* mon: return directly after health_events_cleanup (`pr#16964 <https://github.com/ceph/ceph/pull/16964>`_, wang yang)
+* mon: revert mds metadata argument name change (`issue#22527 <http://tracker.ceph.com/issues/22527>`_, `pr#19926 <https://github.com/ceph/ceph/pull/19926>`_, Patrick Donnelly)
+* mon: show feature flags when printing MonSession (`pr#17535 <https://github.com/ceph/ceph/pull/17535>`_, Paul Emmerich)
+* mon: some cleanup (`pr#17067 <https://github.com/ceph/ceph/pull/17067>`_, huanwen ren)
+* mon,tests: vstart: set osd_pool_default_erasure_code_profile in initial ceph.conf (`pr#21008 <https://github.com/ceph/ceph/pull/21008>`_, Mykola Golub)
+* mon: update get_store_prefixes implementations (`issue#21534 <http://tracker.ceph.com/issues/21534>`_, `pr#17940 <https://github.com/ceph/ceph/pull/17940>`_, John Spray, huanwen ren)
+* mon: update PaxosService::cached_first_committed in PaxosService::maybe_trim() (`issue#11332 <http://tracker.ceph.com/issues/11332>`_, `pr#19397 <https://github.com/ceph/ceph/pull/19397>`_, Xuehan Xu, yupeng chen)
+* mon: use ceph_clock_now if message is self-generated (`pr#17311 <https://github.com/ceph/ceph/pull/17311>`_, huangjun)
+* mon: warn about using osd new instead of osd create (`issue#21023 <http://tracker.ceph.com/issues/21023>`_, `pr#17242 <https://github.com/ceph/ceph/pull/17242>`_, Neha Ojha)
+* msg/async/AsyncConnection: remove legacy feature case handle (`pr#18469 <https://github.com/ceph/ceph/pull/18469>`_, Haomai Wang)
+* msg/async: avoid referencing the temporary string (`pr#20640 <https://github.com/ceph/ceph/pull/20640>`_, Kefu Chai)
+* msg/async: batch handle msg_iovlen (`pr#18415 <https://github.com/ceph/ceph/pull/18415>`_, Jianpeng Ma)
+* msg/async/dpdk: remove xsky copyright and LGPL copying (`pr#21121 <https://github.com/ceph/ceph/pull/21121>`_, Kefu Chai)
+* msg/async/EventKqueue: assert on OOM (`pr#21488 <https://github.com/ceph/ceph/pull/21488>`_, Kefu Chai)
+* msg/async: fix ms_dpdk_coremask and ms_dpdk_coremask conflict (`pr#18678 <https://github.com/ceph/ceph/pull/18678>`_, chunmei)
+* msg/async:fix the incoming parameter type of EventCenter::process_events() (`pr#20607 <https://github.com/ceph/ceph/pull/20607>`_, shangfufei)
+* msg/async misc cleanup (`pr#18531 <https://github.com/ceph/ceph/pull/18531>`_, Jianpeng Ma)
+* msg/async:  misc cleanup (`pr#18575 <https://github.com/ceph/ceph/pull/18575>`_, Jianpeng Ma)
+* msg/async/rdma: a tiny typo fix (`pr#18660 <https://github.com/ceph/ceph/pull/18660>`_, Yan Lei)
+* msg/async/rdma: fix a coredump introduced by PR #18053 (`pr#18204 <https://github.com/ceph/ceph/pull/18204>`_, Yan Lei)
+* msg/async/rdma: fix a potential coredump when handling tx_buffers under heavy RDMA (`pr#18036 <https://github.com/ceph/ceph/pull/18036>`_, Yan Lei)
+* msg/async/rdma: fixes crash for multi rados client within one process (`pr#16981 <https://github.com/ceph/ceph/pull/16981>`_, Alex Mikheev, Haomai Wang, Adir Lev)
+* msg/async/rdma: fix Tx buffer leakage that can introduce "heartbeat no reply" (`pr#18053 <https://github.com/ceph/ceph/pull/18053>`_, Yan Lei)
+* msg/async/rdma: refactor rx buffer pool allocator (`pr#17018 <https://github.com/ceph/ceph/pull/17018>`_, Alex Mikheev)
+* msg/async/rdma: unnecessary reinitiliazation of an iterator (`pr#18190 <https://github.com/ceph/ceph/pull/18190>`_, JustL)
+* msg/async: size of EventCenter::file_events should be greater than fd (`issue#23253 <http://tracker.ceph.com/issues/23253>`_, `pr#20764 <https://github.com/ceph/ceph/pull/20764>`_, Yupeng Chen)
+* msg/async: use bitset<> to do the popcnt (`pr#18681 <https://github.com/ceph/ceph/pull/18681>`_, Kefu Chai)
+* msg/async: use device before checking (`pr#19738 <https://github.com/ceph/ceph/pull/19738>`_, Xiaoyan Li)
+* msg: drop duplicate include (`pr#19623 <https://github.com/ceph/ceph/pull/19623>`_, /bin/bash)
+* msg: drop the unnecessary polling_stop() (`pr#17079 <https://github.com/ceph/ceph/pull/17079>`_, Jos Collin)
+* msg: Initialize lkey,bound,port_cnt,num_chunk,gid_idx (`pr#17797 <https://github.com/ceph/ceph/pull/17797>`_, Amit Kumar)
+* msg: Initializing class members in module msg (`pr#17568 <https://github.com/ceph/ceph/pull/17568>`_, Amit Kumar)
+* msg: reimplement sigpipe blocking (`pr#18105 <https://github.com/ceph/ceph/pull/18105>`_, Greg Farnum)
+* msg: remove the ),it's redundant (`pr#17544 <https://github.com/ceph/ceph/pull/17544>`_, linxuhua)
+* msg: resurrect support for !CEPH_FEATURE_MSG_AUTH (`pr#19044 <https://github.com/ceph/ceph/pull/19044>`_, Ilya Dryomov)
+* msgr: Optimization for connection establishment (`pr#16006 <https://github.com/ceph/ceph/pull/16006>`_, shangfufei)
+* msg/simple: pass a char for reading from shutdown_rd_fd (`pr#19094 <https://github.com/ceph/ceph/pull/19094>`_, Kefu Chai)
+* NVMDevice: fix issued caused by #17002 (`pr#17112 <https://github.com/ceph/ceph/pull/17112>`_, Ziye Yang)
+* objclass-sdk: expose __cls_init() to the world (`pr#21581 <https://github.com/ceph/ceph/pull/21581>`_, Kefu Chai)
+* objecter: minor cleanups (`pr#19994 <https://github.com/ceph/ceph/pull/19994>`_, runsisi)
+* os/bluestore/bluestore_tool: Move redundant code into one method (`pr#19160 <https://github.com/ceph/ceph/pull/19160>`_, Shinobu Kinjo)
+* os/bluestore: implement BlueRocksEnv::AreFilesSame() (`issue#21842 <http://tracker.ceph.com/issues/21842>`_, `pr#18392 <https://github.com/ceph/ceph/pull/18392>`_, Kefu Chai)
+* os/bluestore: simplify and fix SharedBlob::put() (`issue#24211 <http://tracker.ceph.com/issues/24211>`_, `pr#22170 <https://github.com/ceph/ceph/pull/22170>`_, Sage Weil)
+* osd: additional protection for out-of-bounds EC reads (`issue#21629 <http://tracker.ceph.com/issues/21629>`_, `pr#18088 <https://github.com/ceph/ceph/pull/18088>`_, Jason Dillaman)
+* osd: add multiple objecter finishers (`pr#16521 <https://github.com/ceph/ceph/pull/16521>`_, Myoungwon Oh)
+* osd: add num_object_manifest (`pr#20690 <https://github.com/ceph/ceph/pull/20690>`_, Myoungwon Oh)
+* osd: add numpg_removing metric (`pr#18450 <https://github.com/ceph/ceph/pull/18450>`_, Sage Weil)
+* osd: add processed_subop_count for cls_cxx_subop_version() (`issue#21964 <http://tracker.ceph.com/issues/21964>`_, `pr#18610 <https://github.com/ceph/ceph/pull/18610>`_, Casey Bodley)
+* osd: add scrub week day constraint (`pr#18368 <https://github.com/ceph/ceph/pull/18368>`_, kungf)
+* osd: adjust osd_min_pg_log_entries (`issue#21026 <http://tracker.ceph.com/issues/21026>`_, `pr#17075 <https://github.com/ceph/ceph/pull/17075>`_, J. Eric Ivancich)
+* osd: allow FULL_TRY after failsafe (`pr#17177 <https://github.com/ceph/ceph/pull/17177>`_, Pan Liu)
+* osd: allow PG recovery scheduling preemption (`pr#17839 <https://github.com/ceph/ceph/pull/17839>`_, Sage Weil)
+* osd: async recovery (`pr#19811 <https://github.com/ceph/ceph/pull/19811>`_, Neha Ojha)
+* osd: avoid encoding the same log entries repeatedly for different peers (`pr#20201 <https://github.com/ceph/ceph/pull/20201>`_, Jianpeng Ma)
+* osd: avoid the config's get_val() overhead on the read path (`pr#20217 <https://github.com/ceph/ceph/pull/20217>`_, Radoslaw Zarzynski)
+* osd: avoid unnecessary ref-counting across PrimaryLogPG::get_rw_locks (`pr#21028 <https://github.com/ceph/ceph/pull/21028>`_, Radoslaw Zarzynski)
+* osd: be more precise about our asserts and cases when rebuilding missing sets (`issue#20985 <http://tracker.ceph.com/issues/20985>`_, `pr#17000 <https://github.com/ceph/ceph/pull/17000>`_, Greg Farnum)
+* osd: bring in dmclock library changes (`pr#16755 <https://github.com/ceph/ceph/pull/16755>`_, J. Eric Ivancich)
+* osd: bring in latest dmclock library updates (`pr#17997 <https://github.com/ceph/ceph/pull/17997>`_, J. Eric Ivancich)
+* osd: cap snaptrimq_len at 2^32 (`pr#21107 <https://github.com/ceph/ceph/pull/21107>`_, Kefu Chai)
+* osd: change log level when withholding pg creation (`issue#22440 <http://tracker.ceph.com/issues/22440>`_, `pr#20167 <https://github.com/ceph/ceph/pull/20167>`_, Dan van der Ster)
+* osd: change op delayed state to 'waiting for scrub' (`pr#19295 <https://github.com/ceph/ceph/pull/19295>`_, kungf)
+* osd: Change shard digests to hex like object info digests (`pr#21362 <https://github.com/ceph/ceph/pull/21362>`_, David Zafman)
+* osd: change the conditional in _update_calc_stats (`pr#13383 <https://github.com/ceph/ceph/pull/13383>`_, Zhiqiang Wang)
+* osd: check feature bits when encoding objectstore_perf_stat_t (`pr#20378 <https://github.com/ceph/ceph/pull/20378>`_, Kefu Chai)
+* osd: clean up dup index logic; maintain index flag logic in fewer places (`pr#16829 <https://github.com/ceph/ceph/pull/16829>`_, J. Eric Ivancich)
+* osd: clean up pre-luminous compat cruft (part 1 of many) (`pr#17247 <https://github.com/ceph/ceph/pull/17247>`_, Sage Weil)
+* osd: cleanups (`pr#17753 <https://github.com/ceph/ceph/pull/17753>`_, Kefu Chai)
+* osdc/Objecter: using coarse_mono instead (`pr#18473 <https://github.com/ceph/ceph/pull/18473>`_, Haomai Wang)
+* osdc/Objector: use std::shared_mutex instead of boost::shared_mutex (`issue#23910 <http://tracker.ceph.com/issues/23910>`_, `pr#21702 <https://github.com/ceph/ceph/pull/21702>`_, Abhishek Lekshmanan)
+* osd: correct several spell errors in comments (`pr#21064 <https://github.com/ceph/ceph/pull/21064>`_, songweibin)
+* osdc: Remove a bit too redundant public label (`pr#19466 <https://github.com/ceph/ceph/pull/19466>`_, Shinobu Kinjo)
+* osdc: self-managed snapshot helper should catch decode exception (`issue#24103 <http://tracker.ceph.com/issues/24103>`_, `issue#24000 <http://tracker.ceph.com/issues/24000>`_, `pr#21958 <https://github.com/ceph/ceph/pull/21958>`_, Jason Dillaman)
+* osd: debug dispatch_context cases where queries not sent (`pr#20917 <https://github.com/ceph/ceph/pull/20917>`_, Sage Weil)
+* osd: Deleting dead code PrimaryLogPG.cc (`pr#17339 <https://github.com/ceph/ceph/pull/17339>`_, Amit Kumar)
+* osd: don't crash on empty snapset (`issue#23851 <http://tracker.ceph.com/issues/23851>`_, `pr#21058 <https://github.com/ceph/ceph/pull/21058>`_, Mykola Golub, Igor Fedotov)
+* osd: Don't include same header twice (`pr#18319 <https://github.com/ceph/ceph/pull/18319>`_, Shinobu Kinjo)
+* osd: Don't initialze pointers by NULL or 0 (`pr#18311 <https://github.com/ceph/ceph/pull/18311>`_, Shinobu Kinjo)
+* osd: don't memcpy hobject_t in PrimaryLogPG::close_op_ctx() (`pr#21029 <https://github.com/ceph/ceph/pull/21029>`_, Radoslaw Zarzynski)
+* osd: don't process ostream strings when not debugging (`pr#20298 <https://github.com/ceph/ceph/pull/20298>`_, Mark Nelson)
+* osd: drop redundant comment (`pr#20347 <https://github.com/ceph/ceph/pull/20347>`_, songweibin)
+* osd: Drop the unused code in OSD::_collect_metadata (`pr#17131 <https://github.com/ceph/ceph/pull/17131>`_, Luo Kexue)
+* osd: drop unused osd_disk_tp related options (`pr#21339 <https://github.com/ceph/ceph/pull/21339>`_, Gu Zhongyan)
+* osd: eliminate ineffective container operations (`pr#19099 <https://github.com/ceph/ceph/pull/19099>`_, Igor Fedotov)
+* osd: enumerate device names in a simple way (`pr#18453 <https://github.com/ceph/ceph/pull/18453>`_, Sage Weil)
+* osd: exit(1) directly without lock if init fails (`pr#16647 <https://github.com/ceph/ceph/pull/16647>`_, Kefu Chai)
+* osd: fast dispatch of peering events and pg_map + osd sharded wq refactor (`pr#19973 <https://github.com/ceph/ceph/pull/19973>`_, Sage Weil)
+* osd: fine-grained statistics of logical object space usage (`pr#15199 <https://github.com/ceph/ceph/pull/15199>`_, xie xingguo)
+* osd: Fix assert when checking missing version (`issue#21218 <http://tracker.ceph.com/issues/21218>`_, `pr#20410 <https://github.com/ceph/ceph/pull/20410>`_, David Zafman)
+* osd: fix a valgrind issue (conditional jump depends on uninitialized value) (`issue#22641 <http://tracker.ceph.com/issues/22641>`_, `pr#19874 <https://github.com/ceph/ceph/pull/19874>`_, Myoungwon Oh)
+* osd: fix bug which cause can't erase OSDShardPGSlot (`pr#21771 <https://github.com/ceph/ceph/pull/21771>`_, Jianpeng Ma)
+* osd: fix build_initial_pg_history (`issue#21203 <http://tracker.ceph.com/issues/21203>`_, `pr#17423 <https://github.com/ceph/ceph/pull/17423>`_, w11979, Sage Weil)
+* osd: fix crash caused by divide by zero in heartbeat code (`pr#21373 <https://github.com/ceph/ceph/pull/21373>`_, Piotr Dałek)
+* osd: fix dpdk memzon mz_name setting issue (`pr#19809 <https://github.com/ceph/ceph/pull/19809>`_, chunmei Liu)
+* osd: fix dpdk runtime issue based on spdk/dpdk libarary (`pr#19559 <https://github.com/ceph/ceph/pull/19559>`_, chunmei Liu)
+* osd: fix dpdk worker references issue (`pr#19886 <https://github.com/ceph/ceph/pull/19886>`_, chunmei Liu)
+* osd: Fixes for osd_scrub_during_recovery handling (`issue#18206 <http://tracker.ceph.com/issues/18206>`_, `pr#17039 <https://github.com/ceph/ceph/pull/17039>`_, David Zafman)
+* osd: fix out of order caused by letting old msg from down osd be processed (`issue#22570 <http://tracker.ceph.com/issues/22570>`_, `pr#19796 <https://github.com/ceph/ceph/pull/19796>`_, Mingxin Liu)
+* osd: fix _process handling for pg vs slot race (`pr#21745 <https://github.com/ceph/ceph/pull/21745>`_, Sage Weil)
+* osd: fix recovery reservation bugs, and implement remote reservation preemption (`pr#18485 <https://github.com/ceph/ceph/pull/18485>`_, Sage Weil)
+* osd: fix replica/backfill target handling of REJECT (`issue#21613 <http://tracker.ceph.com/issues/21613>`_, `pr#18070 <https://github.com/ceph/ceph/pull/18070>`_, Sage Weil)
+* osd: fix reqid assignment for reply messages in OpRequest (`pr#17060 <https://github.com/ceph/ceph/pull/17060>`_, Yingxin Cheng)
+* osd: fix s390x build failure (`issue#23238 <http://tracker.ceph.com/issues/23238>`_, `pr#20969 <https://github.com/ceph/ceph/pull/20969>`_, Nathan Cutler)
+* osd: fix typos and some cleanups (`pr#19211 <https://github.com/ceph/ceph/pull/19211>`_, Enming Zhang)
+* osd: fix unordered read bug (for chunked object) (`issue#22369 <http://tracker.ceph.com/issues/22369>`_, `pr#19464 <https://github.com/ceph/ceph/pull/19464>`_, Myoungwon Oh)
+* osd: fix waiting_for_peered vs flushing (`issue#21407 <http://tracker.ceph.com/issues/21407>`_, `pr#17759 <https://github.com/ceph/ceph/pull/17759>`_, Sage Weil)
+* osd: flush operations for chunked objects (`pr#19294 <https://github.com/ceph/ceph/pull/19294>`_, Myoungwon Oh)
+* osd: generalize queueing and lock interface for OpWq (`pr#16030 <https://github.com/ceph/ceph/pull/16030>`_, Myoungwon Oh, Kefu Chai, Samuel Just)
+* osd: get loadavg per cpu for scrub load threshold check (`pr#17718 <https://github.com/ceph/ceph/pull/17718>`_, kungf)
+* osd: get rid off extent map in object_info (`pr#19616 <https://github.com/ceph/ceph/pull/19616>`_, Igor Fedotov)
+* osd: hold lock while accessing recovery_needs_sleep (`issue#21566 <http://tracker.ceph.com/issues/21566>`_, `pr#18022 <https://github.com/ceph/ceph/pull/18022>`_, Neha Ojha)
+* osd: Improve recovery stat handling by using peer_missing and missing_loc info (`issue#22837 <http://tracker.ceph.com/issues/22837>`_, `pr#20220 <https://github.com/ceph/ceph/pull/20220>`_, Sage Weil, David Zafman)
+* osd: Improve size scrub error handling and ignore system attrs in xattr checking (`issue#20243 <http://tracker.ceph.com/issues/20243>`_, `issue#18836 <http://tracker.ceph.com/issues/18836>`_, `pr#16407 <https://github.com/ceph/ceph/pull/16407>`_, David Zafman)
+* osd: include front_iface+back_iface in metadata (`issue#20956 <http://tracker.ceph.com/issues/20956>`_, `pr#16941 <https://github.com/ceph/ceph/pull/16941>`_, John Spray)
+* osd: Initialization of data members (`pr#17691 <https://github.com/ceph/ceph/pull/17691>`_, Amit Kumar)
+* osd: Initialization of pointer cls (`pr#17115 <https://github.com/ceph/ceph/pull/17115>`_, amitkuma)
+* osd: Initializing start_offset,last_offset,offset (`pr#19333 <https://github.com/ceph/ceph/pull/19333>`_, Amit Kumar)
+* osd: initial minimal efforts to clean up PG interface (`pr#17708 <https://github.com/ceph/ceph/pull/17708>`_, Sage Weil)
+* osd: introduce sub-chunks to erasure code plugin interface (`issue#19278 <http://tracker.ceph.com/issues/19278>`_, `pr#15193 <https://github.com/ceph/ceph/pull/15193>`_, Myna Vajha)
+* osd: kill snapdirs (`pr#17579 <https://github.com/ceph/ceph/pull/17579>`_, Sage Weil)
+* osd: Make dmclock's anticipation timeout be configurable (`pr#18827 <https://github.com/ceph/ceph/pull/18827>`_, Taewoong Kim)
+* osd: make operations on ReplicatedBackend::in_progress_ops more effective (`pr#19035 <https://github.com/ceph/ceph/pull/19035>`_, Igor Fedotov)
+* osd: make PG::\*Force\* event structs public (`pr#21312 <https://github.com/ceph/ceph/pull/21312>`_, Willem Jan Withagen)
+* osd: make scrub no deadline when max interval is zero (`pr#18354 <https://github.com/ceph/ceph/pull/18354>`_, kungf)
+* osd: make scrub right now when pg stats_invalid is true (`pr#17884 <https://github.com/ceph/ceph/pull/17884>`_, kungf)
+* osd: make scrub wait for ec read/modify/writes to apply (`issue#23339 <http://tracker.ceph.com/issues/23339>`_, `pr#20944 <https://github.com/ceph/ceph/pull/20944>`_, Sage Weil)
+* osd: make snapmapper warn+clean up instead of assert (`issue#22752 <http://tracker.ceph.com/issues/22752>`_, `pr#20040 <https://github.com/ceph/ceph/pull/20040>`_, Sage Weil)
+* osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL (`issue#21981 <http://tracker.ceph.com/issues/21981>`_, `pr#18637 <https://github.com/ceph/ceph/pull/18637>`_, Yao Zongyou)
+* osd: make the PG's SORTBITWISE assert a more generous shutdown (`issue#20416 <http://tracker.ceph.com/issues/20416>`_, `pr#18047 <https://github.com/ceph/ceph/pull/18047>`_, Greg Farnum)
+* osd: Making use of find to reduce computational complexity (`pr#19732 <https://github.com/ceph/ceph/pull/19732>`_, Shinobu Kinjo)
+* osd: migrate PGLOG\_\* macros to constexpr (`issue#20811 <http://tracker.ceph.com/issues/20811>`_, `pr#19352 <https://github.com/ceph/ceph/pull/19352>`_, Jesse Williamson)
+* osd: minor optimizations for op wq (`pr#17704 <https://github.com/ceph/ceph/pull/17704>`_, Sage Weil, J. Eric Ivancich)
+* osd: min_pg_log_entries == max == pg_log_dups_tracked (`pr#20394 <https://github.com/ceph/ceph/pull/20394>`_, Sage Weil)
+* osd: misc cleanups (`pr#17430 <https://github.com/ceph/ceph/pull/17430>`_, songweibin)
+* osd: miscellaneous cleanups (`pr#21431 <https://github.com/ceph/ceph/pull/21431>`_, songweibin)
+* osd: more debugging for snapmapper bug (`issue#21557 <http://tracker.ceph.com/issues/21557>`_, `pr#19366 <https://github.com/ceph/ceph/pull/19366>`_, Sage Weil)
+* osd: object added to missing set for backfill, but is not in recovering, error! (`issue#18162 <http://tracker.ceph.com/issues/18162>`_, `pr#18145 <https://github.com/ceph/ceph/pull/18145>`_, David Zafman)
+* osd: only exit if \*latest\* map(s) say we are destroyed (`issue#22673 <http://tracker.ceph.com/issues/22673>`_, `pr#19988 <https://github.com/ceph/ceph/pull/19988>`_, Sage Weil)
+* osd: Only scan for omap corruption once (`issue#21328 <http://tracker.ceph.com/issues/21328>`_, `pr#17705 <https://github.com/ceph/ceph/pull/17705>`_, David Zafman)
+* osd,os,io: Initializing C_ProxyChunkRead members,queue,request (`pr#19336 <https://github.com/ceph/ceph/pull/19336>`_, amitkuma)
+* osd: pass ops_blocked_by_scrub() to requeue_scrub() (`pr#20319 <https://github.com/ceph/ceph/pull/20319>`_, Kefu Chai)
+* osd: pass pool options to ObjectStore on pg create (`issue#22419 <http://tracker.ceph.com/issues/22419>`_, `pr#19486 <https://github.com/ceph/ceph/pull/19486>`_, Sage Weil)
+* osd/PG: fix clang build vs private state events (`pr#18217 <https://github.com/ceph/ceph/pull/18217>`_, Sage Weil)
+* osd/PG: handle flushed event directly (`pr#19441 <https://github.com/ceph/ceph/pull/19441>`_, wumingqiao)
+* osd/PrimaryLogPG: derr when object size becomes over osd_max_object_size (`pr#19049 <https://github.com/ceph/ceph/pull/19049>`_, Shinobu Kinjo)
+* osd: process _scan_snaps() with all snapshots with head (`issue#22881 <http://tracker.ceph.com/issues/22881>`_, `issue#23909 <http://tracker.ceph.com/issues/23909>`_, `pr#21546 <https://github.com/ceph/ceph/pull/21546>`_, David Zafman)
+* osd: publish osdmap to OSDService before starting wq threads (`issue#21977 <http://tracker.ceph.com/issues/21977>`_, `pr#21623 <https://github.com/ceph/ceph/pull/21623>`_, Sage Weil)
+* osd: pull latest dmclock subtree (`pr#19345 <https://github.com/ceph/ceph/pull/19345>`_, J. Eric Ivancich)
+* osd: put peering events in main sharded wq (`pr#18752 <https://github.com/ceph/ceph/pull/18752>`_, Sage Weil)
+* osd: put pg removal in op_wq (`pr#19433 <https://github.com/ceph/ceph/pull/19433>`_, Sage Weil)
+* osd: reduce all_info map find to get primary (`pr#19425 <https://github.com/ceph/ceph/pull/19425>`_, kungf)
+* osd: refcount for manifest object (redirect, chunked) (`pr#19935 <https://github.com/ceph/ceph/pull/19935>`_, Myoungwon Oh)
+* osd: remove cost from mclock op queues; cost not handled well in dmclock (`pr#21428 <https://github.com/ceph/ceph/pull/21428>`_, J. Eric Ivancich)
+* osd: Remove double space (`pr#19296 <https://github.com/ceph/ceph/pull/19296>`_, Shinobu Kinjo)
+* osd: remove duplicated "commit_queued_for_journal_write" in OpTracker (`issue#23440 <http://tracker.ceph.com/issues/23440>`_, `pr#21018 <https://github.com/ceph/ceph/pull/21018>`_, ashitakasam)
+* osd: remove duplicated function ec_pool in pg_pool_t (`pr#18059 <https://github.com/ceph/ceph/pull/18059>`_, Chang Liu)
+* osd: Remove redundant local variable declaration (`pr#19812 <https://github.com/ceph/ceph/pull/19812>`_, Shinobu Kinjo)
+* osd: Remove unnecessary headers (`pr#19735 <https://github.com/ceph/ceph/pull/19735>`_, Shinobu Kinjo)
+* osd: remove unused ReplicatedBackend::objects_read_async() (`pr#18779 <https://github.com/ceph/ceph/pull/18779>`_, Kefu Chai)
+* osd: remove unused variable in do_proxy_write (`pr#17391 <https://github.com/ceph/ceph/pull/17391>`_, Myoungwon Oh)
+* osd: replace mclock subop opclass w/ rep_op opclass; combine duplicated code (`pr#18194 <https://github.com/ceph/ceph/pull/18194>`_, J. Eric Ivancich)
+* osd: replace vectors_equal() with operator==(vector<>, vector<>) (`pr#18064 <https://github.com/ceph/ceph/pull/18064>`_, Kefu Chai)
+* osd: request new map from PG when needed (`issue#21428 <http://tracker.ceph.com/issues/21428>`_, `pr#17795 <https://github.com/ceph/ceph/pull/17795>`_, Josh Durgin)
+* osd: resend osd_pgtemp if it's not acked (`issue#23610 <http://tracker.ceph.com/issues/23610>`_, `pr#21310 <https://github.com/ceph/ceph/pull/21310>`_, Kefu Chai)
+* osd: Revert use of dmclock message feature bit since not yet finalized (`pr#21398 <https://github.com/ceph/ceph/pull/21398>`_, J. Eric Ivancich)
+* osd,rgw,librbd: SCA fixes (`pr#18495 <https://github.com/ceph/ceph/pull/18495>`_, Danny Al-Gaaf)
+* osd: set min_version to newest version in maybe_force_recovery (`pr#17752 <https://github.com/ceph/ceph/pull/17752>`_, Xinze Chi)
+* osd: Sign in early SIGHUP signal (`issue#22746 <http://tracker.ceph.com/issues/22746>`_, `pr#19958 <https://github.com/ceph/ceph/pull/19958>`_, huanwen ren)
+* osd: silence maybe-uninitialized false positives (`pr#19820 <https://github.com/ceph/ceph/pull/19820>`_, Yao Zongyou)
+* osd: silence warnings from -Wsign-compare (`pr#17872 <https://github.com/ceph/ceph/pull/17872>`_, Jos Collin)
+* osd: skip dumping logical devices (`pr#20740 <https://github.com/ceph/ceph/pull/20740>`_, songweibin)
+* osd: speed up get_key_name (`issue#21026 <http://tracker.ceph.com/issues/21026>`_, `pr#17071 <https://github.com/ceph/ceph/pull/17071>`_, J. Eric Ivancich)
+* osd: s/random_shuffle()/shuffle()/ (`pr#19872 <https://github.com/ceph/ceph/pull/19872>`_, Willem Jan Withagen, Kefu Chai, Greg Farnum)
+* osd: subscribe osdmaps if any pending pgs (`issue#22113 <http://tracker.ceph.com/issues/22113>`_, `pr#18916 <https://github.com/ceph/ceph/pull/18916>`_, Kefu Chai)
+* osd: subscribe to new osdmap while waiting_for_healthy (`issue#21121 <http://tracker.ceph.com/issues/21121>`_, `pr#17244 <https://github.com/ceph/ceph/pull/17244>`_, Sage Weil)
+* osd: support class method whitelisting within caps (`pr#19786 <https://github.com/ceph/ceph/pull/19786>`_, Jason Dillaman)
+* osd: treat successful and erroroneous writes the same for log trimming (`issue#22050 <http://tracker.ceph.com/issues/22050>`_, `pr#20827 <https://github.com/ceph/ceph/pull/20827>`_, Josh Durgin)
+* osd: two cleanups (`pr#20830 <https://github.com/ceph/ceph/pull/20830>`_, songweibin)
+* osd: update dmclock library w git subtree pull (`pr#17737 <https://github.com/ceph/ceph/pull/17737>`_, J. Eric Ivancich)
+* osd: update info only if new_interval (`pr#17437 <https://github.com/ceph/ceph/pull/17437>`_, Kefu Chai)
+* osd: update store with options after pg is created (`issue#22419 <http://tracker.ceph.com/issues/22419>`_, `pr#20044 <https://github.com/ceph/ceph/pull/20044>`_, Kefu Chai)
+* osd: use dmclock library client_info_f function dynamically (`pr#17063 <https://github.com/ceph/ceph/pull/17063>`_, bspark)
+* osd: use existing osd_required variable for messenger policy (`pr#20223 <https://github.com/ceph/ceph/pull/20223>`_, Yan Jun)
+* osd: use prefix increment for non trivial iterator (`pr#19097 <https://github.com/ceph/ceph/pull/19097>`_, Kefu Chai)
+* osd: Use specializations, typedefs instead (`pr#19354 <https://github.com/ceph/ceph/pull/19354>`_, Shinobu Kinjo)
+* osd: Warn about objects with too many omap entries (`pr#16332 <https://github.com/ceph/ceph/pull/16332>`_, Brad Hubbard)
+* os/filestore/HashIndex.h: fixed a typo in comment (`pr#17685 <https://github.com/ceph/ceph/pull/17685>`_, yaoguotao)
+* os: Initializing uninitialized members aio_info (`pr#17066 <https://github.com/ceph/ceph/pull/17066>`_, amitkuma)
+* os: Removing dead code from LFNIndex.cc (`pr#17297 <https://github.com/ceph/ceph/pull/17297>`_, Amit Kumar)
+* prometheus: Handle the TIME perf counter type metrics (`pr#21749 <https://github.com/ceph/ceph/pull/21749>`_, Boris Ranto)
+* pybind: add return note in rbd.pyx (`pr#21768 <https://github.com/ceph/ceph/pull/21768>`_, Zheng Yin)
+* pybind/ceph_daemon: expand the order of magnitude of (`issue#23962 <http://tracker.ceph.com/issues/23962>`_, `pr#21836 <https://github.com/ceph/ceph/pull/21836>`_, Guan yunfei)
+* pybind: fix chart size become bigger when refresh (`issue#20746 <http://tracker.ceph.com/issues/20746>`_, `pr#16857 <https://github.com/ceph/ceph/pull/16857>`_, Yixing Yan)
+* pybind: mgr/dashboard: fix rbd's pool sub menu (`pr#16774 <https://github.com/ceph/ceph/pull/16774>`_, yanyx)
+* pybind,rbd: pybind/rbd: support open the image by image_id (`pr#19361 <https://github.com/ceph/ceph/pull/19361>`_, songweibin)
+* pybind: remove unused get_ceph_version() (`pr#17727 <https://github.com/ceph/ceph/pull/17727>`_, Kefu Chai)
+* qa: add cbt repo parameter (`pr#18543 <https://github.com/ceph/ceph/pull/18543>`_, Neha Ojha)
+* qa: Add cephmetrics suite (`pr#18451 <https://github.com/ceph/ceph/pull/18451>`_, Zack Cerza)
+* qa: add upgrade/luminous-x suite (`pr#17160 <https://github.com/ceph/ceph/pull/17160>`_, Yuri Weinstein)
+* qa/crontab: run the perf-basic suite every day (`pr#21252 <https://github.com/ceph/ceph/pull/21252>`_, Neha Ojha)
+* qa: Decreased amount of jobs on master, kraken, luminous runs (`pr#17069 <https://github.com/ceph/ceph/pull/17069>`_, Yuri Weinstein)
+* qa: install collectl with cbt task (`pr#19324 <https://github.com/ceph/ceph/pull/19324>`_, Neha Ojha)
+* qa: mimic-dev1 backports to avoid trusty nodes (`pr#19600 <https://github.com/ceph/ceph/pull/19600>`_, Kefu Chai)
+* qa: preserve cbt task results (`pr#19364 <https://github.com/ceph/ceph/pull/19364>`_, Neha Ojha)
+* qa:  qa/ceph-disk: enlarge the simulated SCSI disk (`issue#22136 <http://tracker.ceph.com/issues/22136>`_, `pr#19199 <https://github.com/ceph/ceph/pull/19199>`_, Kefu Chai)
+* qa/suites/perf-basic: add desc regarding test machines (`pr#21183 <https://github.com/ceph/ceph/pull/21183>`_, Neha Ojha)
+* qa/suites/rados/multimon/tasks/mon_lock_with_skew: whitelist PG (`pr#17004 <https://github.com/ceph/ceph/pull/17004>`_, Sage Weil)
+* qa/suites/rados/perf: add optimized settings (`pr#17786 <https://github.com/ceph/ceph/pull/17786>`_, Neha Ojha)
+* qa/suites/rados/perf: add workloads (`pr#18573 <https://github.com/ceph/ceph/pull/18573>`_, Neha Ojha)
+* qa/suites/rados/verify/validater/valgrind: whitelist PG (`pr#17005 <https://github.com/ceph/ceph/pull/17005>`_, Sage Weil)
+* qa/suites/upgrade/jewel-x/parallel: tolerate laggy mgr (`pr#17227 <https://github.com/ceph/ceph/pull/17227>`_, Sage Weil)
+* qa/suites/upgrade/kraken-x: fixes (`pr#16881 <https://github.com/ceph/ceph/pull/16881>`_, Sage Weil)
+* qa/suites/upgrade/luminous-x fixes (`pr#22101 <https://github.com/ceph/ceph/pull/22101>`_, Sage Weil)
+* qa/tests - Added options to use both cases: mon.a and installer.0 (`pr#19745 <https://github.com/ceph/ceph/pull/19745>`_, Yuri Weinstein)
+* qa/tests - Fixed typo in crontab entry (`pr#21482 <https://github.com/ceph/ceph/pull/21482>`_, Yuri Weinstein)
+* qa/tests: fixed typo (`pr#21728 <https://github.com/ceph/ceph/pull/21728>`_, Yuri Weinstein)
+* qa/tests - minor clean ups and made perf-suite run 3 times, so we can… (`pr#21309 <https://github.com/ceph/ceph/pull/21309>`_, Yuri Weinstein)
+* qa/tests - one more typo fixed :( (`pr#21483 <https://github.com/ceph/ceph/pull/21483>`_, Yuri Weinstein)
+* qa/tests: removed rest suite from the mix (`pr#21743 <https://github.com/ceph/ceph/pull/21743>`_, Yuri Weinstein)
+* qa: wait longer for osd to flush pg stats (`issue#24321 <http://tracker.ceph.com/issues/24321>`_, `pr#22288 <https://github.com/ceph/ceph/pull/22288>`_, Kefu Chai)
+* qa/workunits/ceph-disk: --no-mon-config (`pr#21956 <https://github.com/ceph/ceph/pull/21956>`_, Kefu Chai)
+* rados: make ceph_perf_msgr_client work for multiple jobs (`issue#22103 <http://tracker.ceph.com/issues/22103>`_, `pr#18877 <https://github.com/ceph/ceph/pull/18877>`_, Jeegn Chen)
+* rbd: add deep cp CLI method (`pr#19996 <https://github.com/ceph/ceph/pull/19996>`_, songweibin)
+* rbd: add group rename methods (`issue#22981 <http://tracker.ceph.com/issues/22981>`_, `pr#20577 <https://github.com/ceph/ceph/pull/20577>`_, songweibin)
+* rbd: add notrim option to rbd map (`pr#21056 <https://github.com/ceph/ceph/pull/21056>`_, Hitoshi Kamei)
+* rbd: add parent info when moving child into trash bin (`pr#19280 <https://github.com/ceph/ceph/pull/19280>`_, songweibin)
+* rbd: adjusted "lock list" JSON and XML formatted output (`pr#19900 <https://github.com/ceph/ceph/pull/19900>`_, Jason Dillaman)
+* rbd: adjusted "showmapped" JSON and XML formatted output (`pr#19937 <https://github.com/ceph/ceph/pull/19937>`_, Mykola Golub)
+* rbd: allow remove all unprotected snapshots (`issue#23126 <http://tracker.ceph.com/issues/23126>`_, `pr#20608 <https://github.com/ceph/ceph/pull/20608>`_, songweibin)
+* rbd: allow trash rm/purge when pool quota is full used (`pr#20697 <https://github.com/ceph/ceph/pull/20697>`_, songweibin)
+* rbd: backport of mimic bug fixes (`issue#24009 <http://tracker.ceph.com/issues/24009>`_, `issue#24008 <http://tracker.ceph.com/issues/24008>`_, `pr#21930 <https://github.com/ceph/ceph/pull/21930>`_, Jason Dillaman)
+* rbd: check if an image is already mapped before rbd map (`issue#20580 <http://tracker.ceph.com/issues/20580>`_, `pr#16517 <https://github.com/ceph/ceph/pull/16517>`_, Jing Li)
+* rbd: children list should support snapshot id optional (`issue#23399 <http://tracker.ceph.com/issues/23399>`_, `pr#20966 <https://github.com/ceph/ceph/pull/20966>`_, Jason Dillaman)
+* rbd: cleanup handling of IEC byte units (`pr#21564 <https://github.com/ceph/ceph/pull/21564>`_, Jason Dillaman)
+* rbd: clean up warnings when mirror commands used on non-setup pool (`issue#21319 <http://tracker.ceph.com/issues/21319>`_, `pr#17636 <https://github.com/ceph/ceph/pull/17636>`_, Jason Dillaman)
+* rbd: cls/journal: ensure tags are properly expired (`issue#21960 <http://tracker.ceph.com/issues/21960>`_, `pr#18604 <https://github.com/ceph/ceph/pull/18604>`_, Jason Dillaman)
+* rbd: cls/journal: fixed possible infinite loop in expire_tags (`issue#21956 <http://tracker.ceph.com/issues/21956>`_, `pr#18592 <https://github.com/ceph/ceph/pull/18592>`_, Jason Dillaman)
+* rbd: cls/journal: possible infinite loop within tag_list class method (`issue#21771 <http://tracker.ceph.com/issues/21771>`_, `pr#18270 <https://github.com/ceph/ceph/pull/18270>`_, Jason Dillaman)
+* rbd: cls/rbd: group_image_list incorrectly flagged as RW (`issue#23388 <http://tracker.ceph.com/issues/23388>`_, `pr#20939 <https://github.com/ceph/ceph/pull/20939>`_, Jason Dillaman)
+* rbd: cls/rbd: metadata_list not honoring max_return parameter (`issue#21247 <http://tracker.ceph.com/issues/21247>`_, `pr#17499 <https://github.com/ceph/ceph/pull/17499>`_, Jason Dillaman)
+* rbd: cls/rbd: Silence gcc7 maybe-uninitialized warning (`pr#18504 <https://github.com/ceph/ceph/pull/18504>`_, Brad Hubbard)
+* rbd: common/options,librbd/Utils: refactor RBD feature validation (`pr#20014 <https://github.com/ceph/ceph/pull/20014>`_, Sage Weil)
+* rbd: disk usage on empty pool no longer returns an error message (`issue#22200 <http://tracker.ceph.com/issues/22200>`_, `pr#19045 <https://github.com/ceph/ceph/pull/19045>`_, Jason Dillaman)
+* rbd: do not show title if there is no group snapshot (`pr#20311 <https://github.com/ceph/ceph/pull/20311>`_, songweibin)
+* rbd: don't overwrite the error code from the remove action (`pr#20481 <https://github.com/ceph/ceph/pull/20481>`_, Jason Dillaman)
+* rbd: drop unnecessary using declaration, etc (`pr#19005 <https://github.com/ceph/ceph/pull/19005>`_, Shinobu Kinjo)
+* rbd: eager-thick provisioning support (`pr#18317 <https://github.com/ceph/ceph/pull/18317>`_, Hitoshi Kamei)
+* rbd: export/import image-meta when we export/import an image (`pr#17134 <https://github.com/ceph/ceph/pull/17134>`_, PCzhangPC)
+* rbd: filter out UserSnapshotNamespace in do_disk_usage (`pr#20532 <https://github.com/ceph/ceph/pull/20532>`_, songweibin)
+* rbd: fix crash during map when "rw" option is specified (`issue#21808 <http://tracker.ceph.com/issues/21808>`_, `pr#18313 <https://github.com/ceph/ceph/pull/18313>`_, Peter Keresztes Schmidt)
+* rbd: fix logically dead code in function list_process_image (`pr#16971 <https://github.com/ceph/ceph/pull/16971>`_, Luo Kexue)
+* rbd: fix rbd children listing when child is in trash (`issue#21893 <http://tracker.ceph.com/issues/21893>`_, `pr#18483 <https://github.com/ceph/ceph/pull/18483>`_, songweibin)
+* rbd: fix thread_offsets calculation of rbd bench (`pr#20590 <https://github.com/ceph/ceph/pull/20590>`_, Hitoshi Kamei)
+* rbd: group misc cleanup and update rbd man page (`pr#20199 <https://github.com/ceph/ceph/pull/20199>`_, songweibin)
+* rbd: group snapshot rename (`pr#12431 <https://github.com/ceph/ceph/pull/12431>`_, Victor Denisov)
+* rbd: implement image qos in tokenbucket algorithm (`pr#17032 <https://github.com/ceph/ceph/pull/17032>`_, Dongsheng Yang)
+* rbd: import with option --export-format 2 fails to protect snapshot (`issue#23038 <http://tracker.ceph.com/issues/23038>`_, `pr#20613 <https://github.com/ceph/ceph/pull/20613>`_, songweibin)
+* rbd: improve 'import-diff' corrupt input error messages (`issue#18844 <http://tracker.ceph.com/issues/18844>`_, `pr#21249 <https://github.com/ceph/ceph/pull/21249>`_, Jason Dillaman)
+* rbd: Initializing m_finalize_ctx (`pr#17563 <https://github.com/ceph/ceph/pull/17563>`_, Amit Kumar)
+* rbd: introduce commands of "image-meta ls/rm" (`pr#16591 <https://github.com/ceph/ceph/pull/16591>`_, PCzhangPC)
+* rbd: journal: limit number of appends sent in one librados op (`issue#23526 <http://tracker.ceph.com/issues/23526>`_, `pr#21157 <https://github.com/ceph/ceph/pull/21157>`_, Mykola Golub)
+* rbd: journal: trivial cleanup (`pr#19317 <https://github.com/ceph/ceph/pull/19317>`_, Shinobu Kinjo)
+* rbd: krbd: include sys/sysmacros.h for major, minor and makedev (`pr#20773 <https://github.com/ceph/ceph/pull/20773>`_, Ilya Dryomov)
+* rbd: krbd: rewrite "already mapped" code (`pr#17638 <https://github.com/ceph/ceph/pull/17638>`_, Ilya Dryomov)
+* rbd: librados/snap_set_diff: don't assert on empty snapset (`pr#20648 <https://github.com/ceph/ceph/pull/20648>`_, Mykola Golub)
+* rbd:  librbd: create+truncate for whole-object layered discards (`issue#23285 <http://tracker.ceph.com/issues/23285>`_, `pr#20809 <https://github.com/ceph/ceph/pull/20809>`_, Ilya Dryomov)
+* rbd:  librbd: make rename request complete with filtered code (`issue#23068 <http://tracker.ceph.com/issues/23068>`_, `pr#20507 <https://github.com/ceph/ceph/pull/20507>`_, Mykola Golub)
+* rbd: librbd misc cleanup (`pr#18419 <https://github.com/ceph/ceph/pull/18419>`_, Jianpeng Ma)
+* rbd:  librbd: skip head object map update when deep copying object beyond image size (`pr#21586 <https://github.com/ceph/ceph/pull/21586>`_, Mykola Golub)
+* rbd:  librbd: sync flush should re-use existing async flush logic (`pr#18403 <https://github.com/ceph/ceph/pull/18403>`_, Jason Dillaman)
+* rbd: librbd,test: address coverity false positives (`pr#17825 <https://github.com/ceph/ceph/pull/17825>`_, Amit Kumar)
+* rbd: mimic: librbd: deep copy optionally support flattening cloned image (`issue#22787 <http://tracker.ceph.com/issues/22787>`_, `pr#22038 <https://github.com/ceph/ceph/pull/22038>`_, Mykola Golub)
+* rbd: mimic: rbd-mirror: optionally support active/active replication (`pr#22105 <https://github.com/ceph/ceph/pull/22105>`_, Jason Dillaman)
+* rbd: mimic: rbd-mirror: potential deadlock when running asok 'flush' command (`issue#24141 <http://tracker.ceph.com/issues/24141>`_, `pr#22039 <https://github.com/ceph/ceph/pull/22039>`_, Mykola Golub)
+* rbd-mirror: additional thrasher testing (`pr#21697 <https://github.com/ceph/ceph/pull/21697>`_, Jason Dillaman)
+* rbd-mirror: clean up spurious error log messages (`issue#21961 <http://tracker.ceph.com/issues/21961>`_, `pr#18601 <https://github.com/ceph/ceph/pull/18601>`_, Jason Dillaman)
+* rbd-mirror: cluster watcher should ensure it has latest OSD map (`issue#22461 <http://tracker.ceph.com/issues/22461>`_, `pr#19550 <https://github.com/ceph/ceph/pull/19550>`_, Jason Dillaman)
+* rbd-mirror: ensure unique service daemon name is utilized (`pr#19492 <https://github.com/ceph/ceph/pull/19492>`_, Jason Dillaman)
+* rbd-mirror: fix potential infinite loop when formatting status message (`issue#22932 <http://tracker.ceph.com/issues/22932>`_, `pr#20349 <https://github.com/ceph/ceph/pull/20349>`_, Mykola Golub)
+* rbd-mirror: forced promotion can result in incorrect status (`issue#21559 <http://tracker.ceph.com/issues/21559>`_, `pr#17979 <https://github.com/ceph/ceph/pull/17979>`_, Jason Dillaman)
+* rbd-mirror: ImageMap memory leak fixes (`pr#19163 <https://github.com/ceph/ceph/pull/19163>`_, Venky Shankar)
+* rbd-mirror: Improve data pool selection when creating images (`pr#18006 <https://github.com/ceph/ceph/pull/18006>`_, Adam Wolfe Gordon)
+* rbd-mirror: integrate image map policy as incremental step to active-active (`pr#21300 <https://github.com/ceph/ceph/pull/21300>`_, Jason Dillaman)
+* rbd-mirror: introduce basic image mapping policy (`issue#18786 <http://tracker.ceph.com/issues/18786>`_, `pr#15691 <https://github.com/ceph/ceph/pull/15691>`_, Venky Shankar)
+* rbd-mirror: missing lock when re-sending update_sync_point (`pr#19011 <https://github.com/ceph/ceph/pull/19011>`_, Mykola Golub)
+* rbd-mirror: persist image map timestamp (`pr#19338 <https://github.com/ceph/ceph/pull/19338>`_, Venky Shankar)
+* rbd-mirror: primary image should register in remote, non-primary image's journal (`issue#21561 <http://tracker.ceph.com/issues/21561>`_, `pr#18136 <https://github.com/ceph/ceph/pull/18136>`_, Jason Dillaman)
+* rbd-mirror: properly translate remote tag mirror uuid for local mirror (`issue#23876 <http://tracker.ceph.com/issues/23876>`_, `pr#21657 <https://github.com/ceph/ceph/pull/21657>`_, Jason Dillaman)
+* rbd-mirror: removed dedicated thread from image deleter (`issue#15322 <http://tracker.ceph.com/issues/15322>`_, `pr#19000 <https://github.com/ceph/ceph/pull/19000>`_, Jason Dillaman)
+* rbd-mirror: rename asok hook to match image name when not replaying (`issue#23888 <http://tracker.ceph.com/issues/23888>`_, `pr#21682 <https://github.com/ceph/ceph/pull/21682>`_, Jason Dillaman)
+* rbd-mirror: rollback state transitions in image policy (`pr#19577 <https://github.com/ceph/ceph/pull/19577>`_, Venky Shankar)
+* rbd-mirror: Set the data pool correctly when creating images (`issue#20567 <http://tracker.ceph.com/issues/20567>`_, `pr#17073 <https://github.com/ceph/ceph/pull/17073>`_, Adam Wolfe Gordon)
+* rbd-mirror: simplify notifications for image assignment (`issue#15764 <http://tracker.ceph.com/issues/15764>`_, `pr#16642 <https://github.com/ceph/ceph/pull/16642>`_, Jason Dillaman)
+* rbd-mirror: strip environment/CLI overrides for remote cluster (`issue#21894 <http://tracker.ceph.com/issues/21894>`_, `pr#18490 <https://github.com/ceph/ceph/pull/18490>`_, Jason Dillaman)
+* rbd-mirror: support deferred deletions of mirrored images (`pr#19536 <https://github.com/ceph/ceph/pull/19536>`_, Jason Dillaman)
+* rbd-mirror: sync image metadata when transfering remote image (`issue#21535 <http://tracker.ceph.com/issues/21535>`_, `pr#18026 <https://github.com/ceph/ceph/pull/18026>`_, Jason Dillaman)
+* rbd-mirror: track images in policy map in support of A/A (`issue#18786 <http://tracker.ceph.com/issues/18786>`_, `pr#15788 <https://github.com/ceph/ceph/pull/15788>`_, Venky Shankar)
+* rbd-mirror: update asok hook name on image rename (`issue#20860 <http://tracker.ceph.com/issues/20860>`_, `pr#16998 <https://github.com/ceph/ceph/pull/16998>`_, Mykola Golub)
+* rbd-mirror: use next transition state to check transition completeness (`pr#18969 <https://github.com/ceph/ceph/pull/18969>`_, Venky Shankar)
+* rbd-nbd: allow to unmap by image or snap spec (`pr#19666 <https://github.com/ceph/ceph/pull/19666>`_, Mykola Golub)
+* rbd-nbd: bug fix when running in container (`issue#22012 <http://tracker.ceph.com/issues/22012>`_, `issue#22011 <http://tracker.ceph.com/issues/22011>`_, `pr#18663 <https://github.com/ceph/ceph/pull/18663>`_, Li Wang)
+* rbd-nbd: certain kernels may not discover resized block devices (`issue#22131 <http://tracker.ceph.com/issues/22131>`_, `pr#18947 <https://github.com/ceph/ceph/pull/18947>`_, Jason Dillaman)
+* rbd-nbd: cleanup for NBDServer shut down (`pr#17283 <https://github.com/ceph/ceph/pull/17283>`_, Pan Liu)
+* rbd-nbd: fix ebusy when do map (`issue#23528 <http://tracker.ceph.com/issues/23528>`_, `pr#21142 <https://github.com/ceph/ceph/pull/21142>`_, Li Wang)
+* rbd-nbd: fix generic option issue (`issue#20426 <http://tracker.ceph.com/issues/20426>`_, `pr#17375 <https://github.com/ceph/ceph/pull/17375>`_, Pan Liu)
+* rbd-nbd: output format support for list-mapped command (`pr#19704 <https://github.com/ceph/ceph/pull/19704>`_, Mykola Golub)
+* rbd-nbd: support optionally setting device timeout (`issue#22333 <http://tracker.ceph.com/issues/22333>`_, `pr#19436 <https://github.com/ceph/ceph/pull/19436>`_, Mykola Golub)
+* rbd: null check before pool_name use (`pr#18790 <https://github.com/ceph/ceph/pull/18790>`_, Amit Kumar)
+* rbd: output notifyOp request name when watching (`pr#20551 <https://github.com/ceph/ceph/pull/20551>`_, shun-s)
+* rbd: parallelize "rbd ls -l" (`pr#15579 <https://github.com/ceph/ceph/pull/15579>`_, Piotr Dałek)
+* rbd: pool_percent_used should not divided by 100 (`pr#20795 <https://github.com/ceph/ceph/pull/20795>`_, songweibin)
+* rbd: properly pass ceph global command line args to subprocess (`pr#19821 <https://github.com/ceph/ceph/pull/19821>`_, Mykola Golub)
+* rbd: pybind/rbd: add deep_copy method (`pr#19406 <https://github.com/ceph/ceph/pull/19406>`_, Mykola Golub)
+* rbd: pybind/rbd: fix metadata functions error handling (`issue#22306 <http://tracker.ceph.com/issues/22306>`_, `pr#19337 <https://github.com/ceph/ceph/pull/19337>`_, Mykola Golub)
+* rbd: python bindings fixes and improvements (`issue#23609 <http://tracker.ceph.com/issues/23609>`_, `pr#21304 <https://github.com/ceph/ceph/pull/21304>`_, Ricardo Dias)
+* rbd: rbd-ggate: fix parsing ceph global options (`pr#19822 <https://github.com/ceph/ceph/pull/19822>`_, Mykola Golub)
+* rbd: rbd-ggate: fix syntax error (`pr#19919 <https://github.com/ceph/ceph/pull/19919>`_, Willem Jan Withagen)
+* rbd: rbd-ggate: make list command produce valid xml format output (`pr#19823 <https://github.com/ceph/ceph/pull/19823>`_, Mykola Golub)
+* rbd: rbd-ggate: small fixes and improvements (`pr#19679 <https://github.com/ceph/ceph/pull/19679>`_, Mykola Golub)
+* rbd: rbd-ggate: tool to map images on FreeBSD via GEOM Gate (`pr#15339 <https://github.com/ceph/ceph/pull/15339>`_, Mykola Golub)
+* rbd:  rbd:introduce rbd bench rw(for read and write mix) test (`pr#17461 <https://github.com/ceph/ceph/pull/17461>`_, PCzhangPC)
+* rbd:  rbd: set a default value for options in `nbd map` (`pr#20529 <https://github.com/ceph/ceph/pull/20529>`_, songweibin)
+* rbd: replace positional_path parameter with arg_index in get_path() (`pr#19722 <https://github.com/ceph/ceph/pull/19722>`_, songweibin)
+* rbd: replace trash delay option, add rbd trash purge command (`pr#18323 <https://github.com/ceph/ceph/pull/18323>`_, Theofilos Mouratidis)
+* rbd: resource agent needs to be executable (`issue#22980 <http://tracker.ceph.com/issues/22980>`_, `issue#22362 <http://tracker.ceph.com/issues/22362>`_, `pr#20397 <https://github.com/ceph/ceph/pull/20397>`_, Tim Bishop)
+* rbd:rm unnecessary conversion from string to char\* in image-meta function (`pr#17184 <https://github.com/ceph/ceph/pull/17184>`_, PCzhangPC)
+* rbd: show read:write proportion in the infomation of readwrite bench test (`pr#18249 <https://github.com/ceph/ceph/pull/18249>`_, PCzhangPC)
+* rbd: snap limit should't be set smaller than the number of  existing snaps (`pr#16597 <https://github.com/ceph/ceph/pull/16597>`_, PCzhangPC)
+* rbd: support cloning an image from a non-primary snapshot (`issue#18480 <http://tracker.ceph.com/issues/18480>`_, `pr#19724 <https://github.com/ceph/ceph/pull/19724>`_, Jason Dillaman)
+* rbd: support iterating over metadata items when listing (`issue#21179 <http://tracker.ceph.com/issues/21179>`_, `pr#17532 <https://github.com/ceph/ceph/pull/17532>`_, Jason Dillaman)
+* rbd: support lock_timeout in rbd mapping (`pr#21344 <https://github.com/ceph/ceph/pull/21344>`_, Dongsheng Yang)
+* rbd: support osd_request_timeout in rbd map command (`issue#23073 <http://tracker.ceph.com/issues/23073>`_, `pr#20792 <https://github.com/ceph/ceph/pull/20792>`_, Dongsheng Yang)
+* rbd: switched from legacy to new-style configuration options (`issue#20737 <http://tracker.ceph.com/issues/20737>`_, `pr#16737 <https://github.com/ceph/ceph/pull/16737>`_, Jason Dillaman)
+* rbd,tests: qa: additional krbd discard test cases (`pr#20499 <https://github.com/ceph/ceph/pull/20499>`_, Ilya Dryomov)
+* rbd,tests: qa: fix POOL_APP_NOT_ENABLED warning in krbd:unmap suite (`pr#16966 <https://github.com/ceph/ceph/pull/16966>`_, Ilya Dryomov)
+* rbd,tests: qa: introduce rbd-mirror thrasher to existing tests (`issue#18753 <http://tracker.ceph.com/issues/18753>`_, `pr#21541 <https://github.com/ceph/ceph/pull/21541>`_, Jason Dillaman)
+* rbd,tests:  qa: krbd_exclusive_option.sh: add lock_timeout test case (`pr#21522 <https://github.com/ceph/ceph/pull/21522>`_, Ilya Dryomov)
+* rbd,tests:  qa: krbd_fallocate.sh: add notrim test case (`pr#21513 <https://github.com/ceph/ceph/pull/21513>`_, Ilya Dryomov)
+* rbd,tests:  qa: krbd huge-image test (`pr#20692 <https://github.com/ceph/ceph/pull/20692>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd latest-osdmap-on-map test (`pr#20591 <https://github.com/ceph/ceph/pull/20591>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd msgr-segments test (`pr#20714 <https://github.com/ceph/ceph/pull/20714>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd parent-overlap test (`pr#20721 <https://github.com/ceph/ceph/pull/20721>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd whole-object-discard test (`pr#20750 <https://github.com/ceph/ceph/pull/20750>`_, Ilya Dryomov)
+* rbd,tests:  qa/suites/krbd: add krbd BLKROSET test (`pr#18652 <https://github.com/ceph/ceph/pull/18652>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/krbd: enable generic/050 and generic/448 (`pr#18795 <https://github.com/ceph/ceph/pull/18795>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/krbd: enable xfstests blockdev tests (`pr#17621 <https://github.com/ceph/ceph/pull/17621>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/krbd: exclude shared/298 (`pr#17971 <https://github.com/ceph/ceph/pull/17971>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/krbd: rbd_xfstests job overhaul (`pr#17346 <https://github.com/ceph/ceph/pull/17346>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/rbd: fewer socket failures (`pr#19617 <https://github.com/ceph/ceph/pull/19617>`_, Sage Weil)
+* rbd,tests: qa/suites/rbd: miscellaneous test fixes (`issue#21251 <http://tracker.ceph.com/issues/21251>`_, `pr#17504 <https://github.com/ceph/ceph/pull/17504>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: segregated v1 image format tests (`issue#22738 <http://tracker.ceph.com/issues/22738>`_, `pr#20729 <https://github.com/ceph/ceph/pull/20729>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: set qemu task time_wait param (`pr#21131 <https://github.com/ceph/ceph/pull/21131>`_, Mykola Golub)
+* rbd,tests: qa/tasks/cram: include /usr/sbin in the PATH for all commands (`pr#18793 <https://github.com/ceph/ceph/pull/18793>`_, Ilya Dryomov)
+* rbd,tests: qa/tasks/rbd: run all xfstests runs to completion (`pr#18583 <https://github.com/ceph/ceph/pull/18583>`_, Ilya Dryomov)
+* rbd,tests: qa/workunits/rbd: fix cli_generic test_purge for rbd default format 1 (`pr#20389 <https://github.com/ceph/ceph/pull/20389>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: fixed variable name for resync image id (`issue#21663 <http://tracker.ceph.com/issues/21663>`_, `pr#18097 <https://github.com/ceph/ceph/pull/18097>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: fix issues within permissions test (`issue#23043 <http://tracker.ceph.com/issues/23043>`_, `pr#20491 <https://github.com/ceph/ceph/pull/20491>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: pool create may fail for small cluster (`pr#18067 <https://github.com/ceph/ceph/pull/18067>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: potential race in mirror disconnect test (`issue#23938 <http://tracker.ceph.com/issues/23938>`_, `pr#21733 <https://github.com/ceph/ceph/pull/21733>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: relax greps to support upgrade formatting change (`issue#21181 <http://tracker.ceph.com/issues/21181>`_, `pr#17559 <https://github.com/ceph/ceph/pull/17559>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: remove sanity check in journal.sh test (`pr#20490 <https://github.com/ceph/ceph/pull/20490>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: remove sanity check in test_admin_socket.sh (`pr#21116 <https://github.com/ceph/ceph/pull/21116>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: remove "trash purge --threshold" test (`issue#22803 <http://tracker.ceph.com/issues/22803>`_, `pr#20170 <https://github.com/ceph/ceph/pull/20170>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: simplify split-brain test to avoid potential race (`issue#22485 <http://tracker.ceph.com/issues/22485>`_, `pr#19604 <https://github.com/ceph/ceph/pull/19604>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: switch devstack tempest to 17.2.0 tag (`issue#22961 <http://tracker.ceph.com/issues/22961>`_, `pr#20599 <https://github.com/ceph/ceph/pull/20599>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: switch devstack to pike release (`pr#20124 <https://github.com/ceph/ceph/pull/20124>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: test data pool is mirrored correctly (`pr#17062 <https://github.com/ceph/ceph/pull/17062>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: unnecessary sleep after failed remove (`pr#18619 <https://github.com/ceph/ceph/pull/18619>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: use command line option to specify watcher asok (`issue#20954 <http://tracker.ceph.com/issues/20954>`_, `pr#16917 <https://github.com/ceph/ceph/pull/16917>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: wait for demote status is propagated (`pr#19073 <https://github.com/ceph/ceph/pull/19073>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: wait for status propagated only if daemon started (`pr#19082 <https://github.com/ceph/ceph/pull/19082>`_, Mykola Golub)
+* rbd,tests: rbd/test: add snap protection test for ex/import (`pr#20689 <https://github.com/ceph/ceph/pull/20689>`_, songweibin)
+* rbd,tests: stop.sh: use --no-mon-config when trying to unmap rbd devices (`pr#21020 <https://github.com/ceph/ceph/pull/21020>`_, Mykola Golub)
+* rbd,tests: test: address coverity false positives (`pr#17803 <https://github.com/ceph/ceph/pull/17803>`_, Amit Kumar)
+* rbd,tests: test/cls_rbd: mask newer feature bits to support upgrade tests (`issue#21217 <http://tracker.ceph.com/issues/21217>`_, `pr#17509 <https://github.com/ceph/ceph/pull/17509>`_, Jason Dillaman)
+* rbd,tests: test/librados_test_stub: always create copy of buffers passed to operation (`pr#21074 <https://github.com/ceph/ceph/pull/21074>`_, Mykola Golub)
+* rbd,tests: test/librbd: added update_features RPC message to test_notify (`issue#21936 <http://tracker.ceph.com/issues/21936>`_, `pr#18561 <https://github.com/ceph/ceph/pull/18561>`_, Jason Dillaman)
+* rbd,tests: test/librbd: clean up for several mock function tests (`pr#18952 <https://github.com/ceph/ceph/pull/18952>`_, Jason Dillaman)
+* rbd,tests: test/librbd: Do not instantiate TrimRequest template class (`pr#19402 <https://github.com/ceph/ceph/pull/19402>`_, Boris Ranto)
+* rbd,tests: test/librbd: ensure OutOfOrder test has enough concurrent management ops (`pr#21436 <https://github.com/ceph/ceph/pull/21436>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix mock method macro of set_journal_policy (`pr#17216 <https://github.com/ceph/ceph/pull/17216>`_, Yan Jun)
+* rbd,tests: test/librbd: fix race condition with OSD map refresh (`issue#20918 <http://tracker.ceph.com/issues/20918>`_, `pr#16877 <https://github.com/ceph/ceph/pull/16877>`_, Jason Dillaman)
+* rbd,tests: test/librbd: fix valgrind memory leak warning (`pr#17187 <https://github.com/ceph/ceph/pull/17187>`_, Mykola Golub)
+* rbd,tests: test/librbd: initialize on_finish,locker,force,snap_id (`pr#17800 <https://github.com/ceph/ceph/pull/17800>`_, Amit Kumar)
+* rbd,tests: test/librbd: make fsx build on non-linux platform (`pr#16939 <https://github.com/ceph/ceph/pull/16939>`_, Mykola Golub)
+* rbd,tests: test/librbd: memory leak in recently added test (`pr#18478 <https://github.com/ceph/ceph/pull/18478>`_, Mykola Golub)
+* rbd,tests: test/librbd: rbd-ggate mode for fsx (`pr#19315 <https://github.com/ceph/ceph/pull/19315>`_, Mykola Golub)
+* rbd,tests: test/librbd: test metadata_set/remove is applied (`pr#18288 <https://github.com/ceph/ceph/pull/18288>`_, Mykola Golub)
+* rbd,tests: test/librbd: TestMirroringWatcher unit tests should ignore duplicates (`issue#21029 <http://tracker.ceph.com/issues/21029>`_, `pr#17078 <https://github.com/ceph/ceph/pull/17078>`_, Jason Dillaman)
+* rbd,tests: test/librbd: utilize unique pool for cache tier testing (`issue#11502 <http://tracker.ceph.com/issues/11502>`_, `pr#20486 <https://github.com/ceph/ceph/pull/20486>`_, Jason Dillaman)
+* rbd,tests: test/librbd: valgrind warning in TestMockManagedLockBreakRequest.DeadLockOwner (`pr#18940 <https://github.com/ceph/ceph/pull/18940>`_, Mykola Golub)
+* rbd,tests: test/pybind/rbd: skip test_deep_copy_clone if layering not enabled (`pr#20295 <https://github.com/ceph/ceph/pull/20295>`_, Mykola Golub)
+* rbd,tests: test/rbd: cli_generic fails if v1 image format or deep-flatten disabled (`issue#22950 <http://tracker.ceph.com/issues/22950>`_, `pr#20364 <https://github.com/ceph/ceph/pull/20364>`_, songweibin)
+* rbd,tests: test/rbd_mirror: fix valgrind warnings in unittest (`pr#19016 <https://github.com/ceph/ceph/pull/19016>`_, Mykola Golub)
+* rbd,tests: test/rbd-mirror: image map policy test (`pr#19320 <https://github.com/ceph/ceph/pull/19320>`_, Venky Shankar)
+* rbd,tests: test/rbd-mirror: improve coverage for dead instance handling (`pr#21403 <https://github.com/ceph/ceph/pull/21403>`_, Jason Dillaman)
+* rbd,tests: test/rbd_mirror: "use of uninitialised value" valgrind warning (`pr#19437 <https://github.com/ceph/ceph/pull/19437>`_, Mykola Golub)
+* rbd,tools: rbd-fuse: make sure PATH_MAX is defined (`pr#18615 <https://github.com/ceph/ceph/pull/18615>`_, Roberto Oliveira)
+* rbd,tools: rbd-replay: remove boost dependency (`pr#21202 <https://github.com/ceph/ceph/pull/21202>`_, Kefu Chai)
+* rbd: tools/rbd: use steady clock in bencher (`pr#20008 <https://github.com/ceph/ceph/pull/20008>`_, Mohamad Gebai)
+* rbd: 'trash list --long' will return a failure on non-cloned images (`pr#19540 <https://github.com/ceph/ceph/pull/19540>`_, Jason Dillaman)
+* rbd: 'trash ls -l' will display column titles if existed non-USER trash image only (`pr#21343 <https://github.com/ceph/ceph/pull/21343>`_, songweibin)
+* rbd: unified way to map images using different drivers (`pr#19711 <https://github.com/ceph/ceph/pull/19711>`_, Mykola Golub)
+* rbd: use different logic to disturb thread's offset in bench seq test (`pr#17218 <https://github.com/ceph/ceph/pull/17218>`_, PCzhangPC)
+* Revert "ceph-fuse: Delete inode's bufferhead was in Tx state would le… (`pr#21976 <https://github.com/ceph/ceph/pull/21976>`_, "Yan, Zheng")
+* Revert "msg/async/rdma: fix multi cephcontext confllicting" (`pr#16980 <https://github.com/ceph/ceph/pull/16980>`_, Haomai Wang)
+* Revert "os/bluestore: compensate for bad freelistmanager size/blocks metadata" (`pr#17275 <https://github.com/ceph/ceph/pull/17275>`_, Xie Xingguo)
+* rgw: ability to list bucket contents in unsorted order for efficiency (`pr#21026 <https://github.com/ceph/ceph/pull/21026>`_, J. Eric Ivancich)
+* rgw: abort multipart if upload meta object doesn't exist (`pr#19918 <https://github.com/ceph/ceph/pull/19918>`_, fang yuxiang)
+* rgw: Access RGWConf through RGWEnv (`pr#17432 <https://github.com/ceph/ceph/pull/17432>`_, Jos Collin)
+* rgw: add "Accept-Ranges" to response header of Swift API (`issue#21554 <http://tracker.ceph.com/issues/21554>`_, `pr#17967 <https://github.com/ceph/ceph/pull/17967>`_, Tone Zhang)
+* rgw: add a default redirect field for zones (`pr#9571 <https://github.com/ceph/ceph/pull/9571>`_, Yehuda Sadeh)
+* rgw: add an option to clear all usage entries (`pr#19322 <https://github.com/ceph/ceph/pull/19322>`_, Abhishek Lekshmanan)
+* rgw: add an option to recalculate user stats (`issue#23335 <http://tracker.ceph.com/issues/23335>`_, `pr#20853 <https://github.com/ceph/ceph/pull/20853>`_, Abhishek Lekshmanan)
+* rgw: add buffering filter to compression for fetch_remote_obj (`issue#23547 <http://tracker.ceph.com/issues/23547>`_, `pr#21479 <https://github.com/ceph/ceph/pull/21479>`_, Casey Bodley)
+* rgw: add cors header rule check in cors option request (`issue#22002 <http://tracker.ceph.com/issues/22002>`_, `pr#18556 <https://github.com/ceph/ceph/pull/18556>`_, yuliyang)
+* rgw: Add dynamic resharding documentation (`issue#21553 <http://tracker.ceph.com/issues/21553>`_, `pr#15941 <https://github.com/ceph/ceph/pull/15941>`_, Orit Wasserman)
+* rgw: add logs if get_data returns error in RGWPutObj::execute (`pr#18642 <https://github.com/ceph/ceph/pull/18642>`_, Zhang Shaowen)
+* rgw: add metadata and data sync related cmd into radosgw-admin usage (`pr#18921 <https://github.com/ceph/ceph/pull/18921>`_, lvshanchun)
+* rgw: add missing override in list_keys_init() (`pr#17254 <https://github.com/ceph/ceph/pull/17254>`_, Jos Collin)
+* rgw: add radosgw-admin sync error trim to trim sync error log (`pr#19854 <https://github.com/ceph/ceph/pull/19854>`_, fang yuxiang)
+* rgw: add reshard commands (`issue#21617 <http://tracker.ceph.com/issues/21617>`_, `pr#18180 <https://github.com/ceph/ceph/pull/18180>`_, Orit Wasserman)
+* rgw: address warnings due to incorrect format code (`pr#18796 <https://github.com/ceph/ceph/pull/18796>`_, J. Eric Ivancich)
+    rgw: Add retry_raced_bucket_write
+* rgw: add rewrite cmd and options into radosgw-admin usage and doc (`pr#18918 <https://github.com/ceph/ceph/pull/18918>`_, Enming Zhang)
+* rgw: add ssl support to beast frontend (`issue#22832 <http://tracker.ceph.com/issues/22832>`_, `pr#20464 <https://github.com/ceph/ceph/pull/20464>`_, Casey Bodley)
+* rgw: add support for Swift's per storage policy statistics (`issue#17932 <http://tracker.ceph.com/issues/17932>`_, `pr#12704 <https://github.com/ceph/ceph/pull/12704>`_, Radoslaw Zarzynski)
+* rgw: add support for Swift's reversed account listings (`issue#21148 <http://tracker.ceph.com/issues/21148>`_, `pr#17320 <https://github.com/ceph/ceph/pull/17320>`_, Radoslaw Zarzynski)
+* rgw: add support for tagging and other conditionals in policy (`pr#17094 <https://github.com/ceph/ceph/pull/17094>`_, Abhishek Lekshmanan)
+* rgw: add tail tag to track tail instance (`issue#20234 <http://tracker.ceph.com/issues/20234>`_, `pr#16145 <https://github.com/ceph/ceph/pull/16145>`_, Yehuda Sadeh)
+* rgw: add tenant to shard_id in RGWDeleteLC::execute() (`pr#10460 <https://github.com/ceph/ceph/pull/10460>`_, Wei Qiaomiao)
+* rgw: add time skew check in function parse_v4_auth_header (`issue#22418 <http://tracker.ceph.com/issues/22418>`_, `pr#19476 <https://github.com/ceph/ceph/pull/19476>`_, Bingyin Zhang)
+    rgw: Add try_refresh_bucket_info function
+* rgw: add xml output header in RGWCopyObj_ObjStore_S3 response msg (`issue#22416 <http://tracker.ceph.com/issues/22416>`_, `pr#19475 <https://github.com/ceph/ceph/pull/19475>`_, Enming Zhang)
+* rgw: adjust log format for lifecycle (`pr#19576 <https://github.com/ceph/ceph/pull/19576>`_, Bingyin Zhang)
+* rgw: admin api - add ability to sync user stats from admin api (`issue#21301 <http://tracker.ceph.com/issues/21301>`_, `pr#17589 <https://github.com/ceph/ceph/pull/17589>`_, Nathan Johnson)
+* rgw: Admin API Support for bucket quota change (`issue#21811 <http://tracker.ceph.com/issues/21811>`_, `pr#18324 <https://github.com/ceph/ceph/pull/18324>`_, Jeegn Chen)
+* rgw: admin rest api shouldn't return error when getting user's stats if the user hasn't create any bucket (`pr#21551 <https://github.com/ceph/ceph/pull/21551>`_, Zhang Shaowen)
+* rgw: allow beast frontend to listen on specific IP address (`issue#22778 <http://tracker.ceph.com/issues/22778>`_, `pr#20000 <https://github.com/ceph/ceph/pull/20000>`_, Yuan Zhou)
+* rgw: Allow swift acls to be deleted (`issue#22897 <http://tracker.ceph.com/issues/22897>`_, `pr#20471 <https://github.com/ceph/ceph/pull/20471>`_, Marcus Watts)
+* rgw: avoid logging keystone revocation messages when not configured (`issue#21400 <http://tracker.ceph.com/issues/21400>`_, `pr#17775 <https://github.com/ceph/ceph/pull/17775>`_, Abhishek Lekshmanan)
+* rgw: aws4 auth supports PutBucketRequestPayment (`issue#23803 <http://tracker.ceph.com/issues/23803>`_, `pr#21569 <https://github.com/ceph/ceph/pull/21569>`_, Casey Bodley)
+* rgw: AWS v4 authorization work when INIT_MULTIPART is chunked (`issue#22129 <http://tracker.ceph.com/issues/22129>`_, `pr#18956 <https://github.com/ceph/ceph/pull/18956>`_, Jeegn Chen)
+* rgw: beast frontend can listen on multiple endpoints (`issue#22779 <http://tracker.ceph.com/issues/22779>`_, `pr#20188 <https://github.com/ceph/ceph/pull/20188>`_, Casey Bodley)
+* rgw: beast frontend no longer experimental (`pr#21272 <https://github.com/ceph/ceph/pull/21272>`_, Casey Bodley)
+* rgw: Better ERANGE error message (`issue#22351 <http://tracker.ceph.com/issues/22351>`_, `pr#20023 <https://github.com/ceph/ceph/pull/20023>`_, Brad Hubbard)
+* rgw: break sending data-log list infinitely (`issue#20951 <http://tracker.ceph.com/issues/20951>`_, `pr#16926 <https://github.com/ceph/ceph/pull/16926>`_, fang.yuxiang)
+* rgw: bucket resharding should not update bucket ACL or user stats (`issue#22742 <http://tracker.ceph.com/issues/22742>`_, `issue#22124 <http://tracker.ceph.com/issues/22124>`_, `pr#20038 <https://github.com/ceph/ceph/pull/20038>`_, Orit Wasserman)
+* rgw: Cache on the barrelhead (`issue#22517 <http://tracker.ceph.com/issues/22517>`_, `pr#19581 <https://github.com/ceph/ceph/pull/19581>`_, Adam C. Emerson)
+* rgw: Cache Register! (`issue#22604 <http://tracker.ceph.com/issues/22604>`_, `issue#22603 <http://tracker.ceph.com/issues/22603>`_, `pr#20144 <https://github.com/ceph/ceph/pull/20144>`_, Adam C. Emerson)
+* rgw: can't download object with range when compression enabled (`issue#22852 <http://tracker.ceph.com/issues/22852>`_, `pr#20226 <https://github.com/ceph/ceph/pull/20226>`_, fang yuxiang)
+* rgw: ceph-dencoder: add missing begin_iter & end_iter item for RGWObjManifest (`pr#19509 <https://github.com/ceph/ceph/pull/19509>`_, wangsongbo)
+* rgw: ceph-dencoder: add support for cls_rgw_lc_obj_head (`pr#18920 <https://github.com/ceph/ceph/pull/18920>`_, Yao Zongyou)
+* rgw: ceph-dencoder: add support for RGWLifecycleConfiguration (`pr#18959 <https://github.com/ceph/ceph/pull/18959>`_, wangsongbo)
+* rgw: change ObjectCache::lru from deque back to list (`issue#22560 <http://tracker.ceph.com/issues/22560>`_, `pr#19768 <https://github.com/ceph/ceph/pull/19768>`_, Casey Bodley)
+* rgw: changes to support ragweed (`pr#13644 <https://github.com/ceph/ceph/pull/13644>`_, Yehuda Sadeh)
+* rgw: Check bucket CORS operations in policy (`issue#21578 <http://tracker.ceph.com/issues/21578>`_, `pr#18000 <https://github.com/ceph/ceph/pull/18000>`_, Adam C. Emerson)
+* rgw: Check bucket GetBucketLocation in policy (`issue#21582 <http://tracker.ceph.com/issues/21582>`_, `pr#18002 <https://github.com/ceph/ceph/pull/18002>`_, Adam C. Emerson)
+* rgw: Check bucket Website operations in policy (`issue#21597 <http://tracker.ceph.com/issues/21597>`_, `pr#18024 <https://github.com/ceph/ceph/pull/18024>`_, Adam C. Emerson)
+* rgw: check going_down() when lifecycle processing (`issue#22099 <http://tracker.ceph.com/issues/22099>`_, `pr#18846 <https://github.com/ceph/ceph/pull/18846>`_, Yao Zongyou)
+* rgw: Check payment operations in policy (`issue#21389 <http://tracker.ceph.com/issues/21389>`_, `pr#17742 <https://github.com/ceph/ceph/pull/17742>`_, Adam C. Emerson)
+* rgw: check read_op.read return value in RGWRados::copy_obj_data (`pr#18962 <https://github.com/ceph/ceph/pull/18962>`_, Enming Zhang)
+* rgw: civetweb fixes for v1.1 upgrade (`pr#21123 <https://github.com/ceph/ceph/pull/21123>`_, Abhishek Lekshmanan)
+* rgw: clean code with helper function dump_header_if_nonempty (`pr#18979 <https://github.com/ceph/ceph/pull/18979>`_, Xinying Song)
+* rgw: clean up and fix some bugs for encryption (`issue#21581 <http://tracker.ceph.com/issues/21581>`_, `pr#17882 <https://github.com/ceph/ceph/pull/17882>`_, Enming Zhang)
+* rgw: cleanup MIN macro with std::min (`pr#17546 <https://github.com/ceph/ceph/pull/17546>`_, Jiaying Ren)
+* rgw: cleanup unused parameters in RGWRados::copy_obj_data (`pr#18917 <https://github.com/ceph/ceph/pull/18917>`_, Enming Zhang)
+* rgw: cloud sync fixes (`pr#21648 <https://github.com/ceph/ceph/pull/21648>`_, Yehuda Sadeh)
+* rgw: cls/log: cls_log_list always returns next marker (`issue#20906 <http://tracker.ceph.com/issues/20906>`_, `pr#17024 <https://github.com/ceph/ceph/pull/17024>`_, Casey Bodley)
+* rgw: cls/rgw: fix bi_log_iterate_entries return wrong truncated (`issue#22737 <http://tracker.ceph.com/issues/22737>`_, `pr#20021 <https://github.com/ceph/ceph/pull/20021>`_, Tianshan Qu)
+* rgw: cls/rgw: Initialization of uninitialized members (`pr#16932 <https://github.com/ceph/ceph/pull/16932>`_, amitkuma)
+* rgw: cls/rgw: mtime in rgw_bucket_dir_entry_meta not really decoded (`issue#22148 <http://tracker.ceph.com/issues/22148>`_, `pr#18981 <https://github.com/ceph/ceph/pull/18981>`_, Yao Zongyou)
+* rgw: cls/rgw: remove unused variable bl (`pr#19570 <https://github.com/ceph/ceph/pull/19570>`_, Yao Zongyou)
+* rgw: cls/rgw: trim all usage entries in cls_rgw (`issue#22234 <http://tracker.ceph.com/issues/22234>`_, `pr#19131 <https://github.com/ceph/ceph/pull/19131>`_, Abhishek Lekshmanan)
+* rgw: cls_rgw: use more effective container operations in get_obj_vals (`pr#19272 <https://github.com/ceph/ceph/pull/19272>`_, Xinying Song)
+* rgw: comparison between signed and unsigned integer expressions (`pr#21105 <https://github.com/ceph/ceph/pull/21105>`_, ashitakasam)
+* rgw: consolidate code that implements hashing algorithms (`pr#18248 <https://github.com/ceph/ceph/pull/18248>`_, J. Eric Ivancich)
+* rgw: copy object add response error messages (`pr#18291 <https://github.com/ceph/ceph/pull/18291>`_, Enming Zhang)
+* rgw: correct comment in function parse_credentials (`pr#19275 <https://github.com/ceph/ceph/pull/19275>`_, Bingyin Zhang)
+* rgw: correct log output for metadata section name in RGWListBucketIndexesCR (`pr#19508 <https://github.com/ceph/ceph/pull/19508>`_, Xinying Song)
+* rgw: Correct permission evaluation to allow only admin users to work with Roles (`pr#20332 <https://github.com/ceph/ceph/pull/20332>`_, Pritha Srivastava)
+* rgw: correct typo refity to refit (`pr#19064 <https://github.com/ceph/ceph/pull/19064>`_, Bingyin Zhang)
+* rgw: correct typo UNKOWN to UNKNOWN (`pr#19273 <https://github.com/ceph/ceph/pull/19273>`_, Bingyin Zhang)
+* rgw: create sync-module instance when execute radosgw-admin data sync run (`issue#22080 <http://tracker.ceph.com/issues/22080>`_, `pr#18898 <https://github.com/ceph/ceph/pull/18898>`_, lvshanchun)
+* rgw: create sync-module instance when radosgw-admin sync run (`pr#20611 <https://github.com/ceph/ceph/pull/20611>`_, lvshanchun)
+* rgw: curl\* reuse and for debian, use openssl not gnutls (`pr#20635 <https://github.com/ceph/ceph/pull/20635>`_, Marcus Watts)
+* rgw: Data encryption is not follow the AWS agreement (`pr#15994 <https://github.com/ceph/ceph/pull/15994>`_, hechuang)
+* rgw: datalog list support --shard-id and --marker (`pr#20649 <https://github.com/ceph/ceph/pull/20649>`_, Tianshan Qu)
+* rgw: data sync: set num_shards when building full maps (`issue#22083 <http://tracker.ceph.com/issues/22083>`_, `pr#18852 <https://github.com/ceph/ceph/pull/18852>`_, Abhishek Lekshmanan)
+* rgw: Delete to_string functions. stringify defined in include/stringify.h can provide the same feature (`pr#18522 <https://github.com/ceph/ceph/pull/18522>`_, zhangwen)
+* rgw: disable dynamic resharding in multisite environment (`issue#21725 <http://tracker.ceph.com/issues/21725>`_, `pr#18184 <https://github.com/ceph/ceph/pull/18184>`_, Orit Wasserman)
+* rgw: do not reflect period if not current (`issue#22844 <http://tracker.ceph.com/issues/22844>`_, `pr#20212 <https://github.com/ceph/ceph/pull/20212>`_, Tianshan Qu)
+* rgw: do not update all gateway caches upon creation of system obj w/ exclusive flag (`pr#19384 <https://github.com/ceph/ceph/pull/19384>`_, J. Eric Ivancich)
+* rgw: don't change rados object's mtime when update olh (`issue#21743 <http://tracker.ceph.com/issues/21743>`_, `pr#18214 <https://github.com/ceph/ceph/pull/18214>`_, Shasha Lu)
+* rgw: don't hold data_lock over frontend io (`pr#20621 <https://github.com/ceph/ceph/pull/20621>`_, Casey Bodley)
+* rgw: don't leak S3 LDAPHelper (`pr#12427 <https://github.com/ceph/ceph/pull/12427>`_, Matt Benjamin)
+* rgw: dont log EBUSY errors in 'sync error list' (`issue#22473 <http://tracker.ceph.com/issues/22473>`_, `pr#19580 <https://github.com/ceph/ceph/pull/19580>`_, Casey Bodley)
+* rgw: dont reuse stale RGWObjectCtx for get_bucket_info() (`issue#21506 <http://tracker.ceph.com/issues/21506>`_, `pr#17916 <https://github.com/ceph/ceph/pull/17916>`_, Casey Bodley)
+* rgw: don't write bucket_header when it is not changed in bucket_link/unlink (`pr#17356 <https://github.com/ceph/ceph/pull/17356>`_, Shasha Lu)
+* rgw: don't write bucket_header when it is not changed in rgw_bucket_prepare_op (`pr#18763 <https://github.com/ceph/ceph/pull/18763>`_, Xinying Song)
+* rgw: download object might fail for local invariable uninitialized (`issue#23146 <http://tracker.ceph.com/issues/23146>`_, `pr#20612 <https://github.com/ceph/ceph/pull/20612>`_, fang yuxiang)
+* rgw: drop a repeated statement for encode_xml() (`pr#20195 <https://github.com/ceph/ceph/pull/20195>`_, luomuyao)
+* rgw: drop commented functions (`pr#19671 <https://github.com/ceph/ceph/pull/19671>`_, Jos Collin)
+* rgw: drop dump_uri_from_state() which isn't used anymore (`pr#19924 <https://github.com/ceph/ceph/pull/19924>`_, Radoslaw Zarzynski)
+* rgw: drop iter in rgw_op.cc (`pr#19583 <https://github.com/ceph/ceph/pull/19583>`_, Bingyin Zhang)
+* rgw: drop marker in RGWLC::process() (`pr#19591 <https://github.com/ceph/ceph/pull/19591>`_, Bingyin Zhang)
+* rgw: drop outdated function doc (`pr#18370 <https://github.com/ceph/ceph/pull/18370>`_, Jiaying Ren)
+* rgw: drop "realm remove" in radosgw-admin (`pr#18212 <https://github.com/ceph/ceph/pull/18212>`_, Shasha Lu)
+* rgw: drop redundant RGW_OP_STAT_OBJ check (`pr#19933 <https://github.com/ceph/ceph/pull/19933>`_, Bingyin Zhang)
+* rgw: drop the unnecessary handling of Swift's X-Storage-Policy on objects (`pr#16383 <https://github.com/ceph/ceph/pull/16383>`_, Jiaying Ren)
+* rgw: drop the unused function init_anon_user() (`pr#16874 <https://github.com/ceph/ceph/pull/16874>`_, Radoslaw Zarzynski)
+* rgw: Drop unnecessary return (`pr#17520 <https://github.com/ceph/ceph/pull/17520>`_, Jos Collin)
+* rgw: drop unused function apply_epoch (`pr#17593 <https://github.com/ceph/ceph/pull/17593>`_, Shasha Lu)
+* rgw: drop unused iter in XMLObj::find_first (`pr#19709 <https://github.com/ceph/ceph/pull/19709>`_, luomuyao)
+* rgw: drop unused variable bucket_instance_ids (`pr#19708 <https://github.com/ceph/ceph/pull/19708>`_, Bingyin Zhang)
+* rgw: drop unused variable in copy_obj_data() (`pr#18477 <https://github.com/ceph/ceph/pull/18477>`_, Enming Zhang)
+* rgw: drop unused vector elements (`pr#19815 <https://github.com/ceph/ceph/pull/19815>`_, Bingyin Zhang)
+* rgw: drop useless includes in rgw\_{main.cc, common.h} (`pr#19109 <https://github.com/ceph/ceph/pull/19109>`_, Jiaying Ren)
+* rgw: drop useless lines (`pr#19817 <https://github.com/ceph/ceph/pull/19817>`_, Bingyin Zhang)
+* rgw: drop useless type conversion (`pr#19824 <https://github.com/ceph/ceph/pull/19824>`_, Bingyin Zhang)
+* rgw: drop variable bl in rgw_op.cc (`pr#19584 <https://github.com/ceph/ceph/pull/19584>`_, Bingyin Zhang)
+* rgw: Drop #warning TODO (`issue#19851 <http://tracker.ceph.com/issues/19851>`_, `pr#17012 <https://github.com/ceph/ceph/pull/17012>`_, Jos Collin)
+* rgw: dump Last-Modified in Swift's responses for GET/HEAD on container (`issue#20883 <http://tracker.ceph.com/issues/20883>`_, `pr#16757 <https://github.com/ceph/ceph/pull/16757>`_, Radoslaw Zarzynski)
+* rgw: enable 'qlen' & 'qactive' performance counters (`pr#20842 <https://github.com/ceph/ceph/pull/20842>`_, Mark Kogan)
+* rgw: encoding fixes (`issue#23779 <http://tracker.ceph.com/issues/23779>`_, `pr#21500 <https://github.com/ceph/ceph/pull/21500>`_, Yehuda Sadeh)
+* rgw: Error check on return of read_line() (`pr#17880 <https://github.com/ceph/ceph/pull/17880>`_, Amit Kumar)
+* rgw: es module: set compression type correctly (`issue#22758 <http://tracker.ceph.com/issues/22758>`_, `pr#20796 <https://github.com/ceph/ceph/pull/20796>`_, Abhishek Lekshmanan)
+* rgw: evaluate the correct bucket action for GetACL bucket operation (`issue#21013 <http://tracker.ceph.com/issues/21013>`_, `pr#17050 <https://github.com/ceph/ceph/pull/17050>`_, Abhishek Lekshmanan)
+* rgw: exit early if rgw_bucket_set_attrs() fails (`pr#17041 <https://github.com/ceph/ceph/pull/17041>`_, dengxiafubi)
+    rgw: Expire entries in bucket info cache
+* rgw_file: fix write error when the write offset overlaps (`issue#21455 <http://tracker.ceph.com/issues/21455>`_, `pr#17809 <https://github.com/ceph/ceph/pull/17809>`_, Yao Zongyou)
+* rgw: fix a bug in rgw cache in delete_system_obj and get_system_obj (`pr#10992 <https://github.com/ceph/ceph/pull/10992>`_, zhangshaowen)
+* rgw: fix accessing expired memory in PrefixableSignatureHelper (`issue#21085 <http://tracker.ceph.com/issues/21085>`_, `pr#17206 <https://github.com/ceph/ceph/pull/17206>`_, Radoslaw Zarzynski)
+* rgw: fix a typo in comment (`pr#19608 <https://github.com/ceph/ceph/pull/19608>`_, luomuyao)
+* rgw: fix a typo in comment (`pr#20164 <https://github.com/ceph/ceph/pull/20164>`_, luomuyao)
+* rgw: fix a typo in comment (`pr#20355 <https://github.com/ceph/ceph/pull/20355>`_, luomuyao)
+* rgw: fix a typo in rgw_perms[] (`pr#20024 <https://github.com/ceph/ceph/pull/20024>`_, luomuyao)
+* rgw: fix bilog entries on multipart complete (`issue#21772 <http://tracker.ceph.com/issues/21772>`_, `pr#18271 <https://github.com/ceph/ceph/pull/18271>`_, Casey Bodley)
+* rgw: fix BZ 1500904, stale bucket index entry remains after obj delete (`pr#18709 <https://github.com/ceph/ceph/pull/18709>`_, J. Eric Ivancich)
+* rgw: fix chained cache invalidation to prevent cache size growth (`issue#22410 <http://tracker.ceph.com/issues/22410>`_, `pr#19455 <https://github.com/ceph/ceph/pull/19455>`_, Mark Kogan)
+* rgw: Fix closing tag for Prefix (`pr#17663 <https://github.com/ceph/ceph/pull/17663>`_, Shasha Lu)
+* rgw: fix cls_bucket_head result order consistency (`pr#18700 <https://github.com/ceph/ceph/pull/18700>`_, Tianshan Qu)
+* rgw: fix collect()'s return in coroutine (`pr#19606 <https://github.com/ceph/ceph/pull/19606>`_, Xinying Song)
+* rgw: fix command argument error for radosgw-admin (`issue#21723 <http://tracker.ceph.com/issues/21723>`_, `pr#18175 <https://github.com/ceph/ceph/pull/18175>`_, Yao Zongyou)
+* rgw: fix 'copy part' without 'x-amz-copy-source-range' (`issue#22729 <http://tracker.ceph.com/issues/22729>`_, `pr#20002 <https://github.com/ceph/ceph/pull/20002>`_, Malcolm Lee)
+* rgw: fix 'copy part' without 'x-amz-copy-source-range' when compression enabled (`issue#23196 <http://tracker.ceph.com/issues/23196>`_, `pr#20686 <https://github.com/ceph/ceph/pull/20686>`_, fang yuxiang)
+* rgw: fix crash with rgw_run_sync_thread false (`issue#20448 <http://tracker.ceph.com/issues/20448>`_, `pr#20769 <https://github.com/ceph/ceph/pull/20769>`_, Orit Wasserman)
+* rgw: Fix dereference of empty optional (`issue#21962 <http://tracker.ceph.com/issues/21962>`_, `pr#18602 <https://github.com/ceph/ceph/pull/18602>`_, Adam C. Emerson)
+* rgw: fix error handling for GET with ?torrent (`issue#23506 <http://tracker.ceph.com/issues/23506>`_, `pr#21576 <https://github.com/ceph/ceph/pull/21576>`_, Casey Bodley)
+* rgw: fix error handling in Browser Uploads (`pr#15054 <https://github.com/ceph/ceph/pull/15054>`_, Radoslaw Zarzynski)
+* rgw: fix error handling in ListBucketIndexesCR (`issue#21735 <http://tracker.ceph.com/issues/21735>`_, `pr#18198 <https://github.com/ceph/ceph/pull/18198>`_, Casey Bodley)
+* rgw: fixes for multisite replication of encrypted objects (`issue#20668 <http://tracker.ceph.com/issues/20668>`_, `issue#20671 <http://tracker.ceph.com/issues/20671>`_, `pr#16612 <https://github.com/ceph/ceph/pull/16612>`_, Casey Bodley)
+* rgw: fix extra_data_len handling in PutObj filters (`issue#21895 <http://tracker.ceph.com/issues/21895>`_, `pr#18489 <https://github.com/ceph/ceph/pull/18489>`_, Casey Bodley)
+* rgw: fix for empty query string in beast frontend (`issue#22797 <http://tracker.ceph.com/issues/22797>`_, `pr#20120 <https://github.com/ceph/ceph/pull/20120>`_, Casey Bodley)
+* rgw: fix for issue #21647 (`issue#23859 <http://tracker.ceph.com/issues/23859>`_, `pr#21647 <https://github.com/ceph/ceph/pull/21647>`_, Yehuda Sadeh)
+* rgw: fix for pause in beast frontend (`issue#21831 <http://tracker.ceph.com/issues/21831>`_, `pr#18402 <https://github.com/ceph/ceph/pull/18402>`_, Casey Bodley)
+* rgw: fix for usage truncated flag (`pr#20926 <https://github.com/ceph/ceph/pull/20926>`_, Yehuda Sadeh, Greg Farnum, Robin H. Johnson)
+* rgw: Fix getter function names in RGWEnv (`pr#18377 <https://github.com/ceph/ceph/pull/18377>`_, Jos Collin)
+* rgw: fix GET website response error code (`issue#22272 <http://tracker.ceph.com/issues/22272>`_, `pr#19236 <https://github.com/ceph/ceph/pull/19236>`_, Dmitry Plyakin)
+* rgw: fix handling of ENOENT in RGWRadosGetOmapKeysCR (`pr#19878 <https://github.com/ceph/ceph/pull/19878>`_, Casey Bodley)
+* rgw: fix index cancel op miss update header (`pr#20396 <https://github.com/ceph/ceph/pull/20396>`_, Tianshan Qu)
+* rgw: Fix infinite call for bi list when resharding a bucket (`issue#22721 <http://tracker.ceph.com/issues/22721>`_, `pr#21584 <https://github.com/ceph/ceph/pull/21584>`_, Orit Wasserman)
+* rgw: fix lc process only schdule the first item of lc objects (`issue#21022 <http://tracker.ceph.com/issues/21022>`_, `pr#17061 <https://github.com/ceph/ceph/pull/17061>`_, Shasha Lu)
+* rgw:fix list objects with marker wrong result  when bucket is enable versioning (`issue#21500 <http://tracker.ceph.com/issues/21500>`_, `pr#17934 <https://github.com/ceph/ceph/pull/17934>`_, yuliyang)
+* rgw: fix memory fragmentation problem reading data from client (`pr#20724 <https://github.com/ceph/ceph/pull/20724>`_, Marcus Watts)
+* rgw: Fix multisite Synchronization failed when read and write delete … (`issue#22804 <http://tracker.ceph.com/issues/22804>`_, `pr#20814 <https://github.com/ceph/ceph/pull/20814>`_, Niu Pengju)
+* rgw: fix not responding when receiving SIGHUP signal (`pr#16854 <https://github.com/ceph/ceph/pull/16854>`_, Yao Zongyou)
+* rgw: fix null pointer crush (`pr#18861 <https://github.com/ceph/ceph/pull/18861>`_, Sibei Gao)
+* rgw: fix obj copied from remote gateway acl full_control issue (`issue#20658 <http://tracker.ceph.com/issues/20658>`_, `pr#16127 <https://github.com/ceph/ceph/pull/16127>`_, Enming Zhang)
+* rgw: fix opslog cannot record remote_addr (`issue#20931 <http://tracker.ceph.com/issues/20931>`_, `pr#16860 <https://github.com/ceph/ceph/pull/16860>`_, Jiaying Ren)
+* rgw: fix opslog can't record referrer when using curl as client (`issue#20935 <http://tracker.ceph.com/issues/20935>`_, `pr#16863 <https://github.com/ceph/ceph/pull/16863>`_, Jiaying Ren)
+* rgw: fix opslog uri as per Amazon s3 (`issue#20971 <http://tracker.ceph.com/issues/20971>`_, `pr#16958 <https://github.com/ceph/ceph/pull/16958>`_, Jiaying Ren)
+* rgw: fix radosgw-admin bucket rm with --purge-objects and --bypass-gc (`issue#22122 <http://tracker.ceph.com/issues/22122>`_, `issue#19959 <http://tracker.ceph.com/issues/19959>`_, `pr#18922 <https://github.com/ceph/ceph/pull/18922>`_, Aleksei Gutikov)
+* rgw: fix radosgw-admin quota enable return value bug (`issue#21608 <http://tracker.ceph.com/issues/21608>`_, `pr#18057 <https://github.com/ceph/ceph/pull/18057>`_, baixueyu)
+* rgw: fix radosgw linkage with WITH_RADOSGW_BEAST_FRONTEND=OFF (`issue#23680 <http://tracker.ceph.com/issues/23680>`_, `pr#21380 <https://github.com/ceph/ceph/pull/21380>`_, Casey Bodley)
+* rgw: fix recursive lock (`pr#19430 <https://github.com/ceph/ceph/pull/19430>`_, Tianshan Qu)
+* rgw: fix resource leak in rgw_bucket.cc and rgw_user.cc (`issue#21214 <http://tracker.ceph.com/issues/21214>`_, `pr#17353 <https://github.com/ceph/ceph/pull/17353>`_, Luo Kexue)
+* rgw: fix return value of auth v2/v4 (`issue#22439 <http://tracker.ceph.com/issues/22439>`_, `pr#19310 <https://github.com/ceph/ceph/pull/19310>`_, Bingyin Zhang)
+* rgw: fix rewrite a versioning object create a new object bug (`issue#21984 <http://tracker.ceph.com/issues/21984>`_, `pr#18662 <https://github.com/ceph/ceph/pull/18662>`_, Enming Zhang)
+* rgw: fix rewrite options usage text (`pr#18968 <https://github.com/ceph/ceph/pull/18968>`_, Jos Collin)
+* rgw: fix RGWCompletionManager get_next stuck after going down (`issue#22799 <http://tracker.ceph.com/issues/22799>`_, `pr#20095 <https://github.com/ceph/ceph/pull/20095>`_, Tianshan Qu)
+* rgw: fix RGWLibIO did not init RGWEnv (`pr#19065 <https://github.com/ceph/ceph/pull/19065>`_, Tianshan Qu)
+* rgw: fix s3 website redirection error (`pr#19252 <https://github.com/ceph/ceph/pull/19252>`_, yuliyang)
+* rgw: fix s3website redirect location string length (`pr#19826 <https://github.com/ceph/ceph/pull/19826>`_, yuliyang)
+* rgw: fix Swift container naming rules (`issue#19264 <http://tracker.ceph.com/issues/19264>`_, `pr#13992 <https://github.com/ceph/ceph/pull/13992>`_, Robin H. Johnson)
+* rgw: Fix swift object expiry not deleting objects (`issue#22084 <http://tracker.ceph.com/issues/22084>`_, `pr#18821 <https://github.com/ceph/ceph/pull/18821>`_, Pavan Rallabhandi)
+* rgw: fix sync status conflict with cloud sync (`pr#21425 <https://github.com/ceph/ceph/pull/21425>`_, Casey Bodley)
+* rgw: fix the bug of radowgw-admin zonegroup set requires realm (`issue#21583 <http://tracker.ceph.com/issues/21583>`_, `pr#19061 <https://github.com/ceph/ceph/pull/19061>`_, lvshanchun)
+* rgw: fix the max-uploads parameter not work (`issue#22825 <http://tracker.ceph.com/issues/22825>`_, `pr#20158 <https://github.com/ceph/ceph/pull/20158>`_, Xin Liao)
+* rgw: fix the return type is wrong (`pr#19773 <https://github.com/ceph/ceph/pull/19773>`_, hechuang)
+* rgw: fix total_time to msec as per AWS S3 (`pr#17541 <https://github.com/ceph/ceph/pull/17541>`_, Jiaying Ren)
+* rgw: fix typo anynoymous to anonymous (`pr#19281 <https://github.com/ceph/ceph/pull/19281>`_, Bingyin Zhang)
+* rgw: fix typo compete to complete (`pr#19675 <https://github.com/ceph/ceph/pull/19675>`_, Bingyin Zhang)
+* rgw: Fix typo in comment (`pr#21032 <https://github.com/ceph/ceph/pull/21032>`_, Simran Singhal)
+* rgw: fix typo in GetOmapKeysCR (`pr#19713 <https://github.com/ceph/ceph/pull/19713>`_, lvshanchun)
+* rgw: fix typo signle to single (`pr#19517 <https://github.com/ceph/ceph/pull/19517>`_, Bingyin Zhang)
+* rgw: fix typo woild to would (`pr#19472 <https://github.com/ceph/ceph/pull/19472>`_, Bingyin Zhang)
+* rgw: Fix use after free in IAM policy parser (`pr#16823 <https://github.com/ceph/ceph/pull/16823>`_, Adam C. Emerson)
+* rgw: fix use of libcurl with empty header values (`issue#23663 <http://tracker.ceph.com/issues/23663>`_, `pr#21358 <https://github.com/ceph/ceph/pull/21358>`_, Casey Bodley)
+* rgw: format logs in file rgw_lc.cc (`pr#19615 <https://github.com/ceph/ceph/pull/19615>`_, Bingyin Zhang)
+* rgw: format rgw_bucket_dir_header in ceph-dencoder (`pr#19753 <https://github.com/ceph/ceph/pull/19753>`_, Bingyin Zhang)
+* rgw: gc use aio (`pr#20546 <https://github.com/ceph/ceph/pull/20546>`_, Yehuda Sadeh)
+    rgw: Handle stale bucket info in RGWDeleteBucketPolicy
+    rgw: Handle stale bucket info in RGWDeleteBucketWebsite
+    rgw: Handle stale bucket info in RGWPutBucketPolicy
+    rgw: Handle stale bucket info in RGWPutMetadataBucket
+    rgw: Handle stale bucket info in RGWSetBucketVersioning
+    rgw: Handle stale bucket info in RGWSetBucketWebsite
+* rgw: honor the tenant part of rgw_bucket during comparisons (`issue#20897 <http://tracker.ceph.com/issues/20897>`_, `pr#16796 <https://github.com/ceph/ceph/pull/16796>`_, Radoslaw Zarzynski)
+* rgw: iam policy printing cleanups (`pr#18961 <https://github.com/ceph/ceph/pull/18961>`_, Kefu Chai)
+* rgw: Ignoring the returned error (`pr#17907 <https://github.com/ceph/ceph/pull/17907>`_, Amit Kumar)
+* rgw: implement ipv4 aws:SourceIp condition for bucket policy (`pr#19167 <https://github.com/ceph/ceph/pull/19167>`_, yuliyang)
+* rgw: improve handling of Swift's error messages and limits (`issue#17938 <http://tracker.ceph.com/issues/17938>`_, `issue#21169 <http://tracker.ceph.com/issues/21169>`_, `issue#17935 <http://tracker.ceph.com/issues/17935>`_, `issue#17934 <http://tracker.ceph.com/issues/17934>`_, `issue#17936 <http://tracker.ceph.com/issues/17936>`_, `pr#15369 <https://github.com/ceph/ceph/pull/15369>`_, Radoslaw Zarzynski)
+* rgw: improve sync status: display behind bucket shards (`pr#20027 <https://github.com/ceph/ceph/pull/20027>`_, lvshanchun)
+* rgw: improve sync status (`pr#19573 <https://github.com/ceph/ceph/pull/19573>`_, lvshanchun)
+* rgw: include SSE-KMS headers in encrypted upload response (`issue#21576 <http://tracker.ceph.com/issues/21576>`_, `pr#17999 <https://github.com/ceph/ceph/pull/17999>`_, Casey Bodley)
+* rgw: incorporate the Transfer-Encoding fix for CivetWeb (`issue#21015 <http://tracker.ceph.com/issues/21015>`_, `pr#17072 <https://github.com/ceph/ceph/pull/17072>`_, Radoslaw Zarzynski)
+* rgw: Initialization of epoch,len (`pr#17722 <https://github.com/ceph/ceph/pull/17722>`_, Amit Kumar)
+* rgw: Initialize is_master, max_aio, size (`pr#16933 <https://github.com/ceph/ceph/pull/16933>`_, amitkuma)
+* rgw: Initializes uninitialized members (`pr#16855 <https://github.com/ceph/ceph/pull/16855>`_, Amit Kumar)
+* rgw: init oldest period after setting run_sync_thread (`issue#21996 <http://tracker.ceph.com/issues/21996>`_, `pr#18664 <https://github.com/ceph/ceph/pull/18664>`_, Orit Wasserman, Casey Bodley)
+* rgw: keep compression type consistent between parts of s3 Multipart (`pr#19740 <https://github.com/ceph/ceph/pull/19740>`_, fang yuxiang)
+* rgw: keystone: bump up logging when error is received (`issue#22151 <http://tracker.ceph.com/issues/22151>`_, `pr#18985 <https://github.com/ceph/ceph/pull/18985>`_, Abhishek Lekshmanan)
+* rgw:lc fix expiration time (`issue#21533 <http://tracker.ceph.com/issues/21533>`_, `pr#17824 <https://github.com/ceph/ceph/pull/17824>`_, Shasha Lu)
+* rgw: lc support Content-MD5 request header and fix a rgw crash bug (`issue#21980 <http://tracker.ceph.com/issues/21980>`_, `pr#18534 <https://github.com/ceph/ceph/pull/18534>`_, Enming Zhang)
+* rgw: lease_cr->go_down is called twice, remove the needless one (`pr#19394 <https://github.com/ceph/ceph/pull/19394>`_, Zhang Shaowen)
+* rgw: librgw: export multitenancy support (`pr#19358 <https://github.com/ceph/ceph/pull/19358>`_, Tao Chen)
+* rgw: librgw: fix shutdown err with resources uncleaned (`issue#22296 <http://tracker.ceph.com/issues/22296>`_, `pr#19279 <https://github.com/ceph/ceph/pull/19279>`_, Tao Chen)
+* rgw: lifecycle omap entry was removed in abnormal situation (`pr#19921 <https://github.com/ceph/ceph/pull/19921>`_, fang yuxiang)
+* rgw: list_objects() honors end_marker regardless of namespace (`issue#18977 <http://tracker.ceph.com/issues/18977>`_, `pr#15273 <https://github.com/ceph/ceph/pull/15273>`_, Radoslaw Zarzynski)
+* rgw: loadgen fix generate random object name rgw crash issue (`issue#22006 <http://tracker.ceph.com/issues/22006>`_, `pr#18536 <https://github.com/ceph/ceph/pull/18536>`_, Enming Zhang)
+* rgw: log the right http status code in civetweb frontend's access log (`issue#22538 <http://tracker.ceph.com/issues/22538>`_, `pr#19678 <https://github.com/ceph/ceph/pull/19678>`_, Yao Zongyou)
+* rgw: log unlink_instance mtime as object's mtime (`issue#18885 <http://tracker.ceph.com/issues/18885>`_, `pr#20016 <https://github.com/ceph/ceph/pull/20016>`_, Yehuda Sadeh)
+* rgw: lttng: Trace rgw data transfer, bi entry and object header update processes (`pr#20556 <https://github.com/ceph/ceph/pull/20556>`_, Yang Honggang)
+* rgw:  make init env methods return an error (`issue#23039 <http://tracker.ceph.com/issues/23039>`_, `pr#20488 <https://github.com/ceph/ceph/pull/20488>`_, Abhishek Lekshmanan)
+* rgw: make radosgw object stat RGW_ATTR_COMPRESSION dump readable (`pr#19846 <https://github.com/ceph/ceph/pull/19846>`_, fang yuxiang)
+* rgw: mfa support (`pr#19283 <https://github.com/ceph/ceph/pull/19283>`_, Yehuda Sadeh)
+* rgw: mimic: rgw: policy: modify s3:ListBucketMultiPartUploads to s3:ListBucketMul (`issue#24062 <http://tracker.ceph.com/issues/24062>`_, `pr#21916 <https://github.com/ceph/ceph/pull/21916>`_, xiangxiang)
+* rgw: modify s3 type subuser access permissions fail through admin rest api (`issue#21983 <http://tracker.ceph.com/issues/21983>`_, `pr#18641 <https://github.com/ceph/ceph/pull/18641>`_, yuliyang)
+* rgw: move all pool creation into rgw_init_ioctx (`issue#23480 <http://tracker.ceph.com/issues/23480>`_, `pr#21534 <https://github.com/ceph/ceph/pull/21534>`_, Casey Bodley)
+* rgw: mrgw.sh uses instance name 'client.rgw' (`pr#18404 <https://github.com/ceph/ceph/pull/18404>`_, Casey Bodley)
+* rgw: multisite log tracing (`pr#16492 <https://github.com/ceph/ceph/pull/16492>`_, Yehuda Sadeh, Casey Bodley)
+* rgw,nfs: Add hint to use -o sync when mouting (`pr#16210 <https://github.com/ceph/ceph/pull/16210>`_, Adam Kupczyk)
+* rgw: no need to deal with md5 header in get_data (`pr#19144 <https://github.com/ceph/ceph/pull/19144>`_, Zhang Shaowen)
+* rgw: optimize function abort_bucket_multiparts (`pr#19710 <https://github.com/ceph/ceph/pull/19710>`_, Bingyin Zhang)
+* rgw: optimize function bucket_lc_prepare (`pr#19613 <https://github.com/ceph/ceph/pull/19613>`_, Bingyin Zhang)
+* rgw: optimize function parse_raw_oid (`pr#19814 <https://github.com/ceph/ceph/pull/19814>`_, Bingyin Zhang)
+* rgw: optimize function RGWHandler::do_init_permissions (`pr#19700 <https://github.com/ceph/ceph/pull/19700>`_, Bingyin Zhang)
+* rgw: optimize memory usage in function rgw_bucket::get_key (`pr#19391 <https://github.com/ceph/ceph/pull/19391>`_, Bingyin Zhang)
+* rgw: optimize next start time for lifecycle (`pr#19596 <https://github.com/ceph/ceph/pull/19596>`_, Bingyin Zhang)
+* rgw: optimize the rgw-attr del code logic (`pr#18895 <https://github.com/ceph/ceph/pull/18895>`_, wangsongbo)
+* rgw: optimize time skew check (`pr#19511 <https://github.com/ceph/ceph/pull/19511>`_, Bingyin Zhang)
+* rgw: parse old rgw_obj with namespace correctly (`issue#22982 <http://tracker.ceph.com/issues/22982>`_, `pr#20425 <https://github.com/ceph/ceph/pull/20425>`_, Yehuda Sadeh)
+* rgw: policy: support for s3 conditionals in ListBucket Op (`pr#16628 <https://github.com/ceph/ceph/pull/16628>`_, Abhishek Lekshmanan)
+* rgw: Potential fix for possible 500 on POST (`pr#18954 <https://github.com/ceph/ceph/pull/18954>`_, Adam C. Emerson)
+* rgw: Prevent overflow of cached stats values (`issue#20934 <http://tracker.ceph.com/issues/20934>`_, `pr#17116 <https://github.com/ceph/ceph/pull/17116>`_, Aleksei Gutikov)
+* rgw: proper error message when tier_type does not exist (`issue#22469 <http://tracker.ceph.com/issues/22469>`_, `pr#19575 <https://github.com/ceph/ceph/pull/19575>`_, lvshanchun, Chang Liu)
+* rgw: pull up beast submodule and update frontend (`pr#17923 <https://github.com/ceph/ceph/pull/17923>`_, Casey Bodley)
+* rgw: put bucket policy panics RGW process (`issue#22541 <http://tracker.ceph.com/issues/22541>`_, `pr#19687 <https://github.com/ceph/ceph/pull/19687>`_, Bingyin Zhang)
+* rgw: radosgw-admin abort early for user stats for empty uids (`issue#23322 <http://tracker.ceph.com/issues/23322>`_, `pr#20846 <https://github.com/ceph/ceph/pull/20846>`_, Abhishek Lekshmanan)
+* rgw: radosgw-admin should not use metadata cache for readonly commands (`issue#23468 <http://tracker.ceph.com/issues/23468>`_, `pr#21129 <https://github.com/ceph/ceph/pull/21129>`_, Orit Wasserman)
+* rgw: radosgw-admin zonegroup get and zone get return defaults when there is no realm (`issue#21615 <http://tracker.ceph.com/issues/21615>`_, `pr#18667 <https://github.com/ceph/ceph/pull/18667>`_, lvshanchun)
+* rgw: radosgw: fix awsv4 header line sort order (`issue#21607 <http://tracker.ceph.com/issues/21607>`_, `pr#18046 <https://github.com/ceph/ceph/pull/18046>`_, Marcus Watts)
+* rgw: radosgw: usage: fix bytes_sent bug (`issue#19870 <http://tracker.ceph.com/issues/19870>`_, `pr#16834 <https://github.com/ceph/ceph/pull/16834>`_, Marcus Watts)
+* rgw: raise log level on coroutine shutdown errors (`issue#23974 <http://tracker.ceph.com/issues/23974>`_, `pr#21791 <https://github.com/ceph/ceph/pull/21791>`_, Casey Bodley)
+* rgw: Reinstating error codes mapping for Roles (`pr#20309 <https://github.com/ceph/ceph/pull/20309>`_, Pritha Srivastava)
+* rgw: reject encrypted object COPY before supported (`issue#23232 <http://tracker.ceph.com/issues/23232>`_, `pr#20739 <https://github.com/ceph/ceph/pull/20739>`_, Jeegn Chen)
+* rgw: release cls lock if taken in RGWCompleteMultipart (`issue#21596 <http://tracker.ceph.com/issues/21596>`_, `pr#18104 <https://github.com/ceph/ceph/pull/18104>`_, Matt Benjamin)
+* rgw: Remove assertions in IAM Policy (`pr#18225 <https://github.com/ceph/ceph/pull/18225>`_, Adam C. Emerson)
+* rgw: remove get_system_obj_attrs in function RGWDeleteLC::execute and RGWDeleteCORS::execute (`pr#19582 <https://github.com/ceph/ceph/pull/19582>`_, Bingyin Zhang)
+* rgw: remove placement_rule from rgw_link_bucket() (`issue#21990 <http://tracker.ceph.com/issues/21990>`_, `pr#18657 <https://github.com/ceph/ceph/pull/18657>`_, Casey Bodley)
+* rgw: remove redundant parenthesis in logs (`pr#19375 <https://github.com/ceph/ceph/pull/19375>`_, Bingyin Zhang)
+* rgw: remove redundant S3AnonymousEngine (`pr#19474 <https://github.com/ceph/ceph/pull/19474>`_, Bingyin Zhang)
+* rgw: remove redundant signature compare in LocalEngine::authenticate (`pr#19676 <https://github.com/ceph/ceph/pull/19676>`_, Bingyin Zhang)
+* rgw: Remove the useless output when list zones (`pr#17434 <https://github.com/ceph/ceph/pull/17434>`_, iliul)
+* rgw: remove unused cls_user_add_bucket (`pr#19917 <https://github.com/ceph/ceph/pull/19917>`_, Yao Zongyou)
+* rgw: remove unused disable_signal_fd (`pr#18875 <https://github.com/ceph/ceph/pull/18875>`_, Yao Zongyou)
+* rgw: remove unused function get_system_obj_attrs (`pr#19852 <https://github.com/ceph/ceph/pull/19852>`_, Yao Zongyou)
+* rgw: Remove unused Parameter in Function RGWConf::init() (`pr#17129 <https://github.com/ceph/ceph/pull/17129>`_, Wen Zhang)
+* rgw: remove unused param in AWSGeneralAbstractor::get_auth_data_v4 (`pr#19250 <https://github.com/ceph/ceph/pull/19250>`_, Bingyin Zhang)
+* rgw: remove unused param in get_bucket_instance_policy_from_attr (`pr#19129 <https://github.com/ceph/ceph/pull/19129>`_, Bingyin Zhang)
+* rgw: remove unused variables (`pr#16649 <https://github.com/ceph/ceph/pull/16649>`_, Zhang Lei)
+* rgw: remove useless lines in RGWDeleteBucket::execute (`pr#19699 <https://github.com/ceph/ceph/pull/19699>`_, Bingyin Zhang)
+* rgw: reshard cancel command should clear bucket resharding flag (`issue#21619 <http://tracker.ceph.com/issues/21619>`_, `pr#21120 <https://github.com/ceph/ceph/pull/21120>`_, Orit Wasserman)
+* rgw: reshard should not update stats when linking new bucket instance (`issue#22124 <http://tracker.ceph.com/issues/22124>`_, `pr#19253 <https://github.com/ceph/ceph/pull/19253>`_, Orit Wasserman)
+* rgw: retry CORS put/delete operations on ECANCELLED (`issue#22517 <http://tracker.ceph.com/issues/22517>`_, `pr#19601 <https://github.com/ceph/ceph/pull/19601>`_, Adam C. Emerson)
+* rgw: return 'Access-Control-Allow-Origin' header when the set and delete bucket website through XMLHttpRequest (`pr#17632 <https://github.com/ceph/ceph/pull/17632>`_, yuliyang)
+* rgw: return 'Access-Control-Allow-Origin' header when the set bucket versioning through XMLHttpRequest (`pr#17631 <https://github.com/ceph/ceph/pull/17631>`_, yuliyang)
+* rgw: return bucket's location no matter which zonegroup it located in (`issue#21125 <http://tracker.ceph.com/issues/21125>`_, `pr#17250 <https://github.com/ceph/ceph/pull/17250>`_, Shasha Lu)
+* rgw: return EINVAL if max_keys can not convert correctly (`issue#23586 <http://tracker.ceph.com/issues/23586>`_, `pr#21285 <https://github.com/ceph/ceph/pull/21285>`_, yuliyang)
+* rgw: Return Error if Bucket Policy Contians Undefined Action (`pr#17433 <https://github.com/ceph/ceph/pull/17433>`_, zhangwen)
+* rgw: Returning when dst_ioctx.operate() returns error (`pr#17873 <https://github.com/ceph/ceph/pull/17873>`_, Amit Kumar)
+* rgw: return valid Location element, CompleteMultipartUpload (`pr#19902 <https://github.com/ceph/ceph/pull/19902>`_, Matt Benjamin)
+* rgw: revert PR #8765 (`pr#16807 <https://github.com/ceph/ceph/pull/16807>`_, fang.yuxiang)
+* rgw: Revert "radosgw: fix awsv4 header line sort order." (`issue#21832 <http://tracker.ceph.com/issues/21832>`_, `pr#18381 <https://github.com/ceph/ceph/pull/18381>`_, Casey Bodley)
+* rgw: Revert "rgw_file: disable FLAG_EXACT_MATCH enforcement" (`issue#22827 <http://tracker.ceph.com/issues/22827>`_, `pr#20171 <https://github.com/ceph/ceph/pull/20171>`_, Matt Benjamin)
+* rgw: Revert "rgw: reshard should not update stats when linking new bucket instance" (`pr#20052 <https://github.com/ceph/ceph/pull/20052>`_, Orit Wasserman)
+* rgw: rework json/xml escape usage follow #19806 (`pr#19845 <https://github.com/ceph/ceph/pull/19845>`_, fang yuxiang)
+* rgw: rgw-admin: check input parameters for friendly prompt (`pr#17343 <https://github.com/ceph/ceph/pull/17343>`_, Yao Zongyou)
+* rgw: rgw-admin: check the data extra pool supports omap (`pr#18978 <https://github.com/ceph/ceph/pull/18978>`_, Yao Zongyou)
+* rgw: rgw-admin: properly filtering bucket stats by user_id or bucket_name (`pr#19401 <https://github.com/ceph/ceph/pull/19401>`_, Yao Zongyou)
+* rgw: rgw-admin: require --yes-i-really-mean-it when using --inconsistent_index (`issue#20777 <http://tracker.ceph.com/issues/20777>`_, `pr#17185 <https://github.com/ceph/ceph/pull/17185>`_, Orit Wasserman)
+* rgw: rgw-admin: support for processing all gc objects including unexpired (`pr#17482 <https://github.com/ceph/ceph/pull/17482>`_, Yao Zongyou)
+* rgw: RGW: change function parameters from value to refrence (`pr#18355 <https://github.com/ceph/ceph/pull/18355>`_, Sibei Gao)
+* rgw: RGWCivetWeb::read_data: fix arguments to mg_read() call (`issue#23596 <http://tracker.ceph.com/issues/23596>`_, `pr#21291 <https://github.com/ceph/ceph/pull/21291>`_, Nathan Cutler)
+* rgw: rgw clean-up: remove unreferenced pure virtual class StreamObjData (`pr#18799 <https://github.com/ceph/ceph/pull/18799>`_, J. Eric Ivancich)
+* rgw: rgw clean-up: remove unused var & func in RGWRados::SystemObject (`pr#18987 <https://github.com/ceph/ceph/pull/18987>`_, J. Eric Ivancich)
+* rgw: rgw cleanup: some unnecessary function called and repeated assignment (`pr#18817 <https://github.com/ceph/ceph/pull/18817>`_, Enming Zhang)
+* rgw: rgw cloud sync (`issue#21802 <http://tracker.ceph.com/issues/21802>`_, `pr#18932 <https://github.com/ceph/ceph/pull/18932>`_, lvshanchun, Yehuda Sadeh, Chang Liu, Abhishek Lekshmanan)
+* rgw: RGWEnv::set() takes std::string (`issue#22101 <http://tracker.ceph.com/issues/22101>`_, `pr#18866 <https://github.com/ceph/ceph/pull/18866>`_, Casey Bodley)
+* rgw: rgw_file: alternate fix deadlock on lru eviction (`pr#20034 <https://github.com/ceph/ceph/pull/20034>`_, Matt Benjamin)
+* rgw: rgw_file: avoid evaluating nullptr for readdir offset (`pr#20145 <https://github.com/ceph/ceph/pull/20145>`_, Matt Benjamin)
+* rgw: rgw_file: conditionally unlink handles when direct deleted (`issue#23299 <http://tracker.ceph.com/issues/23299>`_, `pr#20834 <https://github.com/ceph/ceph/pull/20834>`_, Matt Benjamin)
+* rgw: rgw_file: explicit NFSv3 open() emulation (`pr#18365 <https://github.com/ceph/ceph/pull/18365>`_, Matt Benjamin)
+* rgw: rgw_file:  fix LRU lane lock in evict_block() (`issue#21141 <http://tracker.ceph.com/issues/21141>`_, `pr#17267 <https://github.com/ceph/ceph/pull/17267>`_, Matt Benjamin)
+* rgw: rgw_file: implement variant offset readdir processing (`pr#18335 <https://github.com/ceph/ceph/pull/18335>`_, Matt Benjamin)
+* rgw: rgw_file: introduce new fsid and rgw_mount (`pr#15330 <https://github.com/ceph/ceph/pull/15330>`_, Gui Hecheng)
+* rgw: rgw_file:  set s->obj_size from bytes_written (`issue#21940 <http://tracker.ceph.com/issues/21940>`_, `pr#18571 <https://github.com/ceph/ceph/pull/18571>`_, Matt Benjamin)
+* rgw: rgw_file: Silence unused-function warnings (`pr#19278 <https://github.com/ceph/ceph/pull/19278>`_, Brad Hubbard)
+* rgw: RGW: fix a bug about inconsistent unit of comparison (`issue#21590 <http://tracker.ceph.com/issues/21590>`_, `pr#17958 <https://github.com/ceph/ceph/pull/17958>`_, gaosibei)
+* rgw: rgw.iam: change '1' to '1ULL' in function print_actions (`pr#18900 <https://github.com/ceph/ceph/pull/18900>`_, Bingyin Zhang)
+* rgw: rgw_lc: add support for optional filter argument and make ID optional (`issue#19587 <http://tracker.ceph.com/issues/19587>`_, `issue#20872 <http://tracker.ceph.com/issues/20872>`_, `pr#16818 <https://github.com/ceph/ceph/pull/16818>`_, Abhishek Lekshmanan)
+* rgw: rgw_lc: support for AWSv4 authentication (`pr#16734 <https://github.com/ceph/ceph/pull/16734>`_, Abhishek Lekshmanan)
+* rgw: rgw_log, rgw_file: account for new required envvars (`issue#21942 <http://tracker.ceph.com/issues/21942>`_, `pr#18572 <https://github.com/ceph/ceph/pull/18572>`_, Matt Benjamin)
+* rgw: Rgw master fix plus (`issue#21000 <http://tracker.ceph.com/issues/21000>`_, `issue#21003 <http://tracker.ceph.com/issues/21003>`_, `issue#20501 <http://tracker.ceph.com/issues/20501>`_, `pr#17040 <https://github.com/ceph/ceph/pull/17040>`_, Zhang Shaowen, Marcus Watts)
+* rgw: rgw, mon: normalize delete/remove in admin console (cleanup 22444) (`issue#14363 <http://tracker.ceph.com/issues/14363>`_, `issue#22444 <http://tracker.ceph.com/issues/22444>`_, `pr#19439 <https://github.com/ceph/ceph/pull/19439>`_, Jesse Williamson)
+* rgw: RGW: Multipart upload may double the quota (`issue#21586 <http://tracker.ceph.com/issues/21586>`_, `pr#17959 <https://github.com/ceph/ceph/pull/17959>`_, Sibei Gao)
+* rgw: rgw multisite: automated trimming for bucket index logs (`issue#18229 <http://tracker.ceph.com/issues/18229>`_, `pr#17761 <https://github.com/ceph/ceph/pull/17761>`_, Casey Bodley)
+* rgw: RGW NFS:  mount cmdline example missing -osync (`pr#15855 <https://github.com/ceph/ceph/pull/15855>`_, Matt Benjamin)
+* rgw: RGW-NFS: Use rados cluster_stat to report filesystem usage (`issue#22202 <http://tracker.ceph.com/issues/22202>`_, `pr#20093 <https://github.com/ceph/ceph/pull/20093>`_, Supriti Singh)
+* rgw: rgw_op: Drop the Old LifecycleConfiguration from logs (`pr#16821 <https://github.com/ceph/ceph/pull/16821>`_, Abhishek Lekshmanan)
+* rgw: rgw_op: exit early if object has no attrs in GetObjectTagging (`issue#21010 <http://tracker.ceph.com/issues/21010>`_, `pr#17048 <https://github.com/ceph/ceph/pull/17048>`_, Abhishek Lekshmanan)
+* rgw: RGWPutLC return ERR_MALFORMED_XML  when missing <Rule> tag in lifecycle.xml (`issue#21377 <http://tracker.ceph.com/issues/21377>`_, `pr#17683 <https://github.com/ceph/ceph/pull/17683>`_, Shasha Lu)
+* rgw: rgw_put_system_obj takes bufferlist (`pr#19897 <https://github.com/ceph/ceph/pull/19897>`_, Casey Bodley)
+* rgw: rgw_rados: set_attrs now sets the same time for BI & object (`issue#21200 <http://tracker.ceph.com/issues/21200>`_, `pr#17400 <https://github.com/ceph/ceph/pull/17400>`_, Abhishek Lekshmanan)
+* rgw: rgw/rgw_op.cc: Fix error message in rgw_user_get_all_buckets_stats (`pr#18781 <https://github.com/ceph/ceph/pull/18781>`_, iliul)
+* rgw:  rgw: source data in 'default.rgw.buckets.data' may not be deleted after inter-bucket copy (`issue#21819 <http://tracker.ceph.com/issues/21819>`_, `pr#18369 <https://github.com/ceph/ceph/pull/18369>`_, baixueyu)
+* rgw: RGW: support for tagging in lifecycle policies (`pr#17305 <https://github.com/ceph/ceph/pull/17305>`_, Abhishek Lekshmanan)
+* rgw: RGW: update S3 POST policy handling of Content-Type (`issue#20201 <http://tracker.ceph.com/issues/20201>`_, `pr#18658 <https://github.com/ceph/ceph/pull/18658>`_, Matt Benjamin)
+* rgw:  rgw: use camelcase format in request headers (`pr#19210 <https://github.com/ceph/ceph/pull/19210>`_, lvshanchun, Chang Liu)
+* rgw: RGWUser::init no longer overwrites user_id (`issue#21685 <http://tracker.ceph.com/issues/21685>`_, `pr#18137 <https://github.com/ceph/ceph/pull/18137>`_, Casey Bodley)
+* rgw: S3 Bucket Policy Conditions IpAddress and NotIpAddress do not work (`issue#20991 <http://tracker.ceph.com/issues/20991>`_, `pr#17010 <https://github.com/ceph/ceph/pull/17010>`_, John Gibson)
+* rgw: s3website error handler uses original object name (`issue#23201 <http://tracker.ceph.com/issues/23201>`_, `pr#20693 <https://github.com/ceph/ceph/pull/20693>`_, Casey Bodley)
+* rgw:send x-amz-version-id header when upload files (`pr#18935 <https://github.com/ceph/ceph/pull/18935>`_, Xinying Song)
+* rgw: set bucket versioninig donot change versioning status if missing status in xml (`issue#21364 <http://tracker.ceph.com/issues/21364>`_, `pr#17662 <https://github.com/ceph/ceph/pull/17662>`_, Shasha Lu)
+* rgw: set num_shards on 'radosgw-admin data sync init' (`issue#22083 <http://tracker.ceph.com/issues/22083>`_, `pr#18883 <https://github.com/ceph/ceph/pull/18883>`_, Casey Bodley)
+* rgw: set priority on perf counters (`pr#20006 <https://github.com/ceph/ceph/pull/20006>`_, John Spray)
+* rgw: set sync_from_all as true when no value is seen (`issue#22062 <http://tracker.ceph.com/issues/22062>`_, `pr#18926 <https://github.com/ceph/ceph/pull/18926>`_, Abhishek Lekshmanan)
+* rgw: setup locks for libopenssl (`issue#22951 <http://tracker.ceph.com/issues/22951>`_, `issue#23203 <http://tracker.ceph.com/issues/23203>`_, `pr#20390 <https://github.com/ceph/ceph/pull/20390>`_, Abhishek Lekshmanan, Jesse Williamson)
+* rgw: share time skew check between v2 and v4 auth (`pr#20013 <https://github.com/ceph/ceph/pull/20013>`_, Casey Bodley)
+* rgw: Silence maybe-uninitialized false positives (`pr#19274 <https://github.com/ceph/ceph/pull/19274>`_, Brad Hubbard)
+* rgw: silence not allow register storage class specifier warning (`pr#19859 <https://github.com/ceph/ceph/pull/19859>`_, Yao Zongyou)
+* rgw: simplify use of map::emplace in iam (`pr#18706 <https://github.com/ceph/ceph/pull/18706>`_, Casey Bodley)
+* rgw: Small refactor and two bug fixes (`issue#21901 <http://tracker.ceph.com/issues/21901>`_, `issue#21896 <http://tracker.ceph.com/issues/21896>`_, `pr#18606 <https://github.com/ceph/ceph/pull/18606>`_, Adam C. Emerson)
+* rgw: some cleanup for sync status (`pr#20894 <https://github.com/ceph/ceph/pull/20894>`_, Enming Zhang)
+* rgw: stop/join TokenCache revoke thread only if started (`issue#21666 <http://tracker.ceph.com/issues/21666>`_, `pr#18106 <https://github.com/ceph/ceph/pull/18106>`_, Karol Mroz)
+* rgw: stream metadata full sync init (`issue#18079 <http://tracker.ceph.com/issues/18079>`_, `pr#12429 <https://github.com/ceph/ceph/pull/12429>`_, Yehuda Sadeh)
+* rgw: submodule: update Beast to ceph/ceph-master branch (`pr#19182 <https://github.com/ceph/ceph/pull/19182>`_, Casey Bodley)
+* rgw: switch beast frontend back to stackful coroutine (`issue#20048 <http://tracker.ceph.com/issues/20048>`_, `pr#20449 <https://github.com/ceph/ceph/pull/20449>`_, Casey Bodley)
+* rgw: sync tracing fixes (`issue#22833 <http://tracker.ceph.com/issues/22833>`_, `pr#20191 <https://github.com/ceph/ceph/pull/20191>`_, Yehuda Sadeh)
+* rgw: tenant fixes for dynamic resharding (`issue#22046 <http://tracker.ceph.com/issues/22046>`_, `pr#18811 <https://github.com/ceph/ceph/pull/18811>`_, Orit Wasserman)
+* rgw,tests: fix s3atests that are failing for sometime (`pr#20678 <https://github.com/ceph/ceph/pull/20678>`_, Vasu Kulkarni)
+* rgw,tests: qa: fix overrides for openssl_keys task (`pr#20981 <https://github.com/ceph/ceph/pull/20981>`_, Casey Bodley)
+* rgw,tests: qa: re enable LC tests (`pr#17020 <https://github.com/ceph/ceph/pull/17020>`_, Abhishek Lekshmanan)
+* rgw,tests: qa/rgw: add beast frontend to some rgw suites (`pr#17977 <https://github.com/ceph/ceph/pull/17977>`_, Casey Bodley)
+* rgw,tests: qa/rgw: combine swift, s3tests, ragweed into single verify task (`pr#20756 <https://github.com/ceph/ceph/pull/20756>`_, Casey Bodley)
+* rgw,tests: qa/rgw: disable log trim in multisite suite (`pr#19438 <https://github.com/ceph/ceph/pull/19438>`_, Casey Bodley)
+* rgw,tests: qa/rgw: hadoop-s3a suite targets centos_latest (`pr#17777 <https://github.com/ceph/ceph/pull/17777>`_, Casey Bodley)
+* rgw,tests: qa/rgw: ignore errors from 'pool application enable' (`issue#21715 <http://tracker.ceph.com/issues/21715>`_, `pr#18193 <https://github.com/ceph/ceph/pull/18193>`_, Casey Bodley)
+* rgw,tests: qa/rgw: remove some civetweb overrides for beast testing (`issue#23002 <http://tracker.ceph.com/issues/23002>`_, `pr#20440 <https://github.com/ceph/ceph/pull/20440>`_, Casey Bodley)
+* rgw,tests: qa/rgw: renamed ssl task to openssl_keys (`pr#20863 <https://github.com/ceph/ceph/pull/20863>`_, Ricardo Dias)
+* rgw,tests: qa/rgw: use 'ceph osd pool application enable' on created pools (`pr#17162 <https://github.com/ceph/ceph/pull/17162>`_, Casey Bodley)
+* rgw,tests: qa/rgw: verify suite tests civetweb with ssl (`pr#20444 <https://github.com/ceph/ceph/pull/20444>`_, Casey Bodley)
+* rgw,tests: qa/smoke: add rgw crypto config for s3tests (`pr#17700 <https://github.com/ceph/ceph/pull/17700>`_, Casey Bodley)
+* rgw,tests: qa/tasks/swift: add support for the "force-branch" configurable (`pr#21027 <https://github.com/ceph/ceph/pull/21027>`_, Radoslaw Zarzynski)
+* rgw,tests: rgw, qa: integrate Tempest to verify RadosGW's compliance with Swift API (`pr#16344 <https://github.com/ceph/ceph/pull/16344>`_, Radoslaw Zarzynski)
+* rgw,tests: test/rgw: fix test_encrypted_object_sync for 3+ zones (`pr#17377 <https://github.com/ceph/ceph/pull/17377>`_, Casey Bodley)
+* rgw: the metavariables in frontends-related config won't be expanded (`pr#19689 <https://github.com/ceph/ceph/pull/19689>`_, root)
+* rgw,tools: tools/rgw: add script to inspect admin socket "cr dump" (`pr#15554 <https://github.com/ceph/ceph/pull/15554>`_, Casey Bodley)
+* rgw: Torrents are not supported for objects encrypted using SSE-C (`issue#21720 <http://tracker.ceph.com/issues/21720>`_, `pr#17956 <https://github.com/ceph/ceph/pull/17956>`_, Zhang Shaowen)
+* rgw: trim all spaces inside a metadata value (`issue#23301 <http://tracker.ceph.com/issues/23301>`_, `pr#20841 <https://github.com/ceph/ceph/pull/20841>`_, Orit Wasserman)
+* rgw: udpate radosgw-admin usage with bi purge (`pr#18245 <https://github.com/ceph/ceph/pull/18245>`_, Yao Zongyou)
+* rgw: unlink deleted bucket from bucket's owner (`issue#22248 <http://tracker.ceph.com/issues/22248>`_, `pr#20017 <https://github.com/ceph/ceph/pull/20017>`_, Casey Bodley)
+* rgw: unreachable return in RGWRados::trim_bi_log_entries (`pr#17367 <https://github.com/ceph/ceph/pull/17367>`_, Amit Kumar)
+* rgw: update life cycle related log level (`pr#18845 <https://github.com/ceph/ceph/pull/18845>`_, Yao Zongyou)
+* rgw: update outdated debug func name (`pr#17440 <https://github.com/ceph/ceph/pull/17440>`_, Jiaying Ren)
+* rgw: update quota is inconsistent at add/del object with compression (`issue#22568 <http://tracker.ceph.com/issues/22568>`_, `pr#19772 <https://github.com/ceph/ceph/pull/19772>`_, fang yuxiang)
+* rgw: update the usage read iterator in truncated scenario (`issue#21196 <http://tracker.ceph.com/issues/21196>`_, `pr#17939 <https://github.com/ceph/ceph/pull/17939>`_, Mark Kogan)
+* rgw: update usage() with status (`pr#18178 <https://github.com/ceph/ceph/pull/18178>`_, Jos Collin)
+* rgw: update vstart.sh to support rgw ssl port notation : '--rgw_port 443s' (`issue#21151 <http://tracker.ceph.com/issues/21151>`_, `pr#17989 <https://github.com/ceph/ceph/pull/17989>`_, Mark Kogan)
+* rgw: upldate the max-buckets when the quota is uploaded (`pr#20063 <https://github.com/ceph/ceph/pull/20063>`_, zhaokun)
+* rgw: URL-decode S3 and Swift object-copy URLs (`issue#22121 <http://tracker.ceph.com/issues/22121>`_, `pr#19936 <https://github.com/ceph/ceph/pull/19936>`_, Matt Benjamin)
+* rgw: url_encode key name and instance in es sync module (`pr#20707 <https://github.com/ceph/ceph/pull/20707>`_, Chang Liu)
+* rgw: use explicit index pool placement (`issue#22928 <http://tracker.ceph.com/issues/22928>`_, `pr#20352 <https://github.com/ceph/ceph/pull/20352>`_, Yehuda Sadeh)
+* rgw: Use namespace for lc_pool and roles_pool (`issue#20177 <http://tracker.ceph.com/issues/20177>`_, `pr#16889 <https://github.com/ceph/ceph/pull/16889>`_, Orit Wasserman)
+* rgw: Various cleanups and options update in rgw_admin.cc (`pr#18302 <https://github.com/ceph/ceph/pull/18302>`_, Jos Collin)
+* rgw: vstart.sh: fix mstop.sh can not stop rgw (`pr#17438 <https://github.com/ceph/ceph/pull/17438>`_, Jiaying Ren)
+* rgw: 'zone placement' commands validate compression type (`issue#21775 <http://tracker.ceph.com/issues/21775>`_, `pr#18273 <https://github.com/ceph/ceph/pull/18273>`_, Casey Bodley)
+* rocksdb: sync with upstream (`issue#21603 <http://tracker.ceph.com/issues/21603>`_, `pr#18262 <https://github.com/ceph/ceph/pull/18262>`_, Kefu Chai)
+* rpm: rm macros in comments (`issue#22250 <http://tracker.ceph.com/issues/22250>`_, `pr#17070 <https://github.com/ceph/ceph/pull/17070>`_, Ken Dreyer)
+* script/build-integration-branch: check errors (`pr#17578 <https://github.com/ceph/ceph/pull/17578>`_, Sage Weil)
+* script/build-integration-branch: python3 compatible and pep8 clean (`pr#18035 <https://github.com/ceph/ceph/pull/18035>`_, Kefu Chai)
+* scripts: new backport-create-issue script (`pr#21480 <https://github.com/ceph/ceph/pull/21480>`_, Nathan Cutler)
+* selinux: Allow ceph to execute ldconfig (`pr#21974 <https://github.com/ceph/ceph/pull/21974>`_, Boris Ranto)
+* selinux: Allow getattr on lnk sysfs files (`pr#17891 <https://github.com/ceph/ceph/pull/17891>`_, Boris Ranto)
+* spdk: advance to upstream dc82989d (`pr#20713 <https://github.com/ceph/ceph/pull/20713>`_, Nathan Cutler)
+* src: fix various log messages (`pr#21112 <https://github.com/ceph/ceph/pull/21112>`_, Gu Zhongyan)
+* src/msg/rdma: fixes failure on assert in notify() (`pr#17007 <https://github.com/ceph/ceph/pull/17007>`_, Alex Mikheev)
+* suites/cephmetrics: Add Centos 7 (`pr#18594 <https://github.com/ceph/ceph/pull/18594>`_, Zack Cerza)
+* test: assert check for negative returns (`pr#17296 <https://github.com/ceph/ceph/pull/17296>`_, Amit Kumar)
+* test/fio: generate db histogram to help debug rocksdb performance (`pr#16808 <https://github.com/ceph/ceph/pull/16808>`_, Pan Liu, Xiaoyan Li)
+* test: fix bash path in shebangs (part 2) (`pr#17955 <https://github.com/ceph/ceph/pull/17955>`_, Alan Somers)
+* test: fix CLI unit formatting tests (`pr#22260 <https://github.com/ceph/ceph/pull/22260>`_, Jason Dillaman)
+* test: Incorrect conversion to double (`pr#18963 <https://github.com/ceph/ceph/pull/18963>`_, Amit Kumar)
+* test/librados: reorder ASSERT_EQ() arguments (`pr#16625 <https://github.com/ceph/ceph/pull/16625>`_, Yan Jun)
+* test,osd,kvstore_tool: silence warnings and prepare test buffer in the right way (`pr#18406 <https://github.com/ceph/ceph/pull/18406>`_, Adam C. Emerson)
+* tests: bluestore/fio: Fixed problem with all objects having the same hash (`pr#17770 <https://github.com/ceph/ceph/pull/17770>`_, Adam Kupczyk)
+* tests: CentOS 7.4 is now the latest (`pr#17776 <https://github.com/ceph/ceph/pull/17776>`_, Nathan Cutler)
+* tests - ceph-ansible vars additions (`issue#21822 <http://tracker.ceph.com/issues/21822>`_, `pr#18378 <https://github.com/ceph/ceph/pull/18378>`_, Yuri Weinstein)
+* tests: ceph-disk: ignore E722 in flake8 test (`issue#22207 <http://tracker.ceph.com/issues/22207>`_, `pr#19072 <https://github.com/ceph/ceph/pull/19072>`_, Nathan Cutler)
+* tests: ceph-disk: mock get fsid (`pr#19254 <https://github.com/ceph/ceph/pull/19254>`_, Kefu Chai)
+* tests: ceph-disk: Remove sitepackages=True (`issue#22823 <http://tracker.ceph.com/issues/22823>`_, `pr#20151 <https://github.com/ceph/ceph/pull/20151>`_, Brad Hubbard)
+* tests: ceph-objectstore-tool: don't destroy SnapMapper until the txn is completed (`issue#23121 <http://tracker.ceph.com/issues/23121>`_, `pr#20593 <https://github.com/ceph/ceph/pull/20593>`_, Kefu Chai)
+* tests: Changes required for teuthology's systemd support (`pr#18380 <https://github.com/ceph/ceph/pull/18380>`_, Zack Cerza)
+* tests: Check for empty output in test_dump_pgstate_history (`pr#20838 <https://github.com/ceph/ceph/pull/20838>`_, Brad Hubbard)
+* tests: cleanup: drop calamari tasks (`pr#17531 <https://github.com/ceph/ceph/pull/17531>`_, Nathan Cutler)
+* tests: cleanup: drop upgrade/jewel-x/point-to-point-x (`issue#22888 <http://tracker.ceph.com/issues/22888>`_, `pr#20245 <https://github.com/ceph/ceph/pull/20245>`_, Nathan Cutler)
+* tests: cmake,test/mgr: restructure dashboard tests and cmake related fixes (`pr#20768 <https://github.com/ceph/ceph/pull/20768>`_, Kefu Chai)
+* tests: common/obj_bencher: set {min,max}_iops if runtime < 1 sec (`pr#17182 <https://github.com/ceph/ceph/pull/17182>`_, Kefu Chai)
+* tests: c_read_operations.cc: Silence tautological-compare compiler warning (`pr#19953 <https://github.com/ceph/ceph/pull/19953>`_, Brad Hubbard)
+* tests: fix uninitialized value found by coverity scan (`pr#17895 <https://github.com/ceph/ceph/pull/17895>`_, J. Eric Ivancich)
+* tests: Increase sleep in test_pidfile.sh (`pr#17052 <https://github.com/ceph/ceph/pull/17052>`_, David Zafman)
+* tests: librgw_file: remove unused `using` statement (`pr#17085 <https://github.com/ceph/ceph/pull/17085>`_, Yao Zongyou)
+* tests: mark_unfound_lost fix and some other minor changes (`issue#21907 <http://tracker.ceph.com/issues/21907>`_, `pr#18449 <https://github.com/ceph/ceph/pull/18449>`_, David Zafman)
+* tests: mgr/dashboard: Allow sourcing `run-backend-api-tests.sh` (`pr#20874 <https://github.com/ceph/ceph/pull/20874>`_, Sebastian Wagner)
+* tests:  mgr/dashboard: create venv for running tox (`pr#21490 <https://github.com/ceph/ceph/pull/21490>`_, Kefu Chai)
+* tests: mgr/dashboard: notification queue: fix priority tests (`pr#21147 <https://github.com/ceph/ceph/pull/21147>`_, Ricardo Dias)
+* tests: mimic: qa: fix test on "ceph fs set cephfs allow_new_snaps" (`pr#21830 <https://github.com/ceph/ceph/pull/21830>`_, Kefu Chai)
+* tests: mimic: qa/workunits/rados/test_envlibrados_for_rocksdb: install g++ not g++-4.7 (`pr#22117 <https://github.com/ceph/ceph/pull/22117>`_, Kefu Chai)
+* tests: mimic: test: Need to escape parens in log-whitelist for grep (`pr#22075 <https://github.com/ceph/ceph/pull/22075>`_, David Zafman)
+* tests: mimic: test: wait_for_pg_stats() should do another check after last 13 secon… (`pr#22199 <https://github.com/ceph/ceph/pull/22199>`_, David Zafman)
+* tests: misc: Fix bash path in shebangs (`pr#16494 <https://github.com/ceph/ceph/pull/16494>`_, Alan Somers)
+* tests: mstart.sh: remove bashizm in /bin/sh script (`pr#18541 <https://github.com/ceph/ceph/pull/18541>`_, Mykola Golub)
+* tests: point-to-point-x: upgrade client.1 to -x along with cluster nodes (`issue#21499 <http://tracker.ceph.com/issues/21499>`_, `pr#17910 <https://github.com/ceph/ceph/pull/17910>`_, Nathan Cutler)
+* tests: qa: add cbt task for performance testing (`pr#17583 <https://github.com/ceph/ceph/pull/17583>`_, Neha Ojha)
+* tests: qa: add cosbench workloads and override teuthology default settings (`pr#21710 <https://github.com/ceph/ceph/pull/21710>`_, Neha Ojha)
+* tests/qa: Adding $ distro mix - rgw (`pr#22070 <https://github.com/ceph/ceph/pull/22070>`_, Yuri Weinstein)
+* tests/qa: adding rados/.. dirs (`pr#22068 <https://github.com/ceph/ceph/pull/22068>`_, Yuri Weinstein)
+* tests: qa: add "restful" to ceph_mgr_modules in ceph-ansible suite (`pr#18634 <https://github.com/ceph/ceph/pull/18634>`_, Kefu Chai)
+* tests: qa: add simple and dirty script to find ports being used (`pr#19102 <https://github.com/ceph/ceph/pull/19102>`_, Joao Eduardo Luis)
+* tests: qa: big: add openstack.yaml (`pr#16864 <https://github.com/ceph/ceph/pull/16864>`_, Nathan Cutler)
+* tests: qa: clean up dnsmasq task and fix EPERM error (`pr#20680 <https://github.com/ceph/ceph/pull/20680>`_, Casey Bodley)
+* tests: qa: create_cache_pool no longer runs 'pool application enable' (`issue#21155 <http://tracker.ceph.com/issues/21155>`_, `pr#17312 <https://github.com/ceph/ceph/pull/17312>`_, Casey Bodley)
+* tests: qa: decrease the msg_inject_socket_failures from 1/500 to 1/1000 (`issue#22093 <http://tracker.ceph.com/issues/22093>`_, `pr#19542 <https://github.com/ceph/ceph/pull/19542>`_, Kefu Chai)
+* tests: qa: disable mon-health-to-clog in upgrade test (`pr#19233 <https://github.com/ceph/ceph/pull/19233>`_, Kefu Chai)
+* tests: qa: disable -Werror when compiling env_librados_test (`pr#21433 <https://github.com/ceph/ceph/pull/21433>`_, Kefu Chai)
+* tests: qa: do not "ceph fs get cephfs" w/o a cephfs fs (`pr#18533 <https://github.com/ceph/ceph/pull/18533>`_, Kefu Chai)
+* tests: qa: do not wait for down/out osd for pg convergence (`pr#18808 <https://github.com/ceph/ceph/pull/18808>`_, Kefu Chai)
+* tests/qa - enabled `ceph-deploy` runs on `mira` nodes (`pr#21253 <https://github.com/ceph/ceph/pull/21253>`_, Yuri Weinstein)
+* tests: qa: fix pool-quota related tests (`issue#21409 <http://tracker.ceph.com/issues/21409>`_, `pr#17763 <https://github.com/ceph/ceph/pull/17763>`_, xie xingguo)
+* tests: qa: Fix shebangs on openstack scripts (`pr#16546 <https://github.com/ceph/ceph/pull/16546>`_, Alan Somers)
+* tests: qa: reduce "mon client hunt interval max multiple" to 2 for all clients (`pr#21658 <https://github.com/ceph/ceph/pull/21658>`_, Kefu Chai)
+* tests:  qa: reduce mon-client-hunt-interval-max-multiple to 2 (`pr#18283 <https://github.com/ceph/ceph/pull/18283>`_, Kefu Chai)
+* tests: qa: revert "qa: use config_path property instead of literal" (`pr#17850 <https://github.com/ceph/ceph/pull/17850>`_, Patrick Donnelly)
+* tests: qa/run-standalone.sh: set PYTHONPATH for FreeBSD also (`pr#20646 <https://github.com/ceph/ceph/pull/20646>`_, Kefu Chai)
+* tests: qa: s/backfill/backfilling/ (`pr#18235 <https://github.com/ceph/ceph/pull/18235>`_, Kefu Chai)
+* tests: qa/stanalone: pass options using --<option-name>=<value> (`pr#19544 <https://github.com/ceph/ceph/pull/19544>`_, Kefu Chai)
+* tests: qa/standalone: Add trap for signals to restore the kernel core pattern (`pr#17026 <https://github.com/ceph/ceph/pull/17026>`_, David Zafman)
+* tests: qa/standalone/ceph-helpers.sh: provide argument to dirname (`issue#23805 <http://tracker.ceph.com/issues/23805>`_, `pr#21552 <https://github.com/ceph/ceph/pull/21552>`_, Nathan Cutler)
+* tests: qa/standalone/ceph-helpers.sh: silence ceph-disk DEPRECATION_WARNING (`pr#19478 <https://github.com/ceph/ceph/pull/19478>`_, Kefu Chai)
+* tests: qa/standalone: extract delete_pool() (`pr#20634 <https://github.com/ceph/ceph/pull/20634>`_, Kefu Chai)
+* tests: qa/standalone: misc fixes (`issue#20465 <http://tracker.ceph.com/issues/20465>`_, `issue#20921 <http://tracker.ceph.com/issues/20921>`_, `pr#16709 <https://github.com/ceph/ceph/pull/16709>`_, David Zafman)
+* tests: qa/standalone/mon/misc.sh: Add osdmap-prune tests (`issue#23621 <http://tracker.ceph.com/issues/23621>`_, `pr#21318 <https://github.com/ceph/ceph/pull/21318>`_, Brad Hubbard)
+* tests: qa/standalone/osd/osd-mark-down: create pool to get updated osdmap faster (`pr#18191 <https://github.com/ceph/ceph/pull/18191>`_, huangjun)
+* tests: qa/standalone: remove osd-map-max-advance related tests (`issue#22596 <http://tracker.ceph.com/issues/22596>`_, `pr#19816 <https://github.com/ceph/ceph/pull/19816>`_, Kefu Chai)
+* tests: qa/standalone: respect $TEMPDIR (`pr#17747 <https://github.com/ceph/ceph/pull/17747>`_, Kefu Chai)
+* tests: qa/standalone/scrub/osd-scrub-repair.sh: add extents flag into object_info_t (`issue#21618 <http://tracker.ceph.com/issues/21618>`_, `pr#18094 <https://github.com/ceph/ceph/pull/18094>`_, xie xingguo)
+* tests: qa/standalone/scrub/osd-scrub-repair.sh: drop omap_digest flag (`pr#18150 <https://github.com/ceph/ceph/pull/18150>`_, xie xingguo, Sage Weil)
+* tests: qa/standalone: s/delete_erasure_pool/delete_erasure_coded_pool/ (`pr#20667 <https://github.com/ceph/ceph/pull/20667>`_, Kefu Chai)
+* tests: qa: stop testing deprecated "ceph osd create" (`issue#21993 <http://tracker.ceph.com/issues/21993>`_, `pr#18659 <https://github.com/ceph/ceph/pull/18659>`_, Kefu Chai)
+* tests: qa/suites: add minimal performance suite (`pr#21104 <https://github.com/ceph/ceph/pull/21104>`_, Neha Ojha)
+* tests: qa/suites/cephmetrics: Updates for new version (`pr#21146 <https://github.com/ceph/ceph/pull/21146>`_, Zack Cerza)
+* tests: qa/suites: change fixed-2.yaml users to get 4 openstack disks (`pr#16873 <https://github.com/ceph/ceph/pull/16873>`_, Sage Weil)
+* tests: qa/suites: mds.0 -> mds.a (`pr#20848 <https://github.com/ceph/ceph/pull/20848>`_, Sage Weil)
+* tests: qa/suites/rados: Disable scrub backoff (`issue#23578 <http://tracker.ceph.com/issues/23578>`_, `pr#21295 <https://github.com/ceph/ceph/pull/21295>`_, Brad Hubbard)
+* tests: qa/suites/rados/mgr/tasks/dashboard: add MDS_ALL_DOWN to whitelist (`pr#21549 <https://github.com/ceph/ceph/pull/21549>`_, Ricardo Dias)
+* tests: qa/suites/rados/mgr/tasks/dashboard_v2: add fail_on_skip = false (`pr#20925 <https://github.com/ceph/ceph/pull/20925>`_, Ricardo Dias)
+* tests: qa/suites/rados/multimon: whitelist mgr down vs clock skew test (`pr#16996 <https://github.com/ceph/ceph/pull/16996>`_, Sage Weil)
+* tests: qa/suites/rados/singleton: more whitelist (`pr#19225 <https://github.com/ceph/ceph/pull/19225>`_, Kefu Chai)
+* tests: qa/suites/rados/thrash-old-clients: ms_type=simple (`issue#23922 <http://tracker.ceph.com/issues/23922>`_, `pr#21739 <https://github.com/ceph/ceph/pull/21739>`_, Kefu Chai)
+* tests: qa/suites/rados/upgrade/jewel-x-singleton: tolerate sloppy past_intervals (`pr#17293 <https://github.com/ceph/ceph/pull/17293>`_, Kefu Chai)
+* tests: qa/suites/rest/basic/tasks/rest_test: more whitelisting (`issue#21425 <http://tracker.ceph.com/issues/21425>`_, `pr#17794 <https://github.com/ceph/ceph/pull/17794>`_, huangjun)
+* tests: qa/suites/rest/basic/tasks/rest_test: whiltelist OSD_DOWN (`issue#21425 <http://tracker.ceph.com/issues/21425>`_, `pr#18144 <https://github.com/ceph/ceph/pull/18144>`_, huangjun)
+* tests: qa/suites/upgarde/jewel-x/parallel: tolerate mgr warning (`pr#17203 <https://github.com/ceph/ceph/pull/17203>`_, Sage Weil)
+* tests: qa/suites/upgarde/jewel-x/point-to-point-x: disable app warnings (`pr#16947 <https://github.com/ceph/ceph/pull/16947>`_, Sage Weil)
+* tests: qa/suites: whitelist SLOW_OPS (`issue#23495 <http://tracker.ceph.com/issues/23495>`_, `pr#21324 <https://github.com/ceph/ceph/pull/21324>`_, Kefu Chai)
+* tests: qa/tasks: Add default timeout for wait for pg clean task (`pr#21313 <https://github.com/ceph/ceph/pull/21313>`_, Vasu Kulkarni)
+* tests: qa/tasks/ceph_deploy: gatherkeys before mgr deploy (`pr#17224 <https://github.com/ceph/ceph/pull/17224>`_, Sage Weil)
+* tests:  qa/tasks/ceph_manager: use set_config on revived osd (`pr#20901 <https://github.com/ceph/ceph/pull/20901>`_, Neha Ojha)
+* tests: qa/tasks/mgr/dashboard: Fix login expires too soon (`pr#21021 <https://github.com/ceph/ceph/pull/21021>`_, Sebastian Wagner)
+* tests: qa/tasks: prolong revive_osd() timeout to 6 min (`issue#21474 <http://tracker.ceph.com/issues/21474>`_, `pr#17902 <https://github.com/ceph/ceph/pull/17902>`_, Kefu Chai)
+* tests: qa/tasks: prolong revive_osd() timeout to 6 min (`issue#21474 <http://tracker.ceph.com/issues/21474>`_, `pr#19024 <https://github.com/ceph/ceph/pull/19024>`_, Kefu Chai)
+* tests: qa/tasks: run cosbench using the CBT task (`pr#21656 <https://github.com/ceph/ceph/pull/21656>`_, Neha Ojha)
+* tests: qa/tasks: update ceph-deploy task to use newer ceph-volume syntax (`pr#19244 <https://github.com/ceph/ceph/pull/19244>`_, Vasu Kulkarni)
+* tests: qa/tests: Add additional required ceph-ansible vars due to upstream changes (`pr#17757 <https://github.com/ceph/ceph/pull/17757>`_, Vasu Kulkarni)
+* tests: qa/tests: add ceph-deploy upgrade tests (`issue#20950 <http://tracker.ceph.com/issues/20950>`_, `pr#16826 <https://github.com/ceph/ceph/pull/16826>`_, Vasu Kulkarni)
+* tests: qa/tests: add openstack volume info + lvs for ceph-volume (`pr#20243 <https://github.com/ceph/ceph/pull/20243>`_, Vasu Kulkarni)
+* tests: qa/tests: Fix get_system_type failure due to invalid remote name (`pr#17650 <https://github.com/ceph/ceph/pull/17650>`_, Vasu Kulkarni)
+* tests: qa/tests: fix rbd pool creation for systemd tests (`pr#17536 <https://github.com/ceph/ceph/pull/17536>`_, Vasu Kulkarni)
+* tests: [qa/tests]: misc ceph-ansible fixes and udpate (`pr#17096 <https://github.com/ceph/ceph/pull/17096>`_, Vasu Kulkarni)
+* tests: qa/tests/rados: Remove unsupported 2-size-1-min-size config (`pr#17576 <https://github.com/ceph/ceph/pull/17576>`_, Vasu Kulkarni)
+* tests: qa/tests: use ceph-deploy stable branch for single node tests (`pr#20979 <https://github.com/ceph/ceph/pull/20979>`_, Vasu Kulkarni)
+* tests: qa/tests: Various whitelists for smoke suite (`issue#21376 <http://tracker.ceph.com/issues/21376>`_, `pr#17680 <https://github.com/ceph/ceph/pull/17680>`_, Vasu Kulkarni)
+* tests: qa/tests: Wip ceph deploy upgrade (`pr#17651 <https://github.com/ceph/ceph/pull/17651>`_, Vasu Kulkarni)
+* tests: qa/workunits/rados/test_large_omap_detection: Scrub pgs instead of OSDs (`pr#21410 <https://github.com/ceph/ceph/pull/21410>`_, Brad Hubbard)
+* tests: qa/workunits: silence py warnings for ceph-disk tests (`issue#22154 <http://tracker.ceph.com/issues/22154>`_, `pr#19075 <https://github.com/ceph/ceph/pull/19075>`_, Kefu Chai)
+* tests: rados: Copy payload in ceph_perf_msgr_client (`issue#22100 <http://tracker.ceph.com/issues/22100>`_, `pr#18862 <https://github.com/ceph/ceph/pull/18862>`_, Jeegn Chen)
+* tests: rados: Intializing members class StriperTest (`pr#16843 <https://github.com/ceph/ceph/pull/16843>`_, amitkuma)
+* tests: remove TestPGLog ASSERT_DEATH test (`issue#23504 <http://tracker.ceph.com/issues/23504>`_, `pr#21117 <https://github.com/ceph/ceph/pull/21117>`_, Nathan Cutler)
+* tests: run-standalone.sh improve error message (`pr#17093 <https://github.com/ceph/ceph/pull/17093>`_, David Zafman)
+* tests: run-standalone.sh skip core_pattern if already set (`pr#17098 <https://github.com/ceph/ceph/pull/17098>`_, David Zafman)
+* tests: test/admin_socket_output: add --vstart=path/to/asok option (`pr#20371 <https://github.com/ceph/ceph/pull/20371>`_, Kefu Chai)
+* tests: test/admin_socket_output: better error reporting (`pr#20409 <https://github.com/ceph/ceph/pull/20409>`_, Brad Hubbard)
+* tests: test/admin_socket_output: switch to std::experimental::filesystem (`pr#20307 <https://github.com/ceph/ceph/pull/20307>`_, Kefu Chai)
+* tests: test/ceph_test_objectstore: make settings update and restore less error prone (`pr#21145 <https://github.com/ceph/ceph/pull/21145>`_, Igor Fedotov)
+* tests: test: checking negative returns from creat() (`pr#18090 <https://github.com/ceph/ceph/pull/18090>`_, amitkuma)
+* tests: test/CMakeLists: disable test_pidfile.sh (`issue#20975 <http://tracker.ceph.com/issues/20975>`_, `pr#16977 <https://github.com/ceph/ceph/pull/16977>`_, Sage Weil)
+* tests: test/CMakeLists: disable test-pidfile.sh (`pr#17401 <https://github.com/ceph/ceph/pull/17401>`_, Sage Weil)
+* tests: test/coredumpctl: support freebsd (`pr#17447 <https://github.com/ceph/ceph/pull/17447>`_, Kefu Chai)
+* tests: test/dashboard: hardcode .coverage path to workaround tox bugs (`pr#21485 <https://github.com/ceph/ceph/pull/21485>`_, Kefu Chai)
+* tests: test/dashboard: specify workdir using tox.ini (`issue#23709 <http://tracker.ceph.com/issues/23709>`_, `pr#21416 <https://github.com/ceph/ceph/pull/21416>`_, Kefu Chai)
+* tests: test: Don't dump core when using EXPECT_DEATH (`pr#17390 <https://github.com/ceph/ceph/pull/17390>`_, Kefu Chai)
+* tests: test/fio: extend fio objectstore plugin to better simulate OSD behavior (`pr#19101 <https://github.com/ceph/ceph/pull/19101>`_, Igor Fedotov)
+* tests: test/fio: fix building of the fio_ceph_objectstore plugin (`pr#18332 <https://github.com/ceph/ceph/pull/18332>`_, Radoslaw Zarzynski)
+* tests: test: Fix and enable test_pidfile.sh (`issue#20770 <http://tracker.ceph.com/issues/20770>`_, `pr#16987 <https://github.com/ceph/ceph/pull/16987>`_, David Zafman)
+* tests: test: Fix ceph-objectstore-tool usage check (`pr#17785 <https://github.com/ceph/ceph/pull/17785>`_, David Zafman)
+* tests: test: fix misc fiemap testing (`issue#21716 <http://tracker.ceph.com/issues/21716>`_, `pr#18240 <https://github.com/ceph/ceph/pull/18240>`_, Kefu Chai, Ning Yao)
+* tests: test: Initialization of \*comp_racing_read class CopyFromOp (`pr#17369 <https://github.com/ceph/ceph/pull/17369>`_, Amit Kumar)
+* tests: test: Initializing ChunkReadOp members (`pr#19334 <https://github.com/ceph/ceph/pull/19334>`_, amitkuma)
+* tests: test/journal: Initialize member variable m_work_queue (`pr#17089 <https://github.com/ceph/ceph/pull/17089>`_, amitkuma)
+* tests: test/librados: be more tolerant with timed lock tests (`issue#20086 <http://tracker.ceph.com/issues/20086>`_, `pr#20161 <https://github.com/ceph/ceph/pull/20161>`_, Kefu Chai)
+* tests: test/librados: increase pgp_num along with pg_num (`issue#23763 <http://tracker.ceph.com/issues/23763>`_, `pr#21555 <https://github.com/ceph/ceph/pull/21555>`_, Kefu Chai)
+* tests: test/librados: s/invoke_result_t/result_of_t/ (`pr#20379 <https://github.com/ceph/ceph/pull/20379>`_, Kefu Chai)
+* tests: test/librados_test_stub: pass snap context to zero op (`pr#17186 <https://github.com/ceph/ceph/pull/17186>`_, Mykola Golub)
+* tests: test/log: fix for crash with libc++ (`pr#20233 <https://github.com/ceph/ceph/pull/20233>`_, Casey Bodley)
+* tests: test: Make clearer by moving code out of loop (`pr#20759 <https://github.com/ceph/ceph/pull/20759>`_, David Zafman)
+* tests: test/objectstore/test_bluefs: cleanups (`pr#17909 <https://github.com/ceph/ceph/pull/17909>`_, Kefu Chai)
+* tests: test: only test dashboard_v2 when it is enabled (`pr#20777 <https://github.com/ceph/ceph/pull/20777>`_, Willem Jan Withagen)
+* tests: test: only test enabled python bindings (`pr#21293 <https://github.com/ceph/ceph/pull/21293>`_, Kefu Chai)
+* tests: test/osd: initialize Non-static class members in WeightedTestGenerator (`pr#15922 <https://github.com/ceph/ceph/pull/15922>`_, Jos Collin)
+* tests: test/osd: Non-static class members not initialized in UnsetRedirectOp (`pr#15921 <https://github.com/ceph/ceph/pull/15921>`_, Jos Collin)
+* tests: test/osd: silence warnings from -Wsign-compare (`pr#17027 <https://github.com/ceph/ceph/pull/17027>`_, Jos Collin)
+* tests: test: put new BlueStore tests un ifdef WITH_BLUESTORE (`pr#20576 <https://github.com/ceph/ceph/pull/20576>`_, Willem Jan Withagen)
+* tests: test:qa:infra - Run update daily and use bash (`pr#21218 <https://github.com/ceph/ceph/pull/21218>`_, David Galloway)
+* tests: test:qa:infra - teuthology crontab items as of 3/27/18 (`pr#21075 <https://github.com/ceph/ceph/pull/21075>`_, Yuri Weinstein)
+* tests: test: reduce the chance to have degraded PGs (`issue#22711 <http://tracker.ceph.com/issues/22711>`_, `pr#20046 <https://github.com/ceph/ceph/pull/20046>`_, Kefu Chai)
+* tests: test: remove distro_version assert in distro detect test (`pr#21052 <https://github.com/ceph/ceph/pull/21052>`_, Shengjing Zhu)
+* tests: test: Replace bc command with printf command (`pr#21013 <https://github.com/ceph/ceph/pull/21013>`_, David Zafman)
+* tests: test: silence warning from -Wsign-compare (`pr#17790 <https://github.com/ceph/ceph/pull/17790>`_, Jos Collin)
+* tests: test: silence warnings from -Wsign-compare (`pr#17962 <https://github.com/ceph/ceph/pull/17962>`_, Jos Collin)
+* tests: tests - Replaced requests for "centos 7.3" to centos_latest (`pr#19262 <https://github.com/ceph/ceph/pull/19262>`_, Yuri Weinstein)
+* tests: test/store_test: fix FTBFS as Sequencer is removed (`pr#20382 <https://github.com/ceph/ceph/pull/20382>`_, Kefu Chai)
+* tests: test/store_test: update Many4KWritesTest\* test cases to finalize with… (`pr#20230 <https://github.com/ceph/ceph/pull/20230>`_, Igor Fedotov)
+* tests: test/throttle: kill tests exercising dtor of Throttle classes (`pr#17442 <https://github.com/ceph/ceph/pull/17442>`_, Kefu Chai)
+* tests: test/unittest_bufferlist: check retvals of syscalls (`pr#18238 <https://github.com/ceph/ceph/pull/18238>`_, Kefu Chai)
+* tests: test/unittest_pg_log: silence gcc warning (`pr#17328 <https://github.com/ceph/ceph/pull/17328>`_, Kefu Chai)
+* tests: test: Use jq in a compatible way and for easier diff analysis (`pr#21450 <https://github.com/ceph/ceph/pull/21450>`_, David Zafman)
+* tests: test: Whitelist corrections (`pr#22167 <https://github.com/ceph/ceph/pull/22167>`_, David Zafman)
+* tests,tools: crushtool: print error message to stderr not dout(1) (`issue#21758 <http://tracker.ceph.com/issues/21758>`_, `pr#18242 <https://github.com/ceph/ceph/pull/18242>`_, Kefu Chai)
+* tests: unittest_crypto: Don't exceed limit for getentropy (`pr#18505 <https://github.com/ceph/ceph/pull/18505>`_, Brad Hubbard)
+* tests: vstart: fix initial start when there is no ceph.conf (`pr#21019 <https://github.com/ceph/ceph/pull/21019>`_, Jianpeng Ma)
+* The Day Has Come! (`pr#19657 <https://github.com/ceph/ceph/pull/19657>`_, Adam C. Emerson)
+* tools: Align use of uint64_t in service_daemon::AttributeType (`pr#16938 <https://github.com/ceph/ceph/pull/16938>`_, James Page)
+* tools: ceph-disk: erase 110MB for nuking existing bluestore (`issue#22354 <http://tracker.ceph.com/issues/22354>`_, `pr#20400 <https://github.com/ceph/ceph/pull/20400>`_, Kefu Chai)
+* tools: ceph-disk: fix '--runtime' omission for ceph-osd service (`issue#21498 <http://tracker.ceph.com/issues/21498>`_, `pr#17904 <https://github.com/ceph/ceph/pull/17904>`_, Carl Xiong)
+* tools: ceph-disk: fix signed integer is greater than maximum when call major (`pr#19196 <https://github.com/ceph/ceph/pull/19196>`_, Song Shun)
+* tools: ceph-disk: include output of failed command in exception (`pr#20497 <https://github.com/ceph/ceph/pull/20497>`_, Kefu Chai)
+* tools: ceph-disk: more precise error message when a disk is specified (`pr#18018 <https://github.com/ceph/ceph/pull/18018>`_, Kefu Chai)
+* tools: ceph-disk: reduce the scope of activate_lock (`pr#20114 <https://github.com/ceph/ceph/pull/20114>`_, zhaokun)
+* tools: ceph-disk: retry on OSError (`issue#21728 <http://tracker.ceph.com/issues/21728>`_, `pr#18162 <https://github.com/ceph/ceph/pull/18162>`_, Kefu Chai)
+* tools: ceph-disk: unlock all partitions when activate (`pr#17363 <https://github.com/ceph/ceph/pull/17363>`_, Kefu Chai)
+* tools: ceph-disk: write log to /var/log/ceph not to /var/run/ceph (`pr#18375 <https://github.com/ceph/ceph/pull/18375>`_, Kefu Chai)
+* tools: ceph: fixes for "tell <service>.\*" command (`issue#21230 <http://tracker.ceph.com/issues/21230>`_, `pr#17463 <https://github.com/ceph/ceph/pull/17463>`_, Kefu Chai)
+* tools: ceph-kvstore-tool: make it a bit more friendly (`pr#21477 <https://github.com/ceph/ceph/pull/21477>`_, Sage Weil)
+* tools: ceph-kvstore-tool: use unique_ptr<> to manage the lifecycle of bluestore (`pr#18221 <https://github.com/ceph/ceph/pull/18221>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: Add option "dump-import" to examine an export (`issue#22086 <http://tracker.ceph.com/issues/22086>`_, `pr#19368 <https://github.com/ceph/ceph/pull/19368>`_, David Zafman)
+* tools: ceph-objectstore-tool: Fix set-size to clear data_digest if changing … (`pr#18885 <https://github.com/ceph/ceph/pull/18885>`_, David Zafman)
+* tools: ceph-objectstore-tool: "$OBJ get-omaphdr" and "$OBJ list-omap" scan all pgs instead of using specific pg (`issue#21327 <http://tracker.ceph.com/issues/21327>`_, `pr#17985 <https://github.com/ceph/ceph/pull/17985>`_, David Zafman)
+* tools: ceph-objectstore-tool: skip object with generated version (`pr#18507 <https://github.com/ceph/ceph/pull/18507>`_, huangjun)
+* tools: ceph-syn: silence clang analyzer warning (`pr#18577 <https://github.com/ceph/ceph/pull/18577>`_, Kefu Chai)
+* tools: ceph-volume: Use a delimited CLI output parser instead of JSON (`pr#17097 <https://github.com/ceph/ceph/pull/17097>`_, Alfredo Deza)
+* tools: cleanup: rip out ceph-rest-api (`issue#21264 <http://tracker.ceph.com/issues/21264>`_, `issue#22457 <http://tracker.ceph.com/issues/22457>`_, `pr#17530 <https://github.com/ceph/ceph/pull/17530>`_, Nathan Cutler)
+* tools: correct total size formatting (`pr#21641 <https://github.com/ceph/ceph/pull/21641>`_, Zheng Yin)
+* tools: crushtool: add --add-bucket and --move options (`pr#20183 <https://github.com/ceph/ceph/pull/20183>`_, Kefu Chai)
+* tools: FreeBSD basic getopt does not use --options (`pr#21148 <https://github.com/ceph/ceph/pull/21148>`_, Willem Jan Withagen)
+* tools: Initialization of \*server, command variables (`pr#17135 <https://github.com/ceph/ceph/pull/17135>`_, amitkuma)
+* tools: make rados get/put/append command help txt clear (`issue#22958 <http://tracker.ceph.com/issues/22958>`_, `pr#20363 <https://github.com/ceph/ceph/pull/20363>`_, lvshuhua)
+* tools: Modify "rados df" header's alignment (`pr#17549 <https://github.com/ceph/ceph/pull/17549>`_, iliul)
+* tools: rados add a cli option to clear omap keys (`issue#22255 <http://tracker.ceph.com/issues/22255>`_, `pr#19180 <https://github.com/ceph/ceph/pull/19180>`_, Abhishek Lekshmanan)
+* tools: rados/tool: fixup rados stat command hint (`pr#16983 <https://github.com/ceph/ceph/pull/16983>`_, huanwen ren)
+* tools: script: build-integration-branch: avoid Unicode error (`issue#24003 <http://tracker.ceph.com/issues/24003>`_, `pr#21918 <https://github.com/ceph/ceph/pull/21918>`_, Nathan Cutler)
+* tools: script: ceph-release-notes: minor fixes for split_component (`pr#16605 <https://github.com/ceph/ceph/pull/16605>`_, Abhishek Lekshmanan)
+* tools: Special scrub handling of hinfo_key errors (`issue#23428 <http://tracker.ceph.com/issues/23428>`_, `issue#23364 <http://tracker.ceph.com/issues/23364>`_, `pr#20947 <https://github.com/ceph/ceph/pull/20947>`_, David Zafman)
+* tools: src/vstart.sh: default os to filestore for FreeBSD (`pr#17454 <https://github.com/ceph/ceph/pull/17454>`_, xie xingguo)
+* tools: stop.sh: add ceph configure file location (`pr#20888 <https://github.com/ceph/ceph/pull/20888>`_, Jianpeng Ma)
+* tools: tools/ceph-conf: dump parsed config in plain text or as json (`issue#21862 <http://tracker.ceph.com/issues/21862>`_, `pr#18350 <https://github.com/ceph/ceph/pull/18350>`_, Piotr Dałek)
+* tools: tools/ceph_monstore_tool: include mgrmap in initial paxos epoch (`issue#22266 <http://tracker.ceph.com/issues/22266>`_, `pr#19780 <https://github.com/ceph/ceph/pull/19780>`_, Kefu Chai)
+* tools: tools/ceph_monstore_tool: rebuild initial mgrmap also (`issue#22266 <http://tracker.ceph.com/issues/22266>`_, `pr#19238 <https://github.com/ceph/ceph/pull/19238>`_, Kefu Chai)
+* tools: tools/ceph-objectstore-tool: command to trim the pg log (`issue#23242 <http://tracker.ceph.com/issues/23242>`_, `pr#20786 <https://github.com/ceph/ceph/pull/20786>`_, Josh Durgin, David Zafman)
+* tools: tools/ceph_objectstore_tool: fix 'dup' unable to duplicate meta PG (`pr#17572 <https://github.com/ceph/ceph/pull/17572>`_, xie xingguo)
+* tools: tools/rados: improve the ls command usage (`pr#21553 <https://github.com/ceph/ceph/pull/21553>`_, Li Wang)
+* tools:  tools: rados: make -f be --format for consistency with ceph tool (`issue#15904 <http://tracker.ceph.com/issues/15904>`_, `pr#20147 <https://github.com/ceph/ceph/pull/20147>`_, Nathan Cutler)
+* tools: tools/rados: use the monotonic clock in rados bench (`issue#21375 <http://tracker.ceph.com/issues/21375>`_, `pr#18588 <https://github.com/ceph/ceph/pull/18588>`_, Mohamad Gebai)
+* tools: update monstore tool for fsmap, mgrmap (`issue#21577 <http://tracker.ceph.com/issues/21577>`_, `pr#18005 <https://github.com/ceph/ceph/pull/18005>`_, John Spray)
+* tools: Use --no-mon-config so ceph_objectstore_tool.py test doesn't hang (`pr#21274 <https://github.com/ceph/ceph/pull/21274>`_, David Zafman)
+* tools: vstart.sh: move rgw configuration to client.rgw section (`pr#18331 <https://github.com/ceph/ceph/pull/18331>`_, Yan Jun)
+* tools: vstart.sh: use bluestore as default osd objectstore backend (`pr#17100 <https://github.com/ceph/ceph/pull/17100>`_, mychoxin)
+* vstart: fix option (due to quotes) and allow disabling dashboard (`issue#23345 <http://tracker.ceph.com/issues/23345>`_, `pr#20986 <https://github.com/ceph/ceph/pull/20986>`_, Joao Eduardo Luis)
+* vstart.sh: fix a typo (`pr#18729 <https://github.com/ceph/ceph/pull/18729>`_, iliul)
+* vstart.sh: Fix help text in vstart.sh (`pr#21071 <https://github.com/ceph/ceph/pull/21071>`_, Marc Koderer)
+* vstart.sh: quote cmd params when display executing cmd (`pr#17057 <https://github.com/ceph/ceph/pull/17057>`_, Jiaying Ren)
+* vstart.sh: quote command only when necessary (`pr#18181 <https://github.com/ceph/ceph/pull/18181>`_, Kefu Chai)
+* vstart.sh: should quote the parameters to get them quoted (`pr#18523 <https://github.com/ceph/ceph/pull/18523>`_, Kefu Chai)
+* vstart.sh: simplify the objectstore related logic (`pr#17749 <https://github.com/ceph/ceph/pull/17749>`_, Kefu Chai)
diff --git a/doc/releases/nautilus.rst b/doc/releases/nautilus.rst
new file mode 100644
index 000000000..fd27f7124
--- /dev/null
+++ b/doc/releases/nautilus.rst
@@ -0,0 +1,5155 @@
+========
+Nautilus
+========
+
+Nautilus is the 14th stable release of Ceph.  It is named after the
+nautilus, a family of cephalopods characterized by a whorled shell.
+
+v14.2.22 Nautilus
+=================
+
+This is the 22nd and likely the last backport release in the Nautilus series.
+Ultimately, we recommend all users upgrade to newer Ceph releases.
+
+Notable Changes
+---------------
+
+* This release sets ``bluefs_buffered_io`` to true by default to improve performance
+  for metadata heavy workloads. Enabling this option has been reported to
+  occasionally cause excessive kernel swapping under certain workloads.
+  Currently, the most consistent performing combination is to enable
+  bluefs_buffered_io and disable system level swap.
+
+* The default value of ``bluestore_cache_trim_max_skip_pinned`` has been
+  increased to 1000 to control memory growth due to onodes.
+
+* Several other bug fixes in BlueStore, including a fix for an unexpected
+  ENOSPC bug in Avl/Hybrid allocators.
+
+* The trimming logic in the monitor has been made dynamic, with the
+  introduction of ``paxos_service_trim_max_multiplier``, a factor by which
+  ``paxos_service_trim_max`` is multiplied to make trimming faster,
+  when required. Setting it to 0 disables the upper bound check for trimming
+  and makes the monitors trim at the maximum rate.
+
+* A ``--max <n>`` option is available with the ``osd ok-to-stop`` command to
+  provide up to N OSDs that can be stopped together without making PGs
+  unavailable.
+
+* OSD: the option ``osd_fast_shutdown_notify_mon`` has been introduced to allow
+  the OSD to notify the monitor it is shutting down even if ``osd_fast_shutdown``
+  is enabled. This helps with the monitor logs on larger clusters, that may get
+  many 'osd.X reported immediately failed by osd.Y' messages, and confuse tools.
+
+* A long-standing bug that prevented 32-bit and 64-bit client/server
+  interoperability under msgr v2 has been fixed.  In particular, mixing armv7l
+  (armhf) and x86_64 or aarch64 servers in the same cluster now works.
+
+Changelog
+---------
+
+* PendingReleaseNotes: note about 14.2.18 mgr fixes (`pr#40121 <https://github.com/ceph/ceph/pull/40121>`_, Josh Durgin)
+* bind on loopback address if no other addresses are available (`pr#41137 <https://github.com/ceph/ceph/pull/41137>`_, Kefu Chai, Matthew Oliver)
+* build python extensions using distutils (`pr#41167 <https://github.com/ceph/ceph/pull/41167>`_, Kefu Chai)
+* ceph-monstore-tool: use a large enough paxos/{first,last}_committed (`issue#38219 <http://tracker.ceph.com/issues/38219>`_, `pr#41874 <https://github.com/ceph/ceph/pull/41874>`_, Kefu Chai)
+* ceph-volume: disable cache for blkid calls (`pr#41114 <https://github.com/ceph/ceph/pull/41114>`_, Rafał Wądołowski)
+* ceph-volume: fix batch report and respect ceph.conf config values (`pr#41716 <https://github.com/ceph/ceph/pull/41716>`_, Andrew Schoen)
+* ceph-volume: fix batch report and respect ceph.conf config values (`pr#41713 <https://github.com/ceph/ceph/pull/41713>`_, Andrew Schoen)
+* ceph-volume: implement bluefs volume migration (`pr#41676 <https://github.com/ceph/ceph/pull/41676>`_, Kefu Chai, Igor Fedotov)
+* ceph.spec.in: Enable tcmalloc on IBM Power and Z (`pr#40283 <https://github.com/ceph/ceph/pull/40283>`_, Nathan Cutler, Yaakov Selkowitz)
+* cephfs: client: add ability to lookup snapped inodes by inode number (`pr#40769 <https://github.com/ceph/ceph/pull/40769>`_, Jeff Layton, Xiubo Li)
+* cephfs: client: only check pool permissions for regular files (`pr#40730 <https://github.com/ceph/ceph/pull/40730>`_, Xiubo Li)
+* cephfs: client: wake up the front pos waiter (`pr#40865 <https://github.com/ceph/ceph/pull/40865>`_, Xiubo Li)
+* client: Fix executeable access check for the root user (`pr#41297 <https://github.com/ceph/ceph/pull/41297>`_, Kotresh HR)
+* client: fire the finish_cap_snap() after buffer being flushed (`pr#40722 <https://github.com/ceph/ceph/pull/40722>`_, Xiubo Li)
+* cls/rgw: look for plain entries in non-ascii plain namespace too (`pr#41776 <https://github.com/ceph/ceph/pull/41776>`_, Mykola Golub)
+* cmake,zstd,debian: allow use libzstd in system (`pr#40516 <https://github.com/ceph/ceph/pull/40516>`_, Kefu Chai, Bryan Stillwell, Dan van der Ster)
+* cmake: build static libs if they are internal ones (`pr#39903 <https://github.com/ceph/ceph/pull/39903>`_, Kefu Chai)
+* cmake: detect gettid() presense (`pr#40333 <https://github.com/ceph/ceph/pull/40333>`_, Igor Fedotov)
+* cmake: set empty RPATH for some test executables (`pr#40619 <https://github.com/ceph/ceph/pull/40619>`_, Nathan Cutler, Kefu Chai)
+* common/buffer: adjust align before calling posix_memalign() (`pr#41246 <https://github.com/ceph/ceph/pull/41246>`_, Ilya Dryomov)
+* common/ipaddr: skip loopback interfaces named 'lo' and test it (`pr#40423 <https://github.com/ceph/ceph/pull/40423>`_, Dan van der Ster)
+* common/mempool: only fail tests if sharding is very bad (`pr#40567 <https://github.com/ceph/ceph/pull/40567>`_, singuliere)
+* common/options/global.yaml.in: increase default value of bluestore_cache_trim_max_skip_pinned (`pr#40920 <https://github.com/ceph/ceph/pull/40920>`_, Neha Ojha)
+* common/options: bluefs_buffered_io=true by default (`pr#40393 <https://github.com/ceph/ceph/pull/40393>`_, Dan van der Ster)
+* common: Fix assertion when disabling and re-enabling clog_to_monitors (`pr#39912 <https://github.com/ceph/ceph/pull/39912>`_, Gerald Yang)
+* common: remove log_early configuration option (`pr#40549 <https://github.com/ceph/ceph/pull/40549>`_, Changcheng Liu)
+* crush/CrushLocation: do not print logging message in constructor (`pr#40750 <https://github.com/ceph/ceph/pull/40750>`_, Alex Wu)
+* crush/CrushWrapper: update shadow trees on update_item() (`pr#39920 <https://github.com/ceph/ceph/pull/39920>`_, Sage Weil)
+* debian/ceph-common.postinst: do not chown cephadm log dirs (`pr#40698 <https://github.com/ceph/ceph/pull/40698>`_, Sage Weil)
+* debian/control: add missing commas, use python3 packages for "make check" on focal (`pr#40485 <https://github.com/ceph/ceph/pull/40485>`_, Kefu Chai, Alfredo Deza)
+* install-deps.sh: remove existing ceph-libboost of different version (`pr#40287 <https://github.com/ceph/ceph/pull/40287>`_, Kefu Chai)
+* libcephfs: ignore restoring the open files limit (`pr#41593 <https://github.com/ceph/ceph/pull/41593>`_, Xiubo Li)
+* librbd: allow interrupted trash move request to be restarted (`pr#40675 <https://github.com/ceph/ceph/pull/40675>`_, Jason Dillaman)
+* librbd: don't stop at the first unremovable image when purging (`pr#41662 <https://github.com/ceph/ceph/pull/41662>`_, Ilya Dryomov)
+* librbd: fix sporadic failures in TestMigration.StressLive (`pr#41788 <https://github.com/ceph/ceph/pull/41788>`_, Jason Dillaman)
+* librbd: race when disabling object map with overlapping in-flight writes (`pr#41787 <https://github.com/ceph/ceph/pull/41787>`_, Jason Dillaman)
+* make-dist: refuse to run if script path contains a colon (`pr#41088 <https://github.com/ceph/ceph/pull/41088>`_, Nathan Cutler)
+* mds: do not trim the inodes from the lru list in standby_replay (`pr#41144 <https://github.com/ceph/ceph/pull/41144>`_, Xiubo Li)
+* mds: fix race of fetching large dirfrag (`pr#40720 <https://github.com/ceph/ceph/pull/40720>`_, Erqi Chen)
+* mds: send scrub status to ceph-mgr only when scrub is running (`issue#45349 <http://tracker.ceph.com/issues/45349>`_, `pr#36183 <https://github.com/ceph/ceph/pull/36183>`_, Kefu Chai, Venky Shankar)
+* mds: trim cache regularly for standby-replay (`pr#40744 <https://github.com/ceph/ceph/pull/40744>`_, Patrick Donnelly)
+* mgr/ActivePyModules.cc: always release GIL before attempting to acquire a lock (`pr#40047 <https://github.com/ceph/ceph/pull/40047>`_, Kefu Chai)
+* mgr/Dashboard: Remove erroneous elements in hosts-overview Grafana dashboard (`pr#41650 <https://github.com/ceph/ceph/pull/41650>`_, Malcolm Holmes)
+* mgr/PyModule: put mgr_module_path before Py_GetPath() (`pr#40753 <https://github.com/ceph/ceph/pull/40753>`_, Kefu Chai)
+* mgr/dashboard: Fix for alert notification message being undefined (`pr#40590 <https://github.com/ceph/ceph/pull/40590>`_, Nizamudeen A)
+* mgr/dashboard: Fix missing root path of each session for CephFS (`pr#39869 <https://github.com/ceph/ceph/pull/39869>`_, Yongseok Oh)
+* mgr/dashboard: Monitoring alert badge includes suppressed alerts (`pr#39511 <https://github.com/ceph/ceph/pull/39511>`_, Aashish Sharma)
+* mgr/dashboard: Remove username, password fields from Manager Modules/dashboard,influx (`pr#40490 <https://github.com/ceph/ceph/pull/40490>`_, Aashish Sharma)
+* mgr/dashboard: Revoke read-only user's access to Manager modules (`pr#40650 <https://github.com/ceph/ceph/pull/40650>`_, Nizamudeen A)
+* mgr/dashboard: debug nodeenv hangs (`pr#40818 <https://github.com/ceph/ceph/pull/40818>`_, Ernesto Puerta)
+* mgr/dashboard: decouple unit tests from build artifacts (`pr#40547 <https://github.com/ceph/ceph/pull/40547>`_, Alfonso Martínez)
+* mgr/dashboard: encode non-ascii string before passing it to exec_cmd() (`pr#40522 <https://github.com/ceph/ceph/pull/40522>`_, Kefu Chai)
+* mgr/dashboard: filesystem pool size should use stored stat (`pr#41021 <https://github.com/ceph/ceph/pull/41021>`_, Avan Thakkar)
+* mgr/dashboard: fix API docs link (`pr#41521 <https://github.com/ceph/ceph/pull/41521>`_, Avan Thakkar)
+* mgr/dashboard: fix OSDs Host details/overview grafana graphs (`issue#49769 <http://tracker.ceph.com/issues/49769>`_, `pr#41531 <https://github.com/ceph/ceph/pull/41531>`_, Alfonso Martínez, Michael Wodniok)
+* mgr/dashboard: fix base-href: revert it to previous approach (`pr#41253 <https://github.com/ceph/ceph/pull/41253>`_, Avan Thakkar)
+* mgr/dashboard: fix bucket objects and size calculations (`pr#41648 <https://github.com/ceph/ceph/pull/41648>`_, Avan Thakkar)
+* mgr/dashboard: fix dashboard instance ssl certificate functionality (`pr#40003 <https://github.com/ceph/ceph/pull/40003>`_, Avan Thakkar)
+* mgr/dashboard: grafana panels for rgw multisite sync performance (`pr#41386 <https://github.com/ceph/ceph/pull/41386>`_, Alfonso Martínez)
+* mgr/dashboard: python 2: fix error when non-ASCII password (`pr#40610 <https://github.com/ceph/ceph/pull/40610>`_, Alfonso Martínez)
+* mgr/dashboard: report mgr fsid (`pr#39853 <https://github.com/ceph/ceph/pull/39853>`_, Ernesto Puerta)
+* mgr/dashboard: show partially deleted RBDs (`pr#41738 <https://github.com/ceph/ceph/pull/41738>`_, Tatjana Dehler)
+* mgr/dashboard: test prometheus rules through promtool (`pr#39984 <https://github.com/ceph/ceph/pull/39984>`_, Aashish Sharma, Kefu Chai)
+* mgr/progress: ensure progress stays between [0,1] (`pr#41310 <https://github.com/ceph/ceph/pull/41310>`_, Dan van der Ster)
+* mgr/telemetry: check if 'ident' channel is active (`pr#39923 <https://github.com/ceph/ceph/pull/39923>`_, Yaarit Hatuka)
+* mgr/telemetry: pass leaderboard flag even w/o ident (`pr#41839 <https://github.com/ceph/ceph/pull/41839>`_, Sage Weil)
+* mgr/volumes: Retain suid guid bits in clone (`pr#40270 <https://github.com/ceph/ceph/pull/40270>`_, Kotresh HR)
+* mgr: add --max <n> to 'osd ok-to-stop' command (`pr#40676 <https://github.com/ceph/ceph/pull/40676>`_, Sage Weil, Xuehan Xu)
+* mgr: add mon metada using type of "mon" (`pr#40359 <https://github.com/ceph/ceph/pull/40359>`_, Kefu Chai)
+* mon/ConfigMap: fix stray option leak (`pr#40299 <https://github.com/ceph/ceph/pull/40299>`_, Sage Weil)
+* mon/MonClient: reset authenticate_err in _reopen_session() (`pr#41016 <https://github.com/ceph/ceph/pull/41016>`_, Ilya Dryomov)
+* mon/MonClient: tolerate a rotating key that is slightly out of date (`pr#41448 <https://github.com/ceph/ceph/pull/41448>`_, Ilya Dryomov)
+* mon/OSDMonitor: drop stale failure_info after a grace period (`pr#41213 <https://github.com/ceph/ceph/pull/41213>`_, Kefu Chai)
+* mon/OSDMonitor: drop stale failure_info even if can_mark_down() (`pr#41519 <https://github.com/ceph/ceph/pull/41519>`_, Kefu Chai)
+* mon: Modifying trim logic to change paxos_service_trim_max dynamically (`pr#41099 <https://github.com/ceph/ceph/pull/41099>`_, Aishwarya Mathuria)
+* mon: ensure progress is [0,1] before printing (`pr#41098 <https://github.com/ceph/ceph/pull/41098>`_, Dan van der Ster)
+* mon: load stashed map before mkfs monmap (`pr#41762 <https://github.com/ceph/ceph/pull/41762>`_, Dan van der Ster)
+* monmaptool: Don't call set_port on an invalid address (`pr#40700 <https://github.com/ceph/ceph/pull/40700>`_, Brad Hubbard, Kefu Chai)
+* os/FileStore: don't propagate split/merge error to "create"/"remove" (`pr#40987 <https://github.com/ceph/ceph/pull/40987>`_, Mykola Golub)
+* os/FileStore: fix to handle readdir error correctly (`pr#41238 <https://github.com/ceph/ceph/pull/41238>`_, Misono Tomohiro)
+* os/bluestore/BlueFS: do not _flush_range deleted files (`pr#40752 <https://github.com/ceph/ceph/pull/40752>`_, weixinwei)
+* os/bluestore/BlueFS: use iterator_impl::copy instead of bufferlist::c_str() to avoid bufferlist rebuild (`pr#39883 <https://github.com/ceph/ceph/pull/39883>`_, weixinwei)
+* os/bluestore: be more verbose in _open_super_meta by default (`pr#41060 <https://github.com/ceph/ceph/pull/41060>`_, Igor Fedotov)
+* os/bluestore: do not count pinned entries as trimmed ones (`pr#41173 <https://github.com/ceph/ceph/pull/41173>`_, Igor Fedotov)
+* os/bluestore: fix unexpected ENOSPC in Avl/Hybrid allocators (`pr#41673 <https://github.com/ceph/ceph/pull/41673>`_, Igor Fedotov)
+* os/bluestore: introduce multithireading sync for bluestore's repairer (`pr#41749 <https://github.com/ceph/ceph/pull/41749>`_, Igor Fedotov)
+* os/bluestore: tolerate zero length for allocators' init\_[add/rm]_free() (`pr#41750 <https://github.com/ceph/ceph/pull/41750>`_, Igor Fedotov)
+* osd/PG.cc: handle removal of pgmeta object (`pr#41682 <https://github.com/ceph/ceph/pull/41682>`_, Neha Ojha)
+* osd/PeeringState: fix acting_set_writeable min_size check (`pr#41611 <https://github.com/ceph/ceph/pull/41611>`_, Dan van der Ster)
+* osd: add osd_fast_shutdown_notify_mon option (default false) (`issue#46978 <http://tracker.ceph.com/issues/46978>`_, `pr#40014 <https://github.com/ceph/ceph/pull/40014>`_, Mauricio Faria de Oliveira)
+* osd: compute OSD's space usage ratio via raw space utilization (`pr#41111 <https://github.com/ceph/ceph/pull/41111>`_, Igor Fedotov)
+* osd: do not dump an osd multiple times (`pr#40747 <https://github.com/ceph/ceph/pull/40747>`_, Xue Yantao)
+* pybind/ceph_daemon: do not fail if prettytable is not available (`pr#40335 <https://github.com/ceph/ceph/pull/40335>`_, Kefu Chai)
+* pybind/cephfs: DT_REG and DT_LNK values are wrong (`pr#40704 <https://github.com/ceph/ceph/pull/40704>`_, Varsha Rao)
+* pybind/mgr/balancer/module.py: assign weight-sets to all buckets before balancing (`pr#40128 <https://github.com/ceph/ceph/pull/40128>`_, Neha Ojha)
+* pybind/mgr/volumes: deadlock on async job hangs finisher thread (`pr#41394 <https://github.com/ceph/ceph/pull/41394>`_, Patrick Donnelly)
+* pybind/rados: should pass "name" to cstr() (`pr#41318 <https://github.com/ceph/ceph/pull/41318>`_, Kefu Chai)
+* pybind: volume_client handle purge of directory names encoded in utf-8 (`pr#36679 <https://github.com/ceph/ceph/pull/36679>`_, Jose Castro Leon)
+* qa/tasks/mgr/test_progress: fix wait_until_equal (`pr#39397 <https://github.com/ceph/ceph/pull/39397>`_, Kamoltat, Ricardo Dias)
+* qa/tasks/qemu: precise repos have been archived (`pr#41641 <https://github.com/ceph/ceph/pull/41641>`_, Ilya Dryomov)
+* qa/tasks/vstart_runner.py: start max required mgrs (`pr#40751 <https://github.com/ceph/ceph/pull/40751>`_, Alfonso Martínez)
+* qa/tests: added client-upgrade-nautilus-pacific tests (`pr#39818 <https://github.com/ceph/ceph/pull/39818>`_, Yuri Weinstein)
+* qa/tests: advanced nautilus initial version to 14.2.20 (`pr#41227 <https://github.com/ceph/ceph/pull/41227>`_, Yuri Weinstein)
+* qa/upgrade: disable update_features test_notify with older client as lockowner (`pr#41513 <https://github.com/ceph/ceph/pull/41513>`_, Deepika Upadhyay)
+* qa: add sleep for blocklisting to take effect (`pr#40714 <https://github.com/ceph/ceph/pull/40714>`_, Patrick Donnelly)
+* qa: bump osd heartbeat grace for ffsb workload (`pr#40713 <https://github.com/ceph/ceph/pull/40713>`_, Nathan Cutler)
+* qa: delete all fs during tearDown (`pr#40709 <https://github.com/ceph/ceph/pull/40709>`_, Patrick Donnelly)
+* qa: krbd_blkroset.t: update for separate hw and user read-only flags (`pr#40212 <https://github.com/ceph/ceph/pull/40212>`_, Ilya Dryomov)
+* qa: vstart_runner: TypeError: lstat: path should be string, bytes or os.PathLike, not NoneType (`pr#41485 <https://github.com/ceph/ceph/pull/41485>`_, Patrick Donnelly)
+* rbd-mirror: image replayer stop might race with instance replayer shut down (`pr#41792 <https://github.com/ceph/ceph/pull/41792>`_, Mykola Golub, Jason Dillaman)
+* rgw : catch non int exception (`pr#40356 <https://github.com/ceph/ceph/pull/40356>`_, caolei)
+* rgw/http: add timeout to http client (`pr#40667 <https://github.com/ceph/ceph/pull/40667>`_, Yuval Lifshitz)
+* rgw: Added caching for S3 credentials retrieved from keystone (`pr#41158 <https://github.com/ceph/ceph/pull/41158>`_, James Weaver)
+* rgw: Use correct bucket info when put or get large object with swift (`pr#40106 <https://github.com/ceph/ceph/pull/40106>`_, zhiming zhang, yupeng chen)
+* rgw: allow rgw-orphan-list to handle intermediate files w/ binary data (`pr#39767 <https://github.com/ceph/ceph/pull/39767>`_, J. Eric Ivancich)
+* rgw: beast frontend uses 512k mprotected coroutine stacks (`pr#39947 <https://github.com/ceph/ceph/pull/39947>`_, Yaakov Selkowitz, Mauricio Faria de Oliveira, Daniel Gryniewicz, Casey Bodley)
+* rgw: check object locks in multi-object delete (`issue#47586 <http://tracker.ceph.com/issues/47586>`_, `pr#41164 <https://github.com/ceph/ceph/pull/41164>`_, Mark Houghton, Matt Benjamin)
+* rgw: during reshard lock contention, adjust logging (`pr#41156 <https://github.com/ceph/ceph/pull/41156>`_, J. Eric Ivancich)
+* rgw: limit rgw_gc_max_objs to RGW_SHARDS_PRIME_1 (`pr#40670 <https://github.com/ceph/ceph/pull/40670>`_, Rafał Wądołowski)
+* rgw: radoslist incomplete multipart parts marker (`pr#40827 <https://github.com/ceph/ceph/pull/40827>`_, J. Eric Ivancich)
+* rgw: return ERR_NO_SUCH_BUCKET early while evaluating bucket policy (`issue#38420 <http://tracker.ceph.com/issues/38420>`_, `pr#40668 <https://github.com/ceph/ceph/pull/40668>`_, Abhishek Lekshmanan)
+* rgw: return error when trying to copy encrypted object without key (`pr#40671 <https://github.com/ceph/ceph/pull/40671>`_, Ilsoo Byun)
+* rgw: tooling to locate rgw objects with missing rados components (`pr#39771 <https://github.com/ceph/ceph/pull/39771>`_, Michael Kidd, J. Eric Ivancich)
+* run-make-check.sh: let ctest generate XML output (`pr#40407 <https://github.com/ceph/ceph/pull/40407>`_, Kefu Chai)
+* src/global/signal_handler.h: fix preprocessor logic for alpine (`pr#39942 <https://github.com/ceph/ceph/pull/39942>`_, Duncan Bellamy)
+* test/TestOSDScrub: fix mktime() error (`pr#40621 <https://github.com/ceph/ceph/pull/40621>`_, luo rixin)
+* test/pybind: s/nosetests/python3/ (`pr#40536 <https://github.com/ceph/ceph/pull/40536>`_, Kefu Chai)
+* test/rgw: test_datalog_autotrim filters out new entries (`pr#40674 <https://github.com/ceph/ceph/pull/40674>`_, Casey Bodley)
+* test: use std::atomic<bool> instead of volatile for cb_done var (`pr#40701 <https://github.com/ceph/ceph/pull/40701>`_, Jeff Layton)
+* tests: ceph_test_rados_api_watch_notify: Allow for reconnect (`pr#40697 <https://github.com/ceph/ceph/pull/40697>`_, Brad Hubbard)
+* vstart.sh: disable "auth_allow_insecure_global_id_reclaim" (`pr#40959 <https://github.com/ceph/ceph/pull/40959>`_, Kefu Chai)
+
+
+v14.2.21 Nautilus
+=================
+
+This is a hotfix release addressing a number of security issues and regressions. We recommend all users update to this release.
+
+Changelog
+---------
+
+* mgr/dashboard: fix base-href: revert it to previous approach (`issue#50684 <https://tracker.ceph.com/issues/50684>`_, Avan Thakkar)
+* mgr/dashboard: fix cookie injection issue (:ref:`CVE-2021-3509`, Ernesto Puerta)
+* rgw: RGWSwiftWebsiteHandler::is_web_dir checks empty subdir_name (:ref:`CVE-2021-3531`, Felix Huettner)
+* rgw: sanitize \r in s3 CORSConfiguration's ExposeHeader (:ref:`CVE-2021-3524`, Sergey Bobrov, Casey Bodley)
+
+
+v14.2.20 Nautilus
+=================
+
+This is the 20th bugfix release in the Nautilus stable series.  It addresses a
+security vulnerability in the Ceph authentication framework.
+
+We recommend all Nautilus users upgrade.
+
+Security fixes
+--------------
+
+* This release includes a security fix that ensures the global_id
+  value (a numeric value that should be unique for every authenticated
+  client or daemon in the cluster) is reclaimed after a network
+  disconnect or ticket renewal in a secure fashion.  Two new health
+  alerts may appear during the upgrade indicating that there are
+  clients or daemons that are not yet patched with the appropriate
+  fix.
+
+  It is possible to disable the health alerts around insecure clients::
+
+    ceph config set mon mon_warn_on_insecure_global_id_reclaim false
+    ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
+
+  However, if you disable these alerts, we strongly recommend that you
+  follow up by removing these settings after clients have been
+  upgraded or after upgrading to Octopus.  (Starting in Octopus, these
+  health alerts can be muted for a specific period of time.)
+
+  For more information, see :ref:`CVE-2021-20288`.
+
+
+v14.2.19 Nautilus
+=================
+
+This is the 19th update to the Ceph Nautilus release series. This is a hotfix
+release to prevent daemons from binding to loopback network interfaces. All
+nautilus users are advised to upgrade to this release.
+
+Notable Changes
+---------------
+
+* This release fixes a regression introduced in v14.2.17 whereby in certain environments, OSDs will bind to 127.0.0.1.  See `issue#49938 <https://tracker.ceph.com/issues/49938>`_.
+
+Changelog
+---------
+
+* common/ipaddr: also skip just `lo` (`pr#40423 <https://github.com/ceph/ceph/pull/40423>`_, Dan van der Ster)
+
+
+v14.2.18 Nautilus
+=================
+
+This is the 18th backport release in the Nautilus series. It fixes a regression
+introduced in 14.2.17 in which the manager module tries to use a couple python 
+modules that do not exist in some environments. We recommend users to 
+update to this release.
+
+Notable Changes
+---------------
+
+* This release fixes issues loading the dashboard and volumes manager
+  modules in some environments.
+
+Changelog
+---------
+* nautilus: .github: add workflow for adding labels and milestone (`pr#39926 <https://github.com/ceph/ceph/pull/39926>`_, Kefu Chai, Ernesto Puerta)
+* nautilus: mgr/dashboard: Python2 Cookie module import fails on Python3 (`pr#40116 <https://github.com/ceph/ceph/pull/40116>`_, Volker Theile)
+* nautilus: mgr/volumes: don't require typing (`pr#40095 <https://github.com/ceph/ceph/pull/40095>`_, Josh Durgin)
+* nautilus: qa/suites/krbd: address recent issues caused by newer kernels (`pr#40064 <https://github.com/ceph/ceph/pull/40064>`_, Ilya Dryomov)
+
+v14.2.17 Nautilus
+=================
+
+This is the 17th backport release in the Nautilus series. We recommend
+users to update to this release.
+
+Notable Changes
+---------------
+
+* $pid expansion in config paths like ``admin_socket`` will now properly expand
+  to the daemon pid for commands like ``ceph-mds`` or ``ceph-osd``. Previously
+  only ``ceph-fuse``/``rbd-nbd`` expanded ``$pid`` with the actual daemon pid.
+* RADOS: PG removal has been optimized in this release.
+* RADOS: Memory allocations are tracked in finer detail in BlueStore and displayed as a part of the ``dump_mempools`` command.
+* cephfs: clients which acquire capabilities too quickly are throttled to prevent instability.  See new config option ``mds_session_cap_acquisition_throttle`` to control this behavior.
+
+Changelog
+---------
+
+* nautilus mgr/dashboard: fix 'ceph dashboard iscsi-gateway-add' (`pr#39175 <https://github.com/ceph/ceph/pull/39175>`_, Alfonso Martínez)
+* nautilus: Do not add sensitive information in Ceph log files (`pr#38614 <https://github.com/ceph/ceph/pull/38614>`_, Neha Ojha)
+* nautilus: bluestore: Add protection against bluefs log file growth (`pr#37948 <https://github.com/ceph/ceph/pull/37948>`_, Adam Kupczyk)
+* nautilus: bluestore: provide a different name for fallback allocator (`pr#37793 <https://github.com/ceph/ceph/pull/37793>`_, Igor Fedotov)
+* nautilus: build-integration-branch: take PRs in chronological order (`pr#37693 <https://github.com/ceph/ceph/pull/37693>`_, Nathan Cutler)
+* nautilus: build/ops: install-deps.sh,deb,rpm: move python-saml deps into debian/control and ceph.spec.in (`pr#39184 <https://github.com/ceph/ceph/pull/39184>`_, Kefu Chai)
+* nautilus: ceph-volume batch: reject partitions in argparser (`pr#38279 <https://github.com/ceph/ceph/pull/38279>`_, Jan Fajerski)
+* nautilus: ceph-volume: Fix usage of is_lv (`pr#39221 <https://github.com/ceph/ceph/pull/39221>`_, Michał Nasiadka)
+* nautilus: ceph-volume: Update batch.py (`pr#39470 <https://github.com/ceph/ceph/pull/39470>`_, shenjiatong)
+* nautilus: ceph-volume: add no-systemd argument to zap (`pr#37723 <https://github.com/ceph/ceph/pull/37723>`_, wanghongxu)
+* nautilus: ceph-volume: add some flexibility to bytes_to_extents (`pr#39270 <https://github.com/ceph/ceph/pull/39270>`_, Jan Fajerski)
+* nautilus: ceph-volume: consume mount opt in simple activate (`pr#38015 <https://github.com/ceph/ceph/pull/38015>`_, Dimitri Savineau)
+* nautilus: ceph-volume: implement the --log-level flag (`pr#38372 <https://github.com/ceph/ceph/pull/38372>`_, Andrew Schoen)
+* nautilus: ceph-volume: remove mention of dmcache from docs and help text (`pr#38048 <https://github.com/ceph/ceph/pull/38048>`_, Dimitri Savineau, Andrew Schoen)
+* nautilus: cephfs: client: check rdonly file handle on truncate (`pr#39129 <https://github.com/ceph/ceph/pull/39129>`_, Patrick Donnelly)
+* nautilus: cephfs: client: dump which fs is used by client for multiple-fs (`pr#38552 <https://github.com/ceph/ceph/pull/38552>`_, Zhi Zhang)
+* nautilus: cephfs: client: ensure we take Fs caps when fetching directory link count from cached inode (`pr#38950 <https://github.com/ceph/ceph/pull/38950>`_, Jeff Layton)
+* nautilus: cephfs: client: fix inode ll_ref reference count leak (`pr#37838 <https://github.com/ceph/ceph/pull/37838>`_, sepia-liu)
+* nautilus: cephfs: client: increment file position on _read_sync near eof (`pr#37991 <https://github.com/ceph/ceph/pull/37991>`_, Patrick Donnelly)
+* nautilus: cephfs: client: set CEPH_STAT_RSTAT mask for dir in readdir_r_cb (`pr#38948 <https://github.com/ceph/ceph/pull/38948>`_, chencan)
+* nautilus: cephfs: mds: throttle cap acquisition via readdir (`pr#38101 <https://github.com/ceph/ceph/pull/38101>`_, Kotresh HR)
+* nautilus: cephfs: mount.ceph: collect v2 addresses for non-legacy ms_mode options (`pr#39133 <https://github.com/ceph/ceph/pull/39133>`_, Jeff Layton)
+* nautilus: cephfs: osdc: restart read on truncate/discard (`pr#37988 <https://github.com/ceph/ceph/pull/37988>`_, Patrick Donnelly)
+* nautilus: cephfs: release client dentry_lease before send caps release to mds (`pr#39127 <https://github.com/ceph/ceph/pull/39127>`_, Wei Qiaomiao)
+* nautilus: client: add ceph.{cluster_fsid/client_id} vxattrs suppport (`pr#39001 <https://github.com/ceph/ceph/pull/39001>`_, Xiubo Li)
+* nautilus: client: do not use g_conf().get_val<>() in libcephfs (`pr#38467 <https://github.com/ceph/ceph/pull/38467>`_, Xiubo Li)
+* nautilus: cmake: define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT for Boost.Asio users (`pr#38760 <https://github.com/ceph/ceph/pull/38760>`_, Kefu Chai)
+* nautilus: cmake: detect and use sigdescr_np() if available (`pr#38952 <https://github.com/ceph/ceph/pull/38952>`_, David Disseldorp)
+* nautilus: common/mempool: Improve mempool shard selection (`pr#39651 <https://github.com/ceph/ceph/pull/39651>`_, Nathan Cutler, Adam Kupczyk)
+* nautilus: common: fix logfile create perms (`issue#7849 <http://tracker.ceph.com/issues/7849>`_, `pr#38558 <https://github.com/ceph/ceph/pull/38558>`_, Kefu Chai, Roman Penyaev)
+* nautilus: common: skip interfaces starting with "lo" in find_ipv{4,6}_in_subnet() (`pr#39342 <https://github.com/ceph/ceph/pull/39342>`_, Thomas Goirand, Jiawei Li)
+* nautilus: core: osd: An empty bucket or OSD is not an error (`pr#39126 <https://github.com/ceph/ceph/pull/39126>`_, Brad Hubbard)
+* nautilus: crush/CrushWrapper: rebuild reverse maps after rebuilding crush map (`pr#39197 <https://github.com/ceph/ceph/pull/39197>`_, Jason Dillaman)
+* nautilus: krbd: add support for msgr2 (kernel 5.11) (`pr#39202 <https://github.com/ceph/ceph/pull/39202>`_, Ilya Dryomov)
+* nautilus: librados, tests: allow to list objects with the NUL character in names (`pr#39324 <https://github.com/ceph/ceph/pull/39324>`_, Radoslaw Zarzynski)
+* nautilus: librbd: clear implicitly enabled feature bits when creating images (`pr#39121 <https://github.com/ceph/ceph/pull/39121>`_, Jason Dillaman)
+* nautilus: log: fix timestap precision of log can't set to millisecond (`pr#37659 <https://github.com/ceph/ceph/pull/37659>`_, Guan yunfei)
+* nautilus: lvm/create.py: fix a typo in the help message (`pr#38371 <https://github.com/ceph/ceph/pull/38371>`_, ZhenLiu94)
+* nautilus: mds : move start_files_to_recover() to recovery_done (`pr#37986 <https://github.com/ceph/ceph/pull/37986>`_, Simon Gao)
+* nautilus: mds: account for closing sessions in hit_session (`pr#37820 <https://github.com/ceph/ceph/pull/37820>`_, Dan van der Ster)
+* nautilus: mds: avoid spurious sleeps (`pr#39130 <https://github.com/ceph/ceph/pull/39130>`_, Patrick Donnelly)
+* nautilus: mds: dir->mark_new() should together with dir->mark_dirty() (`pr#39128 <https://github.com/ceph/ceph/pull/39128>`_, "Yan, Zheng")
+* nautilus: mds: update defaults for recall configs (`pr#39134 <https://github.com/ceph/ceph/pull/39134>`_, Patrick Donnelly)
+* nautilus: mgr/PyModule: correctly remove config options (`pr#38803 <https://github.com/ceph/ceph/pull/38803>`_, Tim Serong)
+* nautilus: mgr/crash: Serialize command handling (`pr#39338 <https://github.com/ceph/ceph/pull/39338>`_, Boris Ranto)
+* nautilus: mgr/dashboard: CLI commands: read passwords from file (`pr#38832 <https://github.com/ceph/ceph/pull/38832>`_, Ernesto Puerta, Alfonso Martínez, Juan Miguel Olmo Martínez)
+* nautilus: mgr/dashboard: Datatable catches select events from other datatables (`pr#37756 <https://github.com/ceph/ceph/pull/37756>`_, Volker Theile, Tiago Melo)
+* nautilus: mgr/dashboard: Disable TLS 1.0 and 1.1 (`pr#38332 <https://github.com/ceph/ceph/pull/38332>`_, Volker Theile)
+* nautilus: mgr/dashboard: Disable sso without python3-saml (`pr#38404 <https://github.com/ceph/ceph/pull/38404>`_, Kevin Meijer)
+* nautilus: mgr/dashboard: Display a warning message in Dashboard when debug mode is enabled (`pr#38799 <https://github.com/ceph/ceph/pull/38799>`_, Volker Theile)
+* nautilus: mgr/dashboard: Display users current bucket quota usage (`pr#38024 <https://github.com/ceph/ceph/pull/38024>`_, Avan Thakkar)
+* nautilus: mgr/dashboard: Drop invalid RGW client instances, improve logging (`pr#38584 <https://github.com/ceph/ceph/pull/38584>`_, Volker Theile)
+* nautilus: mgr/dashboard: Fix for datatable item not showing details after getting selected (`pr#38813 <https://github.com/ceph/ceph/pull/38813>`_, Nizamudeen A)
+* nautilus: mgr/dashboard: Fix for incorrect validation in rgw user form (`pr#39117 <https://github.com/ceph/ceph/pull/39117>`_, Nizamudeen A)
+* nautilus: mgr/dashboard: RGW User Form is validating disabled fields (`pr#39543 <https://github.com/ceph/ceph/pull/39543>`_, Aashish Sharma)
+* nautilus: mgr/dashboard: The /rgw/status endpoint does not check for running service (`pr#38771 <https://github.com/ceph/ceph/pull/38771>`_, Volker Theile)
+* nautilus: mgr/dashboard: Updating the inbuilt ssl providers error (`pr#38509 <https://github.com/ceph/ceph/pull/38509>`_, Nizamudeen A)
+* nautilus: mgr/dashboard: Use secure cookies to store JWT Token (`pr#38839 <https://github.com/ceph/ceph/pull/38839>`_, Avan Thakkar, Aashish Sharma)
+* nautilus: mgr/dashboard: add `--ssl` to `ng serve` (`pr#38972 <https://github.com/ceph/ceph/pull/38972>`_, Tatjana Dehler)
+* nautilus: mgr/dashboard: avoid using document.write() (`pr#39526 <https://github.com/ceph/ceph/pull/39526>`_, Avan Thakkar)
+* nautilus: mgr/dashboard: customize CherryPy Server Header (`pr#39419 <https://github.com/ceph/ceph/pull/39419>`_, anurag)
+* nautilus: mgr/dashboard: delete EOF when reading passwords from file (`pr#39438 <https://github.com/ceph/ceph/pull/39438>`_, Alfonso Martínez)
+* nautilus: mgr/dashboard: disable cluster selection in NFS export editing form (`pr#37995 <https://github.com/ceph/ceph/pull/37995>`_, Kiefer Chang)
+* nautilus: mgr/dashboard: enable different URL for users of browser to Grafana (`pr#39136 <https://github.com/ceph/ceph/pull/39136>`_, Patrick Seidensal)
+* nautilus: mgr/dashboard: fix MTU Mismatch alert (`pr#39518 <https://github.com/ceph/ceph/pull/39518>`_, Aashish Sharma)
+* nautilus: mgr/dashboard: fix issues related with PyJWT versions >=2.0.0 (`pr#39837 <https://github.com/ceph/ceph/pull/39837>`_, Alfonso Martínez)
+* nautilus: mgr/dashboard: fix security scopes of some NFS-Ganesha endpoints (`pr#37961 <https://github.com/ceph/ceph/pull/37961>`_, Kiefer Chang)
+* nautilus: mgr/dashboard: fix tooltip for Provisioned/Total Provisioned fields (`pr#39646 <https://github.com/ceph/ceph/pull/39646>`_, Avan Thakkar)
+* nautilus: mgr/dashboard: minimize console log traces of Ceph backend API tests (`pr#39544 <https://github.com/ceph/ceph/pull/39544>`_, Aashish Sharma)
+* nautilus: mgr/dashboard: prometheus alerting: add some leeway for package drops and errors (`pr#39509 <https://github.com/ceph/ceph/pull/39509>`_, Patrick Seidensal)
+* nautilus: mgr/dashboard: python 2: error when setting non-ASCII password (`pr#39441 <https://github.com/ceph/ceph/pull/39441>`_, Alfonso Martínez)
+* nautilus: mgr/dashboard: remove pyOpenSSL version pinning (`pr#38504 <https://github.com/ceph/ceph/pull/38504>`_, Kiefer Chang)
+* nautilus: mgr/dashboard: set security headers (`pr#39626 <https://github.com/ceph/ceph/pull/39626>`_, Avan Thakkar)
+* nautilus: mgr/dashboard: test_standby\* (tasks.mgr.test_dashboard.TestDashboard) failed locally (`pr#38527 <https://github.com/ceph/ceph/pull/38527>`_, Volker Theile)
+* nautilus: mgr/dashboard: trigger alert if some nodes have a MTU different than the median value (`pr#39104 <https://github.com/ceph/ceph/pull/39104>`_, Aashish Sharma)
+* nautilus: mgr/insights: Test environment requires 'six' (`pr#38382 <https://github.com/ceph/ceph/pull/38382>`_, Brad Hubbard)
+* nautilus: mgr/progress: delete all events over the wire (`pr#38416 <https://github.com/ceph/ceph/pull/38416>`_, Sage Weil)
+* nautilus: mgr/progress: make it so progress bar does not get stuck forever (`issue#40618 <http://tracker.ceph.com/issues/40618>`_, `pr#37589 <https://github.com/ceph/ceph/pull/37589>`_, Kamoltat (Junior) Sirivadhna, Kamoltat)
+* nautilus: mgr/prometheus: Add SLOW_OPS healthcheck as a metric (`pr#39747 <https://github.com/ceph/ceph/pull/39747>`_, Paul Cuzner)
+* nautilus: mgr/prometheus: Fix 'pool filling up' with >50% usage (`pr#39076 <https://github.com/ceph/ceph/pull/39076>`_, Daniël Vos)
+* nautilus: mgr/prometheus: Make module more stable (`pr#38334 <https://github.com/ceph/ceph/pull/38334>`_, Boris Ranto, Ken Dreyer)
+* nautilus: mgr/restful: fix TypeError occurring in _gather_osds() (`issue#48488 <http://tracker.ceph.com/issues/48488>`_, `pr#39339 <https://github.com/ceph/ceph/pull/39339>`_, Jerry Pu)
+* nautilus: mgr/telemetry: fix proxy usage (`pr#38816 <https://github.com/ceph/ceph/pull/38816>`_, Nathan Cutler)
+* nautilus: mgr/volume: subvolume auth_id management and few bug fixes (`pr#39292 <https://github.com/ceph/ceph/pull/39292>`_, Rishabh Dave, Patrick Donnelly, Kotresh HR, Ramana Raja)
+* nautilus: mgr/volumes: Make number of cloner threads configurable (`pr#37936 <https://github.com/ceph/ceph/pull/37936>`_, Kotresh HR)
+* nautilus: mgr: Pin importlib_metadata version 2.1.0 (`pr#38296 <https://github.com/ceph/ceph/pull/38296>`_, Brad Hubbard)
+* nautilus: mgr: don't update osd stat which is already out (`pr#38354 <https://github.com/ceph/ceph/pull/38354>`_, Zhi Zhang)
+* nautilus: mgr: fix deadlock in ActivePyModules::get_osdmap() (`pr#39340 <https://github.com/ceph/ceph/pull/39340>`_, peng jiaqi)
+* nautilus: mgr: update mon metadata when monmap is updated (`pr#39075 <https://github.com/ceph/ceph/pull/39075>`_, Kefu Chai)
+* nautilus: mon scrub testing (`pr#38362 <https://github.com/ceph/ceph/pull/38362>`_, Brad Hubbard)
+* nautilus: mon/MDSMonitor do not ignore mds's down:dne request (`pr#37822 <https://github.com/ceph/ceph/pull/37822>`_, chencan)
+* nautilus: mon/MDSMonitor: divide mds identifier and mds real name with dot (`pr#37821 <https://github.com/ceph/ceph/pull/37821>`_, Zhi Zhang)
+* nautilus: mon: Log "ceph health detail" periodically in cluster log (`pr#38118 <https://github.com/ceph/ceph/pull/38118>`_, Prashant Dhange)
+* nautilus: mon: have 'mon stat' output json as well (`pr#37706 <https://github.com/ceph/ceph/pull/37706>`_, Joao Eduardo Luis, Sage Weil)
+* nautilus: mon: paxos: Delete logger in destructor (`pr#39160 <https://github.com/ceph/ceph/pull/39160>`_, Brad Hubbard)
+* nautilus: mon: validate crush-failure-domain (`pr#39124 <https://github.com/ceph/ceph/pull/39124>`_, Prashant Dhange)
+* nautilus: monitoring: Use null yaxes min for OSD read latency (`pr#37959 <https://github.com/ceph/ceph/pull/37959>`_, Seena Fallah)
+* nautilus: msg/async/ProtocolV2: allow rxbuf/txbuf get bigger in testing, again (`pr#38268 <https://github.com/ceph/ceph/pull/38268>`_, Ilya Dryomov)
+* nautilus: ocf: add support for mapping images within an RBD namespace (`pr#39047 <https://github.com/ceph/ceph/pull/39047>`_, Jason Dillaman)
+* nautilus: os/bluestore: Add option to check BlueFS reads (`pr#39756 <https://github.com/ceph/ceph/pull/39756>`_, Adam Kupczyk)
+* nautilus: os/bluestore: detect and fix "zombie" spanning blobs using fsck (`pr#39255 <https://github.com/ceph/ceph/pull/39255>`_, Igor Fedotov)
+* nautilus: os/bluestore: fix huge read/writes in BlueFS (`pr#39698 <https://github.com/ceph/ceph/pull/39698>`_, Jianpeng Ma, Kefu Chai, Igor Fedotov)
+* nautilus: os/bluestore: fix inappropriate ENOSPC from avl/hybrid allocator (`pr#38475 <https://github.com/ceph/ceph/pull/38475>`_, Igor Fedotov)
+* nautilus: os/bluestore: fix segfault on out-of-bound offset provided to  claim\_… (`pr#38637 <https://github.com/ceph/ceph/pull/38637>`_, Igor Fedotov)
+* nautilus: os/bluestore: go beyond pinned onodes while trimming the cache (`pr#39720 <https://github.com/ceph/ceph/pull/39720>`_, Igor Fedotov)
+* nautilus: os/bluestore: mempool's finer granularity + adding missed structs (`pr#38310 <https://github.com/ceph/ceph/pull/38310>`_, Deepika Upadhyay, Igor Fedotov, Adam Kupczyk)
+* nautilus: osd: Check for nosrub/nodeep-scrub in between chunks, to avoid races (`pr#38411 <https://github.com/ceph/ceph/pull/38411>`_, David Zafman)
+* nautilus: osd: fix bluestore bitmap allocator calculate wrong last_pos with hint (`pr#39708 <https://github.com/ceph/ceph/pull/39708>`_, Xue Yantao)
+* nautilus: osd: optimize PG removal (part1) (`pr#38478 <https://github.com/ceph/ceph/pull/38478>`_, Neha Ojha, Igor Fedotov)
+* nautilus: pybind/ceph_volume_client: Update the 'volumes' key to 'subvolumes' in auth-metadata file (`pr#39658 <https://github.com/ceph/ceph/pull/39658>`_, Kotresh HR, Michael Fritch)
+* nautilus: pybind/cephfs: add special values for not reading conffile (`pr#37725 <https://github.com/ceph/ceph/pull/37725>`_, Kefu Chai)
+* nautilus: pybind/cephfs: fix missing terminating NULL char in readlink()'s C string (`pr#38894 <https://github.com/ceph/ceph/pull/38894>`_, Tuan Hoang)
+* nautilus: pybind/mgr/rbd_support: delay creation of progress module events (`pr#38833 <https://github.com/ceph/ceph/pull/38833>`_, Jason Dillaman)
+* nautilus: qa/cephfs: add session_timeout option support (`pr#37840 <https://github.com/ceph/ceph/pull/37840>`_, Xiubo Li)
+* nautilus: qa/distros: add rhel 7.9 (`pr#38188 <https://github.com/ceph/ceph/pull/38188>`_, rakeshgm)
+* nautilus: qa/tasks/ceph_manager.py: don't use log-early in raw_cluster_cmd (`pr#39960 <https://github.com/ceph/ceph/pull/39960>`_, Neha Ojha)
+* nautilus: qa/tasks/{ceph,ceph_manager}: drop py2 support (`pr#37906 <https://github.com/ceph/ceph/pull/37906>`_, Rishabh Dave, Deepika Upadhyay, Kefu Chai)
+* nautilus: qa: fix tox failures (`pr#38627 <https://github.com/ceph/ceph/pull/38627>`_, Patrick Donnelly)
+* nautilus: qa: krbd_stable_pages_required.sh: move to stable_writes attribute (`pr#38834 <https://github.com/ceph/ceph/pull/38834>`_, Ilya Dryomov)
+* nautilus: qa: restore file name (`pr#38772 <https://github.com/ceph/ceph/pull/38772>`_, Patrick Donnelly)
+* nautilus: qa: unmount volumes before removal (`pr#38690 <https://github.com/ceph/ceph/pull/38690>`_, Patrick Donnelly)
+* nautilus: qa: use normal build for valgrind (`pr#39584 <https://github.com/ceph/ceph/pull/39584>`_, Sage Weil)
+* nautilus: rados/upgrade/nautilus-x-singleton fails due to cluster [WRN] evicting unresponsive client (`pr#39706 <https://github.com/ceph/ceph/pull/39706>`_, Patrick Donnelly)
+* nautilus: rbd-nbd: reexpand the conf meta in child process (`pr#38830 <https://github.com/ceph/ceph/pull/38830>`_, Xiubo Li)
+* nautilus: rbd/bench: include used headers (`pr#39123 <https://github.com/ceph/ceph/pull/39123>`_, Kefu Chai)
+* nautilus: rbd: librbd: ensure that thread pool lock is held when processing throttled IOs (`pr#37895 <https://github.com/ceph/ceph/pull/37895>`_, Jason Dillaman)
+* nautilus: rbd: librbd: update hidden global config when removing pool config override (`pr#38831 <https://github.com/ceph/ceph/pull/38831>`_, Jason Dillaman)
+* nautilus: rgw: Disable prefetch of entire head object when GET request with range header (`pr#38556 <https://github.com/ceph/ceph/pull/38556>`_, Or Friedmann)
+* nautilus: rgw: S3 Put Bucket Policy should return 204 on success (`pr#38623 <https://github.com/ceph/ceph/pull/38623>`_, Matthew Oliver)
+* nautilus: rgw: avoid expiration early triggering caused by overflow (`pr#38823 <https://github.com/ceph/ceph/pull/38823>`_, jiahuizeng)
+* nautilus: rgw: cls/rgw/cls_rgw.cc: fix multiple lastest version problem (`pr#38085 <https://github.com/ceph/ceph/pull/38085>`_, Yang Honggang, Ruan Zitao)
+* nautilus: rgw: cls/user: set from_index for reset stats calls (`pr#38822 <https://github.com/ceph/ceph/pull/38822>`_, Mykola Golub, Abhishek Lekshmanan)
+* nautilus: rgw: distribute cache for exclusive put (`pr#38827 <https://github.com/ceph/ceph/pull/38827>`_, Or Friedmann)
+* nautilus: rgw: fix bucket limit check fill_status warnings (`issue#40255 <http://tracker.ceph.com/issues/40255>`_, `pr#38825 <https://github.com/ceph/ceph/pull/38825>`_, Paul Emmerich)
+* nautilus: rgw: fix invalid payload issue when serving s3website error page (`pr#38590 <https://github.com/ceph/ceph/pull/38590>`_, Ilsoo Byun)
+* nautilus: rgw: fix trailing null in object names of multipart reuploads (`pr#39276 <https://github.com/ceph/ceph/pull/39276>`_, Casey Bodley)
+* nautilus: rgw: in ordered bucket listing skip namespaced entries internally when possible (`pr#38493 <https://github.com/ceph/ceph/pull/38493>`_, J. Eric Ivancich)
+* nautilus: rgw: keep syncstopped flag when copying bucket shard headers (`pr#38589 <https://github.com/ceph/ceph/pull/38589>`_, Ilsoo Byun)
+* nautilus: rgw: multisite: Verify if the synced object is identical to source (`pr#38885 <https://github.com/ceph/ceph/pull/38885>`_, Prasad Krishnan, Yang Honggang, Casey Bodley)
+* nautilus: rgw: radosgw-admin: clarify error when email address already in use (`pr#39661 <https://github.com/ceph/ceph/pull/39661>`_, Matthew Vernon)
+* nautilus: rgw: rgw-admin: fixes BucketInfo for missing buckets (`pr#38588 <https://github.com/ceph/ceph/pull/38588>`_, Nick Janus, caolei)
+* nautilus: rgw_file: return common_prefixes in lexical order (`pr#38828 <https://github.com/ceph/ceph/pull/38828>`_, Matt Benjamin)
+* nautilus: rpm,deb: change sudoers file mode to 440 (`pr#39090 <https://github.com/ceph/ceph/pull/39090>`_, David Turner)
+* nautilus: rpm: ceph-mgr-dashboard recommends python3-saml on SUSE (`pr#38818 <https://github.com/ceph/ceph/pull/38818>`_, Nathan Cutler)
+* nautilus: run-make-check.sh: Don't run tests if build fails (`pr#38295 <https://github.com/ceph/ceph/pull/38295>`_, Brad Hubbard)
+* nautilus: test/librados: fix endian bugs in checksum test cases (`pr#37605 <https://github.com/ceph/ceph/pull/37605>`_, Ulrich Weigand)
+* nautilus: test/rbd-mirror: fix broken ceph_test_rbd_mirror_random_write (`pr#39650 <https://github.com/ceph/ceph/pull/39650>`_, Jason Dillaman)
+* nautilus: test/run-cli-tests: use cram from github (`pr#39072 <https://github.com/ceph/ceph/pull/39072>`_, Kefu Chai)
+* nautilus: tests: cancelling both noscrub \*and\* nodeep-scrub (`pr#39125 <https://github.com/ceph/ceph/pull/39125>`_, Ronen Friedman)
+* nautilus: tools/rados: add support for binary object names in the rados CLI (`pr#39329 <https://github.com/ceph/ceph/pull/39329>`_, Radoslaw Zarzynski, Kefu Chai)
+* nautilus: tools/rados: flush formatter periodically during json output of "rados ls" (`pr#37834 <https://github.com/ceph/ceph/pull/37834>`_, J. Eric Ivancich)
+* nautilus: vstart.sh: fix fs set max_mds bug (`pr#37836 <https://github.com/ceph/ceph/pull/37836>`_, Jinmyeong Lee)
+
+
+v14.2.16 Nautilus
+=================
+
+This is the 16th backport release in the Nautilus series. This release fixes a
+security flaw in CephFS. We recommend users to update to this release.
+
+Notable Changes
+---------------
+
+* CVE-2020-27781 : OpenStack Manila use of ceph_volume_client.py library allowed
+  tenant access to any Ceph credential's secret. (Kotresh Hiremath Ravishankar,
+  Ramana Raja)
+
+
+Changelog
+---------
+
+* pybind/ceph_volume_client: disallow authorize on existing auth ids (Kotresh
+  Hiremath Ravishankar, Ramana Raja)
+
+
+v14.2.15 Nautilus
+=================
+
+This is the 15th backport release in the Nautilus series. This release fixes a
+ceph-volume regression introduced in v14.2.13 and includes few other fixes. We
+recommend users to update to this release.
+
+Notable Changes
+---------------
+
+* ceph-volume: Fixes lvm batch --auto, which breaks backward compatibility
+  when using non rotational devices only (SSD and/or NVMe).
+* BlueStore: Fixes a bug in collection_list_legacy which makes pgs inconsistent
+  during scrub when running mixed versions of osds, prior to 14.2.12 with newer.
+* MGR: progress module can now be turned on/off, using the commands:
+  ``ceph progress on`` and ``ceph progress off``.
+
+Changelog
+---------
+
+* ceph-volume: fix filestore/dmcrypt activate (`pr#38198 <https://github.com/ceph/ceph/pull/38198>`_, Guillaume Abrioux)
+* ceph-volume: fix lvm batch auto with full SSDs (`pr#38046 <https://github.com/ceph/ceph/pull/38046>`_, Dimitri Savineau, Guillaume Abrioux)
+* os/bluestore: fix "end reached" check in collection_list_legacy (`pr#38100 <https://github.com/ceph/ceph/pull/38100>`_, Mykola Golub)
+* mgr/progress: introduce turn off/on feature (`pr#38173 <https://github.com/ceph/ceph/pull/38173>`_, kamoltat)
+
+
+v14.2.14 Nautilus
+=================
+
+This is the 14th backport release in the Nautilus series. This release fixes
+a security flaw affecting Messenger v2, among other fixes across components.
+We recommend users to update to this release.
+
+Notable Changes
+---------------
+
+* CVE 2020-25660: CEPHX_V2 replay attack protection lost, for Messenger v2 (Ilya Dryomov)
+
+Changelog
+---------
+
+* mgr/dashboard: Strange iSCSI discovery auth behavior (`pr#37333 <https://github.com/ceph/ceph/pull/37333>`_, Volker Theile)
+* mgr/dashboard: redirect to original URL after successful login (`pr#36834 <https://github.com/ceph/ceph/pull/36834>`_, Avan Thakkar)
+* mgr/prometheus: add pool compression stats (`pr#37563 <https://github.com/ceph/ceph/pull/37563>`_, Paul Cuzner)
+* bluestore: test/objectstore/store_test: kill ExcessiveFragmentation test case (`pr#37824 <https://github.com/ceph/ceph/pull/37824>`_, Igor Fedotov)
+* bluestore: BlockDevice.cc: use pending_aios instead of iovec size as ios num (`pr#37823 <https://github.com/ceph/ceph/pull/37823>`_, weixinwei)
+* bluestore: Support flock retry (`pr#37842 <https://github.com/ceph/ceph/pull/37842>`_, Kefu Chai, wanghongxu)
+* bluestore: attach csum for compressed blobs (`pr#37843 <https://github.com/ceph/ceph/pull/37843>`_, Igor Fedotov)
+* osdc/ObjectCacher: overwrite might cause stray read request callbacks (`pr#37813 <https://github.com/ceph/ceph/pull/37813>`_, Jason Dillaman)
+* mgr: avoid false alarm of MGR_MODULE_ERROR (`pr#38069 <https://github.com/ceph/ceph/pull/38069>`_, Kefu Chai, Sage Weil)
+* mgr: fix race between module load and notify (`pr#37844 <https://github.com/ceph/ceph/pull/37844>`_, Mykola Golub, Patrick Donnelly)
+* mon: set session_timeout when adding to session_map (`pr#37554 <https://github.com/ceph/ceph/pull/37554>`_, Ilya Dryomov)
+* mon/MonClient: bring back CEPHX_V2 authorizer challenges (Ilya Dryomov)
+* osd/osd-rep-recov-eio.sh: TEST_rados_repair_warning:  return 1 (`pr#37815 <https://github.com/ceph/ceph/pull/37815>`_, David Zafman)
+* rbd: librbd: ignore -ENOENT error when disabling object-map (`pr#37814 <https://github.com/ceph/ceph/pull/37814>`_, Jason Dillaman)
+* rbd: rbd-nbd: don't ignore namespace when unmapping by image spec (`pr#37811 <https://github.com/ceph/ceph/pull/37811>`_, Mykola Golub)
+* rgw/rgw_file: Fix the incorrect lru object eviction (`pr#37804 <https://github.com/ceph/ceph/pull/37804>`_, luo rixin)
+* rgw: fix expiration header returned even if there is only one tag in the object the same as the rule (`pr#37806 <https://github.com/ceph/ceph/pull/37806>`_, Or Friedmann)
+* rgw: fix: S3 API KeyCount incorrect return (`pr#37810 <https://github.com/ceph/ceph/pull/37810>`_, 胡玮文)
+* rgw: radosgw-admin should paginate internally when listing bucket (`pr#37802 <https://github.com/ceph/ceph/pull/37802>`_, J. Eric Ivancich)
+* rgw: rgw_file: avoid long-ish delay on shutdown (`pr#37552 <https://github.com/ceph/ceph/pull/37552>`_, Matt Benjamin)
+* rgw: use yum rather than dnf for teuthology testing of rgw-orphan-list (`pr#37805 <https://github.com/ceph/ceph/pull/37805>`_, J. Eric Ivancich)
+
+
+v14.2.13 Nautilus
+=================
+
+This is the 13th backport release in the Nautilus series. This release fixes a
+regression introduced in v14.2.12, and a few ceph-volume & RGW fixes. We
+recommend users to update to this release.
+
+Notable Changes
+---------------
+
+* Fixed a regression that caused breakage in clusters that referred to ceph-mon
+  hosts using dns names instead of ip addresses in the ``mon_host`` param in
+  ``ceph.conf`` (`issue#47951 <https://tracker.ceph.com/issues/47951>`_)
+
+* ceph-volume: the ``lvm batch`` subcommand received a major rewrite
+
+Changelog
+---------
+
+* ceph-volume: major batch refactor (`pr#37522 <https://github.com/ceph/ceph/pull/37522>`_, Jan Fajerski)
+* mgr/dashboard: Proper format iSCSI target portals (`pr#37060 <https://github.com/ceph/ceph/pull/37060>`_, Volker Theile)
+* rpm: move python-enum34 into rhel 7 conditional (`pr#37747 <https://github.com/ceph/ceph/pull/37747>`_, Nathan Cutler)
+* mon/MonMap: fix unconditional failure for init_with_hosts (`pr#37816 <https://github.com/ceph/ceph/pull/37816>`_, Nathan Cutler, Patrick Donnelly)
+* rgw: allow rgw-orphan-list to note when rados objects are in namespace (`pr#37799 <https://github.com/ceph/ceph/pull/37799>`_, J. Eric Ivancich)
+* rgw: fix setting of namespace in ordered and unordered bucket listing (`pr#37798 <https://github.com/ceph/ceph/pull/37798>`_, J. Eric Ivancich)
+
+
+v14.2.12 Nautilus
+=================
+
+This is the 12th backport release in the Nautilus series. This release
+brings a number of bugfixes across all major components of Ceph. We recommend
+that all Nautilus users upgrade to this release.
+
+Notable Changes
+---------------
+
+* The ``ceph df`` command now lists the number of pgs in each pool.
+
+* Monitors now have a config option ``mon_osd_warn_num_repaired``, 10 by default.
+  If any OSD has repaired more than this many I/O errors in stored data a
+  ``OSD_TOO_MANY_REPAIRS`` health warning is generated.  In order to allow
+  clearing of the warning, a new command ``ceph tell osd.# clear_shards_repaired [count]``
+  has been added.  By default it will set the repair count to 0.  If you wanted
+  to be warned again if additional repairs are performed you can provide a value
+  to the command and specify the value of ``mon_osd_warn_num_repaired``.
+  This command will be replaced in future releases by the health mute/unmute feature.
+
+* It is now possible to specify the initial monitor to contact for Ceph tools
+  and daemons using the ``mon_host_override`` config option or
+  ``--mon-host-override <ip>`` command-line switch. This generally should only
+  be used for debugging and only affects initial communication with Ceph's
+  monitor cluster.
+
+* Fix an issue with osdmaps not being trimmed in a healthy cluster (`issue#47296
+  <https://tracker.ceph.com/issues/47296>`_, `pr#36982
+  <https://github.com/ceph/ceph/pull/36982>`_)
+
+Changelog
+---------
+* bluestore/bluefs: make accounting resiliant to unlock() (`pr#36909 <https://github.com/ceph/ceph/pull/36909>`_, Adam Kupczyk)
+* bluestore: Rescue procedure for extremely large bluefs log (`pr#36930 <https://github.com/ceph/ceph/pull/36930>`_, Adam Kupczyk)
+* bluestore: dump onode that has too many spanning blobs (`pr#36756 <https://github.com/ceph/ceph/pull/36756>`_, Igor Fedotov)
+* bluestore: enable more flexible bluefs space management by default (`pr#37091 <https://github.com/ceph/ceph/pull/37091>`_, Igor Fedotov)
+* bluestore: fix collection_list ordering (`pr#37051 <https://github.com/ceph/ceph/pull/37051>`_, Mykola Golub)
+* ceph-iscsi: selinux fixes (`pr#36304 <https://github.com/ceph/ceph/pull/36304>`_, Mike Christie)
+* ceph-volume: add tests for new functions that run LVM commands (`pr#36615 <https://github.com/ceph/ceph/pull/36615>`_, Rishabh Dave)
+* ceph-volume: dont use container classes in api/lvm.py (`pr#35878 <https://github.com/ceph/ceph/pull/35878>`_, Guillaume Abrioux, Rishabh Dave')
+* ceph-volume: fix journal size argument not work (`pr#37377 <https://github.com/ceph/ceph/pull/37377>`_, wanghongxu)
+* ceph-volume: fix simple activate when legacy osd (`pr#37195 <https://github.com/ceph/ceph/pull/37195>`_, Guillaume Abrioux)
+* ceph-volume: fix test_lvm.TestVolume.test_is_not_ceph_device (`pr#36493 <https://github.com/ceph/ceph/pull/36493>`_, Jan Fajerski)
+* ceph-volume: handle idempotency with batch and explicit scenarios (`pr#35881 <https://github.com/ceph/ceph/pull/35881>`_, Andrew Schoen)
+* ceph-volume: remove container classes from api/lvm.py (`pr#36610 <https://github.com/ceph/ceph/pull/36610>`_, Rishabh Dave)
+* ceph-volume: remove unneeded call to get_devices() (`pr#37413 <https://github.com/ceph/ceph/pull/37413>`_, Marc Gariepy)
+* ceph-volume: report correct rejected reason in inventory if device type is invalid (`pr#36453 <https://github.com/ceph/ceph/pull/36453>`_, Satoru Takeuchi)
+* ceph-volume: retry when acquiring lock fails (`pr#36926 <https://github.com/ceph/ceph/pull/36926>`_, S\xc3\xa9bastien Han)
+* ceph-volume: simple scan should ignore tmpfs (`pr#36952 <https://github.com/ceph/ceph/pull/36952>`_, Andrew Schoen)
+* ceph.in: ignore failures to flush stdout (`pr#37226 <https://github.com/ceph/ceph/pull/37226>`_, Dan van der Ster)
+* ceph.spec.in, debian/control: add smartmontools and nvme-cli dependen\xe2\x80\xa6 (`pr#37288 <https://github.com/ceph/ceph/pull/37288>`_, Yaarit Hatuka)
+* cephfs-journal-tool: fix incorrect read_offset when finding missing objects (`pr#37479 <https://github.com/ceph/ceph/pull/37479>`_, Xue Yantao)
+* cephfs: client: fix extra open ref decrease (`pr#36966 <https://github.com/ceph/ceph/pull/36966>`_, Xiubo Li)
+* cephfs: client: make Client::open() pass proper cap mask to path_walk (`pr#37231 <https://github.com/ceph/ceph/pull/37231>`_, "Yan, Zheng")
+* cephfs: mds/CInode: Optimize only pinned by subtrees check (`pr#36965 <https://github.com/ceph/ceph/pull/36965>`_, Mark Nelson)
+* cephfs: mds: After restarting an mds, its standy-replay mds remained in the "resolve" state (`pr#37179 <https://github.com/ceph/ceph/pull/37179>`_, Wei Qiaomiao)
+* cephfs: mds: do not defer incoming mgrmap when mds is laggy (`issue#44638 <http://tracker.ceph.com/issues/44638>`_, `pr#36168 <https://github.com/ceph/ceph/pull/36168>`_, Nathan Cutler, Venky Shankar)
+* cephfs: mds: fix incorrect check for if dirfrag is being fragmented (`pr#37035 <https://github.com/ceph/ceph/pull/37035>`_, "Yan, Zheng")
+* cephfs: mds: fix mds forwarding request no_available_op_found (`pr#36963 <https://github.com/ceph/ceph/pull/36963>`_, Yanhu Cao')
+* cephfs: mds: fix purge_queues _calculate_ops is inaccurate (`pr#37481 <https://github.com/ceph/ceph/pull/37481>`_, Yanhu Cao')
+* cephfs: mds: kcephfs parse dirfrags ndist is always 0 (`pr#37177 <https://github.com/ceph/ceph/pull/37177>`_, Yanhu Cao')
+* cephfs: mds: place MDSGatherBuilder on the stack (`pr#36967 <https://github.com/ceph/ceph/pull/36967>`_, Patrick Donnelly)
+* cephfs: mds: recover files after normal session close (`pr#37178 <https://github.com/ceph/ceph/pull/37178>`_, "Yan, Zheng")
+* cephfs: mds: resolve SIGSEGV in waiting for uncommitted fragments (`pr#36968 <https://github.com/ceph/ceph/pull/36968>`_, Patrick Donnelly)
+* cephfs: osdc/Journaler: do not call onsafe->complete() if onsafe is 0 (`pr#37229 <https://github.com/ceph/ceph/pull/37229>`_, Xiubo Li)
+* client: handle readdir reply without Fs cap (`pr#37232 <https://github.com/ceph/ceph/pull/37232>`_, "Yan, Zheng")
+* common, osd: add sanity checks around osd_scrub_max_preemptions (`pr#37470 <https://github.com/ceph/ceph/pull/37470>`_, xie xingguo)
+* common/config: less noise about configs from mon we cant apply (`pr#36289 <https://github.com/ceph/ceph/pull/36289>`_, Sage Weil')
+* common:  ignore SIGHUP prior to fork (`issue#46269 <http://tracker.ceph.com/issues/46269>`_, `pr#36181 <https://github.com/ceph/ceph/pull/36181>`_, Willem Jan Withagen, hzwuhongsong)
+* compressor: Add a config option to specify Zstd compression level (`pr#37254 <https://github.com/ceph/ceph/pull/37254>`_, Bryan Stillwell)
+* core: include/encoding: Fix encode/decode of float types on big-endian systems (`pr#37033 <https://github.com/ceph/ceph/pull/37033>`_, Ulrich Weigand)
+* doc/rados: Fix osd_op_queue default value (`pr#36354 <https://github.com/ceph/ceph/pull/36354>`_, Beno\xc3\xaet Knecht)
+* doc/rados: Fix osd_scrub_during_recovery default value (`pr#37472 <https://github.com/ceph/ceph/pull/37472>`_, Beno\xc3\xaet Knecht)
+* doc/rbd: add rbd-target-gw enable and start (`pr#36415 <https://github.com/ceph/ceph/pull/36415>`_, Zac Dover)
+* doc: enable Read the Docs (`pr#37204 <https://github.com/ceph/ceph/pull/37204>`_, Kefu Chai)
+* krbd: optionally skip waiting for udev events (`pr#37284 <https://github.com/ceph/ceph/pull/37284>`_, Ilya Dryomov)
+* kv/RocksDBStore: make options compaction_threads/disableWAL/flusher_t\xe2\x80\xa6 (`pr#37055 <https://github.com/ceph/ceph/pull/37055>`_, Jianpeng Ma)
+* librados: add LIBRADOS_SUPPORTS_GETADDRS support (`pr#36853 <https://github.com/ceph/ceph/pull/36853>`_, Xiubo Li, Jason Dillaman, Kaleb S. KEITHLEY, Kefu Chai)
+* messages,mds: Fix decoding of enum types on big-endian systems (`pr#36814 <https://github.com/ceph/ceph/pull/36814>`_, Ulrich Weigand)
+* mgr/balancer: use "==" and "!=" for comparing str (`pr#37471 <https://github.com/ceph/ceph/pull/37471>`_, Kefu Chai)
+* mgr/dashboard/api: increase API health timeout (`pr#36607 <https://github.com/ceph/ceph/pull/36607>`_, Ernesto Puerta)
+* mgr/dashboard: Allow editing iSCSI targets with initiators logged-in (`pr#37278 <https://github.com/ceph/ceph/pull/37278>`_, Tiago Melo)
+* mgr/dashboard: Disabling the form inputs for the read_only modals (`pr#37241 <https://github.com/ceph/ceph/pull/37241>`_, Nizamudeen)
+* mgr/dashboard: Dont use any xlf file when building the default language (`pr#37550 <https://github.com/ceph/ceph/pull/37550>`_, Sebastian Krah')
+* mgr/dashboard: Fix many-to-many issue in host-details Grafana dashboard (`pr#37306 <https://github.com/ceph/ceph/pull/37306>`_, Patrick Seidensal)
+* mgr/dashboard: Fix pool renaming functionality (`pr#37510 <https://github.com/ceph/ceph/pull/37510>`_, Stephan M\xc3\xbcller, Ernesto Puerta)
+* mgr/dashboard: Hide table action input field if limit=0 (`pr#36783 <https://github.com/ceph/ceph/pull/36783>`_, Volker Theile)
+* mgr/dashboard: Monitoring: Fix for the infinite loading bar action (`pr#37161 <https://github.com/ceph/ceph/pull/37161>`_, Nizamudeen A)
+* mgr/dashboard: REST API returns 500 when no Content-Type is specified (`pr#37307 <https://github.com/ceph/ceph/pull/37307>`_, Avan Thakkar)
+* mgr/dashboard: Unable to edit iSCSI logged-in client (`pr#36613 <https://github.com/ceph/ceph/pull/36613>`_, Ricardo Marques)
+* mgr/dashboard: cpu stats incorrectly displayed (`pr#37295 <https://github.com/ceph/ceph/pull/37295>`_, Avan Thakkar)
+* mgr/dashboard: document Prometheus security model (`pr#36920 <https://github.com/ceph/ceph/pull/36920>`_, Patrick Seidensal)
+* mgr/dashboard: fix broken backporting (`pr#37505 <https://github.com/ceph/ceph/pull/37505>`_, Ernesto Puerta)
+* mgr/dashboard: fix perf. issue when listing large amounts of buckets (`pr#37280 <https://github.com/ceph/ceph/pull/37280>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: fix pool usage calculation (`pr#37309 <https://github.com/ceph/ceph/pull/37309>`_, Ernesto Puerta)
+* mgr/dashboard: remove "This week/month/year" and "Today" time stamps (`pr#36790 <https://github.com/ceph/ceph/pull/36790>`_, Avan Thakkar)
+* mgr/dashboard: table detail rows overflow (`pr#37324 <https://github.com/ceph/ceph/pull/37324>`_, Aashish Sharma)
+* mgr/dashboard: wait longer for health status to be cleared (`pr#36784 <https://github.com/ceph/ceph/pull/36784>`_, Tatjana Dehler)
+* mgr/devicehealth: fix daemon filtering before scraping device (`pr#36741 <https://github.com/ceph/ceph/pull/36741>`_, Yaarit Hatuka)
+* mgr/diskprediction_local: Fix array size error (`pr#36578 <https://github.com/ceph/ceph/pull/36578>`_, Beno\xc3\xaet Knecht)
+* mgr/prometheus: automatically discover RBD pools for stats gathering (`pr#36412 <https://github.com/ceph/ceph/pull/36412>`_, Jason Dillaman)
+* mgr/restful: use dict.items() for py3 compatible (`pr#36670 <https://github.com/ceph/ceph/pull/36670>`_, Kefu Chai)
+* mgr/status: metadata is fetched async (`pr#37558 <https://github.com/ceph/ceph/pull/37558>`_, Michael Fritch)
+* mgr/telemetry: fix device id splitting when anonymizing serial (`pr#37318 <https://github.com/ceph/ceph/pull/37318>`_, Yaarit Hatuka)
+* mgr/volumes: add global lock debug (`pr#36828 <https://github.com/ceph/ceph/pull/36828>`_, Patrick Donnelly)
+* mgr: Add missing states to PG_STATES in mgr_module.py (`pr#36785 <https://github.com/ceph/ceph/pull/36785>`_, Harley Gorrell)
+* mgr: decrease pool stats if pg was removed (`pr#37476 <https://github.com/ceph/ceph/pull/37476>`_, Aleksei Gutikov)
+* mgr: don't update pending service map epoch on receiving map from mon (`pr#37181 <https://github.com/ceph/ceph/pull/37181>`_, Mykola Golub')
+* minor tweaks to fix compile issues under latest Fedora (`pr#36726 <https://github.com/ceph/ceph/pull/36726>`_, Willem Jan Withagen, Kaleb S. KEITHLEY, Kefu Chai)
+* mon/OSDMonitor: only take in osd into consideration when trimming osdmaps (`pr#36982 <https://github.com/ceph/ceph/pull/36982>`_, Kefu Chai)
+* mon/PGMap: add pg count for pools in the ceph df command (`pr#36944 <https://github.com/ceph/ceph/pull/36944>`_, Vikhyat Umrao)
+* mon: Warn when too many reads are repaired on an OSD (`pr#36379 <https://github.com/ceph/ceph/pull/36379>`_, David Zafman)
+* mon: fix the \Error ERANGE\ message when conf "osd_objectstore" is filestore' (`pr#37474 <https://github.com/ceph/ceph/pull/37474>`_, wangyunqing')
+* mon: mark pgtemp messages as no_reply more consistenly in preprocess\\_\xe2\x80\xa6 (`pr#37171 <https://github.com/ceph/ceph/pull/37171>`_, Greg Farnum)
+* mon: store mon updates in ceph context for future MonMap instantiation (`pr#36704 <https://github.com/ceph/ceph/pull/36704>`_, Patrick Donnelly, Shyamsundar Ranganathan)
+* monclient: schedule first tick using mon_client_hunt_interval (`pr#36634 <https://github.com/ceph/ceph/pull/36634>`_, Mykola Golub)
+* msg/async/ProtocolV2: allow rxbuf/txbuf get bigger in testing (`pr#37081 <https://github.com/ceph/ceph/pull/37081>`_, Ilya Dryomov)
+* osd/OSDCap: rbd profile permits use of "rbd_info" (`pr#36413 <https://github.com/ceph/ceph/pull/36413>`_, Florian Florensa)
+* osd/PeeringState: prevent peers num_objects going negative (`pr#37473 <https://github.com/ceph/ceph/pull/37473>`_, xie xingguo')
+* prometheus: Properly split the port off IPv6 addresses (`pr#36984 <https://github.com/ceph/ceph/pull/36984>`_, Matthew Oliver)
+* rbd: include RADOS namespace in krbd symlinks (`pr#37468 <https://github.com/ceph/ceph/pull/37468>`_, Ilya Dryomov)
+* rbd: librbd: Align rbd_write_zeroes declarations (`pr#36712 <https://github.com/ceph/ceph/pull/36712>`_, Corey Bryant)
+* rbd: librbd: dont resend async_complete if watcher is unregistered (`pr#37040 <https://github.com/ceph/ceph/pull/37040>`_, Mykola Golub')
+* rbd: librbd: global and pool-level config overrides require image refresh to apply (`pr#36725 <https://github.com/ceph/ceph/pull/36725>`_, Jason Dillaman)
+* rbd: librbd: using migration abort can result in the loss of data (`pr#37165 <https://github.com/ceph/ceph/pull/37165>`_, Jason Dillaman)
+* rbd: make common options override krbd-specific options (`pr#37407 <https://github.com/ceph/ceph/pull/37407>`_, Ilya Dryomov)
+* rgw/cls: preserve olh entrys name on last unlink (`pr#37462 <https://github.com/ceph/ceph/pull/37462>`_, Casey Bodley')
+* rgw: Add bucket name to bucket stats error logging (`pr#37378 <https://github.com/ceph/ceph/pull/37378>`_, Seena Fallah)
+* rgw: Empty reqs_change_state queue before unregistered_reqs (`pr#37461 <https://github.com/ceph/ceph/pull/37461>`_, Soumya Koduri)
+* rgw: Expiration days cant be zero and  transition days can be zero (`pr#37465 <https://github.com/ceph/ceph/pull/37465>`_, zhang Shaowen')
+* rgw: RGWObjVersionTracker tracks version over increments (`pr#37459 <https://github.com/ceph/ceph/pull/37459>`_, Casey Bodley)
+* rgw: Swift API anonymous access should 401 (`pr#37438 <https://github.com/ceph/ceph/pull/37438>`_, Matthew Oliver)
+* rgw: add access log to the beast frontend (`pr#36727 <https://github.com/ceph/ceph/pull/36727>`_, Mark Kogan)
+* rgw: add negative cache to the system object (`pr#37460 <https://github.com/ceph/ceph/pull/37460>`_, Or Friedmann)
+* rgw: append obj: prevent tail from being GCed (`pr#36390 <https://github.com/ceph/ceph/pull/36390>`_, Abhishek Lekshmanan')
+* rgw: dump transitions in RGWLifecycleConfiguration::dump() (`pr#36880 <https://github.com/ceph/ceph/pull/36880>`_, Shengming Zhang)
+* rgw: fail when get/set-bucket-versioning attempted on a non-existent \xe2\x80\xa6 (`pr#36188 <https://github.com/ceph/ceph/pull/36188>`_, Matt Benjamin)
+* rgw: fix boost::asio::async_write() does not return error (`pr#37157 <https://github.com/ceph/ceph/pull/37157>`_, Mark Kogan)
+* rgw: fix double slash (//) killing the gateway (`pr#36682 <https://github.com/ceph/ceph/pull/36682>`_, Theofilos Mouratidis)
+* rgw: fix shutdown crash in RGWAsyncReadMDLogEntries (`pr#37463 <https://github.com/ceph/ceph/pull/37463>`_, Casey Bodley)
+* rgw: hold reloader using unique_ptr (`pr#36770 <https://github.com/ceph/ceph/pull/36770>`_, Kefu Chai)
+* rgw: log resharding events at level 1 (formerly 20) (`pr#36843 <https://github.com/ceph/ceph/pull/36843>`_, Or Friedmann)
+* rgw: ordered bucket listing code clean-up (`pr#37169 <https://github.com/ceph/ceph/pull/37169>`_, J. Eric Ivancich)
+* rgw: policy: reuse eval_principal to evaluate the policy principal (`pr#36637 <https://github.com/ceph/ceph/pull/36637>`_, Abhishek Lekshmanan)
+* rgw: radosgw-admin: period pull command is not always a raw_storage_op (`pr#37464 <https://github.com/ceph/ceph/pull/37464>`_, Casey Bodley)
+* rgw: replace \+\ with "%20" in canonical query string for s3 v4 auth' (`pr#37467 <https://github.com/ceph/ceph/pull/37467>`_, yuliyang_yewu')
+* rgw: urlencode bucket name when forwarding request (`pr#37435 <https://github.com/ceph/ceph/pull/37435>`_, caolei)
+* run-make-check.sh: extract run-make.sh + run sudo with absolute path (`pr#36494 <https://github.com/ceph/ceph/pull/36494>`_, Kefu Chai, Ernesto Puerta)
+* systemd: Support Graceful Reboot for AIO Node (`pr#37301 <https://github.com/ceph/ceph/pull/37301>`_, Wong Hoi Sing Edison)
+* tools/osdmaptool.cc: add ability to clean_temps (`pr#37477 <https://github.com/ceph/ceph/pull/37477>`_, Neha Ojha)
+* tools/rados: Set locator key when exporting or importing a pool (`pr#37475 <https://github.com/ceph/ceph/pull/37475>`_, Iain Buclaw)
+
+
+v14.2.11 Nautilus
+=================
+
+This is the eleventh backport release in the Nautilus series. This release
+brings a number of bugfixes across all major components of Ceph. We recommend
+that all Nautilus users upgrade to this release.
+
+Notable Changes
+---------------
+
+* RGW: The ``radosgw-admin`` sub-commands dealing with orphans --
+  ``radosgw-admin orphans find``, ``radosgw-admin orphans finish``,
+  ``radosgw-admin orphans list-jobs`` -- have been deprecated. They
+  have not been actively maintained and they store intermediate
+  results on the cluster, which could fill a nearly-full cluster.
+  They have been replaced by a tool, currently considered
+  experimental, ``rgw-orphan-list``.
+
+* Now when noscrub and/or nodeep-scrub flags are set globally or per pool,
+  scheduled scrubs of the type disabled will be aborted. All user initiated
+  scrubs are NOT interrupted.
+
+* Fixed a ceph-osd crash in _committed_osd_maps when there is a failure to encode
+  the first incremental map. `issue#46443 <https://tracker.ceph.com/issues/46443>`_
+
+Changelog
+---------
+
+* bluestore: core: os/bluestore: fix large (>2GB) writes when bluefs_buffered_io = true (`pr#35404 <https://github.com/ceph/ceph/pull/35404>`_, Igor Fedotov)
+* bluestore: os/bluestore: implement Hybrid allocator (`pr#35500 <https://github.com/ceph/ceph/pull/35500>`_, Adam Kupczyk, Kefu Chai, Igor Fedotov, xie xingguo)
+* build/ops: build/ops: selinux: allow ceph_t amqp_port_t:tcp_socket (`pr#36190 <https://github.com/ceph/ceph/pull/36190>`_, Kaleb S. KEITHLEY, Thomas Serlin)
+* ceph-volume: add dmcrypt support in raw mode (`pr#35831 <https://github.com/ceph/ceph/pull/35831>`_, Guillaume Abrioux)
+* cephfs,pybind: pybind/cephfs: fix custom exception raised by cephfs.pyx (`pr#36180 <https://github.com/ceph/ceph/pull/36180>`_, Ramana Raja)
+* cephfs: ceph_fuse: add the '-d' option back for libfuse (`pr#35398 <https://github.com/ceph/ceph/pull/35398>`_, Xiubo Li)
+* cephfs: client: fix directory inode can not call release callback (`pr#36177 <https://github.com/ceph/ceph/pull/36177>`_, sepia-liu)
+* cephfs: client: fix setxattr for 0 size value (NULL value) (`pr#36173 <https://github.com/ceph/ceph/pull/36173>`_, Sidharth Anupkrishnan)
+* cephfs: client: fix snap directory atime (`pr#36169 <https://github.com/ceph/ceph/pull/36169>`_, Luis Henriques)
+* cephfs: client: introduce timeout for client shutdown (`issue#44276 <http://tracker.ceph.com/issues/44276>`_, `pr#36215 <https://github.com/ceph/ceph/pull/36215>`_, Venky Shankar)
+* cephfs: client: release the client_lock before copying data in read (`pr#36294 <https://github.com/ceph/ceph/pull/36294>`_, Chencan)
+* cephfs: client: static dirent for readdir is not thread-safe (`pr#36511 <https://github.com/ceph/ceph/pull/36511>`_, Patrick Donnelly)
+* cephfs: mds: add config to require forward to auth MDS (`pr#35377 <https://github.com/ceph/ceph/pull/35377>`_, simon gao)
+* cephfs: mds: cleanup uncommitted fragments before mds goes to active (`pr#35397 <https://github.com/ceph/ceph/pull/35397>`_, "Yan, Zheng")
+* cephfs: mds: do not raise "client failing to respond to cap release" when client working set is reasonable (`pr#36513 <https://github.com/ceph/ceph/pull/36513>`_, Patrick Donnelly)
+* cephfs: mds: do not submit omap_rm_keys if the dir is the basedir of merge (`pr#36178 <https://github.com/ceph/ceph/pull/36178>`_, Chencan)
+* cephfs: mds: fix filelock state when Fc is issued (`pr#35841 <https://github.com/ceph/ceph/pull/35841>`_, Xiubo Li)
+* cephfs: mds: fix hang issue when accessing a file under a lost parent directory (`pr#36179 <https://github.com/ceph/ceph/pull/36179>`_, Zhi Zhang)
+* cephfs: mds: fix nullptr dereference in MDCache::finish_rollback (`pr#36439 <https://github.com/ceph/ceph/pull/36439>`_, "Yan, Zheng")
+* cephfs: mds: flag backtrace scrub failures for new files as okay (`pr#35400 <https://github.com/ceph/ceph/pull/35400>`_, Milind Changire)
+* cephfs: mds: initialize MDSlaveUpdate::waiter (`pr#36462 <https://github.com/ceph/ceph/pull/36462>`_, "Yan, Zheng")
+* cephfs: mds: make threshold for MDS_TRIM configurable (`pr#36175 <https://github.com/ceph/ceph/pull/36175>`_, Paul Emmerich)
+* cephfs: mds: preserve ESlaveUpdate logevent until receiving OP_FINISH (`pr#35394 <https://github.com/ceph/ceph/pull/35394>`_, Varsha Rao, songxinying)
+* cephfs: mds: reset heartbeat in EMetaBlob replay (`pr#36170 <https://github.com/ceph/ceph/pull/36170>`_, Yanhu Cao)
+* cephfs: mgr/fs/volumes misc fixes (`pr#36167 <https://github.com/ceph/ceph/pull/36167>`_, Patrick Donnelly, Kotresh HR, Ramana Raja)
+* cephfs: mgr/volumes: Add snapshot info command (`pr#35672 <https://github.com/ceph/ceph/pull/35672>`_, Kotresh HR)
+* cephfs: mgr/volumes: Deprecate protect/unprotect CLI calls for subvolume snapshots (`pr#36166 <https://github.com/ceph/ceph/pull/36166>`_, Shyamsundar Ranganathan)
+* cephfs: qa: add debugging for volumes plugin use of libcephfs (`pr#36512 <https://github.com/ceph/ceph/pull/36512>`_, Patrick Donnelly)
+* cephfs: qa: skip cache_size check (`pr#36526 <https://github.com/ceph/ceph/pull/36526>`_, Patrick Donnelly)
+* cephfs: tools/cephfs: don't bind to public_addr (`pr#35401 <https://github.com/ceph/ceph/pull/35401>`_, "Yan, Zheng")
+* cephfs: vstart_runner: set mounted to True at the end of mount() (`pr#35396 <https://github.com/ceph/ceph/pull/35396>`_, Rishabh Dave)
+* core,mon: mon/OSDMonitor: Reset grace period if failure interval exceeds a threshold (`pr#35798 <https://github.com/ceph/ceph/pull/35798>`_, Sridhar Seshasayee)
+* core: mgr/DaemonServer.cc: make 'config show' on fsid work (`pr#36074 <https://github.com/ceph/ceph/pull/36074>`_, Neha Ojha)
+* core: mgr/alert: can't set inventory_cache_timeout/service_cache_timeout from CLI (`pr#36104 <https://github.com/ceph/ceph/pull/36104>`_, Kiefer Chang)
+* core: osd/PG: fix history.same_interval_since of merge target again (`pr#36161 <https://github.com/ceph/ceph/pull/36161>`_, xie xingguo)
+* core: osd/PeeringState.h: Fix pg stuck in WaitActingChange (`pr#35389 <https://github.com/ceph/ceph/pull/35389>`_, chen qiuzhang)
+* core: osd: Cancel in-progress scrubs (not user requested) (`pr#36292 <https://github.com/ceph/ceph/pull/36292>`_, David Zafman)
+* core: osd: fix crash in _committed_osd_maps if incremental osdmap crc fails (`pr#36339 <https://github.com/ceph/ceph/pull/36339>`_, Neha Ojha, Dan van der Ster)
+* core: osd: make "missing incremental map" a debug log message (`pr#35386 <https://github.com/ceph/ceph/pull/35386>`_, Nathan Cutler)
+* core: osd: make message cap option usable again (`pr#35738 <https://github.com/ceph/ceph/pull/35738>`_, Neha Ojha, Josh Durgin)
+* mgr/dashboard: Allow to edit iSCSI target with active session (`pr#35998 <https://github.com/ceph/ceph/pull/35998>`_, Ricardo Marques)
+* mgr/dashboard: Prevent dashboard breakdown on bad pool selection (`pr#35367 <https://github.com/ceph/ceph/pull/35367>`_, Stephan Müller)
+* mgr/dashboard: Prometheus query error in the metrics of Pools, OSDs and RBD images (`pr#35884 <https://github.com/ceph/ceph/pull/35884>`_, Avan Thakkar)
+* mgr/dashboard: add popover list of Stand-by Managers & Metadata Servers (MDS) in landing page (`pr#34095 <https://github.com/ceph/ceph/pull/34095>`_, Kiefer Chang, Avan Thakkar)
+* mgr/dashboard: fix Source column i18n issue in RBD configuration tables (`pr#35822 <https://github.com/ceph/ceph/pull/35822>`_, Kiefer Chang)
+* mgr/k8sevents: sanitise kubernetes events (`pr#35563 <https://github.com/ceph/ceph/pull/35563>`_, Paul Cuzner)
+* mgr/prometheus: improve Prometheus module cache (`pr#35918 <https://github.com/ceph/ceph/pull/35918>`_, Patrick Seidensal)
+* mgr: mgr/progress: Skip pg_summary update if _events dict is empty (`pr#36075 <https://github.com/ceph/ceph/pull/36075>`_, Manuel Lausch)
+* mgr: mgr/telemetry: force --license when sending while opted-out (`pr#35390 <https://github.com/ceph/ceph/pull/35390>`_, Yaarit Hatuka)
+* mgr: mon/PGMap: do not consider changing pg stuck (`pr#35959 <https://github.com/ceph/ceph/pull/35959>`_, Kefu Chai)
+* monitoring: fixing some issues in RBD detail dashboard (`pr#35464 <https://github.com/ceph/ceph/pull/35464>`_, Kiefer Chang)
+* msgr: New msgr2 crc and secure modes (msgr2.1) (`pr#35733 <https://github.com/ceph/ceph/pull/35733>`_, Jianpeng Ma, Ilya Dryomov)
+* rbd: librbd: new 'write_zeroes' API methods to suppliment the `discard` APIs (`pr#36250 <https://github.com/ceph/ceph/pull/36250>`_, Jason Dillaman)
+* rbd: mgr/dashboard: work with v1 RBD images (`pr#35712 <https://github.com/ceph/ceph/pull/35712>`_, Ernesto Puerta)
+* rbd: rbd: librbd: Watcher should not attempt to re-watch after detecting blacklisting (`pr#35385 <https://github.com/ceph/ceph/pull/35385>`_, Jason Dillaman)
+* rgw,tests: test/rgw: update hadoop versions (`pr#35778 <https://github.com/ceph/ceph/pull/35778>`_, Casey Bodley, Vasu Kulkarni)
+* rgw: Add subuser to OPA request (`pr#36187 <https://github.com/ceph/ceph/pull/36187>`_, Seena Fallah)
+* rgw: Add support wildcard subuser for bucket policy (`pr#36186 <https://github.com/ceph/ceph/pull/36186>`_, Seena Fallah)
+* rgw: add "rgw-orphan-list" tool and "radosgw-admin bucket radoslist ..." (`pr#34127 <https://github.com/ceph/ceph/pull/34127>`_, J. Eric Ivancich)
+* rgw: add check for index entry's existing when adding bucket stats during bucket reshard (`pr#36189 <https://github.com/ceph/ceph/pull/36189>`_, zhang Shaowen)
+* rgw: add quota enforcement to CopyObj (`pr#36184 <https://github.com/ceph/ceph/pull/36184>`_, Casey Bodley)
+* rgw: bucket list/stats truncates for user w/ >1000 buckets (`pr#36165 <https://github.com/ceph/ceph/pull/36165>`_, J. Eric Ivancich)
+* rgw: cls_bucket_list\_(un)ordered should clear results collection (`pr#36163 <https://github.com/ceph/ceph/pull/36163>`_, J. Eric Ivancich)
+* rgw: fix loop problem with swift stat on account (`pr#36185 <https://github.com/ceph/ceph/pull/36185>`_, Marcus Watts)
+* rgw: lc: fix Segmentation Fault when the tag of the object was not found (`pr#36086 <https://github.com/ceph/ceph/pull/36086>`_, yupeng chen, zhuo li)
+* rgw: ordered listing lcv not managed correctly (`pr#35882 <https://github.com/ceph/ceph/pull/35882>`_, J. Eric Ivancich)
+* rgw: radoslist incomplete multipart uploads fix marker progression (`pr#36191 <https://github.com/ceph/ceph/pull/36191>`_, J. Eric Ivancich)
+* rgw: rgw/iam: correcting the result of get role policy (`pr#36193 <https://github.com/ceph/ceph/pull/36193>`_, Pritha Srivastava)
+* rgw: rgw/url: fix amqp urls with vhosts (`pr#35384 <https://github.com/ceph/ceph/pull/35384>`_, Yuval Lifshitz)
+* rgw: stop realm reloader before store shutdown (`pr#36192 <https://github.com/ceph/ceph/pull/36192>`_, Casey Bodley)
+* tools: Add statfs operation to ceph-objecstore-tool (`pr#35713 <https://github.com/ceph/ceph/pull/35713>`_, David Zafman)
+
+
+v14.2.10 Nautilus
+=================
+
+This is the tenth release in the Nautilus series. In addition to fixing
+a security-related bug in RGW, this release brings a number of bugfixes
+across all major components of Ceph. We recommend that all Nautilus users
+upgrade to this release.
+
+Notable Changes
+---------------
+
+* CVE-2020-10753: rgw: sanitize newlines in s3 CORSConfiguration's ExposeHeader
+  (William Bowling, Adam Mohammed, Casey Bodley)
+
+* RGW: Bucket notifications now support Kafka endpoints. This requires librdkafka of
+  version 0.9.2 and up. Note that Ubuntu 16.04.6 LTS (Xenial Xerus) has an older
+  version of librdkafka, and would require an update to the library.
+
+* The pool parameter ``target_size_ratio``, used by the pg autoscaler,
+  has changed meaning. It is now normalized across pools, rather than
+  specifying an absolute ratio. For details, see :ref:`pg-autoscaler`.
+  If you have set target size ratios on any pools, you may want to set
+  these pools to autoscale ``warn`` mode to avoid data movement during
+  the upgrade::
+
+    ceph osd pool set <pool-name> pg_autoscale_mode warn
+
+* The behaviour of the ``-o`` argument to the rados tool has been reverted to
+  its original behaviour of indicating an output file. This reverts it to a more
+  consistent behaviour when compared to other tools. Specifying object size is now
+  accomplished by using an upper case O ``-O``.
+
+* The format of MDSs in `ceph fs dump` has changed.
+
+* Ceph will issue a health warning if a RADOS pool's ``size`` is set to 1
+  or in other words the pool is configured with no redundancy. This can
+  be fixed by setting the pool size to the minimum recommended value
+  with::
+
+    ceph osd pool set <pool-name> size <num-replicas>
+
+  The warning can be silenced with::
+
+    ceph config set global mon_warn_on_pool_no_redundancy false
+
+* RGW: bucket listing performance on sharded bucket indexes has been
+  notably improved by heuristically -- and significantly, in many
+  cases -- reducing the number of entries requested from each bucket
+  index shard.
+
+Changelog
+---------
+
+* build/ops: address SElinux denials observed in rgw/multisite test run (`pr#34539 <https://github.com/ceph/ceph/pull/34539>`_, Kefu Chai, Kaleb S. Keithley)
+* build/ops: ceph.spec.in: build on el8 (`pr#35599 <https://github.com/ceph/ceph/pull/35599>`_, Kefu Chai, Brad Hubbard, Alfonso Martínez, Nathan Cutler, Sage Weil, luo.runbing)
+* build/ops: cmake: Improve test for 16-byte atomic support on IBM Z (`pr#33716 <https://github.com/ceph/ceph/pull/33716>`_, Ulrich Weigand)
+* build/ops: do_cmake.sh: fix application of -DWITH_RADOSGW_KAFKA_ENDPOINT=OFF (`pr#34008 <https://github.com/ceph/ceph/pull/34008>`_, Nathan Cutler, Kefu Chai)
+* build/ops: install-deps.sh: Use dnf for rhel/centos 8 (`pr#35461 <https://github.com/ceph/ceph/pull/35461>`_, Brad Hubbard)
+* build/ops: rpm: add python3-saml as install dependency (`pr#34475 <https://github.com/ceph/ceph/pull/34475>`_, Kefu Chai, Ernesto Puerta)
+* build/ops: selinux: Allow ceph to setsched (`pr#34433 <https://github.com/ceph/ceph/pull/34433>`_, Brad Hubbard)
+* build/ops: selinux: Allow ceph-mgr access to httpd dir (`pr#34434 <https://github.com/ceph/ceph/pull/34434>`_, Brad Hubbard)
+* build/ops: selinux: Allow getattr access to /proc/kcore (`pr#34870 <https://github.com/ceph/ceph/pull/34870>`_, Brad Hubbard)
+* build/ops: spec: address some warnings raised by RPM 4.15.1 (`pr#34527 <https://github.com/ceph/ceph/pull/34527>`_, Nathan Cutler)
+* ceph-volume/batch: check lvs list before access (`pr#34481 <https://github.com/ceph/ceph/pull/34481>`_, Jan Fajerski)
+* ceph-volume/batch: return success when all devices are filtered (`pr#34478 <https://github.com/ceph/ceph/pull/34478>`_, Jan Fajerski)
+* ceph-volume: add and delete lvm tags in a single lvchange call (`pr#35453 <https://github.com/ceph/ceph/pull/35453>`_, Jan Fajerski)
+* ceph-volume: add ceph.osdspec_affinity tag (`pr#35132 <https://github.com/ceph/ceph/pull/35132>`_, Joshua Schmid)
+* ceph-volume: devices/simple/scan: Fix string in log statement (`pr#34445 <https://github.com/ceph/ceph/pull/34445>`_, Jan Fajerski)
+* ceph-volume: fix nautilus functional tests (`pr#33391 <https://github.com/ceph/ceph/pull/33391>`_, Jan Fajerski)
+* ceph-volume: lvm: get_device_vgs() filter by provided prefix (`pr#33616 <https://github.com/ceph/ceph/pull/33616>`_, Jan Fajerski, Yehuda Sadeh)
+* ceph-volume: prepare: use \*-slots arguments for implicit sizing (`pr#34278 <https://github.com/ceph/ceph/pull/34278>`_, Jan Fajerski)
+* ceph-volume: silence 'ceph-bluestore-tool' failures (`pr#33428 <https://github.com/ceph/ceph/pull/33428>`_, Sébastien Han)
+* ceph-volume: strip _dmcrypt suffix in simple scan json output (`pr#33722 <https://github.com/ceph/ceph/pull/33722>`_, Jan Fajerski)
+* cephfs/tools: add accounted_rstat/rstat when building file dentry (`pr#35185 <https://github.com/ceph/ceph/pull/35185>`_, Xiubo Li)
+* cephfs/tools: cephfs-journal-tool: correctly parse --dry_run argument (`pr#34784 <https://github.com/ceph/ceph/pull/34784>`_, Milind Changire)
+* cephfs: allow pool names with hyphen and period (`pr#35391 <https://github.com/ceph/ceph/pull/35391>`_, Rishabh Dave, Ramana Raja)
+* cephfs: ceph-fuse: link to libfuse3 and pass "-o big_writes" to libfuse if libfuse < 3.0.0 (`pr#34771 <https://github.com/ceph/ceph/pull/34771>`_, Kefu Chai, Xiubo Li, "Yan, Zheng")
+* cephfs: client: expose Client::ll_register_callback via libcephfs (`pr#35393 <https://github.com/ceph/ceph/pull/35393>`_, Kefu Chai, Jeff Layton)
+* cephfs: client: fix Finisher assert failure (`pr#35000 <https://github.com/ceph/ceph/pull/35000>`_, Xiubo Li)
+* cephfs: client: fix bad error handling in lseek SEEK_HOLE / SEEK_DATA (`pr#34308 <https://github.com/ceph/ceph/pull/34308>`_, Jeff Layton)
+* cephfs: client: only set MClientCaps::FLAG_SYNC when flushing dirty auth caps (`pr#35118 <https://github.com/ceph/ceph/pull/35118>`_, Jeff Layton)
+* cephfs: client: reset requested_max_size if file write is not wanted (`pr#34767 <https://github.com/ceph/ceph/pull/34767>`_, "Yan, Zheng")
+* cephfs: mds: Handle blacklisted error in purge queue (`pr#35149 <https://github.com/ceph/ceph/pull/35149>`_, Varsha Rao)
+* cephfs: mds: SIGSEGV in Migrator::export_sessions_flushed (`pr#33751 <https://github.com/ceph/ceph/pull/33751>`_, "Yan, Zheng")
+* cephfs: mds: Using begin() and empty() to iterate the xlist (`pr#34338 <https://github.com/ceph/ceph/pull/34338>`_, Shen Hang, "Yan, Zheng")
+* cephfs: mds: add configurable snapshot limit (`pr#33295 <https://github.com/ceph/ceph/pull/33295>`_, Milind Changire)
+* cephfs: mds: display scrub status in ceph status (`issue#41508 <http://tracker.ceph.com/issues/41508>`_, `issue#42713 <http://tracker.ceph.com/issues/42713>`_, `issue#44520 <http://tracker.ceph.com/issues/44520>`_, `issue#42168 <http://tracker.ceph.com/issues/42168>`_, `issue#42169 <http://tracker.ceph.com/issues/42169>`_, `issue#42569 <http://tracker.ceph.com/issues/42569>`_, `issue#41424 <http://tracker.ceph.com/issues/41424>`_, `issue#42835 <http://tracker.ceph.com/issues/42835>`_, `issue#36370 <http://tracker.ceph.com/issues/36370>`_, `issue#42325 <http://tracker.ceph.com/issues/42325>`_, `pr#30704 <https://github.com/ceph/ceph/pull/30704>`_, Venky Shankar, Patrick Donnelly, Sage Weil, Kefu Chai)
+* cephfs: mds: don't shallow copy when decoding xattr map (`pr#35199 <https://github.com/ceph/ceph/pull/35199>`_, "Yan, Zheng")
+* cephfs: mds: handle bad purge queue item encoding (`pr#34307 <https://github.com/ceph/ceph/pull/34307>`_, "Yan, Zheng")
+* cephfs: mds: handle ceph_assert on blacklisting (`pr#34435 <https://github.com/ceph/ceph/pull/34435>`_, Milind Changire)
+* cephfs: mds: just delete MDSIOContextBase during shutdown (`pr#34343 <https://github.com/ceph/ceph/pull/34343>`_, "Yan, Zheng", Patrick Donnelly)
+* cephfs: mds: take xlock in the order requests start locking (`pr#35392 <https://github.com/ceph/ceph/pull/35392>`_, "Yan, Zheng")
+* common/bl: fix memory corruption in bufferlist::claim_append() (`pr#34516 <https://github.com/ceph/ceph/pull/34516>`_, Radoslaw Zarzynski)
+* common/blkdev: compilation of telemetry and device backports (`pr#33726 <https://github.com/ceph/ceph/pull/33726>`_, Sage Weil, Difan Zhang, Patrick Seidensal, Kefu Chai)
+* common/blkdev: fix some problems with smart scraping (`pr#33421 <https://github.com/ceph/ceph/pull/33421>`_, Sage Weil)
+* common/ceph_time: tolerate mono time going backwards (`pr#34542 <https://github.com/ceph/ceph/pull/34542>`_, Sage Weil)
+* common/options: Disable bluefs_buffered_io by default again (`pr#34297 <https://github.com/ceph/ceph/pull/34297>`_, Mark Nelson)
+* compressor/lz4: work around bug in liblz4 versions <1.8.2 (`pr#35004 <https://github.com/ceph/ceph/pull/35004>`_, Sage Weil, Dan van der Ster)
+* core: bluestore/bdev: initialize size when creating object (`pr#34832 <https://github.com/ceph/ceph/pull/34832>`_, Willem Jan Withagen)
+* core: bluestore: Don't pollute old journal when add new device (`pr#34796 <https://github.com/ceph/ceph/pull/34796>`_, Yang Honggang)
+* core: bluestore: fix 'unused' calculation (`pr#34794 <https://github.com/ceph/ceph/pull/34794>`_, xie xingguo, Igor Fedotov)
+* core: bluestore: fix extent leak after main device expand (`pr#34711 <https://github.com/ceph/ceph/pull/34711>`_, Igor Fedotov)
+* core: bluestore: more flexible DB volume space usage (`pr#33889 <https://github.com/ceph/ceph/pull/33889>`_, Igor Fedotov)
+* core: bluestore: open DB in read-only when expanding DB/WAL (`pr#34611 <https://github.com/ceph/ceph/pull/34611>`_, Igor Fedotov, Jianpeng Ma, Adam Kupczyk)
+* core: bluestore: prevent BlueFS::dirty_files from being leaked when syncing metadata (`pr#34515 <https://github.com/ceph/ceph/pull/34515>`_, Xuehan Xu)
+* core: msg/async/rdma: fix bug event center is blocked by rdma construct connection for transport ib sync msg (`pr#34780 <https://github.com/ceph/ceph/pull/34780>`_, Peng Liu)
+* core: msgr: backport the EventCenter-related fixes (`pr#33820 <https://github.com/ceph/ceph/pull/33820>`_, Radoslaw Zarzynski, Jeff Layton, Kefu Chai)
+* core: rados: prevent ShardedOpWQ suicide_grace drop when waiting for work (`pr#34882 <https://github.com/ceph/ceph/pull/34882>`_, Dan Hill)
+* doc/mgr/telemetry: added device channel details (`pr#33684 <https://github.com/ceph/ceph/pull/33684>`_, Yaarit Hatuka)
+* doc/releases/nautilus: restart OSDs to make them bind to v2 addr (`pr#34524 <https://github.com/ceph/ceph/pull/34524>`_, Nathan Cutler)
+* doc: fix parameter to set pg autoscale mode (`pr#34518 <https://github.com/ceph/ceph/pull/34518>`_, Changcheng Liu)
+* doc: mds-config-ref: update 'mds_log_max_segments' value (`pr#35278 <https://github.com/ceph/ceph/pull/35278>`_, Konstantin Shalygin)
+* doc: reset PendingReleaseNotes following 14.2.8 release (`pr#33863 <https://github.com/ceph/ceph/pull/33863>`_, Nathan Cutler)
+* global: ensure CEPH_ARGS is decoded before early arg processing (`pr#33261 <https://github.com/ceph/ceph/pull/33261>`_, Kefu Chai, Jason Dillaman)
+* mgr/DaemonServer: fix pg merge checks (`pr#34354 <https://github.com/ceph/ceph/pull/34354>`_, Sage Weil)
+* mgr/PyModule: fix missing tracebacks in handle_pyerror() (`pr#34627 <https://github.com/ceph/ceph/pull/34627>`_, Tim Serong)
+* mgr/balancer: tolerate pgs outside of target weight map (`pr#34761 <https://github.com/ceph/ceph/pull/34761>`_, Sage Weil)
+* mgr/dashboard/grafana: Add rbd-image details dashboard (`pr#35248 <https://github.com/ceph/ceph/pull/35248>`_, Enno Gotthold)
+* mgr/dashboard: 'destroyed' view in CRUSH map viewer (`pr#33764 <https://github.com/ceph/ceph/pull/33764>`_, Avan Thakkar)
+* mgr/dashboard: Add more debug information to Dashboard RGW backend (`pr#34399 <https://github.com/ceph/ceph/pull/34399>`_, Volker Theile)
+* mgr/dashboard: Dashboard does not allow you to set norebalance OSD flag (`pr#33927 <https://github.com/ceph/ceph/pull/33927>`_, Nizamudeen)
+* mgr/dashboard: Disable cache for static files (`pr#33763 <https://github.com/ceph/ceph/pull/33763>`_, Tiago Melo)
+* mgr/dashboard: Display the aggregated number of request (`pr#35212 <https://github.com/ceph/ceph/pull/35212>`_, Tiago Melo)
+* mgr/dashboard: Fix HomeTest setup (`pr#35086 <https://github.com/ceph/ceph/pull/35086>`_, Tiago Melo)
+* mgr/dashboard: Fix cherrypy request logging error (`pr#31586 <https://github.com/ceph/ceph/pull/31586>`_, Kiefer Chang)
+* mgr/dashboard: Fix error in unit test caused by timezone (`pr#34473 <https://github.com/ceph/ceph/pull/34473>`_, Tiago Melo)
+* mgr/dashboard: Fix error when listing RBD while deleting or moving (`pr#34120 <https://github.com/ceph/ceph/pull/34120>`_, Tiago Melo)
+* mgr/dashboard: Fix iSCSI's username and password validation (`pr#34550 <https://github.com/ceph/ceph/pull/34550>`_, Tiago Melo)
+* mgr/dashboard: Fixes rbd image 'purge trash' button & modal text (`pr#33697 <https://github.com/ceph/ceph/pull/33697>`_, anurag)
+* mgr/dashboard: Improve workaround to redraw datatables (`pr#34413 <https://github.com/ceph/ceph/pull/34413>`_, Volker Theile)
+* mgr/dashboard: Not able to restrict bucket creation for new user (`pr#34692 <https://github.com/ceph/ceph/pull/34692>`_, Volker Theile)
+* mgr/dashboard: Pool read/write OPS shows too many decimal places (`pr#34039 <https://github.com/ceph/ceph/pull/34039>`_, anurag, Ernesto Puerta)
+* mgr/dashboard: Prevent iSCSI target recreation when editing controls (`pr#34551 <https://github.com/ceph/ceph/pull/34551>`_, Tiago Melo)
+* mgr/dashboard: REST API: OpenAPI docs require internet connection (`pr#33032 <https://github.com/ceph/ceph/pull/33032>`_, Patrick Seidensal)
+* mgr/dashboard: RGW port autodetection does not support "Beast" RGW frontend (`pr#34400 <https://github.com/ceph/ceph/pull/34400>`_, Volker Theile)
+* mgr/dashboard: Refactor Python unittests and controller (`pr#34662 <https://github.com/ceph/ceph/pull/34662>`_, Volker Theile)
+* mgr/dashboard: Repair broken grafana panels (`pr#34417 <https://github.com/ceph/ceph/pull/34417>`_, Kristoffer Grönlund)
+* mgr/dashboard: Searchable objects for table (`pr#32891 <https://github.com/ceph/ceph/pull/32891>`_, Stephan Müller)
+* mgr/dashboard: Tabs does not handle click events (`issue#39326 <http://tracker.ceph.com/issues/39326>`_, `pr#34282 <https://github.com/ceph/ceph/pull/34282>`_, Tiago Melo)
+* mgr/dashboard: UI fixes (`pr#34038 <https://github.com/ceph/ceph/pull/34038>`_, Avan Thakkar)
+* mgr/dashboard: Updated existing E2E tests to match new format (`pr#33024 <https://github.com/ceph/ceph/pull/33024>`_, Nathan Weinberg)
+* mgr/dashboard: Use booleanText pipe (`pr#33234 <https://github.com/ceph/ceph/pull/33234>`_, Alfonso Martínez, Volker Theile)
+* mgr/dashboard: Use default language when running "npm run build" (`pr#33668 <https://github.com/ceph/ceph/pull/33668>`_, Tiago Melo)
+* mgr/dashboard: do not show RGW API keys if only read-only privileges (`pr#33665 <https://github.com/ceph/ceph/pull/33665>`_, Alfonso Martínez)
+* mgr/dashboard: fix COVERAGE_PATH in run-backend-api-tests.sh (`pr#34489 <https://github.com/ceph/ceph/pull/34489>`_, Alfonso Martínez)
+* mgr/dashboard: fix backport #33764 (`pr#34640 <https://github.com/ceph/ceph/pull/34640>`_, Ernesto Puerta)
+* mgr/dashboard: fix error when enabling SSO with cert. file (`pr#34129 <https://github.com/ceph/ceph/pull/34129>`_, Alfonso Martínez)
+* mgr/dashboard: fix py2 strptime ImportError (not thread safe) (`pr#35016 <https://github.com/ceph/ceph/pull/35016>`_, Alfonso Martínez)
+* mgr/dashboard: fixing RBD purge error in backend (`pr#34847 <https://github.com/ceph/ceph/pull/34847>`_, Kiefer Chang)
+* mgr/dashboard: install teuthology using pip (`pr#35174 <https://github.com/ceph/ceph/pull/35174>`_, Nathan Cutler, Kefu Chai)
+* mgr/dashboard: list configured prometheus alerts (`pr#34373 <https://github.com/ceph/ceph/pull/34373>`_, Patrick Seidensal, Tiago Melo)
+* mgr/dashboard: monitoring menu entry should indicate firing alerts (`pr#34823 <https://github.com/ceph/ceph/pull/34823>`_, Tiago Melo, Volker Theile)
+* mgr/dashboard: remove 'config-opt: read' perm. from system roles (`pr#33739 <https://github.com/ceph/ceph/pull/33739>`_, Alfonso Martínez)
+* mgr/dashboard: show checkboxes for booleans (`pr#33388 <https://github.com/ceph/ceph/pull/33388>`_, Tatjana Dehler)
+* mgr/dashboard: use FQDN for failover redirection (`pr#34497 <https://github.com/ceph/ceph/pull/34497>`_, Ernesto Puerta)
+* mgr/insights: fix prune-health-history (`pr#35214 <https://github.com/ceph/ceph/pull/35214>`_, Sage Weil)
+* mgr/pg_autoscaler: fix division by zero (`pr#33420 <https://github.com/ceph/ceph/pull/33420>`_, Sage Weil)
+* mgr/pg_autoscaler: treat target ratios as weights (`pr#34087 <https://github.com/ceph/ceph/pull/34087>`_, Josh Durgin)
+* mgr/prometheus: ceph_pg\_\* metrics contains last value instead of sum across all reported states (`pr#34162 <https://github.com/ceph/ceph/pull/34162>`_, Jacek Suchenia)
+* mgr/run-tox-tests: Fix issue with PYTHONPATH (`pr#33688 <https://github.com/ceph/ceph/pull/33688>`_, Brad Hubbard)
+* mgr/telegraf: catch FileNotFoundError exception (`pr#34628 <https://github.com/ceph/ceph/pull/34628>`_, Kefu Chai)
+* mgr/telemetry: add 'last_upload' to status (`pr#33409 <https://github.com/ceph/ceph/pull/33409>`_, Yaarit Hatuka)
+* mgr/telemetry: catch exception during requests.put (`pr#33141 <https://github.com/ceph/ceph/pull/33141>`_, Sage Weil)
+* mgr/telemetry: fix UUID and STR concat (`pr#33666 <https://github.com/ceph/ceph/pull/33666>`_, Yaarit Hatuka)
+* mgr/telemetry: fix and document proxy usage (`pr#33649 <https://github.com/ceph/ceph/pull/33649>`_, Lars Marowsky-Bree)
+* mgr/volumes: Add interface to get subvolume metadata (`pr#34679 <https://github.com/ceph/ceph/pull/34679>`_, Kotresh HR)
+* mgr/volumes: fs subvolume clone cancel (`issue#44208 <http://tracker.ceph.com/issues/44208>`_, `pr#34036 <https://github.com/ceph/ceph/pull/34036>`_, Venky Shankar, Michael Fritch)
+* mgr/volumes: minor fixes (`pr#35482 <https://github.com/ceph/ceph/pull/35482>`_, Kotresh HR)
+* mgr/volumes: synchronize ownership (for symlinks) and inode timestamps for cloned subvolumes (`issue#24880 <http://tracker.ceph.com/issues/24880>`_, `issue#43965 <http://tracker.ceph.com/issues/43965>`_, `pr#33877 <https://github.com/ceph/ceph/pull/33877>`_, Ramana Raja, Rishabh Dave, huanwen ren, Venky Shankar, Jos Collin)
+* mgr: Add get_rates_from_data to mgr_util.py (`pr#33893 <https://github.com/ceph/ceph/pull/33893>`_, Stephan Müller, Ernesto Puerta)
+* mgr: Improve internal python to c++ interface (`pr#34356 <https://github.com/ceph/ceph/pull/34356>`_, David Zafman)
+* mgr: close restful socket after exec (`pr#35213 <https://github.com/ceph/ceph/pull/35213>`_, liushi)
+* mgr: force purge normal ceph entities from service map (`issue#44677 <http://tracker.ceph.com/issues/44677>`_, `pr#34563 <https://github.com/ceph/ceph/pull/34563>`_, Venky Shankar)
+* mgr: synchronize ClusterState's health and mon_status (`pr#34326 <https://github.com/ceph/ceph/pull/34326>`_, Radoslaw Zarzynski)
+* mgr: update "hostname" when we already have the daemon state from that entity (`pr#33834 <https://github.com/ceph/ceph/pull/33834>`_, Kefu Chai)
+* mon/FSCommands: Fix 'add_data_pool' command and 'fs new' command (`pr#34774 <https://github.com/ceph/ceph/pull/34774>`_, Ramana Raja)
+* mon/OSDMonitor: Always tune priority cache manager memory on all mons (`pr#34916 <https://github.com/ceph/ceph/pull/34916>`_, Sridhar Seshasayee)
+* mon/OSDMonitor: allow trimming maps even if osds are down (`pr#34983 <https://github.com/ceph/ceph/pull/34983>`_, Joao Eduardo Luis)
+* mon/PGMap: fix summary display of >32bit pg states (`pr#33275 <https://github.com/ceph/ceph/pull/33275>`_, Sage Weil, Adam C. Emerson)
+* mon: Get session_map_lock before remove_session (`pr#34677 <https://github.com/ceph/ceph/pull/34677>`_, Xiaofei Cui)
+* mon: calculate min_size on osd pool set size (`pr#34585 <https://github.com/ceph/ceph/pull/34585>`_, Deepika Upadhyay)
+* mon: disable min pg per osd warning (`pr#34618 <https://github.com/ceph/ceph/pull/34618>`_, Sage Weil)
+* mon: fix/improve mon sync over small keys (`pr#33765 <https://github.com/ceph/ceph/pull/33765>`_, Sage Weil)
+* mon: stash newer map on bootstrap when addr doesn't match (`pr#34500 <https://github.com/ceph/ceph/pull/34500>`_, Sage Weil)
+* monitoring: Fix "10% OSDs down" alert description (`pr#35211 <https://github.com/ceph/ceph/pull/35211>`_, Benoît Knecht)
+* monitoring: Fix pool capacity incorrect (`pr#34450 <https://github.com/ceph/ceph/pull/34450>`_, James Cheng)
+* monitoring: alert for pool fill up broken (`pr#35137 <https://github.com/ceph/ceph/pull/35137>`_, Volker Theile)
+* monitoring: alert for prediction of disk and pool fill up broken (`pr#34394 <https://github.com/ceph/ceph/pull/34394>`_, Patrick Seidensal)
+* monitoring: fix RGW grafana chart 'Average GET/PUT Latencies' (`pr#33860 <https://github.com/ceph/ceph/pull/33860>`_, Alfonso Martínez)
+* monitoring: fix decimal precision in Grafana %percentages (`pr#34829 <https://github.com/ceph/ceph/pull/34829>`_, Ernesto Puerta)
+* monitoring: root volume full alert fires false positives (`pr#34419 <https://github.com/ceph/ceph/pull/34419>`_, Patrick Seidensal)
+* osd/OSD: Log slow ops/types to cluster logs (`pr#33503 <https://github.com/ceph/ceph/pull/33503>`_, Sage Weil, Sridhar Seshasayee)
+* osd/OSDMap: Show health warning if a pool is configured with size 1 (`pr#31842 <https://github.com/ceph/ceph/pull/31842>`_, Sridhar Seshasayee)
+* osd/PeeringState.h: ignore RemoteBackfillReserved in WaitLocalBackfillReserved (`pr#34512 <https://github.com/ceph/ceph/pull/34512>`_, Neha Ojha)
+* osd/PeeringState: do not trim pg log past last_update_ondisk (`pr#34957 <https://github.com/ceph/ceph/pull/34957>`_, Samuel Just, xie xingguo)
+* osd/PeeringState: transit async_recovery_targets back into acting before backfilling (`pr#32849 <https://github.com/ceph/ceph/pull/32849>`_, xie xingguo)
+* osd: dispatch_context and queue split finish on early bail-out (`pr#35024 <https://github.com/ceph/ceph/pull/35024>`_, Sage Weil)
+* osd: fix racy accesses to OSD::osdmap (`pr#33530 <https://github.com/ceph/ceph/pull/33530>`_, Radoslaw Zarzynski)
+* pybind/mgr/\*: fix config_notify handling of default values (`pr#34116 <https://github.com/ceph/ceph/pull/34116>`_, Nathan Cutler, Sage Weil)
+* pybind/mgr: use six==1.14.0 (`pr#34316 <https://github.com/ceph/ceph/pull/34316>`_, Kefu Chai)
+* pybind/rbd: RBD.create() method's 'old_format' parameter now defaults to False (`pr#35183 <https://github.com/ceph/ceph/pull/35183>`_, Jason Dillaman)
+* pybind/rbd: ensure image is open before permitting operations (`pr#34424 <https://github.com/ceph/ceph/pull/34424>`_, Mykola Golub)
+* pybind/rbd: fix no lockers are obtained, ImageNotFound exception will be output (`pr#34388 <https://github.com/ceph/ceph/pull/34388>`_, zhangdaolong)
+* rbd: librbd: copy API should not inherit v1 image format by default (`pr#35182 <https://github.com/ceph/ceph/pull/35182>`_, Jason Dillaman)
+* rbd: rbd-mirror: improve detection of blacklisted state (`pr#33533 <https://github.com/ceph/ceph/pull/33533>`_, Mykola Golub)
+* rgw/kafka: add kafka endpoint support (`pr#32960 <https://github.com/ceph/ceph/pull/32960>`_, Yuval Lifshitz, Willem Jan Withagen, Kefu Chai)
+* rgw/notifications: backporting features and bug fix (`pr#34107 <https://github.com/ceph/ceph/pull/34107>`_, Yuval Lifshitz)
+* rgw/notifications: fix topic action fail with "MethodNotAllowed" (`issue#44614 <http://tracker.ceph.com/issues/44614>`_, `pr#33978 <https://github.com/ceph/ceph/pull/33978>`_, Yuval Lifshitz)
+* rgw/notifications: version id was not sent in versioned buckets (`pr#35181 <https://github.com/ceph/ceph/pull/35181>`_, Yuval Lifshitz)
+* rgw:  when you abort a multipart upload request, the quota may be not updated (`pr#33268 <https://github.com/ceph/ceph/pull/33268>`_, Richard Bai(白学余))
+* rgw: Add support bucket policy for subuser (`pr#33714 <https://github.com/ceph/ceph/pull/33714>`_, Seena Fallah)
+* rgw: Fix dynamic resharding not working for empty zonegroup in period (`pr#33266 <https://github.com/ceph/ceph/pull/33266>`_, Or Friedmann)
+* rgw: Fix upload part copy range able to get almost any string (`pr#33265 <https://github.com/ceph/ceph/pull/33265>`_, Or Friedmann)
+* rgw: GET/HEAD and PUT operations on buckets w/lifecycle expiration configured do not return x-amz-expiration header (`pr#32924 <https://github.com/ceph/ceph/pull/32924>`_, Matt Benjamin, Yuval Lifshitz)
+* rgw: MultipartObjectProcessor supports stripe size > chunk size (`pr#33271 <https://github.com/ceph/ceph/pull/33271>`_, Casey Bodley)
+* rgw: ReplaceKeyPrefixWith and ReplaceKeyWith can not set at the same … (`pr#34599 <https://github.com/ceph/ceph/pull/34599>`_, yuliyang)
+* rgw: anonomous swift to obj that dont exist should 401 (`pr#35045 <https://github.com/ceph/ceph/pull/35045>`_, Matthew Oliver)
+* rgw: clear ent_list for each loop of bucket list (`issue#44394 <http://tracker.ceph.com/issues/44394>`_, `pr#34099 <https://github.com/ceph/ceph/pull/34099>`_, Yao Zongyou)
+* rgw: dmclock: wait until the request is handled (`pr#34954 <https://github.com/ceph/ceph/pull/34954>`_, GaryHyg)
+* rgw: find oldest period and update RGWMetadataLogHistory() (`pr#34597 <https://github.com/ceph/ceph/pull/34597>`_, Shilpa Jagannath)
+* rgw: fix SignatureDoesNotMatch when use ipv6 address in s3 client (`pr#33267 <https://github.com/ceph/ceph/pull/33267>`_, yuliyang)
+* rgw: fix bug with (un)ordered bucket listing and marker w/ namespace (`pr#34609 <https://github.com/ceph/ceph/pull/34609>`_, J. Eric Ivancich)
+* rgw: fix lc does not delete objects that do not have exactly the same tags as the rule (`pr#35002 <https://github.com/ceph/ceph/pull/35002>`_, Or Friedmann)
+* rgw: fix multipart upload's error response (`pr#35019 <https://github.com/ceph/ceph/pull/35019>`_, GaryHyg)
+* rgw: fix rgw crash when duration is invalid in sts request (`pr#33273 <https://github.com/ceph/ceph/pull/33273>`_, yuliyang)
+* rgw: fix some list buckets handle leak (`pr#34986 <https://github.com/ceph/ceph/pull/34986>`_, Tianshan Qu)
+* rgw: get barbican secret key request maybe return error code (`pr#33965 <https://github.com/ceph/ceph/pull/33965>`_, Richard Bai(白学余))
+* rgw: increase log level for same or older period pull msg (`pr#34833 <https://github.com/ceph/ceph/pull/34833>`_, Ali Maredia)
+* rgw: make max_connections configurable in beast (`pr#33340 <https://github.com/ceph/ceph/pull/33340>`_, Tiago Pasqualini)
+* rgw: making implicit_tenants backwards compatible (`issue#24348 <http://tracker.ceph.com/issues/24348>`_, `pr#33749 <https://github.com/ceph/ceph/pull/33749>`_, Marcus Watts)
+* rgw: multisite: enforce spawn window for incremental data sync (`pr#33270 <https://github.com/ceph/ceph/pull/33270>`_, Casey Bodley)
+* rgw: radosgw-admin: add support for --bucket-id in bucket stats command (`pr#34815 <https://github.com/ceph/ceph/pull/34815>`_, Vikhyat Umrao)
+* rgw: radosgw-admin: fix infinite loops in 'datalog list' (`pr#35001 <https://github.com/ceph/ceph/pull/35001>`_, Casey Bodley)
+* rgw: reshard: skip stale bucket id entries from reshard queue (`pr#34735 <https://github.com/ceph/ceph/pull/34735>`_, Abhishek Lekshmanan)
+* rgw: set bucket attr twice when delete lifecycle config (`pr#34598 <https://github.com/ceph/ceph/pull/34598>`_, zhang Shaowen)
+* rgw: set correct storage class for append (`pr#34064 <https://github.com/ceph/ceph/pull/34064>`_, yuliyang)
+* rgw: sts: add all http args to req_info (`pr#33355 <https://github.com/ceph/ceph/pull/33355>`_, yuliyang)
+* rgw: tune sharded bucket listing (`pr#33675 <https://github.com/ceph/ceph/pull/33675>`_, J. Eric Ivancich)
+* tests: migrate qa/ to python3 (`pr#34171 <https://github.com/ceph/ceph/pull/34171>`_, Kefu Chai, Sage Weil, Casey Bodley, Rishabh Dave, Patrick Donnelly, Kyr Shatskyy, Michael Fritch, Xiubo Li, Ilya Dryomov, Alfonso Martínez, Thomas Bechtold)
+* tools/cli: bash_completion: Do not auto complete obsolete and hidden cmds (`pr#35117 <https://github.com/ceph/ceph/pull/35117>`_, Kotresh HR)
+* tools/cli: ceph_argparse: increment matchcnt on kwargs (`pr#33160 <https://github.com/ceph/ceph/pull/33160>`_, Matthew Oliver, Shyukri Shyukriev)
+* tools/rados: Unmask '-o' to restore original behaviour (`pr#33641 <https://github.com/ceph/ceph/pull/33641>`_, Brad Hubbard)
+
+v14.2.9 Nautilus
+================
+
+This is the ninth bugfix release of Nautilus. This release fixes a
+couple of security issues in RGW & Messenger V2. We recommend all users
+to upgrade to this release.
+
+Notable Changes
+---------------
+
+- CVE-2020-1759: Fixed nonce reuse in msgr V2 secure mode
+- CVE-2020-1760: Fixed XSS due to RGW GetObject header-splitting
+
+v14.2.8 Nautilus
+================
+
+This is the eighth update to the Ceph Nautilus release series. This release
+fixes issues across a range of subsystems. We recommend that all users upgrade
+to this release.
+
+Notable Changes
+---------------
+
+* The default value of ``bluestore_min_alloc_size_ssd`` has been changed to 4K to improve performance across all workloads.
+
+* The following OSD memory config options related to bluestore cache autotuning can now
+  be configured during runtime:
+
+    - osd_memory_base (default: 768 MB)
+    - osd_memory_cache_min (default: 128 MB)
+    - osd_memory_expected_fragmentation (default: 0.15)
+    - osd_memory_target (default: 4 GB)
+
+  The above options can be set with::
+
+    ceph config set osd <option> <value>
+
+* The MGR now accepts ``profile rbd`` and ``profile rbd-read-only`` user caps.
+  These caps can be used to provide users access to MGR-based RBD functionality
+  such as ``rbd perf image iostat`` an ``rbd perf image iotop``.
+
+* The configuration value ``osd_calc_pg_upmaps_max_stddev`` used for upmap
+  balancing has been removed. Instead use the mgr balancer config
+  ``upmap_max_deviation`` which now is an integer number of PGs of deviation
+  from the target PGs per OSD.  This can be set with a command like
+  ``ceph config set mgr mgr/balancer/upmap_max_deviation 2``.  The default
+  ``upmap_max_deviation`` is 5.  There are situations where crush rules
+  would not allow a pool to ever have completely balanced PGs.  For example, if
+  crush requires 1 replica on each of 3 racks, but there are fewer OSDs in 1 of
+  the racks.  In those cases, the configuration value can be increased.
+
+* RGW: a mismatch between the bucket notification documentation and the actual
+  message format was fixed. This means that any endpoints receiving bucket
+  notification, will now receive the same notifications inside a JSON array
+  named 'Records'. Note that this does not affect pulling bucket notification
+  from a subscription in a 'pubsub' zone, as these are already wrapped inside
+  that array.
+
+* CephFS: multiple active MDS forward scrub is now rejected. Scrub currently
+  only is permitted on a file system with a single rank. Reduce the ranks to one
+  via ``ceph fs set <fs_name> max_mds 1``.
+
+* Ceph now refuses to create a file system with a default EC data pool. For
+  further explanation, see:
+  https://docs.ceph.com/docs/nautilus/cephfs/createfs/#creating-pools
+
+* Ceph will now issue a health warning if a RADOS pool has a ``pg_num``
+  value that is not a power of two. This can be fixed by adjusting
+  the pool to a nearby power of two::
+
+    ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+  Alternatively, the warning can be silenced with::
+
+    ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false
+
+
+Changelog
+---------
+
+* bluestore: common/options: bluestore 4k min_alloc_size for SSD (`pr#32998 <https://github.com/ceph/ceph/pull/32998>`_, Mark Nelson, Sage Weil)
+* bluestore: os/bluestore: Add config observer for osd memory specific options (`pr#31852 <https://github.com/ceph/ceph/pull/31852>`_, Sridhar Seshasayee)
+* bluestore: os/bluestore/BlueStore.cc: set priorities for compression stats (`pr#32845 <https://github.com/ceph/ceph/pull/32845>`_, Neha Ojha)
+* bluestore: os/bluestore: default bluestore_block_size 1T -> 100G (`pr#32283 <https://github.com/ceph/ceph/pull/32283>`_, Sage Weil)
+* build/ops: cmake: remove seastar tests from "make check" (`pr#32658 <https://github.com/ceph/ceph/pull/32658>`_, Kefu Chai)
+* build/ops: install-deps,rpm: enable devtoolset-8 on aarch64 also (`issue#38892 <http://tracker.ceph.com/issues/38892>`_, `pr#32651 <https://github.com/ceph/ceph/pull/32651>`_, Kefu Chai)
+* build/ops: rpm: add rpm-build to SUSE-specific make check deps (`pr#32208 <https://github.com/ceph/ceph/pull/32208>`_, Nathan Cutler)
+* build/ops: switch to boost 1.72 (`pr#32441 <https://github.com/ceph/ceph/pull/32441>`_, Willem Jan Withagen, Kefu Chai)
+* build/ops: tools/setup-virtualenv.sh: do not default to python2.7 (`pr#30739 <https://github.com/ceph/ceph/pull/30739>`_, Nathan Cutler)
+* cephfs: cephfs-journal-tool: fix crash and usage (`pr#32913 <https://github.com/ceph/ceph/pull/32913>`_, Xiubo Li)
+* cephfs: client: Add is_dir() check before changing directory (`pr#32916 <https://github.com/ceph/ceph/pull/32916>`_, Varsha Rao)
+* cephfs: client: add procession of SEEK_HOLE and SEEK_DATA in lseek (`pr#30764 <https://github.com/ceph/ceph/pull/30764>`_, Shen Hang)
+* cephfs: client: add warning when cap != in->auth_cap (`pr#32065 <https://github.com/ceph/ceph/pull/32065>`_, Shen Hang)
+* cephfs: client: EINVAL may be returned when offset is 0 (`pr#30762 <https://github.com/ceph/ceph/pull/30762>`_, wenpengLi)
+* cephfs: client: fix lazyio_synchronize() to update file size and libcephfs: Add Tests for LazyIO (`pr#30769 <https://github.com/ceph/ceph/pull/30769>`_, Sidharth Anupkrishnan)
+* cephfs: client: _readdir_cache_cb() may use the readdir_cache already clear (`issue#41148 <http://tracker.ceph.com/issues/41148>`_, `pr#30763 <https://github.com/ceph/ceph/pull/30763>`_, huanwen ren)
+* cephfs: client: remove Inode.dir_contacts field and handle bad whence value to llseek gracefully (`pr#30766 <https://github.com/ceph/ceph/pull/30766>`_, Jeff Layton)
+* cephfs,common: osdc/objecter: Fix last_sent in scientific format and add age to ops (`pr#31081 <https://github.com/ceph/ceph/pull/31081>`_, Varsha Rao)
+* cephfs: disallow changing fuse_default_permissions option at runtime (`pr#32915 <https://github.com/ceph/ceph/pull/32915>`_, Zhi Zhang)
+* cephfs: mds: add command that config individual client session (`issue#40811 <http://tracker.ceph.com/issues/40811>`_, `pr#32245 <https://github.com/ceph/ceph/pull/32245>`_, "Yan, Zheng")
+* cephfs: mds: "apply configuration changes through MDSRank" and "recall caps from quiescent sessions" and "drive cap recall while dropping cache" (`pr#30761 <https://github.com/ceph/ceph/pull/30761>`_, Patrick Donnelly, Jeff Layton)
+* cephfs: mds: fix assert(omap_num_objs <= MAX_OBJECTS) of OpenFileTable (`pr#32756 <https://github.com/ceph/ceph/pull/32756>`_, "Yan, Zheng")
+* cephfs: mds: fix revoking caps after after stale->resume circle (`pr#32909 <https://github.com/ceph/ceph/pull/32909>`_, "Yan, Zheng")
+* cephfs: mds: free heap memory may grow too large for some workloads (`pr#31802 <https://github.com/ceph/ceph/pull/31802>`_, Patrick Donnelly)
+* cephfs: MDSMonitor: warn if a new file system is being created with an EC default data pool (`pr#32600 <https://github.com/ceph/ceph/pull/32600>`_, Patrick Donnelly)
+* cephfs: mds: no assert on frozen dir when scrub path (`pr#32071 <https://github.com/ceph/ceph/pull/32071>`_, Zhi Zhang)
+* cephfs: mds: note client features when rejecting client (`pr#32914 <https://github.com/ceph/ceph/pull/32914>`_, Patrick Donnelly)
+* cephfs:  mds/OpenFileTable: match MAX_ITEMS_PER_OBJ to osd_deep_scrub_large_omap_object_key_threshold (`pr#32921 <https://github.com/ceph/ceph/pull/32921>`_, Vikhyat Umrao, Varsha Rao)
+* cephfs: mds: properly evaluate unstable locks when evicting client (`pr#32073 <https://github.com/ceph/ceph/pull/32073>`_, "Yan, Zheng")
+* cephfs: mds: reject forward scrubs when cluster has multiple active MDS (more than one rank) (`pr#32602 <https://github.com/ceph/ceph/pull/32602>`_, Patrick Donnelly, Milind Changire)
+* cephfs: mds: reject sessionless messages (`issue#40784 <http://tracker.ceph.com/issues/40784>`_, `pr#30843 <https://github.com/ceph/ceph/pull/30843>`_, "Yan, Zheng", Xiao Guodong, Shen Hang)
+* cephfs: mds: remove unnecessary debug warning (`pr#32077 <https://github.com/ceph/ceph/pull/32077>`_, Patrick Donnelly)
+* cephfs: mds returns -5(EIO) error when the deleted file does not exist (`pr#30767 <https://github.com/ceph/ceph/pull/30767>`_, huanwen ren)
+* cephfs: mds: split the dir if the op makes it oversized, because some ops maybe in flight (`pr#31302 <https://github.com/ceph/ceph/pull/31302>`_, simon gao)
+* cephfs: mds: tolerate no snaprealm encoded in on-disk root inode (`pr#32079 <https://github.com/ceph/ceph/pull/32079>`_, "Yan, Zheng")
+* cephfs: mgr: "mds metadata" to setup new DaemonState races with fsmap (`pr#31905 <https://github.com/ceph/ceph/pull/31905>`_, Patrick Donnelly)
+* cephfs: mgr/volumes: allow setting uid, gid of subvolume and subvolume group during creation (`issue#42923 <http://tracker.ceph.com/issues/42923>`_, `pr#31741 <https://github.com/ceph/ceph/pull/31741>`_, Venky Shankar, Jos Collin)
+* cephfs:  mgr/volumes: fetch trash and clone entries without blocking volume access (`issue#44282 <http://tracker.ceph.com/issues/44282>`_, `pr#33526 <https://github.com/ceph/ceph/pull/33526>`_, Venky Shankar)
+* cephfs:  mgr/volumes: fs subvolume resize command (`pr#31332 <https://github.com/ceph/ceph/pull/31332>`_, Jos Collin)
+* cephfs: mgr/volumes: misc fix and feature enhancements (`issue#42646 <http://tracker.ceph.com/issues/42646>`_, `issue#43645 <http://tracker.ceph.com/issues/43645>`_, `pr#33122 <https://github.com/ceph/ceph/pull/33122>`_, Rishabh Dave, Joshua Schmid, Venky Shankar, Ramana Raja, Jos Collin)
+* cephfs:  mgr/volumes: unregister job upon async threads exception (`issue#44315 <http://tracker.ceph.com/issues/44315>`_, `pr#33569 <https://github.com/ceph/ceph/pull/33569>`_, Venky Shankar)
+* cephfs:  mon: print FSMap regardless of file system count (`pr#32912 <https://github.com/ceph/ceph/pull/32912>`_, Patrick Donnelly)
+* cephfs:  pybind/mgr/volumes: idle connection drop is not working (`pr#33116 <https://github.com/ceph/ceph/pull/33116>`_, Patrick Donnelly)
+* cephfs: RuntimeError: Files in flight high water is unexpectedly low (0 / 6) (`pr#33115 <https://github.com/ceph/ceph/pull/33115>`_, Patrick Donnelly)
+* ceph.in: check ceph-conf returncode (`pr#31367 <https://github.com/ceph/ceph/pull/31367>`_, Dimitri Savineau)
+* ceph-monstore-tool: correct the key for storing mgr_command_descs (`pr#33278 <https://github.com/ceph/ceph/pull/33278>`_, Kefu Chai)
+* ceph-volume: add db and wal support to raw mode (`pr#32979 <https://github.com/ceph/ceph/pull/32979>`_, Sébastien Han)
+* ceph-volume: add methods to pass filters to pvs, vgs and lvs commands (`pr#33217 <https://github.com/ceph/ceph/pull/33217>`_, Rishabh Dave)
+* ceph-volume: add raw (--bluestore) mode (`pr#32733 <https://github.com/ceph/ceph/pull/32733>`_, Jan Fajerski, Sage Weil)
+* ceph-volume: add sizing arguments to prepare (`pr#33231 <https://github.com/ceph/ceph/pull/33231>`_, Jan Fajerski)
+* ceph-volume: allow raw block devices everywhere (`pr#32868 <https://github.com/ceph/ceph/pull/32868>`_, Jan Fajerski)
+* ceph-volume: assume msgrV1 for all branches containing mimic (`pr#31616 <https://github.com/ceph/ceph/pull/31616>`_, Jan Fajerski)
+* ceph-volume: avoid calling zap_lv with a LV-less VG (`pr#33297 <https://github.com/ceph/ceph/pull/33297>`_, Jan Fajerski)
+* ceph-volume: batch bluestore fix create_lvs call (`pr#33232 <https://github.com/ceph/ceph/pull/33232>`_, Jan Fajerski)
+* ceph-volume: batch bluestore fix create_lvs call (`pr#33301 <https://github.com/ceph/ceph/pull/33301>`_, Jan Fajerski)
+* ceph-volume/batch: fail on filtered devices when non-interactive (`pr#33202 <https://github.com/ceph/ceph/pull/33202>`_, Jan Fajerski)
+* ceph-volume: Dereference symlink in lvm list (`pr#32877 <https://github.com/ceph/ceph/pull/32877>`_, Benoît Knecht)
+* ceph-volume: don't remove vg twice when zapping filestore (`pr#33337 <https://github.com/ceph/ceph/pull/33337>`_, Jan Fajerski)
+* ceph-volume: finer grained availability notion in inventory (`pr#33240 <https://github.com/ceph/ceph/pull/33240>`_, Jan Fajerski)
+* ceph-volume: fix has_bluestore_label() function (`pr#33239 <https://github.com/ceph/ceph/pull/33239>`_, Guillaume Abrioux)
+* ceph-volume: fix is_ceph_device for lvm batch (`pr#33253 <https://github.com/ceph/ceph/pull/33253>`_, Jan Fajerski, Dimitri Savineau)
+* ceph-volume: fix the integer overflow (`pr#32873 <https://github.com/ceph/ceph/pull/32873>`_, dongdong tao)
+* ceph-volume: import mock.mock instead of unittest.mock (py2) (`pr#32870 <https://github.com/ceph/ceph/pull/32870>`_, Jan Fajerski)
+* ceph-volume/lvm/activate.py: clarify error message: fsid refers to osd_fsid (`pr#32864 <https://github.com/ceph/ceph/pull/32864>`_, Yaniv Kaul)
+* ceph-volume: lvm/deactivate: add unit tests, remove --all (`pr#32863 <https://github.com/ceph/ceph/pull/32863>`_, Jan Fajerski)
+* ceph-volume: lvm deactivate command (`pr#33209 <https://github.com/ceph/ceph/pull/33209>`_, Jan Fajerski)
+* ceph-volume: make get_devices fs location independent (`pr#33200 <https://github.com/ceph/ceph/pull/33200>`_, Jan Fajerski)
+* ceph-volume: minor clean-up of "simple scan" subcommand help (`pr#32556 <https://github.com/ceph/ceph/pull/32556>`_, Michael Fritch)
+* ceph-volume: pass journal_size as Size not string (`pr#33334 <https://github.com/ceph/ceph/pull/33334>`_, Jan Fajerski)
+* ceph-volume: refactor listing.py + fixes (`pr#33238 <https://github.com/ceph/ceph/pull/33238>`_, Jan Fajerski, Rishabh Dave, Guillaume Abrioux)
+* ceph-volume: reject disks smaller then 5GB in inventory (`issue#40776 <http://tracker.ceph.com/issues/40776>`_, `pr#31554 <https://github.com/ceph/ceph/pull/31554>`_, Jan Fajerski)
+* ceph-volume: skip osd creation when already done (`pr#33242 <https://github.com/ceph/ceph/pull/33242>`_, Guillaume Abrioux)
+* ceph-volume/test: patch VolumeGroups (`pr#32558 <https://github.com/ceph/ceph/pull/32558>`_, Jan Fajerski)
+* ceph-volume: use correct extents if using db-devices and >1 osds_per_device (`pr#32874 <https://github.com/ceph/ceph/pull/32874>`_, Fabian Niepelt)
+* ceph-volume: use fsync for dd command (`pr#31553 <https://github.com/ceph/ceph/pull/31553>`_, Rishabh Dave)
+* ceph-volume: use get_device_vgs in has_common_vg (`pr#33254 <https://github.com/ceph/ceph/pull/33254>`_, Jan Fajerski)
+* ceph-volume: util: look for executable in $PATH (`pr#32860 <https://github.com/ceph/ceph/pull/32860>`_, Shyukri Shyukriev)
+* ceph-volume/zfs: add the inventory command (`pr#31295 <https://github.com/ceph/ceph/pull/31295>`_, Willem Jan Withagen)
+* common/admin_socket: Increase socket timeouts (`pr#32063 <https://github.com/ceph/ceph/pull/32063>`_, Brad Hubbard)
+* common/bl: fix the dangling last_p issue (`pr#33277 <https://github.com/ceph/ceph/pull/33277>`_, Radoslaw Zarzynski)
+* common/config: update values when they are removed via mon (`pr#32846 <https://github.com/ceph/ceph/pull/32846>`_, Sage Weil)
+* common: FIPS: audit and switch some memset & bzero users (`pr#32167 <https://github.com/ceph/ceph/pull/32167>`_, Radoslaw Zarzynski)
+* common: fix deadlocky inflight op visiting in OpTracker (`pr#32858 <https://github.com/ceph/ceph/pull/32858>`_, Radoslaw Zarzynski)
+* common/options: remove unused ms_msgr2\_{sign,encrypt} (`pr#31850 <https://github.com/ceph/ceph/pull/31850>`_, Ilya Dryomov)
+* common/util: use ifstream to read from /proc files (`pr#32901 <https://github.com/ceph/ceph/pull/32901>`_, Kefu Chai, songweibin)
+* core: auth/Crypto: fallback to /dev/urandom if getentropy() fails (`pr#31301 <https://github.com/ceph/ceph/pull/31301>`_, Kefu Chai)
+* core: mon: keep v1 address type when explicitly set (`pr#32028 <https://github.com/ceph/ceph/pull/32028>`_, Ricardo Dias)
+* core: mon/OSDMonitor: Fix pool set target_size_bytes (etc) with unit suffix (`pr#31740 <https://github.com/ceph/ceph/pull/31740>`_, Prashant D)
+* core: osd/OSDMap: health alert for non-power-of-two pg_num (`pr#30689 <https://github.com/ceph/ceph/pull/30689>`_, Sage Weil)
+* crush/CrushWrapper: behave with empty weight vector (`pr#32905 <https://github.com/ceph/ceph/pull/32905>`_, Kefu Chai)
+* doc/cephfs/client-auth: description and example are inconsistent (`pr#32781 <https://github.com/ceph/ceph/pull/32781>`_, Ilya Dryomov)
+* doc/cephfs: improve add/remove MDS section (`issue#39620 <http://tracker.ceph.com/issues/39620>`_, `pr#31116 <https://github.com/ceph/ceph/pull/31116>`_, Patrick Donnelly)
+* doc/ceph-fuse: mention -k option in ceph-fuse man page (`pr#30765 <https://github.com/ceph/ceph/pull/30765>`_, Rishabh Dave)
+* doc/ceph-volume: initial docs for zfs/inventory and zfs/api (`pr#32746 <https://github.com/ceph/ceph/pull/32746>`_, Willem Jan Withagen)
+* doc: remove invalid option mon_pg_warn_max_per_osd (`pr#31300 <https://github.com/ceph/ceph/pull/31300>`_, zhang daolong)
+* doc/_templates/page.html: redirect to etherpad (`pr#32248 <https://github.com/ceph/ceph/pull/32248>`_, Neha Ojha)
+* doc: wrong datatype describing crush_rule (`pr#32254 <https://github.com/ceph/ceph/pull/32254>`_, Kefu Chai)
+* global: disable THP for Ceph daemons (`pr#31646 <https://github.com/ceph/ceph/pull/31646>`_, Patrick Donnelly, Mark Nelson)
+* kv: fix shutdown vs async compaction (`pr#32715 <https://github.com/ceph/ceph/pull/32715>`_, Sage Weil)
+* librbd: diff iterate with fast-diff now correctly includes parent (`pr#32469 <https://github.com/ceph/ceph/pull/32469>`_, Jason Dillaman)
+* librbd: fix rbd_open_by_id, rbd_open_by_id_read_only (`pr#32837 <https://github.com/ceph/ceph/pull/32837>`_, yangjun)
+* librbd: remove pool objects when removing a namespace (`pr#32839 <https://github.com/ceph/ceph/pull/32839>`_, Jason Dillaman)
+* librbd: skip stale child with non-existent pool for list descendants (`pr#32841 <https://github.com/ceph/ceph/pull/32841>`_, songweibin)
+* librbd: support compression allocation hints to the OSD (`pr#32842 <https://github.com/ceph/ceph/pull/32842>`_, Jason Dillaman)
+* mgr: add 'rbd' profiles to support 'rbd_support' module commands (`pr#32086 <https://github.com/ceph/ceph/pull/32086>`_, Jason Dillaman)
+* mgr/alerts: simple health alerts (`pr#30820 <https://github.com/ceph/ceph/pull/30820>`_, Sage Weil)
+* mgr: Balancer fixes (`pr#31956 <https://github.com/ceph/ceph/pull/31956>`_, Neha Ojha, Kefu Chai, David Zafman)
+* mgr/DaemonServer: fix 'osd ok-to-stop' for EC pools (`pr#32844 <https://github.com/ceph/ceph/pull/32844>`_, Sage Weil)
+* mgr/dashboard: add debug mode, and accept expected exception when SSL handshaking (`pr#31190 <https://github.com/ceph/ceph/pull/31190>`_, Kefu Chai, Ernesto Puerta, Joshua Schmid)
+* mgr/dashboard: block mirroring page results in internal server error (`pr#32133 <https://github.com/ceph/ceph/pull/32133>`_, Jason Dillaman)
+* mgr/dashboard: check embedded Grafana dashboard references (`issue#40008 <http://tracker.ceph.com/issues/40008>`_, `pr#31808 <https://github.com/ceph/ceph/pull/31808>`_, Kiefer Chang)
+* mgr/dashboard: check if user has config-opt permissions (`pr#32827 <https://github.com/ceph/ceph/pull/32827>`_, Alfonso Martínez)
+* mgr/dashboard: Cross sign button not working for some modals (`pr#32012 <https://github.com/ceph/ceph/pull/32012>`_, Ricardo Marques)
+* mgr/dashboard: Dashboard can't handle self-signed cert on Grafana API (`pr#31792 <https://github.com/ceph/ceph/pull/31792>`_, Volker Theile)
+* mgr/dashboard: disable 'Add Capability' button in rgw user edit (`pr#32930 <https://github.com/ceph/ceph/pull/32930>`_, Alfonso Martínez)
+* mgr/dashboard: fix restored RBD image naming issue (`pr#31810 <https://github.com/ceph/ceph/pull/31810>`_, Kiefer Chang)
+* mgr/dashboard: grafana charts match time picker selection (`pr#31999 <https://github.com/ceph/ceph/pull/31999>`_, Alfonso Martínez)
+* mgr/dashboard,grafana: remove shortcut menu (`pr#31980 <https://github.com/ceph/ceph/pull/31980>`_, Ernesto Puerta)
+* mgr/dashboard: Handle always-on Ceph Manager modules correctly (`pr#31782 <https://github.com/ceph/ceph/pull/31782>`_, Volker Theile)
+* mgr/dashboard: Hardening accessing the metadata (`pr#32128 <https://github.com/ceph/ceph/pull/32128>`_, Volker Theile)
+* mgr/dashboard: iSCSI targets not available if any gateway is down (and more...) (`pr#32304 <https://github.com/ceph/ceph/pull/32304>`_, Ricardo Marques)
+* mgr/dashboard: KeyError on dashboard reload (`pr#32233 <https://github.com/ceph/ceph/pull/32233>`_, Patrick Seidensal)
+* mgr/dashboard: key-value-table doesn't render booleans (`pr#31789 <https://github.com/ceph/ceph/pull/31789>`_, Patrick Seidensal)
+* mgr/dashboard: Remove compression mode unset in pool from (`pr#31784 <https://github.com/ceph/ceph/pull/31784>`_, Stephan Müller)
+* mgr/dashboard: show "Rename" in header & button when renaming RBD (`pr#31779 <https://github.com/ceph/ceph/pull/31779>`_, Alfonso Martínez)
+* mgr/dashboard: sort monitors by open sessions correctly (`pr#31791 <https://github.com/ceph/ceph/pull/31791>`_, Alfonso Martínez)
+* mgr/dashboard: Standby Dashboards don't handle all requests properly (`pr#32299 <https://github.com/ceph/ceph/pull/32299>`_, Volker Theile)
+* mgr/dashboard: Trim IQN on iSCSI target form (`pr#31942 <https://github.com/ceph/ceph/pull/31942>`_, Ricardo Marques)
+* mgr/dashboard: Unable to set boolean values to false when default is true (`pr#31941 <https://github.com/ceph/ceph/pull/31941>`_, Ricardo Marques)
+* mgr/dashboard: Using wrong identifiers in RGW user/bucket datatables (`pr#32888 <https://github.com/ceph/ceph/pull/32888>`_, Volker Theile)
+* mgr/devicehealth: ensure we don't store empty objects (`pr#31735 <https://github.com/ceph/ceph/pull/31735>`_, Sage Weil)
+* mgr/devicehealth: fix telemetry stops sending device reports after 48 hours (`pr#33346 <https://github.com/ceph/ceph/pull/33346>`_, Yaarit Hatuka, Sage Weil)
+* mgr: drop reference to msg on return (`pr#33498 <https://github.com/ceph/ceph/pull/33498>`_, Patrick Donnelly)
+* mgr/MgrClient: fix open condition (`pr#32769 <https://github.com/ceph/ceph/pull/32769>`_, Sage Weil)
+* mgr/pg_autoscaler: calculate pool_pg_target using pool size (`pr#33170 <https://github.com/ceph/ceph/pull/33170>`_, Dan van der Ster)
+* mgr/pg_autoscaler: default to pg_num[_min] = 16 (`pr#32069 <https://github.com/ceph/ceph/pull/32069>`_, Sage Weil)
+* mgr/pg_autoscaler: default to pg_num[_min] = 32 (`pr#32931 <https://github.com/ceph/ceph/pull/32931>`_, Neha Ojha)
+* mgr/pg_autoscaler: implement shutdown method (`pr#32068 <https://github.com/ceph/ceph/pull/32068>`_, Patrick Donnelly)
+* mgr/pg_autoscaler: only generate target\_\* health warnings if targets set (`pr#32067 <https://github.com/ceph/ceph/pull/32067>`_, Sage Weil)
+* mgr/prometheus: assign a value to osd_dev_node when obj_store is not filestore or bluestore (`pr#31556 <https://github.com/ceph/ceph/pull/31556>`_, jiahuizeng)
+* mgr/prometheus: report per-pool pg states (`pr#33157 <https://github.com/ceph/ceph/pull/33157>`_, Aleksei Zakharov)
+* mgr/telemetry: anonymizing smartctl report itself (`pr#33082 <https://github.com/ceph/ceph/pull/33082>`_, Yaarit Hatuka)
+* mgr/telemetry: check get_metadata return val (`pr#33095 <https://github.com/ceph/ceph/pull/33095>`_, Yaarit Hatuka)
+* mgr/telemetry: split entity_name only once (handle ids with dots) (`pr#33168 <https://github.com/ceph/ceph/pull/33168>`_, Dan Mick)
+* mgr/zabbix: Adds possibility to send data to multiple zabbix servers (`pr#30009 <https://github.com/ceph/ceph/pull/30009>`_, slivik, Jakub Sliva)
+* mon/ConfigMonitor: fix handling of NO_MON_UPDATE settings (`pr#32856 <https://github.com/ceph/ceph/pull/32856>`_, Sage Weil)
+* mon/ConfigMonitor: only propose if leader (`pr#33155 <https://github.com/ceph/ceph/pull/33155>`_, Sage Weil)
+* mon: Don't put session during feature change (`pr#33152 <https://github.com/ceph/ceph/pull/33152>`_, Brad Hubbard)
+* mon: elector: return after triggering a new election (`pr#33007 <https://github.com/ceph/ceph/pull/33007>`_, Greg Farnum)
+* monitoring: wait before firing osd full alert (`pr#32070 <https://github.com/ceph/ceph/pull/32070>`_, Patrick Seidensal)
+* mon/MgrMonitor.cc: add always_on_modules to the output of "ceph mgr module ls" (`pr#32997 <https://github.com/ceph/ceph/pull/32997>`_, Neha Ojha)
+* mon/MgrMonitor.cc: warn about missing mgr in a cluster with osds (`pr#33142 <https://github.com/ceph/ceph/pull/33142>`_, Neha Ojha)
+* mon/OSDMonitor: Don't update mon cache settings if rocksdb is not used (`pr#32520 <https://github.com/ceph/ceph/pull/32520>`_, Sridhar Seshasayee, Sage Weil)
+* mon/OSDMonitor: fix format error ceph osd stat --format json (`pr#32062 <https://github.com/ceph/ceph/pull/32062>`_, Zheng Yin)
+* mon/PGMap.h: disable network stats in dump_osd_stats (`pr#32466 <https://github.com/ceph/ceph/pull/32466>`_, Neha Ojha, David Zafman)
+* mon: remove the restriction of address type in init_with_hosts (`pr#31844 <https://github.com/ceph/ceph/pull/31844>`_, Hao Xiong)
+* mon/Session: only index osd ids >= 0 (`pr#32908 <https://github.com/ceph/ceph/pull/32908>`_, Sage Weil)
+* mount.ceph: give a hint message when no mds is up or cluster is laggy (`pr#32910 <https://github.com/ceph/ceph/pull/32910>`_, Xiubo Li)
+* mount.ceph: remove arbitrary limit on size of name= option (`pr#32807 <https://github.com/ceph/ceph/pull/32807>`_, Jeff Layton)
+* msg: async/net_handler.cc: Fix compilation (`pr#31736 <https://github.com/ceph/ceph/pull/31736>`_, Carlos Valiente)
+* osd: add osd_fast_shutdown option (default true) (`pr#32743 <https://github.com/ceph/ceph/pull/32743>`_, Sage Weil)
+* osd: Allow 64-char hostname to be added as the "host" in CRUSH (`pr#33147 <https://github.com/ceph/ceph/pull/33147>`_, Michal Skalski)
+* osd: Diagnostic logging for upmap cleaning (`pr#32716 <https://github.com/ceph/ceph/pull/32716>`_, David Zafman)
+* osd/OSD: enhance osd numa affinity compatibility (`pr#32843 <https://github.com/ceph/ceph/pull/32843>`_, luo rixin, Dai zhiwei)
+* osd/PeeringState.cc: don't let num_objects become negative (`pr#32857 <https://github.com/ceph/ceph/pull/32857>`_, Neha Ojha)
+* osd/PeeringState.cc: skip peer_purged when discovering all missing (`pr#32847 <https://github.com/ceph/ceph/pull/32847>`_, Neha Ojha)
+* osd/PeeringState: do not exclude up from acting_recovery_backfill (`pr#32064 <https://github.com/ceph/ceph/pull/32064>`_, Nathan Cutler, xie xingguo)
+* osd/PrimaryLogPG: skip obcs that don't exist during backfill scan_range (`pr#31028 <https://github.com/ceph/ceph/pull/31028>`_, Sage Weil)
+* osd: set affinity for \*all\* threads (`pr#31359 <https://github.com/ceph/ceph/pull/31359>`_, Sage Weil)
+* osd: set collection pool opts on collection create, pg load (`pr#32123 <https://github.com/ceph/ceph/pull/32123>`_, Sage Weil)
+* osd: Use physical ratio for nearfull (doesn't include backfill resserve) (`pr#32773 <https://github.com/ceph/ceph/pull/32773>`_, David Zafman)
+* pybind/mgr: Cancel output color control (`pr#31697 <https://github.com/ceph/ceph/pull/31697>`_, Zheng Yin)
+* rbd:  creating thick-provision image progress percent info exceeds 100% (`pr#32840 <https://github.com/ceph/ceph/pull/32840>`_, Xiangdong Mu)
+* rbd: librbd: don't call refresh from mirror::GetInfoRequest state machine (`pr#32900 <https://github.com/ceph/ceph/pull/32900>`_, Mykola Golub)
+* rbd-mirror: clone v2 mirroring improvements (`pr#31518 <https://github.com/ceph/ceph/pull/31518>`_, Mykola Golub)
+* rbd-mirror: fix 'rbd mirror status' asok command output (`pr#32447 <https://github.com/ceph/ceph/pull/32447>`_, Mykola Golub)
+* rbd-mirror: make logrotate work (`pr#32593 <https://github.com/ceph/ceph/pull/32593>`_, Mykola Golub)
+* rgw: add bucket permission verify when copy obj (`pr#31089 <https://github.com/ceph/ceph/pull/31089>`_, NancySu05)
+* rgw: Adding 'iam' namespace for Role and User Policy related REST APIs (`pr#32437 <https://github.com/ceph/ceph/pull/32437>`_, Pritha Srivastava)
+* rgw: adding mfa code validation when bucket versioning status is changed (`pr#32759 <https://github.com/ceph/ceph/pull/32759>`_, Pritha Srivastava)
+* rgw: add num_shards to radosgw-admin bucket stats (`pr#31182 <https://github.com/ceph/ceph/pull/31182>`_, Paul Emmerich)
+* rgw: allow reshard log entries for non-existent buckets to be cancelled (`pr#32056 <https://github.com/ceph/ceph/pull/32056>`_, J. Eric Ivancich)
+* rgw: auto-clean reshard queue entries for non-existent buckets (`pr#32055 <https://github.com/ceph/ceph/pull/32055>`_, J. Eric Ivancich)
+* rgw: build_linked_oids_for_bucket and build_buckets_instance_index should return negative value if it fails (`pr#32820 <https://github.com/ceph/ceph/pull/32820>`_, zhangshaowen)
+* rgw: crypt: permit RGW-AUTO/default with SSE-S3 headers (`pr#31862 <https://github.com/ceph/ceph/pull/31862>`_, Matt Benjamin)
+* rgw: data sync markers include timestamp from datalog entry (`pr#32819 <https://github.com/ceph/ceph/pull/32819>`_, Casey Bodley)
+* rgw_file: avoid string::front() on empty path (`pr#33008 <https://github.com/ceph/ceph/pull/33008>`_, Matt Benjamin)
+* rgw: fix a bug that bucket instance obj can't be removed after resharding completed (`pr#32822 <https://github.com/ceph/ceph/pull/32822>`_, zhang Shaowen)
+* rgw: fix an endless loop error when to show usage (`pr#31684 <https://github.com/ceph/ceph/pull/31684>`_, lvshuhua)
+* rgw: fix bugs in listobjectsv1 (`pr#32239 <https://github.com/ceph/ceph/pull/32239>`_, Albin Antony)
+* rgw: fix compile errors with boost 1.70 (`pr#31289 <https://github.com/ceph/ceph/pull/31289>`_, Casey Bodley)
+* rgw: fix data consistency error casued by rgw sent timeout (`pr#32821 <https://github.com/ceph/ceph/pull/32821>`_, �纲彬82225)
+* rgw: fix list versions starts with version_id=null (`pr#30743 <https://github.com/ceph/ceph/pull/30743>`_, Tianshan Qu)
+* rgw: fix one part of the bulk delete(RGWDeleteMultiObj_ObjStore_S3) fails but no error messages (`pr#33151 <https://github.com/ceph/ceph/pull/33151>`_, Snow Si)
+* rgw: fix opslog operation field as per Amazon s3 (`issue#20978 <http://tracker.ceph.com/issues/20978>`_, `pr#32834 <https://github.com/ceph/ceph/pull/32834>`_, Jiaying Ren)
+* rgw: fix refcount tags to match and update object's idtag (`pr#30741 <https://github.com/ceph/ceph/pull/30741>`_, J. Eric Ivancich)
+* rgw: fix rgw crash when token is not base64 encode (`pr#32050 <https://github.com/ceph/ceph/pull/32050>`_, yuliyang)
+* rgw: gc remove tag after all sub io finish (`issue#40903 <http://tracker.ceph.com/issues/40903>`_, `pr#30733 <https://github.com/ceph/ceph/pull/30733>`_, Tianshan Qu)
+* rgw: Incorrectly calling ceph::buffer::list::decode_base64 in bucket policy (`pr#32832 <https://github.com/ceph/ceph/pull/32832>`_, GaryHyg)
+* rgw: maybe coredump when reload operator happened (`pr#33149 <https://github.com/ceph/ceph/pull/33149>`_, Richard Bai(白学余))
+* rgw: move forward marker even in case of many rgw.none indexes (`pr#32824 <https://github.com/ceph/ceph/pull/32824>`_, Ilsoo Byun)
+* rgw multisite: fixes for concurrent version creation (`pr#32057 <https://github.com/ceph/ceph/pull/32057>`_, Or Friedmann, Casey Bodley)
+* rgw: prevent bucket reshard scheduling if bucket is resharding (`pr#31298 <https://github.com/ceph/ceph/pull/31298>`_, J. Eric Ivancich)
+* rgw/pubsub: fix records/event json format to match documentation (`pr#32221 <https://github.com/ceph/ceph/pull/32221>`_, Yuval Lifshitz)
+* rgw: radosgw-admin: sync status displays id of shard furthest behind (`pr#32818 <https://github.com/ceph/ceph/pull/32818>`_, Casey Bodley)
+* rgw: return error if lock log shard fails (`pr#32825 <https://github.com/ceph/ceph/pull/32825>`_, zhangshaowen)
+* rgw/rgw_rest_conn.h: fix build with clang (`pr#32489 <https://github.com/ceph/ceph/pull/32489>`_, Bernd Zeimetz)
+* rgw: Select the std::bitset to resolv ambiguity (`pr#32504 <https://github.com/ceph/ceph/pull/32504>`_, Willem Jan Withagen)
+* rgw: support radosgw-admin zone/zonegroup placement get command (`pr#32835 <https://github.com/ceph/ceph/pull/32835>`_, jiahuizeng)
+* rgw: the http response code of delete bucket should not be 204-no-content (`pr#32833 <https://github.com/ceph/ceph/pull/32833>`_, Chang Liu)
+* rgw:  update s3-test download code for s3-test tasks (`pr#32229 <https://github.com/ceph/ceph/pull/32229>`_, Ali Maredia)
+* rgw: update the hash source for multipart entries during resharding (`pr#33183 <https://github.com/ceph/ceph/pull/33183>`_, dongdong tao)
+* rgw: url encode common prefixes for List Objects response (`pr#32058 <https://github.com/ceph/ceph/pull/32058>`_, Abhishek Lekshmanan)
+* rgw: when resharding store progress json (`pr#31683 <https://github.com/ceph/ceph/pull/31683>`_, Mark Kogan, Mark Nelson)
+* selinux: Allow ceph to read udev db (`pr#32259 <https://github.com/ceph/ceph/pull/32259>`_, Boris Ranto)
+
+
+v14.2.7 Nautilus
+================
+
+This is the seventh update to the Ceph Nautilus release series. This is
+a hotfix release primarily fixing a couple of security issues. We
+recommend that all users upgrade to this release.
+
+Notable Changes
+---------------
+
+* CVE-2020-1699: Fixed a path traversal flaw in Ceph dashboard that
+  could allow for potential information disclosure (Ernesto Puerta)
+* CVE-2020-1700: Fixed a flaw in RGW beast frontend that could lead to
+  denial of service from an unauthenticated client (Or Friedmann)
+
+
+v14.2.6 Nautilus
+================
+
+This is the sixth update to the Ceph Nautilus release series. This is a hotfix
+release primarily fixing a regression introduced in v14.2.5, all nautilus users
+are advised to upgrade to this release.
+
+Notable Changes
+---------------
+
+* This release fixes a ``ceph-mgr`` bug that caused mgr becoming unresponsive on
+  larger clusters `issue#43364 <https://tracker.ceph.com/issues/43364>`_ (`pr#32466 <https://github.com/ceph/ceph/pull/32466>`_, David Zafman, Neha Ojha)
+
+
+v14.2.5 Nautilus
+================
+
+This is the fifth release of the Ceph Nautilus release series. Among the many
+notable changes, this release fixes a critical BlueStore bug that was introduced
+in 14.2.3. All Nautilus users are advised to upgrade to this release.
+
+Notable Changes
+---------------
+
+Critical fix:
+
+* This release fixes a `critical BlueStore bug <https://tracker.ceph.com/issues/42223>`_
+  introduced in 14.2.3 (and also present in 14.2.4) that can lead to data
+  corruption when a separate "WAL" device is used.
+
+New health warnings:
+
+* Ceph will now issue health warnings if daemons have recently crashed. Ceph
+  has been collecting crash reports since the initial Nautilus release, but the
+  health alerts are new. To view new crashes (or all crashes, if you've just
+  upgraded)::
+
+    ceph crash ls-new
+
+  To acknowledge a particular crash (or all crashes) and silence the health warning::
+
+    ceph crash archive <crash-id>
+    ceph crash archive-all
+
+* Ceph will issue a health warning if a RADOS pool's ``size`` is set to 1
+  or, in other words, if the pool is configured with no redundancy. Ceph will
+  stop issuing the warning if the pool size is set to the minimum
+  recommended value::
+
+    ceph osd pool set <pool-name> size <num-replicas>
+
+  The warning can be silenced with::
+
+    ceph config set global mon_warn_on_pool_no_redundancy false
+
+* A health warning is now generated if the average osd heartbeat ping
+  time exceeds a configurable threshold for any of the intervals
+  computed. The OSD computes 1 minute, 5 minute and 15 minute
+  intervals with average, minimum and maximum values.  New configuration
+  option `mon_warn_on_slow_ping_ratio` specifies a percentage of
+  `osd_heartbeat_grace` to determine the threshold.  A value of zero
+  disables the warning. New configuration option `mon_warn_on_slow_ping_time`
+  specified in milliseconds over-rides the computed value, causes a warning
+  when OSD heartbeat pings take longer than the specified amount.
+  A new admin command, `ceph daemon mgr.# dump_osd_network [threshold]`, will
+  list all connections with a ping time longer than the specified threshold or
+  value determined by the config options, for the average for any of the 3 intervals.
+  Another new admin command, `ceph daemon osd.# dump_osd_network [threshold]`,
+  will do the same but only including heartbeats initiated by the specified OSD.
+
+Changes in the telemetry module:
+
+* The telemetry module now reports more information.
+
+  First, there is a new 'device' channel, enabled by default, that
+  will report anonymized hard disk and SSD health metrics to
+  telemetry.ceph.com in order to build and improve device failure
+  prediction algorithms.  If you are not comfortable sharing device
+  metrics, you can disable that channel first before re-opting-in::
+
+    ceph config set mgr mgr/telemetry/channel_device false
+
+  Second, we now report more information about CephFS file systems,
+  including:
+
+    - how many MDS daemons (in total and per file system)
+    - which features are (or have been) enabled
+    - how many data pools
+    - approximate file system age (year + month of creation)
+    - how many files, bytes, and snapshots
+    - how much metadata is being cached
+
+  We have also added:
+
+    - which Ceph release the monitors are running
+    - whether msgr v1 or v2 addresses are used for the monitors
+    - whether IPv4 or IPv6 addresses are used for the monitors
+    - whether RADOS cache tiering is enabled (and which mode)
+    - whether pools are replicated or erasure coded, and
+      which erasure code profile plugin and parameters are in use
+    - how many hosts are in the cluster, and how many hosts have each type of daemon
+    - whether a separate OSD cluster network is being used
+    - how many RBD pools and images are in the cluster, and how many pools have RBD mirroring enabled
+    - how many RGW daemons, zones, and zonegroups are present; which RGW frontends are in use
+    - aggregate stats about the CRUSH map, like which algorithms are used, how
+      big buckets are, how many rules are defined, and what tunables are in
+      use
+
+  If you had telemetry enabled, you will need to re-opt-in with::
+
+    ceph telemetry on
+
+  You can view exactly what information will be reported first with::
+
+    ceph telemetry show        # see everything
+    ceph telemetry show basic  # basic cluster info (including all of the new info)
+
+OSD:
+
+* A new OSD daemon command, 'dump_recovery_reservations', reveals the
+  recovery locks held (in_progress) and waiting in priority queues.
+
+* Another new OSD daemon command, 'dump_scrub_reservations', reveals the
+  scrub reservations that are held for local (primary) and remote (replica) PGs.
+
+RGW:
+
+* RGW now supports S3 Object Lock set of APIs allowing for a WORM model for
+  storing objects. 6 new APIs have been added put/get bucket object lock,
+  put/get object retention, put/get object legal hold.
+
+* RGW now supports List Objects V2
+        
+Changelog
+---------
+
+* bluestore/KernelDevice: fix RW_IO_MAX constant (`pr#31397 <https://github.com/ceph/ceph/pull/31397>`_, Sage Weil)
+* bluestore: Don't forget sub kv_submitted_waiters (`pr#30048 <https://github.com/ceph/ceph/pull/30048>`_, Jianpeng Ma)
+* bluestore: apply garbage collection against excessive blob count growth (`pr#30144 <https://github.com/ceph/ceph/pull/30144>`_, Igor Fedotov)
+* bluestore: apply shared_alloc_size to shared device with log level change (`pr#30229 <https://github.com/ceph/ceph/pull/30229>`_, Vikhyat Umrao, Sage Weil, Igor Fedotov, Neha Ojha)
+* bluestore: consolidate extents from the same device only (`pr#31644 <https://github.com/ceph/ceph/pull/31644>`_, Igor Fedotov)
+* bluestore: fix improper setting of STATE_KV_SUBMITTED (`pr#30755 <https://github.com/ceph/ceph/pull/30755>`_, Igor Fedotov)
+* bluestore: shallow fsck mode and legacy statfs auto repair (`pr#30685 <https://github.com/ceph/ceph/pull/30685>`_, Sage Weil, Igor Fedotov)
+* bluestore: tool to check fragmentation (`pr#29949 <https://github.com/ceph/ceph/pull/29949>`_, Adam Kupczyk)
+* build/ops: admin/build-doc: use python3 (`pr#30664 <https://github.com/ceph/ceph/pull/30664>`_, Kefu Chai)
+* build/ops: backport endian fixes (`issue#40114 <http://tracker.ceph.com/issues/40114>`_, `pr#30697 <https://github.com/ceph/ceph/pull/30697>`_, Ulrich Weigand, Jeff Layton)
+* build/ops: cmake,rgw: IBM Z build fixes (`pr#30696 <https://github.com/ceph/ceph/pull/30696>`_, Ulrich Weigand)
+* build/ops: cmake/BuildDPDK: ignore gcc8/9 warnings (`pr#30360 <https://github.com/ceph/ceph/pull/30360>`_, Yuval Lifshitz)
+* build/ops: cmake: Allow cephfs and ceph-mds to be build when building on FreeBSD (`pr#31011 <https://github.com/ceph/ceph/pull/31011>`_, Willem Jan Withagen)
+* build/ops: cmake: enforce C++17 instead of relying on cmake-compile-features (`pr#30283 <https://github.com/ceph/ceph/pull/30283>`_, Kefu Chai)
+* build/ops: fix build fail related to PYTHON_EXECUTABLE variable (`pr#30261 <https://github.com/ceph/ceph/pull/30261>`_, Ilsoo Byun)
+* build/ops: hidden corei7 requirement in binary packages (`pr#29772 <https://github.com/ceph/ceph/pull/29772>`_, Kefu Chai)
+* build/ops: install-deps.sh: add EPEL repo for non-x86_64 archs as well (`pr#30601 <https://github.com/ceph/ceph/pull/30601>`_, Kefu Chai, Nathan Cutler)
+* build/ops: install-deps.sh: install `python\*-devel` for python\*rpm-macros (`pr#30322 <https://github.com/ceph/ceph/pull/30322>`_, Kefu Chai)
+* build/ops: install-deps: do not install if rpm already installed and ceph.spec.in: s/pkgversion/version_nodots/ (`pr#30708 <https://github.com/ceph/ceph/pull/30708>`_, Jeff Layton, Kefu Chai)
+* build/ops: make patch build dependency explicit (`issue#40175 <http://tracker.ceph.com/issues/40175>`_, `pr#30046 <https://github.com/ceph/ceph/pull/30046>`_, Nathan Cutler)
+* build/ops: python3-cephfs should provide python36-cephfs (`pr#30983 <https://github.com/ceph/ceph/pull/30983>`_, Kefu Chai)
+* build/ops: rpm: always build ceph-test package (`pr#30049 <https://github.com/ceph/ceph/pull/30049>`_, Nathan Cutler)
+* build/ops: rpm: fdupes in SUSE builds to conform with packaging guidelines (`issue#40973 <http://tracker.ceph.com/issues/40973>`_, `pr#29784 <https://github.com/ceph/ceph/pull/29784>`_, Nathan Cutler)
+* build/ops: rpm: make librados2, libcephfs2 own (create) /etc/ceph (`pr#31125 <https://github.com/ceph/ceph/pull/31125>`_, Nathan Cutler)
+* build/ops: rpm: put librgw lttng SOs in the librgw-devel package (`issue#40975 <http://tracker.ceph.com/issues/40975>`_, `pr#29785 <https://github.com/ceph/ceph/pull/29785>`_, Nathan Cutler)
+* build/ops: seastar,dmclock: use CXX_FLAGS from parent project (`pr#30114 <https://github.com/ceph/ceph/pull/30114>`_, Kefu Chai)
+* build/ops: use gcc-8 (`issue#38892 <http://tracker.ceph.com/issues/38892>`_, `pr#30089 <https://github.com/ceph/ceph/pull/30089>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: update-mon-db: do not fail if incmap is missing (`pr#30740 <https://github.com/ceph/ceph/pull/30740>`_, Kefu Chai)
+* ceph-volume: PVolumes.filter shouldn't purge itself (`pr#30805 <https://github.com/ceph/ceph/pull/30805>`_, Rishabh Dave)
+* ceph-volume: VolumeGroups.filter shouldn't purge itself (`pr#30807 <https://github.com/ceph/ceph/pull/30807>`_, Rishabh Dave)
+* ceph-volume: add Ceph's device id to inventory (`pr#31210 <https://github.com/ceph/ceph/pull/31210>`_, Sebastian Wagner)
+* ceph-volume: allow to skip restorecon calls (`pr#31555 <https://github.com/ceph/ceph/pull/31555>`_, Alfredo Deza)
+* ceph-volume: api/lvm: check if list of LVs is empty (`pr#31228 <https://github.com/ceph/ceph/pull/31228>`_, Rishabh Dave)
+* ceph-volume: check if we run in an selinux environment (`pr#31812 <https://github.com/ceph/ceph/pull/31812>`_, Jan Fajerski)
+* ceph-volume: do not fail when trying to remove crypt mapper (`pr#30554 <https://github.com/ceph/ceph/pull/30554>`_, Guillaume Abrioux)
+* ceph-volume: fix stderr failure to decode/encode when redirected (`pr#30300 <https://github.com/ceph/ceph/pull/30300>`_, Alfredo Deza)
+* ceph-volume: fix warnings raised by pytest (`pr#30676 <https://github.com/ceph/ceph/pull/30676>`_, Rishabh Dave)
+* ceph-volume: lvm list is O(n^2) (`pr#30093 <https://github.com/ceph/ceph/pull/30093>`_, Rishabh Dave)
+* ceph-volume: lvm.zap fix cleanup for db partitions (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30304 <https://github.com/ceph/ceph/pull/30304>`_, Dominik Csapak)
+* ceph-volume: mokeypatch calls to lvm related binaries (`pr#31405 <https://github.com/ceph/ceph/pull/31405>`_, Jan Fajerski)
+* ceph-volume: pre-install python-apt and its variants before test runs (`pr#30294 <https://github.com/ceph/ceph/pull/30294>`_, Alfredo Deza)
+* ceph-volume: rearrange api/lvm.py (`pr#31408 <https://github.com/ceph/ceph/pull/31408>`_, Rishabh Dave)
+* ceph-volume: systemd fix typo in log message (`pr#30520 <https://github.com/ceph/ceph/pull/30520>`_, Manu Zurmühl)
+* ceph-volume: use the OSD identifier when reporting success (`pr#29769 <https://github.com/ceph/ceph/pull/29769>`_, Alfredo Deza)
+* ceph-volume: zap always skips block.db, leaves them around (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#30307 <https://github.com/ceph/ceph/pull/30307>`_, Alfredo Deza)
+* tools: ceph.in: do not preload ASan unless necessary (`pr#31676 <https://github.com/ceph/ceph/pull/31676>`_, Kefu Chai)
+* build/ops: ceph.spec.in: reserve 2500MB per build job (`pr#30370 <https://github.com/ceph/ceph/pull/30370>`_, Dan van der Ster)
+* tools: ceph_volume_client: convert string to bytes object (`issue#39405 <http://tracker.ceph.com/issues/39405>`_, `issue#40369 <http://tracker.ceph.com/issues/40369>`_, `issue#39510 <http://tracker.ceph.com/issues/39510>`_, `issue#40800 <http://tracker.ceph.com/issues/40800>`_, `issue#40460 <http://tracker.ceph.com/issues/40460>`_, `pr#30030 <https://github.com/ceph/ceph/pull/30030>`_, Rishabh Dave)
+* cephfs-shell: Convert paths type from string to bytes (`pr#30057 <https://github.com/ceph/ceph/pull/30057>`_, Varsha Rao)
+* cephfs: Allow mount.ceph to get mount info from ceph configs and keyrings (`pr#30521 <https://github.com/ceph/ceph/pull/30521>`_, Jeff Layton)
+* cephfs: avoid map been inserted by mistake (`pr#29878 <https://github.com/ceph/ceph/pull/29878>`_, XiaoGuoDong2019)
+* cephfs: client: more precise CEPH_CLIENT_CAPS_PENDING_CAPSNAP (`pr#30032 <https://github.com/ceph/ceph/pull/30032>`_, "Yan, Zheng")
+* cephfs: client: nfs-ganesha with cephfs client, removing dir reports not empty (`issue#40746 <http://tracker.ceph.com/issues/40746>`_, `pr#30442 <https://github.com/ceph/ceph/pull/30442>`_, Peng Xie)
+* cephfs: client: return -eio when sync file which unsafe reqs have been dropped (`issue#40877 <http://tracker.ceph.com/issues/40877>`_, `pr#30043 <https://github.com/ceph/ceph/pull/30043>`_, simon gao)
+* cephfs: fix a memory leak (`pr#29879 <https://github.com/ceph/ceph/pull/29879>`_, XiaoGuoDong2019)
+* cephfs: mds: Fix duplicate client entries in eviction list (`pr#30951 <https://github.com/ceph/ceph/pull/30951>`_, Sidharth Anupkrishnan)
+* cephfs: mds: cleanup truncating inodes when standby replay mds trim log segments (`pr#29591 <https://github.com/ceph/ceph/pull/29591>`_, "Yan, Zheng")
+* cephfs: mds: delay exporting directory whose pin value exceeds max rank id (`issue#40603 <http://tracker.ceph.com/issues/40603>`_, `pr#29938 <https://github.com/ceph/ceph/pull/29938>`_, Zhi Zhang)
+* cephfs: mds: evict an unresponsive client only when another client wants its caps (`pr#30031 <https://github.com/ceph/ceph/pull/30031>`_, Rishabh Dave)
+* cephfs: mds: fix InoTable::force_consume_to() (`pr#30041 <https://github.com/ceph/ceph/pull/30041>`_, "Yan, Zheng")
+* cephfs: mds: fix infinite loop in Locker::file_update_finish (`pr#31079 <https://github.com/ceph/ceph/pull/31079>`_, "Yan, Zheng")
+* cephfs: mds: make MDSIOContextBase delete itself when shutting down (`pr#30418 <https://github.com/ceph/ceph/pull/30418>`_, Xuehan Xu)
+* cephfs: mds: trim cache on regular schedule (`pr#30040 <https://github.com/ceph/ceph/pull/30040>`_, Patrick Donnelly)
+* cephfs: mds: wake up lock waiters after forcibly changing lock state (`issue#39987 <http://tracker.ceph.com/issues/39987>`_, `pr#30508 <https://github.com/ceph/ceph/pull/30508>`_, "Yan, Zheng")
+* cephfs: mount.ceph: properly handle -o strictatime (`pr#30039 <https://github.com/ceph/ceph/pull/30039>`_, Jeff Layton)
+* cephfs: qa: ignore expected MDS_CLIENT_LATE_RELEASE warning (`issue#40968 <http://tracker.ceph.com/issues/40968>`_, `pr#29811 <https://github.com/ceph/ceph/pull/29811>`_, Patrick Donnelly)
+* cephfs: qa: wait for MDS to come back after removing it (`issue#40967 <http://tracker.ceph.com/issues/40967>`_, `pr#29832 <https://github.com/ceph/ceph/pull/29832>`_, Patrick Donnelly)
+* cephfs: tests: power off still resulted in client sending session close (`issue#37681 <http://tracker.ceph.com/issues/37681>`_, `pr#29983 <https://github.com/ceph/ceph/pull/29983>`_, Patrick Donnelly)
+* common/ceph_context: avoid unnecessary wait during service thread shutdown (`pr#31097 <https://github.com/ceph/ceph/pull/31097>`_, Jason Dillaman)
+* common/config_proxy: hold lock while accessing mutable container (`pr#30661 <https://github.com/ceph/ceph/pull/30661>`_, Jason Dillaman)
+* common: fix typo in rgw_user_max_buckets option long description (`pr#31605 <https://github.com/ceph/ceph/pull/31605>`_, Alfonso Martínez)
+* core/osd: do not trust partially simplified pg_upmap_item (`issue#42052 <http://tracker.ceph.com/issues/42052>`_, `pr#30899 <https://github.com/ceph/ceph/pull/30899>`_, xie xingguo)
+* core: Health warnings on long network ping times (`issue#40640 <http://tracker.ceph.com/issues/40640>`_, `pr#30195 <https://github.com/ceph/ceph/pull/30195>`_, David Zafman)
+* core: If the nodeep-scrub/noscrub flags are set in pools instead of global cluster. List the pool names in the ceph status (`issue#38029 <http://tracker.ceph.com/issues/38029>`_, `pr#29991 <https://github.com/ceph/ceph/pull/29991>`_, Mohamad Gebai)
+* core: Improve health status for backfill_toofull and recovery_toofull and fix backfill_toofull seen on cluster where the most full OSD is at 1% (`pr#29999 <https://github.com/ceph/ceph/pull/29999>`_, David Zafman)
+* core: Make dumping of reservation info congruent between scrub and recovery (`pr#31444 <https://github.com/ceph/ceph/pull/31444>`_, David Zafman)
+* core: Revert "rocksdb: enable rocksdb_rmrange=true by default" (`pr#31612 <https://github.com/ceph/ceph/pull/31612>`_, Neha Ojha)
+* core: filestore pre-split may not split enough directories (`issue#39390 <http://tracker.ceph.com/issues/39390>`_, `pr#29988 <https://github.com/ceph/ceph/pull/29988>`_, Jeegn Chen)
+* core: kv/RocksDBStore: tell rocksdb to set mode to 0600, not 0644 (`pr#31031 <https://github.com/ceph/ceph/pull/31031>`_, Sage Weil)
+* core: mon/MonClient: ENXIO when sending command to down mon (`pr#31037 <https://github.com/ceph/ceph/pull/31037>`_, Sage Weil, Greg Farnum)
+* core: mon/MonCommands: "smart" only needs read permission (`pr#31111 <https://github.com/ceph/ceph/pull/31111>`_, Kefu Chai)
+* core: mon/MonMap: encode (more) valid compat monmap when we have v2-only addrs (`pr#31658 <https://github.com/ceph/ceph/pull/31658>`_, Sage Weil)
+* core: mon/Monitor.cc: fix condition that checks for unrecognized auth mode (`pr#31038 <https://github.com/ceph/ceph/pull/31038>`_, Neha Ojha)
+* core: mon/OSDMonitor: Use generic priority cache tuner for mon caches (`pr#30419 <https://github.com/ceph/ceph/pull/30419>`_, Sridhar Seshasayee, Kefu Chai, Mykola Golub, Mark Nelson)
+* core: mon/OSDMonitor: add check for crush rule size in pool set size command (`pr#30941 <https://github.com/ceph/ceph/pull/30941>`_, Vikhyat Umrao)
+* core: mon/OSDMonitor: trim not-longer-exist failure reporters (`pr#30904 <https://github.com/ceph/ceph/pull/30904>`_, NancySu05)
+* core: mon/PGMap: fix incorrect pg_pool_sum when delete pool (`pr#31704 <https://github.com/ceph/ceph/pull/31704>`_, luo rixin)
+* core: mon: C_AckMarkedDown has not handled the Callback Arguments (`pr#29997 <https://github.com/ceph/ceph/pull/29997>`_, NancySu05)
+* core: mon: ensure prepare_failure() marks no_reply on op (`pr#30480 <https://github.com/ceph/ceph/pull/30480>`_, Joao Eduardo Luis)
+* core: mon: show pool id in pool ls command (`issue#40287 <http://tracker.ceph.com/issues/40287>`_, `pr#30486 <https://github.com/ceph/ceph/pull/30486>`_, Chang Liu)
+* core: msg,mon/MonClient: fix auth for clients without CEPHX_V2 feature (`pr#30524 <https://github.com/ceph/ceph/pull/30524>`_, Sage Weil)
+* core: msg/auth: handle decode errors instead of throwing exceptions (`pr#31099 <https://github.com/ceph/ceph/pull/31099>`_, Sage Weil)
+* core: msg/simple: reset in_seq_acked to zero when session is reset (`pr#29592 <https://github.com/ceph/ceph/pull/29592>`_, Xiangyang Yu)
+* core: os/bluestore: fix objectstore_blackhole read-after-write (`pr#31019 <https://github.com/ceph/ceph/pull/31019>`_, Sage Weil)
+* core: osd/OSDCap: Check for empty namespace (`issue#40835 <http://tracker.ceph.com/issues/40835>`_, `pr#29998 <https://github.com/ceph/ceph/pull/29998>`_, Brad Hubbard)
+* core: mon/OSDMonitor: make memory autotune disable itself if no rocksdb (`pr#32045 <https://github.com/ceph/ceph/pull/32045>`_, Sage Weil)
+* core: osd/PG: Add PG to large omap log message (`pr#30923 <https://github.com/ceph/ceph/pull/30923>`_, Brad Hubbard)
+* core: osd/PGLog: persist num_objects_missing for replicas when peering is done (`pr#31077 <https://github.com/ceph/ceph/pull/31077>`_, xie xingguo)
+* core: osd/PeeringState: do not complain about past_intervals constrained by oldest epoch (`pr#30000 <https://github.com/ceph/ceph/pull/30000>`_, Sage Weil)
+* core: osd/PeeringState: fix wrong history of merge target (`pr#30280 <https://github.com/ceph/ceph/pull/30280>`_, xie xingguo)
+* core: osd/PeeringState: recover_got - add special handler for empty log and improvements to standalone tests (`pr#30528 <https://github.com/ceph/ceph/pull/30528>`_, Sage Weil, David Zafman, xie xingguo)
+* core: osd/PrimaryLogPG: Avoid accessing destroyed references in finish_degr… (`pr#29994 <https://github.com/ceph/ceph/pull/29994>`_, Tao Ning)
+* core: osd/PrimaryLogPG: update oi.size on write op implicitly truncating ob… (`pr#30278 <https://github.com/ceph/ceph/pull/30278>`_, xie xingguo)
+* core: osd/ReplicatedBackend: check against empty data_included before enabling crc (`pr#29716 <https://github.com/ceph/ceph/pull/29716>`_, xie xingguo)
+* core: osd/osd_types: fix {omap,hitset_bytes}_stats_invalid handling on spli… (`pr#30643 <https://github.com/ceph/ceph/pull/30643>`_, Sage Weil)
+* core: osd: Better error message when OSD count is less than osd_pool_default_size (`issue#38617 <http://tracker.ceph.com/issues/38617>`_, `pr#29992 <https://github.com/ceph/ceph/pull/29992>`_, Kefu Chai, Sage Weil, zjh)
+* core: osd: Remove unused osdmap flags full, nearfull from output (`pr#30900 <https://github.com/ceph/ceph/pull/30900>`_, David Zafman)
+* core: osd: add log information to record the cause of do_osd_ops failure (`pr#30546 <https://github.com/ceph/ceph/pull/30546>`_, NancySu05)
+* core: osd: clear PG_STATE_CLEAN when repair object (`pr#30050 <https://github.com/ceph/ceph/pull/30050>`_, Zengran Zhang)
+* core: osd: fix possible crash on sending dynamic perf stats report (`pr#30648 <https://github.com/ceph/ceph/pull/30648>`_, Mykola Golub)
+* core: osd: merge replica log on primary need according to replica log's crt (`pr#30051 <https://github.com/ceph/ceph/pull/30051>`_, Zengran Zhang)
+* core: osd: prime splits/merges for any potential fabricated split/merge par… (`issue#38483 <http://tracker.ceph.com/issues/38483>`_, `pr#30371 <https://github.com/ceph/ceph/pull/30371>`_, xie xingguo)
+* core: osd: release backoffs during merge (`pr#31822 <https://github.com/ceph/ceph/pull/31822>`_, Sage Weil)
+* core: osd: rollforward may need to mark pglog dirty (`issue#40403 <http://tracker.ceph.com/issues/40403>`_, `pr#31034 <https://github.com/ceph/ceph/pull/31034>`_, Zengran Zhang)
+* core: osd: scrub error on big objects; make bluestore refuse to start on big objects (`pr#30783 <https://github.com/ceph/ceph/pull/30783>`_, David Zafman, Sage Weil)
+* core: osd: support osd_repair_during_recovery (`issue#40620 <http://tracker.ceph.com/issues/40620>`_, `pr#29748 <https://github.com/ceph/ceph/pull/29748>`_, Jeegn Chen)
+* core: pool_stat.dump() - value of num_store_stats is wrong (`issue#39340 <http://tracker.ceph.com/issues/39340>`_, `pr#29946 <https://github.com/ceph/ceph/pull/29946>`_, xie xingguo)
+* doc/ceph-kvstore-tool: add description for 'stats' command (`pr#30245 <https://github.com/ceph/ceph/pull/30245>`_, Josh Durgin, Adam Kupczyk)
+* doc/mgr/telemetry: update default interval (`pr#31009 <https://github.com/ceph/ceph/pull/31009>`_, Tim Serong)
+* doc/rbd: s/guess/xml/ for codeblock lexer (`pr#31074 <https://github.com/ceph/ceph/pull/31074>`_, Kefu Chai)
+* doc: Fix rbd namespace documentation (`pr#29731 <https://github.com/ceph/ceph/pull/29731>`_, Ricardo Marques)
+* doc: cephfs: add section on fsync error reporting to posix.rst (`issue#24641 <http://tracker.ceph.com/issues/24641>`_, `pr#30025 <https://github.com/ceph/ceph/pull/30025>`_, Jeff Layton)
+* doc: default values for mon_health_to_clog\_\* were flipped (`pr#30003 <https://github.com/ceph/ceph/pull/30003>`_, James McClune)
+* doc: fix urls in posix.rst (`pr#30686 <https://github.com/ceph/ceph/pull/30686>`_, Jos Collin)
+* doc: max_misplaced option was renamed in Nautilus (`pr#30649 <https://github.com/ceph/ceph/pull/30649>`_, Nathan Fish)
+* doc: pg_num should always be a power of two (`pr#30004 <https://github.com/ceph/ceph/pull/30004>`_, Lars Marowsky-Bree, Kai Wagner)
+* doc: update bluestore cache settings and clarify data fraction (`issue#39522 <http://tracker.ceph.com/issues/39522>`_, `pr#31259 <https://github.com/ceph/ceph/pull/31259>`_, Jan Fajerski)
+* mgr/ActivePyModules: behave if a module queries a devid that does not exist (`pr#31411 <https://github.com/ceph/ceph/pull/31411>`_, Sage Weil)
+* mgr/BaseMgrStandbyModule: drop GIL in ceph_get_module_option() (`pr#30773 <https://github.com/ceph/ceph/pull/30773>`_, Kefu Chai)
+* mgr/balancer: python3 compatibility issue (`pr#31012 <https://github.com/ceph/ceph/pull/31012>`_, Mykola Golub)
+* mgr/crash: backport archive feature, health alerts (`pr#30851 <https://github.com/ceph/ceph/pull/30851>`_, Sage Weil)
+* mgr/crash: try client.crash[.host] before client.admin; add mon profile (`issue#40781 <http://tracker.ceph.com/issues/40781>`_, `pr#30844 <https://github.com/ceph/ceph/pull/30844>`_, Sage Weil, Dan Mick)
+* mgr/dashboard: Add transifex-i18ntool (`pr#31160 <https://github.com/ceph/ceph/pull/31160>`_, Sebastian Krah)
+* mgr/dashboard: Allow disabling redirection on standby dashboards (`issue#41813 <https://tracker.ceph.com/issues/41813>`_, `pr#30382 <https://github.com/ceph/ceph/pull/30382>`_, Volker Theile)
+* mgr/dashboard: Configuring an URL prefix does not work as expected (`pr#31375 <https://github.com/ceph/ceph/pull/31375>`_, Volker Theile)
+* mgr/dashbaord: Fix calculation of PG status percentage (`issue#41809 <https://tracker.ceph.com/issues/41089>`_, `pr#30394 <https://github.com/ceph/ceph/pull/30394>`_, Tiago Melo)
+* mgr/dashboard: Fix CephFS chart (`pr#30691 <https://github.com/ceph/ceph/pull/30691>`_, Stephan Müller)
+* mgr/dashboard: Fix grafana dashboards (`pr#31733 <https://github.com/ceph/ceph/pull/31733>`_, Radu Toader)
+* mgr/dashboard: Improve position of MDS chart tooltip (`pr#31565 <https://github.com/ceph/ceph/pull/31565>`_, Tiago Melo)
+* mgr/dashboard: Provide the name of the object being deleted (`pr#31263 <https://github.com/ceph/ceph/pull/31263>`_, Ricardo Marques)
+* mgr/dashboard: RBD tests must use pools with power-of-two pg_num (`pr#31522 <https://github.com/ceph/ceph/pull/31522>`_, Ricardo Marques)
+* mgr/dashboard: Set RO as the default access_type for RGW NFS exports (`pr#30516 <https://github.com/ceph/ceph/pull/30516>`_, Tiago Melo)
+* mgr/dashboard: Wait for breadcrumb text is present in e2e tests (`pr#31576 <https://github.com/ceph/ceph/pull/31576>`_, Volker Theile)
+* mgr/dashboard: access_control: add grafana scope read access to \*-manager roles (`pr#30259 <https://github.com/ceph/ceph/pull/30259>`_, Ricardo Dias)
+* mgr/dashboard: do not log tokens (`pr#31413 <https://github.com/ceph/ceph/pull/31413>`_, Kefu Chai)
+* mgr/dashboard: do not show non-pool data in pool details (`pr#31516 <https://github.com/ceph/ceph/pull/31516>`_, Alfonso Martínez)
+* mgr/dashboard: edit/clone/copy rbd image after its data is received (`pr#31349 <https://github.com/ceph/ceph/pull/31349>`_, Alfonso Martínez)
+* mgr/dashboard: internationalization support with AOT enabled (`pr#30910 <https://github.com/ceph/ceph/pull/30910>`_, Ricardo Dias, Tiago Melo)
+* mgr/dashboard: run-backend-api-tests.sh improvements (`pr#29487 <https://github.com/ceph/ceph/pull/29487>`_, Alfonso Martínez, Kefu Chai)
+* mgr/dashboard: tasks: only unblock controller thread after TaskManager thread (`pr#31526 <https://github.com/ceph/ceph/pull/31526>`_, Ricardo Dias)
+* mgr/devicehealth: do not scrape mon devices (`pr#31446 <https://github.com/ceph/ceph/pull/31446>`_, Sage Weil)
+* mgr/devicehealth: import _strptime directly (`pr#32082 <https://github.com/ceph/ceph/pull/32082>`_, Sage Weil)
+* mgr/k8sevents: Initial ceph -> k8s events integration (`pr#30215 <https://github.com/ceph/ceph/pull/30215>`_, Paul Cuzner, Sebastian Wagner)
+* mgr/pg_autoscaler: fix pool_logical_used (`pr#31100 <https://github.com/ceph/ceph/pull/31100>`_, Ansgar Jazdzewski)
+* mgr/pg_autoscaler: fix race with pool deletion (`pr#30008 <https://github.com/ceph/ceph/pull/30008>`_, Sage Weil)
+* mgr/prometheus: Cast collect_timeout (scrape_interval) to float (`pr#30007 <https://github.com/ceph/ceph/pull/30007>`_, Ben Meekhof)
+* mgr/prometheus: Fix KeyError in get_mgr_status (`pr#30774 <https://github.com/ceph/ceph/pull/30774>`_, Sebastian Wagner)
+* mgr/rbd_support: module.py:1088: error: Name 'image_spec' is not defined (`pr#29978 <https://github.com/ceph/ceph/pull/29978>`_, Jason Dillaman)
+* mgr/restful: requests api adds support multiple commands (`pr#31334 <https://github.com/ceph/ceph/pull/31334>`_, Duncan Chiang)
+* mgr/telemetry: backport a ton of stuff (`pr#30849 <https://github.com/ceph/ceph/pull/30849>`_, alfonsomthd, Kefu Chai, Sage Weil, Dan Mick)
+* mgr/volumes: fix incorrect snapshot path creation (`pr#31076 <https://github.com/ceph/ceph/pull/31076>`_, Ramana Raja)
+* mgr/volumes: handle exceptions in purge thread with retry (`issue#41218 <http://tracker.ceph.com/issues/41218>`_, `pr#30455 <https://github.com/ceph/ceph/pull/30455>`_, Venky Shankar)
+* mgr/volumes: list FS subvolumes, subvolume groups, and their snapshots (`pr#30827 <https://github.com/ceph/ceph/pull/30827>`_, Jos Collin)
+* mgr/volumes: minor fixes (`pr#29926 <https://github.com/ceph/ceph/pull/29926>`_, Venky Shankar, Jos Collin, Ramana Raja)
+* mgr/volumes: protection for "fs volume rm" command (`pr#30768 <https://github.com/ceph/ceph/pull/30768>`_, Jos Collin, Ramana Raja)
+* mgr/zabbix: Fix typo in key name for PGs in backfill_wait state (`issue#39666 <http://tracker.ceph.com/issues/39666>`_, `pr#30006 <https://github.com/ceph/ceph/pull/30006>`_, Wido den Hollander)
+* mgr/zabbix: encode string for Python 3 compatibility (`pr#30016 <https://github.com/ceph/ceph/pull/30016>`_, Nathan Cutler)
+* mgr/{dashboard,prometheus}: return FQDN instead of '0.0.0.0' (`pr#31482 <https://github.com/ceph/ceph/pull/31482>`_, Patrick Seidensal)
+* mgr: Release GIL before calling OSDMap::calc_pg_upmaps() (`pr#31682 <https://github.com/ceph/ceph/pull/31682>`_, David Zafman, Shyukri Shyukriev)
+* mgr: Unable to reset / unset module options (`issue#40779 <http://tracker.ceph.com/issues/40779>`_, `pr#29550 <https://github.com/ceph/ceph/pull/29550>`_, Sebastian Wagner)
+* mgr: do not reset reported if a new metric is not collected (`pr#30390 <https://github.com/ceph/ceph/pull/30390>`_, Ilsoo Byun)
+* mgr: fix weird health-alert daemon key (`pr#31039 <https://github.com/ceph/ceph/pull/31039>`_, xie xingguo)
+* mgr: set hostname in DeviceState::set_metadata() (`pr#30624 <https://github.com/ceph/ceph/pull/30624>`_, Kefu Chai)
+* pybind/cephfs: Modification to error message (`pr#30026 <https://github.com/ceph/ceph/pull/30026>`_, Varsha Rao)
+* pybind/rados: fix set_omap() crash on py3 (`pr#30622 <https://github.com/ceph/ceph/pull/30622>`_, Sage Weil)
+* pybind/rbd: deprecate `parent_info` (`pr#30818 <https://github.com/ceph/ceph/pull/30818>`_, Ricardo Marques)
+* rbd: rbd-mirror: cannot restore deferred deletion mirrored images (`pr#30825 <https://github.com/ceph/ceph/pull/30825>`_, Jason Dillaman, Mykola Golub)
+* rbd: rbd-mirror: don't overwrite status error returned by replay (`pr#29870 <https://github.com/ceph/ceph/pull/29870>`_, Mykola Golub)
+* rbd: rbd-mirror: ignore errors relating to parsing the cluster config file (`pr#30116 <https://github.com/ceph/ceph/pull/30116>`_, Jason Dillaman)
+* rbd: rbd-mirror: simplify peer bootstrapping (`pr#30821 <https://github.com/ceph/ceph/pull/30821>`_, Jason Dillaman)
+* rbd: rbd-nbd: add netlink support and nl resize (`pr#30532 <https://github.com/ceph/ceph/pull/30532>`_, Mike Christie)
+* rbd: cls/rbd: sanitize entity instance messenger version type (`pr#30822 <https://github.com/ceph/ceph/pull/30822>`_, Jason Dillaman)
+* rbd: cls/rbd: sanitize the mirror image status peer address after reading from disk (`pr#31833 <https://github.com/ceph/ceph/pull/31833>`_, Jason Dillaman)
+* rbd: krbd: avoid udev netlink socket overrun and retry on transient errors from udev_enumerate_scan_devices() (`pr#31075 <https://github.com/ceph/ceph/pull/31075>`_, Ilya Dryomov, Adam C. Emerson)
+* rbd: librbd: always try to acquire exclusive lock when removing image (`pr#29869 <https://github.com/ceph/ceph/pull/29869>`_, Mykola Golub)
+* rbd: librbd: behave more gracefully when data pool removed (`pr#30824 <https://github.com/ceph/ceph/pull/30824>`_, Mykola Golub)
+* rbd: librbd: v1 clones are restricted to the same namespace (`pr#30823 <https://github.com/ceph/ceph/pull/30823>`_, Jason Dillaman)
+* mgr/restful: Query nodes_by_id for items (`pr#31261 <https://github.com/ceph/ceph/pull/31261>`_, Boris Ranto)
+* rgw/amqp: fix race condition in AMQP unit test (`pr#30889 <https://github.com/ceph/ceph/pull/30889>`_, Yuval Lifshitz)
+* rgw/amqp: remove flaky amqp test (`pr#31628 <https://github.com/ceph/ceph/pull/31628>`_, Yuval Lifshitz)
+* rgw/pubsub: backport notifications and pubsub (`pr#30579 <https://github.com/ceph/ceph/pull/30579>`_, Yuval Lifshitz)
+* rgw/rgw_op: Remove get_val from hotpath via legacy options (`pr#30160 <https://github.com/ceph/ceph/pull/30160>`_, Mark Nelson)
+* rgw: Potential crash in putbj (`pr#29898 <https://github.com/ceph/ceph/pull/29898>`_, Adam C. Emerson)
+* rgw: Put User Policy is sensitive to whitespace (`pr#29970 <https://github.com/ceph/ceph/pull/29970>`_, Abhishek Lekshmanan)
+* rgw: RGWCoroutine::call(nullptr) sets retcode=0 (`pr#30248 <https://github.com/ceph/ceph/pull/30248>`_, Casey Bodley)
+* rgw: Swift metadata dropped after S3 bucket versioning enabled (`pr#29961 <https://github.com/ceph/ceph/pull/29961>`_, Marcus Watts)
+* rgw: add S3 object lock feature to support object worm (`pr#29905 <https://github.com/ceph/ceph/pull/29905>`_, Chang Liu, Casey Bodley, zhang Shaowen)
+* rgw: add minssing admin property when sync user info (`pr#30680 <https://github.com/ceph/ceph/pull/30680>`_, zhang Shaowen)
+* rgw: beast frontend throws an exception when running out of FDs (`pr#29963 <https://github.com/ceph/ceph/pull/29963>`_, Yuval Lifshitz)
+* rgw: data/bilogs are trimmed when no peers are reading them (`issue#39487 <http://tracker.ceph.com/issues/39487>`_, `pr#30999 <https://github.com/ceph/ceph/pull/30999>`_, Casey Bodley)
+* rgw: datalog/mdlog trim commands loop until done (`pr#30869 <https://github.com/ceph/ceph/pull/30869>`_, Casey Bodley)
+* rgw: dns name is not case sensitive (`issue#40995 <http://tracker.ceph.com/issues/40995>`_, `pr#29971 <https://github.com/ceph/ceph/pull/29971>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: fix a bug that lifecycle expiraton generates delete marker continuously (`issue#40393 <http://tracker.ceph.com/issues/40393>`_, `pr#30037 <https://github.com/ceph/ceph/pull/30037>`_, zhang Shaowen)
+* rgw: fix cls_bucket_list_unordered() partial results (`pr#30252 <https://github.com/ceph/ceph/pull/30252>`_, Mark Kogan)
+* rgw: fix data sync start delay if remote haven't init data_log (`pr#30509 <https://github.com/ceph/ceph/pull/30509>`_, Tianshan Qu)
+* rgw: fix default storage class for get_compression_type (`pr#31026 <https://github.com/ceph/ceph/pull/31026>`_, Casey Bodley)
+* rgw: fix drain handles error when deleting bucket with bypass-gc option (`pr#29956 <https://github.com/ceph/ceph/pull/29956>`_, dongdong tao)
+* rgw: fix list bucket with delimiter wrongly skip some special keys (`issue#40905 <http://tracker.ceph.com/issues/40905>`_, `pr#30068 <https://github.com/ceph/ceph/pull/30068>`_, Tianshan Qu)
+* rgw: fix memory growth while deleteing objects with (`pr#30472 <https://github.com/ceph/ceph/pull/30472>`_, Mark Kogan)
+* rgw: fix the bug of rgw not doing necessary checking to website configuration (`issue#40678 <http://tracker.ceph.com/issues/40678>`_, `pr#30325 <https://github.com/ceph/ceph/pull/30325>`_, Enming Zhang)
+* rgw: fixed "unrecognized arg" error when using "radosgw-admin zone rm" (`pr#30247 <https://github.com/ceph/ceph/pull/30247>`_, Hongang Chen)
+* rgw: housekeeping reset stats (`pr#29803 <https://github.com/ceph/ceph/pull/29803>`_, J. Eric Ivancich)
+* rgw: increase beast parse buffer size to 64k (`pr#30437 <https://github.com/ceph/ceph/pull/30437>`_, Casey Bodley)
+* rgw: ldap auth: S3 auth failure should return InvalidAccessKeyId (`pr#30651 <https://github.com/ceph/ceph/pull/30651>`_, Matt Benjamin)
+* rgw: lifecycle days may be 0 (`pr#31073 <https://github.com/ceph/ceph/pull/31073>`_, Matt Benjamin)
+* rgw: lifecycle transitions on non existent placement targets (`pr#29955 <https://github.com/ceph/ceph/pull/29955>`_, Abhishek Lekshmanan)
+* rgw: list objects version 2 (`pr#29849 <https://github.com/ceph/ceph/pull/29849>`_, Albin Antony, zhang Shaowen)
+* rgw: multisite: radosgw-admin bucket sync status incorrectly reports "caught up" during full sync (`issue#40806 <http://tracker.ceph.com/issues/40806>`_, `pr#29974 <https://github.com/ceph/ceph/pull/29974>`_, Casey Bodley)
+* rgw: potential realm watch lost (`issue#40991 <http://tracker.ceph.com/issues/40991>`_, `pr#29972 <https://github.com/ceph/ceph/pull/29972>`_, Tianshan Qu)
+* rgw: protect AioResultList by a lock to avoid race condition (`pr#30746 <https://github.com/ceph/ceph/pull/30746>`_, Ilsoo Byun)
+* rgw: radosgw-admin: add --uid check in bucket list command (`pr#30604 <https://github.com/ceph/ceph/pull/30604>`_, Vikhyat Umrao)
+* rgw: returns one byte more data than the requested range from the SLO object (`pr#29960 <https://github.com/ceph/ceph/pull/29960>`_, Andrey Groshev)
+* rgw: rgw-admin: search for user by access key (`pr#29959 <https://github.com/ceph/ceph/pull/29959>`_, Matt Benjamin)
+* rgw: rgw-log issues the wrong message when decompression fails (`pr#29965 <https://github.com/ceph/ceph/pull/29965>`_, Han Fengzhe)
+* rgw: rgw_file: directory enumeration can be accelerated 1-2 orders of magnitude taking stats from bucket index Part I (stats from S3/Swift only) (`issue#40456 <http://tracker.ceph.com/issues/40456>`_, `pr#29954 <https://github.com/ceph/ceph/pull/29954>`_, Matt Benjamin)
+* rgw: rgw_file: readdir: do not construct markers w/leading '/' (`pr#29969 <https://github.com/ceph/ceph/pull/29969>`_, Matt Benjamin)
+* rgw: silence warning "control reaches end of non-void function" (`issue#40747 <http://tracker.ceph.com/issues/40747>`_, `pr#31742 <https://github.com/ceph/ceph/pull/31742>`_, Jos Collin)
+* rgw: sync with elastic search v7 (`pr#31027 <https://github.com/ceph/ceph/pull/31027>`_, Chang Liu)
+* rgw: use explicit to_string() overload for boost::string_ref (`issue#39611 <http://tracker.ceph.com/issues/39611>`_, `pr#31650 <https://github.com/ceph/ceph/pull/31650>`_, Casey Bodley, Ulrich Weigand)
+* rgw: when using radosgw-admin to list bucket, can set --max-entries excessively high (`pr#29777 <https://github.com/ceph/ceph/pull/29777>`_, J. Eric Ivancich)
+* tests: "CMake Error" in test_envlibrados_for_rocksdb.sh (`pr#29979 <https://github.com/ceph/ceph/pull/29979>`_, Kefu Chai)
+* tests: Get libcephfs and cephfs to compile with FreeBSD (`pr#31136 <https://github.com/ceph/ceph/pull/31136>`_, Willem Jan Withagen)
+* tests: add debugging failed osd-release setting (`pr#31040 <https://github.com/ceph/ceph/pull/31040>`_, Patrick Donnelly)
+* tests: cephfs: fix malformed qa suite config (`pr#30038 <https://github.com/ceph/ceph/pull/30038>`_, Patrick Donnelly)
+* tests: cls_rbd/test_cls_rbd: update TestClsRbd.sparsify (`pr#30354 <https://github.com/ceph/ceph/pull/30354>`_, Kefu Chai)
+* tests: cls_rbd: removed mirror peer pool test cases (`pr#30948 <https://github.com/ceph/ceph/pull/30948>`_, Jason Dillaman)
+* tests: enable dashboard tests to be run with "--suite rados/dashboard" (`pr#31248 <https://github.com/ceph/ceph/pull/31248>`_, Nathan Cutler)
+* tests: librbd: set nbd timeout due to newer kernels defaulting it on (`pr#30423 <https://github.com/ceph/ceph/pull/30423>`_, Jason Dillaman)
+* tests: qa/suites/krbd: run unmap subsuite with msgr1 only (`pr#31290 <https://github.com/ceph/ceph/pull/31290>`_, Ilya Dryomov)
+* tests: qa/tasks/cbt: run stop-all.sh while shutting down (`pr#31304 <https://github.com/ceph/ceph/pull/31304>`_, Sage Weil)
+* tests: qa/tasks/ceph.conf.template: increase mon tell retries (`pr#31641 <https://github.com/ceph/ceph/pull/31641>`_, Sage Weil)
+* tests: qa/workunits/rbd: stress test `rbd mirror pool status --verbose` (`pr#29871 <https://github.com/ceph/ceph/pull/29871>`_, Mykola Golub)
+* tests: qa: avoid page cache for krbd discard round off tests (`pr#30464 <https://github.com/ceph/ceph/pull/30464>`_, Ilya Dryomov)
+* tests: qa: sleep briefly after resetting kclient (`pr#29750 <https://github.com/ceph/ceph/pull/29750>`_, Patrick Donnelly)
+* tests: rados/mgr/tasks/module_selftest: whitelist mgr client getting blacklisted (`issue#40867 <http://tracker.ceph.com/issues/40867>`_, `pr#29649 <https://github.com/ceph/ceph/pull/29649>`_, Sage Weil)
+* tests: test_librados_build.sh: grab from nautilus branch in nautilus (`pr#31604 <https://github.com/ceph/ceph/pull/31604>`_, Nathan Cutler)
+* tests: valgrind: UninitCondition in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update_final() (`issue#38827 <http://tracker.ceph.com/issues/38827>`_, `pr#29928 <https://github.com/ceph/ceph/pull/29928>`_, Radoslaw Zarzynski)
+* tools/rados: add --pgid in help (`pr#30607 <https://github.com/ceph/ceph/pull/30607>`_, Vikhyat Umrao)
+* tools/rados: call pool_lookup() after rados is connected (`pr#30605 <https://github.com/ceph/ceph/pull/30605>`_, Vikhyat Umrao)
+* tools/rbd-ggate: close log before running postfork (`pr#30120 <https://github.com/ceph/ceph/pull/30120>`_, Willem Jan Withagen)
+* tools: ceph-backport.sh: add deprecation warning (`pr#30748 <https://github.com/ceph/ceph/pull/30748>`_, Nathan Cutler)
+* tools: ceph-objectstore-tool can't remove head with bad snapset (`pr#30080 <https://github.com/ceph/ceph/pull/30080>`_, David Zafman)
+
+
+v14.2.4 Nautilus
+================
+
+This is the fourth release in the Ceph Nautilus stable release series. Its sole
+purpose is to fix a regression that found its way into the previous release.
+
+Notable Changes
+---------------
+
+* The ceph-volume in Nautilus v14.2.3 was found to contain a serious
+  regression, described in ``https://tracker.ceph.com/issues/41660``, which 
+  prevented deployment tools like ceph-ansible, DeepSea, Rook, etc. from 
+  deploying/removing OSDs.
+
+Changelog
+---------
+
+* ceph-volume: fix stderr failure to decode/encode when redirected (`pr#30300 <https://github.com/ceph/ceph/pull/30300>`_, Alfredo Deza)
+
+
+v14.2.3 Nautilus
+================
+
+This is the third bug fix release of Ceph Nautilus release series. We recommend
+all Nautilus users upgrade to this release. For upgrading from older releases of
+ceph, general guidelines for upgrade to nautilus must be followed
+:ref:`nautilus-old-upgrade`.
+
+Notable Changes
+---------------
+
+* `CVE-2019-10222` - Fixed a denial of service vulnerability where an
+  unauthenticated client of Ceph Object Gateway could trigger a crash from an
+  uncaught exception
+
+* Nautilus-based librbd clients can now open images on Jewel clusters.
+
+* The RGW `num_rados_handles` has been removed. If you were using a value of
+  `num_rados_handles` greater than 1, multiply your current
+  `objecter_inflight_ops` and `objecter_inflight_op_bytes` parameters by the
+  old `num_rados_handles` to get the same throttle behavior.
+
+* The secure mode of Messenger v2 protocol is no longer experimental with this
+  release. This mode is now the preferred mode of connection for monitors.
+
+* "osd_deep_scrub_large_omap_object_key_threshold" has been lowered to detect an
+  object with large number of omap keys more easily.
+
+* The Ceph Dashboard now supports silencing Prometheus alert notifications.
+
+Changelog
+---------
+
+* bluestore: 50-100% iops lost due to bluefs_preextend_wal_files = false (`issue#38559 <http://tracker.ceph.com/issues/38559>`_, `pr#28573 <https://github.com/ceph/ceph/pull/28573>`_, Vitaliy Filippov)
+* bluestore: add slow op detection for collection_listing (`pr#29227 <https://github.com/ceph/ceph/pull/29227>`_, Igor Fedotov)
+* bluestore: avoid length overflow in extents returned by Stupid Allocator (`issue#40703 <http://tracker.ceph.com/issues/40703>`_, `pr#29023 <https://github.com/ceph/ceph/pull/29023>`_, Igor Fedotov)
+* bluestore/bluefs_types: consolidate contiguous extents (`pr#28862 <https://github.com/ceph/ceph/pull/28862>`_, Sage Weil)
+* bluestore/bluestore-tool: minor fixes around migrate (`pr#28893 <https://github.com/ceph/ceph/pull/28893>`_, Igor Fedotov)
+* bluestore: create the tail when first set FLAG_OMAP (`issue#36482 <http://tracker.ceph.com/issues/36482>`_, `pr#28963 <https://github.com/ceph/ceph/pull/28963>`_, Tao Ning)
+* bluestore: do not set osd_memory_target default from cgroup limit (`pr#29745 <https://github.com/ceph/ceph/pull/29745>`_, Sage Weil)
+* bluestore: fix >2GB bluefs writes (`pr#28966 <https://github.com/ceph/ceph/pull/28966>`_, kungf, Sage Weil)
+* bluestore: load OSD all compression settings unconditionally (`issue#40480 <http://tracker.ceph.com/issues/40480>`_, `pr#28892 <https://github.com/ceph/ceph/pull/28892>`_, Igor Fedotov)
+* bluestore: more smart allocator dump when lacking space for bluefs (`issue#40623 <http://tracker.ceph.com/issues/40623>`_, `pr#28891 <https://github.com/ceph/ceph/pull/28891>`_, Igor Fedotov)
+* bluestore: Set concurrent max_background_compactions in rocksdb to 2 (`issue#40769 <http://tracker.ceph.com/issues/40769>`_, `pr#29162 <https://github.com/ceph/ceph/pull/29162>`_, Mark Nelson)
+* bluestore: support RocksDB prefetch in buffered read mode (`pr#28962 <https://github.com/ceph/ceph/pull/28962>`_, Igor Fedotov)
+* build/ops: Module 'dashboard' has failed: No module named routes (`issue#24420 <http://tracker.ceph.com/issues/24420>`_, `pr#28992 <https://github.com/ceph/ceph/pull/28992>`_, Paul Emmerich)
+* build/ops: rpm: drop SuSEfirewall2 (`issue#40738 <http://tracker.ceph.com/issues/40738>`_, `pr#29007 <https://github.com/ceph/ceph/pull/29007>`_, Matthias Gerstner)
+* build/ops: rpm: Require ceph-grafana-dashboards (`pr#29682 <https://github.com/ceph/ceph/pull/29682>`_, Boris Ranto)
+* cephfs: ceph-fuse: mount does not support the fallocate() (`issue#40615 <http://tracker.ceph.com/issues/40615>`_, `pr#29157 <https://github.com/ceph/ceph/pull/29157>`_, huanwen ren)
+* cephfs: ceph_volume_client: d_name needs to be converted to string before using (`issue#39406 <http://tracker.ceph.com/issues/39406>`_, `pr#28609 <https://github.com/ceph/ceph/pull/28609>`_, Rishabh Dave)
+* cephfs: client: bump ll_ref from int32 to uint64_t (`pr#29186 <https://github.com/ceph/ceph/pull/29186>`_, Xiaoxi CHEN)
+* cephfs: client: set snapdir's link count to 1 (`issue#40101 <http://tracker.ceph.com/issues/40101>`_, `pr#29343 <https://github.com/ceph/ceph/pull/29343>`_, "Yan, Zheng")
+* cephfs: client: unlink dentry for inode with llref=0 (`issue#40960 <http://tracker.ceph.com/issues/40960>`_, `pr#29478 <https://github.com/ceph/ceph/pull/29478>`_, Xiaoxi CHEN)
+* cephfs: getattr on snap inode stuck (`issue#40361 <http://tracker.ceph.com/issues/40361>`_, `pr#29231 <https://github.com/ceph/ceph/pull/29231>`_, "Yan, Zheng")
+* cephfs: mds: cannot switch mds state from standby-replay to active (`issue#40213 <http://tracker.ceph.com/issues/40213>`_, `pr#29233 <https://github.com/ceph/ceph/pull/29233>`_, simon gao)
+* cephfs: mds: cleanup unneeded client_snap_caps when splitting snap inode (`issue#39987 <http://tracker.ceph.com/issues/39987>`_, `pr#29344 <https://github.com/ceph/ceph/pull/29344>`_, "Yan, Zheng")
+* cephfs-shell: name 'files' is not defined error in do_rm() (`issue#40489 <http://tracker.ceph.com/issues/40489>`_, `pr#29158 <https://github.com/ceph/ceph/pull/29158>`_, Varsha Rao)
+* cephfs-shell: TypeError in poutput (`issue#40679 <http://tracker.ceph.com/issues/40679>`_, `pr#29156 <https://github.com/ceph/ceph/pull/29156>`_, Varsha Rao)
+* ceph.spec.in: Drop systemd BuildRequires in case of building for SUSE (`pr#28937 <https://github.com/ceph/ceph/pull/28937>`_, Dominique Leuenberger)
+* ceph-volume: batch functional idempotency test fails since message is now on stderr (`pr#29689 <https://github.com/ceph/ceph/pull/29689>`_, Jan Fajerski)
+* ceph-volume: batch gets confused when the same device is passed in two device lists (`pr#29690 <https://github.com/ceph/ceph/pull/29690>`_, Jan Fajerski)
+* ceph-volume: does not recognize wal/db partitions created by ceph-disk (`pr#29464 <https://github.com/ceph/ceph/pull/29464>`_, Jan Fajerski)
+* ceph-volume: [filestore,bluestore] single type strategies fail after tracking devices as sets (`pr#29702 <https://github.com/ceph/ceph/pull/29702>`_, Jan Fajerski)
+* ceph-volume: lvm.activate: Return an error if WAL/DB devices absent (`pr#29040 <https://github.com/ceph/ceph/pull/29040>`_, David Casier)
+* ceph-volume: missing string substitution when reporting mounts (`issue#25030 <http://tracker.ceph.com/issues/25030>`_, `pr#29260 <https://github.com/ceph/ceph/pull/29260>`_, Shyukri Shyukriev)
+* ceph-volume: prints errors to stdout with --format json (`issue#38548 <http://tracker.ceph.com/issues/38548>`_, `pr#29506 <https://github.com/ceph/ceph/pull/29506>`_, Jan Fajerski)
+* ceph-volume: prints log messages to stdout (`pr#29600 <https://github.com/ceph/ceph/pull/29600>`_, Jan Fajerski, Kefu Chai, Alfredo Deza)
+* ceph-volume: run functional tests without dashboard (`pr#29694 <https://github.com/ceph/ceph/pull/29694>`_, Andrew Schoen)
+* ceph-volume: simple functional tests drop test for lvm zap (`pr#29660 <https://github.com/ceph/ceph/pull/29660>`_, Jan Fajerski)
+* ceph-volume: tests set the noninteractive flag for Debian (`pr#29899 <https://github.com/ceph/ceph/pull/29899>`_, Alfredo Deza)
+* ceph-volume: when 'type' file is not present activate fails (`pr#29416 <https://github.com/ceph/ceph/pull/29416>`_, Alfredo Deza)
+* cmake: update FindBoost.cmake (`pr#29436 <https://github.com/ceph/ceph/pull/29436>`_, Willem Jan Withagen)
+* common/config: respect POD_MEMORY_REQUEST \*and\* POD_MEMORY_LIMIT env vars (`pr#29562 <https://github.com/ceph/ceph/pull/29562>`_, Patrick Donnelly, Sage Weil)
+* common: Keyrings created by ceph auth get are not suitable for ceph auth import (`issue#22227 <http://tracker.ceph.com/issues/22227>`_, `pr#28740 <https://github.com/ceph/ceph/pull/28740>`_, Kefu Chai)
+* common: OutputDataSocket retakes mutex on error path (`issue#40188 <http://tracker.ceph.com/issues/40188>`_, `pr#29147 <https://github.com/ceph/ceph/pull/29147>`_, Casey Bodley)
+* core: Better default value for osd_snap_trim_sleep (`pr#29678 <https://github.com/ceph/ceph/pull/29678>`_, Neha Ojha)
+* core: Change default for bluestore_fsck_on_mount_deep as false (`pr#29697 <https://github.com/ceph/ceph/pull/29697>`_, Neha Ojha)
+* core: lazy omap stat collection (`pr#29188 <https://github.com/ceph/ceph/pull/29188>`_, Brad Hubbard)
+* core: librados: move buffer free functions to inline namespace (`issue#39972 <http://tracker.ceph.com/issues/39972>`_, `pr#29244 <https://github.com/ceph/ceph/pull/29244>`_, Jason Dillaman)
+* core: maybe_remove_pg_upmap can be super inefficient for large clusters (`issue#40104 <http://tracker.ceph.com/issues/40104>`_, `pr#28756 <https://github.com/ceph/ceph/pull/28756>`_, xie xingguo)
+* core: MDSMonitor: use stringstream instead of dout for mds repaired (`issue#40472 <http://tracker.ceph.com/issues/40472>`_, `pr#29159 <https://github.com/ceph/ceph/pull/29159>`_, Zhi Zhang)
+* core: osd beacon sometimes has empty pg list (`issue#40377 <http://tracker.ceph.com/issues/40377>`_, `pr#29254 <https://github.com/ceph/ceph/pull/29254>`_, Sage Weil)
+* core: s3tests-test-readwrite failed in rados run (Connection refused) (`issue#17882 <http://tracker.ceph.com/issues/17882>`_, `pr#29325 <https://github.com/ceph/ceph/pull/29325>`_, Casey Bodley)
+* doc: Document more cache modes (`issue#14153 <http://tracker.ceph.com/issues/14153>`_, `pr#28958 <https://github.com/ceph/ceph/pull/28958>`_, Nathan Cutler)
+* doc: fix rgw ldap username token (`pr#29455 <https://github.com/ceph/ceph/pull/29455>`_, Thomas Kriechbaumer)
+* doc: Improved dashboard feature overview (`pr#28919 <https://github.com/ceph/ceph/pull/28919>`_, Lenz Grimmer)
+* doc: Object Gateway multisite document read-only argument error (`issue#40458 <http://tracker.ceph.com/issues/40458>`_, `pr#29306 <https://github.com/ceph/ceph/pull/29306>`_, Chenjiong Deng)
+* doc/rados: Correcting some typos in the clay code documentation (`pr#29191 <https://github.com/ceph/ceph/pull/29191>`_, Myna)
+* doc/rbd: initial live-migration documentation (`issue#40486 <http://tracker.ceph.com/issues/40486>`_, `pr#29724 <https://github.com/ceph/ceph/pull/29724>`_, Jason Dillaman)
+* doc/rgw: document use of 'realm pull' instead of 'period pull' (`issue#39655 <http://tracker.ceph.com/issues/39655>`_, `pr#29484 <https://github.com/ceph/ceph/pull/29484>`_, Casey Bodley)
+* doc: steps to disable metadata_heap on existing rgw zones (`issue#18174 <http://tracker.ceph.com/issues/18174>`_, `pr#28738 <https://github.com/ceph/ceph/pull/28738>`_, Dan van der Ster)
+* doc: Update 'ceph-iscsi' min version (`pr#29444 <https://github.com/ceph/ceph/pull/29444>`_, Ricardo Marques)
+* journal: properly advance read offset after skipping invalid range (`pr#28816 <https://github.com/ceph/ceph/pull/28816>`_, Mykola Golub)
+* librbd: improve journal performance to match expected degredation (`issue#40072 <http://tracker.ceph.com/issues/40072>`_, `pr#29723 <https://github.com/ceph/ceph/pull/29723>`_, Mykola Golub, Jason Dillaman)
+* librbd: properly track in-flight flush requests (`issue#40555 <http://tracker.ceph.com/issues/40555>`_, `pr#28769 <https://github.com/ceph/ceph/pull/28769>`_, Jason Dillaman)
+* librbd: snapshot object maps can go inconsistent during copyup (`issue#39435 <http://tracker.ceph.com/issues/39435>`_, `pr#29722 <https://github.com/ceph/ceph/pull/29722>`_, Ilya Dryomov)
+* mds: change how mds revoke stale caps (`issue#17854 <http://tracker.ceph.com/issues/17854>`_, `pr#28583 <https://github.com/ceph/ceph/pull/28583>`_, Rishabh Dave, "Yan, Zheng")
+* mgr: Add mgr metdata to prometheus exporter module (`pr#29168 <https://github.com/ceph/ceph/pull/29168>`_, Paul Cuzner)
+* mgr/dashboard: Add, update and remove translations (`issue#39701 <http://tracker.ceph.com/issues/39701>`_, `pr#28938 <https://github.com/ceph/ceph/pull/28938>`_, Sebastian Krah)
+* mgr/dashboard: cephfs multimds graphs stack together (`issue#37579 <http://tracker.ceph.com/issues/37579>`_, `pr#28889 <https://github.com/ceph/ceph/pull/28889>`_, Kiefer Chang)
+* mgr/dashboard: Changing rgw-api-host does not get effective without disable/enable dashboard mgr module (`issue#40252 <http://tracker.ceph.com/issues/40252>`_, `pr#29044 <https://github.com/ceph/ceph/pull/29044>`_, Ricardo Marques)
+* mgr/dashboard: controllers/grafana is not Python3 compatible (`issue#40428 <http://tracker.ceph.com/issues/40428>`_, `pr#29524 <https://github.com/ceph/ceph/pull/29524>`_, Patrick Nawracay)
+* mgr/dashboard: Dentries value of MDS daemon in Filesystems page is inconsistent with ceph fs status output (`issue#40097 <http://tracker.ceph.com/issues/40097>`_, `pr#28912 <https://github.com/ceph/ceph/pull/28912>`_, Kiefer Chang)
+* mgr/dashboard: Display logged in information for each iSCSI client (`issue#40046 <http://tracker.ceph.com/issues/40046>`_, `pr#29045 <https://github.com/ceph/ceph/pull/29045>`_, Ricardo Marques)
+* mgr/dashboard: Fix e2e failures caused by webdriver version (`pr#29491 <https://github.com/ceph/ceph/pull/29491>`_, Tiago Melo)
+* mgr/dashboard: Fix npm vulnerabilities (`issue#40677 <http://tracker.ceph.com/issues/40677>`_, `pr#29102 <https://github.com/ceph/ceph/pull/29102>`_, Tiago Melo)
+* mgr/dashboard: Fix the table mouseenter event handling test (`issue#40580 <http://tracker.ceph.com/issues/40580>`_, `pr#29354 <https://github.com/ceph/ceph/pull/29354>`_, Stephan Müller)
+* mgr/dashboard: Interlock `fast-diff` and `object-map` (`issue#39451 <http://tracker.ceph.com/issues/39451>`_, `pr#29442 <https://github.com/ceph/ceph/pull/29442>`_, Patrick Nawracay)
+* mgr/dashboard: notify the user about unset 'mon_allow_pool_delete' flag beforehand (`issue#39533 <http://tracker.ceph.com/issues/39533>`_, `pr#28833 <https://github.com/ceph/ceph/pull/28833>`_, Tatjana Dehler)
+* mgr/dashboard: Optimize the calculation of portal IPs (`issue#39580 <http://tracker.ceph.com/issues/39580>`_, `pr#29061 <https://github.com/ceph/ceph/pull/29061>`_, Ricardo Marques, Kefu Chai)
+* mgr/dashboard: Pool graph/sparkline points do not display the correct values (`issue#39650 <http://tracker.ceph.com/issues/39650>`_, `pr#29352 <https://github.com/ceph/ceph/pull/29352>`_, Stephan Müller)
+* mgr/dashboard: RGW User quota validation is not working correctly (`pr#29650 <https://github.com/ceph/ceph/pull/29650>`_, Volker Theile)
+* mgr/dashboard: Silence Alertmanager alerts (`issue#36722 <http://tracker.ceph.com/issues/36722>`_, `pr#28968 <https://github.com/ceph/ceph/pull/28968>`_, Stephan Müller)
+* mgr/dashboard: SSL certificate upload command throws deprecation warning (`issue#39123 <http://tracker.ceph.com/issues/39123>`_, `pr#29065 <https://github.com/ceph/ceph/pull/29065>`_, Ricardo Dias)
+* mgr/dashboard: switch ng2-toastr to ngx-toastr (`pr#29050 <https://github.com/ceph/ceph/pull/29050>`_, Tiago Melo, Ernesto Puerta)
+* mgr/dashboard: Upgrade to ceph-iscsi config v10 (`issue#40566 <http://tracker.ceph.com/issues/40566>`_, `pr#28974 <https://github.com/ceph/ceph/pull/28974>`_, Ricardo Marques)
+* mgr/diskprediction_cloud: Service unavailable (`issue#40478 <http://tracker.ceph.com/issues/40478>`_, `pr#29454 <https://github.com/ceph/ceph/pull/29454>`_, Rick Chen)
+* mgr/influx: module fails due to missing close() method (`issue#40174 <http://tracker.ceph.com/issues/40174>`_, `pr#29207 <https://github.com/ceph/ceph/pull/29207>`_, Kefu Chai)
+* mgr/orchestrator: Cache and DeepSea iSCSI + NFS (`pr#29060 <https://github.com/ceph/ceph/pull/29060>`_, Sebastian Wagner, Tim Serong)
+* mgr/rbd_support: support scheduling long-running background operations (`issue#40621 <http://tracker.ceph.com/issues/40621>`_, `issue#40790 <http://tracker.ceph.com/issues/40790>`_, `pr#29725 <https://github.com/ceph/ceph/pull/29725>`_, Venky Shankar, Jason Dillaman)
+* mgr: use ipv4 default when ipv6 was disabled (`issue#40023 <http://tracker.ceph.com/issues/40023>`_, `pr#29194 <https://github.com/ceph/ceph/pull/29194>`_, kungf)
+* mgr/volumes: background purge queue for subvolumes (`issue#40036 <http://tracker.ceph.com/issues/40036>`_, `pr#29079 <https://github.com/ceph/ceph/pull/29079>`_, Patrick Donnelly, Venky Shankar, Kefu Chai)
+* mgr/volumes: minor enhancement and bug fix (`issue#40927 <http://tracker.ceph.com/issues/40927>`_, `issue#40617 <http://tracker.ceph.com/issues/40617>`_, `pr#29490 <https://github.com/ceph/ceph/pull/29490>`_, Ramana Raja)
+* mon: auth mon isn't loading full KeyServerData after restart (`issue#40634 <http://tracker.ceph.com/issues/40634>`_, `pr#28993 <https://github.com/ceph/ceph/pull/28993>`_, Sage Weil)
+* mon/MgrMonitor: fix null deref when invalid formatter is specified (`pr#29566 <https://github.com/ceph/ceph/pull/29566>`_, Sage Weil)
+* mon/OSDMonitor: allow pg_num to increase when require_osd_release < N (`issue#39570 <http://tracker.ceph.com/issues/39570>`_, `pr#29671 <https://github.com/ceph/ceph/pull/29671>`_, Neha Ojha, Sage Weil)
+* mon/OSDMonitor.cc: better error message about min_size (`pr#29617 <https://github.com/ceph/ceph/pull/29617>`_, Neha Ojha)
+* mon: paxos: introduce new reset_pending_committing_finishers for safety (`issue#39484 <http://tracker.ceph.com/issues/39484>`_, `pr#28528 <https://github.com/ceph/ceph/pull/28528>`_, Greg Farnum)
+* mon: set recovery priority etc on cephfs metadata pool (`pr#29275 <https://github.com/ceph/ceph/pull/29275>`_, Sage Weil)
+* mon: take the mon lock in handle_conf_change (`issue#39625 <http://tracker.ceph.com/issues/39625>`_, `pr#29373 <https://github.com/ceph/ceph/pull/29373>`_, huangjun)
+* msg/async: avoid unnecessary costly wakeups for outbound messages (`pr#29141 <https://github.com/ceph/ceph/pull/29141>`_, Jason Dillaman)
+* msg/async: enable secure mode by default, no longer experimental (`pr#29143 <https://github.com/ceph/ceph/pull/29143>`_, Sage Weil)
+* msg/async: no-need set connection for Message (`pr#29142 <https://github.com/ceph/ceph/pull/29142>`_, Jianpeng Ma)
+* msg/async, v2: make the reset_recv_state() unconditional (`issue#40115 <http://tracker.ceph.com/issues/40115>`_, `pr#29140 <https://github.com/ceph/ceph/pull/29140>`_, Radoslaw Zarzynski, Sage Weil)
+* nautilus:common/options.cc: Lower the default value of osd_deep_scrub_large_omap_object_key_threshold (`pr#29173 <https://github.com/ceph/ceph/pull/29173>`_, Neha Ojha)
+* osd: Don't randomize deep scrubs when noscrub set (`issue#40198 <http://tracker.ceph.com/issues/40198>`_, `pr#28768 <https://github.com/ceph/ceph/pull/28768>`_, David Zafman)
+* osd: Fix the way that auto repair triggers after regular scrub (`issue#40530 <http://tracker.ceph.com/issues/40530>`_, `issue#40073 <http://tracker.ceph.com/issues/40073>`_, `pr#28869 <https://github.com/ceph/ceph/pull/28869>`_, sjust@redhat.com, David Zafman)
+* osd/OSD: auto mark heartbeat sessions as stale and tear them down (`issue#40586 <http://tracker.ceph.com/issues/40586>`_, `pr#29391 <https://github.com/ceph/ceph/pull/29391>`_, xie xingguo)
+* osd/OSD: keep synchronizing with mon if stuck at booting (`pr#28639 <https://github.com/ceph/ceph/pull/28639>`_, xie xingguo)
+* osd/PG: do not queue scrub if PG is not active when unblock (`issue#40451 <http://tracker.ceph.com/issues/40451>`_, `pr#29372 <https://github.com/ceph/ceph/pull/29372>`_, Sage Weil)
+* osd/PG: fix cleanup of pgmeta-like objects on PG deletion (`pr#29115 <https://github.com/ceph/ceph/pull/29115>`_, Sage Weil)
+* pybind/mgr/rbd_support: ignore missing support for RBD namespaces (`issue#41475 <https://tracker.ceph.com/issues/41475>`_, `pr#29945 <https://github.com/ceph/ceph/pull/29945>`_, Mykola Golub)
+* rbd/action: fix error getting positional argument (`issue#40095 <http://tracker.ceph.com/issues/40095>`_, `pr#28870 <https://github.com/ceph/ceph/pull/28870>`_, songweibin)
+* rbd: [cli] 'export' should handle concurrent IO completions (`issue#40435 <http://tracker.ceph.com/issues/40435>`_, `pr#29329 <https://github.com/ceph/ceph/pull/29329>`_, Jason Dillaman)
+* rbd: librbd: do not unblock IO prior to growing object map during resize (`issue#39952 <http://tracker.ceph.com/issues/39952>`_, `pr#29246 <https://github.com/ceph/ceph/pull/29246>`_, Jason Dillaman)
+* rbd-mirror: handle duplicates in image sync throttler queue (`issue#40519 <http://tracker.ceph.com/issues/40519>`_, `pr#28817 <https://github.com/ceph/ceph/pull/28817>`_, Mykola Golub)
+* rbd-mirror: link against the specified alloc library (`issue#40110 <http://tracker.ceph.com/issues/40110>`_, `pr#29193 <https://github.com/ceph/ceph/pull/29193>`_, Jason Dillaman)
+* rbd-nbd: sscanf return 0 mean not-match (`issue#39269 <http://tracker.ceph.com/issues/39269>`_, `pr#29315 <https://github.com/ceph/ceph/pull/29315>`_, Jianpeng Ma)
+* rbd: profile rbd OSD cap should add class rbd metadata_list cap by default (`issue#39973 <http://tracker.ceph.com/issues/39973>`_, `pr#29328 <https://github.com/ceph/ceph/pull/29328>`_, songweibin)
+* rbd: Reduce log level for cls/journal and cls/rbd expected errors (`issue#40865 <http://tracker.ceph.com/issues/40865>`_, `pr#29551 <https://github.com/ceph/ceph/pull/29551>`_, Jason Dillaman)
+* rbd: tests: add "rbd diff" coverage to suite (`issue#39447 <http://tracker.ceph.com/issues/39447>`_, `pr#28575 <https://github.com/ceph/ceph/pull/28575>`_, Shyukri Shyukriev, Nathan Cutler)
+* rgw: add 'GET /admin/realm?list' api to list realms (`issue#39626 <http://tracker.ceph.com/issues/39626>`_, `pr#28751 <https://github.com/ceph/ceph/pull/28751>`_, Casey Bodley)
+* rgw: allow radosgw-admin to list bucket w --allow-unordered (`issue#39637 <http://tracker.ceph.com/issues/39637>`_, `pr#28230 <https://github.com/ceph/ceph/pull/28230>`_, J. Eric Ivancich)
+* rgw: conditionally allow builtin users with non-unique email addresses (`issue#40089 <http://tracker.ceph.com/issues/40089>`_, `pr#28715 <https://github.com/ceph/ceph/pull/28715>`_, Matt Benjamin)
+* rgw: deleting bucket can fail when it contains unfinished multipart uploads (`issue#40526 <http://tracker.ceph.com/issues/40526>`_, `pr#29154 <https://github.com/ceph/ceph/pull/29154>`_, J. Eric Ivancich)
+* rgw: Don't crash on copy when metadata directive not supplied (`issue#40416 <http://tracker.ceph.com/issues/40416>`_, `pr#29499 <https://github.com/ceph/ceph/pull/29499>`_, Adam C. Emerson)
+* rgw_file: advance_mtime() should consider namespace expiration (`issue#40415 <http://tracker.ceph.com/issues/40415>`_, `pr#29410 <https://github.com/ceph/ceph/pull/29410>`_, Matt Benjamin)
+* rgw_file:  advance_mtime() takes RGWFileHandle::mutex unconditionally (`pr#29801 <https://github.com/ceph/ceph/pull/29801>`_, Matt Benjamin)
+* rgw_file: all directories are virtual with respect to contents (`issue#40204 <http://tracker.ceph.com/issues/40204>`_, `pr#28886 <https://github.com/ceph/ceph/pull/28886>`_, Matt Benjamin)
+* rgw_file:  fix invalidation of top-level directories (`issue#40196 <http://tracker.ceph.com/issues/40196>`_, `pr#29309 <https://github.com/ceph/ceph/pull/29309>`_, Matt Benjamin)
+* rgw_file: fix readdir eof() calc--caller stop implies !eof (`issue#40375 <http://tracker.ceph.com/issues/40375>`_, `pr#29409 <https://github.com/ceph/ceph/pull/29409>`_, Matt Benjamin)
+* rgw_file: include tenant when hashing bucket names (`issue#40118 <http://tracker.ceph.com/issues/40118>`_, `pr#28854 <https://github.com/ceph/ceph/pull/28854>`_, Matt Benjamin)
+* rgw: fix miss get ret in STSService::storeARN (`issue#40386 <http://tracker.ceph.com/issues/40386>`_, `pr#28713 <https://github.com/ceph/ceph/pull/28713>`_, Tianshan Qu)
+* rgw: fix prefix handling in LCFilter (`issue#37879 <http://tracker.ceph.com/issues/37879>`_, `pr#28550 <https://github.com/ceph/ceph/pull/28550>`_, Matt Benjamin)
+* rgw: fix rgw crash and set correct error code (`pr#28729 <https://github.com/ceph/ceph/pull/28729>`_, yuliyang)
+* rgw: hadoop-s3a suite failing with more ansible errors (`issue#39706 <http://tracker.ceph.com/issues/39706>`_, `pr#28735 <https://github.com/ceph/ceph/pull/28735>`_, Casey Bodley)
+* rgw: hadoop-s3a suite failing with more ansible errors (`issue#39706 <http://tracker.ceph.com/issues/39706>`_, `pr#29265 <https://github.com/ceph/ceph/pull/29265>`_, Casey Bodley)
+* rgw: Librgw doesn't GC deleted object correctly (`issue#37734 <http://tracker.ceph.com/issues/37734>`_, `pr#28648 <https://github.com/ceph/ceph/pull/28648>`_, Tao Chen, Matt Benjamin)
+* rgw: multisite: DELETE Bucket CORS is not forwarded to master zone (`issue#39629 <http://tracker.ceph.com/issues/39629>`_, `pr#28714 <https://github.com/ceph/ceph/pull/28714>`_, Chang Liu)
+* rgw: multisite: fix --bypass-gc flag for 'radosgw-admin bucket rm' (`issue#24991 <http://tracker.ceph.com/issues/24991>`_, `pr#28549 <https://github.com/ceph/ceph/pull/28549>`_, Casey Bodley)
+* rgw: multisite: 'radosgw-admin bilog trim' stops after 1000 entries (`issue#40187 <http://tracker.ceph.com/issues/40187>`_, `pr#29326 <https://github.com/ceph/ceph/pull/29326>`_, Casey Bodley)
+* rgw: multisite: 'radosgw-admin bucket sync status' should call syncs_from(source.name) instead of id (`issue#40022 <http://tracker.ceph.com/issues/40022>`_, `pr#28739 <https://github.com/ceph/ceph/pull/28739>`_, Casey Bodley)
+* rgw: multisite: radosgw-admin commands should not modify metadata on a non-master zone (`issue#39548 <http://tracker.ceph.com/issues/39548>`_, `pr#29163 <https://github.com/ceph/ceph/pull/29163>`_, Shilpa Jagannath)
+* rgw: multisite: RGWListBucketIndexesCR for data full sync needs pagination (`issue#39551 <http://tracker.ceph.com/issues/39551>`_, `pr#29311 <https://github.com/ceph/ceph/pull/29311>`_, Shilpa Jagannath)
+* rgw/OutputDataSocket: append_output(buffer::list&) says it will (but does not) discard output at data_max_backlog (`issue#40178 <http://tracker.ceph.com/issues/40178>`_, `pr#29310 <https://github.com/ceph/ceph/pull/29310>`_, Matt Benjamin)
+* rgw, Policy should be url_decode when assume_role (`pr#28728 <https://github.com/ceph/ceph/pull/28728>`_, yuliyang)
+* rgw: provide admin-friendly reshard status output (`issue#37615 <http://tracker.ceph.com/issues/37615>`_, `pr#29286 <https://github.com/ceph/ceph/pull/29286>`_, Mark Kogan)
+* rgw: Put LC doesn't clear existing lifecycle (`issue#39654 <http://tracker.ceph.com/issues/39654>`_, `pr#29313 <https://github.com/ceph/ceph/pull/29313>`_, Abhishek Lekshmanan)
+* rgw: remove rgw_num_rados_handles; set autoscale parameters or rgw metadata pools (`pr#27684 <https://github.com/ceph/ceph/pull/27684>`_, Adam C. Emerson, Casey Bodley, Sage Weil)
+* rgw: RGWGC add perfcounter retire counter (`issue#38251 <http://tracker.ceph.com/issues/38251>`_, `pr#29308 <https://github.com/ceph/ceph/pull/29308>`_, Matt Benjamin)
+* rgw: Save an unnecessary copy of RGWEnv (`issue#40183 <http://tracker.ceph.com/issues/40183>`_, `pr#29205 <https://github.com/ceph/ceph/pull/29205>`_, Mark Kogan)
+* rgw: set null version object issues (`issue#36763 <http://tracker.ceph.com/issues/36763>`_, `pr#29287 <https://github.com/ceph/ceph/pull/29287>`_, Tianshan Qu)
+* rgw: Swift interface: server side copy fails if object name contains "?" (`issue#27217 <http://tracker.ceph.com/issues/27217>`_, `pr#28736 <https://github.com/ceph/ceph/pull/28736>`_, Casey Bodley)
+* rgw: TempURL should not allow PUTs with the X-Object-Manifest (`issue#20797 <http://tracker.ceph.com/issues/20797>`_, `pr#28712 <https://github.com/ceph/ceph/pull/28712>`_, Radoslaw Zarzynski)
+* rgw: the Multi-Object Delete operation of S3 API wrongly handles the Code response element (`issue#18241 <http://tracker.ceph.com/issues/18241>`_, `pr#28737 <https://github.com/ceph/ceph/pull/28737>`_, Radoslaw Zarzynski)
+* rocksdb: rocksdb_rmrange related improvements (`pr#29439 <https://github.com/ceph/ceph/pull/29439>`_, Zengran Zhang, Sage Weil)
+* rocksdb: Updated to v6.1.2 (`pr#29440 <https://github.com/ceph/ceph/pull/29440>`_, Mark Nelson)
+* tools: ceph-kvstore-tool: print db stats (`pr#28810 <https://github.com/ceph/ceph/pull/28810>`_, Igor Fedotov)
+
+
+v14.2.2 Nautilus
+================
+
+This is the second bug fix release of Ceph Nautilus release series. We recommend
+all Nautilus users upgrade to this release. For upgrading from older releases of
+ceph, general guidelines for upgrade to nautilus must be followed
+:ref:`nautilus-old-upgrade`.
+
+Notable Changes
+---------------
+
+* The no{up,down,in,out} related commands have been revamped.
+  There are now 2 ways to set the no{up,down,in,out} flags:
+  the old 'ceph osd [un]set <flag>' command, which sets cluster-wide flags;
+  and the new 'ceph osd [un]set-group <flags> <who>' command,
+  which sets flags in batch at the granularity of any crush node,
+  or device class.
+
+* radosgw-admin introduces two subcommands that allow the
+  managing of expire-stale objects that might be left behind after a
+  bucket reshard in earlier versions of RGW. One subcommand lists such
+  objects and the other deletes them. Read the troubleshooting section
+  of the dynamic resharding docs for details.
+
+* Earlier Nautilus releases (14.2.1 and 14.2.0) have an issue where
+  deploying a single new (Nautilus) BlueStore OSD on an upgraded
+  cluster (i.e. one that was originally deployed pre-Nautilus) breaks
+  the pool utilization stats reported by ``ceph df``.  Until all OSDs
+  have been reprovisioned or updated (via ``ceph-bluestore-tool
+  repair``), the pool stats will show values that are lower than the
+  true value.  This is resolved in 14.2.2, such that the cluster only
+  switches to using the more accurate per-pool stats after *all* OSDs
+  are 14.2.2 (or later), are BlueStore, and (if they were created
+  prior to Nautilus) have been updated via the ``repair`` function.
+
+* The default value for `mon_crush_min_required_version` has been
+  changed from `firefly` to `hammer`, which means the cluster will
+  issue a health warning if your CRUSH tunables are older than hammer.
+  There is generally a small (but non-zero) amount of data that will
+  move around by making the switch to hammer tunables; for more information,
+  see :ref:`crush-map-tunables`.
+
+  If possible, we recommend that you set the oldest allowed client to `hammer`
+  or later.  You can tell what the current oldest allowed client is with::
+
+    ceph osd dump | grep min_compat_client
+
+  If the current value is older than hammer, you can tell whether it
+  is safe to make this change by verifying that there are no clients
+  older than hammer current connected to the cluster with::
+
+    ceph features
+
+  The newer `straw2` CRUSH bucket type was introduced in hammer, and
+  ensuring that all clients are hammer or newer allows new features
+  only supported for `straw2` buckets to be used, including the
+  `crush-compat` mode for the :ref:`balancer`.
+
+Changelog
+---------
+
+* bluestore: backport more bluestore alerts (`pr#27645 <https://github.com/ceph/ceph/pull/27645>`_, Sage Weil, Igor Fedotov)
+* bluestore: call fault_range prior to looking for blob to reuse (`pr#27525 <https://github.com/ceph/ceph/pull/27525>`_, Igor Fedotov)
+* bluestore: correctly measure deferred writes into new blobs (`issue#38816 <http://tracker.ceph.com/issues/38816>`_, `pr#27819 <https://github.com/ceph/ceph/pull/27819>`_, Sage Weil)
+* bluestore: dump before "no-spanning blob id" abort (`pr#28028 <https://github.com/ceph/ceph/pull/28028>`_, Igor Fedotov)
+* bluestore: fix for FreeBSD iocb structure (`issue#39612 <http://tracker.ceph.com/issues/39612>`_, `pr#28007 <https://github.com/ceph/ceph/pull/28007>`_, Willem Jan Withagen)
+* bluestore: fix missing discard in BlueStore::_kv_sync_thread (`issue#39672 <http://tracker.ceph.com/issues/39672>`_, `pr#28258 <https://github.com/ceph/ceph/pull/28258>`_, Junhui Tang)
+* bluestore: fix out-of-bound access in bmap allocator (`pr#27740 <https://github.com/ceph/ceph/pull/27740>`_, Igor Fedotov)
+* bluestore: fix duplicate allocations in bmap allocator (`issue#40080 <http://tracker.ceph.com/issues/40080>`_, `pr#28646 <https://github.com/ceph/ceph/pull/28646>`_, Igor Fedotov)
+* build/ops: Ceph RPM build fails on openSUSE Tumbleweed with GCC 9 (`issue#40067 <http://tracker.ceph.com/issues/40067>`_, `issue#39974 <http://tracker.ceph.com/issues/39974>`_, `pr#28299 <https://github.com/ceph/ceph/pull/28299>`_, Martin Liška)
+* build/ops: cmake: Fix build against ncurses with separate libtinfo (`pr#27532 <https://github.com/ceph/ceph/pull/27532>`_, Lars Wendler)
+* build/ops: cmake: set empty-string RPATH for ceph-osd (`issue#40301 <http://tracker.ceph.com/issues/40301>`_, `issue#40295 <http://tracker.ceph.com/issues/40295>`_, `pr#28516 <https://github.com/ceph/ceph/pull/28516>`_, Nathan Cutler)
+* build/ops: do_cmake.sh: source not found (`issue#39981 <http://tracker.ceph.com/issues/39981>`_, `issue#40003 <http://tracker.ceph.com/issues/40003>`_, `pr#28215 <https://github.com/ceph/ceph/pull/28215>`_, Nathan Cutler)
+* build/ops: python3 pybind RPMs do not replace their python2 counterparts on upgrade even though they should (`issue#40099 <http://tracker.ceph.com/issues/40099>`_, `issue#40232 <http://tracker.ceph.com/issues/40232>`_, `pr#28469 <https://github.com/ceph/ceph/pull/28469>`_, Nathan Cutler)
+* build/ops: rpm: install grafana dashboards world readable (`pr#28392 <https://github.com/ceph/ceph/pull/28392>`_, Jan Fajerski)
+* build/ops: selinux: Update the policy for RHEL8 (`pr#28511 <https://github.com/ceph/ceph/pull/28511>`_, Boris Ranto)
+* ceph-volume: add utility functions (`pr#27791 <https://github.com/ceph/ceph/pull/27791>`_, Mohamad Gebai)
+* ceph-volume: broken assertion errors after pytest changes (`pr#28925 <https://github.com/ceph/ceph/pull/28925>`_, Alfredo Deza)
+* ceph-volume: look for rotational data in lsblk (`pr#27723 <https://github.com/ceph/ceph/pull/27723>`_, Andrew Schoen)
+* ceph-volume: tests add a sleep in tox for slow OSDs after booting (`pr#28924 <https://github.com/ceph/ceph/pull/28924>`_, Alfredo Deza)
+* ceph-volume: use the Device.rotational property instead of sys_api (`pr#29028 <https://github.com/ceph/ceph/pull/29028>`_, Andrew Schoen)
+* cephfs-shell: Revert "cephfs.pyx: add py3 compatibility (`pr#28641 <https://github.com/ceph/ceph/pull/28641>`_, Varsha Rao)
+* cephfs-shell: ls command produces error: no colorize attribute found error (`issue#39376 <http://tracker.ceph.com/issues/39376>`_, `issue#39378 <http://tracker.ceph.com/issues/39378>`_, `issue#38740 <http://tracker.ceph.com/issues/38740>`_, `issue#39379 <http://tracker.ceph.com/issues/39379>`_, `issue#39197 <http://tracker.ceph.com/issues/39197>`_, `issue#39377 <http://tracker.ceph.com/issues/39377>`_, `pr#27677 <https://github.com/ceph/ceph/pull/27677>`_, Milind Changire, Varsha Rao)
+* cephfs-shell: misc. cephfs-shell backports (`issue#40314 <http://tracker.ceph.com/issues/40314>`_, `issue#40471 <http://tracker.ceph.com/issues/40471>`_, `issue#40418 <http://tracker.ceph.com/issues/40418>`_, `issue#40469 <http://tracker.ceph.com/issues/40469>`_, `issue#40313 <http://tracker.ceph.com/issues/40313>`_, `issue#39937 <http://tracker.ceph.com/issues/39937>`_, `issue#39678 <http://tracker.ceph.com/issues/39678>`_, `issue#40244 <http://tracker.ceph.com/issues/40244>`_, `issue#39404 <http://tracker.ceph.com/issues/39404>`_, `issue#40243 <http://tracker.ceph.com/issues/40243>`_, `issue#39165 <http://tracker.ceph.com/issues/39165>`_, `issue#40470 <http://tracker.ceph.com/issues/40470>`_, `issue#40455 <http://tracker.ceph.com/issues/40455>`_, `issue#39936 <http://tracker.ceph.com/issues/39936>`_, `issue#40217 <http://tracker.ceph.com/issues/40217>`_, `pr#28681 <https://github.com/ceph/ceph/pull/28681>`_, Patrick Donnelly, Varsha Rao, Milind Changire)
+* cephfs-shell: mkdir error for relative path (`issue#39960 <http://tracker.ceph.com/issues/39960>`_, `pr#28616 <https://github.com/ceph/ceph/pull/28616>`_, Varsha Rao)
+* cephfs: FSAL_CEPH assertion failed in Client::_lookup_name: "parent->is_dir() (`issue#40085 <http://tracker.ceph.com/issues/40085>`_, `issue#40161 <http://tracker.ceph.com/issues/40161>`_, `pr#28612 <https://github.com/ceph/ceph/pull/28612>`_, Jeff Layton)
+* cephfs: ceph_volume_client: Too many arguments for "WriteOpCtx (`issue#39050 <http://tracker.ceph.com/issues/39050>`_, `issue#38946 <http://tracker.ceph.com/issues/38946>`_, `pr#27893 <https://github.com/ceph/ceph/pull/27893>`_, Ramana Raja)
+* cephfs: client: ceph.dir.rctime xattr value incorrectly prefixes 09 to the nanoseconds component (`issue#40167 <http://tracker.ceph.com/issues/40167>`_, `pr#28500 <https://github.com/ceph/ceph/pull/28500>`_, David Disseldorp)
+* cephfs: client: fix "ceph.snap.btime" vxattr value (`issue#40169 <http://tracker.ceph.com/issues/40169>`_, `pr#28499 <https://github.com/ceph/ceph/pull/28499>`_, David Disseldorp)
+* cephfs: client: fix fuse client hang because its bad session PipeConnection (`issue#39686 <http://tracker.ceph.com/issues/39686>`_, `issue#39305 <http://tracker.ceph.com/issues/39305>`_, `pr#28375 <https://github.com/ceph/ceph/pull/28375>`_, Guan yunfei)
+* cephfs: kclient: nofail option not supported (`issue#39232 <http://tracker.ceph.com/issues/39232>`_, `pr#27851 <https://github.com/ceph/ceph/pull/27851>`_, Kenneth Waegeman)
+* cephfs: mds: Expose CephFS snapshot creation time to clients (`issue#39471 <http://tracker.ceph.com/issues/39471>`_, `pr#27901 <https://github.com/ceph/ceph/pull/27901>`_, David Disseldorp)
+* cephfs: mds: MDSTableServer.cc: 83: FAILED assert(version == tid) (`issue#39211 <http://tracker.ceph.com/issues/39211>`_, `issue#38835 <http://tracker.ceph.com/issues/38835>`_, `pr#27853 <https://github.com/ceph/ceph/pull/27853>`_, "Yan, Zheng")
+* cephfs: mds: avoid sending too many osd requests at once after mds restarts (`issue#40028 <http://tracker.ceph.com/issues/40028>`_, `issue#40040 <http://tracker.ceph.com/issues/40040>`_, `pr#28582 <https://github.com/ceph/ceph/pull/28582>`_, simon gao)
+* cephfs: mds: behind on trimming and "[dentry] was purgeable but no longer is! (`issue#39222 <http://tracker.ceph.com/issues/39222>`_, `issue#38679 <http://tracker.ceph.com/issues/38679>`_, `pr#27879 <https://github.com/ceph/ceph/pull/27879>`_, "Yan, Zheng")
+* cephfs: mds: better output of 'ceph health detail (`issue#39266 <http://tracker.ceph.com/issues/39266>`_, `pr#27846 <https://github.com/ceph/ceph/pull/27846>`_, Shen Hang')
+* cephfs: mds: check dir fragment to split dir if mkdir makes it oversized (`issue#39690 <http://tracker.ceph.com/issues/39690>`_, `pr#28394 <https://github.com/ceph/ceph/pull/28394>`_, Erqi Chen)
+* cephfs: mds: check directory split after rename (`issue#39199 <http://tracker.ceph.com/issues/39199>`_, `issue#38994 <http://tracker.ceph.com/issues/38994>`_, `pr#27736 <https://github.com/ceph/ceph/pull/27736>`_, Shen Hang)
+* cephfs: mds: drop reconnect message from non-existent session (`issue#39026 <http://tracker.ceph.com/issues/39026>`_, `issue#39192 <http://tracker.ceph.com/issues/39192>`_, `pr#27714 <https://github.com/ceph/ceph/pull/27714>`_, Shen Hang)
+* cephfs: mds: fail to resolve snapshot name contains '_' (`issue#39473 <http://tracker.ceph.com/issues/39473>`_, `pr#27849 <https://github.com/ceph/ceph/pull/27849>`_, "Yan, Zheng')
+* cephfs: mds: fix 'is session in blacklist' check in Server::apply_blacklist() (`issue#40236 <http://tracker.ceph.com/issues/40236>`_, `issue#40061 <http://tracker.ceph.com/issues/40061>`_, `pr#28618 <https://github.com/ceph/ceph/pull/28618>`_, "Yan, Zheng')
+* cephfs: mds: fix corner case of replaying open sessions (`pr#28580 <https://github.com/ceph/ceph/pull/28580>`_, "Yan, Zheng")
+* cephfs: mds: high debug logging with many subtrees is slow (`issue#38876 <http://tracker.ceph.com/issues/38876>`_, `pr#27892 <https://github.com/ceph/ceph/pull/27892>`_, Rishabh Dave)
+* cephfs: mds: initialize cap_revoke_eviction_timeout with conf (`issue#39209 <http://tracker.ceph.com/issues/39209>`_, `issue#38844 <http://tracker.ceph.com/issues/38844>`_, `pr#27842 <https://github.com/ceph/ceph/pull/27842>`_, simon gao)
+* cephfs: mds: output lock state in format dump (`issue#39645 <http://tracker.ceph.com/issues/39645>`_, `issue#39670 <http://tracker.ceph.com/issues/39670>`_, `pr#28233 <https://github.com/ceph/ceph/pull/28233>`_, Zhi Zhang)
+* cephfs: mds: reset heartbeat during long-running loops in recovery (`issue#40223 <http://tracker.ceph.com/issues/40223>`_, `pr#28611 <https://github.com/ceph/ceph/pull/28611>`_, "Yan, Zheng")
+* cephfs: mds: there is an assertion when calling Beacon::shutdown() (`issue#39214 <http://tracker.ceph.com/issues/39214>`_, `issue#38822 <http://tracker.ceph.com/issues/38822>`_, `pr#27852 <https://github.com/ceph/ceph/pull/27852>`_, huanwen ren)
+* cephfs: mount: key parsing fail when doing a remount (`issue#40164 <http://tracker.ceph.com/issues/40164>`_, `pr#28610 <https://github.com/ceph/ceph/pull/28610>`_, Luis Henriques)
+* cephfs: pybind: added lseek() (`pr#28333 <https://github.com/ceph/ceph/pull/28333>`_, Xiaowei Chu)
+* common/assert: include ceph_abort_msg(arg) arg in log output (`pr#27824 <https://github.com/ceph/ceph/pull/27824>`_, Sage Weil)
+* common/options: annotate some options; enable some runtime updates (`pr#27818 <https://github.com/ceph/ceph/pull/27818>`_, Sage Weil)
+* common/options: update mon_crush_min_required_version=hammer (`pr#27625 <https://github.com/ceph/ceph/pull/27625>`_, Sage Weil)
+* common/util: handle long lines in /proc/cpuinfo (`issue#38296 <http://tracker.ceph.com/issues/38296>`_, `issue#39476 <http://tracker.ceph.com/issues/39476>`_, `pr#28141 <https://github.com/ceph/ceph/pull/28141>`_, Sage Weil)
+* common: Clang requires a default constructor, but it can be empty (`issue#39561 <http://tracker.ceph.com/issues/39561>`_, `issue#39573 <http://tracker.ceph.com/issues/39573>`_, `pr#28131 <https://github.com/ceph/ceph/pull/28131>`_, Willem Jan Withagen)
+* common: fix parse_env nullptr deref (`pr#28382 <https://github.com/ceph/ceph/pull/28382>`_, Patrick Donnelly)
+* common: make cluster_network work (`issue#39671 <http://tracker.ceph.com/issues/39671>`_, `pr#28248 <https://github.com/ceph/ceph/pull/28248>`_, Jianpeng Ma)
+* common: parse ISO 8601 datetime format (`issue#40087 <http://tracker.ceph.com/issues/40087>`_, `pr#28325 <https://github.com/ceph/ceph/pull/28325>`_, Sage Weil)
+* core: Give recovery for inactive PGs a higher priority (`issue#39504 <http://tracker.ceph.com/issues/39504>`_, `issue#38195 <http://tracker.ceph.com/issues/38195>`_, `pr#27854 <https://github.com/ceph/ceph/pull/27854>`_, David Zafman)
+* core: mon,osd: add no{out,down,in,out} flags on CRUSH nodes (`pr#27623 <https://github.com/ceph/ceph/pull/27623>`_, xie xingguo, Sage Weil)
+* core: mon/Elector: format mon_release correctly (`issue#39419 <http://tracker.ceph.com/issues/39419>`_, `pr#27771 <https://github.com/ceph/ceph/pull/27771>`_, Sage Weil)
+* core: mon/Monitor: allow probe if MMonProbe::mon_release == 0 (`issue#38850 <http://tracker.ceph.com/issues/38850>`_, `pr#28262 <https://github.com/ceph/ceph/pull/28262>`_, Sage Weil)
+* core: mon: fix off-by-one rendering progress bar (`pr#28398 <https://github.com/ceph/ceph/pull/28398>`_, Sage Weil)
+* core: mon: use per-pool stats only when all OSDs are reporting (`pr#29032 <https://github.com/ceph/ceph/pull/29032>`_, Sage Weil)
+* core: monitoring: Provide a base set of Prometheus alert manager rules that notify the user about common Ceph error conditions (`issue#39540 <http://tracker.ceph.com/issues/39540>`_, `pr#27998 <https://github.com/ceph/ceph/pull/27998>`_, Jan Fajerski)
+* core: monitoring: update Grafana dashboards (`issue#39652 <http://tracker.ceph.com/issues/39652>`_, `issue#40006 <http://tracker.ceph.com/issues/40006>`_, `issue#39971 <http://tracker.ceph.com/issues/39971>`_, `issue#39932 <http://tracker.ceph.com/issues/39932>`_, `pr#28101 <https://github.com/ceph/ceph/pull/28101>`_, Kiefer Chang, Jan Fajerski)
+* core: osd/OSD.cc: make osd bench description consistent with parameters (`issue#39006 <http://tracker.ceph.com/issues/39006>`_, `issue#39375 <http://tracker.ceph.com/issues/39375>`_, `pr#28035 <https://github.com/ceph/ceph/pull/28035>`_, Neha Ojha)
+* core: osd/OSDMap: Replace get_out_osds with get_out_existing_osds (`issue#39421 <http://tracker.ceph.com/issues/39421>`_, `issue#39154 <http://tracker.ceph.com/issues/39154>`_, `pr#28072 <https://github.com/ceph/ceph/pull/28072>`_, Brad Hubbard)
+* core: osd/PG: discover missing objects when an OSD peers and PG is degraded (`pr#27744 <https://github.com/ceph/ceph/pull/27744>`_, Jonas Jelten)
+* core: osd/PG: do not use approx_missing_objects pre-nautilus (`issue#39512 <http://tracker.ceph.com/issues/39512>`_, `pr#28160 <https://github.com/ceph/ceph/pull/28160>`_, Neha Ojha)
+* core: osd/PG: fix last_complete re-calculation on splitting (`issue#39539 <http://tracker.ceph.com/issues/39539>`_, `issue#26958 <http://tracker.ceph.com/issues/26958>`_, `pr#28219 <https://github.com/ceph/ceph/pull/28219>`_, xie xingguo)
+* core: osd/PG: skip rollforward when !transaction_applied during append_log() (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `issue#38881 <http://tracker.ceph.com/issues/38881>`_, `pr#27654 <https://github.com/ceph/ceph/pull/27654>`_, Neha Ojha)
+* core: osd/PGLog: preserve original_crt to check rollbackability (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `issue#39043 <http://tracker.ceph.com/issues/39043>`_, `pr#27632 <https://github.com/ceph/ceph/pull/27632>`_, Neha Ojha)
+* core: osd: Don't evict after a flush if intersecting scrub range (`issue#38840 <http://tracker.ceph.com/issues/38840>`_, `issue#39519 <http://tracker.ceph.com/issues/39519>`_, `pr#28205 <https://github.com/ceph/ceph/pull/28205>`_, David Zafman')
+* core: osd: Don't include user changeable flag in snaptrim related assert (`issue#39699 <http://tracker.ceph.com/issues/39699>`_, `issue#38124 <http://tracker.ceph.com/issues/38124>`_, `pr#28203 <https://github.com/ceph/ceph/pull/28203>`_, David Zafman')
+* core: osd: FAILED ceph_assert(attrs || !pg_log.get_missing().is_missing(soid) || (it_objects != pg_log.get_log().objects.end() && it_objects->second->op == pg_log_entry_t::LOST_REVERT)) in PrimaryLogPG::get_object_context() (`issue#38931 <http://tracker.ceph.com/issues/38931>`_, `issue#39219 <http://tracker.ceph.com/issues/39219>`_, `issue#38784 <http://tracker.ceph.com/issues/38784>`_, `pr#27839 <https://github.com/ceph/ceph/pull/27839>`_, xie xingguo)
+* core: osd: Include dups in copy_after() and copy_up_to() (`issue#39304 <http://tracker.ceph.com/issues/39304>`_, `pr#28088 <https://github.com/ceph/ceph/pull/28088>`_, David Zafman)
+* core: osd: Increase log level of messages which unnecessarily fill up logs (`pr#27687 <https://github.com/ceph/ceph/pull/27687>`_, David Zafman)
+* core: osd: Output Base64 encoding of CRC header if binary data present (`issue#39738 <http://tracker.ceph.com/issues/39738>`_, `pr#28504 <https://github.com/ceph/ceph/pull/28504>`_, David Zafman)
+* core: osd: Primary won't automatically repair replica on pulling error (`issue#39101 <http://tracker.ceph.com/issues/39101>`_, `issue#39184 <http://tracker.ceph.com/issues/39184>`_, `pr#27711 <https://github.com/ceph/ceph/pull/27711>`_, xie xingguo, David Zafman')
+* core: osd: revamp {noup,nodown,noin,noout} related commands (`pr#28400 <https://github.com/ceph/ceph/pull/28400>`_, xie xingguo)
+* core: osd: shutdown recovery_request_timer earlier (`issue#39205 <http://tracker.ceph.com/issues/39205>`_, `pr#27803 <https://github.com/ceph/ceph/pull/27803>`_, Zengran Zhang)
+* core: osd: take heartbeat_lock when calling heartbeat() (`issue#39514 <http://tracker.ceph.com/issues/39514>`_, `issue#39439 <http://tracker.ceph.com/issues/39439>`_, `pr#28164 <https://github.com/ceph/ceph/pull/28164>`_, Sage Weil)
+* doc: add LAZYIO (`issue#39051 <http://tracker.ceph.com/issues/39051>`_, `issue#38729 <http://tracker.ceph.com/issues/38729>`_, `pr#27899 <https://github.com/ceph/ceph/pull/27899>`_, "Yan, Zheng")
+* doc: add documentation for "fs set min_compat_client" (`issue#39130 <http://tracker.ceph.com/issues/39130>`_, `issue#39176 <http://tracker.ceph.com/issues/39176>`_, `pr#27900 <https://github.com/ceph/ceph/pull/27900>`_, Patrick Donnelly)
+* doc: cleanup HTTP Frontends documentation (`issue#38874 <http://tracker.ceph.com/issues/38874>`_, `pr#27922 <https://github.com/ceph/ceph/pull/27922>`_, Casey Bodley)
+* doc: dashboard documentation changes (`pr#27642 <https://github.com/ceph/ceph/pull/27642>`_, Tatjana Dehler, Lenz Grimmer)
+* doc: orchestrator_cli: Rook orch supports mon update (`issue#39169 <http://tracker.ceph.com/issues/39169>`_, `issue#39137 <http://tracker.ceph.com/issues/39137>`_, `pr#27488 <https://github.com/ceph/ceph/pull/27488>`_, Sebastian Wagner)
+* doc: osd_internals/async_recovery: update cost calculation (`pr#28046 <https://github.com/ceph/ceph/pull/28046>`_, Neha Ojha)
+* doc: rados/operations/devices: document device prediction (`pr#27752 <https://github.com/ceph/ceph/pull/27752>`_, Sage Weil)
+* mgr/ActivePyModules: handle_command - fix broken lock (`issue#39235 <http://tracker.ceph.com/issues/39235>`_, `issue#39308 <http://tracker.ceph.com/issues/39308>`_, `pr#27939 <https://github.com/ceph/ceph/pull/27939>`_, xie xingguo)
+* mgr/BaseMgrModule: run MonCommandCompletion on the finisher (`issue#39397 <http://tracker.ceph.com/issues/39397>`_, `issue#39335 <http://tracker.ceph.com/issues/39335>`_, `pr#27699 <https://github.com/ceph/ceph/pull/27699>`_, Sage Weil)
+* mgr/ansible: Host ls implementation (`issue#39559 <http://tracker.ceph.com/issues/39559>`_, `pr#27919 <https://github.com/ceph/ceph/pull/27919>`_, Juan Miguel Olmo Mart\xc3\xadnez)
+* mgr/balancer: various compat weight-set fixes (`pr#28279 <https://github.com/ceph/ceph/pull/28279>`_, xie xingguo)
+* mgr/dashboard: Add custom dialogue for configuring PG scrub parameters (`issue#40059 <http://tracker.ceph.com/issues/40059>`_, `pr#28555 <https://github.com/ceph/ceph/pull/28555>`_, Tatjana Dehler)
+* mgr/dashboard: Admin resource not honored (`issue#39338 <http://tracker.ceph.com/issues/39338>`_, `issue#39467 <http://tracker.ceph.com/issues/39467>`_, `pr#27868 <https://github.com/ceph/ceph/pull/27868>`_, Wido den Hollander)
+* mgr/dashboard: Angular is creating multiple instances of the same service (`issue#39996 <http://tracker.ceph.com/issues/39996>`_, `issue#40075 <http://tracker.ceph.com/issues/40075>`_, `pr#28312 <https://github.com/ceph/ceph/pull/28312>`_, Tiago Melo)
+* mgr/dashboard: Avoid merge conflicts in messages.xlf by auto-generating it at build time? (`issue#39658 <http://tracker.ceph.com/issues/39658>`_, `pr#28178 <https://github.com/ceph/ceph/pull/28178>`_, Sebastian Krah)
+* mgr/dashboard: Display correct dialog title (`pr#28189 <https://github.com/ceph/ceph/pull/28189>`_, Volker Theile)
+* mgr/dashboard: Error creating NFS client without squash (`issue#40074 <http://tracker.ceph.com/issues/40074>`_, `pr#28311 <https://github.com/ceph/ceph/pull/28311>`_, Tiago Melo)
+* mgr/dashboard: KV-table transforms dates through pipe (`issue#39558 <http://tracker.ceph.com/issues/39558>`_, `pr#28021 <https://github.com/ceph/ceph/pull/28021>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Localization for date picker module (`issue#39371 <http://tracker.ceph.com/issues/39371>`_, `pr#27673 <https://github.com/ceph/ceph/pull/27673>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Manager should complain about wrong dashboard certificate (`issue#39346 <http://tracker.ceph.com/issues/39346>`_, `pr#27742 <https://github.com/ceph/ceph/pull/27742>`_, Volker Theile)
+* mgr/dashboard: NFS clients information is not displayed in the details view (`issue#40057 <http://tracker.ceph.com/issues/40057>`_, `pr#28318 <https://github.com/ceph/ceph/pull/28318>`_, Tiago Melo)
+* mgr/dashboard: NFS export creation: Add more info to the validation message of the field Pseudo (`issue#39975 <http://tracker.ceph.com/issues/39975>`_, `issue#39327 <http://tracker.ceph.com/issues/39327>`_, `pr#28320 <https://github.com/ceph/ceph/pull/28320>`_, Tiago Melo)
+* mgr/dashboard: Only one root node is shown in the crush map viewer (`issue#39647 <http://tracker.ceph.com/issues/39647>`_, `issue#40077 <http://tracker.ceph.com/issues/40077>`_, `pr#28316 <https://github.com/ceph/ceph/pull/28316>`_, Tiago Melo)
+* mgr/dashboard: Push Grafana dashboards on startup (`pr#28635 <https://github.com/ceph/ceph/pull/28635>`_, Zack Cerza)
+* mgr/dashboard: Queue notifications as default (`issue#39560 <http://tracker.ceph.com/issues/39560>`_, `pr#28022 <https://github.com/ceph/ceph/pull/28022>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: RBD snapshot name suggestion with local time suffix (`issue#39534 <http://tracker.ceph.com/issues/39534>`_, `pr#27890 <https://github.com/ceph/ceph/pull/27890>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Reduce the number of renders on the tables (`issue#39944 <http://tracker.ceph.com/issues/39944>`_, `issue#40076 <http://tracker.ceph.com/issues/40076>`_, `pr#28315 <https://github.com/ceph/ceph/pull/28315>`_, Tiago Melo)
+* mgr/dashboard: Some validations are not updated and prevent the submission of a form (`issue#40030 <http://tracker.ceph.com/issues/40030>`_, `pr#28319 <https://github.com/ceph/ceph/pull/28319>`_, Tiago Melo)
+* mgr/dashboard: Unable to see tcmu-runner perf counters (`issue#39988 <http://tracker.ceph.com/issues/39988>`_, `pr#28191 <https://github.com/ceph/ceph/pull/28191>`_, Ricardo Marques)
+* mgr/dashboard: Unify the look of dashboard charts (`issue#39384 <http://tracker.ceph.com/issues/39384>`_, `issue#39961 <http://tracker.ceph.com/issues/39961>`_, `pr#28175 <https://github.com/ceph/ceph/pull/28175>`_, Tiago Melo)
+* mgr/dashboard: Validate if any client belongs to more than one group (`issue#39036 <http://tracker.ceph.com/issues/39036>`_, `issue#39454 <http://tracker.ceph.com/issues/39454>`_, `pr#27760 <https://github.com/ceph/ceph/pull/27760>`_, Tiago Melo)
+* mgr/dashboard: code documentation (`issue#39345 <http://tracker.ceph.com/issues/39345>`_, `issue#36243 <http://tracker.ceph.com/issues/36243>`_, `pr#27746 <https://github.com/ceph/ceph/pull/27746>`_, Ernesto Puerta)
+* mgr/dashboard: iSCSI GET requests should not be logged (`pr#28024 <https://github.com/ceph/ceph/pull/28024>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI form does not support IPv6 (`pr#28026 <https://github.com/ceph/ceph/pull/28026>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI form is showing a warning (`issue#39452 <http://tracker.ceph.com/issues/39452>`_, `issue#39324 <http://tracker.ceph.com/issues/39324>`_, `pr#27758 <https://github.com/ceph/ceph/pull/27758>`_, Tiago Melo)
+* mgr/dashboard: iSCSI should allow exporting an RBD image with Journaling enabled (`pr#28011 <https://github.com/ceph/ceph/pull/28011>`_, Ricardo Marques)
+* mgr/dashboard: inconsistent result when editing a RBD image's features (`issue#39993 <http://tracker.ceph.com/issues/39993>`_, `issue#39933 <http://tracker.ceph.com/issues/39933>`_, `pr#28218 <https://github.com/ceph/ceph/pull/28218>`_, Kiefer Chang')
+* mgr/dashboard: incorrect help message for minimum blob size (`issue#39624 <http://tracker.ceph.com/issues/39624>`_, `issue#39664 <http://tracker.ceph.com/issues/39664>`_, `pr#28062 <https://github.com/ceph/ceph/pull/28062>`_, Kiefer Chang)
+* mgr/dashboard: local variable 'cluster_id' referenced before assignment error when trying to list NFS Ganesha daemons (`issue#40031 <http://tracker.ceph.com/issues/40031>`_, `pr#28261 <https://github.com/ceph/ceph/pull/28261>`_, Nur Faizin')
+* mgr/dashboard: make auth token work with UTC times only (`issue#39524 <http://tracker.ceph.com/issues/39524>`_, `issue#39300 <http://tracker.ceph.com/issues/39300>`_, `pr#27942 <https://github.com/ceph/ceph/pull/27942>`_, Ricardo Dias)
+* mgr/dashboard: openssl exception when verifying certificates of HTTPS requests (`issue#39962 <http://tracker.ceph.com/issues/39962>`_, `issue#39628 <http://tracker.ceph.com/issues/39628>`_, `pr#28163 <https://github.com/ceph/ceph/pull/28163>`_, Ricardo Dias)
+* mgr/dashboard: orchestrator mgr modules assert failure on iscsi service request (`issue#40037 <http://tracker.ceph.com/issues/40037>`_, `pr#28552 <https://github.com/ceph/ceph/pull/28552>`_, Sebastian Wagner)
+* mgr/dashboard: show degraded/misplaced/unfound objects (`pr#28584 <https://github.com/ceph/ceph/pull/28584>`_, Alfonso Mart\xc3\xadnez)
+* mgr/orchestrator: Remove "(add|test|remove)_stateful_service_rule (`issue#38808 <http://tracker.ceph.com/issues/38808>`_, `pr#27043 <https://github.com/ceph/ceph/pull/27043>`_, Sebastian Wagner)
+* mgr/orchestrator: add progress events to all orchestrators (`pr#28040 <https://github.com/ceph/ceph/pull/28040>`_, Sebastian Wagner)
+* mgr/progress: behave if pgs disappear (due to a racing pg merge) (`issue#38157 <http://tracker.ceph.com/issues/38157>`_, `issue#39344 <http://tracker.ceph.com/issues/39344>`_, `pr#27608 <https://github.com/ceph/ceph/pull/27608>`_, Sage Weil)
+* mgr/prometheus: replace whitespaces in metrics' names (`pr#27886 <https://github.com/ceph/ceph/pull/27886>`_, Alfonso Mart\xc3\xadnez')
+* mgr/rook: Added missing rgw daemons in service ls (`issue#39171 <http://tracker.ceph.com/issues/39171>`_, `issue#39312 <http://tracker.ceph.com/issues/39312>`_, `pr#27864 <https://github.com/ceph/ceph/pull/27864>`_, Sebastian Wagner)
+* mgr/rook: Fix RGW creation (`issue#39158 <http://tracker.ceph.com/issues/39158>`_, `issue#39313 <http://tracker.ceph.com/issues/39313>`_, `pr#27863 <https://github.com/ceph/ceph/pull/27863>`_, Sebastian Wagner)
+* mgr/rook: Remove support for Rook older than v0.9 (`issue#39356 <http://tracker.ceph.com/issues/39356>`_, `issue#39278 <http://tracker.ceph.com/issues/39278>`_, `pr#27862 <https://github.com/ceph/ceph/pull/27862>`_, Sebastian Wagner)
+* mgr/test_orchestrator: AttributeError: 'TestWriteCompletion' object has no attribute 'id (`issue#39536 <http://tracker.ceph.com/issues/39536>`_, `pr#27920 <https://github.com/ceph/ceph/pull/27920>`_, Sebastian Wagner')
+* mgr/volumes: FS subvolumes enhancements (`issue#40429 <http://tracker.ceph.com/issues/40429>`_, `pr#28767 <https://github.com/ceph/ceph/pull/28767>`_, Ramana Raja)
+* mgr/volumes: add CephFS subvolumes library (`issue#39750 <http://tracker.ceph.com/issues/39750>`_, `issue#40152 <http://tracker.ceph.com/issues/40152>`_, `issue#39949 <http://tracker.ceph.com/issues/39949>`_, `issue#40014 <http://tracker.ceph.com/issues/40014>`_, `issue#39610 <http://tracker.ceph.com/issues/39610>`_, `pr#28429 <https://github.com/ceph/ceph/pull/28429>`_, Sage Weil, Venky Shankar, Ramana Raja, Rishabh Dave)
+* mgr/volumes: refactor volume module (`issue#40378 <http://tracker.ceph.com/issues/40378>`_, `issue#39969 <http://tracker.ceph.com/issues/39969>`_, `pr#28595 <https://github.com/ceph/ceph/pull/28595>`_, Venky Shankar)
+* mgr: Update the restful module in nautilus (`pr#28291 <https://github.com/ceph/ceph/pull/28291>`_, Kefu Chai, Boris Ranto)
+* mgr: deadlock (`issue#39040 <http://tracker.ceph.com/issues/39040>`_, `issue#39425 <http://tracker.ceph.com/issues/39425>`_, `pr#28098 <https://github.com/ceph/ceph/pull/28098>`_, xie xingguo)
+* mgr: fix pgp_num adjustments (`issue#38626 <http://tracker.ceph.com/issues/38626>`_, `pr#27876 <https://github.com/ceph/ceph/pull/27876>`_, Sage Weil, Marius Schiffer)
+* mgr: log an error if we can't find any modules to load (`issue#40090 <http://tracker.ceph.com/issues/40090>`_, `pr#28347 <https://github.com/ceph/ceph/pull/28347>`_, Tim Serong')
+* monitoring: pybind/mgr: fix format for rbd-mirror prometheus metrics (`pr#28485 <https://github.com/ceph/ceph/pull/28485>`_, Mykola Golub)
+* msg/async: connection race + winner fault can leave connection stuck at replacing foreve (`issue#39241 <http://tracker.ceph.com/issues/39241>`_, `issue#37499 <http://tracker.ceph.com/issues/37499>`_, `issue#39448 <http://tracker.ceph.com/issues/39448>`_, `issue#38493 <http://tracker.ceph.com/issues/38493>`_, `pr#27915 <https://github.com/ceph/ceph/pull/27915>`_, Jason Dillaman, xie xingguo)
+* msg/async/ProtocolV[12]: add ms_learn_addr_from_peer (`pr#28589 <https://github.com/ceph/ceph/pull/28589>`_, Sage Weil)
+* msg: output peer address when detecting bad CRCs (`issue#39367 <http://tracker.ceph.com/issues/39367>`_, `pr#27857 <https://github.com/ceph/ceph/pull/27857>`_, Greg Farnum)
+* pybind: Add 'RBD_FEATURE_MIGRATING' to rbd.pyx (`issue#39609 <http://tracker.ceph.com/issues/39609>`_, `issue#39736 <http://tracker.ceph.com/issues/39736>`_, `pr#28482 <https://github.com/ceph/ceph/pull/28482>`_, Ricardo Marques')
+* pybind: Rados.get_fsid() returning bytes in python3 (`issue#40192 <http://tracker.ceph.com/issues/40192>`_, `issue#38381 <http://tracker.ceph.com/issues/38381>`_, `pr#28476 <https://github.com/ceph/ceph/pull/28476>`_, Jason Dillaman)
+* rbd: krbd: fix rbd map hang due to udev return subsystem unordered (`issue#39089 <http://tracker.ceph.com/issues/39089>`_, `issue#39315 <http://tracker.ceph.com/issues/39315>`_, `pr#28019 <https://github.com/ceph/ceph/pull/28019>`_, Zhi Zhang)
+* rbd: librbd: async open/close should free ImageCtx before issuing callback (`issue#39428 <http://tracker.ceph.com/issues/39428>`_, `issue#39031 <http://tracker.ceph.com/issues/39031>`_, `pr#28121 <https://github.com/ceph/ceph/pull/28121>`_, Jason Dillaman)
+* rbd: librbd: avoid dereferencing an empty container during deep-copy (`issue#40368 <http://tracker.ceph.com/issues/40368>`_, `issue#40379 <http://tracker.ceph.com/issues/40379>`_, `pr#28577 <https://github.com/ceph/ceph/pull/28577>`_, Jason Dillaman)
+* rbd: librbd: do not allow to deep copy migrating image (`issue#39224 <http://tracker.ceph.com/issues/39224>`_, `pr#27882 <https://github.com/ceph/ceph/pull/27882>`_, Mykola Golub)
+* rbd: librbd: fix issues with object-map/fast-diff feature interlock (`issue#39946 <http://tracker.ceph.com/issues/39946>`_, `issue#39521 <http://tracker.ceph.com/issues/39521>`_, `pr#28127 <https://github.com/ceph/ceph/pull/28127>`_, Jason Dillaman)
+* rbd: librbd: fixed several race conditions related to copyup (`issue#39195 <http://tracker.ceph.com/issues/39195>`_, `issue#39021 <http://tracker.ceph.com/issues/39021>`_, `pr#28132 <https://github.com/ceph/ceph/pull/28132>`_, Jason Dillaman)
+* rbd: librbd: make flush be queued by QOS throttler (`issue#38869 <http://tracker.ceph.com/issues/38869>`_, `pr#28120 <https://github.com/ceph/ceph/pull/28120>`_, Mykola Golub)
+* rbd: librbd: re-add support for nautilus clients talking to jewel clusters (`issue#39450 <http://tracker.ceph.com/issues/39450>`_, `pr#27936 <https://github.com/ceph/ceph/pull/27936>`_, Jason Dillaman)
+* rbd: librbd: support EC data pool images sparsify (`issue#39226 <http://tracker.ceph.com/issues/39226>`_, `pr#27903 <https://github.com/ceph/ceph/pull/27903>`_, Mykola Golub)
+* rbd: rbd-mirror: clear out bufferlist prior to listing mirror images (`issue#39462 <http://tracker.ceph.com/issues/39462>`_, `issue#39407 <http://tracker.ceph.com/issues/39407>`_, `pr#28122 <https://github.com/ceph/ceph/pull/28122>`_, Jason Dillaman)
+* rbd: rbd-mirror: image replayer should periodically flush IO and commit positions (`issue#39257 <http://tracker.ceph.com/issues/39257>`_, `issue#39288 <http://tracker.ceph.com/issues/39288>`_, `pr#27937 <https://github.com/ceph/ceph/pull/27937>`_, Jason Dillaman)
+* rgw: Evaluating bucket policies also while reading permissions for an\xe2\x80\xa6 (`issue#38638 <http://tracker.ceph.com/issues/38638>`_, `issue#39273 <http://tracker.ceph.com/issues/39273>`_, `pr#27918 <https://github.com/ceph/ceph/pull/27918>`_, Pritha Srivastava)
+* rgw: admin: handle delete_at attr in object stat output (`pr#27827 <https://github.com/ceph/ceph/pull/27827>`_, Abhishek Lekshmanan)
+* rgw: beast: multiple v4 and v6 endpoints with the same port will cause failure (`issue#39746 <http://tracker.ceph.com/issues/39746>`_, `issue#39038 <http://tracker.ceph.com/issues/39038>`_, `pr#28541 <https://github.com/ceph/ceph/pull/28541>`_, Abhishek Lekshmanan)
+* rgw: beast: set a default port for endpoints (`issue#39048 <http://tracker.ceph.com/issues/39048>`_, `issue#39000 <http://tracker.ceph.com/issues/39000>`_, `pr#27660 <https://github.com/ceph/ceph/pull/27660>`_, Abhishek Lekshmanan)
+* rgw: bucket stats report mtime in UTC (`pr#27826 <https://github.com/ceph/ceph/pull/27826>`_, Alfonso Mart\xc3\xadnez, Casey Bodley)
+* rgw: clean up some logging (`issue#39503 <http://tracker.ceph.com/issues/39503>`_, `pr#27953 <https://github.com/ceph/ceph/pull/27953>`_, J. Eric Ivancich)
+* rgw: cloud sync module fails to sync multipart objects (`issue#39684 <http://tracker.ceph.com/issues/39684>`_, `pr#28064 <https://github.com/ceph/ceph/pull/28064>`_, Abhishek Lekshmanan)
+* rgw: cloud sync module logs attrs in the log (`issue#39574 <http://tracker.ceph.com/issues/39574>`_, `pr#27954 <https://github.com/ceph/ceph/pull/27954>`_, Nathan Cutler)
+* rgw: crypto: throw DigestException from Digest and HMAC (`issue#39676 <http://tracker.ceph.com/issues/39676>`_, `issue#39456 <http://tracker.ceph.com/issues/39456>`_, `pr#28309 <https://github.com/ceph/ceph/pull/28309>`_, Matt Benjamin)
+* rgw: document CreateBucketConfiguration for s3 PUT Bucket request (`issue#39597 <http://tracker.ceph.com/issues/39597>`_, `issue#39601 <http://tracker.ceph.com/issues/39601>`_, `pr#28512 <https://github.com/ceph/ceph/pull/28512>`_, Casey Bodley)
+* rgw: fix Multisite sync corruption (`pr#28383 <https://github.com/ceph/ceph/pull/28383>`_, Tianshan Qu, Casey Bodley, Xiaoxi CHEN)
+* rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR (`issue#39984 <http://tracker.ceph.com/issues/39984>`_, `issue#40148 <http://tracker.ceph.com/issues/40148>`_, `pr#28410 <https://github.com/ceph/ceph/pull/28410>`_, Casey Bodley, Tianshan Qu)
+* rgw: fix default_placement containing "/" when storage_class is standard (`issue#39745 <http://tracker.ceph.com/issues/39745>`_, `issue#39380 <http://tracker.ceph.com/issues/39380>`_, `pr#28538 <https://github.com/ceph/ceph/pull/28538>`_, mkogan1)
+* rgw: inefficient unordered bucket listing (`issue#39410 <http://tracker.ceph.com/issues/39410>`_, `issue#39393 <http://tracker.ceph.com/issues/39393>`_, `pr#27924 <https://github.com/ceph/ceph/pull/27924>`_, Casey Bodley)
+* rgw: librgw: unexpected crash when creating bucket (`issue#39575 <http://tracker.ceph.com/issues/39575>`_, `pr#27955 <https://github.com/ceph/ceph/pull/27955>`_, Tao CHEN)
+* rgw: limit entries in remove_olh_pending_entries() (`issue#39178 <http://tracker.ceph.com/issues/39178>`_, `issue#39118 <http://tracker.ceph.com/issues/39118>`_, `pr#27664 <https://github.com/ceph/ceph/pull/27664>`_, Casey Bodley)
+* rgw: list bucket with start marker and delimiter will miss next object with char '0' (`issue#40762 <http://tracker.ceph.com/issues/40762>`_, `issue#39989 <http://tracker.ceph.com/issues/39989>`_, `pr#29022 <https://github.com/ceph/ceph/pull/29022>`_, Tianshan Qu)
+* rgw: multisite log trimming only checks peers that sync from us (`issue#39283 <http://tracker.ceph.com/issues/39283>`_, `pr#27814 <https://github.com/ceph/ceph/pull/27814>`_, Casey Bodley)
+* rgw: multisite: add perf counters to data sync (`issue#38549 <http://tracker.ceph.com/issues/38549>`_, `issue#38918 <http://tracker.ceph.com/issues/38918>`_, `pr#27921 <https://github.com/ceph/ceph/pull/27921>`_, Abhishek Lekshmanan, Casey Bodley)
+* rgw: multisite: mismatch of bucket creation times from List Buckets (`issue#39635 <http://tracker.ceph.com/issues/39635>`_, `issue#39735 <http://tracker.ceph.com/issues/39735>`_, `pr#28444 <https://github.com/ceph/ceph/pull/28444>`_, Casey Bodley)
+* rgw: multisite: period pusher gets 403 Forbidden against other zonegroups (`issue#39287 <http://tracker.ceph.com/issues/39287>`_, `issue#39414 <http://tracker.ceph.com/issues/39414>`_, `pr#27952 <https://github.com/ceph/ceph/pull/27952>`_, Casey Bodley)
+* rgw: race condition between resharding and ops waiting on resharding (`issue#39202 <http://tracker.ceph.com/issues/39202>`_, `pr#27800 <https://github.com/ceph/ceph/pull/27800>`_, J. Eric Ivancich)
+* rgw: radosgw-admin: add tenant argument to reshard cancel (`issue#39018 <http://tracker.ceph.com/issues/39018>`_, `pr#27630 <https://github.com/ceph/ceph/pull/27630>`_, Abhishek Lekshmanan)
+* rgw: rgw_file: save etag and acl info in setattr (`issue#39228 <http://tracker.ceph.com/issues/39228>`_, `pr#27904 <https://github.com/ceph/ceph/pull/27904>`_, Tao Chen)
+* rgw: swift object expiry fails when a bucket reshards (`issue#39740 <http://tracker.ceph.com/issues/39740>`_, `pr#28537 <https://github.com/ceph/ceph/pull/28537>`_, Abhishek Lekshmanan)
+* rgw: unittest_rgw_dmclock_scheduler does not need Boost_LIBRARIES (`issue#39577 <http://tracker.ceph.com/issues/39577>`_, `pr#27944 <https://github.com/ceph/ceph/pull/27944>`_, Willem Jan Withagen)
+* rgw: update resharding documentation (`issue#39046 <http://tracker.ceph.com/issues/39046>`_, `pr#27923 <https://github.com/ceph/ceph/pull/27923>`_, J. Eric Ivancich)
+* tests: added `bluestore_warn_on_legacy_statfs: false` setting (`issue#40467 <http://tracker.ceph.com/issues/40467>`_, `pr#28723 <https://github.com/ceph/ceph/pull/28723>`_, Yuri Weinstein)
+* tests: added ragweed coverage to stress-split\\* upgrade suites (`issue#40452 <http://tracker.ceph.com/issues/40452>`_, `issue#40467 <http://tracker.ceph.com/issues/40467>`_, `pr#28661 <https://github.com/ceph/ceph/pull/28661>`_, Yuri Weinstein)
+* tests: added v14.2.1 (`issue#40181 <http://tracker.ceph.com/issues/40181>`_, `pr#28416 <https://github.com/ceph/ceph/pull/28416>`_, Yuri Weinstein)
+* tests: cannot schedule kcephfs/multimds (`issue#40116 <http://tracker.ceph.com/issues/40116>`_, `pr#28369 <https://github.com/ceph/ceph/pull/28369>`_, Patrick Donnelly)
+* tests: centos 7.6 etc (`pr#27439 <https://github.com/ceph/ceph/pull/27439>`_, Sage Weil)
+* tests: ceph-ansible: ceph-ansible requires ansible 2.8 (`issue#40602 <http://tracker.ceph.com/issues/40602>`_, `issue#40669 <http://tracker.ceph.com/issues/40669>`_, `pr#28871 <https://github.com/ceph/ceph/pull/28871>`_, Brad Hubbard)
+* tests: ceph-ansible: cephfs_pools variable pgs should be pg_num (`issue#40670 <http://tracker.ceph.com/issues/40670>`_, `issue#40605 <http://tracker.ceph.com/issues/40605>`_, `pr#28872 <https://github.com/ceph/ceph/pull/28872>`_, Brad Hubbard)
+* tests: cephfs-shell: teuthology tests (`issue#39935 <http://tracker.ceph.com/issues/39935>`_, `issue#39526 <http://tracker.ceph.com/issues/39526>`_, `pr#28614 <https://github.com/ceph/ceph/pull/28614>`_, Milind Changire)
+* tests: cephfs: TestMisc.test_evict_client fails (`issue#40220 <http://tracker.ceph.com/issues/40220>`_, `pr#28613 <https://github.com/ceph/ceph/pull/28613>`_, "Yan, Zheng")
+* tests: cleaned up supported distro for nautilus (`pr#28065 <https://github.com/ceph/ceph/pull/28065>`_, Yuri Weinstein)
+* tests: ignore legacy bluestore stats errors (`issue#40374 <http://tracker.ceph.com/issues/40374>`_, `pr#28563 <https://github.com/ceph/ceph/pull/28563>`_, Patrick Donnelly)
+* tests: librbd: drop 'ceph_test_librbd_api' target (`issue#39423 <http://tracker.ceph.com/issues/39423>`_, `issue#39072 <http://tracker.ceph.com/issues/39072>`_, `pr#28091 <https://github.com/ceph/ceph/pull/28091>`_, Jason Dillaman')
+* tests: mgr: tox failures when running make check (`issue#39323 <http://tracker.ceph.com/issues/39323>`_, `issue#39530 <http://tracker.ceph.com/issues/39530>`_, `pr#27884 <https://github.com/ceph/ceph/pull/27884>`_, Nathan Cutler)
+* tests: pass --ssh-config to pytest to resolve hosts when connecting (`pr#28923 <https://github.com/ceph/ceph/pull/28923>`_, Alfredo Deza)
+* tests: rbd: qemu-iotests tests fail under latest Ubuntu kernel (`issue#39541 <http://tracker.ceph.com/issues/39541>`_, `issue#24668 <http://tracker.ceph.com/issues/24668>`_, `pr#27988 <https://github.com/ceph/ceph/pull/27988>`_, Jason Dillaman)
+* tests: removed `1node` and `systemd` tests as ceph-deploy is not a\xe2\x80\xa6 (`pr#28458 <https://github.com/ceph/ceph/pull/28458>`_, Yuri Weinstein)
+* tests: rgw: fix race in test_rgw_reshard_wait and test_rgw_reshard_wait uses same clock for timing (`issue#39479 <http://tracker.ceph.com/issues/39479>`_, `pr#27779 <https://github.com/ceph/ceph/pull/27779>`_, Casey Bodley)
+* tests: rgw: fix swift warning message (`issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28698 <https://github.com/ceph/ceph/pull/28698>`_, Casey Bodley)
+* tests: rgw: more fixes for swift task (`issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28922 <https://github.com/ceph/ceph/pull/28922>`_, Casey Bodley)
+* tests: rgw: skip swift tests on rhel 7.6+ (`issue#40402 <http://tracker.ceph.com/issues/40402>`_, `issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28604 <https://github.com/ceph/ceph/pull/28604>`_, Casey Bodley)
+* tests: stop testing simple messenger in fs qa (`issue#40373 <http://tracker.ceph.com/issues/40373>`_, `pr#28562 <https://github.com/ceph/ceph/pull/28562>`_, Patrick Donnelly)
+* tests: tasks/rbd_fio: fixed missing delimiter between 'cd' and 'configure (`issue#39590 <http://tracker.ceph.com/issues/39590>`_, `pr#27989 <https://github.com/ceph/ceph/pull/27989>`_, Jason Dillaman')
+* tests: test_sessionmap assumes simple messenger (`issue#39430 <http://tracker.ceph.com/issues/39430>`_, `pr#27772 <https://github.com/ceph/ceph/pull/27772>`_, Patrick Donnelly)
+* tests: use curl in wait_for_radosgw() in util/rgw.py (`issue#40346 <http://tracker.ceph.com/issues/40346>`_, `pr#28598 <https://github.com/ceph/ceph/pull/28598>`_, Ali Maredia)
+* tests: workunits/rbd: use https protocol for devstack git operations (`issue#39656 <http://tracker.ceph.com/issues/39656>`_, `issue#39729 <http://tracker.ceph.com/issues/39729>`_, `pr#28128 <https://github.com/ceph/ceph/pull/28128>`_, Jason Dillaman)
+* tests: workunits/rbd: wait for rbd-nbd unmap to complete (`issue#39675 <http://tracker.ceph.com/issues/39675>`_, `issue#39598 <http://tracker.ceph.com/issues/39598>`_, `pr#28273 <https://github.com/ceph/ceph/pull/28273>`_, Jason Dillaman)
+
+
+v14.2.1 Nautilus
+================
+
+This is the first bug fix release of Ceph Nautilus release series. We recommend
+all nautilus users upgrade to this release. For upgrading from older releases of
+ceph, general guidelines for upgrade to nautilus must be followed
+:ref:`nautilus-old-upgrade`.
+
+Notable Changes
+---------------
+
+* Ceph now packages python bindings for python3.6 instead of
+  python3.4, because EPEL7 recently switched from python3.4 to
+  python3.6 as the native python3. see the `announcement <https://lists.fedoraproject.org/archives/list/epel-announce@lists.fedoraproject.org/message/EGUMKAIMPK2UD5VSHXM53BH2MBDGDWMO/>`_
+  for more details on the background of this change.
+
+Known Issues
+------------
+
+* Nautilus-based librbd clients cannot open images stored on pre-Luminous
+  clusters
+
+Changelog
+---------
+* bluestore: ceph-bluestore-tool: bluefs-bdev-expand cmd might assert if no WAL is configured (`issue#39253 <http://tracker.ceph.com/issues/39253>`_, `pr#27523 <https://github.com/ceph/ceph/pull/27523>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix bitmap allocator issues (`pr#27139 <https://github.com/ceph/ceph/pull/27139>`_, Igor Fedotov)
+* build/ops,rgw: rgw: build async scheduler only when beast is built (`pr#27191 <https://github.com/ceph/ceph/pull/27191>`_, Abhishek Lekshmanan)
+* build/ops: build/ops: Running ceph under Pacemaker control not supported by SUSE Linux Enterprise (`issue#38862 <http://tracker.ceph.com/issues/38862>`_, `pr#27127 <https://github.com/ceph/ceph/pull/27127>`_, Nathan Cutler)
+* build/ops: build/ops: ceph-mgr-diskprediction-local requires numpy and scipy on SUSE, but these packages do not exist on SUSE (`issue#38863 <http://tracker.ceph.com/issues/38863>`_, `pr#27125 <https://github.com/ceph/ceph/pull/27125>`_, Nathan Cutler)
+* build/ops: cmake/FindRocksDB: fix IMPORTED_LOCATION for ROCKSDB_LIBRARIES (`issue#38993 <http://tracker.ceph.com/issues/38993>`_, `pr#27601 <https://github.com/ceph/ceph/pull/27601>`_, dudengke)
+* build/ops: cmake: revert librados_tp.so version from 3 to 2 (`issue#39291 <http://tracker.ceph.com/issues/39291>`_, `issue#39293 <http://tracker.ceph.com/issues/39293>`_, `pr#27597 <https://github.com/ceph/ceph/pull/27597>`_, Nathan Cutler)
+* build/ops: qa,rpm,cmake: switch over to python3.6 (`issue#39236 <http://tracker.ceph.com/issues/39236>`_, `issue#39164 <http://tracker.ceph.com/issues/39164>`_, `pr#27505 <https://github.com/ceph/ceph/pull/27505>`_, Boris Ranto, Kefu Chai)
+* cephfs: fs: we lack a feature bit for nautilus (`issue#39078 <http://tracker.ceph.com/issues/39078>`_, `issue#39187 <http://tracker.ceph.com/issues/39187>`_, `pr#27497 <https://github.com/ceph/ceph/pull/27497>`_, Patrick Donnelly)
+* cephfs: ls -S command produces AttributeError: 'str' object has no attribute 'decode' (`pr#27531 <https://github.com/ceph/ceph/pull/27531>`_, Varsha Rao)
+* cephfs: mds|kclient: MDS_CLIENT_LATE_RELEASE warning caused by inline bug on RHEL 7.5 (`issue#39225 <http://tracker.ceph.com/issues/39225>`_, `pr#27500 <https://github.com/ceph/ceph/pull/27500>`_, "Yan, Zheng")
+* common,core: crush: various fixes for weight-sets, the osd_crush_update_weight_set option, and tests (`pr#27119 <https://github.com/ceph/ceph/pull/27119>`_, Sage Weil)
+* common/blkdev: get_device_id: behave if model is lvm and id_model_enc isn't there (`pr#27158 <https://github.com/ceph/ceph/pull/27158>`_, Sage Weil)
+* common/config: parse --default-$option as a default value (`pr#27217 <https://github.com/ceph/ceph/pull/27217>`_, Sage Weil)
+* core,mgr: mgr: autoscale down can lead to max_pg_per_osd limit (`issue#39271 <http://tracker.ceph.com/issues/39271>`_, `issue#38786 <http://tracker.ceph.com/issues/38786>`_, `pr#27547 <https://github.com/ceph/ceph/pull/27547>`_, Sage Weil)
+* core,mon: mon/Monitor.cc: print min_mon_release correctly (`pr#27168 <https://github.com/ceph/ceph/pull/27168>`_, Neha Ojha)
+* core,tests: tests: osd-markdown.sh can fail with CLI_DUP_COMMAND=1 (`issue#38359 <http://tracker.ceph.com/issues/38359>`_, `issue#39275 <http://tracker.ceph.com/issues/39275>`_, `pr#27550 <https://github.com/ceph/ceph/pull/27550>`_, Sage Weil)
+* core: Improvements to auto repair (`issue#38616 <http://tracker.ceph.com/issues/38616>`_, `pr#27220 <https://github.com/ceph/ceph/pull/27220>`_, xie xingguo, David Zafman)
+* core: Rook: Fix creation of Bluestore OSDs (`issue#39167 <http://tracker.ceph.com/issues/39167>`_, `issue#39062 <http://tracker.ceph.com/issues/39062>`_, `pr#27486 <https://github.com/ceph/ceph/pull/27486>`_, Sebastian Wagner)
+* core: ceph-objectstore-tool: rename dump-import to dump-export (`issue#39325 <http://tracker.ceph.com/issues/39325>`_, `issue#39284 <http://tracker.ceph.com/issues/39284>`_, `pr#27610 <https://github.com/ceph/ceph/pull/27610>`_, David Zafman)
+* core: common/blkdev: handle devices with ID_MODEL as "LVM PV ..." but valid ID_MODEL_ENC (`pr#27096 <https://github.com/ceph/ceph/pull/27096>`_, Sage Weil)
+* core: common: fix deferred log starting (`pr#27388 <https://github.com/ceph/ceph/pull/27388>`_, Sage Weil, Jason Dillaman)
+* core: crush/CrushCompiler: Fix __replacement_assert (`issue#39174 <http://tracker.ceph.com/issues/39174>`_, `pr#27620 <https://github.com/ceph/ceph/pull/27620>`_, Brad Hubbard)
+* core: global: explicitly call out EIO events in crash dumps (`pr#27440 <https://github.com/ceph/ceph/pull/27440>`_, Sage Weil)
+* core: log: log_to_file + --default-\* + fixes and improvements (`pr#27278 <https://github.com/ceph/ceph/pull/27278>`_, Sage Weil)
+* core: mon/MgrStatMonitor: ensure only one copy of initial service map (`issue#38839 <http://tracker.ceph.com/issues/38839>`_, `pr#27116 <https://github.com/ceph/ceph/pull/27116>`_, Sage Weil)
+* core: mon/OSDMonitor: allow 'osd pool set pgp_num_actual' (`pr#27060 <https://github.com/ceph/ceph/pull/27060>`_, Sage Weil)
+* core: mon: make mon_osd_down_out_subtree_limit update at runtime (`pr#27582 <https://github.com/ceph/ceph/pull/27582>`_, Sage Weil)
+* core: mon: ok-to-stop commands for mon and mds (`pr#27347 <https://github.com/ceph/ceph/pull/27347>`_, Sage Weil)
+* core: mon: quiet devname log noise (`pr#27314 <https://github.com/ceph/ceph/pull/27314>`_, Sage Weil)
+* core: osd/OSDMap: add 'zone' to default crush map (`pr#27117 <https://github.com/ceph/ceph/pull/27117>`_, Sage Weil)
+* core: osd/PGLog.h: print olog_can_rollback_to before deciding to rollback (`issue#38906 <http://tracker.ceph.com/issues/38906>`_, `issue#38894 <http://tracker.ceph.com/issues/38894>`_, `pr#27302 <https://github.com/ceph/ceph/pull/27302>`_, Neha Ojha)
+* core: osd/osd_types: fix object_stat_sum_t fast-path decode (`issue#39320 <http://tracker.ceph.com/issues/39320>`_, `issue#39281 <http://tracker.ceph.com/issues/39281>`_, `pr#27555 <https://github.com/ceph/ceph/pull/27555>`_, David Zafman)
+* core: osd: backport recent upmap fixes (`issue#38860 <http://tracker.ceph.com/issues/38860>`_, `issue#38967 <http://tracker.ceph.com/issues/38967>`_, `issue#38897 <http://tracker.ceph.com/issues/38897>`_, `issue#38826 <http://tracker.ceph.com/issues/38826>`_, `pr#27225 <https://github.com/ceph/ceph/pull/27225>`_, huangjun, xie xingguo)
+* core: osd: process_copy_chunk remove obc ref before pg unlock (`issue#38842 <http://tracker.ceph.com/issues/38842>`_, `issue#38973 <http://tracker.ceph.com/issues/38973>`_, `pr#27478 <https://github.com/ceph/ceph/pull/27478>`_, Zengran Zhang)
+* dashboard: NFS: failed to disable NFSv3 in export create (`issue#39104 <http://tracker.ceph.com/issues/39104>`_, `issue#38997 <http://tracker.ceph.com/issues/38997>`_, `pr#27368 <https://github.com/ceph/ceph/pull/27368>`_, Tiago Melo)
+* doc/releases/nautilus: fix config update step (`pr#27502 <https://github.com/ceph/ceph/pull/27502>`_, Sage Weil)
+* doc: doc/orchestrator: Fix broken bullet points (`issue#39168 <http://tracker.ceph.com/issues/39168>`_, `pr#27487 <https://github.com/ceph/ceph/pull/27487>`_, Sebastian Wagner)
+* doc: doc: Minor rados related documentation fixes (`issue#38896 <http://tracker.ceph.com/issues/38896>`_, `issue#38903 <http://tracker.ceph.com/issues/38903>`_, `pr#27189 <https://github.com/ceph/ceph/pull/27189>`_, David Zafman)
+* doc: doc: rgw: Added library/package for Golang (`issue#38730 <http://tracker.ceph.com/issues/38730>`_, `issue#38867 <http://tracker.ceph.com/issues/38867>`_, `pr#27549 <https://github.com/ceph/ceph/pull/27549>`_, Irek Fasikhov)
+* install-deps.sh: install '\*rpm-macros' (`issue#39164 <http://tracker.ceph.com/issues/39164>`_, `pr#27544 <https://github.com/ceph/ceph/pull/27544>`_, Kefu Chai)
+* mgr/dashboard add polish language (`issue#39052 <http://tracker.ceph.com/issues/39052>`_, `pr#27287 <https://github.com/ceph/ceph/pull/27287>`_, Sebastian Krah)
+* mgr/dashboard/qa: Improve tasks.mgr.test_dashboard.TestDashboard.test_standby (`pr#27237 <https://github.com/ceph/ceph/pull/27237>`_, Volker Theile)
+* mgr/dashboard: 1 osds exist in the crush map but not in the osdmap breaks OSD page (`issue#38885 <http://tracker.ceph.com/issues/38885>`_, `issue#36086 <http://tracker.ceph.com/issues/36086>`_, `pr#27543 <https://github.com/ceph/ceph/pull/27543>`_, Patrick Nawracay)
+* mgr/dashboard: Adapt iSCSI overview page to make use of ceph-iscsi (`pr#27541 <https://github.com/ceph/ceph/pull/27541>`_, Ricardo Marques)
+* mgr/dashboard: Add date range and log search functionality (`issue#37387 <http://tracker.ceph.com/issues/37387>`_, `issue#38878 <http://tracker.ceph.com/issues/38878>`_, `pr#27283 <https://github.com/ceph/ceph/pull/27283>`_, guodan1)
+* mgr/dashboard: Add refresh interval to the dashboard landing page (`issue#26872 <http://tracker.ceph.com/issues/26872>`_, `issue#38988 <http://tracker.ceph.com/issues/38988>`_, `pr#27267 <https://github.com/ceph/ceph/pull/27267>`_, guodan1)
+* mgr/dashboard: Add separate option to config SSL port (`issue#39001 <http://tracker.ceph.com/issues/39001>`_, `pr#27393 <https://github.com/ceph/ceph/pull/27393>`_, Volker Theile)
+* mgr/dashboard: Added breadcrumb tests to NFS menu (`issue#38981 <http://tracker.ceph.com/issues/38981>`_, `pr#27589 <https://github.com/ceph/ceph/pull/27589>`_, Nathan Weinberg)
+* mgr/dashboard: Back button component (`issue#39058 <http://tracker.ceph.com/issues/39058>`_, `pr#27405 <https://github.com/ceph/ceph/pull/27405>`_, Stephan Müller)
+* mgr/dashboard: Cannot submit NFS export form when NFSv4 is not selected (`issue#39105 <http://tracker.ceph.com/issues/39105>`_, `issue#39063 <http://tracker.ceph.com/issues/39063>`_, `pr#27370 <https://github.com/ceph/ceph/pull/27370>`_, Tiago Melo)
+* mgr/dashboard: Error creating NFS export without UDP (`issue#39107 <http://tracker.ceph.com/issues/39107>`_, `issue#39090 <http://tracker.ceph.com/issues/39090>`_, `pr#27372 <https://github.com/ceph/ceph/pull/27372>`_, Tiago Melo)
+* mgr/dashboard: Error on iSCSI disk diff (`pr#27460 <https://github.com/ceph/ceph/pull/27460>`_, Ricardo Marques)
+* mgr/dashboard: Filter iSCSI target images based on required features (`issue#39002 <http://tracker.ceph.com/issues/39002>`_, `pr#27363 <https://github.com/ceph/ceph/pull/27363>`_, Ricardo Marques)
+* mgr/dashboard: Fix env vars of `run-tox.sh` (`issue#38798 <http://tracker.ceph.com/issues/38798>`_, `issue#38864 <http://tracker.ceph.com/issues/38864>`_, `pr#27361 <https://github.com/ceph/ceph/pull/27361>`_, Patrick Nawracay)
+* mgr/dashboard: Fixes tooltip behavior (`pr#27395 <https://github.com/ceph/ceph/pull/27395>`_, Stephan Müller)
+* mgr/dashboard: FixtureHelper (`issue#39041 <http://tracker.ceph.com/issues/39041>`_, `pr#27398 <https://github.com/ceph/ceph/pull/27398>`_, Stephan Müller)
+* mgr/dashboard: NFS Squash field should be required (`issue#39106 <http://tracker.ceph.com/issues/39106>`_, `issue#39064 <http://tracker.ceph.com/issues/39064>`_, `pr#27371 <https://github.com/ceph/ceph/pull/27371>`_, Tiago Melo)
+* mgr/dashboard: PreventDefault isn't working on 400 errors (`pr#27389 <https://github.com/ceph/ceph/pull/27389>`_, Stephan Müller)
+* mgr/dashboard: Typo in "CephFS Name" field on NFS form (`issue#39067 <http://tracker.ceph.com/issues/39067>`_, `pr#27449 <https://github.com/ceph/ceph/pull/27449>`_, Tiago Melo)
+* mgr/dashboard: dashboard giving 401 unauthorized (`issue#38871 <http://tracker.ceph.com/issues/38871>`_, `pr#27219 <https://github.com/ceph/ceph/pull/27219>`_, ming416)
+* mgr/dashboard: fix sparkline component (`issue#38866 <http://tracker.ceph.com/issues/38866>`_, `pr#27260 <https://github.com/ceph/ceph/pull/27260>`_, Alfonso Martínez)
+* mgr/dashboard: readonly user can't see any pages (`issue#39240 <http://tracker.ceph.com/issues/39240>`_, `pr#27611 <https://github.com/ceph/ceph/pull/27611>`_, Stephan Müller)
+* mgr/dashboard: unify button/URL actions naming + bugfix (add whitelist to guard) (`issue#37337 <http://tracker.ceph.com/issues/37337>`_, `issue#39003 <http://tracker.ceph.com/issues/39003>`_, `pr#27492 <https://github.com/ceph/ceph/pull/27492>`_, Ernesto Puerta)
+* mgr/dashboard: update vstart to use new ssl_server_port (`issue#39124 <http://tracker.ceph.com/issues/39124>`_, `pr#27394 <https://github.com/ceph/ceph/pull/27394>`_, Ernesto Puerta)
+* mgr/deepsea: use ceph_volume output in get_inventory() (`issue#39083 <http://tracker.ceph.com/issues/39083>`_, `pr#27319 <https://github.com/ceph/ceph/pull/27319>`_, Tim Serong)
+* mgr/diskprediction_cloud: Correct base64 encode translate table (`pr#27167 <https://github.com/ceph/ceph/pull/27167>`_, Rick Chen)
+* mgr/orchestrator: Add error handling to interface (`issue#38837 <http://tracker.ceph.com/issues/38837>`_, `pr#27095 <https://github.com/ceph/ceph/pull/27095>`_, Sebastian Wagner)
+* mgr/pg_autoscaler: add pg_autoscale_bias (`pr#27387 <https://github.com/ceph/ceph/pull/27387>`_, Sage Weil)
+* mgr:  mgr/dashboard: Error on iSCSI target submission (`pr#27461 <https://github.com/ceph/ceph/pull/27461>`_, Ricardo Marques)
+* mgr: ceph-mgr:  ImportError: Interpreter change detected - this module can only be loaded into one interprer per process (`issue#38865 <http://tracker.ceph.com/issues/38865>`_, `pr#27128 <https://github.com/ceph/ceph/pull/27128>`_, Tim Serong)
+* mgr: mgr/DaemonServer: handle_conf_change - fix broken locking (`issue#38964 <http://tracker.ceph.com/issues/38964>`_, `issue#38899 <http://tracker.ceph.com/issues/38899>`_, `pr#27454 <https://github.com/ceph/ceph/pull/27454>`_, xie xingguo)
+* mgr: mgr/balancer: Python 3 compatibility fix (`issue#38831 <http://tracker.ceph.com/issues/38831>`_, `issue#38855 <http://tracker.ceph.com/issues/38855>`_, `pr#27227 <https://github.com/ceph/ceph/pull/27227>`_, Marius Schiffer)
+* mgr: mgr/dashboard: Check if gateway is in use before allowing the deletion via `iscsi-gateway-rm` command (`pr#27457 <https://github.com/ceph/ceph/pull/27457>`_, Ricardo Marques)
+* mgr: mgr/dashboard: Display the number of active sessions for each iSCSI target (`pr#27450 <https://github.com/ceph/ceph/pull/27450>`_, Ricardo Marques)
+* mgr: mgr/devicehealth: Fix python 3 incompatiblity (`issue#38957 <http://tracker.ceph.com/issues/38957>`_, `issue#38939 <http://tracker.ceph.com/issues/38939>`_, `pr#27390 <https://github.com/ceph/ceph/pull/27390>`_, Marius Schiffer)
+* mgr: mgr/telemetry: add report_timestamp to sent reports (`pr#27701 <https://github.com/ceph/ceph/pull/27701>`_, Dan Mick)
+* mgr: mgr/telemetry: use list; redact host; 24h default interval (`pr#27709 <https://github.com/ceph/ceph/pull/27709>`_, Sage Weil, Dan Mick)
+* mgr: mgr: Configure Py root logger for Mgr modules (`issue#38969 <http://tracker.ceph.com/issues/38969>`_, `pr#27261 <https://github.com/ceph/ceph/pull/27261>`_, Volker Theile)
+* mgr: mgr: Diskprediction unable to transfer data into the cloud server (`issue#38970 <http://tracker.ceph.com/issues/38970>`_, `pr#27240 <https://github.com/ceph/ceph/pull/27240>`_, Rick Chen)
+* mon/MonClient: do not dereference auth_supported.end() (`pr#27215 <https://github.com/ceph/ceph/pull/27215>`_, Kefu Chai)
+* mon/MonmapMonitor: clean up empty created stamp in monmap (`issue#39085 <http://tracker.ceph.com/issues/39085>`_, `pr#27399 <https://github.com/ceph/ceph/pull/27399>`_, Sage Weil)
+* mon: mon: add cluster log to file option (`pr#27346 <https://github.com/ceph/ceph/pull/27346>`_, Sage Weil)
+* msg/async v2: make v2 work on rdma (`pr#27216 <https://github.com/ceph/ceph/pull/27216>`_, Jianpeng Ma)
+* msg: default to debug_ms=0 (`pr#27197 <https://github.com/ceph/ceph/pull/27197>`_, Sage Weil)
+* osd: OSDMapRef access by multiple threads is unsafe (`pr#27402 <https://github.com/ceph/ceph/pull/27402>`_, Zengran Zhang, Kefu Chai)
+* qa/valgrind (`pr#27320 <https://github.com/ceph/ceph/pull/27320>`_, Radoslaw Zarzynski)
+* rbd,tests: backport krbd discard qa fixes to nautilus (`issue#38861 <http://tracker.ceph.com/issues/38861>`_, `pr#27258 <https://github.com/ceph/ceph/pull/27258>`_, Ilya Dryomov)
+* rbd,tests: backport krbd discard qa fixes to stable branches (`issue#38956 <http://tracker.ceph.com/issues/38956>`_, `pr#27239 <https://github.com/ceph/ceph/pull/27239>`_, Ilya Dryomov)
+* rbd: librbd: ignore -EOPNOTSUPP errors when retrieving image group membership (`issue#38834 <http://tracker.ceph.com/issues/38834>`_, `pr#27080 <https://github.com/ceph/ceph/pull/27080>`_, Jason Dillaman)
+* rbd: librbd: look for pool metadata in default namespace (`issue#38961 <http://tracker.ceph.com/issues/38961>`_, `pr#27423 <https://github.com/ceph/ceph/pull/27423>`_, Mykola Golub)
+* rbd: librbd: trash move return EBUSY instead of EINVAL for migrating image (`issue#38968 <http://tracker.ceph.com/issues/38968>`_, `pr#27475 <https://github.com/ceph/ceph/pull/27475>`_, Mykola Golub)
+* rbd: rbd: krbd: return -ETIMEDOUT in polling (`issue#38792 <http://tracker.ceph.com/issues/38792>`_, `issue#38977 <http://tracker.ceph.com/issues/38977>`_, `pr#27539 <https://github.com/ceph/ceph/pull/27539>`_, Dongsheng Yang)
+* rgw: Adding tcp_nodelay option to Beast (`issue#38926 <http://tracker.ceph.com/issues/38926>`_, `pr#27355 <https://github.com/ceph/ceph/pull/27355>`_, Or Friedmann)
+* rgw: Fix S3 compatibility bug when CORS is not found (`issue#38923 <http://tracker.ceph.com/issues/38923>`_, `issue#37945 <http://tracker.ceph.com/issues/37945>`_, `pr#27331 <https://github.com/ceph/ceph/pull/27331>`_, Nick Janus)
+* rgw: LC: handle resharded buckets (`pr#27559 <https://github.com/ceph/ceph/pull/27559>`_, Abhishek Lekshmanan)
+* rgw: Make rgw admin ops api get user info consistent with the command line (`issue#39135 <http://tracker.ceph.com/issues/39135>`_, `pr#27501 <https://github.com/ceph/ceph/pull/27501>`_, Li Shuhao)
+* rgw: don't crash on missing /etc/mime.types (`issue#38921 <http://tracker.ceph.com/issues/38921>`_, `issue#38328 <http://tracker.ceph.com/issues/38328>`_, `pr#27329 <https://github.com/ceph/ceph/pull/27329>`_, Casey Bodley)
+* rgw: don't recalculate etags for slo/dlo (`pr#27561 <https://github.com/ceph/ceph/pull/27561>`_, Casey Bodley)
+* rgw: fix RGWDeleteMultiObj::verify_permission() (`issue#38980 <http://tracker.ceph.com/issues/38980>`_, `pr#27586 <https://github.com/ceph/ceph/pull/27586>`_, Irek Fasikhov)
+* rgw: fix read not exists null version return wrong (`issue#38811 <http://tracker.ceph.com/issues/38811>`_, `issue#38909 <http://tracker.ceph.com/issues/38909>`_, `pr#27306 <https://github.com/ceph/ceph/pull/27306>`_, Tianshan Qu)
+* rgw: ldap: fix early return in LDAPAuthEngine::init w/uri not empty() (`issue#38754 <http://tracker.ceph.com/issues/38754>`_, `pr#26972 <https://github.com/ceph/ceph/pull/26972>`_, Matt Benjamin)
+* rgw: multisite: data sync loops back to the start of the datalog after reaching the end (`issue#39075 <http://tracker.ceph.com/issues/39075>`_, `issue#39033 <http://tracker.ceph.com/issues/39033>`_, `pr#27498 <https://github.com/ceph/ceph/pull/27498>`_, Casey Bodley)
+* rgw: nfs: skip empty (non-POSIX) path segments (`issue#38744 <http://tracker.ceph.com/issues/38744>`_, `issue#38773 <http://tracker.ceph.com/issues/38773>`_, `pr#27208 <https://github.com/ceph/ceph/pull/27208>`_, Matt Benjamin)
+* rgw: nfs: svc-enable RGWLib (`issue#38774 <http://tracker.ceph.com/issues/38774>`_, `pr#27232 <https://github.com/ceph/ceph/pull/27232>`_, Matt Benjamin)
+* rgw: orphans find perf improvements (`issue#39181 <http://tracker.ceph.com/issues/39181>`_, `pr#27560 <https://github.com/ceph/ceph/pull/27560>`_, Abhishek Lekshmanan)
+* rgw: rgw admin: disable stale instance deletion in multisite (`issue#39015 <http://tracker.ceph.com/issues/39015>`_, `pr#27602 <https://github.com/ceph/ceph/pull/27602>`_, Abhishek Lekshmanan)
+* rgw: sse c fixes (`issue#38700 <http://tracker.ceph.com/issues/38700>`_, `pr#27296 <https://github.com/ceph/ceph/pull/27296>`_, Adam Kupczyk, Casey Bodley, Abhishek Lekshmanan)
+* rgw: support delimiter longer then one symbol (`issue#38777 <http://tracker.ceph.com/issues/38777>`_, `pr#27548 <https://github.com/ceph/ceph/pull/27548>`_, Matt Benjamin)
+* rook-ceph-system namespace hardcoded in the rook orchestrator (`issue#38799 <http://tracker.ceph.com/issues/38799>`_, `issue#39250 <http://tracker.ceph.com/issues/39250>`_, `pr#27496 <https://github.com/ceph/ceph/pull/27496>`_, Sebastian Wagner)
+* rpm,cmake: use specified python3 version if any (`pr#27382 <https://github.com/ceph/ceph/pull/27382>`_, Kefu Chai)
+
+
+v14.2.0 Nautilus
+================
+
+This is the first stable release of Ceph Nautilus.
+
+Major Changes from Mimic
+------------------------
+
+- *Dashboard*:
+
+  The :ref:`mgr-dashboard` has gained a lot of new functionality:
+
+  * Support for multiple users / roles
+  * SSO (SAMLv2) for user authentication
+  * Auditing support
+  * New landing page, showing more metrics and health info
+  * I18N support
+  * REST API documentation with Swagger API
+
+  New Ceph management features include:
+
+  * OSD management (mark as down/out, change OSD settings, recovery profiles)
+  * Cluster config settings editor
+  * Ceph Pool management (create/modify/delete)
+  * ECP management
+  * RBD mirroring configuration
+  * Embedded Grafana Dashboards (derived from Ceph Metrics)
+  * CRUSH map viewer
+  * NFS Ganesha management
+  * iSCSI target management (via :ref:`ceph-iscsi`)
+  * RBD QoS configuration
+  * Ceph Manager (ceph-mgr) module management
+  * Prometheus alert Management
+
+  Also, the Ceph Dashboard is now split into its own package named
+  ``ceph-mgr-dashboard``. You might want to install it separately,
+  if your package management software fails to do so when it installs
+  ``ceph-mgr``.
+
+- *RADOS*:
+
+  * The number of placement groups (PGs) per pool can now be decreased
+    at any time, and the cluster can :ref:`automatically tune the PG count <pg-autoscaler>`
+    based on cluster utilization or administrator hints.
+  * The new :ref:`v2 wire protocol <msgr2>` brings support for encryption on the wire.
+  * Physical :ref:`storage devices <devices>` consumed by OSD and Monitor daemons are
+    now tracked by the cluster along with health metrics (i.e.,
+    SMART), and the cluster can apply a pre-trained prediction model
+    or a cloud-based prediction service to :ref:`warn about expected
+    HDD or SSD failures <diskprediction>`.
+  * The NUMA node for OSD daemons can easily be monitored via the
+    ``ceph osd numa-status`` command, and configured via the
+    ``osd_numa_node`` config option.
+  * When BlueStore OSDs are used, space utilization is now broken down
+    by object data, omap data, and internal metadata, by pool, and by
+    pre- and post- compression sizes.
+  * OSDs more effectively prioritize the most important PGs and
+    objects when performing recovery and backfill.
+  * Progress for long-running background processes--like recovery
+    after a device failure--is now reported as part of ``ceph
+    status``.
+  * An experimental `Coupled-Layer "Clay" erasure code
+    <https://www.usenix.org/conference/fast18/presentation/vajha>`_
+    plugin has been added that reduces network bandwidth and IO needed
+    for most recovery operations.
+
+- *RGW*:
+
+  * S3 lifecycle transition for tiering between storage classes.
+  * A new web frontend (Beast) has replaced civetweb as the default,
+    improving overall performance.
+  * A new publish/subscribe infrastructure allows RGW to feed events
+    to serverless frameworks like knative or data pipelies like Kafka.
+  * A range of authentication features, including STS federation using
+    OAuth2 and OpenID::connect and an OPA (Open Policy Agent)
+    authentication delegation prototype.
+  * The new archive zone federation feature enables full preservation
+    of all objects (including history) in a separate zone.
+
+- *CephFS*:
+
+  * MDS stability has been greatly improved for large caches and
+    long-running clients with a lot of RAM. Cache trimming and client
+    capability recall is now throttled to prevent overloading the MDS.
+  * CephFS may now be exported via NFS-Ganesha clusters in environments managed
+    by Rook. Ceph manages the clusters and ensures high-availability and
+    scalability. An `introductory demo
+    <https://ceph.com/community/deploying-a-cephnfs-server-cluster-with-rook/>`_
+    is available. More automation of this feature is expected to be forthcoming
+    in future minor releases of Nautilus.
+  * The MDS ``mds_standby_for_*``, ``mon_force_standby_active``, and
+    ``mds_standby_replay`` configuration options have been obsoleted. Instead,
+    the operator :ref:`may now set <mds-standby-replay>` the new
+    ``allow_standby_replay`` flag on the CephFS file system. This setting
+    causes standbys to become standby-replay for any available rank in the file
+    system.
+  * MDS now supports dropping its cache which concurrently asks clients
+    to trim their caches. This is done using MDS admin socket ``cache drop``
+    command.
+  * It is now possible to check the progress of an on-going scrub in the MDS.
+    Additionally, a scrub may be paused or aborted. See :ref:`the scrub
+    documentation <mds-scrub>` for more information.
+  * A new interface for creating volumes is provided via the ``ceph volume``
+    command-line-interface.
+  * A new cephfs-shell tool is available for manipulating a CephFS file
+    system without mounting.
+  * CephFS-related output from ``ceph status`` has been reformatted for brevity,
+    clarity, and usefulness.
+  * Lazy IO has been revamped. It can be turned on by the client using the new
+    CEPH_O_LAZY flag to the ``ceph_open`` C/C++ API or via the config option
+    ``client_force_lazyio``.
+  * CephFS file system can now be brought down rapidly via the ``ceph fs fail``
+    command. See :ref:`the administration page <cephfs-administration>` for
+    more information.
+
+- *RBD*:
+
+  * Images can be live-migrated with minimal downtime to assist with moving
+    images between pools or to new layouts.
+  * New ``rbd perf image iotop`` and ``rbd perf image iostat`` commands provide
+    an iotop- and iostat-like IO monitor for all RBD images.
+  * The *ceph-mgr* Prometheus exporter now optionally includes an IO monitor
+    for all RBD images.
+  * Support for separate image namespaces within a pool for tenant isolation.
+
+- *Misc*:
+
+  * Ceph has a new set of :ref:`orchestrator modules
+    <orchestrator-cli-module>` to directly interact with external
+    orchestrators like ceph-ansible, DeepSea, Rook, or simply ssh via
+    a consistent CLI (and, eventually, Dashboard) interface.
+
+.. _nautilus-old-upgrade:
+
+Upgrading from Mimic or Luminous
+--------------------------------
+
+Notes
+~~~~~
+
+* During the upgrade from Luminous to Nautilus, it will not be
+  possible to create a new OSD using a Luminous ceph-osd daemon after
+  the monitors have been upgraded to Nautilus.  We recommend you avoid adding
+  or replacing any OSDs while the upgrade is in progress.
+
+* We recommend you avoid creating any RADOS pools while the upgrade is
+  in progress.
+
+* You can monitor the progress of your upgrade at each stage with the
+  ``ceph versions`` command, which will tell you what ceph version(s) are
+  running for each type of daemon.
+
+Instructions
+~~~~~~~~~~~~
+
+#. If your cluster was originally installed with a version prior to
+   Luminous, ensure that it has completed at least one full scrub of
+   all PGs while running Luminous.  Failure to do so will cause your
+   monitor daemons to refuse to join the quorum on start, leaving them
+   non-functional.
+
+   If you are unsure whether or not your Luminous cluster has
+   completed a full scrub of all PGs, you can check your cluster's
+   state by running::
+
+     # ceph osd dump | grep ^flags
+
+   In order to be able to proceed to Nautilus, your OSD map must include
+   the ``recovery_deletes`` and ``purged_snapdirs`` flags.
+
+   If your OSD map does not contain both these flags, you can simply
+   wait for approximately 24-48 hours, which in a standard cluster
+   configuration should be ample time for all your placement groups to
+   be scrubbed at least once, and then repeat the above process to
+   recheck.
+
+   However, if you have just completed an upgrade to Luminous and want
+   to proceed to Mimic in short order, you can force a scrub on all
+   placement groups with a one-line shell command, like::
+
+     # ceph pg dump pgs_brief | cut -d " " -f 1 | xargs -n1 ceph pg scrub
+
+   You should take into consideration that this forced scrub may
+   possibly have a negative impact on your Ceph clients' performance.
+
+#. Make sure your cluster is stable and healthy (no down or
+   recovering OSDs).  (Optional, but recommended.)
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional,
+   but recommended.)::
+
+     # ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.  For example, on each monitor host,::
+
+     # systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is
+   complete by looking for the ``nautilus`` string in the mon
+   map.  The command::
+
+     # ceph mon dump | grep min_mon_release
+
+   should report::
+
+     min_mon_release 14 (nautilus)
+
+   If it doesn't, that implies that one or more monitors hasn't been
+   upgraded and restarted and/or the quorum does not include all monitors.
+
+#. Upgrade ``ceph-mgr`` daemons by installing the new packages and
+   restarting all manager daemons.  For example, on each manager host,::
+
+     # systemctl restart ceph-mgr.target
+
+   Please note, if you are using Ceph Dashboard, you will probably need to
+   install ``ceph-mgr-dashboard`` separately after upgrading ``ceph-mgr``
+   package. The install script of ``ceph-mgr-dashboard`` will restart the
+   manager daemons automatically for you. So in this case, you can just skip
+   the step to restart the daemons.
+
+   Verify the ``ceph-mgr`` daemons are running by checking ``ceph
+   -s``::
+
+     # ceph -s
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+  
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all OSD hosts::
+
+     # systemctl restart ceph-osd.target
+
+   You can monitor the progress of the OSD upgrades with the
+   ``ceph versions`` or ``ceph osd versions`` commands::
+
+     # ceph osd versions
+     {
+        "ceph version 13.2.5 (...) mimic (stable)": 12,
+        "ceph version 14.2.0 (...) nautilus (stable)": 22,
+     }
+
+#. If there are any OSDs in the cluster deployed with ceph-disk (e.g.,
+   almost any OSDs that were created before the Mimic release), you
+   need to tell ceph-volume to adopt responsibility for starting the
+   daemons.  On each host containing OSDs, ensure the OSDs are
+   currently running, and then::
+
+     # ceph-volume simple scan
+     # ceph-volume simple activate --all
+
+   We recommend that each OSD host be rebooted following this step to
+   verify that the OSDs start up automatically.
+
+   Note that ceph-volume doesn't have the same hot-plug capability
+   that ceph-disk did, where a newly attached disk is automatically
+   detected via udev events.  If the OSD isn't currently running when the
+   above ``scan`` command is run, or a ceph-disk-based OSD is moved to
+   a new host, or the host OSD is reinstalled, or the
+   ``/etc/ceph/osd`` directory is lost, you will need to scan the main
+   data partition for each ceph-disk OSD explicitly.  For example,::
+
+     # ceph-volume simple scan /dev/sdb1
+
+   The output will include the appropriate ``ceph-volume simple
+   activate`` command to enable the OSD.
+
+#. Upgrade all CephFS MDS daemons.  For each CephFS file system,
+
+   #. Reduce the number of ranks to 1.  (Make note of the original
+      number of MDS daemons first if you plan to restore it later.)::
+
+	# ceph status
+	# ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by
+      periodically checking the status::
+
+	# ceph status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with::
+
+	# systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS::
+
+	# ceph status
+
+   #. Upgrade the last remaining MDS daemon by installing the new
+      packages and restarting the daemon::
+
+        # systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline::
+
+	# systemctl start ceph-mds.target
+
+   #. Restore the original value of ``max_mds`` for the volume::
+
+	# ceph fs set <fs_name> max_mds <original_max_mds>
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart ceph-radosgw.target
+
+#. Complete the upgrade by disallowing pre-Nautilus OSDs and enabling
+   all new Nautilus-only functionality::
+
+     # ceph osd require-osd-release nautilus
+
+   .. important:: This step is mandatory. Failure to execute this step will make it impossible for OSDs to communicate after msgrv2 is enabled.
+
+#. If you set ``noout`` at the beginning, be sure to clear it with::
+
+     # ceph osd unset noout
+
+#. Verify the cluster is healthy with ``ceph health``.
+
+   If your CRUSH tunables are older than Hammer, Ceph will now issue a
+   health warning.  If you see a health alert to that effect, you can
+   revert this change with::
+
+     ceph config set mon mon_crush_min_required_version firefly
+
+   If Ceph does not complain, however, then we recommend you also
+   switch any existing CRUSH buckets to straw2, which was added back
+   in the Hammer release.  If you have any 'straw' buckets, this will
+   result in a modest amount of data movement, but generally nothing
+   too severe.::
+
+     ceph osd getcrushmap -o backup-crushmap
+     ceph osd crush set-all-straw-buckets-to-straw2
+
+   If there are problems, you can easily revert with::
+
+     ceph osd setcrushmap -i backup-crushmap
+
+   Moving to 'straw2' buckets will unlock a few recent features, like
+   the `crush-compat` :ref:`balancer <balancer>` mode added back in Luminous.
+
+#. To enable the new :ref:`v2 network protocol <msgr2>`, issue the
+   following command::
+
+     ceph mon enable-msgr2
+
+   This will instruct all monitors that bind to the old default port
+   6789 for the legacy v1 protocol to also bind to the new 3300 v2
+   protocol port.  To see if all monitors have been updated,::
+
+     ceph mon dump
+
+   and verify that each monitor has both a ``v2:`` and ``v1:`` address
+   listed.
+
+   Running nautilus OSDs will not bind to their v2 address automatically.
+   They must be restarted for that to happen.
+
+   .. important:: 
+      Before this step is run, the following command must already have been run:
+
+      # ceph osd require-osd-release nautilus
+
+      If this command (step 10 in this procedure) has not been run, OSDs will lose the ability to communicate.
+
+#. For each host that has been upgraded, you should update your
+   ``ceph.conf`` file so that it either specifies no monitor port (if
+   you are running the monitors on the default ports) or references
+   both the v2 and v1 addresses and ports explicitly.  Things will
+   still work if only the v1 IP and port are listed, but each CLI
+   instantiation or daemon will need to reconnect after learning the
+   monitors also speak the v2 protocol, slowing things down a bit and
+   preventing a full transition to the v2 protocol.
+
+   This is also a good time to fully transition any config options in
+   ``ceph.conf`` into the cluster's configuration database.  On each host,
+   you can use the following command to import any options into the
+   monitors with::
+
+     ceph config assimilate-conf -i /etc/ceph/ceph.conf
+
+   You can see the cluster's configuration database with::
+
+     ceph config dump
+     
+   To create a minimal but sufficient ``ceph.conf`` for each host,::
+
+     ceph config generate-minimal-conf > /etc/ceph/ceph.conf.new
+     mv /etc/ceph/ceph.conf.new /etc/ceph/ceph.conf
+
+   Be sure to use this new config only on hosts that have been
+   upgraded to Nautilus, as it may contain a ``mon_host`` value that
+   includes the new ``v2:`` and ``v1:`` prefixes for IP addresses that
+   is only understood by Nautilus.
+
+   For more information, see :ref:`msgr2_ceph_conf`.
+
+#. Consider enabling the :ref:`telemetry module <telemetry>` to send
+   anonymized usage statistics and crash information to the Ceph
+   upstream developers.  To see what would be reported (without actually
+   sending any information to anyone),::
+
+     ceph mgr module enable telemetry
+     ceph telemetry show
+
+   If you are comfortable with the data that is reported, you can opt-in to
+   automatically report the high-level cluster metadata with::
+
+     ceph telemetry on
+
+   For more information about the telemetry module, see :ref:`the
+   documentation <telemetry>`.
+
+
+Upgrading from pre-Luminous releases (like Jewel)
+-------------------------------------------------
+
+You *must* first upgrade to Luminous (12.2.z) before attempting an
+upgrade to Nautilus.  In addition, your cluster must have completed at
+least one scrub of all PGs while running Luminous, setting the
+``recovery_deletes`` and ``purged_snapdirs`` flags in the OSD map.
+
+
+Upgrade compatibility notes
+---------------------------
+
+These changes occurred between the Mimic and Nautilus releases.
+
+* ``ceph pg stat`` output has been modified in json
+  format to match ``ceph df`` output:
+
+  - "raw_bytes" field renamed to "total_bytes"
+  - "raw_bytes_avail" field renamed to "total_bytes_avail"
+  - "raw_bytes_avail" field renamed to "total_bytes_avail"
+  - "raw_bytes_used" field renamed to "total_bytes_raw_used"
+  - "total_bytes_used" field added to represent the space (accumulated over
+     all OSDs) allocated purely for data objects kept at block(slow) device
+  
+* ``ceph df [detail]`` output (GLOBAL section) has been modified in plain
+  format:
+
+  - new 'USED' column shows the space (accumulated over all OSDs) allocated
+    purely for data objects kept at block(slow) device.
+  - 'RAW USED' is now a sum of 'USED' space and space allocated/reserved at
+     block device for Ceph purposes, e.g. BlueFS part for BlueStore.
+
+* ``ceph df [detail]`` output (GLOBAL section) has been modified in json
+  format:
+  
+  - 'total_used_bytes' column now shows the space (accumulated over all OSDs)
+    allocated purely for data objects kept at block(slow) device
+  - new 'total_used_raw_bytes' column shows a sum of 'USED' space and space
+    allocated/reserved at block device for Ceph purposes, e.g. BlueFS part for
+    BlueStore.
+
+* ``ceph df [detail]`` output (POOLS section) has been modified in plain
+  format:
+  
+  - 'BYTES USED' column renamed to 'STORED'. Represents amount of data
+    stored by the user.
+  - 'USED' column now represent amount of space allocated purely for data
+    by all OSD nodes in KB.
+  - 'QUOTA BYTES', 'QUOTA OBJECTS' aren't showed anymore in non-detailed mode.
+  - new column 'USED COMPR' - amount of space allocated for compressed
+    data. i.e., compressed data plus all the allocation, replication and erasure
+    coding overhead.
+  - new column 'UNDER COMPR' - amount of data passed through compression
+    (summed over all replicas) and beneficial enough to be stored in a
+    compressed form.
+  - Some columns reordering
+
+* ``ceph df [detail]`` output (POOLS section) has been modified in json
+  format:
+  
+  - 'bytes used' column renamed to 'stored'. Represents amount of data
+    stored by the user.
+  - 'raw bytes used' column renamed to "stored_raw". Totals of user data
+     over all OSD excluding degraded.
+  - new 'bytes_used' column now represent amount of space allocated by 
+    all OSD nodes.
+  - 'kb_used' column - the same as 'bytes_used' but in KB.
+  - new column 'compress_bytes_used' - amount of space allocated for compressed
+    data. i.e., compressed data plus all the allocation, replication and erasure
+    coding overhead.
+  - new column 'compress_under_bytes' amount of data passed through compression
+    (summed over all replicas) and beneficial enough to be stored in a
+    compressed form.
+
+* ``rados df [detail]`` output (POOLS section) has been modified in plain
+  format:
+  
+  - 'USED' column now shows the space (accumulated over all OSDs) allocated
+    purely for data objects kept at block(slow) device.
+  - new column 'USED COMPR' - amount of space allocated for compressed
+    data. i.e., compressed data plus all the allocation, replication and erasure
+    coding overhead.
+  - new column 'UNDER COMPR' - amount of data passed through compression
+    (summed over all replicas) and beneficial enough to be stored in a
+    compressed form.
+
+* ``rados df [detail]`` output (POOLS section) has been modified in json
+  format:
+  
+  - 'size_bytes' and 'size_kb' columns now show the space (accumulated
+    over all OSDs) allocated purely for data objects kept at block
+    device.
+  - new column 'compress_bytes_used' - amount of space allocated for compressed
+    data. i.e., compressed data plus all the allocation, replication and erasure
+    coding overhead.
+  - new column 'compress_under_bytes' amount of data passed through compression
+    (summed over all replicas) and beneficial enough to be stored in a
+    compressed form.
+
+* ``ceph pg dump`` output (totals section) has been modified in json
+  format:
+  
+  - new 'USED' column shows the space (accumulated over all OSDs) allocated
+    purely for data objects kept at block(slow) device.
+  - 'USED_RAW' is now a sum of 'USED' space and space allocated/reserved at
+    block device for Ceph purposes, e.g. BlueFS part for BlueStore.
+
+* The ``ceph osd rm`` command has been deprecated.  Users should use
+  ``ceph osd destroy`` or ``ceph osd purge`` (but after first confirming it is
+  safe to do so via the ``ceph osd safe-to-destroy`` command).
+
+* The MDS now supports dropping its cache for the purposes of benchmarking.::
+
+    ceph tell mds.* cache drop <timeout>
+
+  Note that the MDS cache is cooperatively managed by the clients. It is
+  necessary for clients to give up capabilities in order for the MDS to fully
+  drop its cache. This is accomplished by asking all clients to trim as many
+  caps as possible. The timeout argument to the ``cache drop`` command controls
+  how long the MDS waits for clients to complete trimming caps. This is optional
+  and is 0 by default (no timeout). Keep in mind that clients may still retain
+  caps to open files which will prevent the metadata for those files from being
+  dropped by both the client and the MDS. (This is an equivalent scenario to
+  dropping the Linux page/buffer/inode/dentry caches with some processes pinning
+  some inodes/dentries/pages in cache.)
+
+* The ``mon_health_preluminous_compat`` and
+  ``mon_health_preluminous_compat_warning`` config options are
+  removed, as the related functionality is more than two versions old.
+  Any legacy monitoring system expecting Jewel-style health output
+  will need to be updated to work with Nautilus.
+
+* Nautilus is not supported on any distros still running upstart so upstart
+  specific files and references have been removed.
+
+* The ``ceph pg <pgid> list_missing`` command has been renamed to
+  ``ceph pg <pgid> list_unfound`` to better match its behaviour.
+
+* The *rbd-mirror* daemon can now retrieve remote peer cluster configuration
+  secrets from the monitor. To use this feature, the rbd-mirror daemon
+  CephX user for the local cluster must use the ``profile rbd-mirror`` mon cap.
+  The secrets can be set using the ``rbd mirror pool peer add`` and
+  ``rbd mirror pool peer set`` actions.
+
+* The 'rbd-mirror' daemon will now run in active/active mode by default, where
+  mirrored images are evenly distributed between all active 'rbd-mirror'
+  daemons. To revert to active/passive mode, override the
+  'rbd_mirror_image_policy_type' config key to 'none'.
+
+* The ``ceph mds deactivate`` is fully obsolete and references to it in the docs
+  have been removed or clarified.
+
+* The libcephfs bindings added the ``ceph_select_filesystem`` function
+  for use with multiple filesystems.
+
+* The cephfs python bindings now include ``mount_root`` and ``filesystem_name``
+  options in the mount() function.
+
+* erasure-code: add experimental *Coupled LAYer (CLAY)* erasure codes
+  support. It features less network traffic and disk I/O when performing
+  recovery.
+
+* The ``cache drop`` OSD command has been added to drop an OSD's caches:
+
+    - ``ceph tell osd.x cache drop``
+
+* The ``cache status`` OSD command has been added to get the cache stats of an
+  OSD:
+
+    - ``ceph tell osd.x cache status``
+
+* The libcephfs added several functions that allow restarted client to destroy
+  or reclaim state held by a previous incarnation. These functions are for NFS
+  servers.
+
+* The ``ceph`` command line tool now accepts keyword arguments in
+  the format ``--arg=value`` or ``--arg value``.
+
+* ``librados::IoCtx::nobjects_begin()`` and
+  ``librados::NObjectIterator`` now communicate errors by throwing a
+  ``std::system_error`` exception instead of ``std::runtime_error``.
+
+* The callback function passed to ``LibRGWFS.readdir()`` now accepts a ``flags``
+  parameter. it will be the last parameter passed to  ``readdir()`` method.
+
+* The ``cephfs-data-scan scan_links`` now automatically repair inotables and
+  snaptable.
+
+* Configuration values ``mon_warn_not_scrubbed`` and
+  ``mon_warn_not_deep_scrubbed`` have been renamed.  They are now
+  ``mon_warn_pg_not_scrubbed_ratio`` and ``mon_warn_pg_not_deep_scrubbed_ratio``
+  respectively.  This is to clarify that these warnings are related to
+  pg scrubbing and are a ratio of the related interval.  These options
+  are now enabled by default.
+
+* The MDS cache trimming is now throttled. Dropping the MDS cache
+  via the ``ceph tell mds.<foo> cache drop`` command or large reductions in the
+  cache size will no longer cause service unavailability.
+
+* The CephFS MDS behavior with recalling caps has been significantly improved
+  to not attempt recalling too many caps at once, leading to instability.
+  MDS with a large cache (64GB+) should be more stable.
+
+* MDS now provides a config option ``mds_max_caps_per_client`` (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
+* The MDS config ``mds_recall_state_timeout`` has been removed. Late
+  client recall warnings are now generated based on the number of caps
+  the MDS has recalled which have not been released. The new configs
+  ``mds_recall_warning_threshold`` (default: 32K) and
+  ``mds_recall_warning_decay_rate`` (default: 60s) sets the threshold
+  for this warning.
+
+* The Telegraf module for the Manager allows for sending statistics to
+  an Telegraf Agent over TCP, UDP or a UNIX Socket. Telegraf can then
+  send the statistics to databases like InfluxDB, ElasticSearch, Graphite
+  and many more.
+
+* The graylog fields naming the originator of a log event have
+  changed: the string-form name is now included (e.g., ``"name":
+  "mgr.foo"``), and the rank-form name is now in a nested section
+  (e.g., ``"rank": {"type": "mgr", "num": 43243}``).
+
+* If the cluster log is directed at syslog, the entries are now
+  prefixed by both the string-form name and the rank-form name (e.g.,
+  ``mgr.x mgr.12345 ...`` instead of just ``mgr.12345 ...``).
+
+* The JSON output of the ``ceph osd find`` command has replaced the ``ip``
+  field with an ``addrs`` section to reflect that OSDs may bind to
+  multiple addresses.
+
+* CephFS clients without the 's' flag in their authentication capability
+  string will no longer be able to create/delete snapshots. To allow
+  ``client.foo`` to create/delete snapshots in the ``bar`` directory of
+  filesystem ``cephfs_a``, use command:
+
+    - ``ceph auth caps client.foo mon 'allow r' osd 'allow rw tag cephfs data=cephfs_a' mds 'allow rw, allow rws path=/bar'``
+
+* The ``osd_heartbeat_addr`` option has been removed as it served no
+  (good) purpose: the OSD should always check heartbeats on both the
+  public and cluster networks.
+
+* The ``rados`` tool's ``mkpool`` and ``rmpool`` commands have been
+  removed because they are redundant; please use the ``ceph osd pool
+  create`` and ``ceph osd pool rm`` commands instead.
+
+* The ``auid`` property for cephx users and RADOS pools has been
+  removed.  This was an undocumented and partially implemented
+  capability that allowed cephx users to map capabilities to RADOS
+  pools that they "owned".  Because there are no users we have removed
+  this support.  If any cephx capabilities exist in the cluster that
+  restrict based on auid then they will no longer parse, and the
+  cluster will report a health warning like::
+
+    AUTH_BAD_CAPS 1 auth entities have invalid capabilities
+        client.bad osd capability parse failed, stopped at 'allow rwx auid 123' of 'allow rwx auid 123'
+
+  The capability can be adjusted with the ``ceph auth caps``
+  command. For example,::
+
+    ceph auth caps client.bad osd 'allow rwx pool foo'
+
+* The ``ceph-kvstore-tool`` ``repair`` command has been renamed
+  ``destructive-repair`` since we have discovered it can corrupt an
+  otherwise healthy rocksdb database.  It should be used only as a last-ditch
+  attempt to recover data from an otherwise corrupted store.
+
+
+* The default memory utilization for the mons has been increased
+  somewhat.  Rocksdb now uses 512 MB of RAM by default, which should
+  be sufficient for small to medium-sized clusters; large clusters
+  should tune this up.  Also, the ``mon_osd_cache_size`` has been
+  increase from 10 OSDMaps to 500, which will translate to an
+  additional 500 MB to 1 GB of RAM for large clusters, and much less
+  for small clusters.
+
+* The ``mgr/balancer/max_misplaced`` option has been replaced by a new
+  global ``target_max_misplaced_ratio`` option that throttles both
+  balancer activity and automated adjustments to ``pgp_num`` (normally as a
+  result of ``pg_num`` changes).  If you have customized the balancer module
+  option, you will need to adjust your config to set the new global option
+  or revert to the default of .05 (5%).
+
+* By default, Ceph no longer issues a health warning when there are
+  misplaced objects (objects that are fully replicated but not stored
+  on the intended OSDs).  You can reenable the old warning by setting
+  ``mon_warn_on_misplaced`` to ``true``.
+
+* The ``ceph-create-keys`` tool is now obsolete.  The monitors
+  automatically create these keys on their own.  For now the script
+  prints a warning message and exits, but it will be removed in the
+  next release.  Note that ``ceph-create-keys`` would also write the
+  admin and bootstrap keys to /etc/ceph and /var/lib/ceph, but this
+  script no longer does that.  Any deployment tools that relied on
+  this behavior should instead make use of the ``ceph auth export
+  <entity-name>`` command for whichever key(s) they need.
+
+* The ``mon_osd_pool_ec_fast_read`` option has been renamed
+  ``osd_pool_default_ec_fast_read`` to be more consistent with other
+  ``osd_pool_default_*`` options that affect default values for newly
+  created RADOS pools.
+
+* The ``mon addr`` configuration option is now deprecated.  It can
+  still be used to specify an address for each monitor in the
+  ``ceph.conf`` file, but it only affects cluster creation and
+  bootstrapping, and it does not support listing multiple addresses
+  (e.g., both a v2 and v1 protocol address).  We strongly recommend
+  the option be removed and instead a single ``mon host`` option be
+  specified in the ``[global]`` section to allow daemons and clients
+  to discover the monitors.
+
+* New command ``ceph fs fail`` has been added to quickly bring down a file
+  system. This is a single command that unsets the joinable flag on the file
+  system and brings down all of its ranks.
+
+* The ``cache drop`` admin socket command has been removed. The ``ceph
+  tell mds.X cache drop`` remains.
+
+
+Detailed Changelog
+------------------
+* add monitoring subdir and Grafana cluster dashboard (`pr#21850 <https://github.com/ceph/ceph/pull/21850>`_, Jan Fajerski)
+* auth,common: include cleanups (`pr#23774 <https://github.com/ceph/ceph/pull/23774>`_, Kefu Chai)
+* bluestore: bluestore/NVMEDevice.cc: fix ceph_assert() when enable SPDK with 64KB kernel page size (`issue#36624 <http://tracker.ceph.com/issues/36624>`_, `pr#24817 <https://github.com/ceph/ceph/pull/24817>`_, tone.zhang)
+* bluestore: bluestore/NVMEDevice.cc: fix NVMEManager thread hang (`issue#37720 <http://tracker.ceph.com/issues/37720>`_, `pr#25646 <https://github.com/ceph/ceph/pull/25646>`_, tone.zhang, Steve Capper)
+* bluestore: bluestore/NVMe: use PCIe selector as the path name (`pr#24144 <https://github.com/ceph/ceph/pull/24144>`_, Kefu Chai)
+* bluestore,cephfs,core,rbd,rgw: buffer,denc: use ptr::const_iterator for decode (`pr#22015 <https://github.com/ceph/ceph/pull/22015>`_, Kefu Chai, Casey Bodley)
+* bluestore: ceph-kvstore-tool: dump fixes (`pr#25262 <https://github.com/ceph/ceph/pull/25262>`_, Adam Kupczyk)
+* bluestore: common/blkdev: check retval of stat() (`pr#26040 <https://github.com/ceph/ceph/pull/26040>`_, Kefu Chai)
+* bluestore,core: ceph-dencoder: add bluefs types (`pr#22463 <https://github.com/ceph/ceph/pull/22463>`_, Sage Weil)
+* bluestore,core,mon,performance: osd,mon: enable level_compaction_dynamic_level_bytes for rocksdb (`issue#24361 <http://tracker.ceph.com/issues/24361>`_, `pr#22337 <https://github.com/ceph/ceph/pull/22337>`_, Kefu Chai)
+* bluestore,core: os/bluestore: don't store/use path_block.{db,wal} from meta (`pr#22462 <https://github.com/ceph/ceph/pull/22462>`_, Sage Weil, Alfredo Deza)
+* bluestore: os/bluestore: add bluestore_ignore_data_csum option (`pr#26233 <https://github.com/ceph/ceph/pull/26233>`_, Sage Weil)
+* bluestore: os/bluestore: add boundary check for cache-autotune related settings (`issue#37507 <http://tracker.ceph.com/issues/37507>`_, `pr#25421 <https://github.com/ceph/ceph/pull/25421>`_, xie xingguo)
+* bluestore: os/bluestore/BlueFS: only flush dirty devices when do _fsync (`pr#22110 <https://github.com/ceph/ceph/pull/22110>`_, Jianpeng Ma)
+* bluestore: os/bluestore: bluestore_buffer_hit_bytes perf counter doesn't reset (`pr#23576 <https://github.com/ceph/ceph/pull/23576>`_, Igor Fedotov)
+* bluestore: os/bluestore: check return value of _open_bluefs (`pr#25471 <https://github.com/ceph/ceph/pull/25471>`_, Jianpeng Ma)
+* bluestore: os/bluestore: cleanups (`pr#22556 <https://github.com/ceph/ceph/pull/22556>`_, Jianpeng Ma)
+* bluestore: os/bluestore: deep fsck fails on inspecting very large onodes (`pr#26170 <https://github.com/ceph/ceph/pull/26170>`_, Igor Fedotov)
+* bluestore: os/bluestore: do not assert on non-zero err codes from  compress() call (`pr#25891 <https://github.com/ceph/ceph/pull/25891>`_, Igor Fedotov)
+* bluestore: os/bluestore: firstly delete db then delete bluefs if open db met error (`pr#22336 <https://github.com/ceph/ceph/pull/22336>`_, Jianpeng Ma)
+* bluestore: os/bluestore: fix and unify log output on allocation failure (`pr#25335 <https://github.com/ceph/ceph/pull/25335>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix assertion in StupidAllocator::get_fragmentation (`pr#23606 <https://github.com/ceph/ceph/pull/23606>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix bloom filter num entry miscalculation in repairer (`issue#25001 <http://tracker.ceph.com/issues/25001>`_, `pr#24076 <https://github.com/ceph/ceph/pull/24076>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix bluefs extent miscalculations on small slow device (`pr#22563 <https://github.com/ceph/ceph/pull/22563>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix race between remove_collection and object removals (`pr#23257 <https://github.com/ceph/ceph/pull/23257>`_, Igor Fedotov)
+* bluestore: os/bluestore: fixup access a destroy cond cause deadlock or undefine behavior (`pr#25659 <https://github.com/ceph/ceph/pull/25659>`_, linbing)
+* bluestore: os/bluestore: introduce new BlueFS perf counter to track the amount of (`pr#22086 <https://github.com/ceph/ceph/pull/22086>`_, Igor Fedotov)
+* bluestore: os/bluestore/KernelDevice: misc cleanup (`pr#21491 <https://github.com/ceph/ceph/pull/21491>`_, Jianpeng Ma)
+* bluestore: os/bluestore/KernelDevice: use flock(2) for block device lock (`issue#38150 <http://tracker.ceph.com/issues/38150>`_, `pr#26245 <https://github.com/ceph/ceph/pull/26245>`_, Sage Weil)
+* bluestore: os/bluestore: misc cleanup (`pr#22472 <https://github.com/ceph/ceph/pull/22472>`_, Jianpeng Ma)
+* bluestore: os/bluestore: Only use F_SET_FILE_RW_HINT when available (`pr#26431 <https://github.com/ceph/ceph/pull/26431>`_, Willem Jan Withagen)
+* bluestore: os/bluestore: Only use ``WRITE_LIFE_`` when available (`pr#25735 <https://github.com/ceph/ceph/pull/25735>`_, Willem Jan Withagen)
+* bluestore: os/bluestore: remove redundant fault_range (`pr#22898 <https://github.com/ceph/ceph/pull/22898>`_, Jianpeng Ma)
+* bluestore: os/bluestore: remove useless condtion (`pr#22335 <https://github.com/ceph/ceph/pull/22335>`_, Jianpeng Ma)
+* bluestore: os/bluestore: simplify and fix SharedBlob::put() (`issue#24211 <http://tracker.ceph.com/issues/24211>`_, `pr#22123 <https://github.com/ceph/ceph/pull/22123>`_, Sage Weil)
+* bluestore: os/bluestore: support for FreeBSD (`pr#25608 <https://github.com/ceph/ceph/pull/25608>`_, Alan Somers, Kefu Chai)
+* bluestore: osd/osd_types: fix pg_t::contains() to check pool id too (`issue#32731 <http://tracker.ceph.com/issues/32731>`_, `pr#24085 <https://github.com/ceph/ceph/pull/24085>`_, Sage Weil)
+* bluestore: os/objectstore: add a new op OP_CREATE (`pr#22385 <https://github.com/ceph/ceph/pull/22385>`_, Jianpeng Ma)
+* bluestore,performance: common/PriorityCache: First Step toward priority based caching (`pr#22009 <https://github.com/ceph/ceph/pull/22009>`_, Mark Nelson)
+* bluestore,performance: os/bluestore: allocator pruning (`pr#21854 <https://github.com/ceph/ceph/pull/21854>`_, Igor Fedotov)
+* bluestore,performance: os/bluestore/BlueFS: reduce bufferlist rebuilds during WAL writes (`pr#21689 <https://github.com/ceph/ceph/pull/21689>`_, Piotr Dałek)
+* bluestore,performance: os/bluestore: use the monotonic clock for perf counters latencies (`pr#22121 <https://github.com/ceph/ceph/pull/22121>`_, Mohamad Gebai)
+* bluestore: silence Clang warning on possible uninitialize usuage (`pr#25702 <https://github.com/ceph/ceph/pull/25702>`_, Willem Jan Withagen)
+* bluestore: spdk: fix ceph-osd crash when activate SPDK (`issue#24371 <http://tracker.ceph.com/issues/24371>`_, `pr#22356 <https://github.com/ceph/ceph/pull/22356>`_, tone-zhang)
+* bluestore: test/fio: add option single_pool_mode in ceph-bluestore.fio (`pr#21929 <https://github.com/ceph/ceph/pull/21929>`_, Jianpeng Ma)
+* bluestore,tests: test/objectstore: fix random generator in allocator_bench (`pr#22544 <https://github.com/ceph/ceph/pull/22544>`_, Igor Fedotov)
+* bluestore,tools: os/bluestore: allow ceph-bluestore-tool to coalesce, add and migrate BlueFS backing volumes (`pr#23103 <https://github.com/ceph/ceph/pull/23103>`_, Igor Fedotov)
+* bluestore,tools: tools/ceph-bluestore-tool: avoid mon/config access when calling global… (`pr#22085 <https://github.com/ceph/ceph/pull/22085>`_, Igor Fedotov)
+* build/ops: Add new OpenSUSE Leap id for install-deps.sh (`issue#25064 <http://tracker.ceph.com/issues/25064>`_, `pr#22793 <https://github.com/ceph/ceph/pull/22793>`_, Kyr Shatskyy)
+* build/ops: arch/arm: Allow ceph_crc32c_aarch64 to be chosen only if it is compil… (`pr#24126 <https://github.com/ceph/ceph/pull/24126>`_, David Wang)
+* build/ops:  auth: do not use GSS/KRB5 if ! HAVE_GSSAPI (`pr#25460 <https://github.com/ceph/ceph/pull/25460>`_, Kefu Chai)
+* build/ops: build: 32 bit architecture fixes (`pr#23485 <https://github.com/ceph/ceph/pull/23485>`_, James Page)
+* build/ops: build: further removal of `subman` configuration (`issue#38261 <http://tracker.ceph.com/issues/38261>`_, `pr#26368 <https://github.com/ceph/ceph/pull/26368>`_, Alfredo Deza)
+* build/ops: build: LLVM ld does not like the versioning scheme (`pr#26801 <https://github.com/ceph/ceph/pull/26801>`_, Willem Jan Withagen)
+* build/ops: ceph-create-keys: Misc Python 3 fixes (`issue#37641 <http://tracker.ceph.com/issues/37641>`_, `pr#25411 <https://github.com/ceph/ceph/pull/25411>`_, James Page)
+* build/ops,cephfs: deb,rpm: fix python-cephfs dependencies (`issue#24919 <http://tracker.ceph.com/issues/24919>`_, `issue#24918 <http://tracker.ceph.com/issues/24918>`_, `pr#23043 <https://github.com/ceph/ceph/pull/23043>`_, Kefu Chai)
+* build/ops: ceph.in: Add support for python 3 (`pr#24739 <https://github.com/ceph/ceph/pull/24739>`_, Tiago Melo)
+* build/ops: ceph.spec.in: Don't use noarch for mgr module subpackages, fix /usr/lib64/ceph/mgr dir ownership (`pr#26398 <https://github.com/ceph/ceph/pull/26398>`_, Tim Serong)
+* build/ops: change ceph-mgr package depency from py-bcrypt to python2-bcrypt (`issue#27206 <http://tracker.ceph.com/issues/27206>`_, `pr#23648 <https://github.com/ceph/ceph/pull/23648>`_, Konstantin Sakhinov)
+* build/ops: civetweb: pull up to ceph-master (`pr#26515 <https://github.com/ceph/ceph/pull/26515>`_, Abhishek Lekshmanan)
+* build/ops: cmake,do_freebsd.sh: disable rdma features (`pr#22752 <https://github.com/ceph/ceph/pull/22752>`_, Kefu Chai)
+* build/ops: cmake/modules/BuildDPDK.cmake: Build required DPDK libraries (`issue#36341 <http://tracker.ceph.com/issues/36341>`_, `pr#24487 <https://github.com/ceph/ceph/pull/24487>`_, Brad Hubbard)
+* build/ops: cmake/modules/BuildRocksDB.cmake: enable compressions for rocksdb (`issue#24025 <http://tracker.ceph.com/issues/24025>`_, `pr#22181 <https://github.com/ceph/ceph/pull/22181>`_, Kefu Chai)
+* build/ops: cmake,rgw: make amqp support optional (`pr#26555 <https://github.com/ceph/ceph/pull/26555>`_, Kefu Chai)
+* build/ops: cmake,rpm,deb: install mgr plugins into /usr/share/ceph/mgr (`pr#26446 <https://github.com/ceph/ceph/pull/26446>`_, Kefu Chai)
+* build/ops: cmake,seastar: pick up latest seastar (`pr#25474 <https://github.com/ceph/ceph/pull/25474>`_, Kefu Chai)
+* build/ops,common: compressor: Fix build of Brotli Compressor (`pr#24967 <https://github.com/ceph/ceph/pull/24967>`_, BI SHUN KE)
+* build/ops,common,core: test: make readable.sh fail if it doesn't run anything (`pr#24812 <https://github.com/ceph/ceph/pull/24812>`_, Greg Farnum)
+* build/ops,core: cmake,common,filestore: silence gcc-8 warnings/errors (`pr#21837 <https://github.com/ceph/ceph/pull/21837>`_, Kefu Chai)
+* build/ops,core,rbd: include/memory.h: remove memory.h (`pr#22690 <https://github.com/ceph/ceph/pull/22690>`_, Kefu Chai)
+* build/ops,core: systemd: only restart 3 times in 30 minutes, as fast as possible (`issue#24368 <http://tracker.ceph.com/issues/24368>`_, `pr#22349 <https://github.com/ceph/ceph/pull/22349>`_, Greg Farnum)
+* build/ops,core,tests: objectstore/test/fio: Fixed fio compilation when tcmalloc is used (`pr#23962 <https://github.com/ceph/ceph/pull/23962>`_, Adam Kupczyk)
+* build/ops: credits.sh: Ignore package-lock.json and .xlf files (`pr#24762 <https://github.com/ceph/ceph/pull/24762>`_, Tiago Melo)
+* build/ops: deb: drop redundant ceph-common recommends (`pr#20133 <https://github.com/ceph/ceph/pull/20133>`_, Nathan Cutler)
+* build/ops: debian/control: change Architecture python plugins to "all" (`pr#26377 <https://github.com/ceph/ceph/pull/26377>`_, Kefu Chai)
+* build/ops: debian/control: require fuse for ceph-fuse (`issue#21057 <http://tracker.ceph.com/issues/21057>`_, `pr#23675 <https://github.com/ceph/ceph/pull/23675>`_, Thomas Serlin)
+* build/ops: debian: correct ceph-common relationship with older radosgw package (`pr#24996 <https://github.com/ceph/ceph/pull/24996>`_, Matthew Vernon)
+* build/ops: debian: drop '-DUSE_CRYPTOPP=OFF' from cmake options (`pr#22471 <https://github.com/ceph/ceph/pull/22471>`_, Kefu Chai)
+* build/ops: debian: librados-dev should replace librados2-dev (`pr#25916 <https://github.com/ceph/ceph/pull/25916>`_, Kefu Chai)
+* build/ops: debian/rules: fix ceph-mgr .pyc files left behind (`issue#26883 <http://tracker.ceph.com/issues/26883>`_, `pr#23615 <https://github.com/ceph/ceph/pull/23615>`_, Dan Mick)
+* build/ops: deb,rpm,do_cmake: switch to cmake3 (`pr#22896 <https://github.com/ceph/ceph/pull/22896>`_, Kefu Chai)
+* build/ops: dmclock, cmake: sync up with ceph/dmclock, dmclock related cleanups (`issue#26998 <http://tracker.ceph.com/issues/26998>`_, `pr#23643 <https://github.com/ceph/ceph/pull/23643>`_, Kefu Chai)
+* build/ops: dmclock: update dmclock submodule sha1 to tip of ceph/dmclock.git master (`pr#23837 <https://github.com/ceph/ceph/pull/23837>`_, Ricardo Dias)
+* build/ops: do_cmake.sh: automate py3 build options for certain distros (`pr#25205 <https://github.com/ceph/ceph/pull/25205>`_, Nathan Cutler)
+* build/ops: do_cmake.sh: SUSE builds need WITH_RADOSGW_AMQP_ENDPOINT=OFF (`pr#26695 <https://github.com/ceph/ceph/pull/26695>`_, Nathan Cutler)
+* build/ops: do_freebsd.sh: FreeBSD building needs the llvm linker (`pr#25247 <https://github.com/ceph/ceph/pull/25247>`_, Willem Jan Withagen)
+* build/ops: dout: declare dpp using `decltype(auto)` instead of `auto` (`pr#22207 <https://github.com/ceph/ceph/pull/22207>`_, Kefu Chai)
+* build/ops: dpdk: drop dpdk submodule (`issue#24032 <http://tracker.ceph.com/issues/24032>`_, `pr#21856 <https://github.com/ceph/ceph/pull/21856>`_, Kefu Chai)
+* build/ops: examples/Makefile: add -Wno-unused-parameter to avoid compile error (`pr#23581 <https://github.com/ceph/ceph/pull/23581>`_, You Ji)
+* build/ops: Improving make check reliability (`pr#22441 <https://github.com/ceph/ceph/pull/22441>`_, Erwan Velu)
+* build/ops: include: define errnos if not defined for better portablity (`pr#25302 <https://github.com/ceph/ceph/pull/25302>`_, Willem Jan Withagen)
+* build/ops: install-deps: check the exit status for the $builddepcmd (`pr#22682 <https://github.com/ceph/ceph/pull/22682>`_, Yunchuan Wen)
+* build/ops: install-deps: do not specify unknown options (`pr#24315 <https://github.com/ceph/ceph/pull/24315>`_, Kefu Chai)
+* build/ops: install-deps: install setuptools before upgrading virtualenv (`pr#25039 <https://github.com/ceph/ceph/pull/25039>`_, Kefu Chai)
+* build/ops: install-deps: nuke wheelhouse if it's stale (`pr#22028 <https://github.com/ceph/ceph/pull/22028>`_, Kefu Chai)
+* build/ops: install-deps,run-make-check: use ceph-libboost repo (`issue#25186 <http://tracker.ceph.com/issues/25186>`_, `pr#23995 <https://github.com/ceph/ceph/pull/23995>`_, Kefu Chai)
+* build/ops: install-deps.sh: Add Kerberos requirement for FreeBSD (`pr#25688 <https://github.com/ceph/ceph/pull/25688>`_, Willem Jan Withagen)
+* build/ops: install-deps.sh: disable centos-sclo-rh-source (`issue#37707 <http://tracker.ceph.com/issues/37707>`_, `pr#25629 <https://github.com/ceph/ceph/pull/25629>`_, Brad Hubbard)
+* build/ops: install-deps.sh: fix gcc detection and install pre-built libboost on bionic (`pr#25169 <https://github.com/ceph/ceph/pull/25169>`_, Changcheng Liu, Kefu Chai)
+* build/ops: install-deps.sh: fix installing gcc on ubuntu when no old compiler (`pr#22488 <https://github.com/ceph/ceph/pull/22488>`_, Tomasz Setkowski)
+* build/ops: install-deps.sh: import ubuntu-toolchain-r's key without keyserver (`pr#22964 <https://github.com/ceph/ceph/pull/22964>`_, Kefu Chai)
+* build/ops: install-deps.sh: install libtool-ltdl-devel for building python-saml (`pr#25071 <https://github.com/ceph/ceph/pull/25071>`_, Kefu Chai)
+* build/ops: install-deps.sh: refrain from installing/using lsb_release, and other cleanup (`issue#18163 <http://tracker.ceph.com/issues/18163>`_, `pr#23361 <https://github.com/ceph/ceph/pull/23361>`_, Nathan Cutler)
+* build/ops: install-deps.sh: Remove CR repo (`issue#13997 <http://tracker.ceph.com/issues/13997>`_, `pr#25211 <https://github.com/ceph/ceph/pull/25211>`_, Brad Hubbard, Alfredo Deza)
+* build/ops:  install-deps.sh: selectively install dependencies (`pr#26402 <https://github.com/ceph/ceph/pull/26402>`_, Kefu Chai)
+* build/ops: install-deps.sh: set with_seastar (`pr#23079 <https://github.com/ceph/ceph/pull/23079>`_, Nathan Cutler)
+* build/ops: install-deps.sh: support install gcc7 in xenial aarch64 (`pr#22451 <https://github.com/ceph/ceph/pull/22451>`_, Yunchuan Wen)
+* build/ops: install-deps.sh: Update python requirements for FreeBSD (`pr#25245 <https://github.com/ceph/ceph/pull/25245>`_, Willem Jan Withagen)
+* build/ops: install-deps.sh: use the latest setuptools (`pr#26156 <https://github.com/ceph/ceph/pull/26156>`_, Kefu Chai)
+* build/ops: install-deps: s/openldap-client/openldap24-client/ (`pr#23912 <https://github.com/ceph/ceph/pull/23912>`_, Kefu Chai)
+* build/ops: libradosstriper: conditional compile (`pr#21983 <https://github.com/ceph/ceph/pull/21983>`_, Jesse Williamson)
+* build/ops: make-debs.sh: clean dir to allow building deb packages multiple times (`pr#25177 <https://github.com/ceph/ceph/pull/25177>`_, Changcheng Liu)
+* build/ops: man: skip directive starting with ".." (`pr#23580 <https://github.com/ceph/ceph/pull/23580>`_, Kefu Chai)
+* build/ops,mgr: build: mgr: check for python's ssl version linkage (`issue#24282 <http://tracker.ceph.com/issues/24282>`_, `pr#22659 <https://github.com/ceph/ceph/pull/22659>`_, Kefu Chai, Abhishek Lekshmanan)
+* build/ops,mgr: cmake,deb,rpm: remove cython 0.29's subinterpreter check, re-enable build with cython 0.29+ (`pr#25585 <https://github.com/ceph/ceph/pull/25585>`_, Tim Serong)
+* build/ops: mgr/dashboard: Add html-linter (`pr#24273 <https://github.com/ceph/ceph/pull/24273>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Add i18n validation script (`pr#25179 <https://github.com/ceph/ceph/pull/25179>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Add package-lock.json (`pr#23285 <https://github.com/ceph/ceph/pull/23285>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Disable showing xi18n's progress (`pr#25427 <https://github.com/ceph/ceph/pull/25427>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Fix run-frontend-e2e-tests.sh (`pr#25157 <https://github.com/ceph/ceph/pull/25157>`_, Tiago Melo)
+* build/ops: mgr/dashboard: fix the version of all frontend dependencies (`pr#22712 <https://github.com/ceph/ceph/pull/22712>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Remove angular build progress logs during cmake (`pr#23115 <https://github.com/ceph/ceph/pull/23115>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Update Node.js to current LTS (`pr#24932 <https://github.com/ceph/ceph/pull/24932>`_, Tiago Melo)
+* build/ops: mgr/dashboard: Update node version (`pr#22639 <https://github.com/ceph/ceph/pull/22639>`_, Tiago Melo)
+* build/ops: mgr/diskprediction: Replace local predictor model file (`pr#24484 <https://github.com/ceph/ceph/pull/24484>`_, Rick Chen)
+* build/ops,mgr: mgr/dashboard: Fix building under FreeBSD (`pr#22562 <https://github.com/ceph/ceph/pull/22562>`_, Willem Jan Withagen)
+* build/ops: move dmclock subtree into submodule (`pr#21651 <https://github.com/ceph/ceph/pull/21651>`_, Danny Al-Gaaf)
+* build/ops,pybind: ceph: do not raise StopIteration within generator (`pr#25400 <https://github.com/ceph/ceph/pull/25400>`_, Jason Dillaman)
+* build/ops,rbd: osd,mon,pybind: Make able to compile with Clang (`pr#21861 <https://github.com/ceph/ceph/pull/21861>`_, Adam C. Emerson)
+* build/ops,rbd: selinux: add support for ceph iscsi (`pr#24936 <https://github.com/ceph/ceph/pull/24936>`_, Mike Christie)
+* build/ops,rbd: systemd: enable ceph-rbd-mirror.target (`pr#24935 <https://github.com/ceph/ceph/pull/24935>`_, Sébastien Han)
+* build/ops,rgw: build/rgw: unittest_rgw_dmclock_scheduler does not need Boost_LIBRARIES (`pr#26799 <https://github.com/ceph/ceph/pull/26799>`_, Willem Jan Withagen)
+* build/ops,rgw: cls: build cls_otp only WITH_RADOSGW (`pr#22548 <https://github.com/ceph/ceph/pull/22548>`_, Piotr Dałek)
+* build/ops,rgw: deb,rpm: package librgw_admin_user.{h,so.\*} (`pr#22205 <https://github.com/ceph/ceph/pull/22205>`_, Kefu Chai)
+* build/ops: rocksdb: sync with upstream (`issue#23653 <http://tracker.ceph.com/issues/23653>`_, `pr#22236 <https://github.com/ceph/ceph/pull/22236>`_, Kefu Chai)
+* build/ops: rpm: bump up required GCC version to 7.3.1 (`pr#24130 <https://github.com/ceph/ceph/pull/24130>`_, Kefu Chai)
+* build/ops: rpm,deb: remove python-jinja2 dependency (`pr#26379 <https://github.com/ceph/ceph/pull/26379>`_, Kefu Chai)
+* build/ops: rpm: do not exclude s390x build on openSUSE (`pr#26268 <https://github.com/ceph/ceph/pull/26268>`_, Nathan Cutler)
+* build/ops: rpm: Fix Fedora error "No matching package to install: 'Cython3'" (`issue#35831 <http://tracker.ceph.com/issues/35831>`_, `pr#23993 <https://github.com/ceph/ceph/pull/23993>`_, Brad Hubbard)
+* build/ops: rpm: fix libradospp-devel runtime dependency (`pr#25491 <https://github.com/ceph/ceph/pull/25491>`_, Nathan Cutler)
+* build/ops: rpm: fix seastar build dependencies for SUSE (`pr#23089 <https://github.com/ceph/ceph/pull/23089>`_, Nathan Cutler)
+* build/ops: rpm: fix seastar build dependencies (`pr#23386 <https://github.com/ceph/ceph/pull/23386>`_, Nathan Cutler)
+* build/ops: rpm: fix xmlsec1 build dependency for dashboard make check (`pr#26119 <https://github.com/ceph/ceph/pull/26119>`_, Nathan Cutler)
+* build/ops: rpm: Install python2-Cython on f28 (`pr#26756 <https://github.com/ceph/ceph/pull/26756>`_, Brad Hubbard)
+* build/ops: rpm: make ceph-grafana-dashboards own its directories (`issue#37485 <http://tracker.ceph.com/issues/37485>`_, `pr#25347 <https://github.com/ceph/ceph/pull/25347>`_, Nathan Cutler, Tim Serong)
+* build/ops: rpm: make Python dependencies somewhat less confusing (`pr#25963 <https://github.com/ceph/ceph/pull/25963>`_, Nathan Cutler)
+* build/ops: rpm: make sudo a build dependency (`pr#23077 <https://github.com/ceph/ceph/pull/23077>`_, Nathan Cutler)
+* build/ops: rpm: package crypto libraries for all archs (`pr#26202 <https://github.com/ceph/ceph/pull/26202>`_, Nathan Cutler)
+* build/ops: rpm: Package grafana dashboards (`pr#24735 <https://github.com/ceph/ceph/pull/24735>`_, Boris Ranto)
+* build/ops: rpm: provide files moved from ceph-test … (`issue#22558 <http://tracker.ceph.com/issues/22558>`_, `pr#20401 <https://github.com/ceph/ceph/pull/20401>`_, Nathan Cutler)
+* build/ops: rpm: RHEL 8 fixes (`pr#26520 <https://github.com/ceph/ceph/pull/26520>`_, Ken Dreyer)
+* build/ops: rpm: RHEL 8 needs Python 3 build (`pr#25223 <https://github.com/ceph/ceph/pull/25223>`_, Nathan Cutler)
+* build/ops: rpm: stop install-deps.sh clobbering spec file Python build setting (`issue#37301 <http://tracker.ceph.com/issues/37301>`_, `pr#25181 <https://github.com/ceph/ceph/pull/25181>`_, Nathan Cutler, Brad Hubbard)
+* build/ops: rpm: Use hardened LDFLAGS (`issue#36316 <http://tracker.ceph.com/issues/36316>`_, `pr#24425 <https://github.com/ceph/ceph/pull/24425>`_, Boris Ranto)
+* build/ops: rpm: use updated gperftools (`issue#35969 <http://tracker.ceph.com/issues/35969>`_, `pr#24124 <https://github.com/ceph/ceph/pull/24124>`_, Kefu Chai)
+* build/ops: rpm: Use updated gperftools-libs at runtime (`issue#36508 <http://tracker.ceph.com/issues/36508>`_, `pr#24652 <https://github.com/ceph/ceph/pull/24652>`_, Brad Hubbard)
+* build/ops: run-make-check: enable --with-seastar option (`pr#22809 <https://github.com/ceph/ceph/pull/22809>`_, Kefu Chai)
+* build/ops: run-make-check: set WITH_SEASTAR with a non-empty string (`pr#23108 <https://github.com/ceph/ceph/pull/23108>`_, Kefu Chai)
+* build/ops: run-make-check.sh: Adding ccache tuning for the CI (`pr#22847 <https://github.com/ceph/ceph/pull/22847>`_, Erwan Velu)
+* build/ops: run-make-check.sh: ccache goodness for everyone (`issue#24817 <http://tracker.ceph.com/issues/24817>`_, `issue#24777 <http://tracker.ceph.com/issues/24777>`_, `pr#22867 <https://github.com/ceph/ceph/pull/22867>`_, Nathan Cutler)
+* build/ops: run-make-check: should use sudo for running sysctl (`pr#23708 <https://github.com/ceph/ceph/pull/23708>`_, Kefu Chai)
+* build/ops: run-make-check: Showing configuration before the build (`pr#23609 <https://github.com/ceph/ceph/pull/23609>`_, Erwan Velu)
+* build/ops: seastar: lower the required yaml-cpp version to 0.5.1 (`pr#23255 <https://github.com/ceph/ceph/pull/23255>`_, Kefu Chai)
+* build/ops: seastar: pickup the change to link pthread (`pr#25671 <https://github.com/ceph/ceph/pull/25671>`_, Kefu Chai)
+* build/ops: selinux: Allow ceph to execute ldconfig (`pr#20118 <https://github.com/ceph/ceph/pull/20118>`_, Boris Ranto)
+* build/ops: spdk: update to latest spdk-18.05 branch (`pr#22547 <https://github.com/ceph/ceph/pull/22547>`_, Kefu Chai)
+* build/ops: spec: requires ceph base instead of common (`issue#37620 <http://tracker.ceph.com/issues/37620>`_, `pr#25503 <https://github.com/ceph/ceph/pull/25503>`_, Sébastien Han)
+* build/ops: test: move ceph-dencoder to src/tools (`pr#23228 <https://github.com/ceph/ceph/pull/23228>`_, Kefu Chai)
+* build/ops: test,qa: s/.libs/lib/ (`pr#20734 <https://github.com/ceph/ceph/pull/20734>`_, Kefu Chai)
+* build/ops,tests: cmake,run-make-check: always enable WITH_GTEST_PARALLEL (`pr#23382 <https://github.com/ceph/ceph/pull/23382>`_, Kefu Chai)
+* build/ops,tests: deb,rpm,qa: split dashboard package (`pr#26380 <https://github.com/ceph/ceph/pull/26380>`_, Kefu Chai)
+* build/ops,tests: mgr/dashboard: Fix localStorage problem in Jest (`pr#23281 <https://github.com/ceph/ceph/pull/23281>`_, Tiago Melo)
+* build/ops,tests: mgr/dashboard: Object Gateway user configuration (`pr#25494 <https://github.com/ceph/ceph/pull/25494>`_, Laura Paduano)
+* build/ops,tests: src/test: Using gtest-parallel to speedup unittests (`pr#22577 <https://github.com/ceph/ceph/pull/22577>`_, Kefu Chai, Erwan Velu)
+* build/ops,tests: tests/fio: fix build failures and ensure this is covered by run-make-check.sh (`pr#23231 <https://github.com/ceph/ceph/pull/23231>`_, Kefu Chai, Igor Fedotov)
+* build/ops,tests: tests/qa: Adding $ distro mix - rgw (`pr#21932 <https://github.com/ceph/ceph/pull/21932>`_, Yuri Weinstein)
+* build/ops,tests: tools/ceph-dencoder: conditionally link against mds (`pr#25255 <https://github.com/ceph/ceph/pull/25255>`_, Kefu Chai)
+* build/ops,tools: tool: link rbd-ggate against librados-cxx (`pr#24901 <https://github.com/ceph/ceph/pull/24901>`_, Willem Jan Withagen)
+* ceph-disk: get_partition_dev() should fail until get_dev_path(partnam… (`pr#21415 <https://github.com/ceph/ceph/pull/21415>`_, Erwan Velu)
+* cephfs: doc/releases: update CephFS mimic notes (`issue#23775 <http://tracker.ceph.com/issues/23775>`_, `pr#22232 <https://github.com/ceph/ceph/pull/22232>`_, Patrick Donnelly)
+* cephfs: mgr/dashboard: NFS Ganesha management REST API (`pr#25918 <https://github.com/ceph/ceph/pull/25918>`_, Lenz Grimmer, Ricardo Dias, Jeff Layton)
+* cephfs,mgr,pybind: pybind/mgr: Unified bits of volumes and orchestrator (`pr#25492 <https://github.com/ceph/ceph/pull/25492>`_, Sebastian Wagner)
+* cephfs,mon: MDSMonitor: silence unable to load metadata (`pr#25693 <https://github.com/ceph/ceph/pull/25693>`_, Song Shun)
+* cephfs,mon: mon/MDSMonitor: do not send redundant MDS health messages to cluster log (`issue#24308 <http://tracker.ceph.com/issues/24308>`_, `pr#22252 <https://github.com/ceph/ceph/pull/22252>`_, Sage Weil)
+* cephfs: qa: fix symlink (`pr#23997 <https://github.com/ceph/ceph/pull/23997>`_, Patrick Donnelly)
+* cephfs,rbd: osdc: Fix the wrong BufferHead offset (`pr#22495 <https://github.com/ceph/ceph/pull/22495>`_, dongdong tao)
+* cephfs,rbd: osdc: optimize the code doing the BufferHead mapping (`pr#22509 <https://github.com/ceph/ceph/pull/22509>`_, dongdong tao)
+* cephfs,rbd: osdc: reduce ObjectCacher's memory fragments (`issue#36192 <http://tracker.ceph.com/issues/36192>`_, `pr#24297 <https://github.com/ceph/ceph/pull/24297>`_, "Yan, Zheng")
+* cephfs,tests: qa: fix run call args (`issue#36450 <http://tracker.ceph.com/issues/36450>`_, `pr#24597 <https://github.com/ceph/ceph/pull/24597>`_, Patrick Donnelly)
+* cephfs,tests: qa: install python3-cephfs for fs suite (`pr#23411 <https://github.com/ceph/ceph/pull/23411>`_, Kefu Chai)
+* cephfs,tests: qa/suites/powercycle: whitelist MDS_SLOW_REQUEST (`pr#23151 <https://github.com/ceph/ceph/pull/23151>`_, Neha Ojha)
+* cephfs,tests: qa/workunits/suites/pjd.sh: use correct dir name (`pr#22233 <https://github.com/ceph/ceph/pull/22233>`_, Neha Ojha)
+* ceph-volume:  activate option --auto-detect-objectstore respects --no-systemd (`issue#36249 <http://tracker.ceph.com/issues/36249>`_, `pr#24355 <https://github.com/ceph/ceph/pull/24355>`_, Alfredo Deza)
+* ceph-volume: Adapt code to support Python3 (`pr#25324 <https://github.com/ceph/ceph/pull/25324>`_, Volker Theile)
+* ceph-volume: add --all flag to simple activate (`pr#26225 <https://github.com/ceph/ceph/pull/26225>`_, Jan Fajerski)
+* ceph-volume add a __release__ string, to help version-conditional calls (`issue#25171 <http://tracker.ceph.com/issues/25171>`_, `pr#23332 <https://github.com/ceph/ceph/pull/23332>`_, Alfredo Deza)
+* ceph-volume: add inventory command (`issue#24972 <http://tracker.ceph.com/issues/24972>`_, `pr#24859 <https://github.com/ceph/ceph/pull/24859>`_, Jan Fajerski)
+* ceph-volume: Additional work on ceph-volume to add some choose_disk capabilities (`issue#36446 <http://tracker.ceph.com/issues/36446>`_, `pr#24504 <https://github.com/ceph/ceph/pull/24504>`_, Erwan Velu)
+* ceph-volume add new ceph-handlers role from ceph-ansible (`issue#36251 <http://tracker.ceph.com/issues/36251>`_, `pr#24336 <https://github.com/ceph/ceph/pull/24336>`_, Alfredo Deza)
+* ceph-volume: adds a --prepare flag to `lvm batch` (`issue#36363 <http://tracker.ceph.com/issues/36363>`_, `pr#24587 <https://github.com/ceph/ceph/pull/24587>`_, Andrew Schoen)
+* ceph-volume: add space between words (`pr#26246 <https://github.com/ceph/ceph/pull/26246>`_, Sébastien Han)
+* ceph-volume: adds test for `ceph-volume lvm list /dev/sda` (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `issue#24957 <http://tracker.ceph.com/issues/24957>`_, `pr#23348 <https://github.com/ceph/ceph/pull/23348>`_, Andrew Schoen)
+* ceph-volume: Add unit test (`pr#25321 <https://github.com/ceph/ceph/pull/25321>`_, Volker Theile)
+* ceph-volume: allow to specify --cluster-fsid instead of reading from ceph.conf (`issue#26953 <http://tracker.ceph.com/issues/26953>`_, `pr#24407 <https://github.com/ceph/ceph/pull/24407>`_, Alfredo Deza)
+* ceph-volume: an OSD ID must be exist and be destroyed before reuse (`pr#23093 <https://github.com/ceph/ceph/pull/23093>`_, Andrew Schoen, Ron Allred)
+* ceph-volume  batch: allow journal+block.db sizing on the CLI (`issue#36088 <http://tracker.ceph.com/issues/36088>`_, `pr#24201 <https://github.com/ceph/ceph/pull/24201>`_, Alfredo Deza)
+* ceph-volume batch: allow --osds-per-device, default it to 1 (`issue#35913 <http://tracker.ceph.com/issues/35913>`_, `pr#24060 <https://github.com/ceph/ceph/pull/24060>`_, Alfredo Deza)
+* ceph-volume  batch carve out lvs for bluestore (`pr#24019 <https://github.com/ceph/ceph/pull/24019>`_, Alfredo Deza)
+* ceph-volume batch command (`issue#24492 <http://tracker.ceph.com/issues/24492>`_, `pr#23075 <https://github.com/ceph/ceph/pull/23075>`_, Alfredo Deza)
+* ceph-volume:  batch tests for mixed-type of devices (`issue#35535 <http://tracker.ceph.com/issues/35535>`_, `issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#23963 <https://github.com/ceph/ceph/pull/23963>`_, Alfredo Deza)
+* ceph-volume custom cluster names fail on filestore trigger (`issue#27210 <http://tracker.ceph.com/issues/27210>`_, `pr#24251 <https://github.com/ceph/ceph/pull/24251>`_, Alfredo Deza)
+* ceph-volume: do not pin the testinfra version for the simple tests (`pr#23268 <https://github.com/ceph/ceph/pull/23268>`_, Andrew Schoen)
+* ceph-volume: do not send (lvm) stderr/stdout to the terminal, use the logfile (`issue#36492 <http://tracker.ceph.com/issues/36492>`_, `pr#24738 <https://github.com/ceph/ceph/pull/24738>`_, Alfredo Deza)
+* ceph-volume do not use stdin in luminous (`issue#25173 <http://tracker.ceph.com/issues/25173>`_, `pr#23355 <https://github.com/ceph/ceph/pull/23355>`_, Alfredo Deza)
+* ceph-volume: don't create osd['block.db'] by default (`issue#38472 <http://tracker.ceph.com/issues/38472>`_, `pr#26627 <https://github.com/ceph/ceph/pull/26627>`_, Jan Fajerski)
+* ceph-volume: earlier detection for --journal and --filestore flag requirements (`issue#24794 <http://tracker.ceph.com/issues/24794>`_, `pr#24150 <https://github.com/ceph/ceph/pull/24150>`_, Alfredo Deza)
+* ceph-volume: enable device discards (`issue#36532 <http://tracker.ceph.com/issues/36532>`_, `pr#24676 <https://github.com/ceph/ceph/pull/24676>`_, Jonas Jelten)
+* ceph-volume enable  --no-systemd flag for simple sub-command (`issue#36470 <http://tracker.ceph.com/issues/36470>`_, `pr#24998 <https://github.com/ceph/ceph/pull/24998>`_, Alfredo Deza)
+* ceph-volume: enable the ceph-osd during lvm activation (`issue#24152 <http://tracker.ceph.com/issues/24152>`_, `pr#23321 <https://github.com/ceph/ceph/pull/23321>`_, Dan van der Ster)
+* ceph-volume ensure encoded bytes are always used (`issue#24993 <http://tracker.ceph.com/issues/24993>`_, `pr#23289 <https://github.com/ceph/ceph/pull/23289>`_, Alfredo Deza)
+* ceph-volume: error on commands that need ceph.conf to operate (`issue#23941 <http://tracker.ceph.com/issues/23941>`_, `pr#22724 <https://github.com/ceph/ceph/pull/22724>`_, Andrew Schoen)
+* ceph-volume  expand auto engine for multiple devices on filestore (`issue#24553 <http://tracker.ceph.com/issues/24553>`_, `pr#23731 <https://github.com/ceph/ceph/pull/23731>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume:  expand auto engine for single type devices on filestore (`issue#24960 <http://tracker.ceph.com/issues/24960>`_, `pr#23532 <https://github.com/ceph/ceph/pull/23532>`_, Alfredo Deza)
+* ceph-volume expand on the LVM API to create multiple LVs at different sizes (`issue#24020 <http://tracker.ceph.com/issues/24020>`_, `pr#22426 <https://github.com/ceph/ceph/pull/22426>`_, Alfredo Deza)
+* ceph-volume: extract flake8 config (`pr#24674 <https://github.com/ceph/ceph/pull/24674>`_, Mehdi Abaakouk)
+* ceph-volume: fix Batch object in py3 environments (`pr#25203 <https://github.com/ceph/ceph/pull/25203>`_, Jan Fajerski)
+* ceph-volume: fix journal and filestore data size in `lvm batch --report` (`issue#36242 <http://tracker.ceph.com/issues/36242>`_, `pr#24274 <https://github.com/ceph/ceph/pull/24274>`_, Andrew Schoen)
+* ceph-volume: fix JSON output in `inventory` (`issue#37390 <http://tracker.ceph.com/issues/37390>`_, `pr#25224 <https://github.com/ceph/ceph/pull/25224>`_, Sebastian Wagner)
+* ceph-volume: Fix TypeError: join() takes exactly one argument (2 given) (`issue#37595 <http://tracker.ceph.com/issues/37595>`_, `pr#25469 <https://github.com/ceph/ceph/pull/25469>`_, Sebastian Wagner)
+* ceph-volume fix TypeError on dmcrypt when using Python3 (`pr#26034 <https://github.com/ceph/ceph/pull/26034>`_, Alfredo Deza)
+* ceph-volume  fix zap not working with LVs (`issue#35970 <http://tracker.ceph.com/issues/35970>`_, `pr#24077 <https://github.com/ceph/ceph/pull/24077>`_, Alfredo Deza)
+* ceph-volume: implement __format__ in Size to format sizes in py3 (`issue#38291 <http://tracker.ceph.com/issues/38291>`_, `pr#26401 <https://github.com/ceph/ceph/pull/26401>`_, Jan Fajerski)
+* ceph-volume initial take on auto sub-command (`pr#21803 <https://github.com/ceph/ceph/pull/21803>`_, Alfredo Deza)
+* ceph-volume: introduce class hierachy for strategies (`issue#37389 <http://tracker.ceph.com/issues/37389>`_, `pr#25238 <https://github.com/ceph/ceph/pull/25238>`_, Jan Fajerski)
+* ceph-volume:  lsblk can fail to find PARTLABEL, must fallback to blkid (`issue#36098 <http://tracker.ceph.com/issues/36098>`_, `pr#24330 <https://github.com/ceph/ceph/pull/24330>`_, Alfredo Deza)
+* ceph-volume lvm.activate conditional mon-config on prime-osd-dir (`issue#25216 <http://tracker.ceph.com/issues/25216>`_, `pr#23375 <https://github.com/ceph/ceph/pull/23375>`_, Alfredo Deza)
+* ceph-volume lvm.activate Do not search for a MON configuration (`pr#22393 <https://github.com/ceph/ceph/pull/22393>`_, Wido den Hollander)
+* ceph-volume: `lvm batch` allow extra flags (like dmcrypt) for bluestore (`issue#26862 <http://tracker.ceph.com/issues/26862>`_, `pr#23448 <https://github.com/ceph/ceph/pull/23448>`_, Alfredo Deza)
+* ceph-volume lvm.batch remove non-existent sys_api property (`issue#34310 <http://tracker.ceph.com/issues/34310>`_, `pr#23787 <https://github.com/ceph/ceph/pull/23787>`_, Alfredo Deza)
+* ceph-volume lvm.listing only include devices if they exist (`issue#24952 <http://tracker.ceph.com/issues/24952>`_, `pr#23129 <https://github.com/ceph/ceph/pull/23129>`_, Alfredo Deza)
+* ceph-volume lvm.prepare update help to indicate partitions are needed, not devices (`issue#24795 <http://tracker.ceph.com/issues/24795>`_, `pr#24394 <https://github.com/ceph/ceph/pull/24394>`_, Alfredo Deza)
+* ceph-volume: make Device hashable to allow set of Device list in py3 (`issue#38290 <http://tracker.ceph.com/issues/38290>`_, `pr#26399 <https://github.com/ceph/ceph/pull/26399>`_, Jan Fajerski)
+* ceph-volume: make `lvm batch` idempotent (`issue#26864 <http://tracker.ceph.com/issues/26864>`_, `pr#24404 <https://github.com/ceph/ceph/pull/24404>`_, Andrew Schoen)
+* ceph-volume: mark a device not available if it belongs to ceph-disk (`pr#26084 <https://github.com/ceph/ceph/pull/26084>`_, Andrew Schoen)
+* ceph-volume normalize comma to dot for string to int conversions (`issue#37442 <http://tracker.ceph.com/issues/37442>`_, `pr#25674 <https://github.com/ceph/ceph/pull/25674>`_, Alfredo Deza)
+* ceph-volume: patch Device when testing (`issue#36768 <http://tracker.ceph.com/issues/36768>`_, `pr#25063 <https://github.com/ceph/ceph/pull/25063>`_, Alfredo Deza)
+* ceph-volume process.call with stdin in Python 3 fix (`issue#24993 <http://tracker.ceph.com/issues/24993>`_, `pr#23141 <https://github.com/ceph/ceph/pull/23141>`_, Alfredo Deza)
+* ceph-volume: provide a nice errror message when missing ceph.conf (`pr#22828 <https://github.com/ceph/ceph/pull/22828>`_, Andrew Schoen)
+* ceph-volume: PVolumes.get() should return one PV when using name or uuid (`issue#24784 <http://tracker.ceph.com/issues/24784>`_, `pr#23234 <https://github.com/ceph/ceph/pull/23234>`_, Andrew Schoen)
+* ceph-volume: refuse to zap mapper devices (`issue#24504 <http://tracker.ceph.com/issues/24504>`_, `pr#22764 <https://github.com/ceph/ceph/pull/22764>`_, Andrew Schoen)
+* ceph-volume: reject devices that have existing GPT headers (`issue#27062 <http://tracker.ceph.com/issues/27062>`_, `pr#25098 <https://github.com/ceph/ceph/pull/25098>`_, Andrew Schoen)
+* ceph-volume: remove iteritems instances (`issue#38299 <http://tracker.ceph.com/issues/38299>`_, `pr#26403 <https://github.com/ceph/ceph/pull/26403>`_, Jan Fajerski)
+* ceph-volume: remove LVs when using zap --destroy (`pr#25093 <https://github.com/ceph/ceph/pull/25093>`_, Alfredo Deza)
+* ceph-volume remove version reporting from help menu (`issue#36386 <http://tracker.ceph.com/issues/36386>`_, `pr#24531 <https://github.com/ceph/ceph/pull/24531>`_, Alfredo Deza)
+* ceph-volume: rename Device property valid to available (`issue#36701 <http://tracker.ceph.com/issues/36701>`_, `pr#25007 <https://github.com/ceph/ceph/pull/25007>`_, Jan Fajerski)
+* ceph-volume: replace testinfra command with py.test (`issue#38568 <http://tracker.ceph.com/issues/38568>`_, `pr#26739 <https://github.com/ceph/ceph/pull/26739>`_, Alfredo Deza)
+* ceph-volume: Restore SELinux context (`pr#23278 <https://github.com/ceph/ceph/pull/23278>`_, Boris Ranto)
+* ceph-volume: revert partition as disk (`issue#37506 <http://tracker.ceph.com/issues/37506>`_, `pr#25390 <https://github.com/ceph/ceph/pull/25390>`_, Jan Fajerski)
+* ceph-volume: run tests without waiting on ceph repos (`pr#23697 <https://github.com/ceph/ceph/pull/23697>`_, Andrew Schoen)
+* ceph-volume: set number of osd ports in the tests (`pr#26753 <https://github.com/ceph/ceph/pull/26753>`_, Andrew Schoen)
+* ceph-volume: set permissions right before prime-osd-dir (`issue#37486 <http://tracker.ceph.com/issues/37486>`_, `pr#25477 <https://github.com/ceph/ceph/pull/25477>`_, Andrew Schoen, Alfredo Deza)
+* ceph-volume: `simple scan` will now scan all running ceph-disk OSDs (`pr#26826 <https://github.com/ceph/ceph/pull/26826>`_, Andrew Schoen)
+* ceph-volume: skip processing devices that don't exist when scanning system disks (`issue#36247 <http://tracker.ceph.com/issues/36247>`_, `pr#24372 <https://github.com/ceph/ceph/pull/24372>`_, Alfredo Deza)
+* ceph-volume: sort and align `lvm list` output (`pr#21812 <https://github.com/ceph/ceph/pull/21812>`_, Theofilos Mouratidis)
+* ceph-volume systemd import main so console_scripts work for executable (`issue#36648 <http://tracker.ceph.com/issues/36648>`_, `pr#24840 <https://github.com/ceph/ceph/pull/24840>`_, Alfredo Deza)
+* ceph-volume tests destroy osds on monitor hosts (`pr#22437 <https://github.com/ceph/ceph/pull/22437>`_, Alfredo Deza)
+* ceph-volume tests do not include admin keyring in OSD nodes (`issue#24417 <http://tracker.ceph.com/issues/24417>`_, `pr#22399 <https://github.com/ceph/ceph/pull/22399>`_, Alfredo Deza)
+* ceph-volume tests/functional add mgrs daemons to lvm tests (`issue#26879 <http://tracker.ceph.com/issues/26879>`_, `pr#23489 <https://github.com/ceph/ceph/pull/23489>`_, Alfredo Deza)
+* ceph-volume tests.functional add notario dep for ceph-ansible (`pr#22116 <https://github.com/ceph/ceph/pull/22116>`_, Alfredo Deza)
+* ceph-volume tests/functional declare ceph-ansible roles instead of importing them (`issue#37805 <http://tracker.ceph.com/issues/37805>`_, `pr#25820 <https://github.com/ceph/ceph/pull/25820>`_, Alfredo Deza)
+* ceph-volume tests.functional fix typo when stopping osd.0 in filestore (`issue#37675 <http://tracker.ceph.com/issues/37675>`_, `pr#25594 <https://github.com/ceph/ceph/pull/25594>`_, Alfredo Deza)
+* ceph-volume: tests.functional inherit SSH_ARGS from ansible (`issue#34311 <http://tracker.ceph.com/issues/34311>`_, `pr#23788 <https://github.com/ceph/ceph/pull/23788>`_, Alfredo Deza)
+* ceph-volume tests/functional run lvm list after OSD provisioning (`issue#24961 <http://tracker.ceph.com/issues/24961>`_, `pr#23116 <https://github.com/ceph/ceph/pull/23116>`_, Alfredo Deza)
+* ceph-volume tests/functional use Ansible 2.6 (`pr#23182 <https://github.com/ceph/ceph/pull/23182>`_, Alfredo Deza)
+* ceph-volume tests install ceph-ansible's requirements.txt dependencies (`issue#36672 <http://tracker.ceph.com/issues/36672>`_, `pr#24881 <https://github.com/ceph/ceph/pull/24881>`_, Alfredo Deza)
+* ceph-volume tests patch __release__ to mimic always for stdin keys (`pr#23398 <https://github.com/ceph/ceph/pull/23398>`_, Alfredo Deza)
+* ceph-volume tests.systemd update imports for systemd module (`issue#36704 <http://tracker.ceph.com/issues/36704>`_, `pr#24937 <https://github.com/ceph/ceph/pull/24937>`_, Alfredo Deza)
+* ceph-volume: test with multiple NVME drives (`issue#37409 <http://tracker.ceph.com/issues/37409>`_, `pr#25354 <https://github.com/ceph/ceph/pull/25354>`_, Andrew Schoen)
+* ceph-volume: unmount lvs correctly before zapping (`issue#24796 <http://tracker.ceph.com/issues/24796>`_, `pr#23117 <https://github.com/ceph/ceph/pull/23117>`_, Andrew Schoen)
+* ceph-volume: update testing playbook 'deploy.yml' (`pr#26397 <https://github.com/ceph/ceph/pull/26397>`_, Guillaume Abrioux)
+* ceph-volume: update version of ansible to 2.6.x for simple tests (`pr#23263 <https://github.com/ceph/ceph/pull/23263>`_, Andrew Schoen)
+* ceph-volume: use console_scripts (`issue#36601 <http://tracker.ceph.com/issues/36601>`_, `pr#24773 <https://github.com/ceph/ceph/pull/24773>`_, Mehdi Abaakouk)
+* ceph-volume: use our own testinfra suite for functional testing (`pr#26685 <https://github.com/ceph/ceph/pull/26685>`_, Andrew Schoen)
+* ceph-volume util.encryption don't push stderr to terminal (`issue#36246 <http://tracker.ceph.com/issues/36246>`_, `pr#24399 <https://github.com/ceph/ceph/pull/24399>`_, Alfredo Deza)
+* ceph-volume util.encryption robust blkid+lsblk detection of lockbox (`pr#24977 <https://github.com/ceph/ceph/pull/24977>`_, Alfredo Deza)
+* ceph-volume zap devices associated with an OSD ID and/or OSD FSID (`pr#25429 <https://github.com/ceph/ceph/pull/25429>`_, Alfredo Deza)
+* ceph-volume  zap: improve zapping to remove all partitions and all LVs, encrypted or not (`issue#37449 <http://tracker.ceph.com/issues/37449>`_, `pr#25330 <https://github.com/ceph/ceph/pull/25330>`_, Alfredo Deza)
+* cleanup: Clean up warnings (`pr#23919 <https://github.com/ceph/ceph/pull/23919>`_, Adam C. Emerson)
+* cli: dump osd-fsid as part of osd find <id> (`pr#26015 <https://github.com/ceph/ceph/pull/26015>`_, Noah Watkins)
+* cmake: add "add_npm_command()" command (`pr#22636 <https://github.com/ceph/ceph/pull/22636>`_, Kefu Chai)
+* cmake: Add cls_opt for vstart target (`pr#22538 <https://github.com/ceph/ceph/pull/22538>`_, Ali Maredia)
+* cmake: add dpdk::dpdk if dpdk is built or found (`issue#24948 <http://tracker.ceph.com/issues/24948>`_, `pr#23620 <https://github.com/ceph/ceph/pull/23620>`_, Nathan Cutler, Kefu Chai)
+* cmake: add option WITH_LIBRADOSSTRIPER (`pr#23732 <https://github.com/ceph/ceph/pull/23732>`_, Kefu Chai)
+* cmake: allow setting of the CTest timeout during building (`pr#22800 <https://github.com/ceph/ceph/pull/22800>`_, Willem Jan Withagen)
+* cmake: always prefer local symbols (`issue#25154 <http://tracker.ceph.com/issues/25154>`_, `pr#23320 <https://github.com/ceph/ceph/pull/23320>`_, Kefu Chai)
+* cmake: always turn off bjam debugging output (`pr#22204 <https://github.com/ceph/ceph/pull/22204>`_, Kefu Chai)
+* cmake: bump up the required boost version to 1.67 (`pr#22392 <https://github.com/ceph/ceph/pull/22392>`_, Kefu Chai)
+* cmake: bump up the required fmt version (`pr#23283 <https://github.com/ceph/ceph/pull/23283>`_, Kefu Chai)
+* cmake: cleanups (`pr#23166 <https://github.com/ceph/ceph/pull/23166>`_, Kefu Chai)
+* cmake: cleanups (`pr#23279 <https://github.com/ceph/ceph/pull/23279>`_, Kefu Chai)
+* cmake: cleanups (`pr#23300 <https://github.com/ceph/ceph/pull/23300>`_, Kefu Chai)
+* cmake,crimson/net: add keepalive support, and enable unittest_seastar_messenger in "make check" (`pr#23642 <https://github.com/ceph/ceph/pull/23642>`_, Kefu Chai)
+* cmake: detect armv8 crc and crypto feature using CHECK_C_COMPILER_FLAG (`issue#17516 <http://tracker.ceph.com/issues/17516>`_, `pr#24168 <https://github.com/ceph/ceph/pull/24168>`_, Kefu Chai)
+* cmake: disable -Werror-stringop-truncation for rocksdb (`pr#22591 <https://github.com/ceph/ceph/pull/22591>`_, Kefu Chai)
+* cmake: do not check for aligned_alloc() anymore (`issue#23653 <http://tracker.ceph.com/issues/23653>`_, `pr#22046 <https://github.com/ceph/ceph/pull/22046>`_, Kefu Chai)
+* cmake: do not depend on ${DPDK_LIBRARIES} if not using bundled dpdk (`issue#24449 <http://tracker.ceph.com/issues/24449>`_, `pr#22938 <https://github.com/ceph/ceph/pull/22938>`_, Kefu Chai)
+* cmake: do not install `hello` demo module (`pr#21886 <https://github.com/ceph/ceph/pull/21886>`_, John Spray)
+* cmake: do not link against common_crc_aarch64 (`pr#23366 <https://github.com/ceph/ceph/pull/23366>`_, Kefu Chai)
+* cmake: do not pass -B{symbolic,symbolic-functions} to linker on FreeBSD (`pr#24920 <https://github.com/ceph/ceph/pull/24920>`_, Willem Jan Withagen)
+* cmake: do not pass unnecessary param to setup.py (`pr#25186 <https://github.com/ceph/ceph/pull/25186>`_, Kefu Chai)
+* cmake: do not use Findfmt.cmake for checking libfmt-dev (`pr#23390 <https://github.com/ceph/ceph/pull/23390>`_, Kefu Chai)
+* cmake: do not use plain target_link_libraries(rgw_a ...) (`pr#24515 <https://github.com/ceph/ceph/pull/24515>`_, Kefu Chai)
+* cmake: enable RTTI for both debug and release RocksDB builds (`pr#22286 <https://github.com/ceph/ceph/pull/22286>`_, Igor Fedotov)
+* cmake: find a python2 interpreter for gtest-parallel (`pr#22931 <https://github.com/ceph/ceph/pull/22931>`_, Kefu Chai)
+* cmake: find liboath using the correct name (`pr#22430 <https://github.com/ceph/ceph/pull/22430>`_, Kefu Chai)
+* cmake: fix a cmake error when with -DALLOCATOR=jemalloc (`pr#23380 <https://github.com/ceph/ceph/pull/23380>`_, Jianpeng Ma)
+* cmake: fix build WITH_SYSTEM_BOOST=ON (`pr#23510 <https://github.com/ceph/ceph/pull/23510>`_, Kefu Chai)
+* cmake: fix compilation with distcc and other compiler wrappers (`pr#24605 <https://github.com/ceph/ceph/pull/24605>`_, Alexey Sheplyakov, Kefu Chai)
+* cmake: fix cython target in test/CMakeFile.txt (`pr#22295 <https://github.com/ceph/ceph/pull/22295>`_, Jan Fajerski)
+* cmake: fix Debug build `WITH_SEASTAR=ON` (`pr#23567 <https://github.com/ceph/ceph/pull/23567>`_, Kefu Chai)
+* cmake: fixes to enable WITH_ASAN with clang and GCC (`pr#24692 <https://github.com/ceph/ceph/pull/24692>`_, Kefu Chai)
+* cmake: fix find system rockdb (`pr#22439 <https://github.com/ceph/ceph/pull/22439>`_, Alexey Shabalin)
+* cmake: fix std::filesystem detection and extract sanitizer detection into its own module (`pr#23384 <https://github.com/ceph/ceph/pull/23384>`_, Kefu Chai)
+* cmake: fix syntax error of set() (`pr#26582 <https://github.com/ceph/ceph/pull/26582>`_, Kefu Chai)
+* cmake: fix the build WITH_DPDK=ON (`pr#23650 <https://github.com/ceph/ceph/pull/23650>`_, Kefu Chai, Casey Bodley)
+* cmake: fix version matching for Findfmt (`pr#23996 <https://github.com/ceph/ceph/pull/23996>`_, Mohamad Gebai)
+* cmake: fix "WITH_STATIC_LIBSTDCXX" (`pr#22990 <https://github.com/ceph/ceph/pull/22990>`_, Kefu Chai)
+* cmake: let rbd_api depend on librbd-tp (`pr#25641 <https://github.com/ceph/ceph/pull/25641>`_, Kefu Chai)
+* cmake: link against gtest in a better way (`pr#23628 <https://github.com/ceph/ceph/pull/23628>`_, Kefu Chai)
+* cmake: link ceph-osd with common statically (`pr#22720 <https://github.com/ceph/ceph/pull/22720>`_, Radoslaw Zarzynski)
+* cmake: link compressor plugins against lib the modern way (`pr#23852 <https://github.com/ceph/ceph/pull/23852>`_, Kefu Chai)
+* cmake: make -DWITH_MGR=OFF work (`pr#22077 <https://github.com/ceph/ceph/pull/22077>`_, Jianpeng Ma)
+* cmake: Make the tests for finding Filesystem with more serious functions (`pr#26316 <https://github.com/ceph/ceph/pull/26316>`_, Willem Jan Withagen)
+* cmake: modularize src/perfglue (`pr#23254 <https://github.com/ceph/ceph/pull/23254>`_, Kefu Chai)
+* cmake: move ceph-osdomap-tool, ceph-monstore-tool out of ceph-test (`pr#19964 <https://github.com/ceph/ceph/pull/19964>`_, runsisi)
+* cmake: move crypto_plugins target (`pr#21891 <https://github.com/ceph/ceph/pull/21891>`_, Casey Bodley)
+* cmake: no libradosstriper headers if WITH_LIBRADOSSTRIPER=OFF (`issue#35922 <http://tracker.ceph.com/issues/35922>`_, `pr#24029 <https://github.com/ceph/ceph/pull/24029>`_, Nathan Cutler, Kefu Chai)
+* cmake: no need to add "-D" before definitions (`pr#23795 <https://github.com/ceph/ceph/pull/23795>`_, Kefu Chai)
+* cmake: oath lives in liboath (`pr#22494 <https://github.com/ceph/ceph/pull/22494>`_, Willem Jan Withagen)
+* cmake: only build extra boost libraries only if WITH_SEASTAR (`pr#22521 <https://github.com/ceph/ceph/pull/22521>`_, Kefu Chai)
+* cmake: remove checking for GCC 5.1 (`pr#24477 <https://github.com/ceph/ceph/pull/24477>`_, Kefu Chai)
+* cmake: remove deleted rgw_request.cc from CMakeLists.txt (`pr#22186 <https://github.com/ceph/ceph/pull/22186>`_, Casey Bodley)
+* cmake: Remove embedded 'cephd' code (`pr#21940 <https://github.com/ceph/ceph/pull/21940>`_, Dan Mick)
+* cmake: remove workarounds for supporting cmake 2.x (`pr#22912 <https://github.com/ceph/ceph/pull/22912>`_, Kefu Chai)
+* cmake: rgw_common should depend on tracing headers (`pr#22367 <https://github.com/ceph/ceph/pull/22367>`_, Kefu Chai)
+* cmake: rocksdb related cleanup (`pr#23441 <https://github.com/ceph/ceph/pull/23441>`_, Kefu Chai)
+* cmake: should link against libatomic if libcxx/libstdc++ does not off… (`pr#22952 <https://github.com/ceph/ceph/pull/22952>`_, Kefu Chai)
+* cmake: update fio version from 3.5 to 540e235dcd276e63c57 (`pr#22019 <https://github.com/ceph/ceph/pull/22019>`_, Jianpeng Ma)
+* cmake: use $CMAKE_BINARY_DIR for default $CEPH_BUILD_VIRTUALENV (`issue#36737 <http://tracker.ceph.com/issues/36737>`_, `pr#26091 <https://github.com/ceph/ceph/pull/26091>`_, Kefu Chai)
+* cmake: use javac -h for creating JNI native headers (`issue#24012 <http://tracker.ceph.com/issues/24012>`_, `pr#21822 <https://github.com/ceph/ceph/pull/21822>`_, Kefu Chai)
+* cmake: use OpenSSL::Crypto instead of OPENSSL_LIBRARIES (`pr#24368 <https://github.com/ceph/ceph/pull/24368>`_, Kefu Chai)
+* cmake: vstart target can build WITH_CEPHFS/RBD/MGR=OFF (`pr#25204 <https://github.com/ceph/ceph/pull/25204>`_, Casey Bodley)
+* common: add adaptor for seastar::temporary_buffer (`pr#22454 <https://github.com/ceph/ceph/pull/22454>`_, Kefu Chai, Casey Bodley)
+* common: add a generic async Completion for use with boost::asio (`pr#21914 <https://github.com/ceph/ceph/pull/21914>`_, Casey Bodley)
+* common: add lockless `md_config_t` (`pr#22710 <https://github.com/ceph/ceph/pull/22710>`_, Kefu Chai)
+* common: async/dpdk: when enable dpdk, multiple message queue defect (`pr#25404 <https://github.com/ceph/ceph/pull/25404>`_, zhangyongsheng)
+* common: auth/cephx: minor code cleanup (`pr#21155 <https://github.com/ceph/ceph/pull/21155>`_, runsisi)
+* common: auth, common: cleanups (`pr#26383 <https://github.com/ceph/ceph/pull/26383>`_, Kefu Chai)
+* common: auth,common: use ceph::mutex instead of LockMutex (`pr#24263 <https://github.com/ceph/ceph/pull/24263>`_, Kefu Chai)
+* common: avoid the overhead of ``ANNOTATE_HAPPENS_*`` in NDEBUG builds (`pr#25129 <https://github.com/ceph/ceph/pull/25129>`_, Radoslaw Zarzynski)
+* common: be more informative if set PID-file fails (`pr#23647 <https://github.com/ceph/ceph/pull/23647>`_, Willem Jan Withagen)
+* common: blkdev: Rework API and add FreeBSD support (`pr#24658 <https://github.com/ceph/ceph/pull/24658>`_, Alan Somers)
+* common: buffer: mark the iterator traits "public" (`pr#25409 <https://github.com/ceph/ceph/pull/25409>`_, Kefu Chai)
+* common: calculate stddev on the fly (`pr#21461 <https://github.com/ceph/ceph/pull/21461>`_, Yao Zongyou)
+* common: ceph.in: use correct module for cmd flags (`pr#26454 <https://github.com/ceph/ceph/pull/26454>`_, Patrick Donnelly)
+* common: ceph-volume add device_id to inventory listing (`pr#25201 <https://github.com/ceph/ceph/pull/25201>`_, Jan Fajerski)
+* common: changes to address FTBFS on fc30 (`pr#26301 <https://github.com/ceph/ceph/pull/26301>`_, Kefu Chai)
+* common: common/admin_socket: add new api unregister_commands(AdminSocketHook … (`pr#21718 <https://github.com/ceph/ceph/pull/21718>`_, Jianpeng Ma)
+* common: common,auth,crimson: add logging to crimson (`pr#23957 <https://github.com/ceph/ceph/pull/23957>`_, Kefu Chai)
+* common: common/buffer: fix compiler bug when enable DEBUG_BUFFER (`pr#25848 <https://github.com/ceph/ceph/pull/25848>`_, Jianpeng Ma)
+* common: common/buffer: remove repeated condtion-check (`pr#25420 <https://github.com/ceph/ceph/pull/25420>`_, Jianpeng Ma)
+* common: common/config: add ConfigProxy for crimson (`pr#23074 <https://github.com/ceph/ceph/pull/23074>`_, Kefu Chai)
+* common: common/config: fix the lock in ConfigProxy::diff() (`pr#23276 <https://github.com/ceph/ceph/pull/23276>`_, Kefu Chai)
+* common: common/config_values: friend md_config_impl<> (`pr#23020 <https://github.com/ceph/ceph/pull/23020>`_, Mykola Golub, Kefu Chai)
+* common:  common: drop the unused methods from SharedLRU (`pr#26224 <https://github.com/ceph/ceph/pull/26224>`_, Radoslaw Zarzynski)
+* common: common/KeyValueDB: Get rid of validate parameter (`pr#25377 <https://github.com/ceph/ceph/pull/25377>`_, Adam Kupczyk)
+* common: common/numa: Add shim routines for NUMA on FreeBSD (`pr#25920 <https://github.com/ceph/ceph/pull/25920>`_, Willem Jan Withagen)
+* common: common, osd: set mclock priority as 1 by default (`pr#26022 <https://github.com/ceph/ceph/pull/26022>`_, Abhishek Lekshmanan)
+* common: common/random_cache: remove unused RandomCache (`pr#26253 <https://github.com/ceph/ceph/pull/26253>`_, Kefu Chai)
+* common: common/shared_cache: add lockless SharedLRU (`pr#22736 <https://github.com/ceph/ceph/pull/22736>`_, Kefu Chai)
+* common: common/shared_cache: bumps it to the front of the LRU if key existed (`pr#25370 <https://github.com/ceph/ceph/pull/25370>`_, Jianpeng Ma)
+* common: common/shared_cache: fix racing issues (`pr#25150 <https://github.com/ceph/ceph/pull/25150>`_, Jianpeng Ma)
+* common: common/util: pass real hostname when running in kubernetes/rook container (`pr#23798 <https://github.com/ceph/ceph/pull/23798>`_, Sage Weil)
+* common: complete all throttle blockers when we set average or max to 0 (`issue#36715 <http://tracker.ceph.com/issues/36715>`_, `pr#24965 <https://github.com/ceph/ceph/pull/24965>`_, Dongsheng Yang)
+* common,core: msg/async: clean up local buffers on dispatch (`issue#35987 <http://tracker.ceph.com/issues/35987>`_, `pr#24111 <https://github.com/ceph/ceph/pull/24111>`_, Greg Farnum)
+* common,core,tests: qa/tests: update links for centos latest to point to 7.5 (`pr#22923 <https://github.com/ceph/ceph/pull/22923>`_, Vasu Kulkarni)
+* common/crc/aarch64: Added cpu feature pmull and make aarch64 specific… (`pr#22178 <https://github.com/ceph/ceph/pull/22178>`_, Adam Kupczyk)
+* common:  crimson/common: write configs synchronously on shard.0 (`pr#23284 <https://github.com/ceph/ceph/pull/23284>`_, Kefu Chai)
+* common,crimson: port perfcounters to seastar (`pr#24141 <https://github.com/ceph/ceph/pull/24141>`_, chunmei Liu)
+* common: crypto: QAT based Encryption for RGW (`pr#19386 <https://github.com/ceph/ceph/pull/19386>`_, Ganesh Maharaj Mahalingam)
+* common: crypto: use ceph_assert_always for assertions (`pr#23654 <https://github.com/ceph/ceph/pull/23654>`_, Casey Bodley)
+* common: define BOOST_COROUTINES_NO_DEPRECATION_WARNING if not yet (`pr#26502 <https://github.com/ceph/ceph/pull/26502>`_, Kefu Chai)
+* common: drop allocation tracking from bufferlist (`pr#25454 <https://github.com/ceph/ceph/pull/25454>`_, Radoslaw Zarzynski)
+* common: drop append_buffer from bufferlist. Use simple carriage instead (`pr#25077 <https://github.com/ceph/ceph/pull/25077>`_, Radoslaw Zarzynski)
+* common: drop at_buffer_{head,tail} from buffer::ptr (`pr#25422 <https://github.com/ceph/ceph/pull/25422>`_, Radoslaw Zarzynski)
+* common: drop/mark-as-final getters of buffer::raw for palign (`pr#24087 <https://github.com/ceph/ceph/pull/24087>`_, Radoslaw Zarzynski)
+* common: drop static_assert.h as it looks unused (`pr#22743 <https://github.com/ceph/ceph/pull/22743>`_, Radoslaw Zarzynski)
+* common: drop the unused buffer::raw_mmap_pages (`pr#24040 <https://github.com/ceph/ceph/pull/24040>`_, Radoslaw Zarzynski)
+* common: drop the unused zero-copy facilities in ceph::bufferlist (`pr#24031 <https://github.com/ceph/ceph/pull/24031>`_, Radoslaw Zarzynski)
+* common: drop unused get_max_pipe_size() in buffer.cc (`pr#25432 <https://github.com/ceph/ceph/pull/25432>`_, Radoslaw Zarzynski)
+* common: ec: lrc doesn't depend on crosstalks between bufferlists anymore (`pr#25595 <https://github.com/ceph/ceph/pull/25595>`_, Radoslaw Zarzynski)
+* common: expand meta in parse_argv() (`pr#23474 <https://github.com/ceph/ceph/pull/23474>`_, Kefu Chai)
+* common: fix access and add name for the token bucket throttle (`pr#25372 <https://github.com/ceph/ceph/pull/25372>`_, Shiyang Ruan)
+* common: Fix Alpine compatability for TEMP_FAILURE_RETRY and ACCESSPERMS (`pr#24813 <https://github.com/ceph/ceph/pull/24813>`_, Willem Jan Withagen)
+* common: fix a racing in PerfCounters::perf_counter_data_any_d::read_avg (`issue#25211 <http://tracker.ceph.com/issues/25211>`_, `pr#23362 <https://github.com/ceph/ceph/pull/23362>`_, ludehp)
+* common: fix for broken rbdmap parameter parsing (`pr#24446 <https://github.com/ceph/ceph/pull/24446>`_, Marc Schoechlin)
+* common: fix missing include boost/noncopyable.hpp (`pr#24278 <https://github.com/ceph/ceph/pull/24278>`_, Willem Jan Withagen)
+* common: fix typo in rados bench write JSON output (`issue#24199 <http://tracker.ceph.com/issues/24199>`_, `pr#22112 <https://github.com/ceph/ceph/pull/22112>`_, Sandor Zeestraten)
+* common: fix typos in BackoffThrottle (`pr#24691 <https://github.com/ceph/ceph/pull/24691>`_, Shiyang Ruan)
+* common: Formatters: improve precision of double numbers (`pr#25745 <https://github.com/ceph/ceph/pull/25745>`_, Коренберг Марк)
+* common: .gitignore: Ignore .idea directory (`pr#24237 <https://github.com/ceph/ceph/pull/24237>`_, Volker Theile)
+* common: hint bufferlist's buffer_track_c_str accordingly (`pr#25424 <https://github.com/ceph/ceph/pull/25424>`_, Radoslaw Zarzynski)
+* common: hypercombined bufferlist (`pr#24882 <https://github.com/ceph/ceph/pull/24882>`_, Radoslaw Zarzynski)
+* common: include/compat.h: make pthread_get_name_np work when available (`pr#23641 <https://github.com/ceph/ceph/pull/23641>`_, Willem Jan Withagen)
+* common: include include/types.h early, otherwise Clang will error (`pr#22493 <https://github.com/ceph/ceph/pull/22493>`_, Willem Jan Withagen)
+* common: include/types: move operator<< into the proper namespace (`pr#23767 <https://github.com/ceph/ceph/pull/23767>`_, Kefu Chai)
+* common: include/types: space between number and units (`pr#22063 <https://github.com/ceph/ceph/pull/22063>`_, Sage Weil)
+* common: librados,rpm,deb: various fixes to address librados3 transition and cleanups in librados (`pr#24896 <https://github.com/ceph/ceph/pull/24896>`_, Kefu Chai)
+* common: make CEPH_BUFFER_ALLOC_UNIT known at compile-time (`pr#26259 <https://github.com/ceph/ceph/pull/26259>`_, Radoslaw Zarzynski)
+* common: mark BlkDev::serial() const to match with its declaration (`pr#24702 <https://github.com/ceph/ceph/pull/24702>`_, Willem Jan Withagen)
+* common: messages: define HEAD_VERSION and COMPAT_VERSION inlined (`pr#23623 <https://github.com/ceph/ceph/pull/23623>`_, Kefu Chai)
+* common,mgr: mgr/MgrClient: make some noise for a user if no mgr daemon is running (`pr#23492 <https://github.com/ceph/ceph/pull/23492>`_, Sage Weil)
+* common: mon/MonClient: set configs via finisher (`issue#24118 <http://tracker.ceph.com/issues/24118>`_, `pr#21984 <https://github.com/ceph/ceph/pull/21984>`_, Sage Weil)
+* common: msg/async: fix FTBFS of dpdk (`pr#23168 <https://github.com/ceph/ceph/pull/23168>`_, Kefu Chai)
+* common: msg/async: Skip the duplicated processing of the same link (`pr#20952 <https://github.com/ceph/ceph/pull/20952>`_, shangfufei)
+* common: msg/msg_types.h: do not cast `ceph_entity_name` to `entity_name_t` for printing (`pr#26315 <https://github.com/ceph/ceph/pull/26315>`_, Kefu Chai)
+* common: msgr/async/rdma: Return from poll system call with EINTR should be retried (`pr#25138 <https://github.com/ceph/ceph/pull/25138>`_, Stig Telfer)
+* common: Mutex -> ceph::mutex (`issue#12614 <http://tracker.ceph.com/issues/12614>`_, `pr#25105 <https://github.com/ceph/ceph/pull/25105>`_, Kefu Chai, Sage Weil)
+* common: optimize reference counting in bufferlist (`pr#25082 <https://github.com/ceph/ceph/pull/25082>`_, Radoslaw Zarzynski)
+* common: OpTracker doesn't visit TrackedOp when nref == 0 (`issue#24037 <http://tracker.ceph.com/issues/24037>`_, `pr#22156 <https://github.com/ceph/ceph/pull/22156>`_, Radoslaw Zarzynski)
+* common: os/filestore: fix throttle configurations (`pr#21926 <https://github.com/ceph/ceph/pull/21926>`_, Li Wang)
+* common,performance: auth,common: add lockless auth (`pr#23591 <https://github.com/ceph/ceph/pull/23591>`_, Kefu Chai)
+* common,performance: common/assert: mark assert helpers with [[gnu::cold]] (`pr#23326 <https://github.com/ceph/ceph/pull/23326>`_, Kefu Chai)
+* common,performance: compressor: add QAT support (`pr#19714 <https://github.com/ceph/ceph/pull/19714>`_, Qiaowei Ren)
+* common,performance: denc: fix internal fragmentation when decoding ptr in bl (`pr#25264 <https://github.com/ceph/ceph/pull/25264>`_, Kefu Chai)
+* common,rbd: misc: mark constructors as explicit (`pr#21637 <https://github.com/ceph/ceph/pull/21637>`_, Danny Al-Gaaf)
+* common: reinit StackStringStream on clear (`pr#25751 <https://github.com/ceph/ceph/pull/25751>`_, Patrick Donnelly)
+* common: reintroduce async SharedMutex (`issue#24124 <http://tracker.ceph.com/issues/24124>`_, `pr#22698 <https://github.com/ceph/ceph/pull/22698>`_, Casey Bodley)
+* common: Reverse deleted include (`pr#23838 <https://github.com/ceph/ceph/pull/23838>`_, Willem Jan Withagen)
+* common: Revert "common: add an async SharedMutex" (`issue#24124 <http://tracker.ceph.com/issues/24124>`_, `pr#21986 <https://github.com/ceph/ceph/pull/21986>`_, Casey Bodley)
+* common,rgw: cls/rbd: init local var with known value (`pr#25588 <https://github.com/ceph/ceph/pull/25588>`_, Kefu Chai)
+* common,tests: run-standalone.sh: Need double-quotes to handle | in core_pattern on all distributions (`issue#38325 <http://tracker.ceph.com/issues/38325>`_, `pr#26436 <https://github.com/ceph/ceph/pull/26436>`_, David Zafman)
+* common,tests: test_shared_cache: fix memory leak (`pr#25215 <https://github.com/ceph/ceph/pull/25215>`_, Jianpeng Ma)
+* common: vstart: do not attempt to re-initialize dashboard for existing cluster (`pr#23261 <https://github.com/ceph/ceph/pull/23261>`_, Jason Dillaman)
+* core: Add support for osd_delete_sleep configuration value (`issue#36474 <http://tracker.ceph.com/issues/36474>`_, `pr#24749 <https://github.com/ceph/ceph/pull/24749>`_, David Zafman)
+* core: auth: drop the RWLock in AuthClientHandler (`pr#23699 <https://github.com/ceph/ceph/pull/23699>`_, Kefu Chai)
+* core: auth/krb: Fix Kerberos build warnings (`pr#25639 <https://github.com/ceph/ceph/pull/25639>`_, Daniel Oliveira)
+* core: build: disable kerberos for nautilus (`pr#26258 <https://github.com/ceph/ceph/pull/26258>`_, Sage Weil)
+* core: ceph_argparse: fix --verbose (`pr#25961 <https://github.com/ceph/ceph/pull/25961>`_, Patrick Nawracay)
+* core: ceph.in: friendlier message on EPERM (`issue#25172 <http://tracker.ceph.com/issues/25172>`_, `pr#23330 <https://github.com/ceph/ceph/pull/23330>`_, John Spray)
+* core: ceph.in: write bytes to stdout in raw_write() (`pr#25280 <https://github.com/ceph/ceph/pull/25280>`_, Kefu Chai)
+* core: ceph_test_rados_api_misc: remove obsolete LibRadosMiscPool.PoolCreationRace (`issue#24150 <http://tracker.ceph.com/issues/24150>`_, `pr#22042 <https://github.com/ceph/ceph/pull/22042>`_, Sage Weil)
+* core: Clang misses <optional> include (`pr#23768 <https://github.com/ceph/ceph/pull/23768>`_, Willem Jan Withagen)
+* core: common/blkdev.h: use std::string (`pr#25783 <https://github.com/ceph/ceph/pull/25783>`_, Neha Ojha)
+* core: common/options: remove unused ms async affinity options (`pr#26099 <https://github.com/ceph/ceph/pull/26099>`_, Josh Durgin)
+* core: common/util.cc: add CONTAINER_NAME processing for metadata (`pr#25383 <https://github.com/ceph/ceph/pull/25383>`_, Dan Mick)
+* core: compressor: building error for QAT decompress (`pr#22609 <https://github.com/ceph/ceph/pull/22609>`_, Qiaowei Ren)
+* core: crush, osd: handle multiple parents properly when applying pg upmaps (`issue#23921 <http://tracker.ceph.com/issues/23921>`_, `pr#21815 <https://github.com/ceph/ceph/pull/21815>`_, xiexingguo)
+* core: erasure-code: add clay codes (`issue#19278 <http://tracker.ceph.com/issues/19278>`_, `pr#24291 <https://github.com/ceph/ceph/pull/24291>`_, Myna V, Sage Weil)
+* core: erasure-code: fixes alignment issue when clay code is used with jerasure, cauchy_orig (`pr#24586 <https://github.com/ceph/ceph/pull/24586>`_, Myna)
+* core: global/signal_handler.cc: report assert_file as correct name (`pr#23738 <https://github.com/ceph/ceph/pull/23738>`_, Dan Mick)
+* core: include/rados: clarify which flags go where for copy_from (`pr#24497 <https://github.com/ceph/ceph/pull/24497>`_, Ilya Dryomov)
+* core: include/rados.h: hide CEPH_OSDMAP_PGLOG_HARDLIMIT from ceph -s (`pr#25887 <https://github.com/ceph/ceph/pull/25887>`_, Neha Ojha)
+* core: kv/KeyValueDB: Move PriCache implementation to ShardedCache (`pr#25925 <https://github.com/ceph/ceph/pull/25925>`_, Mark Nelson)
+* core: kv/KeyValueDB: return const char\* from MergeOperator::name() (`issue#26875 <http://tracker.ceph.com/issues/26875>`_, `pr#23477 <https://github.com/ceph/ceph/pull/23477>`_, Sage Weil)
+* core: messages/MOSDPGScan: fix initialization of query_epoch (`pr#22408 <https://github.com/ceph/ceph/pull/22408>`_, wumingqiao)
+* core: mgr/balancer: add cmd to list all plans (`issue#37418 <http://tracker.ceph.com/issues/37418>`_, `pr#21937 <https://github.com/ceph/ceph/pull/21937>`_, Yang Honggang)
+* core: mgr/BaseMgrModule: drop GIL for ceph_send_command (`issue#38537 <http://tracker.ceph.com/issues/38537>`_, `pr#26723 <https://github.com/ceph/ceph/pull/26723>`_, Sage Weil)
+* core: mgr/MgrClient: Protect daemon_health_metrics (`issue#23352 <http://tracker.ceph.com/issues/23352>`_, `pr#23404 <https://github.com/ceph/ceph/pull/23404>`_, Kjetil Joergensen, Brad Hubbard)
+* core,mgr: mon/MgrMonitor: change 'unresponsive' message to info level (`issue#24222 <http://tracker.ceph.com/issues/24222>`_, `pr#22158 <https://github.com/ceph/ceph/pull/22158>`_, Sage Weil)
+* core,mgr,rbd:  mgr: generalize osd perf query and make counters accessible from modules (`pr#25114 <https://github.com/ceph/ceph/pull/25114>`_, Mykola Golub)
+* core,mgr,rbd:  osd: support more dynamic perf query subkey types (`pr#25371 <https://github.com/ceph/ceph/pull/25371>`_, Mykola Golub)
+* core,mgr,rbd,rgw: rgw, common: Fixes SCA issues (`pr#22007 <https://github.com/ceph/ceph/pull/22007>`_, Danny Al-Gaaf)
+* core: mgr/smart: remove obsolete smart module (`pr#26411 <https://github.com/ceph/ceph/pull/26411>`_, Sage Weil)
+* core: mon/LogMonitor: call no_reply() on ignored log message (`pr#22098 <https://github.com/ceph/ceph/pull/22098>`_, Sage Weil)
+* core: mon/MonClient: avoid using magic number for the `MAuth::protocol` (`pr#23747 <https://github.com/ceph/ceph/pull/23747>`_, Kefu Chai)
+* core: mon/MonClient: extract MonSub out (`pr#23688 <https://github.com/ceph/ceph/pull/23688>`_, Kefu Chai)
+* core: mon/MonClient: use scoped_guard instead of goto (`pr#24304 <https://github.com/ceph/ceph/pull/24304>`_, Kefu Chai)
+* core,mon: mon,osd: dump "compression_algorithms" in "mon metadata" (`issue#22420 <http://tracker.ceph.com/issues/22420>`_, `pr#21809 <https://github.com/ceph/ceph/pull/21809>`_, Kefu Chai, Casey Bodley)
+* core,mon: mon/OSDMonitor: no_reply on MOSDFailure messages (`issue#24322 <http://tracker.ceph.com/issues/24322>`_, `pr#22259 <https://github.com/ceph/ceph/pull/22259>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: Warnings for expected_num_objects (`issue#24687 <http://tracker.ceph.com/issues/24687>`_, `pr#23072 <https://github.com/ceph/ceph/pull/23072>`_, Douglas Fuller)
+* core: mon/OSDMonitor: two "ceph osd crush class rm" fixes (`pr#24657 <https://github.com/ceph/ceph/pull/24657>`_, xie xingguo)
+* core: mon/PGMap: fix PGMapDigest decode (`pr#22066 <https://github.com/ceph/ceph/pull/22066>`_, Sage Weil)
+* core: mon/PGMap: include unknown PGs in 'pg ls' (`pr#24032 <https://github.com/ceph/ceph/pull/24032>`_, Sage Weil)
+* core: msg/async: do not trigger RESETSESSION from connect fault during connection phase (`issue#36612 <http://tracker.ceph.com/issues/36612>`_, `pr#25343 <https://github.com/ceph/ceph/pull/25343>`_, Sage Weil)
+* core: msg/async/Event: clear time_events on shutdown (`issue#24162 <http://tracker.ceph.com/issues/24162>`_, `pr#22093 <https://github.com/ceph/ceph/pull/22093>`_, Sage Weil)
+* core: msg/async: fix banner_v1 check in ProtocolV2 (`pr#26714 <https://github.com/ceph/ceph/pull/26714>`_, Yingxin Cheng)
+* core: msg/async: fix include in frames_v2.h (`pr#26711 <https://github.com/ceph/ceph/pull/26711>`_, Yingxin Cheng)
+* core: msg/async: fix is_queued() semantics (`pr#24693 <https://github.com/ceph/ceph/pull/24693>`_, Ilya Dryomov)
+* core: msg/async: keep connection alive only actually sending (`pr#24301 <https://github.com/ceph/ceph/pull/24301>`_, Haomai Wang, Kefu Chai)
+* core: os/bluestore: fix deep-scrub operation againest disk silent errors (`pr#23629 <https://github.com/ceph/ceph/pull/23629>`_, Xiaoguang Wang)
+* core: os/bluestore: fix flush_commit locking (`issue#21480 <http://tracker.ceph.com/issues/21480>`_, `pr#22083 <https://github.com/ceph/ceph/pull/22083>`_, Sage Weil)
+* core: OSD: add impl for filestore to get dbstatistics (`issue#24591 <http://tracker.ceph.com/issues/24591>`_, `pr#22633 <https://github.com/ceph/ceph/pull/22633>`_, lvshuhua)
+* core: osdc: Change 'bool budgeted' to 'int budget' to avoid recalculating (`pr#21242 <https://github.com/ceph/ceph/pull/21242>`_, Jianpeng Ma)
+* core: OSD: ceph-osd parent process need to restart log service after fork (`issue#24956 <http://tracker.ceph.com/issues/24956>`_, `pr#23090 <https://github.com/ceph/ceph/pull/23090>`_, redickwang)
+* core: osdc/Objecter: fix split vs reconnect race (`issue#22544 <http://tracker.ceph.com/issues/22544>`_, `pr#23850 <https://github.com/ceph/ceph/pull/23850>`_, Sage Weil)
+* core: osdc/Objecter: no need null pointer check for op->session anymore (`pr#25230 <https://github.com/ceph/ceph/pull/25230>`_, runsisi)
+* core: osdc/Objecter: possible race condition with connection reset (`issue#36183 <http://tracker.ceph.com/issues/36183>`_, `pr#24276 <https://github.com/ceph/ceph/pull/24276>`_, Jason Dillaman)
+* core: osdc: self-managed snapshot helper should catch decode exception (`issue#24000 <http://tracker.ceph.com/issues/24000>`_, `pr#21804 <https://github.com/ceph/ceph/pull/21804>`_, Jason Dillaman)
+* core: osd, librados: add unset-manifest op (`pr#21999 <https://github.com/ceph/ceph/pull/21999>`_, Myoungwon Oh)
+* core: osd,mds: make 'config rm ...' idempotent (`issue#24408 <http://tracker.ceph.com/issues/24408>`_, `pr#22395 <https://github.com/ceph/ceph/pull/22395>`_, Sage Weil)
+* core: osd/mon: fix upgrades for pg log hard limit (`issue#36686 <http://tracker.ceph.com/issues/36686>`_, `pr#25816 <https://github.com/ceph/ceph/pull/25816>`_, Neha Ojha, Yuri Weinstein)
+* core: osd,mon: increase mon_max_pg_per_osd to 250 (`pr#23251 <https://github.com/ceph/ceph/pull/23251>`_, Neha Ojha)
+* core: osd,mon,msg: use intrusive_ptr for holding Connection::priv (`issue#20924 <http://tracker.ceph.com/issues/20924>`_, `pr#22292 <https://github.com/ceph/ceph/pull/22292>`_, Kefu Chai)
+* core: osd/OSD: choose heartbeat peers more carefully (`pr#23487 <https://github.com/ceph/ceph/pull/23487>`_, xie xingguo)
+* core: osd/OSD: drop extra/wrong \*unregister_pg\* (`pr#21816 <https://github.com/ceph/ceph/pull/21816>`_, xiexingguo)
+* core: osd/OSDMap: be more aggressive when trying to balance (`issue#37940 <http://tracker.ceph.com/issues/37940>`_, `pr#26039 <https://github.com/ceph/ceph/pull/26039>`_, xie xingguo)
+* core: osd/OSDMap: drop local pool filter in calc_pg_upmaps (`pr#26605 <https://github.com/ceph/ceph/pull/26605>`_, xie xingguo)
+* core: osd/OSDMap: fix CEPHX_V2 osd requirement to nautilus, not mimic (`pr#23249 <https://github.com/ceph/ceph/pull/23249>`_, Sage Weil)
+* core: osd/OSDMap: fix upmap mis-killing for erasure-coded PGs (`pr#25365 <https://github.com/ceph/ceph/pull/25365>`_, ningtao, xie xingguo)
+* core: osd/OSDMap: potential access violation fix (`issue#37881 <http://tracker.ceph.com/issues/37881>`_, `pr#25930 <https://github.com/ceph/ceph/pull/25930>`_, xie xingguo)
+* core: osd/OSDMap: using std::vector::reserve to reduce memory reallocation (`pr#26478 <https://github.com/ceph/ceph/pull/26478>`_, xie xingguo)
+* core: osd/OSD: ping monitor if we are stuck at __waiting_for_healthy__ (`pr#23958 <https://github.com/ceph/ceph/pull/23958>`_, xie xingguo)
+* core: osd/OSD: preallocate for _get_pgs/_get_pgids to avoid reallocate (`pr#25434 <https://github.com/ceph/ceph/pull/25434>`_, Jianpeng Ma)
+* core: osd/PG: async-recovery should respect historical missing objects (`pr#24004 <https://github.com/ceph/ceph/pull/24004>`_, xie xingguo)
+* core: osd/PG.cc: account for missing set irrespective of last_complete (`issue#37919 <http://tracker.ceph.com/issues/37919>`_, `pr#26175 <https://github.com/ceph/ceph/pull/26175>`_, Neha Ojha)
+* core: osd/PG: create new PGs from activate in last_peering_reset epoch (`issue#24452 <http://tracker.ceph.com/issues/24452>`_, `pr#22478 <https://github.com/ceph/ceph/pull/22478>`_, Sage Weil)
+* core: osd/PG: do not choose stray osds as async_recovery_targets (`pr#22330 <https://github.com/ceph/ceph/pull/22330>`_, Neha Ojha)
+* core: osd/PG: fix misused FORCE_RECOVERY[BACKFILL] flags (`issue#27985 <http://tracker.ceph.com/issues/27985>`_, `pr#23904 <https://github.com/ceph/ceph/pull/23904>`_, xie xingguo)
+* core: osd/PGLog.cc: check if complete_to points to log.end() (`pr#23450 <https://github.com/ceph/ceph/pull/23450>`_, Neha Ojha)
+* core: osd/PGLog: trim - avoid dereferencing invalid iter (`pr#23546 <https://github.com/ceph/ceph/pull/23546>`_, xie xingguo)
+* core: osd/PG: remove unused functions (`pr#26155 <https://github.com/ceph/ceph/pull/26155>`_, Kefu Chai)
+* core: osd/PG: reset PG on osd down->up; normalize query processing (`issue#24373 <http://tracker.ceph.com/issues/24373>`_, `pr#22456 <https://github.com/ceph/ceph/pull/22456>`_, Sage Weil)
+* core: osd/PG: restrict async_recovery_targets to up osds (`pr#22664 <https://github.com/ceph/ceph/pull/22664>`_, Neha Ojha)
+* core: osd/PG: unset history_les_bound if local-les is used (`pr#22524 <https://github.com/ceph/ceph/pull/22524>`_, Kefu Chai)
+* core: osd/PG: write pg epoch when resurrecting pg after delete vs merge race (`issue#35923 <http://tracker.ceph.com/issues/35923>`_, `pr#24061 <https://github.com/ceph/ceph/pull/24061>`_, Sage Weil)
+* core: osd/PrimaryLogPG: do not count failed read in delta_stats (`pr#25687 <https://github.com/ceph/ceph/pull/25687>`_, Kefu Chai)
+* core: osd/PrimaryLogPG: fix last_peering_reset checking on manifest flushing (`pr#26778 <https://github.com/ceph/ceph/pull/26778>`_, xie xingguo)
+* core: osd/PrimaryLogPG: fix on_local_recover crash on stray clone (`pr#22396 <https://github.com/ceph/ceph/pull/22396>`_, Sage Weil)
+* core: osd/PrimaryLogPG: fix potential pg-log overtrimming (`pr#23317 <https://github.com/ceph/ceph/pull/23317>`_, xie xingguo)
+* core: osd/PrimaryLogPG: fix the extent length error of the sync read (`pr#25584 <https://github.com/ceph/ceph/pull/25584>`_, Xiaofei Cui)
+* core: osd/PrimaryLogPG: fix try_flush_mark_clean write contention case (`issue#24174 <http://tracker.ceph.com/issues/24174>`_, `pr#22084 <https://github.com/ceph/ceph/pull/22084>`_, Sage Weil)
+* core: osd/PrimaryLogPG: optimize recover order (`pr#23587 <https://github.com/ceph/ceph/pull/23587>`_, xie xingguo)
+* core: osd/PrimaryLogPG: update missing_loc more carefully (`issue#35546 <http://tracker.ceph.com/issues/35546>`_, `pr#23895 <https://github.com/ceph/ceph/pull/23895>`_, xie xingguo)
+* core: osd/ReplicatedBackend: remove useless assert (`pr#21243 <https://github.com/ceph/ceph/pull/21243>`_, Jianpeng Ma)
+* core: osd/Session: fix invalid iterator dereference in Session::have_backoff() (`issue#24486 <http://tracker.ceph.com/issues/24486>`_, `pr#22497 <https://github.com/ceph/ceph/pull/22497>`_, Sage Weil)
+* core:  osd: write "debug dump_missing" output to stdout (`pr#21960 <https://github.com/ceph/ceph/pull/21960>`_, Коренберг Маркr)
+* core: os/kstore: support db statistic (`pr#21487 <https://github.com/ceph/ceph/pull/21487>`_, Yang Honggang)
+* core: os/memstore: use ceph::mutex and friends (`pr#26026 <https://github.com/ceph/ceph/pull/26026>`_, Kefu Chai)
+* core,performance:  core: avoid unnecessary refcounting of OSDMap on OSD's hot paths (`pr#24743 <https://github.com/ceph/ceph/pull/24743>`_, Radoslaw Zarzynski)
+* core,performance: msg/async: avoid put message within write_lock (`pr#20731 <https://github.com/ceph/ceph/pull/20731>`_, Haomai Wang)
+* core,performance: os/bluestore: make osd shard-thread do oncommits (`pr#22739 <https://github.com/ceph/ceph/pull/22739>`_, Jianpeng Ma)
+* core,performance: osd/filestore: Change default filestore_merge_threshold to -10 (`issue#24686 <http://tracker.ceph.com/issues/24686>`_, `pr#22761 <https://github.com/ceph/ceph/pull/22761>`_, Douglas Fuller)
+* core,performance: osd/OSDMap: map pgs with smaller batchs in calc_pg_upmaps (`pr#23734 <https://github.com/ceph/ceph/pull/23734>`_, huangjun)
+* core: PG: release reservations after backfill completes (`issue#23614 <http://tracker.ceph.com/issues/23614>`_, `pr#22255 <https://github.com/ceph/ceph/pull/22255>`_, Neha Ojha)
+* core: pg stuck in backfill_wait with plenty of disk space (`issue#38034 <http://tracker.ceph.com/issues/38034>`_, `pr#26375 <https://github.com/ceph/ceph/pull/26375>`_, xie xingguo, David Zafman)
+* core,pybind: pybind/rados: new methods for manipulating self-managed snapshots (`pr#22579 <https://github.com/ceph/ceph/pull/22579>`_, Jason Dillaman)
+* core: qa/suites/rados: minor fixes (`pr#22195 <https://github.com/ceph/ceph/pull/22195>`_, Neha Ojha)
+* core: qa/suites/rados/thrash-erasure-code\*/thrashers/\*: less likely resv rejection injection (`pr#24667 <https://github.com/ceph/ceph/pull/24667>`_, Sage Weil)
+* core: qa/suites/rados/thrash-old-clients: only centos and 16.04 (`pr#22106 <https://github.com/ceph/ceph/pull/22106>`_, Sage Weil)
+* core: qa/suites: set osd_pg_log_dups_tracked in cfuse_workunit_suites_fsync.yaml (`pr#21909 <https://github.com/ceph/ceph/pull/21909>`_, Neha Ojha)
+* core: qa/suites/upgrade/luminous-x: disable c-o-t import/export tests between versions (`issue#38294 <http://tracker.ceph.com/issues/38294>`_, `pr#27018 <https://github.com/ceph/ceph/pull/27018>`_, Sage Weil)
+* core: qa/suites/upgrade/mimic-x/parallel: enable all classes (`pr#27011 <https://github.com/ceph/ceph/pull/27011>`_, Sage Weil)
+* core: qa/workunits/mgr/test_localpool.sh: use new config syntax (`pr#22496 <https://github.com/ceph/ceph/pull/22496>`_, Sage Weil)
+* core: qa/workunits/rados/test_health_warnings: prevent out osds (`issue#37776 <http://tracker.ceph.com/issues/37776>`_, `pr#25732 <https://github.com/ceph/ceph/pull/25732>`_, Sage Weil)
+* core: rados.pyx: make all exceptions accept keyword arguments (`issue#24033 <http://tracker.ceph.com/issues/24033>`_, `pr#21853 <https://github.com/ceph/ceph/pull/21853>`_, Rishabh Dave)
+* core: rados: return legacy address in 'lock info' (`pr#26150 <https://github.com/ceph/ceph/pull/26150>`_, Jason Dillaman)
+* core: scrub warning check incorrectly uses mon scrub interval (`issue#37264 <http://tracker.ceph.com/issues/37264>`_, `pr#25112 <https://github.com/ceph/ceph/pull/25112>`_, David Zafman)
+* core: src: no 'dne' acronym in user cmd output (`pr#21094 <https://github.com/ceph/ceph/pull/21094>`_, Gu Zhongyan)
+* core,tests: Minor cleanups in tests and log output (`issue#38631 <http://tracker.ceph.com/issues/38631>`_, `issue#38678 <http://tracker.ceph.com/issues/38678>`_, `pr#26899 <https://github.com/ceph/ceph/pull/26899>`_, David Zafman)
+* core,tests: qa/overrides/short_pg_log.yaml: reduce osd_{min,max}_pg_log_entries (`issue#38025 <http://tracker.ceph.com/issues/38025>`_, `pr#26101 <https://github.com/ceph/ceph/pull/26101>`_, Neha Ojha)
+* core,tests: qa/suites/rados/thrash: change crush_tunables to jewel in rados_api_tests (`issue#38042 <http://tracker.ceph.com/issues/38042>`_, `pr#26122 <https://github.com/ceph/ceph/pull/26122>`_, Neha Ojha)
+* core,tests: qa/suites/upgrade/luminous-x: a few fixes (`pr#22092 <https://github.com/ceph/ceph/pull/22092>`_, Sage Weil)
+* core,tests: qa/tests: Set ansible-version: 2.5 (`issue#24926 <http://tracker.ceph.com/issues/24926>`_, `pr#23123 <https://github.com/ceph/ceph/pull/23123>`_, Yuri Weinstein)
+* core,tests: Removal of snapshot with corrupt replica crashes osd (`issue#23875 <http://tracker.ceph.com/issues/23875>`_, `pr#22476 <https://github.com/ceph/ceph/pull/22476>`_, David Zafman)
+* core,tests: test: Verify a log trim trims the dup_index (`pr#26533 <https://github.com/ceph/ceph/pull/26533>`_, Brad Hubbard)
+* core,tools: osdmaptool: fix wrong test_map_pgs_dump_all output (`pr#22280 <https://github.com/ceph/ceph/pull/22280>`_, huangjun)
+* core,tools: rados: provide user with more meaningful error message (`pr#26275 <https://github.com/ceph/ceph/pull/26275>`_, Mykola Golub)
+* core,tools: tools/rados: allow reuse object for write test (`pr#25128 <https://github.com/ceph/ceph/pull/25128>`_, Li Wang)
+* core: vstart.sh: Support SPDK in Ceph development deployment (`pr#22975 <https://github.com/ceph/ceph/pull/22975>`_, tone.zhang)
+* crimson: add MonClient (`pr#23849 <https://github.com/ceph/ceph/pull/23849>`_, Kefu Chai)
+* crimson: cache osdmap using LRU cache (`pr#26254 <https://github.com/ceph/ceph/pull/26254>`_, Kefu Chai, Jianpeng Ma)
+* crimson/common: apply config changes also on shard.0 (`pr#23631 <https://github.com/ceph/ceph/pull/23631>`_, Yingxin)
+* crimson/connection: misc changes (`pr#23044 <https://github.com/ceph/ceph/pull/23044>`_, Kefu Chai)
+* crimson: crimson/mon: remove timeout support from mon::Client::authenticate() (`pr#24660 <https://github.com/ceph/ceph/pull/24660>`_, Kefu Chai)
+* crimson/mon: move mon::Connection into .cc (`pr#24619 <https://github.com/ceph/ceph/pull/24619>`_, Kefu Chai)
+* crimson/net: concurrent dispatch for SocketMessenger (`pr#24090 <https://github.com/ceph/ceph/pull/24090>`_, Casey Bodley)
+* crimson/net: encapsulate protocol implementations with states (`pr#25176 <https://github.com/ceph/ceph/pull/25176>`_, Yingxin, Kefu Chai)
+* crimson/net: encapsulate protocol implementations with states (remaining part) (`pr#25207 <https://github.com/ceph/ceph/pull/25207>`_, Yingxin)
+* crimson/net: fix addresses during banner exchange (`pr#25580 <https://github.com/ceph/ceph/pull/25580>`_, Yingxin)
+* crimson/net: fix compile errors in test_alien_echo.cc (`pr#24629 <https://github.com/ceph/ceph/pull/24629>`_, Yingxin)
+* crimson/net: fix crimson msgr error leaks to caller (`pr#25716 <https://github.com/ceph/ceph/pull/25716>`_, Yingxin)
+* crimson/net: fix misc issues for segment-fault and test-failures (`pr#25939 <https://github.com/ceph/ceph/pull/25939>`_, Yingxin Cheng, Kefu Chai)
+* crimson/net: Fix racing for promise on_message (`pr#24097 <https://github.com/ceph/ceph/pull/24097>`_, Yingxin)
+* crimson/net: fix unittest_seastar_messenger errors (`pr#23539 <https://github.com/ceph/ceph/pull/23539>`_, Yingxin)
+* crimson/net: implement accepting/connecting states (`pr#24608 <https://github.com/ceph/ceph/pull/24608>`_, Yingxin)
+* crimson/net: miscellaneous fixes to seastar-msgr (`pr#23816 <https://github.com/ceph/ceph/pull/23816>`_, Yingxin, Casey Bodley)
+* crimson/net: misc fixes and features for crimson-messenger tests (`pr#26221 <https://github.com/ceph/ceph/pull/26221>`_, Yingxin Cheng)
+* crimson/net: seastar-msgr refactoring (`pr#24576 <https://github.com/ceph/ceph/pull/24576>`_, Yingxin)
+* crimson/net: s/repeat/keep_doing/ (`pr#23898 <https://github.com/ceph/ceph/pull/23898>`_, Kefu Chai)
+* crimson/osd: add heartbeat support (`pr#26222 <https://github.com/ceph/ceph/pull/26222>`_, Kefu Chai)
+* crimson/osd: add more heartbeat peers (`pr#26255 <https://github.com/ceph/ceph/pull/26255>`_, Kefu Chai)
+* crimson/osd: correct the order of parameters passed to OSD::_preboot() (`pr#26774 <https://github.com/ceph/ceph/pull/26774>`_, chunmei Liu)
+* crimson/osd: crimson osd driver (`pr#25304 <https://github.com/ceph/ceph/pull/25304>`_, Radoslaw Zarzynski, Kefu Chai)
+* crimson/osd: remove "force_new" from ms_get_authorizer() (`pr#26054 <https://github.com/ceph/ceph/pull/26054>`_, Kefu Chai)
+* crimson/osd: send known addresses at boot (`pr#26452 <https://github.com/ceph/ceph/pull/26452>`_, Kefu Chai)
+* crimson: persist/load osdmap to/from store (`pr#26090 <https://github.com/ceph/ceph/pull/26090>`_, Kefu Chai)
+* crimson: port messenger to seastar (`pr#22491 <https://github.com/ceph/ceph/pull/22491>`_, Kefu Chai, Casey Bodley)
+* crimson/thread: add thread pool (`pr#22565 <https://github.com/ceph/ceph/pull/22565>`_, Kefu Chai)
+* crimson/thread: pin thread pool to given CPU (`pr#22776 <https://github.com/ceph/ceph/pull/22776>`_, Kefu Chai)
+* crush/CrushWrapper: silence compiler warning (`pr#25336 <https://github.com/ceph/ceph/pull/25336>`_, Li Wang)
+* crush: fix device_class_clone for unpopulated/empty weight-sets (`issue#23386 <http://tracker.ceph.com/issues/23386>`_, `pr#22127 <https://github.com/ceph/ceph/pull/22127>`_, Sage Weil)
+* crush: fix memory leak (`pr#25959 <https://github.com/ceph/ceph/pull/25959>`_, xie xingguo)
+* crush: fix upmap overkill (`issue#37968 <http://tracker.ceph.com/issues/37968>`_, `pr#26179 <https://github.com/ceph/ceph/pull/26179>`_, xie xingguo)
+* dashboard/mgr: Save button doesn't prevent saving an invalid form (`issue#36426 <http://tracker.ceph.com/issues/36426>`_, `pr#24577 <https://github.com/ceph/ceph/pull/24577>`_, Patrick Nawracay)
+* dashboard: Return float if rate not available (`pr#22313 <https://github.com/ceph/ceph/pull/22313>`_, Boris Ranto)
+* doc: add Ceph Manager Dashboard to top-level TOC (`pr#26390 <https://github.com/ceph/ceph/pull/26390>`_, Nathan Cutler)
+* doc: add ceph-volume inventory sections (`pr#25092 <https://github.com/ceph/ceph/pull/25092>`_, Jan Fajerski)
+* doc: add documentation for iostat (`pr#22034 <https://github.com/ceph/ceph/pull/22034>`_, Mohamad Gebai)
+* doc: added demo document changes section (`pr#24791 <https://github.com/ceph/ceph/pull/24791>`_, James McClune)
+* doc: added rbd default features (`pr#24720 <https://github.com/ceph/ceph/pull/24720>`_, Gaurav Sitlani)
+* doc: added some Civetweb configuration options (`pr#24073 <https://github.com/ceph/ceph/pull/24073>`_, Anton Oks)
+* doc: Added some hints on how to further accelerate builds with ccache (`pr#25394 <https://github.com/ceph/ceph/pull/25394>`_, Lenz Grimmer)
+* doc: add instructions about using "serve-doc" to preview built document (`pr#24471 <https://github.com/ceph/ceph/pull/24471>`_, Kefu Chai)
+* doc: add mds state transition diagram (`issue#22989 <http://tracker.ceph.com/issues/22989>`_, `pr#22996 <https://github.com/ceph/ceph/pull/22996>`_, Patrick Donnelly)
+* doc: Add mention of ceph osd pool stats (`pr#25575 <https://github.com/ceph/ceph/pull/25575>`_, Thore Kruess)
+* doc: add missing 12.2.11 release note (`pr#26596 <https://github.com/ceph/ceph/pull/26596>`_, Nathan Cutler)
+* doc: add note about LVM volumes to ceph-deploy quick start (`pr#23879 <https://github.com/ceph/ceph/pull/23879>`_, David Wahler)
+* doc: add release notes for 12.2.11 luminous (`pr#26228 <https://github.com/ceph/ceph/pull/26228>`_, Abhishek Lekshmanan)
+* doc: add spacing to subcommand references (`pr#24669 <https://github.com/ceph/ceph/pull/24669>`_, James McClune)
+* doc: add "--timeout" option to rbd-nbd (`pr#24302 <https://github.com/ceph/ceph/pull/24302>`_, Stefan Kooman)
+* doc/bluestore: fix minor typos in compression section (`pr#22874 <https://github.com/ceph/ceph/pull/22874>`_, David Disseldorp)
+* doc: broken link on troubleshooting-mon page (`pr#25312 <https://github.com/ceph/ceph/pull/25312>`_, James McClune)
+* doc: bump up sphinx and pyyaml versions (`pr#26044 <https://github.com/ceph/ceph/pull/26044>`_, Kefu Chai)
+* doc: ceph-deploy would not support --cluster option anymore (`pr#26471 <https://github.com/ceph/ceph/pull/26471>`_, Tatsuya Naganawa)
+* doc: ceph: describe application subcommand in ceph man page (`pr#20645 <https://github.com/ceph/ceph/pull/20645>`_, Rishabh Dave)
+* doc: ceph-iscsi-api ports should not be public facing (`pr#24248 <https://github.com/ceph/ceph/pull/24248>`_, Jason Dillaman)
+* doc: ceph-volume describe better the options for migrating away from ceph-disk (`issue#24036 <http://tracker.ceph.com/issues/24036>`_, `pr#21890 <https://github.com/ceph/ceph/pull/21890>`_, Alfredo Deza)
+* doc: ceph-volume dmcrypt and activate --all documentation updates (`issue#24031 <http://tracker.ceph.com/issues/24031>`_, `pr#22062 <https://github.com/ceph/ceph/pull/22062>`_, Alfredo Deza)
+* doc: ceph-volume: expand on why ceph-disk was replaced (`pr#23194 <https://github.com/ceph/ceph/pull/23194>`_, Alfredo Deza)
+* doc: ceph-volume: `lvm batch` documentation and man page updates (`issue#24970 <http://tracker.ceph.com/issues/24970>`_, `pr#23443 <https://github.com/ceph/ceph/pull/23443>`_, Alfredo Deza)
+* doc: ceph-volume:  update batch documentation to explain filestore strategies (`issue#34309 <http://tracker.ceph.com/issues/34309>`_, `pr#23785 <https://github.com/ceph/ceph/pull/23785>`_, Alfredo Deza)
+* doc: ceph-volume: zfs, the initial first submit (`pr#23674 <https://github.com/ceph/ceph/pull/23674>`_, Willem Jan Withagen)
+* doc: cleaned up troubleshooting OSDs documentation (`pr#23519 <https://github.com/ceph/ceph/pull/23519>`_, James McClune)
+* doc: Clean up field names in ServiceDescription and add a service field (`pr#26006 <https://github.com/ceph/ceph/pull/26006>`_, Jeff Layton)
+* doc: cleanup: prune Argonaut-specific verbiage (`pr#22899 <https://github.com/ceph/ceph/pull/22899>`_, Nathan Cutler)
+* doc: cleanup rendering syntax (`pr#22389 <https://github.com/ceph/ceph/pull/22389>`_, Mahati Chamarthy)
+* doc: Clean up the snapshot consistency note (`pr#25655 <https://github.com/ceph/ceph/pull/25655>`_, Greg Farnum)
+* doc: common,mon: add implicit `#include` headers (`pr#23930 <https://github.com/ceph/ceph/pull/23930>`_, Kefu Chai)
+* doc: common/options: add description of osd objectstore backends (`issue#24147 <http://tracker.ceph.com/issues/24147>`_, `pr#22040 <https://github.com/ceph/ceph/pull/22040>`_, Alfredo Deza)
+* doc: corrected options of iscsiadm command (`pr#26395 <https://github.com/ceph/ceph/pull/26395>`_, ZhuJieWen)
+* doc: correct rbytes description (`pr#24966 <https://github.com/ceph/ceph/pull/24966>`_, Xiang Dai)
+* doc: describe RBD QoS settings (`pr#25202 <https://github.com/ceph/ceph/pull/25202>`_, Mykola Golub)
+* doc: doc/bluestore: data doesn't use two partitions (ceph-disk era) (`pr#22604 <https://github.com/ceph/ceph/pull/22604>`_, Alfredo Deza)
+* doc: doc/cephfs: fixup add/remove mds docs (`pr#23836 <https://github.com/ceph/ceph/pull/23836>`_, liu wei)
+* doc: doc/cephfs: remove lingering "experimental" note about multimds (`pr#22852 <https://github.com/ceph/ceph/pull/22852>`_, John Spray)
+* doc: doc/dashboard: don't advise mgr_initial_modules (`pr#22808 <https://github.com/ceph/ceph/pull/22808>`_, John Spray)
+* doc: doc/dashboard: fix formatting on Grafana instructions-2 (`pr#22706 <https://github.com/ceph/ceph/pull/22706>`_, Jos Collin)
+* doc: doc/dashboard: fix formatting on Grafana instructions (`pr#22657 <https://github.com/ceph/ceph/pull/22657>`_, John Spray)
+* doc: doc/dev/cephx_protocol: fix couple errors (`pr#23750 <https://github.com/ceph/ceph/pull/23750>`_, Kefu Chai)
+* doc: doc/dev/index: update rados lead (`pr#24160 <https://github.com/ceph/ceph/pull/24160>`_, Josh Durgin)
+* doc: doc/dev/msgr2.rst: update of the banner and authentication phases (`pr#20094 <https://github.com/ceph/ceph/pull/20094>`_, Ricardo Dias)
+* doc: doc/dev/seastore.rst: initial draft notes (`pr#21381 <https://github.com/ceph/ceph/pull/21381>`_, Sage Weil)
+* doc: doc/dev: Updated component leads table (`pr#24238 <https://github.com/ceph/ceph/pull/24238>`_, Lenz Grimmer)
+* doc:  doc: fix the links in releases/schedule.rst (`pr#22364 <https://github.com/ceph/ceph/pull/22364>`_, Kefu Chai)
+* doc: doc/man: mention import and export commands in rados manpage (`issue#4640 <http://tracker.ceph.com/issues/4640>`_, `pr#23186 <https://github.com/ceph/ceph/pull/23186>`_, Nathan Cutler)
+* doc:  doc: Mention PURGED_SNAPDIRS and RECOVERY_DELETES in Mimic release notes (`pr#22711 <https://github.com/ceph/ceph/pull/22711>`_, Florian Haas)
+* doc: doc/mgr/dashboard: fix typo in mgr ssl setup (`pr#24790 <https://github.com/ceph/ceph/pull/24790>`_, Mehdi Abaakouk)
+* doc: doc/mgr: mention how to clear config setting (`pr#22157 <https://github.com/ceph/ceph/pull/22157>`_, John Spray)
+* doc: doc/mgr: note need for module.py file in plugins (`pr#22622 <https://github.com/ceph/ceph/pull/22622>`_, John Spray)
+* doc: doc/mgr/orchestrator: Add Architecture Image (`pr#26331 <https://github.com/ceph/ceph/pull/26331>`_, Sebastian Wagner, Kefu Chai)
+* doc: doc/mgr/orchestrator: add `wal` to blink lights (`pr#25634 <https://github.com/ceph/ceph/pull/25634>`_, Sebastian Wagner)
+* doc: doc/mgr/prometheus: readd section about custom instance labels (`pr#25182 <https://github.com/ceph/ceph/pull/25182>`_, Jan Fajerski)
+* doc: doc/orchestrator: Aligned Documentation with specification (`pr#25893 <https://github.com/ceph/ceph/pull/25893>`_, Sebastian Wagner)
+* doc: doc/orchestrator: Integrate CLI specification into the documentation (`pr#25119 <https://github.com/ceph/ceph/pull/25119>`_, Sebastian Wagner)
+* doc:  doc: purge subcommand link broken (`pr#24785 <https://github.com/ceph/ceph/pull/24785>`_, James McClune)
+* doc: doc/rados: Add bluestore memory autotuning docs (`pr#25069 <https://github.com/ceph/ceph/pull/25069>`_, Mark Nelson)
+* doc: doc/rados/configuration: add osd scrub {begin,end} week day (`pr#25924 <https://github.com/ceph/ceph/pull/25924>`_, Neha Ojha)
+* doc: doc/rados/configuration/msgr2: some documentation about msgr2 (`pr#26867 <https://github.com/ceph/ceph/pull/26867>`_, Sage Weil)
+* doc: doc/rados/configuration: refresh osdmap section (`pr#26120 <https://github.com/ceph/ceph/pull/26120>`_, Ilya Dryomov)
+* doc: doc/rados: correct osd path in troubleshooting-mon.rst (`pr#24964 <https://github.com/ceph/ceph/pull/24964>`_, songweibin)
+* doc: doc/rados: fixed hit set type link (`pr#23833 <https://github.com/ceph/ceph/pull/23833>`_, James McClune)
+* doc: doc/radosgw/s3.rst: Adding AWS S3 `Storage Class` as `Not Supported` (`pr#19571 <https://github.com/ceph/ceph/pull/19571>`_, Katie Holly)
+* doc: doc/rados/operations: add balancer.rst to TOC (`pr#23684 <https://github.com/ceph/ceph/pull/23684>`_, Kefu Chai)
+* doc: doc/rados/operations: add clay to erasure-code-profile (`pr#26902 <https://github.com/ceph/ceph/pull/26902>`_, Kefu Chai)
+* doc: doc/rados/operations/crush-map-edits: fix 'take' syntax (`pr#24868 <https://github.com/ceph/ceph/pull/24868>`_, Remy Zandwijk, Sage Weil)
+* doc: doc/rados/operations/pg-states: fix PG state names, part 2 (`pr#23165 <https://github.com/ceph/ceph/pull/23165>`_, Nathan Cutler)
+* doc: doc/rados/operations/pg-states: fix PG state names (`pr#21520 <https://github.com/ceph/ceph/pull/21520>`_, Jan Fajerski)
+* doc: doc/rados update invalid bash on bluestore migration (`issue#34317 <http://tracker.ceph.com/issues/34317>`_, `pr#23801 <https://github.com/ceph/ceph/pull/23801>`_, Alfredo Deza)
+* doc: doc/rbd: corrected OpenStack Cinder permissions for Glance pool (`pr#22443 <https://github.com/ceph/ceph/pull/22443>`_, Jason Dillaman)
+* doc: doc/rbd: explicitly state that mirroring requires connectivity to clusters (`pr#24433 <https://github.com/ceph/ceph/pull/24433>`_, Jason Dillaman)
+* doc: doc/rbd/iscsi-target-cli: Update auth command (`pr#26788 <https://github.com/ceph/ceph/pull/26788>`_, Ricardo Marques)
+* doc: doc/rbd/iscsi-target-cli: Update disk separator (`pr#26669 <https://github.com/ceph/ceph/pull/26669>`_, Ricardo Marques)
+* doc: doc/release/luminous: v12.2.6 and v12.2.7 release notes (`pr#23057 <https://github.com/ceph/ceph/pull/23057>`_, Abhishek Lekshmanan, Sage Weil)
+* doc: doc/releases: Add luminous releases 12.2.9 and 10 (`pr#25361 <https://github.com/ceph/ceph/pull/25361>`_, Brad Hubbard)
+* doc: doc/releases: Add Mimic release 13.2.2 (`pr#24509 <https://github.com/ceph/ceph/pull/24509>`_, Brad Hubbard)
+* doc: doc/releases: Mark Jewel EOL (`pr#23698 <https://github.com/ceph/ceph/pull/23698>`_, Brad Hubbard)
+* doc: doc/releases: Mark Mimic first release as June (`pr#24099 <https://github.com/ceph/ceph/pull/24099>`_, Brad Hubbard)
+* doc: doc/releases/mimic.rst: make note of 13.2.2 upgrade bug (`pr#24979 <https://github.com/ceph/ceph/pull/24979>`_, Neha Ojha)
+* doc: doc/releases/mimic: tweak RBD major features (`pr#22011 <https://github.com/ceph/ceph/pull/22011>`_, Jason Dillaman)
+* doc: doc/releases/mimic: Updated dashboard description (`pr#22016 <https://github.com/ceph/ceph/pull/22016>`_, Lenz Grimmer)
+* doc: doc/releases/mimic: upgrade steps (`pr#21987 <https://github.com/ceph/ceph/pull/21987>`_, Sage Weil)
+* doc: doc/releases/nautilus: dashboard package notes (`pr#26815 <https://github.com/ceph/ceph/pull/26815>`_, Kefu Chai)
+* doc: doc/releases/schedule: Add Luminous 12.2.8 (`pr#23972 <https://github.com/ceph/ceph/pull/23972>`_, Brad Hubbard)
+* doc: doc/releases/schedule: add mimic column (`pr#22006 <https://github.com/ceph/ceph/pull/22006>`_, Sage Weil)
+* doc: doc/releases: Update releases to August '18 (`pr#23360 <https://github.com/ceph/ceph/pull/23360>`_, Brad Hubbard)
+* doc: doc/rgw: document placement targets and storage classes (`issue#24508 <http://tracker.ceph.com/issues/24508>`_, `issue#38008 <http://tracker.ceph.com/issues/38008>`_, `pr#26997 <https://github.com/ceph/ceph/pull/26997>`_, Casey Bodley)
+* doc: docs: add Clay code plugin documentation (`pr#24422 <https://github.com/ceph/ceph/pull/24422>`_, Myna)
+* doc: docs: Fixed swift client authentication fail (`pr#23729 <https://github.com/ceph/ceph/pull/23729>`_, Dai Dang Van)
+* doc: docs: radosgw: ldap-auth: fixed option name 'rgw_ldap_searchfilter' (`issue#23081 <http://tracker.ceph.com/issues/23081>`_, `pr#20526 <https://github.com/ceph/ceph/pull/20526>`_, Konstantin Shalygin)
+* doc: doc/start: fix kube-helm.rst typo: docuiment -> document (`pr#23423 <https://github.com/ceph/ceph/pull/23423>`_, Zhou Peng)
+* doc: doc/SubmittingPatches.rst: use Google style guide for doc patches (`pr#22190 <https://github.com/ceph/ceph/pull/22190>`_, Nathan Cutler)
+* doc: Document correction (`pr#23926 <https://github.com/ceph/ceph/pull/23926>`_, Gangbiao Liu)
+* doc: Document mappings of S3 Operations to ACL grants (`pr#26827 <https://github.com/ceph/ceph/pull/26827>`_, Adam C. Emerson)
+* doc: document sizing for `block.db` (`pr#23210 <https://github.com/ceph/ceph/pull/23210>`_, Alfredo Deza)
+* doc: document vstart options (`pr#22467 <https://github.com/ceph/ceph/pull/22467>`_, Mao Zhongyi)
+* doc: doc/user-management: Remove obsolete reset caps command (`issue#37663 <http://tracker.ceph.com/issues/37663>`_, `pr#25550 <https://github.com/ceph/ceph/pull/25550>`_, Brad Hubbard)
+* doc: edit on github (`pr#24452 <https://github.com/ceph/ceph/pull/24452>`_, Neha Ojha, Noah Watkins)
+* doc: erasure-code-clay fixes typos (`pr#24653 <https://github.com/ceph/ceph/pull/24653>`_, Myna)
+* doc: erasure-code-jerasure: removed default section of crush-device-class (`pr#21279 <https://github.com/ceph/ceph/pull/21279>`_, Junyoung Sung)
+* doc: examples/librados: Remove not needed else clauses (`pr#24939 <https://github.com/ceph/ceph/pull/24939>`_, Marcos Paulo de Souza)
+* doc: explain 'firstn v indep' in the CRUSH docs (`pr#24255 <https://github.com/ceph/ceph/pull/24255>`_, Greg Farnum)
+* doc: Fix a couple typos and improve diagram formatting (`pr#23496 <https://github.com/ceph/ceph/pull/23496>`_, Bryan Stillwell)
+* doc: fix a typo in doc/mgr/telegraf.rst (`pr#22267 <https://github.com/ceph/ceph/pull/22267>`_, Enming Zhang)
+* doc: fix cephfs spelling errors (`pr#23763 <https://github.com/ceph/ceph/pull/23763>`_, Chen Zhenghua)
+* doc: fix/cleanup freebsd osd disk creation (`pr#23600 <https://github.com/ceph/ceph/pull/23600>`_, Willem Jan Withagen)
+* doc: Fix Create a Cluster url in Running Multiple Clusters (`issue#37764 <http://tracker.ceph.com/issues/37764>`_, `pr#25705 <https://github.com/ceph/ceph/pull/25705>`_, Jos Collin)
+* doc: Fix EC k=3 m=2 profile overhead calculation example (`pr#20581 <https://github.com/ceph/ceph/pull/20581>`_, Charles Alva)
+* doc: fixed broken urls (`pr#23564 <https://github.com/ceph/ceph/pull/23564>`_, James McClune)
+* doc: fixed grammar in restore rbd image section (`pr#22944 <https://github.com/ceph/ceph/pull/22944>`_, James McClune)
+* doc: fixed links in Pools section (`pr#23431 <https://github.com/ceph/ceph/pull/23431>`_, James McClune)
+* doc: fixed minor typo in Debian packages section (`pr#22878 <https://github.com/ceph/ceph/pull/22878>`_, James McClune)
+* doc: fixed restful mgr module SSL configuration commands (`pr#21864 <https://github.com/ceph/ceph/pull/21864>`_, Lenz Grimmer)
+* doc: Fixed spelling errors in configuration section (`pr#23719 <https://github.com/ceph/ceph/pull/23719>`_, Bryan Stillwell)
+* doc: Fixed syntax in iscsi initiator windows doc (`pr#25467 <https://github.com/ceph/ceph/pull/25467>`_, Michel Raabe)
+* doc: Fixed the paragraph and boxes (`pr#25094 <https://github.com/ceph/ceph/pull/25094>`_, Scoots Hamilton)
+* doc: Fixed the wrong numbers in mgr/dashboard.rst (`pr#22658 <https://github.com/ceph/ceph/pull/22658>`_, Jos Collin)
+* doc: fixed typo in add-or-rm-mons.rst (`pr#26250 <https://github.com/ceph/ceph/pull/26250>`_, James McClune)
+* doc: fixed typo in cephfs snapshots (`pr#23764 <https://github.com/ceph/ceph/pull/23764>`_, Kai Wagner)
+* doc: fixed typo in CRUSH map docs (`pr#25953 <https://github.com/ceph/ceph/pull/25953>`_, James McClune)
+* doc: fixed typo in man page (`pr#24792 <https://github.com/ceph/ceph/pull/24792>`_, James McClune)
+* doc: Fix incorrect mention of 'osd_deep_mon_scrub_interval' (`pr#26522 <https://github.com/ceph/ceph/pull/26522>`_, Ashish Singh)
+* doc: Fix iSCSI docs URL (`pr#26296 <https://github.com/ceph/ceph/pull/26296>`_, Ricardo Marques)
+* doc: fix iscsi target name when configuring target (`pr#21906 <https://github.com/ceph/ceph/pull/21906>`_, Venky Shankar)
+* doc: fix long description error for rgw_period_root_pool (`pr#23814 <https://github.com/ceph/ceph/pull/23814>`_, yuliyang)
+* doc: fix some it's -> its typos (`pr#22802 <https://github.com/ceph/ceph/pull/22802>`_, Brad Fitzpatrick)
+* doc: Fix some typos (`pr#25060 <https://github.com/ceph/ceph/pull/25060>`_, mooncake)
+* doc: Fix Spelling Error In File "ceph.rst" (`pr#23917 <https://github.com/ceph/ceph/pull/23917>`_, Gangbiao Liu)
+* doc: Fix Spelling Error In File dynamicresharding.rst (`pr#24175 <https://github.com/ceph/ceph/pull/24175>`_, xiaomanh)
+* doc: Fix Spelling Error of Rados Deployment/Operations (`pr#23746 <https://github.com/ceph/ceph/pull/23746>`_, Li Bingyang)
+* doc: Fix Spelling Error of Radosgw (`pr#23948 <https://github.com/ceph/ceph/pull/23948>`_, Li Bingyang)
+* doc: Fix Spelling Error of Radosgw (`pr#24000 <https://github.com/ceph/ceph/pull/24000>`_, Li Bingyang)
+* doc: Fix Spelling Error of Radosgw (`pr#24021 <https://github.com/ceph/ceph/pull/24021>`_, Li Bingyang)
+* doc: Fix Spelling Error of Rados Operations (`pr#23891 <https://github.com/ceph/ceph/pull/23891>`_, Li Bingyang)
+* doc: Fix Spelling Error of Rados Operations (`pr#23900 <https://github.com/ceph/ceph/pull/23900>`_, Li Bingyang)
+* doc: Fix Spelling Error of Rados Operations (`pr#23903 <https://github.com/ceph/ceph/pull/23903>`_, Li Bingyang)
+* doc: fix spelling errors in rbd doc (`pr#23765 <https://github.com/ceph/ceph/pull/23765>`_, Chen Zhenghua)
+* doc: fix spelling errors of cephfs (`pr#23745 <https://github.com/ceph/ceph/pull/23745>`_, Chen Zhenghua)
+* doc: fix the broken urls (`issue#25185 <http://tracker.ceph.com/issues/25185>`_, `pr#23310 <https://github.com/ceph/ceph/pull/23310>`_, Jos Collin)
+* doc: fix the formatting of HTTP Frontends documentation (`pr#25723 <https://github.com/ceph/ceph/pull/25723>`_, James McClune)
+* doc: fix typo and format issues in quick start documentation (`pr#23705 <https://github.com/ceph/ceph/pull/23705>`_, Chen Zhenghua)
+* doc: fix typo in add-or-rm-mons (`pr#25661 <https://github.com/ceph/ceph/pull/25661>`_, Jos Collin)
+* doc: Fix typo in ceph-fuse(8) (`pr#22214 <https://github.com/ceph/ceph/pull/22214>`_, Jos Collin)
+* doc: fix typo in erasure coding example (`pr#25737 <https://github.com/ceph/ceph/pull/25737>`_, Arthur Liu)
+* doc: Fix typos in Developer Guide (`pr#24067 <https://github.com/ceph/ceph/pull/24067>`_, Li Bingyang)
+* doc: fix typos in doc/releases (`pr#24186 <https://github.com/ceph/ceph/pull/24186>`_, Li Bingyang)
+* doc: \*/: fix typos in docs,messages,logs,comments (`pr#24139 <https://github.com/ceph/ceph/pull/24139>`_, Kefu Chai)
+* doc: Fix Typos of Developer Guide (`pr#24094 <https://github.com/ceph/ceph/pull/24094>`_, Li Bingyang)
+* doc: fix typos (`pr#22174 <https://github.com/ceph/ceph/pull/22174>`_, Mao Zhongyi)
+* doc: .githubmap, .mailmap, .organizationmap: update contributors (`pr#24756 <https://github.com/ceph/ceph/pull/24756>`_, Tiago Melo)
+* doc: githubmap, organizationmap: cleanup and add/update contributors/affiliation (`pr#22734 <https://github.com/ceph/ceph/pull/22734>`_, Tatjana Dehler)
+* doc: give pool name if default pool rbd is not created (`pr#24750 <https://github.com/ceph/ceph/pull/24750>`_, Changcheng Liu)
+* doc: Improve docs osd_recovery_priority, osd_recovery_op_priority and related (`pr#26705 <https://github.com/ceph/ceph/pull/26705>`_, David Zafman)
+* doc: Improve OpenStack integration and multitenancy docs for radosgw (`issue#36765 <http://tracker.ceph.com/issues/36765>`_, `pr#25056 <https://github.com/ceph/ceph/pull/25056>`_, Florian Haas)
+* doc: install build-doc deps without git clone (`pr#24416 <https://github.com/ceph/ceph/pull/24416>`_, Noah Watkins)
+* doc: Luminous v12.2.10 release notes (`pr#25034 <https://github.com/ceph/ceph/pull/25034>`_, Nathan Cutler)
+* doc: Luminous v12.2.9 release notes (`pr#24779 <https://github.com/ceph/ceph/pull/24779>`_, Nathan Cutler)
+* doc: make it easier to reach the old dev doc TOC (`pr#23253 <https://github.com/ceph/ceph/pull/23253>`_, Nathan Cutler)
+* doc: mention CVEs in luminous v12.2.11 release notes (`pr#26312 <https://github.com/ceph/ceph/pull/26312>`_, Nathan Cutler, Abhishek Lekshmanan)
+* doc: mgr/dashboard: Add documentation about supported browsers (`issue#27207 <http://tracker.ceph.com/issues/27207>`_, `pr#23712 <https://github.com/ceph/ceph/pull/23712>`_, Tiago Melo)
+* doc: mgr/dashboard: Added missing tooltip to settings icon (`pr#23935 <https://github.com/ceph/ceph/pull/23935>`_, Lenz Grimmer)
+* doc: mgr/dashboard: Add hints to resolve unit test failures (`pr#23627 <https://github.com/ceph/ceph/pull/23627>`_, Stephan Müller)
+* doc: mgr/dashboard: Cleaner notifications (`pr#23315 <https://github.com/ceph/ceph/pull/23315>`_, Stephan Müller)
+* doc: mgr/dashboard: Cleanup of summary refresh test (`pr#25504 <https://github.com/ceph/ceph/pull/25504>`_, Stephan Müller)
+* doc: mgr/dashboard: Document custom RESTController endpoints (`pr#25322 <https://github.com/ceph/ceph/pull/25322>`_, Stephan Müller)
+* doc: mgr/dashboard: Fixed documentation link on RGW page (`pr#24612 <https://github.com/ceph/ceph/pull/24612>`_, Tina Kallio)
+* doc: mgr/dashboard: Fix some setup steps in HACKING.rst (`pr#24788 <https://github.com/ceph/ceph/pull/24788>`_, Ranjitha G)
+* doc: mgr/dashboard: Improve prettier scripts and documentation (`pr#22994 <https://github.com/ceph/ceph/pull/22994>`_, Tiago Melo)
+* doc: mgr/dashboard/qa: add missing dashboard suites (`pr#25084 <https://github.com/ceph/ceph/pull/25084>`_, Tatjana Dehler)
+* doc: mgr/dashboard: updated SSO documentation (`pr#25943 <https://github.com/ceph/ceph/pull/25943>`_, Alfonso Martínez)
+* doc: mgr/dashboard: Update I18N documentation (`pr#25159 <https://github.com/ceph/ceph/pull/25159>`_, Tiago Melo)
+* doc: mgr/orch: Fix remote_host doc reference (`issue#38254 <http://tracker.ceph.com/issues/38254>`_, `pr#26360 <https://github.com/ceph/ceph/pull/26360>`_, Ernesto Puerta)
+* doc/mgr/plugins.rst: explain more about the plugin command protocol (`pr#22629 <https://github.com/ceph/ceph/pull/22629>`_, Dan Mick)
+* doc: mimic is stable! (`pr#22350 <https://github.com/ceph/ceph/pull/22350>`_, Abhishek Lekshmanan)
+* doc: mimic rc1 release notes (`pr#20975 <https://github.com/ceph/ceph/pull/20975>`_, Abhishek Lekshmanan)
+* doc: Multiple spelling fixes (`pr#23514 <https://github.com/ceph/ceph/pull/23514>`_, Bryan Stillwell)
+* doc: numbered eviction situations (`pr#24618 <https://github.com/ceph/ceph/pull/24618>`_, Scoots Hamilton)
+* doc: osdmaptool/cleanup: Completed osdmaptool's usage (`issue#3214 <http://tracker.ceph.com/issues/3214>`_, `pr#13925 <https://github.com/ceph/ceph/pull/13925>`_, Vedant Nanda)
+* doc: osd/PrimaryLogPG: avoid dereferencing invalid complete_to (`pr#23894 <https://github.com/ceph/ceph/pull/23894>`_, xie xingguo)
+* doc: osd/PrimaryLogPG: rename list_missing -> list_unfound command (`pr#23723 <https://github.com/ceph/ceph/pull/23723>`_, xie xingguo)
+* doc: PendingReleaseNotes: note newly added CLAY code (`pr#24491 <https://github.com/ceph/ceph/pull/24491>`_, Kefu Chai)
+* doc: print pg peering in SVG instead of PNG (`pr#20366 <https://github.com/ceph/ceph/pull/20366>`_, Aleksei Gutikov)
+* doc: Put command template into literal block (`pr#24999 <https://github.com/ceph/ceph/pull/24999>`_, Alexey Stupnikov)
+* doc: qa/mgr/selftest: handle always-on module fall out (`issue#26994 <http://tracker.ceph.com/issues/26994>`_, `pr#23681 <https://github.com/ceph/ceph/pull/23681>`_, Noah Watkins)
+* doc: qa: Task to emulate network delay and packet drop between two given h… (`pr#23602 <https://github.com/ceph/ceph/pull/23602>`_, Shilpa Jagannath)
+* doc: qa/workunits/rbd: replace usage of 'rados rmpool' (`pr#23942 <https://github.com/ceph/ceph/pull/23942>`_, Mykola Golub)
+* doc: release/mimic: correct the changelog to the latest version (`pr#22319 <https://github.com/ceph/ceph/pull/22319>`_, Abhishek Lekshmanan)
+* doc: release notes for 12.2.8 luminous (`pr#23909 <https://github.com/ceph/ceph/pull/23909>`_, Abhishek Lekshmanan)
+* doc: release notes for 13.2.2 mimic (`pr#24266 <https://github.com/ceph/ceph/pull/24266>`_, Abhishek Lekshmanan)
+* doc: releases: mimic 13.2.1 release notes (`pr#23288 <https://github.com/ceph/ceph/pull/23288>`_, Abhishek Lekshmanan)
+* doc: releases: release notes for v10.2.11 Jewel (`pr#22989 <https://github.com/ceph/ceph/pull/22989>`_, Abhishek Lekshmanan)
+* doc: remove CZ mirror (`pr#21797 <https://github.com/ceph/ceph/pull/21797>`_, Tomáš Kukrál)
+* doc: remove deprecated 'scrubq' from ceph(8) (`issue#35813 <http://tracker.ceph.com/issues/35813>`_, `pr#23959 <https://github.com/ceph/ceph/pull/23959>`_, Ruben Kerkhof)
+* doc: remove documentation for installing google-perftools on Debian systems (`pr#22701 <https://github.com/ceph/ceph/pull/22701>`_, James McClune)
+* doc: remove duplicate python packages (`pr#22203 <https://github.com/ceph/ceph/pull/22203>`_, Stefan Kooman)
+* doc: Remove upstart files and references (`pr#23582 <https://github.com/ceph/ceph/pull/23582>`_, Brad Hubbard)
+* doc: Remove value 'mon_osd_max_split_count' (`pr#26584 <https://github.com/ceph/ceph/pull/26584>`_, Kai Wagner)
+* doc: replace rgw_namespace_expire_secs with rgw_nfs_namespace_expire_secs (`pr#20794 <https://github.com/ceph/ceph/pull/20794>`_, chnmagnus)
+* doc: rewrote the iscsi-target-cli installation (`pr#23190 <https://github.com/ceph/ceph/pull/23190>`_, Massimiliano Cuttini)
+* doc: rgw: fix tagging support status (`issue#24164 <http://tracker.ceph.com/issues/24164>`_, `pr#22206 <https://github.com/ceph/ceph/pull/22206>`_, Abhishek Lekshmanan)
+* doc: rgw: fix the default value of usage log setting (`issue#37856 <http://tracker.ceph.com/issues/37856>`_, `pr#25892 <https://github.com/ceph/ceph/pull/25892>`_, Abhishek Lekshmanan)
+* doc: Rook/orchestrator doc fixes (`pr#23472 <https://github.com/ceph/ceph/pull/23472>`_, John Spray)
+* doc: s/doc/ref for dashboard urls (`pr#22772 <https://github.com/ceph/ceph/pull/22772>`_, Jos Collin)
+* doc: sort releases by date and version (`pr#25972 <https://github.com/ceph/ceph/pull/25972>`_, Noah Watkins)
+* doc: Spelling fixes in BlueStore config reference (`pr#23715 <https://github.com/ceph/ceph/pull/23715>`_, Bryan Stillwell)
+* doc: Spelling fixes in Network config reference (`pr#23727 <https://github.com/ceph/ceph/pull/23727>`_, libingyang)
+* doc: SubmittingPatches: added inline markup to important references (`pr#25978 <https://github.com/ceph/ceph/pull/25978>`_, James McClune)
+* docs: update rgw info for mimic (`pr#22305 <https://github.com/ceph/ceph/pull/22305>`_, Yehuda Sadeh)
+* doc: test/crimson: do not use unit.cc as the driver of unittest_seastar_denc (`pr#23937 <https://github.com/ceph/ceph/pull/23937>`_, Kefu Chai)
+* doc: test/fio: Added tips for compilation of fio with 'rados' engine (`pr#24199 <https://github.com/ceph/ceph/pull/24199>`_, Adam Kupczyk)
+* doc: test/msgr: add missing #include (`pr#23947 <https://github.com/ceph/ceph/pull/23947>`_, Kefu Chai)
+* doc: Tidy up description wording and spelling (`pr#22599 <https://github.com/ceph/ceph/pull/22599>`_, Anthony D'Atri)
+* doc: tweak RBD iSCSI docs to point to merged tooling repo (`pr#24963 <https://github.com/ceph/ceph/pull/24963>`_, Jason Dillaman)
+* doc: typo fixes, s/Requered/Required/ (`pr#26406 <https://github.com/ceph/ceph/pull/26406>`_, Drunkard Zhang)
+* doc: update blkin changes (`pr#22317 <https://github.com/ceph/ceph/pull/22317>`_, Mahati Chamarthy)
+* doc: Update cpp.rst to accommodate the new APIs in libs3 (`pr#22162 <https://github.com/ceph/ceph/pull/22162>`_, Zhanhao Liu)
+* doc: Updated Ceph Dashboard documentation (`pr#26626 <https://github.com/ceph/ceph/pull/26626>`_, Lenz Grimmer)
+* doc: updated Ceph documentation links (`pr#25797 <https://github.com/ceph/ceph/pull/25797>`_, James McClune)
+* doc: updated cluster map reference link (`pr#24460 <https://github.com/ceph/ceph/pull/24460>`_, James McClune)
+* doc: updated crush map tunables link (`pr#24462 <https://github.com/ceph/ceph/pull/24462>`_, James McClune)
+* doc: Updated dashboard documentation (features, SSL config) (`pr#22059 <https://github.com/ceph/ceph/pull/22059>`_, Lenz Grimmer)
+* doc: Updated feature list and overview in dashboard.rst (`pr#26143 <https://github.com/ceph/ceph/pull/26143>`_, Lenz Grimmer)
+* doc: updated get-involved.rst for ceph-dashboard (`pr#22663 <https://github.com/ceph/ceph/pull/22663>`_, Jos Collin)
+* doc: Updated Mgr Dashboard documentation (`pr#24030 <https://github.com/ceph/ceph/pull/24030>`_, Lenz Grimmer)
+* doc: updated multisite documentation (`issue#26997 <http://tracker.ceph.com/issues/26997>`_, `pr#23660 <https://github.com/ceph/ceph/pull/23660>`_, James McClune)
+* doc: updated reference link for creating new disk offerings in cloudstack (`pr#22250 <https://github.com/ceph/ceph/pull/22250>`_, James McClune)
+* doc: updated reference link for log based PG (`pr#26611 <https://github.com/ceph/ceph/pull/26611>`_, James McClune)
+* doc: updated rgw multitenancy link (`pr#25929 <https://github.com/ceph/ceph/pull/25929>`_, James McClune)
+* doc: updated the overview and glossary for dashboard (`pr#22750 <https://github.com/ceph/ceph/pull/22750>`_, Jos Collin)
+* doc: updated wording from federated to multisite (`pr#24670 <https://github.com/ceph/ceph/pull/24670>`_, James McClune)
+* doc: Update mgr/zabbix plugin documentation with link to Zabbix template (`pr#24584 <https://github.com/ceph/ceph/pull/24584>`_, Wido den Hollander)
+* doc: update the description for SPDK in bluestore-config-ref.rst (`pr#22365 <https://github.com/ceph/ceph/pull/22365>`_, tone-zhang)
+* doc: use :command: for subcommands in ceph-bluestore-tool manpage (`issue#24800 <http://tracker.ceph.com/issues/24800>`_, `pr#23114 <https://github.com/ceph/ceph/pull/23114>`_, Nathan Cutler)
+* doc: use preferred commands for ceph config-key (`pr#26527 <https://github.com/ceph/ceph/pull/26527>`_, Changcheng Liu)
+* doc: warn about how 'rados put' works in the manpage (`pr#25757 <https://github.com/ceph/ceph/pull/25757>`_, Greg Farnum)
+* doc: Wip githubmap (`pr#25950 <https://github.com/ceph/ceph/pull/25950>`_, Greg Farnum)
+* erasure-code,test: silence -Wunused-variable warnings (`pr#25200 <https://github.com/ceph/ceph/pull/25200>`_, Kefu Chai)
+* example/librados: remove dependency on Boost system library (`issue#25054 <http://tracker.ceph.com/issues/25054>`_, `pr#23159 <https://github.com/ceph/ceph/pull/23159>`_, Nathan Cutler)
+* githubmap: update contributors (`pr#22522 <https://github.com/ceph/ceph/pull/22522>`_, Kefu Chai)
+* git: Ignore tags anywhere (`pr#26159 <https://github.com/ceph/ceph/pull/26159>`_, David Zafman)
+* include/buffer.h: do not use ceph_assert() unless __CEPH__ is defined (`pr#23803 <https://github.com/ceph/ceph/pull/23803>`_, Kefu Chai)
+* install-deps.sh: Fixes for RHEL 7 (`pr#26393 <https://github.com/ceph/ceph/pull/26393>`_, Zack Cerza)
+* kv/MemDB: add perfcounter (`pr#10305 <https://github.com/ceph/ceph/pull/10305>`_, Jianpeng Ma)
+* librados: add a rados_omap_iter_size function (`issue#26948 <http://tracker.ceph.com/issues/26948>`_, `pr#23593 <https://github.com/ceph/ceph/pull/23593>`_, Jeff Layton)
+* librados: block MgrClient::start_command until mgrmap (`pr#21811 <https://github.com/ceph/ceph/pull/21811>`_, John Spray, Kefu Chai)
+* librados: fix admin/build-doc warning (`pr#25706 <https://github.com/ceph/ceph/pull/25706>`_, Jos Collin)
+* librados: fix buffer overflow for aio_exec python binding (`pr#21775 <https://github.com/ceph/ceph/pull/21775>`_, Aleksei Gutikov)
+* librados: fix unitialized timeout in wait_for_osdmap (`pr#24721 <https://github.com/ceph/ceph/pull/24721>`_, Casey Bodley)
+* librados: Include memory for unique_ptr definition (`issue#35833 <http://tracker.ceph.com/issues/35833>`_, `pr#23992 <https://github.com/ceph/ceph/pull/23992>`_, Brad Hubbard)
+* librados: Reject the invalid pool create request at client side, rath… (`pr#21299 <https://github.com/ceph/ceph/pull/21299>`_, Yang Honggang)
+* librados: return ENOENT if pool_id invalid (`pr#21609 <https://github.com/ceph/ceph/pull/21609>`_, Li Wang)
+* librados: split C++ and C APIs into different source files (`pr#24616 <https://github.com/ceph/ceph/pull/24616>`_, Kefu Chai)
+* librados: use ceph::async::Completion for asio bindings (`pr#21920 <https://github.com/ceph/ceph/pull/21920>`_, Casey Bodley)
+* librados: use steady clock for rados_mon_op_timeout (`pr#20004 <https://github.com/ceph/ceph/pull/20004>`_, Mohamad Gebai)
+* librbd: add missing shutdown states to managed lock helper (`issue#38387 <http://tracker.ceph.com/issues/38387>`_, `pr#26523 <https://github.com/ceph/ceph/pull/26523>`_, Jason Dillaman)
+* librbd: add new configuration option to always move deleted items to the trash (`pr#24476 <https://github.com/ceph/ceph/pull/24476>`_, Jason Dillaman)
+* librbd: add rbd image access/modified timestamps (`pr#21114 <https://github.com/ceph/ceph/pull/21114>`_, Julien Collet)
+* librbd: add trash purge api calls (`pr#24427 <https://github.com/ceph/ceph/pull/24427>`_, Julien Collet, Theofilos Mouratidis, Jason Dillaman)
+* librbd: always open first parent image if it exists for a snapshot (`pr#23733 <https://github.com/ceph/ceph/pull/23733>`_, Jason Dillaman)
+* librbd: avoid aggregate-initializing any static_visitor (`pr#26876 <https://github.com/ceph/ceph/pull/26876>`_, Willem Jan Withagen)
+* librbd: blacklisted client might not notice it lost the lock (`issue#34534 <http://tracker.ceph.com/issues/34534>`_, `pr#23829 <https://github.com/ceph/ceph/pull/23829>`_, Jason Dillaman)
+* librbd: block_name_prefix is not created randomly (`issue#24634 <http://tracker.ceph.com/issues/24634>`_, `pr#22675 <https://github.com/ceph/ceph/pull/22675>`_, hyun-ha)
+* librbd: bypass pool validation if "rbd_validate_pool" is false (`pr#26878 <https://github.com/ceph/ceph/pull/26878>`_, Jason Dillaman)
+* librbd: commit IO as safe when complete if writeback cache is disabled (`issue#23516 <http://tracker.ceph.com/issues/23516>`_, `pr#22342 <https://github.com/ceph/ceph/pull/22342>`_, Jason Dillaman)
+* librbd: corrected usage of ImageState::open flag parameter (`pr#25428 <https://github.com/ceph/ceph/pull/25428>`_, Mykola Golub)
+* librbd: deep_copy: don't hide parent if zero overlap for snapshot (`issue#24545 <http://tracker.ceph.com/issues/24545>`_, `pr#22587 <https://github.com/ceph/ceph/pull/22587>`_, Mykola Golub)
+* librbd: deep copy optionally support flattening cloned image (`issue#22787 <http://tracker.ceph.com/issues/22787>`_, `pr#21624 <https://github.com/ceph/ceph/pull/21624>`_, Mykola Golub)
+* librbd: deep_copy: resize head object map if needed (`issue#24399 <http://tracker.ceph.com/issues/24399>`_, `pr#22415 <https://github.com/ceph/ceph/pull/22415>`_, Mykola Golub)
+* librbd: deep-copy should not write to objects that cannot exist (`issue#25000 <http://tracker.ceph.com/issues/25000>`_, `pr#23132 <https://github.com/ceph/ceph/pull/23132>`_, Jason Dillaman)
+* librbd: disable image mirroring when moving to trash (`pr#25509 <https://github.com/ceph/ceph/pull/25509>`_, Mykola Golub)
+* librbd: disallow trash restoring when image being migrated (`pr#25529 <https://github.com/ceph/ceph/pull/25529>`_, songweibin)
+* librbd: don't do create+truncate for discards with copyup (`pr#26825 <https://github.com/ceph/ceph/pull/26825>`_, Ilya Dryomov)
+* librbd: ensure compare-and-write doesn't skip compare after copyup (`issue#38383 <http://tracker.ceph.com/issues/38383>`_, `pr#26519 <https://github.com/ceph/ceph/pull/26519>`_, Ilya Dryomov)
+* librbd: extend API to include parent/child namespaces and image ids (`issue#36650 <http://tracker.ceph.com/issues/36650>`_, `pr#25194 <https://github.com/ceph/ceph/pull/25194>`_, Jason Dillaman)
+* librbd: fix crash when opening nonexistent snapshot (`issue#24637 <http://tracker.ceph.com/issues/24637>`_, `pr#22676 <https://github.com/ceph/ceph/pull/22676>`_, Mykola Golub)
+* librbd: fixed assert when flattening clone with zero overlap (`issue#35702 <http://tracker.ceph.com/issues/35702>`_, `pr#24045 <https://github.com/ceph/ceph/pull/24045>`_, Jason Dillaman)
+* librbd: fix missing unblock_writes if shrink is not allowed (`issue#36778 <http://tracker.ceph.com/issues/36778>`_, `pr#25055 <https://github.com/ceph/ceph/pull/25055>`_, runsisi)
+* librbd: fix possible unnecessary latency when requeue request (`pr#23815 <https://github.com/ceph/ceph/pull/23815>`_, Song Shun)
+* librbd: fix potential live migration after commit issues due to not refreshed image header (`pr#23839 <https://github.com/ceph/ceph/pull/23839>`_, Mykola Golub)
+* librbd: fix were_all_throttled() to avoid incorrect ret-value (`issue#38504 <http://tracker.ceph.com/issues/38504>`_, `pr#26688 <https://github.com/ceph/ceph/pull/26688>`_, Dongsheng Yang)
+* librbd: flatten operation should use object map (`issue#23445 <http://tracker.ceph.com/issues/23445>`_, `pr#23941 <https://github.com/ceph/ceph/pull/23941>`_, Mykola Golub)
+* librbd: force 'invalid object map' flag on-disk update (`issue#24434 <http://tracker.ceph.com/issues/24434>`_, `pr#22444 <https://github.com/ceph/ceph/pull/22444>`_, Mykola Golub)
+* librbd: get_parent API method should properly handle migrating image (`issue#37998 <http://tracker.ceph.com/issues/37998>`_, `pr#26337 <https://github.com/ceph/ceph/pull/26337>`_, Jason Dillaman)
+* librbd: handle aio failure in ManagedLock and PreReleaseRequest (`pr#20112 <https://github.com/ceph/ceph/pull/20112>`_, liyichao)
+* librbd: improve object map performance under high IOPS workloads (`issue#38538 <http://tracker.ceph.com/issues/38538>`_, `pr#26721 <https://github.com/ceph/ceph/pull/26721>`_, Jason Dillaman)
+* librbd: journaling unable request can not be sent to remote lock owner (`issue#26939 <http://tracker.ceph.com/issues/26939>`_, `pr#23649 <https://github.com/ceph/ceph/pull/23649>`_, Mykola Golub)
+* librbd: keep access/modified timestamp updates out of IO path (`issue#37745 <http://tracker.ceph.com/issues/37745>`_, `pr#25883 <https://github.com/ceph/ceph/pull/25883>`_, Jason Dillaman)
+* librbd: make it possible to migrate parent images (`pr#25945 <https://github.com/ceph/ceph/pull/25945>`_, Mykola Golub)
+* librbd: move mirror peer attribute handling from CLI to API (`pr#25096 <https://github.com/ceph/ceph/pull/25096>`_, Jason Dillaman)
+* librbd: namespace create/remove/list support (`pr#22608 <https://github.com/ceph/ceph/pull/22608>`_, Jason Dillaman)
+* librbd: object copy state machine might dereference a deleted object (`issue#36220 <http://tracker.ceph.com/issues/36220>`_, `pr#24293 <https://github.com/ceph/ceph/pull/24293>`_, Jason Dillaman)
+* librbd: object map improperly flagged as invalidated (`issue#24516 <http://tracker.ceph.com/issues/24516>`_, `pr#24105 <https://github.com/ceph/ceph/pull/24105>`_, Jason Dillaman)
+* librbd: optionally limit journal in-flight appends (`pr#22983 <https://github.com/ceph/ceph/pull/22983>`_, Mykola Golub)
+* librbd:optionally support FUA (force unit access) on write requests (`issue#19366 <http://tracker.ceph.com/issues/19366>`_, `pr#22945 <https://github.com/ceph/ceph/pull/22945>`_, ningtao)
+* librbd: pool and image level config overrides (`pr#23743 <https://github.com/ceph/ceph/pull/23743>`_, Mykola Golub)
+* librbd: potential object map race with copyup state machine (`issue#24516 <http://tracker.ceph.com/issues/24516>`_, `pr#24253 <https://github.com/ceph/ceph/pull/24253>`_, Jason Dillaman)
+* librbd: potential race on image create request complete (`issue#24910 <http://tracker.ceph.com/issues/24910>`_, `pr#23639 <https://github.com/ceph/ceph/pull/23639>`_, Mykola Golub)
+* librbd: prevent the use of internal feature bits from external users (`issue#24165 <http://tracker.ceph.com/issues/24165>`_, `pr#22072 <https://github.com/ceph/ceph/pull/22072>`_, Jason Dillaman)
+* librbd: prevent use of namespaces on pre-nautilus OSDs (`pr#23823 <https://github.com/ceph/ceph/pull/23823>`_, Jason Dillaman)
+* librbd: properly filter out trashed non-user images on purge (`pr#26079 <https://github.com/ceph/ceph/pull/26079>`_, Mykola Golub)
+* librbd: properly handle potential object map failures (`issue#36074 <http://tracker.ceph.com/issues/36074>`_, `pr#24179 <https://github.com/ceph/ceph/pull/24179>`_, Jason Dillaman)
+* librbd: race condition possible when validating RBD pool (`issue#38500 <http://tracker.ceph.com/issues/38500>`_, `pr#26683 <https://github.com/ceph/ceph/pull/26683>`_, Jason Dillaman)
+* librbd: reduce the TokenBucket fill cycle and support bursting io configuration (`pr#24214 <https://github.com/ceph/ceph/pull/24214>`_, Shiyang Ruan)
+* librbd: remove template declaration of a non-template function (`pr#23790 <https://github.com/ceph/ceph/pull/23790>`_, Shiyang Ruan)
+* librbd: reset snaps in rbd_snap_list() (`issue#37508 <http://tracker.ceph.com/issues/37508>`_, `pr#25379 <https://github.com/ceph/ceph/pull/25379>`_, Kefu Chai)
+* librbd: restart io if migration parent gone (`issue#36710 <http://tracker.ceph.com/issues/36710>`_, `pr#25175 <https://github.com/ceph/ceph/pull/25175>`_, Mykola Golub)
+* librbd: send_copyup() fixes and cleanups (`pr#26483 <https://github.com/ceph/ceph/pull/26483>`_, Ilya Dryomov)
+* librbd: simplify config override handling (`pr#24450 <https://github.com/ceph/ceph/pull/24450>`_, Jason Dillaman)
+* librbd: skip small, unaligned discard extents by default (`issue#38146 <http://tracker.ceph.com/issues/38146>`_, `pr#26432 <https://github.com/ceph/ceph/pull/26432>`_, Jason Dillaman)
+* librbd: support bps throttle and throttle read and write seperately (`pr#21635 <https://github.com/ceph/ceph/pull/21635>`_, Dongsheng Yang)
+* librbd: support migrating images with minimal downtime (`issue#18430 <http://tracker.ceph.com/issues/18430>`_, `issue#24439 <http://tracker.ceph.com/issues/24439>`_, `issue#26874 <http://tracker.ceph.com/issues/26874>`_, `issue#23659 <http://tracker.ceph.com/issues/23659>`_, `pr#15831 <https://github.com/ceph/ceph/pull/15831>`_, Patrick Donnelly, Sage Weil, Alfredo Deza, Kefu Chai, Patrick Nawracay, Pavani Rajula, Mykola Golub, Casey Bodley, Yingxin, Jason Dillaman)
+* librbd: support v2 cloning across namespaces (`pr#23662 <https://github.com/ceph/ceph/pull/23662>`_, Jason Dillaman)
+* librbd: use object map when doing snap rollback (`pr#23110 <https://github.com/ceph/ceph/pull/23110>`_, songweibin)
+* librbd: utilize the journal disabled policy when removing images (`issue#23512 <http://tracker.ceph.com/issues/23512>`_, `pr#22327 <https://github.com/ceph/ceph/pull/22327>`_, Jason Dillaman)
+* librbd: validate data pool for self-managed snapshot support (`pr#22737 <https://github.com/ceph/ceph/pull/22737>`_, Mykola Golub)
+* librbd: workaround an ICE of GCC (`issue#37719 <http://tracker.ceph.com/issues/37719>`_, `pr#25733 <https://github.com/ceph/ceph/pull/25733>`_, Kefu Chai)
+* log: avoid heap allocations for most log entries (`pr#23721 <https://github.com/ceph/ceph/pull/23721>`_, Patrick Donnelly)
+* lvm: when osd creation fails log the exception (`issue#24456 <http://tracker.ceph.com/issues/24456>`_, `pr#22627 <https://github.com/ceph/ceph/pull/22627>`_, Andrew Schoen)
+* mailmap,organization: Update sangfor affiliation (`pr#25225 <https://github.com/ceph/ceph/pull/25225>`_, Zengran Zhang)
+* mds: add reference when setting Connection::priv to existing session (`pr#22384 <https://github.com/ceph/ceph/pull/22384>`_, "Yan, Zheng")
+* mds: fix leak of MDSCacheObject::waiting (`issue#24289 <http://tracker.ceph.com/issues/24289>`_, `pr#22307 <https://github.com/ceph/ceph/pull/22307>`_, "Yan, Zheng")
+* mds: fix some memory leak (`issue#24289 <http://tracker.ceph.com/issues/24289>`_, `pr#22240 <https://github.com/ceph/ceph/pull/22240>`_, "Yan, Zheng")
+* mds,messages: silence -Wclass-memaccess warnings (`pr#21845 <https://github.com/ceph/ceph/pull/21845>`_, Kefu Chai)
+* mds: properly journal root inode's snaprealm (`issue#24343 <http://tracker.ceph.com/issues/24343>`_, `pr#22320 <https://github.com/ceph/ceph/pull/22320>`_, "Yan, Zheng")
+* mds: remove obsolete comments (`pr#25549 <https://github.com/ceph/ceph/pull/25549>`_, Patrick Donnelly)
+* mds: reply session reject for open request from blacklisted client (`pr#21941 <https://github.com/ceph/ceph/pull/21941>`_, Yan, Zheng, "Yan, Zheng")
+* mgr: Add ability to trigger a cluster/audit log message from Python (`pr#24239 <https://github.com/ceph/ceph/pull/24239>`_, Volker Theile)
+* mgr: Add `HandleCommandResult` namedtuple (`pr#25261 <https://github.com/ceph/ceph/pull/25261>`_, Sebastian Wagner)
+* mgr: add limit param to osd perf query (`pr#25151 <https://github.com/ceph/ceph/pull/25151>`_, Mykola Golub)
+* mgr: add per pool force-recovery/backfill commands (`issue#38456 <http://tracker.ceph.com/issues/38456>`_, `pr#26560 <https://github.com/ceph/ceph/pull/26560>`_, xie xingguo)
+* mgr: add per pool scrub commands (`pr#26532 <https://github.com/ceph/ceph/pull/26532>`_, xie xingguo)
+* mgr: Allow modules to get/set other module options (`pr#25651 <https://github.com/ceph/ceph/pull/25651>`_, Volker Theile)
+* mgr: Allow rook to scale the mon count (`pr#26405 <https://github.com/ceph/ceph/pull/26405>`_, Jeff Layton)
+* mgr: always on modules v2 (`pr#23970 <https://github.com/ceph/ceph/pull/23970>`_, Noah Watkins)
+* mgr/ansible: Add/remove hosts (`pr#26241 <https://github.com/ceph/ceph/pull/26241>`_, Juan Miguel Olmo Martínez)
+* mgr/ansible: Replace Ansible playbook used to retrieve storage devices data (`pr#26023 <https://github.com/ceph/ceph/pull/26023>`_, Juan Miguel Olmo Martínez)
+* mgr/ansible: Replace deprecated <get_config> calls (`pr#25964 <https://github.com/ceph/ceph/pull/25964>`_, Juan Miguel Olmo Martínez)
+* mgr: Centralize PG_STATES to MgrModule (`pr#22594 <https://github.com/ceph/ceph/pull/22594>`_, Wido den Hollander)
+* mgr: ceph-mgr: hold lock while accessing the request list and submitting request (`pr#25048 <https://github.com/ceph/ceph/pull/25048>`_, Jerry Lee)
+* mgr: change 'bytes' dynamic perf counters to COUNTER type (`pr#25908 <https://github.com/ceph/ceph/pull/25908>`_, Mykola Golub)
+* mgr: create always on class of modules (`pr#23106 <https://github.com/ceph/ceph/pull/23106>`_, Noah Watkins)
+* mgr: create shell OSD performance query class (`pr#24117 <https://github.com/ceph/ceph/pull/24117>`_, Mykola Golub)
+* mgr/dashboard: About modal proposed changes (`issue#35693 <http://tracker.ceph.com/issues/35693>`_, `pr#25376 <https://github.com/ceph/ceph/pull/25376>`_, Kanika Murarka)
+* mgr/dashboard: Add ability to list,set and unset cluster-wide OSD flags to the backend (`issue#24056 <http://tracker.ceph.com/issues/24056>`_, `pr#21998 <https://github.com/ceph/ceph/pull/21998>`_, Patrick Nawracay)
+* mgr/dashboard: Add a 'clear filter' button to configuration page (`issue#36173 <http://tracker.ceph.com/issues/36173>`_, `pr#25712 <https://github.com/ceph/ceph/pull/25712>`_, familyuu)
+* mgr/dashboard: add a script to run an API request on a rook cluster (`pr#25991 <https://github.com/ceph/ceph/pull/25991>`_, Jeff Layton)
+* mgr/dashboard: Add a unit test form helper class (`pr#24633 <https://github.com/ceph/ceph/pull/24633>`_, Stephan Müller)
+* mgr/dashboard: Add backend support for changing dashboard configuration settings via the REST API (`pr#22457 <https://github.com/ceph/ceph/pull/22457>`_, Patrick Nawracay)
+* mgr/dashboard: Add breadcrumbs component (`issue#24781 <http://tracker.ceph.com/issues/24781>`_, `pr#23414 <https://github.com/ceph/ceph/pull/23414>`_, Tiago Melo)
+* mgr/dashboard: add columns to Pools table (`pr#25791 <https://github.com/ceph/ceph/pull/25791>`_, Alfonso Martínez)
+* mgr/dashboard: Add decorator to skip parameter encoding (`issue#26856 <http://tracker.ceph.com/issues/26856>`_, `pr#23419 <https://github.com/ceph/ceph/pull/23419>`_, Tiago Melo)
+* mgr/dashboard: Add description to menu items on mobile navigation (`pr#26198 <https://github.com/ceph/ceph/pull/26198>`_, Sebastian Krah)
+* mgr/dashboard: added command to tox.ini (`pr#26073 <https://github.com/ceph/ceph/pull/26073>`_, Alfonso Martínez)
+* mgr/dashboard: added 'env_build' to 'npm run e2e' (`pr#26165 <https://github.com/ceph/ceph/pull/26165>`_, Alfonso Martínez)
+* mgr/dashboard: Added new validators (`pr#22526 <https://github.com/ceph/ceph/pull/22526>`_, Stephan Müller)
+* mgr/dashboard: Add error handling on the frontend (`pr#21820 <https://github.com/ceph/ceph/pull/21820>`_, Tiago Melo)
+* mgr/dashboard: add Feature Toggles (`issue#37530 <http://tracker.ceph.com/issues/37530>`_, `pr#26102 <https://github.com/ceph/ceph/pull/26102>`_, Ernesto Puerta)
+* mgr/dashboard: Add Filesystems list component (`pr#21913 <https://github.com/ceph/ceph/pull/21913>`_, Tiago Melo)
+* mgr/dashboard: Add filtered rows number in table footer (`pr#22504 <https://github.com/ceph/ceph/pull/22504>`_, Tiago Melo)
+* mgr/dashboard: Add gap between panel footer buttons (`pr#23796 <https://github.com/ceph/ceph/pull/23796>`_, Volker Theile)
+* mgr/dashboard: Add guideline how to brand the UI and update the color scheme (`pr#25988 <https://github.com/ceph/ceph/pull/25988>`_, Sebastian Krah)
+* mgr/dashboard: Add help menu entry (`pr#22303 <https://github.com/ceph/ceph/pull/22303>`_, Ricardo Marques)
+* mgr/dashboard: Add i18n support (`pr#24803 <https://github.com/ceph/ceph/pull/24803>`_, Sebastian Krah, Tiago Melo)
+* mgr/dashboard: Add implicit wait in e2e tests (`pr#26384 <https://github.com/ceph/ceph/pull/26384>`_, Tiago Melo)
+* mgr/dashboard: Add info to Pools table (`pr#25489 <https://github.com/ceph/ceph/pull/25489>`_, Alfonso Martínez)
+* mgr/dashboard: Add iSCSI discovery authentication UI (`pr#26320 <https://github.com/ceph/ceph/pull/26320>`_, Tiago Melo)
+* mgr/dashboard: Add iSCSI Target Edit UI (`issue#38014 <http://tracker.ceph.com/issues/38014>`_, `pr#26367 <https://github.com/ceph/ceph/pull/26367>`_, Tiago Melo)
+* mgr/dashboard: Add left padding to helper icon (`pr#24631 <https://github.com/ceph/ceph/pull/24631>`_, Stephan Müller)
+* mgr/dashboard: Add missing frontend I18N (`issue#36719 <http://tracker.ceph.com/issues/36719>`_, `pr#25654 <https://github.com/ceph/ceph/pull/25654>`_, Tiago Melo)
+* mgr/dashboard: Add missing test requirement "werkzeug" (`pr#24628 <https://github.com/ceph/ceph/pull/24628>`_, Stephan Müller)
+* mgr/dashboard: Add NFS status endpoint (`issue#38399 <http://tracker.ceph.com/issues/38399>`_, `pr#26539 <https://github.com/ceph/ceph/pull/26539>`_, Tiago Melo)
+* mgr/dashboard: Add 'no-unused-variable' rule to tslint (`pr#22328 <https://github.com/ceph/ceph/pull/22328>`_, Tiago Melo)
+* mgr/dashboard: Add permission validation to the  "Purge Trash" button (`issue#36272 <http://tracker.ceph.com/issues/36272>`_, `pr#24370 <https://github.com/ceph/ceph/pull/24370>`_, Tiago Melo)
+* mgr/dashboard: Add pool cache tiering details tab (`issue#25158 <http://tracker.ceph.com/issues/25158>`_, `pr#25602 <https://github.com/ceph/ceph/pull/25602>`_, familyuu)
+* mgr/dashboard: Add Pool update endpoint (`pr#21881 <https://github.com/ceph/ceph/pull/21881>`_, Sebastian Wagner, Stephan Müller)
+* mgr/dashboard: Add Prettier formatter to the frontend (`pr#21819 <https://github.com/ceph/ceph/pull/21819>`_, Tiago Melo)
+* mgr/dashboard: add profiles to set cluster's rebuild performance (`pr#24968 <https://github.com/ceph/ceph/pull/24968>`_, Tatjana Dehler)
+* mgr/dashboard: add pytest plugin: faulthandler (`pr#25053 <https://github.com/ceph/ceph/pull/25053>`_, Alfonso Martínez)
+* mgr/dashboard: Add REST API for role management (`pr#23322 <https://github.com/ceph/ceph/pull/23322>`_, Ricardo Marques)
+* mgr/dashboard: Add scrub action to the OSDs table (`pr#22122 <https://github.com/ceph/ceph/pull/22122>`_, Tiago Melo)
+* mgr/dashboard: Adds custom timepicker for grafana iframes (`pr#25583 <https://github.com/ceph/ceph/pull/25583>`_, Kanika Murarka)
+* mgr/dashboard: Adds ECP management to the frontend (`pr#24627 <https://github.com/ceph/ceph/pull/24627>`_, Stephan Müller)
+* mgr/dashboard: Add shared Confirmation Modal (`pr#22601 <https://github.com/ceph/ceph/pull/22601>`_, Tiago Melo)
+* mgr/dashboard: add supported flag information to config options documentation (`pr#22760 <https://github.com/ceph/ceph/pull/22760>`_, Tatjana Dehler)
+* mgr/dashboard: Add support for iSCSI's multi backstores (UI) (`pr#26575 <https://github.com/ceph/ceph/pull/26575>`_, Tiago Melo)
+* mgr/dashboard: Add support for managing individual OSD settings/characteristics in the frontend (`issue#36487 <http://tracker.ceph.com/issues/36487>`_, `issue#36444 <http://tracker.ceph.com/issues/36444>`_, `issue#35448 <http://tracker.ceph.com/issues/35448>`_, `issue#36188 <http://tracker.ceph.com/issues/36188>`_, `issue#35811 <http://tracker.ceph.com/issues/35811>`_, `issue#35816 <http://tracker.ceph.com/issues/35816>`_, `issue#36086 <http://tracker.ceph.com/issues/36086>`_, `pr#24606 <https://github.com/ceph/ceph/pull/24606>`_, Patrick Nawracay)
+* mgr/dashboard: Add support for managing individual OSD settings in the backend (`issue#24270 <http://tracker.ceph.com/issues/24270>`_, `pr#23491 <https://github.com/ceph/ceph/pull/23491>`_, Patrick Nawracay)
+* mgr/dashboard: Add support for managing RBD QoS (`issue#37572 <http://tracker.ceph.com/issues/37572>`_, `issue#38004 <http://tracker.ceph.com/issues/38004>`_, `issue#37570 <http://tracker.ceph.com/issues/37570>`_, `issue#37936 <http://tracker.ceph.com/issues/37936>`_, `issue#37574 <http://tracker.ceph.com/issues/37574>`_, `issue#36191 <http://tracker.ceph.com/issues/36191>`_, `issue#37845 <http://tracker.ceph.com/issues/37845>`_, `issue#37569 <http://tracker.ceph.com/issues/37569>`_, `pr#25233 <https://github.com/ceph/ceph/pull/25233>`_, Patrick Nawracay)
+* mgr/dashboard: Add support for RBD Trash (`issue#24272 <http://tracker.ceph.com/issues/24272>`_, `pr#23351 <https://github.com/ceph/ceph/pull/23351>`_, Tiago Melo)
+* mgr/dashboard: Add support for URI encode (`issue#24621 <http://tracker.ceph.com/issues/24621>`_, `pr#22672 <https://github.com/ceph/ceph/pull/22672>`_, Tiago Melo)
+* mgr/dashboard: Add table actions component (`pr#23779 <https://github.com/ceph/ceph/pull/23779>`_, Stephan Müller)
+* mgr/dashboard: Add table of contents to HACKING.rst (`pr#25812 <https://github.com/ceph/ceph/pull/25812>`_, Sebastian Krah)
+* mgr/dashboard: Add token authentication to Grafana proxy (`pr#22459 <https://github.com/ceph/ceph/pull/22459>`_, Patrick Nawracay)
+* mgr/dashboard: Add TSLint rule "no-unused-variable" (`pr#24699 <https://github.com/ceph/ceph/pull/24699>`_, Alfonso Martínez)
+* mgr/dashboard: Add UI for Cluster-wide OSD Flags configuration (`pr#22461 <https://github.com/ceph/ceph/pull/22461>`_, Tiago Melo)
+* mgr/dashboard: Add UI for disabling ACL authentication (`issue#38218 <http://tracker.ceph.com/issues/38218>`_, `pr#26388 <https://github.com/ceph/ceph/pull/26388>`_, Tiago Melo)
+* mgr/dashboard: Add UI to configure the telemetry mgr plugin (`pr#25989 <https://github.com/ceph/ceph/pull/25989>`_, Volker Theile)
+* mgr/dashboard: Add unique validator (`pr#23802 <https://github.com/ceph/ceph/pull/23802>`_, Volker Theile)
+* mgr/dashboard: Allow "/" in pool name (`issue#38302 <http://tracker.ceph.com/issues/38302>`_, `pr#26408 <https://github.com/ceph/ceph/pull/26408>`_, Tiago Melo)
+* mgr/dashboard: Allow insecure HTTPS in run-backend-api-request (`pr#21882 <https://github.com/ceph/ceph/pull/21882>`_, Sebastian Wagner)
+* mgr/dashboard: Allow renaming an existing Pool (`issue#36560 <http://tracker.ceph.com/issues/36560>`_, `pr#25107 <https://github.com/ceph/ceph/pull/25107>`_, guodan1)
+* mgr/dashboard: Audit REST API calls (`pr#24475 <https://github.com/ceph/ceph/pull/24475>`_, Volker Theile)
+* mgr/dashboard: Auto-create a name for RBD image snapshots (`pr#23735 <https://github.com/ceph/ceph/pull/23735>`_, Volker Theile)
+* mgr/dashboard: avoid blank content in Read/Write Card (`pr#25563 <https://github.com/ceph/ceph/pull/25563>`_, Alfonso Martínez)
+* mgr/dashboard: awsauth: fix python3 string decode problem (`pr#21794 <https://github.com/ceph/ceph/pull/21794>`_, Ricardo Dias)
+* mgr/dashboard: Can't handle user editing when tenants are specified (`pr#24757 <https://github.com/ceph/ceph/pull/24757>`_, Volker Theile)
+* mgr/dashboard: Catch LookupError when checking the RGW status (`pr#24028 <https://github.com/ceph/ceph/pull/24028>`_, Volker Theile)
+* mgr/dashboard: CdFormGroup (`pr#22644 <https://github.com/ceph/ceph/pull/22644>`_, Stephan Müller)
+* mgr/dashboard: Ceph dashboard user management from the UI (`pr#22758 <https://github.com/ceph/ceph/pull/22758>`_, Ricardo Marques)
+* mgr/dashboard: Change 'Client Recovery' title (`pr#26883 <https://github.com/ceph/ceph/pull/26883>`_, Ernesto Puerta)
+* mgr/dashboard: Changed background color of Masthead to brand gray (`issue#35690 <http://tracker.ceph.com/issues/35690>`_, `pr#25628 <https://github.com/ceph/ceph/pull/25628>`_, Neha Gupta)
+* mgr/dashboard: Changed default value of decimal point to 1 (`pr#22386 <https://github.com/ceph/ceph/pull/22386>`_, Tiago Melo)
+* mgr/dashboard: Change icon color in notifications (`pr#26586 <https://github.com/ceph/ceph/pull/26586>`_, Volker Theile)
+* mgr/dashboard: Check content-type before decode json response (`pr#24350 <https://github.com/ceph/ceph/pull/24350>`_, Ricardo Marques)
+* mgr/dashboard: check for existence of Grafana dashboard (`issue#36356 <http://tracker.ceph.com/issues/36356>`_, `pr#25154 <https://github.com/ceph/ceph/pull/25154>`_, Kanika Murarka)
+* mgr/dashboard: Cleanup of OSD list methods (`pr#24823 <https://github.com/ceph/ceph/pull/24823>`_, Stephan Müller)
+* mgr/dashboard: Cleanup of the cluster and audit log (`pr#26188 <https://github.com/ceph/ceph/pull/26188>`_, Sebastian Krah)
+* mgr/dashboard: Cleanup (`pr#24831 <https://github.com/ceph/ceph/pull/24831>`_, Patrick Nawracay)
+* mgr/dashboard: Clean up pylint's `disable:no-else-return` (`pr#26509 <https://github.com/ceph/ceph/pull/26509>`_, Patrick Nawracay)
+* mgr/dashboard: Cleanup Python code (`pr#26743 <https://github.com/ceph/ceph/pull/26743>`_, Volker Theile)
+* mgr/dashboard: Cleanup RGW config checks (`pr#22669 <https://github.com/ceph/ceph/pull/22669>`_, Volker Theile)
+* mgr/dashboard: Close modal dialogs on login screen (`pr#23328 <https://github.com/ceph/ceph/pull/23328>`_, Volker Theile)
+* mgr/dashboard: code cleanup (`pr#25502 <https://github.com/ceph/ceph/pull/25502>`_, Alfonso Martínez)
+* mgr/dashboard: Color variables for color codes (`issue#24575 <http://tracker.ceph.com/issues/24575>`_, `pr#22695 <https://github.com/ceph/ceph/pull/22695>`_, Kanika Murarka)
+* mgr/dashboard config options add (`issue#34528 <http://tracker.ceph.com/issues/34528>`_, `issue#24996 <http://tracker.ceph.com/issues/24996>`_, `issue#24455 <http://tracker.ceph.com/issues/24455>`_, `issue#36173 <http://tracker.ceph.com/issues/36173>`_, `pr#23230 <https://github.com/ceph/ceph/pull/23230>`_, Tatjana Dehler)
+* mgr/dashboard: Config options integration (read-only) depends on #22422 (`pr#21460 <https://github.com/ceph/ceph/pull/21460>`_, Tatjana Dehler)
+* mgr/dashboard: config options table cleanup (`issue#34533 <http://tracker.ceph.com/issues/34533>`_, `pr#24523 <https://github.com/ceph/ceph/pull/24523>`_, Tatjana Dehler)
+* mgr/dashboard: config option type names update (`issue#37843 <http://tracker.ceph.com/issues/37843>`_, `pr#25876 <https://github.com/ceph/ceph/pull/25876>`_, Tatjana Dehler)
+* mgr/dashboard: configs textarea disallow horizontal resize (`issue#36452 <http://tracker.ceph.com/issues/36452>`_, `pr#24614 <https://github.com/ceph/ceph/pull/24614>`_, Tatjana Dehler)
+* mgr/dashboard: Configure all mgr modules in UI (`pr#26116 <https://github.com/ceph/ceph/pull/26116>`_, Volker Theile)
+* mgr/dashboard: Confirmation modal doesn't close (`pr#24544 <https://github.com/ceph/ceph/pull/24544>`_, Volker Theile)
+* mgr/dashboard: Confusing tilted time stamps in the CephFS performance graph (`pr#25909 <https://github.com/ceph/ceph/pull/25909>`_, Volker Theile)
+* mgr/dashboard: consider config option default values (`issue#37683 <http://tracker.ceph.com/issues/37683>`_, `pr#25616 <https://github.com/ceph/ceph/pull/25616>`_, Tatjana Dehler)
+* mgr/dashboard: controller infrastructure refactor and new features (`pr#22210 <https://github.com/ceph/ceph/pull/22210>`_, Patrick Nawracay, Ricardo Dias)
+* mgr/dashboard: Correct permission decorator (`pr#26135 <https://github.com/ceph/ceph/pull/26135>`_, Tina Kallio)
+* mgr/dashboard: CRUSH map viewer (`issue#35684 <http://tracker.ceph.com/issues/35684>`_, `pr#24766 <https://github.com/ceph/ceph/pull/24766>`_, familyuu)
+* mgr/dashboard: CRUSH map viewer RFE (`issue#37794 <http://tracker.ceph.com/issues/37794>`_, `pr#26162 <https://github.com/ceph/ceph/pull/26162>`_, familyuu)
+* mgr/dashboard: Dashboard info cards refactoring (`pr#22902 <https://github.com/ceph/ceph/pull/22902>`_, Alfonso Martínez)
+* mgr/dashboard: Datatable error panel blinking on page loading (`pr#23316 <https://github.com/ceph/ceph/pull/23316>`_, Volker Theile)
+* mgr/dashboard: Deletion dialog falsely executes deletion when pressing 'Cancel' (`pr#22003 <https://github.com/ceph/ceph/pull/22003>`_, Volker Theile)
+* mgr/dashboard: Disable package-lock.json creation (`pr#22061 <https://github.com/ceph/ceph/pull/22061>`_, Tiago Melo)
+* mgr/dashboard: Disable RBD actions during task execution (`pr#23445 <https://github.com/ceph/ceph/pull/23445>`_, Ricardo Marques)
+* mgr/dashboard: disallow editing read-only config options (part 2) (`pr#26450 <https://github.com/ceph/ceph/pull/26450>`_, Tatjana Dehler)
+* mgr/dashboard: disallow editing read-only config options (`pr#26297 <https://github.com/ceph/ceph/pull/26297>`_, Tatjana Dehler)
+* mgr/dashboard: Display logged in user (`issue#24822 <http://tracker.ceph.com/issues/24822>`_, `pr#24213 <https://github.com/ceph/ceph/pull/24213>`_, guodan1, guodan)
+* mgr/dashboard: Display notification if RGW is not configured (`pr#21785 <https://github.com/ceph/ceph/pull/21785>`_, Volker Theile)
+* mgr/dashboard: Display RGW user/bucket quota max size in human readable form (`pr#23842 <https://github.com/ceph/ceph/pull/23842>`_, Volker Theile)
+* mgr/dashboard: Do not fetch pool list on RBD edit (`pr#22404 <https://github.com/ceph/ceph/pull/22404>`_, Ricardo Marques)
+* mgr/dashboard: Do not require cert for http (`issue#36069 <http://tracker.ceph.com/issues/36069>`_, `pr#24103 <https://github.com/ceph/ceph/pull/24103>`_, Boris Ranto)
+* mgr/dashboard: Drop iSCSI gateway name parameter (`pr#26984 <https://github.com/ceph/ceph/pull/26984>`_, Ricardo Marques)
+* mgr/dashboard: enable coverage for API tests (`pr#26851 <https://github.com/ceph/ceph/pull/26851>`_, Alfonso Martínez)
+* mgr/dashboard: Escape regex pattern in DeletionModalComponent (`issue#24902 <http://tracker.ceph.com/issues/24902>`_, `pr#23420 <https://github.com/ceph/ceph/pull/23420>`_, Tiago Melo)
+* mgr/dashboard: Exception.message doesn't exist on Python 3 (`pr#24349 <https://github.com/ceph/ceph/pull/24349>`_, Ricardo Marques)
+* mgr/dashboard: Extract/Refactor Task merge (`pr#23555 <https://github.com/ceph/ceph/pull/23555>`_, Stephan Müller, Tiago Melo)
+* mgr/dashboard: Filter out tasks depending on permissions (`pr#25426 <https://github.com/ceph/ceph/pull/25426>`_, Tina Kallio)
+* mgr/dashboard: Fix /api/grafana/validation (`pr#25997 <https://github.com/ceph/ceph/pull/25997>`_, Zack Cerza)
+* mgr/dashboard: Fix bug in user form when changing password (`pr#23939 <https://github.com/ceph/ceph/pull/23939>`_, Volker Theile)
+* mgr/dashboard: Fix cherrypy static content URL prefix config (`pr#23183 <https://github.com/ceph/ceph/pull/23183>`_, Ricardo Marques)
+* mgr/dashboard: Fix duplicate error messages (`pr#23287 <https://github.com/ceph/ceph/pull/23287>`_, Stephan Müller)
+* mgr/dashboard: Fix duplicate tasks (`pr#24930 <https://github.com/ceph/ceph/pull/24930>`_, Tiago Melo)
+* mgr/dashboard: Fix e2e script (`pr#22903 <https://github.com/ceph/ceph/pull/22903>`_, Tiago Melo)
+* mgr/dashboard: Fixed performance details context for host list row selection (`issue#37854 <http://tracker.ceph.com/issues/37854>`_, `pr#26020 <https://github.com/ceph/ceph/pull/26020>`_, Neha Gupta)
+* mgr/dashboard: Fixed typos in environment.build.js (`pr#26650 <https://github.com/ceph/ceph/pull/26650>`_, Lenz Grimmer)
+* mgr/dashboard: Fix error when clicking on newly created OSD (`issue#36245 <http://tracker.ceph.com/issues/36245>`_, `pr#24369 <https://github.com/ceph/ceph/pull/24369>`_, Patrick Nawracay)
+* mgr/dashboard: Fixes documentation link- to open in new tab (`pr#22237 <https://github.com/ceph/ceph/pull/22237>`_, a2batic)
+* mgr/dashboard: Fixes Grafana 500 error (`issue#37809 <http://tracker.ceph.com/issues/37809>`_, `pr#25830 <https://github.com/ceph/ceph/pull/25830>`_, Kanika Murarka)
+* mgr/dashboard: Fix failing QA test: test_safe_to_destroy (`issue#37290 <http://tracker.ceph.com/issues/37290>`_, `pr#25149 <https://github.com/ceph/ceph/pull/25149>`_, Patrick Nawracay)
+* mgr/dashboard: Fix flaky QA tests (`pr#24024 <https://github.com/ceph/ceph/pull/24024>`_, Patrick Nawracay)
+* mgr/dashboard: Fix Forbidden Error with some roles (`issue#37293 <http://tracker.ceph.com/issues/37293>`_, `pr#25141 <https://github.com/ceph/ceph/pull/25141>`_, Ernesto Puerta)
+* mgr/dashboard: fix for 'Cluster >> Hosts' page (`pr#24974 <https://github.com/ceph/ceph/pull/24974>`_, Alfonso Martínez)
+* mgr/dashboard: Fix formatter service unit test (`pr#22323 <https://github.com/ceph/ceph/pull/22323>`_, Tiago Melo)
+* mgr/dashboard: fix for using '::' on hosts without ipv6 (`pr#26635 <https://github.com/ceph/ceph/pull/26635>`_, Noah Watkins)
+* mgr/dashboard: Fix growing table in firefox (`issue#26999 <http://tracker.ceph.com/issues/26999>`_, `pr#23711 <https://github.com/ceph/ceph/pull/23711>`_, Tiago Melo)
+* mgr/dashboard: Fix HttpClient Module imports in unit tests (`pr#24679 <https://github.com/ceph/ceph/pull/24679>`_, Tiago Melo)
+* mgr/dashboard: Fix iSCSI mutual password input type (`pr#26854 <https://github.com/ceph/ceph/pull/26854>`_, Ricardo Marques)
+* mgr/dashboard: Fix iSCSI service unit tests (`pr#26319 <https://github.com/ceph/ceph/pull/26319>`_, Tiago Melo)
+* mgr/dashboard: Fix issues in controllers/docs (`pr#26738 <https://github.com/ceph/ceph/pull/26738>`_, Volker Theile)
+* mgr/dashboard: Fix Jest conflict with coverage files (`pr#22155 <https://github.com/ceph/ceph/pull/22155>`_, Tiago Melo)
+* mgr/dashboard: Fix layout issues in UI (`issue#24525 <http://tracker.ceph.com/issues/24525>`_, `pr#22597 <https://github.com/ceph/ceph/pull/22597>`_, Volker Theile)
+* mgr/dashboard: Fix links to external documentation (`pr#24829 <https://github.com/ceph/ceph/pull/24829>`_, Patrick Nawracay)
+* mgr/dashboard: fix lint error caused by codelyzer update (`pr#22693 <https://github.com/ceph/ceph/pull/22693>`_, Tiago Melo)
+* mgr/dashboard: fix lint error (`pr#22417 <https://github.com/ceph/ceph/pull/22417>`_, Tiago Melo)
+* mgr/dashboard: Fix long running RBD cloning / copying message (`pr#24641 <https://github.com/ceph/ceph/pull/24641>`_, Ricardo Marques)
+* mgr/dashboard: Fix missing failed restore notification (`issue#36513 <http://tracker.ceph.com/issues/36513>`_, `pr#24664 <https://github.com/ceph/ceph/pull/24664>`_, Tiago Melo)
+* mgr/dashboard: Fix modified files only (frontend) (`pr#25346 <https://github.com/ceph/ceph/pull/25346>`_, Patrick Nawracay)
+* mgr/dashboard: Fix moment.js deprecation warning (`pr#21981 <https://github.com/ceph/ceph/pull/21981>`_, Tiago Melo)
+* mgr/dashboard: Fix more layout issues in UI (`pr#22600 <https://github.com/ceph/ceph/pull/22600>`_, Volker Theile)
+* mgr/dashboard: Fix navbar focused color (`pr#25769 <https://github.com/ceph/ceph/pull/25769>`_, Volker Theile)
+* mgr/dashboard: Fix notifications in user list and form (`pr#23797 <https://github.com/ceph/ceph/pull/23797>`_, Volker Theile)
+* mgr/dashboard: Fix OSD down error display (`issue#24530 <http://tracker.ceph.com/issues/24530>`_, `pr#23754 <https://github.com/ceph/ceph/pull/23754>`_, Patrick Nawracay)
+* mgr/dashboard: Fix pool usage not displaying on filesystem page (`pr#22453 <https://github.com/ceph/ceph/pull/22453>`_, Tiago Melo)
+* mgr/dashboard: Fix problem with ErasureCodeProfileService (`pr#24694 <https://github.com/ceph/ceph/pull/24694>`_, Tiago Melo)
+* mgr/dashboard: Fix Python3 issue (`pr#24617 <https://github.com/ceph/ceph/pull/24617>`_, Patrick Nawracay)
+* mgr/dashboard: fix query parameters in task annotated endpoints (`issue#25096 <http://tracker.ceph.com/issues/25096>`_, `pr#23229 <https://github.com/ceph/ceph/pull/23229>`_, Ricardo Dias)
+* mgr/dashboard: Fix RBD actions disable (`pr#24637 <https://github.com/ceph/ceph/pull/24637>`_, Ricardo Marques)
+* mgr/dashboard: Fix RBD features style (`pr#22759 <https://github.com/ceph/ceph/pull/22759>`_, Ricardo Marques)
+* mgr/dashboard: Fix RBD object size dropdown options (`pr#22830 <https://github.com/ceph/ceph/pull/22830>`_, Ricardo Marques)
+* mgr/dashboard: Fix RBD task metadata (`pr#22088 <https://github.com/ceph/ceph/pull/22088>`_, Tiago Melo)
+* mgr/dashboard: Fix redirect to login page on session lost (`pr#23388 <https://github.com/ceph/ceph/pull/23388>`_, Ricardo Marques)
+* mgr/dashboard: fix reference to oA (`pr#24343 <https://github.com/ceph/ceph/pull/24343>`_, Joao Eduardo Luis)
+* mgr/dashboard: Fix regression on rbd form component (`issue#24757 <http://tracker.ceph.com/issues/24757>`_, `pr#22829 <https://github.com/ceph/ceph/pull/22829>`_, Tiago Melo)
+* mgr/dashboard: Fix reloading of pool listing (`pr#26182 <https://github.com/ceph/ceph/pull/26182>`_, Patrick Nawracay)
+* mgr/dashboard: Fix renaming of pools (`pr#25423 <https://github.com/ceph/ceph/pull/25423>`_, Patrick Nawracay)
+* mgr/dashboard: Fix search in `Source` column of RBD configuration list (`issue#37569 <http://tracker.ceph.com/issues/37569>`_, `pr#26765 <https://github.com/ceph/ceph/pull/26765>`_, Patrick Nawracay)
+* mgr/dashboard: fix skipped backend API tests (`pr#26172 <https://github.com/ceph/ceph/pull/26172>`_, Alfonso Martínez)
+* mgr/dashboard: Fix some datatable CSS issues (`pr#22216 <https://github.com/ceph/ceph/pull/22216>`_, Volker Theile)
+* mgr/dashboard: Fix spaces around status labels on OSD list (`pr#24607 <https://github.com/ceph/ceph/pull/24607>`_, Patrick Nawracay)
+* mgr/dashboard: Fix summary refresh call stack (`pr#25984 <https://github.com/ceph/ceph/pull/25984>`_, Tiago Melo)
+* mgr/dashboard: Fix test_full_health test (`issue#37872 <http://tracker.ceph.com/issues/37872>`_, `pr#25913 <https://github.com/ceph/ceph/pull/25913>`_, Tatjana Dehler)
+* mgr/dashboard: Fix test_remove_not_expired_trash qa test (`issue#37354 <http://tracker.ceph.com/issues/37354>`_, `pr#25221 <https://github.com/ceph/ceph/pull/25221>`_, Tiago Melo)
+* mgr/dashboard: fix: toast notifications hiding utility menu (`pr#26429 <https://github.com/ceph/ceph/pull/26429>`_, Alfonso Martínez)
+* mgr/dashboard: fix: tox not detecting deps changes (`pr#26409 <https://github.com/ceph/ceph/pull/26409>`_, Alfonso Martínez)
+* mgr/dashboard: Fix ts error on iSCSI page (`pr#24715 <https://github.com/ceph/ceph/pull/24715>`_, Ricardo Marques)
+* mgr/dashboard: Fix typo in NoOrchesrtatorConfiguredException class name (`pr#26334 <https://github.com/ceph/ceph/pull/26334>`_, Volker Theile)
+* mgr/dashboard: Fix typo in pools management (`pr#26323 <https://github.com/ceph/ceph/pull/26323>`_, Lenz Grimmer)
+* mgr/dashboard: Fix typo (`pr#23363 <https://github.com/ceph/ceph/pull/23363>`_, Volker Theile)
+* mgr/dashboard: Fix unit tests cli warnings (`pr#21933 <https://github.com/ceph/ceph/pull/21933>`_, Tiago Melo)
+* mgr/dashboard: Format small numbers correctly (`issue#24081 <http://tracker.ceph.com/issues/24081>`_, `pr#21980 <https://github.com/ceph/ceph/pull/21980>`_, Stephan Müller)
+* mgr/dashboard: Get user ID via RGW Admin Ops API (`pr#22416 <https://github.com/ceph/ceph/pull/22416>`_, Volker Theile)
+* mgr/dashboard: Grafana dashboard updates and additions (`pr#24314 <https://github.com/ceph/ceph/pull/24314>`_, Paul Cuzner)
+* mgr/dashboard: Grafana graphs integration with dashboard (`pr#23666 <https://github.com/ceph/ceph/pull/23666>`_, Kanika Murarka)
+* mgr/dashboard: Grafana proxy backend (`pr#21644 <https://github.com/ceph/ceph/pull/21644>`_, Patrick Nawracay)
+* mgr/dashboard: Group buttons together into one menu on OSD page (`issue#37380 <http://tracker.ceph.com/issues/37380>`_, `pr#26189 <https://github.com/ceph/ceph/pull/26189>`_, Tatjana Dehler)
+* mgr/dashboard: Handle class objects as regular objects in KV-table (`pr#24632 <https://github.com/ceph/ceph/pull/24632>`_, Stephan Müller)
+* mgr/dashboard: Handle errors during deletion (`pr#22002 <https://github.com/ceph/ceph/pull/22002>`_, Volker Theile)
+* mgr/dashboard: Hide empty fields and render all objects in KV-table (`pr#25894 <https://github.com/ceph/ceph/pull/25894>`_, Stephan Müller)
+* mgr/dashboard: Hide progress bar in case of an error (`pr#22419 <https://github.com/ceph/ceph/pull/22419>`_, Volker Theile)
+* mgr/dashboard: Implement OSD purge (`issue#35811 <http://tracker.ceph.com/issues/35811>`_, `pr#26242 <https://github.com/ceph/ceph/pull/26242>`_, Patrick Nawracay)
+* mgr/dashboard: Improve CRUSH map viewer (`pr#24934 <https://github.com/ceph/ceph/pull/24934>`_, Volker Theile)
+* mgr/dashboard: Improved support for generating OpenAPI Spec documentation (`issue#24763 <http://tracker.ceph.com/issues/24763>`_, `pr#26227 <https://github.com/ceph/ceph/pull/26227>`_, Tina Kallio)
+* mgr/dashboard: Improve error message handling (`pr#24322 <https://github.com/ceph/ceph/pull/24322>`_, Volker Theile)
+* mgr/dashboard: Improve error panel (`pr#21851 <https://github.com/ceph/ceph/pull/21851>`_, Volker Theile)
+* mgr/dashboard: Improve exception handling in /api/rgw/status (`pr#25836 <https://github.com/ceph/ceph/pull/25836>`_, Volker Theile)
+* mgr/dashboard: Improve exception handling (`issue#23823 <http://tracker.ceph.com/issues/23823>`_, `pr#21066 <https://github.com/ceph/ceph/pull/21066>`_, Sebastian Wagner)
+* mgr/dashboard: Improve `HACKING.rst` (`pr#22281 <https://github.com/ceph/ceph/pull/22281>`_, Patrick Nawracay)
+* mgr/dashboard: Improve 'no pool' message on rbd form (`pr#22150 <https://github.com/ceph/ceph/pull/22150>`_, Ricardo Marques)
+* mgr/dashboard: Improve RBD form (`issue#38303 <http://tracker.ceph.com/issues/38303>`_, `pr#26433 <https://github.com/ceph/ceph/pull/26433>`_, Tiago Melo)
+* mgr/dashboard: Improve RGW address parser (`pr#25870 <https://github.com/ceph/ceph/pull/25870>`_, Volker Theile)
+* mgr/dashboard: Improve RgwUser controller (`pr#25300 <https://github.com/ceph/ceph/pull/25300>`_, Volker Theile)
+* mgr/dashboard: Improves documentation for Grafana Setting (`issue#36371 <http://tracker.ceph.com/issues/36371>`_, `pr#24511 <https://github.com/ceph/ceph/pull/24511>`_, Kanika Murarka)
+* mgr/dashboard: Improve str_to_bool (`pr#22757 <https://github.com/ceph/ceph/pull/22757>`_, Volker Theile)
+* mgr/dashboard: Improve SummaryService and TaskWrapperService (`pr#22906 <https://github.com/ceph/ceph/pull/22906>`_, Tiago Melo)
+* mgr/dashboard: Improve table pagination style (`pr#22065 <https://github.com/ceph/ceph/pull/22065>`_, Ricardo Marques)
+* mgr/dashboard: Introduce pipe to convert bool to text (`pr#26507 <https://github.com/ceph/ceph/pull/26507>`_, Volker Theile)
+* mgr/dashboard: iscsi: adds CLI command to enable/disable API SSL verification (`pr#26891 <https://github.com/ceph/ceph/pull/26891>`_, Ricardo Dias)
+* mgr/dashboard: iSCSI - Adds support for pool/image names with dots (`pr#26503 <https://github.com/ceph/ceph/pull/26503>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI - Add support for disabling ACL authentication (backend) (`pr#26382 <https://github.com/ceph/ceph/pull/26382>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI discovery authentication API (`pr#26115 <https://github.com/ceph/ceph/pull/26115>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI - Infrastructure for multiple backstores (backend) (`pr#26506 <https://github.com/ceph/ceph/pull/26506>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI management API (`pr#25638 <https://github.com/ceph/ceph/pull/25638>`_, Ricardo Marques, Ricardo Dias)
+* mgr/dashboard: iSCSI management UI (`pr#25995 <https://github.com/ceph/ceph/pull/25995>`_, Ricardo Marques, Tiago Melo)
+* mgr/dashboard: iSCSI - Support iSCSI passwords with '/' (`pr#26790 <https://github.com/ceph/ceph/pull/26790>`_, Ricardo Marques)
+* mgr/dashboard: JWT authentication (`pr#22833 <https://github.com/ceph/ceph/pull/22833>`_, Ricardo Dias)
+* mgr/dashboard: Landing Page: chart improvements (`pr#24810 <https://github.com/ceph/ceph/pull/24810>`_, Alfonso Martínez)
+* mgr/dashboard: Landing Page: info visibility (`pr#24513 <https://github.com/ceph/ceph/pull/24513>`_, Alfonso Martínez)
+* mgr/dashboard: Log frontend errors + @UiController (`pr#22285 <https://github.com/ceph/ceph/pull/22285>`_, Ricardo Marques)
+* mgr/dashboard: Login failure should return HTTP 400 (`pr#22403 <https://github.com/ceph/ceph/pull/22403>`_, Ricardo Marques)
+* mgr/dashboard: 'Logs' links permission in Landing Page (`pr#25231 <https://github.com/ceph/ceph/pull/25231>`_, Alfonso Martínez)
+* mgr/dashboard: Make deletion dialog more touch device friendly (`pr#23897 <https://github.com/ceph/ceph/pull/23897>`_, Volker Theile)
+* mgr/dashboard: Map dev 'releases' to master (`pr#24763 <https://github.com/ceph/ceph/pull/24763>`_, Zack Cerza)
+* mgr/dashboard: Module dashboard.services.ganesha has several lint issues (`pr#26378 <https://github.com/ceph/ceph/pull/26378>`_, Volker Theile)
+* mgr/dashboard: More configs for table `updateSelectionOnRefresh` (`pr#24015 <https://github.com/ceph/ceph/pull/24015>`_, Ricardo Marques)
+* mgr/dashboard: Move Cluster/Audit logs from front page to dedicated Logs page (`pr#23834 <https://github.com/ceph/ceph/pull/23834>`_, Diksha Godbole)
+* mgr/dashboard: Move unit-test-helper into the new testing folder (`pr#22857 <https://github.com/ceph/ceph/pull/22857>`_, Tiago Melo)
+* mgr/dashboard: Navbar dropdown button does not respond for mobile browsers (`pr#21967 <https://github.com/ceph/ceph/pull/21967>`_, Volker Theile)
+* mgr/dashboard: New Landing Page: Milestone 2 (`pr#24326 <https://github.com/ceph/ceph/pull/24326>`_, Alfonso Martínez)
+* mgr/dashboard: New Landing Page (`pr#23568 <https://github.com/ceph/ceph/pull/23568>`_, Alfonso Martínez)
+* mgr/dashboard: nfs-ganesha: controller API documentation (`pr#26716 <https://github.com/ceph/ceph/pull/26716>`_, Ricardo Dias)
+* mgr/dashboard: NFS management UI (`pr#26085 <https://github.com/ceph/ceph/pull/26085>`_, Tiago Melo)
+* mgr/dashboard: ng serve bind to 0.0.0.0 (`pr#22058 <https://github.com/ceph/ceph/pull/22058>`_, Ricardo Marques)
+* mgr/dashboard: no side-effects on failed user creation (`pr#24200 <https://github.com/ceph/ceph/pull/24200>`_, Joao Eduardo Luis)
+* mgr/dashboard: Notification queue (`pr#25325 <https://github.com/ceph/ceph/pull/25325>`_, Stephan Müller)
+* mgr/dashboard: npm run e2e:dev (`pr#25136 <https://github.com/ceph/ceph/pull/25136>`_, Stephan Müller)
+* mgr/dashboard: Performance counter progress bar keeps infinitely looping (`pr#24448 <https://github.com/ceph/ceph/pull/24448>`_, Volker Theile)
+* mgr/dashboard: permanent pie chart slice hiding (`pr#25276 <https://github.com/ceph/ceph/pull/25276>`_, Alfonso Martínez)
+* mgr/dashboard: PGs will update as expected (`pr#26589 <https://github.com/ceph/ceph/pull/26589>`_, Stephan Müller)
+* mgr/dashboard: Pool management (`pr#21614 <https://github.com/ceph/ceph/pull/21614>`_, Stephan Müller)
+* mgr/dashboard: pool stats not returned by default (`pr#25635 <https://github.com/ceph/ceph/pull/25635>`_, Alfonso Martínez)
+* mgr/dashboard: Possible fix for some dashboard timing issues (`issue#36107 <http://tracker.ceph.com/issues/36107>`_, `pr#24219 <https://github.com/ceph/ceph/pull/24219>`_, Patrick Nawracay)
+* mgr/dashboard: Prettify package.json (`pr#22401 <https://github.com/ceph/ceph/pull/22401>`_, Ricardo Marques)
+* mgr/dashboard: Prettify RGW JS code (`pr#22278 <https://github.com/ceph/ceph/pull/22278>`_, Volker Theile)
+* mgr/dashboard: Prevent API call on every keystroke (`pr#23391 <https://github.com/ceph/ceph/pull/23391>`_, Volker Theile)
+* mgr/dashboard: Print a blank space between value and unit (`pr#22387 <https://github.com/ceph/ceph/pull/22387>`_, Volker Theile)
+* mgr/dashboard: Progress bar does not stop in TableKeyValueComponent (`pr#24016 <https://github.com/ceph/ceph/pull/24016>`_, Volker Theile)
+* mgr/dashboard: Prometheus integration (`pr#25309 <https://github.com/ceph/ceph/pull/25309>`_, Stephan Müller)
+* mgr/dashboard: Provide all four 'mandatory' OSD flags (`issue#37857 <http://tracker.ceph.com/issues/37857>`_, `pr#25905 <https://github.com/ceph/ceph/pull/25905>`_, Tatjana Dehler)
+* mgr/dashboard/qa: Fix ECP creation test (`pr#25120 <https://github.com/ceph/ceph/pull/25120>`_, Stephan Müller)
+* mgr/dashboard/qa: Fix various vstart_runner.py issues (`issue#36581 <http://tracker.ceph.com/issues/36581>`_, `pr#24767 <https://github.com/ceph/ceph/pull/24767>`_, Volker Theile)
+* mgr/dashboard: Redirect /block to /block/rbd (`pr#24722 <https://github.com/ceph/ceph/pull/24722>`_, Zack Cerza)
+* mgr/dashboard: Reduce Jest logs in CI (`pr#24764 <https://github.com/ceph/ceph/pull/24764>`_, Tiago Melo)
+* mgr/dashboard: Refactor autofocus directive (`pr#23910 <https://github.com/ceph/ceph/pull/23910>`_, Volker Theile)
+* mgr/dashboard: Refactoring of `DeletionModalComponent` (`pr#24005 <https://github.com/ceph/ceph/pull/24005>`_, Patrick Nawracay)
+* mgr/dashboard: Refactor perf counters (`pr#21673 <https://github.com/ceph/ceph/pull/21673>`_, Volker Theile)
+* mgr/dashboard: Refactor RGW backend (`pr#21784 <https://github.com/ceph/ceph/pull/21784>`_, Volker Theile)
+* mgr/dashboard: Refactor role management (`pr#23960 <https://github.com/ceph/ceph/pull/23960>`_, Volker Theile)
+* mgr/dashboard: Relocate empty pipe (`pr#26588 <https://github.com/ceph/ceph/pull/26588>`_, Volker Theile)
+* mgr/dashboard: Removed unnecessary fake services from unit tests (`pr#22473 <https://github.com/ceph/ceph/pull/22473>`_, Stephan Müller)
+* mgr/dashboard: Remove fieldsets when using CdTable (`pr#23730 <https://github.com/ceph/ceph/pull/23730>`_, Tiago Melo)
+* mgr/dashboard: Remove _filterValue from CdFormGroup (`issue#26861 <http://tracker.ceph.com/issues/26861>`_, `pr#24719 <https://github.com/ceph/ceph/pull/24719>`_, Stephan Müller)
+* mgr/dashboard: Remove husky package (`pr#21971 <https://github.com/ceph/ceph/pull/21971>`_, Tiago Melo)
+* mgr/dashboard: Remove karma packages (`pr#23181 <https://github.com/ceph/ceph/pull/23181>`_, Tiago Melo)
+* mgr/dashboard: Remove param when calling notificationService.show (`pr#26447 <https://github.com/ceph/ceph/pull/26447>`_, Volker Theile)
+* mgr/dashboard: Remove top-right actions text and add "About" page (`pr#22762 <https://github.com/ceph/ceph/pull/22762>`_, Ricardo Marques)
+* mgr/dashboard: Remove unused code (`pr#25439 <https://github.com/ceph/ceph/pull/25439>`_, Patrick Nawracay)
+* mgr/dashboard: Remove useless code (`pr#23911 <https://github.com/ceph/ceph/pull/23911>`_, Volker Theile)
+* mgr/dashboard: Remove useless observable unsubscriptions (`pr#21928 <https://github.com/ceph/ceph/pull/21928>`_, Ricardo Marques)
+* mgr/dashboard: replace configuration html table with cd-table (`pr#21643 <https://github.com/ceph/ceph/pull/21643>`_, Tatjana Dehler)
+* mgr/dashboard: Replaced "Pool" with "Pools" in navigation bar (`pr#22715 <https://github.com/ceph/ceph/pull/22715>`_, Lenz Grimmer)
+* mgr/dashboard: Replace RGW proxy controller (`issue#24436 <http://tracker.ceph.com/issues/24436>`_, `pr#22470 <https://github.com/ceph/ceph/pull/22470>`_, Volker Theile)
+* mgr/dashboard: Reset settings to their default values (`pr#22298 <https://github.com/ceph/ceph/pull/22298>`_, Patrick Nawracay)
+* mgr/dashboard: Resolve TestBed performance issue (`pr#21783 <https://github.com/ceph/ceph/pull/21783>`_, Stephan Müller)
+* mgr/dashboard: rest: add support for query params (`pr#22318 <https://github.com/ceph/ceph/pull/22318>`_, Ricardo Dias)
+* mgr/dashboard: RestClient can't handle ProtocolError exceptions (`pr#23347 <https://github.com/ceph/ceph/pull/23347>`_, Volker Theile)
+* mgr/dashboard: restcontroller: minor improvements and bug fixes (`pr#22528 <https://github.com/ceph/ceph/pull/22528>`_, Ricardo Dias)
+* mgr/dashboard: RGW is not working if an URL prefix is defined (`pr#23200 <https://github.com/ceph/ceph/pull/23200>`_, Volker Theile)
+* mgr/dashboard: RGW proxy can't handle self-signed SSL certificates (`pr#22735 <https://github.com/ceph/ceph/pull/22735>`_, Volker Theile)
+* mgr/dashboard: role based authentication/authorization system (`issue#23796 <http://tracker.ceph.com/issues/23796>`_, `pr#22283 <https://github.com/ceph/ceph/pull/22283>`_, Ricardo Marques, Ricardo Dias)
+* mgr/dashboard: Role management from the UI (`pr#23409 <https://github.com/ceph/ceph/pull/23409>`_, Ricardo Marques)
+* mgr/dashboard: Search broken for entries with null values (`issue#38583 <http://tracker.ceph.com/issues/38583>`_, `pr#26766 <https://github.com/ceph/ceph/pull/26766>`_, Patrick Nawracay)
+* mgr/dashboard: set errno via the parent class (`pr#21945 <https://github.com/ceph/ceph/pull/21945>`_, Kefu Chai, Ricardo Dias)
+* mgr/dashboard: Set MODULE_OPTIONS types and defaults (`pr#26386 <https://github.com/ceph/ceph/pull/26386>`_, Volker Theile)
+* mgr/dashboard: Set timeout in RestClient calls (`pr#23224 <https://github.com/ceph/ceph/pull/23224>`_, Volker Theile)
+* mgr/dashboard: Settings service (`pr#25327 <https://github.com/ceph/ceph/pull/25327>`_, Stephan Müller)
+* mgr/dashboard: Show/Hide Grafana tabs according to user role (`issue#36655 <http://tracker.ceph.com/issues/36655>`_, `pr#24851 <https://github.com/ceph/ceph/pull/24851>`_, Kanika Murarka)
+* mgr/dashboard: Show pool dropdown for block-mgr (`issue#37295 <http://tracker.ceph.com/issues/37295>`_, `pr#25144 <https://github.com/ceph/ceph/pull/25144>`_, Ernesto Puerta)
+* mgr/dashboard: Show success notification in RGW forms (`pr#26482 <https://github.com/ceph/ceph/pull/26482>`_, Volker Theile)
+* mgr/dashboard: Simplification of PoolForm method (`pr#24892 <https://github.com/ceph/ceph/pull/24892>`_, Patrick Nawracay)
+* mgr/dashboard: Simplify OSD disabled action test (`pr#24824 <https://github.com/ceph/ceph/pull/24824>`_, Stephan Müller)
+* mgr/dashboard: special casing for minikube in run-backend-rook-api-request.sh (`pr#26600 <https://github.com/ceph/ceph/pull/26600>`_, Jeff Layton)
+* mgr/dashboard: SSO - SAML 2.0 support (`pr#24489 <https://github.com/ceph/ceph/pull/24489>`_, Ricardo Marques, Ricardo Dias)
+* mgr/dashboard: SSO - UserDoesNotExist page (`pr#26058 <https://github.com/ceph/ceph/pull/26058>`_, Alfonso Martínez)
+* mgr/dashboard: Stacktrace is optional on 'js-error' endpoint (`pr#22402 <https://github.com/ceph/ceph/pull/22402>`_, Ricardo Marques)
+* mgr/dashboard: Status info cards' improvements (`pr#25155 <https://github.com/ceph/ceph/pull/25155>`_, Alfonso Martínez)
+* mgr/dashboard: Store user table configurations (`pr#20822 <https://github.com/ceph/ceph/pull/20822>`_, Stephan Müller)
+* mgr/dashboard: Stringify object[] in KV-table (`pr#22422 <https://github.com/ceph/ceph/pull/22422>`_, Stephan Müller)
+* mgr/dashboard: Swagger-UI based Dashboard REST API page (`issue#23898 <http://tracker.ceph.com/issues/23898>`_, `pr#22282 <https://github.com/ceph/ceph/pull/22282>`_, Ricardo Dias)
+* mgr/dashboard: Sync column style with the rest of the UI (`pr#26407 <https://github.com/ceph/ceph/pull/26407>`_, Volker Theile)
+* mgr/dashboard: tasks.mgr.dashboard.test_osd.OsdTest failures (`pr#24947 <https://github.com/ceph/ceph/pull/24947>`_, Volker Theile)
+* mgr/dashboard: Task wrapper service (`pr#22014 <https://github.com/ceph/ceph/pull/22014>`_, Stephan Müller)
+* mgr/dashboard: The RGW backend doesn't handle IPv6 properly (`pr#24222 <https://github.com/ceph/ceph/pull/24222>`_, Volker Theile)
+* mgr/dashboard: typescript cleanup (`pr#26338 <https://github.com/ceph/ceph/pull/26338>`_, Alfonso Martínez)
+* mgr/dashboard: Unit Tests cleanup (`pr#24591 <https://github.com/ceph/ceph/pull/24591>`_, Tiago Melo)
+* mgr/dashboard: Update Angular packages (`pr#23706 <https://github.com/ceph/ceph/pull/23706>`_, Tiago Melo)
+* mgr/dashboard: Update Angular to version 6 (`pr#22082 <https://github.com/ceph/ceph/pull/22082>`_, Tiago Melo)
+* mgr/dashboard: Update bootstrap to v3.4.1 (`pr#26410 <https://github.com/ceph/ceph/pull/26410>`_, Tiago Melo)
+* mgr/dashboard: Updated colors in PG Status chart (`pr#26203 <https://github.com/ceph/ceph/pull/26203>`_, Alfonso Martínez)
+* mgr/dashboard: updated health API test (`pr#25813 <https://github.com/ceph/ceph/pull/25813>`_, Alfonso Martínez)
+* mgr/dashboard: Updated image on 404 page (`pr#23820 <https://github.com/ceph/ceph/pull/23820>`_, Lenz Grimmer)
+* mgr/dashboard: Update frontend packages (`pr#23466 <https://github.com/ceph/ceph/pull/23466>`_, Tiago Melo)
+* mgr/dashboard: Update I18N translation (`pr#26649 <https://github.com/ceph/ceph/pull/26649>`_, Tiago Melo)
+* mgr/dashboard: Update npm packages (`pr#24681 <https://github.com/ceph/ceph/pull/24681>`_, Tiago Melo)
+* mgr/dashboard: Update npm packages (`pr#25656 <https://github.com/ceph/ceph/pull/25656>`_, Tiago Melo)
+* mgr/dashboard: Update npm packages (`pr#26437 <https://github.com/ceph/ceph/pull/26437>`_, Tiago Melo)
+* mgr/dashboard: Update npm packages (`pr#26647 <https://github.com/ceph/ceph/pull/26647>`_, Tiago Melo)
+* mgr/dashboard: update python dependency (`pr#24928 <https://github.com/ceph/ceph/pull/24928>`_, Alfonso Martínez)
+* mgr/dashboard: Update RxJS to version 6 (`pr#21826 <https://github.com/ceph/ceph/pull/21826>`_, Tiago Melo)
+* mgr/dashboard: upgraded python dev dependencies (`pr#26007 <https://github.com/ceph/ceph/pull/26007>`_, Alfonso Martínez)
+* mgr/dashboard: Upgrade Swimlane's data-table (`pr#21880 <https://github.com/ceph/ceph/pull/21880>`_, Volker Theile)
+* mgr/dashboard: Use HTTPS in dev proxy configuration and HACKING.rst (`pr#21777 <https://github.com/ceph/ceph/pull/21777>`_, Volker Theile)
+* mgr/dashboard: Use human readable units on the sparkline graphs (`issue#25075 <http://tracker.ceph.com/issues/25075>`_, `pr#23446 <https://github.com/ceph/ceph/pull/23446>`_, Tiago Melo)
+* mgr/dashboard: User password should be optional (`pr#24128 <https://github.com/ceph/ceph/pull/24128>`_, Ricardo Marques)
+* mgr/dashboard: Validate the OSD recovery priority form input values (`issue#37436 <http://tracker.ceph.com/issues/37436>`_, `pr#25472 <https://github.com/ceph/ceph/pull/25472>`_, Tatjana Dehler)
+* mgr/dashboard: Validation for duplicate RGW user email (`issue#37369 <http://tracker.ceph.com/issues/37369>`_, `pr#25334 <https://github.com/ceph/ceph/pull/25334>`_, Kanika Murarka)
+* mgr: define option defaults for MgrStandbyModule as well (`pr#25734 <https://github.com/ceph/ceph/pull/25734>`_, Kefu Chai)
+* mgr: devicehealth: dont error on dict iteritems (`pr#22827 <https://github.com/ceph/ceph/pull/22827>`_, Abhishek Lekshmanan)
+* mgr: Diskprediction cloud activate when config changes (`pr#25165 <https://github.com/ceph/ceph/pull/25165>`_, Rick Chen)
+* mgr: don't write to output if EOPNOTSUPP (`issue#37444 <http://tracker.ceph.com/issues/37444>`_, `pr#25317 <https://github.com/ceph/ceph/pull/25317>`_, Kefu Chai)
+* mgr: enable inter-module calls (`pr#22951 <https://github.com/ceph/ceph/pull/22951>`_, John Spray)
+* mgr: Expose avgcount to the python modules (`pr#22010 <https://github.com/ceph/ceph/pull/22010>`_, Boris Ranto)
+* mgr: expose avg data for long running avgs (`pr#22420 <https://github.com/ceph/ceph/pull/22420>`_, Boris Ranto)
+* mgr: expose ec profiles through manager (`pr#23010 <https://github.com/ceph/ceph/pull/23010>`_, Noah Watkins)
+* mgr: Extend batch to accept explicit device lists (`issue#37502 <http://tracker.ceph.com/issues/37502>`_, `issue#37086 <http://tracker.ceph.com/issues/37086>`_, `issue#37590 <http://tracker.ceph.com/issues/37590>`_, `pr#25542 <https://github.com/ceph/ceph/pull/25542>`_, Jan Fajerski)
+* mgr: fix beacon interruption caused by deadlock (`pr#23482 <https://github.com/ceph/ceph/pull/23482>`_, Yan Jun)
+* mgr: fix crash due to multiple sessions from daemons with same name (`pr#25534 <https://github.com/ceph/ceph/pull/25534>`_, Mykola Golub)
+* mgr: fix permissions on `balancer execute` (`issue#25345 <http://tracker.ceph.com/issues/25345>`_, `pr#23387 <https://github.com/ceph/ceph/pull/23387>`_, John Spray)
+* mgr: Fix rook spec and have service_describe provide rados_config_location field for nfs services (`pr#25970 <https://github.com/ceph/ceph/pull/25970>`_, Jeff Layton)
+* mgr: fix typo in variable name and cleanups (`pr#22069 <https://github.com/ceph/ceph/pull/22069>`_, Kefu Chai)
+* mgr: fixup pgs show in unknown state (`issue#25103 <http://tracker.ceph.com/issues/25103>`_, `pr#23622 <https://github.com/ceph/ceph/pull/23622>`_, huanwen ren)
+* mgr: Ignore daemon if no metadata was returned (`pr#22794 <https://github.com/ceph/ceph/pull/22794>`_, Wido den Hollander)
+* mgr: Ignore __pycache__ and wheelhouse dirs (`pr#26481 <https://github.com/ceph/ceph/pull/26481>`_, Volker Theile)
+* mgr: Improve ActivePyModules::get_typed_config implementation (`pr#26149 <https://github.com/ceph/ceph/pull/26149>`_, Volker Theile)
+* mgr: improve docs for MgrModule methods (`pr#22792 <https://github.com/ceph/ceph/pull/22792>`_, John Spray)
+* mgr: improvements for dynamic osd perf counters (`pr#25488 <https://github.com/ceph/ceph/pull/25488>`_, Mykola Golub)
+* mgr: Include daemon details in SLOW_OPS output (`issue#23205 <http://tracker.ceph.com/issues/23205>`_, `pr#21750 <https://github.com/ceph/ceph/pull/21750>`_, Brad Hubbard)
+* mgr: `#include <vector>` for clang (`pr#22756 <https://github.com/ceph/ceph/pull/22756>`_, Willem Jan Withagen)
+* mgr: keep status, balancer always on (`pr#23558 <https://github.com/ceph/ceph/pull/23558>`_, Sage Weil)
+* mgr: make module error message more descriptive (`pr#25537 <https://github.com/ceph/ceph/pull/25537>`_, Joao Eduardo Luis)
+* mgr: mgr/ansible: Ansible orchestrator module (`pr#24445 <https://github.com/ceph/ceph/pull/24445>`_, Juan Miguel Olmo Martínez)
+* mgr: mgr/ansible: Create/Remove OSDs (`pr#25497 <https://github.com/ceph/ceph/pull/25497>`_, Juan Miguel Olmo Martínez)
+* mgr: mgr/ansible: Python 3 fix (`pr#25645 <https://github.com/ceph/ceph/pull/25645>`_, Sebastian Wagner)
+* mgr: mgr/balancer: add min/max fields for weekday and be compatible with C (`pr#26505 <https://github.com/ceph/ceph/pull/26505>`_, xie xingguo)
+* mgr: mgr/balancer: auto balance a list of pools (`pr#25940 <https://github.com/ceph/ceph/pull/25940>`_, xie xingguo)
+* mgr: mgr/balancer: blame if upmap won't actually work (`pr#25941 <https://github.com/ceph/ceph/pull/25941>`_, xie xingguo)
+* mgr: mgr/balancer: deepcopy best plan - otherwise we get latest (`issue#27000 <http://tracker.ceph.com/issues/27000>`_, `pr#23682 <https://github.com/ceph/ceph/pull/23682>`_, Stefan Priebe)
+* mgr: mgr/balancer: restrict automatic balancing to specific weekdays (`pr#26440 <https://github.com/ceph/ceph/pull/26440>`_, xie xingguo)
+* mgr: mgr/balancer: skip auto-balancing for pools with pending pg-merge (`pr#25626 <https://github.com/ceph/ceph/pull/25626>`_, xie xingguo)
+* mgr: mgrc: enable disabling stats via mgr_stats_threshold (`issue#25197 <http://tracker.ceph.com/issues/25197>`_, `pr#23352 <https://github.com/ceph/ceph/pull/23352>`_, John Spray)
+* mgr: mgr/crash: add hour granularity crash summary (`pr#23121 <https://github.com/ceph/ceph/pull/23121>`_, Noah Watkins)
+* mgr: mgr/crash: add process name to crash metadata (`pr#25244 <https://github.com/ceph/ceph/pull/25244>`_, Mykola Golub)
+* mgr: mgr/crash: fix python3 invalid syntax problems (`pr#23800 <https://github.com/ceph/ceph/pull/23800>`_, Ricardo Dias)
+* mgr: mgr/DaemonServer: add js-output for "ceph osd safe-to-destroy" (`pr#24799 <https://github.com/ceph/ceph/pull/24799>`_, xie xingguo)
+* mgr: mgr/DaemonServer: log pgmap usage to cluster log (`pr#26105 <https://github.com/ceph/ceph/pull/26105>`_, Neha Ojha)
+* mgr: mgr/dashboard: Add option to disable SSL (`pr#22593 <https://github.com/ceph/ceph/pull/22593>`_, Wido den Hollander)
+* mgr: mgr/dashboard: disable backend tests coverage (`pr#24193 <https://github.com/ceph/ceph/pull/24193>`_, Alfonso Martínez)
+* mgr: mgr/dashboard: Fix dashboard shutdown/restart (`pr#22159 <https://github.com/ceph/ceph/pull/22159>`_, Boris Ranto)
+* mgr: mgr/dashboard: Listen on port 8443 by default and not 8080 (`pr#22409 <https://github.com/ceph/ceph/pull/22409>`_, Wido den Hollander)
+* mgr: mgr/dashboard: use the orchestrator_cli backend setting (`pr#26325 <https://github.com/ceph/ceph/pull/26325>`_, Jeff Layton)
+* mgr: mgr/deepsea: always use 'password' parameter for salt-api auth (`pr#26904 <https://github.com/ceph/ceph/pull/26904>`_, Tim Serong)
+* mgr: mgr/deepsea: check for inflight completions when starting event reader, cleanup logging and comments (`pr#25391 <https://github.com/ceph/ceph/pull/25391>`_, Tim Serong)
+* mgr: mgr/deepsea: DeepSea orchestrator module (`pr#24610 <https://github.com/ceph/ceph/pull/24610>`_, Tim Serong)
+* mgr: mgr/devicehealth: clean up error handling (`pr#23205 <https://github.com/ceph/ceph/pull/23205>`_, John Spray)
+* mgr: mgr/devicehealth: fix is_valid_daemon_name typo error (`pr#24822 <https://github.com/ceph/ceph/pull/24822>`_, Lan Liu)
+* mgr: mgr/diskprediction_cloud: fix divide by zero when total_size is 0 (`pr#26045 <https://github.com/ceph/ceph/pull/26045>`_, Rick Chen)
+* mgr: mgr/diskprediction_cloud: Remove needless library in the requirements file (`issue#37533 <http://tracker.ceph.com/issues/37533>`_, `pr#25433 <https://github.com/ceph/ceph/pull/25433>`_, Rick Chen)
+* mgr: mgr/influx: Use Queue to store points which need to be written (`pr#23464 <https://github.com/ceph/ceph/pull/23464>`_, Wido den Hollander)
+* mgr: mgr/insights: insights reporting module (`pr#23497 <https://github.com/ceph/ceph/pull/23497>`_, Noah Watkins)
+* mgr: mgr/mgr_module.py: fix doc for set_store/set_store_json (`pr#22654 <https://github.com/ceph/ceph/pull/22654>`_, Dan Mick)
+* mgr: mgr/orchestrator: Add RGW service support (`pr#23702 <https://github.com/ceph/ceph/pull/23702>`_, Rubab-Syed)
+* mgr: mgr/orchestrator: Add service_action method (`pr#25649 <https://github.com/ceph/ceph/pull/25649>`_, Tim Serong)
+* mgr: mgr/orchestrator: Add support for "ceph orchestrator service ls" (`pr#24863 <https://github.com/ceph/ceph/pull/24863>`_, Jeff Layton)
+* mgr: mgr/orchestrator: Improve debuggability (`pr#24147 <https://github.com/ceph/ceph/pull/24147>`_, Sebastian Wagner)
+* mgr: mgr/orchestrator: Improve docstrings, add type hinting (`pr#25669 <https://github.com/ceph/ceph/pull/25669>`_, Sebastian Wagner)
+* mgr: mgr/orchestrator: Simplify Orchestrator wait implementation (`pr#25401 <https://github.com/ceph/ceph/pull/25401>`_, Juan Miguel Olmo Martínez)
+* mgr: mgr/orchestrator: use result property in Completion classes (`pr#24672 <https://github.com/ceph/ceph/pull/24672>`_, Tim Serong)
+* mgr: mgr/progress: improve+test OSD out handling (`pr#23146 <https://github.com/ceph/ceph/pull/23146>`_, John Spray)
+* mgr: mgr/progress: introduce the `progress` module (`pr#22993 <https://github.com/ceph/ceph/pull/22993>`_, John Spray)
+* mgr: mgr/prometheus: Add recovery metrics (`pr#26880 <https://github.com/ceph/ceph/pull/26880>`_, Paul Cuzner)
+* mgr: mgr/prometheus: get osd_objectstore once instead twice (`pr#26558 <https://github.com/ceph/ceph/pull/26558>`_, Konstantin Shalygin)
+* mgr: mgr/restful: Fix deep-scrub typo (`issue#36720 <http://tracker.ceph.com/issues/36720>`_, `pr#24841 <https://github.com/ceph/ceph/pull/24841>`_, Boris Ranto)
+* mgr: mgr/restful: fix py got exception when get osd info (`pr#21138 <https://github.com/ceph/ceph/pull/21138>`_, zouaiguo)
+* mgr: mgr/restful: updated string formatting to str.format() (`pr#26210 <https://github.com/ceph/ceph/pull/26210>`_, James McClune)
+* mgr: mgr/rook: fix API version and object types for recent rook changes (`pr#25452 <https://github.com/ceph/ceph/pull/25452>`_, Jeff Layton)
+* mgr: mgr/rook: Fix Rook cluster name detection (`pr#24560 <https://github.com/ceph/ceph/pull/24560>`_, Sebastian Wagner)
+* mgr: mgr/rook: update for v1beta1 API (`pr#23570 <https://github.com/ceph/ceph/pull/23570>`_, John Spray)
+* mgr: mgr/status: Add standby-replay MDS ceph version (`pr#23624 <https://github.com/ceph/ceph/pull/23624>`_, Zhi Zhang)
+* mgr: mgr/status: output to stdout, not stderr (`issue#24175 <http://tracker.ceph.com/issues/24175>`_, `pr#22089 <https://github.com/ceph/ceph/pull/22089>`_, John Spray)
+* mgr: mgr/telegraf: Send more PG status information to Telegraf (`pr#22436 <https://github.com/ceph/ceph/pull/22436>`_, Wido den Hollander)
+* mgr: mgr/telegraf: Telegraf module for Ceph Mgr (`pr#21782 <https://github.com/ceph/ceph/pull/21782>`_, Wido den Hollander)
+* mgr: mgr/telegraf: Use Python generator and catch OSError (`pr#22418 <https://github.com/ceph/ceph/pull/22418>`_, Wido den Hollander)
+* mgr: mgr/telemetry: Add Ceph Telemetry module to send reports back to project (`pr#21982 <https://github.com/ceph/ceph/pull/21982>`_, Wido den Hollander)
+* mgr: mgr/telemetry: Check if boolean is False or not present (`pr#22223 <https://github.com/ceph/ceph/pull/22223>`_, Wido den Hollander)
+* mgr: mgr/telemetry: Fix various issues (`pr#25770 <https://github.com/ceph/ceph/pull/25770>`_, Volker Theile)
+* mgr: mgr/volumes: fix orchestrator remove operation (`pr#25339 <https://github.com/ceph/ceph/pull/25339>`_, Jeff Layton)
+* mgr: mgr/zabbix: drop "total_objects" field (`pr#26052 <https://github.com/ceph/ceph/pull/26052>`_, Kefu Chai)
+* mgr: mgr/zabbix: Send more PG information to Zabbix (`pr#22434 <https://github.com/ceph/ceph/pull/22434>`_, Wido den Hollander)
+* mgr: Miscellaneous small mgr fixes (`pr#22893 <https://github.com/ceph/ceph/pull/22893>`_, John Spray)
+* mgr: modules CLI commands declaration using @CLICommand decorator (`pr#25543 <https://github.com/ceph/ceph/pull/25543>`_, Ricardo Dias)
+* mgr,mon: mgr,mon: fix to apply changed mon_stat_smooth_intervals (`pr#23481 <https://github.com/ceph/ceph/pull/23481>`_, Yan Jun)
+* mgr/orchestrator: added useful attributes to ServiceDescription (`pr#25468 <https://github.com/ceph/ceph/pull/25468>`_, Ricardo Dias)
+* mgr/orchestrator: Add host mon mgr management to interface (`pr#26314 <https://github.com/ceph/ceph/pull/26314>`_, Sebastian Wagner, Noah Watkins)
+* mgr/orchestrator: Add JSON output to CLI commands (`pr#25340 <https://github.com/ceph/ceph/pull/25340>`_, Sebastian Wagner)
+* mgr: orchestrator: add the ability to remove services (`pr#25366 <https://github.com/ceph/ceph/pull/25366>`_, Jeff Layton)
+* mgr/orchestrator: Allow the orchestrator to scale the NFS server count (`pr#26633 <https://github.com/ceph/ceph/pull/26633>`_, Jeff Layton)
+* mgr/orchestrator: clarify error message about kubernetes python module (`pr#24525 <https://github.com/ceph/ceph/pull/24525>`_, Jeff Layton)
+* mgr/orchestrator_cli: Fix README.md (`pr#26443 <https://github.com/ceph/ceph/pull/26443>`_, Sebastian Wagner)
+* mgr/orchestrator: Extend DriveGroupSpec (`pr#25912 <https://github.com/ceph/ceph/pull/25912>`_, Sebastian Wagner)
+* mgr/orchestrator: fix device pretty print with None attributes (`pr#26357 <https://github.com/ceph/ceph/pull/26357>`_, Ricardo Dias)
+* mgr/orchestrator: fix _list_services display (`pr#25610 <https://github.com/ceph/ceph/pull/25610>`_, Jeff Layton)
+* mgr/orchestrator: Fix up rook osd create dispatcher (`pr#26317 <https://github.com/ceph/ceph/pull/26317>`_, Jeff Layton)
+* mgr/orchestrator: make use of @CLICommand (`pr#26094 <https://github.com/ceph/ceph/pull/26094>`_, Sebastian Wagner)
+* mgr/orchestrator: remove unicode whitespaces (`pr#25323 <https://github.com/ceph/ceph/pull/25323>`_, Sebastian Wagner)
+* mgr/orchestrator/rook: allow the creation of OSDs in directories (`pr#26570 <https://github.com/ceph/ceph/pull/26570>`_, Jeff Layton)
+* mgr/orchestrator: Unify `osd create` and `osd add` (`pr#26171 <https://github.com/ceph/ceph/pull/26171>`_, Sebastian Wagner)
+* mgr/orch: refresh option for inventory query (`pr#26346 <https://github.com/ceph/ceph/pull/26346>`_, Noah Watkins)
+* mgr: prometheus: added bluestore db and wal/journal devices to ceph_disk_occupation metric (`issue#36627 <http://tracker.ceph.com/issues/36627>`_, `pr#24821 <https://github.com/ceph/ceph/pull/24821>`_, Konstantin Shalygin)
+* mgr: prometheus: Expose number of degraded/misplaced/unfound objects (`pr#21793 <https://github.com/ceph/ceph/pull/21793>`_, Boris Ranto)
+* mgr: prometheus: Fix metric resets (`pr#22732 <https://github.com/ceph/ceph/pull/22732>`_, Boris Ranto)
+* mgr: prometheus: Fix prometheus shutdown/restart (`pr#21748 <https://github.com/ceph/ceph/pull/21748>`_, Boris Ranto)
+* mgr: pybind/mgr: add osd space utilization to insights report (`pr#25122 <https://github.com/ceph/ceph/pull/25122>`_, Noah Watkins)
+* mgr: pybind/mgr: PEP 8 code clean and fix typo (`pr#26181 <https://github.com/ceph/ceph/pull/26181>`_, Lei Liu)
+* mgr,pybind: mgr/prometheus: add interface and objectstore to osd metadata (`pr#25234 <https://github.com/ceph/ceph/pull/25234>`_, Jan Fajerski)
+* mgr: pybind/mgr/restful: Decode the output of b64decode (`issue#38522 <http://tracker.ceph.com/issues/38522>`_, `pr#26712 <https://github.com/ceph/ceph/pull/26712>`_, Brad Hubbard)
+* mgr,pybind: mgr/rook: fix urljoin import (`pr#24626 <https://github.com/ceph/ceph/pull/24626>`_, Jeff Layton)
+* mgr,pybind: mgr/volumes: Fix Python 3 import error (`pr#25344 <https://github.com/ceph/ceph/pull/25344>`_, Sebastian Wagner)
+* mgr,pybind: pybind/mgr: drop unnecessary iterkeys usage to make py-3 compatible (`issue#37581 <http://tracker.ceph.com/issues/37581>`_, `pr#25457 <https://github.com/ceph/ceph/pull/25457>`_, Mykola Golub)
+* mgr,pybind: pybind/mgr: identify invalid fs (`pr#24392 <https://github.com/ceph/ceph/pull/24392>`_, Jos Collin)
+* mgr,pybind: src/script: add run_mypy to run static type checking on Python code (`pr#26715 <https://github.com/ceph/ceph/pull/26715>`_, Sebastian Wagner)
+* mgr: race between daemon state and service map in 'service status' (`issue#36656 <http://tracker.ceph.com/issues/36656>`_, `pr#24878 <https://github.com/ceph/ceph/pull/24878>`_, Mykola Golub)
+* mgr,rbd: mgr/prometheus: provide RBD stats via osd dynamic perf counters (`pr#25358 <https://github.com/ceph/ceph/pull/25358>`_, Mykola Golub)
+* mgr,rbd: pybind/mgr/prometheus: improve 'rbd_stats_pools' param parsing (`pr#25860 <https://github.com/ceph/ceph/pull/25860>`_, Mykola Golub)
+* mgr,rbd: pybind/mgr/prometheus: rbd stats namespace support (`pr#25636 <https://github.com/ceph/ceph/pull/25636>`_, Mykola Golub)
+* mgr: replace "Unknown error" string on always_on (`pr#23645 <https://github.com/ceph/ceph/pull/23645>`_, John Spray)
+* mgr: restful: Fix regression when traversing leaf nodes (`pr#26421 <https://github.com/ceph/ceph/pull/26421>`_, Boris Ranto)
+* mgr/rook: remove dead code and fix bug in url fetching code (`pr#26032 <https://github.com/ceph/ceph/pull/26032>`_, Jeff Layton)
+* mgr: silence GCC warning (`pr#25199 <https://github.com/ceph/ceph/pull/25199>`_, Kefu Chai)
+* mgr/ssh: fix type and doc errors (`pr#26630 <https://github.com/ceph/ceph/pull/26630>`_, Sebastian Wagner)
+* mgr/telemetry: fix total_objects (`issue#37976 <http://tracker.ceph.com/issues/37976>`_, `pr#26046 <https://github.com/ceph/ceph/pull/26046>`_, Sage Weil)
+* mgr,tests: mgr/dashboard: use dedicated tox working dir (`pr#25290 <https://github.com/ceph/ceph/pull/25290>`_, Noah Watkins)
+* mgr,tests: mgr/insights: use dedicated tox working dir (`pr#25146 <https://github.com/ceph/ceph/pull/25146>`_, Noah Watkins)
+* mgr,tests: mgr/selftest: fix disabled module selection (`pr#24517 <https://github.com/ceph/ceph/pull/24517>`_, John Spray)
+* mgr: timely health updates between monitor and manager (`pr#23294 <https://github.com/ceph/ceph/pull/23294>`_, Noah Watkins)
+* mgr: update daemon_state when necessary (`issue#37753 <http://tracker.ceph.com/issues/37753>`_, `pr#25725 <https://github.com/ceph/ceph/pull/25725>`_, Xinying Song)
+* mgr: update MMgrConfigure message to include optional OSD perf queries (`pr#24180 <https://github.com/ceph/ceph/pull/24180>`_, Julien Collet)
+* mgr: Use Py_BuildValue to create the argument tuple (`pr#26240 <https://github.com/ceph/ceph/pull/26240>`_, Volker Theile)
+* mgr: volumes mgr module fixes (`pr#25331 <https://github.com/ceph/ceph/pull/25331>`_, Jeff Layton)
+* misc: mark functions with 'override' specifier (`pr#21790 <https://github.com/ceph/ceph/pull/21790>`_, Danny Al-Gaaf)
+* mon: add 'osd destroy-new' command that only destroys NEW osd slots (`issue#24428 <http://tracker.ceph.com/issues/24428>`_, `pr#22429 <https://github.com/ceph/ceph/pull/22429>`_, Sage Weil)
+* mon: A PG with PG_STATE_REPAIR doesn't mean damaged data, PG_STATE_IN… (`issue#38070 <http://tracker.ceph.com/issues/38070>`_, `pr#26178 <https://github.com/ceph/ceph/pull/26178>`_, David Zafman)
+* mon: change monitor compact command to run asynchronously (`issue#24160 <http://tracker.ceph.com/issues/24160>`_, `issue#24159 <http://tracker.ceph.com/issues/24159>`_, `pr#22056 <https://github.com/ceph/ceph/pull/22056>`_, penglaiyxy)
+* mon: common/cmdparse: cmd_getval_throws -> cmd_getval (`pr#23557 <https://github.com/ceph/ceph/pull/23557>`_, Sage Weil)
+* mon: don't commit osdmap on no-op application ops (`pr#23528 <https://github.com/ceph/ceph/pull/23528>`_, John Spray)
+* mon: fix mgr module config option handling (`issue#35076 <http://tracker.ceph.com/issues/35076>`_, `pr#23846 <https://github.com/ceph/ceph/pull/23846>`_, Sage Weil)
+* mon: fix pg_sum_old not copied correctly (`pr#26110 <https://github.com/ceph/ceph/pull/26110>`_, Yao Zongyou)
+* monitoring/grafana: Fix OSD Capacity Utlization Grafana graph (`pr#24426 <https://github.com/ceph/ceph/pull/24426>`_, Maxime)
+* mon: make rank ordering explicit (not tied to mon address sort order) (`pr#22193 <https://github.com/ceph/ceph/pull/22193>`_, Sage Weil)
+* mon: mon/config-key: increase max key entry size (`pr#24250 <https://github.com/ceph/ceph/pull/24250>`_, Joao Eduardo Luis)
+* mon: mon/MonClient: drop my_addr (`pr#26449 <https://github.com/ceph/ceph/pull/26449>`_, Kefu Chai)
+* mon: mon/MonClient: use mon_client_ping_timeout during ping_monitor (`pr#23563 <https://github.com/ceph/ceph/pull/23563>`_, Yao Zongyou)
+* mon: mon/MonMap: add more const'ness to its methods (`pr#23709 <https://github.com/ceph/ceph/pull/23709>`_, Kefu Chai)
+* mon: mon/MonMap: remove duplicate code in get_rank (`pr#23547 <https://github.com/ceph/ceph/pull/23547>`_, Yao Zongyou)
+* mon: mon,osd: avoid str copy in parse (`pr#25640 <https://github.com/ceph/ceph/pull/25640>`_, Jos Collin)
+* mon: mon/OSDMonitor: add boundary check for pool recovery_priority (`issue#38578 <http://tracker.ceph.com/issues/38578>`_, `pr#26729 <https://github.com/ceph/ceph/pull/26729>`_, xie xingguo)
+* mon: mon/PGMap: add more #include (`pr#26420 <https://github.com/ceph/ceph/pull/26420>`_, Kefu Chai)
+* mon: mon/PGMap: command 'ceph df -f json' output add total_percent_used (`pr#23588 <https://github.com/ceph/ceph/pull/23588>`_, Yanhu Cao)
+* mon: only share monmap after authenticating (`pr#23741 <https://github.com/ceph/ceph/pull/23741>`_, Sage Weil)
+* mon: shutdown messenger early to avoid accessing deleted logger (`issue#37780 <http://tracker.ceph.com/issues/37780>`_, `pr#25760 <https://github.com/ceph/ceph/pull/25760>`_, ningtao)
+* mon: some tiny cleanups related class forward declaration (`pr#26219 <https://github.com/ceph/ceph/pull/26219>`_, Yao Zongyou)
+* mon,tests: qa/cephtool: test bounds on pool's `hit_set_\*` (`pr#24858 <https://github.com/ceph/ceph/pull/24858>`_, Joao Eduardo Luis)
+* mon:validate hit_set values before set (`issue#22659 <http://tracker.ceph.com/issues/22659>`_, `pr#19983 <https://github.com/ceph/ceph/pull/19983>`_, lijing)
+* msg: addr -> addrvec (part 1) (`pr#22306 <https://github.com/ceph/ceph/pull/22306>`_, Sage Weil)
+* msg/async: do not force updating rotating keys inline (`pr#25859 <https://github.com/ceph/ceph/pull/25859>`_, yanjun, xie xingguo)
+* msg/async/Protocol\*: send keep alive if existing wins (`issue#38493 <http://tracker.ceph.com/issues/38493>`_, `pr#26668 <https://github.com/ceph/ceph/pull/26668>`_, xie xingguo)
+* msg/async/rdma: add iWARP RDMA protocol support (`pr#20297 <https://github.com/ceph/ceph/pull/20297>`_, Haodong Tang)
+* msg/async/rdma: Delete duplicate header file (`pr#25392 <https://github.com/ceph/ceph/pull/25392>`_, Jianpeng Ma)
+* msg/async/rdma: parse IBSYNMsg.lid as hex when receiving message (`pr#26525 <https://github.com/ceph/ceph/pull/26525>`_, Peng Liu)
+* msg/async: reduce additional ceph_msg_header copy (`pr#25938 <https://github.com/ceph/ceph/pull/25938>`_, Jianpeng Ma)
+* msg/async: the ceph_abort is needless in handle_connect_msg (`pr#21751 <https://github.com/ceph/ceph/pull/21751>`_, shangfufei)
+* msg: ceph_abort() when there are enough accepter errors in msg server (`issue#23649 <http://tracker.ceph.com/issues/23649>`_, `pr#23306 <https://github.com/ceph/ceph/pull/23306>`_, penglaiyxy@gmail.com)
+* msg: clear message middle when clearing encoded message buffer (`pr#24289 <https://github.com/ceph/ceph/pull/24289>`_, "Yan, Zheng")
+* msg: entity_addr_t::parse doesn't do memset(this, 0, ...) for clean-up (`issue#26937 <http://tracker.ceph.com/issues/26937>`_, `pr#23573 <https://github.com/ceph/ceph/pull/23573>`_, Radoslaw Zarzynski)
+* nautilus: mgr/dashboard: Validate `ceph-iscsi` config version (`pr#26951 <https://github.com/ceph/ceph/pull/26951>`_, Ricardo Marques)
+* objecter: avoid race when reset down osd's session (`pr#25437 <https://github.com/ceph/ceph/pull/25437>`_, Zengran Zhang)
+* orchestrator_cli: fix HandleCommandResult invocations in _status() (`pr#25329 <https://github.com/ceph/ceph/pull/25329>`_, Jeff Layton)
+* osd: add creating to pg_string_state (`issue#36174 <http://tracker.ceph.com/issues/36174>`_, `pr#24262 <https://github.com/ceph/ceph/pull/24262>`_, Dan van der Ster)
+* osd: add --dump-journal option in ceph-osd help info (`pr#24969 <https://github.com/ceph/ceph/pull/24969>`_, yuliyang)
+* osd: Additional fields for osd "bench" command (`pr#21962 <https://github.com/ceph/ceph/pull/21962>`_, Коренберг Маркr)
+* osd: add log when pg reg next scrub (`pr#23690 <https://github.com/ceph/ceph/pull/23690>`_, lvshuhua)
+* osd: add required cls libraries as dependencies of osd (`pr#24373 <https://github.com/ceph/ceph/pull/24373>`_, Mohamad Gebai)
+* osd: Allow repair of an object with a bad data_digest in object_info on all replicas (`pr#23217 <https://github.com/ceph/ceph/pull/23217>`_, David Zafman)
+* osd: always set query_epoch explicitly for MOSDPGLog (`pr#22487 <https://github.com/ceph/ceph/pull/22487>`_, Kefu Chai)
+* osd: avoid using null agent_state (`pr#25393 <https://github.com/ceph/ceph/pull/25393>`_, Zengran Zhang)
+* osd: Change assert() to ceph_assert() missed in the transition (`pr#23918 <https://github.com/ceph/ceph/pull/23918>`_, David Zafman)
+* osd: Change osd_skip_data_digest default to false and make it LEVEL_DEV (`issue#24950 <http://tracker.ceph.com/issues/24950>`_, `pr#23083 <https://github.com/ceph/ceph/pull/23083>`_, Sage Weil, David Zafman)
+* osdc: invoke notify finish context on linger commit failure (`issue#23966 <http://tracker.ceph.com/issues/23966>`_, `pr#21831 <https://github.com/ceph/ceph/pull/21831>`_, Kefu Chai, Jason Dillaman)
+* osd: clean up and avoid extra ref-counting in PrimaryLogPG::log_op_stats (`pr#23016 <https://github.com/ceph/ceph/pull/23016>`_, Radoslaw Zarzynski)
+* osd: clean up smart probe (`issue#23899 <http://tracker.ceph.com/issues/23899>`_, `pr#21950 <https://github.com/ceph/ceph/pull/21950>`_, Sage Weil, Gu Zhongyan)
+* osd: collect client perf stats when query is enabled (`pr#24265 <https://github.com/ceph/ceph/pull/24265>`_, Julien Collet, Mykola Golub)
+* osd: combine recovery/scrub/snap sleep timer into one (`pr#21711 <https://github.com/ceph/ceph/pull/21711>`_, Jianpeng Ma)
+* osd: Deny reservation if expected backfill size would put us over bac… (`issue#24801 <http://tracker.ceph.com/issues/24801>`_, `issue#19753 <http://tracker.ceph.com/issues/19753>`_, `pr#22797 <https://github.com/ceph/ceph/pull/22797>`_, David Zafman)
+* osd: do not include Messenger.h if not necessary (`pr#22483 <https://github.com/ceph/ceph/pull/22483>`_, Kefu Chai)
+* osd: do not overestimate the size of the object for reads with trimtrunc (`issue#21931 <http://tracker.ceph.com/issues/21931>`_, `issue#22330 <http://tracker.ceph.com/issues/22330>`_, `pr#24564 <https://github.com/ceph/ceph/pull/24564>`_, Neha Ojha)
+* osd: do not treat an IO hint as an IOP for PG stats (`issue#24909 <http://tracker.ceph.com/issues/24909>`_, `pr#23029 <https://github.com/ceph/ceph/pull/23029>`_, Jason Dillaman)
+* osd: don't check overwrite flag when handling copy-get (`issue#21756 <http://tracker.ceph.com/issues/21756>`_, `pr#18241 <https://github.com/ceph/ceph/pull/18241>`_, huangjun)
+* osd: Don't evict even when preemption has restarted with smaller chunk (`pr#21892 <https://github.com/ceph/ceph/pull/21892>`_, David Zafman)
+* osd: do_sparse_read(): Verify checksum earlier so we will try to repair (`issue#24875 <http://tracker.ceph.com/issues/24875>`_, `pr#23377 <https://github.com/ceph/ceph/pull/23377>`_, David Zafman)
+* osd: drop the unused request_redirect_t::osd_instructions (`pr#24458 <https://github.com/ceph/ceph/pull/24458>`_, Radoslaw Zarzynski)
+* osd: ec saves a write access to the memory under most circumstances (`pr#26053 <https://github.com/ceph/ceph/pull/26053>`_, Zengran Zhang, Kefu Chai)
+* osd: fix build_incremental_map_msg (`issue#38282 <http://tracker.ceph.com/issues/38282>`_, `pr#26413 <https://github.com/ceph/ceph/pull/26413>`_, Sage Weil)
+* osd: fix memory leak in EC fast and error read (`pr#22500 <https://github.com/ceph/ceph/pull/22500>`_, xiaofei cui)
+* osd: Fix recovery and backfill priority handling (`issue#38041 <http://tracker.ceph.com/issues/38041>`_, `pr#26213 <https://github.com/ceph/ceph/pull/26213>`_, David Zafman)
+* osd: fix shard_info_wrapper encode (`issue#37653 <http://tracker.ceph.com/issues/37653>`_, `pr#25548 <https://github.com/ceph/ceph/pull/25548>`_, David Zafman)
+* osd: Handle omap and data digests independently (`issue#24366 <http://tracker.ceph.com/issues/24366>`_, `pr#22346 <https://github.com/ceph/ceph/pull/22346>`_, David Zafman)
+* osd: increase default hard pg limit (`pr#22187 <https://github.com/ceph/ceph/pull/22187>`_, Josh Durgin)
+* osd: keep using cache even if op will invalid cache (`pr#25490 <https://github.com/ceph/ceph/pull/25490>`_, Zengran Zhang)
+* osd: limit pg log length under all circumstances (`pr#23098 <https://github.com/ceph/ceph/pull/23098>`_, Neha Ojha)
+* osd: make OSD::HEARTBEAT_MAX_CONN inline (`pr#23424 <https://github.com/ceph/ceph/pull/23424>`_, Kefu Chai)
+* osd: make random shuffle comply with C++17 (`pr#23533 <https://github.com/ceph/ceph/pull/23533>`_, Willem Jan Withagen)
+* osd/OSDMap: add osd status to utilization dumper (`issue#35544 <http://tracker.ceph.com/issues/35544>`_, `pr#23921 <https://github.com/ceph/ceph/pull/23921>`_, Paul Emmerich)
+* osd: per-pool osd stats collection (`pr#19454 <https://github.com/ceph/ceph/pull/19454>`_, Igor Fedotv, Igor Fedotov)
+* osd: Prevent negative local num_bytes sent to peer for backfill reser… (`issue#38344 <http://tracker.ceph.com/issues/38344>`_, `pr#26465 <https://github.com/ceph/ceph/pull/26465>`_, David Zafman)
+* osd: read object attrs failed at EC recovery (`pr#22196 <https://github.com/ceph/ceph/pull/22196>`_, xiaofei cui)
+* osd: refuse to start if we're > N+2 from recorded require_osd_release (`issue#38076 <http://tracker.ceph.com/issues/38076>`_, `pr#26177 <https://github.com/ceph/ceph/pull/26177>`_, Sage Weil)
+* osd: reliably send pg_created messages to the mon (`issue#37775 <http://tracker.ceph.com/issues/37775>`_, `pr#25731 <https://github.com/ceph/ceph/pull/25731>`_, Sage Weil)
+* osd: Remove old bft= which has been superceded by backfill (`issue#36170 <http://tracker.ceph.com/issues/36170>`_, `pr#24256 <https://github.com/ceph/ceph/pull/24256>`_, David Zafman)
+* osd: remove stray derr (`pr#24042 <https://github.com/ceph/ceph/pull/24042>`_, Sage Weil)
+* osd: remove unused class read_log_and_missing_error (`pr#26057 <https://github.com/ceph/ceph/pull/26057>`_, Yao Zongyou)
+* osd: remove unused fields (`pr#26021 <https://github.com/ceph/ceph/pull/26021>`_, Jianpeng Ma)
+* osd: remove unused function (`pr#26223 <https://github.com/ceph/ceph/pull/26223>`_, Jianpeng Ma)
+* osd: Remove useless conditon (`pr#21766 <https://github.com/ceph/ceph/pull/21766>`_, Jianpeng Ma)
+* osd: some recovery improvements and cleanups (`pr#23663 <https://github.com/ceph/ceph/pull/23663>`_, xie xingguo)
+* osd: two heartbeat fixes (`pr#25126 <https://github.com/ceph/ceph/pull/25126>`_, xie xingguo)
+* osd: unlock osd_lock when tweaking osd settings (`issue#37751 <http://tracker.ceph.com/issues/37751>`_, `pr#25726 <https://github.com/ceph/ceph/pull/25726>`_, Kefu Chai)
+* osd: unmount store after service.shutdown() (`issue#37975 <http://tracker.ceph.com/issues/37975>`_, `pr#26043 <https://github.com/ceph/ceph/pull/26043>`_, Kefu Chai)
+* osd: Weighted Random Sampling for dynamic perf stats (`pr#25582 <https://github.com/ceph/ceph/pull/25582>`_, Mykola Golub)
+* osd: When possible check CRC in build_push_op() so repair can eventually stop (`issue#25084 <http://tracker.ceph.com/issues/25084>`_, `pr#23518 <https://github.com/ceph/ceph/pull/23518>`_, David Zafman)
+* osd: write "bench" output to stdout (`issue#24022 <http://tracker.ceph.com/issues/24022>`_, `pr#21905 <https://github.com/ceph/ceph/pull/21905>`_, John Spray)
+* os: Minor fixes in comments describing a transaction (`pr#22329 <https://github.com/ceph/ceph/pull/22329>`_, Bryan Stillwell)
+* performance: Add performance counters breadcrumb (`pr#22060 <https://github.com/ceph/ceph/pull/22060>`_, Ricardo Marques)
+* performance: mgr/dashboard: Enable gzip compression (`issue#36453 <http://tracker.ceph.com/issues/36453>`_, `pr#24727 <https://github.com/ceph/ceph/pull/24727>`_, Zack Cerza)
+* performance: mgr/dashboard: Replace dashboard service (`issue#36675 <http://tracker.ceph.com/issues/36675>`_, `pr#24900 <https://github.com/ceph/ceph/pull/24900>`_, Zack Cerza)
+* performance: msg/async: improve read-prefetch logic (`pr#25758 <https://github.com/ceph/ceph/pull/25758>`_, xie xingguo)
+* performance: qa/tasks/cbt.py: changes to run on bionic (`pr#22405 <https://github.com/ceph/ceph/pull/22405>`_, Neha Ojha)
+* performance,rbd: common/Throttle: TokenBucketThrottle: use reference to m_blockers.front() (`issue#36475 <http://tracker.ceph.com/issues/36475>`_, `pr#24604 <https://github.com/ceph/ceph/pull/24604>`_, Dongsheng Yang)
+* performance,rbd: pybind/rbd: optimize rbd_list2 (`pr#25445 <https://github.com/ceph/ceph/pull/25445>`_, Mykola Golub)
+* Prevent duplicated rows during async tasks (`pr#22148 <https://github.com/ceph/ceph/pull/22148>`_, Ricardo Marques)
+* prometheus: Fix order of occupation values (`pr#22149 <https://github.com/ceph/ceph/pull/22149>`_, Boris Ranto)
+* pybind: do not check MFLAGS (`pr#23601 <https://github.com/ceph/ceph/pull/23601>`_, Kefu Chai)
+* pybind: pybind/ceph_daemon: expand the order of magnitude of daemonperf statistics to ZB (`issue#23962 <http://tracker.ceph.com/issues/23962>`_, `pr#21765 <https://github.com/ceph/ceph/pull/21765>`_, Guan yunfei)
+* pybind: pybind/rbd: make the code more concise (`pr#23664 <https://github.com/ceph/ceph/pull/23664>`_, Zheng Yin)
+* pybind,rbd: pybind/rbd: add allow_shrink=True as a parameter to def resize (`pr#23605 <https://github.com/ceph/ceph/pull/23605>`_, Zheng Yin)
+* pybind,rbd: pybind/rbd: fix a typo in metadata_get comments (`pr#26138 <https://github.com/ceph/ceph/pull/26138>`_, songweibin)
+* pybind,rgw: pybind/rgw: pass the flags to callback function (`pr#25766 <https://github.com/ceph/ceph/pull/25766>`_, Kefu Chai)
+* pybind: simplify timeout handling in run_in_thread() (`pr#24733 <https://github.com/ceph/ceph/pull/24733>`_, Kefu Chai)
+* qa/btrfs/test_rmdir_async_snap: remove binary file (`pr#24108 <https://github.com/ceph/ceph/pull/24108>`_, Cleber Rosa)
+* qa,pybind,tools: Correct usage of collections.abc (`pr#25318 <https://github.com/ceph/ceph/pull/25318>`_, James Page)
+* qa/test: Added rados, rbd and fs to run two time a week only (`pr#21839 <https://github.com/ceph/ceph/pull/21839>`_, Yuri Weinstein)
+* qa/tests: added 1st draft of mimic-x suite (`pr#23292 <https://github.com/ceph/ceph/pull/23292>`_, Yuri Weinstein)
+* qa/tests - added all supported distro (`pr#22647 <https://github.com/ceph/ceph/pull/22647>`_, Yuri Weinstein)
+* qa/tests - added all supported distro to the mix, … (`pr#22674 <https://github.com/ceph/ceph/pull/22674>`_, Yuri Weinstein)
+* qa/tests: added client-upgrade-luminous suit (`pr#21947 <https://github.com/ceph/ceph/pull/21947>`_, Yuri Weinstein)
+* qa/tests: added --filter-out="ubuntu_14.04" (`pr#21949 <https://github.com/ceph/ceph/pull/21949>`_, Yuri Weinstein)
+* qa/tests - added luminous-p2p suite to the schedule (`pr#22666 <https://github.com/ceph/ceph/pull/22666>`_, Yuri Weinstein)
+* qa/tests: added mimic-x to the schedule (`pr#23302 <https://github.com/ceph/ceph/pull/23302>`_, Yuri Weinstein)
+* qa/tests - added powercycle suite to run on weekly basis on master and mimic (`pr#22606 <https://github.com/ceph/ceph/pull/22606>`_, Yuri Weinstein)
+* qa/tests:  added supported distro for powercycle suite (`pr#22185 <https://github.com/ceph/ceph/pull/22185>`_, Yuri Weinstein)
+* qa/tests: changed ceph qa email address to bypass dreamhost's spam filter (`pr#23456 <https://github.com/ceph/ceph/pull/23456>`_, Yuri Weinstein)
+* qa/tests: changed disto symlink to point to new way using supported OS'es (`pr#22536 <https://github.com/ceph/ceph/pull/22536>`_, Yuri Weinstein)
+* qa/tests: fixed typo (`pr#21858 <https://github.com/ceph/ceph/pull/21858>`_, Yuri Weinstein)
+* qa/tests: removed all jewel runs and reduced runs on ovh (`pr#22531 <https://github.com/ceph/ceph/pull/22531>`_, Yuri Weinstein)
+* rbd: add 'config global' command to get/store overrides in mon config db (`pr#24428 <https://github.com/ceph/ceph/pull/24428>`_, Mykola Golub)
+* rbd: add data pool support to trash purge (`issue#22872 <http://tracker.ceph.com/issues/22872>`_, `pr#21247 <https://github.com/ceph/ceph/pull/21247>`_, Mahati Chamarthy)
+* rbd: add group snap rollback method (`issue#23550 <http://tracker.ceph.com/issues/23550>`_, `pr#23896 <https://github.com/ceph/ceph/pull/23896>`_, songweibin)
+* rbd: add protected in snap list (`pr#23853 <https://github.com/ceph/ceph/pull/23853>`_, Zheng Yin)
+* rbd: add snapshot count in rbd info (`pr#21292 <https://github.com/ceph/ceph/pull/21292>`_, Zheng Yin)
+* rbd: add the judgment of resizing the image (`pr#21770 <https://github.com/ceph/ceph/pull/21770>`_, zhengyin)
+* rbd: basic support for images within namespaces (`issue#24558 <http://tracker.ceph.com/issues/24558>`_, `pr#22673 <https://github.com/ceph/ceph/pull/22673>`_, Jason Dillaman)
+* rbd: close image when bench is interrupted (`pr#26693 <https://github.com/ceph/ceph/pull/26693>`_, Mykola Golub)
+* rbd: cls/lock: always store v1 addr in locker_info_t (`pr#25948 <https://github.com/ceph/ceph/pull/25948>`_, Sage Weil)
+* rbd: cls/rbd: fix build (`pr#22078 <https://github.com/ceph/ceph/pull/22078>`_, Kefu Chai)
+* rbd: cls/rbd: fixed uninitialized variable compiler warning (`pr#26896 <https://github.com/ceph/ceph/pull/26896>`_, Jason Dillaman)
+* rbd: cls/rbd: fix method comment (`pr#23277 <https://github.com/ceph/ceph/pull/23277>`_, Zheng Yin)
+* rbd: cls/rbd: silence the log of get metadata error (`pr#25436 <https://github.com/ceph/ceph/pull/25436>`_, songweibin)
+* rbd: correct parameter of namespace and verify it before set_namespace (`pr#23770 <https://github.com/ceph/ceph/pull/23770>`_, songweibin)
+* rbd: dashboard: support configuring block mirroring pools and peers (`pr#25210 <https://github.com/ceph/ceph/pull/25210>`_, Jason Dillaman)
+* rbd: disable cache for actions that open multiple images (`issue#24092 <http://tracker.ceph.com/issues/24092>`_, `pr#21946 <https://github.com/ceph/ceph/pull/21946>`_, Jason Dillaman)
+* rbd: disk-usage can now optionally compute exact on-disk usage (`issue#24064 <http://tracker.ceph.com/issues/24064>`_, `pr#21912 <https://github.com/ceph/ceph/pull/21912>`_, Jason Dillaman)
+* rbd: Document new RBD feature flags and version support (`pr#25192 <https://github.com/ceph/ceph/pull/25192>`_, Valentin Lorentz)
+* rbd: don't load config overrides from monitor initially (`pr#21910 <https://github.com/ceph/ceph/pull/21910>`_, Jason Dillaman)
+* rbd: error if new size is equal to original size (`pr#22637 <https://github.com/ceph/ceph/pull/22637>`_, zhengyin)
+* rbd: expose pool stats summary tool (`pr#24830 <https://github.com/ceph/ceph/pull/24830>`_, Jason Dillaman)
+* rbd: filter out group/trash snapshots from snap_list (`pr#23638 <https://github.com/ceph/ceph/pull/23638>`_, songweibin)
+* rbd: fix a typo in error output (`pr#25931 <https://github.com/ceph/ceph/pull/25931>`_, Dongsheng Yang)
+* rbd: fix delay time calculation for trash move (`pr#25896 <https://github.com/ceph/ceph/pull/25896>`_, Mykola Golub)
+* rbd: fix error import when the input is a pipe (`issue#34536 <http://tracker.ceph.com/issues/34536>`_, `pr#23835 <https://github.com/ceph/ceph/pull/23835>`_, songweibin)
+* rbd: fix segmentation fault when rbd_group_image_list() getting -ENOENT (`issue#38468 <http://tracker.ceph.com/issues/38468>`_, `pr#26622 <https://github.com/ceph/ceph/pull/26622>`_, songweibin)
+* rbd: fix some typos (`pr#25083 <https://github.com/ceph/ceph/pull/25083>`_, Shiyang Ruan)
+* rbd: implement new 'rbd perf image iostat/iotop' commands (`issue#37913 <http://tracker.ceph.com/issues/37913>`_, `pr#26133 <https://github.com/ceph/ceph/pull/26133>`_, Jason Dillaman)
+* rbd: improved trash snapshot namespace handling (`issue#23398 <http://tracker.ceph.com/issues/23398>`_, `pr#23191 <https://github.com/ceph/ceph/pull/23191>`_, Jason Dillaman)
+* rbd: interlock object-map/fast-diff features together (`pr#21969 <https://github.com/ceph/ceph/pull/21969>`_, Mao Zhongyi)
+* rbd: introduce abort_on_full option for rbd map (`pr#25662 <https://github.com/ceph/ceph/pull/25662>`_, Dongsheng Yang)
+* rbd: journal: allow remove set when jounal pool is full (`pr#25166 <https://github.com/ceph/ceph/pull/25166>`_, kungf)
+* rbd: journal: fix potential race when closing object recorder (`pr#26425 <https://github.com/ceph/ceph/pull/26425>`_, Mykola Golub)
+* rbd:  journal: set max journal order to 26 (`issue#37541 <http://tracker.ceph.com/issues/37541>`_, `pr#25743 <https://github.com/ceph/ceph/pull/25743>`_, Mykola Golub)
+* rbd: krbd: support for images within namespaces (`pr#23841 <https://github.com/ceph/ceph/pull/23841>`_, Ilya Dryomov)
+* rbd: librbd/api: misc fix migration (`pr#25765 <https://github.com/ceph/ceph/pull/25765>`_, songweibin)
+* rbd:  librbd: ensure exclusive lock acquired when removing sync point snapshots (`issue#24898 <http://tracker.ceph.com/issues/24898>`_, `pr#23095 <https://github.com/ceph/ceph/pull/23095>`_, Mykola Golub)
+* rbd:  librbd: misc fix potential invalid pointer (`pr#25462 <https://github.com/ceph/ceph/pull/25462>`_, songweibin)
+* rbd: make sure the return-value 'r' will be returned (`pr#24891 <https://github.com/ceph/ceph/pull/24891>`_, Shiyang Ruan)
+* rbd: mgr/dashboard: incorporate RBD overall performance grafana dashboard (`issue#37867 <http://tracker.ceph.com/issues/37867>`_, `pr#25927 <https://github.com/ceph/ceph/pull/25927>`_, Jason Dillaman)
+* rbd-mirror: always attempt to restart canceled status update task (`issue#36500 <http://tracker.ceph.com/issues/36500>`_, `pr#24646 <https://github.com/ceph/ceph/pull/24646>`_, Jason Dillaman)
+* rbd-mirror: bootstrap needs to handle local image id collision (`issue#24139 <http://tracker.ceph.com/issues/24139>`_, `pr#22043 <https://github.com/ceph/ceph/pull/22043>`_, Jason Dillaman)
+* rbd-mirror: create and export replication perf counters to mgr (`pr#25834 <https://github.com/ceph/ceph/pull/25834>`_, Mykola Golub)
+* rbd-mirror: ensure daemon can cleanly exit if pool is deleted (`pr#22348 <https://github.com/ceph/ceph/pull/22348>`_, Jason Dillaman)
+* rbd-mirror: ensure remote demotion is replayed locally (`issue#24009 <http://tracker.ceph.com/issues/24009>`_, `pr#21823 <https://github.com/ceph/ceph/pull/21823>`_, Jason Dillaman)
+* rbd-mirror: fixed potential crashes during shut down (`issue#24008 <http://tracker.ceph.com/issues/24008>`_, `pr#21817 <https://github.com/ceph/ceph/pull/21817>`_, Jason Dillaman)
+* rbd-mirror: guard access to image replayer perf counters (`pr#26097 <https://github.com/ceph/ceph/pull/26097>`_, Mykola Golub)
+* rbd-mirror: instantiate the status formatter before changing state (`issue#36084 <http://tracker.ceph.com/issues/36084>`_, `pr#24181 <https://github.com/ceph/ceph/pull/24181>`_, Jason Dillaman)
+* rbd-mirror: optionally extract peer secrets from config-key (`issue#24688 <http://tracker.ceph.com/issues/24688>`_, `pr#24036 <https://github.com/ceph/ceph/pull/24036>`_, Jason Dillaman)
+* rbd-mirror: optionally support active/active replication (`pr#21915 <https://github.com/ceph/ceph/pull/21915>`_, Mykola Golub, Jason Dillaman)
+* rbd-mirror: potential deadlock when running asok 'flush' command (`issue#24141 <http://tracker.ceph.com/issues/24141>`_, `pr#22027 <https://github.com/ceph/ceph/pull/22027>`_, Mykola Golub)
+* rbd-mirror: prevent creation of clones when parents are syncing (`issue#24140 <http://tracker.ceph.com/issues/24140>`_, `pr#24063 <https://github.com/ceph/ceph/pull/24063>`_, Jason Dillaman)
+* rbd-mirror: schedule rebalancer to level-load instances (`issue#24161 <http://tracker.ceph.com/issues/24161>`_, `pr#22304 <https://github.com/ceph/ceph/pull/22304>`_, Venky Shankar)
+* rbd-mirror: update mirror status when stopping (`issue#36659 <http://tracker.ceph.com/issues/36659>`_, `pr#24864 <https://github.com/ceph/ceph/pull/24864>`_, Jason Dillaman)
+* rbd-mirror: use active/active policy by default (`issue#38453 <http://tracker.ceph.com/issues/38453>`_, `pr#26603 <https://github.com/ceph/ceph/pull/26603>`_, Jason Dillaman)
+* rbd: move image to trash as first step when removing (`issue#24226 <http://tracker.ceph.com/issues/24226>`_, `issue#38404 <http://tracker.ceph.com/issues/38404>`_, `pr#25438 <https://github.com/ceph/ceph/pull/25438>`_, Mahati Chamarthy, Jason Dillaman)
+* rbd-nbd: do not ceph_abort() after print the usages (`issue#36660 <http://tracker.ceph.com/issues/36660>`_, `pr#24815 <https://github.com/ceph/ceph/pull/24815>`_, Shiyang Ruan)
+* rbd-nbd: support namespaces (`issue#24609 <http://tracker.ceph.com/issues/24609>`_, `pr#25260 <https://github.com/ceph/ceph/pull/25260>`_, Mykola Golub)
+* rbd: not allowed to restore an image when it is being deleted (`issue#25346 <http://tracker.ceph.com/issues/25346>`_, `pr#24078 <https://github.com/ceph/ceph/pull/24078>`_, songweibin)
+* rbd: online re-sparsify of images (`pr#26226 <https://github.com/ceph/ceph/pull/26226>`_, Mykola Golub)
+* rbd: pybind/rbd: add namespace helper API methods (`issue#36622 <http://tracker.ceph.com/issues/36622>`_, `pr#25206 <https://github.com/ceph/ceph/pull/25206>`_, Jason Dillaman)
+* rbd: qa/workunits: fixed mon address parsing for rbd-mirror (`issue#38385 <http://tracker.ceph.com/issues/38385>`_, `pr#26521 <https://github.com/ceph/ceph/pull/26521>`_, Jason Dillaman)
+* rbd:  rbd: fix error parse arg when getting key (`pr#25152 <https://github.com/ceph/ceph/pull/25152>`_, songweibin)
+* rbd: rbd-fuse: look for ceph.conf in standard locations (`issue#12219 <http://tracker.ceph.com/issues/12219>`_, `pr#20598 <https://github.com/ceph/ceph/pull/20598>`_, Jason Dillaman)
+* rbd: rbd-fuse: namespace support (`pr#25265 <https://github.com/ceph/ceph/pull/25265>`_, Mykola Golub)
+* rbd: rbd-ggate: support namespaces (`issue#24608 <http://tracker.ceph.com/issues/24608>`_, `pr#25266 <https://github.com/ceph/ceph/pull/25266>`_, Mykola Golub)
+* rbd: rbd-ggate: tag "level" with need_dynamic (`pr#22557 <https://github.com/ceph/ceph/pull/22557>`_, Kefu Chai)
+* rbd: rbd_mirror: assert no requests on destroying InstanceWatcher (`pr#25666 <https://github.com/ceph/ceph/pull/25666>`_, Mykola Golub)
+* rbd: rbd_mirror: don't report error if image replay canceled (`pr#25789 <https://github.com/ceph/ceph/pull/25789>`_, Mykola Golub)
+* rbd:  rbd-mirror: use pool level config overrides (`pr#24348 <https://github.com/ceph/ceph/pull/24348>`_, Mykola Golub)
+* rbd:  rbd: show info about mirror daemon instance in image mirror status output (`pr#24717 <https://github.com/ceph/ceph/pull/24717>`_, Mykola Golub)
+* rbd: return error code when the source and distination namespace are different (`pr#24893 <https://github.com/ceph/ceph/pull/24893>`_, Shiyang Ruan)
+* rbd: simplified code to remove do_clear_limit function (`pr#23954 <https://github.com/ceph/ceph/pull/23954>`_, Zheng Yin)
+* rbd: support namespaces for image migration (`issue#26951 <http://tracker.ceph.com/issues/26951>`_, `pr#24836 <https://github.com/ceph/ceph/pull/24836>`_, Jason Dillaman)
+* rbd:  systemd/rbdmap.service: order us before remote-fs-pre.target (`issue#24713 <http://tracker.ceph.com/issues/24713>`_, `pr#22769 <https://github.com/ceph/ceph/pull/22769>`_, Ilya Dryomov)
+* rbd: test/librbd: drop unused variable ‘num_aios’ (`pr#23085 <https://github.com/ceph/ceph/pull/23085>`_, songweibin)
+* rbd,tests: krbd: alloc_size map option and tests (`pr#26244 <https://github.com/ceph/ceph/pull/26244>`_, Ilya Dryomov)
+* rbd,tests: librbd,test: remove unused context_cb() function, silence GCC warnings (`pr#24673 <https://github.com/ceph/ceph/pull/24673>`_, Kefu Chai)
+* rbd,tests: pybind/rbd: add assert_raise in test set_snap (`pr#22570 <https://github.com/ceph/ceph/pull/22570>`_, Zheng Yin)
+* rbd,tests: qa: krbd_exclusive_option.sh: bump lock_timeout to 60 seconds (`issue#25080 <http://tracker.ceph.com/issues/25080>`_, `pr#22648 <https://github.com/ceph/ceph/pull/22648>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd_msgr_segments.t: filter lvcreate output (`pr#22665 <https://github.com/ceph/ceph/pull/22665>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd namespaces test (`pr#26339 <https://github.com/ceph/ceph/pull/26339>`_, Ilya Dryomov)
+* rbd,tests: qa: objectstore snippets for krbd (`pr#26279 <https://github.com/ceph/ceph/pull/26279>`_, Ilya Dryomov)
+* rbd,tests: qa: rbd_workunit_kernel_untar_build: install build dependencies (`issue#35074 <http://tracker.ceph.com/issues/35074>`_, `pr#23840 <https://github.com/ceph/ceph/pull/23840>`_, Ilya Dryomov)
+* rbd,tests: qa: rbd/workunits : Replace "rbd bench-write" with "rbd bench --io-type write" (`pr#26168 <https://github.com/ceph/ceph/pull/26168>`_, Shyukri Shyukriev)
+* rbd,tests: qa/suites/krbd: more fsx tests (`pr#24354 <https://github.com/ceph/ceph/pull/24354>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/rbd: randomly select a supported distro (`pr#22008 <https://github.com/ceph/ceph/pull/22008>`_, Jason Dillaman)
+* rbd,tests:  qa/tasks/cram: tasks now must live in the repository (`pr#23976 <https://github.com/ceph/ceph/pull/23976>`_, Ilya Dryomov)
+* rbd,tests: qa/tasks/cram: use suite_repo repository for all cram jobs (`pr#23905 <https://github.com/ceph/ceph/pull/23905>`_, Ilya Dryomov)
+* rbd,tests: qa/tasks/qemu: use unique clone directory to avoid race with workunit (`issue#36542 <http://tracker.ceph.com/issues/36542>`_, `pr#24696 <https://github.com/ceph/ceph/pull/24696>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: fix cli generic namespace test (`pr#24457 <https://github.com/ceph/ceph/pull/24457>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: force v2 image format for namespace test (`pr#24512 <https://github.com/ceph/ceph/pull/24512>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: replace usage of 'rados mkpool' (`pr#23938 <https://github.com/ceph/ceph/pull/23938>`_, Jason Dillaman)
+* rbd,tests: qa/workunits: replace 'realpath' with 'readlink -f' in fsstress.sh (`issue#36409 <http://tracker.ceph.com/issues/36409>`_, `pr#24550 <https://github.com/ceph/ceph/pull/24550>`_, Jason Dillaman)
+* rbd,tests: test/cli-integration/rbd: added new parent image attributes (`pr#25415 <https://github.com/ceph/ceph/pull/25415>`_, Jason Dillaman)
+* rbd,tests: test/librados_test_stub: deterministically load cls shared libraries (`pr#21524 <https://github.com/ceph/ceph/pull/21524>`_, Jason Dillaman)
+* rbd,tests: test/librados_test_stub: handle object doesn't exist gracefully (`pr#25667 <https://github.com/ceph/ceph/pull/25667>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix compiler -Wsign-compare warnings (`pr#23657 <https://github.com/ceph/ceph/pull/23657>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix gmock warning in snapshot rollback test (`pr#23736 <https://github.com/ceph/ceph/pull/23736>`_, Jason Dillaman)
+* rbd,tests: test/librbd: fix gmock warning in TestMockIoImageRequestWQ.AcquireLockError (`pr#22778 <https://github.com/ceph/ceph/pull/22778>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix gmock warnings for get_modify_timestamp call (`pr#23707 <https://github.com/ceph/ceph/pull/23707>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix 'Uninteresting mock function call' warning (`pr#26322 <https://github.com/ceph/ceph/pull/26322>`_, Mykola Golub)
+* rbd,tests:  test/librbd: fix valgrind warnings (`pr#23827 <https://github.com/ceph/ceph/pull/23827>`_, Mykola Golub)
+* rbd,tests: test/librbd: fix -Wsign-compare warnings (`pr#23608 <https://github.com/ceph/ceph/pull/23608>`_, Kefu Chai)
+* rbd,tests: test/librbd: metadata key for config should be prefixed with ``conf_`` (`pr#25209 <https://github.com/ceph/ceph/pull/25209>`_, runsisi)
+* rbd,tests: test/librbd: migration supporting namespace tests (`pr#24919 <https://github.com/ceph/ceph/pull/24919>`_, Mykola Golub)
+* rbd,tests: test/librbd: migration tests did not delete additional pool (`pr#24009 <https://github.com/ceph/ceph/pull/24009>`_, Mykola Golub)
+* rbd,tests: test: move OpenStack devstack test to rocky release (`issue#36410 <http://tracker.ceph.com/issues/36410>`_, `pr#24563 <https://github.com/ceph/ceph/pull/24563>`_, Jason Dillaman)
+* rbd,tests: test/pybind: fix test_rbd.TestClone.test_trash_snapshot (`issue#25114 <http://tracker.ceph.com/issues/25114>`_, `pr#23256 <https://github.com/ceph/ceph/pull/23256>`_, Mykola Golub)
+* rbd,tests: test/pybind/test_rbd: filter out unknown list_children2 keys (`issue#37729 <http://tracker.ceph.com/issues/37729>`_, `pr#25832 <https://github.com/ceph/ceph/pull/25832>`_, Mykola Golub)
+* rbd,tests: test/rbd-mirror: disable use of gtest-parallel (`pr#22694 <https://github.com/ceph/ceph/pull/22694>`_, Jason Dillaman)
+* rbd,tests:  test/rbd_mirror: fix gmock warnings (`pr#25863 <https://github.com/ceph/ceph/pull/25863>`_, Mykola Golub)
+* rbd,tests: test/rbd_mirror: race in TestMockImageMap.AddInstancePingPongImageTest (`issue#36683 <http://tracker.ceph.com/issues/36683>`_, `pr#24897 <https://github.com/ceph/ceph/pull/24897>`_, Mykola Golub)
+* rbd,tests: test/rbd_mirror: race in WaitingOnLeaderReleaseLeader (`issue#36236 <http://tracker.ceph.com/issues/36236>`_, `pr#24300 <https://github.com/ceph/ceph/pull/24300>`_, Mykola Golub)
+* rbd,tests: test/rbd_mirror: wait for release leader lock fully complete (`pr#25935 <https://github.com/ceph/ceph/pull/25935>`_, Mykola Golub)
+* rbd,tests: test/rbd: rbd_ggate test improvements (`pr#23630 <https://github.com/ceph/ceph/pull/23630>`_, Willem Jan Withagen)
+* rbd,tests: test: silence -Wsign-compare warnings (`pr#23655 <https://github.com/ceph/ceph/pull/23655>`_, Kefu Chai)
+* rbd: tools/rbd/action: align column headers left (`pr#22566 <https://github.com/ceph/ceph/pull/22566>`_, Sage Weil)
+* rbd: tools/rbd: assert(g_ceph_context) not g_conf (`pr#23167 <https://github.com/ceph/ceph/pull/23167>`_, Kefu Chai)
+* rbd: tools/rbd: minor fixes for rbd du display (`pr#23311 <https://github.com/ceph/ceph/pull/23311>`_, songweibin)
+* rbd,tools: rbd-mirror,common: fix typos in logging messages and comments (`pr#25197 <https://github.com/ceph/ceph/pull/25197>`_, Shiyang Ruan)
+* rbd,tools: tools/rbd: assert(g_ceph_context) not g_conf (`pr#23008 <https://github.com/ceph/ceph/pull/23008>`_, Kefu Chai)
+* rbd: wait for all io complete when bench is interrupted (`pr#26918 <https://github.com/ceph/ceph/pull/26918>`_, Mykola Golub)
+* rbd: workaround for llvm linker problem, avoid std:pair dtor (`pr#25301 <https://github.com/ceph/ceph/pull/25301>`_, Willem Jan Withagen)
+* Revert "cephfs-journal-tool: enable purge_queue journal's event comma… (`pr#23465 <https://github.com/ceph/ceph/pull/23465>`_, "Yan, Zheng")
+* Revert "ceph-fuse: Delete inode's bufferhead was in Tx state would le… (`pr#21975 <https://github.com/ceph/ceph/pull/21975>`_, "Yan, Zheng")
+* rgw: abort_bucket_multiparts() ignores individual NoSuchUpload errors (`issue#35986 <http://tracker.ceph.com/issues/35986>`_, `pr#24110 <https://github.com/ceph/ceph/pull/24110>`_, Casey Bodley)
+* rgw: adapt AioThrottle for RGWGetObj (`pr#25208 <https://github.com/ceph/ceph/pull/25208>`_, Casey Bodley)
+* rgw: Add append object api (`pr#22755 <https://github.com/ceph/ceph/pull/22755>`_, zhang Shaowen, Zhang Shaowen)
+* rgw: add bucket as option when show/trim usage (`pr#23819 <https://github.com/ceph/ceph/pull/23819>`_, lvshuhua)
+* rgw: add configurable AWS-compat invalid range get behavior (`issue#24317 <http://tracker.ceph.com/issues/24317>`_, `pr#22231 <https://github.com/ceph/ceph/pull/22231>`_, Matt Benjamin)
+* rgw: add curl_low_speed_limit and curl_low_speed_time config to avoid (`pr#23058 <https://github.com/ceph/ceph/pull/23058>`_, Mark Kogan, Zhang Shaowen)
+* rgw: add Http header 'Server' in response headers (`pr#23282 <https://github.com/ceph/ceph/pull/23282>`_, Zhang Shaowen)
+* rgw: Adding documentation for Roles (`pr#24714 <https://github.com/ceph/ceph/pull/24714>`_, Pritha Srivastava)
+* rgw: add latency info in the log of req done (`pr#23906 <https://github.com/ceph/ceph/pull/23906>`_, lvshuhua)
+* rgw: add list user admin OP API (`pr#25073 <https://github.com/ceph/ceph/pull/25073>`_, Oshyn Song)
+* rgw: add --op-mask in radosgw-admin help info (`pr#24848 <https://github.com/ceph/ceph/pull/24848>`_, yuliyang)
+* rgw: add optional_yield to block_while_resharding() (`pr#25357 <https://github.com/ceph/ceph/pull/25357>`_, Casey Bodley)
+* rgw: add option for relaxed region enforcement (`issue#24507 <http://tracker.ceph.com/issues/24507>`_, `pr#22533 <https://github.com/ceph/ceph/pull/22533>`_, Matt Benjamin)
+* rgw: Add rgw xml unit tests (`pr#26682 <https://github.com/ceph/ceph/pull/26682>`_, Yuval Lifshitz)
+* rgw: add s3 notification sub resources (`pr#23405 <https://github.com/ceph/ceph/pull/23405>`_, yuliyang)
+* rgw: admin rest api support op-mask (`pr#24869 <https://github.com/ceph/ceph/pull/24869>`_, yuliyang)
+* rgw: admin/user ops dump user 'system' flag (`pr#17414 <https://github.com/ceph/ceph/pull/17414>`_, fang.yuxiang)
+* rgw: All Your Fault (`issue#24962 <http://tracker.ceph.com/issues/24962>`_, `pr#23099 <https://github.com/ceph/ceph/pull/23099>`_, Adam C. Emerson)
+* rgw: apply quota config to users created via external auth (`issue#24595 <http://tracker.ceph.com/issues/24595>`_, `pr#24177 <https://github.com/ceph/ceph/pull/24177>`_, Casey Bodley)
+* rgw: archive zone (`pr#25137 <https://github.com/ceph/ceph/pull/25137>`_, Yehuda Sadeh, Javier M. Mellid)
+* rgw: async sync_object and remove_object does not access coroutine me… (`issue#35905 <http://tracker.ceph.com/issues/35905>`_, `pr#24007 <https://github.com/ceph/ceph/pull/24007>`_, Tianshan Qu)
+* rgw: async watch registration (`pr#21838 <https://github.com/ceph/ceph/pull/21838>`_, Yehuda Sadeh)
+* rgw: avoid race condition in RGWHTTPClient::wait() (`pr#21767 <https://github.com/ceph/ceph/pull/21767>`_, cfanz)
+* rgw: beast frontend logs socket errors at level 4 (`pr#24677 <https://github.com/ceph/ceph/pull/24677>`_, Casey Bodley)
+* rgw: beast frontend parses ipv6 addrs (`issue#36662 <http://tracker.ceph.com/issues/36662>`_, `pr#24887 <https://github.com/ceph/ceph/pull/24887>`_, Casey Bodley)
+* rgw: beast frontend reworks pause/stop and yields during body io (`pr#21271 <https://github.com/ceph/ceph/pull/21271>`_, Casey Bodley)
+* rgw: bucket full sync handles delete markers (`issue#38007 <http://tracker.ceph.com/issues/38007>`_, `pr#26081 <https://github.com/ceph/ceph/pull/26081>`_, Casey Bodley)
+* rgw: bucket limit check misbehaves for > max-entries buckets (usually… (`pr#26800 <https://github.com/ceph/ceph/pull/26800>`_, Matt Benjamin)
+* rgw: bucket sync status improvements, part 1 (`pr#21788 <https://github.com/ceph/ceph/pull/21788>`_, Casey Bodley)
+* rgw: bug in versioning concurrent, list and get have consistency issue (`pr#26197 <https://github.com/ceph/ceph/pull/26197>`_, Wang Hao)
+* rgw: catch exceptions from librados::NObjectIterator (`issue#37091 <http://tracker.ceph.com/issues/37091>`_, `pr#25081 <https://github.com/ceph/ceph/pull/25081>`_, Casey Bodley)
+* rgw: change default rgw_thread_pool_size to 512 (`issue#24544 <http://tracker.ceph.com/issues/24544>`_, `pr#22581 <https://github.com/ceph/ceph/pull/22581>`_, Douglas Fuller)
+* rgw: change the "rgw admin status" 'num_shards' output to signed int (`issue#37645 <http://tracker.ceph.com/issues/37645>`_, `pr#25538 <https://github.com/ceph/ceph/pull/25538>`_, Mark Kogan)
+* rgw: check for non-existent bucket in RGWGetACLs (`pr#26212 <https://github.com/ceph/ceph/pull/26212>`_, Matt Benjamin)
+* rgw: civetweb: update for url validation fixes (`issue#24158 <http://tracker.ceph.com/issues/24158>`_, `pr#22054 <https://github.com/ceph/ceph/pull/22054>`_, Abhishek Lekshmanan)
+* rgw: civetweb: use poll instead of select while waiting on sockets (`issue#24364 <http://tracker.ceph.com/issues/24364>`_, `pr#24027 <https://github.com/ceph/ceph/pull/24027>`_, Abhishek Lekshmanan)
+* rgw: clean-up -- insure C++ source code files contain editor directives (`pr#25495 <https://github.com/ceph/ceph/pull/25495>`_, J. Eric Ivancich)
+* rgw: cleanups for sync tracing (`pr#23828 <https://github.com/ceph/ceph/pull/23828>`_, Casey Bodley)
+* rgw: clean-up -- use enum class for stats category (`pr#25450 <https://github.com/ceph/ceph/pull/25450>`_, J. Eric Ivancich)
+* rgw: cls/rgw: don't assert in decode_list_index_key() (`issue#24117 <http://tracker.ceph.com/issues/24117>`_, `pr#22440 <https://github.com/ceph/ceph/pull/22440>`_, Yehuda Sadeh)
+* rgw: cls/rgw: raise debug level of bi_log_iterate_entries output (`pr#25570 <https://github.com/ceph/ceph/pull/25570>`_, Casey Bodley)
+* rgw: cls/user: cls_user_remove_bucket writes modified header (`issue#36496 <http://tracker.ceph.com/issues/36496>`_, `pr#24645 <https://github.com/ceph/ceph/pull/24645>`_, Casey Bodley)
+* rgw: Code for STS Authentication (`pr#23504 <https://github.com/ceph/ceph/pull/23504>`_, Pritha Srivastava)
+* rgw: common/options: correct the description of rgw_enable_lc_threads option (`pr#23511 <https://github.com/ceph/ceph/pull/23511>`_, excellentkf)
+* rgw: continue enoent index in dir_suggest (`issue#24640 <http://tracker.ceph.com/issues/24640>`_, `pr#22937 <https://github.com/ceph/ceph/pull/22937>`_, Tianshan Qu)
+* rgw: copy actual stats from the source shards during reshard (`issue#36290 <http://tracker.ceph.com/issues/36290>`_, `pr#24444 <https://github.com/ceph/ceph/pull/24444>`_, Abhishek Lekshmanan)
+* rgw: Copying object data should generate new tail tag for the new object (`issue#24562 <http://tracker.ceph.com/issues/24562>`_, `pr#22613 <https://github.com/ceph/ceph/pull/22613>`_, Zhang Shaowen)
+* rgw: Correcting logic for signature calculation for non s3 ops (`pr#26098 <https://github.com/ceph/ceph/pull/26098>`_, Pritha Srivastava)
+* rgw: cors rules num limit (`pr#23434 <https://github.com/ceph/ceph/pull/23434>`_, yuliyang)
+* rgw: crypto: add openssl support for RGW encryption (`pr#15168 <https://github.com/ceph/ceph/pull/15168>`_, Qiaowei Ren)
+* rgw: data sync accepts ERR_PRECONDITION_FAILED on remove_object() (`issue#37448 <http://tracker.ceph.com/issues/37448>`_, `pr#25310 <https://github.com/ceph/ceph/pull/25310>`_, Casey Bodley)
+* rgw: data sync drains lease stack on lease failure (`issue#38479 <http://tracker.ceph.com/issues/38479>`_, `pr#26639 <https://github.com/ceph/ceph/pull/26639>`_, Casey Bodley)
+* rgw: data sync respects error_retry_time for backoff on error_repo (`issue#26938 <http://tracker.ceph.com/issues/26938>`_, `pr#23571 <https://github.com/ceph/ceph/pull/23571>`_, Casey Bodley)
+* rgw: delete multi object num limit (`pr#23544 <https://github.com/ceph/ceph/pull/23544>`_, yuliyang)
+* rgw: delete some unused code about std::regex (`pr#23221 <https://github.com/ceph/ceph/pull/23221>`_, Xueyu Bai)
+* rgw: [DNM] rgw: Controlling STS authentication via a Policy (`pr#24818 <https://github.com/ceph/ceph/pull/24818>`_, Pritha Srivastava)
+* rgw: do not ignore EEXIST in RGWPutObj::execute (`issue#22790 <http://tracker.ceph.com/issues/22790>`_, `pr#23033 <https://github.com/ceph/ceph/pull/23033>`_, Matt Benjamin)
+* rgw: Do not modify email if argument is not set (`pr#22024 <https://github.com/ceph/ceph/pull/22024>`_, Volker Theile)
+* rgw: dont access rgw_http_req_data::client of canceled request (`issue#35851 <http://tracker.ceph.com/issues/35851>`_, `pr#23988 <https://github.com/ceph/ceph/pull/23988>`_, Casey Bodley)
+* rgw: Don't treat colons specially when matching resource field of ARNs in S3 Policy (`issue#23817 <http://tracker.ceph.com/issues/23817>`_, `pr#25145 <https://github.com/ceph/ceph/pull/25145>`_, Adam C. Emerson)
+* rgw: drop unused tmp in main() (`pr#23899 <https://github.com/ceph/ceph/pull/23899>`_, luomuyao)
+* rgw: escape markers in RGWOp_Metadata_List::execute (`issue#23099 <http://tracker.ceph.com/issues/23099>`_, `pr#22721 <https://github.com/ceph/ceph/pull/22721>`_, Matt Benjamin)
+* rgw: ES sync: be more restrictive on object system attrs (`issue#36233 <http://tracker.ceph.com/issues/36233>`_, `pr#24492 <https://github.com/ceph/ceph/pull/24492>`_, Abhishek Lekshmanan)
+* rgw: etag in rgw copy result response body rather in header (`pr#23751 <https://github.com/ceph/ceph/pull/23751>`_, yuliyang)
+* rgw: feature -- log successful bucket resharding events (`pr#25510 <https://github.com/ceph/ceph/pull/25510>`_, J. Eric Ivancich)
+* rgw: fetch_remote_obj filters out olh attrs (`issue#37792 <http://tracker.ceph.com/issues/37792>`_, `pr#25794 <https://github.com/ceph/ceph/pull/25794>`_, Casey Bodley)
+* rgw: fix bad user stats on versioned bucket after reshard (`pr#25414 <https://github.com/ceph/ceph/pull/25414>`_, J. Eric Ivancich)
+* rgw: fix build (`pr#22194 <https://github.com/ceph/ceph/pull/22194>`_, Yehuda Sadeh)
+* rgw: fix build (`pr#23248 <https://github.com/ceph/ceph/pull/23248>`_, Matt Benjamin)
+* rgw: fix chunked-encoding for chunks >1MiB (`issue#35990 <http://tracker.ceph.com/issues/35990>`_, `pr#24114 <https://github.com/ceph/ceph/pull/24114>`_, Robin H. Johnson)
+* rgw: fix compilation after pubsub conflict (`pr#25568 <https://github.com/ceph/ceph/pull/25568>`_, Casey Bodley)
+* rgw: fix copy response header etag format not correct (`issue#24563 <http://tracker.ceph.com/issues/24563>`_, `pr#22614 <https://github.com/ceph/ceph/pull/22614>`_, Tianshan Qu)
+* rgw: fix CreateBucket with BucketLocation parameter failed under default zonegroup (`pr#22312 <https://github.com/ceph/ceph/pull/22312>`_, Enming Zhang)
+* rgw: fix deadlock on RGWIndexCompletionManager::stop (`issue#26949 <http://tracker.ceph.com/issues/26949>`_, `pr#23590 <https://github.com/ceph/ceph/pull/23590>`_, Yao Zongyou)
+* rgw: fix dependencies/target_link_libraries (`pr#23056 <https://github.com/ceph/ceph/pull/23056>`_, Michal Jarzabek)
+* rgw: fixes for sync of versioned objects (`issue#24367 <http://tracker.ceph.com/issues/24367>`_, `pr#22347 <https://github.com/ceph/ceph/pull/22347>`_, Casey Bodley)
+* rgw: Fixes to permission evaluation related to user policies (`pr#25180 <https://github.com/ceph/ceph/pull/25180>`_, Pritha Srivastava)
+* rgw: fix Etag error in multipart copy response (`pr#23749 <https://github.com/ceph/ceph/pull/23749>`_, yuliyang)
+* rgw: Fix for buffer overflow in STS op_post() (`issue#36579 <http://tracker.ceph.com/issues/36579>`_, `pr#24510 <https://github.com/ceph/ceph/pull/24510>`_, Pritha Srivastava, Marcus Watts)
+* rgw: Fix for SignatureMismatchError in s3 commands (`pr#26204 <https://github.com/ceph/ceph/pull/26204>`_, Pritha Srivastava)
+* rgw: fix FTBFS introduced by abca9805 (`pr#23046 <https://github.com/ceph/ceph/pull/23046>`_, Kefu Chai)
+* rgw: fix index complete miss zones_trace set (`issue#24590 <http://tracker.ceph.com/issues/24590>`_, `pr#22632 <https://github.com/ceph/ceph/pull/22632>`_, Tianshan Qu)
+* rgw: fix index update in dir_suggest_changes (`issue#24280 <http://tracker.ceph.com/issues/24280>`_, `pr#22217 <https://github.com/ceph/ceph/pull/22217>`_, Tianshan Qu)
+* rgw: fix ldap secret parsing (`pr#25796 <https://github.com/ceph/ceph/pull/25796>`_, Matt Benjamin)
+* rgw: fix leak of curl handle on shutdown (`issue#35715 <http://tracker.ceph.com/issues/35715>`_, `pr#23986 <https://github.com/ceph/ceph/pull/23986>`_, Casey Bodley)
+* rgw: Fix log level of gc_iterate_entries (`issue#23801 <http://tracker.ceph.com/issues/23801>`_, `pr#22868 <https://github.com/ceph/ceph/pull/22868>`_, iliul)
+* rgw: fix max-size in radosgw-admin and REST Admin API (`pr#24062 <https://github.com/ceph/ceph/pull/24062>`_, Nick Erdmann)
+* rgw: fix meta and data notify thread miss stop cr manager (`issue#24589 <http://tracker.ceph.com/issues/24589>`_, `pr#22631 <https://github.com/ceph/ceph/pull/22631>`_, Tianshan Qu)
+* rgw: fix obj can still be deleted even if deleteobject policy is set (`issue#37403 <http://tracker.ceph.com/issues/37403>`_, `pr#25278 <https://github.com/ceph/ceph/pull/25278>`_, Enming.Zhang)
+* rgw: fix radosgw-admin build error (`pr#21599 <https://github.com/ceph/ceph/pull/21599>`_, cfanz)
+* rgw: fix rgw_data_sync_info::json_decode() (`issue#38373 <http://tracker.ceph.com/issues/38373>`_, `pr#26494 <https://github.com/ceph/ceph/pull/26494>`_, Casey Bodley)
+* rgw: fix RGWSyncTraceNode crash in reload (`issue#24432 <http://tracker.ceph.com/issues/24432>`_, `pr#22432 <https://github.com/ceph/ceph/pull/22432>`_, Tianshan Qu)
+* rgw: fix stats for versioned buckets after reshard (`pr#25333 <https://github.com/ceph/ceph/pull/25333>`_, J. Eric Ivancich)
+* rgw: fix uninitialized access (`pr#25002 <https://github.com/ceph/ceph/pull/25002>`_, Yehuda Sadeh)
+* rgw: fix unordered bucket listing when object names are adorned (`issue#38486 <http://tracker.ceph.com/issues/38486>`_, `pr#26658 <https://github.com/ceph/ceph/pull/26658>`_, J. Eric Ivancich)
+* rgw: fix vector index out of range in RGWReadDataSyncRecoveringShardsCR (`issue#36537 <http://tracker.ceph.com/issues/36537>`_, `pr#24680 <https://github.com/ceph/ceph/pull/24680>`_, Casey Bodley)
+* rgw: fix version bucket stats (`issue#21429 <http://tracker.ceph.com/issues/21429>`_, `pr#17789 <https://github.com/ceph/ceph/pull/17789>`_, Shasha Lu)
+* rgw: fix versioned obj copy generating tags (`issue#37588 <http://tracker.ceph.com/issues/37588>`_, `pr#25473 <https://github.com/ceph/ceph/pull/25473>`_, Abhishek Lekshmanan)
+* rgw: fix wrong debug related to user ACLs in rgw_build_bucket_policies() (`issue#19514 <http://tracker.ceph.com/issues/19514>`_, `pr#14369 <https://github.com/ceph/ceph/pull/14369>`_, Radoslaw Zarzynski)
+* rgw: get or set realm zonegroup zone need check user's caps (`pr#25178 <https://github.com/ceph/ceph/pull/25178>`_, yuliyang, Casey Bodley)
+* rgw: Get the user metadata of the user used to sign the request (`pr#22390 <https://github.com/ceph/ceph/pull/22390>`_, Volker Theile)
+* rgw: handle cases around zone deletion (`issue#37328 <http://tracker.ceph.com/issues/37328>`_, `pr#25160 <https://github.com/ceph/ceph/pull/25160>`_, Abhishek Lekshmanan)
+* rgw: handle S3 version 2 pre-signed urls with meta-data (`pr#24683 <https://github.com/ceph/ceph/pull/24683>`_, Matt Benjamin)
+* rgw: have a configurable authentication order (`issue#23089 <http://tracker.ceph.com/issues/23089>`_, `pr#21494 <https://github.com/ceph/ceph/pull/21494>`_, Abhishek Lekshmanan)
+* rgw: http client: print curl error messages during curl failures (`pr#23318 <https://github.com/ceph/ceph/pull/23318>`_, Abhishek Lekshmanan)
+* rgw: Improvements to STS Lite documentation (`pr#24847 <https://github.com/ceph/ceph/pull/24847>`_, Pritha Srivastava)
+* rgw: Initial commit for AssumeRoleWithWebIdentity (`pr#26002 <https://github.com/ceph/ceph/pull/26002>`_, Pritha Srivastava)
+* rgw: initial RGWRados refactoring work (`pr#24014 <https://github.com/ceph/ceph/pull/24014>`_, Yehuda Sadeh, Casey Bodley)
+* rgw: Initial work for OPA-Ceph integration (`pr#22624 <https://github.com/ceph/ceph/pull/22624>`_, Ashutosh Narkar)
+* rgw: librgw: initialize curl and http client for multisite (`issue#36302 <http://tracker.ceph.com/issues/36302>`_, `pr#24402 <https://github.com/ceph/ceph/pull/24402>`_, Casey Bodley)
+* rgw: librgw: support symbolic link (`pr#19684 <https://github.com/ceph/ceph/pull/19684>`_, Tao Chen)
+* rgw: lifcycle: don't reject compound rules with empty prefix (`issue#37879 <http://tracker.ceph.com/issues/37879>`_, `pr#25926 <https://github.com/ceph/ceph/pull/25926>`_, Matt Benjamin)
+* rgw: Limit the number of lifecycle rules on one bucket (`issue#24572 <http://tracker.ceph.com/issues/24572>`_, `pr#22623 <https://github.com/ceph/ceph/pull/22623>`_, Zhang Shaowen)
+* rgw: list bucket can not show the object uploaded by RGWPostObj when enable bucket versioning (`pr#24341 <https://github.com/ceph/ceph/pull/24341>`_, yuliyang)
+* rgw: log http status with op prefix if available (`pr#25102 <https://github.com/ceph/ceph/pull/25102>`_, Casey Bodley)
+* rgw: log refactoring for data sync (`pr#23843 <https://github.com/ceph/ceph/pull/23843>`_, Casey Bodley)
+* rgw: log refactoring for meta sync (`pr#23950 <https://github.com/ceph/ceph/pull/23950>`_, Casey Bodley, Ali Maredia)
+* rgw: make beast the default for rgw_frontends (`pr#26599 <https://github.com/ceph/ceph/pull/26599>`_, Casey Bodley)
+* rgw: Minor fixes to AssumeRole for boto compliance (`pr#24845 <https://github.com/ceph/ceph/pull/24845>`_, Pritha Srivastava)
+* rgw: Minor fixes to radosgw-admin commands for a role (`pr#24730 <https://github.com/ceph/ceph/pull/24730>`_, Pritha Srivastava)
+* rgw: move all reshard config options out of legacy_config_options (`pr#25356 <https://github.com/ceph/ceph/pull/25356>`_, J. Eric Ivancich)
+* rgw: move keystone secrets from ceph.conf to files (`issue#36621 <http://tracker.ceph.com/issues/36621>`_, `pr#24816 <https://github.com/ceph/ceph/pull/24816>`_, Matt Benjamin)
+* rgw: multiple es related fixes and improvements (`issue#22877 <http://tracker.ceph.com/issues/22877>`_, `issue#38028 <http://tracker.ceph.com/issues/38028>`_, `issue#38030 <http://tracker.ceph.com/issues/38030>`_, `issue#36092 <http://tracker.ceph.com/issues/36092>`_, `pr#26106 <https://github.com/ceph/ceph/pull/26106>`_, Yehuda Sadeh, Abhishek Lekshmanan)
+* rgw: need to give a type in list constructor (`pr#25161 <https://github.com/ceph/ceph/pull/25161>`_, Willem Jan Withagen)
+* rgw: new librgw_admin_us (`pr#21439 <https://github.com/ceph/ceph/pull/21439>`_, Orit Wasserman, Matt Benjamin)
+* rgw: policy: fix NotAction, NotPricipal, NotResource does not take effect (`pr#23625 <https://github.com/ceph/ceph/pull/23625>`_, xiangxiang)
+* rgw: policy: fix s3:x-amz-grant-read-acp keyword error (`pr#23610 <https://github.com/ceph/ceph/pull/23610>`_, xiangxiang)
+* rgw: policy: modify some operation permission keyword (`issue#24061 <http://tracker.ceph.com/issues/24061>`_, `pr#20974 <https://github.com/ceph/ceph/pull/20974>`_, xiangxiang)
+* rgw: pub-sub (`pr#23298 <https://github.com/ceph/ceph/pull/23298>`_, Yehuda Sadeh)
+* rgw: qa/suites/rgw/verify/tasks/cls_rgw: test cls_rgw (`pr#22919 <https://github.com/ceph/ceph/pull/22919>`_, Sage Weil)
+* rgw: radogw-admin reshard status command should print text for reshard status (`issue#23257 <http://tracker.ceph.com/issues/23257>`_, `pr#20779 <https://github.com/ceph/ceph/pull/20779>`_, Orit Wasserman)
+* rgw: radosgw-admin: add mfa related command and options (`pr#23416 <https://github.com/ceph/ceph/pull/23416>`_, Enming.Zhang)
+* rgw: `radosgw-admin bucket rm ... --purge-objects` can hang (`issue#38134 <http://tracker.ceph.com/issues/38134>`_, `pr#26231 <https://github.com/ceph/ceph/pull/26231>`_, J. Eric Ivancich)
+* rgw: "radosgw-admin objects expire" always returns ok even if the process fails (`issue#24592 <http://tracker.ceph.com/issues/24592>`_, `pr#22635 <https://github.com/ceph/ceph/pull/22635>`_, Zhang Shaowen)
+* rgw: radosgw-admin: 'sync error trim' loops until complete (`issue#24873 <http://tracker.ceph.com/issues/24873>`_, `pr#23032 <https://github.com/ceph/ceph/pull/23032>`_, Casey Bodley)
+* rgw: radosgw-admin: translate reshard status codes (trivial) (`issue#36486 <http://tracker.ceph.com/issues/36486>`_, `pr#24638 <https://github.com/ceph/ceph/pull/24638>`_, Matt Benjamin)
+* rgw: RADOS::Obj::operate takes optional_yield (`pr#25068 <https://github.com/ceph/ceph/pull/25068>`_, Casey Bodley)
+* rgw: rados tiering (`issue#19510 <http://tracker.ceph.com/issues/19510>`_, `pr#25774 <https://github.com/ceph/ceph/pull/25774>`_, yuliyang, Yehuda Sadeh, Zhang Shaowen)
+* rgw: raise debug level on redundant data sync error messages (`issue#35830 <http://tracker.ceph.com/issues/35830>`_, `pr#23981 <https://github.com/ceph/ceph/pull/23981>`_, Casey Bodley)
+* rgw: raise default rgw_curl_low_speed_time to 300 seconds (`issue#27989 <http://tracker.ceph.com/issues/27989>`_, `pr#23759 <https://github.com/ceph/ceph/pull/23759>`_, Casey Bodley)
+* rgw: refactor logging in gc and lc (`pr#24530 <https://github.com/ceph/ceph/pull/24530>`_, Ali Maredia)
+* rgw: refactor PutObjProcessor stack (`pr#24453 <https://github.com/ceph/ceph/pull/24453>`_, Casey Bodley)
+* rgw: reject invalid methods in validate_cors_rule_method (`issue#24223 <http://tracker.ceph.com/issues/24223>`_, `pr#22145 <https://github.com/ceph/ceph/pull/22145>`_, Jeegn Chen)
+* rgw: remove all traces of cls replica_log (`pr#21680 <https://github.com/ceph/ceph/pull/21680>`_, Casey Bodley)
+* rgw: remove duplicated ``RGWRados::list_buckets_`` helpers (`pr#25240 <https://github.com/ceph/ceph/pull/25240>`_, Casey Bodley)
+* rgw: remove expired entries from the cache (`issue#23379 <http://tracker.ceph.com/issues/23379>`_, `pr#22410 <https://github.com/ceph/ceph/pull/22410>`_, Mark Kogan)
+* rgw: remove repetitive conditional statement in RGWHandler_REST_Obj_S3 (`pr#24162 <https://github.com/ceph/ceph/pull/24162>`_, Zhang Shaowen)
+* rgw: remove rgw_aclparser.cc (`issue#36665 <http://tracker.ceph.com/issues/36665>`_, `pr#24866 <https://github.com/ceph/ceph/pull/24866>`_, Matt Benjamin)
+* rgw: remove the useless is_cors_op in RGWHandler_REST_Obj_S3 (`pr#22114 <https://github.com/ceph/ceph/pull/22114>`_, Zhang Shaowen)
+* rgw: remove unused aio helper functions (`pr#25239 <https://github.com/ceph/ceph/pull/25239>`_, Casey Bodley)
+* rgw: renew resharding locks to prevent expiration (`issue#27219 <http://tracker.ceph.com/issues/27219>`_, `issue#34307 <http://tracker.ceph.com/issues/34307>`_, `pr#24406 <https://github.com/ceph/ceph/pull/24406>`_, Orit Wasserman, J. Eric Ivancich)
+* rgw: repair olh attributes that were broken by sync (`issue#37792 <http://tracker.ceph.com/issues/37792>`_, `pr#26157 <https://github.com/ceph/ceph/pull/26157>`_, Casey Bodley)
+* rgw: require --yes-i-really-mean-it to run radosgw-admin orphans find (`issue#24146 <http://tracker.ceph.com/issues/24146>`_, `pr#22036 <https://github.com/ceph/ceph/pull/22036>`_, Matt Benjamin)
+* rgw: reshard add: fail correctly on a non existant bucket (`issue#36449 <http://tracker.ceph.com/issues/36449>`_, `pr#24594 <https://github.com/ceph/ceph/pull/24594>`_, Abhishek Lekshmanan)
+* rgw: reshard clean-up and associated commits (`pr#25142 <https://github.com/ceph/ceph/pull/25142>`_, J. Eric Ivancich)
+* rgw: reshard improvements (`pr#25003 <https://github.com/ceph/ceph/pull/25003>`_, J. Eric Ivancich)
+* rgw: reshard stale instance cleanup (`issue#24082 <http://tracker.ceph.com/issues/24082>`_, `pr#24662 <https://github.com/ceph/ceph/pull/24662>`_, Abhishek Lekshmanan)
+* rgw: resolve bugs and clean up garbage collection code (`issue#38454 <http://tracker.ceph.com/issues/38454>`_, `pr#26601 <https://github.com/ceph/ceph/pull/26601>`_, J. Eric Ivancich)
+* rgw: resolve bug where marker was not advanced during garbage collection (`issue#38408 <http://tracker.ceph.com/issues/38408>`_, `pr#26545 <https://github.com/ceph/ceph/pull/26545>`_, J. Eric Ivancich)
+* rgw: return err_malformed_xml when MaxAgeSeconds is an invalid integer (`issue#26957 <http://tracker.ceph.com/issues/26957>`_, `pr#23626 <https://github.com/ceph/ceph/pull/23626>`_, Chang Liu)
+* rgw: Return tenant field in bucket_stats function (`pr#24895 <https://github.com/ceph/ceph/pull/24895>`_, Volker Theile)
+* rgw: return valid Location element, PostObj (`issue#22927 <http://tracker.ceph.com/issues/22927>`_, `pr#20330 <https://github.com/ceph/ceph/pull/20330>`_, yuliyang)
+* rgw: return x-amz-version-id: null when delete obj in versioning suspended bucket (`issue#35814 <http://tracker.ceph.com/issues/35814>`_, `pr#23927 <https://github.com/ceph/ceph/pull/23927>`_, yuliyang)
+* rgw: Revert "rgw: lifcycle: don't reject compound rules with empty prefix" (`pr#26491 <https://github.com/ceph/ceph/pull/26491>`_, Matt Benjamin)
+* rgw: rgw-admin: add "--trim-delay-ms" introduction for 'sync error trim' (`pr#23342 <https://github.com/ceph/ceph/pull/23342>`_, Enming.Zhang)
+* rgw: rgw-admin: fix data sync report for master zone (`pr#23925 <https://github.com/ceph/ceph/pull/23925>`_, cfanz)
+* rgw: RGWAsyncGetBucketInstanceInfo does not access coroutine memory (`issue#35812 <http://tracker.ceph.com/issues/35812>`_, `pr#23987 <https://github.com/ceph/ceph/pull/23987>`_, Casey Bodley)
+* rgw: rgw/beast: drop privileges after binding ports (`issue#36041 <http://tracker.ceph.com/issues/36041>`_, `pr#24271 <https://github.com/ceph/ceph/pull/24271>`_, Paul Emmerich)
+* rgw: RGWBucket::link supports tenant (`issue#22666 <http://tracker.ceph.com/issues/22666>`_, `pr#23119 <https://github.com/ceph/ceph/pull/23119>`_, Casey Bodley)
+* rgw:     rgw: change the way sysobj filters raw attributes, fix bucket sync state xattrs (`issue#37281 <http://tracker.ceph.com/issues/37281>`_, `pr#25123 <https://github.com/ceph/ceph/pull/25123>`_, Yehuda Sadeh)
+* rgw: rgw, cls: remove cls_statelog and rgw opstate tracking (`pr#24059 <https://github.com/ceph/ceph/pull/24059>`_, Casey Bodley)
+* rgw: rgw_file: deep stat handling (`issue#24915 <http://tracker.ceph.com/issues/24915>`_, `pr#23038 <https://github.com/ceph/ceph/pull/23038>`_, Matt Benjamin)
+* rgw: rgw_file: not check max_objects when creating file (`pr#24846 <https://github.com/ceph/ceph/pull/24846>`_, Tao Chen)
+* rgw: rgw_file: use correct secret key to check auth (`pr#26130 <https://github.com/ceph/ceph/pull/26130>`_, MinSheng Lin)
+* rgw: rgw_file: user info never synced since librgw init (`pr#25406 <https://github.com/ceph/ceph/pull/25406>`_, Tao Chen)
+* rgw: [rgw]: Fix help of radosgw-admin user info in case no uid (`pr#25078 <https://github.com/ceph/ceph/pull/25078>`_, Marc Koderer)
+* rgw: rgwgc:process coredump in some special case (`issue#23199 <http://tracker.ceph.com/issues/23199>`_, `pr#25430 <https://github.com/ceph/ceph/pull/25430>`_, zhaokun)
+* rgw: rgw multisite: async rados requests don't access coroutine memory (`issue#35543 <http://tracker.ceph.com/issues/35543>`_, `pr#23920 <https://github.com/ceph/ceph/pull/23920>`_, Casey Bodley)
+* rgw: rgw multisite: bucket sync transitions back to StateInit on OP_SYNCSTOP (`issue#26895 <http://tracker.ceph.com/issues/26895>`_, `pr#23574 <https://github.com/ceph/ceph/pull/23574>`_, Casey Bodley)
+* rgw: rgw multisite: enforce spawn_window for data full sync (`issue#26897 <http://tracker.ceph.com/issues/26897>`_, `pr#23534 <https://github.com/ceph/ceph/pull/23534>`_, Casey Bodley)
+* rgw: rgw-multisite: fix endless loop in RGWBucketShardIncrementalSyncCR (`issue#24603 <http://tracker.ceph.com/issues/24603>`_, `pr#22660 <https://github.com/ceph/ceph/pull/22660>`_, cfanz)
+* rgw: rgw multisite: incremental data sync uses truncated flag to detect end of listing (`issue#26952 <http://tracker.ceph.com/issues/26952>`_, `pr#23596 <https://github.com/ceph/ceph/pull/23596>`_, Casey Bodley)
+* rgw: rgw multisite: only update last_trim marker on ENODATA (`issue#38075 <http://tracker.ceph.com/issues/38075>`_, `pr#26190 <https://github.com/ceph/ceph/pull/26190>`_, Casey Bodley)
+* rgw: rgw multisite: uses local DataChangesLog to track active buckets for trim (`issue#36034 <http://tracker.ceph.com/issues/36034>`_, `pr#24221 <https://github.com/ceph/ceph/pull/24221>`_, Casey Bodley)
+* rgw: rgw/pubsub: add amqp push endpoint (`pr#25866 <https://github.com/ceph/ceph/pull/25866>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: add pubsub tests (`pr#26299 <https://github.com/ceph/ceph/pull/26299>`_, Yuval Lifshitz)
+* rgw: RGWRadosGetOmapKeysCR takes result by shared_ptr (`issue#21154 <http://tracker.ceph.com/issues/21154>`_, `pr#23634 <https://github.com/ceph/ceph/pull/23634>`_, Casey Bodley)
+* rgw: RGWRadosGetOmapKeysCR uses 'more' flag from omap_get_keys2() (`pr#23401 <https://github.com/ceph/ceph/pull/23401>`_, Casey Bodley, Sage Weil)
+* rgw: remove duplicate include header files in rgw_rados.cc (`pr#18578 <https://github.com/ceph/ceph/pull/18578>`_, Sibei Gao)
+* rgw: rgw_sync: drop ENOENT error logs from mdlog (`pr#26971 <https://github.com/ceph/ceph/pull/26971>`_, Abhishek Lekshmanan)
+* rgw: Robustly notify (`issue#24963 <http://tracker.ceph.com/issues/24963>`_, `pr#23100 <https://github.com/ceph/ceph/pull/23100>`_, Adam C. Emerson)
+* rgw: s3: awsv4 drop special handling for x-amz-credential (`issue#26965 <http://tracker.ceph.com/issues/26965>`_, `pr#23652 <https://github.com/ceph/ceph/pull/23652>`_, Abhishek Lekshmanan)
+* rgw: sanitize customer encryption keys from log output in v4 auth (`issue#37847 <http://tracker.ceph.com/issues/37847>`_, `pr#25881 <https://github.com/ceph/ceph/pull/25881>`_, Casey Bodley)
+* rgw: scheduler (`pr#26008 <https://github.com/ceph/ceph/pull/26008>`_, Casey Bodley, Abhishek Lekshmanan)
+* rgw: set cr state if aio_read err return in RGWCloneMetaLogCoroutine (`issue#24566 <http://tracker.ceph.com/issues/24566>`_, `pr#22617 <https://github.com/ceph/ceph/pull/22617>`_, Tianshan Qu)
+* rgw: set default objecter_inflight_ops = 24576 (`issue#25109 <http://tracker.ceph.com/issues/25109>`_, `pr#23242 <https://github.com/ceph/ceph/pull/23242>`_, Matt Benjamin)
+* rgw:  should recode  canonical_uri when caculate s3 v4 auth (`issue#23587 <http://tracker.ceph.com/issues/23587>`_, `pr#21286 <https://github.com/ceph/ceph/pull/21286>`_, yuliyang)
+* rgw: some fix for es sync (`issue#23842 <http://tracker.ceph.com/issues/23842>`_, `issue#23841 <http://tracker.ceph.com/issues/23841>`_, `pr#21622 <https://github.com/ceph/ceph/pull/21622>`_, Tianshan Qu, Shang Ding)
+* rgw: support admin rest api get user info through user's access-key (`pr#22790 <https://github.com/ceph/ceph/pull/22790>`_, yuliyang)
+* rgw: support server-side encryption when SSL is terminated in a proxy (`issue#27221 <http://tracker.ceph.com/issues/27221>`_, `pr#24700 <https://github.com/ceph/ceph/pull/24700>`_, Casey Bodley)
+* rgw: Swift SLO size_bytes member is optional (`issue#18936 <http://tracker.ceph.com/issues/18936>`_, `pr#22967 <https://github.com/ceph/ceph/pull/22967>`_, Matt Benjamin)
+* rgw: Swift's TempURL can handle temp_url_expires written in ISO8601 (`issue#20795 <http://tracker.ceph.com/issues/20795>`_, `pr#16658 <https://github.com/ceph/ceph/pull/16658>`_, Radoslaw Zarzynski)
+* rgw: sync module: avoid printing attrs of objects in log (`issue#37646 <http://tracker.ceph.com/issues/37646>`_, `pr#25541 <https://github.com/ceph/ceph/pull/25541>`_, Abhishek Lekshmanan)
+* rgw: test bi list (`issue#24483 <http://tracker.ceph.com/issues/24483>`_, `pr#21772 <https://github.com/ceph/ceph/pull/21772>`_, Orit Wasserman)
+* rgw: test/rgw: add ifdef for HAVE_BOOST_CONTEXT (`pr#25744 <https://github.com/ceph/ceph/pull/25744>`_, Casey Bodley)
+* rgw,tests: qa: add test for https://github.com/ceph/ceph/pull/22790 (`pr#23143 <https://github.com/ceph/ceph/pull/23143>`_, yuliyang)
+* rgw,tests: qa/rgw: add cls_lock/log/refcount/version tests to verify suite (`pr#25381 <https://github.com/ceph/ceph/pull/25381>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add missing import line (`pr#25298 <https://github.com/ceph/ceph/pull/25298>`_, Shilpa Jagannath)
+* rgw,tests: qa/rgw: add radosgw-admin-rest task to singleton suite (`pr#23145 <https://github.com/ceph/ceph/pull/23145>`_, Casey Bodley)
+* rgw,tests: qa/rgw: disable testing on ec-cache pools (`issue#23965 <http://tracker.ceph.com/issues/23965>`_, `pr#22126 <https://github.com/ceph/ceph/pull/22126>`_, Casey Bodley)
+* rgw,tests: qa/rgw: fix invalid syntax error in radosgw_admin_rest.py (`issue#37440 <http://tracker.ceph.com/issues/37440>`_, `pr#25305 <https://github.com/ceph/ceph/pull/25305>`_, Casey Bodley)
+* rgw,tests: qa/rgw: move ragweed upgrade test into upgrade/luminous-x (`pr#21707 <https://github.com/ceph/ceph/pull/21707>`_, Casey Bodley)
+* rgw,tests: qa/rgw: override valgrind --max-threads for radosgw (`issue#25214 <http://tracker.ceph.com/issues/25214>`_, `pr#23372 <https://github.com/ceph/ceph/pull/23372>`_, Casey Bodley)
+* rgw,tests: qa/rgw: patch keystone requirements.txt (`issue#23659 <http://tracker.ceph.com/issues/23659>`_, `pr#23402 <https://github.com/ceph/ceph/pull/23402>`_, Casey Bodley)
+* rgw,tests: qa/rgw: reduce number of multisite log shards (`pr#24011 <https://github.com/ceph/ceph/pull/24011>`_, Casey Bodley)
+* rgw,tests: qa/rgw: reorganize verify tasks (`pr#22249 <https://github.com/ceph/ceph/pull/22249>`_, Casey Bodley)
+* rgw,tests: qa/rgw/tempest: either force os_type or select random distro (`pr#25996 <https://github.com/ceph/ceph/pull/25996>`_, Yehuda Sadeh)
+* rgw,tests: test/rgw: fix for bucket checkpoints (`issue#24212 <http://tracker.ceph.com/issues/24212>`_, `pr#22124 <https://github.com/ceph/ceph/pull/22124>`_, Casey Bodley)
+* rgw,tests: test/rgw: fix race in test_rgw_reshard_wait (`pr#26741 <https://github.com/ceph/ceph/pull/26741>`_, Casey Bodley)
+* rgw,tests: test/rgw: silence -Wsign-compare warnings (`pr#26364 <https://github.com/ceph/ceph/pull/26364>`_, Kefu Chai)
+* rgw: The delete markers generated by object expiration should have owner attribute (`issue#24568 <http://tracker.ceph.com/issues/24568>`_, `pr#22619 <https://github.com/ceph/ceph/pull/22619>`_, Zhang Shaowen)
+* rgw: the error code returned by rgw is different from amz s3 when getting cors (`issue#26964 <http://tracker.ceph.com/issues/26964>`_, `pr#23646 <https://github.com/ceph/ceph/pull/23646>`_, ashitakasam)
+* rgw: thread DoutPrefixProvider into RGW::Auth_S3::authorize (`pr#24409 <https://github.com/ceph/ceph/pull/24409>`_, Ali Maredia)
+* rgw,tools: ceph-dencoder: add RGWRealm and RGWPeriod  support (`pr#25057 <https://github.com/ceph/ceph/pull/25057>`_, yuliyang)
+* rgw,tools: cls: refcount: add obj_refcount to ceph-dencoder (`pr#25441 <https://github.com/ceph/ceph/pull/25441>`_, Abhishek Lekshmanan)
+* rgw,tools: cls/rgw: ready rgw_usage_log_entry for extraction via ceph-dencoder (`issue#34537 <http://tracker.ceph.com/issues/34537>`_, `pr#22344 <https://github.com/ceph/ceph/pull/22344>`_, Vaibhav Bhembre)
+* rgw,tools: vstart: make beast as the default frontend for rgw (`pr#26566 <https://github.com/ceph/ceph/pull/26566>`_, Abhishek Lekshmanan)
+* rgw,tools: vstart: rgw: disable the lc debug interval option (`pr#25487 <https://github.com/ceph/ceph/pull/25487>`_, Abhishek Lekshmanan)
+* rgw,tools: vstart: set admin socket for RGW in conf (`pr#23983 <https://github.com/ceph/ceph/pull/23983>`_, Abhishek Lekshmanan)
+* rgw: update cls_rgw.cc and cls_rgw_const.h (`pr#24001 <https://github.com/ceph/ceph/pull/24001>`_, yuliyang)
+* rgw: update ObjectCacheInfo::time_added on overwrite (`issue#24346 <http://tracker.ceph.com/issues/24346>`_, `pr#22324 <https://github.com/ceph/ceph/pull/22324>`_, Casey Bodley)
+* rgw: update --url in usage and doc (`pr#22100 <https://github.com/ceph/ceph/pull/22100>`_, Jos Collin)
+* rgw: use chunked encoding to get partial results out faster (`issue#12713 <http://tracker.ceph.com/issues/12713>`_, `pr#23940 <https://github.com/ceph/ceph/pull/23940>`_, Robin H. Johnson)
+* rgw: use coarse_real_clock for req_state::time (`pr#21893 <https://github.com/ceph/ceph/pull/21893>`_, Casey Bodley)
+* rgw: use DoutPrefixProvider to add more context to log output (`pr#21700 <https://github.com/ceph/ceph/pull/21700>`_, Casey Bodley)
+* rgw: use partial-order bucket listing in RGWLC, add configurable processing delay (`issue#23956 <http://tracker.ceph.com/issues/23956>`_, `pr#21755 <https://github.com/ceph/ceph/pull/21755>`_, Matt Benjamin)
+* rgw: User Policy (`pr#21379 <https://github.com/ceph/ceph/pull/21379>`_, Pritha Srivastava)
+* rgw: user stats account for resharded buckets (`pr#24595 <https://github.com/ceph/ceph/pull/24595>`_, Casey Bodley)
+* rgw: warn if zone doesn't contain all zg's placement targets (`pr#22452 <https://github.com/ceph/ceph/pull/22452>`_, Abhishek Lekshmanan)
+* rgw: website routing rules num limit (`pr#23429 <https://github.com/ceph/ceph/pull/23429>`_, yuliyang)
+* rgw: when exclusive lock fails due existing lock, log add'l info (`issue#38171 <http://tracker.ceph.com/issues/38171>`_, `pr#26272 <https://github.com/ceph/ceph/pull/26272>`_, J. Eric Ivancich)
+* rgw: zone service only provides const access to its data (`pr#25412 <https://github.com/ceph/ceph/pull/25412>`_, Casey Bodley)
+* rocksdb: pick up a fix to be backward compatible (`issue#25146 <http://tracker.ceph.com/issues/25146>`_, `pr#25070 <https://github.com/ceph/ceph/pull/25070>`_, Kefu Chai)
+* script: build-integration-branch: avoid Unicode error (`issue#24003 <http://tracker.ceph.com/issues/24003>`_, `pr#21807 <https://github.com/ceph/ceph/pull/21807>`_, Nathan Cutler)
+* script/kubejacker: Add openSUSE based images (`pr#24055 <https://github.com/ceph/ceph/pull/24055>`_, Sebastian Wagner)
+* scripts: backport-create-issue: complain about duplicates and support mimic (`issue#24071 <http://tracker.ceph.com/issues/24071>`_, `pr#21634 <https://github.com/ceph/ceph/pull/21634>`_, Nathan Cutler)
+* seastar: pickup fix for segfault in POSIX stack (`pr#25861 <https://github.com/ceph/ceph/pull/25861>`_, Kefu Chai)
+* spec: add missing rbd mirror bootstrap directory (`pr#24856 <https://github.com/ceph/ceph/pull/24856>`_, Sébastien Han)
+* src: balance std::hex and std::dec manipulators (`pr#22287 <https://github.com/ceph/ceph/pull/22287>`_, Kefu Chai)
+* src/ceph.in: dev mode: add build path to beginning of PATH, not end (`issue#24578 <http://tracker.ceph.com/issues/24578>`_, `pr#22628 <https://github.com/ceph/ceph/pull/22628>`_, Dan Mick)
+* src: Eliminate new warnings in Fedora 28 (`pr#21898 <https://github.com/ceph/ceph/pull/21898>`_, Adam C. Emerson)
+* test/crimson: fixes of unittest_seastar_echo (`pr#26419 <https://github.com/ceph/ceph/pull/26419>`_, Yingxin Cheng, Kefu Chai)
+* test/fio: fix compiler failure (`pr#22728 <https://github.com/ceph/ceph/pull/22728>`_, Jianpeng Ma)
+* test/fio: new option to control file preallocation (`pr#23410 <https://github.com/ceph/ceph/pull/23410>`_, Igor Fedotov)
+* tests: Add hashinfo testing for dump command of ceph-objectstore-tool (`issue#38053 <http://tracker.ceph.com/issues/38053>`_, `pr#26158 <https://github.com/ceph/ceph/pull/26158>`_, David Zafman)
+* tests: add ubuntu 18.04 dockerfile (`pr#25251 <https://github.com/ceph/ceph/pull/25251>`_, Kefu Chai)
+* tests: auth, test: fix building on ARMs after the NSS -> OpenSSL transition (`pr#22129 <https://github.com/ceph/ceph/pull/22129>`_, Radoslaw Zarzynski)
+* tests: ceph_kvstorebench: include <errno.h> not asm-generic/errno.h (`pr#25256 <https://github.com/ceph/ceph/pull/25256>`_, Kefu Chai)
+* tests: ceph-volume: functional tests, add libvirt customization (`pr#25895 <https://github.com/ceph/ceph/pull/25895>`_, Jan Fajerski)
+* tests: do not check for invalid k/m combinations (`issue#16500 <http://tracker.ceph.com/issues/16500>`_, `pr#25046 <https://github.com/ceph/ceph/pull/25046>`_, Kefu Chai)
+* tests: Fixes for standalone tests (`pr#22480 <https://github.com/ceph/ceph/pull/22480>`_, David Zafman)
+* tests: fix to check server_conn in MessengerTest.NameAddrTest (`pr#23931 <https://github.com/ceph/ceph/pull/23931>`_, Yingxin)
+* tests: make ceph-admin-commands.sh log what it does (`issue#37089 <http://tracker.ceph.com/issues/37089>`_, `pr#25080 <https://github.com/ceph/ceph/pull/25080>`_, Nathan Cutler)
+* tests: make test_ceph_argparse.py pass on py3-only systems (`issue#24816 <http://tracker.ceph.com/issues/24816>`_, `pr#22922 <https://github.com/ceph/ceph/pull/22922>`_, Nathan Cutler)
+* tests: mgr/ansible: add install tox==2.9.1 (`pr#26313 <https://github.com/ceph/ceph/pull/26313>`_, Kefu Chai)
+* tests: mgr/dashboard: Added additional breadcrumb and tab tests to Cluster menu (`pr#26151 <https://github.com/ceph/ceph/pull/26151>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added additional breadcrumb tests to Cluster (`pr#25010 <https://github.com/ceph/ceph/pull/25010>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added breadcrumb and tab tests to Pools menu (`pr#25572 <https://github.com/ceph/ceph/pull/25572>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added breadcrumb tests to Block menu items (`pr#25143 <https://github.com/ceph/ceph/pull/25143>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added breadcrumb tests to Filesystems menu (`pr#26592 <https://github.com/ceph/ceph/pull/26592>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added NFS Ganesha suite to QA tests (`pr#26510 <https://github.com/ceph/ceph/pull/26510>`_, Laura Paduano)
+* tests: mgr/dashboard: Added tab tests to Block menu items (`pr#26243 <https://github.com/ceph/ceph/pull/26243>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Add Jest Runner (`pr#22031 <https://github.com/ceph/ceph/pull/22031>`_, Tiago Melo)
+* tests: mgr/dashboard: Add unit test case for controller/erasure_code_profile.py (`pr#24789 <https://github.com/ceph/ceph/pull/24789>`_, Ranjitha G)
+* tests: mgr/dashboard: Add unit test for frontend api services (`pr#22284 <https://github.com/ceph/ceph/pull/22284>`_, Tiago Melo)
+* tests: mgr/dashboard: Add unit tests for all frontend pipes (`pr#22182 <https://github.com/ceph/ceph/pull/22182>`_, Tiago Melo)
+* tests: mgr/dashboard: Add unit test to the frontend services (`pr#22244 <https://github.com/ceph/ceph/pull/22244>`_, Tiago Melo)
+* tests: mgr/dashboard: Fix a broken ECP controller test (`pr#25363 <https://github.com/ceph/ceph/pull/25363>`_, Zack Cerza)
+* tests: mgr/dashboard: Fix PYTHONPATH for test runner (`pr#25359 <https://github.com/ceph/ceph/pull/25359>`_, Zack Cerza)
+* tests: mgr/dashboard: Improve max-line-length tslint rule (`pr#22279 <https://github.com/ceph/ceph/pull/22279>`_, Tiago Melo)
+* tests: mgr/dashboard: RbdMirroringService test suite fails in dev mode (`issue#37841 <http://tracker.ceph.com/issues/37841>`_, `pr#25865 <https://github.com/ceph/ceph/pull/25865>`_, Stephan Müller)
+* tests: mgr/dashboard: Small improvements for running teuthology tests (`pr#25121 <https://github.com/ceph/ceph/pull/25121>`_, Zack Cerza)
+* tests: mgr/dashboard: updated API test (`pr#25653 <https://github.com/ceph/ceph/pull/25653>`_, Alfonso Martínez)
+* tests: mgr/dashboard: updated API test to reflect changes in ModuleInfo (`pr#25761 <https://github.com/ceph/ceph/pull/25761>`_, Kefu Chai)
+* tests: mgr/test_orchestrator: correct ceph-volume path (`issue#37773 <http://tracker.ceph.com/issues/37773>`_, `pr#25839 <https://github.com/ceph/ceph/pull/25839>`_, Kefu Chai)
+* tests: object errors found in be_select_auth_object() aren't logged the same (`issue#25108 <http://tracker.ceph.com/issues/25108>`_, `pr#23376 <https://github.com/ceph/ceph/pull/23376>`_, David Zafman)
+* tests: osd/OSDMap: set pg_autoscale_mode with setting from conf (`pr#25746 <https://github.com/ceph/ceph/pull/25746>`_, Kefu Chai)
+* tests: os/tests: fix garbageCollection test case from store_test suite (`pr#23752 <https://github.com/ceph/ceph/pull/23752>`_, Igor Fedotov)
+* tests: os/tests: silence -Wsign-compare warning (`pr#25072 <https://github.com/ceph/ceph/pull/25072>`_, Kefu Chai)
+* tests: qa: add librados3 to exclude_packages for ugprade tests (`pr#25037 <https://github.com/ceph/ceph/pull/25037>`_, Kefu Chai)
+* tests: qa: add test that builds example librados programs (`issue#35989 <http://tracker.ceph.com/issues/35989>`_, `issue#15100 <http://tracker.ceph.com/issues/15100>`_, `pr#23131 <https://github.com/ceph/ceph/pull/23131>`_, Nathan Cutler)
+* tests: qa/ceph-ansible: Set ceph_stable_release to mimic (`issue#38231 <http://tracker.ceph.com/issues/38231>`_, `pr#26328 <https://github.com/ceph/ceph/pull/26328>`_, Brad Hubbard)
+* tests: qa/distros: add openSUSE Leap 42.3 and 15.0 (`pr#24380 <https://github.com/ceph/ceph/pull/24380>`_, Nathan Cutler)
+* tests: qa: Don't use sudo when moving logs (`pr#22763 <https://github.com/ceph/ceph/pull/22763>`_, David Zafman)
+* tests: qa: downgrade librados2,librbd1 for thrash-old-clients tests (`issue#37618 <http://tracker.ceph.com/issues/37618>`_, `pr#25463 <https://github.com/ceph/ceph/pull/25463>`_, Kefu Chai)
+* tests: qa: fix manager module paths (`pr#23637 <https://github.com/ceph/ceph/pull/23637>`_, Noah Watkins, David Zafman)
+* tests/qa - fix mimic subset for nightlies (`pr#21931 <https://github.com/ceph/ceph/pull/21931>`_, Yuri Weinstein)
+* tests: qa: fix test on "ceph fs set cephfs allow_new_snaps" (`pr#21829 <https://github.com/ceph/ceph/pull/21829>`_, Kefu Chai)
+* tests: qa: fix upgrade tests and test_envlibrados_for_rocksdb.sh (`pr#25106 <https://github.com/ceph/ceph/pull/25106>`_, Kefu Chai)
+* tests: qa: For teuthology copy logs to teuthology expected location (`pr#22702 <https://github.com/ceph/ceph/pull/22702>`_, David Zafman)
+* tests: qa/mgr/dashboard: Fix type annotation error (`pr#25235 <https://github.com/ceph/ceph/pull/25235>`_, Sebastian Wagner)
+* tests: qa/mon: fix cluster support for monmap bootstrap (`issue#38115 <http://tracker.ceph.com/issues/38115>`_, `pr#26205 <https://github.com/ceph/ceph/pull/26205>`_, Casey Bodley)
+* tests: qa/standalone: Minor test improvements (`issue#35912 <http://tracker.ceph.com/issues/35912>`_, `pr#24018 <https://github.com/ceph/ceph/pull/24018>`_, David Zafman)
+* tests: qa/standalone/scrub: When possible show side-by-side diff in addition to regular diff (`pr#22727 <https://github.com/ceph/ceph/pull/22727>`_, David Zafman)
+* tests:  qa/standalone: Standalone test corrections (`issue#35982 <http://tracker.ceph.com/issues/35982>`_, `pr#24088 <https://github.com/ceph/ceph/pull/24088>`_, David Zafman)
+* tests: qa/suites/rados/upgrade: remove stray link (`pr#22460 <https://github.com/ceph/ceph/pull/22460>`_, Sage Weil)
+* tests: qa/suites/rados/upgrade: set require-osd-release to nautilus (`issue#37432 <http://tracker.ceph.com/issues/37432>`_, `pr#25314 <https://github.com/ceph/ceph/pull/25314>`_, Kefu Chai)
+* tests: qa/suites/rados/verify: remove random-distro$ (`pr#22057 <https://github.com/ceph/ceph/pull/22057>`_, Kefu Chai)
+* tests: qa/suites/upgrade/mimic-x: fix rhel runs (`pr#25781 <https://github.com/ceph/ceph/pull/25781>`_, Neha Ojha)
+* tests: qa/tasks/mgr: fix test_pool.py (`issue#24077 <http://tracker.ceph.com/issues/24077>`_, `pr#21943 <https://github.com/ceph/ceph/pull/21943>`_, Kefu Chai)
+* tests: qa/tasks/thrashosds-health.yaml: whitelist slow requests (`issue#25104 <http://tracker.ceph.com/issues/25104>`_, `pr#23237 <https://github.com/ceph/ceph/pull/23237>`_, Neha Ojha)
+* tests: qa/tasks: update mirror link for maven (`pr#23944 <https://github.com/ceph/ceph/pull/23944>`_, Vasu Kulkarni)
+* tests: qa/tests: added filters to support distro tests for client-upgrade tests (`pr#22096 <https://github.com/ceph/ceph/pull/22096>`_, Yuri Weinstein)
+* tests: qa/tests - added mimic-p2p suite (`pr#22726 <https://github.com/ceph/ceph/pull/22726>`_, Yuri Weinstein)
+* tests: qa/tests: Added mimic runs, removed large suites (rados, rbd, etc) ru… (`pr#21827 <https://github.com/ceph/ceph/pull/21827>`_, Yuri Weinstein)
+* tests: qa/tests: added "-n 7" to make sure mimic-x runs on built master branch (`pr#25038 <https://github.com/ceph/ceph/pull/25038>`_, Yuri Weinstein)
+* tests: qa/tests: added rhel 7.6 (`pr#25919 <https://github.com/ceph/ceph/pull/25919>`_, Yuri Weinstein)
+* tests: qa/tests: fix volume size when running in ovh (`pr#21961 <https://github.com/ceph/ceph/pull/21961>`_, Vasu Kulkarni)
+* tests: qa/tests: Move ceph-ansible tests to ansible version 2.7 (`issue#37973 <http://tracker.ceph.com/issues/37973>`_, `pr#26068 <https://github.com/ceph/ceph/pull/26068>`_, Brad Hubbard)
+* tests: qa/tests: remove ceph-disk tests from ceph-deploy and default all tests to use ceph-volume (`pr#22921 <https://github.com/ceph/ceph/pull/22921>`_, Vasu Kulkarni)
+* tests: qa/upgrade: cleanup for nautilus (`pr#23305 <https://github.com/ceph/ceph/pull/23305>`_, Nathan Cutler)
+* tests: qa: use $TESTDIR for testing mkfs (`pr#22246 <https://github.com/ceph/ceph/pull/22246>`_, Kefu Chai)
+* tests: qa: wait longer for osd to flush pg stats (`issue#24321 <http://tracker.ceph.com/issues/24321>`_, `pr#22275 <https://github.com/ceph/ceph/pull/22275>`_, Kefu Chai)
+* tests: qa/workunits/ceph-disk: --no-mon-config (`pr#21942 <https://github.com/ceph/ceph/pull/21942>`_, Kefu Chai)
+* tests: qa/workunits/mon/test_mon_config_key.py: bump up the size limit (`issue#36260 <http://tracker.ceph.com/issues/36260>`_, `pr#24340 <https://github.com/ceph/ceph/pull/24340>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_envlibrados_for_rocksdb: install g++ not g++-4.7 (`pr#22103 <https://github.com/ceph/ceph/pull/22103>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_librados_build.sh: grab files from explicit git branch (`pr#25268 <https://github.com/ceph/ceph/pull/25268>`_, Nathan Cutler)
+* tests: run-make-check: increase fs.aio-max-nr to 1048576 (`pr#23689 <https://github.com/ceph/ceph/pull/23689>`_, Kefu Chai)
+* tests: test,common: silence GCC warnings (`pr#23692 <https://github.com/ceph/ceph/pull/23692>`_, Kefu Chai)
+* tests: test/crimson: add dummy_auth to test_async_echo (`pr#26783 <https://github.com/ceph/ceph/pull/26783>`_, Yingxin Cheng)
+* tests: test/crimson: fix build failure of test_alien_echo (`pr#26308 <https://github.com/ceph/ceph/pull/26308>`_, chunmei Liu)
+* tests: test/crimson: fix FTBFS of unittest_seastar_perfcounters on arm64 (`pr#25647 <https://github.com/ceph/ceph/pull/25647>`_, Kefu Chai)
+* tests: test/crimson: split async-msgr out of alien_echo (`pr#26620 <https://github.com/ceph/ceph/pull/26620>`_, Yingxin Cheng)
+* tests: test/dashboard: fix segfault when importing dm.xmlsec.binding (`issue#37081 <http://tracker.ceph.com/issues/37081>`_, `pr#25139 <https://github.com/ceph/ceph/pull/25139>`_, Kefu Chai)
+* tests: test: Disable duplicate request command test during scrub testing (`pr#25675 <https://github.com/ceph/ceph/pull/25675>`_, David Zafman)
+* tests: test/docker-test-helper.sh: move "cp .git/HEAD" out of loop (`pr#22978 <https://github.com/ceph/ceph/pull/22978>`_, Kefu Chai)
+* tests: test/encoding: Fix typo in encoding/types.h file (`pr#22332 <https://github.com/ceph/ceph/pull/22332>`_, TommyLike)
+* tests: test/fio:  pass config params to object store in a different manner (`pr#23267 <https://github.com/ceph/ceph/pull/23267>`_, Igor Fedotov)
+* tests: test: fix compile error in test/crimson/test_config.cc (`pr#23724 <https://github.com/ceph/ceph/pull/23724>`_, Yingxin)
+* tests: test: fix libc++ crash in Log.GarbleRecovery (`pr#25135 <https://github.com/ceph/ceph/pull/25135>`_, Casey Bodley)
+* tests: test/librados: fix LibRadosList.ListObjectsNS (`pr#22771 <https://github.com/ceph/ceph/pull/22771>`_, Kefu Chai)
+* tests: test: Limit loops waiting for force-backfill/force-recovery to happen (`issue#38309 <http://tracker.ceph.com/issues/38309>`_, `pr#26416 <https://github.com/ceph/ceph/pull/26416>`_, David Zafman)
+* tests: test: Need to escape parens in log-whitelist for grep (`pr#22074 <https://github.com/ceph/ceph/pull/22074>`_, David Zafman)
+* tests: test: osd-backfill-stats.sh Fix check of multi backfill OSDs, skip re… (`pr#26330 <https://github.com/ceph/ceph/pull/26330>`_, David Zafman)
+* tests: test/pybind/test_rados.py: collect output in stdout for "bench" cmd (`pr#21957 <https://github.com/ceph/ceph/pull/21957>`_, Kefu Chai)
+* tests: test: run-standalone.sh: point LD_LIBRARY_PATH to $(pwd)/lib (`issue#38262 <http://tracker.ceph.com/issues/38262>`_, `pr#26371 <https://github.com/ceph/ceph/pull/26371>`_, David Zafman)
+* tests: tests/qa: trying $ distro mix (`pr#21895 <https://github.com/ceph/ceph/pull/21895>`_, Yuri Weinstein)
+* tests: test: Start using GNU awk and fix archiving directory (`pr#23955 <https://github.com/ceph/ceph/pull/23955>`_, Willem Jan Withagen)
+* tests: test/strtol: add test case for parsing hex numbers (`pr#21582 <https://github.com/ceph/ceph/pull/21582>`_, Jan Fajerski)
+* tests: test: suppress core dumping in there tests as well (`pr#25311 <https://github.com/ceph/ceph/pull/25311>`_, Willem Jan Withagen)
+* tests: test: switch to GNU sed on FreeBSD (`pr#26318 <https://github.com/ceph/ceph/pull/26318>`_, Willem Jan Withagen)
+* tests: test: test_get_timeout_delays() fix (`pr#22837 <https://github.com/ceph/ceph/pull/22837>`_, David Zafman)
+* tests: test: Use a file that should be on all OSes (`pr#22428 <https://github.com/ceph/ceph/pull/22428>`_, David Zafman)
+* tests: test: Use a grep pattern that works across releases (`issue#35845 <http://tracker.ceph.com/issues/35845>`_, `pr#24013 <https://github.com/ceph/ceph/pull/24013>`_, David Zafman)
+* tests: test: Use pids instead of jobspecs which were wrong (`issue#27056 <http://tracker.ceph.com/issues/27056>`_, `pr#23695 <https://github.com/ceph/ceph/pull/23695>`_, David Zafman)
+* tests: test: wait_for_pg_stats() should do another check after last 13 secon… (`pr#22198 <https://github.com/ceph/ceph/pull/22198>`_, David Zafman)
+* tests: test: Whitelist corrections (`pr#22164 <https://github.com/ceph/ceph/pull/22164>`_, David Zafman)
+* tests: test: write log file to current directory (`issue#36737 <http://tracker.ceph.com/issues/36737>`_, `pr#25704 <https://github.com/ceph/ceph/pull/25704>`_, Kefu Chai)
+* tests,tools: ceph-objectstore-tool: Dump hashinfo (`issue#37597 <http://tracker.ceph.com/issues/37597>`_, `pr#25483 <https://github.com/ceph/ceph/pull/25483>`_, David Zafman)
+* tests: update Dockerfile to support fc-29 (`pr#26311 <https://github.com/ceph/ceph/pull/26311>`_, Kefu Chai)
+* tests: upgrade/luminous-x: fix order of final-workload directory (`pr#23162 <https://github.com/ceph/ceph/pull/23162>`_, Nathan Cutler)
+* tests: upgrade/luminous-x: whitelist REQUEST_SLOW for rados_mon_thrash (`issue#25051 <http://tracker.ceph.com/issues/25051>`_, `pr#23160 <https://github.com/ceph/ceph/pull/23160>`_, Nathan Cutler)
+* tests: Wip 38027 38195: osd/osd-backfill-space.sh fails (`issue#38027 <http://tracker.ceph.com/issues/38027>`_, `issue#38195 <http://tracker.ceph.com/issues/38195>`_, `pr#26290 <https://github.com/ceph/ceph/pull/26290>`_, David Zafman)
+* tools: Add clear-data-digest command to objectstore tool (`pr#25403 <https://github.com/ceph/ceph/pull/25403>`_, Li Yichao)
+* tools: add offset-align option to "rados" load-gen (`pr#20683 <https://github.com/ceph/ceph/pull/20683>`_, Zengran Zhang)
+* tools: backport-create-issue: rate-limit to avoid seeming like a spammer (`pr#24243 <https://github.com/ceph/ceph/pull/24243>`_, Nathan Cutler)
+* tools: ceph-menv: mrun shell environment (`pr#22132 <https://github.com/ceph/ceph/pull/22132>`_, Yehuda Sadeh)
+* tools: ceph-objectstore-tool: Allow target level as first positional argument (`issue#35846 <http://tracker.ceph.com/issues/35846>`_, `pr#23989 <https://github.com/ceph/ceph/pull/23989>`_, David Zafman)
+* tools: correct the description of Allowed options in osdomap tool (`pr#23488 <https://github.com/ceph/ceph/pull/23488>`_, xiaomanh)
+* tools, mgr: silence clang warnings (`pr#23430 <https://github.com/ceph/ceph/pull/23430>`_, Kefu Chai)
+* tools: mstop.sh allow kill -9 after failing to kill procs (`pr#26680 <https://github.com/ceph/ceph/pull/26680>`_, Yuval Lifshitz)
+* tools/rados: fix memory leak in error path (`pr#25410 <https://github.com/ceph/ceph/pull/25410>`_, Li Wang)
+* tools: script/kubejacker: include cls libs (`pr#23569 <https://github.com/ceph/ceph/pull/23569>`_, John Spray)
+* tools: script: new ceph-backport.sh script (`pr#22875 <https://github.com/ceph/ceph/pull/22875>`_, Nathan Cutler)
+* tools:  tools: ceph-authtool: report correct number of caps when creating keyring (`pr#23304 <https://github.com/ceph/ceph/pull/23304>`_, Nathan Cutler)
+* tools: tools/ceph_kvstore_tool: do not open rocksdb when repairing it (`pr#25108 <https://github.com/ceph/ceph/pull/25108>`_, Kefu Chai)
+* tools: tools/ceph_kvstore_tool: extract StoreTool into kvstore_tool.cc (`pr#26041 <https://github.com/ceph/ceph/pull/26041>`_, Kefu Chai)
+* tools: tools/ceph_kvstore_tool: Move summary output to print_summary (`pr#26666 <https://github.com/ceph/ceph/pull/26666>`_, Brad Hubbard)
+* tools: tools/rados: allow list objects in a specific pg in a pool (`pr#19041 <https://github.com/ceph/ceph/pull/19041>`_, Li Wang)
+* tools: tools/rados: always call rados.shutdown() before exit() (`issue#36732 <http://tracker.ceph.com/issues/36732>`_, `pr#24990 <https://github.com/ceph/ceph/pull/24990>`_, Li Wang)
+* tools: tools/rados: correct the read offset of bench (`pr#23667 <https://github.com/ceph/ceph/pull/23667>`_, Xiaofei Cui)
+* tools: tools/rados: fix the unit of target-throughput (`pr#23683 <https://github.com/ceph/ceph/pull/23683>`_, Xiaofei Cui)
+* vstart: disable dashboard when rbd not built (`pr#23336 <https://github.com/ceph/ceph/pull/23336>`_, Noah Watkins)
+* vstart.sh: fix params generation for monmaptool (`issue#38174 <http://tracker.ceph.com/issues/38174>`_, `pr#26273 <https://github.com/ceph/ceph/pull/26273>`_, Yehuda Sadeh)
diff --git a/doc/releases/octopus.rst b/doc/releases/octopus.rst
new file mode 100644
index 000000000..eb2e7d779
--- /dev/null
+++ b/doc/releases/octopus.rst
@@ -0,0 +1,6137 @@
+=======
+Octopus
+=======
+
+Octopus is the 15th stable release of Ceph.  It is named after an
+order of 8-limbed cephalopods.
+
+v15.2.17 Octopus
+================
+
+This is the 17th and final backport release in the Octopus series. We recommend
+all users update to this release.
+
+Notable Changes
+---------------
+
+* Octopus modified the SnapMapper key format from
+  <LEGACY_MAPPING_PREFIX><snapid>_<shardid>_<hobject_t::to_str()>
+  to
+  <MAPPING_PREFIX><pool>_<snapid>_<shardid>_<hobject_t::to_str()>
+  When this change was introduced, 94ebe0e also introduced a conversion
+  with a crucial bug which essentially destroyed legacy keys by mapping them
+  to
+  <MAPPING_PREFIX><poolid>_<snapid>_
+  without the object-unique suffix. The conversion is fixed in this release.
+  Relevant tracker: https://tracker.ceph.com/issues/5614
+
+* The ability to blend all RBD pools together into a single view by invoking
+  "rbd perf image iostat" or "rbd perf image iotop" commands without any options
+  or positional arguments is resurrected.  Such invocations accidentally became
+  limited to just the default pool (``rbd_default_pool``) in v15.2.14.
+
+* Users who were running OpenStack Manila to export native CephFS, who
+  upgraded their Ceph cluster from Nautilus (or earlier) to a later
+  major version, were vulnerable to an attack by malicious users
+  (:ref:`CVE-2022-0670`). The vulnerability allowed users to obtain
+  access to arbitrary portions of the CephFS filesystem hierarchy,
+  instead of being properly restricted to their own subvolumes. The
+  vulnerability is due to a bug in the "volumes" plugin in Ceph Manager.
+  This plugin is responsible for managing Ceph File System subvolumes
+  which are used by OpenStack Manila services as a way to provide shares
+  to Manila users.
+
+  With this release, the vulnerability is fixed. Administrators who are
+  concerned they may have been impacted should audit the CephX keys in
+  their cluster for proper path restrictions.
+
+  Again, this vulnerability only impacts OpenStack Manila clusters which
+  provided native CephFS access to their users.
+
+Changelog
+---------------
+
+* admin/doc-requirements: bump sphinx to 4.4.0 (`pr#45972 <https://github.com/ceph/ceph/pull/45972>`_, Kefu Chai)
+* backport qemu-iotests fixup for centos stream 8 (`pr#45206 <https://github.com/ceph/ceph/pull/45206>`_, Ken Dreyer, Ilya Dryomov)
+* Catch exception if thrown by __generate_command_map() (`pr#45891 <https://github.com/ceph/ceph/pull/45891>`_, Nikhil Kshirsagar)
+* ceph-volume: abort when passed devices have partitions (`pr#45147 <https://github.com/ceph/ceph/pull/45147>`_, Guillaume Abrioux)
+* ceph-volume: fix error 'KeyError' with inventory (`pr#44883 <https://github.com/ceph/ceph/pull/44883>`_, Guillaume Abrioux)
+* ceph-volume: fix tags dict output in `lvm list` (`pr#44768 <https://github.com/ceph/ceph/pull/44768>`_, Guillaume Abrioux)
+* ceph-volume: zap osds in rollback_osd() (`pr#44770 <https://github.com/ceph/ceph/pull/44770>`_, Guillaume Abrioux)
+* ceph/admin: s/master/main (`pr#46219 <https://github.com/ceph/ceph/pull/46219>`_, Zac Dover)
+* cephadm: infer the default container image during pull (`pr#45570 <https://github.com/ceph/ceph/pull/45570>`_, Michael Fritch)
+* cephadm: preserve `authorized_keys` file during upgrade (`pr#45356 <https://github.com/ceph/ceph/pull/45356>`_, Michael Fritch)
+* client: do not dump mds twice in Inode::dump() (`pr#45162 <https://github.com/ceph/ceph/pull/45162>`_, Xue Yantao)
+* cls/rbd: GroupSnapshotNamespace comparator violates ordering rules (`pr#45076 <https://github.com/ceph/ceph/pull/45076>`_, Ilya Dryomov)
+* cls/rgw: rgw_dir_suggest_changes detects race with completion (`pr#45902 <https://github.com/ceph/ceph/pull/45902>`_, Casey Bodley)
+* cmake: pass RTE_DEVEL_BUILD=n when building dpdk (`pr#45261 <https://github.com/ceph/ceph/pull/45261>`_, Kefu Chai)
+* common: avoid pthread_mutex_unlock twice (`pr#45465 <https://github.com/ceph/ceph/pull/45465>`_, Dai Zhiwei)
+* common: replace BitVector::NoInitAllocator with wrapper struct (`pr#45180 <https://github.com/ceph/ceph/pull/45180>`_, Casey Bodley)
+* crush: cancel upmaps with up set size != pool size (`pr#43416 <https://github.com/ceph/ceph/pull/43416>`_, huangjun)
+* doc/dev: update basic-workflow.rst (`pr#46308 <https://github.com/ceph/ceph/pull/46308>`_, Zac Dover)
+* doc/start: s/3/three/ in intro.rst (`pr#46328 <https://github.com/ceph/ceph/pull/46328>`_, Zac Dover)
+* doc/start: update "memory" in hardware-recs.rst (`pr#46451 <https://github.com/ceph/ceph/pull/46451>`_, Zac Dover)
+* Fixes for make check (`pr#46230 <https://github.com/ceph/ceph/pull/46230>`_, Kefu Chai, Adam C. Emerson)
+* krbd: return error when no initial monitor address found (`pr#45004 <https://github.com/ceph/ceph/pull/45004>`_, Burt Holzman)
+* librados: check latest osdmap on ENOENT in pool_reverse_lookup() (`pr#45587 <https://github.com/ceph/ceph/pull/45587>`_, Ilya Dryomov)
+* librbd: bail from schedule_request_lock() if already lock owner (`pr#47160 <https://github.com/ceph/ceph/pull/47160>`_, Christopher Hoffman)
+* librbd: fix use-after-free on ictx in list_descendants() (`pr#45000 <https://github.com/ceph/ceph/pull/45000>`_, Ilya Dryomov, Wang ShuaiChao)
+* librbd: honor FUA op flag for write_same() in write-around cache (`pr#44992 <https://github.com/ceph/ceph/pull/44992>`_, Ilya Dryomov)
+* librbd: readv/writev fix iovecs length computation overflow (`pr#45560 <https://github.com/ceph/ceph/pull/45560>`_, Jonas Pfefferle)
+* librbd: track complete async operation requests (`pr#45019 <https://github.com/ceph/ceph/pull/45019>`_, Mykola Golub)
+* librbd: unlink newest mirror snapshot when at capacity, bump capacity (`pr#46592 <https://github.com/ceph/ceph/pull/46592>`_, Ilya Dryomov)
+* librbd: update progress for non-existent objects on deep-copy (`pr#46912 <https://github.com/ceph/ceph/pull/46912>`_, Ilya Dryomov)
+* librgw: make rgw file handle versioned (`pr#45496 <https://github.com/ceph/ceph/pull/45496>`_, Xuehan Xu)
+* mds: add heartbeat_reset() in start_files_to_reover() (`pr#45157 <https://github.com/ceph/ceph/pull/45157>`_, Yongseok Oh)
+* mds: check rejoin_ack_gather before enter rejoin_gather_finish (`pr#45161 <https://github.com/ceph/ceph/pull/45161>`_, chencan)
+* mds: directly return just after responding the link request (`pr#44624 <https://github.com/ceph/ceph/pull/44624>`_, Xiubo Li)
+* mds: ensure that we send the btime in cap messages (`pr#45164 <https://github.com/ceph/ceph/pull/45164>`_, Jeff Layton)
+* mds: fix possible mds_lock not locked assert (`pr#45156 <https://github.com/ceph/ceph/pull/45156>`_, Xiubo Li)
+* mds: fix seg fault in expire_recursive (`pr#45055 <https://github.com/ceph/ceph/pull/45055>`_, 胡玮文)
+* mds: ignore unknown client op when tracking op latency (`pr#44976 <https://github.com/ceph/ceph/pull/44976>`_, Venky Shankar)
+* mds: mds_oft_prefetch_dirfrags default to false (`pr#45015 <https://github.com/ceph/ceph/pull/45015>`_, Dan van der Ster)
+* mds: progress the recover queue immediately after the inode is enqueued (`pr#45158 <https://github.com/ceph/ceph/pull/45158>`_, "Yan, Zheng", Xiubo Li)
+* mds: reset the return value for heap command (`pr#45155 <https://github.com/ceph/ceph/pull/45155>`_, Xiubo Li)
+* mds: skip directory size checks for reintegration (`pr#44668 <https://github.com/ceph/ceph/pull/44668>`_, Patrick Donnelly)
+* mgr/cephadm: fix and improve osd draining (`pr#46645 <https://github.com/ceph/ceph/pull/46645>`_, Sage Weil)
+* mgr/cephadm: try to get FQDN for active instance (`pr#46787 <https://github.com/ceph/ceph/pull/46787>`_, Tatjana Dehler)
+* mgr/cephadm: try to get FQDN for configuration files (`pr#45621 <https://github.com/ceph/ceph/pull/45621>`_, Tatjana Dehler)
+* mgr/dashboard: dashboard turns telemetry off when configuring report (`pr#45110 <https://github.com/ceph/ceph/pull/45110>`_, Sarthak0702, Aaryan Porwal)
+* mgr/dashboard: fix "NullInjectorError: No provider for I18n (`pr#45613 <https://github.com/ceph/ceph/pull/45613>`_, Nizamudeen A)
+* mgr/dashboard: fix Grafana OSD/host panels (`pr#44924 <https://github.com/ceph/ceph/pull/44924>`_, Patrick Seidensal)
+* mgr/dashboard: Notification banners at the top of the UI have fixed height (`pr#44763 <https://github.com/ceph/ceph/pull/44763>`_, Waad AlKhoury)
+* mgr/dashboard: Table columns hiding fix (`issue#51119 <http://tracker.ceph.com/issues/51119>`_, `pr#45726 <https://github.com/ceph/ceph/pull/45726>`_, Daniel Persson)
+* mgr/devicehealth: fix missing timezone from time delta calculation (`pr#45287 <https://github.com/ceph/ceph/pull/45287>`_, Yaarit Hatuka)
+* mgr/prometheus: Added `avail_raw` field for Pools DF Prometheus mgr module (`pr#45238 <https://github.com/ceph/ceph/pull/45238>`_, Konstantin Shalygin)
+* mgr/rbd_support: cast pool_id from int to str when collecting LevelSpec (`pr#45530 <https://github.com/ceph/ceph/pull/45530>`_, Ilya Dryomov)
+* mgr/rbd_support: fix schedule remove (`pr#45006 <https://github.com/ceph/ceph/pull/45006>`_, Sunny Kumar)
+* mgr/telemetry: fix waiting for mgr to warm up (`pr#45772 <https://github.com/ceph/ceph/pull/45772>`_, Yaarit Hatuka)
+* mgr/volumes: A few volumes plugin backport (`issue#51271 <http://tracker.ceph.com/issues/51271>`_, `pr#44800 <https://github.com/ceph/ceph/pull/44800>`_, Kotresh HR, Venky Shankar, Jan Fajerski)
+* mgr/volumes: Fix permission during subvol creation with mode (`pr#43224 <https://github.com/ceph/ceph/pull/43224>`_, Kotresh HR)
+* mgr/volumes: Fix subvolume discover during upgrade (`pr#47236 <https://github.com/ceph/ceph/pull/47236>`_, Kotresh HR)
+* mgr: limit changes to pg_num (`pr#44541 <https://github.com/ceph/ceph/pull/44541>`_, Sage Weil)
+* mirror snapshot schedule and trash purge schedule fixes (`pr#46777 <https://github.com/ceph/ceph/pull/46777>`_, Ilya Dryomov)
+* mon/MonCommands.h: fix target_size_ratio range (`pr#45398 <https://github.com/ceph/ceph/pull/45398>`_, Kamoltat)
+* mon: Abort device health when device not found (`pr#44960 <https://github.com/ceph/ceph/pull/44960>`_, Benoît Knecht)
+* octopus rgw: on FIPS enabled, fix segfault performing s3 multipart PUT (`pr#46701 <https://github.com/ceph/ceph/pull/46701>`_, Mark Kogan)
+* octopus rgw: under fips, set flag to allow md5 in select rgw ops (`pr#44806 <https://github.com/ceph/ceph/pull/44806>`_, Mark Kogan)
+* os/bluestore: Always update the cursor position in AVL near-fit search (`pr#46687 <https://github.com/ceph/ceph/pull/46687>`_, Mark Nelson)
+* osd/OSD: Log aggregated slow ops detail to cluster logs (`pr#45154 <https://github.com/ceph/ceph/pull/45154>`_, Prashant D)
+* osd/OSD: osd_fast_shutdown_notify_mon not quite right (`pr#45655 <https://github.com/ceph/ceph/pull/45655>`_, Nitzan Mordechai, Satoru Takeuchi)
+* osd/OSDMap: Add health warning if 'require-osd-release' != current release (`pr#44260 <https://github.com/ceph/ceph/pull/44260>`_, Sridhar Seshasayee)
+* osd/OSDMapMapping: fix spurious threadpool timeout errors (`pr#44546 <https://github.com/ceph/ceph/pull/44546>`_, Sage Weil)
+* osd/PGLog.cc: Trim duplicates by number of entries (`pr#46253 <https://github.com/ceph/ceph/pull/46253>`_, Nitzan Mordechai)
+* osd/PrimaryLogPG.cc: CEPH_OSD_OP_OMAPRMKEYRANGE should mark omap dirty (`pr#45593 <https://github.com/ceph/ceph/pull/45593>`_, Neha Ojha)
+* osd/SnapMapper: fix pacific legacy key conversion and introduce test (`pr#47108 <https://github.com/ceph/ceph/pull/47108>`_, Manuel Lausch, Matan Breizman)
+* osd: log the number of 'dups' entries in a PG Log (`pr#46609 <https://github.com/ceph/ceph/pull/46609>`_, Radoslaw Zarzynski)
+* osd: require osd_pg_max_concurrent_snap_trims > 0 (`pr#45324 <https://github.com/ceph/ceph/pull/45324>`_, Dan van der Ster)
+* qa/rgw: add failing tempest test to blocklist (`pr#45437 <https://github.com/ceph/ceph/pull/45437>`_, Casey Bodley)
+* qa/rgw: update apache-maven mirror for rgw/hadoop-s3a (`pr#45446 <https://github.com/ceph/ceph/pull/45446>`_, Casey Bodley)
+* qa/suites/rados/thrash-erasure-code-big/thrashers: add `osd max backfills` setting to mapgap and pggrow (`pr#46392 <https://github.com/ceph/ceph/pull/46392>`_, Laura Flores)
+* qa/suites: clean up client-upgrade-octopus-pacific test (`pr#45334 <https://github.com/ceph/ceph/pull/45334>`_, Ilya Dryomov)
+* qa/tasks/qemu: make sure block-rbd.so is installed (`pr#45071 <https://github.com/ceph/ceph/pull/45071>`_, Ilya Dryomov)
+* qa/tasks: teuthology octopus backport (`pr#46149 <https://github.com/ceph/ceph/pull/46149>`_, Kefu Chai, Shraddha Agrawal)
+* qa/tests: added upgrade-clients/client-upgrade-octopus-quincy tests (`pr#45282 <https://github.com/ceph/ceph/pull/45282>`_, Yuri Weinstein)
+* qa: always format the pgid in hex (`pr#45159 <https://github.com/ceph/ceph/pull/45159>`_, Xiubo Li)
+* qa: check mounts attribute in ctx (`pr#45633 <https://github.com/ceph/ceph/pull/45633>`_, Jos Collin)
+* qa: remove .teuthology_branch file (`pr#46489 <https://github.com/ceph/ceph/pull/46489>`_, Jeff Layton)
+* radosgw-admin: 'reshard list' doesn't log ENOENT errors (`pr#45452 <https://github.com/ceph/ceph/pull/45452>`_, Casey Bodley)
+* radosgw-admin: 'sync status' is not behind if there are no mdlog entries (`pr#45443 <https://github.com/ceph/ceph/pull/45443>`_, Casey Bodley)
+* radosgw-admin: skip GC init on read-only admin ops (`pr#45423 <https://github.com/ceph/ceph/pull/45423>`_, Mark Kogan)
+* rbd-fuse: librados will filter out -r option from command-line (`pr#46952 <https://github.com/ceph/ceph/pull/46952>`_, wanwencong)
+* rbd-mirror: don't prune non-primary snapshot when restarting delta sync (`pr#46589 <https://github.com/ceph/ceph/pull/46589>`_, Ilya Dryomov)
+* rbd-mirror: generally skip replay/resync if remote image is not primary (`pr#46812 <https://github.com/ceph/ceph/pull/46812>`_, Ilya Dryomov)
+* rbd-mirror: make mirror properly detect pool replayer needs restart (`pr#45169 <https://github.com/ceph/ceph/pull/45169>`_, Mykola Golub)
+* rbd-mirror: remove bogus completed_non_primary_snapshots_exist check (`pr#47117 <https://github.com/ceph/ceph/pull/47117>`_, Ilya Dryomov)
+* rbd-mirror: synchronize with in-flight stop in ImageReplayer::stop() (`pr#45177 <https://github.com/ceph/ceph/pull/45177>`_, Ilya Dryomov)
+* rbd: don't default empty pool name unless namespace is specified (`pr#47142 <https://github.com/ceph/ceph/pull/47142>`_, Ilya Dryomov)
+* rbd: mark optional positional arguments as such in help output (`pr#45009 <https://github.com/ceph/ceph/pull/45009>`_, Ilya Dryomov, Jason Dillaman)
+* rbd: recognize rxbounce map option (`pr#45001 <https://github.com/ceph/ceph/pull/45001>`_, Ilya Dryomov)
+* Revert "rocksdb: do not use non-zero recycle_log_file_num setting" (`pr#47053 <https://github.com/ceph/ceph/pull/47053>`_, Laura Flores)
+* revert of #46253, add tools: ceph-objectstore-tool is able to trim solely pg log dups' entries (`pr#46611 <https://github.com/ceph/ceph/pull/46611>`_, Radosław Zarzyński, Radoslaw Zarzynski)
+* rgw/amqp: add default case to silence compiler warning (`pr#45479 <https://github.com/ceph/ceph/pull/45479>`_, Casey Bodley)
+* rgw: add the condition of lock mode conversion to PutObjRentention (`pr#45441 <https://github.com/ceph/ceph/pull/45441>`_, wangzhong)
+* rgw: bucket chown bad memory usage (`pr#45492 <https://github.com/ceph/ceph/pull/45492>`_, Mohammad Fatemipour)
+* rgw: change order of xml elements in ListRoles response (`pr#45449 <https://github.com/ceph/ceph/pull/45449>`_, Casey Bodley)
+* rgw: cls_bucket_list_unordered() might return one redundent entry every time is_truncated is true (`pr#45458 <https://github.com/ceph/ceph/pull/45458>`_, Peng Zhang)
+* rgw: document rgw_lc_debug_interval configuration option (`pr#45454 <https://github.com/ceph/ceph/pull/45454>`_, J. Eric Ivancich)
+* rgw: document S3 bucket replication support (`pr#45485 <https://github.com/ceph/ceph/pull/45485>`_, Matt Benjamin)
+* rgw: Dump Object Lock Retain Date as ISO 8601 (`pr#43656 <https://github.com/ceph/ceph/pull/43656>`_, Preben Berg)
+* rgw: fix leak of RGWBucketList memory (octopus only) (`pr#45283 <https://github.com/ceph/ceph/pull/45283>`_, Casey Bodley)
+* rgw: fix md5 not match for RGWBulkUploadOp upload when enable rgw com… (`pr#45433 <https://github.com/ceph/ceph/pull/45433>`_, yuliyang_yewu)
+* rgw: fix segfault in UserAsyncRefreshHandler::init_fetch (`pr#45412 <https://github.com/ceph/ceph/pull/45412>`_, Cory Snyder)
+* rgw: have "bucket check --fix" fix pool ids correctly (`pr#45456 <https://github.com/ceph/ceph/pull/45456>`_, J. Eric Ivancich)
+* rgw: init bucket index only if putting bucket instance info succeeds (`pr#45481 <https://github.com/ceph/ceph/pull/45481>`_, Huber-ming)
+* rgw: parse tenant name out of rgwx-bucket-instance (`pr#45523 <https://github.com/ceph/ceph/pull/45523>`_, Casey Bodley)
+* rgw: resolve empty ordered bucket listing results w/ CLS filtering \*and\* bucket index list produces incorrect result when non-ascii entries (`pr#45088 <https://github.com/ceph/ceph/pull/45088>`_, J. Eric Ivancich)
+* rgw: return OK on consecutive complete-multipart reqs (`pr#45488 <https://github.com/ceph/ceph/pull/45488>`_, Mark Kogan)
+* rgw: RGWCoroutine::set_sleeping() checks for null stack (`pr#46042 <https://github.com/ceph/ceph/pull/46042>`_, Or Friedmann, Casey Bodley)
+* rgw: RGWPostObj::execute() may lost data (`pr#45503 <https://github.com/ceph/ceph/pull/45503>`_, Lei Zhang)
+* rgw: url_decode before parsing copysource in copyobject (`issue#43259 <http://tracker.ceph.com/issues/43259>`_, `pr#45431 <https://github.com/ceph/ceph/pull/45431>`_, Paul Reece)
+* rgw:When KMS encryption is used and the key does not exist, we should… (`pr#45462 <https://github.com/ceph/ceph/pull/45462>`_, wangyingbin)
+* rgwlc: fix segfault resharding during lc (`pr#46745 <https://github.com/ceph/ceph/pull/46745>`_, Mark Kogan)
+* rocksdb: do not use non-zero recycle_log_file_num setting (`pr#45040 <https://github.com/ceph/ceph/pull/45040>`_, Igor Fedotov)
+* src/rgw: Fix for malformed url (`pr#45460 <https://github.com/ceph/ceph/pull/45460>`_, Kalpesh Pandya)
+* test/bufferlist: ensure rebuild_aligned_size_and_memory() always rebuilds (`pr#46216 <https://github.com/ceph/ceph/pull/46216>`_, Radoslaw Zarzynski)
+* test/librbd: add test to verify diff_iterate size (`pr#45554 <https://github.com/ceph/ceph/pull/45554>`_, Christopher Hoffman)
+* test: fix wrong alarm (HitSetWrite) (`pr#45320 <https://github.com/ceph/ceph/pull/45320>`_, Myoungwon Oh)
+* tools/rbd: expand where option rbd_default_map_options can be set (`pr#45182 <https://github.com/ceph/ceph/pull/45182>`_, Christopher Hoffman, Ilya Dryomov)
+
+v15.2.16 Octopus
+================
+
+This is the 16th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* Fix in the read lease logic to prevent PGs from going into WAIT state
+  after OSD restart.
+
+* Several bug fixes in BlueStore, including a fix for object listing bug, which
+  could cause stat mismatch scrub errors.
+
+Changelog
+---------
+
+* Fix data corruption in bluefs truncate() (`pr#44860 <https://github.com/ceph/ceph/pull/44860>`_, Adam Kupczyk)
+* Octopus: mds: just respawn mds daemon when osd op requests timeout (`pr#43785 <https://github.com/ceph/ceph/pull/43785>`_, Xiubo Li)
+* admin/doc-requirements.txt: pin Sphinx at 3.5.4 (`pr#43758 <https://github.com/ceph/ceph/pull/43758>`_, Casey Bodley, Kefu Chai, Nizamudeen A, Varsha Rao)
+* backport diff-iterate include_parent tests (`pr#44673 <https://github.com/ceph/ceph/pull/44673>`_, Ilya Dryomov)
+* ceph-volume: `get_first_lv()` refactor (`pr#43959 <https://github.com/ceph/ceph/pull/43959>`_, Guillaume Abrioux)
+* ceph-volume: don't use MultiLogger in find_executable_on_host() (`pr#44766 <https://github.com/ceph/ceph/pull/44766>`_, Guillaume Abrioux)
+* ceph-volume: fix a typo causing AttributeError (`pr#43950 <https://github.com/ceph/ceph/pull/43950>`_, Taha Jahangir)
+* ceph-volume: fix bug with miscalculation of required db/wal slot size for VGs with multiple PVs (`pr#43947 <https://github.com/ceph/ceph/pull/43947>`_, Guillaume Abrioux, Cory Snyder)
+* ceph-volume: fix regression introcuded via #43536 (`pr#44757 <https://github.com/ceph/ceph/pull/44757>`_, Guillaume Abrioux)
+* ceph-volume: honour osd_dmcrypt_key_size option (`pr#44974 <https://github.com/ceph/ceph/pull/44974>`_, Guillaume Abrioux)
+* ceph-volume: human_readable_size() refactor (`pr#44210 <https://github.com/ceph/ceph/pull/44210>`_, Guillaume Abrioux)
+* ceph-volume: improve mpath devices support (`pr#44791 <https://github.com/ceph/ceph/pull/44791>`_, Guillaume Abrioux)
+* ceph-volume: make it possible to skip needs_root() (`pr#44320 <https://github.com/ceph/ceph/pull/44320>`_, Guillaume Abrioux)
+* ceph-volume: show RBD devices as not available (`pr#44709 <https://github.com/ceph/ceph/pull/44709>`_, Michael Fritch)
+* ceph-volume: util/prepare fix osd_id_available() (`pr#43952 <https://github.com/ceph/ceph/pull/43952>`_, Guillaume Abrioux)
+* cephadm/ceph-volume: do not use lvm binary in containers (`pr#43953 <https://github.com/ceph/ceph/pull/43953>`_, Guillaume Abrioux)
+* cephadm: Fix iscsi client caps (allow mgr <service status> calls) (`pr#43822 <https://github.com/ceph/ceph/pull/43822>`_, Juan Miguel Olmo Martínez)
+* cephfs: client: Fix executeable access check for the root user (`pr#41295 <https://github.com/ceph/ceph/pull/41295>`_, Kotresh HR)
+* cls/journal: skip disconnected clients when calculating min_commit_position (`pr#44689 <https://github.com/ceph/ceph/pull/44689>`_, Mykola Golub)
+* common/PriorityCache: low perf counters priorities for submodules (`pr#44176 <https://github.com/ceph/ceph/pull/44176>`_, Igor Fedotov)
+* doc: Use older mistune (`pr#44227 <https://github.com/ceph/ceph/pull/44227>`_, David Galloway)
+* doc: prerequisites fix for cephFS mount (`pr#44271 <https://github.com/ceph/ceph/pull/44271>`_, Nikhilkumar Shelke)
+* librbd/object_map: rbd diff between two snapshots lists entire image content (`pr#43806 <https://github.com/ceph/ceph/pull/43806>`_, Sunny Kumar)
+* librbd: diff-iterate reports incorrect offsets in fast-diff mode (`pr#44548 <https://github.com/ceph/ceph/pull/44548>`_, Ilya Dryomov)
+* mds: Add new flag to MClientSession (`pr#43252 <https://github.com/ceph/ceph/pull/43252>`_, Kotresh HR)
+* mds: PurgeQueue.cc fix for 32bit compilation (`pr#44169 <https://github.com/ceph/ceph/pull/44169>`_, Duncan Bellamy)
+* mds: do not trim stray dentries during opening the root (`pr#43816 <https://github.com/ceph/ceph/pull/43816>`_, Xiubo Li)
+* mds: skip journaling blocklisted clients when in `replay` state (`pr#43842 <https://github.com/ceph/ceph/pull/43842>`_, Venky Shankar)
+* mgr/dashboard/api: set a UTF-8 locale when running pip (`pr#43607 <https://github.com/ceph/ceph/pull/43607>`_, Kefu Chai)
+* mgr/dashboard: all pyfakefs must be pinned on same version (`pr#44159 <https://github.com/ceph/ceph/pull/44159>`_, Rishabh Dave)
+* mgr/dashboard: upgrade Cypress to the latest stable version (`pr#44373 <https://github.com/ceph/ceph/pull/44373>`_, Alfonso Martínez)
+* mgr: Add check to prevent mgr from crashing (`pr#43446 <https://github.com/ceph/ceph/pull/43446>`_, Aswin Toni)
+* mgr: fix locking for MetadataUpdate::finish (`pr#44720 <https://github.com/ceph/ceph/pull/44720>`_, Sage Weil)
+* mgr: set debug_mgr=2/5 (so INFO goes to mgr log by default) (`pr#42677 <https://github.com/ceph/ceph/pull/42677>`_, Sage Weil)
+* mon/MgrStatMonitor: do not spam subscribers (mgr) with service_map (`pr#44722 <https://github.com/ceph/ceph/pull/44722>`_, Sage Weil)
+* mon/MgrStatMonitor: ignore MMgrReport from non-active mgr (`pr#43861 <https://github.com/ceph/ceph/pull/43861>`_, Sage Weil)
+* mon/OSDMonitor: avoid null dereference if stats are not available (`pr#44700 <https://github.com/ceph/ceph/pull/44700>`_, Josh Durgin)
+* mon: prevent new sessions during shutdown (`pr#44544 <https://github.com/ceph/ceph/pull/44544>`_, Sage Weil)
+* msg/async: allow connection reaping to be tuned; fix cephfs test (`pr#43310 <https://github.com/ceph/ceph/pull/43310>`_, Sage Weil, Gerald Yang)
+* msgr/async: fix unsafe access in unregister_conn() (`pr#43325 <https://github.com/ceph/ceph/pull/43325>`_, Sage Weil, Gerald Yang)
+* os/bluestore/AvlAllocator: introduce bluestore_avl_alloc_ff_max\_\* options (`pr#43747 <https://github.com/ceph/ceph/pull/43747>`_, Kefu Chai, Mauricio Faria de Oliveira, Adam Kupczyk, Xue Yantao)
+* os/bluestore: _do_write_small fix head_pad (`pr#43757 <https://github.com/ceph/ceph/pull/43757>`_, dheart)
+* os/bluestore: avoid premature onode release (`pr#44724 <https://github.com/ceph/ceph/pull/44724>`_, Igor Fedotov)
+* os/bluestore: cap omap naming scheme upgrade transactoin (`pr#42958 <https://github.com/ceph/ceph/pull/42958>`_, Adam Kupczyk, Igor Fedotov)
+* os/bluestore: fix additional errors during missed shared blob repair (`pr#43887 <https://github.com/ceph/ceph/pull/43887>`_, Igor Fedotov)
+* os/bluestore: fix writing to invalid offset when repairing (`pr#43885 <https://github.com/ceph/ceph/pull/43885>`_, Igor Fedotov)
+* os/bluestore: list obj which equals to pend (`pr#44978 <https://github.com/ceph/ceph/pull/44978>`_, Mykola Golub, Kefu Chai)
+* os/bluestore: make shared blob fsck much less RAM-greedy (`pr#44614 <https://github.com/ceph/ceph/pull/44614>`_, Igor Fedotov)
+* os/bluestore: use proper prefix when removing undecodable Share Blob (`pr#43883 <https://github.com/ceph/ceph/pull/43883>`_, Igor Fedotov)
+* osd/OSDMap.cc: clean up pg_temp for nonexistent pgs (`pr#44097 <https://github.com/ceph/ceph/pull/44097>`_, Cory Snyder)
+* osd/PeeringState: separate history's pruub from pg's (`pr#44585 <https://github.com/ceph/ceph/pull/44585>`_, Sage Weil)
+* osd: fix 'ceph osd stop <osd.nnn>' doesn't take effect (`pr#43962 <https://github.com/ceph/ceph/pull/43962>`_, tan changzhi)
+* osd: fix partial recovery become whole object recovery after restart osd (`pr#44165 <https://github.com/ceph/ceph/pull/44165>`_, Jianwei Zhang)
+* osd: re-cache peer_bytes on every peering state activate (`pr#43438 <https://github.com/ceph/ceph/pull/43438>`_, Mykola Golub)
+* osd: set r only if succeed in FillInVerifyExtent (`pr#44174 <https://github.com/ceph/ceph/pull/44174>`_, yanqiang-ux)
+* osdc: add set_error in BufferHead, when split set_error to right (`pr#44726 <https://github.com/ceph/ceph/pull/44726>`_, jiawd)
+* pybind/mgr/balancer: define Plan.{dump,show}() (`pr#43965 <https://github.com/ceph/ceph/pull/43965>`_, Kefu Chai)
+* qa/ceph-ansible: Bump OS version for centos (`pr#43658 <https://github.com/ceph/ceph/pull/43658>`_, Brad Hubbard)
+* qa/ceph-ansible: Pin to last compatible stable release (`pr#43557 <https://github.com/ceph/ceph/pull/43557>`_, Brad Hubbard)
+* qa/distros: Remove stale kubic distros (`pr#43788 <https://github.com/ceph/ceph/pull/43788>`_, Sebastian Wagner)
+* qa/mgr/dashboard/test_pool: don't check HEALTH_OK (`pr#43441 <https://github.com/ceph/ceph/pull/43441>`_, Ernesto Puerta)
+* qa/rgw: Fix vault token file access.case (`issue#51539 <http://tracker.ceph.com/issues/51539>`_, `pr#43963 <https://github.com/ceph/ceph/pull/43963>`_, Marcus Watts)
+* qa/rgw: bump tempest version to resolve dependency issue (`pr#43967 <https://github.com/ceph/ceph/pull/43967>`_, Casey Bodley)
+* qa/rgw: octopus branch targets ceph-octopus branch of java_s3tests (`pr#43810 <https://github.com/ceph/ceph/pull/43810>`_, Casey Bodley)
+* qa/run-tox-mgr-dashboard: Do not write to /tmp/test_sanitize_password… (`pr#44728 <https://github.com/ceph/ceph/pull/44728>`_, Kevin Zhao)
+* qa/run_xfstests_qemu.sh: stop reporting success without actually running any tests (`pr#44595 <https://github.com/ceph/ceph/pull/44595>`_, Ilya Dryomov)
+* qa/suites/rados/cephadm: use centos 8.stream (`pr#44929 <https://github.com/ceph/ceph/pull/44929>`_, Adam King, Sage Weil)
+* qa: account for split of the kclient "metrics" debugfs file (`pr#44270 <https://github.com/ceph/ceph/pull/44270>`_, Jeff Layton, Xiubo Li)
+* qa: miscellaneous perf suite fixes (`pr#44254 <https://github.com/ceph/ceph/pull/44254>`_, Neha Ojha)
+* qa: remove centos8 from supported distros (`pr#44864 <https://github.com/ceph/ceph/pull/44864>`_, Casey Bodley, Sage Weil)
+* rbd-mirror: fix mirror image removal (`pr#43663 <https://github.com/ceph/ceph/pull/43663>`_, Arthur Outhenin-Chalandre)
+* rbd-mirror: fix races in snapshot-based mirroring deletion propagation (`pr#44753 <https://github.com/ceph/ceph/pull/44753>`_, Ilya Dryomov)
+* rbd: add missing switch arguments for recognition by get_command_spec() (`pr#44741 <https://github.com/ceph/ceph/pull/44741>`_, Ilya Dryomov)
+* rgw/beast: optimizations for request timeout (`pr#43961 <https://github.com/ceph/ceph/pull/43961>`_, Mark Kogan, Casey Bodley)
+* rgw/rgw_rados: make RGW request IDs non-deterministic (`pr#43696 <https://github.com/ceph/ceph/pull/43696>`_, Cory Snyder)
+* rgw: clear buckets before calling list_buckets() (`pr#43381 <https://github.com/ceph/ceph/pull/43381>`_, Nikhil Kshirsagar)
+* rgw: disable prefetch in rgw_file to fix 3x read amplification (`pr#44170 <https://github.com/ceph/ceph/pull/44170>`_, Kajetan Janiak)
+* rgw: fix `bi put` not using right bucket index shard (`pr#44167 <https://github.com/ceph/ceph/pull/44167>`_, J. Eric Ivancich)
+* rgw: fix bucket purge incomplete multipart uploads (`pr#43863 <https://github.com/ceph/ceph/pull/43863>`_, J. Eric Ivancich)
+* rgw: user stats showing 0 value for "size_utilized" and "size_kb_utilized" fields (`pr#44172 <https://github.com/ceph/ceph/pull/44172>`_, J. Eric Ivancich)
+* rgwlc:  remove lc entry on bucket delete (`pr#44730 <https://github.com/ceph/ceph/pull/44730>`_, Matt Benjamin)
+* rpm, debian: move smartmontools and nvme-cli to ceph-base (`pr#44177 <https://github.com/ceph/ceph/pull/44177>`_, Yaarit Hatuka)
+
+v15.2.15 Octopus
+================
+
+This is the 15th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* The default value of `osd_client_message_cap` has been set to 256, to provide
+  better flow control by limiting maximum number of in-flight client requests.
+
+* A new ceph-erasure-code-tool has been added to help manually recover an
+  object from a damaged PG.
+
+Changelog
+---------
+
+* auth,mon: don't log "unable to find a keyring" error when key is given (`pr#43312 <https://github.com/ceph/ceph/pull/43312>`_, Ilya Dryomov)
+* ceph-monstore-tool: use a large enough paxos/{first,last}_committed (`issue#38219 <http://tracker.ceph.com/issues/38219>`_, `pr#43263 <https://github.com/ceph/ceph/pull/43263>`_, Kefu Chai)
+* ceph-volume/tests: retry when destroying osd (`pr#42547 <https://github.com/ceph/ceph/pull/42547>`_, Guillaume Abrioux)
+* ceph-volume: disable cache for blkid calls (`pr#41115 <https://github.com/ceph/ceph/pull/41115>`_, Rafał Wądołowski)
+* ceph-volume: fix batch report and respect ceph.conf config values (`pr#41715 <https://github.com/ceph/ceph/pull/41715>`_, Andrew Schoen)
+* ceph-volume: fix lvm activate --all --no-systemd (`pr#43268 <https://github.com/ceph/ceph/pull/43268>`_, Dimitri Savineau)
+* ceph-volume: fix lvm activate arguments (`pr#43117 <https://github.com/ceph/ceph/pull/43117>`_, Dimitri Savineau)
+* ceph-volume: fix lvm migrate without args (`pr#43111 <https://github.com/ceph/ceph/pull/43111>`_, Dimitri Savineau)
+* ceph-volume: fix raw list with logical partition (`pr#43088 <https://github.com/ceph/ceph/pull/43088>`_, Guillaume Abrioux, Dimitri Savineau)
+* ceph-volume: lvm batch: fast_allocations(): avoid ZeroDivisionError (`pr#42494 <https://github.com/ceph/ceph/pull/42494>`_, Jonas Zeiger)
+* ceph-volume: pvs --noheadings replace pvs --no-heading (`pr#43077 <https://github.com/ceph/ceph/pull/43077>`_, FengJiankui)
+* ceph-volume: remove --all ref from deactivate help (`pr#43097 <https://github.com/ceph/ceph/pull/43097>`_, Dimitri Savineau)
+* ceph-volume: support no_systemd with lvm migrate (`pr#43092 <https://github.com/ceph/ceph/pull/43092>`_, Dimitri Savineau)
+* ceph-volume: work around phantom atari partitions (`pr#42752 <https://github.com/ceph/ceph/pull/42752>`_, Blaine Gardner)
+* ceph.spec: selinux scripts respect CEPH_AUTO_RESTART_ON_UPGRADE (`pr#43234 <https://github.com/ceph/ceph/pull/43234>`_, Dan van der Ster)
+* cephadm: add thread ident to log messages (`pr#43133 <https://github.com/ceph/ceph/pull/43133>`_, Michael Fritch)
+* cephadm: default to quay.io, not docker.io (`pr#42533 <https://github.com/ceph/ceph/pull/42533>`_, Sage Weil)
+* cephadm: use quay, not docker (`pr#43094 <https://github.com/ceph/ceph/pull/43094>`_, Sage Weil, Juan Miguel Olmo Martínez)
+* cmake: Replace boost download url (`pr#42694 <https://github.com/ceph/ceph/pull/42694>`_, Rafał Wądołowski)
+* cmake: s/Python_EXECUTABLE/Python3_EXECUTABLE/ (`pr#43265 <https://github.com/ceph/ceph/pull/43265>`_, Michael Fritch)
+* common/buffer: fix SIGABRT in  rebuild_aligned_size_and_memory (`pr#42975 <https://github.com/ceph/ceph/pull/42975>`_, Yin Congmin)
+* common/options: Set osd_client_message_cap to 256 (`pr#42616 <https://github.com/ceph/ceph/pull/42616>`_, Mark Nelson)
+* doc/ceph-volume: add lvm migrate/new-db/new-wal (`pr#43090 <https://github.com/ceph/ceph/pull/43090>`_, Dimitri Savineau)
+* Don't persist report data (`pr#42670 <https://github.com/ceph/ceph/pull/42670>`_, Brad Hubbard)
+* krbd: escape udev_enumerate_add_match_sysattr values (`pr#42968 <https://github.com/ceph/ceph/pull/42968>`_, Ilya Dryomov)
+* mgr/cephadm: pass --container-init to cephadm if specified (`pr#42666 <https://github.com/ceph/ceph/pull/42666>`_, Tim Serong)
+* mgr/dashboard: cephadm e2e start script: add --expanded option (`pr#42794 <https://github.com/ceph/ceph/pull/42794>`_, Alfonso Martínez)
+* mgr/dashboard: deprecated variable usage in Grafana dashboards (`pr#43189 <https://github.com/ceph/ceph/pull/43189>`_, Patrick Seidensal)
+* mgr/dashboard: Incorrect MTU mismatch warning (`pr#43186 <https://github.com/ceph/ceph/pull/43186>`_, Aashish Sharma)
+* mgr/dashboard: stats=false not working when listing buckets (`pr#42892 <https://github.com/ceph/ceph/pull/42892>`_, Avan Thakkar)
+* mgr/influx: use "N/A" for unknown hostname (`pr#43369 <https://github.com/ceph/ceph/pull/43369>`_, Kefu Chai)
+* mgr/prometheus: Fix metric types from gauge to counter (`pr#42674 <https://github.com/ceph/ceph/pull/42674>`_, Patrick Seidensal)
+* mon/OSDMonitor: account for PG merging in epoch_by_pg accounting (`pr#42837 <https://github.com/ceph/ceph/pull/42837>`_, Dan van der Ster)
+* mon/PGMap: remove DIRTY field in `ceph df detail` when cache tiering  is not in use (`pr#42862 <https://github.com/ceph/ceph/pull/42862>`_, Deepika Upadhyay)
+* mon: return -EINVAL when handling unknown option in 'ceph osd pool get' (`pr#43266 <https://github.com/ceph/ceph/pull/43266>`_, Zhao Cuicui)
+* monitoring/grafana/cluster: use per-unit max and limit values (`pr#42675 <https://github.com/ceph/ceph/pull/42675>`_, David Caro)
+* monitoring: fix Physical Device Latency unit (`pr#42676 <https://github.com/ceph/ceph/pull/42676>`_, Seena Fallah)
+* os/bluestore: accept undecodable multi-block bluefs transactions on log (`pr#43024 <https://github.com/ceph/ceph/pull/43024>`_, Igor Fedotov)
+* os/bluestore: fix bluefs migrate command (`pr#43140 <https://github.com/ceph/ceph/pull/43140>`_, Igor Fedotov)
+* os/bluestore: fix using incomplete bluefs log when dumping it (`pr#43008 <https://github.com/ceph/ceph/pull/43008>`_, Igor Fedotov)
+* osd/OSD: mkfs need wait for transcation completely finish (`pr#43418 <https://github.com/ceph/ceph/pull/43418>`_, Chen Fan)
+* pybind/rbd: fix mirror_image_get_status (`pr#42971 <https://github.com/ceph/ceph/pull/42971>`_, Ilya Dryomov, Will Smith)
+* qa/mgr/dashboard: add extra wait to test (`pr#43352 <https://github.com/ceph/ceph/pull/43352>`_, Ernesto Puerta)
+* qa/suites/rados: use centos_8.3_container_tools_3.0.yaml (`pr#43102 <https://github.com/ceph/ceph/pull/43102>`_, Sebastian Wagner)
+* qa/tests: advanced version to 15.2.14 to match the latest release (`pr#42761 <https://github.com/ceph/ceph/pull/42761>`_, Yuri Weinstein)
+* qa/workunits/mon/test_mon_config_key: use subprocess.run() instead of proc.communicate() (`pr#42498 <https://github.com/ceph/ceph/pull/42498>`_, Kefu Chai)
+* rbd-mirror: add perf counters to snapshot replayed (`pr#42986 <https://github.com/ceph/ceph/pull/42986>`_, Arthur Outhenin-Chalandre)
+* rbd-mirror: fix potential async op tracker leak in start_image_replayers (`pr#42978 <https://github.com/ceph/ceph/pull/42978>`_, Mykola Golub)
+* rbd-mirror: unbreak one-way snapshot-based mirroring (`pr#43314 <https://github.com/ceph/ceph/pull/43314>`_, Ilya Dryomov)
+* rgw : add check for tenant provided in RGWCreateRole (`pr#43270 <https://github.com/ceph/ceph/pull/43270>`_, caolei)
+* rgw: avoid infinite loop when deleting a bucket (`issue#49206 <http://tracker.ceph.com/issues/49206>`_, `pr#43272 <https://github.com/ceph/ceph/pull/43272>`_, Jeegn Chen)
+* rgw: fail as expected when set/delete-bucket-website attempted on a non-exis… (`pr#43424 <https://github.com/ceph/ceph/pull/43424>`_, xiangrui meng)
+* rgw: fix sts memory leak (`pr#43349 <https://github.com/ceph/ceph/pull/43349>`_, yuliyang_yewu)
+* rgw: remove quota soft threshold (`pr#43271 <https://github.com/ceph/ceph/pull/43271>`_, Zulai Wang)
+* rgw: when deleted obj removed in versioned bucket, extra del-marker added (`pr#43273 <https://github.com/ceph/ceph/pull/43273>`_, J. Eric Ivancich)
+* run-make-check.sh: Increase failure output log size (`pr#42849 <https://github.com/ceph/ceph/pull/42849>`_, David Galloway)
+* tools/erasure-code: new tool to encode/decode files (`pr#43407 <https://github.com/ceph/ceph/pull/43407>`_, Mykola Golub)
+
+v15.2.14 Octopus
+================
+
+This is the 14th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* RGW: It is possible to specify ssl options and ciphers for beast frontend now.
+  The default ssl options setting is "no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1".
+  If you want to return back the old behavior add 'ssl_options=' (empty) to
+  ``rgw frontends`` configuration.
+
+* CephFS: old clusters (pre-Jewel) that did not use CephFS have legacy data
+  structures in the ceph-mon stores. These structures are not understood by
+  Pacific monitors. With Octopus v15.2.14, the monitors have been taught to flush
+  and trim these old structures out in preparation for an upgrade to Pacific or
+  Quincy. For more information, see
+  `Issue 51673 <https://tracker.ceph.com/issues/51673>`.
+
+* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook`
+  anymore. As the latter depends on `python3-numpy` which cannot be imported in
+  different Python sub-interpreters multi-times if the version of
+  `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends`
+  packages by default, `ceph-mgr-rook` was always installed along with
+  `ceph-mgr` debian package as an indirect dependency. If your workflow depends
+  on this behavior, you might want to install `ceph-mgr-rook` separately.
+
+* Several bug fixes in BlueStore, including a fix for an unexpected
+  ENOSPC bug in Avl/Hybrid allocators.
+
+* Includes a fix for a bug that affects recovery below *min_size* for EC pools.
+
+Changelog
+---------
+
+* bind on loopback address if no other addresses are available (`pr#42478 <https://github.com/ceph/ceph/pull/42478>`_, Kefu Chai, Matthew Oliver)
+* bluestore: use string_view and strip trailing slash for dir listing (`pr#41757 <https://github.com/ceph/ceph/pull/41757>`_, Jonas Jelten, Kefu Chai)
+* ceph-volume/tests: update ansible environment variables in tox (`pr#42491 <https://github.com/ceph/ceph/pull/42491>`_, Dimitri Savineau)
+* ceph-volume: Consider /dev/root as mounted (`pr#41584 <https://github.com/ceph/ceph/pull/41584>`_, David Caro)
+* ceph-volume: implement bluefs volume migration (`pr#42377 <https://github.com/ceph/ceph/pull/42377>`_, Igor Fedotov, Kefu Chai)
+* ceph: ignore BrokenPipeError when printing help (`pr#41586 <https://github.com/ceph/ceph/pull/41586>`_, Ernesto Puerta)
+* cephadm: fix escaping/quoting of stderr-prefix arg for ceph daemons (`pr#40948 <https://github.com/ceph/ceph/pull/40948>`_, Michael Fritch, Sage Weil)
+* cephadm: fix port_in_use when IPv6 is disabled (`pr#41602 <https://github.com/ceph/ceph/pull/41602>`_, Patrick Seidensal)
+* cephfs: client: add ability to lookup snapped inodes by inode number (`pr#40768 <https://github.com/ceph/ceph/pull/40768>`_, Jeff Layton, Xiubo Li)
+* cls/rgw: look for plain entries in non-ascii plain namespace too (`pr#41775 <https://github.com/ceph/ceph/pull/41775>`_, Mykola Golub)
+* cmake: build static libs if they are internal ones (`pr#39904 <https://github.com/ceph/ceph/pull/39904>`_, Kefu Chai)
+* crush/crush: ensure alignof(crush_work_bucket) is 1 (`pr#41622 <https://github.com/ceph/ceph/pull/41622>`_, Kefu Chai)
+* debian/control: ceph-mgr-modules-core does not Recommend ceph-mgr-rook (`pr#41878 <https://github.com/ceph/ceph/pull/41878>`_, Kefu Chai)
+* doc/rados/operations: s/max_misplaced/target_max_misplaced_ratio/ (`pr#41624 <https://github.com/ceph/ceph/pull/41624>`_, Kefu Chai)
+* librbd: don't stop at the first unremovable image when purging (`pr#41663 <https://github.com/ceph/ceph/pull/41663>`_, Ilya Dryomov)
+* librbd: global config overrides do not apply to in-use images (`pr#41763 <https://github.com/ceph/ceph/pull/41763>`_, Jason Dillaman)
+* make-dist: refuse to run if script path contains a colon (`pr#41087 <https://github.com/ceph/ceph/pull/41087>`_, Nathan Cutler)
+* mds: avoid journaling overhead for setxattr("ceph.dir.subvolume") for no-op case (`pr#41996 <https://github.com/ceph/ceph/pull/41996>`_, Patrick Donnelly)
+* mds: completed_requests -> num_completed_requests and dump num_completed_flushes (`pr#41625 <https://github.com/ceph/ceph/pull/41625>`_, Dan van der Ster)
+* mds: fix cpu_profiler asok crash (`pr#41767 <https://github.com/ceph/ceph/pull/41767>`_, liu shi)
+* mds: place the journaler pointer under the mds_lock (`pr#41626 <https://github.com/ceph/ceph/pull/41626>`_, Xiubo Li)
+* MDSMonitor: monitor crash after upgrade from ceph 15.2.13 to 16.2.4 (`pr#42537 <https://github.com/ceph/ceph/pull/42537>`_, Patrick Donnelly)
+* mds: reject lookup ino requests for mds dirs (`pr#40782 <https://github.com/ceph/ceph/pull/40782>`_, Xiubo Li, Patrick Donnelly)
+* mgr/DaemonServer.cc: prevent mgr crashes caused by integer underflow that is triggered by large increases to pg_num/pgp_num (`pr#41764 <https://github.com/ceph/ceph/pull/41764>`_, Cory Snyder)
+* mgr/DaemonServer: skip redundant update of pgp_num_actual (`pr#42420 <https://github.com/ceph/ceph/pull/42420>`_, Dan van der Ster)
+* mgr/Dashboard: Remove erroneous elements in hosts-overview Grafana dashboard (`pr#41649 <https://github.com/ceph/ceph/pull/41649>`_, Malcolm Holmes)
+* mgr/cephadm: fix prometheus alerts (`pr#41660 <https://github.com/ceph/ceph/pull/41660>`_, Paul Cuzner, Sage Weil, Patrick Seidensal)
+* mgr/dashboard: Add configurable MOTD or wall notification (`pr#42412 <https://github.com/ceph/ceph/pull/42412>`_, Volker Theile)
+* mgr/dashboard: Fix bucket name input allowing space in the value (`pr#42241 <https://github.com/ceph/ceph/pull/42241>`_, Nizamudeen A)
+* mgr/dashboard: RGW buckets async validator performance enhancement and name constraints (`pr#42123 <https://github.com/ceph/ceph/pull/42123>`_, Nizamudeen A)
+* mgr/dashboard: User database migration has been cut out (`pr#42142 <https://github.com/ceph/ceph/pull/42142>`_, Volker Theile)
+* mgr/dashboard: disable NFSv3 support in dashboard (`pr#41199 <https://github.com/ceph/ceph/pull/41199>`_, Volker Theile)
+* mgr/dashboard: fix API docs link (`pr#41508 <https://github.com/ceph/ceph/pull/41508>`_, Avan Thakkar)
+* mgr/dashboard: fix OSD out count (`pr#42154 <https://github.com/ceph/ceph/pull/42154>`_, 胡玮文)
+* mgr/dashboard: fix OSDs Host details/overview grafana graphs (`issue#49769 <http://tracker.ceph.com/issues/49769>`_, `pr#41530 <https://github.com/ceph/ceph/pull/41530>`_, Alfonso Martínez, Michael Wodniok)
+* mgr/dashboard: fix bucket objects and size calculations (`pr#41647 <https://github.com/ceph/ceph/pull/41647>`_, Avan Thakkar)
+* mgr/dashboard: fix for right sidebar nav icon not clickable (`pr#42015 <https://github.com/ceph/ceph/pull/42015>`_, Aaryan Porwal)
+* mgr/dashboard: run cephadm-backend e2e tests with KCLI (`pr#42243 <https://github.com/ceph/ceph/pull/42243>`_, Alfonso Martínez)
+* mgr/dashboard: show partially deleted RBDs (`pr#41887 <https://github.com/ceph/ceph/pull/41887>`_, Tatjana Dehler)
+* mgr/telemetry: pass leaderboard flag even w/o ident (`pr#41870 <https://github.com/ceph/ceph/pull/41870>`_, Sage Weil)
+* mgr: do not load disabled modules (`pr#41617 <https://github.com/ceph/ceph/pull/41617>`_, Kefu Chai)
+* mon/MonClient: tolerate a rotating key that is slightly out of date (`pr#41449 <https://github.com/ceph/ceph/pull/41449>`_, Ilya Dryomov)
+* mon/OSDMonitor: drop stale failure_info even if can_mark_down() (`pr#41618 <https://github.com/ceph/ceph/pull/41618>`_, Kefu Chai)
+* mon: load stashed map before mkfs monmap (`pr#41621 <https://github.com/ceph/ceph/pull/41621>`_, Dan van der Ster)
+* os/bluestore: Remove possibility of replay log and file inconsistency (`pr#42374 <https://github.com/ceph/ceph/pull/42374>`_, Adam Kupczyk)
+* os/bluestore: compact db after bulk omap naming upgrade (`pr#42375 <https://github.com/ceph/ceph/pull/42375>`_, Igor Fedotov)
+* os/bluestore: fix erroneous SharedBlob record removal during repair (`pr#42373 <https://github.com/ceph/ceph/pull/42373>`_, Igor Fedotov)
+* os/bluestore: fix unexpected ENOSPC in Avl/Hybrid allocators (`pr#41658 <https://github.com/ceph/ceph/pull/41658>`_, Igor Fedotov)
+* os/bluestore: introduce multithireading sync for bluestore's repairer (`pr#41613 <https://github.com/ceph/ceph/pull/41613>`_, Igor Fedotov)
+* os/bluestore: tolerate zero length for allocators' init\_[add/rm]_free() (`pr#41612 <https://github.com/ceph/ceph/pull/41612>`_, Igor Fedotov)
+* osd/PG.cc: handle removal of pgmeta object (`pr#41623 <https://github.com/ceph/ceph/pull/41623>`_, Neha Ojha)
+* osd/PeeringState: fix acting_set_writeable min_size check (`pr#41609 <https://github.com/ceph/ceph/pull/41609>`_, Samuel Just)
+* osd/osd_type: use f->dump_unsigned() when appropriate (`pr#42257 <https://github.com/ceph/ceph/pull/42257>`_, Kefu Chai)
+* osd: clear data digest when write_trunc (`pr#41620 <https://github.com/ceph/ceph/pull/41620>`_, Zengran Zhang)
+* osd: fix scrub reschedule bug (`pr#41972 <https://github.com/ceph/ceph/pull/41972>`_, wencong wan)
+* osd: log snaptrim message to dout (`pr#42484 <https://github.com/ceph/ceph/pull/42484>`_, Arthur Outhenin-Chalandre)
+* osd: move down peers out from peer_purged (`pr#42239 <https://github.com/ceph/ceph/pull/42239>`_, Mykola Golub)
+* pacific: pybind/ceph_volume_client: stat on empty string (`pr#42161 <https://github.com/ceph/ceph/pull/42161>`_, Patrick Donnelly)
+* qa/\*/test_envlibrados_for_rocksdb.sh: install libarchive-3.3.3 (`pr#42421 <https://github.com/ceph/ceph/pull/42421>`_, Neha Ojha)
+* qa/cephadm/upgrade: use v15.2.9 for cephadm tests (`pr#41568 <https://github.com/ceph/ceph/pull/41568>`_, Deepika Upadhyay)
+* qa/config/rados: add dispatch delay testing params (`pr#42180 <https://github.com/ceph/ceph/pull/42180>`_, Deepika Upadhyay)
+* qa/distros: move to latest version on supported distro's (`pr#41478 <https://github.com/ceph/ceph/pull/41478>`_, Josh Durgin, Yuri Weinstein, Deepika Upadhyay, Sage Weil, Kefu Chai, Patrick Donnelly, rakeshgm)
+* qa/suites/rados/perf: pin to 18.04 (`pr#41922 <https://github.com/ceph/ceph/pull/41922>`_, Neha Ojha)
+* qa/suites/rados: add simultaneous scrubs to the thrasher (`pr#42422 <https://github.com/ceph/ceph/pull/42422>`_, Ronen Friedman)
+* qa/tasks/qemu: precise repos have been archived (`pr#41642 <https://github.com/ceph/ceph/pull/41642>`_, Ilya Dryomov)
+* qa/upgrade: disable update_features test_notify with older client as lockowner (`pr#41511 <https://github.com/ceph/ceph/pull/41511>`_, Deepika Upadhyay)
+* qa/workunits/rbd: use bionic version of qemu-iotests for focal (`pr#42025 <https://github.com/ceph/ceph/pull/42025>`_, Ilya Dryomov)
+* rbd-mirror: fix segfault in snapshot replayer shutdown (`pr#41502 <https://github.com/ceph/ceph/pull/41502>`_, Arthur Outhenin-Chalandre)
+* rbd: retrieve global config overrides from the MONs (`pr#41836 <https://github.com/ceph/ceph/pull/41836>`_, Ilya Dryomov, Jason Dillaman)
+* rgw : add check empty for sync url (`pr#41766 <https://github.com/ceph/ceph/pull/41766>`_, caolei)
+* rgw/amqp/kafka: prevent concurrent shutdowns from happening (`pr#40381 <https://github.com/ceph/ceph/pull/40381>`_, Yuval Lifshitz)
+* rgw/amqp/test: fix mock prototype for librabbitmq-0.11.0 (`pr#41418 <https://github.com/ceph/ceph/pull/41418>`_, Yuval Lifshitz)
+* rgw/notifications: delete bucket notification object when empty (`pr#41412 <https://github.com/ceph/ceph/pull/41412>`_, Yuval Lifshitz)
+* rgw/rgw_file: Fix the return value of read() and readlink() (`pr#41416 <https://github.com/ceph/ceph/pull/41416>`_, Dai zhiwei, luo rixin)
+* rgw/sts: read_obj_policy() consults iam_user_policies on ENOENT (`pr#41415 <https://github.com/ceph/ceph/pull/41415>`_, Casey Bodley)
+* rgw: Backport 51674 to Octopus (`pr#42347 <https://github.com/ceph/ceph/pull/42347>`_, Adam C. Emerson)
+* rgw: Improve error message on email id reuse (`pr#41784 <https://github.com/ceph/ceph/pull/41784>`_, Ponnuvel Palaniyappan)
+* rgw: allow rgw-orphan-list to process multiple data pools (`pr#41417 <https://github.com/ceph/ceph/pull/41417>`_, J. Eric Ivancich)
+* rgw: allow to set ssl options and ciphers for beast frontend (`pr#42368 <https://github.com/ceph/ceph/pull/42368>`_, Mykola Golub)
+* rgw: check object locks in multi-object delete (`issue#47586 <http://tracker.ceph.com/issues/47586>`_, `pr#41031 <https://github.com/ceph/ceph/pull/41031>`_, Mark Houghton)
+* rgw: fix bucket object listing when marker matches prefix (`pr#41413 <https://github.com/ceph/ceph/pull/41413>`_, J. Eric Ivancich)
+* rgw: fix segfault related to explicit object manifest handling (`pr#41420 <https://github.com/ceph/ceph/pull/41420>`_, Mark Kogan)
+* rgw: limit rgw_gc_max_objs to RGW_SHARDS_PRIME_1 (`pr#40383 <https://github.com/ceph/ceph/pull/40383>`_, Rafał Wądołowski)
+* rgw: qa/tasks/barbican.py: fix year2021 problem (`pr#40385 <https://github.com/ceph/ceph/pull/40385>`_, Marcus Watts)
+* rgw: radoslist incomplete multipart parts marker (`pr#40820 <https://github.com/ceph/ceph/pull/40820>`_, J. Eric Ivancich)
+* rgw: require bucket name in bucket chown (`pr#41765 <https://github.com/ceph/ceph/pull/41765>`_, Zulai Wang)
+* rgw: send headers of quota settings (`pr#41419 <https://github.com/ceph/ceph/pull/41419>`_, Or Friedmann)
+* rpm: drop use of $FIRST_ARG in ceph-immutable-object-cache (`pr#42509 <https://github.com/ceph/ceph/pull/42509>`_, Nathan Cutler)
+* rpm: three spec file cleanups (`pr#42440 <https://github.com/ceph/ceph/pull/42440>`_, Nathan Cutler, Franck Bui)
+* test: bump DecayCounter.steady acceptable error (`pr#41619 <https://github.com/ceph/ceph/pull/41619>`_, Patrick Donnelly)
+
+
+v15.2.13 Octopus
+================
+
+This is the 13th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* RADOS: Ability to dynamically adjust trimming rate in the monitor and several other bug fixes.
+
+* A long-standing bug that prevented 32-bit and 64-bit client/server
+  interoperability under msgr v2 has been fixed.  In particular, mixing armv7l
+  (armhf) and x86_64 or aarch64 servers in the same cluster now works.
+
+Changelog
+---------
+
+* blk/kernel: fix io_uring got (4) Interrupted system call (`pr#39899 <https://github.com/ceph/ceph/pull/39899>`_, Yanhu Cao)
+* ceph.spec.in: Enable tcmalloc on IBM Power and Z (`pr#39487 <https://github.com/ceph/ceph/pull/39487>`_, Nathan Cutler, Yaakov Selkowitz)
+* cephadm: `cephadm ls` broken for SUSE downstream alertmanager container (`pr#39802 <https://github.com/ceph/ceph/pull/39802>`_, Patrick Seidensal)
+* cephadm: Allow to use paths in all <_devices> drivegroup sections (`pr#40838 <https://github.com/ceph/ceph/pull/40838>`_, Juan Miguel Olmo Martínez)
+* cephadm: add docker.service dependency in systemd units (`pr#39804 <https://github.com/ceph/ceph/pull/39804>`_, Sage Weil)
+* cephadm: allow redeploy of daemons in error state if container running (`pr#39717 <https://github.com/ceph/ceph/pull/39717>`_, Adam King)
+* cephadm: fix failure when using --apply-spec and --shh-user (`pr#40737 <https://github.com/ceph/ceph/pull/40737>`_, Daniel Pivonka)
+* cephadm: run containers using `--init` (`pr#39914 <https://github.com/ceph/ceph/pull/39914>`_, Michael Fritch, Sage Weil)
+* cephfs: client: only check pool permissions for regular files (`pr#40779 <https://github.com/ceph/ceph/pull/40779>`_, Xiubo Li)
+* cephfs: client: wake up the front pos waiter (`pr#40771 <https://github.com/ceph/ceph/pull/40771>`_, Xiubo Li)
+* client: fire the finish_cap_snap() after buffer being flushed (`pr#40778 <https://github.com/ceph/ceph/pull/40778>`_, Xiubo Li)
+* cmake: build static libs if they are internal ones (`pr#40789 <https://github.com/ceph/ceph/pull/40789>`_, Kefu Chai)
+* cmake: define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT globaly (`pr#40784 <https://github.com/ceph/ceph/pull/40784>`_, Kefu Chai)
+* common/buffer: adjust align before calling posix_memalign() (`pr#41247 <https://github.com/ceph/ceph/pull/41247>`_, Ilya Dryomov)
+* common/ipaddr: Allow binding on lo (`pr#39343 <https://github.com/ceph/ceph/pull/39343>`_, Thomas Goirand)
+* common/ipaddr: skip loopback interfaces named 'lo' and test it (`pr#40424 <https://github.com/ceph/ceph/pull/40424>`_, Dan van der Ster)
+* common/mempool: Improve mempool shard selection (`pr#39978 <https://github.com/ceph/ceph/pull/39978>`_, singuliere, Adam Kupczyk)
+* common/options/global.yaml.in: increase default value of bluestore_cache_trim_max_skip_pinned (`pr#40919 <https://github.com/ceph/ceph/pull/40919>`_, Neha Ojha)
+* common/options: bluefs_buffered_io=true by default (`pr#40392 <https://github.com/ceph/ceph/pull/40392>`_, Dan van der Ster)
+* common: Fix assertion when disabling and re-enabling clog_to_monitors (`pr#39935 <https://github.com/ceph/ceph/pull/39935>`_, Gerald Yang)
+* common: remove log_early configuration option (`pr#40550 <https://github.com/ceph/ceph/pull/40550>`_, Changcheng Liu)
+* crush/CrushLocation: do not print logging message in constructor (`pr#40791 <https://github.com/ceph/ceph/pull/40791>`_, Alex Wu)
+* crush/CrushWrapper: update shadow trees on update_item() (`pr#39919 <https://github.com/ceph/ceph/pull/39919>`_, Sage Weil)
+* debian/ceph-common.postinst: do not chown cephadm log dirs (`pr#40275 <https://github.com/ceph/ceph/pull/40275>`_, Sage Weil)
+* doc/cephfs/nfs: Add note about cephadm NFS-Ganesha daemon port (`pr#40777 <https://github.com/ceph/ceph/pull/40777>`_, Varsha Rao)
+* doc/cephfs/nfs: Add rook pod restart note, export and log block example (`pr#40766 <https://github.com/ceph/ceph/pull/40766>`_, Varsha Rao)
+* doc: snap-schedule documentation (`pr#40775 <https://github.com/ceph/ceph/pull/40775>`_, Jan Fajerski)
+* install-deps.sh: remove existing ceph-libboost of different version (`pr#40286 <https://github.com/ceph/ceph/pull/40286>`_, Kefu Chai)
+* krbd: make sure the device node is accessible after the mapping (`pr#39968 <https://github.com/ceph/ceph/pull/39968>`_, Ilya Dryomov)
+* librbd/api: avoid retrieving more than max mirror image info records (`pr#39964 <https://github.com/ceph/ceph/pull/39964>`_, Jason Dillaman)
+* librbd/io: conditionally disable move optimization (`pr#39958 <https://github.com/ceph/ceph/pull/39958>`_, Jason Dillaman)
+* librbd/io: send alloc_hint when compression hint is set (`pr#40386 <https://github.com/ceph/ceph/pull/40386>`_, Jason Dillaman)
+* librbd/mirror/snapshot: avoid UnlinkPeerRequest with a unlinked peer (`pr#41302 <https://github.com/ceph/ceph/pull/41302>`_, Arthur Outhenin-Chalandre)
+* librbd: allow interrupted trash move request to be restarted (`pr#40387 <https://github.com/ceph/ceph/pull/40387>`_, Jason Dillaman)
+* librbd: explicitly disable readahead for writearound cache (`pr#39962 <https://github.com/ceph/ceph/pull/39962>`_, Jason Dillaman)
+* librbd: refuse to release exclusive lock when removing (`pr#39966 <https://github.com/ceph/ceph/pull/39966>`_, Ilya Dryomov)
+* mds: fix race of fetching large dirfrag (`pr#40774 <https://github.com/ceph/ceph/pull/40774>`_, Erqi Chen)
+* mds: trim cache regularly for standby-replay (`pr#40743 <https://github.com/ceph/ceph/pull/40743>`_, Xiubo Li, Patrick Donnelly)
+* mds: update defaults for recall configs (`pr#40764 <https://github.com/ceph/ceph/pull/40764>`_, Patrick Donnelly)
+* mgr/PyModule: put mgr_module_path before Py_GetPath() (`pr#40534 <https://github.com/ceph/ceph/pull/40534>`_, Kefu Chai)
+* mgr/cephadm: alias rgw-nfs -> nfs (`pr#40009 <https://github.com/ceph/ceph/pull/40009>`_, Michael Fritch)
+* mgr/cephadm: on ssh connection error, advice chmod 0600 (`pr#40823 <https://github.com/ceph/ceph/pull/40823>`_, Sebastian Wagner)
+* mgr/dashboard: Add badge to the Label column in Host List (`pr#40433 <https://github.com/ceph/ceph/pull/40433>`_, Nizamudeen A)
+* mgr/dashboard: Device health status is not getting listed under hosts section (`pr#40495 <https://github.com/ceph/ceph/pull/40495>`_, Aashish Sharma)
+* mgr/dashboard: Fix for alert notification message being undefined (`pr#40589 <https://github.com/ceph/ceph/pull/40589>`_, Nizamudeen A)
+* mgr/dashboard: Fix for broken User management role cloning (`pr#40399 <https://github.com/ceph/ceph/pull/40399>`_, Nizamudeen A)
+* mgr/dashboard: OSDs placement text is unreadable (`pr#41124 <https://github.com/ceph/ceph/pull/41124>`_, Aashish Sharma)
+* mgr/dashboard: Remove redundant pytest requirement (`pr#40657 <https://github.com/ceph/ceph/pull/40657>`_, Kefu Chai)
+* mgr/dashboard: Remove username and password from request body (`pr#41057 <https://github.com/ceph/ceph/pull/41057>`_, Nizamudeen A)
+* mgr/dashboard: Remove username, password fields from Manager Modules/dashboard,influx (`pr#40491 <https://github.com/ceph/ceph/pull/40491>`_, Aashish Sharma)
+* mgr/dashboard: Revoke read-only user's access to Manager modules (`pr#40649 <https://github.com/ceph/ceph/pull/40649>`_, Nizamudeen A)
+* mgr/dashboard: Splitting tenant$user when creating rgw user (`pr#40297 <https://github.com/ceph/ceph/pull/40297>`_, Nizamudeen A)
+* mgr/dashboard: additional logging to SMART data retrieval (`pr#37972 <https://github.com/ceph/ceph/pull/37972>`_, Kiefer Chang, Patrick Seidensal)
+* mgr/dashboard: allow getting fresh inventory data from the orchestrator (`pr#41387 <https://github.com/ceph/ceph/pull/41387>`_, Kiefer Chang)
+* mgr/dashboard: debug nodeenv hangs (`pr#40816 <https://github.com/ceph/ceph/pull/40816>`_, Ernesto Puerta)
+* mgr/dashboard: filesystem pool size should use stored stat (`pr#41020 <https://github.com/ceph/ceph/pull/41020>`_, Avan Thakkar)
+* mgr/dashboard: fix base-href: revert it to previous approach (`pr#41252 <https://github.com/ceph/ceph/pull/41252>`_, Avan Thakkar)
+* mgr/dashboard: fix dashboard instance ssl certificate functionality (`pr#40001 <https://github.com/ceph/ceph/pull/40001>`_, Avan Thakkar)
+* mgr/dashboard: improve telemetry opt-in reminder notification message (`pr#40894 <https://github.com/ceph/ceph/pull/40894>`_, Waad Alkhoury)
+* mgr/dashboard: test prometheus rules through promtool (`pr#39987 <https://github.com/ceph/ceph/pull/39987>`_, Aashish Sharma, Kefu Chai)
+* mgr/progress: ensure progress stays between [0,1] (`pr#41311 <https://github.com/ceph/ceph/pull/41311>`_, Dan van der Ster)
+* mgr/rook: Add timezone info (`pr#39716 <https://github.com/ceph/ceph/pull/39716>`_, Varsha Rao)
+* mgr/telemetry: check if 'ident' channel is active (`pr#39922 <https://github.com/ceph/ceph/pull/39922>`_, Sage Weil, Yaarit Hatuka)
+* mgr/volumes: Retain suid guid bits in clone (`pr#40268 <https://github.com/ceph/ceph/pull/40268>`_, Kotresh HR)
+* mgr: fix deadlock in ActivePyModules::get_osdmap() (`pr#39341 <https://github.com/ceph/ceph/pull/39341>`_, peng jiaqi)
+* mgr: relax osd ok-to-stop condition on degraded pgs (`pr#39887 <https://github.com/ceph/ceph/pull/39887>`_, Xuehan Xu)
+* mgr: update mon metadata when monmap is updated (`pr#39219 <https://github.com/ceph/ceph/pull/39219>`_, Kefu Chai)
+* mon/ConfigMap: fix stray option leak (`pr#40298 <https://github.com/ceph/ceph/pull/40298>`_, Sage Weil)
+* mon/MgrMonitor: populate available_modules from promote_standby() (`pr#40757 <https://github.com/ceph/ceph/pull/40757>`_, Sage Weil)
+* mon/MonClient: reset authenticate_err in _reopen_session() (`pr#41017 <https://github.com/ceph/ceph/pull/41017>`_, Ilya Dryomov)
+* mon/OSDMonitor: drop stale failure_info after a grace period (`pr#40558 <https://github.com/ceph/ceph/pull/40558>`_, Kefu Chai)
+* mon/OSDMonitor: fix safety/idempotency of {set,rm}-device-class (`pr#40276 <https://github.com/ceph/ceph/pull/40276>`_, Sage Weil)
+* mon: Modifying trim logic to change paxos_service_trim_max dynamically (`pr#40699 <https://github.com/ceph/ceph/pull/40699>`_, Aishwarya Mathuria)
+* mon: check mdsmap is resizeable before promoting standby-replay (`pr#40783 <https://github.com/ceph/ceph/pull/40783>`_, Patrick Donnelly)
+* monmaptool: Don't call set_port on an invalid address (`pr#40758 <https://github.com/ceph/ceph/pull/40758>`_, Brad Hubbard, Kefu Chai)
+* mount.ceph: collect v2 addresses for non-legacy ms_mode options (`pr#40763 <https://github.com/ceph/ceph/pull/40763>`_, Jeff Layton)
+* os/FileStore: don't propagate split/merge error to "create"/"remove" (`pr#40988 <https://github.com/ceph/ceph/pull/40988>`_, Mykola Golub)
+* os/FileStore: fix to handle readdir error correctly (`pr#41237 <https://github.com/ceph/ceph/pull/41237>`_, Misono Tomohiro)
+* os/bluestore/BlueFS: do not _flush_range deleted files (`pr#40793 <https://github.com/ceph/ceph/pull/40793>`_, weixinwei)
+* os/bluestore/BlueFS: use iterator_impl::copy instead of bufferlist::c_str() to avoid bufferlist rebuild (`pr#39884 <https://github.com/ceph/ceph/pull/39884>`_, weixinwei)
+* os/bluestore: Make Onode::put/get resiliant to split_cache (`pr#40441 <https://github.com/ceph/ceph/pull/40441>`_, Igor Fedotov, Adam Kupczyk)
+* os/bluestore: be more verbose in _open_super_meta by default (`pr#41061 <https://github.com/ceph/ceph/pull/41061>`_, Igor Fedotov)
+* osd/OSDMap: An empty bucket or OSD is not an error (`pr#39970 <https://github.com/ceph/ceph/pull/39970>`_, Brad Hubbard)
+* osd: add osd_fast_shutdown_notify_mon option (default false) (`issue#46978 <http://tracker.ceph.com/issues/46978>`_, `pr#40013 <https://github.com/ceph/ceph/pull/40013>`_, Mauricio Faria de Oliveira)
+* osd: compute OSD's space usage ratio via raw space utilization (`pr#41112 <https://github.com/ceph/ceph/pull/41112>`_, Igor Fedotov)
+* osd: do not dump an osd multiple times (`pr#40788 <https://github.com/ceph/ceph/pull/40788>`_, Xue Yantao)
+* osd: don't assert in-flight backfill is always in recovery list (`pr#41321 <https://github.com/ceph/ceph/pull/41321>`_, Mykola Golub)
+* osd: fix potential null pointer dereference when sending ping (`pr#40277 <https://github.com/ceph/ceph/pull/40277>`_, Mykola Golub)
+* osd: propagate base pool application_metadata to tiers (`pr#40274 <https://github.com/ceph/ceph/pull/40274>`_, Sage Weil)
+* packaging: require ceph-common for immutable object cache daemon (`pr#40666 <https://github.com/ceph/ceph/pull/40666>`_, Ilya Dryomov)
+* pybind/ceph_argparse.py: use a safe value for timeout (`pr#40476 <https://github.com/ceph/ceph/pull/40476>`_, Kefu Chai)
+* pybind/cephfs: DT_REG and DT_LNK values are wrong (`pr#40770 <https://github.com/ceph/ceph/pull/40770>`_, Varsha Rao)
+* pybind/mgr/balancer/module.py: assign weight-sets to all buckets before balancing (`pr#40127 <https://github.com/ceph/ceph/pull/40127>`_, Neha Ojha)
+* pybind/mgr/dashboard: bump flake8 to 3.9.0 (`pr#40492 <https://github.com/ceph/ceph/pull/40492>`_, Kefu Chai, Volker Theile)
+* qa/\*/thrash_cache_writeback_proxy_none.yaml: disable writeback overlay tests (`pr#39578 <https://github.com/ceph/ceph/pull/39578>`_, Neha Ojha)
+* qa/ceph-ansible: Update ansible version and ceph_stable_release (`pr#40945 <https://github.com/ceph/ceph/pull/40945>`_, Brad Hubbard)
+* qa/suites/krbd: address recent issues caused by newer kernels (`pr#40065 <https://github.com/ceph/ceph/pull/40065>`_, Ilya Dryomov)
+* qa/suites/rados/cephadm/upgrade: change starting version by distro (`pr#40364 <https://github.com/ceph/ceph/pull/40364>`_, Sage Weil)
+* qa/suites/rados/cephadm: rm ubuntu_18.04_podman (`pr#39949 <https://github.com/ceph/ceph/pull/39949>`_, Sebastian Wagner)
+* qa/suites/rados/singletone: whitelist MON_DOWN when injecting msgr errors (`pr#40138 <https://github.com/ceph/ceph/pull/40138>`_, Sage Weil)
+* qa/tasks/mgr/test_progress.py:  remove calling of _osd_in_out_completed_events_count() (`pr#40225 <https://github.com/ceph/ceph/pull/40225>`_, Kamoltat)
+* qa/tasks/mgr/test_progress: fix wait_until_equal (`pr#39360 <https://github.com/ceph/ceph/pull/39360>`_, Kamoltat)
+* qa/tasks/vstart_runner.py: start max required mgrs (`pr#40792 <https://github.com/ceph/ceph/pull/40792>`_, Alfonso Martínez)
+* qa/tests: advanced octopus initial version to 15.2.10 (`pr#41228 <https://github.com/ceph/ceph/pull/41228>`_, Yuri Weinstein)
+* qa: add sleep for blocklisting to take effect (`pr#40773 <https://github.com/ceph/ceph/pull/40773>`_, Patrick Donnelly)
+* qa: bump osd heartbeat grace for ffsb workload (`pr#40767 <https://github.com/ceph/ceph/pull/40767>`_, Patrick Donnelly)
+* qa: delete all fs during tearDown (`pr#40772 <https://github.com/ceph/ceph/pull/40772>`_, Patrick Donnelly)
+* qa: for the latest kclient it will also return EIO (`pr#40765 <https://github.com/ceph/ceph/pull/40765>`_, Xiubo Li)
+* qa: krbd_blkroset.t: update for separate hw and user read-only flags (`pr#40211 <https://github.com/ceph/ceph/pull/40211>`_, Ilya Dryomov)
+* rbd-mirror: bad state and crashes in snapshot-based mirroring (`pr#39961 <https://github.com/ceph/ceph/pull/39961>`_, Jason Dillaman)
+* rbd-mirror: delay update snapshot mirror image state (`pr#39967 <https://github.com/ceph/ceph/pull/39967>`_, Jason Dillaman)
+* rbd-mirror: fix UB while registering perf counters (`pr#40790 <https://github.com/ceph/ceph/pull/40790>`_, Arthur Outhenin-Chalandre)
+* rbd/bench: include used headers (`pr#40388 <https://github.com/ceph/ceph/pull/40388>`_, Kefu Chai)
+* rgw/amqp: fix race condition in amqp manager initialization (`pr#40382 <https://github.com/ceph/ceph/pull/40382>`_, Yuval Lifshitz)
+* rgw/http: add timeout to http client (`pr#40384 <https://github.com/ceph/ceph/pull/40384>`_, Yuval Lifshitz)
+* rgw/notification: support GetTopicAttributes API (`pr#40812 <https://github.com/ceph/ceph/pull/40812>`_, Yuval Lifshitz)
+* rgw/notification: trigger notifications on changes from any user (`pr#40029 <https://github.com/ceph/ceph/pull/40029>`_, Yuval Lifshitz)
+* rgw: Use correct bucket info when put or get large object with swift (`pr#40296 <https://github.com/ceph/ceph/pull/40296>`_, zhiming zhang, yupeng chen)
+* rgw: add MD5 in forward_request (`pr#39758 <https://github.com/ceph/ceph/pull/39758>`_, caolei)
+* rgw: allow rgw-orphan-list to handle intermediate files w/ binary data (`pr#39766 <https://github.com/ceph/ceph/pull/39766>`_, J. Eric Ivancich)
+* rgw: catch non int exception (`pr#39746 <https://github.com/ceph/ceph/pull/39746>`_, caolei)
+* rgw: during reshard lock contention, adjust logging (`pr#41157 <https://github.com/ceph/ceph/pull/41157>`_, J. Eric Ivancich)
+* rgw: fix sts get_session_token duration check failed (`pr#39954 <https://github.com/ceph/ceph/pull/39954>`_, yuliyang_yewu)
+* rgw: multisite: fix single-part-MPU object etag misidentify problem (`pr#39611 <https://github.com/ceph/ceph/pull/39611>`_, Yang Honggang)
+* rgw: objectlock: improve client error messages (`pr#40755 <https://github.com/ceph/ceph/pull/40755>`_, Matt Benjamin)
+* rgw: return error when trying to copy encrypted object without key (`pr#40672 <https://github.com/ceph/ceph/pull/40672>`_, Ilsoo Byun)
+* rgw: tooling to locate rgw objects with missing rados components (`pr#39785 <https://github.com/ceph/ceph/pull/39785>`_, Michael Kidd, J. Eric Ivancich)
+* run-make-check.sh: let ctest generate XML output (`pr#40406 <https://github.com/ceph/ceph/pull/40406>`_, Kefu Chai)
+* src/global/signal_handler.h: fix preprocessor logic for alpine (`pr#39940 <https://github.com/ceph/ceph/pull/39940>`_, Duncan Bellamy)
+* test/rbd-mirror: fix broken ceph_test_rbd_mirror_random_write (`pr#39965 <https://github.com/ceph/ceph/pull/39965>`_, Jason Dillaman)
+* test/rgw: test_datalog_autotrim filters out new entries (`pr#40673 <https://github.com/ceph/ceph/pull/40673>`_, Casey Bodley)
+* test: cancelling both noscrub \*and\* nodeep-scrub (`pr#40278 <https://github.com/ceph/ceph/pull/40278>`_, Ronen Friedman)
+* test: reduce number of threads to 32 in LibCephFS.ShutdownRace (`pr#40776 <https://github.com/ceph/ceph/pull/40776>`_, Jeff Layton)
+* test: use std::atomic<bool> instead of volatile for cb_done var (`pr#40708 <https://github.com/ceph/ceph/pull/40708>`_, Jeff Layton)
+* tests: ceph_test_rados_api_watch_notify: Allow for reconnect (`pr#40756 <https://github.com/ceph/ceph/pull/40756>`_, Brad Hubbard)
+* tools/cephfs: don't bind to public_addr (`pr#40762 <https://github.com/ceph/ceph/pull/40762>`_, "Yan, Zheng")
+* vstart.sh: disable "auth_allow_insecure_global_id_reclaim" (`pr#40958 <https://github.com/ceph/ceph/pull/40958>`_, Kefu Chai)
+
+v15.2.12 Octopus
+================
+
+This is a hotfix release addressing a number of security issues and regressions. We recommend all users update to this release.
+
+Changelog
+---------
+
+* mgr/dashboard: fix base-href: revert it to previous approach (`issue#50684 <https://tracker.ceph.com/issues/50684>`_, Avan Thakkar)
+* mgr/dashboard: fix cookie injection issue (:ref:`CVE-2021-3509`, Ernesto Puerta)
+* rgw: RGWSwiftWebsiteHandler::is_web_dir checks empty subdir_name (:ref:`CVE-2021-3531`, Felix Huettner)
+* rgw: sanitize \r in s3 CORSConfiguration's ExposeHeader (:ref:`CVE-2021-3524`, Sergey Bobrov, Casey Bodley)
+
+
+v15.2.11 Octopus
+================
+
+This is the 11th bugfix release in the Octopus stable series.  It addresses a
+security vulnerability in the Ceph authentication framework.
+
+We recommend all Octopus users upgrade.
+
+Security fixes
+--------------
+
+* This release includes a security fix that ensures the global_id
+  value (a numeric value that should be unique for every authenticated
+  client or daemon in the cluster) is reclaimed after a network
+  disconnect or ticket renewal in a secure fashion.  Two new health
+  alerts may appear during the upgrade indicating that there are
+  clients or daemons that are not yet patched with the appropriate
+  fix.
+
+  To temporarily mute the health alerts around insecure clients for the duration of the
+  upgrade, you may want to::
+
+    ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1h
+    ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1h
+
+  For more information, see :ref:`CVE-2021-20288`.
+
+
+v15.2.10 Octopus
+================
+
+This is the 10th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* The containers include an updated tcmalloc that avoids crashes seen on 15.2.9.  See `issue#49618 <https://tracker.ceph.com/issues/49618>`_ for details.
+
+* RADOS: BlueStore handling of huge(>4GB) writes from RocksDB to BlueFS has been fixed.
+
+* When upgrading from a previous cephadm release, systemctl may hang when trying to start or restart the monitoring containers. (This is caused by a change in the systemd unit to use ``type=forking``.) After the upgrade, please run::
+
+    ceph orch redeploy nfs
+    ceph orch redeploy iscsi
+    ceph orch redeploy node-exporter
+    ceph orch redeploy prometheus
+    ceph orch redeploy grafana
+    ceph orch redeploy alertmanager
+
+Changelog
+---------
+
+* octopus: .github: add workflow for adding label and milestone (`pr#39890 <https://github.com/ceph/ceph/pull/39890>`_, Kefu Chai, Ernesto Puerta)
+* octopus: ceph-volume: Fix usage of is_lv (`pr#39220 <https://github.com/ceph/ceph/pull/39220>`_, Michał Nasiadka)
+* octopus: ceph-volume: Update batch.py (`pr#39469 <https://github.com/ceph/ceph/pull/39469>`_, shenjiatong)
+* octopus: ceph-volume: add some flexibility to bytes_to_extents (`pr#39271 <https://github.com/ceph/ceph/pull/39271>`_, Jan Fajerski)
+* octopus: ceph-volume: pass --filter-for-batch from drive-group subcommand (`pr#39523 <https://github.com/ceph/ceph/pull/39523>`_, Jan Fajerski)
+* octopus: cephadm: Delete the unnecessary error line in open_ports (`pr#39633 <https://github.com/ceph/ceph/pull/39633>`_, Donggyu Park)
+* octopus: cephadm: fix 'inspect' and 'pull' (`pr#39715 <https://github.com/ceph/ceph/pull/39715>`_, Sage Weil)
+* octopus: cephfs: pybind/ceph_volume_client: Update the 'volumes' key to 'subvolumes' in auth-metadata file (`pr#39906 <https://github.com/ceph/ceph/pull/39906>`_, Kotresh HR)
+* octopus: cmake: boost>=1.74 adds BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT to radosgw (`pr#39885 <https://github.com/ceph/ceph/pull/39885>`_, Casey Bodley)
+* octopus: librbd: allow disabling journaling for snapshot based mirroring image (`pr#39864 <https://github.com/ceph/ceph/pull/39864>`_, Mykola Golub)
+* octopus: librbd: correct incremental deep-copy object-map inconsistencies (`pr#39577 <https://github.com/ceph/ceph/pull/39577>`_, Mykola Golub, Jason Dillaman)
+* octopus: librbd: don't log error if get mirror status fails due to mirroring disabled (`pr#39862 <https://github.com/ceph/ceph/pull/39862>`_, Mykola Golub)
+* octopus: librbd: use on-disk image name when storing mirror snapshot state (`pr#39866 <https://github.com/ceph/ceph/pull/39866>`_, Mykola Golub)
+* octopus: mgr/dashboard/monitoring: upgrade Grafana version due to CVE-2020-13379 (`pr#39306 <https://github.com/ceph/ceph/pull/39306>`_, Alfonso Martínez)
+* octopus: mgr/dashboard: CLI commands: read passwords from file (`pr#39436 <https://github.com/ceph/ceph/pull/39436>`_, Ernesto Puerta, Alfonso Martínez, Juan Miguel Olmo Martínez)
+* octopus: mgr/dashboard: Fix for incorrect validation in rgw user form (`pr#39027 <https://github.com/ceph/ceph/pull/39027>`_, Nizamudeen A)
+* octopus: mgr/dashboard: Fix missing root path of each session for CephFS (`pr#39868 <https://github.com/ceph/ceph/pull/39868>`_, Yongseok Oh)
+* octopus: mgr/dashboard: Monitoring alert badge includes suppressed alerts (`pr#39512 <https://github.com/ceph/ceph/pull/39512>`_, Aashish Sharma)
+* octopus: mgr/dashboard: add ssl verify option for prometheus and alert manager (`pr#39872 <https://github.com/ceph/ceph/pull/39872>`_, Jean "henyxia" Wasilewski)
+* octopus: mgr/dashboard: avoid using document.write() (`pr#39527 <https://github.com/ceph/ceph/pull/39527>`_, Avan Thakkar)
+* octopus: mgr/dashboard: delete EOF when reading passwords from file (`pr#40155 <https://github.com/ceph/ceph/pull/40155>`_, Alfonso Martínez)
+* octopus: mgr/dashboard: fix MTU Mismatch alert (`pr#39854 <https://github.com/ceph/ceph/pull/39854>`_, Aashish Sharma)
+* octopus: mgr/dashboard: fix issues related with PyJWT versions >=2.0.0 (`pr#39836 <https://github.com/ceph/ceph/pull/39836>`_, Alfonso Martínez)
+* octopus: mgr/dashboard: fix tooltip for Provisioned/Total Provisioned fields (`pr#39645 <https://github.com/ceph/ceph/pull/39645>`_, Avan Thakkar)
+* octopus: mgr/dashboard: prometheus alerting: add some leeway for package drops and errors (`pr#39507 <https://github.com/ceph/ceph/pull/39507>`_, Patrick Seidensal)
+* octopus: mgr/dashboard: report mgr fsid (`pr#39852 <https://github.com/ceph/ceph/pull/39852>`_, Ernesto Puerta)
+* octopus: mgr/dashboard: set security headers (`pr#39627 <https://github.com/ceph/ceph/pull/39627>`_, Avan Thakkar)
+* octopus: mgr/dashboard: trigger alert if some nodes have a MTU different than the median value (`pr#39103 <https://github.com/ceph/ceph/pull/39103>`_, Aashish Sharma)
+* octopus: mgr/dashboard:minimize console log traces of Ceph backend API tests (`pr#39545 <https://github.com/ceph/ceph/pull/39545>`_, Aashish Sharma)
+* octopus: mgr/rbd_support: create mirror snapshots asynchronously (`pr#39376 <https://github.com/ceph/ceph/pull/39376>`_, Mykola Golub, Kefu Chai)
+* octopus: mgr/rbd_support: mirror snapshot schedule should skip non-primary images (`pr#39863 <https://github.com/ceph/ceph/pull/39863>`_, Mykola Golub)
+* octopus: mgr/volume: subvolume auth_id management and few bug fixes (`pr#39390 <https://github.com/ceph/ceph/pull/39390>`_, Rishabh Dave, Patrick Donnelly, Kotresh HR, Ramana Raja)
+* octopus: mgr/zabbix: format ceph.[{#POOL},percent_used as float (`pr#39235 <https://github.com/ceph/ceph/pull/39235>`_, Kefu Chai)
+* octopus: os/bluestore: Add option to check BlueFS reads (`pr#39754 <https://github.com/ceph/ceph/pull/39754>`_, Adam Kupczyk)
+* octopus: os/bluestore: fix huge reads/writes at BlueFS (`pr#39701 <https://github.com/ceph/ceph/pull/39701>`_, Jianpeng Ma, Igor Fedotov)
+* octopus: os/bluestore: introduce bluestore_rocksdb_options_annex config parame… (`pr#39325 <https://github.com/ceph/ceph/pull/39325>`_, Igor Fedotov)
+* octopus: qa/suites/rados/dashboard: whitelist TELEMETRY_CHANGED (`pr#39704 <https://github.com/ceph/ceph/pull/39704>`_, Sage Weil)
+* octopus: qa/suites/upgrade: s/whitelist/ignorelist for octopus specific tests (`pr#40074 <https://github.com/ceph/ceph/pull/40074>`_, Deepika Upadhyay)
+* octopus: qa: use normal build for valgrind (`pr#39583 <https://github.com/ceph/ceph/pull/39583>`_, Sage Weil)
+* octopus: rbd-mirror: reset update_status_task pointer in timer thread (`pr#39867 <https://github.com/ceph/ceph/pull/39867>`_, Mykola Golub)
+* octopus: rgw: fix trailing null in object names of multipart reuploads (`pr#39277 <https://github.com/ceph/ceph/pull/39277>`_, Casey Bodley)
+* octopus: rgw: radosgw-admin: clarify error when email address already in use (`pr#39662 <https://github.com/ceph/ceph/pull/39662>`_, Matthew Vernon)
+* octopus: whitelist -> ignorelist for qa/\* only (`pr#39534 <https://github.com/ceph/ceph/pull/39534>`_, Neha Ojha, Sage Weil)
+* qa/tests: fixed branch entry (`pr#39819 <https://github.com/ceph/ceph/pull/39819>`_, Yuri Weinstein)
+
+
+v15.2.9 Octopus
+===============
+
+This is the 9th backport release in the Octopus series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* MGR: progress module can now be turned on/off, using the commands:
+  ``ceph progress on`` and ``ceph progress off``.
+
+* OSD: PG removal has been optimized in this release.
+
+Changelog
+---------
+
+* octopus: Do not add sensitive information in Ceph log files (`pr#38620 <https://github.com/ceph/ceph/pull/38620>`_, Neha Ojha)
+* octopus: PendingReleaseNotes: mgr/pg_autoscaler (`pr#39393 <https://github.com/ceph/ceph/pull/39393>`_, Kamoltat)
+* octopus: Revert "mgr/pg_autoscaler: avoid scale-down until there is pressure" (`pr#39560 <https://github.com/ceph/ceph/pull/39560>`_, Neha Ojha)
+* octopus: bluestore: Make mempool assignment same after bufferlist rebuild (`pr#38429 <https://github.com/ceph/ceph/pull/38429>`_, Adam Kupczyk)
+* octopus: bluestore: Support flock retry (`pr#37860 <https://github.com/ceph/ceph/pull/37860>`_, wanghongxu)
+* octopus: bluestore: attach csum for compressed blobs (`pr#37861 <https://github.com/ceph/ceph/pull/37861>`_, Igor Fedotov)
+* octopus: bluestore: fix "end reached" check in collection_list_legacy (`pr#38098 <https://github.com/ceph/ceph/pull/38098>`_, Mykola Golub)
+* octopus: bluestore: provide a different name for fallback allocator (`pr#37794 <https://github.com/ceph/ceph/pull/37794>`_, Igor Fedotov)
+* octopus: build/ops: doc: pass --use-feature=2020-resolver to pip (`pr#37859 <https://github.com/ceph/ceph/pull/37859>`_, Kefu Chai)
+* octopus: ceph-volume: lvm/create.py: fix a typo in the help message (`pr#38425 <https://github.com/ceph/ceph/pull/38425>`_, ZhenLiu94)
+* octopus: cephadm: Don't make sysctl spam the log file (`pr#39020 <https://github.com/ceph/ceph/pull/39020>`_, Sebastian Wagner)
+* octopus: cephadm: Revert "spec: Podman (temporarily) requires apparmor-abstractions on suse" (`pr#37766 <https://github.com/ceph/ceph/pull/37766>`_, Nathan Cutler)
+* octopus: cephadm: Various properties like 'last_refresh' do not contain timezone (`pr#39059 <https://github.com/ceph/ceph/pull/39059>`_, Volker Theile)
+* octopus: cephadm: batch backport January (1) (`pr#38782 <https://github.com/ceph/ceph/pull/38782>`_, Ricardo Marques, Patrick Donnelly, Ken Dreyer, Paul Cuzner, Daniel-Pivonka, Juan Miguel Olmo Martínez, Volker Theile, Sebastian Wagner, Varsha Rao, Adam King, Patrick Seidensal, Michael Fritch, Dan Mick)
+* octopus: cephadm: fix rgw osd cap tag (`pr#39170 <https://github.com/ceph/ceph/pull/39170>`_, Patrick Donnelly)
+* octopus: cephadm: make "ceph orch {restart|...}" asynchronous (`pr#39018 <https://github.com/ceph/ceph/pull/39018>`_, Sebastian Wagner)
+* octopus: cephadm: silence "Failed to evict container" log msg (`pr#39166 <https://github.com/ceph/ceph/pull/39166>`_, Sebastian Wagner, Sage Weil)
+* octopus: cephadm: use `apt-get` for package install/update (`pr#39297 <https://github.com/ceph/ceph/pull/39297>`_, Michael Fritch)
+* octopus: cephfs: client: add ceph.{cluster_fsid/client_id} vxattrs suppport (`pr#39000 <https://github.com/ceph/ceph/pull/39000>`_, Xiubo Li)
+* octopus: cephfs: client: check rdonly file handle on truncate (`pr#38424 <https://github.com/ceph/ceph/pull/38424>`_, Patrick Donnelly)
+* octopus: cephfs: client: do not use g_conf().get_val<>() in libcephfs (`pr#38466 <https://github.com/ceph/ceph/pull/38466>`_, Xiubo Li)
+* octopus: cephfs: client: ensure we take Fs caps when fetching directory link count from cached inode (`pr#38949 <https://github.com/ceph/ceph/pull/38949>`_, Jeff Layton)
+* octopus: cephfs: client: increment file position on _read_sync near eof (`pr#37989 <https://github.com/ceph/ceph/pull/37989>`_, Patrick Donnelly)
+* octopus: cephfs: client: set CEPH_STAT_RSTAT mask for dir in readdir_r_cb (`pr#38947 <https://github.com/ceph/ceph/pull/38947>`_, chencan)
+* octopus: cephfs: mds: dir->mark_new() should together with dir->mark_dirty() (`pr#38352 <https://github.com/ceph/ceph/pull/38352>`_, "Yan, Zheng")
+* octopus: cephfs: mds: move start_files_to_recover() to recovery_done (`pr#37985 <https://github.com/ceph/ceph/pull/37985>`_, Simon Gao)
+* octopus: cephfs: osdc: restart read on truncate/discard (`pr#37987 <https://github.com/ceph/ceph/pull/37987>`_, Patrick Donnelly)
+* octopus: cephfs: release client dentry_lease before send caps release to mds (`pr#38349 <https://github.com/ceph/ceph/pull/38349>`_, Wei Qiaomiao)
+* octopus: client: dump which fs is used by client for multiple-fs (`pr#38551 <https://github.com/ceph/ceph/pull/38551>`_, Zhi Zhang)
+* octopus: cmake: add empty RPATH to ceph-diff-sorted (`pr#38847 <https://github.com/ceph/ceph/pull/38847>`_, Nathan Cutler)
+* octopus: cmake: define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT for Boost.Asio users (`pr#38759 <https://github.com/ceph/ceph/pull/38759>`_, Kefu Chai)
+* octopus: cmake: detect and use sigdescr_np() if available (`pr#38951 <https://github.com/ceph/ceph/pull/38951>`_, David Disseldorp)
+* octopus: do_cmake.sh: use python-3.9 with fedora version 33 (`pr#38943 <https://github.com/ceph/ceph/pull/38943>`_, Sunny Kumar)
+* octopus: doc: document MDS cache configuration (`pr#38202 <https://github.com/ceph/ceph/pull/38202>`_, Patrick Donnelly)
+* octopus: global: reexpand conf meta in child process (`pr#38340 <https://github.com/ceph/ceph/pull/38340>`_, Xiubo Li)
+* octopus: install-deps.sh: Make powertools repo case insensitive (`pr#38808 <https://github.com/ceph/ceph/pull/38808>`_, Brad Hubbard)
+* octopus: krbd: add support for msgr2 (kernel 5.11) (`pr#39203 <https://github.com/ceph/ceph/pull/39203>`_, Ilya Dryomov)
+* octopus: librbd: clear implicitly enabled feature bits when creating images (`pr#39122 <https://github.com/ceph/ceph/pull/39122>`_, Jason Dillaman)
+* octopus: librbd: fix regression in object map diff request (`pr#38455 <https://github.com/ceph/ceph/pull/38455>`_, Mykola Golub, Jason Dillaman)
+* octopus: librbd: update hidden global config when removing pool config override (`pr#38343 <https://github.com/ceph/ceph/pull/38343>`_, Jason Dillaman)
+* octopus: mds: dump granular cap info in mds_sessions (`pr#37362 <https://github.com/ceph/ceph/pull/37362>`_, Yanhu Cao)
+* octopus: mds: provide altrenatives to increase the total cephfs subvolume snapshot counts to greater than the current 400 across a Cephfs volume (`pr#38553 <https://github.com/ceph/ceph/pull/38553>`_, "Yan, Zheng")
+* octopus: mds: throttle cap acquisition via readdir (`pr#38095 <https://github.com/ceph/ceph/pull/38095>`_, Kotresh HR)
+* octopus: mgr/ActivePyModules.cc: always release GIL before attempting to acquire a lock (`pr#38801 <https://github.com/ceph/ceph/pull/38801>`_, Cory Snyder)
+* octopus: mgr/balancer: fix available pgs sent to calc_pg_upmaps (`pr#38337 <https://github.com/ceph/ceph/pull/38337>`_, Dan van der Ster)
+* octopus: mgr/cephadm: fix host refresh (`pr#39532 <https://github.com/ceph/ceph/pull/39532>`_, Sage Weil)
+* octopus: mgr/cephadm: lock multithreaded access to OSDRemovalQueue (`pr#39019 <https://github.com/ceph/ceph/pull/39019>`_, Sebastian Wagner)
+* octopus: mgr/cephadm: raise HEALTH_WARN when cephadm daemon in 'error' state (`pr#39169 <https://github.com/ceph/ceph/pull/39169>`_, Sage Weil)
+* octopus: mgr/cephadm: tolerate old host inventory without 'hostname' key (`pr#39167 <https://github.com/ceph/ceph/pull/39167>`_, Sage Weil)
+* octopus: mgr/cephadm: try again calling ceph-volume without --filter-for-batch (`pr#39300 <https://github.com/ceph/ceph/pull/39300>`_, Sebastian Wagner)
+* octopus: mgr/crash: Serialize command handling (`pr#38592 <https://github.com/ceph/ceph/pull/38592>`_, Boris Ranto)
+* octopus: mgr/dashboard: Add clay plugin support (`pr#38489 <https://github.com/ceph/ceph/pull/38489>`_, Stephan Müller)
+* octopus: mgr/dashboard: Create Ceph services via Orchestrator by using ServiceSpec (`pr#38888 <https://github.com/ceph/ceph/pull/38888>`_, Volker Theile)
+* octopus: mgr/dashboard: Display a warning message in Dashboard when debug mode is enabled (`pr#38798 <https://github.com/ceph/ceph/pull/38798>`_, Volker Theile)
+* octopus: mgr/dashboard: Drop invalid RGW client instances, improve logging (`pr#38583 <https://github.com/ceph/ceph/pull/38583>`_, Volker Theile)
+* octopus: mgr/dashboard: Fix CRUSH map viewer VirtualScroll (`pr#38607 <https://github.com/ceph/ceph/pull/38607>`_, Avan Thakkar)
+* octopus: mgr/dashboard: Fix for misleading "Orchestrator is not available" error (`pr#38598 <https://github.com/ceph/ceph/pull/38598>`_, Nizamudeen A)
+* octopus: mgr/dashboard: Fixing dashboard logs e2e test (`pr#38797 <https://github.com/ceph/ceph/pull/38797>`_, Nizamudeen A)
+* octopus: mgr/dashboard: Prevent table items from getting selected while expanding (`pr#37930 <https://github.com/ceph/ceph/pull/37930>`_, Nizamudeen A)
+* octopus: mgr/dashboard: RGW User Form is validating disabled fields (`pr#38594 <https://github.com/ceph/ceph/pull/38594>`_, Aashish Sharma)
+* octopus: mgr/dashboard: Temporary User Lockout if 10 Invalid Login attempts (`pr#38810 <https://github.com/ceph/ceph/pull/38810>`_, Nizamudeen A)
+* octopus: mgr/dashboard: The /rgw/status endpoint does not check for running service (`pr#38770 <https://github.com/ceph/ceph/pull/38770>`_, Volker Theile)
+* octopus: mgr/dashboard: Updating the inbuilt ssl providers error (`pr#38508 <https://github.com/ceph/ceph/pull/38508>`_, Nizamudeen A)
+* octopus: mgr/dashboard: Use secure cookies to store JWT Token (`pr#39120 <https://github.com/ceph/ceph/pull/39120>`_, Avan Thakkar, Aashish Sharma)
+* octopus: mgr/dashboard: add `--ssl` to `ng serve` (`pr#38973 <https://github.com/ceph/ceph/pull/38973>`_, Tatjana Dehler)
+* octopus: mgr/dashboard: adjust refresh intervals of Services and Daemons (`pr#38597 <https://github.com/ceph/ceph/pull/38597>`_, Kiefer Chang)
+* octopus: mgr/dashboard: allow selecting all daemons for Orchestrator NFS clusters (`pr#38496 <https://github.com/ceph/ceph/pull/38496>`_, Kiefer Chang)
+* octopus: mgr/dashboard: assign flags to single OSDs (`pr#38469 <https://github.com/ceph/ceph/pull/38469>`_, Tatjana Dehler)
+* octopus: mgr/dashboard: disable cluster selection in NFS export editing form (`pr#37969 <https://github.com/ceph/ceph/pull/37969>`_, Kiefer Chang)
+* octopus: mgr/dashboard: display placement column in service table (`pr#38336 <https://github.com/ceph/ceph/pull/38336>`_, Volker Theile)
+* octopus: mgr/dashboard: enable different URL for users of browser to Grafana (`pr#38761 <https://github.com/ceph/ceph/pull/38761>`_, Patrick Seidensal)
+* octopus: mgr/dashboard: fix Reads/Writes ratio of Clients IOPS donut chart (`pr#38867 <https://github.com/ceph/ceph/pull/38867>`_, Kiefer Chang)
+* octopus: mgr/dashboard: remove pyOpenSSL version pinning (`pr#38503 <https://github.com/ceph/ceph/pull/38503>`_, Kiefer Chang)
+* octopus: mgr/dashboard: test_standby\* (tasks.mgr.test_dashboard.TestDashboard) failed locally (`pr#38526 <https://github.com/ceph/ceph/pull/38526>`_, Volker Theile)
+* octopus: mgr/pg_autoscaler: avoid scale-down until there is pressure (`pr#39248 <https://github.com/ceph/ceph/pull/39248>`_, Kamoltat)
+* octopus: mgr/progress: introduce turn off/on feature (`pr#39289 <https://github.com/ceph/ceph/pull/39289>`_, kamoltat)
+* octopus: mgr/prometheus: Fix 'pool filling up' with >50% usage (`pr#38593 <https://github.com/ceph/ceph/pull/38593>`_, Daniël Vos)
+* octopus: mgr/prometheus: Sync and backport prometheus fixes (`pr#38333 <https://github.com/ceph/ceph/pull/38333>`_, Paul Cuzner, Boris Ranto, Kefu Chai, Ken Dreyer)
+* octopus: mgr/rbd_support: store global schedule without localized prefix (`pr#38342 <https://github.com/ceph/ceph/pull/38342>`_, Mykola Golub)
+* octopus: mgr/restful: fix TypeError occurring in _gather_osds() (`issue#48488 <http://tracker.ceph.com/issues/48488>`_, `pr#38595 <https://github.com/ceph/ceph/pull/38595>`_, Jerry Pu)
+* octopus: mgr/volumes: Add a per subvolume trash (`pr#38612 <https://github.com/ceph/ceph/pull/38612>`_, Venky Shankar, Shyamsundar Ranganathan)
+* octopus: mgr/volumes: Implement subvolume version v2 (`pr#36803 <https://github.com/ceph/ceph/pull/36803>`_, Shyamsundar Ranganathan)
+* octopus: mgr: Fix for dashboard/prometheus failure due to laggy pg state (`pr#38596 <https://github.com/ceph/ceph/pull/38596>`_, Alexander Sushko)
+* octopus: mgr: don't update osd stat which is already out (`pr#38353 <https://github.com/ceph/ceph/pull/38353>`_, Zhi Zhang)
+* octopus: mon: paxos: Delete logger in destructor (`pr#39161 <https://github.com/ceph/ceph/pull/39161>`_, Brad Hubbard)
+* octopus: mon: validate crush-failure-domain (`pr#38347 <https://github.com/ceph/ceph/pull/38347>`_, Prashant Dhange)
+* octopus: monitoring: Use null yaxes min for OSD read latency (`pr#37960 <https://github.com/ceph/ceph/pull/37960>`_, Seena Fallah)
+* octopus: msg/async/ProtocolV2: allow rxbuf/txbuf get bigger in testing, again (`pr#38267 <https://github.com/ceph/ceph/pull/38267>`_, Ilya Dryomov)
+* octopus: ocf: add support for mapping images within an RBD namespace (`pr#39046 <https://github.com/ceph/ceph/pull/39046>`_, Jason Dillaman)
+* octopus: os/bluestore: detect and fix "zombie" spanning blobs using fsck (`pr#39256 <https://github.com/ceph/ceph/pull/39256>`_, Igor Fedotov)
+* octopus: os/bluestore: fix huge (>4GB) bluefs reads (`pr#39253 <https://github.com/ceph/ceph/pull/39253>`_, Igor Fedotov)
+* octopus: os/bluestore: fix inappropriate ENOSPC from avl/hybrid allocator (`pr#38474 <https://github.com/ceph/ceph/pull/38474>`_, Igor Fedotov)
+* octopus: os/bluestore: fix segfault on out-of-bound offset provided to  claim_free_to_right() call (`pr#38428 <https://github.com/ceph/ceph/pull/38428>`_, Igor Fedotov)
+* octopus: os/bluestore: fixing onode pinning and more (`pr#39230 <https://github.com/ceph/ceph/pull/39230>`_, Adam Kupczyk, Igor Fedotov)
+* octopus: osd: fix bluestore bitmap allocator calculate wrong last_pos with hint (`pr#38430 <https://github.com/ceph/ceph/pull/38430>`_, Xue Yantao)
+* octopus: osd: optimize PG removal (part1) (`pr#38477 <https://github.com/ceph/ceph/pull/38477>`_, Igor Fedotov)
+* octopus: pybind/cephfs: fix missing terminating NULL char in readlink()'s C string (`pr#38893 <https://github.com/ceph/ceph/pull/38893>`_, Tuan Hoang)
+* octopus: pybind/mgr/rbd_support: delay creation of progress module events (`pr#38344 <https://github.com/ceph/ceph/pull/38344>`_, Jason Dillaman)
+* octopus: python-common/drivegroups: avoid dropping "rotational: 0" from Device Selection (`issue#49014 <http://tracker.ceph.com/issues/49014>`_, `pr#39171 <https://github.com/ceph/ceph/pull/39171>`_, Lukas Stockner)
+* octopus: python-common: fix test_datetime_to_str_2 on non-UTC hosts (`pr#39296 <https://github.com/ceph/ceph/pull/39296>`_, Sage Weil)
+* octopus: qa/cephadm: Add yaml output to smoke test (`pr#39168 <https://github.com/ceph/ceph/pull/39168>`_, Sebastian Wagner)
+* octopus: qa/mgr: mgr_test_case: raise SkipTest instead of calling skipTest() (`pr#38165 <https://github.com/ceph/ceph/pull/38165>`_, Rishabh Dave)
+* octopus: qa/tasks/cephfs/nfs: Check if host ip is in cluster info output (`pr#39004 <https://github.com/ceph/ceph/pull/39004>`_, Varsha Rao)
+* octopus: qa/tasks/mgr/test_progress: update test suite to check for specific progress events (`pr#38555 <https://github.com/ceph/ceph/pull/38555>`_, Kamoltat)
+* octopus: qa/tasks/vstart_runner: do not teardown test_path if "create-cluster-only" (`pr#39540 <https://github.com/ceph/ceph/pull/39540>`_, Kefu Chai)
+* octopus: qa/workunits/rbd: fix permission issue when removing mirror peer (`pr#38341 <https://github.com/ceph/ceph/pull/38341>`_, Jason Dillaman)
+* octopus: qa: accept timeout argument in run_shell (`pr#38550 <https://github.com/ceph/ceph/pull/38550>`_, Patrick Donnelly)
+* octopus: qa: ignore evicted client warnings (`pr#38422 <https://github.com/ceph/ceph/pull/38422>`_, Patrick Donnelly)
+* octopus: qa: ignore logrotate state rename error (`pr#37690 <https://github.com/ceph/ceph/pull/37690>`_, Patrick Donnelly)
+* octopus: qa: krbd_stable_pages_required.sh: move to stable_writes attribute (`pr#39321 <https://github.com/ceph/ceph/pull/39321>`_, Ilya Dryomov)
+* octopus: qa: tox failures (`pr#38626 <https://github.com/ceph/ceph/pull/38626>`_, Patrick Donnelly)
+* octopus: qa: unmount volumes before removal (`pr#38688 <https://github.com/ceph/ceph/pull/38688>`_, Patrick Donnelly)
+* octopus: rgw/multisite: Verify if the synced object is identical to source (`pr#38981 <https://github.com/ceph/ceph/pull/38981>`_, Prasad Krishnan, Casey Bodley)
+* octopus: rgw/rgw-admin: fixes BucketInfo for missing buckets (`pr#38184 <https://github.com/ceph/ceph/pull/38184>`_, Nick Janus, caolei)
+* octopus: rgw: S3 Put Bucket Policy should return 204 on success (`pr#38420 <https://github.com/ceph/ceph/pull/38420>`_, Matthew Oliver)
+* octopus: rgw: adding user related web token claims to ops log (`pr#38970 <https://github.com/ceph/ceph/pull/38970>`_, Pritha Srivastava)
+* octopus: rgw: avoid expiration early triggering caused by overflow (`pr#38421 <https://github.com/ceph/ceph/pull/38421>`_, jiahuizeng)
+* octopus: rgw: cls/rgw/cls_rgw.cc: fix multiple lastest version problem (`pr#38086 <https://github.com/ceph/ceph/pull/38086>`_, Yang Honggang, Ruan Zitao)
+* octopus: rgw: cls/user: set from_index for reset stats calls (`pr#38821 <https://github.com/ceph/ceph/pull/38821>`_, Mykola Golub, Abhishek Lekshmanan)
+* octopus: rgw: distribute cache for exclusive put (`pr#38971 <https://github.com/ceph/ceph/pull/38971>`_, Or Friedmann)
+* octopus: rgw: fix bucket limit check fill_status warnings (`issue#40255 <http://tracker.ceph.com/issues/40255>`_, `pr#38826 <https://github.com/ceph/ceph/pull/38826>`_, Paul Emmerich)
+* octopus: rgw: fix invalid payload issue when serving s3website error page (`pr#38339 <https://github.com/ceph/ceph/pull/38339>`_, Ilsoo Byun)
+* octopus: rgw: keep syncstopped flag when copying bucket shard headers (`pr#38338 <https://github.com/ceph/ceph/pull/38338>`_, Ilsoo Byun)
+* octopus: rgw: lc: correctly dimension lc shard index vector (`pr#38824 <https://github.com/ceph/ceph/pull/38824>`_, Matt Benjamin)
+* octopus: rgw_file: return common_prefixes in lexical order (`pr#38829 <https://github.com/ceph/ceph/pull/38829>`_, Matt Benjamin)
+* octopus: rpm,deb: change sudoers file mode to 440 (`pr#38427 <https://github.com/ceph/ceph/pull/38427>`_, David Turner)
+* octopus: rpm: require smartmontools on SUSE (`pr#38755 <https://github.com/ceph/ceph/pull/38755>`_, Nathan Cutler)
+* octopus: test/run-cli-tests: use cram from github (`pr#39071 <https://github.com/ceph/ceph/pull/39071>`_, Kefu Chai)
+* octopus: tests: qa/task/cephadm: run cephadm only on bootstrap_remote (`pr#38040 <https://github.com/ceph/ceph/pull/38040>`_, Kyr Shatskyy)
+
+v15.2.8 Octopus
+===============
+
+This is the 8th backport release in the Octopus series. This release fixes
+a security flaw in CephFS and includes a number of bug fixes. We recommend
+users to update to this release.
+
+Notable Changes
+---------------
+
+* CVE-2020-27781 : OpenStack Manila use of ceph_volume_client.py library allowed
+  tenant access to any Ceph credential's secret. (Kotresh Hiremath Ravishankar,
+  Ramana Raja)
+
+* ceph-volume: The ``lvm batch`` subcommand received a major rewrite. This closed
+  a number of bugs and improves usability in terms of size specification and
+  calculation, as well as idempotency behaviour and disk replacement process.
+  Please refer to https://docs.ceph.com/en/latest/ceph-volume/lvm/batch/ for
+  more detailed information.
+
+* MON: The cluster log now logs health detail every ``mon_health_to_clog_interval``,
+  which has been changed from 1hr to 10min. Logging of health detail will be
+  skipped if there is no change in health summary since last known.
+
+* The ``ceph df`` command now lists the number of pgs in each pool.
+
+* The ``bluefs_preextend_wal_files`` option has been removed.
+
+* It is now possible to specify the initial monitor to contact for Ceph tools
+  and daemons using the ``mon_host_override`` config option or
+  ``--mon-host-override <ip>`` command-line switch. This generally should only
+  be used for debugging and only affects initial communication with Ceph's
+  monitor cluster.
+
+
+Changelog
+---------
+
+* pybind/ceph_volume_client: disallow authorize on existing auth ids (Kotresh
+  Hiremath Ravishankar, Ramana Raja)
+* Enable per-RBD image monitoring (`pr#37697 <https://github.com/ceph/ceph/pull/37697>`_, Patrick Seidensal)
+* [ceph-volume]: remove unneeded call to get_devices() (`pr#37412 <https://github.com/ceph/ceph/pull/37412>`_, Marc Gariepy)
+* bluestore: fix collection_list ordering (`pr#37048 <https://github.com/ceph/ceph/pull/37048>`_, Mykola Golub)
+* bluestore: mempool's finer granularity + adding missed structs (`pr#37264 <https://github.com/ceph/ceph/pull/37264>`_, Deepika Upadhyay, Igor Fedotov, Adam Kupczyk)
+* bluestore: remove preextended WAL support (`pr#37373 <https://github.com/ceph/ceph/pull/37373>`_, Igor Fedotov)
+* ceph-volume batch: reject partitions in argparser (`pr#38280 <https://github.com/ceph/ceph/pull/38280>`_, Jan Fajerski)
+* ceph-volume inventory: make libstoragemgmt data retrieval optional (`pr#38299 <https://github.com/ceph/ceph/pull/38299>`_, Jan Fajerski)
+* ceph-volume: add libstoragemgmt support (`pr#36852 <https://github.com/ceph/ceph/pull/36852>`_, Paul Cuzner, Satoru Takeuchi)
+* ceph-volume: add no-systemd argument to zap (`pr#37722 <https://github.com/ceph/ceph/pull/37722>`_, wanghongxu)
+* ceph-volume: avoid format strings for now (`pr#37345 <https://github.com/ceph/ceph/pull/37345>`_, Jan Fajerski)
+* ceph-volume: consume mount opt in simple activate (`pr#38014 <https://github.com/ceph/ceph/pull/38014>`_, Dimitri Savineau)
+* ceph-volume: fix filestore/dmcrypt activate (`pr#38199 <https://github.com/ceph/ceph/pull/38199>`_, Guillaume Abrioux)
+* ceph-volume: fix journal size argument not work (`pr#37344 <https://github.com/ceph/ceph/pull/37344>`_, wanghongxu)
+* ceph-volume: fix lvm batch auto with full SSDs (`pr#38045 <https://github.com/ceph/ceph/pull/38045>`_, Dimitri Savineau, Guillaume Abrioux)
+* ceph-volume: fix simple activate when legacy osd (`pr#37194 <https://github.com/ceph/ceph/pull/37194>`_, Guillaume Abrioux)
+* ceph-volume: implement the --log-level flag (`pr#38426 <https://github.com/ceph/ceph/pull/38426>`_, Andrew Schoen)
+* ceph-volume: major batch refactor (`pr#37520 <https://github.com/ceph/ceph/pull/37520>`_, Jan Fajerski, Joshua Schmid)
+* ceph-volume: prepare: use \*-slots arguments for implicit sizing (`pr#38205 <https://github.com/ceph/ceph/pull/38205>`_, Jan Fajerski)
+* ceph-volume: remove mention of dmcache from docs and help text (`pr#38047 <https://github.com/ceph/ceph/pull/38047>`_, Dimitri Savineau, Andrew Schoen)
+* ceph-volume: retry when acquiring lock fails (`pr#36925 <https://github.com/ceph/ceph/pull/36925>`_, Sébastien Han)
+* ceph-volume: simple scan should ignore tmpfs (`pr#36953 <https://github.com/ceph/ceph/pull/36953>`_, Andrew Schoen)
+* ceph-volume: support for mpath devices (`pr#36928 <https://github.com/ceph/ceph/pull/36928>`_, Jan Fajerski)
+* ceph.in: ignore failures to flush stdout (`pr#37225 <https://github.com/ceph/ceph/pull/37225>`_, Dan van der Ster)
+* ceph.spec, debian: add smartmontools, nvme-cli dependencies (`pr#37257 <https://github.com/ceph/ceph/pull/37257>`_, Yaarit Hatuka)
+* cephadm batch backport November (`pr#38155 <https://github.com/ceph/ceph/pull/38155>`_, Ricardo Marques, Sebastian Wagner, Kyr Shatskyy, Dan Williams, Volker Theile, Varsha Rao, Tim Serong, Adam King, Dimitri Savineau, Patrick Seidensal, Dan Mick, Michael Fritch, Joshua Schmid)
+* cephadm batch backport September (1) (`pr#36975 <https://github.com/ceph/ceph/pull/36975>`_, Stephan Müller, Matthew Oliver, Sebastian Wagner, Paul Cuzner, Adam King, Patrick Seidensal, Shraddha Agrawal, Michael Fritch, Dan Mick)
+* cephadm batch backport September (2) (`pr#37436 <https://github.com/ceph/ceph/pull/37436>`_, Varsha Rao, Kiefer Chang, Patrick Donnelly, Sebastian Wagner, Kefu Chai, Guillaume Abrioux, Juan Miguel Olmo Martínez, Paul Cuzner, Volker Theile, Tim Serong, Zac Dover, Adam King, Michael Fritch, Joshua Schmid)
+* cephfs-journal-tool: fix incorrect read_offset when finding missing objects (`pr#37854 <https://github.com/ceph/ceph/pull/37854>`_, Xue Yantao)
+* cephfs: client: fix directory inode can not call release callback (`pr#37017 <https://github.com/ceph/ceph/pull/37017>`_, sepia-liu)
+* cephfs: client: fix extra open ref decrease (`pr#37249 <https://github.com/ceph/ceph/pull/37249>`_, Xiubo Li)
+* cephfs: client: fix inode ll_ref reference count leak (`pr#37839 <https://github.com/ceph/ceph/pull/37839>`_, sepia-liu)
+* cephfs: client: handle readdir reply without Fs cap (`pr#37370 <https://github.com/ceph/ceph/pull/37370>`_, "Yan, Zheng")
+* cephfs: client: make Client::open() pass proper cap mask to path_walk (`pr#37369 <https://github.com/ceph/ceph/pull/37369>`_, "Yan, Zheng")
+* cephfs: client: use non-static dirent for thread-safety (`pr#37351 <https://github.com/ceph/ceph/pull/37351>`_, Patrick Donnelly)
+* cephfs: libcephfs: ignore restoring the open files limit (`pr#37358 <https://github.com/ceph/ceph/pull/37358>`_, Xiubo Li)
+* cephfs: osdc/Journaler: do not call onsafe->complete() if onsafe is 0 (`pr#37368 <https://github.com/ceph/ceph/pull/37368>`_, Xiubo Li)
+* common/admin_socket: always validate the parameters (`pr#37341 <https://github.com/ceph/ceph/pull/37341>`_, Kefu Chai)
+* compressor: Add a config option to specify Zstd compression level (`pr#37253 <https://github.com/ceph/ceph/pull/37253>`_, Bryan Stillwell)
+* core: include/encoding: Fix encode/decode of float types on big-endian systems (`pr#37032 <https://github.com/ceph/ceph/pull/37032>`_, Ulrich Weigand)
+* debian: Add missing Python dependency for ceph-mgr (`pr#37422 <https://github.com/ceph/ceph/pull/37422>`_, Johannes M. Scheuermann)
+* doc/PendingReleaseNotes: mention bluefs_preextend_wal_files (`pr#37549 <https://github.com/ceph/ceph/pull/37549>`_, Nathan Cutler)
+* doc/mgr/orchestrator: Add hints related to custom containers to the docs (`pr#37962 <https://github.com/ceph/ceph/pull/37962>`_, Volker Theile)
+* doc: cephfs: improve documentation of "ceph nfs cluster create" and "ceph fs volume create" commands (`pr#37691 <https://github.com/ceph/ceph/pull/37691>`_, Nathan Cutler)
+* doc: enable Read the Docs (`pr#37201 <https://github.com/ceph/ceph/pull/37201>`_, Kefu Chai)
+* erasure-code: enable isa-l EC for aarch64 platform (`pr#37504 <https://github.com/ceph/ceph/pull/37504>`_, luo rixin, Hang Li)
+* krbd: optionally skip waiting for udev events (`pr#37285 <https://github.com/ceph/ceph/pull/37285>`_, Ilya Dryomov)
+* librbd: ensure that thread pool lock is held when processing throttled IOs (`pr#37116 <https://github.com/ceph/ceph/pull/37116>`_, Jason Dillaman)
+* librbd: handle DNE from immutable-object-cache (`pr#36860 <https://github.com/ceph/ceph/pull/36860>`_, Feng Hualong, Mykola Golub, Yin Congmin, Jason Dillaman)
+* librbd: using migration abort can result in the loss of data (`pr#37164 <https://github.com/ceph/ceph/pull/37164>`_, Jason Dillaman)
+* mds/CInode: Optimize only pinned by subtrees check (`pr#37248 <https://github.com/ceph/ceph/pull/37248>`_, Mark Nelson)
+* mds: account for closing sessions in hit_session (`pr#37856 <https://github.com/ceph/ceph/pull/37856>`_, Dan van der Ster)
+* mds: add request to batch_op before taking auth pins and locks (`pr#37022 <https://github.com/ceph/ceph/pull/37022>`_, "Yan, Zheng")
+* mds: do not raise "client failing to respond to cap release" when client working set is reasonable (`pr#37353 <https://github.com/ceph/ceph/pull/37353>`_, Patrick Donnelly)
+* mds: do not submit omap_rm_keys if the dir is the basedir of merge (`pr#37034 <https://github.com/ceph/ceph/pull/37034>`_, "Yan, Zheng", Chencan)
+* mds: don't recover files after normal session close (`pr#37334 <https://github.com/ceph/ceph/pull/37334>`_, "Yan, Zheng")
+* mds: fix 'forward loop' when forward_all_requests_to_auth is set (`pr#37360 <https://github.com/ceph/ceph/pull/37360>`_, "Yan, Zheng")
+* mds: fix hang issue when accessing a file under a lost parent directory (`pr#37020 <https://github.com/ceph/ceph/pull/37020>`_, Zhi Zhang)
+* mds: fix kcephfs parse dirfrag's ndist is always 0 (`pr#37357 <https://github.com/ceph/ceph/pull/37357>`_, Yanhu Cao)
+* mds: fix mds forwarding request 'no_available_op_found' (`pr#37240 <https://github.com/ceph/ceph/pull/37240>`_, Yanhu Cao)
+* mds: fix nullptr dereference in MDCache::finish_rollback (`pr#37243 <https://github.com/ceph/ceph/pull/37243>`_, "Yan, Zheng")
+* mds: fix purge_queue's _calculate_ops is inaccurate (`pr#37372 <https://github.com/ceph/ceph/pull/37372>`_, Yanhu Cao)
+* mds: make threshold for MDS_TRIM configurable (`pr#36970 <https://github.com/ceph/ceph/pull/36970>`_, Paul Emmerich)
+* mds: optimize random threshold lookup for dentry load (`pr#37247 <https://github.com/ceph/ceph/pull/37247>`_, Patrick Donnelly)
+* mds: place MDSGatherBuilder on the stack (`pr#37354 <https://github.com/ceph/ceph/pull/37354>`_, Patrick Donnelly)
+* mds: reduce memory usage of open file table prefetch #37382 (`pr#37383 <https://github.com/ceph/ceph/pull/37383>`_, "Yan, Zheng")
+* mds: resolve SIGSEGV in waiting for uncommitted fragments (`pr#37355 <https://github.com/ceph/ceph/pull/37355>`_, Patrick Donnelly)
+* mds: revert the decode version (`pr#37356 <https://github.com/ceph/ceph/pull/37356>`_, Jos Collin)
+* mds: send scrub status to ceph-mgr only when scrub is running (`issue#45349 <http://tracker.ceph.com/issues/45349>`_, `pr#36047 <https://github.com/ceph/ceph/pull/36047>`_, Kefu Chai, Venky Shankar)
+* mds: standy-replay mds remained in the "resolve" state after resta… (`pr#37363 <https://github.com/ceph/ceph/pull/37363>`_, Wei Qiaomiao)
+* messages,mds: Fix decoding of enum types on big-endian systems (`pr#36813 <https://github.com/ceph/ceph/pull/36813>`_, Ulrich Weigand)
+* mgr/dashboard/api: move/create OSD histogram in separate endpoint (`pr#37973 <https://github.com/ceph/ceph/pull/37973>`_, Aashish Sharma)
+* mgr/dashboard: Add short descriptions to the telemetry report preview (`pr#37597 <https://github.com/ceph/ceph/pull/37597>`_, Nizamudeen A)
+* mgr/dashboard: Allow editing iSCSI targets with initiators logged-in (`pr#37277 <https://github.com/ceph/ceph/pull/37277>`_, Tiago Melo)
+* mgr/dashboard: Auto close table column dropdown on click outside (`pr#36862 <https://github.com/ceph/ceph/pull/36862>`_, Tiago Melo)
+* mgr/dashboard: Copy to clipboard does not work in Firefox (`pr#37493 <https://github.com/ceph/ceph/pull/37493>`_, Volker Theile)
+* mgr/dashboard: Datatable catches select events from other datatables (`pr#36899 <https://github.com/ceph/ceph/pull/36899>`_, Volker Theile, Tiago Melo)
+* mgr/dashboard: Disable TLS 1.0 and 1.1 (`pr#38331 <https://github.com/ceph/ceph/pull/38331>`_, Volker Theile)
+* mgr/dashboard: Disable autocomplete on user form (`pr#36901 <https://github.com/ceph/ceph/pull/36901>`_, Volker Theile)
+* mgr/dashboard: Disable sso without python3-saml (`pr#38405 <https://github.com/ceph/ceph/pull/38405>`_, Kevin Meijer)
+* mgr/dashboard: Disabling the form inputs for the read_only modals (`pr#37239 <https://github.com/ceph/ceph/pull/37239>`_, Nizamudeen)
+* mgr/dashboard: Fix bugs in a unit test and i18n translation (`pr#36991 <https://github.com/ceph/ceph/pull/36991>`_, Volker Theile)
+* mgr/dashboard: Fix for CrushMap viewer items getting compressed vertically (`pr#36871 <https://github.com/ceph/ceph/pull/36871>`_, Nizamudeen A)
+* mgr/dashboard: Fix many-to-many issue in host-details Grafana dashboard (`pr#37299 <https://github.com/ceph/ceph/pull/37299>`_, Patrick Seidensal)
+* mgr/dashboard: Fix npm package's vulnerabilities (`pr#36921 <https://github.com/ceph/ceph/pull/36921>`_, Tiago Melo)
+* mgr/dashboard: Hide table action input field if limit=0 (`pr#36872 <https://github.com/ceph/ceph/pull/36872>`_, Volker Theile)
+* mgr/dashboard: Host delete action should be disabled if not managed by Orchestrator (`pr#36874 <https://github.com/ceph/ceph/pull/36874>`_, Volker Theile)
+* mgr/dashboard: Improve notification badge (`pr#37090 <https://github.com/ceph/ceph/pull/37090>`_, Aashish Sharma)
+* mgr/dashboard: Landing Page improvements (`pr#37390 <https://github.com/ceph/ceph/pull/37390>`_, Tiago Melo, Alfonso Martínez)
+* mgr/dashboard: Merge disable and disableDesc (`pr#37763 <https://github.com/ceph/ceph/pull/37763>`_, Tiago Melo)
+* mgr/dashboard: Proper format iSCSI target portals (`pr#36870 <https://github.com/ceph/ceph/pull/36870>`_, Volker Theile)
+* mgr/dashboard: REST API returns 500 when no Content-Type is specified (`pr#37308 <https://github.com/ceph/ceph/pull/37308>`_, Avan Thakkar)
+* mgr/dashboard: Remove useless tab in monitoring/alerts datatable details (`pr#36875 <https://github.com/ceph/ceph/pull/36875>`_, Volker Theile)
+* mgr/dashboard: Show warning when replicated size is 1 (`pr#37578 <https://github.com/ceph/ceph/pull/37578>`_, Sebastian Krah)
+* mgr/dashboard: The performance 'Client Read/Write' widget shows incorrect write values (`pr#38189 <https://github.com/ceph/ceph/pull/38189>`_, Volker Theile)
+* mgr/dashboard: Update datatable only when necessary (`pr#37331 <https://github.com/ceph/ceph/pull/37331>`_, Volker Theile)
+* mgr/dashboard: Use pipe instead of calling function within template (`pr#38094 <https://github.com/ceph/ceph/pull/38094>`_, Volker Theile)
+* mgr/dashboard: cluster > manager modules (`pr#37434 <https://github.com/ceph/ceph/pull/37434>`_, Avan Thakkar)
+* mgr/dashboard: display devices' health information within a tabset (`pr#37784 <https://github.com/ceph/ceph/pull/37784>`_, Kiefer Chang)
+* mgr/dashboard: fix error when typing existing paths in the Ganesha form (`pr#37688 <https://github.com/ceph/ceph/pull/37688>`_, Kiefer Chang)
+* mgr/dashboard: fix perf. issue when listing large amounts of buckets (`pr#37405 <https://github.com/ceph/ceph/pull/37405>`_, Alfonso Martínez)
+* mgr/dashboard: fix security scopes of some NFS-Ganesha endpoints (`pr#37450 <https://github.com/ceph/ceph/pull/37450>`_, Kiefer Chang)
+* mgr/dashboard: fix the error when exporting CephFS path "/" in NFS exports (`pr#37686 <https://github.com/ceph/ceph/pull/37686>`_, Kiefer Chang)
+* mgr/dashboard: get rgw daemon zonegroup name from mgr (`pr#37620 <https://github.com/ceph/ceph/pull/37620>`_, Alfonso Martinez)
+* mgr/dashboard: increase Grafana iframe height to avoid scroll bar (`pr#37182 <https://github.com/ceph/ceph/pull/37182>`_, Ngwa Sedrick Meh)
+* mgr/dashboard: log in non-admin users successfully if the telemetry notification is shown (`pr#37452 <https://github.com/ceph/ceph/pull/37452>`_, Tatjana Dehler)
+* mgr/dashboard: support Orchestrator and user-defined Ganesha cluster (`pr#37885 <https://github.com/ceph/ceph/pull/37885>`_, Kiefer Chang)
+* mgr/dashboard: table detail rows overflow (`pr#37332 <https://github.com/ceph/ceph/pull/37332>`_, Aashish Sharma)
+* mgr/devicehealth: device_health_metrics pool gets created even without any OSDs in the cluster (`pr#37533 <https://github.com/ceph/ceph/pull/37533>`_, Sunny Kumar)
+* mgr/insights: Test environment requires 'six' (`pr#38396 <https://github.com/ceph/ceph/pull/38396>`_, Brad Hubbard)
+* mgr/prometheus: add pool compression stats (`pr#37562 <https://github.com/ceph/ceph/pull/37562>`_, Paul Cuzner)
+* mgr/telemetry: fix device id splitting when anonymizing serial (`pr#37302 <https://github.com/ceph/ceph/pull/37302>`_, Yaarit Hatuka)
+* mgr/volumes/nfs: Check if orchestrator spec service_id is valid (`pr#37371 <https://github.com/ceph/ceph/pull/37371>`_, Varsha Rao)
+* mgr/volumes/nfs: Fix wrong error message for pseudo path (`pr#37855 <https://github.com/ceph/ceph/pull/37855>`_, Varsha Rao)
+* mgr/volumes: Make number of cloner threads configurable (`pr#37671 <https://github.com/ceph/ceph/pull/37671>`_, Kotresh HR)
+* mgr/zabbix: indent the output of "zabbix config-show" (`pr#37128 <https://github.com/ceph/ceph/pull/37128>`_, Kefu Chai)
+* mgr: PyModuleRegistry::unregister_client() can run endlessly (`issue#47329 <http://tracker.ceph.com/issues/47329>`_, `pr#37217 <https://github.com/ceph/ceph/pull/37217>`_, Venky Shankar)
+* mgr: don't update pending service map epoch on receiving map from mon (`pr#37180 <https://github.com/ceph/ceph/pull/37180>`_, Mykola Golub)
+* mon scrub testing (`pr#38361 <https://github.com/ceph/ceph/pull/38361>`_, Brad Hubbard)
+* mon/MDSMonitor do not ignore mds's down:dne request (`pr#37858 <https://github.com/ceph/ceph/pull/37858>`_, chencan)
+* mon/MDSMonitor: divide mds identifier and mds real name with dot (`pr#37857 <https://github.com/ceph/ceph/pull/37857>`_, Zhi Zhang)
+* mon/MonMap: fix unconditional failure for init_with_hosts (`pr#37817 <https://github.com/ceph/ceph/pull/37817>`_, Nathan Cutler, Patrick Donnelly)
+* mon/PGMap: add pg count for pools in the ceph df command (`pr#36945 <https://github.com/ceph/ceph/pull/36945>`_, Vikhyat Umrao)
+* mon: Log "ceph health detail" periodically in cluster log (`pr#38345 <https://github.com/ceph/ceph/pull/38345>`_, Prashant Dhange)
+* mon: deleting a CephFS and its pools causes MONs to crash (`pr#37256 <https://github.com/ceph/ceph/pull/37256>`_, Patrick Donnelly)
+* mon: have 'mon stat' output json as well (`pr#37705 <https://github.com/ceph/ceph/pull/37705>`_, Joao Eduardo Luis)
+* mon: mark pgtemp messages as no_reply more consistenly in preprocess\_… (`pr#37347 <https://github.com/ceph/ceph/pull/37347>`_, Greg Farnum)
+* mon: set session_timeout when adding to session_map (`pr#37553 <https://github.com/ceph/ceph/pull/37553>`_, Ilya Dryomov)
+* mon: store mon updates in ceph context for future MonMap instantiation (`pr#36705 <https://github.com/ceph/ceph/pull/36705>`_, Patrick Donnelly, Shyamsundar Ranganathan)
+* msg/async/ProtocolV2: allow rxbuf/txbuf get bigger in testing (`pr#37080 <https://github.com/ceph/ceph/pull/37080>`_, Ilya Dryomov)
+* os/bluestore: enable more flexible bluefs space management by default (`pr#37092 <https://github.com/ceph/ceph/pull/37092>`_, Igor Fedotov)
+* osd/osd-rep-recov-eio.sh: TEST_rados_repair_warning:  return 1 (`pr#37853 <https://github.com/ceph/ceph/pull/37853>`_, David Zafman)
+* osd: Check for nosrub/nodeep-scrub in between chunks, to avoid races (`pr#38359 <https://github.com/ceph/ceph/pull/38359>`_, David Zafman)
+* osdc/ObjectCacher: overwrite might cause stray read request callbacks (`pr#37674 <https://github.com/ceph/ceph/pull/37674>`_, Jason Dillaman)
+* osdc: add timeout configs for mons/osds (`pr#37530 <https://github.com/ceph/ceph/pull/37530>`_, Patrick Donnelly)
+* prometheus: Properly split the port off IPv6 addresses (`pr#36985 <https://github.com/ceph/ceph/pull/36985>`_, Matthew Oliver)
+* pybind/cephfs: add special values for not reading conffile (`pr#37724 <https://github.com/ceph/ceph/pull/37724>`_, Kefu Chai)
+* pybind/cephfs: fix custom exception raised by cephfs.pyx (`pr#37350 <https://github.com/ceph/ceph/pull/37350>`_, Ramana Raja)
+* pybind/mgr/volumes: add global lock debug (`pr#37366 <https://github.com/ceph/ceph/pull/37366>`_, Patrick Donnelly)
+* qa/\*/mon/mon-last-epoch-clean.sh: mark osd out instead of down (`pr#37349 <https://github.com/ceph/ceph/pull/37349>`_, Neha Ojha)
+* qa/cephfs: add session_timeout option support (`pr#37841 <https://github.com/ceph/ceph/pull/37841>`_, Xiubo Li)
+* qa/tasks/nfs: Test mounting of export created with nfs command (`pr#37365 <https://github.com/ceph/ceph/pull/37365>`_, Varsha Rao)
+* qa/tasks/{ceph,ceph_manager}: drop py2 support (`pr#37863 <https://github.com/ceph/ceph/pull/37863>`_, Kefu Chai)
+* qa/tests: added rhel 8.2 (`pr#38287 <https://github.com/ceph/ceph/pull/38287>`_, Yuri Weinstein)
+* qa/tests: use bionic only for old clients in rados/thrash-old-clients (`pr#36931 <https://github.com/ceph/ceph/pull/36931>`_, Yuri Weinstein)
+* qa/workunits/mon: fixed excessively large pool PG count (`pr#37346 <https://github.com/ceph/ceph/pull/37346>`_, Jason Dillaman)
+* qa: Enable debug_client for mgr tests (`pr#37270 <https://github.com/ceph/ceph/pull/37270>`_, Brad Hubbard)
+* qa: Fix traceback during fs cleanup between tests (`pr#36713 <https://github.com/ceph/ceph/pull/36713>`_, Kotresh HR)
+* qa: add debugging for volumes plugin use of libcephfs (`pr#37352 <https://github.com/ceph/ceph/pull/37352>`_, Patrick Donnelly)
+* qa: drop hammer branch qa tests (`pr#37728 <https://github.com/ceph/ceph/pull/37728>`_, Neha Ojha, Deepika Upadhyay)
+* qa: ignore expected mds failover message (`pr#37367 <https://github.com/ceph/ceph/pull/37367>`_, Patrick Donnelly)
+* rbd-mirror: peer setup can still race and fail creation of peer (`pr#37342 <https://github.com/ceph/ceph/pull/37342>`_, Jason Dillaman)
+* rbd: include RADOS namespace in krbd symlinks (`pr#37343 <https://github.com/ceph/ceph/pull/37343>`_, Ilya Dryomov)
+* rbd: journal: possible race condition between flush and append callback (`pr#37850 <https://github.com/ceph/ceph/pull/37850>`_, Jason Dillaman)
+* rbd: librbd: ignore -ENOENT error when disabling object-map (`pr#37852 <https://github.com/ceph/ceph/pull/37852>`_, Jason Dillaman)
+* rbd: librbd: update AioCompletion return value before evaluating pending count (`pr#37851 <https://github.com/ceph/ceph/pull/37851>`_, Jason Dillaman)
+* rbd: make common options override krbd-specific options (`pr#37408 <https://github.com/ceph/ceph/pull/37408>`_, Ilya Dryomov)
+* rbd: rbd-nbd: don't ignore namespace when unmapping by image spec (`pr#37812 <https://github.com/ceph/ceph/pull/37812>`_, Mykola Golub)
+* rgw/gc: fix for incrementing the perf counter 'gc_retire_object' (`pr#37847 <https://github.com/ceph/ceph/pull/37847>`_, Pritha Srivastava)
+* rgw/gc: fixing the condition when marker for a queue is (`pr#37846 <https://github.com/ceph/ceph/pull/37846>`_, Pritha Srivastava)
+* rgw/rgw_file: Fix the incorrect lru object eviction (`pr#37672 <https://github.com/ceph/ceph/pull/37672>`_, luo rixin)
+* rgw: Add bucket name to bucket stats error logging (`pr#37335 <https://github.com/ceph/ceph/pull/37335>`_, Seena Fallah)
+* rgw: Add request timeout to beast (`pr#37809 <https://github.com/ceph/ceph/pull/37809>`_, Adam C. Emerson, Or Friedmann)
+* rgw: RGWObjVersionTracker tracks version over increments (`pr#37337 <https://github.com/ceph/ceph/pull/37337>`_, Casey Bodley)
+* rgw: Swift API anonymous access should 401 (`pr#37339 <https://github.com/ceph/ceph/pull/37339>`_, Matthew Oliver)
+* rgw: adds code for creating and managing oidc provider entities in rgw and for offline validation of OpenID Connect Access and ID Token (`pr#37640 <https://github.com/ceph/ceph/pull/37640>`_, Pritha Srivastava, Casey Bodley)
+* rgw: allow rgw-orphan-list to note when rados objects are in namespace (`pr#37800 <https://github.com/ceph/ceph/pull/37800>`_, J. Eric Ivancich)
+* rgw: dump transitions in RGWLifecycleConfiguration::dump() (`pr#36812 <https://github.com/ceph/ceph/pull/36812>`_, Shengming Zhang)
+* rgw: during GC defer, prevent new GC enqueue (`pr#38249 <https://github.com/ceph/ceph/pull/38249>`_, Casey Bodley, J. Eric Ivancich)
+* rgw: fix expiration header returned even if there is only one tag in the object the same as the rule (`pr#37807 <https://github.com/ceph/ceph/pull/37807>`_, Or Friedmann)
+* rgw: fix setting of namespace in ordered and unordered bucket listing (`pr#37673 <https://github.com/ceph/ceph/pull/37673>`_, J. Eric Ivancich)
+* rgw: fix user stats iterative increment (`pr#37779 <https://github.com/ceph/ceph/pull/37779>`_, Mark Kogan)
+* rgw: fix: S3 API KeyCount incorrect return (`pr#37849 <https://github.com/ceph/ceph/pull/37849>`_, 胡玮文)
+* rgw: log resharding events at level 1 (formerly 20) (`pr#36840 <https://github.com/ceph/ceph/pull/36840>`_, Or Friedmann)
+* rgw: radosgw-admin should paginate internally when listing bucket (`pr#37803 <https://github.com/ceph/ceph/pull/37803>`_, J. Eric Ivancich)
+* rgw: radosgw-admin: period pull command is not always a raw_storage_op (`pr#37336 <https://github.com/ceph/ceph/pull/37336>`_, Casey Bodley)
+* rgw: replace '+' with "%20" in canonical query string for s3 v4 auth (`pr#37338 <https://github.com/ceph/ceph/pull/37338>`_, yuliyang_yewu)
+* rgw: rgw_file: avoid long-ish delay on shutdown (`pr#37551 <https://github.com/ceph/ceph/pull/37551>`_, Matt Benjamin)
+* rgw: s3: mark bucket encryption as not implemented (`pr#36691 <https://github.com/ceph/ceph/pull/36691>`_, Abhishek Lekshmanan)
+* rgw: urlencode bucket name when forwarding request (`pr#37340 <https://github.com/ceph/ceph/pull/37340>`_, caolei)
+* rgw: use yum rather than dnf for teuthology testing of rgw-orphan-list (`pr#37845 <https://github.com/ceph/ceph/pull/37845>`_, J. Eric Ivancich)
+* rpm,deb: drop /etc/sudoers.d/cephadm (`pr#37401 <https://github.com/ceph/ceph/pull/37401>`_, Nathan Cutler)
+* run-make-check.sh: Don't run tests if build fails (`pr#38294 <https://github.com/ceph/ceph/pull/38294>`_, Brad Hubbard)
+* systemd: Support Graceful Reboot for AIO Node (`pr#37300 <https://github.com/ceph/ceph/pull/37300>`_, Wong Hoi Sing Edison)
+* test/librados: fix endian bugs in checksum test cases (`pr#37604 <https://github.com/ceph/ceph/pull/37604>`_, Ulrich Weigand)
+* test/rbd-mirror: pool watcher registration error might result in race (`pr#37208 <https://github.com/ceph/ceph/pull/37208>`_, Jason Dillaman)
+* test/store_test: use 'threadsafe' style for death tests (`pr#37819 <https://github.com/ceph/ceph/pull/37819>`_, Igor Fedotov)
+* tools/osdmaptool.cc: add ability to clean_temps (`pr#37348 <https://github.com/ceph/ceph/pull/37348>`_, Neha Ojha)
+* tools/rados: flush formatter periodically during json output of "rados ls" (`pr#37835 <https://github.com/ceph/ceph/pull/37835>`_, J. Eric Ivancich)
+* vstart.sh: fix fs set max_mds bug (`pr#37837 <https://github.com/ceph/ceph/pull/37837>`_, Jinmyeong Lee)
+
+
+v15.2.7 Octopus
+===============
+
+This is the 7th backport release in the Octopus series. This release fixes
+a serious bug in RGW that has been shown to cause data loss when a read of
+a large RGW object (i.e., one with at least one tail segment) takes longer than
+one half the time specified in the configuration option ``rgw_gc_obj_min_wait``.
+The bug causes the tail segments of that read object to be added to the RGW
+garbage collection queue, which will in turn cause them to be deleted after
+a period of time.
+
+Changelog
+---------
+
+* rgw: during GC defer, prevent new GC enqueue (`issue#47866 <https://tracker.ceph.com/issues/47866>`_, `pr#38249 <https://github.com/ceph/ceph/pull/38249>`_, Eric Ivancich, Casey Bodley)
+
+
+v15.2.6 Octopus
+===============
+
+This is the 6th backport release in the Octopus series. This release fixes
+a security flaw affecting Messenger v1 & v2. We recommend users to update to
+this release.
+
+Notable Changes
+---------------
+
+* CVE 2020-25660: CEPHX_V2 replay attack protection lost, for Messenger v1 & v2 (Ilya Dryomov)
+
+Changelog
+---------
+
+* mon/MonClient: bring back CEPHX_V2 authorizer challenges (Ilya Dryomov)
+
+
+v15.2.5 Octopus
+===============
+
+This is the fifth release of the Ceph Octopus stable release series. This
+release brings a range of fixes across all components. We recommend that all
+Octopus users upgrade to this release.
+
+Notable Changes
+---------------
+
+* CephFS: Automatic static subtree partitioning policies may now be configured
+  using the new distributed and random ephemeral pinning extended attributes on
+  directories. See the documentation for more information:
+  https://docs.ceph.com/docs/master/cephfs/multimds/
+
+* Monitors now have a config option ``mon_osd_warn_num_repaired``, 10 by default.
+  If any OSD has repaired more than this many I/O errors in stored data a
+  ``OSD_TOO_MANY_REPAIRS`` health warning is generated.
+
+* Now when noscrub and/or no deep-scrub flags are set globally or per pool,
+  scheduled scrubs of the type disabled will be aborted. All user initiated
+  scrubs are NOT interrupted.
+
+* Fix an issue with osdmaps not being trimmed in a healthy cluster (
+  `issue#47297 <https://tracker.ceph.com/issues/47297>`_,
+  `pr#36981 <https://github.com/ceph/ceph/pull/36981>`_)
+
+Changelog
+---------
+
+* bluestore,core: bluestore: blk:BlockDevice.cc: use pending_aios instead of iovec size as ios num (`pr#36668 <https://github.com/ceph/ceph/pull/36668>`_, weixinwei)
+* bluestore,tests: test/store_test: refactor bluestore spillover test (`pr#34943 <https://github.com/ceph/ceph/pull/34943>`_, Igor Fedotov)
+* bluestore,tests: tests: objectstore/store_test: kill ExcessiveFragmentation test case (`pr#36049 <https://github.com/ceph/ceph/pull/36049>`_, Igor Fedotov)
+* bluestore: bluestore: Rescue procedure for extremely large bluefs log (`pr#36123 <https://github.com/ceph/ceph/pull/36123>`_, Adam Kupczyk)
+* bluestore: octopus:os/bluestore: improve/fix bluefs stats reporting (`pr#35748 <https://github.com/ceph/ceph/pull/35748>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix bluefs log growth (`pr#36621 <https://github.com/ceph/ceph/pull/36621>`_, Adam Kupczyk, Jianpeng Ma)
+* bluestore: os/bluestore: simplify Onode pin/unpin logic (`pr#36795 <https://github.com/ceph/ceph/pull/36795>`_, Igor Fedotov)
+* build/ops: Revert "mgr/osd_support: remove module and all traces" (`pr#36973 <https://github.com/ceph/ceph/pull/36973>`_, Sebastian Wagner)
+* build/ops: ceph-iscsi: selinux fixes (`pr#36302 <https://github.com/ceph/ceph/pull/36302>`_, Mike Christie)
+* build/ops: mgr/dashboard/api: reduce amount of daemon logs (`pr#36693 <https://github.com/ceph/ceph/pull/36693>`_, Ernesto Puerta)
+* ceph-volume: add dmcrypt support in raw mode (`pr#35830 <https://github.com/ceph/ceph/pull/35830>`_, Guillaume Abrioux)
+* ceph-volume: add drive-group subcommand (`pr#36558 <https://github.com/ceph/ceph/pull/36558>`_, Jan Fajerski, Sebastian Wagner)
+* ceph-volume: add tests for new functions that run LVM commands (`pr#36614 <https://github.com/ceph/ceph/pull/36614>`_, Rishabh Dave)
+* ceph-volume: don't use container classes in api/lvm.py (`pr#35879 <https://github.com/ceph/ceph/pull/35879>`_, Rishabh Dave, Guillaume Abrioux)
+* ceph-volume: fix lvm functional tests (`pr#36409 <https://github.com/ceph/ceph/pull/36409>`_, Jan Fajerski)
+* ceph-volume: handle idempotency with batch and explicit scenarios (`pr#35880 <https://github.com/ceph/ceph/pull/35880>`_, Andrew Schoen)
+* ceph-volume: remove container classes from api/lvm.py (`pr#36608 <https://github.com/ceph/ceph/pull/36608>`_, Rishabh Dave)
+* ceph-volume: report correct rejected reason in inventory if device type is invalid (`pr#36410 <https://github.com/ceph/ceph/pull/36410>`_, Satoru Takeuchi)
+* ceph-volume: run flake8 in python3 (`pr#36588 <https://github.com/ceph/ceph/pull/36588>`_, Jan Fajerski)
+* cephfs,common: common:  ignore SIGHUP prior to fork (`issue#46269 <http://tracker.ceph.com/issues/46269>`_, `pr#36195 <https://github.com/ceph/ceph/pull/36195>`_, Willem Jan Withagen, hzwuhongsong)
+* cephfs,core,mgr: mgr/status: metadata is fetched async (`pr#36630 <https://github.com/ceph/ceph/pull/36630>`_, Michael Fritch)
+* cephfs,core,rbd,rgw: librados: add LIBRADOS_SUPPORTS_GETADDRS support (`pr#36643 <https://github.com/ceph/ceph/pull/36643>`_, Xiubo Li)
+* cephfs,mgr: mgr/volumes/nfs: Add interface for adding user defined configuration (`pr#36635 <https://github.com/ceph/ceph/pull/36635>`_, Varsha Rao)
+* cephfs,mon: mon/MDSMonitor: copy MDS info which may be removed (`pr#36035 <https://github.com/ceph/ceph/pull/36035>`_, Patrick Donnelly)
+* cephfs,pybind: pybind/ceph_volume_client: Fix PEP-8 SyntaxWarning (`pr#36100 <https://github.com/ceph/ceph/pull/36100>`_, �ặng Minh Dũng)
+* cephfs,tests: mgr/fs/volumes: misc fixes (`pr#36327 <https://github.com/ceph/ceph/pull/36327>`_, Patrick Donnelly, Kotresh HR)
+* cephfs,tests: tests: Revert "Revert "qa/suites/rados/mgr/tasks/module_selftest: whitelist … (`issue#43943 <http://tracker.ceph.com/issues/43943>`_, `pr#36042 <https://github.com/ceph/ceph/pull/36042>`_, Venky Shankar)
+* cephfs,tests: tests: qa/tasks/cephfs/cephfs_test_case.py: skip cleaning the core dumps when in program case (`pr#36043 <https://github.com/ceph/ceph/pull/36043>`_, Xiubo Li)
+* cephfs,tests: tests: qa/tasks: make sh() in vstart_runner.py identical with teuthology.orchestra.remote.sh (`pr#36044 <https://github.com/ceph/ceph/pull/36044>`_, Jos Collin)
+* cephfs: Update nfs-ganesha package requirements doc backport (`pr#36063 <https://github.com/ceph/ceph/pull/36063>`_, Varsha Rao)
+* cephfs: cephfs: client: fix setxattr for 0 size value (NULL value) (`pr#36045 <https://github.com/ceph/ceph/pull/36045>`_, Sidharth Anupkrishnan)
+* cephfs: cephfs: client: fix snap directory atime (`pr#36039 <https://github.com/ceph/ceph/pull/36039>`_, Luis Henriques)
+* cephfs: cephfs: client: release the client_lock before copying data in read (`pr#36046 <https://github.com/ceph/ceph/pull/36046>`_, Chencan)
+* cephfs: client: expose ceph.quota.max_bytes xattr within snapshots (`pr#36403 <https://github.com/ceph/ceph/pull/36403>`_, Shyamsundar Ranganathan)
+* cephfs: client: introduce timeout for client shutdown (`issue#44276 <http://tracker.ceph.com/issues/44276>`_, `pr#35962 <https://github.com/ceph/ceph/pull/35962>`_, "Yan, Zheng", Venky Shankar)
+* cephfs: mds/MDSRank: fix typo in "unrecognized" (`pr#36197 <https://github.com/ceph/ceph/pull/36197>`_, Nathan Cutler)
+* cephfs: mds: add ephemeral random and distributed export pins (`pr#35759 <https://github.com/ceph/ceph/pull/35759>`_, Patrick Donnelly, Sidharth Anupkrishnan)
+* cephfs: mds: fix filelock state when Fc is issued (`pr#35842 <https://github.com/ceph/ceph/pull/35842>`_, Xiubo Li)
+* cephfs: mds: reset heartbeat in EMetaBlob replay (`pr#36040 <https://github.com/ceph/ceph/pull/36040>`_, Yanhu Cao)
+* cephfs: mgr/nfs: Check if pseudo path is absolute path (`pr#36299 <https://github.com/ceph/ceph/pull/36299>`_, Varsha Rao)
+* cephfs: mgr/nfs: Update MDCACHE block in ganesha config and doc about nfs-cephadm in vstart (`pr#36224 <https://github.com/ceph/ceph/pull/36224>`_, Varsha Rao)
+* cephfs: mgr/volumes: Deprecate protect/unprotect CLI calls for subvolume snapshots (`pr#36126 <https://github.com/ceph/ceph/pull/36126>`_, Shyamsundar Ranganathan)
+* cephfs: mgr/volumes: fix "ceph nfs export" help messages (`pr#36220 <https://github.com/ceph/ceph/pull/36220>`_, Nathan Cutler)
+* cephfs: nfs backport (`pr#35499 <https://github.com/ceph/ceph/pull/35499>`_, Jeff Layton, Varsha Rao, Ramana Raja, Kefu Chai)
+* common,core: common, osd: add sanity checks around osd_scrub_max_preemptions (`pr#36034 <https://github.com/ceph/ceph/pull/36034>`_, xie xingguo)
+* common,rbd,tools: rbd: immutable-object-cache: fixed crashes on start up (`pr#36660 <https://github.com/ceph/ceph/pull/36660>`_, Jason Dillaman)
+* common,rbd: crush/CrushWrapper: rebuild reverse maps after rebuilding crush map (`pr#36662 <https://github.com/ceph/ceph/pull/36662>`_, Jason Dillaman)
+* common: common: log: fix timestap precision of log can't set to millisecond (`pr#36048 <https://github.com/ceph/ceph/pull/36048>`_, Guan yunfei)
+* core,mgr: mgr: decrease pool stats if pg was removed (`pr#36667 <https://github.com/ceph/ceph/pull/36667>`_, Aleksei Gutikov)
+* core,rbd: osd/OSDCap: rbd profile permits use of "rbd_info" (`pr#36414 <https://github.com/ceph/ceph/pull/36414>`_, Florian Florensa)
+* core,tools: tools/rados: Set locator key when exporting or importing a pool (`pr#36666 <https://github.com/ceph/ceph/pull/36666>`_, Iain Buclaw)
+* core: mon/OSDMonitor: Reset grace period if failure interval exceeds a threshold (`pr#35799 <https://github.com/ceph/ceph/pull/35799>`_, Sridhar Seshasayee)
+* core: mon/OSDMonitor: only take in osd into consideration when trimming osd… (`pr#36981 <https://github.com/ceph/ceph/pull/36981>`_, Kefu Chai)
+* core: mon: fix the 'Error ERANGE' message when conf "osd_objectstore" is filestore (`pr#36665 <https://github.com/ceph/ceph/pull/36665>`_, wangyunqing)
+* core: monclient: schedule first tick using mon_client_hunt_interval (`pr#36633 <https://github.com/ceph/ceph/pull/36633>`_, Mykola Golub)
+* core: osd/OSD.cc: remove osd_lock for bench (`pr#36664 <https://github.com/ceph/ceph/pull/36664>`_, Neha Ojha, Adam Kupczyk)
+* core: osd/PG: fix history.same_interval_since of merge target again (`pr#36033 <https://github.com/ceph/ceph/pull/36033>`_, xie xingguo)
+* core: osd/PeeringState: prevent peer's num_objects going negative (`pr#36663 <https://github.com/ceph/ceph/pull/36663>`_, xie xingguo)
+* core: osd/PrimaryLogPG: don't populate watchers if replica (`pr#36029 <https://github.com/ceph/ceph/pull/36029>`_, Ilya Dryomov)
+* core: osd: Cancel in-progress scrubs (not user requested) (`pr#36291 <https://github.com/ceph/ceph/pull/36291>`_, David Zafman)
+* core: osd: expose osdspec_affinity to osd_metadata (`pr#35957 <https://github.com/ceph/ceph/pull/35957>`_, Joshua Schmid)
+* core: osd: fix crash in _committed_osd_maps if incremental osdmap crc fails (`pr#36340 <https://github.com/ceph/ceph/pull/36340>`_, Neha Ojha, Dan van der Ster)
+* core: osd: make message cap option usable again (`pr#35737 <https://github.com/ceph/ceph/pull/35737>`_, Neha Ojha, Josh Durgin)
+* core: osd: wakeup all threads of shard rather than one thread (`pr#36032 <https://github.com/ceph/ceph/pull/36032>`_, Jianpeng Ma)
+* core: test: osd-backfill-stats.sh use nobackfill to avoid races in remainin… (`pr#36030 <https://github.com/ceph/ceph/pull/36030>`_, David Zafman)
+* doc: cephadm batch backport (`pr#36450 <https://github.com/ceph/ceph/pull/36450>`_, Varsha Rao, Ricardo Marques, Kiefer Chang, Matthew Oliver, Paul Cuzner, Kefu Chai, Daniel-Pivonka, Sebastian Wagner, Volker Theile, Adam King, Michael Fritch, Joshua Schmid)
+* doc: doc/mgr/crash: Add missing command in rm example (`pr#36690 <https://github.com/ceph/ceph/pull/36690>`_, Daniël Vos)
+* doc: doc/rados: Fix osd_scrub_during_recovery default value (`pr#36661 <https://github.com/ceph/ceph/pull/36661>`_, Benoît Knecht)
+* doc: doc/rbd: add rbd-target-gw enable and start (`pr#36416 <https://github.com/ceph/ceph/pull/36416>`_, Zac Dover)
+* doc: doc: PendingReleaseNotes: clean slate for 15.2.5 (`pr#35753 <https://github.com/ceph/ceph/pull/35753>`_, Nathan Cutler)
+* mgr,pybind: pybind/mgr/balancer: use "==" and "!=" for comparing str (`pr#36036 <https://github.com/ceph/ceph/pull/36036>`_, Kefu Chai)
+* mgr,pybind: pybind/mgr/pg_autoscaler/module.py: do not update event if ev.pg_num== ev.pg_num_target (`pr#36037 <https://github.com/ceph/ceph/pull/36037>`_, Neha Ojha)
+* mgr,rbd: mgr/prometheus: automatically discover RBD pools for stats gathering (`pr#36411 <https://github.com/ceph/ceph/pull/36411>`_, Jason Dillaman)
+* mgr/dashboard/api: increase API health timeout (`pr#36562 <https://github.com/ceph/ceph/pull/36562>`_, Ernesto Puerta)
+* mgr/dashboard: Add button to copy the bootstrap token into the clipboard (`pr#35796 <https://github.com/ceph/ceph/pull/35796>`_, Ishan Rai)
+* mgr/dashboard: Add host labels in UI (`pr#35893 <https://github.com/ceph/ceph/pull/35893>`_, Volker Theile)
+* mgr/dashboard: Add hosts page unit tests (`pr#36350 <https://github.com/ceph/ceph/pull/36350>`_, Volker Theile)
+* mgr/dashboard: Allow to edit iSCSI target with active session (`pr#35997 <https://github.com/ceph/ceph/pull/35997>`_, Ricardo Marques)
+* mgr/dashboard: Always use fast angular unit tests (`pr#36267 <https://github.com/ceph/ceph/pull/36267>`_, Stephan Müller)
+* mgr/dashboard: Configure overflow of popover in health page (`pr#36460 <https://github.com/ceph/ceph/pull/36460>`_, Tiago Melo)
+* mgr/dashboard: Display check icon instead of true|false in various datatables (`pr#35892 <https://github.com/ceph/ceph/pull/35892>`_, Volker Theile)
+* mgr/dashboard: Display users current bucket quota usage (`pr#35926 <https://github.com/ceph/ceph/pull/35926>`_, Ernesto Puerta, Avan Thakkar)
+* mgr/dashboard: Extract documentation link to a component (`pr#36587 <https://github.com/ceph/ceph/pull/36587>`_, Tiago Melo)
+* mgr/dashboard: Fix host attributes like labels are not returned (`pr#36678 <https://github.com/ceph/ceph/pull/36678>`_, Kiefer Chang)
+* mgr/dashboard: Hide password notification when expiration date is far (`pr#35975 <https://github.com/ceph/ceph/pull/35975>`_, Tiago Melo)
+* mgr/dashboard: Improve Summary's subscribe methods (`pr#35705 <https://github.com/ceph/ceph/pull/35705>`_, Tiago Melo)
+* mgr/dashboard: Prometheus query error in the metrics of Pools, OSDs and RBD images (`pr#35885 <https://github.com/ceph/ceph/pull/35885>`_, Avan Thakkar)
+* mgr/dashboard: Re-enable OSD's table autoReload (`pr#36226 <https://github.com/ceph/ceph/pull/36226>`_, Kiefer Chang, Tiago Melo)
+* mgr/dashboard: Strange iSCSI discovery auth behavior (`pr#36782 <https://github.com/ceph/ceph/pull/36782>`_, Volker Theile)
+* mgr/dashboard: The max. buckets field in RGW user form should be pre-filled (`pr#35795 <https://github.com/ceph/ceph/pull/35795>`_, Volker Theile)
+* mgr/dashboard: Unable to edit iSCSI logged-in client (`pr#36611 <https://github.com/ceph/ceph/pull/36611>`_, Ricardo Marques)
+* mgr/dashboard: Use right size in pool form (`pr#35925 <https://github.com/ceph/ceph/pull/35925>`_, Stephan Müller)
+* mgr/dashboard: Use same required field message accross the UI (`pr#36277 <https://github.com/ceph/ceph/pull/36277>`_, Volker Theile)
+* mgr/dashboard: add API team to CODEOWNERS (`pr#36143 <https://github.com/ceph/ceph/pull/36143>`_, Ernesto Puerta)
+* mgr/dashboard: allow preserving OSD IDs when deleting OSDs (`pr#35766 <https://github.com/ceph/ceph/pull/35766>`_, Kiefer Chang)
+* mgr/dashboard: cpu stats incorrectly displayed (`pr#36322 <https://github.com/ceph/ceph/pull/36322>`_, Avan Thakkar)
+* mgr/dashboard: cropped actions menu in nested details (`pr#35620 <https://github.com/ceph/ceph/pull/35620>`_, Avan Thakkar)
+* mgr/dashboard: fix Source column i18n issue in RBD configuration tables (`pr#35819 <https://github.com/ceph/ceph/pull/35819>`_, Kiefer Chang)
+* mgr/dashboard: fix backporting issue #35926 (`pr#36073 <https://github.com/ceph/ceph/pull/36073>`_, Ernesto Puerta)
+* mgr/dashboard: fix pool usage calculation (`pr#36137 <https://github.com/ceph/ceph/pull/36137>`_, Ernesto Puerta)
+* mgr/dashboard: fix rbdmirroring dropdown menu (`pr#36382 <https://github.com/ceph/ceph/pull/36382>`_, Avan Thakkar)
+* mgr/dashboard: fix regression in delete OSD modal (`pr#36419 <https://github.com/ceph/ceph/pull/36419>`_, Kiefer Chang)
+* mgr/dashboard: fix tasks.mgr.dashboard.test_rbd.RbdTest.test_move_image_to_trash error (`pr#36563 <https://github.com/ceph/ceph/pull/36563>`_, Kiefer Chang)
+* mgr/dashboard: fix ui api endpoints (`pr#36160 <https://github.com/ceph/ceph/pull/36160>`_, Fabrizio D'Angelo)
+* mgr/dashboard: fix wal/db slots controls in the OSD form (`pr#35883 <https://github.com/ceph/ceph/pull/35883>`_, Kiefer Chang)
+* mgr/dashboard: increase API test coverage in API controllers (`pr#36260 <https://github.com/ceph/ceph/pull/36260>`_, Kefu Chai, Aashish Sharma)
+* mgr/dashboard: redirect to original URL after successful login (`pr#36831 <https://github.com/ceph/ceph/pull/36831>`_, Avan Thakkar)
+* mgr/dashboard: remove "This week/month/year" and "Today" time stamps (`pr#36789 <https://github.com/ceph/ceph/pull/36789>`_, Avan Thakkar)
+* mgr/dashboard: remove cdCopy2ClipboardButton `formatted` attribute (`pr#35889 <https://github.com/ceph/ceph/pull/35889>`_, Tatjana Dehler)
+* mgr/dashboard: remove password field if login is using SSO and fix error message in confirm password (`pr#36689 <https://github.com/ceph/ceph/pull/36689>`_, Ishan Rai)
+* mgr/dashboard: right-align dropdown menu of column filters (`pr#36369 <https://github.com/ceph/ceph/pull/36369>`_, Kiefer Chang)
+* mgr/dashboard: telemetry activation notification (`pr#35772 <https://github.com/ceph/ceph/pull/35772>`_, Tatjana Dehler)
+* mgr/dashboard: wait longer for health status to be cleared (`pr#36346 <https://github.com/ceph/ceph/pull/36346>`_, Tatjana Dehler)
+* mgr/k8sevents: sanitise kubernetes events (`pr#35684 <https://github.com/ceph/ceph/pull/35684>`_, Paul Cuzner)
+* mgr/prometheus: improve cache (`pr#35847 <https://github.com/ceph/ceph/pull/35847>`_, Patrick Seidensal)
+* mgr: avoid false alarm of MGR_MODULE_ERROR (`pr#35995 <https://github.com/ceph/ceph/pull/35995>`_, Kefu Chai)
+* mgr: mgr/DaemonServer.cc: make 'config show' on fsid work (`pr#35793 <https://github.com/ceph/ceph/pull/35793>`_, Neha Ojha)
+* mgr: mgr/cephadm: Adapt Vagrantfile to use octopus instead of master repo on shaman (`pr#35988 <https://github.com/ceph/ceph/pull/35988>`_, Volker Theile)
+* mgr: mgr/diskprediction_local: Fix array size error (`pr#36577 <https://github.com/ceph/ceph/pull/36577>`_, Benoît Knecht)
+* mgr: mgr/progress: Skip pg_summary update if _events dict is empty (`pr#36076 <https://github.com/ceph/ceph/pull/36076>`_, Manuel Lausch)
+* mgr: mgr/prometheus: log time it takes to collect metrics (`pr#36581 <https://github.com/ceph/ceph/pull/36581>`_, Patrick Seidensal)
+* mgr: mgr: Add missing states to PG_STATES in mgr_module.py (`pr#36786 <https://github.com/ceph/ceph/pull/36786>`_, Harley Gorrell)
+* mgr: mgr: fix race between module load and notify (`pr#35794 <https://github.com/ceph/ceph/pull/35794>`_, Mykola Golub)
+* mgr: mon/PGMap: do not consider changing pg stuck (`pr#35958 <https://github.com/ceph/ceph/pull/35958>`_, Kefu Chai)
+* monitoring: alert for pool fill up broken (`pr#35136 <https://github.com/ceph/ceph/pull/35136>`_, Volker Theile)
+* msgr: New msgr2 crc and secure modes (msgr2.1) (`pr#35720 <https://github.com/ceph/ceph/pull/35720>`_, Ilya Dryomov)
+* rbd,tests: tests/rbd_mirror: fix race on test shut down (`pr#36657 <https://github.com/ceph/ceph/pull/36657>`_, Mykola Golub)
+* rbd: librbd:  global and pool-level config overrides require image refresh to apply (`pr#36638 <https://github.com/ceph/ceph/pull/36638>`_, Jason Dillaman)
+* rbd: librbd: new 'write_zeroes' API methods to suppliment the `discard` APIs (`pr#36247 <https://github.com/ceph/ceph/pull/36247>`_, Jason Dillaman)
+* rbd: librbd: potential race conditions handling API IO completions (`pr#36331 <https://github.com/ceph/ceph/pull/36331>`_, Jason Dillaman)
+* rbd: mgr/dashboard: work with v1 RBD images (`pr#35711 <https://github.com/ceph/ceph/pull/35711>`_, Ernesto Puerta)
+* rbd: rbd: librbd: Align rbd_write_zeroes declarations (`pr#36717 <https://github.com/ceph/ceph/pull/36717>`_, Corey Bryant)
+* rbd: rbd: librbd: don't resend async_complete if watcher is unregistered (`pr#36659 <https://github.com/ceph/ceph/pull/36659>`_, Mykola Golub)
+* rbd: rbd: librbd: flush all queued object IO from simple scheduler (`pr#36658 <https://github.com/ceph/ceph/pull/36658>`_, Jason Dillaman)
+* rbd: rbd: librbd: race when disabling object map with overlapping in-flight writes (`pr#36656 <https://github.com/ceph/ceph/pull/36656>`_, Jason Dillaman)
+* rbd: rbd: recognize crush_location, read_from_replica and compression_hint map options (`pr#36061 <https://github.com/ceph/ceph/pull/36061>`_, Ilya Dryomov)
+* rgw,tests: qa/tasks/ragweed: always set ragweed_repo (`pr#36651 <https://github.com/ceph/ceph/pull/36651>`_, Kefu Chai)
+* rgw:  rgw: lc: fix Segmentation Fault when the tag of the object was not found (`pr#36085 <https://github.com/ceph/ceph/pull/36085>`_, yupeng chen, zhuo li)
+* rgw: Add subuser to OPA request (`pr#36023 <https://github.com/ceph/ceph/pull/36023>`_, Seena Fallah)
+* rgw: Add support wildcard subuser for bucket policy (`pr#36022 <https://github.com/ceph/ceph/pull/36022>`_, Seena Fallah)
+* rgw: Adding data cache and CDN capabilities (`pr#36646 <https://github.com/ceph/ceph/pull/36646>`_, Mark Kogan, Or Friedmann)
+* rgw: Empty reqs_change_state queue before unregistered_reqs (`pr#36650 <https://github.com/ceph/ceph/pull/36650>`_, Soumya Koduri)
+* rgw: add abort multipart date and rule-id header to init multipart upload response (`pr#36649 <https://github.com/ceph/ceph/pull/36649>`_, zhang Shaowen, zhangshaowen)
+* rgw: add access log to the beast frontend (`pr#36024 <https://github.com/ceph/ceph/pull/36024>`_, Mark Kogan)
+* rgw: add check for index entry's existing when adding bucket stats during bucket reshard (`pr#36025 <https://github.com/ceph/ceph/pull/36025>`_, zhang Shaowen)
+* rgw: add negative cache to the system object (`pr#36648 <https://github.com/ceph/ceph/pull/36648>`_, Or Friedmann)
+* rgw: add quota enforcement to CopyObj (`pr#36020 <https://github.com/ceph/ceph/pull/36020>`_, Casey Bodley)
+* rgw: append obj: prevent tail from being GC'ed (`pr#36389 <https://github.com/ceph/ceph/pull/36389>`_, Abhishek Lekshmanan)
+* rgw: bucket list/stats truncates for user w/ >1000 buckets (`pr#36019 <https://github.com/ceph/ceph/pull/36019>`_, J. Eric Ivancich)
+* rgw: cls/rgw: preserve olh entry's name on last unlink (`pr#36652 <https://github.com/ceph/ceph/pull/36652>`_, Casey Bodley)
+* rgw: cls/rgw_gc: Fixing the iterator used to access urgent data map (`pr#36017 <https://github.com/ceph/ceph/pull/36017>`_, Pritha Srivastava)
+* rgw: fix boost::asio::async_write() does not return error (`pr#36647 <https://github.com/ceph/ceph/pull/36647>`_, Mark Kogan)
+* rgw: fix bug where ordered bucket listing gets stuck (`pr#35877 <https://github.com/ceph/ceph/pull/35877>`_, J. Eric Ivancich)
+* rgw: fix double slash (//) killing the gateway (`pr#36654 <https://github.com/ceph/ceph/pull/36654>`_, Theofilos Mouratidis)
+* rgw: fix loop problem with swift stat on account (`pr#36021 <https://github.com/ceph/ceph/pull/36021>`_, Marcus Watts)
+* rgw: fix shutdown crash in RGWAsyncReadMDLogEntries (`pr#36653 <https://github.com/ceph/ceph/pull/36653>`_, Casey Bodley)
+* rgw: introduce safe user-reset-stats (`pr#36655 <https://github.com/ceph/ceph/pull/36655>`_, Yuval Lifshitz, Matt Benjamin)
+* rgw: lc: add lifecycle perf counters (`pr#36018 <https://github.com/ceph/ceph/pull/36018>`_, Mark Kogan, Matt Benjamin)
+* rgw: orphan list teuthology test & fully-qualified domain issue (`pr#36027 <https://github.com/ceph/ceph/pull/36027>`_, J. Eric Ivancich)
+* rgw: orphan-list timestamp fix (`pr#35929 <https://github.com/ceph/ceph/pull/35929>`_, J. Eric Ivancich)
+* rgw: policy: reuse eval_principal to evaluate the policy principal (`pr#36636 <https://github.com/ceph/ceph/pull/36636>`_, Abhishek Lekshmanan)
+* rgw: radoslist incomplete multipart uploads fix marker progression (`pr#36028 <https://github.com/ceph/ceph/pull/36028>`_, J. Eric Ivancich)
+* rgw: rgw/iam: correcting the result of get role policy (`pr#36645 <https://github.com/ceph/ceph/pull/36645>`_, Pritha Srivastava)
+* rgw: selinux: allow ceph_t amqp_port_t:tcp_socket (`pr#36026 <https://github.com/ceph/ceph/pull/36026>`_, Kaleb S. KEITHLEY, Thomas Serlin)
+* rgw: stop realm reloader before store shutdown (`pr#36644 <https://github.com/ceph/ceph/pull/36644>`_, Kefu Chai, Casey Bodley)
+* tools: tools: Add statfs operation to ceph-objecstore-tool (`pr#35715 <https://github.com/ceph/ceph/pull/35715>`_, David Zafman)
+
+v15.2.4 Octopus
+===============
+
+This is the fourth release of the Ceph Octopus stable release series.
+In addition to a security fix in RGW, this release brings a range of fixes
+across all components. We recommend that all Octopus users upgrade to this
+release.
+
+Notable Changes
+---------------
+
+* CVE-2020-10753: rgw: sanitize newlines in s3 CORSConfiguration's ExposeHeader
+  (William Bowling, Adam Mohammed, Casey Bodley)
+
+* Cephadm: There were a lot of small usability improvements and bug fixes:
+
+  * Grafana when deployed by Cephadm now binds to all network interfaces.
+  * ``cephadm check-host`` now prints all detected problems at once.
+  * Cephadm now calls ``ceph dashboard set-grafana-api-ssl-verify false``
+    when generating an SSL certificate for Grafana.
+  * The Alertmanager is now correctly pointed to the Ceph Dashboard
+  * ``cephadm adopt`` now supports adopting an Alertmanager
+  * ``ceph orch ps`` now supports filtering by service name
+  * ``ceph orch host ls`` now marks hosts as offline, if they are not
+    accessible.
+
+* Cephadm can now deploy NFS Ganesha services. For example, to deploy NFS with
+  a service id of mynfs, that will use the RADOS pool nfs-ganesha and namespace
+  nfs-ns::
+
+    ceph orch apply nfs mynfs nfs-ganesha nfs-ns
+
+* Cephadm: ``ceph orch ls --export`` now returns all service specifications in
+  yaml representation that is consumable by ``ceph orch apply``. In addition,
+  the commands ``orch ps`` and ``orch ls`` now support ``--format yaml`` and
+  ``--format json-pretty``.
+
+* Cephadm: ``ceph orch apply osd`` supports a ``--preview`` flag that prints a preview of
+  the OSD specification before deploying OSDs. This makes it possible to
+  verify that the specification is correct, before applying it.
+
+* RGW: The ``radosgw-admin`` sub-commands dealing with orphans --
+  ``radosgw-admin orphans find``, ``radosgw-admin orphans finish``, and
+  ``radosgw-admin orphans list-jobs`` -- have been deprecated. They have
+  not been actively maintained and they store intermediate results on
+  the cluster, which could fill a nearly-full cluster.  They have been
+  replaced by a tool, currently considered experimental,
+  ``rgw-orphan-list``.
+
+* RBD: The name of the rbd pool object that is used to store
+  rbd trash purge schedule is changed from "rbd_trash_trash_purge_schedule"
+  to "rbd_trash_purge_schedule". Users that have already started using
+  ``rbd trash purge schedule`` functionality and have per pool or namespace
+  schedules configured should copy "rbd_trash_trash_purge_schedule"
+  object to "rbd_trash_purge_schedule" before the upgrade and remove
+  "rbd_trash_purge_schedule" using the following commands in every RBD
+  pool and namespace where a trash purge schedule was previously
+  configured::
+
+    rados -p <pool-name> [-N namespace] cp rbd_trash_trash_purge_schedule rbd_trash_purge_schedule
+    rados -p <pool-name> [-N namespace] rm rbd_trash_trash_purge_schedule
+
+  or use any other convenient way to restore the schedule after the
+  upgrade.
+
+Changelog
+---------
+
+* build/ops: address SElinux denials observed in rgw/multisite test run (`pr#34538 <https://github.com/ceph/ceph/pull/34538>`_, Kefu Chai, Kaleb S. Keithley)
+* ceph-volume: add and delete lvm tags in a single lvchange call (`pr#35452 <https://github.com/ceph/ceph/pull/35452>`_, Jan Fajerski)
+* ceph-volume: add ceph.osdspec_affinity tag (`pr#35134 <https://github.com/ceph/ceph/pull/35134>`_, Joshua Schmid)
+* cephadm: batch backport May (1) (`pr#34893 <https://github.com/ceph/ceph/pull/34893>`_, Michael Fritch, Ricardo Marques, Matthew Oliver, Sebastian Wagner, Joshua Schmid, Zac Dover, Varsha Rao)
+* cephadm: batch backport May (2) (`pr#35188 <https://github.com/ceph/ceph/pull/35188>`_, Michael Fritch, Sebastian Wagner, Kefu Chai, Georgios Kyratsas, Kiefer Chang, Joshua Schmid, Patrick Seidensal, Varsha Rao, Matthew Oliver, Zac Dover, Juan Miguel Olmo Martínez, Tim Serong, Alexey Miasoedov, Ricardo Marques, Satoru Takeuchi)
+* cephadm: batch backport June (1) (`pr#35347 <https://github.com/ceph/ceph/pull/35347>`_, Sebastian Wagner, Zac Dover, Georgios Kyratsas, Kiefer Chang, Ricardo Marques, Patrick Seidensal, Patrick Donnelly, Joshua Schmid, Matthew Oliver, Varsha Rao, Juan Miguel Olmo Martínez, Michael Fritch)
+* cephadm: batch backport June (2) (`pr#35475 <https://github.com/ceph/ceph/pull/35475>`_, Sebastian Wagner, Kiefer Chang, Joshua Schmid, Michael Fritch, shinhwagk, Kefu Chai, Juan Miguel Olmo Martínez, Daniel Pivonka)
+* cephfs: allow pool names with hyphen and period (`pr#35251 <https://github.com/ceph/ceph/pull/35251>`_, Ramana Raja)
+* cephfs: bash_completion: Do not auto complete obsolete and hidden cmds (`pr#34996 <https://github.com/ceph/ceph/pull/34996>`_, Kotresh HR)
+* cephfs: cephfs-shell: Change tox testenv name to py3 (`pr#34998 <https://github.com/ceph/ceph/pull/34998>`_, Kefu Chai, Varsha Rao, Aditya Srivastava)
+* cephfs: client: expose Client::ll_register_callback via libcephfs (`pr#35150 <https://github.com/ceph/ceph/pull/35150>`_, Jeff Layton)
+* cephfs: client: fix Finisher assert failure (`pr#34999 <https://github.com/ceph/ceph/pull/34999>`_, Xiubo Li)
+* cephfs: client: only set MClientCaps::FLAG_SYNC when flushing dirty auth caps (`pr#34997 <https://github.com/ceph/ceph/pull/34997>`_, Jeff Layton)
+* cephfs: fuse: add the '-d' option back for libfuse (`pr#35449 <https://github.com/ceph/ceph/pull/35449>`_, Xiubo Li)
+* cephfs: mds: Handle blacklisted error in purge queue (`pr#35148 <https://github.com/ceph/ceph/pull/35148>`_, Varsha Rao)
+* cephfs: mds: preserve ESlaveUpdate logevent until receiving OP_FINISH (`pr#35253 <https://github.com/ceph/ceph/pull/35253>`_, songxinying)
+* cephfs: mds: take xlock in the order requests start locking (`pr#35252 <https://github.com/ceph/ceph/pull/35252>`_, "Yan, Zheng")
+* cephfs: src/client/fuse_ll: compatible with libfuse3.5 or higher (`pr#35450 <https://github.com/ceph/ceph/pull/35450>`_, Jeff Layton, Xiubo Li)
+* cephfs: vstart_runner: set mounted to True at the end of mount() (`pr#35447 <https://github.com/ceph/ceph/pull/35447>`_, Rishabh Dave)
+* core: bluestore: fix large (>2GB) writes when bluefs_buffered_io = true (`pr#35446 <https://github.com/ceph/ceph/pull/35446>`_, Igor Fedotov)
+* core: bluestore: introduce hybrid allocator (`pr#35498 <https://github.com/ceph/ceph/pull/35498>`_, Igor Fedotov, Adam Kupczyk)
+* core: cls/queue: fix empty markers when listing entries (`pr#35241 <https://github.com/ceph/ceph/pull/35241>`_, Pritha Srivastava, Yuval Lifshitz)
+* core: objecter: don't attempt to read from non-primary on EC pools (`pr#35444 <https://github.com/ceph/ceph/pull/35444>`_, Ilya Dryomov)
+* core: osd: add --osdspec-affinity flag (`pr#35382 <https://github.com/ceph/ceph/pull/35382>`_, Joshua Schmid)
+* core: osd: make "missing incremental map" a debug log message (`pr#35442 <https://github.com/ceph/ceph/pull/35442>`_, Nathan Cutler)
+* core: osd: prevent ShardedOpWQ suicide_grace drop when waiting for work (`pr#34881 <https://github.com/ceph/ceph/pull/34881>`_, Dan Hill)
+* core: rocksdb: Update to ceph-octopus-v5.8-1436 (`pr#35036 <https://github.com/ceph/ceph/pull/35036>`_, Brad Hubbard)
+* doc: drop obsolete cache tier options (`pr#35105 <https://github.com/ceph/ceph/pull/35105>`_, Nathan Cutler)
+* doc: mgr/dashboard: Add troubleshooting guide (`pr#34947 <https://github.com/ceph/ceph/pull/34947>`_, Tatjana Dehler)
+* doc: rgw: document 'rgw gc max concurrent io' (`pr#34987 <https://github.com/ceph/ceph/pull/34987>`_, Casey Bodley)
+* mds: cleanup uncommitted fragments before mds goes to active (`pr#35448 <https://github.com/ceph/ceph/pull/35448>`_, "Yan, Zheng")
+* mds: don't assert empty io context list when shutting down (`pr#34509 <https://github.com/ceph/ceph/pull/34509>`_, "Yan, Zheng")
+* mds: don't shallow copy when decoding xattr map (`pr#35147 <https://github.com/ceph/ceph/pull/35147>`_, "Yan, Zheng")
+* mds: flag backtrace scrub failures for new files as okay (`pr#35555 <https://github.com/ceph/ceph/pull/35555>`_, Milind Changire)
+* mgr/dashboard/grafana: Add rbd-image details dashboard (`pr#35247 <https://github.com/ceph/ceph/pull/35247>`_, Enno Gotthold)
+* mgr/dashboard: Asynchronous unique username validation for User Component (`pr#34849 <https://github.com/ceph/ceph/pull/34849>`_, Nizamudeen)
+* mgr/dashboard: ECP modal enhancement (`pr#35152 <https://github.com/ceph/ceph/pull/35152>`_, Stephan Müller)
+* mgr/dashboard: Fix HomeTest setup (`pr#35085 <https://github.com/ceph/ceph/pull/35085>`_, Tiago Melo)
+* mgr/dashboard: Fix e2e chromium binary validation (`pr#35679 <https://github.com/ceph/ceph/pull/35679>`_, Tiago Melo)
+* mgr/dashboard: Fix random E2E error in mgr-modules (`pr#35706 <https://github.com/ceph/ceph/pull/35706>`_, Tiago Melo)
+* mgr/dashboard: Fix redirect after changing password (`pr#35243 <https://github.com/ceph/ceph/pull/35243>`_, Tiago Melo)
+* mgr/dashboard: Prevent dashboard breakdown on bad pool selection (`pr#35135 <https://github.com/ceph/ceph/pull/35135>`_, Stephan Müller)
+* mgr/dashboard: Proposed About Modal box (`pr#35291 <https://github.com/ceph/ceph/pull/35291>`_, Ngwa Sedrick Meh, Tiago Melo)
+* mgr/dashboard: Reduce requests in Mirroring page (`pr#34992 <https://github.com/ceph/ceph/pull/34992>`_, Tiago Melo)
+* mgr/dashboard: Replace Protractor with Cypress (`pr#34910 <https://github.com/ceph/ceph/pull/34910>`_, Tiago Melo)
+* mgr/dashboard: Show labels in hosts page (`pr#35517 <https://github.com/ceph/ceph/pull/35517>`_, Volker Theile)
+* mgr/dashboard: Show table details inside the datatable (`pr#35270 <https://github.com/ceph/ceph/pull/35270>`_, Sebastian Krah)
+* mgr/dashboard: add telemetry report component (`pr#34850 <https://github.com/ceph/ceph/pull/34850>`_, Tatjana Dehler)
+* mgr/dashboard: displaying Service detail inside table (`pr#35269 <https://github.com/ceph/ceph/pull/35269>`_, Kiefer Chang)
+* mgr/dashboard: fix autocomplete input backgrounds in chrome and firefox (`pr#35718 <https://github.com/ceph/ceph/pull/35718>`_, Ishan Rai)
+* mgr/dashboard: grafana panels for rgw multisite sync performance (`pr#35693 <https://github.com/ceph/ceph/pull/35693>`_, Alfonso Martínez)
+* mgr/dashboard: monitoring menu entry should indicate firing alerts (`pr#34822 <https://github.com/ceph/ceph/pull/34822>`_, Tiago Melo, Volker Theile)
+* mgr/dashboard: redesign the login screen (`pr#35268 <https://github.com/ceph/ceph/pull/35268>`_, Ishan Rai)
+* mgr/dashboard: remove space after service name in the Hosts List table (`pr#35531 <https://github.com/ceph/ceph/pull/35531>`_, Kiefer Chang)
+* mgr/dashboard: replace hard coded telemetry URLs (`pr#35231 <https://github.com/ceph/ceph/pull/35231>`_, Tatjana Dehler)
+* mgr/rbd_support: rename "rbd_trash_trash_purge_schedule" oid (`pr#35436 <https://github.com/ceph/ceph/pull/35436>`_, Nathan Cutler, Mykola Golub)
+* mgr/status: Fix "ceph fs status" json format writing to stderr (`pr#34727 <https://github.com/ceph/ceph/pull/34727>`_, Kotresh HR)
+* mgr/test_orchestrator: fix _get_ceph_daemons() (`pr#34979 <https://github.com/ceph/ceph/pull/34979>`_, Alfonso Martínez)
+* mgr/volumes: Add snapshot info command (`pr#35670 <https://github.com/ceph/ceph/pull/35670>`_, Kotresh HR)
+* mgr/volumes: Create subvolume with isolated rados namespace (`pr#35671 <https://github.com/ceph/ceph/pull/35671>`_, Kotresh HR)
+* mgr/volumes: Fix subvolume create idempotency (`pr#35256 <https://github.com/ceph/ceph/pull/35256>`_, Kotresh HR)
+* mgr: synchronize ClusterState's health and mon_status (`pr#34995 <https://github.com/ceph/ceph/pull/34995>`_, Radoslaw Zarzynski)
+* monitoring: Fix "10% OSDs down" alert description (`pr#35151 <https://github.com/ceph/ceph/pull/35151>`_, Benoît Knecht)
+* monitoring: fixing some issues in RBD detail dashboard (`pr#35463 <https://github.com/ceph/ceph/pull/35463>`_, Kiefer Chang)
+* rbd: librbd: Watcher should not attempt to re-watch after detecting blacklisting (`pr#35439 <https://github.com/ceph/ceph/pull/35439>`_, Jason Dillaman)
+* rbd: librbd: avoid completing mirror:DisableRequest while holding its lock (`pr#35126 <https://github.com/ceph/ceph/pull/35126>`_, Jason Dillaman)
+* rbd: librbd: copy API should not inherit v1 image format by default (`pr#35255 <https://github.com/ceph/ceph/pull/35255>`_, Jason Dillaman)
+* rbd: librbd: make rbd_read_from_replica_policy actually work (`pr#35438 <https://github.com/ceph/ceph/pull/35438>`_, Ilya Dryomov)
+* rbd: pybind: RBD.create() method's 'old_format' parameter now defaults to False (`pr#35435 <https://github.com/ceph/ceph/pull/35435>`_, Jason Dillaman)
+* rbd: rbd-mirror: don't hold (stale) copy of local image journal pointer (`pr#35430 <https://github.com/ceph/ceph/pull/35430>`_, Jason Dillaman)
+* rbd: rbd-mirror: stop local journal replayer first during shut down (`pr#35440 <https://github.com/ceph/ceph/pull/35440>`_, Jason Dillaman, Mykola Golub)
+* rbd: rbd-mirror: wait for in-flight start/stop/restart (`pr#35437 <https://github.com/ceph/ceph/pull/35437>`_, Mykola Golub)
+* rgw: add "rgw-orphan-list" tool and "radosgw-admin bucket radoslist ..." (`pr#34991 <https://github.com/ceph/ceph/pull/34991>`_, J. Eric Ivancich)
+* rgw: amqp: fix the "routable" delivery mode (`pr#35433 <https://github.com/ceph/ceph/pull/35433>`_, Yuval Lifshitz)
+* rgw: anonomous swift to obj that dont exist should 401 (`pr#35120 <https://github.com/ceph/ceph/pull/35120>`_, Matthew Oliver)
+* rgw: fix bug where bucket listing end marker not always set correctly (`pr#34993 <https://github.com/ceph/ceph/pull/34993>`_, J. Eric Ivancich)
+* rgw: fix rgw tries to fetch anonymous user (`pr#34988 <https://github.com/ceph/ceph/pull/34988>`_, Or Friedmann)
+* rgw: fix some list buckets handle leak (`pr#34985 <https://github.com/ceph/ceph/pull/34985>`_, Tianshan Qu)
+* rgw: gc: Clearing off urgent data in bufferlist, before (`pr#35434 <https://github.com/ceph/ceph/pull/35434>`_, Pritha Srivastava)
+* rgw: lc: enable thread-parallelism in RGWLC (`pr#35431 <https://github.com/ceph/ceph/pull/35431>`_, Matt Benjamin)
+* rgw: notifications: fix zero size in notifications (`pr#34940 <https://github.com/ceph/ceph/pull/34940>`_, J. Eric Ivancich, Yuval Lifshitz)
+* rgw: notifications: version id was not sent in versioned buckets (`pr#35254 <https://github.com/ceph/ceph/pull/35254>`_, Yuval Lifshitz)
+* rgw: radosgw-admin: fix infinite loops in 'datalog list' (`pr#34989 <https://github.com/ceph/ceph/pull/34989>`_, Casey Bodley)
+* rgw: url: fix amqp urls with vhosts (`pr#35432 <https://github.com/ceph/ceph/pull/35432>`_, Yuval Lifshitz)
+* tests: migrate qa/ to Python3 (`pr#35364 <https://github.com/ceph/ceph/pull/35364>`_, Kyr Shatskyy, Ilya Dryomov, Xiubo Li, Kefu Chai, Casey Bodley, Rishabh Dave, Patrick Donnelly, Sidharth Anupkrishnan, Michael Fritch)
+
+
+v15.2.3 Octopus
+===============
+
+This is the third bug-fix release of the Ceph Octopus stable release series.
+This release mainly is a workaround for a potential OSD corruption in v15.2.2.
+We advise users to upgrade to v15.2.3 directly. For users running v15.2.2 please
+execute the following::
+
+  ceph config set osd bluefs_preextend_wal_files false
+
+Changelog
+---------
+
+* bluestore: remove preextended WAL support (`issue#45613 <http://tracker.ceph.com/issues/45613>`_, Igor Fedotov, Neha Ojha)
+
+
+v15.2.2 Octopus
+===============
+
+This is the second bug-fix release of the Ceph Octopus stable release series. This release
+brings a range of fixes across all components, as well as patching a security flaw. We
+recommend that all Octopus users upgrade.
+
+Notable Changes
+---------------
+
+* CVE-2020-10736: Fixed an authorization bypass in mons & mgrs (Olle SegerDahl, Josh Durgin)
+
+Changelog
+---------
+
+* bluestore,core: common/options: Disable bluefs_buffered_io by default again (`pr#34353 <https://github.com/ceph/ceph/pull/34353>`_, Mark Nelson)
+* bluestore: os/bluestore: Don't pollute old journal when add new device (`pr#34795 <https://github.com/ceph/ceph/pull/34795>`_, Yang Honggang)
+* bluestore: os/bluestore: fix 'unused' calculation (`pr#34793 <https://github.com/ceph/ceph/pull/34793>`_, Igor Fedotov, xie xingguo)
+* bluestore: os/bluestore: open DB in read-only when expanding DB/WAL (`pr#34610 <https://github.com/ceph/ceph/pull/34610>`_, Adam Kupczyk, Igor Fedotov)
+* build/ops: rpm: add python3-saml as install dependency (`pr#34474 <https://github.com/ceph/ceph/pull/34474>`_, Ernesto Puerta)
+* build/ops: rpm: drop "is_opensuse" conditional in SUSE-specific bcond block (`pr#34790 <https://github.com/ceph/ceph/pull/34790>`_, Nathan Cutler)
+* build/ops: spec: address some warnings raised by RPM 4.15.1 (`pr#34526 <https://github.com/ceph/ceph/pull/34526>`_, Nathan Cutler)
+* ceph-volume/batch: check lvs list before access (`pr#34480 <https://github.com/ceph/ceph/pull/34480>`_, Jan Fajerski)
+* ceph-volume/batch: return success when all devices are filtered (`pr#34477 <https://github.com/ceph/ceph/pull/34477>`_, Jan Fajerski)
+* ceph-volume: update functional testing deploy.yml playbook (`pr#34886 <https://github.com/ceph/ceph/pull/34886>`_, Guillaume Abrioux)
+* cephadm: Fix check_ip_port to work with IPv6 (`pr#34350 <https://github.com/ceph/ceph/pull/34350>`_, Ricardo Marques)
+* cephadm: Update images used (`pr#34686 <https://github.com/ceph/ceph/pull/34686>`_, Sebastian Wagner)
+* cephadm: ceph-volume: disallow concurrent execution (`pr#34423 <https://github.com/ceph/ceph/pull/34423>`_, Sage Weil)
+* cephadm: rm-cluster clean up /etc/ceph (`pr#34299 <https://github.com/ceph/ceph/pull/34299>`_, Daniel-Pivonka)
+* cephfs,mgr: mgr/volumes: Add interface to get subvolume metadata (`pr#34681 <https://github.com/ceph/ceph/pull/34681>`_, Kotresh HR)
+* cephfs,mgr: mgr: force purge normal ceph entities from service map (`issue#44677 <http://tracker.ceph.com/issues/44677>`_, `pr#34800 <https://github.com/ceph/ceph/pull/34800>`_, Venky Shankar)
+* cephfs,tools: cephfs-journal-tool: correctly parse --dry_run argument (`pr#34804 <https://github.com/ceph/ceph/pull/34804>`_, Milind Changire)
+* cephfs,tools: tools/cephfs: add accounted_rstat/rstat when building file dentry (`pr#34803 <https://github.com/ceph/ceph/pull/34803>`_, Xiubo Li)
+* cephfs: ceph-fuse: link to libfuse3 and pass `-o big_writes` to libfuse if libfuse < 3.0.0 (`pr#34769 <https://github.com/ceph/ceph/pull/34769>`_, Xiubo Li, "Yan, Zheng", Kefu Chai)
+* cephfs: client: reset requested_max_size if file write is not wanted (`pr#34766 <https://github.com/ceph/ceph/pull/34766>`_, "Yan, Zheng")
+* cephfs: mds: fix 'if there is lock cache on dir' check (`pr#34273 <https://github.com/ceph/ceph/pull/34273>`_, "Yan, Zheng")
+* cephfs: mon/FSCommands: Fix 'add_data_pool' command and 'fs new' command (`pr#34775 <https://github.com/ceph/ceph/pull/34775>`_, Ramana Raja)
+* cephfs: qa: install task runs twice with double unwind causing fatal errors (`pr#34912 <https://github.com/ceph/ceph/pull/34912>`_, Patrick Donnelly)
+* core,mon: mon/OSDMonitor: allow trimming maps even if osds are down (`pr#34924 <https://github.com/ceph/ceph/pull/34924>`_, Joao Eduardo Luis)
+* core: ceph-object-corpus: update to octopus (`pr#34797 <https://github.com/ceph/ceph/pull/34797>`_, Josh Durgin)
+* core: mgr/DaemonServer: fetch metadata for new daemons (e.g., mons) (`pr#34416 <https://github.com/ceph/ceph/pull/34416>`_, Sage Weil)
+* core: mon/OSDMonitor: Always tune priority cache manager memory on all mons (`pr#34917 <https://github.com/ceph/ceph/pull/34917>`_, Sridhar Seshasayee)
+* core: mon: calculate min_size on osd pool set size (`pr#34528 <https://github.com/ceph/ceph/pull/34528>`_, Deepika Upadhyay)
+* core: osd/PeeringState: do not trim pg log past last_update_ondisk (`pr#34807 <https://github.com/ceph/ceph/pull/34807>`_, xie xingguo, Samuel Just)
+* core: osd/PrimaryLogPG: fix SPARSE_READ stat (`pr#34809 <https://github.com/ceph/ceph/pull/34809>`_, Yan Jun)
+* devices/simple/scan: Fix string in log statement (`pr#34446 <https://github.com/ceph/ceph/pull/34446>`_, Jan Fajerski)
+* doc: cephadm: Batch backport April (1) (`pr#34554 <https://github.com/ceph/ceph/pull/34554>`_, Matthew Oliver, Sage Weil, Sebastian Wagner, Michael Fritch, Tim, Jeff Layton, Juan Miguel Olmo Martínez, Joshua Schmid)
+* doc: cephadm: Batch backport April (2) (`issue#45029 <http://tracker.ceph.com/issues/45029>`_, `pr#34687 <https://github.com/ceph/ceph/pull/34687>`_, Maran Hidskes, Kiefer Chang, Matthew Oliver, Sebastian Wagner, Andreas Haase, Tim Serong, Zac Dover, Michael Fritch, Joshua Schmid)
+* doc: cephadm: Batch backport April (3) (`pr#34742 <https://github.com/ceph/ceph/pull/34742>`_, Sebastian Wagner, Dimitri Savineau, Michael Fritch)
+* doc: cephadm: batch backport March (`pr#34438 <https://github.com/ceph/ceph/pull/34438>`_, Jan Fajerski, Sebastian Wagner, Daniel-Pivonka, Michael Fritch, Sage Weil)
+* doc: doc/releases/nautilus: restart OSDs to make them bind to v2 addr (`pr#34523 <https://github.com/ceph/ceph/pull/34523>`_, Nathan Cutler)
+* mgr/dashboard: 'Prometheus / All Alerts' page shows progress bar (`pr#34631 <https://github.com/ceph/ceph/pull/34631>`_, Volker Theile)
+* mgr/dashboard: Fix ServiceDetails and PoolDetails unit tests (`pr#34760 <https://github.com/ceph/ceph/pull/34760>`_, Tiago Melo)
+* mgr/dashboard: Fix iSCSI's username and password validation (`pr#34547 <https://github.com/ceph/ceph/pull/34547>`_, Tiago Melo)
+* mgr/dashboard: Improve iSCSI CHAP message (`pr#34630 <https://github.com/ceph/ceph/pull/34630>`_, Ricardo Marques)
+* mgr/dashboard: Prevent iSCSI target recreation when editing controls (`pr#34548 <https://github.com/ceph/ceph/pull/34548>`_, Tiago Melo)
+* mgr/dashboard: RGW auto refresh is not working (`pr#34739 <https://github.com/ceph/ceph/pull/34739>`_, Avan Thakkar)
+* mgr/dashboard: Repair broken grafana panels (`pr#34495 <https://github.com/ceph/ceph/pull/34495>`_, Kristoffer Grönlund)
+* mgr/dashboard: Update translations on octopus (`pr#34309 <https://github.com/ceph/ceph/pull/34309>`_, Sebastian Krah)
+* mgr/dashboard: add crush rule test suite (`pr#34211 <https://github.com/ceph/ceph/pull/34211>`_, Tatjana Dehler)
+* mgr/dashboard: fix API tests to be py3 compatible (`pr#34759 <https://github.com/ceph/ceph/pull/34759>`_, Kefu Chai, Laura Paduano, Alfonso Martínez)
+* mgr/dashboard: fix errors related to frontend service subscriptions (`pr#34467 <https://github.com/ceph/ceph/pull/34467>`_, Alfonso Martínez)
+* mgr/dashboard: fix tasks.mgr.dashboard.test_rgw.RgwBucketTest.test_all (`pr#34708 <https://github.com/ceph/ceph/pull/34708>`_, Alfonso Martínez)
+* mgr/dashboard: lint error on plugins/debug.py (`pr#34625 <https://github.com/ceph/ceph/pull/34625>`_, Volker Theile)
+* mgr/dashboard: shorten "Container ID" and "Container image ID" in Services page (`pr#34648 <https://github.com/ceph/ceph/pull/34648>`_, Volker Theile)
+* mgr/dashboard: use FQDN for failover redirection (`pr#34498 <https://github.com/ceph/ceph/pull/34498>`_, Ernesto Puerta)
+* mgr: mgr/PyModule: fix missing tracebacks in handle_pyerror() (`pr#34626 <https://github.com/ceph/ceph/pull/34626>`_, Tim Serong)
+* mgr: mgr/telegraf: catch FileNotFoundError exception (`pr#34629 <https://github.com/ceph/ceph/pull/34629>`_, Kefu Chai)
+* monitoring: Fix pool capacity incorrect (`pr#34449 <https://github.com/ceph/ceph/pull/34449>`_, James Cheng)
+* monitoring: alert for prediction of disk and pool fill up broken (`pr#34395 <https://github.com/ceph/ceph/pull/34395>`_, Patrick Seidensal)
+* monitoring: fix decimal precision in Grafana %percentages (`pr#34828 <https://github.com/ceph/ceph/pull/34828>`_, Ernesto Puerta)
+* monitoring: root volume full alert fires false positives (`pr#34418 <https://github.com/ceph/ceph/pull/34418>`_, Patrick Seidensal)
+* pybind,rbd: pybind/rbd: ensure image is open before permitting operations (`pr#34425 <https://github.com/ceph/ceph/pull/34425>`_, Mykola Golub)
+* pybind,rbd: pybind/rbd: fix no lockers are obtained, ImageNotFound exception will be output (`pr#34387 <https://github.com/ceph/ceph/pull/34387>`_, zhangdaolong)
+* qa/suites/rados/cephadm/upgrade: start from v15.2.0 (`pr#34440 <https://github.com/ceph/ceph/pull/34440>`_, Sage Weil)
+* qa/tasks/cephadm: add 'roleless' mode (`pr#34407 <https://github.com/ceph/ceph/pull/34407>`_, Sage Weil)
+* rbd,tests: tests: update unmap.t for table spacing changes (`pr#34819 <https://github.com/ceph/ceph/pull/34819>`_, Ilya Dryomov)
+* rbd: rbd-mirror: improved replication statistics (`pr#34810 <https://github.com/ceph/ceph/pull/34810>`_, Mykola Golub, Jason Dillaman)
+* rbd: rbd: ignore tx-only mirror peers when adding new peers (`pr#34638 <https://github.com/ceph/ceph/pull/34638>`_, Jason Dillaman)
+* rgw: Disable prefetch of entire head object when GET request with range header (`pr#34826 <https://github.com/ceph/ceph/pull/34826>`_, Or Friedmann)
+* rgw: pubsub sync module ignores ERR_USER_EXIST (`pr#34825 <https://github.com/ceph/ceph/pull/34825>`_, Casey Bodley)
+* rgw: radosgw-admin: add support for --bucket-id in bucket stats command (`pr#34816 <https://github.com/ceph/ceph/pull/34816>`_, Vikhyat Umrao)
+* rgw: reshard: skip stale bucket id entries from reshard queue (`pr#34734 <https://github.com/ceph/ceph/pull/34734>`_, Abhishek Lekshmanan)
+* rgw: use DEFER_DROP_PRIVILEGES flag unconditionally (`pr#34731 <https://github.com/ceph/ceph/pull/34731>`_, Casey Bodley)
+
+
+v15.2.1 Octopus
+===============
+
+This is the first bugfix release of Ceph Octopus, we recommend all Octopus users
+upgrade. This release fixes an upgrade issue and also has 2 security fixes
+
+Notable Changes
+---------------
+
+* issue#44759: Fixed luminous->nautilus->octopus upgrade asserts
+* CVE-2020-1759: Fixed nonce reuse in msgr V2 secure mode
+* CVE-2020-1760: Fixed XSS due to RGW GetObject header-splitting
+
+Changelog
+---------
+
+* build/ops: fix ceph_release type to 'stable' (`pr#34194 <https://github.com/ceph/ceph/pull/34194>`_, Sage Weil)
+* build/ops: vstart_runner.py: fix OSError when checking if non-existent path is mounted (`pr#34132 <https://github.com/ceph/ceph/pull/34132>`_, Alfonso Martínez)
+* cephadm: Add alertmanager adopt (`pr#34157 <https://github.com/ceph/ceph/pull/34157>`_, Eric Jackson)
+* cephadm: Add alertmanager sample (`pr#34158 <https://github.com/ceph/ceph/pull/34158>`_, Eric Jackson)
+* cephadm: Fix truncated output of "ceph mgr dump" (`pr#34258 <https://github.com/ceph/ceph/pull/34258>`_, Sebastian Wagner)
+* mgr/cephadm: Add example to run when debugging ssh failures (`pr#34153 <https://github.com/ceph/ceph/pull/34153>`_, Sebastian Wagner)
+* mgr/cephadm: DriveGroupSpec needs to support/ignore _unmanaged_ (`pr#34185 <https://github.com/ceph/ceph/pull/34185>`_, Joshua Schmid)
+* mgr/cephadm: bind grafana to all interfaces (`pr#34191 <https://github.com/ceph/ceph/pull/34191>`_, Sage Weil)
+* mgr/cephadm: fix 'orch ps --refresh' (`pr#34190 <https://github.com/ceph/ceph/pull/34190>`_, Sage Weil)
+* mgr/cephadm: fix 'upgrade start' message when specifying a version (`pr#34186 <https://github.com/ceph/ceph/pull/34186>`_, Sage Weil)
+* mgr/cephadm: include alerts in prometheus deployment (`pr#34155 <https://github.com/ceph/ceph/pull/34155>`_, Sage Weil)
+* mgr/cephadm: point alertmanager at all mgr/dashboard URLs (`pr#34154 <https://github.com/ceph/ceph/pull/34154>`_, Sage Weil)
+* mgr/cephadm: provision nfs-ganesha via orchestrator (`pr#34192 <https://github.com/ceph/ceph/pull/34192>`_, Michael Fritch)
+* mgr/dashboard: Check for missing npm resolutions (`pr#34202 <https://github.com/ceph/ceph/pull/34202>`_, Tiago Melo)
+* mgr/dashboard: NoRebalance flag is added to the Dashboard (`pr#33939 <https://github.com/ceph/ceph/pull/33939>`_, Nizamudeen)
+* mgr/dashboard: correct Orchestrator documentation link (`pr#34212 <https://github.com/ceph/ceph/pull/34212>`_, Tatjana Dehler)
+* mgr/dashboard: do not fail on user creation (CLI) (`pr#34280 <https://github.com/ceph/ceph/pull/34280>`_, Tatjana Dehler)
+* mgr/orch: allow list daemons by service_name (`pr#34160 <https://github.com/ceph/ceph/pull/34160>`_, Kiefer Chang)
+* mgr/prometheus: ceph_pg\_\* metrics contains last value instead of sum across all reported states (`pr#34163 <https://github.com/ceph/ceph/pull/34163>`_, Jacek Suchenia)
+* mgr/rook: Blinking lights (`pr#34199 <https://github.com/ceph/ceph/pull/34199>`_, Juan Miguel Olmo Martínez)
+* osd/PeeringState: drop mimic assert (`pr#34204 <https://github.com/ceph/ceph/pull/34204>`_, Sage Weil)
+* osd/PeeringState: fix pending want_acting vs osd offline race (`pr#34123 <https://github.com/ceph/ceph/pull/34123>`_, xie xingguo)
+* pybind/mgr: fix config_notify handling of default values (`pr#34178 <https://github.com/ceph/ceph/pull/34178>`_, Nathan Cutler)
+* rbd: librbd: fix client backwards compatibility issues (`issue#39450 <http://tracker.ceph.com/issues/39450>`_, `issue#38834 <http://tracker.ceph.com/issues/38834>`_, `pr#34323 <https://github.com/ceph/ceph/pull/34323>`_, Jason Dillaman)
+* tools: ceph-backport.sh: add deprecation warning (`pr#34125 <https://github.com/ceph/ceph/pull/34125>`_, Nathan Cutler)
+
+
+v15.2.0 Octopus
+===============
+
+This is the first stable release of Ceph Octopus.
+
+Major Changes from Nautilus
+---------------------------
+
+General
+~~~~~~~
+  
+* A new deployment tool called **cephadm** has been introduced that
+  integrates Ceph daemon deployment and management via containers
+  into the orchestration layer.  For more information see
+  :ref:`cephadm`.
+* Health alerts can now be muted, either temporarily or permanently.
+* Health alerts are now raised for recent Ceph daemons crashes.
+* A simple 'alerts' module has been introduced to send email
+  health alerts for clusters deployed without the benefit of an
+  existing external monitoring infrastructure.
+* :ref:`Packages <packages>` are built for the following distributions:
+
+  - CentOS 8
+  - CentOS 7 (partial--see below)
+  - Ubuntu 18.04 (Bionic)
+  - Debian Buster
+  - :ref:`Container image <containers>` (based on CentOS 8)
+
+  Note that the dashboard, prometheus, and restful manager modules
+  will not work on the CentOS 7 build due to Python 3 module
+  dependencies that are missing in CentOS 7.
+
+  Besides this packages built by the community will also available for the
+  following distros:
+
+  - Fedora (33/rawhide)
+  - openSUSE (15.2, Tumbleweed)
+
+
+Dashboard
+~~~~~~~~~
+
+The :ref:`mgr-dashboard` has gained a lot of new features and functionality:
+
+* UI Enhancements
+
+  - New vertical navigation bar
+  - New unified sidebar: better background task and events notification
+  - Shows all progress mgr module notifications
+  - Multi-select on tables to perform bulk operations
+
+* Dashboard user account security enhancements
+
+  - Disabling/enabling existing user accounts
+  - Clone an existing user role
+  - Users can change their own password
+  - Configurable password policies: Minimum password complexity/length
+    requirements
+  - Configurable password expiration
+  - Change password after first login
+
+New and enhanced management of Ceph features/services:
+
+* OSD/device management
+
+  - List all disks associated with an OSD
+  - Add support for blinking enclosure LEDs via the orchestrator
+  - List all hosts known by the orchestrator
+  - List all disks and their properties attached to a node
+  - Display disk health information (health prediction and SMART data)
+  - Deploy new OSDs on new disks/hosts
+  - Display and allow sorting by an OSD's default device class in the OSD
+    table
+  - Explicitly set/change the device class of an OSD, display and sort OSDs by
+    device class
+
+* Pool management
+
+  - Viewing and setting pool quotas
+  - Define and change per-pool PG autoscaling mode
+
+* RGW management enhancements
+
+  - Enable bucket versioning
+  - Enable MFA support
+  - Select placement target on bucket creation
+
+* CephFS management enhancements
+
+  - CephFS client eviction
+  - CephFS snapshot management
+  - CephFS quota management
+  - Browse CephFS directory
+
+* iSCSI management enhancements
+
+  - Show iSCSI GW status on landing page
+  - Prevent deletion of IQNs with open sessions
+  - Display iSCSI "logged in" info
+
+* Prometheus alert management
+
+  - List configured Prometheus alerts
+
+
+RADOS
+~~~~~
+  
+* Objects can now be brought in sync during recovery by copying only
+  the modified portion of the object, reducing tail latencies during
+  recovery.
+* Ceph will allow recovery below *min_size* for Erasure coded pools,
+  wherever possible.
+* The PG autoscaler feature introduced in Nautilus is enabled for
+  new pools by default, allowing new clusters to autotune *pg num*
+  without any user intervention.  The default values for new pools
+  and RGW/CephFS metadata pools have also been adjusted to perform
+  well for most users.
+* BlueStore has received several improvements and performance
+  updates, including improved accounting for "omap" (key/value)
+  object data by pool, improved cache memory management, and a
+  reduced allocation unit size for SSD devices.  (Note that by
+  default, the first time each OSD starts after upgrading to octopus
+  it will trigger a conversion that may take from a few minutes to a
+  few hours, depending on the amount of stored "omap" data.)
+* Snapshot trimming metadata is now managed in a more efficient and
+  scalable fashion.
+
+
+RBD block storage
+~~~~~~~~~~~~~~~~~
+  
+* Mirroring now supports a new snapshot-based mode that no longer requires
+  the journaling feature and its related impacts in exchange for the loss
+  of point-in-time consistency (it remains crash consistent).
+* Clone operations now preserve the sparseness of the underlying RBD image.
+* The trash feature has been improved to (optionally) automatically
+  move old parent images to the trash when their children are all
+  deleted or flattened.
+* The trash can be configured to automatically purge on a defined schedule.
+* Images can be online re-sparsified to reduce the usage of zeroed extents.
+* The ``rbd-nbd`` tool has been improved to use more modern kernel interfaces.
+* Caching has been improved to be more efficient and performant.
+* ``rbd-mirror`` automatically adjusts its per-image memory usage based
+  upon its memory target.
+* A new persistent read-only caching daemon is available to offload reads from
+  shared parent images.
+
+RGW object storage
+~~~~~~~~~~~~~~~~~~
+  
+* New `Multisite Sync Policy`_ primitives for per-bucket replication. (EXPERIMENTAL)
+* S3 feature support:
+
+  - Bucket Replication (EXPERIMENTAL)
+  - `Bucket Notifications`_ via HTTP/S, AMQP and Kafka
+  - Bucket Tagging
+  - Object Lock
+  - Public Access Block for buckets
+
+* Bucket sharding:
+
+  - Significantly improved listing performance on buckets with many shards.
+  - Dynamic resharding prefers prime shard counts for improved distribution.
+  - Raised the default number of bucket shards to 11.
+
+* Added `HashiCorp Vault Integration`_ for SSE-KMS.
+* Added Keystone token cache for S3 requests.
+
+CephFS distributed file system
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  
+* Inline data support in CephFS has been deprecated and will likely be
+  removed in a future release.
+* MDS daemons can now be assigned to manage a particular file system via the
+  new ``mds_join_fs`` option.
+* MDS now aggressively asks idle clients to trim caps which improves stability
+  when file system load changes.
+* The mgr volumes plugin has received numerous improvements to support CephFS
+  via CSI, including snapshots and cloning.
+* cephfs-shell has had numerous incremental improvements and bug fixes.
+
+
+Upgrading from Mimic or Nautilus
+--------------------------------
+
+.. note::
+
+  You can monitor the progress of your upgrade at each stage with the
+  ``ceph versions`` command, which will tell you what ceph version(s) are
+  running for each type of daemon.
+
+Instructions
+~~~~~~~~~~~~
+
+.. highlight:: console
+
+#. Make sure your cluster is stable and healthy (no down or
+   recovering OSDs).  (Optional, but recommended.)
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional,
+   but recommended.)::
+
+     # ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.  For example, on each monitor host,::
+
+     # systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is
+   complete by looking for the ``octopus`` string in the mon
+   map.  The command::
+
+     # ceph mon dump | grep min_mon_release
+
+   should report::
+
+     min_mon_release 15 (octopus)
+
+   If it doesn't, that implies that one or more monitors hasn't been
+   upgraded and restarted and/or the quorum does not include all monitors.
+
+#. Upgrade ``ceph-mgr`` daemons by installing the new packages and
+   restarting all manager daemons.  For example, on each manager host,::
+
+     # systemctl restart ceph-mgr.target
+
+   Verify the ``ceph-mgr`` daemons are running by checking ``ceph
+   -s``::
+
+     # ceph -s
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all OSD hosts::
+
+     # systemctl restart ceph-osd.target
+
+   Note that the first time each OSD starts, it will do a format
+   conversion to improve the accounting for "omap" data.  This may
+   take a few minutes to as much as a few hours (for an HDD with lots
+   of omap data).  You can disable this automatic conversion with::
+
+     # ceph config set osd bluestore_fsck_quick_fix_on_mount false
+
+   You can monitor the progress of the OSD upgrades with the
+   ``ceph versions`` or ``ceph osd versions`` commands::
+
+     # ceph osd versions
+     {
+        "ceph version 13.2.5 (...) mimic (stable)": 12,
+        "ceph version 15.2.0 (...) octopus (stable)": 22,
+     }
+
+#. Upgrade all CephFS MDS daemons.  For each CephFS file system,
+
+   #. Reduce the number of ranks to 1.  (Make note of the original
+      number of MDS daemons first if you plan to restore it later.)::
+
+	# ceph status
+	# ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by
+      periodically checking the status::
+
+	# ceph status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with::
+
+	# systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS::
+
+	# ceph status
+
+   #. Upgrade the last remaining MDS daemon by installing the new
+      packages and restarting the daemon::
+
+        # systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline::
+
+	# systemctl start ceph-mds.target
+
+   #. Restore the original value of ``max_mds`` for the volume::
+
+	# ceph fs set <fs_name> max_mds <original_max_mds>
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart ceph-radosgw.target
+
+#. Complete the upgrade by disallowing pre-Octopus OSDs and enabling
+   all new Octopus-only functionality::
+
+     # ceph osd require-osd-release octopus
+
+#. If you set ``noout`` at the beginning, be sure to clear it with::
+
+     # ceph osd unset noout
+
+#. Verify the cluster is healthy with ``ceph health``.
+
+   If your CRUSH tunables are older than Hammer, Ceph will now issue a
+   health warning.  If you see a health alert to that effect, you can
+   revert this change with::
+
+     ceph config set mon mon_crush_min_required_version firefly
+
+   If Ceph does not complain, however, then we recommend you also
+   switch any existing CRUSH buckets to straw2, which was added back
+   in the Hammer release.  If you have any 'straw' buckets, this will
+   result in a modest amount of data movement, but generally nothing
+   too severe.::
+
+     ceph osd getcrushmap -o backup-crushmap
+     ceph osd crush set-all-straw-buckets-to-straw2
+
+   If there are problems, you can easily revert with::
+
+     ceph osd setcrushmap -i backup-crushmap
+
+   Moving to 'straw2' buckets will unlock a few recent features, like
+   the `crush-compat` :ref:`balancer <balancer>` mode added back in Luminous.
+
+
+#. If you are upgrading from Mimic, or did not already do so when you
+   upgraded to Nautlius, we recommended you enable the new :ref:`v2
+   network protocol <msgr2>`, issue the following command::
+
+     ceph mon enable-msgr2
+
+   This will instruct all monitors that bind to the old default port
+   6789 for the legacy v1 protocol to also bind to the new 3300 v2
+   protocol port.  To see if all monitors have been updated,::
+
+     ceph mon dump
+
+   and verify that each monitor has both a ``v2:`` and ``v1:`` address
+   listed.
+
+#. Consider enabling the :ref:`telemetry module <telemetry>` to send
+   anonymized usage statistics and crash information to the Ceph
+   upstream developers.  To see what would be reported (without actually
+   sending any information to anyone),::
+
+     ceph mgr module enable telemetry
+     ceph telemetry show
+
+   If you are comfortable with the data that is reported, you can opt-in to
+   automatically report the high-level cluster metadata with::
+
+     ceph telemetry on
+
+   For more information about the telemetry module, see :ref:`the
+   documentation <telemetry>`.
+
+
+Upgrading from pre-Mimic releases (like Luminous)
+-------------------------------------------------
+
+You *must* first upgrade to Mimic (13.2.z) or Nautilus (14.2.z) before
+upgrading to Octopus.
+
+
+Upgrade compatibility notes
+---------------------------
+
+* Starting with Octopus, there is now a separate repository directory
+  for each version on `download.ceph.com` (e.g., ``rpm-15.2.0`` and
+  ``debian-15.2.0``).  The traditional package directory that is named
+  after the release (e.g., ``rpm-octopus`` and ``debian-octopus``) is
+  now a symlink to the most recently bug fix version for that release.
+  We no longer generate a single repository that combines all bug fix
+  versions for a single named release.
+
+* The RGW "num_rados_handles" has been removed.
+  If you were using a value of "num_rados_handles" greater than 1
+  multiply your current "objecter_inflight_ops" and
+  "objecter_inflight_op_bytes" parameters by the old
+  "num_rados_handles" to get the same throttle behavior.
+
+* Ceph now packages python bindings for python3.6 instead of
+  python3.4, because python3 in EL7/EL8 is now using python3.6
+  as the native python3. see the `announcement`_
+  for more details on the background of this change.
+
+* librbd now uses a write-around cache policy be default,
+  replacing the previous write-back cache policy default.
+  This cache policy allows librbd to immediately complete
+  write IOs while they are still in-flight to the OSDs.
+  Subsequent flush requests will ensure all in-flight
+  write IOs are completed prior to completing. The
+  librbd cache policy can be controlled via a new
+  "rbd_cache_policy" configuration option.
+
+* librbd now includes a simple IO scheduler which attempts to
+  batch together multiple IOs against the same backing RBD
+  data block object. The librbd IO scheduler policy can be
+  controlled via a new "rbd_io_scheduler" configuration
+  option.
+
+* RGW: radosgw-admin introduces two subcommands that allow the
+  managing of expire-stale objects that might be left behind after a
+  bucket reshard in earlier versions of RGW. One subcommand lists such
+  objects and the other deletes them. Read the troubleshooting section
+  of the dynamic resharding docs for details.
+
+* RGW: Bucket naming restrictions have changed and likely to cause
+  InvalidBucketName errors. We recommend to set ``rgw_relaxed_s3_bucket_names``
+  option to true as a workaround.
+
+* In the Zabbix Mgr Module there was a typo in the key being send
+  to Zabbix for PGs in backfill_wait state. The key that was sent
+  was 'wait_backfill' and the correct name is 'backfill_wait'.
+  Update your Zabbix template accordingly so that it accepts the
+  new key being send to Zabbix.
+
+* zabbix plugin for ceph manager now includes osd and pool
+  discovery. Update of zabbix_template.xml is needed
+  to receive per-pool (read/write throughput, diskspace usage)
+  and per-osd (latency, status, pgs) statistics
+
+* The format of all date + time stamps has been modified to fully
+  conform to ISO 8601.  The old format (``YYYY-MM-DD
+  HH:MM:SS.ssssss``) excluded the ``T`` separator between the date and
+  time and was rendered using the local time zone without any explicit
+  indication.  The new format includes the separator as well as a
+  ``+nnnn`` or ``-nnnn`` suffix to indicate the time zone, or a ``Z``
+  suffix if the time is UTC.  For example,
+  ``2019-04-26T18:40:06.225953+0100``.
+
+  Any code or scripts that was previously parsing date and/or time
+  values from the JSON or XML structure CLI output should be checked
+  to ensure it can handle ISO 8601 conformant values.  Any code
+  parsing date or time values from the unstructured human-readable
+  output should be modified to parse the structured output instead, as
+  the human-readable output may change without notice.
+
+* The ``bluestore_no_per_pool_stats_tolerance`` config option has been
+  replaced with ``bluestore_fsck_error_on_no_per_pool_stats``
+  (default: false).  The overall default behavior has not changed:
+  fsck will warn but not fail on legacy stores, and repair will
+  convert to per-pool stats.
+
+* The disaster-recovery related 'ceph mon sync force' command has been
+  replaced with 'ceph daemon <...> sync_force'.
+
+* The ``osd_recovery_max_active`` option now has
+  ``osd_recovery_max_active_hdd`` and ``osd_recovery_max_active_ssd``
+  variants, each with different default values for HDD and SSD-backed
+  OSDs, respectively.  By default ``osd_recovery_max_active`` now
+  defaults to zero, which means that the OSD will conditionally use
+  the HDD or SSD option values.  Administrators who have customized
+  this value may want to consider whether they have set this to a
+  value similar to the new defaults (3 for HDDs and 10 for SSDs) and,
+  if so, remove the option from their configuration entirely.
+
+* monitors now have a `ceph osd info` command that will provide information
+  on all osds, or provided osds, thus simplifying the process of having to
+  parse `osd dump` for the same information.
+
+* The structured output of ``ceph status`` or ``ceph -s`` is now more
+  concise, particularly the `mgrmap` and `monmap` sections, and the
+  structure of the `osdmap` section has been cleaned up.
+
+* A health warning is now generated if the average osd heartbeat ping
+  time exceeds a configurable threshold for any of the intervals
+  computed.  The OSD computes 1 minute, 5 minute and 15 minute
+  intervals with average, minimum and maximum values.  New
+  configuration option ``mon_warn_on_slow_ping_ratio`` specifies a
+  percentage of ``osd_heartbeat_grace`` to determine the threshold.  A
+  value of zero disables the warning.  New configuration option
+  ``mon_warn_on_slow_ping_time`` specified in milliseconds over-rides
+  the computed value, causes a warning when OSD heartbeat pings take
+  longer than the specified amount.  New admin command ``ceph daemon
+  mgr.# dump_osd_network [threshold]`` command will list all
+  connections with a ping time longer than the specified threshold or
+  value determined by the config options, for the average for any of
+  the 3 intervals.  New admin command ``ceph daemon osd.#
+  dump_osd_network [threshold]`` will do the same but only including
+  heartbeats initiated by the specified OSD.
+
+* Inline data support for CephFS has been deprecated. When setting the flag,
+  users will see a warning to that effect, and enabling it now requires the
+  ``--yes-i-really-really-mean-it`` flag. If the MDS is started on a
+  filesystem that has it enabled, a health warning is generated. Support for
+  this feature will be removed in a future release.
+
+* ``ceph {set,unset} full`` is not supported anymore. We have been using
+  ``full`` and ``nearfull`` flags in OSD map for tracking the fullness status
+  of a cluster back since the Hammer release, if the OSD map is marked ``full``
+  all write operations will be blocked until this flag is removed. In the
+  Infernalis release and Linux kernel 4.7 client, we introduced the per-pool
+  full/nearfull flags to track the status for a finer-grained control, so the
+  clients will hold the write operations if either the cluster-wide ``full``
+  flag or the per-pool ``full`` flag is set. This was a compromise, as we
+  needed to support the cluster with and without per-pool ``full`` flags
+  support. But this practically defeated the purpose of introducing the
+  per-pool flags. So, in the Mimic release, the new flags finally took the
+  place of their cluster-wide counterparts, as the monitor started removing
+  these two flags from OSD map. So the clients of Infernalis and up can benefit
+  from this change, as they won't be blocked by the full pools which they are
+  not writing to. In this release, ``ceph {set,unset} full`` is now considered
+  as an invalid command. And the clients will continue honoring both the
+  cluster-wide and per-pool flags to be backward compatible with pre-infernalis
+  clusters.
+
+* The telemetry module now reports more information.
+
+  First, there is a new 'device' channel, enabled by default, that
+  will report anonymized hard disk and SSD health metrics to
+  telemetry.ceph.com in order to build and improve device failure
+  prediction algorithms.  If you are not comfortable sharing device
+  metrics, you can disable that channel first before re-opting-in::
+
+    ceph config set mgr mgr/telemetry/channel_device false
+
+  Second, we now report more information about CephFS file systems,
+  including:
+
+  - how many MDS daemons (in total and per file system)
+  - which features are (or have been) enabled
+  - how many data pools
+  - approximate file system age (year + month of creation)
+  - how many files, bytes, and snapshots
+  - how much metadata is being cached
+
+  We have also added:
+
+  - which Ceph release the monitors are running
+  - whether msgr v1 or v2 addresses are used for the monitors
+  - whether IPv4 or IPv6 addresses are used for the monitors
+  - whether RADOS cache tiering is enabled (and which mode)
+  - whether pools are replicated or erasure coded, and
+    which erasure code profile plugin and parameters are in use
+  - how many hosts are in the cluster, and how many hosts have each type of daemon
+  - whether a separate OSD cluster network is being used
+  - how many RBD pools and images are in the cluster, and how many pools have RBD mirroring enabled
+  - how many RGW daemons, zones, and zonegroups are present; which RGW frontends are in use
+  - aggregate stats about the CRUSH map, like which algorithms are used, how
+    big buckets are, how many rules are defined, and what tunables are in
+    use
+
+  If you had telemetry enabled, you will need to re-opt-in with::
+
+    ceph telemetry on
+
+  You can view exactly what information will be reported first with::
+
+    $ ceph telemetry show        # see everything
+    $ ceph telemetry show basic  # basic cluster info (including all of the new info)
+
+* Following invalid settings now are not tolerated anymore
+  for the command `ceph osd erasure-code-profile set xxx`.
+  * invalid `m` for "reed_sol_r6_op" erasure technique
+  * invalid `m` and invalid `w` for "liber8tion" erasure technique
+
+* New OSD daemon command dump_recovery_reservations which reveals the
+  recovery locks held (in_progress) and waiting in priority queues.
+
+* New OSD daemon command dump_scrub_reservations which reveals the
+  scrub reservations that are held for local (primary) and remote (replica) PGs.
+
+* Previously, ``ceph tell mgr ...`` could be used to call commands
+  implemented by mgr modules.  This is no longer supported.  Since
+  Luminous, using ``tell`` has not been necessary: those same commands
+  are also accessible without the ``tell mgr`` portion (e.g., ``ceph
+  tell mgr influx foo`` is the same as ``ceph influx foo``.  ``ceph
+  tell mgr ...`` will now call admin commands--the same set of
+  commands accessible via ``ceph daemon ...`` when you are logged into
+  the appropriate host.
+
+* The ``ceph tell`` and ``ceph daemon`` commands have been unified,
+  such that all such commands are accessible via either interface.
+  Note that ceph-mgr tell commands are accessible via either ``ceph
+  tell mgr ...`` or ``ceph tell mgr.<id> ...``, and it is only
+  possible to send tell commands to the active daemon (the standbys do
+  not accept incoming connections over the network).
+
+* Ceph will now issue a health warning if a RADOS pool as a ``pg_num``
+  value that is not a power of two.  This can be fixed by adjusting
+  the pool to a nearby power of two::
+
+    ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+  Alternatively, the warning can be silenced with::
+
+    ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false
+
+* The format of MDSs in ``ceph fs dump`` has changed.
+
+* The ``mds_cache_size`` config option is completely removed. Since Luminous,
+  the ``mds_cache_memory_limit`` config option has been preferred to configure
+  the MDS's cache limits.
+
+* The ``pg_autoscale_mode`` is now set to ``on`` by default for newly
+  created pools, which means that Ceph will automatically manage the
+  number of PGs.  To change this behavior, or to learn more about PG
+  autoscaling, see :ref:`pg-autoscaler`.  Note that existing pools in
+  upgraded clusters will still be set to ``warn`` by default.
+
+* The pool parameter ``target_size_ratio``, used by the pg autoscaler,
+  has changed meaning. It is now normalized across pools, rather than
+  specifying an absolute ratio. For details, see :ref:`pg-autoscaler`.
+  If you have set target size ratios on any pools, you may want to set
+  these pools to autoscale ``warn`` mode to avoid data movement during
+  the upgrade::
+
+    ceph osd pool set <pool-name> pg_autoscale_mode warn
+
+* The ``upmap_max_iterations`` config option of mgr/balancer has been
+  renamed to ``upmap_max_optimizations`` to better match its behaviour.
+
+* ``mClockClientQueue`` and ``mClockClassQueue`` OpQueue
+  implementations have been removed in favor of of a single
+  ``mClockScheduler`` implementation of a simpler OSD interface.
+  Accordingly, the ``osd_op_queue_mclock*`` family of config options
+  has been removed in favor of the ``osd_mclock_scheduler*`` family
+  of options.
+
+* The config subsystem now searches dot ('.') delimited prefixes for
+  options.  That means for an entity like ``client.foo.bar``, its
+  overall configuration will be a combination of the global options,
+  ``client``, ``client.foo``, and ``client.foo.bar``.  Previously,
+  only global, ``client``, and ``client.foo.bar`` options would apply.
+  This change may affect the configuration for clients that include a
+  ``.`` in their name.
+
+* MDS default cache memory limit is now 4GB.
+
+* The behaviour of the ``-o`` argument to the rados tool has been 
+  reverted to its original behaviour of indicating an output file. This 
+  reverts it to a more consistent behaviour when compared to other 
+  tools. Specifying object size is now accomplished by using an 
+  upper-case O ``-O``.
+
+* In certain rare cases, OSDs would self-classify themselves as type
+  'nvme' instead of 'hdd' or 'ssd'.  This appears to be limited to
+  cases where BlueStore was deployed with older versions of ceph-disk,
+  or manually without ceph-volume and LVM.  Going forward, the OSD
+  will limit itself to only 'hdd' and 'ssd' (or whatever device class
+  the user manually specifies).
+
+* RGW: a mismatch between the bucket notification documentation and 
+  the actual message format was fixed. This means that any endpoints 
+  receiving bucket notification, will now receive the same notifications 
+  inside an JSON array named 'Records'. Note that this does not affect 
+  pulling bucket notification from a subscription in a 'pubsub' zone, 
+  as these are already wrapped inside that array.
+
+* The configuration value ``osd_calc_pg_upmaps_max_stddev`` used for 
+  upmap balancing has been removed. Instead use the mgr balancer config
+  ``upmap_max_deviation`` which now is an integer number of PGs of 
+  deviation from the target PGs per OSD.  This can be set with a command 
+  like ``ceph config set mgr mgr/balancer/upmap_max_deviation 2``. The 
+  default ``upmap_max_deviation`` is 1.  There are situations where 
+  crush rules would not allow a pool to ever have completely balanced 
+  PGs. For example, if crush requires 1 replica on each of 3 racks, but
+  there are fewer OSDs in one of the racks. In those cases, the 
+  configuration value can be increased.
+
+* MDS daemons can now be assigned to manage a particular file system via the
+  new ``mds_join_fs`` option. The monitors will try to use only MDS for a file
+  system with mds_join_fs equal to the file system name (strong affinity).
+  Monitors may also deliberately failover an active MDS to a standby when the
+  cluster is otherwise healthy if the standby has stronger affinity.
+
+* RGW Multisite: A new fine grained bucket-granularity policy configuration
+  system has been introduced and it supersedes the previous coarse zone sync
+  configuration (specifically the ``sync_from`` and ``sync_from_all`` fields
+  in the zonegroup configuration. New configuration should only be configured
+  after all relevant zones in the zonegroup have been upgraded.
+
+* RGW S3: Support has been added for BlockPublicAccess set of APIs at a bucket
+  level, currently blocking/ignoring public acls & policies are supported.
+  User/Account level APIs are planned to be added in the future
+
+* RGW: The default number of bucket index shards for new buckets was raised
+  from 1 to 11 to increase the amount of write throughput for small buckets
+  and delay the onset of dynamic resharding. This change only affects new
+  deployments/zones. To change this default value on existing deployments,
+  use ``radosgw-admin zonegroup modify --bucket-index-max-shards=11``.
+  If the zonegroup is part of a realm, the change must be committed with
+  ``radosgw-admin period update --commit`` - otherwise the change will take
+  effect after radosgws are restarted.
+
+
+Changelog
+---------
+* .gitignore: add more stuff (`pr#29568 <https://github.com/ceph/ceph/pull/29568>`_, Volker Theile)
+* async/dpdk: fix compile errors from ceph::mutex update (`pr#30066 <https://github.com/ceph/ceph/pull/30066>`_, yehu)
+* bluestore,build/ops,common,rgw: Enable _GLIBCXX_ASSERTIONS and fix unittest problems (`pr#32387 <https://github.com/ceph/ceph/pull/32387>`_, Samuel Just)
+* bluestore,cephfs,common,core,mgr,mon,rbd,rgw: src/: s/Mutex/ceph::mutex/ (`pr#29113 <https://github.com/ceph/ceph/pull/29113>`_, Kefu Chai)
+* bluestore,common,core,mgr,rbd: common/RefCountedObj: cleanup con/des (`pr#29672 <https://github.com/ceph/ceph/pull/29672>`_, Patrick Donnelly)
+* bluestore,common,core,rgw: common, \\*: kill the bl::last_p member. Use iterator instead (`pr#32831 <https://github.com/ceph/ceph/pull/32831>`_, Radoslaw Zarzynski)
+* bluestore,common: os/bluestore: s/align_down/p2align/ (`pr#29379 <https://github.com/ceph/ceph/pull/29379>`_, Kefu Chai)
+* bluestore,core: common/options: Set bluestore min_alloc size to 4K (`pr#30698 <https://github.com/ceph/ceph/pull/30698>`_, Mark Nelson)
+* bluestore,core: common/options: Set concurrent bluestore rocksdb compactions to 2 (`pr#29027 <https://github.com/ceph/ceph/pull/29027>`_, Mark Nelson)
+* bluestore,core: mon,osd: only use new per-pool usage stats once \\*all\\* osds are reporting (`pr#28978 <https://github.com/ceph/ceph/pull/28978>`_, Sage Weil)
+* bluestore,core: os/bluestore,mon: segregate omap keys by pool; report via df (`pr#29292 <https://github.com/ceph/ceph/pull/29292>`_, Sage Weil)
+* bluestore,core: os/bluestore/BlueFS: explicit check for too-granular allocations (`pr#33027 <https://github.com/ceph/ceph/pull/33027>`_, Sage Weil)
+* bluestore,core: os/bluestore/bluefs_types: consolidate contiguous extents (`pr#28821 <https://github.com/ceph/ceph/pull/28821>`_, Sage Weil)
+* bluestore,core: os/bluestore/KernelDevice: fix RW_IO_MAX constant (`pr#29577 <https://github.com/ceph/ceph/pull/29577>`_, Sage Weil)
+* bluestore,core: os/bluestore: do not set osd_memory_target default from cgroup limit (`pr#29581 <https://github.com/ceph/ceph/pull/29581>`_, Sage Weil)
+* bluestore,core: os/bluestore: drop (semi-broken) nvme automatic class (`pr#31796 <https://github.com/ceph/ceph/pull/31796>`_, Sage Weil)
+* bluestore,core: os/bluestore: expand lttng tracepoints, improve fio_ceph_objectstore backend (`pr#29674 <https://github.com/ceph/ceph/pull/29674>`_, Samuel Just)
+* bluestore,core: os/bluestore: Keep separate onode cache pinned list (`pr#30964 <https://github.com/ceph/ceph/pull/30964>`_, Mark Nelson)
+* bluestore,core: os/bluestore: prefix omap of temp objects by real pool (`pr#29717 <https://github.com/ceph/ceph/pull/29717>`_, xie xingguo)
+* bluestore,core: os/bluestore: Unify on preadv for io_uring and future refactor (`pr#28025 <https://github.com/ceph/ceph/pull/28025>`_, Mark Nelson)
+* bluestore,core: os/bluestore: v.2 framework for more intelligent DB space usage (`pr#29687 <https://github.com/ceph/ceph/pull/29687>`_, Igor Fedotov)
+* bluestore,mgr,rgw: rgw,bluestore: fixes to address failures from check-generated.sh (`pr#29862 <https://github.com/ceph/ceph/pull/29862>`_, Kefu Chai)
+* bluestore,mon: os/bluestore: create the tail when first set FLAG_OMAP (`pr#27627 <https://github.com/ceph/ceph/pull/27627>`_, Tao Ning)
+* bluestore,tools: os/bluestore/bluestore-tool: minor fixes around migrate (`pr#28651 <https://github.com/ceph/ceph/pull/28651>`_, Igor Fedotov)
+* bluestore,tools: tools/ceph-objectstore-tool: implement onode metadata dump (`pr#27869 <https://github.com/ceph/ceph/pull/27869>`_, Igor Fedotov)
+* bluestore,tools: tools/ceph-objectstore-tool: introduce list-slow-omap command (`pr#27985 <https://github.com/ceph/ceph/pull/27985>`_, Igor Fedotov)
+* bluestore: BlueFS: prevent BlueFS::dirty_files from being leaked when syncing metadata (`pr#30631 <https://github.com/ceph/ceph/pull/30631>`_, Xuehan Xu)
+* bluestore: bluestore/allocator: Ageing test for bluestore allocators (`pr#22574 <https://github.com/ceph/ceph/pull/22574>`_, Adam Kupczyk)
+* bluestore: bluestore/bdev: initialize size when creating object (`pr#29968 <https://github.com/ceph/ceph/pull/29968>`_, Willem Jan Withagen)
+* bluestore: bluestore/bluefs: make accounting resiliant to unlock() (`pr#32584 <https://github.com/ceph/ceph/pull/32584>`_, Adam Kupczyk)
+* bluestore: common/options.cc: change default value of bluestore_fsck_on_mount_deep to false (`pr#29408 <https://github.com/ceph/ceph/pull/29408>`_, Neha Ojha)
+* bluestore: common/options: bluestore 64k min_alloc_size for HDD (`pr#32809 <https://github.com/ceph/ceph/pull/32809>`_, Sage Weil)
+* bluestore: NVMEDevice: Remove the unnecessary aio_wait in sync read (`pr#33597 <https://github.com/ceph/ceph/pull/33597>`_, Ziye Yang)
+* bluestore: NVMEDevice: Split the read I/O if the io size is large (`pr#32647 <https://github.com/ceph/ceph/pull/32647>`_, Ziye Yang)
+* bluestore: os/bluestore/Blue(FS|Store): uint64_t alloc_size (`pr#32484 <https://github.com/ceph/ceph/pull/32484>`_, Bernd Zeimetz)
+* bluestore: os/bluestore/BlueFS: clear newly allocated space for WAL logs (`pr#30549 <https://github.com/ceph/ceph/pull/30549>`_, Adam Kupczyk)
+* bluestore: os/bluestore/BlueFS: fixed printing stats (`pr#33235 <https://github.com/ceph/ceph/pull/33235>`_, Adam Kupczyk)
+* bluestore: os/bluestore/BlueFS: less verbose about alloc adjustments (`pr#33512 <https://github.com/ceph/ceph/pull/33512>`_, Sage Weil)
+* bluestore: os/bluestore/BlueFS: Move bluefs alloc size initialization log message to log level 1 (`pr#29822 <https://github.com/ceph/ceph/pull/29822>`_, Vikhyat Umrao)
+* bluestore: os/bluestore/BlueFS: replace flush_log with sync_metadata (`pr#32563 <https://github.com/ceph/ceph/pull/32563>`_, Jianpeng Ma)
+* bluestore: os/bluestore/BlueFS: use 64K alloc_size on the shared device (`pr#29537 <https://github.com/ceph/ceph/pull/29537>`_, Sage Weil, Neha Ojha)
+* bluestore: os/bluestore/BlueStore.cc: set priorities for compression stats (`pr#31959 <https://github.com/ceph/ceph/pull/31959>`_, Neha Ojha)
+* bluestore: os/bluestore/spdk: Fix the overflow error of parsing spdk coremask (`pr#32440 <https://github.com/ceph/ceph/pull/32440>`_, Hu Ye, Chunsong Feng, luo rixin)
+* bluestore: os/bluestore: Actually wait until completion in write_sync (`pr#26909 <https://github.com/ceph/ceph/pull/26909>`_, Vitaliy Filippov)
+* bluestore: os/bluestore: add bluestore_bluefs_max_free; smooth space balancing a bit (`pr#30231 <https://github.com/ceph/ceph/pull/30231>`_, xie xingguo)
+* bluestore: os/bluestore: add slow op detection for collection_listing (`issue#40741 <http://tracker.ceph.com/issues/40741>`_, `pr#29085 <https://github.com/ceph/ceph/pull/29085>`_, Igor Fedotov)
+* bluestore: os/bluestore: allocate Task on stack (`pr#33358 <https://github.com/ceph/ceph/pull/33358>`_, Jun Su)
+* bluestore: os/bluestore: apply garbage collection against excessive blob count growth (`pr#28229 <https://github.com/ceph/ceph/pull/28229>`_, Igor Fedotov)
+* bluestore: os/bluestore: AVL-tree & extent - based space allocator (`pr#30897 <https://github.com/ceph/ceph/pull/30897>`_, Adam Kupczyk, xie xingguo, Kefu Chai)
+* bluestore: os/bluestore: avoid length overflow in extents returned by Stupid (`issue#40703 <http://tracker.ceph.com/issues/40703>`_, `pr#28945 <https://github.com/ceph/ceph/pull/28945>`_, Igor Fedotov)
+* bluestore: os/bluestore: avoid race between split_cache and get/put pin/unpin (`pr#32665 <https://github.com/ceph/ceph/pull/32665>`_, Sage Weil)
+* bluestore: os/bluestore: avoid unnecessary notify (`pr#29345 <https://github.com/ceph/ceph/pull/29345>`_, Jianpeng Ma)
+* bluestore: os/bluestore: be more verbose doing bluefs log replay (`pr#27615 <https://github.com/ceph/ceph/pull/27615>`_, Igor Fedotov)
+* bluestore: os/bluestore: bluefs_preextend_wal_files=true (`pr#28322 <https://github.com/ceph/ceph/pull/28322>`_, Sage Weil)
+* bluestore: os/bluestore: call fault_range prior to looking for blob to reuse (`pr#27444 <https://github.com/ceph/ceph/pull/27444>`_, Igor Fedotov)
+* bluestore: os/bluestore: check bluefs allocations on log replay (`pr#31513 <https://github.com/ceph/ceph/pull/31513>`_, Igor Fedotov)
+* bluestore: os/bluestore: check return value of func _open_db_and_around (`pr#27477 <https://github.com/ceph/ceph/pull/27477>`_, Jianpeng Ma)
+* bluestore: os/bluestore: cleanup around allocator calls (`pr#29068 <https://github.com/ceph/ceph/pull/29068>`_, Igor Fedotov)
+* bluestore: os/bluestore: cleanups (`pr#30737 <https://github.com/ceph/ceph/pull/30737>`_, Kefu Chai)
+* bluestore: os/bluestore: consolidate extents from the same device only (`pr#31621 <https://github.com/ceph/ceph/pull/31621>`_, Igor Fedotov)
+* bluestore: os/bluestore: correctly measure deferred writes into new blobs (`issue#38816 <http://tracker.ceph.com/issues/38816>`_, `pr#27789 <https://github.com/ceph/ceph/pull/27789>`_, Sage Weil)
+* bluestore: os/bluestore: deferred IO notify and locking optimization (`pr#29522 <https://github.com/ceph/ceph/pull/29522>`_, Jianpeng Ma)
+* bluestore: os/bluestore: do not check osd_max_object_size in _open_path() (`pr#26176 <https://github.com/ceph/ceph/pull/26176>`_, Igor Fedotov)
+* bluestore: os/bluestore: do not mark per_pool_omap updated unless we fixed it (`pr#31167 <https://github.com/ceph/ceph/pull/31167>`_, Sage Weil)
+* bluestore: os/bluestore: dont round_up_to in apply_for_bitset_range (`pr#31903 <https://github.com/ceph/ceph/pull/31903>`_, Jianpeng Ma)
+* bluestore: os/bluestore: dump onode before no available blob id abort (`pr#27911 <https://github.com/ceph/ceph/pull/27911>`_, Igor Fedotov)
+* bluestore: os/bluestore: dump onode that has too many spanning blobs (`pr#28010 <https://github.com/ceph/ceph/pull/28010>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix >2GB writes (`pr#27871 <https://github.com/ceph/ceph/pull/27871>`_, Sage Weil, kungf)
+* bluestore: os/bluestore: fix bitmap allocator issues (`pr#26939 <https://github.com/ceph/ceph/pull/26939>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix duplicate allocations in bmap allocator (`issue#40080 <http://tracker.ceph.com/issues/40080>`_, `pr#28496 <https://github.com/ceph/ceph/pull/28496>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix duplicative and misleading debug in KernelDevice::open() (`pr#28630 <https://github.com/ceph/ceph/pull/28630>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: fix for FreeBSD iocb structure (`pr#27458 <https://github.com/ceph/ceph/pull/27458>`_, Willem Jan Withagen)
+* bluestore: os/bluestore: fix invalid stray shared blob detection in fsck (`pr#30616 <https://github.com/ceph/ceph/pull/30616>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix missing discard in BlueStore::_kv_sync_thread (`pr#27843 <https://github.com/ceph/ceph/pull/27843>`_, Junhui Tang)
+* bluestore: os/bluestore: fix origin reference in logging slow ops (`pr#27951 <https://github.com/ceph/ceph/pull/27951>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix out-of-bound access in bmap allocator (`pr#27691 <https://github.com/ceph/ceph/pull/27691>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix per-pool omap repair (`pr#32925 <https://github.com/ceph/ceph/pull/32925>`_, Igor Fedotov)
+* bluestore: os/bluestore: fix space balancing overflow (`pr#30255 <https://github.com/ceph/ceph/pull/30255>`_, xie xingguo)
+* bluestore: os/bluestore: fix wakeup bug (`pr#31931 <https://github.com/ceph/ceph/pull/31931>`_, Jianpeng Ma)
+* bluestore: os/bluestore: introduce legacy statfs and dev size mismatch alerts (`pr#27519 <https://github.com/ceph/ceph/pull/27519>`_, Sage Weil, Igor Fedotov)
+* bluestore: os/bluestore: introduce new io_uring IO engine (`pr#27392 <https://github.com/ceph/ceph/pull/27392>`_, Roman Penyaev)
+* bluestore: os/bluestore: its better to erase spanning blob once (`pr#29238 <https://github.com/ceph/ceph/pull/29238>`_, Xiangyang Yu)
+* bluestore: os/bluestore: load OSD all compression settings unconditionally (`issue#40480 <http://tracker.ceph.com/issues/40480>`_, `pr#28688 <https://github.com/ceph/ceph/pull/28688>`_, Igor Fedotov)
+* bluestore: os/bluestore: log allocation stats on a daily basis (`pr#33565 <https://github.com/ceph/ceph/pull/33565>`_, Igor Fedotov)
+* bluestore: os/bluestore: memorize layout of BlueFS on management (`pr#30593 <https://github.com/ceph/ceph/pull/30593>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore: Merge deferred_finisher and finisher (`pr#29623 <https://github.com/ceph/ceph/pull/29623>`_, Jianpeng Ma)
+* bluestore: os/bluestore: minor improvements/cleanup around allocator (`pr#29738 <https://github.com/ceph/ceph/pull/29738>`_, Igor Fedotov)
+* bluestore: os/bluestore: more aggressive deferred submit when onode trim skipping (`issue#21531 <http://tracker.ceph.com/issues/21531>`_, `pr#25697 <https://github.com/ceph/ceph/pull/25697>`_, Zengran Zhang)
+* bluestore: os/bluestore: more smart allocator dump when lacking space for bluefs (`issue#40623 <http://tracker.ceph.com/issues/40623>`_, `pr#28845 <https://github.com/ceph/ceph/pull/28845>`_, Igor Fedotov)
+* bluestore: os/bluestore: new bluestore_debug_enforce_settings option (`pr#27132 <https://github.com/ceph/ceph/pull/27132>`_, Igor Fedotov)
+* bluestore: os/bluestore: no need protected by OpSequencer::qlock (`pr#29488 <https://github.com/ceph/ceph/pull/29488>`_, Jianpeng Ma)
+* bluestore: os/bluestore: no need to add tail length (revert PR#29185) (`pr#29465 <https://github.com/ceph/ceph/pull/29465>`_, Xiangyang Yu)
+* bluestore: os/bluestore: print correctly info (`pr#29939 <https://github.com/ceph/ceph/pull/29939>`_, Jianpeng Ma)
+* bluestore: os/bluestore: print error if spdk_nvme_ns_cmd_writev() fails (`pr#31932 <https://github.com/ceph/ceph/pull/31932>`_, NancySu05)
+* bluestore: os/bluestore: proper locking for BlueFS prefetching (`pr#29012 <https://github.com/ceph/ceph/pull/29012>`_, Igor Fedotov)
+* bluestore: os/bluestore: reduce wakeups (`pr#29130 <https://github.com/ceph/ceph/pull/29130>`_, Jianpeng Ma)
+* bluestore: os/bluestore: Refactor Bluestore Caches (`pr#28597 <https://github.com/ceph/ceph/pull/28597>`_, Mark Nelson)
+* bluestore: os/bluestore: remove unused arg to _get_deferred_op() (`issue#40918 <http://tracker.ceph.com/issues/40918>`_, `pr#29320 <https://github.com/ceph/ceph/pull/29320>`_, Sage Weil)
+* bluestore: os/bluestore: remove unused _tune_cache_size() method declaration (`pr#29393 <https://github.com/ceph/ceph/pull/29393>`_, Igor Fedotov)
+* bluestore: os/bluestore: restore and fix bug with onode cache pinning (`pr#31778 <https://github.com/ceph/ceph/pull/31778>`_, Josh Durgin)
+* bluestore: os/bluestore: revert cache pinned list (`pr#31180 <https://github.com/ceph/ceph/pull/31180>`_, Sage Weil)
+* bluestore: os/bluestore: set STATE_KV_SUBMITTED properly (`pr#30753 <https://github.com/ceph/ceph/pull/30753>`_, Igor Fedotov)
+* bluestore: os/bluestore: show device name in osd metadata output (`pr#28107 <https://github.com/ceph/ceph/pull/28107>`_, Igor Fedotov)
+* bluestore: os/bluestore: silence StupidAllocator reorder warning (`pr#29866 <https://github.com/ceph/ceph/pull/29866>`_, Jos Collin)
+* bluestore: os/bluestore: simplify multithreaded shallow fsck (`pr#31473 <https://github.com/ceph/ceph/pull/31473>`_, Igor Fedotov)
+* bluestore: os/bluestore: simplify per-pool-stat config options (`pr#30350 <https://github.com/ceph/ceph/pull/30350>`_, Sage Weil, Igor Fedotov)
+* bluestore: os/bluestore: support RocksDB prefetch in buffered read mode (`issue#36482 <http://tracker.ceph.com/issues/36482>`_, `pr#27782 <https://github.com/ceph/ceph/pull/27782>`_, Igor Fedotov)
+* bluestore: os/bluestore: tiny tracepoints improvement (`pr#31669 <https://github.com/ceph/ceph/pull/31669>`_, Adam Kupczyk)
+* bluestore: os/bluestore: upgrade legacy omap to per-pool format automatically (`pr#32758 <https://github.com/ceph/ceph/pull/32758>`_, Igor Fedotov)
+* bluestore: os/bluestore: verify disk layout of BlueFS (`issue#25098 <http://tracker.ceph.com/issues/25098>`_, `pr#30109 <https://github.com/ceph/ceph/pull/30109>`_, Radoslaw Zarzynski)
+* bluestore: os/bluestore:fix two calculation bugs (`pr#29185 <https://github.com/ceph/ceph/pull/29185>`_, Xiangyang Yu)
+* bluestore: os/ceph-bluestore-tool: bluefs-bdev-expand asserts if no WAL (`pr#27445 <https://github.com/ceph/ceph/pull/27445>`_, Igor Fedotov)
+* bluestore: os/objectstore: add new op OP_CREATE for create a new object (`pr#26251 <https://github.com/ceph/ceph/pull/26251>`_, Jianpeng Ma)
+* bluestore: Revert os/bluestore: add kv_drain_preceding_waiters indicate drain_preceding. (`pr#31503 <https://github.com/ceph/ceph/pull/31503>`_, Sage Weil)
+* bluestore: test/fio: handle nullptr when parsing throttle params (`pr#31681 <https://github.com/ceph/ceph/pull/31681>`_, Igor Fedotov)
+* bluestore: [bluestore][tools] Inspect allocations in bluestore (`pr#29425 <https://github.com/ceph/ceph/pull/29425>`_, Adam Kupczyk)
+* build(deps): bump lodash from 4.17.11 to 4.17.13 in /src/pybind/mgr/dashboard/frontend (`pr#29192 <https://github.com/ceph/ceph/pull/29192>`_, dependabot[bot])
+* build/ops,cephfs,common,core,rbd: Fix big-endian handling (`pr#30079 <https://github.com/ceph/ceph/pull/30079>`_, Ulrich Weigand)
+* build/ops,cephfs: mgr/ssh: make mds add work (`pr#31059 <https://github.com/ceph/ceph/pull/31059>`_, Sage Weil)
+* build/ops,common,core: common, include: bump the version of ceph::buffers C++ API (`pr#33373 <https://github.com/ceph/ceph/pull/33373>`_, Radoslaw Zarzynski)
+* build/ops,common,mgr: python-common:  Python common package (`pr#28915 <https://github.com/ceph/ceph/pull/28915>`_, Kefu Chai, Sebastian Wagner)
+* build/ops,common,rgw: rgw, common, build: drop NSS support (`pr#27834 <https://github.com/ceph/ceph/pull/27834>`_, Radoslaw Zarzynski)
+* build/ops,core,rbd: Windows support [part 1] (`pr#31981 <https://github.com/ceph/ceph/pull/31981>`_, Lucian Petrut, Alin Gabriel Serdean)
+* build/ops,core: ceph-crash: use client.crash[.host] to post, and provsion keys via mgr/ssh + ceph-daemon (`pr#30734 <https://github.com/ceph/ceph/pull/30734>`_, Sage Weil)
+* build/ops,core: debian: fix ceph-mgr-modules-core files (`pr#33468 <https://github.com/ceph/ceph/pull/33468>`_, Sage Weil)
+* build/ops,core: os/bluestore: fix pmem osd build problem (`pr#28761 <https://github.com/ceph/ceph/pull/28761>`_, Peterson, Scott, Li, Xiaoyan)
+* build/ops,core: qa: stop testing on 16.04 xenial (`pr#28943 <https://github.com/ceph/ceph/pull/28943>`_, Sage Weil)
+* build/ops,mgr: mgr/diskprediction_local: Replaced old models and updated predictor (`pr#29437 <https://github.com/ceph/ceph/pull/29437>`_, Karanraj Chauhan)
+* build/ops,mgr: systemd: ceph-mgr: set MemoryDenyWriteExecute to false (`issue#39628 <http://tracker.ceph.com/issues/39628>`_, `pr#28023 <https://github.com/ceph/ceph/pull/28023>`_, Ricardo Dias)
+* build/ops,pybind: cmake, pybind: fix build on armhf (`pr#28843 <https://github.com/ceph/ceph/pull/28843>`_, Kefu Chai)
+* build/ops,rbd: rpm,deb: fix python dateutil module dependency (`pr#33624 <https://github.com/ceph/ceph/pull/33624>`_, Mykola Golub)
+* build/ops,rgw: build/rgw: unittest_rgw_dmclock_scheduler does not need Boost_LIBRARIES (`pr#27466 <https://github.com/ceph/ceph/pull/27466>`_, Willem Jan Withagen)
+* build/ops,rgw: install-deps.sh, cmake: use boost 1.72 on bionic (`pr#32391 <https://github.com/ceph/ceph/pull/32391>`_, Kefu Chai)
+* build/ops,tests: ceph-daemon: a few fixes; functional test (`pr#31094 <https://github.com/ceph/ceph/pull/31094>`_, Sage Weil)
+* build/ops,tests: googletest: pick up change to suppress CMP0048 warning (`pr#29471 <https://github.com/ceph/ceph/pull/29471>`_, Kefu Chai)
+* build/ops,tests: install-deps.sh,deb,rpm: move python-saml deps into debian/control an\xe2\x80\xa6 (`pr#29840 <https://github.com/ceph/ceph/pull/29840>`_, Kefu Chai)
+* build/ops,tools: src/script/credits.sh - switch to bash (`pr#32736 <https://github.com/ceph/ceph/pull/32736>`_, Kai Wagner)
+* build/ops,tools: vstart: Now all OSDs are starting in parallel. Use --no-parallel to revert to sequential (`pr#31732 <https://github.com/ceph/ceph/pull/31732>`_, Adam Kupczyk)
+* build/ops: .github/stale.yml: warn at 60, close at 90; adjust message (`pr#24744 <https://github.com/ceph/ceph/pull/24744>`_, Lenz Grimmer, Sage Weil)
+* build/ops: admin/build-doc: keep-going when finding warnings (`pr#27050 <https://github.com/ceph/ceph/pull/27050>`_, Abhishek Lekshmanan)
+* build/ops: build-doc: allow building docs on fedora 30 (`pr#30136 <https://github.com/ceph/ceph/pull/30136>`_, Yuval Lifshitz)
+* build/ops: build-integration-branch: s/prefix/postfix/ (`pr#32303 <https://github.com/ceph/ceph/pull/32303>`_, Kefu Chai)
+* build/ops: build: add static analysis targets (`pr#31579 <https://github.com/ceph/ceph/pull/31579>`_, Yuval Lifshitz)
+* build/ops: build: FreeBSD does not have /etc/os-release (`pr#26731 <https://github.com/ceph/ceph/pull/26731>`_, Willem Jan Withagen)
+* build/ops: ceph-daemon: a couple fixes (`pr#31060 <https://github.com/ceph/ceph/pull/31060>`_, Sage Weil)
+* build/ops: ceph-daemon: add a logrotate.d file for each cluster (`pr#30882 <https://github.com/ceph/ceph/pull/30882>`_, Sage Weil)
+* build/ops: ceph-daemon: deploy ceph daemons with podman and systemd (`pr#30603 <https://github.com/ceph/ceph/pull/30603>`_, Sage Weil)
+* build/ops: ceph-daemon: fix logrotate su line (`pr#31823 <https://github.com/ceph/ceph/pull/31823>`_, Sage Weil)
+* build/ops: ceph-daemon: misc improvements (`pr#30826 <https://github.com/ceph/ceph/pull/30826>`_, Sage Weil)
+* build/ops: ceph-daemon: use /usr/bin/python, not /usr/bin/env python (`pr#31318 <https://github.com/ceph/ceph/pull/31318>`_, Sage Weil)
+* build/ops: ceph.spec.in: add missing python-yaml dependency for mgr-k8sevents (`pr#31178 <https://github.com/ceph/ceph/pull/31178>`_, Kefu Chai)
+* build/ops: ceph.spec.in: add runtime deps for mgr-diskprediction-cloud (`pr#32232 <https://github.com/ceph/ceph/pull/32232>`_, Kefu Chai)
+* build/ops: ceph.spec.in: always depends on python3.6-pyOpenSSL (`pr#32317 <https://github.com/ceph/ceph/pull/32317>`_, Kefu Chai)
+* build/ops: ceph.spec.in: Drop systemd BuildRequires in case of building for SUSE (`pr#28884 <https://github.com/ceph/ceph/pull/28884>`_, Dominique Leuenberger)
+* build/ops: ceph.spec.in: enable amqp_endpoint on RHEL8 by default (`pr#31143 <https://github.com/ceph/ceph/pull/31143>`_, Brad Hubbard)
+* build/ops: ceph.spec.in: fix Cython package dependency for Fedora (`pr#30590 <https://github.com/ceph/ceph/pull/30590>`_, Jeff Layton)
+* build/ops: ceph.spec.in: fix make check deps for centos8 (`pr#32798 <https://github.com/ceph/ceph/pull/32798>`_, Alfonso Mart\xc3\xadnez)
+* build/ops: ceph.spec.in: fix python coverage dependency for non-rhel distros (`pr#33361 <https://github.com/ceph/ceph/pull/33361>`_, Kiefer Chang)
+* build/ops: ceph.spec.in: fix python3 dependencies in centos7 (`pr#32775 <https://github.com/ceph/ceph/pull/32775>`_, liushi)
+* build/ops: ceph.spec.in: grafana-dashboards package depends on grafana (`pr#28228 <https://github.com/ceph/ceph/pull/28228>`_, Jan Fajerski)
+* build/ops: ceph.spec.in: move distro-conditional deps to dedicated section (`pr#32080 <https://github.com/ceph/ceph/pull/32080>`_, Nathan Cutler)
+* build/ops: ceph.spec.in: package prometheus default alerts for SUSE (`pr#27996 <https://github.com/ceph/ceph/pull/27996>`_, Jan Fajerski)
+* build/ops: ceph.spec.in: pin to gcc-c++-8.2.1 (`pr#28859 <https://github.com/ceph/ceph/pull/28859>`_, Kefu Chai)
+* build/ops: ceph.spec.in: re-enable make check deps for el8 (`pr#32412 <https://github.com/ceph/ceph/pull/32412>`_, Kefu Chai)
+* build/ops: ceph.spec.in: reserve more memory per build jo (`pr#30126 <https://github.com/ceph/ceph/pull/30126>`_, Dan van der Ster)
+* build/ops: ceph.spec.in: s/pkgversion/version_nodots/ (`pr#30036 <https://github.com/ceph/ceph/pull/30036>`_, Kefu Chai)
+* build/ops: ceph.spec.in: use g++ >= 8.3.1-3.1 (`pr#30088 <https://github.com/ceph/ceph/pull/30088>`_, Kefu Chai)
+* build/ops: ceph.spec.in: Use pkgconfig() style BuildRequires for udev/libudev-devel (`pr#32933 <https://github.com/ceph/ceph/pull/32933>`_, Dominique Leuenberger)
+* build/ops: ceph.spec.in: use python3 to bytecompile .py files (`pr#32608 <https://github.com/ceph/ceph/pull/32608>`_, Kefu Chai)
+* build/ops: ceph.spec: Recommend (but do not require) podman (`pr#33221 <https://github.com/ceph/ceph/pull/33221>`_, Sage Weil)
+* build/ops: ceph_release: octopus rc 15.1.0 (`pr#32623 <https://github.com/ceph/ceph/pull/32623>`_, Sage Weil)
+* build/ops: cmake,crimson: pick up latest seastar (`pr#27088 <https://github.com/ceph/ceph/pull/27088>`_, Kefu Chai)
+* build/ops: cmake,run-make-check.sh: disable SPDK by default (`pr#29728 <https://github.com/ceph/ceph/pull/29728>`_, Kefu Chai)
+* build/ops: cmake/Boost: Fix python3 version (`pr#32344 <https://github.com/ceph/ceph/pull/32344>`_, Kotresh HR)
+* build/ops: cmake/FindRocksDB: fix IMPORTED_LOCATION for ROCKSDB_LIBRARIES (`pr#26813 <https://github.com/ceph/ceph/pull/26813>`_, dudengke)
+* build/ops: cmake/modules/GetGitRevisionDescription: update to work with git-worktree (`pr#30772 <https://github.com/ceph/ceph/pull/30772>`_, Sage Weil)
+* build/ops: cmake/modules: replace ; with   in compile flags (`pr#28339 <https://github.com/ceph/ceph/pull/28339>`_, Kefu Chai)
+* build/ops: CMakeLists: add std::move warnings in gcc9 (`pr#27569 <https://github.com/ceph/ceph/pull/27569>`_, Patrick Donnelly)
+* build/ops: crimson: clang related cleanups (`pr#33680 <https://github.com/ceph/ceph/pull/33680>`_, Kefu Chai)
+* build/ops: crimson: fix build seastar with dpdk (`pr#31426 <https://github.com/ceph/ceph/pull/31426>`_, Yingxin Cheng)
+* build/ops: deb,rpm,doc: s/plugin/module/ (`pr#33435 <https://github.com/ceph/ceph/pull/33435>`_, Kefu Chai)
+* build/ops: debian/: use ceph-osd for packaging crimson-osd (`pr#28535 <https://github.com/ceph/ceph/pull/28535>`_, Kefu Chai)
+* build/ops: debian/control: add python-routes dependency for dashboard (`pr#28835 <https://github.com/ceph/ceph/pull/28835>`_, Paul Emmerich)
+* build/ops: debian/control: Build-Depends on g++ (`pr#30410 <https://github.com/ceph/ceph/pull/30410>`_, Kefu Chai)
+* build/ops: debian/control: fix Build-Depends (`pr#29913 <https://github.com/ceph/ceph/pull/29913>`_, Kefu Chai)
+* build/ops: debian/radosgw.install: correct path to libradosgw.so\\* (`pr#32539 <https://github.com/ceph/ceph/pull/32539>`_, Kefu Chai)
+* build/ops: debian/rules: run dh_python2 with ceph-daemon (`pr#31313 <https://github.com/ceph/ceph/pull/31313>`_, Kefu Chai)
+* build/ops: debian: modules-core replaces and breaks older ceph-mgr (`pr#33501 <https://github.com/ceph/ceph/pull/33501>`_, Kefu Chai)
+* build/ops: debian: remove dup ceph-fuse line (`pr#28788 <https://github.com/ceph/ceph/pull/28788>`_, huangjun)
+* build/ops: dmclock:  pick up change to use specified C++ settings if any (`pr#30113 <https://github.com/ceph/ceph/pull/30113>`_, Kefu Chai)
+* build/ops: do_cmake.sh: Add a heading to the minimal config (`pr#28776 <https://github.com/ceph/ceph/pull/28776>`_, Brad Hubbard)
+* build/ops: do_cmake.sh: Add CEPH_GIT_DIR (`pr#30863 <https://github.com/ceph/ceph/pull/30863>`_, Matthew Oliver)
+* build/ops: do_cmake.sh: bail out if something goes wrong (`pr#33016 <https://github.com/ceph/ceph/pull/33016>`_, Kefu Chai)
+* build/ops: do_cmake.sh: enable amqp and rdma for EL8 (`pr#30974 <https://github.com/ceph/ceph/pull/30974>`_, Kefu Chai)
+* build/ops: do_cmake.sh: optionally specify build dir with $BUILD_DIR env var (`pr#29786 <https://github.com/ceph/ceph/pull/29786>`_, Yuval Lifshitz)
+* build/ops: do_cmake.sh: remove -DCMAKE_BUILD_TYPE=Debug from cmake options (`pr#30250 <https://github.com/ceph/ceph/pull/30250>`_, Kefu Chai)
+* build/ops: do_cmake.sh: use bash (`issue#39981 <http://tracker.ceph.com/issues/39981>`_, `pr#28181 <https://github.com/ceph/ceph/pull/28181>`_, Nathan Cutler)
+* build/ops: do_cmake: Warn user about slow debug performance only for not set (`pr#31113 <https://github.com/ceph/ceph/pull/31113>`_, Junyoung, Sung)
+* build/ops: do_freebsd.sh: update build scripts to resemble Jenkins scripts (`pr#29400 <https://github.com/ceph/ceph/pull/29400>`_, Willem Jan Withagen)
+* build/ops: dpdk: drop dpdk submodule (`issue#24032 <http://tracker.ceph.com/issues/24032>`_, `pr#33001 <https://github.com/ceph/ceph/pull/33001>`_, Kefu Chai)
+* build/ops: fix build fail related to PYTHON_EXECUTABLE variable (`pr#30199 <https://github.com/ceph/ceph/pull/30199>`_, Ilsoo Byun)
+* build/ops: github: display phrase for signed-off check (`pr#29890 <https://github.com/ceph/ceph/pull/29890>`_, Ernesto Puerta)
+* build/ops: install-dep,rpm: use devtools-8 on amd64 (`issue#38892 <http://tracker.ceph.com/issues/38892>`_, `pr#27134 <https://github.com/ceph/ceph/pull/27134>`_, Kefu Chai)
+* build/ops: install-deps, rpm: use python_provide macro and cleanups (`pr#30830 <https://github.com/ceph/ceph/pull/30830>`_, Kefu Chai)
+* build/ops: install-deps,rpm,do_cmake: build on RHEL/CentOS 8 (`pr#30630 <https://github.com/ceph/ceph/pull/30630>`_, Kefu Chai)
+* build/ops: install-deps.sh,src: drop python2 support (`pr#31525 <https://github.com/ceph/ceph/pull/31525>`_, Kefu Chai)
+* build/ops: install-deps.sh: Actually set gpgcheck to false (`pr#33591 <https://github.com/ceph/ceph/pull/33591>`_, Brad Hubbard)
+* build/ops: install-deps.sh: add EPEL repo for non-x86_64 archs as well (`pr#30557 <https://github.com/ceph/ceph/pull/30557>`_, Kefu Chai, Nathan Cutler)
+* build/ops: install-deps.sh: add kens copr repo for el8 build (`pr#32324 <https://github.com/ceph/ceph/pull/32324>`_, Kefu Chai)
+* build/ops: install-deps.sh: add option to skip prebuilt boost-\\* pkgs installation (`pr#27776 <https://github.com/ceph/ceph/pull/27776>`_, Jun He)
+* build/ops: install-deps.sh: add support for Ubuntu Disco Dingo (`pr#30405 <https://github.com/ceph/ceph/pull/30405>`_, Patrick Seidensal)
+* build/ops: install-deps.sh: download wheel using pip wheel (`pr#29903 <https://github.com/ceph/ceph/pull/29903>`_, Kefu Chai)
+* build/ops: install-deps.sh: enable PowerTool repo for EL8 (`pr#30656 <https://github.com/ceph/ceph/pull/30656>`_, Kefu Chai)
+* build/ops: install-deps.sh: fix typo for krb5 on FreeBSD (`pr#28269 <https://github.com/ceph/ceph/pull/28269>`_, Thomas Johnson)
+* build/ops: install-deps.sh: install binutils 2.28 for xenial (`pr#31601 <https://github.com/ceph/ceph/pull/31601>`_, Kefu Chai)
+* build/ops: install-deps.sh: install libboost-test for seastar (`pr#28015 <https://github.com/ceph/ceph/pull/28015>`_, Kefu Chai)
+* build/ops: install-deps.sh: install python2-{virtualenv,devel} on SUSE if needed (`pr#32153 <https://github.com/ceph/ceph/pull/32153>`_, Nathan Cutler)
+* build/ops: install-deps.sh: install \\*rpm-macros (`issue#39164 <http://tracker.ceph.com/issues/39164>`_, `pr#27524 <https://github.com/ceph/ceph/pull/27524>`_, Kefu Chai)
+* build/ops: install-deps.sh: install `python\\*-devel` for python\\*rpm-macros (`pr#30190 <https://github.com/ceph/ceph/pull/30190>`_, Kefu Chai)
+* build/ops: install-deps.sh: only prepare wheels for make check (`pr#29912 <https://github.com/ceph/ceph/pull/29912>`_, Kefu Chai)
+* build/ops: install-deps.sh: use chacra for cmake repo (`pr#29475 <https://github.com/ceph/ceph/pull/29475>`_, Kefu Chai)
+* build/ops: install-deps.sh: Use dnf for rhel/centos 8 (`pr#31144 <https://github.com/ceph/ceph/pull/31144>`_, Brad Hubbard)
+* build/ops: install-deps.sh: use gcc-8 on xenial and trusty (`pr#28094 <https://github.com/ceph/ceph/pull/28094>`_, Kefu Chai)
+* build/ops: install-deps.sh: use GCC-9 on bionic (`pr#28454 <https://github.com/ceph/ceph/pull/28454>`_, Kefu Chai)
+* build/ops: install-deps.sh: use sepia/lab-extra/8 (`pr#31238 <https://github.com/ceph/ceph/pull/31238>`_, Kefu Chai)
+* build/ops: install-deps: do not install if rpm already installed (`pr#30612 <https://github.com/ceph/ceph/pull/30612>`_, Kefu Chai)
+* build/ops: install-deps: enable homebrew repos for RHEL8 (`pr#33905 <https://github.com/ceph/ceph/pull/33905>`_, Kefu Chai, Dan Mick)
+* build/ops: install-deps: revert 47d4351d (`pr#30122 <https://github.com/ceph/ceph/pull/30122>`_, Kefu Chai)
+* build/ops: make patch build dependency explicit (`issue#40175 <http://tracker.ceph.com/issues/40175>`_, `pr#28414 <https://github.com/ceph/ceph/pull/28414>`_, Nathan Cutler)
+* build/ops: make perf_async_msgr link jemalloc/tcmalloc (`pr#28039 <https://github.com/ceph/ceph/pull/28039>`_, Jianpeng Ma)
+* build/ops: make-dist: Bump Node.js to v10.18.1 (`pr#33059 <https://github.com/ceph/ceph/pull/33059>`_, Tiago Melo)
+* build/ops: make-dist: default to no dashboard frontend build parallelism (`pr#32037 <https://github.com/ceph/ceph/pull/32037>`_, Nathan Cutler)
+* build/ops: make-dist: drop Python 2/3 autoselect (`pr#27792 <https://github.com/ceph/ceph/pull/27792>`_, Nathan Cutler)
+* build/ops: make-dist: set version number only once (`pr#26281 <https://github.com/ceph/ceph/pull/26281>`_, Nathan Cutler)
+* build/ops: mgr/dashboard: Prevent angular of getting stuck during installation (`pr#29929 <https://github.com/ceph/ceph/pull/29929>`_, Tiago Melo)
+* build/ops: mgr/rook: Make use of rook-client-python when talking to Rook (`pr#29427 <https://github.com/ceph/ceph/pull/29427>`_, Sebastian Wagner)
+* build/ops: pybind/mgr/CMakeLists: exclude tox.ini, requirements.txt from install (`pr#31577 <https://github.com/ceph/ceph/pull/31577>`_, Sage Weil)
+* build/ops: pybind/mgr: Exclude tests/ (`pr#31671 <https://github.com/ceph/ceph/pull/31671>`_, Sebastian Wagner)
+* build/ops: pybind/mgr: Rename orchestrator_cli to orchestrator (`pr#32817 <https://github.com/ceph/ceph/pull/32817>`_, Sebastian Wagner)
+* build/ops: qa/tasks/ceph_deploy: do not rely on ceph-create-keys (`pr#29002 <https://github.com/ceph/ceph/pull/29002>`_, Sage Weil)
+* build/ops: Revert dpdk: drop dpdk submodule (`pr#32992 <https://github.com/ceph/ceph/pull/32992>`_, David Galloway)
+* build/ops: rpm,cmake: use specified python3 version if any (`pr#27358 <https://github.com/ceph/ceph/pull/27358>`_, Kefu Chai)
+* build/ops: rpm,deb: package always-enabled plugins in a separated package (`pr#33422 <https://github.com/ceph/ceph/pull/33422>`_, Kefu Chai)
+* build/ops: rpm,deb: python-requests is not needed for ceph-common (`pr#30420 <https://github.com/ceph/ceph/pull/30420>`_, luo.runbing)
+* build/ops: rpm,debian,install-deps: package crimson-osd (`pr#28428 <https://github.com/ceph/ceph/pull/28428>`_, Kefu Chai)
+* build/ops: rpm,etc/sysconfig: remove SuSEfirewall2 support (`issue#40738 <http://tracker.ceph.com/issues/40738>`_, `pr#28957 <https://github.com/ceph/ceph/pull/28957>`_, Matthias Gerstner)
+* build/ops: rpm/cephadm: move HOMEDIR to /var/lib and make scriptlets idempotent on SUSE (`pr#32212 <https://github.com/ceph/ceph/pull/32212>`_, Nathan Cutler)
+* build/ops: rpm: add cmake_verbose_logging switch (`pr#32805 <https://github.com/ceph/ceph/pull/32805>`_, Nathan Cutler)
+* build/ops: rpm: add Provides: python3-\\* for python packages and cleanup (`pr#27468 <https://github.com/ceph/ceph/pull/27468>`_, Kefu Chai)
+* build/ops: rpm: add rpm-build to SUSE-specific make check deps (`pr#32083 <https://github.com/ceph/ceph/pull/32083>`_, Nathan Cutler)
+* build/ops: rpm: always build ceph-test package (`pr#29685 <https://github.com/ceph/ceph/pull/29685>`_, Nathan Cutler)
+* build/ops: rpm: define weak_deps for el8 (`pr#33229 <https://github.com/ceph/ceph/pull/33229>`_, Kefu Chai)
+* build/ops: rpm: Disable LTO in spec when being used (`issue#39974 <http://tracker.ceph.com/issues/39974>`_, `pr#28170 <https://github.com/ceph/ceph/pull/28170>`_, Martin Li\xc5\xa1ka)
+* build/ops: rpm: drop vim-specific header (`pr#32331 <https://github.com/ceph/ceph/pull/32331>`_, Nathan Cutler)
+* build/ops: rpm: enable devtoolset-8 on aarch64 also (`issue#38892 <http://tracker.ceph.com/issues/38892>`_, `pr#27333 <https://github.com/ceph/ceph/pull/27333>`_, Kefu Chai)
+* build/ops: rpm: fdupes in SUSE builds to conform with packaging guidelines (`issue#40973 <http://tracker.ceph.com/issues/40973>`_, `pr#29346 <https://github.com/ceph/ceph/pull/29346>`_, Nathan Cutler)
+* build/ops: rpm: fix rhel <= 7 conditional (`pr#27045 <https://github.com/ceph/ceph/pull/27045>`_, Nathan Cutler)
+* build/ops: rpm: fix up a specfile syntax error (`pr#33066 <https://github.com/ceph/ceph/pull/33066>`_, Greg Farnum)
+* build/ops: rpm: have pybind RPMs provide/obsolete their python2 predecessors (`issue#40099 <http://tracker.ceph.com/issues/40099>`_, `pr#28352 <https://github.com/ceph/ceph/pull/28352>`_, Nathan Cutler)
+* build/ops: rpm: immutable-object-cache related changes (`pr#27150 <https://github.com/ceph/ceph/pull/27150>`_, Kefu Chai)
+* build/ops: rpm: improve ceph-mgr plugin package summaries (`issue#40974 <http://tracker.ceph.com/issues/40974>`_, `pr#29347 <https://github.com/ceph/ceph/pull/29347>`_, Nathan Cutler)
+* build/ops: rpm: make librados2, libcephfs2 own (create) /etc/ceph (`pr#30975 <https://github.com/ceph/ceph/pull/30975>`_, Nathan Cutler)
+* build/ops: rpm: put librgw lttng SOs in the librgw-devel package (`issue#40975 <http://tracker.ceph.com/issues/40975>`_, `pr#29349 <https://github.com/ceph/ceph/pull/29349>`_, Nathan Cutler)
+* build/ops: rpm: refrain from building ceph-resource-agents on SLE (`pr#27046 <https://github.com/ceph/ceph/pull/27046>`_, Nathan Cutler)
+* build/ops: rpm: Relax the selinux policy version for centos builds (`pr#32700 <https://github.com/ceph/ceph/pull/32700>`_, Boris Ranto)
+* build/ops: rpm: s/devtoolset-7/devtoolset-8/ (`pr#27183 <https://github.com/ceph/ceph/pull/27183>`_, Kefu Chai)
+* build/ops: rpm: use python 3.6 as the default python3 (`pr#27417 <https://github.com/ceph/ceph/pull/27417>`_, Kefu Chai)
+* build/ops: rpm: use python3.4 on RHEL7 by default (`pr#27407 <https://github.com/ceph/ceph/pull/27407>`_, Kefu Chai)
+* build/ops: rpm: use Recommends on fedora also (`pr#26819 <https://github.com/ceph/ceph/pull/26819>`_, Kefu Chai)
+* build/ops: run npm ci with a one-hour timeout (`pr#28994 <https://github.com/ceph/ceph/pull/28994>`_, Nathan Cutler)
+* build/ops: run-make-check.sh: extract run-make.sh (`pr#30184 <https://github.com/ceph/ceph/pull/30184>`_, Kefu Chai)
+* build/ops: run-make-check.sh: run sudo with absolute path (`pr#29753 <https://github.com/ceph/ceph/pull/29753>`_, Kefu Chai)
+* build/ops: run-make-check.sh: WITH_SEASTAR on demand (`pr#33723 <https://github.com/ceph/ceph/pull/33723>`_, Kefu Chai)
+* build/ops: script,doc: add gen-corpus.sh (`pr#28950 <https://github.com/ceph/ceph/pull/28950>`_, Kefu Chai)
+* build/ops: script/build-integration-branch: Add usage (`pr#32293 <https://github.com/ceph/ceph/pull/32293>`_, Sebastian Wagner)
+* build/ops: script/run-make.sh: do not pass cmake options twice (`pr#30318 <https://github.com/ceph/ceph/pull/30318>`_, Kefu Chai)
+* build/ops: script/run_tox.sh: Dont overwrite the build dir (`pr#29925 <https://github.com/ceph/ceph/pull/29925>`_, Sebastian Wagner)
+* build/ops: script: remove dep-report.sh (`pr#29296 <https://github.com/ceph/ceph/pull/29296>`_, Kefu Chai)
+* build/ops: scripts: ceph_dump_log.py (`pr#21729 <https://github.com/ceph/ceph/pull/21729>`_, Brad Hubbard)
+* build/ops: seastar: pickup change to add pthread linkage (`pr#33453 <https://github.com/ceph/ceph/pull/33453>`_, Kefu Chai)
+* build/ops: spec, debian: cephadm requires lvm2 (`pr#32323 <https://github.com/ceph/ceph/pull/32323>`_, Sebastian Wagner)
+* build/ops: spec,debian: ceph-mgr-ssh depends on openssh{-client{s}} (`pr#31806 <https://github.com/ceph/ceph/pull/31806>`_, Sebastian Wagner)
+* build/ops: spec: add missing python3-pyyaml (`pr#33387 <https://github.com/ceph/ceph/pull/33387>`_, Sebastian Wagner)
+* build/ops: spec: Podman (temporarily) requires apparmor-abstractions on suse (`pr#33850 <https://github.com/ceph/ceph/pull/33850>`_, Sebastian Wagner)
+* build/ops: src/CMakeLists: remove leading v from git describe version (`pr#31387 <https://github.com/ceph/ceph/pull/31387>`_, Sage Weil)
+* build/ops: test/fio: bump to fio-3.15 (`pr#31544 <https://github.com/ceph/ceph/pull/31544>`_, Igor Fedotov)
+* build/ops: test: only compile ceph_test_bmap_alloc_replay WITH_BLUESTORE (`pr#31306 <https://github.com/ceph/ceph/pull/31306>`_, Willem Jan Withagen)
+* build/ops: vstart: Remove duplicate option -N (`pr#31917 <https://github.com/ceph/ceph/pull/31917>`_, Kotresh HR)
+* ceph-crash: use ceph-crash as logger name (`pr#30989 <https://github.com/ceph/ceph/pull/30989>`_, Kefu Chai)
+* ceph-daemon -> cephadm, mgr/ssh -> mgr/cephadm (`pr#32193 <https://github.com/ceph/ceph/pull/32193>`_, Sage Weil)
+* ceph-daemon,mgr/ssh: add check-host (`pr#31795 <https://github.com/ceph/ceph/pull/31795>`_, Sage Weil)
+* ceph-daemon: -v|--verbose, not -d|--debug (`pr#31583 <https://github.com/ceph/ceph/pull/31583>`_, Sage Weil)
+* ceph-daemon: a few more py2 compatibility hacks (`pr#31264 <https://github.com/ceph/ceph/pull/31264>`_, Sage Weil)
+* ceph-daemon: add additional debug logging (`pr#31837 <https://github.com/ceph/ceph/pull/31837>`_, Michael Fritch)
+* ceph-daemon: Add basic mypy support (`pr#31609 <https://github.com/ceph/ceph/pull/31609>`_, Thomas Bechtold)
+* ceph-daemon: add explicit pull at bootstrap start (`pr#31478 <https://github.com/ceph/ceph/pull/31478>`_, Sage Weil)
+* ceph-daemon: Add more type hints (`pr#31631 <https://github.com/ceph/ceph/pull/31631>`_, Thomas Bechtold)
+* ceph-daemon: add osd create test (`pr#31679 <https://github.com/ceph/ceph/pull/31679>`_, Michael Fritch)
+* ceph-daemon: add standalone `adopt` tests (`pr#31486 <https://github.com/ceph/ceph/pull/31486>`_, Michael Fritch)
+* ceph-daemon: add `--base-dir` arg to `adopt` command (`pr#31487 <https://github.com/ceph/ceph/pull/31487>`_, Michael Fritch)
+* ceph-daemon: add `--legacy-dir` arg to `ls` command (`pr#31585 <https://github.com/ceph/ceph/pull/31585>`_, Michael Fritch)
+* ceph-daemon: Allow env var for setting the used image (`pr#31913 <https://github.com/ceph/ceph/pull/31913>`_, Thomas Bechtold)
+* ceph-daemon: append newline before public key string (`pr#31788 <https://github.com/ceph/ceph/pull/31788>`_, Ricardo Dias)
+* ceph-daemon: behave on rm-cluster when legacy dirs exist and ceph isnt installed (`pr#31499 <https://github.com/ceph/ceph/pull/31499>`_, Sage Weil)
+* ceph-daemon: bootstrap: make --output-\\* args optional (`pr#31695 <https://github.com/ceph/ceph/pull/31695>`_, Sage Weil)
+* ceph-daemon: ceph/daemon-base:latest-master-devel (`pr#31507 <https://github.com/ceph/ceph/pull/31507>`_, Sage Weil)
+* ceph-daemon: clean-up tempfiles on EXIT (`pr#32052 <https://github.com/ceph/ceph/pull/32052>`_, Michael Fritch)
+* ceph-daemon: combine SUDO and ARGS into a single var (`pr#32138 <https://github.com/ceph/ceph/pull/32138>`_, Michael Fritch)
+* ceph-daemon: configure firewalld for new daemons (`pr#31869 <https://github.com/ceph/ceph/pull/31869>`_, Sage Weil)
+* ceph-daemon: consolidate NamedTemporaryFile logic (`pr#31908 <https://github.com/ceph/ceph/pull/31908>`_, Michael Fritch)
+* ceph-daemon: create ~/.ssh if not exist (`pr#31315 <https://github.com/ceph/ceph/pull/31315>`_, Kefu Chai)
+* ceph-daemon: customize the bash prompt for shell + enter (`pr#31498 <https://github.com/ceph/ceph/pull/31498>`_, Sage Weil)
+* ceph-daemon: do not pass -it unless it is an interactive shell (`pr#31181 <https://github.com/ceph/ceph/pull/31181>`_, Sage Weil)
+* ceph-daemon: do not relabel system directories (`pr#31321 <https://github.com/ceph/ceph/pull/31321>`_, Sage Weil)
+* ceph-daemon: dont deref symlinks during chown (`pr#32137 <https://github.com/ceph/ceph/pull/32137>`_, Michael Fritch)
+* ceph-daemon: enable dashboard during bootstrap (`pr#31464 <https://github.com/ceph/ceph/pull/31464>`_, Sage Weil)
+* ceph-daemon: fix bootstrap ownership of tmp monmap file (`pr#32097 <https://github.com/ceph/ceph/pull/32097>`_, Sage Weil)
+* ceph-daemon: fix extract_uid_gid (`pr#31832 <https://github.com/ceph/ceph/pull/31832>`_, Sage Weil)
+* ceph-daemon: fix firewalld error case (`pr#32096 <https://github.com/ceph/ceph/pull/32096>`_, Sage Weil)
+* ceph-daemon: Fix handling for symlinks on python2 (`pr#31838 <https://github.com/ceph/ceph/pull/31838>`_, Michael Fritch)
+* ceph-daemon: fix os.mkdir call (`pr#31320 <https://github.com/ceph/ceph/pull/31320>`_, Sage Weil)
+* ceph-daemon: fix pod stop (`pr#32157 <https://github.com/ceph/ceph/pull/32157>`_, Sage Weil)
+* ceph-daemon: fix prompt (`pr#31603 <https://github.com/ceph/ceph/pull/31603>`_, Sage Weil)
+* ceph-daemon: fix standalone `adopt` OSD test (`pr#31772 <https://github.com/ceph/ceph/pull/31772>`_, Sage Weil, Michael Fritch)
+* ceph-daemon: fix traceback during `ls` command (`pr#31439 <https://github.com/ceph/ceph/pull/31439>`_, Michael Fritch)
+* ceph-daemon: fix version field for legacy `ls` (`pr#31443 <https://github.com/ceph/ceph/pull/31443>`_, Michael Fritch)
+* ceph-daemon: fix `systemctl is-enabled` bool (`pr#31870 <https://github.com/ceph/ceph/pull/31870>`_, Michael Fritch)
+* ceph-daemon: infer fsid for some commands (`pr#31702 <https://github.com/ceph/ceph/pull/31702>`_, Michael Fritch)
+* ceph-daemon: logs command (`pr#31575 <https://github.com/ceph/ceph/pull/31575>`_, Sage Weil)
+* ceph-daemon: make /var/run/ceph behavior better (`pr#31141 <https://github.com/ceph/ceph/pull/31141>`_, Sage Weil)
+* ceph-daemon: make infer_fsid behave when /var/lib/ceph dne (`pr#31831 <https://github.com/ceph/ceph/pull/31831>`_, Sage Weil)
+* ceph-daemon: make ls log less noisy (`pr#31448 <https://github.com/ceph/ceph/pull/31448>`_, Sage Weil)
+* ceph-daemon: make mon container privileged (`pr#31476 <https://github.com/ceph/ceph/pull/31476>`_, Sage Weil)
+* ceph-daemon: make ps1 a raw string (`pr#31540 <https://github.com/ceph/ceph/pull/31540>`_, Michael Fritch)
+* ceph-daemon: make rm-cluster faster (`pr#31538 <https://github.com/ceph/ceph/pull/31538>`_, Sage Weil)
+* ceph-daemon: make rm-cluster handle failed unit cleanup (`pr#31365 <https://github.com/ceph/ceph/pull/31365>`_, Sage Weil)
+* ceph-daemon: Move ceph-daemon executable to own directory (`pr#31467 <https://github.com/ceph/ceph/pull/31467>`_, Thomas Bechtold)
+* ceph-daemon: nicer errors (`pr#31886 <https://github.com/ceph/ceph/pull/31886>`_, Sage Weil, Michael Fritch)
+* ceph-daemon: Only run in the __main__ scope (`pr#31458 <https://github.com/ceph/ceph/pull/31458>`_, Thomas Bechtold)
+* ceph-daemon: only set up /var/run/ceph/$fsid if it exists (`pr#31341 <https://github.com/ceph/ceph/pull/31341>`_, Sage Weil)
+* ceph-daemon: only set up crash dir mount if it exists (`pr#31130 <https://github.com/ceph/ceph/pull/31130>`_, Sage Weil)
+* ceph-daemon: py2 compatibility (`pr#31168 <https://github.com/ceph/ceph/pull/31168>`_, Sage Weil)
+* ceph-daemon: py2: tolerate whitespace before config key name (`pr#32098 <https://github.com/ceph/ceph/pull/32098>`_, Sage Weil)
+* ceph-daemon: raise RuntimeError when CephContainer.run() fails (`pr#31328 <https://github.com/ceph/ceph/pull/31328>`_, Michael Fritch)
+* ceph-daemon: Remove data dir during adopt (`pr#31437 <https://github.com/ceph/ceph/pull/31437>`_, Michael Fritch)
+* ceph-daemon: remove prepare-host (`pr#32108 <https://github.com/ceph/ceph/pull/32108>`_, Sage Weil)
+* ceph-daemon: replace podman variables by container (`pr#31618 <https://github.com/ceph/ceph/pull/31618>`_, Dimitri Savineau)
+* ceph-daemon: seek relative to the start of file (`pr#31892 <https://github.com/ceph/ceph/pull/31892>`_, Michael Fritch)
+* ceph-daemon: set container_image during bootstrap (`pr#31445 <https://github.com/ceph/ceph/pull/31445>`_, Sage Weil)
+* ceph-daemon: set ssh public identity (`pr#31500 <https://github.com/ceph/ceph/pull/31500>`_, Sage Weil)
+* ceph-daemon: several fsid inference fixes (`pr#31798 <https://github.com/ceph/ceph/pull/31798>`_, Sage Weil)
+* ceph-daemon: switch default image (`pr#31463 <https://github.com/ceph/ceph/pull/31463>`_, Sage Weil)
+* ceph-daemon: unmount osd data dir during `adopt` (`pr#31477 <https://github.com/ceph/ceph/pull/31477>`_, Michael Fritch)
+* ceph-daemon: use client.admin keyring during bootstrap (`pr#31270 <https://github.com/ceph/ceph/pull/31270>`_, Sage Weil)
+* ceph-daemon: use `-e` instead of `--env` (`pr#31614 <https://github.com/ceph/ceph/pull/31614>`_, Michael Fritch)
+* ceph-daemon: Use `shutil.move` to move log files (`pr#31331 <https://github.com/ceph/ceph/pull/31331>`_, Michael Fritch)
+* ceph-daemon: `imp` module  DeprecationWarning (`pr#32161 <https://github.com/ceph/ceph/pull/32161>`_, Michael Fritch)
+* ceph-mon: keep v1 address type when explicitly set (`pr#31765 <https://github.com/ceph/ceph/pull/31765>`_, Ricardo Dias)
+* ceph-object-corpus: forward_incompat pg_missing_item and pg_missing_t (`pr#28034 <https://github.com/ceph/ceph/pull/28034>`_, lishuhao)
+* ceph-volume  simple: better detection when type file is not present (`pr#29386 <https://github.com/ceph/ceph/pull/29386>`_, Alfredo Deza)
+* ceph-volume  zap always skips block.db, leaves them around (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#28998 <https://github.com/ceph/ceph/pull/28998>`_, Alfredo Deza)
+* ceph-volume broken assertion errors after pytest changes (`issue#40665 <http://tracker.ceph.com/issues/40665>`_, `pr#28866 <https://github.com/ceph/ceph/pull/28866>`_, Alfredo Deza)
+* ceph-volume lvm.zap fix cleanup for db partitions (`issue#40664 <http://tracker.ceph.com/issues/40664>`_, `pr#28267 <https://github.com/ceph/ceph/pull/28267>`_, Dominik Csapak)
+* ceph-volume tests add a sleep in tox for slow OSDs after booting (`issue#40619 <http://tracker.ceph.com/issues/40619>`_, `pr#28836 <https://github.com/ceph/ceph/pull/28836>`_, Alfredo Deza)
+* ceph-volume tests remove xenial from functional testing (`pr#31159 <https://github.com/ceph/ceph/pull/31159>`_, Alfredo Deza)
+* ceph-volume tests set the noninteractive flag for Debian (`pr#29804 <https://github.com/ceph/ceph/pull/29804>`_, Alfredo Deza)
+* ceph-volume-zfs: add the inventory command (`pr#30995 <https://github.com/ceph/ceph/pull/30995>`_, Willem Jan Withagen)
+* ceph-volume/batch: fail on filtered devices when non-interactive (`pr#31978 <https://github.com/ceph/ceph/pull/31978>`_, Jan Fajerski)
+* ceph-volume/lvm/activate.py: clarify error message: fsid refers to osd_fsid (`pr#32351 <https://github.com/ceph/ceph/pull/32351>`_, Yaniv Kaul)
+* ceph-volume/test: patch VolumeGroups (`pr#31979 <https://github.com/ceph/ceph/pull/31979>`_, Jan Fajerski)
+* ceph-volume: add Cephs device id to inventory (`pr#31072 <https://github.com/ceph/ceph/pull/31072>`_, Sebastian Wagner)
+* ceph-volume: add db and wal support to raw mode (`pr#32828 <https://github.com/ceph/ceph/pull/32828>`_, S\xc3\xa9bastien Han)
+* ceph-volume: add methods to pass filters to pvs, vgs and lvs commands (`pr#32242 <https://github.com/ceph/ceph/pull/32242>`_, Rishabh Dave)
+* ceph-volume: add proper size attribute to partitions (`pr#31492 <https://github.com/ceph/ceph/pull/31492>`_, Jan Fajerski)
+* ceph-volume: add raw (--bluestore) mode (`pr#32095 <https://github.com/ceph/ceph/pull/32095>`_, Sage Weil)
+* ceph-volume: add sizing arguments to prepare (`pr#32235 <https://github.com/ceph/ceph/pull/32235>`_, Jan Fajerski)
+* ceph-volume: add utility functions (`pr#27282 <https://github.com/ceph/ceph/pull/27282>`_, Mohamad Gebai)
+* ceph-volume: allow raw block devices everywhere (`pr#31410 <https://github.com/ceph/ceph/pull/31410>`_, Jan Fajerski)
+* ceph-volume: allow to skip restorecon calls (`pr#31421 <https://github.com/ceph/ceph/pull/31421>`_, Alfredo Deza)
+* ceph-volume: api/lvm: check if list of LVs is empty (`pr#30101 <https://github.com/ceph/ceph/pull/30101>`_, Rishabh Dave)
+* ceph-volume: assume msgrV1 for all branches containing mimic (`pr#31592 <https://github.com/ceph/ceph/pull/31592>`_, Jan Fajerski)
+* ceph-volume: avoid calling zap_lv with a LV-less VG (`pr#33283 <https://github.com/ceph/ceph/pull/33283>`_, Jan Fajerski)
+* ceph-volume: batch bluestore fix create_lvs call (`pr#32929 <https://github.com/ceph/ceph/pull/32929>`_, Jan Fajerski)
+* ceph-volume: batch ensure device lists are disjoint (`pr#27754 <https://github.com/ceph/ceph/pull/27754>`_, Jan Fajerski)
+* ceph-volume: check if we run in an selinux environment (`pr#31809 <https://github.com/ceph/ceph/pull/31809>`_, Jan Fajerski)
+* ceph-volume: check if we run in an selinux environment, now also in py2 (`pr#31814 <https://github.com/ceph/ceph/pull/31814>`_, Jan Fajerski)
+* ceph-volume: Dereference symlink in lvm list (`pr#32525 <https://github.com/ceph/ceph/pull/32525>`_, Beno\xc3\xaet Knecht)
+* ceph-volume: detect ceph-disk osd if PARTLABEL is missing (`issue#40917 <http://tracker.ceph.com/issues/40917>`_, `pr#29401 <https://github.com/ceph/ceph/pull/29401>`_, Jan Fajerski)
+* ceph-volume: do not fail when trying to remove crypt mapper (`pr#30490 <https://github.com/ceph/ceph/pull/30490>`_, Guillaume Abrioux)
+* ceph-volume: dont keep device lists as sets (`pr#29683 <https://github.com/ceph/ceph/pull/29683>`_, Jan Fajerski)
+* ceph-volume: dont remove vg twice when zapping filestore (`pr#33332 <https://github.com/ceph/ceph/pull/33332>`_, Jan Fajerski)
+* ceph-volume: dont try to test lvm zap on simple tests (`pr#29659 <https://github.com/ceph/ceph/pull/29659>`_, Jan Fajerski)
+* ceph-volume: finer grained availability notion in inventory (`pr#32634 <https://github.com/ceph/ceph/pull/32634>`_, Jan Fajerski)
+* ceph-volume: fix batch functional tests, idempotent test must check s\xe2\x80\xa6 (`pr#29684 <https://github.com/ceph/ceph/pull/29684>`_, Jan Fajerski)
+* ceph-volume: fix device unittest, mock has_bluestore_label (`pr#32655 <https://github.com/ceph/ceph/pull/32655>`_, Jan Fajerski)
+* ceph-volume: fix has_bluestore_label() function (`pr#33074 <https://github.com/ceph/ceph/pull/33074>`_, Guillaume Abrioux)
+* ceph-volume: fix is_ceph_device for lvm batch (`pr#33223 <https://github.com/ceph/ceph/pull/33223>`_, Jan Fajerski, Dimitri Savineau)
+* ceph-volume: fix lvm list (`pr#33077 <https://github.com/ceph/ceph/pull/33077>`_, Guillaume Abrioux)
+* ceph-volume: fix regression and improve output in lvm list (`pr#33112 <https://github.com/ceph/ceph/pull/33112>`_, Jan Fajerski)
+* ceph-volume: fix stderr failure to decode/encode when redirected (`pr#30274 <https://github.com/ceph/ceph/pull/30274>`_, Alfredo Deza)
+* ceph-volume: fix the integer overflow (`pr#32106 <https://github.com/ceph/ceph/pull/32106>`_, dongdong tao)
+* ceph-volume: fix warnings raised by pytest (`pr#30422 <https://github.com/ceph/ceph/pull/30422>`_, Rishabh Dave)
+* ceph-volume: import mock.mock instead of unittest.mock (py2) (`pr#31816 <https://github.com/ceph/ceph/pull/31816>`_, Jan Fajerski)
+* ceph-volume: look for rotational data in lsblk (`pr#26957 <https://github.com/ceph/ceph/pull/26957>`_, Andrew Schoen)
+* ceph-volume: lvm: get_device_vgs() filter by provided prefix (`pr#33478 <https://github.com/ceph/ceph/pull/33478>`_, Jan Fajerski, Yehuda Sadeh)
+* ceph-volume: make get_devices fs location independent (`pr#31574 <https://github.com/ceph/ceph/pull/31574>`_, Jan Fajerski)
+* ceph-volume: minor clean-up of `simple scan` subcommand help (`pr#31821 <https://github.com/ceph/ceph/pull/31821>`_, Michael Fritch)
+* ceph-volume: minor optimizations related to class Volumess use (`pr#29665 <https://github.com/ceph/ceph/pull/29665>`_, Rishabh Dave)
+* ceph-volume: mokeypatch calls to lvm related binaries (`pr#31197 <https://github.com/ceph/ceph/pull/31197>`_, Jan Fajerski)
+* ceph-volume: never log to stdout, use stderr instead (`pr#29547 <https://github.com/ceph/ceph/pull/29547>`_, Jan Fajerski)
+* ceph-volume: pass --ssh-config to pytest to resolve hosts when connecting (`issue#40063 <http://tracker.ceph.com/issues/40063>`_, `pr#28294 <https://github.com/ceph/ceph/pull/28294>`_, Alfredo Deza)
+* ceph-volume: pass journal_size as Size not string (`pr#33320 <https://github.com/ceph/ceph/pull/33320>`_, Jan Fajerski)
+* ceph-volume: pre-install python-apt and its variants before test runs (`pr#30115 <https://github.com/ceph/ceph/pull/30115>`_, Alfredo Deza)
+* ceph-volume: print most logging messages to stderr (`issue#38548 <http://tracker.ceph.com/issues/38548>`_, `pr#27675 <https://github.com/ceph/ceph/pull/27675>`_, Jan Fajerski)
+* ceph-volume: PVolumes.filter shouldnt purge itself (`pr#30703 <https://github.com/ceph/ceph/pull/30703>`_, Rishabh Dave)
+* ceph-volume: rearrange api/lvm.py (`pr#30867 <https://github.com/ceph/ceph/pull/30867>`_, Rishabh Dave)
+* ceph-volume: refactor listing.py (`pr#31700 <https://github.com/ceph/ceph/pull/31700>`_, Rishabh Dave)
+* ceph-volume: reject disks smaller then 5GB in inventory (`issue#40776 <http://tracker.ceph.com/issues/40776>`_, `pr#29041 <https://github.com/ceph/ceph/pull/29041>`_, Jan Fajerski)
+* ceph-volume: revert --no-tmpfs change (`pr#30788 <https://github.com/ceph/ceph/pull/30788>`_, Sage Weil)
+* ceph-volume: silence ceph-bluestore-tool failures (`pr#33371 <https://github.com/ceph/ceph/pull/33371>`_, S\xc3\xa9bastien Han)
+* ceph-volume: skip osd creation when already done (`pr#33086 <https://github.com/ceph/ceph/pull/33086>`_, Guillaume Abrioux)
+* ceph-volume: strip _dmcrypt suffix in simple scan json output (`pr#33079 <https://github.com/ceph/ceph/pull/33079>`_, Jan Fajerski)
+* ceph-volume: systemd fix typo in log message (`pr#30497 <https://github.com/ceph/ceph/pull/30497>`_, Manu Zurm\xc3\xbchl)
+* ceph-volume: terminal: encode unicode when writing to stdout (`pr#27148 <https://github.com/ceph/ceph/pull/27148>`_, Alfredo Deza, Kefu Chai)
+* ceph-volume: use centos8 for functional testing (`pr#33174 <https://github.com/ceph/ceph/pull/33174>`_, Jan Fajerski)
+* ceph-volume: use correct extents if using db-devices and >1 osds_per_device (`pr#32177 <https://github.com/ceph/ceph/pull/32177>`_, Fabian Niepelt)
+* ceph-volume: use fsync for dd command (`pr#31479 <https://github.com/ceph/ceph/pull/31479>`_, Rishabh Dave)
+* ceph-volume: use get_device_vgs in has_common_vg (`pr#33246 <https://github.com/ceph/ceph/pull/33246>`_, Jan Fajerski)
+* ceph-volume: use python3 compatible print (`pr#30790 <https://github.com/ceph/ceph/pull/30790>`_, Kyr Shatskyy)
+* ceph-volume: use the Device.rotational property instead of sys_api (`pr#28060 <https://github.com/ceph/ceph/pull/28060>`_, Andrew Schoen)
+* ceph-volume: use the OSD identifier when reporting success (`pr#29762 <https://github.com/ceph/ceph/pull/29762>`_, Alfredo Deza)
+* ceph-volume: util: look for executable in $PATH (`pr#31787 <https://github.com/ceph/ceph/pull/31787>`_, Shyukri Shyukriev)
+* ceph-volume: util: Use proper param substition (`pr#28448 <https://github.com/ceph/ceph/pull/28448>`_, Shyukri Shyukriev)
+* ceph-volume: VolumeGroups.filter shouldnt purge itself (`pr#30707 <https://github.com/ceph/ceph/pull/30707>`_, Rishabh Dave)
+* ceph-volume: when testing disable the dashboard (`pr#29387 <https://github.com/ceph/ceph/pull/29387>`_, Andrew Schoen)
+* ceph.in: disable ASAN if libasan is not found (`pr#28247 <https://github.com/ceph/ceph/pull/28247>`_, Kefu Chai)
+* ceph.in: do not preload asan even if not needed (`pr#28703 <https://github.com/ceph/ceph/pull/28703>`_, Kefu Chai)
+* ceph.in: do not preload libasan if it is found (`pr#28275 <https://github.com/ceph/ceph/pull/28275>`_, Kefu Chai)
+* ceph.in: print decoded output in interactive mode (`pr#33099 <https://github.com/ceph/ceph/pull/33099>`_, Jun Su)
+* cephadm: --cap-add=SYS_PTRACE (`pr#33442 <https://github.com/ceph/ceph/pull/33442>`_, Sage Weil)
+* cephadm: Add ability to deploy grafana container (`pr#32491 <https://github.com/ceph/ceph/pull/32491>`_, Paul Cuzner)
+* cephadm: add ability to specify a timeout (`pr#32049 <https://github.com/ceph/ceph/pull/32049>`_, Michael Fritch)
+* cephadm: add alertmanager deployment feature (`pr#32949 <https://github.com/ceph/ceph/pull/32949>`_, Sage Weil, Paul Cuzner)
+* cephadm: add assert foo is not None for mypy check (`pr#33876 <https://github.com/ceph/ceph/pull/33876>`_, Kefu Chai)
+* cephadm: add grafana adopt (`pr#33746 <https://github.com/ceph/ceph/pull/33746>`_, Eric Jackson)
+* cephadm: add locking (`pr#32334 <https://github.com/ceph/ceph/pull/32334>`_, Sage Weil)
+* cephadm: add nfs-ganesha deployment (`pr#33064 <https://github.com/ceph/ceph/pull/33064>`_, Michael Fritch)
+* cephadm: add prepare-host (`pr#33374 <https://github.com/ceph/ceph/pull/33374>`_, Sage Weil)
+* cephadm: add prometheus adopt (`pr#33438 <https://github.com/ceph/ceph/pull/33438>`_, Eric Jackson)
+* cephadm: add reconfig service action (`pr#32281 <https://github.com/ceph/ceph/pull/32281>`_, Sage Weil)
+* cephadm: add start/stop hooks and c-v activate on container start (`pr#32158 <https://github.com/ceph/ceph/pull/32158>`_, Sage Weil)
+* cephadm: Add Zypper packager (openSUSE/SLES) (`pr#33461 <https://github.com/ceph/ceph/pull/33461>`_, Kristoffer Gr\xc3\xb6nlund)
+* cephadm: add `--retry` arg (`pr#33342 <https://github.com/ceph/ceph/pull/33342>`_, Michael Fritch)
+* cephadm: add {add,rm}-repo commands (`pr#33062 <https://github.com/ceph/ceph/pull/33062>`_, Sage Weil)
+* cephadm: add-repo: add --version (`pr#33961 <https://github.com/ceph/ceph/pull/33961>`_, Sage Weil)
+* cephadm: adopt fixes (`pr#32995 <https://github.com/ceph/ceph/pull/32995>`_, Sage Weil)
+* cephadm: allow multiple get_parm() calls (`pr#33437 <https://github.com/ceph/ceph/pull/33437>`_, Sage Weil)
+* cephadm: allow skipping prepare_host in bootstrap step (`pr#33504 <https://github.com/ceph/ceph/pull/33504>`_, Kiefer Chang)
+* cephadm: allow users to provide their dashboard cert during bootstrap (`pr#33472 <https://github.com/ceph/ceph/pull/33472>`_, Daniel-Pivonka)
+* cephadm: also return JSON decode error (`pr#33433 <https://github.com/ceph/ceph/pull/33433>`_, Sebastian Wagner)
+* cephadm: bootstrap: avoid repeat chars in generated password (`pr#32332 <https://github.com/ceph/ceph/pull/32332>`_, Sage Weil)
+* cephadm: bootstrap: deploy monitoring stack by default (`pr#33936 <https://github.com/ceph/ceph/pull/33936>`_, Sage Weil)
+* cephadm: bootstrap: nag about telemetry (`pr#33517 <https://github.com/ceph/ceph/pull/33517>`_, Sage Weil)
+* cephadm: bootstrap: wait for mgr to restart after enabling a module (`pr#33857 <https://github.com/ceph/ceph/pull/33857>`_, Sage Weil)
+* cephadm: bootstrap: warn on fqdn hostname (`pr#33042 <https://github.com/ceph/ceph/pull/33042>`_, Sage Weil)
+* cephadm: check for both chrony service names (`pr#33369 <https://github.com/ceph/ceph/pull/33369>`_, Sage Weil)
+* cephadm: check for both ntp.service and ntpd.service (`pr#32302 <https://github.com/ceph/ceph/pull/32302>`_, Sage Weil)
+* cephadm: clean up the systemd unit and ceph-crash shutdown behavior (`pr#32685 <https://github.com/ceph/ceph/pull/32685>`_, Sage Weil)
+* cephadm: correct ipv6 support in port open detection (`pr#32286 <https://github.com/ceph/ceph/pull/32286>`_, Paul Cuzner)
+* cephadm: create /var/run/ceph/$fsid as needed (`pr#32390 <https://github.com/ceph/ceph/pull/32390>`_, Sage Weil)
+* cephadm: disable node-exporter cpu/memory limits for the time being (`pr#33133 <https://github.com/ceph/ceph/pull/33133>`_, Sage Weil)
+* cephadm: drop sha256: prefix on container id (`pr#32300 <https://github.com/ceph/ceph/pull/32300>`_, Sage Weil)
+* cephadm: error out on filestore OSDs (`pr#33395 <https://github.com/ceph/ceph/pull/33395>`_, Sage Weil)
+* cephadm: fix adoption safety check (`pr#33445 <https://github.com/ceph/ceph/pull/33445>`_, Sage Weil)
+* cephadm: fix ceph version probe (`pr#33136 <https://github.com/ceph/ceph/pull/33136>`_, Sage Weil)
+* cephadm: fix container cleanup (`pr#32282 <https://github.com/ceph/ceph/pull/32282>`_, Sage Weil)
+* cephadm: fix datetime regexp to capture at most 6 digits (`pr#33932 <https://github.com/ceph/ceph/pull/33932>`_, Michael Fritch)
+* cephadm: fix deploy crash when no `args.fsid` (`pr#33248 <https://github.com/ceph/ceph/pull/33248>`_, Michael Fritch)
+* cephadm: fix error handing in `command_check_host()` (`pr#33048 <https://github.com/ceph/ceph/pull/33048>`_, Guillaume Abrioux)
+* cephadm: fix failure when getting keyring for deploying daemons (`pr#33679 <https://github.com/ceph/ceph/pull/33679>`_, Kiefer Chang)
+* cephadm: fix help message for bootstrap --mgr-id (`pr#32640 <https://github.com/ceph/ceph/pull/32640>`_, Sage Weil)
+* cephadm: fix inspect-image (`pr#33109 <https://github.com/ceph/ceph/pull/33109>`_, Sage Weil)
+* cephadm: fix logging defaults (`pr#32641 <https://github.com/ceph/ceph/pull/32641>`_, Sage Weil)
+* cephadm: fix name argument parsing during image check for non-ceph components (`pr#33114 <https://github.com/ceph/ceph/pull/33114>`_, Daniel-Pivonka)
+* cephadm: Fix Py3 ConfigParser deprecation warnings (`pr#32218 <https://github.com/ceph/ceph/pull/32218>`_, Michael Fritch)
+* cephadm: fix tox DeprecationWarning (`pr#32753 <https://github.com/ceph/ceph/pull/32753>`_, Michael Fritch)
+* cephadm: fix v1/v2 ip/addrv handling; explicitly check bind to ip:port (`pr#32392 <https://github.com/ceph/ceph/pull/32392>`_, Sage Weil)
+* cephadm: fix `alertmanager not implemented yet` (`pr#33694 <https://github.com/ceph/ceph/pull/33694>`_, Patrick Seidensal)
+* cephadm: flag dashboard user to change password (`pr#32990 <https://github.com/ceph/ceph/pull/32990>`_, Daniel-Pivonka)
+* cephadm: further simplify mon setup (`pr#33952 <https://github.com/ceph/ceph/pull/33952>`_, Sage Weil)
+* cephadm: implement install command (`pr#33979 <https://github.com/ceph/ceph/pull/33979>`_, Sage Weil)
+* cephadm: improve handling of crash agent container (`pr#33189 <https://github.com/ceph/ceph/pull/33189>`_, Sage Weil)
+* cephadm: include daemon/unit id in unit name (`pr#32970 <https://github.com/ceph/ceph/pull/32970>`_, Sage Weil)
+* cephadm: Infer ceph image (`pr#33829 <https://github.com/ceph/ceph/pull/33829>`_, Sage Weil, Ricardo Marques)
+* cephadm: infer the fsid by name (`pr#32795 <https://github.com/ceph/ceph/pull/32795>`_, Michael Fritch)
+* cephadm: KillMode=none in unit file (`pr#33162 <https://github.com/ceph/ceph/pull/33162>`_, Sage Weil)
+* cephadm: leave backup when removing stateful daemons (`pr#33973 <https://github.com/ceph/ceph/pull/33973>`_, Sage Weil)
+* cephadm: make add-repo --release and --version independent (`pr#34034 <https://github.com/ceph/ceph/pull/34034>`_, Sage Weil)
+* cephadm: merge `--config-and-keyring` and `--config-json` args (`pr#33870 <https://github.com/ceph/ceph/pull/33870>`_, Michael Fritch)
+* cephadm: misc upgrade fixes (`pr#32794 <https://github.com/ceph/ceph/pull/32794>`_, Sage Weil)
+* cephadm: no --no-systemd arg to ceph-volume deactivate (`pr#32886 <https://github.com/ceph/ceph/pull/32886>`_, Sage Weil)
+* cephadm: only infer image for shell, run, inspect-image, pull, ceph-volume (`pr#34030 <https://github.com/ceph/ceph/pull/34030>`_, Sage Weil)
+* cephadm: podman inspect: image field was called `ImageID` (`pr#32616 <https://github.com/ceph/ceph/pull/32616>`_, Sebastian Wagner)
+* cephadm: prepare-host: do not create Packager unless we need it (`pr#33443 <https://github.com/ceph/ceph/pull/33443>`_, Sage Weil)
+* cephadm: pull: strip newline from version string (`pr#33446 <https://github.com/ceph/ceph/pull/33446>`_, Sage Weil)
+* cephadm: python3 shebang (`pr#32378 <https://github.com/ceph/ceph/pull/32378>`_, Sage Weil)
+* cephadm: re-introduce the `podman logs` command (`pr#33089 <https://github.com/ceph/ceph/pull/33089>`_, Michael Fritch)
+* cephadm: Read ceph version from io.ceph.version label if set (`pr#32982 <https://github.com/ceph/ceph/pull/32982>`_, Kristoffer Gr\xc3\xb6nlund)
+* cephadm: Refactor, prepare for other adoptions (`pr#33672 <https://github.com/ceph/ceph/pull/33672>`_, Eric Jackson)
+* cephadm: relabel /etc/ganesha mount (`pr#34098 <https://github.com/ceph/ceph/pull/34098>`_, Sage Weil)
+* cephadm: remove orphan daemons (`pr#33830 <https://github.com/ceph/ceph/pull/33830>`_, Sage Weil)
+* cephadm: remove `logs` command (`pr#32752 <https://github.com/ceph/ceph/pull/32752>`_, Michael Fritch)
+* cephadm: Rename tox tests ceph-daemon -> cephadm (`pr#32353 <https://github.com/ceph/ceph/pull/32353>`_, Michael Fritch)
+* cephadm: report image name for stopped daemons (`pr#33190 <https://github.com/ceph/ceph/pull/33190>`_, Sage Weil)
+* cephadm: report version for grafana prom etc (`pr#33804 <https://github.com/ceph/ceph/pull/33804>`_, Sage Weil)
+* cephadm: shell: allow -e (`pr#33191 <https://github.com/ceph/ceph/pull/33191>`_, Sage Weil)
+* cephadm: shell: default to config and keyring in /etc/ceph, if present (`pr#33793 <https://github.com/ceph/ceph/pull/33793>`_, Sage Weil)
+* cephadm: shell: do not bind ceph.conf twice (`pr#32425 <https://github.com/ceph/ceph/pull/32425>`_, Sage Weil)
+* cephadm: shell: keep .bash_history in /var/log/ceph/$fsid (`pr#33519 <https://github.com/ceph/ceph/pull/33519>`_, Sage Weil)
+* cephadm: show contextual message when port is in use (`pr#32560 <https://github.com/ceph/ceph/pull/32560>`_, Michael Fritch)
+* cephadm: simplify Monitoring.components structure (`pr#32977 <https://github.com/ceph/ceph/pull/32977>`_, Michael Fritch)
+* cephadm: SO_REUSEADDR when doing bind check (`pr#32712 <https://github.com/ceph/ceph/pull/32712>`_, Sage Weil)
+* cephadm: streamline bootstrap a bit (`pr#33980 <https://github.com/ceph/ceph/pull/33980>`_, Sage Weil)
+* cephadm: support deployment of node-exporter (`pr#32340 <https://github.com/ceph/ceph/pull/32340>`_, Paul Cuzner)
+* cephadm: support deployment of prometheus container (`pr#32198 <https://github.com/ceph/ceph/pull/32198>`_, Sebastian Wagner, Paul Cuzner)
+* cephadm: switch grafana image to the ceph repo (`pr#34082 <https://github.com/ceph/ceph/pull/34082>`_, Paul Cuzner)
+* cephadm: update unit.\\* atomically (`pr#33895 <https://github.com/ceph/ceph/pull/33895>`_, Sage Weil)
+* cephadm: use appropriate default image for non-ceph components (`pr#33069 <https://github.com/ceph/ceph/pull/33069>`_, Sage Weil)
+* cephadm: use spec to deploy crash on every host (`pr#33658 <https://github.com/ceph/ceph/pull/33658>`_, Sage Weil)
+* cephadm: use `sh` instead of `bash` during enter (`pr#33822 <https://github.com/ceph/ceph/pull/33822>`_, Michael Fritch)
+* cephadm: wait longer for things to come up (`pr#33216 <https://github.com/ceph/ceph/pull/33216>`_, Sage Weil)
+* cephfs,common,core: global: disable THP for Ceph daemons (`pr#31582 <https://github.com/ceph/ceph/pull/31582>`_, Patrick Donnelly, Mark Nelson)
+* cephfs,common,rbd: common/config_proxy: hold lock while accessing mutable container (`pr#29809 <https://github.com/ceph/ceph/pull/29809>`_, Jason Dillaman)
+* cephfs,common: common/secret.c: fix key parsing when doing a remount (`pr#28148 <https://github.com/ceph/ceph/pull/28148>`_, Luis Henriques)
+* cephfs,common: osdc: should release the rwlock before waiting (`pr#29686 <https://github.com/ceph/ceph/pull/29686>`_, Kefu Chai)
+* cephfs,core: mds/MDSDaemon: fix asok exit and respawn commands (`pr#32251 <https://github.com/ceph/ceph/pull/32251>`_, Sage Weil)
+* cephfs,core: msg/async: perform the v2 resets in proper EventCenter (`pr#30717 <https://github.com/ceph/ceph/pull/30717>`_, Radoslaw Zarzynski)
+* cephfs,core: qa/suites/rados/mgr/tasks/module_selftest: whitelist mgr client getting backlisted (`issue#40867 <http://tracker.ceph.com/issues/40867>`_, `pr#29169 <https://github.com/ceph/ceph/pull/29169>`_, Sage Weil)
+* cephfs,core: qa/suites/upgrade: a few more octopus fixes (`pr#32853 <https://github.com/ceph/ceph/pull/32853>`_, Sage Weil)
+* cephfs,core: qa: log warning on scrub error (`pr#32739 <https://github.com/ceph/ceph/pull/32739>`_, Patrick Donnelly)
+* cephfs,core: src/: define ceph_release_t and use it (`pr#27855 <https://github.com/ceph/ceph/pull/27855>`_, Kefu Chai)
+* cephfs,mgr,mon: mon/MDSMonitor: enforce mds_join_fs cluster affinity (`pr#33194 <https://github.com/ceph/ceph/pull/33194>`_, Patrick Donnelly)
+* cephfs,mgr,mon: mon/MgrMonitor: blacklist previous instance of ceph-mgr during failover (`pr#31797 <https://github.com/ceph/ceph/pull/31797>`_, Patrick Donnelly)
+* cephfs,mgr,pybind: mgr/prometheus: export standby mds metadata (`pr#29996 <https://github.com/ceph/ceph/pull/29996>`_, lei01.liu)
+* cephfs,mgr,pybind: mgr/volumes: minor enhancements and fixes (`issue#40429 <http://tracker.ceph.com/issues/40429>`_, `pr#28706 <https://github.com/ceph/ceph/pull/28706>`_, Ramana Raja)
+* cephfs,mgr: mds/MDSRank: report state to mgr as mds id, not rank (`pr#31231 <https://github.com/ceph/ceph/pull/31231>`_, Patrick Donnelly, Sage Weil)
+* cephfs,mgr: mgr/volume: ceph cephfs metadata pool pg_num_min and bias (`pr#27374 <https://github.com/ceph/ceph/pull/27374>`_, Sage Weil)
+* cephfs,mgr: mgr/volumes: cleanup libcephfs handles on plugin shutdown (`issue#42299 <http://tracker.ceph.com/issues/42299>`_, `pr#30890 <https://github.com/ceph/ceph/pull/30890>`_, Venky Shankar)
+* cephfs,mgr: pybind/mgr/volumes: use py3 items iterator (`pr#31986 <https://github.com/ceph/ceph/pull/31986>`_, Patrick Donnelly)
+* cephfs,mgr: qa: use skipTest method instead of exception (`pr#27761 <https://github.com/ceph/ceph/pull/27761>`_, Patrick Donnelly)
+* cephfs,mon: mon/MDSMonitor: cleanup check_subs (`pr#32308 <https://github.com/ceph/ceph/pull/32308>`_, Patrick Donnelly)
+* cephfs,mon: mon/MDSMonitor: handle standby already without fscid (`pr#32585 <https://github.com/ceph/ceph/pull/32585>`_, Patrick Donnelly)
+* cephfs,pybind: libcephfs: add missing declaration of ceph_getaddrs() (`pr#32629 <https://github.com/ceph/ceph/pull/32629>`_, Kefu Chai)
+* cephfs,pybind: mgr/volumes: add `ceph fs subvolumegroup getpath` command (`issue#40617 <http://tracker.ceph.com/issues/40617>`_, `pr#29103 <https://github.com/ceph/ceph/pull/29103>`_, Ramana Raja)
+* cephfs,pybind: mgr/volumes: set uid/gid of FS clients mount as 0/0 (`issue#40927 <http://tracker.ceph.com/issues/40927>`_, `pr#29355 <https://github.com/ceph/ceph/pull/29355>`_, Ramana Raja)
+* cephfs,pybind: pybind/cephfs: add cephfs python API removexattr() (`pr#30641 <https://github.com/ceph/ceph/pull/30641>`_, bingyi zhang)
+* cephfs,pybind: pybind/cephfs: Add listxattr (`pr#32804 <https://github.com/ceph/ceph/pull/32804>`_, Varsha Rao)
+* cephfs,rbd,tests: qa/tasks: drop object inherit (`pr#29843 <https://github.com/ceph/ceph/pull/29843>`_, Jos Collin)
+* cephfs,rbd: osdc: using decltype(auto) instead of trailing return type (`pr#29931 <https://github.com/ceph/ceph/pull/29931>`_, Yao Zongyou)
+* cephfs,tests: cephfs-shell: teuthology tests (`issue#39526 <http://tracker.ceph.com/issues/39526>`_, `pr#27872 <https://github.com/ceph/ceph/pull/27872>`_, Milind Changire)
+* cephfs,tests: mgr/volumes: fs subvolume resize command (`pr#30054 <https://github.com/ceph/ceph/pull/30054>`_, Jos Collin)
+* cephfs,tests: qa/cephfs: add test for ACLs (`pr#29421 <https://github.com/ceph/ceph/pull/29421>`_, Rishabh Dave)
+* cephfs,tests: qa/cephfs: change deps for xfstests-dev on centos8 (`pr#32524 <https://github.com/ceph/ceph/pull/32524>`_, Rishabh Dave)
+* cephfs,tests: qa/cephfs: dont test kclient on RHEL 7 (`pr#32582 <https://github.com/ceph/ceph/pull/32582>`_, Rishabh Dave)
+* cephfs,tests: qa/cephfs: update xfstests-dev deps for RHEL 8 (`pr#33427 <https://github.com/ceph/ceph/pull/33427>`_, Rishabh Dave)
+* cephfs,tests: qa/suites/powercycle: install build deps for building xfstest (`pr#33874 <https://github.com/ceph/ceph/pull/33874>`_, Kefu Chai)
+* cephfs,tests: qa/tasks/cephfs/fuse_mount: use python3 (`pr#32339 <https://github.com/ceph/ceph/pull/32339>`_, Sage Weil)
+* cephfs,tests: qa/tasks: add exception in do_thrash() (`pr#29067 <https://github.com/ceph/ceph/pull/29067>`_, Jos Collin)
+* cephfs,tests: qa/tasks: DaemonWatchdog Expansion (`issue#10369 <http://tracker.ceph.com/issues/10369>`_, `issue#11314 <http://tracker.ceph.com/issues/11314>`_, `pr#28378 <https://github.com/ceph/ceph/pull/28378>`_, Jos Collin)
+* cephfs,tests: qa/tasks: Fix raises that doesnt re-raise (`pr#30201 <https://github.com/ceph/ceph/pull/30201>`_, Jos Collin)
+* cephfs,tests: qa/tasks: fixed typo in the comment (`pr#29759 <https://github.com/ceph/ceph/pull/29759>`_, Jos Collin)
+* cephfs,tests: qa/tasks: improvements in vstart_runner.py and mount.py (`pr#27481 <https://github.com/ceph/ceph/pull/27481>`_, Rishabh Dave)
+* cephfs,tests: qa/tasks: upgrade command arguments checks in vstart_runner.py (`pr#28198 <https://github.com/ceph/ceph/pull/28198>`_, Rishabh Dave)
+* cephfs,tests: qa/tests: reduce number of jobs for `kcephfs` (`pr#27328 <https://github.com/ceph/ceph/pull/27328>`_, Yuri Weinstein)
+* cephfs,tests: qa/tests: reduced number of jobs for `kcephfs` (`pr#27165 <https://github.com/ceph/ceph/pull/27165>`_, Yuri Weinstein)
+* cephfs,tests: qa/vstart_runner.py: make run()s interface same as teuthologys run (`pr#33263 <https://github.com/ceph/ceph/pull/33263>`_, Rishabh Dave)
+* cephfs,tests: qa: note timeout in debug message (`pr#32162 <https://github.com/ceph/ceph/pull/32162>`_, Patrick Donnelly)
+* cephfs,tests: qa: stop DaemonWatchdog for each cluster in daemon roles (`pr#29821 <https://github.com/ceph/ceph/pull/29821>`_, Patrick Donnelly)
+* cephfs,tests: qa: test fs:upgrade when running upgrade suite (`pr#31206 <https://github.com/ceph/ceph/pull/31206>`_, Patrick Donnelly)
+* cephfs,tests: test: define ALLPERMS if not yet (`pr#30726 <https://github.com/ceph/ceph/pull/30726>`_, Kefu Chai)
+* cephfs,tests: test_cephfs_shell: fix test_du_works_for_hardlinks (`pr#32168 <https://github.com/ceph/ceph/pull/32168>`_, Rishabh Dave)
+* cephfs,tests: test_cephfs_shell: initialize stderr for run_cephfs_shell_cmd() (`pr#31626 <https://github.com/ceph/ceph/pull/31626>`_, Rishabh Dave)
+* cephfs,tests: test_sessionmap: use sudo_write_file() from teuthology.misc (`pr#29123 <https://github.com/ceph/ceph/pull/29123>`_, Rishabh Dave)
+* cephfs,tools: cephfs-journal-tool: fix crash and usage (`pr#32452 <https://github.com/ceph/ceph/pull/32452>`_, Xiubo Li)
+* cephfs,tools: mount.ceph: fix incorrect options parsing (`pr#33197 <https://github.com/ceph/ceph/pull/33197>`_, Xiubo Li)
+* cephfs,tools: vstart.sh: highlight presence of stray conf (`pr#31403 <https://github.com/ceph/ceph/pull/31403>`_, Milind Changire)
+* cephfs:  client: more precise CEPH_CLIENT_CAPS_PENDING_CAPSNAP (`pr#28685 <https://github.com/ceph/ceph/pull/28685>`_, Yan, Zheng)
+* cephfs:  mds: change how mds revoke stale caps (`issue#17854 <http://tracker.ceph.com/issues/17854>`_, `pr#26737 <https://github.com/ceph/ceph/pull/26737>`_, Yan, Zheng, Rishabh Dave)
+* cephfs:  mds: fix corner case of replaying open sessions (`pr#28456 <https://github.com/ceph/ceph/pull/28456>`_, Yan, Zheng)
+* cephfs: Add doc for deploying cephfs-nfs cluster using rook (`pr#30914 <https://github.com/ceph/ceph/pull/30914>`_, Varsha Rao)
+* cephfs: Allow mount.ceph to get mount info from ceph configs and keyrings (`pr#29817 <https://github.com/ceph/ceph/pull/29817>`_, Jeff Layton)
+* cephfs: avoid map client_caps  been inserted by mistake (`pr#29304 <https://github.com/ceph/ceph/pull/29304>`_, XiaoGuoDong2019)
+* cephfs: ceph-mds: dump all info of ceph_file_layout, InodeStoreBase, frag_inf\xe2\x80\xa6 (`pr#28874 <https://github.com/ceph/ceph/pull/28874>`_, simon gao)
+* cephfs: ceph-mds: set ceph_mds cpu affinity (`pr#31712 <https://github.com/ceph/ceph/pull/31712>`_, qilianghong)
+* cephfs: cephfs pybind: added lseek() function to cephfs pybind (`pr#27688 <https://github.com/ceph/ceph/pull/27688>`_, Xiaowei Chu)
+* cephfs: cephfs-shell: Add command for setxattr, getxattr and listxattr (`pr#32570 <https://github.com/ceph/ceph/pull/32570>`_, Varsha Rao)
+* cephfs: cephfs-shell: Add error message for invalid ls commands (`pr#28652 <https://github.com/ceph/ceph/pull/28652>`_, Varsha Rao)
+* cephfs: cephfs-shell: add quota management (`issue#39165 <http://tracker.ceph.com/issues/39165>`_, `pr#27483 <https://github.com/ceph/ceph/pull/27483>`_, Milind Changire)
+* cephfs: cephfs-shell: add snapshot management (`issue#38681 <http://tracker.ceph.com/issues/38681>`_, `pr#27467 <https://github.com/ceph/ceph/pull/27467>`_, Milind Changire)
+* cephfs: cephfs-shell: Add stat command (`pr#27753 <https://github.com/ceph/ceph/pull/27753>`_, Varsha Rao)
+* cephfs: cephfs-shell: Add tox for testing with flake8 (`pr#28239 <https://github.com/ceph/ceph/pull/28239>`_, Varsha Rao)
+* cephfs: cephfs-shell: better complain info, when deleting non-empty directory (`issue#40864 <http://tracker.ceph.com/issues/40864>`_, `pr#30341 <https://github.com/ceph/ceph/pull/30341>`_, Shen Hang)
+* cephfs: cephfs-shell: Catch OSError exceptions in lcd (`issue#40243 <http://tracker.ceph.com/issues/40243>`_, `pr#28473 <https://github.com/ceph/ceph/pull/28473>`_, Varsha Rao)
+* cephfs: cephfs-shell: cd with no args must change CWD to root (`issue#40476 <http://tracker.ceph.com/issues/40476>`_, `pr#28793 <https://github.com/ceph/ceph/pull/28793>`_, Rishabh Dave)
+* cephfs: cephfs-shell: changes related to read_ceph_conf() (`pr#32347 <https://github.com/ceph/ceph/pull/32347>`_, Rishabh Dave)
+* cephfs: cephfs-shell: changes to stderr and stdout messages (`pr#30365 <https://github.com/ceph/ceph/pull/30365>`_, Rishabh Dave)
+* cephfs: cephfs-shell: Convert paths type from string to bytes (`pr#29552 <https://github.com/ceph/ceph/pull/29552>`_, Varsha Rao)
+* cephfs: cephfs-shell: du should ignore non-directory files (`issue#40371 <http://tracker.ceph.com/issues/40371>`_, `pr#28560 <https://github.com/ceph/ceph/pull/28560>`_, Rishabh Dave, Varsha Rao)
+* cephfs: cephfs-shell: Fix df command errors (`pr#27894 <https://github.com/ceph/ceph/pull/27894>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix flake8 blank line and indentation error (`pr#29149 <https://github.com/ceph/ceph/pull/29149>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix hidden files and directories list by ls command (`pr#27266 <https://github.com/ceph/ceph/pull/27266>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix lls command errors (`issue#40244 <http://tracker.ceph.com/issues/40244>`_, `pr#28475 <https://github.com/ceph/ceph/pull/28475>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix ls -l (`pr#32801 <https://github.com/ceph/ceph/pull/32801>`_, Kotresh HR)
+* cephfs: cephfs-shell: Fix mkdir relative path error (`pr#27822 <https://github.com/ceph/ceph/pull/27822>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix multiple flake8 errors (`pr#28080 <https://github.com/ceph/ceph/pull/28080>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix multiple flake8 errors (`pr#28433 <https://github.com/ceph/ceph/pull/28433>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix multiple flake8 errors (`pr#29374 <https://github.com/ceph/ceph/pull/29374>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix onecmd TypeError (`pr#29554 <https://github.com/ceph/ceph/pull/29554>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix print of error messages to stdout (`pr#28447 <https://github.com/ceph/ceph/pull/28447>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix rmdir -p issues and add tests for rmdir (`pr#31633 <https://github.com/ceph/ceph/pull/31633>`_, Varsha Rao)
+* cephfs: cephfs-shell: fix string decoding for ls command (`issue#39404 <http://tracker.ceph.com/issues/39404>`_, `pr#27716 <https://github.com/ceph/ceph/pull/27716>`_, Milind Changire)
+* cephfs: cephfs-shell: Fix TypeError in poutput() (`pr#28906 <https://github.com/ceph/ceph/pull/28906>`_, Varsha Rao)
+* cephfs: cephfs-shell: Fix typo for mounting (`pr#28718 <https://github.com/ceph/ceph/pull/28718>`_, Varsha Rao)
+* cephfs: cephfs-shell: fix unecessary usage of to_bytes for file paths (`issue#40455 <http://tracker.ceph.com/issues/40455>`_, `pr#28663 <https://github.com/ceph/ceph/pull/28663>`_, Patrick Donnelly)
+* cephfs: cephfs-shell: fix various tracebacks (`issue#38743 <http://tracker.ceph.com/issues/38743>`_, `issue#38739 <http://tracker.ceph.com/issues/38739>`_, `issue#38741 <http://tracker.ceph.com/issues/38741>`_, `issue#38740 <http://tracker.ceph.com/issues/38740>`_, `pr#27235 <https://github.com/ceph/ceph/pull/27235>`_, Milind Changire)
+* cephfs: cephfs-shell: make compatible with cmd2 versions after 0.9.13 (`pr#30585 <https://github.com/ceph/ceph/pull/30585>`_, Rishabh Dave)
+* cephfs: cephfs-shell: make every command set a return value on failure (`pr#32213 <https://github.com/ceph/ceph/pull/32213>`_, Rishabh Dave)
+* cephfs: cephfs-shell: print helpful message when conf file is not found (`pr#31460 <https://github.com/ceph/ceph/pull/31460>`_, Rishabh Dave)
+* cephfs: cephfs-shell: py version fixes (`issue#40418 <http://tracker.ceph.com/issues/40418>`_, `pr#28638 <https://github.com/ceph/ceph/pull/28638>`_, Patrick Donnelly)
+* cephfs: cephfs-shell: read options from ceph.conf (`pr#29964 <https://github.com/ceph/ceph/pull/29964>`_, Rishabh Dave)
+* cephfs: cephfs-shell: rearrange code for convenience (`pr#31629 <https://github.com/ceph/ceph/pull/31629>`_, Rishabh Dave)
+* cephfs: cephfs-shell: Remove extra length argument passed to setxattr() (`pr#30802 <https://github.com/ceph/ceph/pull/30802>`_, Varsha Rao)
+* cephfs: cephfs-shell: Remove str object references to attribute decode (`pr#27345 <https://github.com/ceph/ceph/pull/27345>`_, Varsha Rao)
+* cephfs: cephfs-shell: Remove undefined variable files in do_rm() (`pr#28710 <https://github.com/ceph/ceph/pull/28710>`_, Varsha Rao)
+* cephfs: cephfs-shell: return non-zero value on error (`pr#30657 <https://github.com/ceph/ceph/pull/30657>`_, Rishabh Dave)
+* cephfs: cephfs-shell: rewrite help text for put and get commands (`pr#30297 <https://github.com/ceph/ceph/pull/30297>`_, Rishabh Dave)
+* cephfs: cephfs-shell: Use colorama module instead of colorize (`pr#27427 <https://github.com/ceph/ceph/pull/27427>`_, Varsha Rao)
+* cephfs: ceph_volume_client: convert string to bytes object (`issue#40369 <http://tracker.ceph.com/issues/40369>`_, `issue#40800 <http://tracker.ceph.com/issues/40800>`_, `pr#28557 <https://github.com/ceph/ceph/pull/28557>`_, Rishabh Dave)
+* cephfs: ceph_volume_client: decode d_name before using it (`issue#39406 <http://tracker.ceph.com/issues/39406>`_, `pr#28196 <https://github.com/ceph/ceph/pull/28196>`_, Rishabh Dave)
+* cephfs: client: add client_fs mount option support (`pr#33506 <https://github.com/ceph/ceph/pull/33506>`_, Xiubo Li)
+* cephfs: client: Add is_dir() check before changing directory (`pr#32637 <https://github.com/ceph/ceph/pull/32637>`_, Varsha Rao)
+* cephfs: client: add procession of SEEK_HOLE and SEEK_DATA in lseek (`pr#30416 <https://github.com/ceph/ceph/pull/30416>`_, Shen Hang)
+* cephfs: client: add stx_btime and stx_version in cephfs.pyx (`pr#30206 <https://github.com/ceph/ceph/pull/30206>`_, huanwen ren)
+* cephfs: client: add warning when cap != in->auth_cap (`pr#30402 <https://github.com/ceph/ceph/pull/30402>`_, Shen Hang)
+* cephfs: client: avoid length overflow by calling the lseek function (`pr#29626 <https://github.com/ceph/ceph/pull/29626>`_, wenpengLi)
+* cephfs: Client: bump ll_ref from int32 to uint64_t (`pr#29136 <https://github.com/ceph/ceph/pull/29136>`_, Xiaoxi CHEN)
+* cephfs: client: directory size always is zero lead to is_quota_bytes_approaching lose efficacy (`pr#26104 <https://github.com/ceph/ceph/pull/26104>`_, guoyong)
+* cephfs: client: disallow changing fuse_default_permissions option at runtime (`pr#32315 <https://github.com/ceph/ceph/pull/32315>`_, Zhi Zhang)
+* cephfs: client: dont report any vxattrs to listxattr (`pr#29339 <https://github.com/ceph/ceph/pull/29339>`_, Jeff Layton)
+* cephfs: client: fix bad error handling in ll_lookup_inode (`issue#40085 <http://tracker.ceph.com/issues/40085>`_, `pr#28324 <https://github.com/ceph/ceph/pull/28324>`_, Jeff Layton)
+* cephfs: client: fix bad error handling in lseek SEEK_HOLE / SEEK_DATA (`pr#33480 <https://github.com/ceph/ceph/pull/33480>`_, Jeff Layton)
+* cephfs: client: fix dir.rctime and snap.btime vxattr values (`pr#28116 <https://github.com/ceph/ceph/pull/28116>`_, David Disseldorp)
+* cephfs: client: fix fuse client hang because its bad session PipeConnection to mds (`issue#39305 <http://tracker.ceph.com/issues/39305>`_, `pr#27482 <https://github.com/ceph/ceph/pull/27482>`_, Guan yunfei)
+* cephfs: client: fix lazyio_synchronize() to update file size (`pr#29705 <https://github.com/ceph/ceph/pull/29705>`_, Sidharth Anupkrishnan)
+* cephfs: client: Fixes for missing consts SEEK_DATA and SEEK_HOLE on alpine linux (`pr#33104 <https://github.com/ceph/ceph/pull/33104>`_, Stefan Bischoff)
+* cephfs: client: nfs-ganesha with cephfs client, removing dir reports not empty (`issue#40746 <http://tracker.ceph.com/issues/40746>`_, `pr#29005 <https://github.com/ceph/ceph/pull/29005>`_, Peng Xie)
+* cephfs: client: optimize rename operation under different quota root (`issue#39715 <http://tracker.ceph.com/issues/39715>`_, `pr#28077 <https://github.com/ceph/ceph/pull/28077>`_, Zhi Zhang)
+* cephfs: client: remove Inode.dir_contacts field and handle bad whence value to llseek gracefully (`pr#30580 <https://github.com/ceph/ceph/pull/30580>`_, Jeff Layton)
+* cephfs: client: remove unused variable (`pr#31509 <https://github.com/ceph/ceph/pull/31509>`_, su_nan@inspur.com)
+* cephfs: client: return -EIO when sync file which unsafe reqs have been dropped (`issue#40877 <http://tracker.ceph.com/issues/40877>`_, `pr#29167 <https://github.com/ceph/ceph/pull/29167>`_, simon gao)
+* cephfs: client: set snapdirs link count to 1 (`pr#28545 <https://github.com/ceph/ceph/pull/28545>`_, Yan, Zheng)
+* cephfs: client: support the fallocate() when fuse version >= 2.9 (`issue#40615 <http://tracker.ceph.com/issues/40615>`_, `pr#28831 <https://github.com/ceph/ceph/pull/28831>`_, huanwen ren)
+* cephfs: Client: unlink dentry for inode with llref=0 (`issue#40960 <http://tracker.ceph.com/issues/40960>`_, `pr#29321 <https://github.com/ceph/ceph/pull/29321>`_, Xiaoxi CHEN)
+* cephfs: client: _readdir_cache_cb() may use the readdir_cache already clear (`issue#41148 <http://tracker.ceph.com/issues/41148>`_, `pr#29526 <https://github.com/ceph/ceph/pull/29526>`_, huanwen ren)
+* cephfs: client\xef\xbc\x9aEINVAL may be returned when offset is 0 (`pr#30312 <https://github.com/ceph/ceph/pull/30312>`_, wenpengLi)
+* cephfs: Deploy ganesha daemons with vstart (`pr#31527 <https://github.com/ceph/ceph/pull/31527>`_, Varsha Rao)
+* cephfs: expose snapshot creation time as new ceph.snap.btime vxattr (`pr#27077 <https://github.com/ceph/ceph/pull/27077>`_, David Disseldorp)
+* cephfs: include: fix interval_set const_iterator call operator type (`pr#32185 <https://github.com/ceph/ceph/pull/32185>`_, Patrick Donnelly)
+* cephfs: libcephfs: Add Tests for LazyIO (`issue#40283 <http://tracker.ceph.com/issues/40283>`_, `pr#28834 <https://github.com/ceph/ceph/pull/28834>`_, Sidharth Anupkrishnan)
+* cephfs: mds : clean up data written to unsafe inodes (`pr#30969 <https://github.com/ceph/ceph/pull/30969>`_, simon gao)
+* cephfs: mds : optimization functions,get_dirfrags_under, to speed up processing directories with tens of millions of files (`pr#31123 <https://github.com/ceph/ceph/pull/31123>`_, simon gao)
+* cephfs: mds,mon: deprecate CephFS inline_data support (`pr#29824 <https://github.com/ceph/ceph/pull/29824>`_, Jeff Layton)
+* cephfs: mds/client: inode number delegation (`pr#31817 <https://github.com/ceph/ceph/pull/31817>`_, Jeff Layton)
+* cephfs: mds/FSMap: fix adjust_standby_fscid (`pr#32709 <https://github.com/ceph/ceph/pull/32709>`_, Sage Weil)
+* cephfs: mds/OpenFileTable: match MAX_ITEMS_PER_OBJ to osd_deep_scrub_large_omap_object_key_threshold (`pr#31232 <https://github.com/ceph/ceph/pull/31232>`_, Vikhyat Umrao)
+* cephfs: mds/server:mds: drop reconnect message from non-existent session (`issue#39026 <http://tracker.ceph.com/issues/39026>`_, `pr#27256 <https://github.com/ceph/ceph/pull/27256>`_, Shen Hang)
+* cephfs: messages: make CephFS messages safe (`pr#31330 <https://github.com/ceph/ceph/pull/31330>`_, Patrick Donnelly)
+* cephfs: mgr / volume: refactor [sub]volume (`issue#39969 <http://tracker.ceph.com/issues/39969>`_, `pr#28082 <https://github.com/ceph/ceph/pull/28082>`_, Venky Shankar)
+* cephfs: mgr / volumes: background purge queue for subvolumes (`issue#40036 <http://tracker.ceph.com/issues/40036>`_, `pr#28003 <https://github.com/ceph/ceph/pull/28003>`_, Patrick Donnelly, Venky Shankar)
+* cephfs: mgr/dashboard: CephFS class issues with strings (`pr#29353 <https://github.com/ceph/ceph/pull/29353>`_, Volker Theile)
+* cephfs: mgr/volume: adapt arg passing to ServiceSpec (`pr#33687 <https://github.com/ceph/ceph/pull/33687>`_, Joshua Schmid)
+* cephfs: mgr/volumes: add `mypy` support (`pr#33674 <https://github.com/ceph/ceph/pull/33674>`_, Michael Fritch)
+* cephfs: mgr/volumes: check for string values in uid/gid (`pr#31961 <https://github.com/ceph/ceph/pull/31961>`_, Jos Collin)
+* cephfs: mgr/volumes: cleanup leftovers from earlier purge job implementation (`pr#30886 <https://github.com/ceph/ceph/pull/30886>`_, Venky Shankar)
+* cephfs: mgr/volumes: cleanup on fs create error (`pr#32459 <https://github.com/ceph/ceph/pull/32459>`_, Jos Collin)
+* cephfs: mgr/volumes: clone from snapshot (`issue#24880 <http://tracker.ceph.com/issues/24880>`_, `pr#32030 <https://github.com/ceph/ceph/pull/32030>`_, Venky Shankar)
+* cephfs: mgr/volumes: convert string to bytes object (`issue#39750 <http://tracker.ceph.com/issues/39750>`_, `pr#28380 <https://github.com/ceph/ceph/pull/28380>`_, Rishabh Dave)
+* cephfs: mgr/volumes: drop unused size (`pr#30185 <https://github.com/ceph/ceph/pull/30185>`_, Jos Collin)
+* cephfs: mgr/volumes: drop unused variable vol_name (`pr#31780 <https://github.com/ceph/ceph/pull/31780>`_, Joshua Schmid)
+* cephfs: mgr/volumes: fail removing subvolume with snapshots (`issue#43645 <http://tracker.ceph.com/issues/43645>`_, `pr#32696 <https://github.com/ceph/ceph/pull/32696>`_, Venky Shankar)
+* cephfs: mgr/volumes: fetch trash and clone entries without blocking volume access (`issue#44207 <http://tracker.ceph.com/issues/44207>`_, `pr#33413 <https://github.com/ceph/ceph/pull/33413>`_, Venky Shankar)
+* cephfs: mgr/volumes: fix error message (`issue#40014 <http://tracker.ceph.com/issues/40014>`_, `pr#28407 <https://github.com/ceph/ceph/pull/28407>`_, Ramana Raja)
+* cephfs: mgr/volumes: fix incorrect snapshot path creation (`pr#30654 <https://github.com/ceph/ceph/pull/30654>`_, Ramana Raja)
+* cephfs: mgr/volumes: fix placement default value (`pr#33476 <https://github.com/ceph/ceph/pull/33476>`_, Sage Weil)
+* cephfs: mgr/volumes: fix subvolume creation with quota (`issue#40152 <http://tracker.ceph.com/issues/40152>`_, `pr#28384 <https://github.com/ceph/ceph/pull/28384>`_, Ramana Raja)
+* cephfs: mgr/volumes: fs subvolume resize inf command (`pr#31157 <https://github.com/ceph/ceph/pull/31157>`_, Jos Collin)
+* cephfs: mgr/volumes: handle exceptions in purge thread with retry (`issue#41218 <http://tracker.ceph.com/issues/41218>`_, `issue#41219 <http://tracker.ceph.com/issues/41219>`_, `pr#29735 <https://github.com/ceph/ceph/pull/29735>`_, Venky Shankar)
+* cephfs: mgr/volumes: improve volume deletion process (`pr#31762 <https://github.com/ceph/ceph/pull/31762>`_, Joshua Schmid)
+* cephfs: mgr/volumes: list FS subvolumes, subvolume groups, and their snapshots (`pr#30476 <https://github.com/ceph/ceph/pull/30476>`_, Jos Collin)
+* cephfs: mgr/volumes: minor fixes (`pr#29760 <https://github.com/ceph/ceph/pull/29760>`_, Ramana Raja)
+* cephfs: mgr/volumes: prevent negative subvolume size (`pr#30058 <https://github.com/ceph/ceph/pull/30058>`_, Jos Collin)
+* cephfs: mgr/volumes: protection for `fs volume rm` command (`pr#30407 <https://github.com/ceph/ceph/pull/30407>`_, Jos Collin)
+* cephfs: mgr/volumes: refactor dir handle cleanup (`pr#30887 <https://github.com/ceph/ceph/pull/30887>`_, Jos Collin)
+* cephfs: mgr/volumes: remove stale subvolume module (`pr#32645 <https://github.com/ceph/ceph/pull/32645>`_, Venky Shankar)
+* cephfs: mgr/volumes: return string type to ceph-manager (`pr#30451 <https://github.com/ceph/ceph/pull/30451>`_, Venky Shankar)
+* cephfs: mgr/volumes: sync inode attributes for cloned subvolumes (`issue#43965 <http://tracker.ceph.com/issues/43965>`_, `pr#33120 <https://github.com/ceph/ceph/pull/33120>`_, Venky Shankar)
+* cephfs: mgr/volumes: uid, gid for subvolume create and subvolumegroup create commands (`pr#30336 <https://github.com/ceph/ceph/pull/30336>`_, Jos Collin)
+* cephfs: mgr/volumes: unregister job upon async threads exception (`issue#44293 <http://tracker.ceph.com/issues/44293>`_, `pr#33547 <https://github.com/ceph/ceph/pull/33547>`_, Venky Shankar)
+* cephfs: mgr/volumes: versioned subvolume provisioning (`pr#31763 <https://github.com/ceph/ceph/pull/31763>`_, Venky Shankar)
+* cephfs: mon,mds: map mds daemons to a particular fs (`pr#32015 <https://github.com/ceph/ceph/pull/32015>`_, Sage Weil)
+* cephfs: mon/MDSMonitor: use stringstream instead of dout for mds repaired (`issue#40472 <http://tracker.ceph.com/issues/40472>`_, `pr#28683 <https://github.com/ceph/ceph/pull/28683>`_, Zhi Zhang)
+* cephfs: mon/MDSMonitor: warn when creating fs with default EC data pool (`pr#31494 <https://github.com/ceph/ceph/pull/31494>`_, Patrick Donnelly)
+* cephfs: mount.ceph.c: do not pass nofail to the kernel (`pr#26992 <https://github.com/ceph/ceph/pull/26992>`_, Kenneth Waegeman)
+* cephfs: mount.ceph: give a hint message when no mds is up or cluster is laggy (`pr#32164 <https://github.com/ceph/ceph/pull/32164>`_, Xiubo Li)
+* cephfs: mount.ceph: new mount option alias -- translate fs= option to mds_namespace= (`pr#33491 <https://github.com/ceph/ceph/pull/33491>`_, Xiubo Li)
+* cephfs: mount.ceph: properly handle -o strictatime (`pr#29518 <https://github.com/ceph/ceph/pull/29518>`_, Jeff Layton)
+* cephfs: mount.ceph: remove arbitrary limit on size of name= option (`pr#32706 <https://github.com/ceph/ceph/pull/32706>`_, Jeff Layton)
+* cephfs: mount: fix the debug log when keyring getting secret failed (`pr#33499 <https://github.com/ceph/ceph/pull/33499>`_, Xiubo Li)
+* cephfs: octopus: Add FS subvolume clone cancel (`issue#44208 <http://tracker.ceph.com/issues/44208>`_, `pr#34018 <https://github.com/ceph/ceph/pull/34018>`_, Venky Shankar)
+* cephfs: osdc/objecter: Fix last_sent in scientific format and add age to ops (`pr#29818 <https://github.com/ceph/ceph/pull/29818>`_, Varsha Rao)
+* cephfs: propagate ll_releasedir errors (`pr#32548 <https://github.com/ceph/ceph/pull/32548>`_, David Disseldorp)
+* cephfs: pybind / cephfs: remove static typing in LibCephFS.chown (`issue#42923 <http://tracker.ceph.com/issues/42923>`_, `pr#31756 <https://github.com/ceph/ceph/pull/31756>`_, Venky Shankar)
+* cephfs: pybind/cephfs: Modification to error message (`pr#28628 <https://github.com/ceph/ceph/pull/28628>`_, Varsha Rao)
+* cephfs: pybind/mgr: add cephfs subvolumes module (`issue#39610 <http://tracker.ceph.com/issues/39610>`_, `pr#27594 <https://github.com/ceph/ceph/pull/27594>`_, Ramana Raja)
+* cephfs: pybind/test_volume_client: print python version correctly (`issue#40184 <http://tracker.ceph.com/issues/40184>`_, `pr#28221 <https://github.com/ceph/ceph/pull/28221>`_, Lianne)
+* cephfs: qa/cephfs: fix test_evict_client (`pr#28411 <https://github.com/ceph/ceph/pull/28411>`_, Yan, Zheng)
+* cephfs: qa/cephfs: make filelock_interrupt.py work with python3 (`pr#32741 <https://github.com/ceph/ceph/pull/32741>`_, Yan, Zheng)
+* cephfs: qa/cephfs: test case for auto reconnect after blacklisted (`pr#31200 <https://github.com/ceph/ceph/pull/31200>`_, Yan, Zheng)
+* cephfs: qa/suites/fs/multifs/tasks/failover.yaml: disable RECENT_CRASH (`pr#29363 <https://github.com/ceph/ceph/pull/29363>`_, Sage Weil)
+* cephfs: qa/suites/fs: mon_thrash test for fs (`issue#17309 <http://tracker.ceph.com/issues/17309>`_, `pr#27073 <https://github.com/ceph/ceph/pull/27073>`_, Jos Collin)
+* cephfs: qa/tasks/cephfs: os.write takes bytes, not str (`pr#32359 <https://github.com/ceph/ceph/pull/32359>`_, Sage Weil)
+* cephfs: qa/tasks: add remaining tests for fs volume (`pr#31884 <https://github.com/ceph/ceph/pull/31884>`_, Jos Collin)
+* cephfs: qa/tasks: Better handling of thrasher names and __init__ calls (`pr#31207 <https://github.com/ceph/ceph/pull/31207>`_, Jos Collin)
+* cephfs: qa/tasks: check if fs mounted in umount_wait (`pr#30553 <https://github.com/ceph/ceph/pull/30553>`_, Jos Collin)
+* cephfs: qa/tasks: Fix AttributeError: cant set attribute (`pr#31428 <https://github.com/ceph/ceph/pull/31428>`_, Jos Collin)
+* cephfs: qa/tasks: upgrade the check for -c sudo option in vstart_runner.py (`issue#39385 <http://tracker.ceph.com/issues/39385>`_, `pr#28199 <https://github.com/ceph/ceph/pull/28199>`_, Rishabh Dave)
+* cephfs: qa/vstart_runner.py: add more options (`pr#29906 <https://github.com/ceph/ceph/pull/29906>`_, Rishabh Dave)
+* cephfs: qa: add debugging failed osd-release setting (`pr#29715 <https://github.com/ceph/ceph/pull/29715>`_, Patrick Donnelly)
+* cephfs: qa: add upgrade test for volume upgrade from legacy (`pr#33636 <https://github.com/ceph/ceph/pull/33636>`_, Patrick Donnelly)
+* cephfs: qa: allow client mount to reset fully (`issue#42213 <http://tracker.ceph.com/issues/42213>`_, `pr#30986 <https://github.com/ceph/ceph/pull/30986>`_, Venky Shankar)
+* cephfs: qa: avoid subtree rep in test_version_splitting (`pr#33078 <https://github.com/ceph/ceph/pull/33078>`_, Patrick Donnelly)
+* cephfs: qa: build v5.4 kernel (`pr#32763 <https://github.com/ceph/ceph/pull/32763>`_, Patrick Donnelly)
+* cephfs: qa: decouple session map test from simple msgr (`issue#38803 <http://tracker.ceph.com/issues/38803>`_, `pr#27415 <https://github.com/ceph/ceph/pull/27415>`_, Patrick Donnelly)
+* cephfs: qa: define centos version for fs:verify (`pr#32535 <https://github.com/ceph/ceph/pull/32535>`_, Patrick Donnelly)
+* cephfs: qa: detect RHEL8 for yum package installation (`pr#32507 <https://github.com/ceph/ceph/pull/32507>`_, Patrick Donnelly)
+* cephfs: qa: do not check pg count for new data_isolated volume (`pr#31095 <https://github.com/ceph/ceph/pull/31095>`_, Patrick Donnelly)
+* cephfs: qa: fix malformed suite config (`pr#29431 <https://github.com/ceph/ceph/pull/29431>`_, Patrick Donnelly)
+* cephfs: qa: fix output check to not be sensitive to debugging (`pr#32163 <https://github.com/ceph/ceph/pull/32163>`_, Patrick Donnelly)
+* cephfs: qa: fix testing kernel branch link (`pr#32854 <https://github.com/ceph/ceph/pull/32854>`_, Patrick Donnelly)
+* cephfs: qa: fix various py3 cephfs qa bugs (`pr#32467 <https://github.com/ceph/ceph/pull/32467>`_, Patrick Donnelly)
+* cephfs: qa: fix various py3 cephfs qa bugs x2 (`pr#32533 <https://github.com/ceph/ceph/pull/32533>`_, Patrick Donnelly)
+* cephfs: qa: fs Ignore ceph.dir.pin: No such attribute errors in getfattr tests for old kernel client (`pr#27377 <https://github.com/ceph/ceph/pull/27377>`_, Sidharth Anupkrishnan)
+* cephfs: qa: fs/upgrade test fixes and cephfs feature bit updates for Octopus/Nautilus (`issue#39078 <http://tracker.ceph.com/issues/39078>`_, `issue#39077 <http://tracker.ceph.com/issues/39077>`_, `issue#39020 <http://tracker.ceph.com/issues/39020>`_, `pr#27303 <https://github.com/ceph/ceph/pull/27303>`_, Patrick Donnelly)
+* cephfs: qa: have kclient tests use new mount.ceph functionality (`pr#30462 <https://github.com/ceph/ceph/pull/30462>`_, Jeff Layton)
+* cephfs: qa: ignore expected MDS_CLIENT_LATE_RELEASE warning (`issue#40968 <http://tracker.ceph.com/issues/40968>`_, `pr#29338 <https://github.com/ceph/ceph/pull/29338>`_, Patrick Donnelly)
+* cephfs: qa: ignore RECENT_CRASH for multimds snapshot testing (`pr#29911 <https://github.com/ceph/ceph/pull/29911>`_, Patrick Donnelly)
+* cephfs: qa: ignore slow ops for ffsb workunit (`pr#32668 <https://github.com/ceph/ceph/pull/32668>`_, Patrick Donnelly)
+* cephfs: qa: ignore trimmed cache items for dead cache drop (`pr#32644 <https://github.com/ceph/ceph/pull/32644>`_, Patrick Donnelly)
+* cephfs: qa: install some dependencies for xfstests (`pr#32478 <https://github.com/ceph/ceph/pull/32478>`_, Patrick Donnelly)
+* cephfs: qa: only restart MDS between tests (`pr#32532 <https://github.com/ceph/ceph/pull/32532>`_, Patrick Donnelly)
+* cephfs: qa: remove requirement on simple msgr (`issue#39079 <http://tracker.ceph.com/issues/39079>`_, `pr#27301 <https://github.com/ceph/ceph/pull/27301>`_, Patrick Donnelly)
+* cephfs: qa: rename kcephfs distro overrides (`pr#32639 <https://github.com/ceph/ceph/pull/32639>`_, Patrick Donnelly)
+* cephfs: qa: save MDS epoch barrier (`pr#32642 <https://github.com/ceph/ceph/pull/32642>`_, Patrick Donnelly)
+* cephfs: qa: sleep briefly after resetting kclient (`pr#29388 <https://github.com/ceph/ceph/pull/29388>`_, Patrick Donnelly)
+* cephfs: qa: specify random distros in multimds (`pr#33080 <https://github.com/ceph/ceph/pull/33080>`_, Patrick Donnelly)
+* cephfs: qa: tolerate ECONNRESET errcode during logrotate (`issue#41800 <http://tracker.ceph.com/issues/41800>`_, `pr#30809 <https://github.com/ceph/ceph/pull/30809>`_, Venky Shankar)
+* cephfs: qa: update kclient testing to RHEL 7.6 (`pr#26662 <https://github.com/ceph/ceph/pull/26662>`_, Patrick Donnelly)
+* cephfs: qa: use -D_GNU_SOURCE when compiling fsync-tester.c (`pr#32480 <https://github.com/ceph/ceph/pull/32480>`_, Patrick Donnelly)
+* cephfs: qa: use hard_reset to reboot kclient (`issue#37681 <http://tracker.ceph.com/issues/37681>`_, `pr#28825 <https://github.com/ceph/ceph/pull/28825>`_, Patrick Donnelly)
+* cephfs: qa: use mimic-O upgrade process (`pr#27731 <https://github.com/ceph/ceph/pull/27731>`_, Patrick Donnelly)
+* cephfs: qa: use small default pg count for CephFS pools (`pr#30816 <https://github.com/ceph/ceph/pull/30816>`_, Patrick Donnelly)
+* cephfs: qa: wait for MDS to come back after removing it (`issue#40967 <http://tracker.ceph.com/issues/40967>`_, `pr#29336 <https://github.com/ceph/ceph/pull/29336>`_, Patrick Donnelly)
+* cephfs: qa: whitelist Error recovering journal for cephfs-data-scan (`pr#30971 <https://github.com/ceph/ceph/pull/30971>`_, Yan, Zheng)
+* cephfs: qa: whitelist TOO_FEW_PGS during Mimic deploy (`pr#31063 <https://github.com/ceph/ceph/pull/31063>`_, Patrick Donnelly)
+* cephfs: Resolve a memory leak in cephfs/Resetter.cc (`pr#29302 <https://github.com/ceph/ceph/pull/29302>`_, XiaoGuoDong2019)
+* cephfs: src/common: fix help text for echo option of cephfs-shell (`pr#33285 <https://github.com/ceph/ceph/pull/33285>`_, Rishabh Dave)
+* cephfs: stop: Cleanly umount cephFS volumes (`pr#32024 <https://github.com/ceph/ceph/pull/32024>`_, Kotresh HR)
+* cephfs: test/{fs,cephfs}: Get libcephfs and cephfs to compile with FreeBSD (`pr#30505 <https://github.com/ceph/ceph/pull/30505>`_, Willem Jan Withagen)
+* cephfs: test: extend `fs subvolume` test to cover new interfaces (`issue#39949 <http://tracker.ceph.com/issues/39949>`_, `pr#27856 <https://github.com/ceph/ceph/pull/27856>`_, Venky Shankar)
+* cephfs: test: use distinct subvolume/group/snapshot names (`issue#42646 <http://tracker.ceph.com/issues/42646>`_, `pr#31418 <https://github.com/ceph/ceph/pull/31418>`_, Venky Shankar)
+* cephfs: test_volumes: fix _verify_clone_attrs call (`pr#33788 <https://github.com/ceph/ceph/pull/33788>`_, Ramana Raja)
+* cephfs: test_volume_client: declare only one default for python version (`issue#40460 <http://tracker.ceph.com/issues/40460>`_, `pr#28194 <https://github.com/ceph/ceph/pull/28194>`_, Rishabh Dave)
+* cephfs: test_volume_client: fix test_put_object_versioned() (`issue#39405 <http://tracker.ceph.com/issues/39405>`_, `issue#39510 <http://tracker.ceph.com/issues/39510>`_, `pr#28692 <https://github.com/ceph/ceph/pull/28692>`_, Rishabh Dave)
+* cephfs: test_volume_client: simplify test_get_authorized_ids() (`pr#28171 <https://github.com/ceph/ceph/pull/28171>`_, Rishabh Dave)
+* cephfs: tools/cephfs: make cephfs-data-scan scan_links fix dentrys first (`pr#31680 <https://github.com/ceph/ceph/pull/31680>`_, Yan, Zheng)
+* cephfs: Trivial comment and cleanup fixes for cephfs (`pr#27199 <https://github.com/ceph/ceph/pull/27199>`_, Jeff Layton)
+* cephfs: vstart: add an alias for cephfs-shell to vstart_environment.sh (`pr#27437 <https://github.com/ceph/ceph/pull/27437>`_, Jeff Layton)
+* cephfs: vstart: generate environment script suitable for sourcing (`pr#27198 <https://github.com/ceph/ceph/pull/27198>`_, Jeff Layton)
+* cephfs: vstart_runner: allow the use of it with kernel mounts (`pr#30463 <https://github.com/ceph/ceph/pull/30463>`_, Jeff Layton)
+* ceph_argparse: increment matchcnt on kwargs (`pr#33004 <https://github.com/ceph/ceph/pull/33004>`_, Matthew Oliver)
+* check rdma configuration and fix some logic problem (`pr#28344 <https://github.com/ceph/ceph/pull/28344>`_, Changcheng Liu)
+* client/Client : Fix sign compare compiler warning (`pr#30719 <https://github.com/ceph/ceph/pull/30719>`_, Prashant D)
+* cls/queue: fix data corruption in urgent data (`pr#33686 <https://github.com/ceph/ceph/pull/33686>`_, Yuval Lifshitz)
+* cmake:  support parallel build for rocksd (`pr#31781 <https://github.com/ceph/ceph/pull/31781>`_, Deepika Upadhyay)
+* cmake: add add_tox_test() (`pr#29446 <https://github.com/ceph/ceph/pull/29446>`_, Kefu Chai)
+* cmake: add cython_cephfs to vstart target (`pr#28876 <https://github.com/ceph/ceph/pull/28876>`_, Kefu Chai)
+* cmake: Add dpdk numa support (`pr#31841 <https://github.com/ceph/ceph/pull/31841>`_, Chunsong Feng, Hu Ye)
+* cmake: Allow cephfs and ceph-mds to be build when building on FreeBSD (`pr#30494 <https://github.com/ceph/ceph/pull/30494>`_, Willem Jan Withagen)
+* cmake: avoid rebuilding extensions, and using python-config (`pr#28920 <https://github.com/ceph/ceph/pull/28920>`_, Kefu Chai)
+* cmake: boost fixes for ARM 32 bit (`pr#25729 <https://github.com/ceph/ceph/pull/25729>`_, Daniel Glaser)
+* cmake: bump libceph-common SO version for compliance (`pr#30976 <https://github.com/ceph/ceph/pull/30976>`_, Nathan Cutler)
+* cmake: check for MAJOR.MINOR version of python3 (`pr#27383 <https://github.com/ceph/ceph/pull/27383>`_, Kefu Chai, Boris Ranto)
+* cmake: check for unaligned access (`pr#28936 <https://github.com/ceph/ceph/pull/28936>`_, Kefu Chai)
+* cmake: check version of librdkafka (`pr#32237 <https://github.com/ceph/ceph/pull/32237>`_, Kefu Chai)
+* cmake: cleanups (`pr#28252 <https://github.com/ceph/ceph/pull/28252>`_, Kefu Chai)
+* cmake: cleanups (`pr#33500 <https://github.com/ceph/ceph/pull/33500>`_, Kefu Chai)
+* cmake: compile crimson-auth with crimson::cflags (`pr#33296 <https://github.com/ceph/ceph/pull/33296>`_, Kefu Chai)
+* cmake: dashboard: enable frontend on arm64 (`pr#30958 <https://github.com/ceph/ceph/pull/30958>`_, Kefu Chai)
+* cmake: define mgr_cap_obj library when WITH_MGR=OFF (`pr#31326 <https://github.com/ceph/ceph/pull/31326>`_, Casey Bodley)
+* cmake: detect librt for POSIX time functions (`pr#31543 <https://github.com/ceph/ceph/pull/31543>`_, Kefu Chai)
+* cmake: detect linker support (`pr#30781 <https://github.com/ceph/ceph/pull/30781>`_, Kefu Chai)
+* cmake: Do a debug build by default (`pr#30799 <https://github.com/ceph/ceph/pull/30799>`_, Brad Hubbard)
+* cmake: do not assume ${CMAKE_GENERATOR} == make (`pr#27089 <https://github.com/ceph/ceph/pull/27089>`_, Kefu Chai)
+* cmake: do not include ${CMAKE_SOURCE_DIR}/src/fmt/include (`pr#31761 <https://github.com/ceph/ceph/pull/31761>`_, Kefu Chai)
+* cmake: do not include global_context.cc multiple times (`pr#32607 <https://github.com/ceph/ceph/pull/32607>`_, Kefu Chai)
+* cmake: do not link against unused libs (`pr#33247 <https://github.com/ceph/ceph/pull/33247>`_, Kefu Chai)
+* cmake: do not use CMP0074 unless it is supported (`pr#31958 <https://github.com/ceph/ceph/pull/31958>`_, Kefu Chai)
+* cmake: do not use CMP0093 unless it is supported (`pr#31960 <https://github.com/ceph/ceph/pull/31960>`_, Kefu Chai)
+* cmake: exclude unittest_alloc_aging from all (`pr#33466 <https://github.com/ceph/ceph/pull/33466>`_, Kefu Chai)
+* cmake: Fix build against ncurses with separate libtinfo (`pr#27443 <https://github.com/ceph/ceph/pull/27443>`_, Lars Wendler)
+* cmake: Fix unaligned check on big-endian systems (`pr#30362 <https://github.com/ceph/ceph/pull/30362>`_, Ulrich Weigand)
+* cmake: fix WITH_UBSAN (`pr#28725 <https://github.com/ceph/ceph/pull/28725>`_, Casey Bodley)
+* cmake: Improve test for 16-byte atomic support on IBM Z (`pr#32802 <https://github.com/ceph/ceph/pull/32802>`_, Ulrich Weigand)
+* cmake: let vstart depend on radosgwd (`pr#32564 <https://github.com/ceph/ceph/pull/32564>`_, Kefu Chai)
+* cmake: link ceph-fuse against librt (`pr#31531 <https://github.com/ceph/ceph/pull/31531>`_, Yong Wang)
+* cmake: move crimson-crush to crimson/ (`pr#33481 <https://github.com/ceph/ceph/pull/33481>`_, Kefu Chai)
+* cmake: one run_tox.sh to rule them all (`pr#29457 <https://github.com/ceph/ceph/pull/29457>`_, Kefu Chai)
+* cmake: pass arguments to crimson tests (`pr#30655 <https://github.com/ceph/ceph/pull/30655>`_, Kefu Chai)
+* cmake: pmem/pmdk changes to cmake (`pr#28802 <https://github.com/ceph/ceph/pull/28802>`_, Scott Peterson, Xiaoyan Li)
+* cmake: remove cython 0.29s subinterpreter check during install (`pr#27067 <https://github.com/ceph/ceph/pull/27067>`_, Tim Serong)
+* cmake: Removed unittest_alloc_aging from make check (`pr#33397 <https://github.com/ceph/ceph/pull/33397>`_, Adam Kupczyk)
+* cmake: require CMake v3.10.2 (`pr#29291 <https://github.com/ceph/ceph/pull/29291>`_, Kefu Chai)
+* cmake: require RocksDB 5.14 or higher (`pr#29930 <https://github.com/ceph/ceph/pull/29930>`_, Ilsoo Byun)
+* cmake: revert librados_tp.so version from 3 to 2 (`issue#39291 <http://tracker.ceph.com/issues/39291>`_, `pr#27593 <https://github.com/ceph/ceph/pull/27593>`_, Nathan Cutler)
+* cmake: rewrite Findgenl to support components argument (`pr#28460 <https://github.com/ceph/ceph/pull/28460>`_, Kefu Chai)
+* cmake: s/bortli_libs/brotli_libs/ (`pr#30374 <https://github.com/ceph/ceph/pull/30374>`_, Kefu Chai)
+* cmake: selectively rewrite install rpath (`pr#30028 <https://github.com/ceph/ceph/pull/30028>`_, Kefu Chai)
+* cmake: set empty INSTALL_RPATH on crypto shared libs (`issue#40398 <http://tracker.ceph.com/issues/40398>`_, `pr#28593 <https://github.com/ceph/ceph/pull/28593>`_, Nathan Cutler)
+* cmake: set empty RPATH for some test executables (`pr#29922 <https://github.com/ceph/ceph/pull/29922>`_, Nathan Cutler)
+* cmake: set empty-string RPATH for ceph-osd (`issue#40295 <http://tracker.ceph.com/issues/40295>`_, `pr#28508 <https://github.com/ceph/ceph/pull/28508>`_, Nathan Cutler)
+* cmake: should expose ${C-ARES_BINARY_DIR} from c-ares (`pr#33256 <https://github.com/ceph/ceph/pull/33256>`_, Kefu Chai)
+* cmake: silence messages when cppcheck/IWYU is not found (`pr#32140 <https://github.com/ceph/ceph/pull/32140>`_, Kefu Chai)
+* cmake: support `Seastar_DPDK=ON` option (`pr#31110 <https://github.com/ceph/ceph/pull/31110>`_, Kefu Chai)
+* cmake: Test for 16-byte atomic support on IBM Z (`pr#30638 <https://github.com/ceph/ceph/pull/30638>`_, Ulrich Weigand)
+* cmake: update FindBoost.cmake (`pr#29396 <https://github.com/ceph/ceph/pull/29396>`_, Willem Jan Withagen)
+* cmake: update FindBoost.cmake for 1.71 (`pr#31317 <https://github.com/ceph/ceph/pull/31317>`_, Willem Jan Withagen)
+* cmake: Update pmdk version to 1.7 (`pr#32693 <https://github.com/ceph/ceph/pull/32693>`_, Yin, Congmin)
+* cmake: update SPDK to build with GCC-9 (`pr#28507 <https://github.com/ceph/ceph/pull/28507>`_, Kefu Chai)
+* cmake: use BUILD_ALWAYS for rebuilding external project (`pr#28984 <https://github.com/ceph/ceph/pull/28984>`_, Kefu Chai)
+* cmake: use GNU linker on FreeBSD (`pr#30621 <https://github.com/ceph/ceph/pull/30621>`_, Willem Jan Withagen)
+* cmake: use latest FindPython\\*.cmake (`pr#29100 <https://github.com/ceph/ceph/pull/29100>`_, Kefu Chai)
+* cmake: use python2 by default (`pr#29148 <https://github.com/ceph/ceph/pull/29148>`_, Kefu Chai)
+* cmake: use StdFilesystem::filesystem instead of stdc++fs (`pr#27149 <https://github.com/ceph/ceph/pull/27149>`_, Willem Jan Withagen)
+* cmake: workaround of false alarm from ubsan (`pr#27094 <https://github.com/ceph/ceph/pull/27094>`_, Kefu Chai)
+* CMakeLists.txt: fix typo in error message (`pr#28795 <https://github.com/ceph/ceph/pull/28795>`_, Kefu Chai)
+* codeowners: Add ceph2.py to @ceph/orchestrators (`pr#32131 <https://github.com/ceph/ceph/pull/32131>`_, Sebastian Wagner)
+* common,core,mon: src/: drop cct from cmd_getval() (`pr#33010 <https://github.com/ceph/ceph/pull/33010>`_, Kefu Chai)
+* common,core: common, auth: use boost::spirit to parse ceph.conf, escape quotes in exported auths (`issue#22227 <http://tracker.ceph.com/issues/22227>`_, `pr#28634 <https://github.com/ceph/ceph/pull/28634>`_, Kefu Chai, Gu Zhongyan)
+* common,core: common,mgr,osd: pass string_view as name (`pr#33167 <https://github.com/ceph/ceph/pull/33167>`_, Kefu Chai)
+* common,core: common,osd: add hash algorithms for dedup fingerprint (`pr#28254 <https://github.com/ceph/ceph/pull/28254>`_, Myoungwon Oh)
+* common,core: include/cpp-btree: use the same type when allocate/deallocate (`pr#33638 <https://github.com/ceph/ceph/pull/33638>`_, Kefu Chai)
+* common,core: message,mgr: drop MessageFactory and friends and use ref_t<> in mgr (`pr#27592 <https://github.com/ceph/ceph/pull/27592>`_, Patrick Donnelly, Kefu Chai)
+* common,core: Remove dependence on `using namespace`: Build of common through osdc/Objecter.cc (`pr#27255 <https://github.com/ceph/ceph/pull/27255>`_, Adam C. Emerson)
+* common,mgr: vstart.sh: set prometheus port for each mgr (`pr#33698 <https://github.com/ceph/ceph/pull/33698>`_, Alfonso Mart\xc3\xadnez)
+* common,mon: common/options: make mon_clean_pg_upmaps_per_chunk unsigned (`pr#28509 <https://github.com/ceph/ceph/pull/28509>`_, Kefu Chai)
+* common,rbd: common/ceph_context: avoid unnecessary wait during service thread shutdown (`pr#30947 <https://github.com/ceph/ceph/pull/30947>`_, Jason Dillaman)
+* common,rgw: common/Formatter: escape printed buffer in XMLFormatter::dump_format_va() (`issue#38121 <http://tracker.ceph.com/issues/38121>`_, `pr#26220 <https://github.com/ceph/ceph/pull/26220>`_, ashitakasam)
+* common,rgw: rgw/OutputDataSocket: actually discard data on full buffer (`issue#40178 <http://tracker.ceph.com/issues/40178>`_, `pr#28415 <https://github.com/ceph/ceph/pull/28415>`_, Matt Benjamin)
+* common,tests: python-common: Add mypy testing (`pr#31071 <https://github.com/ceph/ceph/pull/31071>`_, Sebastian Wagner)
+* common,tests: test/test_mempool: test accounting for btree_map (`pr#33621 <https://github.com/ceph/ceph/pull/33621>`_, Adam Kupczyk)
+* common,tools: src/common: add rabin chunking for dedup (`pr#26730 <https://github.com/ceph/ceph/pull/26730>`_, Myoungwon Oh, Hsuan-Heng, Wu)
+* common,tools: vstart.sh: enable creating multiple OSDs backed by spdk backend (`pr#27841 <https://github.com/ceph/ceph/pull/27841>`_, Richael Zhuang)
+* common,tools: vstart.sh: enable nfs-ganesha mgmt. in dashboard (`pr#33691 <https://github.com/ceph/ceph/pull/33691>`_, Alfonso Mart\xc3\xadnez)
+* common/config_values: set seastar logging level per that of ceph (`pr#28792 <https://github.com/ceph/ceph/pull/28792>`_, Kefu Chai)
+* common/options: remove unused ms_msgr2\\_{sign,encrypt}_messages (`pr#31818 <https://github.com/ceph/ceph/pull/31818>`_, Ilya Dryomov)
+* common:  crimson/osd: add --mkkey support (`pr#28534 <https://github.com/ceph/ceph/pull/28534>`_, Kefu Chai)
+* common: .gitignore: ignore /src/python-common/build (`pr#32967 <https://github.com/ceph/ceph/pull/32967>`_, Alfonso Mart\xc3\xadnez)
+* common: add --log-early command line option (`pr#27419 <https://github.com/ceph/ceph/pull/27419>`_, Sage Weil)
+* common: add bool log_to_file option (`pr#27044 <https://github.com/ceph/ceph/pull/27044>`_, Sage Weil)
+* common: add comment about pod memory requests/limits (`pr#29331 <https://github.com/ceph/ceph/pull/29331>`_, Patrick Donnelly)
+* common: add iterator-based string splitter (`pr#33696 <https://github.com/ceph/ceph/pull/33696>`_, Casey Bodley)
+* common: add ref header (`pr#29119 <https://github.com/ceph/ceph/pull/29119>`_, Patrick Donnelly)
+* common: auth/cephx: always initialize local variables (`pr#31154 <https://github.com/ceph/ceph/pull/31154>`_, Kefu Chai)
+* common: auth/krb: fix Kerberos compile error (`issue#39948 <http://tracker.ceph.com/issues/39948>`_, `pr#28113 <https://github.com/ceph/ceph/pull/28113>`_, huangjun)
+* common: avoid use of size_t in options (`pr#28277 <https://github.com/ceph/ceph/pull/28277>`_, James Page)
+* common: blobhash.h: remove extra [[fallthrough]] (`pr#28270 <https://github.com/ceph/ceph/pull/28270>`_, Thomas Johnson)
+* common: blobhash: do not use cast for unaligned access (`pr#28099 <https://github.com/ceph/ceph/pull/28099>`_, Kefu Chai)
+* common: buffer, denc: more constness (`pr#27767 <https://github.com/ceph/ceph/pull/27767>`_, Kefu Chai)
+* common: buffer,crypto,tools: extract digest methods out of bufferlist (`pr#28486 <https://github.com/ceph/ceph/pull/28486>`_, Kefu Chai)
+* common: buffer.h: remove list::iterator_impl::advance(size_t) (`pr#28278 <https://github.com/ceph/ceph/pull/28278>`_, Kefu Chai)
+* common: ceph.in: use sys._exit when we dont shut down (`pr#33950 <https://github.com/ceph/ceph/pull/33950>`_, Sage Weil)
+* common: ceph_argparse: put args from env before existing ones (`pr#33243 <https://github.com/ceph/ceph/pull/33243>`_, Kefu Chai)
+* common: Clang requires a default constructor, but it can be empty (`issue#39561 <http://tracker.ceph.com/issues/39561>`_, `pr#27844 <https://github.com/ceph/ceph/pull/27844>`_, Willem Jan Withagen)
+* common: clean up CLUSTER_CREATE and CREATE options (`pr#31584 <https://github.com/ceph/ceph/pull/31584>`_, Sage Weil)
+* common: common,crimson: fixes to compile with clang and libc++ (`pr#32485 <https://github.com/ceph/ceph/pull/32485>`_, Kefu Chai)
+* common: common,crimson: supporting admin-socket commands (`pr#32174 <https://github.com/ceph/ceph/pull/32174>`_, Ronen Friedman, Kefu Chai)
+* common: common,log: use ISO 8601 datetime format (`pr#27799 <https://github.com/ceph/ceph/pull/27799>`_, Sage Weil, Casey Bodley)
+* common: common,os: address string truncated warnings from GCC-9 (`pr#28289 <https://github.com/ceph/ceph/pull/28289>`_, Kefu Chai)
+* common: common/admin_socket: Added printing of error message (`pr#33380 <https://github.com/ceph/ceph/pull/33380>`_, Adam Kupczyk)
+* common: common/bl: carry the bufferlist::_carriage over std::moves (`pr#32937 <https://github.com/ceph/ceph/pull/32937>`_, Radoslaw Zarzynski)
+* common: common/bl: fix memory corruption in bufferlist::claim_append() (`pr#32823 <https://github.com/ceph/ceph/pull/32823>`_, Radoslaw Zarzynski)
+* common: common/bl: fix the dangling last_p issue (`pr#32702 <https://github.com/ceph/ceph/pull/32702>`_, Radoslaw Zarzynski)
+* common: common/bloom_filter: Fix endian issues (`pr#30527 <https://github.com/ceph/ceph/pull/30527>`_, Ulrich Weigand)
+* common: common/ceph_time: tolerate mono time going backwards (`pr#33699 <https://github.com/ceph/ceph/pull/33699>`_, Sage Weil)
+* common: common/config: cleanups (`pr#33362 <https://github.com/ceph/ceph/pull/33362>`_, Jianpeng Ma)
+* common: common/config: fix lack of normalize_key_name() apply (`pr#33558 <https://github.com/ceph/ceph/pull/33558>`_, Igor Fedotov)
+* common: common/config: Remove unused code (`pr#28940 <https://github.com/ceph/ceph/pull/28940>`_, Jianpeng Ma)
+* common: common/Finisher: remove some lock acquisitions (`pr#29495 <https://github.com/ceph/ceph/pull/29495>`_, Igor Fedotov)
+* common: common/options: change default erasure-code-profile to k=2 m=2 (`pr#27656 <https://github.com/ceph/ceph/pull/27656>`_, Sage Weil)
+* common: common/pick_address.cc: silence GCC warning (`pr#32025 <https://github.com/ceph/ceph/pull/32025>`_, Kefu Chai)
+* common: common/secret.c: dont pass uninitialized stack data to the kernel (`pr#30675 <https://github.com/ceph/ceph/pull/30675>`_, Ilya Dryomov)
+* common: common/thread: Fix race condition in make_named_thread (`pr#31057 <https://github.com/ceph/ceph/pull/31057>`_, Adam C. Emerson)
+* common: common/util: use ifstream to read from /proc files (`pr#32630 <https://github.com/ceph/ceph/pull/32630>`_, Kefu Chai)
+* common: common/WorkQueue: narrow ThreadPool::_lock in func worker (`pr#22411 <https://github.com/ceph/ceph/pull/22411>`_, Jianpeng Ma)
+* common: crimson, common: introduce ceph::atomic and apply it on bufferlist (`pr#32766 <https://github.com/ceph/ceph/pull/32766>`_, Radoslaw Zarzynski)
+* common: crimson, common: RefCountedObj doesnt use atomics in Seastar builds (`pr#28085 <https://github.com/ceph/ceph/pull/28085>`_, Radoslaw Zarzynski)
+* common: crimson/osd: implement readable/lease related methods (`pr#30639 <https://github.com/ceph/ceph/pull/30639>`_, Kefu Chai)
+* common: crimson/osd: Message has non-null ref to SocketConnection now (`pr#30124 <https://github.com/ceph/ceph/pull/30124>`_, Radoslaw Zarzynski)
+* common: crimson: cleanups (`pr#33797 <https://github.com/ceph/ceph/pull/33797>`_, Kefu Chai)
+* common: crimson: cleanups for clang build (`pr#32605 <https://github.com/ceph/ceph/pull/32605>`_, Kefu Chai)
+* common: Cycles: Add support for IBM Z (`pr#30874 <https://github.com/ceph/ceph/pull/30874>`_, Ulrich Weigand)
+* common: default pg_autoscale_mode=on for new pools (`pr#30112 <https://github.com/ceph/ceph/pull/30112>`_, Sage Weil)
+* common: default pg_autoscale_mode=on for new pools (`pr#30475 <https://github.com/ceph/ceph/pull/30475>`_, Sage Weil)
+* common: denc: fix build error by calling global snprintf (`pr#27572 <https://github.com/ceph/ceph/pull/27572>`_, Changcheng Liu)
+* common: denc: slightly optimize container_base::bound_encode (`pr#24636 <https://github.com/ceph/ceph/pull/24636>`_, Radoslaw Zarzynski, Kefu Chai)
+* common: denc: support enums wider than 8 bits (`pr#33673 <https://github.com/ceph/ceph/pull/33673>`_, Casey Bodley)
+* common: dmclock: pick up fix to replace uint (`pr#28829 <https://github.com/ceph/ceph/pull/28829>`_, Kefu Chai)
+* common: drop sharing of buffer::raw outside bufferlist (`pr#32806 <https://github.com/ceph/ceph/pull/32806>`_, Radoslaw Zarzynski)
+* common: encode for std::list<T> doesnt use bl::copy_in() anymore (`pr#32785 <https://github.com/ceph/ceph/pull/32785>`_, Radoslaw Zarzynski)
+* common: FIPS: audit and switch some memset & bzero users (`pr#31692 <https://github.com/ceph/ceph/pull/31692>`_, Radoslaw Zarzynski)
+* common: Fix 44373 and make a couple cleanups in ceph::timer (`pr#33771 <https://github.com/ceph/ceph/pull/33771>`_, Adam C. Emerson)
+* common: fix clang build failures, and clean up warnings (`pr#26701 <https://github.com/ceph/ceph/pull/26701>`_, Adam C. Emerson)
+* common: fix clang compile errors from cython_modules (`pr#33056 <https://github.com/ceph/ceph/pull/33056>`_, Mark Kogan)
+* common: fix compat of strerror_r (`pr#30279 <https://github.com/ceph/ceph/pull/30279>`_, luo.runbing)
+* common: fix deadlocky inflight op visiting in OpTracker (`pr#32364 <https://github.com/ceph/ceph/pull/32364>`_, Radoslaw Zarzynski)
+* common: fix missing <stdio.h> include (`pr#31209 <https://github.com/ceph/ceph/pull/31209>`_, Willem Jan Withagen)
+* common: fix parse_env nullptr deref (`pr#28159 <https://github.com/ceph/ceph/pull/28159>`_, Patrick Donnelly)
+* common: Fix the error handling logic in get_device_id (`pr#30636 <https://github.com/ceph/ceph/pull/30636>`_, Difan Zhang)
+* common: fix typo in rgw_user_max_buckets option long description (`pr#31571 <https://github.com/ceph/ceph/pull/31571>`_, Alfonso Mart\xc3\xadnez)
+* common: give lockdeps group name to OpenSSLs mutexes (`issue#40698 <http://tracker.ceph.com/issues/40698>`_, `pr#28987 <https://github.com/ceph/ceph/pull/28987>`_, Radoslaw Zarzynski)
+* common: global/global_context: always add \\0 after strncpy() (`pr#28365 <https://github.com/ceph/ceph/pull/28365>`_, Kefu Chai)
+* common: global/global_init: do first transport connection after setuid() (`pr#28012 <https://github.com/ceph/ceph/pull/28012>`_, Roman Penyaev)
+* common: global/pidfile: pass string_view instead of ConfigProxy to pidfile_wr\xe2\x80\xa6 (`pr#27975 <https://github.com/ceph/ceph/pull/27975>`_, Kefu Chai)
+* common: handle return value from read(2) (`pr#32192 <https://github.com/ceph/ceph/pull/32192>`_, Patrick Donnelly)
+* common: include, common: make ceph::bufferlist 32 bytes long on x86 (`pr#32934 <https://github.com/ceph/ceph/pull/32934>`_, Radoslaw Zarzynski)
+* common: include/buffer: add operator+=() for list::iterator (`pr#33003 <https://github.com/ceph/ceph/pull/33003>`_, Kefu Chai)
+* common: include/cpp-btree: drop btree::dump() (`pr#32692 <https://github.com/ceph/ceph/pull/32692>`_, Kefu Chai)
+* common: include/interval_set: rename some types (`pr#32415 <https://github.com/ceph/ceph/pull/32415>`_, Kefu Chai)
+* common: include: switch mempool.h to ceph::atomic (`pr#33034 <https://github.com/ceph/ceph/pull/33034>`_, Radoslaw Zarzynski)
+* common: json: JSONDecoder::err inherits from std::runtime_error (`pr#27957 <https://github.com/ceph/ceph/pull/27957>`_, Casey Bodley)
+* common: make cluster_network work (`pr#27811 <https://github.com/ceph/ceph/pull/27811>`_, Jianpeng Ma)
+* common: messages: MOSDPGCreate2 doesnt assume using namespace std (`pr#28342 <https://github.com/ceph/ceph/pull/28342>`_, Radoslaw Zarzynski)
+* common: messages: remove MNop (`pr#27585 <https://github.com/ceph/ceph/pull/27585>`_, Kefu Chai)
+* common: mgr/test_orchestrator: Add dummy data (`pr#32182 <https://github.com/ceph/ceph/pull/32182>`_, Sebastian Wagner, Volker Theile)
+* common: move gen_rand_alphanumeric() helpers into common (`pr#31567 <https://github.com/ceph/ceph/pull/31567>`_, Casey Bodley)
+* common: move xattr -> os/filestore/os_xattr (`pr#32219 <https://github.com/ceph/ceph/pull/32219>`_, David Disseldorp)
+* common: msg/Message: remove unused local variables (`pr#29155 <https://github.com/ceph/ceph/pull/29155>`_, Kefu Chai)
+* common: msg/msg_types: use inet_ntop(3) to render IP addresses (`pr#26987 <https://github.com/ceph/ceph/pull/26987>`_, Sage Weil)
+* common: no need to include ceph_assert.h (`pr#28255 <https://github.com/ceph/ceph/pull/28255>`_, Kefu Chai)
+* common: octopus (`pr#27009 <https://github.com/ceph/ceph/pull/27009>`_, Sage Weil)
+* common: optimize check_utf8 (`pr#27628 <https://github.com/ceph/ceph/pull/27628>`_, Yibo Cai)
+* common: optimize encode_utf8 (`pr#27807 <https://github.com/ceph/ceph/pull/27807>`_, Yibo Cai)
+* common: OutputDataSocket retakes mutex on error path (`issue#40188 <http://tracker.ceph.com/issues/40188>`_, `pr#28431 <https://github.com/ceph/ceph/pull/28431>`_, Casey Bodley)
+* common: preforker: remove useless code (`pr#31714 <https://github.com/ceph/ceph/pull/31714>`_, Xiubo Li)
+* common: python-common: Add drive selection (`pr#31021 <https://github.com/ceph/ceph/pull/31021>`_, Sebastian Wagner)
+* common: python-common: add py.typed (PEP 561) (`pr#33236 <https://github.com/ceph/ceph/pull/33236>`_, Sebastian Wagner)
+* common: python-common: Add small Readme (`pr#30587 <https://github.com/ceph/ceph/pull/30587>`_, Sebastian Wagner)
+* common: python-common: avoid using setup_requires in setup.py (`pr#31222 <https://github.com/ceph/ceph/pull/31222>`_, Sebastian Wagner)
+* common: python-common: enable lint in tox tests (`pr#31068 <https://github.com/ceph/ceph/pull/31068>`_, Kiefer Chang)
+* common: python-common: Fix typo in device type (`pr#31758 <https://github.com/ceph/ceph/pull/31758>`_, Volker Theile)
+* common: python-common: Make Drive Group filter by AND, instead of OR (`pr#33625 <https://github.com/ceph/ceph/pull/33625>`_, Sage Weil, Sebastian Wagner)
+* common: python-common: Make DriveGroupSpec a sub type of ServiceSpec (`pr#33817 <https://github.com/ceph/ceph/pull/33817>`_, Sebastian Wagner)
+* common: random: added a deduction guide to make using the function ob\xe2\x80\xa6 (`pr#30224 <https://github.com/ceph/ceph/pull/30224>`_, Jesse Williamson)
+* common: remove dead code in {safe,mutable}_item_history (`pr#32698 <https://github.com/ceph/ceph/pull/32698>`_, Radoslaw Zarzynski)
+* common: remove unused _STR and STRINGIFY macro (`pr#29605 <https://github.com/ceph/ceph/pull/29605>`_, Yao Zongyou)
+* common: rename image to container_image (`pr#30800 <https://github.com/ceph/ceph/pull/30800>`_, Sage Weil)
+* common: Revert Merge pull request #33673 from cbodley/wip-denc-enum (`pr#33832 <https://github.com/ceph/ceph/pull/33832>`_, Sage Weil)
+* common: selinux: Allow ceph to setsched (`pr#33404 <https://github.com/ceph/ceph/pull/33404>`_, Brad Hubbard)
+* common: skip interfaces starting with lo in find_ipv{4,6}_in_subnet() (`pr#32420 <https://github.com/ceph/ceph/pull/32420>`_, Jiawei Li)
+* common: sort best-matched commond by req argument count (`issue#40292 <http://tracker.ceph.com/issues/40292>`_, `pr#28510 <https://github.com/ceph/ceph/pull/28510>`_, Chang Liu)
+* common: src/: remove execute permissions on nine source files (`pr#28781 <https://github.com/ceph/ceph/pull/28781>`_, J. Eric Ivancich)
+* common: start logging for non-global_init users (`pr#27352 <https://github.com/ceph/ceph/pull/27352>`_, Sage Weil)
+* common: systemd: Wait 5 seconds before attempting a restart of an OSD (`pr#31550 <https://github.com/ceph/ceph/pull/31550>`_, Wido den Hollander)
+* common: use of malloc.h is deprecated (`pr#29397 <https://github.com/ceph/ceph/pull/29397>`_, Willem Jan Withagen)
+* common: zstd: upgrade to v1.4.0 (`pr#28656 <https://github.com/ceph/ceph/pull/28656>`_, Dan van der Ster)
+* core,mgr,tools: osd,tools: Balancer fixes without all of the calc_pg_upmaps() rewrites (`pr#31774 <https://github.com/ceph/ceph/pull/31774>`_, David Zafman)
+* core,mgr: mgr/ActivePyModules: drop GIL to register/unregister clients (`pr#33464 <https://github.com/ceph/ceph/pull/33464>`_, Sage Weil)
+* core,mgr: mgr/alerts: simple module to send health alerts (`pr#30738 <https://github.com/ceph/ceph/pull/30738>`_, Sage Weil)
+* core,mgr: mgr/DaemonServer: warn when we reject reports (`pr#31471 <https://github.com/ceph/ceph/pull/31471>`_, Sage Weil)
+* core,mgr: mgr/pg_autoscaler: add pg_autoscale_bias pool property and apply it to pg_num selection (`pr#27154 <https://github.com/ceph/ceph/pull/27154>`_, Sage Weil)
+* core,mgr: mgr/prometheus: report per-pool pg states (`pr#32370 <https://github.com/ceph/ceph/pull/32370>`_, Aleksei Zakharov)
+* core,mgr: mgr/telemetry: add report_timestamp to sent reports (`pr#27571 <https://github.com/ceph/ceph/pull/27571>`_, Dan Mick)
+* core,mgr: mgr/telemetry: catch exception during requests.put (`pr#33070 <https://github.com/ceph/ceph/pull/33070>`_, Sage Weil)
+* core,mgr: mgr/telemetry: obscure entity_name with a salt (`pr#29330 <https://github.com/ceph/ceph/pull/29330>`_, Sage Weil)
+* core,mgr: osd,mon,mgr: report /dev/disk/by-path paths for devices (`pr#32261 <https://github.com/ceph/ceph/pull/32261>`_, Sage Weil)
+* core,mon: mon,osd: use get_req<> instead of static_cast<>(get_req()) (`pr#30023 <https://github.com/ceph/ceph/pull/30023>`_, Kefu Chai)
+* core,mon: mon/AuthMonitor: fix initial creation of rotating keys (`issue#40634 <http://tracker.ceph.com/issues/40634>`_, `pr#28850 <https://github.com/ceph/ceph/pull/28850>`_, Sage Weil)
+* core,mon: mon/MonClient: add proper SRV priority support (`pr#27126 <https://github.com/ceph/ceph/pull/27126>`_, Kefu Chai)
+* core,mon: mon/Monitor.cc: fix condition that checks for unrecognized auth mode (`pr#30015 <https://github.com/ceph/ceph/pull/30015>`_, Neha Ojha)
+* core,mon: mon/Monitor.cc: print min_mon_release correctly (`pr#27107 <https://github.com/ceph/ceph/pull/27107>`_, Neha Ojha)
+* core,mon: mon/OSDMonitor: clean up removed_snap keys (`pr#30518 <https://github.com/ceph/ceph/pull/30518>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: expand iec_options for osd pool set (`pr#31196 <https://github.com/ceph/ceph/pull/31196>`_, Sage Weil)
+* core,mon: mon/OSDMonitor: Use generic priority cache tuner for mon caches (`issue#40870 <http://tracker.ceph.com/issues/40870>`_, `pr#28227 <https://github.com/ceph/ceph/pull/28227>`_, Sridhar Seshasayee)
+* core,pybind: pybind/ceph_argparse: avoid int overflow (`pr#33101 <https://github.com/ceph/ceph/pull/33101>`_, Kefu Chai)
+* core,pybind: pybind/rados: fix set_omap() crash on py3 (`pr#29096 <https://github.com/ceph/ceph/pull/29096>`_, Sage Weil)
+* core,pybind: pybind/rados: fixed Python3 string conversion issue on get_fsid (`issue#38381 <http://tracker.ceph.com/issues/38381>`_, `pr#26514 <https://github.com/ceph/ceph/pull/26514>`_, Jason Dillaman)
+* core,rbd: common/config: use string_view for keys (`pr#27097 <https://github.com/ceph/ceph/pull/27097>`_, Kefu Chai)
+* core,rbd: osd/OSDCap: rbd profile permits use of rbd_info (`issue#39973 <http://tracker.ceph.com/issues/39973>`_, `pr#28253 <https://github.com/ceph/ceph/pull/28253>`_, songweibin)
+* core,rbd: osd/PrimaryLogPG: do not append outdata to TMAPUP ops (`pr#30457 <https://github.com/ceph/ceph/pull/30457>`_, Jason Dillaman)
+* core,rgw,tests: librados,test,rgw: cleanups to deprecate safe_cb related functions (`pr#31045 <https://github.com/ceph/ceph/pull/31045>`_, Kefu Chai)
+* core,tests: ceph_test_cls_hello: set RETURNVEC on the expected EINVAL request (`pr#33708 <https://github.com/ceph/ceph/pull/33708>`_, Sage Weil)
+* core,tests: ceph_test_rados_api\\_{watch_notify,misc}: tolerate some timeouts (`pr#34011 <https://github.com/ceph/ceph/pull/34011>`_, Sage Weil)
+* core,tests: Improvements to standalone tests (`pr#27279 <https://github.com/ceph/ceph/pull/27279>`_, David Zafman)
+* core,tests: kv_store_bench: fix teuthology_tests() return value (`pr#30293 <https://github.com/ceph/ceph/pull/30293>`_, luo rixin)
+* core,tests: mon.test: improve validation and add a test for osd pool create (`pr#30538 <https://github.com/ceph/ceph/pull/30538>`_, Kefu Chai)
+* core,tests: qa/objectstore: test with reduced value of osd_memory_target (`pr#27083 <https://github.com/ceph/ceph/pull/27083>`_, Neha Ojha)
+* core,tests: qa/standalone/ceph-helpers: more osd debug (`issue#40666 <http://tracker.ceph.com/issues/40666>`_, `pr#28867 <https://github.com/ceph/ceph/pull/28867>`_, Sage Weil)
+* core,tests: qa/standalone/misc/ok-to-stop: improve test (`pr#32738 <https://github.com/ceph/ceph/pull/32738>`_, Sage Weil)
+* core,tests: qa/standalone/mon/health-mute.sh: misc fixes (`pr#29744 <https://github.com/ceph/ceph/pull/29744>`_, Sage Weil)
+* core,tests: qa/standalone/osd/osd-backfill-recovery-log.sh: fix TEST_backfill_log\\_[1, 2] (`pr#32851 <https://github.com/ceph/ceph/pull/32851>`_, Neha Ojha)
+* core,tests: qa/standalone/scrub/osd-scrub-snaps: snapmapper omap is now m (`pr#29774 <https://github.com/ceph/ceph/pull/29774>`_, Sage Weil)
+* core,tests: qa/standalone/scrub/osd-scrub-test: wait longer for update (`pr#33809 <https://github.com/ceph/ceph/pull/33809>`_, Sage Weil)
+* core,tests: qa/suites/rados/multimon: whitelist SLOW_OPS while thrashing mons (`pr#29121 <https://github.com/ceph/ceph/pull/29121>`_, Sage Weil)
+* core,tests: qa/suites/rados/perf: run on ubuntu (`pr#32355 <https://github.com/ceph/ceph/pull/32355>`_, Sage Weil)
+* core,tests: qa/suites/rados/rest: run restful test on el8 (`pr#32920 <https://github.com/ceph/ceph/pull/32920>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton-bluestore/cephtool: whitelist MON_DOWN (`pr#33645 <https://github.com/ceph/ceph/pull/33645>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton/all/lost-unfound\\*: whitelist SLOW_OPS (`pr#32958 <https://github.com/ceph/ceph/pull/32958>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton/all/recovery-preemption: fix pg log length (`pr#32898 <https://github.com/ceph/ceph/pull/32898>`_, Sage Weil)
+* core,tests: qa/suites/rados/singleton/all/thrash-eio: whitelist slow request (`pr#33497 <https://github.com/ceph/ceph/pull/33497>`_, Sage Weil, Sridhar Seshasayee)
+* core,tests: qa/suites/rados/thrash-old-clients: exclude ceph-daemon on nautilus installs (`pr#30817 <https://github.com/ceph/ceph/pull/30817>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: rejigger v1 vs v2 settings (`pr#27249 <https://github.com/ceph/ceph/pull/27249>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: tolerate MON_DOWN (`pr#30577 <https://github.com/ceph/ceph/pull/30577>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash-old-clients: use cephadm (`pr#32377 <https://github.com/ceph/ceph/pull/32377>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash: force normal pg log length with cache tiering (`issue#38358 <http://tracker.ceph.com/issues/38358>`_, `issue#24320 <http://tracker.ceph.com/issues/24320>`_, `pr#28658 <https://github.com/ceph/ceph/pull/28658>`_, Sage Weil)
+* core,tests: qa/suites/rados/thrash: increase async and partial recovery test coverage (`pr#30699 <https://github.com/ceph/ceph/pull/30699>`_, Neha Ojha)
+* core,tests: qa/suites/rados/valgrind-leaks: independently verify we detect leaks on mon, osd, mgr (`pr#32946 <https://github.com/ceph/ceph/pull/32946>`_, Sage Weil)
+* core,tests: qa/suites/rados/verify/tasks/mon_recovery: whitelist SLOW_OPS (`pr#33644 <https://github.com/ceph/ceph/pull/33644>`_, Sage Weil)
+* core,tests: qa/suites/rados/verify: debug monc = 20 (`pr#32968 <https://github.com/ceph/ceph/pull/32968>`_, Sage Weil)
+* core,tests: qa/suites/rados/verify: debug_ms = 1 (`pr#33871 <https://github.com/ceph/ceph/pull/33871>`_, Sage Weil)
+* core,tests: qa/suites/rados: move cephadm_orchestrator to el8 (`pr#32407 <https://github.com/ceph/ceph/pull/32407>`_, Sage Weil)
+* core,tests: qa/suites/upgrade/mimic-x-singleton: suppress TOO_FEW_PGS warning (`pr#31054 <https://github.com/ceph/ceph/pull/31054>`_, Sage Weil)
+* core,tests: qa/suites/upgrade: fix mimic-x-singleton (`pr#32719 <https://github.com/ceph/ceph/pull/32719>`_, Sage Weil)
+* core,tests: qa/suites/upgrade: misc fixes for octopus (`pr#32750 <https://github.com/ceph/ceph/pull/32750>`_, Sage Weil, Josh Durgin)
+* core,tests: qa/tasks/cbt: run stop-all.sh while shutting down (`pr#31171 <https://github.com/ceph/ceph/pull/31171>`_, Sage Weil)
+* core,tests: qa/tasks/ceph: restart: stop osd, mark down, then start (`pr#30196 <https://github.com/ceph/ceph/pull/30196>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: add --log-early to raw_cluster_cmd (`pr#32989 <https://github.com/ceph/ceph/pull/32989>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: enable ceph-objectstore-tool via cephadm (`pr#32411 <https://github.com/ceph/ceph/pull/32411>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: fix ceph-objectstore-tool incantations (`pr#32701 <https://github.com/ceph/ceph/pull/32701>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: fix chmod on log dir during pg export copy (`pr#32943 <https://github.com/ceph/ceph/pull/32943>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: fix post-osd-kill pg peered check (`pr#32737 <https://github.com/ceph/ceph/pull/32737>`_, Sage Weil)
+* core,tests: qa/tasks/ceph_manager: make is\\_{clean,recovered,active_or_down} less racy (`pr#28969 <https://github.com/ceph/ceph/pull/28969>`_, Sage Weil)
+* core,tests: qa/tasks/mon_thrash: sync force requires some force flags (`pr#30361 <https://github.com/ceph/ceph/pull/30361>`_, Sage Weil)
+* core,tests: qa/tasks/radosbench: fix usage of -O (`pr#33744 <https://github.com/ceph/ceph/pull/33744>`_, Sage Weil)
+* core,tests: qa/tasks/thrashosds-health: disable osd_max_markdown behavior (`pr#33601 <https://github.com/ceph/ceph/pull/33601>`_, Sage Weil)
+* core,tests: qa/workunits/cephtool/test.sh: delete test_erasure pool (`pr#33188 <https://github.com/ceph/ceph/pull/33188>`_, Sage Weil)
+* core,tests: qa/workunits/rados/test_crash.sh: suppress core files (`pr#32724 <https://github.com/ceph/ceph/pull/32724>`_, Sage Weil)
+* core,tests: qa: add basic omap testing capability (`pr#29120 <https://github.com/ceph/ceph/pull/29120>`_, Neha Ojha)
+* core,tests: remove ceph_test_rados_watch_notify (`pr#34044 <https://github.com/ceph/ceph/pull/34044>`_, Sage Weil)
+* core,tests: test/CMakeLists: disable memstore make check test (`pr#33473 <https://github.com/ceph/ceph/pull/33473>`_, Sage Weil)
+* core,tests: test/librados: dont release handler if set_pg_num failed (`pr#32112 <https://github.com/ceph/ceph/pull/32112>`_, huangjun)
+* core,tests: test/osd/safe-to-destroy.sh: fix typo (`pr#27651 <https://github.com/ceph/ceph/pull/27651>`_, Sage Weil)
+* core,tests: test/pybind/test_rados.py: test test_aio_remove (`pr#31003 <https://github.com/ceph/ceph/pull/31003>`_, Zhang Jiao)
+* core,tests: test/unittest_lockdep: do not start extra threads (`pr#32772 <https://github.com/ceph/ceph/pull/32772>`_, Kefu Chai)
+* core,tests: test: Bump sleep time for slower machines (`pr#29494 <https://github.com/ceph/ceph/pull/29494>`_, David Zafman)
+* core,tests: test: Make sure that extra scheduled scrubs dont confuse test (`issue#40078 <http://tracker.ceph.com/issues/40078>`_, `pr#28302 <https://github.com/ceph/ceph/pull/28302>`_, David Zafman)
+* core,tests: tests/osd: fix typo in unittest_osdmap (`pr#29790 <https://github.com/ceph/ceph/pull/29790>`_, huangjun)
+* core,tests: tools/rados: use num ops instead of num objs for tracking outstanding IO (`pr#29734 <https://github.com/ceph/ceph/pull/29734>`_, Albert H Chen)
+* core,tests: unittest_lockdep: avoid any threads for death test (`pr#32765 <https://github.com/ceph/ceph/pull/32765>`_, Sage Weil)
+* core,tools: ceph-objectstore-tool cant remove head with bad snapset (`pr#29919 <https://github.com/ceph/ceph/pull/29919>`_, David Zafman)
+* core,tools: ceph.in: check ceph-conf returncode (`pr#30695 <https://github.com/ceph/ceph/pull/30695>`_, Dimitri Savineau)
+* core,tools: src/tools/ceph-dedup-tool: Fix chunk scru (`pr#28765 <https://github.com/ceph/ceph/pull/28765>`_, Myoungwon Oh)
+* core:  ceph.in: only preload asan library for Debug build (`pr#27190 <https://github.com/ceph/ceph/pull/27190>`_, Kefu Chai)
+* core:  osd/ClassHandler: cleanups (`pr#28363 <https://github.com/ceph/ceph/pull/28363>`_, Kefu Chai)
+* core:  osd: add hdd, ssd and hybrid variants for osd_snap_trim_sleep (`pr#28772 <https://github.com/ceph/ceph/pull/28772>`_, Neha Ojha)
+* core:  osdc/Objecter: use unique_ptr<OSDMap> for Objecter::osdmap (`issue#38403 <http://tracker.ceph.com/issues/38403>`_, `pr#28397 <https://github.com/ceph/ceph/pull/28397>`_, Kefu Chai)
+* core: Add structures for tracking in progress operations (`pr#28395 <https://github.com/ceph/ceph/pull/28395>`_, Samuel Just)
+* core: auth: treat mgr the same as mon when selecting auth mode (`pr#33226 <https://github.com/ceph/ceph/pull/33226>`_, Yehuda Sadeh)
+* core: backfill_toofull seen on cluster where the most full OSD is at 1% (`pr#29857 <https://github.com/ceph/ceph/pull/29857>`_, David Zafman)
+* core: ceph,pybind/mgr: a few py3 fixes (`pr#32187 <https://github.com/ceph/ceph/pull/32187>`_, Sage Weil)
+* core: ceph-objectstore-tool: better error message if pgid and object do not match (`pr#30501 <https://github.com/ceph/ceph/pull/30501>`_, Sage Weil)
+* core: ceph.in: Fix name retval is not defined error (`pr#33516 <https://github.com/ceph/ceph/pull/33516>`_, Varsha Rao)
+* core: ceph.in: improve control-c handling (`pr#33352 <https://github.com/ceph/ceph/pull/33352>`_, Sage Weil)
+* core: ceph.in: only shut down rados on clean exit (`pr#33825 <https://github.com/ceph/ceph/pull/33825>`_, Sage Weil)
+* core: client: fix FTBFS due to bl::iterator::advance() (`pr#33085 <https://github.com/ceph/ceph/pull/33085>`_, Radoslaw Zarzynski)
+* core: cls_hello: fix typo (`pr#32976 <https://github.com/ceph/ceph/pull/32976>`_, Sage Weil)
+* core: common,mon,osd: unify ceph tell and ceph daemon command sets (`pr#30217 <https://github.com/ceph/ceph/pull/30217>`_, Sage Weil)
+* core: common,tools,crush,test: misc converity & klocwork fixes (`pr#29316 <https://github.com/ceph/ceph/pull/29316>`_, songweibin)
+* core: common/admin_socket: Increase socket timeouts (`pr#31623 <https://github.com/ceph/ceph/pull/31623>`_, Brad Hubbard)
+* core: common/assert: include ceph_abort_msg(arg) arg in log output (`pr#27732 <https://github.com/ceph/ceph/pull/27732>`_, Sage Weil)
+* core: common/blkdev: fix some problems with smart scraping (`pr#28848 <https://github.com/ceph/ceph/pull/28848>`_, Sage Weil)
+* core: common/blkdev: get_device_id: behave if model is lvm and id_model_enc isnt there (`pr#27156 <https://github.com/ceph/ceph/pull/27156>`_, Sage Weil)
+* core: common/blkdev: handle devices with ID_MODEL as LVM PV ... but valid ID_MODEL_ENC (`pr#27020 <https://github.com/ceph/ceph/pull/27020>`_, Sage Weil)
+* core: common/condition_variable_debug: do not assert() if sloppy (`pr#29854 <https://github.com/ceph/ceph/pull/29854>`_, Kefu Chai)
+* core: common/config: behave when both POD_MEMORY_REQUEST and POD_MEMORY_LIMIT are set (`pr#29511 <https://github.com/ceph/ceph/pull/29511>`_, Sage Weil)
+* core: common/config: less noise about configs from mon we cant apply (`pr#31988 <https://github.com/ceph/ceph/pull/31988>`_, Sage Weil)
+* core: common/config: parse --default-$option as a default value (`pr#27169 <https://github.com/ceph/ceph/pull/27169>`_, Sage Weil)
+* core: common/config: update values when they are removed via mon (`pr#32091 <https://github.com/ceph/ceph/pull/32091>`_, Sage Weil)
+* core: common/kv/rocksdb: Fixed async compations (`pr#26786 <https://github.com/ceph/ceph/pull/26786>`_, Adam Kupczyk)
+* core: common/options.cc: Lower the default value of osd_deep_scrub_large_omap_object_key_threshold (`pr#28782 <https://github.com/ceph/ceph/pull/28782>`_, Neha Ojha)
+* core: common/options.cc: make rocksdb_delete_range_threshold very high (`pr#33439 <https://github.com/ceph/ceph/pull/33439>`_, Neha Ojha)
+* core: common/options: allow osd_pool_default_pg_autoscale_mode to update a runtime (`pr#27821 <https://github.com/ceph/ceph/pull/27821>`_, Sage Weil)
+* core: common/options: annotate some options; enable some runtime updates (`pr#27655 <https://github.com/ceph/ceph/pull/27655>`_, Sage Weil)
+* core: common/options: decrease the default max_omap_entries_per_request (`pr#31506 <https://github.com/ceph/ceph/pull/31506>`_, Yan Jun)
+* core: common/options: make secure mode non-experimental, and prefer/require it for mons (`pr#27012 <https://github.com/ceph/ceph/pull/27012>`_, Sage Weil)
+* core: common/options: update mon_crush_min_required_version=hammer (`pr#27568 <https://github.com/ceph/ceph/pull/27568>`_, Sage Weil)
+* core: common/PriorityCache: fix over-aggressive assert when mem limited (`pr#27763 <https://github.com/ceph/ceph/pull/27763>`_, Mark Nelson)
+* core: common/PriorityCache: Implement a Cache Manager (`pr#27381 <https://github.com/ceph/ceph/pull/27381>`_, Mark Nelson)
+* core: common/TextTable,mgr: standardize on 2 spaces between table columns (`pr#33138 <https://github.com/ceph/ceph/pull/33138>`_, Sage Weil)
+* core: common/util: handle long lines in /proc/cpuinfo (`issue#38296 <http://tracker.ceph.com/issues/38296>`_, `pr#27707 <https://github.com/ceph/ceph/pull/27707>`_, Sage Weil)
+* core: compressor/lz4: work around bug in liblz4 versions <1.8.2 (`pr#33584 <https://github.com/ceph/ceph/pull/33584>`_, Sage Weil, Dan van der Ster)
+* core: crimson, osd: add support for Ceph Classes, part 1 (`pr#29651 <https://github.com/ceph/ceph/pull/29651>`_, Radoslaw Zarzynski)
+* core: crimson/osd: add osd to crush when it boots (`pr#28689 <https://github.com/ceph/ceph/pull/28689>`_, Kefu Chai)
+* core: crush/CrushCompiler: Fix __replacement_assert (`issue#39174 <http://tracker.ceph.com/issues/39174>`_, `pr#27506 <https://github.com/ceph/ceph/pull/27506>`_, Brad Hubbard)
+* core: crush/CrushWrapper.cc: Fix sign compare compiler warning (`pr#31184 <https://github.com/ceph/ceph/pull/31184>`_, Prashant D)
+* core: crush/CrushWrapper: behave with empty weight vector (`pr#32673 <https://github.com/ceph/ceph/pull/32673>`_, Kefu Chai)
+* core: dencoder: include some missed types (`pr#27804 <https://github.com/ceph/ceph/pull/27804>`_, Greg Farnum)
+* core: dmclock server side refactor (`pr#30650 <https://github.com/ceph/ceph/pull/30650>`_, Samuel Just)
+* core: examples/librados: fix bufferlist::copy() in hello_world.cc (`pr#33075 <https://github.com/ceph/ceph/pull/33075>`_, Radoslaw Zarzynski)
+* core: Extract peering logic into a module for use in crimson (`pr#27874 <https://github.com/ceph/ceph/pull/27874>`_, Samuel Just, sjust@redhat.com)
+* core: feature: Health warnings on long network ping times, add dump_osd_network to get a report (`issue#40640 <http://tracker.ceph.com/issues/40640>`_, `pr#28755 <https://github.com/ceph/ceph/pull/28755>`_, David Zafman)
+* core: Feature: Improvements to auto repair (`issue#38616 <http://tracker.ceph.com/issues/38616>`_, `pr#26942 <https://github.com/ceph/ceph/pull/26942>`_, David Zafman)
+* core: global: ensure CEPH_ARGS is decoded before early arg processing (`pr#32830 <https://github.com/ceph/ceph/pull/32830>`_, Jason Dillaman)
+* core: global: explicitly call out EIO events in crash dumps (`pr#27386 <https://github.com/ceph/ceph/pull/27386>`_, Sage Weil)
+* core: include,os: Make ceph_le member private (`pr#30526 <https://github.com/ceph/ceph/pull/30526>`_, Ulrich Weigand)
+* core: include/ceph_features: fix typo (`pr#27353 <https://github.com/ceph/ceph/pull/27353>`_, Sage Weil)
+* core: include/cpp-btree: cleanups (`pr#32443 <https://github.com/ceph/ceph/pull/32443>`_, Kefu Chai)
+* core: init-ceph: wait longer before resending $signal (`pr#27308 <https://github.com/ceph/ceph/pull/27308>`_, Kefu Chai)
+* core: kv/KeyValueDB: fix estimate_prefix_size() (`pr#29842 <https://github.com/ceph/ceph/pull/29842>`_, Adam Kupczyk)
+* core: kv/RocksDBStore: Add minimum key limit before invoking DeleteRange (`pr#31442 <https://github.com/ceph/ceph/pull/31442>`_, Mark Nelson)
+* core: kv/RocksDBStore: make option: compaction_threads/disableWAL/flusher_t\xe2\x80\xa6 (`pr#32453 <https://github.com/ceph/ceph/pull/32453>`_, Jianpeng Ma)
+* core: kv/RocksDBStore: tell rocksdb to set mode to 0600, not 0644 (`pr#30679 <https://github.com/ceph/ceph/pull/30679>`_, Sage Weil)
+* core: kv: fix shutdown vs async compaction (`pr#32619 <https://github.com/ceph/ceph/pull/32619>`_, Sage Weil)
+* core: kv: make delete range optional on number of keys (`pr#27317 <https://github.com/ceph/ceph/pull/27317>`_, Zengran Zhang)
+* core: librados,osd,mon: remove traces of CEPH_OSDMAP_FULL (`pr#30614 <https://github.com/ceph/ceph/pull/30614>`_, Kefu Chai)
+* core: Make dumping of reservation info congruent between scrub and recovery (`pr#30192 <https://github.com/ceph/ceph/pull/30192>`_, David Zafman)
+* core: messages,osd: remove MPGStats::had_map_for (`pr#27026 <https://github.com/ceph/ceph/pull/27026>`_, Kefu Chai)
+* core: messages: #include necessary header (`pr#27590 <https://github.com/ceph/ceph/pull/27590>`_, Kefu Chai)
+* core: mgr/balancer: sort pool names in balancer ls output (`pr#32424 <https://github.com/ceph/ceph/pull/32424>`_, Sage Weil)
+* core: mgr/balancer: tolerate pgs outside of target weight map (`pr#34014 <https://github.com/ceph/ceph/pull/34014>`_, Sage Weil)
+* core: mgr/cephadm: health alert for stray services or hosts (`pr#32754 <https://github.com/ceph/ceph/pull/32754>`_, Sage Weil)
+* core: mgr/crash: behave when posted crash has no backtrace (`pr#31643 <https://github.com/ceph/ceph/pull/31643>`_, Sage Weil)
+* core: mgr/crash: raise warning about recent crashes and other improvements (`pr#29034 <https://github.com/ceph/ceph/pull/29034>`_, Sage Weil)
+* core: mgr/DaemonServer: fix osd ok-to-stop for EC pools (`pr#32046 <https://github.com/ceph/ceph/pull/32046>`_, Sage Weil)
+* core: mgr/DaemonServer: fix pg merge checks (`pr#34067 <https://github.com/ceph/ceph/pull/34067>`_, Sage Weil)
+* core: mgr/DaemonServer: prevent pgp_num reductions from outpacing pg_num merges (`issue#38786 <http://tracker.ceph.com/issues/38786>`_, `pr#27473 <https://github.com/ceph/ceph/pull/27473>`_, Sage Weil)
+* core: mgr/devicehealth: fix telemetry stops sending device reports after 48\xe2\x80\xa6 (`pr#32903 <https://github.com/ceph/ceph/pull/32903>`_, Yaarit Hatuka)
+* core: mgr/diskprediction_cloud: Service unavailable (`issue#40478 <http://tracker.ceph.com/issues/40478>`_, `pr#28687 <https://github.com/ceph/ceph/pull/28687>`_, Rick Chen)
+* core: mgr/diskprediction_local: import scipy early to fix self-test deadlock (`pr#32102 <https://github.com/ceph/ceph/pull/32102>`_, Sage Weil)
+* core: mgr/diskprediction_local: some debug output during predict (and self-test) (`pr#31572 <https://github.com/ceph/ceph/pull/31572>`_, Sage Weil)
+* core: mgr/MgrClient: fix open condition (`pr#31256 <https://github.com/ceph/ceph/pull/31256>`_, Sage Weil)
+* core: mgr/MgrClient: fix open condition fix (`pr#31422 <https://github.com/ceph/ceph/pull/31422>`_, Sage Weil)
+* core: mgr/MgrClient: fix tell mgr.x ... (`pr#31989 <https://github.com/ceph/ceph/pull/31989>`_, Sage Weil)
+* core: mgr/pg_autoscaler: complete event if pool disappears (`pr#30819 <https://github.com/ceph/ceph/pull/30819>`_, Sage Weil)
+* core: mgr/pg_autoscaler: default to pg_num[_min] = 16 (`pr#31636 <https://github.com/ceph/ceph/pull/31636>`_, Sage Weil)
+* core: mgr/pg_autoscaler: default to pg_num[_min] = 32 (`pr#32788 <https://github.com/ceph/ceph/pull/32788>`_, Neha Ojha)
+* core: mgr/pg_autoscaler: fix division by zero (`pr#33402 <https://github.com/ceph/ceph/pull/33402>`_, Sage Weil)
+* core: mgr/pg_autoscaler: only generate target\\_\\* health warnings if targets set (`pr#31638 <https://github.com/ceph/ceph/pull/31638>`_, Sage Weil)
+* core: mgr/progress: behave if pgs disappear (due to a racing pg merge) (`issue#38157 <http://tracker.ceph.com/issues/38157>`_, `pr#27546 <https://github.com/ceph/ceph/pull/27546>`_, Sage Weil)
+* core: mgr/progress: fix duration strings (`pr#34045 <https://github.com/ceph/ceph/pull/34045>`_, Sage Weil)
+* core: mgr/progress: progress clear command should clear events in ceph -s (`pr#33400 <https://github.com/ceph/ceph/pull/33400>`_, Sage Weil)
+* core: mgr/telemetry: add some more telemetry (`pr#31226 <https://github.com/ceph/ceph/pull/31226>`_, Sage Weil)
+* core: mgr/telemetry: include pg_autoscaler and balancer status (`pr#30871 <https://github.com/ceph/ceph/pull/30871>`_, Sage Weil)
+* core: mgr/telemetry: send device telemetry via per-host POST to device endpoint (`pr#31225 <https://github.com/ceph/ceph/pull/31225>`_, Sage Weil)
+* core: mgr/telemetry: split entity_name only once (handle ids with dots) (`pr#33094 <https://github.com/ceph/ceph/pull/33094>`_, Dan Mick)
+* core: Miscellaneous lost fixes (`pr#27599 <https://github.com/ceph/ceph/pull/27599>`_, Xinze Chi, Greg Farnum, linbing, shangfufei)
+* core: mon, osd: parallel clean_pg_upmaps (`issue#40104 <http://tracker.ceph.com/issues/40104>`_, `pr#28373 <https://github.com/ceph/ceph/pull/28373>`_, xie xingguo)
+* core: mon,msg/async: fix mon to mon authentication (`pr#27823 <https://github.com/ceph/ceph/pull/27823>`_, Sage Weil)
+* core: mon,osd: add dead_epoch, --dead flag to osd down (`pr#29221 <https://github.com/ceph/ceph/pull/29221>`_, Sage Weil)
+* core: mon,osd: add no{out,down,in,out} flags on CRUSH nodes (`pr#27563 <https://github.com/ceph/ceph/pull/27563>`_, Sage Weil)
+* core: mon,osd: deprecate forward and readforward cache modes (`pr#28944 <https://github.com/ceph/ceph/pull/28944>`_, Sage Weil)
+* core: mon,osd: track history and past_intervals for creating pgs (`pr#27696 <https://github.com/ceph/ceph/pull/27696>`_, Sage Weil)
+* core: mon,osd: various octopus feature bits (`pr#27141 <https://github.com/ceph/ceph/pull/27141>`_, Sage Weil)
+* core: mon/ConfigMap: search nested sections (`pr#31327 <https://github.com/ceph/ceph/pull/31327>`_, Sage Weil)
+* core: mon/ConfigMonitor: fix handling of NO_MON_UPDATE settings (`pr#32726 <https://github.com/ceph/ceph/pull/32726>`_, Sage Weil)
+* core: mon/ConfigMonitor: only propose if leader (`pr#32975 <https://github.com/ceph/ceph/pull/32975>`_, Sage Weil)
+* core: mon/ConfigMonitor: prefix all global config options with global/ (`pr#32786 <https://github.com/ceph/ceph/pull/32786>`_, Sage Weil)
+* core: mon/LogMonitor: add mon_cluster_log_to_file bool option (`pr#27343 <https://github.com/ceph/ceph/pull/27343>`_, Sage Weil)
+* core: mon/MgrMonitor: fix null deref when invalid formatter is specified (`pr#29089 <https://github.com/ceph/ceph/pull/29089>`_, Sage Weil)
+* core: mon/MgrMonitor: make mgr fail work with no arguments (`pr#33997 <https://github.com/ceph/ceph/pull/33997>`_, Sage Weil)
+* core: mon/MgrStatMonitor: ensure only one copy of initial service map (`issue#38839 <http://tracker.ceph.com/issues/38839>`_, `pr#27101 <https://github.com/ceph/ceph/pull/27101>`_, Sage Weil)
+* core: mon/MonClient: do not dereference auth_supported.end() (`pr#27196 <https://github.com/ceph/ceph/pull/27196>`_, Kefu Chai)
+* core: mon/MonClient: ENXIO when sending command to down mon (`pr#29090 <https://github.com/ceph/ceph/pull/29090>`_, Sage Weil, Greg Farnum)
+* core: mon/MonClient: send logs to mon on separate schedule than pings (`pr#33732 <https://github.com/ceph/ceph/pull/33732>`_, Sage Weil)
+* core: mon/MonClient: skip CEPHX_V2 challenge if client doesnt support it (`pr#30523 <https://github.com/ceph/ceph/pull/30523>`_, Sage Weil)
+* core: mon/Monitor: fail forwarded tell commands (`pr#33542 <https://github.com/ceph/ceph/pull/33542>`_, Sage Weil)
+* core: mon/MonMap: encode (more) valid compat monmap when we have v2-only addrs (`pr#31472 <https://github.com/ceph/ceph/pull/31472>`_, Sage Weil)
+* core: mon/MonmapMonitor: clean up empty created stamp in monmap (`issue#39085 <http://tracker.ceph.com/issues/39085>`_, `pr#27327 <https://github.com/ceph/ceph/pull/27327>`_, Sage Weil)
+* core: mon/OSDMonitor.cc: Add current numbers of objects and bytes (`pr#18694 <https://github.com/ceph/ceph/pull/18694>`_, Shinobu Kinjo)
+* core: mon/OSDMonitor.cc: better error message about min_size (`pr#29184 <https://github.com/ceph/ceph/pull/29184>`_, Neha Ojha)
+* core: mon/OSDMonitor: accept autoscale_mode argument to osd pool create (`pr#33092 <https://github.com/ceph/ceph/pull/33092>`_, Sage Weil)
+* core: mon/OSDMonitor: add check for crush rule size in pool set size command (`pr#30723 <https://github.com/ceph/ceph/pull/30723>`_, Vikhyat Umrao)
+* core: mon/OSDMonitor: allow osd pool set pgp_num_actual (`pr#27010 <https://github.com/ceph/ceph/pull/27010>`_, Sage Weil)
+* core: mon/OSDMonitor: allow pg_num to increase when require_osd_release < N (`issue#39570 <http://tracker.ceph.com/issues/39570>`_, `pr#27928 <https://github.com/ceph/ceph/pull/27928>`_, Sage Weil)
+* core: mon/OSDMonitor: Dont update mon cache settings if rocksdb is not used (`pr#32473 <https://github.com/ceph/ceph/pull/32473>`_, Sridhar Seshasayee)
+* core: mon/OSDMonitor: fix format error ceph osd stat --format json (`pr#31399 <https://github.com/ceph/ceph/pull/31399>`_, Zheng Yin)
+* core: mon/OSDMonitor: make memory autotune disable itself if no rocksd (`pr#32044 <https://github.com/ceph/ceph/pull/32044>`_, Sage Weil)
+* core: mon/OSDMonitor: tolerate duplicate MRemoveSnaps messages (`issue#40774 <http://tracker.ceph.com/issues/40774>`_, `pr#29051 <https://github.com/ceph/ceph/pull/29051>`_, Sage Weil)
+* core: mon/PGMap.h: disable network stats in dump_osd_stats (`pr#32406 <https://github.com/ceph/ceph/pull/32406>`_, Neha Ojha, David Zafman)
+* core: mon/PGMap: drop indentation on df human output (`pr#30848 <https://github.com/ceph/ceph/pull/30848>`_, Sage Weil)
+* core: mon/PGMap: fix summary display of >32bit pg states (`pr#33137 <https://github.com/ceph/ceph/pull/33137>`_, Sage Weil)
+* core: mon/PGMap: use NONE for pg ls[-\\*] output too (`pr#32048 <https://github.com/ceph/ceph/pull/32048>`_, Sage Weil)
+* core: mon/Session: only index osd ids >= 0 (`pr#32764 <https://github.com/ceph/ceph/pull/32764>`_, Sage Weil)
+* core: More PeeringState and related cleanups to ease use in crimson (`pr#28048 <https://github.com/ceph/ceph/pull/28048>`_, Samuel Just)
+* core: msg,auth: migrate msg/async V1 implementation to new Auth{Server,Client} interfaces (`pr#27566 <https://github.com/ceph/ceph/pull/27566>`_, Sage Weil)
+* core: msg/async/frames_v2.h: fix warning (`pr#27464 <https://github.com/ceph/ceph/pull/27464>`_, Sage Weil)
+* core: msg/async/ProtocolV2: fix typo in register_lossy_clients fix (`pr#33559 <https://github.com/ceph/ceph/pull/33559>`_, Sage Weil)
+* core: msg/async/ProtocolV[12]: add ms_learn_addr_from_peer (`pr#27341 <https://github.com/ceph/ceph/pull/27341>`_, Sage Weil)
+* core: msg/async: clear_payload when requeue_sent (`pr#30211 <https://github.com/ceph/ceph/pull/30211>`_, Jianpeng Ma)
+* core: msg/async: optimizations (`pr#26531 <https://github.com/ceph/ceph/pull/26531>`_, Jianpeng Ma)
+* core: msg/auth: handle decode errors instead of throwing exceptions (`pr#31052 <https://github.com/ceph/ceph/pull/31052>`_, Sage Weil)
+* core: msg/DispatchQueue: Set throttle stamp for local_delivery (`pr#31137 <https://github.com/ceph/ceph/pull/31137>`_, Brad Hubbard)
+* core: msg/Policy: limit unregistered anon connections to mon (`pr#33163 <https://github.com/ceph/ceph/pull/33163>`_, Sage Weil)
+* core: msg/Policy: make stateless_server default to anon (again) (`pr#33633 <https://github.com/ceph/ceph/pull/33633>`_, Sage Weil)
+* core: objclass, osd: clean up the cls-host interface. Turn ClassHandler into singleton (`pr#29322 <https://github.com/ceph/ceph/pull/29322>`_, Radoslaw Zarzynski)
+* core: object_stat_sum_t decode broken if given older version (`issue#39284 <http://tracker.ceph.com/issues/39284>`_, `issue#39281 <http://tracker.ceph.com/issues/39281>`_, `pr#27564 <https://github.com/ceph/ceph/pull/27564>`_, David Zafman)
+* core: os, osd: readv (`pr#30061 <https://github.com/ceph/ceph/pull/30061>`_, xie xingguo)
+* core: os/bluestore: Add config observer for osd memory specific options (`pr#29606 <https://github.com/ceph/ceph/pull/29606>`_, Sridhar Seshasayee)
+* core: os/filestore: assure sufficient leaves in pre-split (`issue#39390 <http://tracker.ceph.com/issues/39390>`_, `pr#27689 <https://github.com/ceph/ceph/pull/27689>`_, Jeegn Chen)
+* core: os/Transaction: dump alloc hint flags in op (`pr#28881 <https://github.com/ceph/ceph/pull/28881>`_, Zengran Zhang)
+* core: os: remove KineticStore (`pr#30653 <https://github.com/ceph/ceph/pull/30653>`_, Kefu Chai)
+* core: osd,crimson: use make_message for creating message (`pr#30412 <https://github.com/ceph/ceph/pull/30412>`_, Kefu Chai)
+* core: osd,messages: changes for preparing for crimson-osd (`pr#27003 <https://github.com/ceph/ceph/pull/27003>`_, Kefu Chai)
+* core: osd,mon: remove pg_pool_t::removed_snaps (`pr#28330 <https://github.com/ceph/ceph/pull/28330>`_, Sage Weil)
+* core: osd/ECTransaction,ReplicatedBackend: create op is new in octopus (`pr#29092 <https://github.com/ceph/ceph/pull/29092>`_, Sage Weil)
+* core: osd/MissingLoc, PeeringState: remove osd from missing loc in purge_strays() (`pr#30119 <https://github.com/ceph/ceph/pull/30119>`_, Neha Ojha)
+* core: osd/MissingLoc.cc: do not rely on missing_loc_sources only (`pr#30226 <https://github.com/ceph/ceph/pull/30226>`_, Neha Ojha)
+* core: osd/OSD.cc: make osd bench description consistent with parameters (`issue#39006 <http://tracker.ceph.com/issues/39006>`_, `pr#27600 <https://github.com/ceph/ceph/pull/27600>`_, Neha Ojha)
+* core: osd/osd: add an err log to set_numa_affinty (`pr#30870 <https://github.com/ceph/ceph/pull/30870>`_, luo rixin)
+* core: osd/OSD: auto mark heartbeat sessions as stale and tear them down (`issue#40586 <http://tracker.ceph.com/issues/40586>`_, `pr#28752 <https://github.com/ceph/ceph/pull/28752>`_, xie xingguo)
+* core: osd/OSD: choose more heartbeat peers from different subtrees (`pr#33037 <https://github.com/ceph/ceph/pull/33037>`_, xie xingguo)
+* core: osd/OSD: enhance osd numa affinity compatibility (`pr#31274 <https://github.com/ceph/ceph/pull/31274>`_, Dai zhiwei)
+* core: osd/OSD: keep synchronizing with mon if stuck at booting (`pr#28404 <https://github.com/ceph/ceph/pull/28404>`_, xie xingguo)
+* core: osd/OSD: Log slow ops/types to cluster logs (`pr#33328 <https://github.com/ceph/ceph/pull/33328>`_, Sridhar Seshasayee)
+* core: osd/OSD: only wake up empty pqueue (`pr#28832 <https://github.com/ceph/ceph/pull/28832>`_, Jianpeng Ma)
+* core: osd/OSD: prevent down osds from immediately rejoining the culster (`pr#33039 <https://github.com/ceph/ceph/pull/33039>`_, xie xingguo)
+* core: osd/osd: Refactor get_iface_numa_node (`pr#31965 <https://github.com/ceph/ceph/pull/31965>`_, Dai zhiwei, luo rixin)
+* core: osd/OSD: remove unused func enqueue_peering_evt_front (`pr#32496 <https://github.com/ceph/ceph/pull/32496>`_, Jianpeng Ma)
+* core: osd/OSD: remove unused parameter osdmap_lock_name (`pr#32514 <https://github.com/ceph/ceph/pull/32514>`_, Jianpeng Ma)
+* core: osd/OSDCap: Check for empty namespace (`issue#40835 <http://tracker.ceph.com/issues/40835>`_, `pr#29146 <https://github.com/ceph/ceph/pull/29146>`_, Brad Hubbard)
+* core: osd/OSDMap.cc: add more info in json output of osd stat (`pr#30344 <https://github.com/ceph/ceph/pull/30344>`_, Shen Hang)
+* core: osd/OSDMap.cc: dont output over/underfull messages to lderr (`pr#31542 <https://github.com/ceph/ceph/pull/31542>`_, Neha Ojha)
+* core: osd/OSDMap: add zone to default crush map (`pr#27070 <https://github.com/ceph/ceph/pull/27070>`_, Sage Weil)
+* core: osd/OSDMap: calc_pg_upmaps - restrict optimization to origin pools only (`issue#38897 <http://tracker.ceph.com/issues/38897>`_, `pr#27142 <https://github.com/ceph/ceph/pull/27142>`_, xie xingguo)
+* core: osd/OSDMap: consider overfull osds only when trying to do upmap (`pr#32368 <https://github.com/ceph/ceph/pull/32368>`_, xie xingguo)
+* core: osd/OSDMap: do not trust partially simplified pg_upmap_item (`pr#30576 <https://github.com/ceph/ceph/pull/30576>`_, xie xingguo)
+* core: osd/OSDMap: fix calc_pg_role (`pr#32132 <https://github.com/ceph/ceph/pull/32132>`_, Sage Weil)
+* core: osd/OSDMap: health alert for non-power-of-two pg_num (`pr#30525 <https://github.com/ceph/ceph/pull/30525>`_, Sage Weil)
+* core: osd/OSDMap: Replace get_out_osds with get_out_existing_osds (`issue#39154 <http://tracker.ceph.com/issues/39154>`_, `pr#27663 <https://github.com/ceph/ceph/pull/27663>`_, Brad Hubbard)
+* core: osd/OSDMap: Show health warning if a pool is configured with size 1 (`pr#31416 <https://github.com/ceph/ceph/pull/31416>`_, Sridhar Seshasayee)
+* core: osd/OSDMap: stop encoding osd_state with >8 bits wide states only for old client (`pr#33814 <https://github.com/ceph/ceph/pull/33814>`_, xie xingguo)
+* core: osd/osd_types: bump up some encoding versions (`pr#29923 <https://github.com/ceph/ceph/pull/29923>`_, xie xingguo)
+* core: osd/osd_types: drop last_backfill_bitwise member (`pr#28766 <https://github.com/ceph/ceph/pull/28766>`_, Sage Weil)
+* core: osd/osd_types: fix {omap,hitset_bytes}_stats_invalid handling on split/merge (`pr#30479 <https://github.com/ceph/ceph/pull/30479>`_, Sage Weil)
+* core: osd/osd_types: inc-recovery - add special handler for lost_revert (`pr#29893 <https://github.com/ceph/ceph/pull/29893>`_, xie xingguo)
+* core: osd/osd_types: pool_stat_t::dump - fix num_store_stats field (`issue#39340 <http://tracker.ceph.com/issues/39340>`_, `pr#27633 <https://github.com/ceph/ceph/pull/27633>`_, xie xingguo)
+* core: osd/PeeringState.cc: dont let num_objects become negative (`pr#32305 <https://github.com/ceph/ceph/pull/32305>`_, Neha Ojha)
+* core: osd/PeeringState.cc: skip peer_purged when discovering all missing (`pr#32195 <https://github.com/ceph/ceph/pull/32195>`_, Neha Ojha)
+* core: osd/PeeringState.h: Fix pg stuck in WaitActingChange (`pr#29669 <https://github.com/ceph/ceph/pull/29669>`_, chen qiuzhang)
+* core: osd/PeeringState.h: get_num_missing() should report num_missing() (`pr#30414 <https://github.com/ceph/ceph/pull/30414>`_, Neha Ojha)
+* core: osd/PeeringState.h: ignore RemoteBackfillReserved in WaitLocalBackfillReserved (`pr#33525 <https://github.com/ceph/ceph/pull/33525>`_, Neha Ojha)
+* core: osd/PeeringState: base lease support checks on features, not require_osd_release (`pr#30721 <https://github.com/ceph/ceph/pull/30721>`_, Sage Weil)
+* core: osd/PeeringState: clear LAGGY and WAIT states on exiting Started (`pr#31864 <https://github.com/ceph/ceph/pull/31864>`_, Sage Weil)
+* core: osd/PeeringState: disable read lease until require_osd_release >= octopus (`pr#30692 <https://github.com/ceph/ceph/pull/30692>`_, Sage Weil)
+* core: osd/PeeringState: do not complain about past_intervals constrained by oldest epoch (`pr#29747 <https://github.com/ceph/ceph/pull/29747>`_, Sage Weil)
+* core: osd/PeeringState: do not exclude up from acting_recovery_backfill (`pr#31703 <https://github.com/ceph/ceph/pull/31703>`_, xie xingguo)
+* core: osd/PeeringState: do not start renewing leases until PG is activated (`pr#33129 <https://github.com/ceph/ceph/pull/33129>`_, Sage Weil)
+* core: osd/PeeringState: fix wrong history of merge target (`pr#29835 <https://github.com/ceph/ceph/pull/29835>`_, xie xingguo)
+* core: osd/PeeringState: on_new_interval on child PG after split (`pr#29780 <https://github.com/ceph/ceph/pull/29780>`_, Sage Weil)
+* core: osd/PeeringState: recover_got - add special handler for empty log (`pr#30503 <https://github.com/ceph/ceph/pull/30503>`_, xie xingguo)
+* core: osd/PeeringState: require SERVER_OCTOPUS to respond to RenewLease (`pr#33339 <https://github.com/ceph/ceph/pull/33339>`_, Neha Ojha)
+* core: osd/PeeringState: send pg_info2 if release >= octopus (`pr#30836 <https://github.com/ceph/ceph/pull/30836>`_, Kefu Chai)
+* core: osd/PeeringState: transit async_recovery_targets back into acting before backfilling (`pr#32202 <https://github.com/ceph/ceph/pull/32202>`_, xie xingguo)
+* core: osd/PG: Add PG to large omap log message (`pr#30682 <https://github.com/ceph/ceph/pull/30682>`_, Brad Hubbard)
+* core: osd/PG: adjust pg history on fabricated merge target if necessary (`issue#38623 <http://tracker.ceph.com/issues/38623>`_, `pr#26822 <https://github.com/ceph/ceph/pull/26822>`_, Sage Weil)
+* core: osd/PG: clean up fastinfo key when last_update does not increase (`pr#32615 <https://github.com/ceph/ceph/pull/32615>`_, Sage Weil, Kefu Chai)
+* core: osd/PG: discover missing objects when an OSD peers and PG is degraded (`pr#27288 <https://github.com/ceph/ceph/pull/27288>`_, Jonas Jelten)
+* core: osd/PG: do not leak cluster message when theres no con (`pr#32897 <https://github.com/ceph/ceph/pull/32897>`_, Sage Weil)
+* core: osd/PG: do not queue scrub if PG is not active when unblock (`issue#40451 <http://tracker.ceph.com/issues/40451>`_, `pr#28660 <https://github.com/ceph/ceph/pull/28660>`_, Sage Weil)
+* core: osd/PG: do not use approx_missing_objects pre-nautilus (`pr#27798 <https://github.com/ceph/ceph/pull/27798>`_, Neha Ojha)
+* core: osd/PG: fix cleanup of pgmeta-like objects on PG deletion; disallow empty object names (`pr#27929 <https://github.com/ceph/ceph/pull/27929>`_, Sage Weil)
+* core: osd/PG: fix last_complete re-calculation on splitting (`issue#26958 <http://tracker.ceph.com/issues/26958>`_, `pr#27702 <https://github.com/ceph/ceph/pull/27702>`_, xie xingguo)
+* core: osd/PG: fix _finish_recovery vs repair race (`pr#30059 <https://github.com/ceph/ceph/pull/30059>`_, xie xingguo)
+* core: osd/PG: introduce all_missing_unfound helper (`issue#38784 <http://tracker.ceph.com/issues/38784>`_, `issue#38931 <http://tracker.ceph.com/issues/38931>`_, `pr#27205 <https://github.com/ceph/ceph/pull/27205>`_, xie xingguo)
+* core: osd/PG: move down peers out from peer_purged (`issue#38931 <http://tracker.ceph.com/issues/38931>`_, `pr#27182 <https://github.com/ceph/ceph/pull/27182>`_, xie xingguo)
+* core: osd/PG: move } to the proper place (`pr#27204 <https://github.com/ceph/ceph/pull/27204>`_, xie xingguo)
+* core: osd/PG: remove unused code (`pr#30930 <https://github.com/ceph/ceph/pull/30930>`_, Jianpeng Ma)
+* core: osd/PG: restart peering for undersized PG on any down stray peer coming back (`pr#33106 <https://github.com/ceph/ceph/pull/33106>`_, xie xingguo, Yan Jun)
+* core: osd/PG: skip rollforward when !transaction_applied during append_log() (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `pr#26996 <https://github.com/ceph/ceph/pull/26996>`_, Neha Ojha)
+* core: osd/PG: the warning seems more serious than what it wanna transmit (`pr#27509 <https://github.com/ceph/ceph/pull/27509>`_, Zengran Zhang)
+* core: osd/PG: use emplace() to construct new element in-place (`pr#27124 <https://github.com/ceph/ceph/pull/27124>`_, Zengran Zhang)
+* core: osd/PGLog.h: print olog_can_rollback_to before deciding to rollback (`issue#38894 <http://tracker.ceph.com/issues/38894>`_, `issue#21174 <http://tracker.ceph.com/issues/21174>`_, `pr#27105 <https://github.com/ceph/ceph/pull/27105>`_, Neha Ojha)
+* core: osd/PGLog: persist num_objects_missing for replicas when peering is done (`pr#30466 <https://github.com/ceph/ceph/pull/30466>`_, xie xingguo)
+* core: osd/PGLog: preserve original_crt to check rollbackability (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `pr#27200 <https://github.com/ceph/ceph/pull/27200>`_, Neha Ojha)
+* core: osd/PGLog: reset log.complete_to when recover obect failed (`pr#30533 <https://github.com/ceph/ceph/pull/30533>`_, Tao Ning)
+* core: osd/PGStateUtils: initialize NamedState::enter_time (`pr#33813 <https://github.com/ceph/ceph/pull/33813>`_, Jianpeng Ma)
+* core: osd/PrimaryLogPG: always use strict priority ordering for kicked recovery ops (`pr#30632 <https://github.com/ceph/ceph/pull/30632>`_, xie xingguo)
+* core: osd/PrimaryLogPG: Avoid accessing destroyed references in finish_degr\xe2\x80\xa6 (`pr#29663 <https://github.com/ceph/ceph/pull/29663>`_, Tao Ning)
+* core: osd/PrimaryLogPG: cancel in-flight manifest ops on interval changing; fix race with scru (`pr#29985 <https://github.com/ceph/ceph/pull/29985>`_, xie xingguo)
+* core: osd/PrimaryLogPG: do_op - do not create head object twice (`pr#28785 <https://github.com/ceph/ceph/pull/28785>`_, xie xingguo)
+* core: osd/PrimaryLogPG: finish_copyfrom - dirty omap if necessary (`pr#29729 <https://github.com/ceph/ceph/pull/29729>`_, xie xingguo)
+* core: osd/PrimaryLogPG: fix dirty range of write_full (`pr#29726 <https://github.com/ceph/ceph/pull/29726>`_, xie xingguo)
+* core: osd/PrimaryLogPG: fix warning (`pr#30716 <https://github.com/ceph/ceph/pull/30716>`_, Sage Weil)
+* core: osd/PrimaryLogPG: include op_returns in dup replies (`pr#30640 <https://github.com/ceph/ceph/pull/30640>`_, Sage Weil)
+* core: osd/PrimaryLogPG: kill obsolete ondisk\\_{read,write}_lock comments (`pr#29719 <https://github.com/ceph/ceph/pull/29719>`_, xie xingguo)
+* core: osd/PrimaryLogPG: more constness (`pr#28786 <https://github.com/ceph/ceph/pull/28786>`_, Kefu Chai)
+* core: osd/PrimaryLogPG: remove unused parent pgls-filter (`pr#29675 <https://github.com/ceph/ceph/pull/29675>`_, Radoslaw Zarzynski, Kefu Chai)
+* core: osd/PrimaryLogPG: simple debug message (`pr#32444 <https://github.com/ceph/ceph/pull/32444>`_, Jianpeng Ma)
+* core: osd/PrimaryLogPG: skip obcs that dont exist during backfill scan_range (`pr#30715 <https://github.com/ceph/ceph/pull/30715>`_, Sage Weil)
+* core: osd/PrimaryLogPG: update oi.size on write op implicitly truncating object up (`pr#30085 <https://github.com/ceph/ceph/pull/30085>`_, xie xingguo)
+* core: osd/PrimaryLogPG: use legacy timestamp rendering for hit_set objects (`pr#33117 <https://github.com/ceph/ceph/pull/33117>`_, Sage Weil)
+* core: osd/ReplicatedBackend: check against empty data_included before enabling crc (`pr#29621 <https://github.com/ceph/ceph/pull/29621>`_, xie xingguo)
+* core: osd/scheduler/OpSchedulerItem: schedule backoffs as client ops (`pr#32382 <https://github.com/ceph/ceph/pull/32382>`_, Samuel Just)
+* core: osd/SnapMapper: remove pre-octopus snapmapper keys after conversion (`pr#30368 <https://github.com/ceph/ceph/pull/30368>`_, Sage Weil)
+* core: osd/SnapMirror: no need to record purged_snaps every epoch (`pr#31866 <https://github.com/ceph/ceph/pull/31866>`_, Sage Weil)
+* core: OSD: modify n.cookie to op.notify.cookie (`pr#29418 <https://github.com/ceph/ceph/pull/29418>`_, yangjun)
+* core: osdc/Objecter: always add `\\0 after strncpy() (`pr#27286 <https://github.com/ceph/ceph/pull/27286>`_, Kefu Chai)
+* core: osdc/Objecter: Boost.Asio (I object!) (`pr#16715 <https://github.com/ceph/ceph/pull/16715>`_, Adam C. Emerson)
+* core: osdc/Objecter: debug pause/unpause transition (`pr#32850 <https://github.com/ceph/ceph/pull/32850>`_, Sage Weil)
+* core: osdc/Objecter: fix OSDMap leak in handle_osd_map (`issue#20491 <http://tracker.ceph.com/issues/20491>`_, `pr#28242 <https://github.com/ceph/ceph/pull/28242>`_, Sage Weil)
+* core: osdc/Objecter: only pause if respects_full() (`pr#33020 <https://github.com/ceph/ceph/pull/33020>`_, Sage Weil)
+* core: osdc/Objecter: pg-mapping cache (`pr#28487 <https://github.com/ceph/ceph/pull/28487>`_, xie xingguo)
+* core: osdc/Objecter: _calc_target - inline spgid (`pr#28570 <https://github.com/ceph/ceph/pull/28570>`_, xie xingguo)
+* core: osdc: Fix a missing : for the correct namespace (`pr#29472 <https://github.com/ceph/ceph/pull/29472>`_, Willem Jan Withagen)
+* core: pybind/ceph_argparse: improve ceph -h syntax (`pr#30431 <https://github.com/ceph/ceph/pull/30431>`_, Sage Weil)
+* core: pybind/mgr/mgr_module: fix standby module logging options (`pr#33639 <https://github.com/ceph/ceph/pull/33639>`_, Sage Weil)
+* core: pybind/mgr/mgr_util: fix pretty time delta (`pr#33794 <https://github.com/ceph/ceph/pull/33794>`_, Sage Weil)
+* core: pybind/mgr/\\*: fix config_notify handling of default values (`pr#32755 <https://github.com/ceph/ceph/pull/32755>`_, Sage Weil)
+* core: qa/distros: add rhel/centos 8.1 (`pr#33026 <https://github.com/ceph/ceph/pull/33026>`_, Sage Weil)
+* core: qa/distros: centos 7.6; update centos and ubuntu latest symlinks (`pr#27349 <https://github.com/ceph/ceph/pull/27349>`_, Sage Weil)
+* core: qa/standalone/mon/osd-create-pool: fix utf-8 grep LANG (`pr#32711 <https://github.com/ceph/ceph/pull/32711>`_, Sage Weil)
+* core: qa/standalone/osd/divergent-priors: add reproducer for bug 41816 (`pr#30506 <https://github.com/ceph/ceph/pull/30506>`_, Sage Weil)
+* core: qa/standalone/osd/osd-bench: debug bluestore (`pr#32961 <https://github.com/ceph/ceph/pull/32961>`_, Sage Weil)
+* core: qa/standalone/osd/osd-markdown: fix dup command disabling (`issue#38359 <http://tracker.ceph.com/issues/38359>`_, `pr#27499 <https://github.com/ceph/ceph/pull/27499>`_, Sage Weil)
+* core: qa/standalone/scrub/osd-scrub-snaps: misc fixes for removed_snaps change (`issue#40725 <http://tracker.ceph.com/issues/40725>`_, `pr#29003 <https://github.com/ceph/ceph/pull/29003>`_, Sage Weil)
+* core: qa/standalone: python -> python3 (`pr#32383 <https://github.com/ceph/ceph/pull/32383>`_, Sage Weil)
+* core: qa/suites/rados/multimon/tasks/mon_clock_with_skews: disable ntpd etc (`pr#33184 <https://github.com/ceph/ceph/pull/33184>`_, Sage Weil)
+* core: qa/suites/rados/multimon: fix failures (`issue#40112 <http://tracker.ceph.com/issues/40112>`_, `pr#28353 <https://github.com/ceph/ceph/pull/28353>`_, Sage Weil)
+* core: qa/suites/rados/singleton-nomsgr/all/balancer: whitelist PG_AVAILABILITY (`pr#31747 <https://github.com/ceph/ceph/pull/31747>`_, Sage Weil)
+* core: qa/suites/rados/singleton/all/ec-lost-unfound: no rbd pool (`pr#30596 <https://github.com/ceph/ceph/pull/30596>`_, Sage Weil)
+* core: qa/suites/rados/thrash-old-clients: centos -> ubuntu (`pr#32356 <https://github.com/ceph/ceph/pull/32356>`_, Sage Weil)
+* core: qa/suites/rados/thrash-old-clients: skip TestClsRbd.mirror test (`pr#31745 <https://github.com/ceph/ceph/pull/31745>`_, Sage Weil)
+* core: qa/suites/rados/thrash: debug monc (`pr#32885 <https://github.com/ceph/ceph/pull/32885>`_, Sage Weil)
+* core: qa/suites/upgrade/nautilus-x: misc updates (`pr#27138 <https://github.com/ceph/ceph/pull/27138>`_, Sage Weil)
+* core: qa/suites/upgrade/\\*-x-singleton: enable bluestore debugging settings (`pr#27786 <https://github.com/ceph/ceph/pull/27786>`_, Sage Weil)
+* core: qa/suites/upgrade: all upgrades to octopus on ubuntu only (`pr#32275 <https://github.com/ceph/ceph/pull/32275>`_, Sage Weil)
+* core: qa/suits/rados/basic/tasks/rados_api_tests: pgs can go degraded (`pr#30627 <https://github.com/ceph/ceph/pull/30627>`_, Sage Weil)
+* core: qa/tasks/ceph2: teuthology task to bring up a ceph-daemon+ssh cluster (`pr#31502 <https://github.com/ceph/ceph/pull/31502>`_, Sage Weil)
+* core: qa/tasks/ceph: only re-request scrub on unscrubbed pgs (`pr#32988 <https://github.com/ceph/ceph/pull/32988>`_, Sage Weil)
+* core: qa/tasks/ceph_manager: fix thrash_pg_upmap_items when no pools (`pr#29144 <https://github.com/ceph/ceph/pull/29144>`_, Sage Weil)
+* core: qa/tasks/ceph_manager: make upmap thrasher behave when no pools/pgs (`pr#29069 <https://github.com/ceph/ceph/pull/29069>`_, Sage Weil)
+* core: qa/tasks/ceph_manager: remove race from all_active_or_peered() (`pr#29498 <https://github.com/ceph/ceph/pull/29498>`_, Sage Weil)
+* core: qa/tasks/ceph_manager: wait for clean before asserting clean on minsize test (`pr#29109 <https://github.com/ceph/ceph/pull/29109>`_, Sage Weil)
+* core: qa/workunits/rados/test_large_omap_detection: py3-ify (`pr#32405 <https://github.com/ceph/ceph/pull/32405>`_, Sage Weil)
+* core: qa: increase mon tell retries when injecting msgr failures (`pr#30872 <https://github.com/ceph/ceph/pull/30872>`_, Sage Weil)
+* core: qa: more fixes for the removed_snaps changeset (`issue#40674 <http://tracker.ceph.com/issues/40674>`_, `pr#28901 <https://github.com/ceph/ceph/pull/28901>`_, Sage Weil)
+* core: qa: run various tests on ubuntu (`pr#32278 <https://github.com/ceph/ceph/pull/32278>`_, Sage Weil)
+* core: rados bench: fix the delayed checking of completed ops (`pr#32928 <https://github.com/ceph/ceph/pull/32928>`_, Jianshen Liu)
+* core: Revert common: default pg_autoscale_mode=on for new pools (`pr#30440 <https://github.com/ceph/ceph/pull/30440>`_, David Zafman)
+* core: Revert crush: remove invalid upmap items (`pr#32017 <https://github.com/ceph/ceph/pull/32017>`_, David Zafman)
+* core: Revert Merge pull request #16715 from adamemerson/wip-I-Object! (`pr#31790 <https://github.com/ceph/ceph/pull/31790>`_, Sage Weil)
+* core: Revert test: librados startup/shutdown racer test (`pr#31092 <https://github.com/ceph/ceph/pull/31092>`_, Sage Weil)
+* core: rgw/rgw_tools: fix osd pool set json syntax (`pr#27967 <https://github.com/ceph/ceph/pull/27967>`_, Sage Weil)
+* core: rocksdb: enable rocksdb_rmrange=true by default (`pr#29323 <https://github.com/ceph/ceph/pull/29323>`_, Sage Weil)
+* core: rocksdb: Updated to v6.1.2 (`pr#29026 <https://github.com/ceph/ceph/pull/29026>`_, Mark Nelson)
+* core: sample.ceph.conf: correct the default value of filestore merge threshold (`pr#28653 <https://github.com/ceph/ceph/pull/28653>`_, zhang Shaowen)
+* core: selinux: Allow ceph to read udev d (`pr#29071 <https://github.com/ceph/ceph/pull/29071>`_, Boris Ranto)
+* core: src/: Clean up endian handling (`pr#30409 <https://github.com/ceph/ceph/pull/30409>`_, Ulrich Weigand)
+* core: src/dmclock: bring in fixes for indirect_intrusive_heap (`pr#32380 <https://github.com/ceph/ceph/pull/32380>`_, Samuel Just)
+* core: src/osd: add tier-flush op (`pr#28778 <https://github.com/ceph/ceph/pull/28778>`_, Myoungwon Oh)
+* core: test: add librados-based startup/shutdown racer test (`pr#30552 <https://github.com/ceph/ceph/pull/30552>`_, Jeff Layton)
+* core: tools/rados: call pool_lookup() after rados is connected (`pr#30413 <https://github.com/ceph/ceph/pull/30413>`_, Vikhyat Umrao)
+* core: tools/rados: prevent put operation from recreating object when --offset=0 (`pr#31230 <https://github.com/ceph/ceph/pull/31230>`_, Adam Kupczyk)
+* core: tools/rados: Unmask -o to restore original behaviour (`pr#31310 <https://github.com/ceph/ceph/pull/31310>`_, Brad Hubbard)
+* core: Wip lazy omap test (`pr#28070 <https://github.com/ceph/ceph/pull/28070>`_, Brad Hubbard)
+* crimon/osd: serve read requests (`pr#26697 <https://github.com/ceph/ceph/pull/26697>`_, Kefu Chai)
+* Crimson build fixes (`pr#33345 <https://github.com/ceph/ceph/pull/33345>`_, Samuel Just)
+* crimson, common: Add ephemeral ObjectContext state to crimson (`pr#31202 <https://github.com/ceph/ceph/pull/31202>`_, Samuel Just)
+* crimson,auth: fix FTBFS of crimson-osd and fix v1/v2 auth (`pr#27809 <https://github.com/ceph/ceph/pull/27809>`_, Kefu Chai, Yingxin Cheng)
+* crimson,osd: performance fixes (`pr#28071 <https://github.com/ceph/ceph/pull/28071>`_, Kefu Chai, Radoslaw Zarzynski)
+* crimson/common/errorator.h: add handle_error() method (`pr#31856 <https://github.com/ceph/ceph/pull/31856>`_, Radoslaw Zarzynski)
+* crimson/common/errorator.h: simplify the compound safe_then() variant (`pr#31918 <https://github.com/ceph/ceph/pull/31918>`_, Radoslaw Zarzynski)
+* crimson/common: more friendly to seastar::do_with() (`pr#33199 <https://github.com/ceph/ceph/pull/33199>`_, Kefu Chai)
+* crimson/common: remove unused file .#log.cc (`pr#28828 <https://github.com/ceph/ceph/pull/28828>`_, Changcheng Liu)
+* crimson/mon: fix the v1 auth (`pr#28041 <https://github.com/ceph/ceph/pull/28041>`_, Kefu Chai)
+* crimson/mon: use shared_future for waiting MauthReply (`pr#30366 <https://github.com/ceph/ceph/pull/30366>`_, chunmei Liu)
+* crimson/net: bug fixes from v2 failover tests (`pr#29882 <https://github.com/ceph/ceph/pull/29882>`_, Yingxin Cheng)
+* crimson/net: clean-up and fixes of messenger (`pr#29057 <https://github.com/ceph/ceph/pull/29057>`_, Yingxin Cheng)
+* crimson/net: extract do_write_dispatch_sweep() (`pr#27428 <https://github.com/ceph/ceph/pull/27428>`_, Yingxin Cheng)
+* crimson/net: implement preemptive shutdown/close (`pr#28682 <https://github.com/ceph/ceph/pull/28682>`_, Yingxin Cheng)
+* crimson/net: improve batching in the write path (`pr#27788 <https://github.com/ceph/ceph/pull/27788>`_, Yingxin Cheng)
+* crimson/net: lossless policy for v2 protocol (`pr#29378 <https://github.com/ceph/ceph/pull/29378>`_, Yingxin Cheng)
+* crimson/net: lossy connection for ProtocolV2 (`pr#26710 <https://github.com/ceph/ceph/pull/26710>`_, Yingxin Cheng)
+* crimson/net: misc fixes in v1 read path (`pr#27837 <https://github.com/ceph/ceph/pull/27837>`_, Yingxin Cheng)
+* crimson/net: prefer <fmt/chrono.h> over <fmt/time.h> (`pr#27831 <https://github.com/ceph/ceph/pull/27831>`_, Kefu Chai)
+* crimson/net: prevent reusing the sent messages (`pr#28890 <https://github.com/ceph/ceph/pull/28890>`_, Yingxin Cheng)
+* crimson/net: print tx/rx messages using logger().info() (`pr#28798 <https://github.com/ceph/ceph/pull/28798>`_, Kefu Chai)
+* crimson/net: remove redundant std::move() (`pr#28317 <https://github.com/ceph/ceph/pull/28317>`_, Kefu Chai)
+* crimson/net: v2 racing tests, stall tests and bug fixes (`pr#30313 <https://github.com/ceph/ceph/pull/30313>`_, Yingxin Cheng)
+* crimson/os: do not fail if fsid file exists when mkfs (`pr#27006 <https://github.com/ceph/ceph/pull/27006>`_, chunmei Liu, Kefu Chai)
+* crimson/os: init PG with pg coll not meta coll (`pr#33084 <https://github.com/ceph/ceph/pull/33084>`_, Kefu Chai)
+* crimson/os: Object::read() returns bufferlist instead of never used errcode (`pr#30380 <https://github.com/ceph/ceph/pull/30380>`_, Radoslaw Zarzynski)
+* crimson/osd/osd_operation.h: clean up duplicative check (`pr#31859 <https://github.com/ceph/ceph/pull/31859>`_, Radoslaw Zarzynski)
+* crimson/osd/pg: start_operation for read_state, schedule_event_on_commit (`pr#28771 <https://github.com/ceph/ceph/pull/28771>`_, Samuel Just)
+* crimson/osd/pg_meta: use initializer list for passing set<> (`pr#28461 <https://github.com/ceph/ceph/pull/28461>`_, Kefu Chai)
+* crimson/osd: abort on unsupported objectstore type (`pr#28790 <https://github.com/ceph/ceph/pull/28790>`_, Kefu Chai)
+* crimson/osd: add --help-seastar command line option (`pr#28794 <https://github.com/ceph/ceph/pull/28794>`_, Kefu Chai)
+* crimson/osd: add minimal state machine for PG peering (`pr#27071 <https://github.com/ceph/ceph/pull/27071>`_, Kefu Chai)
+* crimson/osd: add pgls support (`pr#30433 <https://github.com/ceph/ceph/pull/30433>`_, Kefu Chai)
+* crimson/osd: cache object_info and snapset in PGBackend (`pr#27310 <https://github.com/ceph/ceph/pull/27310>`_, Kefu Chai)
+* crimson/osd: call at_exit() before stopping the engine (`pr#27177 <https://github.com/ceph/ceph/pull/27177>`_, Kefu Chai)
+* crimson/osd: call engine().exit(0) after mkfs (`pr#27061 <https://github.com/ceph/ceph/pull/27061>`_, Kefu Chai)
+* crimson/osd: capture watcher when calling its member function (`pr#33425 <https://github.com/ceph/ceph/pull/33425>`_, Kefu Chai)
+* crimson/osd: cleanups (`pr#30736 <https://github.com/ceph/ceph/pull/30736>`_, Kefu Chai)
+* crimson/osd: consolidate the code to initialize msgrs (`pr#27426 <https://github.com/ceph/ceph/pull/27426>`_, Kefu Chai)
+* crimson/osd: create msgrs in main.cc (`pr#27066 <https://github.com/ceph/ceph/pull/27066>`_, Kefu Chai)
+* crimson/osd: crimson/osd: do not load fullmap.0 (`pr#27004 <https://github.com/ceph/ceph/pull/27004>`_, chunmei Liu, Kefu Chai)
+* crimson/osd: differentiate write from writefull (`pr#28959 <https://github.com/ceph/ceph/pull/28959>`_, Kefu Chai)
+* crimson/osd: do not add whoami as hb peer and cleanups (`pr#27307 <https://github.com/ceph/ceph/pull/27307>`_, Kefu Chai)
+* crimson/osd: extend OpsExecuter to carry about op effects (`pr#30310 <https://github.com/ceph/ceph/pull/30310>`_, Radoslaw Zarzynski)
+* crimson/osd: fix the build broken by df771861 (`pr#28053 <https://github.com/ceph/ceph/pull/28053>`_, chunmei Liu)
+* crimson/osd: fix the Clang build in create_watch_info() (`pr#33350 <https://github.com/ceph/ceph/pull/33350>`_, Radoslaw Zarzynski)
+* crimson/osd: implement replicated write (`pr#29076 <https://github.com/ceph/ceph/pull/29076>`_, Kefu Chai)
+* crimson/osd: init PG with more info (`pr#27064 <https://github.com/ceph/ceph/pull/27064>`_, Kefu Chai)
+* crimson/osd: lower debug level on i/o path (`pr#27338 <https://github.com/ceph/ceph/pull/27338>`_, Kefu Chai)
+* crimson/osd: misc fixes and cleanup (`pr#33528 <https://github.com/ceph/ceph/pull/33528>`_, Yingxin Cheng)
+* crimson/osd: misc fixes for OSD reboot-ability (`pr#33595 <https://github.com/ceph/ceph/pull/33595>`_, Yingxin Cheng)
+* crimson/osd: partition args the right way (`pr#27211 <https://github.com/ceph/ceph/pull/27211>`_, Kefu Chai)
+* crimson/osd: pass unknown args to ConfigProxy::parse_args() (`pr#27062 <https://github.com/ceph/ceph/pull/27062>`_, Kefu Chai)
+* crimson/osd: remove unneeded captures - pg.cc (`pr#33349 <https://github.com/ceph/ceph/pull/33349>`_, Ronen Friedman)
+* crimson/osd: report pg_stats to mgr (`pr#27065 <https://github.com/ceph/ceph/pull/27065>`_, Kefu Chai)
+* crimson/osd: should handle pg_lease messages (`pr#30834 <https://github.com/ceph/ceph/pull/30834>`_, Kefu Chai)
+* crimson/osd: shutdown services in the right order (`pr#27987 <https://github.com/ceph/ceph/pull/27987>`_, Kefu Chai)
+* crimson/osd: some cleanups (`pr#28402 <https://github.com/ceph/ceph/pull/28402>`_, Kefu Chai)
+* crimson/osd: support write pid_file when osd start (`pr#27413 <https://github.com/ceph/ceph/pull/27413>`_, chunmei Liu)
+* crimson/osd: update peering_state in PG::on_activate_complete() (`pr#28747 <https://github.com/ceph/ceph/pull/28747>`_, Kefu Chai)
+* crimson/osd: use single-pg peering ops (`pr#30372 <https://github.com/ceph/ceph/pull/30372>`_, Kefu Chai)
+* crimson/thread: generalize Task so it works w/ func returns void (`pr#32742 <https://github.com/ceph/ceph/pull/32742>`_, Kefu Chai)
+* crimson/{net,mon,osd}: misc logging changes (`pr#27099 <https://github.com/ceph/ceph/pull/27099>`_, Kefu Chai)
+* crimson/{osd,heartbeat}: allow heartbeat to have access to authorizer (`pr#27059 <https://github.com/ceph/ceph/pull/27059>`_, Kefu Chai)
+* crimson/{osd,mon}: lower log level when sending a replicated op (`pr#30957 <https://github.com/ceph/ceph/pull/30957>`_, Kefu Chai)
+* crimson: add editor properties header (`pr#33408 <https://github.com/ceph/ceph/pull/33408>`_, Kefu Chai)
+* crimson: add FuturizedStore to encapsulate CyanStore (`pr#28358 <https://github.com/ceph/ceph/pull/28358>`_, chunmei Liu)
+* crimson: add missing include in common/errorator.h (`pr#32490 <https://github.com/ceph/ceph/pull/32490>`_, Radoslaw Zarzynski)
+* crimson: add support for basic write path (`pr#27873 <https://github.com/ceph/ceph/pull/27873>`_, Radoslaw Zarzynski)
+* crimson: add support for watch / notify, part 1 (`pr#32679 <https://github.com/ceph/ceph/pull/32679>`_, Radoslaw Zarzynski)
+* crimson: bring ceph::errorator with its first appliances (`pr#30387 <https://github.com/ceph/ceph/pull/30387>`_, Radoslaw Zarzynski)
+* crimson: CLANG-related fixes to errorator.h (`pr#32488 <https://github.com/ceph/ceph/pull/32488>`_, Ronen Friedman, Radoslaw Zarzynski)
+* crimson: clean up and refactor asok (`pr#33357 <https://github.com/ceph/ceph/pull/33357>`_, Kefu Chai)
+* crimson: enable cephx for v2 msgr (`pr#27514 <https://github.com/ceph/ceph/pull/27514>`_, Kefu Chai)
+* crimson: fix build with GCC-10 (`pr#33233 <https://github.com/ceph/ceph/pull/33233>`_, Kefu Chai)
+* crimson: fix crimson pg coll usage error (`pr#33076 <https://github.com/ceph/ceph/pull/33076>`_, Chunmei Liu)
+* crimson: fix lambda captures of non-variables (`pr#32494 <https://github.com/ceph/ceph/pull/32494>`_, Ronen Friedman)
+* crimson: futurized CyanStores member functions and Collection (`pr#29470 <https://github.com/ceph/ceph/pull/29470>`_, Kefu Chai, chunmei Liu)
+* crimson: handle MOSDPGQuery2 properly (`pr#30399 <https://github.com/ceph/ceph/pull/30399>`_, Kefu Chai)
+* crimson: make seastar::do_with() a friend of errorated futures (`pr#32175 <https://github.com/ceph/ceph/pull/32175>`_, Radoslaw Zarzynski)
+* crimson: move dummy impl of AuthServer to DummyAuth (`pr#27452 <https://github.com/ceph/ceph/pull/27452>`_, Kefu Chai)
+* crimson: move os/cyan\\_\\* down to os/cyanstore/\\* (`pr#31874 <https://github.com/ceph/ceph/pull/31874>`_, Kefu Chai)
+* crimson: pass `Connection\\*` to Dispatch::ms_dispatch() (`pr#27690 <https://github.com/ceph/ceph/pull/27690>`_, Yingxin Cheng, Kefu Chai)
+* crimson: pickup change to fix --cpuset support and cleanups (`pr#33250 <https://github.com/ceph/ceph/pull/33250>`_, Kefu Chai)
+* crimson: remove some attributes from lambda (`pr#32604 <https://github.com/ceph/ceph/pull/32604>`_, Ronen Friedman)
+* crimson: run in foreground if possible, silence warnings (`pr#30474 <https://github.com/ceph/ceph/pull/30474>`_, Samuel Just, Kefu Chai)
+* crimson: s/ceph/crimson/ in namespace names (`pr#31069 <https://github.com/ceph/ceph/pull/31069>`_, Kefu Chai)
+* crimson: serve basic RBD traffic coming from fio (`pr#30339 <https://github.com/ceph/ceph/pull/30339>`_, Radoslaw Zarzynski)
+* crimson: solve the problem that crimson-osd\s created pgs stuck in unknown state (`pr#33780 <https://github.com/ceph/ceph/pull/33780>`_, Xuehan Xu)
+* crimson: stop osd before stopping messengers (`pr#31904 <https://github.com/ceph/ceph/pull/31904>`_, Kefu Chai)
+* crimson: support pgnls and delete op (`pr#28079 <https://github.com/ceph/ceph/pull/28079>`_, Kefu Chai)
+* crimson: update osd when peer gets authenticated (`pr#27416 <https://github.com/ceph/ceph/pull/27416>`_, Kefu Chai)
+* crimson: use given osd_fsid when mkfs (`pr#28800 <https://github.com/ceph/ceph/pull/28800>`_, Kefu Chai)
+* crimson:: add alien blue store (`pr#31041 <https://github.com/ceph/ceph/pull/31041>`_, Samuel Just, Chunmei Liu, Kefu Chai)
+* crush: add root_bucket to identify underfull buckets (`issue#38826 <http://tracker.ceph.com/issues/38826>`_, `pr#27068 <https://github.com/ceph/ceph/pull/27068>`_, huangjun)
+* crush: remove invalid upmap items (`pr#31131 <https://github.com/ceph/ceph/pull/31131>`_, huangjun)
+* crush: remove invalid upmap items (`pr#32099 <https://github.com/ceph/ceph/pull/32099>`_, huangjun)
+* crush: various fixes for weight-sets, the osd_crush_update_weight_set option, and tests (`pr#26955 <https://github.com/ceph/ceph/pull/26955>`_, Sage Weil)
+* dashboard/services: fix lint error (`pr#30289 <https://github.com/ceph/ceph/pull/30289>`_, Willem Jan Withagen)
+* deb,rpm: switch to python 3 (`pr#32252 <https://github.com/ceph/ceph/pull/32252>`_, Sage Weil, Alfredo Deza)
+* debian: add python3-jsonpatch as dependency (`pr#33298 <https://github.com/ceph/ceph/pull/33298>`_, Sebastian Wagner)
+* denc: allow DencDumper to dump OOB buffer (`pr#27704 <https://github.com/ceph/ceph/pull/27704>`_, Kefu Chai)
+* doc/bootstrap: fixed default --keyring target (`pr#32643 <https://github.com/ceph/ceph/pull/32643>`_, Yaarit Hatuka)
+* doc/foundation: fix amihan (`pr#32999 <https://github.com/ceph/ceph/pull/32999>`_, Sage Weil)
+* doc: .organizationmap: Wido 42on -> 42on (`pr#32260 <https://github.com/ceph/ceph/pull/32260>`_, Sage Weil)
+* doc: add a deduplication document (`pr#28462 <https://github.com/ceph/ceph/pull/28462>`_, Myoungwon Oh)
+* doc: add a doc for vstart_runner.py (`pr#29907 <https://github.com/ceph/ceph/pull/29907>`_, Rishabh Dave)
+* doc: add a new document on distributed cephfs metadata cache (`pr#30265 <https://github.com/ceph/ceph/pull/30265>`_, Jeff Layton)
+* doc: Add a new document on Dynamic Metadata Management in CephFS (`pr#30348 <https://github.com/ceph/ceph/pull/30348>`_, Sidharth Anupkrishnan)
+* doc: Add a RGW swift auth note (`pr#31309 <https://github.com/ceph/ceph/pull/31309>`_, Matthew Oliver)
+* doc: add ceph fs volumes and subvolumes documentation (`pr#30381 <https://github.com/ceph/ceph/pull/30381>`_, Ramana Raja)
+* doc: add CephFS Octopus release notes (`pr#33450 <https://github.com/ceph/ceph/pull/33450>`_, Patrick Donnelly)
+* doc: add changelog for nautilus (`pr#27048 <https://github.com/ceph/ceph/pull/27048>`_, Abhishek Lekshmanan)
+* doc: add chrony to preflight checklist for Ubuntu 18.04 (`pr#31948 <https://github.com/ceph/ceph/pull/31948>`_, Zac Dover)
+* doc: add config help/get/set section for runtime client configuration (`issue#41688 <http://tracker.ceph.com/issues/41688>`_, `pr#32117 <https://github.com/ceph/ceph/pull/32117>`_, Venky Shankar)
+* doc: Add Dashboard Octopus release notes (`pr#33555 <https://github.com/ceph/ceph/pull/33555>`_, Lenz Grimmer)
+* doc: add description for fuse_disable_pagecache (`pr#31902 <https://github.com/ceph/ceph/pull/31902>`_, Yan, Zheng)
+* doc: add doc for blacklisting older CephFS clients (`issue#39130 <http://tracker.ceph.com/issues/39130>`_, `pr#27412 <https://github.com/ceph/ceph/pull/27412>`_, Patrick Donnelly)
+* doc: add doc for cephfs lazyio (`issue#38729 <http://tracker.ceph.com/issues/38729>`_, `pr#26976 <https://github.com/ceph/ceph/pull/26976>`_, Yan, Zheng)
+* doc: add guide for running tests with teuthology (`pr#32114 <https://github.com/ceph/ceph/pull/32114>`_, Rishabh Dave)
+* doc: add mds map to list of ceph monitor assets (`pr#32631 <https://github.com/ceph/ceph/pull/32631>`_, Zac Dover)
+* doc: add missed word than in doc/man/8/rbd.rst (`pr#31022 <https://github.com/ceph/ceph/pull/31022>`_, Drunkard Zhang)
+* doc: Add missing mgr cap for the bootstrap keyring (`pr#27201 <https://github.com/ceph/ceph/pull/27201>`_, Bryan Stillwell)
+* doc: add missing virtualenv for build-doc (`pr#31896 <https://github.com/ceph/ceph/pull/31896>`_, Rodrigo Severo)
+* doc: Add note to execute cephfs-shell (`pr#27369 <https://github.com/ceph/ceph/pull/27369>`_, Varsha Rao)
+* doc: add package for Golang (`issue#38730 <http://tracker.ceph.com/issues/38730>`_, `pr#26937 <https://github.com/ceph/ceph/pull/26937>`_, Irek Fasikhov)
+* doc: add Python 2 to Ubuntu 18.04 installations (`pr#31947 <https://github.com/ceph/ceph/pull/31947>`_, Zac Dover)
+* doc: add release notes for 13.2.5 mimic (`pr#26913 <https://github.com/ceph/ceph/pull/26913>`_, Abhishek Lekshmanan)
+* doc: add release notes for v13.2.6 mimic (`pr#28385 <https://github.com/ceph/ceph/pull/28385>`_, Abhishek Lekshmanan)
+* doc: Add sphinx_autodoc_typehints extension (`pr#33577 <https://github.com/ceph/ceph/pull/33577>`_, Sebastian Wagner)
+* doc: Add stat command usage in cephfs-shell (`pr#28236 <https://github.com/ceph/ceph/pull/28236>`_, Varsha Rao)
+* doc: Add usage for shortcuts command in cephfs-shell (`pr#27373 <https://github.com/ceph/ceph/pull/27373>`_, Varsha Rao)
+* doc: Add warning that the root directory cannot be fragmented (`pr#28354 <https://github.com/ceph/ceph/pull/28354>`_, Nathan Fish)
+* doc: Added a link to Ceph Community Calendar (`pr#31475 <https://github.com/ceph/ceph/pull/31475>`_, Zac Dover)
+* doc: added a remark to always use powers of two for pg_num (`pr#31541 <https://github.com/ceph/ceph/pull/31541>`_, Thomas Schneider)
+* doc: added an is where it was needed (`pr#32374 <https://github.com/ceph/ceph/pull/32374>`_, Zac Dover)
+* doc: Added dashboard features, improved wording (`pr#27997 <https://github.com/ceph/ceph/pull/27997>`_, Lenz Grimmer)
+* doc: added section on creating RESTful API user (`pr#26016 <https://github.com/ceph/ceph/pull/26016>`_, James McClune)
+* doc: Added the crisp getting started guide to index.rst (`pr#32531 <https://github.com/ceph/ceph/pull/32531>`_, Zac Dover)
+* doc: Adding US-Mid-West Mirror to docs (`pr#25099 <https://github.com/ceph/ceph/pull/25099>`_, Mike Perez)
+* doc: Adds cmake build options for optionally skipping few components (`pr#31066 <https://github.com/ceph/ceph/pull/31066>`_, Deepika Upadhyay)
+* doc: adjust for mon_status changes in octopus (`pr#33703 <https://github.com/ceph/ceph/pull/33703>`_, Nathan Cutler)
+* doc: admin,doc/_ext/ceph_releases.py: use yaml.safe_load() (`pr#28463 <https://github.com/ceph/ceph/pull/28463>`_, Kefu Chai)
+* doc: admin/build-doc: always install python3-\\* for build deps (`pr#32481 <https://github.com/ceph/ceph/pull/32481>`_, Kefu Chai)
+* doc: admin/build-doc: do not use system site-packages (`pr#32285 <https://github.com/ceph/ceph/pull/32285>`_, Sage Weil)
+* doc: admin/build-doc: Fix doxygen typo (`pr#32572 <https://github.com/ceph/ceph/pull/32572>`_, Varsha Rao)
+* doc: admin/build-doc: use python3 (`pr#29528 <https://github.com/ceph/ceph/pull/29528>`_, Kefu Chai)
+* doc: admin/doc-requirements.txt: bump up Sphinx and breathe (`pr#32301 <https://github.com/ceph/ceph/pull/32301>`_, Kefu Chai)
+* doc: admin/serve-doc: Switch to python3 only (`pr#33596 <https://github.com/ceph/ceph/pull/33596>`_, Brad Hubbard)
+* doc: always load resources via HTTPS (`pr#29544 <https://github.com/ceph/ceph/pull/29544>`_, Tiago Melo)
+* doc: ceph-monstore-tool: correct the key for storing mgr_command_descs (`pr#33172 <https://github.com/ceph/ceph/pull/33172>`_, Kefu Chai)
+* doc: cephfs: add section on fsync error reporting to posix.rst (`issue#24641 <http://tracker.ceph.com/issues/24641>`_, `pr#28300 <https://github.com/ceph/ceph/pull/28300>`_, Jeff Layton)
+* doc: change case from `apis` to `APIs` (`pr#33664 <https://github.com/ceph/ceph/pull/33664>`_, Deepika Upadhyay)
+* doc: clarify difference between fs and kcephfs suite (`pr#32144 <https://github.com/ceph/ceph/pull/32144>`_, Rishabh Dave)
+* doc: clarify priority use (`pr#32191 <https://github.com/ceph/ceph/pull/32191>`_, Yuri Weinstein)
+* doc: clarify support for rbd fancy striping (`pr#32176 <https://github.com/ceph/ceph/pull/32176>`_, Ilya Dryomov)
+* doc: cleanup CephFS Landing Page (`pr#30542 <https://github.com/ceph/ceph/pull/30542>`_, Milind Changire)
+* doc: coding-style: update a link and fix typos (`pr#33128 <https://github.com/ceph/ceph/pull/33128>`_, Ponnuvel Palaniyappan)
+* doc: common/admin_socket: Add doxygen for call and call_async (`pr#32547 <https://github.com/ceph/ceph/pull/32547>`_, Adam Kupczyk)
+* doc: common/hobject: Error invocation of formula in documentation (`pr#28366 <https://github.com/ceph/ceph/pull/28366>`_, Albert)
+* doc: config-ref: add a note on current scheduler settings (`pr#27243 <https://github.com/ceph/ceph/pull/27243>`_, Abhishek Lekshmanan)
+* doc: correct example to use vstart to run up cluster (`pr#26816 <https://github.com/ceph/ceph/pull/26816>`_, Changcheng Liu)
+* doc: cover more cache modes in rados/operations/cache-tiering.rst (`issue#14153 <http://tracker.ceph.com/issues/14153>`_, `pr#17614 <https://github.com/ceph/ceph/pull/17614>`_, Nathan Cutler)
+* doc: default values for mon_health_to_clog\\_\\* were flipped (`pr#29867 <https://github.com/ceph/ceph/pull/29867>`_, James McClune)
+* doc: describe metadata_heap cleanup (`issue#18174 <http://tracker.ceph.com/issues/18174>`_, `pr#26915 <https://github.com/ceph/ceph/pull/26915>`_, Dan van der Ster)
+* doc: Describe recovery and backfill prioritizations (`issue#39011 <http://tracker.ceph.com/issues/39011>`_, `pr#27941 <https://github.com/ceph/ceph/pull/27941>`_, David Zafman)
+* doc: doc : fixed capitalization (`pr#27379 <https://github.com/ceph/ceph/pull/27379>`_, Servesha Dudhgaonkar)
+* doc: doc, qa: remove invalid option mon_pg_warn_max_per_osd (`pr#30787 <https://github.com/ceph/ceph/pull/30787>`_, zhang daolong)
+* doc: doc,admin: fix the builtin search (`pr#33592 <https://github.com/ceph/ceph/pull/33592>`_, Kefu Chai)
+* doc: doc/architecture.rst: fix a typo in EC section (`pr#33241 <https://github.com/ceph/ceph/pull/33241>`_, Nag Pavan Chilakam)
+* doc: doc/bootstrap.rst: fix githus url (`pr#31086 <https://github.com/ceph/ceph/pull/31086>`_, Alexandre Bruyelles)
+* doc: doc/bootstrap: add mds and rgw steps to bootstrap (`pr#33088 <https://github.com/ceph/ceph/pull/33088>`_, Sage Weil)
+* doc: doc/ceph-fuse: describe -n option (`pr#30911 <https://github.com/ceph/ceph/pull/30911>`_, Rishabh Dave)
+* doc: doc/ceph-fuse: mention -k option in ceph-fuse man page (`pr#30561 <https://github.com/ceph/ceph/pull/30561>`_, Rishabh Dave)
+* doc: doc/ceph-kvstore-tool: add description for stats command (`pr#29990 <https://github.com/ceph/ceph/pull/29990>`_, Josh Durgin, Adam Kupczyk)
+* doc: doc/ceph-volume: initial docs for zfs/inventory and zfs/api (`pr#31252 <https://github.com/ceph/ceph/pull/31252>`_, Willem Jan Withagen)
+* doc: doc/cephadm/administration: clarify log gathering (`pr#33627 <https://github.com/ceph/ceph/pull/33627>`_, Nathan Cutler)
+* doc: doc/cephadm: adjust syntax for config set (`pr#33600 <https://github.com/ceph/ceph/pull/33600>`_, Joshua Schmid)
+* doc: doc/cephadm: big cleanup of cephadm docs (`pr#33981 <https://github.com/ceph/ceph/pull/33981>`_, Sage Weil)
+* doc: doc/cephadm: Troubleshooting (`pr#33460 <https://github.com/ceph/ceph/pull/33460>`_, Sebastian Wagner)
+* doc: doc/cephfs/client-auth: description and example are inconsistent (`pr#32762 <https://github.com/ceph/ceph/pull/32762>`_, Ilya Dryomov)
+* doc: doc/cephfs/disaster-recovery-experts: Add link for scrub and note for scrub_path (`pr#32124 <https://github.com/ceph/ceph/pull/32124>`_, Varsha Rao)
+* doc: doc/cephfs: add doc for cephfs io path (`pr#30369 <https://github.com/ceph/ceph/pull/30369>`_, Yan, Zheng)
+* doc: doc/cephfs: correct a description mistake about mds states (`issue#41893 <http://tracker.ceph.com/issues/41893>`_, `pr#30427 <https://github.com/ceph/ceph/pull/30427>`_, Xiao Guodong)
+* doc: doc/cephfs: improve add/remove MDS section (`issue#39620 <http://tracker.ceph.com/issues/39620>`_, `pr#28700 <https://github.com/ceph/ceph/pull/28700>`_, Patrick Donnelly)
+* doc: doc/cephfs: migrate best practices recommendations to relevant docs (`pr#32522 <https://github.com/ceph/ceph/pull/32522>`_, Rishabh Dave)
+* doc: doc/cleanup: drop repo-access.rst (`pr#32276 <https://github.com/ceph/ceph/pull/32276>`_, Nathan Cutler)
+* doc: doc/corpus: update to adapt the change from autotools to cmake (`pr#27552 <https://github.com/ceph/ceph/pull/27552>`_, Kefu Chai)
+* doc: doc/dev/corpus.rst: correct instructions (`pr#27741 <https://github.com/ceph/ceph/pull/27741>`_, Kefu Chai)
+* doc: doc/dev/corpus.rst: minor tweaks (`pr#28877 <https://github.com/ceph/ceph/pull/28877>`_, Kefu Chai)
+* doc: doc/dev/crimson.rst: document CBT testing (`pr#30290 <https://github.com/ceph/ceph/pull/30290>`_, Kefu Chai)
+* doc: doc/dev/crimson: transpose options of compare.py (`pr#30453 <https://github.com/ceph/ceph/pull/30453>`_, Kefu Chai)
+* doc: doc/dev/developer_guide/index.rst: add youtube reference for Getting Started (`pr#29712 <https://github.com/ceph/ceph/pull/29712>`_, Neha Ojha)
+* doc: doc/dev/developer_guide/index.rst: add youtube references (`pr#29033 <https://github.com/ceph/ceph/pull/29033>`_, Neha Ojha)
+* doc: doc/dev/developer_guide: fix heading level (`pr#30428 <https://github.com/ceph/ceph/pull/30428>`_, Nathan Cutler)
+* doc: doc/dev/developer_guide: remove web address (`pr#29183 <https://github.com/ceph/ceph/pull/29183>`_, gabriellasroman)
+* doc: doc/dev/kubernetes: Update (`pr#28081 <https://github.com/ceph/ceph/pull/28081>`_, Sebastian Wagner)
+* doc: doc/dev/osd_internals/async_recovery: update cost calculation (`pr#28036 <https://github.com/ceph/ceph/pull/28036>`_, Neha Ojha)
+* doc: doc/dev: add crimson.rst (`pr#28674 <https://github.com/ceph/ceph/pull/28674>`_, Kefu Chai)
+* doc: doc/dev: add teuthology priority recommendations (`pr#30308 <https://github.com/ceph/ceph/pull/30308>`_, Patrick Donnelly)
+* doc: doc/developer: fix dev mailing list address (`pr#32442 <https://github.com/ceph/ceph/pull/32442>`_, Willem Jan Withagen)
+* doc: doc/drivegroups: add docs for DriveGroups with excessive examples (`pr#33044 <https://github.com/ceph/ceph/pull/33044>`_, Joshua Schmid)
+* doc: doc/foundation: add ceph foundation info here (`pr#31955 <https://github.com/ceph/ceph/pull/31955>`_, Sage Weil)
+* doc: doc/foundation: add cloudbase and vexxhost (`pr#32013 <https://github.com/ceph/ceph/pull/32013>`_, Sage Weil)
+* doc: doc/foundation: add Samsung Electronics (`pr#33518 <https://github.com/ceph/ceph/pull/33518>`_, Sage Weil)
+* doc: doc/governance: add cbodey (`pr#27708 <https://github.com/ceph/ceph/pull/27708>`_, Sage Weil)
+* doc: doc/index: remove quick start from front page for now (`pr#33207 <https://github.com/ceph/ceph/pull/33207>`_, Sage Weil)
+* doc: doc/install/containers: add summary of containers and branches (`pr#31465 <https://github.com/ceph/ceph/pull/31465>`_, Sage Weil)
+* doc: doc/install/containers: note vX.Y.Z[-YYYYMMDD] tags (`pr#31975 <https://github.com/ceph/ceph/pull/31975>`_, Sage Weil)
+* doc: doc/install/manual-deployment: Change owner to ceph for the keyring file (`pr#31452 <https://github.com/ceph/ceph/pull/31452>`_, Jeffrey Chu)
+* doc: doc/install/upgrading-ceph: systemctl in Ubuntu instructions (`pr#32595 <https://github.com/ceph/ceph/pull/32595>`_, Rodrigo Severo)
+* doc: doc/install: rethink install doc installation methods order (`pr#33890 <https://github.com/ceph/ceph/pull/33890>`_, Zac Dover, Sebastian Wagner)
+* doc: doc/man/ceph: document ceph config (`pr#30645 <https://github.com/ceph/ceph/pull/30645>`_, Kefu Chai)
+* doc: doc/man: improve bluefs-bdev-expand option (`pr#32590 <https://github.com/ceph/ceph/pull/32590>`_, Kefu Chai)
+* doc: doc/mgr/ansible.rst: fix typo (`pr#28827 <https://github.com/ceph/ceph/pull/28827>`_, Lan Liu)
+* doc: doc/mgr/cephadm: document adoption process (`pr#33459 <https://github.com/ceph/ceph/pull/33459>`_, Sage Weil)
+* doc: doc/mgr/orchestrator.rst: updated current implementation status (`pr#33410 <https://github.com/ceph/ceph/pull/33410>`_, Kai Wagner)
+* doc: doc/mgr/orchestrator: Add Cephfs (`pr#33574 <https://github.com/ceph/ceph/pull/33574>`_, Sebastian Wagner)
+* doc: doc/mgr/orchestrator_cli: Rook orch supports mon update (`issue#39137 <http://tracker.ceph.com/issues/39137>`_, `pr#27431 <https://github.com/ceph/ceph/pull/27431>`_, Sebastian Wagner)
+* doc: doc/mgr/telemetry: added device channel details (`pr#33113 <https://github.com/ceph/ceph/pull/33113>`_, Yaarit Hatuka)
+* doc: doc/mgr/telemetry: update default interval (`pr#31008 <https://github.com/ceph/ceph/pull/31008>`_, Tim Serong)
+* doc: doc/mgr: Enhance placement specs (`pr#33924 <https://github.com/ceph/ceph/pull/33924>`_, Sebastian Wagner)
+* doc: doc/orchestrator: Fix broken bullet points (`issue#39094 <http://tracker.ceph.com/issues/39094>`_, `pr#27121 <https://github.com/ceph/ceph/pull/27121>`_, Sebastian Wagner)
+* doc: doc/orchestrator: Fix various issues in Orchestrator CLI documentation (`pr#31353 <https://github.com/ceph/ceph/pull/31353>`_, Volker Theile)
+* doc: doc/orchestrator: Sync status with reality (`pr#30281 <https://github.com/ceph/ceph/pull/30281>`_, Sebastian Wagner)
+* doc: doc/orchestrator: update rgw creation (`pr#33540 <https://github.com/ceph/ceph/pull/33540>`_, Yehuda Sadeh)
+* doc: doc/rados/api/python: Add documentation for mon_command (`pr#26934 <https://github.com/ceph/ceph/pull/26934>`_, Sebastian Wagner)
+* doc: doc/rados/configuration/osd-config-ref.rst: document osd_delete_sleep (`pr#28775 <https://github.com/ceph/ceph/pull/28775>`_, Neha Ojha)
+* doc: doc/rados/configuration: fix typo in mon-lookup-dns (`pr#27362 <https://github.com/ceph/ceph/pull/27362>`_, Vanush Misha Paturyan)
+* doc: doc/rados/configuration: fix typos in osd-config-ref.rst (`pr#28805 <https://github.com/ceph/ceph/pull/28805>`_, Lan Liu)
+* doc: doc/rados/configuration: update to be in sync with ConfUtils changes (`pr#28753 <https://github.com/ceph/ceph/pull/28753>`_, Kefu Chai)
+* doc: doc/rados/deployment/ceph-deploy-mon: fix typo (`pr#31164 <https://github.com/ceph/ceph/pull/31164>`_, Kefu Chai)
+* doc: doc/rados/operations/crush-map-edits: recompile and set instructions (`pr#32451 <https://github.com/ceph/ceph/pull/32451>`_, Rodrigo Severo)
+* doc: doc/rados/operations/devices: document device failure prediction (`pr#27472 <https://github.com/ceph/ceph/pull/27472>`_, Sage Weil)
+* doc: doc/rados/operations/erasure-code.rst: allow recovery below min_size (`pr#28750 <https://github.com/ceph/ceph/pull/28750>`_, Greg Farnum, Neha Ojha)
+* doc: doc/rados/operations: add safe-to-destroy check to OSD replacement workflow (`pr#28491 <https://github.com/ceph/ceph/pull/28491>`_, Sage Weil)
+* doc: doc/rados/operations: crush_rule is a name (`pr#29367 <https://github.com/ceph/ceph/pull/29367>`_, Kefu Chai)
+* doc: doc/rados/operations: document BLUEFS_SPILLOVER (`pr#27316 <https://github.com/ceph/ceph/pull/27316>`_, Sage Weil)
+* doc: doc/rados/operations: min_size is applicable to EC (`pr#33543 <https://github.com/ceph/ceph/pull/33543>`_, Brad Hubbard)
+* doc: doc/rados/operations: OSD_OUT_OF_ORDER_FULL fullness order is wrong (`pr#31588 <https://github.com/ceph/ceph/pull/31588>`_, Tsung-Ju Lii)
+* doc: doc/rados: Better block.db size recommendations for bluestore (`pr#32226 <https://github.com/ceph/ceph/pull/32226>`_, Neha Ojha)
+* doc: doc/rados: Correcting some typos in the clay code documentation (`pr#29889 <https://github.com/ceph/ceph/pull/29889>`_, Myna)
+* doc: doc/rados: update osd_min_pg_log_entries and add osd_max_pg_log_entries (`pr#32790 <https://github.com/ceph/ceph/pull/32790>`_, Neha Ojha)
+* doc: doc/radosgw/admin:fix how to modify subuser info (`pr#29839 <https://github.com/ceph/ceph/pull/29839>`_, Feng Hualong)
+* doc: doc/radosgw/compression.rst: fix typo (`pr#28749 <https://github.com/ceph/ceph/pull/28749>`_, hydro-)
+* doc: doc/radosgw/config-ref: paragraph to explain the gc settings (`pr#32367 <https://github.com/ceph/ceph/pull/32367>`_, Kai Wagner)
+* doc: doc/radosgw/multisite-sync-policy.rst: fix typo (`pr#33230 <https://github.com/ceph/ceph/pull/33230>`_, Liu Lan)
+* doc: doc/radosgw: fix typos (`pr#30642 <https://github.com/ceph/ceph/pull/30642>`_, Liu Lan)
+* doc: doc/radosgw: update documentation examples with the current S3 PHP client (`pr#25985 <https://github.com/ceph/ceph/pull/25985>`_, Laurent VOULLEMIER)
+* doc: doc/rbd/rbd-cloudstack: update disk offering URL to new docs (`pr#27713 <https://github.com/ceph/ceph/pull/27713>`_, Kefu Chai)
+* doc: doc/rbd: document the new snapshot-based mirroring feature (`pr#33561 <https://github.com/ceph/ceph/pull/33561>`_, Jason Dillaman)
+* doc: doc/rbd: fix small typos (`pr#33689 <https://github.com/ceph/ceph/pull/33689>`_, songweibin)
+* doc: doc/rbd: initial kubernetes / ceph-csi integration documentation (`pr#29429 <https://github.com/ceph/ceph/pull/29429>`_, Jason Dillaman)
+* doc: doc/rbd: re-organize top-level and add live-migration docs (`issue#40486 <http://tracker.ceph.com/issues/40486>`_, `pr#29135 <https://github.com/ceph/ceph/pull/29135>`_, Jason Dillaman)
+* doc: doc/rbd: refine rbd/libvirt usage (`pr#32273 <https://github.com/ceph/ceph/pull/32273>`_, Changcheng Liu)
+* doc: doc/rbd: s/guess/xml/ for codeblock lexer (`pr#30953 <https://github.com/ceph/ceph/pull/30953>`_, Kefu Chai)
+* doc: doc/rbd: simplify libvirt usage (`pr#32142 <https://github.com/ceph/ceph/pull/32142>`_, Changcheng Liu)
+* doc: doc/rbd: update krbd version support for RBD features (`issue#40802 <http://tracker.ceph.com/issues/40802>`_, `pr#29083 <https://github.com/ceph/ceph/pull/29083>`_, Jason Dillaman)
+* doc: doc/release/nautilus: 14.2.2 changes redone (`pr#29145 <https://github.com/ceph/ceph/pull/29145>`_, Sage Weil)
+* doc: doc/release/octopus: note about upgrade times (`pr#33401 <https://github.com/ceph/ceph/pull/33401>`_, Sage Weil)
+* doc: doc/releases/nautilus,PendingReleaseNotes: consolidate telemetry note (`pr#32160 <https://github.com/ceph/ceph/pull/32160>`_, Sage Weil)
+* doc: doc/releases/nautilus.rst: fix command to check min_compat_client (`pr#28526 <https://github.com/ceph/ceph/pull/28526>`_, Osama Elswah)
+* doc: doc/releases/nautilus.rst: remove a redundant \\* (`pr#32577 <https://github.com/ceph/ceph/pull/32577>`_, Servesha Dudhgaonkar)
+* doc: doc/releases/nautilus: Correct a systemctl command in an upgrade guide (`pr#27773 <https://github.com/ceph/ceph/pull/27773>`_, Teeranai Kormongkolkul)
+* doc: doc/releases/nautilus: final notes for v14.2.0 (`pr#27019 <https://github.com/ceph/ceph/pull/27019>`_, Sage Weil)
+* doc: doc/releases/nautilus: fix config update step (`pr#27495 <https://github.com/ceph/ceph/pull/27495>`_, Sage Weil)
+* doc: doc/releases/nautilus: fix release notes (crash->device) (`pr#32148 <https://github.com/ceph/ceph/pull/32148>`_, Sage Weil)
+* doc: doc/releases/octopus.rst: add note about ec recovery below min_size (`pr#34092 <https://github.com/ceph/ceph/pull/34092>`_, Neha Ojha)
+* doc: doc/releases/octopus.rst: format tweaks (`pr#33971 <https://github.com/ceph/ceph/pull/33971>`_, Kefu Chai)
+* doc: doc/releases/octopus.rst: formatting tweaks (`pr#33987 <https://github.com/ceph/ceph/pull/33987>`_, Kefu Chai)
+* doc: doc/releases/octopus: add additional RBD improvements (`pr#34032 <https://github.com/ceph/ceph/pull/34032>`_, Jason Dillaman)
+* doc: doc/releases/schedule.rst: add 14.2.3, 14.2.4, 15.0.0 and drop dumpling (`pr#30430 <https://github.com/ceph/ceph/pull/30430>`_, Nathan Cutler)
+* doc: doc/releases: access main releases page from top-level TOC (`pr#30598 <https://github.com/ceph/ceph/pull/30598>`_, Nathan Cutler)
+* doc: doc/releases: add 14.2.8 to release timeline (`pr#33721 <https://github.com/ceph/ceph/pull/33721>`_, Nathan Cutler)
+* doc: doc/releases: add mimic v13.2.7 to releases timeline (`pr#31872 <https://github.com/ceph/ceph/pull/31872>`_, Nathan Cutler)
+* doc: doc/releases: add release notes for mimic v13.2.7 (`pr#31777 <https://github.com/ceph/ceph/pull/31777>`_, Nathan Cutler)
+* doc: doc/releases: add release notes for mimic v13.2.8 (`pr#32040 <https://github.com/ceph/ceph/pull/32040>`_, Nathan Cutler)
+* doc: doc/releases: add release notes for nautilus v14.2.5 (`pr#31970 <https://github.com/ceph/ceph/pull/31970>`_, Nathan Cutler)
+* doc: doc/releases: Ceph Nautilus v14.2.4 Release Notes (`pr#30429 <https://github.com/ceph/ceph/pull/30429>`_, Nathan Cutler)
+* doc: doc/releases: octopus draft notes (`pr#33043 <https://github.com/ceph/ceph/pull/33043>`_, Sage Weil)
+* doc: doc/releases: Octopus is not stable yet (`pr#33729 <https://github.com/ceph/ceph/pull/33729>`_, Nathan Cutler)
+* doc: doc/releases: update for 12 month cycle (`pr#28864 <https://github.com/ceph/ceph/pull/28864>`_, Sage Weil)
+* doc: doc/rgw: add design doc for multisite resharding (`pr#33539 <https://github.com/ceph/ceph/pull/33539>`_, Casey Bodley)
+* doc: doc/rgw: document CreateBucketConfiguration for s3 PUT Bucket api (`issue#39597 <http://tracker.ceph.com/issues/39597>`_, `pr#27977 <https://github.com/ceph/ceph/pull/27977>`_, Casey Bodley)
+* doc: doc/rgw: document use of realm pull instead of period pull (`issue#39655 <http://tracker.ceph.com/issues/39655>`_, `pr#28052 <https://github.com/ceph/ceph/pull/28052>`_, Casey Bodley)
+* doc: doc/rgw: fix broken link to boto s3 extensions document (`pr#32740 <https://github.com/ceph/ceph/pull/32740>`_, Casey Bodley)
+* doc: doc/rgw: update civetweb rgw_frontends config example (`pr#27054 <https://github.com/ceph/ceph/pull/27054>`_, Casey Bodley)
+* doc: doc/start/documenting-ceph.rst: make better doc recommendations (`pr#30273 <https://github.com/ceph/ceph/pull/30273>`_, Neha Ojha)
+* doc: doc/start/hardware-recommendations.rst: minor tweaks (`pr#30837 <https://github.com/ceph/ceph/pull/30837>`_, Amrita Sakthivel)
+* doc: doc/_templates/page.html: redirect to etherpad (`pr#32197 <https://github.com/ceph/ceph/pull/32197>`_, Neha Ojha)
+* doc: Doc: Add Nautilus 14.2.2 to schedule and releases (`issue#40988 <http://tracker.ceph.com/issues/40988>`_, `pr#29362 <https://github.com/ceph/ceph/pull/29362>`_, JuanJose Galvez)
+* doc: Doc: update release schedule (`pr#28466 <https://github.com/ceph/ceph/pull/28466>`_, Torben H\xc3\xb8rup)
+* doc: docs: fix rgw_ldap_dnattr username token (`pr#27964 <https://github.com/ceph/ceph/pull/27964>`_, Thomas Kriechbaumer)
+* doc: docs: improve rgw ldap auth options (`pr#28157 <https://github.com/ceph/ceph/pull/28157>`_, Thomas Kriechbaumer)
+* doc: docs: rgw: fix bucket operation spelling: ListBucketMultipartUploads (`pr#28885 <https://github.com/ceph/ceph/pull/28885>`_, Thomas Kriechbaumer)
+* doc: docs: Update au.ceph.com maintainers, update README.md (`pr#32814 <https://github.com/ceph/ceph/pull/32814>`_, Matthew Taylor)
+* doc: Document Export Process during Subtree Migrations (`pr#30751 <https://github.com/ceph/ceph/pull/30751>`_, Sidharth Anupkrishnan)
+* doc: document mds journal event types (`issue#42190 <http://tracker.ceph.com/issues/42190>`_, `pr#30749 <https://github.com/ceph/ceph/pull/30749>`_, Venky Shankar)
+* doc: document mds journaling (`issue#41783 <http://tracker.ceph.com/issues/41783>`_, `pr#30396 <https://github.com/ceph/ceph/pull/30396>`_, Venky Shankar)
+* doc: document mode param for rbd mirror image enable command (`pr#32735 <https://github.com/ceph/ceph/pull/32735>`_, Mykola Golub)
+* doc: document rank option for journal reset (`pr#31201 <https://github.com/ceph/ceph/pull/31201>`_, Patrick Donnelly)
+* doc: document the new --addv argument (`issue#40568 <http://tracker.ceph.com/issues/40568>`_, `pr#28819 <https://github.com/ceph/ceph/pull/28819>`_, Luca Castoro)
+* doc: Documentation: Add missing ceph-volume lvm batch argument to ceph-volume.rst (`pr#29081 <https://github.com/ceph/ceph/pull/29081>`_, Andreas Krebs)
+* doc: Documentation: Centos ceph-deploys python dependencies (`pr#32591 <https://github.com/ceph/ceph/pull/32591>`_, Cl\xc3\xa9ment Hampa\xc3\xaf)
+* doc: documentation: Updated Dashboard Features, improved flow (`pr#33919 <https://github.com/ceph/ceph/pull/33919>`_, Lenz Grimmer)
+* doc: drop and update troubleshooting (`pr#28900 <https://github.com/ceph/ceph/pull/28900>`_, Jos Collin)
+* doc: emphasize the importance of require-osd-release nautilus (`pr#32587 <https://github.com/ceph/ceph/pull/32587>`_, Zac Dover)
+* doc: fix a typo in a command (`pr#32230 <https://github.com/ceph/ceph/pull/32230>`_, taeuk_kim)
+* doc: Fix a typo in balancer documentation (`pr#30210 <https://github.com/ceph/ceph/pull/30210>`_, Francois Deppierraz)
+* doc: fix boot transition in mds state diagram (`pr#27685 <https://github.com/ceph/ceph/pull/27685>`_, Patrick Donnelly)
+* doc: fix errors in search page and use relative address for releases.json (`pr#33423 <https://github.com/ceph/ceph/pull/33423>`_, Kefu Chai)
+* doc: Fix for new ceph-devel mailing list (`pr#29492 <https://github.com/ceph/ceph/pull/29492>`_, David Zafman)
+* doc: Fix FUSE expansion (`pr#30473 <https://github.com/ceph/ceph/pull/30473>`_, Sidharth Anupkrishnan)
+* doc: fix Getting Started with CephFS (`pr#32457 <https://github.com/ceph/ceph/pull/32457>`_, Jos Collin)
+* doc: fix links in developer_guide (`pr#32728 <https://github.com/ceph/ceph/pull/32728>`_, Rishabh Dave)
+* doc: fix LRC documentation (`pr#27106 <https://github.com/ceph/ceph/pull/27106>`_, Danny Al-Gaaf)
+* doc: fix parameter to set pg autoscale mode (`pr#27422 <https://github.com/ceph/ceph/pull/27422>`_, Changcheng Liu)
+* doc: Fix rbd namespace documentation (`pr#29445 <https://github.com/ceph/ceph/pull/29445>`_, Ricardo Marques)
+* doc: Fix the pg states and auto repair config options (`issue#38896 <http://tracker.ceph.com/issues/38896>`_, `pr#27143 <https://github.com/ceph/ceph/pull/27143>`_, David Zafman)
+* doc: fix typo (`pr#28888 <https://github.com/ceph/ceph/pull/28888>`_, Jos Collin)
+* doc: fix typo in doc/radosgw/layout.rst (`pr#29932 <https://github.com/ceph/ceph/pull/29932>`_, ypdai)
+* doc: fix typo to auto scale pg number (`pr#31065 <https://github.com/ceph/ceph/pull/31065>`_, Changcheng Liu)
+* doc: fix typos (`pr#30583 <https://github.com/ceph/ceph/pull/30583>`_, Michael Prokop)
+* doc: fix urls (`pr#29300 <https://github.com/ceph/ceph/pull/29300>`_, Jos Collin)
+* doc: fixed --read-only argument value in multisite doc (`pr#28655 <https://github.com/ceph/ceph/pull/28655>`_, Chenjiong Deng)
+* doc: fixed broken link in Swift Settings section (`pr#28774 <https://github.com/ceph/ceph/pull/28774>`_, James McClune)
+* doc: fixed broken links in nautilus release page (`pr#28074 <https://github.com/ceph/ceph/pull/28074>`_, James McClune)
+* doc: fixed broken reference link for Graphviz (`pr#32021 <https://github.com/ceph/ceph/pull/32021>`_, James McClune)
+* doc: fixed caps (`pr#27397 <https://github.com/ceph/ceph/pull/27397>`_, Servesha Dudhgaonkar)
+* doc: fixed telemetry module reference link (`pr#27624 <https://github.com/ceph/ceph/pull/27624>`_, James McClune)
+* doc: fixed typo in leadership names (`pr#27396 <https://github.com/ceph/ceph/pull/27396>`_, Servesha Dudhgaonkar)
+* doc: Fixes OSD node labels which based on the osd_devices name (`pr#23312 <https://github.com/ceph/ceph/pull/23312>`_, Siyu Sun)
+* doc: Fixes typo for ceph dashboard command (`pr#30292 <https://github.com/ceph/ceph/pull/30292>`_, Fabian Bonk)
+* doc: hide page contents for Ceph Internals (`pr#31046 <https://github.com/ceph/ceph/pull/31046>`_, Milind Changire)
+* doc: improve ceph-backport.sh comment block (`pr#28042 <https://github.com/ceph/ceph/pull/28042>`_, Nathan Cutler)
+* doc: improve developer guide doc (`pr#30435 <https://github.com/ceph/ceph/pull/30435>`_, Rishabh Dave)
+* doc: improve in mount.ceph man page (`pr#31024 <https://github.com/ceph/ceph/pull/31024>`_, Rishabh Dave)
+* doc: Improved the dashboard proxy config section (`pr#27581 <https://github.com/ceph/ceph/pull/27581>`_, Lenz Grimmer)
+* doc: indicate imperative mood for commit titles (`pr#29509 <https://github.com/ceph/ceph/pull/29509>`_, Patrick Donnelly)
+* doc: Make ceph-dashboard require grafana dashboards (`pr#28997 <https://github.com/ceph/ceph/pull/28997>`_, Boris Ranto)
+* doc: mds-config-ref: update mds_log_max_segments value (`pr#29412 <https://github.com/ceph/ceph/pull/29412>`_, Konstantin Shalygin)
+* doc: mention --namespace option in rados manpage (`pr#31871 <https://github.com/ceph/ceph/pull/31871>`_, Nathan Cutler)
+* doc: mgr/dashboard: Add frontend code documentation (`issue#36243 <http://tracker.ceph.com/issues/36243>`_, `pr#27433 <https://github.com/ceph/ceph/pull/27433>`_, Ernesto Puerta)
+* doc: mgr/dashboard: Document UiApiController with ApiController usage (`pr#29819 <https://github.com/ceph/ceph/pull/29819>`_, Stephan M\xc3\xbcller)
+* doc: mgr/dashboard: Extend Writing End-to-End Tests section (describe vs it) (`pr#29707 <https://github.com/ceph/ceph/pull/29707>`_, Adam King, Rafael Quintero)
+* doc: mgr/dashboard: fix hacking.rst (`pr#27222 <https://github.com/ceph/ceph/pull/27222>`_, Ernesto Puerta)
+* doc: mgr/dashboard: Fix link format to HACKING.rst (`pr#28897 <https://github.com/ceph/ceph/pull/28897>`_, Ernesto Puerta)
+* doc: mgr/dashboard: fix typos in HACKING.rst (`pr#30847 <https://github.com/ceph/ceph/pull/30847>`_, Ernesto Puerta)
+* doc: mgr/orchestrator: Add error handling to interface (`pr#26404 <https://github.com/ceph/ceph/pull/26404>`_, Sebastian Wagner)
+* doc: mgr/orchestrator: Fix disabling the orchestrator (`issue#40779 <http://tracker.ceph.com/issues/40779>`_, `pr#29042 <https://github.com/ceph/ceph/pull/29042>`_, Sebastian Wagner)
+* doc: mgr/orchestrator_cli: Update doc link in README (`pr#31731 <https://github.com/ceph/ceph/pull/31731>`_, Varsha Rao)
+* doc: mgr/ssh: HACKING.rst: Add Understanding `AsyncCompletion` (`pr#31967 <https://github.com/ceph/ceph/pull/31967>`_, Sebastian Wagner)
+* doc: mgr/ssh: update ssh-orch bootstrap guide (Vagrantfile & docs) (`pr#31457 <https://github.com/ceph/ceph/pull/31457>`_, Joshua Schmid)
+* doc: mgr/telemetry: force --license when sending while opted-out (`pr#33747 <https://github.com/ceph/ceph/pull/33747>`_, Yaarit Hatuka)
+* doc: minor fix in mount.ceph (`pr#32748 <https://github.com/ceph/ceph/pull/32748>`_, Rishabh Dave)
+* doc: Miscellaneous spelling fixes (`pr#27202 <https://github.com/ceph/ceph/pull/27202>`_, Bryan Stillwell)
+* doc: Modify nature theme (`pr#32312 <https://github.com/ceph/ceph/pull/32312>`_, Brad Hubbard)
+* doc: mon/OSDMonitor: Fix pool set target_size_bytes (etc) with unit suffix (`pr#30701 <https://github.com/ceph/ceph/pull/30701>`_, Prashant D)
+* doc: mounting CephFS subdirectory and Persistent Mounts cleanup (`pr#32498 <https://github.com/ceph/ceph/pull/32498>`_, Jos Collin)
+* doc: Move ceph-deploy docs to doc/install/ceph-deploy (`pr#33953 <https://github.com/ceph/ceph/pull/33953>`_, Sebastian Wagner)
+* doc: move cephadm files to its own directory (`pr#33551 <https://github.com/ceph/ceph/pull/33551>`_, Alexandra Settle, Sebastian Wagner)
+* doc: move Developer Guide to its own subdirectory (`pr#27159 <https://github.com/ceph/ceph/pull/27159>`_, Nathan Cutler)
+* doc: nautilus 14.2.2 release notes, take three (`pr#29171 <https://github.com/ceph/ceph/pull/29171>`_, Nathan Cutler)
+* doc: Nautilus mailmaps (`pr#27092 <https://github.com/ceph/ceph/pull/27092>`_, Abhishek Lekshmanan)
+* doc: note explicitly that profile rbd allows blacklisting (`pr#28296 <https://github.com/ceph/ceph/pull/28296>`_, Matthew Vernon)
+* doc: obsolete entries for allow_standby_replay (`pr#31897 <https://github.com/ceph/ceph/pull/31897>`_, Rodrigo Severo)
+* doc: operations: correct comma-delimited (`pr#29644 <https://github.com/ceph/ceph/pull/29644>`_, Anthony DAtri)
+* doc: operations: improve reweight-by-utilization (`pr#27657 <https://github.com/ceph/ceph/pull/27657>`_, Anthony DAtri)
+* doc: PendingReleaseNotes: 14.2.1 note on crush required version (`pr#27649 <https://github.com/ceph/ceph/pull/27649>`_, Sage Weil)
+* doc: PendingReleaseNotes: fix typo (`pr#31853 <https://github.com/ceph/ceph/pull/31853>`_, Sage Weil)
+* doc: PendingReleaseNotes: note on python3.6 changes (`issue#39164 <http://tracker.ceph.com/issues/39164>`_, `pr#27490 <https://github.com/ceph/ceph/pull/27490>`_, Kefu Chai)
+* doc: pg_num should always be a power of two (`pr#29364 <https://github.com/ceph/ceph/pull/29364>`_, Lars Marowsky-Bree, Kai Wagner)
+* doc: QAT Acceleration for Encryption and Compression (`pr#26967 <https://github.com/ceph/ceph/pull/26967>`_, Qiaowei Ren)
+* doc: quick-rbd.rst de-duplicate (`pr#32965 <https://github.com/ceph/ceph/pull/32965>`_, Tim)
+* doc: RBD exclusive locks (`pr#31893 <https://github.com/ceph/ceph/pull/31893>`_, Florian Haas)
+* doc: README.md: remove stale cmake prerequisite (`pr#32751 <https://github.com/ceph/ceph/pull/32751>`_, Kefu Chai)
+* doc: release note: Add pending release notes for already merged code (`pr#32041 <https://github.com/ceph/ceph/pull/32041>`_, David Zafman)
+* doc: release notes for 14.2.1 (`pr#27793 <https://github.com/ceph/ceph/pull/27793>`_, Abhishek Lekshmanan)
+* doc: release notes for Luminous v12.2.13 (`pr#33030 <https://github.com/ceph/ceph/pull/33030>`_, Nathan Cutler)
+* doc: release notes for nautilus 14.2.2 (`pr#29011 <https://github.com/ceph/ceph/pull/29011>`_, Sage Weil, Nathan Cutler)
+* doc: release notes for Nautilus 14.2.7 (`pr#33031 <https://github.com/ceph/ceph/pull/33031>`_, Nathan Cutler)
+* doc: release notes for v14.2.3 nautilus (`pr#29973 <https://github.com/ceph/ceph/pull/29973>`_, Abhishek Lekshmanan)
+* doc: release notes for v14.2.6 (`pr#32551 <https://github.com/ceph/ceph/pull/32551>`_, Abhishek Lekshmanan)
+* doc: releases/luminous: release notes for 12.2.12 (`pr#27553 <https://github.com/ceph/ceph/pull/27553>`_, Abhishek Lekshmanan)
+* doc: releases: 14.2.3 dashboard note (`pr#30145 <https://github.com/ceph/ceph/pull/30145>`_, Abhishek Lekshmanan)
+* doc: releases: v14.2.8 release notes (`pr#33670 <https://github.com/ceph/ceph/pull/33670>`_, Abhishek Lekshmanan)
+* doc: relicense LGPL-2.1 code as LGPL-2.1 or LGPL-3.0 (`pr#22446 <https://github.com/ceph/ceph/pull/22446>`_, Sage Weil)
+* doc: remove prod cluster examples from hardware recs (`pr#32670 <https://github.com/ceph/ceph/pull/32670>`_, Zac Dover)
+* doc: remove recommendation for kernel.pid_max (`pr#27965 <https://github.com/ceph/ceph/pull/27965>`_, Ben England)
+* doc: remove reference to obsolete scrub command (`pr#32508 <https://github.com/ceph/ceph/pull/32508>`_, Patrick Donnelly)
+* doc: remove the CephFS-Hadoop instructions (`pr#32980 <https://github.com/ceph/ceph/pull/32980>`_, Greg Farnum)
+* doc: removed OpenStack Kilo references in Keystone docs (`pr#27203 <https://github.com/ceph/ceph/pull/27203>`_, James McClune)
+* doc: removes kube-helm installation instructions (`pr#32009 <https://github.com/ceph/ceph/pull/32009>`_, Zac Dover)
+* doc: reorganize CephFS landing page and ToC (`pr#32038 <https://github.com/ceph/ceph/pull/32038>`_, Patrick Donnelly)
+* doc: Revert doc: do not add suffix for search result links (`pr#33562 <https://github.com/ceph/ceph/pull/33562>`_, Jason Dillaman)
+* doc: rgw/pubsub: add S3 compliant API to master zone (`pr#28971 <https://github.com/ceph/ceph/pull/28971>`_, Yuval Lifshitz)
+* doc: rgw/pubsub: clarify pubsub zone configuration (`pr#27493 <https://github.com/ceph/ceph/pull/27493>`_, Yuval Lifshitz)
+* doc: rgw/pubsub: fix topic arn. tenant support to multisite tests (`pr#27671 <https://github.com/ceph/ceph/pull/27671>`_, Yuval Lifshitz)
+* doc: rgw: Fixed bug on wrong name for user_id for OPA (`pr#31972 <https://github.com/ceph/ceph/pull/31972>`_, Seena Fallah)
+* doc: s/achieve/achieves/ (Fixed a verb disagreement) (`pr#32036 <https://github.com/ceph/ceph/pull/32036>`_, Zac Dover)
+* doc: script/ceph-backport.sh: add Troubleshooting notes (`pr#29948 <https://github.com/ceph/ceph/pull/29948>`_, Nathan Cutler)
+* doc: set ceph_perf_msgr_server arguments (`pr#29847 <https://github.com/ceph/ceph/pull/29847>`_, Changcheng Liu)
+* doc: show how to count jobs before triggering them (`pr#32145 <https://github.com/ceph/ceph/pull/32145>`_, Rishabh Dave)
+* doc: Show Jenkins commands (`pr#29423 <https://github.com/ceph/ceph/pull/29423>`_, Ernesto Puerta)
+* doc: Small update of SubmittingPatches-backports (`pr#31163 <https://github.com/ceph/ceph/pull/31163>`_, Laura Paduano)
+* doc: split up SubmittingPatches.rst (`issue#20953 <http://tracker.ceph.com/issues/20953>`_, `pr#30705 <https://github.com/ceph/ceph/pull/30705>`_, Nathan Cutler)
+* doc: Switch spelling of utilization (`pr#32537 <https://github.com/ceph/ceph/pull/32537>`_, Bryan Stillwell)
+* doc: tools/rados: add --pgid in help (`pr#30383 <https://github.com/ceph/ceph/pull/30383>`_, Vikhyat Umrao)
+* doc: typo fix in doc/dev/dev_cluster_deployement.rst: s/hostanme/hostname/ (`pr#31515 <https://github.com/ceph/ceph/pull/31515>`_, Drunkard Zhang)
+* doc: update --force flag to be precise (`pr#32343 <https://github.com/ceph/ceph/pull/32343>`_, Jos Collin)
+* doc: update adding an MDS (`pr#32291 <https://github.com/ceph/ceph/pull/32291>`_, Jos Collin)
+* doc: update and improve mounting with fuse/kernel docs (`pr#30754 <https://github.com/ceph/ceph/pull/30754>`_, Rishabh Dave)
+* doc: update bluestore cache settings and clarify data fraction (`issue#39522 <http://tracker.ceph.com/issues/39522>`_, `pr#27859 <https://github.com/ceph/ceph/pull/27859>`_, Jan Fajerski)
+* doc: update ceph ansible iscsi info (`pr#28665 <https://github.com/ceph/ceph/pull/28665>`_, Mike Christie)
+* doc: Update ceph-deploy docs from dumpling to nautilus (`pr#30269 <https://github.com/ceph/ceph/pull/30269>`_, Danny Abukalam)
+* doc: Update ceph-iscsi min version (`pr#29195 <https://github.com/ceph/ceph/pull/29195>`_, Ricardo Marques)
+* doc: update CephFS overview in introductory page (`pr#30014 <https://github.com/ceph/ceph/pull/30014>`_, Patrick Donnelly)
+* doc: update CephFS Quick Start doc (`pr#30406 <https://github.com/ceph/ceph/pull/30406>`_, Rishabh Dave)
+* doc: Update commands in bootstrap.rst (`pr#31800 <https://github.com/ceph/ceph/pull/31800>`_, Zac Dover)
+* doc: update default container images (`pr#33974 <https://github.com/ceph/ceph/pull/33974>`_, Sage Weil)
+* doc: Update documentation for LazyIO methods lazyio_synchronize() and lazyio_propagate() (`pr#29711 <https://github.com/ceph/ceph/pull/29711>`_, Sidharth Anupkrishnan)
+* doc: update documentation for the MANY_OBJECTS_PER_PG warning (`pr#27403 <https://github.com/ceph/ceph/pull/27403>`_, Vangelis Tasoulas)
+* doc: update documents on using kcephfs (`pr#30626 <https://github.com/ceph/ceph/pull/30626>`_, Jeff Layton)
+* doc: update erasure-code-profile.rst (`pr#33707 <https://github.com/ceph/ceph/pull/33707>`_, Guillaume Abrioux)
+* doc: Update link to Red Hat documentation (`pr#27976 <https://github.com/ceph/ceph/pull/27976>`_, Yaniv Kaul)
+* doc: update list of formats for --format flag for ceph pg dump (`pr#32373 <https://github.com/ceph/ceph/pull/32373>`_, Zac Dover)
+* doc: Update mailing lists (`pr#31666 <https://github.com/ceph/ceph/pull/31666>`_, hrchu)
+* doc: update mondb recovery script (`pr#28515 <https://github.com/ceph/ceph/pull/28515>`_, Hannes von Haugwitz)
+* doc: Update mount CephFS index (`pr#28955 <https://github.com/ceph/ceph/pull/28955>`_, Jos Collin)
+* doc: Update python-rtsli and tcmu-runner min versions (`pr#28494 <https://github.com/ceph/ceph/pull/28494>`_, Ricardo Marques)
+* doc: Update requirements for using CephFS (`pr#30251 <https://github.com/ceph/ceph/pull/30251>`_, Varsha Rao)
+* doc: update with osd addition (`pr#31244 <https://github.com/ceph/ceph/pull/31244>`_, Changcheng Liu)
+* doc: update with zone bucket and straw2 addition (`pr#31177 <https://github.com/ceph/ceph/pull/31177>`_, Changcheng Liu)
+* doc: update Zabbix template reference (`pr#33661 <https://github.com/ceph/ceph/pull/33661>`_, Mathijs Smit)
+* doc: updated ceph monitor config options (`pr#29982 <https://github.com/ceph/ceph/pull/29982>`_, James McClune)
+* doc: Updated dashboard iSCSI configuration, added labels (`pr#27074 <https://github.com/ceph/ceph/pull/27074>`_, Lenz Grimmer)
+* doc: updated OpenStack rbd documentation (`pr#28979 <https://github.com/ceph/ceph/pull/28979>`_, James McClune)
+* doc: updated OS recommendations and distro list (`pr#28643 <https://github.com/ceph/ceph/pull/28643>`_, Kai Wagner)
+* doc: Updates link to Sepia la (`pr#28780 <https://github.com/ceph/ceph/pull/28780>`_, Varsha Rao)
+* doc: use subsection for representing components in release notes (`pr#33940 <https://github.com/ceph/ceph/pull/33940>`_, Kefu Chai)
+* doc: use the console lexer for rendering command line sessions (`pr#32141 <https://github.com/ceph/ceph/pull/32141>`_, Kefu Chai)
+* do_cmake.sh: fedora-32 (rawhide) build with python-3.8 (`pr#32474 <https://github.com/ceph/ceph/pull/32474>`_, Kaleb S. Keithley)
+* errorator: improve general error handlers (`pr#33344 <https://github.com/ceph/ceph/pull/33344>`_, Samuel Just)
+* github/codeowners: Add orchestrator team (`pr#31441 <https://github.com/ceph/ceph/pull/31441>`_, Sebastian Wagner)
+* github: Add ceph-volume to list of jenkins commands (`pr#31191 <https://github.com/ceph/ceph/pull/31191>`_, Sebastian Wagner)
+* include/config-h.in.cmake: remove HAVE_XIO (`pr#28465 <https://github.com/ceph/ceph/pull/28465>`_, Kefu Chai)
+* include/utime: do not cast sec to time_t (`pr#27861 <https://github.com/ceph/ceph/pull/27861>`_, Kefu Chai)
+* include: buffer_raw.h: Copyright time fix (`pr#28481 <https://github.com/ceph/ceph/pull/28481>`_, Changcheng Liu)
+* install-deps.sh: remove failing error catching (`pr#29403 <https://github.com/ceph/ceph/pull/29403>`_, Ernesto Puerta)
+* Integrate PeeringState into crimson, fix related bugs (`pr#28180 <https://github.com/ceph/ceph/pull/28180>`_, Samuel Just)
+* krbd: do away with explicit memory management and other cleanups (`pr#31919 <https://github.com/ceph/ceph/pull/31919>`_, Ilya Dryomov)
+* librados: allow passing flags to operate sync APIs (`pr#33536 <https://github.com/ceph/ceph/pull/33536>`_, Yuval Lifshitz)
+* librados: fix leak in getxattr and getxattrs (`pr#32183 <https://github.com/ceph/ceph/pull/32183>`_, Adam Kupczyk)
+* librados: move buffer free functions to inline namespace (`issue#39972 <http://tracker.ceph.com/issues/39972>`_, `pr#28167 <https://github.com/ceph/ceph/pull/28167>`_, Jason Dillaman)
+* librados: prefer reinterpret_cast over c-style cast (`pr#33038 <https://github.com/ceph/ceph/pull/33038>`_, Kefu Chai)
+* librbd: add reference counting (`pr#30397 <https://github.com/ceph/ceph/pull/30397>`_, Mahati Chamarthy, Venky Shankar)
+* librbd: add snap_get_name and snap_get_id method API (`pr#31280 <https://github.com/ceph/ceph/pull/31280>`_, Zheng Yin)
+* librbd: added missing <string> include to PoolMetadata header (`pr#32614 <https://github.com/ceph/ceph/pull/32614>`_, Kaleb S. Keithley)
+* librbd: adjust the else-if conditions in validate_striping() (`pr#30053 <https://github.com/ceph/ceph/pull/30053>`_, mxdInspur)
+* librbd: always initialize local variables (`pr#31311 <https://github.com/ceph/ceph/pull/31311>`_, Kefu Chai)
+* librbd: always try to acquire exclusive lock when removing image (`pr#29775 <https://github.com/ceph/ceph/pull/29775>`_, Mykola Golub)
+* librbd: async open/close should free ImageCtx before issuing callback (`issue#39031 <http://tracker.ceph.com/issues/39031>`_, `pr#27682 <https://github.com/ceph/ceph/pull/27682>`_, Jason Dillaman)
+* librbd: avoid dereferencing an empty container during deep-copy (`issue#40368 <http://tracker.ceph.com/issues/40368>`_, `pr#28559 <https://github.com/ceph/ceph/pull/28559>`_, Jason Dillaman)
+* librbd: behave more gracefully when data pool removed (`pr#29613 <https://github.com/ceph/ceph/pull/29613>`_, Mykola Golub)
+* librbd: bump minor version to match octopus (`pr#32402 <https://github.com/ceph/ceph/pull/32402>`_, Jason Dillaman)
+* librbd: clean up unused variable (`pr#30019 <https://github.com/ceph/ceph/pull/30019>`_, mxdInspur)
+* librbd: clone copy-on-write operations should preserve sparseness (`pr#27999 <https://github.com/ceph/ceph/pull/27999>`_, Mykola Golub)
+* librbd: copyup read stats were incorrectly tied to child (`pr#27757 <https://github.com/ceph/ceph/pull/27757>`_, Jason Dillaman)
+* librbd: defer event socket completion until after callback issued (`pr#33994 <https://github.com/ceph/ceph/pull/33994>`_, Jason Dillaman)
+* librbd: diff iterate with fast-diff now correctly includes parent (`pr#32403 <https://github.com/ceph/ceph/pull/32403>`_, Jason Dillaman)
+* librbd: disable zero-copy writes by default (`pr#31794 <https://github.com/ceph/ceph/pull/31794>`_, Jason Dillaman)
+* librbd: dispatch delayed requests only if read intersects (`pr#27446 <https://github.com/ceph/ceph/pull/27446>`_, Mykola Golub)
+* librbd: do not allow to deep copy migrating image (`pr#27194 <https://github.com/ceph/ceph/pull/27194>`_, Mykola Golub)
+* librbd: do not unblock IO prior to growing object map during resize (`issue#39952 <http://tracker.ceph.com/issues/39952>`_, `pr#28295 <https://github.com/ceph/ceph/pull/28295>`_, Jason Dillaman)
+* librbd: dont call refresh from mirror::GetInfoRequest state machine (`pr#32734 <https://github.com/ceph/ceph/pull/32734>`_, Mykola Golub)
+* librbd: dont use complete_external_callback if ImageCtx destroyed (`pr#29263 <https://github.com/ceph/ceph/pull/29263>`_, Mykola Golub)
+* librbd: explicitly specify mode on mirror image enable (`pr#32217 <https://github.com/ceph/ceph/pull/32217>`_, Mykola Golub)
+* librbd: features converting bitmask and string API (`pr#31188 <https://github.com/ceph/ceph/pull/31188>`_, Zheng Yin)
+* librbd: finish write request early (`pr#32113 <https://github.com/ceph/ceph/pull/32113>`_, Li, Xiaoyan)
+* librbd: fix broken group snapshot handling (`pr#33448 <https://github.com/ceph/ceph/pull/33448>`_, Jason Dillaman)
+* librbd: fix build on freebsd (`pr#32938 <https://github.com/ceph/ceph/pull/32938>`_, Mykola Golub)
+* librbd: fix issues with object-map/fast-diff feature interlock (`issue#39521 <http://tracker.ceph.com/issues/39521>`_, `pr#28051 <https://github.com/ceph/ceph/pull/28051>`_, Jason Dillaman)
+* librbd: fix potential race conditions (`pr#33563 <https://github.com/ceph/ceph/pull/33563>`_, Mahati Chamarthy)
+* librbd: fix potential snapshot remove failure due to duplicate RPC messages (`pr#32760 <https://github.com/ceph/ceph/pull/32760>`_, Mykola Golub)
+* librbd: fix rbd_features_to_string output (`pr#31006 <https://github.com/ceph/ceph/pull/31006>`_, Zheng Yin)
+* librbd: fix rbd_open_by_id, rbd_open_by_id_read_only (`pr#32105 <https://github.com/ceph/ceph/pull/32105>`_, yangjun)
+* librbd: fix some edge cases for snapshot mirror mode promote (`pr#32567 <https://github.com/ceph/ceph/pull/32567>`_, Mykola Golub)
+* librbd: fix typo in deep_copy::ObjectCopyRequest::compute_read_ops (`pr#27049 <https://github.com/ceph/ceph/pull/27049>`_, Mykola Golub)
+* librbd: fixed several race conditions related to copyup (`issue#39021 <http://tracker.ceph.com/issues/39021>`_, `pr#27357 <https://github.com/ceph/ceph/pull/27357>`_, Jason Dillaman)
+* librbd: force reacquire lock if blacklist is disabled (`pr#30955 <https://github.com/ceph/ceph/pull/30955>`_, luo.runbing)
+* librbd: implement ordering for overlapping IOs (`pr#28952 <https://github.com/ceph/ceph/pull/28952>`_, Mahati Chamarthy)
+* librbd: improve journal performance to match expected degradation (`issue#40072 <http://tracker.ceph.com/issues/40072>`_, `pr#28539 <https://github.com/ceph/ceph/pull/28539>`_, Jason Dillaman)
+* librbd: improved support for balanced and localized reads (`pr#33493 <https://github.com/ceph/ceph/pull/33493>`_, Zheng Yin)
+* librbd: initial consolidation of internal locks (`pr#27756 <https://github.com/ceph/ceph/pull/27756>`_, Jason Dillaman)
+* librbd: introduce new default write-around cache policy (`pr#27229 <https://github.com/ceph/ceph/pull/27229>`_, Jason Dillaman)
+* librbd: leak on canceling simple io scheduler timer task (`pr#27755 <https://github.com/ceph/ceph/pull/27755>`_, Mykola Golub)
+* librbd: look for mirror peers in default namespace (`pr#32338 <https://github.com/ceph/ceph/pull/32338>`_, Mykola Golub)
+* librbd: look for pool metadata in default namespace (`pr#27151 <https://github.com/ceph/ceph/pull/27151>`_, Mykola Golub)
+* librbd: make flush be queued by QOS throttler (`pr#26931 <https://github.com/ceph/ceph/pull/26931>`_, Mykola Golub)
+* librbd: mirror image enable/disable should enable/disable journaling (`pr#28553 <https://github.com/ceph/ceph/pull/28553>`_, Mykola Golub)
+* librbd: optimize image copy state machine to use fast-diff (`pr#33867 <https://github.com/ceph/ceph/pull/33867>`_, Jason Dillaman)
+* librbd: optionally move parent image to trash on remove (`pr#27521 <https://github.com/ceph/ceph/pull/27521>`_, Mykola Golub)
+* librbd: prevent concurrent AIO callbacks to external clients (`issue#40417 <http://tracker.ceph.com/issues/40417>`_, `pr#28743 <https://github.com/ceph/ceph/pull/28743>`_, Jason Dillaman)
+* librbd: Remove duplicated AsyncOpTracker in librbd/Utils.h (`pr#29653 <https://github.com/ceph/ceph/pull/29653>`_, Xiaoyan Li)
+* librbd: remove pool objects when removing a namespace (`pr#32401 <https://github.com/ceph/ceph/pull/32401>`_, Jason Dillaman)
+* librbd: shared read-only cache hook (`pr#27285 <https://github.com/ceph/ceph/pull/27285>`_, Dehao Shang, Yuan Zhou)
+* librbd: silence -Wunused-variable warnings (`pr#27513 <https://github.com/ceph/ceph/pull/27513>`_, David Disseldorp)
+* librbd: simple scheduler plugin for object dispatcher layer (`pr#26675 <https://github.com/ceph/ceph/pull/26675>`_, Mykola Golub)
+* librbd: snapshot object maps can go inconsistent during copyup (`issue#39435 <http://tracker.ceph.com/issues/39435>`_, `pr#27724 <https://github.com/ceph/ceph/pull/27724>`_, Ilya Dryomov)
+* librbd: support compression allocation hints to the OSD (`pr#32687 <https://github.com/ceph/ceph/pull/32687>`_, Jason Dillaman)
+* librbd: support EC data pool images sparsify (`pr#27268 <https://github.com/ceph/ceph/pull/27268>`_, Mykola Golub)
+* librbd: support zero-copy writes via the C API (`pr#27895 <https://github.com/ceph/ceph/pull/27895>`_, Jason Dillaman)
+* librbd: trash move return EBUSY instead of EINVAL for migrating image (`pr#27136 <https://github.com/ceph/ceph/pull/27136>`_, Mykola Golub)
+* librbd: tweak deep-copy to avoid creating last snapshot until sync is complete (`pr#33097 <https://github.com/ceph/ceph/pull/33097>`_, Jason Dillaman)
+* librbd: tweaks to increase IOPS and reduce CPU usage (`pr#28044 <https://github.com/ceph/ceph/pull/28044>`_, Jason Dillaman)
+* librbd: use custom allocator for aligned boost::lockfree::queue (`issue#39703 <http://tracker.ceph.com/issues/39703>`_, `pr#28093 <https://github.com/ceph/ceph/pull/28093>`_, Jason Dillaman)
+* librbd: v1 clones are restricted to the same namespace (`pr#30711 <https://github.com/ceph/ceph/pull/30711>`_, Jason Dillaman)
+* librbd: when unlinking peer from mirror snaps do it in all namespaces (`pr#32463 <https://github.com/ceph/ceph/pull/32463>`_, Mykola Golub)
+* librbd:move all snapshot API functions in internal.cc over to api/Snapshot.cc (`pr#31589 <https://github.com/ceph/ceph/pull/31589>`_, Zheng Yin)
+* log: avoid logging anything when log_to_file=false (`pr#27133 <https://github.com/ceph/ceph/pull/27133>`_, Sage Weil)
+* log: fix store_statfs log line (`pr#28564 <https://github.com/ceph/ceph/pull/28564>`_, Mohamad Gebai)
+* log: just return if t is empty (`pr#31243 <https://github.com/ceph/ceph/pull/31243>`_, Xiubo Li)
+* log: print pthread ID / name mapping in recent events dump (`pr#32354 <https://github.com/ceph/ceph/pull/32354>`_, Radoslaw Zarzynski)
+* lvm deactivate command (`pr#32179 <https://github.com/ceph/ceph/pull/32179>`_, Jan Fajerski)
+* mds: add command that config individual client session (`issue#40811 <http://tracker.ceph.com/issues/40811>`_, `pr#29104 <https://github.com/ceph/ceph/pull/29104>`_, Yan, Zheng)
+* mds: add config to require forward to auth MDS (`pr#29995 <https://github.com/ceph/ceph/pull/29995>`_, simon gao)
+* mds: add configurable snapshot limit (`pr#30710 <https://github.com/ceph/ceph/pull/30710>`_, Milind Changire)
+* mds: add perf counter for finisher of MDSRank (`pr#29377 <https://github.com/ceph/ceph/pull/29377>`_, simon gao)
+* mds: add perf counters for openfiletable (`pr#33363 <https://github.com/ceph/ceph/pull/33363>`_, Milind Changire)
+* mds: add scrub_info_t into mempool (`pr#33180 <https://github.com/ceph/ceph/pull/33180>`_, Jun Su)
+* mds: answering all pending getattr/lookups targeting the same inode in one go (`issue#36608 <http://tracker.ceph.com/issues/36608>`_, `pr#24794 <https://github.com/ceph/ceph/pull/24794>`_, Patrick Donnelly, Xuehan Xu)
+* mds: apply configuration changes through MDSRank (`pr#28951 <https://github.com/ceph/ceph/pull/28951>`_, Patrick Donnelly)
+* mds: async dir operation support (`pr#27866 <https://github.com/ceph/ceph/pull/27866>`_, Yan, Zheng)
+* mds: async dirop support (`pr#32816 <https://github.com/ceph/ceph/pull/32816>`_, Yan, Zheng)
+* mds: avoid check session connections features when issuing caps (`pr#26881 <https://github.com/ceph/ceph/pull/26881>`_, Yan, Zheng)
+* mds: avoid revoking Fsx from loner during directory fragmentation (`pr#26817 <https://github.com/ceph/ceph/pull/26817>`_, Yan, Zheng)
+* mds: avoid sending too many osd requests at once after mds restarts (`issue#40028 <http://tracker.ceph.com/issues/40028>`_, `pr#27436 <https://github.com/ceph/ceph/pull/27436>`_, simon gao)
+* mds: better output of ceph health detail when some client is failing to advance oldest client/flush tid (`issue#39266 <http://tracker.ceph.com/issues/39266>`_, `pr#27537 <https://github.com/ceph/ceph/pull/27537>`_, Shen Hang)
+* mds: check dir fragment to split dir if mkdir makes it oversized (`pr#27480 <https://github.com/ceph/ceph/pull/27480>`_, Erqi Chen)
+* mds: check directory split after rename (`issue#38994 <http://tracker.ceph.com/issues/38994>`_, `pr#27214 <https://github.com/ceph/ceph/pull/27214>`_, Shen Hang)
+* mds: clarify comment (`pr#31401 <https://github.com/ceph/ceph/pull/31401>`_, Patrick Donnelly)
+* mds: cleanup truncating inodes when standby replay mds trim log segments (`pr#28686 <https://github.com/ceph/ceph/pull/28686>`_, Yan, Zheng)
+* mds: cleanup unneeded client_snap_caps when splitting snap inode (`issue#39987 <http://tracker.ceph.com/issues/39987>`_, `pr#28190 <https://github.com/ceph/ceph/pull/28190>`_, Yan, Zheng)
+* mds: complete all the replay op when mds is restarted (`issue#40784 <http://tracker.ceph.com/issues/40784>`_, `pr#29059 <https://github.com/ceph/ceph/pull/29059>`_, Shen Hang)
+* mds: convert unnecessary usage of std::list to std::vector (`pr#26895 <https://github.com/ceph/ceph/pull/26895>`_, Patrick Donnelly)
+* mds: count purge queue items left in journal (`issue#40121 <http://tracker.ceph.com/issues/40121>`_, `pr#28376 <https://github.com/ceph/ceph/pull/28376>`_, Zhi Zhang)
+* mds: delay exporting directory whose pin value exceeds max rank id (`issue#40603 <http://tracker.ceph.com/issues/40603>`_, `pr#28804 <https://github.com/ceph/ceph/pull/28804>`_, Zhi Zhang)
+* mds: display scrub status in ceph status (`pr#28855 <https://github.com/ceph/ceph/pull/28855>`_, Venky Shankar)
+* mds: do not include metric_spec in MClientSession from MDS (`pr#32659 <https://github.com/ceph/ceph/pull/32659>`_, Patrick Donnelly)
+* mds: dont add metadata to session close message (`pr#32318 <https://github.com/ceph/ceph/pull/32318>`_, Yan, Zheng)
+* mds: dont mark cap NEEDSNAPFLUSH if client has no pending capsnap (`pr#28551 <https://github.com/ceph/ceph/pull/28551>`_, Yan, Zheng)
+* mds: dont print subtrees if they are too big or too many (`pr#26056 <https://github.com/ceph/ceph/pull/26056>`_, Rishabh Dave)
+* mds: dont respond getattr with -EROFS when mds is readonly (`pr#32676 <https://github.com/ceph/ceph/pull/32676>`_, Yan, Zheng)
+* mds: drive cap recall while dropping cache (`pr#30389 <https://github.com/ceph/ceph/pull/30389>`_, Patrick Donnelly)
+* mds: evict an unresponsive client only when another client wants its caps (`issue#17854 <http://tracker.ceph.com/issues/17854>`_, `pr#22645 <https://github.com/ceph/ceph/pull/22645>`_, Rishabh Dave)
+* mds: execute PurgeQueue on_error handler in finisher (`pr#29064 <https://github.com/ceph/ceph/pull/29064>`_, Yan, Zheng)
+* mds: fix assert(omap_num_objs <= MAX_OBJECTS) of OpenFileTable (`pr#32020 <https://github.com/ceph/ceph/pull/32020>`_, Yan, Zheng)
+* mds: fix bug of batch getattr/lookup (`pr#32268 <https://github.com/ceph/ceph/pull/32268>`_, Yan, Zheng)
+* mds: fix can wrlock check in Locker::acquire_locks() (`pr#33005 <https://github.com/ceph/ceph/pull/33005>`_, Yan, Zheng)
+* mds: fix infinite loop in Locker::file_update_finish (`pr#29902 <https://github.com/ceph/ceph/pull/29902>`_, Yan, Zheng)
+* mds: fix InoTable::force_consume_to() (`pr#29411 <https://github.com/ceph/ceph/pull/29411>`_, Yan, Zheng)
+* mds: fix invalid access of mdr->dn[0].back() (`pr#31534 <https://github.com/ceph/ceph/pull/31534>`_, Yan, Zheng)
+* mds: fix is session in blacklist check in Server::apply_blacklist() (`issue#40061 <http://tracker.ceph.com/issues/40061>`_, `pr#28293 <https://github.com/ceph/ceph/pull/28293>`_, Yan, Zheng)
+* mds: Fix MDCache.h reorder compiler warnings (`pr#31409 <https://github.com/ceph/ceph/pull/31409>`_, Varsha Rao)
+* mds: fix null pointer dereference in Server::handle_client_link() (`pr#32722 <https://github.com/ceph/ceph/pull/32722>`_, Yan, Zheng)
+* mds: fix revoking caps after after stale->resume circle (`pr#31662 <https://github.com/ceph/ceph/pull/31662>`_, Yan, Zheng)
+* mds: fix SnapRealm::resolve_snapname for long name (`pr#27511 <https://github.com/ceph/ceph/pull/27511>`_, Yan, Zheng)
+* mds: fix use-after-free in Migrater (`pr#33291 <https://github.com/ceph/ceph/pull/33291>`_, Yan, Zheng)
+* mds: handle bad purge queue item encoding (`pr#33449 <https://github.com/ceph/ceph/pull/33449>`_, Yan, Zheng)
+* mds: handle ceph_assert on blacklisting (`pr#33662 <https://github.com/ceph/ceph/pull/33662>`_, Milind Changire)
+* mds: increase default cache memory limit to 4G (`pr#32042 <https://github.com/ceph/ceph/pull/32042>`_, Patrick Donnelly)
+* mds: initialize cap_revoke_eviction_timeout with conf (`issue#38844 <http://tracker.ceph.com/issues/38844>`_, `pr#26970 <https://github.com/ceph/ceph/pull/26970>`_, simon gao)
+* mds: initialize the monc later in init() (`pr#31715 <https://github.com/ceph/ceph/pull/31715>`_, Xiubo Li)
+* mds: just delete MDSIOContextBase during shutdown (`pr#33538 <https://github.com/ceph/ceph/pull/33538>`_, Patrick Donnelly)
+* mds: maintain client provided metric flags in client metadata (`pr#32201 <https://github.com/ceph/ceph/pull/32201>`_, Venky Shankar)
+* mds: make mds-mds per-message versioned (`issue#12107 <http://tracker.ceph.com/issues/12107>`_, `pr#20160 <https://github.com/ceph/ceph/pull/20160>`_, dongdong tao)
+* mds: make MDSIOContextBase delete itself when shutting down (`pr#29752 <https://github.com/ceph/ceph/pull/29752>`_, Xuehan Xu)
+* mds: mds returns -5(EIO) error when the deleted file does not exist (`pr#30403 <https://github.com/ceph/ceph/pull/30403>`_, huanwen ren)
+* mds: move some MDCache member init to header (`pr#29543 <https://github.com/ceph/ceph/pull/29543>`_, Patrick Donnelly)
+* mds: no assert on frozen dir when scrub path (`pr#30835 <https://github.com/ceph/ceph/pull/30835>`_, Zhi Zhang)
+* mds: note client features when rejecting client (`pr#32505 <https://github.com/ceph/ceph/pull/32505>`_, Patrick Donnelly)
+* mds: obsoleting mds_cache_size (`pr#31729 <https://github.com/ceph/ceph/pull/31729>`_, Patrick Donnelly, Ramana Raja)
+* mds: optimize function, fragset_t::simplify, to improve the efficiency of  merging fragment (`pr#31595 <https://github.com/ceph/ceph/pull/31595>`_, simon gao)
+* mds: output lock state in format dump (`issue#39645 <http://tracker.ceph.com/issues/39645>`_, `pr#27717 <https://github.com/ceph/ceph/pull/27717>`_, Zhi Zhang)
+* mds: pass proper MutationImpl::LockOp to Locker::wrlock_start() (`pr#33719 <https://github.com/ceph/ceph/pull/33719>`_, Yan, Zheng)
+* mds: preparation for async dir operation support (`pr#30972 <https://github.com/ceph/ceph/pull/30972>`_, Yan, Zheng)
+* mds: properly evaluate unstable locks when evicting client (`pr#31548 <https://github.com/ceph/ceph/pull/31548>`_, Yan, Zheng)
+* mds: recall caps from quiescent sessions (`pr#28702 <https://github.com/ceph/ceph/pull/28702>`_, Patrick Donnelly)
+* mds: register with mgr only after added to FSMap (`pr#31400 <https://github.com/ceph/ceph/pull/31400>`_, Patrick Donnelly)
+* mds: reject sessionless messages (`pr#29594 <https://github.com/ceph/ceph/pull/29594>`_, Xiao Guodong)
+* mds: release free heap pages after trim (`pr#31793 <https://github.com/ceph/ceph/pull/31793>`_, Patrick Donnelly)
+* mds: relevel debug message levels for balancer/migrator (`pr#33471 <https://github.com/ceph/ceph/pull/33471>`_, Patrick Donnelly)
+* mds: remove dead get_commands code (`pr#33390 <https://github.com/ceph/ceph/pull/33390>`_, Patrick Donnelly)
+* mds: remove duplicated check on balance amount (`pr#27087 <https://github.com/ceph/ceph/pull/27087>`_, Zhi Zhang)
+* mds: remove superfluous error in StrayManager::advance_delayed() (`issue#38679 <http://tracker.ceph.com/issues/38679>`_, `pr#27051 <https://github.com/ceph/ceph/pull/27051>`_, Yan, Zheng)
+* mds: remove the code that skip evicting the only client (`pr#28642 <https://github.com/ceph/ceph/pull/28642>`_, Yan, Zheng)
+* mds: remove the incorrect comments (`pr#31775 <https://github.com/ceph/ceph/pull/31775>`_, Xiubo Li)
+* mds: remove unnecessary debug warning (`pr#31898 <https://github.com/ceph/ceph/pull/31898>`_, Patrick Donnelly)
+* mds: remove unused CDir members (`pr#33227 <https://github.com/ceph/ceph/pull/33227>`_, Jun Su)
+* mds: Reorganize class members in Anchor header (`pr#30090 <https://github.com/ceph/ceph/pull/30090>`_, Varsha Rao)
+* mds: Reorganize class members in Capability header (`pr#29166 <https://github.com/ceph/ceph/pull/29166>`_, Varsha Rao)
+* mds: Reorganize class members in CDir header (`pr#28860 <https://github.com/ceph/ceph/pull/28860>`_, Varsha Rao)
+* mds: Reorganize class members in CInode header (`pr#29066 <https://github.com/ceph/ceph/pull/29066>`_, Varsha Rao)
+* mds: Reorganize class members in DamageTable header (`pr#29569 <https://github.com/ceph/ceph/pull/29569>`_, Varsha Rao)
+* mds: Reorganize class members in FSMap header (`pr#29572 <https://github.com/ceph/ceph/pull/29572>`_, Varsha Rao)
+* mds: Reorganize class members in FSMapUser header (`pr#29574 <https://github.com/ceph/ceph/pull/29574>`_, Varsha Rao)
+* mds: Reorganize class members in InoTable header (`pr#29883 <https://github.com/ceph/ceph/pull/29883>`_, Varsha Rao)
+* mds: Reorganize class members in JournalPointer header (`pr#29888 <https://github.com/ceph/ceph/pull/29888>`_, Varsha Rao)
+* mds: Reorganize class members in LocalLock header (`pr#30143 <https://github.com/ceph/ceph/pull/30143>`_, Varsha Rao)
+* mds: Reorganize class members in Locker header (`pr#30164 <https://github.com/ceph/ceph/pull/30164>`_, Varsha Rao)
+* mds: Reorganize class members in LogEvent header (`pr#30205 <https://github.com/ceph/ceph/pull/30205>`_, Varsha Rao)
+* mds: Reorganize class members in LogSegment header (`pr#30202 <https://github.com/ceph/ceph/pull/30202>`_, Varsha Rao)
+* mds: Reorganize class members in MDBalancer header (`pr#30559 <https://github.com/ceph/ceph/pull/30559>`_, Varsha Rao)
+* mds: Reorganize class members in MDCache header (`pr#30745 <https://github.com/ceph/ceph/pull/30745>`_, Varsha Rao)
+* mds: Reorganize class members in MDLog header (`pr#30744 <https://github.com/ceph/ceph/pull/30744>`_, Varsha Rao)
+* mds: Reorganize class members in MDSAuthCaps header (`pr#30915 <https://github.com/ceph/ceph/pull/30915>`_, Varsha Rao)
+* mds: Reorganize class members in MDSCacheObject header (`pr#30938 <https://github.com/ceph/ceph/pull/30938>`_, Varsha Rao)
+* mds: Reorganize class members in MDSDaemon header (`pr#30990 <https://github.com/ceph/ceph/pull/30990>`_, Varsha Rao)
+* mds: Reorganize class members in MDSMap header (`pr#31118 <https://github.com/ceph/ceph/pull/31118>`_, Varsha Rao)
+* mds: Reorganize class members in MDSRank header (`pr#31120 <https://github.com/ceph/ceph/pull/31120>`_, Varsha Rao)
+* mds: Reorganize class members in MDSTable header (`pr#31122 <https://github.com/ceph/ceph/pull/31122>`_, Varsha Rao)
+* mds: Reorganize class members in MDSTableClient header (`pr#31115 <https://github.com/ceph/ceph/pull/31115>`_, Varsha Rao)
+* mds: Reorganize class members in MDSTableServer header (`pr#31250 <https://github.com/ceph/ceph/pull/31250>`_, Varsha Rao)
+* mds: Reorganize class members in Migrator header (`pr#31253 <https://github.com/ceph/ceph/pull/31253>`_, Varsha Rao)
+* mds: Reorganize class members in OpenFileTable header (`pr#31597 <https://github.com/ceph/ceph/pull/31597>`_, Varsha Rao)
+* mds: Reorganize class members in PurgeQueue header (`pr#31596 <https://github.com/ceph/ceph/pull/31596>`_, Varsha Rao)
+* mds: Reorganize class members in RecoveryQueue header (`pr#31635 <https://github.com/ceph/ceph/pull/31635>`_, Varsha Rao)
+* mds: Reorganize class members in ScatterLock header (`pr#31716 <https://github.com/ceph/ceph/pull/31716>`_, Varsha Rao)
+* mds: Reorganize class members in ScrubHeader header (`pr#31717 <https://github.com/ceph/ceph/pull/31717>`_, Varsha Rao)
+* mds: Reorganize class members in ScrubStack header (`pr#31718 <https://github.com/ceph/ceph/pull/31718>`_, Varsha Rao)
+* mds: Reorganize class members in Server header (`pr#31719 <https://github.com/ceph/ceph/pull/31719>`_, Varsha Rao)
+* mds: Reorganize class members in SessionMap header (`pr#32320 <https://github.com/ceph/ceph/pull/32320>`_, Varsha Rao)
+* mds: Reorganize class members in SimpleLock header (`pr#32322 <https://github.com/ceph/ceph/pull/32322>`_, Varsha Rao)
+* mds: Reorganize class members in SnapClient header (`pr#32326 <https://github.com/ceph/ceph/pull/32326>`_, Varsha Rao)
+* mds: Reorganize class members in SnapServer header (`pr#32350 <https://github.com/ceph/ceph/pull/32350>`_, Varsha Rao)
+* mds: Reorganize struct members in Mutation header (`pr#31481 <https://github.com/ceph/ceph/pull/31481>`_, Varsha Rao)
+* mds: Reorganize structure and class members in mdstypes header (`pr#32435 <https://github.com/ceph/ceph/pull/32435>`_, Varsha Rao)
+* mds: Reorganize structure members in flock header (`pr#32416 <https://github.com/ceph/ceph/pull/32416>`_, Varsha Rao)
+* mds: Reorganize structure members in inode_backtrace header (`pr#32431 <https://github.com/ceph/ceph/pull/32431>`_, Varsha Rao)
+* mds: Reorganize structure members in snap header (`pr#32432 <https://github.com/ceph/ceph/pull/32432>`_, Varsha Rao)
+* mds: Reorganize structure members in SnapRealm header (`pr#32348 <https://github.com/ceph/ceph/pull/32348>`_, Varsha Rao)
+* mds: Reorganize structure members in StrayManager header (`pr#32397 <https://github.com/ceph/ceph/pull/32397>`_, Varsha Rao)
+* mds: reset heartbeat inside big loop (`pr#28406 <https://github.com/ceph/ceph/pull/28406>`_, Yan, Zheng)
+* mds: split the dir if the op makes it oversized, because some ops maybe in flight (`pr#29921 <https://github.com/ceph/ceph/pull/29921>`_, simon gao)
+* mds: there is an assertion when calling Beacon::shutdown() (`issue#38822 <http://tracker.ceph.com/issues/38822>`_, `pr#27063 <https://github.com/ceph/ceph/pull/27063>`_, huanwen ren)
+* mds: throttle scrub start for multiple active MDS (`pr#32521 <https://github.com/ceph/ceph/pull/32521>`_, Patrick Donnelly, Milind Changire)
+* mds: tolerate no snaprealm encoded in on-disk root inode (`pr#31455 <https://github.com/ceph/ceph/pull/31455>`_, Yan, Zheng)
+* mds: track high water mark for purges (`pr#32667 <https://github.com/ceph/ceph/pull/32667>`_, Patrick Donnelly)
+* mds: trim cache during standby-replay (`issue#40213 <http://tracker.ceph.com/issues/40213>`_, `pr#28212 <https://github.com/ceph/ceph/pull/28212>`_, simon gao)
+* mds: trim cache on regular schedule (`pr#29542 <https://github.com/ceph/ceph/pull/29542>`_, Patrick Donnelly)
+* mds: unify daemon and tell commands (`pr#31255 <https://github.com/ceph/ceph/pull/31255>`_, Sage Weil)
+* mds: update projected_version when upgrading snaptable (`issue#38835 <http://tracker.ceph.com/issues/38835>`_, `pr#27238 <https://github.com/ceph/ceph/pull/27238>`_, Yan, Zheng)
+* mds: use set to store to evict client (`pr#30029 <https://github.com/ceph/ceph/pull/30029>`_, Erqi Chen)
+* mds: use vector::empty in feature_bitset_t (`pr#32541 <https://github.com/ceph/ceph/pull/32541>`_, Jos Collin)
+* mds: wake up lock waiters after forcibly changing lock state (`issue#39987 <http://tracker.ceph.com/issues/39987>`_, `pr#28459 <https://github.com/ceph/ceph/pull/28459>`_, Yan, Zheng)
+* mgr,mon,rbd: mon/mgr: add rbd_support to list of always-on mgr modules (`issue#40790 <http://tracker.ceph.com/issues/40790>`_, `pr#29073 <https://github.com/ceph/ceph/pull/29073>`_, Jason Dillaman)
+* mgr,mon: mon,mgr: pass MessageRef to monc.send_mon_message()  \xe2\x80\xa6 (`pr#30449 <https://github.com/ceph/ceph/pull/30449>`_, Kefu Chai)
+* mgr,mon: mon/MgrMonitor.cc: add always_on_modules to the output of ceph mgr module ls (`pr#32939 <https://github.com/ceph/ceph/pull/32939>`_, Neha Ojha)
+* mgr,mon: mon/MgrMonitor.cc: warn about missing mgr in a cluster with osds (`pr#33025 <https://github.com/ceph/ceph/pull/33025>`_, Neha Ojha)
+* mgr,pybind: pybind/mgr/prometheus: remove scrape_duration metric (`pr#27034 <https://github.com/ceph/ceph/pull/27034>`_, Jan Fajerski)
+* mgr,rbd: mgr/dashboard: block mirroring page results in internal server error (`pr#31907 <https://github.com/ceph/ceph/pull/31907>`_, Jason Dillaman)
+* mgr,rbd: mgr/rbd_support: dont scan pools that dont have schedules (`pr#33840 <https://github.com/ceph/ceph/pull/33840>`_, Mykola Golub)
+* mgr,rbd: mgr/rbd_support: implement mirror snapshot scheduler (`pr#32434 <https://github.com/ceph/ceph/pull/32434>`_, Mykola Golub)
+* mgr,rbd: mgr/rbd_support: support scheduling long-running background operations (`issue#40621 <http://tracker.ceph.com/issues/40621>`_, `pr#29054 <https://github.com/ceph/ceph/pull/29054>`_, Jason Dillaman)
+* mgr,rbd: pybind/mgr: fix format for rbd-mirror prometheus metrics (`pr#28200 <https://github.com/ceph/ceph/pull/28200>`_, Mykola Golub)
+* mgr,rgw: mgr/ansible: RGW service (`pr#28468 <https://github.com/ceph/ceph/pull/28468>`_, Juan Miguel Olmo Mart\xc3\xadnez)
+* mgr,tests: install-deps.sh: preload wheel for all mgr requirements.txt files (`pr#32151 <https://github.com/ceph/ceph/pull/32151>`_, Sage Weil)
+* mgr,tests: mgr/orchestrator_cli: remove tox and move test to parent dir (`pr#31561 <https://github.com/ceph/ceph/pull/31561>`_, Sebastian Wagner)
+* mgr,tests: mgr/progress: Created first unit test for progress module (`pr#28758 <https://github.com/ceph/ceph/pull/28758>`_, Kamoltat (Junior) Sirivadhna)
+* mgr,tests: pybind/mgr: Add ceph_module.pyi to improve type checking (`pr#32502 <https://github.com/ceph/ceph/pull/32502>`_, Sebastian Wagner)
+* mgr,tests: pybind/mgr: install setuptools >= 12 (`pr#29414 <https://github.com/ceph/ceph/pull/29414>`_, Kefu Chai)
+* mgr,tests: pybind/tox: handle possible WITH_PYTHON3 values other than 3 (`pr#28002 <https://github.com/ceph/ceph/pull/28002>`_, Nathan Cutler)
+* mgr,tests: qa/mgr/balancer: Add cram based test for altering target_max_misplaced_ratio setting (`pr#30646 <https://github.com/ceph/ceph/pull/30646>`_, Shyukri Shyukriev)
+* mgr,tests: qa/mgr/progress: update the test suite for progress module (`issue#40618 <http://tracker.ceph.com/issues/40618>`_, `pr#29111 <https://github.com/ceph/ceph/pull/29111>`_, Kamoltat (Junior) Sirivadhna)
+* mgr,tools: Remove use of rules batching for upmap balancer and default for upmap_max_deviation to 5 (`pr#32247 <https://github.com/ceph/ceph/pull/32247>`_, David Zafman)
+* mgr/ansible: Host ls implementation (`pr#26185 <https://github.com/ceph/ceph/pull/26185>`_, Juan Miguel Olmo Mart\xc3\xadnez)
+* mgr/ansible: Integrate mgr/ansible/tox into mgr/tox (`pr#32149 <https://github.com/ceph/ceph/pull/32149>`_, Sebastian Wagner)
+* mgr/ansible: TLS Mutual Authentication (`pr#27512 <https://github.com/ceph/ceph/pull/27512>`_, Juan Miguel Olmo Mart\xc3\xadnez)
+* mgr/cephadm: a few fixes around daemon and device caches (`pr#33495 <https://github.com/ceph/ceph/pull/33495>`_, Sage Weil)
+* mgr/cephadm: adapt osd deployment to service_apply (`pr#33922 <https://github.com/ceph/ceph/pull/33922>`_, Sage Weil, Joshua Schmid)
+* mgr/cephadm: add drivegroup support; workaround c-v batch shortcoming (`pr#32972 <https://github.com/ceph/ceph/pull/32972>`_, Sage Weil, Joshua Schmid)
+* mgr/cephadm: add HostAssignment.validate() (`pr#34005 <https://github.com/ceph/ceph/pull/34005>`_, Sebastian Wagner)
+* mgr/cephadm: Add progress to update_mgr() (`pr#32372 <https://github.com/ceph/ceph/pull/32372>`_, Sebastian Wagner)
+* mgr/cephadm: Add unittest for osd removal (`pr#33602 <https://github.com/ceph/ceph/pull/33602>`_, Sage Weil, Sebastian Wagner)
+* mgr/cephadm: Add unittest for service_action (`pr#32209 <https://github.com/ceph/ceph/pull/32209>`_, Sebastian Wagner)
+* mgr/cephadm: allow osd replacement/removal in the background (`pr#32983 <https://github.com/ceph/ceph/pull/32983>`_, Joshua Schmid)
+* mgr/cephadm: auto-select python version to use remotely (`pr#32327 <https://github.com/ceph/ceph/pull/32327>`_, Sage Weil)
+* mgr/cephadm: cache device inventory; zap (`pr#33394 <https://github.com/ceph/ceph/pull/33394>`_, Sage Weil)
+* mgr/cephadm: catch exceptions when scraping ceph-volume inventory (`pr#33484 <https://github.com/ceph/ceph/pull/33484>`_, Sage Weil)
+* mgr/cephadm: catch excpetions in serve() thread (`pr#33139 <https://github.com/ceph/ceph/pull/33139>`_, Sage Weil)
+* mgr/cephadm: check-host on host add (`pr#32385 <https://github.com/ceph/ceph/pull/32385>`_, Sage Weil)
+* mgr/cephadm: clean up client.crash.\\* container_image settings after upgrade (`pr#34068 <https://github.com/ceph/ceph/pull/34068>`_, Sage Weil)
+* mgr/cephadm: consolidate/refactor all add\\_ and apply\\_ methods (`pr#33496 <https://github.com/ceph/ceph/pull/33496>`_, Sage Weil)
+* mgr/cephadm: Convert HostNotFound to OrchestratorError (`pr#33310 <https://github.com/ceph/ceph/pull/33310>`_, Sebastian Wagner)
+* mgr/cephadm: deploy Grafana (`pr#33515 <https://github.com/ceph/ceph/pull/33515>`_, Patrick Seidensal)
+* mgr/cephadm: do not include osd service in orch ls output (`pr#33968 <https://github.com/ceph/ceph/pull/33968>`_, Sage Weil)
+* mgr/cephadm: do not reconfig orphan daemons; fix test to not remote orphans (`pr#34027 <https://github.com/ceph/ceph/pull/34027>`_, Sage Weil)
+* mgr/cephadm: do not refresh daemon and device inventory as often (`pr#33734 <https://github.com/ceph/ceph/pull/33734>`_, Sage Weil)
+* mgr/cephadm: drop mixin parent (`pr#33514 <https://github.com/ceph/ceph/pull/33514>`_, Sage Weil)
+* mgr/cephadm: Enable provisioning alertmanager via orchestrator (`pr#33554 <https://github.com/ceph/ceph/pull/33554>`_, Kristoffer Gr\xc3\xb6nlund)
+* mgr/cephadm: fix dump output by formatting to yaml first (`pr#33891 <https://github.com/ceph/ceph/pull/33891>`_, Joshua Schmid)
+* mgr/cephadm: fix listing services by host (`pr#32314 <https://github.com/ceph/ceph/pull/32314>`_, Kiefer Chang)
+* mgr/cephadm: fix orch rm and upgrade (`pr#33772 <https://github.com/ceph/ceph/pull/33772>`_, Sage Weil)
+* mgr/cephadm: fix osd reconfig/redeploy (`pr#32812 <https://github.com/ceph/ceph/pull/32812>`_, Sage Weil)
+* mgr/cephadm: Fix placement for new services (`pr#33205 <https://github.com/ceph/ceph/pull/33205>`_, Sebastian Wagner)
+* mgr/cephadm: fix placement when existing + specified dont overlap (`pr#33766 <https://github.com/ceph/ceph/pull/33766>`_, Sage Weil)
+* mgr/cephadm: fix prom config generation when hosts have no labels or addrs (`pr#33800 <https://github.com/ceph/ceph/pull/33800>`_, Sage Weil)
+* mgr/cephadm: Fix remove_osds() (`pr#32146 <https://github.com/ceph/ceph/pull/32146>`_, Sebastian Wagner)
+* mgr/cephadm: fix section name for mon options in ceph.conf (`pr#32681 <https://github.com/ceph/ceph/pull/32681>`_, Sage Weil)
+* mgr/cephadm: fix service list filtering (`pr#33838 <https://github.com/ceph/ceph/pull/33838>`_, Kiefer Chang)
+* mgr/cephadm: fix type of timeout options (`pr#32316 <https://github.com/ceph/ceph/pull/32316>`_, Kiefer Chang)
+* mgr/cephadm: fix upgrade ok-to-stop condition check (`pr#33469 <https://github.com/ceph/ceph/pull/33469>`_, Sage Weil)
+* mgr/cephadm: fix upgrade order (`pr#33811 <https://github.com/ceph/ceph/pull/33811>`_, Sage Weil)
+* mgr/cephadm: fix upgrade wait loop (`pr#33447 <https://github.com/ceph/ceph/pull/33447>`_, Sage Weil)
+* mgr/cephadm: fix upgrade when daemon is stopped (`pr#33678 <https://github.com/ceph/ceph/pull/33678>`_, Sage Weil)
+* mgr/cephadm: if we had no record of deps, and deps are [], do not reconfig (`pr#33733 <https://github.com/ceph/ceph/pull/33733>`_, Sage Weil)
+* mgr/cephadm: implement apply mon, mon removal checks (`pr#33792 <https://github.com/ceph/ceph/pull/33792>`_, Sage Weil)
+* mgr/cephadm: implement pause/resume to suspect non-monitoring background work (`pr#33930 <https://github.com/ceph/ceph/pull/33930>`_, Sage Weil)
+* mgr/cephadm: improve pull behavior for upgrade (`pr#32878 <https://github.com/ceph/ceph/pull/32878>`_, Sage Weil)
+* mgr/cephadm: init attrs created by settattr() (`pr#32957 <https://github.com/ceph/ceph/pull/32957>`_, Kefu Chai)
+* mgr/cephadm: leverage service specs (`pr#33553 <https://github.com/ceph/ceph/pull/33553>`_, Sage Weil, Joshua Schmid)
+* mgr/cephadm: limit number of times check host is performed in the serve loop (`pr#33866 <https://github.com/ceph/ceph/pull/33866>`_, Daniel-Pivonka)
+* mgr/cephadm: log information to cluster log (`pr#33488 <https://github.com/ceph/ceph/pull/33488>`_, Sage Weil)
+* mgr/cephadm: make apply move daemons, do its work synchronously (`pr#33704 <https://github.com/ceph/ceph/pull/33704>`_, Sage Weil)
+* mgr/cephadm: make NodeAssignment return a simple host list (`pr#33669 <https://github.com/ceph/ceph/pull/33669>`_, Sage Weil)
+* mgr/cephadm: make osd create on an existing LV idempotent (`pr#33755 <https://github.com/ceph/ceph/pull/33755>`_, Sage Weil)
+* mgr/cephadm: make prometheus scrape all mgrs, node-exporters (`pr#33444 <https://github.com/ceph/ceph/pull/33444>`_, Sage Weil)
+* mgr/cephadm: Make sure we dont co-locate the same daemon (`pr#33853 <https://github.com/ceph/ceph/pull/33853>`_, Sebastian Wagner)
+* mgr/cephadm: misc fixes (`pr#33119 <https://github.com/ceph/ceph/pull/33119>`_, Sage Weil)
+* mgr/cephadm: misc fixes + smoke test (`pr#33730 <https://github.com/ceph/ceph/pull/33730>`_, Sage Weil)
+* mgr/cephadm: mon: Dont show traceback for user errors (`pr#33333 <https://github.com/ceph/ceph/pull/33333>`_, Sebastian Wagner)
+* mgr/cephadm: nicer error from cephadm check-host (`pr#33935 <https://github.com/ceph/ceph/pull/33935>`_, Sage Weil)
+* mgr/cephadm: point dashboard at cephadms grafana automatically (`pr#33700 <https://github.com/ceph/ceph/pull/33700>`_, Sage Weil)
+* mgr/cephadm: prefix daemon ids with hostname (`pr#33012 <https://github.com/ceph/ceph/pull/33012>`_, Sage Weil)
+* mgr/cephadm: progress for upgrade (`pr#33415 <https://github.com/ceph/ceph/pull/33415>`_, Sage Weil)
+* mgr/cephadm: provision node-exporter (`pr#33123 <https://github.com/ceph/ceph/pull/33123>`_, Sage Weil, Patrick Seidensal)
+* mgr/cephadm: provision prometheus (`pr#33073 <https://github.com/ceph/ceph/pull/33073>`_, Sage Weil)
+* mgr/cephadm: reduce boilerplate for unittests (`pr#33663 <https://github.com/ceph/ceph/pull/33663>`_, Joshua Schmid)
+* mgr/cephadm: refresh ceph.conf when mons change (`pr#33855 <https://github.com/ceph/ceph/pull/33855>`_, Sage Weil)
+* mgr/cephadm: refresh configs when dependencies change (`pr#33671 <https://github.com/ceph/ceph/pull/33671>`_, Sage Weil)
+* mgr/cephadm: refresh service state in the background (`pr#32859 <https://github.com/ceph/ceph/pull/32859>`_, Sebastian Wagner, Sage Weil)
+* mgr/cephadm: remove item from cache when removing (`pr#33071 <https://github.com/ceph/ceph/pull/33071>`_, Sage Weil)
+* mgr/cephadm: remove redundant /dev when blinking device light (`pr#32246 <https://github.com/ceph/ceph/pull/32246>`_, Sage Weil)
+* mgr/cephadm: revamp scheduling (`pr#33523 <https://github.com/ceph/ceph/pull/33523>`_, Sage Weil)
+* mgr/cephadm: set thread pool size to 10 (`pr#33463 <https://github.com/ceph/ceph/pull/33463>`_, Sebastian Wagner)
+* mgr/cephadm: show age of service ls (`pr#32686 <https://github.com/ceph/ceph/pull/32686>`_, Sage Weil)
+* mgr/cephadm: simplify and improve placement (`pr#33808 <https://github.com/ceph/ceph/pull/33808>`_, Sage Weil)
+* mgr/cephadm: simplify tracking of daemon inventory (`pr#33249 <https://github.com/ceph/ceph/pull/33249>`_, Sage Weil)
+* mgr/cephadm: two minor fixes (`pr#33736 <https://github.com/ceph/ceph/pull/33736>`_, Sage Weil)
+* mgr/cephadm: update osd removal report immediately (`pr#33713 <https://github.com/ceph/ceph/pull/33713>`_, Kiefer Chang)
+* mgr/cephadm: update type annotation (`pr#33784 <https://github.com/ceph/ceph/pull/33784>`_, Kefu Chai)
+* mgr/cephadm: upgrade requires root mode for now (`pr#33802 <https://github.com/ceph/ceph/pull/33802>`_, Sage Weil)
+* mgr/cephadm: upgrade: fix daemons missing image_id (`pr#33745 <https://github.com/ceph/ceph/pull/33745>`_, Sage Weil)
+* mgr/cephadm: upgrade: handle stopped daemons (`pr#33487 <https://github.com/ceph/ceph/pull/33487>`_, Sage Weil)
+* mgr/cephadm: verify hosts hostname matches cephadm host (`pr#33058 <https://github.com/ceph/ceph/pull/33058>`_, Sage Weil)
+* mgr/dashbaord: Fix E2E pools page failure (`pr#32635 <https://github.com/ceph/ceph/pull/32635>`_, Stephan M\xc3\xbcller)
+* mgr/dashboad: Improve iSCSI overview page (`pr#27254 <https://github.com/ceph/ceph/pull/27254>`_, Ricardo Marques)
+* mgr/dashboard Displays progress bar in notification tray for background tasks (`pr#27420 <https://github.com/ceph/ceph/pull/27420>`_, Pooja)
+* mgr/dashboard/qa: Improve tasks.mgr.test_dashboard.TestDashboard.test_standby (`pr#26925 <https://github.com/ceph/ceph/pull/26925>`_, Volker Theile)
+* mgr/dashboard/qa: Increase timeout for test_disable (tasks.mgr.dashboard.test_mgr_module.MgrModuleTelemetryTest) (`pr#27187 <https://github.com/ceph/ceph/pull/27187>`_, Volker Theile)
+* mgr/dashboard: 1 osds exist in the crush map but not in the osdmap breaks OSD page (`issue#36086 <http://tracker.ceph.com/issues/36086>`_, `pr#26836 <https://github.com/ceph/ceph/pull/26836>`_, Patrick Nawracay)
+* mgr/dashboard: A block-manager can not access the pool page (`pr#30001 <https://github.com/ceph/ceph/pull/30001>`_, Volker Theile)
+* mgr/dashboard: accept expected exception when SSL handshaking (`pr#31014 <https://github.com/ceph/ceph/pull/31014>`_, Kefu Chai)
+* mgr/dashboard: Access control database does not restore disabled users correctly (`pr#29614 <https://github.com/ceph/ceph/pull/29614>`_, Volker Theile)
+* mgr/dashboard: adapt bucket tenant API tests to new behaviour (`pr#29570 <https://github.com/ceph/ceph/pull/29570>`_, alfonsomthd)
+* mgr/dashboard: adapt create_osds interface change (`pr#34000 <https://github.com/ceph/ceph/pull/34000>`_, Kiefer Chang)
+* mgr/dashboard: Add Always-on column to mgr module list (`pr#33429 <https://github.com/ceph/ceph/pull/33429>`_, Volker Theile)
+* mgr/dashboard: Add date range and log search functionality (`issue#37387 <http://tracker.ceph.com/issues/37387>`_, `pr#26562 <https://github.com/ceph/ceph/pull/26562>`_, guodan1)
+* mgr/dashboard: add debug mode (`pr#30522 <https://github.com/ceph/ceph/pull/30522>`_, Ernesto Puerta)
+* mgr/dashboard: add feature toggle for NFS and fix feature toggles regression (`pr#32419 <https://github.com/ceph/ceph/pull/32419>`_, Ernesto Puerta)
+* mgr/dashboard: Add invalid pattern message for Pool name (`pr#31607 <https://github.com/ceph/ceph/pull/31607>`_, Tiago Melo)
+* mgr/dashboard: Add missing text translation (`pr#29934 <https://github.com/ceph/ceph/pull/29934>`_, Volker Theile)
+* mgr/dashboard: Add polish translation (`pr#27247 <https://github.com/ceph/ceph/pull/27247>`_, Sebastian Krah)
+* mgr/dashboard: Add protractor-screenshoter-plugin (`pr#27166 <https://github.com/ceph/ceph/pull/27166>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Add refresh  interval to the dashboard landing page (`issue#26872 <http://tracker.ceph.com/issues/26872>`_, `pr#26396 <https://github.com/ceph/ceph/pull/26396>`_, guodan1)
+* mgr/dashboard: Add separate option to config SSL port (`pr#26914 <https://github.com/ceph/ceph/pull/26914>`_, Volker Theile)
+* mgr/dashboard: Add support for blinking enclosure LEDs (`pr#31851 <https://github.com/ceph/ceph/pull/31851>`_, Volker Theile)
+* mgr/dashboard: Add time-diff unittest and docs (`pr#31357 <https://github.com/ceph/ceph/pull/31357>`_, Volker Theile)
+* mgr/dashboard: Add vertical menu (`pr#31923 <https://github.com/ceph/ceph/pull/31923>`_, Tiago Melo)
+* mgr/dashboard: Add whitelist to guard (`pr#27406 <https://github.com/ceph/ceph/pull/27406>`_, Ernesto Puerta)
+* mgr/dashboard: Allow deletion of RBD with snapshots (`pr#33067 <https://github.com/ceph/ceph/pull/33067>`_, Tiago Melo)
+* mgr/dashboard: Allow disabling redirection on standby Dashboards (`pr#29088 <https://github.com/ceph/ceph/pull/29088>`_, Volker Theile)
+* mgr/dashboard: allow refreshing inventory page (`pr#32423 <https://github.com/ceph/ceph/pull/32423>`_, Kiefer Chang)
+* mgr/dashboard: Allow users to change their password on the UI (`pr#28935 <https://github.com/ceph/ceph/pull/28935>`_, Volker Theile)
+* mgr/dashboard: auth ttl expired error (`pr#27098 <https://github.com/ceph/ceph/pull/27098>`_, ming416)
+* mgr/dashboard: Back button component (`pr#27164 <https://github.com/ceph/ceph/pull/27164>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: behave when pwdUpdateRequired key is missing (`pr#33513 <https://github.com/ceph/ceph/pull/33513>`_, Sage Weil)
+* mgr/dashboard: Bucket names cannot be formatted as IP address (`pr#30620 <https://github.com/ceph/ceph/pull/30620>`_, Volker Theile)
+* mgr/dashboard: ceph dashboard i18ntool (`pr#26953 <https://github.com/ceph/ceph/pull/26953>`_, Sebastian Krah)
+* mgr/dashboard: CephFS client tab switch (`pr#29556 <https://github.com/ceph/ceph/pull/29556>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: CephFS tab component (`pr#29800 <https://github.com/ceph/ceph/pull/29800>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Change the provider of services to root (`issue#39996 <http://tracker.ceph.com/issues/39996>`_, `pr#28211 <https://github.com/ceph/ceph/pull/28211>`_, Tiago Melo)
+* mgr/dashboard: change warn_explicit to warn (`pr#30075 <https://github.com/ceph/ceph/pull/30075>`_, Ernesto Puerta)
+* mgr/dashboard: Check if gateway is in use before deletion (`pr#27262 <https://github.com/ceph/ceph/pull/27262>`_, Ricardo Marques)
+* mgr/dashboard: Check if `num_sessions` is available (`pr#30270 <https://github.com/ceph/ceph/pull/30270>`_, Ricardo Marques)
+* mgr/dashboard: cheroot moved into a separate project (`pr#31431 <https://github.com/ceph/ceph/pull/31431>`_, Joshua Schmid)
+* mgr/dashboard: Cleanup code (`pr#33107 <https://github.com/ceph/ceph/pull/33107>`_, Volker Theile)
+* mgr/dashboard: Cleanup feature toggle status output (`pr#32569 <https://github.com/ceph/ceph/pull/32569>`_, Volker Theile)
+* mgr/dashboard: Cleanup Python code (`pr#29604 <https://github.com/ceph/ceph/pull/29604>`_, Volker Theile)
+* mgr/dashboard: Clone an existing user role (`pr#32653 <https://github.com/ceph/ceph/pull/32653>`_, Volker Theile)
+* mgr/dashboard: commands to set SSL certificate and key (`pr#27463 <https://github.com/ceph/ceph/pull/27463>`_, Ricardo Dias)
+* mgr/dashboard: Configuring an URL prefix does not work as expected (`pr#30599 <https://github.com/ceph/ceph/pull/30599>`_, Volker Theile)
+* mgr/dashboard: consider mon_allow_pool_delete flag (`pr#28260 <https://github.com/ceph/ceph/pull/28260>`_, Tatjana Dehler)
+* mgr/dashboard: Controls UI inputs based on type (`pr#30208 <https://github.com/ceph/ceph/pull/30208>`_, Ricardo Marques)
+* mgr/dashboard: coverage venv python version same as mgr (`pr#33407 <https://github.com/ceph/ceph/pull/33407>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Create bucket with x-amz-bucket-object-lock-enabled (`pr#33821 <https://github.com/ceph/ceph/pull/33821>`_, Volker Theile)
+* mgr/dashboard: Crush rule modal (`pr#33620 <https://github.com/ceph/ceph/pull/33620>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: decouple backend unit tests from build (`pr#32565 <https://github.com/ceph/ceph/pull/32565>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: destroyed view in CRUSH map viewer (`pr#33405 <https://github.com/ceph/ceph/pull/33405>`_, Avan Thakkar)
+* mgr/dashboard: Disable event propagation in the helper icon (`issue#40715 <http://tracker.ceph.com/issues/40715>`_, `pr#29105 <https://github.com/ceph/ceph/pull/29105>`_, Tiago Melo)
+* mgr/dashboard: Display correct dialog title (`pr#28168 <https://github.com/ceph/ceph/pull/28168>`_, Volker Theile)
+* mgr/dashboard: Display iSCSI logged in info (`pr#28265 <https://github.com/ceph/ceph/pull/28265>`_, Ricardo Marques)
+* mgr/dashboard: Display legend for CephFS standbys (`pr#29927 <https://github.com/ceph/ceph/pull/29927>`_, Volker Theile)
+* mgr/dashboard: display OSD IDs on inventory page (`pr#31189 <https://github.com/ceph/ceph/pull/31189>`_, Kiefer Chang)
+* mgr/dashboard: Display the number of iSCSI active sessions (`pr#27248 <https://github.com/ceph/ceph/pull/27248>`_, Ricardo Marques)
+* mgr/dashboard: Display WWN and LUN number in iSCSI target details (`pr#30288 <https://github.com/ceph/ceph/pull/30288>`_, Ricardo Marques)
+* mgr/dashboard: do not log tokens (`pr#30445 <https://github.com/ceph/ceph/pull/30445>`_, Kefu Chai)
+* mgr/dashboard: do not show RGW API keys if only read-only privileges (`pr#33178 <https://github.com/ceph/ceph/pull/33178>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Editing RGW bucket fails because of name is already in use (`pr#29767 <https://github.com/ceph/ceph/pull/29767>`_, Volker Theile)
+* mgr/dashboard: Enable compiler options used by Angular --strict flag (`pr#32553 <https://github.com/ceph/ceph/pull/32553>`_, Tiago Melo)
+* mgr/dashboard: Enable read only users to read again (`pr#27348 <https://github.com/ceph/ceph/pull/27348>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: enable/disable versioning on RGW bucket (`pr#29460 <https://github.com/ceph/ceph/pull/29460>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Enforce password change upon first login (`pr#32680 <https://github.com/ceph/ceph/pull/32680>`_, Volker Theile, Tatjana Dehler)
+* mgr/dashboard: Enhance user create CLI command to force password change (`pr#33552 <https://github.com/ceph/ceph/pull/33552>`_, Volker Theile)
+* mgr/dashboard: Evict a CephFS client (`pr#28898 <https://github.com/ceph/ceph/pull/28898>`_, Ricardo Marques)
+* mgr/dashboard: Explicitly set/change the device class of an OSD (`pr#32150 <https://github.com/ceph/ceph/pull/32150>`_, Ricardo Marques)
+* mgr/dashboard: Extend E2E test section (`pr#28858 <https://github.com/ceph/ceph/pull/28858>`_, Laura Paduano)
+* mgr/dashboard: extend types of `smart` response (`pr#30595 <https://github.com/ceph/ceph/pull/30595>`_, Patrick Seidensal)
+* mgr/dashboard: fix adding/removing host errors (`pr#34023 <https://github.com/ceph/ceph/pull/34023>`_, Kiefer Chang)
+* mgr/dashboard: fix backend error when updating RBD interlocked features (`issue#39933 <http://tracker.ceph.com/issues/39933>`_, `pr#28147 <https://github.com/ceph/ceph/pull/28147>`_, Kiefer Chang)
+* mgr/dashboard: fix cdEncode decorator is not working on class (`pr#30064 <https://github.com/ceph/ceph/pull/30064>`_, Kiefer Chang)
+* mgr/dashboard: Fix CephFS chart (`pr#29557 <https://github.com/ceph/ceph/pull/29557>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Fix dashboard health test failure (`pr#29172 <https://github.com/ceph/ceph/pull/29172>`_, Ricardo Marques)
+* mgr/dashboard: Fix deletion of NFS protocol properties (`issue#38997 <http://tracker.ceph.com/issues/38997>`_, `pr#27244 <https://github.com/ceph/ceph/pull/27244>`_, Tiago Melo)
+* mgr/dashboard: Fix deletion of NFS transports properties (`issue#39090 <http://tracker.ceph.com/issues/39090>`_, `pr#27350 <https://github.com/ceph/ceph/pull/27350>`_, Tiago Melo)
+* mgr/dashboard: Fix e2e chromedriver problem (`pr#32224 <https://github.com/ceph/ceph/pull/32224>`_, Tiago Melo)
+* mgr/dashboard: Fix env vars of `run-tox.sh` (`issue#38798 <http://tracker.ceph.com/issues/38798>`_, `pr#26977 <https://github.com/ceph/ceph/pull/26977>`_, Patrick Nawracay)
+* mgr/dashboard: Fix error in unit test caused by timezone (`pr#31632 <https://github.com/ceph/ceph/pull/31632>`_, Tiago Melo)
+* mgr/dashboard: fix failing user test (`pr#32461 <https://github.com/ceph/ceph/pull/32461>`_, Tatjana Dehler)
+* mgr/dashboard: fix improper URL checking (`pr#32652 <https://github.com/ceph/ceph/pull/32652>`_, Ernesto Puerta)
+* mgr/dashboard: Fix iSCSI + Rook issues (`issue#39586 <http://tracker.ceph.com/issues/39586>`_, `pr#26341 <https://github.com/ceph/ceph/pull/26341>`_, Sebastian Wagner)
+* mgr/dashboard: Fix iSCSI Discovery user permissions (`issue#39328 <http://tracker.ceph.com/issues/39328>`_, `pr#27678 <https://github.com/ceph/ceph/pull/27678>`_, Tiago Melo)
+* mgr/dashboard: Fix iSCSI disk diff calculation (`pr#27378 <https://github.com/ceph/ceph/pull/27378>`_, Ricardo Marques)
+* mgr/dashboard: Fix iSCSI form when using IPv6 (`pr#27946 <https://github.com/ceph/ceph/pull/27946>`_, Ricardo Marques)
+* mgr/dashboard: Fix iSCSI target form warning (`issue#39324 <http://tracker.ceph.com/issues/39324>`_, `pr#27609 <https://github.com/ceph/ceph/pull/27609>`_, Tiago Melo)
+* mgr/dashboard: Fix iSCSI target submission (`pr#27380 <https://github.com/ceph/ceph/pull/27380>`_, Ricardo Marques)
+* mgr/dashboard: Fix issues in user form (`pr#28863 <https://github.com/ceph/ceph/pull/28863>`_, Volker Theile)
+* mgr/dashboard: fix LazyUUID4 not serializable (`pr#31266 <https://github.com/ceph/ceph/pull/31266>`_, Ernesto Puerta)
+* mgr/dashboard: fix MDS counter chart is not displayed (`pr#29371 <https://github.com/ceph/ceph/pull/29371>`_, Kiefer Chang)
+* mgr/dashboard: fix mgr module API tests (`pr#29634 <https://github.com/ceph/ceph/pull/29634>`_, alfonsomthd, Kefu Chai)
+* mgr/dashboard: fix missing constraints file in backend API tests (`pr#30720 <https://github.com/ceph/ceph/pull/30720>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Fix missing i18n (`pr#32650 <https://github.com/ceph/ceph/pull/32650>`_, Volker Theile)
+* mgr/dashboard: Fix mypy issues and enable it by default (`pr#33454 <https://github.com/ceph/ceph/pull/33454>`_, Volker Theile)
+* mgr/dashboard: Fix NFS pseudo validation (`issue#39063 <http://tracker.ceph.com/issues/39063>`_, `pr#27293 <https://github.com/ceph/ceph/pull/27293>`_, Tiago Melo)
+* mgr/dashboard: Fix NFS squash default value (`issue#39064 <http://tracker.ceph.com/issues/39064>`_, `pr#27294 <https://github.com/ceph/ceph/pull/27294>`_, Tiago Melo)
+* mgr/dashboard: Fix npm vulnerabilities (`pr#32699 <https://github.com/ceph/ceph/pull/32699>`_, Tiago Melo)
+* mgr/dashboard: Fix OSD IDs are not displayed when using cephadm backend (`pr#32207 <https://github.com/ceph/ceph/pull/32207>`_, Kiefer Chang)
+* mgr/dashboard: Fix pool deletion e2e (`pr#29993 <https://github.com/ceph/ceph/pull/29993>`_, Volker Theile)
+* mgr/dashboard: Fix pool renaming functionality (`pr#31617 <https://github.com/ceph/ceph/pull/31617>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: fix python2 failure in home controller (`pr#30937 <https://github.com/ceph/ceph/pull/30937>`_, Ricardo Dias)
+* mgr/dashboard: fix RGW subuser auto-generate key (`pr#32186 <https://github.com/ceph/ceph/pull/32186>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Fix RGW user/bucket quota issues (`pr#28174 <https://github.com/ceph/ceph/pull/28174>`_, Volker Theile)
+* mgr/dashboard: fix SAML input argument handling (`pr#29848 <https://github.com/ceph/ceph/pull/29848>`_, Ernesto Puerta)
+* mgr/dashboard: fix small typos in description message (`pr#30647 <https://github.com/ceph/ceph/pull/30647>`_, Tatjana Dehler)
+* mgr/dashboard: fix some performance data are not displayed (`issue#39971 <http://tracker.ceph.com/issues/39971>`_, `pr#28169 <https://github.com/ceph/ceph/pull/28169>`_, Kiefer Chang)
+* mgr/dashboard: fix sparkline component (`pr#26985 <https://github.com/ceph/ceph/pull/26985>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: fix tasks.mgr.dashboard.test_rgw suite (`pr#33718 <https://github.com/ceph/ceph/pull/33718>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Fix the table mouseenter event handling test (`pr#28879 <https://github.com/ceph/ceph/pull/28879>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: fix tox test failure (`pr#29125 <https://github.com/ceph/ceph/pull/29125>`_, Kiefer Chang)
+* mgr/dashboard: Fix translation of variables (`pr#30671 <https://github.com/ceph/ceph/pull/30671>`_, Tiago Melo)
+* mgr/dashboard: Fix typo in NFS form (`issue#39067 <http://tracker.ceph.com/issues/39067>`_, `pr#27245 <https://github.com/ceph/ceph/pull/27245>`_, Tiago Melo)
+* mgr/dashboard: fix visibility of pwdExpirationDate field (`pr#32703 <https://github.com/ceph/ceph/pull/32703>`_, Tatjana Dehler)
+* mgr/dashboard: Fix zsh support in run-backend-api-tests.sh (`pr#31070 <https://github.com/ceph/ceph/pull/31070>`_, Sebastian Wagner)
+* mgr/dashboard: Fix `npm run fixmod` command (`pr#28176 <https://github.com/ceph/ceph/pull/28176>`_, Patrick Nawracay)
+* mgr/dashboard: Fixes defaultBuilder is not a function (`pr#29420 <https://github.com/ceph/ceph/pull/29420>`_, Ricardo Marques)
+* mgr/dashboard: Fixes random cephfs tab test failure (`pr#30814 <https://github.com/ceph/ceph/pull/30814>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Fixes rbd image purge trash button & modal text (`pr#33321 <https://github.com/ceph/ceph/pull/33321>`_, anurag)
+* mgr/dashboard: Fixes tooltip behavior (`pr#27153 <https://github.com/ceph/ceph/pull/27153>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: FixtureHelper (`pr#27157 <https://github.com/ceph/ceph/pull/27157>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Form fields do not show error messages/hints (`pr#29043 <https://github.com/ceph/ceph/pull/29043>`_, Volker Theile)
+* mgr/dashboard: ganesha: Specify the name of the filesystem (create_path) (`pr#29182 <https://github.com/ceph/ceph/pull/29182>`_, David Casier)
+* mgr/dashboard: hide daemon table when orchestrator is disabled (`pr#33941 <https://github.com/ceph/ceph/pull/33941>`_, Kiefer Chang)
+* mgr/dashboard: hide in-use devices when creating OSDs (`pr#31927 <https://github.com/ceph/ceph/pull/31927>`_, Kiefer Chang)
+* mgr/dashboard: improve device selection modal for creating OSDs (`pr#33081 <https://github.com/ceph/ceph/pull/33081>`_, Kiefer Chang)
+* mgr/dashboard: Improve hints shown when message.xlf is invalid (`issue#40064 <http://tracker.ceph.com/issues/40064>`_, `pr#28377 <https://github.com/ceph/ceph/pull/28377>`_, Patrick Nawracay)
+* mgr/dashboard: Improve NFS Pseudo pattern message (`issue#39327 <http://tracker.ceph.com/issues/39327>`_, `pr#27653 <https://github.com/ceph/ceph/pull/27653>`_, Tiago Melo)
+* mgr/dashboard: Improve Notification sidebar (`pr#32895 <https://github.com/ceph/ceph/pull/32895>`_, Tiago Melo)
+* mgr/dashboard: Improve RestClient error logging (`pr#29794 <https://github.com/ceph/ceph/pull/29794>`_, Volker Theile)
+* mgr/dashboard: Increase column size on mgr module form (`pr#29107 <https://github.com/ceph/ceph/pull/29107>`_, Ricardo Marques)
+* mgr/dashboard: install teuthology using pip (`pr#31815 <https://github.com/ceph/ceph/pull/31815>`_, Kefu Chai)
+* mgr/dashboard: internationalization support with AOT enabled (`pr#30694 <https://github.com/ceph/ceph/pull/30694>`_, Tiago Melo, Ricardo Dias)
+* mgr/dashboard: Invalid SSO configuration when certificate path does not exist (`pr#31920 <https://github.com/ceph/ceph/pull/31920>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI GET requests should not be logged (`pr#27813 <https://github.com/ceph/ceph/pull/27813>`_, Ricardo Marques)
+* mgr/dashboard: iSCSI targets not available if any gateway is down (`pr#31819 <https://github.com/ceph/ceph/pull/31819>`_, Ricardo Marques)
+* mgr/dashboard: Isolate each RBD component (`pr#33520 <https://github.com/ceph/ceph/pull/33520>`_, Tiago Melo)
+* mgr/dashboard: KeyError on dashboard reload (`pr#31469 <https://github.com/ceph/ceph/pull/31469>`_, Patrick Seidensal)
+* mgr/dashboard: KV-table transforms dates through pipe (`pr#27612 <https://github.com/ceph/ceph/pull/27612>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Left align badge datatable columns (`pr#32053 <https://github.com/ceph/ceph/pull/32053>`_, Volker Theile)
+* mgr/dashboard: list services and daemons (`pr#33531 <https://github.com/ceph/ceph/pull/33531>`_, Sage Weil, Kiefer Chang)
+* mgr/dashboard: Localization for date picker module (`pr#27275 <https://github.com/ceph/ceph/pull/27275>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Make all columns sortable (`pr#27784 <https://github.com/ceph/ceph/pull/27784>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: make check mypy failure (`pr#33573 <https://github.com/ceph/ceph/pull/33573>`_, Volker Theile)
+* mgr/dashboard: Make password policy check configurable (`pr#32546 <https://github.com/ceph/ceph/pull/32546>`_, Volker Theile)
+* mgr/dashboard: Make preventDefault work with 400 errors (`pr#26561 <https://github.com/ceph/ceph/pull/26561>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: monitoring: improve generic Could not reach external API message (`pr#32648 <https://github.com/ceph/ceph/pull/32648>`_, Patrick Seidensal)
+* mgr/dashboard: Not able to restrict bucket creation for new user (`pr#33612 <https://github.com/ceph/ceph/pull/33612>`_, Volker Theile)
+* mgr/dashboard: Optimize portal IPs calculation (`pr#28084 <https://github.com/ceph/ceph/pull/28084>`_, Ricardo Marques)
+* mgr/dashboard: orchestrator integration initial works (`pr#29127 <https://github.com/ceph/ceph/pull/29127>`_, Kiefer Chang)
+* mgr/dashboard: OSD custom action button removal (`pr#28095 <https://github.com/ceph/ceph/pull/28095>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: OSD improvements (`pr#30493 <https://github.com/ceph/ceph/pull/30493>`_, Patrick Seidensal)
+* mgr/dashboard: pass a list of drive_group to create_osds (`pr#33014 <https://github.com/ceph/ceph/pull/33014>`_, Kefu Chai)
+* mgr/dashboard: Pool form uses different loading spinner (`pr#28649 <https://github.com/ceph/ceph/pull/28649>`_, Volker Theile)
+* mgr/dashboard: Prevent deletion of iSCSI IQNs with open sessions (`pr#29133 <https://github.com/ceph/ceph/pull/29133>`_, Ricardo Marques)
+* mgr/dashboard: Prevent KeyError when requesting always_on_modules (`pr#30426 <https://github.com/ceph/ceph/pull/30426>`_, Volker Theile)
+* mgr/dashboard: Process password complexity checks immediately (`pr#32032 <https://github.com/ceph/ceph/pull/32032>`_, Volker Theile, Tatjana Dehler)
+* mgr/dashboard: Provide the name of the object being deleted (`pr#30658 <https://github.com/ceph/ceph/pull/30658>`_, Ricardo Marques)
+* mgr/dashboard: Provide user enable/disable capability (`issue#25229 <http://tracker.ceph.com/issues/25229>`_, `pr#29046 <https://github.com/ceph/ceph/pull/29046>`_, Ricardo Dias, Patrick Nawracay)
+* mgr/dashboard: Push Grafana dashboards on startup (`pr#26415 <https://github.com/ceph/ceph/pull/26415>`_, Zack Cerza)
+* mgr/dashboard: qa: fix RBD test when matching error strings (`pr#29264 <https://github.com/ceph/ceph/pull/29264>`_, Ricardo Dias)
+* mgr/dashboard: qa: whitelist client eviction warning (`pr#29114 <https://github.com/ceph/ceph/pull/29114>`_, Ricardo Dias)
+* mgr/dashboard: RBD snapshot name suggestion with local time suffix (`pr#27613 <https://github.com/ceph/ceph/pull/27613>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Reduce the number of renders on the tables (`issue#39944 <http://tracker.ceph.com/issues/39944>`_, `pr#28118 <https://github.com/ceph/ceph/pull/28118>`_, Tiago Melo)
+* mgr/dashboard: Refactor and cleanup tasks.mgr.dashboard.test_user (`pr#33743 <https://github.com/ceph/ceph/pull/33743>`_, Volker Theile)
+* mgr/dashboard: Refactor Python unittests and controller (`pr#31165 <https://github.com/ceph/ceph/pull/31165>`_, Volker Theile)
+* mgr/dashboard: Reload all CephFS directories (`pr#32552 <https://github.com/ceph/ceph/pull/32552>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: remove config-opt: read perm. from system roles (`pr#33690 <https://github.com/ceph/ceph/pull/33690>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Remove ngx-store (`pr#33756 <https://github.com/ceph/ceph/pull/33756>`_, Tiago Melo)
+* mgr/dashboard: remove traceback/version assertions (`pr#31720 <https://github.com/ceph/ceph/pull/31720>`_, Ernesto Puerta)
+* mgr/dashboard: Remove unused RBD configuration endpoint (`pr#30815 <https://github.com/ceph/ceph/pull/30815>`_, Ricardo Marques)
+* mgr/dashboard: Remove unused variable (`pr#31785 <https://github.com/ceph/ceph/pull/31785>`_, Volker Theile)
+* mgr/dashboard: Removes distracting search behavior (`pr#27438 <https://github.com/ceph/ceph/pull/27438>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Rename pipe list -> join (`pr#31843 <https://github.com/ceph/ceph/pull/31843>`_, Volker Theile)
+* mgr/dashboard: Replace IP address validation with Python standard library functions (`pr#26184 <https://github.com/ceph/ceph/pull/26184>`_, Ashish Singh)
+* mgr/dashboard: Replace ng2-tree with angular-tree-component (`pr#33758 <https://github.com/ceph/ceph/pull/33758>`_, Tiago Melo)
+* mgr/dashboard: RGW bucket creation when no placement target received (`pr#29280 <https://github.com/ceph/ceph/pull/29280>`_, alfonsomthd)
+* mgr/dashboard: RGW port autodetection does not support Beast RGW frontend (`pr#33060 <https://github.com/ceph/ceph/pull/33060>`_, Volker Theile)
+* mgr/dashboard: RGW User quota validation is not working correctly (`pr#29132 <https://github.com/ceph/ceph/pull/29132>`_, Volker Theile)
+* mgr/dashboard: run e2e tests against prod build (jenkins job) (`pr#29198 <https://github.com/ceph/ceph/pull/29198>`_, alfonsomthd)
+* mgr/dashboard: run-frontend-e2e-tests.sh: allow user defined BASE_URL\xe2\x80\xa6 (`pr#32211 <https://github.com/ceph/ceph/pull/32211>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: select placement target on RGW bucket creation (`pr#28764 <https://github.com/ceph/ceph/pull/28764>`_, alfonsomthd)
+* mgr/dashboard: Set RO as the default access_type for RGW NFS exports (`pr#30111 <https://github.com/ceph/ceph/pull/30111>`_, Tiago Melo)
+* mgr/dashboard: show checkboxes for booleans (`pr#32836 <https://github.com/ceph/ceph/pull/32836>`_, Tatjana Dehler)
+* mgr/dashboard: show correct RGW user system info (`pr#33206 <https://github.com/ceph/ceph/pull/33206>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: Show iSCSI gateways status in the health page (`pr#29112 <https://github.com/ceph/ceph/pull/29112>`_, Ricardo Marques)
+* mgr/dashboard: smart: smart data read out on down osd causes error popup (`pr#32953 <https://github.com/ceph/ceph/pull/32953>`_, Volker Theile)
+* mgr/dashboard: Standby Dashboards dont handle all requests properly (`pr#30478 <https://github.com/ceph/ceph/pull/30478>`_, Volker Theile)
+* mgr/dashboard: Support ceph-iscsi config v9 (`pr#27448 <https://github.com/ceph/ceph/pull/27448>`_, Ricardo Marques)
+* mgr/dashboard: support multiple DriveGroups when creating OSDs (`pr#32678 <https://github.com/ceph/ceph/pull/32678>`_, Kiefer Chang)
+* mgr/dashboard: support removing OSDs in OSDs page (`pr#31997 <https://github.com/ceph/ceph/pull/31997>`_, Kiefer Chang)
+* mgr/dashboard: support setting password hashes (`pr#29138 <https://github.com/ceph/ceph/pull/29138>`_, Fabian Bonk)
+* mgr/dashboard: tasks: only unblock controller thread after TaskManager thread (`pr#30747 <https://github.com/ceph/ceph/pull/30747>`_, Ricardo Dias)
+* mgr/dashboard: Throw a more meaningful exception (`pr#32234 <https://github.com/ceph/ceph/pull/32234>`_, Volker Theile)
+* mgr/dashboard: tox.ini fixes (`pr#30779 <https://github.com/ceph/ceph/pull/30779>`_, Alfonso Mart\xc3\xadnez)
+* mgr/dashboard: UI fixes (`pr#33171 <https://github.com/ceph/ceph/pull/33171>`_, Avan Thakkar)
+* mgr/dashboard: Unable to set boolean values to false when default is true (`pr#31738 <https://github.com/ceph/ceph/pull/31738>`_, Ricardo Marques)
+* mgr/dashboard: unify button/URL actions naming (`issue#37337 <http://tracker.ceph.com/issues/37337>`_, `pr#26572 <https://github.com/ceph/ceph/pull/26572>`_, Ernesto Puerta)
+* mgr/dashboard: Unify the look of dashboard charts (`issue#39384 <http://tracker.ceph.com/issues/39384>`_, `pr#27681 <https://github.com/ceph/ceph/pull/27681>`_, Tiago Melo)
+* mgr/dashboard: update dashboard CODEOWNERShip (`pr#31193 <https://github.com/ceph/ceph/pull/31193>`_, Ernesto Puerta)
+* mgr/dashboard: Update tar to v4.4.8 (`pr#28092 <https://github.com/ceph/ceph/pull/28092>`_, Kefu Chai)
+* mgr/dashboard: update vstart to use new ssl port (`issue#26914 <http://tracker.ceph.com/issues/26914>`_, `pr#27269 <https://github.com/ceph/ceph/pull/27269>`_, Ernesto Puerta)
+* mgr/dashboard: Updated octopus image on 404 page (`pr#33920 <https://github.com/ceph/ceph/pull/33920>`_, Lenz Grimmer)
+* mgr/dashboard: Use booleanText pipe (`pr#26733 <https://github.com/ceph/ceph/pull/26733>`_, Volker Theile)
+* mgr/dashboard: Use default language when running npm run build (`pr#31563 <https://github.com/ceph/ceph/pull/31563>`_, Tiago Melo)
+* mgr/dashboard: Use ModalComponent in all modals (`pr#33858 <https://github.com/ceph/ceph/pull/33858>`_, Tiago Melo)
+* mgr/dashboard: Use Observable in auth.service (`pr#32084 <https://github.com/ceph/ceph/pull/32084>`_, Volker Theile)
+* mgr/dashboard: Use onCancel on any modal event (`pr#29402 <https://github.com/ceph/ceph/pull/29402>`_, Stephan M\xc3\xbcller)
+* mgr/dashboard: Validate iSCSI controls min/max value (`pr#28942 <https://github.com/ceph/ceph/pull/28942>`_, Ricardo Marques)
+* mgr/dashboard: Validate iSCSI images features (`pr#27135 <https://github.com/ceph/ceph/pull/27135>`_, Ricardo Marques)
+* mgr/dashboard: Validate `ceph-iscsi` config version (`pr#26835 <https://github.com/ceph/ceph/pull/26835>`_, Ricardo Marques)
+* mgr/dashboard: Various UI issues related to CephFS (`pr#29272 <https://github.com/ceph/ceph/pull/29272>`_, Volker Theile)
+* mgr/dashboard: Vertically align the Refresh label (`pr#29737 <https://github.com/ceph/ceph/pull/29737>`_, Tiago Melo)
+* mgr/dashboard: vstart: Fix /dev/tty No such device or address (`pr#31195 <https://github.com/ceph/ceph/pull/31195>`_, Volker Theile)
+* mgr/dashboard: wait for PG unknown state to be cleared (`pr#33013 <https://github.com/ceph/ceph/pull/33013>`_, Tatjana Dehler)
+* mgr/dashboard: Watch for pool pgs increase and decrease (`pr#28006 <https://github.com/ceph/ceph/pull/28006>`_, Ricardo Dias, Stephan M\xc3\xbcller)
+* mgr/modules: outsource SSL certificate creation (`pr#33550 <https://github.com/ceph/ceph/pull/33550>`_, Patrick Seidensal)
+* mgr/orch,cephadm: add timestamps to daemons and services (`pr#33728 <https://github.com/ceph/ceph/pull/33728>`_, Sage Weil)
+* mgr/orch: add --all-available-devices to orch apply osd (`pr#33990 <https://github.com/ceph/ceph/pull/33990>`_, Sage Weil)
+* mgr/orch: add missing CLI commands for grafana, alertmanager (`pr#33695 <https://github.com/ceph/ceph/pull/33695>`_, Sage Weil)
+* mgr/orch: associate addresses with hosts (`pr#33098 <https://github.com/ceph/ceph/pull/33098>`_, Sage Weil)
+* mgr/orch: ceph orchestrator ... -> ceph orch ... (`pr#33131 <https://github.com/ceph/ceph/pull/33131>`_, Sage Weil)
+* mgr/orch: ceph upgrade ... -> ceph orch upgrade ... (`pr#34046 <https://github.com/ceph/ceph/pull/34046>`_, Sage Weil)
+* mgr/orch: collapse SPEC and PLACEMENT columns in orch ls (`pr#33795 <https://github.com/ceph/ceph/pull/33795>`_, Sage Weil)
+* mgr/orch: dump service spec by name (`pr#33951 <https://github.com/ceph/ceph/pull/33951>`_, Michael Fritch)
+* mgr/orch: first phase of new cli (`pr#33212 <https://github.com/ceph/ceph/pull/33212>`_, Sage Weil)
+* mgr/orch: fix host ls (`pr#33486 <https://github.com/ceph/ceph/pull/33486>`_, Sage Weil)
+* mgr/orch: fix orch ls table spacing (`pr#33586 <https://github.com/ceph/ceph/pull/33586>`_, Sage Weil)
+* mgr/orch: fix ServiceSpec deserialization error (`pr#33779 <https://github.com/ceph/ceph/pull/33779>`_, Kiefer Chang)
+* mgr/orch: improve commandline parsing for update\\_\\* (`pr#31672 <https://github.com/ceph/ceph/pull/31672>`_, Joshua Schmid)
+* mgr/orch: include spec ref in ServiceDescription (`pr#33667 <https://github.com/ceph/ceph/pull/33667>`_, Sage Weil)
+* mgr/orch: make arg hostname, not host (`pr#33474 <https://github.com/ceph/ceph/pull/33474>`_, Sage Weil)
+* mgr/orch: new cli, phase 2 (`pr#33244 <https://github.com/ceph/ceph/pull/33244>`_, Sage Weil)
+* mgr/orch: pass unicode string to ipaddress.ip_network() (`pr#31755 <https://github.com/ceph/ceph/pull/31755>`_, Kefu Chai)
+* mgr/orch: PlacementSpec: add all_hosts property (`pr#33465 <https://github.com/ceph/ceph/pull/33465>`_, Sage Weil)
+* mgr/orch: Properly handle NotImplementedError (`pr#33914 <https://github.com/ceph/ceph/pull/33914>`_, Sebastian Wagner)
+* mgr/orch: remove ansible and deepsea (`pr#33126 <https://github.com/ceph/ceph/pull/33126>`_, Sage Weil)
+* mgr/orch: resurrect ServiceDescription, orch ls (`pr#33359 <https://github.com/ceph/ceph/pull/33359>`_, Sage Weil)
+* mgr/orch: take a single placement argument (`pr#33706 <https://github.com/ceph/ceph/pull/33706>`_, Sage Weil)
+* mgr/orchestrator,mgr/ssh: add host labels (`pr#31854 <https://github.com/ceph/ceph/pull/31854>`_, Sage Weil)
+* mgr/orchestrator: Add doc about how to use OrchestratorClientMixin (`pr#32893 <https://github.com/ceph/ceph/pull/32893>`_, Sebastian Wagner)
+* mgr/orchestrator: Add mypy static type checking (`pr#32010 <https://github.com/ceph/ceph/pull/32010>`_, Sebastian Wagner)
+* mgr/orchestrator: add optional format param for orchestrator host ls (`pr#31930 <https://github.com/ceph/ceph/pull/31930>`_, Kefu Chai)
+* mgr/orchestrator: add progress events to all orchestrators (`pr#26654 <https://github.com/ceph/ceph/pull/26654>`_, Sebastian Wagner)
+* mgr/orchestrator: Add simple scheduler (`pr#32003 <https://github.com/ceph/ceph/pull/32003>`_, Joshua Schmid)
+* mgr/orchestrator: addr is optional for constructing InventoryNode (`pr#33347 <https://github.com/ceph/ceph/pull/33347>`_, Kefu Chai)
+* mgr/orchestrator: device lights (`pr#26768 <https://github.com/ceph/ceph/pull/26768>`_, Sebastian Wagner, Sage Weil)
+* mgr/orchestrator: do not try to iterate through None (`pr#31705 <https://github.com/ceph/ceph/pull/31705>`_, Kefu Chai)
+* mgr/orchestrator: Document OSD replacement (`pr#29792 <https://github.com/ceph/ceph/pull/29792>`_, Sebastian Wagner)
+* mgr/orchestrator: fix orch host label rm help text (`pr#33585 <https://github.com/ceph/ceph/pull/33585>`_, Sage Weil)
+* mgr/orchestrator: Fix raise_if_exception for Python 3 (`pr#31015 <https://github.com/ceph/ceph/pull/31015>`_, Sebastian Wagner)
+* mgr/orchestrator: fix refs property of progresses (`pr#30197 <https://github.com/ceph/ceph/pull/30197>`_, Kiefer Chang)
+* mgr/orchestrator: fix `ceph orch apply -i` + yaml cleanup + Completion cleanup (`pr#34001 <https://github.com/ceph/ceph/pull/34001>`_, Sebastian Wagner)
+* mgr/orchestrator: functools.partial doesnt work for methods (`pr#33432 <https://github.com/ceph/ceph/pull/33432>`_, Sebastian Wagner)
+* mgr/orchestrator: get_hosts return `HostSpec` instead of `InventoryDevice` (`pr#33258 <https://github.com/ceph/ceph/pull/33258>`_, Sebastian Wagner)
+* mgr/orchestrator: Make Completions composable (`pr#30262 <https://github.com/ceph/ceph/pull/30262>`_, Sebastian Wagner, Tim Serong)
+* mgr/orchestrator: make hosts and label args consistent (`pr#32253 <https://github.com/ceph/ceph/pull/32253>`_, Sage Weil)
+* mgr/orchestrator: Raise more expressive Error, if completion already \xe2\x80\xa6 (`pr#32270 <https://github.com/ceph/ceph/pull/32270>`_, Sebastian Wagner)
+* mgr/orchestrator: raise_if_exception: Add exception type to message (`pr#32574 <https://github.com/ceph/ceph/pull/32574>`_, Sebastian Wagner)
+* mgr/orchestrator: Remove `(add|test|remove)_stateful_service_rule` (`pr#26772 <https://github.com/ceph/ceph/pull/26772>`_, Sebastian Wagner)
+* mgr/orchestrator: set node labels to empty list if none specified (`pr#31914 <https://github.com/ceph/ceph/pull/31914>`_, Tim Serong)
+* mgr/orchestrator: Split \\*_stateless_service and add get_feature_set (`pr#29063 <https://github.com/ceph/ceph/pull/29063>`_, Sebastian Wagner)
+* mgr/orchestrator: Substitute `hostname` for `nodename`, globally (`pr#33467 <https://github.com/ceph/ceph/pull/33467>`_, Sebastian Wagner)
+* mgr/orchestrator: unify StatelessServiceSpec and StatefulServiceSpec (`pr#33175 <https://github.com/ceph/ceph/pull/33175>`_, Sebastian Wagner)
+* mgr/orchestrator: use deepcopy for copying exceptions (`pr#32881 <https://github.com/ceph/ceph/pull/32881>`_, Kefu Chai)
+* mgr/orchestrator: Use `pickle` to pass exceptions across sub-interpreters (`pr#33179 <https://github.com/ceph/ceph/pull/33179>`_, Sebastian Wagner)
+* mgr/orchestrator_cli: clean up device ls table (`pr#32279 <https://github.com/ceph/ceph/pull/32279>`_, Sage Weil)
+* mgr/orchestrator_cli: Fix NFS (`pr#32272 <https://github.com/ceph/ceph/pull/32272>`_, Sebastian Wagner)
+* mgr/orchestrator_cli: improve service ls output, sorting (`pr#31539 <https://github.com/ceph/ceph/pull/31539>`_, Sage Weil)
+* mgr/orchestrator_cli: set type for orchestrator option (`pr#32189 <https://github.com/ceph/ceph/pull/32189>`_, Sage Weil)
+* mgr/orchestrator_cli: sort host list (`pr#33370 <https://github.com/ceph/ceph/pull/33370>`_, Sage Weil)
+* mgr/orchestrator_cli: _update_mons require host spec only (`pr#32499 <https://github.com/ceph/ceph/pull/32499>`_, Sebastian Wagner)
+* mgr/progress/module.py: s/events/_events/ (`pr#29625 <https://github.com/ceph/ceph/pull/29625>`_, Kamoltat (Junior) Sirivadhna)
+* mgr/rook: Add caching for the Dashboard (`pr#29131 <https://github.com/ceph/ceph/pull/29131>`_, Sebastian Wagner, Paul Cuzner)
+* mgr/rook: Added missing `rgw` daemons in `service ls` (`issue#39171 <http://tracker.ceph.com/issues/39171>`_, `pr#27491 <https://github.com/ceph/ceph/pull/27491>`_, Sebastian Wagner)
+* mgr/rook: Added Mypy static type checking (`pr#32127 <https://github.com/ceph/ceph/pull/32127>`_, Sebastian Wagner)
+* mgr/rook: Fix creation of bluestore OSDs (`issue#39062 <http://tracker.ceph.com/issues/39062>`_, `pr#27289 <https://github.com/ceph/ceph/pull/27289>`_, Sebastian Wagner)
+* mgr/rook: Fix error creating OSDs (`pr#33176 <https://github.com/ceph/ceph/pull/33176>`_, Juan Miguel Olmo Mart\xc3\xadnez)
+* mgr/rook: Fix Python 2 regression (`issue#39250 <http://tracker.ceph.com/issues/39250>`_, `pr#27516 <https://github.com/ceph/ceph/pull/27516>`_, Sebastian Wagner)
+* mgr/rook: Fix RGW creation (`issue#39158 <http://tracker.ceph.com/issues/39158>`_, `pr#27462 <https://github.com/ceph/ceph/pull/27462>`_, Sebastian Wagner)
+* mgr/rook: misc fixes for orch ps (`pr#33868 <https://github.com/ceph/ceph/pull/33868>`_, Sage Weil)
+* mgr/rook: provide full path for devices names in inventory (`pr#32654 <https://github.com/ceph/ceph/pull/32654>`_, Sage Weil)
+* mgr/rook: Remove support for Rook older than v0.9 (`issue#39278 <http://tracker.ceph.com/issues/39278>`_, `pr#27556 <https://github.com/ceph/ceph/pull/27556>`_, Sebastian Wagner)
+* mgr/rook: Support other system namespaces (`issue#38799 <http://tracker.ceph.com/issues/38799>`_, `pr#27290 <https://github.com/ceph/ceph/pull/27290>`_, Sebastian Wagner)
+* mgr/ssh/tests: fix RGWSpec test (`pr#31983 <https://github.com/ceph/ceph/pull/31983>`_, Sage Weil)
+* mgr/ssh: add per-service operations: start, stop, restart, redeploy (`pr#31292 <https://github.com/ceph/ceph/pull/31292>`_, Sage Weil)
+* mgr/ssh: add TemporaryDirectory impl for py2 compat (`pr#31835 <https://github.com/ceph/ceph/pull/31835>`_, Sage Weil)
+* mgr/ssh: allow passing LV to orchestrator osd create (`pr#31512 <https://github.com/ceph/ceph/pull/31512>`_, Sage Weil)
+* mgr/ssh: annotate object representation (`pr#31602 <https://github.com/ceph/ceph/pull/31602>`_, Joshua Schmid)
+* mgr/ssh: cache service inventory (`pr#31385 <https://github.com/ceph/ceph/pull/31385>`_, Sage Weil)
+* mgr/ssh: deploy and remove rgw daemons (`pr#31303 <https://github.com/ceph/ceph/pull/31303>`_, Sage Weil)
+* mgr/ssh: deploy rbd-mirror daemons (`pr#31493 <https://github.com/ceph/ceph/pull/31493>`_, Sage Weil)
+* mgr/ssh: fix redeploy (`pr#31613 <https://github.com/ceph/ceph/pull/31613>`_, Sage Weil)
+* mgr/ssh: fix service_action, remove_osds (`pr#31952 <https://github.com/ceph/ceph/pull/31952>`_, Sage Weil)
+* mgr/ssh: Fix various Python issues (`pr#31524 <https://github.com/ceph/ceph/pull/31524>`_, Volker Theile)
+* mgr/ssh: Ignore ssh-config file (`pr#31710 <https://github.com/ceph/ceph/pull/31710>`_, Volker Theile)
+* mgr/ssh: implement blink_device_light (`pr#31438 <https://github.com/ceph/ceph/pull/31438>`_, Sage Weil)
+* mgr/ssh: implement service ls (`pr#31169 <https://github.com/ceph/ceph/pull/31169>`_, Sage Weil)
+* mgr/ssh: improve service ls (`pr#31828 <https://github.com/ceph/ceph/pull/31828>`_, Sage Weil)
+* mgr/ssh: Install SSH public key in Vagrantfile box fails (`pr#31519 <https://github.com/ceph/ceph/pull/31519>`_, Volker Theile)
+* mgr/ssh: optionally specify service names (`pr#31537 <https://github.com/ceph/ceph/pull/31537>`_, Sage Weil)
+* mgr/ssh: packaged-ceph-daemon mode; ssh key mgmt (`pr#31698 <https://github.com/ceph/ceph/pull/31698>`_, Sage Weil)
+* mgr/ssh: Port raising exceptions from completion handlers to Py2 (`pr#31940 <https://github.com/ceph/ceph/pull/31940>`_, Sebastian Wagner)
+* mgr/ssh: raise RuntimeError when ceph-daemon invocation fails (`pr#31420 <https://github.com/ceph/ceph/pull/31420>`_, Sage Weil)
+* mgr/ssh: remove superfluous parameters (`pr#31462 <https://github.com/ceph/ceph/pull/31462>`_, Joshua Schmid)
+* mgr/ssh: set up dummy known_hosts file (`pr#31721 <https://github.com/ceph/ceph/pull/31721>`_, Sage Weil)
+* mgr/ssh: take IP, CIDR, or addrvec for new mon(s) (`pr#31505 <https://github.com/ceph/ceph/pull/31505>`_, Sage Weil)
+* mgr/ssh: upgrade check command (`pr#31827 <https://github.com/ceph/ceph/pull/31827>`_, Sage Weil)
+* mgr/ssh: `test_mon_update` needs to set a mon name (`pr#31933 <https://github.com/ceph/ceph/pull/31933>`_, Sebastian Wagner)
+* mgr/telemetry: anonymizing smartctl report itself (`pr#33029 <https://github.com/ceph/ceph/pull/33029>`_, Yaarit Hatuka)
+* mgr/telemetry: dict.pop() errs on nonexistent key (`pr#30854 <https://github.com/ceph/ceph/pull/30854>`_, Dan Mick)
+* mgr/telemetry: fix log typo (`pr#31984 <https://github.com/ceph/ceph/pull/31984>`_, Sage Weil)
+* mgr/test_orchestrator: Allow initializing dummy data (`pr#29595 <https://github.com/ceph/ceph/pull/29595>`_, Kiefer Chang)
+* mgr/test_orchestrator: fix tests (`pr#33541 <https://github.com/ceph/ceph/pull/33541>`_, Sage Weil)
+* mgr/test_orchestrator: Fix TestWriteCompletion object has no attribute id (`pr#27607 <https://github.com/ceph/ceph/pull/27607>`_, Sebastian Wagner)
+* mgr/test_orchestrator: fix update_mgrs assert (`pr#32417 <https://github.com/ceph/ceph/pull/32417>`_, Sage Weil)
+* mgr/volumes: add arg to fs volume create for mds daemons placement (`pr#33441 <https://github.com/ceph/ceph/pull/33441>`_, Daniel-Pivonka)
+* mgr: Add get_rates_from_data to mgr_util.py (`pr#28603 <https://github.com/ceph/ceph/pull/28603>`_, Stephan M\xc3\xbcller)
+* mgr: add rbd profiles to support rbd_support module commands (`pr#30912 <https://github.com/ceph/ceph/pull/30912>`_, Jason Dillaman)
+* mgr: better error handling when reading option (`pr#32730 <https://github.com/ceph/ceph/pull/32730>`_, Kefu Chai)
+* mgr: ceph fs status support json format (`pr#30985 <https://github.com/ceph/ceph/pull/30985>`_, Erqi Chen)
+* mgr: change perf-counter precision to float (`pr#30400 <https://github.com/ceph/ceph/pull/30400>`_, Ernesto Puerta)
+* mgr: check for unicode passed to set_health_checks() (`pr#29117 <https://github.com/ceph/ceph/pull/29117>`_, Kefu Chai)
+* mgr: cleanup idle debug log at level 4 (`pr#29164 <https://github.com/ceph/ceph/pull/29164>`_, Sebastian Wagner)
+* mgr: close restful socket after exec (`pr#32396 <https://github.com/ceph/ceph/pull/32396>`_, liushi)
+* mgr: Configure Py root logger for Mgr modules (`pr#27069 <https://github.com/ceph/ceph/pull/27069>`_, Volker Theile)
+* mgr: do not reset reported if a new metric is not collected (`pr#30285 <https://github.com/ceph/ceph/pull/30285>`_, Ilsoo Byun)
+* mgr: drop session with Ceph daemon when not ready (`pr#31899 <https://github.com/ceph/ceph/pull/31899>`_, Patrick Donnelly)
+* mgr: fix a few bugs with teh pgp_num adjustments (`pr#27875 <https://github.com/ceph/ceph/pull/27875>`_, Sage Weil)
+* mgr: fix ceph native option value types (`pr#29855 <https://github.com/ceph/ceph/pull/29855>`_, Sage Weil)
+* mgr: fix debug typo (`pr#31900 <https://github.com/ceph/ceph/pull/31900>`_, Patrick Donnelly)
+* mgr: fix errors on using a reference in a Lambda function (`pr#31786 <https://github.com/ceph/ceph/pull/31786>`_, Willem Jan Withagen)
+* mgr: fix reporting of per-module logging options to mon (`pr#33897 <https://github.com/ceph/ceph/pull/33897>`_, Sage Weil)
+* mgr: fix weird health-alert daemon key (`pr#30617 <https://github.com/ceph/ceph/pull/30617>`_, xie xingguo)
+* mgr: handle race with finisher after shutdown (`pr#31620 <https://github.com/ceph/ceph/pull/31620>`_, Patrick Donnelly)
+* mgr: Improve internal python to c++ interface (`pr#32554 <https://github.com/ceph/ceph/pull/32554>`_, David Zafman)
+* mgr: install tox deps from wheelhouse (`pr#30034 <https://github.com/ceph/ceph/pull/30034>`_, Kefu Chai)
+* mgr: mgr, osd: osd df by pool (`pr#28629 <https://github.com/ceph/ceph/pull/28629>`_, xie xingguo)
+* mgr: mgr/ActivePyModules: behave if a module queries a devid that does not exist (`pr#31291 <https://github.com/ceph/ceph/pull/31291>`_, Sage Weil)
+* mgr: mgr/ActivePyModules: drop GIL while we wait for mon reply in set_store, set_config (`issue#39335 <http://tracker.ceph.com/issues/39335>`_, `pr#27619 <https://github.com/ceph/ceph/pull/27619>`_, Sage Weil)
+* mgr: mgr/ActivePyModules: handle_command - fix broken lock (`issue#39235 <http://tracker.ceph.com/issues/39235>`_, `pr#27485 <https://github.com/ceph/ceph/pull/27485>`_, xie xingguo)
+* mgr: mgr/balancer: avoid pulling pg_dump twice (`pr#32266 <https://github.com/ceph/ceph/pull/32266>`_, xie xingguo)
+* mgr: mgr/balancer: eliminate usage of MS infrastructure for upmap mode (`pr#32289 <https://github.com/ceph/ceph/pull/32289>`_, xie xingguo)
+* mgr: mgr/balancer: enable pg_upmap cli for future use (`pr#30560 <https://github.com/ceph/ceph/pull/30560>`_, xie xingguo)
+* mgr: mgr/balancer: fix fudge (`pr#27994 <https://github.com/ceph/ceph/pull/27994>`_, xie xingguo)
+* mgr: mgr/balancer: fix initial weight-set value for newly created osds (`pr#28251 <https://github.com/ceph/ceph/pull/28251>`_, xie xingguo)
+* mgr: mgr/balancer: Python 3 compatibility fix (`issue#38831 <http://tracker.ceph.com/issues/38831>`_, `pr#27076 <https://github.com/ceph/ceph/pull/27076>`_, Marius Schiffer)
+* mgr: mgr/balancer: python3 compatibility issue (`pr#30987 <https://github.com/ceph/ceph/pull/30987>`_, Mykola Golub)
+* mgr: mgr/balancer: upmap_max_iterations -> upmap_max_optimizations; behave as it is per pool (`pr#30591 <https://github.com/ceph/ceph/pull/30591>`_, xie xingguo)
+* mgr: mgr/BaseMgrModule: tolerate Int or Long for health count (`pr#29806 <https://github.com/ceph/ceph/pull/29806>`_, Sage Weil)
+* mgr: mgr/BaseMgrModule: use PyInt_Check() to compatible with py2 (`pr#29831 <https://github.com/ceph/ceph/pull/29831>`_, Kefu Chai)
+* mgr: mgr/BaseMgrStandbyModule: drop GIL in ceph_get_module_option() (`pr#30625 <https://github.com/ceph/ceph/pull/30625>`_, Kefu Chai)
+* mgr: mgr/cephadm: custom certificates for Grafana deployment (`pr#33614 <https://github.com/ceph/ceph/pull/33614>`_, Patrick Seidensal)
+* mgr: mgr/cephadm: support (point release) upgrades (`pr#32006 <https://github.com/ceph/ceph/pull/32006>`_, Sage Weil)
+* mgr: mgr/crash: Calculate and add stack_sig to metadata (`pr#31394 <https://github.com/ceph/ceph/pull/31394>`_, Dan Mick)
+* mgr: mgr/crash: fix crash ls[-new] sorting (`pr#31973 <https://github.com/ceph/ceph/pull/31973>`_, Sage Weil)
+* mgr: mgr/DaemonServer: handle caps more carefully (`pr#26903 <https://github.com/ceph/ceph/pull/26903>`_, xie xingguo)
+* mgr: mgr/DaemonServer: handle_conf_change - fix broken locking (`issue#38899 <http://tracker.ceph.com/issues/38899>`_, `pr#27184 <https://github.com/ceph/ceph/pull/27184>`_, xie xingguo)
+* mgr: mgr/DaemonServer: refactor pgp_num changes throttling (`pr#27891 <https://github.com/ceph/ceph/pull/27891>`_, Kefu Chai)
+* mgr: mgr/DaemonServer: safe-to-destroy - do not consider irrelevant pgs (`pr#27962 <https://github.com/ceph/ceph/pull/27962>`_, xie xingguo)
+* mgr: mgr/DaemonServer: skip adjusting pgp_num when merging is in-progress (`pr#30139 <https://github.com/ceph/ceph/pull/30139>`_, xie xingguo)
+* mgr: mgr/dashboard: Do not default to admin as Admin Resource (`issue#39338 <http://tracker.ceph.com/issues/39338>`_, `pr#27626 <https://github.com/ceph/ceph/pull/27626>`_, Wido den Hollander)
+* mgr: mgr/dashboard: Handle always-on Ceph Manager modules correctly (`pr#30142 <https://github.com/ceph/ceph/pull/30142>`_, Volker Theile)
+* mgr: mgr/dashboard: integrate progress mgr module events into dashboard tasks list (`pr#29048 <https://github.com/ceph/ceph/pull/29048>`_, Ricardo Dias)
+* mgr: mgr/dashboard: Manager should complain about wrong dashboard certificate (`pr#27036 <https://github.com/ceph/ceph/pull/27036>`_, Volker Theile)
+* mgr: mgr/deepsea: return ganesha and iscsi endpoint URLs (`pr#27336 <https://github.com/ceph/ceph/pull/27336>`_, Tim Serong)
+* mgr: mgr/deepsea: use ceph_volume output in get_inventory() (`pr#26966 <https://github.com/ceph/ceph/pull/26966>`_, Tim Serong)
+* mgr: mgr/devicehealth: ensure we dont store empty objects (`pr#31474 <https://github.com/ceph/ceph/pull/31474>`_, Sage Weil)
+* mgr: mgr/devicehealth: Fix python 3 incompatiblity (`issue#38939 <http://tracker.ceph.com/issues/38939>`_, `pr#27172 <https://github.com/ceph/ceph/pull/27172>`_, Marius Schiffer)
+* mgr: mgr/devicehealth: set default monitoring to on (`pr#33091 <https://github.com/ceph/ceph/pull/33091>`_, Sage Weil, Yaarit Hatuka)
+* mgr: mgr/diskprediction: Add diskprediction local plugin dependencies (`pr#25530 <https://github.com/ceph/ceph/pull/25530>`_, Rick Chen)
+* mgr: mgr/diskprediction_cloud: Correct base64 encode translate table (`issue#38848 <http://tracker.ceph.com/issues/38848>`_, `pr#27113 <https://github.com/ceph/ceph/pull/27113>`_, Rick Chen)
+* mgr: mgr/diskprediction_cloud: refactor timeout() decorator (`pr#31176 <https://github.com/ceph/ceph/pull/31176>`_, Kefu Chai)
+* mgr: mgr/hello: some clean up and modernization (`pr#29514 <https://github.com/ceph/ceph/pull/29514>`_, Sage Weil)
+* mgr: mgr/influx: try to call close() (`issue#40174 <http://tracker.ceph.com/issues/40174>`_, `pr#28427 <https://github.com/ceph/ceph/pull/28427>`_, Kefu Chai)
+* mgr: mgr/insights: fix prune-health-history (`pr#32973 <https://github.com/ceph/ceph/pull/32973>`_, Sage Weil)
+* mgr: mgr/k8sevents: Add mgr module for kubernetes event integration (`pr#29520 <https://github.com/ceph/ceph/pull/29520>`_, Paul Cuzner)
+* mgr: mgr/k8sevents: Add support for remote kubernetes (`pr#30482 <https://github.com/ceph/ceph/pull/30482>`_, Paul Cuzner)
+* mgr: mgr/Mgr: kill redundant sub_unwant call (`pr#26950 <https://github.com/ceph/ceph/pull/26950>`_, xie xingguo)
+* mgr: mgr/MgrMonitor: print pending.always_on_modules before updating it (`pr#29917 <https://github.com/ceph/ceph/pull/29917>`_, Kefu Chai)
+* mgr: mgr/orch: logging - handle lists output (`pr#32879 <https://github.com/ceph/ceph/pull/32879>`_, Shyukri Shyukriev)
+* mgr: mgr/orchestrator: Add cache for Inventory and Services (`pr#28213 <https://github.com/ceph/ceph/pull/28213>`_, Tim Serong, Sebastian Wagner)
+* mgr: mgr/orchestrator_cli: pass default value to req=False params (`pr#31314 <https://github.com/ceph/ceph/pull/31314>`_, Kefu Chai)
+* mgr: mgr/osd_support: new module for osd utility (`pr#32677 <https://github.com/ceph/ceph/pull/32677>`_, Joshua Schmid)
+* mgr: mgr/pg_autoscaler: calculate pool_pg_target using pool size (`pr#32592 <https://github.com/ceph/ceph/pull/32592>`_, Dan van der Ster)
+* mgr: mgr/pg_autoscaler: fix pool_logical_used (`pr#29986 <https://github.com/ceph/ceph/pull/29986>`_, Ansgar Jazdzewski)
+* mgr: mgr/pg_autoscaler: Fix python3 incompatibility (`issue#38626 <http://tracker.ceph.com/issues/38626>`_, `pr#27079 <https://github.com/ceph/ceph/pull/27079>`_, Marius Schiffer)
+* mgr: mgr/pg_autoscaler: fix race with pool deletion (`pr#29807 <https://github.com/ceph/ceph/pull/29807>`_, Sage Weil)
+* mgr: mgr/pg_autoscaler: treat target ratios as weights (`pr#33035 <https://github.com/ceph/ceph/pull/33035>`_, Josh Durgin)
+* mgr: mgr/progress & mgr/pg_autoscaler: Added Pg Autoscaler Event (`pr#29035 <https://github.com/ceph/ceph/pull/29035>`_, Kamoltat (Junior) Sirivadhna)
+* mgr: mgr/progress: Add integration to pybind/mgr/tox.ini (`pr#32985 <https://github.com/ceph/ceph/pull/32985>`_, Sebastian Wagner)
+* mgr: mgr/progress: Add recovery event when OSD marked in (`pr#28498 <https://github.com/ceph/ceph/pull/28498>`_, Kamoltat (Junior) Sirivadhna)
+* mgr: mgr/progress: added the time an event has been in progress (`pr#28907 <https://github.com/ceph/ceph/pull/28907>`_, Kamoltat (Junior) Sirivadhna)
+* mgr: mgr/progress: Bug fix complete event when OSD marked in (`pr#28695 <https://github.com/ceph/ceph/pull/28695>`_, Kamoltat (Junior) Sirivadhna)
+* mgr: mgr/progress: clamp pg recovery ratio to 0 (`pr#29126 <https://github.com/ceph/ceph/pull/29126>`_, xie xingguo)
+* mgr: mgr/progress: estimated remaining time for events (`pr#30615 <https://github.com/ceph/ceph/pull/30615>`_, xie xingguo)
+* mgr: mgr/progress: Look at PG state when PG epoch >= OSDMap epoch (`pr#28368 <https://github.com/ceph/ceph/pull/28368>`_, Kamoltat (Junior) Sirivadhna)
+* mgr: mgr/progress: remove since from duration string (`pr#31007 <https://github.com/ceph/ceph/pull/31007>`_, Kefu Chai)
+* mgr: mgr/prometheus: Add mgr metdata to prometheus exporter module (`pr#28372 <https://github.com/ceph/ceph/pull/28372>`_, Paul Cuzner)
+* mgr: mgr/prometheus: assign a value to osd_dev_node when obj_store is not filestore or bluestore (`pr#30534 <https://github.com/ceph/ceph/pull/30534>`_, jiahuizeng)
+* mgr: mgr/prometheus: Cast collect_timeout (scrape_interval) to float (`pr#29382 <https://github.com/ceph/ceph/pull/29382>`_, Ben Meekhof)
+* mgr: mgr/prometheus: Fix KeyError in get_mgr_status (`pr#30421 <https://github.com/ceph/ceph/pull/30421>`_, Sebastian Wagner)
+* mgr: mgr/prometheus: replace whitespaces in metrics names (`pr#27722 <https://github.com/ceph/ceph/pull/27722>`_, Alfonso Mart\xc3\xadnez)
+* mgr: mgr/PyModule: correctly remove config options (`pr#31807 <https://github.com/ceph/ceph/pull/31807>`_, Tim Serong)
+* mgr: mgr/PyModuleRegistry: log error if we cant find any modules to load (`pr#28055 <https://github.com/ceph/ceph/pull/28055>`_, Tim Serong)
+* mgr: mgr/restful: allow shutdown before weve fully started up (`pr#32004 <https://github.com/ceph/ceph/pull/32004>`_, Sage Weil)
+* mgr: mgr/restful: do not use filter() for list (`pr#27925 <https://github.com/ceph/ceph/pull/27925>`_, Kefu Chai)
+* mgr: mgr/restful: jsonify lists instead of maps (`pr#32421 <https://github.com/ceph/ceph/pull/32421>`_, Kefu Chai)
+* mgr: mgr/restful: requests api adds support multiple commands (`pr#31152 <https://github.com/ceph/ceph/pull/31152>`_, Duncan Chiang)
+* mgr: mgr/status: fix ceph osd status ZeroDivisionError (`pr#28797 <https://github.com/ceph/ceph/pull/28797>`_, simon gao)
+* mgr: mgr/telemetry: add last_upload to status (`pr#33125 <https://github.com/ceph/ceph/pull/33125>`_, Yaarit Hatuka)
+* mgr: mgr/telemetry: change crash dict to a list (`pr#27631 <https://github.com/ceph/ceph/pull/27631>`_, Dan Mick)
+* mgr: mgr/telemetry: channels (`pr#28847 <https://github.com/ceph/ceph/pull/28847>`_, Sage Weil)
+* mgr: mgr/telemetry: check get_metadata return val (`pr#33051 <https://github.com/ceph/ceph/pull/33051>`_, Yaarit Hatuka)
+* mgr: mgr/telemetry: clear the event after being awaken by it (`pr#29546 <https://github.com/ceph/ceph/pull/29546>`_, Kefu Chai)
+* mgr: mgr/telemetry: exclude hostname field in crash reports (`pr#27693 <https://github.com/ceph/ceph/pull/27693>`_, Sage Weil)
+* mgr: mgr/telemetry: fix and document proxy usage (`pr#33575 <https://github.com/ceph/ceph/pull/33575>`_, Lars Marowsky-Bree)
+* mgr: mgr/telemetry: fix device serial number anonymization (`pr#32492 <https://github.com/ceph/ceph/pull/32492>`_, Yaarit Hatuka)
+* mgr: mgr/telemetry: include any config options that are customized (`pr#29334 <https://github.com/ceph/ceph/pull/29334>`_, Sage Weil)
+* mgr: mgr/telemetry: include device health telemetry (`pr#30724 <https://github.com/ceph/ceph/pull/30724>`_, Sage Weil)
+* mgr: mgr/telemetry: re-opt-in when telemetry content changes; nag on major releases (`pr#29337 <https://github.com/ceph/ceph/pull/29337>`_, Sage Weil)
+* mgr: mgr/telemetry: salt osd ids too (`pr#29358 <https://github.com/ceph/ceph/pull/29358>`_, Sage Weil)
+* mgr: mgr/telemetry: specify license when opting in (`pr#29340 <https://github.com/ceph/ceph/pull/29340>`_, Sage Weil)
+* mgr: mgr/volumes: do not import unused module (`pr#28875 <https://github.com/ceph/ceph/pull/28875>`_, Kefu Chai)
+* mgr: mgr/zabbix Added pools discovery and per-pool statistics (`pr#26152 <https://github.com/ceph/ceph/pull/26152>`_, Dmitriy Rabotjagov)
+* mgr: mgr/zabbix: Adds possibility to send data to multiple zabbix servers (`issue#38409 <http://tracker.ceph.com/issues/38409>`_, `pr#26547 <https://github.com/ceph/ceph/pull/26547>`_, slivik, Jakub Sliva)
+* mgr: mgr/zabbix: encode string for Python 3 compatibility (`pr#28624 <https://github.com/ceph/ceph/pull/28624>`_, Nathan Cutler)
+* mgr: mgr/zabbix: Fix raw_bytes_used key name (`pr#28058 <https://github.com/ceph/ceph/pull/28058>`_, Dmitriy Rabotjagov)
+* mgr: mgr/zabbix: Fix typo in key name for PGs in backfill_wait state (`issue#39666 <http://tracker.ceph.com/issues/39666>`_, `pr#28057 <https://github.com/ceph/ceph/pull/28057>`_, Wido den Hollander)
+* mgr: missing lock release in DaemonServer::handle_report() (`issue#42169 <http://tracker.ceph.com/issues/42169>`_, `pr#30706 <https://github.com/ceph/ceph/pull/30706>`_, Venky Shankar)
+* mgr: module logging infrastructure (`pr#30961 <https://github.com/ceph/ceph/pull/30961>`_, Ricardo Dias)
+* mgr: more GIL fixes (`issue#39040 <http://tracker.ceph.com/issues/39040>`_, `pr#27280 <https://github.com/ceph/ceph/pull/27280>`_, xie xingguo)
+* mgr: pybind/mgr/balancer/module.py: add max/min info in stats_by_root (`pr#30432 <https://github.com/ceph/ceph/pull/30432>`_, Yang Honggang)
+* mgr: pybind/mgr/pg_autoscaler: implement shutdown method (`pr#31398 <https://github.com/ceph/ceph/pull/31398>`_, Patrick Donnelly)
+* mgr: pybind/mgr/restful: use dict.items() for py3 compatible (`pr#29356 <https://github.com/ceph/ceph/pull/29356>`_, Kefu Chai)
+* mgr: pybind/mgr: Cancel output color control (`pr#31427 <https://github.com/ceph/ceph/pull/31427>`_, Zheng Yin)
+* mgr: pybind/mgr: convert str to int using int() (`pr#27926 <https://github.com/ceph/ceph/pull/27926>`_, Kefu Chai)
+* mgr: pybind/mgr: Make it easier to create a Module instance without the mgr (`pr#31969 <https://github.com/ceph/ceph/pull/31969>`_, Sebastian Wagner)
+* mgr: pybind/mgr: Remove code duplication (`issue#40698 <http://tracker.ceph.com/issues/40698>`_, `pr#28986 <https://github.com/ceph/ceph/pull/28986>`_, Sebastian Wagner)
+* mgr: pyind/mgr: add mgr_module.py and mgr_util.py to mypy (`pr#32597 <https://github.com/ceph/ceph/pull/32597>`_, Sebastian Wagner)
+* mgr: Python cleanup and type check (`pr#31559 <https://github.com/ceph/ceph/pull/31559>`_, Volker Theile)
+* mgr: qa/mgr/progress: fix timeout error when waiting for osd in event (`pr#30095 <https://github.com/ceph/ceph/pull/30095>`_, Ricardo Dias)
+* mgr: re-enable mds `scrub status` info in ceph status (`issue#42835 <http://tracker.ceph.com/issues/42835>`_, `pr#32657 <https://github.com/ceph/ceph/pull/32657>`_, Venky Shankar)
+* mgr: Reduce logging noise when handling commands (`pr#29305 <https://github.com/ceph/ceph/pull/29305>`_, Sebastian Wagner)
+* mgr: Release GIL before calling OSDMap::calc_pg_upmaps() (`pr#31064 <https://github.com/ceph/ceph/pull/31064>`_, David Zafman)
+* mgr: remove unused variable pool_name (`pr#28340 <https://github.com/ceph/ceph/pull/28340>`_, Alex Wu)
+* mgr: restful: Expose perf counters (`pr#27885 <https://github.com/ceph/ceph/pull/27885>`_, Boris Ranto)
+* mgr: restful: Query nodes_by_id for items (`pr#31153 <https://github.com/ceph/ceph/pull/31153>`_, Boris Ranto)
+* mgr: return perf_counters data timestamps in nanosecs (`pr#28882 <https://github.com/ceph/ceph/pull/28882>`_, Ricardo Dias)
+* mgr: Revert mgr/DaemonServer: safe-to-destroy - do not consider irrelevant pgs (`pr#32203 <https://github.com/ceph/ceph/pull/32203>`_, xie xingguo)
+* mgr: set hostname in DeviceState::set_metadata() (`pr#30448 <https://github.com/ceph/ceph/pull/30448>`_, Kefu Chai)
+* mgr: simply exit on SIGINT or SIGTERM (`pr#32051 <https://github.com/ceph/ceph/pull/32051>`_, Sage Weil)
+* mgr: telemetry/server: misc fixes (`pr#29365 <https://github.com/ceph/ceph/pull/29365>`_, user.email, Sage Weil)
+* mgr: telemetry: misc scripts (`pr#29781 <https://github.com/ceph/ceph/pull/29781>`_, sage@newdream.net, Sage Weil)
+* mgr: templatize metrics collection interface (`pr#29214 <https://github.com/ceph/ceph/pull/29214>`_, Venky Shankar)
+* mgr: update hostname when we already have the daemon state from the same entity (`pr#33752 <https://github.com/ceph/ceph/pull/33752>`_, Kefu Chai)
+* mgr: use a struct for DaemonKey (`pr#30635 <https://github.com/ceph/ceph/pull/30635>`_, Kefu Chai)
+* mgr: use ipv4 default when ipv6 was disabled (`pr#28246 <https://github.com/ceph/ceph/pull/28246>`_, kungf)
+* mgr: use new MMgrCommand for CLI commands sent to mgr (`pr#30155 <https://github.com/ceph/ceph/pull/30155>`_, Sage Weil)
+* mgr: zabbix triggers never triggered due to wrong trigger function (`pr#26146 <https://github.com/ceph/ceph/pull/26146>`_, Sebastiaan Nijhuis)
+* mgr: _exit(0) from signal handler even if we are standby (`pr#31685 <https://github.com/ceph/ceph/pull/31685>`_, Sage Weil)
+* mon,rbd,tests: mon,test: silence warnings from GCC and test (`pr#28250 <https://github.com/ceph/ceph/pull/28250>`_, Kefu Chai)
+* mon,tests: qa/tasks: Fix ambiguous store_thrash, thrash_store (`issue#39159 <http://tracker.ceph.com/issues/39159>`_, `pr#27542 <https://github.com/ceph/ceph/pull/27542>`_, Jos Collin)
+* mon,tools: monmaptool: added --addv option to usage description (`pr#29307 <https://github.com/ceph/ceph/pull/29307>`_, Ricardo Dias)
+* mon/MonClient: fix mon tell to older mons (`pr#31121 <https://github.com/ceph/ceph/pull/31121>`_, Sage Weil)
+* mon/OSDMonitor.cc: Allow pool set target_max\\_(objects/bytes) with SI/IEC units (`pr#31010 <https://github.com/ceph/ceph/pull/31010>`_, Prashant D)
+* mon/OSDMonitor: osd add-no{up,down,in,out} - remove state checker (`pr#27605 <https://github.com/ceph/ceph/pull/27605>`_, xie xingguo)
+* mon/pgmap: fix bluestore alerts output (`pr#30342 <https://github.com/ceph/ceph/pull/30342>`_, Igor Fedotov)
+* mon: add ability to mute health alerts (`pr#29422 <https://github.com/ceph/ceph/pull/29422>`_, Sage Weil)
+* mon: add mon, osd, mds ok-to-stop and related commands (`pr#27146 <https://github.com/ceph/ceph/pull/27146>`_, Sage Weil)
+* mon: add `ceph osd info` to obtain info on osds rather than parsing `osd dump` (`pr#26724 <https://github.com/ceph/ceph/pull/26724>`_, Joao Eduardo Luis)
+* mon: allow running without a config file (`pr#30498 <https://github.com/ceph/ceph/pull/30498>`_, Joao Eduardo Luis)
+* mon: always enable pg_autoscaler (`pr#29072 <https://github.com/ceph/ceph/pull/29072>`_, Sage Weil)
+* mon: disable min pg per osd warning (`pr#30352 <https://github.com/ceph/ceph/pull/30352>`_, Sage Weil)
+* mon: Dont put session during feature change (`pr#32365 <https://github.com/ceph/ceph/pull/32365>`_, Brad Hubbard)
+* mon: dump json from sessions asok/tell command (`pr#32974 <https://github.com/ceph/ceph/pull/32974>`_, Sage Weil)
+* mon: elector: return after triggering a new election (`pr#32981 <https://github.com/ceph/ceph/pull/32981>`_, Greg Farnum)
+* mon: ensure prepare_failure() marks no_reply on op (`pr#28177 <https://github.com/ceph/ceph/pull/28177>`_, Joao Eduardo Luis)
+* mon: fix INCOMPAT_OCTOPUS feature number (`pr#27622 <https://github.com/ceph/ceph/pull/27622>`_, Sage Weil)
+* mon: fix misc asok commands (`pr#30859 <https://github.com/ceph/ceph/pull/30859>`_, Sage Weil, Patrick Donnelly)
+* mon: fix off-by-one rendering progress bar (`pr#28268 <https://github.com/ceph/ceph/pull/28268>`_, Sage Weil)
+* mon: fix tell command description (and ceph CLI help behavior) (`pr#33135 <https://github.com/ceph/ceph/pull/33135>`_, Sage Weil)
+* mon: fix tell to hybrid octopus/pre-octopus mons (`pr#31138 <https://github.com/ceph/ceph/pull/31138>`_, Sage Weil)
+* mon: fix/improve mon sync over small keys (`pr#31581 <https://github.com/ceph/ceph/pull/31581>`_, Sage Weil)
+* mon: Get session_map_lock before remove_session (`pr#33682 <https://github.com/ceph/ceph/pull/33682>`_, Xiaofei Cui)
+* mon: Improve health status for backfill_toofull and recovery_toofull (`pr#28204 <https://github.com/ceph/ceph/pull/28204>`_, David Zafman)
+* mon: Improvements to slow heartbeat health messages (`pr#32342 <https://github.com/ceph/ceph/pull/32342>`_, David Zafman)
+* mon: make ceph -s much  more concise (`pr#29493 <https://github.com/ceph/ceph/pull/29493>`_, Sage Weil)
+* mon: make compact tell command, and add deprecate/obsolete check for tell commands (`pr#31722 <https://github.com/ceph/ceph/pull/31722>`_, Kefu Chai)
+* mon: make mon_osd_down_out_subtree_limit update at runtime (`pr#27517 <https://github.com/ceph/ceph/pull/27517>`_, Sage Weil)
+* mon: mon/ConfigMonitor: make config reset idempotent (`pr#27155 <https://github.com/ceph/ceph/pull/27155>`_, xie xingguo)
+* mon: mon/ConfigMonitor: make num of config reset optional; allow target version 0 (`pr#27090 <https://github.com/ceph/ceph/pull/27090>`_, xie xingguo)
+* mon: mon/HealthMonitor: remove unused label (`pr#29749 <https://github.com/ceph/ceph/pull/29749>`_, Kefu Chai)
+* mon: mon/MonClient: weight-based mon selection (`pr#26940 <https://github.com/ceph/ceph/pull/26940>`_, xie xingguo)
+* mon: mon/Monitor: no need to create a local variable for capturing it (`pr#28744 <https://github.com/ceph/ceph/pull/28744>`_, Kefu Chai)
+* mon: mon/MonMap: always set mon priority; add it to dump (`pr#26975 <https://github.com/ceph/ceph/pull/26975>`_, xie xingguo)
+* mon: mon/OSDMonitor: crush node flags - two fixes; add tests (`pr#27719 <https://github.com/ceph/ceph/pull/27719>`_, xie xingguo)
+* mon: mon/OSDMonitor: fix off-by-one when updating new_last_in_change (`pr#28568 <https://github.com/ceph/ceph/pull/28568>`_, xie xingguo)
+* mon: mon/OSDMonitor: report pg[pgp]_num_target instead of pg[pgp]_num (`issue#40193 <http://tracker.ceph.com/issues/40193>`_, `pr#28490 <https://github.com/ceph/ceph/pull/28490>`_, xie xingguo)
+* mon: mon/OSDMonitor: trim not-longer-exist failure reporters (`pr#30200 <https://github.com/ceph/ceph/pull/30200>`_, NancySu05)
+* mon: mon/OSDMonitor: use initializer_list<> for {si,iec}_options (`pr#31175 <https://github.com/ceph/ceph/pull/31175>`_, Kefu Chai)
+* mon: mon/PGMap: fix incorrect pg_pool_sum when delete pool (`pr#31560 <https://github.com/ceph/ceph/pull/31560>`_, luo rixin)
+* mon: optionally bind to public_addrv (instead of public_addr or public_network) (`pr#31501 <https://github.com/ceph/ceph/pull/31501>`_, Sage Weil)
+* mon: paxos: empty pending_finishers before retrying any of committing\xe2\x80\xa6 (`issue#39484 <http://tracker.ceph.com/issues/39484>`_, `pr#27877 <https://github.com/ceph/ceph/pull/27877>`_, Greg Farnum)
+* mon: print FSMap regardless of file system count (`pr#32307 <https://github.com/ceph/ceph/pull/32307>`_, Patrick Donnelly)
+* mon: quiet devname noise (`pr#27313 <https://github.com/ceph/ceph/pull/27313>`_, Sage Weil)
+* mon: remove the restriction of address type in init_with_hosts (`pr#31691 <https://github.com/ceph/ceph/pull/31691>`_, Hao Xiong)
+* mon: Revert mon/OSDMonitor: report pg[pgp]_num_target instead of pg[pgp]_\xe2\x80\xa6 (`pr#28567 <https://github.com/ceph/ceph/pull/28567>`_, xie xingguo)
+* mon: set recovery_priority, pg_num_min, pg_autoscale_bias via fs new command (`pr#29180 <https://github.com/ceph/ceph/pull/29180>`_, Sage Weil)
+* mon: should not take non-tell commands as tell ones (`pr#32517 <https://github.com/ceph/ceph/pull/32517>`_, Kefu Chai)
+* mon: show no[deep-]scrub flags per pool in the status (`issue#38029 <http://tracker.ceph.com/issues/38029>`_, `pr#26488 <https://github.com/ceph/ceph/pull/26488>`_, Mohamad Gebai)
+* mon: show pool id in pool ls command (`issue#40287 <http://tracker.ceph.com/issues/40287>`_, `pr#28488 <https://github.com/ceph/ceph/pull/28488>`_, Chang Liu)
+* mon: Split Elector into message-passing and logic/state components (`pr#28727 <https://github.com/ceph/ceph/pull/28727>`_, Greg Farnum)
+* mon: stash newer map on bootstrap when addr doesnt match (`pr#33418 <https://github.com/ceph/ceph/pull/33418>`_, Sage Weil)
+* mon: take the mon lock in handle_conf_change (`issue#39625 <http://tracker.ceph.com/issues/39625>`_, `pr#28018 <https://github.com/ceph/ceph/pull/28018>`_, huangjun)
+* mon: use non-obsolete mon scrub cmd (`pr#32510 <https://github.com/ceph/ceph/pull/32510>`_, Patrick Donnelly)
+* mon:C_AckMarkedDown has not handled the Callback Arguments (`pr#29624 <https://github.com/ceph/ceph/pull/29624>`_, NancySu05)
+* monitoring: fix prometheus alert for full pools (`pr#32325 <https://github.com/ceph/ceph/pull/32325>`_, Thomas Kriechbaumer)
+* monitoring: fix RGW grafana chart Average GET/PUT Latencies (`pr#33839 <https://github.com/ceph/ceph/pull/33839>`_, Alfonso Mart\xc3\xadnez)
+* monitoring: restore lost fix for `pool full` alert (`pr#33655 <https://github.com/ceph/ceph/pull/33655>`_, Patrick Seidensal)
+* monitoring: SNMP OID per every Prometheus alert rule (`pr#27978 <https://github.com/ceph/ceph/pull/27978>`_, Volker Theile)
+* monitoring: wait before firing osd full alert (`pr#31711 <https://github.com/ceph/ceph/pull/31711>`_, Patrick Seidensal)
+* msg/async, v2: make the reset_recv_state() unconditional (`issue#40115 <http://tracker.ceph.com/issues/40115>`_, `pr#28453 <https://github.com/ceph/ceph/pull/28453>`_, Sage Weil, Radoslaw Zarzynski)
+* msg/async/AsyncConnection: optimize check loopback connection (`pr#26923 <https://github.com/ceph/ceph/pull/26923>`_, Jianpeng Ma)
+* msg/async/dpdk: destroy fd in do_request (`pr#32690 <https://github.com/ceph/ceph/pull/32690>`_, Chunsong Feng, luo rixin)
+* msg/async/dpdk: Fix build when DPDK enabled (`pr#33203 <https://github.com/ceph/ceph/pull/33203>`_, Jun Su)
+* msg/async/dpdk: fix compilation errors when WITH_DPDK=on (`pr#31840 <https://github.com/ceph/ceph/pull/31840>`_, Chunsong Feng)
+* msg/async/dpdk: fix complie errors from fix FTBFS (`pr#30086 <https://github.com/ceph/ceph/pull/30086>`_, yehu)
+* msg/async/dpdk: fix FTBFS (`pr#28763 <https://github.com/ceph/ceph/pull/28763>`_, Kefu Chai)
+* msg/async/dpdk: Fix infinite loop when sending packets (`pr#32691 <https://github.com/ceph/ceph/pull/32691>`_, Chunsong Feng, luo rixin)
+* msg/async/dpdk: fix SEGV caused by zero length packet (`pr#31876 <https://github.com/ceph/ceph/pull/31876>`_, Chunsong Feng)
+* msg/async/dpdk: Fix the overflow while parsing dpdk coremask (`pr#32173 <https://github.com/ceph/ceph/pull/32173>`_, Hu Ye, Chunsong Feng, luo rixin)
+* msg/async/DPDK: refactor set_rss_table to support DPDK 19.05 (`pr#32170 <https://github.com/ceph/ceph/pull/32170>`_, Chunsong Feng, luo rixin)
+* msg/async/EventEpoll: set EPOLLET flag on del_event() (`pr#26926 <https://github.com/ceph/ceph/pull/26926>`_, Roman Penyaev)
+* msg/async/ProtocolV1: avoid unnecessary bufferlist::swap (`pr#30125 <https://github.com/ceph/ceph/pull/30125>`_, Jianpeng Ma)
+* msg/async/ProtocolV2: make v2 work on rdma (`pr#27022 <https://github.com/ceph/ceph/pull/27022>`_, Jianpeng Ma)
+* msg/async/ProtocolV2: optimize check state by replace (`pr#26812 <https://github.com/ceph/ceph/pull/26812>`_, Jianpeng Ma)
+* msg/async/rdma: add an option for choosing different RoCE protocol (`pr#31517 <https://github.com/ceph/ceph/pull/31517>`_, Changcheng Liu)
+* msg/async/rdma: do not init mutex before lockdeps is ready (`pr#31532 <https://github.com/ceph/ceph/pull/31532>`_, Kefu Chai)
+* msg/async/rdma: fix memory leak (`pr#27574 <https://github.com/ceph/ceph/pull/27574>`_, Changcheng Liu)
+* msg/async/rdma: set/get silence warning (`pr#26581 <https://github.com/ceph/ceph/pull/26581>`_, Kefu Chai)
+* msg/async/rdma: unblock event center if the peer is down when connecting (`pr#31109 <https://github.com/ceph/ceph/pull/31109>`_, Peng Liu)
+* msg/async: add comments for commit 294c41f18adada6a (`pr#28667 <https://github.com/ceph/ceph/pull/28667>`_, Jianpeng Ma)
+* msg/async: add timeout for connections which are not ready (`issue#38493 <http://tracker.ceph.com/issues/38493>`_, `issue#37499 <http://tracker.ceph.com/issues/37499>`_, `pr#27337 <https://github.com/ceph/ceph/pull/27337>`_, xie xingguo)
+* msg/async: avoid creating unnecessary AsyncConnectionRef (`pr#27323 <https://github.com/ceph/ceph/pull/27323>`_, Patrick Donnelly)
+* msg/async: Dont dec(msgr_active_connections) if conn still in accept\xe2\x80\xa6 (`pr#29836 <https://github.com/ceph/ceph/pull/29836>`_, Jianpeng Ma)
+* msg/async: Don\t miss record l_msgr_running_recv_time if pendingRead\xe2\x80\xa6 (`pr#27734 <https://github.com/ceph/ceph/pull/27734>`_, Jianpeng Ma)
+* msg/async: drop zero_copy_read() & co from ConnectedSocket (`pr#28921 <https://github.com/ceph/ceph/pull/28921>`_, Radoslaw Zarzynski)
+* msg/async: fix typo in Errormessage (`pr#31825 <https://github.com/ceph/ceph/pull/31825>`_, Willem Jan Withagen)
+* msg/async: mark down local_connection before draining the stack (`pr#32732 <https://github.com/ceph/ceph/pull/32732>`_, Radoslaw Zarzynski)
+* msg/async: move submit_message() into send_to() (`pr#30883 <https://github.com/ceph/ceph/pull/30883>`_, Jianpeng Ma)
+* msg/async: narrow scope of AsyncMessenger::lock in fun connect_to (`pr#30840 <https://github.com/ceph/ceph/pull/30840>`_, Jianpeng Ma)
+* msg/async: No need lock for func _filter_addrs (`pr#31995 <https://github.com/ceph/ceph/pull/31995>`_, Jianpeng Ma)
+* msg/async: no-need set connection for Message (`pr#27766 <https://github.com/ceph/ceph/pull/27766>`_, Jianpeng Ma)
+* msg/async: open() should be called with connection locked (`pr#33015 <https://github.com/ceph/ceph/pull/33015>`_, Roman Penyaev)
+* msg/async: perform recv reset immediately if called inside EC (`pr#33742 <https://github.com/ceph/ceph/pull/33742>`_, Radoslaw Zarzynski)
+* msg/async: remove unsued code (`pr#30833 <https://github.com/ceph/ceph/pull/30833>`_, Jianpeng Ma)
+* msg/async: rename outcoming_bl -> outgoing_bl in AsyncConnection (`pr#30709 <https://github.com/ceph/ceph/pull/30709>`_, Radoslaw Zarzynski)
+* msg/async: reset the V1s session_security in proper EventCenter (`pr#32352 <https://github.com/ceph/ceph/pull/32352>`_, Radoslaw Zarzynski)
+* msg/async: resolve gcc warning (`pr#27414 <https://github.com/ceph/ceph/pull/27414>`_, Patrick Donnelly)
+* msg/async: skip repeat calc crc header in Message::encode (`pr#26534 <https://github.com/ceph/ceph/pull/26534>`_, Jianpeng Ma)
+* msg/async: update refcount and perf counter properly (`pr#31929 <https://github.com/ceph/ceph/pull/31929>`_, Jianpeng Ma)
+* msg/async: use faster clear method to delete containers (`pr#27324 <https://github.com/ceph/ceph/pull/27324>`_, Patrick Donnelly)
+* msg/Message: Remove used code about XioMessenger (`pr#28719 <https://github.com/ceph/ceph/pull/28719>`_, Jianpeng Ma)
+* msg: add func is_blackhole to reduce duplicated code (`pr#30356 <https://github.com/ceph/ceph/pull/30356>`_, Jianpeng Ma)
+* msg: add some anonymous connection infrastructure (`pr#30223 <https://github.com/ceph/ceph/pull/30223>`_, Sage Weil)
+* msg: default to debug_ms=0 (`pr#26936 <https://github.com/ceph/ceph/pull/26936>`_, Sage Weil)
+* msg: fix addr2 encoding for sockaddrs (`issue#40114 <http://tracker.ceph.com/issues/40114>`_, `pr#28379 <https://github.com/ceph/ceph/pull/28379>`_, Jeff Layton)
+* msg: fix comments in Messenger.h after the set -> std::set switch (`pr#30693 <https://github.com/ceph/ceph/pull/30693>`_, Radoslaw Zarzynski)
+* msg: output peer address when detecting bad CRCs (`issue#39367 <http://tracker.ceph.com/issues/39367>`_, `pr#27658 <https://github.com/ceph/ceph/pull/27658>`_, Greg Farnum)
+* msg: remove unused header file in Messenger.h (`pr#27086 <https://github.com/ceph/ceph/pull/27086>`_, Jianpeng Ma)
+* msg: remove xiomessenger (`pr#27021 <https://github.com/ceph/ceph/pull/27021>`_, Sage Weil)
+* msg: set_require_authorizer on messenger, not dispatcher (`pr#27832 <https://github.com/ceph/ceph/pull/27832>`_, Sage Weil)
+* orchestrator: usability fixes (`pr#33118 <https://github.com/ceph/ceph/pull/33118>`_, Yehuda Sadeh)
+* os/bluestore,comon,erasure-code: chmod -x source files (`pr#31179 <https://github.com/ceph/ceph/pull/31179>`_, Sage Weil)
+* os/bluestore: default bluestore_block_size 1T -> 100G (`pr#32043 <https://github.com/ceph/ceph/pull/32043>`_, Sage Weil)
+* os/kstore: do not cache in-fight stripes on read ops to avoid leaks (`issue#39665 <http://tracker.ceph.com/issues/39665>`_, `pr#32538 <https://github.com/ceph/ceph/pull/32538>`_, Chang Liu)
+* os/memstore, crimson/os: introduce memstore_debug_omit_block_device_write (`pr#28601 <https://github.com/ceph/ceph/pull/28601>`_, Radoslaw Zarzynski)
+* osd: a few fixes for the removed_snaps changes (`pr#28865 <https://github.com/ceph/ceph/pull/28865>`_, Sage Weil)
+* osd: accident of rollforward may need to mark pglog dirty (`issue#40403 <http://tracker.ceph.com/issues/40403>`_, `pr#28621 <https://github.com/ceph/ceph/pull/28621>`_, Zengran Zhang)
+* osd: add a copy-from2 operation that includes truncate\\_{seq,size} parameters (`pr#31728 <https://github.com/ceph/ceph/pull/31728>`_, Luis Henriques)
+* osd: add ceph osd stop <osd.nnn> command (`pr#27595 <https://github.com/ceph/ceph/pull/27595>`_, xie xingguo)
+* osd: add cls_cxx_map_remove_range() (`issue#19975 <http://tracker.ceph.com/issues/19975>`_, `pr#15183 <https://github.com/ceph/ceph/pull/15183>`_, Casey Bodley)
+* osd: add common smartctl output to JSON output (`pr#30408 <https://github.com/ceph/ceph/pull/30408>`_, Patrick Seidensal)
+* osd: add device_id to list_devices to help get smart info easily (`pr#29548 <https://github.com/ceph/ceph/pull/29548>`_, Song Shun)
+* osd: add duration field to dump_historic_ops method (`pr#28801 <https://github.com/ceph/ceph/pull/28801>`_, Deepika Upadhyay)
+* osd: add flag to prevent truncate_seq copy in copy-from operation (`pr#25374 <https://github.com/ceph/ceph/pull/25374>`_, Luis Henriques)
+* osd: add hdd and ssd variants for osd_recovery_max_active (`pr#28677 <https://github.com/ceph/ceph/pull/28677>`_, Sage Weil)
+* osd: add log information to record the cause of do_osd_ops failure (`issue#41210 <http://tracker.ceph.com/issues/41210>`_, `pr#29787 <https://github.com/ceph/ceph/pull/29787>`_, NancySu05)
+* osd: add osd_fast_shutdown option (default true) (`pr#31677 <https://github.com/ceph/ceph/pull/31677>`_, Sage Weil)
+* osd: Again remove deprecated full/nearfull from osdmap (`pr#32506 <https://github.com/ceph/ceph/pull/32506>`_, David Zafman)
+* osd: Allow 64-char hostname to be added as the host in CRUSH (`pr#32947 <https://github.com/ceph/ceph/pull/32947>`_, Michal Skalski)
+* osd: allow EC PGs to do recovery below min_size (`issue#18749 <http://tracker.ceph.com/issues/18749>`_, `pr#17619 <https://github.com/ceph/ceph/pull/17619>`_, Chang Liu, Greg Farnum)
+* osd: allow rados write ops to return data and error codes (`pr#30581 <https://github.com/ceph/ceph/pull/30581>`_, Sage Weil)
+* osd: always initialize local variable (`pr#29757 <https://github.com/ceph/ceph/pull/29757>`_, Kefu Chai)
+* osd: assert that write ops have result==0 and no payload (`pr#30191 <https://github.com/ceph/ceph/pull/30191>`_, Sage Weil)
+* osd: automatically repair replicated replica on pulling error (`issue#39101 <http://tracker.ceph.com/issues/39101>`_, `pr#26806 <https://github.com/ceph/ceph/pull/26806>`_, xie xingguo, David Zafman)
+* osd: avoid prep_object_replica_pushes() on clone object when head missing (`issue#39286 <http://tracker.ceph.com/issues/39286>`_, `pr#27575 <https://github.com/ceph/ceph/pull/27575>`_, Zengran Zhang)
+* osd: Better error message when OSD count is less than osd_pool_default_size (`issue#38617 <http://tracker.ceph.com/issues/38617>`_, `pr#27806 <https://github.com/ceph/ceph/pull/27806>`_, Sage Weil, zjh)
+* osd: Change osd op queue cut off default to high (`pr#30441 <https://github.com/ceph/ceph/pull/30441>`_, Anthony DAtri)
+* osd: clean up osdmap sharing (`pr#27932 <https://github.com/ceph/ceph/pull/27932>`_, Sage Weil)
+* osd: clear osd op reply output only when writes success (`issue#38492 <http://tracker.ceph.com/issues/38492>`_, `pr#26652 <https://github.com/ceph/ceph/pull/26652>`_, huangjun)
+* osd: clear PG_STATE_CLEAN when repair object (`pr#29756 <https://github.com/ceph/ceph/pull/29756>`_, Zengran Zhang)
+* osd: copy (dont move) pg list when sending beacon (`issue#40377 <http://tracker.ceph.com/issues/40377>`_, `pr#28566 <https://github.com/ceph/ceph/pull/28566>`_, Sage Weil)
+* osd: copy ObjectOperation::BufferUpdate::Write::fadvise_flag to ceph::os::Transaction (`pr#29944 <https://github.com/ceph/ceph/pull/29944>`_, Xuehan Xu)
+* osd: copyfrom omitted to set mtime (`pr#28581 <https://github.com/ceph/ceph/pull/28581>`_, Zengran Zhang)
+* osd: correct a local variable type (`pr#26672 <https://github.com/ceph/ceph/pull/26672>`_, Kefu Chai)
+* osd: Diagnostic logging for upmap cleaning (`pr#32663 <https://github.com/ceph/ceph/pull/32663>`_, David Zafman)
+* osd: dispatch peering messages as messages, inside the PG lock (`pr#29820 <https://github.com/ceph/ceph/pull/29820>`_, Sage Weil)
+* osd: dispatch_context and queue split finish on early bail-out (`pr#32942 <https://github.com/ceph/ceph/pull/32942>`_, Sage Weil)
+* osd: do not hold osd_lock while requeuing snaps to purge (`pr#28941 <https://github.com/ceph/ceph/pull/28941>`_, Sage Weil)
+* osd: do not invalidate clear_regions of missing item at boot (`pr#29755 <https://github.com/ceph/ceph/pull/29755>`_, xie xingguo)
+* osd: dont carry PGLSFilter between multiple ops in MOSDOp (`pr#29575 <https://github.com/ceph/ceph/pull/29575>`_, Radoslaw Zarzynski)
+* osd: Dont evict after a flush if intersecting scrub range (`issue#38840 <http://tracker.ceph.com/issues/38840>`_, `pr#27209 <https://github.com/ceph/ceph/pull/27209>`_, David Zafman)
+* osd: Dont include user changeable flag in snaptrim related assert (`issue#38124 <http://tracker.ceph.com/issues/38124>`_, `pr#27830 <https://github.com/ceph/ceph/pull/27830>`_, David Zafman)
+* osd: Dont randomize deep scrubs when noscrub set (`issue#40198 <http://tracker.ceph.com/issues/40198>`_, `pr#28443 <https://github.com/ceph/ceph/pull/28443>`_, David Zafman)
+* osd: drop unnecessary includes of messages/MOSDPGTrim.h (`pr#33660 <https://github.com/ceph/ceph/pull/33660>`_, Radoslaw Zarzynski)
+* osd: Fix assert in the case that snapset is missing (`pr#29941 <https://github.com/ceph/ceph/pull/29941>`_, David Zafman)
+* osd: fix possible crash on sending dynamic perf stats report (`pr#30454 <https://github.com/ceph/ceph/pull/30454>`_, Mykola Golub)
+* osd: fix racy accesses to OSD::osdmap (`pr#33336 <https://github.com/ceph/ceph/pull/33336>`_, Radoslaw Zarzynski)
+* osd: fix the missing default value m=2 of reed_sol_r6_op in profile (`pr#29892 <https://github.com/ceph/ceph/pull/29892>`_, Yan Jun)
+* osd: Fix the way that auto repair triggers after regular scru (`issue#40073 <http://tracker.ceph.com/issues/40073>`_, `issue#40530 <http://tracker.ceph.com/issues/40530>`_, `pr#28334 <https://github.com/ceph/ceph/pull/28334>`_, David Zafman)
+* osd: fix wrong arguments when dropping refcount (`pr#29348 <https://github.com/ceph/ceph/pull/29348>`_, Myoungwon Oh)
+* osd: Give recovery for inactive PGs a higher priority (`issue#38195 <http://tracker.ceph.com/issues/38195>`_, `pr#27503 <https://github.com/ceph/ceph/pull/27503>`_, David Zafman)
+* osd: give recovery ops initialized by client op a higher priority (`pr#28418 <https://github.com/ceph/ceph/pull/28418>`_, xie xingguo)
+* osd: implement per-pg leases to avoid stale reads (`pr#29236 <https://github.com/ceph/ceph/pull/29236>`_, Sage Weil)
+* osd: Improve dump_pgstate_history json output (`issue#38846 <http://tracker.ceph.com/issues/38846>`_, `pr#27665 <https://github.com/ceph/ceph/pull/27665>`_, Brad Hubbard)
+* osd: Include dups in copy_after() and copy_up_to() (`issue#39304 <http://tracker.ceph.com/issues/39304>`_, `pr#27914 <https://github.com/ceph/ceph/pull/27914>`_, David Zafman)
+* osd: Increase log level of messages which unnecessarily fill up logs (`pr#27686 <https://github.com/ceph/ceph/pull/27686>`_, David Zafman)
+* osd: make osd recover more smoothly by avoiding failure peer info to resent (`pr#30404 <https://github.com/ceph/ceph/pull/30404>`_, \xe5\xae\x8b\xe9\xa1\xba10180185)
+* osd: make PastIntervals a member of pg_notify_t (`pr#29517 <https://github.com/ceph/ceph/pull/29517>`_, Sage Weil)
+* osd: merge replica log on primary need according to replica logs crt (`pr#29590 <https://github.com/ceph/ceph/pull/29590>`_, Zengran Zhang)
+* osd: misc cleanups (`pr#30022 <https://github.com/ceph/ceph/pull/30022>`_, Yan Jun)
+* osd: misc inc-recovery compat fixes (`pr#29754 <https://github.com/ceph/ceph/pull/29754>`_, xie xingguo)
+* osd: optimize send_message to peers (`pr#30968 <https://github.com/ceph/ceph/pull/30968>`_, Jianpeng Ma)
+* osd: OSDMapRef access by multiple threads is unsafe (`pr#26874 <https://github.com/ceph/ceph/pull/26874>`_, Kefu Chai, Zengran Zhang)
+* osd: Output Base64 encoding of CRC header if binary data present (`pr#27961 <https://github.com/ceph/ceph/pull/27961>`_, David Zafman)
+* osd: partial recovery strategy based on PGLog (`pr#21722 <https://github.com/ceph/ceph/pull/21722>`_, lishuhao, Ning Yao)
+* osd: peering updates peer_last_complete_ondisk via setter (`pr#33659 <https://github.com/ceph/ceph/pull/33659>`_, Radoslaw Zarzynski)
+* osd: pg as a mutex (`pr#29477 <https://github.com/ceph/ceph/pull/29477>`_, Kefu Chai)
+* osd: prime splits/merges for any potential fabricated split/merge participant (`issue#38483 <http://tracker.ceph.com/issues/38483>`_, `pr#30018 <https://github.com/ceph/ceph/pull/30018>`_, xie xingguo)
+* osd: process_copy_chunk remove obc ref before pg unlock (`issue#38842 <http://tracker.ceph.com/issues/38842>`_, `pr#27084 <https://github.com/ceph/ceph/pull/27084>`_, Zengran Zhang)
+* osd: propagate mlcod to replicas and fix problems with read from replica (`pr#32381 <https://github.com/ceph/ceph/pull/32381>`_, Samuel Just, Sage Weil)
+* osd: release backoffs during merge (`pr#31657 <https://github.com/ceph/ceph/pull/31657>`_, Sage Weil)
+* osd: remove orphan include after PGLSParentFilter (`pr#29709 <https://github.com/ceph/ceph/pull/29709>`_, Radoslaw Zarzynski)
+* osd: remove unused function (`pr#30644 <https://github.com/ceph/ceph/pull/30644>`_, Jianpeng Ma)
+* osd: remove unused functions (`pr#32515 <https://github.com/ceph/ceph/pull/32515>`_, Jianpeng Ma)
+* osd: Remove unused osdmap flags full, nearfull from output (`pr#30530 <https://github.com/ceph/ceph/pull/30530>`_, David Zafman)
+* osd: remove useless ceph_assert (`pr#31915 <https://github.com/ceph/ceph/pull/31915>`_, Jianpeng Ma)
+* osd: revamp {noup,nodown,noin,noout} related commands (`pr#27735 <https://github.com/ceph/ceph/pull/27735>`_, xie xingguo)
+* osd: rollforward may need to mark pglog dirty (`issue#36739 <http://tracker.ceph.com/issues/36739>`_, `pr#27015 <https://github.com/ceph/ceph/pull/27015>`_, Zengran Zhang)
+* osd: scrub error on big objects; make bluestore refuse to start on big objects (`pr#29579 <https://github.com/ceph/ceph/pull/29579>`_, David Zafman, Sage Weil)
+* osd: send smart asok result to stdout, not stderr (`pr#31412 <https://github.com/ceph/ceph/pull/31412>`_, Sage Weil)
+* osd: set affinity for \\*all\\* threads (`pr#30712 <https://github.com/ceph/ceph/pull/30712>`_, Sage Weil)
+* osd: set collection pool opts on collection create, pg load (`pr#29093 <https://github.com/ceph/ceph/pull/29093>`_, Sage Weil)
+* osd: share curmap in handle_osd_ping (`pr#28662 <https://github.com/ceph/ceph/pull/28662>`_, Sage Weil)
+* osd: shutdown recovery_request_timer earlier (`pr#27206 <https://github.com/ceph/ceph/pull/27206>`_, Zengran Zhang)
+* osd: some prelim changes (`pr#29052 <https://github.com/ceph/ceph/pull/29052>`_, Sage Weil)
+* osd: support osd_repair_during_recovery (`issue#40620 <http://tracker.ceph.com/issues/40620>`_, `pr#28839 <https://github.com/ceph/ceph/pull/28839>`_, Jeegn Chen)
+* osd: support osd_scrub_extended_sleep (`issue#40955 <http://tracker.ceph.com/issues/40955>`_, `pr#29342 <https://github.com/ceph/ceph/pull/29342>`_, Jeegn Chen)
+* osd: take heartbeat_lock when calling heartbeat() (`issue#39439 <http://tracker.ceph.com/issues/39439>`_, `pr#27729 <https://github.com/ceph/ceph/pull/27729>`_, Sage Weil)
+* osd: tiny clean-ups around the backfill (`pr#33583 <https://github.com/ceph/ceph/pull/33583>`_, Radoslaw Zarzynski)
+* osd: track monotonic clock deltas between osds who ping each other (`pr#29116 <https://github.com/ceph/ceph/pull/29116>`_, Sage Weil, Samuel Just)
+* osd: transpose two wait lists in comment (`pr#27017 <https://github.com/ceph/ceph/pull/27017>`_, Kefu Chai)
+* osd: trim pg logs based on a per-osd budget (`pr#32683 <https://github.com/ceph/ceph/pull/32683>`_, Sage Weil, Kefu Chai)
+* osd: Turn off repair pg state when leaving recovery (`pr#30852 <https://github.com/ceph/ceph/pull/30852>`_, David Zafman)
+* osd: unify sources of no{up,down,in,out} flags into singleton helpers (`pr#28403 <https://github.com/ceph/ceph/pull/28403>`_, xie xingguo)
+* osd: update comment as sub_op_scrub_map has been removed (`pr#28338 <https://github.com/ceph/ceph/pull/28338>`_, Jing Wenjun)
+* osd: Use physical ratio for nearfull (doesnt include backfill resserve) (`pr#31954 <https://github.com/ceph/ceph/pull/31954>`_, David Zafman)
+* osd: use steady clock in prepare_to_stop() (`pr#26457 <https://github.com/ceph/ceph/pull/26457>`_, Mohamad Gebai)
+* osd: use unique_ptr for managing life cycles (`pr#32007 <https://github.com/ceph/ceph/pull/32007>`_, Kefu Chai)
+* osdc/Striper: specialize std::min<> (`pr#28732 <https://github.com/ceph/ceph/pull/28732>`_, Kefu Chai)
+* osd_types: add ec profile to plain text osd pool ls detail output (`issue#40009 <http://tracker.ceph.com/issues/40009>`_, `pr#28224 <https://github.com/ceph/ceph/pull/28224>`_, Jan Fajerski)
+* pybind,rbd: Add RBD_FEATURE_MIGRATING to rbd.pyx (`issue#39609 <http://tracker.ceph.com/issues/39609>`_, `pr#28009 <https://github.com/ceph/ceph/pull/28009>`_, Ricardo Marques)
+* pybind,rbd: pybind/rbd: add config_set/get/remove api in rbd.pyx (`pr#29459 <https://github.com/ceph/ceph/pull/29459>`_, Zheng Yin)
+* pybind,rbd: pybind/rbd: add pool config_set/get/remove api in rbd.pyx (`pr#30865 <https://github.com/ceph/ceph/pull/30865>`_, Zheng Yin)
+* pybind,rbd: pybind/rbd: parent_info should return pool namespace (`pr#30793 <https://github.com/ceph/ceph/pull/30793>`_, Ricardo Marques)
+* pybind,rbd: rbd/pybind: fix unsupported format character of %lx (`pr#30314 <https://github.com/ceph/ceph/pull/30314>`_, songweibin)
+* pybind,tests: pybind/rados: do not slice zip() (`pr#31044 <https://github.com/ceph/ceph/pull/31044>`_, Kefu Chai)
+* pybind,tests: test/pybind/test_rados.py: test test_operate_aio_write_op() (`pr#31158 <https://github.com/ceph/ceph/pull/31158>`_, Zhang Jiao)
+* pybind/mgr: Add test_orchestrator to mypy (`pr#32500 <https://github.com/ceph/ceph/pull/32500>`_, Sebastian Wagner)
+* pybind/mgr: add_tox_test: Add mypy to TOX_ENVS (`pr#32236 <https://github.com/ceph/ceph/pull/32236>`_, Sebastian Wagner)
+* pybind/mgr: bump six to 1.14 (`pr#33185 <https://github.com/ceph/ceph/pull/33185>`_, Kefu Chai)
+* pybind/tox: pass additional command line arguments through to tox (`pr#27947 <https://github.com/ceph/ceph/pull/27947>`_, Nathan Cutler)
+* pybind: .gitignore: Add .mypy_cache to .gitignore (`pr#33510 <https://github.com/ceph/ceph/pull/33510>`_, Kristoffer Gr\xc3\xb6nlund)
+* pybind: add verbose error message (`pr#28054 <https://github.com/ceph/ceph/pull/28054>`_, Daniel Badea, Changcheng Liu, Ovidiu Poncea)
+* pybind: add WriteOp::set_xattr() & rm_xattr() (`pr#31829 <https://github.com/ceph/ceph/pull/31829>`_, Zhang Jiao)
+* pybind: add writesame API (`pr#31489 <https://github.com/ceph/ceph/pull/31489>`_, Zhang Jiao)
+* pybind: check CEPH_LIBDIR not MAKEFLAGS (`pr#29080 <https://github.com/ceph/ceph/pull/29080>`_, Kefu Chai)
+* pybind: customize compiler before checking cflags (`pr#33177 <https://github.com/ceph/ceph/pull/33177>`_, Kefu Chai)
+* pybind: fix use of WriteOpCtx and ReadOpCtx (`issue#38946 <http://tracker.ceph.com/issues/38946>`_, `pr#27213 <https://github.com/ceph/ceph/pull/27213>`_, Ramana Raja)
+* pybind: pybind/rados/rados.pyx: improve Rados.create_pool() (`pr#31241 <https://github.com/ceph/ceph/pull/31241>`_, Zhang Jiao)
+* pybind: pybind/rados: add application_metadata_get (`pr#30504 <https://github.com/ceph/ceph/pull/30504>`_, songweibin)
+* pybind: pybind/rados: add Ioctx.get_pool_id() and Ioctx.get_pool_name() (`pr#29646 <https://github.com/ceph/ceph/pull/29646>`_, Zheng Yin)
+* pybind: pybind/rados: add WriteOp::execute() (`pr#31546 <https://github.com/ceph/ceph/pull/31546>`_, Zhang Jiao)
+* pybind: pybind/rados: should pass name to cstr() (`pr#27111 <https://github.com/ceph/ceph/pull/27111>`_, Kefu Chai)
+* pybind: refactor monkey_with_compiler() (`pr#33061 <https://github.com/ceph/ceph/pull/33061>`_, Kefu Chai)
+* pybind: set language_level for cythonize explicitly (`pr#26607 <https://github.com/ceph/ceph/pull/26607>`_, Kefu Chai)
+* python-common, mgr/orchestrator, mgr/dashboard: Use common Devices (`pr#30662 <https://github.com/ceph/ceph/pull/30662>`_, Kiefer Chang, Sebastian Wagner)
+* python-common: add unmanaged property to PlacementSpec (`pr#33955 <https://github.com/ceph/ceph/pull/33955>`_, Sage Weil)
+* python-common: all:true -> \\* (`pr#33970 <https://github.com/ceph/ceph/pull/33970>`_, Sage Weil)
+* python-common: move pytest integration from setup.py to tox.ini (`pr#31943 <https://github.com/ceph/ceph/pull/31943>`_, Sebastian Wagner)
+* python-common: remove `all_hosts` from `PlacementSpec` (`pr#33948 <https://github.com/ceph/ceph/pull/33948>`_, Sebastian Wagner)
+* qa/distros: rhel and centos: whitelist cephadm logrotate selinux denial (`pr#33110 <https://github.com/ceph/ceph/pull/33110>`_, Sage Weil)
+* qa/standalone/test_ceph_daemon.sh: disable adoption for the moment (`pr#32178 <https://github.com/ceph/ceph/pull/32178>`_, Sage Weil)
+* qa/standalone/test_ceph_daemon.sh: fix overwrites of temp files (`pr#31748 <https://github.com/ceph/ceph/pull/31748>`_, Sage Weil)
+* qa/standalone/test_ceph_daemon: fix multi-version python test (`pr#31342 <https://github.com/ceph/ceph/pull/31342>`_, Sage Weil)
+* qa/suites/cephadm: move orchestrator_cli test into rados/cephadm (`pr#33648 <https://github.com/ceph/ceph/pull/33648>`_, Sage Weil)
+* qa/suites/rados/ceph: drop opensuse for now (`pr#33801 <https://github.com/ceph/ceph/pull/33801>`_, Sage Weil)
+* qa/suites/rados/cephadm/smoke: disable rgw role for now (`pr#33360 <https://github.com/ceph/ceph/pull/33360>`_, Sage Weil)
+* qa/suites/rados/cephadm/upgrade: change start version (`pr#33475 <https://github.com/ceph/ceph/pull/33475>`_, Sage Weil)
+* qa/suites/rados/cephadm/upgrade: fix initial version (`pr#33396 <https://github.com/ceph/ceph/pull/33396>`_, Sage Weil)
+* qa/suites/rados/cephadm: explicitly test many distros (`pr#32969 <https://github.com/ceph/ceph/pull/32969>`_, Sage Weil)
+* qa/suites/rados/cephadm: fix conflicts, missing .qa link (`pr#33132 <https://github.com/ceph/ceph/pull/33132>`_, Sage Weil)
+* qa/suites/rados/cephadm[-smoke]: test podman on ubuntu 18.04 (`pr#33111 <https://github.com/ceph/ceph/pull/33111>`_, Sage Weil)
+* qa/tasks/cephadm: ceph.git branches are now pushed to quay.io (`pr#32375 <https://github.com/ceph/ceph/pull/32375>`_, Sage Weil)
+* qa/tasks/cephadm: deploy rgw daemons too (`pr#33289 <https://github.com/ceph/ceph/pull/33289>`_, Sage Weil)
+* qa/tasks/cephadm: learn to pull cephadm from githu (`pr#32787 <https://github.com/ceph/ceph/pull/32787>`_, Sage Weil)
+* qa/tasks/cephadm: misc fixes (`pr#32713 <https://github.com/ceph/ceph/pull/32713>`_, Sage Weil)
+* qa/tasks/ceph_manager.py: always use self.logger (`pr#29239 <https://github.com/ceph/ceph/pull/29239>`_, Kefu Chai)
+* qa/tasks/ceph_manager: 5s -> 15s for osd out to be visible (`pr#29013 <https://github.com/ceph/ceph/pull/29013>`_, Sage Weil)
+* qa/tasks/ceph_manager: fix movement of cot exports with cephadm (`pr#32986 <https://github.com/ceph/ceph/pull/32986>`_, Sage Weil)
+* qa/tasks/ceph_manager: fix shell osd for ceph-objectstore-tool commands (`pr#32725 <https://github.com/ceph/ceph/pull/32725>`_, Sage Weil)
+* qa/tasks/ceph_manager: make fix_pgp_num behave when no pool is found (`pr#32987 <https://github.com/ceph/ceph/pull/32987>`_, Sage Weil)
+* qa/tasks/mgr/dashboard/test_health: update schema (`pr#30507 <https://github.com/ceph/ceph/pull/30507>`_, Kefu Chai)
+* qa/tasks/mgr/dashboard/test_orchestrator: support addr attribute in inventory (`pr#33211 <https://github.com/ceph/ceph/pull/33211>`_, Kiefer Chang)
+* qa/tasks/mgr/test_orchestrator_cli: fix device ls test (`pr#32384 <https://github.com/ceph/ceph/pull/32384>`_, Sage Weil)
+* qa/tasks/mgr/test_orchestrator_cli: fix rgw add test (`pr#32101 <https://github.com/ceph/ceph/pull/32101>`_, Sage Weil)
+* qa/tasks/mgr/test_orchestrator_cli: support multiple DriveGroups (`pr#33055 <https://github.com/ceph/ceph/pull/33055>`_, Kiefer Chang)
+* qa/test: reduce over all number of runs (`pr#27979 <https://github.com/ceph/ceph/pull/27979>`_, Yuri Weinstein)
+* qa/tests - cleaned up distro settings (`pr#27956 <https://github.com/ceph/ceph/pull/27956>`_, Yuri Weinstein)
+* qa/tests - upped priority for upgrades on master, otherwise they neve\xe2\x80\xa6 (`pr#29666 <https://github.com/ceph/ceph/pull/29666>`_, Yuri Weinstein)
+* qa/tests: added nautilus-x-singleton suite to rados as symlink (`pr#27291 <https://github.com/ceph/ceph/pull/27291>`_, Sage Weil)
+* qa/tests: added rados on master, reduced fs, rbd, multimds (`pr#27535 <https://github.com/ceph/ceph/pull/27535>`_, Yuri Weinstein)
+* qa/tests: added the subset clause for nautilus branch (`pr#27129 <https://github.com/ceph/ceph/pull/27129>`_, Yuri Weinstein)
+* qa/tests: changed the TO email to ceph-qa@ceph.io (`pr#28721 <https://github.com/ceph/ceph/pull/28721>`_, Yuri Weinstein)
+* qa/tests: moved some runs from ovh, removed ceph-disk/nautilus (`pr#27616 <https://github.com/ceph/ceph/pull/27616>`_, Yuri Weinstein)
+* qa/tests: reduced runs for nautilus, added runs for octopus (`pr#33214 <https://github.com/ceph/ceph/pull/33214>`_, Yuri Weinstein)
+* qa/tests: removed all runs on ovh (`pr#27960 <https://github.com/ceph/ceph/pull/27960>`_, Yuri Weinstein)
+* qa/tests: removed filters for client-upgrade-\\* suites (`pr#28271 <https://github.com/ceph/ceph/pull/28271>`_, Yuri Weinstein)
+* qa/tests: run luminous-x and mimic-x 2 times a week but with high priority (`pr#27527 <https://github.com/ceph/ceph/pull/27527>`_, Yuri Weinstein)
+* qa/tests: trying to fix syntax error that prevented mimic-x to be add\xe2\x80\xa6 (`pr#31799 <https://github.com/ceph/ceph/pull/31799>`_, Yuri Weinstein)
+* qa/valgrind.supp: abstract from ceph::buffers symbol versioning (`pr#33757 <https://github.com/ceph/ceph/pull/33757>`_, Radoslaw Zarzynski)
+* qa/workunits/cephadm/test_adoption: run as root (`pr#33485 <https://github.com/ceph/ceph/pull/33485>`_, Sage Weil)
+* qa/workunits/cephadm/test_cephadm.sh: consolidate wait loop logic (`pr#33544 <https://github.com/ceph/ceph/pull/33544>`_, Michael Fritch)
+* qa/workunits/cephadm/test_cephadm.sh: dump logs on exit (`pr#33634 <https://github.com/ceph/ceph/pull/33634>`_, Michael Fritch)
+* qa/workunits/cephadm/test_cephadm.sh: need --fsid always (`pr#32220 <https://github.com/ceph/ceph/pull/32220>`_, Sage Weil)
+* qa/workunits/cephadm/test_cephadm.sh: re-enable `adopt` tests (`pr#32244 <https://github.com/ceph/ceph/pull/32244>`_, Michael Fritch)
+* qa/workunits/cephadm/test_cephadm.sh: skip docker when service is disabled (`pr#33018 <https://github.com/ceph/ceph/pull/33018>`_, Michael Fritch)
+* qa/workunits/cephadm/test_cephadm.sh: use avialable pythons; test on ubuntu and centos (`pr#32333 <https://github.com/ceph/ceph/pull/32333>`_, Sage Weil)
+* qa/workunits/cephadm/test_cephadm: --skip-monitoring-stack (`pr#34013 <https://github.com/ceph/ceph/pull/34013>`_, Sage Weil)
+* qa/workunits/cephadm/test_cephadm: fix typo (`pr#33181 <https://github.com/ceph/ceph/pull/33181>`_, Sage Weil)
+* qa/workunits/cephadm/test_cephadm: workunit test cleanup (`pr#32625 <https://github.com/ceph/ceph/pull/32625>`_, Michael Fritch)
+* qa/workunits/cephadm/test_repos: dont try to use the refspec (`pr#33134 <https://github.com/ceph/ceph/pull/33134>`_, Sage Weil)
+* qa/workunits/cephadm: separate out test_adoption.sh; fix (`pr#33457 <https://github.com/ceph/ceph/pull/33457>`_, Sage Weil)
+* qa: fixes (`pr#29361 <https://github.com/ceph/ceph/pull/29361>`_, Kefu Chai)
+* qa: misc fixes for rados and py3 (`pr#32362 <https://github.com/ceph/ceph/pull/32362>`_, Sage Weil)
+* qa: pin rgw/verify to 8.0 (`pr#32761 <https://github.com/ceph/ceph/pull/32761>`_, Ali Maredia)
+* qa: Run flake8 on python2 and python3 (`pr#32222 <https://github.com/ceph/ceph/pull/32222>`_, Thomas Bechtold)
+* qa: vstart_runner fails because of string index out of range (`pr#28990 <https://github.com/ceph/ceph/pull/28990>`_, Volker Theile)
+* rbd,tests: cls/rbd: add snapshot limit UINT64_MAX test case (`pr#31350 <https://github.com/ceph/ceph/pull/31350>`_, Chen Pan)
+* rbd,tests: cls/rbd: add snapshot_add raise -ESTALE test case (`pr#31149 <https://github.com/ceph/ceph/pull/31149>`_, wonderpow)
+* rbd,tests: journal: always shutdown JournalRecoreder before destructing it (`pr#29501 <https://github.com/ceph/ceph/pull/29501>`_, Kefu Chai)
+* rbd,tests: journal: fix flush by age and in-flight byte tracking (`pr#31392 <https://github.com/ceph/ceph/pull/31392>`_, Jason Dillaman)
+* rbd,tests: mgr/dashboard: s/fsid/mirror_uuid/ (`pr#33348 <https://github.com/ceph/ceph/pull/33348>`_, Kefu Chai)
+* rbd,tests: qa/rbd: add cram-based snap diff test (`issue#39447 <http://tracker.ceph.com/issues/39447>`_, `pr#28346 <https://github.com/ceph/ceph/pull/28346>`_, Shyukri Shyukriev, Nathan Cutler)
+* rbd,tests: qa/suites/krbd: run unmap subsuite with msgr1 only (`pr#31265 <https://github.com/ceph/ceph/pull/31265>`_, Ilya Dryomov)
+* rbd,tests: qa/suites/rbd: add random distro selection to librbd tests (`pr#27577 <https://github.com/ceph/ceph/pull/27577>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: added writearound cache test permutations (`issue#39386 <http://tracker.ceph.com/issues/39386>`_, `pr#27694 <https://github.com/ceph/ceph/pull/27694>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: fix errant tab in yaml which is causing parsing failures (`pr#30942 <https://github.com/ceph/ceph/pull/30942>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: fixed download path for Ubuntu Bionic (`pr#32408 <https://github.com/ceph/ceph/pull/32408>`_, Jason Dillaman)
+* rbd,tests: qa/suites/rbd: removed OpenStack tempest test cases (`pr#33900 <https://github.com/ceph/ceph/pull/33900>`_, Jason Dillaman)
+* rbd,tests: qa/tests: added rbd task on ec (`pr#29541 <https://github.com/ceph/ceph/pull/29541>`_, Yuri Weinstein)
+* rbd,tests: qa/workunit/rbd: fixed QoS throughput unit parsing (`pr#32280 <https://github.com/ceph/ceph/pull/32280>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: fix compare_images and compare_image_snapshots (`pr#28524 <https://github.com/ceph/ceph/pull/28524>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: fixed python interpreter for EL8 (`pr#32409 <https://github.com/ceph/ceph/pull/32409>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: fixups for the new krbd discard behavior (`pr#27192 <https://github.com/ceph/ceph/pull/27192>`_, Ilya Dryomov)
+* rbd,tests: qa/workunits/rbd: override CEPH_ARGS when initializing the site name (`pr#33187 <https://github.com/ceph/ceph/pull/33187>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: remove fast-diff from dynamic features test (`issue#39946 <http://tracker.ceph.com/issues/39946>`_, `pr#28135 <https://github.com/ceph/ceph/pull/28135>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: stress test `rbd mirror pool status --verbose` (`pr#29655 <https://github.com/ceph/ceph/pull/29655>`_, Mykola Golub)
+* rbd,tests: qa/workunits/rbd: use context managers to control Rados lifespan (`pr#34035 <https://github.com/ceph/ceph/pull/34035>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: use https protocol for devstack git operations (`issue#39656 <http://tracker.ceph.com/issues/39656>`_, `pr#28063 <https://github.com/ceph/ceph/pull/28063>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: use more recent qemu-iotests that support Bionic (`issue#24668 <http://tracker.ceph.com/issues/24668>`_, `pr#27683 <https://github.com/ceph/ceph/pull/27683>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: wait for nbd map to close after unmap (`pr#33898 <https://github.com/ceph/ceph/pull/33898>`_, Jason Dillaman)
+* rbd,tests: qa/workunits/rbd: wait for rbd-nbd unmap to complete (`issue#39598 <http://tracker.ceph.com/issues/39598>`_, `pr#27981 <https://github.com/ceph/ceph/pull/27981>`_, Jason Dillaman)
+* rbd,tests: qa: add device mapper and lvm test cases for stable pages (`pr#27271 <https://github.com/ceph/ceph/pull/27271>`_, Ilya Dryomov)
+* rbd,tests: qa: add krbd_discard_granularity.t test (`pr#27042 <https://github.com/ceph/ceph/pull/27042>`_, Ilya Dryomov)
+* rbd,tests: qa: add RBD QOS functional test (`pr#27137 <https://github.com/ceph/ceph/pull/27137>`_, Mykola Golub)
+* rbd,tests: qa: add script to test how libceph handles huge osdmaps (`pr#30363 <https://github.com/ceph/ceph/pull/30363>`_, Ilya Dryomov)
+* rbd,tests: qa: avoid hexdump skip and length options (`pr#30502 <https://github.com/ceph/ceph/pull/30502>`_, Ilya Dryomov)
+* rbd,tests: qa: avoid page cache for krbd discard round off tests (`pr#30452 <https://github.com/ceph/ceph/pull/30452>`_, Ilya Dryomov)
+* rbd,tests: qa: krbd_parent_overlap.t: fix read test (`pr#29966 <https://github.com/ceph/ceph/pull/29966>`_, Ilya Dryomov)
+* rbd,tests: test/cli-integration/rbd: fixed missing image and snap ids (`pr#29853 <https://github.com/ceph/ceph/pull/29853>`_, Jason Dillaman)
+* rbd,tests: test/cli-integration: fixed spacing issue for RBD formatted tables (`pr#33902 <https://github.com/ceph/ceph/pull/33902>`_, Jason Dillaman)
+* rbd,tests: test/cls_rbd/test_cls_rbd: update TestClsRbd.sparsify (`pr#30258 <https://github.com/ceph/ceph/pull/30258>`_, Kefu Chai)
+* rbd,tests: test/cls_rbd: include compat.h for ERESTART (`pr#32172 <https://github.com/ceph/ceph/pull/32172>`_, Willem Jan Withagen)
+* rbd,tests: test/journal: always close object (`pr#29476 <https://github.com/ceph/ceph/pull/29476>`_, Kefu Chai)
+* rbd,tests: test/librados_test_stub: ensure the log flusher thread is started (`pr#27326 <https://github.com/ceph/ceph/pull/27326>`_, Jason Dillaman)
+* rbd,tests: test/librbd: allow parallel runs of run-rbd-unit-tests (`pr#30072 <https://github.com/ceph/ceph/pull/30072>`_, Willem Jan Withagen)
+* rbd,tests: test/librbd: drop ceph_test_librbd_api target (`issue#39072 <http://tracker.ceph.com/issues/39072>`_, `pr#27695 <https://github.com/ceph/ceph/pull/27695>`_, Jason Dillaman)
+* rbd,tests: test/librbd: fix mock warnings in TestMockIoImageRequest (`pr#31497 <https://github.com/ceph/ceph/pull/31497>`_, Mykola Golub)
+* rbd,tests: test/librbd: set nbd timeout due to newer kernels defaulting it on (`pr#29858 <https://github.com/ceph/ceph/pull/29858>`_, Jason Dillaman)
+* rbd,tests: test/pybind/rbd.pyx: add test_remove_snap_by_id case in test_rbd.py (`pr#30927 <https://github.com/ceph/ceph/pull/30927>`_, Zhang Jiao)
+* rbd,tests: test/pybind: add create_snap rasie ImageExists test case (`pr#31140 <https://github.com/ceph/ceph/pull/31140>`_, Gangbiao Liu)
+* rbd,tests: test/pybind: inconsistent use of tabs and spaces in indentation (`pr#31606 <https://github.com/ceph/ceph/pull/31606>`_, Mykola Golub)
+* rbd,tests: test/rbd_mirror: fix mock warnings (`pr#31608 <https://github.com/ceph/ceph/pull/31608>`_, Mykola Golub)
+* rbd,tests: test/run-rbd-tests: properly initialize newly created rbd pool (`pr#33642 <https://github.com/ceph/ceph/pull/33642>`_, Mykola Golub)
+* rbd,tests: test: add test_remove_snap_ImageNotFound test case in remove snap part (`pr#31221 <https://github.com/ceph/ceph/pull/31221>`_, Yingze Wei)
+* rbd,tests: test:add test_remove_snap2 interface to remove  snap when its protected (`pr#31208 <https://github.com/ceph/ceph/pull/31208>`_, Yingze Wei)
+* rbd,tools: tools/rbd-ggate: close log before running postfork (`pr#30010 <https://github.com/ceph/ceph/pull/30010>`_, Willem Jan Withagen)
+* rbd,tools: tools/rbd_nbd: use POSIX basename() (`pr#28856 <https://github.com/ceph/ceph/pull/28856>`_, Kefu Chai)
+* rbd-ggate: fix fallout from bufferlist.copy() change (`pr#33057 <https://github.com/ceph/ceph/pull/33057>`_, Willem Jan Withagen)
+* rbd-mirror: add namespace support (`issue#37529 <http://tracker.ceph.com/issues/37529>`_, `pr#28939 <https://github.com/ceph/ceph/pull/28939>`_, Mykola Golub)
+* rbd-mirror: add namespace support to service daemon (`pr#31642 <https://github.com/ceph/ceph/pull/31642>`_, Mykola Golub)
+* rbd-mirror: add support for snapshot-based mirroring resyncs (`pr#33490 <https://github.com/ceph/ceph/pull/33490>`_, Jason Dillaman)
+* rbd-mirror: apply image state during snapshot replay (`pr#33335 <https://github.com/ceph/ceph/pull/33335>`_, Jason Dillaman)
+* rbd-mirror: cannot restore deferred deletion mirrored images (`pr#30351 <https://github.com/ceph/ceph/pull/30351>`_, Jason Dillaman)
+* rbd-mirror: clear out bufferlist prior to listing mirror images (`issue#39407 <http://tracker.ceph.com/issues/39407>`_, `pr#27720 <https://github.com/ceph/ceph/pull/27720>`_, Jason Dillaman)
+* rbd-mirror: continue to isolate journal replay logic (`pr#32399 <https://github.com/ceph/ceph/pull/32399>`_, Jason Dillaman)
+* rbd-mirror: do not auto-create peers in non-default namespaces (`pr#32341 <https://github.com/ceph/ceph/pull/32341>`_, Jason Dillaman)
+* rbd-mirror: dont expect image map is always initialized (`pr#33368 <https://github.com/ceph/ceph/pull/33368>`_, Mykola Golub)
+* rbd-mirror: dont overwrite status error returned by replay (`pr#28179 <https://github.com/ceph/ceph/pull/28179>`_, Mykola Golub)
+* rbd-mirror: ensure deterministic ordering of method calls (`pr#32274 <https://github.com/ceph/ceph/pull/32274>`_, Jason Dillaman)
+* rbd-mirror: extract journal replaying logic from image replayer (`pr#32257 <https://github.com/ceph/ceph/pull/32257>`_, Jason Dillaman)
+* rbd-mirror: fix pool replayer status for case when init failed (`pr#32483 <https://github.com/ceph/ceph/pull/32483>`_, Mykola Golub)
+* rbd-mirror: fix race on namespace replayer initialization failure (`pr#32243 <https://github.com/ceph/ceph/pull/32243>`_, Mykola Golub)
+* rbd-mirror: handle duplicates in image sync throttler queue (`issue#40519 <http://tracker.ceph.com/issues/40519>`_, `pr#28730 <https://github.com/ceph/ceph/pull/28730>`_, Mykola Golub)
+* rbd-mirror: hold lock while updating local image name (`pr#33988 <https://github.com/ceph/ceph/pull/33988>`_, Jason Dillaman)
+* rbd-mirror: ignore errors relating to parsing the cluster config file (`pr#29808 <https://github.com/ceph/ceph/pull/29808>`_, Jason Dillaman)
+* rbd-mirror: image status should report remote status (`pr#30558 <https://github.com/ceph/ceph/pull/30558>`_, Jason Dillaman)
+* rbd-mirror: improve detection of blacklisted state (`pr#33411 <https://github.com/ceph/ceph/pull/33411>`_, Mykola Golub)
+* rbd-mirror: initial end-to-end test and associated bug fixes (`pr#33588 <https://github.com/ceph/ceph/pull/33588>`_, Jason Dillaman)
+* rbd-mirror: initial snapshot replay state machine (`pr#33166 <https://github.com/ceph/ceph/pull/33166>`_, Jason Dillaman)
+* rbd-mirror: initial snapshot-based mirroring bootstrap logic (`pr#33002 <https://github.com/ceph/ceph/pull/33002>`_, Jason Dillaman)
+* rbd-mirror: link against the specified alloc library (`issue#40110 <http://tracker.ceph.com/issues/40110>`_, `pr#28434 <https://github.com/ceph/ceph/pull/28434>`_, Jason Dillaman)
+* rbd-mirror: make logrotate work (`pr#32456 <https://github.com/ceph/ceph/pull/32456>`_, Mykola Golub)
+* rbd-mirror: mirrored clone should be same format (`pr#31161 <https://github.com/ceph/ceph/pull/31161>`_, Mykola Golub)
+* rbd-mirror: peer_ping should send the local fsid to the remote (`pr#31950 <https://github.com/ceph/ceph/pull/31950>`_, Jason Dillaman)
+* rbd-mirror: periodically flush IO and commit positions (`issue#39257 <http://tracker.ceph.com/issues/39257>`_, `pr#27533 <https://github.com/ceph/ceph/pull/27533>`_, Jason Dillaman)
+* rbd-mirror: periodically poll remote mirror configuration (`pr#32671 <https://github.com/ceph/ceph/pull/32671>`_, Jason Dillaman)
+* rbd-mirror: potential nullptr dereference in ImageReplayer::handle_start_replay (`pr#30484 <https://github.com/ceph/ceph/pull/30484>`_, Mykola Golub)
+* rbd-mirror: prevent I/O modifications against a non-primary image (`pr#33831 <https://github.com/ceph/ceph/pull/33831>`_, Jason Dillaman)
+* rbd-mirror: provide initial snapshot replay status (`pr#33440 <https://github.com/ceph/ceph/pull/33440>`_, Jason Dillaman)
+* rbd-mirror: remove journal-specific logic from image replay and bootstrap state machines (`pr#32578 <https://github.com/ceph/ceph/pull/32578>`_, Jason Dillaman)
+* rbd-mirror: removing non-primary trash snapshot (`pr#31260 <https://github.com/ceph/ceph/pull/31260>`_, Mykola Golub)
+* rbd-mirror: rename per-image replication perf counters (`pr#32184 <https://github.com/ceph/ceph/pull/32184>`_, Mykola Golub)
+* rbd-mirror: simplify peer bootstrapping (`pr#30411 <https://github.com/ceph/ceph/pull/30411>`_, Jason Dillaman)
+* rbd-mirror: snapshot mirror mode (`pr#30548 <https://github.com/ceph/ceph/pull/30548>`_, Mykola Golub)
+* rbd-mirror: snapshot-based mirroring should use image sync throttler (`pr#34040 <https://github.com/ceph/ceph/pull/34040>`_, Jason Dillaman)
+* rbd-nbd: add netlink map/unmap support (`pr#27902 <https://github.com/ceph/ceph/pull/27902>`_, Mike Christie)
+* rbd-nbd: add nl resize (`pr#29036 <https://github.com/ceph/ceph/pull/29036>`_, Mike Christie)
+* rbd-nbd: sscanf return 0 mean not-match (`issue#39269 <http://tracker.ceph.com/issues/39269>`_, `pr#27484 <https://github.com/ceph/ceph/pull/27484>`_, Jianpeng Ma)
+* rbd:  creating thick-provision image progress percent info exceeds 100% (`pr#30954 <https://github.com/ceph/ceph/pull/30954>`_, Xiangdong Mu)
+* rbd:  journal: add support for aligned appends (`pr#28351 <https://github.com/ceph/ceph/pull/28351>`_, Mykola Golub)
+* rbd:  librbd: skip stale child with non-existent pool for list descendants (`pr#29654 <https://github.com/ceph/ceph/pull/29654>`_, songweibin)
+* rbd: add --merge to disk-usage (`pr#30994 <https://github.com/ceph/ceph/pull/30994>`_, Alexandre Bruyelles)
+* rbd: add mirror snapshot schedule commands (`pr#32882 <https://github.com/ceph/ceph/pull/32882>`_, Mykola Golub)
+* rbd: add snap_exists method API (`pr#32497 <https://github.com/ceph/ceph/pull/32497>`_, Zheng Yin)
+* rbd: client,common,mgr,rbd: clang related cleanups (`pr#33657 <https://github.com/ceph/ceph/pull/33657>`_, Kefu Chai)
+* rbd: cls/rbd: improve efficiency of mirror image status queries (`pr#31865 <https://github.com/ceph/ceph/pull/31865>`_, Jason Dillaman)
+* rbd: cls/rbd: sanitize entity instance messenger version type (`pr#30438 <https://github.com/ceph/ceph/pull/30438>`_, Jason Dillaman)
+* rbd: cls/rbd: sanitize the mirror image status peer address after reading from disk (`pr#31824 <https://github.com/ceph/ceph/pull/31824>`_, Jason Dillaman)
+* rbd: cls: reduce log level for non-fatal errors (`issue#40865 <http://tracker.ceph.com/issues/40865>`_, `pr#29165 <https://github.com/ceph/ceph/pull/29165>`_, Jason Dillaman)
+* rbd: delete redundant words when trash restore fails because of same name (`pr#30952 <https://github.com/ceph/ceph/pull/30952>`_, Xiangdong Mu)
+* rbd: fixed additional issues with CEPH_ARGS processing (`pr#33219 <https://github.com/ceph/ceph/pull/33219>`_, Jason Dillaman)
+* rbd: incorporate rbd-mirror daemon status in mirror pool status (`pr#31949 <https://github.com/ceph/ceph/pull/31949>`_, Jason Dillaman)
+* rbd: journal: fix race between player shut down and cache rebalance (`pr#28748 <https://github.com/ceph/ceph/pull/28748>`_, Mykola Golub)
+* rbd: journal: fix race between player shut down and cache rebalance (`pr#29796 <https://github.com/ceph/ceph/pull/29796>`_, Mykola Golub)
+* rbd: journal: optimize object overflow detection (`pr#28240 <https://github.com/ceph/ceph/pull/28240>`_, Mykola Golub)
+* rbd: journal: properly advance read offset after skipping invalid range (`pr#28627 <https://github.com/ceph/ceph/pull/28627>`_, Mykola Golub)
+* rbd: journal: return error after first corruption detected (`pr#28820 <https://github.com/ceph/ceph/pull/28820>`_, Mykola Golub)
+* rbd: journal: wait for in flight advance sets on stopping recorder (`pr#28529 <https://github.com/ceph/ceph/pull/28529>`_, Mykola Golub)
+* rbd: krbd: avoid udev netlink socket overrun (`pr#30965 <https://github.com/ceph/ceph/pull/30965>`_, Ilya Dryomov)
+* rbd: krbd: fix rbd map hang due to udev return subsystem unordered (`issue#39089 <http://tracker.ceph.com/issues/39089>`_, `pr#27339 <https://github.com/ceph/ceph/pull/27339>`_, Zhi Zhang)
+* rbd: krbd: modprobe before calling build_map_buf() (`pr#30978 <https://github.com/ceph/ceph/pull/30978>`_, Ilya Dryomov)
+* rbd: krbd: retry on transient errors from udev_enumerate_scan_devices() (`pr#31023 <https://github.com/ceph/ceph/pull/31023>`_, Ilya Dryomov)
+* rbd: krbd: return -ETIMEDOUT in polling (`issue#38792 <http://tracker.ceph.com/issues/38792>`_, `pr#27025 <https://github.com/ceph/ceph/pull/27025>`_, Dongsheng Yang)
+* rbd: mgr/dashboard: support RBD mirroring bootstrap create/import (`issue#42355 <http://tracker.ceph.com/issues/42355>`_, `pr#31062 <https://github.com/ceph/ceph/pull/31062>`_, Jason Dillaman)
+* rbd: msg/async: avoid unnecessary costly wakeups for outbound messages (`pr#28388 <https://github.com/ceph/ceph/pull/28388>`_, Jason Dillaman)
+* rbd: msg/async: reduce verbosity of connection timeout failures (`issue#39448 <http://tracker.ceph.com/issues/39448>`_, `pr#28050 <https://github.com/ceph/ceph/pull/28050>`_, Jason Dillaman)
+* rbd: pybind/mgr/rbd_support: fix missing variable in error path (`pr#29773 <https://github.com/ceph/ceph/pull/29773>`_, Jason Dillaman)
+* rbd: pybind/mgr/rbd_support: ignore missing support for RBD namespaces (`pr#29433 <https://github.com/ceph/ceph/pull/29433>`_, Jason Dillaman)
+* rbd: pybind/mgr/rbd_support: use image ids to detect duplicate tasks (`pr#29468 <https://github.com/ceph/ceph/pull/29468>`_, Jason Dillaman)
+* rbd: pybind/mgr/rbd_support: wait for latest OSD map prior to handling commands (`pr#33451 <https://github.com/ceph/ceph/pull/33451>`_, Jason Dillaman)
+* rbd: pybind/rbd: fix call to unregister_osd_perf_queries (`pr#29419 <https://github.com/ceph/ceph/pull/29419>`_, Venky Shankar)
+* rbd: pybind/rbd: provide snap remove flags (`pr#31627 <https://github.com/ceph/ceph/pull/31627>`_, Mykola Golub)
+* rbd: qa/suites/rbd/openstack: use 18.04, not 16.04 (`pr#32284 <https://github.com/ceph/ceph/pull/32284>`_, Sage Weil)
+* rbd: rbd-ggate: fix compile errors from ceph::mutex update (`pr#29474 <https://github.com/ceph/ceph/pull/29474>`_, Willem Jan Withagen)
+* rbd: rbd-mirror: adjust journal fetch properties based on memory target (`pr#27670 <https://github.com/ceph/ceph/pull/27670>`_, Mykola Golub)
+* rbd: rbd/action: display image id in rbd du/list output (`pr#29376 <https://github.com/ceph/ceph/pull/29376>`_, songweibin)
+* rbd: rbd/action: fix error getting positional argument (`issue#40095 <http://tracker.ceph.com/issues/40095>`_, `pr#28313 <https://github.com/ceph/ceph/pull/28313>`_, songweibin)
+* rbd: rbd/bench: outputs bytes/s format dynamically (`pr#31491 <https://github.com/ceph/ceph/pull/31491>`_, Zheng Yin)
+* rbd: rbd/cache: Replicated Write Log core codes part 1 (`pr#31279 <https://github.com/ceph/ceph/pull/31279>`_, Peterson, Scott, Li, Xiaoyan, Lu, Yuan, Chamarthy, Mahati)
+* rbd: rbd/cache: Replicated Write Log core codes part 2 (`pr#31963 <https://github.com/ceph/ceph/pull/31963>`_, Peterson, Scott, Li, Xiaoyan, Lu, Yuan, Chamarthy, Mahati)
+* rbd: rbd_replay: call the member decode() explicitly (`pr#27703 <https://github.com/ceph/ceph/pull/27703>`_, Kefu Chai)
+* rbd: schedule for running trash purge operations (`pr#33389 <https://github.com/ceph/ceph/pull/33389>`_, Mykola Golub)
+* rbd: src: use un-deprecated version of aio_create_completion (`pr#31333 <https://github.com/ceph/ceph/pull/31333>`_, Adam C. Emerson)
+* rbd: use the ordered throttle for the export action (`issue#40435 <http://tracker.ceph.com/issues/40435>`_, `pr#28657 <https://github.com/ceph/ceph/pull/28657>`_, Jason Dillaman)
+* remove cephadm-adoption-corpus as submodule (`pr#33587 <https://github.com/ceph/ceph/pull/33587>`_, Sage Weil)
+* Return an error, for Bluestore OSD, if WAL or DB are defined in the tags of the OSD but not present on the system (`pr#28791 <https://github.com/ceph/ceph/pull/28791>`_, David Casier)
+* rgw,tests: qa/rgw/pubsub: fix tests to sync from master (`pr#33049 <https://github.com/ceph/ceph/pull/33049>`_, Yuval Lifshitz)
+* rgw,tests: qa/rgw/pubsub: verify incremental sync is used in pubsu (`pr#33068 <https://github.com/ceph/ceph/pull/33068>`_, Yuval Lifshitz)
+* rgw,tests: qa/rgw: add integration test for sse-kms with barbican (`pr#30218 <https://github.com/ceph/ceph/pull/30218>`_, Casey Bodley, Adam Kupczyk)
+* rgw,tests: qa/rgw: add new rgw/website suite for static website tests (`pr#30193 <https://github.com/ceph/ceph/pull/30193>`_, Casey Bodley)
+* rgw,tests: qa/rgw: add rgw_obj and throttle tests to rgw verify suite (`pr#32188 <https://github.com/ceph/ceph/pull/32188>`_, Casey Bodley)
+* rgw,tests: qa/rgw: disable debuginfo packages (`pr#27528 <https://github.com/ceph/ceph/pull/27528>`_, Casey Bodley)
+* rgw,tests: qa/rgw: dont use ceph-ansible in s3a-hadoop suite (`issue#39706 <http://tracker.ceph.com/issues/39706>`_, `pr#28068 <https://github.com/ceph/ceph/pull/28068>`_, Casey Bodley)
+* rgw,tests: qa/rgw: drop some objectstore types (`pr#30997 <https://github.com/ceph/ceph/pull/30997>`_, Casey Bodley)
+* rgw,tests: qa/rgw: exercise DeleteRange in test_bucket_index_log_trim (`pr#33047 <https://github.com/ceph/ceph/pull/33047>`_, Casey Bodley)
+* rgw,tests: qa/rgw: extra s3tests tasks use rgw endpoint configuration (`issue#17882 <http://tracker.ceph.com/issues/17882>`_, `pr#28631 <https://github.com/ceph/ceph/pull/28631>`_, Casey Bodley)
+* rgw,tests: qa/rgw: fix import error in tasks/swift.py (`issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28605 <https://github.com/ceph/ceph/pull/28605>`_, Casey Bodley)
+* rgw,tests: qa/rgw: fix swift warning message (`pr#28697 <https://github.com/ceph/ceph/pull/28697>`_, Casey Bodley)
+* rgw,tests: qa/rgw: more fixes for swift task (`issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28823 <https://github.com/ceph/ceph/pull/28823>`_, Casey Bodley)
+* rgw,tests: qa/rgw: multisite checkpoints consider pubsub zone (`pr#32941 <https://github.com/ceph/ceph/pull/32941>`_, Casey Bodley)
+* rgw,tests: qa/rgw: refactor the kms backend configuration (`pr#30940 <https://github.com/ceph/ceph/pull/30940>`_, Casey Bodley)
+* rgw,tests: qa/rgw: remove failing radosgw_admin_rest from multisite suite (`pr#32550 <https://github.com/ceph/ceph/pull/32550>`_, Casey Bodley)
+* rgw,tests: qa/rgw: remove whitelist for SLOW_OPS against ec pools (`pr#31363 <https://github.com/ceph/ceph/pull/31363>`_, Casey Bodley)
+* rgw,tests: qa/rgw: s3a-hadoop task defaults to maven-version 3.6.3 (`pr#32620 <https://github.com/ceph/ceph/pull/32620>`_, Casey Bodley)
+* rgw,tests: qa/rgw: skip swift tests on rhel 7.6+ (`issue#40304 <http://tracker.ceph.com/issues/40304>`_, `pr#28532 <https://github.com/ceph/ceph/pull/28532>`_, Casey Bodley)
+* rgw,tests: qa/rgw: update run-s3tests.sh (`pr#28964 <https://github.com/ceph/ceph/pull/28964>`_, Casey Bodley)
+* rgw,tests: qa/rgw: use testing kms backend for multisite tests (`pr#31374 <https://github.com/ceph/ceph/pull/31374>`_, Casey Bodley)
+* rgw,tests: qa/rgw: use testing kms backend for other rgw subsuites (`pr#31414 <https://github.com/ceph/ceph/pull/31414>`_, Casey Bodley)
+* rgw,tests: qa/rgw: whitelist SLOW_OPS failures against ec pools (`pr#30944 <https://github.com/ceph/ceph/pull/30944>`_, Casey Bodley)
+* rgw,tests: qa/suites/rgw/website: run test on ubuntu (`pr#32791 <https://github.com/ceph/ceph/pull/32791>`_, Sage Weil)
+* rgw,tests: qa/suites/rgw: reenable ragweed (now py3) (`pr#32310 <https://github.com/ceph/ceph/pull/32310>`_, Sage Weil)
+* rgw,tests: qa/suites: use s3-tests with python3 support (`pr#32624 <https://github.com/ceph/ceph/pull/32624>`_, Ali Maredia)
+* rgw,tests: qa/tasks/swift: remove swift tests (`pr#32357 <https://github.com/ceph/ceph/pull/32357>`_, Sage Weil)
+* rgw,tests: qa/tests: added rgw into upgrade sequence to improve coverage (`pr#29234 <https://github.com/ceph/ceph/pull/29234>`_, Yuri Weinstein)
+* rgw,tests: qa/tests: added rgw into upgrade sequence to improve coverage - splits (`pr#29282 <https://github.com/ceph/ceph/pull/29282>`_, Yuri Weinstein)
+* rgw,tests: qa: add force-branch to suites running s3readwrite & s3roundtrip tasks (`pr#32225 <https://github.com/ceph/ceph/pull/32225>`_, Ali Maredia)
+* rgw,tests: qa: bump maven repo version in s3a_hadoop.py (`pr#30531 <https://github.com/ceph/ceph/pull/30531>`_, Ali Maredia)
+* rgw,tests: qa: radosgw-admin: remove dependency on bunch package (`pr#32100 <https://github.com/ceph/ceph/pull/32100>`_, Yehuda Sadeh)
+* rgw,tests: qa: radosgw_admin: validate a simple user stats output (`pr#30684 <https://github.com/ceph/ceph/pull/30684>`_, Abhishek Lekshmanan)
+* rgw,tests: qa: remove mon valgrind check in rgw verfiy suite (`issue#38827 <http://tracker.ceph.com/issues/38827>`_, `pr#28155 <https://github.com/ceph/ceph/pull/28155>`_, Ali Maredia)
+* rgw,tests: qa: remove s3-tests from rados/basic/tasks/rgw_snaps.yml (`pr#32940 <https://github.com/ceph/ceph/pull/32940>`_, Ali Maredia)
+* rgw,tests: qa: rgw: add user-policy caps for the s3tests users (`pr#31127 <https://github.com/ceph/ceph/pull/31127>`_, Abhishek Lekshmanan)
+* rgw,tests: qa: use curl in wait_for_radosgw() in util/rgw.py (`pr#28521 <https://github.com/ceph/ceph/pull/28521>`_, Ali Maredia)
+* rgw,tests: rgw/amqp: fix race condition in AMQP unit test (`pr#30735 <https://github.com/ceph/ceph/pull/30735>`_, Yuval Lifshitz)
+* rgw,tests: rgw/amqp: remove flaky amqp test (`pr#31510 <https://github.com/ceph/ceph/pull/31510>`_, Yuval Lifshitz)
+* rgw,tests: rgw/pubsub: add multisite pubsub tests to teuthology (`pr#27838 <https://github.com/ceph/ceph/pull/27838>`_, Yuval Lifshitz)
+* rgw,tests: rgw/pubsub: tests enhancements and fixes (`pr#28910 <https://github.com/ceph/ceph/pull/28910>`_, Yuval Lifshitz)
+* rgw,tests: rgw/pubsub: use incremental sync for pubsub module by default (`pr#28470 <https://github.com/ceph/ceph/pull/28470>`_, Yuval Lifshitz)
+* rgw,tests: test/rgw: fix test-rgw-multisite.sh script for creating multisite clusters (`pr#27984 <https://github.com/ceph/ceph/pull/27984>`_, Casey Bodley)
+* rgw,tests: test/rgw: fixes for test-rgw-multisite.sh (`pr#33537 <https://github.com/ceph/ceph/pull/33537>`_, Casey Bodley)
+* rgw,tests: test/rgw: raise timer durations for unittest_rgw_reshard_wait (`pr#32094 <https://github.com/ceph/ceph/pull/32094>`_, Casey Bodley)
+* rgw,tests: test/rgw: test_rgw_reshard_wait uses same clock for timing (`pr#27035 <https://github.com/ceph/ceph/pull/27035>`_, Casey Bodley)
+* rgw,tests: vstart: move common rgw config to [client.rgw] (`pr#29449 <https://github.com/ceph/ceph/pull/29449>`_, Casey Bodley)
+* rgw,tools: ceph-dencoder: add RGWPeriodLatestEpochInfo support (`pr#30613 <https://github.com/ceph/ceph/pull/30613>`_, yuliyang)
+* rgw,tools: rgw/examples: adding examples for boto3 extensions to AWS S3 (`pr#30600 <https://github.com/ceph/ceph/pull/30600>`_, Yuval Lifshitz)
+* rgw,tools: vstart.sh: run multiple rgws with different ids (`pr#26690 <https://github.com/ceph/ceph/pull/26690>`_, Joao Eduardo Luis)
+* rgw:  rgw: cls_bucket_list_unordered lists a single shard (`issue#39393 <http://tracker.ceph.com/issues/39393>`_, `pr#27697 <https://github.com/ceph/ceph/pull/27697>`_, Casey Bodley)
+* rgw:  rgw: make radosgw-admin user create and modify distinct (`pr#31901 <https://github.com/ceph/ceph/pull/31901>`_, Matthew Oliver)
+* rgw:  rgw: returns LimitExceeded when user creates too many ACLs (`issue#26835 <http://tracker.ceph.com/issues/26835>`_, `pr#25692 <https://github.com/ceph/ceph/pull/25692>`_, Chang Liu)
+* rgw: A task to run S3 Java tests against RGW (`pr#22788 <https://github.com/ceph/ceph/pull/22788>`_, Antoaneta Damyanova)
+* rgw: add --object-version in radosgw-admin help info (`pr#30091 <https://github.com/ceph/ceph/pull/30091>`_, yuliyang)
+* rgw: add a small efficiency (`pr#29178 <https://github.com/ceph/ceph/pull/29178>`_, J. Eric Ivancich)
+* rgw: add admin rest api for bucket sync (`pr#19020 <https://github.com/ceph/ceph/pull/19020>`_, zhang Shaowen, Zhang Shaowen)
+* rgw: add cls_queue and cls_rgw_gc for omap offload (`pr#28421 <https://github.com/ceph/ceph/pull/28421>`_, Pritha Srivastava, Casey Bodley)
+* rgw: add const correctness to some rest functions (`pr#31660 <https://github.com/ceph/ceph/pull/31660>`_, J. Eric Ivancich)
+* rgw: add creation time information into bucket stats (`pr#30384 <https://github.com/ceph/ceph/pull/30384>`_, Enming Zhang)
+* rgw: Add days0 to rgw lc (`pr#29937 <https://github.com/ceph/ceph/pull/29937>`_, Or Friedmann)
+* rgw: add detailed error message for PutACLs (`pr#30385 <https://github.com/ceph/ceph/pull/30385>`_, Enming Zhang)
+* rgw: add editor directive comments to rgw services source files (`pr#27897 <https://github.com/ceph/ceph/pull/27897>`_, J. Eric Ivancich)
+* rgw: add GET /admin/realm?list api to list realms (`pr#28156 <https://github.com/ceph/ceph/pull/28156>`_, Casey Bodley)
+* rgw: add missing admin property when sync user info (`pr#30127 <https://github.com/ceph/ceph/pull/30127>`_, zhang Shaowen)
+* rgw: add missing bilog status to help info (`pr#30357 <https://github.com/ceph/ceph/pull/30357>`_, zhang Shaowen)
+* rgw: add missing close_section in send_versioned_response (`pr#28946 <https://github.com/ceph/ceph/pull/28946>`_, Casey Bodley)
+* rgw: Add more details to the LC delete and transit log (`pr#30913 <https://github.com/ceph/ceph/pull/30913>`_, Or Friedmann)
+* rgw: add num_shards to radosgw-admin bucket stats (`pr#30845 <https://github.com/ceph/ceph/pull/30845>`_, Paul Emmerich)
+* rgw: add option to specify shard-id for bi list admin command (`pr#29394 <https://github.com/ceph/ceph/pull/29394>`_, Mark Kogan)
+* rgw: add optional_yield to http client interface (`pr#25355 <https://github.com/ceph/ceph/pull/25355>`_, Casey Bodley)
+* rgw: add optional_yield to SysObj service interfaces (`pr#25353 <https://github.com/ceph/ceph/pull/25353>`_, Casey Bodley)
+* rgw: add PublicAccessBlock set of APIs on buckets (`pr#30033 <https://github.com/ceph/ceph/pull/30033>`_, Abhishek Lekshmanan)
+* rgw: add rgw_rados_pool_recovery_priority (default 5) (`pr#29181 <https://github.com/ceph/ceph/pull/29181>`_, Sage Weil)
+* rgw: add roles_pool in RGWZoneParams dump/decode json (`issue#22162 <http://tracker.ceph.com/issues/22162>`_, `pr#17338 <https://github.com/ceph/ceph/pull/17338>`_, Tianshan Qu)
+* rgw: add S3 object lock feature to support object worm (`pr#26538 <https://github.com/ceph/ceph/pull/26538>`_, zhang Shaowen)
+* rgw: add some comments to rgw code to help explain functionality (`pr#27896 <https://github.com/ceph/ceph/pull/27896>`_, J. Eric Ivancich)
+* rgw: add SSE-KMS with Vault using token auth (`pr#29783 <https://github.com/ceph/ceph/pull/29783>`_, Andrea Baglioni, Sergio de Carvalho)
+* rgw: Add support bucket policy for subuser (`pr#33165 <https://github.com/ceph/ceph/pull/33165>`_, Seena Fallah)
+* rgw: add tenant as parameter to User in multisite tests (`pr#27969 <https://github.com/ceph/ceph/pull/27969>`_, Yuval Lifshitz)
+* rgw: add transaction id to ops log (`pr#30163 <https://github.com/ceph/ceph/pull/30163>`_, zhang Shaowen)
+* rgw: add YieldingAioThrottle for async PutObj/GetObj (`pr#26173 <https://github.com/ceph/ceph/pull/26173>`_, Casey Bodley)
+* rgw: Added caching for S3 credentials retrieved from keystone (`pr#26095 <https://github.com/ceph/ceph/pull/26095>`_, James Weaver)
+* rgw: adding documentation for AssumeRoleWithWebIdentity (`pr#31994 <https://github.com/ceph/ceph/pull/31994>`_, Pritha Srivastava)
+* rgw: Adding iam namespace for Role and User Policy related REST APIs (`pr#27178 <https://github.com/ceph/ceph/pull/27178>`_, Pritha Srivastava)
+* rgw: adding mfa code validation when bucket versioning status is changed (`pr#31767 <https://github.com/ceph/ceph/pull/31767>`_, Pritha Srivastava)
+* rgw: Adding tcp_nodelay option to Beast (`pr#27008 <https://github.com/ceph/ceph/pull/27008>`_, Or Friedmann)
+* rgw: address 0-length listing results when non-vis entries dominate (`pr#32636 <https://github.com/ceph/ceph/pull/32636>`_, J. Eric Ivancich)
+* rgw: adjust allowable bucket index shard counts for dynamic resharding (`pr#30795 <https://github.com/ceph/ceph/pull/30795>`_, J. Eric Ivancich)
+* rgw: admin: handle delete_at attr in object stat output (`pr#27781 <https://github.com/ceph/ceph/pull/27781>`_, Abhishek Lekshmanan)
+* rgw: Allow admin APIs that write metadata to be executed first on the mast\xe2\x80\xa6 (`issue#39549 <http://tracker.ceph.com/issues/39549>`_, `pr#29549 <https://github.com/ceph/ceph/pull/29549>`_, Shilpa Jagannath)
+* rgw: allow radosgw-admin to list bucket w --allow-unordered (`issue#39637 <http://tracker.ceph.com/issues/39637>`_, `pr#28031 <https://github.com/ceph/ceph/pull/28031>`_, J. Eric Ivancich)
+* rgw: allow reshard log entries for non-existent buckets to be cancelled (`pr#31271 <https://github.com/ceph/ceph/pull/31271>`_, J. Eric Ivancich)
+* rgw: apply_olh_log ignores RGW_ATTR_OLH_VER decode error (`pr#31976 <https://github.com/ceph/ceph/pull/31976>`_, Casey Bodley)
+* rgw: asio: check the remote endpoint before processing requests (`pr#29967 <https://github.com/ceph/ceph/pull/29967>`_, Abhishek Lekshmanan)
+* rgw: auth/Crypto: fallback to /dev/urandom if getentropy() fails (`pr#30544 <https://github.com/ceph/ceph/pull/30544>`_, Kefu Chai)
+* rgw: auto-clean reshard queue entries for non-existent buckets (`pr#31323 <https://github.com/ceph/ceph/pull/31323>`_, J. Eric Ivancich)
+* rgw: az: add archive zone tests (`pr#29359 <https://github.com/ceph/ceph/pull/29359>`_, Javier M. Mellid)
+* rgw: beast frontend uses 512k mprotected coroutine stacks (`pr#31580 <https://github.com/ceph/ceph/pull/31580>`_, Daniel Gryniewicz, Casey Bodley)
+* rgw: beast frontend uses yield_context to read/write body (`pr#27795 <https://github.com/ceph/ceph/pull/27795>`_, Casey Bodley)
+* rgw: beast port parsing (`issue#39000 <http://tracker.ceph.com/issues/39000>`_, `pr#27242 <https://github.com/ceph/ceph/pull/27242>`_, Abhishek Lekshmanan)
+* rgw: beast ssl certs config through config-key (`pr#33287 <https://github.com/ceph/ceph/pull/33287>`_, Yehuda Sadeh)
+* rgw: bucket granularity sync (`pr#31686 <https://github.com/ceph/ceph/pull/31686>`_, Yehuda Sadeh)
+* rgw: bucket re-creation fixes (`pr#32121 <https://github.com/ceph/ceph/pull/32121>`_, Yehuda Sadeh)
+* rgw: bucket stats report mtime in UTC (`pr#27617 <https://github.com/ceph/ceph/pull/27617>`_, Casey Bodley)
+* rgw: bucket tagging (`pr#27993 <https://github.com/ceph/ceph/pull/27993>`_, Chang Liu)
+* rgw: build async scheduler only when beast is built (`pr#26634 <https://github.com/ceph/ceph/pull/26634>`_, Abhishek Lekshmanan)
+* rgw: build radosgw daemon as a shared lib + small executable (`pr#32404 <https://github.com/ceph/ceph/pull/32404>`_, Kaleb S. Keithley)
+* rgw: build_linked_oids_for_bucket and build_buckets_instance_index should return negative value if it fails (`pr#31346 <https://github.com/ceph/ceph/pull/31346>`_, zhangshaowen)
+* rgw: change cls rgw reshard status to enum class (`pr#30611 <https://github.com/ceph/ceph/pull/30611>`_, J. Eric Ivancich)
+* rgw: change MAX_USAGE_TRIM_ENTRIES value from 128 to 1000 (`pr#30392 <https://github.com/ceph/ceph/pull/30392>`_, zhang Shaowen)
+* rgw: check lc objs not empty after fetching (`pr#26167 <https://github.com/ceph/ceph/pull/26167>`_, Yao Zongyou)
+* rgw: clean index and remove bucket instance info when setting resharding status fails (`pr#31103 <https://github.com/ceph/ceph/pull/31103>`_, zhangshaowen)
+* rgw: clean up ordered list (`pr#31338 <https://github.com/ceph/ceph/pull/31338>`_, J. Eric Ivancich)
+* rgw: clean up some logging (`pr#27411 <https://github.com/ceph/ceph/pull/27411>`_, J. Eric Ivancich)
+* rgw: cleanup the magic string usage in cls_rgw_client.cc (`pr#31432 <https://github.com/ceph/ceph/pull/31432>`_, zhangshaowen)
+* rgw: cleanup:remove un-used class member in RGWDeleteLC (`pr#31404 <https://github.com/ceph/ceph/pull/31404>`_, zhang Shaowen)
+* rgw: cleanup:remove un-used create_new_bucket_instance in rgw_admin.cc (`pr#31345 <https://github.com/ceph/ceph/pull/31345>`_, zhangshaowen)
+* rgw: clear ent_list for each loop of bucket list (`issue#44394 <http://tracker.ceph.com/issues/44394>`_, `pr#33693 <https://github.com/ceph/ceph/pull/33693>`_, Yao Zongyou)
+* rgw: cls/rgw: fix bilog trim tests in ceph_test_cls_rgw (`pr#30268 <https://github.com/ceph/ceph/pull/30268>`_, Casey Bodley)
+* rgw: cls/rgw: keep issuing bilog trim ops after reset (`issue#40187 <http://tracker.ceph.com/issues/40187>`_, `pr#28430 <https://github.com/ceph/ceph/pull/28430>`_, Casey Bodley)
+* rgw: cls/rgw: test before accessing pkeys->rbegin() (`issue#39984 <http://tracker.ceph.com/issues/39984>`_, `pr#28391 <https://github.com/ceph/ceph/pull/28391>`_, Casey Bodley)
+* rgw: cls/rgw: when object is versioned and lc transition it, the object is becoming non-current (`pr#32458 <https://github.com/ceph/ceph/pull/32458>`_, Or Friedmann)
+* rgw: cls/user: cls_user_set_buckets_info overwrites creation_time (`issue#39635 <http://tracker.ceph.com/issues/39635>`_, `pr#28045 <https://github.com/ceph/ceph/pull/28045>`_, Casey Bodley)
+* rgw: cls_bucket_list\\_(un)ordered should clear results collection (`pr#33702 <https://github.com/ceph/ceph/pull/33702>`_, J. Eric Ivancich)
+* rgw: compression info should be same during multipart uploading (`pr#30574 <https://github.com/ceph/ceph/pull/30574>`_, zhang Shaowen)
+* rgw: conditionally allow non-unique email addresses (`issue#40089 <http://tracker.ceph.com/issues/40089>`_, `pr#28327 <https://github.com/ceph/ceph/pull/28327>`_, Matt Benjamin)
+* rgw: continuation token doesnt work in list object v2 request (`pr#28988 <https://github.com/ceph/ceph/pull/28988>`_, zhang Shaowen)
+* rgw: continuationToken or startAfter shouldnt be returned if not specified (`pr#29298 <https://github.com/ceph/ceph/pull/29298>`_, zhang Shaowen)
+* rgw: correct some error log about reshard in cls_rgw.cc (`pr#31429 <https://github.com/ceph/ceph/pull/31429>`_, zhangshaowen)
+* rgw: crypt: permit RGW-AUTO/default with SSE-S3 headers (`pr#30189 <https://github.com/ceph/ceph/pull/30189>`_, Matt Benjamin)
+* rgw: crypto: throw DigestException from Digest and HMAC (`issue#39456 <http://tracker.ceph.com/issues/39456>`_, `pr#27765 <https://github.com/ceph/ceph/pull/27765>`_, Matt Benjamin)
+* rgw: data sync markers include timestamp from datalog entry (`pr#32309 <https://github.com/ceph/ceph/pull/32309>`_, Casey Bodley)
+* rgw: data/bilogs are trimmed when no peers are reading them (`issue#39487 <http://tracker.ceph.com/issues/39487>`_, `pr#27794 <https://github.com/ceph/ceph/pull/27794>`_, Casey Bodley)
+* rgw: datalog/mdlog trim commands loop until done (`pr#29448 <https://github.com/ceph/ceph/pull/29448>`_, Casey Bodley)
+* rgw: data_sync_source_zones only contains exporting zones (`pr#33193 <https://github.com/ceph/ceph/pull/33193>`_, Casey Bodley)
+* rgw: decrypt filter does not cross multipart boundaries (`issue#38700 <http://tracker.ceph.com/issues/38700>`_, `pr#27130 <https://github.com/ceph/ceph/pull/27130>`_, Adam Kupczyk, Casey Bodley, Abhishek Lekshmanan)
+* rgw: DefaultRetention requires either Days or Years (`pr#29680 <https://github.com/ceph/ceph/pull/29680>`_, Chang Liu)
+* rgw: delete_obj_index() takes mtime for bilog (`issue#24991 <http://tracker.ceph.com/issues/24991>`_, `pr#27980 <https://github.com/ceph/ceph/pull/27980>`_, Casey Bodley)
+* rgw: distinguish different get_usage for usage log (`pr#17719 <https://github.com/ceph/ceph/pull/17719>`_, Jiaying Ren)
+* rgw: dmclock: wait until the request is handled (`pr#30777 <https://github.com/ceph/ceph/pull/30777>`_, GaryHyg)
+* rgw: do not miss the 1000th element of every iteration during lifecycle processing (`pr#30861 <https://github.com/ceph/ceph/pull/30861>`_, Ilsoo Byun)
+* rgw: do not remove delete marker when fixing versioned bucket (`pr#32562 <https://github.com/ceph/ceph/pull/32562>`_, Ilsoo Byun)
+* rgw: Dont crash on copy when metadata directive not supplied (`issue#40416 <http://tracker.ceph.com/issues/40416>`_, `pr#28949 <https://github.com/ceph/ceph/pull/28949>`_, Adam C. Emerson)
+* rgw: dont crash on missing /etc/mime.types (`issue#38328 <http://tracker.ceph.com/issues/38328>`_, `pr#26998 <https://github.com/ceph/ceph/pull/26998>`_, Casey Bodley)
+* rgw: dont print error log when list reshard result is not truncated (`pr#31142 <https://github.com/ceph/ceph/pull/31142>`_, zhangshaowen)
+* rgw: dont recalculate etags for slo/dlo (`pr#27470 <https://github.com/ceph/ceph/pull/27470>`_, Casey Bodley)
+* rgw: dont throw when accept errors are happening on frontend (`pr#29587 <https://github.com/ceph/ceph/pull/29587>`_, Yuval Lifshitz)
+* rgw: drop cloud sync module logs attrs from the log (`pr#27820 <https://github.com/ceph/ceph/pull/27820>`_, Nathan Cutler)
+* rgw: drop dead flush_read_list declaration (`pr#29458 <https://github.com/ceph/ceph/pull/29458>`_, Jiaying Ren)
+* rgw: drop unused rgw_decode_pki_token() (`pr#27052 <https://github.com/ceph/ceph/pull/27052>`_, Radoslaw Zarzynski)
+* rgw: dump s3_code as the Code response element in RGWDeleteMultiObj_ObjStore_S3 (`issue#18241 <http://tracker.ceph.com/issues/18241>`_, `pr#12470 <https://github.com/ceph/ceph/pull/12470>`_, Radoslaw Zarzynski)
+* rgw: eliminates duplicated tags_bl var (`pr#27970 <https://github.com/ceph/ceph/pull/27970>`_, Chang Liu)
+* rgw: Evaluating bucket policies also while reading permissions for an\xe2\x80\xa6 (`issue#38638 <http://tracker.ceph.com/issues/38638>`_, `pr#27309 <https://github.com/ceph/ceph/pull/27309>`_, Pritha Srivastava)
+* rgw: examples: rgw: add boto3 append & get usage api extensions (`pr#33063 <https://github.com/ceph/ceph/pull/33063>`_, Abhishek Lekshmanan)
+* rgw: Expiration days cant be zero and  transition days can be zero (`pr#30878 <https://github.com/ceph/ceph/pull/30878>`_, zhang Shaowen)
+* rgw: extend SSE-KMS with Vault using transit secrets engine (`pr#31361 <https://github.com/ceph/ceph/pull/31361>`_, Andrea Baglioni, Sergio de Carvalho)
+* rgw: fetch_remote_obj() compares expected object size (`pr#28303 <https://github.com/ceph/ceph/pull/28303>`_, Xiaoxi CHEN, Casey Bodley)
+* rgw: find oldest period and update RGWMetadataLogHistory() (`pr#31873 <https://github.com/ceph/ceph/pull/31873>`_, Shilpa Jagannath)
+* rgw: fix a bug that bucket instance obj cant be removed after resharding completed (`pr#31483 <https://github.com/ceph/ceph/pull/31483>`_, zhang Shaowen)
+* rgw: fix a bug that lifecycle expiraton generates delete marker continuously (`issue#40393 <http://tracker.ceph.com/issues/40393>`_, `pr#28587 <https://github.com/ceph/ceph/pull/28587>`_, zhang Shaowen)
+* rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR (`issue#39984 <http://tracker.ceph.com/issues/39984>`_, `pr#28188 <https://github.com/ceph/ceph/pull/28188>`_, Tianshan Qu)
+* rgw: Fix bucket versioning vs. swift metadata bug (`pr#29240 <https://github.com/ceph/ceph/pull/29240>`_, Marcus Watts)
+* rgw: Fix bug on subuser policy identity checker (`pr#33398 <https://github.com/ceph/ceph/pull/33398>`_, Seena Fallah)
+* rgw: fix bug with (un)ordered bucket listing and marker w/ namespace (`pr#33046 <https://github.com/ceph/ceph/pull/33046>`_, J. Eric Ivancich)
+* rgw: fix bugs in listobjectsv1 (`pr#28873 <https://github.com/ceph/ceph/pull/28873>`_, Albin Antony)
+* rgw: fix cls_bucket_list_unordered() partial results (`pr#29692 <https://github.com/ceph/ceph/pull/29692>`_, Mark Kogan)
+* rgw: fix compile errors with boost 1.70 (`pr#27730 <https://github.com/ceph/ceph/pull/27730>`_, Casey Bodley)
+* rgw: fix data consistency error casued by rgw sent timeout (`pr#30257 <https://github.com/ceph/ceph/pull/30257>`_, \xe6\x9d\x8e\xe7\xba\xb2\xe5\xbd\xac82225)
+* rgw: fix data sync start delay if remote havent init data_log (`pr#30393 <https://github.com/ceph/ceph/pull/30393>`_, Tianshan Qu)
+* rgw: fix default storage class for get_compression_type (`pr#29909 <https://github.com/ceph/ceph/pull/29909>`_, Casey Bodley)
+* rgw: fix default_placement containing / when storage_class is standard (`issue#39380 <http://tracker.ceph.com/issues/39380>`_, `pr#27676 <https://github.com/ceph/ceph/pull/27676>`_, mkogan1)
+* rgw: fix dns name comparison for virtual hosting (`pr#30221 <https://github.com/ceph/ceph/pull/30221>`_, Casey Bodley)
+* rgw: Fix documentation for rgw_ldap_secret (`pr#29816 <https://github.com/ceph/ceph/pull/29816>`_, Robin M\xc3\xbcller)
+* rgw: fix drain handles error when deleting bucket with bypass-gc option (`pr#28789 <https://github.com/ceph/ceph/pull/28789>`_, dongdong tao)
+* rgw: Fix dynamic resharding not working for empty zonegroup in period (`pr#31977 <https://github.com/ceph/ceph/pull/31977>`_, Or Friedmann)
+* rgw: Fix expiration header does not return the earliest rule (`pr#29399 <https://github.com/ceph/ceph/pull/29399>`_, Or Friedmann)
+* rgw: fix incorrect radosgw-admin zonegroup rm info (`pr#30319 <https://github.com/ceph/ceph/pull/30319>`_, zhang Shaowen)
+* rgw: fix indentation for listobjectsv2 (`pr#28830 <https://github.com/ceph/ceph/pull/28830>`_, Albin Antony)
+* rgw: fix list bucket with delimiter wrongly skip some special keys (`issue#40905 <http://tracker.ceph.com/issues/40905>`_, `pr#29215 <https://github.com/ceph/ceph/pull/29215>`_, Tianshan Qu)
+* rgw: fix list bucket with start maker and delimiter / will miss next object\xe2\x80\xa6 (`issue#39989 <http://tracker.ceph.com/issues/39989>`_, `pr#28192 <https://github.com/ceph/ceph/pull/28192>`_, Tianshan Qu)
+* rgw: fix list versions starts with version_id=null (`pr#29897 <https://github.com/ceph/ceph/pull/29897>`_, Tianshan Qu)
+* rgw: fix MalformedXML errors in PutBucketObjectLock/PutObjRetention (`pr#28783 <https://github.com/ceph/ceph/pull/28783>`_, Casey Bodley)
+* rgw: fix memory growth while deleting objects with (`pr#30174 <https://github.com/ceph/ceph/pull/30174>`_, Mark Kogan)
+* rgw: fix minimum of unordered bucket listing (`pr#30146 <https://github.com/ceph/ceph/pull/30146>`_, J. Eric Ivancich)
+* rgw: fix minor compiler warning in keystone auth (`pr#27100 <https://github.com/ceph/ceph/pull/27100>`_, David Disseldorp)
+* rgw: fix miss get ret in STSService::storeARN (`issue#40386 <http://tracker.ceph.com/issues/40386>`_, `pr#28527 <https://github.com/ceph/ceph/pull/28527>`_, Tianshan Qu)
+* rgw: fix miss handle curl error return (`pr#28345 <https://github.com/ceph/ceph/pull/28345>`_, Casey Bodley, Tianshan Qu)
+* rgw: fix missing tenant prefix in bucket name during bucket link (`pr#29815 <https://github.com/ceph/ceph/pull/29815>`_, Shilpa Jagannath)
+* rgw: fix multipart uploads error response (`pr#32771 <https://github.com/ceph/ceph/pull/32771>`_, GaryHyg)
+* rgw: Fix narrowing conversion error (`pr#28905 <https://github.com/ceph/ceph/pull/28905>`_, Adam C. Emerson)
+* rgw: fix one part of the bulk delete(RGWDeleteMultiObj_ObjStore_S3) fails but no error messages (`pr#29795 <https://github.com/ceph/ceph/pull/29795>`_, Snow Si)
+* rgw: fix opslog operation field as per Amazon s3 (`issue#20978 <http://tracker.ceph.com/issues/20978>`_, `pr#30539 <https://github.com/ceph/ceph/pull/30539>`_, Jiaying Ren)
+* rgw: fix potential realm watch lost (`issue#40991 <http://tracker.ceph.com/issues/40991>`_, `pr#29369 <https://github.com/ceph/ceph/pull/29369>`_, Tianshan Qu)
+* rgw: fix read not exists null version return wrong (`issue#38811 <http://tracker.ceph.com/issues/38811>`_, `pr#27047 <https://github.com/ceph/ceph/pull/27047>`_, Tianshan Qu)
+* rgw: fix refcount tags to match and update objects idtag (`pr#30013 <https://github.com/ceph/ceph/pull/30013>`_, J. Eric Ivancich)
+* rgw: fix REQUEST_URI setting in the rgw_asio_client.cc (`pr#30540 <https://github.com/ceph/ceph/pull/30540>`_, Jiaying Ren)
+* rgw: fix rgw crash and set correct error code (`pr#28172 <https://github.com/ceph/ceph/pull/28172>`_, yuliyang)
+* rgw: fix rgw crash when duration is invalid in sts request (`pr#32119 <https://github.com/ceph/ceph/pull/32119>`_, yuliyang)
+* rgw: fix rgw crash when token is not base64 encode (`pr#31830 <https://github.com/ceph/ceph/pull/31830>`_, yuliyang)
+* rgw: fix rgw decompression log-print (`pr#29633 <https://github.com/ceph/ceph/pull/29633>`_, Han Fengzhe)
+* rgw: fix rgw lc does not delete objects that do not have exactly the same tags as the rule (`pr#30151 <https://github.com/ceph/ceph/pull/30151>`_, Or Friedmann)
+* rgw: fix RGWDeleteMultiObj::verify_permission() (`pr#26947 <https://github.com/ceph/ceph/pull/26947>`_, Irek Fasikhov)
+* rgw: fix RGWUserInfo decode current version (`pr#31591 <https://github.com/ceph/ceph/pull/31591>`_, Chang Liu)
+* rgw: fix S3 compatibility bug when CORS is not found (`issue#37945 <http://tracker.ceph.com/issues/37945>`_, `pr#25999 <https://github.com/ceph/ceph/pull/25999>`_, Nick Janus)
+* rgw: fix sharded bucket listing with prefix/delimiter (`pr#33628 <https://github.com/ceph/ceph/pull/33628>`_, Casey Bodley)
+* rgw: fix SignatureDoesNotMatch when use ipv6 address in s3 client (`pr#30778 <https://github.com/ceph/ceph/pull/30778>`_, yuliyang)
+* rgw: fix signed char truncation in delimiter check (`pr#27001 <https://github.com/ceph/ceph/pull/27001>`_, Matt Benjamin)
+* rgw: fix string_view formatting in RGWFormatter_Plain (`pr#33754 <https://github.com/ceph/ceph/pull/33754>`_, Casey Bodley)
+* rgw: fix the bug of rgw not doing necessary checking to website configuration (`issue#40678 <http://tracker.ceph.com/issues/40678>`_, `pr#28904 <https://github.com/ceph/ceph/pull/28904>`_, Enming Zhang)
+* rgw: fix unlock of shared lock in RGWCache (`pr#29558 <https://github.com/ceph/ceph/pull/29558>`_, Abhishek Lekshmanan)
+* rgw: fix unlock of shared lock in RGWDataChangesLog (`pr#29538 <https://github.com/ceph/ceph/pull/29538>`_, Casey Bodley)
+* rgw: Fix upload part copy range able to get almost any string (`pr#32487 <https://github.com/ceph/ceph/pull/32487>`_, Or Friedmann)
+* rgw: fix version tracking across bucket link steps (`pr#29851 <https://github.com/ceph/ceph/pull/29851>`_, Matt Benjamin)
+* rgw: fixed unrecognized arg error when using radosgw-admin zone rm (`pr#30060 <https://github.com/ceph/ceph/pull/30060>`_, Hongang Chen)
+* rgw: Fixes related to omap offload and gc (`pr#33372 <https://github.com/ceph/ceph/pull/33372>`_, Pritha Srivastava)
+* rgw: followup for user rename (`pr#29540 <https://github.com/ceph/ceph/pull/29540>`_, Casey Bodley)
+* rgw: forwarded some requests to master zone (`pr#28276 <https://github.com/ceph/ceph/pull/28276>`_, Chang Liu)
+* rgw: gc remove tag after all sub io finish (`issue#40903 <http://tracker.ceph.com/issues/40903>`_, `pr#29199 <https://github.com/ceph/ceph/pull/29199>`_, Tianshan Qu)
+* rgw: get barbican secret key request maybe return error code (`pr#29639 <https://github.com/ceph/ceph/pull/29639>`_, Richard Bai(\xe7\x99\xbd\xe5\xad\xa6\xe4\xbd\x99))
+* rgw: get elastic search info in start_sync, avoid creating new coroutines manager (`pr#32269 <https://github.com/ceph/ceph/pull/32269>`_, Chang Liu)
+* rgw: housekeeping of reset stats operation in radosgw-admin and cls back-end (`pr#29515 <https://github.com/ceph/ceph/pull/29515>`_, J. Eric Ivancich)
+* rgw: http client drops lock before suspending coroutine (`pr#29553 <https://github.com/ceph/ceph/pull/29553>`_, Casey Bodley)
+* rgw: iam: add all http args to req_info (`pr#31124 <https://github.com/ceph/ceph/pull/31124>`_, Abhishek Lekshmanan)
+* rgw: iam: use a function to calculate the Action Bit string (`pr#30152 <https://github.com/ceph/ceph/pull/30152>`_, Abhishek Lekshmanan)
+* rgw: ignore If-Unmodified-Since if If-Match exists, and ignore If-Modified-Since if If-None-Match exists (`pr#28625 <https://github.com/ceph/ceph/pull/28625>`_, zhang Shaowen)
+* rgw: improve beast (`pr#33017 <https://github.com/ceph/ceph/pull/33017>`_, Or Friedmann, Matt Benjamin)
+* rgw: improve data sync restart after failure (`pr#30175 <https://github.com/ceph/ceph/pull/30175>`_, Tianshan Qu)
+* rgw: improve debugs on the path of RGWRados::cls_bucket_head (`pr#12709 <https://github.com/ceph/ceph/pull/12709>`_, Radoslaw Zarzynski)
+* rgw: improvements to SSE-KMS with Vault (`pr#31025 <https://github.com/ceph/ceph/pull/31025>`_, Andrea Baglioni, Sergio de Carvalho)
+* rgw: Improving doc for Cross Project(Tenant) access with Openstack Ke\xe2\x80\xa6 (`pr#27507 <https://github.com/ceph/ceph/pull/27507>`_, Pritha Srivastava)
+* rgw: incorrect return value when processing CORS headers (`pr#28622 <https://github.com/ceph/ceph/pull/28622>`_, Ilsoo Byun)
+* rgw: Incorrectly calling ceph::buffer::list::decode_base64 in bucket policy (`pr#31356 <https://github.com/ceph/ceph/pull/31356>`_, GaryHyg)
+* rgw: increase beast parse buffer size to 64k (`pr#29776 <https://github.com/ceph/ceph/pull/29776>`_, Casey Bodley)
+* rgw: increase log level for same or older period pull msg (`pr#33527 <https://github.com/ceph/ceph/pull/33527>`_, Ali Maredia)
+* rgw: Increase the default number of RGW bucket shards (`pr#32660 <https://github.com/ceph/ceph/pull/32660>`_, Casey Bodley, Mark Nelson)
+* rgw: init-radosgw: use ceph-conf to get cluster configuration value (`pr#27538 <https://github.com/ceph/ceph/pull/27538>`_, Daniel Badea)
+* rgw: Initialize member variables in rgw_sync.h, rgw_rados.h (`pr#16929 <https://github.com/ceph/ceph/pull/16929>`_, amitkuma)
+* rgw: initialize member variables of rgw_log_entry (`pr#32430 <https://github.com/ceph/ceph/pull/32430>`_, Kefu Chai)
+* rgw: kill compile warnning in rgw_object_lock.h (`pr#30489 <https://github.com/ceph/ceph/pull/30489>`_, Chang Liu)
+* rgw: LC expiration header should present midnight expiration date (`pr#31887 <https://github.com/ceph/ceph/pull/31887>`_, Or Friedmann)
+* rgw: lc: check for valid placement target before processing transitions (`pr#28256 <https://github.com/ceph/ceph/pull/28256>`_, Abhishek Lekshmanan)
+* rgw: LC: handle resharded buckets (`pr#26564 <https://github.com/ceph/ceph/pull/26564>`_, Abhishek Lekshmanan)
+* rgw: ldap auth: S3 auth failure should return InvalidAccessKeyId (`pr#30332 <https://github.com/ceph/ceph/pull/30332>`_, Matt Benjamin)
+* rgw: ldap: fix LDAPAuthEngine::init() when uri !empty() (`pr#26911 <https://github.com/ceph/ceph/pull/26911>`_, Matt Benjamin)
+* rgw: lifecycle days may be 0 (`pr#26524 <https://github.com/ceph/ceph/pull/26524>`_, Matt Benjamin)
+* rgw: lifecycle: alternate solution to prefix_map conflict (`issue#37879 <http://tracker.ceph.com/issues/37879>`_, `pr#26518 <https://github.com/ceph/ceph/pull/26518>`_, Matt Benjamin)
+* rgw: limit entries in remove_olh_pending_entries() (`issue#39118 <http://tracker.ceph.com/issues/39118>`_, `pr#27400 <https://github.com/ceph/ceph/pull/27400>`_, Casey Bodley)
+* rgw: list buckets: dont return buckets if limit=0 (`pr#32109 <https://github.com/ceph/ceph/pull/32109>`_, Yehuda Sadeh)
+* rgw: list_bucket versions return NextVersionIdMarker = null if next_marker.instance is empty (`pr#17591 <https://github.com/ceph/ceph/pull/17591>`_, Shasha Lu)
+* rgw: log refactoring for putobj_processor (`pr#26107 <https://github.com/ceph/ceph/pull/26107>`_, Ali Maredia)
+* rgw: log refactoring for rgw_rest_s3/swift ops (`pr#27037 <https://github.com/ceph/ceph/pull/27037>`_, Ali Maredia)
+* rgw: make dns hostnames matching case insensitive (`issue#40995 <http://tracker.ceph.com/issues/40995>`_, `pr#29380 <https://github.com/ceph/ceph/pull/29380>`_, Abhishek Lekshmanan)
+* rgw: make max_connections configurable in beast (`pr#33053 <https://github.com/ceph/ceph/pull/33053>`_, Tiago Pasqualini)
+* rgw: Make rgw admin ops api get user info consistent with the command line (`pr#26183 <https://github.com/ceph/ceph/pull/26183>`_, Li Shuhao)
+* rgw: make sure modelines are correct for all files (`pr#29742 <https://github.com/ceph/ceph/pull/29742>`_, Daniel Gryniewicz)
+* rgw: maybe coredump when reload operator happened (`pr#29733 <https://github.com/ceph/ceph/pull/29733>`_, Richard Bai(\xe7\x99\xbd\xe5\xad\xa6\xe4\xbd\x99))
+* rgw: metadata refactoring (`pr#29118 <https://github.com/ceph/ceph/pull/29118>`_, Casey Bodley, Yehuda Sadeh)
+* rgw: mgr/ansible: Change default realm and zonegroup (`pr#29793 <https://github.com/ceph/ceph/pull/29793>`_, Sebastian Wagner)
+* rgw: mgr/dashboard: enable/disable MFA Delete on RGW bucket (`pr#31922 <https://github.com/ceph/ceph/pull/31922>`_, Alfonso Mart\xc3\xadnez)
+* rgw: mgr/orchestrator: name rgw by client.rgw.$realm.$zone[.$id] (`pr#31890 <https://github.com/ceph/ceph/pull/31890>`_, Sage Weil)
+* rgw: mitigate bucket list with max-entries excessively high (`pr#29179 <https://github.com/ceph/ceph/pull/29179>`_, J. Eric Ivancich)
+* rgw: move bucket reshard checks out of write path (`pr#29852 <https://github.com/ceph/ceph/pull/29852>`_, Casey Bodley)
+* rgw: move delimiter-based bucket listing/filtering logic to cls (`pr#30272 <https://github.com/ceph/ceph/pull/30272>`_, J. Eric Ivancich)
+* rgw: move forward marker even in case of many rgw.none indexes (`pr#32513 <https://github.com/ceph/ceph/pull/32513>`_, Ilsoo Byun)
+* rgw: Move upload_info declaration out of conditional (`pr#29559 <https://github.com/ceph/ceph/pull/29559>`_, Adam C. Emerson)
+* rgw: multipart upload abort is best-effort (`issue#40526 <http://tracker.ceph.com/issues/40526>`_, `pr#28724 <https://github.com/ceph/ceph/pull/28724>`_, J. Eric Ivancich)
+* rgw: MultipartObjectProcessor supports stripe size > chunk size (`pr#32996 <https://github.com/ceph/ceph/pull/32996>`_, Casey Bodley)
+* rgw: multisite log trimming only checks peers that sync from us (`issue#39283 <http://tracker.ceph.com/issues/39283>`_, `pr#27567 <https://github.com/ceph/ceph/pull/27567>`_, Casey Bodley)
+* rgw: nfs: skip empty (non-POSIX) path segments (`issue#38744 <http://tracker.ceph.com/issues/38744>`_, `pr#26954 <https://github.com/ceph/ceph/pull/26954>`_, Matt Benjamin)
+* rgw: nfs: svc-enable RGWLi (`pr#26981 <https://github.com/ceph/ceph/pull/26981>`_, Matt Benjamin)
+* rgw: normalize v6 endpoint behaviour for the beast frontend (`issue#39038 <http://tracker.ceph.com/issues/39038>`_, `pr#27270 <https://github.com/ceph/ceph/pull/27270>`_, Abhishek Lekshmanan)
+* rgw: object expirer fixes (`pr#27870 <https://github.com/ceph/ceph/pull/27870>`_, Abhishek Lekshmanan)
+* rgw: Object tags shouldnt work with deletemarker or multipart expiration (`issue#40405 <http://tracker.ceph.com/issues/40405>`_, `pr#28617 <https://github.com/ceph/ceph/pull/28617>`_, zhang Shaowen)
+* rgw: one log shard fails shouldnt block other shards process when reshard buckets (`pr#31155 <https://github.com/ceph/ceph/pull/31155>`_, zhangshaowen)
+* rgw: One Rados Handle to Rule Them All (`pr#27102 <https://github.com/ceph/ceph/pull/27102>`_, Adam C. Emerson)
+* rgw: orphan fixes (`pr#26412 <https://github.com/ceph/ceph/pull/26412>`_, Abhishek Lekshmanan)
+* rgw: parse_copy_location defers url-decode (`issue#27217 <http://tracker.ceph.com/issues/27217>`_, `pr#25498 <https://github.com/ceph/ceph/pull/25498>`_, Casey Bodley)
+* rgw: perfcounters: add gc retire counter (`pr#26351 <https://github.com/ceph/ceph/pull/26351>`_, Matt Benjamin)
+* rgw: permit rgw-admin to populate user info by access-key (`pr#28331 <https://github.com/ceph/ceph/pull/28331>`_, Matt Benjamin)
+* rgw: Policy should be url_decode when assume_role (`pr#28704 <https://github.com/ceph/ceph/pull/28704>`_, yuliyang)
+* rgw: prefix-delimiter listing: support >1 character delimiter (`pr#26863 <https://github.com/ceph/ceph/pull/26863>`_, Matt Benjamin)
+* rgw: prevent bucket reshard scheduling if bucket is resharding (`pr#30610 <https://github.com/ceph/ceph/pull/30610>`_, J. Eric Ivancich)
+* rgw: prevent LC from reading stale head when transitioning object (`pr#31214 <https://github.com/ceph/ceph/pull/31214>`_, Ilsoo Byun)
+* rgw: project and return lc expiration from GET/HEAD and PUT ops (`pr#26160 <https://github.com/ceph/ceph/pull/26160>`_, Matt Benjamin)
+* rgw: Project Zipper - Bucket (`pr#31436 <https://github.com/ceph/ceph/pull/31436>`_, Daniel Gryniewicz)
+* rgw: Project Zipper - Bucketlist (`pr#30619 <https://github.com/ceph/ceph/pull/30619>`_, Daniel Gryniewicz)
+* rgw: Project Zipper part 1 (`pr#28824 <https://github.com/ceph/ceph/pull/28824>`_, Daniel Gryniewicz)
+* rgw: qa/suite/rgw/verify: valgrind on centos again! (`pr#32727 <https://github.com/ceph/ceph/pull/32727>`_, Sage Weil)
+* rgw: qa/tasks/s3tests_java: move to gradle 6.0.1 (`pr#32335 <https://github.com/ceph/ceph/pull/32335>`_, Sage Weil)
+* rgw: qa/tests: update s3a hadoop versions used for test (`pr#26100 <https://github.com/ceph/ceph/pull/26100>`_, Vasu Kulkarni)
+* rgw: qa: remove force-branch from overrides of s3-tests (`pr#32462 <https://github.com/ceph/ceph/pull/32462>`_, Ali Maredia)
+* rgw: qa: update s3-test download code for s3-test tasks (`pr#31839 <https://github.com/ceph/ceph/pull/31839>`_, Ali Maredia)
+* rgw: queue like an Egyptian(`pr#26461 <https://github.com/ceph/ceph/pull/26461>`_, Adam C. Emerson)
+* rgw: race condition between resharding and ops waiting on resharding (`issue#38990 <http://tracker.ceph.com/issues/38990>`_, `pr#27223 <https://github.com/ceph/ceph/pull/27223>`_, J. Eric Ivancich)
+* rgw: radosgw-admin flush user stats output (`pr#30669 <https://github.com/ceph/ceph/pull/30669>`_, Abhishek Lekshmanan)
+* rgw: radosgw-admin zone placement rm and radosgw-admin zonegroup placement rm support --storage-class (`pr#31239 <https://github.com/ceph/ceph/pull/31239>`_, yuliyang)
+* rgw: radosgw-admin: add --uid check in bucket list command (`pr#30194 <https://github.com/ceph/ceph/pull/30194>`_, Vikhyat Umrao)
+* rgw: radosgw-admin: bucket sync status not caught up during full sync (`issue#40806 <http://tracker.ceph.com/issues/40806>`_, `pr#29094 <https://github.com/ceph/ceph/pull/29094>`_, Casey Bodley)
+* rgw: radosgw-admin: fix syncs_from in bucket sync status (`issue#40022 <http://tracker.ceph.com/issues/40022>`_, `pr#28243 <https://github.com/ceph/ceph/pull/28243>`_, Casey Bodley)
+* rgw: radosgw-admin: sync status displays id of shard furthest behind (`pr#32311 <https://github.com/ceph/ceph/pull/32311>`_, Casey Bodley)
+* rgw: radosgw-admin: update help for max-concurrent-ios (`pr#30742 <https://github.com/ceph/ceph/pull/30742>`_, Paul Emmerich)
+* rgw: reduce per-shard entry count during ordered bucket listing (`pr#30853 <https://github.com/ceph/ceph/pull/30853>`_, J. Eric Ivancich)
+* rgw: reject bucket tagging requests and document unsupported (`pr#26952 <https://github.com/ceph/ceph/pull/26952>`_, Casey Bodley)
+* rgw: relax es zone validity check (`pr#32290 <https://github.com/ceph/ceph/pull/32290>`_, jiahuizeng)
+* rgw: release unused callback argument (`pr#32669 <https://github.com/ceph/ceph/pull/32669>`_, Ilsoo Byun)
+* rgw: remove re-defined is_tagging_op in RGWHandler_REST_Bucket_S3 (`pr#29004 <https://github.com/ceph/ceph/pull/29004>`_, zhang Shaowen)
+* rgw: remove unused bucket parameter in check_bucket_shards (`pr#31186 <https://github.com/ceph/ceph/pull/31186>`_, zhang Shaowen)
+* rgw: remove unused last_run in reshard thread entry (`pr#31150 <https://github.com/ceph/ceph/pull/31150>`_, zhangshaowen)
+* rgw: Replace COMPLETE_MULTIPART_MAX_LEN with rgw_max_put_param_size (`issue#38002 <http://tracker.ceph.com/issues/38002>`_, `pr#26070 <https://github.com/ceph/ceph/pull/26070>`_, Lei Liu)
+* rgw: replace direct calls to ioctx.operate() (`pr#28569 <https://github.com/ceph/ceph/pull/28569>`_, Ali Maredia)
+* rgw: ReplaceKeyPrefixWith and ReplaceKeyWith can not set at the same \xe2\x80\xa6 (`pr#32609 <https://github.com/ceph/ceph/pull/32609>`_, yuliyang)
+* rgw: reshard list may return more than specified max_entries (`pr#31355 <https://github.com/ceph/ceph/pull/31355>`_, zhangshaowen)
+* rgw: rest client fixes for cloud sync XML outputs (`pr#27680 <https://github.com/ceph/ceph/pull/27680>`_, Abhishek Lekshmanan)
+* rgw: return error if lock log shard fails (`pr#31344 <https://github.com/ceph/ceph/pull/31344>`_, zhangshaowen)
+* rgw: return ERR_NO_SUCH_BUCKET early while evaluating bucket policy (`issue#38420 <http://tracker.ceph.com/issues/38420>`_, `pr#26569 <https://github.com/ceph/ceph/pull/26569>`_, Abhishek Lekshmanan)
+* rgw: rgw : Bucket mv, bucket chown and user rename utilities (`issue#35885 <http://tracker.ceph.com/issues/35885>`_, `issue#24348 <http://tracker.ceph.com/issues/24348>`_, `pr#28813 <https://github.com/ceph/ceph/pull/28813>`_, Shilpa Jagannath, Marcus Watts)
+* rgw: rgw admin: add tenant argument to reshard cancel (`pr#26887 <https://github.com/ceph/ceph/pull/26887>`_, Abhishek Lekshmanan)
+* rgw: rgw admin: disable stale instance delete in a multiste env (`pr#26852 <https://github.com/ceph/ceph/pull/26852>`_, Abhishek Lekshmanan)
+* rgw: rgw multisite: add perf counters to data sync (`issue#38549 <http://tracker.ceph.com/issues/38549>`_, `pr#26722 <https://github.com/ceph/ceph/pull/26722>`_, Casey Bodley)
+* rgw: rgw multisite: avoid writing bilog entries on PREPARE and CANCEL (`pr#26755 <https://github.com/ceph/ceph/pull/26755>`_, Casey Bodley)
+* rgw: rgw multisite: data sync checks empty next_marker for datalog (`issue#39033 <http://tracker.ceph.com/issues/39033>`_, `pr#27276 <https://github.com/ceph/ceph/pull/27276>`_, Casey Bodley)
+* rgw: rgw multisite: enforce spawn window for incremental data sync (`pr#32534 <https://github.com/ceph/ceph/pull/32534>`_, Casey Bodley)
+* rgw: rgw multisite: fixes for concurrent version creation (`pr#31325 <https://github.com/ceph/ceph/pull/31325>`_, Casey Bodley)
+* rgw: rgw/kafka: add ssl+sasl security to kafka (`pr#31834 <https://github.com/ceph/ceph/pull/31834>`_, Yuval Lifshitz)
+* rgw: rgw/multisite: Dont allow certain radosgw-admin commands to run on non-master zone (`issue#39548 <http://tracker.ceph.com/issues/39548>`_, `pr#28861 <https://github.com/ceph/ceph/pull/28861>`_, Shilpa Jagannath)
+* rgw: rgw/multisite: warn if bucket chown command is run on non-master zone (`pr#32932 <https://github.com/ceph/ceph/pull/32932>`_, Shilpa Jagannath)
+* rgw: rgw/multisite:RGWListBucketIndexesCR for data full sync pagination (`issue#39551 <http://tracker.ceph.com/issues/39551>`_, `pr#28146 <https://github.com/ceph/ceph/pull/28146>`_, Shilpa Jagannath)
+* rgw: rgw/notification: add opaque data (`pr#32723 <https://github.com/ceph/ceph/pull/32723>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: add kafka notification endpoint (`pr#30960 <https://github.com/ceph/ceph/pull/30960>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: fix doc on updates. fix multi-notifications (`pr#27931 <https://github.com/ceph/ceph/pull/27931>`_, Yuval Lifshitz, Casey Bodley)
+* rgw: rgw/pubsub: fix records/event json format to match documentation (`pr#31926 <https://github.com/ceph/ceph/pull/31926>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: handle subscription conf errors better (`pr#27530 <https://github.com/ceph/ceph/pull/27530>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: notification filtering by object tags (`pr#31878 <https://github.com/ceph/ceph/pull/31878>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: prevent kafka thread from spinning when there are no messages (`pr#31998 <https://github.com/ceph/ceph/pull/31998>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: send notifications from multi-delete op (`pr#32155 <https://github.com/ceph/ceph/pull/32155>`_, Yuval Lifshitz)
+* rgw: rgw/pubsub: service reordering issue (`pr#29877 <https://github.com/ceph/ceph/pull/29877>`_, Yuval Lifshitz)
+* rgw: rgw/rgw_client_io_filters.h: print size_t the portable way (`pr#28838 <https://github.com/ceph/ceph/pull/28838>`_, Kefu Chai)
+* rgw: rgw/rgw_crypt.cc: silence -Wsign-compare GCC warning (`pr#29151 <https://github.com/ceph/ceph/pull/29151>`_, Kefu Chai)
+* rgw: rgw/rgw_main: auto set radosgws cpu affinity according to numa_node configuration (`pr#31001 <https://github.com/ceph/ceph/pull/31001>`_, luo rixin)
+* rgw: rgw/rgw_op: Remove get_val from hotpath via legacy options (`pr#29943 <https://github.com/ceph/ceph/pull/29943>`_, Mark Nelson)
+* rgw: rgw/rgw_rados: set pg_autoscale_bias=4 for omap pools (`pr#27375 <https://github.com/ceph/ceph/pull/27375>`_, Sage Weil, Casey Bodley)
+* rgw: rgw/rgw_reshard: Dont dump RGWBucketReshard JSON in process_single_logshard (`pr#29894 <https://github.com/ceph/ceph/pull/29894>`_, Mark Nelson)
+* rgw: rgw/rgw_user: add [[maybe_unused]] for silencing -Wunused-variable wa\xe2\x80\xa6 (`pr#30035 <https://github.com/ceph/ceph/pull/30035>`_, Kefu Chai)
+* rgw: rgw/services: silence -Wunused-variable warning (`pr#30063 <https://github.com/ceph/ceph/pull/30063>`_, Lan Liu)
+* rgw: RGW: add bucket permission verify when copy obj (`pr#29628 <https://github.com/ceph/ceph/pull/29628>`_, NancySu05)
+* rgw: RGW: fix an endless loop error when to show usage (`pr#30470 <https://github.com/ceph/ceph/pull/30470>`_, lvshuhua)
+* rgw: RGW: Set appropriate bucket quota value (when quota value is less than 0) (`pr#30920 <https://github.com/ceph/ceph/pull/30920>`_, GaryHyg)
+* rgw: RGW:Listobjectsv2 (`pr#28102 <https://github.com/ceph/ceph/pull/28102>`_, Albin Antony)
+* rgw: RGWCoroutine::call(nullptr) sets retcode=0 (`pr#29856 <https://github.com/ceph/ceph/pull/29856>`_, Casey Bodley)
+* rgw: rgwfile reqid:  absorbs rgw_file: allocate new id for continued request #25664 (`issue#37734 <http://tracker.ceph.com/issues/37734>`_, `pr#28108 <https://github.com/ceph/ceph/pull/28108>`_, Matt Benjamin, Tao Chen)
+* rgw: RGWPeriodPusher uses zone system key for inter-zonegroup messages (`issue#39287 <http://tracker.ceph.com/issues/39287>`_, `pr#27576 <https://github.com/ceph/ceph/pull/27576>`_, Casey Bodley)
+* rgw: RGWSI_User_Module filters .buckets objects out of user listing (`pr#29695 <https://github.com/ceph/ceph/pull/29695>`_, Casey Bodley)
+* rgw: rgw_file: advance_mtime() should consider namespace expiration (`issue#40415 <http://tracker.ceph.com/issues/40415>`_, `pr#28632 <https://github.com/ceph/ceph/pull/28632>`_, Matt Benjamin)
+* rgw: rgw_file: all directories are virtual with respect to contents (`issue#40204 <http://tracker.ceph.com/issues/40204>`_, `pr#28451 <https://github.com/ceph/ceph/pull/28451>`_, Matt Benjamin)
+* rgw: rgw_file: avoid string::front() on empty path (`pr#32596 <https://github.com/ceph/ceph/pull/32596>`_, Matt Benjamin)
+* rgw: rgw_file: dont deadlock in advance_mtime() (`pr#29560 <https://github.com/ceph/ceph/pull/29560>`_, Matt Benjamin)
+* rgw: rgw_file: fix readdir eof() calc--caller stop implies !eof (`issue#40375 <http://tracker.ceph.com/issues/40375>`_, `pr#28565 <https://github.com/ceph/ceph/pull/28565>`_, Matt Benjamin)
+* rgw: rgw_file: include tenant when hashing bucket names (`issue#40118 <http://tracker.ceph.com/issues/40118>`_, `pr#28370 <https://github.com/ceph/ceph/pull/28370>`_, Matt Benjamin)
+* rgw: rgw_file: introduce fast S3 Unix stats (immutable) (`issue#40456 <http://tracker.ceph.com/issues/40456>`_, `pr#28664 <https://github.com/ceph/ceph/pull/28664>`_, Matt Benjamin)
+* rgw: rgw_file: permit lookup_handle to lookup root_fh (`pr#28440 <https://github.com/ceph/ceph/pull/28440>`_, Matt Benjamin)
+* rgw: rgw_file: readdir: do not construct markers w/leading / (`pr#29670 <https://github.com/ceph/ceph/pull/29670>`_, Matt Benjamin)
+* rgw: rgw_file: save etag and acl info in setattr (`pr#26439 <https://github.com/ceph/ceph/pull/26439>`_, Tao Chen)
+* rgw: rgw_lc: use a new bl while encoding RGW_ATTR_LC (`pr#28049 <https://github.com/ceph/ceph/pull/28049>`_, Abhishek Lekshmanan)
+* rgw: rgw_sync: drop ENOENT error logs from mdlog (`pr#26908 <https://github.com/ceph/ceph/pull/26908>`_, Abhishek Lekshmanan)
+* rgw: s/std::map/boost::container::flat_map/ cls_bucket_list_ordered (`pr#28637 <https://github.com/ceph/ceph/pull/28637>`_, Matt Benjamin)
+* rgw: S3 compatible pubsub API (`pr#27091 <https://github.com/ceph/ceph/pull/27091>`_, Yuval Lifshitz)
+* rgw: s3: dont require a body in S3 put-object-acl (`pr#31987 <https://github.com/ceph/ceph/pull/31987>`_, Matt Benjamin)
+* rgw: save an unnecessary copy of RGWEnv (`pr#28426 <https://github.com/ceph/ceph/pull/28426>`_, Mark Kogan)
+* rgw: Select the std::bitset to resolv ambiguity (`pr#31126 <https://github.com/ceph/ceph/pull/31126>`_, Willem Jan Withagen)
+* rgw: set bucket attr twice when delete lifecycle config (`pr#30862 <https://github.com/ceph/ceph/pull/30862>`_, zhang Shaowen)
+* rgw: set correct storage class for append (`pr#31088 <https://github.com/ceph/ceph/pull/31088>`_, yuliyang)
+* rgw: set correct storage class for post object upload (`pr#30956 <https://github.com/ceph/ceph/pull/30956>`_, yuliyang)
+* rgw: set null version object acl issues (`issue#36763 <http://tracker.ceph.com/issues/36763>`_, `pr#25044 <https://github.com/ceph/ceph/pull/25044>`_, Tianshan Qu)
+* rgw: shard number must be non-negative when resharding the bucket (`pr#29037 <https://github.com/ceph/ceph/pull/29037>`_, zhang Shaowen)
+* rgw: silence a -Wunused-function warning in pubsu (`pr#27578 <https://github.com/ceph/ceph/pull/27578>`_, Casey Bodley)
+* rgw: Silence warning: control reaches end of non-void function (`issue#40747 <http://tracker.ceph.com/issues/40747>`_, `pr#28809 <https://github.com/ceph/ceph/pull/28809>`_, Jos Collin)
+* rgw: split mdlog/datalog trimming into separate files (`pr#27579 <https://github.com/ceph/ceph/pull/27579>`_, Casey Bodley)
+* rgw: sts: add all http args to req_info (`pr#31661 <https://github.com/ceph/ceph/pull/31661>`_, yuliyang)
+* rgw: support encoding-type param for list bucket multiparts (`pr#30993 <https://github.com/ceph/ceph/pull/30993>`_, Abhishek Lekshmanan)
+* rgw: support radosgw-admin zone/zonegroup placement get command (`pr#30880 <https://github.com/ceph/ceph/pull/30880>`_, jiahuizeng)
+* rgw: support specify user default placement and placement_tags when create or modify user (`pr#31185 <https://github.com/ceph/ceph/pull/31185>`_, yuliyang)
+* rgw: svc.bucket: assign to optional<> using = (`pr#32433 <https://github.com/ceph/ceph/pull/32433>`_, Kefu Chai)
+* rgw: swift: bugfix: https://tracker.ceph.com/issues/37765 (`pr#25962 <https://github.com/ceph/ceph/pull/25962>`_, Andrey Groshev)
+* rgw: sync counters: drop spaces from counter names (`pr#27725 <https://github.com/ceph/ceph/pull/27725>`_, Abhishek Lekshmanan)
+* rgw: sync with elastic search v7 (`pr#29637 <https://github.com/ceph/ceph/pull/29637>`_, Chang Liu)
+* rgw: TempURL should not allow PUTs with the X-Object-Manifest (`issue#20797 <http://tracker.ceph.com/issues/20797>`_, `pr#16659 <https://github.com/ceph/ceph/pull/16659>`_, Radoslaw Zarzynski)
+* rgw: test/rgw: fix test_rgw_reshard_wait with -DHAVE_BOOST_CONTEXT=OFF (`pr#32811 <https://github.com/ceph/ceph/pull/32811>`_, Yaakov Selkowitz)
+* rgw: test: modify iam tests to use a function to set bits (`pr#32808 <https://github.com/ceph/ceph/pull/32808>`_, Abhishek Lekshmanan)
+* rgw: tests: Fix building with -DWITH_BOOST_CONTEXT=OFF (`pr#29430 <https://github.com/ceph/ceph/pull/29430>`_, Ulrich Weigand)
+* rgw: the http response code of delete bucket should not be 204-no-content (`pr#30471 <https://github.com/ceph/ceph/pull/30471>`_, Chang Liu)
+* rgw: Thread optional yield context through  get_bucket_info call path (`pr#27898 <https://github.com/ceph/ceph/pull/27898>`_, Ali Maredia)
+* rgw: thread option_yield through bucket index transaction prepare (`pr#28152 <https://github.com/ceph/ceph/pull/28152>`_, Ali Maredia)
+* rgw: unexpected crash when creating bucket in librgw (`pr#26089 <https://github.com/ceph/ceph/pull/26089>`_, Tao CHEN)
+* rgw: update op_mask of user via admin rest api (`issue#39084 <http://tracker.ceph.com/issues/39084>`_, `pr#21154 <https://github.com/ceph/ceph/pull/21154>`_, Ning Yao)
+* rgw: update the hash source for multipart entries during resharding (`pr#32617 <https://github.com/ceph/ceph/pull/32617>`_, dongdong tao)
+* rgw: update the radosgw-admin reshard status (`issue#37615 <http://tracker.ceph.com/issues/37615>`_, `pr#25496 <https://github.com/ceph/ceph/pull/25496>`_, Mark Kogan)
+* rgw: updates to resharding documentation (`issue#39007 <http://tracker.ceph.com/issues/39007>`_, `pr#27250 <https://github.com/ceph/ceph/pull/27250>`_, J. Eric Ivancich)
+* rgw: url decode PutUserPolicy params (`pr#29578 <https://github.com/ceph/ceph/pull/29578>`_, Abhishek Lekshmanan)
+* rgw: url encode common prefixes for List Objects response (`pr#30970 <https://github.com/ceph/ceph/pull/30970>`_, Abhishek Lekshmanan)
+* rgw: usage dump_unsigned instead dump_int (`pr#28308 <https://github.com/ceph/ceph/pull/28308>`_, yuliyang)
+* rgw: usage dump_unsigned instead dump_int in dump_usage_categories_info (`pr#25808 <https://github.com/ceph/ceph/pull/25808>`_, yuliyang)
+* rgw: use bucket creation time from bucket instance info (`pr#32180 <https://github.com/ceph/ceph/pull/32180>`_, Yehuda Sadeh)
+* rgw: use explicit to_string() overload for boost::string_ref (`issue#39611 <http://tracker.ceph.com/issues/39611>`_, `pr#28013 <https://github.com/ceph/ceph/pull/28013>`_, Casey Bodley)
+* rgw: use new Stopped state for special handling of bucket sync disable (`pr#33054 <https://github.com/ceph/ceph/pull/33054>`_, Casey Bodley)
+* rgw: use STSEngine::authenticate when post upload with x_amz_security_token (`pr#31879 <https://github.com/ceph/ceph/pull/31879>`_, yuliyang)
+* rgw: use the compatibilty function for pthread_setname (`pr#27456 <https://github.com/ceph/ceph/pull/27456>`_, Willem Jan Withagen)
+* rgw: user policy: forward write requests to master zone (`pr#32476 <https://github.com/ceph/ceph/pull/32476>`_, Abhishek Lekshmanan)
+* rgw: vstart: move [client.rgw] config into [client] (`pr#29778 <https://github.com/ceph/ceph/pull/29778>`_, Casey Bodley)
+* rgw: vstart: only add --debug-ms=1 in RGWDEBUG (`pr#27409 <https://github.com/ceph/ceph/pull/27409>`_, Casey Bodley)
+* rgw: warn on potential insecure mon connection (`pr#33777 <https://github.com/ceph/ceph/pull/33777>`_, Yehuda Sadeh)
+* rgw: when resharding store progress json (`pr#30575 <https://github.com/ceph/ceph/pull/30575>`_, Mark Kogan)
+* rgw: when you abort a multipart upload request, the quota may be not updated (`pr#29703 <https://github.com/ceph/ceph/pull/29703>`_, Richard Bai(\xe7\x99\xbd\xe5\xad\xa6\xe4\xbd\x99))
+* rgw: Zipper - RGWUser (`pr#32298 <https://github.com/ceph/ceph/pull/32298>`_, Daniel Gryniewicz)
+* rgw: [RFC] rgw: raise default rgw_bucket_index_max_aio to 128 (`pr#28558 <https://github.com/ceph/ceph/pull/28558>`_, Casey Bodley)
+* rgw: [rgw]:Validate bucket names as per revised s3 spec (`pr#26787 <https://github.com/ceph/ceph/pull/26787>`_, Soumya Koduri)
+* seastar,crimson: pickup change to pin socket to fixed core (`pr#32797 <https://github.com/ceph/ceph/pull/32797>`_, Kefu Chai)
+* seastar: pick up changes for better performance (`pr#28008 <https://github.com/ceph/ceph/pull/28008>`_, Kefu Chai)
+* seastar: pick up latest changes and cleanups (`pr#29942 <https://github.com/ceph/ceph/pull/29942>`_, Kefu Chai)
+* seastar: pick up the latest seastar (`pr#28709 <https://github.com/ceph/ceph/pull/28709>`_, Kefu Chai)
+* seastar: pickup change to fix cgroups V2 support (`pr#32978 <https://github.com/ceph/ceph/pull/32978>`_, Kefu Chai)
+* seastar: pickup the recent future optimizations (`pr#32296 <https://github.com/ceph/ceph/pull/32296>`_, Radoslaw Zarzynski)
+* seastar: pickup unix domain socket support (`pr#30578 <https://github.com/ceph/ceph/pull/30578>`_, Kefu Chai)
+* src/: silence GCC warnings (`pr#28684 <https://github.com/ceph/ceph/pull/28684>`_, Adam C. Emerson, Kefu Chai)
+* src/msg/async/net_handler.cc: Fix compilation (`pr#31637 <https://github.com/ceph/ceph/pull/31637>`_, Carlos Valiente)
+* src/script/kubejacker: Fix and simplify (`issue#39065 <http://tracker.ceph.com/issues/39065>`_, `pr#27292 <https://github.com/ceph/ceph/pull/27292>`_, Sebastian Wagner)
+* src/script: extract mypy config to mypy.ini (`pr#28264 <https://github.com/ceph/ceph/pull/28264>`_, Alfonso Mart\xc3\xadnez)
+* src/telemetry: remove, now lives in ceph-telemetry.git (`pr#31170 <https://github.com/ceph/ceph/pull/31170>`_, Dan Mick)
+* src: polish the wording (`pr#33224 <https://github.com/ceph/ceph/pull/33224>`_, Jun Su)
+* stop.sh: add --crimson option (`pr#28676 <https://github.com/ceph/ceph/pull/28676>`_, Kefu Chai)
+* stop.sh: do not try to contact mon unless cluster is up (`pr#32295 <https://github.com/ceph/ceph/pull/32295>`_, Kefu Chai)
+* support RDMA NIC without SRQ in msg/async/rdma (`pr#29947 <https://github.com/ceph/ceph/pull/29947>`_, Changcheng Liu, Roman Penyaev)
+* tasks/ceph_deploy: get rid of iteritems for python3 (`pr#30791 <https://github.com/ceph/ceph/pull/30791>`_, Kyr Shatskyy)
+* telemetry: make server compensate for older mgr modules, elasticsearch (`pr#27802 <https://github.com/ceph/ceph/pull/27802>`_, Dan Mick)
+* test/crimson: fix interpretability with perf_async_msgr (`pr#28913 <https://github.com/ceph/ceph/pull/28913>`_, Yingxin Cheng)
+* tests,tools: ceph-objectstore-tool: call collection_bits() crashes on the meta col\xe2\x80\xa6 (`pr#31133 <https://github.com/ceph/ceph/pull/31133>`_, David Zafman)
+* tests,tools: ceph-objectstore-tool: set log date format (`pr#29297 <https://github.com/ceph/ceph/pull/29297>`_, Robert Church)
+* tests,tools: tools/ceph-dencoder: split types.h into smaller pieces (`issue#39595 <http://tracker.ceph.com/issues/39595>`_, `pr#28359 <https://github.com/ceph/ceph/pull/28359>`_, Kefu Chai)
+* tests,tools: tools/setup-virtualenv.sh: do not default to python2.7 (`pr#30379 <https://github.com/ceph/ceph/pull/30379>`_, Nathan Cutler)
+* tests: add missing header cmath to test/mon/test_mon_memory_target.cc (`pr#30284 <https://github.com/ceph/ceph/pull/30284>`_, Su Yue)
+* tests: ceph-object-corpus: pick up 15.0.0-539-g191ab33faf (`pr#27867 <https://github.com/ceph/ceph/pull/27867>`_, Kefu Chai)
+* tests: cls/queue: add unit tests (`pr#33218 <https://github.com/ceph/ceph/pull/33218>`_, Yuval Lifshitz)
+* tests: corrected issues with RBD tests under EL8 distros (`pr#32684 <https://github.com/ceph/ceph/pull/32684>`_, Jason Dillaman)
+* tests: crimson/net: configure seastar to accept on a fixed core (`pr#32632 <https://github.com/ceph/ceph/pull/32632>`_, Yingxin Cheng)
+* tests: crimson/test: add CBT based perf tests (`pr#29612 <https://github.com/ceph/ceph/pull/29612>`_, Kefu Chai)
+* tests: crimson/test: v2 failover tests with crimson FailoverTestPeer (`pr#30162 <https://github.com/ceph/ceph/pull/30162>`_, Yingxin Cheng)
+* tests: crush, test: update editor variables (`pr#30537 <https://github.com/ceph/ceph/pull/30537>`_, Kefu Chai)
+* tests: fio_ceph_messenger: catch up v2 proto changes by using dummy auth (`pr#27264 <https://github.com/ceph/ceph/pull/27264>`_, Roman Penyaev)
+* tests: import-generated.sh: use PATH to get ceph-dencoder (`pr#27573 <https://github.com/ceph/ceph/pull/27573>`_, Changcheng Liu)
+* tests: introduce compiletest_cxx11_client for C++11 conformity (`pr#25395 <https://github.com/ceph/ceph/pull/25395>`_, Radoslaw Zarzynski)
+* tests: lvm/deactivate: add unit tests, remove --all (`pr#32277 <https://github.com/ceph/ceph/pull/32277>`_, Jan Fajerski)
+* tests: mgr/dashboard: ability to provide custom credentials for E2E tests (`pr#33549 <https://github.com/ceph/ceph/pull/33549>`_, Alfonso Mart\xc3\xadnez)
+* tests: mgr/dashboard: Add linter for unclosed HTML tags (`issue#40686 <http://tracker.ceph.com/issues/40686>`_, `pr#28916 <https://github.com/ceph/ceph/pull/28916>`_, Patrick Nawracay)
+* tests: mgr/dashboard: add python-common to $PYTHONPATH (`pr#29525 <https://github.com/ceph/ceph/pull/29525>`_, Kefu Chai)
+* tests: mgr/dashboard: Added breadcrumb tests to Manager modules and Alerts menu (`pr#26853 <https://github.com/ceph/ceph/pull/26853>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added breadcrumb tests to NFS menu (`pr#26850 <https://github.com/ceph/ceph/pull/26850>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Added breadcrumb tests to Object Gateway menu items (`pr#25451 <https://github.com/ceph/ceph/pull/25451>`_, Nathan Weinberg, Tiago Melo)
+* tests: mgr/dashboard: comment failing QA suites out (`pr#30864 <https://github.com/ceph/ceph/pull/30864>`_, Tatjana Dehler)
+* tests: mgr/dashboard: disable pylints --py3k flag (`pr#30078 <https://github.com/ceph/ceph/pull/30078>`_, Ernesto Puerta)
+* tests: mgr/dashboard: E2E test to verify Configuration editing functionality (`pr#29216 <https://github.com/ceph/ceph/pull/29216>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: Explicitly type page variables (`pr#29324 <https://github.com/ceph/ceph/pull/29324>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: Fix e2e host test (`pr#30377 <https://github.com/ceph/ceph/pull/30377>`_, Tiago Melo)
+* tests: mgr/dashboard: fix existing issues in user integration tests (`pr#30789 <https://github.com/ceph/ceph/pull/30789>`_, Tatjana Dehler)
+* tests: mgr/dashboard: fix stray requests/error in Grafana unit test (`pr#33572 <https://github.com/ceph/ceph/pull/33572>`_, Patrick Seidensal)
+* tests: mgr/dashboard: fix tasks.mgr.dashboard.test_rgw suite (`pr#33426 <https://github.com/ceph/ceph/pull/33426>`_, Alfonso Mart\xc3\xadnez)
+* tests: mgr/dashboard: fix tests in order to match pg num conventions (`pr#31906 <https://github.com/ceph/ceph/pull/31906>`_, Tatjana Dehler)
+* tests: mgr/dashboard: Improve e2e script (`pr#29101 <https://github.com/ceph/ceph/pull/29101>`_, Valentin Bajrami)
+* tests: mgr/dashboard: RBD Image Purge Trash, Move to Trash and Restore (`pr#29673 <https://github.com/ceph/ceph/pull/29673>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: reactivate dashboard test suites (`pr#32005 <https://github.com/ceph/ceph/pull/32005>`_, Tatjana Dehler)
+* tests: mgr/dashboard: Reduce code duplication through TableActionComponent UnitTests (`issue#40399 <http://tracker.ceph.com/issues/40399>`_, `pr#28633 <https://github.com/ceph/ceph/pull/28633>`_, Patrick Nawracay)
+* tests: mgr/dashboard: restore working directory after creating venv (`pr#32371 <https://github.com/ceph/ceph/pull/32371>`_, Kefu Chai)
+* tests: mgr/dashboard: RGW bucket E2E Tests (`pr#28999 <https://github.com/ceph/ceph/pull/28999>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: RGW user E2E Tests (`pr#29237 <https://github.com/ceph/ceph/pull/29237>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: take portal_ip_addresses as a list (`pr#28495 <https://github.com/ceph/ceph/pull/28495>`_, Kefu Chai)
+* tests: mgr/dashboard: Update formatting of e2e test files (`pr#29070 <https://github.com/ceph/ceph/pull/29070>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: Updated existing E2E tests to match new format (`pr#27408 <https://github.com/ceph/ceph/pull/27408>`_, Nathan Weinberg)
+* tests: mgr/dashboard: Verify fields on Configuration page (`pr#29583 <https://github.com/ceph/ceph/pull/29583>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: Verify fields on OSDs page (`pr#29447 <https://github.com/ceph/ceph/pull/29447>`_, Adam King, Rafael Quintero)
+* tests: mgr/dashboard: Wait for iSCSI target put and delete (`pr#30588 <https://github.com/ceph/ceph/pull/30588>`_, Ricardo Marques)
+* tests: mgr/dashboard: Write E2E tests for pool creation, deletion and verification (`issue#40693 <http://tracker.ceph.com/issues/40693>`_, `issue#38093 <http://tracker.ceph.com/issues/38093>`_, `pr#28928 <https://github.com/ceph/ceph/pull/28928>`_, Patrick Nawracay)
+* tests: mgr/orch: try harder when pickle fails to marshal an exception (`pr#33701 <https://github.com/ceph/ceph/pull/33701>`_, Kefu Chai)
+* tests: mgr/ssh: add make check integration (`pr#31523 <https://github.com/ceph/ceph/pull/31523>`_, Sebastian Wagner)
+* tests: mgr/tox: make run-tox.sh scripts more robust (`issue#39323 <http://tracker.ceph.com/issues/39323>`_, `pr#27614 <https://github.com/ceph/ceph/pull/27614>`_, Nathan Cutler)
+* tests: osd-backfill-space.sh test failed in TEST_backfill_multi_partial() (`issue#39333 <http://tracker.ceph.com/issues/39333>`_, `pr#27769 <https://github.com/ceph/ceph/pull/27769>`_, David Zafman)
+* tests: pybind/mgr: apply_drivegroups should return Sequence[Completion] (`pr#33977 <https://github.com/ceph/ceph/pull/33977>`_, Kefu Chai)
+* tests: python: pin mypy requirement to mypy==0.770 (`pr#33926 <https://github.com/ceph/ceph/pull/33926>`_, Sebastian Wagner)
+* tests: qa.tests: added smoke suite to the schedule on mimic,nautilus (`pr#28479 <https://github.com/ceph/ceph/pull/28479>`_, Yuri Weinstein)
+* tests: qa/ceph-ansible: Disable dashboard (`pr#29916 <https://github.com/ceph/ceph/pull/29916>`_, Brad Hubbard)
+* tests: qa/ceph-ansible: Move to ansible 2.8 (`issue#40602 <http://tracker.ceph.com/issues/40602>`_, `pr#28803 <https://github.com/ceph/ceph/pull/28803>`_, Brad Hubbard)
+* tests: qa/ceph-ansible: Move to Nautilus (`pr#27013 <https://github.com/ceph/ceph/pull/27013>`_, Brad Hubbard)
+* tests: qa/ceph-ansible: Replace pgs with pg_num (`issue#40605 <http://tracker.ceph.com/issues/40605>`_, `pr#28807 <https://github.com/ceph/ceph/pull/28807>`_, Brad Hubbard)
+* tests: qa/ceph-ansible: Upgrade ansible version (`pr#33379 <https://github.com/ceph/ceph/pull/33379>`_, Brad Hubbard)
+* tests: qa/cephadm/smoke: run on opensuse_15.1 (`pr#33338 <https://github.com/ceph/ceph/pull/33338>`_, Nathan Cutler)
+* tests: qa/crontab/teuthology-cronjobs: fix suite-branch (`pr#27140 <https://github.com/ceph/ceph/pull/27140>`_, Neha Ojha)
+* tests: qa/distros/all: add openSUSE 15.1, drop openSUSE 12.2 (`pr#30597 <https://github.com/ceph/ceph/pull/30597>`_, Nathan Cutler)
+* tests: qa/distros: add SLE-12-SP3 and SLE-15-SP1 (`pr#31112 <https://github.com/ceph/ceph/pull/31112>`_, Nathan Cutler)
+* tests: qa/orchestrator: do not test mon update 3 host1 (`pr#32023 <https://github.com/ceph/ceph/pull/32023>`_, Sage Weil, Kefu Chai)
+* tests: qa/standalone/ceph-helpers: resurrect all OSD before waiting for health (`pr#28328 <https://github.com/ceph/ceph/pull/28328>`_, Kefu Chai)
+* tests: qa/standalone/test_ceph_daemon: Fix ceph daemon standalone test (`pr#31440 <https://github.com/ceph/ceph/pull/31440>`_, Thomas Bechtold)
+* tests: qa/suites/krbd: fsx with object-map and fast-diff (`pr#32376 <https://github.com/ceph/ceph/pull/32376>`_, Ilya Dryomov)
+* tests: qa/suites/rados/cephadm/upgrade: add simple upgrade test (`pr#33343 <https://github.com/ceph/ceph/pull/33343>`_, Sage Weil)
+* tests: qa/suites/rados/cephadm: deploy all monitoring components (`pr#33785 <https://github.com/ceph/ceph/pull/33785>`_, Sage Weil)
+* tests: qa/suites/rados/perf/objectstore: do not symlink to qa/objectstore (`pr#30309 <https://github.com/ceph/ceph/pull/30309>`_, Neha Ojha)
+* tests: qa/suites/rados/perf: test min recommended osd_memory_target (`pr#30347 <https://github.com/ceph/ceph/pull/30347>`_, Neha Ojha)
+* tests: qa/suites/rados: whitelist POOL_APP_NOT_ENABLED warning (`pr#29763 <https://github.com/ceph/ceph/pull/29763>`_, Kefu Chai)
+* tests: qa/suites/upgrade/nautilus-x/parallel: restart mgr.x before mons (`pr#33705 <https://github.com/ceph/ceph/pull/33705>`_, Neha Ojha)
+* tests: qa/suites/upgrade: use correct branch names (`pr#27764 <https://github.com/ceph/ceph/pull/27764>`_, Neha Ojha)
+* tests: qa/suites: do not test luminous-x upgrade path (`pr#27112 <https://github.com/ceph/ceph/pull/27112>`_, Kefu Chai)
+* tests: qa/tasks/cbt.py: add support for client_endpoints (`pr#28522 <https://github.com/ceph/ceph/pull/28522>`_, Neha Ojha)
+* tests: qa/tasks/cbt.py: change port to work with client_endpoints (`pr#28442 <https://github.com/ceph/ceph/pull/28442>`_, Neha Ojha)
+* tests: qa/tasks/cbt.py: use git --depth 1 for faster clone (`pr#29597 <https://github.com/ceph/ceph/pull/29597>`_, Kefu Chai)
+* tests: qa/tasks/ceph.py: quote <kind> in command line (`pr#33775 <https://github.com/ceph/ceph/pull/33775>`_, Kefu Chai)
+* tests: qa/tasks/ceph.py: remove unused variables (`pr#31005 <https://github.com/ceph/ceph/pull/31005>`_, Kefu Chai)
+* tests: qa/tasks/ceph2: add support for shell, packaged ceph-daemon (`pr#31891 <https://github.com/ceph/ceph/pull/31891>`_, Sage Weil)
+* tests: qa/tasks/cephfs_test_runner: setattr to class not instance (`pr#32571 <https://github.com/ceph/ceph/pull/32571>`_, Kefu Chai)
+* tests: qa/tasks/ceph_deploy: assume systemd and simplify shutdown wonkiness (`pr#29030 <https://github.com/ceph/ceph/pull/29030>`_, Sage Weil)
+* tests: qa/tasks/ceph_deploy: install python3.6 instead of python3.4 for py3 tests (`pr#27504 <https://github.com/ceph/ceph/pull/27504>`_, Kefu Chai)
+* tests: qa/tasks/ceph_manager.py: ignore errors in test_pool_min_size (`issue#40533 <http://tracker.ceph.com/issues/40533>`_, `pr#28731 <https://github.com/ceph/ceph/pull/28731>`_, Kefu Chai)
+* tests: qa/tasks/ceph_manager: capture stderr for COT (`pr#33805 <https://github.com/ceph/ceph/pull/33805>`_, Kefu Chai)
+* tests: qa/tasks/ceph_manager: do not panic if pg_num_target is missing (`pr#30973 <https://github.com/ceph/ceph/pull/30973>`_, Kefu Chai)
+* tests: qa/tasks/ceph_manager: do not pick a pool is there is no pools (`pr#32519 <https://github.com/ceph/ceph/pull/32519>`_, Kefu Chai)
+* tests: qa/tasks/mgr/dashboard/test_health: add allow_unknown in mgr_map (`pr#30517 <https://github.com/ceph/ceph/pull/30517>`_, Kefu Chai)
+* tests: qa/tasks/mgr/dashboard/test_health: add missing field for test_full_health (`pr#29615 <https://github.com/ceph/ceph/pull/29615>`_, Kefu Chai)
+* tests: qa/tasks/mgr/dashboard/test_health: update schema (`pr#32122 <https://github.com/ceph/ceph/pull/32122>`_, Tatjana Dehler)
+* tests: qa/tasks/mgr/dashboard/test_mgr_module: sync w/ telemetry (`pr#29461 <https://github.com/ceph/ceph/pull/29461>`_, Kefu Chai)
+* tests: qa/tasks/mgr/dashboard: set pg_num to 16 (`pr#32575 <https://github.com/ceph/ceph/pull/32575>`_, Kefu Chai)
+* tests: qa/tasks/mgr/test_orchestrator_cli: fix mon update test (`pr#32428 <https://github.com/ceph/ceph/pull/32428>`_, Kefu Chai)
+* tests: qa/tasks/mgr/test_orchestrator_cli: fix service action tests (`pr#32518 <https://github.com/ceph/ceph/pull/32518>`_, Kefu Chai)
+* tests: qa/tasks/mgr/test_orchestrator_cli: fix test_host_ls (`pr#33477 <https://github.com/ceph/ceph/pull/33477>`_, Sage Weil)
+* tests: qa/tasks/mgr/test_progress.py: fix bug in 9b4dbf0 (`pr#29385 <https://github.com/ceph/ceph/pull/29385>`_, Kamoltat (Junior) Sirivadhna)
+* tests: qa/tasks/mgr/test_progress.py: s/ev/new_event/ (`issue#40618 <http://tracker.ceph.com/issues/40618>`_, `pr#29368 <https://github.com/ceph/ceph/pull/29368>`_, Kefu Chai)
+* tests: qa/tasks/mgr: set mgr module option with --force (`pr#32588 <https://github.com/ceph/ceph/pull/32588>`_, Kefu Chai)
+* tests: qa/tasks/vstart_runner: write string to StringIO (`pr#32438 <https://github.com/ceph/ceph/pull/32438>`_, Kefu Chai)
+* tests: qa/tasks: call super classs setUp() (`pr#33325 <https://github.com/ceph/ceph/pull/33325>`_, Kefu Chai)
+* tests: qa/tasks: py3 compat (tasks exercised by rados suites) (`pr#33709 <https://github.com/ceph/ceph/pull/33709>`_, Kyr Shatskyy, Kefu Chai)
+* tests: qa/tasks: use items() for py3 compatibility (`pr#30813 <https://github.com/ceph/ceph/pull/30813>`_, Kyr Shatskyy)
+* tests: qa/tests:  filtered in only trusty (`issue#40195 <http://tracker.ceph.com/issues/40195>`_, `pr#28439 <https://github.com/ceph/ceph/pull/28439>`_, Yuri Weinstein)
+* tests: qa/tests: added mimic-x on master run (`pr#29428 <https://github.com/ceph/ceph/pull/29428>`_, Yuri Weinstein)
+* tests: qa/tests: added nautilus-p2p to cron (`pr#27218 <https://github.com/ceph/ceph/pull/27218>`_, Yuri Weinstein)
+* tests: qa/tests: added nautilus-x run (`pr#27252 <https://github.com/ceph/ceph/pull/27252>`_, Yuri Weinstein)
+* tests: qa/tests: added new client-upgrade-\\*-nautilus suites for jewel, luminous, mimic (`pr#28067 <https://github.com/ceph/ceph/pull/28067>`_, Yuri Weinstein)
+* tests: qa/tests: added ragweed coverage to stress-split\\* upgrade suites (`issue#40467 <http://tracker.ceph.com/issues/40467>`_, `issue#40452 <http://tracker.ceph.com/issues/40452>`_, `pr#28931 <https://github.com/ceph/ceph/pull/28931>`_, Yuri Weinstein)
+* tests: qa/tests: added ragweed coverage to stress-split\\* upgrade suites (`issue#40467 <http://tracker.ceph.com/issues/40467>`_, `issue#40452 <http://tracker.ceph.com/issues/40452>`_, `pr#28932 <https://github.com/ceph/ceph/pull/28932>`_, Yuri Weinstein)
+* tests: qa/tests: added rgw into upgrade sequence to improve coverage (`pr#29406 <https://github.com/ceph/ceph/pull/29406>`_, Yuri Weinstein)
+* tests: qa/tests: reduced distro to run to be random (`pr#28435 <https://github.com/ceph/ceph/pull/28435>`_, Yuri Weinstein)
+* tests: qa/tests: reduced frequency for luminous and mimic runs (`pr#27057 <https://github.com/ceph/ceph/pull/27057>`_, Yuri Weinstein)
+* tests: qa/tests: removed all runs for luminous - EOL (`pr#33186 <https://github.com/ceph/ceph/pull/33186>`_, Yuri Weinstein)
+* tests: qa/tests: removed upgrade/client-upgrade-hammer becasue ubuntu 14.04 \xe2\x80\xa6 (`pr#28518 <https://github.com/ceph/ceph/pull/28518>`_, Yuri Weinstein)
+* tests: qa/tests: removed `1node` and `systemd` tests as ceph-deploy is not actively developed (`issue#40207 <http://tracker.ceph.com/issues/40207>`_, `issue#40208 <http://tracker.ceph.com/issues/40208>`_, `pr#28455 <https://github.com/ceph/ceph/pull/28455>`_, Yuri Weinstein)
+* tests: qa/valgrind.supp: generalize the whiterule for aes-128-gcm to help rgw suite (`issue#38827 <http://tracker.ceph.com/issues/38827>`_, `pr#28305 <https://github.com/ceph/ceph/pull/28305>`_, Radoslaw Zarzynski)
+* tests: qa/workunits/cephadm/test_cephadm: drop stray exit 0 (`pr#32622 <https://github.com/ceph/ceph/pull/32622>`_, Sage Weil)
+* tests: qa/workunits/cephtool/test.sh: a handful fixes (`pr#31689 <https://github.com/ceph/ceph/pull/31689>`_, Kefu Chai)
+* tests: qa/workunits/mon/config.sh: s|bin/ceph|ceph| (`pr#27147 <https://github.com/ceph/ceph/pull/27147>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_crash.sh: do not rm coredump (`pr#32883 <https://github.com/ceph/ceph/pull/32883>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_envlibrados_for_rocksdb: accomodate rocksdb c\xe2\x80\xa6 (`pr#32143 <https://github.com/ceph/ceph/pull/32143>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_envlibrados_for_rocksdb: install newer cmake (`pr#29584 <https://github.com/ceph/ceph/pull/29584>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_librados_build.sh: download from current branch (`pr#31693 <https://github.com/ceph/ceph/pull/31693>`_, Kefu Chai)
+* tests: qa/workunits/rados/test_librados_build.sh: install build deps (`pr#28484 <https://github.com/ceph/ceph/pull/28484>`_, Kefu Chai)
+* tests: qa/workunits/rest: Better detection of rest url (`pr#26604 <https://github.com/ceph/ceph/pull/26604>`_, Brad Hubbard)
+* tests: qa: add .qa link (`pr#32363 <https://github.com/ceph/ceph/pull/32363>`_, Patrick Donnelly)
+* tests: qa: Add basic mypy support for the qa directory (`pr#32495 <https://github.com/ceph/ceph/pull/32495>`_, Thomas Bechtold)
+* tests: qa: add path to device output schema (`pr#32427 <https://github.com/ceph/ceph/pull/32427>`_, Kefu Chai)
+* tests: qa: add RHEL 7.7 and use as RHEL7 default (`pr#29908 <https://github.com/ceph/ceph/pull/29908>`_, Patrick Donnelly)
+* tests: qa: correct zap disk with ceph-deploy tool (`pr#31312 <https://github.com/ceph/ceph/pull/31312>`_, Changcheng Liu, Alfredo Deza)
+* tests: qa: distro helper symlinks (`pr#28371 <https://github.com/ceph/ceph/pull/28371>`_, Patrick Donnelly)
+* tests: qa: enable CRB repo for RHEL8 (`pr#32426 <https://github.com/ceph/ceph/pull/32426>`_, Kefu Chai)
+* tests: qa: enable dashboard tests to be run with --suite rados/dashboard (`pr#30434 <https://github.com/ceph/ceph/pull/30434>`_, Nathan Cutler)
+* tests: qa: Enable flake8 tox and fix failures (`pr#32129 <https://github.com/ceph/ceph/pull/32129>`_, Thomas Bechtold)
+* tests: qa: fix all the fsx.sh-invoking yaml files to install dependencies (`pr#33959 <https://github.com/ceph/ceph/pull/33959>`_, Greg Farnum)
+* tests: qa: fix lingering ceph-mgr-ssh -> ceph-mgr-cephadm refs (`pr#32250 <https://github.com/ceph/ceph/pull/32250>`_, Sage Weil)
+* tests: qa: get rid of iterkeys for py3 compatibility (`pr#30873 <https://github.com/ceph/ceph/pull/30873>`_, Kyr Shatskyy)
+* tests: qa: kernel.sh: update for read-only changes (`pr#31773 <https://github.com/ceph/ceph/pull/31773>`_, Ilya Dryomov)
+* tests: qa: krbd_exclusive_option.sh: fixup for json.tool ordering change (`pr#32358 <https://github.com/ceph/ceph/pull/32358>`_, Ilya Dryomov)
+* tests: qa: krbd_exclusive_option.sh: update for recent kernel changes (`pr#32088 <https://github.com/ceph/ceph/pull/32088>`_, Ilya Dryomov)
+* tests: qa: rbd_workunit_suites_fsx: install build dependencies (`pr#33412 <https://github.com/ceph/ceph/pull/33412>`_, Ilya Dryomov)
+* tests: qa: run cephadm/smoke on opensuse 15.2 instead of 15.1 (`pr#33535 <https://github.com/ceph/ceph/pull/33535>`_, Nathan Cutler)
+* tests: qa: update krbd tests for python3 (`pr#31968 <https://github.com/ceph/ceph/pull/31968>`_, Ilya Dryomov)
+* tests: qa: update krbd_blkroset.t and add krbd_get_features.t (`pr#31771 <https://github.com/ceph/ceph/pull/31771>`_, Ilya Dryomov)
+* tests: qa: whitelist FS_DEGRADED (`pr#32549 <https://github.com/ceph/ceph/pull/32549>`_, Kefu Chai)
+* tests: remove spurious whitespace (`pr#33848 <https://github.com/ceph/ceph/pull/33848>`_, Milind Changire)
+* tests: Revert qa/tasks/cbt: include py2 deps on ubuntu for now (`pr#32512 <https://github.com/ceph/ceph/pull/32512>`_, Kefu Chai)
+* tests: script/run-cbt.sh: add support for ceph-osd testing (`pr#30811 <https://github.com/ceph/ceph/pull/30811>`_, Radoslaw Zarzynski)
+* tests: script/run-cbt.sh: always use python3 (`pr#30321 <https://github.com/ceph/ceph/pull/30321>`_, Kefu Chai)
+* tests: script/run-cbt.sh: check option correctly (`pr#30287 <https://github.com/ceph/ceph/pull/30287>`_, Kefu Chai)
+* tests: script/run-cbt.sh: set fs.aio-max-nr for seastar (`pr#31667 <https://github.com/ceph/ceph/pull/31667>`_, Kefu Chai)
+* tests: script/run_mypy: Support mypy 0.740 (`pr#31192 <https://github.com/ceph/ceph/pull/31192>`_, Sebastian Wagner)
+* tests: script/run_tox.sh: do not use python2 if we have python3 (`pr#31751 <https://github.com/ceph/ceph/pull/31751>`_, Kefu Chai)
+* tests: selinux: Update the policy for RHEL8 (`pr#28290 <https://github.com/ceph/ceph/pull/28290>`_, Boris Ranto)
+* tests: src/test, qa/suites/rados/thrash: add dedup test (`pr#28983 <https://github.com/ceph/ceph/pull/28983>`_, Myoungwon Oh)
+* tests: src/test/compressor: Add missing gtest (`pr#33731 <https://github.com/ceph/ceph/pull/33731>`_, Willem Jan Withagen)
+* tests: src/test: fix creating two different objects for testing chunked object (`issue#39282 <http://tracker.ceph.com/issues/39282>`_, `pr#27667 <https://github.com/ceph/ceph/pull/27667>`_, Myoungwon Oh)
+* tests: src/valgrind.supp: replace with the teuthologys file. Whitelist OpenSSL (`pr#27265 <https://github.com/ceph/ceph/pull/27265>`_, Radoslaw Zarzynski)
+* tests: tasks/ceph: drop testdir replacement in skeleton_config (`pr#30829 <https://github.com/ceph/ceph/pull/30829>`_, Kyr Shatskyy)
+* tests: tasks/ceph: get rid of iteritems for python3 (`pr#30792 <https://github.com/ceph/ceph/pull/30792>`_, Kyr Shatskyy)
+* tests: test/bench_log: add usage function (`pr#31723 <https://github.com/ceph/ceph/pull/31723>`_, Xuqiang Chen)
+* tests: test/bufferlist.cc: encode/decode int64_t instead of long (`pr#29881 <https://github.com/ceph/ceph/pull/29881>`_, Alexandre Oliva)
+* tests: test/cli/ceph-conf: fix test (`pr#28818 <https://github.com/ceph/ceph/pull/28818>`_, Kefu Chai)
+* tests: test/cli: Make the ceph-conf test more liberal (`pr#29405 <https://github.com/ceph/ceph/pull/29405>`_, Willem Jan Withagen)
+* tests: test/common/test_util: skip it if /etc/os-release does not exist (`pr#27927 <https://github.com/ceph/ceph/pull/27927>`_, Kefu Chai)
+* tests: test/crimson/: use 256M mem and 1 cpu core for each test (`pr#29152 <https://github.com/ceph/ceph/pull/29152>`_, Kefu Chai)
+* tests: test/crimson/perf_async_msgr: remove unsued header file (`pr#28707 <https://github.com/ceph/ceph/pull/28707>`_, Jianpeng Ma)
+* tests: test/crimson: add acceptable section to tests (`pr#30315 <https://github.com/ceph/ceph/pull/30315>`_, Kefu Chai)
+* tests: test/crimson: add unit-test for ceph::net::Socket (`pr#28623 <https://github.com/ceph/ceph/pull/28623>`_, Yingxin Cheng)
+* tests: test/crimson: cbt test does rand-reads instead of seq-reads (`pr#30794 <https://github.com/ceph/ceph/pull/30794>`_, Radoslaw Zarzynski)
+* tests: test/crimson: fix a compiler error (`pr#27883 <https://github.com/ceph/ceph/pull/27883>`_, Jianpeng Ma)
+* tests: test/crimson: fix build of unittest_seastar_monc (`pr#27515 <https://github.com/ceph/ceph/pull/27515>`_, Kefu Chai, Yingxin Cheng)
+* tests: test/crimson: fix FTBFS (`pr#28902 <https://github.com/ceph/ceph/pull/28902>`_, Kefu Chai)
+* tests: test/crimson: fix msgr test of ref counter racing (`issue#36405 <http://tracker.ceph.com/issues/36405>`_, `pr#28362 <https://github.com/ceph/ceph/pull/28362>`_, Yingxin Cheng)
+* tests: test/crimson: implement a remote async TestPeer for crimson msgr tests (`pr#31156 <https://github.com/ceph/ceph/pull/31156>`_, Yingxin Cheng)
+* tests: test/crimson: improved perf_crimson_msgr with timer and sampled lat (`pr#28542 <https://github.com/ceph/ceph/pull/28542>`_, Yingxin Cheng)
+* tests: test/crimson: include writes in perf_crimson/async_server (`pr#27429 <https://github.com/ceph/ceph/pull/27429>`_, Yingxin Cheng)
+* tests: test/crimson: lower the bar for cbt test (`pr#30458 <https://github.com/ceph/ceph/pull/30458>`_, Kefu Chai)
+* tests: test/crimson: remove unittest_seastar_socket temporarily (`pr#32720 <https://github.com/ceph/ceph/pull/32720>`_, Kefu Chai)
+* tests: test/crimson: update to accomodate Dispatcher changes (`pr#27093 <https://github.com/ceph/ceph/pull/27093>`_, Kefu Chai)
+* tests: test/crimson: v2 failover tests with ack/keepalive (`pr#30803 <https://github.com/ceph/ceph/pull/30803>`_, Yingxin Cheng)
+* tests: test/crimson: verify msgr v2 behavior with different policies (`pr#30925 <https://github.com/ceph/ceph/pull/30925>`_, Yingxin Cheng)
+* tests: test/erasure-code: add exception handling to k & m (`pr#30087 <https://github.com/ceph/ceph/pull/30087>`_, Hang Li)
+* tests: test/fio/fio_ceph_messenger: make exec multi client on the same host (`pr#28464 <https://github.com/ceph/ceph/pull/28464>`_, Jianpeng Ma)
+* tests: test/fio: fix a compiler error (`pr#27880 <https://github.com/ceph/ceph/pull/27880>`_, Jianpeng Ma)
+* tests: test/fio: introduce fio ioengine: fio_ceph_messenger (`pr#24678 <https://github.com/ceph/ceph/pull/24678>`_, Roman Penyaev)
+* tests: test/kv_store_bench: Fix double free error (`pr#32439 <https://github.com/ceph/ceph/pull/32439>`_, Xuqiang Chen, luo rixin)
+* tests: test/librados: avoid residual crush rule after test case execution (`issue#40970 <http://tracker.ceph.com/issues/40970>`_, `pr#29341 <https://github.com/ceph/ceph/pull/29341>`_, Bingyi Zhang)
+* tests: test/librados: free AioCompletion using AioCompletion::release() (`pr#30204 <https://github.com/ceph/ceph/pull/30204>`_, Kefu Chai)
+* tests: test/librados: use GTEST_SKIP() to skip test (`pr#32770 <https://github.com/ceph/ceph/pull/32770>`_, Kefu Chai)
+* tests: test/msgr: fix ComplexTest fail when using DPDK protocal stack (`pr#31910 <https://github.com/ceph/ceph/pull/31910>`_, Chunsong Feng)
+* tests: test/msgr: make ceph_perf_msgr_client/server work (`pr#28842 <https://github.com/ceph/ceph/pull/28842>`_, Jianpeng Ma)
+* tests: test/objectstore: silence -Wsign-compare warning (`pr#27750 <https://github.com/ceph/ceph/pull/27750>`_, Kefu Chai)
+* tests: test/old: remove stale tests (`pr#29124 <https://github.com/ceph/ceph/pull/29124>`_, Kefu Chai)
+* tests: test/pybind/test_ceph_argparse.py: pg_num of pool creation now optional (`pr#30535 <https://github.com/ceph/ceph/pull/30535>`_, xie xingguo)
+* tests: test/python: remove stale tests (`pr#29413 <https://github.com/ceph/ceph/pull/29413>`_, Kefu Chai)
+* tests: test/TestOSDScrub: fix mktime() error (`pr#33430 <https://github.com/ceph/ceph/pull/33430>`_, luo rixin)
+* tests: test/test_socket: fix dispatch_sockets() unexpected exception (`pr#33482 <https://github.com/ceph/ceph/pull/33482>`_, luo rixin)
+* tests: test/test_weighted_shuffle: enlarge epsilon (`pr#27181 <https://github.com/ceph/ceph/pull/27181>`_, Kefu Chai)
+* tests: test/unittest_bluefs: always remove temp bdev file (`pr#29676 <https://github.com/ceph/ceph/pull/29676>`_, Kefu Chai)
+* tests: test/venv: do not hardwire to py2.7 for tox tests (`pr#29761 <https://github.com/ceph/ceph/pull/29761>`_, Willem Jan Withagen)
+* tests: test: Add flush_pg_stats to avoid race with getting num_shards_repaired (`pr#33776 <https://github.com/ceph/ceph/pull/33776>`_, David Zafman)
+* tests: test: Add `#include <array>` (`pr#27455 <https://github.com/ceph/ceph/pull/27455>`_, Willem Jan Withagen)
+* tests: test: Allow fractional milliseconds to make test possible (`pr#30220 <https://github.com/ceph/ceph/pull/30220>`_, David Zafman)
+* tests: test: do not include unnecessary includes (`pr#30065 <https://github.com/ceph/ceph/pull/30065>`_, Kefu Chai)
+* tests: test: Do not test unicode if boost::spirit >= 1.72 (`pr#32388 <https://github.com/ceph/ceph/pull/32388>`_, Willem Jan Withagen)
+* tests: test: Expect being off by up to 2 and make sure all PGs are active+clean (`pr#33566 <https://github.com/ceph/ceph/pull/33566>`_, David Zafman)
+* tests: test: Fix failing ceph_objectstore_tool.py test (`pr#33593 <https://github.com/ceph/ceph/pull/33593>`_, David Zafman)
+* tests: test: Fix race with osd restart and doing a scru (`pr#32039 <https://github.com/ceph/ceph/pull/32039>`_, David Zafman)
+* tests: test: fix unused asserts variable in ceph_test_osd_stale_read.cc (`pr#32789 <https://github.com/ceph/ceph/pull/32789>`_, Radoslaw Zarzynski)
+* tests: test: Fix wait_for_state() to wait for a PG to get into a state (`pr#32628 <https://github.com/ceph/ceph/pull/32628>`_, David Zafman)
+* tests: test: Ignore OSD_SLOW_PING_TIME\\* if injecting socket failures (`pr#30714 <https://github.com/ceph/ceph/pull/30714>`_, David Zafman)
+* tests: test: move bluestore dependent code under WITH_BLUESTORE (`pr#31335 <https://github.com/ceph/ceph/pull/31335>`_, Willem Jan Withagen)
+* tests: test: remove Dockerfile for centos7 and add Dockerfile for centos8 (`pr#33452 <https://github.com/ceph/ceph/pull/33452>`_, Kefu Chai)
+* tests: test: remove useless ASSERT_XXX macros for rgw test (`pr#30062 <https://github.com/ceph/ceph/pull/30062>`_, Zhi Zhang)
+* tests: test: silence warning unused variable nvme (`pr#33650 <https://github.com/ceph/ceph/pull/33650>`_, Jos Collin)
+* tests: test: Update pg log test for new trimming behavior (`pr#32945 <https://github.com/ceph/ceph/pull/32945>`_, David Zafman)
+* tests: use python3 compatible print (`pr#30758 <https://github.com/ceph/ceph/pull/30758>`_, Kyr Shatskyy)
+* tests: vstart.sh: Make sure mkdir succeeds (`pr#30005 <https://github.com/ceph/ceph/pull/30005>`_, Willem Jan Withagen)
+* test_alien_echo: update to use crimson:: namespace (`pr#31135 <https://github.com/ceph/ceph/pull/31135>`_, Samuel Just)
+* test_cephadm.sh: pass --fsid to shell command (`pr#32389 <https://github.com/ceph/ceph/pull/32389>`_, Sage Weil)
+* test_cephadm: use container shell for ceph cmds (`pr#32627 <https://github.com/ceph/ceph/pull/32627>`_, Michael Fritch)
+* tools: add maxread in rados listomapkeys (`pr#30637 <https://github.com/ceph/ceph/pull/30637>`_, lvshuhua)
+* tools: adding ceph level immutable obj cache daemon (`pr#25545 <https://github.com/ceph/ceph/pull/25545>`_, Yuan Zhou, Dehao Shang)
+* tools: backport-create-issue: flush line before overprinting (`pr#31688 <https://github.com/ceph/ceph/pull/31688>`_, Nathan Cutler)
+* tools: backport-create-issue: read redmine key from file (`pr#31533 <https://github.com/ceph/ceph/pull/31533>`_, Tiago Melo)
+* tools: backport-create-issue: resolve parent if all backports resolved/rejected (`pr#30752 <https://github.com/ceph/ceph/pull/30752>`_, Nathan Cutler)
+* tools: backport-create-issue: resolve parent only if parent has backport issues (`pr#31753 <https://github.com/ceph/ceph/pull/31753>`_, Nathan Cutler)
+* tools: backport-resolve-issue: narrow regular expression and read key/token from files (`pr#31594 <https://github.com/ceph/ceph/pull/31594>`_, Nathan Cutler)
+* tools: backport-resolve-issue: populate tracker_description method (`pr#33105 <https://github.com/ceph/ceph/pull/33105>`_, Nathan Cutler)
+* tools: backport-resolve-issue: recognize that Target version is populated and prune duplicate URLs (`pr#31247 <https://github.com/ceph/ceph/pull/31247>`_, Nathan Cutler)
+* tools: backport-resolve-issue: resolve multiple backport issues (`pr#30988 <https://github.com/ceph/ceph/pull/30988>`_, Nathan Cutler)
+* tools: backport-resolve-issue: use Basic Authentication instead of access_token (`pr#33173 <https://github.com/ceph/ceph/pull/33173>`_, Nathan Cutler)
+* tools: build-integration-branch: dont fail on existing branch (`pr#33093 <https://github.com/ceph/ceph/pull/33093>`_, Sage Weil)
+* tools: build-integration-branch: take PRs in chronological order (`pr#31132 <https://github.com/ceph/ceph/pull/31132>`_, Nathan Cutler)
+* tools: ceph-backport.sh: allow user to specify --fork explicitly (`pr#31734 <https://github.com/ceph/ceph/pull/31734>`_, Nathan Cutler)
+* tools: ceph-backport.sh: automate setting of milestone and component label, implement --version option (`pr#30725 <https://github.com/ceph/ceph/pull/30725>`_, Nathan Cutler)
+* tools: ceph-backport.sh: cherry-pick individual commits (`pr#30097 <https://github.com/ceph/ceph/pull/30097>`_, Jan Fajerski)
+* tools: ceph-backport.sh: fix setup routine (`pr#33456 <https://github.com/ceph/ceph/pull/33456>`_, Nathan Cutler)
+* tools: ceph-backport.sh: guess component with --existing-pr (`pr#31419 <https://github.com/ceph/ceph/pull/31419>`_, Nathan Cutler)
+* tools: ceph-backport.sh: implement --milestones feature and more-careful vetting (`pr#30879 <https://github.com/ceph/ceph/pull/30879>`_, Nathan Cutler)
+* tools: ceph-backport.sh: implement interactive setup routine and new options (`pr#31366 <https://github.com/ceph/ceph/pull/31366>`_, Nathan Cutler)
+* tools: ceph-backport.sh: use Basic Authentication instead of access_token (`pr#33182 <https://github.com/ceph/ceph/pull/33182>`_, Nathan Cutler)
+* tools: ceph-conf: added --show-config-value to usage (`pr#29981 <https://github.com/ceph/ceph/pull/29981>`_, James McClune)
+* tools: ceph-crash: use open(..,r) to read bytes for Python3 (`issue#40781 <http://tracker.ceph.com/issues/40781>`_, `pr#29053 <https://github.com/ceph/ceph/pull/29053>`_, Dan Mick)
+* tools: ceph-daemon: ExecStart=/bin/bash script (`pr#31319 <https://github.com/ceph/ceph/pull/31319>`_, Sage Weil)
+* tools: ceph-daemon: fix typo in the output_pub_ssh_key argument (`pr#31337 <https://github.com/ceph/ceph/pull/31337>`_, John McGowan)
+* tools: ceph-daemon: Fix `ls` cmd for legacy confs (`pr#31329 <https://github.com/ceph/ceph/pull/31329>`_, Michael Fritch)
+* tools: ceph-monstore-tool: print out caps when rebuilding monstore (`pr#27340 <https://github.com/ceph/ceph/pull/27340>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: return 0 if incmap is sane (`pr#29704 <https://github.com/ceph/ceph/pull/29704>`_, Kefu Chai)
+* tools: ceph-objectstore-tool: update-mon-db: do not fail if incmap is missing (`pr#29571 <https://github.com/ceph/ceph/pull/29571>`_, Kefu Chai)
+* tools: ceph.in: fix verbose print (`pr#29486 <https://github.com/ceph/ceph/pull/29486>`_, luo.runbing)
+* tools: cls: add timeindex types to ceph-dencoder (`pr#27780 <https://github.com/ceph/ceph/pull/27780>`_, Abhishek Lekshmanan)
+* tools: github/codeowners: add ceph-volume (`pr#31883 <https://github.com/ceph/ceph/pull/31883>`_, Jan Fajerski)
+* tools: github: Add CODEOWNERs for designated code-owner reviews (`pr#29451 <https://github.com/ceph/ceph/pull/29451>`_, Ernesto Puerta)
+* tools: no-mon-config switch for ceph-objectstore-tool (`pr#26717 <https://github.com/ceph/ceph/pull/26717>`_, Igor Fedotov)
+* tools: pin the version of breathe that works with Python2 (`pr#27721 <https://github.com/ceph/ceph/pull/27721>`_, Alfredo Deza)
+* tools: script/backport-create-issue: add --resolve-parent feature (`pr#29904 <https://github.com/ceph/ceph/pull/29904>`_, Nathan Cutler)
+* tools: script/backport-create-issue: handle long Redmine issue names (`pr#27887 <https://github.com/ceph/ceph/pull/27887>`_, Nathan Cutler)
+* tools: script/backport-resolve-issue: better error message (`pr#30187 <https://github.com/ceph/ceph/pull/30187>`_, Nathan Cutler)
+* tools: script/backport-resolve-issue: handle tracker URLs better (`pr#29950 <https://github.com/ceph/ceph/pull/29950>`_, Nathan Cutler)
+* tools: script/ceph-backport-sh: add access_token parameter to all ghub api c\xe2\x80\xa6 (`pr#29261 <https://github.com/ceph/ceph/pull/29261>`_, Jan Fajerski)
+* tools: script/ceph-backport.sh: Add prepare function (`pr#28446 <https://github.com/ceph/ceph/pull/28446>`_, Tiago Melo)
+* tools: script/ceph-backport.sh: Allow to set component label (`pr#29318 <https://github.com/ceph/ceph/pull/29318>`_, Tiago Melo)
+* tools: script/ceph-backport.sh: allow user to specify remote repo (`pr#27233 <https://github.com/ceph/ceph/pull/27233>`_, Kefu Chai)
+* tools: script/ceph-backport.sh: carry https through to logical conclusion (`pr#29743 <https://github.com/ceph/ceph/pull/29743>`_, Nathan Cutler)
+* tools: script/ceph-backport.sh: Fix verification of git repository (`pr#30398 <https://github.com/ceph/ceph/pull/30398>`_, Tiago Melo)
+* tools: script/ceph-backport.sh: make the script idempotent (`pr#30106 <https://github.com/ceph/ceph/pull/30106>`_, Nathan Cutler)
+* tools: script/ceph-backport.sh: Use secure access for tracker.ceph.com (`pr#29438 <https://github.com/ceph/ceph/pull/29438>`_, Willem Jan Withagen)
+* tools: script/ceph-backport.sh: wholesale refactor (`pr#29957 <https://github.com/ceph/ceph/pull/29957>`_, Nathan Cutler)
+* tools: script/ceph-release-notes: alternate merge commit format (`pr#27281 <https://github.com/ceph/ceph/pull/27281>`_, Nathan Cutler)
+* tools: script/ptl-tool: update for python3 (`pr#29095 <https://github.com/ceph/ceph/pull/29095>`_, Patrick Donnelly)
+* tools: script/run_mypy: Sort groups (`pr#28225 <https://github.com/ceph/ceph/pull/28225>`_, Sebastian Wagner)
+* tools: script/run_tox.sh: remove unused code (`pr#30386 <https://github.com/ceph/ceph/pull/30386>`_, Kefu Chai)
+* tools: script/sepia_bt.sh: remove stale script (`pr#29129 <https://github.com/ceph/ceph/pull/29129>`_, Kefu Chai)
+* tools: script: add backport-resolve-issue (`pr#29797 <https://github.com/ceph/ceph/pull/29797>`_, Nathan Cutler)
+* tools: script: enable nautilus in backport scripts (`pr#26973 <https://github.com/ceph/ceph/pull/26973>`_, Nathan Cutler)
+* tools: script: Obtain milestones via github API (`pr#27221 <https://github.com/ceph/ceph/pull/27221>`_, Lenz Grimmer)
+* tools: script: raw_input was renamed to input in py3 (`pr#30346 <https://github.com/ceph/ceph/pull/30346>`_, Patrick Donnelly)
+* tools: scripts/kubejacker: Fix mgr_plugins target for centos (`pr#28078 <https://github.com/ceph/ceph/pull/28078>`_, Sebastian Wagner)
+* tools: scripts/run_mypy: add .gitignore (`pr#27118 <https://github.com/ceph/ceph/pull/27118>`_, Sebastian Wagner)
+* tools: scripts: use https url for redmine (`pr#29536 <https://github.com/ceph/ceph/pull/29536>`_, Patrick Donnelly)
+* tools: src/script/backport-create-issue: implement --force option (`pr#30571 <https://github.com/ceph/ceph/pull/30571>`_, Nathan Cutler)
+* tools: src/script/check_commands.sh: fix grep regex class range (`pr#29161 <https://github.com/ceph/ceph/pull/29161>`_, Valentin Bajrami)
+* tools: src/script/unhexdump-C: script to reverse a hexdump -C style hexdump (`pr#29098 <https://github.com/ceph/ceph/pull/29098>`_, Sage Weil)
+* tools: stop.sh: use bash shell to solve syntax error (`pr#32263 <https://github.com/ceph/ceph/pull/32263>`_, luo rixin)
+* tools: tool/ceph-conf: s/global_pre_init()/global_init()/ (`issue#7849 <http://tracker.ceph.com/issues/7849>`_, `pr#29058 <https://github.com/ceph/ceph/pull/29058>`_, Kefu Chai)
+* tools: tool: ceph_monstore_tool: --readable=0 => --readable (`pr#32265 <https://github.com/ceph/ceph/pull/32265>`_, simon gao)
+* tools: tools/ceph-kvstore-tool: print db stats (`pr#27162 <https://github.com/ceph/ceph/pull/27162>`_, Igor Fedotov)
+* tools: tools/osdmaptool.cc: do not use deprecated std::random_shuffle() (`pr#31990 <https://github.com/ceph/ceph/pull/31990>`_, Kefu Chai)
+* tools: tools/rados: update advisory lock break usage with --lock-cookie required (`pr#31348 <https://github.com/ceph/ceph/pull/31348>`_, Zhi Zhang)
+* tools: vstart.sh: fix CEPH_PORT check and cleanups (`pr#26782 <https://github.com/ceph/ceph/pull/26782>`_, Changcheng Liu, Kefu Chai)
+* tools: vstart: add --inc-osd option (`pr#30512 <https://github.com/ceph/ceph/pull/30512>`_, xie xingguo)
+* tools: vstart: add new option to pass list of block devices to bluestore (`pr#27518 <https://github.com/ceph/ceph/pull/27518>`_, Jeff Layton)
+* tools: vstart: fix error when getting CMake variables with the same prefix (`pr#31962 <https://github.com/ceph/ceph/pull/31962>`_, Kiefer Chang)
+* tools: vstart: fix run() invocation for rgw (`pr#28386 <https://github.com/ceph/ceph/pull/28386>`_, Casey Bodley)
+* Update grafana dashboards (`issue#39652 <http://tracker.ceph.com/issues/39652>`_, `pr#28043 <https://github.com/ceph/ceph/pull/28043>`_, Jan Fajerski)
+* vstart.sh: add an option to use crimson-osd (`pr#27108 <https://github.com/ceph/ceph/pull/27108>`_, chunmei Liu, Kefu Chai)
+* vstart.sh: correct ceph-run path (`pr#27968 <https://github.com/ceph/ceph/pull/27968>`_, Changcheng Liu)
+* vstart.sh: fix install of cephadm ssh keys from ~/.ssh (`pr#33647 <https://github.com/ceph/ceph/pull/33647>`_, Sage Weil)
+* vstart.sh: Fix problem that all extra_conf got merged into single line (`pr#28586 <https://github.com/ceph/ceph/pull/28586>`_, Adam Kupczyk)
+* vstart.sh: move extra_seastar_args up in vstart.sh (`pr#32366 <https://github.com/ceph/ceph/pull/32366>`_, Chunmei Liu)
+* vstart.sh: unify the indent (`pr#27995 <https://github.com/ceph/ceph/pull/27995>`_, Kefu Chai, Richael Zhuang)
+* vstart_runner: split unicode arguments into lists (`pr#28561 <https://github.com/ceph/ceph/pull/28561>`_, Rishabh Dave)
+
+.. _announcement: https://lists.fedoraproject.org/archives/list/epel-announce@lists.fedoraproject.org/message/EGUMKAIMPK2UD5VSHXM53BH2MBDGDWMO/
+.. _Bucket Notifications: ../../radosgw/notifications
+.. _HashiCorp Vault Integration: ../../radosgw/vault
+.. _Multisite Sync Policy: ../../radosgw/multisite-sync-policy
diff --git a/doc/releases/pacific.rst b/doc/releases/pacific.rst
new file mode 100644
index 000000000..c360ef87d
--- /dev/null
+++ b/doc/releases/pacific.rst
@@ -0,0 +1,2381 @@
+=======
+Pacific
+=======
+
+v16.2.11 Pacific
+================
+
+This is the eleventh backport release in the Pacific series.
+
+Notable Changes
+---------------
+
+* Cephfs: The 'AT_NO_ATTR_SYNC' macro is deprecated, please use the standard
+  'AT_STATX_DONT_SYNC' macro. The 'AT_NO_ATTR_SYNC' macro will be removed in
+  the future.
+
+* Trimming of PGLog dups is now controlled by the size instead of the version.
+  This fixes the PGLog inflation issue that was happening when the on-line
+  (in OSD) trimming got jammed after a PG split operation. Also, a new off-line
+  mechanism has been added: `ceph-objectstore-tool` got `trim-pg-log-dups` op
+  that targets situations where OSD is unable to boot due to those inflated dups.
+  If that is the case, in OSD logs the "You can be hit by THE DUPS BUG" warning
+  will be visible.
+  Relevant tracker: https://tracker.ceph.com/issues/53729
+
+* RBD: `rbd device unmap` command gained `--namespace` option.  Support for
+  namespaces was added to RBD in Nautilus 14.2.0 and it has been possible to
+  map and unmap images in namespaces using the `image-spec` syntax since then
+  but the corresponding option available in most other commands was missing.
+
+Changelog
+---------
+
+* .github/CODEOWNERS: tag core devs on core PRs (`pr#46520 <https://github.com/ceph/ceph/pull/46520>`_, Neha Ojha)
+* .github: continue on error and reorder milestone step (`pr#46448 <https://github.com/ceph/ceph/pull/46448>`_, Ernesto Puerta)
+* .readthedocs.yml: Always build latest doc/releases pages (`pr#47443 <https://github.com/ceph/ceph/pull/47443>`_, David Galloway)
+* mgr/alerts: Add Message-Id and Date header to sent emails (`pr#46312 <https://github.com/ceph/ceph/pull/46312>`_, Lorenz Bausch)
+* Add mapping for ernno:13 and adding path in error msg in opendir()/cephfs.pyx (`pr#46646 <https://github.com/ceph/ceph/pull/46646>`_, Sarthak0702)
+* backport of cephadm: fix osd adoption with custom cluster name (`pr#46552 <https://github.com/ceph/ceph/pull/46552>`_, Adam King)
+* bluestore: Improve deferred write decision (`pr#49170 <https://github.com/ceph/ceph/pull/49170>`_, Adam Kupczyk, Igor Fedotov)
+* Catch exception if thrown by __generate_command_map() (`pr#45893 <https://github.com/ceph/ceph/pull/45893>`_, Nikhil Kshirsagar)
+* ceph-fuse: add dedicated snap stag map for each directory (`pr#46949 <https://github.com/ceph/ceph/pull/46949>`_, Xiubo Li)
+* ceph-mixin: backport of recent cleanups (`pr#46549 <https://github.com/ceph/ceph/pull/46549>`_, Arthur Outhenin-Chalandre)
+* ceph mixin: backports (`pr#47868 <https://github.com/ceph/ceph/pull/47868>`_, Aswin Toni, Kefu Chai, Anthony D'Atri)
+* ceph-volume/tests: fix lvm centos8-filestore-create job (`pr#48123 <https://github.com/ceph/ceph/pull/48123>`_, Guillaume Abrioux)
+* ceph-volume: add a retry in util.disk.remove_partition (`pr#47990 <https://github.com/ceph/ceph/pull/47990>`_, Guillaume Abrioux)
+* ceph-volume: allow listing devices by OSD ID (`pr#47018 <https://github.com/ceph/ceph/pull/47018>`_, Rishabh Dave)
+* ceph-volume: avoid unnecessary subprocess calls (`pr#46969 <https://github.com/ceph/ceph/pull/46969>`_, Guillaume Abrioux)
+* ceph-volume: decrease number of `pvs` calls in `lvm list` (`pr#46967 <https://github.com/ceph/ceph/pull/46967>`_, Guillaume Abrioux)
+* ceph-volume: do not log sensitive details (`pr#46729 <https://github.com/ceph/ceph/pull/46729>`_, Guillaume Abrioux)
+* ceph-volume: fix activate (`pr#46511 <https://github.com/ceph/ceph/pull/46511>`_, Guillaume Abrioux, Sage Weil)
+* ceph-volume: fix inventory with device arg (`pr#48126 <https://github.com/ceph/ceph/pull/48126>`_, Guillaume Abrioux)
+* ceph-volume: make is_valid() optional (`pr#46731 <https://github.com/ceph/ceph/pull/46731>`_, Guillaume Abrioux)
+* ceph-volume: only warn when config file isn't found (`pr#46069 <https://github.com/ceph/ceph/pull/46069>`_, Guillaume Abrioux)
+* ceph-volume: Pacific backports (`pr#47413 <https://github.com/ceph/ceph/pull/47413>`_, Guillaume Abrioux, Zack Cerza, Arthur Outhenin-Chalandre)
+* ceph-volume: system.get_mounts() refactor (`pr#47535 <https://github.com/ceph/ceph/pull/47535>`_, Guillaume Abrioux)
+* ceph-volume: zap osds in rollback_osd() (`pr#44769 <https://github.com/ceph/ceph/pull/44769>`_, Guillaume Abrioux)
+* ceph.spec.in: disable annobin plugin if compile with gcc-toolset (`pr#46368 <https://github.com/ceph/ceph/pull/46368>`_, Kefu Chai)
+* ceph.spec.in: remove build directory at end of %install (`pr#45698 <https://github.com/ceph/ceph/pull/45698>`_, Tim Serong)
+* ceph_test_librados_service: wait longer for servicemap to update (`pr#46677 <https://github.com/ceph/ceph/pull/46677>`_, Sage Weil)
+* cephadm batch backport May (`pr#46327 <https://github.com/ceph/ceph/pull/46327>`_, Adam King, Redouane Kachach, Moritz Röhrich)
+* cephadm/ceph-volume: fix rm-cluster --zap (`pr#47627 <https://github.com/ceph/ceph/pull/47627>`_, Guillaume Abrioux)
+* cephadm: add "su root root" to cephadm.log logrotate config (`pr#47319 <https://github.com/ceph/ceph/pull/47319>`_, Adam King)
+* cephadm: add 'is_paused' field in orch status output (`pr#46570 <https://github.com/ceph/ceph/pull/46570>`_, Guillaume Abrioux)
+* cephadm: add `ip_nonlocal_bind` to haproxy deployment (`pr#48212 <https://github.com/ceph/ceph/pull/48212>`_, Michael Fritch)
+* Cephadm: Allow multiple virtual IP addresses for keepalived and haproxy (`pr#47611 <https://github.com/ceph/ceph/pull/47611>`_, Luis Domingues)
+* cephadm: consider stdout to get container version (`pr#48210 <https://github.com/ceph/ceph/pull/48210>`_, Tatjana Dehler)
+* cephadm: Fix disk size calculation (`pr#48098 <https://github.com/ceph/ceph/pull/48098>`_, Paul Cuzner)
+* cephadm: Fix repo_gpgkey should return 2 vars (`pr#47376 <https://github.com/ceph/ceph/pull/47376>`_, Laurent Barbe)
+* cephadm: improve network handling during bootstrap (`pr#46309 <https://github.com/ceph/ceph/pull/46309>`_, Redouane Kachach)
+* cephadm: pin flake8 to 5.0.4 (`pr#49058 <https://github.com/ceph/ceph/pull/49058>`_, Kefu Chai)
+* cephadm: preserve cephadm user during RPM upgrade (`pr#46553 <https://github.com/ceph/ceph/pull/46553>`_, Scott Shambarger)
+* cephadm: prometheus: The generatorURL in alerts is only using hostname (`pr#46352 <https://github.com/ceph/ceph/pull/46352>`_, Volker Theile)
+* cephadm: return nonzero exit code when applying spec fails in bootstrap (`pr#48102 <https://github.com/ceph/ceph/pull/48102>`_, Adam King)
+* cephadm: run tests as root (`pr#48470 <https://github.com/ceph/ceph/pull/48470>`_, Kefu Chai)
+* cephadm: support for Oracle Linux 8 (`pr#47661 <https://github.com/ceph/ceph/pull/47661>`_, Adam King)
+* cephadm: support quotes around public/cluster network in config passed to bootstrap (`pr#47664 <https://github.com/ceph/ceph/pull/47664>`_, Adam King)
+* cephfs-data-scan: make scan_links more verbose (`pr#48443 <https://github.com/ceph/ceph/pull/48443>`_, Mykola Golub)
+* cephfs-shell: fix put and get cmd (`pr#46297 <https://github.com/ceph/ceph/pull/46297>`_, Dhairya Parmar, dparmar18)
+* cephfs-shell: move source to separate subdirectory (`pr#47401 <https://github.com/ceph/ceph/pull/47401>`_, Tim Serong)
+* cephfs-top: adding filesystem menu option (`pr#47998 <https://github.com/ceph/ceph/pull/47998>`_, Neeraj Pratap Singh)
+* cephfs-top: display average read/write/metadata latency (`issue#48619 <http://tracker.ceph.com/issues/48619>`_, `pr#47978 <https://github.com/ceph/ceph/pull/47978>`_, Venky Shankar)
+* cephfs-top: fix the rsp/wsp display (`pr#47647 <https://github.com/ceph/ceph/pull/47647>`_, Jos Collin)
+* cephfs-top: make cephfs-top display scrollable (`pr#48734 <https://github.com/ceph/ceph/pull/48734>`_, Jos Collin)
+* cephfs-top: Multiple filesystem support (`pr#46146 <https://github.com/ceph/ceph/pull/46146>`_, Neeraj Pratap Singh)
+* client: always return ESTALE directly in handle_reply (`pr#46557 <https://github.com/ceph/ceph/pull/46557>`_, Xiubo Li)
+* client: stop forwarding the request when exceeding 256 times (`pr#46179 <https://github.com/ceph/ceph/pull/46179>`_, Xiubo Li)
+* client: switch AT_NO_ATTR_SYNC to AT_STATX_DONT_SYNC (`pr#46679 <https://github.com/ceph/ceph/pull/46679>`_, Xiubo Li)
+* client/fuse: Fix directory DACs overriding for root (`pr#46596 <https://github.com/ceph/ceph/pull/46596>`_, Kotresh HR)
+* client: abort the client if we couldn't invalidate dentry caches (`pr#48109 <https://github.com/ceph/ceph/pull/48109>`_, Xiubo Li)
+* client: add option to disable collecting and sending metrics (`pr#46798 <https://github.com/ceph/ceph/pull/46798>`_, Xiubo Li)
+* client: allow overwrites to file with size greater than the max_file_size (`pr#47972 <https://github.com/ceph/ceph/pull/47972>`_, Tamar Shacked)
+* client: buffer the truncate if we have the Fx caps (`pr#45792 <https://github.com/ceph/ceph/pull/45792>`_, Xiubo Li)
+* client: choose auth MDS for getxattr with the Xs caps (`pr#46799 <https://github.com/ceph/ceph/pull/46799>`_, Xiubo Li)
+* client: do not uninline data for read (`pr#48133 <https://github.com/ceph/ceph/pull/48133>`_, Xiubo Li)
+* client: fix incorrectly showing the .snap size for stat (`pr#48413 <https://github.com/ceph/ceph/pull/48413>`_, Xiubo Li)
+* client: Inode::hold_caps_until is time from monotonic clock now (`pr#46626 <https://github.com/ceph/ceph/pull/46626>`_, Laura Flores, Neeraj Pratap Singh)
+* client: stop the remount_finisher thread in the Client::unmount() (`pr#48108 <https://github.com/ceph/ceph/pull/48108>`_, Xiubo Li)
+* client: use parent directory POSIX ACLs for snapshot dir (`issue#57084 <http://tracker.ceph.com/issues/57084>`_, `pr#48553 <https://github.com/ceph/ceph/pull/48553>`_, Venky Shankar)
+* cls/rbd: update last_read in group::snap_list (`pr#49195 <https://github.com/ceph/ceph/pull/49195>`_, Ilya Dryomov, Prasanna Kumar Kalever)
+* cls/rgw: rgw_dir_suggest_changes detects race with completion (`pr#45900 <https://github.com/ceph/ceph/pull/45900>`_, Casey Bodley)
+* cmake: check for python(\d)\.(\d+) when building boost (`pr#46365 <https://github.com/ceph/ceph/pull/46365>`_, Kefu Chai)
+* cmake: remove spaces in macro used for compiling cython code (`pr#47484 <https://github.com/ceph/ceph/pull/47484>`_, Kefu Chai)
+* CODEOWNERS: add RBD team (`pr#46541 <https://github.com/ceph/ceph/pull/46541>`_, Ilya Dryomov)
+* common: use boost::shared_mutex on Windows (`pr#47492 <https://github.com/ceph/ceph/pull/47492>`_, Lucian Petrut)
+* crash: pthread_mutex_lock() (`pr#47684 <https://github.com/ceph/ceph/pull/47684>`_, Patrick Donnelly)
+* doc/cephadm: add prompts to host-management.rst (`pr#48590 <https://github.com/ceph/ceph/pull/48590>`_, Zac Dover)
+* doc/rados: add prompts to placement-groups.rst (`pr#49272 <https://github.com/ceph/ceph/pull/49272>`_, Zac Dover)
+* doc: Wip pr 46109 backport to pacific (`pr#46117 <https://github.com/ceph/ceph/pull/46117>`_, Ville Ojamo)
+* doc: Wip min hardware typo pacific backport 2022 05 19 (`pr#46347 <https://github.com/ceph/ceph/pull/46347>`_, Zac Dover)
+* doc/_static: add scroll-margin-top to custom.css (`pr#49645 <https://github.com/ceph/ceph/pull/49645>`_, Zac Dover)
+* doc/architecture: correct PDF link (`pr#48796 <https://github.com/ceph/ceph/pull/48796>`_, Zac Dover)
+* doc/ceph-volume: add A. D'Atri's suggestions (`pr#48646 <https://github.com/ceph/ceph/pull/48646>`_, Zac Dover)
+* doc/ceph-volume: improve prepare.rst (`pr#48669 <https://github.com/ceph/ceph/pull/48669>`_, Zac Dover)
+* doc/ceph-volume: refine "bluestore" section (`pr#48635 <https://github.com/ceph/ceph/pull/48635>`_, Zac Dover)
+* doc/ceph-volume: refine "filestore" section (`pr#48637 <https://github.com/ceph/ceph/pull/48637>`_, Zac Dover)
+* doc/ceph-volume: refine "prepare" top matter (`pr#48652 <https://github.com/ceph/ceph/pull/48652>`_, Zac Dover)
+* doc/ceph-volume: refine Filestore docs (`pr#48671 <https://github.com/ceph/ceph/pull/48671>`_, Zac Dover)
+* doc/cephadm/services: fix example for specifying rgw placement (`pr#47948 <https://github.com/ceph/ceph/pull/47948>`_, Redouane Kachach)
+* doc/cephadm/services: the config section of service specs (`pr#47321 <https://github.com/ceph/ceph/pull/47321>`_, Redouane Kachach)
+* doc/cephadm: add airgapped install procedure (`pr#49146 <https://github.com/ceph/ceph/pull/49146>`_, Zac Dover)
+* doc/cephadm: add note about OSDs being recreated to OSD removal section (`pr#47103 <https://github.com/ceph/ceph/pull/47103>`_, Adam King)
+* doc/cephadm: Add post-upgrade section (`pr#46977 <https://github.com/ceph/ceph/pull/46977>`_, Redouane Kachach)
+* doc/cephadm: alphabetize external tools list (`pr#48726 <https://github.com/ceph/ceph/pull/48726>`_, Zac Dover)
+* doc/cephadm: arrange "listing hosts" section (`pr#48724 <https://github.com/ceph/ceph/pull/48724>`_, Zac Dover)
+* doc/cephadm: clean colons in host-management.rst (`pr#48604 <https://github.com/ceph/ceph/pull/48604>`_, Zac Dover)
+* doc/cephadm: correct version staggered upgrade got in pacific (`pr#48056 <https://github.com/ceph/ceph/pull/48056>`_, Adam King)
+* doc/cephadm: document recommended syntax for mounting files with ECA (`pr#48069 <https://github.com/ceph/ceph/pull/48069>`_, Adam King)
+* doc/cephadm: enhancing daemon operations documentation (`pr#46976 <https://github.com/ceph/ceph/pull/46976>`_, Redouane Kachach)
+* doc/cephadm: fix example for specifying networks for rgw (`pr#47807 <https://github.com/ceph/ceph/pull/47807>`_, Adam King)
+* doc/cephadm: fix grammar in compatibility.rst (`pr#48715 <https://github.com/ceph/ceph/pull/48715>`_, Zac Dover)
+* doc/cephadm: format airgap install procedure (`pr#49149 <https://github.com/ceph/ceph/pull/49149>`_, Zac Dover)
+* doc/cephadm: improve airgapping procedure grammar (`pr#49158 <https://github.com/ceph/ceph/pull/49158>`_, Zac Dover)
+* doc/cephadm: improve front matter (`pr#48607 <https://github.com/ceph/ceph/pull/48607>`_, Zac Dover)
+* doc/cephadm: improve grammar in "listing hosts" (`pr#49165 <https://github.com/ceph/ceph/pull/49165>`_, Zac Dover)
+* doc/cephadm: improve lone sentence (`pr#48738 <https://github.com/ceph/ceph/pull/48738>`_, Zac Dover)
+* doc/cephadm: refine "Removing Hosts" (`pr#49707 <https://github.com/ceph/ceph/pull/49707>`_, Zac Dover)
+* doc/cephadm: s/osd/OSD/ where appropriate (`pr#49718 <https://github.com/ceph/ceph/pull/49718>`_, Zac Dover)
+* doc/cephadm: s/ssh/SSH/ in doc/cephadm (complete) (`pr#48612 <https://github.com/ceph/ceph/pull/48612>`_, Zac Dover)
+* doc/cephadm: s/ssh/SSH/ in troubleshooting.rst (`pr#48602 <https://github.com/ceph/ceph/pull/48602>`_, Zac Dover)
+* doc/cephadm: update install.rst (`pr#48595 <https://github.com/ceph/ceph/pull/48595>`_, Zac Dover)
+* doc/cephfs - s/yet to here/yet to hear/ posix.rst (`pr#49449 <https://github.com/ceph/ceph/pull/49449>`_, Zac Dover)
+* doc/cephfs/add-remove-mds: added cephadm note, refined "Adding an MDS" (`pr#45878 <https://github.com/ceph/ceph/pull/45878>`_, Dhairya Parmar)
+* doc/cephfs: fix "e.g." in posix.rst (`pr#49451 <https://github.com/ceph/ceph/pull/49451>`_, Zac Dover)
+* doc/cephfs: s/all of there are/all of these are/ (`pr#49447 <https://github.com/ceph/ceph/pull/49447>`_, Zac Dover)
+* doc/conf.py: run ditaa with java (`pr#48906 <https://github.com/ceph/ceph/pull/48906>`_, Kefu Chai)
+* doc/css: add "span" padding to custom.css (`pr#49694 <https://github.com/ceph/ceph/pull/49694>`_, Zac Dover)
+* doc/css: add scroll-margin-top to dt elements (`pr#49640 <https://github.com/ceph/ceph/pull/49640>`_, Zac Dover)
+* doc/css: Add scroll-margin-top to h2 html element (`pr#49662 <https://github.com/ceph/ceph/pull/49662>`_, Zac Dover)
+* doc/css: add top-bar padding for h3 html element (`pr#49702 <https://github.com/ceph/ceph/pull/49702>`_, Zac Dover)
+* doc/dev/cephadm: fix host maintenance enter/exit syntax (`pr#49647 <https://github.com/ceph/ceph/pull/49647>`_, Ranjini Mandyam Narasiodeyar)
+* doc/dev/developer_guide/tests-unit-tests: Add unit test caveat (`pr#49013 <https://github.com/ceph/ceph/pull/49013>`_, Matan Breizman)
+* doc/dev: add context note to dev guide config (`pr#46817 <https://github.com/ceph/ceph/pull/46817>`_, Zac Dover)
+* doc/dev: add Dependabot section to essentials.rst (`pr#47043 <https://github.com/ceph/ceph/pull/47043>`_, Zac Dover)
+* doc/dev: add explanation of how to use deduplication (`pr#48568 <https://github.com/ceph/ceph/pull/48568>`_, Myoungwon Oh)
+* doc/dev: add IRC registration instructions (`pr#46939 <https://github.com/ceph/ceph/pull/46939>`_, Zac Dover)
+* doc/dev: add submodule-update link to dev guide (`pr#48480 <https://github.com/ceph/ceph/pull/48480>`_, Zac Dover)
+* doc/dev: alphabetize EC glossary (`pr#48686 <https://github.com/ceph/ceph/pull/48686>`_, Zac Dover)
+* doc/dev: edit delayed-delete.rst (`pr#47050 <https://github.com/ceph/ceph/pull/47050>`_, Zac Dover)
+* doc/dev: Elaborate on boost .deb creation (`pr#47416 <https://github.com/ceph/ceph/pull/47416>`_, David Galloway)
+* doc/dev: fix graphviz diagram (`pr#48923 <https://github.com/ceph/ceph/pull/48923>`_, Zac Dover)
+* doc/dev: improve Basic Workflow wording (`pr#49078 <https://github.com/ceph/ceph/pull/49078>`_, Zac Dover)
+* doc/dev: improve EC glossary (`pr#48676 <https://github.com/ceph/ceph/pull/48676>`_, Zac Dover)
+* doc/dev: improve lone sentence (`pr#48741 <https://github.com/ceph/ceph/pull/48741>`_, Zac Dover)
+* doc/dev: improve presentation of note (git remote) (`pr#48236 <https://github.com/ceph/ceph/pull/48236>`_, Zac Dover)
+* doc/dev: link to Dot User's Manual (`pr#48926 <https://github.com/ceph/ceph/pull/48926>`_, Zac Dover)
+* doc/dev: refine erasure_coding.rst (`pr#48701 <https://github.com/ceph/ceph/pull/48701>`_, Zac Dover)
+* doc/dev: remove deduplication.rst from pacific (`pr#48571 <https://github.com/ceph/ceph/pull/48571>`_, Zac Dover)
+* doc/dev: s/github/GitHub/ in essentials.rst (`pr#47049 <https://github.com/ceph/ceph/pull/47049>`_, Zac Dover)
+* doc/dev: s/master/main/ essentials.rst dev guide (`pr#46662 <https://github.com/ceph/ceph/pull/46662>`_, Zac Dover)
+* doc/dev: s/master/main/ in basic workflow (`pr#46704 <https://github.com/ceph/ceph/pull/46704>`_, Zac Dover)
+* doc/dev: s/master/main/ in title (`pr#46722 <https://github.com/ceph/ceph/pull/46722>`_, Zac Dover)
+* doc/dev: s/the the/the/ in basic-workflow.rst (`pr#46934 <https://github.com/ceph/ceph/pull/46934>`_, Zac Dover)
+* doc/dev: update basic-workflow.rst (`pr#46288 <https://github.com/ceph/ceph/pull/46288>`_, Zac Dover)
+* doc/dev_guide: s/master/main in merging.rst (`pr#46710 <https://github.com/ceph/ceph/pull/46710>`_, Zac Dover)
+* doc/glosary.rst: add "Ceph Block Device" term (`pr#48745 <https://github.com/ceph/ceph/pull/48745>`_, Zac Dover)
+* doc/glossary - add "secrets" (`pr#49398 <https://github.com/ceph/ceph/pull/49398>`_, Zac Dover)
+* doc/glossary.rst: add "Ceph Dashboard" term (`pr#48749 <https://github.com/ceph/ceph/pull/48749>`_, Zac Dover)
+* doc/glossary.rst: alphabetize glossary terms (`pr#48339 <https://github.com/ceph/ceph/pull/48339>`_, Zac Dover)
+* doc/glossary.rst: define "Ceph Manager" (`pr#48765 <https://github.com/ceph/ceph/pull/48765>`_, Zac Dover)
+* doc/glossary.rst: remove duplicates (`pr#48358 <https://github.com/ceph/ceph/pull/48358>`_, Zac Dover)
+* doc/glossary.rst: remove old front matter (`pr#48755 <https://github.com/ceph/ceph/pull/48755>`_, Zac Dover)
+* doc/glossary: add "BlueStore" (`pr#48778 <https://github.com/ceph/ceph/pull/48778>`_, Zac Dover)
+* doc/glossary: add "ceph monitor" entry (`pr#48448 <https://github.com/ceph/ceph/pull/48448>`_, Zac Dover)
+* doc/glossary: add "Ceph Object Store" (`pr#49031 <https://github.com/ceph/ceph/pull/49031>`_, Zac Dover)
+* doc/glossary: add "Dashboard Module" (`pr#49138 <https://github.com/ceph/ceph/pull/49138>`_, Zac Dover)
+* doc/glossary: add "FQDN" entry (`pr#49425 <https://github.com/ceph/ceph/pull/49425>`_, Zac Dover)
+* doc/glossary: add "mds" term (`pr#48872 <https://github.com/ceph/ceph/pull/48872>`_, Zac Dover)
+* doc/glossary: add "RADOS Cluster" (`pr#49135 <https://github.com/ceph/ceph/pull/49135>`_, Zac Dover)
+* doc/glossary: add "RADOS" definition (`pr#48951 <https://github.com/ceph/ceph/pull/48951>`_, Zac Dover)
+* doc/glossary: Add "SDS" (`pr#48977 <https://github.com/ceph/ceph/pull/48977>`_, Zac Dover)
+* doc/glossary: add DAS (`pr#49255 <https://github.com/ceph/ceph/pull/49255>`_, Zac Dover)
+* doc/glossary: add matter to "RBD" (`pr#49266 <https://github.com/ceph/ceph/pull/49266>`_, Zac Dover)
+* doc/glossary: add oxford comma to "Cluster Map" (`pr#48993 <https://github.com/ceph/ceph/pull/48993>`_, Zac Dover)
+* doc/glossary: beef up "Ceph Block Storage" (`pr#48965 <https://github.com/ceph/ceph/pull/48965>`_, Zac Dover)
+* doc/glossary: capitalize "DAS" correctly (`pr#49604 <https://github.com/ceph/ceph/pull/49604>`_, Zac Dover)
+* doc/glossary: clean OSD id-related entries (`pr#49590 <https://github.com/ceph/ceph/pull/49590>`_, Zac Dover)
+* doc/glossary: Clean up "Ceph Object Storage" (`pr#49668 <https://github.com/ceph/ceph/pull/49668>`_, Zac Dover)
+* doc/glossary: collate "releases" entries (`pr#49601 <https://github.com/ceph/ceph/pull/49601>`_, Zac Dover)
+* doc/glossary: Define "Ceph Node" (`pr#48995 <https://github.com/ceph/ceph/pull/48995>`_, Zac Dover)
+* doc/glossary: define "Ceph Object Gateway" (`pr#48902 <https://github.com/ceph/ceph/pull/48902>`_, Zac Dover)
+* doc/glossary: define "Ceph OSD" (`pr#48771 <https://github.com/ceph/ceph/pull/48771>`_, Zac Dover)
+* doc/glossary: define "Ceph Storage Cluster" (`pr#49003 <https://github.com/ceph/ceph/pull/49003>`_, Zac Dover)
+* doc/glossary: define "OSD" (`pr#48760 <https://github.com/ceph/ceph/pull/48760>`_, Zac Dover)
+* doc/glossary: define "RGW" (`pr#48961 <https://github.com/ceph/ceph/pull/48961>`_, Zac Dover)
+* doc/glossary: disambiguate "OSD" (`pr#48791 <https://github.com/ceph/ceph/pull/48791>`_, Zac Dover)
+* doc/glossary: disambiguate clauses (`pr#49575 <https://github.com/ceph/ceph/pull/49575>`_, Zac Dover)
+* doc/glossary: fix "Ceph Client" (`pr#49033 <https://github.com/ceph/ceph/pull/49033>`_, Zac Dover)
+* doc/glossary: improve "Ceph Manager Dashboard" (`pr#48825 <https://github.com/ceph/ceph/pull/48825>`_, Zac Dover)
+* doc/glossary: improve "Ceph Manager" term (`pr#48812 <https://github.com/ceph/ceph/pull/48812>`_, Zac Dover)
+* doc/glossary: improve "Ceph Point Release" entry (`pr#48891 <https://github.com/ceph/ceph/pull/48891>`_, Zac Dover)
+* doc/glossary: improve "ceph" term (`pr#48821 <https://github.com/ceph/ceph/pull/48821>`_, Zac Dover)
+* doc/glossary: improve wording (`pr#48752 <https://github.com/ceph/ceph/pull/48752>`_, Zac Dover)
+* doc/glossary: link to "Ceph Manager" (`pr#49064 <https://github.com/ceph/ceph/pull/49064>`_, Zac Dover)
+* doc/glossary: link to OSD material (`pr#48785 <https://github.com/ceph/ceph/pull/48785>`_, Zac Dover)
+* doc/glossary: redirect entries to "Ceph OSD" (`pr#48834 <https://github.com/ceph/ceph/pull/48834>`_, Zac Dover)
+* doc/glossary: remove "Ceph System" (`pr#49073 <https://github.com/ceph/ceph/pull/49073>`_, Zac Dover)
+* doc/glossary: remove "Ceph Test Framework" (`pr#48842 <https://github.com/ceph/ceph/pull/48842>`_, Zac Dover)
+* doc/glossary: rewrite "Ceph File System" (`pr#48918 <https://github.com/ceph/ceph/pull/48918>`_, Zac Dover)
+* doc/glossary: s/an/each/ where it's needed (`pr#49596 <https://github.com/ceph/ceph/pull/49596>`_, Zac Dover)
+* doc/glossary: s/Ceph System/Ceph Cluster/ (`pr#49081 <https://github.com/ceph/ceph/pull/49081>`_, Zac Dover)
+* doc/glossary: s/comprising/consisting of/ (`pr#49019 <https://github.com/ceph/ceph/pull/49019>`_, Zac Dover)
+* doc/glossary: update "Cluster Map" (`pr#48798 <https://github.com/ceph/ceph/pull/48798>`_, Zac Dover)
+* doc/glossary: update "pool/pools" (`pr#48858 <https://github.com/ceph/ceph/pull/48858>`_, Zac Dover)
+* doc/index.rst: add link to Dev Guide basic workfl (`pr#46903 <https://github.com/ceph/ceph/pull/46903>`_, Zac Dover)
+* doc/install: clone-source.rst s/master/main (`pr#48381 <https://github.com/ceph/ceph/pull/48381>`_, Zac Dover)
+* doc/install: improve updating submodules procedure (`pr#48465 <https://github.com/ceph/ceph/pull/48465>`_, Zac Dover)
+* doc/install: update "Official Releases" sources (`pr#49039 <https://github.com/ceph/ceph/pull/49039>`_, Zac Dover)
+* doc/install: update clone-source.rst (`pr#49378 <https://github.com/ceph/ceph/pull/49378>`_, Zac Dover)
+* doc/man/ceph-rbdnamer: remove obsolete udev rule (`pr#49696 <https://github.com/ceph/ceph/pull/49696>`_, Ilya Dryomov)
+* doc/man/rbd: Mention changed `bluestore_min_alloc_size` (`pr#47578 <https://github.com/ceph/ceph/pull/47578>`_, Niklas Hambüchen)
+* doc/man: define --num-rep, --min-rep and --max-rep (`pr#49660 <https://github.com/ceph/ceph/pull/49660>`_, Zac Dover)
+* doc/mgr: add prompt directives to dashboard.rst (`pr#47823 <https://github.com/ceph/ceph/pull/47823>`_, Zac Dover)
+* doc/mgr: edit orchestrator.rst (`pr#47781 <https://github.com/ceph/ceph/pull/47781>`_, Zac Dover)
+* doc/mgr: name data source in "Man Install & Config" (`pr#48371 <https://github.com/ceph/ceph/pull/48371>`_, Zac Dover)
+* doc/mgr: update prompts in dboard.rst includes (`pr#47870 <https://github.com/ceph/ceph/pull/47870>`_, Zac Dover)
+* doc/monitoring: add min vers of apps in mon stack (`pr#48062 <https://github.com/ceph/ceph/pull/48062>`_, Zac Dover, Himadri Maheshwari)
+* doc/osd: Fixes the introduction for writeback mode of cache tier (`pr#48883 <https://github.com/ceph/ceph/pull/48883>`_, Mingyuan Liang)
+* doc/rados/operations: add prompts to operating.rst (`pr#47587 <https://github.com/ceph/ceph/pull/47587>`_, Zac Dover)
+* doc/rados: add prompts to monitoring-osd-pg.rst (`pr#49240 <https://github.com/ceph/ceph/pull/49240>`_, Zac Dover)
+* doc/rados: add prompts to add-or-remove-osds (`pr#49071 <https://github.com/ceph/ceph/pull/49071>`_, Zac Dover)
+* doc/rados: add prompts to add-or-rm-prompts.rst (`pr#48986 <https://github.com/ceph/ceph/pull/48986>`_, Zac Dover)
+* doc/rados: add prompts to add-or-rm-prompts.rst (`pr#48980 <https://github.com/ceph/ceph/pull/48980>`_, Zac Dover)
+* doc/rados: add prompts to auth-config-ref.rst (`pr#49516 <https://github.com/ceph/ceph/pull/49516>`_, Zac Dover)
+* doc/rados: add prompts to balancer.rst (`pr#49112 <https://github.com/ceph/ceph/pull/49112>`_, Zac Dover)
+* doc/rados: add prompts to bluestore-config-ref.rst (`pr#49536 <https://github.com/ceph/ceph/pull/49536>`_, Zac Dover)
+* doc/rados: add prompts to bluestore-migration.rst (`pr#49123 <https://github.com/ceph/ceph/pull/49123>`_, Zac Dover)
+* doc/rados: add prompts to cache-tiering.rst (`pr#49125 <https://github.com/ceph/ceph/pull/49125>`_, Zac Dover)
+* doc/rados: add prompts to ceph-conf.rst (`pr#49493 <https://github.com/ceph/ceph/pull/49493>`_, Zac Dover)
+* doc/rados: add prompts to change-mon-elections.rst (`pr#49130 <https://github.com/ceph/ceph/pull/49130>`_, Zac Dover)
+* doc/rados: add prompts to control.rst (`pr#49128 <https://github.com/ceph/ceph/pull/49128>`_, Zac Dover)
+* doc/rados: add prompts to crush-map.rst (`pr#49184 <https://github.com/ceph/ceph/pull/49184>`_, Zac Dover)
+* doc/rados: add prompts to devices.rst (`pr#49188 <https://github.com/ceph/ceph/pull/49188>`_, Zac Dover)
+* doc/rados: add prompts to erasure-code-clay.rst (`pr#49206 <https://github.com/ceph/ceph/pull/49206>`_, Zac Dover)
+* doc/rados: add prompts to erasure-code-isa (`pr#49208 <https://github.com/ceph/ceph/pull/49208>`_, Zac Dover)
+* doc/rados: add prompts to erasure-code-jerasure.rst (`pr#49210 <https://github.com/ceph/ceph/pull/49210>`_, Zac Dover)
+* doc/rados: add prompts to erasure-code-lrc.rst (`pr#49219 <https://github.com/ceph/ceph/pull/49219>`_, Zac Dover)
+* doc/rados: add prompts to erasure-code-shec.rst (`pr#49221 <https://github.com/ceph/ceph/pull/49221>`_, Zac Dover)
+* doc/rados: add prompts to health-checks (1 of 5) (`pr#49223 <https://github.com/ceph/ceph/pull/49223>`_, Zac Dover)
+* doc/rados: add prompts to health-checks (2 of 5) (`pr#49225 <https://github.com/ceph/ceph/pull/49225>`_, Zac Dover)
+* doc/rados: add prompts to health-checks (3 of 5) (`pr#49227 <https://github.com/ceph/ceph/pull/49227>`_, Zac Dover)
+* doc/rados: add prompts to health-checks (4 of 5) (`pr#49229 <https://github.com/ceph/ceph/pull/49229>`_, Zac Dover)
+* doc/rados: add prompts to health-checks (5 of 5) (`pr#49231 <https://github.com/ceph/ceph/pull/49231>`_, Zac Dover)
+* doc/rados: add prompts to librados-intro.rst (`pr#49552 <https://github.com/ceph/ceph/pull/49552>`_, Zac Dover)
+* doc/rados: add prompts to monitoring.rst (`pr#49245 <https://github.com/ceph/ceph/pull/49245>`_, Zac Dover)
+* doc/rados: add prompts to msgr2.rst (`pr#49512 <https://github.com/ceph/ceph/pull/49512>`_, Zac Dover)
+* doc/rados: add prompts to pg-repair.rst (`pr#49247 <https://github.com/ceph/ceph/pull/49247>`_, Zac Dover)
+* doc/rados: add prompts to placement-groups.rst (`pr#49274 <https://github.com/ceph/ceph/pull/49274>`_, Zac Dover)
+* doc/rados: add prompts to placement-groups.rst (3) (`pr#49276 <https://github.com/ceph/ceph/pull/49276>`_, Zac Dover)
+* doc/rados: add prompts to pools.rst (`pr#48060 <https://github.com/ceph/ceph/pull/48060>`_, Zac Dover)
+* doc/rados: add prompts to stretch-mode.rst (`pr#49370 <https://github.com/ceph/ceph/pull/49370>`_, Zac Dover)
+* doc/rados: add prompts to upmap.rst (`pr#49372 <https://github.com/ceph/ceph/pull/49372>`_, Zac Dover)
+* doc/rados: add prompts to user-management.rst (`pr#49385 <https://github.com/ceph/ceph/pull/49385>`_, Zac Dover)
+* doc/rados: clarify default EC pool from simplest (`pr#49469 <https://github.com/ceph/ceph/pull/49469>`_, Zac Dover)
+* doc/rados: cleanup "erasure code profiles" (`pr#49051 <https://github.com/ceph/ceph/pull/49051>`_, Zac Dover)
+* doc/rados: correct typo in python.rst (`pr#49560 <https://github.com/ceph/ceph/pull/49560>`_, Zac Dover)
+* doc/rados: fix grammar in configuration/index.rst (`pr#48885 <https://github.com/ceph/ceph/pull/48885>`_, Zac Dover)
+* doc/rados: fix prompts in erasure-code.rst (`pr#48335 <https://github.com/ceph/ceph/pull/48335>`_, Zac Dover)
+* doc/rados: improve pools.rst (`pr#48868 <https://github.com/ceph/ceph/pull/48868>`_, Zac Dover)
+* doc/rados: link to cephadm replacing osd section (`pr#49681 <https://github.com/ceph/ceph/pull/49681>`_, Zac Dover)
+* doc/rados: move colon (`pr#49705 <https://github.com/ceph/ceph/pull/49705>`_, Zac Dover)
+* doc/rados: refine English in crush-map-edits.rst (`pr#48366 <https://github.com/ceph/ceph/pull/48366>`_, Zac Dover)
+* doc/rados: remove prompt from php.ini line (`pr#49562 <https://github.com/ceph/ceph/pull/49562>`_, Zac Dover)
+* doc/rados: reword part of cache-tiering.rst (`pr#48888 <https://github.com/ceph/ceph/pull/48888>`_, Zac Dover)
+* doc/rados: rewrite EC intro (`pr#48324 <https://github.com/ceph/ceph/pull/48324>`_, Zac Dover)
+* doc/rados: s/backend/back end/ (`pr#48782 <https://github.com/ceph/ceph/pull/48782>`_, Zac Dover)
+* doc/rados: update "Pools" material (`pr#48856 <https://github.com/ceph/ceph/pull/48856>`_, Zac Dover)
+* doc/rados: update bluestore-config-ref.rst (`pr#46485 <https://github.com/ceph/ceph/pull/46485>`_, Zac Dover)
+* doc/rados: update prompts in crush-map-edits.rst (`pr#48364 <https://github.com/ceph/ceph/pull/48364>`_, Zac Dover)
+* doc/rados: update prompts in network-config-ref (`pr#48158 <https://github.com/ceph/ceph/pull/48158>`_, Zac Dover)
+* doc/radosgw: add prompts to multisite.rst (`pr#48660 <https://github.com/ceph/ceph/pull/48660>`_, Zac Dover)
+* doc/radosgw: add push_endpoint for rabbitmq (`pr#48488 <https://github.com/ceph/ceph/pull/48488>`_, Zac Dover)
+* doc/radosgw: improve "Ceph Object Gateway" text (`pr#48864 <https://github.com/ceph/ceph/pull/48864>`_, Zac Dover)
+* doc/radosgw: improve grammar - notifications.rst (`pr#48495 <https://github.com/ceph/ceph/pull/48495>`_, Zac Dover)
+* doc/radosgw: refine "bucket notifications" (`pr#48562 <https://github.com/ceph/ceph/pull/48562>`_, Zac Dover)
+* doc/radosgw: refine "notification reliability" (`pr#48530 <https://github.com/ceph/ceph/pull/48530>`_, Zac Dover)
+* doc/radosgw: refine "notifications" and "events" (`pr#48580 <https://github.com/ceph/ceph/pull/48580>`_, Zac Dover)
+* doc/radosgw: refine notifications.rst - top part (`pr#48503 <https://github.com/ceph/ceph/pull/48503>`_, Zac Dover)
+* doc/radosgw: update notifications.rst - grammar (`pr#48500 <https://github.com/ceph/ceph/pull/48500>`_, Zac Dover)
+* doc/radosgw: Uppercase s3 (`pr#47360 <https://github.com/ceph/ceph/pull/47360>`_, Anthony D'Atri)
+* doc/radosw: improve radosgw text (`pr#48967 <https://github.com/ceph/ceph/pull/48967>`_, Zac Dover)
+* doc/radowsgw: add prompts to notifications.rst (`pr#48536 <https://github.com/ceph/ceph/pull/48536>`_, Zac Dover)
+* doc/rbd: improve grammar in "immutable object..." (`pr#48970 <https://github.com/ceph/ceph/pull/48970>`_, Zac Dover)
+* doc/rbd: refine "Create a Block Device Pool" (`pr#49308 <https://github.com/ceph/ceph/pull/49308>`_, Zac Dover)
+* doc/rbd: refine "Create a Block Device User" (`pr#49319 <https://github.com/ceph/ceph/pull/49319>`_, Zac Dover)
+* doc/rbd: refine "Create a Block Device User" (`pr#49301 <https://github.com/ceph/ceph/pull/49301>`_, Zac Dover)
+* doc/rbd: refine "Creating a Block Device Image" (`pr#49347 <https://github.com/ceph/ceph/pull/49347>`_, Zac Dover)
+* doc/rbd: refine "Listing Block Device Images" (`pr#49349 <https://github.com/ceph/ceph/pull/49349>`_, Zac Dover)
+* doc/rbd: refine "Removing a Block Device Image" (`pr#49357 <https://github.com/ceph/ceph/pull/49357>`_, Zac Dover)
+* doc/rbd: refine "Resizing a Block Device Image" (`pr#49353 <https://github.com/ceph/ceph/pull/49353>`_, Zac Dover)
+* doc/rbd: refine "Restoring a Block Device Image" (`pr#49355 <https://github.com/ceph/ceph/pull/49355>`_, Zac Dover)
+* doc/rbd: refine "Retrieving Image Information" (`pr#49351 <https://github.com/ceph/ceph/pull/49351>`_, Zac Dover)
+* doc/rbd: refine rbd-exclusive-locks.rst (`pr#49598 <https://github.com/ceph/ceph/pull/49598>`_, Zac Dover)
+* doc/rbd: refine rbd-snapshot.rst (`pr#49485 <https://github.com/ceph/ceph/pull/49485>`_, Zac Dover)
+* doc/rbd: remove typo and ill-formed command (`pr#49366 <https://github.com/ceph/ceph/pull/49366>`_, Zac Dover)
+* doc/rbd: s/wuold/would/ in rados-rbd-cmds.rst (`pr#49592 <https://github.com/ceph/ceph/pull/49592>`_, Zac Dover)
+* doc/rbd: update iSCSI gateway info (`pr#49069 <https://github.com/ceph/ceph/pull/49069>`_, Zac Dover)
+* doc/releases: improve grammar in pacific.rst (`pr#48426 <https://github.com/ceph/ceph/pull/48426>`_, Zac Dover)
+* doc/releases: update pacific release notes (`pr#48404 <https://github.com/ceph/ceph/pull/48404>`_, Zac Dover)
+* doc/security: improve grammar in CVE-2022-0670.rst (`pr#48431 <https://github.com/ceph/ceph/pull/48431>`_, Zac Dover)
+* doc/start: add Anthony D'Atri's suggestions (`pr#49616 <https://github.com/ceph/ceph/pull/49616>`_, Zac Dover)
+* doc/start: add link-related metadocumentation (`pr#49607 <https://github.com/ceph/ceph/pull/49607>`_, Zac Dover)
+* doc/start: alphabetize hardware-recs links (`pr#46340 <https://github.com/ceph/ceph/pull/46340>`_, Zac Dover)
+* doc/start: improve documenting-ceph.rst (`pr#49566 <https://github.com/ceph/ceph/pull/49566>`_, Zac Dover)
+* doc/start: make OSD and MDS structures parallel (`pr#46656 <https://github.com/ceph/ceph/pull/46656>`_, Zac Dover)
+* doc/start: Polish network section of hardware-recommendations.rst (`pr#46663 <https://github.com/ceph/ceph/pull/46663>`_, Anthony D'Atri)
+* doc/start: refine "Quirks of RST" (`pr#49611 <https://github.com/ceph/ceph/pull/49611>`_, Zac Dover)
+* doc/start: rewrite CRUSH para (`pr#46657 <https://github.com/ceph/ceph/pull/46657>`_, Zac Dover)
+* doc/start: rewrite hardware-recs networks section (`pr#46653 <https://github.com/ceph/ceph/pull/46653>`_, Zac Dover)
+* doc/start: s/3/three/ in intro.rst (`pr#46326 <https://github.com/ceph/ceph/pull/46326>`_, Zac Dover)
+* doc/start: update documenting-ceph branch names (`pr#47956 <https://github.com/ceph/ceph/pull/47956>`_, Zac Dover)
+* doc/start: update documenting-ceph.rst (`pr#49571 <https://github.com/ceph/ceph/pull/49571>`_, Zac Dover)
+* doc/start: update hardware recs (`pr#47122 <https://github.com/ceph/ceph/pull/47122>`_, Zac Dover)
+* doc/various: update link to CRUSH pdf (`pr#48403 <https://github.com/ceph/ceph/pull/48403>`_, Zac Dover)
+* doc: add disk benchmarking and cache recommendations (`pr#46348 <https://github.com/ceph/ceph/pull/46348>`_, Dan van der Ster)
+* doc: backport pacific release notes into pacific branch (`pr#46484 <https://github.com/ceph/ceph/pull/46484>`_, Zac Dover, David Galloway)
+* doc: Change 'ReST' to 'REST' in doc/radosgw/layout.rst (`pr#48654 <https://github.com/ceph/ceph/pull/48654>`_, wangyingbin)
+* doc: fix a couple grammatical things (`pr#49622 <https://github.com/ceph/ceph/pull/49622>`_, Brad Fitzpatrick)
+* doc: fix a typo (`pr#49684 <https://github.com/ceph/ceph/pull/49684>`_, Brad Fitzpatrick)
+* doc: Install graphviz (`pr#48905 <https://github.com/ceph/ceph/pull/48905>`_, David Galloway)
+* doc: point to main branch for release info (`pr#48958 <https://github.com/ceph/ceph/pull/48958>`_, Patrick Donnelly)
+* doc: Update release process doc to accurately reflect current process (`pr#47838 <https://github.com/ceph/ceph/pull/47838>`_, David Galloway)
+* docs/start: fixes typo and empty headline in hardware recommendation … (`pr#48392 <https://github.com/ceph/ceph/pull/48392>`_, Sebastian Schmid)
+* docs: correct add system user to the master zone command (`pr#48656 <https://github.com/ceph/ceph/pull/48656>`_, Salar Nosrati-Ershad)
+* docs: fix doc link pointing to master in dashboard.rst (`pr#47791 <https://github.com/ceph/ceph/pull/47791>`_, Nizamudeen A)
+* Fix data corruption in bluefs truncate() (`pr#45171 <https://github.com/ceph/ceph/pull/45171>`_, Adam Kupczyk)
+* fsmap: switch to using iterator based loop (`pr#48269 <https://github.com/ceph/ceph/pull/48269>`_, Aliaksei Makarau)
+* Implement CIDR blocklisting (`pr#46470 <https://github.com/ceph/ceph/pull/46470>`_, Jos Collin, Greg Farnum)
+* include/buffer: include <memory> (`pr#47295 <https://github.com/ceph/ceph/pull/47295>`_, Kefu Chai, Duncan Bellamy)
+* include: fix IS_ERR on Windows (`pr#47923 <https://github.com/ceph/ceph/pull/47923>`_, Lucian Petrut)
+* libcephfs: define AT_NO_ATTR_SYNC back for backward compatibility (`pr#47862 <https://github.com/ceph/ceph/pull/47862>`_, Xiubo Li)
+* libcephsqlite: ceph-mgr crashes when compiled with gcc12 (`pr#47271 <https://github.com/ceph/ceph/pull/47271>`_, Ganesh Maharaj Mahalingam)
+* librados/watch_notify: reconnect after socket injection (`pr#46499 <https://github.com/ceph/ceph/pull/46499>`_, Nitzan Mordechai)
+* librados: rados_ioctx_destroy check for initialized ioctx (`pr#47451 <https://github.com/ceph/ceph/pull/47451>`_, Nitzan Mordechai)
+* librbd/cache/pwl: fix clean vs bytes_dirty cache state inconsistency (`pr#49054 <https://github.com/ceph/ceph/pull/49054>`_, Yin Congmin)
+* librbd/cache/pwl: fix endianness issue (`pr#46815 <https://github.com/ceph/ceph/pull/46815>`_, Yin Congmin)
+* librbd/cache/pwl: narrow the scope of m_lock in write_image_cache_state() (`pr#47939 <https://github.com/ceph/ceph/pull/47939>`_, Ilya Dryomov, Yin Congmin)
+* librbd: bail from schedule_request_lock() if already lock owner (`pr#47161 <https://github.com/ceph/ceph/pull/47161>`_, Christopher Hoffman)
+* librbd: retry ENOENT in V2_REFRESH_PARENT as well (`pr#47995 <https://github.com/ceph/ceph/pull/47995>`_, Ilya Dryomov)
+* librbd: tweak misleading "image is still primary" error message (`pr#47247 <https://github.com/ceph/ceph/pull/47247>`_, Ilya Dryomov)
+* librbd: unlink newest mirror snapshot when at capacity, bump capacity (`pr#46593 <https://github.com/ceph/ceph/pull/46593>`_, Ilya Dryomov)
+* librbd: update progress for non-existent objects on deep-copy (`pr#46909 <https://github.com/ceph/ceph/pull/46909>`_, Ilya Dryomov)
+* librbd: use actual monitor addresses when creating a peer bootstrap token (`pr#47911 <https://github.com/ceph/ceph/pull/47911>`_, Ilya Dryomov)
+* make-dist: patch boost source to support python 3.10  … (`pr#47027 <https://github.com/ceph/ceph/pull/47027>`_, Tim Serong, Kefu Chai)
+* mds: increment directory inode's change attr by one (`pr#48521 <https://github.com/ceph/ceph/pull/48521>`_, Ramana Raja)
+* mds: clear MDCache::rejoin\_\*_q queues before recovering file inodes (`pr#46682 <https://github.com/ceph/ceph/pull/46682>`_, Xiubo Li)
+* mds: flush mdlog if locked and still has wanted caps not satisfied (`pr#46423 <https://github.com/ceph/ceph/pull/46423>`_, Xiubo Li)
+* mds: reset heartbeat when fetching or committing entries (`pr#46180 <https://github.com/ceph/ceph/pull/46180>`_, Xiubo Li)
+* mds: trigger to flush the mdlog in handle_find_ino() (`pr#46424 <https://github.com/ceph/ceph/pull/46424>`_, Xiubo Li)
+* mds/client: fail the request if the peer MDS doesn't support getvxattr op (`pr#47891 <https://github.com/ceph/ceph/pull/47891>`_, Xiubo Li, Zack Cerza)
+* mds/Server: Do not abort MDS on unknown messages (`pr#48253 <https://github.com/ceph/ceph/pull/48253>`_, Dhairya Parmar, Dhairy Parmar)
+* mds: add a perf counter to record slow replies (`pr#46138 <https://github.com/ceph/ceph/pull/46138>`_, haoyixing)
+* mds: damage table only stores one dentry per dirfrag (`pr#48262 <https://github.com/ceph/ceph/pull/48262>`_, Patrick Donnelly)
+* mds: do not assert early on when issuing client leases (`issue#54701 <http://tracker.ceph.com/issues/54701>`_, `pr#46567 <https://github.com/ceph/ceph/pull/46567>`_, Venky Shankar)
+* mds: Don't blocklist clients in any replay state (`pr#47111 <https://github.com/ceph/ceph/pull/47111>`_, Kotresh HR)
+* mds: fix crash when exporting unlinked dir (`pr#47180 <https://github.com/ceph/ceph/pull/47180>`_, 胡玮文)
+* mds: include encoded stray inode when sending dentry unlink message to replicas (`issue#54046 <http://tracker.ceph.com/issues/54046>`_, `pr#46183 <https://github.com/ceph/ceph/pull/46183>`_, Venky Shankar)
+* mds: notify the xattr_version to replica MDSes (`pr#47056 <https://github.com/ceph/ceph/pull/47056>`_, Xiubo Li)
+* mds: skip fetching the dirfrags if not a directory (`pr#47433 <https://github.com/ceph/ceph/pull/47433>`_, Xiubo Li)
+* mds: standby-replay daemon always removed in MDSMonitor::prepare_beacon (`pr#47282 <https://github.com/ceph/ceph/pull/47282>`_, Patrick Donnelly)
+* mds: switch to use projected inode instead (`pr#47059 <https://github.com/ceph/ceph/pull/47059>`_, Xiubo Li)
+* mds: wait unlink to finish to avoid conflict when creating same entries (`pr#48453 <https://github.com/ceph/ceph/pull/48453>`_, Xiubo Li)
+* mgr, mgr/prometheus: Fix regression with prometheus metrics (`pr#47693 <https://github.com/ceph/ceph/pull/47693>`_, Prashant D)
+* mgr, mgr/prometheus: Fix regression with prometheus metrics (`pr#46429 <https://github.com/ceph/ceph/pull/46429>`_, Prashant D)
+* mgr, mon: Keep upto date metadata with mgr for MONs (`pr#47692 <https://github.com/ceph/ceph/pull/47692>`_, Laura Flores, Prashant D)
+* mgr, mon: Keep upto date metadata with mgr for MONs (`pr#46427 <https://github.com/ceph/ceph/pull/46427>`_, Prashant D)
+* mgr/ActivePyModules.cc: fix cases where GIL is held while attempting to lock mutex (`pr#46302 <https://github.com/ceph/ceph/pull/46302>`_, Cory Snyder)
+* mgr/cephadm: Add disk rescan feature to the orchestrator (`pr#47372 <https://github.com/ceph/ceph/pull/47372>`_, Adam King, Paul Cuzner)
+* mgr/cephadm: adding logic to close ports when removing a daemon (`pr#46780 <https://github.com/ceph/ceph/pull/46780>`_, Redouane Kachach)
+* mgr/cephadm: Adding logic to store grafana cert/key per node (`pr#48103 <https://github.com/ceph/ceph/pull/48103>`_, Redouane Kachach)
+* mgr/cephadm: allow setting prometheus retention time (`pr#48100 <https://github.com/ceph/ceph/pull/48100>`_, Adam King)
+* mgr/cephadm: capture exception when not able to list upgrade tags (`pr#46776 <https://github.com/ceph/ceph/pull/46776>`_, Redouane Kachach)
+* mgr/cephadm: check if a service exists before trying to restart it (`pr#46779 <https://github.com/ceph/ceph/pull/46779>`_, Redouane Kachach)
+* mgr/cephadm: clear error message when resuming upgrade (`pr#47375 <https://github.com/ceph/ceph/pull/47375>`_, Adam King)
+* mgr/cephadm: don't redeploy osds seen in raw list if cephadm knows them (`pr#46545 <https://github.com/ceph/ceph/pull/46545>`_, Adam King)
+* mgr/cephadm: fixing scheduler consistent hashing (`pr#46975 <https://github.com/ceph/ceph/pull/46975>`_, Redouane Kachach)
+* mgr/cephadm: Raw OSD Support (`pr#45964 <https://github.com/ceph/ceph/pull/45964>`_, Guillaume Abrioux, Adam King, Sage Weil)
+* mgr/cephadm: reconfig iscsi daemons if trusted_ip_list changes (`pr#48096 <https://github.com/ceph/ceph/pull/48096>`_, Adam King)
+* mgr/cephadm: recreate osd config when redeploy/reconfiguring (`pr#47663 <https://github.com/ceph/ceph/pull/47663>`_, Adam King)
+* mgr/cephadm: set dashboard grafana-api-password when user provides one (`pr#47662 <https://github.com/ceph/ceph/pull/47662>`_, Adam King)
+* mgr/cephadm: staggered upgrade (`pr#46359 <https://github.com/ceph/ceph/pull/46359>`_, Adam King)
+* mgr/cephadm: try to get FQDN for active instance (`pr#46775 <https://github.com/ceph/ceph/pull/46775>`_, Tatjana Dehler)
+* mgr/cephadm: use host shortname for osd memory autotuning (`pr#46556 <https://github.com/ceph/ceph/pull/46556>`_, Adam King)
+* mgr/dashboard:  don't log 3xx as errors (`pr#46461 <https://github.com/ceph/ceph/pull/46461>`_, Ernesto Puerta)
+* mgr/dashboard:  WDC multipath bug fixes (`pr#46456 <https://github.com/ceph/ceph/pull/46456>`_, Nizamudeen A)
+* mgr/dashboard: Add details to the modal which displays the `safe-to-d… (`pr#48176 <https://github.com/ceph/ceph/pull/48176>`_, Francesco Torchia)
+* mgr/dashboard: add option to resolve ip addr (`pr#48220 <https://github.com/ceph/ceph/pull/48220>`_, Tatjana Dehler)
+* mgr/dashboard: add required validation for frontend and monitor port (`pr#47357 <https://github.com/ceph/ceph/pull/47357>`_, Avan Thakkar)
+* mgr/dashboard: Add text to empty life expectancy column (`pr#48276 <https://github.com/ceph/ceph/pull/48276>`_, Francesco Torchia)
+* mgr/dashboard: allow cross origin when the url is set (`pr#49151 <https://github.com/ceph/ceph/pull/49151>`_, Nizamudeen A)
+* mgr/dashboard: allow Origin url for CORS if present in config (`pr#49429 <https://github.com/ceph/ceph/pull/49429>`_, Avan Thakkar)
+* mgr/dashboard: batch rbd-mirror backports (`pr#46531 <https://github.com/ceph/ceph/pull/46531>`_, Pere Diaz Bou, Pedro Gonzalez Gomez, Nizamudeen A, Melissa Li, Sarthak0702, Avan Thakkar, Aashish Sharma)
+* mgr/dashboard: BDD approach for the dashboard cephadm e2e (`pr#46529 <https://github.com/ceph/ceph/pull/46529>`_, Nizamudeen A)
+* mgr/dashboard: bug fixes for rbd mirroring edit and promotion/demotion (`pr#48806 <https://github.com/ceph/ceph/pull/48806>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: bump moment from 2.29.1 to 2.29.3 in /src/pybind/mgr/dashboard/frontend (`pr#46717 <https://github.com/ceph/ceph/pull/46717>`_, dependabot[bot])
+* mgr/dashboard: bump up teuthology (`pr#47497 <https://github.com/ceph/ceph/pull/47497>`_, Kefu Chai)
+* mgr/dashboard: Creating and editing Prometheus AlertManager silences is buggy (`pr#46277 <https://github.com/ceph/ceph/pull/46277>`_, Volker Theile)
+* mgr/dashboard: customizable log-in page text/banner (`pr#46343 <https://github.com/ceph/ceph/pull/46343>`_, Sarthak0702)
+* mgr/dashboard: dashboard help command showing wrong syntax for login-banner (`pr#46810 <https://github.com/ceph/ceph/pull/46810>`_, Sarthak0702)
+* mgr/dashboard: display helpfull message when the iframe-embedded Grafana dashboard failed to load (`pr#47008 <https://github.com/ceph/ceph/pull/47008>`_, Ngwa Sedrick Meh)
+* mgr/dashboard: do not recommend throughput for ssd's only cluster (`pr#47155 <https://github.com/ceph/ceph/pull/47155>`_, Nizamudeen A)
+* mgr/dashboard: don't log tracebacks on 404s (`pr#47093 <https://github.com/ceph/ceph/pull/47093>`_, Ernesto Puerta)
+* mgr/dashboard: enable addition of custom Prometheus alerts (`pr#48099 <https://github.com/ceph/ceph/pull/48099>`_, Patrick Seidensal)
+* mgr/dashboard: ensure limit 0 returns 0 images (`pr#47888 <https://github.com/ceph/ceph/pull/47888>`_, Pere Diaz Bou)
+* mgr/dashboard: Feature 54330 osd creation workflow (`pr#46690 <https://github.com/ceph/ceph/pull/46690>`_, Pere Diaz Bou, Nizamudeen A, Sarthak0702)
+* mgr/dashboard: fix _rbd_image_refs caching (`pr#47636 <https://github.com/ceph/ceph/pull/47636>`_, Pere Diaz Bou)
+* mgr/dashboard: fix Expected to find element: `cd-modal .badge but never found it (`pr#48142 <https://github.com/ceph/ceph/pull/48142>`_, Nizamudeen A)
+* mgr/dashboard: fix nfs exports form issues with squash field (`pr#47960 <https://github.com/ceph/ceph/pull/47960>`_, Nizamudeen A)
+* mgr/dashboard: fix openapi-check (`pr#48045 <https://github.com/ceph/ceph/pull/48045>`_, Pere Diaz Bou)
+* mgr/dashboard: fix rgw connect when using ssl (`issue#56970 <http://tracker.ceph.com/issues/56970>`_, `pr#48189 <https://github.com/ceph/ceph/pull/48189>`_, Henry Hirsch)
+* mgr/dashboard: fix snapshot creation with duplicate name (`pr#48048 <https://github.com/ceph/ceph/pull/48048>`_, Aashish Sharma)
+* mgr/dashboard: fix ssl cert validation for ingress service creation (`pr#46204 <https://github.com/ceph/ceph/pull/46204>`_, Avan Thakkar)
+* mgr/dashboard: fix unmanaged service creation (`pr#48026 <https://github.com/ceph/ceph/pull/48026>`_, Nizamudeen A)
+* mgr/dashboard: fix wrong pg status processing (`pr#46228 <https://github.com/ceph/ceph/pull/46228>`_, Ernesto Puerta)
+* mgr/dashboard: form field validation icons overlap with other icons (`pr#46379 <https://github.com/ceph/ceph/pull/46379>`_, Sarthak0702)
+* mgr/dashboard: grafana frontend e2e testing and update cypress (`pr#47721 <https://github.com/ceph/ceph/pull/47721>`_, Nizamudeen A)
+* mgr/dashboard: handle the cephfs permission issue in nfs exports (`pr#48316 <https://github.com/ceph/ceph/pull/48316>`_, Nizamudeen A)
+* mgr/dashboard: host list tables doesn't show all services deployed (`pr#47454 <https://github.com/ceph/ceph/pull/47454>`_, Avan Thakkar)
+* mgr/dashboard: ingress backend service should list all supported services (`pr#47084 <https://github.com/ceph/ceph/pull/47084>`_, Avan Thakkar)
+* mgr/dashboard: introduce memory and cpu usage for daemons (`pr#46459 <https://github.com/ceph/ceph/pull/46459>`_, Aashish Sharma, Avan Thakkar)
+* mgr/dashboard: iops optimized option enabled (`pr#46737 <https://github.com/ceph/ceph/pull/46737>`_, Pere Diaz Bou)
+* mgr/dashboard: iterate through copy of items (`pr#46870 <https://github.com/ceph/ceph/pull/46870>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: prevent alert redirect (`pr#47145 <https://github.com/ceph/ceph/pull/47145>`_, Tatjana Dehler)
+* mgr/dashboard: Pull latest languages from Transifex (`pr#46695 <https://github.com/ceph/ceph/pull/46695>`_, Volker Theile)
+* mgr/dashboard: rbd image pagination (`pr#47105 <https://github.com/ceph/ceph/pull/47105>`_, Pere Diaz Bou, Nizamudeen A)
+* mgr/dashboard: rbd striping setting pre-population and pop-over (`pr#47410 <https://github.com/ceph/ceph/pull/47410>`_, Vrushal Chaudhari)
+* mgr/dashboard: remove token logging (`pr#47431 <https://github.com/ceph/ceph/pull/47431>`_, Pere Diaz Bou)
+* mgr/dashboard: Show error on creating service with duplicate service id (`pr#47404 <https://github.com/ceph/ceph/pull/47404>`_, Aashish Sharma)
+* mgr/dashboard: stop polling when page is not visible (`pr#46675 <https://github.com/ceph/ceph/pull/46675>`_, Sarthak0702)
+* mgr/dashboard: unselect rows in datatables (`pr#46322 <https://github.com/ceph/ceph/pull/46322>`_, Sarthak0702)
+* mgr/DaemonServer.cc: fix typo in output gap >= max_pg_num_change (`pr#47211 <https://github.com/ceph/ceph/pull/47211>`_, Kamoltat)
+* mgr/prometheus: expose num objects repaired in pool (`pr#48205 <https://github.com/ceph/ceph/pull/48205>`_, Pere Diaz Bou)
+* mgr/prometheus: use vendored "packaging" instead (`pr#49695 <https://github.com/ceph/ceph/pull/49695>`_, Matan Breizman)
+* mgr/rbd_support: avoid wedging the task queue if pool is removed (`pr#49056 <https://github.com/ceph/ceph/pull/49056>`_, Ilya Dryomov)
+* mgr/snap_schedule: add time zone suffix to snapshot dir name (`pr#45968 <https://github.com/ceph/ceph/pull/45968>`_, Milind Changire, Venky Shankar)
+* mgr/snap_schedule: persist all updates to RADOS (`pr#46797 <https://github.com/ceph/ceph/pull/46797>`_, Milind Changire)
+* mgr/snap_schedule: remove subvol interface (`pr#48221 <https://github.com/ceph/ceph/pull/48221>`_, Milind Changire)
+* mgr/stats: be resilient to offline MDS rank-0 (`pr#45293 <https://github.com/ceph/ceph/pull/45293>`_, Jos Collin)
+* mgr/stats: change in structure of perf_stats o/p (`pr#47851 <https://github.com/ceph/ceph/pull/47851>`_, Neeraj Pratap Singh)
+* mgr/stats: missing clients in perf stats command output (`pr#47866 <https://github.com/ceph/ceph/pull/47866>`_, Neeraj Pratap Singh)
+* mgr/telemetry: reset health warning after re-opting-in (`pr#47307 <https://github.com/ceph/ceph/pull/47307>`_, Yaarit Hatuka)
+* mgr/volumes: A few dependent mgr volumes PRs (`pr#47112 <https://github.com/ceph/ceph/pull/47112>`_, Rishabh Dave, Kotresh HR, John Mulligan, Nikhilkumar Shelke)
+* mgr/volumes: Add human-readable flag to volume info command (`pr#48468 <https://github.com/ceph/ceph/pull/48468>`_, Neeraj Pratap Singh)
+* mgr/volumes: add interface to check the presence of subvolumegroups/subvolumes (`pr#47460 <https://github.com/ceph/ceph/pull/47460>`_, Neeraj Pratap Singh)
+* mgr/volumes: Add volume info command (`pr#47769 <https://github.com/ceph/ceph/pull/47769>`_, Neeraj Pratap Singh)
+* mgr/volumes: filter internal directories in 'subvolumegroup ls' command (`pr#47512 <https://github.com/ceph/ceph/pull/47512>`_, Nikhilkumar Shelke)
+* mgr/volumes: Fix idempotent subvolume rm (`pr#46139 <https://github.com/ceph/ceph/pull/46139>`_, Kotresh HR)
+* mgr/volumes: Fix subvolume creation in FIPS enabled system (`pr#47369 <https://github.com/ceph/ceph/pull/47369>`_, Kotresh HR)
+* mgr/volumes: remove incorrect 'size' from output of 'snapshot info' (`pr#46803 <https://github.com/ceph/ceph/pull/46803>`_, Nikhilkumar Shelke)
+* mgr/volumes: set, get, list and remove metadata of snapshot (`pr#46515 <https://github.com/ceph/ceph/pull/46515>`_, Nikhilkumar Shelke)
+* mgr/volumes: set, get, list and remove metadata of subvolume (`pr#45961 <https://github.com/ceph/ceph/pull/45961>`_, Nikhilkumar Shelke)
+* mgr/volumes: Show clone failure reason in clone status command (`pr#45928 <https://github.com/ceph/ceph/pull/45928>`_, Kotresh HR)
+* mgr/volumes: subvolume ls command crashes if groupname as '_nogroup' (`pr#46806 <https://github.com/ceph/ceph/pull/46806>`_, Nikhilkumar Shelke)
+* mgr/volumes: subvolumegroup quotas (`pr#46668 <https://github.com/ceph/ceph/pull/46668>`_, Kotresh HR)
+* mgr: relax "pending_service_map.epoch > service_map.epoch" assert (`pr#46688 <https://github.com/ceph/ceph/pull/46688>`_, Mykola Golub)
+* mirror snapshot schedule and trash purge schedule fixes (`pr#46778 <https://github.com/ceph/ceph/pull/46778>`_, Ilya Dryomov)
+* mon/ConfigMonitor: fix config get key with whitespaces (`pr#47380 <https://github.com/ceph/ceph/pull/47380>`_, Nitzan Mordechai)
+* mon/Elector.cc: Compress peer >= rank_size sanity check into send_peer_ping (`pr#49444 <https://github.com/ceph/ceph/pull/49444>`_, Kamoltat)
+* mon/Elector: Added sanity check when pinging a peer monitor (`pr#48320 <https://github.com/ceph/ceph/pull/48320>`_, Kamoltat)
+* mon/Elector: Change how we handle removed_ranks and notify_rank_removed() (`pr#49312 <https://github.com/ceph/ceph/pull/49312>`_, Kamoltat)
+* mon/Elector: notify_rank_removed erase rank from both live_pinging and dead_pinging sets for highest ranked MON (`pr#47087 <https://github.com/ceph/ceph/pull/47087>`_, Kamoltat)
+* mon/MDSMonitor: fix standby-replay mds being removed from MDSMap unexpectedly (`pr#48270 <https://github.com/ceph/ceph/pull/48270>`_, 胡玮文)
+* mon/OSDMonitor: Added extra check before mon.go_recovery_stretch_mode() (`pr#48803 <https://github.com/ceph/ceph/pull/48803>`_, Kamoltat)
+* mon/OSDMonitor: Ensure kvmon() is writeable before handling "osd new" cmd (`pr#46691 <https://github.com/ceph/ceph/pull/46691>`_, Sridhar Seshasayee)
+* mon/OSDMonitor: properly set last_force_op_resend in stretch mode (`pr#45870 <https://github.com/ceph/ceph/pull/45870>`_, Ilya Dryomov)
+* mon: allow a MON_DOWN grace period after cluster mkfs (`pr#48558 <https://github.com/ceph/ceph/pull/48558>`_, Sage Weil)
+* monitoring/ceph-mixin: add RGW host to label info (`pr#48035 <https://github.com/ceph/ceph/pull/48035>`_, Tatjana Dehler)
+* monitoring/ceph-mixin: OSD overview typo fix (`pr#47386 <https://github.com/ceph/ceph/pull/47386>`_, Tatjana Dehler)
+* mount/conf: Fix IPv6 parsing (`pr#46112 <https://github.com/ceph/ceph/pull/46112>`_, Matan Breizman)
+* msg: fix deadlock when handling existing but closed v2 connection (`pr#48254 <https://github.com/ceph/ceph/pull/48254>`_, Radosław Zarzyński)
+* msg: Fix Windows IPv6 support (`pr#47303 <https://github.com/ceph/ceph/pull/47303>`_, Lucian Petrut)
+* msg: Log at higher level when Throttle::get_or_fail() fails (`pr#47764 <https://github.com/ceph/ceph/pull/47764>`_, Brad Hubbard)
+* msg: reset ProtocolV2's frame assembler in appropriate thread (`pr#48255 <https://github.com/ceph/ceph/pull/48255>`_, Radoslaw Zarzynski)
+* os/bluestore:  proper locking for Allocators' dump methods (`pr#48167 <https://github.com/ceph/ceph/pull/48167>`_, Igor Fedotov)
+* os/bluestore: add bluefs-import command (`pr#47875 <https://github.com/ceph/ceph/pull/47875>`_, Adam Kupczyk, zhang daolong)
+* os/bluestore: Always update the cursor position in AVL near-fit search (`pr#46642 <https://github.com/ceph/ceph/pull/46642>`_, Mark Nelson)
+* os/bluestore: Better readability of perf output (`pr#47259 <https://github.com/ceph/ceph/pull/47259>`_, Adam Kupczyk)
+* os/bluestore: BlueFS: harmonize log read and writes modes (`pr#49431 <https://github.com/ceph/ceph/pull/49431>`_, Adam Kupczyk)
+* os/bluestore: do not signal deleted dirty file to bluefs log (`pr#48168 <https://github.com/ceph/ceph/pull/48168>`_, Igor Fedotov)
+* os/bluestore: fix AU accounting in bluestore_cache_other mempool (`pr#47337 <https://github.com/ceph/ceph/pull/47337>`_, Igor Fedotov)
+* os/bluestore: Fix collision between BlueFS and BlueStore deferred writes (`pr#47296 <https://github.com/ceph/ceph/pull/47296>`_, Adam Kupczyk)
+* os/bluestore: fix improper bluefs log size tracking in volume selector (`pr#45408 <https://github.com/ceph/ceph/pull/45408>`_, Igor Fedotov)
+* os/bluestore: get rid of fake onode nref increment for pinned entry (`pr#47556 <https://github.com/ceph/ceph/pull/47556>`_, Igor Fedotov)
+* os/bluestore: incremental update mode for bluefs log (`pr#48915 <https://github.com/ceph/ceph/pull/48915>`_, Adam Kupczyk)
+* os/bluestore: update perf counter priorities (`pr#47095 <https://github.com/ceph/ceph/pull/47095>`_, Laura Flores)
+* os/bluestore: use direct write in BlueStore::_write_bdev_label (`pr#48278 <https://github.com/ceph/ceph/pull/48278>`_, luo rixin)
+* osd, mds: fix the "heap" admin cmd printing always to error stream (`pr#48106 <https://github.com/ceph/ceph/pull/48106>`_, Radoslaw Zarzynski)
+* osd, tools, kv: non-aggressive, on-line trimming of accumulated dups (`pr#47701 <https://github.com/ceph/ceph/pull/47701>`_, Radoslaw Zarzynski, Nitzan Mordechai)
+* osd/PGLog.cc: Trim duplicates by number of entries (`pr#46252 <https://github.com/ceph/ceph/pull/46252>`_, Nitzan Mordechai)
+* osd/scrub: mark PG as being scrubbed, from scrub initiation to Inacti… (`pr#46767 <https://github.com/ceph/ceph/pull/46767>`_, Ronen Friedman)
+* osd/scrub: Reintroduce scrub starts message (`pr#48070 <https://github.com/ceph/ceph/pull/48070>`_, Prashant D)
+* osd/scrub: use the actual active set when requesting replicas (`pr#48544 <https://github.com/ceph/ceph/pull/48544>`_, Ronen Friedman)
+* osd/SnapMapper: fix legacy key conversion in snapmapper class (`pr#47134 <https://github.com/ceph/ceph/pull/47134>`_, Manuel Lausch, Matan Breizman)
+* osd: add created_at meta (`pr#49144 <https://github.com/ceph/ceph/pull/49144>`_, Alex Marangone)
+* osd: fix wrong input when calling recover_object() (`pr#46120 <https://github.com/ceph/ceph/pull/46120>`_, Myoungwon Oh)
+* osd: log the number of 'dups' entries in a PG Log (`pr#46608 <https://github.com/ceph/ceph/pull/46608>`_, Radoslaw Zarzynski)
+* osd: remove invalid put on message (`pr#47525 <https://github.com/ceph/ceph/pull/47525>`_, Nitzan Mordechai)
+* osd: set per_pool_stats true when OSD has no PG (`pr#48250 <https://github.com/ceph/ceph/pull/48250>`_, jindengke, lmgdlmgd)
+* osd/scrub: late-arriving reservation grants are not an error (`pr#46873 <https://github.com/ceph/ceph/pull/46873>`_, Ronen Friedman)
+* osd/scrubber/pg_scrubber.cc: fix bug where scrub machine gets stuck (`pr#46845 <https://github.com/ceph/ceph/pull/46845>`_, Cory Snyder)
+* PendingReleaseNotes: document online and offline trimming of PG Log's… (`pr#48020 <https://github.com/ceph/ceph/pull/48020>`_, Radoslaw Zarzynski)
+* pybind/cephfs: fix grammar (`pr#48982 <https://github.com/ceph/ceph/pull/48982>`_, Zac Dover)
+* pybind: fix typo in cephfs.pyx (`pr#48953 <https://github.com/ceph/ceph/pull/48953>`_, Zac Dover)
+* pybind/mgr/cephadm/serve: don't remove ceph.conf which leads to qa failure (`pr#46974 <https://github.com/ceph/ceph/pull/46974>`_, Dhairya Parmar)
+* pybind/mgr/dashboard: move pytest into requirements.txt (`pr#48081 <https://github.com/ceph/ceph/pull/48081>`_, Kefu Chai)
+* pybind/mgr/pg_autoscaler: change overlapping roots to warning (`pr#47522 <https://github.com/ceph/ceph/pull/47522>`_, Kamoltat)
+* pybind/mgr: fix flake8 (`pr#47393 <https://github.com/ceph/ceph/pull/47393>`_, Avan Thakkar)
+* pybind/mgr: fixup after upgrading tox versions (`pr#49363 <https://github.com/ceph/ceph/pull/49363>`_, Adam King, Kefu Chai)
+* pybind/mgr: tox and test fixes (`pr#49542 <https://github.com/ceph/ceph/pull/49542>`_, Kefu Chai)
+* pybind/rados: notify callback reconnect (`pr#48112 <https://github.com/ceph/ceph/pull/48112>`_, Nitzan Mordechai)
+* pybind: add wrapper for rados_write_op_omap_cmp (`pr#48376 <https://github.com/ceph/ceph/pull/48376>`_, Sandy Kaur)
+* python-common: Add 'KB' to supported suffixes in SizeMatcher (`pr#48243 <https://github.com/ceph/ceph/pull/48243>`_, Tim Serong)
+* python-common: allow crush device class to be set from osd service spec (`pr#46555 <https://github.com/ceph/ceph/pull/46555>`_, Cory Snyder)
+* qa/cephadm: remove fsid dir before bootstrap in test_cephadm.sh (`pr#48101 <https://github.com/ceph/ceph/pull/48101>`_, Adam King)
+* qa/cephfs: fallback to older way of get_op_read_count (`pr#46901 <https://github.com/ceph/ceph/pull/46901>`_, Dhairya Parmar)
+* qa/import-legacy: install python3 package for nautilus ceph (`pr#47528 <https://github.com/ceph/ceph/pull/47528>`_, Xiubo Li)
+* qa/suites/rados/thrash-erasure-code-big/thrashers: add `osd max backfills` setting to mapgap and pggrow (`pr#46391 <https://github.com/ceph/ceph/pull/46391>`_, Laura Flores)
+* qa/suites/rbd/pwl-cache: ensure recovery is actually tested (`pr#47128 <https://github.com/ceph/ceph/pull/47128>`_, Ilya Dryomov, Yin Congmin)
+* qa/suites/rbd: disable workunit timeout for dynamic_features_no_cache (`pr#47158 <https://github.com/ceph/ceph/pull/47158>`_, Ilya Dryomov)
+* qa/suites/rbd: place cache file on tmpfs for xfstests (`pr#46597 <https://github.com/ceph/ceph/pull/46597>`_, Ilya Dryomov)
+* qa/tasks/ceph_manager.py: increase test_pool_min_size timeout (`pr#47446 <https://github.com/ceph/ceph/pull/47446>`_, Kamoltat)
+* qa/tasks/kubeadm: set up tigera resources via kubectl create (`pr#48097 <https://github.com/ceph/ceph/pull/48097>`_, John Mulligan)
+* qa/tasks/rbd_fio: bump default to fio 3.32 (`pr#48385 <https://github.com/ceph/ceph/pull/48385>`_, Ilya Dryomov)
+* qa/workunits/cephadm: update test_repos master -> main (`pr#47320 <https://github.com/ceph/ceph/pull/47320>`_, Adam King)
+* qa/workunits/rados: specify redirect in curl command (`pr#49139 <https://github.com/ceph/ceph/pull/49139>`_, Laura Flores)
+* qa: Fix test_subvolume_group_ls_filter_internal_directories (`pr#48328 <https://github.com/ceph/ceph/pull/48328>`_, Kotresh HR)
+* qa: Fix test_subvolume_snapshot_info_if_orphan_clone (`pr#48647 <https://github.com/ceph/ceph/pull/48647>`_, Kotresh HR)
+* qa: Fix test_subvolume_snapshot_info_if_orphan_clone (`pr#48417 <https://github.com/ceph/ceph/pull/48417>`_, Kotresh HR)
+* qa: fix teuthology master branch ref (`pr#46504 <https://github.com/ceph/ceph/pull/46504>`_, Ernesto Puerta)
+* qa: ignore disk quota exceeded failure in test (`pr#48165 <https://github.com/ceph/ceph/pull/48165>`_, Nikhilkumar Shelke)
+* qa: remove .teuthology_branch file (`pr#46490 <https://github.com/ceph/ceph/pull/46490>`_, Jeff Layton)
+* qa: run e2e test on centos only (`pr#49337 <https://github.com/ceph/ceph/pull/49337>`_, Kefu Chai)
+* qa: switch back to git protocol for qemu-xfstests (`pr#49543 <https://github.com/ceph/ceph/pull/49543>`_, Ilya Dryomov)
+* qa: switch to https protocol for repos' server (`pr#49470 <https://github.com/ceph/ceph/pull/49470>`_, Xiubo Li)
+* qa: wait rank 0 to become up:active state before mounting fuse client (`pr#46802 <https://github.com/ceph/ceph/pull/46802>`_, Xiubo Li)
+* qa: add filesystem/file sync stuck test support (`pr#46425 <https://github.com/ceph/ceph/pull/46425>`_, Xiubo Li)
+* radosgw-admin: 'reshard list' doesn't log ENOENT errors (`pr#45451 <https://github.com/ceph/ceph/pull/45451>`_, Casey Bodley)
+* rbd-fuse: librados will filter out -r option from command-line (`pr#46953 <https://github.com/ceph/ceph/pull/46953>`_, wanwencong)
+* rbd-mirror: don't prune non-primary snapshot when restarting delta sync (`pr#46590 <https://github.com/ceph/ceph/pull/46590>`_, Ilya Dryomov)
+* rbd-mirror: generally skip replay/resync if remote image is not primary (`pr#46813 <https://github.com/ceph/ceph/pull/46813>`_, Ilya Dryomov)
+* rbd-mirror: remove bogus completed_non_primary_snapshots_exist check (`pr#47118 <https://github.com/ceph/ceph/pull/47118>`_, Ilya Dryomov)
+* rbd-mirror: resume pending shutdown on error in snapshot replayer (`pr#47913 <https://github.com/ceph/ceph/pull/47913>`_, Ilya Dryomov)
+* rbd: device map/unmap --namespace handling fixes (`pr#48459 <https://github.com/ceph/ceph/pull/48459>`_, Ilya Dryomov, Stefan Chivu)
+* rbd: don't default empty pool name unless namespace is specified (`pr#47143 <https://github.com/ceph/ceph/pull/47143>`_, Ilya Dryomov)
+* rbd: find_action() should sort actions first (`pr#47583 <https://github.com/ceph/ceph/pull/47583>`_, Ilya Dryomov)
+* rgw: Swift retarget needs bucket set on object (`pr#47230 <https://github.com/ceph/ceph/pull/47230>`_, Daniel Gryniewicz)
+* rgw/backport/pacific: Fix crashes with Sync policy APIs (`pr#47994 <https://github.com/ceph/ceph/pull/47994>`_, Soumya Koduri)
+* rgw/notifications: Change in multipart upload notification behavior (`pr#47175 <https://github.com/ceph/ceph/pull/47175>`_, Kalpesh Pandya)
+* rgw/rgw_string.h: add missing includes for alpine and boost 1.75 (`pr#47304 <https://github.com/ceph/ceph/pull/47304>`_, Duncan Bellamy)
+* rgw/sts: adding code for aws:RequestTags as part (`pr#47746 <https://github.com/ceph/ceph/pull/47746>`_, Kalpesh Pandya, Pritha Srivastava)
+* rgw: address bug where object puts could write to decommissioned shard (`pr#48663 <https://github.com/ceph/ceph/pull/48663>`_, J. Eric Ivancich)
+* rgw: better tenant id from the uri on anonymous access (`pr#47341 <https://github.com/ceph/ceph/pull/47341>`_, Rafał Wądołowski, Marcus Watts)
+* rgw: check bucket shard init status in RGWRadosBILogTrimCR (`pr#44907 <https://github.com/ceph/ceph/pull/44907>`_, Mykola Golub)
+* rgw: check object storage_class when check_disk_state (`pr#46579 <https://github.com/ceph/ceph/pull/46579>`_, Huber-ming)
+* rgw: data sync uses yield_spawn_window() (`pr#45713 <https://github.com/ceph/ceph/pull/45713>`_, Casey Bodley)
+* rgw: do not permit locked object version removal (`pr#47041 <https://github.com/ceph/ceph/pull/47041>`_, Igor Fedotov)
+* rgw: fix bool/int logic error when calling get_obj_head_ioctx (`pr#48230 <https://github.com/ceph/ceph/pull/48230>`_, J. Eric Ivancich)
+* rgw: fix bug where variable referenced after data moved out (`pr#48229 <https://github.com/ceph/ceph/pull/48229>`_, J. Eric Ivancich)
+* rgw: fix data corruption due to network jitter (`pr#48274 <https://github.com/ceph/ceph/pull/48274>`_, Shasha Lu)
+* rgw: Fix data race in ChangeStatus (`pr#47196 <https://github.com/ceph/ceph/pull/47196>`_, Adam C. Emerson)
+* rgw: fix ListBucketMultiparts response with common prefixes (`pr#44558 <https://github.com/ceph/ceph/pull/44558>`_, Casey Bodley)
+* rgw: fix segfault in OpsLogRados::log when realm is reloaded (`pr#45410 <https://github.com/ceph/ceph/pull/45410>`_, Cory Snyder)
+* rgw: fix self-comparison for RGWCopyObj optimization (`pr#43802 <https://github.com/ceph/ceph/pull/43802>`_, Casey Bodley)
+* rgw: Guard against malformed bucket URLs (`pr#47194 <https://github.com/ceph/ceph/pull/47194>`_, Adam C. Emerson)
+* rgw: initialize rgw_log_entry::identity_type (`pr#49142 <https://github.com/ceph/ceph/pull/49142>`_, Casey Bodley)
+* rgw: log access key id in ops logs (`pr#46622 <https://github.com/ceph/ceph/pull/46622>`_, Cory Snyder)
+* rgw: log deletion status of individual objects in multi object delete request (`pr#48348 <https://github.com/ceph/ceph/pull/48348>`_, Cory Snyder)
+* rgw: maintain object instance within RGWRadosObject::get_obj_state method (`pr#47266 <https://github.com/ceph/ceph/pull/47266>`_, Casey Bodley, Cory Snyder)
+* rgw: OpsLogFile::stop() signals under mutex (`pr#46039 <https://github.com/ceph/ceph/pull/46039>`_, Casey Bodley)
+* rgw: remove rgw_rados_pool_pg_num_min and its use on pool creation use the cluster defaults for pg_num_min (`pr#46235 <https://github.com/ceph/ceph/pull/46235>`_, Casey Bodley)
+* rgw: reopen ops log file on sighup (`pr#46619 <https://github.com/ceph/ceph/pull/46619>`_, Cory Snyder)
+* rgw: return OK on consecutive complete-multipart reqs (`pr#45486 <https://github.com/ceph/ceph/pull/45486>`_, Mark Kogan)
+* rgw: RGWCoroutine::set_sleeping() checks for null stack (`pr#46040 <https://github.com/ceph/ceph/pull/46040>`_, Or Friedmann, Casey Bodley)
+* rgw: splitting gc chains into smaller parts to prevent (`pr#48240 <https://github.com/ceph/ceph/pull/48240>`_, Pritha Srivastava)
+* rgw: x-amz-date change breaks certain cases of aws sig v4 (`pr#48313 <https://github.com/ceph/ceph/pull/48313>`_, Marcus Watts)
+* rgw: on FIPS enabled, fix segfault performing s3 multipart PUT (`pr#46715 <https://github.com/ceph/ceph/pull/46715>`_, Mark Kogan)
+* rgw_reshard: drop olh entries with empty name (`pr#45847 <https://github.com/ceph/ceph/pull/45847>`_, Dan van der Ster, Casey Bodley)
+* rgw_rest_user_policy: Fix GetUserPolicy & ListUserPolicies responses (`pr#47234 <https://github.com/ceph/ceph/pull/47234>`_, Sumedh A. Kulkarni)
+* rgwlc:  don't incorrectly expire delete markers when !next_key_name (`pr#47231 <https://github.com/ceph/ceph/pull/47231>`_, Matt Benjamin)
+* rgwlc: fix segfault resharding during lc (`pr#46744 <https://github.com/ceph/ceph/pull/46744>`_, Mark Kogan)
+* rpm: use system libpmem on Centos 9 Stream (`pr#46211 <https://github.com/ceph/ceph/pull/46211>`_, Ilya Dryomov)
+* run-make-check.sh: enable RBD persistent caches (`pr#45991 <https://github.com/ceph/ceph/pull/45991>`_, Ilya Dryomov)
+* SimpleRADOSStriper: Avoid moving bufferlists by using deque in read() (`pr#48187 <https://github.com/ceph/ceph/pull/48187>`_, Matan Breizman)
+* test/bufferlist: ensure rebuild_aligned_size_and_memory() always rebuilds (`pr#46215 <https://github.com/ceph/ceph/pull/46215>`_, Radoslaw Zarzynski)
+* test/cli-integration/rbd: iSCSI REST API responses aren't pretty-printed anymore (`pr#47920 <https://github.com/ceph/ceph/pull/47920>`_, Ilya Dryomov)
+* test/{librbd, rgw}: increase delay between and number of bind attempts (`pr#48024 <https://github.com/ceph/ceph/pull/48024>`_, Ilya Dryomov, Kefu Chai)
+* test: bump DecayCounter.steady acceptable error (`pr#48031 <https://github.com/ceph/ceph/pull/48031>`_, Patrick Donnelly)
+* test: fix TierFlushDuringFlush to wait until dedup_tier is set on bas… (`issue#53855 <http://tracker.ceph.com/issues/53855>`_, `pr#46748 <https://github.com/ceph/ceph/pull/46748>`_, Myoungwon Oh, Sungmin Lee)
+* test: No direct use of nose (`pr#46255 <https://github.com/ceph/ceph/pull/46255>`_, Steve Kowalik, Kefu Chai)
+* tooling: Change mrun to use bash (`pr#46077 <https://github.com/ceph/ceph/pull/46077>`_, Adam C. Emerson)
+* tools: ceph-objectstore-tool is able to trim solely pg log dups' entries (`pr#46631 <https://github.com/ceph/ceph/pull/46631>`_, Radosław Zarzyński, Radoslaw Zarzynski)
+* Updates to fix `make check` failures (`pr#47803 <https://github.com/ceph/ceph/pull/47803>`_, Tim Serong, Kefu Chai, Willem Jan Withagen, Nathan Cutler, Boris Ranto, Laura Flores, Pete Zaitcev)
+* v16.2.10 (`pr#47220 <https://github.com/ceph/ceph/pull/47220>`_, Kotresh HR, Seena Fallah)
+* v16.2.9 (`pr#46336 <https://github.com/ceph/ceph/pull/46336>`_, Cory Snyder)
+* win32_deps_build.sh: master -> main for wnbd (`pr#46762 <https://github.com/ceph/ceph/pull/46762>`_, Ilya Dryomov)
+
+v16.2.10 Pacific
+================
+
+This is a hotfix release that resolves two security flaws.
+
+Notable Changes
+---------------
+
+* Users who were running OpenStack Manila to export native CephFS and who
+  upgraded their Ceph cluster from Nautilus (or earlier) to a later
+  major version were vulnerable to an attack by malicious users. The
+  vulnerability allowed users to obtain access to arbitrary portions of
+  the CephFS filesystem hierarchy instead of being properly restricted
+  to their own subvolumes. The vulnerability is due to a bug in the
+  "volumes" plugin in Ceph Manager. This plugin is responsible for
+  managing Ceph File System subvolumes, which are used by OpenStack
+  Manila services as a way to provide shares to Manila users.
+
+  With this hotfix, the vulnerability is fixed. Administrators who are
+  concerned they may have been impacted should audit the CephX keys in
+  their cluster for proper path restrictions.
+
+  Again, this vulnerability impacts only OpenStack Manila clusters that
+  provided native CephFS access to their users.
+
+* A regression made it possible to dereference a null pointer for
+  s3website requests that don't refer to a bucket resulting in an RGW
+  segfault.
+
+Changelog
+---------
+* mgr/volumes: Fix subvolume discover during upgrade (:ref:`CVE-2022-0670`, Kotresh HR)
+* mgr/volumes: V2 Fix for test_subvolume_retain_snapshot_invalid_recreate (:ref:`CVE-2022-0670`, Kotresh HR)
+* qa: validate subvolume discover on upgrade (Kotresh HR)
+* rgw: s3website check for bucket before retargeting (Seena Fallah)
+
+v16.2.9 Pacific
+===============
+
+This is a hotfix release in the Pacific series to address a bug in 16.2.8 that could cause MGRs to deadlock. See https://tracker.ceph.com/issues/55687.
+
+Changelog
+---------
+
+* mgr/ActivePyModules.cc: fix cases where GIL is held while attempting to lock mutex (`pr#46302 <https://github.com/ceph/ceph/pull/46302>`_, Cory Snyder)
+
+v16.2.8 Pacific
+===============
+
+This is the eighth backport release in the Pacific series.
+
+Notable Changes
+---------------
+
+* MON/MGR: Pools can now be created with `--bulk` flag. Any pools created with `bulk`
+  will use a profile of the `pg_autoscaler` that provides more performance from the start.
+  However, any pools created without the `--bulk` flag will remain using it's old behavior
+  by default. For more details, see:
+
+  https://docs.ceph.com/en/latest/rados/operations/placement-groups/
+
+* MGR: The pg_autoscaler can now be turned `on` and `off` globally
+  with the `noautoscale` flag. By default this flag is unset and
+  the default pg_autoscale mode remains the same.
+  For more details, see:
+
+  https://docs.ceph.com/en/latest/rados/operations/placement-groups/
+
+* A health warning will now be reported if the ``require-osd-release`` flag is not
+  set to the appropriate release after a cluster upgrade.
+
+* CephFS: Upgrading Ceph Metadata Servers when using multiple active MDSs requires
+  ensuring no pending stray entries which are directories are present for active
+  ranks except rank 0. See :ref:`upgrading_from_octopus_or_nautilus`.
+
+Changelog
+---------
+
+* [Revert] bluestore: set upper and lower bounds on rocksdb omap iterators (`pr#46092 <https://github.com/ceph/ceph/pull/46092>`_, Neha Ojha)
+* admin/doc-requirements: bump sphinx to 4.4.0 (`pr#45876 <https://github.com/ceph/ceph/pull/45876>`_, Kefu Chai)
+* auth,mon: don't log "unable to find a keyring" error when key is given (`pr#43313 <https://github.com/ceph/ceph/pull/43313>`_, Ilya Dryomov)
+* backport nbd cookie support (`pr#45582 <https://github.com/ceph/ceph/pull/45582>`_, Prasanna Kumar Kalever)
+* backport of monitoring related PRs (`pr#45980 <https://github.com/ceph/ceph/pull/45980>`_, Pere Diaz Bou, Travis Nielsen, Aashish Sharma, Nizamudeen A, Arthur Outhenin-Chalandre)
+* bluestore: set upper and lower bounds on rocksdb omap iterators (`pr#45963 <https://github.com/ceph/ceph/pull/45963>`_, Cory Snyder)
+* build: Add some debugging messages (`pr#45753 <https://github.com/ceph/ceph/pull/45753>`_, David Galloway)
+* build: install-deps failing in docker build (`pr#45849 <https://github.com/ceph/ceph/pull/45849>`_, Nizamudeen A, Ernesto Puerta)
+* ceph-fuse: perform cleanup if test_dentry_handling failed (`pr#45351 <https://github.com/ceph/ceph/pull/45351>`_, Nikhilkumar Shelke)
+* ceph-volume: abort when passed devices have partitions (`pr#45146 <https://github.com/ceph/ceph/pull/45146>`_, Guillaume Abrioux)
+* ceph-volume: don't use MultiLogger in find_executable_on_host() (`pr#44701 <https://github.com/ceph/ceph/pull/44701>`_, Guillaume Abrioux)
+* ceph-volume: fix error 'KeyError' with inventory (`pr#44884 <https://github.com/ceph/ceph/pull/44884>`_, Guillaume Abrioux)
+* ceph-volume: fix regression introcuded via #43536 (`pr#44644 <https://github.com/ceph/ceph/pull/44644>`_, Guillaume Abrioux)
+* ceph-volume: fix tags dict output in `lvm list` (`pr#44767 <https://github.com/ceph/ceph/pull/44767>`_, Guillaume Abrioux)
+* ceph-volume: honour osd_dmcrypt_key_size option (`pr#44973 <https://github.com/ceph/ceph/pull/44973>`_, Guillaume Abrioux)
+* ceph-volume: human_readable_size() refactor (`pr#44209 <https://github.com/ceph/ceph/pull/44209>`_, Guillaume Abrioux)
+* ceph-volume: improve mpath devices support (`pr#44789 <https://github.com/ceph/ceph/pull/44789>`_, Guillaume Abrioux)
+* ceph-volume: make it possible to skip needs_root() (`pr#44319 <https://github.com/ceph/ceph/pull/44319>`_, Guillaume Abrioux)
+* ceph-volume: show RBD devices as not available (`pr#44708 <https://github.com/ceph/ceph/pull/44708>`_, Michael Fritch)
+* ceph/admin: s/master/main (`pr#45596 <https://github.com/ceph/ceph/pull/45596>`_, Zac Dover)
+* Cephadm Pacific Batch Backport April (`pr#45919 <https://github.com/ceph/ceph/pull/45919>`_, Adam King, Teoman ONAY, Redouane Kachach, Lukas Mayer, Melissa Li)
+* Cephadm Pacific Batch Backport March (`pr#45716 <https://github.com/ceph/ceph/pull/45716>`_, Adam King, Redouane Kachach, Matan Breizman, wangyunqing)
+* cephadm/ceph-volume: do not use lvm binary in containers (`pr#43954 <https://github.com/ceph/ceph/pull/43954>`_, Guillaume Abrioux, Sage Weil)
+* cephadm: _parse_ipv6_route: Fix parsing ifs w/o route (`pr#44877 <https://github.com/ceph/ceph/pull/44877>`_, Sebastian Wagner)
+* cephadm: add shared_ceph_folder opt to ceph-volume subcommand (`pr#44880 <https://github.com/ceph/ceph/pull/44880>`_, Guillaume Abrioux)
+* cephadm: check if cephadm is root after cli is parsed (`pr#44634 <https://github.com/ceph/ceph/pull/44634>`_, John Mulligan)
+* cephadm: chown the prometheus data dir during redeploy (`pr#45046 <https://github.com/ceph/ceph/pull/45046>`_, Michael Fritch)
+* cephadm: deal with ambiguity within normalize_image_digest (`pr#44632 <https://github.com/ceph/ceph/pull/44632>`_, Sebastian Wagner)
+* cephadm: fix broken telemetry documentation link (`pr#45803 <https://github.com/ceph/ceph/pull/45803>`_, Laura Flores)
+* cephadm: infer the default container image during pull (`pr#45569 <https://github.com/ceph/ceph/pull/45569>`_, Michael Fritch)
+* cephadm: make extract_uid_gid errors more readable (`pr#44528 <https://github.com/ceph/ceph/pull/44528>`_, Sebastian Wagner)
+* cephadm: November batch 2 (`pr#44446 <https://github.com/ceph/ceph/pull/44446>`_, Sage Weil, Adam King, Sebastian Wagner, Melissa Li, Michael Fritch, Guillaume Abrioux)
+* cephadm: pass `CEPH_VOLUME_SKIP_RESTORECON=yes` (backport) (`pr#44248 <https://github.com/ceph/ceph/pull/44248>`_, Guillaume Abrioux)
+* cephadm: preserve `authorized_keys` file during upgrade (`pr#45355 <https://github.com/ceph/ceph/pull/45355>`_, Michael Fritch)
+* cephadm: Remove containers pids-limit (`pr#45580 <https://github.com/ceph/ceph/pull/45580>`_, Ilya Dryomov, Teoman ONAY)
+* cephadm: revert pids limit (`pr#45936 <https://github.com/ceph/ceph/pull/45936>`_, Adam King)
+* cephadm: validate that the constructed YumDnf baseurl is usable (`pr#44882 <https://github.com/ceph/ceph/pull/44882>`_, John Mulligan)
+* cls/journal: skip disconnected clients when calculating min_commit_position (`pr#44690 <https://github.com/ceph/ceph/pull/44690>`_, Mykola Golub)
+* cls/rbd: GroupSnapshotNamespace comparator violates ordering rules (`pr#45075 <https://github.com/ceph/ceph/pull/45075>`_, Ilya Dryomov)
+* cmake/modules: always use the python3 specified in command line (`pr#45967 <https://github.com/ceph/ceph/pull/45967>`_, Kefu Chai)
+* cmake: pass RTE_DEVEL_BUILD=n when building dpdk (`pr#45262 <https://github.com/ceph/ceph/pull/45262>`_, Kefu Chai)
+* common/PriorityCache: low perf counters priorities for submodules (`pr#44175 <https://github.com/ceph/ceph/pull/44175>`_, Igor Fedotov)
+* common: avoid pthread_mutex_unlock twice (`pr#45464 <https://github.com/ceph/ceph/pull/45464>`_, Dai Zhiwei)
+* common: fix FTBFS due to dout & need_dynamic on GCC-12 (`pr#45373 <https://github.com/ceph/ceph/pull/45373>`_, Radoslaw Zarzynski)
+* common: fix missing name in PriorityCache perf counters (`pr#45588 <https://github.com/ceph/ceph/pull/45588>`_, Laura Flores)
+* common: replace BitVector::NoInitAllocator with wrapper struct (`pr#45179 <https://github.com/ceph/ceph/pull/45179>`_, Casey Bodley)
+* crush: Fix segfault in update_from_hook (`pr#44897 <https://github.com/ceph/ceph/pull/44897>`_, Adam Kupczyk)
+* doc/cephadm: Add CentOS Stream install instructions (`pr#44996 <https://github.com/ceph/ceph/pull/44996>`_, Patrick C. F. Ernzer)
+* doc/cephadm: Co-location of daemons (`pr#44879 <https://github.com/ceph/ceph/pull/44879>`_, Sebastian Wagner)
+* doc/cephadm: Doc backport (`pr#44525 <https://github.com/ceph/ceph/pull/44525>`_, Foad Lind, Sebastian Wagner)
+* doc/cephadm: improve the development doc a bit (`pr#44636 <https://github.com/ceph/ceph/pull/44636>`_, Radoslaw Zarzynski)
+* doc/cephadm: remove duplicate deployment scenario section (`pr#44660 <https://github.com/ceph/ceph/pull/44660>`_, Melissa Li)
+* doc/dev: s/repostory/repository/ (really) (`pr#45789 <https://github.com/ceph/ceph/pull/45789>`_, Zac Dover)
+* doc/start: add testing support information (`pr#45989 <https://github.com/ceph/ceph/pull/45989>`_, Zac Dover)
+* doc/start: include A. D'Atri's hardware-recs recs (`pr#45298 <https://github.com/ceph/ceph/pull/45298>`_, Zac Dover)
+* doc/start: remove journal info from hardware recs (`pr#45123 <https://github.com/ceph/ceph/pull/45123>`_, Zac Dover)
+* doc/start: remove osd stub from hardware recs (`pr#45316 <https://github.com/ceph/ceph/pull/45316>`_, Zac Dover)
+* doc: prerequisites fix for cephFS mount (`pr#44272 <https://github.com/ceph/ceph/pull/44272>`_, Nikhilkumar Shelke)
+* doc: Use older mistune (`pr#44226 <https://github.com/ceph/ceph/pull/44226>`_, David Galloway)
+* Enable autotune for osd_memory_target on bootstrap (`pr#44633 <https://github.com/ceph/ceph/pull/44633>`_, Melissa Li)
+* krbd: return error when no initial monitor address found (`pr#45003 <https://github.com/ceph/ceph/pull/45003>`_, Burt Holzman)
+* librados: check latest osdmap on ENOENT in pool_reverse_lookup() (`pr#45586 <https://github.com/ceph/ceph/pull/45586>`_, Ilya Dryomov)
+* librbd/cache/pwl: misc backports (`pr#44199 <https://github.com/ceph/ceph/pull/44199>`_, Jianpeng Ma, Jason Dillaman)
+* librbd: diff-iterate reports incorrect offsets in fast-diff mode (`pr#44547 <https://github.com/ceph/ceph/pull/44547>`_, Ilya Dryomov)
+* librbd: fix use-after-free on ictx in list_descendants() (`pr#44999 <https://github.com/ceph/ceph/pull/44999>`_, Ilya Dryomov, Wang ShuaiChao)
+* librbd: fix various memory leaks (`pr#44998 <https://github.com/ceph/ceph/pull/44998>`_, Or Ozeri)
+* librbd: make diff-iterate in fast-diff mode sort and merge reported extents (`pr#45638 <https://github.com/ceph/ceph/pull/45638>`_, Ilya Dryomov)
+* librbd: readv/writev fix iovecs length computation overflow (`pr#45561 <https://github.com/ceph/ceph/pull/45561>`_, Jonas Pfefferle)
+* librbd: restore diff-iterate include_parent functionality in fast-diff mode (`pr#44594 <https://github.com/ceph/ceph/pull/44594>`_, Ilya Dryomov)
+* librgw: make rgw file handle versioned (`pr#45495 <https://github.com/ceph/ceph/pull/45495>`_, Xuehan Xu)
+* librgw: treat empty root path as "/" on mount (`pr#43968 <https://github.com/ceph/ceph/pull/43968>`_, Matt Benjamin)
+* mds,client: add new getvxattr op (`pr#45487 <https://github.com/ceph/ceph/pull/45487>`_, Milind Changire)
+* mds: add mds_dir_max_entries config option (`pr#44512 <https://github.com/ceph/ceph/pull/44512>`_, Yongseok Oh)
+* mds: directly return just after responding the link request (`pr#44620 <https://github.com/ceph/ceph/pull/44620>`_, Xiubo Li)
+* mds: dump tree '/' when the path is empty (`pr#44622 <https://github.com/ceph/ceph/pull/44622>`_, Xiubo Li)
+* mds: ensure that we send the btime in cap messages (`pr#45163 <https://github.com/ceph/ceph/pull/45163>`_, Jeff Layton)
+* mds: fails to reintegrate strays if destdn's directory is full (ENOSPC) (`pr#44513 <https://github.com/ceph/ceph/pull/44513>`_, Patrick Donnelly)
+* mds: fix seg fault in expire_recursive (`pr#45099 <https://github.com/ceph/ceph/pull/45099>`_, 胡玮文)
+* mds: ignore unknown client op when tracking op latency (`pr#44975 <https://github.com/ceph/ceph/pull/44975>`_, Venky Shankar)
+* mds: kill session when mds do ms_handle_remote_reset (`issue#53911 <http://tracker.ceph.com/issues/53911>`_, `pr#45100 <https://github.com/ceph/ceph/pull/45100>`_, YunfeiGuan)
+* mds: mds_oft_prefetch_dirfrags default to false (`pr#45016 <https://github.com/ceph/ceph/pull/45016>`_, Dan van der Ster)
+* mds: opening connection to up:replay/up:creating daemon causes message drop (`pr#44296 <https://github.com/ceph/ceph/pull/44296>`_, Patrick Donnelly)
+* mds: PurgeQueue.cc fix for 32bit compilation (`pr#44168 <https://github.com/ceph/ceph/pull/44168>`_, Duncan Bellamy)
+* mds: recursive scrub does not trigger stray reintegration (`pr#44514 <https://github.com/ceph/ceph/pull/44514>`_, Patrick Donnelly)
+* mds: remove the duplicated or incorrect respond (`pr#44623 <https://github.com/ceph/ceph/pull/44623>`_, Xiubo Li)
+* mds: reset heartbeat in each MDSContext complete() (`pr#44551 <https://github.com/ceph/ceph/pull/44551>`_, Xiubo Li)
+* mgr/autoscaler: Introduce noautoscale flag (`pr#44540 <https://github.com/ceph/ceph/pull/44540>`_, Kamoltat)
+* mgr/cephadm/iscsi: use `mon_command` in `post_remove` instead of `check_mon_command` (`pr#44830 <https://github.com/ceph/ceph/pull/44830>`_, Melissa Li)
+* mgr/cephadm: Add client.admin keyring when upgrading from older version (`pr#44625 <https://github.com/ceph/ceph/pull/44625>`_, Sebastian Wagner)
+* mgr/cephadm: add keep-alive requests to ssh connections (`pr#45632 <https://github.com/ceph/ceph/pull/45632>`_, Adam King)
+* mgr/cephadm: Add snmp-gateway service support (`pr#44529 <https://github.com/ceph/ceph/pull/44529>`_, Sebastian Wagner, Paul Cuzner)
+* mgr/cephadm: allow miscellaneous container args at service level (`pr#44829 <https://github.com/ceph/ceph/pull/44829>`_, Adam King)
+* mgr/cephadm: auto-enable mirroring module when deploying service (`pr#44661 <https://github.com/ceph/ceph/pull/44661>`_, John Mulligan)
+* mgr/cephadm: avoid repeated calls to get_module_option (`pr#44535 <https://github.com/ceph/ceph/pull/44535>`_, Sage Weil)
+* mgr/cephadm: block draining last _admin host (`pr#45229 <https://github.com/ceph/ceph/pull/45229>`_, Adam King)
+* mgr/cephadm: block removing last instance of _admin label (`pr#45231 <https://github.com/ceph/ceph/pull/45231>`_, Adam King)
+* mgr/cephadm: Delete ceph.target if last cluster (`pr#45228 <https://github.com/ceph/ceph/pull/45228>`_, Redouane Kachach)
+* mgr/cephadm: extend extra_container_args to other service types (`pr#45234 <https://github.com/ceph/ceph/pull/45234>`_, Adam King)
+* mgr/cephadm: fix 'cephadm osd activate' on existing osd devices (`pr#44627 <https://github.com/ceph/ceph/pull/44627>`_, Sage Weil)
+* mgr/cephadm: fix 'mgr/cephadm: spec.virtual_ip  param should be used by the ingress daemon (`pr#44628 <https://github.com/ceph/ceph/pull/44628>`_, Guillaume Abrioux, Francesco Pantano, Sebastian Wagner)
+* mgr/cephadm: Fix count for OSDs with OSD specs (`pr#44629 <https://github.com/ceph/ceph/pull/44629>`_, Sebastian Wagner)
+* mgr/cephadm: fix minor grammar nit in Dry-Runs message (`pr#44637 <https://github.com/ceph/ceph/pull/44637>`_, James McClune)
+* mgr/cephadm: fix tcmu-runner cephadm_stray_daemon (`pr#44630 <https://github.com/ceph/ceph/pull/44630>`_, Melissa Li)
+* mgr/cephadm: Fix test_facts (`pr#44530 <https://github.com/ceph/ceph/pull/44530>`_, Sebastian Wagner)
+* mgr/cephadm: less log noise when config checks fail (`pr#44526 <https://github.com/ceph/ceph/pull/44526>`_, Sage Weil)
+* mgr/cephadm: nfs migration: avoid port conflicts (`pr#44631 <https://github.com/ceph/ceph/pull/44631>`_, Sebastian Wagner)
+* mgr/cephadm: Show an error when invalid format (`pr#45226 <https://github.com/ceph/ceph/pull/45226>`_, Redouane Kachach)
+* mgr/cephadm: store contianer registry credentials in config-key (`pr#44658 <https://github.com/ceph/ceph/pull/44658>`_, Daniel Pivonka)
+* mgr/cephadm: try to get FQDN for configuration files (`pr#45620 <https://github.com/ceph/ceph/pull/45620>`_, Tatjana Dehler)
+* mgr/cephadm: update monitoring stack versions (`pr#45940 <https://github.com/ceph/ceph/pull/45940>`_, Aashish Sharma, Ernesto Puerta)
+* mgr/cephadm: validating service_id for MDS (`pr#45227 <https://github.com/ceph/ceph/pull/45227>`_, Redouane Kachach)
+* mgr/dashboard: "Please expand your cluster first" shouldn't be shown if cluster is already meaningfully running (`pr#45044 <https://github.com/ceph/ceph/pull/45044>`_, Volker Theile)
+* mgr/dashboard: add test coverage for API docs (SwaggerUI) (`pr#44533 <https://github.com/ceph/ceph/pull/44533>`_, Alfonso Martínez)
+* mgr/dashboard: avoid tooltip if disk_usage=null and fast-diff enabled (`pr#44149 <https://github.com/ceph/ceph/pull/44149>`_, Avan Thakkar)
+* mgr/dashboard: cephadm e2e job improvements (`pr#44938 <https://github.com/ceph/ceph/pull/44938>`_, Nizamudeen A, Alfonso Martínez)
+* mgr/dashboard: cephadm e2e job: improvements (`pr#44382 <https://github.com/ceph/ceph/pull/44382>`_, Alfonso Martínez)
+* mgr/dashboard: change privacy protocol field from required to optional (`pr#45052 <https://github.com/ceph/ceph/pull/45052>`_, Avan Thakkar)
+* mgr/dashboard: Cluster Expansion - Review Section: fixes and improvements (`pr#44389 <https://github.com/ceph/ceph/pull/44389>`_, Aashish Sharma)
+* mgr/dashboard: Compare values of MTU alert by device (`pr#45813 <https://github.com/ceph/ceph/pull/45813>`_, Aashish Sharma, Patrick Seidensal)
+* mgr/dashboard: dashboard does not show degraded objects if they are less than 0.5% under "Dashboard->Capacity->Objects block (`pr#44091 <https://github.com/ceph/ceph/pull/44091>`_, Aashish Sharma)
+* mgr/dashboard: dashboard turns telemetry off when configuring report (`pr#45111 <https://github.com/ceph/ceph/pull/45111>`_, Sarthak0702, Aaryan Porwal)
+* mgr/dashboard: datatable in Cluster Host page hides wrong column on selection (`pr#45861 <https://github.com/ceph/ceph/pull/45861>`_, Sarthak0702)
+* mgr/dashboard: Directories Menu Can't Use on Ceph File System Dashboard (`pr#45028 <https://github.com/ceph/ceph/pull/45028>`_, Sarthak0702)
+* mgr/dashboard: extend daemon actions to host details (`pr#45721 <https://github.com/ceph/ceph/pull/45721>`_, Nizamudeen A)
+* mgr/dashboard: fix api test issue with pip (`pr#45880 <https://github.com/ceph/ceph/pull/45880>`_, Ernesto Puerta)
+* mgr/dashboard: fix frontend deps' vulnerabilities (`pr#44297 <https://github.com/ceph/ceph/pull/44297>`_, Alfonso Martínez)
+* mgr/dashboard: fix Grafana OSD/host panels (`pr#44775 <https://github.com/ceph/ceph/pull/44775>`_, Patrick Seidensal)
+* mgr/dashboard: fix orchestrator/02-hosts-inventory.e2e failure (`pr#44467 <https://github.com/ceph/ceph/pull/44467>`_, Nizamudeen A)
+* mgr/dashboard: fix timeout error in dashboard cephadm e2e job (`pr#44468 <https://github.com/ceph/ceph/pull/44468>`_, Nizamudeen A)
+* mgr/dashboard: fix white screen on Safari (`pr#45301 <https://github.com/ceph/ceph/pull/45301>`_, 胡玮文)
+* mgr/dashboard: fix: get SMART data from single-daemon device (`pr#44597 <https://github.com/ceph/ceph/pull/44597>`_, Alfonso Martínez)
+* mgr/dashboard: highlight the search text in cluster logs (`pr#45678 <https://github.com/ceph/ceph/pull/45678>`_, Sarthak0702)
+* mgr/dashboard: Implement drain host functionality in dashboard (`pr#44376 <https://github.com/ceph/ceph/pull/44376>`_, Nizamudeen A)
+* mgr/dashboard: Improve notifications for osd nearfull, full (`pr#44876 <https://github.com/ceph/ceph/pull/44876>`_, Aashish Sharma)
+* mgr/dashboard: Imrove error message of '/api/grafana/validation' API endpoint (`pr#45956 <https://github.com/ceph/ceph/pull/45956>`_, Volker Theile)
+* mgr/dashboard: introduce HAProxy metrics for RGW (`pr#44273 <https://github.com/ceph/ceph/pull/44273>`_, Avan Thakkar)
+* mgr/dashboard: introduce separate front-end component for API docs (`pr#44400 <https://github.com/ceph/ceph/pull/44400>`_, Aashish Sharma)
+* mgr/dashboard: Language dropdown box is partly hidden on login page (`pr#45618 <https://github.com/ceph/ceph/pull/45618>`_, Volker Theile)
+* mgr/dashboard: monitoring:Implement BlueStore onode hit/miss counters into the dashboard (`pr#44650 <https://github.com/ceph/ceph/pull/44650>`_, Aashish Sharma)
+* mgr/dashboard: NFS non-existent files cleanup (`pr#44046 <https://github.com/ceph/ceph/pull/44046>`_, Alfonso Martínez)
+* mgr/dashboard: NFS pages shows 'Page not found' (`pr#45723 <https://github.com/ceph/ceph/pull/45723>`_, Volker Theile)
+* mgr/dashboard: Notification banners at the top of the UI have fixed height (`pr#44756 <https://github.com/ceph/ceph/pull/44756>`_, Nizamudeen A, Waad AlKhoury)
+* mgr/dashboard: perform daemon actions (`pr#45203 <https://github.com/ceph/ceph/pull/45203>`_, Pere Diaz Bou)
+* mgr/dashboard: Pull latest translations from Transifex (`pr#45418 <https://github.com/ceph/ceph/pull/45418>`_, Volker Theile)
+* mgr/dashboard: Refactoring dashboard cephadm checks (`pr#44652 <https://github.com/ceph/ceph/pull/44652>`_, Nizamudeen A)
+* mgr/dashboard: RGW users and buckets tables are empty if the selected gateway is down (`pr#45868 <https://github.com/ceph/ceph/pull/45868>`_, Volker Theile)
+* mgr/dashboard: run-backend-api-tests.sh: Older setuptools (`pr#44377 <https://github.com/ceph/ceph/pull/44377>`_, David Galloway)
+* mgr/dashboard: set appropriate baseline branch for applitools (`pr#44935 <https://github.com/ceph/ceph/pull/44935>`_, Nizamudeen A)
+* mgr/dashboard: support snmp-gateway service creation from UI (`pr#44977 <https://github.com/ceph/ceph/pull/44977>`_, Avan Thakkar)
+* mgr/dashboard: Table columns hiding fix (`issue#51119 <http://tracker.ceph.com/issues/51119>`_, `pr#45725 <https://github.com/ceph/ceph/pull/45725>`_, Daniel Persson)
+* mgr/dashboard: Update Angular version to 12 (`pr#44534 <https://github.com/ceph/ceph/pull/44534>`_, Ernesto Puerta, Nizamudeen A)
+* mgr/dashboard: upgrade Cypress to the latest stable version (`pr#44086 <https://github.com/ceph/ceph/pull/44086>`_, Sage Weil, Alfonso Martínez)
+* mgr/dashboard: use -f for npm ci to skip fsevents error (`pr#44105 <https://github.com/ceph/ceph/pull/44105>`_, Duncan Bellamy)
+* mgr/devicehealth: fix missing timezone from time delta calculation (`pr#44325 <https://github.com/ceph/ceph/pull/44325>`_, Yaarit Hatuka)
+* mgr/devicehealth: skip null pages when extracting wear level (`pr#45151 <https://github.com/ceph/ceph/pull/45151>`_, Yaarit Hatuka)
+* mgr/nfs: allow dynamic update of cephfs nfs export (`pr#45543 <https://github.com/ceph/ceph/pull/45543>`_, Ramana Raja)
+* mgr/nfs: support managing exports without orchestration enabled (`pr#45508 <https://github.com/ceph/ceph/pull/45508>`_, John Mulligan)
+* mgr/orchestrator: add filtering and count option for orch host ls (`pr#44531 <https://github.com/ceph/ceph/pull/44531>`_, Adam King)
+* mgr/prometheus: Added `avail_raw` field for Pools DF Prometheus mgr module (`pr#45236 <https://github.com/ceph/ceph/pull/45236>`_, Konstantin Shalygin)
+* mgr/prometheus: define module options for standby (`pr#44205 <https://github.com/ceph/ceph/pull/44205>`_, Sage Weil)
+* mgr/prometheus: expose ceph healthchecks as metrics (`pr#44480 <https://github.com/ceph/ceph/pull/44480>`_, Paul Cuzner, Sebastian Wagner)
+* mgr/prometheus: Fix metric types from gauge to counter (`pr#43187 <https://github.com/ceph/ceph/pull/43187>`_, Patrick Seidensal)
+* mgr/prometheus: Fix the per method stats exported (`pr#44146 <https://github.com/ceph/ceph/pull/44146>`_, Paul Cuzner)
+* mgr/prometheus: Make prometheus standby behaviour configurable (`pr#43897 <https://github.com/ceph/ceph/pull/43897>`_, Roland Sommer)
+* mgr/rbd_support: cast pool_id from int to str when collecting LevelSpec (`pr#45532 <https://github.com/ceph/ceph/pull/45532>`_, Ilya Dryomov)
+* mgr/rbd_support: fix schedule remove (`pr#45005 <https://github.com/ceph/ceph/pull/45005>`_, Sunny Kumar)
+* mgr/snap_schedule: backports (`pr#45906 <https://github.com/ceph/ceph/pull/45906>`_, Venky Shankar, Milind Changire)
+* mgr/stats: exception handling for ceph fs perf stats command (`pr#44516 <https://github.com/ceph/ceph/pull/44516>`_, Nikhilkumar Shelke)
+* mgr/telemetry: fix waiting for mgr to warm up (`pr#45773 <https://github.com/ceph/ceph/pull/45773>`_, Yaarit Hatuka)
+* mgr/volumes: A few mgr volumes pacific backports (`pr#45205 <https://github.com/ceph/ceph/pull/45205>`_, Kotresh HR)
+* mgr/volumes: Subvolume removal and clone failure fixes (`pr#42932 <https://github.com/ceph/ceph/pull/42932>`_, Kotresh HR)
+* mgr/volumes: the 'mode' should honor idempotent subvolume creation (`pr#45474 <https://github.com/ceph/ceph/pull/45474>`_, Nikhilkumar Shelke)
+* mgr: Fix ceph_daemon label in ceph_rgw\_\* metrics (`pr#44885 <https://github.com/ceph/ceph/pull/44885>`_, Benoît Knecht)
+* mgr: fix locking for MetadataUpdate::finish (`pr#44212 <https://github.com/ceph/ceph/pull/44212>`_, Sage Weil)
+* mgr: TTL Cache in mgr module (`pr#44750 <https://github.com/ceph/ceph/pull/44750>`_, Waad AlKhoury, Pere Diaz Bou)
+* mgr: various fixes for mgr scalability (`pr#44869 <https://github.com/ceph/ceph/pull/44869>`_, Neha Ojha, Sage Weil)
+* mon/MDSMonitor: sanity assert when inline data turned on in MDSMap from v16.2.4 -> v16.2.[567] (`pr#44910 <https://github.com/ceph/ceph/pull/44910>`_, Patrick Donnelly)
+* mon/MgrStatMonitor: do not spam subscribers (mgr) with service_map (`pr#44721 <https://github.com/ceph/ceph/pull/44721>`_, Sage Weil)
+* mon/MonCommands.h: fix target_size_ratio range (`pr#45397 <https://github.com/ceph/ceph/pull/45397>`_, Kamoltat)
+* mon/OSDMonitor: avoid null dereference if stats are not available (`pr#44698 <https://github.com/ceph/ceph/pull/44698>`_, Josh Durgin)
+* mon: Abort device health when device not found (`pr#44959 <https://github.com/ceph/ceph/pull/44959>`_, Benoît Knecht)
+* mon: do not quickly mark mds laggy when MON_DOWN (`pr#43698 <https://github.com/ceph/ceph/pull/43698>`_, Sage Weil, Patrick Donnelly)
+* mon: Omit MANY_OBJECTS_PER_PG warning when autoscaler is on (`pr#45152 <https://github.com/ceph/ceph/pull/45152>`_, Christopher Hoffman)
+* mon: osd pool create <pool-name> with --bulk flag (`pr#44847 <https://github.com/ceph/ceph/pull/44847>`_, Kamoltat)
+* mon: prevent new sessions during shutdown (`pr#44543 <https://github.com/ceph/ceph/pull/44543>`_, Sage Weil)
+* monitoring/grafana: Grafana query tester (`pr#44316 <https://github.com/ceph/ceph/pull/44316>`_, Ernesto Puerta, Pere Diaz Bou)
+* monitoring: mention PyYAML only once in requirements (`pr#44944 <https://github.com/ceph/ceph/pull/44944>`_, Rishabh Dave)
+* os/bluestore/AvlAllocator: introduce bluestore_avl_alloc_ff_max\_\* options (`pr#43745 <https://github.com/ceph/ceph/pull/43745>`_, Kefu Chai, Mauricio Faria de Oliveira, Adam Kupczyk)
+* os/bluestore: avoid premature onode release (`pr#44723 <https://github.com/ceph/ceph/pull/44723>`_, Igor Fedotov)
+* os/bluestore: make shared blob fsck much less RAM-greedy (`pr#44613 <https://github.com/ceph/ceph/pull/44613>`_, Igor Fedotov)
+* os/bluestore: use proper prefix when removing undecodable Share Blob (`pr#43882 <https://github.com/ceph/ceph/pull/43882>`_, Igor Fedotov)
+* osd/OSD: Log aggregated slow ops detail to cluster logs (`pr#44771 <https://github.com/ceph/ceph/pull/44771>`_, Prashant D)
+* osd/OSDMap.cc: clean up pg_temp for nonexistent pgs (`pr#44096 <https://github.com/ceph/ceph/pull/44096>`_, Cory Snyder)
+* osd/OSDMap: Add health warning if 'require-osd-release' != current release (`pr#44259 <https://github.com/ceph/ceph/pull/44259>`_, Sridhar Seshasayee, Patrick Donnelly, Neha Ojha)
+* osd/OSDMapMapping: fix spurious threadpool timeout errors (`pr#44545 <https://github.com/ceph/ceph/pull/44545>`_, Sage Weil)
+* osd/PeeringState: separate history's pruub from pg's (`pr#44584 <https://github.com/ceph/ceph/pull/44584>`_, Sage Weil)
+* osd/PrimaryLogPG.cc: CEPH_OSD_OP_OMAPRMKEYRANGE should mark omap dirty (`pr#45591 <https://github.com/ceph/ceph/pull/45591>`_, Neha Ojha)
+* osd/scrub: destruct the scrubber shortly before the PG is destructed (`pr#45731 <https://github.com/ceph/ceph/pull/45731>`_, Ronen Friedman)
+* osd/scrub: only telling the scrubber of awaited-for 'updates' events (`pr#45365 <https://github.com/ceph/ceph/pull/45365>`_, Ronen Friedman)
+* osd/scrub: remove reliance of Scrubber objects' logging on the PG (`pr#45729 <https://github.com/ceph/ceph/pull/45729>`_, Ronen Friedman)
+* osd/scrub: restart snap trimming only after scrubbing is done (`pr#45785 <https://github.com/ceph/ceph/pull/45785>`_, Ronen Friedman)
+* osd/scrub: stop sending bogus digest-update events (`issue#54423 <http://tracker.ceph.com/issues/54423>`_, `pr#45194 <https://github.com/ceph/ceph/pull/45194>`_, Ronen Friedman)
+* osd/scrub: tag replica scrub messages to identify stale events (`pr#45374 <https://github.com/ceph/ceph/pull/45374>`_, Ronen Friedman)
+* osd: add pg_num_max value & pg_num_max reordering (`pr#45173 <https://github.com/ceph/ceph/pull/45173>`_, Kamoltat, Sage Weil)
+* osd: fix 'ceph osd stop <osd.nnn>' doesn't take effect (`pr#43955 <https://github.com/ceph/ceph/pull/43955>`_, tan changzhi)
+* osd: fix the truncation of an int by int division (`pr#45376 <https://github.com/ceph/ceph/pull/45376>`_, Ronen Friedman)
+* osd: PeeringState: fix selection order in calc_replicated_acting_stretch (`pr#44664 <https://github.com/ceph/ceph/pull/44664>`_, Greg Farnum)
+* osd: recover unreadable snapshot before reading ref. count info (`pr#44181 <https://github.com/ceph/ceph/pull/44181>`_, Myoungwon Oh)
+* osd: require osd_pg_max_concurrent_snap_trims > 0 (`pr#45323 <https://github.com/ceph/ceph/pull/45323>`_, Dan van der Ster)
+* osd: set r only if succeed in FillInVerifyExtent (`pr#44173 <https://github.com/ceph/ceph/pull/44173>`_, yanqiang-ux)
+* osdc: add set_error in BufferHead, when split set_error to right (`pr#44725 <https://github.com/ceph/ceph/pull/44725>`_, jiawd)
+* pacfic: doc/rados/operations/placement-groups: fix --bulk docs (`pr#45328 <https://github.com/ceph/ceph/pull/45328>`_, Kamoltat)
+* Pacific fast shutdown backports (`pr#45654 <https://github.com/ceph/ceph/pull/45654>`_, Sridhar Seshasayee, Nitzan Mordechai, Satoru Takeuchi)
+* pybind/mgr/balancer: define Plan.{dump,show}() (`pr#43964 <https://github.com/ceph/ceph/pull/43964>`_, Kefu Chai)
+* pybind/mgr/progress: enforced try and except on accessing event dictionary (`pr#44672 <https://github.com/ceph/ceph/pull/44672>`_, Kamoltat)
+* python-common: add int value validation for count and count_per_host (`pr#44527 <https://github.com/ceph/ceph/pull/44527>`_, John Mulligan)
+* python-common: improve OSD spec error messages (`pr#44626 <https://github.com/ceph/ceph/pull/44626>`_, Sebastian Wagner)
+* qa/distros/podman: remove centos_8.2 and centos_8.3 (`pr#44903 <https://github.com/ceph/ceph/pull/44903>`_, Neha Ojha)
+* qa/rgw: add failing tempest test to blocklist (`pr#45436 <https://github.com/ceph/ceph/pull/45436>`_, Casey Bodley)
+* qa/rgw: barbican and pykmip tasks upgrade pip before installing pytz (`pr#45444 <https://github.com/ceph/ceph/pull/45444>`_, Casey Bodley)
+* qa/rgw: bump tempest version to resolve dependency issue (`pr#43966 <https://github.com/ceph/ceph/pull/43966>`_, Casey Bodley)
+* qa/rgw: Fix vault token file access (`issue#51539 <http://tracker.ceph.com/issues/51539>`_, `pr#43951 <https://github.com/ceph/ceph/pull/43951>`_, Marcus Watts)
+* qa/rgw: update apache-maven mirror for rgw/hadoop-s3a (`pr#45445 <https://github.com/ceph/ceph/pull/45445>`_, Casey Bodley)
+* qa/rgw: use symlinks for rgw/sts suite, target supported-random-distro$ (`pr#45245 <https://github.com/ceph/ceph/pull/45245>`_, Casey Bodley)
+* qa/run-tox-mgr-dashboard: Do not write to /tmp/test_sanitize_password… (`pr#44727 <https://github.com/ceph/ceph/pull/44727>`_, Kevin Zhao)
+* qa/run_xfstests_qemu.sh: stop reporting success without actually running any tests (`pr#44596 <https://github.com/ceph/ceph/pull/44596>`_, Ilya Dryomov)
+* qa/suites/fs: add prefetch_dirfrags false to thrasher suite (`pr#44504 <https://github.com/ceph/ceph/pull/44504>`_, Arthur Outhenin-Chalandre)
+* qa/suites/orch/cephadm: Also run the rbd/iscsi suite (`pr#44635 <https://github.com/ceph/ceph/pull/44635>`_, Sebastian Wagner)
+* qa/tasks/qemu: make sure block-rbd.so is installed (`pr#45072 <https://github.com/ceph/ceph/pull/45072>`_, Ilya Dryomov)
+* qa/tasks: improve backfill_toofull test (`pr#44387 <https://github.com/ceph/ceph/pull/44387>`_, Mykola Golub)
+* qa/tests: added upgrade-clients/client-upgrade-pacific-quincy test (`pr#45326 <https://github.com/ceph/ceph/pull/45326>`_, Yuri Weinstein)
+* qa/tests: replaced 16.2.6 with 16.2.7 version (`pr#44369 <https://github.com/ceph/ceph/pull/44369>`_, Yuri Weinstein)
+* qa: adjust for MDSs to get deployed before verifying their availability (`issue#53857 <http://tracker.ceph.com/issues/53857>`_, `pr#44639 <https://github.com/ceph/ceph/pull/44639>`_, Venky Shankar)
+* qa: Default to CentOS 8 Stream (`pr#44889 <https://github.com/ceph/ceph/pull/44889>`_, David Galloway)
+* qa: do not use any time related suffix for \*_op_timeouts (`pr#44621 <https://github.com/ceph/ceph/pull/44621>`_, Xiubo Li)
+* qa: fsync dir for asynchronous creat on stray tests (`pr#45565 <https://github.com/ceph/ceph/pull/45565>`_, Patrick Donnelly, Ramana Raja)
+* qa: ignore expected metadata cluster log error (`pr#45564 <https://github.com/ceph/ceph/pull/45564>`_, Patrick Donnelly)
+* qa: increase the timeout value to wait a litte longer (`pr#43979 <https://github.com/ceph/ceph/pull/43979>`_, Xiubo Li)
+* qa: move certificates for kmip task into /etc/ceph (`pr#45413 <https://github.com/ceph/ceph/pull/45413>`_, Ali Maredia)
+* qa: remove centos8 from supported distros (`pr#44865 <https://github.com/ceph/ceph/pull/44865>`_, Casey Bodley, Sage Weil)
+* qa: skip sanity check during upgrade (`pr#44840 <https://github.com/ceph/ceph/pull/44840>`_, Milind Changire)
+* qa: split distro for rados/cephadm/smoke tests (`pr#44681 <https://github.com/ceph/ceph/pull/44681>`_, Guillaume Abrioux)
+* qa: wait for purge queue operations to finish (`issue#52487 <http://tracker.ceph.com/issues/52487>`_, `pr#44642 <https://github.com/ceph/ceph/pull/44642>`_, Venky Shankar)
+* radosgw-admin: 'sync status' is not behind if there are no mdlog entries (`pr#45442 <https://github.com/ceph/ceph/pull/45442>`_, Casey Bodley)
+* rbd persistent cache UX improvements (status report, metrics, flush command) (`pr#45895 <https://github.com/ceph/ceph/pull/45895>`_, Ilya Dryomov, Yin Congmin)
+* rbd-mirror: fix races in snapshot-based mirroring deletion propagation (`pr#44754 <https://github.com/ceph/ceph/pull/44754>`_, Ilya Dryomov)
+* rbd-mirror: make mirror properly detect pool replayer needs restart (`pr#45170 <https://github.com/ceph/ceph/pull/45170>`_, Mykola Golub)
+* rbd-mirror: make RemoveImmediateUpdate test synchronous (`pr#44094 <https://github.com/ceph/ceph/pull/44094>`_, Arthur Outhenin-Chalandre)
+* rbd-mirror: synchronize with in-flight stop in ImageReplayer::stop() (`pr#45184 <https://github.com/ceph/ceph/pull/45184>`_, Ilya Dryomov)
+* rbd: add missing switch arguments for recognition by get_command_spec() (`pr#44742 <https://github.com/ceph/ceph/pull/44742>`_, Ilya Dryomov)
+* rbd: mark optional positional arguments as such in help output (`pr#45008 <https://github.com/ceph/ceph/pull/45008>`_, Ilya Dryomov)
+* rbd: recognize rxbounce map option (`pr#45002 <https://github.com/ceph/ceph/pull/45002>`_, Ilya Dryomov)
+* Revert "mds: kill session when mds do ms_handle_remote_reset" (`pr#45557 <https://github.com/ceph/ceph/pull/45557>`_, Venky Shankar)
+* revert bootstrap network handling changes (`pr#46085 <https://github.com/ceph/ceph/pull/46085>`_, Adam King)
+* revival and backport of fix for RocksDB optimized iterators (`pr#46096 <https://github.com/ceph/ceph/pull/46096>`_, Adam Kupczyk, Cory Snyder)
+* RGW - Zipper - Make default args match in get_obj_state (`pr#45438 <https://github.com/ceph/ceph/pull/45438>`_, Daniel Gryniewicz)
+* RGW - Zipper - Make sure PostObj has bucket set (`pr#45060 <https://github.com/ceph/ceph/pull/45060>`_, Daniel Gryniewicz)
+* rgw/admin: fix radosgw-admin datalog list max-entries issue (`pr#45500 <https://github.com/ceph/ceph/pull/45500>`_, Yuval Lifshitz)
+* rgw/amqp: add default case to silence compiler warning (`pr#45478 <https://github.com/ceph/ceph/pull/45478>`_, Casey Bodley)
+* rgw/amqp: remove the explicit "disconnect()" interface (`pr#45427 <https://github.com/ceph/ceph/pull/45427>`_, Yuval Lifshitz)
+* rgw/beast: optimizations for request timeout (`pr#43946 <https://github.com/ceph/ceph/pull/43946>`_, Mark Kogan, Casey Bodley)
+* rgw/notification: send correct size in COPY events (`pr#45426 <https://github.com/ceph/ceph/pull/45426>`_, Yuval Lifshitz)
+* rgw/sts: adding role name and role session to ops log (`pr#43956 <https://github.com/ceph/ceph/pull/43956>`_, Pritha Srivastava)
+* rgw: add object null point judging when listing pubsub  topics (`pr#45476 <https://github.com/ceph/ceph/pull/45476>`_, zhipeng li)
+* rgw: add OPT_BUCKET_SYNC_RUN to gc_ops_list, so that (`pr#45421 <https://github.com/ceph/ceph/pull/45421>`_, Pritha Srivastava)
+* rgw: add the condition of lock mode conversion to PutObjRentention (`pr#45440 <https://github.com/ceph/ceph/pull/45440>`_, wangzhong)
+* rgw: bucket chown bad memory usage (`pr#45491 <https://github.com/ceph/ceph/pull/45491>`_, Mohammad Fatemipour)
+* rgw: change order of xml elements in ListRoles response (`pr#45448 <https://github.com/ceph/ceph/pull/45448>`_, Casey Bodley)
+* rgw: clean-up logging of function entering to make thorough and consistent (`pr#45450 <https://github.com/ceph/ceph/pull/45450>`_, J. Eric Ivancich)
+* rgw: cls_bucket_list_unordered() might return one redundent entry every time is_truncated is true (`pr#45457 <https://github.com/ceph/ceph/pull/45457>`_, Peng Zhang)
+* rgw: default ms_mon_client_mode = secure (`pr#45439 <https://github.com/ceph/ceph/pull/45439>`_, Sage Weil)
+* rgw: document rgw_lc_debug_interval configuration option (`pr#45453 <https://github.com/ceph/ceph/pull/45453>`_, J. Eric Ivancich)
+* rgw: document S3 bucket replication support (`pr#45484 <https://github.com/ceph/ceph/pull/45484>`_, Matt Benjamin)
+* rgw: Dump Object Lock Retain Date as ISO 8601 (`pr#44697 <https://github.com/ceph/ceph/pull/44697>`_, Danny Abukalam)
+* rgw: fix `bi put` not using right bucket index shard (`pr#44166 <https://github.com/ceph/ceph/pull/44166>`_, J. Eric Ivancich)
+* rgw: fix lock scope in ObjectCache::get() (`pr#44747 <https://github.com/ceph/ceph/pull/44747>`_, Casey Bodley)
+* rgw: fix md5 not match for RGWBulkUploadOp upload when enable rgw com… (`pr#45432 <https://github.com/ceph/ceph/pull/45432>`_, yuliyang_yewu)
+* rgw: fix rgw.none statistics (`pr#45463 <https://github.com/ceph/ceph/pull/45463>`_, J. Eric Ivancich)
+* rgw: fix segfault in UserAsyncRefreshHandler::init_fetch (`pr#45411 <https://github.com/ceph/ceph/pull/45411>`_, Cory Snyder)
+* rgw: forward request in multisite for RGWDeleteBucketPolicy and RGWDeleteBucketPublicAccessBlock (`pr#45434 <https://github.com/ceph/ceph/pull/45434>`_, yuliyang_yewu)
+* rgw: have "bucket check --fix" fix pool ids correctly (`pr#45455 <https://github.com/ceph/ceph/pull/45455>`_, J. Eric Ivancich)
+* rgw: in bucket reshard list, clarify new num shards is tentative (`pr#45509 <https://github.com/ceph/ceph/pull/45509>`_, J. Eric Ivancich)
+* rgw: init bucket index only if putting bucket instance info succeeds (`pr#45480 <https://github.com/ceph/ceph/pull/45480>`_, Huber-ming)
+* rgw: RadosBucket::get_bucket_info() updates RGWBucketEnt (`pr#45483 <https://github.com/ceph/ceph/pull/45483>`_, Casey Bodley)
+* rgw: remove bucket API returns NoSuchKey than NoSuchBucket (`pr#45489 <https://github.com/ceph/ceph/pull/45489>`_, Satoru Takeuchi)
+* rgw: resolve empty ordered bucket listing results w/ CLS filtering \*and\* bucket index list produces incorrect result when non-ascii entries (`pr#45087 <https://github.com/ceph/ceph/pull/45087>`_, J. Eric Ivancich)
+* rgw: RGWPostObj::execute() may lost data (`pr#45502 <https://github.com/ceph/ceph/pull/45502>`_, Lei Zhang)
+* rgw: under fips, set flag to allow md5 in select rgw ops (`pr#44778 <https://github.com/ceph/ceph/pull/44778>`_, Mark Kogan)
+* rgw: url_decode before parsing copysource in copyobject (`issue#43259 <http://tracker.ceph.com/issues/43259>`_, `pr#45430 <https://github.com/ceph/ceph/pull/45430>`_, Paul Reece)
+* rgw: user stats showing 0 value for "size_utilized" and "size_kb_utilized" fields (`pr#44171 <https://github.com/ceph/ceph/pull/44171>`_, J. Eric Ivancich)
+* rgw: write meta of a MP part to a correct pool (`issue#49128 <http://tracker.ceph.com/issues/49128>`_, `pr#45428 <https://github.com/ceph/ceph/pull/45428>`_, Jeegn Chen)
+* rgw:When KMS encryption is used and the key does not exist, we should… (`pr#45461 <https://github.com/ceph/ceph/pull/45461>`_, wangyingbin)
+* rgwlc:  remove lc entry on bucket delete (`pr#44729 <https://github.com/ceph/ceph/pull/44729>`_, Matt Benjamin)
+* rgwlc:  warn on missing RGW_ATTR_LC (`pr#45497 <https://github.com/ceph/ceph/pull/45497>`_, Matt Benjamin)
+* src/ceph-crash.in: various enhancements and fixes (`pr#45381 <https://github.com/ceph/ceph/pull/45381>`_, Sébastien Han)
+* src/rgw: Fix for malformed url (`pr#45459 <https://github.com/ceph/ceph/pull/45459>`_, Kalpesh Pandya)
+* test/librbd/test_notify.py: effect post object map rebuild assert (`pr#45311 <https://github.com/ceph/ceph/pull/45311>`_, Ilya Dryomov)
+* test/librbd: add test to verify diff_iterate size (`pr#45555 <https://github.com/ceph/ceph/pull/45555>`_, Christopher Hoffman)
+* test/librbd: harden RemoveFullTry tests (`pr#43649 <https://github.com/ceph/ceph/pull/43649>`_, Ilya Dryomov)
+* test/rgw: disable cls_rgw_gc test cases with defer_gc() (`pr#45477 <https://github.com/ceph/ceph/pull/45477>`_, Casey Bodley)
+* test: fix wrong alarm (HitSetWrite) (`pr#45319 <https://github.com/ceph/ceph/pull/45319>`_, Myoungwon Oh)
+* test: increase retry duration when calculating manifest ref. count (`pr#44202 <https://github.com/ceph/ceph/pull/44202>`_, Myoungwon Oh)
+* tools/rbd: expand where option rbd_default_map_options can be set (`pr#45181 <https://github.com/ceph/ceph/pull/45181>`_, Christopher Hoffman, Ilya Dryomov)
+* Wip doc pr 46109 backport to pacific (`pr#46117 <https://github.com/ceph/ceph/pull/46117>`_, Ville Ojamo)
+
+
+v16.2.7 Pacific
+===============
+
+This is the seventh backport release in the Pacific series.
+
+Notable Changes
+---------------
+
+* Critical bug in OMAP format upgrade is fixed. This could cause data corruption
+  (improperly formatted OMAP keys) after pre-Pacific cluster upgrade if
+  bluestore-quick-fix-on-mount parameter is set to true or ceph-bluestore-tool's
+  quick-fix/repair commands are invoked.
+  Relevant tracker: https://tracker.ceph.com/issues/53062
+  ``bluestore-quick-fix-on-mount`` continues to be set to false, by default.
+
+* CephFS:  If you are not using cephadm, you must disable FSMap sanity checks *before starting the upgrade*::
+
+      ceph config set mon mon_mds_skip_sanity true
+
+  After the upgrade has finished and the cluster is stable, please remove that setting::
+
+      ceph config rm mon mon_mds_skip_sanity
+
+  Clusters managed by and upgraded using cephadm take care of this step automatically.
+
+* MGR: The pg_autoscaler will use the 'scale-up' profile as the default profile.
+  16.2.6 changed the default profile to 'scale-down' but we ran into issues
+  with the device_health_metrics pool consuming too many PGs, which is not ideal
+  for performance. So we will continue to use the 'scale-up' profile by default,
+  until we implement a limit on the number of PGs default pools should consume,
+  in combination with the 'scale-down' profile.
+
+* Cephadm & Ceph Dashboard: NFS management has been completely reworked to
+  ensure that NFS exports are managed consistently across the different Ceph
+  components. Prior to this, there were 3 incompatible implementations for
+  configuring the NFS exports: Ceph-Ansible/OpenStack Manila, Ceph Dashboard and
+  'mgr/nfs' module. With this release the 'mgr/nfs' way becomes the official
+  interface, and the remaining components (Cephadm and Ceph Dashboard) adhere to
+  it. While this might require manually migrating from the deprecated
+  implementations, it will simplify the user experience for those heavily
+  relying on NFS exports.
+
+* Dashboard: "Cluster Expansion Wizard". After the 'cephadm bootstrap' step,
+  users that log into the Ceph Dashboard will be presented with a welcome
+  screen. If they choose to follow the installation wizard, they will be guided
+  through a set of steps to help them configure their Ceph cluster: expanding
+  the cluster by adding more hosts, detecting and defining their storage
+  devices, and finally deploying and configuring the different Ceph services.
+
+* OSD: When using mclock_scheduler for QoS, there is no longer a need to run any
+  manual benchmark. The OSD now automatically sets an appropriate value for
+  `osd_mclock_max_capacity_iops` by running a simple benchmark during
+  initialization.
+
+* MGR: The global recovery event in the progress module has been optimized and
+  a `sleep_interval` of 5 seconds has been added between stats collection,
+  to reduce the impact of the progress module on the MGR, especially in large
+  clusters.
+
+Changelog
+---------
+
+* rpm, debian: move smartmontools and nvme-cli to ceph-base (`pr#44164 <https://github.com/ceph/ceph/pull/44164>`_, Yaarit Hatuka)
+* qa: miscellaneous perf suite fixes (`pr#44154 <https://github.com/ceph/ceph/pull/44154>`_, Neha Ojha)
+* qa/suites/orch/cephadm: mgr-nfs-upgrade: add missing 0-distro dir (`pr#44201 <https://github.com/ceph/ceph/pull/44201>`_, Sebastian Wagner)
+* \*: s/virtualenv/python -m venv/ (`pr#43002 <https://github.com/ceph/ceph/pull/43002>`_, Kefu Chai, Ken Dreyer)
+* admin/doc-requirements.txt: pin Sphinx at 3.5.4 (`pr#43748 <https://github.com/ceph/ceph/pull/43748>`_, Kefu Chai)
+* backport mgr/nfs bits (`pr#43811 <https://github.com/ceph/ceph/pull/43811>`_, Sage Weil, Michael Fritch)
+* ceph-volume: `get_first_lv()` refactor (`pr#43960 <https://github.com/ceph/ceph/pull/43960>`_, Guillaume Abrioux)
+* ceph-volume: fix a typo causing AttributeError (`pr#43949 <https://github.com/ceph/ceph/pull/43949>`_, Taha Jahangir)
+* ceph-volume: fix bug with miscalculation of required db/wal slot size for VGs with multiple PVs (`pr#43948 <https://github.com/ceph/ceph/pull/43948>`_, Guillaume Abrioux, Cory Snyder)
+* ceph-volume: fix lvm activate --all --no-systemd (`pr#43267 <https://github.com/ceph/ceph/pull/43267>`_, Dimitri Savineau)
+* ceph-volume: util/prepare fix osd_id_available() (`pr#43708 <https://github.com/ceph/ceph/pull/43708>`_, Guillaume Abrioux)
+* ceph.spec: selinux scripts respect CEPH_AUTO_RESTART_ON_UPGRADE (`pr#43235 <https://github.com/ceph/ceph/pull/43235>`_, Dan van der Ster)
+* cephadm: November batch (`pr#43906 <https://github.com/ceph/ceph/pull/43906>`_, Sebastian Wagner, Sage Weil, Daniel Pivonka, Andrew Sharapov, Paul Cuzner, Adam King, Melissa Li)
+* cephadm: October batch (`pr#43728 <https://github.com/ceph/ceph/pull/43728>`_, Patrick Donnelly, Sage Weil, Cory Snyder, Sebastian Wagner, Paul Cuzner, Joao Eduardo Luis, Zac Dover, Dmitry Kvashnin, Daniel Pivonka, Adam King, jianglong01, Guillaume Abrioux, Melissa Li, Roaa Sakr, Kefu Chai, Brad Hubbard, Michael Fritch, Javier Cacheiro)
+* cephfs-mirror, test: add thrasher for cephfs mirror daemon, HA test yamls (`issue#50372 <http://tracker.ceph.com/issues/50372>`_, `pr#43924 <https://github.com/ceph/ceph/pull/43924>`_, Venky Shankar)
+* cephfs-mirror: shutdown ClusterWatcher on termination (`pr#43198 <https://github.com/ceph/ceph/pull/43198>`_, Willem Jan Withagen, Venky Shankar)
+* cmake: link Threads::Threads instead of CMAKE_THREAD_LIBS_INIT (`pr#43167 <https://github.com/ceph/ceph/pull/43167>`_, Ken Dreyer)
+* cmake: s/Python_EXECUTABLE/Python3_EXECUTABLE/ (`pr#43264 <https://github.com/ceph/ceph/pull/43264>`_, Michael Fritch)
+* crush: cancel upmaps with up set size != pool size (`pr#43415 <https://github.com/ceph/ceph/pull/43415>`_, huangjun)
+* doc/radosgw/nfs: add note about NFSv3 deprecation (`pr#43941 <https://github.com/ceph/ceph/pull/43941>`_, Michael Fritch)
+* doc: document subvolume (group) pins (`pr#43925 <https://github.com/ceph/ceph/pull/43925>`_, Patrick Donnelly)
+* github: add dashboard PRs to Dashboard project (`pr#43610 <https://github.com/ceph/ceph/pull/43610>`_, Ernesto Puerta)
+* librbd/cache/pwl: persistant cache backports (`pr#43772 <https://github.com/ceph/ceph/pull/43772>`_, Kefu Chai, Yingxin Cheng, Yin Congmin, Feng Hualong, Jianpeng Ma, Ilya Dryomov, Hualong Feng)
+* librbd/cache/pwl: SSD caching backports (`pr#43918 <https://github.com/ceph/ceph/pull/43918>`_, Yin Congmin, Jianpeng Ma)
+* librbd/object_map: rbd diff between two snapshots lists entire image content (`pr#43805 <https://github.com/ceph/ceph/pull/43805>`_, Sunny Kumar)
+* librbd: fix pool validation lockup (`pr#43113 <https://github.com/ceph/ceph/pull/43113>`_, Ilya Dryomov)
+* mds/FSMap: do not assert allow_standby_replay on old FSMaps (`pr#43614 <https://github.com/ceph/ceph/pull/43614>`_, Patrick Donnelly)
+* mds: Add new flag to MClientSession (`pr#43251 <https://github.com/ceph/ceph/pull/43251>`_, Kotresh HR)
+* mds: do not trim stray dentries during opening the root (`pr#43815 <https://github.com/ceph/ceph/pull/43815>`_, Xiubo Li)
+* mds: skip journaling blocklisted clients when in `replay` state (`pr#43841 <https://github.com/ceph/ceph/pull/43841>`_, Venky Shankar)
+* mds: switch mds_lock to fair mutex to fix the slow performance issue (`pr#43148 <https://github.com/ceph/ceph/pull/43148>`_, Xiubo Li, Kefu Chai)
+* MDSMonitor: assertion during upgrade to v16.2.5+ (`pr#43890 <https://github.com/ceph/ceph/pull/43890>`_, Patrick Donnelly)
+* MDSMonitor: handle damaged state from standby-replay (`pr#43200 <https://github.com/ceph/ceph/pull/43200>`_, Patrick Donnelly)
+* MDSMonitor: no active MDS after cluster deployment (`pr#43891 <https://github.com/ceph/ceph/pull/43891>`_, Patrick Donnelly)
+* mgr/dashboard,prometheus: fix handling of server_addr (`issue#52002 <http://tracker.ceph.com/issues/52002>`_, `pr#43631 <https://github.com/ceph/ceph/pull/43631>`_, Scott Shambarger)
+* mgr/dashboard: all pyfakefs must be pinned on same version (`pr#43930 <https://github.com/ceph/ceph/pull/43930>`_, Rishabh Dave)
+* mgr/dashboard: BATCH incl.: NFS integration, Cluster Expansion Workflow, and Angular 11 upgrade (`pr#43682 <https://github.com/ceph/ceph/pull/43682>`_, Alfonso Martínez, Avan Thakkar, Aashish Sharma, Nizamudeen A, Pere Diaz Bou, Varsha Rao, Ramana Raja, Sage Weil, Kefu Chai)
+* mgr/dashboard: cephfs MDS Workload to use rate for counter type metric (`pr#43190 <https://github.com/ceph/ceph/pull/43190>`_, Jan Horacek)
+* mgr/dashboard: clean-up controllers and API backward versioning compatibility (`pr#43543 <https://github.com/ceph/ceph/pull/43543>`_, Ernesto Puerta, Avan Thakkar)
+* mgr/dashboard: Daemon Events listing using bootstrap class (`pr#44057 <https://github.com/ceph/ceph/pull/44057>`_, Nizamudeen A)
+* mgr/dashboard: deprecated variable usage in Grafana dashboards (`pr#43188 <https://github.com/ceph/ceph/pull/43188>`_, Patrick Seidensal)
+* mgr/dashboard: Device health status is not getting listed under hosts section (`pr#44053 <https://github.com/ceph/ceph/pull/44053>`_, Aashish Sharma)
+* mgr/dashboard: Edit a service feature (`pr#43939 <https://github.com/ceph/ceph/pull/43939>`_, Nizamudeen A)
+* mgr/dashboard: Fix failing config dashboard e2e check (`pr#43238 <https://github.com/ceph/ceph/pull/43238>`_, Nizamudeen A)
+* mgr/dashboard: fix flaky inventory e2e test (`pr#44056 <https://github.com/ceph/ceph/pull/44056>`_, Nizamudeen A)
+* mgr/dashboard: fix missing alert rule details (`pr#43812 <https://github.com/ceph/ceph/pull/43812>`_, Ernesto Puerta)
+* mgr/dashboard: Fix orchestrator/01-hosts.e2e-spec.ts failure (`pr#43541 <https://github.com/ceph/ceph/pull/43541>`_, Nizamudeen A)
+* mgr/dashboard: include mfa_ids in rgw user-details section (`pr#43893 <https://github.com/ceph/ceph/pull/43893>`_, Avan Thakkar)
+* mgr/dashboard: Incorrect MTU mismatch warning (`pr#43185 <https://github.com/ceph/ceph/pull/43185>`_, Aashish Sharma)
+* mgr/dashboard: monitoring: grafonnet refactoring for radosgw dashboards (`pr#43644 <https://github.com/ceph/ceph/pull/43644>`_, Aashish Sharma)
+* mgr/dashboard: Move force maintenance test to the workflow test suite (`pr#43347 <https://github.com/ceph/ceph/pull/43347>`_, Nizamudeen A)
+* mgr/dashboard: pin a version for autopep8 and pyfakefs (`pr#43646 <https://github.com/ceph/ceph/pull/43646>`_, Nizamudeen A)
+* mgr/dashboard: Predefine labels in create host form (`pr#44077 <https://github.com/ceph/ceph/pull/44077>`_, Nizamudeen A)
+* mgr/dashboard: provisioned values is misleading in RBD image table (`pr#44051 <https://github.com/ceph/ceph/pull/44051>`_, Avan Thakkar)
+* mgr/dashboard: replace "Ceph-cluster" Client connections with active-standby MGRs (`pr#43523 <https://github.com/ceph/ceph/pull/43523>`_, Avan Thakkar)
+* mgr/dashboard: rgw daemon list: add realm column (`pr#44047 <https://github.com/ceph/ceph/pull/44047>`_, Alfonso Martínez)
+* mgr/dashboard: Spelling mistake in host-form Network address field (`pr#43973 <https://github.com/ceph/ceph/pull/43973>`_, Avan Thakkar)
+* mgr/dashboard: Visual regression tests for ceph dashboard (`pr#42678 <https://github.com/ceph/ceph/pull/42678>`_, Aaryan Porwal)
+* mgr/dashboard: visual tests: Add more ignore regions for dashboard component (`pr#43240 <https://github.com/ceph/ceph/pull/43240>`_, Aaryan Porwal)
+* mgr/influx: use "N/A" for unknown hostname (`pr#43368 <https://github.com/ceph/ceph/pull/43368>`_, Kefu Chai)
+* mgr/mirroring: remove unnecessary fs_name arg from daemon status command (`issue#51989 <http://tracker.ceph.com/issues/51989>`_, `pr#43199 <https://github.com/ceph/ceph/pull/43199>`_, Venky Shankar)
+* mgr/nfs: nfs-rgw batch backport (`pr#43075 <https://github.com/ceph/ceph/pull/43075>`_, Sebastian Wagner, Sage Weil, Varsha Rao, Ramana Raja)
+* mgr/progress: optimize global recovery && introduce 5 seconds interval (`pr#43353 <https://github.com/ceph/ceph/pull/43353>`_, Kamoltat, Neha Ojha)
+* mgr/prometheus: offer ability to disable cache (`pr#43931 <https://github.com/ceph/ceph/pull/43931>`_, Patrick Seidensal)
+* mgr/volumes: Fix permission during subvol creation with mode (`pr#43223 <https://github.com/ceph/ceph/pull/43223>`_, Kotresh HR)
+* mgr: Add check to prevent mgr from crashing (`pr#43445 <https://github.com/ceph/ceph/pull/43445>`_, Aswin Toni)
+* mon,auth: fix proposal (and mon db rebuild) of rotating secrets (`pr#43697 <https://github.com/ceph/ceph/pull/43697>`_, Sage Weil)
+* mon/MDSMonitor: avoid crash when decoding old FSMap epochs (`pr#43615 <https://github.com/ceph/ceph/pull/43615>`_, Patrick Donnelly)
+* mon: Allow specifying new tiebreaker monitors (`pr#43457 <https://github.com/ceph/ceph/pull/43457>`_, Greg Farnum)
+* mon: MonMap: display disallowed_leaders whenever they're set (`pr#43972 <https://github.com/ceph/ceph/pull/43972>`_, Greg Farnum)
+* mon: MonMap: do not increase mon_info_t's compatv in stretch mode, really (`pr#43971 <https://github.com/ceph/ceph/pull/43971>`_, Greg Farnum)
+* monitoring: ethernet bonding filter in Network Load (`pr#43694 <https://github.com/ceph/ceph/pull/43694>`_, Pere Diaz Bou)
+* msg/async/ProtocolV2: Set the recv_stamp at the beginning of receiving a message (`pr#43511 <https://github.com/ceph/ceph/pull/43511>`_, dongdong tao)
+* msgr/async: fix unsafe access in unregister_conn() (`pr#43548 <https://github.com/ceph/ceph/pull/43548>`_, Sage Weil, Radoslaw Zarzynski)
+* os/bluestore: _do_write_small fix head_pad (`pr#43756 <https://github.com/ceph/ceph/pull/43756>`_, dheart)
+* os/bluestore: do not select absent device in volume selector (`pr#43970 <https://github.com/ceph/ceph/pull/43970>`_, Igor Fedotov)
+* os/bluestore: fix invalid omap name conversion when upgrading to per-pg (`pr#43793 <https://github.com/ceph/ceph/pull/43793>`_, Igor Fedotov)
+* os/bluestore: list obj which equals to pend (`pr#43512 <https://github.com/ceph/ceph/pull/43512>`_, Mykola Golub, Kefu Chai)
+* os/bluestore: multiple repair fixes (`pr#43731 <https://github.com/ceph/ceph/pull/43731>`_, Igor Fedotov)
+* osd/OSD: mkfs need wait for transcation completely finish (`pr#43417 <https://github.com/ceph/ceph/pull/43417>`_, Chen Fan)
+* osd: fix partial recovery become whole object recovery after restart osd (`pr#43513 <https://github.com/ceph/ceph/pull/43513>`_, Jianwei Zhang)
+* osd: fix to allow inc manifest leaked (`pr#43306 <https://github.com/ceph/ceph/pull/43306>`_, Myoungwon Oh)
+* osd: fix to recover adjacent clone when set_chunk is called (`pr#43099 <https://github.com/ceph/ceph/pull/43099>`_, Myoungwon Oh)
+* osd: handle inconsistent hash info during backfill and deep scrub gracefully (`pr#43544 <https://github.com/ceph/ceph/pull/43544>`_, Ronen Friedman, Mykola Golub)
+* osd: re-cache peer_bytes on every peering state activate (`pr#43437 <https://github.com/ceph/ceph/pull/43437>`_, Mykola Golub)
+* osd: Run osd bench test to override default max osd capacity for mclock (`pr#41731 <https://github.com/ceph/ceph/pull/41731>`_, Sridhar Seshasayee)
+* Pacific: BlueStore: Omap upgrade to per-pg fix fix (`pr#43922 <https://github.com/ceph/ceph/pull/43922>`_, Adam Kupczyk)
+* Pacific: client: do not defer releasing caps when revoking (`pr#43782 <https://github.com/ceph/ceph/pull/43782>`_, Xiubo Li)
+* Pacific: mds: add read/write io size metrics support (`pr#43784 <https://github.com/ceph/ceph/pull/43784>`_, Xiubo Li)
+* Pacific: test/libcephfs: put inodes after lookup (`pr#43562 <https://github.com/ceph/ceph/pull/43562>`_, Patrick Donnelly)
+* pybind/mgr/cephadm: set allow_standby_replay during CephFS upgrade (`pr#43559 <https://github.com/ceph/ceph/pull/43559>`_, Patrick Donnelly)
+* pybind/mgr/CMakeLists.txt: exclude files not used at runtime (`pr#43787 <https://github.com/ceph/ceph/pull/43787>`_, Duncan Bellamy)
+* pybind/mgr/pg_autoscale: revert to default profile scale-up (`pr#44032 <https://github.com/ceph/ceph/pull/44032>`_, Kamoltat)
+* qa/mgr/dashboard/test_pool: don't check HEALTH_OK (`pr#43440 <https://github.com/ceph/ceph/pull/43440>`_, Ernesto Puerta)
+* qa/mgr/dashboard: add extra wait to test (`pr#43351 <https://github.com/ceph/ceph/pull/43351>`_, Ernesto Puerta)
+* qa/rgw: pacific branch targets ceph-pacific branch of java_s3tests (`pr#43809 <https://github.com/ceph/ceph/pull/43809>`_, Casey Bodley)
+* qa/tasks/kubeadm: force docker cgroup engine to systemd (`pr#43937 <https://github.com/ceph/ceph/pull/43937>`_, Sage Weil)
+* qa/tasks/mgr: skip test_diskprediction_local on python>=3.8 (`pr#43421 <https://github.com/ceph/ceph/pull/43421>`_, Kefu Chai)
+* qa/tests: advanced version to reflect the latest 16.2.6 release (`pr#43242 <https://github.com/ceph/ceph/pull/43242>`_, Yuri Weinstein)
+* qa: disable metrics on kernel client during upgrade (`pr#44034 <https://github.com/ceph/ceph/pull/44034>`_, Patrick Donnelly)
+* qa: lengthen grace for fs map showing dead MDS (`pr#43702 <https://github.com/ceph/ceph/pull/43702>`_, Patrick Donnelly)
+* qa: reduce frag split confs for dir_split counter test (`pr#43828 <https://github.com/ceph/ceph/pull/43828>`_, Patrick Donnelly)
+* rbd-mirror: fix mirror image removal (`pr#43662 <https://github.com/ceph/ceph/pull/43662>`_, Arthur Outhenin-Chalandre)
+* rbd-mirror: unbreak one-way snapshot-based mirroring (`pr#43315 <https://github.com/ceph/ceph/pull/43315>`_, Ilya Dryomov)
+* rgw/notification: make notifications agnostic of bucket reshard (`pr#42946 <https://github.com/ceph/ceph/pull/42946>`_, Yuval Lifshitz)
+* rgw/notifications: cache object size to avoid accessing invalid memory (`pr#42949 <https://github.com/ceph/ceph/pull/42949>`_, Yuval Lifshitz)
+* rgw/notifications: send correct size in case of delete marker creation (`pr#42643 <https://github.com/ceph/ceph/pull/42643>`_, Yuval Lifshitz)
+* rgw/notifications: support v4 auth for topics and notifications (`pr#42947 <https://github.com/ceph/ceph/pull/42947>`_, Yuval Lifshitz)
+* rgw/rgw_rados: make RGW request IDs non-deterministic (`pr#43695 <https://github.com/ceph/ceph/pull/43695>`_, Cory Snyder)
+* rgw/sts: fix for copy object operation using sts (`pr#43703 <https://github.com/ceph/ceph/pull/43703>`_, Pritha Srivastava)
+* rgw/tracing: unify SO version numbers within librgw2 package (`pr#43619 <https://github.com/ceph/ceph/pull/43619>`_, Nathan Cutler)
+* rgw: add abstraction for ops log destination and add file logger (`pr#43740 <https://github.com/ceph/ceph/pull/43740>`_, Casey Bodley, Cory Snyder)
+* rgw: Ensure buckets too old to decode a layout have layout logs (`pr#43823 <https://github.com/ceph/ceph/pull/43823>`_, Adam C. Emerson)
+* rgw: fix bucket purge incomplete multipart uploads (`pr#43862 <https://github.com/ceph/ceph/pull/43862>`_, J. Eric Ivancich)
+* rgw: fix spelling of eTag in S3 message structure (`pr#42945 <https://github.com/ceph/ceph/pull/42945>`_, Tom Schoonjans)
+* rgw: fix sts memory leak (`pr#43348 <https://github.com/ceph/ceph/pull/43348>`_, yuliyang_yewu)
+* rgw: remove prefix & delim params for bucket removal & mp upload abort (`pr#43975 <https://github.com/ceph/ceph/pull/43975>`_, J. Eric Ivancich)
+* rgw: use existing s->bucket in s3 website retarget() (`pr#43777 <https://github.com/ceph/ceph/pull/43777>`_, Casey Bodley)
+* snap-schedule: count retained snapshots per retention policy (`pr#43434 <https://github.com/ceph/ceph/pull/43434>`_, Jan Fajerski)
+* test: shutdown the mounter after test finishes (`pr#43475 <https://github.com/ceph/ceph/pull/43475>`_, Xiubo Li)
+
+v16.2.6 Pacific
+===============
+
+.. DANGER:: DATE: 01 NOV 2021.
+
+   DO NOT UPGRADE TO CEPH PACIFIC FROM AN OLDER VERSION.
+
+   A recently-discovered bug (https://tracker.ceph.com/issues/53062) can cause
+   data corruption. This bug occurs during OMAP format conversion for
+   clusters that are updated to Pacific. New clusters are not affected by this
+   bug.
+
+   The trigger for this bug is BlueStore's repair/quick-fix functionality. This
+   bug can be triggered in two known ways:
+
+    (1) manually via the ceph-bluestore-tool, or
+    (2) automatically, by OSD if ``bluestore_fsck_quick_fix_on_mount`` is set
+        to true.
+
+   The fix for this bug is expected to be available in Ceph v16.2.7.
+
+   DO NOT set ``bluestore_quick_fix_on_mount`` to true. If it is currently
+   set to true in your configuration, immediately set it to false.
+
+   DO NOT run ``ceph-bluestore-tool``'s repair/quick-fix commands.
+
+
+This is the sixth backport release in the Pacific series.
+
+
+Notable Changes
+---------------
+
+* MGR: The pg_autoscaler has a new default 'scale-down' profile which provides more
+  performance from the start for new pools (for newly created clusters).
+  Existing clusters will retain the old behavior, now called the 'scale-up' profile.
+  For more details, see:
+  https://docs.ceph.com/en/latest/rados/operations/placement-groups/
+
+* CephFS: the upgrade procedure for CephFS is now simpler. It is no longer
+  necessary to stop all MDS before upgrading the sole active MDS. After
+  disabling standby-replay, reducing max_mds to 1, and waiting for the file
+  systems to become stable (each fs with 1 active and 0 stopping daemons), a
+  rolling upgrade of all MDS daemons can be performed.
+
+* Dashboard: now allows users to set up and display a custom message (MOTD, warning,
+  etc.) in a sticky banner at the top of the page. For more details, see:
+  https://docs.ceph.com/en/pacific/mgr/dashboard/#message-of-the-day-motd
+
+* Several fixes in BlueStore, including a fix for the deferred write regression,
+  which led to excessive RocksDB flushes and compactions. Previously, when
+  `bluestore_prefer_deferred_size_hdd` was equal to or more than
+  `bluestore_max_blob_size_hdd` (both set to 64K), all the data was deferred,
+  which led to increased consumption of the column family used to store
+  deferred writes in RocksDB. Now, the `bluestore_prefer_deferred_size` parameter
+  independently controls deferred writes, and only writes smaller than
+  this size use the deferred write path.
+
+* The default value of `osd_client_message_cap` has been set to 256, to provide
+  better flow control by limiting maximum number of in-flight client requests.
+
+* PGs no longer show a `active+clean+scrubbing+deep+repair` state when
+  `osd_scrub_auto_repair` is set to true, for regular deep-scrubs with no repair
+  required.
+
+* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook`
+  anymore. As the latter depends on `python3-numpy` which cannot be imported in
+  different Python sub-interpreters multi-times if the version of
+  `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends`
+  packages by default, `ceph-mgr-rook` was always installed along with
+  `ceph-mgr` debian package as an indirect dependency. If your workflow depends
+  on this behavior, you might want to install `ceph-mgr-rook` separately.
+
+ * This is the first release built for Debian Bullseye.
+
+Changelog
+---------
+
+* bind on loopback address if no other addresses are available (`pr#42477 <https://github.com/ceph/ceph/pull/42477>`_, Kefu Chai)
+* ceph-monstore-tool: use a large enough paxos/{first,last}_committed (`issue#38219 <http://tracker.ceph.com/issues/38219>`_, `pr#42411 <https://github.com/ceph/ceph/pull/42411>`_, Kefu Chai)
+* ceph-volume/tests: retry when destroying osd (`pr#42546 <https://github.com/ceph/ceph/pull/42546>`_, Guillaume Abrioux)
+* ceph-volume/tests: update ansible environment variables in tox (`pr#42490 <https://github.com/ceph/ceph/pull/42490>`_, Dimitri Savineau)
+* ceph-volume: Consider /dev/root as mounted (`pr#42755 <https://github.com/ceph/ceph/pull/42755>`_, David Caro)
+* ceph-volume: fix lvm activate arguments (`pr#43116 <https://github.com/ceph/ceph/pull/43116>`_, Dimitri Savineau)
+* ceph-volume: fix lvm migrate without args (`pr#43110 <https://github.com/ceph/ceph/pull/43110>`_, Dimitri Savineau)
+* ceph-volume: fix raw list with logical partition (`pr#43087 <https://github.com/ceph/ceph/pull/43087>`_, Guillaume Abrioux, Dimitri Savineau)
+* ceph-volume: implement bluefs volume migration (`pr#42219 <https://github.com/ceph/ceph/pull/42219>`_, Kefu Chai, Igor Fedotov)
+* ceph-volume: lvm batch: fast_allocations(): avoid ZeroDivisionError (`pr#42493 <https://github.com/ceph/ceph/pull/42493>`_, Jonas Zeiger)
+* ceph-volume: pvs --noheadings replace pvs --no-heading (`pr#43076 <https://github.com/ceph/ceph/pull/43076>`_, FengJiankui)
+* ceph-volume: remove --all ref from deactivate help (`pr#43098 <https://github.com/ceph/ceph/pull/43098>`_, Dimitri Savineau)
+* ceph-volume: support no_systemd with lvm migrate (`pr#43091 <https://github.com/ceph/ceph/pull/43091>`_, Dimitri Savineau)
+* ceph-volume: work around phantom atari partitions (`pr#42753 <https://github.com/ceph/ceph/pull/42753>`_, Blaine Gardner)
+* ceph.spec.in: drop gdbm from build deps (`pr#43000 <https://github.com/ceph/ceph/pull/43000>`_, Kefu Chai)
+* cephadm: August batch 1 (`pr#42736 <https://github.com/ceph/ceph/pull/42736>`_, Sage Weil, Dimitri Savineau, Guillaume Abrioux, Sebastian Wagner, Varsha Rao, Zac Dover, Adam King, Cory Snyder, Michael Fritch, Asbjørn Sannes, "Wang,Fei", Javier Cacheiro, 胡玮文, Daniel Pivonka)
+* cephadm: September batch 1 (`issue#52038 <http://tracker.ceph.com/issues/52038>`_, `pr#43029 <https://github.com/ceph/ceph/pull/43029>`_, Sebastian Wagner, Dimitri Savineau, Paul Cuzner, Oleander Reis, Adam King, Yuxiang Zhu, Zac Dover, Alfonso Martínez, Sage Weil, Daniel Pivonka)
+* cephadm: use quay, not docker (`pr#42534 <https://github.com/ceph/ceph/pull/42534>`_, Sage Weil)
+* cephfs-mirror: record directory path cancel in DirRegistry (`issue#51666 <http://tracker.ceph.com/issues/51666>`_, `pr#42458 <https://github.com/ceph/ceph/pull/42458>`_, Venky Shankar)
+* client: flush the mdlog in unsafe requests' relevant and auth MDSes only (`pr#42925 <https://github.com/ceph/ceph/pull/42925>`_, Xiubo Li)
+* client: make sure only to update dir dist from auth mds (`pr#42937 <https://github.com/ceph/ceph/pull/42937>`_, Xue Yantao)
+* cls/cmpomap: empty values are 0 in U64 comparisons (`pr#42908 <https://github.com/ceph/ceph/pull/42908>`_, Casey Bodley)
+* cmake, ceph.spec.in: build with header only fmt on RHEL (`pr#42472 <https://github.com/ceph/ceph/pull/42472>`_, Kefu Chai)
+* cmake: build static libs if they are internal ones (`pr#39902 <https://github.com/ceph/ceph/pull/39902>`_, Kefu Chai)
+* cmake: exclude "grafonnet-lib" target from "all" (`pr#42898 <https://github.com/ceph/ceph/pull/42898>`_, Kefu Chai)
+* cmake: link bundled fmt statically (`pr#42692 <https://github.com/ceph/ceph/pull/42692>`_, Kefu Chai)
+* cmake: Replace boost download url (`pr#42693 <https://github.com/ceph/ceph/pull/42693>`_, Rafał Wądołowski)
+* common/buffer: fix SIGABRT in  rebuild_aligned_size_and_memory (`pr#42976 <https://github.com/ceph/ceph/pull/42976>`_, Yin Congmin)
+* common/Formatter: include used header (`pr#42233 <https://github.com/ceph/ceph/pull/42233>`_, Kefu Chai)
+* common/options: Set osd_client_message_cap to 256 (`pr#42615 <https://github.com/ceph/ceph/pull/42615>`_, Mark Nelson)
+* compression/snappy: use uint32_t to be compatible with 1.1.9 (`pr#42542 <https://github.com/ceph/ceph/pull/42542>`_, Kefu Chai, Nathan Cutler)
+* debian/control: ceph-mgr-modules-core does not Recommend ceph-mgr-roo… (`pr#42300 <https://github.com/ceph/ceph/pull/42300>`_, Kefu Chai)
+* debian/control: dh-systemd is part of debhelper now (`pr#43151 <https://github.com/ceph/ceph/pull/43151>`_, David Galloway)
+* debian/control: remove cython from Build-Depends (`pr#43131 <https://github.com/ceph/ceph/pull/43131>`_, Kefu Chai)
+* doc/ceph-volume: add lvm migrate/new-db/new-wal (`pr#43089 <https://github.com/ceph/ceph/pull/43089>`_, Dimitri Savineau)
+* doc/rados/operations: s/max_misplaced/target_max_misplaced_ratio/ (`pr#42250 <https://github.com/ceph/ceph/pull/42250>`_, Paul Reece, Kefu Chai)
+* doc/releases/pacific.rst: remove notes about autoscaler (`pr#42265 <https://github.com/ceph/ceph/pull/42265>`_, Neha Ojha)
+* Don't persist report data (`pr#42888 <https://github.com/ceph/ceph/pull/42888>`_, Brad Hubbard)
+* krbd: escape udev_enumerate_add_match_sysattr values (`pr#42969 <https://github.com/ceph/ceph/pull/42969>`_, Ilya Dryomov)
+* kv/RocksDBStore: Add handling of block_cache option for resharding (`pr#42844 <https://github.com/ceph/ceph/pull/42844>`_, Adam Kupczyk)
+* kv/RocksDBStore: enrich debug message (`pr#42544 <https://github.com/ceph/ceph/pull/42544>`_, Toshikuni Fukaya, Satoru Takeuchi)
+* librgw/notifications: initialize kafka and amqp (`pr#42648 <https://github.com/ceph/ceph/pull/42648>`_, Yuval Lifshitz)
+* mds: add debugging when rejecting mksnap with EPERM (`pr#42935 <https://github.com/ceph/ceph/pull/42935>`_, Patrick Donnelly)
+* mds: create file system with specific ID (`pr#42900 <https://github.com/ceph/ceph/pull/42900>`_, Ramana Raja)
+* mds: MDCache.cc:5319 FAILED ceph_assert(rejoin_ack_gather.count(mds->get_nodeid())) (`pr#42938 <https://github.com/ceph/ceph/pull/42938>`_, chencan)
+* mds: META_POP_READDIR, META_POP_FETCH, META_POP_STORE, and cache_hit_rate are not updated (`pr#42939 <https://github.com/ceph/ceph/pull/42939>`_, Yongseok Oh)
+* mds: to print the unknow type value (`pr#42088 <https://github.com/ceph/ceph/pull/42088>`_, Xiubo Li, Jos Collin)
+* MDSMonitor: monitor crash after upgrade from ceph 15.2.13 to 16.2.4 (`pr#42536 <https://github.com/ceph/ceph/pull/42536>`_, Patrick Donnelly)
+* mgr/DaemonServer: skip redundant update of pgp_num_actual (`pr#42223 <https://github.com/ceph/ceph/pull/42223>`_, Dan van der Ster)
+* mgr/dashboard/api: set a UTF-8 locale when running pip (`pr#42829 <https://github.com/ceph/ceph/pull/42829>`_, Kefu Chai)
+* mgr/dashboard: Add configurable MOTD or wall notification (`pr#42414 <https://github.com/ceph/ceph/pull/42414>`_, Volker Theile)
+* mgr/dashboard: cephadm e2e start script: add --expanded option (`pr#42789 <https://github.com/ceph/ceph/pull/42789>`_, Alfonso Martínez)
+* mgr/dashboard: cephadm-e2e job script: improvements (`pr#42585 <https://github.com/ceph/ceph/pull/42585>`_, Alfonso Martínez)
+* mgr/dashboard: disable create snapshot with subvolumes (`pr#42819 <https://github.com/ceph/ceph/pull/42819>`_, Pere Diaz Bou)
+* mgr/dashboard: don't notify for suppressed alerts (`pr#42974 <https://github.com/ceph/ceph/pull/42974>`_, Tatjana Dehler)
+* mgr/dashboard: fix Accept-Language header parsing (`pr#42297 <https://github.com/ceph/ceph/pull/42297>`_, 胡玮文)
+* mgr/dashboard: fix rename inventory to disks (`pr#42810 <https://github.com/ceph/ceph/pull/42810>`_, Navin Barnwal)
+* mgr/dashboard: fix ssl cert validation for rgw service creation (`pr#42628 <https://github.com/ceph/ceph/pull/42628>`_, Avan Thakkar)
+* mgr/dashboard: Fix test_error force maintenance dashboard check (`pr#42354 <https://github.com/ceph/ceph/pull/42354>`_, Nizamudeen A)
+* mgr/dashboard: monitoring: replace Grafana JSON with Grafonnet based code (`pr#42812 <https://github.com/ceph/ceph/pull/42812>`_, Aashish Sharma)
+* mgr/dashboard: Refresh button on the iscsi targets page (`pr#42817 <https://github.com/ceph/ceph/pull/42817>`_, Nizamudeen A)
+* mgr/dashboard: remove usage of 'rgw_frontend_ssl_key' (`pr#42316 <https://github.com/ceph/ceph/pull/42316>`_, Avan Thakkar)
+* mgr/dashboard: show perf. counters for rgw svc. on Cluster > Hosts (`pr#42629 <https://github.com/ceph/ceph/pull/42629>`_, Alfonso Martínez)
+* mgr/dashboard: stats=false not working when listing buckets (`pr#42889 <https://github.com/ceph/ceph/pull/42889>`_, Avan Thakkar)
+* mgr/dashboard: tox.ini: delete useless env. 'apidocs' (`pr#42788 <https://github.com/ceph/ceph/pull/42788>`_, Alfonso Martínez)
+* mgr/dashboard: update translations for pacific (`pr#42606 <https://github.com/ceph/ceph/pull/42606>`_, Tatjana Dehler)
+* mgr/mgr_util: switch using unshared cephfs connections whenever possible (`issue#51256 <http://tracker.ceph.com/issues/51256>`_, `pr#42083 <https://github.com/ceph/ceph/pull/42083>`_, Venky Shankar)
+* mgr/pg_autoscaler: Introduce autoscaler scale-down feature (`pr#42428 <https://github.com/ceph/ceph/pull/42428>`_, Kamoltat, Kefu Chai)
+* mgr/rook: Add timezone info (`pr#39834 <https://github.com/ceph/ceph/pull/39834>`_, Varsha Rao, Sebastian Wagner)
+* mgr/telemetry: pass leaderboard flag even w/o ident (`pr#42228 <https://github.com/ceph/ceph/pull/42228>`_, Sage Weil)
+* mgr/volumes: Add config to insert delay at the beginning of the clone (`pr#42086 <https://github.com/ceph/ceph/pull/42086>`_, Kotresh HR)
+* mgr/volumes: use dedicated libcephfs handles for subvolume calls and … (`issue#51271 <http://tracker.ceph.com/issues/51271>`_, `pr#42914 <https://github.com/ceph/ceph/pull/42914>`_, Venky Shankar)
+* mgr: set debug_mgr=2/5 (so INFO goes to mgr log by default) (`pr#42225 <https://github.com/ceph/ceph/pull/42225>`_, Sage Weil)
+* mon/MDSMonitor: do not pointlessly kill standbys that are incompatible with current CompatSet (`pr#42578 <https://github.com/ceph/ceph/pull/42578>`_, Patrick Donnelly, Zhi Zhang)
+* mon/OSDMonitor: resize oversized Lec::epoch_by_pg, after PG merging, preventing osdmap trimming (`pr#42224 <https://github.com/ceph/ceph/pull/42224>`_, Dan van der Ster)
+* mon/PGMap: remove DIRTY field in ceph df detail when cache tiering is not in use (`pr#42860 <https://github.com/ceph/ceph/pull/42860>`_, Deepika Upadhyay)
+* mon: return -EINVAL when handling unknown option in 'ceph osd pool get' (`pr#42229 <https://github.com/ceph/ceph/pull/42229>`_, Zhao Cuicui)
+* mon: Sanely set the default CRUSH rule when creating pools in stretch… (`pr#42909 <https://github.com/ceph/ceph/pull/42909>`_, Greg Farnum)
+* monitoring/grafana/build/Makefile: revamp for arm64 builds, pushes to docker and quay, jenkins (`pr#42211 <https://github.com/ceph/ceph/pull/42211>`_, Dan Mick)
+* monitoring/grafana/cluster: use per-unit max and limit values (`pr#42679 <https://github.com/ceph/ceph/pull/42679>`_, David Caro)
+* monitoring: Clean up Grafana dashboards (`pr#42299 <https://github.com/ceph/ceph/pull/42299>`_, Patrick Seidensal)
+* monitoring: fix Physical Device Latency unit (`pr#42298 <https://github.com/ceph/ceph/pull/42298>`_, Seena Fallah)
+* msg: active_connections regression (`pr#42936 <https://github.com/ceph/ceph/pull/42936>`_, Sage Weil)
+* nfs backport June (`pr#42096 <https://github.com/ceph/ceph/pull/42096>`_, Varsha Rao)
+* os/bluestore: accept undecodable multi-block bluefs transactions on log (`pr#43023 <https://github.com/ceph/ceph/pull/43023>`_, Igor Fedotov)
+* os/bluestore: cap omap naming scheme upgrade transaction (`pr#42956 <https://github.com/ceph/ceph/pull/42956>`_, Igor Fedotov)
+* os/bluestore: compact db after bulk omap naming upgrade (`pr#42426 <https://github.com/ceph/ceph/pull/42426>`_, Igor Fedotov)
+* os/bluestore: fix bluefs migrate command (`pr#43100 <https://github.com/ceph/ceph/pull/43100>`_, Igor Fedotov)
+* os/bluestore: fix erroneous SharedBlob record removal during repair (`pr#42423 <https://github.com/ceph/ceph/pull/42423>`_, Igor Fedotov)
+* os/bluestore: fix using incomplete bluefs log when dumping it (`pr#43007 <https://github.com/ceph/ceph/pull/43007>`_, Igor Fedotov)
+* os/bluestore: make deferred writes less aggressive for large writes (`pr#42773 <https://github.com/ceph/ceph/pull/42773>`_, Igor Fedotov, Adam Kupczyk)
+* os/bluestore: Remove possibility of replay log and file inconsistency (`pr#42424 <https://github.com/ceph/ceph/pull/42424>`_, Adam Kupczyk)
+* os/bluestore: respect bluestore_warn_on_spurious_read_errors setting (`pr#42897 <https://github.com/ceph/ceph/pull/42897>`_, Igor Fedotov)
+* osd/scrub: separate between PG state flags and internal scrubber operation (`pr#42398 <https://github.com/ceph/ceph/pull/42398>`_, Ronen Friedman)
+* osd: log snaptrim message to dout (`pr#42482 <https://github.com/ceph/ceph/pull/42482>`_, Arthur Outhenin-Chalandre)
+* osd: move down peers out from peer_purged (`pr#42238 <https://github.com/ceph/ceph/pull/42238>`_, Mykola Golub)
+* pybind/mgr/stats: validate cmdtag (`pr#42702 <https://github.com/ceph/ceph/pull/42702>`_, Jos Collin)
+* pybind/mgr: Fix IPv6 url generation (`pr#42990 <https://github.com/ceph/ceph/pull/42990>`_, Sebastian Wagner)
+* pybind/rbd: fix mirror_image_get_status (`pr#42972 <https://github.com/ceph/ceph/pull/42972>`_, Ilya Dryomov, Will Smith)
+* qa/\*/test_envlibrados_for_rocksdb.sh: install libarchive-3.3.3 (`pr#42344 <https://github.com/ceph/ceph/pull/42344>`_, Neha Ojha)
+* qa/cephadm: centos_8.x_container_tools_3.0.yaml (`pr#42868 <https://github.com/ceph/ceph/pull/42868>`_, Sebastian Wagner)
+* qa/rgw: move ignore-pg-availability.yaml out of suites/rgw (`pr#40694 <https://github.com/ceph/ceph/pull/40694>`_, Casey Bodley)
+* qa/standalone: Add missing cleanups after completion of a subset of osd and scrub tests (`pr#42258 <https://github.com/ceph/ceph/pull/42258>`_, Sridhar Seshasayee)
+* qa/tests: advanced pacific version to reflect the latest 16.2.5 point (`pr#42264 <https://github.com/ceph/ceph/pull/42264>`_, Yuri Weinstein)
+* qa/workunits/mon/test_mon_config_key: use subprocess.run() instead of proc.communicate() (`pr#42221 <https://github.com/ceph/ceph/pull/42221>`_, Kefu Chai)
+* qa: FileNotFoundError: [Errno 2] No such file or directory: '/sys/kernel/debug/ceph/3fab6bea-f243-47a4-a956-8c03a62b61b5.client4721/mds_sessions' (`pr#42165 <https://github.com/ceph/ceph/pull/42165>`_, Patrick Donnelly)
+* qa: increase the pg_num for cephfs_data/metadata pools (`pr#42923 <https://github.com/ceph/ceph/pull/42923>`_, Xiubo Li)
+* qa: test_ls_H_prints_human_readable_file_size failure (`pr#42166 <https://github.com/ceph/ceph/pull/42166>`_, Patrick Donnelly)
+* radosgw-admin: skip GC init on read-only admin ops (`pr#42655 <https://github.com/ceph/ceph/pull/42655>`_, Mark Kogan)
+* radosgw: include realm\_{id,name} in service map (`pr#42213 <https://github.com/ceph/ceph/pull/42213>`_, Sage Weil)
+* rbd-mirror: add perf counters to snapshot replayer (`pr#42987 <https://github.com/ceph/ceph/pull/42987>`_, Arthur Outhenin-Chalandre)
+* rbd-mirror: fix potential async op tracker leak in start_image_replayers (`pr#42979 <https://github.com/ceph/ceph/pull/42979>`_, Mykola Golub)
+* rbd: fix default pool handling for nbd map/unmap (`pr#42980 <https://github.com/ceph/ceph/pull/42980>`_, Sunny Kumar)
+* Remove dependency on lsb_release (`pr#43001 <https://github.com/ceph/ceph/pull/43001>`_, Ken Dreyer)
+* RGW - Bucket Remove Op: Pass in user (`pr#42135 <https://github.com/ceph/ceph/pull/42135>`_, Daniel Gryniewicz)
+* RGW - Don't move attrs before setting them (`pr#42320 <https://github.com/ceph/ceph/pull/42320>`_, Daniel Gryniewicz)
+* rgw : add check empty for sync url (`pr#42653 <https://github.com/ceph/ceph/pull/42653>`_, caolei)
+* rgw : add check for tenant provided in RGWCreateRole (`pr#42637 <https://github.com/ceph/ceph/pull/42637>`_, caolei)
+* rgw : modfiy error XML for deleterole (`pr#42639 <https://github.com/ceph/ceph/pull/42639>`_, caolei)
+* rgw multisite: metadata sync treats all errors as 'transient' for retry (`pr#42656 <https://github.com/ceph/ceph/pull/42656>`_, Casey Bodley)
+* RGW Zipper - Make sure bucket list progresses (`pr#42625 <https://github.com/ceph/ceph/pull/42625>`_, Daniel Gryniewicz)
+* rgw/amqp/test: fix mock prototype for librabbitmq-0.11.0 (`pr#42649 <https://github.com/ceph/ceph/pull/42649>`_, Yuval Lifshitz)
+* rgw/http/notifications: support content type in HTTP POST messages (`pr#42644 <https://github.com/ceph/ceph/pull/42644>`_, Yuval Lifshitz)
+* rgw/multisite: return correct error code when op fails (`pr#42646 <https://github.com/ceph/ceph/pull/42646>`_, Yuval Lifshitz)
+* rgw/notification: add exception handling for persistent notification thread (`pr#42647 <https://github.com/ceph/ceph/pull/42647>`_, Yuval Lifshitz)
+* rgw/notification: fix persistent notification hang when ack-levl=none (`pr#40696 <https://github.com/ceph/ceph/pull/40696>`_, Yuval Lifshitz)
+* rgw/notification: fixing the "persistent=false" flag (`pr#40695 <https://github.com/ceph/ceph/pull/40695>`_, Yuval Lifshitz)
+* rgw/notifications: delete bucket notification object when empty (`pr#42631 <https://github.com/ceph/ceph/pull/42631>`_, Yuval Lifshitz)
+* rgw/notifications: support metadata filter in CompleteMultipartUpload and Copy events (`pr#42321 <https://github.com/ceph/ceph/pull/42321>`_, Yuval Lifshitz)
+* rgw/notifications: support metadata filter in CompleteMultipartUploa… (`pr#42566 <https://github.com/ceph/ceph/pull/42566>`_, Yuval Lifshitz)
+* rgw/rgw_file: Fix the return value of read() and readlink() (`pr#42654 <https://github.com/ceph/ceph/pull/42654>`_, Dai zhiwei, luo rixin)
+* rgw/sts: correcting the evaluation of session policies (`pr#42632 <https://github.com/ceph/ceph/pull/42632>`_, Pritha Srivastava)
+* rgw/sts: read_obj_policy() consults iam_user_policies on ENOENT (`pr#42650 <https://github.com/ceph/ceph/pull/42650>`_, Casey Bodley)
+* rgw: allow rgw-orphan-list to process multiple data pools (`pr#42635 <https://github.com/ceph/ceph/pull/42635>`_, J. Eric Ivancich)
+* rgw: allow to set ssl options and ciphers for beast frontend (`pr#42363 <https://github.com/ceph/ceph/pull/42363>`_, Mykola Golub)
+* rgw: avoid infinite loop when deleting a bucket (`issue#49206 <http://tracker.ceph.com/issues/49206>`_, `pr#42230 <https://github.com/ceph/ceph/pull/42230>`_, Jeegn Chen)
+* rgw: avoid occuring radosgw daemon crash when access a conditionally … (`pr#42626 <https://github.com/ceph/ceph/pull/42626>`_, xiangrui meng, yupeng chen)
+* rgw: Backport of 51674 to Pacific (`pr#42346 <https://github.com/ceph/ceph/pull/42346>`_, Adam C. Emerson)
+* rgw: deprecate the civetweb frontend (`pr#41367 <https://github.com/ceph/ceph/pull/41367>`_, Casey Bodley)
+* rgw: Don't segfault on datalog trim (`pr#42336 <https://github.com/ceph/ceph/pull/42336>`_, Adam C. Emerson)
+* rgw: during reshard lock contention, adjust logging (`pr#42641 <https://github.com/ceph/ceph/pull/42641>`_, J. Eric Ivancich)
+* rgw: extending existing ssl support for vault KMS (`pr#42093 <https://github.com/ceph/ceph/pull/42093>`_, Jiffin Tony Thottan)
+* rgw: fail as expected when set/delete-bucket-website attempted on a non-exis… (`pr#42642 <https://github.com/ceph/ceph/pull/42642>`_, xiangrui meng)
+* rgw: fix bucket object listing when marker matches prefix (`pr#42638 <https://github.com/ceph/ceph/pull/42638>`_, J. Eric Ivancich)
+* rgw: fix for mfa resync crash when supplied with only one totp_pin (`pr#42652 <https://github.com/ceph/ceph/pull/42652>`_, Pritha Srivastava)
+* rgw: fix segfault related to explicit object manifest handling (`pr#42633 <https://github.com/ceph/ceph/pull/42633>`_, Mark Kogan)
+* rgw: Improve error message on email id reuse (`pr#41783 <https://github.com/ceph/ceph/pull/41783>`_, Ponnuvel Palaniyappan)
+* rgw: objectlock: improve client error messages (`pr#40693 <https://github.com/ceph/ceph/pull/40693>`_, Matt Benjamin)
+* rgw: parse tenant name out of rgwx-bucket-instance (`pr#42231 <https://github.com/ceph/ceph/pull/42231>`_, Casey Bodley)
+* rgw: radosgw-admin errors if marker not specified on data/mdlog trim (`pr#42640 <https://github.com/ceph/ceph/pull/42640>`_, Adam C. Emerson)
+* rgw: remove quota soft threshold (`pr#42634 <https://github.com/ceph/ceph/pull/42634>`_, Zulai Wang)
+* rgw: require bucket name in bucket chown (`pr#42323 <https://github.com/ceph/ceph/pull/42323>`_, Zulai Wang)
+* rgw: when deleted obj removed in versioned bucket, extra del-marker added (`pr#42645 <https://github.com/ceph/ceph/pull/42645>`_, J. Eric Ivancich)
+* rpm/luarocks: simplify conditional and support Leap 15.3 (`pr#42561 <https://github.com/ceph/ceph/pull/42561>`_, Nathan Cutler)
+* rpm: drop use of $FIRST_ARG in ceph-immutable-object-cache (`pr#42480 <https://github.com/ceph/ceph/pull/42480>`_, Nathan Cutler)
+* run-make-check.sh: Increase failure output log size (`pr#42850 <https://github.com/ceph/ceph/pull/42850>`_, David Galloway)
+* SimpleRADOSStriper: use debug_cephsqlite (`pr#42659 <https://github.com/ceph/ceph/pull/42659>`_, Patrick Donnelly)
+* src/pybind/mgr/mirroring/fs/snapshot_mirror.py: do not assume a cephf… (`pr#42226 <https://github.com/ceph/ceph/pull/42226>`_, Sébastien Han)
+* test/rgw: fix use of poll() with timers in unittest_rgw_dmclock_scheduler (`pr#42651 <https://github.com/ceph/ceph/pull/42651>`_, Casey Bodley)
+* Warning Cleanup and Clang Compile Fix (`pr#40692 <https://github.com/ceph/ceph/pull/40692>`_, Adam C. Emerson)
+* workunits/rgw: semicolon terminates perl statements (`pr#43168 <https://github.com/ceph/ceph/pull/43168>`_, Matt Benjamin)
+
+v16.2.5 Pacific
+===============
+
+This is the fifth backport release in the Pacific series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook`
+  anymore. As the latter depends on `python3-numpy` which cannot be imported in
+  different Python sub-interpreters multi-times if the version of
+  `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends`
+  packages by default, `ceph-mgr-rook` was always installed along with
+  `ceph-mgr` debian package as an indirect dependency. If your workflow depends
+  on this behavior, you might want to install `ceph-mgr-rook` separately.
+
+* mgr/nfs: ``nfs`` module is moved out of volumes plugin. Prior using the
+  ``ceph nfs`` commands, ``nfs`` mgr module must be enabled.
+
+* volumes/nfs: The ``cephfs`` cluster type has been removed from the
+  ``nfs cluster create`` subcommand. Clusters deployed by cephadm can
+  support an NFS export of both ``rgw`` and ``cephfs`` from a single
+  NFS cluster instance.
+
+* The ``nfs cluster update`` command has been removed.  You can modify
+  the placement of an existing NFS service (and/or its associated
+  ingress service) using ``orch ls --export`` and ``orch apply -i
+  ...``.
+
+* The ``orch apply nfs`` command no longer requires a pool or
+  namespace argument. We strongly encourage users to use the defaults
+  so that the ``nfs cluster ls`` and related commands will work
+  properly.
+
+* The ``nfs cluster delete`` and ``nfs export delete`` commands are
+  deprecated and will be removed in a future release.  Please use
+  ``nfs cluster rm`` and ``nfs export rm`` instead.
+
+* A long-standing bug that prevented 32-bit and 64-bit client/server
+  interoperability under msgr v2 has been fixed.  In particular, mixing armv7l
+  (armhf) and x86_64 or aarch64 servers in the same cluster now works.
+
+Changelog
+---------
+
+* .github/labeler: add api-change label (`pr#41818 <https://github.com/ceph/ceph/pull/41818>`_, Ernesto Puerta)
+* Improve mon location handling for stretch clusters (`pr#40484 <https://github.com/ceph/ceph/pull/40484>`_, Greg Farnum)
+* MDS heartbeat timed out between during executing MDCache::start_files_to_recover() (`pr#42061 <https://github.com/ceph/ceph/pull/42061>`_, Yongseok Oh)
+* MDS slow request lookupino #0x100 on rank 1 block forever on dispatched (`pr#40856 <https://github.com/ceph/ceph/pull/40856>`_, Xiubo Li, Patrick Donnelly)
+* MDSMonitor: crash when attempting to mount cephfs (`pr#42068 <https://github.com/ceph/ceph/pull/42068>`_, Patrick Donnelly)
+* Pacific stretch mon state [Merge after 40484] (`pr#41130 <https://github.com/ceph/ceph/pull/41130>`_, Greg Farnum)
+* Pacific: Add DoutPrefixProvider for RGW Log Messages in Pacfic (`pr#40054 <https://github.com/ceph/ceph/pull/40054>`_, Ali Maredia, Kalpesh Pandya, Casey Bodley)
+* Pacific: Direct MMonJoin messages to leader, not first rank [Merge after 41130] (`pr#41131 <https://github.com/ceph/ceph/pull/41131>`_, Greg Farnum)
+* Revert "pacific: mgr/dashboard: Generate NPM dependencies manifest" (`pr#41549 <https://github.com/ceph/ceph/pull/41549>`_, Nizamudeen A)
+* Update boost url, fixing windows build (`pr#41259 <https://github.com/ceph/ceph/pull/41259>`_, Lucian Petrut)
+* bluestore: use string_view and strip trailing slash for dir listing (`pr#41755 <https://github.com/ceph/ceph/pull/41755>`_, Jonas Jelten, Kefu Chai)
+* build(deps): bump node-notifier from 8.0.0 to 8.0.1 in /src/pybind/mgr/dashboard/frontend (`pr#40813 <https://github.com/ceph/ceph/pull/40813>`_, Ernesto Puerta, dependabot[bot])
+* ceph-volume: fix batch report and respect ceph.conf config values (`pr#41714 <https://github.com/ceph/ceph/pull/41714>`_, Andrew Schoen)
+* ceph_test_rados_api_service: more retries for servicemkap (`pr#41182 <https://github.com/ceph/ceph/pull/41182>`_, Sage Weil)
+* cephadm june final batch (`pr#42117 <https://github.com/ceph/ceph/pull/42117>`_, Kefu Chai, Sage Weil, Zac Dover, Sebastian Wagner, Varsha Rao, Sandro Bonazzola, Juan Miguel Olmo Martínez)
+* cephadm: batch backport for May (2) (`pr#41219 <https://github.com/ceph/ceph/pull/41219>`_, Adam King, Sage Weil, Zac Dover, Dennis Körner, jianglong01, Avan Thakkar, Juan Miguel Olmo Martínez)
+* cephadm: june batch 1 (`pr#41684 <https://github.com/ceph/ceph/pull/41684>`_, Sage Weil, Paul Cuzner, Juan Miguel Olmo Martínez, VasishtaShastry, Zac Dover, Sebastian Wagner, Adam King, Michael Fritch, Daniel Pivonka, sunilkumarn417)
+* cephadm: june batch 2 (`pr#41815 <https://github.com/ceph/ceph/pull/41815>`_, Sebastian Wagner, Daniel Pivonka, Zac Dover, Michael Fritch)
+* cephadm: june batch 3 (`pr#41913 <https://github.com/ceph/ceph/pull/41913>`_, Zac Dover, Adam King, Michael Fritch, Patrick Donnelly, Sage Weil, Juan Miguel Olmo Martínez, jianglong01)
+* cephadm: may batch 1 (`pr#41151 <https://github.com/ceph/ceph/pull/41151>`_, Juan Miguel Olmo Martínez, Sage Weil, Zac Dover, Daniel Pivonka, Adam King, Stanislav Datskevych, jianglong01, Kefu Chai, Deepika Upadhyay, Joao Eduardo Luis)
+* cephadm: may batch 3 (`pr#41463 <https://github.com/ceph/ceph/pull/41463>`_, Sage Weil, Michael Fritch, Adam King, Patrick Seidensal, Juan Miguel Olmo Martínez, Dimitri Savineau, Zac Dover, Sebastian Wagner)
+* cephfs-mirror backports (`issue#50523 <http://tracker.ceph.com/issues/50523>`_, `issue#50035 <http://tracker.ceph.com/issues/50035>`_, `issue#50266 <http://tracker.ceph.com/issues/50266>`_, `issue#50442 <http://tracker.ceph.com/issues/50442>`_, `issue#50581 <http://tracker.ceph.com/issues/50581>`_, `issue#50229 <http://tracker.ceph.com/issues/50229>`_, `issue#49939 <http://tracker.ceph.com/issues/49939>`_, `issue#50224 <http://tracker.ceph.com/issues/50224>`_, `issue#50298 <http://tracker.ceph.com/issues/50298>`_, `pr#41475 <https://github.com/ceph/ceph/pull/41475>`_, Venky Shankar, Lucian Petrut)
+* cephfs-mirror: backports (`issue#50447 <http://tracker.ceph.com/issues/50447>`_, `issue#50867 <http://tracker.ceph.com/issues/50867>`_, `issue#51204 <http://tracker.ceph.com/issues/51204>`_, `pr#41947 <https://github.com/ceph/ceph/pull/41947>`_, Venky Shankar)
+* cephfs-mirror: reopen logs on SIGHUP (`issue#51413 <http://tracker.ceph.com/issues/51413>`_, `issue#51318 <http://tracker.ceph.com/issues/51318>`_, `pr#42097 <https://github.com/ceph/ceph/pull/42097>`_, Venky Shankar)
+* cephfs-top: self-adapt the display according the window size (`pr#41053 <https://github.com/ceph/ceph/pull/41053>`_, Xiubo Li)
+* client: Fix executeable access check for the root user (`pr#41294 <https://github.com/ceph/ceph/pull/41294>`_, Kotresh HR)
+* client: fix the opened inodes counter increasing (`pr#40685 <https://github.com/ceph/ceph/pull/40685>`_, Xiubo Li)
+* client: make Inode to inherit from RefCountedObject (`pr#41052 <https://github.com/ceph/ceph/pull/41052>`_, Xiubo Li)
+* cls/rgw: look for plain entries in non-ascii plain namespace too (`pr#41774 <https://github.com/ceph/ceph/pull/41774>`_, Mykola Golub)
+* common/buffer: adjust align before calling posix_memalign() (`pr#41249 <https://github.com/ceph/ceph/pull/41249>`_, Ilya Dryomov)
+* common/mempool: only fail tests if sharding is very bad (`pr#40566 <https://github.com/ceph/ceph/pull/40566>`_, singuliere)
+* common/options/global.yaml.in: increase default value of bluestore_cache_trim_max_skip_pinned (`pr#40918 <https://github.com/ceph/ceph/pull/40918>`_, Neha Ojha)
+* crush/crush: ensure alignof(crush_work_bucket) is 1 (`pr#41983 <https://github.com/ceph/ceph/pull/41983>`_, Kefu Chai)
+* debian,cmake,cephsqlite: hide non-public symbols (`pr#40689 <https://github.com/ceph/ceph/pull/40689>`_, Kefu Chai)
+* debian/control: ceph-mgr-modules-core does not Recommend ceph-mgr-rook (`pr#41877 <https://github.com/ceph/ceph/pull/41877>`_, Kefu Chai)
+* doc: pacific updates (`pr#42066 <https://github.com/ceph/ceph/pull/42066>`_, Patrick Donnelly)
+* librbd/cache/pwl: fix parsing of cache_type in create_image_cache_state() (`pr#41244 <https://github.com/ceph/ceph/pull/41244>`_, Ilya Dryomov)
+* librbd/mirror/snapshot: avoid UnlinkPeerRequest with a unlinked peer (`pr#41304 <https://github.com/ceph/ceph/pull/41304>`_, Arthur Outhenin-Chalandre)
+* librbd: don't stop at the first unremovable image when purging (`pr#41664 <https://github.com/ceph/ceph/pull/41664>`_, Ilya Dryomov)
+* make-dist: refuse to run if script path contains a colon (`pr#41086 <https://github.com/ceph/ceph/pull/41086>`_, Nathan Cutler)
+* mds: "FAILED ceph_assert(r == 0 || r == -2)" (`pr#42072 <https://github.com/ceph/ceph/pull/42072>`_, Xiubo Li)
+* mds: "cluster [ERR]   Error recovering journal 0x203: (2) No such file or directory" in cluster log" (`pr#42059 <https://github.com/ceph/ceph/pull/42059>`_, Xiubo Li)
+* mds: Add full caps to avoid osd full check (`pr#41691 <https://github.com/ceph/ceph/pull/41691>`_, Patrick Donnelly, Kotresh HR)
+* mds: CephFS kclient gets stuck when getattr() on a certain file (`pr#42062 <https://github.com/ceph/ceph/pull/42062>`_, "Yan, Zheng", Xiubo Li)
+* mds: Error ENOSYS: mds.a started profiler (`pr#42056 <https://github.com/ceph/ceph/pull/42056>`_, Xiubo Li)
+* mds: MDSLog::journaler pointer maybe crash with use-after-free (`pr#42060 <https://github.com/ceph/ceph/pull/42060>`_, Xiubo Li)
+* mds: avoid journaling overhead for setxattr("ceph.dir.subvolume") for no-op case (`pr#41995 <https://github.com/ceph/ceph/pull/41995>`_, Patrick Donnelly)
+* mds: do not assert when receiving a unknow metric type (`pr#41596 <https://github.com/ceph/ceph/pull/41596>`_, Patrick Donnelly, Xiubo Li)
+* mds: journal recovery thread is possibly asserting with mds_lock not locked (`pr#42058 <https://github.com/ceph/ceph/pull/42058>`_, Xiubo Li)
+* mds: mkdir on ephemerally pinned directory sometimes blocked on journal flush (`pr#42071 <https://github.com/ceph/ceph/pull/42071>`_, Xiubo Li)
+* mds: scrub error on inode 0x1 (`pr#41685 <https://github.com/ceph/ceph/pull/41685>`_, Milind Changire)
+* mds: standby-replay only trims cache when it reaches the end of the replay log (`pr#40855 <https://github.com/ceph/ceph/pull/40855>`_, Xiubo Li, Patrick Donnelly)
+* mgr/DaemonServer.cc: prevent mgr crashes caused by integer underflow that is triggered by large increases to pg_num/pgp_num (`pr#41862 <https://github.com/ceph/ceph/pull/41862>`_, Cory Snyder)
+* mgr/Dashboard: Remove erroneous elements in hosts-overview Grafana dashboard (`pr#40982 <https://github.com/ceph/ceph/pull/40982>`_, Malcolm Holmes)
+* mgr/dashboard: API Version changes do not apply to pre-defined methods (list, create etc.) (`pr#41675 <https://github.com/ceph/ceph/pull/41675>`_, Aashish Sharma)
+* mgr/dashboard: Alertmanager fails to POST alerts (`pr#41987 <https://github.com/ceph/ceph/pull/41987>`_, Avan Thakkar)
+* mgr/dashboard: Fix 500 error while exiting out of maintenance (`pr#41915 <https://github.com/ceph/ceph/pull/41915>`_, Nizamudeen A)
+* mgr/dashboard: Fix bucket name input allowing space in the value (`pr#42119 <https://github.com/ceph/ceph/pull/42119>`_, Nizamudeen A)
+* mgr/dashboard: Fix for query params resetting on change-password (`pr#41440 <https://github.com/ceph/ceph/pull/41440>`_, Nizamudeen A)
+* mgr/dashboard: Generate NPM dependencies manifest (`pr#41204 <https://github.com/ceph/ceph/pull/41204>`_, Nizamudeen A)
+* mgr/dashboard: Host Maintenance Follow ups (`pr#41056 <https://github.com/ceph/ceph/pull/41056>`_, Nizamudeen A)
+* mgr/dashboard: Include Network address and labels on Host Creation form (`pr#42027 <https://github.com/ceph/ceph/pull/42027>`_, Nizamudeen A)
+* mgr/dashboard: OSDs placement text is unreadable (`pr#41096 <https://github.com/ceph/ceph/pull/41096>`_, Aashish Sharma)
+* mgr/dashboard: RGW buckets async validator performance enhancement and name constraints (`pr#41296 <https://github.com/ceph/ceph/pull/41296>`_, Nizamudeen A)
+* mgr/dashboard: User database migration has been cut out (`pr#42140 <https://github.com/ceph/ceph/pull/42140>`_, Volker Theile)
+* mgr/dashboard: avoid data processing in crush-map component (`pr#41203 <https://github.com/ceph/ceph/pull/41203>`_, Avan Thakkar)
+* mgr/dashboard: bucket details: show lock retention period only in days (`pr#41948 <https://github.com/ceph/ceph/pull/41948>`_, Alfonso Martínez)
+* mgr/dashboard: crushmap tree doesn't display crush type other than root (`pr#42007 <https://github.com/ceph/ceph/pull/42007>`_, Kefu Chai, Avan Thakkar)
+* mgr/dashboard: disable NFSv3 support in dashboard (`pr#41200 <https://github.com/ceph/ceph/pull/41200>`_, Volker Theile)
+* mgr/dashboard: drop container image name and id from services list (`pr#41505 <https://github.com/ceph/ceph/pull/41505>`_, Avan Thakkar)
+* mgr/dashboard: fix API docs link (`pr#41507 <https://github.com/ceph/ceph/pull/41507>`_, Avan Thakkar)
+* mgr/dashboard: fix ESOCKETTIMEDOUT E2E failure (`pr#41427 <https://github.com/ceph/ceph/pull/41427>`_, Avan Thakkar)
+* mgr/dashboard: fix HAProxy (now called ingress) (`pr#41298 <https://github.com/ceph/ceph/pull/41298>`_, Avan Thakkar)
+* mgr/dashboard: fix OSD out count (`pr#42153 <https://github.com/ceph/ceph/pull/42153>`_, 胡玮文)
+* mgr/dashboard: fix OSDs Host details/overview grafana graphs (`issue#49769 <http://tracker.ceph.com/issues/49769>`_, `pr#41324 <https://github.com/ceph/ceph/pull/41324>`_, Alfonso Martínez, Michael Wodniok)
+* mgr/dashboard: fix base-href (`pr#41634 <https://github.com/ceph/ceph/pull/41634>`_, Avan Thakkar)
+* mgr/dashboard: fix base-href: revert it to previous approach (`pr#41251 <https://github.com/ceph/ceph/pull/41251>`_, Avan Thakkar)
+* mgr/dashboard: fix bucket objects and size calculations (`pr#41646 <https://github.com/ceph/ceph/pull/41646>`_, Avan Thakkar)
+* mgr/dashboard: fix bucket versioning when locking is enabled (`pr#41197 <https://github.com/ceph/ceph/pull/41197>`_, Avan Thakkar)
+* mgr/dashboard: fix for right sidebar nav icon not clickable (`pr#42008 <https://github.com/ceph/ceph/pull/42008>`_, Aaryan Porwal)
+* mgr/dashboard: fix set-ssl-certificate{,-key} commands (`pr#41170 <https://github.com/ceph/ceph/pull/41170>`_, Alfonso Martínez)
+* mgr/dashboard: fix typo: Filesystems to File Systems (`pr#42016 <https://github.com/ceph/ceph/pull/42016>`_, Navin Barnwal)
+* mgr/dashboard: ingress service creation follow-up (`pr#41428 <https://github.com/ceph/ceph/pull/41428>`_, Avan Thakkar)
+* mgr/dashboard: pass Grafana datasource in URL (`pr#41633 <https://github.com/ceph/ceph/pull/41633>`_, Ernesto Puerta)
+* mgr/dashboard: provide the service events when showing a service in the UI (`pr#41494 <https://github.com/ceph/ceph/pull/41494>`_, Aashish Sharma)
+* mgr/dashboard: run cephadm-backend e2e tests with KCLI (`pr#42156 <https://github.com/ceph/ceph/pull/42156>`_, Alfonso Martínez)
+* mgr/dashboard: set required env. variables in run-backend-api-tests.sh (`pr#41069 <https://github.com/ceph/ceph/pull/41069>`_, Alfonso Martínez)
+* mgr/dashboard: show RGW tenant user id correctly in 'NFS create export' form (`pr#41528 <https://github.com/ceph/ceph/pull/41528>`_, Alfonso Martínez)
+* mgr/dashboard: show partially deleted RBDs (`pr#41891 <https://github.com/ceph/ceph/pull/41891>`_, Tatjana Dehler)
+* mgr/dashboard: simplify object locking fields in 'Bucket Creation' form (`pr#41777 <https://github.com/ceph/ceph/pull/41777>`_, Alfonso Martínez)
+* mgr/dashboard: update frontend deps due to security vulnerabilities (`pr#41402 <https://github.com/ceph/ceph/pull/41402>`_, Alfonso Martínez)
+* mgr/dashboard:include compression stats on pool dashboard (`pr#41577 <https://github.com/ceph/ceph/pull/41577>`_, Ernesto Puerta, Paul Cuzner)
+* mgr/nfs: do not depend on cephadm.utils (`pr#41842 <https://github.com/ceph/ceph/pull/41842>`_, Sage Weil)
+* mgr/progress: ensure progress stays between [0,1] (`pr#41312 <https://github.com/ceph/ceph/pull/41312>`_, Dan van der Ster)
+* mgr/prometheus:Improve the pool metadata (`pr#40804 <https://github.com/ceph/ceph/pull/40804>`_, Paul Cuzner)
+* mgr/pybind/snap_schedule: do not fail when no fs snapshots are available (`pr#41044 <https://github.com/ceph/ceph/pull/41044>`_, Sébastien Han)
+* mgr/volumes/nfs: drop type param during cluster create (`pr#41005 <https://github.com/ceph/ceph/pull/41005>`_, Michael Fritch)
+* mon,doc: deprecate min_compat_client (`pr#41468 <https://github.com/ceph/ceph/pull/41468>`_, Patrick Donnelly)
+* mon/MonClient: reset authenticate_err in _reopen_session() (`pr#41019 <https://github.com/ceph/ceph/pull/41019>`_, Ilya Dryomov)
+* mon/MonClient: tolerate a rotating key that is slightly out of date (`pr#41450 <https://github.com/ceph/ceph/pull/41450>`_, Ilya Dryomov)
+* mon/OSDMonitor: drop stale failure_info after a grace period (`pr#41090 <https://github.com/ceph/ceph/pull/41090>`_, Kefu Chai)
+* mon/OSDMonitor: drop stale failure_info even if can_mark_down() (`pr#41982 <https://github.com/ceph/ceph/pull/41982>`_, Kefu Chai)
+* mon: load stashed map before mkfs monmap (`pr#41768 <https://github.com/ceph/ceph/pull/41768>`_, Dan van der Ster)
+* nfs backport May (`pr#41389 <https://github.com/ceph/ceph/pull/41389>`_, Varsha Rao)
+* os/FileStore: fix to handle readdir error correctly (`pr#41236 <https://github.com/ceph/ceph/pull/41236>`_, Misono Tomohiro)
+* os/bluestore: fix unexpected ENOSPC in Avl/Hybrid allocators (`pr#41655 <https://github.com/ceph/ceph/pull/41655>`_, Igor Fedotov, Neha Ojha)
+* os/bluestore: introduce multithreading sync for bluestore's repairer (`pr#41752 <https://github.com/ceph/ceph/pull/41752>`_, Igor Fedotov)
+* os/bluestore: tolerate zero length for allocators' init\_[add/rm]_free() (`pr#41753 <https://github.com/ceph/ceph/pull/41753>`_, Igor Fedotov)
+* osd/PG.cc: handle removal of pgmeta object (`pr#41680 <https://github.com/ceph/ceph/pull/41680>`_, Neha Ojha)
+* osd/osd_type: use f->dump_unsigned() when appropriate (`pr#42045 <https://github.com/ceph/ceph/pull/42045>`_, Kefu Chai)
+* osd/scrub: replace a ceph_assert() with a test (`pr#41944 <https://github.com/ceph/ceph/pull/41944>`_, Ronen Friedman)
+* osd: Override recovery, backfill and sleep related config options during OSD and mclock scheduler initialization (`pr#41125 <https://github.com/ceph/ceph/pull/41125>`_, Sridhar Seshasayee, Zac Dover)
+* osd: clear data digest when write_trunc (`pr#42019 <https://github.com/ceph/ceph/pull/42019>`_, Zengran Zhang)
+* osd: compute OSD's space usage ratio via raw space utilization (`pr#41113 <https://github.com/ceph/ceph/pull/41113>`_, Igor Fedotov)
+* osd: don't assert in-flight backfill is always in recovery list (`pr#41320 <https://github.com/ceph/ceph/pull/41320>`_, Mykola Golub)
+* osd: fix scrub reschedule bug (`pr#41971 <https://github.com/ceph/ceph/pull/41971>`_, wencong wan)
+* pacific: client: abort after MDS blocklist (`issue#50530 <http://tracker.ceph.com/issues/50530>`_, `pr#42070 <https://github.com/ceph/ceph/pull/42070>`_, Venky Shankar)
+* pybind/ceph_volume_client: use cephfs mkdirs api (`pr#42159 <https://github.com/ceph/ceph/pull/42159>`_, Patrick Donnelly)
+* pybind/mgr/devicehealth: scrape-health-metrics command accidentally renamed to scrape-daemon-health-metrics (`pr#41089 <https://github.com/ceph/ceph/pull/41089>`_, Patrick Donnelly)
+* pybind/mgr/progress: Disregard unreported pgs (`pr#41872 <https://github.com/ceph/ceph/pull/41872>`_, Kamoltat)
+* pybind/mgr/snap_schedule: Invalid command: Unexpected argument 'fs=cephfs' (`pr#42064 <https://github.com/ceph/ceph/pull/42064>`_, Patrick Donnelly)
+* qa/config/rados: add dispatch delay testing params (`pr#41136 <https://github.com/ceph/ceph/pull/41136>`_, Deepika Upadhyay)
+* qa/distros/podman: preserve registries.conf (`pr#40729 <https://github.com/ceph/ceph/pull/40729>`_, Sage Weil)
+* qa/suites/rados/standalone: remove mon_election symlink (`pr#41212 <https://github.com/ceph/ceph/pull/41212>`_, Neha Ojha)
+* qa/suites/rados: add simultaneous scrubs to the thrasher (`pr#42120 <https://github.com/ceph/ceph/pull/42120>`_, Ronen Friedman)
+* qa/tasks/qemu: precise repos have been archived (`pr#41643 <https://github.com/ceph/ceph/pull/41643>`_, Ilya Dryomov)
+* qa/tests: corrected point versions to reflect latest releases (`pr#41313 <https://github.com/ceph/ceph/pull/41313>`_, Yuri Weinstein)
+* qa/tests: initial checkin for pacific-p2p suite (2) (`pr#41208 <https://github.com/ceph/ceph/pull/41208>`_, Yuri Weinstein)
+* qa/tests: replaced ubuntu_latest.yaml with ubuntu 20.04 (`pr#41460 <https://github.com/ceph/ceph/pull/41460>`_, Patrick Donnelly, Kefu Chai)
+* qa/upgrade: conditionally disable update_features tests (`pr#41629 <https://github.com/ceph/ceph/pull/41629>`_, Deepika)
+* qa/workunits/rbd: use bionic version of qemu-iotests for focal (`pr#41195 <https://github.com/ceph/ceph/pull/41195>`_, Ilya Dryomov)
+* qa: AttributeError: 'RemoteProcess' object has no attribute 'split' (`pr#41811 <https://github.com/ceph/ceph/pull/41811>`_, Patrick Donnelly)
+* qa: add async dirops testing (`pr#41823 <https://github.com/ceph/ceph/pull/41823>`_, Patrick Donnelly)
+* qa: check mounts attribute in ctx (`pr#40634 <https://github.com/ceph/ceph/pull/40634>`_, Jos Collin)
+* qa: convert some legacy Filesystem.rados calls (`pr#40996 <https://github.com/ceph/ceph/pull/40996>`_, Patrick Donnelly)
+* qa: drop the distro~HEAD directory from the fs suite (`pr#41169 <https://github.com/ceph/ceph/pull/41169>`_, Radoslaw Zarzynski)
+* qa: fs:bugs does not specify distro (`pr#42063 <https://github.com/ceph/ceph/pull/42063>`_, Patrick Donnelly)
+* qa: fs:upgrade uses teuthology default distro (`pr#42067 <https://github.com/ceph/ceph/pull/42067>`_, Patrick Donnelly)
+* qa: scrub code does not join scrubopts with comma (`pr#42065 <https://github.com/ceph/ceph/pull/42065>`_, Kefu Chai, Patrick Donnelly)
+* qa: test_data_scan.TestDataScan.test_pg_files AssertionError: Items in the second set but not the first (`pr#42069 <https://github.com/ceph/ceph/pull/42069>`_, Xiubo Li)
+* qa: test_ephemeral_pin_distribution failure (`pr#41659 <https://github.com/ceph/ceph/pull/41659>`_, Patrick Donnelly)
+* qa: update RHEL to 8.4 (`pr#41822 <https://github.com/ceph/ceph/pull/41822>`_, Patrick Donnelly)
+* rbd-mirror: fix segfault in snapshot replayer shutdown (`pr#41503 <https://github.com/ceph/ceph/pull/41503>`_, Arthur Outhenin-Chalandre)
+* rbd: --source-spec-file should be --source-spec-path (`pr#41122 <https://github.com/ceph/ceph/pull/41122>`_, Ilya Dryomov)
+* rbd: don't attempt to interpret image cache state json (`pr#41281 <https://github.com/ceph/ceph/pull/41281>`_, Ilya Dryomov)
+* rgw: Simplify log shard probing and err on the side of omap (`pr#41576 <https://github.com/ceph/ceph/pull/41576>`_, Adam C. Emerson)
+* rgw: completion of multipart upload leaves delete marker (`pr#41769 <https://github.com/ceph/ceph/pull/41769>`_, J. Eric Ivancich)
+* rgw: crash on multipart upload to bucket with policy (`pr#41893 <https://github.com/ceph/ceph/pull/41893>`_, Or Friedmann)
+* rgw: radosgw_admin remove bucket not purging past 1,000 objects (`pr#41863 <https://github.com/ceph/ceph/pull/41863>`_, J. Eric Ivancich)
+* rgw: radoslist incomplete multipart parts marker (`pr#40819 <https://github.com/ceph/ceph/pull/40819>`_, J. Eric Ivancich)
+* rocksdb: pickup fix to detect PMULL instruction (`pr#41079 <https://github.com/ceph/ceph/pull/41079>`_, Kefu Chai)
+* session dump includes completed_requests twice, once as an integer and once as a list (`pr#42057 <https://github.com/ceph/ceph/pull/42057>`_, Dan van der Ster)
+* systemd: remove `ProtectClock=true` for `ceph-osd@.service` (`pr#41232 <https://github.com/ceph/ceph/pull/41232>`_, Wong Hoi Sing Edison)
+* test/librbd: use really invalid domain (`pr#42010 <https://github.com/ceph/ceph/pull/42010>`_, Mykola Golub)
+* win32\*.sh: disable libcephsqlite when targeting Windows (`pr#40557 <https://github.com/ceph/ceph/pull/40557>`_, Lucian Petrut)
+
+
+v16.2.4 Pacific
+===============
+
+This is a hotfix release addressing a number of security issues and regressions. We recommend all users update to this release.
+
+Changelog
+---------
+
+* mgr/dashboard: fix base-href: revert it to previous approach (`issue#50684 <https://tracker.ceph.com/issues/50684>`_, Avan Thakkar)
+* mgr/dashboard: fix cookie injection issue (:ref:`CVE-2021-3509`, Ernesto Puerta)
+* mgr/dashboard: fix set-ssl-certificate{,-key} commands (`issue#50519 <https://tracker.ceph.com/issues/50519>`_, Alfonso Martínez)
+* rgw: RGWSwiftWebsiteHandler::is_web_dir checks empty subdir_name (:ref:`CVE-2021-3531`, Felix Huettner)
+* rgw: sanitize \r in s3 CORSConfiguration's ExposeHeader (:ref:`CVE-2021-3524`, Sergey Bobrov, Casey Bodley)
+* systemd: remove ProtectClock=true for ceph-osd@.service (`issue#50347 <https://tracker.ceph.com/issues/50347>`_, Wong Hoi Sing Edison)
+
+v16.2.3 Pacific
+===============
+
+This is the third backport release in the Pacific series.  We recommend all users
+update to this release.
+
+Notable Changes
+---------------
+
+* This release fixes a cephadm upgrade bug that caused some systems to get stuck in a loop
+  restarting the first mgr daemon.
+
+
+v16.2.2 Pacific
+===============
+
+This is the second backport release in the Pacific series. We recommend all
+users update to this release.
+
+Notable Changes
+---------------
+
+* Cephadm now supports an *ingress* service type that provides load
+  balancing and HA (via haproxy and keepalived on a virtual IP) for
+  RGW service (see :ref:`orchestrator-haproxy-service-spec`).  (The experimental
+  *rgw-ha* service has been removed.)
+
+Changelog
+---------
+
+* ceph-fuse: src/include/buffer.h: 1187: FAILED ceph_assert(_num <= 1024) (`pr#40628 <https://github.com/ceph/ceph/pull/40628>`_, Yanhu Cao)
+* ceph-volume: fix "device" output (`pr#41054 <https://github.com/ceph/ceph/pull/41054>`_, Sébastien Han)
+* ceph-volume: fix raw listing when finding OSDs from different clusters (`pr#40985 <https://github.com/ceph/ceph/pull/40985>`_, Sébastien Han)
+* ceph.spec.in: Enable tcmalloc on IBM Power and Z (`pr#39488 <https://github.com/ceph/ceph/pull/39488>`_, Nathan Cutler, Yaakov Selkowitz)
+* cephadm april batch 3 (`issue#49737 <http://tracker.ceph.com/issues/49737>`_, `pr#40922 <https://github.com/ceph/ceph/pull/40922>`_, Adam King, Sage Weil, Daniel Pivonka, Shreyaa Sharma, Sebastian Wagner, Juan Miguel Olmo Martínez, Zac Dover, Jeff Layton, Guillaume Abrioux, 胡玮文, Melissa Li, Nathan Cutler, Yaakov Selkowitz)
+* cephadm: april batch 1 (`pr#40544 <https://github.com/ceph/ceph/pull/40544>`_, Sage Weil, Daniel Pivonka, Joao Eduardo Luis, Adam King)
+* cephadm: april batch backport 2 (`pr#40746 <https://github.com/ceph/ceph/pull/40746>`_, Guillaume Abrioux, Sage Weil, Paul Cuzner)
+* cephadm: specify addr on bootstrap's host add (`pr#40554 <https://github.com/ceph/ceph/pull/40554>`_, Joao Eduardo Luis)
+* cephfs: minor ceph-dokan improvements (`pr#40627 <https://github.com/ceph/ceph/pull/40627>`_, Lucian Petrut)
+* client: items pinned in cache preventing unmount (`pr#40629 <https://github.com/ceph/ceph/pull/40629>`_, Xiubo Li)
+* client: only check pool permissions for regular files (`pr#40686 <https://github.com/ceph/ceph/pull/40686>`_, Xiubo Li)
+* cmake: define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT globally (`pr#40706 <https://github.com/ceph/ceph/pull/40706>`_, Kefu Chai)
+* cmake: pass unparsed args to add_ceph_test() (`pr#40523 <https://github.com/ceph/ceph/pull/40523>`_, Kefu Chai)
+* cmake: use --smp 1 --memory 256M to crimson tests (`pr#40568 <https://github.com/ceph/ceph/pull/40568>`_, Kefu Chai)
+* crush/CrushLocation: do not print logging message in constructor (`pr#40679 <https://github.com/ceph/ceph/pull/40679>`_, Alex Wu)
+* doc/cephfs/nfs: add user id, fs name and key to FSAL block (`pr#40687 <https://github.com/ceph/ceph/pull/40687>`_, Varsha Rao)
+* include/librados: fix doxygen syntax for docs build (`pr#40805 <https://github.com/ceph/ceph/pull/40805>`_, Josh Durgin)
+* mds: "cluster [WRN] Scrub error on inode 0x1000000039d (/client.0/tmp/blogbench-1.0/src/blogtest_in) see mds.a log and `damage ls` output for details" (`pr#40825 <https://github.com/ceph/ceph/pull/40825>`_, Milind Changire)
+* mds: skip the buffer in UnknownPayload::decode() (`pr#40682 <https://github.com/ceph/ceph/pull/40682>`_, Xiubo Li)
+* mgr/PyModule: put mgr_module_path before Py_GetPath() (`pr#40517 <https://github.com/ceph/ceph/pull/40517>`_, Kefu Chai)
+* mgr/dashboard: Device health status is not getting listed under hosts section (`pr#40494 <https://github.com/ceph/ceph/pull/40494>`_, Aashish Sharma)
+* mgr/dashboard: Fix for alert notification message being undefined (`pr#40588 <https://github.com/ceph/ceph/pull/40588>`_, Nizamudeen A)
+* mgr/dashboard: Fix for broken User management role cloning (`pr#40398 <https://github.com/ceph/ceph/pull/40398>`_, Nizamudeen A)
+* mgr/dashboard: Improve descriptions in some parts of the dashboard (`pr#40545 <https://github.com/ceph/ceph/pull/40545>`_, Nizamudeen A)
+* mgr/dashboard: Remove username and password from request body (`pr#40981 <https://github.com/ceph/ceph/pull/40981>`_, Nizamudeen A)
+* mgr/dashboard: Remove username, password fields from Manager Modules/dashboard,influx (`pr#40489 <https://github.com/ceph/ceph/pull/40489>`_, Aashish Sharma)
+* mgr/dashboard: Revoke read-only user's access to Manager modules (`pr#40648 <https://github.com/ceph/ceph/pull/40648>`_, Nizamudeen A)
+* mgr/dashboard: Unable to login to ceph dashboard until clearing cookies manually (`pr#40586 <https://github.com/ceph/ceph/pull/40586>`_, Avan Thakkar)
+* mgr/dashboard: debug nodeenv hangs (`pr#40815 <https://github.com/ceph/ceph/pull/40815>`_, Ernesto Puerta)
+* mgr/dashboard: filesystem pool size should use stored stat (`pr#40980 <https://github.com/ceph/ceph/pull/40980>`_, Avan Thakkar)
+* mgr/dashboard: fix broken feature toggles (`pr#40474 <https://github.com/ceph/ceph/pull/40474>`_, Ernesto Puerta)
+* mgr/dashboard: fix duplicated rows when creating NFS export (`pr#40990 <https://github.com/ceph/ceph/pull/40990>`_, Alfonso Martínez)
+* mgr/dashboard: fix errors when creating NFS export (`pr#40822 <https://github.com/ceph/ceph/pull/40822>`_, Alfonso Martínez)
+* mgr/dashboard: improve telemetry opt-in reminder notification message (`pr#40887 <https://github.com/ceph/ceph/pull/40887>`_, Waad Alkhoury)
+* mgr/dashboard: test prometheus rules through promtool (`pr#40929 <https://github.com/ceph/ceph/pull/40929>`_, Aashish Sharma, Kefu Chai)
+* mon: Modifying trim logic to change paxos_service_trim_max dynamically (`pr#40691 <https://github.com/ceph/ceph/pull/40691>`_, Aishwarya Mathuria)
+* monmaptool: Don't call set_port on an invalid address (`pr#40690 <https://github.com/ceph/ceph/pull/40690>`_, Brad Hubbard, Kefu Chai)
+* os/FileStore: don't propagate split/merge error to "create"/"remove" (`pr#40989 <https://github.com/ceph/ceph/pull/40989>`_, Mykola Golub)
+* os/bluestore/BlueFS: do not _flush_range deleted files (`pr#40677 <https://github.com/ceph/ceph/pull/40677>`_, weixinwei)
+* osd/PeeringState: fix acting_set_writeable min_size check (`pr#40759 <https://github.com/ceph/ceph/pull/40759>`_, Samuel Just)
+* packaging: require ceph-common for immutable object cache daemon (`pr#40665 <https://github.com/ceph/ceph/pull/40665>`_, Ilya Dryomov)
+* pybind/mgr/volumes: deadlock on async job hangs finisher thread (`pr#40630 <https://github.com/ceph/ceph/pull/40630>`_, Kefu Chai, Patrick Donnelly)
+* qa/suites/krbd: don't require CEPHX_V2 for unmap subsuite (`pr#40826 <https://github.com/ceph/ceph/pull/40826>`_, Ilya Dryomov)
+* qa/suites/rados/cephadm: stop testing on broken focal kubic podman (`pr#40512 <https://github.com/ceph/ceph/pull/40512>`_, Sage Weil)
+* qa/tasks/ceph.conf: shorten cephx TTL for testing (`pr#40663 <https://github.com/ceph/ceph/pull/40663>`_, Sage Weil)
+* qa/tasks/cephfs: create enough subvolumes (`pr#40688 <https://github.com/ceph/ceph/pull/40688>`_, Ramana Raja)
+* qa/tasks/vstart_runner.py: start max required mgrs (`pr#40612 <https://github.com/ceph/ceph/pull/40612>`_, Alfonso Martínez)
+* qa/tasks: Add wait_for_clean() check prior to initiating scrubbing (`pr#40461 <https://github.com/ceph/ceph/pull/40461>`_, Sridhar Seshasayee)
+* qa: "AttributeError: 'NoneType' object has no attribute 'mon_manager'" (`pr#40645 <https://github.com/ceph/ceph/pull/40645>`_, Rishabh Dave)
+* qa: "log [ERR] : error reading sessionmap 'mds2_sessionmap'" (`pr#40852 <https://github.com/ceph/ceph/pull/40852>`_, Patrick Donnelly)
+* qa: fix ino_release_cb racy behavior (`pr#40683 <https://github.com/ceph/ceph/pull/40683>`_, Patrick Donnelly)
+* qa: fs:cephadm mount does not wait for mds to be created (`pr#40528 <https://github.com/ceph/ceph/pull/40528>`_, Patrick Donnelly)
+* qa: test standby_replay in workloads (`pr#40853 <https://github.com/ceph/ceph/pull/40853>`_, Patrick Donnelly)
+* rbd-mirror: fix UB while registering perf counters (`pr#40680 <https://github.com/ceph/ceph/pull/40680>`_, Arthur Outhenin-Chalandre)
+* rgw: add latency to the request summary of an op (`pr#40448 <https://github.com/ceph/ceph/pull/40448>`_, Ali Maredia)
+* rgw: Backport of datalog improvements to Pacific (`pr#40559 <https://github.com/ceph/ceph/pull/40559>`_, Yuval Lifshitz, Adam C. Emerson)
+* test: disable mgr/mirroring for `test_mirroring_init_failure_with_recovery` test (`issue#50020 <http://tracker.ceph.com/issues/50020>`_, `pr#40684 <https://github.com/ceph/ceph/pull/40684>`_, Venky Shankar)
+* tools/cephfs_mirror/PeerReplayer.cc: add missing include (`pr#40678 <https://github.com/ceph/ceph/pull/40678>`_, Duncan Bellamy)
+* vstart.sh: disable "auth_allow_insecure_global_id_reclaim" (`pr#40957 <https://github.com/ceph/ceph/pull/40957>`_, Kefu Chai)
+
+
+v16.2.1 Pacific
+===============
+
+This is the first bugfix release in the Pacific stable series.  It addresses a
+security vulnerability in the Ceph authentication framework.
+
+We recommend all Pacific users upgrade.
+
+Security fixes
+--------------
+
+* This release includes a security fix that ensures the global_id
+  value (a numeric value that should be unique for every authenticated
+  client or daemon in the cluster) is reclaimed after a network
+  disconnect or ticket renewal in a secure fashion.  Two new health
+  alerts may appear during the upgrade indicating that there are
+  clients or daemons that are not yet patched with the appropriate
+  fix.
+
+  To temporarily mute the health alerts around insecure clients for the duration of the
+  upgrade, you may want to::
+
+    ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1h
+    ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1h
+
+  For more information, see :ref:`CVE-2021-20288`.
+
+
+v16.2.0 Pacific
+===============
+
+This is the first stable release of Ceph Pacific.
+
+Major Changes from Octopus
+--------------------------
+
+General
+~~~~~~~
+
+* Cephadm can automatically upgrade an Octopus cluster to Pacific with a single
+  command to start the process.
+* Cephadm has improved significantly over the past year, with improved
+  support for RGW (standalone and multisite), and new support for NFS
+  and iSCSI.  Most of these changes have already been backported to
+  recent Octopus point releases, but with the Pacific release we will
+  switch to backporting bug fixes only.
+* :ref:`Packages <packages>` are built for the following distributions:
+
+  - CentOS 8
+  - Ubuntu 20.04 (Focal)
+  - Ubuntu 18.04 (Bionic)
+  - Debian Buster
+  - :ref:`Container image <containers>` (based on CentOS 8)
+
+  With the exception of Debian Buster, packages and containers are
+  built for both x86_64 and aarch64 (arm64) architectures.
+
+  Note that cephadm clusters may work on many other distributions,
+  provided Python 3 and a recent version of Docker or Podman is
+  available to manage containers.  For more information, see
+  :ref:`cephadm-host-requirements`.
+
+
+Dashboard
+~~~~~~~~~
+
+The :ref:`mgr-dashboard` brings improvements in the following management areas:
+
+* Orchestrator/Cephadm:
+
+  - Host management: maintenance mode, labels.
+  - Services: display placement specification.
+  - OSD: disk replacement, display status of ongoing deletion, and improved
+    health/SMART diagnostics reporting.
+
+* Official :ref:`mgr ceph api`:
+
+  - OpenAPI v3 compliant.
+  - Stability commitment starting from Pacific release.
+  - Versioned via HTTP ``Accept`` header (starting with v1.0).
+  - Thoroughly tested (>90% coverage and per Pull Request validation).
+  - Fully documented.
+
+* RGW:
+
+  - Multi-site synchronization monitoring.
+  - Management of multiple RGW daemons and their resources (buckets and users).
+  - Bucket and user quota usage visualization.
+  - Improved configuration of S3 tenanted users.
+
+* Security (multiple enhancements and fixes resulting from a pen testing conducted by IBM):
+
+  - Account lock-out after a configurable number of failed log-in attempts.
+  - Improved cookie policies to mitigate XSS/CSRF attacks.
+  - Reviewed and improved security in HTTP headers.
+  - Sensitive information reviewed and removed from logs and error messages.
+  - TLS 1.0 and 1.1 support disabled.
+  - Debug mode when enabled triggers HEALTH_WARN.
+
+* Pools:
+
+  - Improved visualization of replication and erasure coding modes.
+  - CLAY erasure code plugin supported.
+
+* Alerts and notifications:
+
+  - Alert triggered on MTU mismatches in the cluster network.
+  - Favicon changes according cluster status.
+
+* Other:
+
+  - Landing page: improved charts and visualization.
+  - Telemetry configuration wizard.
+  - OSDs: management of individual OSD flags.
+  - RBD: per-RBD image Grafana dashboards.
+  - CephFS: Dirs and Caps displayed.
+  - NFS: v4 support only (v3 backward compatibility planned).
+  - Front-end: Angular 10 update.
+
+
+RADOS
+~~~~~
+
+* Pacific introduces :ref:`bluestore-rocksdb-sharding`, which reduces disk space requirements.
+
+* Ceph now provides QoS between client I/O and background operations via the
+  mclock scheduler.
+
+* The balancer is now on by default in upmap mode to improve distribution of
+  PGs across OSDs.
+
+* The output of ``ceph -s`` has been improved to show recovery progress in
+  one progress bar. More detailed progress bars are visible via the
+  ``ceph progress`` command.
+
+
+RBD block storage
+~~~~~~~~~~~~~~~~~
+
+* Image live-migration feature has been extended to support external data
+  sources.  Images can now be instantly imported from local files, remote
+  files served over HTTP(S) or remote S3 buckets in ``raw`` (``rbd export v1``)
+  or basic ``qcow`` and ``qcow2`` formats.  Support for ``rbd export v2``
+  format, advanced QCOW features and ``rbd export-diff`` snapshot differentials
+  is expected in future releases.
+
+* Initial support for client-side encryption has been added.  This is based
+  on LUKS and in future releases will allow using per-image encryption keys
+  while maintaining snapshot and clone functionality -- so that parent image
+  and potentially multiple clone images can be encrypted with different keys.
+
+* A new persistent write-back cache is available.  The cache operates in
+  a log-structured manner, providing full point-in-time consistency for the
+  backing image.  It should be particularly suitable for PMEM devices.
+
+* A Windows client is now available in the form of ``librbd.dll`` and
+  ``rbd-wnbd`` (Windows Network Block Device) daemon.  It allows mapping,
+  unmapping and manipulating images similar to ``rbd-nbd``.
+
+* librbd API now offers quiesce/unquiesce hooks, allowing for coordinated
+  snapshot creation.
+
+
+RGW object storage
+~~~~~~~~~~~~~~~~~~
+
+* Initial support for S3 Select. See :ref:`s3-select-feature-table` for supported queries.
+
+* Bucket notification topics can be configured as ``persistent``, where events
+  are recorded in rados for reliable delivery.
+
+* Bucket notifications can be delivered to SSL-enabled AMQP endpoints.
+
+* Lua scripts can be run during requests and access their metadata.
+
+* SSE-KMS now supports KMIP as a key management service.
+
+* Multisite data logs can now be deployed on ``cls_fifo`` to avoid large omap
+  cluster warnings and make their trimming cheaper. See ``rgw_data_log_backing``.
+
+
+CephFS distributed file system
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* The CephFS MDS modifies on-RADOS metadata such that the new format is no
+  longer backwards compatible. It is not possible to downgrade a file system from
+  Pacific (or later) to an older release.
+
+* Multiple file systems in a single Ceph cluster is now stable. New Ceph
+  clusters enable support for multiple file systems by default. Existing clusters
+  must still set the "enable_multiple" flag on the FS. See also
+  :ref:`cephfs-multifs`.
+
+* A new ``mds_autoscaler`` ``ceph-mgr`` plugin is available for automatically
+  deploying MDS daemons in response to changes to the ``max_mds`` configuration.
+  Expect further enhancements in the future to simplify and automate MDS scaling.
+
+* ``cephfs-top`` is a new utility for looking at performance metrics from CephFS
+  clients. It is development preview quality and will have bugs. For more
+  information, see :ref:`cephfs-top`.
+
+* A new ``snap_schedule`` ``ceph-mgr`` plugin provides a command toolset for
+  scheduling snapshots on a CephFS file system. For more information, see
+  :ref:`snap-schedule`.
+
+* First class NFS gateway support in Ceph is here! It's now possible to create
+  scale-out ("active-active") NFS gateway clusters that export CephFS using
+  a few commands. The gateways are deployed via cephadm (or Rook, in the future).
+  For more information, see :ref:`mgr-nfs`.
+
+* Multiple active MDS file system scrub is now stable. It is no longer necessary
+  to set ``max_mds`` to 1 and wait for non-zero ranks to stop. Scrub commands
+  can only be sent to rank 0: ``ceph tell mds.<fs_name>:0 scrub start /path ...``.
+  For more information, see :ref:`mds-scrub`.
+
+* Ephemeral pinning -- policy based subtree pinning -- is considered stable.
+  ``mds_export_ephemeral_random`` and ``mds_export_ephemeral_distributed`` now
+  default to true. For more information, see :ref:`cephfs-ephemeral-pinning`.
+
+* A new ``cephfs-mirror`` daemon is available to mirror CephFS file systems to
+  a remote Ceph cluster. For more information, see :ref:`cephfs-mirroring`.
+
+* A Windows client is now available for connecting to CephFS. This is offered
+  through a new ``ceph-dokan`` utility which operates via the Dokan userspace
+  API, similar to FUSE. For more information, see :ref:`ceph-dokan`.
+
+
+.. _upgrading_from_octopus_or_nautilus:
+
+Upgrading from Octopus or Nautilus
+----------------------------------
+
+Before starting, make sure your cluster is stable and healthy (no down or
+recovering OSDs).  (This is optional, but recommended.)
+
+
+.. note::
+  WARNING: Please do not set `bluestore_fsck_quick_fix_on_mount` to true or
+  run `ceph-bluestore-tool` repair or quick-fix commands in Pacific versions
+  <= 16.2.6, because this can lead to data corruption, details in
+  https://tracker.ceph.com/issues/53062.
+
+.. note::
+   When using multiple active Ceph Metadata Servers, ensure that there are
+   no pending stray entries which are directories for active ranks except rank 0 as
+   starting an upgrade (which sets `max_mds` to 1) could crash the Ceph
+   Metadata Server. The following command should return zero (0) stray entries
+   for all stray directories::
+
+     # for idx in {0..9}; do ceph tell mds.<rank> dump tree ~mdsdir/stray$idx| jq '.[] | select (.nlink == 0 and .dir_layout.dir_hash > 0) | .stray_prior_path' | wc -l; done
+
+   Ensure that all active ranks except rank 0 are checked for absence of stray
+   entries which are directories (using the above command). Details are captured
+   in http://tracker.ceph.com/issues/53597.
+
+Upgrading cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your cluster is deployed with cephadm (first introduced in Octopus), then
+the upgrade process is entirely automated.  To initiate the upgrade,
+
+  .. prompt:: bash #
+
+    ceph orch upgrade start --ceph-version 16.2.0
+
+The same process is used to upgrade to future minor releases.
+
+Upgrade progress can be monitored with ``ceph -s`` (which provides a simple
+progress bar) or more verbosely with
+
+  .. prompt:: bash #
+
+    ceph -W cephadm
+
+The upgrade can be paused or resumed with
+
+  .. prompt:: bash #
+
+    ceph orch upgrade pause   # to pause
+    ceph orch upgrade resume  # to resume
+
+or canceled with
+
+  .. prompt:: bash #
+
+    ceph orch upgrade stop
+
+Note that canceling the upgrade simply stops the process; there is no ability to
+downgrade back to Octopus.
+
+.. note:
+
+   If you have deployed an RGW service on Octopus using the default port (7280), you
+   will need to redeploy it because the default port changed (to 80 or 443, depending
+   on whether SSL is enabled):
+
+   .. prompt: bash #
+
+     ceph orch apply rgw <realm>.<zone> --port 7280
+
+
+Upgrading non-cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+   If you cluster is running Octopus (15.2.x), you might choose
+   to first convert it to use cephadm so that the upgrade to Pacific
+   is automated (see above).  For more information, see
+   :ref:`cephadm-adoption`.
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional,
+   but recommended.)::
+
+     # ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.  For example, on each monitor host,::
+
+     # systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is
+   complete by looking for the ``octopus`` string in the mon
+   map.  The command::
+
+     # ceph mon dump | grep min_mon_release
+
+   should report::
+
+     min_mon_release 16 (pacific)
+
+   If it doesn't, that implies that one or more monitors hasn't been
+   upgraded and restarted and/or the quorum does not include all monitors.
+
+#. Upgrade ``ceph-mgr`` daemons by installing the new packages and
+   restarting all manager daemons.  For example, on each manager host,::
+
+     # systemctl restart ceph-mgr.target
+
+   Verify the ``ceph-mgr`` daemons are running by checking ``ceph
+   -s``::
+
+     # ceph -s
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all OSD hosts::
+
+     # systemctl restart ceph-osd.target
+
+   Note that if you are upgrading from Nautilus, the first time each
+   OSD starts, it will do a format conversion to improve the
+   accounting for "omap" data.  This may take a few minutes to as much
+   as a few hours (for an HDD with lots of omap data).  You can
+   disable this automatic conversion with::
+
+     # ceph config set osd bluestore_fsck_quick_fix_on_mount false
+
+   You can monitor the progress of the OSD upgrades with the
+   ``ceph versions`` or ``ceph osd versions`` commands::
+
+     # ceph osd versions
+     {
+        "ceph version 14.2.5 (...) nautilus (stable)": 12,
+        "ceph version 16.2.0 (...) pacific (stable)": 22,
+     }
+
+#. Upgrade all CephFS MDS daemons. For each CephFS file system,
+
+   #. Disable FSMap sanity checks::
+
+        # ceph config set mon mon_mds_skip_sanity true
+
+   #. Disable standby_replay::
+
+        # ceph fs set <fs_name> allow_standby_replay false
+
+   #. Reduce the number of ranks to 1.  (Make note of the original
+      number of MDS daemons first if you plan to restore it later.)::
+
+	# ceph status
+	# ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by
+      periodically checking the status::
+
+	# ceph status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with::
+
+	# systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS::
+
+	# ceph status
+
+   #. Upgrade the last remaining MDS daemon by installing the new
+      packages and restarting the daemon::
+
+        # systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline::
+
+	# systemctl start ceph-mds.target
+
+   #. Restore the original value of ``max_mds`` for the volume::
+
+	# ceph fs set <fs_name> max_mds <original_max_mds>
+
+   #. Remove `mon_mds_skip_sanity` setting::
+
+        # ceph config rm mon mon_mds_skip_sanity
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts::
+
+     # systemctl restart ceph-radosgw.target
+
+#. Complete the upgrade by disallowing pre-Pacific OSDs and enabling
+   all new Pacific-only functionality::
+
+     # ceph osd require-osd-release pacific
+
+#. If you set ``noout`` at the beginning, be sure to clear it with::
+
+     # ceph osd unset noout
+
+#. Consider transitioning your cluster to use the cephadm deployment
+   and orchestration framework to simplify cluster management and
+   future upgrades.  For more information on converting an existing
+   cluster to cephadm, see :ref:`cephadm-adoption`.
+
+
+Post-upgrade
+~~~~~~~~~~~~
+
+#. Verify the cluster is healthy with ``ceph health``.
+
+   If your CRUSH tunables are older than Hammer, Ceph will now issue a
+   health warning.  If you see a health alert to that effect, you can
+   revert this change with::
+
+     ceph config set mon mon_crush_min_required_version firefly
+
+   If Ceph does not complain, however, then we recommend you also
+   switch any existing CRUSH buckets to straw2, which was added back
+   in the Hammer release.  If you have any 'straw' buckets, this will
+   result in a modest amount of data movement, but generally nothing
+   too severe.::
+
+     ceph osd getcrushmap -o backup-crushmap
+     ceph osd crush set-all-straw-buckets-to-straw2
+
+   If there are problems, you can easily revert with::
+
+     ceph osd setcrushmap -i backup-crushmap
+
+   Moving to 'straw2' buckets will unlock a few recent features, like
+   the `crush-compat` :ref:`balancer <balancer>` mode added back in Luminous.
+
+#. If you did not already do so when upgrading from Mimic, we
+   recommended you enable the new :ref:`v2 network protocol <msgr2>`,
+   issue the following command::
+
+     ceph mon enable-msgr2
+
+   This will instruct all monitors that bind to the old default port
+   6789 for the legacy v1 protocol to also bind to the new 3300 v2
+   protocol port.  To see if all monitors have been updated,::
+
+     ceph mon dump
+
+   and verify that each monitor has both a ``v2:`` and ``v1:`` address
+   listed.
+
+#. Consider enabling the :ref:`telemetry module <telemetry>` to send
+   anonymized usage statistics and crash information to the Ceph
+   upstream developers.  To see what would be reported (without actually
+   sending any information to anyone),::
+
+     ceph mgr module enable telemetry
+     ceph telemetry show
+
+   If you are comfortable with the data that is reported, you can opt-in to
+   automatically report the high-level cluster metadata with::
+
+     ceph telemetry on
+
+   The public dashboard that aggregates Ceph telemetry can be found at
+   `https://telemetry-public.ceph.com/ <https://telemetry-public.ceph.com/>`_.
+
+   For more information about the telemetry module, see :ref:`the
+   documentation <telemetry>`.
+
+
+Upgrade from pre-Nautilus releases (like Mimic or Luminous)
+-----------------------------------------------------------
+
+You must first upgrade to Nautilus (14.2.z) or Octopus (15.2.z) before
+upgrading to Pacific.
+
+
+Notable Changes
+---------------
+
+* A new library is available, libcephsqlite. It provides a SQLite Virtual File
+  System (VFS) on top of RADOS. The database and journals are striped over
+  RADOS across multiple objects for virtually unlimited scaling and throughput
+  only limited by the SQLite client. Applications using SQLite may change to
+  the Ceph VFS with minimal changes, usually just by specifying the alternate
+  VFS. We expect the library to be most impactful and useful for applications
+  that were storing state in RADOS omap, especially without striping which
+  limits scalability.
+
+* New ``bluestore_rocksdb_options_annex`` config parameter. Complements
+  ``bluestore_rocksdb_options`` and allows setting rocksdb options without
+  repeating the existing defaults.
+
+* $pid expansion in config paths like ``admin_socket`` will now properly expand
+  to the daemon pid for commands like ``ceph-mds`` or ``ceph-osd``. Previously
+  only ``ceph-fuse``/``rbd-nbd`` expanded ``$pid`` with the actual daemon pid.
+
+* The allowable options for some ``radosgw-admin`` commands have been changed.
+
+  * ``mdlog-list``, ``datalog-list``, ``sync-error-list`` no longer accepts
+    start and end dates, but does accept a single optional start marker.
+  * ``mdlog-trim``, ``datalog-trim``, ``sync-error-trim`` only accept a
+    single marker giving the end of the trimmed range.
+  * Similarly the date ranges and marker ranges have been removed on
+    the RESTful DATALog and MDLog list and trim operations.
+
+* ceph-volume: The ``lvm batch`` subcommand received a major rewrite. This
+  closed a number of bugs and improves usability in terms of size specification
+  and calculation, as well as idempotency behaviour and disk replacement
+  process.
+  Please refer to https://docs.ceph.com/en/latest/ceph-volume/lvm/batch/ for
+  more detailed information.
+
+* Configuration variables for permitted scrub times have changed.  The legal
+  values for ``osd_scrub_begin_hour`` and ``osd_scrub_end_hour`` are 0 - 23.
+  The use of 24 is now illegal.  Specifying ``0`` for both values causes every
+  hour to be allowed.  The legal values for ``osd_scrub_begin_week_day`` and
+  ``osd_scrub_end_week_day`` are 0 - 6.  The use of 7 is now illegal.
+  Specifying ``0`` for both values causes every day of the week to be allowed.
+
+* volume/nfs: Recently "ganesha-" prefix from cluster id and nfs-ganesha common
+  config object was removed, to ensure consistent namespace across different
+  orchestrator backends. Please delete any existing nfs-ganesha clusters prior
+  to upgrading and redeploy new clusters after upgrading to Pacific.
+
+* A new health check, DAEMON_OLD_VERSION, will warn if different versions of Ceph are running
+  on daemons. It will generate a health error if multiple versions are detected.
+  This condition must exist for over mon_warn_older_version_delay (set to 1 week by default) in order for the
+  health condition to be triggered.  This allows most upgrades to proceed
+  without falsely seeing the warning.  If upgrade is paused for an extended
+  time period, health mute can be used like this
+  "ceph health mute DAEMON_OLD_VERSION --sticky".  In this case after
+  upgrade has finished use "ceph health unmute DAEMON_OLD_VERSION".
+
+* MGR: progress module can now be turned on/off, using the commands:
+  ``ceph progress on`` and ``ceph progress off``.
+
+* An AWS-compliant API: "GetTopicAttributes" was added to replace the existing "GetTopic" API. The new API
+  should be used to fetch information about topics used for bucket notifications.
+
+* librbd: The shared, read-only parent cache's config option ``immutable_object_cache_watermark`` now has been updated
+  to property reflect the upper cache utilization before space is reclaimed. The default ``immutable_object_cache_watermark``
+  now is ``0.9``. If the capacity reaches 90% the daemon will delete cold cache.
+
+* OSD: the option ``osd_fast_shutdown_notify_mon`` has been introduced to allow
+  the OSD to notify the monitor it is shutting down even if ``osd_fast_shutdown``
+  is enabled. This helps with the monitor logs on larger clusters, that may get
+  many 'osd.X reported immediately failed by osd.Y' messages, and confuse tools.
+
+* The mclock scheduler has been refined. A set of built-in profiles are now available that
+  provide QoS between the internal and external clients of Ceph. To enable the mclock
+  scheduler, set the config option "osd_op_queue" to "mclock_scheduler". The
+  "high_client_ops" profile is enabled by default, and allocates more OSD bandwidth to
+  external client operations than to internal client operations (such as background recovery
+  and scrubs). Other built-in profiles include "high_recovery_ops" and "balanced". These
+  built-in profiles optimize the QoS provided to clients of mclock scheduler.
+
+* The balancer is now on by default in upmap mode. Since upmap mode requires
+  ``require_min_compat_client`` luminous, new clusters will only support luminous
+  and newer clients by default. Existing clusters can enable upmap support by running
+  ``ceph osd set-require-min-compat-client luminous``. It is still possible to turn
+  the balancer off using the ``ceph balancer off`` command. In earlier versions,
+  the balancer was included in the ``always_on_modules`` list, but needed to be
+  turned on explicitly using the ``ceph balancer on`` command.
+
+* Version 2 of the cephx authentication protocol (``CEPHX_V2`` feature bit) is
+  now required by default.  It was introduced in 2018, adding replay attack
+  protection for authorizers and making msgr v1 message signatures stronger
+  (CVE-2018-1128 and CVE-2018-1129).  Support is present in Jewel 10.2.11,
+  Luminous 12.2.6, Mimic 13.2.1, Nautilus 14.2.0 and later; upstream kernels
+  4.9.150, 4.14.86, 4.19 and later; various distribution kernels, in particular
+  CentOS 7.6 and later.  To enable older clients, set ``cephx_require_version``
+  and ``cephx_service_require_version`` config options to 1.
+
+* `blacklist` has been replaced with `blocklist` throughout.  The following commands have changed:
+
+  - ``ceph osd blacklist ...`` are now ``ceph osd blocklist ...``
+  - ``ceph <tell|daemon> osd.<NNN> dump_blacklist`` is now ``ceph <tell|daemon> osd.<NNN> dump_blocklist``
+
+* The following config options have changed:
+
+  - ``mon osd blacklist default expire`` is now ``mon osd blocklist default expire``
+  - ``mon mds blacklist interval`` is now ``mon mds blocklist interval``
+  - ``mon mgr blacklist interval`` is now ''mon mgr blocklist interval``
+  - ``rbd blacklist on break lock`` is now ``rbd blocklist on break lock``
+  - ``rbd blacklist expire seconds`` is now ``rbd blocklist expire seconds``
+  - ``mds session blacklist on timeout`` is now ``mds session blocklist on timeout``
+  - ``mds session blacklist on evict`` is now ``mds session blocklist on evict``
+
+* The following librados API calls have changed:
+
+  - ``rados_blacklist_add`` is now ``rados_blocklist_add``; the former will issue a deprecation warning and be removed in a future release.
+  - ``rados.blacklist_add`` is now ``rados.blocklist_add`` in the C++ API.
+
+* The JSON output for the following commands now shows ``blocklist`` instead of ``blacklist``:
+
+  - ``ceph osd dump``
+  - ``ceph <tell|daemon> osd.<N> dump_blocklist``
+
+* Monitors now have config option ``mon_allow_pool_size_one``, which is disabled
+  by default. However, if enabled, user now have to pass the
+  ``--yes-i-really-mean-it`` flag to ``osd pool set size 1``, if they are really
+  sure of configuring pool size 1.
+
+* ``ceph pg #.# list_unfound`` output has been enhanced to provide
+  might_have_unfound information which indicates which OSDs may
+  contain the unfound objects.
+
+* OSD: A new configuration option ``osd_compact_on_start`` has been added which triggers
+  an OSD compaction on start. Setting this option to ``true`` and restarting an OSD
+  will result in an offline compaction of the OSD prior to booting.
+
+* OSD: the option named ``bdev_nvme_retry_count`` has been removed. Because
+  in SPDK v20.07, there is no easy access to bdev_nvme options, and this
+  option is hardly used, so it was removed.
+
+* Alpine build related script, documentation and test have been removed since
+  the most updated APKBUILD script of Ceph is already included by Alpine Linux's
+  aports repository.
+
diff --git a/doc/releases/quincy.rst b/doc/releases/quincy.rst
new file mode 100644
index 000000000..0322a2cf7
--- /dev/null
+++ b/doc/releases/quincy.rst
@@ -0,0 +1,1013 @@
+======
+Quincy
+======
+
+Quincy is the 17th stable release of Ceph.  It is named after Squidward
+Quincy Tentacles from Spongebob Squarepants.
+
+v17.2.5 Quincy
+==============
+
+This is a hotfix release that addresses missing commits in the 17.2.4 release.
+We recommend that all users update to this release.
+
+Related tracker: https://tracker.ceph.com/issues/57858
+
+Notable Changes
+---------------
+
+* A ceph-volume regression introduced in bea9f4b that makes the
+  activate process take a very long time to complete has been
+  fixed.
+  
+  Related tracker: https://tracker.ceph.com/issues/57627
+
+* An exception that occurs with some NFS commands
+  in Rook clusters has been fixed.
+  
+  Related tracker: https://tracker.ceph.com/issues/55605
+
+* A crash in the Telemetry module that may affect some users opted
+  into the perf channel has been fixed.
+
+  Related tracker: https://tracker.ceph.com/issues/57700
+
+Changelog
+---------
+
+* ceph-volume: fix regression in activate (`pr#48201 <https://github.com/ceph/ceph/pull/48201>`_, Guillaume Abrioux)
+* mgr/rook: fix error when trying to get the list of nfs services (`pr#48199 <https://github.com/ceph/ceph/pull/48199>`_, Juan Miguel Olmo)
+* mgr/telemetry: handle daemons with complex ids (`pr#48283 <https://github.com/ceph/ceph/pull/48283>`_, Laura Flores)
+* Revert PR 47901 (`pr#48104 <https://github.com/ceph/ceph/pull/48104>`_, Laura Flores)
+
+v17.2.4 Quincy
+==============
+
+This is the fourth backport release in the Quincy series. We recommend
+that all users update to this release.
+
+Notable Changes
+---------------
+
+* Cephfs: The ``AT_NO_ATTR_SYNC`` macro is deprecated, please use the standard
+  ``AT_STATX_DONT_SYNC`` macro. The ``AT_NO_ATTR_SYNC`` macro will be removed in
+  the future.
+
+* OSD: The issue of high CPU utilization during recovery/backfill operations
+  has been fixed. For more details see: https://tracker.ceph.com/issues/56530.
+
+* Trimming of PGLog dups is now controlled by size instead of the version.
+  This fixes the PGLog inflation issue that was happening when online
+  (in OSD) trimming jammed after a PG split operation. Also, a new offline
+  mechanism has been added: ``ceph-objectstore-tool`` now has a ``trim-pg-log-dups`` op
+  that targets situations where an OSD is unable to boot due to those inflated dups.
+  If that is the case, in OSD logs the "You can be hit by THE DUPS BUG" warning
+  will be visible.
+  Relevant tracker: https://tracker.ceph.com/issues/53729
+
+* OSD: Octopus modified the SnapMapper key format from
+  ``<LEGACY_MAPPING_PREFIX><snapid>_<shardid>_<hobject_t::to_str()>``
+  to
+  ``<MAPPING_PREFIX><pool>_<snapid>_<shardid>_<hobject_t::to_str()>``.
+  When this change was introduced, `94ebe0e <https://github.com/ceph/ceph/commit/94ebe0eab968068c29fdffa1bfe68c72122db633>`_
+  also introduced a conversion with a crucial bug which essentially
+  destroyed legacy keys by mapping them to
+  ``<MAPPING_PREFIX><poolid>_<snapid>_``
+  without the object-unique suffix. The conversion is fixed in this release.
+  Relevant tracker: https://tracker.ceph.com/issues/56147
+
+Changelog
+---------
+
+* .readthedocs.yml: Always build latest doc/releases pages (`pr#47442 <https://github.com/ceph/ceph/pull/47442>`_, David Galloway)
+* Add mapping for ernno:13 and adding path in error msg in opendir()/cephfs.pyx (`pr#46647 <https://github.com/ceph/ceph/pull/46647>`_, Sarthak0702)
+* admin: Fix check if PR or release branch docs build (`pr#47739 <https://github.com/ceph/ceph/pull/47739>`_, David Galloway)
+* bdev: fix FTBFS on FreeBSD, keep the huge paged read buffers (`pr#44641 <https://github.com/ceph/ceph/pull/44641>`_, Radoslaw Zarzynski)
+* build: Silence deprecation warnings from OpenSSL 3 (`pr#47585 <https://github.com/ceph/ceph/pull/47585>`_, Kefu Chai, Adam C. Emerson)
+* Catch exception if thrown by __generate_command_map() (`pr#45892 <https://github.com/ceph/ceph/pull/45892>`_, Nikhil Kshirsagar)
+* ceph-fuse: add dedicated snap stag map for each directory (`pr#46948 <https://github.com/ceph/ceph/pull/46948>`_, Xiubo Li)
+* ceph-mixin: backport of recent cleanups (`pr#46548 <https://github.com/ceph/ceph/pull/46548>`_, Arthur Outhenin-Chalandre)
+* ceph-volume: avoid unnecessary subprocess calls (`pr#46968 <https://github.com/ceph/ceph/pull/46968>`_, Guillaume Abrioux)
+* ceph-volume: decrease number of `pvs` calls in `lvm list` (`pr#46966 <https://github.com/ceph/ceph/pull/46966>`_, Guillaume Abrioux)
+* ceph-volume: do not call get_device_vgs() per devices (`pr#47348 <https://github.com/ceph/ceph/pull/47348>`_, Guillaume Abrioux)
+* ceph-volume: do not log sensitive details (`pr#46728 <https://github.com/ceph/ceph/pull/46728>`_, Guillaume Abrioux)
+* ceph-volume: fix `simple scan` (`pr#47149 <https://github.com/ceph/ceph/pull/47149>`_, Guillaume Abrioux)
+* ceph-volume: fix fast device alloc size on mulitple device (`pr#47293 <https://github.com/ceph/ceph/pull/47293>`_, Arthur Outhenin-Chalandre)
+* ceph-volume: fix regression in activate (`pr#48201 <https://github.com/ceph/ceph/pull/48201>`_, Guillaume Abrioux)
+* ceph-volume: make is_valid() optional (`pr#46730 <https://github.com/ceph/ceph/pull/46730>`_, Guillaume Abrioux)
+* ceph-volume: only warn when config file isn't found (`pr#46070 <https://github.com/ceph/ceph/pull/46070>`_, Guillaume Abrioux)
+* ceph-volume: Quincy backports (`pr#47406 <https://github.com/ceph/ceph/pull/47406>`_, Guillaume Abrioux, Zack Cerza, Michael Fritch)
+* ceph-volume: system.get_mounts() refactor (`pr#47536 <https://github.com/ceph/ceph/pull/47536>`_, Guillaume Abrioux)
+* ceph-volume/tests: fix test_exception_returns_default (`pr#47435 <https://github.com/ceph/ceph/pull/47435>`_, Guillaume Abrioux)
+* ceph.spec.in backports (`pr#47549 <https://github.com/ceph/ceph/pull/47549>`_, David Galloway, Kefu Chai, Tim Serong, Casey Bodley, Radoslaw Zarzynski, Radosław Zarzyński)
+* ceph.spec.in: disable system_pmdk on s390x (`pr#47251 <https://github.com/ceph/ceph/pull/47251>`_, Ken Dreyer)
+* ceph.spec.in: openSUSE: require gcc11-c++, disable parquet (`pr#46155 <https://github.com/ceph/ceph/pull/46155>`_, Tim Serong)
+* ceph.spec: fixing cephadm build deps (`pr#47069 <https://github.com/ceph/ceph/pull/47069>`_, Redouane Kachach)
+* cephadm/ceph-volume: fix rm-cluster --zap (`pr#47626 <https://github.com/ceph/ceph/pull/47626>`_, Guillaume Abrioux)
+* cephadm/mgr: adding logic to handle --no-overwrite for tuned profiles (`pr#47944 <https://github.com/ceph/ceph/pull/47944>`_, Redouane Kachach)
+* cephadm: add "su root root" to cephadm.log logrotate config (`pr#47314 <https://github.com/ceph/ceph/pull/47314>`_, Adam King)
+* cephadm: add 'is_paused' field in orch status output (`pr#46569 <https://github.com/ceph/ceph/pull/46569>`_, Guillaume Abrioux)
+* Cephadm: Allow multiple virtual IP addresses for keepalived and haproxy (`pr#47610 <https://github.com/ceph/ceph/pull/47610>`_, Luis Domingues)
+* cephadm: change default keepalived/haproxy container images (`pr#46714 <https://github.com/ceph/ceph/pull/46714>`_, Guillaume Abrioux)
+* cephadm: fix incorrect warning (`pr#47608 <https://github.com/ceph/ceph/pull/47608>`_, Guillaume Abrioux)
+* cephadm: fix osd adoption with custom cluster name (`pr#46551 <https://github.com/ceph/ceph/pull/46551>`_, Adam King)
+* cephadm: Fix repo_gpgkey should return 2 vars (`pr#47374 <https://github.com/ceph/ceph/pull/47374>`_, Laurent Barbe)
+* cephadm: improve message when removing osd (`pr#47071 <https://github.com/ceph/ceph/pull/47071>`_, Guillaume Abrioux)
+* cephadm: preserve cephadm user during RPM upgrade (`pr#46790 <https://github.com/ceph/ceph/pull/46790>`_, Scott Shambarger)
+* cephadm: reduce spam to cephadm.log (`pr#47313 <https://github.com/ceph/ceph/pull/47313>`_, Adam King)
+* cephadm: Remove duplicated process args in promtail and loki (`pr#47654 <https://github.com/ceph/ceph/pull/47654>`_, jinhong.kim)
+* cephadm: return nonzero exit code when applying spec fails in bootstrap (`pr#47952 <https://github.com/ceph/ceph/pull/47952>`_, Adam King)
+* cephadm: support for Oracle Linux 8 (`pr#47656 <https://github.com/ceph/ceph/pull/47656>`_, Adam King)
+* cephfs-shell: move source to separate subdirectory (`pr#47400 <https://github.com/ceph/ceph/pull/47400>`_, Tim Serong)
+* cephfs-top: display average read/write/metadata latency (`issue#48619 <http://tracker.ceph.com/issues/48619>`_, `pr#47977 <https://github.com/ceph/ceph/pull/47977>`_, Venky Shankar)
+* cephfs-top: fix the rsp/wsp display (`pr#47648 <https://github.com/ceph/ceph/pull/47648>`_, Jos Collin)
+* client/fuse: Fix directory DACs overriding for root (`pr#46595 <https://github.com/ceph/ceph/pull/46595>`_, Kotresh HR)
+* client: allow overwrites to file with size greater than the max_file_size (`pr#47971 <https://github.com/ceph/ceph/pull/47971>`_, Tamar Shacked)
+* client: always return ESTALE directly in handle_reply (`pr#46558 <https://github.com/ceph/ceph/pull/46558>`_, Xiubo Li)
+* client: choose auth MDS for getxattr with the Xs caps (`pr#46800 <https://github.com/ceph/ceph/pull/46800>`_, Xiubo Li)
+* client: do not release the global snaprealm until unmounting (`pr#46495 <https://github.com/ceph/ceph/pull/46495>`_, Xiubo Li)
+* client: Inode::hold_caps_until is time from monotonic clock now (`pr#46563 <https://github.com/ceph/ceph/pull/46563>`_, Laura Flores, Neeraj Pratap Singh)
+* client: switch AT_NO_ATTR_SYNC to AT_STATX_DONT_SYNC (`pr#46680 <https://github.com/ceph/ceph/pull/46680>`_, Xiubo Li)
+* cmake: disable LTO when building pmdk (`pr#47619 <https://github.com/ceph/ceph/pull/47619>`_, Kefu Chai)
+* cmake: pass -Wno-error when building PMDK (`pr#46623 <https://github.com/ceph/ceph/pull/46623>`_, Ilya Dryomov)
+* cmake: remove spaces in macro used for compiling cython code (`pr#47483 <https://github.com/ceph/ceph/pull/47483>`_, Kefu Chai)
+* cmake: set $PATH for tests using jsonnet tools (`pr#47625 <https://github.com/ceph/ceph/pull/47625>`_, Kefu Chai)
+* common/bl: fix FTBFS on C++11 due to C++17's if-with-initializer (`pr#46005 <https://github.com/ceph/ceph/pull/46005>`_, Radosław Zarzyński)
+* common/win32,dokan: include bcrypt.h for NTSTATUS (`pr#48016 <https://github.com/ceph/ceph/pull/48016>`_, Lucian Petrut, Kefu Chai)
+* common: fix FTBFS due to dout & need_dynamic on GCC-12 (`pr#46214 <https://github.com/ceph/ceph/pull/46214>`_, Radoslaw Zarzynski)
+* common: use boost::shared_mutex on Windows (`pr#47493 <https://github.com/ceph/ceph/pull/47493>`_, Lucian Petrut)
+* crash: pthread_mutex_lock() (`pr#47683 <https://github.com/ceph/ceph/pull/47683>`_, Patrick Donnelly)
+* crimson: fixes for compiling with fmtlib v8 (`pr#47603 <https://github.com/ceph/ceph/pull/47603>`_, Adam C. Emerson, Kefu Chai)
+* doc, crimson: document installing crimson with cephadm (`pr#47283 <https://github.com/ceph/ceph/pull/47283>`_, Radoslaw Zarzynski)
+* doc/cephadm/services: fix example for specifying rgw placement (`pr#47947 <https://github.com/ceph/ceph/pull/47947>`_, Redouane Kachach)
+* doc/cephadm/services: the config section of service specs (`pr#47068 <https://github.com/ceph/ceph/pull/47068>`_, Redouane Kachach)
+* doc/cephadm: add note about OSDs being recreated to OSD removal section (`pr#47102 <https://github.com/ceph/ceph/pull/47102>`_, Adam King)
+* doc/cephadm: Add post-upgrade section (`pr#47077 <https://github.com/ceph/ceph/pull/47077>`_, Redouane Kachach)
+* doc/cephadm: document the new per-fsid cephadm conf location (`pr#47076 <https://github.com/ceph/ceph/pull/47076>`_, Redouane Kachach)
+* doc/cephadm: enhancing daemon operations documentation (`pr#47074 <https://github.com/ceph/ceph/pull/47074>`_, Redouane Kachach)
+* doc/cephadm: fix example for specifying networks for rgw (`pr#47806 <https://github.com/ceph/ceph/pull/47806>`_, Adam King)
+* doc/dev: add context note to dev guide config (`pr#46818 <https://github.com/ceph/ceph/pull/46818>`_, Zac Dover)
+* doc/dev: add Dependabot section to essentials.rst (`pr#47042 <https://github.com/ceph/ceph/pull/47042>`_, Zac Dover)
+* doc/dev: add IRC registration instructions (`pr#46940 <https://github.com/ceph/ceph/pull/46940>`_, Zac Dover)
+* doc/dev: edit delayed-delete.rst (`pr#47051 <https://github.com/ceph/ceph/pull/47051>`_, Zac Dover)
+* doc/dev: Elaborate on boost .deb creation (`pr#47415 <https://github.com/ceph/ceph/pull/47415>`_, David Galloway)
+* doc/dev: s/github/GitHub/ in essentials.rst (`pr#47048 <https://github.com/ceph/ceph/pull/47048>`_, Zac Dover)
+* doc/dev: s/master/main/ essentials.rst dev guide (`pr#46661 <https://github.com/ceph/ceph/pull/46661>`_, Zac Dover)
+* doc/dev: s/master/main/ in basic workflow (`pr#46703 <https://github.com/ceph/ceph/pull/46703>`_, Zac Dover)
+* doc/dev: s/master/main/ in title (`pr#46721 <https://github.com/ceph/ceph/pull/46721>`_, Zac Dover)
+* doc/dev: s/the the/the/ in basic-workflow.rst (`pr#46935 <https://github.com/ceph/ceph/pull/46935>`_, Zac Dover)
+* doc/dev_guide: s/master/main in merging.rst (`pr#46709 <https://github.com/ceph/ceph/pull/46709>`_, Zac Dover)
+* doc/index.rst: add link to Dev Guide basic workfl (`pr#46904 <https://github.com/ceph/ceph/pull/46904>`_, Zac Dover)
+* doc/man/rbd: Mention changed `bluestore_min_alloc_size` (`pr#47579 <https://github.com/ceph/ceph/pull/47579>`_, Niklas Hambüchen)
+* doc/mgr: add prompt directives to dashboard.rst (`pr#47822 <https://github.com/ceph/ceph/pull/47822>`_, Zac Dover)
+* doc/mgr: edit orchestrator.rst (`pr#47780 <https://github.com/ceph/ceph/pull/47780>`_, Zac Dover)
+* doc/mgr: update prompts in dboard.rst includes (`pr#47869 <https://github.com/ceph/ceph/pull/47869>`_, Zac Dover)
+* doc/rados/operations: add prompts to operating.rst (`pr#47586 <https://github.com/ceph/ceph/pull/47586>`_, Zac Dover)
+* doc/radosgw: Uppercase s3 (`pr#47359 <https://github.com/ceph/ceph/pull/47359>`_, Anthony D'Atri)
+* doc/start: alphabetize hardware-recs links (`pr#46339 <https://github.com/ceph/ceph/pull/46339>`_, Zac Dover)
+* doc/start: make OSD and MDS structures parallel (`pr#46655 <https://github.com/ceph/ceph/pull/46655>`_, Zac Dover)
+* doc/start: Polish network section of hardware-recommendations.rst (`pr#46665 <https://github.com/ceph/ceph/pull/46665>`_, Anthony D'Atri)
+* doc/start: rewrite CRUSH para (`pr#46658 <https://github.com/ceph/ceph/pull/46658>`_, Zac Dover)
+* doc/start: rewrite hardware-recs networks section (`pr#46652 <https://github.com/ceph/ceph/pull/46652>`_, Zac Dover)
+* doc/start: update documenting-ceph branch names (`pr#47955 <https://github.com/ceph/ceph/pull/47955>`_, Zac Dover)
+* doc/start: update hardware recs (`pr#47123 <https://github.com/ceph/ceph/pull/47123>`_, Zac Dover)
+* doc: update docs for centralized logging (`pr#46946 <https://github.com/ceph/ceph/pull/46946>`_, Aashish Sharma)
+* doc: Update release process doc to accurately reflect current process (`pr#47837 <https://github.com/ceph/ceph/pull/47837>`_, David Galloway)
+* docs: fix doc link pointing to master in dashboard.rst (`pr#47789 <https://github.com/ceph/ceph/pull/47789>`_, Nizamudeen A)
+* exporter: per node metric exporter (`pr#47629 <https://github.com/ceph/ceph/pull/47629>`_, Pere Diaz Bou, Avan Thakkar)
+* include/buffer: include <memory> (`pr#47694 <https://github.com/ceph/ceph/pull/47694>`_, Kefu Chai)
+* install-deps.sh: do not install libpmem from chacra (`pr#46900 <https://github.com/ceph/ceph/pull/46900>`_, Kefu Chai)
+* install-deps: script exit on /ValueError: in centos_stream8 (`pr#47892 <https://github.com/ceph/ceph/pull/47892>`_, Nizamudeen A)
+* libcephfs: define AT_NO_ATTR_SYNC back for backward compatibility (`pr#47861 <https://github.com/ceph/ceph/pull/47861>`_, Xiubo Li)
+* libcephsqlite: ceph-mgr crashes when compiled with gcc12 (`pr#47270 <https://github.com/ceph/ceph/pull/47270>`_, Ganesh Maharaj Mahalingam)
+* librados: rados_ioctx_destroy check for initialized ioctx (`pr#47452 <https://github.com/ceph/ceph/pull/47452>`_, Nitzan Mordechai)
+* librbd/cache/pwl: narrow the scope of m_lock in write_image_cache_state() (`pr#47940 <https://github.com/ceph/ceph/pull/47940>`_, Ilya Dryomov, Yin Congmin)
+* librbd: bail from schedule_request_lock() if already lock owner (`pr#47162 <https://github.com/ceph/ceph/pull/47162>`_, Christopher Hoffman)
+* librbd: retry ENOENT in V2_REFRESH_PARENT as well (`pr#47996 <https://github.com/ceph/ceph/pull/47996>`_, Ilya Dryomov)
+* librbd: tweak misleading "image is still primary" error message (`pr#47248 <https://github.com/ceph/ceph/pull/47248>`_, Ilya Dryomov)
+* librbd: unlink newest mirror snapshot when at capacity, bump capacity (`pr#46594 <https://github.com/ceph/ceph/pull/46594>`_, Ilya Dryomov)
+* librbd: update progress for non-existent objects on deep-copy (`pr#46910 <https://github.com/ceph/ceph/pull/46910>`_, Ilya Dryomov)
+* librbd: use actual monitor addresses when creating a peer bootstrap token (`pr#47912 <https://github.com/ceph/ceph/pull/47912>`_, Ilya Dryomov)
+* mds: clear MDCache::rejoin\_\*_q queues before recovering file inodes (`pr#46681 <https://github.com/ceph/ceph/pull/46681>`_, Xiubo Li)
+* mds: do not assert early on when issuing client leases (`issue#54701 <http://tracker.ceph.com/issues/54701>`_, `pr#46566 <https://github.com/ceph/ceph/pull/46566>`_, Venky Shankar)
+* mds: Don't blocklist clients in any replay state (`pr#47110 <https://github.com/ceph/ceph/pull/47110>`_, Kotresh HR)
+* mds: fix crash when exporting unlinked dir (`pr#47181 <https://github.com/ceph/ceph/pull/47181>`_, 胡玮文)
+* mds: flush mdlog if locked and still has wanted caps not satisfied (`pr#46494 <https://github.com/ceph/ceph/pull/46494>`_, Xiubo Li)
+* mds: notify the xattr_version to replica MDSes (`pr#47057 <https://github.com/ceph/ceph/pull/47057>`_, Xiubo Li)
+* mds: skip fetching the dirfrags if not a directory (`pr#47432 <https://github.com/ceph/ceph/pull/47432>`_, Xiubo Li)
+* mds: standby-replay daemon always removed in MDSMonitor::prepare_beacon (`pr#47281 <https://github.com/ceph/ceph/pull/47281>`_, Patrick Donnelly)
+* mds: switch to use projected inode instead (`pr#47058 <https://github.com/ceph/ceph/pull/47058>`_, Xiubo Li)
+* mgr, mon: Keep upto date metadata with mgr for MONs (`pr#46559 <https://github.com/ceph/ceph/pull/46559>`_, Laura Flores, Prashant D)
+* mgr/cephadm: Add disk rescan feature to the orchestrator (`pr#47311 <https://github.com/ceph/ceph/pull/47311>`_, Adam King, Paul Cuzner)
+* mgr/cephadm: add parsing for config on osd specs (`pr#47268 <https://github.com/ceph/ceph/pull/47268>`_, Luis Domingues)
+* mgr/cephadm: Adding logic to store grafana cert/key per node (`pr#47950 <https://github.com/ceph/ceph/pull/47950>`_, Redouane Kachach)
+* mgr/cephadm: allow binding to loopback for rgw daemons (`pr#47951 <https://github.com/ceph/ceph/pull/47951>`_, Redouane Kachach)
+* mgr/cephadm: capture exception when not able to list upgrade tags (`pr#46783 <https://github.com/ceph/ceph/pull/46783>`_, Redouane Kachach)
+* mgr/cephadm: check for events key before accessing it (`pr#47317 <https://github.com/ceph/ceph/pull/47317>`_, Redouane Kachach)
+* mgr/cephadm: check if a service exists before trying to restart it (`pr#46789 <https://github.com/ceph/ceph/pull/46789>`_, Redouane Kachach)
+* mgr/cephadm: clear error message when resuming upgrade (`pr#47373 <https://github.com/ceph/ceph/pull/47373>`_, Adam King)
+* mgr/cephadm: don't try to write client/os tuning profiles to known offline hosts (`pr#47953 <https://github.com/ceph/ceph/pull/47953>`_, Adam King)
+* mgr/cephadm: fix handling of draining hosts with explicit placement specs (`pr#47657 <https://github.com/ceph/ceph/pull/47657>`_, Adam King)
+* mgr/cephadm: Fix how we check if a host belongs to public network (`pr#47946 <https://github.com/ceph/ceph/pull/47946>`_, Redouane Kachach)
+* mgr/cephadm: fix the loki address in grafana, promtail configuration files (`pr#47171 <https://github.com/ceph/ceph/pull/47171>`_, jinhong.kim)
+* mgr/cephadm: fixing scheduler consistent hashing (`pr#47073 <https://github.com/ceph/ceph/pull/47073>`_, Redouane Kachach)
+* mgr/cephadm: limiting ingress/keepalived pass to 8 chars (`pr#47070 <https://github.com/ceph/ceph/pull/47070>`_, Redouane Kachach)
+* mgr/cephadm: recreate osd config when redeploy/reconfiguring (`pr#47659 <https://github.com/ceph/ceph/pull/47659>`_, Adam King)
+* mgr/cephadm: set dashboard grafana-api-password when user provides one (`pr#47658 <https://github.com/ceph/ceph/pull/47658>`_, Adam King)
+* mgr/cephadm: store device info separately from rest of host cache (`pr#46791 <https://github.com/ceph/ceph/pull/46791>`_, Adam King)
+* mgr/cephadm: support for miscellaneous config files for daemons (`pr#47312 <https://github.com/ceph/ceph/pull/47312>`_, Adam King)
+* mgr/cephadm: support for os tuning profiles (`pr#47316 <https://github.com/ceph/ceph/pull/47316>`_, Adam King)
+* mgr/cephadm: try to get FQDN for active instance (`pr#46793 <https://github.com/ceph/ceph/pull/46793>`_, Tatjana Dehler)
+* mgr/cephadm: use host shortname for osd memory autotuning (`pr#47075 <https://github.com/ceph/ceph/pull/47075>`_, Adam King)
+* mgr/dashboard: Add daemon logs tab to Logs component (`pr#46807 <https://github.com/ceph/ceph/pull/46807>`_, Aashish Sharma)
+* mgr/dashboard: add flag to automatically deploy loki/promtail service at bootstrap (`pr#47623 <https://github.com/ceph/ceph/pull/47623>`_, Aashish Sharma)
+* mgr/dashboard: add required validation for frontend and monitor port (`pr#47356 <https://github.com/ceph/ceph/pull/47356>`_, Avan Thakkar)
+* mgr/dashboard: added pattern validaton for form input (`pr#47329 <https://github.com/ceph/ceph/pull/47329>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: BDD approach for the dashboard cephadm e2e (`pr#46528 <https://github.com/ceph/ceph/pull/46528>`_, Nizamudeen A)
+* mgr/dashboard: bump moment from 2.29.1 to 2.29.3 in /src/pybind/mgr/dashboard/frontend (`pr#46718 <https://github.com/ceph/ceph/pull/46718>`_, dependabot[bot])
+* mgr/dashboard: bump up teuthology (`pr#47498 <https://github.com/ceph/ceph/pull/47498>`_, Kefu Chai)
+* mgr/dashboard: dashboard help command showing wrong syntax for login-banner (`pr#46809 <https://github.com/ceph/ceph/pull/46809>`_, Sarthak0702)
+* mgr/dashboard: display helpfull message when the iframe-embedded Grafana dashboard failed to load (`pr#47007 <https://github.com/ceph/ceph/pull/47007>`_, Ngwa Sedrick Meh)
+* mgr/dashboard: do not recommend throughput for ssd's only cluster (`pr#47156 <https://github.com/ceph/ceph/pull/47156>`_, Nizamudeen A)
+* mgr/dashboard: don't log tracebacks on 404s (`pr#47094 <https://github.com/ceph/ceph/pull/47094>`_, Ernesto Puerta)
+* mgr/dashboard: enable addition of custom Prometheus alerts (`pr#47942 <https://github.com/ceph/ceph/pull/47942>`_, Patrick Seidensal)
+* mgr/dashboard: ensure limit 0 returns 0 images (`pr#47887 <https://github.com/ceph/ceph/pull/47887>`_, Pere Diaz Bou)
+* mgr/dashboard: Feature 54330 osd creation workflow (`pr#46686 <https://github.com/ceph/ceph/pull/46686>`_, Pere Diaz Bou, Nizamudeen A, Sarthak0702)
+* mgr/dashboard: fix _rbd_image_refs caching (`pr#47635 <https://github.com/ceph/ceph/pull/47635>`_, Pere Diaz Bou)
+* mgr/dashboard: fix nfs exports form issues with squash field (`pr#47961 <https://github.com/ceph/ceph/pull/47961>`_, Nizamudeen A)
+* mgr/dashboard: fix unmanaged service creation (`pr#48025 <https://github.com/ceph/ceph/pull/48025>`_, Nizamudeen A)
+* mgr/dashboard: grafana frontend e2e testing and update cypress (`pr#47703 <https://github.com/ceph/ceph/pull/47703>`_, Nizamudeen A)
+* mgr/dashboard: Hide maintenance option on expand cluster (`pr#47724 <https://github.com/ceph/ceph/pull/47724>`_, Nizamudeen A)
+* mgr/dashboard: host list tables doesn't show all services deployed (`pr#47453 <https://github.com/ceph/ceph/pull/47453>`_, Avan Thakkar)
+* mgr/dashboard: Improve monitoring tabs content (`pr#46990 <https://github.com/ceph/ceph/pull/46990>`_, Aashish Sharma)
+* mgr/dashboard: ingress backend service should list all supported services (`pr#47085 <https://github.com/ceph/ceph/pull/47085>`_, Avan Thakkar)
+* mgr/dashboard: iops optimized option enabled (`pr#46819 <https://github.com/ceph/ceph/pull/46819>`_, Pere Diaz Bou)
+* mgr/dashboard: iterate through copy of items (`pr#46871 <https://github.com/ceph/ceph/pull/46871>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: prevent alert redirect (`pr#47146 <https://github.com/ceph/ceph/pull/47146>`_, Tatjana Dehler)
+* mgr/dashboard: rbd image pagination (`pr#47104 <https://github.com/ceph/ceph/pull/47104>`_, Pere Diaz Bou, Nizamudeen A)
+* mgr/dashboard: rbd striping setting pre-population and pop-over (`pr#47409 <https://github.com/ceph/ceph/pull/47409>`_, Vrushal Chaudhari)
+* mgr/dashboard: rbd-mirror batch backport (`pr#46532 <https://github.com/ceph/ceph/pull/46532>`_, Pedro Gonzalez Gomez, Pere Diaz Bou, Nizamudeen A, Melissa Li, Sarthak0702, Avan Thakkar, Aashish Sharma)
+* mgr/dashboard: remove token logging (`pr#47430 <https://github.com/ceph/ceph/pull/47430>`_, Pere Diaz Bou)
+* mgr/dashboard: Show error on creating service with duplicate service id (`pr#47403 <https://github.com/ceph/ceph/pull/47403>`_, Aashish Sharma)
+* mgr/dashboard: stop polling when page is not visible (`pr#46672 <https://github.com/ceph/ceph/pull/46672>`_, Sarthak0702)
+* mgr/dashboard:Get different storage class metrics in Prometheus dashboard (`pr#47201 <https://github.com/ceph/ceph/pull/47201>`_, Aashish Sharma)
+* mgr/nfs: validate virtual_ip parameter (`pr#46794 <https://github.com/ceph/ceph/pull/46794>`_, Redouane Kachach)
+* mgr/orchestrator/tests: don't match exact whitespace in table output (`pr#47858 <https://github.com/ceph/ceph/pull/47858>`_, Adam King)
+* mgr/rook: fix error when trying to get the list of nfs services `pr#48199 <https://github.com/ceph/ceph/pull/48199>`_, Juan Miguel Olmo)
+* mgr/snap_schedule: replace .snap with the client configured snap dir name (`pr#47734 <https://github.com/ceph/ceph/pull/47734>`_, Milind Changire, Venky Shankar, Neeraj Pratap Singh)
+* mgr/snap_schedule: Use rados.Ioctx.remove_object() instead of remove() (`pr#48013 <https://github.com/ceph/ceph/pull/48013>`_, Andreas Teuchert)
+* mgr/telemetry: add `perf_memory_metrics` collection to telemetry (`pr#47826 <https://github.com/ceph/ceph/pull/47826>`_, Laura Flores)
+* mgr/telemetry: handle daemons with complex ids (`pr#48283 <https://github.com/ceph/ceph/pull/48283>`_, Laura Flores)
+* mgr/telemetry: reset health warning after re-opting-in (`pr#47289 <https://github.com/ceph/ceph/pull/47289>`_, Yaarit Hatuka)
+* mgr/volumes: add interface to check the presence of subvolumegroups/subvolumes (`pr#47474 <https://github.com/ceph/ceph/pull/47474>`_, Neeraj Pratap Singh)
+* mgr/volumes: Add volume info command (`pr#47768 <https://github.com/ceph/ceph/pull/47768>`_, Neeraj Pratap Singh)
+* mgr/volumes: Few mgr volumes backports (`pr#47894 <https://github.com/ceph/ceph/pull/47894>`_, Rishabh Dave, Kotresh HR, Nikhilkumar Shelke)
+* mgr/volumes: filter internal directories in 'subvolumegroup ls' command (`pr#47511 <https://github.com/ceph/ceph/pull/47511>`_, Nikhilkumar Shelke)
+* mgr/volumes: Fix subvolume creation in FIPS enabled system (`pr#47368 <https://github.com/ceph/ceph/pull/47368>`_, Kotresh HR)
+* mgr/volumes: prevent intermittent ParsingError failure in "clone cancel" (`pr#47747 <https://github.com/ceph/ceph/pull/47747>`_, John Mulligan)
+* mgr/volumes: remove incorrect 'size' from output of 'snapshot info' (`pr#46804 <https://github.com/ceph/ceph/pull/46804>`_, Nikhilkumar Shelke)
+* mgr/volumes: subvolume ls command crashes if groupname as '_nogroup' (`pr#46805 <https://github.com/ceph/ceph/pull/46805>`_, Nikhilkumar Shelke)
+* mgr/volumes: subvolumegroup quotas (`pr#46667 <https://github.com/ceph/ceph/pull/46667>`_, Kotresh HR)
+* mgr: Define PY_SSIZE_T_CLEAN ahead of every Python.h (`pr#47616 <https://github.com/ceph/ceph/pull/47616>`_, Pete Zaitcev, Kefu Chai)
+* mgr: relax "pending_service_map.epoch > service_map.epoch" assert (`pr#46738 <https://github.com/ceph/ceph/pull/46738>`_, Mykola Golub)
+* mirror snapshot schedule and trash purge schedule fixes (`pr#46781 <https://github.com/ceph/ceph/pull/46781>`_, Ilya Dryomov)
+* mon/ConfigMonitor: fix config get key with whitespace (`pr#47381 <https://github.com/ceph/ceph/pull/47381>`_, Nitzan Mordechai)
+* mon/Elector: notify_rank_removed erase rank from both live_pinging and dead_pinging sets for highest ranked MON (`pr#47086 <https://github.com/ceph/ceph/pull/47086>`_, Kamoltat)
+* mon/MDSMonitor: fix standby-replay mds being removed from MDSMap unexpectedly (`pr#47902 <https://github.com/ceph/ceph/pull/47902>`_, 胡玮文)
+* mon/OSDMonitor: Ensure kvmon() is writeable before handling "osd new" cmd (`pr#46689 <https://github.com/ceph/ceph/pull/46689>`_, Sridhar Seshasayee)
+* monitoring/ceph-mixin: OSD overview typo fix (`pr#47387 <https://github.com/ceph/ceph/pull/47387>`_, Tatjana Dehler)
+* monitoring: ceph mixin backports (`pr#47867 <https://github.com/ceph/ceph/pull/47867>`_, Aswin Toni, Arthur Outhenin-Chalandre, Anthony D'Atri, Tatjana Dehler)
+* msg: fix deadlock when handling existing but closed v2 connection (`pr#47930 <https://github.com/ceph/ceph/pull/47930>`_, Radosław Zarzyński)
+* msg: Fix Windows IPv6 support (`pr#47302 <https://github.com/ceph/ceph/pull/47302>`_, Lucian Petrut)
+* msg: Log at higher level when Throttle::get_or_fail() fails (`pr#47765 <https://github.com/ceph/ceph/pull/47765>`_, Brad Hubbard)
+* msg: reset ProtocolV2's frame assembler in appropriate thread (`pr#47931 <https://github.com/ceph/ceph/pull/47931>`_, Radoslaw Zarzynski)
+* os/bluestore: fix AU accounting in bluestore_cache_other mempool (`pr#47339 <https://github.com/ceph/ceph/pull/47339>`_, Igor Fedotov)
+* os/bluestore: Fix collision between BlueFS and BlueStore deferred writes (`pr#47297 <https://github.com/ceph/ceph/pull/47297>`_, Adam Kupczyk)
+* osd, mds: fix the "heap" admin cmd printing always to error stream (`pr#47825 <https://github.com/ceph/ceph/pull/47825>`_, Radoslaw Zarzynski)
+* osd, tools, kv: non-aggressive, on-line trimming of accumulated dups (`pr#47688 <https://github.com/ceph/ceph/pull/47688>`_, Radoslaw Zarzynski, Nitzan Mordechai)
+* osd/scrub: do not start scrubbing if the PG is snap-trimming (`pr#46498 <https://github.com/ceph/ceph/pull/46498>`_, Ronen Friedman)
+* osd/scrub: late-arriving reservation grants are not an error (`pr#46872 <https://github.com/ceph/ceph/pull/46872>`_, Ronen Friedman)
+* osd/scrub: Reintroduce scrub starts message (`pr#47621 <https://github.com/ceph/ceph/pull/47621>`_, Prashant D)
+* osd/scrubber/pg_scrubber.cc: fix bug where scrub machine gets stuck (`pr#46844 <https://github.com/ceph/ceph/pull/46844>`_, Cory Snyder)
+* osd/SnapMapper: fix legacy key conversion in snapmapper class (`pr#47133 <https://github.com/ceph/ceph/pull/47133>`_, Manuel Lausch, Matan Breizman)
+* osd: Handle oncommits and wait for future work items from mClock queue (`pr#47490 <https://github.com/ceph/ceph/pull/47490>`_, Sridhar Seshasayee)
+* osd: return ENOENT if pool information is invalid during tier-flush (`pr#47929 <https://github.com/ceph/ceph/pull/47929>`_, Myoungwon Oh)
+* osd: Set initial mClock QoS params at CONF_DEFAULT level (`pr#47020 <https://github.com/ceph/ceph/pull/47020>`_, Sridhar Seshasayee)
+* PendingReleaseNotes: Note the fix for high CPU utilization during recovery (`pr#48004 <https://github.com/ceph/ceph/pull/48004>`_, Sridhar Seshasayee)
+* pybind/mgr/cephadm/serve: don't remove ceph.conf which leads to qa failure (`pr#47072 <https://github.com/ceph/ceph/pull/47072>`_, Dhairya Parmar)
+* pybind/mgr/dashboard: do not use distutils.version.StrictVersion (`pr#47602 <https://github.com/ceph/ceph/pull/47602>`_, Kefu Chai)
+* pybind/mgr/pg_autoscaler: change overlapping roots to warning (`pr#47519 <https://github.com/ceph/ceph/pull/47519>`_, Kamoltat)
+* pybind/mgr: ceph osd status crash with ZeroDivisionError (`pr#46697 <https://github.com/ceph/ceph/pull/46697>`_, Nitzan Mordechai)
+* pybind/mgr: fix flake8 (`pr#47391 <https://github.com/ceph/ceph/pull/47391>`_, Avan Thakkar)
+* python-common: allow crush device class to be set from osd service spec (`pr#46792 <https://github.com/ceph/ceph/pull/46792>`_, Cory Snyder)
+* qa/cephadm: specify using container host distros for workunits (`pr#47910 <https://github.com/ceph/ceph/pull/47910>`_, Adam King)
+* qa/cephfs: fallback to older way of get_op_read_count (`pr#46899 <https://github.com/ceph/ceph/pull/46899>`_, Dhairya Parmar)
+* qa/suites/rbd/pwl-cache: ensure recovery is actually tested (`pr#47129 <https://github.com/ceph/ceph/pull/47129>`_, Ilya Dryomov, Yin Congmin)
+* qa/suites/rbd: disable workunit timeout for dynamic_features_no_cache (`pr#47159 <https://github.com/ceph/ceph/pull/47159>`_, Ilya Dryomov)
+* qa/suites/rbd: place cache file on tmpfs for xfstests (`pr#46598 <https://github.com/ceph/ceph/pull/46598>`_, Ilya Dryomov)
+* qa/tasks/ceph_manager.py: increase test_pool_min_size timeout (`pr#47445 <https://github.com/ceph/ceph/pull/47445>`_, Kamoltat)
+* qa/workunits/cephadm: update test_repos master -> main (`pr#47315 <https://github.com/ceph/ceph/pull/47315>`_, Adam King)
+* qa: wait rank 0 to become up:active state before mounting fuse client (`pr#46801 <https://github.com/ceph/ceph/pull/46801>`_, Xiubo Li)
+* quincy -- sse s3 changes (`pr#46467 <https://github.com/ceph/ceph/pull/46467>`_, Casey Bodley, Marcus Watts, Priya Sehgal)
+* rbd-fuse: librados will filter out -r option from command-line (`pr#46954 <https://github.com/ceph/ceph/pull/46954>`_, wanwencong)
+* rbd-mirror: don't prune non-primary snapshot when restarting delta sync (`pr#46591 <https://github.com/ceph/ceph/pull/46591>`_, Ilya Dryomov)
+* rbd-mirror: generally skip replay/resync if remote image is not primary (`pr#46814 <https://github.com/ceph/ceph/pull/46814>`_, Ilya Dryomov)
+* rbd-mirror: remove bogus completed_non_primary_snapshots_exist check (`pr#47126 <https://github.com/ceph/ceph/pull/47126>`_, Ilya Dryomov)
+* rbd-mirror: resume pending shutdown on error in snapshot replayer (`pr#47914 <https://github.com/ceph/ceph/pull/47914>`_, Ilya Dryomov)
+* rbd: don't default empty pool name unless namespace is specified (`pr#47144 <https://github.com/ceph/ceph/pull/47144>`_, Ilya Dryomov)
+* rbd: find_action() should sort actions first (`pr#47584 <https://github.com/ceph/ceph/pull/47584>`_, Ilya Dryomov)
+* RGW - Swift retarget needs bucket set on object (`pr#46719 <https://github.com/ceph/ceph/pull/46719>`_, Daniel Gryniewicz)
+* rgw/backport/quincy: Fix crashes with Sync policy APIs (`pr#47993 <https://github.com/ceph/ceph/pull/47993>`_, Soumya Koduri)
+* rgw/dbstore: Fix build errors on centos9 (`pr#46915 <https://github.com/ceph/ceph/pull/46915>`_, Soumya Koduri)
+* rgw: Avoid segfault when OPA authz is enabled (`pr#46107 <https://github.com/ceph/ceph/pull/46107>`_, Benoît Knecht)
+* rgw: better tenant id from the uri on anonymous access (`pr#47342 <https://github.com/ceph/ceph/pull/47342>`_, Rafał Wądołowski, Marcus Watts)
+* rgw: check object storage_class when check_disk_state (`pr#46580 <https://github.com/ceph/ceph/pull/46580>`_, Huber-ming)
+* rgw: data sync uses yield_spawn_window() (`pr#45714 <https://github.com/ceph/ceph/pull/45714>`_, Casey Bodley)
+* rgw: Fix data race in ChangeStatus (`pr#47195 <https://github.com/ceph/ceph/pull/47195>`_, Adam C. Emerson)
+* rgw: Guard against malformed bucket URLs (`pr#47191 <https://github.com/ceph/ceph/pull/47191>`_, Adam C. Emerson)
+* rgw: log access key id in ops logs (`pr#46624 <https://github.com/ceph/ceph/pull/46624>`_, Cory Snyder)
+* rgw: reopen ops log file on sighup (`pr#46625 <https://github.com/ceph/ceph/pull/46625>`_, Cory Snyder)
+* rgw_rest_user_policy: Fix GetUserPolicy & ListUserPolicies responses (`pr#47235 <https://github.com/ceph/ceph/pull/47235>`_, Sumedh A. Kulkarni)
+* rgwlc: fix segfault resharding during lc (`pr#46742 <https://github.com/ceph/ceph/pull/46742>`_, Mark Kogan)
+* script/build-integration-branch: add quincy to the list of releases (`pr#46361 <https://github.com/ceph/ceph/pull/46361>`_, Yuri Weinstein)
+* SimpleRADOSStriper: Avoid moving bufferlists by using deque in read() (`pr#47909 <https://github.com/ceph/ceph/pull/47909>`_, Matan Breizman)
+* src/mgr/DaemonServer.cc: fix typo in output gap >= max_pg_num_change (`pr#47210 <https://github.com/ceph/ceph/pull/47210>`_, Kamoltat)
+* test/lazy-omap-stats: Various enhancements (`pr#47932 <https://github.com/ceph/ceph/pull/47932>`_, Brad Hubbard)
+* test/{librbd, rgw}: increase delay between and number of bind attempts (`pr#48023 <https://github.com/ceph/ceph/pull/48023>`_, Ilya Dryomov)
+* test/{librbd, rgw}: retry when bind fail with port 0 (`pr#47980 <https://github.com/ceph/ceph/pull/47980>`_, Kefu Chai)
+* tooling: Change mrun to use bash (`pr#46076 <https://github.com/ceph/ceph/pull/46076>`_, Adam C. Emerson)
+* tools: ceph-objectstore-tool is able to trim pg log dups' entries (`pr#46706 <https://github.com/ceph/ceph/pull/46706>`_, Radosław Zarzyński)
+* win32_deps_build.sh: master -> main for wnbd (`pr#46763 <https://github.com/ceph/ceph/pull/46763>`_, Ilya Dryomov)
+
+v17.2.3 Quincy
+==============
+
+This is a hotfix release that addresses a libcephsqlite crash in the mgr.
+
+Notable Changes
+---------------
+* A libcephsqlite bug that caused the mgr to crash repeatedly and die is now
+  fixed. The bug was exposed due to 17.2.2 being built with gcc 8.5.0-14, which contains
+  a new patch to check for invalid regex. 17.2.1 was built using gcc 8.5.0-13, which
+  does not contain the invalid regex patch.
+
+  Relevant tracker: https://tracker.ceph.com/issues/55304
+
+  Relevant BZ: https://bugzilla.redhat.com/show_bug.cgi?id=2110797
+
+Changelog
+---------
+
+* libcephsqlite: ceph-mgr crashes when compiled with gcc12 (`pr#47270 <https://github.com/ceph/ceph/pull/47270>`_, Ganesh Maharaj Mahalingam)
+
+v17.2.2 Quincy
+==============
+
+This is a hotfix release that resolves two security flaws.
+
+Notable Changes
+---------------
+* Users who were running OpenStack Manila to export native CephFS, who
+  upgraded their Ceph cluster from Nautilus (or earlier) to a later
+  major version, were vulnerable to an attack by malicious users. The
+  vulnerability allowed users to obtain access to arbitrary portions of
+  the CephFS filesystem hierarchy, instead of being properly restricted
+  to their own subvolumes. The vulnerability is due to a bug in the
+  "volumes" plugin in Ceph Manager. This plugin is responsible for
+  managing Ceph File System subvolumes which are used by OpenStack
+  Manila services as a way to provide shares to Manila users.
+  
+  With this hotfix, the vulnerability is fixed. Administrators who are
+  concerned they may have been impacted should audit the CephX keys in
+  their cluster for proper path restrictions.
+  
+  Again, this vulnerability only impacts OpenStack Manila clusters which
+  provided native CephFS access to their users.
+
+* A regression made it possible to dereference a null pointer for
+  for s3website requests that don't refer to a bucket resulting in an RGW
+  segfault.
+
+Changelog
+---------
+* mgr/volumes: Fix subvolume discover during upgrade (:ref:`CVE-2022-0670`, Kotresh HR)
+* mgr/volumes: V2 Fix for test_subvolume_retain_snapshot_invalid_recreate (:ref:`CVE-2022-0670`, Kotresh HR)
+* qa: validate subvolume discover on upgrade (Kotresh HR)
+* rgw: s3website check for bucket before retargeting (Seena Fallah)
+
+v17.2.1 Quincy
+==============
+
+This is the first bugfix release of Ceph Quincy.
+
+Notable Changes
+---------------
+* The "BlueStore zero block detection" feature (first introduced to Quincy in
+  https://github.com/ceph/ceph/pull/43337) has been turned off by default with a
+  new global option called `bluestore_zero_block_detection`. This feature,
+  intended for large-scale synthetic testing, does not interact well with some RBD
+  and CephFS features. Any side effects experienced in previous Quincy versions
+  would no longer occur, provided that the config option remains set to false.
+  Relevant tracker: https://tracker.ceph.com/issues/55521
+
+* telemetry: Added new Rook metrics to the 'basic' channel to report Rook's
+  version, Kubernetes version, node metrics, etc.
+  See a sample report with `ceph telemetry preview`.
+  Opt-in with `ceph telemetry on`.
+
+  For more details, see:
+
+  https://docs.ceph.com/en/latest/mgr/telemetry/
+
+* Add offline dup op trimming ability in the ceph-objectstore-tool.
+  Relevant tracker: https://tracker.ceph.com/issues/53729
+
+* Fixes a bug with cluster logs not being populated after log rotation.
+  Relevant tracker: https://tracker.ceph.com/issues/55383
+
+Changelog
+---------
+* .github/CODEOWNERS: tag core devs on core PRs (`pr#46519 <https://github.com/ceph/ceph/pull/46519>`_, Neha Ojha)
+* .github: continue on error and reorder milestone step (`pr#46447 <https://github.com/ceph/ceph/pull/46447>`_, Ernesto Puerta)
+* [quincy] mgr/alerts: Add Message-Id and Date header to sent emails (`pr#46311 <https://github.com/ceph/ceph/pull/46311>`_, Lorenz Bausch)
+* ceph-fuse: ignore fuse mount failure if path is already mounted (`pr#45939 <https://github.com/ceph/ceph/pull/45939>`_, Nikhilkumar Shelke)
+* ceph.in: clarify the usage of `--format` in the ceph command (`pr#46246 <https://github.com/ceph/ceph/pull/46246>`_, Laura Flores)
+* ceph.spec.in: disable annobin plugin if compile with gcc-toolset (`pr#46377 <https://github.com/ceph/ceph/pull/46377>`_, Kefu Chai)
+* ceph.spec.in: remove build directory at end of %install (`pr#45697 <https://github.com/ceph/ceph/pull/45697>`_, Tim Serong)
+* ceph.spec.in: Use libthrift-devel on SUSE distros (`pr#45700 <https://github.com/ceph/ceph/pull/45700>`_, Tim Serong)
+* ceph.spec: make ninja-build package install always (`pr#45875 <https://github.com/ceph/ceph/pull/45875>`_, Deepika Upadhyay)
+* Cephadm Batch Backport April (`pr#46055 <https://github.com/ceph/ceph/pull/46055>`_, Adam King, Lukas Mayer, Ken Dreyer, Redouane Kachach, Aashish Sharma, Avan Thakkar, Moritz Röhrich, Teoman ONAY, Melissa Li, Christoph Glaubitz, Guillaume Abrioux, wangyunqing, Joseph Sawaya, Matan Breizman, Pere Diaz Bou, Michael Fritch, Patrick C. F. Ernzer)
+* Cephadm Batch Backport May (`pr#46360 <https://github.com/ceph/ceph/pull/46360>`_, John Mulligan, Adam King, Prashant D, Redouane Kachach, Aashish Sharma, Ramana Raja, Ville Ojamo)
+* cephadm: infer the default container image during pull (`pr#45568 <https://github.com/ceph/ceph/pull/45568>`_, Michael Fritch)
+* cephadm: preserve `authorized_keys` file during upgrade (`pr#45359 <https://github.com/ceph/ceph/pull/45359>`_, Michael Fritch)
+* cephadm: prometheus: The generatorURL in alerts is only using hostname (`pr#46353 <https://github.com/ceph/ceph/pull/46353>`_, Volker Theile)
+* cephfs-shell: fix put and get cmd (`pr#46300 <https://github.com/ceph/ceph/pull/46300>`_, Dhairya Parmar, dparmar18)
+* cephfs-top: Multiple filesystem support (`pr#46147 <https://github.com/ceph/ceph/pull/46147>`_, Neeraj Pratap Singh)
+* client: add option to disable collecting and sending metrics (`pr#46476 <https://github.com/ceph/ceph/pull/46476>`_, Xiubo Li)
+* cls/rgw: rgw_dir_suggest_changes detects race with completion (`pr#45901 <https://github.com/ceph/ceph/pull/45901>`_, Casey Bodley)
+* cmake/modules: always use the python3 specified in command line (`pr#45966 <https://github.com/ceph/ceph/pull/45966>`_, Kefu Chai)
+* cmake/rgw: add missing dependency on Arrow::Arrow (`pr#46144 <https://github.com/ceph/ceph/pull/46144>`_, Casey Bodley)
+* cmake: resurrect mutex debugging in all Debug builds (`pr#45913 <https://github.com/ceph/ceph/pull/45913>`_, Ilya Dryomov)
+* cmake: WITH_SYSTEM_UTF8PROC defaults to OFF (`pr#45766 <https://github.com/ceph/ceph/pull/45766>`_, Casey Bodley)
+* CODEOWNERS: add RBD team (`pr#46542 <https://github.com/ceph/ceph/pull/46542>`_, Ilya Dryomov)
+* debian: include the new object_format.py file (`pr#46409 <https://github.com/ceph/ceph/pull/46409>`_, John Mulligan)
+* doc/cephfs/add-remove-mds: added cephadm note, refined "Adding an MDS" (`pr#45879 <https://github.com/ceph/ceph/pull/45879>`_, Dhairya Parmar)
+* doc/dev: update basic-workflow.rst (`pr#46287 <https://github.com/ceph/ceph/pull/46287>`_, Zac Dover)
+* doc/mgr/dashboard: Fix typo and double slash missing from URL (`pr#46075 <https://github.com/ceph/ceph/pull/46075>`_, Ville Ojamo)
+* doc/start: add testing support information (`pr#45988 <https://github.com/ceph/ceph/pull/45988>`_, Zac Dover)
+* doc/start: s/3/three/ in intro.rst (`pr#46325 <https://github.com/ceph/ceph/pull/46325>`_, Zac Dover)
+* doc/start: update "memory" in hardware-recs.rst (`pr#46449 <https://github.com/ceph/ceph/pull/46449>`_, Zac Dover)
+* Implement CIDR blocklisting (`pr#46469 <https://github.com/ceph/ceph/pull/46469>`_, Jos Collin, Greg Farnum)
+* librbd/cache/pwl: fix bit field endianness issue (`pr#46094 <https://github.com/ceph/ceph/pull/46094>`_, Yin Congmin)
+* mds: add a perf counter to record slow replies (`pr#46156 <https://github.com/ceph/ceph/pull/46156>`_, haoyixing)
+* mds: include encoded stray inode when sending dentry unlink message to replicas (`issue#54046 <http://tracker.ceph.com/issues/54046>`_, `pr#46184 <https://github.com/ceph/ceph/pull/46184>`_, Venky Shankar)
+* mds: reset heartbeat when fetching or committing entries (`pr#46181 <https://github.com/ceph/ceph/pull/46181>`_, Xiubo Li)
+* mds: trigger to flush the mdlog in handle_find_ino() (`pr#46497 <https://github.com/ceph/ceph/pull/46497>`_, Xiubo Li)
+* mgr/cephadm: Adding python natsort module (`pr#46065 <https://github.com/ceph/ceph/pull/46065>`_, Redouane Kachach)
+* mgr/cephadm: try to get FQDN for configuration files (`pr#45665 <https://github.com/ceph/ceph/pull/45665>`_, Tatjana Dehler)
+* mgr/dashboard:  don't log 3xx as errors (`pr#46453 <https://github.com/ceph/ceph/pull/46453>`_, Ernesto Puerta)
+* mgr/dashboard: Compare values of MTU alert by device (`pr#45814 <https://github.com/ceph/ceph/pull/45814>`_, Aashish Sharma, Patrick Seidensal)
+* mgr/dashboard: Creating and editing Prometheus AlertManager silences is buggy (`pr#46278 <https://github.com/ceph/ceph/pull/46278>`_, Volker Theile)
+* mgr/dashboard: customizable log-in page text/banner (`pr#46342 <https://github.com/ceph/ceph/pull/46342>`_, Sarthak0702)
+* mgr/dashboard: datatable in Cluster Host page hides wrong column on selection (`pr#45862 <https://github.com/ceph/ceph/pull/45862>`_, Sarthak0702)
+* mgr/dashboard: extend daemon actions to host details (`pr#45722 <https://github.com/ceph/ceph/pull/45722>`_, Aashish Sharma, Nizamudeen A)
+* mgr/dashboard: fix columns in host table  with NaN Undefined (`pr#46446 <https://github.com/ceph/ceph/pull/46446>`_, Avan Thakkar)
+* mgr/dashboard: fix ssl cert validation for ingress service creation (`pr#46203 <https://github.com/ceph/ceph/pull/46203>`_, Avan Thakkar)
+* mgr/dashboard: fix wrong pg status processing (`pr#46229 <https://github.com/ceph/ceph/pull/46229>`_, Ernesto Puerta)
+* mgr/dashboard: form field validation icons overlap with other icons (`pr#46380 <https://github.com/ceph/ceph/pull/46380>`_, Sarthak0702)
+* mgr/dashboard: highlight the search text in cluster logs (`pr#45679 <https://github.com/ceph/ceph/pull/45679>`_, Sarthak0702)
+* mgr/dashboard: Imrove error message of '/api/grafana/validation' API endpoint (`pr#45957 <https://github.com/ceph/ceph/pull/45957>`_, Volker Theile)
+* mgr/dashboard: introduce memory and cpu usage for daemons (`pr#46220 <https://github.com/ceph/ceph/pull/46220>`_, Aashish Sharma, Avan Thakkar)
+* mgr/dashboard: Language dropdown box is partly hidden on login page (`pr#45619 <https://github.com/ceph/ceph/pull/45619>`_, Volker Theile)
+* mgr/dashboard: RGW users and buckets tables are empty if the selected gateway is down (`pr#45867 <https://github.com/ceph/ceph/pull/45867>`_, Volker Theile)
+* mgr/dashboard: Table columns hiding fix (`issue#51119 <http://tracker.ceph.com/issues/51119>`_, `pr#45724 <https://github.com/ceph/ceph/pull/45724>`_, Daniel Persson)
+* mgr/dashboard: unselect rows in datatables (`pr#46323 <https://github.com/ceph/ceph/pull/46323>`_, Sarthak0702)
+* mgr/dashboard: WDC multipath bug fixes (`pr#46455 <https://github.com/ceph/ceph/pull/46455>`_, Nizamudeen A)
+* mgr/stats: be resilient to offline MDS rank-0 (`pr#45291 <https://github.com/ceph/ceph/pull/45291>`_, Jos Collin)
+* mgr/telemetry: add Rook data (`pr#46486 <https://github.com/ceph/ceph/pull/46486>`_, Yaarit Hatuka)
+* mgr/volumes: Fix idempotent subvolume rm (`pr#46140 <https://github.com/ceph/ceph/pull/46140>`_, Kotresh HR)
+* mgr/volumes: set, get, list and remove metadata of snapshot (`pr#46508 <https://github.com/ceph/ceph/pull/46508>`_, Nikhilkumar Shelke)
+* mgr/volumes: set, get, list and remove metadata of subvolume (`pr#45994 <https://github.com/ceph/ceph/pull/45994>`_, Nikhilkumar Shelke)
+* mgr/volumes: Show clone failure reason in clone status command (`pr#45927 <https://github.com/ceph/ceph/pull/45927>`_, Kotresh HR)
+* mon/LogMonitor: reopen log files on SIGHUP (`pr#46374 <https://github.com/ceph/ceph/pull/46374>`_, 胡玮文)
+* mon/OSDMonitor: properly set last_force_op_resend in stretch mode (`pr#45871 <https://github.com/ceph/ceph/pull/45871>`_, Ilya Dryomov)
+* mount/conf: Fix IPv6 parsing (`pr#46113 <https://github.com/ceph/ceph/pull/46113>`_, Matan Breizman)
+* os/bluestore: set upper and lower bounds on rocksdb omap iterators (`pr#46175 <https://github.com/ceph/ceph/pull/46175>`_, Adam Kupczyk, Cory Snyder)
+* os/bluestore: turn `bluestore zero block detection` off by default (`pr#46468 <https://github.com/ceph/ceph/pull/46468>`_, Laura Flores)
+* osd/PGLog.cc: Trim duplicates by number of entries (`pr#46251 <https://github.com/ceph/ceph/pull/46251>`_, Nitzan Mordechai)
+* osd/scrub: ignoring unsolicited DigestUpdate events (`pr#45595 <https://github.com/ceph/ceph/pull/45595>`_, Ronen Friedman)
+* osd/scrub: restart snap trimming after a failed scrub (`pr#46418 <https://github.com/ceph/ceph/pull/46418>`_, Ronen Friedman)
+* osd: return appropriate error if the object is not manifest (`pr#46061 <https://github.com/ceph/ceph/pull/46061>`_, Myoungwon Oh)
+* qa/suites/rados/thrash-erasure-code-big/thrashers: add `osd max backfills` setting to mapgap and pggrow (`pr#46384 <https://github.com/ceph/ceph/pull/46384>`_, Laura Flores)
+* qa/tasks/cephadm_cases: increase timeouts in test_cli.py (`pr#45625 <https://github.com/ceph/ceph/pull/45625>`_, Adam King)
+* qa: add filesystem/file sync stuck test support (`pr#46496 <https://github.com/ceph/ceph/pull/46496>`_, Xiubo Li)
+* qa: fix teuthology master branch ref (`pr#46503 <https://github.com/ceph/ceph/pull/46503>`_, Ernesto Puerta)
+* qa: remove .teuthology_branch file (`pr#46491 <https://github.com/ceph/ceph/pull/46491>`_, Jeff Layton)
+* Quincy: client: stop forwarding the request when exceeding 256 times (`pr#46178 <https://github.com/ceph/ceph/pull/46178>`_, Xiubo Li)
+* Quincy: Wip doc backport quincy release notes to quincy branch 2022 05 24 (`pr#46381 <https://github.com/ceph/ceph/pull/46381>`_, Neha Ojha, David Galloway, Josh Durgin, Ilya Dryomov, Ernesto Puerta, Sridhar Seshasayee, Zac Dover, Yaarit Hatuka)
+* rbd persistent cache UX improvements (status report, metrics, flush command) (`pr#45896 <https://github.com/ceph/ceph/pull/45896>`_, Ilya Dryomov, Yin Congmin)
+* rgw: OpsLogFile::stop() signals under mutex (`pr#46038 <https://github.com/ceph/ceph/pull/46038>`_, Casey Bodley)
+* rgw: remove rgw_rados_pool_pg_num_min and its use on pool creation use the cluster defaults for pg_num_min (`pr#46234 <https://github.com/ceph/ceph/pull/46234>`_, Casey Bodley)
+* rgw: RGWCoroutine::set_sleeping() checks for null stack (`pr#46041 <https://github.com/ceph/ceph/pull/46041>`_, Or Friedmann, Casey Bodley)
+* rgw_reshard: drop olh entries with empty name (`pr#45846 <https://github.com/ceph/ceph/pull/45846>`_, Dan van der Ster)
+* rocksdb: build with rocksdb-7.y.z (`pr#46492 <https://github.com/ceph/ceph/pull/46492>`_, Kaleb S. KEITHLEY)
+* rpm: use system libpmem on Centos 9 Stream (`pr#46212 <https://github.com/ceph/ceph/pull/46212>`_, Ilya Dryomov)
+* run-make-check.sh: enable RBD persistent caches (`pr#45992 <https://github.com/ceph/ceph/pull/45992>`_, Ilya Dryomov)
+* test/rbd_mirror: grab timer lock before calling add_event_after() (`pr#45905 <https://github.com/ceph/ceph/pull/45905>`_, Ilya Dryomov)
+* test: fix TierFlushDuringFlush to wait until dedup_tier is set on base pool (`issue#53855 <http://tracker.ceph.com/issues/53855>`_, `pr#45624 <https://github.com/ceph/ceph/pull/45624>`_, Sungmin Lee)
+* test: No direct use of nose (`pr#46254 <https://github.com/ceph/ceph/pull/46254>`_, Steve Kowalik)
+* Wip doc pr 46109 backport to quincy (`pr#46116 <https://github.com/ceph/ceph/pull/46116>`_, Ville Ojamo)
+
+v17.2.0 Quincy
+==============
+
+This is the first stable release of Ceph Quincy.
+
+Major Changes from Pacific
+--------------------------
+
+General
+~~~~~~~
+
+* Filestore has been deprecated in Quincy. BlueStore is Ceph's default object
+  store.
+
+* The `ceph-mgr-modules-core` debian package no longer recommends
+  `ceph-mgr-rook`. `ceph-mgr-rook` depends on `python3-numpy`, which
+  cannot be imported in different Python sub-interpreters multiple times
+  when the version of `python3-numpy` is older than 1.19. Because
+  `apt-get` installs the `Recommends` packages by default, `ceph-mgr-rook`
+  was always installed along with the `ceph-mgr` debian package as an
+  indirect dependency. If your workflow depends on this behavior, you
+  might want to install `ceph-mgr-rook` separately.
+
+* The ``device_health_metrics`` pool has been renamed ``.mgr``. It is now
+  used as a common store for all ``ceph-mgr`` modules. After upgrading to
+  Quincy, the ``device_health_metrics`` pool will be renamed to ``.mgr``
+  on existing clusters.
+
+* The ``ceph pg dump`` command now prints three additional columns:
+  `LAST_SCRUB_DURATION` shows the duration (in seconds) of the last completed
+  scrub;
+  `SCRUB_SCHEDULING` conveys whether a PG is scheduled to be scrubbed at a
+  specified time, whether it is queued for scrubbing, or whether it is being
+  scrubbed;
+  `OBJECTS_SCRUBBED` shows the number of objects scrubbed in a PG after a
+  scrub begins.
+
+* A health warning is now reported if the ``require-osd-release`` flag
+  is not set to the appropriate release after a cluster upgrade.
+
+* LevelDB support has been removed. ``WITH_LEVELDB`` is no longer a supported
+  build option. Users *should* migrate their monitors and OSDs to RocksDB
+  before upgrading to Quincy.
+
+* Cephadm: ``osd_memory_target_autotune`` is enabled by default, which sets
+  ``mgr/cephadm/autotune_memory_target_ratio`` to ``0.7`` of total RAM. This
+  is unsuitable for hyperconverged infrastructures. For hyperconverged Ceph,
+  please refer to the documentation or set
+  ``mgr/cephadm/autotune_memory_target_ratio`` to ``0.2``.
+
+* telemetry: Improved the opt-in flow so that users can keep sharing the same
+  data, even when new data collections are available. A new 'perf' channel that
+  collects various performance metrics is now available for operators to opt
+  into with:
+  `ceph telemetry on`
+  `ceph telemetry enable channel perf`
+  See a sample report with `ceph telemetry preview`.
+  Note that generating a telemetry report with 'perf' channel data might
+  take a few moments in big clusters.
+  For more details, see:
+  https://docs.ceph.com/en/quincy/mgr/telemetry/
+
+* MGR: The progress module disables the pg recovery event by default since the
+  event is expensive and has interrupted other services when there are OSDs
+  being marked in/out from the cluster. However, the user can still enable
+  this event anytime. For more detail, see:
+
+  https://docs.ceph.com/en/quincy/mgr/progress/
+
+* https://tracker.ceph.com/issues/55383 is a known issue -
+  to continue to log cluster log messages to file,
+  run `ceph config set mon mon_cluster_log_to_file true` after every log rotation.
+
+Cephadm
+-------
+
+* SNMP Support
+* Colocation of Daemons (mgr, mds, rgw)
+* osd memory autotuning
+* Integration with new NFS mgr module
+* Ability to zap osds as they are removed
+* cephadm agent for increased performance/scalability
+
+Dashboard
+~~~~~~~~~
+* Day 1: the new "Cluster Expansion Wizard" will guide users through post-install steps:
+  adding new hosts, storage devices or services.
+* NFS: the Dashboard now allows users to fully manage all NFS exports from a single place.
+* New mgr module (feedback): users can quickly report Ceph tracker issues
+  or suggestions directly from the Dashboard or the CLI.
+* New "Message of the Day": cluster admins can publish a custom message in a banner.
+* Cephadm integration improvements:
+   * Host management: maintenance, specs and labelling,
+   * Service management: edit and display logs,
+   * Daemon management (start, stop, restart, reload),
+   * New services supported: ingress (HAProxy) and SNMP-gateway.
+* Monitoring and alerting:
+   * 43 new alerts have been added (totalling 68) improving observability of events affecting:
+     cluster health, monitors, storage devices, PGs and CephFS.
+   * Alerts can now be sent externally as SNMP traps via the new SNMP gateway service
+     (the MIB is provided).
+   * Improved integrated full/nearfull event notifications.
+   * Grafana Dashboards now use grafonnet format (though they're still available
+     in JSON format).
+   * Stack update: images for monitoring containers have been updated.
+     Grafana 8.3.5, Prometheus 2.33.4, Alertmanager 0.23.0 and Node Exporter 1.3.1.
+     This reduced exposure to several Grafana vulnerabilities (CVE-2021-43798,
+     CVE-2021-39226, CVE-2021-43798,  CVE-2020-29510, CVE-2020-29511).
+
+RADOS
+~~~~~
+
+* OSD: Ceph now uses `mclock_scheduler` for BlueStore OSDs as its default
+  `osd_op_queue` to provide QoS. The 'mclock_scheduler' is not supported
+  for Filestore OSDs. Therefore, the default 'osd_op_queue' is set to `wpq`
+  for Filestore OSDs and is enforced even if the user attempts to change it.
+  For more details on configuring mclock see,
+
+  https://docs.ceph.com/en/quincy/rados/configuration/mclock-config-ref/
+
+  An outstanding issue exists during runtime where the mclock config options
+  related to reservation, weight and limit cannot be modified after switching
+  to the `custom` mclock profile using the `ceph config set ...` command.
+  This is tracked by: https://tracker.ceph.com/issues/55153. Until the issue
+  is fixed, users are advised to avoid using the 'custom' profile or use the
+  workaround mentioned in the tracker.
+
+* MGR: The pg_autoscaler can now be turned `on` and `off` globally
+  with the `noautoscale` flag. By default, it is set to `on`, but this flag
+  can come in handy to prevent rebalancing triggered by autoscaling during
+  cluster upgrade and maintenance. Pools can now be created with the `--bulk`
+  flag, which allows the autoscaler to allocate more PGs to such pools. This
+  can be useful to get better out of the box performance for data-heavy pools.
+
+  For more details about autoscaling, see:
+  https://docs.ceph.com/en/quincy/rados/operations/placement-groups/
+
+* OSD: Support for on-wire compression for osd-osd communication, `off` by
+  default.
+
+  For more details about compression modes, see:
+  https://docs.ceph.com/en/quincy/rados/configuration/msgr2/#compression-modes
+
+* OSD: Concise reporting of slow operations in the cluster log. The old
+  and more verbose logging behavior can be regained by setting
+  `osd_aggregated_slow_ops_logging` to false.
+
+* the "kvs" Ceph object class is not packaged anymore. The "kvs" Ceph
+  object class offers a distributed flat b-tree key-value store that
+  is implemented on top of the librados objects omap. Because there
+  are no existing internal users of this object class, it is not
+  packaged anymore.
+
+RBD block storage
+~~~~~~~~~~~~~~~~~
+
+* rbd-nbd: `rbd device attach` and `rbd device detach` commands added,
+  these allow for safe reattach after `rbd-nbd` daemon is restarted since
+  Linux kernel 5.14.
+
+* rbd-nbd: `notrim` map option added to support thick-provisioned images,
+  similar to krbd.
+
+* Large stabilization effort for client-side persistent caching on SSD
+  devices, also available in 16.2.8. For details on usage, see:
+
+  https://docs.ceph.com/en/quincy/rbd/rbd-persistent-write-log-cache/
+
+* Several bug fixes in diff calculation when using fast-diff image
+  feature + whole object (inexact) mode. In some rare cases these
+  long-standing issues could cause an incorrect `rbd export`. Also
+  fixed in 15.2.16 and 16.2.8.
+
+* Fix for a potential performance degradation when running Windows VMs
+  on krbd. For details, see `rxbounce` map option description:
+
+  https://docs.ceph.com/en/quincy/man/8/rbd/#kernel-rbd-krbd-options
+
+RGW object storage
+~~~~~~~~~~~~~~~~~~
+
+* RGW now supports rate limiting by user and/or by bucket. With this
+  feature it is possible to limit user and/or bucket, the total operations
+  and/or bytes per minute can be delivered. This feature allows the
+  admin to limit only READ operations and/or WRITE operations. The
+  rate-limiting configuration could be applied on all users and all buckets
+  by using global configuration.
+
+* `radosgw-admin realm delete` has been renamed to `radosgw-admin realm
+  rm`. This is consistent with the help message.
+
+* S3 bucket notification events now contain an `eTag` key instead of
+  `etag`, and eventName values no longer carry the `s3:` prefix, fixing
+  deviations from the message format that is observed on AWS.
+
+* It is possible to specify ssl options and ciphers for beast frontend
+  now. The default ssl options setting is
+  "no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1". If you want to return to the old
+  behavior, add 'ssl_options=' (empty) to the ``rgw frontends`` configuration.
+
+* The behavior for Multipart Upload was modified so that only
+  CompleteMultipartUpload notification is sent at the end of the multipart
+  upload. The POST notification at the beginning of the upload and the PUT
+  notifications that were sent on each part are no longer sent.
+
+
+CephFS distributed file system
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* fs: A file system can be created with a specific ID ("fscid"). This is
+  useful in certain recovery scenarios (for example, when a monitor
+  database has been lost and rebuilt, and the restored file system is
+  expected to have the same ID as before).
+
+* fs: A file system can be renamed using the `fs rename` command. Any cephx
+  credentials authorized for the old file system name will need to be
+  reauthorized to the new file system name. Since the operations of the clients
+  using these re-authorized IDs may be disrupted, this command requires the
+  "--yes-i-really-mean-it" flag. Also, mirroring is expected to be disabled
+  on the file system.
+
+* MDS upgrades no longer require all standby MDS daemons to be stoped before
+  upgrading a file systems's sole active MDS.
+
+* CephFS: Failure to replay the journal by a standby-replay daemon now
+  causes the rank to be marked "damaged".
+
+Upgrading from Octopus or Pacific
+----------------------------------
+
+Quincy does not support LevelDB. Please migrate your OSDs and monitors
+to RocksDB before upgrading to Quincy.
+
+Before starting, make sure your cluster is stable and healthy (no down or
+recovering OSDs).  (This is optional, but recommended.) You can disable
+the autoscaler for all pools during the upgrade using the noautoscale flag.
+
+.. note::
+
+  You can monitor the progress of your upgrade at each stage with the
+  ``ceph versions`` command, which will tell you what ceph version(s) are
+  running for each type of daemon.
+
+Upgrading cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your cluster is deployed with cephadm (first introduced in Octopus), then
+the upgrade process is entirely automated.  To initiate the upgrade,
+
+  .. prompt:: bash #
+
+    ceph orch upgrade start --ceph-version 17.2.0
+
+The same process is used to upgrade to future minor releases.
+
+Upgrade progress can be monitored with ``ceph -s`` (which provides a simple
+progress bar) or more verbosely with
+
+  .. prompt:: bash #
+
+    ceph -W cephadm
+
+The upgrade can be paused or resumed with
+
+  .. prompt:: bash #
+
+    ceph orch upgrade pause   # to pause
+    ceph orch upgrade resume  # to resume
+
+or canceled with
+
+  .. prompt:: bash #
+
+    ceph orch upgrade stop
+
+Note that canceling the upgrade simply stops the process; there is no ability to
+downgrade back to Octopus or Pacific.
+
+
+Upgrading non-cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+   If you cluster is running Octopus (15.2.x) or later, you might choose
+   to first convert it to use cephadm so that the upgrade to Quincy
+   is automated (see above).  For more information, see
+   :ref:`cephadm-adoption`.
+
+#. Set the ``noout`` flag for the duration of the upgrade. (Optional,
+   but recommended.):
+
+   .. prompt:: bash #
+
+      ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the
+   monitor daemons.  For example, on each monitor host,:
+   
+   .. prompt:: bash #
+
+      systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is
+   complete by looking for the ``quincy`` string in the mon
+   map.  The command:
+   
+   .. prompt:: bash #
+
+      ceph mon dump | grep min_mon_release
+
+   should report::
+
+     min_mon_release 17 (quincy)
+
+   If it doesn't, that implies that one or more monitors hasn't been
+   upgraded and restarted and/or the quorum does not include all monitors.
+
+#. Upgrade ``ceph-mgr`` daemons by installing the new packages and
+   restarting all manager daemons.  For example, on each manager host,:
+   
+   .. prompt:: bash #
+
+      systemctl restart ceph-mgr.target
+
+   Verify the ``ceph-mgr`` daemons are running by checking ``ceph
+   -s``:
+   
+   .. prompt:: bash #
+
+      ceph -s
+
+   ::
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the
+   ceph-osd daemons on all OSD hosts:
+   
+   .. prompt:: bash #
+
+      systemctl restart ceph-osd.target
+
+#. Upgrade all CephFS MDS daemons. For each CephFS file system,
+
+   #. Disable standby_replay.  Before executing, note the current value
+      so that it may be re-enabled after the upgrade (if currently enabled):
+         
+      .. prompt:: bash #
+
+	 ceph fs get <fs_name> | grep allow_standby_replay
+	 ceph fs set <fs_name> allow_standby_replay false
+
+   #. Reduce the number of ranks to 1.  (Make note of the original
+      number of MDS daemons first if you plan to restore it later.):
+      
+      .. prompt:: bash #
+
+	 ceph fs status
+	 ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by
+      periodically checking the status:
+      
+      .. prompt:: bash #
+
+	 ceph fs status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with:
+
+      .. prompt:: bash #
+
+	 systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS:
+
+      .. prompt:: bash #
+
+	 ceph fs status
+
+   #. Upgrade the last remaining MDS daemon by installing the new
+      packages and restarting the daemon:
+
+      .. prompt:: bash #
+
+         systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline:
+
+      .. prompt:: bash #
+
+	 systemctl start ceph-mds.target
+
+   #. Restore the original value of ``max_mds`` for the volume:
+
+      .. prompt:: bash #
+
+	 ceph fs set <fs_name> max_mds <original_max_mds>
+
+    #. Restore the original value of ``allow_standby_replay`` for the volume if
+       it was ``true``:
+
+      .. prompt:: bash #
+
+	 ceph fs set <fs_name> allow_standby_replay true
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting
+   daemons on all hosts:
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw.target
+
+#. Complete the upgrade by disallowing pre-Quincy OSDs and enabling
+   all new Quincy-only functionality:
+
+   .. prompt:: bash #
+
+      ceph osd require-osd-release quincy
+
+#. If you set ``noout`` at the beginning, be sure to clear it with:
+
+   .. prompt:: bash #
+
+      ceph osd unset noout
+
+#. Consider transitioning your cluster to use the cephadm deployment
+   and orchestration framework to simplify cluster management and
+   future upgrades.  For more information on converting an existing
+   cluster to cephadm, see :ref:`cephadm-adoption`.
+
+Post-upgrade
+~~~~~~~~~~~~
+
+#. Verify the cluster is healthy with ``ceph health``. If your cluster is
+   running Filestore, a deprecation warning is expected. This warning can
+   be temporarily muted using the following command:
+
+   .. prompt:: bash #
+
+      ceph health mute OSD_FILESTORE
+
+#. If you are upgrading from Mimic, or did not already do so when you
+   upgraded to Nautilus, we recommend you enable the new :ref:`v2
+   network protocol <msgr2>`, issue the following command:
+
+   .. prompt:: bash #
+
+      ceph mon enable-msgr2
+
+   This will instruct all monitors that bind to the old default port
+   6789 for the legacy v1 protocol to also bind to the new 3300 v2
+   protocol port.  To see if all monitors have been updated, run this:
+
+   .. prompt:: bash #
+
+      ceph mon dump
+
+   and verify that each monitor has both a ``v2:`` and ``v1:`` address
+   listed.
+
+#. Consider enabling the :ref:`telemetry module <telemetry>` to send
+   anonymized usage statistics and crash information to the Ceph
+   upstream developers.  To see what would be reported (without actually
+   sending any information to anyone),:
+
+   .. prompt:: bash #
+
+      ceph telemetry preview-all
+
+   If you are comfortable with the data that is reported, you can opt-in to
+   automatically report the high-level cluster metadata with:
+
+   .. prompt:: bash #
+
+      ceph telemetry on
+
+   The public dashboard that aggregates Ceph telemetry can be found at
+   `https://telemetry-public.ceph.com/ <https://telemetry-public.ceph.com/>`_.
+
+   For more information about the telemetry module, see :ref:`the
+   documentation <telemetry>`.
+
+
+Upgrading from pre-Octopus releases (like Nautilus)
+---------------------------------------------------
+
+
+You *must* first upgrade to Octopus (15.2.z) or Pacific (16.2.z) before
+upgrading to Quincy.
diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
new file mode 100644
index 000000000..6f4f82783
--- /dev/null
+++ b/doc/releases/reef.rst
@@ -0,0 +1,551 @@
+====
+Reef
+====
+
+Reef is the 18th stable release of Ceph. It is named after the reef squid
+(Sepioteuthis).
+
+v18.2.0 Reef
+============
+
+This is the first stable release of Ceph Reef.
+
+.. important::
+
+   We are unable to build Ceph on Debian stable (bookworm) for the 18.2.0
+   release because of Debian bug
+   https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1030129. We will build as
+   soon as this bug is resolved in Debian stable.
+
+   *last updated 2023 Aug 04*
+
+Major Changes from Quincy
+--------------------------
+
+Highlights
+~~~~~~~~~~
+
+See the relevant sections below for more details on these changes.
+
+* **RADOS** FileStore is not supported in Reef.
+* **RADOS:** RocksDB has been upgraded to version 7.9.2.
+* **RADOS:** There have been significant improvements to RocksDB iteration overhead and performance.
+* **RADOS:** The ``perf dump`` and ``perf schema`` commands have been deprecated in
+  favor of the new ``counter dump`` and ``counter schema`` commands.
+* **RADOS:** Cache tiering is now deprecated.
+* **RADOS:** A new feature, the "read balancer", is now available, which allows users to balance primary PGs per pool on their clusters.
+* **RGW:** Bucket resharding is now supported for multi-site configurations.
+* **RGW:** There have been significant improvements to the stability and consistency of multi-site replication.
+* **RGW:** Compression is now supported for objects uploaded with Server-Side Encryption.
+* **Dashboard:** There is a new Dashboard page with improved layout. Active alerts and some important charts are now displayed inside cards.
+* **RBD:** Support for layered client-side encryption has been added.
+* **Telemetry**: Users can now opt in to participate in a leaderboard in the telemetry public dashboards.
+
+CephFS
+~~~~~~
+
+* CephFS: The ``mds_max_retries_on_remount_failure`` option has been renamed to
+  ``client_max_retries_on_remount_failure`` and moved from ``mds.yaml.in`` to
+  ``mds-client.yaml.in``. This change was made because the option has always
+  been used only by the MDS client.
+* CephFS: It is now possible to delete the recovered files in the
+  ``lost+found`` directory after a CephFS post has been recovered in accordance
+  with disaster recovery procedures.
+* The ``AT_NO_ATTR_SYNC`` macro has been deprecated in favor of the standard
+  ``AT_STATX_DONT_SYNC`` macro. The ``AT_NO_ATTR_SYNC`` macro will be removed
+  in the future.
+
+Dashboard
+~~~~~~~~~
+
+* There is a new Dashboard page with improved layout. Active alerts
+  and some important charts are now displayed inside cards.
+
+* Cephx Auth Management: There is a new section dedicated to listing and
+  managing Ceph cluster users.
+  
+* RGW Server Side Encryption: The SSE-S3 and KMS encryption of rgw buckets can
+  now be configured at the time of bucket creation.
+
+* RBD Snapshot mirroring: Snapshot mirroring can now be configured through UI.
+  Snapshots can now be scheduled.
+  
+* 1-Click OSD Creation Wizard: OSD creation has been broken into 3 options:
+
+  #. Cost/Capacity Optimized: Use all HDDs
+
+  #. Throughput Optimized: Combine HDDs and SSDs
+
+  #. IOPS Optimized: Use all NVMes
+
+  The current OSD-creation form has been moved to the Advanced section.
+
+* Centralized Logging: There is now a view that collects all the logs from
+  the Ceph cluster.
+
+* Accessibility WCAG-AA: Dashboard is WCAG 2.1 level A compliant and therefore
+  improved for blind and visually impaired Ceph users.
+
+* Monitoring & Alerting
+
+      * Ceph-exporter: Now the performance metrics for Ceph daemons are
+        exported by ceph-exporter, which deploys on each daemon rather than
+        using prometheus exporter. This will reduce performance bottlenecks.
+
+      * Monitoring stacks updated:
+
+            * Prometheus 2.43.0
+
+            * Node-exporter 1.5.0
+
+            * Grafana 9.4.7
+
+            * Alertmanager 0.25.0
+
+MGR
+~~~
+
+* mgr/snap_schedule: The snap-schedule manager module now retains one snapshot
+  less than the number mentioned against the config option
+  ``mds_max_snaps_per_dir``. This means that a new snapshot can be created and
+  retained during the next schedule run.
+* The ``ceph mgr dump`` command now outputs ``last_failure_osd_epoch`` and
+  ``active_clients`` fields at the top level. Previously, these fields were
+  output under the ``always_on_modules`` field.
+  
+RADOS
+~~~~~
+
+* FileStore is not supported in Reef.
+* RocksDB has been upgraded to version 7.9.2, which incorporates several
+  performance improvements and features. This is the first release that can
+  tune RocksDB settings per column family, which allows for more granular
+  tunings to be applied to different kinds of data stored in RocksDB. New
+  default settings have been used to optimize performance for most workloads, with a
+  slight penalty in some use cases. This slight penalty is outweighed by large
+  improvements in compactions and write amplification in use cases such as RGW
+  (up to a measured 13.59% improvement in 4K random write IOPs).
+* Trimming of PGLog dups is now controlled by the size rather than the version.
+  This change fixes the PGLog inflation issue that was happening when the
+  online (in OSD) trimming got jammed after a PG split operation. Also, a new
+  offline mechanism has been added: ``ceph-objectstore-tool`` has a new
+  operation called ``trim-pg-log-dups`` that targets situations in which an OSD
+  is unable to boot because of the inflated dups. In such situations, the "You
+  can be hit by THE DUPS BUG" warning is visible in OSD logs. Relevant tracker:
+  https://tracker.ceph.com/issues/53729
+* The RADOS Python bindings are now able to process (opt-in) omap keys as bytes
+  objects. This allows interacting with RADOS omap keys that are not
+  decodable as UTF-8 strings.
+* mClock Scheduler: The mClock scheduler (the default scheduler in Quincy) has
+  undergone significant usability and design improvements to address the slow
+  backfill issue. The following is a list of some important changes:
+
+  * The ``balanced`` profile is set as the default mClock profile because it
+    represents a compromise between prioritizing client I/O and prioritizing
+    recovery I/O. Users can then choose either the ``high_client_ops`` profile
+    to prioritize client I/O or the ``high_recovery_ops`` profile to prioritize
+    recovery I/O.
+  * QoS parameters including ``reservation`` and ``limit`` are now specified in 
+    terms of a fraction (range: 0.0 to 1.0) of the OSD's IOPS capacity.
+  * The cost parameters (``osd_mclock_cost_per_io_usec_*`` and
+    ``osd_mclock_cost_per_byte_usec_*``) have been removed. The cost of an 
+    operation is now a function of the random IOPS and maximum sequential
+    bandwidth capability of the OSD's underlying device.
+  * Degraded object recovery is given higher priority than misplaced
+    object recovery because degraded objects present a data safety issue that
+    is not present with objects that are merely misplaced. As a result,
+    backfilling operations with the ``balanced`` and ``high_client_ops`` mClock
+    profiles might progress more slowly than in the past, when backfilling
+    operations used the 'WeightedPriorityQueue' (WPQ) scheduler.
+  * The QoS allocations in all the mClock profiles are optimized in 
+    accordance with the above fixes and enhancements.
+  * For more details, see:
+    https://docs.ceph.com/en/reef/rados/configuration/mclock-config-ref/
+* A new feature, the "read balancer", is now available, which allows
+  users to balance primary PGs per pool on their clusters. The read balancer is
+  currently available as an offline option via the ``osdmaptool``. By providing
+  a copy of their osdmap and a pool they want balanced to the ``osdmaptool``, users
+  can generate a preview of optimal primary PG mappings that they can then choose to
+  apply to their cluster. For more details, see
+  https://docs.ceph.com/en/latest/dev/balancer-design/#read-balancing
+* The ``active_clients`` array displayed by the ``ceph mgr dump`` command now
+  has a ``name`` field that shows the name of the manager module that
+  registered a RADOS client. Previously, the ``active_clients`` array showed
+  the address of a module's RADOS client, but not the name of the module.
+* The ``perf dump`` and ``perf schema`` commands have been deprecated in 
+  favor of the new ``counter dump`` and ``counter schema`` commands. These new
+  commands add support for labeled perf counters and also emit existing
+  unlabeled perf counters. Some unlabeled perf counters became labeled in this
+  release, and more will be labeled in future releases; such converted perf
+  counters are no longer emitted by the ``perf dump`` and ``perf schema``
+  commands.
+* Cache tiering is now deprecated.
+* The SPDK backend for BlueStore can now connect to an NVMeoF target. This
+  is not an officially supported feature.
+
+RBD
+~~~
+
+* The semantics of compare-and-write C++ API (`Image::compare_and_write` and
+  `Image::aio_compare_and_write` methods) now match those of C API. Both
+  compare and write steps operate only on len bytes even if the buffers
+  associated with them are larger. The previous behavior of comparing up to the
+  size of the compare buffer was prone to subtle breakage upon straddling a
+  stripe unit boundary.
+* The ``compare-and-write`` operation is no longer limited to 512-byte
+  sectors. Assuming proper alignment, it now allows operating on stripe units
+  (4MB by default).
+* There is a new ``rbd_aio_compare_and_writev`` API method that supports
+  scatter/gather on compare buffers as well as on write buffers. This
+  complements the existing ``rbd_aio_readv`` and ``rbd_aio_writev`` methods.
+* The ``rbd device unmap`` command now has a ``--namespace`` option.
+  Support for namespaces was added to RBD in Nautilus 14.2.0, and since then it
+  has been possible to map and unmap images in namespaces using the
+  ``image-spec`` syntax. However, the corresponding option available in most
+  other commands was missing.
+* All rbd-mirror daemon perf counters have become labeled and are now
+  emitted only by the new ``counter dump`` and ``counter schema`` commands. As
+  part of the conversion, many were also renamed in order to better
+  disambiguate journal-based and snapshot-based mirroring.
+* The list-watchers C++ API (`Image::list_watchers`) now clears the passed
+  `std::list` before appending to it. This aligns with the semantics of the C
+  API (``rbd_watchers_list``).
+* Trailing newline in passphrase files (for example: the
+  ``<passphrase-file>`` argument of the ``rbd encryption format`` command and
+  the ``--encryption-passphrase-file`` option of other commands) is no longer
+  stripped.
+* Support for layered client-side encryption has been added. It is now
+  possible to encrypt cloned images with a distinct encryption format and
+  passphrase, differing from that of the parent image and from that of every
+  other cloned image. The efficient copy-on-write semantics intrinsic to
+  unformatted (regular) cloned images have been retained.
+
+RGW
+~~~
+
+* Bucket resharding is now supported for multi-site configurations. This
+  feature is enabled by default for new deployments. Existing deployments must
+  enable the ``resharding`` feature manually after all zones have upgraded.
+  See https://docs.ceph.com/en/reef/radosgw/multisite/#zone-features for
+  details.
+* The RGW policy parser now rejects unknown principals by default. If you are
+  mirroring policies between RGW and AWS, you might want to set
+  ``rgw_policy_reject_invalid_principals`` to ``false``. This change affects
+  only newly set policies, not policies that are already in place.
+* RGW's default backend for ``rgw_enable_ops_log`` has changed from ``RADOS``
+  to ``file``. The default value of ``rgw_ops_log_rados`` is now ``false``, and
+  ``rgw_ops_log_file_path`` now defaults to
+  ``/var/log/ceph/ops-log-$cluster-$name.log``.
+* RGW's pubsub interface now returns boolean fields using ``bool``. Before this
+  change, ``/topics/<topic-name>`` returned ``stored_secret`` and
+  ``persistent`` using a string of ``"true"`` or ``"false"`` that contains
+  enclosing quotation marks. After this change, these fields are returned
+  without enclosing quotation marks so that the fields can be decoded as
+  boolean values in JSON. The same is true of the ``is_truncated`` field
+  returned by ``/subscriptions/<sub-name>``.
+* RGW's response of ``Action=GetTopicAttributes&TopicArn=<topic-arn>`` REST 
+  API now returns ``HasStoredSecret`` and ``Persistent`` as boolean in the JSON
+  string that is encoded in ``Attributes/EndPoint``.
+* All boolean fields that were previously rendered as strings by the
+  ``rgw-admin`` command when the JSON format was used are now rendered as
+  boolean. If your scripts and tools rely on this behavior, update them
+  accordingly. The following is a list of the field names impacted by this
+  change:
+
+      * ``absolute``
+      * ``add``
+      * ``admin``
+      * ``appendable``
+      * ``bucket_key_enabled``
+      * ``delete_marker``
+      * ``exists``
+      * ``has_bucket_info``
+      * ``high_precision_time``
+      * ``index``
+      * ``is_master``
+      * ``is_prefix``
+      * ``is_truncated``
+      * ``linked``
+      * ``log_meta``
+      * ``log_op``
+      * ``pending_removal``
+      * ``read_only``
+      * ``retain_head_object``
+      * ``rule_exist``
+      * ``start_with_full_sync``
+      * ``sync_from_all``
+      * ``syncstopped``
+      * ``system``
+      * ``truncated``
+      * ``user_stats_sync``
+* The Beast front end's HTTP access log line now uses a new
+  ``debug_rgw_access`` configurable. It has the same defaults as
+  ``debug_rgw``, but it can be controlled independently.
+* The pubsub functionality for storing bucket notifications inside Ceph
+  has been removed. As a result, the pubsub zone should not be used anymore.
+  The following have also been removed: the REST operations, ``radosgw-admin``
+  commands for manipulating subscriptions, fetching the notifications, and
+  acking the notifications. 
+
+  If the endpoint to which the notifications are sent is down or disconnected,
+  we recommend that you use persistent notifications to guarantee their
+  delivery. If the system that consumes the notifications has to pull them
+  (instead of the notifications being pushed to the system), use an external
+  message bus (for example, RabbitMQ or Kafka) for that purpose. 
+* The serialized format of notification and topics has changed. This means
+  that new and updated topics will be unreadable by old RGWs. We recommend
+  completing the RGW upgrades before creating or modifying any notification
+  topics.
+* Compression is now supported for objects uploaded with Server-Side
+  Encryption. When both compression and encryption are enabled, compression is
+  applied before encryption. Earlier releases of multisite do not replicate
+  such objects correctly, so all zones must upgrade to Reef before enabling the
+  `compress-encrypted` zonegroup feature: see
+  https://docs.ceph.com/en/reef/radosgw/multisite/#zone-features and note the
+  security considerations.
+  
+Telemetry
+~~~~~~~~~
+
+* Users who have opted in to telemetry can also opt in to
+  participate in a leaderboard in the telemetry public dashboards
+  (https://telemetry-public.ceph.com/). In addition, users are now able to
+  provide a description of their cluster that will appear publicly in the
+  leaderboard. For more details, see:
+  https://docs.ceph.com/en/reef/mgr/telemetry/#leaderboard. To see a sample
+  report, run ``ceph telemetry preview``. To opt in to telemetry, run ``ceph
+  telemetry on``. To opt in to the leaderboard, run ``ceph config set mgr
+  mgr/telemetry/leaderboard true``. To add a leaderboard description, run
+  ``ceph config set mgr mgr/telemetry/leaderboard_description ‘Cluster
+  description’`` (entering your own cluster description).
+
+Upgrading from Pacific or Quincy
+--------------------------------
+
+Before starting, make sure your cluster is stable and healthy (no down or recovering OSDs). (This is optional, but recommended.) You can disable the autoscaler for all pools during the upgrade using the noautoscale flag.
+
+
+.. note::
+
+   You can monitor the progress of your upgrade at each stage with the ``ceph versions`` command, which will tell you what ceph version(s) are running for each type of daemon.
+
+Upgrading cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your cluster is deployed with cephadm (first introduced in Octopus), then the upgrade process is entirely automated. To initiate the upgrade,
+
+  .. prompt:: bash #
+
+    ceph orch upgrade start --image quay.io/ceph/ceph:v18.2.0
+
+The same process is used to upgrade to future minor releases.
+
+Upgrade progress can be monitored with
+
+  .. prompt:: bash #
+    
+    ceph orch upgrade status
+
+Upgrade progress can also be monitored with `ceph -s` (which provides a simple progress bar) or more verbosely with
+
+  .. prompt:: bash #
+
+    ceph -W cephadm
+
+The upgrade can be paused or resumed with
+
+  .. prompt:: bash #
+
+    ceph orch upgrade pause  # to pause
+    ceph orch upgrade resume # to resume
+
+or canceled with
+
+.. prompt:: bash #
+
+    ceph orch upgrade stop
+
+Note that canceling the upgrade simply stops the process; there is no ability to downgrade back to Pacific or Quincy.
+
+Upgrading non-cephadm clusters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+
+   1. If your cluster is running Pacific (16.2.x) or later, you might choose to first convert it to use cephadm so that the upgrade to Reef is automated (see above).
+      For more information, see https://docs.ceph.com/en/reef/cephadm/adoption/.
+
+   2. If your cluster is running Pacific (16.2.x) or later, systemd unit file names have changed to include the cluster fsid. To find the correct systemd unit file name for your cluster, run following command:
+
+      ```
+      systemctl -l | grep <daemon type>
+      ```
+
+      Example:
+
+      ```
+      $ systemctl -l | grep mon | grep active
+      ceph-6ce0347c-314a-11ee-9b52-000af7995d6c@mon.f28-h21-000-r630.service                                           loaded active running   Ceph mon.f28-h21-000-r630 for 6ce0347c-314a-11ee-9b52-000af7995d6c
+      ```
+
+#. Set the `noout` flag for the duration of the upgrade. (Optional, but recommended.)
+
+   .. prompt:: bash #
+
+      ceph osd set noout
+
+#. Upgrade monitors by installing the new packages and restarting the monitor daemons. For example, on each monitor host
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-mon.target
+
+   Once all monitors are up, verify that the monitor upgrade is complete by looking for the `reef` string in the mon map. The command
+
+   .. prompt:: bash #
+
+      ceph mon dump | grep min_mon_release
+
+   should report:
+
+   .. prompt:: bash #
+
+      min_mon_release 18 (reef)
+
+   If it does not, that implies that one or more monitors hasn't been upgraded and restarted and/or the quorum does not include all monitors.
+
+#. Upgrade `ceph-mgr` daemons by installing the new packages and restarting all manager daemons. For example, on each manager host,
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-mgr.target
+
+   Verify the `ceph-mgr` daemons are running by checking `ceph -s`:
+
+   .. prompt:: bash #
+
+      ceph -s
+
+   ::
+
+     ...
+       services:
+        mon: 3 daemons, quorum foo,bar,baz
+        mgr: foo(active), standbys: bar, baz
+     ...
+
+#. Upgrade all OSDs by installing the new packages and restarting the ceph-osd daemons on all OSD hosts
+
+   .. prompt:: bash #
+   
+      systemctl restart ceph-osd.target
+
+#. Upgrade all CephFS MDS daemons. For each CephFS file system,
+
+   #. Disable standby_replay:
+
+         .. prompt:: bash #
+         
+            ceph fs set <fs_name> allow_standby_replay false
+
+   #. If upgrading from Pacific <=16.2.5:
+
+         .. prompt:: bash #
+   
+            ceph config set mon mon_mds_skip_sanity true
+
+   #. Reduce the number of ranks to 1. (Make note of the original number of MDS daemons first if you plan to restore it later.)
+   
+      .. prompt:: bash #   
+   
+         ceph status # ceph fs set <fs_name> max_mds 1
+
+   #. Wait for the cluster to deactivate any non-zero ranks by periodically checking the status
+
+      .. prompt:: bash #
+
+         ceph status
+
+   #. Take all standby MDS daemons offline on the appropriate hosts with
+
+      .. prompt:: bash #
+   
+         systemctl stop ceph-mds@<daemon_name>
+
+   #. Confirm that only one MDS is online and is rank 0 for your FS
+
+      .. prompt:: bash #
+
+         ceph status
+
+   #. Upgrade the last remaining MDS daemon by installing the new packages and restarting the daemon
+      
+      .. prompt:: bash #
+
+         systemctl restart ceph-mds.target
+
+   #. Restart all standby MDS daemons that were taken offline
+
+      .. prompt:: bash #
+
+         systemctl start ceph-mds.target
+
+   #. Restore the original value of `max_mds` for the volume
+
+      .. prompt:: bash #
+
+         ceph fs set <fs_name> max_mds <original_max_mds>
+
+   #. If upgrading from Pacific <=16.2.5 (followup to step 5.2):
+
+      .. prompt:: bash #
+
+         ceph config set mon mon_mds_skip_sanity false
+
+#. Upgrade all radosgw daemons by upgrading packages and restarting daemons on all hosts
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw.target
+
+#. Complete the upgrade by disallowing pre-Reef OSDs and enabling all new Reef-only functionality
+   
+   .. prompt:: bash #
+   
+      ceph osd require-osd-release reef
+
+#. If you set `noout` at the beginning, be sure to clear it with
+
+   .. prompt:: bash #
+
+      ceph osd unset noout
+
+#. Consider transitioning your cluster to use the cephadm deployment and orchestration framework to simplify cluster management and future upgrades. For more information on converting an existing cluster to cephadm, see https://docs.ceph.com/en/reef/cephadm/adoption/.
+
+Post-upgrade
+~~~~~~~~~~~~
+
+#. Verify the cluster is healthy with `ceph health`. If your cluster is running Filestore, and you are upgrading directly from Pacific to Reef, a deprecation warning is expected. This warning can be temporarily muted using the following command
+
+   .. prompt:: bash #
+
+      ceph health mute OSD_FILESTORE
+
+#. Consider enabling the `telemetry module <https://docs.ceph.com/en/reef/mgr/telemetry/>`_ to send anonymized usage statistics and crash information to the Ceph upstream developers. To see what would be reported (without actually sending any information to anyone),
+
+   .. prompt:: bash #
+
+      ceph telemetry preview-all
+
+   If you are comfortable with the data that is reported, you can opt-in to automatically report the high-level cluster metadata with
+   
+   .. prompt:: bash #   
+   
+      ceph telemetry on
+
+   The public dashboard that aggregates Ceph telemetry can be found at https://telemetry-public.ceph.com/.
+
+Upgrading from pre-Pacific releases (like Octopus)
+__________________________________________________
+
+You **must** first upgrade to Pacific (16.2.z) or Quincy (17.2.z) before upgrading to Reef.
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
new file mode 100644
index 000000000..5eaffdc65
--- /dev/null
+++ b/doc/releases/releases.yml
@@ -0,0 +1,509 @@
+#
+# there are two sections
+#
+# releases: ... for named releases
+# development: ... for dev releases
+#
+# by default a `version` is interpreted as a sphinx reference when rendered (see
+# schedule.rst for the existing tags such as `_13.2.2`). If a version should not
+# be treated as a reference, add `skip_ref: True` as a key/value pair sibling to
+# the `version` key.
+#
+# If a version might represent an actual number (e.g. 0.80) quote it.
+#
+releases:
+  reef:
+    target_eol: 2025-08-01
+    releases:
+      - version: 18.2.0
+        released: 2023-08-07
+  quincy:
+    target_eol: 2024-06-01
+    releases:
+      - version: 17.2.5
+        released: 2022-10-19
+      - version: 17.2.4
+        released: 2022-09-30
+      - version: 17.2.3
+        released: 2022-07-29
+      - version: 17.2.2
+        released: 2022-07-21
+      - version: 17.2.1
+        released: 2022-06-23
+      - version: 17.2.0
+        released: 2022-04-19
+
+  pacific:
+    target_eol: 2023-10-01
+    releases:
+      - version: 16.2.11
+        released: 2023-01-26
+      - version: 16.2.10
+        released: 2022-07-21
+      - version: 16.2.9
+        released: 2022-05-19
+      - version: 16.2.8
+        released: 2022-05-16
+      - version: 16.2.7
+        released: 2021-12-14
+      - version: 16.2.6
+        released: 2021-09-16
+      - version: 16.2.5
+        released: 2021-07-08
+      - version: 16.2.4
+        released: 2021-05-13
+      - version: 16.2.3
+        released: 2021-05-06
+      - version: 16.2.2
+        released: 2021-05-05
+      - version: 16.2.1
+        released: 2021-04-19
+      - version: 16.2.0
+        released: 2021-03-31
+
+  octopus:
+    target_eol: 2022-06-01
+    actual_eol: 2022-08-09
+    releases:
+      - version: 15.2.17
+        released: 2022-08-09
+      - version: 15.2.16
+        released: 2022-03-01
+      - version: 15.2.15
+        released: 2021-10-20
+      - version: 15.2.14
+        released: 2021-08-05
+      - version: 15.2.13
+        released: 2021-05-26
+      - version: 15.2.12
+        released: 2021-05-13
+      - version: 15.2.11
+        released: 2021-04-19
+      - version: 15.2.10
+        released: 2021-03-18
+      - version: 15.2.9
+        released: 2021-02-23
+      - version: 15.2.8
+        released: 2020-12-16
+      - version: 15.2.7
+        released: 2020-11-30
+      - version: 15.2.6
+        released: 2020-11-18
+      - version: 15.2.5
+        released: 2020-09-16
+      - version: 15.2.4
+        released: 2020-06-30
+      - version: 15.2.3
+        released: 2020-05-29
+      - version: 15.2.2
+        released: 2020-05-18
+      - version: 15.2.1
+        released: 2020-04-09
+      - version: 15.2.0
+        released: 2020-03-23
+
+  nautilus:
+    target_eol: 2021-06-01
+    actual_eol: 2021-06-30
+    releases:
+      - version: 14.2.22
+        released: 2021-06-30
+      - version: 14.2.21
+        released: 2021-05-13
+      - version: 14.2.20
+        released: 2021-04-19
+      - version: 14.2.19
+        released: 2021-03-30
+      - version: 14.2.18
+        released: 2021-03-15
+      - version: 14.2.17
+        released: 2021-03-11
+      - version: 14.2.16
+        released: 2020-12-16
+      - version: 14.2.15
+        released: 2020-11-23
+      - version: 14.2.14
+        released: 2020-11-18
+      - version: 14.2.13
+        released: 2020-11-02
+      - version: 14.2.12
+        released: 2020-09-21
+      - version: 14.2.11
+        released: 2020-08-11
+      - version: 14.2.10
+        released: 2020-06-26
+      - version: 14.2.9
+        released: 2020-04-15
+      - version: 14.2.8
+        released: 2020-03-03
+      - version: 14.2.7
+        released: 2020-01-31
+      - version: 14.2.6
+        released: 2020-01-09
+      - version: 14.2.5
+        released: 2019-12-10
+      - version: 14.2.4
+        released: 2019-09-17
+      - version: 14.2.3
+        released: 2019-09-04
+      - version: 14.2.2
+        released: 2019-07-17
+      - version: 14.2.1
+        released: 2019-04-29
+      - version: 14.2.0
+        released: 2019-03-19
+
+  mimic:
+    target_eol: 2020-06-01
+    actual_eol: 2020-07-22
+    releases:
+      - version: 13.2.10
+        released: 2020-04-23
+      - version: 13.2.9
+        released: 2020-04-16
+      - version: 13.2.8
+        released: 2019-12-13
+      - version: 13.2.7
+        released: 2019-11-25
+      - version: 13.2.6
+        released: 2019-06-04
+      - version: 13.2.5
+        released: 2019-03-13
+      - version: 13.2.4
+        released: 2019-01-07
+      - version: 13.2.3
+        released: 2019-01-07
+      - version: 13.2.2
+        released: 2018-09-01
+      - version: 13.2.1
+        released: 2018-07-01
+      - version: 13.2.0
+        released: 2018-06-01
+
+  luminous:
+    target_eol: 2019-06-01
+    actual_eol: 2020-03-01
+    releases:
+      - version: 12.2.13
+        released: 2020-01-31
+      - version: 12.2.12
+        released: 2019-04-12
+      - version: 12.2.11
+        released: 2019-01-31
+      - version: 12.2.10
+        released: 2018-11-27
+      - version: 12.2.9
+        released: 2018-11-01
+      - version: 12.2.8
+        released: 2018-09-01
+      - version: 12.2.7
+        released: 2018-07-01
+      - version: 12.2.6
+        released: 2018-07-01
+      - version: 12.2.5
+        released: 2018-04-01
+      - version: 12.2.4
+        released: 2018-02-01
+      - version: 12.2.3
+        released: 2018-02-01
+      - version: 12.2.2
+        released: 2017-12-01
+      - version: 12.2.1
+        released: 2017-09-01
+      - version: 12.2.0
+        released: 2017-08-01
+
+  kraken:
+    target_eol: 2017-08-01
+    actual_eol: 2017-08-01
+    releases:
+      - version: 11.2.1
+        released: 2017-08-01
+      - version: 11.2.0
+        released: 2017-01-01
+
+  jewel:
+    target_eol: 2018-06-01
+    actual_eol: 2018-07-01
+    releases:
+      - version: 10.2.11
+        released: 2018-07-11
+      - version: 10.2.10
+        released: 2017-10-01
+      - version: 10.2.9
+        released: 2017-07-01
+      - version: 10.2.8
+        released: 2017-07-01
+      - version: 10.2.7
+        released: 2017-04-01
+      - version: 10.2.6
+        released: 2017-03-01
+      - version: 10.2.5
+        released: 2016-12-01
+      - version: 10.2.4
+        released: 2016-12-01
+      - version: 10.2.3
+        released: 2016-09-01
+      - version: 10.2.2
+        released: 2016-06-01
+      - version: 10.2.1
+        released: 2016-05-01
+      - version: 10.2.0
+        released: 2016-04-01
+
+  infernalis:
+    actual_eol: 2016-04-01
+    releases:
+      - version: 9.2.1
+        released: 2016-02-01
+      - version: 9.2.0
+        released: 2015-11-01
+
+  hammer:
+    target_eol: 2017-05-01
+    actual_eol: 2017-08-01
+    releases:
+      - version: 0.94.10
+        released: 2017-02-01
+      - version: 0.94.9
+        released: 2016-08-01
+      - version: 0.94.8
+        released: 2016-08-01
+      - version: 0.94.7
+        released: 2016-05-01
+      - version: 0.94.6
+        released: 2016-02-01
+      - version: 0.94.5
+        released: 2015-10-01
+      - version: 0.94.4
+        released: 2015-10-01
+      - version: 0.94.3
+        released: 2015-08-01
+      - version: 0.94.2
+        released: 2015-06-01
+      - version: 0.94.1
+        released: 2015-04-01
+      - version: '0.94'
+        released: 2015-04-01
+
+  giant:
+    actual_eol: 2015-04-01
+    releases:
+      - version: 0.87.2
+        released: 2015-04-01
+      - version: 0.87.1
+        released: 2015-02-01
+      - version: '0.87'
+        released: 2014-10-01
+
+  firefly:
+    target_eol: 2016-01-01
+    actual_eol: 2016-04-01
+    releases:
+      - version: 0.80.11
+        released: 2015-11-01
+      - version: 0.80.10
+        released: 2015-07-01
+      - version: 0.80.9
+        released: 2015-03-01
+      - version: 0.80.8
+        released: 2015-01-01
+      - version: 0.80.7
+        released: 2014-10-01
+      - version: 0.80.6
+        released: 2014-10-01
+      - version: 0.80.5
+        released: 2014-07-01
+      - version: 0.80.4
+        released: 2014-07-01
+      - version: 0.80.3
+        released: 2014-07-01
+      - version: 0.80.2
+        released: 2014-07-01
+      - version: 0.80.1
+        released: 2014-05-01
+      - version: '0.80'
+        released: 2014-05-01
+
+  emperor:
+    actual_eol: 2014-05-01
+    releases:
+      - version: 0.72.2
+        released: 2013-12-01
+      - version: 0.72.1
+        released: 2013-11-01
+      - version: '0.72'
+        released: 2013-11-01
+
+  dumpling:
+    target_eol: 2015-03-01
+    actual_eol: 2015-05-01
+    releases:
+      - version: 0.67.11
+        released: 2014-09-01
+      - version: 0.67.10
+        released: 2014-08-01
+      - version: 0.67.9
+        released: 2014-05-01
+      - version: 0.67.8
+        released: 2014-05-01
+      - version: 0.67.7
+        released: 2014-02-01
+      - version: 0.67.6
+        released: 2014-02-01
+      - version: 0.67.5
+        released: 2013-12-01
+      - version: 0.67.4
+        released: 2013-10-01
+      - version: 0.67.3
+        released: 2013-09-01
+      - version: 0.67.2
+        released: 2013-08-01
+      - version: 0.67.1
+        released: 2013-08-01
+      - version: '0.67'
+        released: 2013-08-01
+
+development:
+  releases:
+    - version: 15.1.1
+      released: 2020-03-13
+      skip_ref: true
+    - version: 15.1.0
+      released: 2020-01-29
+      skip_ref: true
+    - version: 15.0.0
+      released: 2019-04-03
+      skip_ref: true
+    - version: 14.1.1
+      released:  2019-03-11
+      skip_ref: true
+    - version: 14.1.0
+      released:  2019-02-22
+      skip_ref: true
+    - version: 14.0.1
+      released:  2018-11-05
+      skip_ref: true
+    - version: 14.0.0
+      released:  2018-05-06
+      skip_ref: true
+    - version: 13.1.0
+      released: 2018-05-01
+      skip_ref: true
+    - version: 13.0.2
+      released: 2018-04-01
+      skip_ref: true
+    - version: 13.0.1
+      released: 2018-02-01
+      skip_ref: true
+    - version: 13.0.0
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.4
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.3
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.2
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.1
+      released: 2017-07-01
+      skip_ref: true
+    - version: 12.1.0
+      released: 2017-06-01
+      skip_ref: true
+    - version: 12.0.3
+      released: 2017-05-01
+      skip_ref: true
+    - version: 12.0.2
+      released: 2017-04-01
+      skip_ref: true
+    - version: 12.0.1
+      released: 2017-03-01
+      skip_ref: true
+    - version: 12.0.0
+      released: 2017-02-01
+      skip_ref: true
+    - version: 11.1.1
+      released: 2017-01-01
+      skip_ref: true
+    - version: 11.1.0
+      released: 2016-12-01
+      skip_ref: true
+    - version: 11.0.2
+      released: 2016-10-01
+    - version: 11.0.1
+      released: 2016-10-01
+      skip_ref: true
+    - version: 11.0.0
+      released: 2016-06-01
+      skip_ref: true
+    - version: 10.1.2
+      released: 2016-04-01
+    - version: 10.1.1
+      released: 2016-04-01
+    - version: 10.1.0
+      released: 2016-03-01
+    - version: 10.0.5
+      released: 2016-03-01
+    - version: 10.0.4
+      released: 2016-03-01
+      skip_ref: true
+    - version: 10.0.3
+      released: 2016-02-01
+    - version: 10.0.2
+      released: 2016-01-01
+    - version: 10.0.1
+      released: 2015-12-01
+    - version: 10.0.0
+      released: 2015-11-01
+    - version: 9.1.0
+      released: 2015-10-01
+    - version: 9.0.3
+      released: 2015-08-01
+    - version: 9.0.2
+      released: 2015-07-01
+    - version: 9.0.1
+      released: 2015-06-01
+    - version: 9.0.0
+      released: 2015-05-01
+    - version: '0.93'
+      released: 2015-02-01
+    - version: '0.92'
+      released: 2015-02-01
+    - version: '0.91'
+      released: 2015-01-01
+    - version: '0.90'
+      released: 2014-12-01
+    - version: '0.89'
+      released: 2014-12-01
+    - version: '0.88'
+      released: 2014-11-01
+    - version: '0.86'
+      released: 2014-10-01
+    - version: '0.85'
+      released: 2014-09-01
+    - version: '0.84'
+      released: 2014-08-01
+    - version: '0.83'
+      released: 2014-07-01
+    - version: '0.82'
+      released: 2014-06-01
+    - version: '0.81'
+      released: 2014-06-01
+    - version: '0.79'
+      released: 2014-04-01
+    - version: '0.78'
+      released: 2014-03-01
+    - version: '0.77'
+      released: 2014-02-01
+    - version: '0.76'
+      released: 2014-01-01
+    - version: '0.75'
+      released: 2014-01-01
+    - version: '0.74'
+      released: 2013-12-01
+    - version: '0.73'
+      released: 2013-12-01
diff --git a/doc/scripts/README.md b/doc/scripts/README.md
new file mode 100644
index 000000000..e3d4c23e8
--- /dev/null
+++ b/doc/scripts/README.md
@@ -0,0 +1,11 @@
+Script Usage
+============
+
+Peering State Model: gen_state_diagram.py
+------------------------------------------
+    $ git clone https://github.com/ceph/ceph.git
+    $ cd ceph
+    $ cat src/osd/PeeringState.h src/osd/PeeringState.cc | doc/scripts/gen_state_diagram.py > doc/dev/peering_graph.generated.dot
+    $ sed -i 's/7,7/1080,1080/' doc/dev/peering_graph.generated.dot
+    $ dot -Tsvg doc/dev/peering_graph.generated.dot > doc/dev/peering_graph.generated.svg
+
diff --git a/doc/scripts/gen_state_diagram.py b/doc/scripts/gen_state_diagram.py
new file mode 100755
index 000000000..b084f8a74
--- /dev/null
+++ b/doc/scripts/gen_state_diagram.py
@@ -0,0 +1,242 @@
+#!/usr/bin/python3
+
+import itertools
+import re
+import sys
+
+
+def do_filter(generator):
+    return acc_lines(remove_multiline_comments(to_char(remove_single_line_comments(generator))))
+
+
+def acc_lines(generator):
+    current = ""
+    for i in generator:
+        current += i
+        if i == ';' or \
+            i == '{' or \
+            i == '}':
+            yield current.lstrip("\n")
+            current = ""
+
+
+def to_char(generator):
+    for line in generator:
+        for char in line:
+            if char != '\n':
+                yield char
+            else:
+                yield ' '
+
+
+def remove_single_line_comments(generator):
+    for i in generator:
+        if len(i) and i[0] == '#':
+            continue
+        yield re.sub(r'//.*', '', i)
+
+
+def remove_multiline_comments(generator):
+    saw = ""
+    in_comment = False
+    for char in generator:
+        if in_comment:
+            if saw == "*":
+                if char == "/":
+                    in_comment = False
+                saw = ""
+            if char == "*":
+                saw = "*"
+            continue
+        if saw == "/":
+            if char == '*':
+                in_comment = True
+                saw = ""
+                continue
+            else:
+                yield saw
+                saw = ""
+        if char == '/':
+            saw = "/"
+            continue
+        yield char
+
+
+class StateMachineRenderer(object):
+    def __init__(self):
+        self.states = {} # state -> parent
+        self.machines = {} # state-> initial
+        self.edges = {} # event -> [(state, state)]
+
+        self.context = [] # [(context, depth_encountered)]
+        self.context_depth = 0
+        self.state_contents = {}
+        self.subgraphnum = 0
+        self.clusterlabel = {}
+
+        self.color_palette = itertools.cycle([
+            "#000000",  # black
+            "#1e90ff",  # dodgerblue
+            "#ff0000",  # red
+            "#0000ff",  # blue
+            "#ffa500",  # orange
+            "#40e0d0",  # turquoise
+            "#c71585",  # mediumvioletred
+        ])
+
+    def __str__(self):
+        return f'''-------------------
+
+ states: {self.states}
+
+ machines: {self.machines}
+
+ edges: {self.edges}
+
+ context: {self.context}
+
+ state_contents: {self.state_contents}
+
+--------------------'''
+
+    def read_input(self, input_lines):
+        previous_line = None
+        for line in input_lines:
+            self.get_state(line)
+            self.get_event(line)
+            # pass two lines at a time to get the context so that regexes can
+            # match on split signatures
+            self.get_context(line, previous_line)
+            previous_line = line
+
+    def get_context(self, line, previous_line):
+        match = re.search(r"(\w+::)*::(?P<tag>\w+)::\w+\(const (?P<event>\w+)", line)
+        if match is None and previous_line is not None:
+            # it is possible that we need to match on the previous line as well, so join
+            # them to make them one line and try and get this matching
+            joined_line = ' '.join([previous_line, line])
+            match = re.search(r"(\w+::)*::(?P<tag>\w+)::\w+\(\s*const (?P<event>\w+)", joined_line)
+        if match is not None:
+            self.context.append((match.group('tag'), self.context_depth, match.group('event')))
+        if '{' in line:
+            self.context_depth += 1
+        if '}' in line:
+            self.context_depth -= 1
+            while len(self.context) and self.context[-1][1] == self.context_depth:
+                self.context.pop()
+
+    def get_state(self, line):
+        if "boost::statechart::state_machine" in line:
+            tokens = re.search(
+                r"boost::statechart::state_machine<\s*(\w*),\s*(\w*)\s*>",
+                line)
+            if tokens is None:
+                raise Exception("Error: malformed state_machine line: " + line)
+            self.machines[tokens.group(1)] = tokens.group(2)
+            self.context.append((tokens.group(1), self.context_depth, ""))
+            return
+        if "boost::statechart::state" in line:
+            tokens = re.search(
+                r"boost::statechart::state<\s*(\w*),\s*(\w*)\s*,?\s*(\w*)\s*>",
+                line)
+            if tokens is None:
+                raise Exception("Error: malformed state line: " + line)
+            self.states[tokens.group(1)] = tokens.group(2)
+            if tokens.group(2) not in self.state_contents.keys():
+                self.state_contents[tokens.group(2)] = []
+            self.state_contents[tokens.group(2)].append(tokens.group(1))
+            if tokens.group(3):
+                self.machines[tokens.group(1)] = tokens.group(3)
+            self.context.append((tokens.group(1), self.context_depth, ""))
+            return
+
+    def get_event(self, line):
+        if "boost::statechart::transition" in line:
+            for i in re.finditer(r'boost::statechart::transition<\s*([\w:]*)\s*,\s*(\w*)\s*>',
+                                 line):
+                if i.group(1) not in self.edges.keys():
+                    self.edges[i.group(1)] = []
+                if not self.context:
+                    raise Exception("no context at line: " + line)
+                self.edges[i.group(1)].append((self.context[-1][0], i.group(2)))
+        i = re.search("return\s+transit<\s*(\w*)\s*>()", line)
+        if i is not None:
+            if not self.context:
+                raise Exception("no context at line: " + line)
+            if not self.context[-1][2]:
+                raise Exception("no event in context at line: " + line)
+            if self.context[-1][2] not in self.edges.keys():
+                self.edges[self.context[-1][2]] = []
+            self.edges[self.context[-1][2]].append((self.context[-1][0], i.group(1)))
+
+    def emit_dot(self, output):
+        top_level = []
+        for state in self.machines.keys():
+            if state not in self.states.keys():
+                top_level.append(state)
+        print('Top Level States: ', top_level, file=sys.stderr)
+        print('digraph G {', file=output)
+        print('\tsize="7,7"', file=output)
+        print('\tcompound=true;', file=output)
+        for i in self.emit_state(top_level[0]):
+            print('\t' + i, file=output)
+        for i in self.edges.keys():
+            for j in self.emit_event(i):
+                print(j, file=output)
+        print('}', file=output)
+
+    def emit_state(self, state):
+        if state in self.state_contents.keys():
+            self.clusterlabel[state] = "cluster%s" % (str(self.subgraphnum),)
+            yield "subgraph cluster%s {" % (str(self.subgraphnum),)
+            self.subgraphnum += 1
+            yield """\tlabel = "%s";""" % (state,)
+            yield """\tcolor = "black";"""
+
+            if state in self.machines.values():
+                yield """\tstyle = "filled";"""
+                yield """\tfillcolor = "lightgrey";"""
+
+            for j in self.state_contents[state]:
+                for i in self.emit_state(j):
+                    yield "\t"+i
+            yield "}"
+        else:
+            found = False
+            for (k, v) in self.machines.items():
+                if v == state:
+                    yield state+"[shape=Mdiamond style=filled fillcolor=lightgrey];"
+                    found = True
+                    break
+            if not found:
+                yield state+";"
+
+    def emit_event(self, event):
+        def append(app):
+            retval = "["
+            for i in app:
+                retval += (i + ",")
+            retval += "]"
+            return retval
+
+        for (fro, to) in self.edges[event]:
+            color = next(self.color_palette)
+            appendix = ['label="%s"' % (event,),
+                        'color="%s"' % (color,),
+                        'fontcolor="%s"' % (color,)]
+            if fro in self.machines.keys():
+                appendix.append("ltail=%s" % (self.clusterlabel[fro],))
+                while fro in self.machines.keys():
+                    fro = self.machines[fro]
+            if to in self.machines.keys():
+                appendix.append("lhead=%s" % (self.clusterlabel[to],))
+                while to in self.machines.keys():
+                    to = self.machines[to]
+            yield("%s -> %s %s;" % (fro, to, append(appendix)))
+
+
+if __name__ == '__main__':
+    INPUT_GENERATOR = do_filter(line for line in sys.stdin)
+    RENDERER = StateMachineRenderer()
+    RENDERER.read_input(INPUT_GENERATOR)
+    RENDERER.emit_dot(output=sys.stdout)
diff --git a/doc/security/CVE-2021-20288.rst b/doc/security/CVE-2021-20288.rst
new file mode 100644
index 000000000..fa3b073cb
--- /dev/null
+++ b/doc/security/CVE-2021-20288.rst
@@ -0,0 +1,183 @@
+.. _CVE-2021-20288:
+
+CVE-2021-20288: Unauthorized global_id reuse in cephx
+=====================================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-20288>`_
+
+Summary
+-------
+
+Ceph was not ensuring that reconnecting/renewing clients were
+presenting an existing ticket when reclaiming their global_id value.
+An attacker that was able to authenticate could claim a global_id in
+use by a different client and potentially disrupt
+other cluster services.
+
+Background
+----------
+
+Each authenticated client or daemon in Ceph is assigned a numeric
+global_id identifier. That value is assumed to be unique across the
+cluster.  When clients reconnect to the monitor (e.g., due to a
+network disconnection) or renew their ticket, they are supposed to
+present their old ticket to prove prior possession of their global_id
+so that it can be reclaimed and thus remain constant over the lifetime
+of that client instance.
+
+Ceph was not correctly checking that the old ticket was valid, allowing
+an arbitrary global_id to be reclaimed, even if it was in use by another
+active client in the system.
+
+Attacker Requirements
+---------------------
+
+Any potential attacker must:
+
+* have a valid authentication key for the cluster
+* know or guess the global_id of another client
+* run a modified version of the Ceph client code to reclaim another client's global_id
+* construct appropriate client messages or requests to disrupt service or exploit
+  Ceph daemon assumptions about global_id uniqueness
+
+Impact
+------
+
+Confidentiality Impact
+______________________
+
+None
+
+Integrity Impact
+________________
+
+Partial.  An attacker could potentially exploit assumptions around
+global_id uniqueness to disrupt other clients' access or disrupt
+Ceph daemons.
+
+Availability Impact
+___________________
+
+High.  An attacker could potentially exploit assumptions around
+global_id uniqueness to disrupt other clients' access or disrupt
+Ceph daemons.
+
+Access Complexity
+_________________
+
+High.  The client must make use of modified client code in order to
+exploit specific assumptions in the behavior of other Ceph daemons.
+
+Authentication
+______________
+
+Yes.  The attacker must also be authenticated and have access to the
+same services as a client it is wishing to impersonate or disrupt.
+
+Gained Access
+_____________
+
+Partial.  An attacker can partially impersonate another client.
+
+Affected versions
+-----------------
+
+All prior versions of Ceph monitors fail to ensure that global_id reclaim
+attempts are authentic.
+
+In addition, all user-space daemons and clients starting from Luminous v12.2.0
+were failing to securely reclaim their global_id following commit a2eb6ae3fb57
+("mon/monclient: hunt for multiple monitor in parallel").
+
+All versions of the Linux kernel client properly authenticate.
+
+Fixed versions
+--------------
+
+* Pacific v16.2.1 (and later)
+* Octopus v15.2.11 (and later)
+* Nautilus v14.2.20 (and later)
+
+
+Fix details
+-----------
+
+#. Patched monitors now properly require that clients securely reclaim
+   their global_id when the ``auth_allow_insecure_global_id_reclaim``
+   is ``false``.  Initially, by default, this option is set to
+   ``true`` so that existing clients can continue to function without
+   disruption until all clients have been upgraded.  When this option
+   is set to false, then an unpatched client will not be able to reconnect
+   to the cluster after an intermittent network disruption breaking
+   its connect to a monitor, or be able to renew its authentication
+   ticket when it times out (by default, after 72 hours).
+
+   Patched monitors raise the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED``
+   health alert if ``auth_allow_insecure_global_id_reclaim`` is enabled.
+   This health alert can be muted with::
+
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w
+
+   Although it is not recommended, the alert can also be disabled with::
+
+     ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
+
+#. Patched monitors can disconnect new clients right after they have
+   authenticated (forcing them to reconnect and reclaim) in order to
+   determine whether they securely reclaim global_ids.  This allows
+   the cluster and users to discover quickly whether clients would be
+   affected by requiring secure global_id reclaim: most clients will
+   report an authentication error immediately.  This behavior can be
+   disabled by setting ``auth_expose_insecure_global_id_reclaim`` to
+   ``false``::
+
+     ceph config set mon auth_expose_insecure_global_id_reclaim false
+
+#. Patched monitors will raise the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` health
+   alert for any clients or daemons that are not securely reclaiming their
+   global_id.  These clients should be upgraded before disabling the
+   ``auth_allow_insecure_global_id_reclaim`` option to avoid disrupting
+   client access.
+
+   By default (if ``auth_expose_insecure_global_id_reclaim`` has not
+   been disabled), clients' failure to securely reclaim global_id will
+   immediately be exposed and raise this health alert.
+   However, if ``auth_expose_insecure_global_id_reclaim`` has been
+   disabled, this alert will not be triggered for a client until it is
+   forced to reconnect to a monitor (e.g., due to a network disruption)
+   or the client renews its authentication ticket (by default, after
+   72 hours).
+
+#. The default time-to-live (TTL) for authentication tickets has been increased
+   from 12 hours to 72 hours.  Because we previously were not ensuring that
+   a client's prior ticket was valid when reclaiming their global_id, a client
+   could tolerate a network outage that lasted longer than the ticket TTL and still
+   reclaim its global_id.  Once the cluster starts requiring secure global_id reclaim,
+   a client that is disconnected for longer than the TTL may fail to reclaim its global_id,
+   fail to reauthenticate, and be unable to continue communicating with the cluster
+   until it is restarted.  The default TTL was increased to minimize the impact of this
+   change on users.
+
+
+Recommendations
+---------------
+
+#. Users should upgrade to a patched version of Ceph at their earliest
+   convenience.
+
+#. Users should upgrade any unpatched clients at their earliest
+   convenience.  By default, these clients can be easily identified by
+   checking the ``ceph health detail`` output for the
+   ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` alert.
+
+#. If all clients cannot be upgraded immediately, the health alerts can be
+   temporarily muted with::
+
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1w  # 1 week
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w  # 1 week
+
+#. After all clients have been updated and the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM``
+   alert is no longer present, the cluster should be set to prevent insecure
+   global_id reclaim with::
+
+     ceph config set mon auth_allow_insecure_global_id_reclaim false
diff --git a/doc/security/CVE-2021-3509.rst b/doc/security/CVE-2021-3509.rst
new file mode 100644
index 000000000..7e865e9b2
--- /dev/null
+++ b/doc/security/CVE-2021-3509.rst
@@ -0,0 +1,28 @@
+.. _CVE-2021-3509:
+
+CVE-2021-3509: Dashboard XSS via token cookie
+=============================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3509>`_
+
+The Ceph Dashboard was vulnerable to an XSS attack that could expose the authentication
+cookie to other sites.
+
+
+Affected versions
+-----------------
+
+* Octopus v15.2.0 and later
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+
+Recommendations
+---------------
+
+All users of the Ceph dashboard should upgrade.
diff --git a/doc/security/CVE-2021-3524.rst b/doc/security/CVE-2021-3524.rst
new file mode 100644
index 000000000..4d627c071
--- /dev/null
+++ b/doc/security/CVE-2021-3524.rst
@@ -0,0 +1,30 @@
+.. _CVE-2021-3524:
+
+CVE-2021-3524: HTTP header injects via CORS in RGW
+==================================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3524>`_
+
+A flaw was found in the radosgw.  The vulnerability is related to the
+injection of HTTP headers via a CORS ExposeHeader tag. The \r
+character in the ExposeHeader tag in the CORS configuration file
+generates a header injection in the response when the CORS request is
+made.
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+Recommendations
+---------------
+
+All users of Ceph object storage (RGW) should upgrade.
+
+Acknowledgements
+----------------
+
+Red Hat would like to thank Sergey Bobrov (Kaspersky) for reporting this issue.
+
diff --git a/doc/security/CVE-2021-3531.rst b/doc/security/CVE-2021-3531.rst
new file mode 100644
index 000000000..907cb4764
--- /dev/null
+++ b/doc/security/CVE-2021-3531.rst
@@ -0,0 +1,28 @@
+.. _CVE-2021-3531:
+
+CVE-2021-3531: Swift API denial of service
+==========================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3531>`_
+
+Unauthenticated users of the Swift API can trigger a server-side assertion with a
+malformed URL, leading to a denial of service.
+
+
+Affected versions
+-----------------
+
+* Nautilus v14.2.0 and later
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+
+Recommendations
+---------------
+
+All users of Ceph object storage (RGW) should upgrade.
diff --git a/doc/security/CVE-2022-0670.rst b/doc/security/CVE-2022-0670.rst
new file mode 100644
index 000000000..f2b90de2e
--- /dev/null
+++ b/doc/security/CVE-2022-0670.rst
@@ -0,0 +1,43 @@
+.. _CVE-2022-0670:
+
+CVE-2022-0670: Native-CephFS Manila Path-restriction bypass
+===========================================================
+
+Summary
+-------
+
+Users who were running OpenStack Manila to export native CephFS and who
+upgraded their Ceph cluster from Nautilus (or earlier) to a later
+major version were vulnerable to an attack by malicious users. The
+vulnerability allowed users to obtain access to arbitrary portions of
+the CephFS filesystem hierarchy instead of being properly restricted
+to their own subvolumes. The vulnerability is due to a bug in the
+"volumes" plugin in Ceph Manager. This plugin is responsible for
+managing Ceph File System subvolumes, which are used by OpenStack
+Manila services as a way to provide shares to Manila users.
+
+Again, this vulnerability impacts only OpenStack Manila clusters that 
+provided native CephFS access to their users.
+
+Affected versions
+-----------------
+
+Any version of Ceph running OpenStack Manila that was upgraded from Nautilus
+or earlier.
+
+Fixed versions
+--------------
+
+* Quincy v17.2.2 (and later)
+* Pacific v16.2.10 (and later)
+* Octopus v15.2.17
+
+Recommendations
+---------------
+
+#. Users should upgrade to a patched version of Ceph at their earliest
+   convenience.
+
+#. Administrators who are
+   concerned they may have been impacted should audit the CephX keys in
+   their cluster for proper path restrictions.
diff --git a/doc/security/cves.rst b/doc/security/cves.rst
new file mode 100644
index 000000000..8bbccbf64
--- /dev/null
+++ b/doc/security/cves.rst
@@ -0,0 +1,114 @@
+
+Past vulnerabilities
+====================
+
++------------+-------------------+-------------+---------------------------------------------+
+| Published  | CVE               | Severity    | Summary                                     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2022-07-21 | `CVE-2022-0670`_  | Medium      | Native-CephFS Manila Path-restriction bypass|
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3531`_  | Medium      | Swift API denial of service                 |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3524`_  | Medium      | HTTP header injects via CORS in RGW         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3509`_  | High        | Dashboard XSS via token cookie              |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-04-14 | `CVE-2021-20288`_ | High        | Unauthorized global_id reuse in cephx       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-12-18 | `CVE-2020-27781`_ | 7.1 High    | CephFS creds read/modified by Manila users  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-01-08 | `CVE-2020-25678`_ | 4.9 Medium  | mgr module passwords in clear text          |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-12-07 | `CVE-2020-25677`_ | 5.5 Medium  | ceph-ansible iscsi-gateway.conf perm        |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-11-23 | `CVE-2020-25660`_ | 8.8 High    | Cephx replay vulnerability                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-22 | `CVE-2020-12059`_ | 7.5 High    | malformed POST could crash RGW              |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-06-26 | `CVE-2020-10753`_ | 6.5 Medium  | HTTP header injects via CORS in RGW         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-06-22 | `CVE-2020-10736`_ | 8.0 High    | authorization bypass in mon and mgr         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-23 | `CVE-2020-1760`_  | 6.1 Medium  | potential RGW XSS attack                    |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-13 | `CVE-2020-1759`_  | 6.8 Medium  | Cephx nonce reuse in secure mode            |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-02-07 | `CVE-2020-1700`_  | 6.5 Medium  | RGW disconnects leak sockets, can DoS       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-21 | `CVE-2020-1699`_  | 7.5 High    | Dashboard path traversal flaw               |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-12-23 | `CVE-2019-19337`_ | 6.5 Medium  | RGW DoS via malformed headers               |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-11-08 | `CVE-2019-10222`_ | 7.5 High    | Invalid HTTP headers could crash RGW        |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-03-27 | `CVE-2019-3821`_  | 7.5 High    | RGW file descriptors could be exhausted     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-28 | `CVE-2018-16889`_ | 7.5 High    | encryption keys logged in plaintext         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-15 | `CVE-2018-16846`_ | 6.5 Medium  | authenticated RGW users can cause DoS       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-15 | `CVE-2018-14662`_ | 5.7 Medium  | read-only users could steal dm-crypt keys   |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-10861`_ | 8.1 High    | authenticated user can create/delete pools  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-03-19 | `CVE-2018-7262`_  | 7.5 High    | malformed headers can cause RGW DoS         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-1129`_  | 6.5 Medium  | network MITM can tamper with messages       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-1128`_  | 7.5 High    | Cephx replay vulnerability                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-27 | `CVE-2017-7519`_  | 4.4 Medium  | libradosstriper unvalidated format string   |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-08-01 | `CVE-2016-9579`_  | 7.6 High    | potential RGW XSS attack                    |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-31 | `CVE-2016-8626`_  | 6.5 Medium  | malformed POST can DoS RGW                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-10-03 | `CVE-2016-7031`_  | 7.5 High    | RGW unauthorized bucket listing             |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-07-12 | `CVE-2016-5009`_  | 6.5 Medium  | mon command handler DoS                     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-12-03 | `CVE-2015-5245`_  |             | RGW header injection                        |
++------------+-------------------+-------------+---------------------------------------------+
+
+.. toctree::
+   :hidden:
+   :maxdepth: 0
+
+    CVE-2022-0670 <CVE-2022-0670.rst>
+    CVE-2021-3531 <CVE-2021-3531.rst>
+    CVE-2021-3524 <CVE-2021-3524.rst>
+    CVE-2021-3509 <CVE-2021-3509.rst>
+    CVE-2021-20288 <CVE-2021-20288.rst>
+
+.. _CVE-2022-0670: ../CVE-2022-0670
+.. _CVE-2021-3531: ../CVE-2021-3531
+.. _CVE-2021-3524: ../CVE-2021-3524
+.. _CVE-2021-3509: ../CVE-2021-3509
+.. _CVE-2021-20288: ../CVE-2021-20288
+.. _CVE-2020-27781: https://nvd.nist.gov/vuln/detail/CVE-2020-27781
+.. _CVE-2020-25678: https://nvd.nist.gov/vuln/detail/CVE-2020-25678
+.. _CVE-2020-25677: https://nvd.nist.gov/vuln/detail/CVE-2020-25677
+.. _CVE-2020-25660: https://nvd.nist.gov/vuln/detail/CVE-2020-25660
+.. _CVE-2020-12059: https://nvd.nist.gov/vuln/detail/CVE-2020-12059
+.. _CVE-2020-10753: https://nvd.nist.gov/vuln/detail/CVE-2020-10753
+.. _CVE-2020-10736: https://nvd.nist.gov/vuln/detail/CVE-2020-10736
+.. _CVE-2020-1760: https://nvd.nist.gov/vuln/detail/CVE-2020-1760
+.. _CVE-2020-1759: https://nvd.nist.gov/vuln/detail/CVE-2020-1759
+.. _CVE-2020-1700: https://nvd.nist.gov/vuln/detail/CVE-2020-1700
+.. _CVE-2020-1699: https://nvd.nist.gov/vuln/detail/CVE-2020-1699
+.. _CVE-2019-19337: https://nvd.nist.gov/vuln/detail/CVE-2019-19337
+.. _CVE-2019-10222: https://nvd.nist.gov/vuln/detail/CVE-2019-10222
+.. _CVE-2019-3821: https://nvd.nist.gov/vuln/detail/CVE-2019-3821
+.. _CVE-2018-16889: https://nvd.nist.gov/vuln/detail/CVE-2018-16889
+.. _CVE-2018-16846: https://nvd.nist.gov/vuln/detail/CVE-2018-16846
+.. _CVE-2018-14662: https://nvd.nist.gov/vuln/detail/CVE-2018-14662
+.. _CVE-2018-10861: https://nvd.nist.gov/vuln/detail/CVE-2018-10861
+.. _CVE-2018-7262: https://nvd.nist.gov/vuln/detail/CVE-2018-7262
+.. _CVE-2018-1129: https://nvd.nist.gov/vuln/detail/CVE-2018-1129
+.. _CVE-2018-1128: https://nvd.nist.gov/vuln/detail/CVE-2018-1128
+.. _CVE-2017-7519: https://nvd.nist.gov/vuln/detail/CVE-2017-7519
+.. _CVE-2016-9579: https://nvd.nist.gov/vuln/detail/CVE-2016-9579
+.. _CVE-2016-8626: https://nvd.nist.gov/vuln/detail/CVE-2016-8626
+.. _CVE-2016-7031: https://nvd.nist.gov/vuln/detail/CVE-2016-7031
+.. _CVE-2016-5009: https://nvd.nist.gov/vuln/detail/CVE-2016-5009
+.. _CVE-2015-5245: https://nvd.nist.gov/vuln/detail/CVE-2015-5245
diff --git a/doc/security/index.rst b/doc/security/index.rst
new file mode 100644
index 000000000..9d158cb96
--- /dev/null
+++ b/doc/security/index.rst
@@ -0,0 +1,42 @@
+==========
+ Security
+==========
+
+.. toctree::
+   :maxdepth: 1
+
+   Past Vulnerabilities / CVEs <cves>
+   Vulnerability Management Process <process>
+
+Reporting a vulnerability
+=========================
+
+To report a vulnerability, please send email to `security@ceph.io
+<security@ceph.io>`_.
+
+* Please do not file a public ceph tracker issue for a vulnerability.
+* We urge reporters to provide as much information as is practicable
+  (a reproducer, versions affected, fix if available, etc.), as this
+  can speed up the process considerably.
+* Please let us know to whom credit should be given and with what
+  affiliations.
+* If this issue is not yet disclosed publicly and you have any
+  disclosure date in mind, please share the same along with the
+  report.
+
+Although you are not required to, you may encrypt your message using 
+the following GPG key:
+
+**6EEF26FFD4093B99: Ceph Security Team (security@ceph.io)**
+
+| **Download:** `MIT PGP Public Key Server <https://pgp.mit.edu/pks/lookup?op=vindex&search=0x6EEF26FFD4093B99>`_
+| **Fingerprint:** A527 D019 21F9 7178 C232 66C1 6EEF 26FF D409 3B99
+
+
+Supported versions
+==================
+
+Security updates are applied only to the current `Active Releases`_.
+
+
+.. _Active Releases: https://docs.ceph.com/en/latest/releases/#active-releases
diff --git a/doc/security/process.rst b/doc/security/process.rst
new file mode 100644
index 000000000..c8a2031ca
--- /dev/null
+++ b/doc/security/process.rst
@@ -0,0 +1,48 @@
+Vulnerability Management Process
+================================
+
+#. The report will be acknowledged within three business days.
+#. The team will investigate the reported issue and will update the email
+   thread with relevant information. The team may ask for additional
+   information regarding the reported issue.
+#. If the team does not confirm the report, no further action will be
+   taken and the issue will be closed.
+#. If the report is confirmed by Ceph team members, a unique CVE identifier
+   will be assigned to the report and then shared with the reporter. The Ceph
+   security team will start working on a fix. 
+#. If a reporter has no disclosure date in mind, a Ceph security team
+   member will coordinate a release date (CRD) with the list members
+   and share the mutually agreed disclosure date with the reporter.
+#. The vulnerability disclosure / release date is set excluding Friday and
+   holiday periods.
+#. Embargoes are preferred for Critical and High impact
+   issues. Embargo should not be held for more than 90 days from the
+   date of vulnerability confirmation, except under unusual
+   circumstances. For Low and Moderate issues with limited impact and
+   an easy workaround or where an issue that is already public, a
+   standard patch release process will be followed to fix the
+   vulnerability once CVE is assigned.
+#. Fixes for issues of "Medium" and "Low" severity will be released as part of
+   the next standard release cycle. List members will receive seven days of
+   advance notice prior to the release date of these fixes. The details of the
+   CVE fix will be included in the release notes, and the release notes will be
+   linked in the public announcement.
+#. Commits will be handled in a private repository for review and
+   testing and a new patch version will be released from this private
+   repository.
+#. If a vulnerability is unintentionally already fixed in the public
+   repository, a few days are given to downstream stakeholders/vendors
+   to prepare for updating before the public disclosure.
+#. An announcement will be made disclosing the vulnerability. The
+   fastest place to receive security announcements is via the
+   `ceph-announce@ceph.io <ceph-announce@ceph.io>`_ or
+   `oss-security@lists.openwall.com <oss-security@lists.openwall.com>`_ mailing
+   lists.  (These lists are low-traffic).
+
+If the report is considered embargoed, we ask you to not disclose the
+vulnerability before it has been fixed and announced, unless you
+received a response from the Ceph security team that you can do
+so. This holds true until the public disclosure date that was agreed
+upon by the list. Thank you for improving the security of Ceph and its
+ecosystem. Your efforts and responsible disclosure are greatly
+appreciated and will be acknowledged.
diff --git a/doc/start/ceph.conf b/doc/start/ceph.conf
new file mode 100644
index 000000000..f3d558eb0
--- /dev/null
+++ b/doc/start/ceph.conf
@@ -0,0 +1,3 @@
+[global]
+	# list your monitors here
+	mon host = {mon-host-1}, {mon-host-2}
diff --git a/doc/start/documenting-ceph.rst b/doc/start/documenting-ceph.rst
new file mode 100644
index 000000000..02d4dccc4
--- /dev/null
+++ b/doc/start/documenting-ceph.rst
@@ -0,0 +1,1085 @@
+.. _documenting_ceph:
+
+==================
+ Documenting Ceph
+==================
+
+You can help the Ceph project by contributing to the documentation.  Even
+small contributions help the Ceph project. 
+
+The easiest way to suggest a correction to the documentation is to send an
+email to `ceph-users@ceph.io`. Include the string "ATTN: DOCS" or
+"Attention: Docs" or "Attention: Documentation" in the subject line.  In
+the body of the email, include the text to be corrected (so that I can find
+it in the repo) and include your correction.
+
+Another way to suggest a documentation correction is to make a pull request.
+The instructions for making a pull request against the Ceph documentation are
+in the section :ref:`making_contributions`.
+
+If this is your first time making an improvement to the documentation or
+if you have noticed a small mistake (such as a spelling error or a typo),
+it will be easier to send an email than to make a pull request. You will
+be credited for the improvement unless you instruct Ceph Upstream
+Documentation not to credit you.
+
+Location of the Documentation in the Repository
+===============================================
+
+The Ceph documentation source is in the ``ceph/doc`` directory of the Ceph
+repository. Python Sphinx renders the source into HTML and manpages. 
+
+Viewing Old Ceph Documentation
+==============================
+The https://docs.ceph.com link displays the latest release branch by default
+(for example, if "Quincy" is the most recent release, then by default
+https://docs.ceph.com displays the documentation for Quincy), but you can view
+the documentation for older versions of Ceph (for example, ``pacific``) by
+replacing the version name in the url (for example, ``quincy`` in
+`https://docs.ceph.com/en/pacific <https://docs.ceph.com/en/quincy>`_) with the
+branch name you prefer (for example, ``pacific``, to create a URL that reads
+`https://docs.ceph.com/en/pacific/ <https://docs.ceph.com/en/pacific/>`_).
+
+.. _making_contributions:
+
+Making Contributions
+====================
+
+Making a documentation contribution involves the same basic procedure as making
+a code contribution, with one exception: you must build documentation source
+instead of compiling program source. This sequence (the sequence of building
+the documentation source) includes the following steps:
+
+#. `Get the Source`_
+#. `Select a Branch`_
+#. `Make a Change`_
+#. `Build the Source`_
+#. `Commit the Change`_
+#. `Push the Change`_
+#. `Make a Pull Request`_
+#. `Notify Us`_
+
+Get the Source
+--------------
+
+The source of the Ceph documentation is a collection of ReStructured Text files
+that are in the Ceph repository in the ``ceph/doc`` directory. For details
+on GitHub and Ceph, see :ref:`Get Involved`.
+
+Use the `Fork and Pull`_ approach to make documentation contributions. To do
+this, you must:
+
+#. Install git locally. In Debian or Ubuntu, run the following command:
+
+   .. prompt:: bash $
+
+	sudo apt-get install git
+
+   In Fedora, run the following command:
+
+   .. prompt:: bash $
+
+	sudo yum install git
+
+   In CentOS/RHEL, run the following command:
+
+   .. prompt:: bash $
+
+	sudo yum install git
+
+#. Make sure that your ``.gitconfig`` file has been configured to include your
+   name and email address:
+
+   .. code-block:: ini
+
+	[user]
+	   email = {your-email-address}
+	   name = {your-name}
+
+   For example:
+
+   .. prompt:: bash $
+
+	git config --global user.name "John Doe"
+	git config --global user.email johndoe@example.com
+
+
+#. Create a  `github`_ account (if you don't have one).
+
+#. Fork the Ceph project. See https://github.com/ceph/ceph.
+
+#. Clone your fork of the Ceph project to your local host. This creates what is
+   known as a "local working copy".
+
+The Ceph documentation is organized by component:
+
+- **Ceph Storage Cluster:** The Ceph Storage Cluster documentation is 
+  in the ``doc/rados`` directory.
+  
+- **Ceph Block Device:** The Ceph Block Device documentation is in 
+  the ``doc/rbd`` directory.
+  
+- **Ceph Object Storage:** The Ceph Object Storage documentation is in 
+  the ``doc/radosgw`` directory.
+
+- **Ceph File System:** The Ceph File System documentation is in the 
+  ``doc/cephfs`` directory.
+  
+- **Installation (Quick):** Quick start documentation is in the
+  ``doc/start`` directory.
+  
+- **Installation (Manual):** Documentaton concerning the manual installation of
+  Ceph is in the ``doc/install`` directory.
+  
+- **Manpage:** Manpage source is in the ``doc/man`` directory.
+
+- **Developer:** Developer documentation is in the ``doc/dev`` 
+  directory.
+
+- **Images:** Images including JPEG and PNG files are stored in the
+  ``doc/images`` directory.
+
+
+Select a Branch
+---------------
+
+When you make small changes to the documentation, such as fixing typographical
+errors or clarifying explanations, use the ``main`` branch (default). You
+should also use the ``main`` branch when making contributions to features that
+are in the current release. ``main`` is the most commonly used branch. :
+
+.. prompt:: bash $
+
+	git checkout main
+
+When you make changes to documentation that affect an upcoming release, use 
+the ``next`` branch. ``next`` is the second most commonly used branch. :
+
+.. prompt:: bash $
+
+	git checkout next
+
+When you are making substantial contributions such as new features that are not
+yet in the current release; if your contribution is related to an issue with a
+tracker ID; or, if you want to see your documentation rendered on the Ceph.com
+website before it gets merged into the ``main`` branch, you should create a
+branch. To distinguish branches that include only documentation updates, we
+prepend them with ``wip-doc`` by convention, following the form
+``wip-doc-{your-branch-name}``. If the branch relates to an issue filed in
+http://tracker.ceph.com/issues, the branch name incorporates the issue number.
+For example, if a documentation branch is a fix for issue #4000, the branch name
+should be ``wip-doc-4000`` by convention and the relevant tracker URL will be
+http://tracker.ceph.com/issues/4000.
+
+.. note:: Please do not mingle documentation contributions and source code
+   contributions in a single commit. When you keep documentation
+   commits separate from source code commits, it simplifies the review
+   process. We highly recommend that any pull request that adds a feature or
+   a configuration option should also include a documentation commit that
+   describes the changes.
+
+Before you create your branch name, ensure that it doesn't already exist in the
+local or remote repository. :
+
+.. prompt:: bash $
+
+	git branch -a | grep wip-doc-{your-branch-name}
+
+If it doesn't exist, create your branch:
+
+.. prompt:: bash $
+
+	git checkout -b wip-doc-{your-branch-name}
+
+
+Make a Change
+-------------
+
+Modifying a document involves opening a reStructuredText file, changing
+its contents, and saving the changes. See `Documentation Style Guide`_ for
+details on syntax requirements.
+
+Adding a document involves creating a new reStructuredText file within the
+``doc`` directory tree with a ``*.rst``
+extension. You must also include a reference to the document: a hyperlink
+or a table of contents entry. The ``index.rst`` file of a top-level directory
+usually contains a TOC, where you can add the new file name. All documents must
+have a title. See `Headings`_ for details.
+
+Your new document doesn't get tracked by ``git`` automatically. When you want 
+to add the document to the repository,  you must use ``git add 
+{path-to-filename}``. For example, from the top level  directory of the
+repository, adding an ``example.rst`` file to the ``rados`` subdirectory would
+look like this:
+
+.. prompt:: bash $
+
+	git add doc/rados/example.rst
+
+Deleting a document involves removing it from the repository with ``git rm
+{path-to-filename}``. For example:
+
+.. prompt:: bash $
+
+	git rm doc/rados/example.rst
+
+You must also remove any reference to a deleted document from other documents.
+
+
+Build the Source
+----------------
+
+To build the documentation, navigate to the ``ceph`` repository directory:
+
+
+.. prompt:: bash $
+
+	cd ceph
+
+.. note::
+   The directory that contains ``build-doc`` and ``serve-doc`` must be included
+   in the ``PATH`` environment variable in order for these commands to work.
+
+
+To build the documentation on Debian/Ubuntu, Fedora, or CentOS/RHEL, execute:
+
+.. prompt:: bash $
+
+	admin/build-doc
+
+To scan for the reachability of external links, execute:
+
+.. prompt:: bash $
+
+	admin/build-doc linkcheck
+
+Executing ``admin/build-doc`` will create a ``build-doc`` directory under
+``ceph``.  You may need to create a directory under ``ceph/build-doc`` for
+output of Javadoc files:
+
+.. prompt:: bash $
+
+	mkdir -p output/html/api/libcephfs-java/javadoc
+
+The build script ``build-doc`` will produce an output of errors and warnings.
+You MUST fix errors in documents you modified before committing a change, and
+you SHOULD fix warnings that are related to syntax you modified.
+
+.. important:: You must validate ALL HYPERLINKS. If a hyperlink is broken,
+   it automatically breaks the build!
+
+Once you build the documentation set, you may start an HTTP server at
+``http://localhost:8080/`` to view it:
+
+.. prompt:: bash $
+
+	admin/serve-doc
+
+You can also navigate to ``build-doc/output`` to inspect the built documents.
+There should be an ``html`` directory and a ``man`` directory containing
+documentation in HTML and manpage formats respectively.
+
+Build the Source (First Time)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ceph uses Python Sphinx, which is generally distribution agnostic. The first
+time you build Ceph documentation, it will generate a doxygen XML tree, which
+is a bit time consuming.
+
+Python Sphinx does have some dependencies that vary across distributions. The
+first time you build the documentation, the script will notify you if you do not
+have the dependencies installed. To run Sphinx and build documentation successfully,
+the following packages are required:
+
+.. raw:: html
+
+	<style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
+	<table cellpadding="10"><colgroup><col width="30%"><col width="30%"><col width="30%"></colgroup><tbody valign="top"><tr><td><h3>Debian/Ubuntu</h3>
+
+- gcc
+- python3-dev
+- python3-pip
+- python3-sphinx
+- python3-venv
+- libxml2-dev
+- libxslt1-dev
+- doxygen
+- graphviz
+- ant
+- ditaa
+
+.. raw:: html
+
+	</td><td><h3>Fedora</h3>
+
+- gcc
+- python-devel
+- python-pip
+- python-docutils
+- python-jinja2
+- python-pygments
+- python-sphinx
+- libxml2-devel
+- libxslt1-devel
+- doxygen
+- graphviz
+- ant
+- ditaa
+
+.. raw:: html
+
+	</td><td><h3>CentOS/RHEL</h3>
+
+- gcc
+- python-devel
+- python-pip
+- python-docutils
+- python-jinja2
+- python-pygments
+- python-sphinx
+- libxml2-dev
+- libxslt1-dev
+- doxygen
+- graphviz
+- ant
+
+.. raw:: html
+
+	</td></tr></tbody></table>
+
+
+Install each dependency that is not installed on your host. For Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+	sudo apt-get install gcc python-dev python3-pip libxml2-dev libxslt-dev doxygen graphviz ant ditaa
+	sudo apt-get install python3-sphinx python3-venv
+
+For Fedora distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install gcc python-devel python-pip libxml2-devel libxslt-devel doxygen graphviz ant
+   sudo pip install html2text
+   sudo yum install python-jinja2 python-pygments python-docutils python-sphinx
+   sudo yum install jericho-html ditaa
+
+For CentOS/RHEL distributions, it is recommended to have ``epel`` (Extra
+Packages for Enterprise Linux) repository as it provides some extra packages
+which are not available in the default repository. To install ``epel``, execute
+the following:
+
+.. prompt:: bash $
+
+        sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+For CentOS/RHEL distributions, execute the following:
+
+.. prompt:: bash $
+
+	sudo yum install gcc python-devel python-pip libxml2-devel libxslt-devel doxygen graphviz ant
+	sudo pip install html2text
+
+For CentOS/RHEL distributions, the remaining python packages are not available
+in the default and ``epel`` repositories. So, use http://rpmfind.net/ to find
+the packages. Then, download them from a mirror and install them. For example:
+
+.. prompt:: bash $
+
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-jinja2-2.7.2-2.el7.noarch.rpm
+	sudo yum install python-jinja2-2.7.2-2.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-pygments-1.4-9.el7.noarch.rpm
+	sudo yum install python-pygments-1.4-9.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm
+	sudo yum install python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-sphinx-1.1.3-11.el7.noarch.rpm
+	sudo yum install python-sphinx-1.1.3-11.el7.noarch.rpm
+
+Ceph documentation makes extensive use of `ditaa`_, which is not presently built
+for CentOS/RHEL7. You must install ``ditaa`` if you are making changes to
+``ditaa`` diagrams so that you can verify that they render properly before you
+commit new or modified ``ditaa`` diagrams. You may retrieve compatible required
+packages for CentOS/RHEL distributions and install them manually. To run
+``ditaa`` on CentOS/RHEL7, following dependencies are required:
+
+- jericho-html
+- jai-imageio-core
+- batik
+
+Use http://rpmfind.net/ to find compatible ``ditaa`` and the dependencies.
+Then, download them from a mirror and install them. For example:
+
+.. prompt:: bash $
+
+	wget http://rpmfind.net/linux/fedora/linux/releases/22/Everything/x86_64/os/Packages/j/jericho-html-3.3-4.fc22.noarch.rpm
+	sudo yum install jericho-html-3.3-4.fc22.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/jai-imageio-core-1.2-0.14.20100217cvs.el7.noarch.rpm
+	sudo yum install jai-imageio-core-1.2-0.14.20100217cvs.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/batik-1.8-0.12.svn1230816.el7.noarch.rpm
+	sudo yum install batik-1.8-0.12.svn1230816.el7.noarch.rpm
+	wget http://rpmfind.net/linux/fedora/linux/releases/22/Everything/x86_64/os/Packages/d/ditaa-0.9-13.r74.fc21.noarch.rpm
+	sudo yum install ditaa-0.9-13.r74.fc21.noarch.rpm
+
+Once you have installed all these packages, build the documentation by following
+the steps given in `Build the Source`_.
+
+
+Commit the Change
+-----------------
+
+Ceph documentation commits are simple, but follow a strict convention:
+
+- A commit SHOULD have 1 file per commit (it simplifies rollback). You MAY
+  commit multiple files with related changes. Unrelated changes SHOULD NOT
+  be put into the same commit.
+- A commit MUST have a comment.
+- A commit comment MUST be prepended with ``doc:``. (strict)
+- The comment summary MUST be one line only. (strict)
+- Additional comments MAY follow a blank line after the summary, 
+  but should be terse.
+- A commit MAY include ``Fixes: https://tracker.ceph.com/issues/{bug number}``.
+- Commits MUST include ``Signed-off-by: Firstname Lastname <email>``. (strict)
+
+.. tip:: Follow the foregoing convention particularly where it says 
+   ``(strict)`` or you will be asked to modify your commit to comply with 
+   this convention.
+
+The following is a common commit comment (preferred):: 
+
+	doc: Fixes a spelling error and a broken hyperlink.
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+The following comment includes a reference to a bug. :: 
+
+	doc: Fixes a spelling error and a broken hyperlink.
+
+	Fixes: https://tracker.ceph.com/issues/1234
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+The following comment includes a terse sentence following the comment summary.
+There is a carriage return between the summary line and the description:: 
+
+	doc: Added mon setting to monitor config reference
+	
+	Describes 'mon setting', which is a new setting added
+	to config_opts.h.
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+To commit changes, execute the following:
+
+.. prompt:: bash $
+
+	git commit -a
+	
+
+An easy way to manage your documentation commits is to use visual tools for
+``git``. For example, ``gitk`` provides a graphical interface for viewing the
+repository history, and ``git-gui`` provides a graphical interface for viewing
+your uncommitted changes, staging them for commit, committing the changes and
+pushing them to your forked Ceph repository.
+
+
+For Debian/Ubuntu, execute:
+
+.. prompt:: bash $
+
+	sudo apt-get install gitk git-gui
+
+For Fedora/CentOS/RHEL, execute:
+
+.. prompt:: bash $
+
+	sudo yum install gitk git-gui
+
+Then, execute:
+
+.. prompt:: bash $
+
+	cd {git-ceph-repo-path}
+	gitk
+	
+Finally, select **File->Start git gui** to activate the graphical user interface.
+
+
+Push the Change
+---------------
+
+Once you have one or more commits, you must push them from the local copy of the
+repository to ``github``. A graphical tool like ``git-gui`` provides a user
+interface for pushing to the repository. If you created a branch previously:
+
+.. prompt:: bash $
+
+	git push origin wip-doc-{your-branch-name}
+
+Otherwise:
+
+.. prompt:: bash $
+
+	git push
+
+
+Make a Pull Request
+-------------------
+
+As noted earlier, you can make documentation contributions using the `Fork and
+Pull`_ approach.
+
+
+Squash Extraneous Commits
+-------------------------
+Each pull request ought to be associated with only a single commit. If you have
+made more than one commit to the feature branch that you are working in, you
+will need to "squash" the multiple commits. "Squashing" is the colloquial term
+for a particular kind of "interactive rebase". Squashing can be done in a great
+number of ways, but the example here will deal with a situation in which there
+are three commits and the changes in all three of the commits are kept. The three
+commits will be squashed into a single commit.
+
+#. Make the commits that you will later squash.
+
+   #. Make the first commit.
+   
+      ::
+   
+         doc/glossary: improve "CephX" entry
+   
+         Improve the glossary entry for "CephX".
+   
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+   
+         # Please enter the commit message for your changes. Lines starting
+         # with '#' will be ignored, and an empty message aborts the commit.
+         #
+         # On branch wip-doc-2023-03-28-glossary-cephx
+         # Changes to be committed:
+         #       modified:   glossary.rst
+         #
+   
+   #. Make the second commit.
+   
+      ::
+   
+         doc/glossary: add link to architecture doc
+         
+         Add a link to a section in the architecture document, which link
+         will be used in the process of improving the "CephX" glossary entry.
+         
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+      
+            # Please enter the commit message for your changes. Lines starting
+            # with '#' will be ignored, and an empty message aborts the commit.
+            #
+            # On branch wip-doc-2023-03-28-glossary-cephx
+            # Your branch is up to date with 'origin/wip-doc-2023-03-28-glossary-cephx'.
+            #
+            # Changes to be committed:
+            #       modified:   architecture.rst
+      
+   #. Make the third commit.
+   
+      ::
+      
+         doc/glossary: link to Arch doc in "CephX" glossary
+         
+         Link to the Architecture document from the "CephX" entry in the
+         Glossary.
+         
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+         
+         # Please enter the commit message for your changes. Lines starting
+         # with '#' will be ignored, and an empty message aborts the commit.
+         #
+         # On branch wip-doc-2023-03-28-glossary-cephx
+         # Your branch is up to date with 'origin/wip-doc-2023-03-28-glossary-cephx'.
+         #
+         # Changes to be committed:
+         #       modified:   glossary.rst
+
+#. There are now three commits in the feature branch. We will now begin the
+   process of squashing them into a single commit. 
+   
+   #. Run the command ``git rebase -i main``, which rebases the current branch 
+      (the feature branch) against the ``main`` branch:
+
+      .. prompt:: bash
+   
+         git rebase -i main
+   
+   #. A list of the commits that have been made to the feature branch now
+      appear, and looks like this:
+
+      ::
+      
+         pick d395e500883 doc/glossary: improve "CephX" entry
+         pick b34986e2922 doc/glossary: add link to architecture doc
+         pick 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
+         
+         # Rebase 0793495b9d1..74d0719735c onto 0793495b9d1 (3 commands)
+         #
+         # Commands:
+         # p, pick <commit> = use commit
+         # r, reword <commit> = use commit, but edit the commit message
+         # e, edit <commit> = use commit, but stop for amending
+         # s, squash <commit> = use commit, but meld into previous commit
+         # f, fixup [-C | -c] <commit> = like "squash" but keep only the previous
+         #                    commit's log message, unless -C is used, in which case
+         #                    keep only this commit's message; -c is same as -C but
+         #                    opens the editor
+         # x, exec <command> = run command (the rest of the line) using shell
+         # b, break = stop here (continue rebase later with 'git rebase --continue')
+         # d, drop <commit> = remove commit
+         # l, label <label> = label current HEAD with a name
+         # t, reset <label> = reset HEAD to a label
+         # m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
+         #         create a merge commit using the original merge commit's
+         #         message (or the oneline, if no original merge commit was
+         #         specified); use -c <commit> to reword the commit message
+         # u, update-ref <ref> = track a placeholder for the <ref> to be updated
+         #                       to this position in the new commits. The <ref> is
+         #                       updated at the end of the rebase
+         #
+         # These lines can be re-ordered; they are executed from top to bottom.
+         #
+         # If you remove a line here THAT COMMIT WILL BE LOST.
+
+      Find the part of the screen that says "pick". This is the part that you will 
+      alter. There are three commits that are currently labeled "pick". We will
+      choose one of them to remain labeled "pick", and we will label the other two
+      commits "squash".
+
+#. Label two of the three commits ``squash``:
+
+   ::
+
+      pick d395e500883 doc/glossary: improve "CephX" entry
+      squash b34986e2922 doc/glossary: add link to architecture doc
+      squash 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
+      
+      # Rebase 0793495b9d1..74d0719735c onto 0793495b9d1 (3 commands)
+      #
+      # Commands:
+      # p, pick <commit> = use commit
+      # r, reword <commit> = use commit, but edit the commit message
+      # e, edit <commit> = use commit, but stop for amending
+      # s, squash <commit> = use commit, but meld into previous commit
+      # f, fixup [-C | -c] <commit> = like "squash" but keep only the previous
+      #                    commit's log message, unless -C is used, in which case
+      #                    keep only this commit's message; -c is same as -C but
+      #                    opens the editor
+      # x, exec <command> = run command (the rest of the line) using shell
+      # b, break = stop here (continue rebase later with 'git rebase --continue')
+      # d, drop <commit> = remove commit
+      # l, label <label> = label current HEAD with a name
+      # t, reset <label> = reset HEAD to a label
+      # m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
+      #         create a merge commit using the original merge commit's
+      #         message (or the oneline, if no original merge commit was
+      #         specified); use -c <commit> to reword the commit message
+      # u, update-ref <ref> = track a placeholder for the <ref> to be updated
+      #                       to this position in the new commits. The <ref> is
+      #                       updated at the end of the rebase
+      #
+      # These lines can be re-ordered; they are executed from top to bottom.
+      #
+      # If you remove a line here THAT COMMIT WILL BE LOST.
+
+#. Now we create a commit message that applies to all the commits that have
+   been squashed together:
+
+   #. When you save and close the list of commits that you have designated for
+      squashing, a list of all three commit messages appears, and it looks
+      like this:
+
+      ::
+      
+         # This is a combination of 3 commits.
+         # This is the 1st commit message:
+      
+         doc/glossary: improve "CephX" entry
+      
+         Improve the glossary entry for "CephX".
+      
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+      
+         # This is the commit message #2:
+      
+         doc/glossary: add link to architecture doc
+      
+         Add a link to a section in the architecture document, which link
+         will be used in the process of improving the "CephX" glossary entry.
+      
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+      
+         # This is the commit message #3:
+      
+         doc/glossary: link to Arch doc in "CephX" glossary
+      
+         Link to the Architecture document from the "CephX" entry in the
+         Glossary.
+      
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+      
+         # Please enter the commit message for your changes. Lines starting
+         # with '#' will be ignored, and an empty message aborts the commit.
+         #
+         # Date:      Tue Mar 28 18:42:11 2023 +1000
+         #
+         # interactive rebase in progress; onto 0793495b9d1
+         # Last commands done (3 commands done):
+         #    squash b34986e2922 doc/glossary: add link to architecture doc
+         #    squash 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
+         # No commands remaining.
+         # You are currently rebasing branch 'wip-doc-2023-03-28-glossary-cephx' on '0793495b9d1'.
+         #
+         # Changes to be committed:
+         #       modified:   doc/architecture.rst
+         #       modified:   doc/glossary.rst
+      
+   #. The commit messages have been revised into the simpler form presented here:   
+            
+      ::
+      
+         doc/glossary: improve "CephX" entry
+      
+         Improve the glossary entry for "CephX".
+      
+         Signed-off-by: Zac Dover <zac.dover@proton.me>
+      
+         # Please enter the commit message for your changes. Lines starting
+         # with '#' will be ignored, and an empty message aborts the commit.
+         #
+         # Date:      Tue Mar 28 18:42:11 2023 +1000
+         #
+         # interactive rebase in progress; onto 0793495b9d1
+         # Last commands done (3 commands done):
+         #    squash b34986e2922 doc/glossary: add link to architecture doc
+         #    squash 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
+         # No commands remaining.
+         # You are currently rebasing branch 'wip-doc-2023-03-28-glossary-cephx' on '0793495b9d1'.
+         #
+         # Changes to be committed:
+         #       modified:   doc/architecture.rst
+         #       modified:   doc/glossary.rst
+
+#. Force push the squashed commit from your local working copy to the remote
+   upstream branch. The force push is necessary because the newly squashed commit
+   does not have an ancestor in the remote. If that confuses you, just run this 
+   command and don't think too much about it:
+
+   .. prompt:: bash $  
+
+      git push -f
+   
+   ::
+
+      Enumerating objects: 9, done.
+      Counting objects: 100% (9/9), done.
+      Delta compression using up to 8 threads
+      Compressing objects: 100% (5/5), done.
+      Writing objects: 100% (5/5), 722 bytes | 722.00 KiB/s, done.
+      Total 5 (delta 4), reused 0 (delta 0), pack-reused 0
+      remote: Resolving deltas: 100% (4/4), completed with 4 local objects.
+      To github.com:zdover23/ceph.git
+       + b34986e2922...02e3a5cb763 wip-doc-2023-03-28-glossary-cephx -> wip-doc-2023-03-28-glossary-cephx (forced update)
+
+
+
+
+
+Notify Us
+---------
+
+If some time has passed and the pull request that you raised has not been
+reviewed, contact the component lead and ask what's taking so long. See
+:ref:`ctl` for a list of component leads.
+
+Documentation Style Guide
+=========================
+
+One objective of the Ceph documentation project is to ensure the readability of
+the documentation in both native reStructuredText format and its rendered
+formats such as HTML. Navigate to your Ceph repository and view a document in
+its native format. You may notice that it is generally as legible in a terminal
+as it is in its rendered HTML format. Additionally, you may also notice that
+diagrams in ``ditaa`` format also render reasonably well in text mode. :
+
+.. prompt:: bash $
+
+	less doc/architecture.rst
+
+Review the following style guides to maintain this consistency.
+
+
+Headings
+--------
+
+#. **Document Titles:** Document titles use the ``=`` character overline and 
+   underline with a leading and trailing space on the title text line. 
+   See `Document Title`_ for details.
+
+#. **Section Titles:** Section tiles use the ``=`` character underline with no
+   leading or trailing spaces for text. Two carriage returns should precede a 
+   section title (unless an inline reference precedes it). See `Sections`_ for
+   details.
+
+#. **Subsection Titles:** Subsection titles use the ``_`` character underline 
+   with no leading or trailing spaces for text.  Two carriage returns should 
+   precede a subsection title (unless an inline reference precedes it).
+
+
+Text Body
+---------
+
+As a general rule, we prefer text to wrap at column 80 so that it is legible in
+a command line interface without leading or trailing white space. Where
+possible, we prefer to maintain this convention with text, lists, literal text
+(exceptions allowed), tables, and ``ditaa`` graphics.
+
+#. **Paragraphs**: Paragraphs have a leading and a trailing carriage return, 
+   and should be 80 characters wide or less so that the documentation can be 
+   read in native format in a command line terminal.
+
+#. **Literal Text:** To create an example of literal text (e.g., command line
+   usage), terminate the preceding paragraph with ``::`` or enter a carriage
+   return to create an empty line after the preceding paragraph; then, enter
+   ``::`` on a separate line followed by another empty line. Then, begin the
+   literal text with tab indentation (preferred) or space indentation of 3 
+   characters.
+
+#. **Indented Text:** Indented text such as bullet points 
+   (e.g., ``- some text``) may span multiple lines. The text of subsequent
+   lines should begin at the same character position as the text of the
+   indented text (less numbers, bullets, etc.).
+
+   Indented text may include literal text examples. Whereas, text indentation
+   should be done with spaces, literal text examples should be indented with
+   tabs. This convention enables you to add an additional indented paragraph
+   following a literal example by leaving a blank line and beginning the
+   subsequent paragraph with space indentation.
+
+#. **Numbered Lists:** Numbered lists should use autonumbering by starting
+   a numbered indent with ``#.`` instead of the actual number so that
+   numbered paragraphs can be repositioned without requiring manual 
+   renumbering.
+
+#. **Code Examples:** Ceph supports the use of the 
+   ``.. code-block::<language>`` role, so that you can add highlighting to 
+   source examples. This is preferred for source code. However, use of this 
+   tag will cause autonumbering to restart at 1 if it is used as an example 
+   within a numbered list. See `Showing code examples`_ for details.
+
+
+Paragraph Level Markup
+----------------------
+
+The Ceph project uses `paragraph level markup`_ to highlight points.
+
+#. **Tip:** Use the ``.. tip::`` directive to provide additional information
+   that assists the reader or steers the reader away from trouble.
+
+#. **Note**: Use the ``.. note::`` directive to highlight an important point.
+
+#. **Important:** Use the ``.. important::`` directive to highlight important
+   requirements or caveats (e.g., anything that could lead to data loss). Use
+   this directive sparingly, because it renders in red.
+
+#. **Version Added:** Use the ``.. versionadded::`` directive for new features
+   or configuration settings so that users know the minimum release for using
+   a feature.
+   
+#. **Version Changed:** Use the ``.. versionchanged::`` directive for changes
+   in usage or configuration settings.
+
+#. **Deprecated:** Use the ``.. deprecated::`` directive when CLI usage, 
+   a feature or a configuration setting is no longer preferred or will be 
+   discontinued.
+
+#. **Topic:** Use the ``.. topic::`` directive to encapsulate text that is
+   outside the main flow of the document. See the `topic directive`_ for
+   additional details.
+
+
+Table of Contents (TOC) and Hyperlinks
+---------------------------------------
+
+The documents in the Ceph documentation suite follow certain conventions that
+are explained in this section.
+
+Every document (every ``.rst`` file) in the Sphinx-controlled Ceph
+documentation suite must be linked either (1) from another document in the
+documentation suite or (2) from a table of contents (TOC). If any document in
+the documentation suite is not linked in this way, the ``build-doc`` script
+generates warnings when it tries to build the documentation. 
+
+The Ceph project uses the ``.. toctree::`` directive. See `The TOC tree`_ for
+details. When rendering a table of contents (TOC), specify the ``:maxdepth:``
+parameter so that the rendered TOC is not too long.
+
+Use the ``:ref:`` syntax where a link target contains a specific unique
+identifier (for example, ``.. _unique-target-id:``). A link to the section
+designated by ``.. _unique-target-id:`` looks like this:
+``:ref:`unique-target-id```. If this convention is followed, the links within
+the ``.rst`` source files will work even if the source files are moved within
+the ``ceph/doc`` directory. See `Cross referencing arbitrary locations`_ for
+details.
+
+.. _start_external_hyperlink_example:
+
+External Hyperlink Example
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is also possible to create a link to a section of the documentation and to
+have custom text appear in the body of the link. This is useful when it is more
+important to preserve the text of the sentence containing the link than it is
+to refer explicitly to the title of the section being linked to.
+
+For example, RST that links to the Sphinx Python Document Generator homepage
+and generates a sentence reading "Click here to learn more about Python
+Sphinx." looks like this: 
+
+::
+
+    ``Click `here <https://www.sphinx-doc.org>`_ to learn more about Python
+    Sphinx.`` 
+
+And here it is, rendered:
+
+Click `here <https://www.sphinx-doc.org>`_ to learn more about Python Sphinx. 
+
+Pay special attention to the underscore after the backtick. If you forget to
+include it and this is your first day working with RST, there's a chance that
+you'll spend all day wondering what went wrong without realizing that you
+omitted that underscore. Also, pay special attention to the space between the
+substitution text (in this case, "here") and the less-than bracket that sets
+the explicit link apart from the substition text. The link will not render
+properly without this space.
+
+Linking Customs
+~~~~~~~~~~~~~~~
+
+By a custom established when Ceph was still being developed by Inktank,
+contributors to the documentation of the Ceph project preferred to use the
+convention of putting ``.. _Link Text: ../path`` links at the bottom of the
+document and linking to them using references of the form ``:ref:`path```. This
+convention was preferred because it made the documents more readable in a
+command line interface. As of 2023, though, we have no preference for one over
+the other. Use whichever convention makes the text easier to read.
+
+Using a part of a sentence as a hyperlink, `like this <docs.ceph.com>`_, is
+discouraged. The convention of writing "See X" is preferred. Here are some
+preferred formulations:
+
+#. For more information, see `docs.ceph.com <docs.ceph.com>`_.
+
+#. See `docs.ceph.com <docs.ceph.com>`_.
+
+
+Quirks of ReStructured Text
+---------------------------
+
+External Links
+~~~~~~~~~~~~~~
+
+.. _external_link_with_inline_text:
+
+Use the formula immediately below to render links that direct the reader to
+addresses external to the Ceph documentation:
+
+::
+
+   `inline text <http:www.foo.com>`_
+
+.. note:: Do not fail to include the space between the inline text and the
+   less-than sign. 
+   
+   Do not fail to include the underscore after the final backtick.
+
+   To link to addresses that are external to the Ceph documentation, include a
+   space between the inline text and the angle bracket that precedes the
+   external address. This is precisely the opposite of the convention for
+   inline text that links to a location inside the Ceph documentation. See
+   :ref:`here <internal_link_with_inline_text>` for an exemplar of this
+   convention.
+
+   If this seems inconsistent and confusing to you, then you're right. It is
+   inconsistent and confusing.
+
+See also ":ref:`External Hyperlink Example<start_external_hyperlink_example>`".
+
+Internal Links
+~~~~~~~~~~~~~~
+
+To link to a section in the Ceph documentation, you must (1) define a target
+link before the section and then (2) link to that target from another location
+in the documentation. Here are the formulas for targets and links to those
+targets:
+
+Target::
+
+   .. _target:
+
+   Title of Targeted Section
+   =========================
+
+   Lorem ipsum...
+
+Link to target::
+
+   :ref:`target`
+
+.. _internal_link_with_inline_text:
+
+Link to target with inline text::
+
+   :ref:`inline text<target>`
+
+.. note:: 
+
+   There is no space between "inline text" and the angle bracket that
+   immediately follows it. This is precisely the opposite of :ref:`the
+   convention for inline text that links to a location outside of the Ceph
+   documentation<external_link_with_inline_text>`. If this seems inconsistent
+   and confusing to you, then you're right. It is inconsistent and confusing.
+
+Escaping Bold Characters within Words
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section explains how to make certain letters within a word bold while
+leaving the other letters in the word regular (non-bold). 
+
+The following single-line paragraph provides an example of this:
+
+**C**\eph **F**\ile **S**\ystem.
+
+In ReStructured Text, the following formula will not work:
+
+::
+
+   **C**eph **F**ile **S**ystem
+
+The bolded notation must be turned off by means of the escape character (\\), as shown here:
+
+::
+
+   **C**\eph **F**\ile **S**\ystem
+
+.. _Python Sphinx: https://www.sphinx-doc.org
+.. _restructuredText: http://docutils.sourceforge.net/rst.html
+.. _Fork and Pull: https://help.github.com/articles/using-pull-requests
+.. _github: http://github.com
+.. _ditaa: http://ditaa.sourceforge.net/
+.. _Document Title: http://docutils.sourceforge.net/docs/user/rst/quickstart.html#document-title-subtitle
+.. _Sections: http://docutils.sourceforge.net/docs/user/rst/quickstart.html#sections
+.. _Cross referencing arbitrary locations: http://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+.. _The TOC tree: http://sphinx-doc.org/markup/toctree.html
+.. _Showing code examples: http://sphinx-doc.org/markup/code.html
+.. _paragraph level markup: http://sphinx-doc.org/markup/para.html
+.. _topic directive: http://docutils.sourceforge.net/docs/ref/rst/directives.html#topic
diff --git a/doc/start/get-involved.rst b/doc/start/get-involved.rst
new file mode 100644
index 000000000..4f5277e37
--- /dev/null
+++ b/doc/start/get-involved.rst
@@ -0,0 +1,99 @@
+.. _Get Involved:
+
+=====================================
+ Get Involved in the Ceph Community!
+=====================================
+
+These are exciting times in the Ceph community! Get involved!
+
++----------------------+-------------------------------------------------+-----------------------------------------------+
+|Channel               | Description                                     | Contact Info                                  |
++======================+=================================================+===============================================+
+| **Blog**             | Check the Ceph Blog_ periodically to keep track | http://ceph.com/community/blog/               |
+|                      | of Ceph progress and important announcements.   |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Planet Ceph**      | Check the blog aggregation on Planet Ceph for   | https://old.ceph.com/category/planet/         |
+|                      | interesting stories, information and            |                                               |
+|                      | experiences from the community. **NOTE: NO      |                                               |
+|                      | longer updated as of 2023.**                    |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Wiki**             | Check the Ceph Wiki is a source for more        | http://wiki.ceph.com/                         |
+|                      | community and development related topics. You   |                                               |
+|                      | can find there information about blueprints,    |                                               |
+|                      | meetups, the Ceph Developer Summits and more.   |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **IRC**              | As you delve into Ceph, you may have questions  |                                               |
+|                      | or feedback for the Ceph development team. Ceph | - **Domain:**                                 |
+|                      | developers are often available on the ``#ceph`` |     ``irc.oftc.net``                          |
+|                      | IRC channel particularly during daytime hours   | - **Channels:**                               |
+|                      | in the US Pacific Standard Time zone.           |     ``#ceph``,                                |
+|                      | While ``#ceph`` is a good starting point for    |     ``#ceph-devel``,                          |
+|                      | cluster operators and users, there is also      |     ``#ceph-dashboard``,                      |
+|                      | ``#ceph-devel``, ``#ceph-dashboard`` and        |     ``#cephfs``                               |
+|                      | ``#cephfs`` dedicated for Ceph developers.      |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **User List**        | Ask and answer user-related questions by        |                                               |
+|                      | subscribing to the email list at                | - `User Subscribe`_                           |
+|                      | ceph-users@ceph.io. You can opt out of the email| - `User Unsubscribe`_                         |
+|                      | list at any time by unsubscribing. A simple     | - `User Archives`_                            |
+|                      | email is all it takes!                          |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Devel List**       | Keep in touch with developer activity by        |                                               |
+|                      | subscribing to the email list at dev@ceph.io.   | - `Devel Subscribe`_                          |
+|                      | You can opt out of the email list at any time by| - `Devel Unsubscribe`_                        |
+|                      | unsubscribing. A simple email is all it takes!  | - `Devel Archives`_                           |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Kernel Client**    | Linux kernel-related traffic, including kernel  | - `Kernel Client Subscribe`_                  |
+|                      | patches and discussion of implementation details| - `Kernel Client Unsubscribe`_                |
+|                      | for the kernel client code.                     | - `Kernel Client Archives`_                   |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Commit List**      | Subscribe to ceph-commit@ceph.com to get        |                                               |
+|                      | commit notifications via email. You can opt out | - `Commit Subscribe`_                         |
+|                      | of the email list at any time by unsubscribing. | - `Commit Unsubscribe`_                       |
+|                      | A simple email is all it takes!                 | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **QA List**          | For Quality Assurance (QA) related activities   |                                               |
+|                      | subscribe to this list. You can opt out         | - `QA Subscribe`_                             |
+|                      | of the email list at any time by unsubscribing. | - `QA Unsubscribe`_                           |
+|                      | A simple email is all it takes!                 | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Community List**   | For all discussions related to the Ceph User    |                                               |
+|                      | Committee and other community topics. You can   | - `Community Subscribe`_                      |
+|                      | opt out of the email list at any time by        | - `Community Unsubscribe`_                    |
+|                      | unsubscribing. A simple email is all it takes!  | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Bug Tracker**      | You can help keep Ceph production worthy by     | http://tracker.ceph.com/projects/ceph         |
+|                      | filing and tracking bugs, and providing feature |                                               |
+|                      | requests using the Bug Tracker_.                |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Source Code**      | If you would like to participate in             |                                               |
+|                      | development, bug fixing, or if you just want    | - http://github.com/ceph/ceph                 |
+|                      | the very latest code for Ceph, you can get it   | - http://download.ceph.com/tarballs/          |
+|                      | at http://github.com. See `Ceph Source Code`_   |                                               |
+|                      | for details on cloning from github.             |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Ceph Calendar**    | Learn about upcoming Ceph events.               | https://ceph.io/contribute/                   |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+
+
+
+.. _Devel Subscribe: mailto:dev-request@ceph.io?body=subscribe
+.. _Devel Unsubscribe: mailto:dev-request@ceph.io?body=unsubscribe
+.. _Kernel Client Subscribe: mailto:majordomo@vger.kernel.org?body=subscribe+ceph-devel
+.. _Kernel Client Unsubscribe: mailto:majordomo@vger.kernel.org?body=unsubscribe+ceph-devel
+.. _User Subscribe: mailto:ceph-users-request@ceph.io?body=subscribe
+.. _User Unsubscribe: mailto:ceph-users-request@ceph.io?body=unsubscribe
+.. _Community Subscribe: mailto:ceph-community-join@lists.ceph.com
+.. _Community Unsubscribe: mailto:ceph-community-leave@lists.ceph.com
+.. _Commit Subscribe: mailto:ceph-commit-join@lists.ceph.com
+.. _Commit Unsubscribe: mailto:ceph-commit-leave@lists.ceph.com
+.. _QA Subscribe: mailto:ceph-qa-join@lists.ceph.com
+.. _QA Unsubscribe: mailto:ceph-qa-leave@lists.ceph.com
+.. _Devel Archives: https://lists.ceph.io/hyperkitty/list/dev@ceph.io/
+.. _User Archives: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/
+.. _Kernel Client Archives: https://www.spinics.net/lists/ceph-devel/
+.. _Mailing list archives: http://lists.ceph.com/
+.. _Blog: http://ceph.com/community/blog/
+.. _Tracker: http://tracker.ceph.com/
+.. _Ceph Source Code: http://github.com/ceph/ceph
+
diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
new file mode 100644
index 000000000..a63b5a457
--- /dev/null
+++ b/doc/start/hardware-recommendations.rst
@@ -0,0 +1,623 @@
+.. _hardware-recommendations:
+
+==========================
+ hardware recommendations
+==========================
+
+Ceph is designed to run on commodity hardware, which makes building and
+maintaining petabyte-scale data clusters flexible and economically feasible. 
+When planning your cluster's hardware, you will need to balance a number 
+of considerations, including failure domains, cost, and performance.
+Hardware planning should include distributing Ceph daemons and 
+other processes that use Ceph across many hosts. Generally, we recommend 
+running Ceph daemons of a specific type on a host configured for that type 
+of daemon. We recommend using separate hosts for processes that utilize your 
+data cluster (e.g., OpenStack, CloudStack, Kubernetes, etc).
+
+The requirements of one Ceph cluster are not the same as the requirements of
+another, but below are some general guidelines. 
+
+.. tip:: check out the `ceph blog`_ too.
+
+CPU
+===
+
+CephFS Metadata Servers (MDS) are CPU-intensive. They are
+are single-threaded and perform best with CPUs with a high clock rate (GHz). MDS
+servers do not need a large number of CPU cores unless they are also hosting other
+services, such as SSD OSDs for the CephFS metadata pool.
+OSD nodes need enough processing power to run the RADOS service, to calculate data
+placement with CRUSH, to replicate data, and to maintain their own copies of the
+cluster map.
+
+With earlier releases of Ceph, we would make hardware recommendations based on
+the number of cores per OSD, but this cores-per-osd metric is no longer as
+useful a metric as the number of cycles per IOP and the number of IOPS per OSD.
+For example, with NVMe OSD drives, Ceph can easily utilize five or six cores on real
+clusters and up to about fourteen cores on single OSDs in isolation. So cores
+per OSD are no longer as pressing a concern as they were. When selecting
+hardware, select for IOPS per core.
+
+.. tip:: When we speak of CPU _cores_, we mean _threads_ when hyperthreading
+	 is enabled.  Hyperthreading is usually beneficial for Ceph servers.
+
+Monitor nodes and Manager nodes do not have heavy CPU demands and require only
+modest processors. if your hosts will run CPU-intensive processes in
+addition to Ceph daemons, make sure that you have enough processing power to
+run both the CPU-intensive processes and the Ceph daemons. (OpenStack Nova is
+one example of a CPU-intensive process.) We recommend that you run
+non-Ceph CPU-intensive processes on separate hosts (that is, on hosts that are
+not your Monitor and Manager nodes) in order to avoid resource contention.
+If your cluster deployes the Ceph Object Gateway, RGW daemons may co-reside
+with your Mon and Manager services if the nodes have sufficient resources.
+
+RAM
+===
+
+Generally, more RAM is better.  Monitor / Manager nodes for a modest cluster
+might do fine with 64GB; for a larger cluster with hundreds of OSDs 128GB
+is advised.
+
+.. tip:: when we speak of RAM and storage requirements, we often describe
+	 the needs of a single daemon of a given type.  A given server as
+	 a whole will thus need at least the sum of the needs of the
+	 daemons that it hosts as well as resources for logs and other operating
+	 system components.  Keep in mind that a server's need for RAM
+	 and storage will be greater at startup and when components
+	 fail or are added and the cluster rebalances.  In other words,
+	 allow headroom past what you might see used during a calm period
+	 on a small initial cluster footprint.
+
+There is an :confval:`osd_memory_target` setting for BlueStore OSDs that
+defaults to 4GB.  Factor in a prudent margin for the operating system and
+administrative tasks (like monitoring and metrics) as well as increased
+consumption during recovery:  provisioning ~8GB *per BlueStore OSD* is thus
+advised.
+
+Monitors and managers (ceph-mon and ceph-mgr)
+---------------------------------------------
+
+Monitor and manager daemon memory usage scales with the size of the
+cluster.  Note that at boot-time and during topology changes and recovery these
+daemons will need more RAM than they do during steady-state operation, so plan
+for peak usage. For very small clusters, 32 GB suffices. For clusters of up to,
+say, 300 OSDs go with 64GB. For clusters built with (or which will grow to)
+even more OSDs you should provision 128GB. You may also want to consider
+tuning the following settings:
+
+* :confval:`mon_osd_cache_size`
+* :confval:`rocksdb_cache_size`
+
+
+Metadata servers (ceph-mds)
+---------------------------
+
+CephFS metadata daemon memory utilization depends on the configured size of
+its cache. We recommend 1 GB as a minimum for most systems.  See
+:confval:`mds_cache_memory_limit`.
+
+
+Memory
+======
+
+Bluestore uses its own memory to cache data rather than relying on the
+operating system's page cache. In Bluestore you can adjust the amount of memory
+that the OSD attempts to consume by changing the :confval:`osd_memory_target`
+configuration option.
+
+- Setting the :confval:`osd_memory_target` below 2GB is not
+  recommended. Ceph may fail to keep the memory consumption under 2GB and 
+  extremely slow performance is likely.
+
+- Setting the memory target between 2GB and 4GB typically works but may result
+  in degraded performance: metadata may need to be read from disk during IO
+  unless the active data set is relatively small.
+
+- 4GB is the current default value for :confval:`osd_memory_target` This default
+  was chosen for typical use cases, and is intended to balance RAM cost and
+  OSD performance.
+
+- Setting the :confval:`osd_memory_target` higher than 4GB can improve
+  performance when there many (small) objects or when large (256GB/OSD 
+  or more) data sets are processed.  This is especially true with fast
+  NVMe OSDs.
+
+.. important:: OSD memory management is "best effort". Although the OSD may
+   unmap memory to allow the kernel to reclaim it, there is no guarantee that
+   the kernel will actually reclaim freed memory within a specific time
+   frame. This applies especially in older versions of Ceph, where transparent
+   huge pages can prevent the kernel from reclaiming memory that was freed from
+   fragmented huge pages. Modern versions of Ceph disable transparent huge
+   pages at the application level to avoid this, but that does not
+   guarantee that the kernel will immediately reclaim unmapped memory. The OSD
+   may still at times exceed its memory target. We recommend budgeting 
+   at least 20% extra memory on your system to prevent OSDs from going OOM
+   (**O**\ut **O**\f **M**\emory) during temporary spikes or due to delay in
+   the kernel reclaiming freed pages. That 20% value might be more or less than
+   needed, depending on the exact configuration of the system.
+
+.. tip:: Configuring the operating system with swap to provide additional
+	 virtual memory for daemons is not advised for modern systems.  Doing
+	 may result in lower performance, and your Ceph cluster may well be
+	 happier with a daemon that crashes vs one that slows to a crawl.
+
+When using the legacy FileStore back end, the OS page cache was used for caching
+data, so tuning was not normally needed. When using the legacy FileStore backend,
+the OSD memory consumption was related to the number of PGs per daemon in the
+system.
+
+
+Data Storage
+============
+
+Plan your data storage configuration carefully. There are significant cost and
+performance tradeoffs to consider when planning for data storage. Simultaneous
+OS operations and simultaneous requests from multiple daemons for read and
+write operations against a single drive can impact performance.
+
+OSDs require substantial storage drive space for RADOS data. We recommend a
+minimum drive size of 1 terabyte. OSD drives much smaller than one terabyte
+use a significant fraction of their capacity for metadata, and drives smaller
+than 100 gigabytes will not be effective at all.
+
+It is *strongly* suggested that (enterprise-class) SSDs are provisioned for, at a
+minimum, Ceph Monitor and Ceph Manager hosts, as well as CephFS Metadata Server
+metadata pools and Ceph Object Gateway (RGW) index pools, even if HDDs are to
+be provisioned for bulk OSD data.
+
+To get the best performance out of Ceph, provision the following on separate
+drives:
+
+* The operating systems
+* OSD data
+* BlueStore WAL+DB
+
+For more
+information on how to effectively use a mix of fast drives and slow drives in
+your Ceph cluster, see the `block and block.db`_ section of the Bluestore
+Configuration Reference.
+
+Hard Disk Drives
+----------------
+
+Consider carefully the cost-per-gigabyte advantage
+of larger disks. We recommend dividing the price of the disk drive by the
+number of gigabytes to arrive at a cost per gigabyte, because larger drives may
+have a significant impact on the cost-per-gigabyte. For example, a 1 terabyte
+hard disk priced at $75.00 has a cost of $0.07 per gigabyte (i.e., $75 / 1024 =
+0.0732). By contrast, a 3 terabyte disk priced at $150.00 has a cost of $0.05
+per gigabyte (i.e., $150 / 3072 = 0.0488). In the foregoing example, using the
+1 terabyte disks would generally increase the cost per gigabyte by
+40%--rendering your cluster substantially less cost efficient.
+
+.. tip:: Hosting multiple OSDs on a single SAS / SATA HDD
+   is **NOT** a good idea.
+
+.. tip:: Hosting an OSD with monitor, manager, or MDS data on a single 
+   drive is also **NOT** a good idea.
+
+.. tip:: With spinning disks, the SATA and SAS interface increasingly
+   becomes a bottleneck at larger capacities. See also the `Storage Networking 
+   Industry Association's Total Cost of Ownership calculator`_.
+
+
+Storage drives are subject to limitations on seek time, access time, read and
+write times, as well as total throughput. These physical limitations affect
+overall system performance--especially during recovery. We recommend using a
+dedicated (ideally mirrored) drive for the operating system and software, and
+one drive for each Ceph OSD Daemon you run on the host.
+Many "slow OSD" issues (when they are not attributable to hardware failure)
+arise from running an operating system and multiple OSDs on the same drive.
+Also be aware that today's 22TB HDD uses the same SATA interface as a
+3TB HDD from ten years ago: more than seven times the data to squeeze
+through the same same interface.  For this reason, when using HDDs for
+OSDs, drives larger than 8TB may be best suited for storage of large
+files / objects that are not at all performance-sensitive.
+
+
+Solid State Drives
+------------------
+
+Ceph performance is much improved when using solid-state drives (SSDs). This
+reduces random access time and reduces latency while increasing throughput. 
+
+SSDs cost more per gigabyte than do HDDs but SSDs often offer
+access times that are, at a minimum, 100 times faster than HDDs.
+SSDs avoid hotspot issues and bottleneck issues within busy clusters, and
+they may offer better economics when TCO is evaluated holistically. Notably,
+the amortized drive cost for a given number of IOPS is much lower with SSDs
+than with HDDs.  SSDs do not suffer rotational or seek latency and in addition
+to improved client performance, they substantially improve the speed and
+client impact of cluster changes including rebalancing when OSDs or Monitors
+are added, removed, or fail.
+
+SSDs do not have moving mechanical parts, so they are not subject
+to many of the limitations of HDDs.  SSDs do have significant
+limitations though. When evaluating SSDs, it is important to consider the
+performance of sequential and random reads and writes.
+
+.. important:: We recommend exploring the use of SSDs to improve performance. 
+   However, before making a significant investment in SSDs, we **strongly
+   recommend** reviewing the performance metrics of an SSD and testing the
+   SSD in a test configuration in order to gauge performance. 
+
+Relatively inexpensive SSDs may appeal to your sense of economy. Use caution.
+Acceptable IOPS are not the only factor to consider when selecting SSDs for
+use with Ceph. Bargain SSDs are often a false economy: they may experience
+"cliffing", which means that after an initial burst, sustained performance
+once a limited cache is filled declines considerably.  Consider also durability:
+a drive rated for 0.3 Drive Writes Per Day (DWPD or equivalent) may be fine for
+OSDs dedicated to certain types of sequentially-written read-mostly data, but
+are not a good choice for Ceph Monitor duty.  Enterprise-class SSDs are best
+for Ceph:  they almost always feature power less protection (PLP) and do
+not suffer the dramatic cliffing that client (desktop) models may experience.
+
+When using a single (or mirrored pair) SSD for both operating system boot
+and Ceph Monitor / Manager purposes, a minimum capacity of 256GB is advised
+and at least 480GB is recommended. A drive model rated at 1+ DWPD (or the
+equivalent in TBW (TeraBytes Written) is suggested.  However, for a given write
+workload, a larger drive than technically required will provide more endurance
+because it effectively has greater overprovsioning. We stress that
+enterprise-class drives are best for production use, as they feature power
+loss protection and increased durability compared to client (desktop) SKUs
+that are intended for much lighter and intermittent duty cycles.
+
+SSDs were historically been cost prohibitive for object storage, but
+QLC SSDs are closing the gap, offering greater density with lower power
+consumption and less power spent on cooling. Also, HDD OSDs may see a
+significant write latency improvement by offloading WAL+DB onto an SSD.
+Many Ceph OSD deployments do not require an SSD with greater endurance than
+1 DWPD (aka "read-optimized").  "Mixed-use" SSDs in the 3 DWPD class are
+often overkill for this purpose and cost signficantly more.
+
+To get a better sense of the factors that determine the total cost of storage,
+you might use the `Storage Networking Industry Association's Total Cost of
+Ownership calculator`_
+
+Partition Alignment
+~~~~~~~~~~~~~~~~~~~
+
+When using SSDs with Ceph, make sure that your partitions are properly aligned.
+Improperly aligned partitions suffer slower data transfer speeds than do
+properly aligned partitions. For more information about proper partition
+alignment and example commands that show how to align partitions properly, see
+`Werner Fischer's blog post on partition alignment`_.
+
+CephFS Metadata Segregation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One way that Ceph accelerates CephFS file system performance is by separating
+the storage of CephFS metadata from the storage of the CephFS file contents.
+Ceph provides a default ``metadata`` pool for CephFS metadata. You will never
+have to manually create a pool for CephFS metadata, but you can create a CRUSH map
+hierarchy for your CephFS metadata pool that includes only SSD storage media.
+See :ref:`CRUSH Device Class<crush-map-device-class>` for details.
+
+
+Controllers
+-----------
+
+Disk controllers (HBAs) can have a significant impact on write throughput.
+Carefully consider your selection of HBAs to ensure that they do not create a
+performance bottleneck. Notably, RAID-mode (IR) HBAs may exhibit higher latency
+than simpler "JBOD" (IT) mode HBAs. The RAID SoC, write cache, and battery
+backup can substantially increase hardware and maintenance costs. Many RAID
+HBAs can be configured with an IT-mode "personality" or "JBOD mode" for
+streamlined operation.
+
+You do not need an RoC (RAID-capable) HBA. ZFS or Linux MD software mirroring
+serve well for boot volume durability.  When using SAS or SATA data drives,
+forgoing HBA RAID capabilities can reduce the gap between HDD and SSD
+media cost.  Moreover, when using NVMe SSDs, you do not need *any* HBA.  This
+additionally reduces the HDD vs SSD cost gap when the system as a whole is
+considered. The initial cost of a fancy RAID HBA plus onboard cache plus
+battery backup (BBU or supercapacitor) can easily exceed more than 1000 US
+dollars even after discounts - a sum that goes a log way toward SSD cost parity.
+An HBA-free system may also cost hundreds of US dollars less every year if one
+purchases an annual maintenance contract or extended warranty.
+
+.. tip:: The `Ceph blog`_ is often an excellent source of information on Ceph
+   performance issues. See `Ceph Write Throughput 1`_ and `Ceph Write 
+   Throughput 2`_ for additional details.
+
+
+Benchmarking
+------------
+
+BlueStore opens storage devices with ``O_DIRECT`` and issues ``fsync()``
+frequently to ensure that data is safely persisted to media. You can evaluate a
+drive's low-level write performance using ``fio``. For example, 4kB random write
+performance is measured as follows:
+
+.. code-block:: console
+
+  # fio --name=/dev/sdX --ioengine=libaio --direct=1 --fsync=1 --readwrite=randwrite --blocksize=4k --runtime=300
+
+Write Caches
+------------
+
+Enterprise SSDs and HDDs normally include power loss protection features which
+ensure data durability when power is lost while operating, and
+use multi-level caches to speed up direct or synchronous writes.  These devices
+can be toggled between two caching modes -- a volatile cache flushed to
+persistent media with fsync, or a non-volatile cache written synchronously.
+
+These two modes are selected by either "enabling" or "disabling" the write
+(volatile) cache.  When the volatile cache is enabled, Linux uses a device in
+"write back" mode, and when disabled, it uses "write through".
+
+The default configuration (usually: caching is enabled) may not be optimal, and
+OSD performance may be dramatically increased in terms of increased IOPS and
+decreased commit latency by disabling this write cache.
+
+Users are therefore encouraged to benchmark their devices with ``fio`` as
+described earlier and persist the optimal cache configuration for their
+devices.
+
+The cache configuration can be queried with ``hdparm``, ``sdparm``,
+``smartctl`` or by reading the values in ``/sys/class/scsi_disk/*/cache_type``,
+for example:
+
+.. code-block:: console
+
+  # hdparm -W /dev/sda
+
+  /dev/sda:
+   write-caching =  1 (on)
+
+  # sdparm --get WCE /dev/sda
+      /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+  WCE           1  [cha: y]
+  # smartctl -g wcache /dev/sda
+  smartctl 7.1 2020-04-05 r5049 [x86_64-linux-4.18.0-305.19.1.el8_4.x86_64] (local build)
+  Copyright (C) 2002-19, Bruce Allen, Christian Franke, www.smartmontools.org
+
+  Write cache is:   Enabled
+
+  # cat /sys/class/scsi_disk/0\:0\:0\:0/cache_type
+  write back
+
+The write cache can be disabled with those same tools:
+
+.. code-block:: console
+
+  # hdparm -W0 /dev/sda
+
+  /dev/sda:
+   setting drive write-caching to 0 (off)
+   write-caching =  0 (off)
+
+  # sdparm --clear WCE /dev/sda
+      /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+  # smartctl -s wcache,off /dev/sda
+  smartctl 7.1 2020-04-05 r5049 [x86_64-linux-4.18.0-305.19.1.el8_4.x86_64] (local build)
+  Copyright (C) 2002-19, Bruce Allen, Christian Franke, www.smartmontools.org
+
+  === START OF ENABLE/DISABLE COMMANDS SECTION ===
+  Write cache disabled
+
+In most cases, disabling this cache  using ``hdparm``, ``sdparm``, or ``smartctl``
+results in the cache_type changing automatically to "write through". If this is
+not the case, you can try setting it directly as follows. (Users should ensure
+that setting cache_type also correctly persists the caching mode of the device
+until the next reboot as some drives require this to be repeated at every boot):
+
+.. code-block:: console
+
+  # echo "write through" > /sys/class/scsi_disk/0\:0\:0\:0/cache_type
+
+  # hdparm -W /dev/sda
+
+  /dev/sda:
+   write-caching =  0 (off)
+
+.. tip:: This udev rule (tested on CentOS 8) will set all SATA/SAS device cache_types to "write
+  through":
+
+  .. code-block:: console
+
+    # cat /etc/udev/rules.d/99-ceph-write-through.rules
+    ACTION=="add", SUBSYSTEM=="scsi_disk", ATTR{cache_type}:="write through"
+
+.. tip:: This udev rule (tested on CentOS 7) will set all SATA/SAS device cache_types to "write
+  through":
+
+  .. code-block:: console
+
+    # cat /etc/udev/rules.d/99-ceph-write-through-el7.rules
+    ACTION=="add", SUBSYSTEM=="scsi_disk", RUN+="/bin/sh -c 'echo write through > /sys/class/scsi_disk/$kernel/cache_type'"
+
+.. tip:: The ``sdparm`` utility can be used to view/change the volatile write
+  cache on several devices at once:
+
+  .. code-block:: console
+
+    # sdparm --get WCE /dev/sd*
+        /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+    WCE           0  [cha: y]
+        /dev/sdb: ATA       TOSHIBA MG07ACA1  0101
+    WCE           0  [cha: y]
+    # sdparm --clear WCE /dev/sd*
+        /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+        /dev/sdb: ATA       TOSHIBA MG07ACA1  0101
+
+Additional Considerations
+-------------------------
+
+Ceph operators typically provision  multiple OSDs per host, but you should
+ensure that the aggregate throughput of your OSD drives doesn't exceed the
+network bandwidth required to service a client's read and write operations.
+You should also each host's percentage of the cluster's overall capacity. If
+the percentage located on a particular host is large and the host fails, it
+can lead to problems such as recovery causing OSDs to exceed the ``full ratio``,
+which in turn causes Ceph to halt operations to prevent data loss.
+
+When you run multiple OSDs per host, you also need to ensure that the kernel
+is up to date. See `OS Recommendations`_ for notes on ``glibc`` and
+``syncfs(2)`` to ensure that your hardware performs as expected when running
+multiple OSDs per host.
+
+
+Networks
+========
+
+Provision at least 10 Gb/s networking in your datacenter, both among Ceph
+hosts and between clients and your Ceph cluster.  Network link active/active
+bonding across separate network switches is strongly recommended both for
+increased throughput and for tolerance of network failures and maintenance.
+Take care that your bonding hash policy distributes traffic across links.
+
+Speed
+-----
+
+It takes three hours to replicate 1 TB of data across a 1 Gb/s network and it
+takes thirty hours to replicate 10 TB across a 1 Gb/s network. But it takes only
+twenty minutes to replicate 1 TB across a 10 Gb/s network, and it takes
+only one hour to replicate 10 TB across a 10 Gb/s network.
+
+Note that a 40 Gb/s network link is effectively four 10 Gb/s channels in
+parallel, and that a 100Gb/s network link is effectively four 25 Gb/s channels
+in parallel.  Thus, and perhaps somewhat counterintuitively, an individual
+packet on a 25 Gb/s network has slightly lower latency compared to a 40 Gb/s
+network.
+
+
+Cost
+----
+
+The larger the Ceph cluster, the more common OSD failures will be.
+The faster that a placement group (PG) can recover from a degraded state to
+an ``active + clean`` state, the better. Notably, fast recovery minimizes
+the likelihood of multiple, overlapping failures that can cause data to become
+temporarily unavailable or even lost. Of course, when provisioning your
+network, you will have to balance price against performance. 
+
+Some deployment tools employ VLANs to make hardware and network cabling more
+manageable. VLANs that use the 802.1q protocol require VLAN-capable NICs and
+switches. The added expense of this hardware may be offset by the operational
+cost savings on network setup and maintenance. When using VLANs to handle VM
+traffic between the cluster and compute stacks (e.g., OpenStack, CloudStack,
+etc.), there is additional value in using 10 Gb/s Ethernet or better; 40 Gb/s or
+increasingly 25/50/100 Gb/s networking as of 2022 is common for production clusters.
+
+Top-of-rack (TOR) switches also need fast and redundant uplinks to
+core / spine network switches or routers, often at least 40 Gb/s.
+
+
+Baseboard Management Controller (BMC)
+-------------------------------------
+
+Your server chassis should have a Baseboard Management Controller (BMC).
+Well-known examples are iDRAC (Dell), CIMC (Cisco UCS), and iLO (HPE).
+Administration and deployment tools may also use BMCs extensively, especially
+via IPMI or Redfish, so consider the cost/benefit tradeoff of an out-of-band
+network for security and administration.  Hypervisor SSH access, VM image uploads,
+OS image installs, management sockets, etc. can impose significant loads on a network.
+Running multiple networks may seem like overkill, but each traffic path represents
+a potential capacity, throughput and/or performance bottleneck that you should
+carefully consider before deploying a large scale data cluster.
+
+Additionally BMCs as of 2023 rarely sport network connections faster than 1 Gb/s,
+so dedicated and inexpensive 1 Gb/s switches for BMC administrative traffic
+may reduce costs by wasting fewer expenive ports on faster host switches.
+ 
+
+Failure Domains
+===============
+
+A failure domain can be thought of as any component loss that prevents access to
+one or more OSDs or other Ceph daemons. These could be a stopped daemon on a host;
+a storage drive failure, an OS crash, a malfunctioning NIC, a failed power supply,
+a network outage, a power outage, and so forth. When planning your hardware
+deployment, you must balance the risk of reducing costs by placing too many
+responsibilities into too few failure domains against the added costs of
+isolating every potential failure domain.
+
+
+Minimum Hardware Recommendations
+================================
+
+Ceph can run on inexpensive commodity hardware. Small production clusters
+and development clusters can run successfully with modest hardware.  As
+we noted above: when we speak of CPU _cores_, we mean _threads_ when
+hyperthreading (HT) is enabled.  Each modern physical x64 CPU core typically
+provides two logical CPU threads; other CPU architectures may vary.
+
+Take care that there are many factors that influence resource choices.  The
+minimum resources that suffice for one purpose will not necessarily suffice for
+another.  A sandbox cluster with one OSD built on a laptop with VirtualBox or on
+a trio of Raspberry PIs will get by with fewer resources than a production
+deployment with a thousand OSDs serving five thousand of RBD clients.  The
+classic Fisher Price PXL 2000 captures video, as does an IMAX or RED camera.
+One would not expect the former to do the job of the latter.  We especially
+cannot stress enough the criticality of using enterprise-quality storage
+media for production workloads.
+
+Additional insights into resource planning for production clusters are
+found above and elsewhere within this documentation.
+
++--------------+----------------+-----------------------------------------+
+|  Process     | Criteria       | Bare Minimum and Recommended            |
++==============+================+=========================================+
+| ``ceph-osd`` | Processor      | - 1 core minimum, 2 recommended         |
+|              |                | - 1 core per 200-500 MB/s throughput    |
+|              |                | - 1 core per 1000-3000 IOPS             |
+|              |                |                                         |
+|              |                | * Results are before replication.       |
+|              |                | * Results may vary across CPU and drive |
+|              |                |   models and Ceph configuration:        |
+|              |                |   (erasure coding, compression, etc)    |
+|              |                | * ARM processors specifically may       |
+|              |                |   require more cores for performance.   |
+|              |                | * SSD OSDs, especially NVMe, will       |
+|              |                |   benefit from additional cores per OSD.|
+|              |                | * Actual performance depends on many    |
+|              |                |   factors including drives, net, and    |
+|              |                |   client throughput and latency.        |
+|              |                |   Benchmarking is highly recommended.   |
+|              +----------------+-----------------------------------------+
+|              | RAM            | - 4GB+ per daemon (more is better)      |
+|              |                | - 2-4GB may function but may be slow    |
+|              |                | - Less than 2GB is not recommended      |
+|              +----------------+-----------------------------------------+
+|              | Storage Drives |  1x storage drive per OSD               |
+|              +----------------+-----------------------------------------+
+|              | DB/WAL         |  1x SSD partion per HDD OSD             |
+|              | (optional)     |  4-5x HDD OSDs per DB/WAL SATA SSD      |
+|              |                |  <= 10 HDD OSDss per DB/WAL NVMe SSD    |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1Gb/s (bonded 10+ Gb/s recommended) |
++--------------+----------------+-----------------------------------------+
+| ``ceph-mon`` | Processor      | - 2 cores minimum                       |
+|              +----------------+-----------------------------------------+
+|              | RAM            |  5GB+ per daemon (large / production    |
+|              |                |  clusters need more)                    |
+|              +----------------+-----------------------------------------+
+|              | Storage        |  100 GB per daemon, SSD is recommended  |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1Gb/s (10+ Gb/s recommended)        |
++--------------+----------------+-----------------------------------------+
+| ``ceph-mds`` | Processor      | - 2 cores minimum                       |
+|              +----------------+-----------------------------------------+
+|              | RAM            |  2GB+ per daemon (more for production)  |
+|              +----------------+-----------------------------------------+
+|              | Disk Space     |  1 GB per daemon                        |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1Gb/s (10+ Gb/s recommended)        |
++--------------+----------------+-----------------------------------------+
+
+.. tip:: If you are running an OSD node with a single storage drive, create a
+   partition for your OSD that is separate from the partition
+   containing the OS. We recommend separate drives for the
+   OS and for OSD storage.
+
+
+
+.. _block and block.db: https://docs.ceph.com/en/latest/rados/configuration/bluestore-config-ref/#block-and-block-db
+.. _Ceph blog: https://ceph.com/community/blog/
+.. _Ceph Write Throughput 1: http://ceph.com/community/ceph-performance-part-1-disk-controller-write-throughput/
+.. _Ceph Write Throughput 2: http://ceph.com/community/ceph-performance-part-2-write-throughput-without-ssd-journals/
+.. _Mapping Pools to Different Types of OSDs: ../../rados/operations/crush-map#placing-different-pools-on-different-osds
+.. _OS Recommendations: ../os-recommendations
+.. _Storage Networking Industry Association's Total Cost of Ownership calculator: https://www.snia.org/forums/cmsi/programs/TCOcalc
+.. _Werner Fischer's blog post on partition alignment: https://www.thomas-krenn.com/en/wiki/Partition_Alignment_detailed_explanation
diff --git a/doc/start/intro.rst b/doc/start/intro.rst
new file mode 100644
index 000000000..3a50a8733
--- /dev/null
+++ b/doc/start/intro.rst
@@ -0,0 +1,99 @@
+===============
+ Intro to Ceph
+===============
+
+Ceph can be used to provide :term:`Ceph Object Storage` to :term:`Cloud
+Platforms` and Ceph can be used to provide :term:`Ceph Block Device` services
+to :term:`Cloud Platforms`. Ceph can be used to deploy a :term:`Ceph File
+System`.  All :term:`Ceph Storage Cluster` deployments begin with setting up
+each :term:`Ceph Node` and then setting up the network. 
+
+A Ceph Storage Cluster requires the following: at least one Ceph Monitor and at
+least one Ceph Manager, and at least as many Ceph OSDs as there are copies of
+an object stored on the Ceph cluster (for example, if three copies of a given
+object are stored on the Ceph cluster, then at least three OSDs must exist in
+that Ceph cluster). 
+
+The Ceph Metadata Server is necessary to run Ceph File System clients.
+
+.. note::
+
+   It is a best practice to have a Ceph Manager for each Monitor, but it is not
+   necessary. 
+
+.. ditaa::
+
+            +---------------+ +------------+ +------------+ +---------------+
+            |      OSDs     | | Monitors   | |  Managers  | |      MDSs     |
+            +---------------+ +------------+ +------------+ +---------------+
+
+- **Monitors**: A :term:`Ceph Monitor` (``ceph-mon``) maintains maps
+  of the cluster state, including the monitor map, manager map, the
+  OSD map, the MDS map, and the CRUSH map.  These maps are critical 
+  cluster state required for Ceph daemons to coordinate with each other.  
+  Monitors are also responsible for managing authentication between 
+  daemons and clients.  At least three monitors are normally required 
+  for redundancy and high availability.
+
+- **Managers**: A :term:`Ceph Manager` daemon (``ceph-mgr``) is
+  responsible for keeping track of runtime metrics and the current
+  state of the Ceph cluster, including storage utilization, current
+  performance metrics, and system load.  The Ceph Manager daemons also
+  host python-based modules to manage and expose Ceph cluster
+  information, including a web-based :ref:`mgr-dashboard` and
+  `REST API`_.  At least two managers are normally required for high
+  availability.
+
+- **Ceph OSDs**: An Object Storage Daemon (:term:`Ceph OSD`,
+  ``ceph-osd``) stores data, handles data replication, recovery,
+  rebalancing, and provides some monitoring information to Ceph
+  Monitors and Managers by checking other Ceph OSD Daemons for a
+  heartbeat. At least three Ceph OSDs are normally required for 
+  redundancy and high availability.
+
+- **MDSs**: A :term:`Ceph Metadata Server` (MDS, ``ceph-mds``) stores
+  metadata on behalf of the :term:`Ceph File System` (i.e., Ceph Block
+  Devices and Ceph Object Storage do not use MDS). Ceph Metadata
+  Servers allow POSIX file system users to execute basic commands (like
+  ``ls``, ``find``, etc.) without placing an enormous burden on the
+  Ceph Storage Cluster.
+
+Ceph stores data as objects within logical storage pools. Using the
+:term:`CRUSH` algorithm, Ceph calculates which placement group (PG) should
+contain the object, and which OSD should store the placement group.  The
+CRUSH algorithm enables the Ceph Storage Cluster to scale, rebalance, and
+recover dynamically.
+
+.. _REST API: ../../mgr/restful
+
+.. container:: columns-2
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Recommendations</h3>
+
+      To begin using Ceph in production, you should review our hardware
+      recommendations and operating system recommendations.
+
+      .. toctree::
+         :maxdepth: 2
+
+         Hardware Recommendations <hardware-recommendations>
+         OS Recommendations <os-recommendations>
+
+   .. container:: column
+
+      .. raw:: html
+
+          <h3>Get Involved</h3>
+
+      You can avail yourself of help or contribute documentation, source
+      code or bugs by getting involved in the Ceph community.
+
+      .. toctree::
+         :maxdepth: 2
+
+         get-involved
+         documenting-ceph
diff --git a/doc/start/os-recommendations.rst b/doc/start/os-recommendations.rst
new file mode 100644
index 000000000..81906569e
--- /dev/null
+++ b/doc/start/os-recommendations.rst
@@ -0,0 +1,82 @@
+====================
+ OS Recommendations
+====================
+
+Ceph Dependencies
+=================
+
+As a general rule, we recommend deploying Ceph on newer releases of Linux. 
+We also recommend deploying on releases with long-term support.
+
+Linux Kernel
+------------
+
+- **Ceph Kernel Client**
+
+  If you are using the kernel client to map RBD block devices or mount
+  CephFS, the general advice is to use a "stable" or "longterm
+  maintenance" kernel series provided by either http://kernel.org or
+  your Linux distribution on any client hosts.
+
+  For RBD, if you choose to *track* long-term kernels, we recommend
+  *at least* 4.19-based "longterm maintenance" kernel series.  If you can
+  use a newer "stable" or "longterm maintenance" kernel series, do it.
+
+  For CephFS, see the section about `Mounting CephFS using Kernel Driver`_
+  for kernel version guidance.
+
+  Older kernel client versions may not support your `CRUSH tunables`_ profile
+  or other newer features of the Ceph cluster, requiring the storage cluster to
+  be configured with those features disabled. For RBD, a kernel of version 5.3
+  or CentOS 8.2 is the minimum necessary for reasonable support for RBD image
+  features.
+
+
+Platforms
+=========
+
+The chart below shows which Linux platforms Ceph provides packages for, and
+which platforms Ceph has been tested on. 
+
+Ceph does not require a specific Linux distribution. Ceph can run on any
+distribution that includes a supported kernel and supported system startup
+framework, for example ``sysvinit`` or ``systemd``. Ceph is sometimes ported to
+non-Linux systems but these are not supported by the core Ceph effort.
+
+
++---------------+---------------+-----------------+------------------+------------------+
+|               | Reef (18.2.z) | Quincy (17.2.z) | Pacific (16.2.z) | Octopus (15.2.z) |
++===============+===============+=================+==================+==================+
+| Centos 7      |               |                 |         A        |      B           |
++---------------+---------------+-----------------+------------------+------------------+
+| Centos 8      |    A          |     A           |         A        |      A           |
++---------------+---------------+-----------------+------------------+------------------+
+| Centos 9      |    A          |                 |                  |                  |
++---------------+---------------+-----------------+------------------+------------------+
+| Debian 10     |    C          |                 |         C        |      C           |
++---------------+---------------+-----------------+------------------+------------------+
+| Debian 11     |    C          |     C           |         C        |                  |
++---------------+---------------+-----------------+------------------+------------------+
+| OpenSUSE 15.2 |    C          |                 |         C        |      C           |
++---------------+---------------+-----------------+------------------+------------------+
+| OpenSUSE 15.3 |    C          |     C           |                  |                  |
++---------------+---------------+-----------------+------------------+------------------+
+| Ubuntu 18.04  |               |                 |         C        |      C           |
++---------------+---------------+-----------------+------------------+------------------+
+| Ubuntu 20.04  |    A          |     A           |         A        |      A           |
++---------------+---------------+-----------------+------------------+------------------+
+| Ubuntu 22.04  |    A          |                 |                  |                  |
++---------------+---------------+-----------------+------------------+------------------+
+
+- **A**: Ceph provides packages and has done comprehensive tests on the software in them.
+- **B**: Ceph provides packages and has done basic tests on the software in them.
+- **C**: Ceph provides packages only. No tests have been done on these releases.
+
+.. note::
+   **For Centos 7 Users** 
+   
+   ``Btrfs`` is no longer tested on Centos 7 in the Octopus release. We recommend using ``bluestore`` instead.
+
+.. _CRUSH Tunables: ../../rados/operations/crush-map#tunables
+
+.. _Mounting CephFS using Kernel Driver: ../../cephfs/mount-using-kernel-driver#which-kernel-version
diff --git a/doc/start/quick-rbd.rst b/doc/start/quick-rbd.rst
new file mode 100644
index 000000000..c1cf77098
--- /dev/null
+++ b/doc/start/quick-rbd.rst
@@ -0,0 +1,69 @@
+==========================
+ Block Device Quick Start
+==========================
+
+Ensure your :term:`Ceph Storage Cluster` is in an ``active + clean`` state
+before working with the :term:`Ceph Block Device`.
+
+.. note:: The Ceph Block Device is also known as :term:`RBD` or :term:`RADOS`
+   Block Device.
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |   ceph-client  |
+           |                  +-------->+ cCCC           |
+           |    ceph-deploy   |         |      ceph      |
+           \------------------/         \----------------/
+
+
+You may use a virtual machine for your ``ceph-client`` node, but do not 
+execute the following procedures on the same physical node as your Ceph 
+Storage Cluster nodes (unless you use a VM). See `FAQ`_ for details.
+
+Create a Block Device Pool
+==========================
+
+#. On the admin node, use the ``ceph`` tool to `create a pool`_
+   (we recommend the name 'rbd').
+
+#. On the admin node, use the ``rbd`` tool to initialize the pool for use by RBD::
+
+        rbd pool init <pool-name>
+
+Configure a Block Device
+========================
+
+#. On the ``ceph-client`` node, create a block device image. :: 
+
+	rbd create foo --size 4096 --image-feature layering [-m {mon-IP}] [-k /path/to/ceph.client.admin.keyring] [-p {pool-name}]
+
+#. On the ``ceph-client`` node, map the image to a block device. :: 
+
+	sudo rbd map foo --name client.admin [-m {mon-IP}] [-k /path/to/ceph.client.admin.keyring] [-p {pool-name}]
+	
+#. Use the block device by creating a file system on the ``ceph-client`` 
+   node. :: 
+
+	sudo mkfs.ext4 -m0 /dev/rbd/{pool-name}/foo
+	
+	This may take a few moments.
+	
+#. Mount the file system on the ``ceph-client`` node. ::
+
+	sudo mkdir /mnt/ceph-block-device
+	sudo mount /dev/rbd/{pool-name}/foo /mnt/ceph-block-device
+	cd /mnt/ceph-block-device
+
+#. Optionally configure the block device to be automatically mapped and mounted
+   at boot (and unmounted/unmapped at shutdown) - see the `rbdmap manpage`_.
+
+
+See `block devices`_ for additional details.
+
+.. _create a pool: ../../rados/operations/pools/#create-a-pool
+.. _block devices: ../../rbd
+.. _FAQ: http://wiki.ceph.com/How_Can_I_Give_Ceph_a_Try
+.. _OS Recommendations: ../os-recommendations
+.. _rbdmap manpage: ../../man/8/rbdmap